From 2bfe3f2e556754b95a66179377c431c6e8dacadb Mon Sep 17 00:00:00 2001 From: logwang Date: Tue, 15 May 2018 17:49:22 +0800 Subject: [PATCH] DPDK: upgrade to 17.11.2 LTS. Changes: 1. This version is downloaded from https://fast.dpdk.org/rel/dpdk-17.11.2.tar.xz. 2. Adapt the new interface `rte_ring_dequeue_burst`. 3. Change the type of `port_id` from uint8_t to uint16_t. 4. Just link libdpdk.a instead of the other libaries. 5. Install libnuma-dev first. 6. Update the documents. --- README.md | 10 +- app/micro_thread/Makefile | 8 +- app/nginx-1.11.10/auto/make | 9 +- app/redis-3.2.8/src/Makefile | 11 +- doc/F-Stack_Development_Guide.md | 16 +- doc/F-Stack_Quick_Start_Guide.md | 31 +- ...Launch_F-Stack_on_AWS_EC2_in_one_minute.md | 11 +- dpdk/.gitattributes | 3 + dpdk/.gitignore | 14 + dpdk/GNUmakefile | 1 + dpdk/MAINTAINERS | 677 +- dpdk/app/Makefile | 14 +- dpdk/app/cmdline_test/Makefile | 55 - dpdk/app/cmdline_test/cmdline_test.py | 113 - dpdk/app/cmdline_test/cmdline_test_data.py | 311 - dpdk/app/pdump/Makefile | 3 - dpdk/app/pdump/main.c | 15 +- dpdk/app/proc_info/Makefile | 3 - dpdk/app/proc_info/main.c | 383 +- dpdk/app/test-acl/Makefile | 48 - dpdk/app/test-acl/main.c | 1121 -- dpdk/app/test-crypto-perf/Makefile | 54 + dpdk/app/test-crypto-perf/cperf.h | 61 + dpdk/app/test-crypto-perf/cperf_ops.c | 651 + dpdk/app/test-crypto-perf/cperf_ops.h | 65 + dpdk/app/test-crypto-perf/cperf_options.h | 140 + .../test-crypto-perf/cperf_options_parsing.c | 1138 ++ dpdk/app/test-crypto-perf/cperf_test_common.c | 244 + dpdk/app/test-crypto-perf/cperf_test_common.h | 52 + .../app/test-crypto-perf/cperf_test_latency.c | 405 + .../app/test-crypto-perf/cperf_test_latency.h | 60 + .../cperf_test_pmd_cyclecount.c | 520 + .../cperf_test_pmd_cyclecount.h | 61 + .../test-crypto-perf/cperf_test_throughput.c | 357 + .../test-crypto-perf/cperf_test_throughput.h | 61 + .../cperf_test_vector_parsing.c | 620 + .../cperf_test_vector_parsing.h | 73 + .../app/test-crypto-perf/cperf_test_vectors.c | 568 + .../app/test-crypto-perf/cperf_test_vectors.h | 114 + dpdk/app/test-crypto-perf/cperf_test_verify.c | 455 + dpdk/app/test-crypto-perf/cperf_test_verify.h | 61 + .../data/aes_cbc_128_sha.data | 502 + .../data/aes_cbc_192_sha.data | 504 + .../data/aes_cbc_256_sha.data | 504 + dpdk/app/test-crypto-perf/main.c | 615 + dpdk/app/test-eventdev/Makefile | 54 + dpdk/app/test-eventdev/evt_common.h | 135 + dpdk/app/test-eventdev/evt_main.c | 227 + dpdk/app/test-eventdev/evt_options.c | 331 + dpdk/app/test-eventdev/evt_options.h | 269 + dpdk/app/test-eventdev/evt_test.c | 70 + dpdk/app/test-eventdev/evt_test.h | 125 + dpdk/app/test-eventdev/parser.c | 388 + dpdk/app/test-eventdev/parser.h | 79 + dpdk/app/test-eventdev/test_order_atq.c | 238 + dpdk/app/test-eventdev/test_order_common.c | 377 + dpdk/app/test-eventdev/test_order_common.h | 153 + dpdk/app/test-eventdev/test_order_queue.c | 248 + dpdk/app/test-eventdev/test_perf_atq.c | 283 + dpdk/app/test-eventdev/test_perf_common.c | 454 + dpdk/app/test-eventdev/test_perf_common.h | 170 + dpdk/app/test-eventdev/test_perf_queue.c | 294 + dpdk/app/test-pipeline/Makefile | 64 - dpdk/app/test-pipeline/config.c | 247 - dpdk/app/test-pipeline/init.c | 280 - dpdk/app/test-pipeline/main.c | 179 - dpdk/app/test-pipeline/main.h | 142 - dpdk/app/test-pipeline/pipeline_hash.c | 493 - dpdk/app/test-pipeline/runtime.c | 184 - dpdk/app/test-pmd/Makefile | 34 +- dpdk/app/test-pmd/cmdline.c | 6055 +++++- dpdk/app/test-pmd/cmdline_flow.c | 2959 +++ dpdk/app/test-pmd/cmdline_mtr.c | 1018 + dpdk/app/test-pmd/cmdline_mtr.h | 49 + dpdk/app/test-pmd/cmdline_tm.c | 2059 ++ dpdk/app/test-pmd/cmdline_tm.h | 56 + dpdk/app/test-pmd/config.c | 1243 +- dpdk/app/test-pmd/csumonly.c | 280 +- dpdk/app/test-pmd/flowgen.c | 17 +- dpdk/app/test-pmd/icmpecho.c | 6 +- dpdk/app/test-pmd/ieee1588fwd.c | 27 +- dpdk/app/test-pmd/iofwd.c | 4 +- dpdk/app/test-pmd/macfwd.c | 6 +- dpdk/app/test-pmd/macswap.c | 6 +- dpdk/app/test-pmd/parameters.c | 235 +- dpdk/app/test-pmd/rxonly.c | 224 +- dpdk/app/test-pmd/testpmd.c | 764 +- dpdk/app/test-pmd/testpmd.h | 184 +- dpdk/app/test-pmd/tm.c | 865 + dpdk/app/test-pmd/txonly.c | 16 +- dpdk/buildtools/auto-config-h.sh | 136 + dpdk/{scripts => buildtools}/gen-build-mk.sh | 0 dpdk/{scripts => buildtools}/gen-config-h.sh | 0 dpdk/buildtools/pmdinfogen/Makefile | 2 - dpdk/buildtools/pmdinfogen/pmdinfogen.c | 30 +- dpdk/buildtools/pmdinfogen/pmdinfogen.h | 11 +- dpdk/buildtools/relpath.sh | 105 + dpdk/config/common_armv8a_linuxapp | 51 + dpdk/config/common_base | 335 +- dpdk/config/common_linuxapp | 7 + dpdk/config/defconfig_arm-armv7a-linuxapp-gcc | 11 +- .../defconfig_arm64-armv8a-linuxapp-clang | 35 + .../defconfig_arm64-armv8a-linuxapp-gcc | 22 +- dpdk/config/defconfig_arm64-dpaa-linuxapp-gcc | 76 + .../config/defconfig_arm64-dpaa2-linuxapp-gcc | 56 +- .../defconfig_arm64-thunderx-linuxapp-gcc | 14 +- .../defconfig_arm64-xgene1-linuxapp-gcc | 5 +- .../config/defconfig_i686-native-linuxapp-gcc | 14 +- .../config/defconfig_i686-native-linuxapp-icc | 15 + .../defconfig_ppc_64-power8-linuxapp-gcc | 12 +- .../config/defconfig_tile-tilegx-linuxapp-gcc | 73 - .../defconfig_x86_64-ivshmem-linuxapp-gcc | 49 - .../defconfig_x86_64-ivshmem-linuxapp-icc | 49 - .../defconfig_x86_64-native-linuxapp-icc | 5 + .../defconfig_x86_x32-native-linuxapp-gcc | 10 + dpdk/devtools/build-tags.sh | 233 + dpdk/devtools/check-dup-includes.sh | 37 + dpdk/devtools/check-git-log.sh | 203 + dpdk/devtools/check-includes.sh | 288 + dpdk/devtools/check-maintainers.sh | 160 + dpdk/devtools/checkpatches.sh | 135 + dpdk/devtools/cocci.sh | 64 + dpdk/devtools/cocci/mtod-offset.cocci | 76 + dpdk/devtools/get-maintainer.sh | 85 + dpdk/devtools/git-log-fixes.sh | 140 + dpdk/devtools/load-devel-config | 12 + dpdk/devtools/test-build.sh | 259 + dpdk/devtools/test-null.sh | 45 + dpdk/devtools/validate-abi.sh | 275 + dpdk/doc/api/doxy-api-index.md | 53 +- dpdk/doc/api/doxy-api.conf | 21 +- dpdk/doc/api/examples.dox | 113 - dpdk/doc/build-sdk-quick.txt | 11 +- dpdk/doc/guides/conf.py | 280 +- dpdk/doc/guides/contributing/coding_style.rst | 33 +- dpdk/doc/guides/contributing/design.rst | 14 + .../doc/guides/contributing/documentation.rst | 54 +- dpdk/doc/guides/contributing/index.rst | 1 + dpdk/doc/guides/contributing/patches.rst | 195 +- dpdk/doc/guides/contributing/stable.rst | 99 + dpdk/doc/guides/contributing/versioning.rst | 68 +- dpdk/doc/guides/cryptodevs/aesni_gcm.rst | 70 +- dpdk/doc/guides/cryptodevs/aesni_mb.rst | 84 +- dpdk/doc/guides/cryptodevs/armv8.rst | 98 + dpdk/doc/guides/cryptodevs/dpaa2_sec.rst | 239 + dpdk/doc/guides/cryptodevs/dpaa_sec.rst | 182 + .../guides/cryptodevs/features/aesni_gcm.ini | 30 + .../guides/cryptodevs/features/aesni_mb.ini | 44 + dpdk/doc/guides/cryptodevs/features/armv8.ini | 28 + .../guides/cryptodevs/features/default.ini | 74 + .../guides/cryptodevs/features/dpaa2_sec.ini | 41 + .../guides/cryptodevs/features/dpaa_sec.ini | 40 + .../doc/guides/cryptodevs/features/kasumi.ini | 24 + dpdk/doc/guides/cryptodevs/features/mrvl.ini | 42 + dpdk/doc/guides/cryptodevs/features/null.ini | 25 + .../guides/cryptodevs/features/openssl.ini | 50 + dpdk/doc/guides/cryptodevs/features/qat.ini | 53 + .../doc/guides/cryptodevs/features/snow3g.ini | 24 + dpdk/doc/guides/cryptodevs/features/zuc.ini | 24 + .../cryptodevs/img/scheduler-overview.svg | 277 + dpdk/doc/guides/cryptodevs/index.rst | 9 +- dpdk/doc/guides/cryptodevs/kasumi.rst | 50 +- dpdk/doc/guides/cryptodevs/mrvl.rst | 206 + dpdk/doc/guides/cryptodevs/null.rst | 7 +- dpdk/doc/guides/cryptodevs/openssl.rst | 131 + dpdk/doc/guides/cryptodevs/overview.rst | 66 +- dpdk/doc/guides/cryptodevs/qat.rst | 404 +- dpdk/doc/guides/cryptodevs/scheduler.rst | 199 + dpdk/doc/guides/cryptodevs/snow3g.rst | 28 +- dpdk/doc/guides/cryptodevs/zuc.rst | 115 + dpdk/doc/guides/eventdevs/dpaa2.rst | 175 + dpdk/doc/guides/eventdevs/index.rst | 42 + dpdk/doc/guides/eventdevs/octeontx.rst | 130 + dpdk/doc/guides/eventdevs/sw.rst | 158 + dpdk/doc/guides/faq/faq.rst | 43 +- dpdk/doc/guides/freebsd_gsg/build_dpdk.rst | 19 +- .../guides/freebsd_gsg/build_sample_apps.rst | 20 +- .../guides/freebsd_gsg/install_from_ports.rst | 2 +- dpdk/doc/guides/howto/flow_bifurcation.rst | 12 +- .../howto/img/packet_capture_framework.svg | 471 + dpdk/doc/guides/howto/img/pvp_2nics.svg | 556 + ..._models_for_running_dpdk_in_containers.svg | 398 + .../guides/howto/img/vf_daemon_overview.svg | 440 + .../img/virtio_user_as_exceptional_path.svg | 207 + .../virtio_user_for_container_networking.svg | 685 + dpdk/doc/guides/howto/index.rst | 10 +- .../doc/guides/howto/lm_bond_virtio_sriov.rst | 12 +- .../doc/guides/howto/lm_virtio_vhost_user.rst | 26 +- .../guides/howto/packet_capture_framework.rst | 140 + .../guides/howto/pvp_reference_benchmark.rst | 398 + dpdk/doc/guides/howto/rte_flow.rst | 333 + dpdk/doc/guides/howto/vfd.rst | 407 + .../howto/virtio_user_as_exceptional_path.rst | 142 + .../virtio_user_for_container_networking.rst | 144 + dpdk/doc/guides/index.rst | 15 +- dpdk/doc/guides/linux_gsg/build_dpdk.rst | 147 +- .../guides/linux_gsg/build_sample_apps.rst | 44 +- dpdk/doc/guides/linux_gsg/enable_func.rst | 37 +- dpdk/doc/guides/linux_gsg/index.rst | 1 + dpdk/doc/guides/linux_gsg/linux_drivers.rst | 204 + .../linux_gsg/nic_perf_intel_platform.rst | 76 +- dpdk/doc/guides/linux_gsg/quick_start.rst | 20 +- dpdk/doc/guides/linux_gsg/sys_reqs.rst | 86 +- dpdk/doc/guides/mempool/index.rst | 40 + dpdk/doc/guides/mempool/octeontx.rst | 104 + dpdk/doc/guides/nics/ark.rst | 258 + dpdk/doc/guides/nics/avp.rst | 111 + dpdk/doc/guides/nics/bnx2x.rst | 184 +- dpdk/doc/guides/nics/bnxt.rst | 35 +- dpdk/doc/guides/nics/build_and_test.rst | 179 + dpdk/doc/guides/nics/cxgbe.rst | 133 +- dpdk/doc/guides/nics/dpaa.rst | 378 + dpdk/doc/guides/nics/dpaa2.rst | 606 + dpdk/doc/guides/nics/ena.rst | 63 +- dpdk/doc/guides/nics/enic.rst | 207 +- dpdk/doc/guides/nics/fail_safe.rst | 221 + dpdk/doc/guides/nics/features.rst | 942 + dpdk/doc/guides/nics/features/afpacket.ini | 6 + dpdk/doc/guides/nics/features/ark.ini | 15 + dpdk/doc/guides/nics/features/avp.ini | 16 + dpdk/doc/guides/nics/features/bnx2x.ini | 17 + dpdk/doc/guides/nics/features/bnx2x_vf.ini | 18 + dpdk/doc/guides/nics/features/bnxt.ini | 43 + dpdk/doc/guides/nics/features/bonding.ini | 6 + dpdk/doc/guides/nics/features/cxgbe.ini | 32 + dpdk/doc/guides/nics/features/default.ini | 80 + dpdk/doc/guides/nics/features/dpaa.ini | 24 + dpdk/doc/guides/nics/features/dpaa2.ini | 28 + dpdk/doc/guides/nics/features/e1000.ini | 32 + dpdk/doc/guides/nics/features/ena.ini | 27 + dpdk/doc/guides/nics/features/enic.ini | 34 + dpdk/doc/guides/nics/features/failsafe.ini | 26 + dpdk/doc/guides/nics/features/fm10k.ini | 35 + dpdk/doc/guides/nics/features/fm10k_vf.ini | 29 + dpdk/doc/guides/nics/features/i40e.ini | 54 + dpdk/doc/guides/nics/features/i40e_vec.ini | 44 + dpdk/doc/guides/nics/features/i40e_vf.ini | 38 + dpdk/doc/guides/nics/features/i40e_vf_vec.ini | 31 + dpdk/doc/guides/nics/features/igb.ini | 49 + dpdk/doc/guides/nics/features/igb_vf.ini | 29 + dpdk/doc/guides/nics/features/ixgbe.ini | 61 + dpdk/doc/guides/nics/features/ixgbe_vec.ini | 50 + dpdk/doc/guides/nics/features/ixgbe_vf.ini | 40 + .../doc/guides/nics/features/ixgbe_vf_vec.ini | 32 + dpdk/doc/guides/nics/features/kni.ini | 7 + dpdk/doc/guides/nics/features/liquidio.ini | 31 + dpdk/doc/guides/nics/features/mlx4.ini | 34 + dpdk/doc/guides/nics/features/mlx5.ini | 43 + dpdk/doc/guides/nics/features/mrvl.ini | 23 + dpdk/doc/guides/nics/features/nfp_pf.ini | 28 + dpdk/doc/guides/nics/features/nfp_vf.ini | 28 + dpdk/doc/guides/nics/features/null.ini | 6 + dpdk/doc/guides/nics/features/octeontx.ini | 19 + dpdk/doc/guides/nics/features/pcap.ini | 15 + dpdk/doc/guides/nics/features/qede.ini | 42 + dpdk/doc/guides/nics/features/qede_vf.ini | 38 + dpdk/doc/guides/nics/features/ring.ini | 6 + dpdk/doc/guides/nics/features/sfc_efx.ini | 37 + dpdk/doc/guides/nics/features/szedata2.ini | 18 + dpdk/doc/guides/nics/features/tap.ini | 28 + dpdk/doc/guides/nics/features/thunderx.ini | 31 + dpdk/doc/guides/nics/features/vhost.ini | 14 + dpdk/doc/guides/nics/features/virtio.ini | 29 + dpdk/doc/guides/nics/features/virtio_vec.ini | 24 + dpdk/doc/guides/nics/features/vmxnet3.ini | 30 + dpdk/doc/guides/nics/fm10k.rst | 11 + dpdk/doc/guides/nics/i40e.rst | 307 +- .../img/intel_perf_test_setup.svg | 0 dpdk/doc/guides/nics/index.rst | 13 + dpdk/doc/guides/nics/intel_vf.rst | 89 +- dpdk/doc/guides/nics/ixgbe.rst | 119 +- dpdk/doc/guides/nics/kni.rst | 197 + dpdk/doc/guides/nics/liquidio.rst | 233 + dpdk/doc/guides/nics/mlx4.rst | 252 +- dpdk/doc/guides/nics/mlx5.rst | 367 +- dpdk/doc/guides/nics/mrvl.rst | 253 + dpdk/doc/guides/nics/nfp.rst | 229 +- dpdk/doc/guides/nics/octeontx.rst | 225 + dpdk/doc/guides/nics/overview.rst | 100 +- dpdk/doc/guides/nics/pcap_ring.rst | 37 +- dpdk/doc/guides/nics/qede.rst | 253 +- dpdk/doc/guides/nics/sfc_efx.rst | 283 + dpdk/doc/guides/nics/szedata2.rst | 52 +- dpdk/doc/guides/nics/tap.rst | 215 + dpdk/doc/guides/nics/thunderx.rst | 327 +- dpdk/doc/guides/nics/vhost.rst | 7 +- dpdk/doc/guides/nics/virtio.rst | 86 +- dpdk/doc/guides/nics/vmxnet3.rst | 12 +- dpdk/doc/guides/platform/index.rst | 39 + dpdk/doc/guides/platform/octeontx.rst | 81 + dpdk/doc/guides/prog_guide/cryptodev_lib.rst | 366 +- .../prog_guide/dev_kit_build_system.rst | 61 +- .../prog_guide/dev_kit_root_make_help.rst | 28 - dpdk/doc/guides/prog_guide/efd_lib.rst | 455 + .../prog_guide/env_abstraction_layer.rst | 41 +- .../prog_guide/event_ethernet_rx_adapter.rst | 168 + dpdk/doc/guides/prog_guide/eventdev.rst | 394 + dpdk/doc/guides/prog_guide/extend_dpdk.rst | 18 +- .../guides/prog_guide/flow_classify_lib.rst | 427 + .../generic_receive_offload_lib.rst | 159 + .../generic_segmentation_offload_lib.rst | 257 + .../prog_guide/img/crypto_xform_chain.svg | 8 +- .../prog_guide/img/cryptodev_sym_sess.svg | 418 +- dpdk/doc/guides/prog_guide/img/efd_i1.svg | 130 + dpdk/doc/guides/prog_guide/img/efd_i10.svg | 384 + dpdk/doc/guides/prog_guide/img/efd_i11.svg | 319 + dpdk/doc/guides/prog_guide/img/efd_i12.svg | 1008 + dpdk/doc/guides/prog_guide/img/efd_i2.svg | 280 + dpdk/doc/guides/prog_guide/img/efd_i3.svg | 634 + dpdk/doc/guides/prog_guide/img/efd_i4.svg | 203 + dpdk/doc/guides/prog_guide/img/efd_i5.svg | 183 + dpdk/doc/guides/prog_guide/img/efd_i6.svg | 1254 ++ dpdk/doc/guides/prog_guide/img/efd_i7.svg | 790 + dpdk/doc/guides/prog_guide/img/efd_i8.svg | 182 + dpdk/doc/guides/prog_guide/img/efd_i9.svg | 390 + .../guides/prog_guide/img/eventdev_usage.svg | 994 + .../img/gso-output-segment-format.svg | 313 + .../prog_guide/img/gso-three-seg-mbuf.svg | 477 + dpdk/doc/guides/prog_guide/img/ivshmem.png | Bin 44920 -> 0 bytes dpdk/doc/guides/prog_guide/img/mbuf1.svg | 2 +- dpdk/doc/guides/prog_guide/img/member_i1.svg | 1613 ++ dpdk/doc/guides/prog_guide/img/member_i2.svg | 36 + dpdk/doc/guides/prog_guide/img/member_i3.svg | 148 + dpdk/doc/guides/prog_guide/img/member_i4.svg | 450 + dpdk/doc/guides/prog_guide/img/member_i5.svg | 163 + dpdk/doc/guides/prog_guide/img/member_i6.svg | 332 + dpdk/doc/guides/prog_guide/img/member_i7.svg | 399 + dpdk/doc/guides/prog_guide/index.rst | 55 +- dpdk/doc/guides/prog_guide/intro.rst | 2 +- dpdk/doc/guides/prog_guide/ivshmem_lib.rst | 160 - .../prog_guide/kernel_nic_interface.rst | 116 +- .../link_bonding_poll_mode_drv_lib.rst | 14 +- dpdk/doc/guides/prog_guide/lpm6_lib.rst | 2 +- dpdk/doc/guides/prog_guide/mbuf_lib.rst | 11 +- dpdk/doc/guides/prog_guide/member_lib.rst | 420 + dpdk/doc/guides/prog_guide/metrics_lib.rst | 300 + .../guides/prog_guide/multi_proc_support.rst | 4 +- dpdk/doc/guides/prog_guide/overview.rst | 2 +- .../prog_guide/packet_classif_access_ctrl.rst | 3 +- .../guides/prog_guide/packet_distrib_lib.rst | 4 + .../guides/prog_guide/perf_opt_guidelines.rst | 2 +- dpdk/doc/guides/prog_guide/poll_mode_drv.rst | 278 +- .../prog_guide/port_hotplug_framework.rst | 10 +- dpdk/doc/guides/prog_guide/power_man.rst | 12 + dpdk/doc/guides/prog_guide/profile_app.rst | 99 +- dpdk/doc/guides/prog_guide/ring_lib.rst | 15 - dpdk/doc/guides/prog_guide/rte_flow.rst | 2354 +++ dpdk/doc/guides/prog_guide/rte_security.rst | 564 + dpdk/doc/guides/prog_guide/service_cores.rst | 81 + dpdk/doc/guides/prog_guide/source_org.rst | 3 - .../guides/prog_guide/traffic_management.rst | 251 + .../traffic_metering_and_policing.rst | 102 + dpdk/doc/guides/prog_guide/vhost_lib.rst | 163 +- .../prog_guide/writing_efficient_code.rst | 17 +- dpdk/doc/guides/rel_notes/deprecation.rst | 90 +- dpdk/doc/guides/rel_notes/index.rst | 6 +- dpdk/doc/guides/rel_notes/known_issues.rst | 102 + dpdk/doc/guides/rel_notes/release_16_07.rst | 141 - dpdk/doc/guides/rel_notes/release_16_11.rst | 600 + dpdk/doc/guides/rel_notes/release_17_02.rst | 692 + dpdk/doc/guides/rel_notes/release_17_05.rst | 830 + dpdk/doc/guides/rel_notes/release_17_08.rst | 614 + dpdk/doc/guides/rel_notes/release_17_11.rst | 1092 + dpdk/doc/guides/rel_notes/supported_os.rst | 49 - dpdk/doc/guides/sample_app_ug/cmd_line.rst | 23 +- dpdk/doc/guides/sample_app_ug/compiling.rst | 122 + dpdk/doc/guides/sample_app_ug/dist_app.rst | 72 +- dpdk/doc/guides/sample_app_ug/ethtool.rst | 23 +- .../eventdev_pipeline_sw_pmd.rst | 173 + .../guides/sample_app_ug/exception_path.rst | 27 +- .../guides/sample_app_ug/flow_classify.rst | 575 + .../guides/sample_app_ug/flow_filtering.rst | 545 + dpdk/doc/guides/sample_app_ug/hello_world.rst | 23 +- .../doc/guides/sample_app_ug/img/dist_app.svg | 276 +- .../guides/sample_app_ug/img/l2_fwd_vm2vm.svg | 311 + .../sample_app_ug/img/qemu_virtio_net.png | Bin 31557 -> 0 bytes .../sample_app_ug/img/server_node_efd.svg | 1254 ++ .../sample_app_ug/img/tx_dpdk_testpmd.png | Bin 76019 -> 0 bytes .../sample_app_ug/img/vhost_net_arch.png | Bin 154920 -> 0 bytes .../img/vhost_net_sample_app.png | Bin 23800 -> 0 bytes .../sample_app_ug/img/virtio_linux_vhost.png | Bin 30290 -> 0 bytes dpdk/doc/guides/sample_app_ug/index.rst | 28 +- .../sample_app_ug/intel_quickassist.rst | 221 - dpdk/doc/guides/sample_app_ug/intro.rst | 129 +- dpdk/doc/guides/sample_app_ug/ip_frag.rst | 31 +- dpdk/doc/guides/sample_app_ug/ip_pipeline.rst | 2 +- .../guides/sample_app_ug/ip_reassembly.rst | 26 +- dpdk/doc/guides/sample_app_ug/ipsec_secgw.rst | 932 +- .../guides/sample_app_ug/ipv4_multicast.rst | 59 +- dpdk/doc/guides/sample_app_ug/keep_alive.rst | 26 +- .../sample_app_ug/kernel_nic_interface.rst | 36 +- .../guides/sample_app_ug/l2_forward_cat.rst | 40 +- .../sample_app_ug/l2_forward_crypto.rst | 114 +- .../sample_app_ug/l2_forward_job_stats.rst | 60 +- .../sample_app_ug/l2_forward_real_virtual.rst | 86 +- dpdk/doc/guides/sample_app_ug/l3_forward.rst | 63 +- .../sample_app_ug/l3_forward_access_ctrl.rst | 70 +- .../sample_app_ug/l3_forward_power_man.rst | 27 +- .../sample_app_ug/l3_forward_virtual.rst | 52 +- .../guides/sample_app_ug/link_status_intr.rst | 37 +- .../guides/sample_app_ug/load_balancer.rst | 21 +- .../guides/sample_app_ug/multi_process.rst | 54 +- .../sample_app_ug/netmap_compatibility.rst | 27 +- .../guides/sample_app_ug/packet_ordering.rst | 23 +- dpdk/doc/guides/sample_app_ug/pdump.rst | 144 - .../sample_app_ug/performance_thread.rst | 46 +- dpdk/doc/guides/sample_app_ug/proc_info.rst | 71 - dpdk/doc/guides/sample_app_ug/ptpclient.rst | 48 +- .../doc/guides/sample_app_ug/qos_metering.rst | 24 +- .../guides/sample_app_ug/qos_scheduler.rst | 27 +- .../guides/sample_app_ug/quota_watermark.rst | 182 +- .../guides/sample_app_ug/rxtx_callbacks.rst | 33 +- .../guides/sample_app_ug/server_node_efd.rst | 477 + .../guides/sample_app_ug/service_cores.rst | 172 + dpdk/doc/guides/sample_app_ug/skeleton.rst | 33 +- .../guides/sample_app_ug/tep_termination.rst | 101 +- .../guides/sample_app_ug/test_pipeline.rst | 22 +- dpdk/doc/guides/sample_app_ug/timer.rst | 23 +- dpdk/doc/guides/sample_app_ug/vhost.rst | 908 +- dpdk/doc/guides/sample_app_ug/vhost_scsi.rst | 106 + .../sample_app_ug/vm_power_management.rst | 13 +- .../sample_app_ug/vmdq_dcb_forwarding.rst | 21 +- dpdk/doc/guides/testpmd_app_ug/index.rst | 2 + dpdk/doc/guides/testpmd_app_ug/run_app.rst | 76 +- .../guides/testpmd_app_ug/testpmd_funcs.rst | 1556 +- dpdk/doc/guides/tools/cryptoperf.rst | 472 + dpdk/doc/guides/tools/devbind.rst | 143 + .../tools/img/eventdev_order_atq_test.svg | 1576 ++ .../tools/img/eventdev_order_queue_test.svg | 1673 ++ .../tools/img/eventdev_perf_atq_test.svg | 3188 +++ .../tools/img/eventdev_perf_queue_test.svg | 2599 +++ dpdk/doc/guides/tools/index.rst | 43 + dpdk/doc/guides/tools/pdump.rst | 152 + dpdk/doc/guides/tools/pmdinfo.rst | 55 + dpdk/doc/guides/tools/proc_info.rst | 70 + dpdk/doc/guides/tools/testeventdev.rst | 455 + .../guides/xen/img/dpdk_xen_pkt_switch.png | Bin 163842 -> 0 bytes dpdk/doc/guides/xen/img/grant_refs.png | Bin 6405 -> 0 bytes dpdk/doc/guides/xen/img/grant_table.png | Bin 96762 -> 0 bytes dpdk/doc/guides/xen/index.rst | 38 - dpdk/doc/guides/xen/pkt_switch.rst | 470 - dpdk/drivers/Makefile | 7 + dpdk/drivers/bus/Makefile | 38 + dpdk/drivers/bus/dpaa/Makefile | 78 + dpdk/drivers/bus/dpaa/base/fman/fman.c | 612 + dpdk/drivers/bus/dpaa/base/fman/fman_hw.c | 590 + .../drivers/bus/dpaa/base/fman/netcfg_layer.c | 214 + dpdk/drivers/bus/dpaa/base/fman/of.c | 576 + dpdk/drivers/bus/dpaa/base/qbman/bman.c | 394 + dpdk/drivers/bus/dpaa/base/qbman/bman.h | 574 + .../drivers/bus/dpaa/base/qbman/bman_driver.c | 323 + dpdk/drivers/bus/dpaa/base/qbman/bman_priv.h | 125 + dpdk/drivers/bus/dpaa/base/qbman/dpaa_alloc.c | 104 + dpdk/drivers/bus/dpaa/base/qbman/dpaa_sys.c | 136 + dpdk/drivers/bus/dpaa/base/qbman/dpaa_sys.h | 61 + dpdk/drivers/bus/dpaa/base/qbman/process.c | 331 + dpdk/drivers/bus/dpaa/base/qbman/qman.c | 2502 +++ dpdk/drivers/bus/dpaa/base/qbman/qman.h | 946 + .../drivers/bus/dpaa/base/qbman/qman_driver.c | 288 + dpdk/drivers/bus/dpaa/base/qbman/qman_priv.h | 310 + dpdk/drivers/bus/dpaa/dpaa_bus.c | 529 + dpdk/drivers/bus/dpaa/include/compat.h | 385 + dpdk/drivers/bus/dpaa/include/dpaa_bits.h | 65 + dpdk/drivers/bus/dpaa/include/dpaa_list.h | 101 + dpdk/drivers/bus/dpaa/include/dpaa_rbtree.h | 143 + dpdk/drivers/bus/dpaa/include/fman.h | 458 + dpdk/drivers/bus/dpaa/include/fsl_bman.h | 375 + dpdk/drivers/bus/dpaa/include/fsl_fman.h | 181 + .../drivers/bus/dpaa/include/fsl_fman_crc64.h | 263 + dpdk/drivers/bus/dpaa/include/fsl_qman.h | 2021 ++ dpdk/drivers/bus/dpaa/include/fsl_usd.h | 107 + dpdk/drivers/bus/dpaa/include/netcfg.h | 96 + dpdk/drivers/bus/dpaa/include/of.h | 190 + dpdk/drivers/bus/dpaa/include/process.h | 107 + .../drivers/bus/dpaa/rte_bus_dpaa_version.map | 66 + dpdk/drivers/bus/dpaa/rte_dpaa_bus.h | 173 + dpdk/drivers/bus/dpaa/rte_dpaa_logs.h | 107 + dpdk/drivers/bus/fslmc/Makefile | 81 + dpdk/drivers/bus/fslmc/fslmc_bus.c | 369 + dpdk/drivers/bus/fslmc/fslmc_logs.h | 76 + dpdk/drivers/bus/fslmc/fslmc_vfio.c | 700 + dpdk/drivers/bus/fslmc/fslmc_vfio.h | 82 + dpdk/drivers/bus/fslmc/mc/dpbp.c | 379 + dpdk/drivers/bus/fslmc/mc/dpci.c | 473 + dpdk/drivers/bus/fslmc/mc/dpcon.c | 357 + dpdk/drivers/bus/fslmc/mc/dpio.c | 503 + dpdk/drivers/bus/fslmc/mc/dpmng.c | 117 + dpdk/drivers/bus/fslmc/mc/fsl_dpbp.h | 128 + dpdk/drivers/bus/fslmc/mc/fsl_dpbp_cmd.h | 131 + dpdk/drivers/bus/fslmc/mc/fsl_dpci.h | 253 + dpdk/drivers/bus/fslmc/mc/fsl_dpci_cmd.h | 165 + dpdk/drivers/bus/fslmc/mc/fsl_dpcon.h | 122 + dpdk/drivers/bus/fslmc/mc/fsl_dpcon_cmd.h | 108 + dpdk/drivers/bus/fslmc/mc/fsl_dpio.h | 165 + dpdk/drivers/bus/fslmc/mc/fsl_dpio_cmd.h | 148 + dpdk/drivers/bus/fslmc/mc/fsl_dpmng.h | 87 + dpdk/drivers/bus/fslmc/mc/fsl_dpmng_cmd.h | 68 + dpdk/drivers/bus/fslmc/mc/fsl_mc_cmd.h | 208 + dpdk/drivers/bus/fslmc/mc/fsl_mc_sys.h | 96 + dpdk/drivers/bus/fslmc/mc/mc_sys.c | 117 + dpdk/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c | 144 + dpdk/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c | 179 + dpdk/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c | 544 + dpdk/drivers/bus/fslmc/portal/dpaa2_hw_dpio.h | 75 + dpdk/drivers/bus/fslmc/portal/dpaa2_hw_pvt.h | 378 + dpdk/drivers/bus/fslmc/qbman/include/compat.h | 124 + .../bus/fslmc/qbman/include/fsl_qbman_base.h | 156 + .../fslmc/qbman/include/fsl_qbman_portal.h | 1190 ++ dpdk/drivers/bus/fslmc/qbman/qbman_portal.c | 1425 ++ dpdk/drivers/bus/fslmc/qbman/qbman_portal.h | 167 + dpdk/drivers/bus/fslmc/qbman/qbman_sys.h | 376 + dpdk/drivers/bus/fslmc/qbman/qbman_sys_decl.h | 52 + .../bus/fslmc/rte_bus_fslmc_version.map | 91 + dpdk/drivers/bus/fslmc/rte_fslmc.h | 201 + dpdk/drivers/bus/pci/Makefile | 62 + dpdk/drivers/bus/pci/bsd/Makefile | 32 + dpdk/drivers/bus/pci/bsd/pci.c | 680 + dpdk/drivers/bus/pci/linux/Makefile | 36 + dpdk/drivers/bus/pci/linux/pci.c | 927 + dpdk/drivers/bus/pci/linux/pci_init.h | 111 + dpdk/drivers/bus/pci/linux/pci_uio.c | 568 + dpdk/drivers/bus/pci/linux/pci_vfio.c | 761 + dpdk/drivers/bus/pci/pci_common.c | 540 + dpdk/drivers/bus/pci/pci_common_uio.c | 234 + dpdk/drivers/bus/pci/private.h | 248 + dpdk/drivers/bus/pci/rte_bus_pci.h | 340 + dpdk/drivers/bus/pci/rte_bus_pci_version.map | 18 + dpdk/drivers/bus/vdev/Makefile | 57 + dpdk/drivers/bus/vdev/rte_bus_vdev.h | 153 + .../drivers/bus/vdev/rte_bus_vdev_version.map | 10 + dpdk/drivers/bus/vdev/vdev.c | 355 + dpdk/drivers/bus/vdev/vdev_logs.h | 45 + dpdk/drivers/crypto/Makefile | 9 +- dpdk/drivers/crypto/aesni_gcm/Makefile | 13 +- dpdk/drivers/crypto/aesni_gcm/aesni_gcm_ops.h | 128 +- dpdk/drivers/crypto/aesni_gcm/aesni_gcm_pmd.c | 649 +- .../crypto/aesni_gcm/aesni_gcm_pmd_ops.c | 120 +- .../crypto/aesni_gcm/aesni_gcm_pmd_private.h | 44 +- dpdk/drivers/crypto/aesni_mb/Makefile | 12 +- dpdk/drivers/crypto/aesni_mb/aesni_mb_ops.h | 52 +- .../crypto/aesni_mb/rte_aesni_mb_pmd.c | 651 +- .../crypto/aesni_mb/rte_aesni_mb_pmd_ops.c | 211 +- .../aesni_mb/rte_aesni_mb_pmd_private.h | 39 +- ...rsion.map => rte_pmd_aesni_mb_version.map} | 0 dpdk/drivers/crypto/armv8/Makefile | 68 + dpdk/drivers/crypto/armv8/rte_armv8_pmd.c | 882 + dpdk/drivers/crypto/armv8/rte_armv8_pmd_ops.c | 393 + .../crypto/armv8/rte_armv8_pmd_private.h | 227 + .../crypto/armv8/rte_pmd_armv8_version.map | 3 + dpdk/drivers/crypto/dpaa2_sec/Makefile | 85 + .../crypto/dpaa2_sec/dpaa2_sec_dpseci.c | 2435 +++ .../drivers/crypto/dpaa2_sec/dpaa2_sec_logs.h | 70 + .../drivers/crypto/dpaa2_sec/dpaa2_sec_priv.h | 477 + dpdk/drivers/crypto/dpaa2_sec/hw/compat.h | 159 + dpdk/drivers/crypto/dpaa2_sec/hw/desc.h | 2601 +++ dpdk/drivers/crypto/dpaa2_sec/hw/desc/algo.h | 679 + .../drivers/crypto/dpaa2_sec/hw/desc/common.h | 131 + dpdk/drivers/crypto/dpaa2_sec/hw/desc/ipsec.h | 1554 ++ dpdk/drivers/crypto/dpaa2_sec/hw/rta.h | 954 + .../dpaa2_sec/hw/rta/fifo_load_store_cmd.h | 346 + .../crypto/dpaa2_sec/hw/rta/header_cmd.h | 251 + .../crypto/dpaa2_sec/hw/rta/jump_cmd.h | 207 + .../drivers/crypto/dpaa2_sec/hw/rta/key_cmd.h | 222 + .../crypto/dpaa2_sec/hw/rta/load_cmd.h | 335 + .../crypto/dpaa2_sec/hw/rta/math_cmd.h | 402 + .../crypto/dpaa2_sec/hw/rta/move_cmd.h | 445 + .../crypto/dpaa2_sec/hw/rta/nfifo_cmd.h | 196 + .../crypto/dpaa2_sec/hw/rta/operation_cmd.h | 599 + .../crypto/dpaa2_sec/hw/rta/protocol_cmd.h | 732 + .../dpaa2_sec/hw/rta/sec_run_time_asm.h | 823 + .../dpaa2_sec/hw/rta/seq_in_out_ptr_cmd.h | 208 + .../crypto/dpaa2_sec/hw/rta/signature_cmd.h | 75 + .../crypto/dpaa2_sec/hw/rta/store_cmd.h | 185 + dpdk/drivers/crypto/dpaa2_sec/mc/dpseci.c | 676 + dpdk/drivers/crypto/dpaa2_sec/mc/fsl_dpseci.h | 432 + .../crypto/dpaa2_sec/mc/fsl_dpseci_cmd.h | 223 + .../dpaa2_sec/rte_pmd_dpaa2_sec_version.map | 4 + dpdk/drivers/crypto/dpaa_sec/Makefile | 73 + dpdk/drivers/crypto/dpaa_sec/dpaa_sec.c | 1548 ++ dpdk/drivers/crypto/dpaa_sec/dpaa_sec.h | 402 + dpdk/drivers/crypto/dpaa_sec/dpaa_sec_log.h | 70 + .../dpaa_sec/rte_pmd_dpaa_sec_version.map | 4 + dpdk/drivers/crypto/kasumi/Makefile | 10 +- dpdk/drivers/crypto/kasumi/rte_kasumi_pmd.c | 298 +- .../crypto/kasumi/rte_kasumi_pmd_ops.c | 70 +- .../crypto/kasumi/rte_kasumi_pmd_private.h | 13 +- dpdk/drivers/crypto/mrvl/Makefile | 66 + dpdk/drivers/crypto/mrvl/rte_mrvl_compat.h | 50 + dpdk/drivers/crypto/mrvl/rte_mrvl_pmd.c | 857 + dpdk/drivers/crypto/mrvl/rte_mrvl_pmd_ops.c | 778 + .../crypto/mrvl/rte_mrvl_pmd_private.h | 123 + .../crypto/mrvl/rte_pmd_mrvl_version.map | 3 + dpdk/drivers/crypto/null/Makefile | 10 +- dpdk/drivers/crypto/null/null_crypto_pmd.c | 186 +- .../drivers/crypto/null/null_crypto_pmd_ops.c | 76 +- .../crypto/null/null_crypto_pmd_private.h | 3 +- dpdk/drivers/crypto/openssl/Makefile | 56 + dpdk/drivers/crypto/openssl/rte_openssl_pmd.c | 1765 ++ .../crypto/openssl/rte_openssl_pmd_ops.c | 785 + .../crypto/openssl/rte_openssl_pmd_private.h | 200 + .../openssl/rte_pmd_openssl_version.map | 3 + dpdk/drivers/crypto/qat/Makefile | 10 +- .../qat/qat_adf/adf_transport_access_macros.h | 11 +- dpdk/drivers/crypto/qat/qat_adf/icp_qat_hw.h | 35 +- dpdk/drivers/crypto/qat/qat_adf/qat_algs.h | 84 +- .../crypto/qat/qat_adf/qat_algs_build_desc.c | 955 +- dpdk/drivers/crypto/qat/qat_crypto.c | 1672 +- dpdk/drivers/crypto/qat/qat_crypto.h | 77 +- .../crypto/qat/qat_crypto_capabilities.h | 586 + dpdk/drivers/crypto/qat/qat_qp.c | 100 +- dpdk/drivers/crypto/qat/rte_qat_cryptodev.c | 131 +- dpdk/drivers/crypto/scheduler/Makefile | 64 + .../scheduler/rte_cryptodev_scheduler.c | 601 + .../scheduler/rte_cryptodev_scheduler.h | 312 + .../rte_cryptodev_scheduler_operations.h | 85 + .../rte_pmd_crypto_scheduler_version.map | 21 + .../crypto/scheduler/scheduler_failover.c | 248 + .../crypto/scheduler/scheduler_multicore.c | 415 + .../scheduler/scheduler_pkt_size_distr.c | 446 + dpdk/drivers/crypto/scheduler/scheduler_pmd.c | 500 + .../crypto/scheduler/scheduler_pmd_ops.c | 570 + .../crypto/scheduler/scheduler_pmd_private.h | 159 + .../crypto/scheduler/scheduler_roundrobin.c | 240 + dpdk/drivers/crypto/snow3g/Makefile | 10 +- dpdk/drivers/crypto/snow3g/rte_snow3g_pmd.c | 286 +- .../crypto/snow3g/rte_snow3g_pmd_ops.c | 72 +- .../crypto/snow3g/rte_snow3g_pmd_private.h | 14 +- dpdk/drivers/crypto/zuc/Makefile | 65 + .../crypto/zuc/rte_pmd_zuc_version.map | 3 + dpdk/drivers/crypto/zuc/rte_zuc_pmd.c | 556 + dpdk/drivers/crypto/zuc/rte_zuc_pmd_ops.c | 366 + dpdk/drivers/crypto/zuc/rte_zuc_pmd_private.h | 120 + dpdk/drivers/event/Makefile | 39 + dpdk/drivers/event/dpaa2/Makefile | 64 + dpdk/drivers/event/dpaa2/dpaa2_eventdev.c | 854 + dpdk/drivers/event/dpaa2/dpaa2_eventdev.h | 122 + dpdk/drivers/event/dpaa2/dpaa2_hw_dpcon.c | 139 + .../dpaa2/rte_pmd_dpaa2_event_version.map | 3 + dpdk/drivers/event/octeontx/Makefile | 71 + .../rte_pmd_octeontx_ssovf_version.map | 3 + dpdk/drivers/event/octeontx/ssovf_evdev.c | 711 + dpdk/drivers/event/octeontx/ssovf_evdev.h | 200 + dpdk/drivers/event/octeontx/ssovf_worker.c | 277 + dpdk/drivers/event/octeontx/ssovf_worker.h | 166 + dpdk/drivers/event/skeleton/Makefile | 54 + .../rte_pmd_skeleton_event_version.map | 4 + .../event/skeleton/skeleton_eventdev.c | 504 + .../event/skeleton/skeleton_eventdev.h | 69 + dpdk/drivers/event/sw/Makefile | 64 + dpdk/drivers/event/sw/event_ring.h | 181 + dpdk/drivers/event/sw/iq_ring.h | 172 + .../event/sw/rte_pmd_sw_event_version.map | 3 + dpdk/drivers/event/sw/sw_evdev.c | 907 + dpdk/drivers/event/sw/sw_evdev.h | 323 + dpdk/drivers/event/sw/sw_evdev_scheduler.c | 603 + dpdk/drivers/event/sw/sw_evdev_worker.c | 212 + dpdk/drivers/event/sw/sw_evdev_xstats.c | 682 + dpdk/drivers/mempool/Makefile | 39 + dpdk/drivers/mempool/dpaa/Makefile | 59 + dpdk/drivers/mempool/dpaa/dpaa_mempool.c | 284 + dpdk/drivers/mempool/dpaa/dpaa_mempool.h | 76 + .../mempool/dpaa/rte_mempool_dpaa_version.map | 8 + dpdk/drivers/mempool/dpaa2/Makefile | 63 + dpdk/drivers/mempool/dpaa2/dpaa2_hw_mempool.c | 391 + dpdk/drivers/mempool/dpaa2/dpaa2_hw_mempool.h | 91 + .../dpaa2/rte_mempool_dpaa2_version.map | 8 + dpdk/drivers/mempool/octeontx/Makefile | 68 + .../drivers/mempool/octeontx/octeontx_fpavf.c | 813 + .../drivers/mempool/octeontx/octeontx_fpavf.h | 133 + dpdk/drivers/mempool/octeontx/octeontx_mbox.c | 242 + dpdk/drivers/mempool/octeontx/octeontx_mbox.h | 64 + .../mempool/octeontx/octeontx_pool_logs.h | 68 + .../drivers/mempool/octeontx/octeontx_ssovf.c | 299 + .../mempool/octeontx/rte_mempool_octeontx.c | 191 + .../octeontx/rte_mempool_octeontx_version.map | 9 + dpdk/drivers/mempool/ring/Makefile | 48 + dpdk/drivers/mempool/ring/rte_mempool_ring.c | 165 + .../mempool/ring/rte_mempool_ring_version.map | 4 + dpdk/drivers/mempool/stack/Makefile | 51 + .../drivers/mempool/stack/rte_mempool_stack.c | 147 + .../stack/rte_mempool_stack_version.map | 4 + dpdk/drivers/net/Makefile | 31 +- dpdk/drivers/net/af_packet/Makefile | 10 +- .../drivers/net/af_packet/rte_eth_af_packet.c | 258 +- dpdk/drivers/net/ark/Makefile | 69 + dpdk/drivers/net/ark/ark_ddm.c | 151 + dpdk/drivers/net/ark/ark_ddm.h | 177 + dpdk/drivers/net/ark/ark_ethdev.c | 1047 + dpdk/drivers/net/ark/ark_ethdev_rx.c | 672 + dpdk/drivers/net/ark/ark_ethdev_rx.h | 65 + dpdk/drivers/net/ark/ark_ethdev_tx.c | 467 + dpdk/drivers/net/ark/ark_ethdev_tx.h | 59 + dpdk/drivers/net/ark/ark_ext.h | 119 + dpdk/drivers/net/ark/ark_global.h | 162 + dpdk/drivers/net/ark/ark_logs.h | 119 + dpdk/drivers/net/ark/ark_mpu.c | 181 + dpdk/drivers/net/ark/ark_mpu.h | 154 + dpdk/drivers/net/ark/ark_pktchkr.c | 478 + dpdk/drivers/net/ark/ark_pktchkr.h | 117 + dpdk/drivers/net/ark/ark_pktdir.c | 85 + dpdk/drivers/net/ark/ark_pktdir.h | 70 + dpdk/drivers/net/ark/ark_pktgen.c | 500 + dpdk/drivers/net/ark/ark_pktgen.h | 108 + dpdk/drivers/net/ark/ark_rqp.c | 97 + dpdk/drivers/net/ark/ark_rqp.h | 86 + dpdk/drivers/net/ark/ark_udm.c | 226 + dpdk/drivers/net/ark/ark_udm.h | 192 + dpdk/drivers/net/ark/rte_pmd_ark_version.map | 4 + dpdk/drivers/net/avp/Makefile | 60 + dpdk/drivers/net/avp/avp_ethdev.c | 2314 +++ dpdk/drivers/net/avp/avp_logs.h | 59 + dpdk/drivers/net/avp/rte_avp_common.h | 433 + dpdk/drivers/net/avp/rte_avp_fifo.h | 169 + dpdk/drivers/net/avp/rte_pmd_avp_version.map | 4 + dpdk/drivers/net/bnx2x/Makefile | 8 +- dpdk/drivers/net/bnx2x/bnx2x.c | 103 +- dpdk/drivers/net/bnx2x/bnx2x.h | 176 +- dpdk/drivers/net/bnx2x/bnx2x_ethdev.c | 142 +- dpdk/drivers/net/bnx2x/bnx2x_ethdev.h | 1 - dpdk/drivers/net/bnx2x/bnx2x_rxtx.c | 34 +- dpdk/drivers/net/bnx2x/bnx2x_rxtx.h | 12 +- dpdk/drivers/net/bnx2x/bnx2x_stats.c | 12 +- dpdk/drivers/net/bnx2x/bnx2x_vfpf.c | 358 +- dpdk/drivers/net/bnx2x/bnx2x_vfpf.h | 19 +- dpdk/drivers/net/bnx2x/debug.c | 96 - dpdk/drivers/net/bnx2x/ecore_hsi.h | 12 +- dpdk/drivers/net/bnx2x/ecore_init.h | 2 +- dpdk/drivers/net/bnx2x/ecore_sp.c | 6 +- dpdk/drivers/net/bnx2x/ecore_sp.h | 26 +- dpdk/drivers/net/bnx2x/elink.c | 409 +- dpdk/drivers/net/bnx2x/elink.h | 4 - dpdk/drivers/net/bnxt/Makefile | 16 +- dpdk/drivers/net/bnxt/bnxt.h | 160 +- dpdk/drivers/net/bnxt/bnxt_cpr.c | 155 +- dpdk/drivers/net/bnxt/bnxt_cpr.h | 37 +- dpdk/drivers/net/bnxt/bnxt_ethdev.c | 2313 ++- dpdk/drivers/net/bnxt/bnxt_filter.c | 1124 +- dpdk/drivers/net/bnxt/bnxt_filter.h | 91 + dpdk/drivers/net/bnxt/bnxt_hwrm.c | 2718 ++- dpdk/drivers/net/bnxt/bnxt_hwrm.h | 92 +- dpdk/drivers/net/bnxt/bnxt_irq.c | 173 + dpdk/drivers/net/bnxt/bnxt_irq.h | 54 + dpdk/drivers/net/bnxt/bnxt_nvm_defs.h | 75 + dpdk/drivers/net/bnxt/bnxt_ring.c | 176 +- dpdk/drivers/net/bnxt/bnxt_ring.h | 10 +- dpdk/drivers/net/bnxt/bnxt_rxq.c | 275 +- dpdk/drivers/net/bnxt/bnxt_rxq.h | 9 +- dpdk/drivers/net/bnxt/bnxt_rxr.c | 491 +- dpdk/drivers/net/bnxt/bnxt_rxr.h | 64 +- dpdk/drivers/net/bnxt/bnxt_stats.c | 381 +- dpdk/drivers/net/bnxt/bnxt_stats.h | 17 +- dpdk/drivers/net/bnxt/bnxt_txq.h | 2 +- dpdk/drivers/net/bnxt/bnxt_txr.c | 49 +- dpdk/drivers/net/bnxt/bnxt_txr.h | 28 +- dpdk/drivers/net/bnxt/bnxt_vnic.c | 72 +- dpdk/drivers/net/bnxt/bnxt_vnic.h | 25 +- dpdk/drivers/net/bnxt/hsi_struct_def_dpdk.h | 16561 +++++++++++----- dpdk/drivers/net/bnxt/rte_pmd_bnxt.c | 839 + dpdk/drivers/net/bnxt/rte_pmd_bnxt.h | 354 + .../drivers/net/bnxt/rte_pmd_bnxt_version.map | 20 +- dpdk/drivers/net/bonding/Makefile | 17 +- dpdk/drivers/net/bonding/rte_eth_bond.h | 41 +- .../drivers/net/bonding/rte_eth_bond_8023ad.c | 450 +- .../drivers/net/bonding/rte_eth_bond_8023ad.h | 102 +- .../net/bonding/rte_eth_bond_8023ad_private.h | 37 +- dpdk/drivers/net/bonding/rte_eth_bond_alb.c | 11 +- dpdk/drivers/net/bonding/rte_eth_bond_alb.h | 6 +- dpdk/drivers/net/bonding/rte_eth_bond_api.c | 359 +- dpdk/drivers/net/bonding/rte_eth_bond_args.c | 80 +- dpdk/drivers/net/bonding/rte_eth_bond_pmd.c | 1058 +- .../net/bonding/rte_eth_bond_private.h | 93 +- .../net/bonding/rte_eth_bond_version.map | 45 - .../net/bonding/rte_pmd_bond_version.map | 54 + dpdk/drivers/net/cxgbe/Makefile | 10 +- dpdk/drivers/net/cxgbe/base/adapter.h | 54 +- dpdk/drivers/net/cxgbe/base/common.h | 30 +- dpdk/drivers/net/cxgbe/base/t4_chip_type.h | 11 +- dpdk/drivers/net/cxgbe/base/t4_hw.c | 1168 +- dpdk/drivers/net/cxgbe/base/t4_hw.h | 18 +- dpdk/drivers/net/cxgbe/base/t4_msg.h | 16 +- dpdk/drivers/net/cxgbe/base/t4_pci_id_tbl.h | 15 +- dpdk/drivers/net/cxgbe/base/t4_regs.h | 51 +- dpdk/drivers/net/cxgbe/base/t4_regs_values.h | 9 +- dpdk/drivers/net/cxgbe/base/t4fw_interface.h | 31 +- dpdk/drivers/net/cxgbe/cxgbe.h | 4 +- dpdk/drivers/net/cxgbe/cxgbe_compat.h | 19 +- dpdk/drivers/net/cxgbe/cxgbe_ethdev.c | 72 +- dpdk/drivers/net/cxgbe/cxgbe_main.c | 279 +- dpdk/drivers/net/cxgbe/sge.c | 367 +- dpdk/drivers/net/dpaa/Makefile | 63 + dpdk/drivers/net/dpaa/dpaa_ethdev.c | 1119 ++ dpdk/drivers/net/dpaa/dpaa_ethdev.h | 182 + dpdk/drivers/net/dpaa/dpaa_rxtx.c | 775 + dpdk/drivers/net/dpaa/dpaa_rxtx.h | 297 + .../drivers/net/dpaa/rte_pmd_dpaa_version.map | 4 + dpdk/drivers/net/dpaa2/Makefile | 73 + dpdk/drivers/net/dpaa2/base/dpaa2_hw_dpni.c | 359 + .../net/dpaa2/base/dpaa2_hw_dpni_annot.h | 257 + dpdk/drivers/net/dpaa2/dpaa2_ethdev.c | 2011 ++ dpdk/drivers/net/dpaa2/dpaa2_ethdev.h | 122 + dpdk/drivers/net/dpaa2/dpaa2_rxtx.c | 687 + dpdk/drivers/net/dpaa2/mc/dpkg.c | 107 + dpdk/drivers/net/dpaa2/mc/dpni.c | 1976 ++ dpdk/drivers/net/dpaa2/mc/fsl_dpkg.h | 237 + dpdk/drivers/net/dpaa2/mc/fsl_dpni.h | 1168 ++ dpdk/drivers/net/dpaa2/mc/fsl_dpni_cmd.h | 638 + dpdk/drivers/net/dpaa2/mc/fsl_net.h | 487 + .../net/dpaa2/rte_pmd_dpaa2_version.map | 12 + dpdk/drivers/net/e1000/Makefile | 14 +- dpdk/drivers/net/e1000/base/README | 25 +- dpdk/drivers/net/e1000/base/e1000_82575.c | 1 - dpdk/drivers/net/e1000/base/e1000_82575.h | 1 + dpdk/drivers/net/e1000/base/e1000_api.c | 19 + dpdk/drivers/net/e1000/base/e1000_defines.h | 9 + dpdk/drivers/net/e1000/base/e1000_hw.h | 21 +- dpdk/drivers/net/e1000/base/e1000_ich8lan.c | 871 +- dpdk/drivers/net/e1000/base/e1000_ich8lan.h | 21 +- dpdk/drivers/net/e1000/base/e1000_mbx.c | 36 +- dpdk/drivers/net/e1000/base/e1000_nvm.c | 1 + dpdk/drivers/net/e1000/base/e1000_osdep.h | 18 +- dpdk/drivers/net/e1000/base/e1000_regs.h | 7 + dpdk/drivers/net/e1000/base/e1000_vf.c | 3 +- dpdk/drivers/net/e1000/e1000_ethdev.h | 116 +- dpdk/drivers/net/e1000/em_ethdev.c | 199 +- dpdk/drivers/net/e1000/em_rxtx.c | 129 +- dpdk/drivers/net/e1000/igb_ethdev.c | 1163 +- dpdk/drivers/net/e1000/igb_flow.c | 1726 ++ dpdk/drivers/net/e1000/igb_pf.c | 9 +- dpdk/drivers/net/e1000/igb_rxtx.c | 323 +- dpdk/drivers/net/ena/Makefile | 8 +- dpdk/drivers/net/ena/base/ena_com.c | 22 +- dpdk/drivers/net/ena/base/ena_plat_dpdk.h | 21 +- dpdk/drivers/net/ena/ena_ethdev.c | 307 +- dpdk/drivers/net/ena/ena_ethdev.h | 26 +- dpdk/drivers/net/enic/Makefile | 10 +- dpdk/drivers/net/enic/base/cq_enet_desc.h | 13 + dpdk/drivers/net/enic/base/vnic_cq.c | 14 +- dpdk/drivers/net/enic/base/vnic_dev.c | 305 +- dpdk/drivers/net/enic/base/vnic_dev.h | 10 +- dpdk/drivers/net/enic/base/vnic_devcmd.h | 421 +- dpdk/drivers/net/enic/base/vnic_rq.c | 9 +- dpdk/drivers/net/enic/base/vnic_rq.h | 1 + dpdk/drivers/net/enic/base/vnic_rss.c | 32 - dpdk/drivers/net/enic/base/vnic_wq.c | 7 +- dpdk/drivers/net/enic/enic.h | 84 +- dpdk/drivers/net/enic/enic_clsf.c | 310 +- dpdk/drivers/net/enic/enic_compat.h | 25 +- dpdk/drivers/net/enic/enic_ethdev.c | 197 +- dpdk/drivers/net/enic/enic_flow.c | 1544 ++ dpdk/drivers/net/enic/enic_main.c | 334 +- dpdk/drivers/net/enic/enic_res.c | 25 +- dpdk/drivers/net/enic/enic_res.h | 5 +- dpdk/drivers/net/enic/enic_rxtx.c | 240 +- dpdk/drivers/net/failsafe/Makefile | 65 + dpdk/drivers/net/failsafe/failsafe.c | 299 + dpdk/drivers/net/failsafe/failsafe_args.c | 472 + dpdk/drivers/net/failsafe/failsafe_eal.c | 124 + dpdk/drivers/net/failsafe/failsafe_ether.c | 486 + dpdk/drivers/net/failsafe/failsafe_flow.c | 244 + dpdk/drivers/net/failsafe/failsafe_ops.c | 770 + dpdk/drivers/net/failsafe/failsafe_private.h | 378 + dpdk/drivers/net/failsafe/failsafe_rxtx.c | 204 + .../net/failsafe/rte_pmd_failsafe_version.map | 4 + dpdk/drivers/net/fm10k/Makefile | 15 +- dpdk/drivers/net/fm10k/base/fm10k_common.c | 11 +- dpdk/drivers/net/fm10k/base/fm10k_mbx.c | 10 +- dpdk/drivers/net/fm10k/base/fm10k_mbx.h | 2 + dpdk/drivers/net/fm10k/base/fm10k_osdep.h | 34 +- dpdk/drivers/net/fm10k/base/fm10k_pf.c | 140 +- dpdk/drivers/net/fm10k/base/fm10k_pf.h | 2 + dpdk/drivers/net/fm10k/base/fm10k_tlv.c | 16 +- dpdk/drivers/net/fm10k/base/fm10k_type.h | 49 +- dpdk/drivers/net/fm10k/base/fm10k_vf.c | 24 +- dpdk/drivers/net/fm10k/fm10k.h | 21 +- dpdk/drivers/net/fm10k/fm10k_ethdev.c | 359 +- dpdk/drivers/net/fm10k/fm10k_rxtx.c | 72 +- dpdk/drivers/net/fm10k/fm10k_rxtx_vec.c | 80 +- dpdk/drivers/net/i40e/Makefile | 35 +- dpdk/drivers/net/i40e/base/README | 59 + dpdk/drivers/net/i40e/base/i40e_adminq.c | 20 +- dpdk/drivers/net/i40e/base/i40e_adminq.h | 4 +- dpdk/drivers/net/i40e/base/i40e_adminq_cmd.h | 363 +- dpdk/drivers/net/i40e/base/i40e_common.c | 1333 +- dpdk/drivers/net/i40e/base/i40e_dcb.c | 2 + dpdk/drivers/net/i40e/base/i40e_devids.h | 5 +- dpdk/drivers/net/i40e/base/i40e_lan_hmc.c | 5 - dpdk/drivers/net/i40e/base/i40e_nvm.c | 72 +- dpdk/drivers/net/i40e/base/i40e_osdep.h | 13 +- dpdk/drivers/net/i40e/base/i40e_prototype.h | 85 +- dpdk/drivers/net/i40e/base/i40e_register.h | 4 +- dpdk/drivers/net/i40e/base/i40e_type.h | 500 +- dpdk/drivers/net/i40e/base/i40e_virtchnl.h | 438 - dpdk/drivers/net/i40e/base/virtchnl.h | 772 + dpdk/drivers/net/i40e/i40e_ethdev.c | 3817 +++- dpdk/drivers/net/i40e/i40e_ethdev.h | 725 +- dpdk/drivers/net/i40e/i40e_ethdev_vf.c | 993 +- dpdk/drivers/net/i40e/i40e_fdir.c | 863 +- dpdk/drivers/net/i40e/i40e_flow.c | 4537 +++++ dpdk/drivers/net/i40e/i40e_logs.h | 17 +- dpdk/drivers/net/i40e/i40e_pf.c | 687 +- dpdk/drivers/net/i40e/i40e_pf.h | 74 +- dpdk/drivers/net/i40e/i40e_rxtx.c | 1151 +- dpdk/drivers/net/i40e/i40e_rxtx.h | 595 +- dpdk/drivers/net/i40e/i40e_rxtx_vec.c | 775 - dpdk/drivers/net/i40e/i40e_rxtx_vec_altivec.c | 645 + dpdk/drivers/net/i40e/i40e_rxtx_vec_common.h | 247 + dpdk/drivers/net/i40e/i40e_rxtx_vec_neon.c | 626 + dpdk/drivers/net/i40e/i40e_rxtx_vec_sse.c | 655 + dpdk/drivers/net/i40e/i40e_tm.c | 1000 + dpdk/drivers/net/i40e/rte_pmd_i40e.c | 3020 +++ dpdk/drivers/net/i40e/rte_pmd_i40e.h | 936 + .../drivers/net/i40e/rte_pmd_i40e_version.map | 56 + dpdk/drivers/net/ixgbe/Makefile | 29 +- dpdk/drivers/net/ixgbe/base/README | 5 +- dpdk/drivers/net/ixgbe/base/ixgbe_82598.c | 10 +- dpdk/drivers/net/ixgbe/base/ixgbe_82598.h | 2 +- dpdk/drivers/net/ixgbe/base/ixgbe_82599.c | 20 +- dpdk/drivers/net/ixgbe/base/ixgbe_82599.h | 2 +- dpdk/drivers/net/ixgbe/base/ixgbe_api.c | 21 +- dpdk/drivers/net/ixgbe/base/ixgbe_api.h | 6 +- dpdk/drivers/net/ixgbe/base/ixgbe_common.c | 216 +- dpdk/drivers/net/ixgbe/base/ixgbe_common.h | 9 +- dpdk/drivers/net/ixgbe/base/ixgbe_hv_vf.c | 240 + dpdk/drivers/net/ixgbe/base/ixgbe_hv_vf.h | 41 + dpdk/drivers/net/ixgbe/base/ixgbe_mbx.h | 15 +- dpdk/drivers/net/ixgbe/base/ixgbe_osdep.h | 24 +- dpdk/drivers/net/ixgbe/base/ixgbe_phy.c | 307 +- dpdk/drivers/net/ixgbe/base/ixgbe_phy.h | 7 +- dpdk/drivers/net/ixgbe/base/ixgbe_type.h | 216 +- dpdk/drivers/net/ixgbe/base/ixgbe_vf.c | 110 +- dpdk/drivers/net/ixgbe/base/ixgbe_vf.h | 6 +- dpdk/drivers/net/ixgbe/base/ixgbe_x540.c | 43 +- dpdk/drivers/net/ixgbe/base/ixgbe_x540.h | 2 +- dpdk/drivers/net/ixgbe/base/ixgbe_x550.c | 1664 +- dpdk/drivers/net/ixgbe/base/ixgbe_x550.h | 65 +- dpdk/drivers/net/ixgbe/ixgbe_bypass.c | 5 +- dpdk/drivers/net/ixgbe/ixgbe_bypass.h | 4 +- dpdk/drivers/net/ixgbe/ixgbe_bypass_api.h | 4 +- dpdk/drivers/net/ixgbe/ixgbe_bypass_defines.h | 4 +- dpdk/drivers/net/ixgbe/ixgbe_ethdev.c | 2445 ++- dpdk/drivers/net/ixgbe/ixgbe_ethdev.h | 345 +- dpdk/drivers/net/ixgbe/ixgbe_fdir.c | 566 +- dpdk/drivers/net/ixgbe/ixgbe_flow.c | 3223 +++ dpdk/drivers/net/ixgbe/ixgbe_ipsec.c | 737 + dpdk/drivers/net/ixgbe/ixgbe_ipsec.h | 151 + dpdk/drivers/net/ixgbe/ixgbe_pf.c | 153 +- dpdk/drivers/net/ixgbe/ixgbe_regs.h | 2 +- dpdk/drivers/net/ixgbe/ixgbe_rxtx.c | 1012 +- dpdk/drivers/net/ixgbe/ixgbe_rxtx.h | 34 +- .../drivers/net/ixgbe/ixgbe_rxtx_vec_common.h | 25 +- dpdk/drivers/net/ixgbe/ixgbe_rxtx_vec_neon.c | 69 +- dpdk/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c | 299 +- dpdk/drivers/net/ixgbe/ixgbe_tm.c | 1060 + dpdk/drivers/net/ixgbe/rte_pmd_ixgbe.c | 1043 + dpdk/drivers/net/ixgbe/rte_pmd_ixgbe.h | 668 + .../net/ixgbe/rte_pmd_ixgbe_version.map | 50 + dpdk/drivers/net/kni/Makefile | 59 + dpdk/drivers/net/kni/rte_eth_kni.c | 508 + dpdk/drivers/net/kni/rte_pmd_kni_version.map | 4 + dpdk/drivers/net/liquidio/Makefile | 61 + dpdk/drivers/net/liquidio/base/lio_23xx_reg.h | 194 + dpdk/drivers/net/liquidio/base/lio_23xx_vf.c | 588 + dpdk/drivers/net/liquidio/base/lio_23xx_vf.h | 97 + dpdk/drivers/net/liquidio/base/lio_hw_defs.h | 266 + dpdk/drivers/net/liquidio/base/lio_mbox.c | 275 + dpdk/drivers/net/liquidio/base/lio_mbox.h | 131 + dpdk/drivers/net/liquidio/lio_ethdev.c | 2182 ++ dpdk/drivers/net/liquidio/lio_ethdev.h | 205 + dpdk/drivers/net/liquidio/lio_logs.h | 91 + dpdk/drivers/net/liquidio/lio_rxtx.c | 1856 ++ dpdk/drivers/net/liquidio/lio_rxtx.h | 769 + dpdk/drivers/net/liquidio/lio_struct.h | 690 + .../net/liquidio/rte_pmd_lio_version.map | 4 + dpdk/drivers/net/mlx4/Makefile | 59 +- dpdk/drivers/net/mlx4/mlx4.c | 5802 +----- dpdk/drivers/net/mlx4/mlx4.h | 204 +- dpdk/drivers/net/mlx4/mlx4_ethdev.c | 1062 + dpdk/drivers/net/mlx4/mlx4_flow.c | 1566 ++ dpdk/drivers/net/mlx4/mlx4_flow.h | 85 + dpdk/drivers/net/mlx4/mlx4_intr.c | 397 + dpdk/drivers/net/mlx4/mlx4_mr.c | 293 + dpdk/drivers/net/mlx4/mlx4_prm.h | 177 + dpdk/drivers/net/mlx4/mlx4_rxq.c | 874 + dpdk/drivers/net/mlx4/mlx4_rxtx.c | 1071 + dpdk/drivers/net/mlx4/mlx4_rxtx.h | 215 + dpdk/drivers/net/mlx4/mlx4_txq.c | 413 + dpdk/drivers/net/mlx4/mlx4_utils.c | 217 + dpdk/drivers/net/mlx4/mlx4_utils.h | 127 + dpdk/drivers/net/mlx5/Makefile | 51 +- dpdk/drivers/net/mlx5/mlx5.c | 652 +- dpdk/drivers/net/mlx5/mlx5.h | 202 +- dpdk/drivers/net/mlx5/mlx5_defs.h | 46 +- dpdk/drivers/net/mlx5/mlx5_ethdev.c | 961 +- dpdk/drivers/net/mlx5/mlx5_fdir.c | 1082 - dpdk/drivers/net/mlx5/mlx5_flow.c | 3050 +++ dpdk/drivers/net/mlx5/mlx5_mac.c | 406 +- dpdk/drivers/net/mlx5/mlx5_mr.c | 291 +- dpdk/drivers/net/mlx5/mlx5_prm.h | 295 +- dpdk/drivers/net/mlx5/mlx5_rss.c | 151 +- dpdk/drivers/net/mlx5/mlx5_rxmode.c | 389 +- dpdk/drivers/net/mlx5/mlx5_rxq.c | 2302 +-- dpdk/drivers/net/mlx5/mlx5_rxtx.c | 1919 +- dpdk/drivers/net/mlx5/mlx5_rxtx.h | 674 +- dpdk/drivers/net/mlx5/mlx5_rxtx_vec.c | 371 + dpdk/drivers/net/mlx5/mlx5_rxtx_vec.h | 130 + dpdk/drivers/net/mlx5/mlx5_rxtx_vec_neon.h | 1031 + dpdk/drivers/net/mlx5/mlx5_rxtx_vec_sse.h | 986 + dpdk/drivers/net/mlx5/mlx5_socket.c | 294 + dpdk/drivers/net/mlx5/mlx5_stats.c | 384 +- dpdk/drivers/net/mlx5/mlx5_trigger.c | 369 +- dpdk/drivers/net/mlx5/mlx5_txq.c | 1017 +- dpdk/drivers/net/mlx5/mlx5_utils.h | 6 + dpdk/drivers/net/mlx5/mlx5_vlan.c | 133 +- dpdk/drivers/net/mpipe/Makefile | 47 - dpdk/drivers/net/mpipe/mpipe_tilegx.c | 1655 -- .../net/mpipe/rte_pmd_mpipe_version.map | 3 - dpdk/drivers/net/mrvl/Makefile | 68 + dpdk/drivers/net/mrvl/mrvl_ethdev.c | 2386 +++ dpdk/drivers/net/mrvl/mrvl_ethdev.h | 113 + dpdk/drivers/net/mrvl/mrvl_qos.c | 636 + dpdk/drivers/net/mrvl/mrvl_qos.h | 113 + .../drivers/net/mrvl/rte_pmd_mrvl_version.map | 3 + dpdk/drivers/net/nfp/Makefile | 10 +- dpdk/drivers/net/nfp/nfp_net.c | 1093 +- dpdk/drivers/net/nfp/nfp_net_ctrl.h | 22 + dpdk/drivers/net/nfp/nfp_net_eth.h | 82 + dpdk/drivers/net/nfp/nfp_net_pmd.h | 31 +- dpdk/drivers/net/nfp/nfp_nfpu.c | 108 + dpdk/drivers/net/nfp/nfp_nfpu.h | 55 + dpdk/drivers/net/nfp/nfp_nspu.c | 642 + dpdk/drivers/net/nfp/nfp_nspu.h | 83 + dpdk/drivers/net/null/Makefile | 17 +- dpdk/drivers/net/null/rte_eth_null.c | 104 +- dpdk/drivers/net/null/rte_eth_null.h | 40 - .../drivers/net/null/rte_pmd_null_version.map | 7 - dpdk/drivers/net/octeontx/Makefile | 79 + dpdk/drivers/net/octeontx/base/octeontx_bgx.c | 273 + dpdk/drivers/net/octeontx/base/octeontx_bgx.h | 150 + dpdk/drivers/net/octeontx/base/octeontx_io.h | 156 + .../net/octeontx/base/octeontx_pki_var.h | 237 + .../net/octeontx/base/octeontx_pkivf.c | 169 + .../net/octeontx/base/octeontx_pkivf.h | 553 + .../net/octeontx/base/octeontx_pkovf.c | 617 + .../net/octeontx/base/octeontx_pkovf.h | 97 + dpdk/drivers/net/octeontx/octeontx_ethdev.c | 1339 ++ dpdk/drivers/net/octeontx/octeontx_ethdev.h | 115 + dpdk/drivers/net/octeontx/octeontx_logs.h | 76 + dpdk/drivers/net/octeontx/octeontx_rxtx.c | 127 + dpdk/drivers/net/octeontx/octeontx_rxtx.h | 137 + .../net/octeontx/rte_pmd_octeontx_version.map | 7 + dpdk/drivers/net/pcap/Makefile | 10 +- dpdk/drivers/net/pcap/rte_eth_pcap.c | 708 +- dpdk/drivers/net/qede/Makefile | 58 +- dpdk/drivers/net/qede/base/bcm_osal.c | 134 +- dpdk/drivers/net/qede/base/bcm_osal.h | 91 +- dpdk/drivers/net/qede/base/common_hsi.h | 1422 +- dpdk/drivers/net/qede/base/ecore.h | 808 +- dpdk/drivers/net/qede/base/ecore_chain.h | 177 +- dpdk/drivers/net/qede/base/ecore_cxt.c | 818 +- dpdk/drivers/net/qede/base/ecore_cxt.h | 108 +- dpdk/drivers/net/qede/base/ecore_cxt_api.h | 47 +- dpdk/drivers/net/qede/base/ecore_dcbx.c | 1287 +- dpdk/drivers/net/qede/base/ecore_dcbx.h | 33 +- dpdk/drivers/net/qede/base/ecore_dcbx_api.h | 197 +- dpdk/drivers/net/qede/base/ecore_dev.c | 4574 +++-- dpdk/drivers/net/qede/base/ecore_dev_api.h | 457 +- .../net/qede/base/ecore_gtt_reg_addr.h | 30 +- dpdk/drivers/net/qede/base/ecore_gtt_values.h | 20 +- dpdk/drivers/net/qede/base/ecore_hsi_common.h | 2213 ++- .../net/qede/base/ecore_hsi_debug_tools.h | 1116 ++ dpdk/drivers/net/qede/base/ecore_hsi_eth.h | 2563 ++- .../net/qede/base/ecore_hsi_init_func.h | 136 + .../net/qede/base/ecore_hsi_init_tool.h | 454 + dpdk/drivers/net/qede/base/ecore_hsi_tools.h | 1081 - dpdk/drivers/net/qede/base/ecore_hw.c | 300 +- dpdk/drivers/net/qede/base/ecore_hw.h | 83 +- dpdk/drivers/net/qede/base/ecore_hw_defs.h | 39 +- .../net/qede/base/ecore_init_fw_funcs.c | 1804 +- .../net/qede/base/ecore_init_fw_funcs.h | 358 +- dpdk/drivers/net/qede/base/ecore_init_ops.c | 104 +- dpdk/drivers/net/qede/base/ecore_init_ops.h | 17 +- dpdk/drivers/net/qede/base/ecore_int.c | 1660 +- dpdk/drivers/net/qede/base/ecore_int.h | 109 +- dpdk/drivers/net/qede/base/ecore_int_api.h | 109 +- dpdk/drivers/net/qede/base/ecore_iov_api.h | 605 +- dpdk/drivers/net/qede/base/ecore_iro.h | 250 +- dpdk/drivers/net/qede/base/ecore_iro_values.h | 148 +- dpdk/drivers/net/qede/base/ecore_l2.c | 1584 +- dpdk/drivers/net/qede/base/ecore_l2.h | 210 +- dpdk/drivers/net/qede/base/ecore_l2_api.h | 282 +- dpdk/drivers/net/qede/base/ecore_mcp.c | 2859 ++- dpdk/drivers/net/qede/base/ecore_mcp.h | 477 +- dpdk/drivers/net/qede/base/ecore_mcp_api.h | 748 +- dpdk/drivers/net/qede/base/ecore_mng_tlv.c | 1534 ++ dpdk/drivers/net/qede/base/ecore_proto_if.h | 84 +- dpdk/drivers/net/qede/base/ecore_rt_defs.h | 949 +- dpdk/drivers/net/qede/base/ecore_sp_api.h | 36 +- .../drivers/net/qede/base/ecore_sp_commands.c | 554 +- .../drivers/net/qede/base/ecore_sp_commands.h | 86 +- dpdk/drivers/net/qede/base/ecore_spq.c | 482 +- dpdk/drivers/net/qede/base/ecore_spq.h | 194 +- dpdk/drivers/net/qede/base/ecore_sriov.c | 3337 +++- dpdk/drivers/net/qede/base/ecore_sriov.h | 287 +- dpdk/drivers/net/qede/base/ecore_status.h | 19 +- dpdk/drivers/net/qede/base/ecore_utils.h | 6 + dpdk/drivers/net/qede/base/ecore_vf.c | 1474 +- dpdk/drivers/net/qede/base/ecore_vf.h | 350 +- dpdk/drivers/net/qede/base/ecore_vf_api.h | 120 +- dpdk/drivers/net/qede/base/ecore_vfpf_if.h | 539 +- dpdk/drivers/net/qede/base/eth_common.h | 457 +- dpdk/drivers/net/qede/base/mcp_public.h | 1392 +- dpdk/drivers/net/qede/base/nvm_cfg.h | 2754 ++- dpdk/drivers/net/qede/base/reg_addr.h | 123 +- dpdk/drivers/net/qede/qede_eth_if.c | 463 - dpdk/drivers/net/qede/qede_eth_if.h | 177 - dpdk/drivers/net/qede/qede_ethdev.c | 2812 ++- dpdk/drivers/net/qede/qede_ethdev.h | 216 +- dpdk/drivers/net/qede/qede_fdir.c | 469 + dpdk/drivers/net/qede/qede_if.h | 115 +- dpdk/drivers/net/qede/qede_logs.h | 31 +- dpdk/drivers/net/qede/qede_main.c | 354 +- dpdk/drivers/net/qede/qede_rxtx.c | 2446 ++- dpdk/drivers/net/qede/qede_rxtx.h | 149 +- dpdk/drivers/net/ring/Makefile | 10 +- dpdk/drivers/net/ring/rte_eth_ring.c | 219 +- ...g_version.map => rte_pmd_ring_version.map} | 0 dpdk/drivers/net/sfc/Makefile | 149 + dpdk/drivers/net/sfc/base/README | 36 + dpdk/drivers/net/sfc/base/ef10_ev.c | 1406 ++ dpdk/drivers/net/sfc/base/ef10_filter.c | 1663 ++ dpdk/drivers/net/sfc/base/ef10_impl.h | 1208 ++ dpdk/drivers/net/sfc/base/ef10_intr.c | 197 + dpdk/drivers/net/sfc/base/ef10_mac.c | 897 + dpdk/drivers/net/sfc/base/ef10_mcdi.c | 342 + dpdk/drivers/net/sfc/base/ef10_nic.c | 1790 ++ dpdk/drivers/net/sfc/base/ef10_nvram.c | 2385 +++ dpdk/drivers/net/sfc/base/ef10_phy.c | 631 + dpdk/drivers/net/sfc/base/ef10_rx.c | 1033 + dpdk/drivers/net/sfc/base/ef10_tlv_layout.h | 941 + dpdk/drivers/net/sfc/base/ef10_tx.c | 722 + dpdk/drivers/net/sfc/base/ef10_vpd.c | 463 + dpdk/drivers/net/sfc/base/efx.h | 2596 +++ dpdk/drivers/net/sfc/base/efx_bootcfg.c | 563 + dpdk/drivers/net/sfc/base/efx_check.h | 346 + dpdk/drivers/net/sfc/base/efx_crc32.c | 122 + dpdk/drivers/net/sfc/base/efx_ev.c | 1470 ++ dpdk/drivers/net/sfc/base/efx_filter.c | 1513 ++ dpdk/drivers/net/sfc/base/efx_hash.c | 328 + dpdk/drivers/net/sfc/base/efx_impl.h | 1214 ++ dpdk/drivers/net/sfc/base/efx_intr.c | 572 + dpdk/drivers/net/sfc/base/efx_lic.c | 1751 ++ dpdk/drivers/net/sfc/base/efx_mac.c | 951 + dpdk/drivers/net/sfc/base/efx_mcdi.c | 2346 +++ dpdk/drivers/net/sfc/base/efx_mcdi.h | 415 + dpdk/drivers/net/sfc/base/efx_mon.c | 255 + dpdk/drivers/net/sfc/base/efx_nic.c | 1110 ++ dpdk/drivers/net/sfc/base/efx_nvram.c | 1044 + dpdk/drivers/net/sfc/base/efx_phy.c | 561 + dpdk/drivers/net/sfc/base/efx_phy_ids.h | 51 + dpdk/drivers/net/sfc/base/efx_port.c | 252 + dpdk/drivers/net/sfc/base/efx_regs.h | 3870 ++++ dpdk/drivers/net/sfc/base/efx_regs_ef10.h | 571 + dpdk/drivers/net/sfc/base/efx_regs_mcdi.h | 15690 +++++++++++++++ dpdk/drivers/net/sfc/base/efx_regs_pci.h | 2356 +++ dpdk/drivers/net/sfc/base/efx_rx.c | 1419 ++ dpdk/drivers/net/sfc/base/efx_sram.c | 331 + dpdk/drivers/net/sfc/base/efx_tx.c | 1097 + dpdk/drivers/net/sfc/base/efx_types.h | 1647 ++ dpdk/drivers/net/sfc/base/efx_vpd.c | 1016 + dpdk/drivers/net/sfc/base/hunt_impl.h | 74 + dpdk/drivers/net/sfc/base/hunt_nic.c | 409 + dpdk/drivers/net/sfc/base/mcdi_mon.c | 565 + dpdk/drivers/net/sfc/base/mcdi_mon.h | 74 + dpdk/drivers/net/sfc/base/medford_impl.h | 67 + dpdk/drivers/net/sfc/base/medford_nic.c | 409 + dpdk/drivers/net/sfc/base/siena_flash.h | 215 + dpdk/drivers/net/sfc/base/siena_impl.h | 431 + dpdk/drivers/net/sfc/base/siena_mac.c | 476 + dpdk/drivers/net/sfc/base/siena_mcdi.c | 263 + dpdk/drivers/net/sfc/base/siena_nic.c | 588 + dpdk/drivers/net/sfc/base/siena_nvram.c | 734 + dpdk/drivers/net/sfc/base/siena_phy.c | 797 + dpdk/drivers/net/sfc/base/siena_sram.c | 178 + dpdk/drivers/net/sfc/base/siena_vpd.c | 618 + dpdk/drivers/net/sfc/efsys.h | 780 + .../net/sfc/rte_pmd_sfc_efx_version.map | 4 + dpdk/drivers/net/sfc/sfc.c | 754 + dpdk/drivers/net/sfc/sfc.h | 362 + dpdk/drivers/net/sfc/sfc_debug.h | 58 + dpdk/drivers/net/sfc/sfc_dp.c | 100 + dpdk/drivers/net/sfc/sfc_dp.h | 125 + dpdk/drivers/net/sfc/sfc_dp_rx.h | 203 + dpdk/drivers/net/sfc/sfc_dp_tx.h | 180 + dpdk/drivers/net/sfc/sfc_ef10.h | 107 + dpdk/drivers/net/sfc/sfc_ef10_rx.c | 721 + dpdk/drivers/net/sfc/sfc_ef10_tx.c | 655 + dpdk/drivers/net/sfc/sfc_ethdev.c | 1954 ++ dpdk/drivers/net/sfc/sfc_ev.c | 925 + dpdk/drivers/net/sfc/sfc_ev.h | 129 + dpdk/drivers/net/sfc/sfc_filter.c | 137 + dpdk/drivers/net/sfc/sfc_filter.h | 62 + dpdk/drivers/net/sfc/sfc_flow.c | 1376 ++ dpdk/drivers/net/sfc/sfc_flow.h | 79 + dpdk/drivers/net/sfc/sfc_intr.c | 345 + dpdk/drivers/net/sfc/sfc_kvargs.c | 145 + dpdk/drivers/net/sfc/sfc_kvargs.h | 93 + dpdk/drivers/net/sfc/sfc_log.h | 75 + dpdk/drivers/net/sfc/sfc_mcdi.c | 331 + dpdk/drivers/net/sfc/sfc_port.c | 509 + dpdk/drivers/net/sfc/sfc_rx.c | 1384 ++ dpdk/drivers/net/sfc/sfc_rx.h | 180 + dpdk/drivers/net/sfc/sfc_tso.c | 201 + dpdk/drivers/net/sfc/sfc_tweak.h | 59 + dpdk/drivers/net/sfc/sfc_tx.c | 1050 + dpdk/drivers/net/sfc/sfc_tx.h | 166 + dpdk/drivers/net/softnic/Makefile | 60 + dpdk/drivers/net/softnic/rte_eth_softnic.c | 851 + dpdk/drivers/net/softnic/rte_eth_softnic.h | 83 + .../net/softnic/rte_eth_softnic_internals.h | 291 + dpdk/drivers/net/softnic/rte_eth_softnic_tm.c | 3452 ++++ .../softnic/rte_pmd_eth_softnic_version.map | 7 + dpdk/drivers/net/szedata2/Makefile | 11 +- dpdk/drivers/net/szedata2/rte_eth_szedata2.c | 233 +- dpdk/drivers/net/szedata2/rte_eth_szedata2.h | 356 +- dpdk/drivers/net/szedata2/szedata2_iobuf.c | 203 + dpdk/drivers/net/szedata2/szedata2_iobuf.h | 356 + dpdk/drivers/net/tap/Makefile | 96 + dpdk/drivers/net/tap/rte_eth_tap.c | 1615 ++ dpdk/drivers/net/tap/rte_eth_tap.h | 101 + dpdk/drivers/net/tap/rte_pmd_tap_version.map | 4 + dpdk/drivers/net/tap/tap_flow.c | 1668 ++ dpdk/drivers/net/tap/tap_flow.h | 83 + dpdk/drivers/net/tap/tap_netlink.c | 367 + dpdk/drivers/net/tap/tap_netlink.h | 69 + dpdk/drivers/net/tap/tap_tcmsgs.c | 323 + dpdk/drivers/net/tap/tap_tcmsgs.h | 61 + dpdk/drivers/net/thunderx/Makefile | 15 +- dpdk/drivers/net/thunderx/base/nicvf_bsvf.c | 72 + dpdk/drivers/net/thunderx/base/nicvf_bsvf.h | 76 + dpdk/drivers/net/thunderx/base/nicvf_hw.c | 52 +- dpdk/drivers/net/thunderx/base/nicvf_hw.h | 33 +- .../drivers/net/thunderx/base/nicvf_hw_defs.h | 77 +- dpdk/drivers/net/thunderx/base/nicvf_mbox.c | 48 +- dpdk/drivers/net/thunderx/base/nicvf_mbox.h | 16 +- dpdk/drivers/net/thunderx/base/nicvf_plat.h | 47 +- dpdk/drivers/net/thunderx/nicvf_ethdev.c | 978 +- dpdk/drivers/net/thunderx/nicvf_ethdev.h | 51 +- dpdk/drivers/net/thunderx/nicvf_logs.h | 4 +- dpdk/drivers/net/thunderx/nicvf_rxtx.c | 55 +- dpdk/drivers/net/thunderx/nicvf_rxtx.h | 35 +- dpdk/drivers/net/thunderx/nicvf_struct.h | 41 +- dpdk/drivers/net/thunderx/nicvf_svf.c | 78 + dpdk/drivers/net/thunderx/nicvf_svf.h | 66 + dpdk/drivers/net/vhost/Makefile | 13 +- dpdk/drivers/net/vhost/rte_eth_vhost.c | 761 +- dpdk/drivers/net/vhost/rte_eth_vhost.h | 43 +- .../net/vhost/rte_pmd_vhost_version.map | 9 +- dpdk/drivers/net/virtio/Makefile | 21 +- dpdk/drivers/net/virtio/virtio_ethdev.c | 1036 +- dpdk/drivers/net/virtio/virtio_ethdev.h | 47 +- dpdk/drivers/net/virtio/virtio_pci.c | 335 +- dpdk/drivers/net/virtio/virtio_pci.h | 63 +- dpdk/drivers/net/virtio/virtio_rxtx.c | 621 +- dpdk/drivers/net/virtio/virtio_rxtx.h | 15 +- dpdk/drivers/net/virtio/virtio_rxtx_simple.c | 280 +- dpdk/drivers/net/virtio/virtio_rxtx_simple.h | 136 + .../net/virtio/virtio_rxtx_simple_neon.c | 238 + .../net/virtio/virtio_rxtx_simple_sse.c | 225 + dpdk/drivers/net/virtio/virtio_user/vhost.h | 51 +- .../net/virtio/virtio_user/vhost_kernel.c | 403 + .../net/virtio/virtio_user/vhost_kernel_tap.c | 133 + .../net/virtio/virtio_user/vhost_kernel_tap.h | 67 + .../net/virtio/virtio_user/vhost_user.c | 102 +- .../net/virtio/virtio_user/virtio_user_dev.c | 276 +- .../net/virtio/virtio_user/virtio_user_dev.h | 25 +- dpdk/drivers/net/virtio/virtio_user_ethdev.c | 226 +- dpdk/drivers/net/virtio/virtqueue.c | 81 +- dpdk/drivers/net/virtio/virtqueue.h | 62 +- dpdk/drivers/net/vmxnet3/Makefile | 8 +- dpdk/drivers/net/vmxnet3/base/vmxnet3_defs.h | 85 +- dpdk/drivers/net/vmxnet3/vmxnet3_ethdev.c | 691 +- dpdk/drivers/net/vmxnet3/vmxnet3_ethdev.h | 74 +- dpdk/drivers/net/vmxnet3/vmxnet3_ring.h | 42 +- dpdk/drivers/net/vmxnet3/vmxnet3_rxtx.c | 332 +- dpdk/drivers/net/xenvirt/Makefile | 63 - dpdk/drivers/net/xenvirt/rte_eth_xenvirt.c | 770 - dpdk/drivers/net/xenvirt/rte_eth_xenvirt.h | 62 - .../net/xenvirt/rte_eth_xenvirt_version.map | 7 - .../net/xenvirt/rte_mempool_gntalloc.c | 295 - dpdk/drivers/net/xenvirt/rte_xen_lib.c | 454 - dpdk/drivers/net/xenvirt/rte_xen_lib.h | 116 - dpdk/drivers/net/xenvirt/virtio_logs.h | 70 - dpdk/drivers/net/xenvirt/virtqueue.h | 273 - dpdk/examples/Makefile | 20 +- dpdk/examples/bond/Makefile | 4 + dpdk/examples/bond/main.c | 55 +- dpdk/examples/cmdline/Makefile | 2 +- dpdk/examples/cmdline/main.c | 1 - dpdk/examples/distributor/Makefile | 2 +- dpdk/examples/distributor/main.c | 441 +- dpdk/examples/dpdk_qat/Makefile | 93 - .../config_files/coleto/dh895xcc_qa_dev0.conf | 65 - .../shumway/dh89xxcc_qa_dev0.conf | 293 - .../shumway/dh89xxcc_qa_dev1.conf | 292 - .../config_files/stargo/dh89xxcc_qa_dev0.conf | 235 - dpdk/examples/dpdk_qat/crypto.c | 944 - dpdk/examples/dpdk_qat/crypto.h | 90 - dpdk/examples/dpdk_qat/main.c | 822 - dpdk/examples/ethtool/Makefile | 2 + dpdk/examples/ethtool/ethtool-app/Makefile | 5 + dpdk/examples/ethtool/ethtool-app/ethapp.c | 3 + dpdk/examples/ethtool/ethtool-app/main.c | 13 +- dpdk/examples/ethtool/lib/Makefile | 9 +- dpdk/examples/ethtool/lib/rte_ethtool.c | 78 +- dpdk/examples/ethtool/lib/rte_ethtool.h | 46 +- .../eventdev_pipeline_sw_pmd/Makefile | 49 + dpdk/examples/eventdev_pipeline_sw_pmd/main.c | 1023 + dpdk/examples/exception_path/Makefile | 10 +- dpdk/examples/exception_path/main.c | 82 +- dpdk/examples/flow_classify/Makefile | 57 + dpdk/examples/flow_classify/flow_classify.c | 852 + .../flow_classify/ipv4_rules_file.txt | 14 + dpdk/examples/flow_filtering/Makefile | 49 + dpdk/examples/flow_filtering/flow_blocks.c | 150 + dpdk/examples/flow_filtering/main.c | 244 + dpdk/examples/helloworld/Makefile | 2 +- dpdk/examples/helloworld/main.c | 1 - dpdk/examples/ip_fragmentation/Makefile | 2 +- dpdk/examples/ip_fragmentation/main.c | 144 +- dpdk/examples/ip_pipeline/Makefile | 4 +- dpdk/examples/ip_pipeline/app.h | 184 +- .../ip_pipeline/config/diagram-generator.py | 13 +- .../ip_pipeline/config/network_layers.cfg | 4 + .../ip_pipeline/config/network_layers.sh | 10 +- .../config/pipeline-to-core-mapping.py | 11 +- dpdk/examples/ip_pipeline/config/tap.cfg | 64 + dpdk/examples/ip_pipeline/config_check.c | 31 + dpdk/examples/ip_pipeline/config_parse.c | 166 +- dpdk/examples/ip_pipeline/init.c | 186 +- .../examples/ip_pipeline/pipeline/hash_func.h | 180 +- .../ip_pipeline/pipeline/hash_func_arm64.h | 261 + .../ip_pipeline/pipeline/pipeline_common_be.c | 1 - .../ip_pipeline/pipeline/pipeline_common_fe.c | 2 +- .../pipeline/pipeline_firewall_be.c | 6 +- .../pipeline/pipeline_flow_classification.c | 10 +- .../pipeline_flow_classification_be.c | 57 +- .../pipeline/pipeline_passthrough.c | 2 +- .../pipeline/pipeline_passthrough_be.c | 349 +- .../pipeline/pipeline_passthrough_be.h | 16 +- .../ip_pipeline/pipeline/pipeline_routing.c | 20 + .../pipeline/pipeline_routing_be.c | 19 +- dpdk/examples/ip_pipeline/pipeline_be.h | 13 + dpdk/examples/ip_pipeline/thread_fe.c | 9 +- dpdk/examples/ip_reassembly/Makefile | 2 +- dpdk/examples/ip_reassembly/main.c | 87 +- dpdk/examples/ipsec-secgw/Makefile | 7 + dpdk/examples/ipsec-secgw/ep0.cfg | 160 + dpdk/examples/ipsec-secgw/ep1.cfg | 160 + dpdk/examples/ipsec-secgw/esp.c | 331 +- dpdk/examples/ipsec-secgw/esp.h | 10 - dpdk/examples/ipsec-secgw/ipip.h | 3 +- dpdk/examples/ipsec-secgw/ipsec-secgw.c | 322 +- dpdk/examples/ipsec-secgw/ipsec.c | 295 +- dpdk/examples/ipsec-secgw/ipsec.h | 91 +- dpdk/examples/ipsec-secgw/parser.c | 591 + dpdk/examples/ipsec-secgw/parser.h | 116 + dpdk/examples/ipsec-secgw/rt.c | 261 +- dpdk/examples/ipsec-secgw/sa.c | 1059 +- dpdk/examples/ipsec-secgw/sp4.c | 540 +- dpdk/examples/ipsec-secgw/sp6.c | 541 +- dpdk/examples/ipv4_multicast/Makefile | 2 +- dpdk/examples/ipv4_multicast/main.c | 55 +- dpdk/examples/kni/Makefile | 2 +- dpdk/examples/kni/main.c | 74 +- dpdk/examples/l2fwd-cat/Makefile | 5 +- dpdk/examples/l2fwd-cat/cat.c | 89 +- dpdk/examples/l2fwd-cat/l2fwd-cat.c | 22 +- dpdk/examples/l2fwd-crypto/main.c | 1173 +- dpdk/examples/l2fwd-ivshmem/Makefile | 43 - dpdk/examples/l2fwd-ivshmem/guest/Makefile | 50 - dpdk/examples/l2fwd-ivshmem/guest/guest.c | 452 - dpdk/examples/l2fwd-ivshmem/host/Makefile | 50 - dpdk/examples/l2fwd-ivshmem/host/host.c | 895 - dpdk/examples/l2fwd-ivshmem/include/common.h | 111 - dpdk/examples/l2fwd-jobstats/main.c | 75 +- dpdk/examples/l2fwd-keepalive/ka-agent/main.c | 1 - dpdk/examples/l2fwd-keepalive/main.c | 94 +- dpdk/examples/l2fwd-keepalive/shm.c | 10 + dpdk/examples/l2fwd-keepalive/shm.h | 9 + dpdk/examples/l2fwd/Makefile | 2 +- dpdk/examples/l2fwd/main.c | 127 +- dpdk/examples/l3fwd-acl/Makefile | 2 +- dpdk/examples/l3fwd-acl/main.c | 62 +- dpdk/examples/l3fwd-power/Makefile | 2 +- dpdk/examples/l3fwd-power/main.c | 231 +- dpdk/examples/l3fwd-vf/Makefile | 2 +- dpdk/examples/l3fwd-vf/main.c | 77 +- dpdk/examples/l3fwd/Makefile | 2 +- dpdk/examples/l3fwd/l3fwd.h | 12 +- dpdk/examples/l3fwd/l3fwd_altivec.h | 284 + dpdk/examples/l3fwd/l3fwd_common.h | 293 + dpdk/examples/l3fwd/l3fwd_em.c | 68 +- dpdk/examples/l3fwd/l3fwd_em.h | 8 +- dpdk/examples/l3fwd/l3fwd_em_hlm.h | 218 + dpdk/examples/l3fwd/l3fwd_em_hlm_neon.h | 74 + dpdk/examples/l3fwd/l3fwd_em_hlm_sse.h | 278 +- dpdk/examples/l3fwd/l3fwd_em_sequential.h | 126 + dpdk/examples/l3fwd/l3fwd_em_sse.h | 112 - dpdk/examples/l3fwd/l3fwd_lpm.c | 102 +- dpdk/examples/l3fwd/l3fwd_lpm.h | 32 +- dpdk/examples/l3fwd/l3fwd_lpm_altivec.h | 164 + dpdk/examples/l3fwd/l3fwd_lpm_neon.h | 193 + dpdk/examples/l3fwd/l3fwd_lpm_sse.h | 72 +- dpdk/examples/l3fwd/l3fwd_neon.h | 260 + dpdk/examples/l3fwd/l3fwd_sse.h | 261 +- dpdk/examples/l3fwd/main.c | 230 +- dpdk/examples/link_status_interrupt/Makefile | 2 +- dpdk/examples/link_status_interrupt/main.c | 48 +- dpdk/examples/load_balancer/Makefile | 2 +- dpdk/examples/load_balancer/config.c | 20 +- dpdk/examples/load_balancer/init.c | 50 +- dpdk/examples/load_balancer/main.c | 4 - dpdk/examples/load_balancer/main.h | 19 +- dpdk/examples/load_balancer/runtime.c | 53 +- dpdk/examples/multi_process/Makefile | 2 +- .../multi_process/client_server_mp/Makefile | 2 +- .../client_server_mp/mp_client/Makefile | 2 +- .../client_server_mp/mp_client/client.c | 35 +- .../client_server_mp/mp_server/Makefile | 4 +- .../client_server_mp/mp_server/args.c | 4 +- .../client_server_mp/mp_server/args.h | 2 +- .../client_server_mp/mp_server/init.c | 21 +- .../client_server_mp/mp_server/main.c | 8 +- .../client_server_mp/shared/common.h | 4 +- .../multi_process/l2fwd_fork/Makefile | 2 +- dpdk/examples/multi_process/l2fwd_fork/flib.c | 4 - dpdk/examples/multi_process/l2fwd_fork/flib.h | 2 +- dpdk/examples/multi_process/l2fwd_fork/main.c | 59 +- .../examples/multi_process/simple_mp/Makefile | 2 +- dpdk/examples/multi_process/simple_mp/main.c | 5 +- .../multi_process/simple_mp/mp_commands.c | 3 +- .../multi_process/simple_mp/mp_commands.h | 1 - .../multi_process/symmetric_mp/Makefile | 2 +- .../multi_process/symmetric_mp/main.c | 39 +- dpdk/examples/netmap_compat/Makefile | 2 +- dpdk/examples/netmap_compat/bridge/Makefile | 3 +- dpdk/examples/netmap_compat/bridge/bridge.c | 2 +- .../netmap_compat/lib/compat_netmap.c | 62 +- .../netmap_compat/lib/compat_netmap.h | 2 +- dpdk/examples/packet_ordering/Makefile | 2 +- dpdk/examples/packet_ordering/main.c | 50 +- dpdk/examples/performance-thread/Makefile | 4 +- .../common/arch/arm64/ctx.c | 90 + .../common/arch/arm64/ctx.h | 83 + .../common/arch/arm64/stack.h | 84 + .../performance-thread/common/arch/x86/ctx.h | 8 + .../common/arch/x86/stack.h | 94 + .../performance-thread/common/common.mk | 26 +- .../performance-thread/common/lthread.c | 11 +- .../performance-thread/common/lthread.h | 10 +- .../performance-thread/common/lthread_api.h | 8 + .../performance-thread/common/lthread_cond.h | 8 + .../performance-thread/common/lthread_diag.c | 3 +- .../performance-thread/common/lthread_diag.h | 9 + .../common/lthread_diag_api.h | 8 + .../performance-thread/common/lthread_int.h | 11 +- .../performance-thread/common/lthread_mutex.c | 2 +- .../performance-thread/common/lthread_mutex.h | 8 + .../common/lthread_objcache.h | 7 + .../performance-thread/common/lthread_pool.h | 17 +- .../performance-thread/common/lthread_queue.h | 17 +- .../performance-thread/common/lthread_sched.c | 21 +- .../performance-thread/common/lthread_sched.h | 19 +- .../performance-thread/common/lthread_timer.h | 24 +- .../performance-thread/common/lthread_tls.c | 14 +- .../performance-thread/common/lthread_tls.h | 7 + .../performance-thread/l3fwd-thread/main.c | 338 +- .../performance-thread/pthread_shim/main.c | 14 +- .../pthread_shim/pthread_shim.c | 43 +- .../pthread_shim/pthread_shim.h | 3 +- dpdk/examples/ptpclient/ptpclient.c | 24 +- dpdk/examples/qos_meter/Makefile | 2 +- dpdk/examples/qos_meter/main.c | 30 +- dpdk/examples/qos_sched/Makefile | 3 +- dpdk/examples/qos_sched/app_thread.c | 14 +- dpdk/examples/qos_sched/args.c | 13 +- dpdk/examples/qos_sched/cmdline.c | 30 +- dpdk/examples/qos_sched/init.c | 42 +- dpdk/examples/qos_sched/main.c | 14 +- dpdk/examples/qos_sched/main.h | 29 +- dpdk/examples/qos_sched/stats.c | 16 +- dpdk/examples/quota_watermark/Makefile | 2 +- dpdk/examples/quota_watermark/qw/Makefile | 2 +- dpdk/examples/quota_watermark/qw/args.c | 57 +- dpdk/examples/quota_watermark/qw/init.c | 174 +- dpdk/examples/quota_watermark/qw/init.h | 4 +- dpdk/examples/quota_watermark/qw/main.c | 386 +- dpdk/examples/quota_watermark/qw/main.h | 9 +- dpdk/examples/quota_watermark/qwctl/Makefile | 2 +- .../examples/quota_watermark/qwctl/commands.c | 196 +- dpdk/examples/quota_watermark/qwctl/qwctl.c | 41 +- dpdk/examples/quota_watermark/qwctl/qwctl.h | 1 + dpdk/examples/rxtx_callbacks/main.c | 20 +- dpdk/examples/server_node_efd/Makefile | 44 + dpdk/examples/server_node_efd/node/Makefile | 48 + dpdk/examples/server_node_efd/node/node.c | 415 + dpdk/examples/server_node_efd/server/Makefile | 57 + dpdk/examples/server_node_efd/server/args.c | 200 + dpdk/examples/server_node_efd/server/args.h | 39 + dpdk/examples/server_node_efd/server/init.c | 377 + dpdk/examples/server_node_efd/server/init.h | 76 + dpdk/examples/server_node_efd/server/main.c | 358 + dpdk/examples/server_node_efd/shared/common.h | 99 + dpdk/examples/service_cores/Makefile | 54 + dpdk/examples/service_cores/main.c | 246 + dpdk/examples/skeleton/basicfwd.c | 22 +- dpdk/examples/tep_termination/main.c | 59 +- dpdk/examples/tep_termination/main.h | 2 + dpdk/examples/tep_termination/vxlan.c | 11 +- dpdk/examples/tep_termination/vxlan_setup.c | 19 +- dpdk/examples/tep_termination/vxlan_setup.h | 10 +- dpdk/examples/timer/Makefile | 2 +- dpdk/examples/timer/main.c | 1 - dpdk/examples/vhost/Makefile | 4 +- dpdk/examples/vhost/main.c | 295 +- dpdk/examples/vhost/main.h | 32 +- dpdk/examples/vhost/virtio_net.c | 476 + dpdk/examples/vhost_scsi/Makefile | 59 + dpdk/examples/vhost_scsi/scsi.c | 541 + dpdk/examples/vhost_scsi/scsi_spec.h | 493 + dpdk/examples/vhost_scsi/vhost_scsi.c | 512 + dpdk/examples/vhost_scsi/vhost_scsi.h | 108 + dpdk/examples/vhost_xen/Makefile | 52 - dpdk/examples/vhost_xen/main.c | 1510 -- dpdk/examples/vhost_xen/main.h | 66 - dpdk/examples/vhost_xen/vhost_monitor.c | 595 - dpdk/examples/vhost_xen/virtio-net.h | 113 - dpdk/examples/vhost_xen/xen_vhost.h | 148 - dpdk/examples/vhost_xen/xenstore_parse.c | 775 - dpdk/examples/vm_power_manager/Makefile | 16 + .../vm_power_manager/channel_manager.c | 67 + .../vm_power_manager/channel_manager.h | 25 + .../vm_power_manager/channel_monitor.c | 336 +- .../vm_power_manager/channel_monitor.h | 18 + .../vm_power_manager/guest_cli/main.c | 2 - .../guest_cli/vm_power_cli_guest.c | 108 +- .../guest_cli/vm_power_cli_guest.h | 6 - dpdk/examples/vm_power_manager/main.c | 263 +- .../examples/vm_power_manager/power_manager.c | 68 +- .../examples/vm_power_manager/power_manager.h | 65 + dpdk/examples/vm_power_manager/vm_power_cli.c | 21 +- dpdk/examples/vmdq/Makefile | 2 +- dpdk/examples/vmdq/main.c | 26 +- dpdk/examples/vmdq_dcb/Makefile | 2 +- dpdk/examples/vmdq_dcb/main.c | 27 +- dpdk/lib/Makefile | 67 +- dpdk/lib/librte_acl/Makefile | 15 +- dpdk/lib/librte_acl/acl.h | 4 + dpdk/lib/librte_acl/acl_bld.c | 8 +- dpdk/lib/librte_acl/acl_run.h | 8 +- dpdk/lib/librte_acl/acl_run_altivec.c | 47 + dpdk/lib/librte_acl/acl_run_altivec.h | 329 + dpdk/lib/librte_acl/acl_run_avx2.h | 2 +- dpdk/lib/librte_acl/acl_run_neon.c | 4 +- dpdk/lib/librte_acl/acl_run_neon.h | 10 +- dpdk/lib/librte_acl/acl_run_sse.h | 4 +- dpdk/lib/librte_acl/rte_acl.c | 19 +- dpdk/lib/librte_acl/rte_acl.h | 5 +- dpdk/lib/librte_acl/rte_acl_osdep.h | 2 - dpdk/lib/librte_bitratestats/Makefile | 50 + dpdk/lib/librte_bitratestats/rte_bitrate.c | 153 + dpdk/lib/librte_bitratestats/rte_bitrate.h | 94 + .../rte_bitratestats_version.map | 9 + dpdk/lib/librte_cfgfile/Makefile | 4 +- dpdk/lib/librte_cfgfile/rte_cfgfile.c | 481 +- dpdk/lib/librte_cfgfile/rte_cfgfile.h | 173 +- .../librte_cfgfile/rte_cfgfile_version.map | 18 + dpdk/lib/librte_cmdline/Makefile | 4 +- dpdk/lib/librte_cmdline/cmdline.c | 3 +- dpdk/lib/librte_cmdline/cmdline.h | 1 + dpdk/lib/librte_cmdline/cmdline_cirbuf.h | 2 + dpdk/lib/librte_cmdline/cmdline_parse.c | 56 +- dpdk/lib/librte_cmdline/cmdline_parse.h | 51 + .../librte_cmdline/cmdline_parse_etheraddr.c | 1 - dpdk/lib/librte_cmdline/cmdline_parse_num.c | 4 +- .../librte_cmdline/cmdline_parse_portlist.h | 1 + dpdk/lib/librte_cmdline/cmdline_socket.h | 3 + dpdk/lib/librte_compat/rte_compat.h | 2 +- dpdk/lib/librte_cryptodev/Makefile | 13 +- dpdk/lib/librte_cryptodev/rte_crypto.h | 52 +- dpdk/lib/librte_cryptodev/rte_crypto_sym.h | 713 +- dpdk/lib/librte_cryptodev/rte_cryptodev.c | 928 +- dpdk/lib/librte_cryptodev/rte_cryptodev.h | 591 +- dpdk/lib/librte_cryptodev/rte_cryptodev_pmd.c | 201 + dpdk/lib/librte_cryptodev/rte_cryptodev_pmd.h | 351 +- .../rte_cryptodev_version.map | 58 +- dpdk/lib/librte_distributor/Makefile | 14 +- dpdk/lib/librte_distributor/rte_distributor.c | 807 +- dpdk/lib/librte_distributor/rte_distributor.h | 76 +- .../rte_distributor_match_generic.c | 43 + .../rte_distributor_match_sse.c | 114 + .../rte_distributor_private.h | 202 + .../rte_distributor_v1705.h | 89 + .../librte_distributor/rte_distributor_v20.c | 430 + .../librte_distributor/rte_distributor_v20.h | 247 + .../rte_distributor_version.map | 14 + dpdk/lib/librte_eal/Makefile | 2 + .../librte_eal/bsdapp/contigmem/contigmem.c | 197 +- dpdk/lib/librte_eal/bsdapp/eal/Makefile | 18 +- dpdk/lib/librte_eal/bsdapp/eal/eal.c | 236 +- dpdk/lib/librte_eal/bsdapp/eal/eal_debug.c | 4 + .../librte_eal/bsdapp/eal/eal_interrupts.c | 79 +- dpdk/lib/librte_eal/bsdapp/eal/eal_lcore.c | 8 +- dpdk/lib/librte_eal/bsdapp/eal/eal_log.c | 57 - dpdk/lib/librte_eal/bsdapp/eal/eal_memory.c | 21 +- dpdk/lib/librte_eal/bsdapp/eal/eal_pci.c | 638 - dpdk/lib/librte_eal/bsdapp/eal/eal_thread.c | 2 - dpdk/lib/librte_eal/bsdapp/eal/eal_timer.c | 1 - .../eal/include/exec-env/rte_dom0_common.h | 107 - .../eal/include/exec-env/rte_interrupts.h | 137 - .../librte_eal/bsdapp/eal/rte_eal_version.map | 164 - dpdk/lib/librte_eal/bsdapp/nic_uio/nic_uio.c | 44 + dpdk/lib/librte_eal/common/Makefile | 19 +- .../librte_eal/common/arch/arm/rte_cpuflags.c | 6 +- .../librte_eal/common/arch/arm/rte_cycles.c | 45 + .../common/arch/ppc_64/rte_cpuflags.c | 2 +- .../common/arch/ppc_64/rte_cycles.c | 7 + .../common/arch/tile/rte_cpuflags.c | 47 - .../librte_eal/common/arch/x86/rte_cpuflags.c | 40 +- .../librte_eal/common/arch/x86/rte_cycles.c | 152 + .../librte_eal/common/arch/x86/rte_memcpy.c | 58 + .../librte_eal/common/arch/x86/rte_spinlock.c | 3 +- dpdk/lib/librte_eal/common/eal_common_bus.c | 243 + .../librte_eal/common/eal_common_cpuflags.c | 19 +- dpdk/lib/librte_eal/common/eal_common_dev.c | 244 +- .../librte_eal/common/eal_common_devargs.c | 162 +- dpdk/lib/librte_eal/common/eal_common_errno.c | 22 +- .../lib/librte_eal/common/eal_common_launch.c | 5 +- dpdk/lib/librte_eal/common/eal_common_lcore.c | 8 +- dpdk/lib/librte_eal/common/eal_common_log.c | 280 +- .../lib/librte_eal/common/eal_common_memory.c | 17 +- .../librte_eal/common/eal_common_memzone.c | 24 +- .../librte_eal/common/eal_common_options.c | 344 +- dpdk/lib/librte_eal/common/eal_common_pci.c | 474 - .../librte_eal/common/eal_common_pci_uio.c | 233 - dpdk/lib/librte_eal/common/eal_common_proc.c | 8 +- .../lib/librte_eal/common/eal_common_tailqs.c | 5 +- .../lib/librte_eal/common/eal_common_thread.c | 14 + dpdk/lib/librte_eal/common/eal_common_timer.c | 22 +- dpdk/lib/librte_eal/common/eal_filesystem.h | 11 - dpdk/lib/librte_eal/common/eal_hugepages.h | 3 - dpdk/lib/librte_eal/common/eal_internal_cfg.h | 4 +- dpdk/lib/librte_eal/common/eal_options.h | 5 +- dpdk/lib/librte_eal/common/eal_private.h | 159 +- .../common/include/arch/arm/rte_atomic.h | 6 - .../common/include/arch/arm/rte_atomic_32.h | 12 + .../common/include/arch/arm/rte_atomic_64.h | 55 +- .../common/include/arch/arm/rte_byteorder.h | 4 +- .../common/include/arch/arm/rte_cpuflags_64.h | 4 +- .../common/include/arch/arm/rte_cycles_64.h | 37 +- .../common/include/arch/arm/rte_io.h | 51 + .../common/include/arch/arm/rte_io_64.h | 199 + .../common/include/arch/arm/rte_memcpy_32.h | 5 +- .../common/include/arch/arm/rte_memcpy_64.h | 4 +- .../common/include/arch/arm/rte_pause.h | 50 + .../common/include/arch/arm/rte_pause_32.h | 51 + .../common/include/arch/arm/rte_pause_64.h | 52 + .../common/include/arch/arm/rte_prefetch_32.h | 1 + .../common/include/arch/arm/rte_prefetch_64.h | 5 +- .../common/include/arch/arm/rte_vect.h | 121 +- .../common/include/arch/ppc_64/rte_atomic.h | 15 +- .../include/arch/ppc_64/rte_byteorder.h | 1 + .../common/include/arch/ppc_64/rte_cycles.h | 2 + .../common/include/arch/ppc_64/rte_io.h | 47 + .../common/include/arch/ppc_64/rte_memcpy.h | 5 +- .../common/include/arch/ppc_64/rte_pause.h | 51 + .../common/include/arch/ppc_64/rte_prefetch.h | 1 + .../common/include/arch/ppc_64/rte_spinlock.h | 1 + .../common/include/arch/ppc_64/rte_vect.h | 61 + .../common/include/arch/tile/rte_atomic.h | 92 - .../common/include/arch/tile/rte_byteorder.h | 91 - .../common/include/arch/tile/rte_cpuflags.h | 53 - .../common/include/arch/tile/rte_cycles.h | 70 - .../common/include/arch/tile/rte_memcpy.h | 87 - .../common/include/arch/tile/rte_prefetch.h | 67 - .../common/include/arch/tile/rte_rwlock.h | 70 - .../common/include/arch/tile/rte_spinlock.h | 92 - .../common/include/arch/x86/rte_atomic.h | 9 + .../common/include/arch/x86/rte_atomic_32.h | 11 +- .../common/include/arch/x86/rte_atomic_64.h | 8 + .../common/include/arch/x86/rte_byteorder.h | 3 + .../include/arch/x86/rte_byteorder_32.h | 7 + .../include/arch/x86/rte_byteorder_64.h | 7 + .../common/include/arch/x86/rte_cycles.h | 34 +- .../common/include/arch/x86/rte_io.h | 47 + .../common/include/arch/x86/rte_memcpy.h | 91 +- .../common/include/arch/x86/rte_pause.h | 53 + .../common/include/arch/x86/rte_prefetch.h | 1 + .../common/include/arch/x86/rte_rtm.h | 1 + .../common/include/arch/x86/rte_spinlock.h | 1 + .../common/include/arch/x86/rte_vect.h | 32 +- .../common/include/generic/rte_atomic.h | 28 + .../common/include/generic/rte_byteorder.h | 111 +- .../common/include/generic/rte_cpuflags.h | 14 + .../common/include/generic/rte_cycles.h | 75 +- .../common/include/generic/rte_io.h | 379 + .../common/include/generic/rte_memcpy.h | 4 + .../common/include/generic/rte_pause.h | 52 + .../common/include/generic/rte_rwlock.h | 1 + .../common/include/generic/rte_spinlock.h | 1 + .../common/include/generic/rte_vect.h | 214 + .../lib/librte_eal/common/include/rte_alarm.h | 2 +- .../librte_eal/common/include/rte_bitmap.h | 562 + dpdk/lib/librte_eal/common/include/rte_bus.h | 339 + .../librte_eal/common/include/rte_common.h | 118 +- .../lib/librte_eal/common/include/rte_debug.h | 2 +- dpdk/lib/librte_eal/common/include/rte_dev.h | 219 +- .../librte_eal/common/include/rte_devargs.h | 79 +- dpdk/lib/librte_eal/common/include/rte_eal.h | 96 +- .../common/include/rte_eal_interrupts.h | 250 + .../common/include/rte_eal_memconfig.h | 2 + .../common/include/rte_interrupts.h | 15 +- .../librte_eal/common/include/rte_keepalive.h | 1 + .../lib/librte_eal/common/include/rte_lcore.h | 18 +- dpdk/lib/librte_eal/common/include/rte_log.h | 218 +- .../librte_eal/common/include/rte_malloc.h | 21 +- .../librte_eal/common/include/rte_memory.h | 109 +- .../librte_eal/common/include/rte_memzone.h | 17 +- dpdk/lib/librte_eal/common/include/rte_pci.h | 587 - .../common/include/rte_pci_dev_ids.h | 326 - .../librte_eal/common/include/rte_random.h | 2 +- .../librte_eal/common/include/rte_service.h | 431 + .../common/include/rte_service_component.h | 170 + .../lib/librte_eal/common/include/rte_tailq.h | 6 +- dpdk/lib/librte_eal/common/include/rte_time.h | 10 +- .../librte_eal/common/include/rte_version.h | 5 +- dpdk/lib/librte_eal/common/include/rte_vfio.h | 153 + .../librte_eal/common/include/rte_warnings.h | 84 - dpdk/lib/librte_eal/common/malloc_elem.c | 26 +- dpdk/lib/librte_eal/common/malloc_elem.h | 4 +- dpdk/lib/librte_eal/common/malloc_heap.c | 14 +- dpdk/lib/librte_eal/common/malloc_heap.h | 2 +- dpdk/lib/librte_eal/common/rte_keepalive.c | 29 +- dpdk/lib/librte_eal/common/rte_malloc.c | 19 +- dpdk/lib/librte_eal/common/rte_service.c | 802 + dpdk/lib/librte_eal/linuxapp/Makefile | 2 +- dpdk/lib/librte_eal/linuxapp/eal/Makefile | 36 +- dpdk/lib/librte_eal/linuxapp/eal/eal.c | 278 +- dpdk/lib/librte_eal/linuxapp/eal/eal_alarm.c | 6 +- dpdk/lib/librte_eal/linuxapp/eal/eal_debug.c | 4 + .../linuxapp/eal/eal_hugepage_info.c | 10 +- .../librte_eal/linuxapp/eal/eal_interrupts.c | 127 +- .../lib/librte_eal/linuxapp/eal/eal_ivshmem.c | 954 - dpdk/lib/librte_eal/linuxapp/eal/eal_log.c | 41 +- dpdk/lib/librte_eal/linuxapp/eal/eal_memory.c | 679 +- dpdk/lib/librte_eal/linuxapp/eal/eal_pci.c | 755 - .../librte_eal/linuxapp/eal/eal_pci_init.h | 96 - .../lib/librte_eal/linuxapp/eal/eal_pci_uio.c | 566 - .../librte_eal/linuxapp/eal/eal_pci_vfio.c | 596 - dpdk/lib/librte_eal/linuxapp/eal/eal_thread.c | 11 +- dpdk/lib/librte_eal/linuxapp/eal/eal_timer.c | 3 +- dpdk/lib/librte_eal/linuxapp/eal/eal_vfio.c | 445 +- dpdk/lib/librte_eal/linuxapp/eal/eal_vfio.h | 110 +- .../linuxapp/eal/eal_vfio_mp_sync.c | 29 +- .../librte_eal/linuxapp/eal/eal_xen_memory.c | 383 - .../eal/include/exec-env/rte_dom0_common.h | 108 - .../eal/include/exec-env/rte_interrupts.h | 228 - .../eal/include/exec-env/rte_kni_common.h | 14 +- .../linuxapp/eal/rte_eal_version.map | 168 - dpdk/lib/librte_eal/linuxapp/igb_uio/compat.h | 21 +- .../lib/librte_eal/linuxapp/igb_uio/igb_uio.c | 347 +- dpdk/lib/librte_eal/linuxapp/kni/Makefile | 65 +- dpdk/lib/librte_eal/linuxapp/kni/compat.h | 64 +- .../linuxapp/kni/ethtool/igb/COPYING | 339 - .../linuxapp/kni/ethtool/igb/e1000_82575.c | 6 +- .../linuxapp/kni/ethtool/igb/e1000_82575.h | 2 +- .../linuxapp/kni/ethtool/igb/e1000_api.c | 2 +- .../linuxapp/kni/ethtool/igb/e1000_api.h | 2 +- .../linuxapp/kni/ethtool/igb/e1000_defines.h | 2 +- .../linuxapp/kni/ethtool/igb/e1000_hw.h | 2 +- .../linuxapp/kni/ethtool/igb/e1000_i210.c | 2 +- .../linuxapp/kni/ethtool/igb/e1000_i210.h | 2 +- .../linuxapp/kni/ethtool/igb/e1000_mac.c | 2 +- .../linuxapp/kni/ethtool/igb/e1000_mac.h | 2 +- .../linuxapp/kni/ethtool/igb/e1000_manage.c | 2 +- .../linuxapp/kni/ethtool/igb/e1000_manage.h | 2 +- .../linuxapp/kni/ethtool/igb/e1000_mbx.c | 2 +- .../linuxapp/kni/ethtool/igb/e1000_mbx.h | 2 +- .../linuxapp/kni/ethtool/igb/e1000_nvm.c | 2 +- .../linuxapp/kni/ethtool/igb/e1000_nvm.h | 2 +- .../linuxapp/kni/ethtool/igb/e1000_osdep.h | 2 +- .../linuxapp/kni/ethtool/igb/e1000_phy.c | 2 +- .../linuxapp/kni/ethtool/igb/e1000_phy.h | 2 +- .../linuxapp/kni/ethtool/igb/e1000_regs.h | 2 +- .../librte_eal/linuxapp/kni/ethtool/igb/igb.h | 4 +- .../linuxapp/kni/ethtool/igb/igb_debugfs.c | 28 - .../linuxapp/kni/ethtool/igb/igb_ethtool.c | 4 +- .../linuxapp/kni/ethtool/igb/igb_hwmon.c | 260 - .../linuxapp/kni/ethtool/igb/igb_main.c | 62 +- .../linuxapp/kni/ethtool/igb/igb_param.c | 2 +- .../linuxapp/kni/ethtool/igb/igb_procfs.c | 363 - .../linuxapp/kni/ethtool/igb/igb_ptp.c | 944 - .../linuxapp/kni/ethtool/igb/igb_regtest.h | 2 +- .../linuxapp/kni/ethtool/igb/igb_vmdq.c | 2 +- .../linuxapp/kni/ethtool/igb/igb_vmdq.h | 2 +- .../linuxapp/kni/ethtool/igb/kcompat.c | 1482 -- .../linuxapp/kni/ethtool/igb/kcompat.h | 32 +- .../kni/ethtool/igb/kcompat_ethtool.c | 1171 -- .../linuxapp/kni/ethtool/ixgbe/COPYING | 339 - .../linuxapp/kni/ethtool/ixgbe/ixgbe.h | 2 +- .../linuxapp/kni/ethtool/ixgbe/ixgbe_82598.c | 2 +- .../linuxapp/kni/ethtool/ixgbe/ixgbe_82598.h | 2 +- .../linuxapp/kni/ethtool/ixgbe/ixgbe_82599.c | 2 +- .../linuxapp/kni/ethtool/ixgbe/ixgbe_82599.h | 2 +- .../linuxapp/kni/ethtool/ixgbe/ixgbe_api.c | 4 +- .../linuxapp/kni/ethtool/ixgbe/ixgbe_api.h | 2 +- .../linuxapp/kni/ethtool/ixgbe/ixgbe_common.c | 4 +- .../linuxapp/kni/ethtool/ixgbe/ixgbe_common.h | 2 +- .../linuxapp/kni/ethtool/ixgbe/ixgbe_dcb.h | 2 +- .../kni/ethtool/ixgbe/ixgbe_ethtool.c | 6 +- .../linuxapp/kni/ethtool/ixgbe/ixgbe_fcoe.h | 2 +- .../linuxapp/kni/ethtool/ixgbe/ixgbe_main.c | 2 +- .../linuxapp/kni/ethtool/ixgbe/ixgbe_mbx.h | 2 +- .../linuxapp/kni/ethtool/ixgbe/ixgbe_osdep.h | 2 +- .../linuxapp/kni/ethtool/ixgbe/ixgbe_phy.c | 2 +- .../linuxapp/kni/ethtool/ixgbe/ixgbe_phy.h | 2 +- .../linuxapp/kni/ethtool/ixgbe/ixgbe_sriov.h | 73 - .../linuxapp/kni/ethtool/ixgbe/ixgbe_type.h | 2 +- .../linuxapp/kni/ethtool/ixgbe/ixgbe_x540.c | 2 +- .../linuxapp/kni/ethtool/ixgbe/ixgbe_x540.h | 2 +- .../linuxapp/kni/ethtool/ixgbe/kcompat.c | 2 +- .../linuxapp/kni/ethtool/ixgbe/kcompat.h | 16 +- dpdk/lib/librte_eal/linuxapp/kni/kni_dev.h | 78 +- .../lib/librte_eal/linuxapp/kni/kni_ethtool.c | 39 +- dpdk/lib/librte_eal/linuxapp/kni/kni_fifo.h | 42 +- dpdk/lib/librte_eal/linuxapp/kni/kni_misc.c | 562 +- dpdk/lib/librte_eal/linuxapp/kni/kni_net.c | 774 +- dpdk/lib/librte_eal/linuxapp/kni/kni_vhost.c | 857 - .../lib/librte_eal/linuxapp/xen_dom0/Makefile | 56 - .../lib/librte_eal/linuxapp/xen_dom0/compat.h | 15 - .../linuxapp/xen_dom0/dom0_mm_dev.h | 107 - .../linuxapp/xen_dom0/dom0_mm_misc.c | 780 - dpdk/lib/librte_eal/rte_eal_version.map | 238 + dpdk/lib/librte_efd/Makefile | 51 + dpdk/lib/librte_efd/rte_efd.c | 1364 ++ dpdk/lib/librte_efd/rte_efd.h | 308 + dpdk/lib/librte_efd/rte_efd_arm64.h | 76 + dpdk/lib/librte_efd/rte_efd_version.map | 13 + dpdk/lib/librte_efd/rte_efd_x86.h | 86 + dpdk/lib/librte_ether/Makefile | 26 +- dpdk/lib/librte_ether/ethdev_profile.c | 164 + dpdk/lib/librte_ether/ethdev_profile.h | 56 + dpdk/lib/librte_ether/rte_dev_info.h | 2 + dpdk/lib/librte_ether/rte_eth_ctrl.h | 5 + dpdk/lib/librte_ether/rte_ethdev.c | 1799 +- dpdk/lib/librte_ether/rte_ethdev.h | 1793 +- dpdk/lib/librte_ether/rte_ethdev_pci.h | 196 + dpdk/lib/librte_ether/rte_ethdev_vdev.h | 84 + dpdk/lib/librte_ether/rte_ethdev_version.map | 217 + dpdk/lib/librte_ether/rte_ether.h | 416 - dpdk/lib/librte_ether/rte_ether_version.map | 141 - dpdk/lib/librte_ether/rte_flow.c | 427 + dpdk/lib/librte_ether/rte_flow.h | 1479 ++ dpdk/lib/librte_ether/rte_flow_driver.h | 149 + dpdk/lib/librte_ether/rte_mtr.c | 229 + dpdk/lib/librte_ether/rte_mtr.h | 730 + dpdk/lib/librte_ether/rte_mtr_driver.h | 221 + dpdk/lib/librte_ether/rte_tm.c | 438 + dpdk/lib/librte_ether/rte_tm.h | 1912 ++ dpdk/lib/librte_ether/rte_tm_driver.h | 366 + dpdk/lib/librte_eventdev/Makefile | 60 + .../rte_event_eth_rx_adapter.c | 1240 ++ .../rte_event_eth_rx_adapter.h | 444 + dpdk/lib/librte_eventdev/rte_event_ring.c | 207 + dpdk/lib/librte_eventdev/rte_event_ring.h | 308 + dpdk/lib/librte_eventdev/rte_eventdev.c | 1284 ++ dpdk/lib/librte_eventdev/rte_eventdev.h | 1778 ++ dpdk/lib/librte_eventdev/rte_eventdev_pmd.h | 693 + .../librte_eventdev/rte_eventdev_pmd_pci.h | 164 + .../librte_eventdev/rte_eventdev_pmd_vdev.h | 135 + .../librte_eventdev/rte_eventdev_version.map | 70 + dpdk/lib/librte_flow_classify/Makefile | 53 + .../librte_flow_classify/rte_flow_classify.c | 691 + .../librte_flow_classify/rte_flow_classify.h | 293 + .../rte_flow_classify_parse.c | 546 + .../rte_flow_classify_parse.h | 74 + .../rte_flow_classify_version.map | 12 + dpdk/lib/librte_gro/Makefile | 52 + dpdk/lib/librte_gro/gro_tcp4.c | 505 + dpdk/lib/librte_gro/gro_tcp4.h | 210 + dpdk/lib/librte_gro/rte_gro.c | 278 + dpdk/lib/librte_gro/rte_gro.h | 222 + dpdk/lib/librte_gro/rte_gro_version.map | 12 + dpdk/lib/librte_gso/Makefile | 54 + dpdk/lib/librte_gso/gso_common.c | 153 + dpdk/lib/librte_gso/gso_common.h | 171 + dpdk/lib/librte_gso/gso_tcp4.c | 102 + dpdk/lib/librte_gso/gso_tcp4.h | 74 + dpdk/lib/librte_gso/gso_tunnel_tcp4.c | 126 + dpdk/lib/librte_gso/gso_tunnel_tcp4.h | 75 + dpdk/lib/librte_gso/rte_gso.c | 110 + dpdk/lib/librte_gso/rte_gso.h | 148 + dpdk/lib/librte_gso/rte_gso_version.map | 7 + dpdk/lib/librte_hash/Makefile | 6 +- dpdk/lib/librte_hash/rte_cmp_arm64.h | 4 +- dpdk/lib/librte_hash/rte_cmp_x86.h | 6 - dpdk/lib/librte_hash/rte_crc_arm64.h | 13 +- dpdk/lib/librte_hash/rte_cuckoo_hash.c | 512 +- dpdk/lib/librte_hash/rte_cuckoo_hash.h | 72 +- dpdk/lib/librte_hash/rte_cuckoo_hash_x86.h | 25 +- dpdk/lib/librte_hash/rte_fbk_hash.c | 1 - dpdk/lib/librte_hash/rte_fbk_hash.h | 5 +- dpdk/lib/librte_hash/rte_hash_crc.h | 34 +- dpdk/lib/librte_hash/rte_jhash.h | 6 +- dpdk/lib/librte_hash/rte_thash.h | 39 +- dpdk/lib/librte_ip_frag/Makefile | 9 +- dpdk/lib/librte_ip_frag/ip_frag_common.h | 20 + dpdk/lib/librte_ip_frag/ip_frag_internal.c | 12 +- dpdk/lib/librte_ip_frag/rte_ip_frag.h | 20 +- dpdk/lib/librte_ip_frag/rte_ip_frag_common.c | 13 + .../librte_ip_frag/rte_ip_frag_version.map | 20 + .../lib/librte_ip_frag/rte_ipfrag_version.map | 13 - .../librte_ip_frag/rte_ipv4_fragmentation.c | 19 +- dpdk/lib/librte_ip_frag/rte_ipv4_reassembly.c | 6 +- dpdk/lib/librte_ip_frag/rte_ipv6_reassembly.c | 2 +- dpdk/lib/librte_ivshmem/Makefile | 54 - dpdk/lib/librte_ivshmem/rte_ivshmem.c | 919 - dpdk/lib/librte_ivshmem/rte_ivshmem.h | 165 - .../librte_ivshmem/rte_ivshmem_version.map | 12 - dpdk/lib/librte_jobstats/Makefile | 4 +- dpdk/lib/librte_jobstats/rte_jobstats.h | 6 +- dpdk/lib/librte_kni/Makefile | 6 +- dpdk/lib/librte_kni/rte_kni.c | 103 +- dpdk/lib/librte_kni/rte_kni.h | 13 +- dpdk/lib/librte_kni/rte_kni_fifo.h | 11 +- dpdk/lib/librte_kvargs/Makefile | 4 +- dpdk/lib/librte_kvargs/rte_kvargs.c | 8 +- dpdk/lib/librte_kvargs/rte_kvargs.h | 3 +- dpdk/lib/librte_latencystats/Makefile | 51 + .../librte_latencystats/rte_latencystats.c | 360 + .../librte_latencystats/rte_latencystats.h | 156 + .../rte_latencystats_version.map | 11 + dpdk/lib/librte_lpm/Makefile | 6 +- dpdk/lib/librte_lpm/rte_lpm.c | 42 +- dpdk/lib/librte_lpm/rte_lpm.h | 10 +- dpdk/lib/librte_lpm/rte_lpm6.c | 141 +- dpdk/lib/librte_lpm/rte_lpm6.h | 32 +- dpdk/lib/librte_lpm/rte_lpm_altivec.h | 154 + dpdk/lib/librte_lpm/rte_lpm_neon.h | 5 +- dpdk/lib/librte_lpm/rte_lpm_sse.h | 4 +- dpdk/lib/librte_lpm/rte_lpm_version.map | 10 + dpdk/lib/librte_mbuf/Makefile | 10 +- dpdk/lib/librte_mbuf/rte_mbuf.c | 205 +- dpdk/lib/librte_mbuf/rte_mbuf.h | 1156 +- dpdk/lib/librte_mbuf/rte_mbuf_ptype.c | 230 + dpdk/lib/librte_mbuf/rte_mbuf_ptype.h | 714 + dpdk/lib/librte_mbuf/rte_mbuf_version.map | 17 + dpdk/lib/librte_member/Makefile | 52 + dpdk/lib/librte_member/rte_member.c | 337 + dpdk/lib/librte_member/rte_member.h | 519 + dpdk/lib/librte_member/rte_member_ht.c | 586 + dpdk/lib/librte_member/rte_member_ht.h | 94 + dpdk/lib/librte_member/rte_member_vbf.c | 350 + dpdk/lib/librte_member/rte_member_vbf.h | 82 + dpdk/lib/librte_member/rte_member_version.map | 16 + dpdk/lib/librte_member/rte_member_x86.h | 107 + dpdk/lib/librte_mempool/Makefile | 7 +- dpdk/lib/librte_mempool/rte_mempool.c | 214 +- dpdk/lib/librte_mempool/rte_mempool.h | 494 +- dpdk/lib/librte_mempool/rte_mempool_ops.c | 29 + dpdk/lib/librte_mempool/rte_mempool_ring.c | 161 - dpdk/lib/librte_mempool/rte_mempool_stack.c | 147 - .../librte_mempool/rte_mempool_version.map | 11 +- dpdk/lib/librte_meter/Makefile | 4 +- dpdk/lib/librte_meter/rte_meter.h | 24 +- dpdk/lib/librte_metrics/Makefile | 50 + dpdk/lib/librte_metrics/rte_metrics.c | 304 + dpdk/lib/librte_metrics/rte_metrics.h | 251 + .../librte_metrics/rte_metrics_version.map | 13 + dpdk/lib/librte_net/Makefile | 17 +- dpdk/lib/librte_net/net_crc_neon.h | 297 + dpdk/lib/librte_net/net_crc_sse.h | 363 + dpdk/lib/librte_net/rte_esp.h | 60 + dpdk/lib/librte_net/rte_ether.h | 420 + dpdk/lib/librte_net/rte_gre.h | 71 + dpdk/lib/librte_net/rte_ip.h | 71 + dpdk/lib/librte_net/rte_net.c | 518 + dpdk/lib/librte_net/rte_net.h | 205 + dpdk/lib/librte_net/rte_net_crc.c | 225 + dpdk/lib/librte_net/rte_net_crc.h | 100 + dpdk/lib/librte_net/rte_net_version.map | 14 + dpdk/lib/librte_pci/Makefile | 49 + dpdk/lib/librte_pci/rte_pci.c | 212 + dpdk/lib/librte_pci/rte_pci.h | 263 + dpdk/lib/librte_pci/rte_pci_version.map | 15 + dpdk/lib/librte_pdump/Makefile | 9 +- dpdk/lib/librte_pdump/rte_pdump.c | 80 +- dpdk/lib/librte_pdump/rte_pdump.h | 10 +- dpdk/lib/librte_pipeline/Makefile | 9 +- dpdk/lib/librte_pipeline/rte_pipeline.c | 1 - dpdk/lib/librte_pipeline/rte_pipeline.h | 6 +- dpdk/lib/librte_port/Makefile | 18 +- dpdk/lib/librte_port/rte_port_ethdev.c | 36 +- dpdk/lib/librte_port/rte_port_ethdev.h | 6 +- dpdk/lib/librte_port/rte_port_fd.c | 547 + dpdk/lib/librte_port/rte_port_fd.h | 105 + dpdk/lib/librte_port/rte_port_frag.c | 9 +- dpdk/lib/librte_port/rte_port_kni.c | 26 +- dpdk/lib/librte_port/rte_port_ras.c | 14 +- dpdk/lib/librte_port/rte_port_ring.c | 90 +- dpdk/lib/librte_port/rte_port_sched.c | 10 +- dpdk/lib/librte_port/rte_port_source_sink.c | 27 +- dpdk/lib/librte_port/rte_port_source_sink.h | 4 +- dpdk/lib/librte_port/rte_port_version.map | 9 + dpdk/lib/librte_power/Makefile | 4 +- dpdk/lib/librte_power/channel_commands.h | 44 + dpdk/lib/librte_power/guest_channel.c | 7 + dpdk/lib/librte_power/guest_channel.h | 15 + dpdk/lib/librte_power/rte_power.c | 9 + dpdk/lib/librte_power/rte_power.h | 41 + .../lib/librte_power/rte_power_acpi_cpufreq.c | 115 +- .../lib/librte_power/rte_power_acpi_cpufreq.h | 42 +- dpdk/lib/librte_power/rte_power_kvm_vm.c | 19 + dpdk/lib/librte_power/rte_power_kvm_vm.h | 35 +- dpdk/lib/librte_power/rte_power_version.map | 10 + dpdk/lib/librte_reorder/Makefile | 6 +- dpdk/lib/librte_reorder/rte_reorder.c | 1 - dpdk/lib/librte_reorder/rte_reorder.h | 6 +- dpdk/lib/librte_ring/Makefile | 3 +- dpdk/lib/librte_ring/rte_ring.c | 101 +- dpdk/lib/librte_ring/rte_ring.h | 930 +- dpdk/lib/librte_sched/Makefile | 9 +- dpdk/lib/librte_sched/rte_bitmap.h | 560 - dpdk/lib/librte_sched/rte_reciprocal.h | 2 + dpdk/lib/librte_sched/rte_red.h | 4 +- dpdk/lib/librte_sched/rte_sched.c | 32 +- dpdk/lib/librte_sched/rte_sched_common.h | 1 + dpdk/lib/librte_security/Makefile | 54 + dpdk/lib/librte_security/rte_security.c | 149 + dpdk/lib/librte_security/rte_security.h | 534 + .../lib/librte_security/rte_security_driver.h | 156 + .../librte_security/rte_security_version.map | 14 + dpdk/lib/librte_table/Makefile | 26 +- dpdk/lib/librte_table/rte_lru.h | 109 +- dpdk/lib/librte_table/rte_lru_arm64.h | 88 + dpdk/lib/librte_table/rte_lru_x86.h | 132 + dpdk/lib/librte_table/rte_table_acl.c | 26 +- dpdk/lib/librte_table/rte_table_array.c | 11 +- dpdk/lib/librte_table/rte_table_hash.h | 286 +- dpdk/lib/librte_table/rte_table_hash_cuckoo.c | 353 + dpdk/lib/librte_table/rte_table_hash_ext.c | 433 +- dpdk/lib/librte_table/rte_table_hash_key16.c | 774 +- dpdk/lib/librte_table/rte_table_hash_key32.c | 450 +- dpdk/lib/librte_table/rte_table_hash_key8.c | 734 +- dpdk/lib/librte_table/rte_table_hash_lru.c | 526 +- dpdk/lib/librte_table/rte_table_lpm.c | 14 +- dpdk/lib/librte_table/rte_table_lpm_ipv6.c | 23 +- dpdk/lib/librte_table/rte_table_stub.c | 2 +- dpdk/lib/librte_table/rte_table_version.map | 19 +- dpdk/lib/librte_timer/Makefile | 4 +- dpdk/lib/librte_timer/rte_timer.c | 12 +- dpdk/lib/librte_timer/rte_timer.h | 3 + dpdk/lib/librte_vhost/Makefile | 25 +- dpdk/lib/librte_vhost/eventfd_link/Makefile | 41 - .../librte_vhost/eventfd_link/eventfd_link.c | 277 - .../librte_vhost/eventfd_link/eventfd_link.h | 94 - dpdk/lib/librte_vhost/fd_man.c | 303 + dpdk/lib/librte_vhost/fd_man.h | 69 + dpdk/lib/librte_vhost/iotlb.c | 358 + dpdk/lib/librte_vhost/iotlb.h | 79 + dpdk/lib/librte_vhost/libvirt/qemu-wrap.py | 387 - dpdk/lib/librte_vhost/rte_vhost.h | 501 + dpdk/lib/librte_vhost/rte_vhost_version.map | 36 +- dpdk/lib/librte_vhost/rte_virtio_net.h | 197 - dpdk/lib/librte_vhost/socket.c | 848 + dpdk/lib/librte_vhost/vhost-net.h | 250 - dpdk/lib/librte_vhost/vhost.c | 627 + dpdk/lib/librte_vhost/vhost.h | 398 + .../librte_vhost/vhost_cuse/eventfd_copy.c | 104 - .../librte_vhost/vhost_cuse/eventfd_copy.h | 45 - .../librte_vhost/vhost_cuse/vhost-net-cdev.c | 431 - .../librte_vhost/vhost_cuse/virtio-net-cdev.c | 433 - .../librte_vhost/vhost_cuse/virtio-net-cdev.h | 56 - dpdk/lib/librte_vhost/vhost_rxtx.c | 931 - dpdk/lib/librte_vhost/vhost_user.c | 1510 ++ dpdk/lib/librte_vhost/vhost_user.h | 149 + dpdk/lib/librte_vhost/vhost_user/fd_man.c | 299 - dpdk/lib/librte_vhost/vhost_user/fd_man.h | 67 - .../librte_vhost/vhost_user/vhost-net-user.c | 795 - .../librte_vhost/vhost_user/vhost-net-user.h | 113 - .../librte_vhost/vhost_user/virtio-net-user.c | 470 - .../librte_vhost/vhost_user/virtio-net-user.h | 62 - dpdk/lib/librte_vhost/virtio-net.c | 847 - dpdk/lib/librte_vhost/virtio_net.c | 1711 ++ dpdk/mk/arch/arm64/rte.vars.mk | 4 +- dpdk/mk/arch/i686/rte.vars.mk | 10 +- dpdk/mk/arch/tile/rte.vars.mk | 39 - dpdk/mk/arch/x86_64/rte.vars.mk | 10 +- dpdk/mk/exec-env/bsdapp/rte.vars.mk | 6 +- dpdk/mk/exec-env/linuxapp/rte.vars.mk | 6 +- dpdk/mk/internal/rte.compile-pre.mk | 12 +- dpdk/mk/internal/rte.depdirs-post.mk | 42 - dpdk/mk/internal/rte.depdirs-pre.mk | 32 - dpdk/mk/internal/rte.extvars.mk | 2 +- dpdk/mk/internal/rte.install-post.mk | 2 +- dpdk/mk/machine/armv7a/rte.vars.mk | 2 +- dpdk/mk/machine/armv8a/rte.vars.mk | 6 +- dpdk/mk/machine/atm/rte.vars.mk | 58 - dpdk/mk/machine/default/rte.vars.mk | 18 +- dpdk/mk/machine/dpaa/rte.vars.mk | 61 + dpdk/mk/machine/dpaa2/rte.vars.mk | 7 +- dpdk/mk/machine/hsw/rte.vars.mk | 16 +- dpdk/mk/machine/ivb/rte.vars.mk | 16 +- dpdk/mk/machine/native/rte.vars.mk | 19 +- dpdk/mk/machine/nhm/rte.vars.mk | 16 +- dpdk/mk/machine/silvermont/rte.vars.mk | 58 + dpdk/mk/machine/snb/rte.vars.mk | 16 +- dpdk/mk/machine/thunderx/rte.vars.mk | 8 +- dpdk/mk/machine/tilegx/rte.vars.mk | 57 - dpdk/mk/machine/wsm/rte.vars.mk | 16 +- dpdk/mk/machine/xgene1/rte.vars.mk | 4 +- dpdk/mk/rte.app.mk | 160 +- dpdk/mk/rte.bsdmodule.mk | 2 - dpdk/mk/rte.combinedlib.mk | 2 +- dpdk/mk/rte.cpuflags.mk | 12 + dpdk/mk/rte.extsubdir.mk | 17 +- dpdk/mk/rte.gnuconfigure.mk | 2 - dpdk/mk/rte.hostapp.mk | 6 +- dpdk/mk/rte.hostlib.mk | 2 - dpdk/mk/rte.install.mk | 2 - dpdk/mk/rte.lib.mk | 35 +- dpdk/mk/rte.module.mk | 2 - dpdk/mk/rte.obj.mk | 2 - dpdk/mk/rte.sdkbuild.mk | 30 +- dpdk/mk/rte.sdkconfig.mk | 38 +- dpdk/mk/rte.sdkdepdirs.mk | 27 - dpdk/mk/rte.sdkdoc.mk | 17 +- dpdk/mk/rte.sdkinstall.mk | 33 +- dpdk/mk/rte.sdkroot.mk | 21 +- dpdk/mk/rte.sdktest.mk | 18 +- dpdk/mk/rte.shared.mk | 6 +- dpdk/mk/rte.subdir.mk | 37 +- dpdk/mk/rte.vars.mk | 7 +- dpdk/mk/target/generic/rte.vars.mk | 48 +- .../toolchain/clang/rte.toolchain-compat.mk | 3 +- dpdk/mk/toolchain/clang/rte.vars.mk | 13 +- dpdk/mk/toolchain/gcc/rte.toolchain-compat.mk | 6 + dpdk/mk/toolchain/gcc/rte.vars.mk | 19 +- dpdk/mk/toolchain/icc/rte.toolchain-compat.mk | 7 +- dpdk/mk/toolchain/icc/rte.vars.mk | 15 +- dpdk/pkg/dpdk.spec | 17 +- dpdk/scripts/auto-config-h.sh | 136 - dpdk/scripts/check-git-log.sh | 190 - dpdk/scripts/check-maintainers.sh | 157 - dpdk/scripts/checkpatches.sh | 117 - dpdk/scripts/cocci.sh | 64 - dpdk/scripts/cocci/mtod-offset.cocci | 76 - dpdk/scripts/depdirs-rule.sh | 95 - dpdk/scripts/load-devel-config.sh | 14 - dpdk/scripts/relpath.sh | 105 - dpdk/scripts/test-build.sh | 249 - dpdk/scripts/test-null.sh | 45 - dpdk/scripts/validate-abi.sh | 262 - dpdk/test/Makefile | 39 + dpdk/test/cmdline_test/Makefile | 52 + .../{app => test}/cmdline_test/cmdline_test.c | 0 .../{app => test}/cmdline_test/cmdline_test.h | 0 dpdk/test/cmdline_test/cmdline_test.py | 118 + dpdk/test/cmdline_test/cmdline_test_data.py | 310 + dpdk/{app => test}/cmdline_test/commands.c | 0 dpdk/test/test-acl/Makefile | 45 + dpdk/test/test-acl/main.c | 1125 ++ dpdk/test/test-pipeline/Makefile | 61 + dpdk/test/test-pipeline/config.c | 262 + dpdk/test/test-pipeline/init.c | 278 + dpdk/test/test-pipeline/main.c | 186 + dpdk/test/test-pipeline/main.h | 155 + .../test-pipeline/pipeline_acl.c | 0 dpdk/test/test-pipeline/pipeline_hash.c | 474 + .../test-pipeline/pipeline_lpm.c | 0 .../test-pipeline/pipeline_lpm_ipv6.c | 0 .../test-pipeline/pipeline_stub.c | 0 dpdk/test/test-pipeline/runtime.c | 184 + dpdk/test/test/Makefile | 272 + dpdk/test/test/autotest.py | 79 + dpdk/test/test/autotest_data.py | 495 + dpdk/test/test/autotest_runner.py | 428 + dpdk/test/test/autotest_test_funcs.py | 304 + dpdk/test/test/commands.c | 403 + dpdk/test/test/packet_burst_generator.c | 476 + dpdk/test/test/packet_burst_generator.h | 106 + dpdk/test/test/process.h | 93 + dpdk/test/test/resource.c | 305 + dpdk/test/test/resource.h | 135 + dpdk/test/test/test.c | 250 + dpdk/test/test/test.h | 259 + dpdk/test/test/test_acl.c | 1652 ++ dpdk/test/test/test_acl.h | 692 + dpdk/test/test/test_alarm.c | 260 + dpdk/test/test/test_atomic.c | 375 + dpdk/test/test/test_bitmap.c | 195 + dpdk/test/test/test_byteorder.c | 95 + dpdk/test/test/test_cfgfile.c | 362 + dpdk/test/test/test_cfgfiles/etc/empty.ini | 0 .../test_cfgfiles/etc/empty_key_value.ini | 3 + .../test_cfgfiles/etc/invalid_section.ini | 3 + .../test/test_cfgfiles/etc/line_too_long.ini | 3 + .../test_cfgfiles/etc/missing_section.ini | 2 + .../test_cfgfiles/etc/realloc_sections.ini | 128 + dpdk/test/test/test_cfgfiles/etc/sample1.ini | 12 + dpdk/test/test/test_cfgfiles/etc/sample2.ini | 12 + dpdk/test/test/test_cmdline.c | 92 + dpdk/test/test/test_cmdline.h | 73 + dpdk/test/test/test_cmdline_cirbuf.c | 1330 ++ dpdk/test/test/test_cmdline_etheraddr.c | 247 + dpdk/test/test/test_cmdline_ipaddr.c | 722 + dpdk/test/test/test_cmdline_lib.c | 263 + dpdk/test/test/test_cmdline_num.c | 623 + dpdk/test/test/test_cmdline_portlist.c | 250 + dpdk/test/test/test_cmdline_string.c | 412 + dpdk/test/test/test_common.c | 194 + dpdk/test/test/test_cpuflags.c | 205 + dpdk/test/test/test_crc.c | 193 + dpdk/test/test/test_cryptodev.c | 9817 +++++++++ dpdk/test/test/test_cryptodev.h | 231 + .../test/test_cryptodev_aead_test_vectors.h | 3803 ++++ .../test/test_cryptodev_aes_test_vectors.h | 1809 ++ dpdk/test/test/test_cryptodev_blockcipher.c | 687 + dpdk/test/test/test_cryptodev_blockcipher.h | 135 + .../test/test_cryptodev_des_test_vectors.h | 1293 ++ .../test/test_cryptodev_hash_test_vectors.h | 557 + .../test/test_cryptodev_hmac_test_vectors.h | 121 + .../test_cryptodev_kasumi_hash_test_vectors.h | 248 + .../test/test_cryptodev_kasumi_test_vectors.h | 387 + .../test_cryptodev_snow3g_hash_test_vectors.h | 526 + .../test/test_cryptodev_snow3g_test_vectors.h | 411 + .../test/test_cryptodev_zuc_test_vectors.h | 1071 + dpdk/test/test/test_cycles.c | 137 + dpdk/test/test/test_debug.c | 149 + dpdk/test/test/test_devargs.c | 131 + dpdk/test/test/test_distributor.c | 726 + dpdk/test/test/test_distributor_perf.c | 296 + dpdk/test/test/test_eal_flags.c | 1362 ++ dpdk/test/test/test_eal_fs.c | 206 + dpdk/test/test/test_efd.c | 500 + dpdk/test/test/test_efd_perf.c | 414 + dpdk/test/test/test_errno.c | 116 + dpdk/test/test/test_event_eth_rx_adapter.c | 454 + dpdk/test/test/test_event_ring.c | 276 + dpdk/test/test/test_eventdev.c | 1012 + dpdk/test/test/test_eventdev_octeontx.c | 1474 ++ dpdk/test/test/test_eventdev_sw.c | 3261 +++ dpdk/test/test/test_flow_classify.c | 672 + dpdk/test/test/test_flow_classify.h | 234 + dpdk/test/test/test_func_reentrancy.c | 509 + dpdk/test/test/test_hash.c | 1516 ++ dpdk/test/test/test_hash_functions.c | 322 + dpdk/test/test/test_hash_multiwriter.c | 281 + dpdk/test/test/test_hash_perf.c | 659 + dpdk/test/test/test_hash_scaling.c | 220 + dpdk/test/test/test_interrupts.c | 552 + dpdk/test/test/test_kni.c | 637 + dpdk/test/test/test_kvargs.c | 235 + dpdk/test/test/test_link_bonding.c | 4963 +++++ dpdk/test/test/test_link_bonding_mode4.c | 1679 ++ dpdk/test/test/test_link_bonding_rssconf.c | 673 + dpdk/test/test/test_logs.c | 88 + dpdk/test/test/test_lpm.c | 1319 ++ dpdk/test/test/test_lpm6.c | 1825 ++ dpdk/test/test/test_lpm6_data.h | 1188 ++ dpdk/test/test/test_lpm6_perf.c | 192 + dpdk/test/test/test_lpm_perf.c | 513 + dpdk/test/test/test_malloc.c | 958 + dpdk/test/test/test_mbuf.c | 1166 ++ dpdk/test/test/test_member.c | 744 + dpdk/test/test/test_member_perf.c | 654 + dpdk/test/test/test_memcpy.c | 162 + dpdk/test/test/test_memcpy_perf.c | 381 + dpdk/test/test/test_memory.c | 89 + dpdk/test/test/test_mempool.c | 652 + dpdk/test/test/test_mempool_perf.c | 428 + dpdk/test/test/test_memzone.c | 1022 + dpdk/test/test/test_meter.c | 497 + dpdk/test/test/test_mp_secondary.c | 264 + dpdk/test/test/test_per_lcore.c | 137 + dpdk/test/test/test_pmd_perf.c | 916 + dpdk/test/test/test_pmd_ring.c | 529 + dpdk/test/test/test_pmd_ring_perf.c | 186 + dpdk/test/test/test_power.c | 107 + dpdk/test/test/test_power_acpi_cpufreq.c | 540 + dpdk/test/test/test_power_kvm_vm.c | 303 + dpdk/test/test/test_prefetch.c | 61 + dpdk/test/test/test_red.c | 1885 ++ dpdk/test/test/test_reorder.c | 395 + dpdk/test/test/test_resource.c | 133 + dpdk/test/test/test_ring.c | 905 + dpdk/test/test/test_ring_perf.c | 430 + dpdk/test/test/test_rwlock.c | 130 + dpdk/test/test/test_sched.c | 216 + dpdk/test/test/test_service_cores.c | 743 + dpdk/test/test/test_spinlock.c | 334 + dpdk/test/test/test_string_fns.c | 169 + dpdk/test/test/test_table.c | 215 + dpdk/test/test/test_table.h | 207 + dpdk/test/test/test_table_acl.c | 762 + dpdk/test/test/test_table_acl.h | 35 + dpdk/test/test/test_table_combined.c | 872 + dpdk/test/test/test_table_combined.h | 56 + dpdk/test/test/test_table_pipeline.c | 600 + dpdk/test/test/test_table_pipeline.h | 35 + dpdk/test/test/test_table_ports.c | 220 + dpdk/test/test/test_table_ports.h | 42 + dpdk/test/test/test_table_tables.c | 1083 + dpdk/test/test/test_table_tables.h | 51 + dpdk/test/test/test_tailq.c | 157 + dpdk/test/test/test_thash.c | 172 + dpdk/test/test/test_timer.c | 629 + dpdk/test/test/test_timer_perf.c | 163 + dpdk/test/test/test_timer_racecond.c | 206 + dpdk/test/test/test_version.c | 57 + dpdk/test/test/test_xmmt_ops.h | 83 + dpdk/test/test/virtual_pmd.c | 624 + dpdk/test/test/virtual_pmd.h | 106 + dpdk/tools/cpu_layout.py | 95 - dpdk/tools/dpdk-devbind.py | 577 - dpdk/tools/dpdk-pmdinfo.py | 636 - dpdk/tools/dpdk-setup.sh | 634 - dpdk/usertools/cpu_layout.py | 103 + dpdk/usertools/dpdk-devbind.py | 718 + dpdk/usertools/dpdk-pmdinfo.py | 610 + dpdk/usertools/dpdk-setup.sh | 634 + example/Makefile | 8 +- lib/ff_dpdk_if.c | 19 +- lib/ff_dpdk_kni.c | 17 +- lib/ff_dpdk_kni.h | 6 +- tools/prog.mk | 7 +- 2336 files changed, 575213 insertions(+), 113360 deletions(-) create mode 100644 dpdk/.gitattributes delete mode 100644 dpdk/app/cmdline_test/Makefile delete mode 100644 dpdk/app/cmdline_test/cmdline_test.py delete mode 100644 dpdk/app/cmdline_test/cmdline_test_data.py delete mode 100644 dpdk/app/test-acl/Makefile delete mode 100644 dpdk/app/test-acl/main.c create mode 100644 dpdk/app/test-crypto-perf/Makefile create mode 100644 dpdk/app/test-crypto-perf/cperf.h create mode 100644 dpdk/app/test-crypto-perf/cperf_ops.c create mode 100644 dpdk/app/test-crypto-perf/cperf_ops.h create mode 100644 dpdk/app/test-crypto-perf/cperf_options.h create mode 100644 dpdk/app/test-crypto-perf/cperf_options_parsing.c create mode 100644 dpdk/app/test-crypto-perf/cperf_test_common.c create mode 100644 dpdk/app/test-crypto-perf/cperf_test_common.h create mode 100644 dpdk/app/test-crypto-perf/cperf_test_latency.c create mode 100644 dpdk/app/test-crypto-perf/cperf_test_latency.h create mode 100644 dpdk/app/test-crypto-perf/cperf_test_pmd_cyclecount.c create mode 100644 dpdk/app/test-crypto-perf/cperf_test_pmd_cyclecount.h create mode 100644 dpdk/app/test-crypto-perf/cperf_test_throughput.c create mode 100644 dpdk/app/test-crypto-perf/cperf_test_throughput.h create mode 100644 dpdk/app/test-crypto-perf/cperf_test_vector_parsing.c create mode 100644 dpdk/app/test-crypto-perf/cperf_test_vector_parsing.h create mode 100644 dpdk/app/test-crypto-perf/cperf_test_vectors.c create mode 100644 dpdk/app/test-crypto-perf/cperf_test_vectors.h create mode 100644 dpdk/app/test-crypto-perf/cperf_test_verify.c create mode 100644 dpdk/app/test-crypto-perf/cperf_test_verify.h create mode 100644 dpdk/app/test-crypto-perf/data/aes_cbc_128_sha.data create mode 100644 dpdk/app/test-crypto-perf/data/aes_cbc_192_sha.data create mode 100644 dpdk/app/test-crypto-perf/data/aes_cbc_256_sha.data create mode 100644 dpdk/app/test-crypto-perf/main.c create mode 100644 dpdk/app/test-eventdev/Makefile create mode 100644 dpdk/app/test-eventdev/evt_common.h create mode 100644 dpdk/app/test-eventdev/evt_main.c create mode 100644 dpdk/app/test-eventdev/evt_options.c create mode 100644 dpdk/app/test-eventdev/evt_options.h create mode 100644 dpdk/app/test-eventdev/evt_test.c create mode 100644 dpdk/app/test-eventdev/evt_test.h create mode 100644 dpdk/app/test-eventdev/parser.c create mode 100644 dpdk/app/test-eventdev/parser.h create mode 100644 dpdk/app/test-eventdev/test_order_atq.c create mode 100644 dpdk/app/test-eventdev/test_order_common.c create mode 100644 dpdk/app/test-eventdev/test_order_common.h create mode 100644 dpdk/app/test-eventdev/test_order_queue.c create mode 100644 dpdk/app/test-eventdev/test_perf_atq.c create mode 100644 dpdk/app/test-eventdev/test_perf_common.c create mode 100644 dpdk/app/test-eventdev/test_perf_common.h create mode 100644 dpdk/app/test-eventdev/test_perf_queue.c delete mode 100644 dpdk/app/test-pipeline/Makefile delete mode 100644 dpdk/app/test-pipeline/config.c delete mode 100644 dpdk/app/test-pipeline/init.c delete mode 100644 dpdk/app/test-pipeline/main.c delete mode 100644 dpdk/app/test-pipeline/main.h delete mode 100644 dpdk/app/test-pipeline/pipeline_hash.c delete mode 100644 dpdk/app/test-pipeline/runtime.c create mode 100644 dpdk/app/test-pmd/cmdline_flow.c create mode 100644 dpdk/app/test-pmd/cmdline_mtr.c create mode 100644 dpdk/app/test-pmd/cmdline_mtr.h create mode 100644 dpdk/app/test-pmd/cmdline_tm.c create mode 100644 dpdk/app/test-pmd/cmdline_tm.h create mode 100644 dpdk/app/test-pmd/tm.c create mode 100644 dpdk/buildtools/auto-config-h.sh rename dpdk/{scripts => buildtools}/gen-build-mk.sh (100%) mode change 100755 => 100644 rename dpdk/{scripts => buildtools}/gen-config-h.sh (100%) mode change 100755 => 100644 create mode 100644 dpdk/buildtools/relpath.sh create mode 100644 dpdk/config/common_armv8a_linuxapp create mode 100644 dpdk/config/defconfig_arm64-armv8a-linuxapp-clang create mode 100644 dpdk/config/defconfig_arm64-dpaa-linuxapp-gcc delete mode 100644 dpdk/config/defconfig_tile-tilegx-linuxapp-gcc delete mode 100644 dpdk/config/defconfig_x86_64-ivshmem-linuxapp-gcc delete mode 100644 dpdk/config/defconfig_x86_64-ivshmem-linuxapp-icc create mode 100644 dpdk/devtools/build-tags.sh create mode 100644 dpdk/devtools/check-dup-includes.sh create mode 100644 dpdk/devtools/check-git-log.sh create mode 100644 dpdk/devtools/check-includes.sh create mode 100644 dpdk/devtools/check-maintainers.sh create mode 100644 dpdk/devtools/checkpatches.sh create mode 100644 dpdk/devtools/cocci.sh create mode 100644 dpdk/devtools/cocci/mtod-offset.cocci create mode 100644 dpdk/devtools/get-maintainer.sh create mode 100644 dpdk/devtools/git-log-fixes.sh create mode 100644 dpdk/devtools/load-devel-config create mode 100644 dpdk/devtools/test-build.sh create mode 100644 dpdk/devtools/test-null.sh create mode 100644 dpdk/devtools/validate-abi.sh delete mode 100644 dpdk/doc/api/examples.dox create mode 100644 dpdk/doc/guides/contributing/stable.rst create mode 100644 dpdk/doc/guides/cryptodevs/armv8.rst create mode 100644 dpdk/doc/guides/cryptodevs/dpaa2_sec.rst create mode 100644 dpdk/doc/guides/cryptodevs/dpaa_sec.rst create mode 100644 dpdk/doc/guides/cryptodevs/features/aesni_gcm.ini create mode 100644 dpdk/doc/guides/cryptodevs/features/aesni_mb.ini create mode 100644 dpdk/doc/guides/cryptodevs/features/armv8.ini create mode 100644 dpdk/doc/guides/cryptodevs/features/default.ini create mode 100644 dpdk/doc/guides/cryptodevs/features/dpaa2_sec.ini create mode 100644 dpdk/doc/guides/cryptodevs/features/dpaa_sec.ini create mode 100644 dpdk/doc/guides/cryptodevs/features/kasumi.ini create mode 100644 dpdk/doc/guides/cryptodevs/features/mrvl.ini create mode 100644 dpdk/doc/guides/cryptodevs/features/null.ini create mode 100644 dpdk/doc/guides/cryptodevs/features/openssl.ini create mode 100644 dpdk/doc/guides/cryptodevs/features/qat.ini create mode 100644 dpdk/doc/guides/cryptodevs/features/snow3g.ini create mode 100644 dpdk/doc/guides/cryptodevs/features/zuc.ini create mode 100644 dpdk/doc/guides/cryptodevs/img/scheduler-overview.svg create mode 100644 dpdk/doc/guides/cryptodevs/mrvl.rst create mode 100644 dpdk/doc/guides/cryptodevs/openssl.rst create mode 100644 dpdk/doc/guides/cryptodevs/scheduler.rst create mode 100644 dpdk/doc/guides/cryptodevs/zuc.rst create mode 100644 dpdk/doc/guides/eventdevs/dpaa2.rst create mode 100644 dpdk/doc/guides/eventdevs/index.rst create mode 100644 dpdk/doc/guides/eventdevs/octeontx.rst create mode 100644 dpdk/doc/guides/eventdevs/sw.rst create mode 100644 dpdk/doc/guides/howto/img/packet_capture_framework.svg create mode 100644 dpdk/doc/guides/howto/img/pvp_2nics.svg create mode 100644 dpdk/doc/guides/howto/img/use_models_for_running_dpdk_in_containers.svg create mode 100644 dpdk/doc/guides/howto/img/vf_daemon_overview.svg create mode 100644 dpdk/doc/guides/howto/img/virtio_user_as_exceptional_path.svg create mode 100644 dpdk/doc/guides/howto/img/virtio_user_for_container_networking.svg create mode 100644 dpdk/doc/guides/howto/packet_capture_framework.rst create mode 100644 dpdk/doc/guides/howto/pvp_reference_benchmark.rst create mode 100644 dpdk/doc/guides/howto/rte_flow.rst create mode 100644 dpdk/doc/guides/howto/vfd.rst create mode 100644 dpdk/doc/guides/howto/virtio_user_as_exceptional_path.rst create mode 100644 dpdk/doc/guides/howto/virtio_user_for_container_networking.rst create mode 100644 dpdk/doc/guides/linux_gsg/linux_drivers.rst create mode 100644 dpdk/doc/guides/mempool/index.rst create mode 100644 dpdk/doc/guides/mempool/octeontx.rst create mode 100644 dpdk/doc/guides/nics/ark.rst create mode 100644 dpdk/doc/guides/nics/avp.rst create mode 100644 dpdk/doc/guides/nics/build_and_test.rst create mode 100644 dpdk/doc/guides/nics/dpaa.rst create mode 100644 dpdk/doc/guides/nics/dpaa2.rst create mode 100644 dpdk/doc/guides/nics/fail_safe.rst create mode 100644 dpdk/doc/guides/nics/features.rst create mode 100644 dpdk/doc/guides/nics/features/afpacket.ini create mode 100644 dpdk/doc/guides/nics/features/ark.ini create mode 100644 dpdk/doc/guides/nics/features/avp.ini create mode 100644 dpdk/doc/guides/nics/features/bnx2x.ini create mode 100644 dpdk/doc/guides/nics/features/bnx2x_vf.ini create mode 100644 dpdk/doc/guides/nics/features/bnxt.ini create mode 100644 dpdk/doc/guides/nics/features/bonding.ini create mode 100644 dpdk/doc/guides/nics/features/cxgbe.ini create mode 100644 dpdk/doc/guides/nics/features/default.ini create mode 100644 dpdk/doc/guides/nics/features/dpaa.ini create mode 100644 dpdk/doc/guides/nics/features/dpaa2.ini create mode 100644 dpdk/doc/guides/nics/features/e1000.ini create mode 100644 dpdk/doc/guides/nics/features/ena.ini create mode 100644 dpdk/doc/guides/nics/features/enic.ini create mode 100644 dpdk/doc/guides/nics/features/failsafe.ini create mode 100644 dpdk/doc/guides/nics/features/fm10k.ini create mode 100644 dpdk/doc/guides/nics/features/fm10k_vf.ini create mode 100644 dpdk/doc/guides/nics/features/i40e.ini create mode 100644 dpdk/doc/guides/nics/features/i40e_vec.ini create mode 100644 dpdk/doc/guides/nics/features/i40e_vf.ini create mode 100644 dpdk/doc/guides/nics/features/i40e_vf_vec.ini create mode 100644 dpdk/doc/guides/nics/features/igb.ini create mode 100644 dpdk/doc/guides/nics/features/igb_vf.ini create mode 100644 dpdk/doc/guides/nics/features/ixgbe.ini create mode 100644 dpdk/doc/guides/nics/features/ixgbe_vec.ini create mode 100644 dpdk/doc/guides/nics/features/ixgbe_vf.ini create mode 100644 dpdk/doc/guides/nics/features/ixgbe_vf_vec.ini create mode 100644 dpdk/doc/guides/nics/features/kni.ini create mode 100644 dpdk/doc/guides/nics/features/liquidio.ini create mode 100644 dpdk/doc/guides/nics/features/mlx4.ini create mode 100644 dpdk/doc/guides/nics/features/mlx5.ini create mode 100644 dpdk/doc/guides/nics/features/mrvl.ini create mode 100644 dpdk/doc/guides/nics/features/nfp_pf.ini create mode 100644 dpdk/doc/guides/nics/features/nfp_vf.ini create mode 100644 dpdk/doc/guides/nics/features/null.ini create mode 100644 dpdk/doc/guides/nics/features/octeontx.ini create mode 100644 dpdk/doc/guides/nics/features/pcap.ini create mode 100644 dpdk/doc/guides/nics/features/qede.ini create mode 100644 dpdk/doc/guides/nics/features/qede_vf.ini create mode 100644 dpdk/doc/guides/nics/features/ring.ini create mode 100644 dpdk/doc/guides/nics/features/sfc_efx.ini create mode 100644 dpdk/doc/guides/nics/features/szedata2.ini create mode 100644 dpdk/doc/guides/nics/features/tap.ini create mode 100644 dpdk/doc/guides/nics/features/thunderx.ini create mode 100644 dpdk/doc/guides/nics/features/vhost.ini create mode 100644 dpdk/doc/guides/nics/features/virtio.ini create mode 100644 dpdk/doc/guides/nics/features/virtio_vec.ini create mode 100644 dpdk/doc/guides/nics/features/vmxnet3.ini rename dpdk/doc/guides/{linux_gsg => nics}/img/intel_perf_test_setup.svg (100%) create mode 100644 dpdk/doc/guides/nics/kni.rst create mode 100644 dpdk/doc/guides/nics/liquidio.rst create mode 100644 dpdk/doc/guides/nics/mrvl.rst create mode 100644 dpdk/doc/guides/nics/octeontx.rst create mode 100644 dpdk/doc/guides/nics/sfc_efx.rst create mode 100644 dpdk/doc/guides/nics/tap.rst create mode 100644 dpdk/doc/guides/platform/index.rst create mode 100644 dpdk/doc/guides/platform/octeontx.rst create mode 100644 dpdk/doc/guides/prog_guide/efd_lib.rst create mode 100644 dpdk/doc/guides/prog_guide/event_ethernet_rx_adapter.rst create mode 100644 dpdk/doc/guides/prog_guide/eventdev.rst create mode 100644 dpdk/doc/guides/prog_guide/flow_classify_lib.rst create mode 100644 dpdk/doc/guides/prog_guide/generic_receive_offload_lib.rst create mode 100644 dpdk/doc/guides/prog_guide/generic_segmentation_offload_lib.rst create mode 100644 dpdk/doc/guides/prog_guide/img/efd_i1.svg create mode 100644 dpdk/doc/guides/prog_guide/img/efd_i10.svg create mode 100644 dpdk/doc/guides/prog_guide/img/efd_i11.svg create mode 100644 dpdk/doc/guides/prog_guide/img/efd_i12.svg create mode 100644 dpdk/doc/guides/prog_guide/img/efd_i2.svg create mode 100644 dpdk/doc/guides/prog_guide/img/efd_i3.svg create mode 100644 dpdk/doc/guides/prog_guide/img/efd_i4.svg create mode 100644 dpdk/doc/guides/prog_guide/img/efd_i5.svg create mode 100644 dpdk/doc/guides/prog_guide/img/efd_i6.svg create mode 100644 dpdk/doc/guides/prog_guide/img/efd_i7.svg create mode 100644 dpdk/doc/guides/prog_guide/img/efd_i8.svg create mode 100644 dpdk/doc/guides/prog_guide/img/efd_i9.svg create mode 100644 dpdk/doc/guides/prog_guide/img/eventdev_usage.svg create mode 100644 dpdk/doc/guides/prog_guide/img/gso-output-segment-format.svg create mode 100644 dpdk/doc/guides/prog_guide/img/gso-three-seg-mbuf.svg delete mode 100644 dpdk/doc/guides/prog_guide/img/ivshmem.png create mode 100644 dpdk/doc/guides/prog_guide/img/member_i1.svg create mode 100644 dpdk/doc/guides/prog_guide/img/member_i2.svg create mode 100644 dpdk/doc/guides/prog_guide/img/member_i3.svg create mode 100644 dpdk/doc/guides/prog_guide/img/member_i4.svg create mode 100644 dpdk/doc/guides/prog_guide/img/member_i5.svg create mode 100644 dpdk/doc/guides/prog_guide/img/member_i6.svg create mode 100644 dpdk/doc/guides/prog_guide/img/member_i7.svg delete mode 100644 dpdk/doc/guides/prog_guide/ivshmem_lib.rst create mode 100644 dpdk/doc/guides/prog_guide/member_lib.rst create mode 100644 dpdk/doc/guides/prog_guide/metrics_lib.rst create mode 100644 dpdk/doc/guides/prog_guide/rte_flow.rst create mode 100644 dpdk/doc/guides/prog_guide/rte_security.rst create mode 100644 dpdk/doc/guides/prog_guide/service_cores.rst create mode 100644 dpdk/doc/guides/prog_guide/traffic_management.rst create mode 100644 dpdk/doc/guides/prog_guide/traffic_metering_and_policing.rst create mode 100644 dpdk/doc/guides/rel_notes/release_16_11.rst create mode 100644 dpdk/doc/guides/rel_notes/release_17_02.rst create mode 100644 dpdk/doc/guides/rel_notes/release_17_05.rst create mode 100644 dpdk/doc/guides/rel_notes/release_17_08.rst create mode 100644 dpdk/doc/guides/rel_notes/release_17_11.rst delete mode 100644 dpdk/doc/guides/rel_notes/supported_os.rst create mode 100644 dpdk/doc/guides/sample_app_ug/compiling.rst create mode 100644 dpdk/doc/guides/sample_app_ug/eventdev_pipeline_sw_pmd.rst create mode 100644 dpdk/doc/guides/sample_app_ug/flow_classify.rst create mode 100644 dpdk/doc/guides/sample_app_ug/flow_filtering.rst create mode 100644 dpdk/doc/guides/sample_app_ug/img/l2_fwd_vm2vm.svg delete mode 100644 dpdk/doc/guides/sample_app_ug/img/qemu_virtio_net.png create mode 100644 dpdk/doc/guides/sample_app_ug/img/server_node_efd.svg delete mode 100644 dpdk/doc/guides/sample_app_ug/img/tx_dpdk_testpmd.png delete mode 100644 dpdk/doc/guides/sample_app_ug/img/vhost_net_arch.png delete mode 100644 dpdk/doc/guides/sample_app_ug/img/vhost_net_sample_app.png delete mode 100644 dpdk/doc/guides/sample_app_ug/img/virtio_linux_vhost.png delete mode 100644 dpdk/doc/guides/sample_app_ug/intel_quickassist.rst delete mode 100644 dpdk/doc/guides/sample_app_ug/pdump.rst delete mode 100644 dpdk/doc/guides/sample_app_ug/proc_info.rst create mode 100644 dpdk/doc/guides/sample_app_ug/server_node_efd.rst create mode 100644 dpdk/doc/guides/sample_app_ug/service_cores.rst create mode 100644 dpdk/doc/guides/sample_app_ug/vhost_scsi.rst create mode 100644 dpdk/doc/guides/tools/cryptoperf.rst create mode 100644 dpdk/doc/guides/tools/devbind.rst create mode 100644 dpdk/doc/guides/tools/img/eventdev_order_atq_test.svg create mode 100644 dpdk/doc/guides/tools/img/eventdev_order_queue_test.svg create mode 100644 dpdk/doc/guides/tools/img/eventdev_perf_atq_test.svg create mode 100644 dpdk/doc/guides/tools/img/eventdev_perf_queue_test.svg create mode 100644 dpdk/doc/guides/tools/index.rst create mode 100644 dpdk/doc/guides/tools/pdump.rst create mode 100644 dpdk/doc/guides/tools/pmdinfo.rst create mode 100644 dpdk/doc/guides/tools/proc_info.rst create mode 100644 dpdk/doc/guides/tools/testeventdev.rst delete mode 100644 dpdk/doc/guides/xen/img/dpdk_xen_pkt_switch.png delete mode 100644 dpdk/doc/guides/xen/img/grant_refs.png delete mode 100644 dpdk/doc/guides/xen/img/grant_table.png delete mode 100644 dpdk/doc/guides/xen/index.rst delete mode 100644 dpdk/doc/guides/xen/pkt_switch.rst create mode 100644 dpdk/drivers/bus/Makefile create mode 100644 dpdk/drivers/bus/dpaa/Makefile create mode 100644 dpdk/drivers/bus/dpaa/base/fman/fman.c create mode 100644 dpdk/drivers/bus/dpaa/base/fman/fman_hw.c create mode 100644 dpdk/drivers/bus/dpaa/base/fman/netcfg_layer.c create mode 100644 dpdk/drivers/bus/dpaa/base/fman/of.c create mode 100644 dpdk/drivers/bus/dpaa/base/qbman/bman.c create mode 100644 dpdk/drivers/bus/dpaa/base/qbman/bman.h create mode 100644 dpdk/drivers/bus/dpaa/base/qbman/bman_driver.c create mode 100644 dpdk/drivers/bus/dpaa/base/qbman/bman_priv.h create mode 100644 dpdk/drivers/bus/dpaa/base/qbman/dpaa_alloc.c create mode 100644 dpdk/drivers/bus/dpaa/base/qbman/dpaa_sys.c create mode 100644 dpdk/drivers/bus/dpaa/base/qbman/dpaa_sys.h create mode 100644 dpdk/drivers/bus/dpaa/base/qbman/process.c create mode 100644 dpdk/drivers/bus/dpaa/base/qbman/qman.c create mode 100644 dpdk/drivers/bus/dpaa/base/qbman/qman.h create mode 100644 dpdk/drivers/bus/dpaa/base/qbman/qman_driver.c create mode 100644 dpdk/drivers/bus/dpaa/base/qbman/qman_priv.h create mode 100644 dpdk/drivers/bus/dpaa/dpaa_bus.c create mode 100644 dpdk/drivers/bus/dpaa/include/compat.h create mode 100644 dpdk/drivers/bus/dpaa/include/dpaa_bits.h create mode 100644 dpdk/drivers/bus/dpaa/include/dpaa_list.h create mode 100644 dpdk/drivers/bus/dpaa/include/dpaa_rbtree.h create mode 100644 dpdk/drivers/bus/dpaa/include/fman.h create mode 100644 dpdk/drivers/bus/dpaa/include/fsl_bman.h create mode 100644 dpdk/drivers/bus/dpaa/include/fsl_fman.h create mode 100644 dpdk/drivers/bus/dpaa/include/fsl_fman_crc64.h create mode 100644 dpdk/drivers/bus/dpaa/include/fsl_qman.h create mode 100644 dpdk/drivers/bus/dpaa/include/fsl_usd.h create mode 100644 dpdk/drivers/bus/dpaa/include/netcfg.h create mode 100644 dpdk/drivers/bus/dpaa/include/of.h create mode 100644 dpdk/drivers/bus/dpaa/include/process.h create mode 100644 dpdk/drivers/bus/dpaa/rte_bus_dpaa_version.map create mode 100644 dpdk/drivers/bus/dpaa/rte_dpaa_bus.h create mode 100644 dpdk/drivers/bus/dpaa/rte_dpaa_logs.h create mode 100644 dpdk/drivers/bus/fslmc/Makefile create mode 100644 dpdk/drivers/bus/fslmc/fslmc_bus.c create mode 100644 dpdk/drivers/bus/fslmc/fslmc_logs.h create mode 100644 dpdk/drivers/bus/fslmc/fslmc_vfio.c create mode 100644 dpdk/drivers/bus/fslmc/fslmc_vfio.h create mode 100644 dpdk/drivers/bus/fslmc/mc/dpbp.c create mode 100644 dpdk/drivers/bus/fslmc/mc/dpci.c create mode 100644 dpdk/drivers/bus/fslmc/mc/dpcon.c create mode 100644 dpdk/drivers/bus/fslmc/mc/dpio.c create mode 100644 dpdk/drivers/bus/fslmc/mc/dpmng.c create mode 100644 dpdk/drivers/bus/fslmc/mc/fsl_dpbp.h create mode 100644 dpdk/drivers/bus/fslmc/mc/fsl_dpbp_cmd.h create mode 100644 dpdk/drivers/bus/fslmc/mc/fsl_dpci.h create mode 100644 dpdk/drivers/bus/fslmc/mc/fsl_dpci_cmd.h create mode 100644 dpdk/drivers/bus/fslmc/mc/fsl_dpcon.h create mode 100644 dpdk/drivers/bus/fslmc/mc/fsl_dpcon_cmd.h create mode 100644 dpdk/drivers/bus/fslmc/mc/fsl_dpio.h create mode 100644 dpdk/drivers/bus/fslmc/mc/fsl_dpio_cmd.h create mode 100644 dpdk/drivers/bus/fslmc/mc/fsl_dpmng.h create mode 100644 dpdk/drivers/bus/fslmc/mc/fsl_dpmng_cmd.h create mode 100644 dpdk/drivers/bus/fslmc/mc/fsl_mc_cmd.h create mode 100644 dpdk/drivers/bus/fslmc/mc/fsl_mc_sys.h create mode 100644 dpdk/drivers/bus/fslmc/mc/mc_sys.c create mode 100644 dpdk/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c create mode 100644 dpdk/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c create mode 100644 dpdk/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c create mode 100644 dpdk/drivers/bus/fslmc/portal/dpaa2_hw_dpio.h create mode 100644 dpdk/drivers/bus/fslmc/portal/dpaa2_hw_pvt.h create mode 100644 dpdk/drivers/bus/fslmc/qbman/include/compat.h create mode 100644 dpdk/drivers/bus/fslmc/qbman/include/fsl_qbman_base.h create mode 100644 dpdk/drivers/bus/fslmc/qbman/include/fsl_qbman_portal.h create mode 100644 dpdk/drivers/bus/fslmc/qbman/qbman_portal.c create mode 100644 dpdk/drivers/bus/fslmc/qbman/qbman_portal.h create mode 100644 dpdk/drivers/bus/fslmc/qbman/qbman_sys.h create mode 100644 dpdk/drivers/bus/fslmc/qbman/qbman_sys_decl.h create mode 100644 dpdk/drivers/bus/fslmc/rte_bus_fslmc_version.map create mode 100644 dpdk/drivers/bus/fslmc/rte_fslmc.h create mode 100644 dpdk/drivers/bus/pci/Makefile create mode 100644 dpdk/drivers/bus/pci/bsd/Makefile create mode 100644 dpdk/drivers/bus/pci/bsd/pci.c create mode 100644 dpdk/drivers/bus/pci/linux/Makefile create mode 100644 dpdk/drivers/bus/pci/linux/pci.c create mode 100644 dpdk/drivers/bus/pci/linux/pci_init.h create mode 100644 dpdk/drivers/bus/pci/linux/pci_uio.c create mode 100644 dpdk/drivers/bus/pci/linux/pci_vfio.c create mode 100644 dpdk/drivers/bus/pci/pci_common.c create mode 100644 dpdk/drivers/bus/pci/pci_common_uio.c create mode 100644 dpdk/drivers/bus/pci/private.h create mode 100644 dpdk/drivers/bus/pci/rte_bus_pci.h create mode 100644 dpdk/drivers/bus/pci/rte_bus_pci_version.map create mode 100644 dpdk/drivers/bus/vdev/Makefile create mode 100644 dpdk/drivers/bus/vdev/rte_bus_vdev.h create mode 100644 dpdk/drivers/bus/vdev/rte_bus_vdev_version.map create mode 100644 dpdk/drivers/bus/vdev/vdev.c create mode 100644 dpdk/drivers/bus/vdev/vdev_logs.h rename dpdk/drivers/crypto/aesni_mb/{rte_pmd_aesni_version.map => rte_pmd_aesni_mb_version.map} (100%) create mode 100644 dpdk/drivers/crypto/armv8/Makefile create mode 100644 dpdk/drivers/crypto/armv8/rte_armv8_pmd.c create mode 100644 dpdk/drivers/crypto/armv8/rte_armv8_pmd_ops.c create mode 100644 dpdk/drivers/crypto/armv8/rte_armv8_pmd_private.h create mode 100644 dpdk/drivers/crypto/armv8/rte_pmd_armv8_version.map create mode 100644 dpdk/drivers/crypto/dpaa2_sec/Makefile create mode 100644 dpdk/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c create mode 100644 dpdk/drivers/crypto/dpaa2_sec/dpaa2_sec_logs.h create mode 100644 dpdk/drivers/crypto/dpaa2_sec/dpaa2_sec_priv.h create mode 100644 dpdk/drivers/crypto/dpaa2_sec/hw/compat.h create mode 100644 dpdk/drivers/crypto/dpaa2_sec/hw/desc.h create mode 100644 dpdk/drivers/crypto/dpaa2_sec/hw/desc/algo.h create mode 100644 dpdk/drivers/crypto/dpaa2_sec/hw/desc/common.h create mode 100644 dpdk/drivers/crypto/dpaa2_sec/hw/desc/ipsec.h create mode 100644 dpdk/drivers/crypto/dpaa2_sec/hw/rta.h create mode 100644 dpdk/drivers/crypto/dpaa2_sec/hw/rta/fifo_load_store_cmd.h create mode 100644 dpdk/drivers/crypto/dpaa2_sec/hw/rta/header_cmd.h create mode 100644 dpdk/drivers/crypto/dpaa2_sec/hw/rta/jump_cmd.h create mode 100644 dpdk/drivers/crypto/dpaa2_sec/hw/rta/key_cmd.h create mode 100644 dpdk/drivers/crypto/dpaa2_sec/hw/rta/load_cmd.h create mode 100644 dpdk/drivers/crypto/dpaa2_sec/hw/rta/math_cmd.h create mode 100644 dpdk/drivers/crypto/dpaa2_sec/hw/rta/move_cmd.h create mode 100644 dpdk/drivers/crypto/dpaa2_sec/hw/rta/nfifo_cmd.h create mode 100644 dpdk/drivers/crypto/dpaa2_sec/hw/rta/operation_cmd.h create mode 100644 dpdk/drivers/crypto/dpaa2_sec/hw/rta/protocol_cmd.h create mode 100644 dpdk/drivers/crypto/dpaa2_sec/hw/rta/sec_run_time_asm.h create mode 100644 dpdk/drivers/crypto/dpaa2_sec/hw/rta/seq_in_out_ptr_cmd.h create mode 100644 dpdk/drivers/crypto/dpaa2_sec/hw/rta/signature_cmd.h create mode 100644 dpdk/drivers/crypto/dpaa2_sec/hw/rta/store_cmd.h create mode 100644 dpdk/drivers/crypto/dpaa2_sec/mc/dpseci.c create mode 100644 dpdk/drivers/crypto/dpaa2_sec/mc/fsl_dpseci.h create mode 100644 dpdk/drivers/crypto/dpaa2_sec/mc/fsl_dpseci_cmd.h create mode 100644 dpdk/drivers/crypto/dpaa2_sec/rte_pmd_dpaa2_sec_version.map create mode 100644 dpdk/drivers/crypto/dpaa_sec/Makefile create mode 100644 dpdk/drivers/crypto/dpaa_sec/dpaa_sec.c create mode 100644 dpdk/drivers/crypto/dpaa_sec/dpaa_sec.h create mode 100644 dpdk/drivers/crypto/dpaa_sec/dpaa_sec_log.h create mode 100644 dpdk/drivers/crypto/dpaa_sec/rte_pmd_dpaa_sec_version.map create mode 100644 dpdk/drivers/crypto/mrvl/Makefile create mode 100644 dpdk/drivers/crypto/mrvl/rte_mrvl_compat.h create mode 100644 dpdk/drivers/crypto/mrvl/rte_mrvl_pmd.c create mode 100644 dpdk/drivers/crypto/mrvl/rte_mrvl_pmd_ops.c create mode 100644 dpdk/drivers/crypto/mrvl/rte_mrvl_pmd_private.h create mode 100644 dpdk/drivers/crypto/mrvl/rte_pmd_mrvl_version.map create mode 100644 dpdk/drivers/crypto/openssl/Makefile create mode 100644 dpdk/drivers/crypto/openssl/rte_openssl_pmd.c create mode 100644 dpdk/drivers/crypto/openssl/rte_openssl_pmd_ops.c create mode 100644 dpdk/drivers/crypto/openssl/rte_openssl_pmd_private.h create mode 100644 dpdk/drivers/crypto/openssl/rte_pmd_openssl_version.map create mode 100644 dpdk/drivers/crypto/qat/qat_crypto_capabilities.h create mode 100644 dpdk/drivers/crypto/scheduler/Makefile create mode 100644 dpdk/drivers/crypto/scheduler/rte_cryptodev_scheduler.c create mode 100644 dpdk/drivers/crypto/scheduler/rte_cryptodev_scheduler.h create mode 100644 dpdk/drivers/crypto/scheduler/rte_cryptodev_scheduler_operations.h create mode 100644 dpdk/drivers/crypto/scheduler/rte_pmd_crypto_scheduler_version.map create mode 100644 dpdk/drivers/crypto/scheduler/scheduler_failover.c create mode 100644 dpdk/drivers/crypto/scheduler/scheduler_multicore.c create mode 100644 dpdk/drivers/crypto/scheduler/scheduler_pkt_size_distr.c create mode 100644 dpdk/drivers/crypto/scheduler/scheduler_pmd.c create mode 100644 dpdk/drivers/crypto/scheduler/scheduler_pmd_ops.c create mode 100644 dpdk/drivers/crypto/scheduler/scheduler_pmd_private.h create mode 100644 dpdk/drivers/crypto/scheduler/scheduler_roundrobin.c create mode 100644 dpdk/drivers/crypto/zuc/Makefile create mode 100644 dpdk/drivers/crypto/zuc/rte_pmd_zuc_version.map create mode 100644 dpdk/drivers/crypto/zuc/rte_zuc_pmd.c create mode 100644 dpdk/drivers/crypto/zuc/rte_zuc_pmd_ops.c create mode 100644 dpdk/drivers/crypto/zuc/rte_zuc_pmd_private.h create mode 100644 dpdk/drivers/event/Makefile create mode 100644 dpdk/drivers/event/dpaa2/Makefile create mode 100644 dpdk/drivers/event/dpaa2/dpaa2_eventdev.c create mode 100644 dpdk/drivers/event/dpaa2/dpaa2_eventdev.h create mode 100644 dpdk/drivers/event/dpaa2/dpaa2_hw_dpcon.c create mode 100644 dpdk/drivers/event/dpaa2/rte_pmd_dpaa2_event_version.map create mode 100644 dpdk/drivers/event/octeontx/Makefile create mode 100644 dpdk/drivers/event/octeontx/rte_pmd_octeontx_ssovf_version.map create mode 100644 dpdk/drivers/event/octeontx/ssovf_evdev.c create mode 100644 dpdk/drivers/event/octeontx/ssovf_evdev.h create mode 100644 dpdk/drivers/event/octeontx/ssovf_worker.c create mode 100644 dpdk/drivers/event/octeontx/ssovf_worker.h create mode 100644 dpdk/drivers/event/skeleton/Makefile create mode 100644 dpdk/drivers/event/skeleton/rte_pmd_skeleton_event_version.map create mode 100644 dpdk/drivers/event/skeleton/skeleton_eventdev.c create mode 100644 dpdk/drivers/event/skeleton/skeleton_eventdev.h create mode 100644 dpdk/drivers/event/sw/Makefile create mode 100644 dpdk/drivers/event/sw/event_ring.h create mode 100644 dpdk/drivers/event/sw/iq_ring.h create mode 100644 dpdk/drivers/event/sw/rte_pmd_sw_event_version.map create mode 100644 dpdk/drivers/event/sw/sw_evdev.c create mode 100644 dpdk/drivers/event/sw/sw_evdev.h create mode 100644 dpdk/drivers/event/sw/sw_evdev_scheduler.c create mode 100644 dpdk/drivers/event/sw/sw_evdev_worker.c create mode 100644 dpdk/drivers/event/sw/sw_evdev_xstats.c create mode 100644 dpdk/drivers/mempool/Makefile create mode 100644 dpdk/drivers/mempool/dpaa/Makefile create mode 100644 dpdk/drivers/mempool/dpaa/dpaa_mempool.c create mode 100644 dpdk/drivers/mempool/dpaa/dpaa_mempool.h create mode 100644 dpdk/drivers/mempool/dpaa/rte_mempool_dpaa_version.map create mode 100644 dpdk/drivers/mempool/dpaa2/Makefile create mode 100644 dpdk/drivers/mempool/dpaa2/dpaa2_hw_mempool.c create mode 100644 dpdk/drivers/mempool/dpaa2/dpaa2_hw_mempool.h create mode 100644 dpdk/drivers/mempool/dpaa2/rte_mempool_dpaa2_version.map create mode 100644 dpdk/drivers/mempool/octeontx/Makefile create mode 100644 dpdk/drivers/mempool/octeontx/octeontx_fpavf.c create mode 100644 dpdk/drivers/mempool/octeontx/octeontx_fpavf.h create mode 100644 dpdk/drivers/mempool/octeontx/octeontx_mbox.c create mode 100644 dpdk/drivers/mempool/octeontx/octeontx_mbox.h create mode 100644 dpdk/drivers/mempool/octeontx/octeontx_pool_logs.h create mode 100644 dpdk/drivers/mempool/octeontx/octeontx_ssovf.c create mode 100644 dpdk/drivers/mempool/octeontx/rte_mempool_octeontx.c create mode 100644 dpdk/drivers/mempool/octeontx/rte_mempool_octeontx_version.map create mode 100644 dpdk/drivers/mempool/ring/Makefile create mode 100644 dpdk/drivers/mempool/ring/rte_mempool_ring.c create mode 100644 dpdk/drivers/mempool/ring/rte_mempool_ring_version.map create mode 100644 dpdk/drivers/mempool/stack/Makefile create mode 100644 dpdk/drivers/mempool/stack/rte_mempool_stack.c create mode 100644 dpdk/drivers/mempool/stack/rte_mempool_stack_version.map create mode 100644 dpdk/drivers/net/ark/Makefile create mode 100644 dpdk/drivers/net/ark/ark_ddm.c create mode 100644 dpdk/drivers/net/ark/ark_ddm.h create mode 100644 dpdk/drivers/net/ark/ark_ethdev.c create mode 100644 dpdk/drivers/net/ark/ark_ethdev_rx.c create mode 100644 dpdk/drivers/net/ark/ark_ethdev_rx.h create mode 100644 dpdk/drivers/net/ark/ark_ethdev_tx.c create mode 100644 dpdk/drivers/net/ark/ark_ethdev_tx.h create mode 100644 dpdk/drivers/net/ark/ark_ext.h create mode 100644 dpdk/drivers/net/ark/ark_global.h create mode 100644 dpdk/drivers/net/ark/ark_logs.h create mode 100644 dpdk/drivers/net/ark/ark_mpu.c create mode 100644 dpdk/drivers/net/ark/ark_mpu.h create mode 100644 dpdk/drivers/net/ark/ark_pktchkr.c create mode 100644 dpdk/drivers/net/ark/ark_pktchkr.h create mode 100644 dpdk/drivers/net/ark/ark_pktdir.c create mode 100644 dpdk/drivers/net/ark/ark_pktdir.h create mode 100644 dpdk/drivers/net/ark/ark_pktgen.c create mode 100644 dpdk/drivers/net/ark/ark_pktgen.h create mode 100644 dpdk/drivers/net/ark/ark_rqp.c create mode 100644 dpdk/drivers/net/ark/ark_rqp.h create mode 100644 dpdk/drivers/net/ark/ark_udm.c create mode 100644 dpdk/drivers/net/ark/ark_udm.h create mode 100644 dpdk/drivers/net/ark/rte_pmd_ark_version.map create mode 100644 dpdk/drivers/net/avp/Makefile create mode 100644 dpdk/drivers/net/avp/avp_ethdev.c create mode 100644 dpdk/drivers/net/avp/avp_logs.h create mode 100644 dpdk/drivers/net/avp/rte_avp_common.h create mode 100644 dpdk/drivers/net/avp/rte_avp_fifo.h create mode 100644 dpdk/drivers/net/avp/rte_pmd_avp_version.map delete mode 100644 dpdk/drivers/net/bnx2x/debug.c create mode 100644 dpdk/drivers/net/bnxt/bnxt_irq.c create mode 100644 dpdk/drivers/net/bnxt/bnxt_irq.h create mode 100644 dpdk/drivers/net/bnxt/bnxt_nvm_defs.h create mode 100644 dpdk/drivers/net/bnxt/rte_pmd_bnxt.c create mode 100644 dpdk/drivers/net/bnxt/rte_pmd_bnxt.h delete mode 100644 dpdk/drivers/net/bonding/rte_eth_bond_version.map create mode 100644 dpdk/drivers/net/bonding/rte_pmd_bond_version.map create mode 100644 dpdk/drivers/net/dpaa/Makefile create mode 100644 dpdk/drivers/net/dpaa/dpaa_ethdev.c create mode 100644 dpdk/drivers/net/dpaa/dpaa_ethdev.h create mode 100644 dpdk/drivers/net/dpaa/dpaa_rxtx.c create mode 100644 dpdk/drivers/net/dpaa/dpaa_rxtx.h create mode 100644 dpdk/drivers/net/dpaa/rte_pmd_dpaa_version.map create mode 100644 dpdk/drivers/net/dpaa2/Makefile create mode 100644 dpdk/drivers/net/dpaa2/base/dpaa2_hw_dpni.c create mode 100644 dpdk/drivers/net/dpaa2/base/dpaa2_hw_dpni_annot.h create mode 100644 dpdk/drivers/net/dpaa2/dpaa2_ethdev.c create mode 100644 dpdk/drivers/net/dpaa2/dpaa2_ethdev.h create mode 100644 dpdk/drivers/net/dpaa2/dpaa2_rxtx.c create mode 100644 dpdk/drivers/net/dpaa2/mc/dpkg.c create mode 100644 dpdk/drivers/net/dpaa2/mc/dpni.c create mode 100644 dpdk/drivers/net/dpaa2/mc/fsl_dpkg.h create mode 100644 dpdk/drivers/net/dpaa2/mc/fsl_dpni.h create mode 100644 dpdk/drivers/net/dpaa2/mc/fsl_dpni_cmd.h create mode 100644 dpdk/drivers/net/dpaa2/mc/fsl_net.h create mode 100644 dpdk/drivers/net/dpaa2/rte_pmd_dpaa2_version.map create mode 100644 dpdk/drivers/net/e1000/igb_flow.c create mode 100644 dpdk/drivers/net/enic/enic_flow.c create mode 100644 dpdk/drivers/net/failsafe/Makefile create mode 100644 dpdk/drivers/net/failsafe/failsafe.c create mode 100644 dpdk/drivers/net/failsafe/failsafe_args.c create mode 100644 dpdk/drivers/net/failsafe/failsafe_eal.c create mode 100644 dpdk/drivers/net/failsafe/failsafe_ether.c create mode 100644 dpdk/drivers/net/failsafe/failsafe_flow.c create mode 100644 dpdk/drivers/net/failsafe/failsafe_ops.c create mode 100644 dpdk/drivers/net/failsafe/failsafe_private.h create mode 100644 dpdk/drivers/net/failsafe/failsafe_rxtx.c create mode 100644 dpdk/drivers/net/failsafe/rte_pmd_failsafe_version.map create mode 100644 dpdk/drivers/net/i40e/base/README delete mode 100644 dpdk/drivers/net/i40e/base/i40e_virtchnl.h create mode 100644 dpdk/drivers/net/i40e/base/virtchnl.h create mode 100644 dpdk/drivers/net/i40e/i40e_flow.c delete mode 100644 dpdk/drivers/net/i40e/i40e_rxtx_vec.c create mode 100644 dpdk/drivers/net/i40e/i40e_rxtx_vec_altivec.c create mode 100644 dpdk/drivers/net/i40e/i40e_rxtx_vec_common.h create mode 100644 dpdk/drivers/net/i40e/i40e_rxtx_vec_neon.c create mode 100644 dpdk/drivers/net/i40e/i40e_rxtx_vec_sse.c create mode 100644 dpdk/drivers/net/i40e/i40e_tm.c create mode 100644 dpdk/drivers/net/i40e/rte_pmd_i40e.c create mode 100644 dpdk/drivers/net/i40e/rte_pmd_i40e.h create mode 100644 dpdk/drivers/net/ixgbe/base/ixgbe_hv_vf.c create mode 100644 dpdk/drivers/net/ixgbe/base/ixgbe_hv_vf.h create mode 100644 dpdk/drivers/net/ixgbe/ixgbe_flow.c create mode 100644 dpdk/drivers/net/ixgbe/ixgbe_ipsec.c create mode 100644 dpdk/drivers/net/ixgbe/ixgbe_ipsec.h create mode 100644 dpdk/drivers/net/ixgbe/ixgbe_tm.c create mode 100644 dpdk/drivers/net/ixgbe/rte_pmd_ixgbe.c create mode 100644 dpdk/drivers/net/ixgbe/rte_pmd_ixgbe.h create mode 100644 dpdk/drivers/net/kni/Makefile create mode 100644 dpdk/drivers/net/kni/rte_eth_kni.c create mode 100644 dpdk/drivers/net/kni/rte_pmd_kni_version.map create mode 100644 dpdk/drivers/net/liquidio/Makefile create mode 100644 dpdk/drivers/net/liquidio/base/lio_23xx_reg.h create mode 100644 dpdk/drivers/net/liquidio/base/lio_23xx_vf.c create mode 100644 dpdk/drivers/net/liquidio/base/lio_23xx_vf.h create mode 100644 dpdk/drivers/net/liquidio/base/lio_hw_defs.h create mode 100644 dpdk/drivers/net/liquidio/base/lio_mbox.c create mode 100644 dpdk/drivers/net/liquidio/base/lio_mbox.h create mode 100644 dpdk/drivers/net/liquidio/lio_ethdev.c create mode 100644 dpdk/drivers/net/liquidio/lio_ethdev.h create mode 100644 dpdk/drivers/net/liquidio/lio_logs.h create mode 100644 dpdk/drivers/net/liquidio/lio_rxtx.c create mode 100644 dpdk/drivers/net/liquidio/lio_rxtx.h create mode 100644 dpdk/drivers/net/liquidio/lio_struct.h create mode 100644 dpdk/drivers/net/liquidio/rte_pmd_lio_version.map create mode 100644 dpdk/drivers/net/mlx4/mlx4_ethdev.c create mode 100644 dpdk/drivers/net/mlx4/mlx4_flow.c create mode 100644 dpdk/drivers/net/mlx4/mlx4_flow.h create mode 100644 dpdk/drivers/net/mlx4/mlx4_intr.c create mode 100644 dpdk/drivers/net/mlx4/mlx4_mr.c create mode 100644 dpdk/drivers/net/mlx4/mlx4_prm.h create mode 100644 dpdk/drivers/net/mlx4/mlx4_rxq.c create mode 100644 dpdk/drivers/net/mlx4/mlx4_rxtx.c create mode 100644 dpdk/drivers/net/mlx4/mlx4_rxtx.h create mode 100644 dpdk/drivers/net/mlx4/mlx4_txq.c create mode 100644 dpdk/drivers/net/mlx4/mlx4_utils.c create mode 100644 dpdk/drivers/net/mlx4/mlx4_utils.h delete mode 100644 dpdk/drivers/net/mlx5/mlx5_fdir.c create mode 100644 dpdk/drivers/net/mlx5/mlx5_flow.c create mode 100644 dpdk/drivers/net/mlx5/mlx5_rxtx_vec.c create mode 100644 dpdk/drivers/net/mlx5/mlx5_rxtx_vec.h create mode 100644 dpdk/drivers/net/mlx5/mlx5_rxtx_vec_neon.h create mode 100644 dpdk/drivers/net/mlx5/mlx5_rxtx_vec_sse.h create mode 100644 dpdk/drivers/net/mlx5/mlx5_socket.c delete mode 100644 dpdk/drivers/net/mpipe/Makefile delete mode 100644 dpdk/drivers/net/mpipe/mpipe_tilegx.c delete mode 100644 dpdk/drivers/net/mpipe/rte_pmd_mpipe_version.map create mode 100644 dpdk/drivers/net/mrvl/Makefile create mode 100644 dpdk/drivers/net/mrvl/mrvl_ethdev.c create mode 100644 dpdk/drivers/net/mrvl/mrvl_ethdev.h create mode 100644 dpdk/drivers/net/mrvl/mrvl_qos.c create mode 100644 dpdk/drivers/net/mrvl/mrvl_qos.h create mode 100644 dpdk/drivers/net/mrvl/rte_pmd_mrvl_version.map create mode 100644 dpdk/drivers/net/nfp/nfp_net_eth.h create mode 100644 dpdk/drivers/net/nfp/nfp_nfpu.c create mode 100644 dpdk/drivers/net/nfp/nfp_nfpu.h create mode 100644 dpdk/drivers/net/nfp/nfp_nspu.c create mode 100644 dpdk/drivers/net/nfp/nfp_nspu.h delete mode 100644 dpdk/drivers/net/null/rte_eth_null.h create mode 100644 dpdk/drivers/net/octeontx/Makefile create mode 100644 dpdk/drivers/net/octeontx/base/octeontx_bgx.c create mode 100644 dpdk/drivers/net/octeontx/base/octeontx_bgx.h create mode 100644 dpdk/drivers/net/octeontx/base/octeontx_io.h create mode 100644 dpdk/drivers/net/octeontx/base/octeontx_pki_var.h create mode 100644 dpdk/drivers/net/octeontx/base/octeontx_pkivf.c create mode 100644 dpdk/drivers/net/octeontx/base/octeontx_pkivf.h create mode 100644 dpdk/drivers/net/octeontx/base/octeontx_pkovf.c create mode 100644 dpdk/drivers/net/octeontx/base/octeontx_pkovf.h create mode 100644 dpdk/drivers/net/octeontx/octeontx_ethdev.c create mode 100644 dpdk/drivers/net/octeontx/octeontx_ethdev.h create mode 100644 dpdk/drivers/net/octeontx/octeontx_logs.h create mode 100644 dpdk/drivers/net/octeontx/octeontx_rxtx.c create mode 100644 dpdk/drivers/net/octeontx/octeontx_rxtx.h create mode 100644 dpdk/drivers/net/octeontx/rte_pmd_octeontx_version.map create mode 100644 dpdk/drivers/net/qede/base/ecore_hsi_debug_tools.h create mode 100644 dpdk/drivers/net/qede/base/ecore_hsi_init_func.h create mode 100644 dpdk/drivers/net/qede/base/ecore_hsi_init_tool.h delete mode 100644 dpdk/drivers/net/qede/base/ecore_hsi_tools.h create mode 100644 dpdk/drivers/net/qede/base/ecore_mng_tlv.c delete mode 100644 dpdk/drivers/net/qede/qede_eth_if.c delete mode 100644 dpdk/drivers/net/qede/qede_eth_if.h create mode 100644 dpdk/drivers/net/qede/qede_fdir.c rename dpdk/drivers/net/ring/{rte_eth_ring_version.map => rte_pmd_ring_version.map} (100%) create mode 100644 dpdk/drivers/net/sfc/Makefile create mode 100644 dpdk/drivers/net/sfc/base/README create mode 100644 dpdk/drivers/net/sfc/base/ef10_ev.c create mode 100644 dpdk/drivers/net/sfc/base/ef10_filter.c create mode 100644 dpdk/drivers/net/sfc/base/ef10_impl.h create mode 100644 dpdk/drivers/net/sfc/base/ef10_intr.c create mode 100644 dpdk/drivers/net/sfc/base/ef10_mac.c create mode 100644 dpdk/drivers/net/sfc/base/ef10_mcdi.c create mode 100644 dpdk/drivers/net/sfc/base/ef10_nic.c create mode 100644 dpdk/drivers/net/sfc/base/ef10_nvram.c create mode 100644 dpdk/drivers/net/sfc/base/ef10_phy.c create mode 100644 dpdk/drivers/net/sfc/base/ef10_rx.c create mode 100644 dpdk/drivers/net/sfc/base/ef10_tlv_layout.h create mode 100644 dpdk/drivers/net/sfc/base/ef10_tx.c create mode 100644 dpdk/drivers/net/sfc/base/ef10_vpd.c create mode 100644 dpdk/drivers/net/sfc/base/efx.h create mode 100644 dpdk/drivers/net/sfc/base/efx_bootcfg.c create mode 100644 dpdk/drivers/net/sfc/base/efx_check.h create mode 100644 dpdk/drivers/net/sfc/base/efx_crc32.c create mode 100644 dpdk/drivers/net/sfc/base/efx_ev.c create mode 100644 dpdk/drivers/net/sfc/base/efx_filter.c create mode 100644 dpdk/drivers/net/sfc/base/efx_hash.c create mode 100644 dpdk/drivers/net/sfc/base/efx_impl.h create mode 100644 dpdk/drivers/net/sfc/base/efx_intr.c create mode 100644 dpdk/drivers/net/sfc/base/efx_lic.c create mode 100644 dpdk/drivers/net/sfc/base/efx_mac.c create mode 100644 dpdk/drivers/net/sfc/base/efx_mcdi.c create mode 100644 dpdk/drivers/net/sfc/base/efx_mcdi.h create mode 100644 dpdk/drivers/net/sfc/base/efx_mon.c create mode 100644 dpdk/drivers/net/sfc/base/efx_nic.c create mode 100644 dpdk/drivers/net/sfc/base/efx_nvram.c create mode 100644 dpdk/drivers/net/sfc/base/efx_phy.c create mode 100644 dpdk/drivers/net/sfc/base/efx_phy_ids.h create mode 100644 dpdk/drivers/net/sfc/base/efx_port.c create mode 100644 dpdk/drivers/net/sfc/base/efx_regs.h create mode 100644 dpdk/drivers/net/sfc/base/efx_regs_ef10.h create mode 100644 dpdk/drivers/net/sfc/base/efx_regs_mcdi.h create mode 100644 dpdk/drivers/net/sfc/base/efx_regs_pci.h create mode 100644 dpdk/drivers/net/sfc/base/efx_rx.c create mode 100644 dpdk/drivers/net/sfc/base/efx_sram.c create mode 100644 dpdk/drivers/net/sfc/base/efx_tx.c create mode 100644 dpdk/drivers/net/sfc/base/efx_types.h create mode 100644 dpdk/drivers/net/sfc/base/efx_vpd.c create mode 100644 dpdk/drivers/net/sfc/base/hunt_impl.h create mode 100644 dpdk/drivers/net/sfc/base/hunt_nic.c create mode 100644 dpdk/drivers/net/sfc/base/mcdi_mon.c create mode 100644 dpdk/drivers/net/sfc/base/mcdi_mon.h create mode 100644 dpdk/drivers/net/sfc/base/medford_impl.h create mode 100644 dpdk/drivers/net/sfc/base/medford_nic.c create mode 100644 dpdk/drivers/net/sfc/base/siena_flash.h create mode 100644 dpdk/drivers/net/sfc/base/siena_impl.h create mode 100644 dpdk/drivers/net/sfc/base/siena_mac.c create mode 100644 dpdk/drivers/net/sfc/base/siena_mcdi.c create mode 100644 dpdk/drivers/net/sfc/base/siena_nic.c create mode 100644 dpdk/drivers/net/sfc/base/siena_nvram.c create mode 100644 dpdk/drivers/net/sfc/base/siena_phy.c create mode 100644 dpdk/drivers/net/sfc/base/siena_sram.c create mode 100644 dpdk/drivers/net/sfc/base/siena_vpd.c create mode 100644 dpdk/drivers/net/sfc/efsys.h create mode 100644 dpdk/drivers/net/sfc/rte_pmd_sfc_efx_version.map create mode 100644 dpdk/drivers/net/sfc/sfc.c create mode 100644 dpdk/drivers/net/sfc/sfc.h create mode 100644 dpdk/drivers/net/sfc/sfc_debug.h create mode 100644 dpdk/drivers/net/sfc/sfc_dp.c create mode 100644 dpdk/drivers/net/sfc/sfc_dp.h create mode 100644 dpdk/drivers/net/sfc/sfc_dp_rx.h create mode 100644 dpdk/drivers/net/sfc/sfc_dp_tx.h create mode 100644 dpdk/drivers/net/sfc/sfc_ef10.h create mode 100644 dpdk/drivers/net/sfc/sfc_ef10_rx.c create mode 100644 dpdk/drivers/net/sfc/sfc_ef10_tx.c create mode 100644 dpdk/drivers/net/sfc/sfc_ethdev.c create mode 100644 dpdk/drivers/net/sfc/sfc_ev.c create mode 100644 dpdk/drivers/net/sfc/sfc_ev.h create mode 100644 dpdk/drivers/net/sfc/sfc_filter.c create mode 100644 dpdk/drivers/net/sfc/sfc_filter.h create mode 100644 dpdk/drivers/net/sfc/sfc_flow.c create mode 100644 dpdk/drivers/net/sfc/sfc_flow.h create mode 100644 dpdk/drivers/net/sfc/sfc_intr.c create mode 100644 dpdk/drivers/net/sfc/sfc_kvargs.c create mode 100644 dpdk/drivers/net/sfc/sfc_kvargs.h create mode 100644 dpdk/drivers/net/sfc/sfc_log.h create mode 100644 dpdk/drivers/net/sfc/sfc_mcdi.c create mode 100644 dpdk/drivers/net/sfc/sfc_port.c create mode 100644 dpdk/drivers/net/sfc/sfc_rx.c create mode 100644 dpdk/drivers/net/sfc/sfc_rx.h create mode 100644 dpdk/drivers/net/sfc/sfc_tso.c create mode 100644 dpdk/drivers/net/sfc/sfc_tweak.h create mode 100644 dpdk/drivers/net/sfc/sfc_tx.c create mode 100644 dpdk/drivers/net/sfc/sfc_tx.h create mode 100644 dpdk/drivers/net/softnic/Makefile create mode 100644 dpdk/drivers/net/softnic/rte_eth_softnic.c create mode 100644 dpdk/drivers/net/softnic/rte_eth_softnic.h create mode 100644 dpdk/drivers/net/softnic/rte_eth_softnic_internals.h create mode 100644 dpdk/drivers/net/softnic/rte_eth_softnic_tm.c create mode 100644 dpdk/drivers/net/softnic/rte_pmd_eth_softnic_version.map create mode 100644 dpdk/drivers/net/szedata2/szedata2_iobuf.c create mode 100644 dpdk/drivers/net/szedata2/szedata2_iobuf.h create mode 100644 dpdk/drivers/net/tap/Makefile create mode 100644 dpdk/drivers/net/tap/rte_eth_tap.c create mode 100644 dpdk/drivers/net/tap/rte_eth_tap.h create mode 100644 dpdk/drivers/net/tap/rte_pmd_tap_version.map create mode 100644 dpdk/drivers/net/tap/tap_flow.c create mode 100644 dpdk/drivers/net/tap/tap_flow.h create mode 100644 dpdk/drivers/net/tap/tap_netlink.c create mode 100644 dpdk/drivers/net/tap/tap_netlink.h create mode 100644 dpdk/drivers/net/tap/tap_tcmsgs.c create mode 100644 dpdk/drivers/net/tap/tap_tcmsgs.h create mode 100644 dpdk/drivers/net/thunderx/base/nicvf_bsvf.c create mode 100644 dpdk/drivers/net/thunderx/base/nicvf_bsvf.h create mode 100644 dpdk/drivers/net/thunderx/nicvf_svf.c create mode 100644 dpdk/drivers/net/thunderx/nicvf_svf.h create mode 100644 dpdk/drivers/net/virtio/virtio_rxtx_simple.h create mode 100644 dpdk/drivers/net/virtio/virtio_rxtx_simple_neon.c create mode 100644 dpdk/drivers/net/virtio/virtio_rxtx_simple_sse.c create mode 100644 dpdk/drivers/net/virtio/virtio_user/vhost_kernel.c create mode 100644 dpdk/drivers/net/virtio/virtio_user/vhost_kernel_tap.c create mode 100644 dpdk/drivers/net/virtio/virtio_user/vhost_kernel_tap.h delete mode 100644 dpdk/drivers/net/xenvirt/Makefile delete mode 100644 dpdk/drivers/net/xenvirt/rte_eth_xenvirt.c delete mode 100644 dpdk/drivers/net/xenvirt/rte_eth_xenvirt.h delete mode 100644 dpdk/drivers/net/xenvirt/rte_eth_xenvirt_version.map delete mode 100644 dpdk/drivers/net/xenvirt/rte_mempool_gntalloc.c delete mode 100644 dpdk/drivers/net/xenvirt/rte_xen_lib.c delete mode 100644 dpdk/drivers/net/xenvirt/rte_xen_lib.h delete mode 100644 dpdk/drivers/net/xenvirt/virtio_logs.h delete mode 100644 dpdk/drivers/net/xenvirt/virtqueue.h delete mode 100644 dpdk/examples/dpdk_qat/Makefile delete mode 100644 dpdk/examples/dpdk_qat/config_files/coleto/dh895xcc_qa_dev0.conf delete mode 100644 dpdk/examples/dpdk_qat/config_files/shumway/dh89xxcc_qa_dev0.conf delete mode 100644 dpdk/examples/dpdk_qat/config_files/shumway/dh89xxcc_qa_dev1.conf delete mode 100644 dpdk/examples/dpdk_qat/config_files/stargo/dh89xxcc_qa_dev0.conf delete mode 100644 dpdk/examples/dpdk_qat/crypto.c delete mode 100644 dpdk/examples/dpdk_qat/crypto.h delete mode 100644 dpdk/examples/dpdk_qat/main.c create mode 100644 dpdk/examples/eventdev_pipeline_sw_pmd/Makefile create mode 100644 dpdk/examples/eventdev_pipeline_sw_pmd/main.c create mode 100644 dpdk/examples/flow_classify/Makefile create mode 100644 dpdk/examples/flow_classify/flow_classify.c create mode 100644 dpdk/examples/flow_classify/ipv4_rules_file.txt create mode 100644 dpdk/examples/flow_filtering/Makefile create mode 100644 dpdk/examples/flow_filtering/flow_blocks.c create mode 100644 dpdk/examples/flow_filtering/main.c create mode 100644 dpdk/examples/ip_pipeline/config/tap.cfg create mode 100644 dpdk/examples/ip_pipeline/pipeline/hash_func_arm64.h create mode 100644 dpdk/examples/ipsec-secgw/ep0.cfg create mode 100644 dpdk/examples/ipsec-secgw/ep1.cfg create mode 100644 dpdk/examples/ipsec-secgw/parser.c create mode 100644 dpdk/examples/ipsec-secgw/parser.h delete mode 100644 dpdk/examples/l2fwd-ivshmem/Makefile delete mode 100644 dpdk/examples/l2fwd-ivshmem/guest/Makefile delete mode 100644 dpdk/examples/l2fwd-ivshmem/guest/guest.c delete mode 100644 dpdk/examples/l2fwd-ivshmem/host/Makefile delete mode 100644 dpdk/examples/l2fwd-ivshmem/host/host.c delete mode 100644 dpdk/examples/l2fwd-ivshmem/include/common.h create mode 100644 dpdk/examples/l3fwd/l3fwd_altivec.h create mode 100644 dpdk/examples/l3fwd/l3fwd_common.h create mode 100644 dpdk/examples/l3fwd/l3fwd_em_hlm.h create mode 100644 dpdk/examples/l3fwd/l3fwd_em_hlm_neon.h create mode 100644 dpdk/examples/l3fwd/l3fwd_em_sequential.h delete mode 100644 dpdk/examples/l3fwd/l3fwd_em_sse.h create mode 100644 dpdk/examples/l3fwd/l3fwd_lpm_altivec.h create mode 100644 dpdk/examples/l3fwd/l3fwd_lpm_neon.h create mode 100644 dpdk/examples/l3fwd/l3fwd_neon.h create mode 100644 dpdk/examples/performance-thread/common/arch/arm64/ctx.c create mode 100644 dpdk/examples/performance-thread/common/arch/arm64/ctx.h create mode 100644 dpdk/examples/performance-thread/common/arch/arm64/stack.h create mode 100644 dpdk/examples/performance-thread/common/arch/x86/stack.h create mode 100644 dpdk/examples/server_node_efd/Makefile create mode 100644 dpdk/examples/server_node_efd/node/Makefile create mode 100644 dpdk/examples/server_node_efd/node/node.c create mode 100644 dpdk/examples/server_node_efd/server/Makefile create mode 100644 dpdk/examples/server_node_efd/server/args.c create mode 100644 dpdk/examples/server_node_efd/server/args.h create mode 100644 dpdk/examples/server_node_efd/server/init.c create mode 100644 dpdk/examples/server_node_efd/server/init.h create mode 100644 dpdk/examples/server_node_efd/server/main.c create mode 100644 dpdk/examples/server_node_efd/shared/common.h create mode 100644 dpdk/examples/service_cores/Makefile create mode 100644 dpdk/examples/service_cores/main.c create mode 100644 dpdk/examples/vhost/virtio_net.c create mode 100644 dpdk/examples/vhost_scsi/Makefile create mode 100644 dpdk/examples/vhost_scsi/scsi.c create mode 100644 dpdk/examples/vhost_scsi/scsi_spec.h create mode 100644 dpdk/examples/vhost_scsi/vhost_scsi.c create mode 100644 dpdk/examples/vhost_scsi/vhost_scsi.h delete mode 100644 dpdk/examples/vhost_xen/Makefile delete mode 100644 dpdk/examples/vhost_xen/main.c delete mode 100644 dpdk/examples/vhost_xen/main.h delete mode 100644 dpdk/examples/vhost_xen/vhost_monitor.c delete mode 100644 dpdk/examples/vhost_xen/virtio-net.h delete mode 100644 dpdk/examples/vhost_xen/xen_vhost.h delete mode 100644 dpdk/examples/vhost_xen/xenstore_parse.c create mode 100644 dpdk/lib/librte_acl/acl_run_altivec.c create mode 100644 dpdk/lib/librte_acl/acl_run_altivec.h create mode 100644 dpdk/lib/librte_bitratestats/Makefile create mode 100644 dpdk/lib/librte_bitratestats/rte_bitrate.c create mode 100644 dpdk/lib/librte_bitratestats/rte_bitrate.h create mode 100644 dpdk/lib/librte_bitratestats/rte_bitratestats_version.map create mode 100644 dpdk/lib/librte_cryptodev/rte_cryptodev_pmd.c create mode 100644 dpdk/lib/librte_distributor/rte_distributor_match_generic.c create mode 100644 dpdk/lib/librte_distributor/rte_distributor_match_sse.c create mode 100644 dpdk/lib/librte_distributor/rte_distributor_private.h create mode 100644 dpdk/lib/librte_distributor/rte_distributor_v1705.h create mode 100644 dpdk/lib/librte_distributor/rte_distributor_v20.c create mode 100644 dpdk/lib/librte_distributor/rte_distributor_v20.h delete mode 100644 dpdk/lib/librte_eal/bsdapp/eal/eal_log.c delete mode 100644 dpdk/lib/librte_eal/bsdapp/eal/eal_pci.c delete mode 100644 dpdk/lib/librte_eal/bsdapp/eal/include/exec-env/rte_dom0_common.h delete mode 100644 dpdk/lib/librte_eal/bsdapp/eal/include/exec-env/rte_interrupts.h delete mode 100644 dpdk/lib/librte_eal/bsdapp/eal/rte_eal_version.map create mode 100644 dpdk/lib/librte_eal/common/arch/arm/rte_cycles.c create mode 100644 dpdk/lib/librte_eal/common/arch/ppc_64/rte_cycles.c delete mode 100644 dpdk/lib/librte_eal/common/arch/tile/rte_cpuflags.c create mode 100644 dpdk/lib/librte_eal/common/arch/x86/rte_cycles.c create mode 100644 dpdk/lib/librte_eal/common/arch/x86/rte_memcpy.c create mode 100644 dpdk/lib/librte_eal/common/eal_common_bus.c delete mode 100644 dpdk/lib/librte_eal/common/eal_common_pci.c delete mode 100644 dpdk/lib/librte_eal/common/eal_common_pci_uio.c create mode 100644 dpdk/lib/librte_eal/common/include/arch/arm/rte_io.h create mode 100644 dpdk/lib/librte_eal/common/include/arch/arm/rte_io_64.h create mode 100644 dpdk/lib/librte_eal/common/include/arch/arm/rte_pause.h create mode 100644 dpdk/lib/librte_eal/common/include/arch/arm/rte_pause_32.h create mode 100644 dpdk/lib/librte_eal/common/include/arch/arm/rte_pause_64.h create mode 100644 dpdk/lib/librte_eal/common/include/arch/ppc_64/rte_io.h create mode 100644 dpdk/lib/librte_eal/common/include/arch/ppc_64/rte_pause.h create mode 100644 dpdk/lib/librte_eal/common/include/arch/ppc_64/rte_vect.h delete mode 100644 dpdk/lib/librte_eal/common/include/arch/tile/rte_atomic.h delete mode 100644 dpdk/lib/librte_eal/common/include/arch/tile/rte_byteorder.h delete mode 100644 dpdk/lib/librte_eal/common/include/arch/tile/rte_cpuflags.h delete mode 100644 dpdk/lib/librte_eal/common/include/arch/tile/rte_cycles.h delete mode 100644 dpdk/lib/librte_eal/common/include/arch/tile/rte_memcpy.h delete mode 100644 dpdk/lib/librte_eal/common/include/arch/tile/rte_prefetch.h delete mode 100644 dpdk/lib/librte_eal/common/include/arch/tile/rte_rwlock.h delete mode 100644 dpdk/lib/librte_eal/common/include/arch/tile/rte_spinlock.h create mode 100644 dpdk/lib/librte_eal/common/include/arch/x86/rte_io.h create mode 100644 dpdk/lib/librte_eal/common/include/arch/x86/rte_pause.h create mode 100644 dpdk/lib/librte_eal/common/include/generic/rte_io.h create mode 100644 dpdk/lib/librte_eal/common/include/generic/rte_pause.h create mode 100644 dpdk/lib/librte_eal/common/include/generic/rte_vect.h create mode 100644 dpdk/lib/librte_eal/common/include/rte_bitmap.h create mode 100644 dpdk/lib/librte_eal/common/include/rte_bus.h create mode 100644 dpdk/lib/librte_eal/common/include/rte_eal_interrupts.h delete mode 100644 dpdk/lib/librte_eal/common/include/rte_pci.h delete mode 100644 dpdk/lib/librte_eal/common/include/rte_pci_dev_ids.h create mode 100644 dpdk/lib/librte_eal/common/include/rte_service.h create mode 100644 dpdk/lib/librte_eal/common/include/rte_service_component.h create mode 100644 dpdk/lib/librte_eal/common/include/rte_vfio.h delete mode 100644 dpdk/lib/librte_eal/common/include/rte_warnings.h create mode 100644 dpdk/lib/librte_eal/common/rte_service.c delete mode 100644 dpdk/lib/librte_eal/linuxapp/eal/eal_ivshmem.c delete mode 100644 dpdk/lib/librte_eal/linuxapp/eal/eal_pci.c delete mode 100644 dpdk/lib/librte_eal/linuxapp/eal/eal_pci_init.h delete mode 100644 dpdk/lib/librte_eal/linuxapp/eal/eal_pci_uio.c delete mode 100644 dpdk/lib/librte_eal/linuxapp/eal/eal_pci_vfio.c delete mode 100644 dpdk/lib/librte_eal/linuxapp/eal/eal_xen_memory.c delete mode 100644 dpdk/lib/librte_eal/linuxapp/eal/include/exec-env/rte_dom0_common.h delete mode 100644 dpdk/lib/librte_eal/linuxapp/eal/include/exec-env/rte_interrupts.h delete mode 100644 dpdk/lib/librte_eal/linuxapp/eal/rte_eal_version.map delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/COPYING delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/igb_debugfs.c delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/igb_hwmon.c delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/igb_procfs.c delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/igb_ptp.c delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/kcompat.c delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/kcompat_ethtool.c delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/COPYING delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_sriov.h delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/kni_vhost.c delete mode 100644 dpdk/lib/librte_eal/linuxapp/xen_dom0/Makefile delete mode 100644 dpdk/lib/librte_eal/linuxapp/xen_dom0/compat.h delete mode 100644 dpdk/lib/librte_eal/linuxapp/xen_dom0/dom0_mm_dev.h delete mode 100644 dpdk/lib/librte_eal/linuxapp/xen_dom0/dom0_mm_misc.c create mode 100644 dpdk/lib/librte_eal/rte_eal_version.map create mode 100644 dpdk/lib/librte_efd/Makefile create mode 100644 dpdk/lib/librte_efd/rte_efd.c create mode 100644 dpdk/lib/librte_efd/rte_efd.h create mode 100644 dpdk/lib/librte_efd/rte_efd_arm64.h create mode 100644 dpdk/lib/librte_efd/rte_efd_version.map create mode 100644 dpdk/lib/librte_efd/rte_efd_x86.h create mode 100644 dpdk/lib/librte_ether/ethdev_profile.c create mode 100644 dpdk/lib/librte_ether/ethdev_profile.h create mode 100644 dpdk/lib/librte_ether/rte_ethdev_pci.h create mode 100644 dpdk/lib/librte_ether/rte_ethdev_vdev.h create mode 100644 dpdk/lib/librte_ether/rte_ethdev_version.map delete mode 100644 dpdk/lib/librte_ether/rte_ether.h delete mode 100644 dpdk/lib/librte_ether/rte_ether_version.map create mode 100644 dpdk/lib/librte_ether/rte_flow.c create mode 100644 dpdk/lib/librte_ether/rte_flow.h create mode 100644 dpdk/lib/librte_ether/rte_flow_driver.h create mode 100644 dpdk/lib/librte_ether/rte_mtr.c create mode 100644 dpdk/lib/librte_ether/rte_mtr.h create mode 100644 dpdk/lib/librte_ether/rte_mtr_driver.h create mode 100644 dpdk/lib/librte_ether/rte_tm.c create mode 100644 dpdk/lib/librte_ether/rte_tm.h create mode 100644 dpdk/lib/librte_ether/rte_tm_driver.h create mode 100644 dpdk/lib/librte_eventdev/Makefile create mode 100644 dpdk/lib/librte_eventdev/rte_event_eth_rx_adapter.c create mode 100644 dpdk/lib/librte_eventdev/rte_event_eth_rx_adapter.h create mode 100644 dpdk/lib/librte_eventdev/rte_event_ring.c create mode 100644 dpdk/lib/librte_eventdev/rte_event_ring.h create mode 100644 dpdk/lib/librte_eventdev/rte_eventdev.c create mode 100644 dpdk/lib/librte_eventdev/rte_eventdev.h create mode 100644 dpdk/lib/librte_eventdev/rte_eventdev_pmd.h create mode 100644 dpdk/lib/librte_eventdev/rte_eventdev_pmd_pci.h create mode 100644 dpdk/lib/librte_eventdev/rte_eventdev_pmd_vdev.h create mode 100644 dpdk/lib/librte_eventdev/rte_eventdev_version.map create mode 100644 dpdk/lib/librte_flow_classify/Makefile create mode 100644 dpdk/lib/librte_flow_classify/rte_flow_classify.c create mode 100644 dpdk/lib/librte_flow_classify/rte_flow_classify.h create mode 100644 dpdk/lib/librte_flow_classify/rte_flow_classify_parse.c create mode 100644 dpdk/lib/librte_flow_classify/rte_flow_classify_parse.h create mode 100644 dpdk/lib/librte_flow_classify/rte_flow_classify_version.map create mode 100644 dpdk/lib/librte_gro/Makefile create mode 100644 dpdk/lib/librte_gro/gro_tcp4.c create mode 100644 dpdk/lib/librte_gro/gro_tcp4.h create mode 100644 dpdk/lib/librte_gro/rte_gro.c create mode 100644 dpdk/lib/librte_gro/rte_gro.h create mode 100644 dpdk/lib/librte_gro/rte_gro_version.map create mode 100644 dpdk/lib/librte_gso/Makefile create mode 100644 dpdk/lib/librte_gso/gso_common.c create mode 100644 dpdk/lib/librte_gso/gso_common.h create mode 100644 dpdk/lib/librte_gso/gso_tcp4.c create mode 100644 dpdk/lib/librte_gso/gso_tcp4.h create mode 100644 dpdk/lib/librte_gso/gso_tunnel_tcp4.c create mode 100644 dpdk/lib/librte_gso/gso_tunnel_tcp4.h create mode 100644 dpdk/lib/librte_gso/rte_gso.c create mode 100644 dpdk/lib/librte_gso/rte_gso.h create mode 100644 dpdk/lib/librte_gso/rte_gso_version.map create mode 100644 dpdk/lib/librte_ip_frag/rte_ip_frag_version.map delete mode 100644 dpdk/lib/librte_ip_frag/rte_ipfrag_version.map delete mode 100644 dpdk/lib/librte_ivshmem/Makefile delete mode 100644 dpdk/lib/librte_ivshmem/rte_ivshmem.c delete mode 100644 dpdk/lib/librte_ivshmem/rte_ivshmem.h delete mode 100644 dpdk/lib/librte_ivshmem/rte_ivshmem_version.map create mode 100644 dpdk/lib/librte_latencystats/Makefile create mode 100644 dpdk/lib/librte_latencystats/rte_latencystats.c create mode 100644 dpdk/lib/librte_latencystats/rte_latencystats.h create mode 100644 dpdk/lib/librte_latencystats/rte_latencystats_version.map create mode 100644 dpdk/lib/librte_lpm/rte_lpm_altivec.h create mode 100644 dpdk/lib/librte_mbuf/rte_mbuf_ptype.c create mode 100644 dpdk/lib/librte_mbuf/rte_mbuf_ptype.h create mode 100644 dpdk/lib/librte_member/Makefile create mode 100644 dpdk/lib/librte_member/rte_member.c create mode 100644 dpdk/lib/librte_member/rte_member.h create mode 100644 dpdk/lib/librte_member/rte_member_ht.c create mode 100644 dpdk/lib/librte_member/rte_member_ht.h create mode 100644 dpdk/lib/librte_member/rte_member_vbf.c create mode 100644 dpdk/lib/librte_member/rte_member_vbf.h create mode 100644 dpdk/lib/librte_member/rte_member_version.map create mode 100644 dpdk/lib/librte_member/rte_member_x86.h delete mode 100644 dpdk/lib/librte_mempool/rte_mempool_ring.c delete mode 100644 dpdk/lib/librte_mempool/rte_mempool_stack.c create mode 100644 dpdk/lib/librte_metrics/Makefile create mode 100644 dpdk/lib/librte_metrics/rte_metrics.c create mode 100644 dpdk/lib/librte_metrics/rte_metrics.h create mode 100644 dpdk/lib/librte_metrics/rte_metrics_version.map create mode 100644 dpdk/lib/librte_net/net_crc_neon.h create mode 100644 dpdk/lib/librte_net/net_crc_sse.h create mode 100644 dpdk/lib/librte_net/rte_esp.h create mode 100644 dpdk/lib/librte_net/rte_ether.h create mode 100644 dpdk/lib/librte_net/rte_gre.h create mode 100644 dpdk/lib/librte_net/rte_net.c create mode 100644 dpdk/lib/librte_net/rte_net.h create mode 100644 dpdk/lib/librte_net/rte_net_crc.c create mode 100644 dpdk/lib/librte_net/rte_net_crc.h create mode 100644 dpdk/lib/librte_net/rte_net_version.map create mode 100644 dpdk/lib/librte_pci/Makefile create mode 100644 dpdk/lib/librte_pci/rte_pci.c create mode 100644 dpdk/lib/librte_pci/rte_pci.h create mode 100644 dpdk/lib/librte_pci/rte_pci_version.map create mode 100644 dpdk/lib/librte_port/rte_port_fd.c create mode 100644 dpdk/lib/librte_port/rte_port_fd.h delete mode 100644 dpdk/lib/librte_sched/rte_bitmap.h create mode 100644 dpdk/lib/librte_security/Makefile create mode 100644 dpdk/lib/librte_security/rte_security.c create mode 100644 dpdk/lib/librte_security/rte_security.h create mode 100644 dpdk/lib/librte_security/rte_security_driver.h create mode 100644 dpdk/lib/librte_security/rte_security_version.map create mode 100644 dpdk/lib/librte_table/rte_lru_arm64.h create mode 100644 dpdk/lib/librte_table/rte_lru_x86.h create mode 100644 dpdk/lib/librte_table/rte_table_hash_cuckoo.c delete mode 100644 dpdk/lib/librte_vhost/eventfd_link/Makefile delete mode 100644 dpdk/lib/librte_vhost/eventfd_link/eventfd_link.c delete mode 100644 dpdk/lib/librte_vhost/eventfd_link/eventfd_link.h create mode 100644 dpdk/lib/librte_vhost/fd_man.c create mode 100644 dpdk/lib/librte_vhost/fd_man.h create mode 100644 dpdk/lib/librte_vhost/iotlb.c create mode 100644 dpdk/lib/librte_vhost/iotlb.h delete mode 100644 dpdk/lib/librte_vhost/libvirt/qemu-wrap.py create mode 100644 dpdk/lib/librte_vhost/rte_vhost.h delete mode 100644 dpdk/lib/librte_vhost/rte_virtio_net.h create mode 100644 dpdk/lib/librte_vhost/socket.c delete mode 100644 dpdk/lib/librte_vhost/vhost-net.h create mode 100644 dpdk/lib/librte_vhost/vhost.c create mode 100644 dpdk/lib/librte_vhost/vhost.h delete mode 100644 dpdk/lib/librte_vhost/vhost_cuse/eventfd_copy.c delete mode 100644 dpdk/lib/librte_vhost/vhost_cuse/eventfd_copy.h delete mode 100644 dpdk/lib/librte_vhost/vhost_cuse/vhost-net-cdev.c delete mode 100644 dpdk/lib/librte_vhost/vhost_cuse/virtio-net-cdev.c delete mode 100644 dpdk/lib/librte_vhost/vhost_cuse/virtio-net-cdev.h delete mode 100644 dpdk/lib/librte_vhost/vhost_rxtx.c create mode 100644 dpdk/lib/librte_vhost/vhost_user.c create mode 100644 dpdk/lib/librte_vhost/vhost_user.h delete mode 100644 dpdk/lib/librte_vhost/vhost_user/fd_man.c delete mode 100644 dpdk/lib/librte_vhost/vhost_user/fd_man.h delete mode 100644 dpdk/lib/librte_vhost/vhost_user/vhost-net-user.c delete mode 100644 dpdk/lib/librte_vhost/vhost_user/vhost-net-user.h delete mode 100644 dpdk/lib/librte_vhost/vhost_user/virtio-net-user.c delete mode 100644 dpdk/lib/librte_vhost/vhost_user/virtio-net-user.h delete mode 100644 dpdk/lib/librte_vhost/virtio-net.c create mode 100644 dpdk/lib/librte_vhost/virtio_net.c delete mode 100644 dpdk/mk/arch/tile/rte.vars.mk delete mode 100644 dpdk/mk/internal/rte.depdirs-post.mk delete mode 100644 dpdk/mk/internal/rte.depdirs-pre.mk delete mode 100644 dpdk/mk/machine/atm/rte.vars.mk create mode 100644 dpdk/mk/machine/dpaa/rte.vars.mk create mode 100644 dpdk/mk/machine/silvermont/rte.vars.mk delete mode 100644 dpdk/mk/machine/tilegx/rte.vars.mk delete mode 100755 dpdk/scripts/auto-config-h.sh delete mode 100755 dpdk/scripts/check-git-log.sh delete mode 100755 dpdk/scripts/check-maintainers.sh delete mode 100755 dpdk/scripts/checkpatches.sh delete mode 100755 dpdk/scripts/cocci.sh delete mode 100644 dpdk/scripts/cocci/mtod-offset.cocci delete mode 100755 dpdk/scripts/depdirs-rule.sh delete mode 100755 dpdk/scripts/load-devel-config.sh delete mode 100755 dpdk/scripts/relpath.sh delete mode 100755 dpdk/scripts/test-build.sh delete mode 100755 dpdk/scripts/test-null.sh delete mode 100755 dpdk/scripts/validate-abi.sh create mode 100644 dpdk/test/Makefile create mode 100644 dpdk/test/cmdline_test/Makefile rename dpdk/{app => test}/cmdline_test/cmdline_test.c (100%) rename dpdk/{app => test}/cmdline_test/cmdline_test.h (100%) create mode 100644 dpdk/test/cmdline_test/cmdline_test.py create mode 100644 dpdk/test/cmdline_test/cmdline_test_data.py rename dpdk/{app => test}/cmdline_test/commands.c (100%) create mode 100644 dpdk/test/test-acl/Makefile create mode 100644 dpdk/test/test-acl/main.c create mode 100644 dpdk/test/test-pipeline/Makefile create mode 100644 dpdk/test/test-pipeline/config.c create mode 100644 dpdk/test/test-pipeline/init.c create mode 100644 dpdk/test/test-pipeline/main.c create mode 100644 dpdk/test/test-pipeline/main.h rename dpdk/{app => test}/test-pipeline/pipeline_acl.c (100%) create mode 100644 dpdk/test/test-pipeline/pipeline_hash.c rename dpdk/{app => test}/test-pipeline/pipeline_lpm.c (100%) rename dpdk/{app => test}/test-pipeline/pipeline_lpm_ipv6.c (100%) rename dpdk/{app => test}/test-pipeline/pipeline_stub.c (100%) create mode 100644 dpdk/test/test-pipeline/runtime.c create mode 100644 dpdk/test/test/Makefile create mode 100644 dpdk/test/test/autotest.py create mode 100644 dpdk/test/test/autotest_data.py create mode 100644 dpdk/test/test/autotest_runner.py create mode 100644 dpdk/test/test/autotest_test_funcs.py create mode 100644 dpdk/test/test/commands.c create mode 100644 dpdk/test/test/packet_burst_generator.c create mode 100644 dpdk/test/test/packet_burst_generator.h create mode 100644 dpdk/test/test/process.h create mode 100644 dpdk/test/test/resource.c create mode 100644 dpdk/test/test/resource.h create mode 100644 dpdk/test/test/test.c create mode 100644 dpdk/test/test/test.h create mode 100644 dpdk/test/test/test_acl.c create mode 100644 dpdk/test/test/test_acl.h create mode 100644 dpdk/test/test/test_alarm.c create mode 100644 dpdk/test/test/test_atomic.c create mode 100644 dpdk/test/test/test_bitmap.c create mode 100644 dpdk/test/test/test_byteorder.c create mode 100644 dpdk/test/test/test_cfgfile.c create mode 100644 dpdk/test/test/test_cfgfiles/etc/empty.ini create mode 100644 dpdk/test/test/test_cfgfiles/etc/empty_key_value.ini create mode 100644 dpdk/test/test/test_cfgfiles/etc/invalid_section.ini create mode 100644 dpdk/test/test/test_cfgfiles/etc/line_too_long.ini create mode 100644 dpdk/test/test/test_cfgfiles/etc/missing_section.ini create mode 100644 dpdk/test/test/test_cfgfiles/etc/realloc_sections.ini create mode 100644 dpdk/test/test/test_cfgfiles/etc/sample1.ini create mode 100644 dpdk/test/test/test_cfgfiles/etc/sample2.ini create mode 100644 dpdk/test/test/test_cmdline.c create mode 100644 dpdk/test/test/test_cmdline.h create mode 100644 dpdk/test/test/test_cmdline_cirbuf.c create mode 100644 dpdk/test/test/test_cmdline_etheraddr.c create mode 100644 dpdk/test/test/test_cmdline_ipaddr.c create mode 100644 dpdk/test/test/test_cmdline_lib.c create mode 100644 dpdk/test/test/test_cmdline_num.c create mode 100644 dpdk/test/test/test_cmdline_portlist.c create mode 100644 dpdk/test/test/test_cmdline_string.c create mode 100644 dpdk/test/test/test_common.c create mode 100644 dpdk/test/test/test_cpuflags.c create mode 100644 dpdk/test/test/test_crc.c create mode 100644 dpdk/test/test/test_cryptodev.c create mode 100644 dpdk/test/test/test_cryptodev.h create mode 100644 dpdk/test/test/test_cryptodev_aead_test_vectors.h create mode 100644 dpdk/test/test/test_cryptodev_aes_test_vectors.h create mode 100644 dpdk/test/test/test_cryptodev_blockcipher.c create mode 100644 dpdk/test/test/test_cryptodev_blockcipher.h create mode 100644 dpdk/test/test/test_cryptodev_des_test_vectors.h create mode 100644 dpdk/test/test/test_cryptodev_hash_test_vectors.h create mode 100644 dpdk/test/test/test_cryptodev_hmac_test_vectors.h create mode 100644 dpdk/test/test/test_cryptodev_kasumi_hash_test_vectors.h create mode 100644 dpdk/test/test/test_cryptodev_kasumi_test_vectors.h create mode 100644 dpdk/test/test/test_cryptodev_snow3g_hash_test_vectors.h create mode 100644 dpdk/test/test/test_cryptodev_snow3g_test_vectors.h create mode 100644 dpdk/test/test/test_cryptodev_zuc_test_vectors.h create mode 100644 dpdk/test/test/test_cycles.c create mode 100644 dpdk/test/test/test_debug.c create mode 100644 dpdk/test/test/test_devargs.c create mode 100644 dpdk/test/test/test_distributor.c create mode 100644 dpdk/test/test/test_distributor_perf.c create mode 100644 dpdk/test/test/test_eal_flags.c create mode 100644 dpdk/test/test/test_eal_fs.c create mode 100644 dpdk/test/test/test_efd.c create mode 100644 dpdk/test/test/test_efd_perf.c create mode 100644 dpdk/test/test/test_errno.c create mode 100644 dpdk/test/test/test_event_eth_rx_adapter.c create mode 100644 dpdk/test/test/test_event_ring.c create mode 100644 dpdk/test/test/test_eventdev.c create mode 100644 dpdk/test/test/test_eventdev_octeontx.c create mode 100644 dpdk/test/test/test_eventdev_sw.c create mode 100644 dpdk/test/test/test_flow_classify.c create mode 100644 dpdk/test/test/test_flow_classify.h create mode 100644 dpdk/test/test/test_func_reentrancy.c create mode 100644 dpdk/test/test/test_hash.c create mode 100644 dpdk/test/test/test_hash_functions.c create mode 100644 dpdk/test/test/test_hash_multiwriter.c create mode 100644 dpdk/test/test/test_hash_perf.c create mode 100644 dpdk/test/test/test_hash_scaling.c create mode 100644 dpdk/test/test/test_interrupts.c create mode 100644 dpdk/test/test/test_kni.c create mode 100644 dpdk/test/test/test_kvargs.c create mode 100644 dpdk/test/test/test_link_bonding.c create mode 100644 dpdk/test/test/test_link_bonding_mode4.c create mode 100644 dpdk/test/test/test_link_bonding_rssconf.c create mode 100644 dpdk/test/test/test_logs.c create mode 100644 dpdk/test/test/test_lpm.c create mode 100644 dpdk/test/test/test_lpm6.c create mode 100644 dpdk/test/test/test_lpm6_data.h create mode 100644 dpdk/test/test/test_lpm6_perf.c create mode 100644 dpdk/test/test/test_lpm_perf.c create mode 100644 dpdk/test/test/test_malloc.c create mode 100644 dpdk/test/test/test_mbuf.c create mode 100644 dpdk/test/test/test_member.c create mode 100644 dpdk/test/test/test_member_perf.c create mode 100644 dpdk/test/test/test_memcpy.c create mode 100644 dpdk/test/test/test_memcpy_perf.c create mode 100644 dpdk/test/test/test_memory.c create mode 100644 dpdk/test/test/test_mempool.c create mode 100644 dpdk/test/test/test_mempool_perf.c create mode 100644 dpdk/test/test/test_memzone.c create mode 100644 dpdk/test/test/test_meter.c create mode 100644 dpdk/test/test/test_mp_secondary.c create mode 100644 dpdk/test/test/test_per_lcore.c create mode 100644 dpdk/test/test/test_pmd_perf.c create mode 100644 dpdk/test/test/test_pmd_ring.c create mode 100644 dpdk/test/test/test_pmd_ring_perf.c create mode 100644 dpdk/test/test/test_power.c create mode 100644 dpdk/test/test/test_power_acpi_cpufreq.c create mode 100644 dpdk/test/test/test_power_kvm_vm.c create mode 100644 dpdk/test/test/test_prefetch.c create mode 100644 dpdk/test/test/test_red.c create mode 100644 dpdk/test/test/test_reorder.c create mode 100644 dpdk/test/test/test_resource.c create mode 100644 dpdk/test/test/test_ring.c create mode 100644 dpdk/test/test/test_ring_perf.c create mode 100644 dpdk/test/test/test_rwlock.c create mode 100644 dpdk/test/test/test_sched.c create mode 100644 dpdk/test/test/test_service_cores.c create mode 100644 dpdk/test/test/test_spinlock.c create mode 100644 dpdk/test/test/test_string_fns.c create mode 100644 dpdk/test/test/test_table.c create mode 100644 dpdk/test/test/test_table.h create mode 100644 dpdk/test/test/test_table_acl.c create mode 100644 dpdk/test/test/test_table_acl.h create mode 100644 dpdk/test/test/test_table_combined.c create mode 100644 dpdk/test/test/test_table_combined.h create mode 100644 dpdk/test/test/test_table_pipeline.c create mode 100644 dpdk/test/test/test_table_pipeline.h create mode 100644 dpdk/test/test/test_table_ports.c create mode 100644 dpdk/test/test/test_table_ports.h create mode 100644 dpdk/test/test/test_table_tables.c create mode 100644 dpdk/test/test/test_table_tables.h create mode 100644 dpdk/test/test/test_tailq.c create mode 100644 dpdk/test/test/test_thash.c create mode 100644 dpdk/test/test/test_timer.c create mode 100644 dpdk/test/test/test_timer_perf.c create mode 100644 dpdk/test/test/test_timer_racecond.c create mode 100644 dpdk/test/test/test_version.c create mode 100644 dpdk/test/test/test_xmmt_ops.h create mode 100644 dpdk/test/test/virtual_pmd.c create mode 100644 dpdk/test/test/virtual_pmd.h delete mode 100755 dpdk/tools/cpu_layout.py delete mode 100755 dpdk/tools/dpdk-devbind.py delete mode 100755 dpdk/tools/dpdk-pmdinfo.py delete mode 100755 dpdk/tools/dpdk-setup.sh create mode 100644 dpdk/usertools/cpu_layout.py create mode 100644 dpdk/usertools/dpdk-devbind.py create mode 100644 dpdk/usertools/dpdk-pmdinfo.py create mode 100644 dpdk/usertools/dpdk-setup.sh diff --git a/README.md b/README.md index 745c7126..d2d4f82b 100644 --- a/README.md +++ b/README.md @@ -33,12 +33,16 @@ Currently, besides authorized DNS server of DNSPod, there are various products i # clone F-Stack mkdir /data/f-stack git clone https://github.com/F-Stack/f-stack.git /data/f-stack - + + # install libnuma-dev + yum install numactl-devel # on Centos + #sudo apt-get install libnuma-dev # on Ubuntu + cd f-stack # compile DPDK - cd dpdk/tools + cd dpdk/usertools ./dpdk-setup.sh # compile with x86_64-native-linuxapp-gcc - + # Set hugepage # single-node system echo 1024 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages diff --git a/app/micro_thread/Makefile b/app/micro_thread/Makefile index 9e3e3381..315ca7f8 100644 --- a/app/micro_thread/Makefile +++ b/app/micro_thread/Makefile @@ -29,11 +29,9 @@ endif DEBUG= -g BINARY = libmt.a -FF_LIBS+= -L${FF_PATH}/lib -L${FF_DPDK}/lib -Wl,--whole-archive,-lfstack,--no-whole-archive -FF_LIBS+= -Wl,--whole-archive -lrte_pmd_vmxnet3_uio -lrte_pmd_i40e -lrte_pmd_ixgbe -lrte_pmd_e1000 -lrte_pmd_ring -FF_LIBS+= -Wl,--whole-archive -lrte_hash -lrte_kvargs -Wl,-lrte_mbuf -lethdev -lrte_eal -Wl,-lrte_mempool -FF_LIBS+= -lrte_ring -lrte_cmdline -lrte_cfgfile -lrte_kni -lrte_timer -Wl,-lrte_pmd_virtio -FF_LIBS+= -Wl,--no-whole-archive -lrt -lm -ldl -lcrypto -pthread +FF_LIBS+= -L${FF_PATH}/lib -Wl,--whole-archive,-lfstack,--no-whole-archive +FF_LIBS+= -L${FF_DPDK}/lib -Wl,--whole-archive,-ldpdk,--no-whole-archive +FF_LIBS+= -Wl,--no-whole-archive -lrt -lm -ldl -lcrypto -lpthread -lnuma # Comment the following line if you are not using the gnu c compiler #C_ARGS = -Wall -g -fPIC -D_DEBUG diff --git a/app/nginx-1.11.10/auto/make b/app/nginx-1.11.10/auto/make index e7928fcf..3e9b8615 100644 --- a/app/nginx-1.11.10/auto/make +++ b/app/nginx-1.11.10/auto/make @@ -28,12 +28,9 @@ if [ ! $FF_DPDK ]; then echo "FF_DPDK environment variable not defined, default:$FF_DPDK" fi -CORE_LIBS="$CORE_LIBS -L$FF_PATH/lib -L$FF_DPDK/lib -Wl,--whole-archive,-lfstack,--no-whole-archive" -CORE_LIBS="$CORE_LIBS -g -Wl,--no-as-needed -fvisibility=default -pthread -lm -lrt" -CORE_LIBS="$CORE_LIBS -Wl,--whole-archive -lrte_pmd_vmxnet3_uio -lrte_pmd_i40e -lrte_pmd_ixgbe -lrte_pmd_e1000 -lrte_pmd_ring" -CORE_LIBS="$CORE_LIBS -Wl,--whole-archive -lrte_hash -lrte_kvargs -Wl,-lrte_mbuf -lethdev -lrte_eal -Wl,-lrte_mempool" -CORE_LIBS="$CORE_LIBS -lrte_ring -lrte_cmdline -lrte_cfgfile -lrte_kni -lrte_timer -Wl,-lrte_pmd_virtio" -CORE_LIBS="$CORE_LIBS -Wl,--no-whole-archive -lrt -lm -ldl -lcrypto" +CORE_LIBS="$CORE_LIBS -L$FF_PATH/lib -Wl,--whole-archive,-lfstack,--no-whole-archive" +CORE_LIBS="$CORE_LIBS -L$FF_DPDK/lib -Wl,--whole-archive,-ldpdk,--no-whole-archive" +CORE_LIBS="$CORE_LIBS -Wl,--no-whole-archive -lrt -lm -ldl -lcrypto -lpthread -lnuma" cat << END > $NGX_MAKEFILE diff --git a/app/redis-3.2.8/src/Makefile b/app/redis-3.2.8/src/Makefile index c1014973..60573614 100644 --- a/app/redis-3.2.8/src/Makefile +++ b/app/redis-3.2.8/src/Makefile @@ -16,7 +16,7 @@ ifeq ($(FF_PATH),) $(error variable FF_PATH is not set) endif ifeq ($(FF_DPDK),) - $(error variable FF_DPDK is not set) + FF_DPDK= $(FF_PATH)/dpdk/x86_64-native-linuxapp-gcc endif release_hdr := $(shell sh -c './mkreleasehdr.sh') @@ -119,12 +119,9 @@ endif FINAL_CFLAGS+= -DHAVE_FF_KQUEUE FINAL_CFLAGS+= -I$(FF_PATH)/lib -FINAL_LIBS+= -L$(FF_PATH)/lib -L$(FF_DPDK)/lib -Wl,--whole-archive,-lfstack,--no-whole-archive -FINAL_LIBS+= -g -Wl,--no-as-needed -fvisibility=default -pthread -lm -lrt -FINAL_LIBS+= -Wl,--whole-archive -lrte_pmd_vmxnet3_uio -lrte_pmd_i40e -lrte_pmd_ixgbe -lrte_pmd_e1000 -lrte_pmd_ring -FINAL_LIBS+= -Wl,--whole-archive -lrte_hash -lrte_kvargs -Wl,-lrte_mbuf -lethdev -lrte_eal -Wl,-lrte_mempool -FINAL_LIBS+= -lrte_ring -lrte_cmdline -lrte_cfgfile -lrte_kni -lrte_timer -Wl,-lrte_pmd_virtio -FINAL_LIBS+= -Wl,--no-whole-archive -lrt -lm -ldl -lm -lcrypto +FINAL_LIBS+= -L${FF_PATH}/lib -Wl,--whole-archive,-lfstack,--no-whole-archive +FINAL_LIBS+= -L${FF_DPDK}/lib -Wl,--whole-archive,-ldpdk,--no-whole-archive +FINAL_LIBS+= -Wl,--no-whole-archive -lrt -lm -ldl -lcrypto -lpthread -lnuma REDIS_CC=$(QUIET_CC)$(CC) $(FINAL_CFLAGS) REDIS_LD=$(QUIET_LINK)$(CC) $(FINAL_LDFLAGS) diff --git a/doc/F-Stack_Development_Guide.md b/doc/F-Stack_Development_Guide.md index 7ce6ba58..da6ff3fd 100644 --- a/doc/F-Stack_Development_Guide.md +++ b/doc/F-Stack_Development_Guide.md @@ -1,15 +1,15 @@ # F-Stack Development Guide -With the rapid development of NIC, the poor performance of data packets processing with Linux kernel has become the bottleneck. However the rapid development of the Internet needs high performance of network processing, kernel bypass has caught more and more attention. There are various similar technologies appear, such as DPDK, NETMAP and PF_RING. The main idea of kernel bypass is that Linux is only used to deal with control flow, all data streams are processed in user space. Therefore kernel bypass can avoid performance bottlenecks caused by kernel packet copy, thread scheduling, system calls and interrupt. Further more, kernel bypass can achieve higher performance with multi optimizing methods. Within various techniques, DPDK has been widely used because of its more thorough isolation from kernel scheduling and active community support. +With the rapid development of Network Interface Cards the poor performance of data packet processing with the Linux kernel has become the bottleneck in modern network systems. Yet, the increasing demands of the Internet's growth demand a higher performant network processing solution. Kernel bypass has emerged to catch more and more attention. There are various similar technologies such as: DPDK, NETMAP and PF_RING. The main idea of kernel bypass is that Linux is only used to deal with control flow; all data streams are processed in user space. Therefore, kernel bypass can avoid performance bottlenecks caused by kernel packet copying, thread scheduling, system calls, and interrupts. Furthermore, kernel bypass can achieve higher performance with multi-optimizing methods. Within various techniques, DPDK has been widely used because of it's more thorough isolation from kernel scheduling and active community support. -F-Stack is an open source network framework with high performance based on DPDK. With follow characteristics +F-Stack is an open source high performant network framework based on DPDK with the following characteristics: -1. Ultra high network performance which can achieve network card under full load, 10 million concurrent, five million RPS, 1 million CPS. -2. Transplant FreeBSD 11.01 user space stack, provides a complete stack function, cut a great amount of irrelevant features. Therefore greatly enhance the performance. -3. Support Nginx, Redis and other mature applications, service can easily use F-Stack -4. With Multi-process architecture, easy to extend -5. Provide micro thread interface. Various applications with long time consuming can easily use F-Stack to get high performance without processing complex asynchronous logic. -6. Provide Epoll/kqueue interface that allow many kinds of applications easily use F-Stack +1. Ultra high network performance which the network card can achieve under full load: 10 million concurrent connections, 5 million RPS, 1 million CPS. +2. Transplant FreeBSD 11.01 user space stack, which provides a complete stack function, and cut a great amount of irrelevant features. This greatly enhances network performance. +3. Support Nginx, Redis, and other mature applications. Services can easily use F-Stack. +4. Easy to extend with multi-process architecture. +5. Provides micro thread interface. Various applications with stateful applications can easily use F-Stack to get high performance without processing complex asynchronous logic. +6. Provide an Epoll/Kqueue interface that allow many kinds of applications to easily use F-Stack. ## Structure of F-Stack code diff --git a/doc/F-Stack_Quick_Start_Guide.md b/doc/F-Stack_Quick_Start_Guide.md index 0abacece..ef9c3103 100644 --- a/doc/F-Stack_Quick_Start_Guide.md +++ b/doc/F-Stack_Quick_Start_Guide.md @@ -7,11 +7,18 @@ See Intel DPDK [linux_gsg](http://dpdk.org/doc/guides/linux_gsg/index.html) -## clone F-Stack +## Clone F-Stack mkdir /data/f-stack git clone https://github.com/F-Stack/f-stack.git /data/f-stack +## Install libnuma-dev + + # on Centos + yum install numactl-devel + # on Ubuntu + sudo apt-get install libnuma-dev + ## Compile DPDK Read DPDK Quick Started Guide or run the command below @@ -43,20 +50,20 @@ The mount point can be made permanent across reboots, by adding the following li ## offload NIC - modprobe uio - insmod /data/f-stack/dpdk/x86_64-native-linuxapp-gcc/kmod/igb_uio.ko - insmod /data/f-stack/dpdk/x86_64-native-linuxapp-gcc/kmod/rte_kni.ko - python dpdk-devbind.py --status - ifconfig eth0 down - python dpdk-devbind.py --bind=igb_uio eth0 # assuming that use 10GE NIC and eth0 + modprobe uio + insmod /data/f-stack/dpdk/x86_64-native-linuxapp-gcc/kmod/igb_uio.ko + insmod /data/f-stack/dpdk/x86_64-native-linuxapp-gcc/kmod/rte_kni.ko + python dpdk-devbind.py --status + ifconfig eth0 down + python dpdk-devbind.py --bind=igb_uio eth0 # assuming that use 10GE NIC and eth0 ## Compile lib - export FF_PATH=/data/f-stack - export FF_DPDK=/data/f-stack/dpdk/x86_64-native-linuxapp-gcc - cd ../../ - cd lib - make + export FF_PATH=/data/f-stack + export FF_DPDK=/data/f-stack/dpdk/x86_64-native-linuxapp-gcc + cd ../../ + cd lib + make ### Compile Nginx diff --git a/doc/Launch_F-Stack_on_AWS_EC2_in_one_minute.md b/doc/Launch_F-Stack_on_AWS_EC2_in_one_minute.md index a0a6969d..cba7d2f6 100644 --- a/doc/Launch_F-Stack_on_AWS_EC2_in_one_minute.md +++ b/doc/Launch_F-Stack_on_AWS_EC2_in_one_minute.md @@ -3,8 +3,8 @@ If you have a Redhat7.3 EC2 instance，and then execute the following cmds, you will get the F-Stack server in one minute sudo -i - yum install -y git gcc openssl-devel kernel-devel-$(uname -r) bc - + yum install -y git gcc openssl-devel kernel-devel-$(uname -r) bc numactl-devel + mkdir /data/f-stack git clone https://github.com/F-Stack/f-stack.git /data/f-stack @@ -17,7 +17,7 @@ echo 1024 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages mkdir /mnt/huge mount -t hugetlbfs nodev /mnt/huge - + # close ASLR; it is necessary in multiple process echo 0 > /proc/sys/kernel/randomize_va_space @@ -40,7 +40,7 @@ #export mybc=`ifconfig eth0 | grep "Bcast" | awk -F ' ' '{print $3}' | awk -F ':' '{print $2}'` #export myhw=`ifconfig eth0 | grep "HWaddr" | awk -F ' ' '{print $5}'` #export mygw=`route -n | grep 0.0.0.0 | grep eth0 | grep UG | awk -F ' ' '{print $2}' - + sed "s/addr=192.168.1.2/addr=${myaddr}/" -i /data/f-stack/config.ini sed "s/netmask=255.255.255.0/netmask=${mymask}/" -i /data/f-stack/config.ini sed "s/broadcast=192.168.1.255/broadcast=${mybc}/" -i /data/f-stack/config.ini @@ -53,7 +53,6 @@ sed "s/#tcp_port=80/tcp_port=80/" -i /data/f-stack/config.ini sed "s/#vlanstrip=1/vlanstrip=1/" -i /data/f-stack/config.ini - # Compile F-Stack lib export FF_PATH=/data/f-stack export FF_DPDK=/data/f-stack/dpdk/build @@ -68,7 +67,7 @@ # offload NIC（if there is only one NIC，the follow commands must run in a script） ifconfig eth0 down - python /data/f-stack/dpdk/tools/dpdk-devbind.py --bind=igb_uio eth0 + python /data/f-stack/dpdk/usertools/dpdk-devbind.py --bind=igb_uio eth0 # copy config.ini to $NGX_PREFIX/conf/f-stack.conf cp /data/f-stack/config.ini /usr/local/nginx_fstack/conf/f-stack.conf diff --git a/dpdk/.gitattributes b/dpdk/.gitattributes new file mode 100644 index 00000000..fe555f80 --- /dev/null +++ b/dpdk/.gitattributes @@ -0,0 +1,3 @@ +*.c diff=cpp +*.h diff=cpp +*.py diff=python diff --git a/dpdk/.gitignore b/dpdk/.gitignore index e69de29b..6df5ba06 100644 --- a/dpdk/.gitignore +++ b/dpdk/.gitignore @@ -0,0 +1,14 @@ +doc/guides/nics/overview_table.txt +doc/guides/cryptodevs/overview_feature_table.txt +doc/guides/cryptodevs/overview_cipher_table.txt +doc/guides/cryptodevs/overview_auth_table.txt +doc/guides/cryptodevs/overview_aead_table.txt +cscope.out.po +cscope.out.in +cscope.out +cscope.files +GTAGS +GPATH +GRTAGS +tags +TAGS diff --git a/dpdk/GNUmakefile b/dpdk/GNUmakefile index 00fe0db7..45b7fbbe 100644 --- a/dpdk/GNUmakefile +++ b/dpdk/GNUmakefile @@ -41,5 +41,6 @@ export RTE_SDK # ROOTDIRS-y := buildtools lib drivers app +ROOTDIRS- := test include $(RTE_SDK)/mk/rte.sdkroot.mk diff --git a/dpdk/MAINTAINERS b/dpdk/MAINTAINERS index 6536c6b1..f0baeb42 100644 --- a/dpdk/MAINTAINERS +++ b/dpdk/MAINTAINERS @@ -21,45 +21,50 @@ Descriptions of section entries: General Project Administration ------------------------------ -M: Thomas Monjalon + +Main Branch +M: Thomas Monjalon +M: Ferruh Yigit T: git://dpdk.org/dpdk -F: MAINTAINERS -F: scripts/check-maintainers.sh -F: scripts/check-git-log.sh -F: scripts/checkpatches.sh -F: scripts/load-devel-config.sh -F: scripts/test-build.sh Stable Branches ---------------- +M: Yuanhan Liu T: git://dpdk.org/dpdk-stable Security Issues ---------------- M: maintainers@dpdk.org - Documentation (with overlaps) -------------- -M: Siobhan Butler M: John McNamara F: README F: doc/ +Developers and Maintainers Tools +M: Thomas Monjalon +F: MAINTAINERS +F: devtools/check-dup-includes.sh +F: devtools/check-maintainers.sh +F: devtools/check-git-log.sh +F: devtools/check-includes.sh +F: devtools/checkpatches.sh +F: devtools/get-maintainer.sh +F: devtools/git-log-fixes.sh +F: devtools/load-devel-config +F: devtools/test-build.sh + Build System ------------ -M: Thomas Monjalon +M: Thomas Monjalon F: GNUmakefile F: Makefile F: config/ F: mk/ F: pkg/ -F: scripts/auto-config-h.sh -F: scripts/depdirs-rule.sh -F: scripts/gen-build-mk.sh -F: scripts/gen-config-h.sh -F: scripts/relpath.sh +F: buildtools/auto-config-h.sh +F: buildtools/gen-build-mk.sh +F: buildtools/gen-config-h.sh +F: buildtools/relpath.sh F: doc/build-sdk-quick.txt F: doc/guides/prog_guide/build_app.rst F: doc/guides/prog_guide/dev_kit_* @@ -69,44 +74,43 @@ ABI versioning M: Neil Horman F: lib/librte_compat/ F: doc/guides/rel_notes/deprecation.rst -F: scripts/validate-abi.sh +F: devtools/validate-abi.sh Driver information F: buildtools/pmdinfogen/ -F: tools/dpdk-pmdinfo.py +F: usertools/dpdk-pmdinfo.py +F: doc/guides/tools/pmdinfo.rst Environment Abstraction Layer ----------------------------- EAL API and common code -M: David Marchand F: lib/librte_eal/common/* F: lib/librte_eal/common/include/* F: lib/librte_eal/common/include/generic/ +F: lib/librte_eal/rte_eal_version.map F: doc/guides/prog_guide/env_abstraction_layer.rst -F: app/test/test_alarm.c -F: app/test/test_atomic.c -F: app/test/test_byteorder.c -F: app/test/test_common.c -F: app/test/test_cpuflags.c -F: app/test/test_cycles.c -F: app/test/test_debug.c -F: app/test/test_devargs.c -F: app/test/test_eal* -F: app/test/test_errno.c -F: app/test/test_interrupts.c -F: app/test/test_logs.c -F: app/test/test_memcpy* -F: app/test/test_pci.c -F: app/test/test_pci_sysfs/ -F: app/test/test_per_lcore.c -F: app/test/test_prefetch.c -F: app/test/test_rwlock.c -F: app/test/test_spinlock.c -F: app/test/test_string_fns.c -F: app/test/test_tailq.c -F: app/test/test_version.c +F: test/test/test_alarm.c +F: test/test/test_atomic.c +F: test/test/test_byteorder.c +F: test/test/test_common.c +F: test/test/test_cpuflags.c +F: test/test/test_cycles.c +F: test/test/test_debug.c +F: test/test/test_devargs.c +F: test/test/test_eal* +F: test/test/test_errno.c +F: test/test/test_interrupts.c +F: test/test/test_logs.c +F: test/test/test_memcpy* +F: test/test/test_per_lcore.c +F: test/test/test_prefetch.c +F: test/test/test_rwlock.c +F: test/test/test_spinlock.c +F: test/test/test_string_fns.c +F: test/test/test_tailq.c +F: test/test/test_version.c Memory Allocation M: Sergio Gonzalez Monroy @@ -116,10 +120,10 @@ F: lib/librte_eal/common/*malloc* F: lib/librte_eal/common/eal_common_mem* F: lib/librte_eal/common/eal_hugepages.h F: doc/guides/prog_guide/env_abstraction_layer.rst -F: app/test/test_func_reentrancy.c -F: app/test/test_malloc.c -F: app/test/test_memory.c -F: app/test/test_memzone.c +F: test/test/test_func_reentrancy.c +F: test/test/test_malloc.c +F: test/test/test_memory.c +F: test/test/test_memzone.c Keep alive M: Remy Horton @@ -132,36 +136,49 @@ Secondary process M: Sergio Gonzalez Monroy K: RTE_PROC_ F: doc/guides/prog_guide/multi_proc_support.rst -F: app/test/test_mp_secondary.c +F: test/test/test_mp_secondary.c F: examples/multi_process/ F: doc/guides/sample_app_ug/multi_process.rst +Service Cores - EXPERIMENTAL +M: Harry van Haaren +F: lib/librte_eal/common/include/rte_service.h +F: lib/librte_eal/common/include/rte_service_component.h +F: lib/librte_eal/common/rte_service.c +F: doc/guides/prog_guide/service_cores.rst +F: test/test/test_service_cores.c + +Bitmap +M: Cristian Dumitrescu +F: lib/librte_eal/common/include/rte_bitmap.h +F: test/test/test_bitmap.c + ARM v7 M: Jan Viktorin -M: Jianbo Liu +M: Jianbo Liu F: lib/librte_eal/common/arch/arm/ F: lib/librte_eal/common/include/arch/arm/ ARM v8 M: Jerin Jacob -M: Jianbo Liu +M: Jianbo Liu F: lib/librte_eal/common/include/arch/arm/*_64.h +F: lib/librte_net/net_crc_neon.h F: lib/librte_acl/acl_run_neon.* F: lib/librte_lpm/rte_lpm_neon.h F: lib/librte_hash/rte*_arm64.h +F: lib/librte_efd/rte*_arm64.h +F: lib/librte_table/rte*_arm64.h F: drivers/net/ixgbe/ixgbe_rxtx_vec_neon.c - -EZchip TILE-Gx -M: Zhigang Lu -M: Liming Sun -F: lib/librte_eal/common/arch/tile/ -F: lib/librte_eal/common/include/arch/tile/ -F: drivers/net/mpipe/ +F: drivers/net/i40e/i40e_rxtx_vec_neon.c +F: drivers/net/virtio/virtio_rxtx_simple_neon.c IBM POWER M: Chao Zhu F: lib/librte_eal/common/arch/ppc_64/ F: lib/librte_eal/common/include/arch/ppc_64/ +F: drivers/net/i40e/i40e_rxtx_vec_altivec.c +F: examples/l3fwd/*altivec.h Intel x86 M: Bruce Richardson @@ -170,7 +187,6 @@ F: lib/librte_eal/common/arch/x86/ F: lib/librte_eal/common/include/arch/x86/ Linux EAL (with overlaps) -M: David Marchand F: lib/librte_eal/linuxapp/Makefile F: lib/librte_eal/linuxapp/eal/ F: doc/guides/linux_gsg/ @@ -178,19 +194,12 @@ F: doc/guides/linux_gsg/ Linux UIO M: Ferruh Yigit F: lib/librte_eal/linuxapp/igb_uio/ -F: lib/librte_eal/linuxapp/eal/*uio* +F: drivers/bus/pci/linux/*uio* Linux VFIO M: Anatoly Burakov F: lib/librte_eal/linuxapp/eal/*vfio* - -Linux Xen -F: lib/librte_eal/linuxapp/xen_dom0/ -F: lib/librte_eal/linuxapp/eal/*xen* -F: lib/librte_eal/linuxapp/eal/include/exec-env/rte_dom0_common.h -F: drivers/net/xenvirt/ -F: doc/guides/xen/ -F: examples/vhost_xen/ +F: drivers/bus/pci/linux/*vfio* FreeBSD EAL (with overlaps) M: Bruce Richardson @@ -216,198 +225,391 @@ Core Libraries Memory pool M: Olivier Matz F: lib/librte_mempool/ +F: drivers/mempool/Makefile +F: drivers/mempool/ring/ +F: drivers/mempool/stack/ F: doc/guides/prog_guide/mempool_lib.rst -F: app/test/test_mempool* -F: app/test/test_func_reentrancy.c +F: test/test/test_mempool* +F: test/test/test_func_reentrancy.c Ring queue M: Olivier Matz F: lib/librte_ring/ F: doc/guides/prog_guide/ring_lib.rst -F: app/test/test_ring* -F: app/test/test_func_reentrancy.c +F: test/test/test_ring* +F: test/test/test_func_reentrancy.c Packet buffer M: Olivier Matz F: lib/librte_mbuf/ F: doc/guides/prog_guide/mbuf_lib.rst -F: app/test/test_mbuf.c +F: test/test/test_mbuf.c Ethernet API -M: Thomas Monjalon +M: Thomas Monjalon +T: git://dpdk.org/next/dpdk-next-net F: lib/librte_ether/ -F: scripts/test-null.sh +F: devtools/test-null.sh + +Flow API +M: Adrien Mazarguil +T: git://dpdk.org/next/dpdk-next-net +F: lib/librte_ether/rte_flow* + +Traffic Management API - EXPERIMENTAL +M: Cristian Dumitrescu +T: git://dpdk.org/next/dpdk-next-tm +F: lib/librte_ether/rte_tm* + +Traffic Metering and Policing API - EXPERIMENTAL +M: Cristian Dumitrescu +F: lib/librte_ether/rte_mtr* Crypto API M: Declan Doherty +T: git://dpdk.org/next/dpdk-next-crypto F: lib/librte_cryptodev/ -F: app/test/test_cryptodev* +F: test/test/test_cryptodev* F: examples/l2fwd-crypto/ +Security API - EXPERIMENTAL +M: Akhil Goyal +M: Declan Doherty +F: lib/librte_security/ +F: doc/guides/prog_guide/rte_security.rst + +Eventdev API +M: Jerin Jacob +T: git://dpdk.org/next/dpdk-next-eventdev +F: lib/librte_eventdev/ +F: drivers/event/skeleton/ +F: test/test/test_eventdev.c + +Eventdev Ethdev Rx Adapter API - EXPERIMENTAL +M: Nikhil Rao +T: git://dpdk.org/next/dpdk-next-eventdev +F: lib/librte_eventdev/*eth_rx_adapter* +F: test/test/test_event_eth_rx_adapter.c +F: doc/guides/prog_guide/event_ethernet_rx_adapter.rst + + +Bus Drivers +----------- + +NXP buses +M: Hemant Agrawal +M: Shreyansh Jain +F: drivers/bus/dpaa/ +F: drivers/bus/fslmc/ + +PCI bus driver +F: drivers/bus/pci/ + +VDEV bus driver +M: Jianfeng Tan +F: drivers/bus/vdev/ + Networking Drivers ------------------ +M: Ferruh Yigit +T: git://dpdk.org/next/dpdk-next-net +F: doc/guides/nics/features/default.ini Link bonding M: Declan Doherty F: drivers/net/bonding/ F: doc/guides/prog_guide/link_bonding_poll_mode_drv_lib.rst -F: app/test/test_link_bonding* +F: test/test/test_link_bonding* F: examples/bond/ +F: doc/guides/nics/features/bonding.ini Linux KNI M: Ferruh Yigit F: lib/librte_eal/linuxapp/kni/ F: lib/librte_kni/ F: doc/guides/prog_guide/kernel_nic_interface.rst -F: app/test/test_kni.c +F: test/test/test_kni.c F: examples/kni/ F: doc/guides/sample_app_ug/kernel_nic_interface.rst Linux AF_PACKET M: John W. Linville F: drivers/net/af_packet/ +F: doc/guides/nics/features/afpacket.ini Amazon ENA -M: Jan Medala -M: Jakub Palider -M: Netanel Belgazal +M: Marcin Wojtas +M: Michal Krawczyk +M: Guy Tzalik M: Evgeny Schemeilin F: drivers/net/ena/ F: doc/guides/nics/ena.rst +F: doc/guides/nics/features/ena.ini + +Atomic Rules ARK +M: Shepard Siegel +M: Ed Czeck +M: John Miller +F: drivers/net/ark/ +F: doc/guides/nics/ark.rst +F: doc/guides/nics/features/ark.ini Broadcom bnxt M: Stephen Hurd +M: Ajit Khaparde F: drivers/net/bnxt/ F: doc/guides/nics/bnxt.rst +F: doc/guides/nics/features/bnxt.ini Cavium ThunderX nicvf M: Jerin Jacob M: Maciej Czekaj F: drivers/net/thunderx/ F: doc/guides/nics/thunderx.rst +F: doc/guides/nics/features/thunderx.ini + +Cavium LiquidIO +M: Shijith Thotton +M: Srisivasubramanian Srinivasan +F: drivers/net/liquidio/ +F: doc/guides/nics/liquidio.rst +F: doc/guides/nics/features/liquidio.ini + +Cavium OcteonTX +M: Santosh Shukla +M: Jerin Jacob +F: drivers/mempool/octeontx/ +F: drivers/net/octeontx/ +F: doc/guides/nics/octeontx.rst +F: doc/guides/nics/features/octeontx.ini Chelsio cxgbe M: Rahul Lakkireddy F: drivers/net/cxgbe/ F: doc/guides/nics/cxgbe.rst +F: doc/guides/nics/features/cxgbe.ini Cisco enic M: John Daley M: Nelson Escobar F: drivers/net/enic/ F: doc/guides/nics/enic.rst - -Combo szedata2 -M: Matej Vido -F: drivers/net/szedata2/ -F: doc/guides/nics/szedata2.rst +F: doc/guides/nics/features/enic.ini Intel e1000 M: Wenzhuo Lu F: drivers/net/e1000/ F: doc/guides/nics/e1000em.rst F: doc/guides/nics/intel_vf.rst +F: doc/guides/nics/features/e1000.ini +F: doc/guides/nics/features/igb*.ini Intel ixgbe -M: Helin Zhang +M: Wenzhuo Lu M: Konstantin Ananyev F: drivers/net/ixgbe/ F: doc/guides/nics/ixgbe.rst F: doc/guides/nics/intel_vf.rst +F: doc/guides/nics/features/ixgbe*.ini Intel i40e -M: Helin Zhang M: Jingjing Wu +M: Beilei Xing F: drivers/net/i40e/ F: doc/guides/nics/i40e.rst F: doc/guides/nics/intel_vf.rst +F: doc/guides/nics/features/i40e*.ini Intel fm10k -M: Jing Chen +M: Qi Zhang +M: Xiao Wang F: drivers/net/fm10k/ +F: doc/guides/nics/features/fm10k*.ini Mellanox mlx4 M: Adrien Mazarguil F: drivers/net/mlx4/ F: doc/guides/nics/mlx4.rst +F: doc/guides/nics/features/mlx4.ini Mellanox mlx5 M: Adrien Mazarguil +M: Nelio Laranjeiro +M: Yongseok Koh F: drivers/net/mlx5/ F: doc/guides/nics/mlx5.rst +F: doc/guides/nics/features/mlx5.ini + +Marvell mrvl +M: Jacek Siuda +M: Tomasz Duszynski +M: Dmitri Epshtein +M: Natalie Samsonov +M: Jianbo Liu +F: drivers/net/mrvl/ +F: doc/guides/nics/mrvl.rst +F: doc/guides/nics/features/mrvl.ini + +Netcope szedata2 +M: Matej Vido +F: drivers/net/szedata2/ +F: doc/guides/nics/szedata2.rst +F: doc/guides/nics/features/szedata2.ini Netronome nfp M: Alejandro Lucero F: drivers/net/nfp/ F: doc/guides/nics/nfp.rst +F: doc/guides/nics/features/nfp*.ini + +NXP dpaa +M: Hemant Agrawal +M: Shreyansh Jain +F: drivers/mempool/dpaa/ +F: drivers/net/dpaa/ +F: doc/guides/nics/dpaa.rst +F: doc/guides/nics/features/dpaa.ini + +NXP dpaa2 +M: Hemant Agrawal +M: Shreyansh Jain +F: drivers/mempool/dpaa2/ +F: drivers/net/dpaa2/ +F: doc/guides/nics/dpaa2.rst +F: doc/guides/nics/features/dpaa2.ini QLogic bnx2x -M: Sony Chacko -M: Harish Patil -M: Rasesh Mody +M: Harish Patil +M: Rasesh Mody F: drivers/net/bnx2x/ F: doc/guides/nics/bnx2x.rst +F: doc/guides/nics/features/bnx2x*.ini QLogic qede PMD -M: Harish Patil -M: Rasesh Mody -M: Sony Chacko +M: Rasesh Mody +M: Harish Patil +M: Shahed Shaikh F: drivers/net/qede/ F: doc/guides/nics/qede.rst +F: doc/guides/nics/features/qede*.ini -RedHat virtio -M: Huawei Xie -M: Yuanhan Liu +Solarflare sfc_efx +M: Andrew Rybchenko +F: drivers/net/sfc/ +F: doc/guides/nics/sfc_efx.rst +F: doc/guides/nics/features/sfc_efx.ini + +VMware vmxnet3 +M: Shrikrishna Khare +F: drivers/net/vmxnet3/ +F: doc/guides/nics/vmxnet3.rst +F: doc/guides/nics/features/vmxnet3.ini + +Vhost-user +M: Yuanhan Liu +M: Maxime Coquelin T: git://dpdk.org/next/dpdk-next-virtio -F: drivers/net/virtio/ -F: doc/guides/nics/virtio.rst F: lib/librte_vhost/ F: doc/guides/prog_guide/vhost_lib.rst F: examples/vhost/ F: doc/guides/sample_app_ug/vhost.rst - -VMware vmxnet3 -M: Yong Wang -F: drivers/net/vmxnet3/ -F: doc/guides/nics/vmxnet3.rst +F: examples/vhost_scsi/ +F: doc/guides/sample_app_ug/vhost_scsi.rst Vhost PMD M: Tetsuya Mukawa -M: Yuanhan Liu +M: Yuanhan Liu +M: Maxime Coquelin T: git://dpdk.org/next/dpdk-next-virtio F: drivers/net/vhost/ +F: doc/guides/nics/features/vhost.ini + +Virtio PMD +M: Yuanhan Liu +M: Maxime Coquelin +T: git://dpdk.org/next/dpdk-next-virtio +F: drivers/net/virtio/ +F: doc/guides/nics/virtio.rst +F: doc/guides/nics/features/virtio*.ini + +Wind River AVP +M: Allain Legacy +M: Matt Peters +F: drivers/net/avp/ +F: doc/guides/nics/avp.rst +F: doc/guides/nics/features/avp.ini PCAP PMD -M: Nicolás Pernas Maradei M: Ferruh Yigit F: drivers/net/pcap/ F: doc/guides/nics/pcap_ring.rst +F: doc/guides/nics/features/pcap.ini + +Tap PMD +M: Pascal Mazon +F: drivers/net/tap/ +F: doc/guides/nics/tap.rst +F: doc/guides/nics/features/tap.ini + +KNI PMD +M: Ferruh Yigit +F: drivers/net/kni/ +F: doc/guides/nics/kni.rst +F: doc/guides/nics/features/kni.ini Ring PMD M: Bruce Richardson F: drivers/net/ring/ F: doc/guides/nics/pcap_ring.rst -F: app/test/test_pmd_ring.c -F: app/test/test_pmd_ring_perf.c +F: test/test/test_pmd_ring.c +F: test/test/test_pmd_ring_perf.c +F: doc/guides/nics/features/ring.ini Null Networking PMD M: Tetsuya Mukawa F: drivers/net/null/ +F: doc/guides/nics/features/null.ini + +Fail-safe PMD +M: Gaetan Rivet +F: drivers/net/failsafe/ +F: doc/guides/nics/fail_safe.rst + +Softnic PMD +M: Jasvinder Singh +M: Cristian Dumitrescu +F: drivers/net/softnic/ Crypto Drivers -------------- +M: Pablo de Lara +T: git://dpdk.org/next/dpdk-next-crypto +F: doc/guides/cryptodevs/features/default.ini -Intel AES-NI GCM PMD +ARMv8 Crypto +M: Jerin Jacob +F: drivers/crypto/armv8/ +F: doc/guides/cryptodevs/armv8.rst +F: doc/guides/cryptodevs/features/armv8.ini + +Crypto Scheduler +M: Fan Zhang +F: drivers/crypto/scheduler/ +F: doc/guides/cryptodevs/scheduler.rst + +Intel AES-NI GCM M: Declan Doherty F: drivers/crypto/aesni_gcm/ F: doc/guides/cryptodevs/aesni_gcm.rst +F: doc/guides/cryptodevs/features/aesni_gcm.ini Intel AES-NI Multi-Buffer M: Declan Doherty F: drivers/crypto/aesni_mb/ F: doc/guides/cryptodevs/aesni_mb.rst +F: doc/guides/cryptodevs/features/aesni_mb.ini Intel QuickAssist M: John Griffin @@ -415,29 +617,103 @@ M: Fiona Trahe M: Deepak Kumar Jain F: drivers/crypto/qat/ F: doc/guides/cryptodevs/qat.rst +F: doc/guides/cryptodevs/features/qat.ini -SNOW 3G PMD -M: Pablo de Lara -F: drivers/crypto/snow3g/ -F: doc/guides/cryptodevs/snow3g.rst - -KASUMI PMD +KASUMI M: Pablo de Lara F: drivers/crypto/kasumi/ F: doc/guides/cryptodevs/kasumi.rst +F: doc/guides/cryptodevs/features/kasumi.ini -Null Crypto PMD +Marvell Mrvl +M: Jacek Siuda +M: Tomasz Duszynski +M: Dmitri Epshtein +M: Natalie Samsonov +M: Jianbo Liu +F: drivers/crypto/mrvl/ +F: doc/guides/cryptodevs/mrvl.rst +F: doc/guides/cryptodevs/features/mrvl.ini + +Null Crypto M: Declan Doherty F: drivers/crypto/null/ F: doc/guides/cryptodevs/null.rst +F: doc/guides/cryptodevs/features/null.ini + +NXP DPAA_SEC +M: Akhil Goyal +M: Hemant Agrawal +F: drivers/crypto/dpaa_sec/ +F: doc/guides/cryptodevs/dpaa_sec.rst +F: doc/guides/cryptodevs/features/dpaa_sec.ini + +NXP DPAA2_SEC +M: Akhil Goyal +M: Hemant Agrawal +F: drivers/crypto/dpaa2_sec/ +F: doc/guides/cryptodevs/dpaa2_sec.rst +F: doc/guides/cryptodevs/features/dpaa2_sec.ini + +OpenSSL +M: Declan Doherty +F: drivers/crypto/openssl/ +F: doc/guides/cryptodevs/openssl.rst +F: doc/guides/cryptodevs/features/openssl.ini + +SNOW 3G +M: Pablo de Lara +F: drivers/crypto/snow3g/ +F: doc/guides/cryptodevs/snow3g.rst +F: doc/guides/cryptodevs/features/snow3g.ini + +ZUC +M: Pablo de Lara +F: drivers/crypto/zuc/ +F: doc/guides/cryptodevs/zuc.rst +F: doc/guides/cryptodevs/features/zuc.ini + + +Eventdev Drivers +---------------- +M: Jerin Jacob +T: git://dpdk.org/next/dpdk-next-eventdev + +Cavium OCTEONTX ssovf +M: Jerin Jacob +M: Santosh Shukla +F: drivers/event/octeontx/ +F: test/test/test_eventdev_octeontx.c +F: doc/guides/eventdevs/octeontx.rst + +NXP DPAA2 eventdev +M: Hemant Agrawal +M: Nipun Gupta +F: drivers/event/dpaa2/ +F: doc/guides/eventdevs/dpaa2.rst + +Software Eventdev PMD +M: Harry van Haaren +F: drivers/event/sw/ +F: test/test/test_eventdev_sw.c +F: doc/guides/eventdevs/sw.rst +F: examples/eventdev_pipeline_sw_pmd/ +F: doc/guides/sample_app_ug/eventdev_pipeline_sw_pmd.rst Packet processing ----------------- Network headers +M: Olivier Matz F: lib/librte_net/ +Packet CRC +M: Jasvinder Singh +F: lib/librte_net/rte_net_crc* +F: lib/librte_net/net_crc_sse.h +F: test/test/test_crc.c + IP fragmentation & reassembly M: Konstantin Ananyev F: lib/librte_ip_frag/ @@ -447,11 +723,31 @@ F: doc/guides/sample_app_ug/ip_frag.rst F: examples/ip_reassembly/ F: doc/guides/sample_app_ug/ip_reassembly.rst +Generic Receive Offload - EXPERIMENTAL +M: Jiayu Hu +F: lib/librte_gro/ +F: doc/guides/prog_guide/generic_receive_offload_lib.rst + +Generic Segmentation Offload +M: Jiayu Hu +M: Mark Kavanagh +F: lib/librte_gso/ +F: doc/guides/prog_guide/generic_segmentation_offload_lib.rst + +Flow Classify - EXPERIMENTAL +M: Bernard Iremonger +F: lib/librte_flow_classify/ +F: test/test/test_flow_classify* +F: doc/guides/prog_guide/flow_classify_lib.rst +F: examples/flow_classify/ +F: doc/guides/sample_app_ug/flow_classify.rst + Distributor M: Bruce Richardson +M: David Hunt F: lib/librte_distributor/ F: doc/guides/prog_guide/packet_distrib_lib.rst -F: app/test/test_distributor* +F: test/test/test_distributor* F: examples/distributor/ F: doc/guides/sample_app_ug/dist_app.rst @@ -459,7 +755,7 @@ Reorder M: Reshma Pattan F: lib/librte_reorder/ F: doc/guides/prog_guide/reorder_lib.rst -F: app/test/test_reorder* +F: test/test/test_reorder* F: examples/packet_ordering/ F: doc/guides/sample_app_ug/packet_ordering.rst @@ -467,8 +763,8 @@ Hierarchical scheduler M: Cristian Dumitrescu F: lib/librte_sched/ F: doc/guides/prog_guide/qos_framework.rst -F: app/test/test_red.c -F: app/test/test_sched.c +F: test/test/test_red.c +F: test/test/test_sched.c F: examples/qos_sched/ F: doc/guides/sample_app_ug/qos_scheduler.rst @@ -477,8 +773,7 @@ M: Reshma Pattan F: lib/librte_pdump/ F: doc/guides/prog_guide/pdump_lib.rst F: app/pdump/ -F: doc/guides/sample_app_ug/pdump.rst - +F: doc/guides/tools/pdump.rst Packet Framework ---------------- @@ -487,8 +782,8 @@ F: lib/librte_pipeline/ F: lib/librte_port/ F: lib/librte_table/ F: doc/guides/prog_guide/packet_framework.rst -F: app/test/test_table* -F: app/test-pipeline/ +F: test/test/test_table* +F: test/test-pipeline/ F: doc/guides/sample_app_ug/test_pipeline.rst F: examples/ip_pipeline/ F: doc/guides/sample_app_ug/ip_pipeline.rst @@ -501,68 +796,82 @@ ACL M: Konstantin Ananyev F: lib/librte_acl/ F: doc/guides/prog_guide/packet_classif_access_ctrl.rst -F: app/test-acl/ -F: app/test/test_acl.* +F: test/test-acl/ +F: test/test/test_acl.* F: examples/l3fwd-acl/ F: doc/guides/sample_app_ug/l3_forward_access_ctrl.rst +EFD +M: Byron Marohn +M: Pablo de Lara Guarch +F: lib/librte_efd/ +F: doc/guides/prog_guide/efd_lib.rst +F: test/test/test_efd* +F: examples/server_node_efd/ +F: doc/guides/sample_app_ug/server_node_efd.rst + Hashes M: Bruce Richardson M: Pablo de Lara F: lib/librte_hash/ F: doc/guides/prog_guide/hash_lib.rst -F: app/test/test_*hash* -F: app/test/test_func_reentrancy.c +F: test/test/test_*hash* +F: test/test/test_func_reentrancy.c LPM M: Bruce Richardson F: lib/librte_lpm/ F: doc/guides/prog_guide/lpm* -F: app/test/test_lpm* -F: app/test/test_func_reentrancy.c -F: app/test/test_xmmt_ops.h +F: test/test/test_lpm* +F: test/test/test_func_reentrancy.c +F: test/test/test_xmmt_ops.h + +Membership - EXPERIMENTAL +M: Yipeng Wang +M: Sameh Gobriel +F: lib/librte_member/ +F: doc/guides/prog_guide/member_lib.rst +F: test/test/test_member* Traffic metering M: Cristian Dumitrescu F: lib/librte_meter/ F: doc/guides/sample_app_ug/qos_scheduler.rst -F: app/test/test_meter.c +F: test/test/test_meter.c F: examples/qos_meter/ F: doc/guides/sample_app_ug/qos_metering.rst - Other libraries --------------- Configuration file M: Cristian Dumitrescu F: lib/librte_cfgfile/ +F: test/test/test_cfgfile.c +F: test/test/test_cfgfiles/ Interactive command line M: Olivier Matz F: lib/librte_cmdline/ -F: app/cmdline_test/ -F: app/test/test_cmdline* +F: test/cmdline_test/ +F: test/test/test_cmdline* F: examples/cmdline/ F: doc/guides/sample_app_ug/cmd_line.rst -Qemu IVSHMEM -M: Anatoly Burakov -F: lib/librte_ivshmem/ -F: lib/librte_eal/linuxapp/eal/eal_ivshmem.c -F: doc/guides/prog_guide/ivshmem_lib.rst -F: app/test/test_ivshmem.c -F: examples/l2fwd-ivshmem/ - Key/Value parsing M: Olivier Matz F: lib/librte_kvargs/ -F: app/test/test_kvargs.c +F: test/test/test_kvargs.c + +PCI +M: Gaetan Rivet +F: lib/librte_pci/ Power management +M: David Hunt F: lib/librte_power/ F: doc/guides/prog_guide/power_man.rst -F: app/test/test_power* +F: test/test/test_power* F: examples/l3fwd-power/ F: doc/guides/sample_app_ug/l3_forward_power_man.rst F: examples/vm_power_manager/ @@ -572,53 +881,75 @@ Timers M: Robert Sanford F: lib/librte_timer/ F: doc/guides/prog_guide/timer_lib.rst -F: app/test/test_timer* +F: test/test/test_timer* F: examples/timer/ F: doc/guides/sample_app_ug/timer.rst Job statistics -M: Pawel Wodkowski +M: Pablo de Lara F: lib/librte_jobstats/ F: examples/l2fwd-jobstats/ F: doc/guides/sample_app_ug/l2_forward_job_stats.rst +Metrics +M: Remy Horton +F: lib/librte_metrics/ + +Bit-rate statistics +M: Remy Horton +F: lib/librte_bitratestats/ + +Latency statistics +M: Reshma Pattan +F: lib/librte_latencystats/ + Test Applications ----------------- Unit tests framework -F: app/test/autotest* -F: app/test/commands.c -F: app/test/packet_burst_generator.c -F: app/test/packet_burst_generator.h -F: app/test/process.h -F: app/test/resource.* -F: app/test/test.c -F: app/test/test.h -F: app/test/test_pmd_perf.c -F: app/test/test_resource.c -F: app/test/virtual_pmd.c -F: app/test/virtual_pmd.h +F: test/Makefile +F: test/test/Makefile +F: test/test/autotest* +F: test/test/commands.c +F: test/test/packet_burst_generator.c +F: test/test/packet_burst_generator.h +F: test/test/process.h +F: test/test/resource.* +F: test/test/test.c +F: test/test/test.h +F: test/test/test_pmd_perf.c +F: test/test/test_resource.c +F: test/test/virtual_pmd.c +F: test/test/virtual_pmd.h Driver testing tool -M: Pablo de Lara +M: Jingjing Wu F: app/test-pmd/ F: doc/guides/testpmd_app_ug/ -Dump tool +Crypto performance test application +M: Declan Doherty +F: app/test-crypto-perf/ +F: doc/guides/tools/cryptoperf.rst + +Eventdev test application +M: Jerin Jacob +F: app/test-eventdev/ +F: doc/guides/tools/testeventdev.rst +F: doc/guides/tools/img/eventdev_* +F: test/test/test_event_ring.c + +Procinfo tool M: Maryam Tahhan -M: John McNamara +M: Reshma Pattan F: app/proc_info/ +F: doc/guides/tools/proc_info.rst Other Example Applications -------------------------- -M: Bruce Richardson -M: Pablo de Lara -F: examples/dpdk_qat/ -F: doc/guides/sample_app_ug/intel_quickassist.rst - M: Remy Horton F: examples/ethtool/ F: doc/guides/sample_app_ug/ethtool.rst @@ -626,12 +957,17 @@ F: doc/guides/sample_app_ug/ethtool.rst F: examples/exception_path/ F: doc/guides/sample_app_ug/exception_path.rst +M: Ori Kam +F: examples/flow_filtering/ +F: doc/guides/sample_app_ug/flow_filtering.rst + M: Bruce Richardson M: Pablo de Lara F: examples/helloworld/ F: doc/guides/sample_app_ug/hello_world.rst M: Sergio Gonzalez Monroy +M: Radu Nicolau F: examples/ipsec-secgw/ F: doc/guides/sample_app_ug/ipsec_secgw.rst @@ -663,12 +999,11 @@ F: examples/netmap_compat/ F: doc/guides/sample_app_ug/netmap_compatibility.rst L-threads - EXPERIMENTAL -M: Ian Betts +M: John McNamara F: examples/performance-thread/ F: doc/guides/sample_app_ug/performance_thread.rst M: Pablo de Lara -M: Daniel Mrzyglod F: examples/ptpclient/ F: examples/quota_watermark/ @@ -679,6 +1014,10 @@ M: John McNamara F: examples/rxtx_callbacks/ F: doc/guides/sample_app_ug/rxtx_callbacks.rst +M: Harry van Haaren +F: examples/service_cores/ +F: doc/guides/sample_app_ug/service_cores.rst + M: Bruce Richardson M: John McNamara F: examples/skeleton/ diff --git a/dpdk/app/Makefile b/dpdk/app/Makefile index 30ec292a..6398c184 100644 --- a/dpdk/app/Makefile +++ b/dpdk/app/Makefile @@ -31,12 +31,16 @@ include $(RTE_SDK)/mk/rte.vars.mk -DIRS-$(CONFIG_RTE_APP_TEST) += test -DIRS-$(CONFIG_RTE_LIBRTE_ACL) += test-acl -DIRS-$(CONFIG_RTE_LIBRTE_PIPELINE) += test-pipeline DIRS-$(CONFIG_RTE_TEST_PMD) += test-pmd -DIRS-$(CONFIG_RTE_LIBRTE_CMDLINE) += cmdline_test -DIRS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += proc_info +DIRS-$(CONFIG_RTE_PROC_INFO) += proc_info DIRS-$(CONFIG_RTE_LIBRTE_PDUMP) += pdump +ifeq ($(CONFIG_RTE_LIBRTE_CRYPTODEV),y) +DIRS-$(CONFIG_RTE_APP_CRYPTO_PERF) += test-crypto-perf +endif + +ifeq ($(CONFIG_RTE_LIBRTE_EVENTDEV),y) +DIRS-$(CONFIG_RTE_APP_EVENTDEV) += test-eventdev +endif + include $(RTE_SDK)/mk/rte.subdir.mk diff --git a/dpdk/app/cmdline_test/Makefile b/dpdk/app/cmdline_test/Makefile deleted file mode 100644 index c6169f56..00000000 --- a/dpdk/app/cmdline_test/Makefile +++ /dev/null @@ -1,55 +0,0 @@ -# BSD LICENSE -# -# Copyright(c) 2010-2014 Intel Corporation. All rights reserved. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in -# the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Intel Corporation nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -include $(RTE_SDK)/mk/rte.vars.mk - -ifeq ($(CONFIG_RTE_LIBRTE_CMDLINE),y) - -# -# library name -# -APP = cmdline_test - -# -# all sources are stored in SRCS-y -# -SRCS-y += cmdline_test.c -SRCS-y += commands.c - -CFLAGS += -O3 -CFLAGS += $(WERROR_FLAGS) - -# this application needs libraries first -DEPDIRS-y += lib drivers - -include $(RTE_SDK)/mk/rte.app.mk - -endif diff --git a/dpdk/app/cmdline_test/cmdline_test.py b/dpdk/app/cmdline_test/cmdline_test.py deleted file mode 100644 index 8efc5ead..00000000 --- a/dpdk/app/cmdline_test/cmdline_test.py +++ /dev/null @@ -1,113 +0,0 @@ -#!/usr/bin/python - -# BSD LICENSE -# -# Copyright(c) 2010-2014 Intel Corporation. All rights reserved. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in -# the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Intel Corporation nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# Script that runs cmdline_test app and feeds keystrokes into it. - -import sys, pexpect, string, os, cmdline_test_data - -# -# function to run test -# -def runTest(child,test): - child.send(test["Sequence"]) - if test["Result"] == None: - return 0 - child.expect(test["Result"],1) - -# -# history test is a special case -# -# This test does the following: -# 1) fills the history with garbage up to its full capacity -# (just enough to remove last entry) -# 2) scrolls back history to the very beginning -# 3) checks if the output is as expected, that is, the first -# number in the sequence (not the last entry before it) -# -# This is a self-contained test, it needs only a pexpect child -# -def runHistoryTest(child): - # find out history size - child.sendline(cmdline_test_data.CMD_GET_BUFSIZE) - child.expect("History buffer size: \\d+", timeout=1) - history_size = int(child.after[len(cmdline_test_data.BUFSIZE_TEMPLATE):]) - i = 0 - - # fill the history with numbers - while i < history_size / 10: - # add 1 to prevent from parsing as octals - child.send("1" + str(i).zfill(8) + cmdline_test_data.ENTER) - # the app will simply print out the number - child.expect(str(i + 100000000), timeout=1) - i += 1 - # scroll back history - child.send(cmdline_test_data.UP * (i + 2) + cmdline_test_data.ENTER) - child.expect("100000000", timeout=1) - -# the path to cmdline_test executable is supplied via command-line. -if len(sys.argv) < 2: - print "Error: please supply cmdline_test app path" - sys.exit(1) - -test_app_path = sys.argv[1] - -if not os.path.exists(test_app_path): - print "Error: please supply cmdline_test app path" - sys.exit(1) - -child = pexpect.spawn(test_app_path) - -print "Running command-line tests..." -for test in cmdline_test_data.tests: - print (test["Name"] + ":").ljust(30), - try: - runTest(child,test) - print "PASS" - except: - print "FAIL" - print child - sys.exit(1) - -# since last test quits the app, run new instance -child = pexpect.spawn(test_app_path) - -print ("History fill test:").ljust(30), -try: - runHistoryTest(child) - print "PASS" -except: - print "FAIL" - print child - sys.exit(1) -child.close() -sys.exit(0) diff --git a/dpdk/app/cmdline_test/cmdline_test_data.py b/dpdk/app/cmdline_test/cmdline_test_data.py deleted file mode 100644 index b1945a57..00000000 --- a/dpdk/app/cmdline_test/cmdline_test_data.py +++ /dev/null @@ -1,311 +0,0 @@ -#!/usr/bin/python - -# BSD LICENSE -# -# Copyright(c) 2010-2014 Intel Corporation. All rights reserved. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in -# the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Intel Corporation nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# collection of static data - -import sys - -# keycode constants -CTRL_A = chr(1) -CTRL_B = chr(2) -CTRL_C = chr(3) -CTRL_D = chr(4) -CTRL_E = chr(5) -CTRL_F = chr(6) -CTRL_K = chr(11) -CTRL_L = chr(12) -CTRL_N = chr(14) -CTRL_P = chr(16) -CTRL_W = chr(23) -CTRL_Y = chr(25) -ALT_B = chr(27) + chr(98) -ALT_D = chr(27) + chr(100) -ALT_F = chr(27) + chr(102) -ALT_BKSPACE = chr(27) + chr(127) -DEL = chr(27) + chr(91) + chr(51) + chr(126) -TAB = chr(9) -HELP = chr(63) -BKSPACE = chr(127) -RIGHT = chr(27) + chr(91) + chr(67) -DOWN = chr(27) + chr(91) + chr(66) -LEFT = chr(27) + chr(91) + chr(68) -UP = chr(27) + chr(91) + chr(65) -ENTER2 = '\r' -ENTER = '\n' - -# expected result constants -NOT_FOUND = "Command not found" -BAD_ARG = "Bad arguments" -AMBIG = "Ambiguous command" -CMD1 = "Command 1 parsed!" -CMD2 = "Command 2 parsed!" -SINGLE = "Single word command parsed!" -SINGLE_LONG = "Single long word command parsed!" -AUTO1 = "Autocomplete command 1 parsed!" -AUTO2 = "Autocomplete command 2 parsed!" - -# misc defines -CMD_QUIT = "quit" -CMD_GET_BUFSIZE = "get_history_bufsize" -BUFSIZE_TEMPLATE = "History buffer size: " -PROMPT = "CMDLINE_TEST>>" - -# test defines -# each test tests progressively diverse set of keys. this way for example -# if we want to use some key sequence in the test, we first need to test -# that it itself does what it is expected to do. Most of the tests are -# designed that way. -# -# example: "arrows & delete test 1". we enter a partially valid command, -# then move 3 chars left and use delete three times. this way we get to -# know that "delete", "left" and "ctrl+B" all work (because if any of -# them fails, the whole test will fail and next tests won't be run). -# -# each test consists of name, character sequence to send to child, -# and expected output (if any). - -tests = [ -# test basic commands - {"Name" : "command test 1", - "Sequence" : "ambiguous first" + ENTER, - "Result" : CMD1}, - {"Name" : "command test 2", - "Sequence" : "ambiguous second" + ENTER, - "Result" : CMD2}, - {"Name" : "command test 3", - "Sequence" : "ambiguous ambiguous" + ENTER, - "Result" : AMBIG}, - {"Name" : "command test 4", - "Sequence" : "ambiguous ambiguous2" + ENTER, - "Result" : AMBIG}, - - {"Name" : "invalid command test 1", - "Sequence" : "ambiguous invalid" + ENTER, - "Result" : BAD_ARG}, -# test invalid commands - {"Name" : "invalid command test 2", - "Sequence" : "invalid" + ENTER, - "Result" : NOT_FOUND}, - {"Name" : "invalid command test 3", - "Sequence" : "ambiguousinvalid" + ENTER2, - "Result" : NOT_FOUND}, - -# test arrows and deletes - {"Name" : "arrows & delete test 1", - "Sequence" : "singlebad" + LEFT*2 + CTRL_B + DEL*3 + ENTER, - "Result" : SINGLE}, - {"Name" : "arrows & delete test 2", - "Sequence" : "singlebad" + LEFT*5 + RIGHT + CTRL_F + DEL*3 + ENTER, - "Result" : SINGLE}, - -# test backspace - {"Name" : "backspace test", - "Sequence" : "singlebad" + BKSPACE*3 + ENTER, - "Result" : SINGLE}, - -# test goto left and goto right - {"Name" : "goto left test", - "Sequence" : "biguous first" + CTRL_A + "am" + ENTER, - "Result" : CMD1}, - {"Name" : "goto right test", - "Sequence" : "biguous fir" + CTRL_A + "am" + CTRL_E + "st" + ENTER, - "Result" : CMD1}, - -# test goto words - {"Name" : "goto left word test", - "Sequence" : "ambiguous st" + ALT_B + "fir" + ENTER, - "Result" : CMD1}, - {"Name" : "goto right word test", - "Sequence" : "ambig first" + CTRL_A + ALT_F + "uous" + ENTER, - "Result" : CMD1}, - -# test removing words - {"Name" : "remove left word 1", - "Sequence" : "single invalid" + CTRL_W + ENTER, - "Result" : SINGLE}, - {"Name" : "remove left word 2", - "Sequence" : "single invalid" + ALT_BKSPACE + ENTER, - "Result" : SINGLE}, - {"Name" : "remove right word", - "Sequence" : "single invalid" + ALT_B + ALT_D + ENTER, - "Result" : SINGLE}, - -# test kill buffer (copy and paste) - {"Name" : "killbuffer test 1", - "Sequence" : "ambiguous" + CTRL_A + CTRL_K + " first" + CTRL_A + CTRL_Y + ENTER, - "Result" : CMD1}, - {"Name" : "killbuffer test 2", - "Sequence" : "ambiguous" + CTRL_A + CTRL_K + CTRL_Y*26 + ENTER, - "Result" : NOT_FOUND}, - -# test newline - {"Name" : "newline test", - "Sequence" : "invalid" + CTRL_C + "single" + ENTER, - "Result" : SINGLE}, - -# test redisplay (nothing should really happen) - {"Name" : "redisplay test", - "Sequence" : "single" + CTRL_L + ENTER, - "Result" : SINGLE}, - -# test autocomplete - {"Name" : "autocomplete test 1", - "Sequence" : "si" + TAB + ENTER, - "Result" : SINGLE}, - {"Name" : "autocomplete test 2", - "Sequence" : "si" + TAB + "_" + TAB + ENTER, - "Result" : SINGLE_LONG}, - {"Name" : "autocomplete test 3", - "Sequence" : "in" + TAB + ENTER, - "Result" : NOT_FOUND}, - {"Name" : "autocomplete test 4", - "Sequence" : "am" + TAB + ENTER, - "Result" : BAD_ARG}, - {"Name" : "autocomplete test 5", - "Sequence" : "am" + TAB + "fir" + TAB + ENTER, - "Result" : CMD1}, - {"Name" : "autocomplete test 6", - "Sequence" : "am" + TAB + "fir" + TAB + TAB + ENTER, - "Result" : CMD1}, - {"Name" : "autocomplete test 7", - "Sequence" : "am" + TAB + "fir" + TAB + " " + TAB + ENTER, - "Result" : CMD1}, - {"Name" : "autocomplete test 8", - "Sequence" : "am" + TAB + " am" + TAB + " " + ENTER, - "Result" : AMBIG}, - {"Name" : "autocomplete test 9", - "Sequence" : "am" + TAB + "inv" + TAB + ENTER, - "Result" : BAD_ARG}, - {"Name" : "autocomplete test 10", - "Sequence" : "au" + TAB + ENTER, - "Result" : NOT_FOUND}, - {"Name" : "autocomplete test 11", - "Sequence" : "au" + TAB + "1" + ENTER, - "Result" : AUTO1}, - {"Name" : "autocomplete test 12", - "Sequence" : "au" + TAB + "2" + ENTER, - "Result" : AUTO2}, - {"Name" : "autocomplete test 13", - "Sequence" : "au" + TAB + "2" + TAB + ENTER, - "Result" : AUTO2}, - {"Name" : "autocomplete test 14", - "Sequence" : "au" + TAB + "2 " + TAB + ENTER, - "Result" : AUTO2}, - {"Name" : "autocomplete test 15", - "Sequence" : "24" + TAB + ENTER, - "Result" : "24"}, - -# test history - {"Name" : "history test 1", - "Sequence" : "invalid" + ENTER + "single" + ENTER + "invalid" + ENTER + UP + CTRL_P + ENTER, - "Result" : SINGLE}, - {"Name" : "history test 2", - "Sequence" : "invalid" + ENTER + "ambiguous first" + ENTER + "invalid" + ENTER + "single" + ENTER + UP * 3 + CTRL_N + DOWN + ENTER, - "Result" : SINGLE}, - -# -# tests that improve coverage -# - -# empty space tests - {"Name" : "empty space test 1", - "Sequence" : RIGHT + LEFT + CTRL_B + CTRL_F + ENTER, - "Result" : PROMPT}, - {"Name" : "empty space test 2", - "Sequence" : BKSPACE + ENTER, - "Result" : PROMPT}, - {"Name" : "empty space test 3", - "Sequence" : CTRL_E*2 + CTRL_A*2 + ENTER, - "Result" : PROMPT}, - {"Name" : "empty space test 4", - "Sequence" : ALT_F*2 + ALT_B*2 + ENTER, - "Result" : PROMPT}, - {"Name" : "empty space test 5", - "Sequence" : " " + CTRL_E*2 + CTRL_A*2 + ENTER, - "Result" : PROMPT}, - {"Name" : "empty space test 6", - "Sequence" : " " + CTRL_A + ALT_F*2 + ALT_B*2 + ENTER, - "Result" : PROMPT}, - {"Name" : "empty space test 7", - "Sequence" : " " + CTRL_A + CTRL_D + CTRL_E + CTRL_D + ENTER, - "Result" : PROMPT}, - {"Name" : "empty space test 8", - "Sequence" : " space" + CTRL_W*2 + ENTER, - "Result" : PROMPT}, - {"Name" : "empty space test 9", - "Sequence" : " space" + ALT_BKSPACE*2 + ENTER, - "Result" : PROMPT}, - {"Name" : "empty space test 10", - "Sequence" : " space " + CTRL_A + ALT_D*3 + ENTER, - "Result" : PROMPT}, - -# non-printable char tests - {"Name" : "non-printable test 1", - "Sequence" : chr(27) + chr(47) + ENTER, - "Result" : PROMPT}, - {"Name" : "non-printable test 2", - "Sequence" : chr(27) + chr(128) + ENTER*7, - "Result" : PROMPT}, - {"Name" : "non-printable test 3", - "Sequence" : chr(27) + chr(91) + chr(127) + ENTER*6, - "Result" : PROMPT}, - -# miscellaneous tests - {"Name" : "misc test 1", - "Sequence" : ENTER, - "Result" : PROMPT}, - {"Name" : "misc test 2", - "Sequence" : "single #comment" + ENTER, - "Result" : SINGLE}, - {"Name" : "misc test 3", - "Sequence" : "#empty line" + ENTER, - "Result" : PROMPT}, - {"Name" : "misc test 4", - "Sequence" : " single " + ENTER, - "Result" : SINGLE}, - {"Name" : "misc test 5", - "Sequence" : "single#" + ENTER, - "Result" : SINGLE}, - {"Name" : "misc test 6", - "Sequence" : 'a' * 257 + ENTER, - "Result" : NOT_FOUND}, - {"Name" : "misc test 7", - "Sequence" : "clear_history" + UP*5 + DOWN*5 + ENTER, - "Result" : PROMPT}, - {"Name" : "misc test 8", - "Sequence" : "a" + HELP + CTRL_C, - "Result" : PROMPT}, - {"Name" : "misc test 9", - "Sequence" : CTRL_D*3, - "Result" : None}, -] diff --git a/dpdk/app/pdump/Makefile b/dpdk/app/pdump/Makefile index 536198fa..38ac3e9a 100644 --- a/dpdk/app/pdump/Makefile +++ b/dpdk/app/pdump/Makefile @@ -41,9 +41,6 @@ CFLAGS += $(WERROR_FLAGS) SRCS-y := main.c -# this application needs libraries first -DEPDIRS-y += lib - include $(RTE_SDK)/mk/rte.app.mk endif diff --git a/dpdk/app/pdump/main.c b/dpdk/app/pdump/main.c index b76cfd0f..66272f59 100644 --- a/dpdk/app/pdump/main.c +++ b/dpdk/app/pdump/main.c @@ -68,8 +68,8 @@ #define CMD_LINE_OPT_SER_SOCK_PATH "server-socket-path" #define CMD_LINE_OPT_CLI_SOCK_PATH "client-socket-path" -#define VDEV_PCAP "eth_pcap_%s_%d,tx_pcap=%s" -#define VDEV_IFACE "eth_pcap_%s_%d,tx_iface=%s" +#define VDEV_PCAP "net_pcap_%s_%d,tx_pcap=%s" +#define VDEV_IFACE "net_pcap_%s_%d,tx_iface=%s" #define TX_STREAM_SIZE 64 #define MP_NAME "pdump_pool_%d" @@ -92,7 +92,6 @@ #define BURST_SIZE 32 #define NUM_VDEVS 2 -#define RTE_RING_SZ_MASK (unsigned)(0x0fffffff) /**< Ring size mask */ /* true if x is a power of 2 */ #define POWEROF2(x) ((((x)-1) & (x)) == 0) @@ -132,7 +131,7 @@ struct pdump_stats { struct pdump_tuples { /* cli params */ - uint8_t port; + uint16_t port; char *device_id; uint16_t queue; char rx_dev[TX_STREAM_SIZE]; @@ -497,7 +496,7 @@ pdump_rxtx(struct rte_ring *ring, uint8_t vdev_id, struct pdump_stats *stats) /* first dequeue packets from ring of primary process */ const uint16_t nb_in_deq = rte_ring_dequeue_burst(ring, - (void *)rxtx_bufs, BURST_SIZE); + (void *)rxtx_bufs, BURST_SIZE, NULL); stats->dequeue_pkts += nb_in_deq; if (nb_in_deq) { @@ -580,7 +579,7 @@ signal_handler(int sig_num) } static inline int -configure_vdev(uint8_t port_id) +configure_vdev(uint16_t port_id) { struct ether_addr addr; const uint16_t rxRings = 0, txRings = 1; @@ -610,7 +609,7 @@ configure_vdev(uint8_t port_id) rte_eth_macaddr_get(port_id, &addr); printf("Port %u MAC: %02"PRIx8" %02"PRIx8" %02"PRIx8 " %02"PRIx8" %02"PRIx8" %02"PRIx8"\n", - (unsigned)port_id, + port_id, addr.addr_bytes[0], addr.addr_bytes[1], addr.addr_bytes[2], addr.addr_bytes[3], addr.addr_bytes[4], addr.addr_bytes[5]); @@ -624,7 +623,7 @@ static void create_mp_ring_vdev(void) { int i; - uint8_t portid; + uint16_t portid; struct pdump_tuples *pt = NULL; struct rte_mempool *mbuf_pool = NULL; char vdev_args[SIZE]; diff --git a/dpdk/app/proc_info/Makefile b/dpdk/app/proc_info/Makefile index e051e032..9e90438e 100644 --- a/dpdk/app/proc_info/Makefile +++ b/dpdk/app/proc_info/Makefile @@ -39,7 +39,4 @@ CFLAGS += $(WERROR_FLAGS) SRCS-y := main.c -# this application needs libraries first -DEPDIRS-y += lib - include $(RTE_SDK)/mk/rte.app.mk diff --git a/dpdk/app/proc_info/main.c b/dpdk/app/proc_info/main.c index 595f79f6..64fbbd0f 100644 --- a/dpdk/app/proc_info/main.c +++ b/dpdk/app/proc_info/main.c @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. + * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -40,6 +40,7 @@ #include #include #include +#include #include #include @@ -52,28 +53,46 @@ #include #include #include -#include #include #include #include #include +#include /* Maximum long option length for option parsing. */ #define MAX_LONG_OPT_SZ 64 #define RTE_LOGTYPE_APP RTE_LOGTYPE_USER1 +#define MAX_STRING_LEN 256 + /**< mask of enabled ports */ static uint32_t enabled_port_mask; /**< Enable stats. */ static uint32_t enable_stats; /**< Enable xstats. */ static uint32_t enable_xstats; +/**< Enable collectd format*/ +static uint32_t enable_collectd_format; +/**< FD to send collectd format messages to STDOUT*/ +static int stdout_fd; +/**< Host id process is running on */ +static char host_id[MAX_LONG_OPT_SZ]; +/**< Enable metrics. */ +static uint32_t enable_metrics; /**< Enable stats reset. */ static uint32_t reset_stats; /**< Enable xstats reset. */ static uint32_t reset_xstats; /**< Enable memory info. */ static uint32_t mem_info; +/**< Enable displaying xstat name. */ +static uint32_t enable_xstats_name; +static char *xstats_name; + +/**< Enable xstats by ids. */ +#define MAX_NB_XSTATS_IDS 1024 +static uint32_t nb_xstats_ids; +static uint64_t xstats_ids[MAX_NB_XSTATS_IDS]; /**< display usage */ static void @@ -85,8 +104,15 @@ proc_info_usage(const char *prgname) " --stats: to display port statistics, enabled by default\n" " --xstats: to display extended port statistics, disabled by " "default\n" + " --metrics: to display derived metrics of the ports, disabled by " + "default\n" + " --xstats-name NAME: to display single xstat id by NAME\n" + " --xstats-ids IDLIST: to display xstat values by id. " + "The argument is comma-separated list of xstat ids to print out.\n" " --stats-reset: to reset port statistics\n" - " --xstats-reset: to reset port extended statistics\n", + " --xstats-reset: to reset port extended statistics\n" + " --collectd-format: to print statistics to STDOUT in expected by collectd format\n" + " --host-id STRING: host id used to identify the system process is running on\n", prgname); } @@ -116,6 +142,66 @@ parse_portmask(const char *portmask) } +/* + * Parse ids value list into array + */ +static int +parse_xstats_ids(char *list, uint64_t *ids, int limit) { + int length; + char *token; + char *ctx = NULL; + char *endptr; + + length = 0; + token = strtok_r(list, ",", &ctx); + while (token != NULL) { + ids[length] = strtoull(token, &endptr, 10); + if (*endptr != '\0') + return -EINVAL; + + length++; + if (length >= limit) + return -E2BIG; + + token = strtok_r(NULL, ",", &ctx); + } + + return length; +} + +static int +proc_info_preparse_args(int argc, char **argv) +{ + char *prgname = argv[0]; + int i; + + for (i = 0; i < argc; i++) { + /* Print stats or xstats to STDOUT in collectd format */ + if (!strncmp(argv[i], "--collectd-format", MAX_LONG_OPT_SZ)) { + enable_collectd_format = 1; + stdout_fd = dup(STDOUT_FILENO); + close(STDOUT_FILENO); + } + if (!strncmp(argv[i], "--host-id", MAX_LONG_OPT_SZ)) { + if ((i + 1) == argc) { + printf("Invalid host id or not specified\n"); + proc_info_usage(prgname); + return -1; + } + strncpy(host_id, argv[i+1], sizeof(host_id)); + } + } + + if (!strlen(host_id)) { + int err = gethostname(host_id, MAX_LONG_OPT_SZ-1); + + if (err) + strcpy(host_id, "unknown"); + } + + return 0; +} + /* Parse the argument given in the command line of the application */ static int proc_info_parse_args(int argc, char **argv) @@ -127,7 +213,12 @@ proc_info_parse_args(int argc, char **argv) {"stats", 0, NULL, 0}, {"stats-reset", 0, NULL, 0}, {"xstats", 0, NULL, 0}, + {"metrics", 0, NULL, 0}, {"xstats-reset", 0, NULL, 0}, + {"xstats-name", required_argument, NULL, 1}, + {"collectd-format", 0, NULL, 0}, + {"xstats-ids", 1, NULL, 1}, + {"host-id", 0, NULL, 0}, {NULL, 0, 0, 0} }; @@ -159,6 +250,10 @@ proc_info_parse_args(int argc, char **argv) else if (!strncmp(long_option[option_index].name, "xstats", MAX_LONG_OPT_SZ)) enable_xstats = 1; + else if (!strncmp(long_option[option_index].name, + "metrics", + MAX_LONG_OPT_SZ)) + enable_metrics = 1; /* Reset stats */ if (!strncmp(long_option[option_index].name, "stats-reset", MAX_LONG_OPT_SZ)) @@ -168,7 +263,28 @@ proc_info_parse_args(int argc, char **argv) MAX_LONG_OPT_SZ)) reset_xstats = 1; break; + case 1: + /* Print xstat single value given by name*/ + if (!strncmp(long_option[option_index].name, + "xstats-name", MAX_LONG_OPT_SZ)) { + enable_xstats_name = 1; + xstats_name = optarg; + printf("name:%s:%s\n", + long_option[option_index].name, + optarg); + } else if (!strncmp(long_option[option_index].name, + "xstats-ids", + MAX_LONG_OPT_SZ)) { + nb_xstats_ids = parse_xstats_ids(optarg, + xstats_ids, MAX_NB_XSTATS_IDS); + if (nb_xstats_ids <= 0) { + printf("xstats-id list parse error.\n"); + return -1; + } + + } + break; default: proc_info_usage(prgname); return -1; @@ -194,7 +310,7 @@ meminfo_display(void) } static void -nic_stats_display(uint8_t port_id) +nic_stats_display(uint16_t port_id) { struct rte_eth_stats stats; uint8_t i; @@ -233,28 +349,92 @@ nic_stats_display(uint8_t port_id) } static void -nic_stats_clear(uint8_t port_id) +nic_stats_clear(uint16_t port_id) { printf("\n Clearing NIC stats for port %d\n", port_id); rte_eth_stats_reset(port_id); printf("\n NIC statistics for port %d cleared\n", port_id); } +static void collectd_resolve_cnt_type(char *cnt_type, size_t cnt_type_len, + const char *cnt_name) { + char *type_end = strrchr(cnt_name, '_'); + + if ((type_end != NULL) && + (strncmp(cnt_name, "rx_", strlen("rx_")) == 0)) { + if (strncmp(type_end, "_errors", strlen("_errors")) == 0) + strncpy(cnt_type, "if_rx_errors", cnt_type_len); + else if (strncmp(type_end, "_dropped", strlen("_dropped")) == 0) + strncpy(cnt_type, "if_rx_dropped", cnt_type_len); + else if (strncmp(type_end, "_bytes", strlen("_bytes")) == 0) + strncpy(cnt_type, "if_rx_octets", cnt_type_len); + else if (strncmp(type_end, "_packets", strlen("_packets")) == 0) + strncpy(cnt_type, "if_rx_packets", cnt_type_len); + else if (strncmp(type_end, "_placement", + strlen("_placement")) == 0) + strncpy(cnt_type, "if_rx_errors", cnt_type_len); + else if (strncmp(type_end, "_buff", strlen("_buff")) == 0) + strncpy(cnt_type, "if_rx_errors", cnt_type_len); + else + /* Does not fit obvious type: use a more generic one */ + strncpy(cnt_type, "derive", cnt_type_len); + } else if ((type_end != NULL) && + (strncmp(cnt_name, "tx_", strlen("tx_"))) == 0) { + if (strncmp(type_end, "_errors", strlen("_errors")) == 0) + strncpy(cnt_type, "if_tx_errors", cnt_type_len); + else if (strncmp(type_end, "_dropped", strlen("_dropped")) == 0) + strncpy(cnt_type, "if_tx_dropped", cnt_type_len); + else if (strncmp(type_end, "_bytes", strlen("_bytes")) == 0) + strncpy(cnt_type, "if_tx_octets", cnt_type_len); + else if (strncmp(type_end, "_packets", strlen("_packets")) == 0) + strncpy(cnt_type, "if_tx_packets", cnt_type_len); + else + /* Does not fit obvious type: use a more generic one */ + strncpy(cnt_type, "derive", cnt_type_len); + } else if ((type_end != NULL) && + (strncmp(cnt_name, "flow_", strlen("flow_"))) == 0) { + if (strncmp(type_end, "_filters", strlen("_filters")) == 0) + strncpy(cnt_type, "operations", cnt_type_len); + else if (strncmp(type_end, "_errors", strlen("_errors")) == 0) + strncpy(cnt_type, "errors", cnt_type_len); + else if (strncmp(type_end, "_filters", strlen("_filters")) == 0) + strncpy(cnt_type, "filter_result", cnt_type_len); + } else if ((type_end != NULL) && + (strncmp(cnt_name, "mac_", strlen("mac_"))) == 0) { + if (strncmp(type_end, "_errors", strlen("_errors")) == 0) + strncpy(cnt_type, "errors", cnt_type_len); + } else { + /* Does not fit obvious type, or strrchr error: */ + /* use a more generic type */ + strncpy(cnt_type, "derive", cnt_type_len); + } +} + static void -nic_xstats_display(uint8_t port_id) +nic_xstats_by_name_display(uint16_t port_id, char *name) +{ + uint64_t id; + + printf("###### NIC statistics for port %-2d, statistic name '%s':\n", + port_id, name); + + if (rte_eth_xstats_get_id_by_name(port_id, name, &id) == 0) + printf("%s: %"PRIu64"\n", name, id); + else + printf("Statistic not found...\n"); + +} + +static void +nic_xstats_by_ids_display(uint16_t port_id, uint64_t *ids, int len) { struct rte_eth_xstat_name *xstats_names; - struct rte_eth_xstat *xstats; - int len, ret, i; + uint64_t *values; + int ret, i; static const char *nic_stats_border = "########################"; - len = rte_eth_xstats_get_names(port_id, NULL, 0); - if (len < 0) { - printf("Cannot get xstats count\n"); - return; - } - xstats = malloc(sizeof(xstats[0]) * len); - if (xstats == NULL) { + values = malloc(sizeof(*values) * len); + if (values == NULL) { printf("Cannot allocate memory for xstats\n"); return; } @@ -262,20 +442,20 @@ nic_xstats_display(uint8_t port_id) xstats_names = malloc(sizeof(struct rte_eth_xstat_name) * len); if (xstats_names == NULL) { printf("Cannot allocate memory for xstat names\n"); - free(xstats); + free(values); return; } - if (len != rte_eth_xstats_get_names( - port_id, xstats_names, len)) { + + if (len != rte_eth_xstats_get_names_by_id( + port_id, xstats_names, len, ids)) { printf("Cannot get xstat names\n"); goto err; } printf("###### NIC extended statistics for port %-2d #########\n", port_id); - printf("%s############################\n", - nic_stats_border); - ret = rte_eth_xstats_get(port_id, xstats, len); + printf("%s############################\n", nic_stats_border); + ret = rte_eth_xstats_get_by_id(port_id, ids, values, len); if (ret < 0 || ret > len) { printf("Cannot get xstats\n"); goto err; @@ -284,23 +464,151 @@ nic_xstats_display(uint8_t port_id) for (i = 0; i < len; i++) printf("%s: %"PRIu64"\n", xstats_names[i].name, - xstats[i].value); + values[i]); - printf("%s############################\n", - nic_stats_border); + printf("%s############################\n", nic_stats_border); err: - free(xstats); + free(values); free(xstats_names); } static void -nic_xstats_clear(uint8_t port_id) +nic_xstats_display(uint16_t port_id) +{ + struct rte_eth_xstat_name *xstats_names; + uint64_t *values; + int len, ret, i; + static const char *nic_stats_border = "########################"; + + len = rte_eth_xstats_get_names_by_id(port_id, NULL, 0, NULL); + if (len < 0) { + printf("Cannot get xstats count\n"); + return; + } + values = malloc(sizeof(*values) * len); + if (values == NULL) { + printf("Cannot allocate memory for xstats\n"); + return; + } + + xstats_names = malloc(sizeof(struct rte_eth_xstat_name) * len); + if (xstats_names == NULL) { + printf("Cannot allocate memory for xstat names\n"); + free(values); + return; + } + if (len != rte_eth_xstats_get_names_by_id( + port_id, xstats_names, len, NULL)) { + printf("Cannot get xstat names\n"); + goto err; + } + + printf("###### NIC extended statistics for port %-2d #########\n", + port_id); + printf("%s############################\n", + nic_stats_border); + ret = rte_eth_xstats_get_by_id(port_id, NULL, values, len); + if (ret < 0 || ret > len) { + printf("Cannot get xstats\n"); + goto err; + } + + for (i = 0; i < len; i++) { + if (enable_collectd_format) { + char counter_type[MAX_STRING_LEN]; + char buf[MAX_STRING_LEN]; + + collectd_resolve_cnt_type(counter_type, + sizeof(counter_type), + xstats_names[i].name); + sprintf(buf, "PUTVAL %s/dpdkstat-port.%u/%s-%s N:%" + PRIu64"\n", host_id, port_id, counter_type, + xstats_names[i].name, values[i]); + ret = write(stdout_fd, buf, strlen(buf)); + if (ret < 0) + goto err; + } else { + printf("%s: %"PRIu64"\n", xstats_names[i].name, + values[i]); + } + } + + printf("%s############################\n", + nic_stats_border); +err: + free(values); + free(xstats_names); +} + +static void +nic_xstats_clear(uint16_t port_id) { printf("\n Clearing NIC xstats for port %d\n", port_id); rte_eth_xstats_reset(port_id); printf("\n NIC extended statistics for port %d cleared\n", port_id); } +static void +metrics_display(int port_id) +{ + struct rte_metric_value *metrics; + struct rte_metric_name *names; + int len, ret; + static const char *nic_stats_border = "########################"; + + len = rte_metrics_get_names(NULL, 0); + if (len < 0) { + printf("Cannot get metrics count\n"); + return; + } + if (len == 0) { + printf("No metrics to display (none have been registered)\n"); + return; + } + + metrics = rte_malloc("proc_info_metrics", + sizeof(struct rte_metric_value) * len, 0); + if (metrics == NULL) { + printf("Cannot allocate memory for metrics\n"); + return; + } + + names = rte_malloc(NULL, sizeof(struct rte_metric_name) * len, 0); + if (names == NULL) { + printf("Cannot allocate memory for metrcis names\n"); + rte_free(metrics); + return; + } + + if (len != rte_metrics_get_names(names, len)) { + printf("Cannot get metrics names\n"); + rte_free(metrics); + rte_free(names); + return; + } + + if (port_id == RTE_METRICS_GLOBAL) + printf("###### Non port specific metrics #########\n"); + else + printf("###### metrics for port %-2d #########\n", port_id); + printf("%s############################\n", nic_stats_border); + ret = rte_metrics_get_values(port_id, metrics, len); + if (ret < 0 || ret > len) { + printf("Cannot get metrics values\n"); + rte_free(metrics); + rte_free(names); + return; + } + + int i; + for (i = 0; i < len; i++) + printf("%s: %"PRIu64"\n", names[i].name, metrics[i].value); + + printf("%s############################\n", nic_stats_border); + rte_free(metrics); + rte_free(names); +} + int main(int argc, char **argv) { @@ -310,7 +618,14 @@ main(int argc, char **argv) char n_flag[] = "-n4"; char mp_flag[] = "--proc-type=secondary"; char *argp[argc + 3]; - uint8_t nb_ports; + uint16_t nb_ports; + + /* preparse app arguments */ + ret = proc_info_preparse_args(argc, argv); + if (ret < 0) { + printf("Failed to parse arguments\n"); + return -1; + } argp[0] = argv[0]; argp[1] = c_flag; @@ -329,6 +644,9 @@ main(int argc, char **argv) argc -= ret; argv += (ret - 3); + if (!rte_eal_primary_proc_alive(NULL)) + rte_exit(EXIT_FAILURE, "No primary DPDK process is running.\n"); + /* parse app arguments */ ret = proc_info_parse_args(argc, argv); if (ret < 0) @@ -357,8 +675,19 @@ main(int argc, char **argv) nic_stats_clear(i); else if (reset_xstats) nic_xstats_clear(i); + else if (enable_xstats_name) + nic_xstats_by_name_display(i, xstats_name); + else if (nb_xstats_ids > 0) + nic_xstats_by_ids_display(i, xstats_ids, + nb_xstats_ids); + else if (enable_metrics) + metrics_display(i); } } + /* print port independent stats */ + if (enable_metrics) + metrics_display(RTE_METRICS_GLOBAL); + return 0; } diff --git a/dpdk/app/test-acl/Makefile b/dpdk/app/test-acl/Makefile deleted file mode 100644 index 43dfdcbd..00000000 --- a/dpdk/app/test-acl/Makefile +++ /dev/null @@ -1,48 +0,0 @@ -# BSD LICENSE -# -# Copyright(c) 2010-2014 Intel Corporation. All rights reserved. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in -# the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Intel Corporation nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -include $(RTE_SDK)/mk/rte.vars.mk - -ifeq ($(CONFIG_RTE_LIBRTE_ACL),y) - -APP = testacl - -CFLAGS += $(WERROR_FLAGS) - -# all source are stored in SRCS-y -SRCS-y := main.c - -# this application needs libraries first -DEPDIRS-y += lib - -include $(RTE_SDK)/mk/rte.app.mk - -endif diff --git a/dpdk/app/test-acl/main.c b/dpdk/app/test-acl/main.c deleted file mode 100644 index d3669814..00000000 --- a/dpdk/app/test-acl/main.c +++ /dev/null @@ -1,1121 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include -#include -#include - -#include -#include -#include -#include - -#define PRINT_USAGE_START "%s [EAL options]\n" - -#define RTE_LOGTYPE_TESTACL RTE_LOGTYPE_USER1 - -#define APP_NAME "TESTACL" - -#define GET_CB_FIELD(in, fd, base, lim, dlm) do { \ - unsigned long val; \ - char *end_fld; \ - errno = 0; \ - val = strtoul((in), &end_fld, (base)); \ - if (errno != 0 || end_fld[0] != (dlm) || val > (lim)) \ - return -EINVAL; \ - (fd) = (typeof(fd))val; \ - (in) = end_fld + 1; \ -} while (0) - -#define OPT_RULE_FILE "rulesf" -#define OPT_TRACE_FILE "tracef" -#define OPT_RULE_NUM "rulenum" -#define OPT_TRACE_NUM "tracenum" -#define OPT_TRACE_STEP "tracestep" -#define OPT_SEARCH_ALG "alg" -#define OPT_BLD_CATEGORIES "bldcat" -#define OPT_RUN_CATEGORIES "runcat" -#define OPT_MAX_SIZE "maxsize" -#define OPT_ITER_NUM "iter" -#define OPT_VERBOSE "verbose" -#define OPT_IPV6 "ipv6" - -#define TRACE_DEFAULT_NUM 0x10000 -#define TRACE_STEP_MAX 0x1000 -#define TRACE_STEP_DEF 0x100 - -#define RULE_NUM 0x10000 - -enum { - DUMP_NONE, - DUMP_SEARCH, - DUMP_PKT, - DUMP_MAX -}; - -struct acl_alg { - const char *name; - enum rte_acl_classify_alg alg; -}; - -static const struct acl_alg acl_alg[] = { - { - .name = "scalar", - .alg = RTE_ACL_CLASSIFY_SCALAR, - }, - { - .name = "sse", - .alg = RTE_ACL_CLASSIFY_SSE, - }, - { - .name = "avx2", - .alg = RTE_ACL_CLASSIFY_AVX2, - }, - { - .name = "neon", - .alg = RTE_ACL_CLASSIFY_NEON, - }, -}; - -static struct { - const char *prgname; - const char *rule_file; - const char *trace_file; - size_t max_size; - uint32_t bld_categories; - uint32_t run_categories; - uint32_t nb_rules; - uint32_t nb_traces; - uint32_t trace_step; - uint32_t trace_sz; - uint32_t iter_num; - uint32_t verbose; - uint32_t ipv6; - struct acl_alg alg; - uint32_t used_traces; - void *traces; - struct rte_acl_ctx *acx; -} config = { - .bld_categories = 3, - .run_categories = 1, - .nb_rules = RULE_NUM, - .nb_traces = TRACE_DEFAULT_NUM, - .trace_step = TRACE_STEP_DEF, - .iter_num = 1, - .verbose = DUMP_MAX, - .alg = { - .name = "default", - .alg = RTE_ACL_CLASSIFY_DEFAULT, - }, - .ipv6 = 0 -}; - -static struct rte_acl_param prm = { - .name = APP_NAME, - .socket_id = SOCKET_ID_ANY, -}; - -/* - * Rule and trace formats definitions. - */ - -struct ipv4_5tuple { - uint8_t proto; - uint32_t ip_src; - uint32_t ip_dst; - uint16_t port_src; - uint16_t port_dst; -}; - -enum { - PROTO_FIELD_IPV4, - SRC_FIELD_IPV4, - DST_FIELD_IPV4, - SRCP_FIELD_IPV4, - DSTP_FIELD_IPV4, - NUM_FIELDS_IPV4 -}; - -/* - * That effectively defines order of IPV4VLAN classifications: - * - PROTO - * - VLAN (TAG and DOMAIN) - * - SRC IP ADDRESS - * - DST IP ADDRESS - * - PORTS (SRC and DST) - */ -enum { - RTE_ACL_IPV4VLAN_PROTO, - RTE_ACL_IPV4VLAN_VLAN, - RTE_ACL_IPV4VLAN_SRC, - RTE_ACL_IPV4VLAN_DST, - RTE_ACL_IPV4VLAN_PORTS, - RTE_ACL_IPV4VLAN_NUM -}; - -struct rte_acl_field_def ipv4_defs[NUM_FIELDS_IPV4] = { - { - .type = RTE_ACL_FIELD_TYPE_BITMASK, - .size = sizeof(uint8_t), - .field_index = PROTO_FIELD_IPV4, - .input_index = RTE_ACL_IPV4VLAN_PROTO, - .offset = offsetof(struct ipv4_5tuple, proto), - }, - { - .type = RTE_ACL_FIELD_TYPE_MASK, - .size = sizeof(uint32_t), - .field_index = SRC_FIELD_IPV4, - .input_index = RTE_ACL_IPV4VLAN_SRC, - .offset = offsetof(struct ipv4_5tuple, ip_src), - }, - { - .type = RTE_ACL_FIELD_TYPE_MASK, - .size = sizeof(uint32_t), - .field_index = DST_FIELD_IPV4, - .input_index = RTE_ACL_IPV4VLAN_DST, - .offset = offsetof(struct ipv4_5tuple, ip_dst), - }, - { - .type = RTE_ACL_FIELD_TYPE_RANGE, - .size = sizeof(uint16_t), - .field_index = SRCP_FIELD_IPV4, - .input_index = RTE_ACL_IPV4VLAN_PORTS, - .offset = offsetof(struct ipv4_5tuple, port_src), - }, - { - .type = RTE_ACL_FIELD_TYPE_RANGE, - .size = sizeof(uint16_t), - .field_index = DSTP_FIELD_IPV4, - .input_index = RTE_ACL_IPV4VLAN_PORTS, - .offset = offsetof(struct ipv4_5tuple, port_dst), - }, -}; - -#define IPV6_ADDR_LEN 16 -#define IPV6_ADDR_U16 (IPV6_ADDR_LEN / sizeof(uint16_t)) -#define IPV6_ADDR_U32 (IPV6_ADDR_LEN / sizeof(uint32_t)) - -struct ipv6_5tuple { - uint8_t proto; - uint32_t ip_src[IPV6_ADDR_U32]; - uint32_t ip_dst[IPV6_ADDR_U32]; - uint16_t port_src; - uint16_t port_dst; -}; - -enum { - PROTO_FIELD_IPV6, - SRC1_FIELD_IPV6, - SRC2_FIELD_IPV6, - SRC3_FIELD_IPV6, - SRC4_FIELD_IPV6, - DST1_FIELD_IPV6, - DST2_FIELD_IPV6, - DST3_FIELD_IPV6, - DST4_FIELD_IPV6, - SRCP_FIELD_IPV6, - DSTP_FIELD_IPV6, - NUM_FIELDS_IPV6 -}; - -struct rte_acl_field_def ipv6_defs[NUM_FIELDS_IPV6] = { - { - .type = RTE_ACL_FIELD_TYPE_BITMASK, - .size = sizeof(uint8_t), - .field_index = PROTO_FIELD_IPV6, - .input_index = PROTO_FIELD_IPV6, - .offset = offsetof(struct ipv6_5tuple, proto), - }, - { - .type = RTE_ACL_FIELD_TYPE_MASK, - .size = sizeof(uint32_t), - .field_index = SRC1_FIELD_IPV6, - .input_index = SRC1_FIELD_IPV6, - .offset = offsetof(struct ipv6_5tuple, ip_src[0]), - }, - { - .type = RTE_ACL_FIELD_TYPE_MASK, - .size = sizeof(uint32_t), - .field_index = SRC2_FIELD_IPV6, - .input_index = SRC2_FIELD_IPV6, - .offset = offsetof(struct ipv6_5tuple, ip_src[1]), - }, - { - .type = RTE_ACL_FIELD_TYPE_MASK, - .size = sizeof(uint32_t), - .field_index = SRC3_FIELD_IPV6, - .input_index = SRC3_FIELD_IPV6, - .offset = offsetof(struct ipv6_5tuple, ip_src[2]), - }, - { - .type = RTE_ACL_FIELD_TYPE_MASK, - .size = sizeof(uint32_t), - .field_index = SRC4_FIELD_IPV6, - .input_index = SRC4_FIELD_IPV6, - .offset = offsetof(struct ipv6_5tuple, ip_src[3]), - }, - { - .type = RTE_ACL_FIELD_TYPE_MASK, - .size = sizeof(uint32_t), - .field_index = DST1_FIELD_IPV6, - .input_index = DST1_FIELD_IPV6, - .offset = offsetof(struct ipv6_5tuple, ip_dst[0]), - }, - { - .type = RTE_ACL_FIELD_TYPE_MASK, - .size = sizeof(uint32_t), - .field_index = DST2_FIELD_IPV6, - .input_index = DST2_FIELD_IPV6, - .offset = offsetof(struct ipv6_5tuple, ip_dst[1]), - }, - { - .type = RTE_ACL_FIELD_TYPE_MASK, - .size = sizeof(uint32_t), - .field_index = DST3_FIELD_IPV6, - .input_index = DST3_FIELD_IPV6, - .offset = offsetof(struct ipv6_5tuple, ip_dst[2]), - }, - { - .type = RTE_ACL_FIELD_TYPE_MASK, - .size = sizeof(uint32_t), - .field_index = DST4_FIELD_IPV6, - .input_index = DST4_FIELD_IPV6, - .offset = offsetof(struct ipv6_5tuple, ip_dst[3]), - }, - { - .type = RTE_ACL_FIELD_TYPE_RANGE, - .size = sizeof(uint16_t), - .field_index = SRCP_FIELD_IPV6, - .input_index = SRCP_FIELD_IPV6, - .offset = offsetof(struct ipv6_5tuple, port_src), - }, - { - .type = RTE_ACL_FIELD_TYPE_RANGE, - .size = sizeof(uint16_t), - .field_index = DSTP_FIELD_IPV6, - .input_index = SRCP_FIELD_IPV6, - .offset = offsetof(struct ipv6_5tuple, port_dst), - }, -}; - - -enum { - CB_FLD_SRC_ADDR, - CB_FLD_DST_ADDR, - CB_FLD_SRC_PORT_LOW, - CB_FLD_SRC_PORT_DLM, - CB_FLD_SRC_PORT_HIGH, - CB_FLD_DST_PORT_LOW, - CB_FLD_DST_PORT_DLM, - CB_FLD_DST_PORT_HIGH, - CB_FLD_PROTO, - CB_FLD_NUM, -}; - -enum { - CB_TRC_SRC_ADDR, - CB_TRC_DST_ADDR, - CB_TRC_SRC_PORT, - CB_TRC_DST_PORT, - CB_TRC_PROTO, - CB_TRC_NUM, -}; - -RTE_ACL_RULE_DEF(acl_rule, RTE_ACL_MAX_FIELDS); - -static const char cb_port_delim[] = ":"; - -static char line[LINE_MAX]; - -#define dump_verbose(lvl, fh, fmt, args...) do { \ - if ((lvl) <= (int32_t)config.verbose) \ - fprintf(fh, fmt, ##args); \ -} while (0) - - -/* - * Parse ClassBench input trace (test vectors and expected results) file. - * Expected format: - * \ - * - */ -static int -parse_cb_ipv4_trace(char *str, struct ipv4_5tuple *v) -{ - int i; - char *s, *sp, *in[CB_TRC_NUM]; - static const char *dlm = " \t\n"; - - s = str; - for (i = 0; i != RTE_DIM(in); i++) { - in[i] = strtok_r(s, dlm, &sp); - if (in[i] == NULL) - return -EINVAL; - s = NULL; - } - - GET_CB_FIELD(in[CB_TRC_SRC_ADDR], v->ip_src, 0, UINT32_MAX, 0); - GET_CB_FIELD(in[CB_TRC_DST_ADDR], v->ip_dst, 0, UINT32_MAX, 0); - GET_CB_FIELD(in[CB_TRC_SRC_PORT], v->port_src, 0, UINT16_MAX, 0); - GET_CB_FIELD(in[CB_TRC_DST_PORT], v->port_dst, 0, UINT16_MAX, 0); - GET_CB_FIELD(in[CB_TRC_PROTO], v->proto, 0, UINT8_MAX, 0); - - /* convert to network byte order. */ - v->ip_src = rte_cpu_to_be_32(v->ip_src); - v->ip_dst = rte_cpu_to_be_32(v->ip_dst); - v->port_src = rte_cpu_to_be_16(v->port_src); - v->port_dst = rte_cpu_to_be_16(v->port_dst); - - return 0; -} - -/* - * Parses IPV6 address, exepcts the following format: - * XXXX:XXXX:XXXX:XXXX:XXXX:XXXX:XXXX:XXXX (where X - is a hexedecimal digit). - */ -static int -parse_ipv6_addr(const char *in, const char **end, uint32_t v[IPV6_ADDR_U32], - char dlm) -{ - uint32_t addr[IPV6_ADDR_U16]; - - GET_CB_FIELD(in, addr[0], 16, UINT16_MAX, ':'); - GET_CB_FIELD(in, addr[1], 16, UINT16_MAX, ':'); - GET_CB_FIELD(in, addr[2], 16, UINT16_MAX, ':'); - GET_CB_FIELD(in, addr[3], 16, UINT16_MAX, ':'); - GET_CB_FIELD(in, addr[4], 16, UINT16_MAX, ':'); - GET_CB_FIELD(in, addr[5], 16, UINT16_MAX, ':'); - GET_CB_FIELD(in, addr[6], 16, UINT16_MAX, ':'); - GET_CB_FIELD(in, addr[7], 16, UINT16_MAX, dlm); - - *end = in; - - v[0] = (addr[0] << 16) + addr[1]; - v[1] = (addr[2] << 16) + addr[3]; - v[2] = (addr[4] << 16) + addr[5]; - v[3] = (addr[6] << 16) + addr[7]; - - return 0; -} - -static int -parse_cb_ipv6_addr_trace(const char *in, uint32_t v[IPV6_ADDR_U32]) -{ - int32_t rc; - const char *end; - - rc = parse_ipv6_addr(in, &end, v, 0); - if (rc != 0) - return rc; - - v[0] = rte_cpu_to_be_32(v[0]); - v[1] = rte_cpu_to_be_32(v[1]); - v[2] = rte_cpu_to_be_32(v[2]); - v[3] = rte_cpu_to_be_32(v[3]); - - return 0; -} - -/* - * Parse ClassBench input trace (test vectors and expected results) file. - * Expected format: - * \ - * - */ -static int -parse_cb_ipv6_trace(char *str, struct ipv6_5tuple *v) -{ - int32_t i, rc; - char *s, *sp, *in[CB_TRC_NUM]; - static const char *dlm = " \t\n"; - - s = str; - for (i = 0; i != RTE_DIM(in); i++) { - in[i] = strtok_r(s, dlm, &sp); - if (in[i] == NULL) - return -EINVAL; - s = NULL; - } - - /* get ip6 src address. */ - rc = parse_cb_ipv6_addr_trace(in[CB_TRC_SRC_ADDR], v->ip_src); - if (rc != 0) - return rc; - - /* get ip6 dst address. */ - rc = parse_cb_ipv6_addr_trace(in[CB_TRC_DST_ADDR], v->ip_dst); - if (rc != 0) - return rc; - - GET_CB_FIELD(in[CB_TRC_SRC_PORT], v->port_src, 0, UINT16_MAX, 0); - GET_CB_FIELD(in[CB_TRC_DST_PORT], v->port_dst, 0, UINT16_MAX, 0); - GET_CB_FIELD(in[CB_TRC_PROTO], v->proto, 0, UINT8_MAX, 0); - - /* convert to network byte order. */ - v->port_src = rte_cpu_to_be_16(v->port_src); - v->port_dst = rte_cpu_to_be_16(v->port_dst); - - return 0; -} - -static void -tracef_init(void) -{ - static const char name[] = APP_NAME; - FILE *f; - size_t sz; - uint32_t n; - struct ipv4_5tuple *v; - struct ipv6_5tuple *w; - - sz = config.nb_traces * (config.ipv6 ? sizeof(*w) : sizeof(*v)); - config.traces = rte_zmalloc_socket(name, sz, RTE_CACHE_LINE_SIZE, - SOCKET_ID_ANY); - if (config.traces == NULL) - rte_exit(EXIT_FAILURE, "Cannot allocate %zu bytes for " - "requested %u number of trace records\n", - sz, config.nb_traces); - - f = fopen(config.trace_file, "r"); - if (f == NULL) - rte_exit(-EINVAL, "failed to open file: %s\n", - config.trace_file); - - v = config.traces; - w = config.traces; - for (n = 0; n != config.nb_traces; n++) { - - if (fgets(line, sizeof(line), f) == NULL) - break; - - if (config.ipv6) { - if (parse_cb_ipv6_trace(line, w + n) != 0) - rte_exit(EXIT_FAILURE, - "%s: failed to parse ipv6 trace " - "record at line %u\n", - config.trace_file, n + 1); - } else { - if (parse_cb_ipv4_trace(line, v + n) != 0) - rte_exit(EXIT_FAILURE, - "%s: failed to parse ipv4 trace " - "record at line %u\n", - config.trace_file, n + 1); - } - } - - config.used_traces = n; - fclose(f); -} - -static int -parse_ipv6_net(const char *in, struct rte_acl_field field[4]) -{ - int32_t rc; - const char *mp; - uint32_t i, m, v[4]; - const uint32_t nbu32 = sizeof(uint32_t) * CHAR_BIT; - - /* get address. */ - rc = parse_ipv6_addr(in, &mp, v, '/'); - if (rc != 0) - return rc; - - /* get mask. */ - GET_CB_FIELD(mp, m, 0, CHAR_BIT * sizeof(v), 0); - - /* put all together. */ - for (i = 0; i != RTE_DIM(v); i++) { - if (m >= (i + 1) * nbu32) - field[i].mask_range.u32 = nbu32; - else - field[i].mask_range.u32 = m > (i * nbu32) ? - m - (i * 32) : 0; - - field[i].value.u32 = v[i]; - } - - return 0; -} - - -static int -parse_cb_ipv6_rule(char *str, struct acl_rule *v) -{ - int i, rc; - char *s, *sp, *in[CB_FLD_NUM]; - static const char *dlm = " \t\n"; - - /* - * Skip leading '@' - */ - if (strchr(str, '@') != str) - return -EINVAL; - - s = str + 1; - - for (i = 0; i != RTE_DIM(in); i++) { - in[i] = strtok_r(s, dlm, &sp); - if (in[i] == NULL) - return -EINVAL; - s = NULL; - } - - rc = parse_ipv6_net(in[CB_FLD_SRC_ADDR], v->field + SRC1_FIELD_IPV6); - if (rc != 0) { - RTE_LOG(ERR, TESTACL, - "failed to read source address/mask: %s\n", - in[CB_FLD_SRC_ADDR]); - return rc; - } - - rc = parse_ipv6_net(in[CB_FLD_DST_ADDR], v->field + DST1_FIELD_IPV6); - if (rc != 0) { - RTE_LOG(ERR, TESTACL, - "failed to read destination address/mask: %s\n", - in[CB_FLD_DST_ADDR]); - return rc; - } - - /* source port. */ - GET_CB_FIELD(in[CB_FLD_SRC_PORT_LOW], - v->field[SRCP_FIELD_IPV6].value.u16, - 0, UINT16_MAX, 0); - GET_CB_FIELD(in[CB_FLD_SRC_PORT_HIGH], - v->field[SRCP_FIELD_IPV6].mask_range.u16, - 0, UINT16_MAX, 0); - - if (strncmp(in[CB_FLD_SRC_PORT_DLM], cb_port_delim, - sizeof(cb_port_delim)) != 0) - return -EINVAL; - - /* destination port. */ - GET_CB_FIELD(in[CB_FLD_DST_PORT_LOW], - v->field[DSTP_FIELD_IPV6].value.u16, - 0, UINT16_MAX, 0); - GET_CB_FIELD(in[CB_FLD_DST_PORT_HIGH], - v->field[DSTP_FIELD_IPV6].mask_range.u16, - 0, UINT16_MAX, 0); - - if (strncmp(in[CB_FLD_DST_PORT_DLM], cb_port_delim, - sizeof(cb_port_delim)) != 0) - return -EINVAL; - - GET_CB_FIELD(in[CB_FLD_PROTO], v->field[PROTO_FIELD_IPV6].value.u8, - 0, UINT8_MAX, '/'); - GET_CB_FIELD(in[CB_FLD_PROTO], v->field[PROTO_FIELD_IPV6].mask_range.u8, - 0, UINT8_MAX, 0); - - return 0; -} - -static int -parse_ipv4_net(const char *in, uint32_t *addr, uint32_t *mask_len) -{ - uint8_t a, b, c, d, m; - - GET_CB_FIELD(in, a, 0, UINT8_MAX, '.'); - GET_CB_FIELD(in, b, 0, UINT8_MAX, '.'); - GET_CB_FIELD(in, c, 0, UINT8_MAX, '.'); - GET_CB_FIELD(in, d, 0, UINT8_MAX, '/'); - GET_CB_FIELD(in, m, 0, sizeof(uint32_t) * CHAR_BIT, 0); - - addr[0] = IPv4(a, b, c, d); - mask_len[0] = m; - - return 0; -} -/* - * Parse ClassBench rules file. - * Expected format: - * '@''/' \ - * '/' \ - * ":" \ - * ":" \ - * '/' - */ -static int -parse_cb_ipv4_rule(char *str, struct acl_rule *v) -{ - int i, rc; - char *s, *sp, *in[CB_FLD_NUM]; - static const char *dlm = " \t\n"; - - /* - * Skip leading '@' - */ - if (strchr(str, '@') != str) - return -EINVAL; - - s = str + 1; - - for (i = 0; i != RTE_DIM(in); i++) { - in[i] = strtok_r(s, dlm, &sp); - if (in[i] == NULL) - return -EINVAL; - s = NULL; - } - - rc = parse_ipv4_net(in[CB_FLD_SRC_ADDR], - &v->field[SRC_FIELD_IPV4].value.u32, - &v->field[SRC_FIELD_IPV4].mask_range.u32); - if (rc != 0) { - RTE_LOG(ERR, TESTACL, - "failed to read source address/mask: %s\n", - in[CB_FLD_SRC_ADDR]); - return rc; - } - - rc = parse_ipv4_net(in[CB_FLD_DST_ADDR], - &v->field[DST_FIELD_IPV4].value.u32, - &v->field[DST_FIELD_IPV4].mask_range.u32); - if (rc != 0) { - RTE_LOG(ERR, TESTACL, - "failed to read destination address/mask: %s\n", - in[CB_FLD_DST_ADDR]); - return rc; - } - - /* source port. */ - GET_CB_FIELD(in[CB_FLD_SRC_PORT_LOW], - v->field[SRCP_FIELD_IPV4].value.u16, - 0, UINT16_MAX, 0); - GET_CB_FIELD(in[CB_FLD_SRC_PORT_HIGH], - v->field[SRCP_FIELD_IPV4].mask_range.u16, - 0, UINT16_MAX, 0); - - if (strncmp(in[CB_FLD_SRC_PORT_DLM], cb_port_delim, - sizeof(cb_port_delim)) != 0) - return -EINVAL; - - /* destination port. */ - GET_CB_FIELD(in[CB_FLD_DST_PORT_LOW], - v->field[DSTP_FIELD_IPV4].value.u16, - 0, UINT16_MAX, 0); - GET_CB_FIELD(in[CB_FLD_DST_PORT_HIGH], - v->field[DSTP_FIELD_IPV4].mask_range.u16, - 0, UINT16_MAX, 0); - - if (strncmp(in[CB_FLD_DST_PORT_DLM], cb_port_delim, - sizeof(cb_port_delim)) != 0) - return -EINVAL; - - GET_CB_FIELD(in[CB_FLD_PROTO], v->field[PROTO_FIELD_IPV4].value.u8, - 0, UINT8_MAX, '/'); - GET_CB_FIELD(in[CB_FLD_PROTO], v->field[PROTO_FIELD_IPV4].mask_range.u8, - 0, UINT8_MAX, 0); - - return 0; -} - -typedef int (*parse_5tuple)(char *text, struct acl_rule *rule); - -static int -add_cb_rules(FILE *f, struct rte_acl_ctx *ctx) -{ - int rc; - uint32_t n; - struct acl_rule v; - parse_5tuple parser; - - memset(&v, 0, sizeof(v)); - parser = (config.ipv6 != 0) ? parse_cb_ipv6_rule : parse_cb_ipv4_rule; - - for (n = 1; fgets(line, sizeof(line), f) != NULL; n++) { - - rc = parser(line, &v); - if (rc != 0) { - RTE_LOG(ERR, TESTACL, "line %u: parse_cb_ipv4vlan_rule" - " failed, error code: %d (%s)\n", - n, rc, strerror(-rc)); - return rc; - } - - v.data.category_mask = RTE_LEN2MASK(RTE_ACL_MAX_CATEGORIES, - typeof(v.data.category_mask)); - v.data.priority = RTE_ACL_MAX_PRIORITY - n; - v.data.userdata = n; - - rc = rte_acl_add_rules(ctx, (struct rte_acl_rule *)&v, 1); - if (rc != 0) { - RTE_LOG(ERR, TESTACL, "line %u: failed to add rules " - "into ACL context, error code: %d (%s)\n", - n, rc, strerror(-rc)); - return rc; - } - } - - return 0; -} - -static void -acx_init(void) -{ - int ret; - FILE *f; - struct rte_acl_config cfg; - - memset(&cfg, 0, sizeof(cfg)); - - /* setup ACL build config. */ - if (config.ipv6) { - cfg.num_fields = RTE_DIM(ipv6_defs); - memcpy(&cfg.defs, ipv6_defs, sizeof(ipv6_defs)); - } else { - cfg.num_fields = RTE_DIM(ipv4_defs); - memcpy(&cfg.defs, ipv4_defs, sizeof(ipv4_defs)); - } - cfg.num_categories = config.bld_categories; - cfg.max_size = config.max_size; - - /* setup ACL creation parameters. */ - prm.rule_size = RTE_ACL_RULE_SZ(cfg.num_fields); - prm.max_rule_num = config.nb_rules; - - config.acx = rte_acl_create(&prm); - if (config.acx == NULL) - rte_exit(rte_errno, "failed to create ACL context\n"); - - /* set default classify method for this context. */ - if (config.alg.alg != RTE_ACL_CLASSIFY_DEFAULT) { - ret = rte_acl_set_ctx_classify(config.acx, config.alg.alg); - if (ret != 0) - rte_exit(ret, "failed to setup %s method " - "for ACL context\n", config.alg.name); - } - - /* add ACL rules. */ - f = fopen(config.rule_file, "r"); - if (f == NULL) - rte_exit(-EINVAL, "failed to open file %s\n", - config.rule_file); - - ret = add_cb_rules(f, config.acx); - if (ret != 0) - rte_exit(ret, "failed to add rules into ACL context\n"); - - fclose(f); - - /* perform build. */ - ret = rte_acl_build(config.acx, &cfg); - - dump_verbose(DUMP_NONE, stdout, - "rte_acl_build(%u) finished with %d\n", - config.bld_categories, ret); - - rte_acl_dump(config.acx); - - if (ret != 0) - rte_exit(ret, "failed to build search context\n"); -} - -static uint32_t -search_ip5tuples_once(uint32_t categories, uint32_t step, const char *alg) -{ - int ret; - uint32_t i, j, k, n, r; - const uint8_t *data[step], *v; - uint32_t results[step * categories]; - - v = config.traces; - for (i = 0; i != config.used_traces; i += n) { - - n = RTE_MIN(step, config.used_traces - i); - - for (j = 0; j != n; j++) { - data[j] = v; - v += config.trace_sz; - } - - ret = rte_acl_classify(config.acx, data, results, - n, categories); - - if (ret != 0) - rte_exit(ret, "classify for ipv%c_5tuples returns %d\n", - config.ipv6 ? '6' : '4', ret); - - for (r = 0, j = 0; j != n; j++) { - for (k = 0; k != categories; k++, r++) { - dump_verbose(DUMP_PKT, stdout, - "ipv%c_5tuple: %u, category: %u, " - "result: %u\n", - config.ipv6 ? '6' : '4', - i + j + 1, k, results[r] - 1); - } - - } - } - - dump_verbose(DUMP_SEARCH, stdout, - "%s(%u, %u, %s) returns %u\n", __func__, - categories, step, alg, i); - return i; -} - -static int -search_ip5tuples(__attribute__((unused)) void *arg) -{ - uint64_t pkt, start, tm; - uint32_t i, lcore; - - lcore = rte_lcore_id(); - start = rte_rdtsc(); - pkt = 0; - - for (i = 0; i != config.iter_num; i++) { - pkt += search_ip5tuples_once(config.run_categories, - config.trace_step, config.alg.name); - } - - tm = rte_rdtsc() - start; - dump_verbose(DUMP_NONE, stdout, - "%s @lcore %u: %" PRIu32 " iterations, %" PRIu64 " pkts, %" - PRIu32 " categories, %" PRIu64 " cycles, %#Lf cycles/pkt\n", - __func__, lcore, i, pkt, config.run_categories, - tm, (pkt == 0) ? 0 : (long double)tm / pkt); - - return 0; -} - -static unsigned long -get_ulong_opt(const char *opt, const char *name, size_t min, size_t max) -{ - unsigned long val; - char *end; - - errno = 0; - val = strtoul(opt, &end, 0); - if (errno != 0 || end[0] != 0 || val > max || val < min) - rte_exit(-EINVAL, "invalid value: \"%s\" for option: %s\n", - opt, name); - return val; -} - -static void -get_alg_opt(const char *opt, const char *name) -{ - uint32_t i; - - for (i = 0; i != RTE_DIM(acl_alg); i++) { - if (strcmp(opt, acl_alg[i].name) == 0) { - config.alg = acl_alg[i]; - return; - } - } - - rte_exit(-EINVAL, "invalid value: \"%s\" for option: %s\n", - opt, name); -} - -static void -print_usage(const char *prgname) -{ - uint32_t i, n, rc; - char buf[PATH_MAX]; - - n = 0; - buf[0] = 0; - - for (i = 0; i < RTE_DIM(acl_alg) - 1; i++) { - rc = snprintf(buf + n, sizeof(buf) - n, "%s|", - acl_alg[i].name); - if (rc > sizeof(buf) - n) - break; - n += rc; - } - - snprintf(buf + n, sizeof(buf) - n, "%s", acl_alg[i].name); - - fprintf(stdout, - PRINT_USAGE_START - "--" OPT_RULE_FILE "=\n" - "[--" OPT_TRACE_FILE "=]\n" - "[--" OPT_RULE_NUM - "=]\n" - "[--" OPT_TRACE_NUM - "=]\n" - "[--" OPT_TRACE_STEP - "=]\n" - "[--" OPT_BLD_CATEGORIES - "=]\n" - "[--" OPT_RUN_CATEGORIES - "= " - "should be either 1 or multiple of %zu, " - "but not greater then %u]\n" - "[--" OPT_MAX_SIZE - "= " - "leave 0 for default behaviour]\n" - "[--" OPT_ITER_NUM "=]\n" - "[--" OPT_VERBOSE "=]\n" - "[--" OPT_SEARCH_ALG "=%s]\n" - "[--" OPT_IPV6 "=]\n", - prgname, RTE_ACL_RESULTS_MULTIPLIER, - (uint32_t)RTE_ACL_MAX_CATEGORIES, - buf); -} - -static void -dump_config(FILE *f) -{ - fprintf(f, "%s:\n", __func__); - fprintf(f, "%s:%s\n", OPT_RULE_FILE, config.rule_file); - fprintf(f, "%s:%s\n", OPT_TRACE_FILE, config.trace_file); - fprintf(f, "%s:%u\n", OPT_RULE_NUM, config.nb_rules); - fprintf(f, "%s:%u\n", OPT_TRACE_NUM, config.nb_traces); - fprintf(f, "%s:%u\n", OPT_TRACE_STEP, config.trace_step); - fprintf(f, "%s:%u\n", OPT_BLD_CATEGORIES, config.bld_categories); - fprintf(f, "%s:%u\n", OPT_RUN_CATEGORIES, config.run_categories); - fprintf(f, "%s:%zu\n", OPT_MAX_SIZE, config.max_size); - fprintf(f, "%s:%u\n", OPT_ITER_NUM, config.iter_num); - fprintf(f, "%s:%u\n", OPT_VERBOSE, config.verbose); - fprintf(f, "%s:%u(%s)\n", OPT_SEARCH_ALG, config.alg.alg, - config.alg.name); - fprintf(f, "%s:%u\n", OPT_IPV6, config.ipv6); -} - -static void -check_config(void) -{ - if (config.rule_file == NULL) { - print_usage(config.prgname); - rte_exit(-EINVAL, "mandatory option %s is not specified\n", - OPT_RULE_FILE); - } -} - - -static void -get_input_opts(int argc, char **argv) -{ - static struct option lgopts[] = { - {OPT_RULE_FILE, 1, 0, 0}, - {OPT_TRACE_FILE, 1, 0, 0}, - {OPT_TRACE_NUM, 1, 0, 0}, - {OPT_RULE_NUM, 1, 0, 0}, - {OPT_MAX_SIZE, 1, 0, 0}, - {OPT_TRACE_STEP, 1, 0, 0}, - {OPT_BLD_CATEGORIES, 1, 0, 0}, - {OPT_RUN_CATEGORIES, 1, 0, 0}, - {OPT_ITER_NUM, 1, 0, 0}, - {OPT_VERBOSE, 1, 0, 0}, - {OPT_SEARCH_ALG, 1, 0, 0}, - {OPT_IPV6, 0, 0, 0}, - {NULL, 0, 0, 0} - }; - - int opt, opt_idx; - - while ((opt = getopt_long(argc, argv, "", lgopts, &opt_idx)) != EOF) { - - if (opt != 0) { - print_usage(config.prgname); - rte_exit(-EINVAL, "unknown option: %c", opt); - } - - if (strcmp(lgopts[opt_idx].name, OPT_RULE_FILE) == 0) { - config.rule_file = optarg; - } else if (strcmp(lgopts[opt_idx].name, OPT_TRACE_FILE) == 0) { - config.trace_file = optarg; - } else if (strcmp(lgopts[opt_idx].name, OPT_RULE_NUM) == 0) { - config.nb_rules = get_ulong_opt(optarg, - lgopts[opt_idx].name, 1, RTE_ACL_MAX_INDEX + 1); - } else if (strcmp(lgopts[opt_idx].name, OPT_MAX_SIZE) == 0) { - config.max_size = get_ulong_opt(optarg, - lgopts[opt_idx].name, 0, SIZE_MAX); - } else if (strcmp(lgopts[opt_idx].name, OPT_TRACE_NUM) == 0) { - config.nb_traces = get_ulong_opt(optarg, - lgopts[opt_idx].name, 1, UINT32_MAX); - } else if (strcmp(lgopts[opt_idx].name, OPT_TRACE_STEP) == 0) { - config.trace_step = get_ulong_opt(optarg, - lgopts[opt_idx].name, 1, TRACE_STEP_MAX); - } else if (strcmp(lgopts[opt_idx].name, - OPT_BLD_CATEGORIES) == 0) { - config.bld_categories = get_ulong_opt(optarg, - lgopts[opt_idx].name, 1, - RTE_ACL_MAX_CATEGORIES); - } else if (strcmp(lgopts[opt_idx].name, - OPT_RUN_CATEGORIES) == 0) { - config.run_categories = get_ulong_opt(optarg, - lgopts[opt_idx].name, 1, - RTE_ACL_MAX_CATEGORIES); - } else if (strcmp(lgopts[opt_idx].name, OPT_ITER_NUM) == 0) { - config.iter_num = get_ulong_opt(optarg, - lgopts[opt_idx].name, 1, INT32_MAX); - } else if (strcmp(lgopts[opt_idx].name, OPT_VERBOSE) == 0) { - config.verbose = get_ulong_opt(optarg, - lgopts[opt_idx].name, DUMP_NONE, DUMP_MAX); - } else if (strcmp(lgopts[opt_idx].name, - OPT_SEARCH_ALG) == 0) { - get_alg_opt(optarg, lgopts[opt_idx].name); - } else if (strcmp(lgopts[opt_idx].name, OPT_IPV6) == 0) { - config.ipv6 = 1; - } - } - config.trace_sz = config.ipv6 ? sizeof(struct ipv6_5tuple) : - sizeof(struct ipv4_5tuple); - -} - -int -main(int argc, char **argv) -{ - int ret; - uint32_t lcore; - - ret = rte_eal_init(argc, argv); - if (ret < 0) - rte_panic("Cannot init EAL\n"); - - argc -= ret; - argv += ret; - - config.prgname = argv[0]; - - get_input_opts(argc, argv); - dump_config(stdout); - check_config(); - - acx_init(); - - if (config.trace_file != NULL) - tracef_init(); - - RTE_LCORE_FOREACH_SLAVE(lcore) - rte_eal_remote_launch(search_ip5tuples, NULL, lcore); - - search_ip5tuples(NULL); - - rte_eal_mp_wait_lcore(); - - rte_acl_free(config.acx); - return 0; -} diff --git a/dpdk/app/test-crypto-perf/Makefile b/dpdk/app/test-crypto-perf/Makefile new file mode 100644 index 00000000..c75d7ed1 --- /dev/null +++ b/dpdk/app/test-crypto-perf/Makefile @@ -0,0 +1,54 @@ +# BSD LICENSE +# +# Copyright(c) 2016-2017 Intel Corporation. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +include $(RTE_SDK)/mk/rte.vars.mk + + +APP = dpdk-test-crypto-perf + +CFLAGS += $(WERROR_FLAGS) + +# all source are stored in SRCS-y +SRCS-y := main.c +SRCS-y += cperf_ops.c +SRCS-y += cperf_options_parsing.c +SRCS-y += cperf_test_vectors.c +SRCS-y += cperf_test_throughput.c +SRCS-y += cperf_test_latency.c +SRCS-y += cperf_test_pmd_cyclecount.c +SRCS-y += cperf_test_verify.c +SRCS-y += cperf_test_vector_parsing.c +SRCS-y += cperf_test_common.c + +ifeq ($(CONFIG_RTE_LIBRTE_PMD_CRYPTO_SCHEDULER),y) +LDLIBS += -lrte_pmd_crypto_scheduler +endif + +include $(RTE_SDK)/mk/rte.app.mk diff --git a/dpdk/app/test-crypto-perf/cperf.h b/dpdk/app/test-crypto-perf/cperf.h new file mode 100644 index 00000000..c9f7f817 --- /dev/null +++ b/dpdk/app/test-crypto-perf/cperf.h @@ -0,0 +1,61 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _CPERF_ +#define _CPERF_ + +#include + +#include "cperf_ops.h" + +struct cperf_options; +struct cperf_test_vector; +struct cperf_op_fns; + +typedef void *(*cperf_constructor_t)( + struct rte_mempool *sess_mp, + uint8_t dev_id, + uint16_t qp_id, + const struct cperf_options *options, + const struct cperf_test_vector *t_vec, + const struct cperf_op_fns *op_fns); + +typedef int (*cperf_runner_t)(void *test_ctx); +typedef void (*cperf_destructor_t)(void *test_ctx); + +struct cperf_test { + cperf_constructor_t constructor; + cperf_runner_t runner; + cperf_destructor_t destructor; +}; + +#endif /* _CPERF_ */ diff --git a/dpdk/app/test-crypto-perf/cperf_ops.c b/dpdk/app/test-crypto-perf/cperf_ops.c new file mode 100644 index 00000000..23d30ca3 --- /dev/null +++ b/dpdk/app/test-crypto-perf/cperf_ops.c @@ -0,0 +1,651 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +#include "cperf_ops.h" +#include "cperf_test_vectors.h" + +static int +cperf_set_ops_null_cipher(struct rte_crypto_op **ops, + uint32_t src_buf_offset, uint32_t dst_buf_offset, + uint16_t nb_ops, struct rte_cryptodev_sym_session *sess, + const struct cperf_options *options, + const struct cperf_test_vector *test_vector __rte_unused, + uint16_t iv_offset __rte_unused) +{ + uint16_t i; + + for (i = 0; i < nb_ops; i++) { + struct rte_crypto_sym_op *sym_op = ops[i]->sym; + + ops[i]->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; + rte_crypto_op_attach_sym_session(ops[i], sess); + + sym_op->m_src = (struct rte_mbuf *)((uint8_t *)ops[i] + + src_buf_offset); + + /* Set dest mbuf to NULL if out-of-place (dst_buf_offset = 0) */ + if (dst_buf_offset == 0) + sym_op->m_dst = NULL; + else + sym_op->m_dst = (struct rte_mbuf *)((uint8_t *)ops[i] + + dst_buf_offset); + + /* cipher parameters */ + sym_op->cipher.data.length = options->test_buffer_size; + sym_op->cipher.data.offset = 0; + } + + return 0; +} + +static int +cperf_set_ops_null_auth(struct rte_crypto_op **ops, + uint32_t src_buf_offset, uint32_t dst_buf_offset, + uint16_t nb_ops, struct rte_cryptodev_sym_session *sess, + const struct cperf_options *options, + const struct cperf_test_vector *test_vector __rte_unused, + uint16_t iv_offset __rte_unused) +{ + uint16_t i; + + for (i = 0; i < nb_ops; i++) { + struct rte_crypto_sym_op *sym_op = ops[i]->sym; + + ops[i]->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; + rte_crypto_op_attach_sym_session(ops[i], sess); + + sym_op->m_src = (struct rte_mbuf *)((uint8_t *)ops[i] + + src_buf_offset); + + /* Set dest mbuf to NULL if out-of-place (dst_buf_offset = 0) */ + if (dst_buf_offset == 0) + sym_op->m_dst = NULL; + else + sym_op->m_dst = (struct rte_mbuf *)((uint8_t *)ops[i] + + dst_buf_offset); + + /* auth parameters */ + sym_op->auth.data.length = options->test_buffer_size; + sym_op->auth.data.offset = 0; + } + + return 0; +} + +static int +cperf_set_ops_cipher(struct rte_crypto_op **ops, + uint32_t src_buf_offset, uint32_t dst_buf_offset, + uint16_t nb_ops, struct rte_cryptodev_sym_session *sess, + const struct cperf_options *options, + const struct cperf_test_vector *test_vector, + uint16_t iv_offset) +{ + uint16_t i; + + for (i = 0; i < nb_ops; i++) { + struct rte_crypto_sym_op *sym_op = ops[i]->sym; + + ops[i]->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; + rte_crypto_op_attach_sym_session(ops[i], sess); + + sym_op->m_src = (struct rte_mbuf *)((uint8_t *)ops[i] + + src_buf_offset); + + /* Set dest mbuf to NULL if out-of-place (dst_buf_offset = 0) */ + if (dst_buf_offset == 0) + sym_op->m_dst = NULL; + else + sym_op->m_dst = (struct rte_mbuf *)((uint8_t *)ops[i] + + dst_buf_offset); + + /* cipher parameters */ + if (options->cipher_algo == RTE_CRYPTO_CIPHER_SNOW3G_UEA2 || + options->cipher_algo == RTE_CRYPTO_CIPHER_KASUMI_F8 || + options->cipher_algo == RTE_CRYPTO_CIPHER_ZUC_EEA3) + sym_op->cipher.data.length = options->test_buffer_size << 3; + else + sym_op->cipher.data.length = options->test_buffer_size; + + sym_op->cipher.data.offset = 0; + } + + if (options->test == CPERF_TEST_TYPE_VERIFY) { + for (i = 0; i < nb_ops; i++) { + uint8_t *iv_ptr = rte_crypto_op_ctod_offset(ops[i], + uint8_t *, iv_offset); + + memcpy(iv_ptr, test_vector->cipher_iv.data, + test_vector->cipher_iv.length); + + } + } + + return 0; +} + +static int +cperf_set_ops_auth(struct rte_crypto_op **ops, + uint32_t src_buf_offset, uint32_t dst_buf_offset, + uint16_t nb_ops, struct rte_cryptodev_sym_session *sess, + const struct cperf_options *options, + const struct cperf_test_vector *test_vector, + uint16_t iv_offset) +{ + uint16_t i; + + for (i = 0; i < nb_ops; i++) { + struct rte_crypto_sym_op *sym_op = ops[i]->sym; + + ops[i]->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; + rte_crypto_op_attach_sym_session(ops[i], sess); + + sym_op->m_src = (struct rte_mbuf *)((uint8_t *)ops[i] + + src_buf_offset); + + /* Set dest mbuf to NULL if out-of-place (dst_buf_offset = 0) */ + if (dst_buf_offset == 0) + sym_op->m_dst = NULL; + else + sym_op->m_dst = (struct rte_mbuf *)((uint8_t *)ops[i] + + dst_buf_offset); + + if (test_vector->auth_iv.length) { + uint8_t *iv_ptr = rte_crypto_op_ctod_offset(ops[i], + uint8_t *, + iv_offset); + memcpy(iv_ptr, test_vector->auth_iv.data, + test_vector->auth_iv.length); + } + + /* authentication parameters */ + if (options->auth_op == RTE_CRYPTO_AUTH_OP_VERIFY) { + sym_op->auth.digest.data = test_vector->digest.data; + sym_op->auth.digest.phys_addr = + test_vector->digest.phys_addr; + } else { + + uint32_t offset = options->test_buffer_size; + struct rte_mbuf *buf, *tbuf; + + if (options->out_of_place) { + buf = sym_op->m_dst; + } else { + tbuf = sym_op->m_src; + while ((tbuf->next != NULL) && + (offset >= tbuf->data_len)) { + offset -= tbuf->data_len; + tbuf = tbuf->next; + } + /* + * If there is not enough room in segment, + * place the digest in the next segment + */ + if ((tbuf->data_len - offset) < options->digest_sz) { + tbuf = tbuf->next; + offset = 0; + } + buf = tbuf; + } + + sym_op->auth.digest.data = rte_pktmbuf_mtod_offset(buf, + uint8_t *, offset); + sym_op->auth.digest.phys_addr = + rte_pktmbuf_iova_offset(buf, offset); + + } + + if (options->auth_algo == RTE_CRYPTO_AUTH_SNOW3G_UIA2 || + options->auth_algo == RTE_CRYPTO_AUTH_KASUMI_F9 || + options->auth_algo == RTE_CRYPTO_AUTH_ZUC_EIA3) + sym_op->auth.data.length = options->test_buffer_size << 3; + else + sym_op->auth.data.length = options->test_buffer_size; + + sym_op->auth.data.offset = 0; + } + + if (options->test == CPERF_TEST_TYPE_VERIFY) { + if (test_vector->auth_iv.length) { + for (i = 0; i < nb_ops; i++) { + uint8_t *iv_ptr = rte_crypto_op_ctod_offset(ops[i], + uint8_t *, iv_offset); + + memcpy(iv_ptr, test_vector->auth_iv.data, + test_vector->auth_iv.length); + } + } + } + return 0; +} + +static int +cperf_set_ops_cipher_auth(struct rte_crypto_op **ops, + uint32_t src_buf_offset, uint32_t dst_buf_offset, + uint16_t nb_ops, struct rte_cryptodev_sym_session *sess, + const struct cperf_options *options, + const struct cperf_test_vector *test_vector, + uint16_t iv_offset) +{ + uint16_t i; + + for (i = 0; i < nb_ops; i++) { + struct rte_crypto_sym_op *sym_op = ops[i]->sym; + + ops[i]->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; + rte_crypto_op_attach_sym_session(ops[i], sess); + + sym_op->m_src = (struct rte_mbuf *)((uint8_t *)ops[i] + + src_buf_offset); + + /* Set dest mbuf to NULL if out-of-place (dst_buf_offset = 0) */ + if (dst_buf_offset == 0) + sym_op->m_dst = NULL; + else + sym_op->m_dst = (struct rte_mbuf *)((uint8_t *)ops[i] + + dst_buf_offset); + + /* cipher parameters */ + if (options->cipher_algo == RTE_CRYPTO_CIPHER_SNOW3G_UEA2 || + options->cipher_algo == RTE_CRYPTO_CIPHER_KASUMI_F8 || + options->cipher_algo == RTE_CRYPTO_CIPHER_ZUC_EEA3) + sym_op->cipher.data.length = options->test_buffer_size << 3; + else + sym_op->cipher.data.length = options->test_buffer_size; + + sym_op->cipher.data.offset = 0; + + /* authentication parameters */ + if (options->auth_op == RTE_CRYPTO_AUTH_OP_VERIFY) { + sym_op->auth.digest.data = test_vector->digest.data; + sym_op->auth.digest.phys_addr = + test_vector->digest.phys_addr; + } else { + + uint32_t offset = options->test_buffer_size; + struct rte_mbuf *buf, *tbuf; + + if (options->out_of_place) { + buf = sym_op->m_dst; + } else { + tbuf = sym_op->m_src; + while ((tbuf->next != NULL) && + (offset >= tbuf->data_len)) { + offset -= tbuf->data_len; + tbuf = tbuf->next; + } + /* + * If there is not enough room in segment, + * place the digest in the next segment + */ + if ((tbuf->data_len - offset) < options->digest_sz) { + tbuf = tbuf->next; + offset = 0; + } + buf = tbuf; + } + + sym_op->auth.digest.data = rte_pktmbuf_mtod_offset(buf, + uint8_t *, offset); + sym_op->auth.digest.phys_addr = + rte_pktmbuf_iova_offset(buf, offset); + } + + if (options->auth_algo == RTE_CRYPTO_AUTH_SNOW3G_UIA2 || + options->auth_algo == RTE_CRYPTO_AUTH_KASUMI_F9 || + options->auth_algo == RTE_CRYPTO_AUTH_ZUC_EIA3) + sym_op->auth.data.length = options->test_buffer_size << 3; + else + sym_op->auth.data.length = options->test_buffer_size; + + sym_op->auth.data.offset = 0; + } + + if (options->test == CPERF_TEST_TYPE_VERIFY) { + for (i = 0; i < nb_ops; i++) { + uint8_t *iv_ptr = rte_crypto_op_ctod_offset(ops[i], + uint8_t *, iv_offset); + + memcpy(iv_ptr, test_vector->cipher_iv.data, + test_vector->cipher_iv.length); + if (test_vector->auth_iv.length) { + /* + * Copy IV after the crypto operation and + * the cipher IV + */ + iv_ptr += test_vector->cipher_iv.length; + memcpy(iv_ptr, test_vector->auth_iv.data, + test_vector->auth_iv.length); + } + } + + } + + return 0; +} + +static int +cperf_set_ops_aead(struct rte_crypto_op **ops, + uint32_t src_buf_offset, uint32_t dst_buf_offset, + uint16_t nb_ops, struct rte_cryptodev_sym_session *sess, + const struct cperf_options *options, + const struct cperf_test_vector *test_vector, + uint16_t iv_offset) +{ + uint16_t i; + /* AAD is placed after the IV */ + uint16_t aad_offset = iv_offset + + RTE_ALIGN_CEIL(test_vector->aead_iv.length, 16); + + for (i = 0; i < nb_ops; i++) { + struct rte_crypto_sym_op *sym_op = ops[i]->sym; + + ops[i]->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; + rte_crypto_op_attach_sym_session(ops[i], sess); + + sym_op->m_src = (struct rte_mbuf *)((uint8_t *)ops[i] + + src_buf_offset); + + /* Set dest mbuf to NULL if out-of-place (dst_buf_offset = 0) */ + if (dst_buf_offset == 0) + sym_op->m_dst = NULL; + else + sym_op->m_dst = (struct rte_mbuf *)((uint8_t *)ops[i] + + dst_buf_offset); + + /* AEAD parameters */ + sym_op->aead.data.length = options->test_buffer_size; + sym_op->aead.data.offset = 0; + + sym_op->aead.aad.data = rte_crypto_op_ctod_offset(ops[i], + uint8_t *, aad_offset); + sym_op->aead.aad.phys_addr = rte_crypto_op_ctophys_offset(ops[i], + aad_offset); + + if (options->aead_op == RTE_CRYPTO_AEAD_OP_DECRYPT) { + sym_op->aead.digest.data = test_vector->digest.data; + sym_op->aead.digest.phys_addr = + test_vector->digest.phys_addr; + } else { + + uint32_t offset = sym_op->aead.data.length + + sym_op->aead.data.offset; + struct rte_mbuf *buf, *tbuf; + + if (options->out_of_place) { + buf = sym_op->m_dst; + } else { + tbuf = sym_op->m_src; + while ((tbuf->next != NULL) && + (offset >= tbuf->data_len)) { + offset -= tbuf->data_len; + tbuf = tbuf->next; + } + /* + * If there is not enough room in segment, + * place the digest in the next segment + */ + if ((tbuf->data_len - offset) < options->digest_sz) { + tbuf = tbuf->next; + offset = 0; + } + buf = tbuf; + } + + sym_op->aead.digest.data = rte_pktmbuf_mtod_offset(buf, + uint8_t *, offset); + sym_op->aead.digest.phys_addr = + rte_pktmbuf_iova_offset(buf, offset); + } + } + + if (options->test == CPERF_TEST_TYPE_VERIFY) { + for (i = 0; i < nb_ops; i++) { + uint8_t *iv_ptr = rte_crypto_op_ctod_offset(ops[i], + uint8_t *, iv_offset); + + /* + * If doing AES-CCM, nonce is copied one byte + * after the start of IV field, and AAD is copied + * 18 bytes after the start of the AAD field. + */ + if (options->aead_algo == RTE_CRYPTO_AEAD_AES_CCM) { + memcpy(iv_ptr + 1, test_vector->aead_iv.data, + test_vector->aead_iv.length); + + memcpy(ops[i]->sym->aead.aad.data + 18, + test_vector->aad.data, + test_vector->aad.length); + } else { + memcpy(iv_ptr, test_vector->aead_iv.data, + test_vector->aead_iv.length); + + memcpy(ops[i]->sym->aead.aad.data, + test_vector->aad.data, + test_vector->aad.length); + } + } + } + + return 0; +} + +static struct rte_cryptodev_sym_session * +cperf_create_session(struct rte_mempool *sess_mp, + uint8_t dev_id, + const struct cperf_options *options, + const struct cperf_test_vector *test_vector, + uint16_t iv_offset) +{ + struct rte_crypto_sym_xform cipher_xform; + struct rte_crypto_sym_xform auth_xform; + struct rte_crypto_sym_xform aead_xform; + struct rte_cryptodev_sym_session *sess = NULL; + + sess = rte_cryptodev_sym_session_create(sess_mp); + /* + * cipher only + */ + if (options->op_type == CPERF_CIPHER_ONLY) { + cipher_xform.type = RTE_CRYPTO_SYM_XFORM_CIPHER; + cipher_xform.next = NULL; + cipher_xform.cipher.algo = options->cipher_algo; + cipher_xform.cipher.op = options->cipher_op; + cipher_xform.cipher.iv.offset = iv_offset; + + /* cipher different than null */ + if (options->cipher_algo != RTE_CRYPTO_CIPHER_NULL) { + cipher_xform.cipher.key.data = + test_vector->cipher_key.data; + cipher_xform.cipher.key.length = + test_vector->cipher_key.length; + cipher_xform.cipher.iv.length = + test_vector->cipher_iv.length; + } else { + cipher_xform.cipher.key.data = NULL; + cipher_xform.cipher.key.length = 0; + cipher_xform.cipher.iv.length = 0; + } + /* create crypto session */ + rte_cryptodev_sym_session_init(dev_id, sess, &cipher_xform, + sess_mp); + /* + * auth only + */ + } else if (options->op_type == CPERF_AUTH_ONLY) { + auth_xform.type = RTE_CRYPTO_SYM_XFORM_AUTH; + auth_xform.next = NULL; + auth_xform.auth.algo = options->auth_algo; + auth_xform.auth.op = options->auth_op; + + /* auth different than null */ + if (options->auth_algo != RTE_CRYPTO_AUTH_NULL) { + auth_xform.auth.digest_length = + options->digest_sz; + auth_xform.auth.key.length = + test_vector->auth_key.length; + auth_xform.auth.key.data = test_vector->auth_key.data; + auth_xform.auth.iv.length = + test_vector->auth_iv.length; + } else { + auth_xform.auth.digest_length = 0; + auth_xform.auth.key.length = 0; + auth_xform.auth.key.data = NULL; + auth_xform.auth.iv.length = 0; + } + /* create crypto session */ + rte_cryptodev_sym_session_init(dev_id, sess, &auth_xform, + sess_mp); + /* + * cipher and auth + */ + } else if (options->op_type == CPERF_CIPHER_THEN_AUTH + || options->op_type == CPERF_AUTH_THEN_CIPHER) { + /* + * cipher + */ + cipher_xform.type = RTE_CRYPTO_SYM_XFORM_CIPHER; + cipher_xform.next = NULL; + cipher_xform.cipher.algo = options->cipher_algo; + cipher_xform.cipher.op = options->cipher_op; + cipher_xform.cipher.iv.offset = iv_offset; + + /* cipher different than null */ + if (options->cipher_algo != RTE_CRYPTO_CIPHER_NULL) { + cipher_xform.cipher.key.data = + test_vector->cipher_key.data; + cipher_xform.cipher.key.length = + test_vector->cipher_key.length; + cipher_xform.cipher.iv.length = + test_vector->cipher_iv.length; + } else { + cipher_xform.cipher.key.data = NULL; + cipher_xform.cipher.key.length = 0; + cipher_xform.cipher.iv.length = 0; + } + + /* + * auth + */ + auth_xform.type = RTE_CRYPTO_SYM_XFORM_AUTH; + auth_xform.next = NULL; + auth_xform.auth.algo = options->auth_algo; + auth_xform.auth.op = options->auth_op; + + /* auth different than null */ + if (options->auth_algo != RTE_CRYPTO_AUTH_NULL) { + auth_xform.auth.digest_length = options->digest_sz; + auth_xform.auth.iv.length = test_vector->auth_iv.length; + auth_xform.auth.key.length = + test_vector->auth_key.length; + auth_xform.auth.key.data = + test_vector->auth_key.data; + } else { + auth_xform.auth.digest_length = 0; + auth_xform.auth.key.length = 0; + auth_xform.auth.key.data = NULL; + auth_xform.auth.iv.length = 0; + } + + /* cipher then auth */ + if (options->op_type == CPERF_CIPHER_THEN_AUTH) { + cipher_xform.next = &auth_xform; + /* create crypto session */ + rte_cryptodev_sym_session_init(dev_id, + sess, &cipher_xform, sess_mp); + } else { /* auth then cipher */ + auth_xform.next = &cipher_xform; + /* create crypto session */ + rte_cryptodev_sym_session_init(dev_id, + sess, &auth_xform, sess_mp); + } + } else { /* options->op_type == CPERF_AEAD */ + aead_xform.type = RTE_CRYPTO_SYM_XFORM_AEAD; + aead_xform.next = NULL; + aead_xform.aead.algo = options->aead_algo; + aead_xform.aead.op = options->aead_op; + aead_xform.aead.iv.offset = iv_offset; + + aead_xform.aead.key.data = + test_vector->aead_key.data; + aead_xform.aead.key.length = + test_vector->aead_key.length; + aead_xform.aead.iv.length = test_vector->aead_iv.length; + + aead_xform.aead.digest_length = options->digest_sz; + aead_xform.aead.aad_length = + options->aead_aad_sz; + + /* Create crypto session */ + rte_cryptodev_sym_session_init(dev_id, + sess, &aead_xform, sess_mp); + } + + return sess; +} + +int +cperf_get_op_functions(const struct cperf_options *options, + struct cperf_op_fns *op_fns) +{ + memset(op_fns, 0, sizeof(struct cperf_op_fns)); + + op_fns->sess_create = cperf_create_session; + + if (options->op_type == CPERF_AEAD) { + op_fns->populate_ops = cperf_set_ops_aead; + return 0; + } + + if (options->op_type == CPERF_AUTH_THEN_CIPHER + || options->op_type == CPERF_CIPHER_THEN_AUTH) { + op_fns->populate_ops = cperf_set_ops_cipher_auth; + return 0; + } + if (options->op_type == CPERF_AUTH_ONLY) { + if (options->auth_algo == RTE_CRYPTO_AUTH_NULL) + op_fns->populate_ops = cperf_set_ops_null_auth; + else + op_fns->populate_ops = cperf_set_ops_auth; + return 0; + } + if (options->op_type == CPERF_CIPHER_ONLY) { + if (options->cipher_algo == RTE_CRYPTO_CIPHER_NULL) + op_fns->populate_ops = cperf_set_ops_null_cipher; + else + op_fns->populate_ops = cperf_set_ops_cipher; + return 0; + } + + return -1; +} diff --git a/dpdk/app/test-crypto-perf/cperf_ops.h b/dpdk/app/test-crypto-perf/cperf_ops.h new file mode 100644 index 00000000..94951cc3 --- /dev/null +++ b/dpdk/app/test-crypto-perf/cperf_ops.h @@ -0,0 +1,65 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _CPERF_OPS_ +#define _CPERF_OPS_ + +#include + +#include "cperf.h" +#include "cperf_options.h" +#include "cperf_test_vectors.h" + + +typedef struct rte_cryptodev_sym_session *(*cperf_sessions_create_t)( + struct rte_mempool *sess_mp, + uint8_t dev_id, const struct cperf_options *options, + const struct cperf_test_vector *test_vector, + uint16_t iv_offset); + +typedef int (*cperf_populate_ops_t)(struct rte_crypto_op **ops, + uint32_t src_buf_offset, uint32_t dst_buf_offset, + uint16_t nb_ops, struct rte_cryptodev_sym_session *sess, + const struct cperf_options *options, + const struct cperf_test_vector *test_vector, + uint16_t iv_offset); + +struct cperf_op_fns { + cperf_sessions_create_t sess_create; + cperf_populate_ops_t populate_ops; +}; + +int +cperf_get_op_functions(const struct cperf_options *options, + struct cperf_op_fns *op_fns); + +#endif /* _CPERF_OPS_ */ diff --git a/dpdk/app/test-crypto-perf/cperf_options.h b/dpdk/app/test-crypto-perf/cperf_options.h new file mode 100644 index 00000000..da4fb47c --- /dev/null +++ b/dpdk/app/test-crypto-perf/cperf_options.h @@ -0,0 +1,140 @@ + +#ifndef _CPERF_OPTIONS_ +#define _CPERF_OPTIONS_ + +#include + +#define CPERF_PTEST_TYPE ("ptest") +#define CPERF_SILENT ("silent") + +#define CPERF_POOL_SIZE ("pool-sz") +#define CPERF_TOTAL_OPS ("total-ops") +#define CPERF_BURST_SIZE ("burst-sz") +#define CPERF_BUFFER_SIZE ("buffer-sz") +#define CPERF_SEGMENT_SIZE ("segment-sz") +#define CPERF_DESC_NB ("desc-nb") + +#define CPERF_DEVTYPE ("devtype") +#define CPERF_OPTYPE ("optype") +#define CPERF_SESSIONLESS ("sessionless") +#define CPERF_OUT_OF_PLACE ("out-of-place") +#define CPERF_TEST_FILE ("test-file") +#define CPERF_TEST_NAME ("test-name") + +#define CPERF_CIPHER_ALGO ("cipher-algo") +#define CPERF_CIPHER_OP ("cipher-op") +#define CPERF_CIPHER_KEY_SZ ("cipher-key-sz") +#define CPERF_CIPHER_IV_SZ ("cipher-iv-sz") + +#define CPERF_AUTH_ALGO ("auth-algo") +#define CPERF_AUTH_OP ("auth-op") +#define CPERF_AUTH_KEY_SZ ("auth-key-sz") +#define CPERF_AUTH_IV_SZ ("auth-iv-sz") + +#define CPERF_AEAD_ALGO ("aead-algo") +#define CPERF_AEAD_OP ("aead-op") +#define CPERF_AEAD_KEY_SZ ("aead-key-sz") +#define CPERF_AEAD_IV_SZ ("aead-iv-sz") +#define CPERF_AEAD_AAD_SZ ("aead-aad-sz") + +#define CPERF_DIGEST_SZ ("digest-sz") + +#define CPERF_CSV ("csv-friendly") + +/* benchmark-specific options */ +#define CPERF_PMDCC_DELAY_MS ("pmd-cyclecount-delay-ms") + +#define MAX_LIST 32 + +enum cperf_perf_test_type { + CPERF_TEST_TYPE_THROUGHPUT, + CPERF_TEST_TYPE_LATENCY, + CPERF_TEST_TYPE_VERIFY, + CPERF_TEST_TYPE_PMDCC +}; + + +extern const char *cperf_test_type_strs[]; + +enum cperf_op_type { + CPERF_CIPHER_ONLY = 1, + CPERF_AUTH_ONLY, + CPERF_CIPHER_THEN_AUTH, + CPERF_AUTH_THEN_CIPHER, + CPERF_AEAD +}; + +extern const char *cperf_op_type_strs[]; + +struct cperf_options { + enum cperf_perf_test_type test; + + uint32_t pool_sz; + uint32_t total_ops; + uint32_t segment_sz; + uint32_t test_buffer_size; + uint32_t nb_descriptors; + uint16_t nb_qps; + + uint32_t sessionless:1; + uint32_t out_of_place:1; + uint32_t silent:1; + uint32_t csv:1; + + enum rte_crypto_cipher_algorithm cipher_algo; + enum rte_crypto_cipher_operation cipher_op; + + uint16_t cipher_key_sz; + uint16_t cipher_iv_sz; + + enum rte_crypto_auth_algorithm auth_algo; + enum rte_crypto_auth_operation auth_op; + + uint16_t auth_key_sz; + uint16_t auth_iv_sz; + + enum rte_crypto_aead_algorithm aead_algo; + enum rte_crypto_aead_operation aead_op; + + uint16_t aead_key_sz; + uint16_t aead_iv_sz; + uint16_t aead_aad_sz; + + uint16_t digest_sz; + + char device_type[RTE_CRYPTODEV_NAME_LEN]; + enum cperf_op_type op_type; + + char *test_file; + char *test_name; + + uint32_t buffer_size_list[MAX_LIST]; + uint8_t buffer_size_count; + uint32_t max_buffer_size; + uint32_t min_buffer_size; + uint32_t inc_buffer_size; + + uint32_t burst_size_list[MAX_LIST]; + uint8_t burst_size_count; + uint32_t max_burst_size; + uint32_t min_burst_size; + uint32_t inc_burst_size; + + /* pmd-cyclecount specific options */ + uint32_t pmdcc_delay; +}; + +void +cperf_options_default(struct cperf_options *options); + +int +cperf_options_parse(struct cperf_options *options, + int argc, char **argv); + +int +cperf_options_check(struct cperf_options *options); + +void +cperf_options_dump(struct cperf_options *options); + +#endif diff --git a/dpdk/app/test-crypto-perf/cperf_options_parsing.c b/dpdk/app/test-crypto-perf/cperf_options_parsing.c new file mode 100644 index 00000000..ad43e84c --- /dev/null +++ b/dpdk/app/test-crypto-perf/cperf_options_parsing.c @@ -0,0 +1,1138 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include + +#include +#include + +#include "cperf_options.h" + +#define AES_BLOCK_SIZE 16 +#define DES_BLOCK_SIZE 8 + +struct name_id_map { + const char *name; + uint32_t id; +}; + +static void +usage(char *progname) +{ + printf("%s [EAL options] --\n" + " --silent: disable options dump\n" + " --ptest throughput / latency / verify / pmd-cycleount :" + " set test type\n" + " --pool_sz N: set the number of crypto ops/mbufs allocated\n" + " --total-ops N: set the number of total operations performed\n" + " --burst-sz N: set the number of packets per burst\n" + " --buffer-sz N: set the size of a single packet\n" + " --segment-sz N: set the size of the segment to use\n" + " --desc-nb N: set number of descriptors for each crypto device\n" + " --devtype TYPE: set crypto device type to use\n" + " --optype cipher-only / auth-only / cipher-then-auth /\n" + " auth-then-cipher / aead : set operation type\n" + " --sessionless: enable session-less crypto operations\n" + " --out-of-place: enable out-of-place crypto operations\n" + " --test-file NAME: set the test vector file path\n" + " --test-name NAME: set specific test name section in test file\n" + " --cipher-algo ALGO: set cipher algorithm\n" + " --cipher-op encrypt / decrypt: set the cipher operation\n" + " --cipher-key-sz N: set the cipher key size\n" + " --cipher-iv-sz N: set the cipher IV size\n" + " --auth-algo ALGO: set auth algorithm\n" + " --auth-op generate / verify: set the auth operation\n" + " --auth-key-sz N: set the auth key size\n" + " --auth-iv-sz N: set the auth IV size\n" + " --aead-algo ALGO: set AEAD algorithm\n" + " --aead-op encrypt / decrypt: set the AEAD operation\n" + " --aead-key-sz N: set the AEAD key size\n" + " --aead-iv-sz N: set the AEAD IV size\n" + " --aead-aad-sz N: set the AEAD AAD size\n" + " --digest-sz N: set the digest size\n" + " --pmd-cyclecount-delay-ms N: set delay between enqueue\n" + " and dequeue in pmd-cyclecount benchmarking mode\n" + " --csv-friendly: enable test result output CSV friendly\n" + " -h: prints this help\n", + progname); +} + +static int +get_str_key_id_mapping(struct name_id_map *map, unsigned int map_len, + const char *str_key) +{ + unsigned int i; + + for (i = 0; i < map_len; i++) { + + if (strcmp(str_key, map[i].name) == 0) + return map[i].id; + } + + return -1; +} + +static int +parse_cperf_test_type(struct cperf_options *opts, const char *arg) +{ + struct name_id_map cperftest_namemap[] = { + { + cperf_test_type_strs[CPERF_TEST_TYPE_THROUGHPUT], + CPERF_TEST_TYPE_THROUGHPUT + }, + { + cperf_test_type_strs[CPERF_TEST_TYPE_VERIFY], + CPERF_TEST_TYPE_VERIFY + }, + { + cperf_test_type_strs[CPERF_TEST_TYPE_LATENCY], + CPERF_TEST_TYPE_LATENCY + }, + { + cperf_test_type_strs[CPERF_TEST_TYPE_PMDCC], + CPERF_TEST_TYPE_PMDCC + } + }; + + int id = get_str_key_id_mapping( + (struct name_id_map *)cperftest_namemap, + RTE_DIM(cperftest_namemap), arg); + if (id < 0) { + RTE_LOG(ERR, USER1, "failed to parse test type"); + return -1; + } + + opts->test = (enum cperf_perf_test_type)id; + + return 0; +} + +static int +parse_uint32_t(uint32_t *value, const char *arg) +{ + char *end = NULL; + unsigned long n = strtoul(arg, &end, 10); + + if ((optarg[0] == '\0') || (end == NULL) || (*end != '\0')) + return -1; + + if (n > UINT32_MAX) + return -ERANGE; + + *value = (uint32_t) n; + + return 0; +} + +static int +parse_uint16_t(uint16_t *value, const char *arg) +{ + uint32_t val = 0; + int ret = parse_uint32_t(&val, arg); + + if (ret < 0) + return ret; + + if (val > UINT16_MAX) + return -ERANGE; + + *value = (uint16_t) val; + + return 0; +} + +static int +parse_range(const char *arg, uint32_t *min, uint32_t *max, uint32_t *inc) +{ + char *token; + uint32_t number; + + char *copy_arg = strdup(arg); + + if (copy_arg == NULL) + return -1; + + errno = 0; + token = strtok(copy_arg, ":"); + + /* Parse minimum value */ + if (token != NULL) { + number = strtoul(token, NULL, 10); + + if (errno == EINVAL || errno == ERANGE || + number == 0) + goto err_range; + + *min = number; + } else + goto err_range; + + token = strtok(NULL, ":"); + + /* Parse increment value */ + if (token != NULL) { + number = strtoul(token, NULL, 10); + + if (errno == EINVAL || errno == ERANGE || + number == 0) + goto err_range; + + *inc = number; + } else + goto err_range; + + token = strtok(NULL, ":"); + + /* Parse maximum value */ + if (token != NULL) { + number = strtoul(token, NULL, 10); + + if (errno == EINVAL || errno == ERANGE || + number == 0 || + number < *min) + goto err_range; + + *max = number; + } else + goto err_range; + + if (strtok(NULL, ":") != NULL) + goto err_range; + + free(copy_arg); + return 0; + +err_range: + free(copy_arg); + return -1; +} + +static int +parse_list(const char *arg, uint32_t *list, uint32_t *min, uint32_t *max) +{ + char *token; + uint32_t number; + uint8_t count = 0; + + char *copy_arg = strdup(arg); + + if (copy_arg == NULL) + return -1; + + errno = 0; + token = strtok(copy_arg, ","); + + /* Parse first value */ + if (token != NULL) { + number = strtoul(token, NULL, 10); + + if (errno == EINVAL || errno == ERANGE || + number == 0) + goto err_list; + + list[count++] = number; + *min = number; + *max = number; + } else + goto err_list; + + token = strtok(NULL, ","); + + while (token != NULL) { + if (count == MAX_LIST) { + RTE_LOG(WARNING, USER1, "Using only the first %u sizes\n", + MAX_LIST); + break; + } + + number = strtoul(token, NULL, 10); + + if (errno == EINVAL || errno == ERANGE || + number == 0) + goto err_list; + + list[count++] = number; + + if (number < *min) + *min = number; + if (number > *max) + *max = number; + + token = strtok(NULL, ","); + } + + free(copy_arg); + return count; + +err_list: + free(copy_arg); + return -1; +} + +static int +parse_total_ops(struct cperf_options *opts, const char *arg) +{ + int ret = parse_uint32_t(&opts->total_ops, arg); + + if (ret) + RTE_LOG(ERR, USER1, "failed to parse total operations count\n"); + + if (opts->total_ops == 0) { + RTE_LOG(ERR, USER1, + "invalid total operations count number specified\n"); + return -1; + } + + return ret; +} + +static int +parse_pool_sz(struct cperf_options *opts, const char *arg) +{ + int ret = parse_uint32_t(&opts->pool_sz, arg); + + if (ret) + RTE_LOG(ERR, USER1, "failed to parse pool size"); + return ret; +} + +static int +parse_burst_sz(struct cperf_options *opts, const char *arg) +{ + int ret; + + /* Try parsing the argument as a range, if it fails, parse it as a list */ + if (parse_range(arg, &opts->min_burst_size, &opts->max_burst_size, + &opts->inc_burst_size) < 0) { + ret = parse_list(arg, opts->burst_size_list, + &opts->min_burst_size, + &opts->max_burst_size); + if (ret < 0) { + RTE_LOG(ERR, USER1, "failed to parse burst size/s\n"); + return -1; + } + opts->burst_size_count = ret; + } + + return 0; +} + +static int +parse_buffer_sz(struct cperf_options *opts, const char *arg) +{ + int ret; + + /* Try parsing the argument as a range, if it fails, parse it as a list */ + if (parse_range(arg, &opts->min_buffer_size, &opts->max_buffer_size, + &opts->inc_buffer_size) < 0) { + ret = parse_list(arg, opts->buffer_size_list, + &opts->min_buffer_size, + &opts->max_buffer_size); + if (ret < 0) { + RTE_LOG(ERR, USER1, "failed to parse buffer size/s\n"); + return -1; + } + opts->buffer_size_count = ret; + } + + return 0; +} + +static int +parse_segment_sz(struct cperf_options *opts, const char *arg) +{ + int ret = parse_uint32_t(&opts->segment_sz, arg); + + if (ret) { + RTE_LOG(ERR, USER1, "failed to parse segment size\n"); + return -1; + } + + if (opts->segment_sz == 0) { + RTE_LOG(ERR, USER1, "Segment size has to be bigger than 0\n"); + return -1; + } + + return 0; +} + +static int +parse_desc_nb(struct cperf_options *opts, const char *arg) +{ + int ret = parse_uint32_t(&opts->nb_descriptors, arg); + + if (ret) { + RTE_LOG(ERR, USER1, "failed to parse descriptors number\n"); + return -1; + } + + if (opts->nb_descriptors == 0) { + RTE_LOG(ERR, USER1, "invalid descriptors number specified\n"); + return -1; + } + + return 0; +} + +static int +parse_device_type(struct cperf_options *opts, const char *arg) +{ + if (strlen(arg) > (sizeof(opts->device_type) - 1)) + return -1; + + strncpy(opts->device_type, arg, sizeof(opts->device_type) - 1); + *(opts->device_type + sizeof(opts->device_type) - 1) = '\0'; + + return 0; +} + +static int +parse_op_type(struct cperf_options *opts, const char *arg) +{ + struct name_id_map optype_namemap[] = { + { + cperf_op_type_strs[CPERF_CIPHER_ONLY], + CPERF_CIPHER_ONLY + }, + { + cperf_op_type_strs[CPERF_AUTH_ONLY], + CPERF_AUTH_ONLY + }, + { + cperf_op_type_strs[CPERF_CIPHER_THEN_AUTH], + CPERF_CIPHER_THEN_AUTH + }, + { + cperf_op_type_strs[CPERF_AUTH_THEN_CIPHER], + CPERF_AUTH_THEN_CIPHER + }, + { + cperf_op_type_strs[CPERF_AEAD], + CPERF_AEAD + } + }; + + int id = get_str_key_id_mapping(optype_namemap, + RTE_DIM(optype_namemap), arg); + if (id < 0) { + RTE_LOG(ERR, USER1, "invalid opt type specified\n"); + return -1; + } + + opts->op_type = (enum cperf_op_type)id; + + return 0; +} + +static int +parse_sessionless(struct cperf_options *opts, + const char *arg __rte_unused) +{ + opts->sessionless = 1; + return 0; +} + +static int +parse_out_of_place(struct cperf_options *opts, + const char *arg __rte_unused) +{ + opts->out_of_place = 1; + return 0; +} + +static int +parse_test_file(struct cperf_options *opts, + const char *arg) +{ + opts->test_file = strdup(arg); + if (access(opts->test_file, F_OK) != -1) + return 0; + RTE_LOG(ERR, USER1, "Test vector file doesn't exist\n"); + + return -1; +} + +static int +parse_test_name(struct cperf_options *opts, + const char *arg) +{ + char *test_name = (char *) rte_zmalloc(NULL, + sizeof(char) * (strlen(arg) + 3), 0); + snprintf(test_name, strlen(arg) + 3, "[%s]", arg); + opts->test_name = test_name; + + return 0; +} + +static int +parse_silent(struct cperf_options *opts, + const char *arg __rte_unused) +{ + opts->silent = 1; + + return 0; +} + +static int +parse_cipher_algo(struct cperf_options *opts, const char *arg) +{ + + enum rte_crypto_cipher_algorithm cipher_algo; + + if (rte_cryptodev_get_cipher_algo_enum(&cipher_algo, arg) < 0) { + RTE_LOG(ERR, USER1, "Invalid cipher algorithm specified\n"); + return -1; + } + + opts->cipher_algo = cipher_algo; + + return 0; +} + +static int +parse_cipher_op(struct cperf_options *opts, const char *arg) +{ + struct name_id_map cipher_op_namemap[] = { + { + rte_crypto_cipher_operation_strings + [RTE_CRYPTO_CIPHER_OP_ENCRYPT], + RTE_CRYPTO_CIPHER_OP_ENCRYPT }, + { + rte_crypto_cipher_operation_strings + [RTE_CRYPTO_CIPHER_OP_DECRYPT], + RTE_CRYPTO_CIPHER_OP_DECRYPT + } + }; + + int id = get_str_key_id_mapping(cipher_op_namemap, + RTE_DIM(cipher_op_namemap), arg); + if (id < 0) { + RTE_LOG(ERR, USER1, "Invalid cipher operation specified\n"); + return -1; + } + + opts->cipher_op = (enum rte_crypto_cipher_operation)id; + + return 0; +} + +static int +parse_cipher_key_sz(struct cperf_options *opts, const char *arg) +{ + return parse_uint16_t(&opts->cipher_key_sz, arg); +} + +static int +parse_cipher_iv_sz(struct cperf_options *opts, const char *arg) +{ + return parse_uint16_t(&opts->cipher_iv_sz, arg); +} + +static int +parse_auth_algo(struct cperf_options *opts, const char *arg) +{ + enum rte_crypto_auth_algorithm auth_algo; + + if (rte_cryptodev_get_auth_algo_enum(&auth_algo, arg) < 0) { + RTE_LOG(ERR, USER1, "Invalid authentication algorithm specified\n"); + return -1; + } + + opts->auth_algo = auth_algo; + + return 0; +} + +static int +parse_auth_op(struct cperf_options *opts, const char *arg) +{ + struct name_id_map auth_op_namemap[] = { + { + rte_crypto_auth_operation_strings + [RTE_CRYPTO_AUTH_OP_GENERATE], + RTE_CRYPTO_AUTH_OP_GENERATE }, + { + rte_crypto_auth_operation_strings + [RTE_CRYPTO_AUTH_OP_VERIFY], + RTE_CRYPTO_AUTH_OP_VERIFY + } + }; + + int id = get_str_key_id_mapping(auth_op_namemap, + RTE_DIM(auth_op_namemap), arg); + if (id < 0) { + RTE_LOG(ERR, USER1, "invalid authentication operation specified" + "\n"); + return -1; + } + + opts->auth_op = (enum rte_crypto_auth_operation)id; + + return 0; +} + +static int +parse_auth_key_sz(struct cperf_options *opts, const char *arg) +{ + return parse_uint16_t(&opts->auth_key_sz, arg); +} + +static int +parse_digest_sz(struct cperf_options *opts, const char *arg) +{ + return parse_uint16_t(&opts->digest_sz, arg); +} + +static int +parse_auth_iv_sz(struct cperf_options *opts, const char *arg) +{ + return parse_uint16_t(&opts->auth_iv_sz, arg); +} + +static int +parse_aead_algo(struct cperf_options *opts, const char *arg) +{ + enum rte_crypto_aead_algorithm aead_algo; + + if (rte_cryptodev_get_aead_algo_enum(&aead_algo, arg) < 0) { + RTE_LOG(ERR, USER1, "Invalid AEAD algorithm specified\n"); + return -1; + } + + opts->aead_algo = aead_algo; + + return 0; +} + +static int +parse_aead_op(struct cperf_options *opts, const char *arg) +{ + struct name_id_map aead_op_namemap[] = { + { + rte_crypto_aead_operation_strings + [RTE_CRYPTO_AEAD_OP_ENCRYPT], + RTE_CRYPTO_AEAD_OP_ENCRYPT }, + { + rte_crypto_aead_operation_strings + [RTE_CRYPTO_AEAD_OP_DECRYPT], + RTE_CRYPTO_AEAD_OP_DECRYPT + } + }; + + int id = get_str_key_id_mapping(aead_op_namemap, + RTE_DIM(aead_op_namemap), arg); + if (id < 0) { + RTE_LOG(ERR, USER1, "invalid AEAD operation specified" + "\n"); + return -1; + } + + opts->aead_op = (enum rte_crypto_aead_operation)id; + + return 0; +} + +static int +parse_aead_key_sz(struct cperf_options *opts, const char *arg) +{ + return parse_uint16_t(&opts->aead_key_sz, arg); +} + +static int +parse_aead_iv_sz(struct cperf_options *opts, const char *arg) +{ + return parse_uint16_t(&opts->aead_iv_sz, arg); +} + +static int +parse_aead_aad_sz(struct cperf_options *opts, const char *arg) +{ + return parse_uint16_t(&opts->aead_aad_sz, arg); +} + +static int +parse_csv_friendly(struct cperf_options *opts, const char *arg __rte_unused) +{ + opts->csv = 1; + opts->silent = 1; + return 0; +} + +static int +parse_pmd_cyclecount_delay_ms(struct cperf_options *opts, + const char *arg) +{ + int ret = parse_uint32_t(&opts->pmdcc_delay, arg); + + if (ret) { + RTE_LOG(ERR, USER1, "failed to parse pmd-cyclecount delay\n"); + return -1; + } + + return 0; +} + +typedef int (*option_parser_t)(struct cperf_options *opts, + const char *arg); + +struct long_opt_parser { + const char *lgopt_name; + option_parser_t parser_fn; + +}; + +static struct option lgopts[] = { + + { CPERF_PTEST_TYPE, required_argument, 0, 0 }, + + { CPERF_POOL_SIZE, required_argument, 0, 0 }, + { CPERF_TOTAL_OPS, required_argument, 0, 0 }, + { CPERF_BURST_SIZE, required_argument, 0, 0 }, + { CPERF_BUFFER_SIZE, required_argument, 0, 0 }, + { CPERF_SEGMENT_SIZE, required_argument, 0, 0 }, + { CPERF_DESC_NB, required_argument, 0, 0 }, + + { CPERF_DEVTYPE, required_argument, 0, 0 }, + { CPERF_OPTYPE, required_argument, 0, 0 }, + + { CPERF_SILENT, no_argument, 0, 0 }, + { CPERF_SESSIONLESS, no_argument, 0, 0 }, + { CPERF_OUT_OF_PLACE, no_argument, 0, 0 }, + { CPERF_TEST_FILE, required_argument, 0, 0 }, + { CPERF_TEST_NAME, required_argument, 0, 0 }, + + { CPERF_CIPHER_ALGO, required_argument, 0, 0 }, + { CPERF_CIPHER_OP, required_argument, 0, 0 }, + + { CPERF_CIPHER_KEY_SZ, required_argument, 0, 0 }, + { CPERF_CIPHER_IV_SZ, required_argument, 0, 0 }, + + { CPERF_AUTH_ALGO, required_argument, 0, 0 }, + { CPERF_AUTH_OP, required_argument, 0, 0 }, + + { CPERF_AUTH_KEY_SZ, required_argument, 0, 0 }, + { CPERF_AUTH_IV_SZ, required_argument, 0, 0 }, + + { CPERF_AEAD_ALGO, required_argument, 0, 0 }, + { CPERF_AEAD_OP, required_argument, 0, 0 }, + + { CPERF_AEAD_KEY_SZ, required_argument, 0, 0 }, + { CPERF_AEAD_AAD_SZ, required_argument, 0, 0 }, + { CPERF_AEAD_IV_SZ, required_argument, 0, 0 }, + + { CPERF_DIGEST_SZ, required_argument, 0, 0 }, + + { CPERF_CSV, no_argument, 0, 0}, + + { CPERF_PMDCC_DELAY_MS, required_argument, 0, 0 }, + + { NULL, 0, 0, 0 } +}; + +void +cperf_options_default(struct cperf_options *opts) +{ + opts->test = CPERF_TEST_TYPE_THROUGHPUT; + + opts->pool_sz = 8192; + opts->total_ops = 10000000; + opts->nb_descriptors = 2048; + + opts->buffer_size_list[0] = 64; + opts->buffer_size_count = 1; + opts->max_buffer_size = 64; + opts->min_buffer_size = 64; + opts->inc_buffer_size = 0; + + opts->burst_size_list[0] = 32; + opts->burst_size_count = 1; + opts->max_burst_size = 32; + opts->min_burst_size = 32; + opts->inc_burst_size = 0; + + /* + * Will be parsed from command line or set to + * maximum buffer size + digest, later + */ + opts->segment_sz = 0; + + strncpy(opts->device_type, "crypto_aesni_mb", + sizeof(opts->device_type)); + opts->nb_qps = 1; + + opts->op_type = CPERF_CIPHER_THEN_AUTH; + + opts->silent = 0; + opts->test_file = NULL; + opts->test_name = NULL; + opts->sessionless = 0; + opts->out_of_place = 0; + opts->csv = 0; + + opts->cipher_algo = RTE_CRYPTO_CIPHER_AES_CBC; + opts->cipher_op = RTE_CRYPTO_CIPHER_OP_ENCRYPT; + opts->cipher_key_sz = 16; + opts->cipher_iv_sz = 16; + + opts->auth_algo = RTE_CRYPTO_AUTH_SHA1_HMAC; + opts->auth_op = RTE_CRYPTO_AUTH_OP_GENERATE; + + opts->auth_key_sz = 64; + opts->auth_iv_sz = 0; + + opts->aead_key_sz = 0; + opts->aead_iv_sz = 0; + opts->aead_aad_sz = 0; + + opts->digest_sz = 12; + + opts->pmdcc_delay = 0; +} + +static int +cperf_opts_parse_long(int opt_idx, struct cperf_options *opts) +{ + struct long_opt_parser parsermap[] = { + { CPERF_PTEST_TYPE, parse_cperf_test_type }, + { CPERF_SILENT, parse_silent }, + { CPERF_POOL_SIZE, parse_pool_sz }, + { CPERF_TOTAL_OPS, parse_total_ops }, + { CPERF_BURST_SIZE, parse_burst_sz }, + { CPERF_BUFFER_SIZE, parse_buffer_sz }, + { CPERF_SEGMENT_SIZE, parse_segment_sz }, + { CPERF_DESC_NB, parse_desc_nb }, + { CPERF_DEVTYPE, parse_device_type }, + { CPERF_OPTYPE, parse_op_type }, + { CPERF_SESSIONLESS, parse_sessionless }, + { CPERF_OUT_OF_PLACE, parse_out_of_place }, + { CPERF_TEST_FILE, parse_test_file }, + { CPERF_TEST_NAME, parse_test_name }, + { CPERF_CIPHER_ALGO, parse_cipher_algo }, + { CPERF_CIPHER_OP, parse_cipher_op }, + { CPERF_CIPHER_KEY_SZ, parse_cipher_key_sz }, + { CPERF_CIPHER_IV_SZ, parse_cipher_iv_sz }, + { CPERF_AUTH_ALGO, parse_auth_algo }, + { CPERF_AUTH_OP, parse_auth_op }, + { CPERF_AUTH_KEY_SZ, parse_auth_key_sz }, + { CPERF_AUTH_IV_SZ, parse_auth_iv_sz }, + { CPERF_AEAD_ALGO, parse_aead_algo }, + { CPERF_AEAD_OP, parse_aead_op }, + { CPERF_AEAD_KEY_SZ, parse_aead_key_sz }, + { CPERF_AEAD_IV_SZ, parse_aead_iv_sz }, + { CPERF_AEAD_AAD_SZ, parse_aead_aad_sz }, + { CPERF_DIGEST_SZ, parse_digest_sz }, + { CPERF_CSV, parse_csv_friendly}, + { CPERF_PMDCC_DELAY_MS, parse_pmd_cyclecount_delay_ms}, + }; + unsigned int i; + + for (i = 0; i < RTE_DIM(parsermap); i++) { + if (strncmp(lgopts[opt_idx].name, parsermap[i].lgopt_name, + strlen(lgopts[opt_idx].name)) == 0) + return parsermap[i].parser_fn(opts, optarg); + } + + return -EINVAL; +} + +int +cperf_options_parse(struct cperf_options *options, int argc, char **argv) +{ + int opt, retval, opt_idx; + + while ((opt = getopt_long(argc, argv, "h", lgopts, &opt_idx)) != EOF) { + switch (opt) { + case 'h': + usage(argv[0]); + rte_exit(EXIT_SUCCESS, "Displayed help\n"); + break; + /* long options */ + case 0: + retval = cperf_opts_parse_long(opt_idx, options); + if (retval != 0) + return retval; + + break; + + default: + usage(argv[0]); + return -EINVAL; + } + } + + return 0; +} + +static int +check_cipher_buffer_length(struct cperf_options *options) +{ + uint32_t buffer_size, buffer_size_idx = 0; + + if (options->cipher_algo == RTE_CRYPTO_CIPHER_AES_CBC || + options->cipher_algo == RTE_CRYPTO_CIPHER_AES_ECB) { + if (options->inc_buffer_size != 0) + buffer_size = options->min_buffer_size; + else + buffer_size = options->buffer_size_list[0]; + + while (buffer_size <= options->max_buffer_size) { + if ((buffer_size % AES_BLOCK_SIZE) != 0) { + RTE_LOG(ERR, USER1, "Some of the buffer sizes are " + "not suitable for the algorithm selected\n"); + return -EINVAL; + } + + if (options->inc_buffer_size != 0) + buffer_size += options->inc_buffer_size; + else { + if (++buffer_size_idx == options->buffer_size_count) + break; + buffer_size = options->buffer_size_list[buffer_size_idx]; + } + + } + } + + if (options->cipher_algo == RTE_CRYPTO_CIPHER_DES_CBC || + options->cipher_algo == RTE_CRYPTO_CIPHER_3DES_CBC || + options->cipher_algo == RTE_CRYPTO_CIPHER_3DES_ECB) { + if (options->inc_buffer_size != 0) + buffer_size = options->min_buffer_size; + else + buffer_size = options->buffer_size_list[0]; + + while (buffer_size <= options->max_buffer_size) { + if ((buffer_size % DES_BLOCK_SIZE) != 0) { + RTE_LOG(ERR, USER1, "Some of the buffer sizes are " + "not suitable for the algorithm selected\n"); + return -EINVAL; + } + + if (options->inc_buffer_size != 0) + buffer_size += options->inc_buffer_size; + else { + if (++buffer_size_idx == options->buffer_size_count) + break; + buffer_size = options->buffer_size_list[buffer_size_idx]; + } + + } + } + + return 0; +} + +int +cperf_options_check(struct cperf_options *options) +{ + if (options->op_type == CPERF_CIPHER_ONLY) + options->digest_sz = 0; + + /* + * If segment size is not set, assume only one segment, + * big enough to contain the largest buffer and the digest + */ + if (options->segment_sz == 0) + options->segment_sz = options->max_buffer_size + + options->digest_sz; + + if (options->segment_sz < options->digest_sz) { + RTE_LOG(ERR, USER1, + "Segment size should be at least " + "the size of the digest\n"); + return -EINVAL; + } + + if (options->test == CPERF_TEST_TYPE_VERIFY && + options->test_file == NULL) { + RTE_LOG(ERR, USER1, "Define path to the file with test" + " vectors.\n"); + return -EINVAL; + } + + if (options->test == CPERF_TEST_TYPE_VERIFY && + options->op_type != CPERF_CIPHER_ONLY && + options->test_name == NULL) { + RTE_LOG(ERR, USER1, "Define test name to get the correct digest" + " from the test vectors.\n"); + return -EINVAL; + } + + if (options->test_name != NULL && options->test_file == NULL) { + RTE_LOG(ERR, USER1, "Define path to the file with test" + " vectors.\n"); + return -EINVAL; + } + + if (options->auth_op == RTE_CRYPTO_AUTH_OP_VERIFY && + options->test_file == NULL) { + RTE_LOG(ERR, USER1, "Define path to the file with test" + " vectors.\n"); + return -EINVAL; + } + + if (options->test == CPERF_TEST_TYPE_VERIFY && + (options->inc_buffer_size != 0 || + options->buffer_size_count > 1)) { + RTE_LOG(ERR, USER1, "Only one buffer size is allowed when " + "using the verify test.\n"); + return -EINVAL; + } + + if (options->test == CPERF_TEST_TYPE_VERIFY && + (options->inc_burst_size != 0 || + options->burst_size_count > 1)) { + RTE_LOG(ERR, USER1, "Only one burst size is allowed when " + "using the verify test.\n"); + return -EINVAL; + } + + if (options->test == CPERF_TEST_TYPE_PMDCC && + options->pool_sz < options->nb_descriptors) { + RTE_LOG(ERR, USER1, "For pmd cyclecount benchmarks, pool size " + "must be equal or greater than the number of " + "cryptodev descriptors.\n"); + return -EINVAL; + } + + if (options->op_type == CPERF_CIPHER_THEN_AUTH) { + if (options->cipher_op != RTE_CRYPTO_CIPHER_OP_ENCRYPT && + options->auth_op != + RTE_CRYPTO_AUTH_OP_GENERATE) { + RTE_LOG(ERR, USER1, "Option cipher then auth must use" + " options: encrypt and generate.\n"); + return -EINVAL; + } + } else if (options->op_type == CPERF_AUTH_THEN_CIPHER) { + if (options->cipher_op != RTE_CRYPTO_CIPHER_OP_DECRYPT && + options->auth_op != + RTE_CRYPTO_AUTH_OP_VERIFY) { + RTE_LOG(ERR, USER1, "Option auth then cipher must use" + " options: decrypt and verify.\n"); + return -EINVAL; + } + } + + if (options->op_type == CPERF_CIPHER_ONLY || + options->op_type == CPERF_CIPHER_THEN_AUTH || + options->op_type == CPERF_AUTH_THEN_CIPHER) { + if (check_cipher_buffer_length(options) < 0) + return -EINVAL; + } + + return 0; +} + +void +cperf_options_dump(struct cperf_options *opts) +{ + uint8_t size_idx; + + printf("# Crypto Performance Application Options:\n"); + printf("#\n"); + printf("# cperf test: %s\n", cperf_test_type_strs[opts->test]); + printf("#\n"); + printf("# size of crypto op / mbuf pool: %u\n", opts->pool_sz); + printf("# total number of ops: %u\n", opts->total_ops); + if (opts->inc_buffer_size != 0) { + printf("# buffer size:\n"); + printf("#\t min: %u\n", opts->min_buffer_size); + printf("#\t max: %u\n", opts->max_buffer_size); + printf("#\t inc: %u\n", opts->inc_buffer_size); + } else { + printf("# buffer sizes: "); + for (size_idx = 0; size_idx < opts->buffer_size_count; size_idx++) + printf("%u ", opts->buffer_size_list[size_idx]); + printf("\n"); + } + if (opts->inc_burst_size != 0) { + printf("# burst size:\n"); + printf("#\t min: %u\n", opts->min_burst_size); + printf("#\t max: %u\n", opts->max_burst_size); + printf("#\t inc: %u\n", opts->inc_burst_size); + } else { + printf("# burst sizes: "); + for (size_idx = 0; size_idx < opts->burst_size_count; size_idx++) + printf("%u ", opts->burst_size_list[size_idx]); + printf("\n"); + } + printf("\n# segment size: %u\n", opts->segment_sz); + printf("#\n"); + printf("# cryptodev type: %s\n", opts->device_type); + printf("#\n"); + printf("# number of queue pairs per device: %u\n", opts->nb_qps); + printf("# crypto operation: %s\n", cperf_op_type_strs[opts->op_type]); + printf("# sessionless: %s\n", opts->sessionless ? "yes" : "no"); + printf("# out of place: %s\n", opts->out_of_place ? "yes" : "no"); + if (opts->test == CPERF_TEST_TYPE_PMDCC) + printf("# inter-burst delay: %u ms\n", opts->pmdcc_delay); + + printf("#\n"); + + if (opts->op_type == CPERF_AUTH_ONLY || + opts->op_type == CPERF_CIPHER_THEN_AUTH || + opts->op_type == CPERF_AUTH_THEN_CIPHER) { + printf("# auth algorithm: %s\n", + rte_crypto_auth_algorithm_strings[opts->auth_algo]); + printf("# auth operation: %s\n", + rte_crypto_auth_operation_strings[opts->auth_op]); + printf("# auth key size: %u\n", opts->auth_key_sz); + printf("# auth iv size: %u\n", opts->auth_iv_sz); + printf("# auth digest size: %u\n", opts->digest_sz); + printf("#\n"); + } + + if (opts->op_type == CPERF_CIPHER_ONLY || + opts->op_type == CPERF_CIPHER_THEN_AUTH || + opts->op_type == CPERF_AUTH_THEN_CIPHER) { + printf("# cipher algorithm: %s\n", + rte_crypto_cipher_algorithm_strings[opts->cipher_algo]); + printf("# cipher operation: %s\n", + rte_crypto_cipher_operation_strings[opts->cipher_op]); + printf("# cipher key size: %u\n", opts->cipher_key_sz); + printf("# cipher iv size: %u\n", opts->cipher_iv_sz); + printf("#\n"); + } + + if (opts->op_type == CPERF_AEAD) { + printf("# aead algorithm: %s\n", + rte_crypto_aead_algorithm_strings[opts->aead_algo]); + printf("# aead operation: %s\n", + rte_crypto_aead_operation_strings[opts->aead_op]); + printf("# aead key size: %u\n", opts->aead_key_sz); + printf("# aead iv size: %u\n", opts->aead_iv_sz); + printf("# aead digest size: %u\n", opts->digest_sz); + printf("# aead aad size: %u\n", opts->aead_aad_sz); + printf("#\n"); + } +} diff --git a/dpdk/app/test-crypto-perf/cperf_test_common.c b/dpdk/app/test-crypto-perf/cperf_test_common.c new file mode 100644 index 00000000..328744ef --- /dev/null +++ b/dpdk/app/test-crypto-perf/cperf_test_common.c @@ -0,0 +1,244 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +#include "cperf_test_common.h" + +struct obj_params { + uint32_t src_buf_offset; + uint32_t dst_buf_offset; + uint16_t segment_sz; + uint16_t segments_nb; +}; + +static void +fill_single_seg_mbuf(struct rte_mbuf *m, struct rte_mempool *mp, + void *obj, uint32_t mbuf_offset, uint16_t segment_sz) +{ + uint32_t mbuf_hdr_size = sizeof(struct rte_mbuf); + + /* start of buffer is after mbuf structure and priv data */ + m->priv_size = 0; + m->buf_addr = (char *)m + mbuf_hdr_size; + m->buf_iova = rte_mempool_virt2iova(obj) + + mbuf_offset + mbuf_hdr_size; + m->buf_len = segment_sz; + m->data_len = segment_sz; + + /* No headroom needed for the buffer */ + m->data_off = 0; + + /* init some constant fields */ + m->pool = mp; + m->nb_segs = 1; + m->port = 0xff; + rte_mbuf_refcnt_set(m, 1); + m->next = NULL; +} + +static void +fill_multi_seg_mbuf(struct rte_mbuf *m, struct rte_mempool *mp, + void *obj, uint32_t mbuf_offset, uint16_t segment_sz, + uint16_t segments_nb) +{ + uint16_t mbuf_hdr_size = sizeof(struct rte_mbuf); + uint16_t remaining_segments = segments_nb; + struct rte_mbuf *next_mbuf; + rte_iova_t next_seg_phys_addr = rte_mempool_virt2iova(obj) + + mbuf_offset + mbuf_hdr_size; + + do { + /* start of buffer is after mbuf structure and priv data */ + m->priv_size = 0; + m->buf_addr = (char *)m + mbuf_hdr_size; + m->buf_iova = next_seg_phys_addr; + next_seg_phys_addr += mbuf_hdr_size + segment_sz; + m->buf_len = segment_sz; + m->data_len = segment_sz; + + /* No headroom needed for the buffer */ + m->data_off = 0; + + /* init some constant fields */ + m->pool = mp; + m->nb_segs = segments_nb; + m->port = 0xff; + rte_mbuf_refcnt_set(m, 1); + next_mbuf = (struct rte_mbuf *) ((uint8_t *) m + + mbuf_hdr_size + segment_sz); + m->next = next_mbuf; + m = next_mbuf; + remaining_segments--; + + } while (remaining_segments > 0); + + m->next = NULL; +} + +static void +mempool_obj_init(struct rte_mempool *mp, + void *opaque_arg, + void *obj, + __attribute__((unused)) unsigned int i) +{ + struct obj_params *params = opaque_arg; + struct rte_crypto_op *op = obj; + struct rte_mbuf *m = (struct rte_mbuf *) ((uint8_t *) obj + + params->src_buf_offset); + /* Set crypto operation */ + op->type = RTE_CRYPTO_OP_TYPE_SYMMETRIC; + op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; + op->sess_type = RTE_CRYPTO_OP_WITH_SESSION; + op->phys_addr = rte_mem_virt2phy(obj); + op->mempool = mp; + + /* Set source buffer */ + op->sym->m_src = m; + if (params->segments_nb == 1) + fill_single_seg_mbuf(m, mp, obj, params->src_buf_offset, + params->segment_sz); + else + fill_multi_seg_mbuf(m, mp, obj, params->src_buf_offset, + params->segment_sz, params->segments_nb); + + + /* Set destination buffer */ + if (params->dst_buf_offset) { + m = (struct rte_mbuf *) ((uint8_t *) obj + + params->dst_buf_offset); + fill_single_seg_mbuf(m, mp, obj, params->dst_buf_offset, + params->segment_sz); + op->sym->m_dst = m; + } else + op->sym->m_dst = NULL; +} + +int +cperf_alloc_common_memory(const struct cperf_options *options, + const struct cperf_test_vector *test_vector, + uint8_t dev_id, uint16_t qp_id, + size_t extra_op_priv_size, + uint32_t *src_buf_offset, + uint32_t *dst_buf_offset, + struct rte_mempool **pool) +{ + char pool_name[32] = ""; + int ret; + + /* Calculate the object size */ + uint16_t crypto_op_size = sizeof(struct rte_crypto_op) + + sizeof(struct rte_crypto_sym_op); + uint16_t crypto_op_private_size; + /* + * If doing AES-CCM, IV field needs to be 16 bytes long, + * and AAD field needs to be long enough to have 18 bytes, + * plus the length of the AAD, and all rounded to a + * multiple of 16 bytes. + */ + if (options->aead_algo == RTE_CRYPTO_AEAD_AES_CCM) { + crypto_op_private_size = extra_op_priv_size + + test_vector->cipher_iv.length + + test_vector->auth_iv.length + + RTE_ALIGN_CEIL(test_vector->aead_iv.length, 16) + + RTE_ALIGN_CEIL(options->aead_aad_sz + 18, 16); + } else { + crypto_op_private_size = extra_op_priv_size + + test_vector->cipher_iv.length + + test_vector->auth_iv.length + + test_vector->aead_iv.length + + options->aead_aad_sz; + } + + uint16_t crypto_op_total_size = crypto_op_size + + crypto_op_private_size; + uint16_t crypto_op_total_size_padded = + RTE_CACHE_LINE_ROUNDUP(crypto_op_total_size); + uint32_t mbuf_size = sizeof(struct rte_mbuf) + options->segment_sz; + uint32_t max_size = options->max_buffer_size + options->digest_sz; + uint16_t segments_nb = (max_size % options->segment_sz) ? + (max_size / options->segment_sz) + 1 : + max_size / options->segment_sz; + uint32_t obj_size = crypto_op_total_size_padded + + (mbuf_size * segments_nb); + + snprintf(pool_name, sizeof(pool_name), "pool_cdev_%u_qp_%u", + dev_id, qp_id); + + *src_buf_offset = crypto_op_total_size_padded; + + struct obj_params params = { + .segment_sz = options->segment_sz, + .segments_nb = segments_nb, + .src_buf_offset = crypto_op_total_size_padded, + .dst_buf_offset = 0 + }; + + if (options->out_of_place) { + *dst_buf_offset = *src_buf_offset + + (mbuf_size * segments_nb); + params.dst_buf_offset = *dst_buf_offset; + /* Destination buffer will be one segment only */ + obj_size += max_size; + } + + *pool = rte_mempool_create_empty(pool_name, + options->pool_sz, obj_size, 512, 0, + rte_socket_id(), 0); + if (*pool == NULL) { + RTE_LOG(ERR, USER1, + "Cannot allocate mempool for device %u\n", + dev_id); + return -1; + } + + ret = rte_mempool_set_ops_byname(*pool, + RTE_MBUF_DEFAULT_MEMPOOL_OPS, NULL); + if (ret != 0) { + RTE_LOG(ERR, USER1, + "Error setting mempool handler for device %u\n", + dev_id); + return -1; + } + + ret = rte_mempool_populate_default(*pool); + if (ret < 0) { + RTE_LOG(ERR, USER1, + "Error populating mempool for device %u\n", + dev_id); + return -1; + } + + rte_mempool_obj_iter(*pool, mempool_obj_init, (void *)¶ms); + + return 0; +} diff --git a/dpdk/app/test-crypto-perf/cperf_test_common.h b/dpdk/app/test-crypto-perf/cperf_test_common.h new file mode 100644 index 00000000..4cee7852 --- /dev/null +++ b/dpdk/app/test-crypto-perf/cperf_test_common.h @@ -0,0 +1,52 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _CPERF_TEST_COMMON_H_ +#define _CPERF_TEST_COMMON_H_ + +#include + +#include + +#include "cperf_options.h" +#include "cperf_test_vectors.h" + +int +cperf_alloc_common_memory(const struct cperf_options *options, + const struct cperf_test_vector *test_vector, + uint8_t dev_id, uint16_t qp_id, + size_t extra_op_priv_size, + uint32_t *src_buf_offset, + uint32_t *dst_buf_offset, + struct rte_mempool **pool); + +#endif /* _CPERF_TEST_COMMON_H_ */ diff --git a/dpdk/app/test-crypto-perf/cperf_test_latency.c b/dpdk/app/test-crypto-perf/cperf_test_latency.c new file mode 100644 index 00000000..ca2a4ba6 --- /dev/null +++ b/dpdk/app/test-crypto-perf/cperf_test_latency.c @@ -0,0 +1,405 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include + +#include "cperf_test_latency.h" +#include "cperf_ops.h" +#include "cperf_test_common.h" + +struct cperf_op_result { + uint64_t tsc_start; + uint64_t tsc_end; + enum rte_crypto_op_status status; +}; + +struct cperf_latency_ctx { + uint8_t dev_id; + uint16_t qp_id; + uint8_t lcore_id; + + struct rte_mempool *pool; + + struct rte_cryptodev_sym_session *sess; + + cperf_populate_ops_t populate_ops; + + uint32_t src_buf_offset; + uint32_t dst_buf_offset; + + const struct cperf_options *options; + const struct cperf_test_vector *test_vector; + struct cperf_op_result *res; +}; + +struct priv_op_data { + struct cperf_op_result *result; +}; + +#define max(a, b) (a > b ? (uint64_t)a : (uint64_t)b) +#define min(a, b) (a < b ? (uint64_t)a : (uint64_t)b) + +static void +cperf_latency_test_free(struct cperf_latency_ctx *ctx) +{ + if (ctx) { + if (ctx->sess) { + rte_cryptodev_sym_session_clear(ctx->dev_id, ctx->sess); + rte_cryptodev_sym_session_free(ctx->sess); + } + + if (ctx->pool) + rte_mempool_free(ctx->pool); + + rte_free(ctx->res); + rte_free(ctx); + } +} + +void * +cperf_latency_test_constructor(struct rte_mempool *sess_mp, + uint8_t dev_id, uint16_t qp_id, + const struct cperf_options *options, + const struct cperf_test_vector *test_vector, + const struct cperf_op_fns *op_fns) +{ + struct cperf_latency_ctx *ctx = NULL; + size_t extra_op_priv_size = sizeof(struct priv_op_data); + + ctx = rte_malloc(NULL, sizeof(struct cperf_latency_ctx), 0); + if (ctx == NULL) + goto err; + + ctx->dev_id = dev_id; + ctx->qp_id = qp_id; + + ctx->populate_ops = op_fns->populate_ops; + ctx->options = options; + ctx->test_vector = test_vector; + + /* IV goes at the end of the crypto operation */ + uint16_t iv_offset = sizeof(struct rte_crypto_op) + + sizeof(struct rte_crypto_sym_op) + + sizeof(struct cperf_op_result *); + + ctx->sess = op_fns->sess_create(sess_mp, dev_id, options, test_vector, + iv_offset); + if (ctx->sess == NULL) + goto err; + + if (cperf_alloc_common_memory(options, test_vector, dev_id, qp_id, + extra_op_priv_size, + &ctx->src_buf_offset, &ctx->dst_buf_offset, + &ctx->pool) < 0) + goto err; + + ctx->res = rte_malloc(NULL, sizeof(struct cperf_op_result) * + ctx->options->total_ops, 0); + + if (ctx->res == NULL) + goto err; + + return ctx; +err: + cperf_latency_test_free(ctx); + + return NULL; +} + +static inline void +store_timestamp(struct rte_crypto_op *op, uint64_t timestamp) +{ + struct priv_op_data *priv_data; + + priv_data = (struct priv_op_data *) (op->sym + 1); + priv_data->result->status = op->status; + priv_data->result->tsc_end = timestamp; +} + +int +cperf_latency_test_runner(void *arg) +{ + struct cperf_latency_ctx *ctx = arg; + uint16_t test_burst_size; + uint8_t burst_size_idx = 0; + + static int only_once; + + if (ctx == NULL) + return 0; + + struct rte_crypto_op *ops[ctx->options->max_burst_size]; + struct rte_crypto_op *ops_processed[ctx->options->max_burst_size]; + uint64_t i; + struct priv_op_data *priv_data; + + uint32_t lcore = rte_lcore_id(); + +#ifdef CPERF_LINEARIZATION_ENABLE + struct rte_cryptodev_info dev_info; + int linearize = 0; + + /* Check if source mbufs require coalescing */ + if (ctx->options->segment_sz < ctx->options->max_buffer_size) { + rte_cryptodev_info_get(ctx->dev_id, &dev_info); + if ((dev_info.feature_flags & + RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER) == 0) + linearize = 1; + } +#endif /* CPERF_LINEARIZATION_ENABLE */ + + ctx->lcore_id = lcore; + + /* Warm up the host CPU before starting the test */ + for (i = 0; i < ctx->options->total_ops; i++) + rte_cryptodev_enqueue_burst(ctx->dev_id, ctx->qp_id, NULL, 0); + + /* Get first size from range or list */ + if (ctx->options->inc_burst_size != 0) + test_burst_size = ctx->options->min_burst_size; + else + test_burst_size = ctx->options->burst_size_list[0]; + + uint16_t iv_offset = sizeof(struct rte_crypto_op) + + sizeof(struct rte_crypto_sym_op) + + sizeof(struct cperf_op_result *); + + while (test_burst_size <= ctx->options->max_burst_size) { + uint64_t ops_enqd = 0, ops_deqd = 0; + uint64_t b_idx = 0; + + uint64_t tsc_val, tsc_end, tsc_start; + uint64_t tsc_max = 0, tsc_min = ~0UL, tsc_tot = 0, tsc_idx = 0; + uint64_t enqd_max = 0, enqd_min = ~0UL, enqd_tot = 0; + uint64_t deqd_max = 0, deqd_min = ~0UL, deqd_tot = 0; + + while (enqd_tot < ctx->options->total_ops) { + + uint16_t burst_size = ((enqd_tot + test_burst_size) + <= ctx->options->total_ops) ? + test_burst_size : + ctx->options->total_ops - + enqd_tot; + + /* Allocate objects containing crypto operations and mbufs */ + if (rte_mempool_get_bulk(ctx->pool, (void **)ops, + burst_size) != 0) { + RTE_LOG(ERR, USER1, + "Failed to allocate more crypto operations " + "from the the crypto operation pool.\n" + "Consider increasing the pool size " + "with --pool-sz\n"); + return -1; + } + + /* Setup crypto op, attach mbuf etc */ + (ctx->populate_ops)(ops, ctx->src_buf_offset, + ctx->dst_buf_offset, + burst_size, ctx->sess, ctx->options, + ctx->test_vector, iv_offset); + + tsc_start = rte_rdtsc_precise(); + +#ifdef CPERF_LINEARIZATION_ENABLE + if (linearize) { + /* PMD doesn't support scatter-gather and source buffer + * is segmented. + * We need to linearize it before enqueuing. + */ + for (i = 0; i < burst_size; i++) + rte_pktmbuf_linearize(ops[i]->sym->m_src); + } +#endif /* CPERF_LINEARIZATION_ENABLE */ + + /* Enqueue burst of ops on crypto device */ + ops_enqd = rte_cryptodev_enqueue_burst(ctx->dev_id, ctx->qp_id, + ops, burst_size); + + /* Dequeue processed burst of ops from crypto device */ + ops_deqd = rte_cryptodev_dequeue_burst(ctx->dev_id, ctx->qp_id, + ops_processed, test_burst_size); + + tsc_end = rte_rdtsc_precise(); + + /* Free memory for not enqueued operations */ + if (ops_enqd != burst_size) + rte_mempool_put_bulk(ctx->pool, + (void **)&ops[ops_enqd], + burst_size - ops_enqd); + + for (i = 0; i < ops_enqd; i++) { + ctx->res[tsc_idx].tsc_start = tsc_start; + /* + * Private data structure starts after the end of the + * rte_crypto_sym_op structure. + */ + priv_data = (struct priv_op_data *) (ops[i]->sym + 1); + priv_data->result = (void *)&ctx->res[tsc_idx]; + tsc_idx++; + } + + if (likely(ops_deqd)) { + /* Free crypto ops so they can be reused. */ + for (i = 0; i < ops_deqd; i++) + store_timestamp(ops_processed[i], tsc_end); + + rte_mempool_put_bulk(ctx->pool, + (void **)ops_processed, ops_deqd); + + deqd_tot += ops_deqd; + deqd_max = max(ops_deqd, deqd_max); + deqd_min = min(ops_deqd, deqd_min); + } + + enqd_tot += ops_enqd; + enqd_max = max(ops_enqd, enqd_max); + enqd_min = min(ops_enqd, enqd_min); + + b_idx++; + } + + /* Dequeue any operations still in the crypto device */ + while (deqd_tot < ctx->options->total_ops) { + /* Sending 0 length burst to flush sw crypto device */ + rte_cryptodev_enqueue_burst(ctx->dev_id, ctx->qp_id, NULL, 0); + + /* dequeue burst */ + ops_deqd = rte_cryptodev_dequeue_burst(ctx->dev_id, ctx->qp_id, + ops_processed, test_burst_size); + + tsc_end = rte_rdtsc_precise(); + + if (ops_deqd != 0) { + for (i = 0; i < ops_deqd; i++) + store_timestamp(ops_processed[i], tsc_end); + + rte_mempool_put_bulk(ctx->pool, + (void **)ops_processed, ops_deqd); + + deqd_tot += ops_deqd; + deqd_max = max(ops_deqd, deqd_max); + deqd_min = min(ops_deqd, deqd_min); + } + } + + for (i = 0; i < tsc_idx; i++) { + tsc_val = ctx->res[i].tsc_end - ctx->res[i].tsc_start; + tsc_max = max(tsc_val, tsc_max); + tsc_min = min(tsc_val, tsc_min); + tsc_tot += tsc_val; + } + + double time_tot, time_avg, time_max, time_min; + + const uint64_t tunit = 1000000; /* us */ + const uint64_t tsc_hz = rte_get_tsc_hz(); + + uint64_t enqd_avg = enqd_tot / b_idx; + uint64_t deqd_avg = deqd_tot / b_idx; + uint64_t tsc_avg = tsc_tot / tsc_idx; + + time_tot = tunit*(double)(tsc_tot) / tsc_hz; + time_avg = tunit*(double)(tsc_avg) / tsc_hz; + time_max = tunit*(double)(tsc_max) / tsc_hz; + time_min = tunit*(double)(tsc_min) / tsc_hz; + + if (ctx->options->csv) { + if (!only_once) + printf("\n# lcore, Buffer Size, Burst Size, Pakt Seq #, " + "Packet Size, cycles, time (us)"); + + for (i = 0; i < ctx->options->total_ops; i++) { + + printf("\n%u;%u;%u;%"PRIu64";%"PRIu64";%.3f", + ctx->lcore_id, ctx->options->test_buffer_size, + test_burst_size, i + 1, + ctx->res[i].tsc_end - ctx->res[i].tsc_start, + tunit * (double) (ctx->res[i].tsc_end + - ctx->res[i].tsc_start) + / tsc_hz); + + } + only_once = 1; + } else { + printf("\n# Device %d on lcore %u\n", ctx->dev_id, + ctx->lcore_id); + printf("\n# total operations: %u", ctx->options->total_ops); + printf("\n# Buffer size: %u", ctx->options->test_buffer_size); + printf("\n# Burst size: %u", test_burst_size); + printf("\n# Number of bursts: %"PRIu64, + b_idx); + + printf("\n#"); + printf("\n# \t Total\t Average\t " + "Maximum\t Minimum"); + printf("\n# enqueued\t%12"PRIu64"\t%10"PRIu64"\t" + "%10"PRIu64"\t%10"PRIu64, enqd_tot, + enqd_avg, enqd_max, enqd_min); + printf("\n# dequeued\t%12"PRIu64"\t%10"PRIu64"\t" + "%10"PRIu64"\t%10"PRIu64, deqd_tot, + deqd_avg, deqd_max, deqd_min); + printf("\n# cycles\t%12"PRIu64"\t%10"PRIu64"\t" + "%10"PRIu64"\t%10"PRIu64, tsc_tot, + tsc_avg, tsc_max, tsc_min); + printf("\n# time [us]\t%12.0f\t%10.3f\t%10.3f\t%10.3f", + time_tot, time_avg, time_max, time_min); + printf("\n\n"); + + } + + /* Get next size from range or list */ + if (ctx->options->inc_burst_size != 0) + test_burst_size += ctx->options->inc_burst_size; + else { + if (++burst_size_idx == ctx->options->burst_size_count) + break; + test_burst_size = + ctx->options->burst_size_list[burst_size_idx]; + } + } + + return 0; +} + +void +cperf_latency_test_destructor(void *arg) +{ + struct cperf_latency_ctx *ctx = arg; + + if (ctx == NULL) + return; + + cperf_latency_test_free(ctx); +} diff --git a/dpdk/app/test-crypto-perf/cperf_test_latency.h b/dpdk/app/test-crypto-perf/cperf_test_latency.h new file mode 100644 index 00000000..1bbedb4e --- /dev/null +++ b/dpdk/app/test-crypto-perf/cperf_test_latency.h @@ -0,0 +1,60 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _CPERF_LATENCY_ +#define _CPERF_LATENCY_ + +#include + +#include + +#include "cperf.h" +#include "cperf_ops.h" +#include "cperf_options.h" +#include "cperf_test_vectors.h" + +void * +cperf_latency_test_constructor( + struct rte_mempool *sess_mp, + uint8_t dev_id, + uint16_t qp_id, + const struct cperf_options *options, + const struct cperf_test_vector *test_vector, + const struct cperf_op_fns *ops_fn); + +int +cperf_latency_test_runner(void *test_ctx); + +void +cperf_latency_test_destructor(void *test_ctx); + +#endif /* _CPERF_LATENCY_ */ diff --git a/dpdk/app/test-crypto-perf/cperf_test_pmd_cyclecount.c b/dpdk/app/test-crypto-perf/cperf_test_pmd_cyclecount.c new file mode 100644 index 00000000..9b41724a --- /dev/null +++ b/dpdk/app/test-crypto-perf/cperf_test_pmd_cyclecount.c @@ -0,0 +1,520 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +#include +#include +#include +#include + +#include "cperf_ops.h" +#include "cperf_test_pmd_cyclecount.h" +#include "cperf_test_common.h" + +#define PRETTY_HDR_FMT "%12s%12s%12s%12s%12s%12s%12s%12s%12s%12s\n\n" +#define PRETTY_LINE_FMT "%12u%12u%12u%12u%12u%12u%12u%12.0f%12.0f%12.0f\n" +#define CSV_HDR_FMT "%s,%s,%s,%s,%s,%s,%s,%s,%s,%s\n" +#define CSV_LINE_FMT "%10u;%10u;%u;%u;%u;%u;%u;%.f3;%.f3;%.f3\n" + +struct cperf_pmd_cyclecount_ctx { + uint8_t dev_id; + uint16_t qp_id; + uint8_t lcore_id; + + struct rte_mempool *pool; + struct rte_crypto_op **ops; + struct rte_crypto_op **ops_processed; + + struct rte_cryptodev_sym_session *sess; + + cperf_populate_ops_t populate_ops; + + uint32_t src_buf_offset; + uint32_t dst_buf_offset; + + const struct cperf_options *options; + const struct cperf_test_vector *test_vector; +}; + +struct pmd_cyclecount_state { + struct cperf_pmd_cyclecount_ctx *ctx; + const struct cperf_options *opts; + uint32_t lcore; + uint64_t delay; + int linearize; + uint32_t ops_enqd; + uint32_t ops_deqd; + uint32_t ops_enq_retries; + uint32_t ops_deq_retries; + double cycles_per_build; + double cycles_per_enq; + double cycles_per_deq; +}; + +static const uint16_t iv_offset = + sizeof(struct rte_crypto_op) + sizeof(struct rte_crypto_sym_op); + +static void +cperf_pmd_cyclecount_test_free(struct cperf_pmd_cyclecount_ctx *ctx) +{ + if (ctx) { + if (ctx->sess) { + rte_cryptodev_sym_session_clear(ctx->dev_id, ctx->sess); + rte_cryptodev_sym_session_free(ctx->sess); + } + + if (ctx->pool) + rte_mempool_free(ctx->pool); + + if (ctx->ops) + rte_free(ctx->ops); + + if (ctx->ops_processed) + rte_free(ctx->ops_processed); + + rte_free(ctx); + } +} + +void * +cperf_pmd_cyclecount_test_constructor(struct rte_mempool *sess_mp, + uint8_t dev_id, uint16_t qp_id, + const struct cperf_options *options, + const struct cperf_test_vector *test_vector, + const struct cperf_op_fns *op_fns) +{ + struct cperf_pmd_cyclecount_ctx *ctx = NULL; + + /* preallocate buffers for crypto ops as they can get quite big */ + size_t alloc_sz = sizeof(struct rte_crypto_op *) * + options->nb_descriptors; + + ctx = rte_malloc(NULL, sizeof(struct cperf_pmd_cyclecount_ctx), 0); + if (ctx == NULL) + goto err; + + ctx->dev_id = dev_id; + ctx->qp_id = qp_id; + + ctx->populate_ops = op_fns->populate_ops; + ctx->options = options; + ctx->test_vector = test_vector; + + /* IV goes at the end of the crypto operation */ + uint16_t iv_offset = sizeof(struct rte_crypto_op) + + sizeof(struct rte_crypto_sym_op); + + ctx->sess = op_fns->sess_create( + sess_mp, dev_id, options, test_vector, iv_offset); + if (ctx->sess == NULL) + goto err; + + if (cperf_alloc_common_memory(options, test_vector, dev_id, qp_id, 0, + &ctx->src_buf_offset, &ctx->dst_buf_offset, + &ctx->pool) < 0) + goto err; + + ctx->ops = rte_malloc("ops", alloc_sz, 0); + if (!ctx->ops) + goto err; + + ctx->ops_processed = rte_malloc("ops_processed", alloc_sz, 0); + if (!ctx->ops_processed) + goto err; + + return ctx; + +err: + cperf_pmd_cyclecount_test_free(ctx); + + return NULL; +} + +/* benchmark alloc-build-free of ops */ +static inline int +pmd_cyclecount_bench_ops(struct pmd_cyclecount_state *state, uint32_t cur_op, + uint16_t test_burst_size) +{ + uint32_t iter_ops_left = state->opts->total_ops - cur_op; + uint32_t iter_ops_needed = + RTE_MIN(state->opts->nb_descriptors, iter_ops_left); + uint32_t cur_iter_op; + + for (cur_iter_op = 0; cur_iter_op < iter_ops_needed; + cur_iter_op += test_burst_size) { + uint32_t burst_size = RTE_MIN(state->opts->total_ops - cur_op, + test_burst_size); + struct rte_crypto_op **ops = &state->ctx->ops[cur_iter_op]; + + /* Allocate objects containing crypto operations and mbufs */ + if (rte_mempool_get_bulk(state->ctx->pool, (void **)ops, + burst_size) != 0) { + RTE_LOG(ERR, USER1, + "Failed to allocate more crypto operations " + "from the the crypto operation pool.\n" + "Consider increasing the pool size " + "with --pool-sz\n"); + return -1; + } + + /* Setup crypto op, attach mbuf etc */ + (state->ctx->populate_ops)(ops, + state->ctx->src_buf_offset, + state->ctx->dst_buf_offset, + burst_size, + state->ctx->sess, state->opts, + state->ctx->test_vector, iv_offset); + +#ifdef CPERF_LINEARIZATION_ENABLE + /* Check if source mbufs require coalescing */ + if (state->linearize) { + uint8_t i; + for (i = 0; i < burst_size; i++) { + struct rte_mbuf *src = ops[i]->sym->m_src; + rte_pktmbuf_linearize(src); + } + } +#endif /* CPERF_LINEARIZATION_ENABLE */ + rte_mempool_put_bulk(state->ctx->pool, (void **)ops, + burst_size); + } + + return 0; +} + +/* allocate and build ops (no free) */ +static int +pmd_cyclecount_build_ops(struct pmd_cyclecount_state *state, + uint32_t iter_ops_needed, uint16_t test_burst_size) +{ + uint32_t cur_iter_op; + + for (cur_iter_op = 0; cur_iter_op < iter_ops_needed; + cur_iter_op += test_burst_size) { + uint32_t burst_size = RTE_MIN( + iter_ops_needed - cur_iter_op, test_burst_size); + struct rte_crypto_op **ops = &state->ctx->ops[cur_iter_op]; + + /* Allocate objects containing crypto operations and mbufs */ + if (rte_mempool_get_bulk(state->ctx->pool, (void **)ops, + burst_size) != 0) { + RTE_LOG(ERR, USER1, + "Failed to allocate more crypto operations " + "from the the crypto operation pool.\n" + "Consider increasing the pool size " + "with --pool-sz\n"); + return -1; + } + + /* Setup crypto op, attach mbuf etc */ + (state->ctx->populate_ops)(ops, + state->ctx->src_buf_offset, + state->ctx->dst_buf_offset, + burst_size, + state->ctx->sess, state->opts, + state->ctx->test_vector, iv_offset); + } + return 0; +} + +/* benchmark enqueue, returns number of ops enqueued */ +static uint32_t +pmd_cyclecount_bench_enq(struct pmd_cyclecount_state *state, + uint32_t iter_ops_needed, uint16_t test_burst_size) +{ + /* Enqueue full descriptor ring of ops on crypto device */ + uint32_t cur_iter_op = 0; + while (cur_iter_op < iter_ops_needed) { + uint32_t burst_size = RTE_MIN(iter_ops_needed - cur_iter_op, + test_burst_size); + struct rte_crypto_op **ops = &state->ctx->ops[cur_iter_op]; + uint32_t burst_enqd; + + burst_enqd = rte_cryptodev_enqueue_burst(state->ctx->dev_id, + state->ctx->qp_id, ops, burst_size); + + /* if we couldn't enqueue anything, the queue is full */ + if (!burst_enqd) { + /* don't try to dequeue anything we didn't enqueue */ + return cur_iter_op; + } + + if (burst_enqd < burst_size) + state->ops_enq_retries++; + state->ops_enqd += burst_enqd; + cur_iter_op += burst_enqd; + } + return iter_ops_needed; +} + +/* benchmark dequeue */ +static void +pmd_cyclecount_bench_deq(struct pmd_cyclecount_state *state, + uint32_t iter_ops_needed, uint16_t test_burst_size) +{ + /* Dequeue full descriptor ring of ops on crypto device */ + uint32_t cur_iter_op = 0; + while (cur_iter_op < iter_ops_needed) { + uint32_t burst_size = RTE_MIN(iter_ops_needed - cur_iter_op, + test_burst_size); + struct rte_crypto_op **ops_processed = + &state->ctx->ops[cur_iter_op]; + uint32_t burst_deqd; + + burst_deqd = rte_cryptodev_dequeue_burst(state->ctx->dev_id, + state->ctx->qp_id, ops_processed, burst_size); + + if (burst_deqd < burst_size) + state->ops_deq_retries++; + state->ops_deqd += burst_deqd; + cur_iter_op += burst_deqd; + } +} + +/* run benchmark per burst size */ +static inline int +pmd_cyclecount_bench_burst_sz( + struct pmd_cyclecount_state *state, uint16_t test_burst_size) +{ + uint64_t tsc_start; + uint64_t tsc_end; + uint64_t tsc_op; + uint64_t tsc_enq; + uint64_t tsc_deq; + uint32_t cur_op; + + /* reset all counters */ + tsc_enq = 0; + tsc_deq = 0; + state->ops_enqd = 0; + state->ops_enq_retries = 0; + state->ops_deqd = 0; + state->ops_deq_retries = 0; + + /* + * Benchmark crypto op alloc-build-free separately. + */ + tsc_start = rte_rdtsc_precise(); + + for (cur_op = 0; cur_op < state->opts->total_ops; + cur_op += state->opts->nb_descriptors) { + if (unlikely(pmd_cyclecount_bench_ops( + state, cur_op, test_burst_size))) + return -1; + } + + tsc_end = rte_rdtsc_precise(); + tsc_op = tsc_end - tsc_start; + + + /* + * Hardware acceleration cyclecount benchmarking loop. + * + * We're benchmarking raw enq/deq performance by filling up the device + * queue, so we never get any failed enqs unless the driver won't accept + * the exact number of descriptors we requested, or the driver won't + * wrap around the end of the TX ring. However, since we're only + * dequeueing once we've filled up the queue, we have to benchmark it + * piecemeal and then average out the results. + */ + cur_op = 0; + while (cur_op < state->opts->total_ops) { + uint32_t iter_ops_left = state->opts->total_ops - cur_op; + uint32_t iter_ops_needed = RTE_MIN( + state->opts->nb_descriptors, iter_ops_left); + uint32_t iter_ops_allocd = iter_ops_needed; + + /* allocate and build ops */ + if (unlikely(pmd_cyclecount_build_ops(state, iter_ops_needed, + test_burst_size))) + return -1; + + tsc_start = rte_rdtsc_precise(); + + /* fill up TX ring */ + iter_ops_needed = pmd_cyclecount_bench_enq(state, + iter_ops_needed, test_burst_size); + + tsc_end = rte_rdtsc_precise(); + + tsc_enq += tsc_end - tsc_start; + + /* allow for HW to catch up */ + if (state->delay) + rte_delay_us_block(state->delay); + + tsc_start = rte_rdtsc_precise(); + + /* drain RX ring */ + pmd_cyclecount_bench_deq(state, iter_ops_needed, + test_burst_size); + + tsc_end = rte_rdtsc_precise(); + + tsc_deq += tsc_end - tsc_start; + + cur_op += iter_ops_needed; + + /* + * we may not have processed all ops that we allocated, so + * free everything we've allocated. + */ + rte_mempool_put_bulk(state->ctx->pool, + (void **)state->ctx->ops, iter_ops_allocd); + } + + state->cycles_per_build = (double)tsc_op / state->opts->total_ops; + state->cycles_per_enq = (double)tsc_enq / state->ops_enqd; + state->cycles_per_deq = (double)tsc_deq / state->ops_deqd; + + return 0; +} + +int +cperf_pmd_cyclecount_test_runner(void *test_ctx) +{ + struct pmd_cyclecount_state state = {0}; + const struct cperf_options *opts; + uint16_t test_burst_size; + uint8_t burst_size_idx = 0; + + state.ctx = test_ctx; + opts = state.ctx->options; + state.opts = opts; + state.lcore = rte_lcore_id(); + state.linearize = 0; + + static int only_once; + static bool warmup = true; + + /* + * We need a small delay to allow for hardware to process all the crypto + * operations. We can't automatically figure out what the delay should + * be, so we leave it up to the user (by default it's 0). + */ + state.delay = 1000 * opts->pmdcc_delay; + +#ifdef CPERF_LINEARIZATION_ENABLE + struct rte_cryptodev_info dev_info; + + /* Check if source mbufs require coalescing */ + if (opts->segments_sz < ctx->options->max_buffer_size) { + rte_cryptodev_info_get(state.ctx->dev_id, &dev_info); + if ((dev_info.feature_flags & + RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER) == + 0) { + state.linearize = 1; + } + } +#endif /* CPERF_LINEARIZATION_ENABLE */ + + state.ctx->lcore_id = state.lcore; + + /* Get first size from range or list */ + if (opts->inc_burst_size != 0) + test_burst_size = opts->min_burst_size; + else + test_burst_size = opts->burst_size_list[0]; + + while (test_burst_size <= opts->max_burst_size) { + /* do a benchmark run */ + if (pmd_cyclecount_bench_burst_sz(&state, test_burst_size)) + return -1; + + /* + * First run is always a warm up run. + */ + if (warmup) { + warmup = false; + continue; + } + + if (!opts->csv) { + if (!only_once) + printf(PRETTY_HDR_FMT, "lcore id", "Buf Size", + "Burst Size", "Enqueued", + "Dequeued", "Enq Retries", + "Deq Retries", "Cycles/Op", + "Cycles/Enq", "Cycles/Deq"); + only_once = 1; + + printf(PRETTY_LINE_FMT, state.ctx->lcore_id, + opts->test_buffer_size, test_burst_size, + state.ops_enqd, state.ops_deqd, + state.ops_enq_retries, + state.ops_deq_retries, + state.cycles_per_build, + state.cycles_per_enq, + state.cycles_per_deq); + } else { + if (!only_once) + printf(CSV_HDR_FMT, "# lcore id", "Buf Size", + "Burst Size", "Enqueued", + "Dequeued", "Enq Retries", + "Deq Retries", "Cycles/Op", + "Cycles/Enq", "Cycles/Deq"); + only_once = 1; + + printf(CSV_LINE_FMT, state.ctx->lcore_id, + opts->test_buffer_size, test_burst_size, + state.ops_enqd, state.ops_deqd, + state.ops_enq_retries, + state.ops_deq_retries, + state.cycles_per_build, + state.cycles_per_enq, + state.cycles_per_deq); + } + + /* Get next size from range or list */ + if (opts->inc_burst_size != 0) + test_burst_size += opts->inc_burst_size; + else { + if (++burst_size_idx == opts->burst_size_count) + break; + test_burst_size = opts->burst_size_list[burst_size_idx]; + } + } + + return 0; +} + +void +cperf_pmd_cyclecount_test_destructor(void *arg) +{ + struct cperf_pmd_cyclecount_ctx *ctx = arg; + + if (ctx == NULL) + return; + + cperf_pmd_cyclecount_test_free(ctx); +} diff --git a/dpdk/app/test-crypto-perf/cperf_test_pmd_cyclecount.h b/dpdk/app/test-crypto-perf/cperf_test_pmd_cyclecount.h new file mode 100644 index 00000000..93f0eae0 --- /dev/null +++ b/dpdk/app/test-crypto-perf/cperf_test_pmd_cyclecount.h @@ -0,0 +1,61 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _CPERF_TEST_PMD_CYCLECOUNT_H_ +#define _CPERF_TEST_PMD_CYCLECOUNT_H_ + +#include + +#include + +#include "cperf.h" +#include "cperf_ops.h" +#include "cperf_options.h" +#include "cperf_test_vectors.h" + + +void * +cperf_pmd_cyclecount_test_constructor( + struct rte_mempool *sess_mp, + uint8_t dev_id, + uint16_t qp_id, + const struct cperf_options *options, + const struct cperf_test_vector *test_vector, + const struct cperf_op_fns *ops_fn); + +int +cperf_pmd_cyclecount_test_runner(void *test_ctx); + +void +cperf_pmd_cyclecount_test_destructor(void *test_ctx); + +#endif /* _CPERF_TEST_PMD_CYCLECOUNT_H_ */ diff --git a/dpdk/app/test-crypto-perf/cperf_test_throughput.c b/dpdk/app/test-crypto-perf/cperf_test_throughput.c new file mode 100644 index 00000000..b84dc630 --- /dev/null +++ b/dpdk/app/test-crypto-perf/cperf_test_throughput.c @@ -0,0 +1,357 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include + +#include "cperf_test_throughput.h" +#include "cperf_ops.h" +#include "cperf_test_common.h" + +struct cperf_throughput_ctx { + uint8_t dev_id; + uint16_t qp_id; + uint8_t lcore_id; + + struct rte_mempool *pool; + + struct rte_cryptodev_sym_session *sess; + + cperf_populate_ops_t populate_ops; + + uint32_t src_buf_offset; + uint32_t dst_buf_offset; + + const struct cperf_options *options; + const struct cperf_test_vector *test_vector; +}; + +static void +cperf_throughput_test_free(struct cperf_throughput_ctx *ctx) +{ + if (ctx) { + if (ctx->sess) { + rte_cryptodev_sym_session_clear(ctx->dev_id, ctx->sess); + rte_cryptodev_sym_session_free(ctx->sess); + } + + if (ctx->pool) + rte_mempool_free(ctx->pool); + + rte_free(ctx); + } +} + +void * +cperf_throughput_test_constructor(struct rte_mempool *sess_mp, + uint8_t dev_id, uint16_t qp_id, + const struct cperf_options *options, + const struct cperf_test_vector *test_vector, + const struct cperf_op_fns *op_fns) +{ + struct cperf_throughput_ctx *ctx = NULL; + + ctx = rte_malloc(NULL, sizeof(struct cperf_throughput_ctx), 0); + if (ctx == NULL) + goto err; + + ctx->dev_id = dev_id; + ctx->qp_id = qp_id; + + ctx->populate_ops = op_fns->populate_ops; + ctx->options = options; + ctx->test_vector = test_vector; + + /* IV goes at the end of the crypto operation */ + uint16_t iv_offset = sizeof(struct rte_crypto_op) + + sizeof(struct rte_crypto_sym_op); + + ctx->sess = op_fns->sess_create(sess_mp, dev_id, options, test_vector, + iv_offset); + if (ctx->sess == NULL) + goto err; + + if (cperf_alloc_common_memory(options, test_vector, dev_id, qp_id, 0, + &ctx->src_buf_offset, &ctx->dst_buf_offset, + &ctx->pool) < 0) + goto err; + + return ctx; +err: + cperf_throughput_test_free(ctx); + + return NULL; +} + +int +cperf_throughput_test_runner(void *test_ctx) +{ + struct cperf_throughput_ctx *ctx = test_ctx; + uint16_t test_burst_size; + uint8_t burst_size_idx = 0; + + static int only_once; + + struct rte_crypto_op *ops[ctx->options->max_burst_size]; + struct rte_crypto_op *ops_processed[ctx->options->max_burst_size]; + uint64_t i; + + uint32_t lcore = rte_lcore_id(); + +#ifdef CPERF_LINEARIZATION_ENABLE + struct rte_cryptodev_info dev_info; + int linearize = 0; + + /* Check if source mbufs require coalescing */ + if (ctx->options->segment_sz < ctx->options->max_buffer_size) { + rte_cryptodev_info_get(ctx->dev_id, &dev_info); + if ((dev_info.feature_flags & + RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER) == 0) + linearize = 1; + } +#endif /* CPERF_LINEARIZATION_ENABLE */ + + ctx->lcore_id = lcore; + + /* Warm up the host CPU before starting the test */ + for (i = 0; i < ctx->options->total_ops; i++) + rte_cryptodev_enqueue_burst(ctx->dev_id, ctx->qp_id, NULL, 0); + + /* Get first size from range or list */ + if (ctx->options->inc_burst_size != 0) + test_burst_size = ctx->options->min_burst_size; + else + test_burst_size = ctx->options->burst_size_list[0]; + + uint16_t iv_offset = sizeof(struct rte_crypto_op) + + sizeof(struct rte_crypto_sym_op); + + while (test_burst_size <= ctx->options->max_burst_size) { + uint64_t ops_enqd = 0, ops_enqd_total = 0, ops_enqd_failed = 0; + uint64_t ops_deqd = 0, ops_deqd_total = 0, ops_deqd_failed = 0; + + uint64_t tsc_start, tsc_end, tsc_duration; + + uint16_t ops_unused = 0; + + tsc_start = rte_rdtsc_precise(); + + while (ops_enqd_total < ctx->options->total_ops) { + + uint16_t burst_size = ((ops_enqd_total + test_burst_size) + <= ctx->options->total_ops) ? + test_burst_size : + ctx->options->total_ops - + ops_enqd_total; + + uint16_t ops_needed = burst_size - ops_unused; + + /* Allocate objects containing crypto operations and mbufs */ + if (rte_mempool_get_bulk(ctx->pool, (void **)ops, + ops_needed) != 0) { + RTE_LOG(ERR, USER1, + "Failed to allocate more crypto operations " + "from the the crypto operation pool.\n" + "Consider increasing the pool size " + "with --pool-sz\n"); + return -1; + } + + /* Setup crypto op, attach mbuf etc */ + (ctx->populate_ops)(ops, ctx->src_buf_offset, + ctx->dst_buf_offset, + ops_needed, ctx->sess, + ctx->options, ctx->test_vector, + iv_offset); + + /** + * When ops_needed is smaller than ops_enqd, the + * unused ops need to be moved to the front for + * next round use. + */ + if (unlikely(ops_enqd > ops_needed)) { + size_t nb_b_to_mov = ops_unused * sizeof( + struct rte_crypto_op *); + + memmove(&ops[ops_needed], &ops[ops_enqd], + nb_b_to_mov); + } + +#ifdef CPERF_LINEARIZATION_ENABLE + if (linearize) { + /* PMD doesn't support scatter-gather and source buffer + * is segmented. + * We need to linearize it before enqueuing. + */ + for (i = 0; i < burst_size; i++) + rte_pktmbuf_linearize(ops[i]->sym->m_src); + } +#endif /* CPERF_LINEARIZATION_ENABLE */ + + /* Enqueue burst of ops on crypto device */ + ops_enqd = rte_cryptodev_enqueue_burst(ctx->dev_id, ctx->qp_id, + ops, burst_size); + if (ops_enqd < burst_size) + ops_enqd_failed++; + + /** + * Calculate number of ops not enqueued (mainly for hw + * accelerators whose ingress queue can fill up). + */ + ops_unused = burst_size - ops_enqd; + ops_enqd_total += ops_enqd; + + + /* Dequeue processed burst of ops from crypto device */ + ops_deqd = rte_cryptodev_dequeue_burst(ctx->dev_id, ctx->qp_id, + ops_processed, test_burst_size); + + if (likely(ops_deqd)) { + /* Free crypto ops so they can be reused. */ + rte_mempool_put_bulk(ctx->pool, + (void **)ops_processed, ops_deqd); + + ops_deqd_total += ops_deqd; + } else { + /** + * Count dequeue polls which didn't return any + * processed operations. This statistic is mainly + * relevant to hw accelerators. + */ + ops_deqd_failed++; + } + + } + + /* Dequeue any operations still in the crypto device */ + + while (ops_deqd_total < ctx->options->total_ops) { + /* Sending 0 length burst to flush sw crypto device */ + rte_cryptodev_enqueue_burst(ctx->dev_id, ctx->qp_id, NULL, 0); + + /* dequeue burst */ + ops_deqd = rte_cryptodev_dequeue_burst(ctx->dev_id, ctx->qp_id, + ops_processed, test_burst_size); + if (ops_deqd == 0) + ops_deqd_failed++; + else { + rte_mempool_put_bulk(ctx->pool, + (void **)ops_processed, ops_deqd); + ops_deqd_total += ops_deqd; + } + } + + tsc_end = rte_rdtsc_precise(); + tsc_duration = (tsc_end - tsc_start); + + /* Calculate average operations processed per second */ + double ops_per_second = ((double)ctx->options->total_ops / + tsc_duration) * rte_get_tsc_hz(); + + /* Calculate average throughput (Gbps) in bits per second */ + double throughput_gbps = ((ops_per_second * + ctx->options->test_buffer_size * 8) / 1000000000); + + /* Calculate average cycles per packet */ + double cycles_per_packet = ((double)tsc_duration / + ctx->options->total_ops); + + if (!ctx->options->csv) { + if (!only_once) + printf("%12s%12s%12s%12s%12s%12s%12s%12s%12s%12s\n\n", + "lcore id", "Buf Size", "Burst Size", + "Enqueued", "Dequeued", "Failed Enq", + "Failed Deq", "MOps", "Gbps", + "Cycles/Buf"); + only_once = 1; + + printf("%12u%12u%12u%12"PRIu64"%12"PRIu64"%12"PRIu64 + "%12"PRIu64"%12.4f%12.4f%12.2f\n", + ctx->lcore_id, + ctx->options->test_buffer_size, + test_burst_size, + ops_enqd_total, + ops_deqd_total, + ops_enqd_failed, + ops_deqd_failed, + ops_per_second/1000000, + throughput_gbps, + cycles_per_packet); + } else { + if (!only_once) + printf("#lcore id,Buffer Size(B)," + "Burst Size,Enqueued,Dequeued,Failed Enq," + "Failed Deq,Ops(Millions),Throughput(Gbps)," + "Cycles/Buf\n\n"); + only_once = 1; + + printf("%u;%u;%u;%"PRIu64";%"PRIu64";%"PRIu64";%"PRIu64";" + "%.3f;%.3f;%.3f\n", + ctx->lcore_id, + ctx->options->test_buffer_size, + test_burst_size, + ops_enqd_total, + ops_deqd_total, + ops_enqd_failed, + ops_deqd_failed, + ops_per_second/1000000, + throughput_gbps, + cycles_per_packet); + } + + /* Get next size from range or list */ + if (ctx->options->inc_burst_size != 0) + test_burst_size += ctx->options->inc_burst_size; + else { + if (++burst_size_idx == ctx->options->burst_size_count) + break; + test_burst_size = ctx->options->burst_size_list[burst_size_idx]; + } + + } + + return 0; +} + + +void +cperf_throughput_test_destructor(void *arg) +{ + struct cperf_throughput_ctx *ctx = arg; + + if (ctx == NULL) + return; + + cperf_throughput_test_free(ctx); +} diff --git a/dpdk/app/test-crypto-perf/cperf_test_throughput.h b/dpdk/app/test-crypto-perf/cperf_test_throughput.h new file mode 100644 index 00000000..987d0c31 --- /dev/null +++ b/dpdk/app/test-crypto-perf/cperf_test_throughput.h @@ -0,0 +1,61 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _CPERF_THROUGHPUT_ +#define _CPERF_THROUGHPUT_ + +#include + +#include + +#include "cperf.h" +#include "cperf_ops.h" +#include "cperf_options.h" +#include "cperf_test_vectors.h" + + +void * +cperf_throughput_test_constructor( + struct rte_mempool *sess_mp, + uint8_t dev_id, + uint16_t qp_id, + const struct cperf_options *options, + const struct cperf_test_vector *test_vector, + const struct cperf_op_fns *ops_fn); + +int +cperf_throughput_test_runner(void *test_ctx); + +void +cperf_throughput_test_destructor(void *test_ctx); + +#endif /* _CPERF_THROUGHPUT_ */ diff --git a/dpdk/app/test-crypto-perf/cperf_test_vector_parsing.c b/dpdk/app/test-crypto-perf/cperf_test_vector_parsing.c new file mode 100644 index 00000000..d4736f9e --- /dev/null +++ b/dpdk/app/test-crypto-perf/cperf_test_vector_parsing.c @@ -0,0 +1,620 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifdef RTE_EXEC_ENV_BSDAPP + #define _WITH_GETLINE +#endif +#include + +#include + +#include "cperf_options.h" +#include "cperf_test_vectors.h" +#include "cperf_test_vector_parsing.h" + +int +free_test_vector(struct cperf_test_vector *vector, struct cperf_options *opts) +{ + if (vector == NULL || opts == NULL) + return -1; + + rte_free(vector->cipher_iv.data); + rte_free(vector->auth_iv.data); + rte_free(vector->aad.data); + rte_free(vector->digest.data); + + if (opts->test_file != NULL) { + rte_free(vector->plaintext.data); + rte_free(vector->cipher_key.data); + rte_free(vector->auth_key.data); + rte_free(vector->ciphertext.data); + } + + rte_free(vector); + + return 0; +} + +void +show_test_vector(struct cperf_test_vector *test_vector) +{ + const uint8_t wrap = 32; + uint32_t i; + + if (test_vector == NULL) + return; + + if (test_vector->plaintext.data) { + printf("\nplaintext =\n"); + for (i = 0; i < test_vector->plaintext.length; ++i) { + if ((i % wrap == 0) && (i != 0)) + printf("\n"); + if (i == test_vector->plaintext.length - 1) + printf("0x%02x", + test_vector->plaintext.data[i]); + else + printf("0x%02x, ", + test_vector->plaintext.data[i]); + } + printf("\n"); + } + + if (test_vector->cipher_key.data) { + printf("\ncipher_key =\n"); + for (i = 0; i < test_vector->cipher_key.length; ++i) { + if ((i % wrap == 0) && (i != 0)) + printf("\n"); + if (i == (uint32_t)(test_vector->cipher_key.length - 1)) + printf("0x%02x", + test_vector->cipher_key.data[i]); + else + printf("0x%02x, ", + test_vector->cipher_key.data[i]); + } + printf("\n"); + } + + if (test_vector->auth_key.data) { + printf("\nauth_key =\n"); + for (i = 0; i < test_vector->auth_key.length; ++i) { + if ((i % wrap == 0) && (i != 0)) + printf("\n"); + if (i == (uint32_t)(test_vector->auth_key.length - 1)) + printf("0x%02x", test_vector->auth_key.data[i]); + else + printf("0x%02x, ", + test_vector->auth_key.data[i]); + } + printf("\n"); + } + + if (test_vector->aead_key.data) { + printf("\naead_key =\n"); + for (i = 0; i < test_vector->aead_key.length; ++i) { + if ((i % wrap == 0) && (i != 0)) + printf("\n"); + if (i == (uint32_t)(test_vector->aead_key.length - 1)) + printf("0x%02x", test_vector->aead_key.data[i]); + else + printf("0x%02x, ", + test_vector->aead_key.data[i]); + } + printf("\n"); + } + + if (test_vector->cipher_iv.data) { + printf("\ncipher_iv =\n"); + for (i = 0; i < test_vector->cipher_iv.length; ++i) { + if ((i % wrap == 0) && (i != 0)) + printf("\n"); + if (i == (uint32_t)(test_vector->cipher_iv.length - 1)) + printf("0x%02x", test_vector->cipher_iv.data[i]); + else + printf("0x%02x, ", test_vector->cipher_iv.data[i]); + } + printf("\n"); + } + + if (test_vector->auth_iv.data) { + printf("\nauth_iv =\n"); + for (i = 0; i < test_vector->auth_iv.length; ++i) { + if ((i % wrap == 0) && (i != 0)) + printf("\n"); + if (i == (uint32_t)(test_vector->auth_iv.length - 1)) + printf("0x%02x", test_vector->auth_iv.data[i]); + else + printf("0x%02x, ", test_vector->auth_iv.data[i]); + } + printf("\n"); + } + + if (test_vector->aead_iv.data) { + printf("\naead_iv =\n"); + for (i = 0; i < test_vector->aead_iv.length; ++i) { + if ((i % wrap == 0) && (i != 0)) + printf("\n"); + if (i == (uint32_t)(test_vector->aead_iv.length - 1)) + printf("0x%02x", test_vector->aead_iv.data[i]); + else + printf("0x%02x, ", test_vector->aead_iv.data[i]); + } + printf("\n"); + } + + if (test_vector->ciphertext.data) { + printf("\nciphertext =\n"); + for (i = 0; i < test_vector->ciphertext.length; ++i) { + if ((i % wrap == 0) && (i != 0)) + printf("\n"); + if (i == test_vector->ciphertext.length - 1) + printf("0x%02x", + test_vector->ciphertext.data[i]); + else + printf("0x%02x, ", + test_vector->ciphertext.data[i]); + } + printf("\n"); + } + + if (test_vector->aad.data) { + printf("\naad =\n"); + for (i = 0; i < test_vector->aad.length; ++i) { + if ((i % wrap == 0) && (i != 0)) + printf("\n"); + if (i == (uint32_t)(test_vector->aad.length - 1)) + printf("0x%02x", test_vector->aad.data[i]); + else + printf("0x%02x, ", test_vector->aad.data[i]); + } + printf("\n"); + } + + if (test_vector->digest.data) { + printf("\ndigest =\n"); + for (i = 0; i < test_vector->digest.length; ++i) { + if ((i % wrap == 0) && (i != 0)) + printf("\n"); + if (i == (uint32_t)(test_vector->digest.length - 1)) + printf("0x%02x", test_vector->digest.data[i]); + else + printf("0x%02x, ", test_vector->digest.data[i]); + } + printf("\n"); + } +} + +/* trim leading and trailing spaces */ +static char * +trim_space(char *str) +{ + char *start, *end; + + for (start = str; *start; start++) { + if (!isspace((unsigned char) start[0])) + break; + } + + for (end = start + strlen(start); end > start + 1; end--) { + if (!isspace((unsigned char) end[-1])) + break; + } + + *end = 0; + + /* Shift from "start" to the beginning of the string */ + if (start > str) + memmove(str, start, (end - start) + 1); + + return str; +} + +/* tokenization test values separated by a comma */ +static int +parse_values(char *tokens, uint8_t **data, uint32_t *data_length) +{ + uint32_t n_tokens; + uint32_t data_size = 32; + + uint8_t *values, *values_resized; + char *tok, *error = NULL; + + tok = strtok(tokens, CPERF_VALUE_DELIMITER); + if (tok == NULL) + return -1; + + values = (uint8_t *) rte_zmalloc(NULL, sizeof(uint8_t) * data_size, 0); + if (values == NULL) + return -1; + + n_tokens = 0; + while (tok != NULL) { + values_resized = NULL; + + if (n_tokens >= data_size) { + data_size *= 2; + + values_resized = (uint8_t *) rte_realloc(values, + sizeof(uint8_t) * data_size, 0); + if (values_resized == NULL) { + rte_free(values); + return -1; + } + values = values_resized; + } + + values[n_tokens] = (uint8_t) strtoul(tok, &error, 0); + if ((error == NULL) || (*error != '\0')) { + printf("Failed with convert '%s'\n", tok); + rte_free(values); + return -1; + } + + tok = strtok(NULL, CPERF_VALUE_DELIMITER); + if (tok == NULL) + break; + + n_tokens++; + } + + values_resized = (uint8_t *) rte_realloc(values, + sizeof(uint8_t) * (n_tokens + 1), 0); + + if (values_resized == NULL) { + rte_free(values); + return -1; + } + + *data = values_resized; + *data_length = n_tokens + 1; + + return 0; +} + +/* checks the type of key and assigns data */ +static int +parse_entry(char *entry, struct cperf_test_vector *vector, + struct cperf_options *opts, uint8_t tc_found) +{ + int status; + uint32_t data_length; + + uint8_t *data = NULL; + char *token, *key_token; + + if (entry == NULL) { + printf("Expected entry value\n"); + return -1; + } + + /* get key */ + token = strtok(entry, CPERF_ENTRY_DELIMITER); + key_token = token; + /* get values for key */ + token = strtok(NULL, CPERF_ENTRY_DELIMITER); + + if (key_token == NULL || token == NULL) { + printf("Expected 'key = values' but was '%.40s'..\n", entry); + return -1; + } + + status = parse_values(token, &data, &data_length); + if (status) + return -1; + + /* compare keys */ + if (strstr(key_token, "plaintext")) { + rte_free(vector->plaintext.data); + vector->plaintext.data = data; + if (tc_found) + vector->plaintext.length = data_length; + else { + if (opts->max_buffer_size > data_length) { + printf("Global plaintext shorter than " + "buffer_sz\n"); + return -1; + } + vector->plaintext.length = opts->max_buffer_size; + } + + } else if (strstr(key_token, "cipher_key")) { + rte_free(vector->cipher_key.data); + vector->cipher_key.data = data; + if (tc_found) + vector->cipher_key.length = data_length; + else { + if (opts->cipher_key_sz > data_length) { + printf("Global cipher_key shorter than " + "cipher_key_sz\n"); + return -1; + } + vector->cipher_key.length = opts->cipher_key_sz; + } + + } else if (strstr(key_token, "auth_key")) { + rte_free(vector->auth_key.data); + vector->auth_key.data = data; + if (tc_found) + vector->auth_key.length = data_length; + else { + if (opts->auth_key_sz > data_length) { + printf("Global auth_key shorter than " + "auth_key_sz\n"); + return -1; + } + vector->auth_key.length = opts->auth_key_sz; + } + + } else if (strstr(key_token, "aead_key")) { + rte_free(vector->aead_key.data); + vector->aead_key.data = data; + if (tc_found) + vector->aead_key.length = data_length; + else { + if (opts->aead_key_sz > data_length) { + printf("Global aead_key shorter than " + "aead_key_sz\n"); + return -1; + } + vector->aead_key.length = opts->aead_key_sz; + } + + } else if (strstr(key_token, "cipher_iv")) { + rte_free(vector->cipher_iv.data); + vector->cipher_iv.data = data; + if (tc_found) + vector->cipher_iv.length = data_length; + else { + if (opts->cipher_iv_sz > data_length) { + printf("Global cipher iv shorter than " + "cipher_iv_sz\n"); + return -1; + } + vector->cipher_iv.length = opts->cipher_iv_sz; + } + + } else if (strstr(key_token, "auth_iv")) { + rte_free(vector->auth_iv.data); + vector->auth_iv.data = data; + if (tc_found) + vector->auth_iv.length = data_length; + else { + if (opts->auth_iv_sz > data_length) { + printf("Global auth iv shorter than " + "auth_iv_sz\n"); + return -1; + } + vector->auth_iv.length = opts->auth_iv_sz; + } + + } else if (strstr(key_token, "aead_iv")) { + rte_free(vector->aead_iv.data); + vector->aead_iv.data = data; + if (tc_found) + vector->aead_iv.length = data_length; + else { + if (opts->aead_iv_sz > data_length) { + printf("Global aead iv shorter than " + "aead_iv_sz\n"); + return -1; + } + vector->aead_iv.length = opts->aead_iv_sz; + } + + } else if (strstr(key_token, "ciphertext")) { + rte_free(vector->ciphertext.data); + vector->ciphertext.data = data; + if (tc_found) + vector->ciphertext.length = data_length; + else { + if (opts->max_buffer_size > data_length) { + printf("Global ciphertext shorter than " + "buffer_sz\n"); + return -1; + } + vector->ciphertext.length = opts->max_buffer_size; + } + + } else if (strstr(key_token, "aad")) { + rte_free(vector->aad.data); + vector->aad.data = data; + vector->aad.phys_addr = rte_malloc_virt2iova(vector->aad.data); + if (tc_found) + vector->aad.length = data_length; + else { + if (opts->aead_aad_sz > data_length) { + printf("Global aad shorter than " + "aead_aad_sz\n"); + return -1; + } + vector->aad.length = opts->aead_aad_sz; + } + + } else if (strstr(key_token, "digest")) { + rte_free(vector->digest.data); + vector->digest.data = data; + vector->digest.phys_addr = rte_malloc_virt2iova( + vector->digest.data); + if (tc_found) + vector->digest.length = data_length; + else { + if (opts->digest_sz > data_length) { + printf("Global digest shorter than " + "digest_sz\n"); + return -1; + } + vector->digest.length = opts->digest_sz; + } + } else { + printf("Not valid key: '%s'\n", trim_space(key_token)); + return -1; + } + + return 0; +} + +/* searches in the file for test keys and values */ +static int +parse_file(struct cperf_test_vector *vector, struct cperf_options *opts) +{ + uint8_t tc_found = 0; + uint8_t tc_data_start = 0; + ssize_t read; + size_t len = 0; + int status = 0; + + FILE *fp; + char *line = NULL; + char *entry = NULL; + + fp = fopen(opts->test_file, "r"); + if (fp == NULL) { + printf("File %s does not exists\n", opts->test_file); + return -1; + } + + while ((read = getline(&line, &len, fp)) != -1) { + + /* ignore comments and new lines */ + if (line[0] == '#' || line[0] == '/' || line[0] == '\n' + || line[0] == '\r' || line[0] == ' ') + continue; + + trim_space(line); + + /* next test case is started */ + if (line[0] == '[' && line[strlen(line) - 1] == ']' && tc_found) + break; + /* test case section started, end of global data */ + else if (line[0] == '[' && line[strlen(line) - 1] == ']') + tc_data_start = 1; + + /* test name unspecified, end after global data */ + if (tc_data_start && opts->test_name == NULL) + break; + /* searching for a suitable test */ + else if (tc_data_start && tc_found == 0) { + if (!strcmp(line, opts->test_name)) { + tc_found = 1; + continue; + } else + continue; + } + + /* buffer for multiline */ + entry = (char *) rte_realloc(entry, + sizeof(char) * strlen(line) + 1, 0); + if (entry == NULL) + return -1; + + memset(entry, 0, strlen(line) + 1); + strncpy(entry, line, strlen(line)); + + /* check if entry ends with , or = */ + if (entry[strlen(entry) - 1] == ',' + || entry[strlen(entry) - 1] == '=') { + while ((read = getline(&line, &len, fp)) != -1) { + trim_space(line); + + /* extend entry about length of new line */ + char *entry_extended = (char *) rte_realloc( + entry, sizeof(char) + * (strlen(line) + strlen(entry)) + + 1, 0); + + if (entry_extended == NULL) + goto err; + entry = entry_extended; + + strncat(entry, line, strlen(line)); + + if (entry[strlen(entry) - 1] != ',') + break; + } + } + status = parse_entry(entry, vector, opts, tc_found); + if (status) { + printf("An error occurred while parsing!\n"); + goto err; + } + } + + if (tc_found == 0 && opts->test_name != NULL) { + printf("Not found '%s' case in test file\n", opts->test_name); + goto err; + } + + fclose(fp); + free(line); + rte_free(entry); + + return 0; + +err: + if (fp) + fclose(fp); + if (line) + free(line); + if (entry) + rte_free(entry); + + return -1; +} + +struct cperf_test_vector* +cperf_test_vector_get_from_file(struct cperf_options *opts) +{ + int status; + struct cperf_test_vector *test_vector = NULL; + + if (opts == NULL || opts->test_file == NULL) + return test_vector; + + test_vector = (struct cperf_test_vector *) rte_zmalloc(NULL, + sizeof(struct cperf_test_vector), 0); + if (test_vector == NULL) + return test_vector; + + /* filling the vector with data from a file */ + status = parse_file(test_vector, opts); + if (status) { + free_test_vector(test_vector, opts); + return NULL; + } + + /* other values not included in the file */ + test_vector->data.cipher_offset = 0; + test_vector->data.cipher_length = opts->max_buffer_size; + + test_vector->data.auth_offset = 0; + test_vector->data.auth_length = opts->max_buffer_size; + + return test_vector; +} diff --git a/dpdk/app/test-crypto-perf/cperf_test_vector_parsing.h b/dpdk/app/test-crypto-perf/cperf_test_vector_parsing.h new file mode 100644 index 00000000..e3df98bd --- /dev/null +++ b/dpdk/app/test-crypto-perf/cperf_test_vector_parsing.h @@ -0,0 +1,73 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef APP_CRYPTO_PERF_CPERF_TEST_VECTOR_PARSING_H_ +#define APP_CRYPTO_PERF_CPERF_TEST_VECTOR_PARSING_H_ + +#define CPERF_VALUE_DELIMITER "," +#define CPERF_ENTRY_DELIMITER "=" + +/** + * Frees the allocated memory for test vector + * + * @param vector + * Destination vector test to release + * @param opts + * Test options + * @return + * 0 on success, (-1) on error. + */ +int +free_test_vector(struct cperf_test_vector *vector, struct cperf_options *opts); + +/** + * Displays data in test vector + * + * @param vector + * Vector to display + */ +void +show_test_vector(struct cperf_test_vector *test_vector); + +/** + * Completes test vector with data from file + * + * @param opts + * Test options + * @return + * NULL on error. + * Test vector pointer on successful. + */ +struct cperf_test_vector* +cperf_test_vector_get_from_file(struct cperf_options *opts); + +#endif /* APP_CRYPTO_PERF_CPERF_TEST_VECTOR_PARSING_H_ */ diff --git a/dpdk/app/test-crypto-perf/cperf_test_vectors.c b/dpdk/app/test-crypto-perf/cperf_test_vectors.c new file mode 100644 index 00000000..fa911ff6 --- /dev/null +++ b/dpdk/app/test-crypto-perf/cperf_test_vectors.c @@ -0,0 +1,568 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include + +#include "cperf_test_vectors.h" + +uint8_t plaintext[2048] = { + 0x71, 0x75, 0x83, 0x98, 0x75, 0x42, 0x51, 0x09, 0x94, 0x02, 0x13, 0x20, + 0x15, 0x64, 0x46, 0x32, 0x08, 0x18, 0x91, 0x82, 0x86, 0x52, 0x23, 0x93, + 0x44, 0x54, 0x28, 0x68, 0x78, 0x78, 0x70, 0x06, 0x42, 0x74, 0x41, 0x27, + 0x73, 0x38, 0x53, 0x77, 0x51, 0x96, 0x53, 0x24, 0x03, 0x88, 0x74, 0x14, + 0x70, 0x23, 0x88, 0x30, 0x85, 0x18, 0x89, 0x27, 0x41, 0x71, 0x61, 0x23, + 0x04, 0x83, 0x30, 0x57, 0x26, 0x47, 0x23, 0x75, 0x25, 0x62, 0x53, 0x80, + 0x38, 0x34, 0x21, 0x33, 0x34, 0x51, 0x46, 0x29, 0x94, 0x64, 0x22, 0x67, + 0x25, 0x45, 0x70, 0x26, 0x74, 0x39, 0x46, 0x71, 0x08, 0x85, 0x27, 0x18, + 0x93, 0x39, 0x72, 0x11, 0x57, 0x26, 0x88, 0x46, 0x47, 0x49, 0x86, 0x92, + 0x03, 0x37, 0x96, 0x40, 0x84, 0x53, 0x67, 0x47, 0x60, 0x60, 0x37, 0x67, + 0x02, 0x68, 0x76, 0x62, 0x42, 0x01, 0x59, 0x11, 0x01, 0x89, 0x40, 0x87, + 0x58, 0x20, 0x51, 0x21, 0x66, 0x26, 0x26, 0x73, 0x03, 0x06, 0x14, 0x25, + 0x98, 0x42, 0x44, 0x67, 0x24, 0x78, 0x71, 0x45, 0x32, 0x61, 0x20, 0x26, + 0x08, 0x88, 0x44, 0x26, 0x40, 0x63, 0x76, 0x23, 0x78, 0x55, 0x81, 0x97, + 0x95, 0x89, 0x39, 0x07, 0x14, 0x50, 0x50, 0x73, 0x07, 0x20, 0x86, 0x83, + 0x74, 0x57, 0x72, 0x36, 0x68, 0x61, 0x14, 0x41, 0x56, 0x49, 0x64, 0x72, + 0x75, 0x81, 0x47, 0x91, 0x08, 0x76, 0x47, 0x06, 0x55, 0x77, 0x61, 0x45, + 0x50, 0x10, 0x07, 0x46, 0x46, 0x89, 0x80, 0x07, 0x24, 0x95, 0x39, 0x43, + 0x03, 0x75, 0x24, 0x35, 0x57, 0x82, 0x09, 0x64, 0x29, 0x24, 0x26, 0x66, + 0x67, 0x29, 0x05, 0x90, 0x82, 0x02, 0x45, 0x71, 0x21, 0x34, 0x25, 0x48, + 0x68, 0x26, 0x01, 0x18, 0x73, 0x18, 0x46, 0x15, 0x14, 0x33, 0x28, 0x44, + 0x24, 0x82, 0x20, 0x12, 0x99, 0x43, 0x68, 0x43, 0x25, 0x14, 0x34, 0x33, + 0x31, 0x13, 0x77, 0x44, 0x95, 0x22, 0x99, 0x02, 0x30, 0x50, 0x74, 0x43, + 0x81, 0x78, 0x32, 0x17, 0x09, 0x85, 0x04, 0x37, 0x31, 0x98, 0x76, 0x79, + 0x64, 0x10, 0x39, 0x89, 0x59, 0x90, 0x50, 0x15, 0x77, 0x39, 0x28, 0x14, + 0x30, 0x19, 0x68, 0x77, 0x89, 0x48, 0x86, 0x16, 0x11, 0x33, 0x84, 0x56, + 0x10, 0x20, 0x94, 0x72, 0x41, 0x69, 0x13, 0x00, 0x56, 0x27, 0x01, 0x57, + 0x46, 0x65, 0x65, 0x19, 0x33, 0x07, 0x62, 0x19, 0x91, 0x60, 0x29, 0x11, + 0x41, 0x25, 0x88, 0x21, 0x93, 0x85, 0x87, 0x40, 0x91, 0x25, 0x32, 0x86, + 0x76, 0x54, 0x92, 0x52, 0x72, 0x46, 0x61, 0x84, 0x20, 0x14, 0x65, 0x83, + 0x69, 0x90, 0x80, 0x11, 0x35, 0x70, 0x42, 0x64, 0x74, 0x85, 0x15, 0x23, + 0x06, 0x55, 0x67, 0x49, 0x76, 0x47, 0x11, 0x95, 0x00, 0x85, 0x05, 0x12, + 0x58, 0x53, 0x25, 0x73, 0x62, 0x81, 0x63, 0x82, 0x32, 0x75, 0x16, 0x48, + 0x04, 0x96, 0x75, 0x16, 0x43, 0x83, 0x41, 0x85, 0x95, 0x67, 0x27, 0x83, + 0x22, 0x43, 0x02, 0x27, 0x69, 0x62, 0x78, 0x50, 0x57, 0x66, 0x99, 0x89, + 0x05, 0x06, 0x35, 0x86, 0x37, 0x27, 0x48, 0x46, 0x50, 0x80, 0x96, 0x40, + 0x42, 0x36, 0x21, 0x54, 0x49, 0x18, 0x63, 0x38, 0x45, 0x76, 0x23, 0x20, + 0x28, 0x06, 0x17, 0x32, 0x58, 0x50, 0x49, 0x54, 0x29, 0x46, 0x18, 0x12, + 0x17, 0x50, 0x02, 0x80, 0x99, 0x53, 0x15, 0x02, 0x07, 0x14, 0x19, 0x60, + 0x56, 0x43, 0x76, 0x71, 0x49, 0x99, 0x54, 0x83, 0x28, 0x94, 0x30, 0x30, + 0x57, 0x05, 0x89, 0x80, 0x11, 0x03, 0x78, 0x35, 0x73, 0x52, 0x67, 0x39, + 0x67, 0x07, 0x04, 0x49, 0x23, 0x83, 0x86, 0x89, 0x57, 0x71, 0x08, 0x41, + 0x15, 0x97, 0x19, 0x72, 0x03, 0x27, 0x72, 0x52, 0x66, 0x67, 0x99, 0x15, + 0x33, 0x64, 0x69, 0x78, 0x07, 0x83, 0x53, 0x71, 0x21, 0x50, 0x05, 0x48, + 0x59, 0x85, 0x01, 0x36, 0x65, 0x02, 0x52, 0x01, 0x09, 0x49, 0x28, 0x77, + 0x25, 0x35, 0x67, 0x77, 0x81, 0x64, 0x24, 0x29, 0x42, 0x32, 0x59, 0x22, + 0x93, 0x48, 0x59, 0x03, 0x85, 0x87, 0x15, 0x55, 0x23, 0x42, 0x58, 0x17, + 0x18, 0x37, 0x70, 0x83, 0x80, 0x12, 0x44, 0x83, 0x45, 0x70, 0x55, 0x86, + 0x03, 0x23, 0x01, 0x56, 0x94, 0x12, 0x41, 0x34, 0x82, 0x90, 0x83, 0x46, + 0x17, 0x56, 0x66, 0x96, 0x75, 0x80, 0x59, 0x07, 0x15, 0x84, 0x19, 0x52, + 0x37, 0x44, 0x44, 0x83, 0x72, 0x43, 0x25, 0x42, 0x26, 0x86, 0x87, 0x86, + 0x91, 0x62, 0x14, 0x90, 0x34, 0x26, 0x14, 0x33, 0x59, 0x70, 0x73, 0x15, + 0x49, 0x40, 0x66, 0x88, 0x42, 0x66, 0x16, 0x42, 0x55, 0x92, 0x82, 0x06, + 0x20, 0x96, 0x36, 0x96, 0x13, 0x07, 0x84, 0x94, 0x37, 0x66, 0x62, 0x78, + 0x60, 0x58, 0x80, 0x50, 0x69, 0x03, 0x97, 0x16, 0x64, 0x45, 0x21, 0x39, + 0x79, 0x28, 0x52, 0x17, 0x14, 0x77, 0x31, 0x60, 0x86, 0x70, 0x09, 0x53, + 0x39, 0x32, 0x52, 0x31, 0x35, 0x79, 0x24, 0x70, 0x25, 0x48, 0x23, 0x49, + 0x10, 0x64, 0x54, 0x30, 0x82, 0x34, 0x51, 0x20, 0x46, 0x04, 0x29, 0x25, + 0x65, 0x09, 0x55, 0x30, 0x30, 0x52, 0x85, 0x32, 0x79, 0x19, 0x59, 0x07, + 0x05, 0x12, 0x11, 0x03, 0x21, 0x90, 0x36, 0x62, 0x23, 0x67, 0x36, 0x67, + 0x47, 0x39, 0x92, 0x88, 0x45, 0x43, 0x71, 0x16, 0x48, 0x27, 0x68, 0x39, + 0x98, 0x38, 0x03, 0x31, 0x85, 0x10, 0x06, 0x95, 0x54, 0x79, 0x28, 0x79, + 0x56, 0x16, 0x65, 0x69, 0x00, 0x54, 0x09, 0x91, 0x06, 0x10, 0x10, 0x86, + 0x75, 0x01, 0x02, 0x71, 0x01, 0x09, 0x32, 0x94, 0x66, 0x43, 0x68, 0x36, + 0x19, 0x52, 0x02, 0x04, 0x45, 0x49, 0x40, 0x94, 0x07, 0x87, 0x86, 0x79, + 0x84, 0x07, 0x75, 0x30, 0x73, 0x02, 0x57, 0x81, 0x65, 0x02, 0x28, 0x96, + 0x57, 0x07, 0x70, 0x34, 0x39, 0x35, 0x75, 0x19, 0x47, 0x57, 0x08, 0x75, + 0x86, 0x57, 0x11, 0x32, 0x09, 0x47, 0x83, 0x93, 0x20, 0x94, 0x90, 0x88, + 0x39, 0x63, 0x22, 0x88, 0x54, 0x54, 0x95, 0x75, 0x67, 0x26, 0x02, 0x49, + 0x26, 0x17, 0x35, 0x16, 0x27, 0x65, 0x64, 0x26, 0x93, 0x92, 0x77, 0x85, + 0x84, 0x40, 0x59, 0x29, 0x49, 0x69, 0x94, 0x71, 0x72, 0x21, 0x55, 0x03, + 0x19, 0x74, 0x09, 0x40, 0x57, 0x68, 0x41, 0x19, 0x11, 0x21, 0x63, 0x56, + 0x29, 0x77, 0x57, 0x81, 0x44, 0x40, 0x76, 0x77, 0x02, 0x71, 0x66, 0x35, + 0x89, 0x02, 0x64, 0x51, 0x61, 0x02, 0x46, 0x91, 0x38, 0x93, 0x62, 0x57, + 0x18, 0x98, 0x12, 0x87, 0x29, 0x48, 0x65, 0x39, 0x99, 0x45, 0x54, 0x69, + 0x51, 0x16, 0x25, 0x75, 0x60, 0x70, 0x33, 0x72, 0x01, 0x60, 0x26, 0x51, + 0x44, 0x14, 0x39, 0x12, 0x95, 0x48, 0x87, 0x33, 0x90, 0x16, 0x42, 0x78, + 0x48, 0x58, 0x96, 0x93, 0x75, 0x23, 0x07, 0x13, 0x86, 0x07, 0x96, 0x30, + 0x22, 0x82, 0x91, 0x36, 0x72, 0x16, 0x48, 0x77, 0x64, 0x99, 0x07, 0x34, + 0x78, 0x60, 0x61, 0x13, 0x48, 0x93, 0x46, 0x62, 0x48, 0x38, 0x37, 0x96, + 0x58, 0x64, 0x39, 0x90, 0x69, 0x46, 0x81, 0x98, 0x61, 0x89, 0x15, 0x59, + 0x78, 0x98, 0x21, 0x34, 0x00, 0x69, 0x97, 0x80, 0x28, 0x81, 0x53, 0x49, + 0x79, 0x53, 0x92, 0x20, 0x29, 0x40, 0x70, 0x06, 0x09, 0x55, 0x99, 0x41, + 0x51, 0x35, 0x55, 0x27, 0x39, 0x06, 0x29, 0x83, 0x66, 0x03, 0x68, 0x14, + 0x11, 0x69, 0x95, 0x51, 0x71, 0x55, 0x24, 0x60, 0x52, 0x58, 0x88, 0x11, + 0x88, 0x25, 0x37, 0x86, 0x01, 0x52, 0x93, 0x52, 0x02, 0x24, 0x91, 0x58, + 0x56, 0x37, 0x50, 0x88, 0x39, 0x09, 0x61, 0x19, 0x08, 0x86, 0x29, 0x51, + 0x63, 0x38, 0x81, 0x14, 0x75, 0x75, 0x39, 0x99, 0x22, 0x04, 0x32, 0x63, + 0x14, 0x68, 0x41, 0x79, 0x09, 0x57, 0x87, 0x29, 0x26, 0x94, 0x05, 0x71, + 0x82, 0x41, 0x26, 0x98, 0x68, 0x18, 0x55, 0x42, 0x78, 0x05, 0x74, 0x17, + 0x34, 0x34, 0x07, 0x62, 0x94, 0x72, 0x21, 0x08, 0x54, 0x72, 0x21, 0x08, + 0x31, 0x53, 0x82, 0x35, 0x27, 0x40, 0x85, 0x77, 0x08, 0x52, 0x58, 0x48, + 0x03, 0x86, 0x65, 0x51, 0x96, 0x43, 0x89, 0x19, 0x15, 0x08, 0x49, 0x62, + 0x57, 0x46, 0x17, 0x68, 0x56, 0x04, 0x70, 0x63, 0x75, 0x88, 0x13, 0x27, + 0x87, 0x44, 0x46, 0x27, 0x02, 0x97, 0x71, 0x07, 0x40, 0x17, 0x24, 0x61, + 0x16, 0x94, 0x86, 0x85, 0x67, 0x58, 0x87, 0x92, 0x02, 0x84, 0x75, 0x19, + 0x43, 0x60, 0x68, 0x03, 0x54, 0x75, 0x33, 0x17, 0x97, 0x75, 0x12, 0x62, + 0x43, 0x08, 0x35, 0x75, 0x32, 0x21, 0x08, 0x82, 0x78, 0x04, 0x74, 0x09, + 0x13, 0x48, 0x63, 0x68, 0x67, 0x09, 0x08, 0x50, 0x11, 0x71, 0x64, 0x72, + 0x63, 0x76, 0x21, 0x62, 0x80, 0x57, 0x19, 0x15, 0x26, 0x88, 0x02, 0x26, + 0x83, 0x17, 0x61, 0x76, 0x28, 0x10, 0x22, 0x37, 0x56, 0x71, 0x51, 0x60, + 0x12, 0x79, 0x24, 0x83, 0x78, 0x47, 0x78, 0x20, 0x52, 0x27, 0x19, 0x88, + 0x81, 0x04, 0x70, 0x20, 0x25, 0x10, 0x04, 0x01, 0x72, 0x57, 0x30, 0x93, + 0x96, 0x23, 0x02, 0x94, 0x61, 0x44, 0x17, 0x65, 0x77, 0x60, 0x27, 0x43, + 0x24, 0x59, 0x46, 0x76, 0x00, 0x11, 0x31, 0x99, 0x41, 0x48, 0x75, 0x32, + 0x05, 0x15, 0x45, 0x31, 0x57, 0x89, 0x10, 0x47, 0x53, 0x14, 0x66, 0x54, + 0x60, 0x55, 0x36, 0x93, 0x30, 0x03, 0x63, 0x80, 0x65, 0x43, 0x17, 0x36, + 0x18, 0x64, 0x21, 0x38, 0x16, 0x19, 0x19, 0x51, 0x73, 0x80, 0x38, 0x27, + 0x30, 0x89, 0x13, 0x43, 0x54, 0x11, 0x78, 0x05, 0x24, 0x38, 0x83, 0x56, + 0x50, 0x59, 0x12, 0x47, 0x69, 0x70, 0x70, 0x91, 0x28, 0x02, 0x08, 0x91, + 0x66, 0x09, 0x31, 0x65, 0x46, 0x20, 0x04, 0x85, 0x89, 0x53, 0x91, 0x42, + 0x34, 0x09, 0x36, 0x92, 0x42, 0x06, 0x87, 0x88, 0x23, 0x54, 0x87, 0x85, + 0x52, 0x98, 0x95, 0x76, 0x13, 0x50, 0x59, 0x89, 0x18, 0x14, 0x17, 0x47, + 0x10, 0x97, 0x39, 0x14, 0x33, 0x79, 0x83, 0x62, 0x55, 0x18, 0x30, 0x83, + 0x03, 0x45, 0x38, 0x37, 0x35, 0x20, 0x94, 0x84, 0x89, 0x80, 0x89, 0x10, + 0x48, 0x77, 0x33, 0x36, 0x50, 0x07, 0x93, 0x02, 0x45, 0x42, 0x91, 0x12, + 0x98, 0x09, 0x77, 0x20, 0x31, 0x95, 0x10, 0x29, 0x89, 0x02, 0x38, 0x92, + 0x90, 0x19, 0x51, 0x10, 0x19, 0x82, 0x23, 0x68, 0x06, 0x00, 0x67, 0x50, + 0x25, 0x03, 0x41, 0x69, 0x53, 0x42, 0x23, 0x99, 0x29, 0x21, 0x63, 0x22, + 0x72, 0x54, 0x72, 0x40, 0x23, 0x39, 0x74, 0x92, 0x53, 0x28, 0x67, 0x56, + 0x46, 0x84, 0x59, 0x85, 0x10, 0x92, 0x31, 0x20, 0x39, 0x95, 0x65, 0x15, + 0x76, 0x35, 0x37, 0x21, 0x98, 0x41, 0x68, 0x74, 0x94, 0x94, 0x86, 0x90, + 0x35, 0x07, 0x06, 0x38, 0x78, 0x32, 0x00, 0x60, 0x86, 0x12, 0x34, 0x65, + 0x67, 0x35, 0x76, 0x94, 0x78, 0x22, 0x99, 0x42, 0x82, 0x40, 0x05, 0x74, + 0x18, 0x59, 0x03, 0x83, 0x89, 0x05, 0x19, 0x28, 0x88, 0x35, 0x59, 0x10, + 0x12, 0x96, 0x48, 0x67, 0x59, 0x87, 0x26, 0x85, 0x74, 0x64, 0x78, 0x56, + 0x91, 0x81, 0x45, 0x90, 0x21, 0x80, 0x32, 0x19, 0x61, 0x38, 0x61, 0x70, + 0x35, 0x08, 0x93, 0x53, 0x21, 0x95, 0x08, 0x27, 0x90, 0x28, 0x94, 0x27, + 0x35, 0x78, 0x03, 0x57, 0x74, 0x84, 0x73, 0x63, 0x27, 0x98, 0x14, 0x21, + 0x22, 0x36, 0x75, 0x31, 0x81, 0x65, 0x85, 0x51, 0x02, 0x45, 0x18, 0x06, + 0x39, 0x13, 0x29, 0x29, 0x73, 0x26, 0x99, 0x51, 0x38, 0x43, 0x35, 0x58, + 0x70, 0x92, 0x32, 0x13, 0x80, 0x16, 0x26, 0x44, 0x22, 0x28, 0x05, 0x45, + 0x86, 0x90, 0x38, 0x19, 0x40, 0x06, 0x30, 0x56, 0x94, 0x09, 0x02, 0x02, + 0x96, 0x29, 0x22, 0x44, 0x87, 0x38, 0x09, 0x95, 0x58, 0x46, 0x42, 0x78, + 0x72, 0x77, 0x86, 0x31, 0x97, 0x19, 0x86, 0x51, 0x73, 0x76, 0x63, 0x98, + 0x39, 0x40, 0x20, 0x20, 0x67, 0x42, 0x55, 0x50, 0x63, 0x76, 0x81, 0x87, + 0x13, 0x81, 0x19, 0x54, 0x11, 0x77, 0x90, 0x26, 0x47, 0x25, 0x92, 0x88, + 0x18, 0x56, 0x23, 0x73, 0x91, 0x52, 0x39, 0x08, 0x59, 0x51, 0x81, 0x57, + 0x78, 0x17, 0x13, 0x90, 0x90, 0x50, 0x65, 0x59, 0x99, 0x77, 0x42, 0x28, + 0x21, 0x59, 0x97, 0x64, 0x25, 0x17, 0x92, 0x24, 0x50, 0x00, 0x28, 0x40, + 0x85, 0x33, 0x78, 0x86, 0x79, 0x40, 0x28, 0x30, 0x14, 0x12, 0x01, 0x72, + 0x41, 0x43, 0x06, 0x87, 0x67, 0x31, 0x66, 0x77, 0x07, 0x50, 0x55, 0x50, + 0x22, 0x80, 0x42, 0x06, 0x38, 0x01, 0x63, 0x66, 0x70, 0x12, 0x52, 0x91, + 0x90, 0x97, 0x21, 0x28, 0x22, 0x65, 0x02, 0x80, 0x72, 0x31, 0x17, 0x76, + 0x35, 0x16, 0x03, 0x56, 0x59, 0x93, 0x36, 0x37, 0x67, 0x54, 0x46, 0x87, + 0x29, 0x01, 0x30, 0x80, 0x47, 0x47, 0x31, 0x98, 0x34, 0x30, 0x23, 0x86, + 0x86, 0x14, 0x05, 0x75, 0x09, 0x88, 0x77, 0x92, 0x59, 0x43, 0x98, 0x72, + 0x55, 0x54, 0x25, 0x59, 0x22, 0x27, 0x21, 0x62, 0x97, 0x10, 0x61, 0x73, + 0x86, 0x95, 0x99, 0x10, 0x62, 0x35, 0x25, 0x16, 0x62, 0x60, 0x51, 0x48, + 0x69, 0x69, 0x92, 0x27, 0x19, 0x43, 0x40, 0x52, 0x70, 0x23, 0x37, 0x28, + 0x73, 0x10, 0x32, 0x55, 0x85, 0x46, 0x97, 0x59, 0x88, 0x48, 0x54, 0x06, + 0x58, 0x04, 0x82, 0x98, 0x88, 0x34, 0x05, 0x41, 0x94, 0x44, 0x35, 0x10, + 0x96, 0x48, 0x21, 0x17, 0x24, 0x40, 0x26, 0x15, 0x49, 0x28, 0x12, 0x17, + 0x10, 0x17, 0x91, 0x42, 0x84, 0x15, 0x83, 0x36, 0x29, 0x49, 0x92, 0x77, + 0x74, 0x11, 0x72, 0x97, 0x64, 0x53, 0x23, 0x29, 0x16, 0x35, 0x22, 0x10, + 0x87, 0x07, 0x44, 0x78, 0x18, 0x19, 0x79, 0x03, 0x58, 0x24, 0x15, 0x63, + 0x55, 0x75, 0x56, 0x14, 0x63, 0x65, 0x86, 0x61, 0x92, 0x94, 0x30, 0x92, + 0x69, 0x78, 0x40, 0x95, 0x19, 0x81, 0x41, 0x66, 0x97, 0x00, 0x17, 0x37, + 0x20, 0x82, 0x14, 0x26, 0x42, 0x63, 0x84, 0x20, 0x96, 0x11, 0x68, 0x37, + 0x60, 0x28, 0x69, 0x85, 0x45, 0x04, 0x62, 0x20, 0x49, 0x39, 0x74, 0x84, + 0x60, 0x23, 0x38, 0x33, 0x42, 0x49, 0x38, 0x82, 0x30, 0x63, 0x21, 0x51, + 0x69, 0x09, 0x05, 0x55, 0x78, 0x90, 0x68, 0x69, 0x22, 0x20, 0x17, 0x26, + 0x54, 0x01, 0x10, 0x04, 0x68, 0x19, 0x88, 0x40, 0x91, 0x74, 0x81, 0x29, + 0x07, 0x45, 0x33, 0x77, 0x12, 0x47, 0x08, 0x60, 0x09, 0x42, 0x84, 0x15, + 0x63, 0x92, 0x64, 0x77, 0x07, 0x44, 0x11, 0x07, 0x79, 0x81, 0x24, 0x05, + 0x21, 0x60, 0x81, 0x70, 0x66, 0x36, 0x69, 0x68, 0x45, 0x01, 0x11, 0x95, + 0x67, 0x95, 0x55, 0x07, 0x96, 0x63, 0x84, 0x04, 0x74, 0x72, 0x61, 0x91, + 0x60, 0x09, 0x90, 0x14, 0x34, 0x94, 0x06, 0x12, 0x01, 0x94, 0x40, 0x14, + 0x12, 0x53, 0x64, 0x81, 0x75, 0x99, 0x36, 0x99, 0x11, 0x69, 0x95, 0x51, + 0x71, 0x55, 0x24, 0x60, 0x52, 0x58, 0x88, 0x11, 0x88, 0x25, 0x37, 0x86, + 0x66, 0x36, 0x69, 0x68, 0x45, 0x01, 0x11, 0x95 +}; + +/* cipher text */ +uint8_t ciphertext[2048] = { + 0xE2, 0x19, 0x24, 0x56, 0x13, 0x59, 0xA6, 0x5D, 0xDF, 0xD0, 0x72, 0xAA, + 0x23, 0xC7, 0x36, 0x3A, 0xBB, 0x3E, 0x8B, 0x64, 0xD5, 0xBF, 0xDE, 0x65, + 0xA2, 0x75, 0xD9, 0x45, 0x6C, 0x3C, 0xD2, 0x6A, 0xC7, 0xD0, 0x9A, 0xD0, + 0x87, 0xB8, 0xE4, 0x94, 0x11, 0x62, 0x5A, 0xC3, 0xC3, 0x01, 0xA3, 0x86, + 0xBC, 0xBC, 0x9C, 0xC0, 0x81, 0x9F, 0xBF, 0x5C, 0x6F, 0x3F, 0x13, 0xF1, + 0xAE, 0xCF, 0x26, 0xB3, 0xBC, 0x49, 0xD6, 0x3B, 0x7A, 0x2E, 0x99, 0x9E, + 0x1B, 0x04, 0x50, 0x6C, 0x48, 0x6B, 0x4E, 0x72, 0xFC, 0xC8, 0xA7, 0x0C, + 0x2C, 0xD9, 0xED, 0xE4, 0x82, 0xC4, 0x81, 0xA6, 0xB4, 0xCC, 0xAD, 0x10, + 0xF3, 0x1C, 0x39, 0x05, 0x41, 0x2D, 0x57, 0x32, 0xE7, 0x16, 0xF8, 0x4D, + 0xF0, 0xDE, 0x40, 0x5B, 0x5F, 0x80, 0xDC, 0xA7, 0xC3, 0x2D, 0x3D, 0x9E, + 0x27, 0xD4, 0xE8, 0x10, 0x8E, 0xEB, 0xA5, 0x68, 0x6F, 0x3D, 0xC0, 0x44, + 0xE7, 0x77, 0x73, 0xB9, 0x92, 0x8E, 0xA2, 0x26, 0x5C, 0x6F, 0x33, 0x4B, + 0x0B, 0xEF, 0x37, 0x55, 0xBE, 0xEC, 0x98, 0x83, 0x1E, 0xDF, 0xB2, 0x9E, + 0x5D, 0x1D, 0x78, 0x14, 0xD7, 0x85, 0x0E, 0xF8, 0x12, 0x30, 0x8E, 0x5D, + 0x08, 0x77, 0x0B, 0x2E, 0x9B, 0xF9, 0xA6, 0x72, 0xD2, 0x41, 0xC1, 0x8E, + 0x6B, 0x5E, 0x11, 0x85, 0x22, 0x6E, 0xE4, 0xA3, 0xEA, 0x4C, 0x91, 0xE1, + 0x7D, 0xD0, 0xEB, 0x9F, 0xD9, 0xD7, 0x05, 0x77, 0xD9, 0xA1, 0xC2, 0xFD, + 0x41, 0x63, 0x51, 0xB4, 0x7A, 0x1F, 0x21, 0xF0, 0xBF, 0x11, 0x4D, 0x9B, + 0x97, 0xAB, 0xB4, 0x94, 0x36, 0x34, 0xC9, 0x2D, 0x8B, 0xE2, 0x61, 0xCF, + 0xAF, 0x69, 0xD5, 0x5C, 0xE9, 0xED, 0xE3, 0xA0, 0x69, 0xD3, 0xE5, 0xAE, + 0x67, 0x6C, 0xC7, 0x11, 0xB1, 0x21, 0x96, 0xD6, 0xDB, 0xA8, 0x1D, 0xC9, + 0x83, 0x0B, 0xE2, 0xC6, 0x6E, 0x94, 0xE9, 0x50, 0x12, 0x9B, 0x01, 0x72, + 0xAA, 0xFD, 0x8B, 0x7C, 0xEC, 0x0D, 0x01, 0xA4, 0x5D, 0x00, 0xE9, 0x79, + 0x58, 0xF5, 0x67, 0xF9, 0x61, 0xC3, 0x11, 0xB4, 0x7E, 0x76, 0x0A, 0x4C, + 0x60, 0xD6, 0xBD, 0xC8, 0x31, 0xD3, 0x0C, 0xD0, 0x5B, 0xDF, 0x7B, 0x05, + 0x9A, 0xBB, 0xC6, 0x2E, 0x9F, 0xF8, 0x18, 0x80, 0x6D, 0x1B, 0x21, 0xE5, + 0xAC, 0x75, 0xBC, 0x0D, 0x72, 0x51, 0x61, 0xD7, 0xEA, 0xA2, 0xAC, 0x0E, + 0xC1, 0xE7, 0x49, 0x37, 0xE7, 0x7C, 0xDE, 0xBD, 0x56, 0x00, 0x44, 0x6D, + 0xAB, 0x81, 0x2B, 0x26, 0x4A, 0xAA, 0x60, 0xE6, 0x43, 0x8D, 0x88, 0x1C, + 0x48, 0x55, 0x53, 0x25, 0xE8, 0x3C, 0x46, 0xF0, 0xA6, 0x33, 0x2D, 0xA2, + 0xDC, 0x99, 0x57, 0x38, 0x59, 0xCF, 0x53, 0xFA, 0x3E, 0x78, 0x46, 0xA0, + 0xA9, 0x50, 0x12, 0x72, 0xAC, 0x15, 0xC6, 0xA7, 0x42, 0x0F, 0x59, 0x6E, + 0xEA, 0xB0, 0x3D, 0xB8, 0x94, 0x32, 0xD1, 0xB6, 0xE8, 0x90, 0x06, 0x66, + 0x0C, 0xDE, 0xA9, 0x35, 0xC7, 0xDD, 0x72, 0x42, 0x38, 0x33, 0x32, 0x2F, + 0x2C, 0x3F, 0xBD, 0x01, 0xD6, 0x47, 0xFC, 0x89, 0x31, 0x38, 0x2E, 0xB9, + 0x6B, 0xED, 0xDB, 0x85, 0x38, 0xB1, 0xA5, 0x50, 0xFA, 0xFB, 0xA7, 0x31, + 0xEC, 0xB6, 0xBB, 0x82, 0x50, 0xB4, 0x88, 0x5C, 0xED, 0xE5, 0x4B, 0x5B, + 0xBF, 0xB3, 0x18, 0xFB, 0xAD, 0x24, 0x41, 0x55, 0x80, 0xCD, 0xA3, 0xA1, + 0xD6, 0xD5, 0xB6, 0x06, 0xE9, 0x85, 0x12, 0x33, 0x52, 0x56, 0xF1, 0xB7, + 0xDC, 0x57, 0x9E, 0xB4, 0x00, 0x1E, 0xCB, 0x62, 0x13, 0x4C, 0x90, 0x9A, + 0x9D, 0x64, 0x80, 0xD1, 0x5E, 0xB3, 0xCB, 0x8A, 0x73, 0x4E, 0x7B, 0xBE, + 0x4D, 0xA7, 0xF7, 0xB7, 0x9C, 0x1C, 0x7F, 0x27, 0x1E, 0x7F, 0x58, 0xB2, + 0x74, 0xAF, 0x94, 0x0E, 0x19, 0x23, 0xE1, 0x6B, 0xD8, 0x20, 0x4F, 0x2C, + 0x13, 0xE8, 0x8C, 0x37, 0x46, 0x27, 0x55, 0x68, 0xDA, 0x3F, 0x7A, 0xC6, + 0xEF, 0x87, 0x1D, 0x3B, 0x95, 0x43, 0x5E, 0x75, 0xE0, 0x02, 0x22, 0x0E, + 0x11, 0x60, 0xAB, 0x1A, 0x91, 0x94, 0xC4, 0xFA, 0xD9, 0x92, 0x2B, 0xE5, + 0x03, 0xE0, 0x7A, 0x17, 0x5C, 0x67, 0x22, 0xB3, 0xCB, 0x77, 0x9E, 0x22, + 0x01, 0x5F, 0x5D, 0x64, 0xE4, 0x2F, 0xC4, 0x61, 0xCA, 0xC7, 0xFD, 0x20, + 0x24, 0x30, 0xAB, 0x3F, 0x1A, 0x08, 0x85, 0x08, 0x39, 0xDE, 0x19, 0x1C, + 0x1A, 0xEA, 0xB8, 0x7E, 0xE5, 0xBC, 0xD9, 0xB2, 0x59, 0xC8, 0x81, 0x02, + 0x1D, 0x5C, 0xC0, 0xDD, 0x8D, 0x56, 0xB6, 0x2E, 0x85, 0x26, 0xA8, 0x34, + 0x92, 0x36, 0x9A, 0x84, 0xBD, 0x27, 0xC1, 0x9D, 0x5E, 0x14, 0xC4, 0xB7, + 0x02, 0xA8, 0xC9, 0xC2, 0xAD, 0xDC, 0x98, 0x42, 0x51, 0xDE, 0x94, 0x28, + 0x39, 0xEF, 0xE9, 0x7F, 0x05, 0x3F, 0x1D, 0x67, 0x72, 0x04, 0xCF, 0x7D, + 0x38, 0x49, 0xC4, 0x59, 0xA5, 0xF6, 0xB6, 0x02, 0x31, 0xD0, 0x05, 0x74, + 0x4B, 0xD0, 0x89, 0xD1, 0x7F, 0xC6, 0xDB, 0x7E, 0x75, 0x62, 0xA3, 0xC2, + 0x2E, 0xB0, 0xCC, 0x9A, 0xD3, 0xA4, 0x14, 0xB6, 0xF2, 0x91, 0x44, 0x3F, + 0x84, 0xE0, 0x90, 0x4A, 0x6A, 0x34, 0x8C, 0x35, 0x3C, 0xB2, 0xA9, 0x35, + 0x88, 0xB0, 0x88, 0xF8, 0x7E, 0x5C, 0xD2, 0x08, 0x5E, 0x08, 0x15, 0x03, + 0xBC, 0xF5, 0x42, 0x6B, 0x28, 0xED, 0xDD, 0xAA, 0x4D, 0x78, 0x10, 0x31, + 0x32, 0xA2, 0xC5, 0xCA, 0xEE, 0x9A, 0x62, 0x52, 0x3E, 0x48, 0x83, 0xA4, + 0xCA, 0xD4, 0xC7, 0xA7, 0xA5, 0x3F, 0x44, 0x1C, 0x86, 0xAD, 0x52, 0x7D, + 0x80, 0x1D, 0x9E, 0x32, 0x3F, 0x2A, 0x2E, 0xD8, 0x89, 0xC1, 0xA4, 0xD6, + 0xC1, 0x90, 0x2E, 0x1A, 0x20, 0x4B, 0x87, 0x32, 0x35, 0x25, 0xD8, 0xB8, + 0x57, 0x15, 0x85, 0x1E, 0x3C, 0x8A, 0xDC, 0x1A, 0x49, 0x3D, 0x70, 0x35, + 0x99, 0xAA, 0xDE, 0x2C, 0xD4, 0xAF, 0x79, 0x72, 0xAB, 0x97, 0x84, 0x20, + 0xB6, 0x4F, 0x34, 0x3F, 0xEA, 0xAE, 0x5F, 0x8F, 0x3A, 0x42, 0xDB, 0x68, + 0xE5, 0x84, 0x63, 0x2E, 0x7A, 0x0E, 0xBD, 0x28, 0x6A, 0x24, 0xB6, 0xAB, + 0xE4, 0xAC, 0x20, 0x7C, 0x81, 0xD0, 0x69, 0x89, 0xF8, 0xDE, 0xA9, 0x02, + 0xFD, 0x1F, 0x08, 0xDA, 0x26, 0xC2, 0x24, 0xCA, 0xEB, 0x44, 0x16, 0x8D, + 0x55, 0x5F, 0xB9, 0xA9, 0x5A, 0x18, 0x50, 0xB1, 0x54, 0xF1, 0xBF, 0x06, + 0xC2, 0xB0, 0x95, 0xC2, 0xAE, 0xE5, 0xBF, 0xB3, 0xFD, 0xC9, 0xBF, 0x75, + 0x42, 0x7D, 0xA0, 0xA8, 0x95, 0xF9, 0x62, 0x3B, 0x9C, 0x0D, 0x81, 0xF3, + 0x9C, 0xFC, 0x19, 0x5B, 0xF7, 0xD1, 0x9C, 0xF0, 0xAA, 0xFE, 0xEF, 0x35, + 0x1E, 0x81, 0x9E, 0x02, 0x46, 0x52, 0x9B, 0x99, 0x0D, 0x12, 0x8B, 0x71, + 0x6C, 0x32, 0xB5, 0x23, 0x17, 0x03, 0xC5, 0xB0, 0xA1, 0xC3, 0x4B, 0x10, + 0x01, 0x4D, 0x4C, 0x4A, 0x46, 0x8F, 0xD9, 0x79, 0xBB, 0x10, 0x44, 0xB0, + 0x3C, 0x7D, 0x46, 0xFD, 0x38, 0xDF, 0xAF, 0x6E, 0x58, 0x7D, 0xE1, 0xEB, + 0xBB, 0x8C, 0xDC, 0x79, 0xDA, 0x41, 0xD1, 0x8B, 0x0B, 0x11, 0x4F, 0xE5, + 0x1C, 0xC1, 0x59, 0xA7, 0x1E, 0x5A, 0xC1, 0xEE, 0x27, 0x33, 0xC8, 0x55, + 0xA9, 0x32, 0xEA, 0xF7, 0x45, 0xB0, 0x08, 0xE9, 0x32, 0xDF, 0x70, 0x24, + 0x82, 0xD3, 0x2A, 0x3E, 0x4F, 0x42, 0xB9, 0x25, 0x10, 0xD1, 0x73, 0xFA, + 0xFD, 0xC1, 0x84, 0xF2, 0xF7, 0x0E, 0xBC, 0x9D, 0x90, 0x39, 0xD7, 0xFD, + 0x45, 0x77, 0xBA, 0x29, 0xF9, 0x87, 0x45, 0xC1, 0x32, 0x44, 0xB0, 0x27, + 0x6B, 0xFC, 0x8A, 0xFE, 0x00, 0x6F, 0x61, 0x98, 0xD0, 0x60, 0xC8, 0x10, + 0xE5, 0xBC, 0x88, 0x13, 0x45, 0x44, 0xA5, 0xEB, 0x6E, 0xCB, 0x11, 0xAF, + 0x30, 0xDC, 0x8B, 0xF8, 0x30, 0x46, 0xDA, 0x76, 0xF1, 0xE5, 0x14, 0x51, + 0x8A, 0x02, 0x5A, 0x5A, 0xAA, 0x7B, 0x2D, 0x57, 0x0A, 0x5C, 0x73, 0xD1, + 0x88, 0xCE, 0xBE, 0x3D, 0x06, 0x3F, 0x48, 0x1D, 0x44, 0x24, 0x6F, 0x4F, + 0x7F, 0x6A, 0xF2, 0x16, 0x34, 0x35, 0x38, 0x73, 0x8A, 0xE5, 0x25, 0xF4, + 0x34, 0x9E, 0x5B, 0x40, 0x90, 0x04, 0x57, 0x1B, 0x57, 0x75, 0x8F, 0xEA, + 0x1C, 0xF8, 0x7A, 0x68, 0x01, 0x1C, 0x8D, 0xBA, 0xF4, 0xE3, 0xD3, 0x8F, + 0x7F, 0xE4, 0x50, 0x35, 0x6B, 0x6B, 0xF6, 0xFC, 0x5F, 0x9B, 0x98, 0x78, + 0x16, 0x68, 0x72, 0x74, 0x71, 0x78, 0x25, 0x68, 0xE5, 0x1E, 0x66, 0xE2, + 0x4E, 0xC8, 0xDB, 0x92, 0x8E, 0x88, 0x64, 0x74, 0xDE, 0xDB, 0x85, 0x56, + 0x9F, 0xF9, 0xC4, 0x29, 0x54, 0xA8, 0xFB, 0xBA, 0xEA, 0xAB, 0xC7, 0x49, + 0x5C, 0x6C, 0xD7, 0x61, 0x8C, 0xE2, 0x2B, 0xF5, 0xA0, 0xA8, 0xD2, 0x41, + 0xC0, 0x54, 0xAB, 0xA7, 0x56, 0x5C, 0xE7, 0xA5, 0xEA, 0xBC, 0x47, 0xD1, + 0x0D, 0xD9, 0xC0, 0xA9, 0xC4, 0xA7, 0x3E, 0xD1, 0x2B, 0x1E, 0x34, 0x31, + 0x36, 0x9D, 0xB9, 0x51, 0xD3, 0xAD, 0x29, 0xE6, 0x9B, 0xD8, 0x4B, 0x93, + 0x33, 0x2F, 0x30, 0xEF, 0x18, 0x90, 0x69, 0x11, 0x09, 0xEA, 0xBA, 0xE0, + 0x10, 0x93, 0x63, 0x71, 0xA8, 0x83, 0x59, 0xDB, 0xFC, 0x12, 0x22, 0x84, + 0xC7, 0x01, 0x20, 0x99, 0xEC, 0x59, 0xA9, 0xE6, 0x9B, 0x5B, 0x8B, 0xB8, + 0x68, 0x52, 0x61, 0x8B, 0x4E, 0xF3, 0x50, 0x69, 0xF1, 0x49, 0x9B, 0xAF, + 0x53, 0xAD, 0xA0, 0x9D, 0x23, 0xE0, 0xE0, 0xC4, 0x31, 0xE4, 0x8E, 0x1C, + 0x51, 0x14, 0xFC, 0x95, 0x9C, 0xA6, 0x34, 0x85, 0xB0, 0x36, 0xFC, 0x7A, + 0x53, 0x03, 0x31, 0x0E, 0xCB, 0x34, 0x3E, 0xDF, 0xD1, 0x71, 0xBC, 0xDB, + 0xA1, 0xAF, 0x59, 0x4A, 0x03, 0x19, 0xA7, 0x8E, 0xB5, 0x82, 0x15, 0x24, + 0x69, 0x68, 0xBD, 0x9C, 0x2E, 0xFA, 0x06, 0xB5, 0x70, 0xC5, 0x70, 0xC4, + 0x14, 0x99, 0x01, 0x49, 0xBD, 0x6E, 0xAE, 0x10, 0xA1, 0xE4, 0xEF, 0xDD, + 0xE5, 0x51, 0x22, 0x9D, 0xF7, 0x93, 0xAB, 0x41, 0xBD, 0x86, 0x7A, 0xCC, + 0x51, 0x94, 0xEC, 0x22, 0xBE, 0x0D, 0x67, 0xFD, 0xA3, 0xFD, 0xCF, 0xF8, + 0x74, 0x0A, 0x5E, 0x1C, 0x71, 0xAD, 0xB6, 0xD0, 0xD7, 0xF8, 0x71, 0x34, + 0xAB, 0x62, 0xE7, 0xA8, 0x6B, 0x8F, 0x1E, 0x43, 0x46, 0xA5, 0xE4, 0xB4, + 0x52, 0x81, 0x66, 0xB3, 0xE5, 0x10, 0x23, 0x21, 0x2B, 0x31, 0x0F, 0xB8, + 0xB6, 0xC5, 0xA5, 0xC9, 0x90, 0x07, 0x83, 0xD0, 0xC3, 0x10, 0x7A, 0x04, + 0xBD, 0x8A, 0x3C, 0x7B, 0xF9, 0x0E, 0x51, 0x81, 0x96, 0xC8, 0xAE, 0xF9, + 0x27, 0xDE, 0x62, 0x7A, 0x41, 0x60, 0x35, 0x8F, 0x77, 0xBC, 0x95, 0x11, + 0x2C, 0xC4, 0x6C, 0x47, 0x7A, 0xEB, 0x29, 0xE5, 0x8E, 0xB5, 0xD6, 0xA5, + 0x54, 0x1B, 0xD0, 0xE0, 0x0F, 0x7D, 0x5C, 0x51, 0xD8, 0x6C, 0x92, 0x2F, + 0x13, 0x4E, 0x90, 0x77, 0xF8, 0x8D, 0x69, 0x78, 0x96, 0x96, 0x49, 0x9F, + 0x3C, 0x2E, 0x5C, 0xA6, 0x73, 0x27, 0x7D, 0xAD, 0x8D, 0xE3, 0x9B, 0x4A, + 0x2F, 0x50, 0x0A, 0x42, 0x7E, 0xF2, 0x3B, 0x50, 0x5C, 0x81, 0xC9, 0x49, + 0x01, 0x96, 0x83, 0x0A, 0xEC, 0x7F, 0xED, 0x1C, 0xA5, 0x7D, 0xF1, 0xE6, + 0xC4, 0xB3, 0x8F, 0xF9, 0x0F, 0xDB, 0x7B, 0xC1, 0x35, 0xF7, 0x63, 0x4A, + 0x39, 0xD4, 0x0E, 0x9E, 0x05, 0xD9, 0x42, 0xAA, 0xAB, 0x52, 0xCA, 0x4E, + 0x98, 0x3B, 0x43, 0x1A, 0x91, 0x25, 0xA9, 0x34, 0xD5, 0x66, 0xB2, 0xF4, + 0xFF, 0xDE, 0x64, 0x91, 0x90, 0xB9, 0x17, 0x70, 0xA0, 0xD6, 0xEA, 0xB6, + 0x36, 0xF4, 0x44, 0xCE, 0x86, 0x7B, 0x18, 0x74, 0x9C, 0x18, 0xAD, 0xB6, + 0xE0, 0x74, 0xC1, 0x0E, 0x29, 0x5D, 0x6A, 0x36, 0xD1, 0x3E, 0xB8, 0x2A, + 0xE4, 0x23, 0x1D, 0xB2, 0xAE, 0xF5, 0x5B, 0x8E, 0x2C, 0xD9, 0xD1, 0xE1, + 0x4F, 0x58, 0xA6, 0xE3, 0x88, 0x2E, 0xF9, 0xCF, 0x32, 0x3E, 0x8E, 0x37, + 0x95, 0xFF, 0xAD, 0x68, 0x11, 0x5E, 0x7F, 0x3D, 0x38, 0x06, 0x7C, 0x33, + 0x32, 0x78, 0x09, 0xEC, 0xCA, 0x3E, 0x08, 0xF1, 0xD0, 0x95, 0x19, 0xC9, + 0x7E, 0x62, 0xB2, 0x02, 0xA3, 0x5D, 0xF8, 0x3F, 0xA2, 0xB0, 0x8B, 0x38, + 0xB1, 0x8C, 0xEA, 0xB3, 0xE4, 0xBF, 0xD3, 0x6C, 0x6D, 0x3D, 0xD1, 0xC6, + 0xDA, 0x6B, 0x7A, 0xBA, 0x05, 0xEA, 0x9E, 0xA5, 0xE9, 0x00, 0xCC, 0x80, + 0x57, 0xAB, 0xD9, 0x0A, 0xD1, 0x00, 0x82, 0x2A, 0x51, 0x4B, 0xA2, 0x96, + 0xEB, 0x96, 0x14, 0xA8, 0x46, 0xDF, 0x1D, 0x48, 0xAE, 0xFA, 0x12, 0xA8, + 0x89, 0x8E, 0xEF, 0xBC, 0x3C, 0xA1, 0x6E, 0xDD, 0x90, 0x66, 0x2E, 0x56, + 0x6B, 0xF7, 0x1D, 0xF0, 0x46, 0x11, 0x4A, 0xA6, 0x07, 0x73, 0xC4, 0xE3, + 0x97, 0xFE, 0x7E, 0x22, 0x6F, 0x22, 0xB4, 0x6F, 0xB0, 0x32, 0x0A, 0x5E, + 0x85, 0x7E, 0x54, 0xB4, 0x24, 0xBD, 0x36, 0xA7, 0x94, 0xE7, 0x37, 0xFD, + 0x1A, 0xAF, 0xF4, 0x44, 0xB4, 0x35, 0x4F, 0xE0, 0x41, 0x0E, 0x7D, 0x73, + 0x29, 0x28, 0xDA, 0xAF, 0x69, 0xB2, 0xC5, 0xA7, 0x2A, 0x0A, 0xB5, 0x9C, + 0xC2, 0xAC, 0x5F, 0x59, 0x5C, 0xEE, 0x44, 0x49, 0x6F, 0x4F, 0x64, 0x43, + 0x6F, 0x43, 0x44, 0xAA, 0xA0, 0x4E, 0x94, 0x7C, 0x26, 0x5A, 0xF1, 0xD9, + 0xE6, 0x09, 0x80, 0x7A, 0x7D, 0x2E, 0xA2, 0xB9, 0x1A, 0x7A, 0x8F, 0x2A, + 0x97, 0x77, 0x23, 0xB4, 0x10, 0xAD, 0x20, 0x7B, 0xA3, 0x0F, 0xFD, 0x44, + 0x38, 0xAD, 0x94, 0x39, 0x88, 0x1C, 0xC4, 0xC8, 0xDF, 0xF1, 0x04, 0xA6, + 0x51, 0x5D, 0x54, 0x53, 0x60, 0xE4, 0x8A, 0x89, 0x4A, 0x9C, 0xE1, 0x68, + 0x4D, 0xFE, 0x69, 0x94, 0x0B, 0x8E, 0xED, 0x6C, 0xFE, 0x11, 0xA7, 0x77, + 0xBF, 0x08, 0x41, 0x67, 0x22, 0x59, 0x51, 0x48, 0xEE, 0x59, 0x02, 0x0E, + 0x60, 0x6D, 0xAE, 0x8C, 0xC6, 0x39, 0xB7, 0x55, 0xC5, 0x3B, 0x87, 0xA9, + 0xBD, 0xD8, 0xEA, 0x48, 0x21, 0xE4, 0x57, 0x51, 0x56, 0x03, 0xF4, 0xBE, + 0xBD, 0xBD, 0xC5, 0x26, 0x9B, 0x27, 0xE3, 0xAE, 0xD5, 0x1E, 0x30, 0xE9, + 0x7C, 0x9D, 0xDB, 0xE1, 0x09, 0x9D, 0x82, 0x49, 0x15, 0x38, 0x69, 0xFC, + 0x1D, 0x52, 0x1A, 0x75, 0xE6, 0xDD, 0x1D, 0xBE, 0x06, 0xC4, 0x9F, 0x14, + 0x4C, 0x12, 0xDE, 0xDF, 0x4A, 0xE1, 0x3B, 0xE7, 0xD1, 0xE3, 0x71, 0xD1, + 0xFA, 0xD8, 0x0E, 0x63, 0x27, 0xA9, 0xC7, 0x9D, 0xC0, 0x01, 0xC2, 0xDD, + 0xFC, 0xA6, 0x1F, 0x59, 0x87, 0xC5, 0x56, 0x99, 0x80, 0xEB, 0xF0, 0xB8, + 0xB3, 0x00, 0x9A, 0x61, 0xDB, 0x50, 0x79, 0x48, 0x37, 0x35, 0xDA, 0xD8, + 0xF2, 0x37, 0xA7, 0x43, 0xA7, 0xEB, 0x88, 0x2C, 0x68, 0xB4, 0xBB, 0x14, + 0x45, 0x31, 0x6B, 0x87, 0x65, 0xE7, 0x82, 0xB4, 0x74, 0xD2, 0xFF, 0x7F, + 0x60, 0x15, 0x94, 0x75, 0xEE, 0x30, 0x3C, 0x4E, 0xFC, 0x41, 0xD1, 0x5B, + 0xDD, 0x84, 0x6E, 0x13, 0x6C, 0xF8, 0x12, 0xE6, 0xB7, 0xA4, 0xB9, 0xC8, + 0x13, 0x89, 0x0C, 0x34, 0xA6, 0xAF, 0x09, 0xEB, 0xF2, 0xB3, 0x79, 0x77, + 0x80, 0xD8, 0x77, 0x64, 0xAD, 0x32, 0x3D, 0xD2, 0x06, 0xDF, 0x72, 0x11, + 0x4A, 0xA7, 0x70, 0xCE, 0xF9, 0xE6, 0x81, 0x35, 0xA4, 0xA7, 0x52, 0xB5, + 0x13, 0x68, 0x5C, 0x69, 0x45, 0xE2, 0x77, 0x2D, 0xBE, 0x2C, 0xE9, 0x38, + 0x25, 0x28, 0x7B, 0x63, 0x2C, 0x19, 0x8F, 0x59 +}; + +/* aad */ +uint8_t aad[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B +}; + +/* iv */ +uint8_t iv[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, + 0x0C, 0x0D, 0x0E, 0x0F +}; + +/* cipher key */ +uint8_t cipher_key[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, + 0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F +}; + +/* auth key */ +uint8_t auth_key[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, + 0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F, 0x20, 0x21, 0x22, 0x23, + 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2A, 0x2B, 0x2C, 0x2D, 0x2E, 0x2F, + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x3B, + 0x3C, 0x3D, 0x3E, 0x3F, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, + 0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x50, 0x51, 0x52, 0x53, + 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, 0x5B, 0x5C, 0x5D, 0x5E, 0x5F, + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6A, 0x6B, + 0x6C, 0x6D, 0x6E, 0x6F, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7A, 0x7B, 0x7C, 0x7D, 0x7E, 0x7F +}; + +/* AEAD key */ +uint8_t aead_key[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, + 0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F +}; + +/* Digests */ +uint8_t digest[2048] = { 0x00 }; + +struct cperf_test_vector* +cperf_test_vector_get_dummy(struct cperf_options *options) +{ + struct cperf_test_vector *t_vec; + + t_vec = (struct cperf_test_vector *)rte_malloc(NULL, + sizeof(struct cperf_test_vector), 0); + if (t_vec == NULL) + return t_vec; + + t_vec->plaintext.data = plaintext; + t_vec->plaintext.length = options->max_buffer_size; + + if (options->op_type == CPERF_CIPHER_ONLY || + options->op_type == CPERF_CIPHER_THEN_AUTH || + options->op_type == CPERF_AUTH_THEN_CIPHER) { + if (options->cipher_algo == RTE_CRYPTO_CIPHER_NULL) { + t_vec->cipher_key.length = 0; + t_vec->ciphertext.data = plaintext; + t_vec->cipher_key.data = NULL; + t_vec->cipher_iv.data = NULL; + } else { + t_vec->cipher_key.length = options->cipher_key_sz; + t_vec->ciphertext.data = ciphertext; + t_vec->cipher_key.data = cipher_key; + t_vec->cipher_iv.data = rte_malloc(NULL, options->cipher_iv_sz, + 16); + if (t_vec->cipher_iv.data == NULL) { + rte_free(t_vec); + return NULL; + } + memcpy(t_vec->cipher_iv.data, iv, options->cipher_iv_sz); + } + t_vec->ciphertext.length = options->max_buffer_size; + + /* Set IV parameters */ + t_vec->cipher_iv.data = rte_malloc(NULL, options->cipher_iv_sz, + 16); + if (options->cipher_iv_sz && t_vec->cipher_iv.data == NULL) { + rte_free(t_vec); + return NULL; + } + memcpy(t_vec->cipher_iv.data, iv, options->cipher_iv_sz); + t_vec->cipher_iv.length = options->cipher_iv_sz; + + t_vec->data.cipher_offset = 0; + t_vec->data.cipher_length = options->max_buffer_size; + + } + + if (options->op_type == CPERF_AUTH_ONLY || + options->op_type == CPERF_CIPHER_THEN_AUTH || + options->op_type == CPERF_AUTH_THEN_CIPHER) { + if (options->auth_algo == RTE_CRYPTO_AUTH_NULL) { + t_vec->auth_key.length = 0; + t_vec->auth_key.data = NULL; + t_vec->digest.data = NULL; + t_vec->digest.length = 0; + } else { + t_vec->auth_key.length = options->auth_key_sz; + t_vec->auth_key.data = auth_key; + + t_vec->digest.data = rte_malloc(NULL, + options->digest_sz, + 16); + if (t_vec->digest.data == NULL) { + rte_free(t_vec->cipher_iv.data); + rte_free(t_vec); + return NULL; + } + t_vec->digest.phys_addr = + rte_malloc_virt2iova(t_vec->digest.data); + t_vec->digest.length = options->digest_sz; + memcpy(t_vec->digest.data, digest, + options->digest_sz); + } + t_vec->data.auth_offset = 0; + t_vec->data.auth_length = options->max_buffer_size; + + /* Set IV parameters */ + t_vec->auth_iv.data = rte_malloc(NULL, options->auth_iv_sz, + 16); + if (options->auth_iv_sz && t_vec->auth_iv.data == NULL) { + if (options->op_type != CPERF_AUTH_ONLY) + rte_free(t_vec->cipher_iv.data); + rte_free(t_vec); + return NULL; + } + memcpy(t_vec->auth_iv.data, iv, options->auth_iv_sz); + t_vec->auth_iv.length = options->auth_iv_sz; + } + + if (options->op_type == CPERF_AEAD) { + t_vec->aead_key.length = options->aead_key_sz; + t_vec->aead_key.data = aead_key; + + if (options->aead_aad_sz) { + t_vec->aad.data = rte_malloc(NULL, + options->aead_aad_sz, 16); + if (t_vec->aad.data == NULL) { + rte_free(t_vec); + return NULL; + } + memcpy(t_vec->aad.data, aad, options->aead_aad_sz); + t_vec->aad.phys_addr = rte_malloc_virt2iova(t_vec->aad.data); + t_vec->aad.length = options->aead_aad_sz; + } else { + t_vec->aad.data = NULL; + t_vec->aad.length = 0; + } + + t_vec->digest.data = rte_malloc(NULL, options->digest_sz, + 16); + if (t_vec->digest.data == NULL) { + rte_free(t_vec->aad.data); + rte_free(t_vec); + return NULL; + } + t_vec->digest.phys_addr = + rte_malloc_virt2iova(t_vec->digest.data); + t_vec->digest.length = options->digest_sz; + memcpy(t_vec->digest.data, digest, options->digest_sz); + t_vec->data.aead_offset = 0; + t_vec->data.aead_length = options->max_buffer_size; + + /* Set IV parameters */ + t_vec->aead_iv.data = rte_malloc(NULL, options->aead_iv_sz, + 16); + if (options->aead_iv_sz && t_vec->aead_iv.data == NULL) { + rte_free(t_vec->aad.data); + rte_free(t_vec->digest.data); + rte_free(t_vec); + return NULL; + } + memcpy(t_vec->aead_iv.data, iv, options->aead_iv_sz); + t_vec->aead_iv.length = options->aead_iv_sz; + } + return t_vec; +} diff --git a/dpdk/app/test-crypto-perf/cperf_test_vectors.h b/dpdk/app/test-crypto-perf/cperf_test_vectors.h new file mode 100644 index 00000000..cb5d8284 --- /dev/null +++ b/dpdk/app/test-crypto-perf/cperf_test_vectors.h @@ -0,0 +1,114 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _CPERF_TEST_VECTRORS_ +#define _CPERF_TEST_VECTRORS_ + +#include "cperf_options.h" + +struct cperf_test_vector { + struct { + uint8_t *data; + uint32_t length; + } plaintext; + + struct { + uint8_t *data; + uint16_t length; + } cipher_key; + + struct { + uint8_t *data; + uint16_t length; + } auth_key; + + struct { + uint8_t *data; + uint16_t length; + } aead_key; + + struct { + uint8_t *data; + uint16_t length; + } cipher_iv; + + struct { + uint8_t *data; + uint16_t length; + } auth_iv; + + struct { + uint8_t *data; + uint16_t length; + } aead_iv; + + struct { + uint8_t *data; + uint32_t length; + } ciphertext; + + struct { + uint8_t *data; + rte_iova_t phys_addr; + uint16_t length; + } aad; + + struct { + uint8_t *data; + rte_iova_t phys_addr; + uint16_t length; + } digest; + + struct { + uint32_t auth_offset; + uint32_t auth_length; + uint32_t cipher_offset; + uint32_t cipher_length; + uint32_t aead_offset; + uint32_t aead_length; + } data; +}; + +struct cperf_test_vector* +cperf_test_vector_get_dummy(struct cperf_options *options); + +extern uint8_t ciphertext[2048]; + +extern uint8_t cipher_key[]; +extern uint8_t auth_key[]; + +extern uint8_t iv[]; +extern uint8_t aad[]; + +extern uint8_t digest[2048]; + +#endif diff --git a/dpdk/app/test-crypto-perf/cperf_test_verify.c b/dpdk/app/test-crypto-perf/cperf_test_verify.c new file mode 100644 index 00000000..6945c8b4 --- /dev/null +++ b/dpdk/app/test-crypto-perf/cperf_test_verify.c @@ -0,0 +1,455 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include + +#include "cperf_test_verify.h" +#include "cperf_ops.h" +#include "cperf_test_common.h" + +struct cperf_verify_ctx { + uint8_t dev_id; + uint16_t qp_id; + uint8_t lcore_id; + + struct rte_mempool *pool; + + struct rte_cryptodev_sym_session *sess; + + cperf_populate_ops_t populate_ops; + + uint32_t src_buf_offset; + uint32_t dst_buf_offset; + + const struct cperf_options *options; + const struct cperf_test_vector *test_vector; +}; + +struct cperf_op_result { + enum rte_crypto_op_status status; +}; + +static void +cperf_verify_test_free(struct cperf_verify_ctx *ctx) +{ + if (ctx) { + if (ctx->sess) { + rte_cryptodev_sym_session_clear(ctx->dev_id, ctx->sess); + rte_cryptodev_sym_session_free(ctx->sess); + } + + if (ctx->pool) + rte_mempool_free(ctx->pool); + + rte_free(ctx); + } +} + +void * +cperf_verify_test_constructor(struct rte_mempool *sess_mp, + uint8_t dev_id, uint16_t qp_id, + const struct cperf_options *options, + const struct cperf_test_vector *test_vector, + const struct cperf_op_fns *op_fns) +{ + struct cperf_verify_ctx *ctx = NULL; + + ctx = rte_malloc(NULL, sizeof(struct cperf_verify_ctx), 0); + if (ctx == NULL) + goto err; + + ctx->dev_id = dev_id; + ctx->qp_id = qp_id; + + ctx->populate_ops = op_fns->populate_ops; + ctx->options = options; + ctx->test_vector = test_vector; + + /* IV goes at the end of the crypto operation */ + uint16_t iv_offset = sizeof(struct rte_crypto_op) + + sizeof(struct rte_crypto_sym_op); + + ctx->sess = op_fns->sess_create(sess_mp, dev_id, options, test_vector, + iv_offset); + if (ctx->sess == NULL) + goto err; + + if (cperf_alloc_common_memory(options, test_vector, dev_id, qp_id, 0, + &ctx->src_buf_offset, &ctx->dst_buf_offset, + &ctx->pool) < 0) + goto err; + + return ctx; +err: + cperf_verify_test_free(ctx); + + return NULL; +} + +static int +cperf_verify_op(struct rte_crypto_op *op, + const struct cperf_options *options, + const struct cperf_test_vector *vector) +{ + const struct rte_mbuf *m; + uint32_t len; + uint16_t nb_segs; + uint8_t *data; + uint32_t cipher_offset, auth_offset; + uint8_t cipher, auth; + int res = 0; + + if (op->status != RTE_CRYPTO_OP_STATUS_SUCCESS) + return 1; + + if (op->sym->m_dst) + m = op->sym->m_dst; + else + m = op->sym->m_src; + nb_segs = m->nb_segs; + len = 0; + while (m && nb_segs != 0) { + len += m->data_len; + m = m->next; + nb_segs--; + } + + data = rte_malloc(NULL, len, 0); + if (data == NULL) + return 1; + + if (op->sym->m_dst) + m = op->sym->m_dst; + else + m = op->sym->m_src; + nb_segs = m->nb_segs; + len = 0; + while (m && nb_segs != 0) { + memcpy(data + len, rte_pktmbuf_mtod(m, uint8_t *), + m->data_len); + len += m->data_len; + m = m->next; + nb_segs--; + } + + switch (options->op_type) { + case CPERF_CIPHER_ONLY: + cipher = 1; + cipher_offset = 0; + auth = 0; + auth_offset = 0; + break; + case CPERF_CIPHER_THEN_AUTH: + cipher = 1; + cipher_offset = 0; + auth = 1; + auth_offset = options->test_buffer_size; + break; + case CPERF_AUTH_ONLY: + cipher = 0; + cipher_offset = 0; + auth = 1; + auth_offset = options->test_buffer_size; + break; + case CPERF_AUTH_THEN_CIPHER: + cipher = 1; + cipher_offset = 0; + auth = 1; + auth_offset = options->test_buffer_size; + break; + case CPERF_AEAD: + cipher = 1; + cipher_offset = 0; + auth = 1; + auth_offset = options->test_buffer_size; + break; + default: + res = 1; + goto out; + } + + if (cipher == 1) { + if (options->cipher_op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) + res += memcmp(data + cipher_offset, + vector->ciphertext.data, + options->test_buffer_size); + else + res += memcmp(data + cipher_offset, + vector->plaintext.data, + options->test_buffer_size); + } + + if (auth == 1) { + if (options->auth_op == RTE_CRYPTO_AUTH_OP_GENERATE) + res += memcmp(data + auth_offset, + vector->digest.data, + options->digest_sz); + } + +out: + rte_free(data); + return !!res; +} + +static void +cperf_mbuf_set(struct rte_mbuf *mbuf, + const struct cperf_options *options, + const struct cperf_test_vector *test_vector) +{ + uint32_t segment_sz = options->segment_sz; + uint8_t *mbuf_data; + uint8_t *test_data = + (options->cipher_op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ? + test_vector->plaintext.data : + test_vector->ciphertext.data; + uint32_t remaining_bytes = options->max_buffer_size; + + while (remaining_bytes) { + mbuf_data = rte_pktmbuf_mtod(mbuf, uint8_t *); + + if (remaining_bytes <= segment_sz) { + memcpy(mbuf_data, test_data, remaining_bytes); + return; + } + + memcpy(mbuf_data, test_data, segment_sz); + remaining_bytes -= segment_sz; + test_data += segment_sz; + mbuf = mbuf->next; + } +} + +int +cperf_verify_test_runner(void *test_ctx) +{ + struct cperf_verify_ctx *ctx = test_ctx; + + uint64_t ops_enqd = 0, ops_enqd_total = 0, ops_enqd_failed = 0; + uint64_t ops_deqd = 0, ops_deqd_total = 0, ops_deqd_failed = 0; + uint64_t ops_failed = 0; + + static int only_once; + + uint64_t i; + uint16_t ops_unused = 0; + + struct rte_crypto_op *ops[ctx->options->max_burst_size]; + struct rte_crypto_op *ops_processed[ctx->options->max_burst_size]; + + uint32_t lcore = rte_lcore_id(); + +#ifdef CPERF_LINEARIZATION_ENABLE + struct rte_cryptodev_info dev_info; + int linearize = 0; + + /* Check if source mbufs require coalescing */ + if (ctx->options->segment_sz < ctx->options->max_buffer_size) { + rte_cryptodev_info_get(ctx->dev_id, &dev_info); + if ((dev_info.feature_flags & + RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER) == 0) + linearize = 1; + } +#endif /* CPERF_LINEARIZATION_ENABLE */ + + ctx->lcore_id = lcore; + + if (!ctx->options->csv) + printf("\n# Running verify test on device: %u, lcore: %u\n", + ctx->dev_id, lcore); + + uint16_t iv_offset = sizeof(struct rte_crypto_op) + + sizeof(struct rte_crypto_sym_op); + + while (ops_enqd_total < ctx->options->total_ops) { + + uint16_t burst_size = ((ops_enqd_total + ctx->options->max_burst_size) + <= ctx->options->total_ops) ? + ctx->options->max_burst_size : + ctx->options->total_ops - + ops_enqd_total; + + uint16_t ops_needed = burst_size - ops_unused; + + /* Allocate objects containing crypto operations and mbufs */ + if (rte_mempool_get_bulk(ctx->pool, (void **)ops, + ops_needed) != 0) { + RTE_LOG(ERR, USER1, + "Failed to allocate more crypto operations " + "from the the crypto operation pool.\n" + "Consider increasing the pool size " + "with --pool-sz\n"); + return -1; + } + + /* Setup crypto op, attach mbuf etc */ + (ctx->populate_ops)(ops, ctx->src_buf_offset, + ctx->dst_buf_offset, + ops_needed, ctx->sess, ctx->options, + ctx->test_vector, iv_offset); + + + /* Populate the mbuf with the test vector, for verification */ + for (i = 0; i < ops_needed; i++) + cperf_mbuf_set(ops[i]->sym->m_src, + ctx->options, + ctx->test_vector); + +#ifdef CPERF_LINEARIZATION_ENABLE + if (linearize) { + /* PMD doesn't support scatter-gather and source buffer + * is segmented. + * We need to linearize it before enqueuing. + */ + for (i = 0; i < burst_size; i++) + rte_pktmbuf_linearize(ops[i]->sym->m_src); + } +#endif /* CPERF_LINEARIZATION_ENABLE */ + + /* Enqueue burst of ops on crypto device */ + ops_enqd = rte_cryptodev_enqueue_burst(ctx->dev_id, ctx->qp_id, + ops, burst_size); + if (ops_enqd < burst_size) + ops_enqd_failed++; + + /** + * Calculate number of ops not enqueued (mainly for hw + * accelerators whose ingress queue can fill up). + */ + ops_unused = burst_size - ops_enqd; + ops_enqd_total += ops_enqd; + + + /* Dequeue processed burst of ops from crypto device */ + ops_deqd = rte_cryptodev_dequeue_burst(ctx->dev_id, ctx->qp_id, + ops_processed, ctx->options->max_burst_size); + + if (ops_deqd == 0) { + /** + * Count dequeue polls which didn't return any + * processed operations. This statistic is mainly + * relevant to hw accelerators. + */ + ops_deqd_failed++; + continue; + } + + for (i = 0; i < ops_deqd; i++) { + if (cperf_verify_op(ops_processed[i], ctx->options, + ctx->test_vector)) + ops_failed++; + } + /* Free crypto ops so they can be reused. */ + rte_mempool_put_bulk(ctx->pool, + (void **)ops_processed, ops_deqd); + ops_deqd_total += ops_deqd; + } + + /* Dequeue any operations still in the crypto device */ + + while (ops_deqd_total < ctx->options->total_ops) { + /* Sending 0 length burst to flush sw crypto device */ + rte_cryptodev_enqueue_burst(ctx->dev_id, ctx->qp_id, NULL, 0); + + /* dequeue burst */ + ops_deqd = rte_cryptodev_dequeue_burst(ctx->dev_id, ctx->qp_id, + ops_processed, ctx->options->max_burst_size); + if (ops_deqd == 0) { + ops_deqd_failed++; + continue; + } + + for (i = 0; i < ops_deqd; i++) { + if (cperf_verify_op(ops_processed[i], ctx->options, + ctx->test_vector)) + ops_failed++; + } + /* Free crypto ops so they can be reused. */ + rte_mempool_put_bulk(ctx->pool, + (void **)ops_processed, ops_deqd); + ops_deqd_total += ops_deqd; + } + + if (!ctx->options->csv) { + if (!only_once) + printf("%12s%12s%12s%12s%12s%12s%12s%12s\n\n", + "lcore id", "Buf Size", "Burst size", + "Enqueued", "Dequeued", "Failed Enq", + "Failed Deq", "Failed Ops"); + only_once = 1; + + printf("%12u%12u%12u%12"PRIu64"%12"PRIu64"%12"PRIu64 + "%12"PRIu64"%12"PRIu64"\n", + ctx->lcore_id, + ctx->options->max_buffer_size, + ctx->options->max_burst_size, + ops_enqd_total, + ops_deqd_total, + ops_enqd_failed, + ops_deqd_failed, + ops_failed); + } else { + if (!only_once) + printf("\n# lcore id, Buffer Size(B), " + "Burst Size,Enqueued,Dequeued,Failed Enq," + "Failed Deq,Failed Ops\n"); + only_once = 1; + + printf("%10u;%10u;%u;%"PRIu64";%"PRIu64";%"PRIu64";%"PRIu64";" + "%"PRIu64"\n", + ctx->lcore_id, + ctx->options->max_buffer_size, + ctx->options->max_burst_size, + ops_enqd_total, + ops_deqd_total, + ops_enqd_failed, + ops_deqd_failed, + ops_failed); + } + + return 0; +} + + + +void +cperf_verify_test_destructor(void *arg) +{ + struct cperf_verify_ctx *ctx = arg; + + if (ctx == NULL) + return; + + cperf_verify_test_free(ctx); +} diff --git a/dpdk/app/test-crypto-perf/cperf_test_verify.h b/dpdk/app/test-crypto-perf/cperf_test_verify.h new file mode 100644 index 00000000..e67b48d3 --- /dev/null +++ b/dpdk/app/test-crypto-perf/cperf_test_verify.h @@ -0,0 +1,61 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _CPERF_VERIFY_ +#define _CPERF_VERIFY_ + +#include + +#include + +#include "cperf.h" +#include "cperf_ops.h" +#include "cperf_options.h" +#include "cperf_test_vectors.h" + + +void * +cperf_verify_test_constructor( + struct rte_mempool *sess_mp, + uint8_t dev_id, + uint16_t qp_id, + const struct cperf_options *options, + const struct cperf_test_vector *test_vector, + const struct cperf_op_fns *ops_fn); + +int +cperf_verify_test_runner(void *test_ctx); + +void +cperf_verify_test_destructor(void *test_ctx); + +#endif /* _CPERF_VERIFY_ */ diff --git a/dpdk/app/test-crypto-perf/data/aes_cbc_128_sha.data b/dpdk/app/test-crypto-perf/data/aes_cbc_128_sha.data new file mode 100644 index 00000000..ff555903 --- /dev/null +++ b/dpdk/app/test-crypto-perf/data/aes_cbc_128_sha.data @@ -0,0 +1,502 @@ +# List of tests for AES-128 CBC: +# 1) [sha1_hmac_buff_x] +# 2) [sha224_hmac_buff_x] +# 3) [sha256_hmac_buff_x] +# 4) [sha384_hmac_buff_x] +# 5) [sha512_hmac_buff_x] +# where x is one of the values: 32, 64, 128, 256, 512, 1024, 2048 + +########## +# GLOBAL # +########## +plaintext = +0xff, 0xca, 0xfb, 0xf1, 0x38, 0x20, 0x2f, 0x7b, 0x24, 0x98, 0x26, 0x7d, 0x1d, 0x9f, 0xb3, 0x93, +0xd9, 0xef, 0xbd, 0xad, 0x4e, 0x40, 0xbd, 0x60, 0xe9, 0x48, 0x59, 0x90, 0x67, 0xd7, 0x2b, 0x7b, +0x8a, 0xe0, 0x4d, 0xb0, 0x70, 0x38, 0xcc, 0x48, 0x61, 0x7d, 0xee, 0xd6, 0x35, 0x49, 0xae, 0xb4, +0xaf, 0x6b, 0xdd, 0xe6, 0x21, 0xc0, 0x60, 0xce, 0x0a, 0xf4, 0x1c, 0x2e, 0x1c, 0x8d, 0xe8, 0x7b, +0x59, 0xda, 0x19, 0x4f, 0xec, 0x07, 0x8e, 0xe2, 0xf0, 0x61, 0xf9, 0x27, 0x61, 0x6f, 0xf8, 0xdf, +0x62, 0x4d, 0xaf, 0x06, 0xfe, 0x41, 0xa6, 0xa6, 0xf9, 0xa2, 0x06, 0x40, 0xb3, 0x04, 0xbd, 0xe6, +0xc8, 0x17, 0xfb, 0x56, 0x6f, 0xa9, 0x3b, 0x8e, 0xa6, 0x58, 0xdc, 0x91, 0x17, 0x58, 0x42, 0x95, +0xa3, 0x7c, 0x81, 0x78, 0xa6, 0x3d, 0x3f, 0x75, 0x74, 0x17, 0x1a, 0xd3, 0x6c, 0x2f, 0x48, 0x39, +0x20, 0x20, 0xc1, 0x9a, 0x29, 0x84, 0x7d, 0x2d, 0x52, 0xa1, 0xf9, 0x5c, 0xf3, 0x4f, 0x91, 0xfc, +0x75, 0xcf, 0xd6, 0x2d, 0xe7, 0x9a, 0x59, 0x6e, 0x00, 0x0e, 0x8d, 0x22, 0x17, 0xbd, 0xa0, 0xdd, +0x79, 0x1f, 0x71, 0xe6, 0xcd, 0x2f, 0xb1, 0xb6, 0xbc, 0xc3, 0xdb, 0x02, 0x91, 0x41, 0x9b, 0x09, +0xa9, 0xd2, 0x7e, 0xbd, 0x2c, 0x18, 0xae, 0xc0, 0x93, 0x0c, 0x02, 0x9a, 0xdb, 0x4e, 0xaa, 0xeb, +0x84, 0x4b, 0x43, 0x5e, 0xf0, 0x98, 0xf2, 0x5f, 0x86, 0x70, 0x96, 0x90, 0x15, 0x30, 0xcf, 0x3a, +0xc9, 0x33, 0x21, 0xec, 0x59, 0x86, 0xfc, 0x65, 0x7d, 0xbe, 0xb9, 0xf8, 0x97, 0xf9, 0x30, 0xa9, +0x6d, 0xfc, 0x0c, 0x6e, 0x36, 0x67, 0xd5, 0xa6, 0x67, 0xd9, 0xbd, 0x9b, 0x34, 0x5d, 0xa7, 0xdd, +0xda, 0x46, 0x33, 0x25, 0x60, 0x4a, 0x18, 0xf1, 0x55, 0x07, 0xb2, 0xb7, 0x26, 0x7b, 0xa6, 0x1e, +0x77, 0xbe, 0x7f, 0x35, 0x46, 0xdf, 0x56, 0x9c, 0x22, 0x19, 0xc8, 0x85, 0xa2, 0x45, 0xb2, 0xad, +0xf9, 0x26, 0x66, 0xab, 0xfc, 0x97, 0x4b, 0x51, 0x32, 0x36, 0xbc, 0xad, 0xcf, 0x54, 0x3a, 0x4f, +0x94, 0xdb, 0xd2, 0xf9, 0x67, 0x1b, 0x3b, 0xe5, 0xb2, 0x1d, 0xc5, 0x52, 0x64, 0x2c, 0x06, 0x44, +0xcf, 0x18, 0x83, 0xe0, 0xd8, 0x04, 0x92, 0xa9, 0xc4, 0x3c, 0x8b, 0xa3, 0x2b, 0xbc, 0x88, 0x7e, +0xc0, 0x76, 0xa7, 0xe2, 0x7b, 0x47, 0x90, 0xf2, 0xaa, 0x0a, 0x34, 0x1b, 0x91, 0x12, 0xd2, 0xd0, +0x82, 0x45, 0xf4, 0x57, 0xf1, 0xbd, 0x91, 0x5e, 0xab, 0x41, 0x4c, 0xdf, 0x91, 0x4c, 0xdd, 0x67, +0x04, 0xa0, 0x98, 0x23, 0x8c, 0x24, 0xbe, 0xd6, 0x80, 0xb3, 0x6d, 0x04, 0xa1, 0x77, 0x43, 0xa5, +0xee, 0xb7, 0xce, 0xb1, 0x48, 0x43, 0x94, 0x61, 0x15, 0x20, 0x9d, 0xce, 0xd0, 0x14, 0x95, 0x37, +0xc8, 0x64, 0xa3, 0x2d, 0x3d, 0xe3, 0xff, 0xb4, 0x55, 0x83, 0x84, 0x41, 0x50, 0x57, 0xbd, 0x5a, +0x0c, 0xe4, 0xda, 0x3b, 0x36, 0x4d, 0x21, 0xb5, 0x6f, 0x73, 0x2a, 0x8c, 0x78, 0x4f, 0x9b, 0x83, +0xda, 0x11, 0x3c, 0xf0, 0xc9, 0x7e, 0xa6, 0x48, 0x34, 0x53, 0x62, 0xd3, 0x0c, 0xff, 0xb1, 0x74, +0xd6, 0xea, 0xa5, 0xfc, 0x13, 0x1c, 0x05, 0xa8, 0xc0, 0xbc, 0x95, 0x9c, 0x8c, 0xf6, 0x8c, 0xc3, +0xf3, 0x69, 0xab, 0x93, 0x65, 0xc0, 0xb7, 0x7e, 0xb0, 0x16, 0x7c, 0xb5, 0x5f, 0x05, 0x28, 0xc9, +0x09, 0x4e, 0x2a, 0x32, 0x87, 0xb3, 0xab, 0xf8, 0x4c, 0xab, 0xeb, 0x3b, 0x6a, 0xa0, 0x1d, 0x7f, +0xef, 0xe5, 0x9b, 0xa4, 0xb7, 0xd7, 0xc2, 0x5e, 0x03, 0x0f, 0x99, 0xeb, 0xb1, 0xb1, 0xa6, 0x9d, +0x1c, 0x7c, 0x5c, 0x94, 0x8b, 0x6e, 0x11, 0x7a, 0xb3, 0x6d, 0x1e, 0x61, 0x64, 0xc3, 0x7d, 0x1c, +0xb3, 0x54, 0x65, 0x08, 0x3b, 0xda, 0x97, 0xb9, 0x75, 0xc1, 0x2b, 0x3e, 0xa8, 0x5c, 0x3c, 0x2d, +0x81, 0x5b, 0xbf, 0x5a, 0x13, 0x0e, 0xeb, 0x66, 0xc0, 0x0b, 0x8f, 0x04, 0x68, 0x68, 0x9b, 0xe3, +0x0d, 0x84, 0xe0, 0xcf, 0x83, 0xd7, 0x62, 0x48, 0xc1, 0x31, 0xa5, 0xd5, 0xbc, 0xe3, 0xa3, 0xa5, +0xb6, 0xd1, 0xfd, 0x81, 0x91, 0x4d, 0xbd, 0xc4, 0x62, 0x4f, 0xe3, 0xd5, 0x99, 0x14, 0xf1, 0xcd, +0xf4, 0x7d, 0x13, 0xda, 0x68, 0x0a, 0xca, 0xd6, 0x82, 0x0b, 0xf6, 0xea, 0xad, 0x78, 0xa4, 0xc8, +0x14, 0x7a, 0xec, 0x11, 0xd3, 0x16, 0x86, 0x9f, 0x17, 0x37, 0x6a, 0x06, 0x56, 0xaa, 0x1b, 0xd1, +0xaf, 0x85, 0x95, 0x21, 0x36, 0x69, 0xec, 0x1b, 0x56, 0x84, 0x01, 0x3f, 0x4d, 0x34, 0x3d, 0x2d, +0x38, 0x57, 0x2d, 0x7e, 0xd9, 0x7b, 0x2d, 0x81, 0x86, 0xd4, 0x7c, 0x83, 0x12, 0x1d, 0x9d, 0x27, +0x72, 0x1b, 0x5e, 0xf4, 0x15, 0xa5, 0xcd, 0xb7, 0x5f, 0xbb, 0x49, 0xa1, 0xd9, 0xdd, 0x8d, 0xad, +0xa9, 0x2c, 0x65, 0x18, 0x91, 0xfd, 0xd2, 0xd4, 0x09, 0x60, 0x0c, 0xfd, 0xa4, 0xe1, 0x25, 0x87, +0x32, 0x64, 0x7b, 0x99, 0xd7, 0x61, 0x2f, 0xd4, 0x73, 0xdd, 0x85, 0x26, 0x08, 0x92, 0xc0, 0xe1, +0x4f, 0x0c, 0x76, 0x5b, 0x26, 0x69, 0xdb, 0x78, 0x35, 0x65, 0xb9, 0x58, 0x1f, 0x9c, 0x0f, 0x18, +0x95, 0xfe, 0x40, 0xfc, 0xf7, 0x93, 0x71, 0x70, 0x8b, 0x73, 0xdc, 0xb0, 0x88, 0x72, 0x19, 0x26, +0x94, 0x26, 0xa7, 0xaa, 0x00, 0x72, 0x61, 0x53, 0xd2, 0x5d, 0x8f, 0x5e, 0x51, 0x88, 0x2d, 0xa4, +0x28, 0xd5, 0xaf, 0x2d, 0xd2, 0x84, 0x39, 0x75, 0x1e, 0xe7, 0xf0, 0x23, 0xc0, 0x4f, 0x8d, 0xdd, +0x5c, 0x90, 0xef, 0x6e, 0x53, 0xe0, 0x54, 0x67, 0xe1, 0x5b, 0x10, 0xf1, 0xf5, 0xf8, 0x64, 0x34, +0x94, 0xeb, 0x37, 0xf7, 0xe9, 0xaa, 0x6c, 0xa4, 0xd8, 0x74, 0x6d, 0xca, 0x8d, 0x1a, 0x31, 0x73, +0xca, 0xb4, 0xc7, 0x47, 0x34, 0x7f, 0xf8, 0x24, 0x9b, 0xfa, 0xc9, 0xcc, 0xa8, 0x61, 0xb4, 0x0e, +0x4d, 0x68, 0xc7, 0xa0, 0xcb, 0xea, 0xf0, 0xcc, 0x0a, 0x6c, 0xf2, 0x33, 0x42, 0x99, 0x6c, 0xd8, +0x74, 0x7f, 0x1e, 0x8a, 0xa3, 0x0a, 0x48, 0x4b, 0x7e, 0xbe, 0xdb, 0x7f, 0x56, 0x69, 0x43, 0xe8, +0xbf, 0x12, 0xc4, 0x7b, 0xc2, 0xd9, 0xfa, 0x5c, 0xeb, 0x45, 0xca, 0x07, 0x3d, 0xc0, 0xcd, 0x68, +0x8b, 0xd0, 0x79, 0xea, 0x0a, 0x78, 0x06, 0xdc, 0x81, 0xd7, 0x32, 0x18, 0xb3, 0x65, 0xbe, 0x47, +0xbb, 0xfa, 0x17, 0x09, 0xe9, 0x31, 0x95, 0x30, 0xef, 0x07, 0x44, 0xec, 0xd0, 0x98, 0x98, 0xc0, +0x6b, 0x71, 0x5b, 0x23, 0xb8, 0xb6, 0xd2, 0x21, 0xff, 0x51, 0xdd, 0xae, 0x48, 0x29, 0x75, 0x0c, +0xc3, 0x3d, 0x91, 0xfe, 0x9d, 0xa8, 0x5e, 0xb2, 0x34, 0xb2, 0xd3, 0x81, 0xf6, 0x27, 0x9c, 0xac, +0x6b, 0x20, 0x56, 0x86, 0xa5, 0x4f, 0x7a, 0xdb, 0xf9, 0xac, 0xa9, 0x8e, 0xe3, 0x73, 0x21, 0x99, +0x71, 0x2d, 0xaf, 0x27, 0x92, 0x0c, 0xc7, 0xd3, 0x85, 0xb3, 0x40, 0xda, 0x13, 0x4a, 0x04, 0x41, +0x54, 0xf8, 0xf2, 0x55, 0xb7, 0x80, 0xdd, 0x77, 0xba, 0x01, 0x7a, 0x31, 0xbd, 0x6b, 0xdc, 0x5c, +0x59, 0xf4, 0x2b, 0xca, 0x25, 0xbb, 0x50, 0xba, 0xfa, 0x42, 0x38, 0xd2, 0x28, 0x10, 0x8b, 0x7b, +0x96, 0x45, 0x30, 0xbb, 0x7f, 0xf4, 0x5a, 0xf7, 0x28, 0x6f, 0x47, 0xdc, 0xd2, 0x82, 0xf2, 0xf7, +0xdd, 0x20, 0xb5, 0x0c, 0x7e, 0x53, 0x85, 0xa7, 0xfc, 0x3b, 0x1a, 0xc0, 0x07, 0x7b, 0xa1, 0x43, +0x05, 0x18, 0x19, 0xd3, 0xfc, 0x41, 0xc2, 0xce, 0xd9, 0x5b, 0x4b, 0x63, 0xe2, 0x8f, 0x86, 0x3a, +0xd1, 0xd0, 0x1d, 0x74, 0x2e, 0xbc, 0xd3, 0xce, 0x08, 0x0c, 0x10, 0x7a, 0x42, 0x60, 0xc5, 0x3a, +0xa6, 0xd8, 0xb0, 0x52, 0xcf, 0x53, 0x28, 0x70, 0x45, 0xb7, 0x72, 0x7d, 0x77, 0x66, 0x54, 0x3d, +0x38, 0x26, 0xcf, 0xd5, 0xbf, 0xe4, 0x80, 0x10, 0xba, 0x2b, 0xe8, 0xdc, 0xc3, 0xfe, 0x28, 0xa3, +0x52, 0x58, 0x70, 0x4a, 0xde, 0x84, 0x33, 0x5e, 0x93, 0x04, 0xa4, 0x7c, 0xe7, 0xea, 0x8e, 0xba, +0xeb, 0x8a, 0x19, 0x26, 0x6a, 0x7f, 0x7c, 0x4a, 0x5b, 0xb4, 0x0d, 0xfc, 0xc8, 0x11, 0x1b, 0x41, +0x68, 0x5d, 0x2a, 0x25, 0x04, 0x4f, 0xc8, 0xf4, 0x65, 0xfc, 0xb9, 0x58, 0xeb, 0xb4, 0x67, 0x50, +0x24, 0xf5, 0x43, 0xf6, 0x91, 0x4a, 0xb0, 0x0f, 0x32, 0xe0, 0x07, 0x75, 0x69, 0x1b, 0x3c, 0xeb, +0xb2, 0x65, 0x26, 0x6f, 0xb8, 0x79, 0xe0, 0x78, 0x8c, 0xdc, 0x39, 0x24, 0x48, 0x76, 0x11, 0xd4, +0x3a, 0xc5, 0xd2, 0x2b, 0xaa, 0x55, 0xfb, 0x92, 0x12, 0x2d, 0x88, 0x05, 0xd1, 0xb1, 0x31, 0x36, +0x1f, 0xc2, 0x44, 0x1c, 0xab, 0x2e, 0xcd, 0x1c, 0x72, 0x86, 0xf6, 0x83, 0x87, 0x2e, 0x8b, 0xdb, +0xaa, 0x16, 0x0e, 0x1b, 0xe6, 0x5c, 0x4d, 0x2f, 0x82, 0xbd, 0x49, 0x11, 0x60, 0x22, 0x0f, 0xde, +0x3b, 0x2b, 0x20, 0x1d, 0x56, 0xb7, 0x21, 0xae, 0x0b, 0x26, 0x4f, 0xde, 0x3d, 0xa6, 0x3f, 0x61, +0x81, 0xe2, 0x76, 0x60, 0x08, 0xc5, 0x4b, 0x18, 0x0b, 0xd1, 0xf5, 0xff, 0x8d, 0x1a, 0x96, 0x76, +0x51, 0x15, 0x05, 0x4d, 0x8c, 0x6b, 0x12, 0x90, 0x47, 0xd4, 0xa4, 0x38, 0xb9, 0x48, 0xe4, 0x4c, +0x05, 0x69, 0x6a, 0x8b, 0x9d, 0x7c, 0xa1, 0xbc, 0x77, 0xeb, 0x86, 0x93, 0x0a, 0x15, 0x84, 0xba, +0x8f, 0xf5, 0x7c, 0x44, 0x75, 0x31, 0x79, 0x16, 0xc1, 0x81, 0x1a, 0xb6, 0xe6, 0x6c, 0x3d, 0xb8, +0x15, 0x46, 0xf5, 0xbe, 0x46, 0x04, 0xa6, 0xec, 0xec, 0xd1, 0x74, 0x8b, 0x87, 0x2b, 0xdb, 0xd0, +0x9f, 0xb3, 0x99, 0x9d, 0x87, 0x8c, 0xc6, 0xaa, 0xd4, 0x64, 0x45, 0xbd, 0xe8, 0xed, 0xa3, 0xc1, +0x2a, 0x41, 0x1e, 0x26, 0xaf, 0x86, 0x16, 0xed, 0x80, 0x08, 0xca, 0x64, 0x21, 0x3a, 0xce, 0x21, +0x4c, 0x41, 0xb9, 0x13, 0x2d, 0xf7, 0x1b, 0xdf, 0x2b, 0x33, 0x69, 0xe7, 0x5c, 0x8c, 0x7b, 0xfb, +0xe3, 0x41, 0xe9, 0xce, 0xd7, 0xff, 0x0e, 0x54, 0xfe, 0xb0, 0x71, 0x78, 0xdc, 0xde, 0x7e, 0xdd, +0x1f, 0x1c, 0x4a, 0x8f, 0x3e, 0x16, 0xfd, 0x91, 0x82, 0x94, 0xd4, 0xc2, 0xf7, 0xb2, 0x77, 0x89, +0x16, 0x2c, 0xba, 0xb6, 0xbd, 0xed, 0x95, 0x43, 0x05, 0x9b, 0xf2, 0xc4, 0xbe, 0x46, 0x43, 0x90, +0x1d, 0xd8, 0x24, 0x02, 0xd2, 0xea, 0xf4, 0x08, 0xd9, 0xf7, 0x84, 0x0e, 0xc6, 0xe7, 0x44, 0xdb, +0xb8, 0xac, 0x0a, 0x53, 0x39, 0x61, 0x43, 0xdc, 0x22, 0x28, 0x8f, 0x22, 0x2f, 0x73, 0xbf, 0x59, +0x2d, 0x3c, 0x8c, 0x0b, 0xcc, 0x2a, 0x67, 0xe0, 0x5b, 0x5c, 0x65, 0x5e, 0x6d, 0x98, 0x99, 0xaa, +0x3b, 0x89, 0x12, 0xe2, 0x99, 0xf6, 0x15, 0xa7, 0xd2, 0x6a, 0x79, 0xb4, 0xf6, 0x0b, 0xf5, 0x0d, +0x2d, 0x4c, 0xcb, 0x1b, 0x35, 0x93, 0x61, 0x32, 0xa1, 0x8a, 0xa8, 0x27, 0xe8, 0x95, 0x5a, 0x56, +0x59, 0x04, 0xfe, 0xce, 0xc2, 0xd8, 0x92, 0x97, 0xb2, 0x54, 0x63, 0xd0, 0x3b, 0xde, 0x10, 0x34, +0x32, 0x16, 0x05, 0x51, 0x1d, 0xfc, 0x96, 0x8e, 0xf1, 0xf6, 0x4b, 0xd7, 0x48, 0x22, 0xce, 0xca, +0x1c, 0x6b, 0xab, 0x1f, 0x59, 0xa2, 0x74, 0xd6, 0xcd, 0x15, 0x07, 0xab, 0xa2, 0xd5, 0x22, 0x81, +0xec, 0x20, 0x14, 0x36, 0xac, 0xe4, 0x25, 0x7d, 0xe6, 0x09, 0x00, 0x2c, 0x92, 0x4d, 0x4e, 0xbf, +0xbf, 0xa1, 0xd4, 0xbe, 0x6b, 0xd4, 0x1f, 0x95, 0x9b, 0xf3, 0xda, 0x99, 0xad, 0xa4, 0x6c, 0x73, +0x55, 0xd1, 0x9d, 0x4b, 0x16, 0xd4, 0x06, 0xec, 0x46, 0x3d, 0xb7, 0xe7, 0xce, 0xd0, 0x1d, 0x94, +0x65, 0xde, 0x61, 0xb3, 0xc1, 0x10, 0x65, 0xe5, 0x68, 0x9b, 0xb0, 0xb4, 0x43, 0x0b, 0x92, 0xaf, +0xb7, 0x40, 0xa2, 0xe5, 0x06, 0x3d, 0x72, 0x00, 0xc5, 0x39, 0xab, 0x35, 0x29, 0x22, 0x4c, 0xa5, +0xa5, 0x3f, 0x22, 0x90, 0x53, 0xd2, 0x36, 0x63, 0x1e, 0xd3, 0x33, 0xa5, 0xbb, 0x3d, 0xa3, 0x0c, +0x14, 0x9c, 0x2e, 0x6d, 0x9a, 0x7a, 0xf7, 0xf1, 0x56, 0x66, 0xe5, 0x8d, 0x53, 0x83, 0x34, 0x3f, +0xa9, 0x83, 0x84, 0x68, 0x90, 0xc9, 0x51, 0xc2, 0xd4, 0x8e, 0x6c, 0xc7, 0x6d, 0xa7, 0x19, 0x61, +0xa7, 0x2e, 0x36, 0xbc, 0xd2, 0x0f, 0x17, 0x49, 0xd4, 0x6b, 0x36, 0x63, 0xfb, 0x1d, 0xf4, 0xb0, +0x6b, 0xcf, 0x34, 0x5f, 0xd2, 0x77, 0xae, 0x12, 0xaf, 0xb3, 0xdf, 0x52, 0xf7, 0xc2, 0xc8, 0xf2, +0x63, 0x61, 0xb6, 0x3e, 0x39, 0xf2, 0xa7, 0x1a, 0x89, 0x9d, 0x0e, 0x8f, 0xaf, 0xe1, 0x01, 0x24, +0xa6, 0x3a, 0xd5, 0x9a, 0x62, 0x67, 0xa3, 0x66, 0xee, 0xbc, 0xc5, 0x94, 0x4b, 0xc3, 0x15, 0xa1, +0x7e, 0x07, 0x07, 0x2b, 0xb7, 0x43, 0x2a, 0xb4, 0xb8, 0x25, 0x88, 0x86, 0x23, 0xab, 0xdf, 0x05, +0xbe, 0x46, 0x56, 0xd7, 0xda, 0xd6, 0x75, 0x53, 0xd9, 0xc8, 0x26, 0x8f, 0x39, 0x67, 0xed, 0x21, +0x53, 0x1c, 0x9c, 0x89, 0x46, 0xd3, 0xfe, 0x54, 0xe6, 0x1d, 0x02, 0xb9, 0x25, 0x82, 0x66, 0xe6, +0xf9, 0x45, 0xd9, 0x3f, 0xa5, 0x71, 0xc1, 0x46, 0x66, 0x7a, 0x27, 0x8a, 0x82, 0xc9, 0x21, 0xe9, +0x17, 0xab, 0x6c, 0xef, 0x45, 0xe5, 0x88, 0x93, 0x87, 0x80, 0xb3, 0x85, 0x25, 0x96, 0x19, 0x41, +0xab, 0xd6, 0xba, 0x92, 0x76, 0x21, 0x8a, 0x58, 0xbd, 0xe2, 0x4b, 0xec, 0x45, 0x59, 0x2c, 0x13, +0x1a, 0xb5, 0x13, 0x25, 0x44, 0xe7, 0x71, 0x26, 0x0a, 0x34, 0x33, 0xb9, 0x57, 0x15, 0xa4, 0x90, +0x60, 0x11, 0x05, 0x8e, 0xc8, 0x8e, 0x74, 0x52, 0x4b, 0x31, 0x71, 0xeb, 0x66, 0x7e, 0xee, 0xb1, +0x0a, 0x21, 0x52, 0xc0, 0x1a, 0xe9, 0xa1, 0x5a, 0xe3, 0x3a, 0x24, 0xfb, 0xf3, 0x1e, 0xd6, 0x83, +0x1d, 0xfb, 0x81, 0xa8, 0x91, 0x60, 0x9e, 0xbc, 0x59, 0x20, 0xc9, 0x9e, 0x71, 0x19, 0x83, 0x2b, +0x6a, 0x48, 0x4e, 0x6b, 0x46, 0x82, 0x89, 0xda, 0x60, 0xff, 0x1a, 0x46, 0x94, 0x55, 0xda, 0xe5, +0x99, 0xfa, 0x84, 0xd7, 0x3b, 0xb9, 0xa5, 0x34, 0x87, 0x86, 0x5e, 0x6d, 0x75, 0x9a, 0xe7, 0x09, +0xb8, 0xe6, 0x71, 0x15, 0x10, 0x56, 0xd7, 0xc1, 0xc8, 0xb2, 0x62, 0xbc, 0xec, 0xe0, 0x94, 0xa0, +0xcd, 0xb4, 0x04, 0xa9, 0xc3, 0x51, 0xee, 0xf8, 0x2e, 0x42, 0x9a, 0xaa, 0x34, 0xd3, 0xb9, 0xb0, +0x36, 0xf9, 0x47, 0xc1, 0x07, 0x49, 0xde, 0xb8, 0x32, 0x8a, 0x87, 0x68, 0x56, 0x9a, 0x35, 0x79, +0xd1, 0xac, 0x49, 0x38, 0xc6, 0xfe, 0xfd, 0xdf, 0x6f, 0x3c, 0xda, 0x48, 0xbd, 0x23, 0xfd, 0x85, +0xf0, 0x96, 0xee, 0x1c, 0x27, 0x18, 0x86, 0xa6, 0xf0, 0x7b, 0xd8, 0x3c, 0xc7, 0x22, 0x3e, 0x2f, +0xac, 0xb1, 0x37, 0xbd, 0x84, 0x4b, 0xe3, 0x92, 0x82, 0xd0, 0x25, 0x14, 0x22, 0x65, 0xed, 0xeb, +0xef, 0xb9, 0xb6, 0xe4, 0x95, 0x18, 0x0d, 0x2b, 0x8d, 0x4f, 0xaf, 0xc0, 0xa0, 0x05, 0x8b, 0x35, +0x5b, 0x94, 0xb2, 0x68, 0x26, 0x4f, 0x4a, 0x9e, 0x85, 0x0e, 0x46, 0xe0, 0x4f, 0x60, 0x66, 0x01, +0xa4, 0x39, 0xe8, 0x8b, 0x2a, 0x50, 0xf5, 0x18, 0x70, 0xe2, 0xfc, 0xd6, 0xbe, 0xd3, 0x46, 0x4b + +ciphertext = +0x75, 0x95, 0xb3, 0x48, 0x38, 0xf9, 0xe4, 0x88, 0xec, 0xf8, 0x3b, 0x09, 0x40, 0xd4, 0xd6, 0xea, +0xf1, 0x80, 0x6d, 0xfb, 0xba, 0x9e, 0xee, 0xac, 0x6a, 0xf9, 0x8f, 0xb6, 0xe1, 0xff, 0xea, 0x19, +0x17, 0xc2, 0x77, 0x8d, 0xc2, 0x8d, 0x6c, 0x89, 0xd1, 0x5f, 0xa6, 0xf3, 0x2c, 0xa7, 0x6a, 0x7f, +0x50, 0x1b, 0xc9, 0x4d, 0xb4, 0x36, 0x64, 0x6e, 0xa6, 0xd9, 0x39, 0x8b, 0xcf, 0x8e, 0x0c, 0x55, +0x4d, 0xb8, 0xe8, 0xf5, 0xb6, 0x5a, 0xd4, 0x49, 0x63, 0x24, 0xa2, 0xdf, 0xf0, 0x7a, 0x1c, 0x3b, +0x74, 0x0c, 0x1d, 0x9b, 0xe2, 0x2b, 0x31, 0xb5, 0xe3, 0xca, 0x9f, 0xe4, 0x23, 0xe8, 0x64, 0x83, +0x1a, 0xf1, 0xaa, 0xbf, 0xd0, 0x6a, 0xd6, 0x43, 0xd1, 0x2d, 0x2b, 0x7f, 0x38, 0x8d, 0x55, 0xd8, +0xb2, 0xa9, 0x96, 0xec, 0xf5, 0xf9, 0x56, 0x50, 0x65, 0xc8, 0x63, 0x35, 0x22, 0xb0, 0xf4, 0x11, +0xf3, 0x96, 0x16, 0xc3, 0x55, 0x90, 0xd0, 0x42, 0x3a, 0x5b, 0xc7, 0xfa, 0x67, 0x9b, 0x9f, 0xbd, +0xca, 0xa5, 0x89, 0xd1, 0xe6, 0xfb, 0xbe, 0x3c, 0x7a, 0x29, 0xcb, 0xd7, 0xd2, 0xaf, 0xfc, 0x2d, +0x1e, 0xb2, 0x6c, 0x4f, 0x5e, 0x31, 0x67, 0x6f, 0xa9, 0x8e, 0x54, 0x1f, 0xdb, 0x87, 0xd4, 0x11, +0xa4, 0x99, 0x50, 0xcc, 0x52, 0xd0, 0xfa, 0x43, 0x70, 0x03, 0xa6, 0xff, 0xe0, 0xcb, 0x22, 0xbd, +0xf3, 0x66, 0xf2, 0x4d, 0x82, 0x13, 0x94, 0x28, 0x89, 0xae, 0x82, 0xdc, 0xa5, 0x3e, 0xf2, 0xd2, +0x01, 0xda, 0xef, 0xb4, 0x81, 0x77, 0x86, 0x29, 0x35, 0xad, 0xca, 0x14, 0x84, 0xdb, 0x86, 0xbd, +0x8c, 0xf0, 0x8b, 0xc4, 0x2a, 0xb0, 0x87, 0xd8, 0x0b, 0xcd, 0x2c, 0x32, 0xe1, 0xce, 0xca, 0x15, +0x82, 0x6e, 0xf5, 0xc8, 0x20, 0x38, 0xa2, 0x60, 0xaf, 0xe1, 0xdc, 0xe7, 0x7b, 0xa4, 0x75, 0x4b, +0xf2, 0xd1, 0xfb, 0x25, 0x36, 0xbe, 0x84, 0x67, 0x94, 0x06, 0x20, 0xc0, 0x21, 0x30, 0x29, 0xdf, +0x63, 0xf5, 0x56, 0x3a, 0x07, 0xef, 0x8d, 0xad, 0x62, 0x0f, 0x60, 0xbe, 0xb0, 0x8e, 0x10, 0x27, +0xc6, 0x46, 0x90, 0xe5, 0x59, 0xaa, 0x16, 0x55, 0xca, 0x68, 0x6c, 0xbf, 0x19, 0x4e, 0xcd, 0xe8, +0xb4, 0xf8, 0x94, 0xc4, 0x55, 0x9d, 0x7c, 0x3c, 0x06, 0x4e, 0x1a, 0x34, 0x1b, 0x16, 0x80, 0xe4, +0x56, 0x98, 0xd4, 0xaa, 0xee, 0x66, 0xea, 0x91, 0x71, 0x44, 0xbd, 0xcb, 0xb1, 0xc5, 0x57, 0x49, +0xa8, 0x4e, 0xe2, 0x03, 0xc6, 0xd5, 0x39, 0xd8, 0x69, 0xa8, 0xa0, 0xad, 0xad, 0x0d, 0x8d, 0xa7, +0x80, 0xd3, 0xba, 0xc1, 0xae, 0xfe, 0xf5, 0xa2, 0x55, 0x8d, 0xa6, 0x2f, 0xb1, 0x4c, 0x67, 0x3e, +0xb4, 0xaa, 0xed, 0xab, 0x89, 0xbc, 0xa5, 0x6c, 0x96, 0xe1, 0xc2, 0x27, 0x80, 0x55, 0xe3, 0x1b, +0x5c, 0x20, 0xad, 0x02, 0x83, 0xa1, 0xc9, 0x51, 0xbd, 0x63, 0xf6, 0x08, 0x64, 0x38, 0x75, 0x7c, +0x50, 0xd9, 0xae, 0xbb, 0x1b, 0xab, 0x33, 0xd5, 0x61, 0xf1, 0xda, 0xe4, 0x52, 0x6d, 0x97, 0x3c, +0xdb, 0xec, 0x62, 0x37, 0x5b, 0xe9, 0x84, 0xda, 0x26, 0x26, 0xa2, 0xc2, 0x00, 0x67, 0x50, 0x82, +0x0c, 0xa2, 0xd4, 0xb0, 0xc9, 0xe7, 0x6e, 0x2a, 0x2a, 0xaa, 0x5a, 0x8a, 0x3c, 0xfa, 0xb8, 0xd6, +0x95, 0xdf, 0xb5, 0x20, 0x08, 0x65, 0xf4, 0x2f, 0x49, 0x2a, 0xeb, 0x06, 0xd4, 0x1a, 0x3a, 0x3d, +0xc7, 0xfe, 0xd2, 0x5c, 0xb5, 0x00, 0x14, 0x67, 0x32, 0xb2, 0x17, 0xe2, 0x17, 0x50, 0x97, 0xf0, +0x11, 0xb5, 0x1a, 0xe4, 0xa9, 0xe1, 0x40, 0x21, 0xbb, 0x75, 0x96, 0xb2, 0xc1, 0x90, 0x8a, 0x66, +0xfd, 0x2c, 0x1a, 0xa6, 0xc0, 0x4a, 0x53, 0xcb, 0x3e, 0x10, 0x0f, 0x0a, 0x73, 0x3c, 0x6b, 0x4f, +0xba, 0x14, 0x57, 0xce, 0xc8, 0xb7, 0xd8, 0x33, 0xf1, 0xc4, 0xba, 0x02, 0x13, 0xaa, 0xc6, 0x15, +0x9e, 0xd6, 0xfd, 0x77, 0x05, 0x81, 0x92, 0x61, 0x3b, 0x35, 0x3f, 0xbd, 0x38, 0x22, 0x2a, 0x5f, +0xc3, 0x09, 0xc5, 0x73, 0x22, 0x2d, 0x27, 0x8a, 0x42, 0xac, 0x06, 0xe2, 0x8b, 0x9e, 0x3d, 0x73, +0xfb, 0xf2, 0x71, 0x06, 0x07, 0x26, 0xc7, 0x25, 0xdd, 0x19, 0x7a, 0x54, 0xd8, 0xb8, 0x66, 0x6b, +0x73, 0xad, 0xc8, 0xa2, 0x24, 0x39, 0x4a, 0xab, 0xdc, 0x5a, 0x6e, 0x32, 0xd9, 0x4a, 0x12, 0xe4, +0xbd, 0x39, 0xf8, 0x72, 0x6a, 0xdc, 0x46, 0xfb, 0x18, 0x08, 0x01, 0xc5, 0xd3, 0xe7, 0xa2, 0xe9, +0xf4, 0xe4, 0xcc, 0xaf, 0x91, 0x1c, 0xc7, 0x57, 0xdc, 0x18, 0x53, 0x2a, 0x66, 0xeb, 0x29, 0xf9, +0xc5, 0x0e, 0x5a, 0x1c, 0x0d, 0xcc, 0xca, 0xb1, 0x67, 0x75, 0xff, 0x91, 0x58, 0x71, 0xff, 0x01, +0x56, 0xaa, 0x51, 0x75, 0xfc, 0x61, 0x8a, 0x2a, 0x1c, 0xb3, 0x0a, 0x4b, 0x9a, 0xea, 0xe3, 0xc4, +0x2a, 0x07, 0xd2, 0xce, 0x6d, 0xfc, 0x34, 0xf8, 0xb0, 0xe9, 0xe3, 0x4b, 0x71, 0x1f, 0x5f, 0x0e, +0xb9, 0x87, 0x25, 0x1c, 0xad, 0x7b, 0x52, 0xa0, 0x56, 0xcf, 0x90, 0xbe, 0x7d, 0xc0, 0x6c, 0x34, +0x28, 0x49, 0x77, 0xd4, 0x66, 0x12, 0x40, 0xa9, 0xd4, 0x32, 0xbe, 0x10, 0xad, 0x11, 0x73, 0xed, +0x10, 0x60, 0xc5, 0x76, 0x63, 0xe8, 0x1a, 0x12, 0x26, 0x94, 0xa4, 0xa7, 0xee, 0xc3, 0xd3, 0x47, +0xb6, 0x2f, 0xa1, 0x18, 0xe5, 0x7a, 0xf4, 0x85, 0x97, 0x1e, 0x09, 0xdf, 0xd6, 0x92, 0xc5, 0x2e, +0x3e, 0xe5, 0xa9, 0x70, 0x7b, 0x89, 0x91, 0x5b, 0x72, 0x9a, 0x53, 0x5c, 0xdd, 0xb9, 0xd5, 0xe0, +0xab, 0xb3, 0xc5, 0x14, 0x74, 0xcb, 0x67, 0xdc, 0xbb, 0x7c, 0x98, 0x31, 0xde, 0x2a, 0x61, 0x79, +0x48, 0xdf, 0xb5, 0x1f, 0xb6, 0x3f, 0xbd, 0x15, 0xc8, 0xdf, 0x69, 0xc1, 0x11, 0xfc, 0xd2, 0xcf, +0x33, 0xac, 0xe3, 0xdf, 0xc9, 0x26, 0xc7, 0x3c, 0x3d, 0xa8, 0x2b, 0xf1, 0xb7, 0x34, 0x01, 0x9e, +0x53, 0x5a, 0x98, 0xe7, 0x45, 0x3a, 0x46, 0x90, 0xe1, 0xa3, 0x5f, 0xd3, 0xc4, 0xbc, 0x64, 0xea, +0x9d, 0x90, 0xcc, 0xfc, 0x35, 0xa3, 0xd1, 0x8b, 0xc1, 0x9b, 0x6f, 0xce, 0xdb, 0xe7, 0x43, 0x3b, +0x3d, 0x2e, 0xff, 0xc5, 0x81, 0x27, 0x2f, 0xd2, 0x66, 0x85, 0x7c, 0x8c, 0x3d, 0x4f, 0x3d, 0xca, +0xce, 0x3a, 0xdf, 0xbc, 0xa2, 0x76, 0x3b, 0xe9, 0xc0, 0xf0, 0x22, 0xd8, 0x3c, 0xb8, 0x67, 0x7b, +0x9b, 0xf1, 0x8d, 0x30, 0x0c, 0x1a, 0xd9, 0xe1, 0xff, 0x85, 0x26, 0xf1, 0xe8, 0x99, 0x92, 0x24, +0xec, 0xb0, 0x7a, 0x20, 0xe3, 0x36, 0xe0, 0xe7, 0xc6, 0x9d, 0xfe, 0x7e, 0x52, 0x08, 0x00, 0x8c, +0xc3, 0x8b, 0x8d, 0x3b, 0xf8, 0x07, 0x8c, 0x1e, 0x26, 0xbd, 0x1b, 0x82, 0x80, 0xe4, 0xec, 0x7a, +0xf5, 0x3b, 0xb6, 0x2a, 0x59, 0xf0, 0x0c, 0x3d, 0x36, 0xaf, 0x8a, 0x59, 0xc1, 0x57, 0xc1, 0x9c, +0xf1, 0x6e, 0x81, 0x98, 0xc2, 0x18, 0x0a, 0xb8, 0xe3, 0xe8, 0xa6, 0xd6, 0x54, 0x3a, 0xfd, 0xb2, +0x3e, 0x13, 0x3e, 0xfb, 0xf9, 0x34, 0xc4, 0x8c, 0x6f, 0xbe, 0x11, 0x5b, 0x2d, 0x81, 0x7b, 0x20, +0xc9, 0xd3, 0xe6, 0x71, 0x3e, 0xae, 0xbf, 0x23, 0x09, 0xa0, 0x87, 0xe7, 0x49, 0x2d, 0xc9, 0x6a, +0x8d, 0xa3, 0x5e, 0x8e, 0xeb, 0x18, 0x33, 0x3a, 0xf8, 0x00, 0x3d, 0x91, 0xf0, 0x6c, 0x80, 0x38, +0x3b, 0x0c, 0xa1, 0xb1, 0x17, 0xb1, 0xe0, 0x6d, 0x63, 0x7c, 0xa4, 0xf5, 0x9a, 0x65, 0xc8, 0x3e, +0x09, 0xc5, 0x57, 0x79, 0x7a, 0x2a, 0x17, 0x8b, 0xbb, 0xe2, 0x75, 0xb8, 0x87, 0x14, 0x7b, 0xc6, +0x21, 0xa8, 0x9e, 0x31, 0xdc, 0x15, 0xee, 0x43, 0xf6, 0xc0, 0x11, 0x30, 0xa8, 0x45, 0xd3, 0x4b, +0x61, 0xfe, 0x9a, 0x19, 0xae, 0x01, 0x40, 0xf7, 0x56, 0xcc, 0xc6, 0xa9, 0x35, 0x10, 0xe7, 0x58, +0xbb, 0x13, 0x79, 0x19, 0x11, 0x47, 0x90, 0xf4, 0xa3, 0x40, 0xf2, 0xa1, 0xe0, 0xd0, 0xb0, 0xe4, +0xca, 0xf3, 0x03, 0x3a, 0xd5, 0xd9, 0x67, 0xbc, 0x35, 0x6d, 0x74, 0xa0, 0xd2, 0x10, 0x9a, 0x5e, +0x14, 0x7e, 0xb9, 0x10, 0x17, 0x1c, 0x1d, 0x44, 0x31, 0xe4, 0xcc, 0xa6, 0x95, 0xd2, 0x45, 0x1a, +0xfc, 0x9a, 0x7c, 0x62, 0xf2, 0xd8, 0xc4, 0x4b, 0x4c, 0x87, 0x13, 0xb7, 0x61, 0xe5, 0x7e, 0xa7, +0x47, 0xac, 0x97, 0xf0, 0x86, 0x2b, 0xe6, 0x1e, 0x8c, 0xd0, 0x66, 0x86, 0xfa, 0x18, 0x1f, 0x12, +0xa7, 0x84, 0x6f, 0x0d, 0x66, 0x1e, 0xe5, 0xf3, 0xb8, 0x1b, 0x37, 0xe4, 0x9a, 0x12, 0x81, 0x10, +0x18, 0xad, 0xdd, 0x9d, 0x5a, 0x4b, 0xce, 0xf5, 0xcb, 0x31, 0x6d, 0x2e, 0xa5, 0x82, 0x40, 0x87, +0x5c, 0x08, 0x62, 0xc2, 0xc2, 0x5d, 0xea, 0x78, 0x0a, 0xc1, 0x96, 0x99, 0xe5, 0xf4, 0x12, 0x5c, +0xf1, 0xee, 0x70, 0x59, 0xc6, 0x5e, 0xc5, 0xfa, 0xb3, 0xa3, 0x62, 0x71, 0xd8, 0x22, 0x6a, 0x99, +0xf9, 0xb7, 0xbe, 0x58, 0x45, 0x9a, 0x5a, 0xc1, 0xa9, 0x3f, 0x99, 0x7a, 0x16, 0x46, 0x52, 0x21, +0x4b, 0x0c, 0x52, 0xce, 0xa6, 0x6c, 0x44, 0xf7, 0x77, 0xc2, 0x10, 0x11, 0x13, 0xe2, 0x19, 0x2e, +0x5e, 0xb5, 0x4a, 0x5b, 0xfc, 0x66, 0x9d, 0xe1, 0xd0, 0x9d, 0xde, 0x46, 0xf2, 0xad, 0x35, 0x97, +0x64, 0xa9, 0x05, 0x0e, 0x3b, 0x0f, 0xf9, 0xc7, 0xe0, 0xcd, 0x3b, 0x8c, 0xff, 0x6b, 0xde, 0xb0, +0x7f, 0x3e, 0x1f, 0x3f, 0x7b, 0x66, 0xbd, 0x52, 0x40, 0x18, 0xde, 0x91, 0x61, 0xca, 0xae, 0x40, +0x56, 0x9b, 0x46, 0x5f, 0xd9, 0x2f, 0x13, 0x62, 0x7e, 0x22, 0xec, 0x4b, 0x64, 0x8d, 0x21, 0xa2, +0xe9, 0x83, 0xbb, 0xec, 0x7f, 0xd9, 0xb4, 0xfb, 0x4f, 0x21, 0x9e, 0xb4, 0x66, 0x15, 0x13, 0x95, +0x0f, 0x50, 0xb4, 0x9f, 0x77, 0xe8, 0xad, 0x24, 0x0e, 0x00, 0xb3, 0x73, 0x29, 0xd0, 0xc4, 0x25, +0xf7, 0x91, 0xe6, 0xac, 0xf4, 0x5f, 0x7f, 0xac, 0xd7, 0x68, 0x6b, 0x94, 0xd8, 0x7a, 0xcb, 0xb8, +0xd8, 0xcb, 0x24, 0x06, 0x88, 0x2e, 0x8e, 0x91, 0xaf, 0xce, 0x6f, 0x36, 0x2f, 0x2d, 0x1a, 0xac, +0xcc, 0x06, 0xb4, 0x0e, 0x66, 0x6e, 0x79, 0x15, 0xe5, 0xaa, 0x33, 0xeb, 0xb1, 0xe5, 0xa3, 0x62, +0x7a, 0x76, 0xfc, 0x4a, 0xbd, 0xa2, 0xbe, 0x85, 0x44, 0x6c, 0x31, 0xae, 0x5b, 0xd9, 0x85, 0x5e, +0xb7, 0x88, 0xdb, 0x29, 0xa1, 0x1e, 0x78, 0x98, 0x56, 0xbf, 0xfb, 0x4c, 0x63, 0xac, 0x96, 0xfb, +0xa1, 0x18, 0x91, 0xc2, 0x21, 0x90, 0x7c, 0xfa, 0x9d, 0x6d, 0x09, 0xb9, 0xae, 0x9e, 0x90, 0xf3, +0x33, 0x31, 0x95, 0xa3, 0xf4, 0xc1, 0xfa, 0x89, 0xad, 0x6d, 0x30, 0x1c, 0x42, 0x5b, 0x56, 0x6f, +0x85, 0x26, 0x6a, 0xf6, 0x95, 0xf6, 0x3c, 0xbc, 0x9b, 0xb1, 0x70, 0x50, 0xeb, 0x9e, 0x40, 0xa2, +0x97, 0x50, 0x2b, 0x90, 0x7b, 0x38, 0x64, 0xf1, 0xae, 0xa1, 0x23, 0xeb, 0x34, 0x22, 0x1a, 0x97, +0x9d, 0xdb, 0x48, 0x44, 0x7d, 0x1a, 0x56, 0xfa, 0xdd, 0x18, 0xc9, 0xac, 0xe9, 0x2a, 0x98, 0x97, +0x48, 0xff, 0x79, 0x66, 0x44, 0xfd, 0x2e, 0xa6, 0x15, 0x0b, 0x51, 0x3e, 0x0a, 0xaf, 0x62, 0x16, +0x1a, 0x37, 0xab, 0x72, 0xa5, 0xf1, 0x0b, 0xa7, 0x8c, 0x00, 0xf2, 0xaa, 0xd3, 0x34, 0x01, 0xb1, +0xd0, 0x2c, 0x88, 0xb8, 0x25, 0xd6, 0x62, 0x02, 0x52, 0xa4, 0x4a, 0xa2, 0xb1, 0xe3, 0x07, 0x91, +0x41, 0x30, 0x55, 0x2f, 0x14, 0x61, 0x29, 0xd0, 0x94, 0x1e, 0x4e, 0xe3, 0x02, 0x39, 0xc9, 0xb1, +0xfc, 0x43, 0xec, 0x83, 0x28, 0xf1, 0x98, 0x0e, 0xe9, 0x26, 0x79, 0x1c, 0x48, 0xa9, 0x22, 0x21, +0x4a, 0x82, 0xaf, 0x43, 0x35, 0xf4, 0x9c, 0x39, 0x08, 0x8b, 0x93, 0xb8, 0x42, 0x40, 0x7b, 0x99, +0x4c, 0xfa, 0x63, 0x90, 0x4e, 0x31, 0x5f, 0x9f, 0x60, 0x60, 0xa7, 0x1b, 0xb8, 0x38, 0x83, 0x63, +0xb3, 0xe7, 0x2e, 0xcc, 0x1a, 0x21, 0xdd, 0x4b, 0xfb, 0x62, 0x2c, 0x30, 0xae, 0x15, 0x6b, 0xe2, +0x37, 0x63, 0xc8, 0xa1, 0x16, 0x57, 0x83, 0x14, 0xcc, 0xae, 0xe4, 0x31, 0x1b, 0x06, 0xf7, 0xbe, +0xf8, 0x56, 0xef, 0xd4, 0x60, 0x9e, 0x68, 0x0c, 0xa0, 0x82, 0x7e, 0x71, 0x87, 0x9e, 0xd2, 0xa7, +0x5d, 0x86, 0xc6, 0x3d, 0x88, 0x4a, 0xd9, 0x01, 0x1e, 0x44, 0xa1, 0xc0, 0x91, 0x42, 0xd2, 0xfc, +0xab, 0xf2, 0x7a, 0x94, 0x16, 0xf1, 0x39, 0x50, 0x83, 0x1c, 0x65, 0x9d, 0xc3, 0x26, 0x93, 0xdf, +0x65, 0x0c, 0xe3, 0x83, 0xb5, 0x7f, 0x72, 0x73, 0xef, 0xd7, 0x62, 0xe9, 0x7f, 0xe2, 0xd1, 0xcc, +0x9e, 0x77, 0x9f, 0xab, 0x30, 0x26, 0x2a, 0x2c, 0x18, 0xb1, 0x3c, 0x64, 0xcf, 0x54, 0x49, 0x75, +0xe1, 0xbe, 0x51, 0xdc, 0xaa, 0xdf, 0xeb, 0xc0, 0x41, 0xc7, 0x24, 0x4b, 0xe3, 0xe0, 0xe7, 0xbc, +0xed, 0x32, 0x15, 0x9f, 0x4b, 0x2f, 0x17, 0xc1, 0xce, 0x39, 0x38, 0x83, 0xcb, 0x97, 0x30, 0x7b, +0x82, 0x46, 0x65, 0x55, 0x2d, 0xd8, 0x16, 0xa7, 0xd3, 0x33, 0x73, 0x5a, 0xb2, 0xe0, 0xae, 0xfc, +0x12, 0x8a, 0xf4, 0x56, 0xd9, 0x7b, 0xd2, 0x02, 0xcf, 0x99, 0x37, 0x04, 0x56, 0x90, 0xab, 0x10, +0x82, 0x3e, 0xcc, 0x2c, 0x8d, 0x53, 0x67, 0x9b, 0x43, 0x59, 0xc0, 0x80, 0xec, 0x18, 0x5e, 0x03, +0x04, 0x5d, 0x1d, 0x5f, 0xb4, 0x03, 0x8f, 0xc7, 0x38, 0x10, 0x6c, 0xd7, 0xfe, 0x8f, 0x2c, 0xd4, +0x0a, 0x1e, 0x47, 0x5f, 0x2a, 0x26, 0xd3, 0x4b, 0x3e, 0x46, 0x87, 0xd4, 0x94, 0xba, 0xe8, 0x19, +0x89, 0x90, 0x70, 0x90, 0xa0, 0xee, 0x8d, 0x74, 0x87, 0x1c, 0x35, 0x6b, 0x48, 0x94, 0x3d, 0x80, +0x4c, 0x8c, 0x84, 0x35, 0x86, 0x97, 0xb4, 0xe2, 0xae, 0x4c, 0xae, 0x30, 0xcf, 0x6e, 0x34, 0xa5, +0xbb, 0xa5, 0xf5, 0xdd, 0x7e, 0xe8, 0xea, 0x37, 0x54, 0xe2, 0xc3, 0x91, 0x03, 0xcb, 0x8c, 0x4b, +0x23, 0x73, 0x63, 0x5b, 0x35, 0x63, 0x5b, 0x89, 0xbb, 0x01, 0xce, 0x8d, 0x73, 0xa3, 0x4f, 0x89, +0x76, 0x15, 0x5d, 0x50, 0x26, 0x01, 0x8c, 0x7b, 0x23, 0x6b, 0x84, 0xa6, 0x60, 0x44, 0x2a, 0x0b, +0x33, 0x8f, 0x00, 0xad, 0x0e, 0x05, 0x75, 0x41, 0xae, 0x96, 0x1b, 0x2d, 0x0b, 0xe9, 0xdb, 0xba, +0xbe, 0xe0, 0xc5, 0x65, 0x35, 0x02, 0xf2, 0x04, 0x6c, 0x3f, 0x81, 0xe0, 0x0c, 0x2c, 0xd7, 0xde, +0xc8, 0xb2, 0x6c, 0x5d, 0x1e, 0x9b, 0xe0, 0x65, 0x1e, 0x13, 0xd8, 0x6a, 0x92, 0xa7, 0x59, 0x14, +0x78, 0x92, 0xb7, 0x11, 0x06, 0xea, 0xc2, 0x8d, 0x61, 0x82, 0x5d, 0xfe, 0x18, 0x66, 0x02, 0x8e, +0x7a, 0x09, 0x7f, 0xdc, 0x7e, 0xca, 0xa7, 0x76, 0x99, 0x50, 0x25, 0xf6, 0x7e, 0x30, 0xaa, 0xf7, +0x82, 0xae, 0xfa, 0xe3, 0xdf, 0x56, 0xa9, 0xab, 0xa4, 0xa3, 0x2c, 0x4d, 0x02, 0x4c, 0x38, 0x02, +0x2d, 0x7f, 0x37, 0x54, 0xca, 0x3f, 0x6e, 0x7b, 0x0e, 0xb2, 0xa5, 0x68, 0x76, 0x98, 0x85, 0xd4, +0x83, 0x5b, 0x26, 0xb3, 0xcd, 0xe9, 0x0c, 0xdf, 0xa5, 0x35, 0x4d, 0xd8, 0x5c, 0x59, 0x53, 0xe8, +0x81, 0xf0, 0x33, 0xc9, 0xc9, 0xef, 0x84, 0xf6, 0x5c, 0xf8, 0x6e, 0x32, 0xe7, 0x20, 0x94, 0x79 + +cipher_key = +0xE4, 0x23, 0x33, 0x8A, 0x35, 0x64, 0x61, 0xE2, 0x49, 0x03, 0xDD, 0xC6, 0xB8, 0xCA, 0x55, 0x7A + +auth_key = +0xaf, 0x96, 0x42, 0xf1, 0x8c, 0x50, 0xdc, 0x67, 0x1a, 0x43, 0x47, 0x62, 0xc7, 0x04, 0xab, 0x05, +0xf5, 0x0c, 0xe7, 0xa2, 0xa6, 0x23, 0xd5, 0x3d, 0x95, 0xd8, 0xcd, 0x86, 0x79, 0xf5, 0x01, 0x47, +0x4f, 0xf9, 0x1d, 0x9d, 0x36, 0xf7, 0x68, 0x1a, 0x64, 0x44, 0x58, 0x5d, 0xe5, 0x81, 0x15, 0x2a, +0x41, 0xe4, 0x0e, 0xaa, 0x1f, 0x04, 0x21, 0xff, 0x2c, 0xf3, 0x73, 0x2b, 0x48, 0x1e, 0xd2, 0xf7, +0xf6, 0xd9, 0xaf, 0xbf, 0x08, 0x3b, 0xbb, 0x19, 0x5f, 0xf6, 0x7d, 0x25, 0x85, 0xdf, 0x6b, 0x54, +0xd0, 0xe7, 0x4b, 0x9e, 0xc7, 0xef, 0xca, 0x48, 0x6f, 0x21, 0xd7, 0x51, 0xc8, 0x21, 0xc1, 0x15, +0xe8, 0x38, 0x36, 0x58, 0x39, 0xd9, 0x9a, 0xc5, 0xe7, 0x3b, 0xc4, 0x47, 0xe2, 0xbd, 0x80, 0x73, +0xf8, 0xd1, 0x9a, 0x5e, 0x4b, 0xfb, 0x52, 0x6b, 0x50, 0xaf, 0x8b, 0xb7, 0xb5, 0x2c, 0x52, 0x84 + +cipher_iv = +0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F + +#################### +# sha_hmac_buff_32 # +#################### +[sha1_hmac_buff_32] +digest = +0xAD, 0x0F, 0xE8, 0xAD, 0x30, 0xD6, 0x0A, 0x2B, 0x6B, 0x3D, 0x24, 0x79, 0x6D, 0xB6, 0x80, 0x43, +0x1D, 0xD1, 0x8A, 0xA8 + +[sha224_hmac_buff_32] +digest = +0xB4, 0x1C, 0xF4, 0x73, 0x04, 0x62, 0x2D, 0xAF, 0x59, 0x05, 0xE8, 0x55, 0xE4, 0x8B, 0x24, 0xB4, +0x0F, 0x4C, 0x3D, 0xBE, 0xFF, 0xFF, 0x8D, 0x19, 0x0D, 0x38, 0xA1, 0xFC + +[sha256_hmac_buff_32] +digest = +0x77, 0x54, 0x8D, 0x73, 0x7D, 0xB9, 0x78, 0x2F, 0x3D, 0xEE, 0xA2, 0xE7, 0xC9, 0x03, 0xF3, 0xB0, +0x17, 0x8E, 0x71, 0x81, 0xB7, 0xA7, 0x5F, 0x9F, 0xF3, 0xED, 0x55, 0x1A, 0x69, 0x68, 0x52, 0xE5 + +[sha384_hmac_buff_32] +digest = +0x1A, 0x3D, 0x1A, 0xD8, 0x0C, 0x04, 0x4B, 0x14, 0x9F, 0xF6, 0x9B, 0x80, 0x5C, 0xA2, 0x78, 0xFC, +0xE6, 0xA3, 0xBA, 0x32, 0x09, 0x04, 0x52, 0x24, 0x76, 0xC9, 0xFC, 0x70, 0xA6, 0x00, 0xB4, 0x41, +0xA1, 0x48, 0x1D, 0xCC, 0x54, 0x59, 0xE6, 0x94, 0x35, 0x36, 0xCC, 0xAB, 0x4B, 0xF5, 0xE4, 0xCA + +[sha512_hmac_buff_32] +digest = +0xED, 0xD7, 0x96, 0x0B, 0xC4, 0x8F, 0xBF, 0xF3, 0xEA, 0x7D, 0x5D, 0x57, 0x2A, 0x50, 0x50, 0xAC, +0x33, 0xF2, 0xED, 0x56, 0x0A, 0xF7, 0x97, 0x4A, 0x36, 0x8D, 0x3B, 0xA5, 0x9F, 0x7A, 0x6D, 0x57, +0xE0, 0x94, 0x10, 0xB9, 0x15, 0xC6, 0x1B, 0x7F, 0x17, 0xB3, 0x48, 0xB5, 0xF9, 0x93, 0xD8, 0xCA, +0x74, 0x56, 0xED, 0xBD, 0x55, 0x14, 0xD2, 0xB6, 0x36, 0x07, 0x06, 0x73, 0x66, 0x41, 0x50, 0x84 + +#################### +# sha_hmac_buff_64 # +#################### +[sha1_hmac_buff_64] +digest = +0xC9, 0xC3, 0x65, 0x23, 0xCE, 0x9D, 0x6E, 0x35, 0xDC, 0x65, 0x3B, 0x21, 0x33, 0xFF, 0x1E, 0x74, +0xAF, 0x48, 0x24, 0xD9 + +[sha224_hmac_buff_64] +digest = +0x9F, 0xC3, 0x34, 0x0F, 0xB0, 0xA5, 0x4D, 0x89, 0xA4, 0x20, 0x01, 0xDF, 0x40, 0xAE, 0xA2, 0x41, +0x4E, 0x97, 0x38, 0xBA, 0xA7, 0xF9, 0x94, 0x1F, 0x56, 0x4E, 0x00, 0x0A + +[sha256_hmac_buff_64] +digest = +0x66, 0x52, 0xF3, 0xEB, 0xC7, 0x06, 0xEF, 0x21, 0x82, 0x7C, 0xCF, 0x7F, 0x5B, 0x6B, 0x77, 0x2F, +0x28, 0x61, 0x06, 0x2B, 0x67, 0x4B, 0x2D, 0x62, 0x71, 0x53, 0xBE, 0x12, 0xF9, 0x5B, 0xD8, 0x64 + +[sha384_hmac_buff_64] +digest = +0x63, 0x50, 0x09, 0x45, 0x87, 0x02, 0x7F, 0x85, 0xD3, 0xC8, 0xF2, 0x26, 0x01, 0xE8, 0x2C, 0x28, +0xB4, 0x86, 0x5A, 0x8E, 0x8E, 0x95, 0x27, 0x6A, 0x74, 0xF0, 0x29, 0xC2, 0x2D, 0x13, 0x91, 0xD5, +0x38, 0xDB, 0x06, 0x7E, 0xB4, 0x8C, 0xD6, 0x30, 0x54, 0x86, 0xE7, 0xBB, 0x5C, 0xB6, 0xDD, 0x90 + +[sha512_hmac_buff_64] +digest = +0xBE, 0x52, 0xBF, 0x69, 0x2B, 0x7F, 0xD5, 0x37, 0x48, 0x76, 0xCF, 0xBD, 0x23, 0x18, 0x45, 0x90, +0x74, 0x25, 0x07, 0x91, 0x13, 0x37, 0xF3, 0x26, 0xEE, 0x68, 0xEC, 0xFC, 0xCB, 0x60, 0x53, 0x96, +0x54, 0xF6, 0xE8, 0xAC, 0xF7, 0xB1, 0x52, 0x31, 0x7E, 0x5D, 0x99, 0xF3, 0x86, 0xF2, 0x98, 0x7D, +0xD4, 0x38, 0xD9, 0xF1, 0x4C, 0x08, 0x87, 0x7F, 0xB9, 0x17, 0x97, 0x39, 0xDB, 0x68, 0x39, 0x19 +##################### +# sha_hmac_buff_128 # +##################### +[sha1_hmac_buff_128] +digest = +0xB0, 0x6F, 0x7D, 0xB3, 0x29, 0xC3, 0x2B, 0x5D, 0xB7, 0xF1, 0x13, 0xFB, 0x9E, 0x90, 0x5D, 0xF1, +0x48, 0xBC, 0x71, 0xA4 + +[sha224_hmac_buff_128] +digest = +0xFC, 0x59, 0xAF, 0xF2, 0x83, 0x2E, 0x07, 0x08, 0xBF, 0xB4, 0x3C, 0x24, 0xA8, 0x52, 0x7B, 0x9E, +0x92, 0x83, 0xCE, 0x96, 0xEE, 0x8B, 0x65, 0x72, 0x00, 0x12, 0xC6, 0x98 + +[sha256_hmac_buff_128] +digest = +0x7E, 0xCA, 0x95, 0xD5, 0x63, 0xA8, 0xCA, 0xA6, 0xC5, 0x41, 0x75, 0x12, 0x60, 0xDF, 0xFE, 0x16, +0x70, 0xEB, 0xCC, 0x4E, 0xEB, 0x00, 0x86, 0xF0, 0xEC, 0x45, 0x44, 0x76, 0x62, 0x55, 0x48, 0x56 + +[sha384_hmac_buff_128] +digest = +0x1B, 0x2A, 0xAA, 0x7F, 0x2E, 0x58, 0x1F, 0x64, 0xB4, 0xE6, 0x29, 0xE4, 0x74, 0x78, 0x09, 0xD7, +0xBA, 0xDD, 0x18, 0xB6, 0xE4, 0x21, 0xF5, 0x8F, 0x40, 0x45, 0x65, 0xD1, 0xBE, 0x4F, 0x7B, 0x27, +0xF4, 0x64, 0x72, 0x55, 0x53, 0xAB, 0x39, 0x05, 0x7A, 0x6D, 0xAA, 0x12, 0x75, 0x03, 0x67, 0x4E + +[sha512_hmac_buff_128] +digest = +0x5F, 0x8F, 0xA8, 0xFA, 0xEA, 0x05, 0x29, 0xBD, 0x3B, 0xBA, 0xF6, 0xA7, 0x93, 0x9E, 0x16, 0xF1, +0x8B, 0x10, 0x2F, 0x6D, 0x08, 0x18, 0x54, 0xD2, 0x39, 0xEB, 0xF9, 0x70, 0xCC, 0x55, 0xA0, 0xC3, +0x08, 0x9B, 0x8E, 0x55, 0x81, 0x1A, 0xCE, 0x0D, 0x09, 0x97, 0x4E, 0x34, 0xD1, 0xE6, 0x25, 0x05, +0x94, 0xC7, 0x05, 0x30, 0xAF, 0x2F, 0x7F, 0x54, 0xAA, 0xB8, 0xC5, 0x8E, 0x3D, 0xBB, 0xF2, 0x12 + +##################### +# sha_hmac_buff_256 # +##################### +[sha1_hmac_buff_256] +digest = +0xF9, 0xCA, 0x12, 0x7D, 0x60, 0x68, 0xB7, 0xAF, 0xDC, 0xAC, 0x41, 0x13, 0x1C, 0xA8, 0xC1, 0x85, +0x65, 0x11, 0x31, 0x4C + +[sha224_hmac_buff_256] +digest = +0x49, 0xED, 0xA4, 0x27, 0x51, 0x6A, 0x46, 0xE4, 0x31, 0x12, 0x72, 0x92, 0xB8, 0x81, 0x16, 0x97, +0x19, 0x4F, 0x3B, 0xAC, 0xD1, 0xCE, 0x06, 0x40, 0xD4, 0xEA, 0x8E, 0xC3 + +[sha256_hmac_buff_256] +digest = +0xB9, 0xFB, 0x21, 0x16, 0x0C, 0x08, 0xD1, 0xE0, 0x49, 0xB8, 0xC8, 0x7E, 0xCC, 0xF0, 0xBA, 0x29, +0x32, 0xCE, 0x53, 0x03, 0xE8, 0xFB, 0xD2, 0x44, 0xB7, 0xB9, 0xFE, 0xE8, 0x03, 0x86, 0xE2, 0x68 + +[sha384_hmac_buff_256] +digest = +0x47, 0xEA, 0x51, 0xA7, 0xAD, 0xA2, 0x34, 0x3D, 0x4A, 0x3A, 0x86, 0x89, 0x78, 0x56, 0xCF, 0x21, +0x94, 0xBF, 0x80, 0x33, 0x6B, 0x42, 0x73, 0x01, 0xAD, 0x6B, 0xE0, 0xEC, 0x10, 0xEE, 0x6E, 0xEC, +0xED, 0x54, 0x50, 0x5E, 0x96, 0x3B, 0xE8, 0x2A, 0x8C, 0x33, 0x67, 0x9B, 0x17, 0x6C, 0xBB, 0xF8 + +[sha512_hmac_buff_256] +digest = +0x01, 0xAE, 0xE7, 0x74, 0xCD, 0x86, 0x43, 0xBC, 0x8A, 0xF6, 0xAF, 0x6C, 0xDE, 0x9E, 0x9A, 0xB7, +0x6B, 0xCF, 0x98, 0x95, 0x31, 0xE8, 0x37, 0x3B, 0x3F, 0xF3, 0xC1, 0x00, 0xA0, 0xA6, 0xE5, 0x15, +0x60, 0x36, 0x7E, 0x7C, 0x96, 0xAB, 0x17, 0xB9, 0x79, 0x3D, 0x3E, 0x43, 0xBC, 0xA0, 0xA0, 0x8B, +0x14, 0x14, 0x22, 0x86, 0xE9, 0xF6, 0x96, 0x38, 0x9F, 0x24, 0x45, 0x9C, 0xE8, 0x63, 0x2A, 0x22 + +##################### +# sha_hmac_buff_512 # +##################### +[sha1_hmac_buff_512] +digest = +0x45, 0x8D, 0x5B, 0x40, 0x0D, 0x34, 0x3A, 0x7B, 0xB2, 0xB1, 0xE7, 0x62, 0xDE, 0x2B, 0xD0, 0x46, +0xCD, 0x4B, 0x55, 0x95 + +[sha224_hmac_buff_512] +digest = +0xE1, 0x82, 0x07, 0x4F, 0x6B, 0x24, 0x4A, 0x57, 0xE9, 0x04, 0x14, 0xB1, 0x7F, 0xD2, 0x4C, 0xA0, +0x89, 0x8B, 0xB2, 0xA2, 0x28, 0x9F, 0xFE, 0x7C, 0xD1, 0x7F, 0x35, 0x07 + +[sha256_hmac_buff_512] +digest = +0xB9, 0x75, 0x4F, 0x70, 0xC7, 0x8C, 0xF2, 0x62, 0x89, 0x3C, 0x41, 0x4D, 0x1D, 0x15, 0x81, 0x2A, +0x5A, 0xCB, 0x56, 0x62, 0xF8, 0xE9, 0x38, 0x13, 0xC9, 0x4D, 0xC3, 0x9D, 0xF0, 0x82, 0xAC, 0xD2 + +[sha384_hmac_buff_512] +digest = +0x9C, 0xAE, 0x77, 0x8D, 0x7E, 0x26, 0x01, 0xA6, 0x46, 0x47, 0xDF, 0xB7, 0x23, 0x6F, 0x17, 0x6B, +0x9F, 0x4D, 0x94, 0xBB, 0x78, 0xD8, 0x2D, 0x90, 0xB1, 0xC1, 0x65, 0x6D, 0x92, 0x4E, 0x54, 0x7A, +0xA5, 0xF6, 0x80, 0x29, 0x82, 0x77, 0xAC, 0xC3, 0x58, 0xE5, 0x14, 0x75, 0x64, 0x9D, 0x02, 0x6E + +[sha512_hmac_buff_512] +digest = +0x33, 0xB6, 0xD1, 0xC4, 0x5F, 0xDB, 0xEF, 0xF4, 0x14, 0xE8, 0xDA, 0x07, 0x30, 0xB6, 0xC6, 0xC9, +0x4F, 0xCF, 0x64, 0x48, 0x08, 0xA2, 0xC1, 0x9D, 0x03, 0xAD, 0x93, 0x62, 0x41, 0xB6, 0xB9, 0xEC, +0x1B, 0xD1, 0xAC, 0xA1, 0xC5, 0x94, 0x67, 0x19, 0xA3, 0x4B, 0x53, 0xCE, 0x0C, 0x8A, 0x27, 0x07, +0x37, 0x75, 0x93, 0xC3, 0xC6, 0x60, 0x19, 0x39, 0x9E, 0x02, 0x23, 0x9A, 0xE6, 0xA9, 0x34, 0x1A + +###################### +# sha_hmac_buff_1024 # +###################### +[sha1_hmac_buff_1024] +digest = +0x0B, 0x26, 0x34, 0xAD, 0x56, 0x44, 0x39, 0x4A, 0x0D, 0x2F, 0x14, 0xFB, 0x60, 0x77, 0xDD, 0xFC, +0x0B, 0x5F, 0x9F, 0x99 + +[sha224_hmac_buff_1024] +digest = +0x56, 0x41, 0xC2, 0xF0, 0x73, 0x5C, 0x21, 0x13, 0x7E, 0x47, 0xCC, 0xAB, 0x21, 0x3D, 0x5E, 0xA7, +0xC6, 0x1E, 0xFF, 0x26, 0x59, 0x0C, 0x71, 0x95, 0x72, 0x76, 0x0D, 0x00 + +[sha256_hmac_buff_1024] +digest = +0x08, 0x91, 0x23, 0x89, 0x0F, 0xB0, 0xE4, 0x25, 0x9F, 0xC7, 0x46, 0x6B, 0xC3, 0x39, 0xE0, 0x9C, +0xE2, 0xAE, 0xA3, 0xCF, 0xB8, 0xA0, 0x0A, 0xCF, 0x29, 0xEE, 0x0D, 0x83, 0x8A, 0xE5, 0xE4, 0x85 + +[sha384_hmac_buff_1024] +digest = +0x38, 0xC7, 0x19, 0xA4, 0x46, 0x14, 0x79, 0xA4, 0xAB, 0x40, 0x61, 0xBC, 0xFB, 0x87, 0x16, 0xE2, +0x08, 0x90, 0xAD, 0x33, 0x5D, 0x37, 0xB6, 0xCA, 0x80, 0xEE, 0x59, 0x9C, 0xBF, 0xA8, 0xEB, 0x78, +0xC2, 0xE2, 0x2D, 0x6E, 0x2E, 0x98, 0x98, 0x6F, 0x07, 0x6A, 0x39, 0x57, 0x8F, 0xCE, 0xEE, 0x64 + +[sha512_hmac_buff_1024] +digest = +0x17, 0x60, 0x08, 0x57, 0x43, 0x20, 0xF6, 0xB5, 0x6D, 0x0D, 0x7F, 0x7C, 0xB9, 0x09, 0x3F, 0x6D, +0x3E, 0x75, 0x2F, 0x17, 0xDA, 0x19, 0x58, 0xF0, 0xEC, 0xED, 0x96, 0xA9, 0x57, 0x05, 0xCD, 0x23, +0x0F, 0x1E, 0x38, 0x55, 0x2D, 0x8E, 0x36, 0x14, 0xF4, 0x99, 0x5E, 0x3C, 0x33, 0xBB, 0x99, 0xC9, +0xCD, 0x7A, 0xF4, 0x87, 0x10, 0xB8, 0x6C, 0xB1, 0x14, 0x2D, 0xA8, 0xCE, 0xFE, 0xF8, 0x6F, 0xD9 + +###################### +# sha_hmac_buff_2048 # +###################### +[sha1_hmac_buff_2048] +digest = +0x58, 0xE4, 0xBB, 0x8D, 0x63, 0x5D, 0x23, 0xF1, 0xAB, 0xB5, 0xBD, 0xD8, 0x71, 0xC9, 0x05, 0x0A, +0x65, 0x5D, 0x2D, 0x2D + +[sha224_hmac_buff_2048] +digest = +0xA3, 0x9A, 0x47, 0x68, 0x32, 0x3A, 0xA8, 0xE4, 0xBE, 0x23, 0xEA, 0xEE, 0x2D, 0x5E, 0x3C, 0x2E, +0xD3, 0x99, 0xBE, 0x87, 0x19, 0x17, 0xC5, 0x13, 0x6D, 0xB7, 0x05, 0x97 + +[sha256_hmac_buff_2048] +digest = +0x5B, 0x36, 0x1A, 0xF0, 0x55, 0xAC, 0xC3, 0xEA, 0x1B, 0x01, 0xCF, 0xCE, 0x89, 0x0D, 0x6A, 0xC3, +0x5F, 0x9A, 0xD3, 0x49, 0xCC, 0xA4, 0xDF, 0xDD, 0x44, 0x1F, 0x9D, 0x6C, 0xB1, 0x92, 0xDF, 0xB9 + +[sha384_hmac_buff_2048] +digest = +0x24, 0x17, 0xA2, 0x61, 0xFF, 0x46, 0xA2, 0x2E, 0xE5, 0xC3, 0xB4, 0x47, 0x10, 0x8C, 0x54, 0xD2, +0xC2, 0x4D, 0x15, 0xA2, 0x8D, 0xEF, 0x98, 0x6E, 0xE0, 0xB1, 0x31, 0x3B, 0x7D, 0xDE, 0x41, 0x8E, +0x98, 0xB9, 0xE9, 0xD2, 0xD8, 0xE5, 0x75, 0x6D, 0xC5, 0xF0, 0x1A, 0xC4, 0x1B, 0x8B, 0xC1, 0xA4 + +[sha512_hmac_buff_2048] +digest = +0xD8, 0x77, 0x7A, 0x0F, 0x63, 0x1E, 0x92, 0x7B, 0x87, 0xCE, 0x07, 0x24, 0x7E, 0xE4, 0x36, 0x30, +0x16, 0x76, 0x0D, 0xEC, 0xEF, 0x01, 0xF5, 0xD5, 0x44, 0xB7, 0xF3, 0x51, 0x31, 0x6A, 0xC2, 0x80, +0xCD, 0x4C, 0x7F, 0xD4, 0xA6, 0x90, 0x85, 0xAE, 0x49, 0xB1, 0xF1, 0xB0, 0xC4, 0x16, 0x79, 0xC3, +0xE3, 0x8B, 0x67, 0xC3, 0xAA, 0xC1, 0x9C, 0x8D, 0xE0, 0x22, 0xB3, 0xFD, 0x09, 0xD5, 0x40, 0xAC diff --git a/dpdk/app/test-crypto-perf/data/aes_cbc_192_sha.data b/dpdk/app/test-crypto-perf/data/aes_cbc_192_sha.data new file mode 100644 index 00000000..3f85a004 --- /dev/null +++ b/dpdk/app/test-crypto-perf/data/aes_cbc_192_sha.data @@ -0,0 +1,504 @@ +# List of tests for AES-192 CBC: +# 1) [sha1_hmac_buff_x] +# 2) [sha224_hmac_buff_x] +# 3) [sha256_hmac_buff_x] +# 4) [sha384_hmac_buff_x] +# 5) [sha512_hmac_buff_x] +# where x is one of the values: 32, 64, 128, 256, 512, 1024, 2048 + +########## +# GLOBAL # +########## +plaintext = +0xff, 0xca, 0xfb, 0xf1, 0x38, 0x20, 0x2f, 0x7b, 0x24, 0x98, 0x26, 0x7d, 0x1d, 0x9f, 0xb3, 0x93, +0xd9, 0xef, 0xbd, 0xad, 0x4e, 0x40, 0xbd, 0x60, 0xe9, 0x48, 0x59, 0x90, 0x67, 0xd7, 0x2b, 0x7b, +0x8a, 0xe0, 0x4d, 0xb0, 0x70, 0x38, 0xcc, 0x48, 0x61, 0x7d, 0xee, 0xd6, 0x35, 0x49, 0xae, 0xb4, +0xaf, 0x6b, 0xdd, 0xe6, 0x21, 0xc0, 0x60, 0xce, 0x0a, 0xf4, 0x1c, 0x2e, 0x1c, 0x8d, 0xe8, 0x7b, +0x59, 0xda, 0x19, 0x4f, 0xec, 0x07, 0x8e, 0xe2, 0xf0, 0x61, 0xf9, 0x27, 0x61, 0x6f, 0xf8, 0xdf, +0x62, 0x4d, 0xaf, 0x06, 0xfe, 0x41, 0xa6, 0xa6, 0xf9, 0xa2, 0x06, 0x40, 0xb3, 0x04, 0xbd, 0xe6, +0xc8, 0x17, 0xfb, 0x56, 0x6f, 0xa9, 0x3b, 0x8e, 0xa6, 0x58, 0xdc, 0x91, 0x17, 0x58, 0x42, 0x95, +0xa3, 0x7c, 0x81, 0x78, 0xa6, 0x3d, 0x3f, 0x75, 0x74, 0x17, 0x1a, 0xd3, 0x6c, 0x2f, 0x48, 0x39, +0x20, 0x20, 0xc1, 0x9a, 0x29, 0x84, 0x7d, 0x2d, 0x52, 0xa1, 0xf9, 0x5c, 0xf3, 0x4f, 0x91, 0xfc, +0x75, 0xcf, 0xd6, 0x2d, 0xe7, 0x9a, 0x59, 0x6e, 0x00, 0x0e, 0x8d, 0x22, 0x17, 0xbd, 0xa0, 0xdd, +0x79, 0x1f, 0x71, 0xe6, 0xcd, 0x2f, 0xb1, 0xb6, 0xbc, 0xc3, 0xdb, 0x02, 0x91, 0x41, 0x9b, 0x09, +0xa9, 0xd2, 0x7e, 0xbd, 0x2c, 0x18, 0xae, 0xc0, 0x93, 0x0c, 0x02, 0x9a, 0xdb, 0x4e, 0xaa, 0xeb, +0x84, 0x4b, 0x43, 0x5e, 0xf0, 0x98, 0xf2, 0x5f, 0x86, 0x70, 0x96, 0x90, 0x15, 0x30, 0xcf, 0x3a, +0xc9, 0x33, 0x21, 0xec, 0x59, 0x86, 0xfc, 0x65, 0x7d, 0xbe, 0xb9, 0xf8, 0x97, 0xf9, 0x30, 0xa9, +0x6d, 0xfc, 0x0c, 0x6e, 0x36, 0x67, 0xd5, 0xa6, 0x67, 0xd9, 0xbd, 0x9b, 0x34, 0x5d, 0xa7, 0xdd, +0xda, 0x46, 0x33, 0x25, 0x60, 0x4a, 0x18, 0xf1, 0x55, 0x07, 0xb2, 0xb7, 0x26, 0x7b, 0xa6, 0x1e, +0x77, 0xbe, 0x7f, 0x35, 0x46, 0xdf, 0x56, 0x9c, 0x22, 0x19, 0xc8, 0x85, 0xa2, 0x45, 0xb2, 0xad, +0xf9, 0x26, 0x66, 0xab, 0xfc, 0x97, 0x4b, 0x51, 0x32, 0x36, 0xbc, 0xad, 0xcf, 0x54, 0x3a, 0x4f, +0x94, 0xdb, 0xd2, 0xf9, 0x67, 0x1b, 0x3b, 0xe5, 0xb2, 0x1d, 0xc5, 0x52, 0x64, 0x2c, 0x06, 0x44, +0xcf, 0x18, 0x83, 0xe0, 0xd8, 0x04, 0x92, 0xa9, 0xc4, 0x3c, 0x8b, 0xa3, 0x2b, 0xbc, 0x88, 0x7e, +0xc0, 0x76, 0xa7, 0xe2, 0x7b, 0x47, 0x90, 0xf2, 0xaa, 0x0a, 0x34, 0x1b, 0x91, 0x12, 0xd2, 0xd0, +0x82, 0x45, 0xf4, 0x57, 0xf1, 0xbd, 0x91, 0x5e, 0xab, 0x41, 0x4c, 0xdf, 0x91, 0x4c, 0xdd, 0x67, +0x04, 0xa0, 0x98, 0x23, 0x8c, 0x24, 0xbe, 0xd6, 0x80, 0xb3, 0x6d, 0x04, 0xa1, 0x77, 0x43, 0xa5, +0xee, 0xb7, 0xce, 0xb1, 0x48, 0x43, 0x94, 0x61, 0x15, 0x20, 0x9d, 0xce, 0xd0, 0x14, 0x95, 0x37, +0xc8, 0x64, 0xa3, 0x2d, 0x3d, 0xe3, 0xff, 0xb4, 0x55, 0x83, 0x84, 0x41, 0x50, 0x57, 0xbd, 0x5a, +0x0c, 0xe4, 0xda, 0x3b, 0x36, 0x4d, 0x21, 0xb5, 0x6f, 0x73, 0x2a, 0x8c, 0x78, 0x4f, 0x9b, 0x83, +0xda, 0x11, 0x3c, 0xf0, 0xc9, 0x7e, 0xa6, 0x48, 0x34, 0x53, 0x62, 0xd3, 0x0c, 0xff, 0xb1, 0x74, +0xd6, 0xea, 0xa5, 0xfc, 0x13, 0x1c, 0x05, 0xa8, 0xc0, 0xbc, 0x95, 0x9c, 0x8c, 0xf6, 0x8c, 0xc3, +0xf3, 0x69, 0xab, 0x93, 0x65, 0xc0, 0xb7, 0x7e, 0xb0, 0x16, 0x7c, 0xb5, 0x5f, 0x05, 0x28, 0xc9, +0x09, 0x4e, 0x2a, 0x32, 0x87, 0xb3, 0xab, 0xf8, 0x4c, 0xab, 0xeb, 0x3b, 0x6a, 0xa0, 0x1d, 0x7f, +0xef, 0xe5, 0x9b, 0xa4, 0xb7, 0xd7, 0xc2, 0x5e, 0x03, 0x0f, 0x99, 0xeb, 0xb1, 0xb1, 0xa6, 0x9d, +0x1c, 0x7c, 0x5c, 0x94, 0x8b, 0x6e, 0x11, 0x7a, 0xb3, 0x6d, 0x1e, 0x61, 0x64, 0xc3, 0x7d, 0x1c, +0xb3, 0x54, 0x65, 0x08, 0x3b, 0xda, 0x97, 0xb9, 0x75, 0xc1, 0x2b, 0x3e, 0xa8, 0x5c, 0x3c, 0x2d, +0x81, 0x5b, 0xbf, 0x5a, 0x13, 0x0e, 0xeb, 0x66, 0xc0, 0x0b, 0x8f, 0x04, 0x68, 0x68, 0x9b, 0xe3, +0x0d, 0x84, 0xe0, 0xcf, 0x83, 0xd7, 0x62, 0x48, 0xc1, 0x31, 0xa5, 0xd5, 0xbc, 0xe3, 0xa3, 0xa5, +0xb6, 0xd1, 0xfd, 0x81, 0x91, 0x4d, 0xbd, 0xc4, 0x62, 0x4f, 0xe3, 0xd5, 0x99, 0x14, 0xf1, 0xcd, +0xf4, 0x7d, 0x13, 0xda, 0x68, 0x0a, 0xca, 0xd6, 0x82, 0x0b, 0xf6, 0xea, 0xad, 0x78, 0xa4, 0xc8, +0x14, 0x7a, 0xec, 0x11, 0xd3, 0x16, 0x86, 0x9f, 0x17, 0x37, 0x6a, 0x06, 0x56, 0xaa, 0x1b, 0xd1, +0xaf, 0x85, 0x95, 0x21, 0x36, 0x69, 0xec, 0x1b, 0x56, 0x84, 0x01, 0x3f, 0x4d, 0x34, 0x3d, 0x2d, +0x38, 0x57, 0x2d, 0x7e, 0xd9, 0x7b, 0x2d, 0x81, 0x86, 0xd4, 0x7c, 0x83, 0x12, 0x1d, 0x9d, 0x27, +0x72, 0x1b, 0x5e, 0xf4, 0x15, 0xa5, 0xcd, 0xb7, 0x5f, 0xbb, 0x49, 0xa1, 0xd9, 0xdd, 0x8d, 0xad, +0xa9, 0x2c, 0x65, 0x18, 0x91, 0xfd, 0xd2, 0xd4, 0x09, 0x60, 0x0c, 0xfd, 0xa4, 0xe1, 0x25, 0x87, +0x32, 0x64, 0x7b, 0x99, 0xd7, 0x61, 0x2f, 0xd4, 0x73, 0xdd, 0x85, 0x26, 0x08, 0x92, 0xc0, 0xe1, +0x4f, 0x0c, 0x76, 0x5b, 0x26, 0x69, 0xdb, 0x78, 0x35, 0x65, 0xb9, 0x58, 0x1f, 0x9c, 0x0f, 0x18, +0x95, 0xfe, 0x40, 0xfc, 0xf7, 0x93, 0x71, 0x70, 0x8b, 0x73, 0xdc, 0xb0, 0x88, 0x72, 0x19, 0x26, +0x94, 0x26, 0xa7, 0xaa, 0x00, 0x72, 0x61, 0x53, 0xd2, 0x5d, 0x8f, 0x5e, 0x51, 0x88, 0x2d, 0xa4, +0x28, 0xd5, 0xaf, 0x2d, 0xd2, 0x84, 0x39, 0x75, 0x1e, 0xe7, 0xf0, 0x23, 0xc0, 0x4f, 0x8d, 0xdd, +0x5c, 0x90, 0xef, 0x6e, 0x53, 0xe0, 0x54, 0x67, 0xe1, 0x5b, 0x10, 0xf1, 0xf5, 0xf8, 0x64, 0x34, +0x94, 0xeb, 0x37, 0xf7, 0xe9, 0xaa, 0x6c, 0xa4, 0xd8, 0x74, 0x6d, 0xca, 0x8d, 0x1a, 0x31, 0x73, +0xca, 0xb4, 0xc7, 0x47, 0x34, 0x7f, 0xf8, 0x24, 0x9b, 0xfa, 0xc9, 0xcc, 0xa8, 0x61, 0xb4, 0x0e, +0x4d, 0x68, 0xc7, 0xa0, 0xcb, 0xea, 0xf0, 0xcc, 0x0a, 0x6c, 0xf2, 0x33, 0x42, 0x99, 0x6c, 0xd8, +0x74, 0x7f, 0x1e, 0x8a, 0xa3, 0x0a, 0x48, 0x4b, 0x7e, 0xbe, 0xdb, 0x7f, 0x56, 0x69, 0x43, 0xe8, +0xbf, 0x12, 0xc4, 0x7b, 0xc2, 0xd9, 0xfa, 0x5c, 0xeb, 0x45, 0xca, 0x07, 0x3d, 0xc0, 0xcd, 0x68, +0x8b, 0xd0, 0x79, 0xea, 0x0a, 0x78, 0x06, 0xdc, 0x81, 0xd7, 0x32, 0x18, 0xb3, 0x65, 0xbe, 0x47, +0xbb, 0xfa, 0x17, 0x09, 0xe9, 0x31, 0x95, 0x30, 0xef, 0x07, 0x44, 0xec, 0xd0, 0x98, 0x98, 0xc0, +0x6b, 0x71, 0x5b, 0x23, 0xb8, 0xb6, 0xd2, 0x21, 0xff, 0x51, 0xdd, 0xae, 0x48, 0x29, 0x75, 0x0c, +0xc3, 0x3d, 0x91, 0xfe, 0x9d, 0xa8, 0x5e, 0xb2, 0x34, 0xb2, 0xd3, 0x81, 0xf6, 0x27, 0x9c, 0xac, +0x6b, 0x20, 0x56, 0x86, 0xa5, 0x4f, 0x7a, 0xdb, 0xf9, 0xac, 0xa9, 0x8e, 0xe3, 0x73, 0x21, 0x99, +0x71, 0x2d, 0xaf, 0x27, 0x92, 0x0c, 0xc7, 0xd3, 0x85, 0xb3, 0x40, 0xda, 0x13, 0x4a, 0x04, 0x41, +0x54, 0xf8, 0xf2, 0x55, 0xb7, 0x80, 0xdd, 0x77, 0xba, 0x01, 0x7a, 0x31, 0xbd, 0x6b, 0xdc, 0x5c, +0x59, 0xf4, 0x2b, 0xca, 0x25, 0xbb, 0x50, 0xba, 0xfa, 0x42, 0x38, 0xd2, 0x28, 0x10, 0x8b, 0x7b, +0x96, 0x45, 0x30, 0xbb, 0x7f, 0xf4, 0x5a, 0xf7, 0x28, 0x6f, 0x47, 0xdc, 0xd2, 0x82, 0xf2, 0xf7, +0xdd, 0x20, 0xb5, 0x0c, 0x7e, 0x53, 0x85, 0xa7, 0xfc, 0x3b, 0x1a, 0xc0, 0x07, 0x7b, 0xa1, 0x43, +0x05, 0x18, 0x19, 0xd3, 0xfc, 0x41, 0xc2, 0xce, 0xd9, 0x5b, 0x4b, 0x63, 0xe2, 0x8f, 0x86, 0x3a, +0xd1, 0xd0, 0x1d, 0x74, 0x2e, 0xbc, 0xd3, 0xce, 0x08, 0x0c, 0x10, 0x7a, 0x42, 0x60, 0xc5, 0x3a, +0xa6, 0xd8, 0xb0, 0x52, 0xcf, 0x53, 0x28, 0x70, 0x45, 0xb7, 0x72, 0x7d, 0x77, 0x66, 0x54, 0x3d, +0x38, 0x26, 0xcf, 0xd5, 0xbf, 0xe4, 0x80, 0x10, 0xba, 0x2b, 0xe8, 0xdc, 0xc3, 0xfe, 0x28, 0xa3, +0x52, 0x58, 0x70, 0x4a, 0xde, 0x84, 0x33, 0x5e, 0x93, 0x04, 0xa4, 0x7c, 0xe7, 0xea, 0x8e, 0xba, +0xeb, 0x8a, 0x19, 0x26, 0x6a, 0x7f, 0x7c, 0x4a, 0x5b, 0xb4, 0x0d, 0xfc, 0xc8, 0x11, 0x1b, 0x41, +0x68, 0x5d, 0x2a, 0x25, 0x04, 0x4f, 0xc8, 0xf4, 0x65, 0xfc, 0xb9, 0x58, 0xeb, 0xb4, 0x67, 0x50, +0x24, 0xf5, 0x43, 0xf6, 0x91, 0x4a, 0xb0, 0x0f, 0x32, 0xe0, 0x07, 0x75, 0x69, 0x1b, 0x3c, 0xeb, +0xb2, 0x65, 0x26, 0x6f, 0xb8, 0x79, 0xe0, 0x78, 0x8c, 0xdc, 0x39, 0x24, 0x48, 0x76, 0x11, 0xd4, +0x3a, 0xc5, 0xd2, 0x2b, 0xaa, 0x55, 0xfb, 0x92, 0x12, 0x2d, 0x88, 0x05, 0xd1, 0xb1, 0x31, 0x36, +0x1f, 0xc2, 0x44, 0x1c, 0xab, 0x2e, 0xcd, 0x1c, 0x72, 0x86, 0xf6, 0x83, 0x87, 0x2e, 0x8b, 0xdb, +0xaa, 0x16, 0x0e, 0x1b, 0xe6, 0x5c, 0x4d, 0x2f, 0x82, 0xbd, 0x49, 0x11, 0x60, 0x22, 0x0f, 0xde, +0x3b, 0x2b, 0x20, 0x1d, 0x56, 0xb7, 0x21, 0xae, 0x0b, 0x26, 0x4f, 0xde, 0x3d, 0xa6, 0x3f, 0x61, +0x81, 0xe2, 0x76, 0x60, 0x08, 0xc5, 0x4b, 0x18, 0x0b, 0xd1, 0xf5, 0xff, 0x8d, 0x1a, 0x96, 0x76, +0x51, 0x15, 0x05, 0x4d, 0x8c, 0x6b, 0x12, 0x90, 0x47, 0xd4, 0xa4, 0x38, 0xb9, 0x48, 0xe4, 0x4c, +0x05, 0x69, 0x6a, 0x8b, 0x9d, 0x7c, 0xa1, 0xbc, 0x77, 0xeb, 0x86, 0x93, 0x0a, 0x15, 0x84, 0xba, +0x8f, 0xf5, 0x7c, 0x44, 0x75, 0x31, 0x79, 0x16, 0xc1, 0x81, 0x1a, 0xb6, 0xe6, 0x6c, 0x3d, 0xb8, +0x15, 0x46, 0xf5, 0xbe, 0x46, 0x04, 0xa6, 0xec, 0xec, 0xd1, 0x74, 0x8b, 0x87, 0x2b, 0xdb, 0xd0, +0x9f, 0xb3, 0x99, 0x9d, 0x87, 0x8c, 0xc6, 0xaa, 0xd4, 0x64, 0x45, 0xbd, 0xe8, 0xed, 0xa3, 0xc1, +0x2a, 0x41, 0x1e, 0x26, 0xaf, 0x86, 0x16, 0xed, 0x80, 0x08, 0xca, 0x64, 0x21, 0x3a, 0xce, 0x21, +0x4c, 0x41, 0xb9, 0x13, 0x2d, 0xf7, 0x1b, 0xdf, 0x2b, 0x33, 0x69, 0xe7, 0x5c, 0x8c, 0x7b, 0xfb, +0xe3, 0x41, 0xe9, 0xce, 0xd7, 0xff, 0x0e, 0x54, 0xfe, 0xb0, 0x71, 0x78, 0xdc, 0xde, 0x7e, 0xdd, +0x1f, 0x1c, 0x4a, 0x8f, 0x3e, 0x16, 0xfd, 0x91, 0x82, 0x94, 0xd4, 0xc2, 0xf7, 0xb2, 0x77, 0x89, +0x16, 0x2c, 0xba, 0xb6, 0xbd, 0xed, 0x95, 0x43, 0x05, 0x9b, 0xf2, 0xc4, 0xbe, 0x46, 0x43, 0x90, +0x1d, 0xd8, 0x24, 0x02, 0xd2, 0xea, 0xf4, 0x08, 0xd9, 0xf7, 0x84, 0x0e, 0xc6, 0xe7, 0x44, 0xdb, +0xb8, 0xac, 0x0a, 0x53, 0x39, 0x61, 0x43, 0xdc, 0x22, 0x28, 0x8f, 0x22, 0x2f, 0x73, 0xbf, 0x59, +0x2d, 0x3c, 0x8c, 0x0b, 0xcc, 0x2a, 0x67, 0xe0, 0x5b, 0x5c, 0x65, 0x5e, 0x6d, 0x98, 0x99, 0xaa, +0x3b, 0x89, 0x12, 0xe2, 0x99, 0xf6, 0x15, 0xa7, 0xd2, 0x6a, 0x79, 0xb4, 0xf6, 0x0b, 0xf5, 0x0d, +0x2d, 0x4c, 0xcb, 0x1b, 0x35, 0x93, 0x61, 0x32, 0xa1, 0x8a, 0xa8, 0x27, 0xe8, 0x95, 0x5a, 0x56, +0x59, 0x04, 0xfe, 0xce, 0xc2, 0xd8, 0x92, 0x97, 0xb2, 0x54, 0x63, 0xd0, 0x3b, 0xde, 0x10, 0x34, +0x32, 0x16, 0x05, 0x51, 0x1d, 0xfc, 0x96, 0x8e, 0xf1, 0xf6, 0x4b, 0xd7, 0x48, 0x22, 0xce, 0xca, +0x1c, 0x6b, 0xab, 0x1f, 0x59, 0xa2, 0x74, 0xd6, 0xcd, 0x15, 0x07, 0xab, 0xa2, 0xd5, 0x22, 0x81, +0xec, 0x20, 0x14, 0x36, 0xac, 0xe4, 0x25, 0x7d, 0xe6, 0x09, 0x00, 0x2c, 0x92, 0x4d, 0x4e, 0xbf, +0xbf, 0xa1, 0xd4, 0xbe, 0x6b, 0xd4, 0x1f, 0x95, 0x9b, 0xf3, 0xda, 0x99, 0xad, 0xa4, 0x6c, 0x73, +0x55, 0xd1, 0x9d, 0x4b, 0x16, 0xd4, 0x06, 0xec, 0x46, 0x3d, 0xb7, 0xe7, 0xce, 0xd0, 0x1d, 0x94, +0x65, 0xde, 0x61, 0xb3, 0xc1, 0x10, 0x65, 0xe5, 0x68, 0x9b, 0xb0, 0xb4, 0x43, 0x0b, 0x92, 0xaf, +0xb7, 0x40, 0xa2, 0xe5, 0x06, 0x3d, 0x72, 0x00, 0xc5, 0x39, 0xab, 0x35, 0x29, 0x22, 0x4c, 0xa5, +0xa5, 0x3f, 0x22, 0x90, 0x53, 0xd2, 0x36, 0x63, 0x1e, 0xd3, 0x33, 0xa5, 0xbb, 0x3d, 0xa3, 0x0c, +0x14, 0x9c, 0x2e, 0x6d, 0x9a, 0x7a, 0xf7, 0xf1, 0x56, 0x66, 0xe5, 0x8d, 0x53, 0x83, 0x34, 0x3f, +0xa9, 0x83, 0x84, 0x68, 0x90, 0xc9, 0x51, 0xc2, 0xd4, 0x8e, 0x6c, 0xc7, 0x6d, 0xa7, 0x19, 0x61, +0xa7, 0x2e, 0x36, 0xbc, 0xd2, 0x0f, 0x17, 0x49, 0xd4, 0x6b, 0x36, 0x63, 0xfb, 0x1d, 0xf4, 0xb0, +0x6b, 0xcf, 0x34, 0x5f, 0xd2, 0x77, 0xae, 0x12, 0xaf, 0xb3, 0xdf, 0x52, 0xf7, 0xc2, 0xc8, 0xf2, +0x63, 0x61, 0xb6, 0x3e, 0x39, 0xf2, 0xa7, 0x1a, 0x89, 0x9d, 0x0e, 0x8f, 0xaf, 0xe1, 0x01, 0x24, +0xa6, 0x3a, 0xd5, 0x9a, 0x62, 0x67, 0xa3, 0x66, 0xee, 0xbc, 0xc5, 0x94, 0x4b, 0xc3, 0x15, 0xa1, +0x7e, 0x07, 0x07, 0x2b, 0xb7, 0x43, 0x2a, 0xb4, 0xb8, 0x25, 0x88, 0x86, 0x23, 0xab, 0xdf, 0x05, +0xbe, 0x46, 0x56, 0xd7, 0xda, 0xd6, 0x75, 0x53, 0xd9, 0xc8, 0x26, 0x8f, 0x39, 0x67, 0xed, 0x21, +0x53, 0x1c, 0x9c, 0x89, 0x46, 0xd3, 0xfe, 0x54, 0xe6, 0x1d, 0x02, 0xb9, 0x25, 0x82, 0x66, 0xe6, +0xf9, 0x45, 0xd9, 0x3f, 0xa5, 0x71, 0xc1, 0x46, 0x66, 0x7a, 0x27, 0x8a, 0x82, 0xc9, 0x21, 0xe9, +0x17, 0xab, 0x6c, 0xef, 0x45, 0xe5, 0x88, 0x93, 0x87, 0x80, 0xb3, 0x85, 0x25, 0x96, 0x19, 0x41, +0xab, 0xd6, 0xba, 0x92, 0x76, 0x21, 0x8a, 0x58, 0xbd, 0xe2, 0x4b, 0xec, 0x45, 0x59, 0x2c, 0x13, +0x1a, 0xb5, 0x13, 0x25, 0x44, 0xe7, 0x71, 0x26, 0x0a, 0x34, 0x33, 0xb9, 0x57, 0x15, 0xa4, 0x90, +0x60, 0x11, 0x05, 0x8e, 0xc8, 0x8e, 0x74, 0x52, 0x4b, 0x31, 0x71, 0xeb, 0x66, 0x7e, 0xee, 0xb1, +0x0a, 0x21, 0x52, 0xc0, 0x1a, 0xe9, 0xa1, 0x5a, 0xe3, 0x3a, 0x24, 0xfb, 0xf3, 0x1e, 0xd6, 0x83, +0x1d, 0xfb, 0x81, 0xa8, 0x91, 0x60, 0x9e, 0xbc, 0x59, 0x20, 0xc9, 0x9e, 0x71, 0x19, 0x83, 0x2b, +0x6a, 0x48, 0x4e, 0x6b, 0x46, 0x82, 0x89, 0xda, 0x60, 0xff, 0x1a, 0x46, 0x94, 0x55, 0xda, 0xe5, +0x99, 0xfa, 0x84, 0xd7, 0x3b, 0xb9, 0xa5, 0x34, 0x87, 0x86, 0x5e, 0x6d, 0x75, 0x9a, 0xe7, 0x09, +0xb8, 0xe6, 0x71, 0x15, 0x10, 0x56, 0xd7, 0xc1, 0xc8, 0xb2, 0x62, 0xbc, 0xec, 0xe0, 0x94, 0xa0, +0xcd, 0xb4, 0x04, 0xa9, 0xc3, 0x51, 0xee, 0xf8, 0x2e, 0x42, 0x9a, 0xaa, 0x34, 0xd3, 0xb9, 0xb0, +0x36, 0xf9, 0x47, 0xc1, 0x07, 0x49, 0xde, 0xb8, 0x32, 0x8a, 0x87, 0x68, 0x56, 0x9a, 0x35, 0x79, +0xd1, 0xac, 0x49, 0x38, 0xc6, 0xfe, 0xfd, 0xdf, 0x6f, 0x3c, 0xda, 0x48, 0xbd, 0x23, 0xfd, 0x85, +0xf0, 0x96, 0xee, 0x1c, 0x27, 0x18, 0x86, 0xa6, 0xf0, 0x7b, 0xd8, 0x3c, 0xc7, 0x22, 0x3e, 0x2f, +0xac, 0xb1, 0x37, 0xbd, 0x84, 0x4b, 0xe3, 0x92, 0x82, 0xd0, 0x25, 0x14, 0x22, 0x65, 0xed, 0xeb, +0xef, 0xb9, 0xb6, 0xe4, 0x95, 0x18, 0x0d, 0x2b, 0x8d, 0x4f, 0xaf, 0xc0, 0xa0, 0x05, 0x8b, 0x35, +0x5b, 0x94, 0xb2, 0x68, 0x26, 0x4f, 0x4a, 0x9e, 0x85, 0x0e, 0x46, 0xe0, 0x4f, 0x60, 0x66, 0x01, +0xa4, 0x39, 0xe8, 0x8b, 0x2a, 0x50, 0xf5, 0x18, 0x70, 0xe2, 0xfc, 0xd6, 0xbe, 0xd3, 0x46, 0x4b + +ciphertext = +0x0F, 0x9B, 0xF5, 0x9F, 0xE2, 0xB3, 0xD9, 0x12, 0x27, 0x51, 0x18, 0xD7, 0x4E, 0x1F, 0x40, 0x4C, +0xC1, 0xDF, 0x66, 0xAB, 0x1A, 0xFA, 0xF8, 0xEE, 0xA4, 0x40, 0x63, 0x72, 0x58, 0xAE, 0x7E, 0x98, +0x76, 0x63, 0x56, 0x1F, 0x71, 0xDB, 0x80, 0x07, 0xFE, 0x34, 0x23, 0x43, 0x1E, 0x3D, 0xDE, 0x7E, +0xC0, 0x72, 0xEF, 0xAD, 0xF4, 0x30, 0xDF, 0x4E, 0x3B, 0x9F, 0xCA, 0x90, 0xC3, 0x95, 0x8A, 0x66, +0x5C, 0xD6, 0xCA, 0xBD, 0x3C, 0xC9, 0xD2, 0xFE, 0x30, 0x02, 0xA9, 0x8E, 0xAA, 0x80, 0xF6, 0xFF, +0xCD, 0x40, 0xBC, 0x99, 0xD2, 0x25, 0x7F, 0xBF, 0xC5, 0xF3, 0x50, 0x31, 0x69, 0xE1, 0xC8, 0x64, +0xC5, 0x5F, 0x30, 0x30, 0xD9, 0xD7, 0xF9, 0xF0, 0xD3, 0x77, 0xE0, 0xD0, 0x11, 0xB8, 0xC9, 0x54, +0xD9, 0x9C, 0x10, 0x74, 0xCA, 0x4A, 0xFE, 0xD2, 0x16, 0xA5, 0xD8, 0x2D, 0xC0, 0xDA, 0x39, 0x24, +0xAF, 0x5E, 0xF2, 0xEB, 0xC7, 0x9D, 0xBC, 0xEF, 0x94, 0xA0, 0x49, 0x56, 0x39, 0xCE, 0x8A, 0x38, +0x3B, 0x70, 0xC7, 0xB2, 0xE0, 0xD4, 0x43, 0xD7, 0xAC, 0xB4, 0xB3, 0xDB, 0xA2, 0x2B, 0x75, 0xE2, +0x0E, 0x38, 0x2B, 0xE6, 0x42, 0x1A, 0x11, 0x08, 0x79, 0x9A, 0x32, 0xD2, 0x41, 0xCC, 0x28, 0xC3, +0x4B, 0x3E, 0xD4, 0xB0, 0x10, 0x89, 0x7B, 0x0D, 0xB7, 0x95, 0xBE, 0x22, 0x01, 0xD0, 0x86, 0xA8, +0xC6, 0xD0, 0xDD, 0xDF, 0x18, 0x2C, 0x1B, 0x49, 0xE3, 0x2B, 0x84, 0x53, 0x54, 0x14, 0xE6, 0x04, +0xE1, 0xD6, 0x98, 0x91, 0x17, 0xE0, 0xD9, 0x39, 0xAF, 0xF9, 0x71, 0x35, 0x90, 0xCE, 0x4B, 0xD2, +0x45, 0xB2, 0x4B, 0x68, 0x26, 0xBB, 0x8C, 0xBD, 0xA3, 0xF7, 0x60, 0xD4, 0x38, 0xAA, 0xDF, 0x5B, +0x3B, 0x53, 0xF6, 0xA4, 0x45, 0x49, 0x4A, 0xEF, 0x6F, 0x04, 0x00, 0xFF, 0xE3, 0x3F, 0x7C, 0x7D, +0xDC, 0xB0, 0x62, 0x9C, 0x6A, 0x99, 0x07, 0x85, 0xB3, 0x13, 0x2B, 0x40, 0x06, 0xAF, 0xE3, 0xA0, +0x17, 0x97, 0x0D, 0x4E, 0xD7, 0xB0, 0x6B, 0xF8, 0x3C, 0x91, 0x0A, 0xF3, 0x17, 0x51, 0x30, 0xC8, +0x58, 0x20, 0x20, 0xE2, 0xA3, 0xE6, 0x3B, 0x2F, 0x77, 0x7C, 0x52, 0x31, 0x4F, 0x4C, 0xA8, 0xD8, +0x84, 0xB1, 0xE9, 0xB4, 0x86, 0xD8, 0x93, 0xBF, 0x2D, 0x6A, 0xDA, 0x5D, 0x39, 0x62, 0x5B, 0x52, +0xFB, 0xBA, 0x9F, 0x83, 0x82, 0x3C, 0x40, 0x57, 0x02, 0x92, 0x6A, 0x97, 0x06, 0x39, 0x17, 0x0B, +0xA7, 0xF5, 0x6A, 0x1A, 0x8E, 0x64, 0x74, 0x90, 0x33, 0xA6, 0xA3, 0x1E, 0x30, 0x1E, 0x67, 0x49, +0x5A, 0x76, 0x43, 0x97, 0x71, 0xE0, 0x4E, 0xCC, 0x5A, 0xFD, 0x44, 0xFD, 0x5C, 0x41, 0x3A, 0x09, +0x8E, 0x4E, 0xD2, 0xF0, 0x9A, 0x52, 0x39, 0x5B, 0x0E, 0xC4, 0xF2, 0xFE, 0xB4, 0x66, 0x6C, 0x60, +0x47, 0x96, 0x80, 0x91, 0xBE, 0x6C, 0xA8, 0x33, 0x66, 0x4D, 0x08, 0x70, 0x27, 0x0C, 0x33, 0x50, +0x8F, 0xEF, 0x05, 0xC9, 0x93, 0x21, 0x8D, 0xA4, 0x94, 0xF3, 0xBC, 0x4D, 0x96, 0x9A, 0x51, 0x29, +0xDA, 0x8E, 0x32, 0xB4, 0xB3, 0xD8, 0x75, 0x20, 0x37, 0x9F, 0x33, 0xE2, 0xF9, 0xEB, 0xFA, 0xF2, +0x6E, 0x3F, 0x71, 0x0C, 0x29, 0x8D, 0xFE, 0xE1, 0xF9, 0xC6, 0x49, 0xB6, 0x6E, 0x53, 0xBC, 0x24, +0x1D, 0x0B, 0x24, 0x75, 0x54, 0x51, 0x0B, 0xEB, 0xDD, 0x67, 0x40, 0x61, 0xCA, 0x3F, 0xD2, 0x85, +0x71, 0x16, 0xFC, 0x77, 0x9C, 0x56, 0xE5, 0xCA, 0x43, 0xC6, 0xC3, 0x2A, 0x47, 0xA8, 0x98, 0x40, +0x02, 0x1D, 0x64, 0x47, 0x85, 0x99, 0x2F, 0x8F, 0x2D, 0xC2, 0x29, 0x7B, 0x8D, 0x64, 0xD9, 0x8F, +0xF4, 0x91, 0x6F, 0x2A, 0xB0, 0x5C, 0xDC, 0xB0, 0xBE, 0xDE, 0x34, 0xA8, 0x99, 0x40, 0x23, 0x9F, +0x8A, 0xF5, 0x0C, 0x32, 0xDE, 0x53, 0xA5, 0x55, 0xFE, 0x4C, 0xF8, 0x87, 0x83, 0xB6, 0xA1, 0x31, +0x2C, 0xB4, 0xE9, 0x78, 0xB8, 0x45, 0xAA, 0x33, 0x6E, 0x8A, 0xBE, 0xDB, 0x76, 0x35, 0xDD, 0xDF, +0xA1, 0x98, 0x21, 0x2B, 0x42, 0xF3, 0xA4, 0x3E, 0x2C, 0x38, 0xA9, 0xB1, 0x07, 0x38, 0xA1, 0x1D, +0xA5, 0x85, 0x61, 0x87, 0xF1, 0xA1, 0x9D, 0x3D, 0x2C, 0xA6, 0x2F, 0x26, 0xFD, 0xE8, 0x46, 0x0D, +0xBD, 0xDA, 0x44, 0xC4, 0xB5, 0xFF, 0x6F, 0xDB, 0xF7, 0xF4, 0xDB, 0x0A, 0x80, 0x7C, 0x81, 0x27, +0xF4, 0x27, 0x41, 0x15, 0x9F, 0xEC, 0xA5, 0xAA, 0x79, 0x30, 0x9B, 0x0D, 0x84, 0xAC, 0x4D, 0x50, +0x68, 0x56, 0x55, 0x32, 0xF9, 0x28, 0x06, 0xC3, 0x96, 0xD6, 0x57, 0x61, 0x04, 0xCF, 0xD8, 0xB9, +0x36, 0x0D, 0x33, 0x11, 0xEE, 0x8A, 0x88, 0x5A, 0x11, 0x6C, 0x11, 0x71, 0x41, 0xFC, 0xD5, 0xF1, +0xB7, 0xC2, 0x94, 0x98, 0x6F, 0xAB, 0x12, 0x5F, 0x34, 0x46, 0xDD, 0xBC, 0x65, 0x5C, 0x76, 0x3A, +0x81, 0x42, 0xF8, 0x6D, 0xC2, 0x08, 0x93, 0x58, 0x30, 0x1B, 0x82, 0x28, 0xD7, 0xFB, 0x90, 0x61, +0x24, 0x38, 0x12, 0xDC, 0xFB, 0x88, 0xFB, 0xC8, 0xB9, 0xB8, 0x23, 0xA5, 0xEB, 0x85, 0xB1, 0x92, +0x55, 0x34, 0xA7, 0x8E, 0x2C, 0x3D, 0xD7, 0x07, 0xF6, 0x2C, 0xF5, 0x5A, 0x72, 0x38, 0x25, 0x5A, +0x3F, 0x1F, 0xF5, 0x7B, 0x02, 0xFC, 0x70, 0xB1, 0x31, 0x01, 0xCA, 0xD6, 0x18, 0x7C, 0xF4, 0x0A, +0x3A, 0xAE, 0x7B, 0x89, 0x0B, 0xE8, 0x5E, 0x7D, 0x64, 0x7A, 0x10, 0xE0, 0x8C, 0x8C, 0x02, 0xF7, +0x2F, 0x52, 0x06, 0xE7, 0xD7, 0xFA, 0x56, 0xBB, 0xC4, 0x5A, 0x82, 0x14, 0x13, 0x7D, 0x41, 0xD0, +0xCE, 0x57, 0xE6, 0x41, 0x3A, 0x08, 0xAD, 0x7C, 0x20, 0xDC, 0x15, 0xA8, 0xBA, 0xE6, 0xB8, 0x58, +0xA6, 0xF4, 0x82, 0x70, 0xDE, 0x09, 0xE2, 0x8B, 0xFC, 0xAA, 0x7D, 0xD9, 0xFD, 0xCE, 0xBC, 0xAC, +0xB8, 0xE8, 0xB2, 0x46, 0xC5, 0xC4, 0xF9, 0xFC, 0xE5, 0xFF, 0xF5, 0xCA, 0xB3, 0x51, 0x3C, 0xB8, +0x25, 0xD7, 0x83, 0x3C, 0x91, 0xCD, 0xFB, 0x86, 0x94, 0x5E, 0x36, 0xCF, 0xDE, 0x8E, 0x37, 0x4F, +0x9D, 0x54, 0xBB, 0x8D, 0x52, 0xE6, 0x86, 0x5C, 0x8B, 0x72, 0x23, 0xAF, 0x93, 0xF5, 0xFF, 0xB5, +0xC5, 0xFE, 0xC1, 0x31, 0xCE, 0x72, 0xD7, 0x3A, 0x2E, 0x25, 0xBF, 0xC1, 0xEE, 0xD5, 0x28, 0xDB, +0xB7, 0xD2, 0x4E, 0xC6, 0x2B, 0xE3, 0x6F, 0x87, 0x2D, 0xA5, 0xFB, 0xDB, 0x03, 0xB0, 0x13, 0x58, +0x3F, 0x3B, 0xBD, 0x2E, 0x0B, 0x78, 0xC2, 0xDC, 0xF8, 0x30, 0x2F, 0x08, 0xA6, 0x6A, 0x00, 0x57, +0x82, 0x79, 0x06, 0x11, 0xC2, 0x20, 0x49, 0xD1, 0xDD, 0x53, 0xF6, 0xA8, 0xB1, 0x0A, 0x1C, 0x97, +0x11, 0x5C, 0xA4, 0x98, 0xBC, 0xE7, 0x10, 0x27, 0x73, 0x54, 0x3C, 0x91, 0x98, 0x58, 0x2F, 0xA8, +0x96, 0xB2, 0xE8, 0x54, 0xC0, 0x6B, 0x4B, 0x69, 0x0E, 0x92, 0xFB, 0xF0, 0x5E, 0xDC, 0x45, 0xAE, +0x6B, 0x12, 0x29, 0xBE, 0x59, 0xA2, 0x0B, 0xD6, 0xD2, 0x09, 0xE0, 0xBA, 0xE2, 0x1D, 0xD9, 0x8A, +0x6D, 0x70, 0xD0, 0x62, 0x0C, 0x9D, 0xC4, 0x38, 0x3A, 0x11, 0xBA, 0xF8, 0x8B, 0x0A, 0xDF, 0xDF, +0xD1, 0xBF, 0x87, 0x98, 0xC8, 0xFF, 0x42, 0x85, 0x13, 0xE7, 0x6C, 0xDE, 0xFE, 0x45, 0xFE, 0xF8, +0x67, 0x28, 0x56, 0xF7, 0x3C, 0x12, 0x4E, 0x88, 0x05, 0x8B, 0x56, 0x53, 0x2E, 0xE8, 0x6F, 0x3B, +0xEE, 0x40, 0x4D, 0x3C, 0x1B, 0x0C, 0xE7, 0xCA, 0xD1, 0x3A, 0xA3, 0x2A, 0xDB, 0xC0, 0xDB, 0x9D, +0xA4, 0xC1, 0xDA, 0x50, 0x81, 0x59, 0x52, 0x87, 0x33, 0x10, 0xE6, 0x58, 0xC3, 0x3E, 0x08, 0x69, +0xFF, 0x3B, 0x4C, 0x82, 0xBA, 0xC3, 0x81, 0x82, 0xB5, 0xF2, 0x9F, 0x0A, 0x20, 0xCF, 0x70, 0xFB, +0x81, 0x63, 0xB7, 0x04, 0x95, 0xFE, 0x08, 0x33, 0x1A, 0xED, 0xB0, 0x0A, 0x86, 0xA6, 0x42, 0x8F, +0xEB, 0x48, 0xD8, 0xF4, 0xFE, 0x50, 0xA2, 0xC6, 0x5E, 0xD8, 0x33, 0xAE, 0x44, 0x83, 0xE6, 0x7C, +0x88, 0xA3, 0xEC, 0x74, 0xA4, 0x26, 0x58, 0xE6, 0x3C, 0xD4, 0x04, 0xAC, 0x71, 0xBD, 0xBE, 0xCC, +0xC6, 0xE8, 0x55, 0xDE, 0x98, 0x7F, 0x6B, 0x4F, 0x62, 0xF6, 0x4E, 0x1C, 0x95, 0x44, 0xBE, 0xD7, +0x77, 0x9C, 0xD2, 0x13, 0xD7, 0xF5, 0x2C, 0x8D, 0xC1, 0xC7, 0xCA, 0xF8, 0x68, 0x91, 0x06, 0x24, +0x01, 0x5D, 0xD0, 0x9E, 0xC9, 0x24, 0x12, 0x83, 0xA1, 0xF4, 0x05, 0xE0, 0xB2, 0xA9, 0xAD, 0xFC, +0xC3, 0x64, 0x08, 0x17, 0x58, 0xAB, 0xB7, 0x39, 0xA4, 0xB6, 0x77, 0x26, 0x37, 0x13, 0x24, 0x20, +0xEB, 0xEE, 0x61, 0x28, 0x0E, 0x33, 0x7C, 0x4D, 0xD5, 0xEA, 0x42, 0x39, 0xB7, 0x20, 0xCB, 0x97, +0x44, 0xBE, 0x6D, 0x67, 0x28, 0xF9, 0x99, 0xE1, 0xE7, 0x3A, 0x5D, 0x7D, 0xAA, 0xBB, 0x1D, 0xEC, +0xA4, 0xF6, 0xAA, 0xED, 0xC5, 0x1F, 0xAD, 0xDF, 0x0E, 0xD3, 0x94, 0xA9, 0xD9, 0xFF, 0x29, 0x56, +0x3C, 0x43, 0x5E, 0xC2, 0x3B, 0xB3, 0x2E, 0x4A, 0xB7, 0x9F, 0xF4, 0xA0, 0xF1, 0xD6, 0x93, 0xFF, +0x4A, 0xC8, 0xF2, 0xDE, 0x1A, 0x96, 0xB1, 0x83, 0x37, 0x45, 0x0C, 0x79, 0x40, 0x51, 0xD1, 0x08, +0xF2, 0xDC, 0xFA, 0xBF, 0xB4, 0x15, 0x9A, 0x60, 0x37, 0x81, 0x5F, 0x76, 0xB2, 0x87, 0x82, 0x2D, +0xA8, 0x0A, 0x70, 0x1B, 0xA2, 0xD6, 0x7E, 0x65, 0xAC, 0x02, 0x36, 0xDA, 0x0F, 0xB4, 0xD5, 0x6E, +0xEE, 0x60, 0x0B, 0xC8, 0xD6, 0x1B, 0x9C, 0xF1, 0xCF, 0x66, 0xCA, 0x1D, 0x9D, 0xE5, 0xC7, 0x0B, +0x1A, 0xE1, 0x9E, 0x68, 0xFF, 0xCE, 0xCE, 0x1B, 0x0A, 0x1E, 0x8D, 0x25, 0x37, 0x40, 0x05, 0x20, +0xE4, 0xA1, 0x4F, 0x4C, 0x33, 0x29, 0xB3, 0x7B, 0x47, 0xFA, 0x7E, 0xEF, 0x87, 0x89, 0x13, 0x10, +0x1C, 0x0F, 0x15, 0x03, 0xFD, 0xD8, 0x81, 0xD7, 0xB9, 0xA7, 0xBA, 0xD0, 0x6F, 0x6F, 0x76, 0xF6, +0xD1, 0xF3, 0xCE, 0x57, 0x21, 0x02, 0x12, 0xAA, 0x5E, 0x61, 0xD4, 0xCA, 0xF2, 0x84, 0x80, 0xF0, +0x90, 0x37, 0x0B, 0xE5, 0xC9, 0xE4, 0xAD, 0xD5, 0x8C, 0x47, 0x6D, 0xFA, 0xE1, 0xE5, 0xC6, 0x3A, +0xE4, 0x0A, 0x82, 0xCE, 0x05, 0xD6, 0x46, 0x39, 0xAE, 0xE9, 0xE4, 0x6A, 0xD7, 0xE5, 0x9E, 0x85, +0x4A, 0x31, 0xFA, 0xA0, 0x86, 0x01, 0x4B, 0xC8, 0x77, 0xC2, 0x10, 0xED, 0x70, 0xD5, 0x7D, 0x0F, +0xF4, 0xDC, 0x97, 0x93, 0xB2, 0x7D, 0x45, 0x7A, 0x24, 0x37, 0x2D, 0x39, 0xAC, 0xB6, 0x53, 0x6B, +0xA5, 0x9B, 0xDC, 0xC1, 0xAC, 0x95, 0x01, 0x3E, 0x52, 0x53, 0x11, 0x0A, 0xB7, 0x1E, 0x1F, 0xCD, +0x29, 0xC0, 0xE9, 0x0A, 0xDF, 0xE5, 0xF5, 0x54, 0xF7, 0xF7, 0x23, 0x7C, 0xC6, 0xB3, 0x18, 0xEA, +0x0E, 0x08, 0xE3, 0x76, 0x8D, 0xDC, 0x01, 0x45, 0xB7, 0x45, 0xAF, 0x5C, 0x88, 0xEA, 0x74, 0xFE, +0xE0, 0xC5, 0xA3, 0x73, 0x2D, 0x2E, 0x47, 0xF1, 0x2E, 0xC0, 0x01, 0x97, 0xE5, 0x64, 0x84, 0x59, +0xC6, 0x83, 0xF5, 0xFC, 0x0D, 0x2D, 0x70, 0x46, 0x0E, 0x17, 0xE1, 0xC9, 0xE2, 0xBF, 0xF2, 0xF4, +0x7B, 0x5C, 0xF1, 0xBE, 0xC3, 0xA7, 0x81, 0x96, 0xA1, 0x24, 0x67, 0x85, 0xF5, 0x6C, 0xD4, 0xA3, +0x2A, 0xBE, 0xF5, 0x05, 0x2A, 0xBB, 0xF2, 0x18, 0xAF, 0xDC, 0x21, 0x8A, 0x76, 0x2E, 0xAC, 0x31, +0xB7, 0x56, 0x55, 0xFD, 0x09, 0x48, 0x3A, 0xA6, 0x66, 0x1C, 0x2F, 0x92, 0x2C, 0x07, 0xDF, 0x09, +0x3A, 0xD1, 0xEA, 0x7A, 0xFA, 0x87, 0xE3, 0xF6, 0x5D, 0x03, 0x45, 0xC9, 0x3F, 0x97, 0x60, 0x4F, +0x9C, 0x6C, 0xCC, 0x55, 0x1F, 0xE1, 0xBD, 0x5E, 0xE5, 0x5D, 0x76, 0x0C, 0x00, 0xE0, 0xDA, 0xB9, +0x60, 0xFC, 0xF4, 0xC5, 0xFC, 0x14, 0x3E, 0x2C, 0xF2, 0x18, 0xEC, 0x7D, 0x2B, 0x56, 0xFC, 0x6D, +0xCE, 0x3D, 0x94, 0x5E, 0xA1, 0x00, 0xDC, 0x00, 0xA6, 0x73, 0xCE, 0xDE, 0x57, 0x87, 0xB3, 0x90, +0xE5, 0x16, 0x82, 0x24, 0x62, 0xAD, 0x27, 0x8B, 0xFF, 0xDD, 0xD6, 0x7C, 0xFB, 0x45, 0x8A, 0x09, +0xB0, 0x80, 0x15, 0x86, 0x90, 0xB9, 0xE9, 0x0F, 0x59, 0x86, 0x9E, 0xBA, 0xA2, 0x6A, 0xBA, 0x33, +0xDD, 0xE5, 0xE8, 0x08, 0x6F, 0x3F, 0xF4, 0x2E, 0xF7, 0x25, 0x28, 0x23, 0x60, 0x51, 0x88, 0x50, +0x66, 0x31, 0xEA, 0xEE, 0x83, 0x98, 0x62, 0x04, 0xBD, 0x48, 0x17, 0x34, 0x15, 0x2E, 0x9F, 0x33, +0x8C, 0x96, 0x83, 0x71, 0x1C, 0x0F, 0x72, 0xEB, 0x5F, 0x1C, 0x1A, 0x79, 0x7C, 0x22, 0x4C, 0x84, +0xC6, 0xA6, 0xD6, 0xDD, 0x29, 0x67, 0x78, 0xC5, 0xEC, 0x37, 0x3E, 0x00, 0xCF, 0xEF, 0x7E, 0x4B, +0x6D, 0xB2, 0xBD, 0xD3, 0x70, 0x41, 0x2A, 0x89, 0x00, 0x0E, 0xCA, 0x89, 0xC6, 0x31, 0x9F, 0xCB, +0x59, 0x0C, 0x9B, 0x47, 0x6A, 0x0C, 0x56, 0x6D, 0x03, 0xF7, 0xD4, 0x81, 0x15, 0x3E, 0x9F, 0x6F, +0x15, 0x22, 0x10, 0x52, 0xCC, 0xD9, 0x2C, 0xA3, 0x85, 0xA5, 0x42, 0x04, 0x7E, 0xD2, 0xE2, 0x16, +0x98, 0xB1, 0x40, 0x57, 0x8A, 0x5A, 0x54, 0xA1, 0xF8, 0x9B, 0x8C, 0x77, 0x7A, 0x5F, 0x26, 0x4B, +0x85, 0xE0, 0x3C, 0xB2, 0xC2, 0xC2, 0x90, 0xDD, 0x8E, 0xD9, 0xF5, 0x94, 0xAC, 0x3E, 0x56, 0x20, +0x63, 0x13, 0x08, 0x48, 0x69, 0x9C, 0xD7, 0xD7, 0x05, 0x1B, 0xB0, 0xF3, 0x8A, 0x81, 0x78, 0x43, +0x62, 0x04, 0xAB, 0x8D, 0xA8, 0xD7, 0x51, 0x2C, 0xFE, 0x66, 0x80, 0x06, 0xC0, 0x10, 0x92, 0xF3, +0xBA, 0xBF, 0x42, 0x95, 0x96, 0xB7, 0xDB, 0xA5, 0x73, 0x06, 0xB8, 0x0D, 0xAE, 0xDA, 0x00, 0xA7, +0xD2, 0xF8, 0x63, 0xF0, 0x35, 0x15, 0x7D, 0x9B, 0x1C, 0x3D, 0x3F, 0x83, 0x9C, 0xAE, 0xC1, 0xFA, +0xE3, 0x62, 0x9A, 0x8A, 0xF3, 0x86, 0x2F, 0xE4, 0xDB, 0x49, 0xDF, 0x2C, 0x17, 0xFE, 0x2C, 0x30, +0xD1, 0x76, 0x88, 0x92, 0x60, 0x60, 0xB9, 0xF9, 0x35, 0x0F, 0xE2, 0xE8, 0x8E, 0x73, 0x9E, 0x7C, +0xF6, 0xD0, 0x5F, 0x81, 0x22, 0x2A, 0x5D, 0x5F, 0x10, 0xFE, 0x7B, 0x06, 0xB5, 0x5F, 0xF3, 0x42, +0x94, 0xC4, 0x3E, 0xE0, 0x0E, 0x11, 0xCC, 0x3B, 0x4F, 0xBD, 0x06, 0xD1, 0x15, 0xF5, 0x32, 0x76, +0xE4, 0x97, 0xBF, 0xA7, 0xC5, 0xD0, 0x5E, 0x5A, 0x1C, 0x35, 0x18, 0x43, 0x6B, 0x10, 0xD2, 0xAB, +0x8B, 0xF6, 0xE1, 0xBB, 0x8C, 0xE9, 0x9C, 0x6C, 0x27, 0x96, 0xF8, 0x19, 0x6F, 0x6E, 0x73, 0x28, +0x8F, 0x51, 0x61, 0x0B, 0x4D, 0x4F, 0x22, 0x15, 0x1A, 0xCD, 0x88, 0x2C, 0x6F, 0x9C, 0xEC, 0x79, +0x51, 0x70, 0x27, 0x8E, 0x58, 0xEF, 0xDE, 0x0B, 0xAA, 0x4D, 0xD2, 0x8F, 0x96, 0xE0, 0x71, 0x6A, +0x8C, 0x41, 0xDB, 0x98, 0xF7, 0x2A, 0x09, 0x59, 0xD9, 0x48, 0x7B, 0x14, 0x87, 0x9B, 0x4A, 0xBB, +0x88, 0xF2, 0x9D, 0x9D, 0x47, 0xF2, 0x80, 0x4B, 0xD9, 0x0E, 0xF3, 0xA9, 0x7E, 0xEF, 0xA1, 0x80, +0x9D, 0x6F, 0x67, 0x6C, 0x14, 0x09, 0x2C, 0xB9, 0x03, 0xF3, 0x58, 0x37, 0xAE, 0x6B, 0xF7, 0x01, +0x86, 0xDF, 0x43, 0xD5, 0xE4, 0xE2, 0x28, 0x8E, 0x8D, 0xA4, 0xF2, 0xC5, 0x7A, 0xA7, 0x89, 0x3A, +0xE2, 0x7E, 0x77, 0xFD, 0x3A, 0x9A, 0x65, 0x5D, 0x87, 0xDC, 0x85, 0x70, 0xAB, 0xF3, 0x18, 0x0A + +cipher_key = +0xE4, 0x23, 0x33, 0x8A, 0x35, 0x64, 0x61, 0xE2, 0x49, 0x03, 0xDD, 0xC6, 0xB8, 0xCA, 0x55, 0x7A, +0xd0, 0xe7, 0x4b, 0xfb, 0x5d, 0xe5, 0x0c, 0xe7 + +auth_key = +0xaf, 0x96, 0x42, 0xf1, 0x8c, 0x50, 0xdc, 0x67, 0x1a, 0x43, 0x47, 0x62, 0xc7, 0x04, 0xab, 0x05, +0xf5, 0x0c, 0xe7, 0xa2, 0xa6, 0x23, 0xd5, 0x3d, 0x95, 0xd8, 0xcd, 0x86, 0x79, 0xf5, 0x01, 0x47, +0x4f, 0xf9, 0x1d, 0x9d, 0x36, 0xf7, 0x68, 0x1a, 0x64, 0x44, 0x58, 0x5d, 0xe5, 0x81, 0x15, 0x2a, +0x41, 0xe4, 0x0e, 0xaa, 0x1f, 0x04, 0x21, 0xff, 0x2c, 0xf3, 0x73, 0x2b, 0x48, 0x1e, 0xd2, 0xf7, +0xf6, 0xd9, 0xaf, 0xbf, 0x08, 0x3b, 0xbb, 0x19, 0x5f, 0xf6, 0x7d, 0x25, 0x85, 0xdf, 0x6b, 0x54, +0xd0, 0xe7, 0x4b, 0x9e, 0xc7, 0xef, 0xca, 0x48, 0x6f, 0x21, 0xd7, 0x51, 0xc8, 0x21, 0xc1, 0x15, +0xe8, 0x38, 0x36, 0x58, 0x39, 0xd9, 0x9a, 0xc5, 0xe7, 0x3b, 0xc4, 0x47, 0xe2, 0xbd, 0x80, 0x73, +0xf8, 0xd1, 0x9a, 0x5e, 0x4b, 0xfb, 0x52, 0x6b, 0x50, 0xaf, 0x8b, 0xb7, 0xb5, 0x2c, 0x52, 0x84 + +cipher_iv = +0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F + +#################### +# sha_hmac_buff_32 # +#################### +[sha1_hmac_buff_32] +digest = +0x34, 0xCB, 0xFA, 0xE6, 0xBF, 0x60, 0x88, 0x82, 0x2C, 0x9F, 0x96, 0x4A, 0x5A, 0xD8, 0xAE, 0x48, +0x1D, 0x2B, 0x0A, 0x22 + +[sha224_hmac_buff_32] +digest = +0x17, 0xDD, 0xF3, 0xC0, 0xFC, 0xBC, 0x89, 0xE1, 0x02, 0x94, 0x51, 0x2B, 0xA0, 0x63, 0x4D, 0x1B, +0x78, 0xCA, 0x0D, 0xD5, 0x99, 0xC0, 0xB8, 0x8E, 0x7F, 0x8A, 0xCB, 0x92 + +[sha256_hmac_buff_32] +digest = +0xF7, 0xC0, 0xFD, 0x09, 0x94, 0x8F, 0x88, 0xF3, 0x26, 0xE1, 0xB1, 0xA6, 0x70, 0xFD, 0x57, 0xA2, +0xB7, 0x63, 0x9C, 0x35, 0x97, 0x60, 0x27, 0x6A, 0xCC, 0xD8, 0x4A, 0xAE, 0xFD, 0xEC, 0x14, 0x56 + +[sha384_hmac_buff_32] +digest = +0x01, 0x90, 0x96, 0x95, 0x8B, 0xE8, 0xF8, 0x30, 0xE0, 0xFE, 0xD4, 0x83, 0xE4, 0xE1, 0x48, 0xEB, +0xEB, 0x3E, 0x6D, 0xBC, 0x72, 0xD8, 0xBF, 0x00, 0x86, 0x0B, 0x80, 0xCB, 0xCF, 0x0E, 0x83, 0x7E, +0x77, 0x29, 0xA0, 0x71, 0xA9, 0x15, 0x8F, 0xE5, 0xC4, 0x32, 0xC8, 0xDB, 0x0A, 0xD1, 0xE3, 0x79 + +[sha512_hmac_buff_32] +digest = +0x45, 0x72, 0x2B, 0xF4, 0x10, 0x82, 0xB6, 0xBC, 0xDB, 0x44, 0x34, 0x47, 0x55, 0xA7, 0x34, 0x4C, +0x1C, 0x5D, 0x4D, 0x88, 0x58, 0x0B, 0xC2, 0x3E, 0xE7, 0x49, 0xF4, 0x6A, 0x99, 0x35, 0xE5, 0x2B, +0xF4, 0x18, 0xD7, 0xD1, 0xAF, 0xC9, 0x81, 0xC5, 0x97, 0xE7, 0x94, 0xC4, 0xFD, 0x95, 0x7E, 0x1D, +0x4E, 0xF4, 0x88, 0xC0, 0x10, 0x31, 0xB5, 0x1E, 0x39, 0x91, 0x1A, 0x48, 0xC2, 0x2F, 0xFE, 0xF6 + +#################### +# sha_hmac_buff_64 # +#################### +[sha1_hmac_buff_64] +digest = +0xD5, 0x0D, 0x9A, 0x30, 0xED, 0x71, 0x88, 0xF3, 0x72, 0x5D, 0x1C, 0xEF, 0x44, 0x36, 0xC1, 0x0C, +0x66, 0x32, 0x15, 0xB5 + +[sha224_hmac_buff_64] +digest = +0x59, 0xF0, 0x38, 0xA5, 0x7F, 0xDC, 0x55, 0xAF, 0x0A, 0x18, 0x0B, 0x34, 0xC1, 0x48, 0xFC, 0xB2, +0xF8, 0x3B, 0xC2, 0x4A, 0x36, 0x0F, 0xEA, 0x4F, 0xD9, 0x46, 0x25, 0x1F + +[sha256_hmac_buff_64] +digest = +0x00, 0xD2, 0x5F, 0xAC, 0x89, 0x4B, 0x16, 0x08, 0x89, 0x8D, 0x4F, 0x8F, 0x56, 0xF5, 0xA2, 0x9B, +0xDB, 0x91, 0x2F, 0xCE, 0x90, 0x0E, 0x4B, 0x17, 0x74, 0x8B, 0xC4, 0x8A, 0x47, 0xF0, 0x7C, 0x7A + +[sha384_hmac_buff_64] +digest = +0xFE, 0x79, 0x5C, 0x3D, 0xEA, 0x2C, 0x04, 0xB9, 0xE9, 0x37, 0x4E, 0x02, 0xE8, 0x56, 0xE8, 0x7E, +0xB4, 0xA7, 0x3D, 0x37, 0xDD, 0x05, 0x58, 0x6A, 0x3D, 0x44, 0x0E, 0x84, 0xA9, 0xB4, 0x1F, 0x26, +0xED, 0xAF, 0xA3, 0x4D, 0xA3, 0x6E, 0x30, 0x8F, 0xE3, 0x79, 0xB9, 0x58, 0x4E, 0xB3, 0x36, 0x1D + +[sha512_hmac_buff_64] +digest = +0x3A, 0xFE, 0x1E, 0xC8, 0x75, 0x0D, 0xF3, 0x15, 0x03, 0xCC, 0x7B, 0x50, 0xD1, 0x3E, 0x35, 0x97, +0x4D, 0x80, 0xB8, 0x0B, 0x5E, 0x22, 0x4D, 0xB9, 0xD2, 0xC9, 0x0E, 0x24, 0xC4, 0xD9, 0xDD, 0x95, +0xA1, 0x7D, 0xED, 0xE4, 0x28, 0x17, 0x7C, 0x50, 0x0B, 0x40, 0x81, 0x18, 0xEA, 0xFF, 0xBA, 0x1C, +0x8C, 0x5D, 0x17, 0xB3, 0x5B, 0x39, 0x70, 0x93, 0x9A, 0xA0, 0x47, 0x59, 0x06, 0x01, 0xE5, 0xD0 + +##################### +# sha_hmac_buff_128 # +##################### +[sha1_hmac_buff_128] +digest = +0xF6, 0x21, 0x56, 0x15, 0xEC, 0xB6, 0xDE, 0x2B, 0x68, 0x79, 0x30, 0x69, 0x69, 0x82, 0x4B, 0x97, +0x57, 0x61, 0xED, 0x01 + +[sha224_hmac_buff_128] +digest = +0xC4, 0x5A, 0x48, 0x07, 0xCC, 0xFD, 0x68, 0x28, 0xCC, 0xDA, 0x13, 0x9C, 0xFE, 0x02, 0x22, 0x1C, +0xD5, 0x2E, 0x9F, 0x0D, 0xED, 0x0B, 0x9B, 0x6D, 0xF3, 0x81, 0xD6, 0xDF + +[sha256_hmac_buff_128] +digest = +0x44, 0xA6, 0x70, 0x6C, 0xD2, 0xA6, 0x3F, 0xFB, 0x98, 0xB9, 0x6A, 0x71, 0x40, 0xCF, 0xD2, 0x40, +0xA3, 0xC0, 0xC6, 0x4E, 0xF6, 0xD8, 0x4D, 0x87, 0xF5, 0x9B, 0xAC, 0xB0, 0x7B, 0xB7, 0x35, 0x5B + +[sha384_hmac_buff_128] +digest = +0xD7, 0xFA, 0xF1, 0xC5, 0xD6, 0xCE, 0xB9, 0x77, 0xD5, 0x6B, 0x4D, 0x7F, 0xFE, 0xAF, 0xDF, 0xCE, +0x68, 0x1A, 0xB7, 0x3E, 0xA5, 0x9A, 0xF5, 0x79, 0x91, 0x96, 0x7C, 0xED, 0xC9, 0x02, 0x31, 0x67, +0xC4, 0xD9, 0xD7, 0x5A, 0xD7, 0xF0, 0x82, 0x2C, 0xBD, 0x4A, 0xFF, 0x03, 0x25, 0xB6, 0x50, 0x17 + +[sha512_hmac_buff_128] +digest = +0x78, 0xE1, 0x86, 0x74, 0xC1, 0x83, 0xDB, 0x3C, 0xAC, 0x2B, 0x17, 0xAC, 0x12, 0xAA, 0x30, 0xDE, +0x2C, 0x86, 0xC4, 0x53, 0x4A, 0xC2, 0x78, 0x86, 0x3A, 0x8A, 0x96, 0x36, 0x2B, 0x09, 0xB1, 0x62, +0xCA, 0xD0, 0x25, 0xB2, 0x5B, 0x3A, 0x76, 0xFA, 0xED, 0x5B, 0xFB, 0xF0, 0x8F, 0xF2, 0x06, 0x76, +0x9B, 0xE0, 0x82, 0x25, 0x10, 0x5E, 0x8A, 0x13, 0xA1, 0x25, 0x3B, 0xB7, 0xFC, 0xC3, 0x15, 0xED + +##################### +# sha_hmac_buff_256 # +##################### +[sha1_hmac_buff_256] +digest = +0x31, 0x36, 0xC8, 0xC7, 0x95, 0xDB, 0xF7, 0xF3, 0xA0, 0xFF, 0x89, 0x94, 0x8D, 0xB2, 0x3D, 0x5F, +0x7A, 0xC1, 0x38, 0x97 + +[sha224_hmac_buff_256] +digest = +0xAE, 0x1B, 0xA4, 0x42, 0xE6, 0x3C, 0x3D, 0xEE, 0xEE, 0xD1, 0x24, 0xD2, 0xFF, 0xD2, 0x1A, 0xF3, +0x28, 0x87, 0x8F, 0x00, 0xE3, 0x74, 0xA1, 0xA2, 0xED, 0x70, 0x2D, 0x9D + +[sha256_hmac_buff_256] +digest = +0x73, 0x89, 0x54, 0x06, 0x65, 0x71, 0x1D, 0xA1, 0xAB, 0x4A, 0x0A, 0x36, 0x63, 0x92, 0xB8, 0x94, +0x98, 0x98, 0xB5, 0x27, 0x9D, 0x9C, 0xF2, 0x91, 0x0C, 0x56, 0xC4, 0xEE, 0x99, 0xC6, 0xDE, 0xC7 + +[sha384_hmac_buff_256] +digest = +0xFE, 0x6E, 0xA5, 0xDC, 0x82, 0x57, 0xBB, 0x4D, 0xA8, 0xF1, 0x2F, 0xD1, 0xA2, 0x05, 0x0A, 0xFE, +0xF2, 0x64, 0x8A, 0xB3, 0x96, 0xBA, 0x06, 0x47, 0xA4, 0x40, 0x76, 0x8E, 0xB8, 0xE3, 0xAD, 0xD1, +0xB2, 0x90, 0x9A, 0x02, 0xD1, 0x13, 0x4F, 0x74, 0x9B, 0xEB, 0x09, 0xFC, 0x7F, 0x99, 0xAC, 0xCD + +[sha512_hmac_buff_256] +digest = +0x9F, 0x71, 0xE0, 0x86, 0xF9, 0x76, 0x3F, 0xAB, 0x16, 0x4D, 0x9C, 0x28, 0x72, 0x3A, 0x17, 0x8F, +0x35, 0xD1, 0x44, 0x18, 0xE0, 0x4A, 0xBD, 0x8B, 0x25, 0x9F, 0x6E, 0x5B, 0xE3, 0xF4, 0x1C, 0x40, +0x2B, 0xF4, 0x61, 0x59, 0x60, 0x8D, 0x55, 0xD8, 0x18, 0x9B, 0x65, 0x8D, 0x9F, 0xAA, 0xB3, 0x71, +0x3D, 0xB5, 0x70, 0xD2, 0x26, 0xF1, 0x55, 0xDC, 0xCE, 0x49, 0x40, 0xD7, 0x23, 0x6B, 0x20, 0x4A + +##################### +# sha_hmac_buff_512 # +##################### +[sha1_hmac_buff_512] +digest = +0x0F, 0xF6, 0x33, 0x61, 0x16, 0x5C, 0xD1, 0x9E, 0xE0, 0x3D, 0x95, 0x93, 0xD2, 0x82, 0xDE, 0x2E, +0xFA, 0x25, 0x56, 0xE4 + +[sha224_hmac_buff_512] +digest = +0x85, 0x61, 0x57, 0x9F, 0x3E, 0x6A, 0xE1, 0x5C, 0x1D, 0xA3, 0x98, 0x9C, 0x28, 0x2C, 0x96, 0x8E, +0xD0, 0x7D, 0x70, 0x7D, 0xF5, 0x98, 0xA4, 0x7C, 0x88, 0x1C, 0xA4, 0x5C + +[sha256_hmac_buff_512] +digest = +0xF4, 0x77, 0xB2, 0x2E, 0xAD, 0xBC, 0xA2, 0xCD, 0x49, 0xE0, 0xD2, 0xB0, 0xB3, 0xDE, 0x51, 0xD0, +0xBC, 0xEF, 0x33, 0x50, 0x4F, 0x39, 0xBC, 0x94, 0x18, 0xF3, 0xDD, 0xFC, 0xB5, 0x8E, 0x44, 0x18 + +[sha384_hmac_buff_512] +digest = +0x03, 0x4F, 0x86, 0xA0, 0xBC, 0x00, 0x44, 0xEB, 0x06, 0x75, 0x61, 0x13, 0x92, 0x60, 0x74, 0x44, +0x1D, 0xCB, 0x2C, 0x8D, 0xEC, 0xE5, 0x5C, 0xBE, 0xA3, 0xAE, 0x5F, 0xE2, 0x71, 0xED, 0xAC, 0x69, +0x9C, 0x6C, 0xE3, 0x20, 0x5C, 0x90, 0xC3, 0xF4, 0x36, 0x76, 0x70, 0xAE, 0x2A, 0x9C, 0xF5, 0x0B + +[sha512_hmac_buff_512] +digest = +0x3B, 0x83, 0x4B, 0x7F, 0x2A, 0x62, 0x9A, 0xF6, 0x42, 0x29, 0x7A, 0xF0, 0xCA, 0xE7, 0xBE, 0x1F, +0x92, 0x5C, 0x66, 0x88, 0x58, 0xFA, 0xA4, 0xC7, 0xE7, 0xF0, 0xEA, 0x83, 0xB2, 0x0A, 0x2C, 0x3B, +0xA7, 0xD4, 0xA4, 0x3E, 0x87, 0x00, 0x44, 0x2B, 0x3F, 0xB2, 0x4B, 0xFF, 0xAD, 0x9B, 0x07, 0x7D, +0xA1, 0x09, 0x09, 0x60, 0x68, 0x2F, 0x7B, 0x16, 0x43, 0x0E, 0x22, 0xAF, 0x78, 0x8D, 0xC7, 0x16 + +###################### +# sha_hmac_buff_1024 # +###################### +[sha1_hmac_buff_1024] +digest = +0xCD, 0x04, 0x78, 0x4A, 0xD5, 0xFE, 0x95, 0xFB, 0x04, 0x85, 0xD5, 0x25, 0x58, 0xE6, 0x6C, 0x81, +0xFA, 0x4B, 0xE7, 0x75 + +[sha224_hmac_buff_1024] +digest = +0x10, 0xA5, 0x18, 0x56, 0x66, 0xDE, 0xE6, 0xF6, 0x71, 0xAF, 0x65, 0xEC, 0xE3, 0x77, 0x08, 0x58, +0xD5, 0x45, 0xE6, 0x21, 0xF5, 0xCC, 0x2B, 0xE2, 0x76, 0x91, 0x51, 0xD9 + +[sha256_hmac_buff_1024] +digest = +0xB4, 0x09, 0xF9, 0xA0, 0x5B, 0x80, 0xFF, 0xBA, 0x21, 0x5F, 0x57, 0xAB, 0x8B, 0x67, 0x89, 0x6D, +0xB4, 0xE9, 0xEA, 0x5D, 0x77, 0x57, 0xBD, 0x0A, 0x60, 0xA4, 0x6B, 0xE8, 0xAA, 0x8A, 0x9B, 0xC7 + +[sha384_hmac_buff_1024] +digest = +0x9D, 0xAE, 0x37, 0x87, 0x2C, 0x36, 0xFD, 0x51, 0xF1, 0xF2, 0x4C, 0x82, 0x27, 0xB5, 0x99, 0x63, +0xAB, 0xD7, 0x62, 0x4A, 0x4E, 0xF1, 0xBF, 0xFB, 0xCA, 0x30, 0x55, 0x3F, 0x43, 0x85, 0xDE, 0x6B, +0xA2, 0xB2, 0x8B, 0x45, 0x2A, 0x9D, 0x39, 0x29, 0x63, 0x1C, 0x04, 0x47, 0x58, 0x94, 0x5C, 0x91 + +[sha512_hmac_buff_1024] +digest = +0xBF, 0x06, 0x94, 0xBB, 0x2E, 0x9C, 0x0A, 0xA4, 0xF3, 0x5F, 0x52, 0x4B, 0x42, 0x6E, 0xE1, 0x6C, +0xA8, 0xAB, 0xB7, 0xE9, 0x6F, 0xAB, 0x77, 0xF8, 0x94, 0xD0, 0xA1, 0x81, 0xB8, 0x17, 0x21, 0xAC, +0xB2, 0x3C, 0x73, 0x71, 0xDD, 0x76, 0xF8, 0x58, 0x84, 0xE7, 0xBB, 0xD4, 0x4A, 0x4F, 0x51, 0xF5, +0x8C, 0x87, 0xAA, 0xAC, 0xCE, 0x5E, 0xFB, 0xD3, 0x1F, 0xA2, 0x49, 0xF2, 0x40, 0xAB, 0xB8, 0x76 + +###################### +# sha_hmac_buff_2048 # +###################### +[sha1_hmac_buff_2048] +digest = +0x56, 0xE7, 0x86, 0x23, 0x2C, 0x77, 0xBE, 0x2B, 0xE6, 0x76, 0xA1, 0xE9, 0xB1, 0x0F, 0x25, 0x15, +0x1D, 0x59, 0x7F, 0x4A + +[sha224_hmac_buff_2048] +digest = +0xFD, 0xEB, 0x9E, 0x04, 0x53, 0xC7, 0xEA, 0x56, 0x21, 0x91, 0x6D, 0x8B, 0xDC, 0xA1, 0x0A, 0x2F, +0x73, 0x4D, 0x05, 0x36, 0x43, 0x58, 0x71, 0xB2, 0x74, 0x6E, 0x7B, 0xAF + +[sha256_hmac_buff_2048] +digest = +0x25, 0x76, 0xA5, 0x64, 0x00, 0x13, 0xF7, 0xE7, 0x2D, 0x6D, 0x17, 0x36, 0x13, 0xF3, 0xC7, 0x57, +0x70, 0x30, 0xB2, 0x76, 0x7A, 0xF4, 0x8F, 0xAF, 0x6B, 0x8C, 0x26, 0x88, 0x73, 0x2E, 0x49, 0xC0 + +[sha384_hmac_buff_2048] +digest = +0xF0, 0x69, 0x78, 0x9A, 0x34, 0x88, 0x25, 0x3C, 0x3D, 0xF5, 0x42, 0x65, 0x25, 0x79, 0xB2, 0xFC, +0x3B, 0x92, 0x46, 0xF9, 0x0D, 0x6D, 0xC1, 0x8E, 0x45, 0xBE, 0x8B, 0x70, 0x7D, 0x1B, 0x7E, 0xDE, +0x93, 0x04, 0xC8, 0x59, 0x4B, 0x37, 0x2C, 0x20, 0x51, 0xB1, 0x91, 0x4F, 0xA4, 0x30, 0x09, 0xED + +[sha512_hmac_buff_2048] +digest = +0xAE, 0x8A, 0x42, 0x03, 0x11, 0x25, 0xE8, 0xC3, 0x4B, 0x4B, 0xC0, 0x29, 0x27, 0xE0, 0x0D, 0x27, +0x13, 0x8F, 0x7D, 0x82, 0x72, 0x94, 0x4B, 0xF8, 0xC3, 0x1A, 0xE1, 0x5A, 0xF3, 0x8E, 0x23, 0x27, +0x76, 0xE4, 0xF5, 0x3E, 0x24, 0x5B, 0xA3, 0xFA, 0x49, 0x8E, 0x57, 0xE3, 0x88, 0x15, 0x1D, 0xF6, +0x27, 0xA5, 0xC8, 0xFB, 0x34, 0x44, 0x18, 0x6A, 0x64, 0xBE, 0xFB, 0x1E, 0x87, 0xA8, 0x36, 0x1E diff --git a/dpdk/app/test-crypto-perf/data/aes_cbc_256_sha.data b/dpdk/app/test-crypto-perf/data/aes_cbc_256_sha.data new file mode 100644 index 00000000..8da81611 --- /dev/null +++ b/dpdk/app/test-crypto-perf/data/aes_cbc_256_sha.data @@ -0,0 +1,504 @@ +# List of tests for AES-256 CBC: +# 1) [sha1_hmac_buff_x] +# 2) [sha224_hmac_buff_x] +# 3) [sha256_hmac_buff_x] +# 4) [sha384_hmac_buff_x] +# 5) [sha512_hmac_buff_x] +# where x is one of the values: 32, 64, 128, 256, 512, 1024, 2048 + +########## +# GLOBAL # +########## +plaintext = +0xff, 0xca, 0xfb, 0xf1, 0x38, 0x20, 0x2f, 0x7b, 0x24, 0x98, 0x26, 0x7d, 0x1d, 0x9f, 0xb3, 0x93, +0xd9, 0xef, 0xbd, 0xad, 0x4e, 0x40, 0xbd, 0x60, 0xe9, 0x48, 0x59, 0x90, 0x67, 0xd7, 0x2b, 0x7b, +0x8a, 0xe0, 0x4d, 0xb0, 0x70, 0x38, 0xcc, 0x48, 0x61, 0x7d, 0xee, 0xd6, 0x35, 0x49, 0xae, 0xb4, +0xaf, 0x6b, 0xdd, 0xe6, 0x21, 0xc0, 0x60, 0xce, 0x0a, 0xf4, 0x1c, 0x2e, 0x1c, 0x8d, 0xe8, 0x7b, +0x59, 0xda, 0x19, 0x4f, 0xec, 0x07, 0x8e, 0xe2, 0xf0, 0x61, 0xf9, 0x27, 0x61, 0x6f, 0xf8, 0xdf, +0x62, 0x4d, 0xaf, 0x06, 0xfe, 0x41, 0xa6, 0xa6, 0xf9, 0xa2, 0x06, 0x40, 0xb3, 0x04, 0xbd, 0xe6, +0xc8, 0x17, 0xfb, 0x56, 0x6f, 0xa9, 0x3b, 0x8e, 0xa6, 0x58, 0xdc, 0x91, 0x17, 0x58, 0x42, 0x95, +0xa3, 0x7c, 0x81, 0x78, 0xa6, 0x3d, 0x3f, 0x75, 0x74, 0x17, 0x1a, 0xd3, 0x6c, 0x2f, 0x48, 0x39, +0x20, 0x20, 0xc1, 0x9a, 0x29, 0x84, 0x7d, 0x2d, 0x52, 0xa1, 0xf9, 0x5c, 0xf3, 0x4f, 0x91, 0xfc, +0x75, 0xcf, 0xd6, 0x2d, 0xe7, 0x9a, 0x59, 0x6e, 0x00, 0x0e, 0x8d, 0x22, 0x17, 0xbd, 0xa0, 0xdd, +0x79, 0x1f, 0x71, 0xe6, 0xcd, 0x2f, 0xb1, 0xb6, 0xbc, 0xc3, 0xdb, 0x02, 0x91, 0x41, 0x9b, 0x09, +0xa9, 0xd2, 0x7e, 0xbd, 0x2c, 0x18, 0xae, 0xc0, 0x93, 0x0c, 0x02, 0x9a, 0xdb, 0x4e, 0xaa, 0xeb, +0x84, 0x4b, 0x43, 0x5e, 0xf0, 0x98, 0xf2, 0x5f, 0x86, 0x70, 0x96, 0x90, 0x15, 0x30, 0xcf, 0x3a, +0xc9, 0x33, 0x21, 0xec, 0x59, 0x86, 0xfc, 0x65, 0x7d, 0xbe, 0xb9, 0xf8, 0x97, 0xf9, 0x30, 0xa9, +0x6d, 0xfc, 0x0c, 0x6e, 0x36, 0x67, 0xd5, 0xa6, 0x67, 0xd9, 0xbd, 0x9b, 0x34, 0x5d, 0xa7, 0xdd, +0xda, 0x46, 0x33, 0x25, 0x60, 0x4a, 0x18, 0xf1, 0x55, 0x07, 0xb2, 0xb7, 0x26, 0x7b, 0xa6, 0x1e, +0x77, 0xbe, 0x7f, 0x35, 0x46, 0xdf, 0x56, 0x9c, 0x22, 0x19, 0xc8, 0x85, 0xa2, 0x45, 0xb2, 0xad, +0xf9, 0x26, 0x66, 0xab, 0xfc, 0x97, 0x4b, 0x51, 0x32, 0x36, 0xbc, 0xad, 0xcf, 0x54, 0x3a, 0x4f, +0x94, 0xdb, 0xd2, 0xf9, 0x67, 0x1b, 0x3b, 0xe5, 0xb2, 0x1d, 0xc5, 0x52, 0x64, 0x2c, 0x06, 0x44, +0xcf, 0x18, 0x83, 0xe0, 0xd8, 0x04, 0x92, 0xa9, 0xc4, 0x3c, 0x8b, 0xa3, 0x2b, 0xbc, 0x88, 0x7e, +0xc0, 0x76, 0xa7, 0xe2, 0x7b, 0x47, 0x90, 0xf2, 0xaa, 0x0a, 0x34, 0x1b, 0x91, 0x12, 0xd2, 0xd0, +0x82, 0x45, 0xf4, 0x57, 0xf1, 0xbd, 0x91, 0x5e, 0xab, 0x41, 0x4c, 0xdf, 0x91, 0x4c, 0xdd, 0x67, +0x04, 0xa0, 0x98, 0x23, 0x8c, 0x24, 0xbe, 0xd6, 0x80, 0xb3, 0x6d, 0x04, 0xa1, 0x77, 0x43, 0xa5, +0xee, 0xb7, 0xce, 0xb1, 0x48, 0x43, 0x94, 0x61, 0x15, 0x20, 0x9d, 0xce, 0xd0, 0x14, 0x95, 0x37, +0xc8, 0x64, 0xa3, 0x2d, 0x3d, 0xe3, 0xff, 0xb4, 0x55, 0x83, 0x84, 0x41, 0x50, 0x57, 0xbd, 0x5a, +0x0c, 0xe4, 0xda, 0x3b, 0x36, 0x4d, 0x21, 0xb5, 0x6f, 0x73, 0x2a, 0x8c, 0x78, 0x4f, 0x9b, 0x83, +0xda, 0x11, 0x3c, 0xf0, 0xc9, 0x7e, 0xa6, 0x48, 0x34, 0x53, 0x62, 0xd3, 0x0c, 0xff, 0xb1, 0x74, +0xd6, 0xea, 0xa5, 0xfc, 0x13, 0x1c, 0x05, 0xa8, 0xc0, 0xbc, 0x95, 0x9c, 0x8c, 0xf6, 0x8c, 0xc3, +0xf3, 0x69, 0xab, 0x93, 0x65, 0xc0, 0xb7, 0x7e, 0xb0, 0x16, 0x7c, 0xb5, 0x5f, 0x05, 0x28, 0xc9, +0x09, 0x4e, 0x2a, 0x32, 0x87, 0xb3, 0xab, 0xf8, 0x4c, 0xab, 0xeb, 0x3b, 0x6a, 0xa0, 0x1d, 0x7f, +0xef, 0xe5, 0x9b, 0xa4, 0xb7, 0xd7, 0xc2, 0x5e, 0x03, 0x0f, 0x99, 0xeb, 0xb1, 0xb1, 0xa6, 0x9d, +0x1c, 0x7c, 0x5c, 0x94, 0x8b, 0x6e, 0x11, 0x7a, 0xb3, 0x6d, 0x1e, 0x61, 0x64, 0xc3, 0x7d, 0x1c, +0xb3, 0x54, 0x65, 0x08, 0x3b, 0xda, 0x97, 0xb9, 0x75, 0xc1, 0x2b, 0x3e, 0xa8, 0x5c, 0x3c, 0x2d, +0x81, 0x5b, 0xbf, 0x5a, 0x13, 0x0e, 0xeb, 0x66, 0xc0, 0x0b, 0x8f, 0x04, 0x68, 0x68, 0x9b, 0xe3, +0x0d, 0x84, 0xe0, 0xcf, 0x83, 0xd7, 0x62, 0x48, 0xc1, 0x31, 0xa5, 0xd5, 0xbc, 0xe3, 0xa3, 0xa5, +0xb6, 0xd1, 0xfd, 0x81, 0x91, 0x4d, 0xbd, 0xc4, 0x62, 0x4f, 0xe3, 0xd5, 0x99, 0x14, 0xf1, 0xcd, +0xf4, 0x7d, 0x13, 0xda, 0x68, 0x0a, 0xca, 0xd6, 0x82, 0x0b, 0xf6, 0xea, 0xad, 0x78, 0xa4, 0xc8, +0x14, 0x7a, 0xec, 0x11, 0xd3, 0x16, 0x86, 0x9f, 0x17, 0x37, 0x6a, 0x06, 0x56, 0xaa, 0x1b, 0xd1, +0xaf, 0x85, 0x95, 0x21, 0x36, 0x69, 0xec, 0x1b, 0x56, 0x84, 0x01, 0x3f, 0x4d, 0x34, 0x3d, 0x2d, +0x38, 0x57, 0x2d, 0x7e, 0xd9, 0x7b, 0x2d, 0x81, 0x86, 0xd4, 0x7c, 0x83, 0x12, 0x1d, 0x9d, 0x27, +0x72, 0x1b, 0x5e, 0xf4, 0x15, 0xa5, 0xcd, 0xb7, 0x5f, 0xbb, 0x49, 0xa1, 0xd9, 0xdd, 0x8d, 0xad, +0xa9, 0x2c, 0x65, 0x18, 0x91, 0xfd, 0xd2, 0xd4, 0x09, 0x60, 0x0c, 0xfd, 0xa4, 0xe1, 0x25, 0x87, +0x32, 0x64, 0x7b, 0x99, 0xd7, 0x61, 0x2f, 0xd4, 0x73, 0xdd, 0x85, 0x26, 0x08, 0x92, 0xc0, 0xe1, +0x4f, 0x0c, 0x76, 0x5b, 0x26, 0x69, 0xdb, 0x78, 0x35, 0x65, 0xb9, 0x58, 0x1f, 0x9c, 0x0f, 0x18, +0x95, 0xfe, 0x40, 0xfc, 0xf7, 0x93, 0x71, 0x70, 0x8b, 0x73, 0xdc, 0xb0, 0x88, 0x72, 0x19, 0x26, +0x94, 0x26, 0xa7, 0xaa, 0x00, 0x72, 0x61, 0x53, 0xd2, 0x5d, 0x8f, 0x5e, 0x51, 0x88, 0x2d, 0xa4, +0x28, 0xd5, 0xaf, 0x2d, 0xd2, 0x84, 0x39, 0x75, 0x1e, 0xe7, 0xf0, 0x23, 0xc0, 0x4f, 0x8d, 0xdd, +0x5c, 0x90, 0xef, 0x6e, 0x53, 0xe0, 0x54, 0x67, 0xe1, 0x5b, 0x10, 0xf1, 0xf5, 0xf8, 0x64, 0x34, +0x94, 0xeb, 0x37, 0xf7, 0xe9, 0xaa, 0x6c, 0xa4, 0xd8, 0x74, 0x6d, 0xca, 0x8d, 0x1a, 0x31, 0x73, +0xca, 0xb4, 0xc7, 0x47, 0x34, 0x7f, 0xf8, 0x24, 0x9b, 0xfa, 0xc9, 0xcc, 0xa8, 0x61, 0xb4, 0x0e, +0x4d, 0x68, 0xc7, 0xa0, 0xcb, 0xea, 0xf0, 0xcc, 0x0a, 0x6c, 0xf2, 0x33, 0x42, 0x99, 0x6c, 0xd8, +0x74, 0x7f, 0x1e, 0x8a, 0xa3, 0x0a, 0x48, 0x4b, 0x7e, 0xbe, 0xdb, 0x7f, 0x56, 0x69, 0x43, 0xe8, +0xbf, 0x12, 0xc4, 0x7b, 0xc2, 0xd9, 0xfa, 0x5c, 0xeb, 0x45, 0xca, 0x07, 0x3d, 0xc0, 0xcd, 0x68, +0x8b, 0xd0, 0x79, 0xea, 0x0a, 0x78, 0x06, 0xdc, 0x81, 0xd7, 0x32, 0x18, 0xb3, 0x65, 0xbe, 0x47, +0xbb, 0xfa, 0x17, 0x09, 0xe9, 0x31, 0x95, 0x30, 0xef, 0x07, 0x44, 0xec, 0xd0, 0x98, 0x98, 0xc0, +0x6b, 0x71, 0x5b, 0x23, 0xb8, 0xb6, 0xd2, 0x21, 0xff, 0x51, 0xdd, 0xae, 0x48, 0x29, 0x75, 0x0c, +0xc3, 0x3d, 0x91, 0xfe, 0x9d, 0xa8, 0x5e, 0xb2, 0x34, 0xb2, 0xd3, 0x81, 0xf6, 0x27, 0x9c, 0xac, +0x6b, 0x20, 0x56, 0x86, 0xa5, 0x4f, 0x7a, 0xdb, 0xf9, 0xac, 0xa9, 0x8e, 0xe3, 0x73, 0x21, 0x99, +0x71, 0x2d, 0xaf, 0x27, 0x92, 0x0c, 0xc7, 0xd3, 0x85, 0xb3, 0x40, 0xda, 0x13, 0x4a, 0x04, 0x41, +0x54, 0xf8, 0xf2, 0x55, 0xb7, 0x80, 0xdd, 0x77, 0xba, 0x01, 0x7a, 0x31, 0xbd, 0x6b, 0xdc, 0x5c, +0x59, 0xf4, 0x2b, 0xca, 0x25, 0xbb, 0x50, 0xba, 0xfa, 0x42, 0x38, 0xd2, 0x28, 0x10, 0x8b, 0x7b, +0x96, 0x45, 0x30, 0xbb, 0x7f, 0xf4, 0x5a, 0xf7, 0x28, 0x6f, 0x47, 0xdc, 0xd2, 0x82, 0xf2, 0xf7, +0xdd, 0x20, 0xb5, 0x0c, 0x7e, 0x53, 0x85, 0xa7, 0xfc, 0x3b, 0x1a, 0xc0, 0x07, 0x7b, 0xa1, 0x43, +0x05, 0x18, 0x19, 0xd3, 0xfc, 0x41, 0xc2, 0xce, 0xd9, 0x5b, 0x4b, 0x63, 0xe2, 0x8f, 0x86, 0x3a, +0xd1, 0xd0, 0x1d, 0x74, 0x2e, 0xbc, 0xd3, 0xce, 0x08, 0x0c, 0x10, 0x7a, 0x42, 0x60, 0xc5, 0x3a, +0xa6, 0xd8, 0xb0, 0x52, 0xcf, 0x53, 0x28, 0x70, 0x45, 0xb7, 0x72, 0x7d, 0x77, 0x66, 0x54, 0x3d, +0x38, 0x26, 0xcf, 0xd5, 0xbf, 0xe4, 0x80, 0x10, 0xba, 0x2b, 0xe8, 0xdc, 0xc3, 0xfe, 0x28, 0xa3, +0x52, 0x58, 0x70, 0x4a, 0xde, 0x84, 0x33, 0x5e, 0x93, 0x04, 0xa4, 0x7c, 0xe7, 0xea, 0x8e, 0xba, +0xeb, 0x8a, 0x19, 0x26, 0x6a, 0x7f, 0x7c, 0x4a, 0x5b, 0xb4, 0x0d, 0xfc, 0xc8, 0x11, 0x1b, 0x41, +0x68, 0x5d, 0x2a, 0x25, 0x04, 0x4f, 0xc8, 0xf4, 0x65, 0xfc, 0xb9, 0x58, 0xeb, 0xb4, 0x67, 0x50, +0x24, 0xf5, 0x43, 0xf6, 0x91, 0x4a, 0xb0, 0x0f, 0x32, 0xe0, 0x07, 0x75, 0x69, 0x1b, 0x3c, 0xeb, +0xb2, 0x65, 0x26, 0x6f, 0xb8, 0x79, 0xe0, 0x78, 0x8c, 0xdc, 0x39, 0x24, 0x48, 0x76, 0x11, 0xd4, +0x3a, 0xc5, 0xd2, 0x2b, 0xaa, 0x55, 0xfb, 0x92, 0x12, 0x2d, 0x88, 0x05, 0xd1, 0xb1, 0x31, 0x36, +0x1f, 0xc2, 0x44, 0x1c, 0xab, 0x2e, 0xcd, 0x1c, 0x72, 0x86, 0xf6, 0x83, 0x87, 0x2e, 0x8b, 0xdb, +0xaa, 0x16, 0x0e, 0x1b, 0xe6, 0x5c, 0x4d, 0x2f, 0x82, 0xbd, 0x49, 0x11, 0x60, 0x22, 0x0f, 0xde, +0x3b, 0x2b, 0x20, 0x1d, 0x56, 0xb7, 0x21, 0xae, 0x0b, 0x26, 0x4f, 0xde, 0x3d, 0xa6, 0x3f, 0x61, +0x81, 0xe2, 0x76, 0x60, 0x08, 0xc5, 0x4b, 0x18, 0x0b, 0xd1, 0xf5, 0xff, 0x8d, 0x1a, 0x96, 0x76, +0x51, 0x15, 0x05, 0x4d, 0x8c, 0x6b, 0x12, 0x90, 0x47, 0xd4, 0xa4, 0x38, 0xb9, 0x48, 0xe4, 0x4c, +0x05, 0x69, 0x6a, 0x8b, 0x9d, 0x7c, 0xa1, 0xbc, 0x77, 0xeb, 0x86, 0x93, 0x0a, 0x15, 0x84, 0xba, +0x8f, 0xf5, 0x7c, 0x44, 0x75, 0x31, 0x79, 0x16, 0xc1, 0x81, 0x1a, 0xb6, 0xe6, 0x6c, 0x3d, 0xb8, +0x15, 0x46, 0xf5, 0xbe, 0x46, 0x04, 0xa6, 0xec, 0xec, 0xd1, 0x74, 0x8b, 0x87, 0x2b, 0xdb, 0xd0, +0x9f, 0xb3, 0x99, 0x9d, 0x87, 0x8c, 0xc6, 0xaa, 0xd4, 0x64, 0x45, 0xbd, 0xe8, 0xed, 0xa3, 0xc1, +0x2a, 0x41, 0x1e, 0x26, 0xaf, 0x86, 0x16, 0xed, 0x80, 0x08, 0xca, 0x64, 0x21, 0x3a, 0xce, 0x21, +0x4c, 0x41, 0xb9, 0x13, 0x2d, 0xf7, 0x1b, 0xdf, 0x2b, 0x33, 0x69, 0xe7, 0x5c, 0x8c, 0x7b, 0xfb, +0xe3, 0x41, 0xe9, 0xce, 0xd7, 0xff, 0x0e, 0x54, 0xfe, 0xb0, 0x71, 0x78, 0xdc, 0xde, 0x7e, 0xdd, +0x1f, 0x1c, 0x4a, 0x8f, 0x3e, 0x16, 0xfd, 0x91, 0x82, 0x94, 0xd4, 0xc2, 0xf7, 0xb2, 0x77, 0x89, +0x16, 0x2c, 0xba, 0xb6, 0xbd, 0xed, 0x95, 0x43, 0x05, 0x9b, 0xf2, 0xc4, 0xbe, 0x46, 0x43, 0x90, +0x1d, 0xd8, 0x24, 0x02, 0xd2, 0xea, 0xf4, 0x08, 0xd9, 0xf7, 0x84, 0x0e, 0xc6, 0xe7, 0x44, 0xdb, +0xb8, 0xac, 0x0a, 0x53, 0x39, 0x61, 0x43, 0xdc, 0x22, 0x28, 0x8f, 0x22, 0x2f, 0x73, 0xbf, 0x59, +0x2d, 0x3c, 0x8c, 0x0b, 0xcc, 0x2a, 0x67, 0xe0, 0x5b, 0x5c, 0x65, 0x5e, 0x6d, 0x98, 0x99, 0xaa, +0x3b, 0x89, 0x12, 0xe2, 0x99, 0xf6, 0x15, 0xa7, 0xd2, 0x6a, 0x79, 0xb4, 0xf6, 0x0b, 0xf5, 0x0d, +0x2d, 0x4c, 0xcb, 0x1b, 0x35, 0x93, 0x61, 0x32, 0xa1, 0x8a, 0xa8, 0x27, 0xe8, 0x95, 0x5a, 0x56, +0x59, 0x04, 0xfe, 0xce, 0xc2, 0xd8, 0x92, 0x97, 0xb2, 0x54, 0x63, 0xd0, 0x3b, 0xde, 0x10, 0x34, +0x32, 0x16, 0x05, 0x51, 0x1d, 0xfc, 0x96, 0x8e, 0xf1, 0xf6, 0x4b, 0xd7, 0x48, 0x22, 0xce, 0xca, +0x1c, 0x6b, 0xab, 0x1f, 0x59, 0xa2, 0x74, 0xd6, 0xcd, 0x15, 0x07, 0xab, 0xa2, 0xd5, 0x22, 0x81, +0xec, 0x20, 0x14, 0x36, 0xac, 0xe4, 0x25, 0x7d, 0xe6, 0x09, 0x00, 0x2c, 0x92, 0x4d, 0x4e, 0xbf, +0xbf, 0xa1, 0xd4, 0xbe, 0x6b, 0xd4, 0x1f, 0x95, 0x9b, 0xf3, 0xda, 0x99, 0xad, 0xa4, 0x6c, 0x73, +0x55, 0xd1, 0x9d, 0x4b, 0x16, 0xd4, 0x06, 0xec, 0x46, 0x3d, 0xb7, 0xe7, 0xce, 0xd0, 0x1d, 0x94, +0x65, 0xde, 0x61, 0xb3, 0xc1, 0x10, 0x65, 0xe5, 0x68, 0x9b, 0xb0, 0xb4, 0x43, 0x0b, 0x92, 0xaf, +0xb7, 0x40, 0xa2, 0xe5, 0x06, 0x3d, 0x72, 0x00, 0xc5, 0x39, 0xab, 0x35, 0x29, 0x22, 0x4c, 0xa5, +0xa5, 0x3f, 0x22, 0x90, 0x53, 0xd2, 0x36, 0x63, 0x1e, 0xd3, 0x33, 0xa5, 0xbb, 0x3d, 0xa3, 0x0c, +0x14, 0x9c, 0x2e, 0x6d, 0x9a, 0x7a, 0xf7, 0xf1, 0x56, 0x66, 0xe5, 0x8d, 0x53, 0x83, 0x34, 0x3f, +0xa9, 0x83, 0x84, 0x68, 0x90, 0xc9, 0x51, 0xc2, 0xd4, 0x8e, 0x6c, 0xc7, 0x6d, 0xa7, 0x19, 0x61, +0xa7, 0x2e, 0x36, 0xbc, 0xd2, 0x0f, 0x17, 0x49, 0xd4, 0x6b, 0x36, 0x63, 0xfb, 0x1d, 0xf4, 0xb0, +0x6b, 0xcf, 0x34, 0x5f, 0xd2, 0x77, 0xae, 0x12, 0xaf, 0xb3, 0xdf, 0x52, 0xf7, 0xc2, 0xc8, 0xf2, +0x63, 0x61, 0xb6, 0x3e, 0x39, 0xf2, 0xa7, 0x1a, 0x89, 0x9d, 0x0e, 0x8f, 0xaf, 0xe1, 0x01, 0x24, +0xa6, 0x3a, 0xd5, 0x9a, 0x62, 0x67, 0xa3, 0x66, 0xee, 0xbc, 0xc5, 0x94, 0x4b, 0xc3, 0x15, 0xa1, +0x7e, 0x07, 0x07, 0x2b, 0xb7, 0x43, 0x2a, 0xb4, 0xb8, 0x25, 0x88, 0x86, 0x23, 0xab, 0xdf, 0x05, +0xbe, 0x46, 0x56, 0xd7, 0xda, 0xd6, 0x75, 0x53, 0xd9, 0xc8, 0x26, 0x8f, 0x39, 0x67, 0xed, 0x21, +0x53, 0x1c, 0x9c, 0x89, 0x46, 0xd3, 0xfe, 0x54, 0xe6, 0x1d, 0x02, 0xb9, 0x25, 0x82, 0x66, 0xe6, +0xf9, 0x45, 0xd9, 0x3f, 0xa5, 0x71, 0xc1, 0x46, 0x66, 0x7a, 0x27, 0x8a, 0x82, 0xc9, 0x21, 0xe9, +0x17, 0xab, 0x6c, 0xef, 0x45, 0xe5, 0x88, 0x93, 0x87, 0x80, 0xb3, 0x85, 0x25, 0x96, 0x19, 0x41, +0xab, 0xd6, 0xba, 0x92, 0x76, 0x21, 0x8a, 0x58, 0xbd, 0xe2, 0x4b, 0xec, 0x45, 0x59, 0x2c, 0x13, +0x1a, 0xb5, 0x13, 0x25, 0x44, 0xe7, 0x71, 0x26, 0x0a, 0x34, 0x33, 0xb9, 0x57, 0x15, 0xa4, 0x90, +0x60, 0x11, 0x05, 0x8e, 0xc8, 0x8e, 0x74, 0x52, 0x4b, 0x31, 0x71, 0xeb, 0x66, 0x7e, 0xee, 0xb1, +0x0a, 0x21, 0x52, 0xc0, 0x1a, 0xe9, 0xa1, 0x5a, 0xe3, 0x3a, 0x24, 0xfb, 0xf3, 0x1e, 0xd6, 0x83, +0x1d, 0xfb, 0x81, 0xa8, 0x91, 0x60, 0x9e, 0xbc, 0x59, 0x20, 0xc9, 0x9e, 0x71, 0x19, 0x83, 0x2b, +0x6a, 0x48, 0x4e, 0x6b, 0x46, 0x82, 0x89, 0xda, 0x60, 0xff, 0x1a, 0x46, 0x94, 0x55, 0xda, 0xe5, +0x99, 0xfa, 0x84, 0xd7, 0x3b, 0xb9, 0xa5, 0x34, 0x87, 0x86, 0x5e, 0x6d, 0x75, 0x9a, 0xe7, 0x09, +0xb8, 0xe6, 0x71, 0x15, 0x10, 0x56, 0xd7, 0xc1, 0xc8, 0xb2, 0x62, 0xbc, 0xec, 0xe0, 0x94, 0xa0, +0xcd, 0xb4, 0x04, 0xa9, 0xc3, 0x51, 0xee, 0xf8, 0x2e, 0x42, 0x9a, 0xaa, 0x34, 0xd3, 0xb9, 0xb0, +0x36, 0xf9, 0x47, 0xc1, 0x07, 0x49, 0xde, 0xb8, 0x32, 0x8a, 0x87, 0x68, 0x56, 0x9a, 0x35, 0x79, +0xd1, 0xac, 0x49, 0x38, 0xc6, 0xfe, 0xfd, 0xdf, 0x6f, 0x3c, 0xda, 0x48, 0xbd, 0x23, 0xfd, 0x85, +0xf0, 0x96, 0xee, 0x1c, 0x27, 0x18, 0x86, 0xa6, 0xf0, 0x7b, 0xd8, 0x3c, 0xc7, 0x22, 0x3e, 0x2f, +0xac, 0xb1, 0x37, 0xbd, 0x84, 0x4b, 0xe3, 0x92, 0x82, 0xd0, 0x25, 0x14, 0x22, 0x65, 0xed, 0xeb, +0xef, 0xb9, 0xb6, 0xe4, 0x95, 0x18, 0x0d, 0x2b, 0x8d, 0x4f, 0xaf, 0xc0, 0xa0, 0x05, 0x8b, 0x35, +0x5b, 0x94, 0xb2, 0x68, 0x26, 0x4f, 0x4a, 0x9e, 0x85, 0x0e, 0x46, 0xe0, 0x4f, 0x60, 0x66, 0x01, +0xa4, 0x39, 0xe8, 0x8b, 0x2a, 0x50, 0xf5, 0x18, 0x70, 0xe2, 0xfc, 0xd6, 0xbe, 0xd3, 0x46, 0x4b + +ciphertext = +0x77, 0xF9, 0xF7, 0x7A, 0xA3, 0xCB, 0x68, 0x1A, 0x11, 0x70, 0xD8, 0x7A, 0xB6, 0xE2, 0x37, 0x7E, +0xD1, 0x57, 0x1C, 0x8E, 0x85, 0xD8, 0x08, 0xBF, 0x57, 0x1F, 0x21, 0x6C, 0xAD, 0xAD, 0x47, 0x1E, +0x0D, 0x6B, 0x79, 0x39, 0x15, 0x4E, 0x5B, 0x59, 0x2D, 0x76, 0x87, 0xA6, 0xD6, 0x47, 0x8F, 0x82, +0xB8, 0x51, 0x91, 0x32, 0x60, 0xCB, 0x97, 0xDE, 0xBE, 0xF0, 0xAD, 0xFC, 0x23, 0x2E, 0x22, 0x02, +0x46, 0x17, 0x3F, 0x8F, 0x24, 0x0E, 0x31, 0x80, 0xEA, 0xD6, 0x85, 0x50, 0xA5, 0xEE, 0xB7, 0x15, +0xD0, 0x2F, 0xA6, 0x92, 0xEF, 0xCD, 0x8B, 0x91, 0x4A, 0xEA, 0x03, 0x92, 0xB4, 0x65, 0x19, 0x66, +0x9E, 0x73, 0x79, 0xCE, 0xA7, 0x4D, 0x8B, 0x77, 0x74, 0x44, 0x1C, 0x9F, 0xEE, 0x15, 0x91, 0xF2, +0xF9, 0x50, 0x7D, 0x2A, 0xC2, 0x6B, 0x58, 0x36, 0xF7, 0x62, 0x25, 0x9C, 0x71, 0x34, 0x43, 0x14, +0x9E, 0x17, 0xF7, 0xB7, 0x56, 0x1D, 0x91, 0x4C, 0xF6, 0x6C, 0xF2, 0x19, 0x39, 0xA2, 0x29, 0xA3, +0x22, 0x4F, 0x14, 0x18, 0x76, 0x8A, 0x59, 0xAD, 0x3E, 0x5F, 0xDC, 0xC9, 0x80, 0x07, 0x51, 0xB2, +0x90, 0x6A, 0xB9, 0x0C, 0xA4, 0x3F, 0x42, 0xBD, 0x40, 0xB7, 0xA7, 0xF5, 0x85, 0xBF, 0xEA, 0xD8, +0x89, 0xA9, 0xE9, 0xC7, 0x25, 0xEC, 0xFF, 0x80, 0xE8, 0x94, 0x3B, 0xE7, 0xD1, 0x68, 0xDA, 0x1C, +0xFA, 0x5D, 0xCD, 0x68, 0x09, 0x72, 0x63, 0xBA, 0x34, 0x56, 0xD4, 0x5F, 0xB0, 0xA7, 0xAE, 0xCF, +0xFB, 0xA8, 0xBD, 0x52, 0x79, 0x98, 0xF6, 0x39, 0x52, 0xD3, 0xA7, 0xE1, 0xFB, 0x75, 0x76, 0x87, +0xBC, 0x11, 0x18, 0x17, 0x47, 0x65, 0xDA, 0xE3, 0x25, 0x3E, 0x17, 0x43, 0x7B, 0x0D, 0x8B, 0x7F, +0x20, 0xFF, 0x03, 0xFA, 0x28, 0xC7, 0xD3, 0xF8, 0xC2, 0xA8, 0xC1, 0xE5, 0xDA, 0x77, 0x41, 0x28, +0x06, 0xC4, 0x20, 0xFC, 0x1B, 0xAA, 0x99, 0x78, 0x5C, 0x28, 0xDA, 0x9A, 0x2B, 0x6C, 0x56, 0x7E, +0x63, 0x34, 0xCF, 0xCD, 0x5D, 0xB6, 0x13, 0xF5, 0x98, 0x08, 0x2E, 0x02, 0x2C, 0x63, 0xEC, 0xE3, +0x43, 0xE8, 0x3B, 0xE6, 0x59, 0x8C, 0x61, 0x60, 0xDD, 0x33, 0x3F, 0x29, 0xA4, 0xA5, 0xD5, 0x33, +0xEF, 0xAA, 0x7E, 0x8C, 0xAE, 0x9C, 0x1B, 0x0D, 0x74, 0xF6, 0x01, 0x8C, 0xF1, 0x04, 0xEB, 0x62, +0x75, 0xC0, 0x98, 0x24, 0xB2, 0xDF, 0xB1, 0xBA, 0x50, 0xC3, 0x01, 0x5B, 0x13, 0x3A, 0xF9, 0x7A, +0xF6, 0xF4, 0x75, 0xAF, 0x55, 0x54, 0x10, 0xBE, 0x11, 0x91, 0x7D, 0xF6, 0x66, 0x79, 0xE6, 0x4D, +0x0E, 0x0B, 0x70, 0x3C, 0x00, 0x68, 0x2E, 0xA3, 0x84, 0xCE, 0xE1, 0x0A, 0xDC, 0xFE, 0xF9, 0xD2, +0x59, 0x23, 0x05, 0xCA, 0x79, 0xF0, 0x89, 0xB9, 0x76, 0xD9, 0xAA, 0x04, 0x43, 0x30, 0x97, 0x15, +0x59, 0x0B, 0x7C, 0x22, 0x0E, 0x72, 0xC6, 0x61, 0x19, 0x35, 0xC3, 0x6A, 0xF2, 0x6B, 0x39, 0xB6, +0x1B, 0xD3, 0xE3, 0xF7, 0xCB, 0x46, 0x26, 0x97, 0x39, 0xBA, 0x41, 0xD8, 0xA4, 0x48, 0x96, 0xBC, +0x22, 0x38, 0xCF, 0xE2, 0xCF, 0xD6, 0x36, 0x30, 0xD9, 0x96, 0x73, 0xAF, 0xA4, 0x0F, 0x52, 0x9D, +0x64, 0x28, 0xAB, 0x3D, 0xF7, 0x1B, 0xA6, 0xDB, 0x47, 0x09, 0x45, 0x48, 0x30, 0x27, 0x4B, 0x37, +0x38, 0x5B, 0xC5, 0x90, 0x8C, 0xCC, 0x82, 0x48, 0x7A, 0x98, 0x1C, 0x46, 0x24, 0xB1, 0xDA, 0xB9, +0x6C, 0x30, 0x48, 0xF3, 0x6C, 0x84, 0xBD, 0x3F, 0x95, 0x3E, 0xC1, 0x27, 0x8B, 0x3C, 0x2F, 0xDA, +0xD9, 0xF6, 0x54, 0x73, 0x04, 0x38, 0xD6, 0x45, 0xC5, 0x5C, 0x92, 0xDE, 0xB2, 0xE3, 0x62, 0x31, +0xCE, 0x70, 0xD7, 0x11, 0x5E, 0x7A, 0x63, 0x0F, 0xA1, 0xD4, 0x8A, 0x2B, 0xDE, 0x38, 0xAA, 0x9F, +0x33, 0x71, 0x67, 0x05, 0xDB, 0x48, 0xE4, 0x09, 0x73, 0x3A, 0x35, 0x1F, 0xC2, 0x0F, 0x44, 0x99, +0x35, 0xBD, 0xBD, 0x7E, 0x85, 0x77, 0x46, 0x3A, 0x41, 0x79, 0xAB, 0x67, 0xA5, 0x87, 0xBD, 0x96, +0xAE, 0xC2, 0x99, 0x35, 0xC3, 0xCA, 0x90, 0x36, 0xB1, 0x15, 0x9C, 0x37, 0x62, 0x54, 0xCA, 0x72, +0x10, 0x07, 0x07, 0x6E, 0x1D, 0xD0, 0xFE, 0x4C, 0xE8, 0x48, 0x92, 0x7A, 0xA1, 0x7B, 0xA5, 0xAC, +0xF7, 0xE1, 0x99, 0xC0, 0x99, 0x20, 0xD5, 0x07, 0x77, 0x1D, 0xE5, 0x14, 0x36, 0xA6, 0xF3, 0x77, +0x9B, 0x61, 0x87, 0x98, 0xD6, 0xD6, 0xF8, 0xE6, 0x34, 0x37, 0x6F, 0x58, 0x29, 0x97, 0x2D, 0xE6, +0xD1, 0x56, 0xB1, 0xBB, 0x35, 0xBD, 0x2B, 0x44, 0xAD, 0x30, 0x0F, 0x1D, 0x48, 0x5F, 0xDD, 0x58, +0x7C, 0xB8, 0x2C, 0x2E, 0x26, 0x9B, 0xDA, 0x55, 0x01, 0x06, 0x66, 0xB0, 0x3C, 0xAB, 0xA0, 0x60, +0x98, 0xF4, 0x72, 0xAF, 0xBC, 0x00, 0xAA, 0x57, 0x6A, 0xD8, 0x47, 0xC7, 0xC1, 0xCE, 0xB1, 0x05, +0x45, 0x84, 0x63, 0x1E, 0x9C, 0x47, 0x11, 0xB4, 0xE6, 0x80, 0x8D, 0x3E, 0xFF, 0xE9, 0xD9, 0x3A, +0x56, 0x3D, 0x41, 0x68, 0x2C, 0x6C, 0xA2, 0x23, 0x4C, 0xD6, 0x08, 0x91, 0x93, 0xCD, 0xAA, 0xF7, +0xAA, 0x2B, 0x55, 0xEC, 0x53, 0xE8, 0xD9, 0x2E, 0xCB, 0xE0, 0x67, 0x1D, 0x9F, 0xFF, 0x94, 0xB8, +0xBA, 0x82, 0xA7, 0x6A, 0x8C, 0x61, 0x7C, 0x58, 0x90, 0xA5, 0x11, 0x57, 0x21, 0xCF, 0x30, 0xB0, +0x97, 0x44, 0x7D, 0x1D, 0xD3, 0x91, 0x3F, 0xC2, 0x4F, 0x0E, 0x3B, 0x57, 0x3A, 0x1F, 0x85, 0x82, +0x79, 0x91, 0x03, 0xB4, 0x9B, 0x70, 0x2A, 0x5F, 0x8B, 0x20, 0x66, 0x6F, 0xF4, 0x10, 0x96, 0x52, +0x4C, 0x77, 0xA2, 0x45, 0x28, 0xF8, 0xAD, 0xA3, 0x8C, 0x99, 0x3F, 0xD2, 0x39, 0x4A, 0x1A, 0x3A, +0x72, 0x7E, 0x47, 0x49, 0x25, 0x63, 0x87, 0xCB, 0xEA, 0x89, 0x1D, 0x7F, 0x0C, 0x86, 0x9A, 0x8E, +0xB1, 0x0C, 0xFF, 0xC6, 0xF2, 0xB1, 0x01, 0x99, 0xEA, 0xF1, 0x4A, 0xF1, 0xF3, 0x71, 0x4B, 0x92, +0xC6, 0xD6, 0xD8, 0x26, 0xE8, 0x28, 0xF2, 0xF5, 0x5B, 0xE8, 0xF1, 0xE4, 0x4B, 0x36, 0x46, 0xD3, +0x12, 0x2F, 0x98, 0x61, 0x12, 0xD9, 0x26, 0x58, 0x5C, 0x80, 0x7C, 0x71, 0x4E, 0x57, 0x9A, 0xAC, +0x59, 0xE0, 0xC3, 0x70, 0x55, 0x57, 0xAE, 0x55, 0xF6, 0xCF, 0x6A, 0xF0, 0x10, 0xDC, 0xF4, 0xED, +0xCC, 0x32, 0x4B, 0xAC, 0xC1, 0x4B, 0x2F, 0x62, 0x69, 0xD2, 0x15, 0x63, 0x39, 0xD5, 0x29, 0x09, +0xA2, 0xB5, 0xC7, 0xBC, 0xFA, 0xC7, 0xC7, 0x8C, 0x64, 0xCF, 0x43, 0x9B, 0x4C, 0x60, 0x97, 0x33, +0xA2, 0xB9, 0x0F, 0x70, 0x05, 0x89, 0x56, 0x62, 0xB1, 0x48, 0x08, 0xB5, 0x77, 0x4C, 0x60, 0x24, +0x1D, 0x35, 0xEF, 0xD6, 0x53, 0xB0, 0x2E, 0x7F, 0xA6, 0x4B, 0x94, 0xE7, 0xCD, 0xC4, 0xFE, 0xC4, +0x12, 0x7A, 0xAB, 0xD4, 0x05, 0xA5, 0x32, 0xD4, 0xA1, 0x8D, 0x9C, 0x22, 0x10, 0xDD, 0x39, 0x66, +0x96, 0x79, 0x49, 0x19, 0x80, 0x1C, 0xE1, 0x1F, 0x01, 0x69, 0x37, 0x03, 0xB5, 0x22, 0x8F, 0x95, +0xF7, 0xBD, 0x36, 0x89, 0x38, 0x37, 0x29, 0x6C, 0x0E, 0x89, 0x55, 0x4D, 0xC9, 0x64, 0xD3, 0xD5, +0x9B, 0xB0, 0x51, 0x43, 0xBB, 0xA6, 0x6B, 0xFF, 0x13, 0xB6, 0x1A, 0x06, 0xF3, 0x86, 0x51, 0xFD, +0xB9, 0xC8, 0x26, 0xA9, 0x8A, 0x4A, 0xC1, 0xE0, 0xD9, 0x3D, 0x31, 0x48, 0x39, 0xC8, 0x48, 0xC7, +0xDE, 0xB1, 0x58, 0x0F, 0x4D, 0xEC, 0x5B, 0x32, 0x9C, 0x8B, 0xF4, 0x3A, 0x02, 0xE2, 0x92, 0x4A, +0x7D, 0xCD, 0x38, 0x07, 0x4F, 0xBA, 0xD1, 0xD4, 0xE4, 0x3C, 0xB0, 0x4D, 0xB7, 0xEF, 0xFB, 0x06, +0xA9, 0x83, 0x20, 0x0D, 0x7A, 0x1F, 0x15, 0x02, 0x70, 0x08, 0x8B, 0x91, 0xE6, 0xFD, 0x8C, 0x0B, +0x3C, 0xEA, 0x1F, 0x94, 0xB6, 0x17, 0xC6, 0xB2, 0x07, 0x2C, 0x73, 0x7A, 0x4A, 0x76, 0x5F, 0x30, +0x38, 0xE5, 0x22, 0xC6, 0xA3, 0xA7, 0x4D, 0x6A, 0x3A, 0xA7, 0x82, 0x90, 0xBE, 0xD1, 0xE9, 0x89, +0x2F, 0xF0, 0xC9, 0x0A, 0xB6, 0xDA, 0x0D, 0x3E, 0x25, 0x8E, 0x99, 0xB2, 0x06, 0xE3, 0x65, 0x53, +0x3F, 0x1A, 0xD9, 0x45, 0xCE, 0x10, 0xBE, 0x2E, 0xF4, 0x4F, 0x60, 0x25, 0xA7, 0x0A, 0xAE, 0x82, +0x92, 0xAE, 0xC0, 0xFF, 0xAB, 0x49, 0x97, 0x5C, 0x53, 0x73, 0x4E, 0x78, 0x1A, 0x65, 0x42, 0xD5, +0x6F, 0x1E, 0xE2, 0x25, 0x76, 0x3B, 0x6D, 0xF8, 0xBC, 0xBD, 0x3A, 0xDE, 0xB5, 0xFB, 0xBD, 0x90, +0xDC, 0xC2, 0xB8, 0x90, 0xD4, 0x03, 0xD2, 0xDD, 0x35, 0x86, 0x48, 0x58, 0xB4, 0xCB, 0x10, 0xB2, +0x31, 0xBD, 0x6C, 0x16, 0x92, 0x7A, 0x3D, 0x67, 0x45, 0x6B, 0x57, 0x26, 0xD2, 0xC2, 0xAF, 0xB1, +0xAB, 0x82, 0x4B, 0x95, 0x08, 0x7D, 0x48, 0x1D, 0x17, 0x9D, 0x8B, 0x16, 0xCF, 0xE0, 0x16, 0x94, +0xE1, 0xA6, 0xFC, 0x6C, 0xE1, 0x71, 0x3C, 0x57, 0x7F, 0x17, 0xC8, 0x4E, 0xFF, 0x16, 0x46, 0x1E, +0x21, 0x27, 0x05, 0x41, 0xD3, 0x19, 0x28, 0x58, 0x86, 0xFB, 0x5A, 0xEF, 0xC3, 0x00, 0xE7, 0xA3, +0x25, 0x1A, 0x94, 0x41, 0xE3, 0x50, 0x98, 0x94, 0x29, 0x42, 0x1F, 0x1C, 0x69, 0x46, 0xF4, 0x89, +0x30, 0x4E, 0x5C, 0xCE, 0x2F, 0x65, 0xC5, 0x34, 0x71, 0xB7, 0xD9, 0x54, 0xB2, 0xC1, 0xCC, 0xED, +0x14, 0x3E, 0xF1, 0x7B, 0x5F, 0xAE, 0xD3, 0x8F, 0xA2, 0x18, 0x12, 0x15, 0x23, 0x92, 0x75, 0x61, +0xFF, 0xFA, 0x8F, 0xD1, 0x77, 0xC8, 0xC7, 0xA3, 0x44, 0x9F, 0x06, 0x2B, 0x1E, 0xA4, 0x4D, 0x4F, +0x8E, 0x9A, 0x02, 0xA8, 0x4A, 0x67, 0x5D, 0x2D, 0x59, 0xFD, 0x1A, 0x8F, 0xE6, 0x52, 0x0C, 0xC7, +0x4A, 0x95, 0xAF, 0xDD, 0x04, 0x76, 0x26, 0xCE, 0x4C, 0x97, 0x4E, 0x55, 0x9C, 0x28, 0xA4, 0x1D, +0x92, 0xD6, 0x84, 0x87, 0x29, 0x28, 0x16, 0x1B, 0x34, 0xE3, 0xBD, 0x2F, 0x9B, 0xF8, 0x6F, 0xDC, +0x9B, 0x6C, 0xF5, 0xEB, 0x26, 0x51, 0x47, 0x78, 0xA2, 0xB5, 0x4C, 0x24, 0x1E, 0x3D, 0xE5, 0x33, +0xA3, 0xD9, 0x04, 0x20, 0x8E, 0xA7, 0x32, 0x88, 0xC6, 0x52, 0x0B, 0x71, 0x0D, 0x26, 0xC3, 0x3F, +0xC4, 0xC8, 0x7F, 0x6F, 0x3A, 0xAD, 0xC7, 0x27, 0x3D, 0xB3, 0xE6, 0x6B, 0x68, 0x66, 0xB3, 0xEE, +0x6D, 0xC7, 0xAB, 0xD4, 0xA2, 0x88, 0xAF, 0xEB, 0x1A, 0x51, 0x76, 0x19, 0xFC, 0xF7, 0x29, 0xF0, +0x4D, 0xC5, 0xAB, 0x42, 0x81, 0x9F, 0x10, 0xD9, 0xB0, 0x5C, 0x9D, 0x1A, 0x5A, 0xFE, 0xB3, 0x71, +0xBC, 0x13, 0x69, 0xDA, 0xCE, 0x15, 0x7C, 0x18, 0x2C, 0x81, 0xFC, 0xA9, 0x1E, 0x0B, 0x33, 0xBF, +0x82, 0x0D, 0xD5, 0x58, 0xD0, 0xB6, 0x17, 0x34, 0xFE, 0x53, 0x45, 0xE7, 0x57, 0x9B, 0xFA, 0x3C, +0x04, 0xCF, 0x89, 0x38, 0x73, 0xE9, 0x60, 0xEA, 0xF4, 0x0F, 0xB2, 0x2E, 0x90, 0x60, 0xAE, 0xFB, +0x57, 0xCB, 0xA5, 0x9D, 0x60, 0x44, 0x46, 0x13, 0x3C, 0xB3, 0xB6, 0x0A, 0x09, 0x12, 0x2B, 0x27, +0x95, 0x45, 0x29, 0x92, 0x86, 0x00, 0x2A, 0x93, 0x77, 0x8D, 0xAA, 0xC3, 0xF8, 0x46, 0xBE, 0x3A, +0x6A, 0x0E, 0x51, 0x9D, 0x94, 0x60, 0x9A, 0x76, 0x93, 0xF4, 0x01, 0x19, 0xC3, 0xB1, 0x86, 0xA9, +0x7E, 0xD1, 0xF6, 0xF1, 0x88, 0x59, 0x4E, 0x9F, 0xCC, 0xF2, 0xF7, 0xDD, 0x1B, 0x91, 0x98, 0xAC, +0xCC, 0xC6, 0x81, 0x57, 0x3F, 0x07, 0xF2, 0x52, 0x5B, 0x79, 0x5D, 0xFB, 0x07, 0xF7, 0x6A, 0x62, +0x30, 0xE5, 0x77, 0x81, 0x00, 0x6C, 0xB1, 0x11, 0x8A, 0x1D, 0x0C, 0x9C, 0x94, 0x1A, 0xAD, 0xB6, +0x85, 0x29, 0x70, 0x19, 0xFB, 0xE1, 0xF5, 0x89, 0x6D, 0xB3, 0x84, 0xC5, 0x56, 0x14, 0x1E, 0x67, +0x46, 0x57, 0xFE, 0x30, 0xD0, 0x81, 0x2B, 0x27, 0xD6, 0x4B, 0x41, 0x74, 0xF3, 0x51, 0xD0, 0x78, +0xCE, 0x3A, 0x5C, 0x46, 0xCC, 0xCE, 0x19, 0xC9, 0xC3, 0x1A, 0x81, 0xF4, 0x62, 0x9A, 0x8B, 0xAD, +0x71, 0x9C, 0x3E, 0x5B, 0x23, 0xA7, 0x9F, 0x7E, 0x26, 0xDD, 0x21, 0xCC, 0x36, 0x75, 0x90, 0x09, +0x61, 0x0B, 0x85, 0xC1, 0x0A, 0xF4, 0x9D, 0x93, 0x9F, 0x5F, 0x73, 0x71, 0xAB, 0x2B, 0xFA, 0x5E, +0xD9, 0xA1, 0xF8, 0x7F, 0x0F, 0xD5, 0x07, 0x59, 0xB2, 0x4F, 0xF9, 0x71, 0xD4, 0x35, 0x3E, 0x5D, +0x85, 0x6A, 0x32, 0x76, 0xDB, 0xBE, 0xC5, 0xD4, 0x2B, 0xC5, 0x70, 0x95, 0x7C, 0x64, 0x04, 0x0E, +0xC0, 0x4E, 0x59, 0x76, 0x10, 0xBF, 0x93, 0xBE, 0xEC, 0x40, 0x2C, 0xDE, 0x2D, 0xE6, 0xD1, 0x77, +0xC7, 0x84, 0x4B, 0xD6, 0x1C, 0x9A, 0xA1, 0x93, 0xE4, 0x50, 0xA8, 0x1B, 0x73, 0x29, 0x92, 0xB0, +0x37, 0x83, 0x15, 0xE3, 0xB5, 0xCD, 0xD1, 0x47, 0x38, 0xD1, 0xB6, 0xB6, 0x04, 0x3D, 0x58, 0x28, +0xB1, 0xB5, 0x9E, 0xF3, 0x95, 0x12, 0x1A, 0xC2, 0xA1, 0x71, 0x72, 0x45, 0x35, 0x0F, 0xB8, 0xC4, +0xEF, 0xF7, 0xAD, 0xD6, 0x82, 0x6A, 0x6A, 0x9E, 0x0E, 0xEF, 0xAB, 0xAD, 0x9D, 0x8D, 0xE4, 0x77, +0xA1, 0x93, 0xAE, 0xE1, 0xBA, 0x0E, 0xAF, 0x83, 0xC4, 0x84, 0x19, 0x6E, 0x5B, 0x15, 0xD7, 0xAE, +0x33, 0xA4, 0x37, 0xE2, 0xA1, 0x18, 0x2A, 0x4A, 0x9C, 0x5E, 0x7C, 0x61, 0x70, 0x76, 0xE9, 0xE6, +0x0E, 0x11, 0xEE, 0x71, 0x45, 0xE0, 0x5E, 0x72, 0x3C, 0x88, 0x0C, 0x34, 0x34, 0x78, 0x39, 0xD7, +0xFB, 0x26, 0x14, 0x1B, 0xCE, 0xEE, 0x15, 0x3C, 0xA4, 0x3F, 0xD3, 0x2A, 0x7C, 0x66, 0x58, 0xDD, +0x56, 0x46, 0xAF, 0x14, 0x04, 0x35, 0x33, 0xD5, 0x83, 0xA0, 0x07, 0xE0, 0xC0, 0x4B, 0x9D, 0x36, +0xF0, 0x72, 0x90, 0x7D, 0xFC, 0x4B, 0x3B, 0xDD, 0x07, 0x5E, 0xCD, 0xBE, 0x0B, 0x30, 0x78, 0x8C, +0x9B, 0x4D, 0xFB, 0xB4, 0x95, 0xC4, 0xDE, 0x57, 0xB3, 0x07, 0xE6, 0x8F, 0x20, 0xE7, 0x54, 0x84, +0xC8, 0x35, 0x3B, 0x68, 0x15, 0x74, 0x0F, 0x6A, 0xAB, 0xCC, 0x3E, 0x90, 0x6B, 0x38, 0x0A, 0xA8, +0x5A, 0x3F, 0xF3, 0xAC, 0x27, 0x12, 0xFC, 0x04, 0xF6, 0x93, 0xB4, 0x84, 0xF2, 0x82, 0xED, 0xAE, +0xF9, 0x64, 0x53, 0x1F, 0x9A, 0x2F, 0xAD, 0xB7, 0x2A, 0x17, 0x60, 0xFC, 0xDB, 0x07, 0xB1, 0x01, +0xC9, 0xF8, 0x02, 0x5F, 0xF3, 0x5B, 0x5B, 0x90, 0xD4, 0x96, 0x92, 0x99, 0x36, 0x22, 0x53, 0xEA, +0x62, 0xAE, 0xB0, 0x22, 0x6A, 0xAB, 0x24, 0xCD, 0x19, 0xBB, 0x86, 0x54, 0x17, 0x0F, 0x9D, 0x1A, +0x4A, 0x3D, 0xE4, 0xF0, 0x0D, 0x03, 0xF2, 0x9A, 0x6D, 0x70, 0xEE, 0xA5, 0x51, 0x5F, 0xE8, 0x74, +0xC1, 0xAC, 0x4B, 0xC6, 0x1C, 0x58, 0x26, 0x8F, 0xBF, 0xE1, 0x1D, 0xDB, 0x2D, 0xCA, 0x7E, 0x56, +0xB9, 0x5E, 0x28, 0x4D, 0x63, 0x21, 0xDA, 0x20, 0xC5, 0xBB, 0xE3, 0x23, 0x92, 0x90, 0xB3, 0x2D, +0xCE, 0x5B, 0x97, 0xF1, 0x66, 0x4A, 0x1D, 0xD0, 0xA4, 0x9E, 0x72, 0xD5, 0x3C, 0xC8, 0x7C, 0xCF, +0x78, 0x1F, 0x5B, 0x34, 0x9B, 0xFF, 0x92, 0x71, 0xF5, 0x02, 0x0E, 0x01, 0xAC, 0x6A, 0x1E, 0xE0, +0x2D, 0x15, 0x05, 0x40, 0x37, 0xF1, 0x7B, 0x24, 0xD8, 0x92, 0x5B, 0xE9, 0xEB, 0xD1, 0x7F, 0xC1, +0xCE, 0x9C, 0xAA, 0x6A, 0x48, 0x38, 0x3A, 0xF5, 0x5A, 0x3F, 0x17, 0xFF, 0x45, 0x09, 0x1B, 0x40 + +cipher_key = +0xE4, 0x23, 0x33, 0x8A, 0x35, 0x64, 0x61, 0xE2, 0x49, 0x03, 0xDD, 0xC6, 0xB8, 0xCA, 0x55, 0x7A, +0xd0, 0xe7, 0x4b, 0xfb, 0x5d, 0xe5, 0x0c, 0xe7, 0x6f, 0x21, 0xb5, 0x52, 0x2a, 0xbb, 0xc7, 0xf7 + +auth_key = +0xaf, 0x96, 0x42, 0xf1, 0x8c, 0x50, 0xdc, 0x67, 0x1a, 0x43, 0x47, 0x62, 0xc7, 0x04, 0xab, 0x05, +0xf5, 0x0c, 0xe7, 0xa2, 0xa6, 0x23, 0xd5, 0x3d, 0x95, 0xd8, 0xcd, 0x86, 0x79, 0xf5, 0x01, 0x47, +0x4f, 0xf9, 0x1d, 0x9d, 0x36, 0xf7, 0x68, 0x1a, 0x64, 0x44, 0x58, 0x5d, 0xe5, 0x81, 0x15, 0x2a, +0x41, 0xe4, 0x0e, 0xaa, 0x1f, 0x04, 0x21, 0xff, 0x2c, 0xf3, 0x73, 0x2b, 0x48, 0x1e, 0xd2, 0xf7, +0xf6, 0xd9, 0xaf, 0xbf, 0x08, 0x3b, 0xbb, 0x19, 0x5f, 0xf6, 0x7d, 0x25, 0x85, 0xdf, 0x6b, 0x54, +0xd0, 0xe7, 0x4b, 0x9e, 0xc7, 0xef, 0xca, 0x48, 0x6f, 0x21, 0xd7, 0x51, 0xc8, 0x21, 0xc1, 0x15, +0xe8, 0x38, 0x36, 0x58, 0x39, 0xd9, 0x9a, 0xc5, 0xe7, 0x3b, 0xc4, 0x47, 0xe2, 0xbd, 0x80, 0x73, +0xf8, 0xd1, 0x9a, 0x5e, 0x4b, 0xfb, 0x52, 0x6b, 0x50, 0xaf, 0x8b, 0xb7, 0xb5, 0x2c, 0x52, 0x84 + +cipher_iv = +0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F + +#################### +# sha_hmac_buff_32 # +#################### +[sha1_hmac_buff_32] +digest = +0x36, 0xCA, 0x49, 0x6A, 0xE3, 0x54, 0xD8, 0x4F, 0x0B, 0x76, 0xD8, 0xAA, 0x78, 0xEB, 0x9D, 0x65, +0x2C, 0xCA, 0x1F, 0x97 + +[sha224_hmac_buff_32] +digest = +0x48, 0xC1, 0x45, 0x25, 0x29, 0xA0, 0x8B, 0x88, 0x72, 0x7A, 0xBC, 0x00, 0x94, 0x37, 0xE1, 0x22, +0xEB, 0xFA, 0x1B, 0x7D, 0x89, 0x81, 0x31, 0xC8, 0x64, 0x76, 0x55, 0xA4 + +[sha256_hmac_buff_32] +digest = +0x1C, 0xB2, 0x3D, 0xD1, 0xF9, 0xC7, 0x6C, 0x49, 0x2E, 0xDA, 0x94, 0x8B, 0xF1, 0xCF, 0x96, 0x43, +0x67, 0x50, 0x39, 0x76, 0xB5, 0xA1, 0xCE, 0xA1, 0xD7, 0x77, 0x10, 0x07, 0x43, 0x37, 0x05, 0xB4 + +[sha384_hmac_buff_32] +digest = +0x6C, 0xBD, 0x1E, 0x2E, 0x75, 0xA7, 0x2C, 0x98, 0xC4, 0x1E, 0x03, 0x4E, 0x39, 0x4B, 0x27, 0x41, +0xFB, 0xC6, 0x56, 0x87, 0x84, 0xEB, 0xFA, 0xB1, 0x20, 0x1F, 0x11, 0x81, 0x8D, 0xDC, 0xB6, 0xA7, +0xAD, 0x1F, 0xAC, 0xA9, 0x43, 0x1D, 0x2B, 0xEB, 0x5F, 0x27, 0xC6, 0x0F, 0x9E, 0xFB, 0x1E, 0xB1 + +[sha512_hmac_buff_32] +digest = +0xA4, 0x60, 0x7E, 0xBE, 0x5F, 0x47, 0x58, 0x3B, 0x41, 0x5F, 0x29, 0xDF, 0xE4, 0xD2, 0xFB, 0x30, +0xF0, 0x2B, 0x09, 0x4E, 0x09, 0x50, 0xEC, 0x1C, 0x0E, 0x34, 0x79, 0xAE, 0xD8, 0x6D, 0xAC, 0xB6, +0x9B, 0x7C, 0xB9, 0x06, 0xC2, 0x4A, 0x4E, 0x22, 0x14, 0x4D, 0x42, 0x46, 0x20, 0xE0, 0x6C, 0xEE, +0x2F, 0xE1, 0x23, 0xA2, 0x7A, 0x2F, 0xDB, 0xAF, 0x78, 0x75, 0x56, 0xF7, 0x3A, 0x5E, 0x74, 0xEF + +#################### +# sha_hmac_buff_64 # +#################### +[sha1_hmac_buff_64] +digest = +0xFC, 0x17, 0x7E, 0x0E, 0x52, 0x94, 0xE3, 0x27, 0xC0, 0x9B, 0x72, 0xAD, 0xC0, 0x5B, 0xCF, 0xFF, +0x65, 0x88, 0x43, 0xE7 + +[sha224_hmac_buff_64] +digest = +0xD7, 0x55, 0x25, 0xC0, 0x26, 0xDD, 0x8E, 0x14, 0x17, 0x8B, 0x89, 0x59, 0x8A, 0xBB, 0xEA, 0xD6, +0x7D, 0x85, 0x00, 0x9F, 0xC2, 0x8A, 0xCB, 0x01, 0x7F, 0x8C, 0x6E, 0x24 + +[sha256_hmac_buff_64] +digest = +0x8F, 0x4B, 0x3B, 0x4C, 0x58, 0x25, 0x3B, 0x07, 0xEB, 0xF8, 0x20, 0x81, 0xD9, 0xD9, 0x92, 0x8F, +0xF4, 0x32, 0x7C, 0x2A, 0xD9, 0xEC, 0x92, 0x60, 0x8F, 0xE3, 0x90, 0x7F, 0xC5, 0x75, 0x05, 0xB6 + +[sha384_hmac_buff_64] +digest = +0xD1, 0xC7, 0x64, 0x27, 0xF0, 0x30, 0x43, 0x8E, 0xD6, 0xA6, 0x78, 0xF7, 0xE9, 0xCC, 0x8E, 0x69, +0x6D, 0xB8, 0x3E, 0xFA, 0xA0, 0x81, 0x9C, 0x61, 0x78, 0x72, 0xF0, 0x1C, 0x29, 0x35, 0x51, 0x3E, +0x4A, 0x95, 0xDE, 0x2C, 0x6A, 0x3F, 0x56, 0xA8, 0x12, 0xBA, 0x44, 0x39, 0x1E, 0xDB, 0xF7, 0xF5 + +[sha512_hmac_buff_64] +digest = +0xE6, 0xE9, 0xD8, 0x1D, 0x90, 0xAE, 0x32, 0x0E, 0xBA, 0x55, 0x58, 0xD5, 0x55, 0x97, 0x40, 0xB3, +0xE9, 0x12, 0xD3, 0x08, 0xEF, 0x21, 0xED, 0xA5, 0x94, 0x8D, 0xF2, 0x4C, 0x52, 0x2C, 0x50, 0xB2, +0xD2, 0xEC, 0xB7, 0xE1, 0x95, 0x2D, 0x68, 0xDB, 0xAD, 0xB5, 0x94, 0x50, 0x67, 0xF3, 0x0A, 0x83, +0x54, 0x03, 0x33, 0x1C, 0xD5, 0x42, 0x7D, 0xB4, 0x3E, 0x69, 0x7C, 0x36, 0x7E, 0x96, 0x0D, 0x3E + +##################### +# sha_hmac_buff_128 # +##################### +[sha1_hmac_buff_128] +digest = +0xAA, 0x90, 0x55, 0xA5, 0x71, 0xC4, 0x2B, 0xA3, 0x02, 0xAA, 0xB1, 0x1C, 0xB3, 0x88, 0x38, 0x6E, +0xAD, 0x26, 0x98, 0xA7 + +[sha224_hmac_buff_128] +digest = +0xBE, 0xCC, 0x83, 0x48, 0x4C, 0x58, 0xF9, 0x86, 0xFA, 0x93, 0x5F, 0xD1, 0x3C, 0x11, 0x8A, 0x37, +0xA6, 0xEE, 0x52, 0x4D, 0xA3, 0x98, 0x3E, 0x35, 0xF1, 0x4F, 0xD9, 0xDB + +[sha256_hmac_buff_128] +digest = +0xE2, 0x9C, 0xE1, 0xDF, 0xCD, 0xAE, 0x50, 0x4B, 0x9A, 0xA6, 0x41, 0xAC, 0x0C, 0xF1, 0x66, 0xED, +0xA1, 0x22, 0x05, 0x72, 0x49, 0x97, 0xA1, 0x30, 0xB8, 0xF9, 0xED, 0x36, 0x0A, 0x19, 0xE4, 0x2A + +[sha384_hmac_buff_128] +digest = +0xD9, 0x3C, 0xEB, 0xF4, 0x20, 0xC6, 0x4F, 0xC7, 0xBD, 0x34, 0xBA, 0xFD, 0x7C, 0xA9, 0xCE, 0xFF, +0x26, 0x2E, 0xB4, 0x4A, 0xB7, 0x47, 0x71, 0x2C, 0x9E, 0xCF, 0x44, 0x0B, 0xD9, 0xAF, 0x8D, 0x17, +0x0A, 0x3A, 0x02, 0xD0, 0xE9, 0xDF, 0xCF, 0x52, 0x5F, 0xDA, 0xA7, 0xB6, 0x51, 0x7C, 0x59, 0x09 + +[sha512_hmac_buff_128] +digest = +0xAD, 0x7E, 0xB7, 0x33, 0xFB, 0x8A, 0x17, 0xD0, 0x3C, 0xB0, 0x80, 0x19, 0xF3, 0x9A, 0x6F, 0x90, +0xDE, 0xF3, 0x53, 0xEA, 0x48, 0x75, 0x0A, 0x1E, 0x49, 0x02, 0xA0, 0x94, 0xC4, 0xE8, 0xFB, 0x87, +0x83, 0x80, 0xD3, 0xFF, 0x6B, 0x79, 0x73, 0x54, 0xF9, 0x2F, 0x2D, 0x59, 0x69, 0x0E, 0x50, 0x29, +0x2A, 0xDA, 0x59, 0x38, 0xDD, 0x62, 0xF9, 0x1A, 0x18, 0xA9, 0x51, 0x5A, 0xFE, 0x8E, 0xFD, 0xBF + +##################### +# sha_hmac_buff_256 # +##################### +[sha1_hmac_buff_256] +digest = +0xB1, 0x18, 0x31, 0xBF, 0xEE, 0x81, 0x7E, 0xFC, 0x68, 0xDA, 0xB6, 0x8A, 0x5D, 0xDE, 0x39, 0x65, +0xC8, 0xF8, 0xC3, 0xE5 + +[sha224_hmac_buff_256] +digest = +0xCD, 0xF6, 0xC2, 0x6D, 0xFD, 0x33, 0x1A, 0xD8, 0x2F, 0x07, 0x4F, 0x1A, 0xE8, 0x18, 0xBD, 0x04, +0xB1, 0xE5, 0x8D, 0xC1, 0x21, 0x95, 0x87, 0x75, 0xC2, 0x27, 0x4B, 0xF2 + +[sha256_hmac_buff_256] +digest = +0xC0, 0xFA, 0x8F, 0x6F, 0x55, 0xFC, 0xF3, 0xDF, 0x8E, 0x5D, 0x93, 0x5E, 0x6B, 0x20, 0x0A, 0x9A, +0x84, 0x3D, 0xCD, 0x4B, 0x57, 0x63, 0x2D, 0x93, 0x51, 0x45, 0xF2, 0x1E, 0xC7, 0xA4, 0xD4, 0x69 + +[sha384_hmac_buff_256] +digest = +0x2B, 0x92, 0x9E, 0x85, 0x5A, 0x89, 0xB5, 0x12, 0x4A, 0x9B, 0x2D, 0xD2, 0xB2, 0x3E, 0xAB, 0xC1, +0x1E, 0x7F, 0x53, 0xD9, 0x88, 0xEB, 0xEE, 0xA2, 0x49, 0x14, 0xDE, 0x1A, 0x9E, 0x20, 0xCE, 0xEC, +0x7A, 0x5D, 0x25, 0xD8, 0x8F, 0xFE, 0x8B, 0xB1, 0xB1, 0x04, 0x5F, 0x46, 0x2D, 0x34, 0x2D, 0x72 + +[sha512_hmac_buff_256] +digest = +0x4F, 0x96, 0x89, 0x9E, 0x9D, 0x53, 0xAC, 0x05, 0xC7, 0xA0, 0x0F, 0x4D, 0xB6, 0x3E, 0x06, 0x03, +0x19, 0x68, 0x41, 0x4F, 0x11, 0x57, 0x77, 0x21, 0xBD, 0x60, 0x3E, 0xB4, 0xFE, 0x6A, 0x0D, 0xBF, +0xE0, 0x4F, 0x32, 0x5B, 0xF9, 0xDF, 0x13, 0xBD, 0x02, 0x73, 0xD4, 0x0C, 0xE9, 0x9D, 0xB7, 0xD5, +0x38, 0xA0, 0x20, 0xD9, 0xD1, 0x66, 0x17, 0x19, 0x54, 0x36, 0x18, 0xE1, 0xF5, 0x34, 0x12, 0x9E + +##################### +# sha_hmac_buff_512 # +##################### +[sha1_hmac_buff_512] +digest = +0x78, 0x14, 0x01, 0xED, 0x93, 0x6F, 0x22, 0xB6, 0x96, 0x5A, 0x32, 0x05, 0xA9, 0xD3, 0x49, 0x04, +0x55, 0xB0, 0x00, 0x06 + +[sha224_hmac_buff_512] +digest = +0x25, 0xD4, 0x8F, 0x92, 0xE1, 0xD0, 0x4E, 0x3F, 0x34, 0x38, 0x01, 0xB8, 0xFE, 0x57, 0x3D, 0x34, +0x39, 0x98, 0x82, 0x8D, 0x68, 0x04, 0x5A, 0x74, 0x28, 0x4F, 0x18, 0xCE + +[sha256_hmac_buff_512] +digest = +0x90, 0x06, 0x97, 0x8A, 0x7A, 0xEF, 0x62, 0x14, 0x4C, 0x14, 0xAA, 0x25, 0x4C, 0xE3, 0x5D, 0xE4, +0xAD, 0x6C, 0xD6, 0x82, 0x2B, 0x87, 0x53, 0x3E, 0xE9, 0xE4, 0x97, 0x82, 0x82, 0x76, 0xE7, 0xF1 + +[sha384_hmac_buff_512] +digest = +0xD5, 0xDA, 0x7C, 0x8A, 0x0D, 0x1B, 0xBE, 0x3E, 0x25, 0x1E, 0x6C, 0xA4, 0x50, 0x32, 0x92, 0x13, +0x91, 0x4F, 0xA2, 0x29, 0x2A, 0x0A, 0x57, 0x62, 0x3D, 0x93, 0xF2, 0x45, 0x96, 0x22, 0xF8, 0x0D, +0xA9, 0xE9, 0xAB, 0xAC, 0x7B, 0x2E, 0x42, 0xC2, 0x3E, 0x75, 0x23, 0xD0, 0xD2, 0xAA, 0x1E, 0xEE + +[sha512_hmac_buff_512] +digest = +0x57, 0x34, 0x65, 0x3D, 0xDE, 0x8B, 0x7B, 0x99, 0x62, 0x24, 0xF3, 0xAF, 0xA6, 0xB1, 0xF0, 0x01, +0x23, 0xD4, 0x94, 0xC2, 0x06, 0x70, 0xA5, 0x8C, 0x80, 0x93, 0x49, 0x88, 0xB4, 0xB6, 0x45, 0xE5, +0x37, 0xC9, 0xE4, 0xA1, 0xAB, 0x6C, 0xA5, 0x23, 0xD2, 0x07, 0x9B, 0x10, 0x4D, 0xFD, 0x75, 0xC0, +0x28, 0xA1, 0x8A, 0x84, 0x03, 0x35, 0x22, 0xCC, 0xAC, 0x6C, 0x97, 0x93, 0x57, 0x08, 0x48, 0x51 + +###################### +# sha_hmac_buff_1024 # +###################### +[sha1_hmac_buff_1024] +digest = +0x74, 0xF7, 0x91, 0x04, 0x06, 0xDB, 0xA9, 0xF0, 0x08, 0x0E, 0x93, 0xCE, 0x55, 0xA8, 0x54, 0xF0, +0x4B, 0x5E, 0x3F, 0xC7 + +[sha224_hmac_buff_1024] +digest = +0x45, 0xDA, 0x2E, 0x83, 0xBD, 0x35, 0xA4, 0x58, 0x14, 0x74, 0xCB, 0xA4, 0x48, 0xA6, 0xBA, 0xDC, +0x7D, 0x56, 0x6A, 0x44, 0xA7, 0xE9, 0x2F, 0x75, 0x20, 0x47, 0x2A, 0x5A + +[sha256_hmac_buff_1024] +digest = +0xA2, 0x81, 0xFE, 0x1A, 0x5C, 0x4F, 0x02, 0x72, 0xEF, 0x4F, 0xC6, 0xEE, 0x54, 0x71, 0x69, 0xAF, +0x5C, 0x71, 0x9F, 0xB0, 0xAC, 0x5B, 0x7F, 0x51, 0xD6, 0x0D, 0x64, 0xD2, 0x2E, 0x0E, 0x30, 0x55 + +[sha384_hmac_buff_1024] +digest = +0x26, 0x44, 0x13, 0x01, 0x25, 0x6E, 0xC7, 0xC3, 0x88, 0x25, 0xD5, 0xDD, 0x1D, 0xCA, 0x0C, 0xB1, +0xB8, 0x82, 0xB2, 0xB8, 0x15, 0x3F, 0xE5, 0x54, 0x43, 0x47, 0x32, 0x3B, 0xB2, 0xE0, 0xC8, 0x58, +0x64, 0xE7, 0x78, 0xC9, 0x1F, 0x81, 0x7B, 0xBD, 0x0D, 0x6D, 0x37, 0x9C, 0x01, 0x20, 0x6A, 0x8E + +[sha512_hmac_buff_1024] +digest = +0xBE, 0xDA, 0x0D, 0x3B, 0x47, 0x24, 0xBA, 0x45, 0xBA, 0xCA, 0x84, 0x5F, 0xEA, 0xAC, 0x33, 0x84, +0x00, 0x62, 0xA5, 0x29, 0xB6, 0x2F, 0xB7, 0x86, 0xD0, 0x94, 0xFF, 0xFF, 0xE4, 0x1E, 0x5C, 0xFD, +0xC8, 0xD5, 0x3A, 0xD4, 0xFC, 0xA6, 0x1C, 0x66, 0x4A, 0x6D, 0xF9, 0x2B, 0x1D, 0x7F, 0xA0, 0xCF, +0x3D, 0x0F, 0x1F, 0x5B, 0xDD, 0x21, 0x12, 0xA8, 0x76, 0xB0, 0xD7, 0x30, 0x66, 0xA6, 0xA0, 0x6C + +###################### +# sha_hmac_buff_2048 # +###################### +[sha1_hmac_buff_2048] +digest = +0x99, 0x32, 0xCD, 0xC3, 0xC9, 0x7F, 0x98, 0x1A, 0x96, 0xF6, 0x52, 0xC8, 0xA2, 0x16, 0x9C, 0x29, +0x9D, 0x6E, 0x96, 0xF5 + +[sha224_hmac_buff_2048] +digest = +0x1A, 0xC4, 0xDC, 0x46, 0xE5, 0x87, 0xFE, 0xE0, 0x47, 0x64, 0x53, 0xA3, 0x6A, 0x1F, 0x78, 0xC0, +0xC0, 0x02, 0x03, 0x64, 0xB1, 0x55, 0x50, 0x66, 0xDA, 0xD6, 0x9E, 0xBC + +[sha256_hmac_buff_2048] +digest = +0xA6, 0xC6, 0x4B, 0x0C, 0x95, 0xDE, 0xD5, 0xE2, 0x40, 0x7D, 0x44, 0xC5, 0xBF, 0x00, 0x5A, 0xFB, +0x6F, 0x3F, 0x5E, 0x69, 0xB1, 0x32, 0x91, 0xAB, 0x6C, 0x90, 0x25, 0xAB, 0xD9, 0x1B, 0x8F, 0x80 + +[sha384_hmac_buff_2048] +digest = +0x16, 0xF1, 0x1B, 0xC1, 0x22, 0xDB, 0x21, 0x90, 0x08, 0xE3, 0x42, 0x0C, 0x9A, 0xF1, 0x0F, 0xF8, +0x7A, 0xE9, 0x50, 0x09, 0xC6, 0x0C, 0x71, 0x65, 0x3A, 0x40, 0xB3, 0x78, 0x11, 0xE8, 0xD2, 0xD4, +0xB0, 0x6C, 0xA9, 0x6A, 0x0C, 0xCD, 0xE1, 0x70, 0x7E, 0x90, 0x86, 0x34, 0xC1, 0x08, 0x9E, 0xFC + +[sha512_hmac_buff_2048] +digest = +0xDF, 0x7F, 0xC3, 0x26, 0x3E, 0x55, 0x80, 0x7D, 0x02, 0x06, 0x5A, 0x4B, 0x8C, 0xFD, 0x2F, 0x33, +0xF8, 0x0E, 0x9C, 0x59, 0xAE, 0x56, 0xAE, 0x86, 0xA9, 0x25, 0x3F, 0xB7, 0xF7, 0x4C, 0x7A, 0xB4, +0xD8, 0x0E, 0x43, 0xC0, 0x86, 0xDF, 0xDB, 0xBA, 0xA8, 0xCB, 0x46, 0x2A, 0x92, 0x34, 0xF5, 0x3B, +0xBD, 0x59, 0x64, 0xDF, 0x30, 0x20, 0xF5, 0x13, 0xD7, 0x78, 0xB9, 0x27, 0xE6, 0xB6, 0x56, 0x19 diff --git a/dpdk/app/test-crypto-perf/main.c b/dpdk/app/test-crypto-perf/main.c new file mode 100644 index 00000000..29373f5b --- /dev/null +++ b/dpdk/app/test-crypto-perf/main.c @@ -0,0 +1,615 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include + +#include +#include +#ifdef RTE_LIBRTE_PMD_CRYPTO_SCHEDULER +#include +#endif + +#include "cperf.h" +#include "cperf_options.h" +#include "cperf_test_vector_parsing.h" +#include "cperf_test_throughput.h" +#include "cperf_test_latency.h" +#include "cperf_test_verify.h" +#include "cperf_test_pmd_cyclecount.h" + +#define NUM_SESSIONS 2048 +#define SESS_MEMPOOL_CACHE_SIZE 64 + +const char *cperf_test_type_strs[] = { + [CPERF_TEST_TYPE_THROUGHPUT] = "throughput", + [CPERF_TEST_TYPE_LATENCY] = "latency", + [CPERF_TEST_TYPE_VERIFY] = "verify", + [CPERF_TEST_TYPE_PMDCC] = "pmd-cyclecount" +}; + +const char *cperf_op_type_strs[] = { + [CPERF_CIPHER_ONLY] = "cipher-only", + [CPERF_AUTH_ONLY] = "auth-only", + [CPERF_CIPHER_THEN_AUTH] = "cipher-then-auth", + [CPERF_AUTH_THEN_CIPHER] = "auth-then-cipher", + [CPERF_AEAD] = "aead" +}; + +const struct cperf_test cperf_testmap[] = { + [CPERF_TEST_TYPE_THROUGHPUT] = { + cperf_throughput_test_constructor, + cperf_throughput_test_runner, + cperf_throughput_test_destructor + }, + [CPERF_TEST_TYPE_LATENCY] = { + cperf_latency_test_constructor, + cperf_latency_test_runner, + cperf_latency_test_destructor + }, + [CPERF_TEST_TYPE_VERIFY] = { + cperf_verify_test_constructor, + cperf_verify_test_runner, + cperf_verify_test_destructor + }, + [CPERF_TEST_TYPE_PMDCC] = { + cperf_pmd_cyclecount_test_constructor, + cperf_pmd_cyclecount_test_runner, + cperf_pmd_cyclecount_test_destructor + } +}; + +static int +cperf_initialize_cryptodev(struct cperf_options *opts, uint8_t *enabled_cdevs, + struct rte_mempool *session_pool_socket[]) +{ + uint8_t enabled_cdev_count = 0, nb_lcores, cdev_id; + unsigned int i, j; + int ret; + + enabled_cdev_count = rte_cryptodev_devices_get(opts->device_type, + enabled_cdevs, RTE_CRYPTO_MAX_DEVS); + if (enabled_cdev_count == 0) { + printf("No crypto devices type %s available\n", + opts->device_type); + return -EINVAL; + } + + nb_lcores = rte_lcore_count() - 1; + + if (enabled_cdev_count > nb_lcores) { + printf("Number of capable crypto devices (%d) " + "has to be less or equal to number of slave " + "cores (%d)\n", enabled_cdev_count, nb_lcores); + return -EINVAL; + } + + /* Create a mempool shared by all the devices */ + uint32_t max_sess_size = 0, sess_size; + + for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) { + sess_size = rte_cryptodev_get_private_session_size(cdev_id); + if (sess_size > max_sess_size) + max_sess_size = sess_size; + } + + /* + * Calculate number of needed queue pairs, based on the amount + * of available number of logical cores and crypto devices. + * For instance, if there are 4 cores and 2 crypto devices, + * 2 queue pairs will be set up per device. + */ + opts->nb_qps = (nb_lcores % enabled_cdev_count) ? + (nb_lcores / enabled_cdev_count) + 1 : + nb_lcores / enabled_cdev_count; + + for (i = 0; i < enabled_cdev_count && + i < RTE_CRYPTO_MAX_DEVS; i++) { + cdev_id = enabled_cdevs[i]; +#ifdef RTE_LIBRTE_PMD_CRYPTO_SCHEDULER + /* + * If multi-core scheduler is used, limit the number + * of queue pairs to 1, as there is no way to know + * how many cores are being used by the PMD, and + * how many will be available for the application. + */ + if (!strcmp((const char *)opts->device_type, "crypto_scheduler") && + rte_cryptodev_scheduler_mode_get(cdev_id) == + CDEV_SCHED_MODE_MULTICORE) + opts->nb_qps = 1; +#endif + + struct rte_cryptodev_info cdev_info; + uint8_t socket_id = rte_cryptodev_socket_id(cdev_id); + + rte_cryptodev_info_get(cdev_id, &cdev_info); + if (opts->nb_qps > cdev_info.max_nb_queue_pairs) { + printf("Number of needed queue pairs is higher " + "than the maximum number of queue pairs " + "per device.\n"); + printf("Lower the number of cores or increase " + "the number of crypto devices\n"); + return -EINVAL; + } + struct rte_cryptodev_config conf = { + .nb_queue_pairs = opts->nb_qps, + .socket_id = socket_id + }; + + struct rte_cryptodev_qp_conf qp_conf = { + .nb_descriptors = opts->nb_descriptors + }; + + if (session_pool_socket[socket_id] == NULL) { + char mp_name[RTE_MEMPOOL_NAMESIZE]; + struct rte_mempool *sess_mp; + + snprintf(mp_name, RTE_MEMPOOL_NAMESIZE, + "sess_mp_%u", socket_id); + + sess_mp = rte_mempool_create(mp_name, + NUM_SESSIONS, + max_sess_size, + SESS_MEMPOOL_CACHE_SIZE, + 0, NULL, NULL, NULL, + NULL, socket_id, + 0); + + if (sess_mp == NULL) { + printf("Cannot create session pool on socket %d\n", + socket_id); + return -ENOMEM; + } + + printf("Allocated session pool on socket %d\n", socket_id); + session_pool_socket[socket_id] = sess_mp; + } + + ret = rte_cryptodev_configure(cdev_id, &conf); + if (ret < 0) { + printf("Failed to configure cryptodev %u", cdev_id); + return -EINVAL; + } + + for (j = 0; j < opts->nb_qps; j++) { + ret = rte_cryptodev_queue_pair_setup(cdev_id, j, + &qp_conf, socket_id, + session_pool_socket[socket_id]); + if (ret < 0) { + printf("Failed to setup queue pair %u on " + "cryptodev %u", j, cdev_id); + return -EINVAL; + } + } + + ret = rte_cryptodev_start(cdev_id); + if (ret < 0) { + printf("Failed to start device %u: error %d\n", + cdev_id, ret); + return -EPERM; + } + } + + return enabled_cdev_count; +} + +static int +cperf_verify_devices_capabilities(struct cperf_options *opts, + uint8_t *enabled_cdevs, uint8_t nb_cryptodevs) +{ + struct rte_cryptodev_sym_capability_idx cap_idx; + const struct rte_cryptodev_symmetric_capability *capability; + + uint8_t i, cdev_id; + int ret; + + for (i = 0; i < nb_cryptodevs; i++) { + + cdev_id = enabled_cdevs[i]; + + if (opts->op_type == CPERF_AUTH_ONLY || + opts->op_type == CPERF_CIPHER_THEN_AUTH || + opts->op_type == CPERF_AUTH_THEN_CIPHER) { + + cap_idx.type = RTE_CRYPTO_SYM_XFORM_AUTH; + cap_idx.algo.auth = opts->auth_algo; + + capability = rte_cryptodev_sym_capability_get(cdev_id, + &cap_idx); + if (capability == NULL) + return -1; + + ret = rte_cryptodev_sym_capability_check_auth( + capability, + opts->auth_key_sz, + opts->digest_sz, + opts->auth_iv_sz); + if (ret != 0) + return ret; + } + + if (opts->op_type == CPERF_CIPHER_ONLY || + opts->op_type == CPERF_CIPHER_THEN_AUTH || + opts->op_type == CPERF_AUTH_THEN_CIPHER) { + + cap_idx.type = RTE_CRYPTO_SYM_XFORM_CIPHER; + cap_idx.algo.cipher = opts->cipher_algo; + + capability = rte_cryptodev_sym_capability_get(cdev_id, + &cap_idx); + if (capability == NULL) + return -1; + + ret = rte_cryptodev_sym_capability_check_cipher( + capability, + opts->cipher_key_sz, + opts->cipher_iv_sz); + if (ret != 0) + return ret; + } + + if (opts->op_type == CPERF_AEAD) { + + cap_idx.type = RTE_CRYPTO_SYM_XFORM_AEAD; + cap_idx.algo.aead = opts->aead_algo; + + capability = rte_cryptodev_sym_capability_get(cdev_id, + &cap_idx); + if (capability == NULL) + return -1; + + ret = rte_cryptodev_sym_capability_check_aead( + capability, + opts->aead_key_sz, + opts->digest_sz, + opts->aead_aad_sz, + opts->aead_iv_sz); + if (ret != 0) + return ret; + } + } + + return 0; +} + +static int +cperf_check_test_vector(struct cperf_options *opts, + struct cperf_test_vector *test_vec) +{ + if (opts->op_type == CPERF_CIPHER_ONLY) { + if (opts->cipher_algo == RTE_CRYPTO_CIPHER_NULL) { + if (test_vec->plaintext.data == NULL) + return -1; + } else if (opts->cipher_algo != RTE_CRYPTO_CIPHER_NULL) { + if (test_vec->plaintext.data == NULL) + return -1; + if (test_vec->plaintext.length < opts->max_buffer_size) + return -1; + if (test_vec->ciphertext.data == NULL) + return -1; + if (test_vec->ciphertext.length < opts->max_buffer_size) + return -1; + if (test_vec->cipher_iv.data == NULL) + return -1; + if (test_vec->cipher_iv.length != opts->cipher_iv_sz) + return -1; + if (test_vec->cipher_key.data == NULL) + return -1; + if (test_vec->cipher_key.length != opts->cipher_key_sz) + return -1; + } + } else if (opts->op_type == CPERF_AUTH_ONLY) { + if (opts->auth_algo != RTE_CRYPTO_AUTH_NULL) { + if (test_vec->plaintext.data == NULL) + return -1; + if (test_vec->plaintext.length < opts->max_buffer_size) + return -1; + if (test_vec->auth_key.data == NULL) + return -1; + if (test_vec->auth_key.length != opts->auth_key_sz) + return -1; + if (test_vec->auth_iv.length != opts->auth_iv_sz) + return -1; + /* Auth IV is only required for some algorithms */ + if (opts->auth_iv_sz && test_vec->auth_iv.data == NULL) + return -1; + if (test_vec->digest.data == NULL) + return -1; + if (test_vec->digest.length < opts->digest_sz) + return -1; + } + + } else if (opts->op_type == CPERF_CIPHER_THEN_AUTH || + opts->op_type == CPERF_AUTH_THEN_CIPHER) { + if (opts->cipher_algo == RTE_CRYPTO_CIPHER_NULL) { + if (test_vec->plaintext.data == NULL) + return -1; + if (test_vec->plaintext.length < opts->max_buffer_size) + return -1; + } else if (opts->cipher_algo != RTE_CRYPTO_CIPHER_NULL) { + if (test_vec->plaintext.data == NULL) + return -1; + if (test_vec->plaintext.length < opts->max_buffer_size) + return -1; + if (test_vec->ciphertext.data == NULL) + return -1; + if (test_vec->ciphertext.length < opts->max_buffer_size) + return -1; + if (test_vec->cipher_iv.data == NULL) + return -1; + if (test_vec->cipher_iv.length != opts->cipher_iv_sz) + return -1; + if (test_vec->cipher_key.data == NULL) + return -1; + if (test_vec->cipher_key.length != opts->cipher_key_sz) + return -1; + } + if (opts->auth_algo != RTE_CRYPTO_AUTH_NULL) { + if (test_vec->auth_key.data == NULL) + return -1; + if (test_vec->auth_key.length != opts->auth_key_sz) + return -1; + if (test_vec->auth_iv.length != opts->auth_iv_sz) + return -1; + /* Auth IV is only required for some algorithms */ + if (opts->auth_iv_sz && test_vec->auth_iv.data == NULL) + return -1; + if (test_vec->digest.data == NULL) + return -1; + if (test_vec->digest.length < opts->digest_sz) + return -1; + } + } else if (opts->op_type == CPERF_AEAD) { + if (test_vec->plaintext.data == NULL) + return -1; + if (test_vec->plaintext.length < opts->max_buffer_size) + return -1; + if (test_vec->ciphertext.data == NULL) + return -1; + if (test_vec->ciphertext.length < opts->max_buffer_size) + return -1; + if (test_vec->aead_iv.data == NULL) + return -1; + if (test_vec->aead_iv.length != opts->aead_iv_sz) + return -1; + if (test_vec->aad.data == NULL) + return -1; + if (test_vec->aad.length != opts->aead_aad_sz) + return -1; + if (test_vec->digest.data == NULL) + return -1; + if (test_vec->digest.length < opts->digest_sz) + return -1; + } + return 0; +} + +int +main(int argc, char **argv) +{ + struct cperf_options opts = {0}; + struct cperf_test_vector *t_vec = NULL; + struct cperf_op_fns op_fns; + + void *ctx[RTE_MAX_LCORE] = { }; + struct rte_mempool *session_pool_socket[RTE_MAX_NUMA_NODES] = { 0 }; + + int nb_cryptodevs = 0; + uint16_t total_nb_qps = 0; + uint8_t cdev_id, i; + uint8_t enabled_cdevs[RTE_CRYPTO_MAX_DEVS] = { 0 }; + + uint8_t buffer_size_idx = 0; + + int ret; + uint32_t lcore_id; + + /* Initialise DPDK EAL */ + ret = rte_eal_init(argc, argv); + if (ret < 0) + rte_exit(EXIT_FAILURE, "Invalid EAL arguments!\n"); + argc -= ret; + argv += ret; + + cperf_options_default(&opts); + + ret = cperf_options_parse(&opts, argc, argv); + if (ret) { + RTE_LOG(ERR, USER1, "Parsing on or more user options failed\n"); + goto err; + } + + ret = cperf_options_check(&opts); + if (ret) { + RTE_LOG(ERR, USER1, + "Checking on or more user options failed\n"); + goto err; + } + + nb_cryptodevs = cperf_initialize_cryptodev(&opts, enabled_cdevs, + session_pool_socket); + + if (!opts.silent) + cperf_options_dump(&opts); + + if (nb_cryptodevs < 1) { + RTE_LOG(ERR, USER1, "Failed to initialise requested crypto " + "device type\n"); + nb_cryptodevs = 0; + goto err; + } + + ret = cperf_verify_devices_capabilities(&opts, enabled_cdevs, + nb_cryptodevs); + if (ret) { + RTE_LOG(ERR, USER1, "Crypto device type does not support " + "capabilities requested\n"); + goto err; + } + + if (opts.test_file != NULL) { + t_vec = cperf_test_vector_get_from_file(&opts); + if (t_vec == NULL) { + RTE_LOG(ERR, USER1, + "Failed to create test vector for" + " specified file\n"); + goto err; + } + + if (cperf_check_test_vector(&opts, t_vec)) { + RTE_LOG(ERR, USER1, "Incomplete necessary test vectors" + "\n"); + goto err; + } + } else { + t_vec = cperf_test_vector_get_dummy(&opts); + if (t_vec == NULL) { + RTE_LOG(ERR, USER1, + "Failed to create test vector for" + " specified algorithms\n"); + goto err; + } + } + + ret = cperf_get_op_functions(&opts, &op_fns); + if (ret) { + RTE_LOG(ERR, USER1, "Failed to find function ops set for " + "specified algorithms combination\n"); + goto err; + } + + if (!opts.silent) + show_test_vector(t_vec); + + total_nb_qps = nb_cryptodevs * opts.nb_qps; + + i = 0; + uint8_t qp_id = 0, cdev_index = 0; + RTE_LCORE_FOREACH_SLAVE(lcore_id) { + + if (i == total_nb_qps) + break; + + cdev_id = enabled_cdevs[cdev_index]; + + uint8_t socket_id = rte_cryptodev_socket_id(cdev_id); + + ctx[i] = cperf_testmap[opts.test].constructor( + session_pool_socket[socket_id], cdev_id, qp_id, + &opts, t_vec, &op_fns); + if (ctx[i] == NULL) { + RTE_LOG(ERR, USER1, "Test run constructor failed\n"); + goto err; + } + qp_id = (qp_id + 1) % opts.nb_qps; + if (qp_id == 0) + cdev_index++; + i++; + } + + /* Get first size from range or list */ + if (opts.inc_buffer_size != 0) + opts.test_buffer_size = opts.min_buffer_size; + else + opts.test_buffer_size = opts.buffer_size_list[0]; + + while (opts.test_buffer_size <= opts.max_buffer_size) { + i = 0; + RTE_LCORE_FOREACH_SLAVE(lcore_id) { + + if (i == total_nb_qps) + break; + + rte_eal_remote_launch(cperf_testmap[opts.test].runner, + ctx[i], lcore_id); + i++; + } + i = 0; + RTE_LCORE_FOREACH_SLAVE(lcore_id) { + + if (i == total_nb_qps) + break; + rte_eal_wait_lcore(lcore_id); + i++; + } + + /* Get next size from range or list */ + if (opts.inc_buffer_size != 0) + opts.test_buffer_size += opts.inc_buffer_size; + else { + if (++buffer_size_idx == opts.buffer_size_count) + break; + opts.test_buffer_size = opts.buffer_size_list[buffer_size_idx]; + } + } + + i = 0; + RTE_LCORE_FOREACH_SLAVE(lcore_id) { + + if (i == total_nb_qps) + break; + + cperf_testmap[opts.test].destructor(ctx[i]); + i++; + } + + for (i = 0; i < nb_cryptodevs && + i < RTE_CRYPTO_MAX_DEVS; i++) + rte_cryptodev_stop(enabled_cdevs[i]); + + free_test_vector(t_vec, &opts); + + printf("\n"); + return EXIT_SUCCESS; + +err: + i = 0; + RTE_LCORE_FOREACH_SLAVE(lcore_id) { + if (i == total_nb_qps) + break; + + cdev_id = enabled_cdevs[i]; + + if (ctx[i] && cperf_testmap[opts.test].destructor) + cperf_testmap[opts.test].destructor(ctx[i]); + i++; + } + + for (i = 0; i < nb_cryptodevs && + i < RTE_CRYPTO_MAX_DEVS; i++) + rte_cryptodev_stop(enabled_cdevs[i]); + + free_test_vector(t_vec, &opts); + + printf("\n"); + return EXIT_FAILURE; +} diff --git a/dpdk/app/test-eventdev/Makefile b/dpdk/app/test-eventdev/Makefile new file mode 100644 index 00000000..dcb2ac47 --- /dev/null +++ b/dpdk/app/test-eventdev/Makefile @@ -0,0 +1,54 @@ +# BSD LICENSE +# +# Copyright(c) 2017 Cavium, Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Cavium, Inc nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +include $(RTE_SDK)/mk/rte.vars.mk + +APP = dpdk-test-eventdev + +CFLAGS += -O3 +CFLAGS += $(WERROR_FLAGS) + +# +# all source are stored in SRCS-y +# +SRCS-y := evt_main.c +SRCS-y += evt_options.c +SRCS-y += evt_test.c +SRCS-y += parser.c + +SRCS-y += test_order_common.c +SRCS-y += test_order_queue.c +SRCS-y += test_order_atq.c + +SRCS-y += test_perf_common.c +SRCS-y += test_perf_queue.c +SRCS-y += test_perf_atq.c + +include $(RTE_SDK)/mk/rte.app.mk diff --git a/dpdk/app/test-eventdev/evt_common.h b/dpdk/app/test-eventdev/evt_common.h new file mode 100644 index 00000000..0fadab4a --- /dev/null +++ b/dpdk/app/test-eventdev/evt_common.h @@ -0,0 +1,135 @@ +/* + * BSD LICENSE + * + * Copyright (C) Cavium, Inc 2017. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _EVT_COMMON_ +#define _EVT_COMMON_ + +#include +#include +#include +#include + +#define CLNRM "\x1b[0m" +#define CLRED "\x1b[31m" +#define CLGRN "\x1b[32m" +#define CLYEL "\x1b[33m" + +#define evt_err(fmt, args...) \ + fprintf(stderr, CLRED"error: %s() "fmt CLNRM "\n", __func__, ## args) + +#define evt_info(fmt, args...) \ + fprintf(stdout, CLYEL""fmt CLNRM "\n", ## args) + +#define EVT_STR_FMT 20 + +#define evt_dump(str, fmt, val...) \ + printf("\t%-*s : "fmt"\n", EVT_STR_FMT, str, ## val) + +#define evt_dump_begin(str) printf("\t%-*s : {", EVT_STR_FMT, str) + +#define evt_dump_end printf("\b}\n") + +#define EVT_MAX_STAGES 64 +#define EVT_MAX_PORTS 256 +#define EVT_MAX_QUEUES 256 + +static inline bool +evt_has_distributed_sched(uint8_t dev_id) +{ + struct rte_event_dev_info dev_info; + + rte_event_dev_info_get(dev_id, &dev_info); + return (dev_info.event_dev_cap & RTE_EVENT_DEV_CAP_DISTRIBUTED_SCHED) ? + true : false; +} + +static inline bool +evt_has_burst_mode(uint8_t dev_id) +{ + struct rte_event_dev_info dev_info; + + rte_event_dev_info_get(dev_id, &dev_info); + return (dev_info.event_dev_cap & RTE_EVENT_DEV_CAP_BURST_MODE) ? + true : false; +} + + +static inline bool +evt_has_all_types_queue(uint8_t dev_id) +{ + struct rte_event_dev_info dev_info; + + rte_event_dev_info_get(dev_id, &dev_info); + return (dev_info.event_dev_cap & RTE_EVENT_DEV_CAP_QUEUE_ALL_TYPES) ? + true : false; +} + +static inline int +evt_service_setup(uint8_t dev_id) +{ + uint32_t service_id; + int32_t core_cnt; + unsigned int lcore = 0; + uint32_t core_array[RTE_MAX_LCORE]; + uint8_t cnt; + uint8_t min_cnt = UINT8_MAX; + + if (evt_has_distributed_sched(dev_id)) + return 0; + + if (!rte_service_lcore_count()) + return -ENOENT; + + if (!rte_event_dev_service_id_get(dev_id, &service_id)) { + core_cnt = rte_service_lcore_list(core_array, + RTE_MAX_LCORE); + if (core_cnt < 0) + return -ENOENT; + /* Get the core which has least number of services running. */ + while (core_cnt--) { + /* Reset default mapping */ + rte_service_map_lcore_set(service_id, + core_array[core_cnt], 0); + cnt = rte_service_lcore_count_services( + core_array[core_cnt]); + if (cnt < min_cnt) { + lcore = core_array[core_cnt]; + min_cnt = cnt; + } + } + if (rte_service_map_lcore_set(service_id, lcore, 1)) + return -ENOENT; + } + return 0; +} + +#endif /* _EVT_COMMON_*/ diff --git a/dpdk/app/test-eventdev/evt_main.c b/dpdk/app/test-eventdev/evt_main.c new file mode 100644 index 00000000..1c3a7fae --- /dev/null +++ b/dpdk/app/test-eventdev/evt_main.c @@ -0,0 +1,227 @@ +/* + * BSD LICENSE + * + * Copyright (C) Cavium, Inc 2017. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include + +#include +#include +#include +#include + +#include "evt_options.h" +#include "evt_test.h" + +struct evt_options opt; +struct evt_test *test; + +static void +signal_handler(int signum) +{ + if (signum == SIGINT || signum == SIGTERM) { + printf("\nSignal %d received, preparing to exit...\n", + signum); + /* request all lcores to exit from the main loop */ + *(int *)test->test_priv = true; + rte_wmb(); + + rte_eal_mp_wait_lcore(); + + if (test->ops.eventdev_destroy) + test->ops.eventdev_destroy(test, &opt); + + if (test->ops.ethdev_destroy) + test->ops.ethdev_destroy(test, &opt); + + if (test->ops.mempool_destroy) + test->ops.mempool_destroy(test, &opt); + + if (test->ops.test_destroy) + test->ops.test_destroy(test, &opt); + + /* exit with the expected status */ + signal(signum, SIG_DFL); + kill(getpid(), signum); + } +} + +static inline void +evt_options_dump_all(struct evt_test *test, struct evt_options *opts) +{ + evt_options_dump(opts); + if (test->ops.opt_dump) + test->ops.opt_dump(opts); +} + +int +main(int argc, char **argv) +{ + uint8_t evdevs; + int ret; + + signal(SIGINT, signal_handler); + signal(SIGTERM, signal_handler); + + ret = rte_eal_init(argc, argv); + if (ret < 0) + rte_panic("invalid EAL arguments\n"); + argc -= ret; + argv += ret; + + evdevs = rte_event_dev_count(); + if (!evdevs) + rte_panic("no eventdev devices found\n"); + + /* Populate the default values of the options */ + evt_options_default(&opt); + + /* Parse the command line arguments */ + ret = evt_options_parse(&opt, argc, argv); + if (ret) { + evt_err("parsing on or more user options failed"); + goto error; + } + + /* Get struct evt_test *test from name */ + test = evt_test_get(opt.test_name); + if (test == NULL) { + evt_err("failed to find requested test: %s", opt.test_name); + goto error; + } + + if (test->ops.test_result == NULL) { + evt_err("%s: ops.test_result not found", opt.test_name); + goto error; + } + + /* Verify the command line options */ + if (opt.dev_id >= rte_event_dev_count()) { + evt_err("invalid event device %d", opt.dev_id); + goto error; + } + if (test->ops.opt_check) { + if (test->ops.opt_check(&opt)) { + evt_err("invalid command line argument"); + evt_options_dump_all(test, &opt); + goto error; + } + } + + /* Check the eventdev capability before proceeding */ + if (test->ops.cap_check) { + if (test->ops.cap_check(&opt) == false) { + evt_info("unsupported test: %s", opt.test_name); + evt_options_dump_all(test, &opt); + ret = EVT_TEST_UNSUPPORTED; + goto nocap; + } + } + + /* Dump the options */ + if (opt.verbose_level) + evt_options_dump_all(test, &opt); + + /* Test specific setup */ + if (test->ops.test_setup) { + if (test->ops.test_setup(test, &opt)) { + evt_err("failed to setup test: %s", opt.test_name); + goto error; + + } + } + + /* Test specific mempool setup */ + if (test->ops.mempool_setup) { + if (test->ops.mempool_setup(test, &opt)) { + evt_err("%s: mempool setup failed", opt.test_name); + goto test_destroy; + } + } + + /* Test specific ethdev setup */ + if (test->ops.ethdev_setup) { + if (test->ops.ethdev_setup(test, &opt)) { + evt_err("%s: ethdev setup failed", opt.test_name); + goto mempool_destroy; + } + } + + /* Test specific eventdev setup */ + if (test->ops.eventdev_setup) { + if (test->ops.eventdev_setup(test, &opt)) { + evt_err("%s: eventdev setup failed", opt.test_name); + goto ethdev_destroy; + } + } + + /* Launch lcores */ + if (test->ops.launch_lcores) { + if (test->ops.launch_lcores(test, &opt)) { + evt_err("%s: failed to launch lcores", opt.test_name); + goto eventdev_destroy; + } + } + + rte_eal_mp_wait_lcore(); + + /* Print the test result */ + ret = test->ops.test_result(test, &opt); +nocap: + if (ret == EVT_TEST_SUCCESS) { + printf("Result: "CLGRN"%s"CLNRM"\n", "Success"); + } else if (ret == EVT_TEST_FAILED) { + printf("Result: "CLRED"%s"CLNRM"\n", "Failed"); + return EXIT_FAILURE; + } else if (ret == EVT_TEST_UNSUPPORTED) { + printf("Result: "CLYEL"%s"CLNRM"\n", "Unsupported"); + } + + return 0; +eventdev_destroy: + if (test->ops.eventdev_destroy) + test->ops.eventdev_destroy(test, &opt); + +ethdev_destroy: + if (test->ops.ethdev_destroy) + test->ops.ethdev_destroy(test, &opt); + +mempool_destroy: + if (test->ops.mempool_destroy) + test->ops.mempool_destroy(test, &opt); + +test_destroy: + if (test->ops.test_destroy) + test->ops.test_destroy(test, &opt); +error: + return EXIT_FAILURE; +} diff --git a/dpdk/app/test-eventdev/evt_options.c b/dpdk/app/test-eventdev/evt_options.c new file mode 100644 index 00000000..e2187dfc --- /dev/null +++ b/dpdk/app/test-eventdev/evt_options.c @@ -0,0 +1,331 @@ +/* + * BSD LICENSE + * + * Copyright (C) Cavium, Inc 2017. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include + +#include +#include +#include + +#include "evt_options.h" +#include "evt_test.h" +#include "parser.h" + +void +evt_options_default(struct evt_options *opt) +{ + memset(opt, 0, sizeof(*opt)); + opt->verbose_level = 1; /* Enable minimal prints */ + opt->dev_id = 0; + strncpy(opt->test_name, "order_queue", EVT_TEST_NAME_MAX_LEN); + opt->nb_flows = 1024; + opt->socket_id = SOCKET_ID_ANY; + opt->pool_sz = 16 * 1024; + opt->wkr_deq_dep = 16; + opt->nb_pkts = (1ULL << 26); /* do ~64M packets */ +} + +typedef int (*option_parser_t)(struct evt_options *opt, + const char *arg); + +struct long_opt_parser { + const char *lgopt_name; + option_parser_t parser_fn; +}; + +static int +evt_parse_nb_flows(struct evt_options *opt, const char *arg) +{ + int ret; + + ret = parser_read_uint32(&(opt->nb_flows), arg); + + return ret; +} + +static int +evt_parse_dev_id(struct evt_options *opt, const char *arg) +{ + int ret; + + ret = parser_read_uint8(&(opt->dev_id), arg); + + return ret; +} + +static int +evt_parse_verbose(struct evt_options *opt, const char *arg __rte_unused) +{ + opt->verbose_level = atoi(arg); + return 0; +} + +static int +evt_parse_fwd_latency(struct evt_options *opt, const char *arg __rte_unused) +{ + opt->fwd_latency = 1; + return 0; +} + +static int +evt_parse_queue_priority(struct evt_options *opt, const char *arg __rte_unused) +{ + opt->q_priority = 1; + return 0; +} + +static int +evt_parse_test_name(struct evt_options *opt, const char *arg) +{ + snprintf(opt->test_name, EVT_TEST_NAME_MAX_LEN, "%s", arg); + return 0; +} + +static int +evt_parse_socket_id(struct evt_options *opt, const char *arg) +{ + opt->socket_id = atoi(arg); + return 0; +} + +static int +evt_parse_wkr_deq_dep(struct evt_options *opt, const char *arg) +{ + int ret; + + ret = parser_read_uint16(&(opt->wkr_deq_dep), arg); + return ret; +} + +static int +evt_parse_nb_pkts(struct evt_options *opt, const char *arg) +{ + int ret; + + ret = parser_read_uint64(&(opt->nb_pkts), arg); + + return ret; +} + +static int +evt_parse_pool_sz(struct evt_options *opt, const char *arg) +{ + opt->pool_sz = atoi(arg); + + return 0; +} + +static int +evt_parse_plcores(struct evt_options *opt, const char *corelist) +{ + int ret; + + ret = parse_lcores_list(opt->plcores, corelist); + if (ret == -E2BIG) + evt_err("duplicate lcores in plcores"); + + return ret; +} + +static int +evt_parse_work_lcores(struct evt_options *opt, const char *corelist) +{ + int ret; + + ret = parse_lcores_list(opt->wlcores, corelist); + if (ret == -E2BIG) + evt_err("duplicate lcores in wlcores"); + + return ret; +} + +static void +usage(char *program) +{ + printf("usage : %s [EAL options] -- [application options]\n", program); + printf("application options:\n"); + printf("\t--verbose : verbose level\n" + "\t--dev : device id of the event device\n" + "\t--test : name of the test application to run\n" + "\t--socket_id : socket_id of application resources\n" + "\t--pool_sz : pool size of the mempool\n" + "\t--plcores : list of lcore ids for producers\n" + "\t--wlcores : list of lcore ids for workers\n" + "\t--stlist : list of scheduled types of the stages\n" + "\t--nb_flows : number of flows to produce\n" + "\t--nb_pkts : number of packets to produce\n" + "\t--worker_deq_depth : dequeue depth of the worker\n" + "\t--fwd_latency : perform fwd_latency measurement\n" + "\t--queue_priority : enable queue priority\n" + ); + printf("available tests:\n"); + evt_test_dump_names(); +} + +static int +evt_parse_sched_type_list(struct evt_options *opt, const char *arg) +{ + char c; + int i = 0, j = -1; + + for (i = 0; i < EVT_MAX_STAGES; i++) + opt->sched_type_list[i] = (uint8_t)-1; + + i = 0; + + do { + c = arg[++j]; + + switch (c) { + case 'o': + case 'O': + opt->sched_type_list[i++] = RTE_SCHED_TYPE_ORDERED; + break; + case 'a': + case 'A': + opt->sched_type_list[i++] = RTE_SCHED_TYPE_ATOMIC; + break; + case 'p': + case 'P': + opt->sched_type_list[i++] = RTE_SCHED_TYPE_PARALLEL; + break; + case ',': + break; + default: + if (c != '\0') { + evt_err("invalid sched_type %c", c); + return -EINVAL; + } + } + } while (c != '\0'); + + opt->nb_stages = i; + return 0; +} + +static struct option lgopts[] = { + { EVT_NB_FLOWS, 1, 0, 0 }, + { EVT_DEVICE, 1, 0, 0 }, + { EVT_VERBOSE, 1, 0, 0 }, + { EVT_TEST, 1, 0, 0 }, + { EVT_PROD_LCORES, 1, 0, 0 }, + { EVT_WORK_LCORES, 1, 0, 0 }, + { EVT_SOCKET_ID, 1, 0, 0 }, + { EVT_POOL_SZ, 1, 0, 0 }, + { EVT_NB_PKTS, 1, 0, 0 }, + { EVT_WKR_DEQ_DEP, 1, 0, 0 }, + { EVT_SCHED_TYPE_LIST, 1, 0, 0 }, + { EVT_FWD_LATENCY, 0, 0, 0 }, + { EVT_QUEUE_PRIORITY, 0, 0, 0 }, + { EVT_HELP, 0, 0, 0 }, + { NULL, 0, 0, 0 } +}; + +static int +evt_opts_parse_long(int opt_idx, struct evt_options *opt) +{ + unsigned int i; + + struct long_opt_parser parsermap[] = { + { EVT_NB_FLOWS, evt_parse_nb_flows}, + { EVT_DEVICE, evt_parse_dev_id}, + { EVT_VERBOSE, evt_parse_verbose}, + { EVT_TEST, evt_parse_test_name}, + { EVT_PROD_LCORES, evt_parse_plcores}, + { EVT_WORK_LCORES, evt_parse_work_lcores}, + { EVT_SOCKET_ID, evt_parse_socket_id}, + { EVT_POOL_SZ, evt_parse_pool_sz}, + { EVT_NB_PKTS, evt_parse_nb_pkts}, + { EVT_WKR_DEQ_DEP, evt_parse_wkr_deq_dep}, + { EVT_SCHED_TYPE_LIST, evt_parse_sched_type_list}, + { EVT_FWD_LATENCY, evt_parse_fwd_latency}, + { EVT_QUEUE_PRIORITY, evt_parse_queue_priority}, + }; + + for (i = 0; i < RTE_DIM(parsermap); i++) { + if (strncmp(lgopts[opt_idx].name, parsermap[i].lgopt_name, + strlen(parsermap[i].lgopt_name)) == 0) + return parsermap[i].parser_fn(opt, optarg); + } + + return -EINVAL; +} + +int +evt_options_parse(struct evt_options *opt, int argc, char **argv) +{ + int opts, retval, opt_idx; + + while ((opts = getopt_long(argc, argv, "", lgopts, &opt_idx)) != EOF) { + switch (opts) { + case 0: /* long options */ + if (!strcmp(lgopts[opt_idx].name, "help")) { + usage(argv[0]); + exit(EXIT_SUCCESS); + } + + retval = evt_opts_parse_long(opt_idx, opt); + if (retval != 0) + return retval; + break; + default: + return -EINVAL; + } + } + return 0; +} + +void +evt_options_dump(struct evt_options *opt) +{ + int lcore_id; + struct rte_event_dev_info dev_info; + + rte_event_dev_info_get(opt->dev_id, &dev_info); + evt_dump("driver", "%s", dev_info.driver_name); + evt_dump("test", "%s", opt->test_name); + evt_dump("dev", "%d", opt->dev_id); + evt_dump("verbose_level", "%d", opt->verbose_level); + evt_dump("socket_id", "%d", opt->socket_id); + evt_dump("pool_sz", "%d", opt->pool_sz); + evt_dump("master lcore", "%d", rte_get_master_lcore()); + evt_dump("nb_pkts", "%"PRIu64, opt->nb_pkts); + evt_dump_begin("available lcores"); + RTE_LCORE_FOREACH(lcore_id) + printf("%d ", lcore_id); + evt_dump_end; + evt_dump_nb_flows(opt); + evt_dump_worker_dequeue_depth(opt); +} diff --git a/dpdk/app/test-eventdev/evt_options.h b/dpdk/app/test-eventdev/evt_options.h new file mode 100644 index 00000000..a9a91252 --- /dev/null +++ b/dpdk/app/test-eventdev/evt_options.h @@ -0,0 +1,269 @@ +/* + * BSD LICENSE + * + * Copyright (C) Cavium, Inc 2017. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _EVT_OPTIONS_ +#define _EVT_OPTIONS_ + +#include +#include + +#include +#include +#include + +#include "evt_common.h" + +#define EVT_BOOL_FMT(x) ((x) ? "true" : "false") + +#define EVT_VERBOSE ("verbose") +#define EVT_DEVICE ("dev") +#define EVT_TEST ("test") +#define EVT_PROD_LCORES ("plcores") +#define EVT_WORK_LCORES ("wlcores") +#define EVT_NB_FLOWS ("nb_flows") +#define EVT_SOCKET_ID ("socket_id") +#define EVT_POOL_SZ ("pool_sz") +#define EVT_WKR_DEQ_DEP ("worker_deq_depth") +#define EVT_NB_PKTS ("nb_pkts") +#define EVT_NB_STAGES ("nb_stages") +#define EVT_SCHED_TYPE_LIST ("stlist") +#define EVT_FWD_LATENCY ("fwd_latency") +#define EVT_QUEUE_PRIORITY ("queue_priority") +#define EVT_HELP ("help") + +struct evt_options { +#define EVT_TEST_NAME_MAX_LEN 32 + char test_name[EVT_TEST_NAME_MAX_LEN]; + bool plcores[RTE_MAX_LCORE]; + bool wlcores[RTE_MAX_LCORE]; + uint8_t sched_type_list[EVT_MAX_STAGES]; + uint32_t nb_flows; + int socket_id; + int pool_sz; + int nb_stages; + int verbose_level; + uint64_t nb_pkts; + uint16_t wkr_deq_dep; + uint8_t dev_id; + uint32_t fwd_latency:1; + uint32_t q_priority:1; +}; + +void evt_options_default(struct evt_options *opt); +int evt_options_parse(struct evt_options *opt, int argc, char **argv); +void evt_options_dump(struct evt_options *opt); + +/* options check helpers */ +static inline bool +evt_lcores_has_overlap(bool lcores[], int lcore) +{ + if (lcores[lcore] == true) { + evt_err("lcore overlaps at %d", lcore); + return true; + } + + return false; +} + +static inline bool +evt_lcores_has_overlap_multi(bool lcoresx[], bool lcoresy[]) +{ + int i; + + for (i = 0; i < RTE_MAX_LCORE; i++) { + if (lcoresx[i] && lcoresy[i]) { + evt_err("lcores overlaps at %d", i); + return true; + } + } + return false; +} + +static inline bool +evt_has_active_lcore(bool lcores[]) +{ + int i; + + for (i = 0; i < RTE_MAX_LCORE; i++) + if (lcores[i]) + return true; + return false; +} + +static inline int +evt_nr_active_lcores(bool lcores[]) +{ + int i; + int c = 0; + + for (i = 0; i < RTE_MAX_LCORE; i++) + if (lcores[i]) + c++; + return c; +} + +static inline int +evt_get_first_active_lcore(bool lcores[]) +{ + int i; + + for (i = 0; i < RTE_MAX_LCORE; i++) + if (lcores[i]) + return i; + return -1; +} + +static inline bool +evt_has_disabled_lcore(bool lcores[]) +{ + int i; + + for (i = 0; i < RTE_MAX_LCORE; i++) + if ((lcores[i] == true) && !(rte_lcore_is_enabled(i))) + return true; + return false; +} + +static inline bool +evt_has_invalid_stage(struct evt_options *opt) +{ + if (!opt->nb_stages) { + evt_err("need minimum one stage, check --stlist"); + return true; + } + if (opt->nb_stages > EVT_MAX_STAGES) { + evt_err("requested changes are beyond EVT_MAX_STAGES=%d", + EVT_MAX_STAGES); + return true; + } + return false; +} + +static inline bool +evt_has_invalid_sched_type(struct evt_options *opt) +{ + int i; + + for (i = 0; i < opt->nb_stages; i++) { + if (opt->sched_type_list[i] > RTE_SCHED_TYPE_PARALLEL) { + evt_err("invalid sched_type %d at %d", + opt->sched_type_list[i], i); + return true; + } + } + return false; +} + +/* option dump helpers */ +static inline void +evt_dump_worker_lcores(struct evt_options *opt) +{ + int c; + + evt_dump_begin("worker lcores"); + for (c = 0; c < RTE_MAX_LCORE; c++) { + if (opt->wlcores[c]) + printf("%d ", c); + } + evt_dump_end; +} + +static inline void +evt_dump_producer_lcores(struct evt_options *opt) +{ + int c; + + evt_dump_begin("producer lcores"); + for (c = 0; c < RTE_MAX_LCORE; c++) { + if (opt->plcores[c]) + printf("%d ", c); + } + evt_dump_end; +} + +static inline void +evt_dump_nb_flows(struct evt_options *opt) +{ + evt_dump("nb_flows", "%d", opt->nb_flows); +} + +static inline void +evt_dump_worker_dequeue_depth(struct evt_options *opt) +{ + evt_dump("worker deq depth", "%d", opt->wkr_deq_dep); +} + +static inline void +evt_dump_nb_stages(struct evt_options *opt) +{ + evt_dump("nb_stages", "%d", opt->nb_stages); +} + +static inline void +evt_dump_fwd_latency(struct evt_options *opt) +{ + evt_dump("fwd_latency", "%s", EVT_BOOL_FMT(opt->fwd_latency)); +} + +static inline void +evt_dump_queue_priority(struct evt_options *opt) +{ + evt_dump("queue_priority", "%s", EVT_BOOL_FMT(opt->q_priority)); +} + +static inline const char* +evt_sched_type_2_str(uint8_t sched_type) +{ + + if (sched_type == RTE_SCHED_TYPE_ORDERED) + return "O"; + else if (sched_type == RTE_SCHED_TYPE_ATOMIC) + return "A"; + else if (sched_type == RTE_SCHED_TYPE_PARALLEL) + return "P"; + else + return "I"; +} + +static inline void +evt_dump_sched_type_list(struct evt_options *opt) +{ + int i; + + evt_dump_begin("sched_type_list"); + for (i = 0; i < opt->nb_stages; i++) + printf("%s ", evt_sched_type_2_str(opt->sched_type_list[i])); + + evt_dump_end; +} + +#endif /* _EVT_OPTIONS_ */ diff --git a/dpdk/app/test-eventdev/evt_test.c b/dpdk/app/test-eventdev/evt_test.c new file mode 100644 index 00000000..3a432233 --- /dev/null +++ b/dpdk/app/test-eventdev/evt_test.c @@ -0,0 +1,70 @@ +/* + * BSD LICENSE + * + * Copyright (C) Cavium, Inc. 2017. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include + +#include "evt_test.h" + +static STAILQ_HEAD(, evt_test_entry) head = STAILQ_HEAD_INITIALIZER(head); + +void +evt_test_register(struct evt_test_entry *entry) +{ + STAILQ_INSERT_TAIL(&head, entry, next); +} + +struct evt_test* +evt_test_get(const char *name) +{ + struct evt_test_entry *entry; + + if (!name) + return NULL; + + STAILQ_FOREACH(entry, &head, next) + if (!strncmp(entry->test.name, name, strlen(name))) + return &entry->test; + + return NULL; +} + +void +evt_test_dump_names(void) +{ + struct evt_test_entry *entry; + + STAILQ_FOREACH(entry, &head, next) + if (entry->test.name) + printf("\t %s\n", entry->test.name); +} diff --git a/dpdk/app/test-eventdev/evt_test.h b/dpdk/app/test-eventdev/evt_test.h new file mode 100644 index 00000000..17bdd165 --- /dev/null +++ b/dpdk/app/test-eventdev/evt_test.h @@ -0,0 +1,125 @@ +/* + * BSD LICENSE + * + * Copyright (C) Cavium, Inc 2017. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _EVT_TEST_ +#define _EVT_TEST_ + +#include +#include +#include + +#include + +enum evt_test_result { + EVT_TEST_SUCCESS, + EVT_TEST_FAILED, + EVT_TEST_UNSUPPORTED, +}; + +struct evt_test; +struct evt_options; + +typedef bool (*evt_test_capability_check_t)(struct evt_options *opt); +typedef int (*evt_test_options_check_t)(struct evt_options *opt); +typedef void (*evt_test_options_dump_t)(struct evt_options *opt); +typedef int (*evt_test_setup_t) + (struct evt_test *test, struct evt_options *opt); +typedef int (*evt_test_mempool_setup_t) + (struct evt_test *test, struct evt_options *opt); +typedef int (*evt_test_ethdev_setup_t) + (struct evt_test *test, struct evt_options *opt); +typedef int (*evt_test_eventdev_setup_t) + (struct evt_test *test, struct evt_options *opt); +typedef int (*evt_test_launch_lcores_t) + (struct evt_test *test, struct evt_options *opt); +typedef int (*evt_test_result_t) + (struct evt_test *test, struct evt_options *opt); +typedef void (*evt_test_eventdev_destroy_t) + (struct evt_test *test, struct evt_options *opt); +typedef void (*evt_test_ethdev_destroy_t) + (struct evt_test *test, struct evt_options *opt); +typedef void (*evt_test_mempool_destroy_t) + (struct evt_test *test, struct evt_options *opt); +typedef void (*evt_test_destroy_t) + (struct evt_test *test, struct evt_options *opt); + +struct evt_test_ops { + evt_test_capability_check_t cap_check; + evt_test_options_check_t opt_check; + evt_test_options_dump_t opt_dump; + evt_test_setup_t test_setup; + evt_test_mempool_setup_t mempool_setup; + evt_test_ethdev_setup_t ethdev_setup; + evt_test_eventdev_setup_t eventdev_setup; + evt_test_launch_lcores_t launch_lcores; + evt_test_result_t test_result; + evt_test_eventdev_destroy_t eventdev_destroy; + evt_test_ethdev_destroy_t ethdev_destroy; + evt_test_mempool_destroy_t mempool_destroy; + evt_test_destroy_t test_destroy; +}; + +struct evt_test { + const char *name; + void *test_priv; + struct evt_test_ops ops; +}; + +struct evt_test_entry { + struct evt_test test; + + STAILQ_ENTRY(evt_test_entry) next; +}; + +void evt_test_register(struct evt_test_entry *test); +void evt_test_dump_names(void); + +#define EVT_TEST_REGISTER(nm) \ +static struct evt_test_entry _evt_test_entry_ ##nm; \ +RTE_INIT(evt_test_ ##nm); \ +static void evt_test_ ##nm(void) \ +{ \ + _evt_test_entry_ ##nm.test.name = RTE_STR(nm);\ + memcpy(&_evt_test_entry_ ##nm.test.ops, &nm, \ + sizeof(struct evt_test_ops)); \ + evt_test_register(&_evt_test_entry_ ##nm); \ +} + +struct evt_test *evt_test_get(const char *name); + +static inline void * +evt_test_priv(struct evt_test *test) +{ + return test->test_priv; +} + +#endif /* _EVT_TEST_ */ diff --git a/dpdk/app/test-eventdev/parser.c b/dpdk/app/test-eventdev/parser.c new file mode 100644 index 00000000..9de41bf4 --- /dev/null +++ b/dpdk/app/test-eventdev/parser.c @@ -0,0 +1,388 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2017 Cavium, Inc. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "parser.h" + +static uint32_t +get_hex_val(char c) +{ + switch (c) { + case '0': case '1': case '2': case '3': case '4': case '5': + case '6': case '7': case '8': case '9': + return c - '0'; + case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': + return c - 'A' + 10; + case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': + return c - 'a' + 10; + default: + return 0; + } +} + +int +parser_read_arg_bool(const char *p) +{ + p = skip_white_spaces(p); + int result = -EINVAL; + + if (((p[0] == 'y') && (p[1] == 'e') && (p[2] == 's')) || + ((p[0] == 'Y') && (p[1] == 'E') && (p[2] == 'S'))) { + p += 3; + result = 1; + } + + if (((p[0] == 'o') && (p[1] == 'n')) || + ((p[0] == 'O') && (p[1] == 'N'))) { + p += 2; + result = 1; + } + + if (((p[0] == 'n') && (p[1] == 'o')) || + ((p[0] == 'N') && (p[1] == 'O'))) { + p += 2; + result = 0; + } + + if (((p[0] == 'o') && (p[1] == 'f') && (p[2] == 'f')) || + ((p[0] == 'O') && (p[1] == 'F') && (p[2] == 'F'))) { + p += 3; + result = 0; + } + + p = skip_white_spaces(p); + + if (p[0] != '\0') + return -EINVAL; + + return result; +} + +int +parser_read_uint64(uint64_t *value, const char *p) +{ + char *next; + uint64_t val; + + p = skip_white_spaces(p); + if (!isdigit(*p)) + return -EINVAL; + + val = strtoul(p, &next, 10); + if (p == next) + return -EINVAL; + + p = next; + switch (*p) { + case 'T': + val *= 1024ULL; + /* fall through */ + case 'G': + val *= 1024ULL; + /* fall through */ + case 'M': + val *= 1024ULL; + /* fall through */ + case 'k': + case 'K': + val *= 1024ULL; + p++; + break; + } + + p = skip_white_spaces(p); + if (*p != '\0') + return -EINVAL; + + *value = val; + return 0; +} + +int +parser_read_int32(int32_t *value, const char *p) +{ + char *next; + int32_t val; + + p = skip_white_spaces(p); + if (!isdigit(*p)) + return -EINVAL; + + val = strtol(p, &next, 10); + if (p == next) + return -EINVAL; + + *value = val; + return 0; +} + +int +parser_read_uint64_hex(uint64_t *value, const char *p) +{ + char *next; + uint64_t val; + + p = skip_white_spaces(p); + + val = strtoul(p, &next, 16); + if (p == next) + return -EINVAL; + + p = skip_white_spaces(next); + if (*p != '\0') + return -EINVAL; + + *value = val; + return 0; +} + +int +parser_read_uint32(uint32_t *value, const char *p) +{ + uint64_t val = 0; + int ret = parser_read_uint64(&val, p); + + if (ret < 0) + return ret; + + if (val > UINT32_MAX) + return -ERANGE; + + *value = val; + return 0; +} + +int +parser_read_uint32_hex(uint32_t *value, const char *p) +{ + uint64_t val = 0; + int ret = parser_read_uint64_hex(&val, p); + + if (ret < 0) + return ret; + + if (val > UINT32_MAX) + return -ERANGE; + + *value = val; + return 0; +} + +int +parser_read_uint16(uint16_t *value, const char *p) +{ + uint64_t val = 0; + int ret = parser_read_uint64(&val, p); + + if (ret < 0) + return ret; + + if (val > UINT16_MAX) + return -ERANGE; + + *value = val; + return 0; +} + +int +parser_read_uint16_hex(uint16_t *value, const char *p) +{ + uint64_t val = 0; + int ret = parser_read_uint64_hex(&val, p); + + if (ret < 0) + return ret; + + if (val > UINT16_MAX) + return -ERANGE; + + *value = val; + return 0; +} + +int +parser_read_uint8(uint8_t *value, const char *p) +{ + uint64_t val = 0; + int ret = parser_read_uint64(&val, p); + + if (ret < 0) + return ret; + + if (val > UINT8_MAX) + return -ERANGE; + + *value = val; + return 0; +} + +int +parser_read_uint8_hex(uint8_t *value, const char *p) +{ + uint64_t val = 0; + int ret = parser_read_uint64_hex(&val, p); + + if (ret < 0) + return ret; + + if (val > UINT8_MAX) + return -ERANGE; + + *value = val; + return 0; +} + +int +parse_tokenize_string(char *string, char *tokens[], uint32_t *n_tokens) +{ + uint32_t i; + + if ((string == NULL) || + (tokens == NULL) || + (*n_tokens < 1)) + return -EINVAL; + + for (i = 0; i < *n_tokens; i++) { + tokens[i] = strtok_r(string, PARSE_DELIMITER, &string); + if (tokens[i] == NULL) + break; + } + + if ((i == *n_tokens) && + (strtok_r(string, PARSE_DELIMITER, &string) != NULL)) + return -E2BIG; + + *n_tokens = i; + return 0; +} + +int +parse_hex_string(char *src, uint8_t *dst, uint32_t *size) +{ + char *c; + uint32_t len, i; + + /* Check input parameters */ + if ((src == NULL) || + (dst == NULL) || + (size == NULL) || + (*size == 0)) + return -1; + + len = strlen(src); + if (((len & 3) != 0) || + (len > (*size) * 2)) + return -1; + *size = len / 2; + + for (c = src; *c != 0; c++) { + if ((((*c) >= '0') && ((*c) <= '9')) || + (((*c) >= 'A') && ((*c) <= 'F')) || + (((*c) >= 'a') && ((*c) <= 'f'))) + continue; + + return -1; + } + + /* Convert chars to bytes */ + for (i = 0; i < *size; i++) + dst[i] = get_hex_val(src[2 * i]) * 16 + + get_hex_val(src[2 * i + 1]); + + return 0; +} + +int +parse_lcores_list(bool lcores[], const char *corelist) +{ + int i, idx = 0; + int min, max; + char *end = NULL; + + if (corelist == NULL) + return -1; + while (isblank(*corelist)) + corelist++; + i = strlen(corelist); + while ((i > 0) && isblank(corelist[i - 1])) + i--; + + /* Get list of lcores */ + min = RTE_MAX_LCORE; + do { + while (isblank(*corelist)) + corelist++; + if (*corelist == '\0') + return -1; + idx = strtoul(corelist, &end, 10); + + if (end == NULL) + return -1; + while (isblank(*end)) + end++; + if (*end == '-') { + min = idx; + } else if ((*end == ',') || (*end == '\0')) { + max = idx; + if (min == RTE_MAX_LCORE) + min = idx; + for (idx = min; idx <= max; idx++) { + if (lcores[idx] == 1) + return -E2BIG; + lcores[idx] = 1; + } + + min = RTE_MAX_LCORE; + } else + return -1; + corelist = end + 1; + } while (*end != '\0'); + + return 0; +} diff --git a/dpdk/app/test-eventdev/parser.h b/dpdk/app/test-eventdev/parser.h new file mode 100644 index 00000000..75a5a3b4 --- /dev/null +++ b/dpdk/app/test-eventdev/parser.h @@ -0,0 +1,79 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __INCLUDE_PARSER_H__ +#define __INCLUDE_PARSER_H__ + +#include + +#define PARSE_DELIMITER " \f\n\r\t\v" + +#define skip_white_spaces(pos) \ +({ \ + __typeof__(pos) _p = (pos); \ + for ( ; isspace(*_p); _p++) \ + ; \ + _p; \ +}) + +static inline size_t +skip_digits(const char *src) +{ + size_t i; + + for (i = 0; isdigit(src[i]); i++) + ; + + return i; +} + +int parser_read_arg_bool(const char *p); + +int parser_read_uint64(uint64_t *value, const char *p); +int parser_read_uint32(uint32_t *value, const char *p); +int parser_read_uint16(uint16_t *value, const char *p); +int parser_read_uint8(uint8_t *value, const char *p); + +int parser_read_uint64_hex(uint64_t *value, const char *p); +int parser_read_uint32_hex(uint32_t *value, const char *p); +int parser_read_uint16_hex(uint16_t *value, const char *p); +int parser_read_uint8_hex(uint8_t *value, const char *p); + +int parser_read_int32(int32_t *value, const char *p); + +int parse_hex_string(char *src, uint8_t *dst, uint32_t *size); + +int parse_tokenize_string(char *string, char *tokens[], uint32_t *n_tokens); + +int parse_lcores_list(bool lcores[], const char *corelist); +#endif diff --git a/dpdk/app/test-eventdev/test_order_atq.c b/dpdk/app/test-eventdev/test_order_atq.c new file mode 100644 index 00000000..4ee0dea8 --- /dev/null +++ b/dpdk/app/test-eventdev/test_order_atq.c @@ -0,0 +1,238 @@ +/* + * BSD LICENSE + * + * Copyright (C) Cavium, Inc 2017. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include + +#include "test_order_common.h" + +/* See http://dpdk.org/doc/guides/tools/testeventdev.html for test details */ + +static inline __attribute__((always_inline)) void +order_atq_process_stage_0(struct rte_event *const ev) +{ + ev->sub_event_type = 1; /* move to stage 1 (atomic) on the same queue */ + ev->op = RTE_EVENT_OP_FORWARD; + ev->sched_type = RTE_SCHED_TYPE_ATOMIC; + ev->event_type = RTE_EVENT_TYPE_CPU; +} + +static int +order_atq_worker(void *arg) +{ + ORDER_WORKER_INIT; + struct rte_event ev; + + while (t->err == false) { + uint16_t event = rte_event_dequeue_burst(dev_id, port, + &ev, 1, 0); + if (!event) { + if (rte_atomic64_read(outstand_pkts) <= 0) + break; + rte_pause(); + continue; + } + + if (ev.sub_event_type == 0) { /* stage 0 from producer */ + order_atq_process_stage_0(&ev); + while (rte_event_enqueue_burst(dev_id, port, &ev, 1) + != 1) + rte_pause(); + } else if (ev.sub_event_type == 1) { /* stage 1 */ + order_process_stage_1(t, &ev, nb_flows, + expected_flow_seq, outstand_pkts); + } else { + order_process_stage_invalid(t, &ev); + } + } + return 0; +} + +static int +order_atq_worker_burst(void *arg) +{ + ORDER_WORKER_INIT; + struct rte_event ev[BURST_SIZE]; + uint16_t i; + + while (t->err == false) { + uint16_t const nb_rx = rte_event_dequeue_burst(dev_id, port, ev, + BURST_SIZE, 0); + + if (nb_rx == 0) { + if (rte_atomic64_read(outstand_pkts) <= 0) + break; + rte_pause(); + continue; + } + + for (i = 0; i < nb_rx; i++) { + if (ev[i].sub_event_type == 0) { /*stage 0 */ + order_atq_process_stage_0(&ev[i]); + } else if (ev[i].sub_event_type == 1) { /* stage 1 */ + order_process_stage_1(t, &ev[i], nb_flows, + expected_flow_seq, outstand_pkts); + ev[i].op = RTE_EVENT_OP_RELEASE; + } else { + order_process_stage_invalid(t, &ev[i]); + } + } + + uint16_t enq; + + enq = rte_event_enqueue_burst(dev_id, port, ev, nb_rx); + while (enq < nb_rx) { + enq += rte_event_enqueue_burst(dev_id, port, + ev + enq, nb_rx - enq); + } + } + return 0; +} + +static int +worker_wrapper(void *arg) +{ + struct worker_data *w = arg; + const bool burst = evt_has_burst_mode(w->dev_id); + + if (burst) + return order_atq_worker_burst(arg); + else + return order_atq_worker(arg); +} + +static int +order_atq_launch_lcores(struct evt_test *test, struct evt_options *opt) +{ + return order_launch_lcores(test, opt, worker_wrapper); +} + +#define NB_QUEUES 1 +static int +order_atq_eventdev_setup(struct evt_test *test, struct evt_options *opt) +{ + int ret; + + const uint8_t nb_workers = evt_nr_active_lcores(opt->wlcores); + /* number of active worker cores + 1 producer */ + const uint8_t nb_ports = nb_workers + 1; + + const struct rte_event_dev_config config = { + .nb_event_queues = NB_QUEUES,/* one all types queue */ + .nb_event_ports = nb_ports, + .nb_events_limit = 4096, + .nb_event_queue_flows = opt->nb_flows, + .nb_event_port_dequeue_depth = 128, + .nb_event_port_enqueue_depth = 128, + }; + + ret = rte_event_dev_configure(opt->dev_id, &config); + if (ret) { + evt_err("failed to configure eventdev %d", opt->dev_id); + return ret; + } + + /* q0 all types queue configuration */ + struct rte_event_queue_conf q0_conf = { + .priority = RTE_EVENT_DEV_PRIORITY_NORMAL, + .event_queue_cfg = RTE_EVENT_QUEUE_CFG_ALL_TYPES, + .nb_atomic_flows = opt->nb_flows, + .nb_atomic_order_sequences = opt->nb_flows, + }; + ret = rte_event_queue_setup(opt->dev_id, 0, &q0_conf); + if (ret) { + evt_err("failed to setup queue0 eventdev %d", opt->dev_id); + return ret; + } + + /* setup one port per worker, linking to all queues */ + ret = order_event_dev_port_setup(test, opt, nb_workers, NB_QUEUES); + if (ret) + return ret; + + ret = evt_service_setup(opt->dev_id); + if (ret) { + evt_err("No service lcore found to run event dev."); + return ret; + } + + ret = rte_event_dev_start(opt->dev_id); + if (ret) { + evt_err("failed to start eventdev %d", opt->dev_id); + return ret; + } + + return 0; +} + +static void +order_atq_opt_dump(struct evt_options *opt) +{ + order_opt_dump(opt); + evt_dump("nb_evdev_queues", "%d", NB_QUEUES); +} + +static bool +order_atq_capability_check(struct evt_options *opt) +{ + struct rte_event_dev_info dev_info; + + rte_event_dev_info_get(opt->dev_id, &dev_info); + if (dev_info.max_event_queues < NB_QUEUES || dev_info.max_event_ports < + order_nb_event_ports(opt)) { + evt_err("not enough eventdev queues=%d/%d or ports=%d/%d", + NB_QUEUES, dev_info.max_event_queues, + order_nb_event_ports(opt), dev_info.max_event_ports); + return false; + } + + if (!evt_has_all_types_queue(opt->dev_id)) + return false; + + return true; +} + +static const struct evt_test_ops order_atq = { + .cap_check = order_atq_capability_check, + .opt_check = order_opt_check, + .opt_dump = order_atq_opt_dump, + .test_setup = order_test_setup, + .mempool_setup = order_mempool_setup, + .eventdev_setup = order_atq_eventdev_setup, + .launch_lcores = order_atq_launch_lcores, + .eventdev_destroy = order_eventdev_destroy, + .mempool_destroy = order_mempool_destroy, + .test_result = order_test_result, + .test_destroy = order_test_destroy, +}; + +EVT_TEST_REGISTER(order_atq); diff --git a/dpdk/app/test-eventdev/test_order_common.c b/dpdk/app/test-eventdev/test_order_common.c new file mode 100644 index 00000000..7cfe7fac --- /dev/null +++ b/dpdk/app/test-eventdev/test_order_common.c @@ -0,0 +1,377 @@ +/* + * BSD LICENSE + * + * Copyright (C) Cavium, Inc 2017. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "test_order_common.h" + +int +order_test_result(struct evt_test *test, struct evt_options *opt) +{ + RTE_SET_USED(opt); + struct test_order *t = evt_test_priv(test); + + return t->result; +} + +static inline int +order_producer(void *arg) +{ + struct prod_data *p = arg; + struct test_order *t = p->t; + struct evt_options *opt = t->opt; + const uint8_t dev_id = p->dev_id; + const uint8_t port = p->port_id; + struct rte_mempool *pool = t->pool; + const uint64_t nb_pkts = t->nb_pkts; + uint32_t *producer_flow_seq = t->producer_flow_seq; + const uint32_t nb_flows = t->nb_flows; + uint64_t count = 0; + struct rte_mbuf *m; + struct rte_event ev; + + if (opt->verbose_level > 1) + printf("%s(): lcore %d dev_id %d port=%d queue=%d\n", + __func__, rte_lcore_id(), dev_id, port, p->queue_id); + + ev.event = 0; + ev.op = RTE_EVENT_OP_NEW; + ev.queue_id = p->queue_id; + ev.sched_type = RTE_SCHED_TYPE_ORDERED; + ev.priority = RTE_EVENT_DEV_PRIORITY_NORMAL; + ev.event_type = RTE_EVENT_TYPE_CPU; + ev.sub_event_type = 0; /* stage 0 */ + + while (count < nb_pkts && t->err == false) { + m = rte_pktmbuf_alloc(pool); + if (m == NULL) + continue; + + const uint32_t flow = (uintptr_t)m % nb_flows; + /* Maintain seq number per flow */ + m->seqn = producer_flow_seq[flow]++; + + ev.flow_id = flow; + ev.mbuf = m; + + while (rte_event_enqueue_burst(dev_id, port, &ev, 1) != 1) { + if (t->err) + break; + rte_pause(); + } + + count++; + } + return 0; +} + +int +order_opt_check(struct evt_options *opt) +{ + /* 1 producer + N workers + 1 master */ + if (rte_lcore_count() < 3) { + evt_err("test need minimum 3 lcores"); + return -1; + } + + /* Validate worker lcores */ + if (evt_lcores_has_overlap(opt->wlcores, rte_get_master_lcore())) { + evt_err("worker lcores overlaps with master lcore"); + return -1; + } + + if (evt_nr_active_lcores(opt->plcores) == 0) { + evt_err("missing the producer lcore"); + return -1; + } + + if (evt_nr_active_lcores(opt->plcores) != 1) { + evt_err("only one producer lcore must be selected"); + return -1; + } + + int plcore = evt_get_first_active_lcore(opt->plcores); + + if (plcore < 0) { + evt_err("failed to find active producer"); + return plcore; + } + + if (evt_lcores_has_overlap(opt->wlcores, plcore)) { + evt_err("worker lcores overlaps producer lcore"); + return -1; + } + if (evt_has_disabled_lcore(opt->wlcores)) { + evt_err("one or more workers lcores are not enabled"); + return -1; + } + if (!evt_has_active_lcore(opt->wlcores)) { + evt_err("minimum one worker is required"); + return -1; + } + + /* Validate producer lcore */ + if (plcore == (int)rte_get_master_lcore()) { + evt_err("producer lcore and master lcore should be different"); + return -1; + } + if (!rte_lcore_is_enabled(plcore)) { + evt_err("producer lcore is not enabled"); + return -1; + } + + /* Fixups */ + if (opt->nb_pkts == 0) + opt->nb_pkts = INT64_MAX; + + return 0; +} + +int +order_test_setup(struct evt_test *test, struct evt_options *opt) +{ + void *test_order; + + test_order = rte_zmalloc_socket(test->name, sizeof(struct test_order), + RTE_CACHE_LINE_SIZE, opt->socket_id); + if (test_order == NULL) { + evt_err("failed to allocate test_order memory"); + goto nomem; + } + test->test_priv = test_order; + + struct test_order *t = evt_test_priv(test); + + t->producer_flow_seq = rte_zmalloc_socket("test_producer_flow_seq", + sizeof(*t->producer_flow_seq) * opt->nb_flows, + RTE_CACHE_LINE_SIZE, opt->socket_id); + + if (t->producer_flow_seq == NULL) { + evt_err("failed to allocate t->producer_flow_seq memory"); + goto prod_nomem; + } + + t->expected_flow_seq = rte_zmalloc_socket("test_expected_flow_seq", + sizeof(*t->expected_flow_seq) * opt->nb_flows, + RTE_CACHE_LINE_SIZE, opt->socket_id); + + if (t->expected_flow_seq == NULL) { + evt_err("failed to allocate t->expected_flow_seq memory"); + goto exp_nomem; + } + rte_atomic64_set(&t->outstand_pkts, opt->nb_pkts); + t->err = false; + t->nb_pkts = opt->nb_pkts; + t->nb_flows = opt->nb_flows; + t->result = EVT_TEST_FAILED; + t->opt = opt; + return 0; + +exp_nomem: + rte_free(t->producer_flow_seq); +prod_nomem: + rte_free(test->test_priv); +nomem: + return -ENOMEM; +} + +void +order_test_destroy(struct evt_test *test, struct evt_options *opt) +{ + RTE_SET_USED(opt); + struct test_order *t = evt_test_priv(test); + + rte_free(t->expected_flow_seq); + rte_free(t->producer_flow_seq); + rte_free(test->test_priv); +} + +int +order_mempool_setup(struct evt_test *test, struct evt_options *opt) +{ + struct test_order *t = evt_test_priv(test); + + t->pool = rte_pktmbuf_pool_create(test->name, opt->pool_sz, + 256 /* Cache */, 0, + 512, /* Use very small mbufs */ + opt->socket_id); + if (t->pool == NULL) { + evt_err("failed to create mempool"); + return -ENOMEM; + } + + return 0; +} + +void +order_mempool_destroy(struct evt_test *test, struct evt_options *opt) +{ + RTE_SET_USED(opt); + struct test_order *t = evt_test_priv(test); + + rte_mempool_free(t->pool); +} + +void +order_eventdev_destroy(struct evt_test *test, struct evt_options *opt) +{ + RTE_SET_USED(test); + + rte_event_dev_stop(opt->dev_id); + rte_event_dev_close(opt->dev_id); +} + +void +order_opt_dump(struct evt_options *opt) +{ + evt_dump_producer_lcores(opt); + evt_dump("nb_wrker_lcores", "%d", evt_nr_active_lcores(opt->wlcores)); + evt_dump_worker_lcores(opt); + evt_dump("nb_evdev_ports", "%d", order_nb_event_ports(opt)); +} + +int +order_launch_lcores(struct evt_test *test, struct evt_options *opt, + int (*worker)(void *)) +{ + int ret, lcore_id; + struct test_order *t = evt_test_priv(test); + + int wkr_idx = 0; + /* launch workers */ + RTE_LCORE_FOREACH_SLAVE(lcore_id) { + if (!(opt->wlcores[lcore_id])) + continue; + + ret = rte_eal_remote_launch(worker, &t->worker[wkr_idx], + lcore_id); + if (ret) { + evt_err("failed to launch worker %d", lcore_id); + return ret; + } + wkr_idx++; + } + + /* launch producer */ + int plcore = evt_get_first_active_lcore(opt->plcores); + + ret = rte_eal_remote_launch(order_producer, &t->prod, plcore); + if (ret) { + evt_err("failed to launch order_producer %d", plcore); + return ret; + } + + uint64_t cycles = rte_get_timer_cycles(); + int64_t old_remaining = -1; + + while (t->err == false) { + uint64_t new_cycles = rte_get_timer_cycles(); + int64_t remaining = rte_atomic64_read(&t->outstand_pkts); + + if (remaining <= 0) { + t->result = EVT_TEST_SUCCESS; + break; + } + + if (new_cycles - cycles > rte_get_timer_hz() * 1) { + printf(CLGRN"\r%"PRId64""CLNRM, remaining); + fflush(stdout); + if (old_remaining == remaining) { + rte_event_dev_dump(opt->dev_id, stdout); + evt_err("No schedules for seconds, deadlock"); + t->err = true; + rte_smp_wmb(); + break; + } + old_remaining = remaining; + cycles = new_cycles; + } + } + printf("\r"); + + return 0; +} + +int +order_event_dev_port_setup(struct evt_test *test, struct evt_options *opt, + uint8_t nb_workers, uint8_t nb_queues) +{ + int ret; + uint8_t port; + struct test_order *t = evt_test_priv(test); + + /* port configuration */ + const struct rte_event_port_conf wkr_p_conf = { + .dequeue_depth = opt->wkr_deq_dep, + .enqueue_depth = 64, + .new_event_threshold = 4096, + }; + + /* setup one port per worker, linking to all queues */ + for (port = 0; port < nb_workers; port++) { + struct worker_data *w = &t->worker[port]; + + w->dev_id = opt->dev_id; + w->port_id = port; + w->t = t; + + ret = rte_event_port_setup(opt->dev_id, port, &wkr_p_conf); + if (ret) { + evt_err("failed to setup port %d", port); + return ret; + } + + ret = rte_event_port_link(opt->dev_id, port, NULL, NULL, 0); + if (ret != nb_queues) { + evt_err("failed to link all queues to port %d", port); + return -EINVAL; + } + } + /* port for producer, no links */ + const struct rte_event_port_conf prod_conf = { + .dequeue_depth = 8, + .enqueue_depth = 32, + .new_event_threshold = 1200, + }; + struct prod_data *p = &t->prod; + + p->dev_id = opt->dev_id; + p->port_id = port; /* last port */ + p->queue_id = 0; + p->t = t; + + ret = rte_event_port_setup(opt->dev_id, port, &prod_conf); + if (ret) { + evt_err("failed to setup producer port %d", port); + return ret; + } + + return ret; +} diff --git a/dpdk/app/test-eventdev/test_order_common.h b/dpdk/app/test-eventdev/test_order_common.h new file mode 100644 index 00000000..57bc76e0 --- /dev/null +++ b/dpdk/app/test-eventdev/test_order_common.h @@ -0,0 +1,153 @@ +/* + * BSD LICENSE + * + * Copyright (C) Cavium, Inc 2017. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _TEST_ORDER_COMMON_ +#define _TEST_ORDER_COMMON_ + +#include +#include + +#include +#include +#include +#include +#include + +#include "evt_common.h" +#include "evt_options.h" +#include "evt_test.h" + +#define BURST_SIZE 16 + +struct test_order; + +struct worker_data { + uint8_t dev_id; + uint8_t port_id; + struct test_order *t; +}; + +struct prod_data { + uint8_t dev_id; + uint8_t port_id; + uint8_t queue_id; + struct test_order *t; +}; + +struct test_order { + /* Don't change the offset of "err". Signal handler use this memory + * to terminate all lcores work. + */ + int err; + /* + * The atomic_* is an expensive operation,Since it is a functional test, + * We are using the atomic_ operation to reduce the code complexity. + */ + rte_atomic64_t outstand_pkts; + enum evt_test_result result; + uint32_t nb_flows; + uint64_t nb_pkts; + struct rte_mempool *pool; + struct prod_data prod; + struct worker_data worker[EVT_MAX_PORTS]; + uint32_t *producer_flow_seq; + uint32_t *expected_flow_seq; + struct evt_options *opt; +} __rte_cache_aligned; + +static inline int +order_nb_event_ports(struct evt_options *opt) +{ + return evt_nr_active_lcores(opt->wlcores) + 1 /* producer */; +} + +static inline __attribute__((always_inline)) void +order_process_stage_1(struct test_order *const t, + struct rte_event *const ev, const uint32_t nb_flows, + uint32_t *const expected_flow_seq, + rte_atomic64_t *const outstand_pkts) +{ + const uint32_t flow = (uintptr_t)ev->mbuf % nb_flows; + /* compare the seqn against expected value */ + if (ev->mbuf->seqn != expected_flow_seq[flow]) { + evt_err("flow=%x seqn mismatch got=%x expected=%x", + flow, ev->mbuf->seqn, expected_flow_seq[flow]); + t->err = true; + rte_smp_wmb(); + } + /* + * Events from an atomic flow of an event queue can be scheduled only to + * a single port at a time. The port is guaranteed to have exclusive + * (atomic) access for given atomic flow.So we don't need to update + * expected_flow_seq in critical section. + */ + expected_flow_seq[flow]++; + rte_pktmbuf_free(ev->mbuf); + rte_atomic64_sub(outstand_pkts, 1); +} + +static inline __attribute__((always_inline)) void +order_process_stage_invalid(struct test_order *const t, + struct rte_event *const ev) +{ + evt_err("invalid queue %d", ev->queue_id); + t->err = true; + rte_smp_wmb(); +} + +#define ORDER_WORKER_INIT\ + struct worker_data *w = arg;\ + struct test_order *t = w->t;\ + struct evt_options *opt = t->opt;\ + const uint8_t dev_id = w->dev_id;\ + const uint8_t port = w->port_id;\ + const uint32_t nb_flows = t->nb_flows;\ + uint32_t *expected_flow_seq = t->expected_flow_seq;\ + rte_atomic64_t *outstand_pkts = &t->outstand_pkts;\ + if (opt->verbose_level > 1)\ + printf("%s(): lcore %d dev_id %d port=%d\n",\ + __func__, rte_lcore_id(), dev_id, port) + +int order_test_result(struct evt_test *test, struct evt_options *opt); +int order_opt_check(struct evt_options *opt); +int order_test_setup(struct evt_test *test, struct evt_options *opt); +int order_mempool_setup(struct evt_test *test, struct evt_options *opt); +int order_launch_lcores(struct evt_test *test, struct evt_options *opt, + int (*worker)(void *)); +int order_event_dev_port_setup(struct evt_test *test, struct evt_options *opt, + uint8_t nb_workers, uint8_t nb_queues); +void order_test_destroy(struct evt_test *test, struct evt_options *opt); +void order_opt_dump(struct evt_options *opt); +void order_mempool_destroy(struct evt_test *test, struct evt_options *opt); +void order_eventdev_destroy(struct evt_test *test, struct evt_options *opt); + +#endif /* _TEST_ORDER_COMMON_ */ diff --git a/dpdk/app/test-eventdev/test_order_queue.c b/dpdk/app/test-eventdev/test_order_queue.c new file mode 100644 index 00000000..eef69a4c --- /dev/null +++ b/dpdk/app/test-eventdev/test_order_queue.c @@ -0,0 +1,248 @@ +/* + * BSD LICENSE + * + * Copyright (C) Cavium, Inc 2017. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include + +#include "test_order_common.h" + +/* See http://dpdk.org/doc/guides/tools/testeventdev.html for test details */ + +static inline __attribute__((always_inline)) void +order_queue_process_stage_0(struct rte_event *const ev) +{ + ev->queue_id = 1; /* q1 atomic queue */ + ev->op = RTE_EVENT_OP_FORWARD; + ev->sched_type = RTE_SCHED_TYPE_ATOMIC; + ev->event_type = RTE_EVENT_TYPE_CPU; +} + +static int +order_queue_worker(void *arg) +{ + ORDER_WORKER_INIT; + struct rte_event ev; + + while (t->err == false) { + uint16_t event = rte_event_dequeue_burst(dev_id, port, + &ev, 1, 0); + if (!event) { + if (rte_atomic64_read(outstand_pkts) <= 0) + break; + rte_pause(); + continue; + } + + if (ev.queue_id == 0) { /* from ordered queue */ + order_queue_process_stage_0(&ev); + while (rte_event_enqueue_burst(dev_id, port, &ev, 1) + != 1) + rte_pause(); + } else if (ev.queue_id == 1) { /* from atomic queue */ + order_process_stage_1(t, &ev, nb_flows, + expected_flow_seq, outstand_pkts); + } else { + order_process_stage_invalid(t, &ev); + } + } + return 0; +} + +static int +order_queue_worker_burst(void *arg) +{ + ORDER_WORKER_INIT; + struct rte_event ev[BURST_SIZE]; + uint16_t i; + + while (t->err == false) { + uint16_t const nb_rx = rte_event_dequeue_burst(dev_id, port, ev, + BURST_SIZE, 0); + + if (nb_rx == 0) { + if (rte_atomic64_read(outstand_pkts) <= 0) + break; + rte_pause(); + continue; + } + + for (i = 0; i < nb_rx; i++) { + if (ev[i].queue_id == 0) { /* from ordered queue */ + order_queue_process_stage_0(&ev[i]); + } else if (ev[i].queue_id == 1) {/* from atomic queue */ + order_process_stage_1(t, &ev[i], nb_flows, + expected_flow_seq, outstand_pkts); + ev[i].op = RTE_EVENT_OP_RELEASE; + } else { + order_process_stage_invalid(t, &ev[i]); + } + } + + uint16_t enq; + + enq = rte_event_enqueue_burst(dev_id, port, ev, nb_rx); + while (enq < nb_rx) { + enq += rte_event_enqueue_burst(dev_id, port, + ev + enq, nb_rx - enq); + } + } + return 0; +} + +static int +worker_wrapper(void *arg) +{ + struct worker_data *w = arg; + const bool burst = evt_has_burst_mode(w->dev_id); + + if (burst) + return order_queue_worker_burst(arg); + else + return order_queue_worker(arg); +} + +static int +order_queue_launch_lcores(struct evt_test *test, struct evt_options *opt) +{ + return order_launch_lcores(test, opt, worker_wrapper); +} + +#define NB_QUEUES 2 +static int +order_queue_eventdev_setup(struct evt_test *test, struct evt_options *opt) +{ + int ret; + + const uint8_t nb_workers = evt_nr_active_lcores(opt->wlcores); + /* number of active worker cores + 1 producer */ + const uint8_t nb_ports = nb_workers + 1; + + const struct rte_event_dev_config config = { + .nb_event_queues = NB_QUEUES,/* q0 ordered, q1 atomic */ + .nb_event_ports = nb_ports, + .nb_events_limit = 4096, + .nb_event_queue_flows = opt->nb_flows, + .nb_event_port_dequeue_depth = 128, + .nb_event_port_enqueue_depth = 128, + }; + + ret = rte_event_dev_configure(opt->dev_id, &config); + if (ret) { + evt_err("failed to configure eventdev %d", opt->dev_id); + return ret; + } + + /* q0 (ordered queue) configuration */ + struct rte_event_queue_conf q0_ordered_conf = { + .priority = RTE_EVENT_DEV_PRIORITY_NORMAL, + .schedule_type = RTE_SCHED_TYPE_ORDERED, + .nb_atomic_flows = opt->nb_flows, + .nb_atomic_order_sequences = opt->nb_flows, + }; + ret = rte_event_queue_setup(opt->dev_id, 0, &q0_ordered_conf); + if (ret) { + evt_err("failed to setup queue0 eventdev %d", opt->dev_id); + return ret; + } + + /* q1 (atomic queue) configuration */ + struct rte_event_queue_conf q1_atomic_conf = { + .priority = RTE_EVENT_DEV_PRIORITY_NORMAL, + .schedule_type = RTE_SCHED_TYPE_ATOMIC, + .nb_atomic_flows = opt->nb_flows, + .nb_atomic_order_sequences = opt->nb_flows, + }; + ret = rte_event_queue_setup(opt->dev_id, 1, &q1_atomic_conf); + if (ret) { + evt_err("failed to setup queue1 eventdev %d", opt->dev_id); + return ret; + } + + /* setup one port per worker, linking to all queues */ + ret = order_event_dev_port_setup(test, opt, nb_workers, NB_QUEUES); + if (ret) + return ret; + + ret = evt_service_setup(opt->dev_id); + if (ret) { + evt_err("No service lcore found to run event dev."); + return ret; + } + + ret = rte_event_dev_start(opt->dev_id); + if (ret) { + evt_err("failed to start eventdev %d", opt->dev_id); + return ret; + } + + return 0; +} + +static void +order_queue_opt_dump(struct evt_options *opt) +{ + order_opt_dump(opt); + evt_dump("nb_evdev_queues", "%d", NB_QUEUES); +} + +static bool +order_queue_capability_check(struct evt_options *opt) +{ + struct rte_event_dev_info dev_info; + + rte_event_dev_info_get(opt->dev_id, &dev_info); + if (dev_info.max_event_queues < NB_QUEUES || dev_info.max_event_ports < + order_nb_event_ports(opt)) { + evt_err("not enough eventdev queues=%d/%d or ports=%d/%d", + NB_QUEUES, dev_info.max_event_queues, + order_nb_event_ports(opt), dev_info.max_event_ports); + return false; + } + + return true; +} + +static const struct evt_test_ops order_queue = { + .cap_check = order_queue_capability_check, + .opt_check = order_opt_check, + .opt_dump = order_queue_opt_dump, + .test_setup = order_test_setup, + .mempool_setup = order_mempool_setup, + .eventdev_setup = order_queue_eventdev_setup, + .launch_lcores = order_queue_launch_lcores, + .eventdev_destroy = order_eventdev_destroy, + .mempool_destroy = order_mempool_destroy, + .test_result = order_test_result, + .test_destroy = order_test_destroy, +}; + +EVT_TEST_REGISTER(order_queue); diff --git a/dpdk/app/test-eventdev/test_perf_atq.c b/dpdk/app/test-eventdev/test_perf_atq.c new file mode 100644 index 00000000..0e9f2db0 --- /dev/null +++ b/dpdk/app/test-eventdev/test_perf_atq.c @@ -0,0 +1,283 @@ +/* + * BSD LICENSE + * + * Copyright (C) Cavium, Inc 2017. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "test_perf_common.h" + +/* See http://dpdk.org/doc/guides/tools/testeventdev.html for test details */ + +static inline int +atq_nb_event_queues(struct evt_options *opt) +{ + /* nb_queues = number of producers */ + return evt_nr_active_lcores(opt->plcores); +} + +static inline __attribute__((always_inline)) void +atq_mark_fwd_latency(struct rte_event *const ev) +{ + if (unlikely(ev->sub_event_type == 0)) { + struct perf_elt *const m = ev->event_ptr; + + m->timestamp = rte_get_timer_cycles(); + } +} + +static inline __attribute__((always_inline)) void +atq_fwd_event(struct rte_event *const ev, uint8_t *const sched_type_list, + const uint8_t nb_stages) +{ + ev->sub_event_type++; + ev->sched_type = sched_type_list[ev->sub_event_type % nb_stages]; + ev->op = RTE_EVENT_OP_FORWARD; + ev->event_type = RTE_EVENT_TYPE_CPU; +} + +static int +perf_atq_worker(void *arg, const int enable_fwd_latency) +{ + PERF_WORKER_INIT; + struct rte_event ev; + + while (t->done == false) { + uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0); + + if (enable_fwd_latency) + rte_prefetch0(ev.event_ptr); + + if (!event) { + rte_pause(); + continue; + } + + if (enable_fwd_latency) + /* first stage in pipeline, mark ts to compute fwd latency */ + atq_mark_fwd_latency(&ev); + + /* last stage in pipeline */ + if (unlikely((ev.sub_event_type % nb_stages) == laststage)) { + if (enable_fwd_latency) + cnt = perf_process_last_stage_latency(pool, + &ev, w, bufs, sz, cnt); + else + cnt = perf_process_last_stage(pool, &ev, w, + bufs, sz, cnt); + } else { + atq_fwd_event(&ev, sched_type_list, nb_stages); + while (rte_event_enqueue_burst(dev, port, &ev, 1) != 1) + rte_pause(); + } + } + return 0; +} + +static int +perf_atq_worker_burst(void *arg, const int enable_fwd_latency) +{ + PERF_WORKER_INIT; + uint16_t i; + /* +1 to avoid prefetch out of array check */ + struct rte_event ev[BURST_SIZE + 1]; + + while (t->done == false) { + uint16_t const nb_rx = rte_event_dequeue_burst(dev, port, ev, + BURST_SIZE, 0); + + if (!nb_rx) { + rte_pause(); + continue; + } + + for (i = 0; i < nb_rx; i++) { + if (enable_fwd_latency) { + rte_prefetch0(ev[i+1].event_ptr); + /* first stage in pipeline. + * mark time stamp to compute fwd latency + */ + atq_mark_fwd_latency(&ev[i]); + } + /* last stage in pipeline */ + if (unlikely((ev[i].sub_event_type % nb_stages) + == laststage)) { + if (enable_fwd_latency) + cnt = perf_process_last_stage_latency( + pool, &ev[i], w, bufs, sz, cnt); + else + cnt = perf_process_last_stage(pool, + &ev[i], w, bufs, sz, cnt); + + ev[i].op = RTE_EVENT_OP_RELEASE; + } else { + atq_fwd_event(&ev[i], sched_type_list, + nb_stages); + } + } + + uint16_t enq; + + enq = rte_event_enqueue_burst(dev, port, ev, nb_rx); + while (enq < nb_rx) { + enq += rte_event_enqueue_burst(dev, port, + ev + enq, nb_rx - enq); + } + } + return 0; +} + +static int +worker_wrapper(void *arg) +{ + struct worker_data *w = arg; + struct evt_options *opt = w->t->opt; + + const bool burst = evt_has_burst_mode(w->dev_id); + const int fwd_latency = opt->fwd_latency; + + /* allow compiler to optimize */ + if (!burst && !fwd_latency) + return perf_atq_worker(arg, 0); + else if (!burst && fwd_latency) + return perf_atq_worker(arg, 1); + else if (burst && !fwd_latency) + return perf_atq_worker_burst(arg, 0); + else if (burst && fwd_latency) + return perf_atq_worker_burst(arg, 1); + + rte_panic("invalid worker\n"); +} + +static int +perf_atq_launch_lcores(struct evt_test *test, struct evt_options *opt) +{ + return perf_launch_lcores(test, opt, worker_wrapper); +} + +static int +perf_atq_eventdev_setup(struct evt_test *test, struct evt_options *opt) +{ + int ret; + uint8_t queue; + + const struct rte_event_dev_config config = { + .nb_event_queues = atq_nb_event_queues(opt), + .nb_event_ports = perf_nb_event_ports(opt), + .nb_events_limit = 4096, + .nb_event_queue_flows = opt->nb_flows, + .nb_event_port_dequeue_depth = 128, + .nb_event_port_enqueue_depth = 128, + }; + + ret = rte_event_dev_configure(opt->dev_id, &config); + if (ret) { + evt_err("failed to configure eventdev %d", opt->dev_id); + return ret; + } + + struct rte_event_queue_conf q_conf = { + .priority = RTE_EVENT_DEV_PRIORITY_NORMAL, + .event_queue_cfg = RTE_EVENT_QUEUE_CFG_ALL_TYPES, + .nb_atomic_flows = opt->nb_flows, + .nb_atomic_order_sequences = opt->nb_flows, + }; + /* queue configurations */ + for (queue = 0; queue < atq_nb_event_queues(opt); queue++) { + ret = rte_event_queue_setup(opt->dev_id, queue, &q_conf); + if (ret) { + evt_err("failed to setup queue=%d", queue); + return ret; + } + } + + ret = perf_event_dev_port_setup(test, opt, 1 /* stride */, + atq_nb_event_queues(opt)); + if (ret) + return ret; + + ret = evt_service_setup(opt->dev_id); + if (ret) { + evt_err("No service lcore found to run event dev."); + return ret; + } + + ret = rte_event_dev_start(opt->dev_id); + if (ret) { + evt_err("failed to start eventdev %d", opt->dev_id); + return ret; + } + + return 0; +} + +static void +perf_atq_opt_dump(struct evt_options *opt) +{ + perf_opt_dump(opt, atq_nb_event_queues(opt)); +} + +static int +perf_atq_opt_check(struct evt_options *opt) +{ + return perf_opt_check(opt, atq_nb_event_queues(opt)); +} + +static bool +perf_atq_capability_check(struct evt_options *opt) +{ + struct rte_event_dev_info dev_info; + + rte_event_dev_info_get(opt->dev_id, &dev_info); + if (dev_info.max_event_queues < atq_nb_event_queues(opt) || + dev_info.max_event_ports < perf_nb_event_ports(opt)) { + evt_err("not enough eventdev queues=%d/%d or ports=%d/%d", + atq_nb_event_queues(opt), dev_info.max_event_queues, + perf_nb_event_ports(opt), dev_info.max_event_ports); + } + if (!evt_has_all_types_queue(opt->dev_id)) + return false; + + return true; +} + +static const struct evt_test_ops perf_atq = { + .cap_check = perf_atq_capability_check, + .opt_check = perf_atq_opt_check, + .opt_dump = perf_atq_opt_dump, + .test_setup = perf_test_setup, + .mempool_setup = perf_mempool_setup, + .eventdev_setup = perf_atq_eventdev_setup, + .launch_lcores = perf_atq_launch_lcores, + .eventdev_destroy = perf_eventdev_destroy, + .mempool_destroy = perf_mempool_destroy, + .test_result = perf_test_result, + .test_destroy = perf_test_destroy, +}; + +EVT_TEST_REGISTER(perf_atq); diff --git a/dpdk/app/test-eventdev/test_perf_common.c b/dpdk/app/test-eventdev/test_perf_common.c new file mode 100644 index 00000000..e77b4723 --- /dev/null +++ b/dpdk/app/test-eventdev/test_perf_common.c @@ -0,0 +1,454 @@ +/* + * BSD LICENSE + * + * Copyright (C) Cavium, Inc 2017. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "test_perf_common.h" + +int +perf_test_result(struct evt_test *test, struct evt_options *opt) +{ + RTE_SET_USED(opt); + struct test_perf *t = evt_test_priv(test); + + return t->result; +} + +static inline int +perf_producer(void *arg) +{ + struct prod_data *p = arg; + struct test_perf *t = p->t; + struct evt_options *opt = t->opt; + const uint8_t dev_id = p->dev_id; + const uint8_t port = p->port_id; + struct rte_mempool *pool = t->pool; + const uint64_t nb_pkts = t->nb_pkts; + const uint32_t nb_flows = t->nb_flows; + uint32_t flow_counter = 0; + uint64_t count = 0; + struct perf_elt *m; + struct rte_event ev; + + if (opt->verbose_level > 1) + printf("%s(): lcore %d dev_id %d port=%d queue %d\n", __func__, + rte_lcore_id(), dev_id, port, p->queue_id); + + ev.event = 0; + ev.op = RTE_EVENT_OP_NEW; + ev.queue_id = p->queue_id; + ev.sched_type = t->opt->sched_type_list[0]; + ev.priority = RTE_EVENT_DEV_PRIORITY_NORMAL; + ev.event_type = RTE_EVENT_TYPE_CPU; + ev.sub_event_type = 0; /* stage 0 */ + + while (count < nb_pkts && t->done == false) { + if (rte_mempool_get(pool, (void **)&m) < 0) + continue; + + ev.flow_id = flow_counter++ % nb_flows; + ev.event_ptr = m; + m->timestamp = rte_get_timer_cycles(); + while (rte_event_enqueue_burst(dev_id, port, &ev, 1) != 1) { + if (t->done) + break; + rte_pause(); + m->timestamp = rte_get_timer_cycles(); + } + count++; + } + + return 0; +} + +static inline uint64_t +processed_pkts(struct test_perf *t) +{ + uint8_t i; + uint64_t total = 0; + + rte_smp_rmb(); + for (i = 0; i < t->nb_workers; i++) + total += t->worker[i].processed_pkts; + + return total; +} + +static inline uint64_t +total_latency(struct test_perf *t) +{ + uint8_t i; + uint64_t total = 0; + + rte_smp_rmb(); + for (i = 0; i < t->nb_workers; i++) + total += t->worker[i].latency; + + return total; +} + + +int +perf_launch_lcores(struct evt_test *test, struct evt_options *opt, + int (*worker)(void *)) +{ + int ret, lcore_id; + struct test_perf *t = evt_test_priv(test); + + int port_idx = 0; + /* launch workers */ + RTE_LCORE_FOREACH_SLAVE(lcore_id) { + if (!(opt->wlcores[lcore_id])) + continue; + + ret = rte_eal_remote_launch(worker, + &t->worker[port_idx], lcore_id); + if (ret) { + evt_err("failed to launch worker %d", lcore_id); + return ret; + } + port_idx++; + } + + /* launch producers */ + RTE_LCORE_FOREACH_SLAVE(lcore_id) { + if (!(opt->plcores[lcore_id])) + continue; + + ret = rte_eal_remote_launch(perf_producer, &t->prod[port_idx], + lcore_id); + if (ret) { + evt_err("failed to launch perf_producer %d", lcore_id); + return ret; + } + port_idx++; + } + + const uint64_t total_pkts = opt->nb_pkts * + evt_nr_active_lcores(opt->plcores); + + uint64_t dead_lock_cycles = rte_get_timer_cycles(); + int64_t dead_lock_remaining = total_pkts; + const uint64_t dead_lock_sample = rte_get_timer_hz() * 5; + + uint64_t perf_cycles = rte_get_timer_cycles(); + int64_t perf_remaining = total_pkts; + const uint64_t perf_sample = rte_get_timer_hz(); + + static float total_mpps; + static uint64_t samples; + + const uint64_t freq_mhz = rte_get_timer_hz() / 1000000; + int64_t remaining = t->outstand_pkts - processed_pkts(t); + + while (t->done == false) { + const uint64_t new_cycles = rte_get_timer_cycles(); + + if ((new_cycles - perf_cycles) > perf_sample) { + const uint64_t latency = total_latency(t); + const uint64_t pkts = processed_pkts(t); + + remaining = t->outstand_pkts - pkts; + float mpps = (float)(perf_remaining-remaining)/1000000; + + perf_remaining = remaining; + perf_cycles = new_cycles; + total_mpps += mpps; + ++samples; + if (opt->fwd_latency && pkts > 0) { + printf(CLGRN"\r%.3f mpps avg %.3f mpps [avg fwd latency %.3f us] "CLNRM, + mpps, total_mpps/samples, + (float)(latency/pkts)/freq_mhz); + } else { + printf(CLGRN"\r%.3f mpps avg %.3f mpps"CLNRM, + mpps, total_mpps/samples); + } + fflush(stdout); + + if (remaining <= 0) { + t->done = true; + t->result = EVT_TEST_SUCCESS; + rte_smp_wmb(); + break; + } + } + + if (new_cycles - dead_lock_cycles > dead_lock_sample) { + remaining = t->outstand_pkts - processed_pkts(t); + if (dead_lock_remaining == remaining) { + rte_event_dev_dump(opt->dev_id, stdout); + evt_err("No schedules for seconds, deadlock"); + t->done = true; + rte_smp_wmb(); + break; + } + dead_lock_remaining = remaining; + dead_lock_cycles = new_cycles; + } + } + printf("\n"); + return 0; +} + +int +perf_event_dev_port_setup(struct evt_test *test, struct evt_options *opt, + uint8_t stride, uint8_t nb_queues) +{ + struct test_perf *t = evt_test_priv(test); + uint8_t port, prod; + int ret = -1; + + /* port configuration */ + const struct rte_event_port_conf wkr_p_conf = { + .dequeue_depth = opt->wkr_deq_dep, + .enqueue_depth = 64, + .new_event_threshold = 4096, + }; + + /* setup one port per worker, linking to all queues */ + for (port = 0; port < evt_nr_active_lcores(opt->wlcores); + port++) { + struct worker_data *w = &t->worker[port]; + + w->dev_id = opt->dev_id; + w->port_id = port; + w->t = t; + w->processed_pkts = 0; + w->latency = 0; + + ret = rte_event_port_setup(opt->dev_id, port, &wkr_p_conf); + if (ret) { + evt_err("failed to setup port %d", port); + return ret; + } + + ret = rte_event_port_link(opt->dev_id, port, NULL, NULL, 0); + if (ret != nb_queues) { + evt_err("failed to link all queues to port %d", port); + return -EINVAL; + } + } + + /* port for producers, no links */ + const struct rte_event_port_conf prod_conf = { + .dequeue_depth = 8, + .enqueue_depth = 32, + .new_event_threshold = 1200, + }; + prod = 0; + for ( ; port < perf_nb_event_ports(opt); port++) { + struct prod_data *p = &t->prod[port]; + + p->dev_id = opt->dev_id; + p->port_id = port; + p->queue_id = prod * stride; + p->t = t; + + ret = rte_event_port_setup(opt->dev_id, port, &prod_conf); + if (ret) { + evt_err("failed to setup port %d", port); + return ret; + } + prod++; + } + + return ret; +} + +int +perf_opt_check(struct evt_options *opt, uint64_t nb_queues) +{ + unsigned int lcores; + + /* N producer + N worker + 1 master */ + lcores = 3; + + if (rte_lcore_count() < lcores) { + evt_err("test need minimum %d lcores", lcores); + return -1; + } + + /* Validate worker lcores */ + if (evt_lcores_has_overlap(opt->wlcores, rte_get_master_lcore())) { + evt_err("worker lcores overlaps with master lcore"); + return -1; + } + if (evt_lcores_has_overlap_multi(opt->wlcores, opt->plcores)) { + evt_err("worker lcores overlaps producer lcores"); + return -1; + } + if (evt_has_disabled_lcore(opt->wlcores)) { + evt_err("one or more workers lcores are not enabled"); + return -1; + } + if (!evt_has_active_lcore(opt->wlcores)) { + evt_err("minimum one worker is required"); + return -1; + } + + /* Validate producer lcores */ + if (evt_lcores_has_overlap(opt->plcores, rte_get_master_lcore())) { + evt_err("producer lcores overlaps with master lcore"); + return -1; + } + if (evt_has_disabled_lcore(opt->plcores)) { + evt_err("one or more producer lcores are not enabled"); + return -1; + } + if (!evt_has_active_lcore(opt->plcores)) { + evt_err("minimum one producer is required"); + return -1; + } + + if (evt_has_invalid_stage(opt)) + return -1; + + if (evt_has_invalid_sched_type(opt)) + return -1; + + if (nb_queues > EVT_MAX_QUEUES) { + evt_err("number of queues exceeds %d", EVT_MAX_QUEUES); + return -1; + } + if (perf_nb_event_ports(opt) > EVT_MAX_PORTS) { + evt_err("number of ports exceeds %d", EVT_MAX_PORTS); + return -1; + } + + /* Fixups */ + if (opt->nb_stages == 1 && opt->fwd_latency) { + evt_info("fwd_latency is valid when nb_stages > 1, disabling"); + opt->fwd_latency = 0; + } + if (opt->fwd_latency && !opt->q_priority) { + evt_info("enabled queue priority for latency measurement"); + opt->q_priority = 1; + } + if (opt->nb_pkts == 0) + opt->nb_pkts = INT64_MAX/evt_nr_active_lcores(opt->plcores); + + return 0; +} + +void +perf_opt_dump(struct evt_options *opt, uint8_t nb_queues) +{ + evt_dump("nb_prod_lcores", "%d", evt_nr_active_lcores(opt->plcores)); + evt_dump_producer_lcores(opt); + evt_dump("nb_worker_lcores", "%d", evt_nr_active_lcores(opt->wlcores)); + evt_dump_worker_lcores(opt); + evt_dump_nb_stages(opt); + evt_dump("nb_evdev_ports", "%d", perf_nb_event_ports(opt)); + evt_dump("nb_evdev_queues", "%d", nb_queues); + evt_dump_queue_priority(opt); + evt_dump_sched_type_list(opt); +} + +void +perf_eventdev_destroy(struct evt_test *test, struct evt_options *opt) +{ + RTE_SET_USED(test); + + rte_event_dev_stop(opt->dev_id); + rte_event_dev_close(opt->dev_id); +} + +static inline void +perf_elt_init(struct rte_mempool *mp, void *arg __rte_unused, + void *obj, unsigned i __rte_unused) +{ + memset(obj, 0, mp->elt_size); +} + +int +perf_mempool_setup(struct evt_test *test, struct evt_options *opt) +{ + struct test_perf *t = evt_test_priv(test); + + t->pool = rte_mempool_create(test->name, /* mempool name */ + opt->pool_sz, /* number of elements*/ + sizeof(struct perf_elt), /* element size*/ + 512, /* cache size*/ + 0, NULL, NULL, + perf_elt_init, /* obj constructor */ + NULL, opt->socket_id, 0); /* flags */ + if (t->pool == NULL) { + evt_err("failed to create mempool"); + return -ENOMEM; + } + + return 0; +} + +void +perf_mempool_destroy(struct evt_test *test, struct evt_options *opt) +{ + RTE_SET_USED(opt); + struct test_perf *t = evt_test_priv(test); + + rte_mempool_free(t->pool); +} + +int +perf_test_setup(struct evt_test *test, struct evt_options *opt) +{ + void *test_perf; + + test_perf = rte_zmalloc_socket(test->name, sizeof(struct test_perf), + RTE_CACHE_LINE_SIZE, opt->socket_id); + if (test_perf == NULL) { + evt_err("failed to allocate test_perf memory"); + goto nomem; + } + test->test_priv = test_perf; + + struct test_perf *t = evt_test_priv(test); + + t->outstand_pkts = opt->nb_pkts * evt_nr_active_lcores(opt->plcores); + t->nb_workers = evt_nr_active_lcores(opt->wlcores); + t->done = false; + t->nb_pkts = opt->nb_pkts; + t->nb_flows = opt->nb_flows; + t->result = EVT_TEST_FAILED; + t->opt = opt; + memcpy(t->sched_type_list, opt->sched_type_list, + sizeof(opt->sched_type_list)); + return 0; +nomem: + return -ENOMEM; +} + +void +perf_test_destroy(struct evt_test *test, struct evt_options *opt) +{ + RTE_SET_USED(opt); + + rte_free(test->test_priv); +} diff --git a/dpdk/app/test-eventdev/test_perf_common.h b/dpdk/app/test-eventdev/test_perf_common.h new file mode 100644 index 00000000..c6fc70cd --- /dev/null +++ b/dpdk/app/test-eventdev/test_perf_common.h @@ -0,0 +1,170 @@ +/* + * BSD LICENSE + * + * Copyright (C) Cavium, Inc 2017. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _TEST_PERF_COMMON_ +#define _TEST_PERF_COMMON_ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "evt_common.h" +#include "evt_options.h" +#include "evt_test.h" + +struct test_perf; + +struct worker_data { + uint64_t processed_pkts; + uint64_t latency; + uint8_t dev_id; + uint8_t port_id; + struct test_perf *t; +} __rte_cache_aligned; + +struct prod_data { + uint8_t dev_id; + uint8_t port_id; + uint8_t queue_id; + struct test_perf *t; +} __rte_cache_aligned; + +struct test_perf { + /* Don't change the offset of "done". Signal handler use this memory + * to terminate all lcores work. + */ + int done; + uint64_t outstand_pkts; + uint8_t nb_workers; + enum evt_test_result result; + uint32_t nb_flows; + uint64_t nb_pkts; + struct rte_mempool *pool; + struct prod_data prod[EVT_MAX_PORTS]; + struct worker_data worker[EVT_MAX_PORTS]; + struct evt_options *opt; + uint8_t sched_type_list[EVT_MAX_STAGES] __rte_cache_aligned; +} __rte_cache_aligned; + +struct perf_elt { + uint64_t timestamp; +} __rte_cache_aligned; + +#define BURST_SIZE 16 + +#define PERF_WORKER_INIT\ + struct worker_data *w = arg;\ + struct test_perf *t = w->t;\ + struct evt_options *opt = t->opt;\ + const uint8_t dev = w->dev_id;\ + const uint8_t port = w->port_id;\ + uint8_t *const sched_type_list = &t->sched_type_list[0];\ + struct rte_mempool *const pool = t->pool;\ + const uint8_t nb_stages = t->opt->nb_stages;\ + const uint8_t laststage = nb_stages - 1;\ + uint8_t cnt = 0;\ + void *bufs[16] __rte_cache_aligned;\ + int const sz = RTE_DIM(bufs);\ + if (opt->verbose_level > 1)\ + printf("%s(): lcore %d dev_id %d port=%d\n", __func__,\ + rte_lcore_id(), dev, port) + +static inline __attribute__((always_inline)) int +perf_process_last_stage(struct rte_mempool *const pool, + struct rte_event *const ev, struct worker_data *const w, + void *bufs[], int const buf_sz, uint8_t count) +{ + bufs[count++] = ev->event_ptr; + w->processed_pkts++; + rte_smp_wmb(); + + if (unlikely(count == buf_sz)) { + count = 0; + rte_mempool_put_bulk(pool, bufs, buf_sz); + } + return count; +} + +static inline __attribute__((always_inline)) uint8_t +perf_process_last_stage_latency(struct rte_mempool *const pool, + struct rte_event *const ev, struct worker_data *const w, + void *bufs[], int const buf_sz, uint8_t count) +{ + uint64_t latency; + struct perf_elt *const m = ev->event_ptr; + + bufs[count++] = ev->event_ptr; + w->processed_pkts++; + + if (unlikely(count == buf_sz)) { + count = 0; + latency = rte_get_timer_cycles() - m->timestamp; + rte_mempool_put_bulk(pool, bufs, buf_sz); + } else { + latency = rte_get_timer_cycles() - m->timestamp; + } + + w->latency += latency; + rte_smp_wmb(); + return count; +} + + +static inline int +perf_nb_event_ports(struct evt_options *opt) +{ + return evt_nr_active_lcores(opt->wlcores) + + evt_nr_active_lcores(opt->plcores); +} + +int perf_test_result(struct evt_test *test, struct evt_options *opt); +int perf_opt_check(struct evt_options *opt, uint64_t nb_queues); +int perf_test_setup(struct evt_test *test, struct evt_options *opt); +int perf_mempool_setup(struct evt_test *test, struct evt_options *opt); +int perf_event_dev_port_setup(struct evt_test *test, struct evt_options *opt, + uint8_t stride, uint8_t nb_queues); +int perf_event_dev_service_setup(uint8_t dev_id); +int perf_launch_lcores(struct evt_test *test, struct evt_options *opt, + int (*worker)(void *)); +void perf_opt_dump(struct evt_options *opt, uint8_t nb_queues); +void perf_test_destroy(struct evt_test *test, struct evt_options *opt); +void perf_eventdev_destroy(struct evt_test *test, struct evt_options *opt); +void perf_mempool_destroy(struct evt_test *test, struct evt_options *opt); + +#endif /* _TEST_PERF_COMMON_ */ diff --git a/dpdk/app/test-eventdev/test_perf_queue.c b/dpdk/app/test-eventdev/test_perf_queue.c new file mode 100644 index 00000000..d843eea1 --- /dev/null +++ b/dpdk/app/test-eventdev/test_perf_queue.c @@ -0,0 +1,294 @@ +/* + * BSD LICENSE + * + * Copyright (C) Cavium, Inc 2017. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "test_perf_common.h" + +/* See http://dpdk.org/doc/guides/tools/testeventdev.html for test details */ + +static inline int +perf_queue_nb_event_queues(struct evt_options *opt) +{ + /* nb_queues = number of producers * number of stages */ + return evt_nr_active_lcores(opt->plcores) * opt->nb_stages; +} + +static inline __attribute__((always_inline)) void +mark_fwd_latency(struct rte_event *const ev, + const uint8_t nb_stages) +{ + if (unlikely((ev->queue_id % nb_stages) == 0)) { + struct perf_elt *const m = ev->event_ptr; + + m->timestamp = rte_get_timer_cycles(); + } +} + +static inline __attribute__((always_inline)) void +fwd_event(struct rte_event *const ev, uint8_t *const sched_type_list, + const uint8_t nb_stages) +{ + ev->queue_id++; + ev->sched_type = sched_type_list[ev->queue_id % nb_stages]; + ev->op = RTE_EVENT_OP_FORWARD; + ev->event_type = RTE_EVENT_TYPE_CPU; +} + +static int +perf_queue_worker(void *arg, const int enable_fwd_latency) +{ + PERF_WORKER_INIT; + struct rte_event ev; + + while (t->done == false) { + uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0); + + if (!event) { + rte_pause(); + continue; + } + if (enable_fwd_latency) + /* first q in pipeline, mark timestamp to compute fwd latency */ + mark_fwd_latency(&ev, nb_stages); + + /* last stage in pipeline */ + if (unlikely((ev.queue_id % nb_stages) == laststage)) { + if (enable_fwd_latency) + cnt = perf_process_last_stage_latency(pool, + &ev, w, bufs, sz, cnt); + else + cnt = perf_process_last_stage(pool, + &ev, w, bufs, sz, cnt); + } else { + fwd_event(&ev, sched_type_list, nb_stages); + while (rte_event_enqueue_burst(dev, port, &ev, 1) != 1) + rte_pause(); + } + } + return 0; +} + +static int +perf_queue_worker_burst(void *arg, const int enable_fwd_latency) +{ + PERF_WORKER_INIT; + uint16_t i; + /* +1 to avoid prefetch out of array check */ + struct rte_event ev[BURST_SIZE + 1]; + + while (t->done == false) { + uint16_t const nb_rx = rte_event_dequeue_burst(dev, port, ev, + BURST_SIZE, 0); + + if (!nb_rx) { + rte_pause(); + continue; + } + + for (i = 0; i < nb_rx; i++) { + if (enable_fwd_latency) { + rte_prefetch0(ev[i+1].event_ptr); + /* first queue in pipeline. + * mark time stamp to compute fwd latency + */ + mark_fwd_latency(&ev[i], nb_stages); + } + /* last stage in pipeline */ + if (unlikely((ev[i].queue_id % nb_stages) == + laststage)) { + if (enable_fwd_latency) + cnt = perf_process_last_stage_latency( + pool, &ev[i], w, bufs, sz, cnt); + else + cnt = perf_process_last_stage(pool, + &ev[i], w, bufs, sz, cnt); + + ev[i].op = RTE_EVENT_OP_RELEASE; + } else { + fwd_event(&ev[i], sched_type_list, nb_stages); + } + } + + uint16_t enq; + + enq = rte_event_enqueue_burst(dev, port, ev, nb_rx); + while (enq < nb_rx) { + enq += rte_event_enqueue_burst(dev, port, + ev + enq, nb_rx - enq); + } + } + return 0; +} + +static int +worker_wrapper(void *arg) +{ + struct worker_data *w = arg; + struct evt_options *opt = w->t->opt; + + const bool burst = evt_has_burst_mode(w->dev_id); + const int fwd_latency = opt->fwd_latency; + + /* allow compiler to optimize */ + if (!burst && !fwd_latency) + return perf_queue_worker(arg, 0); + else if (!burst && fwd_latency) + return perf_queue_worker(arg, 1); + else if (burst && !fwd_latency) + return perf_queue_worker_burst(arg, 0); + else if (burst && fwd_latency) + return perf_queue_worker_burst(arg, 1); + + rte_panic("invalid worker\n"); +} + +static int +perf_queue_launch_lcores(struct evt_test *test, struct evt_options *opt) +{ + return perf_launch_lcores(test, opt, worker_wrapper); +} + +static int +perf_queue_eventdev_setup(struct evt_test *test, struct evt_options *opt) +{ + uint8_t queue; + int nb_stages = opt->nb_stages; + int ret; + + const struct rte_event_dev_config config = { + .nb_event_queues = perf_queue_nb_event_queues(opt), + .nb_event_ports = perf_nb_event_ports(opt), + .nb_events_limit = 4096, + .nb_event_queue_flows = opt->nb_flows, + .nb_event_port_dequeue_depth = 128, + .nb_event_port_enqueue_depth = 128, + }; + + ret = rte_event_dev_configure(opt->dev_id, &config); + if (ret) { + evt_err("failed to configure eventdev %d", opt->dev_id); + return ret; + } + + struct rte_event_queue_conf q_conf = { + .priority = RTE_EVENT_DEV_PRIORITY_NORMAL, + .nb_atomic_flows = opt->nb_flows, + .nb_atomic_order_sequences = opt->nb_flows, + }; + /* queue configurations */ + for (queue = 0; queue < perf_queue_nb_event_queues(opt); queue++) { + q_conf.schedule_type = + (opt->sched_type_list[queue % nb_stages]); + + if (opt->q_priority) { + uint8_t stage_pos = queue % nb_stages; + /* Configure event queues(stage 0 to stage n) with + * RTE_EVENT_DEV_PRIORITY_LOWEST to + * RTE_EVENT_DEV_PRIORITY_HIGHEST. + */ + uint8_t step = RTE_EVENT_DEV_PRIORITY_LOWEST / + (nb_stages - 1); + /* Higher prio for the queues closer to last stage */ + q_conf.priority = RTE_EVENT_DEV_PRIORITY_LOWEST - + (step * stage_pos); + } + ret = rte_event_queue_setup(opt->dev_id, queue, &q_conf); + if (ret) { + evt_err("failed to setup queue=%d", queue); + return ret; + } + } + + ret = perf_event_dev_port_setup(test, opt, nb_stages /* stride */, + perf_queue_nb_event_queues(opt)); + if (ret) + return ret; + + ret = evt_service_setup(opt->dev_id); + if (ret) { + evt_err("No service lcore found to run event dev."); + return ret; + } + + ret = rte_event_dev_start(opt->dev_id); + if (ret) { + evt_err("failed to start eventdev %d", opt->dev_id); + return ret; + } + + return 0; +} + +static void +perf_queue_opt_dump(struct evt_options *opt) +{ + evt_dump_fwd_latency(opt); + perf_opt_dump(opt, perf_queue_nb_event_queues(opt)); +} + +static int +perf_queue_opt_check(struct evt_options *opt) +{ + return perf_opt_check(opt, perf_queue_nb_event_queues(opt)); +} + +static bool +perf_queue_capability_check(struct evt_options *opt) +{ + struct rte_event_dev_info dev_info; + + rte_event_dev_info_get(opt->dev_id, &dev_info); + if (dev_info.max_event_queues < perf_queue_nb_event_queues(opt) || + dev_info.max_event_ports < perf_nb_event_ports(opt)) { + evt_err("not enough eventdev queues=%d/%d or ports=%d/%d", + perf_queue_nb_event_queues(opt), + dev_info.max_event_queues, + perf_nb_event_ports(opt), dev_info.max_event_ports); + } + + return true; +} + +static const struct evt_test_ops perf_queue = { + .cap_check = perf_queue_capability_check, + .opt_check = perf_queue_opt_check, + .opt_dump = perf_queue_opt_dump, + .test_setup = perf_test_setup, + .mempool_setup = perf_mempool_setup, + .eventdev_setup = perf_queue_eventdev_setup, + .launch_lcores = perf_queue_launch_lcores, + .eventdev_destroy = perf_eventdev_destroy, + .mempool_destroy = perf_mempool_destroy, + .test_result = perf_test_result, + .test_destroy = perf_test_destroy, +}; + +EVT_TEST_REGISTER(perf_queue); diff --git a/dpdk/app/test-pipeline/Makefile b/dpdk/app/test-pipeline/Makefile deleted file mode 100644 index 4bab6dc6..00000000 --- a/dpdk/app/test-pipeline/Makefile +++ /dev/null @@ -1,64 +0,0 @@ -# BSD LICENSE -# -# Copyright(c) 2010-2015 Intel Corporation. All rights reserved. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in -# the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Intel Corporation nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -include $(RTE_SDK)/mk/rte.vars.mk - -ifeq ($(CONFIG_RTE_LIBRTE_PIPELINE),y) - -# -# library name -# -APP = testpipeline - -CFLAGS += -O3 -CFLAGS += $(WERROR_FLAGS) - -# -# all source are stored in SRCS-y -# -SRCS-y := main.c -SRCS-y += config.c -SRCS-y += init.c -SRCS-y += runtime.c -SRCS-y += pipeline_stub.c -SRCS-y += pipeline_hash.c -SRCS-y += pipeline_lpm.c -SRCS-y += pipeline_lpm_ipv6.c - -# include ACL lib if available -SRCS-$(CONFIG_RTE_LIBRTE_ACL) += pipeline_acl.c - -# this application needs libraries first -DEPDIRS-y += lib drivers - -include $(RTE_SDK)/mk/rte.app.mk - -endif diff --git a/dpdk/app/test-pipeline/config.c b/dpdk/app/test-pipeline/config.c deleted file mode 100644 index 72e018cf..00000000 --- a/dpdk/app/test-pipeline/config.c +++ /dev/null @@ -1,247 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "main.h" - -struct app_params app; - -static const char usage[] = "\n"; - -void -app_print_usage(void) -{ - printf(usage); -} - -static int -app_parse_port_mask(const char *arg) -{ - char *end = NULL; - uint64_t port_mask; - uint32_t i; - - if (arg[0] == '\0') - return -1; - - port_mask = strtoul(arg, &end, 16); - if ((end == NULL) || (*end != '\0')) - return -2; - - if (port_mask == 0) - return -3; - - app.n_ports = 0; - for (i = 0; i < 64; i++) { - if ((port_mask & (1LLU << i)) == 0) - continue; - - if (app.n_ports >= APP_MAX_PORTS) - return -4; - - app.ports[app.n_ports] = i; - app.n_ports++; - } - - if (!rte_is_power_of_2(app.n_ports)) - return -5; - - return 0; -} - -struct { - const char *name; - uint32_t value; -} app_args_table[] = { - {"none", e_APP_PIPELINE_NONE}, - {"stub", e_APP_PIPELINE_STUB}, - {"hash-8-ext", e_APP_PIPELINE_HASH_KEY8_EXT}, - {"hash-8-lru", e_APP_PIPELINE_HASH_KEY8_LRU}, - {"hash-16-ext", e_APP_PIPELINE_HASH_KEY16_EXT}, - {"hash-16-lru", e_APP_PIPELINE_HASH_KEY16_LRU}, - {"hash-32-ext", e_APP_PIPELINE_HASH_KEY32_EXT}, - {"hash-32-lru", e_APP_PIPELINE_HASH_KEY32_LRU}, - {"hash-spec-8-ext", e_APP_PIPELINE_HASH_SPEC_KEY8_EXT}, - {"hash-spec-8-lru", e_APP_PIPELINE_HASH_SPEC_KEY8_LRU}, - {"hash-spec-16-ext", e_APP_PIPELINE_HASH_SPEC_KEY16_EXT}, - {"hash-spec-16-lru", e_APP_PIPELINE_HASH_SPEC_KEY16_LRU}, - {"hash-spec-32-ext", e_APP_PIPELINE_HASH_SPEC_KEY32_EXT}, - {"hash-spec-32-lru", e_APP_PIPELINE_HASH_SPEC_KEY32_LRU}, - {"acl", e_APP_PIPELINE_ACL}, - {"lpm", e_APP_PIPELINE_LPM}, - {"lpm-ipv6", e_APP_PIPELINE_LPM_IPV6}, -}; - -int -app_parse_args(int argc, char **argv) -{ - int opt, ret; - char **argvopt; - int option_index; - char *prgname = argv[0]; - static struct option lgopts[] = { - {"none", 0, 0, 0}, - {"stub", 0, 0, 0}, - {"hash-8-ext", 0, 0, 0}, - {"hash-8-lru", 0, 0, 0}, - {"hash-16-ext", 0, 0, 0}, - {"hash-16-lru", 0, 0, 0}, - {"hash-32-ext", 0, 0, 0}, - {"hash-32-lru", 0, 0, 0}, - {"hash-spec-8-ext", 0, 0, 0}, - {"hash-spec-8-lru", 0, 0, 0}, - {"hash-spec-16-ext", 0, 0, 0}, - {"hash-spec-16-lru", 0, 0, 0}, - {"hash-spec-32-ext", 0, 0, 0}, - {"hash-spec-32-lru", 0, 0, 0}, - {"acl", 0, 0, 0}, - {"lpm", 0, 0, 0}, - {"lpm-ipv6", 0, 0, 0}, - {NULL, 0, 0, 0} - }; - uint32_t lcores[3], n_lcores, lcore_id, pipeline_type_provided; - - /* EAL args */ - n_lcores = 0; - for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { - if (rte_lcore_is_enabled(lcore_id) == 0) - continue; - - if (n_lcores >= 3) { - RTE_LOG(ERR, USER1, "Number of cores must be 3\n"); - app_print_usage(); - return -1; - } - - lcores[n_lcores] = lcore_id; - n_lcores++; - } - - if (n_lcores != 3) { - RTE_LOG(ERR, USER1, "Number of cores must be 3\n"); - app_print_usage(); - return -1; - } - - app.core_rx = lcores[0]; - app.core_worker = lcores[1]; - app.core_tx = lcores[2]; - - /* Non-EAL args */ - argvopt = argv; - - app.pipeline_type = e_APP_PIPELINE_HASH_KEY16_LRU; - pipeline_type_provided = 0; - - while ((opt = getopt_long(argc, argvopt, "p:", - lgopts, &option_index)) != EOF) { - switch (opt) { - case 'p': - if (app_parse_port_mask(optarg) < 0) { - app_print_usage(); - return -1; - } - break; - - case 0: /* long options */ - if (!pipeline_type_provided) { - uint32_t i; - - for (i = 0; i < e_APP_PIPELINES; i++) { - if (!strcmp(lgopts[option_index].name, - app_args_table[i].name)) { - app.pipeline_type = - app_args_table[i].value; - pipeline_type_provided = 1; - break; - } - } - - break; - } - - app_print_usage(); - return -1; - - default: - return -1; - } - } - - if (optind >= 0) - argv[optind - 1] = prgname; - - ret = optind - 1; - optind = 0; /* reset getopt lib */ - return ret; -} diff --git a/dpdk/app/test-pipeline/init.c b/dpdk/app/test-pipeline/init.c deleted file mode 100644 index aef082fc..00000000 --- a/dpdk/app/test-pipeline/init.c +++ /dev/null @@ -1,280 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "main.h" - -struct app_params app = { - /* Ports*/ - .n_ports = APP_MAX_PORTS, - .port_rx_ring_size = 128, - .port_tx_ring_size = 512, - - /* Rings */ - .ring_rx_size = 128, - .ring_tx_size = 128, - - /* Buffer pool */ - .pool_buffer_size = 2048 + RTE_PKTMBUF_HEADROOM, - .pool_size = 32 * 1024, - .pool_cache_size = 256, - - /* Burst sizes */ - .burst_size_rx_read = 64, - .burst_size_rx_write = 64, - .burst_size_worker_read = 64, - .burst_size_worker_write = 64, - .burst_size_tx_read = 64, - .burst_size_tx_write = 64, -}; - -static struct rte_eth_conf port_conf = { - .rxmode = { - .split_hdr_size = 0, - .header_split = 0, /* Header Split disabled */ - .hw_ip_checksum = 1, /* IP checksum offload enabled */ - .hw_vlan_filter = 0, /* VLAN filtering disabled */ - .jumbo_frame = 0, /* Jumbo Frame Support disabled */ - .hw_strip_crc = 0, /* CRC stripped by hardware */ - }, - .rx_adv_conf = { - .rss_conf = { - .rss_key = NULL, - .rss_hf = ETH_RSS_IP, - }, - }, - .txmode = { - .mq_mode = ETH_MQ_TX_NONE, - }, -}; - -static struct rte_eth_rxconf rx_conf = { - .rx_thresh = { - .pthresh = 8, - .hthresh = 8, - .wthresh = 4, - }, - .rx_free_thresh = 64, - .rx_drop_en = 0, -}; - -static struct rte_eth_txconf tx_conf = { - .tx_thresh = { - .pthresh = 36, - .hthresh = 0, - .wthresh = 0, - }, - .tx_free_thresh = 0, - .tx_rs_thresh = 0, -}; - -static void -app_init_mbuf_pools(void) -{ - /* Init the buffer pool */ - RTE_LOG(INFO, USER1, "Creating the mbuf pool ...\n"); - app.pool = rte_pktmbuf_pool_create("mempool", app.pool_size, - app.pool_cache_size, 0, app.pool_buffer_size, rte_socket_id()); - if (app.pool == NULL) - rte_panic("Cannot create mbuf pool\n"); -} - -static void -app_init_rings(void) -{ - uint32_t i; - - for (i = 0; i < app.n_ports; i++) { - char name[32]; - - snprintf(name, sizeof(name), "app_ring_rx_%u", i); - - app.rings_rx[i] = rte_ring_create( - name, - app.ring_rx_size, - rte_socket_id(), - RING_F_SP_ENQ | RING_F_SC_DEQ); - - if (app.rings_rx[i] == NULL) - rte_panic("Cannot create RX ring %u\n", i); - } - - for (i = 0; i < app.n_ports; i++) { - char name[32]; - - snprintf(name, sizeof(name), "app_ring_tx_%u", i); - - app.rings_tx[i] = rte_ring_create( - name, - app.ring_tx_size, - rte_socket_id(), - RING_F_SP_ENQ | RING_F_SC_DEQ); - - if (app.rings_tx[i] == NULL) - rte_panic("Cannot create TX ring %u\n", i); - } - -} - -static void -app_ports_check_link(void) -{ - uint32_t all_ports_up, i; - - all_ports_up = 1; - - for (i = 0; i < app.n_ports; i++) { - struct rte_eth_link link; - uint8_t port; - - port = (uint8_t) app.ports[i]; - memset(&link, 0, sizeof(link)); - rte_eth_link_get_nowait(port, &link); - RTE_LOG(INFO, USER1, "Port %u (%u Gbps) %s\n", - port, - link.link_speed / 1000, - link.link_status ? "UP" : "DOWN"); - - if (link.link_status == ETH_LINK_DOWN) - all_ports_up = 0; - } - - if (all_ports_up == 0) - rte_panic("Some NIC ports are DOWN\n"); -} - -static void -app_init_ports(void) -{ - uint32_t i; - - /* Init NIC ports, then start the ports */ - for (i = 0; i < app.n_ports; i++) { - uint8_t port; - int ret; - - port = (uint8_t) app.ports[i]; - RTE_LOG(INFO, USER1, "Initializing NIC port %u ...\n", port); - - /* Init port */ - ret = rte_eth_dev_configure( - port, - 1, - 1, - &port_conf); - if (ret < 0) - rte_panic("Cannot init NIC port %u (%d)\n", port, ret); - - rte_eth_promiscuous_enable(port); - - /* Init RX queues */ - ret = rte_eth_rx_queue_setup( - port, - 0, - app.port_rx_ring_size, - rte_eth_dev_socket_id(port), - &rx_conf, - app.pool); - if (ret < 0) - rte_panic("Cannot init RX for port %u (%d)\n", - (uint32_t) port, ret); - - /* Init TX queues */ - ret = rte_eth_tx_queue_setup( - port, - 0, - app.port_tx_ring_size, - rte_eth_dev_socket_id(port), - &tx_conf); - if (ret < 0) - rte_panic("Cannot init TX for port %u (%d)\n", - (uint32_t) port, ret); - - /* Start port */ - ret = rte_eth_dev_start(port); - if (ret < 0) - rte_panic("Cannot start port %u (%d)\n", port, ret); - } - - app_ports_check_link(); -} - -void -app_init(void) -{ - app_init_mbuf_pools(); - app_init_rings(); - app_init_ports(); - - RTE_LOG(INFO, USER1, "Initialization completed\n"); -} diff --git a/dpdk/app/test-pipeline/main.c b/dpdk/app/test-pipeline/main.c deleted file mode 100644 index 685ebd46..00000000 --- a/dpdk/app/test-pipeline/main.c +++ /dev/null @@ -1,179 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "main.h" - -int -main(int argc, char **argv) -{ - uint32_t lcore; - int ret; - - /* Init EAL */ - ret = rte_eal_init(argc, argv); - if (ret < 0) - return -1; - argc -= ret; - argv += ret; - - /* Parse application arguments (after the EAL ones) */ - ret = app_parse_args(argc, argv); - if (ret < 0) { - app_print_usage(); - return -1; - } - - /* Init */ - app_init(); - - /* Launch per-lcore init on every lcore */ - rte_eal_mp_remote_launch(app_lcore_main_loop, NULL, CALL_MASTER); - RTE_LCORE_FOREACH_SLAVE(lcore) { - if (rte_eal_wait_lcore(lcore) < 0) - return -1; - } - - return 0; -} - -int -app_lcore_main_loop(__attribute__((unused)) void *arg) -{ - unsigned lcore; - - lcore = rte_lcore_id(); - - if (lcore == app.core_rx) { - switch (app.pipeline_type) { - case e_APP_PIPELINE_ACL: - app_main_loop_rx(); - return 0; - - default: - app_main_loop_rx_metadata(); - return 0; - } - } - - if (lcore == app.core_worker) { - switch (app.pipeline_type) { - case e_APP_PIPELINE_STUB: - app_main_loop_worker_pipeline_stub(); - return 0; - - case e_APP_PIPELINE_HASH_KEY8_EXT: - case e_APP_PIPELINE_HASH_KEY8_LRU: - case e_APP_PIPELINE_HASH_KEY16_EXT: - case e_APP_PIPELINE_HASH_KEY16_LRU: - case e_APP_PIPELINE_HASH_KEY32_EXT: - case e_APP_PIPELINE_HASH_KEY32_LRU: - case e_APP_PIPELINE_HASH_SPEC_KEY8_EXT: - case e_APP_PIPELINE_HASH_SPEC_KEY8_LRU: - case e_APP_PIPELINE_HASH_SPEC_KEY16_EXT: - case e_APP_PIPELINE_HASH_SPEC_KEY16_LRU: - case e_APP_PIPELINE_HASH_SPEC_KEY32_EXT: - case e_APP_PIPELINE_HASH_SPEC_KEY32_LRU: - app_main_loop_worker_pipeline_hash(); - return 0; - - case e_APP_PIPELINE_ACL: -#ifndef RTE_LIBRTE_ACL - rte_exit(EXIT_FAILURE, "ACL not present in build\n"); -#else - app_main_loop_worker_pipeline_acl(); - return 0; -#endif - - case e_APP_PIPELINE_LPM: - app_main_loop_worker_pipeline_lpm(); - return 0; - - case e_APP_PIPELINE_LPM_IPV6: - app_main_loop_worker_pipeline_lpm_ipv6(); - return 0; - - case e_APP_PIPELINE_NONE: - default: - app_main_loop_worker(); - return 0; - } - } - - if (lcore == app.core_tx) { - app_main_loop_tx(); - return 0; - } - - return 0; -} diff --git a/dpdk/app/test-pipeline/main.h b/dpdk/app/test-pipeline/main.h deleted file mode 100644 index 8dcd4592..00000000 --- a/dpdk/app/test-pipeline/main.h +++ /dev/null @@ -1,142 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef _MAIN_H_ -#define _MAIN_H_ - -#ifndef APP_MBUF_ARRAY_SIZE -#define APP_MBUF_ARRAY_SIZE 256 -#endif - -struct app_mbuf_array { - struct rte_mbuf *array[APP_MBUF_ARRAY_SIZE]; - uint16_t n_mbufs; -}; - -#ifndef APP_MAX_PORTS -#define APP_MAX_PORTS 4 -#endif - -struct app_params { - /* CPU cores */ - uint32_t core_rx; - uint32_t core_worker; - uint32_t core_tx; - - /* Ports*/ - uint32_t ports[APP_MAX_PORTS]; - uint32_t n_ports; - uint32_t port_rx_ring_size; - uint32_t port_tx_ring_size; - - /* Rings */ - struct rte_ring *rings_rx[APP_MAX_PORTS]; - struct rte_ring *rings_tx[APP_MAX_PORTS]; - uint32_t ring_rx_size; - uint32_t ring_tx_size; - - /* Internal buffers */ - struct app_mbuf_array mbuf_rx; - struct app_mbuf_array mbuf_tx[APP_MAX_PORTS]; - - /* Buffer pool */ - struct rte_mempool *pool; - uint32_t pool_buffer_size; - uint32_t pool_size; - uint32_t pool_cache_size; - - /* Burst sizes */ - uint32_t burst_size_rx_read; - uint32_t burst_size_rx_write; - uint32_t burst_size_worker_read; - uint32_t burst_size_worker_write; - uint32_t burst_size_tx_read; - uint32_t burst_size_tx_write; - - /* App behavior */ - uint32_t pipeline_type; -} __rte_cache_aligned; - -extern struct app_params app; - -int app_parse_args(int argc, char **argv); -void app_print_usage(void); -void app_init(void); -int app_lcore_main_loop(void *arg); - -/* Pipeline */ -enum { - e_APP_PIPELINE_NONE = 0, - e_APP_PIPELINE_STUB, - - e_APP_PIPELINE_HASH_KEY8_EXT, - e_APP_PIPELINE_HASH_KEY8_LRU, - e_APP_PIPELINE_HASH_KEY16_EXT, - e_APP_PIPELINE_HASH_KEY16_LRU, - e_APP_PIPELINE_HASH_KEY32_EXT, - e_APP_PIPELINE_HASH_KEY32_LRU, - - e_APP_PIPELINE_HASH_SPEC_KEY8_EXT, - e_APP_PIPELINE_HASH_SPEC_KEY8_LRU, - e_APP_PIPELINE_HASH_SPEC_KEY16_EXT, - e_APP_PIPELINE_HASH_SPEC_KEY16_LRU, - e_APP_PIPELINE_HASH_SPEC_KEY32_EXT, - e_APP_PIPELINE_HASH_SPEC_KEY32_LRU, - - e_APP_PIPELINE_ACL, - e_APP_PIPELINE_LPM, - e_APP_PIPELINE_LPM_IPV6, - e_APP_PIPELINES -}; - -void app_main_loop_rx(void); -void app_main_loop_rx_metadata(void); -uint64_t test_hash(void *key, uint32_t key_size, uint64_t seed); - -void app_main_loop_worker(void); -void app_main_loop_worker_pipeline_stub(void); -void app_main_loop_worker_pipeline_hash(void); -void app_main_loop_worker_pipeline_acl(void); -void app_main_loop_worker_pipeline_lpm(void); -void app_main_loop_worker_pipeline_lpm_ipv6(void); - -void app_main_loop_tx(void); - -#define APP_FLUSH 0 -#ifndef APP_FLUSH -#define APP_FLUSH 0x3FF -#endif - -#define APP_METADATA_OFFSET(offset) (sizeof(struct rte_mbuf) + (offset)) - -#endif /* _MAIN_H_ */ diff --git a/dpdk/app/test-pipeline/pipeline_hash.c b/dpdk/app/test-pipeline/pipeline_hash.c deleted file mode 100644 index f8aac0d8..00000000 --- a/dpdk/app/test-pipeline/pipeline_hash.c +++ /dev/null @@ -1,493 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include -#include -#include - -#include -#include -#include -#include -#include - -#include -#include -#include - -#include "main.h" - -static void -translate_options(uint32_t *special, uint32_t *ext, uint32_t *key_size) -{ - switch (app.pipeline_type) { - case e_APP_PIPELINE_HASH_KEY8_EXT: - *special = 0; *ext = 1; *key_size = 8; return; - case e_APP_PIPELINE_HASH_KEY8_LRU: - *special = 0; *ext = 0; *key_size = 8; return; - case e_APP_PIPELINE_HASH_KEY16_EXT: - *special = 0; *ext = 1; *key_size = 16; return; - case e_APP_PIPELINE_HASH_KEY16_LRU: - *special = 0; *ext = 0; *key_size = 16; return; - case e_APP_PIPELINE_HASH_KEY32_EXT: - *special = 0; *ext = 1; *key_size = 32; return; - case e_APP_PIPELINE_HASH_KEY32_LRU: - *special = 0; *ext = 0; *key_size = 32; return; - - case e_APP_PIPELINE_HASH_SPEC_KEY8_EXT: - *special = 1; *ext = 1; *key_size = 8; return; - case e_APP_PIPELINE_HASH_SPEC_KEY8_LRU: - *special = 1; *ext = 0; *key_size = 8; return; - case e_APP_PIPELINE_HASH_SPEC_KEY16_EXT: - *special = 1; *ext = 1; *key_size = 16; return; - case e_APP_PIPELINE_HASH_SPEC_KEY16_LRU: - *special = 1; *ext = 0; *key_size = 16; return; - case e_APP_PIPELINE_HASH_SPEC_KEY32_EXT: - *special = 1; *ext = 1; *key_size = 32; return; - case e_APP_PIPELINE_HASH_SPEC_KEY32_LRU: - *special = 1; *ext = 0; *key_size = 32; return; - - default: - rte_panic("Invalid hash table type or key size\n"); - } -} -void -app_main_loop_worker_pipeline_hash(void) { - struct rte_pipeline_params pipeline_params = { - .name = "pipeline", - .socket_id = rte_socket_id(), - }; - - struct rte_pipeline *p; - uint32_t port_in_id[APP_MAX_PORTS]; - uint32_t port_out_id[APP_MAX_PORTS]; - uint32_t table_id; - uint32_t i; - uint32_t special, ext, key_size; - - translate_options(&special, &ext, &key_size); - - RTE_LOG(INFO, USER1, "Core %u is doing work " - "(pipeline with hash table, %s, %s, %d-byte key)\n", - rte_lcore_id(), - special ? "specialized" : "non-specialized", - ext ? "extendible bucket" : "LRU", - key_size); - - /* Pipeline configuration */ - p = rte_pipeline_create(&pipeline_params); - if (p == NULL) - rte_panic("Unable to configure the pipeline\n"); - - /* Input port configuration */ - for (i = 0; i < app.n_ports; i++) { - struct rte_port_ring_reader_params port_ring_params = { - .ring = app.rings_rx[i], - }; - - struct rte_pipeline_port_in_params port_params = { - .ops = &rte_port_ring_reader_ops, - .arg_create = (void *) &port_ring_params, - .f_action = NULL, - .arg_ah = NULL, - .burst_size = app.burst_size_worker_read, - }; - - if (rte_pipeline_port_in_create(p, &port_params, - &port_in_id[i])) - rte_panic("Unable to configure input port for " - "ring %d\n", i); - } - - /* Output port configuration */ - for (i = 0; i < app.n_ports; i++) { - struct rte_port_ring_writer_params port_ring_params = { - .ring = app.rings_tx[i], - .tx_burst_sz = app.burst_size_worker_write, - }; - - struct rte_pipeline_port_out_params port_params = { - .ops = &rte_port_ring_writer_ops, - .arg_create = (void *) &port_ring_params, - .f_action = NULL, - .arg_ah = NULL, - }; - - if (rte_pipeline_port_out_create(p, &port_params, - &port_out_id[i])) - rte_panic("Unable to configure output port for " - "ring %d\n", i); - } - - /* Table configuration */ - switch (app.pipeline_type) { - case e_APP_PIPELINE_HASH_KEY8_EXT: - case e_APP_PIPELINE_HASH_KEY16_EXT: - case e_APP_PIPELINE_HASH_KEY32_EXT: - { - struct rte_table_hash_ext_params table_hash_params = { - .key_size = key_size, - .n_keys = 1 << 24, - .n_buckets = 1 << 22, - .n_buckets_ext = 1 << 21, - .f_hash = test_hash, - .seed = 0, - .signature_offset = APP_METADATA_OFFSET(0), - .key_offset = APP_METADATA_OFFSET(32), - }; - - struct rte_pipeline_table_params table_params = { - .ops = &rte_table_hash_ext_ops, - .arg_create = &table_hash_params, - .f_action_hit = NULL, - .f_action_miss = NULL, - .arg_ah = NULL, - .action_data_size = 0, - }; - - if (rte_pipeline_table_create(p, &table_params, &table_id)) - rte_panic("Unable to configure the hash table\n"); - } - break; - - case e_APP_PIPELINE_HASH_KEY8_LRU: - case e_APP_PIPELINE_HASH_KEY16_LRU: - case e_APP_PIPELINE_HASH_KEY32_LRU: - { - struct rte_table_hash_lru_params table_hash_params = { - .key_size = key_size, - .n_keys = 1 << 24, - .n_buckets = 1 << 22, - .f_hash = test_hash, - .seed = 0, - .signature_offset = 0, - .key_offset = 32, - }; - - struct rte_pipeline_table_params table_params = { - .ops = &rte_table_hash_lru_ops, - .arg_create = &table_hash_params, - .f_action_hit = NULL, - .f_action_miss = NULL, - .arg_ah = NULL, - .action_data_size = 0, - }; - - if (rte_pipeline_table_create(p, &table_params, &table_id)) - rte_panic("Unable to configure the hash table\n"); - } - break; - - case e_APP_PIPELINE_HASH_SPEC_KEY8_EXT: - { - struct rte_table_hash_key8_ext_params table_hash_params = { - .n_entries = 1 << 24, - .n_entries_ext = 1 << 23, - .signature_offset = APP_METADATA_OFFSET(0), - .key_offset = APP_METADATA_OFFSET(32), - .key_mask = NULL, - .f_hash = test_hash, - .seed = 0, - }; - - struct rte_pipeline_table_params table_params = { - .ops = &rte_table_hash_key8_ext_ops, - .arg_create = &table_hash_params, - .f_action_hit = NULL, - .f_action_miss = NULL, - .arg_ah = NULL, - .action_data_size = 0, - }; - - if (rte_pipeline_table_create(p, &table_params, &table_id)) - rte_panic("Unable to configure the hash table\n"); - } - break; - - case e_APP_PIPELINE_HASH_SPEC_KEY8_LRU: - { - struct rte_table_hash_key8_lru_params table_hash_params = { - .n_entries = 1 << 24, - .signature_offset = APP_METADATA_OFFSET(0), - .key_offset = APP_METADATA_OFFSET(32), - .key_mask = NULL, - .f_hash = test_hash, - .seed = 0, - }; - - struct rte_pipeline_table_params table_params = { - .ops = &rte_table_hash_key8_lru_ops, - .arg_create = &table_hash_params, - .f_action_hit = NULL, - .f_action_miss = NULL, - .arg_ah = NULL, - .action_data_size = 0, - }; - - if (rte_pipeline_table_create(p, &table_params, &table_id)) - rte_panic("Unable to configure the hash table\n"); - } - break; - - case e_APP_PIPELINE_HASH_SPEC_KEY16_EXT: - { - struct rte_table_hash_key16_ext_params table_hash_params = { - .n_entries = 1 << 24, - .n_entries_ext = 1 << 23, - .signature_offset = APP_METADATA_OFFSET(0), - .key_offset = APP_METADATA_OFFSET(32), - .f_hash = test_hash, - .seed = 0, - .key_mask = NULL, - }; - - struct rte_pipeline_table_params table_params = { - .ops = &rte_table_hash_key16_ext_ops, - .arg_create = &table_hash_params, - .f_action_hit = NULL, - .f_action_miss = NULL, - .arg_ah = NULL, - .action_data_size = 0, - }; - - if (rte_pipeline_table_create(p, &table_params, &table_id)) - rte_panic("Unable to configure the hash table)\n"); - } - break; - - case e_APP_PIPELINE_HASH_SPEC_KEY16_LRU: - { - struct rte_table_hash_key16_lru_params table_hash_params = { - .n_entries = 1 << 24, - .signature_offset = APP_METADATA_OFFSET(0), - .key_offset = APP_METADATA_OFFSET(32), - .f_hash = test_hash, - .seed = 0, - .key_mask = NULL, - }; - - struct rte_pipeline_table_params table_params = { - .ops = &rte_table_hash_key16_lru_ops, - .arg_create = &table_hash_params, - .f_action_hit = NULL, - .f_action_miss = NULL, - .arg_ah = NULL, - .action_data_size = 0, - }; - - if (rte_pipeline_table_create(p, &table_params, &table_id)) - rte_panic("Unable to configure the hash table\n"); - } - break; - - case e_APP_PIPELINE_HASH_SPEC_KEY32_EXT: - { - struct rte_table_hash_key32_ext_params table_hash_params = { - .n_entries = 1 << 24, - .n_entries_ext = 1 << 23, - .signature_offset = APP_METADATA_OFFSET(0), - .key_offset = APP_METADATA_OFFSET(32), - .f_hash = test_hash, - .seed = 0, - }; - - struct rte_pipeline_table_params table_params = { - .ops = &rte_table_hash_key32_ext_ops, - .arg_create = &table_hash_params, - .f_action_hit = NULL, - .f_action_miss = NULL, - .arg_ah = NULL, - .action_data_size = 0, - }; - - if (rte_pipeline_table_create(p, &table_params, &table_id)) - rte_panic("Unable to configure the hash table\n"); - } - break; - - - case e_APP_PIPELINE_HASH_SPEC_KEY32_LRU: - { - struct rte_table_hash_key32_lru_params table_hash_params = { - .n_entries = 1 << 24, - .signature_offset = APP_METADATA_OFFSET(0), - .key_offset = APP_METADATA_OFFSET(32), - .f_hash = test_hash, - .seed = 0, - }; - - struct rte_pipeline_table_params table_params = { - .ops = &rte_table_hash_key32_lru_ops, - .arg_create = &table_hash_params, - .f_action_hit = NULL, - .f_action_miss = NULL, - .arg_ah = NULL, - .action_data_size = 0, - }; - - if (rte_pipeline_table_create(p, &table_params, &table_id)) - rte_panic("Unable to configure the hash table\n"); - } - break; - - default: - rte_panic("Invalid hash table type or key size\n"); - } - - /* Interconnecting ports and tables */ - for (i = 0; i < app.n_ports; i++) - if (rte_pipeline_port_in_connect_to_table(p, port_in_id[i], - table_id)) - rte_panic("Unable to connect input port %u to " - "table %u\n", port_in_id[i], table_id); - - /* Add entries to tables */ - for (i = 0; i < (1 << 24); i++) { - struct rte_pipeline_table_entry entry = { - .action = RTE_PIPELINE_ACTION_PORT, - {.port_id = port_out_id[i & (app.n_ports - 1)]}, - }; - struct rte_pipeline_table_entry *entry_ptr; - uint8_t key[32]; - uint32_t *k32 = (uint32_t *) key; - int key_found, status; - - memset(key, 0, sizeof(key)); - k32[0] = rte_be_to_cpu_32(i); - - status = rte_pipeline_table_entry_add(p, table_id, key, &entry, - &key_found, &entry_ptr); - if (status < 0) - rte_panic("Unable to add entry to table %u (%d)\n", - table_id, status); - } - - /* Enable input ports */ - for (i = 0; i < app.n_ports; i++) - if (rte_pipeline_port_in_enable(p, port_in_id[i])) - rte_panic("Unable to enable input port %u\n", - port_in_id[i]); - - /* Check pipeline consistency */ - if (rte_pipeline_check(p) < 0) - rte_panic("Pipeline consistency check failed\n"); - - /* Run-time */ -#if APP_FLUSH == 0 - for ( ; ; ) - rte_pipeline_run(p); -#else - for (i = 0; ; i++) { - rte_pipeline_run(p); - - if ((i & APP_FLUSH) == 0) - rte_pipeline_flush(p); - } -#endif -} - -uint64_t test_hash( - void *key, - __attribute__((unused)) uint32_t key_size, - __attribute__((unused)) uint64_t seed) -{ - uint32_t *k32 = (uint32_t *) key; - uint32_t ip_dst = rte_be_to_cpu_32(k32[0]); - uint64_t signature = (ip_dst >> 2) | ((ip_dst & 0x3) << 30); - - return signature; -} - -void -app_main_loop_rx_metadata(void) { - uint32_t i, j; - int ret; - - RTE_LOG(INFO, USER1, "Core %u is doing RX (with meta-data)\n", - rte_lcore_id()); - - for (i = 0; ; i = ((i + 1) & (app.n_ports - 1))) { - uint16_t n_mbufs; - - n_mbufs = rte_eth_rx_burst( - app.ports[i], - 0, - app.mbuf_rx.array, - app.burst_size_rx_read); - - if (n_mbufs == 0) - continue; - - for (j = 0; j < n_mbufs; j++) { - struct rte_mbuf *m; - uint8_t *m_data, *key; - struct ipv4_hdr *ip_hdr; - struct ipv6_hdr *ipv6_hdr; - uint32_t ip_dst; - uint8_t *ipv6_dst; - uint32_t *signature, *k32; - - m = app.mbuf_rx.array[j]; - m_data = rte_pktmbuf_mtod(m, uint8_t *); - signature = RTE_MBUF_METADATA_UINT32_PTR(m, - APP_METADATA_OFFSET(0)); - key = RTE_MBUF_METADATA_UINT8_PTR(m, - APP_METADATA_OFFSET(32)); - - if (RTE_ETH_IS_IPV4_HDR(m->packet_type)) { - ip_hdr = (struct ipv4_hdr *) - &m_data[sizeof(struct ether_hdr)]; - ip_dst = ip_hdr->dst_addr; - - k32 = (uint32_t *) key; - k32[0] = ip_dst & 0xFFFFFF00; - } else if (RTE_ETH_IS_IPV6_HDR(m->packet_type)) { - ipv6_hdr = (struct ipv6_hdr *) - &m_data[sizeof(struct ether_hdr)]; - ipv6_dst = ipv6_hdr->dst_addr; - - memcpy(key, ipv6_dst, 16); - } else - continue; - - *signature = test_hash(key, 0, 0); - } - - do { - ret = rte_ring_sp_enqueue_bulk( - app.rings_rx[i], - (void **) app.mbuf_rx.array, - n_mbufs); - } while (ret < 0); - } -} diff --git a/dpdk/app/test-pipeline/runtime.c b/dpdk/app/test-pipeline/runtime.c deleted file mode 100644 index 42a6142c..00000000 --- a/dpdk/app/test-pipeline/runtime.c +++ /dev/null @@ -1,184 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "main.h" - -void -app_main_loop_rx(void) { - uint32_t i; - int ret; - - RTE_LOG(INFO, USER1, "Core %u is doing RX\n", rte_lcore_id()); - - for (i = 0; ; i = ((i + 1) & (app.n_ports - 1))) { - uint16_t n_mbufs; - - n_mbufs = rte_eth_rx_burst( - app.ports[i], - 0, - app.mbuf_rx.array, - app.burst_size_rx_read); - - if (n_mbufs == 0) - continue; - - do { - ret = rte_ring_sp_enqueue_bulk( - app.rings_rx[i], - (void **) app.mbuf_rx.array, - n_mbufs); - } while (ret < 0); - } -} - -void -app_main_loop_worker(void) { - struct app_mbuf_array *worker_mbuf; - uint32_t i; - - RTE_LOG(INFO, USER1, "Core %u is doing work (no pipeline)\n", - rte_lcore_id()); - - worker_mbuf = rte_malloc_socket(NULL, sizeof(struct app_mbuf_array), - RTE_CACHE_LINE_SIZE, rte_socket_id()); - if (worker_mbuf == NULL) - rte_panic("Worker thread: cannot allocate buffer space\n"); - - for (i = 0; ; i = ((i + 1) & (app.n_ports - 1))) { - int ret; - - ret = rte_ring_sc_dequeue_bulk( - app.rings_rx[i], - (void **) worker_mbuf->array, - app.burst_size_worker_read); - - if (ret == -ENOENT) - continue; - - do { - ret = rte_ring_sp_enqueue_bulk( - app.rings_tx[i ^ 1], - (void **) worker_mbuf->array, - app.burst_size_worker_write); - } while (ret < 0); - } -} - -void -app_main_loop_tx(void) { - uint32_t i; - - RTE_LOG(INFO, USER1, "Core %u is doing TX\n", rte_lcore_id()); - - for (i = 0; ; i = ((i + 1) & (app.n_ports - 1))) { - uint16_t n_mbufs, n_pkts; - int ret; - - n_mbufs = app.mbuf_tx[i].n_mbufs; - - ret = rte_ring_sc_dequeue_bulk( - app.rings_tx[i], - (void **) &app.mbuf_tx[i].array[n_mbufs], - app.burst_size_tx_read); - - if (ret == -ENOENT) - continue; - - n_mbufs += app.burst_size_tx_read; - - if (n_mbufs < app.burst_size_tx_write) { - app.mbuf_tx[i].n_mbufs = n_mbufs; - continue; - } - - n_pkts = rte_eth_tx_burst( - app.ports[i], - 0, - app.mbuf_tx[i].array, - n_mbufs); - - if (n_pkts < n_mbufs) { - uint16_t k; - - for (k = n_pkts; k < n_mbufs; k++) { - struct rte_mbuf *pkt_to_free; - - pkt_to_free = app.mbuf_tx[i].array[k]; - rte_pktmbuf_free(pkt_to_free); - } - } - - app.mbuf_tx[i].n_mbufs = 0; - } -} diff --git a/dpdk/app/test-pmd/Makefile b/dpdk/app/test-pmd/Makefile index 2a0b5a5d..4993c918 100644 --- a/dpdk/app/test-pmd/Makefile +++ b/dpdk/app/test-pmd/Makefile @@ -47,6 +47,9 @@ CFLAGS += $(WERROR_FLAGS) SRCS-y := testpmd.c SRCS-y += parameters.c SRCS-$(CONFIG_RTE_LIBRTE_CMDLINE) += cmdline.c +SRCS-$(CONFIG_RTE_LIBRTE_CMDLINE) += cmdline_flow.c +SRCS-$(CONFIG_RTE_LIBRTE_CMDLINE) += cmdline_mtr.c +SRCS-$(CONFIG_RTE_LIBRTE_CMDLINE) += cmdline_tm.c SRCS-y += config.c SRCS-y += iofwd.c SRCS-y += macfwd.c @@ -58,10 +61,35 @@ SRCS-y += csumonly.c SRCS-y += icmpecho.c SRCS-$(CONFIG_RTE_LIBRTE_IEEE1588) += ieee1588fwd.c -CFLAGS_cmdline.o := -D_GNU_SOURCE +ifeq ($(CONFIG_RTE_LIBRTE_PMD_SOFTNIC)$(CONFIG_RTE_LIBRTE_SCHED),yy) +SRCS-y += tm.c +endif -# this application needs libraries first -DEPDIRS-y += lib drivers +ifeq ($(CONFIG_RTE_BUILD_SHARED_LIB),y) + +ifeq ($(CONFIG_RTE_LIBRTE_PMD_BOND),y) +LDLIBS += -lrte_pmd_bond +endif + +ifeq ($(CONFIG_RTE_LIBRTE_IXGBE_PMD),y) +LDLIBS += -lrte_pmd_ixgbe +endif + +ifeq ($(CONFIG_RTE_LIBRTE_I40E_PMD),y) +LDLIBS += -lrte_pmd_i40e +endif + +ifeq ($(CONFIG_RTE_LIBRTE_BNXT_PMD),y) +LDLIBS += -lrte_pmd_bnxt +endif + +ifeq ($(CONFIG_RTE_LIBRTE_PMD_SOFTNIC),y) +LDLIBS += -lrte_pmd_softnic +endif + +endif + +CFLAGS_cmdline.o := -D_GNU_SOURCE include $(RTE_SDK)/mk/rte.app.mk diff --git a/dpdk/app/test-pmd/cmdline.c b/dpdk/app/test-pmd/cmdline.c index 09a28327..b3c3f241 100644 --- a/dpdk/app/test-pmd/cmdline.c +++ b/dpdk/app/test-pmd/cmdline.c @@ -36,7 +36,6 @@ #include #include #include -#include #include #include #include @@ -75,6 +74,8 @@ #include #include #include +#include +#include #include #include @@ -86,9 +87,20 @@ #include #ifdef RTE_LIBRTE_PMD_BOND #include +#include +#endif +#ifdef RTE_LIBRTE_IXGBE_PMD +#include +#endif +#ifdef RTE_LIBRTE_I40E_PMD +#include +#endif +#ifdef RTE_LIBRTE_BNXT_PMD +#include #endif - #include "testpmd.h" +#include "cmdline_mtr.h" +#include "cmdline_tm.h" static struct cmdline *testpmd_cl; @@ -125,7 +137,7 @@ cmdline_parse_token_string_t cmd_help_brief_help = cmdline_parse_inst_t cmd_help_brief = { .f = cmd_help_brief_parsed, .data = NULL, - .help_str = "show help", + .help_str = "help: Show help", .tokens = { (void *)&cmd_help_brief_help, NULL, @@ -180,7 +192,7 @@ static void cmd_help_long_parsed(void *parsed_result, "Display:\n" "--------\n\n" - "show port (info|stats|xstats|fdir|stat_qmap|dcb_tc) (port_id|all)\n" + "show port (info|stats|xstats|fdir|stat_qmap|dcb_tc|cap) (port_id|all)\n" " Display information for port_id, or all.\n\n" "show port X rss reta (size) (mask0,mask1,...)\n" @@ -208,6 +220,39 @@ static void cmd_help_long_parsed(void *parsed_result, "read txd (port_id) (queue_id) (txd_id)\n" " Display a TX descriptor of a port TX queue.\n\n" + + "ddp get list (port_id)\n" + " Get ddp profile info list\n\n" + + "ddp get info (profile_path)\n" + " Get ddp profile information.\n\n" + + "show vf stats (port_id) (vf_id)\n" + " Display a VF's statistics.\n\n" + + "clear vf stats (port_id) (vf_id)\n" + " Reset a VF's statistics.\n\n" + + "show port (port_id) pctype mapping\n" + " Get flow ptype to pctype mapping on a port\n\n" + + "show port meter stats (port_id) (meter_id) (clear)\n" + " Get meter stats on a port\n\n" + "show port tm cap (port_id)\n" + " Display the port TM capability.\n\n" + + "show port tm level cap (port_id) (level_id)\n" + " Display the port TM hierarchical level capability.\n\n" + + "show port tm node cap (port_id) (node_id)\n" + " Display the port TM node capability.\n\n" + + "show port tm node type (port_id) (node_id)\n" + " Display the port TM node type.\n\n" + + "show port tm node stats (port_id) (node_id) (clear)\n" + " Display the port TM node stats.\n\n" + ); } @@ -260,12 +305,66 @@ static void cmd_help_long_parsed(void *parsed_result, "set portlist (x[,y]*)\n" " Set the list of forwarding ports.\n\n" + "set tx loopback (port_id) (on|off)\n" + " Enable or disable tx loopback.\n\n" + + "set all queues drop (port_id) (on|off)\n" + " Set drop enable bit for all queues.\n\n" + + "set vf split drop (port_id) (vf_id) (on|off)\n" + " Set split drop enable bit for a VF from the PF.\n\n" + + "set vf mac antispoof (port_id) (vf_id) (on|off).\n" + " Set MAC antispoof for a VF from the PF.\n\n" + + "set macsec offload (port_id) on encrypt (on|off) replay-protect (on|off)\n" + " Enable MACsec offload.\n\n" + + "set macsec offload (port_id) off\n" + " Disable MACsec offload.\n\n" + + "set macsec sc (tx|rx) (port_id) (mac) (pi)\n" + " Configure MACsec secure connection (SC).\n\n" + + "set macsec sa (tx|rx) (port_id) (idx) (an) (pn) (key)\n" + " Configure MACsec secure association (SA).\n\n" + + "set vf broadcast (port_id) (vf_id) (on|off)\n" + " Set VF broadcast for a VF from the PF.\n\n" + "vlan set strip (on|off) (port_id)\n" " Set the VLAN strip on a port.\n\n" "vlan set stripq (on|off) (port_id,queue_id)\n" " Set the VLAN strip for a queue on a port.\n\n" + "set vf vlan stripq (port_id) (vf_id) (on|off)\n" + " Set the VLAN strip for all queues in a pool for a VF from the PF.\n\n" + + "set vf vlan insert (port_id) (vf_id) (vlan_id)\n" + " Set VLAN insert for a VF from the PF.\n\n" + + "set vf vlan antispoof (port_id) (vf_id) (on|off)\n" + " Set VLAN antispoof for a VF from the PF.\n\n" + + "set vf vlan tag (port_id) (vf_id) (on|off)\n" + " Set VLAN tag for a VF from the PF.\n\n" + + "set vf tx max-bandwidth (port_id) (vf_id) (bandwidth)\n" + " Set a VF's max bandwidth(Mbps).\n\n" + + "set vf tc tx min-bandwidth (port_id) (vf_id) (bw1, bw2, ...)\n" + " Set all TCs' min bandwidth(%%) on a VF.\n\n" + + "set vf tc tx max-bandwidth (port_id) (vf_id) (tc_no) (bandwidth)\n" + " Set a TC's max bandwidth(Mbps) on a VF.\n\n" + + "set tx strict-link-priority (port_id) (tc_bitmap)\n" + " Set some TCs' strict link priority mode on a physical port.\n\n" + + "set tc tx min-bandwidth (port_id) (bw1, bw2, ...)\n" + " Set all TCs' min bandwidth(%%) for all PF and VFs.\n\n" + "vlan set filter (on|off) (port_id)\n" " Set the VLAN filter on a port.\n\n" @@ -347,6 +446,28 @@ static void cmd_help_long_parsed(void *parsed_result, "tso show (portid)" " Display the status of TCP Segmentation Offload.\n\n" + "set port (port_id) gro on|off\n" + " Enable or disable Generic Receive Offload in" + " csum forwarding engine.\n\n" + + "show port (port_id) gro\n" + " Display GRO configuration.\n\n" + + "set gro flush (cycles)\n" + " Set the cycle to flush GROed packets from" + " reassembly tables.\n\n" + + "set port (port_id) gso (on|off)" + " Enable or disable Generic Segmentation Offload in" + " csum forwarding engine.\n\n" + + "set gso segsz (length)\n" + " Set max packet length for output GSO segments," + " including packet header and payload.\n\n" + + "show port (port_id) gso\n" + " Show GSO configuration.\n\n" + "set fwd (%s)\n" " Set packet forwarding mode.\n\n" @@ -356,9 +477,15 @@ static void cmd_help_long_parsed(void *parsed_result, "mac_addr remove (port_id) (XX:XX:XX:XX:XX:XX)\n" " Remove a MAC address from port_id.\n\n" + "mac_addr set (port_id) (XX:XX:XX:XX:XX:XX)\n" + " Set the default MAC address for port_id.\n\n" + "mac_addr add port (port_id) vf (vf_id) (mac_address)\n" " Add a MAC address for a VF on the port.\n\n" + "set vf mac addr (port_id) (vf_id) (XX:XX:XX:XX:XX:XX)\n" + " Set the MAC address for a VF from the PF.\n\n" + "set port (port_id) uta (mac_address|all) (on|off)\n" " Add/Remove a or all unicast hash filter(s)" "from port X.\n\n" @@ -369,6 +496,12 @@ static void cmd_help_long_parsed(void *parsed_result, "set allmulti (port_id|all) (on|off)\n" " Set the allmulti mode on port_id, or all.\n\n" + "set vf promisc (port_id) (vf_id) (on|off)\n" + " Set unicast promiscuous mode for a VF from the PF.\n\n" + + "set vf allmulti (port_id) (vf_id) (on|off)\n" + " Set multicast promiscuous mode for a VF from the PF.\n\n" + "set flow_ctrl rx (on|off) tx (on|off) (high_water)" " (low_water) (pause_time) (send_xon) mac_ctrl_frame_fwd" " (on|off) autoneg (on|off) (port_id)\n" @@ -393,6 +526,10 @@ static void cmd_help_long_parsed(void *parsed_result, " e.g., 'set stat_qmap rx 0 2 5' sets rx queue 2" " on port 0 to mapping 5.\n\n" + "set xstats-hide-zero on|off\n" + " Set the option to hide the zero values" + " for xstats display.\n" + "set port (port_id) vf (vf_id) rx|tx on|off\n" " Enable/Disable a VF receive/tranmit from a port\n\n" @@ -438,7 +575,6 @@ static void cmd_help_long_parsed(void *parsed_result, " Flush (default) or don't flush RX streams before" " forwarding. Mainly used with PCAP drivers.\n\n" - #ifdef RTE_NIC_BYPASS "set bypass mode (normal|bypass|isolate) (port_id)\n" " Set the bypass mode for the lowest port on bypass enabled" " NIC.\n\n" @@ -461,7 +597,7 @@ static void cmd_help_long_parsed(void *parsed_result, "show bypass config (port_id)\n" " Show the bypass configuration for a bypass enabled NIC" " using the lowest port on the NIC.\n\n" -#endif + #ifdef RTE_LIBRTE_PMD_BOND "create bonded device (mode) (socket)\n" " Create a new bonded device with specific bonding mode and socket.\n\n" @@ -484,11 +620,18 @@ static void cmd_help_long_parsed(void *parsed_result, "set bonding mac_addr (port_id) (address)\n" " Set the MAC address of a bonded device.\n\n" + "set bonding mode IEEE802.3AD aggregator policy (port_id) (agg_name)" + " Set Aggregation mode for IEEE802.3AD (mode 4)" + "set bonding xmit_balance_policy (port_id) (l2|l23|l34)\n" " Set the transmit balance policy for bonded device running in balance mode.\n\n" "set bonding mon_period (port_id) (value)\n" " Set the bonding link status monitoring polling period in ms.\n\n" + + "set bonding lacp dedicated_queues (enable|disable)\n" + " Enable/disable dedicated queues for LACP control traffic.\n\n" + #endif "set link-up port (port_id)\n" " Set link up for a port.\n\n" @@ -517,6 +660,123 @@ static void cmd_help_long_parsed(void *parsed_result, "E-tag set filter del e-tag-id (value) port (port_id)\n" " Delete an E-tag forwarding filter on a port\n\n" +#if defined RTE_LIBRTE_PMD_SOFTNIC && defined RTE_LIBRTE_SCHED + "set port tm hierarchy default (port_id)\n" + " Set default traffic Management hierarchy on a port\n\n" + +#endif + "ddp add (port_id) (profile_path[,output_path])\n" + " Load a profile package on a port\n\n" + + "ddp del (port_id) (profile_path)\n" + " Delete a profile package from a port\n\n" + + "ptype mapping get (port_id) (valid_only)\n" + " Get ptype mapping on a port\n\n" + + "ptype mapping replace (port_id) (target) (mask) (pky_type)\n" + " Replace target with the pkt_type in ptype mapping\n\n" + + "ptype mapping reset (port_id)\n" + " Reset ptype mapping on a port\n\n" + + "ptype mapping update (port_id) (hw_ptype) (sw_ptype)\n" + " Update a ptype mapping item on a port\n\n" + + "set port (port_id) queue-region region_id (value) " + "queue_start_index (value) queue_num (value)\n" + " Set a queue region on a port\n\n" + + "set port (port_id) queue-region region_id (value) " + "flowtype (value)\n" + " Set a flowtype region index on a port\n\n" + + "set port (port_id) queue-region UP (value) region_id (value)\n" + " Set the mapping of User Priority to " + "queue region on a port\n\n" + + "set port (port_id) queue-region flush (on|off)\n" + " flush all queue region related configuration\n\n" + + "add port meter profile srtcm_rfc2697 (port_id) (profile_id) (cir) (cbs) (ebs) (color_aware)\n" + " meter profile add - srtcm rfc 2697\n\n" + + "add port meter profile trtcm_rfc2698 (port_id) (profile_id) (cir) (pir) (cbs) (pbs)\n" + " meter profile add - trtcm rfc 2698\n\n" + + "add port meter profile trtcm_rfc4115 (port_id) (profile_id) (cir) (eir) (cbs) (ebs)\n" + " meter profile add - trtcm rfc 4115\n\n" + + "del port meter profile (port_id) (profile_id)\n" + " meter profile delete\n\n" + + "set port meter (port_id) (mtr_id) (profile_id) (g_action) (y_action) (r_action) (stats_mask) (shared)\n" + " meter create\n\n" + + "del port meter (port_id) (mtr_id)\n" + " meter delete\n\n" + + "set port meter profile (port_id) (mtr_id) (profile_id)\n" + " meter update meter profile\n\n" + + "set port meter policer action (port_id) (mtr_id) (color) (action)\n" + " meter update policer action\n\n" + + "set port meter stats mask (port_id) (mtr_id) (stats_mask)\n" + " meter update stats\n\n" + + "show port (port_id) queue-region\n" + " show all queue region related configuration info\n\n" + + "add port tm node shaper profile (port_id) (shaper_profile_id)" + " (tb_rate) (tb_size) (packet_length_adjust)\n" + " Add port tm node private shaper profile.\n\n" + + "del port tm node shaper profile (port_id) (shaper_profile_id)\n" + " Delete port tm node private shaper profile.\n\n" + + "add port tm node shared shaper (port_id) (shared_shaper_id)" + " (shaper_profile_id)\n" + " Add/update port tm node shared shaper.\n\n" + + "del port tm node shared shaper (port_id) (shared_shaper_id)\n" + " Delete port tm node shared shaper.\n\n" + + "set port tm node shaper profile (port_id) (node_id)" + " (shaper_profile_id)\n" + " Set port tm node shaper profile.\n\n" + + "add port tm node wred profile (port_id) (wred_profile_id)" + " (color_g) (min_th_g) (max_th_g) (maxp_inv_g) (wq_log2_g)" + " (color_y) (min_th_y) (max_th_y) (maxp_inv_y) (wq_log2_y)" + " (color_r) (min_th_r) (max_th_r) (maxp_inv_r) (wq_log2_r)\n" + " Add port tm node wred profile.\n\n" + + "del port tm node wred profile (port_id) (wred_profile_id)\n" + " Delete port tm node wred profile.\n\n" + + "add port tm nonleaf node (port_id) (node_id) (parent_node_id)" + " (priority) (weight) (level_id) (shaper_profile_id)" + " (n_sp_priorities) (stats_mask) (n_shared_shapers)" + " [(shared_shaper_id_0) (shared_shaper_id_1)...]\n" + " Add port tm nonleaf node.\n\n" + + "add port tm leaf node (port_id) (node_id) (parent_node_id)" + " (priority) (weight) (level_id) (shaper_profile_id)" + " (cman_mode) (wred_profile_id) (stats_mask) (n_shared_shapers)" + " [(shared_shaper_id_0) (shared_shaper_id_1)...]\n" + " Add port tm leaf node.\n\n" + + "del port tm node (port_id) (node_id)\n" + " Delete port tm node.\n\n" + + "set port tm node parent (port_id) (node_id) (parent_node_id)" + " (priority) (weight)\n" + " Set port tm node parent.\n\n" + + "port tm hierarchy commit (port_id) (clean_on_fail)\n" + " Commit tm hierarchy.\n\n" + , list_pkt_forwarding_modes() ); } @@ -545,7 +805,7 @@ static void cmd_help_long_parsed(void *parsed_result, " Detach physical or virtual dev by port_id\n\n" "port config (port_id|all)" - " speed (10|100|1000|10000|40000|100000|auto)" + " speed (10|100|1000|10000|25000|40000|50000|100000|auto)" " duplex (half|full|auto)\n" " Set speed and duplex for all ports or port_id\n\n" @@ -555,13 +815,14 @@ static void cmd_help_long_parsed(void *parsed_result, "port config all max-pkt-len (value)\n" " Set the max packet length.\n\n" - "port config all (crc-strip|scatter|rx-cksum|hw-vlan|hw-vlan-filter|" + "port config all (crc-strip|scatter|rx-cksum|rx-timestamp|hw-vlan|hw-vlan-filter|" "hw-vlan-strip|hw-vlan-extend|drop-en)" " (on|off)\n" " Set crc-strip/scatter/rx-checksum/hardware-vlan/drop_en" " for ports.\n\n" - "port config all rss (all|ip|tcp|udp|sctp|ether|port|vxlan|geneve|nvgre|none)\n" + "port config all rss (all|ip|tcp|udp|sctp|ether|port|vxlan|" + "geneve|nvgre|none|)\n" " Set the RSS mode.\n\n" "port config port-id rss reta (hash,queue)[,(hash,queue)]\n" @@ -596,6 +857,13 @@ static void cmd_help_long_parsed(void *parsed_result, "port config (port_id|all) l2-tunnel E-tag" " (enable|disable)\n" " Enable/disable the E-tag support.\n\n" + + "port config (port_id) pctype mapping reset\n" + " Reset flow type to pctype mapping on a port\n\n" + + "port config (port_id) pctype mapping update" + " (pctype_id_0[,pctype_id_1]*) (flow_type_id)\n" + " Update a flow type to pctype mapping item on a port\n\n" ); } @@ -758,8 +1026,8 @@ static void cmd_help_long_parsed(void *parsed_result, "set_hash_input_set (port_id) (ipv4|ipv4-frag|" "ipv4-tcp|ipv4-udp|ipv4-sctp|ipv4-other|ipv6|" "ipv6-frag|ipv6-tcp|ipv6-udp|ipv6-sctp|ipv6-other|" - "l2_payload) (ovlan|ivlan|src-ipv4|dst-ipv4|src-ipv6|" - "dst-ipv6|ipv4-tos|ipv4-proto|ipv6-tc|" + "l2_payload|) (ovlan|ivlan|src-ipv4|dst-ipv4|" + "src-ipv6|dst-ipv6|ipv4-tos|ipv4-proto|ipv6-tc|" "ipv6-next-header|udp-src-port|udp-dst-port|" "tcp-src-port|tcp-dst-port|sctp-src-port|" "sctp-dst-port|sctp-veri-tag|udp-key|gre-key|fld-1st|" @@ -777,6 +1045,37 @@ static void cmd_help_long_parsed(void *parsed_result, "sctp-src-port|sctp-dst-port|sctp-veri-tag|none)" " (select|add)\n" " Set the input set for FDir.\n\n" + + "flow validate {port_id}" + " [group {group_id}] [priority {level}]" + " [ingress] [egress]" + " pattern {item} [/ {item} [...]] / end" + " actions {action} [/ {action} [...]] / end\n" + " Check whether a flow rule can be created.\n\n" + + "flow create {port_id}" + " [group {group_id}] [priority {level}]" + " [ingress] [egress]" + " pattern {item} [/ {item} [...]] / end" + " actions {action} [/ {action} [...]] / end\n" + " Create a flow rule.\n\n" + + "flow destroy {port_id} rule {rule_id} [...]\n" + " Destroy specific flow rules.\n\n" + + "flow flush {port_id}\n" + " Destroy all flow rules.\n\n" + + "flow query {port_id} {rule_id} {action}\n" + " Query an existing flow rule.\n\n" + + "flow list {port_id} [group {group_id}] [...]\n" + " List existing flow rules sorted by priority," + " filtered by group identifiers.\n\n" + + "flow isolate {port_id} {boolean}\n" + " Restrict ingress traffic to the defined" + " flow rules\n\n" ); } } @@ -792,7 +1091,8 @@ cmdline_parse_token_string_t cmd_help_long_section = cmdline_parse_inst_t cmd_help_long = { .f = cmd_help_long_parsed, .data = NULL, - .help_str = "show help", + .help_str = "help all|control|display|config|ports|register|filters: " + "Show help", .tokens = { (void *)&cmd_help_long_help, (void *)&cmd_help_long_section, @@ -820,6 +1120,8 @@ static void cmd_operate_port_parsed(void *parsed_result, stop_port(RTE_PORT_ALL); else if (!strcmp(res->name, "close")) close_port(RTE_PORT_ALL); + else if (!strcmp(res->name, "reset")) + reset_port(RTE_PORT_ALL); else printf("Unknown parameter\n"); } @@ -829,14 +1131,14 @@ cmdline_parse_token_string_t cmd_operate_port_all_cmd = "port"); cmdline_parse_token_string_t cmd_operate_port_all_port = TOKEN_STRING_INITIALIZER(struct cmd_operate_port_result, name, - "start#stop#close"); + "start#stop#close#reset"); cmdline_parse_token_string_t cmd_operate_port_all_all = TOKEN_STRING_INITIALIZER(struct cmd_operate_port_result, value, "all"); cmdline_parse_inst_t cmd_operate_port = { .f = cmd_operate_port_parsed, .data = NULL, - .help_str = "port start|stop|close all: start/stop/close all ports", + .help_str = "port start|stop|close all: Start/Stop/Close/Reset all ports", .tokens = { (void *)&cmd_operate_port_all_cmd, (void *)&cmd_operate_port_all_port, @@ -864,6 +1166,8 @@ static void cmd_operate_specific_port_parsed(void *parsed_result, stop_port(res->value); else if (!strcmp(res->name, "close")) close_port(res->value); + else if (!strcmp(res->name, "reset")) + reset_port(res->value); else printf("Unknown parameter\n"); } @@ -873,7 +1177,7 @@ cmdline_parse_token_string_t cmd_operate_specific_port_cmd = keyword, "port"); cmdline_parse_token_string_t cmd_operate_specific_port_port = TOKEN_STRING_INITIALIZER(struct cmd_operate_specific_port_result, - name, "start#stop#close"); + name, "start#stop#close#reset"); cmdline_parse_token_num_t cmd_operate_specific_port_id = TOKEN_NUM_INITIALIZER(struct cmd_operate_specific_port_result, value, UINT8); @@ -881,7 +1185,7 @@ cmdline_parse_token_num_t cmd_operate_specific_port_id = cmdline_parse_inst_t cmd_operate_specific_port = { .f = cmd_operate_specific_port_parsed, .data = NULL, - .help_str = "port start|stop|close X: start/stop/close port X", + .help_str = "port start|stop|close : Start/Stop/Close/Reset port_id", .tokens = { (void *)&cmd_operate_specific_port_cmd, (void *)&cmd_operate_specific_port_port, @@ -922,8 +1226,8 @@ cmdline_parse_token_string_t cmd_operate_attach_port_identifier = cmdline_parse_inst_t cmd_operate_attach_port = { .f = cmd_operate_attach_port_parsed, .data = NULL, - .help_str = "port attach identifier, " - "identifier: pci address or virtual dev name", + .help_str = "port attach : " + "(identifier: pci address or virtual dev name)", .tokens = { (void *)&cmd_operate_attach_port_port, (void *)&cmd_operate_attach_port_keyword, @@ -936,7 +1240,7 @@ cmdline_parse_inst_t cmd_operate_attach_port = { struct cmd_operate_detach_port_result { cmdline_fixed_string_t port; cmdline_fixed_string_t keyword; - uint8_t port_id; + portid_t port_id; }; static void cmd_operate_detach_port_parsed(void *parsed_result, @@ -959,12 +1263,12 @@ cmdline_parse_token_string_t cmd_operate_detach_port_keyword = keyword, "detach"); cmdline_parse_token_num_t cmd_operate_detach_port_port_id = TOKEN_NUM_INITIALIZER(struct cmd_operate_detach_port_result, - port_id, UINT8); + port_id, UINT16); cmdline_parse_inst_t cmd_operate_detach_port = { .f = cmd_operate_detach_port_parsed, .data = NULL, - .help_str = "port detach port_id", + .help_str = "port detach ", .tokens = { (void *)&cmd_operate_detach_port_port, (void *)&cmd_operate_detach_port_keyword, @@ -1016,8 +1320,12 @@ parse_and_check_speed_duplex(char *speedstr, char *duplexstr, uint32_t *speed) *speed = ETH_LINK_SPEED_1G; } else if (!strcmp(speedstr, "10000")) { *speed = ETH_LINK_SPEED_10G; + } else if (!strcmp(speedstr, "25000")) { + *speed = ETH_LINK_SPEED_25G; } else if (!strcmp(speedstr, "40000")) { *speed = ETH_LINK_SPEED_40G; + } else if (!strcmp(speedstr, "50000")) { + *speed = ETH_LINK_SPEED_50G; } else if (!strcmp(speedstr, "100000")) { *speed = ETH_LINK_SPEED_100G; } else if (!strcmp(speedstr, "auto")) { @@ -1049,7 +1357,7 @@ cmd_config_speed_all_parsed(void *parsed_result, &link_speed) < 0) return; - FOREACH_PORT(pid, ports) { + RTE_ETH_FOREACH_DEV(pid) { ports[pid].dev_conf.link_speeds = link_speed; } @@ -1067,7 +1375,7 @@ cmdline_parse_token_string_t cmd_config_speed_all_item1 = TOKEN_STRING_INITIALIZER(struct cmd_config_speed_all, item1, "speed"); cmdline_parse_token_string_t cmd_config_speed_all_value1 = TOKEN_STRING_INITIALIZER(struct cmd_config_speed_all, value1, - "10#100#1000#10000#40000#100000#auto"); + "10#100#1000#10000#25000#40000#50000#100000#auto"); cmdline_parse_token_string_t cmd_config_speed_all_item2 = TOKEN_STRING_INITIALIZER(struct cmd_config_speed_all, item2, "duplex"); cmdline_parse_token_string_t cmd_config_speed_all_value2 = @@ -1077,7 +1385,8 @@ cmdline_parse_token_string_t cmd_config_speed_all_value2 = cmdline_parse_inst_t cmd_config_speed_all = { .f = cmd_config_speed_all_parsed, .data = NULL, - .help_str = "port config all speed 10|100|1000|10000|40000|100000|auto duplex " + .help_str = "port config all speed " + "10|100|1000|10000|25000|40000|50000|100000|auto duplex " "half|full|auto", .tokens = { (void *)&cmd_config_speed_all_port, @@ -1141,7 +1450,7 @@ cmdline_parse_token_string_t cmd_config_speed_specific_item1 = "speed"); cmdline_parse_token_string_t cmd_config_speed_specific_value1 = TOKEN_STRING_INITIALIZER(struct cmd_config_speed_specific, value1, - "10#100#1000#10000#40000#100000#auto"); + "10#100#1000#10000#25000#40000#50000#100000#auto"); cmdline_parse_token_string_t cmd_config_speed_specific_item2 = TOKEN_STRING_INITIALIZER(struct cmd_config_speed_specific, item2, "duplex"); @@ -1152,7 +1461,8 @@ cmdline_parse_token_string_t cmd_config_speed_specific_value2 = cmdline_parse_inst_t cmd_config_speed_specific = { .f = cmd_config_speed_specific_parsed, .data = NULL, - .help_str = "port config X speed 10|100|1000|10000|40000|100000|auto duplex " + .help_str = "port config speed " + "10|100|1000|10000|25000|40000|50000|100000|auto duplex " "half|full|auto", .tokens = { (void *)&cmd_config_speed_specific_port, @@ -1191,6 +1501,8 @@ cmd_config_rx_tx_parsed(void *parsed_result, printf("Warning: Either rx or tx queues should be non zero\n"); return; } + if (check_nb_rxq(res->value) != 0) + return; nb_rxq = res->value; } else if (!strcmp(res->name, "txq")) { @@ -1198,6 +1510,8 @@ cmd_config_rx_tx_parsed(void *parsed_result, printf("Warning: Either rx or tx queues should be non zero\n"); return; } + if (check_nb_txq(res->value) != 0) + return; nb_txq = res->value; } else if (!strcmp(res->name, "rxd")) { @@ -1241,7 +1555,7 @@ cmdline_parse_token_num_t cmd_config_rx_tx_value = cmdline_parse_inst_t cmd_config_rx_tx = { .f = cmd_config_rx_tx_parsed, .data = NULL, - .help_str = "port config all rxq|txq|rxd|txd value", + .help_str = "port config all rxq|txq|rxd|txd ", .tokens = { (void *)&cmd_config_rx_tx_port, (void *)&cmd_config_rx_tx_keyword, @@ -1316,7 +1630,7 @@ cmdline_parse_token_num_t cmd_config_max_pkt_len_value = cmdline_parse_inst_t cmd_config_max_pkt_len = { .f = cmd_config_max_pkt_len_parsed, .data = NULL, - .help_str = "port config all max-pkt-len value", + .help_str = "port config all max-pkt-len ", .tokens = { (void *)&cmd_config_max_pkt_len_port, (void *)&cmd_config_max_pkt_len_keyword, @@ -1332,7 +1646,7 @@ struct cmd_config_mtu_result { cmdline_fixed_string_t port; cmdline_fixed_string_t keyword; cmdline_fixed_string_t mtu; - uint8_t port_id; + portid_t port_id; uint16_t value; }; @@ -1360,14 +1674,14 @@ cmdline_parse_token_string_t cmd_config_mtu_mtu = TOKEN_STRING_INITIALIZER(struct cmd_config_mtu_result, keyword, "mtu"); cmdline_parse_token_num_t cmd_config_mtu_port_id = - TOKEN_NUM_INITIALIZER(struct cmd_config_mtu_result, port_id, UINT8); + TOKEN_NUM_INITIALIZER(struct cmd_config_mtu_result, port_id, UINT16); cmdline_parse_token_num_t cmd_config_mtu_value = TOKEN_NUM_INITIALIZER(struct cmd_config_mtu_result, value, UINT16); cmdline_parse_inst_t cmd_config_mtu = { .f = cmd_config_mtu_parsed, .data = NULL, - .help_str = "port config mtu port_id value", + .help_str = "port config mtu ", .tokens = { (void *)&cmd_config_mtu_port, (void *)&cmd_config_mtu_keyword, @@ -1426,6 +1740,15 @@ cmd_config_rx_mode_flag_parsed(void *parsed_result, printf("Unknown parameter\n"); return; } + } else if (!strcmp(res->name, "rx-timestamp")) { + if (!strcmp(res->value, "on")) + rx_mode.hw_timestamp = 1; + else if (!strcmp(res->value, "off")) + rx_mode.hw_timestamp = 0; + else { + printf("Unknown parameter\n"); + return; + } } else if (!strcmp(res->name, "hw-vlan")) { if (!strcmp(res->value, "on")) { rx_mode.hw_vlan_filter = 1; @@ -1494,7 +1817,7 @@ cmdline_parse_token_string_t cmd_config_rx_mode_flag_all = TOKEN_STRING_INITIALIZER(struct cmd_config_rx_mode_flag, all, "all"); cmdline_parse_token_string_t cmd_config_rx_mode_flag_name = TOKEN_STRING_INITIALIZER(struct cmd_config_rx_mode_flag, name, - "crc-strip#scatter#rx-cksum#hw-vlan#" + "crc-strip#scatter#rx-cksum#rx-timestamp#hw-vlan#" "hw-vlan-filter#hw-vlan-strip#hw-vlan-extend"); cmdline_parse_token_string_t cmd_config_rx_mode_flag_value = TOKEN_STRING_INITIALIZER(struct cmd_config_rx_mode_flag, value, @@ -1503,7 +1826,7 @@ cmdline_parse_token_string_t cmd_config_rx_mode_flag_value = cmdline_parse_inst_t cmd_config_rx_mode_flag = { .f = cmd_config_rx_mode_flag_parsed, .data = NULL, - .help_str = "port config all crc-strip|scatter|rx-cksum|hw-vlan|" + .help_str = "port config all crc-strip|scatter|rx-cksum|rx-timestamp|hw-vlan|" "hw-vlan-filter|hw-vlan-strip|hw-vlan-extend on|off", .tokens = { (void *)&cmd_config_rx_mode_flag_port, @@ -1530,7 +1853,7 @@ cmd_config_rss_parsed(void *parsed_result, __attribute__((unused)) void *data) { struct cmd_config_rss *res = parsed_result; - struct rte_eth_rss_conf rss_conf; + struct rte_eth_rss_conf rss_conf = { .rss_key_len = 0, }; int diag; uint8_t i; @@ -1558,6 +1881,9 @@ cmd_config_rss_parsed(void *parsed_result, rss_conf.rss_hf = ETH_RSS_NVGRE; else if (!strcmp(res->value, "none")) rss_conf.rss_hf = 0; + else if (isdigit(res->value[0]) && atoi(res->value) > 0 && + atoi(res->value) < 64) + rss_conf.rss_hf = 1ULL << atoi(res->value); else { printf("Unknown parameter\n"); return; @@ -1581,13 +1907,13 @@ cmdline_parse_token_string_t cmd_config_rss_all = cmdline_parse_token_string_t cmd_config_rss_name = TOKEN_STRING_INITIALIZER(struct cmd_config_rss, name, "rss"); cmdline_parse_token_string_t cmd_config_rss_value = - TOKEN_STRING_INITIALIZER(struct cmd_config_rss, value, - "all#ip#tcp#udp#sctp#ether#port#vxlan#geneve#nvgre#none"); + TOKEN_STRING_INITIALIZER(struct cmd_config_rss, value, NULL); cmdline_parse_inst_t cmd_config_rss = { .f = cmd_config_rss_parsed, .data = NULL, - .help_str = "port config all rss all|ip|tcp|udp|sctp|ether|port|vxlan|geneve|nvgre|none", + .help_str = "port config all rss " + "all|ip|tcp|udp|sctp|ether|port|vxlan|geneve|nvgre|none|", .tokens = { (void *)&cmd_config_rss_port, (void *)&cmd_config_rss_keyword, @@ -1602,7 +1928,7 @@ cmdline_parse_inst_t cmd_config_rss = { struct cmd_config_rss_hash_key { cmdline_fixed_string_t port; cmdline_fixed_string_t config; - uint8_t port_id; + portid_t port_id; cmdline_fixed_string_t rss_hash_key; cmdline_fixed_string_t rss_type; cmdline_fixed_string_t key; @@ -1684,7 +2010,7 @@ cmdline_parse_token_string_t cmd_config_rss_hash_key_config = TOKEN_STRING_INITIALIZER(struct cmd_config_rss_hash_key, config, "config"); cmdline_parse_token_num_t cmd_config_rss_hash_key_port_id = - TOKEN_NUM_INITIALIZER(struct cmd_config_rss_hash_key, port_id, UINT8); + TOKEN_NUM_INITIALIZER(struct cmd_config_rss_hash_key, port_id, UINT16); cmdline_parse_token_string_t cmd_config_rss_hash_key_rss_hash_key = TOKEN_STRING_INITIALIZER(struct cmd_config_rss_hash_key, rss_hash_key, "rss-hash-key"); @@ -1700,12 +2026,11 @@ cmdline_parse_token_string_t cmd_config_rss_hash_key_value = cmdline_parse_inst_t cmd_config_rss_hash_key = { .f = cmd_config_rss_hash_key_parsed, .data = NULL, - .help_str = - "port config X rss-hash-key ipv4|ipv4-frag|ipv4-tcp|ipv4-udp|" - "ipv4-sctp|ipv4-other|ipv6|ipv6-frag|ipv6-tcp|ipv6-udp|" - "ipv6-sctp|ipv6-other|l2-payload|" - "ipv6-ex|ipv6-tcp-ex|ipv6-udp-ex " - "\n", + .help_str = "port config rss-hash-key " + "ipv4|ipv4-frag|ipv4-tcp|ipv4-udp|ipv4-sctp|ipv4-other|" + "ipv6|ipv6-frag|ipv6-tcp|ipv6-udp|ipv6-sctp|ipv6-other|" + "l2-payload|ipv6-ex|ipv6-tcp-ex|ipv6-udp-ex " + "", .tokens = { (void *)&cmd_config_rss_hash_key_port, (void *)&cmd_config_rss_hash_key_config, @@ -1720,7 +2045,7 @@ cmdline_parse_inst_t cmd_config_rss_hash_key = { /* *** configure port rxq/txq start/stop *** */ struct cmd_config_rxtx_queue { cmdline_fixed_string_t port; - uint8_t portid; + portid_t portid; cmdline_fixed_string_t rxtxq; uint16_t qid; cmdline_fixed_string_t opname; @@ -1788,7 +2113,7 @@ cmd_config_rxtx_queue_parsed(void *parsed_result, cmdline_parse_token_string_t cmd_config_rxtx_queue_port = TOKEN_STRING_INITIALIZER(struct cmd_config_rxtx_queue, port, "port"); cmdline_parse_token_num_t cmd_config_rxtx_queue_portid = - TOKEN_NUM_INITIALIZER(struct cmd_config_rxtx_queue, portid, UINT8); + TOKEN_NUM_INITIALIZER(struct cmd_config_rxtx_queue, portid, UINT16); cmdline_parse_token_string_t cmd_config_rxtx_queue_rxtxq = TOKEN_STRING_INITIALIZER(struct cmd_config_rxtx_queue, rxtxq, "rxq#txq"); cmdline_parse_token_num_t cmd_config_rxtx_queue_qid = @@ -1800,7 +2125,7 @@ cmdline_parse_token_string_t cmd_config_rxtx_queue_opname = cmdline_parse_inst_t cmd_config_rxtx_queue = { .f = cmd_config_rxtx_queue_parsed, .data = NULL, - .help_str = "port X rxq|txq ID start|stop", + .help_str = "port rxq|txq start|stop", .tokens = { (void *)&cmd_config_speed_all_port, (void *)&cmd_config_rxtx_queue_portid, @@ -1815,7 +2140,7 @@ cmdline_parse_inst_t cmd_config_rxtx_queue = { struct cmd_config_rss_reta { cmdline_fixed_string_t port; cmdline_fixed_string_t keyword; - uint8_t port_id; + portid_t port_id; cmdline_fixed_string_t name; cmdline_fixed_string_t list_name; cmdline_fixed_string_t list_of_items; @@ -1924,7 +2249,7 @@ cmdline_parse_token_string_t cmd_config_rss_reta_port = cmdline_parse_token_string_t cmd_config_rss_reta_keyword = TOKEN_STRING_INITIALIZER(struct cmd_config_rss_reta, keyword, "config"); cmdline_parse_token_num_t cmd_config_rss_reta_port_id = - TOKEN_NUM_INITIALIZER(struct cmd_config_rss_reta, port_id, UINT8); + TOKEN_NUM_INITIALIZER(struct cmd_config_rss_reta, port_id, UINT16); cmdline_parse_token_string_t cmd_config_rss_reta_name = TOKEN_STRING_INITIALIZER(struct cmd_config_rss_reta, name, "rss"); cmdline_parse_token_string_t cmd_config_rss_reta_list_name = @@ -1935,7 +2260,7 @@ cmdline_parse_token_string_t cmd_config_rss_reta_list_of_items = cmdline_parse_inst_t cmd_config_rss_reta = { .f = cmd_set_rss_reta_parsed, .data = NULL, - .help_str = "port config X rss reta (hash,queue)[,(hash,queue)]", + .help_str = "port config rss reta ", .tokens = { (void *)&cmd_config_rss_reta_port, (void *)&cmd_config_rss_reta_keyword, @@ -1951,7 +2276,7 @@ cmdline_parse_inst_t cmd_config_rss_reta = { struct cmd_showport_reta { cmdline_fixed_string_t show; cmdline_fixed_string_t port; - uint8_t port_id; + portid_t port_id; cmdline_fixed_string_t rss; cmdline_fixed_string_t reta; uint16_t size; @@ -1968,7 +2293,9 @@ showport_parse_reta_config(struct rte_eth_rss_reta_entry64 *conf, char s[256]; char *end; char *str_fld[8]; - uint16_t i, num = nb_entries / RTE_RETA_GROUP_SIZE; + uint16_t i; + uint16_t num = (nb_entries + RTE_RETA_GROUP_SIZE - 1) / + RTE_RETA_GROUP_SIZE; int ret; p = strchr(p0, '('); @@ -2004,12 +2331,14 @@ cmd_showport_reta_parsed(void *parsed_result, struct cmd_showport_reta *res = parsed_result; struct rte_eth_rss_reta_entry64 reta_conf[8]; struct rte_eth_dev_info dev_info; + uint16_t max_reta_size; memset(&dev_info, 0, sizeof(dev_info)); rte_eth_dev_info_get(res->port_id, &dev_info); - if (dev_info.reta_size == 0 || res->size != dev_info.reta_size || - res->size > ETH_RSS_RETA_SIZE_512) { - printf("Invalid redirection table size: %u\n", res->size); + max_reta_size = RTE_MIN(dev_info.reta_size, ETH_RSS_RETA_SIZE_512); + if (res->size == 0 || res->size > max_reta_size) { + printf("Invalid redirection table size: %u (1-%u)\n", + res->size, max_reta_size); return; } @@ -2028,7 +2357,7 @@ cmdline_parse_token_string_t cmd_showport_reta_show = cmdline_parse_token_string_t cmd_showport_reta_port = TOKEN_STRING_INITIALIZER(struct cmd_showport_reta, port, "port"); cmdline_parse_token_num_t cmd_showport_reta_port_id = - TOKEN_NUM_INITIALIZER(struct cmd_showport_reta, port_id, UINT8); + TOKEN_NUM_INITIALIZER(struct cmd_showport_reta, port_id, UINT16); cmdline_parse_token_string_t cmd_showport_reta_rss = TOKEN_STRING_INITIALIZER(struct cmd_showport_reta, rss, "rss"); cmdline_parse_token_string_t cmd_showport_reta_reta = @@ -2042,7 +2371,7 @@ cmdline_parse_token_string_t cmd_showport_reta_list_of_items = cmdline_parse_inst_t cmd_showport_reta = { .f = cmd_showport_reta_parsed, .data = NULL, - .help_str = "show port X rss reta (size) (mask0,mask1,...)", + .help_str = "show port rss reta ", .tokens = { (void *)&cmd_showport_reta_show, (void *)&cmd_showport_reta_port, @@ -2059,7 +2388,7 @@ cmdline_parse_inst_t cmd_showport_reta = { struct cmd_showport_rss_hash { cmdline_fixed_string_t show; cmdline_fixed_string_t port; - uint8_t port_id; + portid_t port_id; cmdline_fixed_string_t rss_hash; cmdline_fixed_string_t rss_type; cmdline_fixed_string_t key; /* optional argument */ @@ -2080,7 +2409,7 @@ cmdline_parse_token_string_t cmd_showport_rss_hash_show = cmdline_parse_token_string_t cmd_showport_rss_hash_port = TOKEN_STRING_INITIALIZER(struct cmd_showport_rss_hash, port, "port"); cmdline_parse_token_num_t cmd_showport_rss_hash_port_id = - TOKEN_NUM_INITIALIZER(struct cmd_showport_rss_hash, port_id, UINT8); + TOKEN_NUM_INITIALIZER(struct cmd_showport_rss_hash, port_id, UINT16); cmdline_parse_token_string_t cmd_showport_rss_hash_rss_hash = TOKEN_STRING_INITIALIZER(struct cmd_showport_rss_hash, rss_hash, "rss-hash"); @@ -2096,11 +2425,10 @@ cmdline_parse_token_string_t cmd_showport_rss_hash_rss_key = cmdline_parse_inst_t cmd_showport_rss_hash = { .f = cmd_showport_rss_hash_parsed, .data = NULL, - .help_str = - "show port X rss-hash ipv4|ipv4-frag|ipv4-tcp|ipv4-udp|" - "ipv4-sctp|ipv4-other|ipv6|ipv6-frag|ipv6-tcp|ipv6-udp|" - "ipv6-sctp|ipv6-other|l2-payload|" - "ipv6-ex|ipv6-tcp-ex|ipv6-udp-ex (X = port number)\n", + .help_str = "show port rss-hash " + "ipv4|ipv4-frag|ipv4-tcp|ipv4-udp|ipv4-sctp|ipv4-other|" + "ipv6|ipv6-frag|ipv6-tcp|ipv6-udp|ipv6-sctp|ipv6-other|" + "l2-payload|ipv6-ex|ipv6-tcp-ex|ipv6-udp-ex", .tokens = { (void *)&cmd_showport_rss_hash_show, (void *)&cmd_showport_rss_hash_port, @@ -2114,11 +2442,10 @@ cmdline_parse_inst_t cmd_showport_rss_hash = { cmdline_parse_inst_t cmd_showport_rss_hash_key = { .f = cmd_showport_rss_hash_parsed, .data = (void *)1, - .help_str = - "show port X rss-hash ipv4|ipv4-frag|ipv4-tcp|ipv4-udp|" - "ipv4-sctp|ipv4-other|ipv6|ipv6-frag|ipv6-tcp|ipv6-udp|" - "ipv6-sctp|ipv6-other|l2-payload|" - "ipv6-ex|ipv6-tcp-ex|ipv6-udp-ex key (X = port number)\n", + .help_str = "show port rss-hash " + "ipv4|ipv4-frag|ipv4-tcp|ipv4-udp|ipv4-sctp|ipv4-other|" + "ipv6|ipv6-frag|ipv6-tcp|ipv6-udp|ipv6-sctp|ipv6-other|" + "l2-payload|ipv6-ex|ipv6-tcp-ex|ipv6-udp-ex key", .tokens = { (void *)&cmd_showport_rss_hash_show, (void *)&cmd_showport_rss_hash_port, @@ -2134,7 +2461,7 @@ cmdline_parse_inst_t cmd_showport_rss_hash_key = { struct cmd_config_dcb { cmdline_fixed_string_t port; cmdline_fixed_string_t config; - uint8_t port_id; + portid_t port_id; cmdline_fixed_string_t dcb; cmdline_fixed_string_t vt; cmdline_fixed_string_t vt_en; @@ -2200,7 +2527,7 @@ cmdline_parse_token_string_t cmd_config_dcb_port = cmdline_parse_token_string_t cmd_config_dcb_config = TOKEN_STRING_INITIALIZER(struct cmd_config_dcb, config, "config"); cmdline_parse_token_num_t cmd_config_dcb_port_id = - TOKEN_NUM_INITIALIZER(struct cmd_config_dcb, port_id, UINT8); + TOKEN_NUM_INITIALIZER(struct cmd_config_dcb, port_id, UINT16); cmdline_parse_token_string_t cmd_config_dcb_dcb = TOKEN_STRING_INITIALIZER(struct cmd_config_dcb, dcb, "dcb"); cmdline_parse_token_string_t cmd_config_dcb_vt = @@ -2215,10 +2542,10 @@ cmdline_parse_token_string_t cmd_config_dcb_pfc_en = TOKEN_STRING_INITIALIZER(struct cmd_config_dcb, pfc_en, "on#off"); cmdline_parse_inst_t cmd_config_dcb = { - .f = cmd_config_dcb_parsed, - .data = NULL, - .help_str = "port config port-id dcb vt on|off nb-tcs pfc on|off", - .tokens = { + .f = cmd_config_dcb_parsed, + .data = NULL, + .help_str = "port config dcb vt on|off pfc on|off", + .tokens = { (void *)&cmd_config_dcb_port, (void *)&cmd_config_dcb_config, (void *)&cmd_config_dcb_port_id, @@ -2283,7 +2610,7 @@ cmdline_parse_token_num_t cmd_config_burst_value = cmdline_parse_inst_t cmd_config_burst = { .f = cmd_config_burst_parsed, .data = NULL, - .help_str = "port config all burst value", + .help_str = "port config all burst ", .tokens = { (void *)&cmd_config_burst_port, (void *)&cmd_config_burst_keyword, @@ -2352,7 +2679,7 @@ cmdline_parse_token_num_t cmd_config_thresh_value = cmdline_parse_inst_t cmd_config_thresh = { .f = cmd_config_thresh_parsed, .data = NULL, - .help_str = "port config all txpt|txht|txwt|rxpt|rxht|rxwt value", + .help_str = "port config all txpt|txht|txwt|rxpt|rxht|rxwt ", .tokens = { (void *)&cmd_config_thresh_port, (void *)&cmd_config_thresh_keyword, @@ -2416,7 +2743,7 @@ cmdline_parse_token_num_t cmd_config_threshold_value = cmdline_parse_inst_t cmd_config_threshold = { .f = cmd_config_threshold_parsed, .data = NULL, - .help_str = "port config all txfreet|txrst|rxfreet value", + .help_str = "port config all txfreet|txrst|rxfreet ", .tokens = { (void *)&cmd_config_threshold_port, (void *)&cmd_config_threshold_keyword, @@ -2445,7 +2772,7 @@ cmdline_parse_token_string_t cmd_stop_stop = cmdline_parse_inst_t cmd_stop = { .f = cmd_stop_parsed, .data = NULL, - .help_str = "stop - stop packet forwarding", + .help_str = "stop: Stop packet forwarding", .tokens = { (void *)&cmd_stop_stop, NULL, @@ -2575,7 +2902,7 @@ cmdline_parse_token_string_t cmd_set_list_of_items = cmdline_parse_inst_t cmd_set_fwd_list = { .f = cmd_set_list_parsed, .data = NULL, - .help_str = "set corelist|portlist x[,y]*", + .help_str = "set corelist|portlist ", .tokens = { (void *)&cmd_set_list_keyword, (void *)&cmd_set_list_name, @@ -2622,7 +2949,7 @@ cmdline_parse_token_num_t cmd_setmask_value = cmdline_parse_inst_t cmd_set_fwd_mask = { .f = cmd_set_mask_parsed, .data = NULL, - .help_str = "set coremask|portmask hexadecimal value", + .help_str = "set coremask|portmask ", .tokens = { (void *)&cmd_setmask_set, (void *)&cmd_setmask_mask, @@ -2668,7 +2995,7 @@ cmdline_parse_token_num_t cmd_set_value = cmdline_parse_inst_t cmd_set_numbers = { .f = cmd_set_parsed, .data = NULL, - .help_str = "set nbport|nbcore|burst|verbose value", + .help_str = "set nbport|nbcore|burst|verbose ", .tokens = { (void *)&cmd_set_set, (void *)&cmd_set_what, @@ -2714,7 +3041,7 @@ cmdline_parse_token_string_t cmd_set_txpkts_lengths = cmdline_parse_inst_t cmd_set_txpkts = { .f = cmd_set_txpkts_parsed, .data = NULL, - .help_str = "set txpkts x[,y]*", + .help_str = "set txpkts ", .tokens = { (void *)&cmd_set_txpkts_keyword, (void *)&cmd_set_txpkts_name, @@ -2821,7 +3148,7 @@ cmdline_parse_token_num_t cmd_config_txqflags_value = cmdline_parse_inst_t cmd_config_txqflags = { .f = cmd_config_txqflags_parsed, .data = NULL, - .help_str = "port config all txqflags value", + .help_str = "port config all txqflags ", .tokens = { (void *)&cmd_config_txqflags_port, (void *)&cmd_config_txqflags_config, @@ -2837,7 +3164,7 @@ struct cmd_rx_vlan_filter_all_result { cmdline_fixed_string_t rx_vlan; cmdline_fixed_string_t what; cmdline_fixed_string_t all; - uint8_t port_id; + portid_t port_id; }; static void @@ -2864,13 +3191,14 @@ cmdline_parse_token_string_t cmd_rx_vlan_filter_all_all = all, "all"); cmdline_parse_token_num_t cmd_rx_vlan_filter_all_portid = TOKEN_NUM_INITIALIZER(struct cmd_rx_vlan_filter_all_result, - port_id, UINT8); + port_id, UINT16); cmdline_parse_inst_t cmd_rx_vlan_filter_all = { .f = cmd_rx_vlan_filter_all_parsed, .data = NULL, - .help_str = "add/remove all identifiers to/from the set of VLAN " - "Identifiers filtered by a port", + .help_str = "rx_vlan add|rm all : " + "Add/Remove all identifiers to/from the set of VLAN " + "identifiers filtered by a port", .tokens = { (void *)&cmd_rx_vlan_filter_all_rx_vlan, (void *)&cmd_rx_vlan_filter_all_what, @@ -2969,8 +3297,9 @@ cmdline_parse_token_string_t cmd_vlan_offload_portid = cmdline_parse_inst_t cmd_vlan_offload = { .f = cmd_vlan_offload_parsed, .data = NULL, - .help_str = "set strip|filter|qinq|stripq on|off port_id[,queue_id], filter/strip for rx side" - " qinq(extended) for both rx/tx sides ", + .help_str = "vlan set strip|filter|qinq|stripq on|off " + ": " + "Filter/Strip for rx side qinq(extended) for both rx/tx sides", .tokens = { (void *)&cmd_vlan_offload_vlan, (void *)&cmd_vlan_offload_set, @@ -2988,7 +3317,7 @@ struct cmd_vlan_tpid_result { cmdline_fixed_string_t vlan_type; cmdline_fixed_string_t what; uint16_t tp_id; - uint8_t port_id; + portid_t port_id; }; static void @@ -3027,13 +3356,13 @@ cmdline_parse_token_num_t cmd_vlan_tpid_tpid = tp_id, UINT16); cmdline_parse_token_num_t cmd_vlan_tpid_portid = TOKEN_NUM_INITIALIZER(struct cmd_vlan_tpid_result, - port_id, UINT8); + port_id, UINT16); cmdline_parse_inst_t cmd_vlan_tpid = { .f = cmd_vlan_tpid_parsed, .data = NULL, - .help_str = "set inner|outer tpid tp_id port_id, set the VLAN " - "Ether type", + .help_str = "vlan set inner|outer tpid : " + "Set the VLAN Ether type", .tokens = { (void *)&cmd_vlan_tpid_vlan, (void *)&cmd_vlan_tpid_set, @@ -3050,7 +3379,7 @@ struct cmd_rx_vlan_filter_result { cmdline_fixed_string_t rx_vlan; cmdline_fixed_string_t what; uint16_t vlan_id; - uint8_t port_id; + portid_t port_id; }; static void @@ -3077,13 +3406,14 @@ cmdline_parse_token_num_t cmd_rx_vlan_filter_vlanid = vlan_id, UINT16); cmdline_parse_token_num_t cmd_rx_vlan_filter_portid = TOKEN_NUM_INITIALIZER(struct cmd_rx_vlan_filter_result, - port_id, UINT8); + port_id, UINT16); cmdline_parse_inst_t cmd_rx_vlan_filter = { .f = cmd_rx_vlan_filter_parsed, .data = NULL, - .help_str = "add/remove a VLAN identifier to/from the set of VLAN " - "Identifiers filtered by a port", + .help_str = "rx_vlan add|rm : " + "Add/Remove a VLAN identifier to/from the set of VLAN " + "identifiers filtered by a port", .tokens = { (void *)&cmd_rx_vlan_filter_rx_vlan, (void *)&cmd_rx_vlan_filter_what, @@ -3097,7 +3427,7 @@ cmdline_parse_inst_t cmd_rx_vlan_filter = { struct cmd_tx_vlan_set_result { cmdline_fixed_string_t tx_vlan; cmdline_fixed_string_t set; - uint8_t port_id; + portid_t port_id; uint16_t vlan_id; }; @@ -3119,7 +3449,7 @@ cmdline_parse_token_string_t cmd_tx_vlan_set_set = set, "set"); cmdline_parse_token_num_t cmd_tx_vlan_set_portid = TOKEN_NUM_INITIALIZER(struct cmd_tx_vlan_set_result, - port_id, UINT8); + port_id, UINT16); cmdline_parse_token_num_t cmd_tx_vlan_set_vlanid = TOKEN_NUM_INITIALIZER(struct cmd_tx_vlan_set_result, vlan_id, UINT16); @@ -3127,7 +3457,8 @@ cmdline_parse_token_num_t cmd_tx_vlan_set_vlanid = cmdline_parse_inst_t cmd_tx_vlan_set = { .f = cmd_tx_vlan_set_parsed, .data = NULL, - .help_str = "enable hardware insertion of a single VLAN header " + .help_str = "tx_vlan set : " + "Enable hardware insertion of a single VLAN header " "with a given TAG Identifier in packets sent on a port", .tokens = { (void *)&cmd_tx_vlan_set_tx_vlan, @@ -3142,7 +3473,7 @@ cmdline_parse_inst_t cmd_tx_vlan_set = { struct cmd_tx_vlan_set_qinq_result { cmdline_fixed_string_t tx_vlan; cmdline_fixed_string_t set; - uint8_t port_id; + portid_t port_id; uint16_t vlan_id; uint16_t vlan_id_outer; }; @@ -3165,7 +3496,7 @@ cmdline_parse_token_string_t cmd_tx_vlan_set_qinq_set = set, "set"); cmdline_parse_token_num_t cmd_tx_vlan_set_qinq_portid = TOKEN_NUM_INITIALIZER(struct cmd_tx_vlan_set_qinq_result, - port_id, UINT8); + port_id, UINT16); cmdline_parse_token_num_t cmd_tx_vlan_set_qinq_vlanid = TOKEN_NUM_INITIALIZER(struct cmd_tx_vlan_set_qinq_result, vlan_id, UINT16); @@ -3176,7 +3507,8 @@ cmdline_parse_token_num_t cmd_tx_vlan_set_qinq_vlanid_outer = cmdline_parse_inst_t cmd_tx_vlan_set_qinq = { .f = cmd_tx_vlan_set_qinq_parsed, .data = NULL, - .help_str = "enable hardware insertion of double VLAN header " + .help_str = "tx_vlan set : " + "Enable hardware insertion of double VLAN header " "with given TAG Identifiers in packets sent on a port", .tokens = { (void *)&cmd_tx_vlan_set_qinq_tx_vlan, @@ -3193,7 +3525,7 @@ struct cmd_tx_vlan_set_pvid_result { cmdline_fixed_string_t tx_vlan; cmdline_fixed_string_t set; cmdline_fixed_string_t pvid; - uint8_t port_id; + portid_t port_id; uint16_t vlan_id; cmdline_fixed_string_t mode; }; @@ -3222,7 +3554,7 @@ cmdline_parse_token_string_t cmd_tx_vlan_set_pvid_pvid = pvid, "pvid"); cmdline_parse_token_num_t cmd_tx_vlan_set_pvid_port_id = TOKEN_NUM_INITIALIZER(struct cmd_tx_vlan_set_pvid_result, - port_id, UINT8); + port_id, UINT16); cmdline_parse_token_num_t cmd_tx_vlan_set_pvid_vlan_id = TOKEN_NUM_INITIALIZER(struct cmd_tx_vlan_set_pvid_result, vlan_id, UINT16); @@ -3233,7 +3565,7 @@ cmdline_parse_token_string_t cmd_tx_vlan_set_pvid_mode = cmdline_parse_inst_t cmd_tx_vlan_set_pvid = { .f = cmd_tx_vlan_set_pvid_parsed, .data = NULL, - .help_str = "tx_vlan set pvid port_id vlan_id (on|off)", + .help_str = "tx_vlan set pvid on|off", .tokens = { (void *)&cmd_tx_vlan_set_pvid_tx_vlan, (void *)&cmd_tx_vlan_set_pvid_set, @@ -3249,7 +3581,7 @@ cmdline_parse_inst_t cmd_tx_vlan_set_pvid = { struct cmd_tx_vlan_reset_result { cmdline_fixed_string_t tx_vlan; cmdline_fixed_string_t reset; - uint8_t port_id; + portid_t port_id; }; static void @@ -3270,13 +3602,13 @@ cmdline_parse_token_string_t cmd_tx_vlan_reset_reset = reset, "reset"); cmdline_parse_token_num_t cmd_tx_vlan_reset_portid = TOKEN_NUM_INITIALIZER(struct cmd_tx_vlan_reset_result, - port_id, UINT8); + port_id, UINT16); cmdline_parse_inst_t cmd_tx_vlan_reset = { .f = cmd_tx_vlan_reset_parsed, .data = NULL, - .help_str = "disable hardware insertion of a VLAN header in packets " - "sent on a port", + .help_str = "tx_vlan reset : Disable hardware insertion of a " + "VLAN header in packets sent on a port", .tokens = { (void *)&cmd_tx_vlan_reset_tx_vlan, (void *)&cmd_tx_vlan_reset_reset, @@ -3292,7 +3624,7 @@ struct cmd_csum_result { cmdline_fixed_string_t mode; cmdline_fixed_string_t proto; cmdline_fixed_string_t hwsw; - uint8_t port_id; + portid_t port_id; }; static void @@ -3397,13 +3729,14 @@ cmdline_parse_token_string_t cmd_csum_hwsw = hwsw, "hw#sw"); cmdline_parse_token_num_t cmd_csum_portid = TOKEN_NUM_INITIALIZER(struct cmd_csum_result, - port_id, UINT8); + port_id, UINT16); cmdline_parse_inst_t cmd_csum_set = { .f = cmd_csum_parsed, .data = NULL, - .help_str = "enable/disable hardware calculation of L3/L4 checksum when " - "using csum forward engine: csum set ip|tcp|udp|sctp|outer-ip hw|sw ", + .help_str = "csum set ip|tcp|udp|sctp|outer-ip hw|sw : " + "Enable/Disable hardware calculation of L3/L4 checksum when " + "using csum forward engine", .tokens = { (void *)&cmd_csum_csum, (void *)&cmd_csum_mode, @@ -3421,7 +3754,7 @@ cmdline_parse_token_string_t cmd_csum_mode_show = cmdline_parse_inst_t cmd_csum_show = { .f = cmd_csum_parsed, .data = NULL, - .help_str = "show checksum offload configuration: csum show ", + .help_str = "csum show : Show checksum offload configuration", .tokens = { (void *)&cmd_csum_csum, (void *)&cmd_csum_mode_show, @@ -3435,7 +3768,7 @@ struct cmd_csum_tunnel_result { cmdline_fixed_string_t csum; cmdline_fixed_string_t parse; cmdline_fixed_string_t onoff; - uint8_t port_id; + portid_t port_id; }; static void @@ -3469,13 +3802,13 @@ cmdline_parse_token_string_t cmd_csum_tunnel_onoff = onoff, "on#off"); cmdline_parse_token_num_t cmd_csum_tunnel_portid = TOKEN_NUM_INITIALIZER(struct cmd_csum_tunnel_result, - port_id, UINT8); + port_id, UINT16); cmdline_parse_inst_t cmd_csum_tunnel = { .f = cmd_csum_tunnel_parsed, .data = NULL, - .help_str = "enable/disable parsing of tunnels for csum engine: " - "csum parse_tunnel on|off ", + .help_str = "csum parse_tunnel on|off : " + "Enable/Disable parsing of tunnels for csum engine", .tokens = { (void *)&cmd_csum_tunnel_csum, (void *)&cmd_csum_tunnel_parse, @@ -3485,12 +3818,12 @@ cmdline_parse_inst_t cmd_csum_tunnel = { }, }; -/* *** ENABLE HARDWARE SEGMENTATION IN TX PACKETS *** */ +/* *** ENABLE HARDWARE SEGMENTATION IN TX NON-TUNNELED PACKETS *** */ struct cmd_tso_set_result { cmdline_fixed_string_t tso; cmdline_fixed_string_t mode; uint16_t tso_segsz; - uint8_t port_id; + portid_t port_id; }; static void @@ -3508,9 +3841,9 @@ cmd_tso_set_parsed(void *parsed_result, ports[res->port_id].tso_segsz = res->tso_segsz; if (ports[res->port_id].tso_segsz == 0) - printf("TSO is disabled\n"); + printf("TSO for non-tunneled packets is disabled\n"); else - printf("TSO segment size is %d\n", + printf("TSO segment size for non-tunneled packets is %d\n", ports[res->port_id].tso_segsz); /* display warnings if configuration is not supported by the NIC */ @@ -3533,13 +3866,14 @@ cmdline_parse_token_num_t cmd_tso_set_tso_segsz = tso_segsz, UINT16); cmdline_parse_token_num_t cmd_tso_set_portid = TOKEN_NUM_INITIALIZER(struct cmd_tso_set_result, - port_id, UINT8); + port_id, UINT16); cmdline_parse_inst_t cmd_tso_set = { .f = cmd_tso_set_parsed, .data = NULL, - .help_str = "Set TSO segment size for csum engine (0 to disable): " - "tso set ", + .help_str = "tso set : " + "Set TSO segment size of non-tunneled packets for csum engine " + "(0 to disable)", .tokens = { (void *)&cmd_tso_set_tso, (void *)&cmd_tso_set_mode, @@ -3557,8 +3891,8 @@ cmdline_parse_token_string_t cmd_tso_show_mode = cmdline_parse_inst_t cmd_tso_show = { .f = cmd_tso_set_parsed, .data = NULL, - .help_str = "Show TSO segment size for csum engine: " - "tso show ", + .help_str = "tso show : " + "Show TSO segment size of non-tunneled packets for csum engine", .tokens = { (void *)&cmd_tso_set_tso, (void *)&cmd_tso_show_mode, @@ -3567,6 +3901,433 @@ cmdline_parse_inst_t cmd_tso_show = { }, }; +/* *** ENABLE HARDWARE SEGMENTATION IN TX TUNNELED PACKETS *** */ +struct cmd_tunnel_tso_set_result { + cmdline_fixed_string_t tso; + cmdline_fixed_string_t mode; + uint16_t tso_segsz; + portid_t port_id; +}; + +static void +check_tunnel_tso_nic_support(portid_t port_id) +{ + struct rte_eth_dev_info dev_info; + + rte_eth_dev_info_get(port_id, &dev_info); + if (!(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_VXLAN_TNL_TSO)) + printf("Warning: TSO enabled but VXLAN TUNNEL TSO not " + "supported by port %d\n", port_id); + if (!(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_GRE_TNL_TSO)) + printf("Warning: TSO enabled but GRE TUNNEL TSO not " + "supported by port %d\n", port_id); + if (!(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_IPIP_TNL_TSO)) + printf("Warning: TSO enabled but IPIP TUNNEL TSO not " + "supported by port %d\n", port_id); + if (!(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_GENEVE_TNL_TSO)) + printf("Warning: TSO enabled but GENEVE TUNNEL TSO not " + "supported by port %d\n", port_id); +} + +static void +cmd_tunnel_tso_set_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_tunnel_tso_set_result *res = parsed_result; + + if (port_id_is_invalid(res->port_id, ENABLED_WARN)) + return; + + if (!strcmp(res->mode, "set")) + ports[res->port_id].tunnel_tso_segsz = res->tso_segsz; + + if (ports[res->port_id].tunnel_tso_segsz == 0) + printf("TSO for tunneled packets is disabled\n"); + else { + printf("TSO segment size for tunneled packets is %d\n", + ports[res->port_id].tunnel_tso_segsz); + + /* Below conditions are needed to make it work: + * (1) tunnel TSO is supported by the NIC; + * (2) "csum parse_tunnel" must be set so that tunneled pkts + * are recognized; + * (3) for tunneled pkts with outer L3 of IPv4, + * "csum set outer-ip" must be set to hw, because after tso, + * total_len of outer IP header is changed, and the checksum + * of outer IP header calculated by sw should be wrong; that + * is not necessary for IPv6 tunneled pkts because there's no + * checksum in IP header anymore. + */ + check_tunnel_tso_nic_support(res->port_id); + + if (!(ports[res->port_id].tx_ol_flags & + TESTPMD_TX_OFFLOAD_PARSE_TUNNEL)) + printf("Warning: csum parse_tunnel must be set " + "so that tunneled packets are recognized\n"); + if (!(ports[res->port_id].tx_ol_flags & + TESTPMD_TX_OFFLOAD_OUTER_IP_CKSUM)) + printf("Warning: csum set outer-ip must be set to hw " + "if outer L3 is IPv4; not necessary for IPv6\n"); + } +} + +cmdline_parse_token_string_t cmd_tunnel_tso_set_tso = + TOKEN_STRING_INITIALIZER(struct cmd_tunnel_tso_set_result, + tso, "tunnel_tso"); +cmdline_parse_token_string_t cmd_tunnel_tso_set_mode = + TOKEN_STRING_INITIALIZER(struct cmd_tunnel_tso_set_result, + mode, "set"); +cmdline_parse_token_num_t cmd_tunnel_tso_set_tso_segsz = + TOKEN_NUM_INITIALIZER(struct cmd_tunnel_tso_set_result, + tso_segsz, UINT16); +cmdline_parse_token_num_t cmd_tunnel_tso_set_portid = + TOKEN_NUM_INITIALIZER(struct cmd_tunnel_tso_set_result, + port_id, UINT16); + +cmdline_parse_inst_t cmd_tunnel_tso_set = { + .f = cmd_tunnel_tso_set_parsed, + .data = NULL, + .help_str = "tunnel_tso set : " + "Set TSO segment size of tunneled packets for csum engine " + "(0 to disable)", + .tokens = { + (void *)&cmd_tunnel_tso_set_tso, + (void *)&cmd_tunnel_tso_set_mode, + (void *)&cmd_tunnel_tso_set_tso_segsz, + (void *)&cmd_tunnel_tso_set_portid, + NULL, + }, +}; + +cmdline_parse_token_string_t cmd_tunnel_tso_show_mode = + TOKEN_STRING_INITIALIZER(struct cmd_tunnel_tso_set_result, + mode, "show"); + + +cmdline_parse_inst_t cmd_tunnel_tso_show = { + .f = cmd_tunnel_tso_set_parsed, + .data = NULL, + .help_str = "tunnel_tso show " + "Show TSO segment size of tunneled packets for csum engine", + .tokens = { + (void *)&cmd_tunnel_tso_set_tso, + (void *)&cmd_tunnel_tso_show_mode, + (void *)&cmd_tunnel_tso_set_portid, + NULL, + }, +}; + +/* *** SET GRO FOR A PORT *** */ +struct cmd_gro_enable_result { + cmdline_fixed_string_t cmd_set; + cmdline_fixed_string_t cmd_port; + cmdline_fixed_string_t cmd_keyword; + cmdline_fixed_string_t cmd_onoff; + portid_t cmd_pid; +}; + +static void +cmd_gro_enable_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_gro_enable_result *res; + + res = parsed_result; + if (!strcmp(res->cmd_keyword, "gro")) + setup_gro(res->cmd_onoff, res->cmd_pid); +} + +cmdline_parse_token_string_t cmd_gro_enable_set = + TOKEN_STRING_INITIALIZER(struct cmd_gro_enable_result, + cmd_set, "set"); +cmdline_parse_token_string_t cmd_gro_enable_port = + TOKEN_STRING_INITIALIZER(struct cmd_gro_enable_result, + cmd_keyword, "port"); +cmdline_parse_token_num_t cmd_gro_enable_pid = + TOKEN_NUM_INITIALIZER(struct cmd_gro_enable_result, + cmd_pid, UINT16); +cmdline_parse_token_string_t cmd_gro_enable_keyword = + TOKEN_STRING_INITIALIZER(struct cmd_gro_enable_result, + cmd_keyword, "gro"); +cmdline_parse_token_string_t cmd_gro_enable_onoff = + TOKEN_STRING_INITIALIZER(struct cmd_gro_enable_result, + cmd_onoff, "on#off"); + +cmdline_parse_inst_t cmd_gro_enable = { + .f = cmd_gro_enable_parsed, + .data = NULL, + .help_str = "set port gro on|off", + .tokens = { + (void *)&cmd_gro_enable_set, + (void *)&cmd_gro_enable_port, + (void *)&cmd_gro_enable_pid, + (void *)&cmd_gro_enable_keyword, + (void *)&cmd_gro_enable_onoff, + NULL, + }, +}; + +/* *** DISPLAY GRO CONFIGURATION *** */ +struct cmd_gro_show_result { + cmdline_fixed_string_t cmd_show; + cmdline_fixed_string_t cmd_port; + cmdline_fixed_string_t cmd_keyword; + portid_t cmd_pid; +}; + +static void +cmd_gro_show_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_gro_show_result *res; + + res = parsed_result; + if (!strcmp(res->cmd_keyword, "gro")) + show_gro(res->cmd_pid); +} + +cmdline_parse_token_string_t cmd_gro_show_show = + TOKEN_STRING_INITIALIZER(struct cmd_gro_show_result, + cmd_show, "show"); +cmdline_parse_token_string_t cmd_gro_show_port = + TOKEN_STRING_INITIALIZER(struct cmd_gro_show_result, + cmd_port, "port"); +cmdline_parse_token_num_t cmd_gro_show_pid = + TOKEN_NUM_INITIALIZER(struct cmd_gro_show_result, + cmd_pid, UINT16); +cmdline_parse_token_string_t cmd_gro_show_keyword = + TOKEN_STRING_INITIALIZER(struct cmd_gro_show_result, + cmd_keyword, "gro"); + +cmdline_parse_inst_t cmd_gro_show = { + .f = cmd_gro_show_parsed, + .data = NULL, + .help_str = "show port gro", + .tokens = { + (void *)&cmd_gro_show_show, + (void *)&cmd_gro_show_port, + (void *)&cmd_gro_show_pid, + (void *)&cmd_gro_show_keyword, + NULL, + }, +}; + +/* *** SET FLUSH CYCLES FOR GRO *** */ +struct cmd_gro_flush_result { + cmdline_fixed_string_t cmd_set; + cmdline_fixed_string_t cmd_keyword; + cmdline_fixed_string_t cmd_flush; + uint8_t cmd_cycles; +}; + +static void +cmd_gro_flush_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_gro_flush_result *res; + + res = parsed_result; + if ((!strcmp(res->cmd_keyword, "gro")) && + (!strcmp(res->cmd_flush, "flush"))) + setup_gro_flush_cycles(res->cmd_cycles); +} + +cmdline_parse_token_string_t cmd_gro_flush_set = + TOKEN_STRING_INITIALIZER(struct cmd_gro_flush_result, + cmd_set, "set"); +cmdline_parse_token_string_t cmd_gro_flush_keyword = + TOKEN_STRING_INITIALIZER(struct cmd_gro_flush_result, + cmd_keyword, "gro"); +cmdline_parse_token_string_t cmd_gro_flush_flush = + TOKEN_STRING_INITIALIZER(struct cmd_gro_flush_result, + cmd_flush, "flush"); +cmdline_parse_token_num_t cmd_gro_flush_cycles = + TOKEN_NUM_INITIALIZER(struct cmd_gro_flush_result, + cmd_cycles, UINT8); + +cmdline_parse_inst_t cmd_gro_flush = { + .f = cmd_gro_flush_parsed, + .data = NULL, + .help_str = "set gro flush ", + .tokens = { + (void *)&cmd_gro_flush_set, + (void *)&cmd_gro_flush_keyword, + (void *)&cmd_gro_flush_flush, + (void *)&cmd_gro_flush_cycles, + NULL, + }, +}; + +/* *** ENABLE/DISABLE GSO *** */ +struct cmd_gso_enable_result { + cmdline_fixed_string_t cmd_set; + cmdline_fixed_string_t cmd_port; + cmdline_fixed_string_t cmd_keyword; + cmdline_fixed_string_t cmd_mode; + portid_t cmd_pid; +}; + +static void +cmd_gso_enable_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_gso_enable_result *res; + + res = parsed_result; + if (!strcmp(res->cmd_keyword, "gso")) + setup_gso(res->cmd_mode, res->cmd_pid); +} + +cmdline_parse_token_string_t cmd_gso_enable_set = + TOKEN_STRING_INITIALIZER(struct cmd_gso_enable_result, + cmd_set, "set"); +cmdline_parse_token_string_t cmd_gso_enable_port = + TOKEN_STRING_INITIALIZER(struct cmd_gso_enable_result, + cmd_port, "port"); +cmdline_parse_token_string_t cmd_gso_enable_keyword = + TOKEN_STRING_INITIALIZER(struct cmd_gso_enable_result, + cmd_keyword, "gso"); +cmdline_parse_token_string_t cmd_gso_enable_mode = + TOKEN_STRING_INITIALIZER(struct cmd_gso_enable_result, + cmd_mode, "on#off"); +cmdline_parse_token_num_t cmd_gso_enable_pid = + TOKEN_NUM_INITIALIZER(struct cmd_gso_enable_result, + cmd_pid, UINT16); + +cmdline_parse_inst_t cmd_gso_enable = { + .f = cmd_gso_enable_parsed, + .data = NULL, + .help_str = "set port gso on|off", + .tokens = { + (void *)&cmd_gso_enable_set, + (void *)&cmd_gso_enable_port, + (void *)&cmd_gso_enable_pid, + (void *)&cmd_gso_enable_keyword, + (void *)&cmd_gso_enable_mode, + NULL, + }, +}; + +/* *** SET MAX PACKET LENGTH FOR GSO SEGMENTS *** */ +struct cmd_gso_size_result { + cmdline_fixed_string_t cmd_set; + cmdline_fixed_string_t cmd_keyword; + cmdline_fixed_string_t cmd_segsz; + uint16_t cmd_size; +}; + +static void +cmd_gso_size_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_gso_size_result *res = parsed_result; + + if (test_done == 0) { + printf("Before setting GSO segsz, please first" + " stop fowarding\n"); + return; + } + + if (!strcmp(res->cmd_keyword, "gso") && + !strcmp(res->cmd_segsz, "segsz")) { + if (res->cmd_size < RTE_GSO_SEG_SIZE_MIN) + printf("gso_size should be larger than %zu." + " Please input a legal value\n", + RTE_GSO_SEG_SIZE_MIN); + else + gso_max_segment_size = res->cmd_size; + } +} + +cmdline_parse_token_string_t cmd_gso_size_set = + TOKEN_STRING_INITIALIZER(struct cmd_gso_size_result, + cmd_set, "set"); +cmdline_parse_token_string_t cmd_gso_size_keyword = + TOKEN_STRING_INITIALIZER(struct cmd_gso_size_result, + cmd_keyword, "gso"); +cmdline_parse_token_string_t cmd_gso_size_segsz = + TOKEN_STRING_INITIALIZER(struct cmd_gso_size_result, + cmd_segsz, "segsz"); +cmdline_parse_token_num_t cmd_gso_size_size = + TOKEN_NUM_INITIALIZER(struct cmd_gso_size_result, + cmd_size, UINT16); + +cmdline_parse_inst_t cmd_gso_size = { + .f = cmd_gso_size_parsed, + .data = NULL, + .help_str = "set gso segsz ", + .tokens = { + (void *)&cmd_gso_size_set, + (void *)&cmd_gso_size_keyword, + (void *)&cmd_gso_size_segsz, + (void *)&cmd_gso_size_size, + NULL, + }, +}; + +/* *** SHOW GSO CONFIGURATION *** */ +struct cmd_gso_show_result { + cmdline_fixed_string_t cmd_show; + cmdline_fixed_string_t cmd_port; + cmdline_fixed_string_t cmd_keyword; + portid_t cmd_pid; +}; + +static void +cmd_gso_show_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_gso_show_result *res = parsed_result; + + if (!rte_eth_dev_is_valid_port(res->cmd_pid)) { + printf("invalid port id %u\n", res->cmd_pid); + return; + } + if (!strcmp(res->cmd_keyword, "gso")) { + if (gso_ports[res->cmd_pid].enable) { + printf("Max GSO'd packet size: %uB\n" + "Supported GSO types: TCP/IPv4, " + "VxLAN with inner TCP/IPv4 packet, " + "GRE with inner TCP/IPv4 packet\n", + gso_max_segment_size); + } else + printf("GSO is not enabled on Port %u\n", res->cmd_pid); + } +} + +cmdline_parse_token_string_t cmd_gso_show_show = +TOKEN_STRING_INITIALIZER(struct cmd_gso_show_result, + cmd_show, "show"); +cmdline_parse_token_string_t cmd_gso_show_port = +TOKEN_STRING_INITIALIZER(struct cmd_gso_show_result, + cmd_port, "port"); +cmdline_parse_token_string_t cmd_gso_show_keyword = + TOKEN_STRING_INITIALIZER(struct cmd_gso_show_result, + cmd_keyword, "gso"); +cmdline_parse_token_num_t cmd_gso_show_pid = + TOKEN_NUM_INITIALIZER(struct cmd_gso_show_result, + cmd_pid, UINT16); + +cmdline_parse_inst_t cmd_gso_show = { + .f = cmd_gso_show_parsed, + .data = NULL, + .help_str = "show port gso", + .tokens = { + (void *)&cmd_gso_show_show, + (void *)&cmd_gso_show_port, + (void *)&cmd_gso_show_pid, + (void *)&cmd_gso_show_keyword, + NULL, + }, +}; + /* *** ENABLE/DISABLE FLUSH ON RX STREAMS *** */ struct cmd_set_flush_rx { cmdline_fixed_string_t set; @@ -3596,7 +4357,7 @@ cmdline_parse_token_string_t cmd_setflushrx_mode = cmdline_parse_inst_t cmd_set_flush_rx = { .f = cmd_set_flush_rx_parsed, - .help_str = "set flush_rx on|off: enable/disable flush on rx streams", + .help_str = "set flush_rx on|off: Enable/Disable flush on rx streams", .data = NULL, .tokens = { (void *)&cmd_setflushrx_set, @@ -3635,7 +4396,7 @@ cmdline_parse_token_string_t cmd_setlinkcheck_mode = cmdline_parse_inst_t cmd_set_link_check = { .f = cmd_set_link_check_parsed, - .help_str = "set link_check on|off: enable/disable link status check " + .help_str = "set link_check on|off: Enable/Disable link status check " "when starting/stopping a port", .data = NULL, .tokens = { @@ -3646,14 +4407,13 @@ cmdline_parse_inst_t cmd_set_link_check = { }, }; -#ifdef RTE_NIC_BYPASS /* *** SET NIC BYPASS MODE *** */ struct cmd_set_bypass_mode_result { cmdline_fixed_string_t set; cmdline_fixed_string_t bypass; cmdline_fixed_string_t mode; cmdline_fixed_string_t value; - uint8_t port_id; + portid_t port_id; }; static void @@ -3663,19 +4423,23 @@ cmd_set_bypass_mode_parsed(void *parsed_result, { struct cmd_set_bypass_mode_result *res = parsed_result; portid_t port_id = res->port_id; - uint32_t bypass_mode = RTE_BYPASS_MODE_NORMAL; + int32_t rc = -EINVAL; + +#if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS + uint32_t bypass_mode = RTE_PMD_IXGBE_BYPASS_MODE_NORMAL; if (!strcmp(res->value, "bypass")) - bypass_mode = RTE_BYPASS_MODE_BYPASS; + bypass_mode = RTE_PMD_IXGBE_BYPASS_MODE_BYPASS; else if (!strcmp(res->value, "isolate")) - bypass_mode = RTE_BYPASS_MODE_ISOLATE; + bypass_mode = RTE_PMD_IXGBE_BYPASS_MODE_ISOLATE; else - bypass_mode = RTE_BYPASS_MODE_NORMAL; + bypass_mode = RTE_PMD_IXGBE_BYPASS_MODE_NORMAL; /* Set the bypass mode for the relevant port. */ - if (0 != rte_eth_dev_bypass_state_set(port_id, &bypass_mode)) { + rc = rte_pmd_ixgbe_bypass_state_set(port_id, &bypass_mode); +#endif + if (rc != 0) printf("\t Failed to set bypass mode for port = %d.\n", port_id); - } } cmdline_parse_token_string_t cmd_setbypass_mode_set = @@ -3692,11 +4456,11 @@ cmdline_parse_token_string_t cmd_setbypass_mode_value = value, "normal#bypass#isolate"); cmdline_parse_token_num_t cmd_setbypass_mode_port = TOKEN_NUM_INITIALIZER(struct cmd_set_bypass_mode_result, - port_id, UINT8); + port_id, UINT16); cmdline_parse_inst_t cmd_set_bypass_mode = { .f = cmd_set_bypass_mode_parsed, - .help_str = "set bypass mode (normal|bypass|isolate) (port_id): " + .help_str = "set bypass mode normal|bypass|isolate : " "Set the NIC bypass mode for port_id", .data = NULL, .tokens = { @@ -3717,7 +4481,7 @@ struct cmd_set_bypass_event_result { cmdline_fixed_string_t event_value; cmdline_fixed_string_t mode; cmdline_fixed_string_t mode_value; - uint8_t port_id; + portid_t port_id; }; static void @@ -3725,51 +4489,57 @@ cmd_set_bypass_event_parsed(void *parsed_result, __attribute__((unused)) struct cmdline *cl, __attribute__((unused)) void *data) { - int32_t rc; + int32_t rc = -EINVAL; struct cmd_set_bypass_event_result *res = parsed_result; portid_t port_id = res->port_id; - uint32_t bypass_event = RTE_BYPASS_EVENT_NONE; - uint32_t bypass_mode = RTE_BYPASS_MODE_NORMAL; + +#if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS + uint32_t bypass_event = RTE_PMD_IXGBE_BYPASS_EVENT_NONE; + uint32_t bypass_mode = RTE_PMD_IXGBE_BYPASS_MODE_NORMAL; if (!strcmp(res->event_value, "timeout")) - bypass_event = RTE_BYPASS_EVENT_TIMEOUT; + bypass_event = RTE_PMD_IXGBE_BYPASS_EVENT_TIMEOUT; else if (!strcmp(res->event_value, "os_on")) - bypass_event = RTE_BYPASS_EVENT_OS_ON; + bypass_event = RTE_PMD_IXGBE_BYPASS_EVENT_OS_ON; else if (!strcmp(res->event_value, "os_off")) - bypass_event = RTE_BYPASS_EVENT_OS_OFF; + bypass_event = RTE_PMD_IXGBE_BYPASS_EVENT_OS_OFF; else if (!strcmp(res->event_value, "power_on")) - bypass_event = RTE_BYPASS_EVENT_POWER_ON; + bypass_event = RTE_PMD_IXGBE_BYPASS_EVENT_POWER_ON; else if (!strcmp(res->event_value, "power_off")) - bypass_event = RTE_BYPASS_EVENT_POWER_OFF; + bypass_event = RTE_PMD_IXGBE_BYPASS_EVENT_POWER_OFF; else - bypass_event = RTE_BYPASS_EVENT_NONE; + bypass_event = RTE_PMD_IXGBE_BYPASS_EVENT_NONE; if (!strcmp(res->mode_value, "bypass")) - bypass_mode = RTE_BYPASS_MODE_BYPASS; + bypass_mode = RTE_PMD_IXGBE_BYPASS_MODE_BYPASS; else if (!strcmp(res->mode_value, "isolate")) - bypass_mode = RTE_BYPASS_MODE_ISOLATE; + bypass_mode = RTE_PMD_IXGBE_BYPASS_MODE_ISOLATE; else - bypass_mode = RTE_BYPASS_MODE_NORMAL; + bypass_mode = RTE_PMD_IXGBE_BYPASS_MODE_NORMAL; /* Set the watchdog timeout. */ - if (bypass_event == RTE_BYPASS_EVENT_TIMEOUT) { + if (bypass_event == RTE_PMD_IXGBE_BYPASS_EVENT_TIMEOUT) { rc = -EINVAL; - if (!RTE_BYPASS_TMT_VALID(bypass_timeout) || - (rc = rte_eth_dev_wd_timeout_store(port_id, - bypass_timeout)) != 0) { + if (RTE_PMD_IXGBE_BYPASS_TMT_VALID(bypass_timeout)) { + rc = rte_pmd_ixgbe_bypass_wd_timeout_store(port_id, + bypass_timeout); + } + if (rc != 0) { printf("Failed to set timeout value %u " - "for port %d, errto code: %d.\n", - bypass_timeout, port_id, rc); + "for port %d, errto code: %d.\n", + bypass_timeout, port_id, rc); } } /* Set the bypass event to transition to bypass mode. */ - if (0 != rte_eth_dev_bypass_event_store(port_id, - bypass_event, bypass_mode)) { - printf("\t Failed to set bypass event for port = %d.\n", port_id); - } + rc = rte_pmd_ixgbe_bypass_event_store(port_id, bypass_event, + bypass_mode); +#endif + if (rc != 0) + printf("\t Failed to set bypass event for port = %d.\n", + port_id); } cmdline_parse_token_string_t cmd_setbypass_event_set = @@ -3792,13 +4562,13 @@ cmdline_parse_token_string_t cmd_setbypass_event_mode_value = mode_value, "normal#bypass#isolate"); cmdline_parse_token_num_t cmd_setbypass_event_port = TOKEN_NUM_INITIALIZER(struct cmd_set_bypass_event_result, - port_id, UINT8); + port_id, UINT16); cmdline_parse_inst_t cmd_set_bypass_event = { .f = cmd_set_bypass_event_parsed, - .help_str = "set bypass event (timeout|os_on|os_off|power_on|power_off) " - "mode (normal|bypass|isolate) (port_id): " - "Set the NIC bypass event mode for port_id", + .help_str = "set bypass event none|timeout|os_on|os_off|power_on|" + "power_off mode normal|bypass|isolate : " + "Set the NIC bypass event mode for port_id", .data = NULL, .tokens = { (void *)&cmd_setbypass_event_set, @@ -3826,24 +4596,26 @@ cmd_set_bypass_timeout_parsed(void *parsed_result, __attribute__((unused)) struct cmdline *cl, __attribute__((unused)) void *data) { - struct cmd_set_bypass_timeout_result *res = parsed_result; + __rte_unused struct cmd_set_bypass_timeout_result *res = parsed_result; +#if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS if (!strcmp(res->value, "1.5")) - bypass_timeout = RTE_BYPASS_TMT_1_5_SEC; + bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_1_5_SEC; else if (!strcmp(res->value, "2")) - bypass_timeout = RTE_BYPASS_TMT_2_SEC; + bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_2_SEC; else if (!strcmp(res->value, "3")) - bypass_timeout = RTE_BYPASS_TMT_3_SEC; + bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_3_SEC; else if (!strcmp(res->value, "4")) - bypass_timeout = RTE_BYPASS_TMT_4_SEC; + bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_4_SEC; else if (!strcmp(res->value, "8")) - bypass_timeout = RTE_BYPASS_TMT_8_SEC; + bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_8_SEC; else if (!strcmp(res->value, "16")) - bypass_timeout = RTE_BYPASS_TMT_16_SEC; + bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_16_SEC; else if (!strcmp(res->value, "32")) - bypass_timeout = RTE_BYPASS_TMT_32_SEC; + bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_32_SEC; else - bypass_timeout = RTE_BYPASS_TMT_OFF; + bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF; +#endif } cmdline_parse_token_string_t cmd_setbypass_timeout_set = @@ -3861,8 +4633,8 @@ cmdline_parse_token_string_t cmd_setbypass_timeout_value = cmdline_parse_inst_t cmd_set_bypass_timeout = { .f = cmd_set_bypass_timeout_parsed, - .help_str = "set bypass timeout (0|1.5|2|3|4|8|16|32) seconds: " - "Set the NIC bypass watchdog timeout", + .help_str = "set bypass timeout 0|1.5|2|3|4|8|16|32: " + "Set the NIC bypass watchdog timeout in seconds", .data = NULL, .tokens = { (void *)&cmd_setbypass_timeout_set, @@ -3878,7 +4650,7 @@ struct cmd_show_bypass_config_result { cmdline_fixed_string_t show; cmdline_fixed_string_t bypass; cmdline_fixed_string_t config; - uint8_t port_id; + portid_t port_id; }; static void @@ -3887,17 +4659,19 @@ cmd_show_bypass_config_parsed(void *parsed_result, __attribute__((unused)) void *data) { struct cmd_show_bypass_config_result *res = parsed_result; + portid_t port_id = res->port_id; + int rc = -EINVAL; +#if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS uint32_t event_mode; uint32_t bypass_mode; - portid_t port_id = res->port_id; uint32_t timeout = bypass_timeout; int i; - static const char * const timeouts[RTE_BYPASS_TMT_NUM] = + static const char * const timeouts[RTE_PMD_IXGBE_BYPASS_TMT_NUM] = {"off", "1.5", "2", "3", "4", "8", "16", "32"}; - static const char * const modes[RTE_BYPASS_MODE_NUM] = + static const char * const modes[RTE_PMD_IXGBE_BYPASS_MODE_NUM] = {"UNKNOWN", "normal", "bypass", "isolate"}; - static const char * const events[RTE_BYPASS_EVENT_NUM] = { + static const char * const events[RTE_PMD_IXGBE_BYPASS_EVENT_NUM] = { "NONE", "OS/board on", "power supply on", @@ -3907,37 +4681,41 @@ cmd_show_bypass_config_parsed(void *parsed_result, int num_events = (sizeof events) / (sizeof events[0]); /* Display the bypass mode.*/ - if (0 != rte_eth_dev_bypass_state_show(port_id, &bypass_mode)) { + if (rte_pmd_ixgbe_bypass_state_show(port_id, &bypass_mode) != 0) { printf("\tFailed to get bypass mode for port = %d\n", port_id); return; } else { - if (!RTE_BYPASS_MODE_VALID(bypass_mode)) - bypass_mode = RTE_BYPASS_MODE_NONE; + if (!RTE_PMD_IXGBE_BYPASS_MODE_VALID(bypass_mode)) + bypass_mode = RTE_PMD_IXGBE_BYPASS_MODE_NONE; printf("\tbypass mode = %s\n", modes[bypass_mode]); } /* Display the bypass timeout.*/ - if (!RTE_BYPASS_TMT_VALID(timeout)) - timeout = RTE_BYPASS_TMT_OFF; + if (!RTE_PMD_IXGBE_BYPASS_TMT_VALID(timeout)) + timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF; printf("\tbypass timeout = %s\n", timeouts[timeout]); /* Display the bypass events and associated modes. */ - for (i = RTE_BYPASS_EVENT_START; i < num_events; i++) { + for (i = RTE_PMD_IXGBE_BYPASS_EVENT_START; i < num_events; i++) { - if (0 != rte_eth_dev_bypass_event_show(port_id, i, &event_mode)) { + if (rte_pmd_ixgbe_bypass_event_show(port_id, i, &event_mode)) { printf("\tFailed to get bypass mode for event = %s\n", events[i]); } else { - if (!RTE_BYPASS_MODE_VALID(event_mode)) - event_mode = RTE_BYPASS_MODE_NONE; + if (!RTE_PMD_IXGBE_BYPASS_MODE_VALID(event_mode)) + event_mode = RTE_PMD_IXGBE_BYPASS_MODE_NONE; printf("\tbypass event: %-16s = %s\n", events[i], modes[event_mode]); } } +#endif + if (rc != 0) + printf("\tFailed to get bypass configuration for port = %d\n", + port_id); } cmdline_parse_token_string_t cmd_showbypass_config_show = @@ -3951,11 +4729,11 @@ cmdline_parse_token_string_t cmd_showbypass_config_config = config, "config"); cmdline_parse_token_num_t cmd_showbypass_config_port = TOKEN_NUM_INITIALIZER(struct cmd_show_bypass_config_result, - port_id, UINT8); + port_id, UINT16); cmdline_parse_inst_t cmd_show_bypass_config = { .f = cmd_show_bypass_config_parsed, - .help_str = "show bypass config (port_id): " + .help_str = "show bypass config : " "Show the NIC bypass config for port_id", .data = NULL, .tokens = { @@ -3966,7 +4744,6 @@ cmdline_parse_inst_t cmd_show_bypass_config = { NULL, }, }; -#endif #ifdef RTE_LIBRTE_PMD_BOND /* *** SET BONDING MODE *** */ @@ -3975,7 +4752,7 @@ struct cmd_set_bonding_mode_result { cmdline_fixed_string_t bonding; cmdline_fixed_string_t mode; uint8_t value; - uint8_t port_id; + portid_t port_id; }; static void cmd_set_bonding_mode_parsed(void *parsed_result, @@ -4004,11 +4781,12 @@ TOKEN_NUM_INITIALIZER(struct cmd_set_bonding_mode_result, value, UINT8); cmdline_parse_token_num_t cmd_setbonding_mode_port = TOKEN_NUM_INITIALIZER(struct cmd_set_bonding_mode_result, - port_id, UINT8); + port_id, UINT16); cmdline_parse_inst_t cmd_set_bonding_mode = { .f = cmd_set_bonding_mode_parsed, - .help_str = "set bonding mode (mode_value) (port_id): Set the bonding mode for port_id", + .help_str = "set bonding mode : " + "Set the bonding mode for port_id", .data = NULL, .tokens = { (void *) &cmd_setbonding_mode_set, @@ -4020,12 +4798,91 @@ cmdline_parse_inst_t cmd_set_bonding_mode = { } }; +/* *** SET BONDING SLOW_QUEUE SW/HW *** */ +struct cmd_set_bonding_lacp_dedicated_queues_result { + cmdline_fixed_string_t set; + cmdline_fixed_string_t bonding; + cmdline_fixed_string_t lacp; + cmdline_fixed_string_t dedicated_queues; + portid_t port_id; + cmdline_fixed_string_t mode; +}; + +static void cmd_set_bonding_lacp_dedicated_queues_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_set_bonding_lacp_dedicated_queues_result *res = parsed_result; + portid_t port_id = res->port_id; + struct rte_port *port; + + port = &ports[port_id]; + + /** Check if the port is not started **/ + if (port->port_status != RTE_PORT_STOPPED) { + printf("Please stop port %d first\n", port_id); + return; + } + + if (!strcmp(res->mode, "enable")) { + if (rte_eth_bond_8023ad_dedicated_queues_enable(port_id) == 0) + printf("Dedicate queues for LACP control packets" + " enabled\n"); + else + printf("Enabling dedicate queues for LACP control " + "packets on port %d failed\n", port_id); + } else if (!strcmp(res->mode, "disable")) { + if (rte_eth_bond_8023ad_dedicated_queues_disable(port_id) == 0) + printf("Dedicated queues for LACP control packets " + "disabled\n"); + else + printf("Disabling dedicated queues for LACP control " + "traffic on port %d failed\n", port_id); + } +} + +cmdline_parse_token_string_t cmd_setbonding_lacp_dedicated_queues_set = +TOKEN_STRING_INITIALIZER(struct cmd_set_bonding_lacp_dedicated_queues_result, + set, "set"); +cmdline_parse_token_string_t cmd_setbonding_lacp_dedicated_queues_bonding = +TOKEN_STRING_INITIALIZER(struct cmd_set_bonding_lacp_dedicated_queues_result, + bonding, "bonding"); +cmdline_parse_token_string_t cmd_setbonding_lacp_dedicated_queues_lacp = +TOKEN_STRING_INITIALIZER(struct cmd_set_bonding_lacp_dedicated_queues_result, + lacp, "lacp"); +cmdline_parse_token_string_t cmd_setbonding_lacp_dedicated_queues_dedicated_queues = +TOKEN_STRING_INITIALIZER(struct cmd_set_bonding_lacp_dedicated_queues_result, + dedicated_queues, "dedicated_queues"); +cmdline_parse_token_num_t cmd_setbonding_lacp_dedicated_queues_port_id = +TOKEN_NUM_INITIALIZER(struct cmd_set_bonding_lacp_dedicated_queues_result, + port_id, UINT16); +cmdline_parse_token_string_t cmd_setbonding_lacp_dedicated_queues_mode = +TOKEN_STRING_INITIALIZER(struct cmd_set_bonding_lacp_dedicated_queues_result, + mode, "enable#disable"); + +cmdline_parse_inst_t cmd_set_lacp_dedicated_queues = { + .f = cmd_set_bonding_lacp_dedicated_queues_parsed, + .help_str = "set bonding lacp dedicated_queues " + "enable|disable: " + "Enable/disable dedicated queues for LACP control traffic for port_id", + .data = NULL, + .tokens = { + (void *)&cmd_setbonding_lacp_dedicated_queues_set, + (void *)&cmd_setbonding_lacp_dedicated_queues_bonding, + (void *)&cmd_setbonding_lacp_dedicated_queues_lacp, + (void *)&cmd_setbonding_lacp_dedicated_queues_dedicated_queues, + (void *)&cmd_setbonding_lacp_dedicated_queues_port_id, + (void *)&cmd_setbonding_lacp_dedicated_queues_mode, + NULL + } +}; + /* *** SET BALANCE XMIT POLICY *** */ struct cmd_set_bonding_balance_xmit_policy_result { cmdline_fixed_string_t set; cmdline_fixed_string_t bonding; cmdline_fixed_string_t balance_xmit_policy; - uint8_t port_id; + portid_t port_id; cmdline_fixed_string_t policy; }; @@ -4066,14 +4923,16 @@ TOKEN_STRING_INITIALIZER(struct cmd_set_bonding_balance_xmit_policy_result, balance_xmit_policy, "balance_xmit_policy"); cmdline_parse_token_num_t cmd_setbonding_balance_xmit_policy_port = TOKEN_NUM_INITIALIZER(struct cmd_set_bonding_balance_xmit_policy_result, - port_id, UINT8); + port_id, UINT16); cmdline_parse_token_string_t cmd_setbonding_balance_xmit_policy_policy = TOKEN_STRING_INITIALIZER(struct cmd_set_bonding_balance_xmit_policy_result, policy, "l2#l23#l34"); cmdline_parse_inst_t cmd_set_balance_xmit_policy = { .f = cmd_set_bonding_balance_xmit_policy_parsed, - .help_str = "set bonding balance_xmit_policy (port_id) (policy_value): Set the bonding balance_xmit_policy for port_id", + .help_str = "set bonding balance_xmit_policy " + "l2|l23|l34: " + "Set the bonding balance_xmit_policy for port_id", .data = NULL, .tokens = { (void *)&cmd_setbonding_balance_xmit_policy_set, @@ -4090,7 +4949,7 @@ struct cmd_show_bonding_config_result { cmdline_fixed_string_t show; cmdline_fixed_string_t bonding; cmdline_fixed_string_t config; - uint8_t port_id; + portid_t port_id; }; static void cmd_show_bonding_config_parsed(void *parsed_result, @@ -4098,8 +4957,8 @@ static void cmd_show_bonding_config_parsed(void *parsed_result, __attribute__((unused)) void *data) { struct cmd_show_bonding_config_result *res = parsed_result; - int bonding_mode; - uint8_t slaves[RTE_MAX_ETHPORTS]; + int bonding_mode, agg_mode; + portid_t slaves[RTE_MAX_ETHPORTS]; int num_slaves, num_active_slaves; int primary_id; int i; @@ -4139,6 +4998,23 @@ static void cmd_show_bonding_config_parsed(void *parsed_result, } } + if (bonding_mode == BONDING_MODE_8023AD) { + agg_mode = rte_eth_bond_8023ad_agg_selection_get(port_id); + printf("\tIEEE802.3AD Aggregator Mode: "); + switch (agg_mode) { + case AGG_BANDWIDTH: + printf("bandwidth"); + break; + case AGG_STABLE: + printf("stable"); + break; + case AGG_COUNT: + printf("count"); + break; + } + printf("\n"); + } + num_slaves = rte_eth_bond_slaves_get(port_id, slaves, RTE_MAX_ETHPORTS); if (num_slaves < 0) { @@ -4195,11 +5071,12 @@ TOKEN_STRING_INITIALIZER(struct cmd_show_bonding_config_result, config, "config"); cmdline_parse_token_num_t cmd_showbonding_config_port = TOKEN_NUM_INITIALIZER(struct cmd_show_bonding_config_result, - port_id, UINT8); + port_id, UINT16); cmdline_parse_inst_t cmd_show_bonding_config = { .f = cmd_show_bonding_config_parsed, - .help_str = "show bonding config (port_id): Show the bonding config for port_id", + .help_str = "show bonding config : " + "Show the bonding config for port_id", .data = NULL, .tokens = { (void *)&cmd_showbonding_config_show, @@ -4215,8 +5092,8 @@ struct cmd_set_bonding_primary_result { cmdline_fixed_string_t set; cmdline_fixed_string_t bonding; cmdline_fixed_string_t primary; - uint8_t slave_id; - uint8_t port_id; + portid_t slave_id; + portid_t port_id; }; static void cmd_set_bonding_primary_parsed(void *parsed_result, @@ -4247,14 +5124,15 @@ TOKEN_STRING_INITIALIZER(struct cmd_set_bonding_primary_result, primary, "primary"); cmdline_parse_token_num_t cmd_setbonding_primary_slave = TOKEN_NUM_INITIALIZER(struct cmd_set_bonding_primary_result, - slave_id, UINT8); + slave_id, UINT16); cmdline_parse_token_num_t cmd_setbonding_primary_port = TOKEN_NUM_INITIALIZER(struct cmd_set_bonding_primary_result, - port_id, UINT8); + port_id, UINT16); cmdline_parse_inst_t cmd_set_bonding_primary = { .f = cmd_set_bonding_primary_parsed, - .help_str = "set bonding primary (slave_id) (port_id): Set the primary slave for port_id", + .help_str = "set bonding primary : " + "Set the primary slave for port_id", .data = NULL, .tokens = { (void *)&cmd_setbonding_primary_set, @@ -4271,8 +5149,8 @@ struct cmd_add_bonding_slave_result { cmdline_fixed_string_t add; cmdline_fixed_string_t bonding; cmdline_fixed_string_t slave; - uint8_t slave_id; - uint8_t port_id; + portid_t slave_id; + portid_t port_id; }; static void cmd_add_bonding_slave_parsed(void *parsed_result, @@ -4283,7 +5161,7 @@ static void cmd_add_bonding_slave_parsed(void *parsed_result, portid_t master_port_id = res->port_id; portid_t slave_port_id = res->slave_id; - /* Set the primary slave for a bonded device. */ + /* add the slave for a bonded device. */ if (0 != rte_eth_bond_slave_add(master_port_id, slave_port_id)) { printf("\t Failed to add slave %d to master port = %d.\n", slave_port_id, master_port_id); @@ -4304,14 +5182,15 @@ TOKEN_STRING_INITIALIZER(struct cmd_add_bonding_slave_result, slave, "slave"); cmdline_parse_token_num_t cmd_addbonding_slave_slaveid = TOKEN_NUM_INITIALIZER(struct cmd_add_bonding_slave_result, - slave_id, UINT8); + slave_id, UINT16); cmdline_parse_token_num_t cmd_addbonding_slave_port = TOKEN_NUM_INITIALIZER(struct cmd_add_bonding_slave_result, - port_id, UINT8); + port_id, UINT16); cmdline_parse_inst_t cmd_add_bonding_slave = { .f = cmd_add_bonding_slave_parsed, - .help_str = "add bonding slave (slave_id) (port_id): Add a slave device to a bonded device", + .help_str = "add bonding slave : " + "Add a slave device to a bonded device", .data = NULL, .tokens = { (void *)&cmd_addbonding_slave_add, @@ -4328,8 +5207,8 @@ struct cmd_remove_bonding_slave_result { cmdline_fixed_string_t remove; cmdline_fixed_string_t bonding; cmdline_fixed_string_t slave; - uint8_t slave_id; - uint8_t port_id; + portid_t slave_id; + portid_t port_id; }; static void cmd_remove_bonding_slave_parsed(void *parsed_result, @@ -4340,7 +5219,7 @@ static void cmd_remove_bonding_slave_parsed(void *parsed_result, portid_t master_port_id = res->port_id; portid_t slave_port_id = res->slave_id; - /* Set the primary slave for a bonded device. */ + /* remove the slave from a bonded device. */ if (0 != rte_eth_bond_slave_remove(master_port_id, slave_port_id)) { printf("\t Failed to remove slave %d from master port = %d.\n", slave_port_id, master_port_id); @@ -4361,14 +5240,15 @@ cmdline_parse_token_string_t cmd_removebonding_slave_slave = slave, "slave"); cmdline_parse_token_num_t cmd_removebonding_slave_slaveid = TOKEN_NUM_INITIALIZER(struct cmd_remove_bonding_slave_result, - slave_id, UINT8); + slave_id, UINT16); cmdline_parse_token_num_t cmd_removebonding_slave_port = TOKEN_NUM_INITIALIZER(struct cmd_remove_bonding_slave_result, - port_id, UINT8); + port_id, UINT16); cmdline_parse_inst_t cmd_remove_bonding_slave = { .f = cmd_remove_bonding_slave_parsed, - .help_str = "remove bonding slave (slave_id) (port_id): Remove a slave device from a bonded device", + .help_str = "remove bonding slave : " + "Remove a slave device from a bonded device", .data = NULL, .tokens = { (void *)&cmd_removebonding_slave_remove, @@ -4404,7 +5284,7 @@ static void cmd_create_bonded_device_parsed(void *parsed_result, return; } - snprintf(ethdev_name, RTE_ETH_NAME_MAX_LEN, "eth_bond_testpmd_%d", + snprintf(ethdev_name, RTE_ETH_NAME_MAX_LEN, "net_bonding_testpmd_%d", bond_dev_num++); /* Create a new bonded device. */ @@ -4420,7 +5300,6 @@ static void cmd_create_bonded_device_parsed(void *parsed_result, nb_ports = rte_eth_dev_count(); reconfig(port_id, res->socket); rte_eth_promiscuous_enable(port_id); - ports[port_id].enabled = 1; } } @@ -4443,7 +5322,8 @@ cmdline_parse_token_num_t cmd_createbonded_device_socket = cmdline_parse_inst_t cmd_create_bonded_device = { .f = cmd_create_bonded_device_parsed, - .help_str = "create bonded device (mode) (socket): Create a new bonded device with specific bonding mode and socket", + .help_str = "create bonded device : " + "Create a new bonded device with specific bonding mode and socket", .data = NULL, .tokens = { (void *)&cmd_createbonded_device_create, @@ -4460,7 +5340,7 @@ struct cmd_set_bond_mac_addr_result { cmdline_fixed_string_t set; cmdline_fixed_string_t bonding; cmdline_fixed_string_t mac_addr; - uint8_t port_num; + uint16_t port_num; struct ether_addr address; }; @@ -4490,14 +5370,15 @@ cmdline_parse_token_string_t cmd_set_bond_mac_addr_mac = TOKEN_STRING_INITIALIZER(struct cmd_set_bond_mac_addr_result, mac_addr, "mac_addr"); cmdline_parse_token_num_t cmd_set_bond_mac_addr_portnum = - TOKEN_NUM_INITIALIZER(struct cmd_set_bond_mac_addr_result, port_num, UINT8); + TOKEN_NUM_INITIALIZER(struct cmd_set_bond_mac_addr_result, + port_num, UINT16); cmdline_parse_token_etheraddr_t cmd_set_bond_mac_addr_addr = TOKEN_ETHERADDR_INITIALIZER(struct cmd_set_bond_mac_addr_result, address); cmdline_parse_inst_t cmd_set_bond_mac_addr = { .f = cmd_set_bond_mac_addr_parsed, .data = (void *) 0, - .help_str = "set bonding mac_addr (port_id) (address): ", + .help_str = "set bonding mac_addr ", .tokens = { (void *)&cmd_set_bond_mac_addr_set, (void *)&cmd_set_bond_mac_addr_bonding, @@ -4514,7 +5395,7 @@ struct cmd_set_bond_mon_period_result { cmdline_fixed_string_t set; cmdline_fixed_string_t bonding; cmdline_fixed_string_t mon_period; - uint8_t port_num; + uint16_t port_num; uint32_t period_ms; }; @@ -4548,7 +5429,7 @@ cmdline_parse_token_string_t cmd_set_bond_mon_period_mon_period = mon_period, "mon_period"); cmdline_parse_token_num_t cmd_set_bond_mon_period_portnum = TOKEN_NUM_INITIALIZER(struct cmd_set_bond_mon_period_result, - port_num, UINT8); + port_num, UINT16); cmdline_parse_token_num_t cmd_set_bond_mon_period_period_ms = TOKEN_NUM_INITIALIZER(struct cmd_set_bond_mon_period_result, period_ms, UINT32); @@ -4556,7 +5437,7 @@ cmdline_parse_token_num_t cmd_set_bond_mon_period_period_ms = cmdline_parse_inst_t cmd_set_bond_mon_period = { .f = cmd_set_bond_mon_period_parsed, .data = (void *) 0, - .help_str = "set bonding mon_period (port_id) (period_ms): ", + .help_str = "set bonding mon_period ", .tokens = { (void *)&cmd_set_bond_mon_period_set, (void *)&cmd_set_bond_mon_period_bonding, @@ -4567,6 +5448,77 @@ cmdline_parse_inst_t cmd_set_bond_mon_period = { } }; + + +struct cmd_set_bonding_agg_mode_policy_result { + cmdline_fixed_string_t set; + cmdline_fixed_string_t bonding; + cmdline_fixed_string_t agg_mode; + uint16_t port_num; + cmdline_fixed_string_t policy; +}; + + +static void +cmd_set_bonding_agg_mode(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_set_bonding_agg_mode_policy_result *res = parsed_result; + uint8_t policy = AGG_BANDWIDTH; + + if (res->port_num >= nb_ports) { + printf("Port id %d must be less than %d\n", + res->port_num, nb_ports); + return; + } + + if (!strcmp(res->policy, "bandwidth")) + policy = AGG_BANDWIDTH; + else if (!strcmp(res->policy, "stable")) + policy = AGG_STABLE; + else if (!strcmp(res->policy, "count")) + policy = AGG_COUNT; + + rte_eth_bond_8023ad_agg_selection_set(res->port_num, policy); +} + + +cmdline_parse_token_string_t cmd_set_bonding_agg_mode_set = + TOKEN_STRING_INITIALIZER(struct cmd_set_bonding_agg_mode_policy_result, + set, "set"); +cmdline_parse_token_string_t cmd_set_bonding_agg_mode_bonding = + TOKEN_STRING_INITIALIZER(struct cmd_set_bonding_agg_mode_policy_result, + bonding, "bonding"); + +cmdline_parse_token_string_t cmd_set_bonding_agg_mode_agg_mode = + TOKEN_STRING_INITIALIZER(struct cmd_set_bonding_agg_mode_policy_result, + agg_mode, "agg_mode"); + +cmdline_parse_token_num_t cmd_set_bonding_agg_mode_portnum = + TOKEN_NUM_INITIALIZER(struct cmd_set_bonding_agg_mode_policy_result, + port_num, UINT16); + +cmdline_parse_token_string_t cmd_set_bonding_agg_mode_policy_string = + TOKEN_STRING_INITIALIZER( + struct cmd_set_bonding_balance_xmit_policy_result, + policy, "stable#bandwidth#count"); + +cmdline_parse_inst_t cmd_set_bonding_agg_mode_policy = { + .f = cmd_set_bonding_agg_mode, + .data = (void *) 0, + .help_str = "set bonding mode IEEE802.3AD aggregator policy ", + .tokens = { + (void *)&cmd_set_bonding_agg_mode_set, + (void *)&cmd_set_bonding_agg_mode_bonding, + (void *)&cmd_set_bonding_agg_mode_agg_mode, + (void *)&cmd_set_bonding_agg_mode_portnum, + (void *)&cmd_set_bonding_agg_mode_policy_string, + NULL + } +}; + + #endif /* RTE_LIBRTE_PMD_BOND */ /* *** SET FORWARDING MODE *** */ @@ -4614,8 +5566,8 @@ static void cmd_set_fwd_mode_init(void) cmdline_parse_token_string_t *token_struct; modes = list_pkt_forwarding_modes(); - snprintf(help, sizeof help, "set fwd %s - " - "set packet forwarding mode", modes); + snprintf(help, sizeof(help), "set fwd %s: " + "Set packet forwarding mode", modes); cmd_set_fwd_mode.help_str = help; /* string token separator is # */ @@ -4681,8 +5633,8 @@ static void cmd_set_fwd_retry_mode_init(void) cmdline_parse_token_string_t *token_struct; modes = list_pkt_forwarding_retry_modes(); - snprintf(help, sizeof(help), "set fwd %s retry - " - "set packet forwarding mode with retry", modes); + snprintf(help, sizeof(help), "set fwd %s retry: " + "Set packet forwarding mode with retry", modes); cmd_set_fwd_retry_mode.help_str = help; /* string token separator is # */ @@ -4741,7 +5693,7 @@ cmdline_parse_token_num_t cmd_set_burst_tx_retry_retry_num = cmdline_parse_inst_t cmd_set_burst_tx_retry = { .f = cmd_set_burst_tx_retry_parsed, - .help_str = "set burst tx delay (time_by_useconds) retry (retry_num)", + .help_str = "set burst tx delay retry ", .tokens = { (void *)&cmd_set_burst_tx_retry_set, (void *)&cmd_set_burst_tx_retry_burst, @@ -4759,7 +5711,7 @@ struct cmd_set_promisc_mode_result { cmdline_fixed_string_t set; cmdline_fixed_string_t promisc; cmdline_fixed_string_t port_all; /* valid if "allports" argument == 1 */ - uint8_t port_num; /* valid if "allports" argument == 0 */ + uint16_t port_num; /* valid if "allports" argument == 0 */ cmdline_fixed_string_t mode; }; @@ -4778,7 +5730,7 @@ static void cmd_set_promisc_mode_parsed(void *parsed_result, /* all ports */ if (allports) { - FOREACH_PORT(i, ports) { + RTE_ETH_FOREACH_DEV(i) { if (enable) rte_eth_promiscuous_enable(i); else @@ -4811,7 +5763,7 @@ cmdline_parse_token_string_t cmd_setpromisc_mode = cmdline_parse_inst_t cmd_set_promisc_mode_all = { .f = cmd_set_promisc_mode_parsed, .data = (void *)1, - .help_str = "set promisc all on|off: set promisc mode for all ports", + .help_str = "set promisc all on|off: Set promisc mode for all ports", .tokens = { (void *)&cmd_setpromisc_set, (void *)&cmd_setpromisc_promisc, @@ -4824,7 +5776,7 @@ cmdline_parse_inst_t cmd_set_promisc_mode_all = { cmdline_parse_inst_t cmd_set_promisc_mode_one = { .f = cmd_set_promisc_mode_parsed, .data = (void *)0, - .help_str = "set promisc X on|off: set promisc mode on port X", + .help_str = "set promisc on|off: Set promisc mode on port_id", .tokens = { (void *)&cmd_setpromisc_set, (void *)&cmd_setpromisc_promisc, @@ -4839,7 +5791,7 @@ struct cmd_set_allmulti_mode_result { cmdline_fixed_string_t set; cmdline_fixed_string_t allmulti; cmdline_fixed_string_t port_all; /* valid if "allports" argument == 1 */ - uint8_t port_num; /* valid if "allports" argument == 0 */ + uint16_t port_num; /* valid if "allports" argument == 0 */ cmdline_fixed_string_t mode; }; @@ -4858,7 +5810,7 @@ static void cmd_set_allmulti_mode_parsed(void *parsed_result, /* all ports */ if (allports) { - FOREACH_PORT(i, ports) { + RTE_ETH_FOREACH_DEV(i) { if (enable) rte_eth_allmulticast_enable(i); else @@ -4883,7 +5835,7 @@ cmdline_parse_token_string_t cmd_setallmulti_portall = "all"); cmdline_parse_token_num_t cmd_setallmulti_portnum = TOKEN_NUM_INITIALIZER(struct cmd_set_allmulti_mode_result, port_num, - UINT8); + UINT16); cmdline_parse_token_string_t cmd_setallmulti_mode = TOKEN_STRING_INITIALIZER(struct cmd_set_allmulti_mode_result, mode, "on#off"); @@ -4891,7 +5843,7 @@ cmdline_parse_token_string_t cmd_setallmulti_mode = cmdline_parse_inst_t cmd_set_allmulti_mode_all = { .f = cmd_set_allmulti_mode_parsed, .data = (void *)1, - .help_str = "set allmulti all on|off: set allmulti mode for all ports", + .help_str = "set allmulti all on|off: Set allmulti mode for all ports", .tokens = { (void *)&cmd_setallmulti_set, (void *)&cmd_setallmulti_allmulti, @@ -4904,7 +5856,8 @@ cmdline_parse_inst_t cmd_set_allmulti_mode_all = { cmdline_parse_inst_t cmd_set_allmulti_mode_one = { .f = cmd_set_allmulti_mode_parsed, .data = (void *)0, - .help_str = "set allmulti X on|off: set allmulti mode on port X", + .help_str = "set allmulti on|off: " + "Set allmulti mode on port_id", .tokens = { (void *)&cmd_setallmulti_set, (void *)&cmd_setallmulti_allmulti, @@ -4934,7 +5887,7 @@ struct cmd_link_flow_ctrl_set_result { uint16_t pause_time; cmdline_fixed_string_t xon_str; uint16_t send_xon; - uint8_t port_id; + portid_t port_id; }; cmdline_parse_token_string_t cmd_lfc_set_set = @@ -4993,7 +5946,7 @@ cmdline_parse_token_string_t cmd_lfc_set_autoneg = autoneg, "on#off"); cmdline_parse_token_num_t cmd_lfc_set_portid = TOKEN_NUM_INITIALIZER(struct cmd_link_flow_ctrl_set_result, - port_id, UINT8); + port_id, UINT16); /* forward declaration */ static void @@ -5003,9 +5956,9 @@ cmd_link_flow_ctrl_set_parsed(void *parsed_result, struct cmdline *cl, cmdline_parse_inst_t cmd_link_flow_control_set = { .f = cmd_link_flow_ctrl_set_parsed, .data = NULL, - .help_str = "Configure the Ethernet flow control: set flow_ctrl rx on|off \ -tx on|off high_water low_water pause_time send_xon mac_ctrl_frame_fwd on|off \ -autoneg on|off port_id", + .help_str = "set flow_ctrl rx on|off tx on|off " + " mac_ctrl_frame_fwd on|off " + "autoneg on|off : Configure the Ethernet flow control", .tokens = { (void *)&cmd_lfc_set_set, (void *)&cmd_lfc_set_flow_ctrl, @@ -5029,8 +5982,8 @@ autoneg on|off port_id", cmdline_parse_inst_t cmd_link_flow_control_set_rx = { .f = cmd_link_flow_ctrl_set_parsed, .data = (void *)&cmd_link_flow_control_set_rx, - .help_str = "Change rx flow control parameter: set flow_ctrl " - "rx on|off port_id", + .help_str = "set flow_ctrl rx on|off : " + "Change rx flow control parameter", .tokens = { (void *)&cmd_lfc_set_set, (void *)&cmd_lfc_set_flow_ctrl, @@ -5044,8 +5997,8 @@ cmdline_parse_inst_t cmd_link_flow_control_set_rx = { cmdline_parse_inst_t cmd_link_flow_control_set_tx = { .f = cmd_link_flow_ctrl_set_parsed, .data = (void *)&cmd_link_flow_control_set_tx, - .help_str = "Change tx flow control parameter: set flow_ctrl " - "tx on|off port_id", + .help_str = "set flow_ctrl tx on|off : " + "Change tx flow control parameter", .tokens = { (void *)&cmd_lfc_set_set, (void *)&cmd_lfc_set_flow_ctrl, @@ -5059,8 +6012,8 @@ cmdline_parse_inst_t cmd_link_flow_control_set_tx = { cmdline_parse_inst_t cmd_link_flow_control_set_hw = { .f = cmd_link_flow_ctrl_set_parsed, .data = (void *)&cmd_link_flow_control_set_hw, - .help_str = "Change high water flow control parameter: set flow_ctrl " - "high_water value port_id", + .help_str = "set flow_ctrl high_water : " + "Change high water flow control parameter", .tokens = { (void *)&cmd_lfc_set_set, (void *)&cmd_lfc_set_flow_ctrl, @@ -5074,8 +6027,8 @@ cmdline_parse_inst_t cmd_link_flow_control_set_hw = { cmdline_parse_inst_t cmd_link_flow_control_set_lw = { .f = cmd_link_flow_ctrl_set_parsed, .data = (void *)&cmd_link_flow_control_set_lw, - .help_str = "Change low water flow control parameter: set flow_ctrl " - "low_water value port_id", + .help_str = "set flow_ctrl low_water : " + "Change low water flow control parameter", .tokens = { (void *)&cmd_lfc_set_set, (void *)&cmd_lfc_set_flow_ctrl, @@ -5089,8 +6042,8 @@ cmdline_parse_inst_t cmd_link_flow_control_set_lw = { cmdline_parse_inst_t cmd_link_flow_control_set_pt = { .f = cmd_link_flow_ctrl_set_parsed, .data = (void *)&cmd_link_flow_control_set_pt, - .help_str = "Change pause time flow control parameter: set flow_ctrl " - "pause_time value port_id", + .help_str = "set flow_ctrl pause_time : " + "Change pause time flow control parameter", .tokens = { (void *)&cmd_lfc_set_set, (void *)&cmd_lfc_set_flow_ctrl, @@ -5104,8 +6057,8 @@ cmdline_parse_inst_t cmd_link_flow_control_set_pt = { cmdline_parse_inst_t cmd_link_flow_control_set_xon = { .f = cmd_link_flow_ctrl_set_parsed, .data = (void *)&cmd_link_flow_control_set_xon, - .help_str = "Change send_xon flow control parameter: set flow_ctrl " - "send_xon value port_id", + .help_str = "set flow_ctrl send_xon : " + "Change send_xon flow control parameter", .tokens = { (void *)&cmd_lfc_set_set, (void *)&cmd_lfc_set_flow_ctrl, @@ -5119,8 +6072,8 @@ cmdline_parse_inst_t cmd_link_flow_control_set_xon = { cmdline_parse_inst_t cmd_link_flow_control_set_macfwd = { .f = cmd_link_flow_ctrl_set_parsed, .data = (void *)&cmd_link_flow_control_set_macfwd, - .help_str = "Change mac ctrl fwd flow control parameter: set flow_ctrl " - "mac_ctrl_frame_fwd on|off port_id", + .help_str = "set flow_ctrl mac_ctrl_frame_fwd on|off : " + "Change mac ctrl fwd flow control parameter", .tokens = { (void *)&cmd_lfc_set_set, (void *)&cmd_lfc_set_flow_ctrl, @@ -5134,8 +6087,8 @@ cmdline_parse_inst_t cmd_link_flow_control_set_macfwd = { cmdline_parse_inst_t cmd_link_flow_control_set_autoneg = { .f = cmd_link_flow_ctrl_set_parsed, .data = (void *)&cmd_link_flow_control_set_autoneg, - .help_str = "Change autoneg flow control parameter: set flow_ctrl " - "autoneg on|off port_id", + .help_str = "set flow_ctrl autoneg on|off : " + "Change autoneg flow control parameter", .tokens = { (void *)&cmd_lfc_set_set, (void *)&cmd_lfc_set_flow_ctrl, @@ -5220,7 +6173,7 @@ cmd_link_flow_ctrl_set_parsed(void *parsed_result, printf("bad flow contrl parameter, return code = %d \n", ret); } -/* *** SETUP ETHERNET PIRORITY FLOW CONTROL *** */ +/* *** SETUP ETHERNET PRIORITY FLOW CONTROL *** */ struct cmd_priority_flow_ctrl_set_result { cmdline_fixed_string_t set; cmdline_fixed_string_t pfc_ctrl; @@ -5232,7 +6185,7 @@ struct cmd_priority_flow_ctrl_set_result { uint32_t low_water; uint16_t pause_time; uint8_t priority; - uint8_t port_id; + portid_t port_id; }; static void @@ -5300,13 +6253,14 @@ cmdline_parse_token_num_t cmd_pfc_set_priority = priority, UINT8); cmdline_parse_token_num_t cmd_pfc_set_portid = TOKEN_NUM_INITIALIZER(struct cmd_priority_flow_ctrl_set_result, - port_id, UINT8); + port_id, UINT16); cmdline_parse_inst_t cmd_priority_flow_control_set = { .f = cmd_priority_flow_ctrl_set_parsed, .data = NULL, - .help_str = "Configure the Ethernet priority flow control: set pfc_ctrl rx on|off\n\ - tx on|off high_water low_water pause_time priority port_id", + .help_str = "set pfc_ctrl rx on|off tx on|off " + " : " + "Configure the Ethernet priority flow control", .tokens = { (void *)&cmd_pfc_set_set, (void *)&cmd_pfc_set_flow_ctrl, @@ -5346,7 +6300,7 @@ cmdline_parse_token_string_t cmd_reset_def = cmdline_parse_inst_t cmd_reset = { .f = cmd_reset_parsed, .data = NULL, - .help_str = "set default: reset default forwarding configuration", + .help_str = "set default: Reset default forwarding configuration", .tokens = { (void *)&cmd_reset_set, (void *)&cmd_reset_def, @@ -5372,7 +6326,7 @@ static void cmd_start_parsed(__attribute__((unused)) void *parsed_result, cmdline_parse_inst_t cmd_start = { .f = cmd_start_parsed, .data = NULL, - .help_str = "start packet forwarding", + .help_str = "start: Start packet forwarding", .tokens = { (void *)&cmd_start_start, NULL, @@ -5403,7 +6357,8 @@ cmdline_parse_token_string_t cmd_start_tx_first_tx_first = cmdline_parse_inst_t cmd_start_tx_first = { .f = cmd_start_tx_first_parsed, .data = NULL, - .help_str = "start packet forwarding, after sending 1 burst of packets", + .help_str = "start tx_first: Start packet forwarding, " + "after sending 1 burst of packets", .tokens = { (void *)&cmd_start_tx_first_start, (void *)&cmd_start_tx_first_tx_first, @@ -5441,8 +6396,8 @@ cmdline_parse_token_num_t cmd_start_tx_first_n_tx_num = cmdline_parse_inst_t cmd_start_tx_first_n = { .f = cmd_start_tx_first_n_parsed, .data = NULL, - .help_str = "start packet forwarding, after sending " - "bursts of packets", + .help_str = "start tx_first : " + "packet forwarding, after sending bursts of packets", .tokens = { (void *)&cmd_start_tx_first_n_start, (void *)&cmd_start_tx_first_n_tx_first, @@ -5456,7 +6411,7 @@ struct cmd_set_link_up_result { cmdline_fixed_string_t set; cmdline_fixed_string_t link_up; cmdline_fixed_string_t port; - uint8_t port_id; + portid_t port_id; }; cmdline_parse_token_string_t cmd_set_link_up_set = @@ -5467,7 +6422,7 @@ cmdline_parse_token_string_t cmd_set_link_up_link_up = cmdline_parse_token_string_t cmd_set_link_up_port = TOKEN_STRING_INITIALIZER(struct cmd_set_link_up_result, port, "port"); cmdline_parse_token_num_t cmd_set_link_up_port_id = - TOKEN_NUM_INITIALIZER(struct cmd_set_link_up_result, port_id, UINT8); + TOKEN_NUM_INITIALIZER(struct cmd_set_link_up_result, port_id, UINT16); static void cmd_set_link_up_parsed(__attribute__((unused)) void *parsed_result, __attribute__((unused)) struct cmdline *cl, @@ -5480,7 +6435,7 @@ static void cmd_set_link_up_parsed(__attribute__((unused)) void *parsed_result, cmdline_parse_inst_t cmd_set_link_up = { .f = cmd_set_link_up_parsed, .data = NULL, - .help_str = "set link-up port (port id)", + .help_str = "set link-up port ", .tokens = { (void *)&cmd_set_link_up_set, (void *)&cmd_set_link_up_link_up, @@ -5495,7 +6450,7 @@ struct cmd_set_link_down_result { cmdline_fixed_string_t set; cmdline_fixed_string_t link_down; cmdline_fixed_string_t port; - uint8_t port_id; + portid_t port_id; }; cmdline_parse_token_string_t cmd_set_link_down_set = @@ -5506,7 +6461,7 @@ cmdline_parse_token_string_t cmd_set_link_down_link_down = cmdline_parse_token_string_t cmd_set_link_down_port = TOKEN_STRING_INITIALIZER(struct cmd_set_link_down_result, port, "port"); cmdline_parse_token_num_t cmd_set_link_down_port_id = - TOKEN_NUM_INITIALIZER(struct cmd_set_link_down_result, port_id, UINT8); + TOKEN_NUM_INITIALIZER(struct cmd_set_link_down_result, port_id, UINT16); static void cmd_set_link_down_parsed( __attribute__((unused)) void *parsed_result, @@ -5520,7 +6475,7 @@ static void cmd_set_link_down_parsed( cmdline_parse_inst_t cmd_set_link_down = { .f = cmd_set_link_down_parsed, .data = NULL, - .help_str = "set link-down port (port id)", + .help_str = "set link-down port ", .tokens = { (void *)&cmd_set_link_down_set, (void *)&cmd_set_link_down_link_down, @@ -5589,29 +6544,32 @@ static void cmd_showportall_parsed(void *parsed_result, struct cmd_showportall_result *res = parsed_result; if (!strcmp(res->show, "clear")) { if (!strcmp(res->what, "stats")) - FOREACH_PORT(i, ports) + RTE_ETH_FOREACH_DEV(i) nic_stats_clear(i); else if (!strcmp(res->what, "xstats")) - FOREACH_PORT(i, ports) + RTE_ETH_FOREACH_DEV(i) nic_xstats_clear(i); } else if (!strcmp(res->what, "info")) - FOREACH_PORT(i, ports) + RTE_ETH_FOREACH_DEV(i) port_infos_display(i); else if (!strcmp(res->what, "stats")) - FOREACH_PORT(i, ports) + RTE_ETH_FOREACH_DEV(i) nic_stats_display(i); else if (!strcmp(res->what, "xstats")) - FOREACH_PORT(i, ports) + RTE_ETH_FOREACH_DEV(i) nic_xstats_display(i); else if (!strcmp(res->what, "fdir")) - FOREACH_PORT(i, ports) + RTE_ETH_FOREACH_DEV(i) fdir_get_infos(i); else if (!strcmp(res->what, "stat_qmap")) - FOREACH_PORT(i, ports) + RTE_ETH_FOREACH_DEV(i) nic_stats_mapping_display(i); else if (!strcmp(res->what, "dcb_tc")) - FOREACH_PORT(i, ports) + RTE_ETH_FOREACH_DEV(i) port_dcb_info_display(i); + else if (!strcmp(res->what, "cap")) + RTE_ETH_FOREACH_DEV(i) + port_offload_cap_display(i); } cmdline_parse_token_string_t cmd_showportall_show = @@ -5621,13 +6579,14 @@ cmdline_parse_token_string_t cmd_showportall_port = TOKEN_STRING_INITIALIZER(struct cmd_showportall_result, port, "port"); cmdline_parse_token_string_t cmd_showportall_what = TOKEN_STRING_INITIALIZER(struct cmd_showportall_result, what, - "info#stats#xstats#fdir#stat_qmap#dcb_tc"); + "info#stats#xstats#fdir#stat_qmap#dcb_tc#cap"); cmdline_parse_token_string_t cmd_showportall_all = TOKEN_STRING_INITIALIZER(struct cmd_showportall_result, all, "all"); cmdline_parse_inst_t cmd_showportall = { .f = cmd_showportall_parsed, .data = NULL, - .help_str = "show|clear port info|stats|xstats|fdir|stat_qmap|dcb_tc all", + .help_str = "show|clear port " + "info|stats|xstats|fdir|stat_qmap|dcb_tc|cap all", .tokens = { (void *)&cmd_showportall_show, (void *)&cmd_showportall_port, @@ -5642,7 +6601,7 @@ struct cmd_showport_result { cmdline_fixed_string_t show; cmdline_fixed_string_t port; cmdline_fixed_string_t what; - uint8_t portnum; + uint16_t portnum; }; static void cmd_showport_parsed(void *parsed_result, @@ -5667,6 +6626,8 @@ static void cmd_showport_parsed(void *parsed_result, nic_stats_mapping_display(res->portnum); else if (!strcmp(res->what, "dcb_tc")) port_dcb_info_display(res->portnum); + else if (!strcmp(res->what, "cap")) + port_offload_cap_display(res->portnum); } cmdline_parse_token_string_t cmd_showport_show = @@ -5676,14 +6637,16 @@ cmdline_parse_token_string_t cmd_showport_port = TOKEN_STRING_INITIALIZER(struct cmd_showport_result, port, "port"); cmdline_parse_token_string_t cmd_showport_what = TOKEN_STRING_INITIALIZER(struct cmd_showport_result, what, - "info#stats#xstats#fdir#stat_qmap#dcb_tc"); + "info#stats#xstats#fdir#stat_qmap#dcb_tc#cap"); cmdline_parse_token_num_t cmd_showport_portnum = - TOKEN_NUM_INITIALIZER(struct cmd_showport_result, portnum, UINT8); + TOKEN_NUM_INITIALIZER(struct cmd_showport_result, portnum, UINT16); cmdline_parse_inst_t cmd_showport = { .f = cmd_showport_parsed, .data = NULL, - .help_str = "show|clear port info|stats|xstats|fdir|stat_qmap|dcb_tc X (X = port number)", + .help_str = "show|clear port " + "info|stats|xstats|fdir|stat_qmap|dcb_tc|cap " + "", .tokens = { (void *)&cmd_showport_show, (void *)&cmd_showport_port, @@ -5698,7 +6661,7 @@ struct cmd_showqueue_result { cmdline_fixed_string_t show; cmdline_fixed_string_t type; cmdline_fixed_string_t what; - uint8_t portnum; + uint16_t portnum; uint16_t queuenum; }; @@ -5722,14 +6685,14 @@ cmdline_parse_token_string_t cmd_showqueue_type = cmdline_parse_token_string_t cmd_showqueue_what = TOKEN_STRING_INITIALIZER(struct cmd_showqueue_result, what, "info"); cmdline_parse_token_num_t cmd_showqueue_portnum = - TOKEN_NUM_INITIALIZER(struct cmd_showqueue_result, portnum, UINT8); + TOKEN_NUM_INITIALIZER(struct cmd_showqueue_result, portnum, UINT16); cmdline_parse_token_num_t cmd_showqueue_queuenum = TOKEN_NUM_INITIALIZER(struct cmd_showqueue_result, queuenum, UINT16); cmdline_parse_inst_t cmd_showqueue = { .f = cmd_showqueue_parsed, .data = NULL, - .help_str = "show rxq|txq info ", + .help_str = "show rxq|txq info ", .tokens = { (void *)&cmd_showqueue_show, (void *)&cmd_showqueue_type, @@ -5744,7 +6707,7 @@ cmdline_parse_inst_t cmd_showqueue = { struct cmd_read_reg_result { cmdline_fixed_string_t read; cmdline_fixed_string_t reg; - uint8_t port_id; + portid_t port_id; uint32_t reg_off; }; @@ -5762,14 +6725,14 @@ cmdline_parse_token_string_t cmd_read_reg_read = cmdline_parse_token_string_t cmd_read_reg_reg = TOKEN_STRING_INITIALIZER(struct cmd_read_reg_result, reg, "reg"); cmdline_parse_token_num_t cmd_read_reg_port_id = - TOKEN_NUM_INITIALIZER(struct cmd_read_reg_result, port_id, UINT8); + TOKEN_NUM_INITIALIZER(struct cmd_read_reg_result, port_id, UINT16); cmdline_parse_token_num_t cmd_read_reg_reg_off = TOKEN_NUM_INITIALIZER(struct cmd_read_reg_result, reg_off, UINT32); cmdline_parse_inst_t cmd_read_reg = { .f = cmd_read_reg_parsed, .data = NULL, - .help_str = "read reg port_id reg_off", + .help_str = "read reg ", .tokens = { (void *)&cmd_read_reg_read, (void *)&cmd_read_reg_reg, @@ -5783,7 +6746,7 @@ cmdline_parse_inst_t cmd_read_reg = { struct cmd_read_reg_bit_field_result { cmdline_fixed_string_t read; cmdline_fixed_string_t regfield; - uint8_t port_id; + portid_t port_id; uint32_t reg_off; uint8_t bit1_pos; uint8_t bit2_pos; @@ -5807,7 +6770,7 @@ cmdline_parse_token_string_t cmd_read_reg_bit_field_regfield = regfield, "regfield"); cmdline_parse_token_num_t cmd_read_reg_bit_field_port_id = TOKEN_NUM_INITIALIZER(struct cmd_read_reg_bit_field_result, port_id, - UINT8); + UINT16); cmdline_parse_token_num_t cmd_read_reg_bit_field_reg_off = TOKEN_NUM_INITIALIZER(struct cmd_read_reg_bit_field_result, reg_off, UINT32); @@ -5821,8 +6784,8 @@ cmdline_parse_token_num_t cmd_read_reg_bit_field_bit2_pos = cmdline_parse_inst_t cmd_read_reg_bit_field = { .f = cmd_read_reg_bit_field_parsed, .data = NULL, - .help_str = "read regfield port_id reg_off bit_x bit_y " - "(read register bit field between bit_x and bit_y included)", + .help_str = "read regfield : " + "Read register bit field between bit_x and bit_y included", .tokens = { (void *)&cmd_read_reg_bit_field_read, (void *)&cmd_read_reg_bit_field_regfield, @@ -5838,7 +6801,7 @@ cmdline_parse_inst_t cmd_read_reg_bit_field = { struct cmd_read_reg_bit_result { cmdline_fixed_string_t read; cmdline_fixed_string_t regbit; - uint8_t port_id; + portid_t port_id; uint32_t reg_off; uint8_t bit_pos; }; @@ -5858,7 +6821,7 @@ cmdline_parse_token_string_t cmd_read_reg_bit_regbit = TOKEN_STRING_INITIALIZER(struct cmd_read_reg_bit_result, regbit, "regbit"); cmdline_parse_token_num_t cmd_read_reg_bit_port_id = - TOKEN_NUM_INITIALIZER(struct cmd_read_reg_bit_result, port_id, UINT8); + TOKEN_NUM_INITIALIZER(struct cmd_read_reg_bit_result, port_id, UINT16); cmdline_parse_token_num_t cmd_read_reg_bit_reg_off = TOKEN_NUM_INITIALIZER(struct cmd_read_reg_bit_result, reg_off, UINT32); cmdline_parse_token_num_t cmd_read_reg_bit_bit_pos = @@ -5867,7 +6830,7 @@ cmdline_parse_token_num_t cmd_read_reg_bit_bit_pos = cmdline_parse_inst_t cmd_read_reg_bit = { .f = cmd_read_reg_bit_parsed, .data = NULL, - .help_str = "read regbit port_id reg_off bit_x (0 <= bit_x <= 31)", + .help_str = "read regbit : 0 <= bit_x <= 31", .tokens = { (void *)&cmd_read_reg_bit_read, (void *)&cmd_read_reg_bit_regbit, @@ -5882,7 +6845,7 @@ cmdline_parse_inst_t cmd_read_reg_bit = { struct cmd_write_reg_result { cmdline_fixed_string_t write; cmdline_fixed_string_t reg; - uint8_t port_id; + portid_t port_id; uint32_t reg_off; uint32_t value; }; @@ -5901,7 +6864,7 @@ cmdline_parse_token_string_t cmd_write_reg_write = cmdline_parse_token_string_t cmd_write_reg_reg = TOKEN_STRING_INITIALIZER(struct cmd_write_reg_result, reg, "reg"); cmdline_parse_token_num_t cmd_write_reg_port_id = - TOKEN_NUM_INITIALIZER(struct cmd_write_reg_result, port_id, UINT8); + TOKEN_NUM_INITIALIZER(struct cmd_write_reg_result, port_id, UINT16); cmdline_parse_token_num_t cmd_write_reg_reg_off = TOKEN_NUM_INITIALIZER(struct cmd_write_reg_result, reg_off, UINT32); cmdline_parse_token_num_t cmd_write_reg_value = @@ -5910,7 +6873,7 @@ cmdline_parse_token_num_t cmd_write_reg_value = cmdline_parse_inst_t cmd_write_reg = { .f = cmd_write_reg_parsed, .data = NULL, - .help_str = "write reg port_id reg_off reg_value", + .help_str = "write reg ", .tokens = { (void *)&cmd_write_reg_write, (void *)&cmd_write_reg_reg, @@ -5925,7 +6888,7 @@ cmdline_parse_inst_t cmd_write_reg = { struct cmd_write_reg_bit_field_result { cmdline_fixed_string_t write; cmdline_fixed_string_t regfield; - uint8_t port_id; + portid_t port_id; uint32_t reg_off; uint8_t bit1_pos; uint8_t bit2_pos; @@ -5950,7 +6913,7 @@ cmdline_parse_token_string_t cmd_write_reg_bit_field_regfield = regfield, "regfield"); cmdline_parse_token_num_t cmd_write_reg_bit_field_port_id = TOKEN_NUM_INITIALIZER(struct cmd_write_reg_bit_field_result, port_id, - UINT8); + UINT16); cmdline_parse_token_num_t cmd_write_reg_bit_field_reg_off = TOKEN_NUM_INITIALIZER(struct cmd_write_reg_bit_field_result, reg_off, UINT32); @@ -5967,8 +6930,9 @@ cmdline_parse_token_num_t cmd_write_reg_bit_field_value = cmdline_parse_inst_t cmd_write_reg_bit_field = { .f = cmd_write_reg_bit_field_parsed, .data = NULL, - .help_str = "write regfield port_id reg_off bit_x bit_y reg_value" - "(set register bit field between bit_x and bit_y included)", + .help_str = "write regfield " + ": " + "Set register bit field between bit_x and bit_y included", .tokens = { (void *)&cmd_write_reg_bit_field_write, (void *)&cmd_write_reg_bit_field_regfield, @@ -5985,7 +6949,7 @@ cmdline_parse_inst_t cmd_write_reg_bit_field = { struct cmd_write_reg_bit_result { cmdline_fixed_string_t write; cmdline_fixed_string_t regbit; - uint8_t port_id; + portid_t port_id; uint32_t reg_off; uint8_t bit_pos; uint8_t value; @@ -6007,7 +6971,7 @@ cmdline_parse_token_string_t cmd_write_reg_bit_regbit = TOKEN_STRING_INITIALIZER(struct cmd_write_reg_bit_result, regbit, "regbit"); cmdline_parse_token_num_t cmd_write_reg_bit_port_id = - TOKEN_NUM_INITIALIZER(struct cmd_write_reg_bit_result, port_id, UINT8); + TOKEN_NUM_INITIALIZER(struct cmd_write_reg_bit_result, port_id, UINT16); cmdline_parse_token_num_t cmd_write_reg_bit_reg_off = TOKEN_NUM_INITIALIZER(struct cmd_write_reg_bit_result, reg_off, UINT32); cmdline_parse_token_num_t cmd_write_reg_bit_bit_pos = @@ -6018,7 +6982,8 @@ cmdline_parse_token_num_t cmd_write_reg_bit_value = cmdline_parse_inst_t cmd_write_reg_bit = { .f = cmd_write_reg_bit_parsed, .data = NULL, - .help_str = "write regbit port_id reg_off bit_x 0/1 (0 <= bit_x <= 31)", + .help_str = "write regbit 0|1: " + "0 <= bit_x <= 31", .tokens = { (void *)&cmd_write_reg_bit_write, (void *)&cmd_write_reg_bit_regbit, @@ -6034,7 +6999,7 @@ cmdline_parse_inst_t cmd_write_reg_bit = { struct cmd_read_rxd_txd_result { cmdline_fixed_string_t read; cmdline_fixed_string_t rxd_txd; - uint8_t port_id; + portid_t port_id; uint16_t queue_id; uint16_t desc_id; }; @@ -6058,7 +7023,7 @@ cmdline_parse_token_string_t cmd_read_rxd_txd_rxd_txd = TOKEN_STRING_INITIALIZER(struct cmd_read_rxd_txd_result, rxd_txd, "rxd#txd"); cmdline_parse_token_num_t cmd_read_rxd_txd_port_id = - TOKEN_NUM_INITIALIZER(struct cmd_read_rxd_txd_result, port_id, UINT8); + TOKEN_NUM_INITIALIZER(struct cmd_read_rxd_txd_result, port_id, UINT16); cmdline_parse_token_num_t cmd_read_rxd_txd_queue_id = TOKEN_NUM_INITIALIZER(struct cmd_read_rxd_txd_result, queue_id, UINT16); cmdline_parse_token_num_t cmd_read_rxd_txd_desc_id = @@ -6067,7 +7032,7 @@ cmdline_parse_token_num_t cmd_read_rxd_txd_desc_id = cmdline_parse_inst_t cmd_read_rxd_txd = { .f = cmd_read_rxd_txd_parsed, .data = NULL, - .help_str = "read rxd|txd port_id queue_id rxd_id", + .help_str = "read rxd|txd ", .tokens = { (void *)&cmd_read_rxd_txd_read, (void *)&cmd_read_rxd_txd_rxd_txd, @@ -6097,7 +7062,7 @@ cmdline_parse_token_string_t cmd_quit_quit = cmdline_parse_inst_t cmd_quit = { .f = cmd_quit_parsed, .data = NULL, - .help_str = "exit application", + .help_str = "quit: Exit application", .tokens = { (void *)&cmd_quit_quit, NULL, @@ -6108,7 +7073,7 @@ cmdline_parse_inst_t cmd_quit = { struct cmd_mac_addr_result { cmdline_fixed_string_t mac_addr_cmd; cmdline_fixed_string_t what; - uint8_t port_num; + uint16_t port_num; struct ether_addr address; }; @@ -6121,6 +7086,9 @@ static void cmd_mac_addr_parsed(void *parsed_result, if (strcmp(res->what, "add") == 0) ret = rte_eth_dev_mac_addr_add(res->port_num, &res->address, 0); + else if (strcmp(res->what, "set") == 0) + ret = rte_eth_dev_default_mac_addr_set(res->port_num, + &res->address); else ret = rte_eth_dev_mac_addr_remove(res->port_num, &res->address); @@ -6135,17 +7103,18 @@ cmdline_parse_token_string_t cmd_mac_addr_cmd = "mac_addr"); cmdline_parse_token_string_t cmd_mac_addr_what = TOKEN_STRING_INITIALIZER(struct cmd_mac_addr_result, what, - "add#remove"); + "add#remove#set"); cmdline_parse_token_num_t cmd_mac_addr_portnum = - TOKEN_NUM_INITIALIZER(struct cmd_mac_addr_result, port_num, UINT8); + TOKEN_NUM_INITIALIZER(struct cmd_mac_addr_result, port_num, + UINT16); cmdline_parse_token_etheraddr_t cmd_mac_addr_addr = TOKEN_ETHERADDR_INITIALIZER(struct cmd_mac_addr_result, address); cmdline_parse_inst_t cmd_mac_addr = { .f = cmd_mac_addr_parsed, .data = (void *)0, - .help_str = "mac_addr add|remove X
: " - "add/remove MAC address on port X", + .help_str = "mac_addr add|remove|set : " + "Add/Remove/Set MAC address on port_id", .tokens = { (void *)&cmd_mac_addr_cmd, (void *)&cmd_mac_addr_what, @@ -6161,7 +7130,7 @@ struct cmd_set_qmap_result { cmdline_fixed_string_t set; cmdline_fixed_string_t qmap; cmdline_fixed_string_t what; - uint8_t port_id; + portid_t port_id; uint16_t queue_id; uint8_t map_value; }; @@ -6188,7 +7157,7 @@ cmdline_parse_token_string_t cmd_setqmap_what = what, "tx#rx"); cmdline_parse_token_num_t cmd_setqmap_portid = TOKEN_NUM_INITIALIZER(struct cmd_set_qmap_result, - port_id, UINT8); + port_id, UINT16); cmdline_parse_token_num_t cmd_setqmap_queueid = TOKEN_NUM_INITIALIZER(struct cmd_set_qmap_result, queue_id, UINT16); @@ -6199,7 +7168,8 @@ cmdline_parse_token_num_t cmd_setqmap_mapvalue = cmdline_parse_inst_t cmd_set_qmap = { .f = cmd_set_qmap_parsed, .data = NULL, - .help_str = "Set statistics mapping value on tx|rx queue_id of port_id", + .help_str = "set stat_qmap rx|tx : " + "Set statistics mapping value on tx|rx queue_id of port_id", .tokens = { (void *)&cmd_setqmap_set, (void *)&cmd_setqmap_qmap, @@ -6211,11 +7181,53 @@ cmdline_parse_inst_t cmd_set_qmap = { }, }; +/* *** SET OPTION TO HIDE ZERO VALUES FOR XSTATS DISPLAY *** */ +struct cmd_set_xstats_hide_zero_result { + cmdline_fixed_string_t keyword; + cmdline_fixed_string_t name; + cmdline_fixed_string_t on_off; +}; + +static void +cmd_set_xstats_hide_zero_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_set_xstats_hide_zero_result *res; + uint16_t on_off = 0; + + res = parsed_result; + on_off = !strcmp(res->on_off, "on") ? 1 : 0; + set_xstats_hide_zero(on_off); +} + +cmdline_parse_token_string_t cmd_set_xstats_hide_zero_keyword = + TOKEN_STRING_INITIALIZER(struct cmd_set_xstats_hide_zero_result, + keyword, "set"); +cmdline_parse_token_string_t cmd_set_xstats_hide_zero_name = + TOKEN_STRING_INITIALIZER(struct cmd_set_xstats_hide_zero_result, + name, "xstats-hide-zero"); +cmdline_parse_token_string_t cmd_set_xstats_hide_zero_on_off = + TOKEN_STRING_INITIALIZER(struct cmd_set_xstats_hide_zero_result, + on_off, "on#off"); + +cmdline_parse_inst_t cmd_set_xstats_hide_zero = { + .f = cmd_set_xstats_hide_zero_parsed, + .data = NULL, + .help_str = "set xstats-hide-zero on|off", + .tokens = { + (void *)&cmd_set_xstats_hide_zero_keyword, + (void *)&cmd_set_xstats_hide_zero_name, + (void *)&cmd_set_xstats_hide_zero_on_off, + NULL, + }, +}; + /* *** CONFIGURE UNICAST HASH TABLE *** */ struct cmd_set_uc_hash_table { cmdline_fixed_string_t set; cmdline_fixed_string_t port; - uint8_t port_id; + portid_t port_id; cmdline_fixed_string_t what; struct ether_addr address; cmdline_fixed_string_t mode; @@ -6247,7 +7259,7 @@ cmdline_parse_token_string_t cmd_set_uc_hash_port = port, "port"); cmdline_parse_token_num_t cmd_set_uc_hash_portid = TOKEN_NUM_INITIALIZER(struct cmd_set_uc_hash_table, - port_id, UINT8); + port_id, UINT16); cmdline_parse_token_string_t cmd_set_uc_hash_what = TOKEN_STRING_INITIALIZER(struct cmd_set_uc_hash_table, what, "uta"); @@ -6261,7 +7273,7 @@ cmdline_parse_token_string_t cmd_set_uc_hash_mode = cmdline_parse_inst_t cmd_set_uc_hash_filter = { .f = cmd_set_uc_hash_parsed, .data = NULL, - .help_str = "set port X uta Y on|off(X = port number,Y = MAC address)", + .help_str = "set port uta on|off)", .tokens = { (void *)&cmd_set_uc_hash_set, (void *)&cmd_set_uc_hash_port, @@ -6276,7 +7288,7 @@ cmdline_parse_inst_t cmd_set_uc_hash_filter = { struct cmd_set_uc_all_hash_table { cmdline_fixed_string_t set; cmdline_fixed_string_t port; - uint8_t port_id; + portid_t port_id; cmdline_fixed_string_t what; cmdline_fixed_string_t value; cmdline_fixed_string_t mode; @@ -6308,7 +7320,7 @@ cmdline_parse_token_string_t cmd_set_uc_all_hash_port = port, "port"); cmdline_parse_token_num_t cmd_set_uc_all_hash_portid = TOKEN_NUM_INITIALIZER(struct cmd_set_uc_all_hash_table, - port_id, UINT8); + port_id, UINT16); cmdline_parse_token_string_t cmd_set_uc_all_hash_what = TOKEN_STRING_INITIALIZER(struct cmd_set_uc_all_hash_table, what, "uta"); @@ -6322,7 +7334,7 @@ cmdline_parse_token_string_t cmd_set_uc_all_hash_mode = cmdline_parse_inst_t cmd_set_uc_all_hash_filter = { .f = cmd_set_uc_all_hash_parsed, .data = NULL, - .help_str = "set port X uta all on|off (X = port number)", + .help_str = "set port uta all on|off", .tokens = { (void *)&cmd_set_uc_all_hash_set, (void *)&cmd_set_uc_all_hash_port, @@ -6338,7 +7350,7 @@ cmdline_parse_inst_t cmd_set_uc_all_hash_filter = { struct cmd_set_vf_macvlan_filter { cmdline_fixed_string_t set; cmdline_fixed_string_t port; - uint8_t port_id; + portid_t port_id; cmdline_fixed_string_t vf; uint8_t vf_id; struct ether_addr address; @@ -6357,7 +7369,7 @@ cmd_set_vf_macvlan_parsed(void *parsed_result, memset(&filter, 0, sizeof(struct rte_eth_mac_filter)); - (void)rte_memcpy(&filter.mac_addr, &res->address, ETHER_ADDR_LEN); + rte_memcpy(&filter.mac_addr, &res->address, ETHER_ADDR_LEN); /* set VF MAC filter */ filter.is_vf = 1; @@ -6400,7 +7412,7 @@ cmdline_parse_token_string_t cmd_set_vf_macvlan_port = port, "port"); cmdline_parse_token_num_t cmd_set_vf_macvlan_portid = TOKEN_NUM_INITIALIZER(struct cmd_set_vf_macvlan_filter, - port_id, UINT8); + port_id, UINT16); cmdline_parse_token_string_t cmd_set_vf_macvlan_vf = TOKEN_STRING_INITIALIZER(struct cmd_set_vf_macvlan_filter, vf, "vf"); @@ -6421,12 +7433,10 @@ cmdline_parse_token_string_t cmd_set_vf_macvlan_mode = cmdline_parse_inst_t cmd_set_vf_macvlan_filter = { .f = cmd_set_vf_macvlan_parsed, .data = NULL, - .help_str = "set port (portid) vf (vfid) (mac-addr) " - "(exact-mac|exact-mac-vlan|hashmac|hashmac-vlan) " - "on|off\n" - "exact match rule:exact match of MAC or MAC and VLAN; " - "hash match rule: hash match of MAC and exact match " - "of VLAN", + .help_str = "set port vf " + "exact-mac|exact-mac-vlan|hashmac|hashmac-vlan on|off: " + "Exact match rule: exact match of MAC or MAC and VLAN; " + "hash match rule: hash match of MAC and exact match of VLAN", .tokens = { (void *)&cmd_set_vf_macvlan_set, (void *)&cmd_set_vf_macvlan_port, @@ -6444,7 +7454,7 @@ cmdline_parse_inst_t cmd_set_vf_macvlan_filter = { struct cmd_set_vf_traffic { cmdline_fixed_string_t set; cmdline_fixed_string_t port; - uint8_t port_id; + portid_t port_id; cmdline_fixed_string_t vf; uint8_t vf_id; cmdline_fixed_string_t what; @@ -6471,7 +7481,7 @@ cmdline_parse_token_string_t cmd_setvf_traffic_port = port, "port"); cmdline_parse_token_num_t cmd_setvf_traffic_portid = TOKEN_NUM_INITIALIZER(struct cmd_set_vf_traffic, - port_id, UINT8); + port_id, UINT16); cmdline_parse_token_string_t cmd_setvf_traffic_vf = TOKEN_STRING_INITIALIZER(struct cmd_set_vf_traffic, vf, "vf"); @@ -6488,8 +7498,7 @@ cmdline_parse_token_string_t cmd_setvf_traffic_mode = cmdline_parse_inst_t cmd_set_vf_traffic = { .f = cmd_set_vf_traffic_parsed, .data = NULL, - .help_str = "set port X vf Y rx|tx on|off" - "(X = port number,Y = vf id)", + .help_str = "set port vf rx|tx on|off", .tokens = { (void *)&cmd_setvf_traffic_set, (void *)&cmd_setvf_traffic_port, @@ -6506,7 +7515,7 @@ cmdline_parse_inst_t cmd_set_vf_traffic = { struct cmd_set_vf_rxmode { cmdline_fixed_string_t set; cmdline_fixed_string_t port; - uint8_t port_id; + portid_t port_id; cmdline_fixed_string_t vf; uint8_t vf_id; cmdline_fixed_string_t what; @@ -6519,7 +7528,7 @@ cmd_set_vf_rxmode_parsed(void *parsed_result, __attribute__((unused)) struct cmdline *cl, __attribute__((unused)) void *data) { - int ret; + int ret = -ENOTSUP; uint16_t rx_mode = 0; struct cmd_set_vf_rxmode *res = parsed_result; @@ -6535,7 +7544,18 @@ cmd_set_vf_rxmode_parsed(void *parsed_result, rx_mode |= ETH_VMDQ_ACCEPT_MULTICAST; } - ret = rte_eth_dev_set_vf_rxmode(res->port_id,res->vf_id,rx_mode,(uint8_t)is_on); + RTE_SET_USED(is_on); + +#ifdef RTE_LIBRTE_IXGBE_PMD + if (ret == -ENOTSUP) + ret = rte_pmd_ixgbe_set_vf_rxmode(res->port_id, res->vf_id, + rx_mode, (uint8_t)is_on); +#endif +#ifdef RTE_LIBRTE_BNXT_PMD + if (ret == -ENOTSUP) + ret = rte_pmd_bnxt_set_vf_rxmode(res->port_id, res->vf_id, + rx_mode, (uint8_t)is_on); +#endif if (ret < 0) printf("bad VF receive mode parameter, return code = %d \n", ret); @@ -6549,7 +7569,7 @@ cmdline_parse_token_string_t cmd_set_vf_rxmode_port = port, "port"); cmdline_parse_token_num_t cmd_set_vf_rxmode_portid = TOKEN_NUM_INITIALIZER(struct cmd_set_vf_rxmode, - port_id, UINT8); + port_id, UINT16); cmdline_parse_token_string_t cmd_set_vf_rxmode_vf = TOKEN_STRING_INITIALIZER(struct cmd_set_vf_rxmode, vf, "vf"); @@ -6569,7 +7589,8 @@ cmdline_parse_token_string_t cmd_set_vf_rxmode_on = cmdline_parse_inst_t cmd_set_vf_rxmode = { .f = cmd_set_vf_rxmode_parsed, .data = NULL, - .help_str = "set port X vf Y rxmode AUPE|ROPE|BAM|MPE on|off", + .help_str = "set port vf rxmode " + "AUPE|ROPE|BAM|MPE on|off", .tokens = { (void *)&cmd_set_vf_rxmode_set, (void *)&cmd_set_vf_rxmode_port, @@ -6588,7 +7609,7 @@ struct cmd_vf_mac_addr_result { cmdline_fixed_string_t mac_addr_cmd; cmdline_fixed_string_t what; cmdline_fixed_string_t port; - uint8_t port_num; + uint16_t port_num; cmdline_fixed_string_t vf; uint8_t vf_num; struct ether_addr address; @@ -6599,11 +7620,22 @@ static void cmd_vf_mac_addr_parsed(void *parsed_result, __attribute__((unused)) void *data) { struct cmd_vf_mac_addr_result *res = parsed_result; - int ret = 0; + int ret = -ENOTSUP; + + if (strcmp(res->what, "add") != 0) + return; + +#ifdef RTE_LIBRTE_I40E_PMD + if (ret == -ENOTSUP) + ret = rte_pmd_i40e_add_vf_mac_addr(res->port_num, res->vf_num, + &res->address); +#endif +#ifdef RTE_LIBRTE_BNXT_PMD + if (ret == -ENOTSUP) + ret = rte_pmd_bnxt_mac_addr_add(res->port_num, &res->address, + res->vf_num); +#endif - if (strcmp(res->what, "add") == 0) - ret = rte_eth_dev_mac_addr_add(res->port_num, - &res->address, res->vf_num); if(ret < 0) printf("vf_mac_addr_cmd error: (%s)\n", strerror(-ret)); @@ -6620,7 +7652,7 @@ cmdline_parse_token_string_t cmd_vf_mac_addr_port = port,"port"); cmdline_parse_token_num_t cmd_vf_mac_addr_portnum = TOKEN_NUM_INITIALIZER(struct cmd_vf_mac_addr_result, - port_num, UINT8); + port_num, UINT16); cmdline_parse_token_string_t cmd_vf_mac_addr_vf = TOKEN_STRING_INITIALIZER(struct cmd_vf_mac_addr_result, vf,"vf"); @@ -6634,8 +7666,8 @@ cmdline_parse_token_etheraddr_t cmd_vf_mac_addr_addr = cmdline_parse_inst_t cmd_vf_mac_addr_filter = { .f = cmd_vf_mac_addr_parsed, .data = (void *)0, - .help_str = "mac_addr add port X vf Y ethaddr:(X = port number," - "Y = VF number)add MAC address filtering for a VF on port X", + .help_str = "mac_addr add port vf : " + "Add MAC address filtering for a VF on port_id", .tokens = { (void *)&cmd_vf_mac_addr_cmd, (void *)&cmd_vf_mac_addr_what, @@ -6654,7 +7686,7 @@ struct cmd_vf_rx_vlan_filter { cmdline_fixed_string_t what; uint16_t vlan_id; cmdline_fixed_string_t port; - uint8_t port_id; + portid_t port_id; cmdline_fixed_string_t vf; uint64_t vf_mask; }; @@ -6665,11 +7697,42 @@ cmd_vf_rx_vlan_filter_parsed(void *parsed_result, __attribute__((unused)) void *data) { struct cmd_vf_rx_vlan_filter *res = parsed_result; + int ret = -ENOTSUP; - if (!strcmp(res->what, "add")) - set_vf_rx_vlan(res->port_id, res->vlan_id,res->vf_mask, 1); - else - set_vf_rx_vlan(res->port_id, res->vlan_id,res->vf_mask, 0); + __rte_unused int is_add = (strcmp(res->what, "add") == 0) ? 1 : 0; + +#ifdef RTE_LIBRTE_IXGBE_PMD + if (ret == -ENOTSUP) + ret = rte_pmd_ixgbe_set_vf_vlan_filter(res->port_id, + res->vlan_id, res->vf_mask, is_add); +#endif +#ifdef RTE_LIBRTE_I40E_PMD + if (ret == -ENOTSUP) + ret = rte_pmd_i40e_set_vf_vlan_filter(res->port_id, + res->vlan_id, res->vf_mask, is_add); +#endif +#ifdef RTE_LIBRTE_BNXT_PMD + if (ret == -ENOTSUP) + ret = rte_pmd_bnxt_set_vf_vlan_filter(res->port_id, + res->vlan_id, res->vf_mask, is_add); +#endif + + switch (ret) { + case 0: + break; + case -EINVAL: + printf("invalid vlan_id %d or vf_mask %"PRIu64"\n", + res->vlan_id, res->vf_mask); + break; + case -ENODEV: + printf("invalid port_id %d\n", res->port_id); + break; + case -ENOTSUP: + printf("function not implemented or supported\n"); + break; + default: + printf("programming error: (%s)\n", strerror(-ret)); + } } cmdline_parse_token_string_t cmd_vf_rx_vlan_filter_rx_vlan = @@ -6686,7 +7749,7 @@ cmdline_parse_token_string_t cmd_vf_rx_vlan_filter_port = port, "port"); cmdline_parse_token_num_t cmd_vf_rx_vlan_filter_portid = TOKEN_NUM_INITIALIZER(struct cmd_vf_rx_vlan_filter, - port_id, UINT8); + port_id, UINT16); cmdline_parse_token_string_t cmd_vf_rx_vlan_filter_vf = TOKEN_STRING_INITIALIZER(struct cmd_vf_rx_vlan_filter, vf, "vf"); @@ -6697,8 +7760,8 @@ cmdline_parse_token_num_t cmd_vf_rx_vlan_filter_vf_mask = cmdline_parse_inst_t cmd_vf_rxvlan_filter = { .f = cmd_vf_rx_vlan_filter_parsed, .data = NULL, - .help_str = "rx_vlan add|rm X port Y vf Z (X = VLAN ID," - "Y = port number,Z = hexadecimal VF mask)", + .help_str = "rx_vlan add|rm port vf : " + "(vf_mask = hexadecimal VF mask)", .tokens = { (void *)&cmd_vf_rx_vlan_filter_rx_vlan, (void *)&cmd_vf_rx_vlan_filter_what, @@ -6715,7 +7778,7 @@ cmdline_parse_inst_t cmd_vf_rxvlan_filter = { struct cmd_queue_rate_limit_result { cmdline_fixed_string_t set; cmdline_fixed_string_t port; - uint8_t port_num; + uint16_t port_num; cmdline_fixed_string_t queue; uint8_t queue_num; cmdline_fixed_string_t rate; @@ -6747,7 +7810,7 @@ cmdline_parse_token_string_t cmd_queue_rate_limit_port = port, "port"); cmdline_parse_token_num_t cmd_queue_rate_limit_portnum = TOKEN_NUM_INITIALIZER(struct cmd_queue_rate_limit_result, - port_num, UINT8); + port_num, UINT16); cmdline_parse_token_string_t cmd_queue_rate_limit_queue = TOKEN_STRING_INITIALIZER(struct cmd_queue_rate_limit_result, queue, "queue"); @@ -6764,8 +7827,8 @@ cmdline_parse_token_num_t cmd_queue_rate_limit_ratenum = cmdline_parse_inst_t cmd_queue_rate_limit = { .f = cmd_queue_rate_limit_parsed, .data = (void *)0, - .help_str = "set port X queue Y rate Z:(X = port number," - "Y = queue number,Z = rate number)set rate limit for a queue on port X", + .help_str = "set port queue rate : " + "Set rate limit for a queue on port_id", .tokens = { (void *)&cmd_queue_rate_limit_set, (void *)&cmd_queue_rate_limit_port, @@ -6782,7 +7845,7 @@ cmdline_parse_inst_t cmd_queue_rate_limit = { struct cmd_vf_rate_limit_result { cmdline_fixed_string_t set; cmdline_fixed_string_t port; - uint8_t port_num; + uint16_t port_num; cmdline_fixed_string_t vf; uint8_t vf_num; cmdline_fixed_string_t rate; @@ -6817,7 +7880,7 @@ cmdline_parse_token_string_t cmd_vf_rate_limit_port = port, "port"); cmdline_parse_token_num_t cmd_vf_rate_limit_portnum = TOKEN_NUM_INITIALIZER(struct cmd_vf_rate_limit_result, - port_num, UINT8); + port_num, UINT16); cmdline_parse_token_string_t cmd_vf_rate_limit_vf = TOKEN_STRING_INITIALIZER(struct cmd_vf_rate_limit_result, vf, "vf"); @@ -6840,9 +7903,9 @@ cmdline_parse_token_num_t cmd_vf_rate_limit_q_msk_val = cmdline_parse_inst_t cmd_vf_rate_limit = { .f = cmd_vf_rate_limit_parsed, .data = (void *)0, - .help_str = "set port X vf Y rate Z queue_mask V:(X = port number," - "Y = VF number,Z = rate number, V = queue mask value)set rate limit " - "for queues of VF on port X", + .help_str = "set port vf rate " + "queue_mask : " + "Set rate limit for queues of VF on port_id", .tokens = { (void *)&cmd_vf_rate_limit_set, (void *)&cmd_vf_rate_limit_port, @@ -6861,7 +7924,7 @@ cmdline_parse_inst_t cmd_vf_rate_limit = { struct cmd_tunnel_filter_result { cmdline_fixed_string_t cmd; cmdline_fixed_string_t what; - uint8_t port_id; + portid_t port_id; struct ether_addr outer_mac; struct ether_addr inner_mac; cmdline_ipaddr_t ip_value; @@ -6955,7 +8018,7 @@ cmdline_parse_token_string_t cmd_tunnel_filter_what = what, "add#rm"); cmdline_parse_token_num_t cmd_tunnel_filter_port_id = TOKEN_NUM_INITIALIZER(struct cmd_tunnel_filter_result, - port_id, UINT8); + port_id, UINT16); cmdline_parse_token_etheraddr_t cmd_tunnel_filter_outer_mac = TOKEN_ETHERADDR_INITIALIZER(struct cmd_tunnel_filter_result, outer_mac); @@ -6986,12 +8049,10 @@ cmdline_parse_token_num_t cmd_tunnel_filter_queue_num = cmdline_parse_inst_t cmd_tunnel_filter = { .f = cmd_tunnel_filter_parsed, .data = (void *)0, - .help_str = "add/rm tunnel filter of a port: " - "tunnel_filter add port_id outer_mac inner_mac ip " - "inner_vlan tunnel_type(vxlan|nvgre|ipingre) filter_type " - "(oip|iip|imac-ivlan|imac-ivlan-tenid|imac-tenid|" - "imac|omac-imac-tenid) " - "tenant_id queue_num", + .help_str = "tunnel_filter add|rm " + " vxlan|nvgre|ipingre oip|iip|imac-ivlan|" + "imac-ivlan-tenid|imac-tenid|imac|omac-imac-tenid " + ": Add/Rm tunnel filter of a port", .tokens = { (void *)&cmd_tunnel_filter_cmd, (void *)&cmd_tunnel_filter_what, @@ -7013,7 +8074,7 @@ struct cmd_tunnel_udp_config { cmdline_fixed_string_t cmd; cmdline_fixed_string_t what; uint16_t udp_port; - uint8_t port_id; + portid_t port_id; }; static void @@ -7052,13 +8113,13 @@ cmdline_parse_token_num_t cmd_tunnel_udp_config_udp_port = udp_port, UINT16); cmdline_parse_token_num_t cmd_tunnel_udp_config_port_id = TOKEN_NUM_INITIALIZER(struct cmd_tunnel_udp_config, - port_id, UINT8); + port_id, UINT16); cmdline_parse_inst_t cmd_tunnel_udp_config = { .f = cmd_tunnel_udp_config_parsed, .data = (void *)0, - .help_str = "add/rm an tunneling UDP port filter: " - "rx_vxlan_port add udp_port port_id", + .help_str = "rx_vxlan_port add|rm : " + "Add/Remove a tunneling UDP port filter", .tokens = { (void *)&cmd_tunnel_udp_config_cmd, (void *)&cmd_tunnel_udp_config_what, @@ -7071,7 +8132,7 @@ cmdline_parse_inst_t cmd_tunnel_udp_config = { /* *** GLOBAL CONFIG *** */ struct cmd_global_config_result { cmdline_fixed_string_t cmd; - uint8_t port_id; + portid_t port_id; cmdline_fixed_string_t cfg_type; uint8_t len; }; @@ -7098,7 +8159,8 @@ cmdline_parse_token_string_t cmd_global_config_cmd = TOKEN_STRING_INITIALIZER(struct cmd_global_config_result, cmd, "global_config"); cmdline_parse_token_num_t cmd_global_config_port_id = - TOKEN_NUM_INITIALIZER(struct cmd_global_config_result, port_id, UINT8); + TOKEN_NUM_INITIALIZER(struct cmd_global_config_result, port_id, + UINT16); cmdline_parse_token_string_t cmd_global_config_type = TOKEN_STRING_INITIALIZER(struct cmd_global_config_result, cfg_type, "gre-key-len"); @@ -7109,7 +8171,7 @@ cmdline_parse_token_num_t cmd_global_config_gre_key_len = cmdline_parse_inst_t cmd_global_config = { .f = cmd_global_config_parsed, .data = (void *)NULL, - .help_str = "global_config gre-key-len ", + .help_str = "global_config gre-key-len ", .tokens = { (void *)&cmd_global_config_cmd, (void *)&cmd_global_config_port_id, @@ -7123,7 +8185,7 @@ cmdline_parse_inst_t cmd_global_config = { struct cmd_set_mirror_mask_result { cmdline_fixed_string_t set; cmdline_fixed_string_t port; - uint8_t port_id; + portid_t port_id; cmdline_fixed_string_t mirror; uint8_t rule_id; cmdline_fixed_string_t what; @@ -7141,7 +8203,7 @@ cmdline_parse_token_string_t cmd_mirror_mask_port = port, "port"); cmdline_parse_token_num_t cmd_mirror_mask_portid = TOKEN_NUM_INITIALIZER(struct cmd_set_mirror_mask_result, - port_id, UINT8); + port_id, UINT16); cmdline_parse_token_string_t cmd_mirror_mask_mirror = TOKEN_STRING_INITIALIZER(struct cmd_set_mirror_mask_result, mirror, "mirror-rule"); @@ -7217,8 +8279,9 @@ cmd_set_mirror_mask_parsed(void *parsed_result, cmdline_parse_inst_t cmd_set_mirror_mask = { .f = cmd_set_mirror_mask_parsed, .data = NULL, - .help_str = "set port X mirror-rule Y pool-mirror-up|pool-mirror-down|vlan-mirror" - " pool_mask|vlan_id[,vlan_id]* dst-pool Z on|off", + .help_str = "set port mirror-rule " + "pool-mirror-up|pool-mirror-down|vlan-mirror " + " dst-pool on|off", .tokens = { (void *)&cmd_mirror_mask_set, (void *)&cmd_mirror_mask_port, @@ -7234,11 +8297,11 @@ cmdline_parse_inst_t cmd_set_mirror_mask = { }, }; -/* *** CONFIGURE VM MIRROR UDLINK/DOWNLINK RULE *** */ +/* *** CONFIGURE VM MIRROR UPLINK/DOWNLINK RULE *** */ struct cmd_set_mirror_link_result { cmdline_fixed_string_t set; cmdline_fixed_string_t port; - uint8_t port_id; + portid_t port_id; cmdline_fixed_string_t mirror; uint8_t rule_id; cmdline_fixed_string_t what; @@ -7255,7 +8318,7 @@ cmdline_parse_token_string_t cmd_mirror_link_port = port, "port"); cmdline_parse_token_num_t cmd_mirror_link_portid = TOKEN_NUM_INITIALIZER(struct cmd_set_mirror_link_result, - port_id, UINT8); + port_id, UINT16); cmdline_parse_token_string_t cmd_mirror_link_mirror = TOKEN_STRING_INITIALIZER(struct cmd_set_mirror_link_result, mirror, "mirror-rule"); @@ -7308,8 +8371,8 @@ cmd_set_mirror_link_parsed(void *parsed_result, cmdline_parse_inst_t cmd_set_mirror_link = { .f = cmd_set_mirror_link_parsed, .data = NULL, - .help_str = "set port X mirror-rule Y uplink-mirror|" - "downlink-mirror dst-pool Z on|off", + .help_str = "set port mirror-rule " + "uplink-mirror|downlink-mirror dst-pool on|off", .tokens = { (void *)&cmd_mirror_link_set, (void *)&cmd_mirror_link_port, @@ -7328,7 +8391,7 @@ cmdline_parse_inst_t cmd_set_mirror_link = { struct cmd_rm_mirror_rule_result { cmdline_fixed_string_t reset; cmdline_fixed_string_t port; - uint8_t port_id; + portid_t port_id; cmdline_fixed_string_t mirror; uint8_t rule_id; }; @@ -7341,7 +8404,7 @@ cmdline_parse_token_string_t cmd_rm_mirror_rule_port = port, "port"); cmdline_parse_token_num_t cmd_rm_mirror_rule_portid = TOKEN_NUM_INITIALIZER(struct cmd_rm_mirror_rule_result, - port_id, UINT8); + port_id, UINT16); cmdline_parse_token_string_t cmd_rm_mirror_rule_mirror = TOKEN_STRING_INITIALIZER(struct cmd_rm_mirror_rule_result, mirror, "mirror-rule"); @@ -7365,7 +8428,7 @@ cmd_reset_mirror_rule_parsed(void *parsed_result, cmdline_parse_inst_t cmd_reset_mirror_rule = { .f = cmd_reset_mirror_rule_parsed, .data = NULL, - .help_str = "reset port X mirror-rule Y", + .help_str = "reset port mirror-rule ", .tokens = { (void *)&cmd_rm_mirror_rule_reset, (void *)&cmd_rm_mirror_rule_port, @@ -7410,6 +8473,8 @@ static void cmd_dump_parsed(void *parsed_result, rte_mempool_list_dump(stdout); else if (!strcmp(res->dump, "dump_devargs")) rte_eal_devargs_dump(stdout); + else if (!strcmp(res->dump, "dump_log_types")) + rte_log_dump(stdout); } cmdline_parse_token_string_t cmd_dump_dump = @@ -7419,12 +8484,13 @@ cmdline_parse_token_string_t cmd_dump_dump = "dump_struct_sizes#" "dump_ring#" "dump_mempool#" - "dump_devargs"); + "dump_devargs#" + "dump_log_types"); cmdline_parse_inst_t cmd_dump = { .f = cmd_dump_parsed, /* function to call */ .data = NULL, /* 2nd arg of func */ - .help_str = "dump status", + .help_str = "Dump status", .tokens = { /* token list, NULL terminated */ (void *)&cmd_dump_dump, NULL, @@ -7472,7 +8538,7 @@ cmdline_parse_token_string_t cmd_dump_one_name = cmdline_parse_inst_t cmd_dump_one = { .f = cmd_dump_one_parsed, /* function to call */ .data = NULL, /* 2nd arg of func */ - .help_str = "dump one ring/mempool: dump_ring|dump_mempool ", + .help_str = "dump_ring|dump_mempool : Dump one ring/mempool", .tokens = { /* token list, NULL terminated */ (void *)&cmd_dump_one_dump, (void *)&cmd_dump_one_name, @@ -7483,7 +8549,7 @@ cmdline_parse_inst_t cmd_dump_one = { /* *** Add/Del syn filter *** */ struct cmd_syn_filter_result { cmdline_fixed_string_t filter; - uint8_t port_id; + portid_t port_id; cmdline_fixed_string_t ops; cmdline_fixed_string_t priority; cmdline_fixed_string_t high; @@ -7537,7 +8603,7 @@ cmdline_parse_token_string_t cmd_syn_filter_filter = filter, "syn_filter"); cmdline_parse_token_num_t cmd_syn_filter_port_id = TOKEN_NUM_INITIALIZER(struct cmd_syn_filter_result, - port_id, UINT8); + port_id, UINT16); cmdline_parse_token_string_t cmd_syn_filter_ops = TOKEN_STRING_INITIALIZER(struct cmd_syn_filter_result, ops, "add#del"); @@ -7557,7 +8623,8 @@ cmdline_parse_token_num_t cmd_syn_filter_queue_id = cmdline_parse_inst_t cmd_syn_filter = { .f = cmd_syn_filter_parsed, .data = NULL, - .help_str = "add/delete syn filter", + .help_str = "syn_filter add|del priority high|low queue " + ": Add/Delete syn filter", .tokens = { (void *)&cmd_syn_filter_filter, (void *)&cmd_syn_filter_port_id, @@ -7570,10 +8637,452 @@ cmdline_parse_inst_t cmd_syn_filter = { }, }; +/* *** queue region set *** */ +struct cmd_queue_region_result { + cmdline_fixed_string_t set; + cmdline_fixed_string_t port; + portid_t port_id; + cmdline_fixed_string_t cmd; + cmdline_fixed_string_t region; + uint8_t region_id; + cmdline_fixed_string_t queue_start_index; + uint8_t queue_id; + cmdline_fixed_string_t queue_num; + uint8_t queue_num_value; +}; + +static void +cmd_queue_region_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_queue_region_result *res = parsed_result; + int ret = -ENOTSUP; +#ifdef RTE_LIBRTE_I40E_PMD + struct rte_pmd_i40e_queue_region_conf region_conf; + enum rte_pmd_i40e_queue_region_op op_type; +#endif + + if (port_id_is_invalid(res->port_id, ENABLED_WARN)) + return; + +#ifdef RTE_LIBRTE_I40E_PMD + memset(®ion_conf, 0, sizeof(region_conf)); + op_type = RTE_PMD_I40E_RSS_QUEUE_REGION_SET; + region_conf.region_id = res->region_id; + region_conf.queue_num = res->queue_num_value; + region_conf.queue_start_index = res->queue_id; + + ret = rte_pmd_i40e_rss_queue_region_conf(res->port_id, + op_type, ®ion_conf); +#endif + + switch (ret) { + case 0: + break; + case -ENOTSUP: + printf("function not implemented or supported\n"); + break; + default: + printf("queue region config error: (%s)\n", strerror(-ret)); + } +} + +cmdline_parse_token_string_t cmd_queue_region_set = +TOKEN_STRING_INITIALIZER(struct cmd_queue_region_result, + set, "set"); +cmdline_parse_token_string_t cmd_queue_region_port = + TOKEN_STRING_INITIALIZER(struct cmd_queue_region_result, port, "port"); +cmdline_parse_token_num_t cmd_queue_region_port_id = + TOKEN_NUM_INITIALIZER(struct cmd_queue_region_result, + port_id, UINT16); +cmdline_parse_token_string_t cmd_queue_region_cmd = + TOKEN_STRING_INITIALIZER(struct cmd_queue_region_result, + cmd, "queue-region"); +cmdline_parse_token_string_t cmd_queue_region_id = + TOKEN_STRING_INITIALIZER(struct cmd_queue_region_result, + region, "region_id"); +cmdline_parse_token_num_t cmd_queue_region_index = + TOKEN_NUM_INITIALIZER(struct cmd_queue_region_result, + region_id, UINT8); +cmdline_parse_token_string_t cmd_queue_region_queue_start_index = + TOKEN_STRING_INITIALIZER(struct cmd_queue_region_result, + queue_start_index, "queue_start_index"); +cmdline_parse_token_num_t cmd_queue_region_queue_id = + TOKEN_NUM_INITIALIZER(struct cmd_queue_region_result, + queue_id, UINT8); +cmdline_parse_token_string_t cmd_queue_region_queue_num = + TOKEN_STRING_INITIALIZER(struct cmd_queue_region_result, + queue_num, "queue_num"); +cmdline_parse_token_num_t cmd_queue_region_queue_num_value = + TOKEN_NUM_INITIALIZER(struct cmd_queue_region_result, + queue_num_value, UINT8); + +cmdline_parse_inst_t cmd_queue_region = { + .f = cmd_queue_region_parsed, + .data = NULL, + .help_str = "set port queue-region region_id " + "queue_start_index queue_num : Set a queue region", + .tokens = { + (void *)&cmd_queue_region_set, + (void *)&cmd_queue_region_port, + (void *)&cmd_queue_region_port_id, + (void *)&cmd_queue_region_cmd, + (void *)&cmd_queue_region_id, + (void *)&cmd_queue_region_index, + (void *)&cmd_queue_region_queue_start_index, + (void *)&cmd_queue_region_queue_id, + (void *)&cmd_queue_region_queue_num, + (void *)&cmd_queue_region_queue_num_value, + NULL, + }, +}; + +/* *** queue region and flowtype set *** */ +struct cmd_region_flowtype_result { + cmdline_fixed_string_t set; + cmdline_fixed_string_t port; + portid_t port_id; + cmdline_fixed_string_t cmd; + cmdline_fixed_string_t region; + uint8_t region_id; + cmdline_fixed_string_t flowtype; + uint8_t flowtype_id; +}; + +static void +cmd_region_flowtype_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_region_flowtype_result *res = parsed_result; + int ret = -ENOTSUP; +#ifdef RTE_LIBRTE_I40E_PMD + struct rte_pmd_i40e_queue_region_conf region_conf; + enum rte_pmd_i40e_queue_region_op op_type; +#endif + + if (port_id_is_invalid(res->port_id, ENABLED_WARN)) + return; + +#ifdef RTE_LIBRTE_I40E_PMD + memset(®ion_conf, 0, sizeof(region_conf)); + + op_type = RTE_PMD_I40E_RSS_QUEUE_REGION_FLOWTYPE_SET; + region_conf.region_id = res->region_id; + region_conf.hw_flowtype = res->flowtype_id; + + ret = rte_pmd_i40e_rss_queue_region_conf(res->port_id, + op_type, ®ion_conf); +#endif + + switch (ret) { + case 0: + break; + case -ENOTSUP: + printf("function not implemented or supported\n"); + break; + default: + printf("region flowtype config error: (%s)\n", strerror(-ret)); + } +} + +cmdline_parse_token_string_t cmd_region_flowtype_set = +TOKEN_STRING_INITIALIZER(struct cmd_region_flowtype_result, + set, "set"); +cmdline_parse_token_string_t cmd_region_flowtype_port = + TOKEN_STRING_INITIALIZER(struct cmd_region_flowtype_result, + port, "port"); +cmdline_parse_token_num_t cmd_region_flowtype_port_index = + TOKEN_NUM_INITIALIZER(struct cmd_region_flowtype_result, + port_id, UINT16); +cmdline_parse_token_string_t cmd_region_flowtype_cmd = + TOKEN_STRING_INITIALIZER(struct cmd_region_flowtype_result, + cmd, "queue-region"); +cmdline_parse_token_string_t cmd_region_flowtype_index = + TOKEN_STRING_INITIALIZER(struct cmd_region_flowtype_result, + region, "region_id"); +cmdline_parse_token_num_t cmd_region_flowtype_id = + TOKEN_NUM_INITIALIZER(struct cmd_region_flowtype_result, + region_id, UINT8); +cmdline_parse_token_string_t cmd_region_flowtype_flow_index = + TOKEN_STRING_INITIALIZER(struct cmd_region_flowtype_result, + flowtype, "flowtype"); +cmdline_parse_token_num_t cmd_region_flowtype_flow_id = + TOKEN_NUM_INITIALIZER(struct cmd_region_flowtype_result, + flowtype_id, UINT8); +cmdline_parse_inst_t cmd_region_flowtype = { + .f = cmd_region_flowtype_parsed, + .data = NULL, + .help_str = "set port queue-region region_id " + "flowtype : Set a flowtype region index", + .tokens = { + (void *)&cmd_region_flowtype_set, + (void *)&cmd_region_flowtype_port, + (void *)&cmd_region_flowtype_port_index, + (void *)&cmd_region_flowtype_cmd, + (void *)&cmd_region_flowtype_index, + (void *)&cmd_region_flowtype_id, + (void *)&cmd_region_flowtype_flow_index, + (void *)&cmd_region_flowtype_flow_id, + NULL, + }, +}; + +/* *** User Priority (UP) to queue region (region_id) set *** */ +struct cmd_user_priority_region_result { + cmdline_fixed_string_t set; + cmdline_fixed_string_t port; + portid_t port_id; + cmdline_fixed_string_t cmd; + cmdline_fixed_string_t user_priority; + uint8_t user_priority_id; + cmdline_fixed_string_t region; + uint8_t region_id; +}; + +static void +cmd_user_priority_region_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_user_priority_region_result *res = parsed_result; + int ret = -ENOTSUP; +#ifdef RTE_LIBRTE_I40E_PMD + struct rte_pmd_i40e_queue_region_conf region_conf; + enum rte_pmd_i40e_queue_region_op op_type; +#endif + + if (port_id_is_invalid(res->port_id, ENABLED_WARN)) + return; + +#ifdef RTE_LIBRTE_I40E_PMD + memset(®ion_conf, 0, sizeof(region_conf)); + op_type = RTE_PMD_I40E_RSS_QUEUE_REGION_USER_PRIORITY_SET; + region_conf.user_priority = res->user_priority_id; + region_conf.region_id = res->region_id; + + ret = rte_pmd_i40e_rss_queue_region_conf(res->port_id, + op_type, ®ion_conf); +#endif + + switch (ret) { + case 0: + break; + case -ENOTSUP: + printf("function not implemented or supported\n"); + break; + default: + printf("user_priority region config error: (%s)\n", + strerror(-ret)); + } +} + +cmdline_parse_token_string_t cmd_user_priority_region_set = + TOKEN_STRING_INITIALIZER(struct cmd_user_priority_region_result, + set, "set"); +cmdline_parse_token_string_t cmd_user_priority_region_port = + TOKEN_STRING_INITIALIZER(struct cmd_user_priority_region_result, + port, "port"); +cmdline_parse_token_num_t cmd_user_priority_region_port_index = + TOKEN_NUM_INITIALIZER(struct cmd_user_priority_region_result, + port_id, UINT16); +cmdline_parse_token_string_t cmd_user_priority_region_cmd = + TOKEN_STRING_INITIALIZER(struct cmd_user_priority_region_result, + cmd, "queue-region"); +cmdline_parse_token_string_t cmd_user_priority_region_UP = + TOKEN_STRING_INITIALIZER(struct cmd_user_priority_region_result, + user_priority, "UP"); +cmdline_parse_token_num_t cmd_user_priority_region_UP_id = + TOKEN_NUM_INITIALIZER(struct cmd_user_priority_region_result, + user_priority_id, UINT8); +cmdline_parse_token_string_t cmd_user_priority_region_region = + TOKEN_STRING_INITIALIZER(struct cmd_user_priority_region_result, + region, "region_id"); +cmdline_parse_token_num_t cmd_user_priority_region_region_id = + TOKEN_NUM_INITIALIZER(struct cmd_user_priority_region_result, + region_id, UINT8); + +cmdline_parse_inst_t cmd_user_priority_region = { + .f = cmd_user_priority_region_parsed, + .data = NULL, + .help_str = "set port queue-region UP " + "region_id : Set the mapping of User Priority (UP) " + "to queue region (region_id) ", + .tokens = { + (void *)&cmd_user_priority_region_set, + (void *)&cmd_user_priority_region_port, + (void *)&cmd_user_priority_region_port_index, + (void *)&cmd_user_priority_region_cmd, + (void *)&cmd_user_priority_region_UP, + (void *)&cmd_user_priority_region_UP_id, + (void *)&cmd_user_priority_region_region, + (void *)&cmd_user_priority_region_region_id, + NULL, + }, +}; + +/* *** flush all queue region related configuration *** */ +struct cmd_flush_queue_region_result { + cmdline_fixed_string_t set; + cmdline_fixed_string_t port; + portid_t port_id; + cmdline_fixed_string_t cmd; + cmdline_fixed_string_t flush; + cmdline_fixed_string_t what; +}; + +static void +cmd_flush_queue_region_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_flush_queue_region_result *res = parsed_result; + int ret = -ENOTSUP; +#ifdef RTE_LIBRTE_I40E_PMD + struct rte_pmd_i40e_queue_region_conf region_conf; + enum rte_pmd_i40e_queue_region_op op_type; +#endif + + if (port_id_is_invalid(res->port_id, ENABLED_WARN)) + return; + +#ifdef RTE_LIBRTE_I40E_PMD + memset(®ion_conf, 0, sizeof(region_conf)); + + if (strcmp(res->what, "on") == 0) + op_type = RTE_PMD_I40E_RSS_QUEUE_REGION_ALL_FLUSH_ON; + else + op_type = RTE_PMD_I40E_RSS_QUEUE_REGION_ALL_FLUSH_OFF; + + ret = rte_pmd_i40e_rss_queue_region_conf(res->port_id, + op_type, ®ion_conf); +#endif + + switch (ret) { + case 0: + break; + case -ENOTSUP: + printf("function not implemented or supported\n"); + break; + default: + printf("queue region config flush error: (%s)\n", + strerror(-ret)); + } +} + +cmdline_parse_token_string_t cmd_flush_queue_region_set = + TOKEN_STRING_INITIALIZER(struct cmd_flush_queue_region_result, + set, "set"); +cmdline_parse_token_string_t cmd_flush_queue_region_port = + TOKEN_STRING_INITIALIZER(struct cmd_flush_queue_region_result, + port, "port"); +cmdline_parse_token_num_t cmd_flush_queue_region_port_index = + TOKEN_NUM_INITIALIZER(struct cmd_flush_queue_region_result, + port_id, UINT16); +cmdline_parse_token_string_t cmd_flush_queue_region_cmd = + TOKEN_STRING_INITIALIZER(struct cmd_flush_queue_region_result, + cmd, "queue-region"); +cmdline_parse_token_string_t cmd_flush_queue_region_flush = + TOKEN_STRING_INITIALIZER(struct cmd_flush_queue_region_result, + flush, "flush"); +cmdline_parse_token_string_t cmd_flush_queue_region_what = + TOKEN_STRING_INITIALIZER(struct cmd_flush_queue_region_result, + what, "on#off"); + +cmdline_parse_inst_t cmd_flush_queue_region = { + .f = cmd_flush_queue_region_parsed, + .data = NULL, + .help_str = "set port queue-region flush on|off" + ": flush all queue region related configuration", + .tokens = { + (void *)&cmd_flush_queue_region_set, + (void *)&cmd_flush_queue_region_port, + (void *)&cmd_flush_queue_region_port_index, + (void *)&cmd_flush_queue_region_cmd, + (void *)&cmd_flush_queue_region_flush, + (void *)&cmd_flush_queue_region_what, + NULL, + }, +}; + +/* *** get all queue region related configuration info *** */ +struct cmd_show_queue_region_info { + cmdline_fixed_string_t show; + cmdline_fixed_string_t port; + portid_t port_id; + cmdline_fixed_string_t cmd; +}; + +static void +cmd_show_queue_region_info_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_show_queue_region_info *res = parsed_result; + int ret = -ENOTSUP; +#ifdef RTE_LIBRTE_I40E_PMD + struct rte_pmd_i40e_queue_regions rte_pmd_regions; + enum rte_pmd_i40e_queue_region_op op_type; +#endif + + if (port_id_is_invalid(res->port_id, ENABLED_WARN)) + return; + +#ifdef RTE_LIBRTE_I40E_PMD + memset(&rte_pmd_regions, 0, sizeof(rte_pmd_regions)); + + op_type = RTE_PMD_I40E_RSS_QUEUE_REGION_INFO_GET; + + ret = rte_pmd_i40e_rss_queue_region_conf(res->port_id, + op_type, &rte_pmd_regions); + + port_queue_region_info_display(res->port_id, &rte_pmd_regions); +#endif + + switch (ret) { + case 0: + break; + case -ENOTSUP: + printf("function not implemented or supported\n"); + break; + default: + printf("queue region config info show error: (%s)\n", + strerror(-ret)); + } +} + +cmdline_parse_token_string_t cmd_show_queue_region_info_get = +TOKEN_STRING_INITIALIZER(struct cmd_show_queue_region_info, + show, "show"); +cmdline_parse_token_string_t cmd_show_queue_region_info_port = + TOKEN_STRING_INITIALIZER(struct cmd_show_queue_region_info, + port, "port"); +cmdline_parse_token_num_t cmd_show_queue_region_info_port_index = + TOKEN_NUM_INITIALIZER(struct cmd_show_queue_region_info, + port_id, UINT16); +cmdline_parse_token_string_t cmd_show_queue_region_info_cmd = + TOKEN_STRING_INITIALIZER(struct cmd_show_queue_region_info, + cmd, "queue-region"); + +cmdline_parse_inst_t cmd_show_queue_region_info_all = { + .f = cmd_show_queue_region_info_parsed, + .data = NULL, + .help_str = "show port queue-region" + ": show all queue region related configuration info", + .tokens = { + (void *)&cmd_show_queue_region_info_get, + (void *)&cmd_show_queue_region_info_port, + (void *)&cmd_show_queue_region_info_port_index, + (void *)&cmd_show_queue_region_info_cmd, + NULL, + }, +}; + /* *** ADD/REMOVE A 2tuple FILTER *** */ struct cmd_2tuple_filter_result { cmdline_fixed_string_t filter; - uint8_t port_id; + portid_t port_id; cmdline_fixed_string_t ops; cmdline_fixed_string_t dst_port; uint16_t dst_port_value; @@ -7652,7 +9161,7 @@ cmdline_parse_token_string_t cmd_2tuple_filter_filter = filter, "2tuple_filter"); cmdline_parse_token_num_t cmd_2tuple_filter_port_id = TOKEN_NUM_INITIALIZER(struct cmd_2tuple_filter_result, - port_id, UINT8); + port_id, UINT16); cmdline_parse_token_string_t cmd_2tuple_filter_ops = TOKEN_STRING_INITIALIZER(struct cmd_2tuple_filter_result, ops, "add#del"); @@ -7696,7 +9205,9 @@ cmdline_parse_token_num_t cmd_2tuple_filter_queue_id = cmdline_parse_inst_t cmd_2tuple_filter = { .f = cmd_2tuple_filter_parsed, .data = NULL, - .help_str = "add a 2tuple filter", + .help_str = "2tuple_filter add|del dst_port protocol " + " mask tcp_flags priority queue " + ": Add a 2tuple filter", .tokens = { (void *)&cmd_2tuple_filter_filter, (void *)&cmd_2tuple_filter_port_id, @@ -7720,7 +9231,7 @@ cmdline_parse_inst_t cmd_2tuple_filter = { /* *** ADD/REMOVE A 5tuple FILTER *** */ struct cmd_5tuple_filter_result { cmdline_fixed_string_t filter; - uint8_t port_id; + portid_t port_id; cmdline_fixed_string_t ops; cmdline_fixed_string_t dst_ip; cmdline_ipaddr_t dst_ip_value; @@ -7829,7 +9340,7 @@ cmdline_parse_token_string_t cmd_5tuple_filter_filter = filter, "5tuple_filter"); cmdline_parse_token_num_t cmd_5tuple_filter_port_id = TOKEN_NUM_INITIALIZER(struct cmd_5tuple_filter_result, - port_id, UINT8); + port_id, UINT16); cmdline_parse_token_string_t cmd_5tuple_filter_ops = TOKEN_STRING_INITIALIZER(struct cmd_5tuple_filter_result, ops, "add#del"); @@ -7891,7 +9402,10 @@ cmdline_parse_token_num_t cmd_5tuple_filter_queue_id = cmdline_parse_inst_t cmd_5tuple_filter = { .f = cmd_5tuple_filter_parsed, .data = NULL, - .help_str = "add/del a 5tuple filter", + .help_str = "5tuple_filter add|del dst_ip " + "src_ip dst_port src_port " + "protocol mask tcp_flags " + "priority queue : Add/Del a 5tuple filter", .tokens = { (void *)&cmd_5tuple_filter_filter, (void *)&cmd_5tuple_filter_port_id, @@ -7922,7 +9436,7 @@ cmdline_parse_inst_t cmd_5tuple_filter = { struct cmd_flex_filter_result { cmdline_fixed_string_t filter; cmdline_fixed_string_t ops; - uint8_t port_id; + portid_t port_id; cmdline_fixed_string_t len; uint8_t len_value; cmdline_fixed_string_t bytes; @@ -8049,7 +9563,7 @@ cmdline_parse_token_string_t cmd_flex_filter_filter = filter, "flex_filter"); cmdline_parse_token_num_t cmd_flex_filter_port_id = TOKEN_NUM_INITIALIZER(struct cmd_flex_filter_result, - port_id, UINT8); + port_id, UINT16); cmdline_parse_token_string_t cmd_flex_filter_ops = TOKEN_STRING_INITIALIZER(struct cmd_flex_filter_result, ops, "add#del"); @@ -8086,7 +9600,9 @@ cmdline_parse_token_num_t cmd_flex_filter_queue_id = cmdline_parse_inst_t cmd_flex_filter = { .f = cmd_flex_filter_parsed, .data = NULL, - .help_str = "add/del a flex filter", + .help_str = "flex_filter add|del len bytes " + " mask priority queue : " + "Add/Del a flex filter", .tokens = { (void *)&cmd_flex_filter_filter, (void *)&cmd_flex_filter_port_id, @@ -8110,7 +9626,7 @@ cmdline_parse_inst_t cmd_flex_filter = { /* *** deal with ethertype filter *** */ struct cmd_ethertype_filter_result { cmdline_fixed_string_t filter; - uint8_t port_id; + portid_t port_id; cmdline_fixed_string_t ops; cmdline_fixed_string_t mac; struct ether_addr mac_addr; @@ -8126,7 +9642,7 @@ cmdline_parse_token_string_t cmd_ethertype_filter_filter = filter, "ethertype_filter"); cmdline_parse_token_num_t cmd_ethertype_filter_port_id = TOKEN_NUM_INITIALIZER(struct cmd_ethertype_filter_result, - port_id, UINT8); + port_id, UINT16); cmdline_parse_token_string_t cmd_ethertype_filter_ops = TOKEN_STRING_INITIALIZER(struct cmd_ethertype_filter_result, ops, "add#del"); @@ -8172,7 +9688,7 @@ cmd_ethertype_filter_parsed(void *parsed_result, memset(&filter, 0, sizeof(filter)); if (!strcmp(res->mac, "mac_addr")) { filter.flags |= RTE_ETHTYPE_FLAGS_MAC; - (void)rte_memcpy(&filter.mac_addr, &res->mac_addr, + rte_memcpy(&filter.mac_addr, &res->mac_addr, sizeof(struct ether_addr)); } if (!strcmp(res->drop, "drop")) @@ -8198,7 +9714,9 @@ cmd_ethertype_filter_parsed(void *parsed_result, cmdline_parse_inst_t cmd_ethertype_filter = { .f = cmd_ethertype_filter_parsed, .data = NULL, - .help_str = "add or delete an ethertype filter entry", + .help_str = "ethertype_filter add|del mac_addr|mac_ignr " + " ethertype drop|fw queue : " + "Add or delete an ethertype filter entry", .tokens = { (void *)&cmd_ethertype_filter_filter, (void *)&cmd_ethertype_filter_port_id, @@ -8217,7 +9735,7 @@ cmdline_parse_inst_t cmd_ethertype_filter = { /* *** deal with flow director filter *** */ struct cmd_flow_director_result { cmdline_fixed_string_t flow_director_filter; - uint8_t port_id; + portid_t port_id; cmdline_fixed_string_t mode; cmdline_fixed_string_t mode_value; cmdline_fixed_string_t ops; @@ -8233,11 +9751,11 @@ struct cmd_flow_director_result { uint16_t port_dst; cmdline_fixed_string_t verify_tag; uint32_t verify_tag_value; - cmdline_ipaddr_t tos; + cmdline_fixed_string_t tos; uint8_t tos_value; - cmdline_ipaddr_t proto; + cmdline_fixed_string_t proto; uint8_t proto_value; - cmdline_ipaddr_t ttl; + cmdline_fixed_string_t ttl; uint8_t ttl_value; cmdline_fixed_string_t vlan; uint16_t vlan_value; @@ -8323,6 +9841,10 @@ str2flowtype(char *string) if (!strcmp(flowtype_str[i].str, string)) return flowtype_str[i].type; } + + if (isdigit(string[0]) && atoi(string) > 0 && atoi(string) < 64) + return (uint16_t)atoi(string); + return RTE_ETH_FLOW_UNKNOWN; } @@ -8359,7 +9881,7 @@ do { \ #define IPV6_ADDR_TO_ARRAY(ip_addr, ip) \ do { \ if ((ip_addr).family == AF_INET6) \ - (void)rte_memcpy(&(ip), \ + rte_memcpy(&(ip), \ &((ip_addr).addr.ipv6), \ sizeof(struct in6_addr)); \ else { \ @@ -8419,6 +9941,7 @@ cmd_flow_director_filter_parsed(void *parsed_result, case RTE_ETH_FLOW_FRAG_IPV4: case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: entry.input.flow.ip4_flow.proto = res->proto_value; + /* fall-through */ case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: IPV4_ADDR_TO_UINT(res->ip_dst, @@ -8451,6 +9974,7 @@ cmd_flow_director_filter_parsed(void *parsed_result, case RTE_ETH_FLOW_FRAG_IPV6: case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: entry.input.flow.ipv6_flow.proto = res->proto_value; + /* fall-through */ case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: IPV6_ADDR_TO_ARRAY(res->ip_dst, @@ -8489,12 +10013,12 @@ cmd_flow_director_filter_parsed(void *parsed_result, } if (fdir_conf.mode == RTE_FDIR_MODE_PERFECT_MAC_VLAN) - (void)rte_memcpy(&entry.input.flow.mac_vlan_flow.mac_addr, + rte_memcpy(&entry.input.flow.mac_vlan_flow.mac_addr, &res->mac_addr, sizeof(struct ether_addr)); if (fdir_conf.mode == RTE_FDIR_MODE_PERFECT_TUNNEL) { - (void)rte_memcpy(&entry.input.flow.tunnel_flow.mac_addr, + rte_memcpy(&entry.input.flow.tunnel_flow.mac_addr, &res->mac_addr, sizeof(struct ether_addr)); entry.input.flow.tunnel_flow.tunnel_type = @@ -8503,7 +10027,7 @@ cmd_flow_director_filter_parsed(void *parsed_result, rte_cpu_to_be_32(res->tunnel_id_value); } - (void)rte_memcpy(entry.input.flow_ext.flexbytes, + rte_memcpy(entry.input.flow_ext.flexbytes, flexbytes, RTE_ETH_FDIR_MAX_FLEXLEN); @@ -8562,7 +10086,7 @@ cmdline_parse_token_string_t cmd_flow_director_filter = flow_director_filter, "flow_director_filter"); cmdline_parse_token_num_t cmd_flow_director_port_id = TOKEN_NUM_INITIALIZER(struct cmd_flow_director_result, - port_id, UINT8); + port_id, UINT16); cmdline_parse_token_string_t cmd_flow_director_ops = TOKEN_STRING_INITIALIZER(struct cmd_flow_director_result, ops, "add#del#update"); @@ -8686,7 +10210,14 @@ cmdline_parse_token_num_t cmd_flow_director_tunnel_id_value = cmdline_parse_inst_t cmd_add_del_ip_flow_director = { .f = cmd_flow_director_filter_parsed, .data = NULL, - .help_str = "add or delete an ip flow director entry on NIC", + .help_str = "flow_director_filter mode IP add|del|update flow" + " ipv4-other|ipv4-frag|ipv4-tcp|ipv4-udp|ipv4-sctp|" + "ipv6-other|ipv6-frag|ipv6-tcp|ipv6-udp|ipv6-sctp|" + "l2_payload src dst tos " + "proto ttl vlan " + "flexbytes drop|fw queue " + "fd_id : " + "Add or delete an ip flow director entry on NIC", .tokens = { (void *)&cmd_flow_director_filter, (void *)&cmd_flow_director_port_id, @@ -8722,7 +10253,8 @@ cmdline_parse_inst_t cmd_add_del_ip_flow_director = { cmdline_parse_inst_t cmd_add_del_udp_flow_director = { .f = cmd_flow_director_filter_parsed, .data = NULL, - .help_str = "add or delete an udp/tcp flow director entry on NIC", + .help_str = "flow_director_filter ... : Add or delete an udp/tcp flow " + "director entry on NIC", .tokens = { (void *)&cmd_flow_director_filter, (void *)&cmd_flow_director_port_id, @@ -8758,7 +10290,8 @@ cmdline_parse_inst_t cmd_add_del_udp_flow_director = { cmdline_parse_inst_t cmd_add_del_sctp_flow_director = { .f = cmd_flow_director_filter_parsed, .data = NULL, - .help_str = "add or delete a sctp flow director entry on NIC", + .help_str = "flow_director_filter ... : Add or delete a sctp flow " + "director entry on NIC", .tokens = { (void *)&cmd_flow_director_filter, (void *)&cmd_flow_director_port_id, @@ -8769,7 +10302,7 @@ cmdline_parse_inst_t cmd_add_del_sctp_flow_director = { (void *)&cmd_flow_director_flow_type, (void *)&cmd_flow_director_src, (void *)&cmd_flow_director_ip_src, - (void *)&cmd_flow_director_port_dst, + (void *)&cmd_flow_director_port_src, (void *)&cmd_flow_director_dst, (void *)&cmd_flow_director_ip_dst, (void *)&cmd_flow_director_port_dst, @@ -8796,7 +10329,8 @@ cmdline_parse_inst_t cmd_add_del_sctp_flow_director = { cmdline_parse_inst_t cmd_add_del_l2_flow_director = { .f = cmd_flow_director_filter_parsed, .data = NULL, - .help_str = "add or delete a L2 flow director entry on NIC", + .help_str = "flow_director_filter ... : Add or delete a L2 flow " + "director entry on NIC", .tokens = { (void *)&cmd_flow_director_filter, (void *)&cmd_flow_director_port_id, @@ -8822,7 +10356,8 @@ cmdline_parse_inst_t cmd_add_del_l2_flow_director = { cmdline_parse_inst_t cmd_add_del_mac_vlan_flow_director = { .f = cmd_flow_director_filter_parsed, .data = NULL, - .help_str = "add or delete a MAC VLAN flow director entry on NIC", + .help_str = "flow_director_filter ... : Add or delete a MAC VLAN flow " + "director entry on NIC", .tokens = { (void *)&cmd_flow_director_filter, (void *)&cmd_flow_director_port_id, @@ -8847,7 +10382,8 @@ cmdline_parse_inst_t cmd_add_del_mac_vlan_flow_director = { cmdline_parse_inst_t cmd_add_del_tunnel_flow_director = { .f = cmd_flow_director_filter_parsed, .data = NULL, - .help_str = "add or delete a tunnel flow director entry on NIC", + .help_str = "flow_director_filter ... : Add or delete a tunnel flow " + "director entry on NIC", .tokens = { (void *)&cmd_flow_director_filter, (void *)&cmd_flow_director_port_id, @@ -8875,7 +10411,7 @@ cmdline_parse_inst_t cmd_add_del_tunnel_flow_director = { struct cmd_flush_flow_director_result { cmdline_fixed_string_t flush_flow_director; - uint8_t port_id; + portid_t port_id; }; cmdline_parse_token_string_t cmd_flush_flow_director_flush = @@ -8883,7 +10419,7 @@ cmdline_parse_token_string_t cmd_flush_flow_director_flush = flush_flow_director, "flush_flow_director"); cmdline_parse_token_num_t cmd_flush_flow_director_port_id = TOKEN_NUM_INITIALIZER(struct cmd_flush_flow_director_result, - port_id, UINT8); + port_id, UINT16); static void cmd_flush_flow_director_parsed(void *parsed_result, @@ -8910,7 +10446,8 @@ cmd_flush_flow_director_parsed(void *parsed_result, cmdline_parse_inst_t cmd_flush_flow_director = { .f = cmd_flush_flow_director_parsed, .data = NULL, - .help_str = "flush all flow director entries of a device on NIC", + .help_str = "flush_flow_director : " + "Flush all flow director entries of a device on NIC", .tokens = { (void *)&cmd_flush_flow_director_flush, (void *)&cmd_flush_flow_director_port_id, @@ -8921,7 +10458,7 @@ cmdline_parse_inst_t cmd_flush_flow_director = { /* *** deal with flow director mask *** */ struct cmd_flow_director_mask_result { cmdline_fixed_string_t flow_director_mask; - uint8_t port_id; + portid_t port_id; cmdline_fixed_string_t mode; cmdline_fixed_string_t mode_value; cmdline_fixed_string_t vlan; @@ -9005,7 +10542,7 @@ cmdline_parse_token_string_t cmd_flow_director_mask = flow_director_mask, "flow_director_mask"); cmdline_parse_token_num_t cmd_flow_director_mask_port_id = TOKEN_NUM_INITIALIZER(struct cmd_flow_director_mask_result, - port_id, UINT8); + port_id, UINT16); cmdline_parse_token_string_t cmd_flow_director_mask_vlan = TOKEN_STRING_INITIALIZER(struct cmd_flow_director_mask_result, vlan, "vlan"); @@ -9071,7 +10608,8 @@ cmdline_parse_token_num_t cmd_flow_director_mask_tunnel_id_value = cmdline_parse_inst_t cmd_set_flow_director_ip_mask = { .f = cmd_flow_director_mask_parsed, .data = NULL, - .help_str = "set IP mode flow director's mask on NIC", + .help_str = "flow_director_mask ... : " + "Set IP mode flow director's mask on NIC", .tokens = { (void *)&cmd_flow_director_mask, (void *)&cmd_flow_director_mask_port_id, @@ -9094,7 +10632,8 @@ cmdline_parse_inst_t cmd_set_flow_director_ip_mask = { cmdline_parse_inst_t cmd_set_flow_director_mac_vlan_mask = { .f = cmd_flow_director_mask_parsed, .data = NULL, - .help_str = "set MAC VLAN mode flow director's mask on NIC", + .help_str = "flow_director_mask ... : Set MAC VLAN mode " + "flow director's mask on NIC", .tokens = { (void *)&cmd_flow_director_mask, (void *)&cmd_flow_director_mask_port_id, @@ -9109,7 +10648,8 @@ cmdline_parse_inst_t cmd_set_flow_director_mac_vlan_mask = { cmdline_parse_inst_t cmd_set_flow_director_tunnel_mask = { .f = cmd_flow_director_mask_parsed, .data = NULL, - .help_str = "set tunnel mode flow director's mask on NIC", + .help_str = "flow_director_mask ... : Set tunnel mode " + "flow director's mask on NIC", .tokens = { (void *)&cmd_flow_director_mask, (void *)&cmd_flow_director_mask_port_id, @@ -9130,7 +10670,7 @@ cmdline_parse_inst_t cmd_set_flow_director_tunnel_mask = { /* *** deal with flow director mask on flexible payload *** */ struct cmd_flow_director_flex_mask_result { cmdline_fixed_string_t flow_director_flexmask; - uint8_t port_id; + portid_t port_id; cmdline_fixed_string_t flow; cmdline_fixed_string_t flow_type; cmdline_fixed_string_t mask; @@ -9185,7 +10725,7 @@ cmd_flow_director_flex_mask_parsed(void *parsed_result, memset(&port->dev_conf.fdir_conf.flex_conf.flex_mask[i], 0, sizeof(struct rte_eth_fdir_flex_mask)); port->dev_conf.fdir_conf.flex_conf.nb_flexmasks = 1; - (void)rte_memcpy(&port->dev_conf.fdir_conf.flex_conf.flex_mask[0], + rte_memcpy(&port->dev_conf.fdir_conf.flex_conf.flex_mask[0], &flex_mask, sizeof(struct rte_eth_fdir_flex_mask)); cmd_reconfig_device_queue(res->port_id, 1, 1); @@ -9222,7 +10762,7 @@ cmdline_parse_token_string_t cmd_flow_director_flexmask = "flow_director_flex_mask"); cmdline_parse_token_num_t cmd_flow_director_flexmask_port_id = TOKEN_NUM_INITIALIZER(struct cmd_flow_director_flex_mask_result, - port_id, UINT8); + port_id, UINT16); cmdline_parse_token_string_t cmd_flow_director_flexmask_flow = TOKEN_STRING_INITIALIZER(struct cmd_flow_director_flex_mask_result, flow, "flow"); @@ -9237,7 +10777,8 @@ cmdline_parse_token_string_t cmd_flow_director_flexmask_mask = cmdline_parse_inst_t cmd_set_flow_director_flex_mask = { .f = cmd_flow_director_flex_mask_parsed, .data = NULL, - .help_str = "set flow director's flex mask on NIC", + .help_str = "flow_director_flex_mask ... : " + "Set flow director's flex mask on NIC", .tokens = { (void *)&cmd_flow_director_flexmask, (void *)&cmd_flow_director_flexmask_port_id, @@ -9251,7 +10792,7 @@ cmdline_parse_inst_t cmd_set_flow_director_flex_mask = { /* *** deal with flow director flexible payload configuration *** */ struct cmd_flow_director_flexpayload_result { cmdline_fixed_string_t flow_director_flexpayload; - uint8_t port_id; + portid_t port_id; cmdline_fixed_string_t payload_layer; cmdline_fixed_string_t payload_cfg; }; @@ -9344,7 +10885,7 @@ cmdline_parse_token_string_t cmd_flow_director_flexpayload = "flow_director_flex_payload"); cmdline_parse_token_num_t cmd_flow_director_flexpayload_port_id = TOKEN_NUM_INITIALIZER(struct cmd_flow_director_flexpayload_result, - port_id, UINT8); + port_id, UINT16); cmdline_parse_token_string_t cmd_flow_director_flexpayload_payload_layer = TOKEN_STRING_INITIALIZER(struct cmd_flow_director_flexpayload_result, payload_layer, "raw#l2#l3#l4"); @@ -9355,7 +10896,8 @@ cmdline_parse_token_string_t cmd_flow_director_flexpayload_payload_cfg = cmdline_parse_inst_t cmd_set_flow_director_flex_payload = { .f = cmd_flow_director_flxpld_parsed, .data = NULL, - .help_str = "set flow director's flex payload on NIC", + .help_str = "flow_director_flexpayload ... : " + "Set flow director's flex payload on NIC", .tokens = { (void *)&cmd_flow_director_flexpayload, (void *)&cmd_flow_director_flexpayload_port_id, @@ -9365,11 +10907,14 @@ cmdline_parse_inst_t cmd_set_flow_director_flex_payload = { }, }; +/* Generic flow interface command. */ +extern cmdline_parse_inst_t cmd_flow; + /* *** Classification Filters Control *** */ /* *** Get symmetric hash enable per port *** */ struct cmd_get_sym_hash_ena_per_port_result { cmdline_fixed_string_t get_sym_hash_ena_per_port; - uint8_t port_id; + portid_t port_id; }; static void @@ -9408,12 +10953,12 @@ cmdline_parse_token_string_t cmd_get_sym_hash_ena_per_port_all = get_sym_hash_ena_per_port, "get_sym_hash_ena_per_port"); cmdline_parse_token_num_t cmd_get_sym_hash_ena_per_port_port_id = TOKEN_NUM_INITIALIZER(struct cmd_get_sym_hash_ena_per_port_result, - port_id, UINT8); + port_id, UINT16); cmdline_parse_inst_t cmd_get_sym_hash_ena_per_port = { .f = cmd_get_sym_hash_per_port_parsed, .data = NULL, - .help_str = "get_sym_hash_ena_per_port port_id", + .help_str = "get_sym_hash_ena_per_port ", .tokens = { (void *)&cmd_get_sym_hash_ena_per_port_all, (void *)&cmd_get_sym_hash_ena_per_port_port_id, @@ -9425,7 +10970,7 @@ cmdline_parse_inst_t cmd_get_sym_hash_ena_per_port = { struct cmd_set_sym_hash_ena_per_port_result { cmdline_fixed_string_t set_sym_hash_ena_per_port; cmdline_fixed_string_t enable; - uint8_t port_id; + portid_t port_id; }; static void @@ -9464,7 +11009,7 @@ cmdline_parse_token_string_t cmd_set_sym_hash_ena_per_port_all = set_sym_hash_ena_per_port, "set_sym_hash_ena_per_port"); cmdline_parse_token_num_t cmd_set_sym_hash_ena_per_port_port_id = TOKEN_NUM_INITIALIZER(struct cmd_set_sym_hash_ena_per_port_result, - port_id, UINT8); + port_id, UINT16); cmdline_parse_token_string_t cmd_set_sym_hash_ena_per_port_enable = TOKEN_STRING_INITIALIZER(struct cmd_set_sym_hash_ena_per_port_result, enable, "enable#disable"); @@ -9472,7 +11017,7 @@ cmdline_parse_token_string_t cmd_set_sym_hash_ena_per_port_enable = cmdline_parse_inst_t cmd_set_sym_hash_ena_per_port = { .f = cmd_set_sym_hash_per_port_parsed, .data = NULL, - .help_str = "set_sym_hash_ena_per_port port_id enable|disable", + .help_str = "set_sym_hash_ena_per_port enable|disable", .tokens = { (void *)&cmd_set_sym_hash_ena_per_port_all, (void *)&cmd_set_sym_hash_ena_per_port_port_id, @@ -9484,7 +11029,7 @@ cmdline_parse_inst_t cmd_set_sym_hash_ena_per_port = { /* Get global config of hash function */ struct cmd_get_hash_global_config_result { cmdline_fixed_string_t get_hash_global_config; - uint8_t port_id; + portid_t port_id; }; static char * @@ -9585,12 +11130,12 @@ cmdline_parse_token_string_t cmd_get_hash_global_config_all = get_hash_global_config, "get_hash_global_config"); cmdline_parse_token_num_t cmd_get_hash_global_config_port_id = TOKEN_NUM_INITIALIZER(struct cmd_get_hash_global_config_result, - port_id, UINT8); + port_id, UINT16); cmdline_parse_inst_t cmd_get_hash_global_config = { .f = cmd_get_hash_global_config_parsed, .data = NULL, - .help_str = "get_hash_global_config port_id", + .help_str = "get_hash_global_config ", .tokens = { (void *)&cmd_get_hash_global_config_all, (void *)&cmd_get_hash_global_config_port_id, @@ -9601,7 +11146,7 @@ cmdline_parse_inst_t cmd_get_hash_global_config = { /* Set global config of hash function */ struct cmd_set_hash_global_config_result { cmdline_fixed_string_t set_hash_global_config; - uint8_t port_id; + portid_t port_id; cmdline_fixed_string_t hash_func; cmdline_fixed_string_t flow_type; cmdline_fixed_string_t enable; @@ -9657,7 +11202,7 @@ cmdline_parse_token_string_t cmd_set_hash_global_config_all = set_hash_global_config, "set_hash_global_config"); cmdline_parse_token_num_t cmd_set_hash_global_config_port_id = TOKEN_NUM_INITIALIZER(struct cmd_set_hash_global_config_result, - port_id, UINT8); + port_id, UINT16); cmdline_parse_token_string_t cmd_set_hash_global_config_hash_func = TOKEN_STRING_INITIALIZER(struct cmd_set_hash_global_config_result, hash_func, "toeplitz#simple_xor#default"); @@ -9673,11 +11218,11 @@ cmdline_parse_token_string_t cmd_set_hash_global_config_enable = cmdline_parse_inst_t cmd_set_hash_global_config = { .f = cmd_set_hash_global_config_parsed, .data = NULL, - .help_str = "set_hash_global_config port_id " + .help_str = "set_hash_global_config " "toeplitz|simple_xor|default " - "ipv4|ipv4-frag|ipv4-tcp|ipv4-udp|ipv4-sctp|ipv4-other|ipv6|" - "ipv6-frag|ipv6-tcp|ipv6-udp|ipv6-sctp|ipv6-other|l2_payload " - "enable|disable", + "ipv4|ipv4-frag|ipv4-tcp|ipv4-udp|ipv4-sctp|ipv4-other|" + "ipv6|ipv6-frag|ipv6-tcp|ipv6-udp|ipv6-sctp|ipv6-other|" + "l2_payload enable|disable", .tokens = { (void *)&cmd_set_hash_global_config_all, (void *)&cmd_set_hash_global_config_port_id, @@ -9691,7 +11236,7 @@ cmdline_parse_inst_t cmd_set_hash_global_config = { /* Set hash input set */ struct cmd_set_hash_input_set_result { cmdline_fixed_string_t set_hash_input_set; - uint8_t port_id; + portid_t port_id; cmdline_fixed_string_t flow_type; cmdline_fixed_string_t inset_field; cmdline_fixed_string_t select; @@ -9773,12 +11318,10 @@ cmdline_parse_token_string_t cmd_set_hash_input_set_cmd = set_hash_input_set, "set_hash_input_set"); cmdline_parse_token_num_t cmd_set_hash_input_set_port_id = TOKEN_NUM_INITIALIZER(struct cmd_set_hash_input_set_result, - port_id, UINT8); + port_id, UINT16); cmdline_parse_token_string_t cmd_set_hash_input_set_flow_type = TOKEN_STRING_INITIALIZER(struct cmd_set_hash_input_set_result, - flow_type, - "ipv4-frag#ipv4-tcp#ipv4-udp#ipv4-sctp#ipv4-other#" - "ipv6-frag#ipv6-tcp#ipv6-udp#ipv6-sctp#ipv6-other#l2_payload"); + flow_type, NULL); cmdline_parse_token_string_t cmd_set_hash_input_set_field = TOKEN_STRING_INITIALIZER(struct cmd_set_hash_input_set_result, inset_field, @@ -9797,7 +11340,7 @@ cmdline_parse_inst_t cmd_set_hash_input_set = { .data = NULL, .help_str = "set_hash_input_set " "ipv4-frag|ipv4-tcp|ipv4-udp|ipv4-sctp|ipv4-other|" - "ipv6-frag|ipv6-tcp|ipv6-udp|ipv6-sctp|ipv6-other|l2_payload " + "ipv6-frag|ipv6-tcp|ipv6-udp|ipv6-sctp|ipv6-other|l2_payload| " "ovlan|ivlan|src-ipv4|dst-ipv4|src-ipv6|dst-ipv6|ipv4-tos|ipv4-proto|" "ipv6-tc|ipv6-next-header|udp-src-port|udp-dst-port|tcp-src-port|" "tcp-dst-port|sctp-src-port|sctp-dst-port|sctp-veri-tag|udp-key|" @@ -9816,7 +11359,7 @@ cmdline_parse_inst_t cmd_set_hash_input_set = { /* Set flow director input set */ struct cmd_set_fdir_input_set_result { cmdline_fixed_string_t set_fdir_input_set; - uint8_t port_id; + portid_t port_id; cmdline_fixed_string_t flow_type; cmdline_fixed_string_t inset_field; cmdline_fixed_string_t select; @@ -9848,7 +11391,7 @@ cmdline_parse_token_string_t cmd_set_fdir_input_set_cmd = set_fdir_input_set, "set_fdir_input_set"); cmdline_parse_token_num_t cmd_set_fdir_input_set_port_id = TOKEN_NUM_INITIALIZER(struct cmd_set_fdir_input_set_result, - port_id, UINT8); + port_id, UINT16); cmdline_parse_token_string_t cmd_set_fdir_input_set_flow_type = TOKEN_STRING_INITIALIZER(struct cmd_set_fdir_input_set_result, flow_type, @@ -9891,7 +11434,7 @@ cmdline_parse_inst_t cmd_set_fdir_input_set = { struct cmd_mcast_addr_result { cmdline_fixed_string_t mcast_addr_cmd; cmdline_fixed_string_t what; - uint8_t port_num; + uint16_t port_num; struct ether_addr mc_addr; }; @@ -9921,14 +11464,15 @@ cmdline_parse_token_string_t cmd_mcast_addr_what = TOKEN_STRING_INITIALIZER(struct cmd_mcast_addr_result, what, "add#remove"); cmdline_parse_token_num_t cmd_mcast_addr_portnum = - TOKEN_NUM_INITIALIZER(struct cmd_mcast_addr_result, port_num, UINT8); + TOKEN_NUM_INITIALIZER(struct cmd_mcast_addr_result, port_num, UINT16); cmdline_parse_token_etheraddr_t cmd_mcast_addr_addr = TOKEN_ETHERADDR_INITIALIZER(struct cmd_mac_addr_result, address); cmdline_parse_inst_t cmd_mcast_addr = { .f = cmd_mcast_addr_parsed, .data = (void *)0, - .help_str = "mcast_addr add|remove X : add/remove multicast MAC address on port X", + .help_str = "mcast_addr add|remove : " + "Add/Remove multicast MAC address on port_id", .tokens = { (void *)&cmd_mcast_addr_cmd, (void *)&cmd_mcast_addr_what, @@ -10020,7 +11564,7 @@ cmd_config_l2_tunnel_eth_type_all_parsed entry.l2_tunnel_type = str2fdir_l2_tunnel_type(res->l2_tunnel_type); entry.ether_type = res->eth_type_val; - FOREACH_PORT(pid, ports) { + RTE_ETH_FOREACH_DEV(pid) { rte_eth_dev_l2_tunnel_eth_type_conf(pid, &entry); } } @@ -10028,7 +11572,7 @@ cmd_config_l2_tunnel_eth_type_all_parsed cmdline_parse_inst_t cmd_config_l2_tunnel_eth_type_all = { .f = cmd_config_l2_tunnel_eth_type_all_parsed, .data = NULL, - .help_str = "port config all l2-tunnel ether-type", + .help_str = "port config all l2-tunnel E-tag ether-type ", .tokens = { (void *)&cmd_config_l2_tunnel_eth_type_port, (void *)&cmd_config_l2_tunnel_eth_type_config, @@ -10064,7 +11608,7 @@ cmd_config_l2_tunnel_eth_type_specific_parsed( cmdline_parse_inst_t cmd_config_l2_tunnel_eth_type_specific = { .f = cmd_config_l2_tunnel_eth_type_specific_parsed, .data = NULL, - .help_str = "port config l2-tunnel ether-type", + .help_str = "port config l2-tunnel E-tag ether-type ", .tokens = { (void *)&cmd_config_l2_tunnel_eth_type_port, (void *)&cmd_config_l2_tunnel_eth_type_config, @@ -10136,7 +11680,7 @@ cmd_config_l2_tunnel_en_dis_all_parsed( else en = 0; - FOREACH_PORT(pid, ports) { + RTE_ETH_FOREACH_DEV(pid) { rte_eth_dev_l2_tunnel_offload_set(pid, &entry, ETH_L2_TUNNEL_ENABLE_MASK, @@ -10147,7 +11691,7 @@ cmd_config_l2_tunnel_en_dis_all_parsed( cmdline_parse_inst_t cmd_config_l2_tunnel_en_dis_all = { .f = cmd_config_l2_tunnel_en_dis_all_parsed, .data = NULL, - .help_str = "port config all l2-tunnel enable/disable", + .help_str = "port config all l2-tunnel E-tag enable|disable", .tokens = { (void *)&cmd_config_l2_tunnel_en_dis_port, (void *)&cmd_config_l2_tunnel_en_dis_config, @@ -10190,7 +11734,7 @@ cmd_config_l2_tunnel_en_dis_specific_parsed( cmdline_parse_inst_t cmd_config_l2_tunnel_en_dis_specific = { .f = cmd_config_l2_tunnel_en_dis_specific_parsed, .data = NULL, - .help_str = "port config l2-tunnel enable/disable", + .help_str = "port config l2-tunnel E-tag enable|disable", .tokens = { (void *)&cmd_config_l2_tunnel_en_dis_port, (void *)&cmd_config_l2_tunnel_en_dis_config, @@ -10224,7 +11768,7 @@ struct cmd_config_e_tag_result { cmdline_fixed_string_t dst_pool; uint8_t dst_pool_val; cmdline_fixed_string_t port; - uint8_t port_id; + portid_t port_id; cmdline_fixed_string_t vf; uint8_t vf_id; }; @@ -10305,7 +11849,7 @@ cmdline_parse_token_string_t cmd_config_e_tag_port = cmdline_parse_token_num_t cmd_config_e_tag_port_id = TOKEN_NUM_INITIALIZER (struct cmd_config_e_tag_result, - port_id, UINT8); + port_id, UINT16); cmdline_parse_token_string_t cmd_config_e_tag_vf = TOKEN_STRING_INITIALIZER (struct cmd_config_e_tag_result, @@ -10363,7 +11907,7 @@ cmd_config_e_tag_insertion_dis_parsed( cmdline_parse_inst_t cmd_config_e_tag_insertion_en = { .f = cmd_config_e_tag_insertion_en_parsed, .data = NULL, - .help_str = "E-tag insertion enable", + .help_str = "E-tag ... : E-tag insertion enable", .tokens = { (void *)&cmd_config_e_tag_e_tag, (void *)&cmd_config_e_tag_set, @@ -10382,7 +11926,7 @@ cmdline_parse_inst_t cmd_config_e_tag_insertion_en = { cmdline_parse_inst_t cmd_config_e_tag_insertion_dis = { .f = cmd_config_e_tag_insertion_dis_parsed, .data = NULL, - .help_str = "E-tag insertion disable", + .help_str = "E-tag ... : E-tag insertion disable", .tokens = { (void *)&cmd_config_e_tag_e_tag, (void *)&cmd_config_e_tag_set, @@ -10429,7 +11973,7 @@ cmd_config_e_tag_stripping_parsed( cmdline_parse_inst_t cmd_config_e_tag_stripping_en_dis = { .f = cmd_config_e_tag_stripping_parsed, .data = NULL, - .help_str = "E-tag stripping enable/disable", + .help_str = "E-tag ... : E-tag stripping enable/disable", .tokens = { (void *)&cmd_config_e_tag_e_tag, (void *)&cmd_config_e_tag_set, @@ -10473,7 +12017,7 @@ cmd_config_e_tag_forwarding_parsed( cmdline_parse_inst_t cmd_config_e_tag_forwarding_en_dis = { .f = cmd_config_e_tag_forwarding_parsed, .data = NULL, - .help_str = "E-tag forwarding enable/disable", + .help_str = "E-tag ... : E-tag forwarding enable/disable", .tokens = { (void *)&cmd_config_e_tag_e_tag, (void *)&cmd_config_e_tag_set, @@ -10528,7 +12072,7 @@ cmd_config_e_tag_filter_add_parsed( cmdline_parse_inst_t cmd_config_e_tag_filter_add = { .f = cmd_config_e_tag_filter_add_parsed, .data = NULL, - .help_str = "E-tag filter add", + .help_str = "E-tag ... : E-tag filter add", .tokens = { (void *)&cmd_config_e_tag_e_tag, (void *)&cmd_config_e_tag_set, @@ -10585,7 +12129,7 @@ cmd_config_e_tag_filter_del_parsed( cmdline_parse_inst_t cmd_config_e_tag_filter_del = { .f = cmd_config_e_tag_filter_del_parsed, .data = NULL, - .help_str = "E-tag filter delete", + .help_str = "E-tag ... : E-tag filter delete", .tokens = { (void *)&cmd_config_e_tag_e_tag, (void *)&cmd_config_e_tag_set, @@ -10599,6 +12143,3402 @@ cmdline_parse_inst_t cmd_config_e_tag_filter_del = { }, }; +/* vf vlan anti spoof configuration */ + +/* Common result structure for vf vlan anti spoof */ +struct cmd_vf_vlan_anti_spoof_result { + cmdline_fixed_string_t set; + cmdline_fixed_string_t vf; + cmdline_fixed_string_t vlan; + cmdline_fixed_string_t antispoof; + portid_t port_id; + uint32_t vf_id; + cmdline_fixed_string_t on_off; +}; + +/* Common CLI fields for vf vlan anti spoof enable disable */ +cmdline_parse_token_string_t cmd_vf_vlan_anti_spoof_set = + TOKEN_STRING_INITIALIZER + (struct cmd_vf_vlan_anti_spoof_result, + set, "set"); +cmdline_parse_token_string_t cmd_vf_vlan_anti_spoof_vf = + TOKEN_STRING_INITIALIZER + (struct cmd_vf_vlan_anti_spoof_result, + vf, "vf"); +cmdline_parse_token_string_t cmd_vf_vlan_anti_spoof_vlan = + TOKEN_STRING_INITIALIZER + (struct cmd_vf_vlan_anti_spoof_result, + vlan, "vlan"); +cmdline_parse_token_string_t cmd_vf_vlan_anti_spoof_antispoof = + TOKEN_STRING_INITIALIZER + (struct cmd_vf_vlan_anti_spoof_result, + antispoof, "antispoof"); +cmdline_parse_token_num_t cmd_vf_vlan_anti_spoof_port_id = + TOKEN_NUM_INITIALIZER + (struct cmd_vf_vlan_anti_spoof_result, + port_id, UINT16); +cmdline_parse_token_num_t cmd_vf_vlan_anti_spoof_vf_id = + TOKEN_NUM_INITIALIZER + (struct cmd_vf_vlan_anti_spoof_result, + vf_id, UINT32); +cmdline_parse_token_string_t cmd_vf_vlan_anti_spoof_on_off = + TOKEN_STRING_INITIALIZER + (struct cmd_vf_vlan_anti_spoof_result, + on_off, "on#off"); + +static void +cmd_set_vf_vlan_anti_spoof_parsed( + void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_vf_vlan_anti_spoof_result *res = parsed_result; + int ret = -ENOTSUP; + + __rte_unused int is_on = (strcmp(res->on_off, "on") == 0) ? 1 : 0; + + if (port_id_is_invalid(res->port_id, ENABLED_WARN)) + return; + +#ifdef RTE_LIBRTE_IXGBE_PMD + if (ret == -ENOTSUP) + ret = rte_pmd_ixgbe_set_vf_vlan_anti_spoof(res->port_id, + res->vf_id, is_on); +#endif +#ifdef RTE_LIBRTE_I40E_PMD + if (ret == -ENOTSUP) + ret = rte_pmd_i40e_set_vf_vlan_anti_spoof(res->port_id, + res->vf_id, is_on); +#endif +#ifdef RTE_LIBRTE_BNXT_PMD + if (ret == -ENOTSUP) + ret = rte_pmd_bnxt_set_vf_vlan_anti_spoof(res->port_id, + res->vf_id, is_on); +#endif + + switch (ret) { + case 0: + break; + case -EINVAL: + printf("invalid vf_id %d\n", res->vf_id); + break; + case -ENODEV: + printf("invalid port_id %d\n", res->port_id); + break; + case -ENOTSUP: + printf("function not implemented\n"); + break; + default: + printf("programming error: (%s)\n", strerror(-ret)); + } +} + +cmdline_parse_inst_t cmd_set_vf_vlan_anti_spoof = { + .f = cmd_set_vf_vlan_anti_spoof_parsed, + .data = NULL, + .help_str = "set vf vlan antispoof on|off", + .tokens = { + (void *)&cmd_vf_vlan_anti_spoof_set, + (void *)&cmd_vf_vlan_anti_spoof_vf, + (void *)&cmd_vf_vlan_anti_spoof_vlan, + (void *)&cmd_vf_vlan_anti_spoof_antispoof, + (void *)&cmd_vf_vlan_anti_spoof_port_id, + (void *)&cmd_vf_vlan_anti_spoof_vf_id, + (void *)&cmd_vf_vlan_anti_spoof_on_off, + NULL, + }, +}; + +/* vf mac anti spoof configuration */ + +/* Common result structure for vf mac anti spoof */ +struct cmd_vf_mac_anti_spoof_result { + cmdline_fixed_string_t set; + cmdline_fixed_string_t vf; + cmdline_fixed_string_t mac; + cmdline_fixed_string_t antispoof; + portid_t port_id; + uint32_t vf_id; + cmdline_fixed_string_t on_off; +}; + +/* Common CLI fields for vf mac anti spoof enable disable */ +cmdline_parse_token_string_t cmd_vf_mac_anti_spoof_set = + TOKEN_STRING_INITIALIZER + (struct cmd_vf_mac_anti_spoof_result, + set, "set"); +cmdline_parse_token_string_t cmd_vf_mac_anti_spoof_vf = + TOKEN_STRING_INITIALIZER + (struct cmd_vf_mac_anti_spoof_result, + vf, "vf"); +cmdline_parse_token_string_t cmd_vf_mac_anti_spoof_mac = + TOKEN_STRING_INITIALIZER + (struct cmd_vf_mac_anti_spoof_result, + mac, "mac"); +cmdline_parse_token_string_t cmd_vf_mac_anti_spoof_antispoof = + TOKEN_STRING_INITIALIZER + (struct cmd_vf_mac_anti_spoof_result, + antispoof, "antispoof"); +cmdline_parse_token_num_t cmd_vf_mac_anti_spoof_port_id = + TOKEN_NUM_INITIALIZER + (struct cmd_vf_mac_anti_spoof_result, + port_id, UINT16); +cmdline_parse_token_num_t cmd_vf_mac_anti_spoof_vf_id = + TOKEN_NUM_INITIALIZER + (struct cmd_vf_mac_anti_spoof_result, + vf_id, UINT32); +cmdline_parse_token_string_t cmd_vf_mac_anti_spoof_on_off = + TOKEN_STRING_INITIALIZER + (struct cmd_vf_mac_anti_spoof_result, + on_off, "on#off"); + +static void +cmd_set_vf_mac_anti_spoof_parsed( + void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_vf_mac_anti_spoof_result *res = parsed_result; + int ret = -ENOTSUP; + + __rte_unused int is_on = (strcmp(res->on_off, "on") == 0) ? 1 : 0; + + if (port_id_is_invalid(res->port_id, ENABLED_WARN)) + return; + +#ifdef RTE_LIBRTE_IXGBE_PMD + if (ret == -ENOTSUP) + ret = rte_pmd_ixgbe_set_vf_mac_anti_spoof(res->port_id, + res->vf_id, is_on); +#endif +#ifdef RTE_LIBRTE_I40E_PMD + if (ret == -ENOTSUP) + ret = rte_pmd_i40e_set_vf_mac_anti_spoof(res->port_id, + res->vf_id, is_on); +#endif +#ifdef RTE_LIBRTE_BNXT_PMD + if (ret == -ENOTSUP) + ret = rte_pmd_bnxt_set_vf_mac_anti_spoof(res->port_id, + res->vf_id, is_on); +#endif + + switch (ret) { + case 0: + break; + case -EINVAL: + printf("invalid vf_id %d or is_on %d\n", res->vf_id, is_on); + break; + case -ENODEV: + printf("invalid port_id %d\n", res->port_id); + break; + case -ENOTSUP: + printf("function not implemented\n"); + break; + default: + printf("programming error: (%s)\n", strerror(-ret)); + } +} + +cmdline_parse_inst_t cmd_set_vf_mac_anti_spoof = { + .f = cmd_set_vf_mac_anti_spoof_parsed, + .data = NULL, + .help_str = "set vf mac antispoof on|off", + .tokens = { + (void *)&cmd_vf_mac_anti_spoof_set, + (void *)&cmd_vf_mac_anti_spoof_vf, + (void *)&cmd_vf_mac_anti_spoof_mac, + (void *)&cmd_vf_mac_anti_spoof_antispoof, + (void *)&cmd_vf_mac_anti_spoof_port_id, + (void *)&cmd_vf_mac_anti_spoof_vf_id, + (void *)&cmd_vf_mac_anti_spoof_on_off, + NULL, + }, +}; + +/* vf vlan strip queue configuration */ + +/* Common result structure for vf mac anti spoof */ +struct cmd_vf_vlan_stripq_result { + cmdline_fixed_string_t set; + cmdline_fixed_string_t vf; + cmdline_fixed_string_t vlan; + cmdline_fixed_string_t stripq; + portid_t port_id; + uint16_t vf_id; + cmdline_fixed_string_t on_off; +}; + +/* Common CLI fields for vf vlan strip enable disable */ +cmdline_parse_token_string_t cmd_vf_vlan_stripq_set = + TOKEN_STRING_INITIALIZER + (struct cmd_vf_vlan_stripq_result, + set, "set"); +cmdline_parse_token_string_t cmd_vf_vlan_stripq_vf = + TOKEN_STRING_INITIALIZER + (struct cmd_vf_vlan_stripq_result, + vf, "vf"); +cmdline_parse_token_string_t cmd_vf_vlan_stripq_vlan = + TOKEN_STRING_INITIALIZER + (struct cmd_vf_vlan_stripq_result, + vlan, "vlan"); +cmdline_parse_token_string_t cmd_vf_vlan_stripq_stripq = + TOKEN_STRING_INITIALIZER + (struct cmd_vf_vlan_stripq_result, + stripq, "stripq"); +cmdline_parse_token_num_t cmd_vf_vlan_stripq_port_id = + TOKEN_NUM_INITIALIZER + (struct cmd_vf_vlan_stripq_result, + port_id, UINT16); +cmdline_parse_token_num_t cmd_vf_vlan_stripq_vf_id = + TOKEN_NUM_INITIALIZER + (struct cmd_vf_vlan_stripq_result, + vf_id, UINT16); +cmdline_parse_token_string_t cmd_vf_vlan_stripq_on_off = + TOKEN_STRING_INITIALIZER + (struct cmd_vf_vlan_stripq_result, + on_off, "on#off"); + +static void +cmd_set_vf_vlan_stripq_parsed( + void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_vf_vlan_stripq_result *res = parsed_result; + int ret = -ENOTSUP; + + __rte_unused int is_on = (strcmp(res->on_off, "on") == 0) ? 1 : 0; + + if (port_id_is_invalid(res->port_id, ENABLED_WARN)) + return; + +#ifdef RTE_LIBRTE_IXGBE_PMD + if (ret == -ENOTSUP) + ret = rte_pmd_ixgbe_set_vf_vlan_stripq(res->port_id, + res->vf_id, is_on); +#endif +#ifdef RTE_LIBRTE_I40E_PMD + if (ret == -ENOTSUP) + ret = rte_pmd_i40e_set_vf_vlan_stripq(res->port_id, + res->vf_id, is_on); +#endif +#ifdef RTE_LIBRTE_BNXT_PMD + if (ret == -ENOTSUP) + ret = rte_pmd_bnxt_set_vf_vlan_stripq(res->port_id, + res->vf_id, is_on); +#endif + + switch (ret) { + case 0: + break; + case -EINVAL: + printf("invalid vf_id %d or is_on %d\n", res->vf_id, is_on); + break; + case -ENODEV: + printf("invalid port_id %d\n", res->port_id); + break; + case -ENOTSUP: + printf("function not implemented\n"); + break; + default: + printf("programming error: (%s)\n", strerror(-ret)); + } +} + +cmdline_parse_inst_t cmd_set_vf_vlan_stripq = { + .f = cmd_set_vf_vlan_stripq_parsed, + .data = NULL, + .help_str = "set vf vlan stripq on|off", + .tokens = { + (void *)&cmd_vf_vlan_stripq_set, + (void *)&cmd_vf_vlan_stripq_vf, + (void *)&cmd_vf_vlan_stripq_vlan, + (void *)&cmd_vf_vlan_stripq_stripq, + (void *)&cmd_vf_vlan_stripq_port_id, + (void *)&cmd_vf_vlan_stripq_vf_id, + (void *)&cmd_vf_vlan_stripq_on_off, + NULL, + }, +}; + +/* vf vlan insert configuration */ + +/* Common result structure for vf vlan insert */ +struct cmd_vf_vlan_insert_result { + cmdline_fixed_string_t set; + cmdline_fixed_string_t vf; + cmdline_fixed_string_t vlan; + cmdline_fixed_string_t insert; + portid_t port_id; + uint16_t vf_id; + uint16_t vlan_id; +}; + +/* Common CLI fields for vf vlan insert enable disable */ +cmdline_parse_token_string_t cmd_vf_vlan_insert_set = + TOKEN_STRING_INITIALIZER + (struct cmd_vf_vlan_insert_result, + set, "set"); +cmdline_parse_token_string_t cmd_vf_vlan_insert_vf = + TOKEN_STRING_INITIALIZER + (struct cmd_vf_vlan_insert_result, + vf, "vf"); +cmdline_parse_token_string_t cmd_vf_vlan_insert_vlan = + TOKEN_STRING_INITIALIZER + (struct cmd_vf_vlan_insert_result, + vlan, "vlan"); +cmdline_parse_token_string_t cmd_vf_vlan_insert_insert = + TOKEN_STRING_INITIALIZER + (struct cmd_vf_vlan_insert_result, + insert, "insert"); +cmdline_parse_token_num_t cmd_vf_vlan_insert_port_id = + TOKEN_NUM_INITIALIZER + (struct cmd_vf_vlan_insert_result, + port_id, UINT16); +cmdline_parse_token_num_t cmd_vf_vlan_insert_vf_id = + TOKEN_NUM_INITIALIZER + (struct cmd_vf_vlan_insert_result, + vf_id, UINT16); +cmdline_parse_token_num_t cmd_vf_vlan_insert_vlan_id = + TOKEN_NUM_INITIALIZER + (struct cmd_vf_vlan_insert_result, + vlan_id, UINT16); + +static void +cmd_set_vf_vlan_insert_parsed( + void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_vf_vlan_insert_result *res = parsed_result; + int ret = -ENOTSUP; + + if (port_id_is_invalid(res->port_id, ENABLED_WARN)) + return; + +#ifdef RTE_LIBRTE_IXGBE_PMD + if (ret == -ENOTSUP) + ret = rte_pmd_ixgbe_set_vf_vlan_insert(res->port_id, res->vf_id, + res->vlan_id); +#endif +#ifdef RTE_LIBRTE_I40E_PMD + if (ret == -ENOTSUP) + ret = rte_pmd_i40e_set_vf_vlan_insert(res->port_id, res->vf_id, + res->vlan_id); +#endif +#ifdef RTE_LIBRTE_BNXT_PMD + if (ret == -ENOTSUP) + ret = rte_pmd_bnxt_set_vf_vlan_insert(res->port_id, res->vf_id, + res->vlan_id); +#endif + + switch (ret) { + case 0: + break; + case -EINVAL: + printf("invalid vf_id %d or vlan_id %d\n", res->vf_id, res->vlan_id); + break; + case -ENODEV: + printf("invalid port_id %d\n", res->port_id); + break; + case -ENOTSUP: + printf("function not implemented\n"); + break; + default: + printf("programming error: (%s)\n", strerror(-ret)); + } +} + +cmdline_parse_inst_t cmd_set_vf_vlan_insert = { + .f = cmd_set_vf_vlan_insert_parsed, + .data = NULL, + .help_str = "set vf vlan insert ", + .tokens = { + (void *)&cmd_vf_vlan_insert_set, + (void *)&cmd_vf_vlan_insert_vf, + (void *)&cmd_vf_vlan_insert_vlan, + (void *)&cmd_vf_vlan_insert_insert, + (void *)&cmd_vf_vlan_insert_port_id, + (void *)&cmd_vf_vlan_insert_vf_id, + (void *)&cmd_vf_vlan_insert_vlan_id, + NULL, + }, +}; + +/* tx loopback configuration */ + +/* Common result structure for tx loopback */ +struct cmd_tx_loopback_result { + cmdline_fixed_string_t set; + cmdline_fixed_string_t tx; + cmdline_fixed_string_t loopback; + portid_t port_id; + cmdline_fixed_string_t on_off; +}; + +/* Common CLI fields for tx loopback enable disable */ +cmdline_parse_token_string_t cmd_tx_loopback_set = + TOKEN_STRING_INITIALIZER + (struct cmd_tx_loopback_result, + set, "set"); +cmdline_parse_token_string_t cmd_tx_loopback_tx = + TOKEN_STRING_INITIALIZER + (struct cmd_tx_loopback_result, + tx, "tx"); +cmdline_parse_token_string_t cmd_tx_loopback_loopback = + TOKEN_STRING_INITIALIZER + (struct cmd_tx_loopback_result, + loopback, "loopback"); +cmdline_parse_token_num_t cmd_tx_loopback_port_id = + TOKEN_NUM_INITIALIZER + (struct cmd_tx_loopback_result, + port_id, UINT16); +cmdline_parse_token_string_t cmd_tx_loopback_on_off = + TOKEN_STRING_INITIALIZER + (struct cmd_tx_loopback_result, + on_off, "on#off"); + +static void +cmd_set_tx_loopback_parsed( + void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_tx_loopback_result *res = parsed_result; + int ret = -ENOTSUP; + + __rte_unused int is_on = (strcmp(res->on_off, "on") == 0) ? 1 : 0; + + if (port_id_is_invalid(res->port_id, ENABLED_WARN)) + return; + +#ifdef RTE_LIBRTE_IXGBE_PMD + if (ret == -ENOTSUP) + ret = rte_pmd_ixgbe_set_tx_loopback(res->port_id, is_on); +#endif +#ifdef RTE_LIBRTE_I40E_PMD + if (ret == -ENOTSUP) + ret = rte_pmd_i40e_set_tx_loopback(res->port_id, is_on); +#endif +#ifdef RTE_LIBRTE_BNXT_PMD + if (ret == -ENOTSUP) + ret = rte_pmd_bnxt_set_tx_loopback(res->port_id, is_on); +#endif + + switch (ret) { + case 0: + break; + case -EINVAL: + printf("invalid is_on %d\n", is_on); + break; + case -ENODEV: + printf("invalid port_id %d\n", res->port_id); + break; + case -ENOTSUP: + printf("function not implemented\n"); + break; + default: + printf("programming error: (%s)\n", strerror(-ret)); + } +} + +cmdline_parse_inst_t cmd_set_tx_loopback = { + .f = cmd_set_tx_loopback_parsed, + .data = NULL, + .help_str = "set tx loopback on|off", + .tokens = { + (void *)&cmd_tx_loopback_set, + (void *)&cmd_tx_loopback_tx, + (void *)&cmd_tx_loopback_loopback, + (void *)&cmd_tx_loopback_port_id, + (void *)&cmd_tx_loopback_on_off, + NULL, + }, +}; + +/* all queues drop enable configuration */ + +/* Common result structure for all queues drop enable */ +struct cmd_all_queues_drop_en_result { + cmdline_fixed_string_t set; + cmdline_fixed_string_t all; + cmdline_fixed_string_t queues; + cmdline_fixed_string_t drop; + portid_t port_id; + cmdline_fixed_string_t on_off; +}; + +/* Common CLI fields for tx loopback enable disable */ +cmdline_parse_token_string_t cmd_all_queues_drop_en_set = + TOKEN_STRING_INITIALIZER + (struct cmd_all_queues_drop_en_result, + set, "set"); +cmdline_parse_token_string_t cmd_all_queues_drop_en_all = + TOKEN_STRING_INITIALIZER + (struct cmd_all_queues_drop_en_result, + all, "all"); +cmdline_parse_token_string_t cmd_all_queues_drop_en_queues = + TOKEN_STRING_INITIALIZER + (struct cmd_all_queues_drop_en_result, + queues, "queues"); +cmdline_parse_token_string_t cmd_all_queues_drop_en_drop = + TOKEN_STRING_INITIALIZER + (struct cmd_all_queues_drop_en_result, + drop, "drop"); +cmdline_parse_token_num_t cmd_all_queues_drop_en_port_id = + TOKEN_NUM_INITIALIZER + (struct cmd_all_queues_drop_en_result, + port_id, UINT16); +cmdline_parse_token_string_t cmd_all_queues_drop_en_on_off = + TOKEN_STRING_INITIALIZER + (struct cmd_all_queues_drop_en_result, + on_off, "on#off"); + +static void +cmd_set_all_queues_drop_en_parsed( + void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_all_queues_drop_en_result *res = parsed_result; + int ret = -ENOTSUP; + int is_on = (strcmp(res->on_off, "on") == 0) ? 1 : 0; + + if (port_id_is_invalid(res->port_id, ENABLED_WARN)) + return; + +#ifdef RTE_LIBRTE_IXGBE_PMD + if (ret == -ENOTSUP) + ret = rte_pmd_ixgbe_set_all_queues_drop_en(res->port_id, is_on); +#endif +#ifdef RTE_LIBRTE_BNXT_PMD + if (ret == -ENOTSUP) + ret = rte_pmd_bnxt_set_all_queues_drop_en(res->port_id, is_on); +#endif + switch (ret) { + case 0: + break; + case -EINVAL: + printf("invalid is_on %d\n", is_on); + break; + case -ENODEV: + printf("invalid port_id %d\n", res->port_id); + break; + case -ENOTSUP: + printf("function not implemented\n"); + break; + default: + printf("programming error: (%s)\n", strerror(-ret)); + } +} + +cmdline_parse_inst_t cmd_set_all_queues_drop_en = { + .f = cmd_set_all_queues_drop_en_parsed, + .data = NULL, + .help_str = "set all queues drop on|off", + .tokens = { + (void *)&cmd_all_queues_drop_en_set, + (void *)&cmd_all_queues_drop_en_all, + (void *)&cmd_all_queues_drop_en_queues, + (void *)&cmd_all_queues_drop_en_drop, + (void *)&cmd_all_queues_drop_en_port_id, + (void *)&cmd_all_queues_drop_en_on_off, + NULL, + }, +}; + +/* vf split drop enable configuration */ + +/* Common result structure for vf split drop enable */ +struct cmd_vf_split_drop_en_result { + cmdline_fixed_string_t set; + cmdline_fixed_string_t vf; + cmdline_fixed_string_t split; + cmdline_fixed_string_t drop; + portid_t port_id; + uint16_t vf_id; + cmdline_fixed_string_t on_off; +}; + +/* Common CLI fields for vf split drop enable disable */ +cmdline_parse_token_string_t cmd_vf_split_drop_en_set = + TOKEN_STRING_INITIALIZER + (struct cmd_vf_split_drop_en_result, + set, "set"); +cmdline_parse_token_string_t cmd_vf_split_drop_en_vf = + TOKEN_STRING_INITIALIZER + (struct cmd_vf_split_drop_en_result, + vf, "vf"); +cmdline_parse_token_string_t cmd_vf_split_drop_en_split = + TOKEN_STRING_INITIALIZER + (struct cmd_vf_split_drop_en_result, + split, "split"); +cmdline_parse_token_string_t cmd_vf_split_drop_en_drop = + TOKEN_STRING_INITIALIZER + (struct cmd_vf_split_drop_en_result, + drop, "drop"); +cmdline_parse_token_num_t cmd_vf_split_drop_en_port_id = + TOKEN_NUM_INITIALIZER + (struct cmd_vf_split_drop_en_result, + port_id, UINT16); +cmdline_parse_token_num_t cmd_vf_split_drop_en_vf_id = + TOKEN_NUM_INITIALIZER + (struct cmd_vf_split_drop_en_result, + vf_id, UINT16); +cmdline_parse_token_string_t cmd_vf_split_drop_en_on_off = + TOKEN_STRING_INITIALIZER + (struct cmd_vf_split_drop_en_result, + on_off, "on#off"); + +static void +cmd_set_vf_split_drop_en_parsed( + void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_vf_split_drop_en_result *res = parsed_result; + int ret = -ENOTSUP; + int is_on = (strcmp(res->on_off, "on") == 0) ? 1 : 0; + + if (port_id_is_invalid(res->port_id, ENABLED_WARN)) + return; + +#ifdef RTE_LIBRTE_IXGBE_PMD + ret = rte_pmd_ixgbe_set_vf_split_drop_en(res->port_id, res->vf_id, + is_on); +#endif + switch (ret) { + case 0: + break; + case -EINVAL: + printf("invalid vf_id %d or is_on %d\n", res->vf_id, is_on); + break; + case -ENODEV: + printf("invalid port_id %d\n", res->port_id); + break; + case -ENOTSUP: + printf("not supported on port %d\n", res->port_id); + break; + default: + printf("programming error: (%s)\n", strerror(-ret)); + } +} + +cmdline_parse_inst_t cmd_set_vf_split_drop_en = { + .f = cmd_set_vf_split_drop_en_parsed, + .data = NULL, + .help_str = "set vf split drop on|off", + .tokens = { + (void *)&cmd_vf_split_drop_en_set, + (void *)&cmd_vf_split_drop_en_vf, + (void *)&cmd_vf_split_drop_en_split, + (void *)&cmd_vf_split_drop_en_drop, + (void *)&cmd_vf_split_drop_en_port_id, + (void *)&cmd_vf_split_drop_en_vf_id, + (void *)&cmd_vf_split_drop_en_on_off, + NULL, + }, +}; + +/* vf mac address configuration */ + +/* Common result structure for vf mac address */ +struct cmd_set_vf_mac_addr_result { + cmdline_fixed_string_t set; + cmdline_fixed_string_t vf; + cmdline_fixed_string_t mac; + cmdline_fixed_string_t addr; + portid_t port_id; + uint16_t vf_id; + struct ether_addr mac_addr; + +}; + +/* Common CLI fields for vf split drop enable disable */ +cmdline_parse_token_string_t cmd_set_vf_mac_addr_set = + TOKEN_STRING_INITIALIZER + (struct cmd_set_vf_mac_addr_result, + set, "set"); +cmdline_parse_token_string_t cmd_set_vf_mac_addr_vf = + TOKEN_STRING_INITIALIZER + (struct cmd_set_vf_mac_addr_result, + vf, "vf"); +cmdline_parse_token_string_t cmd_set_vf_mac_addr_mac = + TOKEN_STRING_INITIALIZER + (struct cmd_set_vf_mac_addr_result, + mac, "mac"); +cmdline_parse_token_string_t cmd_set_vf_mac_addr_addr = + TOKEN_STRING_INITIALIZER + (struct cmd_set_vf_mac_addr_result, + addr, "addr"); +cmdline_parse_token_num_t cmd_set_vf_mac_addr_port_id = + TOKEN_NUM_INITIALIZER + (struct cmd_set_vf_mac_addr_result, + port_id, UINT16); +cmdline_parse_token_num_t cmd_set_vf_mac_addr_vf_id = + TOKEN_NUM_INITIALIZER + (struct cmd_set_vf_mac_addr_result, + vf_id, UINT16); +cmdline_parse_token_etheraddr_t cmd_set_vf_mac_addr_mac_addr = + TOKEN_ETHERADDR_INITIALIZER(struct cmd_set_vf_mac_addr_result, + mac_addr); + +static void +cmd_set_vf_mac_addr_parsed( + void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_set_vf_mac_addr_result *res = parsed_result; + int ret = -ENOTSUP; + + if (port_id_is_invalid(res->port_id, ENABLED_WARN)) + return; + +#ifdef RTE_LIBRTE_IXGBE_PMD + if (ret == -ENOTSUP) + ret = rte_pmd_ixgbe_set_vf_mac_addr(res->port_id, res->vf_id, + &res->mac_addr); +#endif +#ifdef RTE_LIBRTE_I40E_PMD + if (ret == -ENOTSUP) + ret = rte_pmd_i40e_set_vf_mac_addr(res->port_id, res->vf_id, + &res->mac_addr); +#endif +#ifdef RTE_LIBRTE_BNXT_PMD + if (ret == -ENOTSUP) + ret = rte_pmd_bnxt_set_vf_mac_addr(res->port_id, res->vf_id, + &res->mac_addr); +#endif + + switch (ret) { + case 0: + break; + case -EINVAL: + printf("invalid vf_id %d or mac_addr\n", res->vf_id); + break; + case -ENODEV: + printf("invalid port_id %d\n", res->port_id); + break; + case -ENOTSUP: + printf("function not implemented\n"); + break; + default: + printf("programming error: (%s)\n", strerror(-ret)); + } +} + +cmdline_parse_inst_t cmd_set_vf_mac_addr = { + .f = cmd_set_vf_mac_addr_parsed, + .data = NULL, + .help_str = "set vf mac addr ", + .tokens = { + (void *)&cmd_set_vf_mac_addr_set, + (void *)&cmd_set_vf_mac_addr_vf, + (void *)&cmd_set_vf_mac_addr_mac, + (void *)&cmd_set_vf_mac_addr_addr, + (void *)&cmd_set_vf_mac_addr_port_id, + (void *)&cmd_set_vf_mac_addr_vf_id, + (void *)&cmd_set_vf_mac_addr_mac_addr, + NULL, + }, +}; + +/* MACsec configuration */ + +/* Common result structure for MACsec offload enable */ +struct cmd_macsec_offload_on_result { + cmdline_fixed_string_t set; + cmdline_fixed_string_t macsec; + cmdline_fixed_string_t offload; + portid_t port_id; + cmdline_fixed_string_t on; + cmdline_fixed_string_t encrypt; + cmdline_fixed_string_t en_on_off; + cmdline_fixed_string_t replay_protect; + cmdline_fixed_string_t rp_on_off; +}; + +/* Common CLI fields for MACsec offload disable */ +cmdline_parse_token_string_t cmd_macsec_offload_on_set = + TOKEN_STRING_INITIALIZER + (struct cmd_macsec_offload_on_result, + set, "set"); +cmdline_parse_token_string_t cmd_macsec_offload_on_macsec = + TOKEN_STRING_INITIALIZER + (struct cmd_macsec_offload_on_result, + macsec, "macsec"); +cmdline_parse_token_string_t cmd_macsec_offload_on_offload = + TOKEN_STRING_INITIALIZER + (struct cmd_macsec_offload_on_result, + offload, "offload"); +cmdline_parse_token_num_t cmd_macsec_offload_on_port_id = + TOKEN_NUM_INITIALIZER + (struct cmd_macsec_offload_on_result, + port_id, UINT16); +cmdline_parse_token_string_t cmd_macsec_offload_on_on = + TOKEN_STRING_INITIALIZER + (struct cmd_macsec_offload_on_result, + on, "on"); +cmdline_parse_token_string_t cmd_macsec_offload_on_encrypt = + TOKEN_STRING_INITIALIZER + (struct cmd_macsec_offload_on_result, + encrypt, "encrypt"); +cmdline_parse_token_string_t cmd_macsec_offload_on_en_on_off = + TOKEN_STRING_INITIALIZER + (struct cmd_macsec_offload_on_result, + en_on_off, "on#off"); +cmdline_parse_token_string_t cmd_macsec_offload_on_replay_protect = + TOKEN_STRING_INITIALIZER + (struct cmd_macsec_offload_on_result, + replay_protect, "replay-protect"); +cmdline_parse_token_string_t cmd_macsec_offload_on_rp_on_off = + TOKEN_STRING_INITIALIZER + (struct cmd_macsec_offload_on_result, + rp_on_off, "on#off"); + +static void +cmd_set_macsec_offload_on_parsed( + void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_macsec_offload_on_result *res = parsed_result; + int ret = -ENOTSUP; + portid_t port_id = res->port_id; + int en = (strcmp(res->en_on_off, "on") == 0) ? 1 : 0; + int rp = (strcmp(res->rp_on_off, "on") == 0) ? 1 : 0; + + if (port_id_is_invalid(port_id, ENABLED_WARN)) + return; + + ports[port_id].tx_ol_flags |= TESTPMD_TX_OFFLOAD_MACSEC; +#ifdef RTE_LIBRTE_IXGBE_PMD + ret = rte_pmd_ixgbe_macsec_enable(port_id, en, rp); +#endif + RTE_SET_USED(en); + RTE_SET_USED(rp); + + switch (ret) { + case 0: + break; + case -ENODEV: + printf("invalid port_id %d\n", port_id); + break; + case -ENOTSUP: + printf("not supported on port %d\n", port_id); + break; + default: + printf("programming error: (%s)\n", strerror(-ret)); + } +} + +cmdline_parse_inst_t cmd_set_macsec_offload_on = { + .f = cmd_set_macsec_offload_on_parsed, + .data = NULL, + .help_str = "set macsec offload on " + "encrypt on|off replay-protect on|off", + .tokens = { + (void *)&cmd_macsec_offload_on_set, + (void *)&cmd_macsec_offload_on_macsec, + (void *)&cmd_macsec_offload_on_offload, + (void *)&cmd_macsec_offload_on_port_id, + (void *)&cmd_macsec_offload_on_on, + (void *)&cmd_macsec_offload_on_encrypt, + (void *)&cmd_macsec_offload_on_en_on_off, + (void *)&cmd_macsec_offload_on_replay_protect, + (void *)&cmd_macsec_offload_on_rp_on_off, + NULL, + }, +}; + +/* Common result structure for MACsec offload disable */ +struct cmd_macsec_offload_off_result { + cmdline_fixed_string_t set; + cmdline_fixed_string_t macsec; + cmdline_fixed_string_t offload; + portid_t port_id; + cmdline_fixed_string_t off; +}; + +/* Common CLI fields for MACsec offload disable */ +cmdline_parse_token_string_t cmd_macsec_offload_off_set = + TOKEN_STRING_INITIALIZER + (struct cmd_macsec_offload_off_result, + set, "set"); +cmdline_parse_token_string_t cmd_macsec_offload_off_macsec = + TOKEN_STRING_INITIALIZER + (struct cmd_macsec_offload_off_result, + macsec, "macsec"); +cmdline_parse_token_string_t cmd_macsec_offload_off_offload = + TOKEN_STRING_INITIALIZER + (struct cmd_macsec_offload_off_result, + offload, "offload"); +cmdline_parse_token_num_t cmd_macsec_offload_off_port_id = + TOKEN_NUM_INITIALIZER + (struct cmd_macsec_offload_off_result, + port_id, UINT16); +cmdline_parse_token_string_t cmd_macsec_offload_off_off = + TOKEN_STRING_INITIALIZER + (struct cmd_macsec_offload_off_result, + off, "off"); + +static void +cmd_set_macsec_offload_off_parsed( + void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_macsec_offload_off_result *res = parsed_result; + int ret = -ENOTSUP; + portid_t port_id = res->port_id; + + if (port_id_is_invalid(port_id, ENABLED_WARN)) + return; + + ports[port_id].tx_ol_flags &= ~TESTPMD_TX_OFFLOAD_MACSEC; +#ifdef RTE_LIBRTE_IXGBE_PMD + ret = rte_pmd_ixgbe_macsec_disable(port_id); +#endif + + switch (ret) { + case 0: + break; + case -ENODEV: + printf("invalid port_id %d\n", port_id); + break; + case -ENOTSUP: + printf("not supported on port %d\n", port_id); + break; + default: + printf("programming error: (%s)\n", strerror(-ret)); + } +} + +cmdline_parse_inst_t cmd_set_macsec_offload_off = { + .f = cmd_set_macsec_offload_off_parsed, + .data = NULL, + .help_str = "set macsec offload off", + .tokens = { + (void *)&cmd_macsec_offload_off_set, + (void *)&cmd_macsec_offload_off_macsec, + (void *)&cmd_macsec_offload_off_offload, + (void *)&cmd_macsec_offload_off_port_id, + (void *)&cmd_macsec_offload_off_off, + NULL, + }, +}; + +/* Common result structure for MACsec secure connection configure */ +struct cmd_macsec_sc_result { + cmdline_fixed_string_t set; + cmdline_fixed_string_t macsec; + cmdline_fixed_string_t sc; + cmdline_fixed_string_t tx_rx; + portid_t port_id; + struct ether_addr mac; + uint16_t pi; +}; + +/* Common CLI fields for MACsec secure connection configure */ +cmdline_parse_token_string_t cmd_macsec_sc_set = + TOKEN_STRING_INITIALIZER + (struct cmd_macsec_sc_result, + set, "set"); +cmdline_parse_token_string_t cmd_macsec_sc_macsec = + TOKEN_STRING_INITIALIZER + (struct cmd_macsec_sc_result, + macsec, "macsec"); +cmdline_parse_token_string_t cmd_macsec_sc_sc = + TOKEN_STRING_INITIALIZER + (struct cmd_macsec_sc_result, + sc, "sc"); +cmdline_parse_token_string_t cmd_macsec_sc_tx_rx = + TOKEN_STRING_INITIALIZER + (struct cmd_macsec_sc_result, + tx_rx, "tx#rx"); +cmdline_parse_token_num_t cmd_macsec_sc_port_id = + TOKEN_NUM_INITIALIZER + (struct cmd_macsec_sc_result, + port_id, UINT16); +cmdline_parse_token_etheraddr_t cmd_macsec_sc_mac = + TOKEN_ETHERADDR_INITIALIZER + (struct cmd_macsec_sc_result, + mac); +cmdline_parse_token_num_t cmd_macsec_sc_pi = + TOKEN_NUM_INITIALIZER + (struct cmd_macsec_sc_result, + pi, UINT16); + +static void +cmd_set_macsec_sc_parsed( + void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_macsec_sc_result *res = parsed_result; + int ret = -ENOTSUP; + int is_tx = (strcmp(res->tx_rx, "tx") == 0) ? 1 : 0; + +#ifdef RTE_LIBRTE_IXGBE_PMD + ret = is_tx ? + rte_pmd_ixgbe_macsec_config_txsc(res->port_id, + res->mac.addr_bytes) : + rte_pmd_ixgbe_macsec_config_rxsc(res->port_id, + res->mac.addr_bytes, res->pi); +#endif + RTE_SET_USED(is_tx); + + switch (ret) { + case 0: + break; + case -ENODEV: + printf("invalid port_id %d\n", res->port_id); + break; + case -ENOTSUP: + printf("not supported on port %d\n", res->port_id); + break; + default: + printf("programming error: (%s)\n", strerror(-ret)); + } +} + +cmdline_parse_inst_t cmd_set_macsec_sc = { + .f = cmd_set_macsec_sc_parsed, + .data = NULL, + .help_str = "set macsec sc tx|rx ", + .tokens = { + (void *)&cmd_macsec_sc_set, + (void *)&cmd_macsec_sc_macsec, + (void *)&cmd_macsec_sc_sc, + (void *)&cmd_macsec_sc_tx_rx, + (void *)&cmd_macsec_sc_port_id, + (void *)&cmd_macsec_sc_mac, + (void *)&cmd_macsec_sc_pi, + NULL, + }, +}; + +/* Common result structure for MACsec secure connection configure */ +struct cmd_macsec_sa_result { + cmdline_fixed_string_t set; + cmdline_fixed_string_t macsec; + cmdline_fixed_string_t sa; + cmdline_fixed_string_t tx_rx; + portid_t port_id; + uint8_t idx; + uint8_t an; + uint32_t pn; + cmdline_fixed_string_t key; +}; + +/* Common CLI fields for MACsec secure connection configure */ +cmdline_parse_token_string_t cmd_macsec_sa_set = + TOKEN_STRING_INITIALIZER + (struct cmd_macsec_sa_result, + set, "set"); +cmdline_parse_token_string_t cmd_macsec_sa_macsec = + TOKEN_STRING_INITIALIZER + (struct cmd_macsec_sa_result, + macsec, "macsec"); +cmdline_parse_token_string_t cmd_macsec_sa_sa = + TOKEN_STRING_INITIALIZER + (struct cmd_macsec_sa_result, + sa, "sa"); +cmdline_parse_token_string_t cmd_macsec_sa_tx_rx = + TOKEN_STRING_INITIALIZER + (struct cmd_macsec_sa_result, + tx_rx, "tx#rx"); +cmdline_parse_token_num_t cmd_macsec_sa_port_id = + TOKEN_NUM_INITIALIZER + (struct cmd_macsec_sa_result, + port_id, UINT16); +cmdline_parse_token_num_t cmd_macsec_sa_idx = + TOKEN_NUM_INITIALIZER + (struct cmd_macsec_sa_result, + idx, UINT8); +cmdline_parse_token_num_t cmd_macsec_sa_an = + TOKEN_NUM_INITIALIZER + (struct cmd_macsec_sa_result, + an, UINT8); +cmdline_parse_token_num_t cmd_macsec_sa_pn = + TOKEN_NUM_INITIALIZER + (struct cmd_macsec_sa_result, + pn, UINT32); +cmdline_parse_token_string_t cmd_macsec_sa_key = + TOKEN_STRING_INITIALIZER + (struct cmd_macsec_sa_result, + key, NULL); + +static void +cmd_set_macsec_sa_parsed( + void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_macsec_sa_result *res = parsed_result; + int ret = -ENOTSUP; + int is_tx = (strcmp(res->tx_rx, "tx") == 0) ? 1 : 0; + uint8_t key[16] = { 0 }; + uint8_t xdgt0; + uint8_t xdgt1; + int key_len; + int i; + + key_len = strlen(res->key) / 2; + if (key_len > 16) + key_len = 16; + + for (i = 0; i < key_len; i++) { + xdgt0 = parse_and_check_key_hexa_digit(res->key, (i * 2)); + if (xdgt0 == 0xFF) + return; + xdgt1 = parse_and_check_key_hexa_digit(res->key, (i * 2) + 1); + if (xdgt1 == 0xFF) + return; + key[i] = (uint8_t) ((xdgt0 * 16) + xdgt1); + } + +#ifdef RTE_LIBRTE_IXGBE_PMD + ret = is_tx ? + rte_pmd_ixgbe_macsec_select_txsa(res->port_id, + res->idx, res->an, res->pn, key) : + rte_pmd_ixgbe_macsec_select_rxsa(res->port_id, + res->idx, res->an, res->pn, key); +#endif + RTE_SET_USED(is_tx); + RTE_SET_USED(key); + + switch (ret) { + case 0: + break; + case -EINVAL: + printf("invalid idx %d or an %d\n", res->idx, res->an); + break; + case -ENODEV: + printf("invalid port_id %d\n", res->port_id); + break; + case -ENOTSUP: + printf("not supported on port %d\n", res->port_id); + break; + default: + printf("programming error: (%s)\n", strerror(-ret)); + } +} + +cmdline_parse_inst_t cmd_set_macsec_sa = { + .f = cmd_set_macsec_sa_parsed, + .data = NULL, + .help_str = "set macsec sa tx|rx ", + .tokens = { + (void *)&cmd_macsec_sa_set, + (void *)&cmd_macsec_sa_macsec, + (void *)&cmd_macsec_sa_sa, + (void *)&cmd_macsec_sa_tx_rx, + (void *)&cmd_macsec_sa_port_id, + (void *)&cmd_macsec_sa_idx, + (void *)&cmd_macsec_sa_an, + (void *)&cmd_macsec_sa_pn, + (void *)&cmd_macsec_sa_key, + NULL, + }, +}; + +/* VF unicast promiscuous mode configuration */ + +/* Common result structure for VF unicast promiscuous mode */ +struct cmd_vf_promisc_result { + cmdline_fixed_string_t set; + cmdline_fixed_string_t vf; + cmdline_fixed_string_t promisc; + portid_t port_id; + uint32_t vf_id; + cmdline_fixed_string_t on_off; +}; + +/* Common CLI fields for VF unicast promiscuous mode enable disable */ +cmdline_parse_token_string_t cmd_vf_promisc_set = + TOKEN_STRING_INITIALIZER + (struct cmd_vf_promisc_result, + set, "set"); +cmdline_parse_token_string_t cmd_vf_promisc_vf = + TOKEN_STRING_INITIALIZER + (struct cmd_vf_promisc_result, + vf, "vf"); +cmdline_parse_token_string_t cmd_vf_promisc_promisc = + TOKEN_STRING_INITIALIZER + (struct cmd_vf_promisc_result, + promisc, "promisc"); +cmdline_parse_token_num_t cmd_vf_promisc_port_id = + TOKEN_NUM_INITIALIZER + (struct cmd_vf_promisc_result, + port_id, UINT16); +cmdline_parse_token_num_t cmd_vf_promisc_vf_id = + TOKEN_NUM_INITIALIZER + (struct cmd_vf_promisc_result, + vf_id, UINT32); +cmdline_parse_token_string_t cmd_vf_promisc_on_off = + TOKEN_STRING_INITIALIZER + (struct cmd_vf_promisc_result, + on_off, "on#off"); + +static void +cmd_set_vf_promisc_parsed( + void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_vf_promisc_result *res = parsed_result; + int ret = -ENOTSUP; + + __rte_unused int is_on = (strcmp(res->on_off, "on") == 0) ? 1 : 0; + + if (port_id_is_invalid(res->port_id, ENABLED_WARN)) + return; + +#ifdef RTE_LIBRTE_I40E_PMD + ret = rte_pmd_i40e_set_vf_unicast_promisc(res->port_id, + res->vf_id, is_on); +#endif + + switch (ret) { + case 0: + break; + case -EINVAL: + printf("invalid vf_id %d\n", res->vf_id); + break; + case -ENODEV: + printf("invalid port_id %d\n", res->port_id); + break; + case -ENOTSUP: + printf("function not implemented\n"); + break; + default: + printf("programming error: (%s)\n", strerror(-ret)); + } +} + +cmdline_parse_inst_t cmd_set_vf_promisc = { + .f = cmd_set_vf_promisc_parsed, + .data = NULL, + .help_str = "set vf promisc on|off: " + "Set unicast promiscuous mode for a VF from the PF", + .tokens = { + (void *)&cmd_vf_promisc_set, + (void *)&cmd_vf_promisc_vf, + (void *)&cmd_vf_promisc_promisc, + (void *)&cmd_vf_promisc_port_id, + (void *)&cmd_vf_promisc_vf_id, + (void *)&cmd_vf_promisc_on_off, + NULL, + }, +}; + +/* VF multicast promiscuous mode configuration */ + +/* Common result structure for VF multicast promiscuous mode */ +struct cmd_vf_allmulti_result { + cmdline_fixed_string_t set; + cmdline_fixed_string_t vf; + cmdline_fixed_string_t allmulti; + portid_t port_id; + uint32_t vf_id; + cmdline_fixed_string_t on_off; +}; + +/* Common CLI fields for VF multicast promiscuous mode enable disable */ +cmdline_parse_token_string_t cmd_vf_allmulti_set = + TOKEN_STRING_INITIALIZER + (struct cmd_vf_allmulti_result, + set, "set"); +cmdline_parse_token_string_t cmd_vf_allmulti_vf = + TOKEN_STRING_INITIALIZER + (struct cmd_vf_allmulti_result, + vf, "vf"); +cmdline_parse_token_string_t cmd_vf_allmulti_allmulti = + TOKEN_STRING_INITIALIZER + (struct cmd_vf_allmulti_result, + allmulti, "allmulti"); +cmdline_parse_token_num_t cmd_vf_allmulti_port_id = + TOKEN_NUM_INITIALIZER + (struct cmd_vf_allmulti_result, + port_id, UINT16); +cmdline_parse_token_num_t cmd_vf_allmulti_vf_id = + TOKEN_NUM_INITIALIZER + (struct cmd_vf_allmulti_result, + vf_id, UINT32); +cmdline_parse_token_string_t cmd_vf_allmulti_on_off = + TOKEN_STRING_INITIALIZER + (struct cmd_vf_allmulti_result, + on_off, "on#off"); + +static void +cmd_set_vf_allmulti_parsed( + void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_vf_allmulti_result *res = parsed_result; + int ret = -ENOTSUP; + + __rte_unused int is_on = (strcmp(res->on_off, "on") == 0) ? 1 : 0; + + if (port_id_is_invalid(res->port_id, ENABLED_WARN)) + return; + +#ifdef RTE_LIBRTE_I40E_PMD + ret = rte_pmd_i40e_set_vf_multicast_promisc(res->port_id, + res->vf_id, is_on); +#endif + + switch (ret) { + case 0: + break; + case -EINVAL: + printf("invalid vf_id %d\n", res->vf_id); + break; + case -ENODEV: + printf("invalid port_id %d\n", res->port_id); + break; + case -ENOTSUP: + printf("function not implemented\n"); + break; + default: + printf("programming error: (%s)\n", strerror(-ret)); + } +} + +cmdline_parse_inst_t cmd_set_vf_allmulti = { + .f = cmd_set_vf_allmulti_parsed, + .data = NULL, + .help_str = "set vf allmulti on|off: " + "Set multicast promiscuous mode for a VF from the PF", + .tokens = { + (void *)&cmd_vf_allmulti_set, + (void *)&cmd_vf_allmulti_vf, + (void *)&cmd_vf_allmulti_allmulti, + (void *)&cmd_vf_allmulti_port_id, + (void *)&cmd_vf_allmulti_vf_id, + (void *)&cmd_vf_allmulti_on_off, + NULL, + }, +}; + +/* vf broadcast mode configuration */ + +/* Common result structure for vf broadcast */ +struct cmd_set_vf_broadcast_result { + cmdline_fixed_string_t set; + cmdline_fixed_string_t vf; + cmdline_fixed_string_t broadcast; + portid_t port_id; + uint16_t vf_id; + cmdline_fixed_string_t on_off; +}; + +/* Common CLI fields for vf broadcast enable disable */ +cmdline_parse_token_string_t cmd_set_vf_broadcast_set = + TOKEN_STRING_INITIALIZER + (struct cmd_set_vf_broadcast_result, + set, "set"); +cmdline_parse_token_string_t cmd_set_vf_broadcast_vf = + TOKEN_STRING_INITIALIZER + (struct cmd_set_vf_broadcast_result, + vf, "vf"); +cmdline_parse_token_string_t cmd_set_vf_broadcast_broadcast = + TOKEN_STRING_INITIALIZER + (struct cmd_set_vf_broadcast_result, + broadcast, "broadcast"); +cmdline_parse_token_num_t cmd_set_vf_broadcast_port_id = + TOKEN_NUM_INITIALIZER + (struct cmd_set_vf_broadcast_result, + port_id, UINT16); +cmdline_parse_token_num_t cmd_set_vf_broadcast_vf_id = + TOKEN_NUM_INITIALIZER + (struct cmd_set_vf_broadcast_result, + vf_id, UINT16); +cmdline_parse_token_string_t cmd_set_vf_broadcast_on_off = + TOKEN_STRING_INITIALIZER + (struct cmd_set_vf_broadcast_result, + on_off, "on#off"); + +static void +cmd_set_vf_broadcast_parsed( + void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_set_vf_broadcast_result *res = parsed_result; + int ret = -ENOTSUP; + + __rte_unused int is_on = (strcmp(res->on_off, "on") == 0) ? 1 : 0; + + if (port_id_is_invalid(res->port_id, ENABLED_WARN)) + return; + +#ifdef RTE_LIBRTE_I40E_PMD + ret = rte_pmd_i40e_set_vf_broadcast(res->port_id, + res->vf_id, is_on); +#endif + + switch (ret) { + case 0: + break; + case -EINVAL: + printf("invalid vf_id %d or is_on %d\n", res->vf_id, is_on); + break; + case -ENODEV: + printf("invalid port_id %d\n", res->port_id); + break; + case -ENOTSUP: + printf("function not implemented\n"); + break; + default: + printf("programming error: (%s)\n", strerror(-ret)); + } +} + +cmdline_parse_inst_t cmd_set_vf_broadcast = { + .f = cmd_set_vf_broadcast_parsed, + .data = NULL, + .help_str = "set vf broadcast on|off", + .tokens = { + (void *)&cmd_set_vf_broadcast_set, + (void *)&cmd_set_vf_broadcast_vf, + (void *)&cmd_set_vf_broadcast_broadcast, + (void *)&cmd_set_vf_broadcast_port_id, + (void *)&cmd_set_vf_broadcast_vf_id, + (void *)&cmd_set_vf_broadcast_on_off, + NULL, + }, +}; + +/* vf vlan tag configuration */ + +/* Common result structure for vf vlan tag */ +struct cmd_set_vf_vlan_tag_result { + cmdline_fixed_string_t set; + cmdline_fixed_string_t vf; + cmdline_fixed_string_t vlan; + cmdline_fixed_string_t tag; + portid_t port_id; + uint16_t vf_id; + cmdline_fixed_string_t on_off; +}; + +/* Common CLI fields for vf vlan tag enable disable */ +cmdline_parse_token_string_t cmd_set_vf_vlan_tag_set = + TOKEN_STRING_INITIALIZER + (struct cmd_set_vf_vlan_tag_result, + set, "set"); +cmdline_parse_token_string_t cmd_set_vf_vlan_tag_vf = + TOKEN_STRING_INITIALIZER + (struct cmd_set_vf_vlan_tag_result, + vf, "vf"); +cmdline_parse_token_string_t cmd_set_vf_vlan_tag_vlan = + TOKEN_STRING_INITIALIZER + (struct cmd_set_vf_vlan_tag_result, + vlan, "vlan"); +cmdline_parse_token_string_t cmd_set_vf_vlan_tag_tag = + TOKEN_STRING_INITIALIZER + (struct cmd_set_vf_vlan_tag_result, + tag, "tag"); +cmdline_parse_token_num_t cmd_set_vf_vlan_tag_port_id = + TOKEN_NUM_INITIALIZER + (struct cmd_set_vf_vlan_tag_result, + port_id, UINT16); +cmdline_parse_token_num_t cmd_set_vf_vlan_tag_vf_id = + TOKEN_NUM_INITIALIZER + (struct cmd_set_vf_vlan_tag_result, + vf_id, UINT16); +cmdline_parse_token_string_t cmd_set_vf_vlan_tag_on_off = + TOKEN_STRING_INITIALIZER + (struct cmd_set_vf_vlan_tag_result, + on_off, "on#off"); + +static void +cmd_set_vf_vlan_tag_parsed( + void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_set_vf_vlan_tag_result *res = parsed_result; + int ret = -ENOTSUP; + + __rte_unused int is_on = (strcmp(res->on_off, "on") == 0) ? 1 : 0; + + if (port_id_is_invalid(res->port_id, ENABLED_WARN)) + return; + +#ifdef RTE_LIBRTE_I40E_PMD + ret = rte_pmd_i40e_set_vf_vlan_tag(res->port_id, + res->vf_id, is_on); +#endif + + switch (ret) { + case 0: + break; + case -EINVAL: + printf("invalid vf_id %d or is_on %d\n", res->vf_id, is_on); + break; + case -ENODEV: + printf("invalid port_id %d\n", res->port_id); + break; + case -ENOTSUP: + printf("function not implemented\n"); + break; + default: + printf("programming error: (%s)\n", strerror(-ret)); + } +} + +cmdline_parse_inst_t cmd_set_vf_vlan_tag = { + .f = cmd_set_vf_vlan_tag_parsed, + .data = NULL, + .help_str = "set vf vlan tag on|off", + .tokens = { + (void *)&cmd_set_vf_vlan_tag_set, + (void *)&cmd_set_vf_vlan_tag_vf, + (void *)&cmd_set_vf_vlan_tag_vlan, + (void *)&cmd_set_vf_vlan_tag_tag, + (void *)&cmd_set_vf_vlan_tag_port_id, + (void *)&cmd_set_vf_vlan_tag_vf_id, + (void *)&cmd_set_vf_vlan_tag_on_off, + NULL, + }, +}; + +/* Common definition of VF and TC TX bandwidth configuration */ +struct cmd_vf_tc_bw_result { + cmdline_fixed_string_t set; + cmdline_fixed_string_t vf; + cmdline_fixed_string_t tc; + cmdline_fixed_string_t tx; + cmdline_fixed_string_t min_bw; + cmdline_fixed_string_t max_bw; + cmdline_fixed_string_t strict_link_prio; + portid_t port_id; + uint16_t vf_id; + uint8_t tc_no; + uint32_t bw; + cmdline_fixed_string_t bw_list; + uint8_t tc_map; +}; + +cmdline_parse_token_string_t cmd_vf_tc_bw_set = + TOKEN_STRING_INITIALIZER + (struct cmd_vf_tc_bw_result, + set, "set"); +cmdline_parse_token_string_t cmd_vf_tc_bw_vf = + TOKEN_STRING_INITIALIZER + (struct cmd_vf_tc_bw_result, + vf, "vf"); +cmdline_parse_token_string_t cmd_vf_tc_bw_tc = + TOKEN_STRING_INITIALIZER + (struct cmd_vf_tc_bw_result, + tc, "tc"); +cmdline_parse_token_string_t cmd_vf_tc_bw_tx = + TOKEN_STRING_INITIALIZER + (struct cmd_vf_tc_bw_result, + tx, "tx"); +cmdline_parse_token_string_t cmd_vf_tc_bw_strict_link_prio = + TOKEN_STRING_INITIALIZER + (struct cmd_vf_tc_bw_result, + strict_link_prio, "strict-link-priority"); +cmdline_parse_token_string_t cmd_vf_tc_bw_min_bw = + TOKEN_STRING_INITIALIZER + (struct cmd_vf_tc_bw_result, + min_bw, "min-bandwidth"); +cmdline_parse_token_string_t cmd_vf_tc_bw_max_bw = + TOKEN_STRING_INITIALIZER + (struct cmd_vf_tc_bw_result, + max_bw, "max-bandwidth"); +cmdline_parse_token_num_t cmd_vf_tc_bw_port_id = + TOKEN_NUM_INITIALIZER + (struct cmd_vf_tc_bw_result, + port_id, UINT16); +cmdline_parse_token_num_t cmd_vf_tc_bw_vf_id = + TOKEN_NUM_INITIALIZER + (struct cmd_vf_tc_bw_result, + vf_id, UINT16); +cmdline_parse_token_num_t cmd_vf_tc_bw_tc_no = + TOKEN_NUM_INITIALIZER + (struct cmd_vf_tc_bw_result, + tc_no, UINT8); +cmdline_parse_token_num_t cmd_vf_tc_bw_bw = + TOKEN_NUM_INITIALIZER + (struct cmd_vf_tc_bw_result, + bw, UINT32); +cmdline_parse_token_string_t cmd_vf_tc_bw_bw_list = + TOKEN_STRING_INITIALIZER + (struct cmd_vf_tc_bw_result, + bw_list, NULL); +cmdline_parse_token_num_t cmd_vf_tc_bw_tc_map = + TOKEN_NUM_INITIALIZER + (struct cmd_vf_tc_bw_result, + tc_map, UINT8); + +/* VF max bandwidth setting */ +static void +cmd_vf_max_bw_parsed( + void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_vf_tc_bw_result *res = parsed_result; + int ret = -ENOTSUP; + + if (port_id_is_invalid(res->port_id, ENABLED_WARN)) + return; + +#ifdef RTE_LIBRTE_I40E_PMD + ret = rte_pmd_i40e_set_vf_max_bw(res->port_id, + res->vf_id, res->bw); +#endif + + switch (ret) { + case 0: + break; + case -EINVAL: + printf("invalid vf_id %d or bandwidth %d\n", + res->vf_id, res->bw); + break; + case -ENODEV: + printf("invalid port_id %d\n", res->port_id); + break; + case -ENOTSUP: + printf("function not implemented\n"); + break; + default: + printf("programming error: (%s)\n", strerror(-ret)); + } +} + +cmdline_parse_inst_t cmd_vf_max_bw = { + .f = cmd_vf_max_bw_parsed, + .data = NULL, + .help_str = "set vf tx max-bandwidth ", + .tokens = { + (void *)&cmd_vf_tc_bw_set, + (void *)&cmd_vf_tc_bw_vf, + (void *)&cmd_vf_tc_bw_tx, + (void *)&cmd_vf_tc_bw_max_bw, + (void *)&cmd_vf_tc_bw_port_id, + (void *)&cmd_vf_tc_bw_vf_id, + (void *)&cmd_vf_tc_bw_bw, + NULL, + }, +}; + +static int +vf_tc_min_bw_parse_bw_list(uint8_t *bw_list, + uint8_t *tc_num, + char *str) +{ + uint32_t size; + const char *p, *p0 = str; + char s[256]; + char *end; + char *str_fld[16]; + uint16_t i; + int ret; + + p = strchr(p0, '('); + if (p == NULL) { + printf("The bandwidth-list should be '(bw1, bw2, ...)'\n"); + return -1; + } + p++; + p0 = strchr(p, ')'); + if (p0 == NULL) { + printf("The bandwidth-list should be '(bw1, bw2, ...)'\n"); + return -1; + } + size = p0 - p; + if (size >= sizeof(s)) { + printf("The string size exceeds the internal buffer size\n"); + return -1; + } + snprintf(s, sizeof(s), "%.*s", size, p); + ret = rte_strsplit(s, sizeof(s), str_fld, 16, ','); + if (ret <= 0) { + printf("Failed to get the bandwidth list. "); + return -1; + } + *tc_num = ret; + for (i = 0; i < ret; i++) + bw_list[i] = (uint8_t)strtoul(str_fld[i], &end, 0); + + return 0; +} + +/* TC min bandwidth setting */ +static void +cmd_vf_tc_min_bw_parsed( + void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_vf_tc_bw_result *res = parsed_result; + uint8_t tc_num; + uint8_t bw[16]; + int ret = -ENOTSUP; + + if (port_id_is_invalid(res->port_id, ENABLED_WARN)) + return; + + ret = vf_tc_min_bw_parse_bw_list(bw, &tc_num, res->bw_list); + if (ret) + return; + +#ifdef RTE_LIBRTE_I40E_PMD + ret = rte_pmd_i40e_set_vf_tc_bw_alloc(res->port_id, res->vf_id, + tc_num, bw); +#endif + + switch (ret) { + case 0: + break; + case -EINVAL: + printf("invalid vf_id %d or bandwidth\n", res->vf_id); + break; + case -ENODEV: + printf("invalid port_id %d\n", res->port_id); + break; + case -ENOTSUP: + printf("function not implemented\n"); + break; + default: + printf("programming error: (%s)\n", strerror(-ret)); + } +} + +cmdline_parse_inst_t cmd_vf_tc_min_bw = { + .f = cmd_vf_tc_min_bw_parsed, + .data = NULL, + .help_str = "set vf tc tx min-bandwidth " + " ", + .tokens = { + (void *)&cmd_vf_tc_bw_set, + (void *)&cmd_vf_tc_bw_vf, + (void *)&cmd_vf_tc_bw_tc, + (void *)&cmd_vf_tc_bw_tx, + (void *)&cmd_vf_tc_bw_min_bw, + (void *)&cmd_vf_tc_bw_port_id, + (void *)&cmd_vf_tc_bw_vf_id, + (void *)&cmd_vf_tc_bw_bw_list, + NULL, + }, +}; + +static void +cmd_tc_min_bw_parsed( + void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_vf_tc_bw_result *res = parsed_result; + struct rte_port *port; + uint8_t tc_num; + uint8_t bw[16]; + int ret = -ENOTSUP; + + if (port_id_is_invalid(res->port_id, ENABLED_WARN)) + return; + + port = &ports[res->port_id]; + /** Check if the port is not started **/ + if (port->port_status != RTE_PORT_STOPPED) { + printf("Please stop port %d first\n", res->port_id); + return; + } + + ret = vf_tc_min_bw_parse_bw_list(bw, &tc_num, res->bw_list); + if (ret) + return; + +#ifdef RTE_LIBRTE_IXGBE_PMD + ret = rte_pmd_ixgbe_set_tc_bw_alloc(res->port_id, tc_num, bw); +#endif + + switch (ret) { + case 0: + break; + case -EINVAL: + printf("invalid bandwidth\n"); + break; + case -ENODEV: + printf("invalid port_id %d\n", res->port_id); + break; + case -ENOTSUP: + printf("function not implemented\n"); + break; + default: + printf("programming error: (%s)\n", strerror(-ret)); + } +} + +cmdline_parse_inst_t cmd_tc_min_bw = { + .f = cmd_tc_min_bw_parsed, + .data = NULL, + .help_str = "set tc tx min-bandwidth ", + .tokens = { + (void *)&cmd_vf_tc_bw_set, + (void *)&cmd_vf_tc_bw_tc, + (void *)&cmd_vf_tc_bw_tx, + (void *)&cmd_vf_tc_bw_min_bw, + (void *)&cmd_vf_tc_bw_port_id, + (void *)&cmd_vf_tc_bw_bw_list, + NULL, + }, +}; + +/* TC max bandwidth setting */ +static void +cmd_vf_tc_max_bw_parsed( + void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_vf_tc_bw_result *res = parsed_result; + int ret = -ENOTSUP; + + if (port_id_is_invalid(res->port_id, ENABLED_WARN)) + return; + +#ifdef RTE_LIBRTE_I40E_PMD + ret = rte_pmd_i40e_set_vf_tc_max_bw(res->port_id, res->vf_id, + res->tc_no, res->bw); +#endif + + switch (ret) { + case 0: + break; + case -EINVAL: + printf("invalid vf_id %d, tc_no %d or bandwidth %d\n", + res->vf_id, res->tc_no, res->bw); + break; + case -ENODEV: + printf("invalid port_id %d\n", res->port_id); + break; + case -ENOTSUP: + printf("function not implemented\n"); + break; + default: + printf("programming error: (%s)\n", strerror(-ret)); + } +} + +cmdline_parse_inst_t cmd_vf_tc_max_bw = { + .f = cmd_vf_tc_max_bw_parsed, + .data = NULL, + .help_str = "set vf tc tx max-bandwidth " + " ", + .tokens = { + (void *)&cmd_vf_tc_bw_set, + (void *)&cmd_vf_tc_bw_vf, + (void *)&cmd_vf_tc_bw_tc, + (void *)&cmd_vf_tc_bw_tx, + (void *)&cmd_vf_tc_bw_max_bw, + (void *)&cmd_vf_tc_bw_port_id, + (void *)&cmd_vf_tc_bw_vf_id, + (void *)&cmd_vf_tc_bw_tc_no, + (void *)&cmd_vf_tc_bw_bw, + NULL, + }, +}; + + +#if defined RTE_LIBRTE_PMD_SOFTNIC && defined RTE_LIBRTE_SCHED + +/* *** Set Port default Traffic Management Hierarchy *** */ +struct cmd_set_port_tm_hierarchy_default_result { + cmdline_fixed_string_t set; + cmdline_fixed_string_t port; + cmdline_fixed_string_t tm; + cmdline_fixed_string_t hierarchy; + cmdline_fixed_string_t def; + portid_t port_id; +}; + +cmdline_parse_token_string_t cmd_set_port_tm_hierarchy_default_set = + TOKEN_STRING_INITIALIZER( + struct cmd_set_port_tm_hierarchy_default_result, set, "set"); +cmdline_parse_token_string_t cmd_set_port_tm_hierarchy_default_port = + TOKEN_STRING_INITIALIZER( + struct cmd_set_port_tm_hierarchy_default_result, port, "port"); +cmdline_parse_token_string_t cmd_set_port_tm_hierarchy_default_tm = + TOKEN_STRING_INITIALIZER( + struct cmd_set_port_tm_hierarchy_default_result, tm, "tm"); +cmdline_parse_token_string_t cmd_set_port_tm_hierarchy_default_hierarchy = + TOKEN_STRING_INITIALIZER( + struct cmd_set_port_tm_hierarchy_default_result, + hierarchy, "hierarchy"); +cmdline_parse_token_string_t cmd_set_port_tm_hierarchy_default_default = + TOKEN_STRING_INITIALIZER( + struct cmd_set_port_tm_hierarchy_default_result, + def, "default"); +cmdline_parse_token_num_t cmd_set_port_tm_hierarchy_default_port_id = + TOKEN_NUM_INITIALIZER( + struct cmd_set_port_tm_hierarchy_default_result, + port_id, UINT16); + +static void cmd_set_port_tm_hierarchy_default_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_set_port_tm_hierarchy_default_result *res = parsed_result; + struct rte_port *p; + portid_t port_id = res->port_id; + + if (port_id_is_invalid(port_id, ENABLED_WARN)) + return; + + p = &ports[port_id]; + + /* Port tm flag */ + if (p->softport.tm_flag == 0) { + printf(" tm not enabled on port %u (error)\n", port_id); + return; + } + + /* Forward mode: tm */ + if (strcmp(cur_fwd_config.fwd_eng->fwd_mode_name, "tm")) { + printf(" tm mode not enabled(error)\n"); + return; + } + + /* Set the default tm hierarchy */ + p->softport.tm.default_hierarchy_enable = 1; +} + +cmdline_parse_inst_t cmd_set_port_tm_hierarchy_default = { + .f = cmd_set_port_tm_hierarchy_default_parsed, + .data = NULL, + .help_str = "set port tm hierarchy default ", + .tokens = { + (void *)&cmd_set_port_tm_hierarchy_default_set, + (void *)&cmd_set_port_tm_hierarchy_default_port, + (void *)&cmd_set_port_tm_hierarchy_default_tm, + (void *)&cmd_set_port_tm_hierarchy_default_hierarchy, + (void *)&cmd_set_port_tm_hierarchy_default_default, + (void *)&cmd_set_port_tm_hierarchy_default_port_id, + NULL, + }, +}; +#endif + +/* Strict link priority scheduling mode setting */ +static void +cmd_strict_link_prio_parsed( + void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_vf_tc_bw_result *res = parsed_result; + int ret = -ENOTSUP; + + if (port_id_is_invalid(res->port_id, ENABLED_WARN)) + return; + +#ifdef RTE_LIBRTE_I40E_PMD + ret = rte_pmd_i40e_set_tc_strict_prio(res->port_id, res->tc_map); +#endif + + switch (ret) { + case 0: + break; + case -EINVAL: + printf("invalid tc_bitmap 0x%x\n", res->tc_map); + break; + case -ENODEV: + printf("invalid port_id %d\n", res->port_id); + break; + case -ENOTSUP: + printf("function not implemented\n"); + break; + default: + printf("programming error: (%s)\n", strerror(-ret)); + } +} + +cmdline_parse_inst_t cmd_strict_link_prio = { + .f = cmd_strict_link_prio_parsed, + .data = NULL, + .help_str = "set tx strict-link-priority ", + .tokens = { + (void *)&cmd_vf_tc_bw_set, + (void *)&cmd_vf_tc_bw_tx, + (void *)&cmd_vf_tc_bw_strict_link_prio, + (void *)&cmd_vf_tc_bw_port_id, + (void *)&cmd_vf_tc_bw_tc_map, + NULL, + }, +}; + +/* Load dynamic device personalization*/ +struct cmd_ddp_add_result { + cmdline_fixed_string_t ddp; + cmdline_fixed_string_t add; + portid_t port_id; + char filepath[]; +}; + +cmdline_parse_token_string_t cmd_ddp_add_ddp = + TOKEN_STRING_INITIALIZER(struct cmd_ddp_add_result, ddp, "ddp"); +cmdline_parse_token_string_t cmd_ddp_add_add = + TOKEN_STRING_INITIALIZER(struct cmd_ddp_add_result, add, "add"); +cmdline_parse_token_num_t cmd_ddp_add_port_id = + TOKEN_NUM_INITIALIZER(struct cmd_ddp_add_result, port_id, UINT16); +cmdline_parse_token_string_t cmd_ddp_add_filepath = + TOKEN_STRING_INITIALIZER(struct cmd_ddp_add_result, filepath, NULL); + +static void +cmd_ddp_add_parsed( + void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_ddp_add_result *res = parsed_result; + uint8_t *buff; + uint32_t size; + char *filepath; + char *file_fld[2]; + int file_num; + int ret = -ENOTSUP; + + if (res->port_id > nb_ports) { + printf("Invalid port, range is [0, %d]\n", nb_ports - 1); + return; + } + + if (!all_ports_stopped()) { + printf("Please stop all ports first\n"); + return; + } + + filepath = strdup(res->filepath); + if (filepath == NULL) { + printf("Failed to allocate memory\n"); + return; + } + file_num = rte_strsplit(filepath, strlen(filepath), file_fld, 2, ','); + + buff = open_ddp_package_file(file_fld[0], &size); + if (!buff) { + free((void *)filepath); + return; + } + +#ifdef RTE_LIBRTE_I40E_PMD + if (ret == -ENOTSUP) + ret = rte_pmd_i40e_process_ddp_package(res->port_id, + buff, size, + RTE_PMD_I40E_PKG_OP_WR_ADD); +#endif + + if (ret == -EEXIST) + printf("Profile has already existed.\n"); + else if (ret < 0) + printf("Failed to load profile.\n"); + else if (file_num == 2) + save_ddp_package_file(file_fld[1], buff, size); + + close_ddp_package_file(buff); + free((void *)filepath); +} + +cmdline_parse_inst_t cmd_ddp_add = { + .f = cmd_ddp_add_parsed, + .data = NULL, + .help_str = "ddp add ", + .tokens = { + (void *)&cmd_ddp_add_ddp, + (void *)&cmd_ddp_add_add, + (void *)&cmd_ddp_add_port_id, + (void *)&cmd_ddp_add_filepath, + NULL, + }, +}; + +/* Delete dynamic device personalization*/ +struct cmd_ddp_del_result { + cmdline_fixed_string_t ddp; + cmdline_fixed_string_t del; + portid_t port_id; + char filepath[]; +}; + +cmdline_parse_token_string_t cmd_ddp_del_ddp = + TOKEN_STRING_INITIALIZER(struct cmd_ddp_del_result, ddp, "ddp"); +cmdline_parse_token_string_t cmd_ddp_del_del = + TOKEN_STRING_INITIALIZER(struct cmd_ddp_del_result, del, "del"); +cmdline_parse_token_num_t cmd_ddp_del_port_id = + TOKEN_NUM_INITIALIZER(struct cmd_ddp_del_result, port_id, UINT16); +cmdline_parse_token_string_t cmd_ddp_del_filepath = + TOKEN_STRING_INITIALIZER(struct cmd_ddp_del_result, filepath, NULL); + +static void +cmd_ddp_del_parsed( + void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_ddp_del_result *res = parsed_result; + uint8_t *buff; + uint32_t size; + int ret = -ENOTSUP; + + if (res->port_id > nb_ports) { + printf("Invalid port, range is [0, %d]\n", nb_ports - 1); + return; + } + + if (!all_ports_stopped()) { + printf("Please stop all ports first\n"); + return; + } + + buff = open_ddp_package_file(res->filepath, &size); + if (!buff) + return; + +#ifdef RTE_LIBRTE_I40E_PMD + if (ret == -ENOTSUP) + ret = rte_pmd_i40e_process_ddp_package(res->port_id, + buff, size, + RTE_PMD_I40E_PKG_OP_WR_DEL); +#endif + + if (ret == -EACCES) + printf("Profile does not exist.\n"); + else if (ret < 0) + printf("Failed to delete profile.\n"); + + close_ddp_package_file(buff); +} + +cmdline_parse_inst_t cmd_ddp_del = { + .f = cmd_ddp_del_parsed, + .data = NULL, + .help_str = "ddp del ", + .tokens = { + (void *)&cmd_ddp_del_ddp, + (void *)&cmd_ddp_del_del, + (void *)&cmd_ddp_del_port_id, + (void *)&cmd_ddp_del_filepath, + NULL, + }, +}; + +/* Get dynamic device personalization profile info */ +struct cmd_ddp_info_result { + cmdline_fixed_string_t ddp; + cmdline_fixed_string_t get; + cmdline_fixed_string_t info; + char filepath[]; +}; + +cmdline_parse_token_string_t cmd_ddp_info_ddp = + TOKEN_STRING_INITIALIZER(struct cmd_ddp_info_result, ddp, "ddp"); +cmdline_parse_token_string_t cmd_ddp_info_get = + TOKEN_STRING_INITIALIZER(struct cmd_ddp_info_result, get, "get"); +cmdline_parse_token_string_t cmd_ddp_info_info = + TOKEN_STRING_INITIALIZER(struct cmd_ddp_info_result, info, "info"); +cmdline_parse_token_string_t cmd_ddp_info_filepath = + TOKEN_STRING_INITIALIZER(struct cmd_ddp_info_result, filepath, NULL); + +static void +cmd_ddp_info_parsed( + void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_ddp_info_result *res = parsed_result; + uint8_t *pkg; + uint32_t pkg_size; + int ret = -ENOTSUP; +#ifdef RTE_LIBRTE_I40E_PMD + uint32_t i, j, n; + uint8_t *buff; + uint32_t buff_size = 0; + struct rte_pmd_i40e_profile_info info; + uint32_t dev_num = 0; + struct rte_pmd_i40e_ddp_device_id *devs; + uint32_t proto_num = 0; + struct rte_pmd_i40e_proto_info *proto = NULL; + uint32_t pctype_num = 0; + struct rte_pmd_i40e_ptype_info *pctype; + uint32_t ptype_num = 0; + struct rte_pmd_i40e_ptype_info *ptype; + uint8_t proto_id; + +#endif + + pkg = open_ddp_package_file(res->filepath, &pkg_size); + if (!pkg) + return; + +#ifdef RTE_LIBRTE_I40E_PMD + ret = rte_pmd_i40e_get_ddp_info(pkg, pkg_size, + (uint8_t *)&info, sizeof(info), + RTE_PMD_I40E_PKG_INFO_GLOBAL_HEADER); + if (!ret) { + printf("Global Track id: 0x%x\n", info.track_id); + printf("Global Version: %d.%d.%d.%d\n", + info.version.major, + info.version.minor, + info.version.update, + info.version.draft); + printf("Global Package name: %s\n\n", info.name); + } + + ret = rte_pmd_i40e_get_ddp_info(pkg, pkg_size, + (uint8_t *)&info, sizeof(info), + RTE_PMD_I40E_PKG_INFO_HEADER); + if (!ret) { + printf("i40e Profile Track id: 0x%x\n", info.track_id); + printf("i40e Profile Version: %d.%d.%d.%d\n", + info.version.major, + info.version.minor, + info.version.update, + info.version.draft); + printf("i40e Profile name: %s\n\n", info.name); + } + + ret = rte_pmd_i40e_get_ddp_info(pkg, pkg_size, + (uint8_t *)&buff_size, sizeof(buff_size), + RTE_PMD_I40E_PKG_INFO_GLOBAL_NOTES_SIZE); + if (!ret && buff_size) { + buff = (uint8_t *)malloc(buff_size); + if (buff) { + ret = rte_pmd_i40e_get_ddp_info(pkg, pkg_size, + buff, buff_size, + RTE_PMD_I40E_PKG_INFO_GLOBAL_NOTES); + if (!ret) + printf("Package Notes:\n%s\n\n", buff); + free(buff); + } + } + + ret = rte_pmd_i40e_get_ddp_info(pkg, pkg_size, + (uint8_t *)&dev_num, sizeof(dev_num), + RTE_PMD_I40E_PKG_INFO_DEVID_NUM); + if (!ret && dev_num) { + buff_size = dev_num * sizeof(struct rte_pmd_i40e_ddp_device_id); + devs = (struct rte_pmd_i40e_ddp_device_id *)malloc(buff_size); + if (devs) { + ret = rte_pmd_i40e_get_ddp_info(pkg, pkg_size, + (uint8_t *)devs, buff_size, + RTE_PMD_I40E_PKG_INFO_DEVID_LIST); + if (!ret) { + printf("List of supported devices:\n"); + for (i = 0; i < dev_num; i++) { + printf(" %04X:%04X %04X:%04X\n", + devs[i].vendor_dev_id >> 16, + devs[i].vendor_dev_id & 0xFFFF, + devs[i].sub_vendor_dev_id >> 16, + devs[i].sub_vendor_dev_id & 0xFFFF); + } + printf("\n"); + } + free(devs); + } + } + + /* get information about protocols and packet types */ + ret = rte_pmd_i40e_get_ddp_info(pkg, pkg_size, + (uint8_t *)&proto_num, sizeof(proto_num), + RTE_PMD_I40E_PKG_INFO_PROTOCOL_NUM); + if (ret || !proto_num) + goto no_print_return; + + buff_size = proto_num * sizeof(struct rte_pmd_i40e_proto_info); + proto = (struct rte_pmd_i40e_proto_info *)malloc(buff_size); + if (!proto) + goto no_print_return; + + ret = rte_pmd_i40e_get_ddp_info(pkg, pkg_size, (uint8_t *)proto, + buff_size, + RTE_PMD_I40E_PKG_INFO_PROTOCOL_LIST); + if (!ret) { + printf("List of used protocols:\n"); + for (i = 0; i < proto_num; i++) + printf(" %2u: %s\n", proto[i].proto_id, + proto[i].name); + printf("\n"); + } + ret = rte_pmd_i40e_get_ddp_info(pkg, pkg_size, + (uint8_t *)&pctype_num, sizeof(pctype_num), + RTE_PMD_I40E_PKG_INFO_PCTYPE_NUM); + if (ret || !pctype_num) + goto no_print_pctypes; + + buff_size = pctype_num * sizeof(struct rte_pmd_i40e_ptype_info); + pctype = (struct rte_pmd_i40e_ptype_info *)malloc(buff_size); + if (!pctype) + goto no_print_pctypes; + + ret = rte_pmd_i40e_get_ddp_info(pkg, pkg_size, (uint8_t *)pctype, + buff_size, + RTE_PMD_I40E_PKG_INFO_PCTYPE_LIST); + if (ret) { + free(pctype); + goto no_print_pctypes; + } + + printf("List of defined packet classification types:\n"); + for (i = 0; i < pctype_num; i++) { + printf(" %2u:", pctype[i].ptype_id); + for (j = 0; j < RTE_PMD_I40E_PROTO_NUM; j++) { + proto_id = pctype[i].protocols[j]; + if (proto_id != RTE_PMD_I40E_PROTO_UNUSED) { + for (n = 0; n < proto_num; n++) { + if (proto[n].proto_id == proto_id) { + printf(" %s", proto[n].name); + break; + } + } + } + } + printf("\n"); + } + printf("\n"); + free(pctype); + +no_print_pctypes: + + ret = rte_pmd_i40e_get_ddp_info(pkg, pkg_size, (uint8_t *)&ptype_num, + sizeof(ptype_num), + RTE_PMD_I40E_PKG_INFO_PTYPE_NUM); + if (ret || !ptype_num) + goto no_print_return; + + buff_size = ptype_num * sizeof(struct rte_pmd_i40e_ptype_info); + ptype = (struct rte_pmd_i40e_ptype_info *)malloc(buff_size); + if (!ptype) + goto no_print_return; + + ret = rte_pmd_i40e_get_ddp_info(pkg, pkg_size, (uint8_t *)ptype, + buff_size, + RTE_PMD_I40E_PKG_INFO_PTYPE_LIST); + if (ret) { + free(ptype); + goto no_print_return; + } + printf("List of defined packet types:\n"); + for (i = 0; i < ptype_num; i++) { + printf(" %2u:", ptype[i].ptype_id); + for (j = 0; j < RTE_PMD_I40E_PROTO_NUM; j++) { + proto_id = ptype[i].protocols[j]; + if (proto_id != RTE_PMD_I40E_PROTO_UNUSED) { + for (n = 0; n < proto_num; n++) { + if (proto[n].proto_id == proto_id) { + printf(" %s", proto[n].name); + break; + } + } + } + } + printf("\n"); + } + free(ptype); + printf("\n"); + + ret = 0; +no_print_return: + if (proto) + free(proto); +#endif + if (ret == -ENOTSUP) + printf("Function not supported in PMD driver\n"); + close_ddp_package_file(pkg); +} + +cmdline_parse_inst_t cmd_ddp_get_info = { + .f = cmd_ddp_info_parsed, + .data = NULL, + .help_str = "ddp get info ", + .tokens = { + (void *)&cmd_ddp_info_ddp, + (void *)&cmd_ddp_info_get, + (void *)&cmd_ddp_info_info, + (void *)&cmd_ddp_info_filepath, + NULL, + }, +}; + +/* Get dynamic device personalization profile info list*/ +#define PROFILE_INFO_SIZE 48 +#define MAX_PROFILE_NUM 16 + +struct cmd_ddp_get_list_result { + cmdline_fixed_string_t ddp; + cmdline_fixed_string_t get; + cmdline_fixed_string_t list; + portid_t port_id; +}; + +cmdline_parse_token_string_t cmd_ddp_get_list_ddp = + TOKEN_STRING_INITIALIZER(struct cmd_ddp_get_list_result, ddp, "ddp"); +cmdline_parse_token_string_t cmd_ddp_get_list_get = + TOKEN_STRING_INITIALIZER(struct cmd_ddp_get_list_result, get, "get"); +cmdline_parse_token_string_t cmd_ddp_get_list_list = + TOKEN_STRING_INITIALIZER(struct cmd_ddp_get_list_result, list, "list"); +cmdline_parse_token_num_t cmd_ddp_get_list_port_id = + TOKEN_NUM_INITIALIZER(struct cmd_ddp_get_list_result, port_id, UINT16); + +static void +cmd_ddp_get_list_parsed( + void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_ddp_get_list_result *res = parsed_result; +#ifdef RTE_LIBRTE_I40E_PMD + struct rte_pmd_i40e_profile_list *p_list; + struct rte_pmd_i40e_profile_info *p_info; + uint32_t p_num; + uint32_t size; + uint32_t i; +#endif + int ret = -ENOTSUP; + + if (res->port_id > nb_ports) { + printf("Invalid port, range is [0, %d]\n", nb_ports - 1); + return; + } + +#ifdef RTE_LIBRTE_I40E_PMD + size = PROFILE_INFO_SIZE * MAX_PROFILE_NUM + 4; + p_list = (struct rte_pmd_i40e_profile_list *)malloc(size); + if (!p_list) + printf("%s: Failed to malloc buffer\n", __func__); + + if (ret == -ENOTSUP) + ret = rte_pmd_i40e_get_ddp_list(res->port_id, + (uint8_t *)p_list, size); + + if (!ret) { + p_num = p_list->p_count; + printf("Profile number is: %d\n\n", p_num); + + for (i = 0; i < p_num; i++) { + p_info = &p_list->p_info[i]; + printf("Profile %d:\n", i); + printf("Track id: 0x%x\n", p_info->track_id); + printf("Version: %d.%d.%d.%d\n", + p_info->version.major, + p_info->version.minor, + p_info->version.update, + p_info->version.draft); + printf("Profile name: %s\n\n", p_info->name); + } + } + + free(p_list); +#endif + + if (ret < 0) + printf("Failed to get ddp list\n"); +} + +cmdline_parse_inst_t cmd_ddp_get_list = { + .f = cmd_ddp_get_list_parsed, + .data = NULL, + .help_str = "ddp get list ", + .tokens = { + (void *)&cmd_ddp_get_list_ddp, + (void *)&cmd_ddp_get_list_get, + (void *)&cmd_ddp_get_list_list, + (void *)&cmd_ddp_get_list_port_id, + NULL, + }, +}; + +/* show vf stats */ + +/* Common result structure for show vf stats */ +struct cmd_show_vf_stats_result { + cmdline_fixed_string_t show; + cmdline_fixed_string_t vf; + cmdline_fixed_string_t stats; + portid_t port_id; + uint16_t vf_id; +}; + +/* Common CLI fields show vf stats*/ +cmdline_parse_token_string_t cmd_show_vf_stats_show = + TOKEN_STRING_INITIALIZER + (struct cmd_show_vf_stats_result, + show, "show"); +cmdline_parse_token_string_t cmd_show_vf_stats_vf = + TOKEN_STRING_INITIALIZER + (struct cmd_show_vf_stats_result, + vf, "vf"); +cmdline_parse_token_string_t cmd_show_vf_stats_stats = + TOKEN_STRING_INITIALIZER + (struct cmd_show_vf_stats_result, + stats, "stats"); +cmdline_parse_token_num_t cmd_show_vf_stats_port_id = + TOKEN_NUM_INITIALIZER + (struct cmd_show_vf_stats_result, + port_id, UINT16); +cmdline_parse_token_num_t cmd_show_vf_stats_vf_id = + TOKEN_NUM_INITIALIZER + (struct cmd_show_vf_stats_result, + vf_id, UINT16); + +static void +cmd_show_vf_stats_parsed( + void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_show_vf_stats_result *res = parsed_result; + struct rte_eth_stats stats; + int ret = -ENOTSUP; + static const char *nic_stats_border = "########################"; + + if (port_id_is_invalid(res->port_id, ENABLED_WARN)) + return; + + memset(&stats, 0, sizeof(stats)); + +#ifdef RTE_LIBRTE_I40E_PMD + if (ret == -ENOTSUP) + ret = rte_pmd_i40e_get_vf_stats(res->port_id, + res->vf_id, + &stats); +#endif +#ifdef RTE_LIBRTE_BNXT_PMD + if (ret == -ENOTSUP) + ret = rte_pmd_bnxt_get_vf_stats(res->port_id, + res->vf_id, + &stats); +#endif + + switch (ret) { + case 0: + break; + case -EINVAL: + printf("invalid vf_id %d\n", res->vf_id); + break; + case -ENODEV: + printf("invalid port_id %d\n", res->port_id); + break; + case -ENOTSUP: + printf("function not implemented\n"); + break; + default: + printf("programming error: (%s)\n", strerror(-ret)); + } + + printf("\n %s NIC statistics for port %-2d vf %-2d %s\n", + nic_stats_border, res->port_id, res->vf_id, nic_stats_border); + + printf(" RX-packets: %-10"PRIu64" RX-missed: %-10"PRIu64" RX-bytes: " + "%-"PRIu64"\n", + stats.ipackets, stats.imissed, stats.ibytes); + printf(" RX-errors: %-"PRIu64"\n", stats.ierrors); + printf(" RX-nombuf: %-10"PRIu64"\n", + stats.rx_nombuf); + printf(" TX-packets: %-10"PRIu64" TX-errors: %-10"PRIu64" TX-bytes: " + "%-"PRIu64"\n", + stats.opackets, stats.oerrors, stats.obytes); + + printf(" %s############################%s\n", + nic_stats_border, nic_stats_border); +} + +cmdline_parse_inst_t cmd_show_vf_stats = { + .f = cmd_show_vf_stats_parsed, + .data = NULL, + .help_str = "show vf stats ", + .tokens = { + (void *)&cmd_show_vf_stats_show, + (void *)&cmd_show_vf_stats_vf, + (void *)&cmd_show_vf_stats_stats, + (void *)&cmd_show_vf_stats_port_id, + (void *)&cmd_show_vf_stats_vf_id, + NULL, + }, +}; + +/* clear vf stats */ + +/* Common result structure for clear vf stats */ +struct cmd_clear_vf_stats_result { + cmdline_fixed_string_t clear; + cmdline_fixed_string_t vf; + cmdline_fixed_string_t stats; + portid_t port_id; + uint16_t vf_id; +}; + +/* Common CLI fields clear vf stats*/ +cmdline_parse_token_string_t cmd_clear_vf_stats_clear = + TOKEN_STRING_INITIALIZER + (struct cmd_clear_vf_stats_result, + clear, "clear"); +cmdline_parse_token_string_t cmd_clear_vf_stats_vf = + TOKEN_STRING_INITIALIZER + (struct cmd_clear_vf_stats_result, + vf, "vf"); +cmdline_parse_token_string_t cmd_clear_vf_stats_stats = + TOKEN_STRING_INITIALIZER + (struct cmd_clear_vf_stats_result, + stats, "stats"); +cmdline_parse_token_num_t cmd_clear_vf_stats_port_id = + TOKEN_NUM_INITIALIZER + (struct cmd_clear_vf_stats_result, + port_id, UINT16); +cmdline_parse_token_num_t cmd_clear_vf_stats_vf_id = + TOKEN_NUM_INITIALIZER + (struct cmd_clear_vf_stats_result, + vf_id, UINT16); + +static void +cmd_clear_vf_stats_parsed( + void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_clear_vf_stats_result *res = parsed_result; + int ret = -ENOTSUP; + + if (port_id_is_invalid(res->port_id, ENABLED_WARN)) + return; + +#ifdef RTE_LIBRTE_I40E_PMD + if (ret == -ENOTSUP) + ret = rte_pmd_i40e_reset_vf_stats(res->port_id, + res->vf_id); +#endif +#ifdef RTE_LIBRTE_BNXT_PMD + if (ret == -ENOTSUP) + ret = rte_pmd_bnxt_reset_vf_stats(res->port_id, + res->vf_id); +#endif + + switch (ret) { + case 0: + break; + case -EINVAL: + printf("invalid vf_id %d\n", res->vf_id); + break; + case -ENODEV: + printf("invalid port_id %d\n", res->port_id); + break; + case -ENOTSUP: + printf("function not implemented\n"); + break; + default: + printf("programming error: (%s)\n", strerror(-ret)); + } +} + +cmdline_parse_inst_t cmd_clear_vf_stats = { + .f = cmd_clear_vf_stats_parsed, + .data = NULL, + .help_str = "clear vf stats ", + .tokens = { + (void *)&cmd_clear_vf_stats_clear, + (void *)&cmd_clear_vf_stats_vf, + (void *)&cmd_clear_vf_stats_stats, + (void *)&cmd_clear_vf_stats_port_id, + (void *)&cmd_clear_vf_stats_vf_id, + NULL, + }, +}; + +/* port config pctype mapping reset */ + +/* Common result structure for port config pctype mapping reset */ +struct cmd_pctype_mapping_reset_result { + cmdline_fixed_string_t port; + cmdline_fixed_string_t config; + portid_t port_id; + cmdline_fixed_string_t pctype; + cmdline_fixed_string_t mapping; + cmdline_fixed_string_t reset; +}; + +/* Common CLI fields for port config pctype mapping reset*/ +cmdline_parse_token_string_t cmd_pctype_mapping_reset_port = + TOKEN_STRING_INITIALIZER + (struct cmd_pctype_mapping_reset_result, + port, "port"); +cmdline_parse_token_string_t cmd_pctype_mapping_reset_config = + TOKEN_STRING_INITIALIZER + (struct cmd_pctype_mapping_reset_result, + config, "config"); +cmdline_parse_token_num_t cmd_pctype_mapping_reset_port_id = + TOKEN_NUM_INITIALIZER + (struct cmd_pctype_mapping_reset_result, + port_id, UINT16); +cmdline_parse_token_string_t cmd_pctype_mapping_reset_pctype = + TOKEN_STRING_INITIALIZER + (struct cmd_pctype_mapping_reset_result, + pctype, "pctype"); +cmdline_parse_token_string_t cmd_pctype_mapping_reset_mapping = + TOKEN_STRING_INITIALIZER + (struct cmd_pctype_mapping_reset_result, + mapping, "mapping"); +cmdline_parse_token_string_t cmd_pctype_mapping_reset_reset = + TOKEN_STRING_INITIALIZER + (struct cmd_pctype_mapping_reset_result, + reset, "reset"); + +static void +cmd_pctype_mapping_reset_parsed( + void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_pctype_mapping_reset_result *res = parsed_result; + int ret = -ENOTSUP; + + if (port_id_is_invalid(res->port_id, ENABLED_WARN)) + return; + +#ifdef RTE_LIBRTE_I40E_PMD + ret = rte_pmd_i40e_flow_type_mapping_reset(res->port_id); +#endif + + switch (ret) { + case 0: + break; + case -ENODEV: + printf("invalid port_id %d\n", res->port_id); + break; + case -ENOTSUP: + printf("function not implemented\n"); + break; + default: + printf("programming error: (%s)\n", strerror(-ret)); + } +} + +cmdline_parse_inst_t cmd_pctype_mapping_reset = { + .f = cmd_pctype_mapping_reset_parsed, + .data = NULL, + .help_str = "port config pctype mapping reset", + .tokens = { + (void *)&cmd_pctype_mapping_reset_port, + (void *)&cmd_pctype_mapping_reset_config, + (void *)&cmd_pctype_mapping_reset_port_id, + (void *)&cmd_pctype_mapping_reset_pctype, + (void *)&cmd_pctype_mapping_reset_mapping, + (void *)&cmd_pctype_mapping_reset_reset, + NULL, + }, +}; + +/* show port pctype mapping */ + +/* Common result structure for show port pctype mapping */ +struct cmd_pctype_mapping_get_result { + cmdline_fixed_string_t show; + cmdline_fixed_string_t port; + portid_t port_id; + cmdline_fixed_string_t pctype; + cmdline_fixed_string_t mapping; +}; + +/* Common CLI fields for pctype mapping get */ +cmdline_parse_token_string_t cmd_pctype_mapping_get_show = + TOKEN_STRING_INITIALIZER + (struct cmd_pctype_mapping_get_result, + show, "show"); +cmdline_parse_token_string_t cmd_pctype_mapping_get_port = + TOKEN_STRING_INITIALIZER + (struct cmd_pctype_mapping_get_result, + port, "port"); +cmdline_parse_token_num_t cmd_pctype_mapping_get_port_id = + TOKEN_NUM_INITIALIZER + (struct cmd_pctype_mapping_get_result, + port_id, UINT16); +cmdline_parse_token_string_t cmd_pctype_mapping_get_pctype = + TOKEN_STRING_INITIALIZER + (struct cmd_pctype_mapping_get_result, + pctype, "pctype"); +cmdline_parse_token_string_t cmd_pctype_mapping_get_mapping = + TOKEN_STRING_INITIALIZER + (struct cmd_pctype_mapping_get_result, + mapping, "mapping"); + +static void +cmd_pctype_mapping_get_parsed( + void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_pctype_mapping_get_result *res = parsed_result; + int ret = -ENOTSUP; +#ifdef RTE_LIBRTE_I40E_PMD + struct rte_pmd_i40e_flow_type_mapping + mapping[RTE_PMD_I40E_FLOW_TYPE_MAX]; + int i, j, first_pctype; +#endif + + if (port_id_is_invalid(res->port_id, ENABLED_WARN)) + return; + +#ifdef RTE_LIBRTE_I40E_PMD + ret = rte_pmd_i40e_flow_type_mapping_get(res->port_id, mapping); +#endif + + switch (ret) { + case 0: + break; + case -ENODEV: + printf("invalid port_id %d\n", res->port_id); + return; + case -ENOTSUP: + printf("function not implemented\n"); + return; + default: + printf("programming error: (%s)\n", strerror(-ret)); + return; + } + +#ifdef RTE_LIBRTE_I40E_PMD + for (i = 0; i < RTE_PMD_I40E_FLOW_TYPE_MAX; i++) { + if (mapping[i].pctype != 0ULL) { + first_pctype = 1; + + printf("pctype: "); + for (j = 0; j < RTE_PMD_I40E_PCTYPE_MAX; j++) { + if (mapping[i].pctype & (1ULL << j)) { + printf(first_pctype ? + "%02d" : ",%02d", j); + first_pctype = 0; + } + } + printf(" -> flowtype: %02d\n", mapping[i].flow_type); + } + } +#endif +} + +cmdline_parse_inst_t cmd_pctype_mapping_get = { + .f = cmd_pctype_mapping_get_parsed, + .data = NULL, + .help_str = "show port pctype mapping", + .tokens = { + (void *)&cmd_pctype_mapping_get_show, + (void *)&cmd_pctype_mapping_get_port, + (void *)&cmd_pctype_mapping_get_port_id, + (void *)&cmd_pctype_mapping_get_pctype, + (void *)&cmd_pctype_mapping_get_mapping, + NULL, + }, +}; + +/* port config pctype mapping update */ + +/* Common result structure for port config pctype mapping update */ +struct cmd_pctype_mapping_update_result { + cmdline_fixed_string_t port; + cmdline_fixed_string_t config; + portid_t port_id; + cmdline_fixed_string_t pctype; + cmdline_fixed_string_t mapping; + cmdline_fixed_string_t update; + cmdline_fixed_string_t pctype_list; + uint16_t flow_type; +}; + +/* Common CLI fields for pctype mapping update*/ +cmdline_parse_token_string_t cmd_pctype_mapping_update_port = + TOKEN_STRING_INITIALIZER + (struct cmd_pctype_mapping_update_result, + port, "port"); +cmdline_parse_token_string_t cmd_pctype_mapping_update_config = + TOKEN_STRING_INITIALIZER + (struct cmd_pctype_mapping_update_result, + config, "config"); +cmdline_parse_token_num_t cmd_pctype_mapping_update_port_id = + TOKEN_NUM_INITIALIZER + (struct cmd_pctype_mapping_update_result, + port_id, UINT16); +cmdline_parse_token_string_t cmd_pctype_mapping_update_pctype = + TOKEN_STRING_INITIALIZER + (struct cmd_pctype_mapping_update_result, + pctype, "pctype"); +cmdline_parse_token_string_t cmd_pctype_mapping_update_mapping = + TOKEN_STRING_INITIALIZER + (struct cmd_pctype_mapping_update_result, + mapping, "mapping"); +cmdline_parse_token_string_t cmd_pctype_mapping_update_update = + TOKEN_STRING_INITIALIZER + (struct cmd_pctype_mapping_update_result, + update, "update"); +cmdline_parse_token_string_t cmd_pctype_mapping_update_pc_type = + TOKEN_STRING_INITIALIZER + (struct cmd_pctype_mapping_update_result, + pctype_list, NULL); +cmdline_parse_token_num_t cmd_pctype_mapping_update_flow_type = + TOKEN_NUM_INITIALIZER + (struct cmd_pctype_mapping_update_result, + flow_type, UINT16); + +static void +cmd_pctype_mapping_update_parsed( + void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_pctype_mapping_update_result *res = parsed_result; + int ret = -ENOTSUP; +#ifdef RTE_LIBRTE_I40E_PMD + struct rte_pmd_i40e_flow_type_mapping mapping; + unsigned int i; + unsigned int nb_item; + unsigned int pctype_list[RTE_PMD_I40E_PCTYPE_MAX]; +#endif + + if (port_id_is_invalid(res->port_id, ENABLED_WARN)) + return; + +#ifdef RTE_LIBRTE_I40E_PMD + nb_item = parse_item_list(res->pctype_list, "pctypes", + RTE_PMD_I40E_PCTYPE_MAX, pctype_list, 1); + mapping.flow_type = res->flow_type; + for (i = 0, mapping.pctype = 0ULL; i < nb_item; i++) + mapping.pctype |= (1ULL << pctype_list[i]); + ret = rte_pmd_i40e_flow_type_mapping_update(res->port_id, + &mapping, + 1, + 0); +#endif + + switch (ret) { + case 0: + break; + case -EINVAL: + printf("invalid pctype or flow type\n"); + break; + case -ENODEV: + printf("invalid port_id %d\n", res->port_id); + break; + case -ENOTSUP: + printf("function not implemented\n"); + break; + default: + printf("programming error: (%s)\n", strerror(-ret)); + } +} + +cmdline_parse_inst_t cmd_pctype_mapping_update = { + .f = cmd_pctype_mapping_update_parsed, + .data = NULL, + .help_str = "port config pctype mapping update" + " ", + .tokens = { + (void *)&cmd_pctype_mapping_update_port, + (void *)&cmd_pctype_mapping_update_config, + (void *)&cmd_pctype_mapping_update_port_id, + (void *)&cmd_pctype_mapping_update_pctype, + (void *)&cmd_pctype_mapping_update_mapping, + (void *)&cmd_pctype_mapping_update_update, + (void *)&cmd_pctype_mapping_update_pc_type, + (void *)&cmd_pctype_mapping_update_flow_type, + NULL, + }, +}; + +/* ptype mapping get */ + +/* Common result structure for ptype mapping get */ +struct cmd_ptype_mapping_get_result { + cmdline_fixed_string_t ptype; + cmdline_fixed_string_t mapping; + cmdline_fixed_string_t get; + portid_t port_id; + uint8_t valid_only; +}; + +/* Common CLI fields for ptype mapping get */ +cmdline_parse_token_string_t cmd_ptype_mapping_get_ptype = + TOKEN_STRING_INITIALIZER + (struct cmd_ptype_mapping_get_result, + ptype, "ptype"); +cmdline_parse_token_string_t cmd_ptype_mapping_get_mapping = + TOKEN_STRING_INITIALIZER + (struct cmd_ptype_mapping_get_result, + mapping, "mapping"); +cmdline_parse_token_string_t cmd_ptype_mapping_get_get = + TOKEN_STRING_INITIALIZER + (struct cmd_ptype_mapping_get_result, + get, "get"); +cmdline_parse_token_num_t cmd_ptype_mapping_get_port_id = + TOKEN_NUM_INITIALIZER + (struct cmd_ptype_mapping_get_result, + port_id, UINT16); +cmdline_parse_token_num_t cmd_ptype_mapping_get_valid_only = + TOKEN_NUM_INITIALIZER + (struct cmd_ptype_mapping_get_result, + valid_only, UINT8); + +static void +cmd_ptype_mapping_get_parsed( + void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_ptype_mapping_get_result *res = parsed_result; + int ret = -ENOTSUP; +#ifdef RTE_LIBRTE_I40E_PMD + int max_ptype_num = 256; + struct rte_pmd_i40e_ptype_mapping mapping[max_ptype_num]; + uint16_t count; + int i; +#endif + + if (port_id_is_invalid(res->port_id, ENABLED_WARN)) + return; + +#ifdef RTE_LIBRTE_I40E_PMD + ret = rte_pmd_i40e_ptype_mapping_get(res->port_id, + mapping, + max_ptype_num, + &count, + res->valid_only); +#endif + + switch (ret) { + case 0: + break; + case -ENODEV: + printf("invalid port_id %d\n", res->port_id); + break; + case -ENOTSUP: + printf("function not implemented\n"); + break; + default: + printf("programming error: (%s)\n", strerror(-ret)); + } + +#ifdef RTE_LIBRTE_I40E_PMD + if (!ret) { + for (i = 0; i < count; i++) + printf("%3d\t0x%08x\n", + mapping[i].hw_ptype, mapping[i].sw_ptype); + } +#endif +} + +cmdline_parse_inst_t cmd_ptype_mapping_get = { + .f = cmd_ptype_mapping_get_parsed, + .data = NULL, + .help_str = "ptype mapping get ", + .tokens = { + (void *)&cmd_ptype_mapping_get_ptype, + (void *)&cmd_ptype_mapping_get_mapping, + (void *)&cmd_ptype_mapping_get_get, + (void *)&cmd_ptype_mapping_get_port_id, + (void *)&cmd_ptype_mapping_get_valid_only, + NULL, + }, +}; + +/* ptype mapping replace */ + +/* Common result structure for ptype mapping replace */ +struct cmd_ptype_mapping_replace_result { + cmdline_fixed_string_t ptype; + cmdline_fixed_string_t mapping; + cmdline_fixed_string_t replace; + portid_t port_id; + uint32_t target; + uint8_t mask; + uint32_t pkt_type; +}; + +/* Common CLI fields for ptype mapping replace */ +cmdline_parse_token_string_t cmd_ptype_mapping_replace_ptype = + TOKEN_STRING_INITIALIZER + (struct cmd_ptype_mapping_replace_result, + ptype, "ptype"); +cmdline_parse_token_string_t cmd_ptype_mapping_replace_mapping = + TOKEN_STRING_INITIALIZER + (struct cmd_ptype_mapping_replace_result, + mapping, "mapping"); +cmdline_parse_token_string_t cmd_ptype_mapping_replace_replace = + TOKEN_STRING_INITIALIZER + (struct cmd_ptype_mapping_replace_result, + replace, "replace"); +cmdline_parse_token_num_t cmd_ptype_mapping_replace_port_id = + TOKEN_NUM_INITIALIZER + (struct cmd_ptype_mapping_replace_result, + port_id, UINT16); +cmdline_parse_token_num_t cmd_ptype_mapping_replace_target = + TOKEN_NUM_INITIALIZER + (struct cmd_ptype_mapping_replace_result, + target, UINT32); +cmdline_parse_token_num_t cmd_ptype_mapping_replace_mask = + TOKEN_NUM_INITIALIZER + (struct cmd_ptype_mapping_replace_result, + mask, UINT8); +cmdline_parse_token_num_t cmd_ptype_mapping_replace_pkt_type = + TOKEN_NUM_INITIALIZER + (struct cmd_ptype_mapping_replace_result, + pkt_type, UINT32); + +static void +cmd_ptype_mapping_replace_parsed( + void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_ptype_mapping_replace_result *res = parsed_result; + int ret = -ENOTSUP; + + if (port_id_is_invalid(res->port_id, ENABLED_WARN)) + return; + +#ifdef RTE_LIBRTE_I40E_PMD + ret = rte_pmd_i40e_ptype_mapping_replace(res->port_id, + res->target, + res->mask, + res->pkt_type); +#endif + + switch (ret) { + case 0: + break; + case -EINVAL: + printf("invalid ptype 0x%8x or 0x%8x\n", + res->target, res->pkt_type); + break; + case -ENODEV: + printf("invalid port_id %d\n", res->port_id); + break; + case -ENOTSUP: + printf("function not implemented\n"); + break; + default: + printf("programming error: (%s)\n", strerror(-ret)); + } +} + +cmdline_parse_inst_t cmd_ptype_mapping_replace = { + .f = cmd_ptype_mapping_replace_parsed, + .data = NULL, + .help_str = + "ptype mapping replace ", + .tokens = { + (void *)&cmd_ptype_mapping_replace_ptype, + (void *)&cmd_ptype_mapping_replace_mapping, + (void *)&cmd_ptype_mapping_replace_replace, + (void *)&cmd_ptype_mapping_replace_port_id, + (void *)&cmd_ptype_mapping_replace_target, + (void *)&cmd_ptype_mapping_replace_mask, + (void *)&cmd_ptype_mapping_replace_pkt_type, + NULL, + }, +}; + +/* ptype mapping reset */ + +/* Common result structure for ptype mapping reset */ +struct cmd_ptype_mapping_reset_result { + cmdline_fixed_string_t ptype; + cmdline_fixed_string_t mapping; + cmdline_fixed_string_t reset; + portid_t port_id; +}; + +/* Common CLI fields for ptype mapping reset*/ +cmdline_parse_token_string_t cmd_ptype_mapping_reset_ptype = + TOKEN_STRING_INITIALIZER + (struct cmd_ptype_mapping_reset_result, + ptype, "ptype"); +cmdline_parse_token_string_t cmd_ptype_mapping_reset_mapping = + TOKEN_STRING_INITIALIZER + (struct cmd_ptype_mapping_reset_result, + mapping, "mapping"); +cmdline_parse_token_string_t cmd_ptype_mapping_reset_reset = + TOKEN_STRING_INITIALIZER + (struct cmd_ptype_mapping_reset_result, + reset, "reset"); +cmdline_parse_token_num_t cmd_ptype_mapping_reset_port_id = + TOKEN_NUM_INITIALIZER + (struct cmd_ptype_mapping_reset_result, + port_id, UINT16); + +static void +cmd_ptype_mapping_reset_parsed( + void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_ptype_mapping_reset_result *res = parsed_result; + int ret = -ENOTSUP; + + if (port_id_is_invalid(res->port_id, ENABLED_WARN)) + return; + +#ifdef RTE_LIBRTE_I40E_PMD + ret = rte_pmd_i40e_ptype_mapping_reset(res->port_id); +#endif + + switch (ret) { + case 0: + break; + case -ENODEV: + printf("invalid port_id %d\n", res->port_id); + break; + case -ENOTSUP: + printf("function not implemented\n"); + break; + default: + printf("programming error: (%s)\n", strerror(-ret)); + } +} + +cmdline_parse_inst_t cmd_ptype_mapping_reset = { + .f = cmd_ptype_mapping_reset_parsed, + .data = NULL, + .help_str = "ptype mapping reset ", + .tokens = { + (void *)&cmd_ptype_mapping_reset_ptype, + (void *)&cmd_ptype_mapping_reset_mapping, + (void *)&cmd_ptype_mapping_reset_reset, + (void *)&cmd_ptype_mapping_reset_port_id, + NULL, + }, +}; + +/* ptype mapping update */ + +/* Common result structure for ptype mapping update */ +struct cmd_ptype_mapping_update_result { + cmdline_fixed_string_t ptype; + cmdline_fixed_string_t mapping; + cmdline_fixed_string_t reset; + portid_t port_id; + uint8_t hw_ptype; + uint32_t sw_ptype; +}; + +/* Common CLI fields for ptype mapping update*/ +cmdline_parse_token_string_t cmd_ptype_mapping_update_ptype = + TOKEN_STRING_INITIALIZER + (struct cmd_ptype_mapping_update_result, + ptype, "ptype"); +cmdline_parse_token_string_t cmd_ptype_mapping_update_mapping = + TOKEN_STRING_INITIALIZER + (struct cmd_ptype_mapping_update_result, + mapping, "mapping"); +cmdline_parse_token_string_t cmd_ptype_mapping_update_update = + TOKEN_STRING_INITIALIZER + (struct cmd_ptype_mapping_update_result, + reset, "update"); +cmdline_parse_token_num_t cmd_ptype_mapping_update_port_id = + TOKEN_NUM_INITIALIZER + (struct cmd_ptype_mapping_update_result, + port_id, UINT16); +cmdline_parse_token_num_t cmd_ptype_mapping_update_hw_ptype = + TOKEN_NUM_INITIALIZER + (struct cmd_ptype_mapping_update_result, + hw_ptype, UINT8); +cmdline_parse_token_num_t cmd_ptype_mapping_update_sw_ptype = + TOKEN_NUM_INITIALIZER + (struct cmd_ptype_mapping_update_result, + sw_ptype, UINT32); + +static void +cmd_ptype_mapping_update_parsed( + void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_ptype_mapping_update_result *res = parsed_result; + int ret = -ENOTSUP; +#ifdef RTE_LIBRTE_I40E_PMD + struct rte_pmd_i40e_ptype_mapping mapping; +#endif + if (port_id_is_invalid(res->port_id, ENABLED_WARN)) + return; + +#ifdef RTE_LIBRTE_I40E_PMD + mapping.hw_ptype = res->hw_ptype; + mapping.sw_ptype = res->sw_ptype; + ret = rte_pmd_i40e_ptype_mapping_update(res->port_id, + &mapping, + 1, + 0); +#endif + + switch (ret) { + case 0: + break; + case -EINVAL: + printf("invalid ptype 0x%8x\n", res->sw_ptype); + break; + case -ENODEV: + printf("invalid port_id %d\n", res->port_id); + break; + case -ENOTSUP: + printf("function not implemented\n"); + break; + default: + printf("programming error: (%s)\n", strerror(-ret)); + } +} + +cmdline_parse_inst_t cmd_ptype_mapping_update = { + .f = cmd_ptype_mapping_update_parsed, + .data = NULL, + .help_str = "ptype mapping update ", + .tokens = { + (void *)&cmd_ptype_mapping_update_ptype, + (void *)&cmd_ptype_mapping_update_mapping, + (void *)&cmd_ptype_mapping_update_update, + (void *)&cmd_ptype_mapping_update_port_id, + (void *)&cmd_ptype_mapping_update_hw_ptype, + (void *)&cmd_ptype_mapping_update_sw_ptype, + NULL, + }, +}; + +/* Common result structure for file commands */ +struct cmd_cmdfile_result { + cmdline_fixed_string_t load; + cmdline_fixed_string_t filename; +}; + +/* Common CLI fields for file commands */ +cmdline_parse_token_string_t cmd_load_cmdfile = + TOKEN_STRING_INITIALIZER(struct cmd_cmdfile_result, load, "load"); +cmdline_parse_token_string_t cmd_load_cmdfile_filename = + TOKEN_STRING_INITIALIZER(struct cmd_cmdfile_result, filename, NULL); + +static void +cmd_load_from_file_parsed( + void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_cmdfile_result *res = parsed_result; + + cmdline_read_from_file(res->filename); +} + +cmdline_parse_inst_t cmd_load_from_file = { + .f = cmd_load_from_file_parsed, + .data = NULL, + .help_str = "load ", + .tokens = { + (void *)&cmd_load_cmdfile, + (void *)&cmd_load_cmdfile_filename, + NULL, + }, +}; + /* ******************************************************************************** */ /* list of instructions */ @@ -10606,6 +15546,7 @@ cmdline_parse_ctx_t main_ctx[] = { (cmdline_parse_inst_t *)&cmd_help_brief, (cmdline_parse_inst_t *)&cmd_help_long, (cmdline_parse_inst_t *)&cmd_quit, + (cmdline_parse_inst_t *)&cmd_load_from_file, (cmdline_parse_inst_t *)&cmd_showport, (cmdline_parse_inst_t *)&cmd_showqueue, (cmdline_parse_inst_t *)&cmd_showportall, @@ -10630,12 +15571,10 @@ cmdline_parse_ctx_t main_ctx[] = { (cmdline_parse_inst_t *)&cmd_set_allmulti_mode_all, (cmdline_parse_inst_t *)&cmd_set_flush_rx, (cmdline_parse_inst_t *)&cmd_set_link_check, -#ifdef RTE_NIC_BYPASS (cmdline_parse_inst_t *)&cmd_set_bypass_mode, (cmdline_parse_inst_t *)&cmd_set_bypass_event, (cmdline_parse_inst_t *)&cmd_set_bypass_timeout, (cmdline_parse_inst_t *)&cmd_show_bypass_config, -#endif #ifdef RTE_LIBRTE_PMD_BOND (cmdline_parse_inst_t *) &cmd_set_bonding_mode, (cmdline_parse_inst_t *) &cmd_show_bonding_config, @@ -10646,6 +15585,8 @@ cmdline_parse_ctx_t main_ctx[] = { (cmdline_parse_inst_t *) &cmd_set_bond_mac_addr, (cmdline_parse_inst_t *) &cmd_set_balance_xmit_policy, (cmdline_parse_inst_t *) &cmd_set_bond_mon_period, + (cmdline_parse_inst_t *) &cmd_set_lacp_dedicated_queues, + (cmdline_parse_inst_t *) &cmd_set_bonding_agg_mode_policy, #endif (cmdline_parse_inst_t *)&cmd_vlan_offload, (cmdline_parse_inst_t *)&cmd_vlan_tpid, @@ -10660,6 +15601,14 @@ cmdline_parse_ctx_t main_ctx[] = { (cmdline_parse_inst_t *)&cmd_csum_tunnel, (cmdline_parse_inst_t *)&cmd_tso_set, (cmdline_parse_inst_t *)&cmd_tso_show, + (cmdline_parse_inst_t *)&cmd_tunnel_tso_set, + (cmdline_parse_inst_t *)&cmd_tunnel_tso_show, + (cmdline_parse_inst_t *)&cmd_gro_enable, + (cmdline_parse_inst_t *)&cmd_gro_flush, + (cmdline_parse_inst_t *)&cmd_gro_show, + (cmdline_parse_inst_t *)&cmd_gso_enable, + (cmdline_parse_inst_t *)&cmd_gso_size, + (cmdline_parse_inst_t *)&cmd_gso_show, (cmdline_parse_inst_t *)&cmd_link_flow_control_set, (cmdline_parse_inst_t *)&cmd_link_flow_control_set_rx, (cmdline_parse_inst_t *)&cmd_link_flow_control_set_tx, @@ -10681,6 +15630,7 @@ cmdline_parse_ctx_t main_ctx[] = { (cmdline_parse_inst_t *)&cmd_stop, (cmdline_parse_inst_t *)&cmd_mac_addr, (cmdline_parse_inst_t *)&cmd_set_qmap, + (cmdline_parse_inst_t *)&cmd_set_xstats_hide_zero, (cmdline_parse_inst_t *)&cmd_operate_port, (cmdline_parse_inst_t *)&cmd_operate_specific_port, (cmdline_parse_inst_t *)&cmd_operate_attach_port, @@ -10699,15 +15649,11 @@ cmdline_parse_ctx_t main_ctx[] = { (cmdline_parse_inst_t *)&cmd_config_burst, (cmdline_parse_inst_t *)&cmd_config_thresh, (cmdline_parse_inst_t *)&cmd_config_threshold, - (cmdline_parse_inst_t *)&cmd_set_vf_rxmode, (cmdline_parse_inst_t *)&cmd_set_uc_hash_filter, (cmdline_parse_inst_t *)&cmd_set_uc_all_hash_filter, (cmdline_parse_inst_t *)&cmd_vf_mac_addr_filter, (cmdline_parse_inst_t *)&cmd_set_vf_macvlan_filter, - (cmdline_parse_inst_t *)&cmd_set_vf_traffic, - (cmdline_parse_inst_t *)&cmd_vf_rxvlan_filter, (cmdline_parse_inst_t *)&cmd_queue_rate_limit, - (cmdline_parse_inst_t *)&cmd_vf_rate_limit, (cmdline_parse_inst_t *)&cmd_tunnel_filter, (cmdline_parse_inst_t *)&cmd_tunnel_udp_config, (cmdline_parse_inst_t *)&cmd_global_config, @@ -10742,6 +15688,16 @@ cmdline_parse_ctx_t main_ctx[] = { (cmdline_parse_inst_t *)&cmd_set_hash_global_config, (cmdline_parse_inst_t *)&cmd_set_hash_input_set, (cmdline_parse_inst_t *)&cmd_set_fdir_input_set, + (cmdline_parse_inst_t *)&cmd_flow, + (cmdline_parse_inst_t *)&cmd_add_port_meter_profile_srtcm, + (cmdline_parse_inst_t *)&cmd_add_port_meter_profile_trtcm, + (cmdline_parse_inst_t *)&cmd_del_port_meter_profile, + (cmdline_parse_inst_t *)&cmd_set_port_meter, + (cmdline_parse_inst_t *)&cmd_del_port_meter, + (cmdline_parse_inst_t *)&cmd_set_port_meter_profile, + (cmdline_parse_inst_t *)&cmd_set_port_meter_policer_action, + (cmdline_parse_inst_t *)&cmd_set_port_meter_stats_mask, + (cmdline_parse_inst_t *)&cmd_show_port_meter_stats, (cmdline_parse_inst_t *)&cmd_mcast_addr, (cmdline_parse_inst_t *)&cmd_config_l2_tunnel_eth_type_all, (cmdline_parse_inst_t *)&cmd_config_l2_tunnel_eth_type_specific, @@ -10753,9 +15709,94 @@ cmdline_parse_ctx_t main_ctx[] = { (cmdline_parse_inst_t *)&cmd_config_e_tag_forwarding_en_dis, (cmdline_parse_inst_t *)&cmd_config_e_tag_filter_add, (cmdline_parse_inst_t *)&cmd_config_e_tag_filter_del, + (cmdline_parse_inst_t *)&cmd_set_vf_vlan_anti_spoof, + (cmdline_parse_inst_t *)&cmd_set_vf_mac_anti_spoof, + (cmdline_parse_inst_t *)&cmd_set_vf_vlan_stripq, + (cmdline_parse_inst_t *)&cmd_set_vf_vlan_insert, + (cmdline_parse_inst_t *)&cmd_set_tx_loopback, + (cmdline_parse_inst_t *)&cmd_set_all_queues_drop_en, + (cmdline_parse_inst_t *)&cmd_set_vf_split_drop_en, + (cmdline_parse_inst_t *)&cmd_set_macsec_offload_on, + (cmdline_parse_inst_t *)&cmd_set_macsec_offload_off, + (cmdline_parse_inst_t *)&cmd_set_macsec_sc, + (cmdline_parse_inst_t *)&cmd_set_macsec_sa, + (cmdline_parse_inst_t *)&cmd_set_vf_traffic, + (cmdline_parse_inst_t *)&cmd_set_vf_rxmode, + (cmdline_parse_inst_t *)&cmd_vf_rate_limit, + (cmdline_parse_inst_t *)&cmd_vf_rxvlan_filter, + (cmdline_parse_inst_t *)&cmd_set_vf_mac_addr, + (cmdline_parse_inst_t *)&cmd_set_vf_promisc, + (cmdline_parse_inst_t *)&cmd_set_vf_allmulti, + (cmdline_parse_inst_t *)&cmd_set_vf_broadcast, + (cmdline_parse_inst_t *)&cmd_set_vf_vlan_tag, + (cmdline_parse_inst_t *)&cmd_vf_max_bw, + (cmdline_parse_inst_t *)&cmd_vf_tc_min_bw, + (cmdline_parse_inst_t *)&cmd_vf_tc_max_bw, + (cmdline_parse_inst_t *)&cmd_strict_link_prio, + (cmdline_parse_inst_t *)&cmd_tc_min_bw, +#if defined RTE_LIBRTE_PMD_SOFTNIC && defined RTE_LIBRTE_SCHED + (cmdline_parse_inst_t *)&cmd_set_port_tm_hierarchy_default, +#endif + (cmdline_parse_inst_t *)&cmd_ddp_add, + (cmdline_parse_inst_t *)&cmd_ddp_del, + (cmdline_parse_inst_t *)&cmd_ddp_get_list, + (cmdline_parse_inst_t *)&cmd_ddp_get_info, + (cmdline_parse_inst_t *)&cmd_show_vf_stats, + (cmdline_parse_inst_t *)&cmd_clear_vf_stats, + (cmdline_parse_inst_t *)&cmd_ptype_mapping_get, + (cmdline_parse_inst_t *)&cmd_ptype_mapping_replace, + (cmdline_parse_inst_t *)&cmd_ptype_mapping_reset, + (cmdline_parse_inst_t *)&cmd_ptype_mapping_update, + + (cmdline_parse_inst_t *)&cmd_pctype_mapping_get, + (cmdline_parse_inst_t *)&cmd_pctype_mapping_reset, + (cmdline_parse_inst_t *)&cmd_pctype_mapping_update, + (cmdline_parse_inst_t *)&cmd_queue_region, + (cmdline_parse_inst_t *)&cmd_region_flowtype, + (cmdline_parse_inst_t *)&cmd_user_priority_region, + (cmdline_parse_inst_t *)&cmd_flush_queue_region, + (cmdline_parse_inst_t *)&cmd_show_queue_region_info_all, + (cmdline_parse_inst_t *)&cmd_show_port_tm_cap, + (cmdline_parse_inst_t *)&cmd_show_port_tm_level_cap, + (cmdline_parse_inst_t *)&cmd_show_port_tm_node_cap, + (cmdline_parse_inst_t *)&cmd_show_port_tm_node_type, + (cmdline_parse_inst_t *)&cmd_show_port_tm_node_stats, + (cmdline_parse_inst_t *)&cmd_add_port_tm_node_shaper_profile, + (cmdline_parse_inst_t *)&cmd_del_port_tm_node_shaper_profile, + (cmdline_parse_inst_t *)&cmd_add_port_tm_node_shared_shaper, + (cmdline_parse_inst_t *)&cmd_del_port_tm_node_shared_shaper, + (cmdline_parse_inst_t *)&cmd_add_port_tm_node_wred_profile, + (cmdline_parse_inst_t *)&cmd_del_port_tm_node_wred_profile, + (cmdline_parse_inst_t *)&cmd_set_port_tm_node_shaper_profile, + (cmdline_parse_inst_t *)&cmd_add_port_tm_nonleaf_node, + (cmdline_parse_inst_t *)&cmd_add_port_tm_leaf_node, + (cmdline_parse_inst_t *)&cmd_del_port_tm_node, + (cmdline_parse_inst_t *)&cmd_set_port_tm_node_parent, + (cmdline_parse_inst_t *)&cmd_port_tm_hierarchy_commit, NULL, }; +/* read cmdline commands from file */ +void +cmdline_read_from_file(const char *filename) +{ + struct cmdline *cl; + + cl = cmdline_file_new(main_ctx, "testpmd> ", filename); + if (cl == NULL) { + printf("Failed to create file based cmdline context: %s\n", + filename); + return; + } + + cmdline_interact(cl); + cmdline_quit(cl); + + cmdline_free(cl); + + printf("Read CLI commands from %s\n", filename); +} + /* prompt function, called from main on MASTER lcore */ void prompt(void) @@ -10784,7 +15825,7 @@ cmd_reconfig_device_queue(portid_t id, uint8_t dev, uint8_t queue) if (id == (portid_t)RTE_PORT_ALL) { portid_t pid; - FOREACH_PORT(pid, ports) { + RTE_ETH_FOREACH_DEV(pid) { /* check if need_reconfig has been set to 1 */ if (ports[pid].need_reconfig == 0) ports[pid].need_reconfig = dev; diff --git a/dpdk/app/test-pmd/cmdline_flow.c b/dpdk/app/test-pmd/cmdline_flow.c new file mode 100644 index 00000000..df16d2ab --- /dev/null +++ b/dpdk/app/test-pmd/cmdline_flow.c @@ -0,0 +1,2959 @@ +/*- + * BSD LICENSE + * + * Copyright 2016 6WIND S.A. + * Copyright 2016 Mellanox. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of 6WIND S.A. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "testpmd.h" + +/** Parser token indices. */ +enum index { + /* Special tokens. */ + ZERO = 0, + END, + + /* Common tokens. */ + INTEGER, + UNSIGNED, + PREFIX, + BOOLEAN, + STRING, + MAC_ADDR, + IPV4_ADDR, + IPV6_ADDR, + RULE_ID, + PORT_ID, + GROUP_ID, + PRIORITY_LEVEL, + + /* Top-level command. */ + FLOW, + + /* Sub-level commands. */ + VALIDATE, + CREATE, + DESTROY, + FLUSH, + QUERY, + LIST, + ISOLATE, + + /* Destroy arguments. */ + DESTROY_RULE, + + /* Query arguments. */ + QUERY_ACTION, + + /* List arguments. */ + LIST_GROUP, + + /* Validate/create arguments. */ + GROUP, + PRIORITY, + INGRESS, + EGRESS, + + /* Validate/create pattern. */ + PATTERN, + ITEM_PARAM_IS, + ITEM_PARAM_SPEC, + ITEM_PARAM_LAST, + ITEM_PARAM_MASK, + ITEM_PARAM_PREFIX, + ITEM_NEXT, + ITEM_END, + ITEM_VOID, + ITEM_INVERT, + ITEM_ANY, + ITEM_ANY_NUM, + ITEM_PF, + ITEM_VF, + ITEM_VF_ID, + ITEM_PORT, + ITEM_PORT_INDEX, + ITEM_RAW, + ITEM_RAW_RELATIVE, + ITEM_RAW_SEARCH, + ITEM_RAW_OFFSET, + ITEM_RAW_LIMIT, + ITEM_RAW_PATTERN, + ITEM_ETH, + ITEM_ETH_DST, + ITEM_ETH_SRC, + ITEM_ETH_TYPE, + ITEM_VLAN, + ITEM_VLAN_TPID, + ITEM_VLAN_TCI, + ITEM_VLAN_PCP, + ITEM_VLAN_DEI, + ITEM_VLAN_VID, + ITEM_IPV4, + ITEM_IPV4_TOS, + ITEM_IPV4_TTL, + ITEM_IPV4_PROTO, + ITEM_IPV4_SRC, + ITEM_IPV4_DST, + ITEM_IPV6, + ITEM_IPV6_TC, + ITEM_IPV6_FLOW, + ITEM_IPV6_PROTO, + ITEM_IPV6_HOP, + ITEM_IPV6_SRC, + ITEM_IPV6_DST, + ITEM_ICMP, + ITEM_ICMP_TYPE, + ITEM_ICMP_CODE, + ITEM_UDP, + ITEM_UDP_SRC, + ITEM_UDP_DST, + ITEM_TCP, + ITEM_TCP_SRC, + ITEM_TCP_DST, + ITEM_TCP_FLAGS, + ITEM_SCTP, + ITEM_SCTP_SRC, + ITEM_SCTP_DST, + ITEM_SCTP_TAG, + ITEM_SCTP_CKSUM, + ITEM_VXLAN, + ITEM_VXLAN_VNI, + ITEM_E_TAG, + ITEM_E_TAG_GRP_ECID_B, + ITEM_NVGRE, + ITEM_NVGRE_TNI, + ITEM_MPLS, + ITEM_MPLS_LABEL, + ITEM_GRE, + ITEM_GRE_PROTO, + ITEM_FUZZY, + ITEM_FUZZY_THRESH, + ITEM_GTP, + ITEM_GTP_TEID, + ITEM_GTPC, + ITEM_GTPU, + + /* Validate/create actions. */ + ACTIONS, + ACTION_NEXT, + ACTION_END, + ACTION_VOID, + ACTION_PASSTHRU, + ACTION_MARK, + ACTION_MARK_ID, + ACTION_FLAG, + ACTION_QUEUE, + ACTION_QUEUE_INDEX, + ACTION_DROP, + ACTION_COUNT, + ACTION_DUP, + ACTION_DUP_INDEX, + ACTION_RSS, + ACTION_RSS_QUEUES, + ACTION_RSS_QUEUE, + ACTION_PF, + ACTION_VF, + ACTION_VF_ORIGINAL, + ACTION_VF_ID, + ACTION_METER, + ACTION_METER_ID, +}; + +/** Size of pattern[] field in struct rte_flow_item_raw. */ +#define ITEM_RAW_PATTERN_SIZE 36 + +/** Storage size for struct rte_flow_item_raw including pattern. */ +#define ITEM_RAW_SIZE \ + (offsetof(struct rte_flow_item_raw, pattern) + ITEM_RAW_PATTERN_SIZE) + +/** Number of queue[] entries in struct rte_flow_action_rss. */ +#define ACTION_RSS_NUM 32 + +/** Storage size for struct rte_flow_action_rss including queues. */ +#define ACTION_RSS_SIZE \ + (offsetof(struct rte_flow_action_rss, queue) + \ + sizeof(*((struct rte_flow_action_rss *)0)->queue) * ACTION_RSS_NUM) + +/** Maximum number of subsequent tokens and arguments on the stack. */ +#define CTX_STACK_SIZE 16 + +/** Parser context. */ +struct context { + /** Stack of subsequent token lists to process. */ + const enum index *next[CTX_STACK_SIZE]; + /** Arguments for stacked tokens. */ + const void *args[CTX_STACK_SIZE]; + enum index curr; /**< Current token index. */ + enum index prev; /**< Index of the last token seen. */ + int next_num; /**< Number of entries in next[]. */ + int args_num; /**< Number of entries in args[]. */ + uint32_t eol:1; /**< EOL has been detected. */ + uint32_t last:1; /**< No more arguments. */ + portid_t port; /**< Current port ID (for completions). */ + uint32_t objdata; /**< Object-specific data. */ + void *object; /**< Address of current object for relative offsets. */ + void *objmask; /**< Object a full mask must be written to. */ +}; + +/** Token argument. */ +struct arg { + uint32_t hton:1; /**< Use network byte ordering. */ + uint32_t sign:1; /**< Value is signed. */ + uint32_t offset; /**< Relative offset from ctx->object. */ + uint32_t size; /**< Field size. */ + const uint8_t *mask; /**< Bit-mask to use instead of offset/size. */ +}; + +/** Parser token definition. */ +struct token { + /** Type displayed during completion (defaults to "TOKEN"). */ + const char *type; + /** Help displayed during completion (defaults to token name). */ + const char *help; + /** Private data used by parser functions. */ + const void *priv; + /** + * Lists of subsequent tokens to push on the stack. Each call to the + * parser consumes the last entry of that stack. + */ + const enum index *const *next; + /** Arguments stack for subsequent tokens that need them. */ + const struct arg *const *args; + /** + * Token-processing callback, returns -1 in case of error, the + * length of the matched string otherwise. If NULL, attempts to + * match the token name. + * + * If buf is not NULL, the result should be stored in it according + * to context. An error is returned if not large enough. + */ + int (*call)(struct context *ctx, const struct token *token, + const char *str, unsigned int len, + void *buf, unsigned int size); + /** + * Callback that provides possible values for this token, used for + * completion. Returns -1 in case of error, the number of possible + * values otherwise. If NULL, the token name is used. + * + * If buf is not NULL, entry index ent is written to buf and the + * full length of the entry is returned (same behavior as + * snprintf()). + */ + int (*comp)(struct context *ctx, const struct token *token, + unsigned int ent, char *buf, unsigned int size); + /** Mandatory token name, no default value. */ + const char *name; +}; + +/** Static initializer for the next field. */ +#define NEXT(...) (const enum index *const []){ __VA_ARGS__, NULL, } + +/** Static initializer for a NEXT() entry. */ +#define NEXT_ENTRY(...) (const enum index []){ __VA_ARGS__, ZERO, } + +/** Static initializer for the args field. */ +#define ARGS(...) (const struct arg *const []){ __VA_ARGS__, NULL, } + +/** Static initializer for ARGS() to target a field. */ +#define ARGS_ENTRY(s, f) \ + (&(const struct arg){ \ + .offset = offsetof(s, f), \ + .size = sizeof(((s *)0)->f), \ + }) + +/** Static initializer for ARGS() to target a bit-field. */ +#define ARGS_ENTRY_BF(s, f, b) \ + (&(const struct arg){ \ + .size = sizeof(s), \ + .mask = (const void *)&(const s){ .f = (1 << (b)) - 1 }, \ + }) + +/** Static initializer for ARGS() to target an arbitrary bit-mask. */ +#define ARGS_ENTRY_MASK(s, f, m) \ + (&(const struct arg){ \ + .offset = offsetof(s, f), \ + .size = sizeof(((s *)0)->f), \ + .mask = (const void *)(m), \ + }) + +/** Same as ARGS_ENTRY_MASK() using network byte ordering for the value. */ +#define ARGS_ENTRY_MASK_HTON(s, f, m) \ + (&(const struct arg){ \ + .hton = 1, \ + .offset = offsetof(s, f), \ + .size = sizeof(((s *)0)->f), \ + .mask = (const void *)(m), \ + }) + +/** Static initializer for ARGS() to target a pointer. */ +#define ARGS_ENTRY_PTR(s, f) \ + (&(const struct arg){ \ + .size = sizeof(*((s *)0)->f), \ + }) + +/** Static initializer for ARGS() with arbitrary size. */ +#define ARGS_ENTRY_USZ(s, f, sz) \ + (&(const struct arg){ \ + .offset = offsetof(s, f), \ + .size = (sz), \ + }) + +/** Same as ARGS_ENTRY() using network byte ordering. */ +#define ARGS_ENTRY_HTON(s, f) \ + (&(const struct arg){ \ + .hton = 1, \ + .offset = offsetof(s, f), \ + .size = sizeof(((s *)0)->f), \ + }) + +/** Parser output buffer layout expected by cmd_flow_parsed(). */ +struct buffer { + enum index command; /**< Flow command. */ + portid_t port; /**< Affected port ID. */ + union { + struct { + struct rte_flow_attr attr; + struct rte_flow_item *pattern; + struct rte_flow_action *actions; + uint32_t pattern_n; + uint32_t actions_n; + uint8_t *data; + } vc; /**< Validate/create arguments. */ + struct { + uint32_t *rule; + uint32_t rule_n; + } destroy; /**< Destroy arguments. */ + struct { + uint32_t rule; + enum rte_flow_action_type action; + } query; /**< Query arguments. */ + struct { + uint32_t *group; + uint32_t group_n; + } list; /**< List arguments. */ + struct { + int set; + } isolate; /**< Isolated mode arguments. */ + } args; /**< Command arguments. */ +}; + +/** Private data for pattern items. */ +struct parse_item_priv { + enum rte_flow_item_type type; /**< Item type. */ + uint32_t size; /**< Size of item specification structure. */ +}; + +#define PRIV_ITEM(t, s) \ + (&(const struct parse_item_priv){ \ + .type = RTE_FLOW_ITEM_TYPE_ ## t, \ + .size = s, \ + }) + +/** Private data for actions. */ +struct parse_action_priv { + enum rte_flow_action_type type; /**< Action type. */ + uint32_t size; /**< Size of action configuration structure. */ +}; + +#define PRIV_ACTION(t, s) \ + (&(const struct parse_action_priv){ \ + .type = RTE_FLOW_ACTION_TYPE_ ## t, \ + .size = s, \ + }) + +static const enum index next_vc_attr[] = { + GROUP, + PRIORITY, + INGRESS, + EGRESS, + PATTERN, + ZERO, +}; + +static const enum index next_destroy_attr[] = { + DESTROY_RULE, + END, + ZERO, +}; + +static const enum index next_list_attr[] = { + LIST_GROUP, + END, + ZERO, +}; + +static const enum index item_param[] = { + ITEM_PARAM_IS, + ITEM_PARAM_SPEC, + ITEM_PARAM_LAST, + ITEM_PARAM_MASK, + ITEM_PARAM_PREFIX, + ZERO, +}; + +static const enum index next_item[] = { + ITEM_END, + ITEM_VOID, + ITEM_INVERT, + ITEM_ANY, + ITEM_PF, + ITEM_VF, + ITEM_PORT, + ITEM_RAW, + ITEM_ETH, + ITEM_VLAN, + ITEM_IPV4, + ITEM_IPV6, + ITEM_ICMP, + ITEM_UDP, + ITEM_TCP, + ITEM_SCTP, + ITEM_VXLAN, + ITEM_E_TAG, + ITEM_NVGRE, + ITEM_MPLS, + ITEM_GRE, + ITEM_FUZZY, + ITEM_GTP, + ITEM_GTPC, + ITEM_GTPU, + ZERO, +}; + +static const enum index item_fuzzy[] = { + ITEM_FUZZY_THRESH, + ITEM_NEXT, + ZERO, +}; + +static const enum index item_any[] = { + ITEM_ANY_NUM, + ITEM_NEXT, + ZERO, +}; + +static const enum index item_vf[] = { + ITEM_VF_ID, + ITEM_NEXT, + ZERO, +}; + +static const enum index item_port[] = { + ITEM_PORT_INDEX, + ITEM_NEXT, + ZERO, +}; + +static const enum index item_raw[] = { + ITEM_RAW_RELATIVE, + ITEM_RAW_SEARCH, + ITEM_RAW_OFFSET, + ITEM_RAW_LIMIT, + ITEM_RAW_PATTERN, + ITEM_NEXT, + ZERO, +}; + +static const enum index item_eth[] = { + ITEM_ETH_DST, + ITEM_ETH_SRC, + ITEM_ETH_TYPE, + ITEM_NEXT, + ZERO, +}; + +static const enum index item_vlan[] = { + ITEM_VLAN_TPID, + ITEM_VLAN_TCI, + ITEM_VLAN_PCP, + ITEM_VLAN_DEI, + ITEM_VLAN_VID, + ITEM_NEXT, + ZERO, +}; + +static const enum index item_ipv4[] = { + ITEM_IPV4_TOS, + ITEM_IPV4_TTL, + ITEM_IPV4_PROTO, + ITEM_IPV4_SRC, + ITEM_IPV4_DST, + ITEM_NEXT, + ZERO, +}; + +static const enum index item_ipv6[] = { + ITEM_IPV6_TC, + ITEM_IPV6_FLOW, + ITEM_IPV6_PROTO, + ITEM_IPV6_HOP, + ITEM_IPV6_SRC, + ITEM_IPV6_DST, + ITEM_NEXT, + ZERO, +}; + +static const enum index item_icmp[] = { + ITEM_ICMP_TYPE, + ITEM_ICMP_CODE, + ITEM_NEXT, + ZERO, +}; + +static const enum index item_udp[] = { + ITEM_UDP_SRC, + ITEM_UDP_DST, + ITEM_NEXT, + ZERO, +}; + +static const enum index item_tcp[] = { + ITEM_TCP_SRC, + ITEM_TCP_DST, + ITEM_TCP_FLAGS, + ITEM_NEXT, + ZERO, +}; + +static const enum index item_sctp[] = { + ITEM_SCTP_SRC, + ITEM_SCTP_DST, + ITEM_SCTP_TAG, + ITEM_SCTP_CKSUM, + ITEM_NEXT, + ZERO, +}; + +static const enum index item_vxlan[] = { + ITEM_VXLAN_VNI, + ITEM_NEXT, + ZERO, +}; + +static const enum index item_e_tag[] = { + ITEM_E_TAG_GRP_ECID_B, + ITEM_NEXT, + ZERO, +}; + +static const enum index item_nvgre[] = { + ITEM_NVGRE_TNI, + ITEM_NEXT, + ZERO, +}; + +static const enum index item_mpls[] = { + ITEM_MPLS_LABEL, + ITEM_NEXT, + ZERO, +}; + +static const enum index item_gre[] = { + ITEM_GRE_PROTO, + ITEM_NEXT, + ZERO, +}; + +static const enum index item_gtp[] = { + ITEM_GTP_TEID, + ITEM_NEXT, + ZERO, +}; + +static const enum index next_action[] = { + ACTION_END, + ACTION_VOID, + ACTION_PASSTHRU, + ACTION_MARK, + ACTION_FLAG, + ACTION_QUEUE, + ACTION_DROP, + ACTION_COUNT, + ACTION_DUP, + ACTION_RSS, + ACTION_PF, + ACTION_VF, + ACTION_METER, + ZERO, +}; + +static const enum index action_mark[] = { + ACTION_MARK_ID, + ACTION_NEXT, + ZERO, +}; + +static const enum index action_queue[] = { + ACTION_QUEUE_INDEX, + ACTION_NEXT, + ZERO, +}; + +static const enum index action_dup[] = { + ACTION_DUP_INDEX, + ACTION_NEXT, + ZERO, +}; + +static const enum index action_rss[] = { + ACTION_RSS_QUEUES, + ACTION_NEXT, + ZERO, +}; + +static const enum index action_vf[] = { + ACTION_VF_ORIGINAL, + ACTION_VF_ID, + ACTION_NEXT, + ZERO, +}; + +static const enum index action_meter[] = { + ACTION_METER_ID, + ACTION_NEXT, + ZERO, +}; + +static int parse_init(struct context *, const struct token *, + const char *, unsigned int, + void *, unsigned int); +static int parse_vc(struct context *, const struct token *, + const char *, unsigned int, + void *, unsigned int); +static int parse_vc_spec(struct context *, const struct token *, + const char *, unsigned int, void *, unsigned int); +static int parse_vc_conf(struct context *, const struct token *, + const char *, unsigned int, void *, unsigned int); +static int parse_vc_action_rss_queue(struct context *, const struct token *, + const char *, unsigned int, void *, + unsigned int); +static int parse_destroy(struct context *, const struct token *, + const char *, unsigned int, + void *, unsigned int); +static int parse_flush(struct context *, const struct token *, + const char *, unsigned int, + void *, unsigned int); +static int parse_query(struct context *, const struct token *, + const char *, unsigned int, + void *, unsigned int); +static int parse_action(struct context *, const struct token *, + const char *, unsigned int, + void *, unsigned int); +static int parse_list(struct context *, const struct token *, + const char *, unsigned int, + void *, unsigned int); +static int parse_isolate(struct context *, const struct token *, + const char *, unsigned int, + void *, unsigned int); +static int parse_int(struct context *, const struct token *, + const char *, unsigned int, + void *, unsigned int); +static int parse_prefix(struct context *, const struct token *, + const char *, unsigned int, + void *, unsigned int); +static int parse_boolean(struct context *, const struct token *, + const char *, unsigned int, + void *, unsigned int); +static int parse_string(struct context *, const struct token *, + const char *, unsigned int, + void *, unsigned int); +static int parse_mac_addr(struct context *, const struct token *, + const char *, unsigned int, + void *, unsigned int); +static int parse_ipv4_addr(struct context *, const struct token *, + const char *, unsigned int, + void *, unsigned int); +static int parse_ipv6_addr(struct context *, const struct token *, + const char *, unsigned int, + void *, unsigned int); +static int parse_port(struct context *, const struct token *, + const char *, unsigned int, + void *, unsigned int); +static int comp_none(struct context *, const struct token *, + unsigned int, char *, unsigned int); +static int comp_boolean(struct context *, const struct token *, + unsigned int, char *, unsigned int); +static int comp_action(struct context *, const struct token *, + unsigned int, char *, unsigned int); +static int comp_port(struct context *, const struct token *, + unsigned int, char *, unsigned int); +static int comp_rule_id(struct context *, const struct token *, + unsigned int, char *, unsigned int); +static int comp_vc_action_rss_queue(struct context *, const struct token *, + unsigned int, char *, unsigned int); + +/** Token definitions. */ +static const struct token token_list[] = { + /* Special tokens. */ + [ZERO] = { + .name = "ZERO", + .help = "null entry, abused as the entry point", + .next = NEXT(NEXT_ENTRY(FLOW)), + }, + [END] = { + .name = "", + .type = "RETURN", + .help = "command may end here", + }, + /* Common tokens. */ + [INTEGER] = { + .name = "{int}", + .type = "INTEGER", + .help = "integer value", + .call = parse_int, + .comp = comp_none, + }, + [UNSIGNED] = { + .name = "{unsigned}", + .type = "UNSIGNED", + .help = "unsigned integer value", + .call = parse_int, + .comp = comp_none, + }, + [PREFIX] = { + .name = "{prefix}", + .type = "PREFIX", + .help = "prefix length for bit-mask", + .call = parse_prefix, + .comp = comp_none, + }, + [BOOLEAN] = { + .name = "{boolean}", + .type = "BOOLEAN", + .help = "any boolean value", + .call = parse_boolean, + .comp = comp_boolean, + }, + [STRING] = { + .name = "{string}", + .type = "STRING", + .help = "fixed string", + .call = parse_string, + .comp = comp_none, + }, + [MAC_ADDR] = { + .name = "{MAC address}", + .type = "MAC-48", + .help = "standard MAC address notation", + .call = parse_mac_addr, + .comp = comp_none, + }, + [IPV4_ADDR] = { + .name = "{IPv4 address}", + .type = "IPV4 ADDRESS", + .help = "standard IPv4 address notation", + .call = parse_ipv4_addr, + .comp = comp_none, + }, + [IPV6_ADDR] = { + .name = "{IPv6 address}", + .type = "IPV6 ADDRESS", + .help = "standard IPv6 address notation", + .call = parse_ipv6_addr, + .comp = comp_none, + }, + [RULE_ID] = { + .name = "{rule id}", + .type = "RULE ID", + .help = "rule identifier", + .call = parse_int, + .comp = comp_rule_id, + }, + [PORT_ID] = { + .name = "{port_id}", + .type = "PORT ID", + .help = "port identifier", + .call = parse_port, + .comp = comp_port, + }, + [GROUP_ID] = { + .name = "{group_id}", + .type = "GROUP ID", + .help = "group identifier", + .call = parse_int, + .comp = comp_none, + }, + [PRIORITY_LEVEL] = { + .name = "{level}", + .type = "PRIORITY", + .help = "priority level", + .call = parse_int, + .comp = comp_none, + }, + /* Top-level command. */ + [FLOW] = { + .name = "flow", + .type = "{command} {port_id} [{arg} [...]]", + .help = "manage ingress/egress flow rules", + .next = NEXT(NEXT_ENTRY + (VALIDATE, + CREATE, + DESTROY, + FLUSH, + LIST, + QUERY, + ISOLATE)), + .call = parse_init, + }, + /* Sub-level commands. */ + [VALIDATE] = { + .name = "validate", + .help = "check whether a flow rule can be created", + .next = NEXT(next_vc_attr, NEXT_ENTRY(PORT_ID)), + .args = ARGS(ARGS_ENTRY(struct buffer, port)), + .call = parse_vc, + }, + [CREATE] = { + .name = "create", + .help = "create a flow rule", + .next = NEXT(next_vc_attr, NEXT_ENTRY(PORT_ID)), + .args = ARGS(ARGS_ENTRY(struct buffer, port)), + .call = parse_vc, + }, + [DESTROY] = { + .name = "destroy", + .help = "destroy specific flow rules", + .next = NEXT(NEXT_ENTRY(DESTROY_RULE), NEXT_ENTRY(PORT_ID)), + .args = ARGS(ARGS_ENTRY(struct buffer, port)), + .call = parse_destroy, + }, + [FLUSH] = { + .name = "flush", + .help = "destroy all flow rules", + .next = NEXT(NEXT_ENTRY(PORT_ID)), + .args = ARGS(ARGS_ENTRY(struct buffer, port)), + .call = parse_flush, + }, + [QUERY] = { + .name = "query", + .help = "query an existing flow rule", + .next = NEXT(NEXT_ENTRY(QUERY_ACTION), + NEXT_ENTRY(RULE_ID), + NEXT_ENTRY(PORT_ID)), + .args = ARGS(ARGS_ENTRY(struct buffer, args.query.action), + ARGS_ENTRY(struct buffer, args.query.rule), + ARGS_ENTRY(struct buffer, port)), + .call = parse_query, + }, + [LIST] = { + .name = "list", + .help = "list existing flow rules", + .next = NEXT(next_list_attr, NEXT_ENTRY(PORT_ID)), + .args = ARGS(ARGS_ENTRY(struct buffer, port)), + .call = parse_list, + }, + [ISOLATE] = { + .name = "isolate", + .help = "restrict ingress traffic to the defined flow rules", + .next = NEXT(NEXT_ENTRY(BOOLEAN), + NEXT_ENTRY(PORT_ID)), + .args = ARGS(ARGS_ENTRY(struct buffer, args.isolate.set), + ARGS_ENTRY(struct buffer, port)), + .call = parse_isolate, + }, + /* Destroy arguments. */ + [DESTROY_RULE] = { + .name = "rule", + .help = "specify a rule identifier", + .next = NEXT(next_destroy_attr, NEXT_ENTRY(RULE_ID)), + .args = ARGS(ARGS_ENTRY_PTR(struct buffer, args.destroy.rule)), + .call = parse_destroy, + }, + /* Query arguments. */ + [QUERY_ACTION] = { + .name = "{action}", + .type = "ACTION", + .help = "action to query, must be part of the rule", + .call = parse_action, + .comp = comp_action, + }, + /* List arguments. */ + [LIST_GROUP] = { + .name = "group", + .help = "specify a group", + .next = NEXT(next_list_attr, NEXT_ENTRY(GROUP_ID)), + .args = ARGS(ARGS_ENTRY_PTR(struct buffer, args.list.group)), + .call = parse_list, + }, + /* Validate/create attributes. */ + [GROUP] = { + .name = "group", + .help = "specify a group", + .next = NEXT(next_vc_attr, NEXT_ENTRY(GROUP_ID)), + .args = ARGS(ARGS_ENTRY(struct rte_flow_attr, group)), + .call = parse_vc, + }, + [PRIORITY] = { + .name = "priority", + .help = "specify a priority level", + .next = NEXT(next_vc_attr, NEXT_ENTRY(PRIORITY_LEVEL)), + .args = ARGS(ARGS_ENTRY(struct rte_flow_attr, priority)), + .call = parse_vc, + }, + [INGRESS] = { + .name = "ingress", + .help = "affect rule to ingress", + .next = NEXT(next_vc_attr), + .call = parse_vc, + }, + [EGRESS] = { + .name = "egress", + .help = "affect rule to egress", + .next = NEXT(next_vc_attr), + .call = parse_vc, + }, + /* Validate/create pattern. */ + [PATTERN] = { + .name = "pattern", + .help = "submit a list of pattern items", + .next = NEXT(next_item), + .call = parse_vc, + }, + [ITEM_PARAM_IS] = { + .name = "is", + .help = "match value perfectly (with full bit-mask)", + .call = parse_vc_spec, + }, + [ITEM_PARAM_SPEC] = { + .name = "spec", + .help = "match value according to configured bit-mask", + .call = parse_vc_spec, + }, + [ITEM_PARAM_LAST] = { + .name = "last", + .help = "specify upper bound to establish a range", + .call = parse_vc_spec, + }, + [ITEM_PARAM_MASK] = { + .name = "mask", + .help = "specify bit-mask with relevant bits set to one", + .call = parse_vc_spec, + }, + [ITEM_PARAM_PREFIX] = { + .name = "prefix", + .help = "generate bit-mask from a prefix length", + .call = parse_vc_spec, + }, + [ITEM_NEXT] = { + .name = "/", + .help = "specify next pattern item", + .next = NEXT(next_item), + }, + [ITEM_END] = { + .name = "end", + .help = "end list of pattern items", + .priv = PRIV_ITEM(END, 0), + .next = NEXT(NEXT_ENTRY(ACTIONS)), + .call = parse_vc, + }, + [ITEM_VOID] = { + .name = "void", + .help = "no-op pattern item", + .priv = PRIV_ITEM(VOID, 0), + .next = NEXT(NEXT_ENTRY(ITEM_NEXT)), + .call = parse_vc, + }, + [ITEM_INVERT] = { + .name = "invert", + .help = "perform actions when pattern does not match", + .priv = PRIV_ITEM(INVERT, 0), + .next = NEXT(NEXT_ENTRY(ITEM_NEXT)), + .call = parse_vc, + }, + [ITEM_ANY] = { + .name = "any", + .help = "match any protocol for the current layer", + .priv = PRIV_ITEM(ANY, sizeof(struct rte_flow_item_any)), + .next = NEXT(item_any), + .call = parse_vc, + }, + [ITEM_ANY_NUM] = { + .name = "num", + .help = "number of layers covered", + .next = NEXT(item_any, NEXT_ENTRY(UNSIGNED), item_param), + .args = ARGS(ARGS_ENTRY(struct rte_flow_item_any, num)), + }, + [ITEM_PF] = { + .name = "pf", + .help = "match packets addressed to the physical function", + .priv = PRIV_ITEM(PF, 0), + .next = NEXT(NEXT_ENTRY(ITEM_NEXT)), + .call = parse_vc, + }, + [ITEM_VF] = { + .name = "vf", + .help = "match packets addressed to a virtual function ID", + .priv = PRIV_ITEM(VF, sizeof(struct rte_flow_item_vf)), + .next = NEXT(item_vf), + .call = parse_vc, + }, + [ITEM_VF_ID] = { + .name = "id", + .help = "destination VF ID", + .next = NEXT(item_vf, NEXT_ENTRY(UNSIGNED), item_param), + .args = ARGS(ARGS_ENTRY(struct rte_flow_item_vf, id)), + }, + [ITEM_PORT] = { + .name = "port", + .help = "device-specific physical port index to use", + .priv = PRIV_ITEM(PORT, sizeof(struct rte_flow_item_port)), + .next = NEXT(item_port), + .call = parse_vc, + }, + [ITEM_PORT_INDEX] = { + .name = "index", + .help = "physical port index", + .next = NEXT(item_port, NEXT_ENTRY(UNSIGNED), item_param), + .args = ARGS(ARGS_ENTRY(struct rte_flow_item_port, index)), + }, + [ITEM_RAW] = { + .name = "raw", + .help = "match an arbitrary byte string", + .priv = PRIV_ITEM(RAW, ITEM_RAW_SIZE), + .next = NEXT(item_raw), + .call = parse_vc, + }, + [ITEM_RAW_RELATIVE] = { + .name = "relative", + .help = "look for pattern after the previous item", + .next = NEXT(item_raw, NEXT_ENTRY(BOOLEAN), item_param), + .args = ARGS(ARGS_ENTRY_BF(struct rte_flow_item_raw, + relative, 1)), + }, + [ITEM_RAW_SEARCH] = { + .name = "search", + .help = "search pattern from offset (see also limit)", + .next = NEXT(item_raw, NEXT_ENTRY(BOOLEAN), item_param), + .args = ARGS(ARGS_ENTRY_BF(struct rte_flow_item_raw, + search, 1)), + }, + [ITEM_RAW_OFFSET] = { + .name = "offset", + .help = "absolute or relative offset for pattern", + .next = NEXT(item_raw, NEXT_ENTRY(INTEGER), item_param), + .args = ARGS(ARGS_ENTRY(struct rte_flow_item_raw, offset)), + }, + [ITEM_RAW_LIMIT] = { + .name = "limit", + .help = "search area limit for start of pattern", + .next = NEXT(item_raw, NEXT_ENTRY(UNSIGNED), item_param), + .args = ARGS(ARGS_ENTRY(struct rte_flow_item_raw, limit)), + }, + [ITEM_RAW_PATTERN] = { + .name = "pattern", + .help = "byte string to look for", + .next = NEXT(item_raw, + NEXT_ENTRY(STRING), + NEXT_ENTRY(ITEM_PARAM_IS, + ITEM_PARAM_SPEC, + ITEM_PARAM_MASK)), + .args = ARGS(ARGS_ENTRY(struct rte_flow_item_raw, length), + ARGS_ENTRY_USZ(struct rte_flow_item_raw, + pattern, + ITEM_RAW_PATTERN_SIZE)), + }, + [ITEM_ETH] = { + .name = "eth", + .help = "match Ethernet header", + .priv = PRIV_ITEM(ETH, sizeof(struct rte_flow_item_eth)), + .next = NEXT(item_eth), + .call = parse_vc, + }, + [ITEM_ETH_DST] = { + .name = "dst", + .help = "destination MAC", + .next = NEXT(item_eth, NEXT_ENTRY(MAC_ADDR), item_param), + .args = ARGS(ARGS_ENTRY_HTON(struct rte_flow_item_eth, dst)), + }, + [ITEM_ETH_SRC] = { + .name = "src", + .help = "source MAC", + .next = NEXT(item_eth, NEXT_ENTRY(MAC_ADDR), item_param), + .args = ARGS(ARGS_ENTRY_HTON(struct rte_flow_item_eth, src)), + }, + [ITEM_ETH_TYPE] = { + .name = "type", + .help = "EtherType", + .next = NEXT(item_eth, NEXT_ENTRY(UNSIGNED), item_param), + .args = ARGS(ARGS_ENTRY_HTON(struct rte_flow_item_eth, type)), + }, + [ITEM_VLAN] = { + .name = "vlan", + .help = "match 802.1Q/ad VLAN tag", + .priv = PRIV_ITEM(VLAN, sizeof(struct rte_flow_item_vlan)), + .next = NEXT(item_vlan), + .call = parse_vc, + }, + [ITEM_VLAN_TPID] = { + .name = "tpid", + .help = "tag protocol identifier", + .next = NEXT(item_vlan, NEXT_ENTRY(UNSIGNED), item_param), + .args = ARGS(ARGS_ENTRY_HTON(struct rte_flow_item_vlan, tpid)), + }, + [ITEM_VLAN_TCI] = { + .name = "tci", + .help = "tag control information", + .next = NEXT(item_vlan, NEXT_ENTRY(UNSIGNED), item_param), + .args = ARGS(ARGS_ENTRY_HTON(struct rte_flow_item_vlan, tci)), + }, + [ITEM_VLAN_PCP] = { + .name = "pcp", + .help = "priority code point", + .next = NEXT(item_vlan, NEXT_ENTRY(UNSIGNED), item_param), + .args = ARGS(ARGS_ENTRY_MASK_HTON(struct rte_flow_item_vlan, + tci, "\xe0\x00")), + }, + [ITEM_VLAN_DEI] = { + .name = "dei", + .help = "drop eligible indicator", + .next = NEXT(item_vlan, NEXT_ENTRY(UNSIGNED), item_param), + .args = ARGS(ARGS_ENTRY_MASK_HTON(struct rte_flow_item_vlan, + tci, "\x10\x00")), + }, + [ITEM_VLAN_VID] = { + .name = "vid", + .help = "VLAN identifier", + .next = NEXT(item_vlan, NEXT_ENTRY(UNSIGNED), item_param), + .args = ARGS(ARGS_ENTRY_MASK_HTON(struct rte_flow_item_vlan, + tci, "\x0f\xff")), + }, + [ITEM_IPV4] = { + .name = "ipv4", + .help = "match IPv4 header", + .priv = PRIV_ITEM(IPV4, sizeof(struct rte_flow_item_ipv4)), + .next = NEXT(item_ipv4), + .call = parse_vc, + }, + [ITEM_IPV4_TOS] = { + .name = "tos", + .help = "type of service", + .next = NEXT(item_ipv4, NEXT_ENTRY(UNSIGNED), item_param), + .args = ARGS(ARGS_ENTRY_HTON(struct rte_flow_item_ipv4, + hdr.type_of_service)), + }, + [ITEM_IPV4_TTL] = { + .name = "ttl", + .help = "time to live", + .next = NEXT(item_ipv4, NEXT_ENTRY(UNSIGNED), item_param), + .args = ARGS(ARGS_ENTRY_HTON(struct rte_flow_item_ipv4, + hdr.time_to_live)), + }, + [ITEM_IPV4_PROTO] = { + .name = "proto", + .help = "next protocol ID", + .next = NEXT(item_ipv4, NEXT_ENTRY(UNSIGNED), item_param), + .args = ARGS(ARGS_ENTRY_HTON(struct rte_flow_item_ipv4, + hdr.next_proto_id)), + }, + [ITEM_IPV4_SRC] = { + .name = "src", + .help = "source address", + .next = NEXT(item_ipv4, NEXT_ENTRY(IPV4_ADDR), item_param), + .args = ARGS(ARGS_ENTRY_HTON(struct rte_flow_item_ipv4, + hdr.src_addr)), + }, + [ITEM_IPV4_DST] = { + .name = "dst", + .help = "destination address", + .next = NEXT(item_ipv4, NEXT_ENTRY(IPV4_ADDR), item_param), + .args = ARGS(ARGS_ENTRY_HTON(struct rte_flow_item_ipv4, + hdr.dst_addr)), + }, + [ITEM_IPV6] = { + .name = "ipv6", + .help = "match IPv6 header", + .priv = PRIV_ITEM(IPV6, sizeof(struct rte_flow_item_ipv6)), + .next = NEXT(item_ipv6), + .call = parse_vc, + }, + [ITEM_IPV6_TC] = { + .name = "tc", + .help = "traffic class", + .next = NEXT(item_ipv6, NEXT_ENTRY(UNSIGNED), item_param), + .args = ARGS(ARGS_ENTRY_MASK_HTON(struct rte_flow_item_ipv6, + hdr.vtc_flow, + "\x0f\xf0\x00\x00")), + }, + [ITEM_IPV6_FLOW] = { + .name = "flow", + .help = "flow label", + .next = NEXT(item_ipv6, NEXT_ENTRY(UNSIGNED), item_param), + .args = ARGS(ARGS_ENTRY_MASK_HTON(struct rte_flow_item_ipv6, + hdr.vtc_flow, + "\x00\x0f\xff\xff")), + }, + [ITEM_IPV6_PROTO] = { + .name = "proto", + .help = "protocol (next header)", + .next = NEXT(item_ipv6, NEXT_ENTRY(UNSIGNED), item_param), + .args = ARGS(ARGS_ENTRY_HTON(struct rte_flow_item_ipv6, + hdr.proto)), + }, + [ITEM_IPV6_HOP] = { + .name = "hop", + .help = "hop limit", + .next = NEXT(item_ipv6, NEXT_ENTRY(UNSIGNED), item_param), + .args = ARGS(ARGS_ENTRY_HTON(struct rte_flow_item_ipv6, + hdr.hop_limits)), + }, + [ITEM_IPV6_SRC] = { + .name = "src", + .help = "source address", + .next = NEXT(item_ipv6, NEXT_ENTRY(IPV6_ADDR), item_param), + .args = ARGS(ARGS_ENTRY_HTON(struct rte_flow_item_ipv6, + hdr.src_addr)), + }, + [ITEM_IPV6_DST] = { + .name = "dst", + .help = "destination address", + .next = NEXT(item_ipv6, NEXT_ENTRY(IPV6_ADDR), item_param), + .args = ARGS(ARGS_ENTRY_HTON(struct rte_flow_item_ipv6, + hdr.dst_addr)), + }, + [ITEM_ICMP] = { + .name = "icmp", + .help = "match ICMP header", + .priv = PRIV_ITEM(ICMP, sizeof(struct rte_flow_item_icmp)), + .next = NEXT(item_icmp), + .call = parse_vc, + }, + [ITEM_ICMP_TYPE] = { + .name = "type", + .help = "ICMP packet type", + .next = NEXT(item_icmp, NEXT_ENTRY(UNSIGNED), item_param), + .args = ARGS(ARGS_ENTRY_HTON(struct rte_flow_item_icmp, + hdr.icmp_type)), + }, + [ITEM_ICMP_CODE] = { + .name = "code", + .help = "ICMP packet code", + .next = NEXT(item_icmp, NEXT_ENTRY(UNSIGNED), item_param), + .args = ARGS(ARGS_ENTRY_HTON(struct rte_flow_item_icmp, + hdr.icmp_code)), + }, + [ITEM_UDP] = { + .name = "udp", + .help = "match UDP header", + .priv = PRIV_ITEM(UDP, sizeof(struct rte_flow_item_udp)), + .next = NEXT(item_udp), + .call = parse_vc, + }, + [ITEM_UDP_SRC] = { + .name = "src", + .help = "UDP source port", + .next = NEXT(item_udp, NEXT_ENTRY(UNSIGNED), item_param), + .args = ARGS(ARGS_ENTRY_HTON(struct rte_flow_item_udp, + hdr.src_port)), + }, + [ITEM_UDP_DST] = { + .name = "dst", + .help = "UDP destination port", + .next = NEXT(item_udp, NEXT_ENTRY(UNSIGNED), item_param), + .args = ARGS(ARGS_ENTRY_HTON(struct rte_flow_item_udp, + hdr.dst_port)), + }, + [ITEM_TCP] = { + .name = "tcp", + .help = "match TCP header", + .priv = PRIV_ITEM(TCP, sizeof(struct rte_flow_item_tcp)), + .next = NEXT(item_tcp), + .call = parse_vc, + }, + [ITEM_TCP_SRC] = { + .name = "src", + .help = "TCP source port", + .next = NEXT(item_tcp, NEXT_ENTRY(UNSIGNED), item_param), + .args = ARGS(ARGS_ENTRY_HTON(struct rte_flow_item_tcp, + hdr.src_port)), + }, + [ITEM_TCP_DST] = { + .name = "dst", + .help = "TCP destination port", + .next = NEXT(item_tcp, NEXT_ENTRY(UNSIGNED), item_param), + .args = ARGS(ARGS_ENTRY_HTON(struct rte_flow_item_tcp, + hdr.dst_port)), + }, + [ITEM_TCP_FLAGS] = { + .name = "flags", + .help = "TCP flags", + .next = NEXT(item_tcp, NEXT_ENTRY(UNSIGNED), item_param), + .args = ARGS(ARGS_ENTRY_HTON(struct rte_flow_item_tcp, + hdr.tcp_flags)), + }, + [ITEM_SCTP] = { + .name = "sctp", + .help = "match SCTP header", + .priv = PRIV_ITEM(SCTP, sizeof(struct rte_flow_item_sctp)), + .next = NEXT(item_sctp), + .call = parse_vc, + }, + [ITEM_SCTP_SRC] = { + .name = "src", + .help = "SCTP source port", + .next = NEXT(item_sctp, NEXT_ENTRY(UNSIGNED), item_param), + .args = ARGS(ARGS_ENTRY_HTON(struct rte_flow_item_sctp, + hdr.src_port)), + }, + [ITEM_SCTP_DST] = { + .name = "dst", + .help = "SCTP destination port", + .next = NEXT(item_sctp, NEXT_ENTRY(UNSIGNED), item_param), + .args = ARGS(ARGS_ENTRY_HTON(struct rte_flow_item_sctp, + hdr.dst_port)), + }, + [ITEM_SCTP_TAG] = { + .name = "tag", + .help = "validation tag", + .next = NEXT(item_sctp, NEXT_ENTRY(UNSIGNED), item_param), + .args = ARGS(ARGS_ENTRY_HTON(struct rte_flow_item_sctp, + hdr.tag)), + }, + [ITEM_SCTP_CKSUM] = { + .name = "cksum", + .help = "checksum", + .next = NEXT(item_sctp, NEXT_ENTRY(UNSIGNED), item_param), + .args = ARGS(ARGS_ENTRY_HTON(struct rte_flow_item_sctp, + hdr.cksum)), + }, + [ITEM_VXLAN] = { + .name = "vxlan", + .help = "match VXLAN header", + .priv = PRIV_ITEM(VXLAN, sizeof(struct rte_flow_item_vxlan)), + .next = NEXT(item_vxlan), + .call = parse_vc, + }, + [ITEM_VXLAN_VNI] = { + .name = "vni", + .help = "VXLAN identifier", + .next = NEXT(item_vxlan, NEXT_ENTRY(UNSIGNED), item_param), + .args = ARGS(ARGS_ENTRY_HTON(struct rte_flow_item_vxlan, vni)), + }, + [ITEM_E_TAG] = { + .name = "e_tag", + .help = "match E-Tag header", + .priv = PRIV_ITEM(E_TAG, sizeof(struct rte_flow_item_e_tag)), + .next = NEXT(item_e_tag), + .call = parse_vc, + }, + [ITEM_E_TAG_GRP_ECID_B] = { + .name = "grp_ecid_b", + .help = "GRP and E-CID base", + .next = NEXT(item_e_tag, NEXT_ENTRY(UNSIGNED), item_param), + .args = ARGS(ARGS_ENTRY_MASK_HTON(struct rte_flow_item_e_tag, + rsvd_grp_ecid_b, + "\x3f\xff")), + }, + [ITEM_NVGRE] = { + .name = "nvgre", + .help = "match NVGRE header", + .priv = PRIV_ITEM(NVGRE, sizeof(struct rte_flow_item_nvgre)), + .next = NEXT(item_nvgre), + .call = parse_vc, + }, + [ITEM_NVGRE_TNI] = { + .name = "tni", + .help = "virtual subnet ID", + .next = NEXT(item_nvgre, NEXT_ENTRY(UNSIGNED), item_param), + .args = ARGS(ARGS_ENTRY_HTON(struct rte_flow_item_nvgre, tni)), + }, + [ITEM_MPLS] = { + .name = "mpls", + .help = "match MPLS header", + .priv = PRIV_ITEM(MPLS, sizeof(struct rte_flow_item_mpls)), + .next = NEXT(item_mpls), + .call = parse_vc, + }, + [ITEM_MPLS_LABEL] = { + .name = "label", + .help = "MPLS label", + .next = NEXT(item_mpls, NEXT_ENTRY(UNSIGNED), item_param), + .args = ARGS(ARGS_ENTRY_MASK_HTON(struct rte_flow_item_mpls, + label_tc_s, + "\xff\xff\xf0")), + }, + [ITEM_GRE] = { + .name = "gre", + .help = "match GRE header", + .priv = PRIV_ITEM(GRE, sizeof(struct rte_flow_item_gre)), + .next = NEXT(item_gre), + .call = parse_vc, + }, + [ITEM_GRE_PROTO] = { + .name = "protocol", + .help = "GRE protocol type", + .next = NEXT(item_gre, NEXT_ENTRY(UNSIGNED), item_param), + .args = ARGS(ARGS_ENTRY_HTON(struct rte_flow_item_gre, + protocol)), + }, + [ITEM_FUZZY] = { + .name = "fuzzy", + .help = "fuzzy pattern match, expect faster than default", + .priv = PRIV_ITEM(FUZZY, + sizeof(struct rte_flow_item_fuzzy)), + .next = NEXT(item_fuzzy), + .call = parse_vc, + }, + [ITEM_FUZZY_THRESH] = { + .name = "thresh", + .help = "match accuracy threshold", + .next = NEXT(item_fuzzy, NEXT_ENTRY(UNSIGNED), item_param), + .args = ARGS(ARGS_ENTRY(struct rte_flow_item_fuzzy, + thresh)), + }, + [ITEM_GTP] = { + .name = "gtp", + .help = "match GTP header", + .priv = PRIV_ITEM(GTP, sizeof(struct rte_flow_item_gtp)), + .next = NEXT(item_gtp), + .call = parse_vc, + }, + [ITEM_GTP_TEID] = { + .name = "teid", + .help = "tunnel endpoint identifier", + .next = NEXT(item_gtp, NEXT_ENTRY(UNSIGNED), item_param), + .args = ARGS(ARGS_ENTRY_HTON(struct rte_flow_item_gtp, teid)), + }, + [ITEM_GTPC] = { + .name = "gtpc", + .help = "match GTP header", + .priv = PRIV_ITEM(GTPC, sizeof(struct rte_flow_item_gtp)), + .next = NEXT(item_gtp), + .call = parse_vc, + }, + [ITEM_GTPU] = { + .name = "gtpu", + .help = "match GTP header", + .priv = PRIV_ITEM(GTPU, sizeof(struct rte_flow_item_gtp)), + .next = NEXT(item_gtp), + .call = parse_vc, + }, + + /* Validate/create actions. */ + [ACTIONS] = { + .name = "actions", + .help = "submit a list of associated actions", + .next = NEXT(next_action), + .call = parse_vc, + }, + [ACTION_NEXT] = { + .name = "/", + .help = "specify next action", + .next = NEXT(next_action), + }, + [ACTION_END] = { + .name = "end", + .help = "end list of actions", + .priv = PRIV_ACTION(END, 0), + .call = parse_vc, + }, + [ACTION_VOID] = { + .name = "void", + .help = "no-op action", + .priv = PRIV_ACTION(VOID, 0), + .next = NEXT(NEXT_ENTRY(ACTION_NEXT)), + .call = parse_vc, + }, + [ACTION_PASSTHRU] = { + .name = "passthru", + .help = "let subsequent rule process matched packets", + .priv = PRIV_ACTION(PASSTHRU, 0), + .next = NEXT(NEXT_ENTRY(ACTION_NEXT)), + .call = parse_vc, + }, + [ACTION_MARK] = { + .name = "mark", + .help = "attach 32 bit value to packets", + .priv = PRIV_ACTION(MARK, sizeof(struct rte_flow_action_mark)), + .next = NEXT(action_mark), + .call = parse_vc, + }, + [ACTION_MARK_ID] = { + .name = "id", + .help = "32 bit value to return with packets", + .next = NEXT(action_mark, NEXT_ENTRY(UNSIGNED)), + .args = ARGS(ARGS_ENTRY(struct rte_flow_action_mark, id)), + .call = parse_vc_conf, + }, + [ACTION_FLAG] = { + .name = "flag", + .help = "flag packets", + .priv = PRIV_ACTION(FLAG, 0), + .next = NEXT(NEXT_ENTRY(ACTION_NEXT)), + .call = parse_vc, + }, + [ACTION_QUEUE] = { + .name = "queue", + .help = "assign packets to a given queue index", + .priv = PRIV_ACTION(QUEUE, + sizeof(struct rte_flow_action_queue)), + .next = NEXT(action_queue), + .call = parse_vc, + }, + [ACTION_QUEUE_INDEX] = { + .name = "index", + .help = "queue index to use", + .next = NEXT(action_queue, NEXT_ENTRY(UNSIGNED)), + .args = ARGS(ARGS_ENTRY(struct rte_flow_action_queue, index)), + .call = parse_vc_conf, + }, + [ACTION_DROP] = { + .name = "drop", + .help = "drop packets (note: passthru has priority)", + .priv = PRIV_ACTION(DROP, 0), + .next = NEXT(NEXT_ENTRY(ACTION_NEXT)), + .call = parse_vc, + }, + [ACTION_COUNT] = { + .name = "count", + .help = "enable counters for this rule", + .priv = PRIV_ACTION(COUNT, 0), + .next = NEXT(NEXT_ENTRY(ACTION_NEXT)), + .call = parse_vc, + }, + [ACTION_DUP] = { + .name = "dup", + .help = "duplicate packets to a given queue index", + .priv = PRIV_ACTION(DUP, sizeof(struct rte_flow_action_dup)), + .next = NEXT(action_dup), + .call = parse_vc, + }, + [ACTION_DUP_INDEX] = { + .name = "index", + .help = "queue index to duplicate packets to", + .next = NEXT(action_dup, NEXT_ENTRY(UNSIGNED)), + .args = ARGS(ARGS_ENTRY(struct rte_flow_action_dup, index)), + .call = parse_vc_conf, + }, + [ACTION_RSS] = { + .name = "rss", + .help = "spread packets among several queues", + .priv = PRIV_ACTION(RSS, ACTION_RSS_SIZE), + .next = NEXT(action_rss), + .call = parse_vc, + }, + [ACTION_RSS_QUEUES] = { + .name = "queues", + .help = "queue indices to use", + .next = NEXT(action_rss, NEXT_ENTRY(ACTION_RSS_QUEUE)), + .call = parse_vc_conf, + }, + [ACTION_RSS_QUEUE] = { + .name = "{queue}", + .help = "queue index", + .call = parse_vc_action_rss_queue, + .comp = comp_vc_action_rss_queue, + }, + [ACTION_PF] = { + .name = "pf", + .help = "redirect packets to physical device function", + .priv = PRIV_ACTION(PF, 0), + .next = NEXT(NEXT_ENTRY(ACTION_NEXT)), + .call = parse_vc, + }, + [ACTION_VF] = { + .name = "vf", + .help = "redirect packets to virtual device function", + .priv = PRIV_ACTION(VF, sizeof(struct rte_flow_action_vf)), + .next = NEXT(action_vf), + .call = parse_vc, + }, + [ACTION_VF_ORIGINAL] = { + .name = "original", + .help = "use original VF ID if possible", + .next = NEXT(action_vf, NEXT_ENTRY(BOOLEAN)), + .args = ARGS(ARGS_ENTRY_BF(struct rte_flow_action_vf, + original, 1)), + .call = parse_vc_conf, + }, + [ACTION_VF_ID] = { + .name = "id", + .help = "VF ID to redirect packets to", + .next = NEXT(action_vf, NEXT_ENTRY(UNSIGNED)), + .args = ARGS(ARGS_ENTRY(struct rte_flow_action_vf, id)), + .call = parse_vc_conf, + }, + [ACTION_METER] = { + .name = "meter", + .help = "meter the directed packets at given id", + .priv = PRIV_ACTION(METER, + sizeof(struct rte_flow_action_meter)), + .next = NEXT(action_meter), + .call = parse_vc, + }, + [ACTION_METER_ID] = { + .name = "mtr_id", + .help = "meter id to use", + .next = NEXT(action_meter, NEXT_ENTRY(UNSIGNED)), + .args = ARGS(ARGS_ENTRY(struct rte_flow_action_meter, mtr_id)), + .call = parse_vc_conf, + }, +}; + +/** Remove and return last entry from argument stack. */ +static const struct arg * +pop_args(struct context *ctx) +{ + return ctx->args_num ? ctx->args[--ctx->args_num] : NULL; +} + +/** Add entry on top of the argument stack. */ +static int +push_args(struct context *ctx, const struct arg *arg) +{ + if (ctx->args_num == CTX_STACK_SIZE) + return -1; + ctx->args[ctx->args_num++] = arg; + return 0; +} + +/** Spread value into buffer according to bit-mask. */ +static size_t +arg_entry_bf_fill(void *dst, uintmax_t val, const struct arg *arg) +{ + uint32_t i = arg->size; + uint32_t end = 0; + int sub = 1; + int add = 0; + size_t len = 0; + + if (!arg->mask) + return 0; +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + if (!arg->hton) { + i = 0; + end = arg->size; + sub = 0; + add = 1; + } +#endif + while (i != end) { + unsigned int shift = 0; + uint8_t *buf = (uint8_t *)dst + arg->offset + (i -= sub); + + for (shift = 0; arg->mask[i] >> shift; ++shift) { + if (!(arg->mask[i] & (1 << shift))) + continue; + ++len; + if (!dst) + continue; + *buf &= ~(1 << shift); + *buf |= (val & 1) << shift; + val >>= 1; + } + i += add; + } + return len; +} + +/** Compare a string with a partial one of a given length. */ +static int +strcmp_partial(const char *full, const char *partial, size_t partial_len) +{ + int r = strncmp(full, partial, partial_len); + + if (r) + return r; + if (strlen(full) <= partial_len) + return 0; + return full[partial_len]; +} + +/** + * Parse a prefix length and generate a bit-mask. + * + * Last argument (ctx->args) is retrieved to determine mask size, storage + * location and whether the result must use network byte ordering. + */ +static int +parse_prefix(struct context *ctx, const struct token *token, + const char *str, unsigned int len, + void *buf, unsigned int size) +{ + const struct arg *arg = pop_args(ctx); + static const uint8_t conv[] = "\x00\x80\xc0\xe0\xf0\xf8\xfc\xfe\xff"; + char *end; + uintmax_t u; + unsigned int bytes; + unsigned int extra; + + (void)token; + /* Argument is expected. */ + if (!arg) + return -1; + errno = 0; + u = strtoumax(str, &end, 0); + if (errno || (size_t)(end - str) != len) + goto error; + if (arg->mask) { + uintmax_t v = 0; + + extra = arg_entry_bf_fill(NULL, 0, arg); + if (u > extra) + goto error; + if (!ctx->object) + return len; + extra -= u; + while (u--) + (v <<= 1, v |= 1); + v <<= extra; + if (!arg_entry_bf_fill(ctx->object, v, arg) || + !arg_entry_bf_fill(ctx->objmask, -1, arg)) + goto error; + return len; + } + bytes = u / 8; + extra = u % 8; + size = arg->size; + if (bytes > size || bytes + !!extra > size) + goto error; + if (!ctx->object) + return len; + buf = (uint8_t *)ctx->object + arg->offset; +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + if (!arg->hton) { + memset((uint8_t *)buf + size - bytes, 0xff, bytes); + memset(buf, 0x00, size - bytes); + if (extra) + ((uint8_t *)buf)[size - bytes - 1] = conv[extra]; + } else +#endif + { + memset(buf, 0xff, bytes); + memset((uint8_t *)buf + bytes, 0x00, size - bytes); + if (extra) + ((uint8_t *)buf)[bytes] = conv[extra]; + } + if (ctx->objmask) + memset((uint8_t *)ctx->objmask + arg->offset, 0xff, size); + return len; +error: + push_args(ctx, arg); + return -1; +} + +/** Default parsing function for token name matching. */ +static int +parse_default(struct context *ctx, const struct token *token, + const char *str, unsigned int len, + void *buf, unsigned int size) +{ + (void)ctx; + (void)buf; + (void)size; + if (strcmp_partial(token->name, str, len)) + return -1; + return len; +} + +/** Parse flow command, initialize output buffer for subsequent tokens. */ +static int +parse_init(struct context *ctx, const struct token *token, + const char *str, unsigned int len, + void *buf, unsigned int size) +{ + struct buffer *out = buf; + + /* Token name must match. */ + if (parse_default(ctx, token, str, len, NULL, 0) < 0) + return -1; + /* Nothing else to do if there is no buffer. */ + if (!out) + return len; + /* Make sure buffer is large enough. */ + if (size < sizeof(*out)) + return -1; + /* Initialize buffer. */ + memset(out, 0x00, sizeof(*out)); + memset((uint8_t *)out + sizeof(*out), 0x22, size - sizeof(*out)); + ctx->objdata = 0; + ctx->object = out; + ctx->objmask = NULL; + return len; +} + +/** Parse tokens for validate/create commands. */ +static int +parse_vc(struct context *ctx, const struct token *token, + const char *str, unsigned int len, + void *buf, unsigned int size) +{ + struct buffer *out = buf; + uint8_t *data; + uint32_t data_size; + + /* Token name must match. */ + if (parse_default(ctx, token, str, len, NULL, 0) < 0) + return -1; + /* Nothing else to do if there is no buffer. */ + if (!out) + return len; + if (!out->command) { + if (ctx->curr != VALIDATE && ctx->curr != CREATE) + return -1; + if (sizeof(*out) > size) + return -1; + out->command = ctx->curr; + ctx->objdata = 0; + ctx->object = out; + ctx->objmask = NULL; + out->args.vc.data = (uint8_t *)out + size; + return len; + } + ctx->objdata = 0; + ctx->object = &out->args.vc.attr; + ctx->objmask = NULL; + switch (ctx->curr) { + case GROUP: + case PRIORITY: + return len; + case INGRESS: + out->args.vc.attr.ingress = 1; + return len; + case EGRESS: + out->args.vc.attr.egress = 1; + return len; + case PATTERN: + out->args.vc.pattern = + (void *)RTE_ALIGN_CEIL((uintptr_t)(out + 1), + sizeof(double)); + ctx->object = out->args.vc.pattern; + ctx->objmask = NULL; + return len; + case ACTIONS: + out->args.vc.actions = + (void *)RTE_ALIGN_CEIL((uintptr_t) + (out->args.vc.pattern + + out->args.vc.pattern_n), + sizeof(double)); + ctx->object = out->args.vc.actions; + ctx->objmask = NULL; + return len; + default: + if (!token->priv) + return -1; + break; + } + if (!out->args.vc.actions) { + const struct parse_item_priv *priv = token->priv; + struct rte_flow_item *item = + out->args.vc.pattern + out->args.vc.pattern_n; + + data_size = priv->size * 3; /* spec, last, mask */ + data = (void *)RTE_ALIGN_FLOOR((uintptr_t) + (out->args.vc.data - data_size), + sizeof(double)); + if ((uint8_t *)item + sizeof(*item) > data) + return -1; + *item = (struct rte_flow_item){ + .type = priv->type, + }; + ++out->args.vc.pattern_n; + ctx->object = item; + ctx->objmask = NULL; + } else { + const struct parse_action_priv *priv = token->priv; + struct rte_flow_action *action = + out->args.vc.actions + out->args.vc.actions_n; + + data_size = priv->size; /* configuration */ + data = (void *)RTE_ALIGN_FLOOR((uintptr_t) + (out->args.vc.data - data_size), + sizeof(double)); + if ((uint8_t *)action + sizeof(*action) > data) + return -1; + *action = (struct rte_flow_action){ + .type = priv->type, + }; + ++out->args.vc.actions_n; + ctx->object = action; + ctx->objmask = NULL; + } + memset(data, 0, data_size); + out->args.vc.data = data; + ctx->objdata = data_size; + return len; +} + +/** Parse pattern item parameter type. */ +static int +parse_vc_spec(struct context *ctx, const struct token *token, + const char *str, unsigned int len, + void *buf, unsigned int size) +{ + struct buffer *out = buf; + struct rte_flow_item *item; + uint32_t data_size; + int index; + int objmask = 0; + + (void)size; + /* Token name must match. */ + if (parse_default(ctx, token, str, len, NULL, 0) < 0) + return -1; + /* Parse parameter types. */ + switch (ctx->curr) { + static const enum index prefix[] = NEXT_ENTRY(PREFIX); + + case ITEM_PARAM_IS: + index = 0; + objmask = 1; + break; + case ITEM_PARAM_SPEC: + index = 0; + break; + case ITEM_PARAM_LAST: + index = 1; + break; + case ITEM_PARAM_PREFIX: + /* Modify next token to expect a prefix. */ + if (ctx->next_num < 2) + return -1; + ctx->next[ctx->next_num - 2] = prefix; + /* Fall through. */ + case ITEM_PARAM_MASK: + index = 2; + break; + default: + return -1; + } + /* Nothing else to do if there is no buffer. */ + if (!out) + return len; + if (!out->args.vc.pattern_n) + return -1; + item = &out->args.vc.pattern[out->args.vc.pattern_n - 1]; + data_size = ctx->objdata / 3; /* spec, last, mask */ + /* Point to selected object. */ + ctx->object = out->args.vc.data + (data_size * index); + if (objmask) { + ctx->objmask = out->args.vc.data + (data_size * 2); /* mask */ + item->mask = ctx->objmask; + } else + ctx->objmask = NULL; + /* Update relevant item pointer. */ + *((const void **[]){ &item->spec, &item->last, &item->mask })[index] = + ctx->object; + return len; +} + +/** Parse action configuration field. */ +static int +parse_vc_conf(struct context *ctx, const struct token *token, + const char *str, unsigned int len, + void *buf, unsigned int size) +{ + struct buffer *out = buf; + struct rte_flow_action *action; + + (void)size; + /* Token name must match. */ + if (parse_default(ctx, token, str, len, NULL, 0) < 0) + return -1; + /* Nothing else to do if there is no buffer. */ + if (!out) + return len; + if (!out->args.vc.actions_n) + return -1; + action = &out->args.vc.actions[out->args.vc.actions_n - 1]; + /* Point to selected object. */ + ctx->object = out->args.vc.data; + ctx->objmask = NULL; + /* Update configuration pointer. */ + action->conf = ctx->object; + return len; +} + +/** + * Parse queue field for RSS action. + * + * Valid tokens are queue indices and the "end" token. + */ +static int +parse_vc_action_rss_queue(struct context *ctx, const struct token *token, + const char *str, unsigned int len, + void *buf, unsigned int size) +{ + static const enum index next[] = NEXT_ENTRY(ACTION_RSS_QUEUE); + int ret; + int i; + + (void)token; + (void)buf; + (void)size; + if (ctx->curr != ACTION_RSS_QUEUE) + return -1; + i = ctx->objdata >> 16; + if (!strcmp_partial("end", str, len)) { + ctx->objdata &= 0xffff; + return len; + } + if (i >= ACTION_RSS_NUM) + return -1; + if (push_args(ctx, ARGS_ENTRY(struct rte_flow_action_rss, queue[i]))) + return -1; + ret = parse_int(ctx, token, str, len, NULL, 0); + if (ret < 0) { + pop_args(ctx); + return -1; + } + ++i; + ctx->objdata = i << 16 | (ctx->objdata & 0xffff); + /* Repeat token. */ + if (ctx->next_num == RTE_DIM(ctx->next)) + return -1; + ctx->next[ctx->next_num++] = next; + if (!ctx->object) + return len; + ((struct rte_flow_action_rss *)ctx->object)->num = i; + return len; +} + +/** Parse tokens for destroy command. */ +static int +parse_destroy(struct context *ctx, const struct token *token, + const char *str, unsigned int len, + void *buf, unsigned int size) +{ + struct buffer *out = buf; + + /* Token name must match. */ + if (parse_default(ctx, token, str, len, NULL, 0) < 0) + return -1; + /* Nothing else to do if there is no buffer. */ + if (!out) + return len; + if (!out->command) { + if (ctx->curr != DESTROY) + return -1; + if (sizeof(*out) > size) + return -1; + out->command = ctx->curr; + ctx->objdata = 0; + ctx->object = out; + ctx->objmask = NULL; + out->args.destroy.rule = + (void *)RTE_ALIGN_CEIL((uintptr_t)(out + 1), + sizeof(double)); + return len; + } + if (((uint8_t *)(out->args.destroy.rule + out->args.destroy.rule_n) + + sizeof(*out->args.destroy.rule)) > (uint8_t *)out + size) + return -1; + ctx->objdata = 0; + ctx->object = out->args.destroy.rule + out->args.destroy.rule_n++; + ctx->objmask = NULL; + return len; +} + +/** Parse tokens for flush command. */ +static int +parse_flush(struct context *ctx, const struct token *token, + const char *str, unsigned int len, + void *buf, unsigned int size) +{ + struct buffer *out = buf; + + /* Token name must match. */ + if (parse_default(ctx, token, str, len, NULL, 0) < 0) + return -1; + /* Nothing else to do if there is no buffer. */ + if (!out) + return len; + if (!out->command) { + if (ctx->curr != FLUSH) + return -1; + if (sizeof(*out) > size) + return -1; + out->command = ctx->curr; + ctx->objdata = 0; + ctx->object = out; + ctx->objmask = NULL; + } + return len; +} + +/** Parse tokens for query command. */ +static int +parse_query(struct context *ctx, const struct token *token, + const char *str, unsigned int len, + void *buf, unsigned int size) +{ + struct buffer *out = buf; + + /* Token name must match. */ + if (parse_default(ctx, token, str, len, NULL, 0) < 0) + return -1; + /* Nothing else to do if there is no buffer. */ + if (!out) + return len; + if (!out->command) { + if (ctx->curr != QUERY) + return -1; + if (sizeof(*out) > size) + return -1; + out->command = ctx->curr; + ctx->objdata = 0; + ctx->object = out; + ctx->objmask = NULL; + } + return len; +} + +/** Parse action names. */ +static int +parse_action(struct context *ctx, const struct token *token, + const char *str, unsigned int len, + void *buf, unsigned int size) +{ + struct buffer *out = buf; + const struct arg *arg = pop_args(ctx); + unsigned int i; + + (void)size; + /* Argument is expected. */ + if (!arg) + return -1; + /* Parse action name. */ + for (i = 0; next_action[i]; ++i) { + const struct parse_action_priv *priv; + + token = &token_list[next_action[i]]; + if (strcmp_partial(token->name, str, len)) + continue; + priv = token->priv; + if (!priv) + goto error; + if (out) + memcpy((uint8_t *)ctx->object + arg->offset, + &priv->type, + arg->size); + return len; + } +error: + push_args(ctx, arg); + return -1; +} + +/** Parse tokens for list command. */ +static int +parse_list(struct context *ctx, const struct token *token, + const char *str, unsigned int len, + void *buf, unsigned int size) +{ + struct buffer *out = buf; + + /* Token name must match. */ + if (parse_default(ctx, token, str, len, NULL, 0) < 0) + return -1; + /* Nothing else to do if there is no buffer. */ + if (!out) + return len; + if (!out->command) { + if (ctx->curr != LIST) + return -1; + if (sizeof(*out) > size) + return -1; + out->command = ctx->curr; + ctx->objdata = 0; + ctx->object = out; + ctx->objmask = NULL; + out->args.list.group = + (void *)RTE_ALIGN_CEIL((uintptr_t)(out + 1), + sizeof(double)); + return len; + } + if (((uint8_t *)(out->args.list.group + out->args.list.group_n) + + sizeof(*out->args.list.group)) > (uint8_t *)out + size) + return -1; + ctx->objdata = 0; + ctx->object = out->args.list.group + out->args.list.group_n++; + ctx->objmask = NULL; + return len; +} + +/** Parse tokens for isolate command. */ +static int +parse_isolate(struct context *ctx, const struct token *token, + const char *str, unsigned int len, + void *buf, unsigned int size) +{ + struct buffer *out = buf; + + /* Token name must match. */ + if (parse_default(ctx, token, str, len, NULL, 0) < 0) + return -1; + /* Nothing else to do if there is no buffer. */ + if (!out) + return len; + if (!out->command) { + if (ctx->curr != ISOLATE) + return -1; + if (sizeof(*out) > size) + return -1; + out->command = ctx->curr; + ctx->objdata = 0; + ctx->object = out; + ctx->objmask = NULL; + } + return len; +} + +/** + * Parse signed/unsigned integers 8 to 64-bit long. + * + * Last argument (ctx->args) is retrieved to determine integer type and + * storage location. + */ +static int +parse_int(struct context *ctx, const struct token *token, + const char *str, unsigned int len, + void *buf, unsigned int size) +{ + const struct arg *arg = pop_args(ctx); + uintmax_t u; + char *end; + + (void)token; + /* Argument is expected. */ + if (!arg) + return -1; + errno = 0; + u = arg->sign ? + (uintmax_t)strtoimax(str, &end, 0) : + strtoumax(str, &end, 0); + if (errno || (size_t)(end - str) != len) + goto error; + if (!ctx->object) + return len; + if (arg->mask) { + if (!arg_entry_bf_fill(ctx->object, u, arg) || + !arg_entry_bf_fill(ctx->objmask, -1, arg)) + goto error; + return len; + } + buf = (uint8_t *)ctx->object + arg->offset; + size = arg->size; +objmask: + switch (size) { + case sizeof(uint8_t): + *(uint8_t *)buf = u; + break; + case sizeof(uint16_t): + *(uint16_t *)buf = arg->hton ? rte_cpu_to_be_16(u) : u; + break; + case sizeof(uint8_t [3]): +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + if (!arg->hton) { + ((uint8_t *)buf)[0] = u; + ((uint8_t *)buf)[1] = u >> 8; + ((uint8_t *)buf)[2] = u >> 16; + break; + } +#endif + ((uint8_t *)buf)[0] = u >> 16; + ((uint8_t *)buf)[1] = u >> 8; + ((uint8_t *)buf)[2] = u; + break; + case sizeof(uint32_t): + *(uint32_t *)buf = arg->hton ? rte_cpu_to_be_32(u) : u; + break; + case sizeof(uint64_t): + *(uint64_t *)buf = arg->hton ? rte_cpu_to_be_64(u) : u; + break; + default: + goto error; + } + if (ctx->objmask && buf != (uint8_t *)ctx->objmask + arg->offset) { + u = -1; + buf = (uint8_t *)ctx->objmask + arg->offset; + goto objmask; + } + return len; +error: + push_args(ctx, arg); + return -1; +} + +/** + * Parse a string. + * + * Two arguments (ctx->args) are retrieved from the stack to store data and + * its length (in that order). + */ +static int +parse_string(struct context *ctx, const struct token *token, + const char *str, unsigned int len, + void *buf, unsigned int size) +{ + const struct arg *arg_data = pop_args(ctx); + const struct arg *arg_len = pop_args(ctx); + char tmp[16]; /* Ought to be enough. */ + int ret; + + /* Arguments are expected. */ + if (!arg_data) + return -1; + if (!arg_len) { + push_args(ctx, arg_data); + return -1; + } + size = arg_data->size; + /* Bit-mask fill is not supported. */ + if (arg_data->mask || size < len) + goto error; + if (!ctx->object) + return len; + /* Let parse_int() fill length information first. */ + ret = snprintf(tmp, sizeof(tmp), "%u", len); + if (ret < 0) + goto error; + push_args(ctx, arg_len); + ret = parse_int(ctx, token, tmp, ret, NULL, 0); + if (ret < 0) { + pop_args(ctx); + goto error; + } + buf = (uint8_t *)ctx->object + arg_data->offset; + /* Output buffer is not necessarily NUL-terminated. */ + memcpy(buf, str, len); + memset((uint8_t *)buf + len, 0x55, size - len); + if (ctx->objmask) + memset((uint8_t *)ctx->objmask + arg_data->offset, 0xff, len); + return len; +error: + push_args(ctx, arg_len); + push_args(ctx, arg_data); + return -1; +} + +/** + * Parse a MAC address. + * + * Last argument (ctx->args) is retrieved to determine storage size and + * location. + */ +static int +parse_mac_addr(struct context *ctx, const struct token *token, + const char *str, unsigned int len, + void *buf, unsigned int size) +{ + const struct arg *arg = pop_args(ctx); + struct ether_addr tmp; + int ret; + + (void)token; + /* Argument is expected. */ + if (!arg) + return -1; + size = arg->size; + /* Bit-mask fill is not supported. */ + if (arg->mask || size != sizeof(tmp)) + goto error; + /* Only network endian is supported. */ + if (!arg->hton) + goto error; + ret = cmdline_parse_etheraddr(NULL, str, &tmp, size); + if (ret < 0 || (unsigned int)ret != len) + goto error; + if (!ctx->object) + return len; + buf = (uint8_t *)ctx->object + arg->offset; + memcpy(buf, &tmp, size); + if (ctx->objmask) + memset((uint8_t *)ctx->objmask + arg->offset, 0xff, size); + return len; +error: + push_args(ctx, arg); + return -1; +} + +/** + * Parse an IPv4 address. + * + * Last argument (ctx->args) is retrieved to determine storage size and + * location. + */ +static int +parse_ipv4_addr(struct context *ctx, const struct token *token, + const char *str, unsigned int len, + void *buf, unsigned int size) +{ + const struct arg *arg = pop_args(ctx); + char str2[len + 1]; + struct in_addr tmp; + int ret; + + /* Argument is expected. */ + if (!arg) + return -1; + size = arg->size; + /* Bit-mask fill is not supported. */ + if (arg->mask || size != sizeof(tmp)) + goto error; + /* Only network endian is supported. */ + if (!arg->hton) + goto error; + memcpy(str2, str, len); + str2[len] = '\0'; + ret = inet_pton(AF_INET, str2, &tmp); + if (ret != 1) { + /* Attempt integer parsing. */ + push_args(ctx, arg); + return parse_int(ctx, token, str, len, buf, size); + } + if (!ctx->object) + return len; + buf = (uint8_t *)ctx->object + arg->offset; + memcpy(buf, &tmp, size); + if (ctx->objmask) + memset((uint8_t *)ctx->objmask + arg->offset, 0xff, size); + return len; +error: + push_args(ctx, arg); + return -1; +} + +/** + * Parse an IPv6 address. + * + * Last argument (ctx->args) is retrieved to determine storage size and + * location. + */ +static int +parse_ipv6_addr(struct context *ctx, const struct token *token, + const char *str, unsigned int len, + void *buf, unsigned int size) +{ + const struct arg *arg = pop_args(ctx); + char str2[len + 1]; + struct in6_addr tmp; + int ret; + + (void)token; + /* Argument is expected. */ + if (!arg) + return -1; + size = arg->size; + /* Bit-mask fill is not supported. */ + if (arg->mask || size != sizeof(tmp)) + goto error; + /* Only network endian is supported. */ + if (!arg->hton) + goto error; + memcpy(str2, str, len); + str2[len] = '\0'; + ret = inet_pton(AF_INET6, str2, &tmp); + if (ret != 1) + goto error; + if (!ctx->object) + return len; + buf = (uint8_t *)ctx->object + arg->offset; + memcpy(buf, &tmp, size); + if (ctx->objmask) + memset((uint8_t *)ctx->objmask + arg->offset, 0xff, size); + return len; +error: + push_args(ctx, arg); + return -1; +} + +/** Boolean values (even indices stand for false). */ +static const char *const boolean_name[] = { + "0", "1", + "false", "true", + "no", "yes", + "N", "Y", + NULL, +}; + +/** + * Parse a boolean value. + * + * Last argument (ctx->args) is retrieved to determine storage size and + * location. + */ +static int +parse_boolean(struct context *ctx, const struct token *token, + const char *str, unsigned int len, + void *buf, unsigned int size) +{ + const struct arg *arg = pop_args(ctx); + unsigned int i; + int ret; + + /* Argument is expected. */ + if (!arg) + return -1; + for (i = 0; boolean_name[i]; ++i) + if (!strcmp_partial(boolean_name[i], str, len)) + break; + /* Process token as integer. */ + if (boolean_name[i]) + str = i & 1 ? "1" : "0"; + push_args(ctx, arg); + ret = parse_int(ctx, token, str, strlen(str), buf, size); + return ret > 0 ? (int)len : ret; +} + +/** Parse port and update context. */ +static int +parse_port(struct context *ctx, const struct token *token, + const char *str, unsigned int len, + void *buf, unsigned int size) +{ + struct buffer *out = &(struct buffer){ .port = 0 }; + int ret; + + if (buf) + out = buf; + else { + ctx->objdata = 0; + ctx->object = out; + ctx->objmask = NULL; + size = sizeof(*out); + } + ret = parse_int(ctx, token, str, len, out, size); + if (ret >= 0) + ctx->port = out->port; + if (!buf) + ctx->object = NULL; + return ret; +} + +/** No completion. */ +static int +comp_none(struct context *ctx, const struct token *token, + unsigned int ent, char *buf, unsigned int size) +{ + (void)ctx; + (void)token; + (void)ent; + (void)buf; + (void)size; + return 0; +} + +/** Complete boolean values. */ +static int +comp_boolean(struct context *ctx, const struct token *token, + unsigned int ent, char *buf, unsigned int size) +{ + unsigned int i; + + (void)ctx; + (void)token; + for (i = 0; boolean_name[i]; ++i) + if (buf && i == ent) + return snprintf(buf, size, "%s", boolean_name[i]); + if (buf) + return -1; + return i; +} + +/** Complete action names. */ +static int +comp_action(struct context *ctx, const struct token *token, + unsigned int ent, char *buf, unsigned int size) +{ + unsigned int i; + + (void)ctx; + (void)token; + for (i = 0; next_action[i]; ++i) + if (buf && i == ent) + return snprintf(buf, size, "%s", + token_list[next_action[i]].name); + if (buf) + return -1; + return i; +} + +/** Complete available ports. */ +static int +comp_port(struct context *ctx, const struct token *token, + unsigned int ent, char *buf, unsigned int size) +{ + unsigned int i = 0; + portid_t p; + + (void)ctx; + (void)token; + RTE_ETH_FOREACH_DEV(p) { + if (buf && i == ent) + return snprintf(buf, size, "%u", p); + ++i; + } + if (buf) + return -1; + return i; +} + +/** Complete available rule IDs. */ +static int +comp_rule_id(struct context *ctx, const struct token *token, + unsigned int ent, char *buf, unsigned int size) +{ + unsigned int i = 0; + struct rte_port *port; + struct port_flow *pf; + + (void)token; + if (port_id_is_invalid(ctx->port, DISABLED_WARN) || + ctx->port == (portid_t)RTE_PORT_ALL) + return -1; + port = &ports[ctx->port]; + for (pf = port->flow_list; pf != NULL; pf = pf->next) { + if (buf && i == ent) + return snprintf(buf, size, "%u", pf->id); + ++i; + } + if (buf) + return -1; + return i; +} + +/** Complete queue field for RSS action. */ +static int +comp_vc_action_rss_queue(struct context *ctx, const struct token *token, + unsigned int ent, char *buf, unsigned int size) +{ + static const char *const str[] = { "", "end", NULL }; + unsigned int i; + + (void)ctx; + (void)token; + for (i = 0; str[i] != NULL; ++i) + if (buf && i == ent) + return snprintf(buf, size, "%s", str[i]); + if (buf) + return -1; + return i; +} + +/** Internal context. */ +static struct context cmd_flow_context; + +/** Global parser instance (cmdline API). */ +cmdline_parse_inst_t cmd_flow; + +/** Initialize context. */ +static void +cmd_flow_context_init(struct context *ctx) +{ + /* A full memset() is not necessary. */ + ctx->curr = ZERO; + ctx->prev = ZERO; + ctx->next_num = 0; + ctx->args_num = 0; + ctx->eol = 0; + ctx->last = 0; + ctx->port = 0; + ctx->objdata = 0; + ctx->object = NULL; + ctx->objmask = NULL; +} + +/** Parse a token (cmdline API). */ +static int +cmd_flow_parse(cmdline_parse_token_hdr_t *hdr, const char *src, void *result, + unsigned int size) +{ + struct context *ctx = &cmd_flow_context; + const struct token *token; + const enum index *list; + int len; + int i; + + (void)hdr; + token = &token_list[ctx->curr]; + /* Check argument length. */ + ctx->eol = 0; + ctx->last = 1; + for (len = 0; src[len]; ++len) + if (src[len] == '#' || isspace(src[len])) + break; + if (!len) + return -1; + /* Last argument and EOL detection. */ + for (i = len; src[i]; ++i) + if (src[i] == '#' || src[i] == '\r' || src[i] == '\n') + break; + else if (!isspace(src[i])) { + ctx->last = 0; + break; + } + for (; src[i]; ++i) + if (src[i] == '\r' || src[i] == '\n') { + ctx->eol = 1; + break; + } + /* Initialize context if necessary. */ + if (!ctx->next_num) { + if (!token->next) + return 0; + ctx->next[ctx->next_num++] = token->next[0]; + } + /* Process argument through candidates. */ + ctx->prev = ctx->curr; + list = ctx->next[ctx->next_num - 1]; + for (i = 0; list[i]; ++i) { + const struct token *next = &token_list[list[i]]; + int tmp; + + ctx->curr = list[i]; + if (next->call) + tmp = next->call(ctx, next, src, len, result, size); + else + tmp = parse_default(ctx, next, src, len, result, size); + if (tmp == -1 || tmp != len) + continue; + token = next; + break; + } + if (!list[i]) + return -1; + --ctx->next_num; + /* Push subsequent tokens if any. */ + if (token->next) + for (i = 0; token->next[i]; ++i) { + if (ctx->next_num == RTE_DIM(ctx->next)) + return -1; + ctx->next[ctx->next_num++] = token->next[i]; + } + /* Push arguments if any. */ + if (token->args) + for (i = 0; token->args[i]; ++i) { + if (ctx->args_num == RTE_DIM(ctx->args)) + return -1; + ctx->args[ctx->args_num++] = token->args[i]; + } + return len; +} + +/** Return number of completion entries (cmdline API). */ +static int +cmd_flow_complete_get_nb(cmdline_parse_token_hdr_t *hdr) +{ + struct context *ctx = &cmd_flow_context; + const struct token *token = &token_list[ctx->curr]; + const enum index *list; + int i; + + (void)hdr; + /* Count number of tokens in current list. */ + if (ctx->next_num) + list = ctx->next[ctx->next_num - 1]; + else + list = token->next[0]; + for (i = 0; list[i]; ++i) + ; + if (!i) + return 0; + /* + * If there is a single token, use its completion callback, otherwise + * return the number of entries. + */ + token = &token_list[list[0]]; + if (i == 1 && token->comp) { + /* Save index for cmd_flow_get_help(). */ + ctx->prev = list[0]; + return token->comp(ctx, token, 0, NULL, 0); + } + return i; +} + +/** Return a completion entry (cmdline API). */ +static int +cmd_flow_complete_get_elt(cmdline_parse_token_hdr_t *hdr, int index, + char *dst, unsigned int size) +{ + struct context *ctx = &cmd_flow_context; + const struct token *token = &token_list[ctx->curr]; + const enum index *list; + int i; + + (void)hdr; + /* Count number of tokens in current list. */ + if (ctx->next_num) + list = ctx->next[ctx->next_num - 1]; + else + list = token->next[0]; + for (i = 0; list[i]; ++i) + ; + if (!i) + return -1; + /* If there is a single token, use its completion callback. */ + token = &token_list[list[0]]; + if (i == 1 && token->comp) { + /* Save index for cmd_flow_get_help(). */ + ctx->prev = list[0]; + return token->comp(ctx, token, index, dst, size) < 0 ? -1 : 0; + } + /* Otherwise make sure the index is valid and use defaults. */ + if (index >= i) + return -1; + token = &token_list[list[index]]; + snprintf(dst, size, "%s", token->name); + /* Save index for cmd_flow_get_help(). */ + ctx->prev = list[index]; + return 0; +} + +/** Populate help strings for current token (cmdline API). */ +static int +cmd_flow_get_help(cmdline_parse_token_hdr_t *hdr, char *dst, unsigned int size) +{ + struct context *ctx = &cmd_flow_context; + const struct token *token = &token_list[ctx->prev]; + + (void)hdr; + if (!size) + return -1; + /* Set token type and update global help with details. */ + snprintf(dst, size, "%s", (token->type ? token->type : "TOKEN")); + if (token->help) + cmd_flow.help_str = token->help; + else + cmd_flow.help_str = token->name; + return 0; +} + +/** Token definition template (cmdline API). */ +static struct cmdline_token_hdr cmd_flow_token_hdr = { + .ops = &(struct cmdline_token_ops){ + .parse = cmd_flow_parse, + .complete_get_nb = cmd_flow_complete_get_nb, + .complete_get_elt = cmd_flow_complete_get_elt, + .get_help = cmd_flow_get_help, + }, + .offset = 0, +}; + +/** Populate the next dynamic token. */ +static void +cmd_flow_tok(cmdline_parse_token_hdr_t **hdr, + cmdline_parse_token_hdr_t **hdr_inst) +{ + struct context *ctx = &cmd_flow_context; + + /* Always reinitialize context before requesting the first token. */ + if (!(hdr_inst - cmd_flow.tokens)) + cmd_flow_context_init(ctx); + /* Return NULL when no more tokens are expected. */ + if (!ctx->next_num && ctx->curr) { + *hdr = NULL; + return; + } + /* Determine if command should end here. */ + if (ctx->eol && ctx->last && ctx->next_num) { + const enum index *list = ctx->next[ctx->next_num - 1]; + int i; + + for (i = 0; list[i]; ++i) { + if (list[i] != END) + continue; + *hdr = NULL; + return; + } + } + *hdr = &cmd_flow_token_hdr; +} + +/** Dispatch parsed buffer to function calls. */ +static void +cmd_flow_parsed(const struct buffer *in) +{ + switch (in->command) { + case VALIDATE: + port_flow_validate(in->port, &in->args.vc.attr, + in->args.vc.pattern, in->args.vc.actions); + break; + case CREATE: + port_flow_create(in->port, &in->args.vc.attr, + in->args.vc.pattern, in->args.vc.actions); + break; + case DESTROY: + port_flow_destroy(in->port, in->args.destroy.rule_n, + in->args.destroy.rule); + break; + case FLUSH: + port_flow_flush(in->port); + break; + case QUERY: + port_flow_query(in->port, in->args.query.rule, + in->args.query.action); + break; + case LIST: + port_flow_list(in->port, in->args.list.group_n, + in->args.list.group); + break; + case ISOLATE: + port_flow_isolate(in->port, in->args.isolate.set); + break; + default: + break; + } +} + +/** Token generator and output processing callback (cmdline API). */ +static void +cmd_flow_cb(void *arg0, struct cmdline *cl, void *arg2) +{ + if (cl == NULL) + cmd_flow_tok(arg0, arg2); + else + cmd_flow_parsed(arg0); +} + +/** Global parser instance (cmdline API). */ +cmdline_parse_inst_t cmd_flow = { + .f = cmd_flow_cb, + .data = NULL, /**< Unused. */ + .help_str = NULL, /**< Updated by cmd_flow_get_help(). */ + .tokens = { + NULL, + }, /**< Tokens are returned by cmd_flow_tok(). */ +}; diff --git a/dpdk/app/test-pmd/cmdline_mtr.c b/dpdk/app/test-pmd/cmdline_mtr.c new file mode 100644 index 00000000..d8d806d7 --- /dev/null +++ b/dpdk/app/test-pmd/cmdline_mtr.c @@ -0,0 +1,1018 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include + +#include +#include +#include + +#include "testpmd.h" +#include "cmdline_mtr.h" + +/** Display Meter Error Message */ +static void +print_err_msg(struct rte_mtr_error *error) +{ + static const char *const errstrlist[] = { + [RTE_MTR_ERROR_TYPE_NONE] = "no error", + [RTE_MTR_ERROR_TYPE_UNSPECIFIED] = "cause unspecified", + [RTE_MTR_ERROR_TYPE_METER_PROFILE_ID] = "meter profile id", + [RTE_MTR_ERROR_TYPE_METER_PROFILE] = "meter profile null", + [RTE_MTR_ERROR_TYPE_MTR_ID] = "meter id", + [RTE_MTR_ERROR_TYPE_MTR_PARAMS] = "meter params null", + [RTE_MTR_ERROR_TYPE_POLICER_ACTION_GREEN] + = "policer action(green)", + [RTE_MTR_ERROR_TYPE_POLICER_ACTION_YELLOW] + = "policer action(yellow)", + [RTE_MTR_ERROR_TYPE_POLICER_ACTION_RED] + = "policer action(red)", + [RTE_MTR_ERROR_TYPE_STATS_MASK] = "stats mask", + [RTE_MTR_ERROR_TYPE_STATS] = "stats", + [RTE_MTR_ERROR_TYPE_SHARED] + = "shared meter", + }; + + const char *errstr; + char buf[64]; + + if ((unsigned int)error->type >= RTE_DIM(errstrlist) || + !errstrlist[error->type]) + errstr = "unknown type"; + else + errstr = errstrlist[error->type]; + + if (error->cause) + snprintf(buf, sizeof(buf), "cause: %p, ", error->cause); + + printf("%s: %s%s (error %d)\n", errstr, error->cause ? buf : "", + error->message ? error->message : "(no stated reason)", + error->type); +} + +static int +string_to_policer_action(char *s) +{ + if (strcmp(s, "G") == 0) + return MTR_POLICER_ACTION_COLOR_GREEN; + + if (strcmp(s, "Y") == 0) + return MTR_POLICER_ACTION_COLOR_YELLOW; + + if (strcmp(s, "R") == 0) + return MTR_POLICER_ACTION_COLOR_RED; + + if (strcmp(s, "D") == 0) + return MTR_POLICER_ACTION_DROP; + + return -1; +} + +/* *** Add Port Meter Profile srtcm_rfc2697 *** */ +struct cmd_add_port_meter_profile_srtcm_result { + cmdline_fixed_string_t add; + cmdline_fixed_string_t port; + cmdline_fixed_string_t meter; + cmdline_fixed_string_t profile; + cmdline_fixed_string_t srtcm_rfc2697; + uint16_t port_id; + uint32_t profile_id; + uint64_t cir; + uint64_t cbs; + uint64_t ebs; + uint8_t color_aware; +}; + +cmdline_parse_token_string_t cmd_add_port_meter_profile_srtcm_add = + TOKEN_STRING_INITIALIZER( + struct cmd_add_port_meter_profile_srtcm_result, add, "add"); +cmdline_parse_token_string_t cmd_add_port_meter_profile_srtcm_port = + TOKEN_STRING_INITIALIZER( + struct cmd_add_port_meter_profile_srtcm_result, + port, "port"); +cmdline_parse_token_string_t cmd_add_port_meter_profile_srtcm_meter = + TOKEN_STRING_INITIALIZER( + struct cmd_add_port_meter_profile_srtcm_result, + meter, "meter"); +cmdline_parse_token_string_t cmd_add_port_meter_profile_srtcm_profile = + TOKEN_STRING_INITIALIZER( + struct cmd_add_port_meter_profile_srtcm_result, + profile, "profile"); +cmdline_parse_token_string_t cmd_add_port_meter_profile_srtcm_srtcm_rfc2697 = + TOKEN_STRING_INITIALIZER( + struct cmd_add_port_meter_profile_srtcm_result, + srtcm_rfc2697, "srtcm_rfc2697"); +cmdline_parse_token_num_t cmd_add_port_meter_profile_srtcm_port_id = + TOKEN_NUM_INITIALIZER( + struct cmd_add_port_meter_profile_srtcm_result, + port_id, UINT16); +cmdline_parse_token_num_t cmd_add_port_meter_profile_srtcm_profile_id = + TOKEN_NUM_INITIALIZER( + struct cmd_add_port_meter_profile_srtcm_result, + profile_id, UINT32); +cmdline_parse_token_num_t cmd_add_port_meter_profile_srtcm_cir = + TOKEN_NUM_INITIALIZER( + struct cmd_add_port_meter_profile_srtcm_result, + cir, UINT64); +cmdline_parse_token_num_t cmd_add_port_meter_profile_srtcm_cbs = + TOKEN_NUM_INITIALIZER( + struct cmd_add_port_meter_profile_srtcm_result, + cbs, UINT64); +cmdline_parse_token_num_t cmd_add_port_meter_profile_srtcm_ebs = + TOKEN_NUM_INITIALIZER( + struct cmd_add_port_meter_profile_srtcm_result, + ebs, UINT64); + +static void cmd_add_port_meter_profile_srtcm_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_add_port_meter_profile_srtcm_result *res = parsed_result; + struct rte_mtr_meter_profile mp; + struct rte_mtr_error error; + uint32_t profile_id = res->profile_id; + uint16_t port_id = res->port_id; + int ret; + + if (port_id_is_invalid(port_id, ENABLED_WARN)) + return; + + /* Private shaper profile params */ + memset(&mp, 0, sizeof(struct rte_mtr_meter_profile)); + mp.alg = 0; + mp.srtcm_rfc2697.cir = res->cir; + mp.srtcm_rfc2697.cbs = res->cbs; + mp.srtcm_rfc2697.ebs = res->ebs; + + ret = rte_mtr_meter_profile_add(port_id, profile_id, &mp, &error); + if (ret != 0) { + print_err_msg(&error); + return; + } +} + +cmdline_parse_inst_t cmd_add_port_meter_profile_srtcm = { + .f = cmd_add_port_meter_profile_srtcm_parsed, + .data = NULL, + .help_str = "Add port meter profile srtcm (rfc2697)", + .tokens = { + (void *)&cmd_add_port_meter_profile_srtcm_add, + (void *)&cmd_add_port_meter_profile_srtcm_port, + (void *)&cmd_add_port_meter_profile_srtcm_meter, + (void *)&cmd_add_port_meter_profile_srtcm_profile, + (void *)&cmd_add_port_meter_profile_srtcm_port_id, + (void *)&cmd_add_port_meter_profile_srtcm_profile_id, + (void *)&cmd_add_port_meter_profile_srtcm_srtcm_rfc2697, + (void *)&cmd_add_port_meter_profile_srtcm_cir, + (void *)&cmd_add_port_meter_profile_srtcm_cbs, + (void *)&cmd_add_port_meter_profile_srtcm_ebs, + NULL, + }, +}; + +/* *** Add Port Meter Profile trtcm_rfc2698 *** */ +struct cmd_add_port_meter_profile_trtcm_result { + cmdline_fixed_string_t add; + cmdline_fixed_string_t port; + cmdline_fixed_string_t meter; + cmdline_fixed_string_t profile; + cmdline_fixed_string_t trtcm_rfc2698; + uint16_t port_id; + uint32_t profile_id; + uint64_t cir; + uint64_t pir; + uint64_t cbs; + uint64_t pbs; +}; + +cmdline_parse_token_string_t cmd_add_port_meter_profile_trtcm_add = + TOKEN_STRING_INITIALIZER( + struct cmd_add_port_meter_profile_trtcm_result, add, "add"); +cmdline_parse_token_string_t cmd_add_port_meter_profile_trtcm_port = + TOKEN_STRING_INITIALIZER( + struct cmd_add_port_meter_profile_trtcm_result, + port, "port"); +cmdline_parse_token_string_t cmd_add_port_meter_profile_trtcm_meter = + TOKEN_STRING_INITIALIZER( + struct cmd_add_port_meter_profile_trtcm_result, + meter, "meter"); +cmdline_parse_token_string_t cmd_add_port_meter_profile_trtcm_profile = + TOKEN_STRING_INITIALIZER( + struct cmd_add_port_meter_profile_trtcm_result, + profile, "profile"); +cmdline_parse_token_string_t cmd_add_port_meter_profile_trtcm_trtcm_rfc2698 = + TOKEN_STRING_INITIALIZER( + struct cmd_add_port_meter_profile_trtcm_result, + trtcm_rfc2698, "trtcm_rfc2698"); +cmdline_parse_token_num_t cmd_add_port_meter_profile_trtcm_port_id = + TOKEN_NUM_INITIALIZER( + struct cmd_add_port_meter_profile_trtcm_result, + port_id, UINT16); +cmdline_parse_token_num_t cmd_add_port_meter_profile_trtcm_profile_id = + TOKEN_NUM_INITIALIZER( + struct cmd_add_port_meter_profile_trtcm_result, + profile_id, UINT32); +cmdline_parse_token_num_t cmd_add_port_meter_profile_trtcm_cir = + TOKEN_NUM_INITIALIZER( + struct cmd_add_port_meter_profile_trtcm_result, + cir, UINT64); +cmdline_parse_token_num_t cmd_add_port_meter_profile_trtcm_pir = + TOKEN_NUM_INITIALIZER( + struct cmd_add_port_meter_profile_trtcm_result, + pir, UINT64); +cmdline_parse_token_num_t cmd_add_port_meter_profile_trtcm_cbs = + TOKEN_NUM_INITIALIZER( + struct cmd_add_port_meter_profile_trtcm_result, + cbs, UINT64); +cmdline_parse_token_num_t cmd_add_port_meter_profile_trtcm_pbs = + TOKEN_NUM_INITIALIZER( + struct cmd_add_port_meter_profile_trtcm_result, + pbs, UINT64); + +static void cmd_add_port_meter_profile_trtcm_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_add_port_meter_profile_trtcm_result *res = parsed_result; + struct rte_mtr_meter_profile mp; + struct rte_mtr_error error; + uint32_t profile_id = res->profile_id; + uint16_t port_id = res->port_id; + int ret; + + if (port_id_is_invalid(port_id, ENABLED_WARN)) + return; + + /* Private shaper profile params */ + memset(&mp, 0, sizeof(struct rte_mtr_meter_profile)); + mp.alg = 0; + mp.trtcm_rfc2698.cir = res->cir; + mp.trtcm_rfc2698.pir = res->pir; + mp.trtcm_rfc2698.cbs = res->cbs; + mp.trtcm_rfc2698.pbs = res->pbs; + + ret = rte_mtr_meter_profile_add(port_id, profile_id, &mp, &error); + if (ret != 0) { + print_err_msg(&error); + return; + } +} + +cmdline_parse_inst_t cmd_add_port_meter_profile_trtcm = { + .f = cmd_add_port_meter_profile_trtcm_parsed, + .data = NULL, + .help_str = "Add port meter profile trtcm (rfc2698)", + .tokens = { + (void *)&cmd_add_port_meter_profile_trtcm_add, + (void *)&cmd_add_port_meter_profile_trtcm_port, + (void *)&cmd_add_port_meter_profile_trtcm_meter, + (void *)&cmd_add_port_meter_profile_trtcm_profile, + (void *)&cmd_add_port_meter_profile_trtcm_port_id, + (void *)&cmd_add_port_meter_profile_trtcm_profile_id, + (void *)&cmd_add_port_meter_profile_trtcm_trtcm_rfc2698, + (void *)&cmd_add_port_meter_profile_trtcm_cir, + (void *)&cmd_add_port_meter_profile_trtcm_pir, + (void *)&cmd_add_port_meter_profile_trtcm_cbs, + (void *)&cmd_add_port_meter_profile_trtcm_pbs, + NULL, + }, +}; + +/* *** Add Port Meter Profile trtcm_rfc4115 *** */ +struct cmd_add_port_meter_profile_trtcm_rfc4115_result { + cmdline_fixed_string_t add; + cmdline_fixed_string_t port; + cmdline_fixed_string_t meter; + cmdline_fixed_string_t profile; + cmdline_fixed_string_t trtcm_rfc4115; + uint16_t port_id; + uint32_t profile_id; + uint64_t cir; + uint64_t eir; + uint64_t cbs; + uint64_t ebs; +}; + +cmdline_parse_token_string_t cmd_add_port_meter_profile_trtcm_rfc4115_add = + TOKEN_STRING_INITIALIZER( + struct cmd_add_port_meter_profile_trtcm_rfc4115_result, add, + "add"); +cmdline_parse_token_string_t cmd_add_port_meter_profile_trtcm_rfc4115_port = + TOKEN_STRING_INITIALIZER( + struct cmd_add_port_meter_profile_trtcm_rfc4115_result, + port, "port"); +cmdline_parse_token_string_t cmd_add_port_meter_profile_trtcm_rfc4115_meter = + TOKEN_STRING_INITIALIZER( + struct cmd_add_port_meter_profile_trtcm_rfc4115_result, + meter, "meter"); +cmdline_parse_token_string_t cmd_add_port_meter_profile_trtcm_rfc4115_profile = + TOKEN_STRING_INITIALIZER( + struct cmd_add_port_meter_profile_trtcm_rfc4115_result, + profile, "profile"); +cmdline_parse_token_string_t + cmd_add_port_meter_profile_trtcm_rfc4115_trtcm_rfc4115 = + TOKEN_STRING_INITIALIZER( + struct cmd_add_port_meter_profile_trtcm_rfc4115_result, + trtcm_rfc4115, "trtcm_rfc4115"); +cmdline_parse_token_num_t cmd_add_port_meter_profile_trtcm_rfc4115_port_id = + TOKEN_NUM_INITIALIZER( + struct cmd_add_port_meter_profile_trtcm_rfc4115_result, + port_id, UINT16); +cmdline_parse_token_num_t cmd_add_port_meter_profile_trtcm_rfc4115_profile_id = + TOKEN_NUM_INITIALIZER( + struct cmd_add_port_meter_profile_trtcm_rfc4115_result, + profile_id, UINT32); +cmdline_parse_token_num_t cmd_add_port_meter_profile_trtcm_rfc4115_cir = + TOKEN_NUM_INITIALIZER( + struct cmd_add_port_meter_profile_trtcm_rfc4115_result, + cir, UINT64); +cmdline_parse_token_num_t cmd_add_port_meter_profile_trtcm_rfc4115_eir = + TOKEN_NUM_INITIALIZER( + struct cmd_add_port_meter_profile_trtcm_rfc4115_result, + eir, UINT64); +cmdline_parse_token_num_t cmd_add_port_meter_profile_trtcm_rfc4115_cbs = + TOKEN_NUM_INITIALIZER( + struct cmd_add_port_meter_profile_trtcm_rfc4115_result, + cbs, UINT64); +cmdline_parse_token_num_t cmd_add_port_meter_profile_trtcm_rfc4115_ebs = + TOKEN_NUM_INITIALIZER( + struct cmd_add_port_meter_profile_trtcm_rfc4115_result, + ebs, UINT64); + +static void cmd_add_port_meter_profile_trtcm_rfc4115_parsed( + void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_add_port_meter_profile_trtcm_rfc4115_result *res = + parsed_result; + struct rte_mtr_meter_profile mp; + struct rte_mtr_error error; + uint32_t profile_id = res->profile_id; + uint16_t port_id = res->port_id; + int ret; + + if (port_id_is_invalid(port_id, ENABLED_WARN)) + return; + + /* Private shaper profile params */ + memset(&mp, 0, sizeof(struct rte_mtr_meter_profile)); + mp.alg = 0; + mp.trtcm_rfc4115.cir = res->cir; + mp.trtcm_rfc4115.eir = res->eir; + mp.trtcm_rfc4115.cbs = res->cbs; + mp.trtcm_rfc4115.ebs = res->ebs; + + ret = rte_mtr_meter_profile_add(port_id, profile_id, &mp, &error); + if (ret != 0) { + print_err_msg(&error); + return; + } +} + +cmdline_parse_inst_t cmd_add_port_meter_profile_trtcm_rfc4115 = { + .f = cmd_add_port_meter_profile_trtcm_rfc4115_parsed, + .data = NULL, + .help_str = "Add port meter profile trtcm (rfc4115)", + .tokens = { + (void *)&cmd_add_port_meter_profile_trtcm_rfc4115_add, + (void *)&cmd_add_port_meter_profile_trtcm_rfc4115_port, + (void *)&cmd_add_port_meter_profile_trtcm_rfc4115_meter, + (void *)&cmd_add_port_meter_profile_trtcm_rfc4115_profile, + (void *)&cmd_add_port_meter_profile_trtcm_rfc4115_port_id, + (void *)&cmd_add_port_meter_profile_trtcm_rfc4115_profile_id, + (void *)&cmd_add_port_meter_profile_trtcm_rfc4115_trtcm_rfc4115, + (void *)&cmd_add_port_meter_profile_trtcm_rfc4115_cir, + (void *)&cmd_add_port_meter_profile_trtcm_rfc4115_eir, + (void *)&cmd_add_port_meter_profile_trtcm_rfc4115_cbs, + (void *)&cmd_add_port_meter_profile_trtcm_rfc4115_ebs, + NULL, + }, +}; + +/* *** Delete Port Meter Profile *** */ +struct cmd_del_port_meter_profile_result { + cmdline_fixed_string_t del; + cmdline_fixed_string_t port; + cmdline_fixed_string_t meter; + cmdline_fixed_string_t profile; + uint16_t port_id; + uint32_t profile_id; +}; + +cmdline_parse_token_string_t cmd_del_port_meter_profile_del = + TOKEN_STRING_INITIALIZER( + struct cmd_del_port_meter_profile_result, del, "del"); +cmdline_parse_token_string_t cmd_del_port_meter_profile_port = + TOKEN_STRING_INITIALIZER( + struct cmd_del_port_meter_profile_result, + port, "port"); +cmdline_parse_token_string_t cmd_del_port_meter_profile_meter = + TOKEN_STRING_INITIALIZER( + struct cmd_del_port_meter_profile_result, + meter, "meter"); +cmdline_parse_token_string_t cmd_del_port_meter_profile_profile = + TOKEN_STRING_INITIALIZER( + struct cmd_del_port_meter_profile_result, + profile, "profile"); +cmdline_parse_token_num_t cmd_del_port_meter_profile_port_id = + TOKEN_NUM_INITIALIZER( + struct cmd_del_port_meter_profile_result, + port_id, UINT16); +cmdline_parse_token_num_t cmd_del_port_meter_profile_profile_id = + TOKEN_NUM_INITIALIZER( + struct cmd_del_port_meter_profile_result, + profile_id, UINT32); + +static void cmd_del_port_meter_profile_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_del_port_meter_profile_result *res = parsed_result; + struct rte_mtr_error error; + uint32_t profile_id = res->profile_id; + uint16_t port_id = res->port_id; + int ret; + + if (port_id_is_invalid(port_id, ENABLED_WARN)) + return; + + /* Delete meter profile */ + ret = rte_mtr_meter_profile_delete(port_id, profile_id, &error); + if (ret != 0) { + print_err_msg(&error); + return; + } +} + +cmdline_parse_inst_t cmd_del_port_meter_profile = { + .f = cmd_del_port_meter_profile_parsed, + .data = NULL, + .help_str = "Delete port meter profile", + .tokens = { + (void *)&cmd_del_port_meter_profile_del, + (void *)&cmd_del_port_meter_profile_port, + (void *)&cmd_del_port_meter_profile_meter, + (void *)&cmd_del_port_meter_profile_profile, + (void *)&cmd_del_port_meter_profile_port_id, + (void *)&cmd_del_port_meter_profile_profile_id, + NULL, + }, +}; + +/* *** Create Port Meter Object *** */ +struct cmd_set_port_meter_result { + cmdline_fixed_string_t set; + cmdline_fixed_string_t port; + cmdline_fixed_string_t meter; + uint16_t port_id; + uint32_t mtr_id; + uint32_t profile_id; + cmdline_fixed_string_t g_action; + cmdline_fixed_string_t y_action; + cmdline_fixed_string_t r_action; + uint64_t statistics_mask; + uint32_t shared; +}; + +cmdline_parse_token_string_t cmd_set_port_meter_set = + TOKEN_STRING_INITIALIZER( + struct cmd_set_port_meter_result, set, "set"); +cmdline_parse_token_string_t cmd_set_port_meter_port = + TOKEN_STRING_INITIALIZER( + struct cmd_set_port_meter_result, port, "port"); +cmdline_parse_token_string_t cmd_set_port_meter_meter = + TOKEN_STRING_INITIALIZER( + struct cmd_set_port_meter_result, meter, "meter"); +cmdline_parse_token_num_t cmd_set_port_meter_port_id = + TOKEN_NUM_INITIALIZER( + struct cmd_set_port_meter_result, port_id, UINT16); +cmdline_parse_token_num_t cmd_set_port_meter_mtr_id = + TOKEN_NUM_INITIALIZER( + struct cmd_set_port_meter_result, mtr_id, UINT32); +cmdline_parse_token_num_t cmd_set_port_meter_profile_id = + TOKEN_NUM_INITIALIZER( + struct cmd_set_port_meter_result, profile_id, UINT32); +cmdline_parse_token_string_t cmd_set_port_meter_g_action = + TOKEN_STRING_INITIALIZER(struct cmd_set_port_meter_result, + g_action, "R#Y#G#D"); +cmdline_parse_token_string_t cmd_set_port_meter_y_action = + TOKEN_STRING_INITIALIZER(struct cmd_set_port_meter_result, + y_action, "R#Y#G#D"); +cmdline_parse_token_string_t cmd_set_port_meter_r_action = + TOKEN_STRING_INITIALIZER(struct cmd_set_port_meter_result, + r_action, "R#Y#G#D"); +cmdline_parse_token_num_t cmd_set_port_meter_statistics_mask = + TOKEN_NUM_INITIALIZER(struct cmd_set_port_meter_result, + statistics_mask, UINT64); +cmdline_parse_token_num_t cmd_set_port_meter_shared = + TOKEN_NUM_INITIALIZER(struct cmd_set_port_meter_result, + shared, UINT32); + +static void cmd_set_port_meter_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_set_port_meter_result *res = parsed_result; + struct rte_mtr_error error; + struct rte_mtr_params params; + uint32_t mtr_id = res->mtr_id; + uint32_t shared = res->shared; + uint16_t port_id = res->port_id; + + int ret; + + if (port_id_is_invalid(port_id, ENABLED_WARN)) + return; + + /* Meter params */ + memset(¶ms, 0, sizeof(struct rte_mtr_params)); + params.meter_profile_id = res->profile_id; + params.use_prev_mtr_color = 1; + params.dscp_table = NULL; + params.meter_enable = 1; + params.action[RTE_MTR_GREEN] = + string_to_policer_action(res->g_action); + params.action[RTE_MTR_YELLOW] = + string_to_policer_action(res->y_action); + params.action[RTE_MTR_RED] = + string_to_policer_action(res->r_action); + params.stats_mask = res->statistics_mask; + + ret = rte_mtr_create(port_id, mtr_id, ¶ms, shared, &error); + if (ret != 0) { + print_err_msg(&error); + return; + } +} + +cmdline_parse_inst_t cmd_set_port_meter = { + .f = cmd_set_port_meter_parsed, + .data = NULL, + .help_str = "Set port meter", + .tokens = { + (void *)&cmd_set_port_meter_set, + (void *)&cmd_set_port_meter_port, + (void *)&cmd_set_port_meter_meter, + (void *)&cmd_set_port_meter_port_id, + (void *)&cmd_set_port_meter_mtr_id, + (void *)&cmd_set_port_meter_profile_id, + (void *)&cmd_set_port_meter_g_action, + (void *)&cmd_set_port_meter_y_action, + (void *)&cmd_set_port_meter_r_action, + (void *)&cmd_set_port_meter_statistics_mask, + (void *)&cmd_set_port_meter_shared, + NULL, + }, +}; + +/* *** Delete Port Meter Object *** */ +struct cmd_del_port_meter_result { + cmdline_fixed_string_t del; + cmdline_fixed_string_t port; + cmdline_fixed_string_t meter; + uint16_t port_id; + uint32_t mtr_id; +}; + +cmdline_parse_token_string_t cmd_del_port_meter_del = + TOKEN_STRING_INITIALIZER( + struct cmd_del_port_meter_result, del, "del"); +cmdline_parse_token_string_t cmd_del_port_meter_port = + TOKEN_STRING_INITIALIZER( + struct cmd_del_port_meter_result, port, "port"); +cmdline_parse_token_string_t cmd_del_port_meter_meter = + TOKEN_STRING_INITIALIZER( + struct cmd_del_port_meter_result, meter, "meter"); +cmdline_parse_token_num_t cmd_del_port_meter_port_id = + TOKEN_NUM_INITIALIZER( + struct cmd_del_port_meter_result, port_id, UINT16); +cmdline_parse_token_num_t cmd_del_port_meter_mtr_id = + TOKEN_NUM_INITIALIZER( + struct cmd_del_port_meter_result, mtr_id, UINT32); + +static void cmd_del_port_meter_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_del_port_meter_result *res = parsed_result; + struct rte_mtr_error error; + uint32_t mtr_id = res->mtr_id; + uint16_t port_id = res->port_id; + + int ret; + + if (port_id_is_invalid(port_id, ENABLED_WARN)) + return; + + /* Destroy Meter */ + ret = rte_mtr_destroy(port_id, mtr_id, &error); + if (ret != 0) { + print_err_msg(&error); + return; + } +} + +cmdline_parse_inst_t cmd_del_port_meter = { + .f = cmd_del_port_meter_parsed, + .data = NULL, + .help_str = "Delete port meter", + .tokens = { + (void *)&cmd_del_port_meter_del, + (void *)&cmd_del_port_meter_port, + (void *)&cmd_del_port_meter_meter, + (void *)&cmd_del_port_meter_port_id, + (void *)&cmd_del_port_meter_mtr_id, + NULL, + }, +}; + +/* *** Set Port Meter Profile *** */ +struct cmd_set_port_meter_profile_result { + cmdline_fixed_string_t set; + cmdline_fixed_string_t port; + cmdline_fixed_string_t meter; + cmdline_fixed_string_t profile; + uint16_t port_id; + uint32_t mtr_id; + uint32_t profile_id; +}; + +cmdline_parse_token_string_t cmd_set_port_meter_profile_set = + TOKEN_STRING_INITIALIZER( + struct cmd_set_port_meter_profile_result, set, "set"); +cmdline_parse_token_string_t cmd_set_port_meter_profile_port = + TOKEN_STRING_INITIALIZER( + struct cmd_set_port_meter_profile_result, port, "port"); +cmdline_parse_token_string_t cmd_set_port_meter_profile_meter = + TOKEN_STRING_INITIALIZER( + struct cmd_set_port_meter_profile_result, meter, "meter"); +cmdline_parse_token_string_t cmd_set_port_meter_profile_profile = + TOKEN_STRING_INITIALIZER( + struct cmd_set_port_meter_profile_result, profile, "profile"); +cmdline_parse_token_num_t cmd_set_port_meter_profile_port_id = + TOKEN_NUM_INITIALIZER( + struct cmd_set_port_meter_profile_result, port_id, UINT16); +cmdline_parse_token_num_t cmd_set_port_meter_profile_mtr_id = + TOKEN_NUM_INITIALIZER( + struct cmd_set_port_meter_profile_result, mtr_id, UINT32); +cmdline_parse_token_num_t cmd_set_port_meter_profile_profile_id = + TOKEN_NUM_INITIALIZER( + struct cmd_set_port_meter_profile_result, profile_id, UINT32); + +static void cmd_set_port_meter_profile_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_set_port_meter_profile_result *res = parsed_result; + struct rte_mtr_error error; + uint32_t mtr_id = res->mtr_id; + uint32_t profile_id = res->profile_id; + uint16_t port_id = res->port_id; + + int ret; + + if (port_id_is_invalid(port_id, ENABLED_WARN)) + return; + + /* Set meter profile */ + ret = rte_mtr_meter_profile_update(port_id, mtr_id, + profile_id, &error); + if (ret != 0) { + print_err_msg(&error); + return; + } +} + +cmdline_parse_inst_t cmd_set_port_meter_profile = { + .f = cmd_set_port_meter_profile_parsed, + .data = NULL, + .help_str = "Set port meter profile", + .tokens = { + (void *)&cmd_set_port_meter_profile_set, + (void *)&cmd_set_port_meter_profile_port, + (void *)&cmd_set_port_meter_profile_meter, + (void *)&cmd_set_port_meter_profile_profile, + (void *)&cmd_set_port_meter_profile_port_id, + (void *)&cmd_set_port_meter_profile_mtr_id, + (void *)&cmd_set_port_meter_profile_profile_id, + NULL, + }, +}; + +/* *** Set Port Meter Policer Action *** */ +struct cmd_set_port_meter_policer_action_result { + cmdline_fixed_string_t set; + cmdline_fixed_string_t port; + cmdline_fixed_string_t meter; + cmdline_fixed_string_t policer; + cmdline_fixed_string_t action; + uint16_t port_id; + uint32_t mtr_id; + cmdline_fixed_string_t color; + cmdline_fixed_string_t policer_action; +}; + +cmdline_parse_token_string_t cmd_set_port_meter_policer_action_set = + TOKEN_STRING_INITIALIZER( + struct cmd_set_port_meter_policer_action_result, set, "set"); +cmdline_parse_token_string_t cmd_set_port_meter_policer_action_port = + TOKEN_STRING_INITIALIZER( + struct cmd_set_port_meter_policer_action_result, port, "port"); +cmdline_parse_token_string_t cmd_set_port_meter_policer_action_meter = + TOKEN_STRING_INITIALIZER( + struct cmd_set_port_meter_policer_action_result, meter, + "meter"); +cmdline_parse_token_string_t cmd_set_port_meter_policer_action_policer = + TOKEN_STRING_INITIALIZER( + struct cmd_set_port_meter_policer_action_result, policer, + "policer"); +cmdline_parse_token_string_t cmd_set_port_meter_policer_action_action = + TOKEN_STRING_INITIALIZER( + struct cmd_set_port_meter_policer_action_result, action, + "action"); +cmdline_parse_token_num_t cmd_set_port_meter_policer_action_port_id = + TOKEN_NUM_INITIALIZER( + struct cmd_set_port_meter_policer_action_result, port_id, + UINT16); +cmdline_parse_token_num_t cmd_set_port_meter_policer_action_mtr_id = + TOKEN_NUM_INITIALIZER( + struct cmd_set_port_meter_policer_action_result, mtr_id, + UINT32); +cmdline_parse_token_string_t cmd_set_port_meter_policer_action_color = + TOKEN_STRING_INITIALIZER( + struct cmd_set_port_meter_policer_action_result, color, + "G#Y#R"); +cmdline_parse_token_string_t cmd_set_port_meter_policer_action_policer_action = + TOKEN_STRING_INITIALIZER( + struct cmd_set_port_meter_policer_action_result, + policer_action, "G#Y#R#D"); + +static void cmd_set_port_meter_policer_action_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_set_port_meter_policer_action_result *res = parsed_result; + enum rte_mtr_color color; + enum rte_mtr_policer_action action[RTE_MTR_COLORS]; + struct rte_mtr_error error; + uint32_t mtr_id = res->mtr_id; + uint16_t port_id = res->port_id; + char *c = res->color; + char *a = res->policer_action; + int ret; + + if (port_id_is_invalid(port_id, ENABLED_WARN)) + return; + + /* Color */ + if (strcmp(c, "G") == 0) + color = RTE_MTR_GREEN; + else if (strcmp(c, "Y") == 0) + color = RTE_MTR_YELLOW; + else + color = RTE_MTR_RED; + + /* Action */ + if (strcmp(a, "G") == 0) + action[color] = MTR_POLICER_ACTION_COLOR_GREEN; + else if (strcmp(a, "Y") == 0) + action[color] = MTR_POLICER_ACTION_COLOR_YELLOW; + else if (strcmp(a, "R") == 0) + action[color] = MTR_POLICER_ACTION_COLOR_RED; + else + action[color] = MTR_POLICER_ACTION_DROP; + + ret = rte_mtr_policer_actions_update(port_id, mtr_id, + 1 << color, action, &error); + if (ret != 0) { + print_err_msg(&error); + return; + } +} + +cmdline_parse_inst_t cmd_set_port_meter_policer_action = { + .f = cmd_set_port_meter_policer_action_parsed, + .data = NULL, + .help_str = "Set port meter policer action", + .tokens = { + (void *)&cmd_set_port_meter_policer_action_set, + (void *)&cmd_set_port_meter_policer_action_port, + (void *)&cmd_set_port_meter_policer_action_meter, + (void *)&cmd_set_port_meter_policer_action_policer, + (void *)&cmd_set_port_meter_policer_action_action, + (void *)&cmd_set_port_meter_policer_action_port_id, + (void *)&cmd_set_port_meter_policer_action_mtr_id, + (void *)&cmd_set_port_meter_policer_action_color, + (void *)&cmd_set_port_meter_policer_action_policer_action, + NULL, + }, +}; + +/* *** Set Port Meter Stats Mask *** */ +struct cmd_set_port_meter_stats_mask_result { + cmdline_fixed_string_t set; + cmdline_fixed_string_t port; + cmdline_fixed_string_t meter; + cmdline_fixed_string_t stats; + cmdline_fixed_string_t mask; + uint16_t port_id; + uint32_t mtr_id; + uint64_t stats_mask; +}; + +cmdline_parse_token_string_t cmd_set_port_meter_stats_mask_set = + TOKEN_STRING_INITIALIZER( + struct cmd_set_port_meter_stats_mask_result, set, "set"); +cmdline_parse_token_string_t cmd_set_port_meter_stats_mask_port = + TOKEN_STRING_INITIALIZER( + struct cmd_set_port_meter_stats_mask_result, port, "port"); +cmdline_parse_token_string_t cmd_set_port_meter_stats_mask_meter = + TOKEN_STRING_INITIALIZER( + struct cmd_set_port_meter_stats_mask_result, meter, "meter"); +cmdline_parse_token_string_t cmd_set_port_meter_stats_mask_stats = + TOKEN_STRING_INITIALIZER( + struct cmd_set_port_meter_stats_mask_result, stats, "stats"); +cmdline_parse_token_string_t cmd_set_port_meter_stats_mask_mask = + TOKEN_STRING_INITIALIZER( + struct cmd_set_port_meter_stats_mask_result, mask, "mask"); +cmdline_parse_token_num_t cmd_set_port_meter_stats_mask_port_id = + TOKEN_NUM_INITIALIZER( + struct cmd_set_port_meter_stats_mask_result, port_id, UINT16); +cmdline_parse_token_num_t cmd_set_port_meter_stats_mask_mtr_id = + TOKEN_NUM_INITIALIZER( + struct cmd_set_port_meter_stats_mask_result, mtr_id, UINT32); +cmdline_parse_token_num_t cmd_set_port_meter_stats_mask_stats_mask = + TOKEN_NUM_INITIALIZER( + struct cmd_set_port_meter_stats_mask_result, stats_mask, + UINT64); + +static void cmd_set_port_meter_stats_mask_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_set_port_meter_stats_mask_result *res = parsed_result; + struct rte_mtr_error error; + uint64_t stats_mask = res->stats_mask; + uint32_t mtr_id = res->mtr_id; + uint16_t port_id = res->port_id; + int ret; + + if (port_id_is_invalid(port_id, ENABLED_WARN)) + return; + + ret = rte_mtr_stats_update(port_id, mtr_id, stats_mask, &error); + if (ret != 0) { + print_err_msg(&error); + return; + } +} + +cmdline_parse_inst_t cmd_set_port_meter_stats_mask = { + .f = cmd_set_port_meter_stats_mask_parsed, + .data = NULL, + .help_str = "Set port meter stats mask", + .tokens = { + (void *)&cmd_set_port_meter_stats_mask_set, + (void *)&cmd_set_port_meter_stats_mask_port, + (void *)&cmd_set_port_meter_stats_mask_meter, + (void *)&cmd_set_port_meter_stats_mask_stats, + (void *)&cmd_set_port_meter_stats_mask_mask, + (void *)&cmd_set_port_meter_stats_mask_port_id, + (void *)&cmd_set_port_meter_stats_mask_mtr_id, + (void *)&cmd_set_port_meter_stats_mask_stats_mask, + NULL, + }, +}; + +/* *** Show Port Meter Stats *** */ +struct cmd_show_port_meter_stats_result { + cmdline_fixed_string_t show; + cmdline_fixed_string_t port; + cmdline_fixed_string_t meter; + cmdline_fixed_string_t stats; + uint16_t port_id; + uint32_t mtr_id; + uint32_t clear; +}; + +cmdline_parse_token_string_t cmd_show_port_meter_stats_show = + TOKEN_STRING_INITIALIZER( + struct cmd_show_port_meter_stats_result, show, "show"); +cmdline_parse_token_string_t cmd_show_port_meter_stats_port = + TOKEN_STRING_INITIALIZER( + struct cmd_show_port_meter_stats_result, port, "port"); +cmdline_parse_token_string_t cmd_show_port_meter_stats_meter = + TOKEN_STRING_INITIALIZER( + struct cmd_show_port_meter_stats_result, meter, "meter"); +cmdline_parse_token_string_t cmd_show_port_meter_stats_stats = + TOKEN_STRING_INITIALIZER( + struct cmd_show_port_meter_stats_result, stats, "stats"); +cmdline_parse_token_num_t cmd_show_port_meter_stats_port_id = + TOKEN_NUM_INITIALIZER( + struct cmd_show_port_meter_stats_result, port_id, UINT16); +cmdline_parse_token_num_t cmd_show_port_meter_stats_mtr_id = + TOKEN_NUM_INITIALIZER( + struct cmd_show_port_meter_stats_result, mtr_id, UINT32); +cmdline_parse_token_num_t cmd_show_port_meter_stats_clear = + TOKEN_NUM_INITIALIZER( + struct cmd_show_port_meter_stats_result, clear, UINT32); + +static void cmd_show_port_meter_stats_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_show_port_meter_stats_result *res = parsed_result; + struct rte_mtr_stats stats; + uint64_t stats_mask = 0; + struct rte_mtr_error error; + uint32_t mtr_id = res->mtr_id; + uint32_t clear = res->clear; + uint16_t port_id = res->port_id; + int ret; + + if (port_id_is_invalid(port_id, ENABLED_WARN)) + return; + + memset(&stats, 0, sizeof(struct rte_mtr_stats)); + ret = rte_mtr_stats_read(port_id, mtr_id, &stats, + &stats_mask, clear, &error); + if (ret != 0) { + print_err_msg(&error); + return; + } + + /* Display stats */ + if (stats_mask & RTE_MTR_STATS_N_PKTS_GREEN) + printf("\tPkts G: %" PRIu64 "\n", + stats.n_pkts[RTE_MTR_GREEN]); + if (stats_mask & RTE_MTR_STATS_N_BYTES_GREEN) + printf("\tBytes G: %" PRIu64 "\n", + stats.n_bytes[RTE_MTR_GREEN]); + if (stats_mask & RTE_MTR_STATS_N_PKTS_YELLOW) + printf("\tPkts Y: %" PRIu64 "\n", + stats.n_pkts[RTE_MTR_YELLOW]); + if (stats_mask & RTE_MTR_STATS_N_BYTES_YELLOW) + printf("\tBytes Y: %" PRIu64 "\n", + stats.n_bytes[RTE_MTR_YELLOW]); + if (stats_mask & RTE_MTR_STATS_N_PKTS_RED) + printf("\tPkts R: %" PRIu64 "\n", + stats.n_pkts[RTE_MTR_RED]); + if (stats_mask & RTE_MTR_STATS_N_BYTES_RED) + printf("\tBytes Y: %" PRIu64 "\n", + stats.n_bytes[RTE_MTR_RED]); + if (stats_mask & RTE_MTR_STATS_N_PKTS_DROPPED) + printf("\tPkts DROPPED: %" PRIu64 "\n", + stats.n_pkts_dropped); + if (stats_mask & RTE_MTR_STATS_N_BYTES_DROPPED) + printf("\tBytes DROPPED: %" PRIu64 "\n", + stats.n_bytes_dropped); +} + +cmdline_parse_inst_t cmd_show_port_meter_stats = { + .f = cmd_show_port_meter_stats_parsed, + .data = NULL, + .help_str = "Show port meter stats", + .tokens = { + (void *)&cmd_show_port_meter_stats_show, + (void *)&cmd_show_port_meter_stats_port, + (void *)&cmd_show_port_meter_stats_meter, + (void *)&cmd_show_port_meter_stats_stats, + (void *)&cmd_show_port_meter_stats_port_id, + (void *)&cmd_show_port_meter_stats_mtr_id, + (void *)&cmd_show_port_meter_stats_clear, + NULL, + }, +}; diff --git a/dpdk/app/test-pmd/cmdline_mtr.h b/dpdk/app/test-pmd/cmdline_mtr.h new file mode 100644 index 00000000..5d599efc --- /dev/null +++ b/dpdk/app/test-pmd/cmdline_mtr.h @@ -0,0 +1,49 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _CMDLINE_MTR_H_ +#define _CMDLINE_MTR_H_ + +/* Traffic Metering and Policing */ +extern cmdline_parse_inst_t cmd_add_port_meter_profile_srtcm; +extern cmdline_parse_inst_t cmd_add_port_meter_profile_trtcm; +extern cmdline_parse_inst_t cmd_add_port_meter_profile_trtcm_rfc4115; +extern cmdline_parse_inst_t cmd_del_port_meter_profile; +extern cmdline_parse_inst_t cmd_set_port_meter; +extern cmdline_parse_inst_t cmd_del_port_meter; +extern cmdline_parse_inst_t cmd_set_port_meter_profile; +extern cmdline_parse_inst_t cmd_set_port_meter_policer_action; +extern cmdline_parse_inst_t cmd_set_port_meter_stats_mask; +extern cmdline_parse_inst_t cmd_show_port_meter_stats; + +#endif /* _CMDLINE_MTR_H_ */ diff --git a/dpdk/app/test-pmd/cmdline_tm.c b/dpdk/app/test-pmd/cmdline_tm.c new file mode 100644 index 00000000..803fae44 --- /dev/null +++ b/dpdk/app/test-pmd/cmdline_tm.c @@ -0,0 +1,2059 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include + +#include +#include +#include + +#include "testpmd.h" +#include "cmdline_tm.h" + +#define PARSE_DELIMITER " \f\n\r\t\v" +#define MAX_NUM_SHARED_SHAPERS 256 + +#define skip_white_spaces(pos) \ +({ \ + __typeof__(pos) _p = (pos); \ + for ( ; isspace(*_p); _p++) \ + ; \ + _p; \ +}) + +/** Display TM Error Message */ +static void +print_err_msg(struct rte_tm_error *error) +{ + static const char *const errstrlist[] = { + [RTE_TM_ERROR_TYPE_NONE] = "no error", + [RTE_TM_ERROR_TYPE_UNSPECIFIED] = "cause unspecified", + [RTE_TM_ERROR_TYPE_CAPABILITIES] + = "capability parameter null", + [RTE_TM_ERROR_TYPE_LEVEL_ID] = "level id", + [RTE_TM_ERROR_TYPE_WRED_PROFILE] + = "wred profile null", + [RTE_TM_ERROR_TYPE_WRED_PROFILE_GREEN] = "wred profile(green)", + [RTE_TM_ERROR_TYPE_WRED_PROFILE_YELLOW] + = "wred profile(yellow)", + [RTE_TM_ERROR_TYPE_WRED_PROFILE_RED] = "wred profile(red)", + [RTE_TM_ERROR_TYPE_WRED_PROFILE_ID] = "wred profile id", + [RTE_TM_ERROR_TYPE_SHARED_WRED_CONTEXT_ID] + = "shared wred context id", + [RTE_TM_ERROR_TYPE_SHAPER_PROFILE] = "shaper profile null", + [RTE_TM_ERROR_TYPE_SHAPER_PROFILE_COMMITTED_RATE] + = "committed rate field (shaper profile)", + [RTE_TM_ERROR_TYPE_SHAPER_PROFILE_COMMITTED_SIZE] + = "committed size field (shaper profile)", + [RTE_TM_ERROR_TYPE_SHAPER_PROFILE_PEAK_RATE] + = "peak rate field (shaper profile)", + [RTE_TM_ERROR_TYPE_SHAPER_PROFILE_PEAK_SIZE] + = "peak size field (shaper profile)", + [RTE_TM_ERROR_TYPE_SHAPER_PROFILE_PKT_ADJUST_LEN] + = "packet adjust length field (shaper profile)", + [RTE_TM_ERROR_TYPE_SHAPER_PROFILE_ID] = "shaper profile id", + [RTE_TM_ERROR_TYPE_SHARED_SHAPER_ID] = "shared shaper id", + [RTE_TM_ERROR_TYPE_NODE_PARENT_NODE_ID] = "parent node id", + [RTE_TM_ERROR_TYPE_NODE_PRIORITY] = "node priority", + [RTE_TM_ERROR_TYPE_NODE_WEIGHT] = "node weight", + [RTE_TM_ERROR_TYPE_NODE_PARAMS] = "node parameter null", + [RTE_TM_ERROR_TYPE_NODE_PARAMS_SHAPER_PROFILE_ID] + = "shaper profile id field (node params)", + [RTE_TM_ERROR_TYPE_NODE_PARAMS_SHARED_SHAPER_ID] + = "shared shaper id field (node params)", + [RTE_TM_ERROR_TYPE_NODE_PARAMS_N_SHARED_SHAPERS] + = "num shared shapers field (node params)", + [RTE_TM_ERROR_TYPE_NODE_PARAMS_WFQ_WEIGHT_MODE] + = "wfq weght mode field (node params)", + [RTE_TM_ERROR_TYPE_NODE_PARAMS_N_SP_PRIORITIES] + = "num strict priorities field (node params)", + [RTE_TM_ERROR_TYPE_NODE_PARAMS_CMAN] + = "congestion management mode field (node params)", + [RTE_TM_ERROR_TYPE_NODE_PARAMS_WRED_PROFILE_ID] = + "wred profile id field (node params)", + [RTE_TM_ERROR_TYPE_NODE_PARAMS_SHARED_WRED_CONTEXT_ID] + = "shared wred context id field (node params)", + [RTE_TM_ERROR_TYPE_NODE_PARAMS_N_SHARED_WRED_CONTEXTS] + = "num shared wred contexts field (node params)", + [RTE_TM_ERROR_TYPE_NODE_PARAMS_STATS] + = "stats field (node params)", + [RTE_TM_ERROR_TYPE_NODE_ID] = "node id", + }; + + const char *errstr; + char buf[64]; + + if ((unsigned int)error->type >= RTE_DIM(errstrlist) || + !errstrlist[error->type]) + errstr = "unknown type"; + else + errstr = errstrlist[error->type]; + + if (error->cause) + snprintf(buf, sizeof(buf), "cause: %p, ", error->cause); + + printf("%s: %s%s (error %d)\n", errstr, error->cause ? buf : "", + error->message ? error->message : "(no stated reason)", + error->type); +} + +static int +read_uint64(uint64_t *value, const char *p) +{ + char *next; + uint64_t val; + + p = skip_white_spaces(p); + if (!isdigit(*p)) + return -EINVAL; + + val = strtoul(p, &next, 10); + if (p == next) + return -EINVAL; + + p = next; + switch (*p) { + case 'T': + val *= 1024ULL; + /* fall through */ + case 'G': + val *= 1024ULL; + /* fall through */ + case 'M': + val *= 1024ULL; + /* fall through */ + case 'k': + case 'K': + val *= 1024ULL; + p++; + break; + } + + p = skip_white_spaces(p); + if (*p != '\0') + return -EINVAL; + + *value = val; + return 0; +} + +static int +read_uint32(uint32_t *value, const char *p) +{ + uint64_t val = 0; + int ret = read_uint64(&val, p); + + if (ret < 0) + return ret; + + if (val > UINT32_MAX) + return -ERANGE; + + *value = val; + return 0; +} + +static int +parse_multi_ss_id_str(char *s_str, uint32_t *n_ssp, uint32_t shaper_id[]) +{ + uint32_t n_shared_shapers = 0, i = 0; + char *token; + + /* First token: num of shared shapers */ + token = strtok_r(s_str, PARSE_DELIMITER, &s_str); + if (token == NULL) + return -1; + + if (read_uint32(&n_shared_shapers, token)) + return -1; + + /* Check: num of shared shaper */ + if (n_shared_shapers >= MAX_NUM_SHARED_SHAPERS) { + printf(" Number of shared shapers exceed the max (error)\n"); + return -1; + } + + /* Parse shared shaper ids */ + while (1) { + token = strtok_r(s_str, PARSE_DELIMITER, &s_str); + if ((token != NULL && n_shared_shapers == 0) || + (token == NULL && i < n_shared_shapers)) + return -1; + + if (token == NULL) + break; + + if (read_uint32(&shaper_id[i], token)) + return -1; + i++; + } + *n_ssp = n_shared_shapers; + + return 0; +} +/* *** Port TM Capability *** */ +struct cmd_show_port_tm_cap_result { + cmdline_fixed_string_t show; + cmdline_fixed_string_t port; + cmdline_fixed_string_t tm; + cmdline_fixed_string_t cap; + uint16_t port_id; +}; + +cmdline_parse_token_string_t cmd_show_port_tm_cap_show = + TOKEN_STRING_INITIALIZER(struct cmd_show_port_tm_cap_result, + show, "show"); +cmdline_parse_token_string_t cmd_show_port_tm_cap_port = + TOKEN_STRING_INITIALIZER(struct cmd_show_port_tm_cap_result, + port, "port"); +cmdline_parse_token_string_t cmd_show_port_tm_cap_tm = + TOKEN_STRING_INITIALIZER(struct cmd_show_port_tm_cap_result, + tm, "tm"); +cmdline_parse_token_string_t cmd_show_port_tm_cap_cap = + TOKEN_STRING_INITIALIZER(struct cmd_show_port_tm_cap_result, + cap, "cap"); +cmdline_parse_token_num_t cmd_show_port_tm_cap_port_id = + TOKEN_NUM_INITIALIZER(struct cmd_show_port_tm_cap_result, + port_id, UINT16); + +static void cmd_show_port_tm_cap_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_show_port_tm_cap_result *res = parsed_result; + struct rte_tm_capabilities cap; + struct rte_tm_error error; + portid_t port_id = res->port_id; + uint32_t i; + int ret; + + if (port_id_is_invalid(port_id, ENABLED_WARN)) + return; + + memset(&cap, 0, sizeof(struct rte_tm_capabilities)); + ret = rte_tm_capabilities_get(port_id, &cap, &error); + if (ret) { + print_err_msg(&error); + return; + } + + printf("\n**** Port TM Capabilities ****\n\n"); + printf("cap.n_nodes_max %" PRIu32 "\n", cap.n_nodes_max); + printf("cap.n_levels_max %" PRIu32 "\n", cap.n_levels_max); + printf("cap.non_leaf_nodes_identical %" PRId32 "\n", + cap.non_leaf_nodes_identical); + printf("cap.leaf_nodes_identical %" PRId32 "\n", + cap.leaf_nodes_identical); + printf("cap.shaper_n_max %u\n", cap.shaper_n_max); + printf("cap.shaper_private_n_max %" PRIu32 "\n", + cap.shaper_private_n_max); + printf("cap.shaper_private_dual_rate_n_max %" PRId32 "\n", + cap.shaper_private_dual_rate_n_max); + printf("cap.shaper_private_rate_min %" PRIu64 "\n", + cap.shaper_private_rate_min); + printf("cap.shaper_private_rate_max %" PRIu64 "\n", + cap.shaper_private_rate_max); + printf("cap.shaper_shared_n_max %" PRIu32 "\n", + cap.shaper_shared_n_max); + printf("cap.shaper_shared_n_nodes_per_shaper_max %" PRIu32 "\n", + cap.shaper_shared_n_nodes_per_shaper_max); + printf("cap.shaper_shared_n_shapers_per_node_max %" PRIu32 "\n", + cap.shaper_shared_n_shapers_per_node_max); + printf("cap.shaper_shared_dual_rate_n_max %" PRIu32 "\n", + cap.shaper_shared_dual_rate_n_max); + printf("cap.shaper_shared_rate_min %" PRIu64 "\n", + cap.shaper_shared_rate_min); + printf("cap.shaper_shared_rate_max %" PRIu64 "\n", + cap.shaper_shared_rate_max); + printf("cap.shaper_pkt_length_adjust_min %" PRId32 "\n", + cap.shaper_pkt_length_adjust_min); + printf("cap.shaper_pkt_length_adjust_max %" PRId32 "\n", + cap.shaper_pkt_length_adjust_max); + printf("cap.sched_n_children_max %" PRIu32 "\n", + cap.sched_n_children_max); + printf("cap.sched_sp_n_priorities_max %" PRIu32 "\n", + cap.sched_sp_n_priorities_max); + printf("cap.sched_wfq_n_children_per_group_max %" PRIu32 "\n", + cap.sched_wfq_n_children_per_group_max); + printf("cap.sched_wfq_n_groups_max %" PRIu32 "\n", + cap.sched_wfq_n_groups_max); + printf("cap.sched_wfq_weight_max %" PRIu32 "\n", + cap.sched_wfq_weight_max); + printf("cap.cman_head_drop_supported %" PRId32 "\n", + cap.cman_head_drop_supported); + printf("cap.cman_wred_context_n_max %" PRIu32 "\n", + cap.cman_wred_context_n_max); + printf("cap.cman_wred_context_private_n_max %" PRIu32 "\n", + cap.cman_wred_context_private_n_max); + printf("cap.cman_wred_context_shared_n_max %" PRIu32 "\n", + cap.cman_wred_context_shared_n_max); + printf("cap.cman_wred_context_shared_n_nodes_per_context_max %" PRIu32 + "\n", cap.cman_wred_context_shared_n_nodes_per_context_max); + printf("cap.cman_wred_context_shared_n_contexts_per_node_max %" PRIu32 + "\n", cap.cman_wred_context_shared_n_contexts_per_node_max); + + for (i = 0; i < RTE_TM_COLORS; i++) { + printf("cap.mark_vlan_dei_supported %" PRId32 "\n", + cap.mark_vlan_dei_supported[i]); + printf("cap.mark_ip_ecn_tcp_supported %" PRId32 "\n", + cap.mark_ip_ecn_tcp_supported[i]); + printf("cap.mark_ip_ecn_sctp_supported %" PRId32 "\n", + cap.mark_ip_ecn_sctp_supported[i]); + printf("cap.mark_ip_dscp_supported %" PRId32 "\n", + cap.mark_ip_dscp_supported[i]); + } + + printf("cap.dynamic_update_mask %" PRIx64 "\n", + cap.dynamic_update_mask); + printf("cap.stats_mask %" PRIx64 "\n", cap.stats_mask); +} + +cmdline_parse_inst_t cmd_show_port_tm_cap = { + .f = cmd_show_port_tm_cap_parsed, + .data = NULL, + .help_str = "Show Port TM Capabilities", + .tokens = { + (void *)&cmd_show_port_tm_cap_show, + (void *)&cmd_show_port_tm_cap_port, + (void *)&cmd_show_port_tm_cap_tm, + (void *)&cmd_show_port_tm_cap_cap, + (void *)&cmd_show_port_tm_cap_port_id, + NULL, + }, +}; + +/* *** Port TM Hierarchical Level Capability *** */ +struct cmd_show_port_tm_level_cap_result { + cmdline_fixed_string_t show; + cmdline_fixed_string_t port; + cmdline_fixed_string_t tm; + cmdline_fixed_string_t level; + cmdline_fixed_string_t cap; + uint16_t port_id; + uint32_t level_id; +}; + +cmdline_parse_token_string_t cmd_show_port_tm_level_cap_show = + TOKEN_STRING_INITIALIZER(struct cmd_show_port_tm_level_cap_result, + show, "show"); +cmdline_parse_token_string_t cmd_show_port_tm_level_cap_port = + TOKEN_STRING_INITIALIZER(struct cmd_show_port_tm_level_cap_result, + port, "port"); +cmdline_parse_token_string_t cmd_show_port_tm_level_cap_tm = + TOKEN_STRING_INITIALIZER(struct cmd_show_port_tm_level_cap_result, + tm, "tm"); +cmdline_parse_token_string_t cmd_show_port_tm_level_cap_level = + TOKEN_STRING_INITIALIZER(struct cmd_show_port_tm_level_cap_result, + level, "level"); +cmdline_parse_token_string_t cmd_show_port_tm_level_cap_cap = + TOKEN_STRING_INITIALIZER(struct cmd_show_port_tm_level_cap_result, + cap, "cap"); +cmdline_parse_token_num_t cmd_show_port_tm_level_cap_port_id = + TOKEN_NUM_INITIALIZER(struct cmd_show_port_tm_level_cap_result, + port_id, UINT16); +cmdline_parse_token_num_t cmd_show_port_tm_level_cap_level_id = + TOKEN_NUM_INITIALIZER(struct cmd_show_port_tm_level_cap_result, + level_id, UINT32); + + +static void cmd_show_port_tm_level_cap_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_show_port_tm_level_cap_result *res = parsed_result; + struct rte_tm_level_capabilities lcap; + struct rte_tm_error error; + portid_t port_id = res->port_id; + uint32_t level_id = res->level_id; + int ret; + + if (port_id_is_invalid(port_id, ENABLED_WARN)) + return; + + memset(&lcap, 0, sizeof(struct rte_tm_level_capabilities)); + ret = rte_tm_level_capabilities_get(port_id, level_id, &lcap, &error); + if (ret) { + print_err_msg(&error); + return; + } + printf("\n** Port TM Hierarchy level %" PRIu32 " Capability **\n\n", + level_id); + + printf("cap.n_nodes_max %" PRIu32 "\n", lcap.n_nodes_max); + printf("cap.n_nodes_nonleaf_max %" PRIu32 "\n", + lcap.n_nodes_nonleaf_max); + printf("cap.n_nodes_leaf_max %" PRIu32 "\n", lcap.n_nodes_leaf_max); + printf("cap.non_leaf_nodes_identical %" PRId32 "\n", + lcap.non_leaf_nodes_identical); + printf("cap.leaf_nodes_identical %" PRId32 "\n", + lcap.leaf_nodes_identical); + if (level_id <= 3) { + printf("cap.nonleaf.shaper_private_supported %" PRId32 "\n", + lcap.nonleaf.shaper_private_supported); + printf("cap.nonleaf.shaper_private_dual_rate_supported %" PRId32 + "\n", lcap.nonleaf.shaper_private_dual_rate_supported); + printf("cap.nonleaf.shaper_private_rate_min %" PRIu64 "\n", + lcap.nonleaf.shaper_private_rate_min); + printf("cap.nonleaf.shaper_private_rate_max %" PRIu64 "\n", + lcap.nonleaf.shaper_private_rate_max); + printf("cap.nonleaf.shaper_shared_n_max %" PRIu32 "\n", + lcap.nonleaf.shaper_shared_n_max); + printf("cap.nonleaf.sched_n_children_max %" PRIu32 "\n", + lcap.nonleaf.sched_n_children_max); + printf("cap.nonleaf.sched_sp_n_priorities_max %" PRIu32 "\n", + lcap.nonleaf.sched_sp_n_priorities_max); + printf("cap.nonleaf.sched_wfq_n_children_per_group_max %" PRIu32 + "\n", lcap.nonleaf.sched_wfq_n_children_per_group_max); + printf("cap.nonleaf.sched_wfq_n_groups_max %" PRIu32 "\n", + lcap.nonleaf.sched_wfq_n_groups_max); + printf("cap.nonleaf.sched_wfq_weight_max %" PRIu32 "\n", + lcap.nonleaf.sched_wfq_weight_max); + printf("cap.nonleaf.stats_mask %" PRIx64 "\n", + lcap.nonleaf.stats_mask); + } else { + printf("cap.leaf.shaper_private_supported %" PRId32 "\n", + lcap.leaf.shaper_private_supported); + printf("cap.leaf.shaper_private_dual_rate_supported %" PRId32 + "\n", lcap.leaf.shaper_private_dual_rate_supported); + printf("cap.leaf.shaper_private_rate_min %" PRIu64 "\n", + lcap.leaf.shaper_private_rate_min); + printf("cap.leaf.shaper_private_rate_max %" PRIu64 "\n", + lcap.leaf.shaper_private_rate_max); + printf("cap.leaf.shaper_shared_n_max %" PRIu32 "\n", + lcap.leaf.shaper_shared_n_max); + printf("cap.leaf.cman_head_drop_supported %" PRId32 "\n", + lcap.leaf.cman_head_drop_supported); + printf("cap.leaf.cman_wred_context_private_supported %" PRId32 + "\n", lcap.leaf.cman_wred_context_private_supported); + printf("cap.leaf.cman_wred_context_shared_n_max %" PRIu32 "\n", + lcap.leaf.cman_wred_context_shared_n_max); + printf("cap.leaf.stats_mask %" PRIx64 "\n", + lcap.leaf.stats_mask); + } +} + +cmdline_parse_inst_t cmd_show_port_tm_level_cap = { + .f = cmd_show_port_tm_level_cap_parsed, + .data = NULL, + .help_str = "Show Port TM Hierarhical level Capabilities", + .tokens = { + (void *)&cmd_show_port_tm_level_cap_show, + (void *)&cmd_show_port_tm_level_cap_port, + (void *)&cmd_show_port_tm_level_cap_tm, + (void *)&cmd_show_port_tm_level_cap_level, + (void *)&cmd_show_port_tm_level_cap_cap, + (void *)&cmd_show_port_tm_level_cap_port_id, + (void *)&cmd_show_port_tm_level_cap_level_id, + NULL, + }, +}; + +/* *** Port TM Hierarchy Node Capability *** */ +struct cmd_show_port_tm_node_cap_result { + cmdline_fixed_string_t show; + cmdline_fixed_string_t port; + cmdline_fixed_string_t tm; + cmdline_fixed_string_t node; + cmdline_fixed_string_t cap; + uint16_t port_id; + uint32_t node_id; +}; + +cmdline_parse_token_string_t cmd_show_port_tm_node_cap_show = + TOKEN_STRING_INITIALIZER(struct cmd_show_port_tm_node_cap_result, + show, "show"); +cmdline_parse_token_string_t cmd_show_port_tm_node_cap_port = + TOKEN_STRING_INITIALIZER(struct cmd_show_port_tm_node_cap_result, + port, "port"); +cmdline_parse_token_string_t cmd_show_port_tm_node_cap_tm = + TOKEN_STRING_INITIALIZER(struct cmd_show_port_tm_node_cap_result, + tm, "tm"); +cmdline_parse_token_string_t cmd_show_port_tm_node_cap_node = + TOKEN_STRING_INITIALIZER(struct cmd_show_port_tm_node_cap_result, + node, "node"); +cmdline_parse_token_string_t cmd_show_port_tm_node_cap_cap = + TOKEN_STRING_INITIALIZER(struct cmd_show_port_tm_node_cap_result, + cap, "cap"); +cmdline_parse_token_num_t cmd_show_port_tm_node_cap_port_id = + TOKEN_NUM_INITIALIZER(struct cmd_show_port_tm_node_cap_result, + port_id, UINT16); +cmdline_parse_token_num_t cmd_show_port_tm_node_cap_node_id = + TOKEN_NUM_INITIALIZER(struct cmd_show_port_tm_node_cap_result, + node_id, UINT32); + +static void cmd_show_port_tm_node_cap_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_show_port_tm_node_cap_result *res = parsed_result; + struct rte_tm_node_capabilities ncap; + struct rte_tm_error error; + uint32_t node_id = res->node_id; + portid_t port_id = res->port_id; + int ret, is_leaf = 0; + + if (port_id_is_invalid(port_id, ENABLED_WARN)) + return; + + /* Node id must be valid */ + ret = rte_tm_node_type_get(port_id, node_id, &is_leaf, &error); + if (ret != 0) { + print_err_msg(&error); + return; + } + + memset(&ncap, 0, sizeof(struct rte_tm_node_capabilities)); + ret = rte_tm_node_capabilities_get(port_id, node_id, &ncap, &error); + if (ret != 0) { + print_err_msg(&error); + return; + } + printf("\n** Port TM Hierarchy node %" PRIu32 " Capability **\n\n", + node_id); + printf("cap.shaper_private_supported %" PRId32 "\n", + ncap.shaper_private_supported); + printf("cap.shaper_private_dual_rate_supported %" PRId32 "\n", + ncap.shaper_private_dual_rate_supported); + printf("cap.shaper_private_rate_min %" PRIu64 "\n", + ncap.shaper_private_rate_min); + printf("cap.shaper_private_rate_max %" PRIu64 "\n", + ncap.shaper_private_rate_max); + printf("cap.shaper_shared_n_max %" PRIu32 "\n", + ncap.shaper_shared_n_max); + if (!is_leaf) { + printf("cap.nonleaf.sched_n_children_max %" PRIu32 "\n", + ncap.nonleaf.sched_n_children_max); + printf("cap.nonleaf.sched_sp_n_priorities_max %" PRIu32 "\n", + ncap.nonleaf.sched_sp_n_priorities_max); + printf("cap.nonleaf.sched_wfq_n_children_per_group_max %" PRIu32 + "\n", ncap.nonleaf.sched_wfq_n_children_per_group_max); + printf("cap.nonleaf.sched_wfq_n_groups_max %" PRIu32 "\n", + ncap.nonleaf.sched_wfq_n_groups_max); + printf("cap.nonleaf.sched_wfq_weight_max %" PRIu32 "\n", + ncap.nonleaf.sched_wfq_weight_max); + } else { + printf("cap.leaf.cman_head_drop_supported %" PRId32 "\n", + ncap.leaf.cman_head_drop_supported); + printf("cap.leaf.cman_wred_context_private_supported %" PRId32 + "\n", ncap.leaf.cman_wred_context_private_supported); + printf("cap.leaf.cman_wred_context_shared_n_max %" PRIu32 "\n", + ncap.leaf.cman_wred_context_shared_n_max); + } + printf("cap.stats_mask %" PRIx64 "\n", ncap.stats_mask); +} + +cmdline_parse_inst_t cmd_show_port_tm_node_cap = { + .f = cmd_show_port_tm_node_cap_parsed, + .data = NULL, + .help_str = "Show Port TM Hierarchy node capabilities", + .tokens = { + (void *)&cmd_show_port_tm_node_cap_show, + (void *)&cmd_show_port_tm_node_cap_port, + (void *)&cmd_show_port_tm_node_cap_tm, + (void *)&cmd_show_port_tm_node_cap_node, + (void *)&cmd_show_port_tm_node_cap_cap, + (void *)&cmd_show_port_tm_node_cap_port_id, + (void *)&cmd_show_port_tm_node_cap_node_id, + NULL, + }, +}; + +/* *** Show Port TM Node Statistics *** */ +struct cmd_show_port_tm_node_stats_result { + cmdline_fixed_string_t show; + cmdline_fixed_string_t port; + cmdline_fixed_string_t tm; + cmdline_fixed_string_t node; + cmdline_fixed_string_t stats; + uint16_t port_id; + uint32_t node_id; + uint32_t clear; +}; + +cmdline_parse_token_string_t cmd_show_port_tm_node_stats_show = + TOKEN_STRING_INITIALIZER( + struct cmd_show_port_tm_node_stats_result, show, "show"); +cmdline_parse_token_string_t cmd_show_port_tm_node_stats_port = + TOKEN_STRING_INITIALIZER( + struct cmd_show_port_tm_node_stats_result, port, "port"); +cmdline_parse_token_string_t cmd_show_port_tm_node_stats_tm = + TOKEN_STRING_INITIALIZER( + struct cmd_show_port_tm_node_stats_result, tm, "tm"); +cmdline_parse_token_string_t cmd_show_port_tm_node_stats_node = + TOKEN_STRING_INITIALIZER( + struct cmd_show_port_tm_node_stats_result, node, "node"); +cmdline_parse_token_string_t cmd_show_port_tm_node_stats_stats = + TOKEN_STRING_INITIALIZER( + struct cmd_show_port_tm_node_stats_result, stats, "stats"); +cmdline_parse_token_num_t cmd_show_port_tm_node_stats_port_id = + TOKEN_NUM_INITIALIZER(struct cmd_show_port_tm_node_stats_result, + port_id, UINT16); +cmdline_parse_token_num_t cmd_show_port_tm_node_stats_node_id = + TOKEN_NUM_INITIALIZER( + struct cmd_show_port_tm_node_stats_result, + node_id, UINT32); +cmdline_parse_token_num_t cmd_show_port_tm_node_stats_clear = + TOKEN_NUM_INITIALIZER( + struct cmd_show_port_tm_node_stats_result, clear, UINT32); + +static void cmd_show_port_tm_node_stats_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_show_port_tm_node_stats_result *res = parsed_result; + struct rte_tm_node_stats stats; + struct rte_tm_error error; + uint64_t stats_mask = 0; + uint32_t node_id = res->node_id; + uint32_t clear = res->clear; + portid_t port_id = res->port_id; + int ret; + + if (port_id_is_invalid(port_id, ENABLED_WARN)) + return; + + /* Port status */ + if (!port_is_started(port_id)) { + printf(" Port %u not started (error)\n", port_id); + return; + } + + memset(&stats, 0, sizeof(struct rte_tm_node_stats)); + ret = rte_tm_node_stats_read(port_id, node_id, &stats, + &stats_mask, clear, &error); + if (ret != 0) { + print_err_msg(&error); + return; + } + + /* Display stats */ + if (stats_mask & RTE_TM_STATS_N_PKTS) + printf("\tPkts scheduled from node: %" PRIu64 "\n", + stats.n_pkts); + if (stats_mask & RTE_TM_STATS_N_BYTES) + printf("\tBytes scheduled from node: %" PRIu64 "\n", + stats.n_bytes); + if (stats_mask & RTE_TM_STATS_N_PKTS_GREEN_DROPPED) + printf("\tPkts dropped (green): %" PRIu64 "\n", + stats.leaf.n_pkts_dropped[RTE_TM_GREEN]); + if (stats_mask & RTE_TM_STATS_N_PKTS_YELLOW_DROPPED) + printf("\tPkts dropped (yellow): %" PRIu64 "\n", + stats.leaf.n_pkts_dropped[RTE_TM_YELLOW]); + if (stats_mask & RTE_TM_STATS_N_PKTS_RED_DROPPED) + printf("\tPkts dropped (red): %" PRIu64 "\n", + stats.leaf.n_pkts_dropped[RTE_TM_RED]); + if (stats_mask & RTE_TM_STATS_N_BYTES_GREEN_DROPPED) + printf("\tBytes dropped (green): %" PRIu64 "\n", + stats.leaf.n_bytes_dropped[RTE_TM_GREEN]); + if (stats_mask & RTE_TM_STATS_N_BYTES_YELLOW_DROPPED) + printf("\tBytes dropped (yellow): %" PRIu64 "\n", + stats.leaf.n_bytes_dropped[RTE_TM_YELLOW]); + if (stats_mask & RTE_TM_STATS_N_BYTES_RED_DROPPED) + printf("\tBytes dropped (red): %" PRIu64 "\n", + stats.leaf.n_bytes_dropped[RTE_TM_RED]); + if (stats_mask & RTE_TM_STATS_N_PKTS_QUEUED) + printf("\tPkts queued: %" PRIu64 "\n", + stats.leaf.n_pkts_queued); + if (stats_mask & RTE_TM_STATS_N_BYTES_QUEUED) + printf("\tBytes queued: %" PRIu64 "\n", + stats.leaf.n_bytes_queued); +} + +cmdline_parse_inst_t cmd_show_port_tm_node_stats = { + .f = cmd_show_port_tm_node_stats_parsed, + .data = NULL, + .help_str = "Show port tm node stats", + .tokens = { + (void *)&cmd_show_port_tm_node_stats_show, + (void *)&cmd_show_port_tm_node_stats_port, + (void *)&cmd_show_port_tm_node_stats_tm, + (void *)&cmd_show_port_tm_node_stats_node, + (void *)&cmd_show_port_tm_node_stats_stats, + (void *)&cmd_show_port_tm_node_stats_port_id, + (void *)&cmd_show_port_tm_node_stats_node_id, + (void *)&cmd_show_port_tm_node_stats_clear, + NULL, + }, +}; + +/* *** Show Port TM Node Type *** */ +struct cmd_show_port_tm_node_type_result { + cmdline_fixed_string_t show; + cmdline_fixed_string_t port; + cmdline_fixed_string_t tm; + cmdline_fixed_string_t node; + cmdline_fixed_string_t type; + uint16_t port_id; + uint32_t node_id; +}; + +cmdline_parse_token_string_t cmd_show_port_tm_node_type_show = + TOKEN_STRING_INITIALIZER( + struct cmd_show_port_tm_node_type_result, show, "show"); +cmdline_parse_token_string_t cmd_show_port_tm_node_type_port = + TOKEN_STRING_INITIALIZER( + struct cmd_show_port_tm_node_type_result, port, "port"); +cmdline_parse_token_string_t cmd_show_port_tm_node_type_tm = + TOKEN_STRING_INITIALIZER( + struct cmd_show_port_tm_node_type_result, tm, "tm"); +cmdline_parse_token_string_t cmd_show_port_tm_node_type_node = + TOKEN_STRING_INITIALIZER( + struct cmd_show_port_tm_node_type_result, node, "node"); +cmdline_parse_token_string_t cmd_show_port_tm_node_type_type = + TOKEN_STRING_INITIALIZER( + struct cmd_show_port_tm_node_type_result, type, "type"); +cmdline_parse_token_num_t cmd_show_port_tm_node_type_port_id = + TOKEN_NUM_INITIALIZER( + struct cmd_show_port_tm_node_type_result, + port_id, UINT16); +cmdline_parse_token_num_t cmd_show_port_tm_node_type_node_id = + TOKEN_NUM_INITIALIZER( + struct cmd_show_port_tm_node_type_result, + node_id, UINT32); + +static void cmd_show_port_tm_node_type_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_show_port_tm_node_type_result *res = parsed_result; + struct rte_tm_error error; + uint32_t node_id = res->node_id; + portid_t port_id = res->port_id; + int ret, is_leaf = 0; + + if (port_id_is_invalid(port_id, ENABLED_WARN)) + return; + + ret = rte_tm_node_type_get(port_id, node_id, &is_leaf, &error); + if (ret != 0) { + print_err_msg(&error); + return; + } + + if (is_leaf == 1) + printf("leaf node\n"); + else + printf("nonleaf node\n"); + +} + +cmdline_parse_inst_t cmd_show_port_tm_node_type = { + .f = cmd_show_port_tm_node_type_parsed, + .data = NULL, + .help_str = "Show port tm node type", + .tokens = { + (void *)&cmd_show_port_tm_node_type_show, + (void *)&cmd_show_port_tm_node_type_port, + (void *)&cmd_show_port_tm_node_type_tm, + (void *)&cmd_show_port_tm_node_type_node, + (void *)&cmd_show_port_tm_node_type_type, + (void *)&cmd_show_port_tm_node_type_port_id, + (void *)&cmd_show_port_tm_node_type_node_id, + NULL, + }, +}; + +/* *** Add Port TM Private Shaper Profile *** */ +struct cmd_add_port_tm_node_shaper_profile_result { + cmdline_fixed_string_t add; + cmdline_fixed_string_t port; + cmdline_fixed_string_t tm; + cmdline_fixed_string_t node; + cmdline_fixed_string_t shaper; + cmdline_fixed_string_t profile; + uint16_t port_id; + uint32_t shaper_id; + uint64_t tb_rate; + uint64_t tb_size; + uint32_t pktlen_adjust; +}; + +cmdline_parse_token_string_t cmd_add_port_tm_node_shaper_profile_add = + TOKEN_STRING_INITIALIZER( + struct cmd_add_port_tm_node_shaper_profile_result, add, "add"); +cmdline_parse_token_string_t cmd_add_port_tm_node_shaper_profile_port = + TOKEN_STRING_INITIALIZER( + struct cmd_add_port_tm_node_shaper_profile_result, + port, "port"); +cmdline_parse_token_string_t cmd_add_port_tm_node_shaper_profile_tm = + TOKEN_STRING_INITIALIZER( + struct cmd_add_port_tm_node_shaper_profile_result, + tm, "tm"); +cmdline_parse_token_string_t cmd_add_port_tm_node_shaper_profile_node = + TOKEN_STRING_INITIALIZER( + struct cmd_add_port_tm_node_shaper_profile_result, + node, "node"); +cmdline_parse_token_string_t cmd_add_port_tm_node_shaper_profile_shaper = + TOKEN_STRING_INITIALIZER( + struct cmd_add_port_tm_node_shaper_profile_result, + shaper, "shaper"); +cmdline_parse_token_string_t cmd_add_port_tm_node_shaper_profile_profile = + TOKEN_STRING_INITIALIZER( + struct cmd_add_port_tm_node_shaper_profile_result, + profile, "profile"); +cmdline_parse_token_num_t cmd_add_port_tm_node_shaper_profile_port_id = + TOKEN_NUM_INITIALIZER( + struct cmd_add_port_tm_node_shaper_profile_result, + port_id, UINT16); +cmdline_parse_token_num_t cmd_add_port_tm_node_shaper_profile_shaper_id = + TOKEN_NUM_INITIALIZER( + struct cmd_add_port_tm_node_shaper_profile_result, + shaper_id, UINT32); +cmdline_parse_token_num_t cmd_add_port_tm_node_shaper_profile_tb_rate = + TOKEN_NUM_INITIALIZER( + struct cmd_add_port_tm_node_shaper_profile_result, + tb_rate, UINT64); +cmdline_parse_token_num_t cmd_add_port_tm_node_shaper_profile_tb_size = + TOKEN_NUM_INITIALIZER( + struct cmd_add_port_tm_node_shaper_profile_result, + tb_size, UINT64); +cmdline_parse_token_num_t cmd_add_port_tm_node_shaper_profile_pktlen_adjust = + TOKEN_NUM_INITIALIZER( + struct cmd_add_port_tm_node_shaper_profile_result, + pktlen_adjust, UINT32); + +static void cmd_add_port_tm_node_shaper_profile_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_add_port_tm_node_shaper_profile_result *res = parsed_result; + struct rte_tm_shaper_params sp; + struct rte_tm_error error; + uint32_t shaper_id = res->shaper_id; + uint32_t pkt_len_adjust = res->pktlen_adjust; + portid_t port_id = res->port_id; + int ret; + + if (port_id_is_invalid(port_id, ENABLED_WARN)) + return; + + /* Private shaper profile params */ + memset(&sp, 0, sizeof(struct rte_tm_shaper_params)); + sp.peak.rate = res->tb_rate; + sp.peak.size = res->tb_size; + sp.pkt_length_adjust = pkt_len_adjust; + + ret = rte_tm_shaper_profile_add(port_id, shaper_id, &sp, &error); + if (ret != 0) { + print_err_msg(&error); + return; + } +} + +cmdline_parse_inst_t cmd_add_port_tm_node_shaper_profile = { + .f = cmd_add_port_tm_node_shaper_profile_parsed, + .data = NULL, + .help_str = "Add port tm node private shaper profile", + .tokens = { + (void *)&cmd_add_port_tm_node_shaper_profile_add, + (void *)&cmd_add_port_tm_node_shaper_profile_port, + (void *)&cmd_add_port_tm_node_shaper_profile_tm, + (void *)&cmd_add_port_tm_node_shaper_profile_node, + (void *)&cmd_add_port_tm_node_shaper_profile_shaper, + (void *)&cmd_add_port_tm_node_shaper_profile_profile, + (void *)&cmd_add_port_tm_node_shaper_profile_port_id, + (void *)&cmd_add_port_tm_node_shaper_profile_shaper_id, + (void *)&cmd_add_port_tm_node_shaper_profile_tb_rate, + (void *)&cmd_add_port_tm_node_shaper_profile_tb_size, + (void *)&cmd_add_port_tm_node_shaper_profile_pktlen_adjust, + NULL, + }, +}; + +/* *** Delete Port TM Private Shaper Profile *** */ +struct cmd_del_port_tm_node_shaper_profile_result { + cmdline_fixed_string_t del; + cmdline_fixed_string_t port; + cmdline_fixed_string_t tm; + cmdline_fixed_string_t node; + cmdline_fixed_string_t shaper; + cmdline_fixed_string_t profile; + uint16_t port_id; + uint32_t shaper_id; +}; + +cmdline_parse_token_string_t cmd_del_port_tm_node_shaper_profile_del = + TOKEN_STRING_INITIALIZER( + struct cmd_del_port_tm_node_shaper_profile_result, del, "del"); +cmdline_parse_token_string_t cmd_del_port_tm_node_shaper_profile_port = + TOKEN_STRING_INITIALIZER( + struct cmd_del_port_tm_node_shaper_profile_result, + port, "port"); +cmdline_parse_token_string_t cmd_del_port_tm_node_shaper_profile_tm = + TOKEN_STRING_INITIALIZER( + struct cmd_del_port_tm_node_shaper_profile_result, tm, "tm"); +cmdline_parse_token_string_t cmd_del_port_tm_node_shaper_profile_node = + TOKEN_STRING_INITIALIZER( + struct cmd_del_port_tm_node_shaper_profile_result, + node, "node"); +cmdline_parse_token_string_t cmd_del_port_tm_node_shaper_profile_shaper = + TOKEN_STRING_INITIALIZER( + struct cmd_del_port_tm_node_shaper_profile_result, + shaper, "shaper"); +cmdline_parse_token_string_t cmd_del_port_tm_node_shaper_profile_profile = + TOKEN_STRING_INITIALIZER( + struct cmd_del_port_tm_node_shaper_profile_result, + profile, "profile"); +cmdline_parse_token_num_t cmd_del_port_tm_node_shaper_profile_port_id = + TOKEN_NUM_INITIALIZER( + struct cmd_del_port_tm_node_shaper_profile_result, + port_id, UINT16); +cmdline_parse_token_num_t cmd_del_port_tm_node_shaper_profile_shaper_id = + TOKEN_NUM_INITIALIZER( + struct cmd_del_port_tm_node_shaper_profile_result, + shaper_id, UINT32); + +static void cmd_del_port_tm_node_shaper_profile_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_del_port_tm_node_shaper_profile_result *res = parsed_result; + struct rte_tm_error error; + uint32_t shaper_id = res->shaper_id; + portid_t port_id = res->port_id; + int ret; + + if (port_id_is_invalid(port_id, ENABLED_WARN)) + return; + + ret = rte_tm_shaper_profile_delete(port_id, shaper_id, &error); + if (ret != 0) { + print_err_msg(&error); + return; + } +} + +cmdline_parse_inst_t cmd_del_port_tm_node_shaper_profile = { + .f = cmd_del_port_tm_node_shaper_profile_parsed, + .data = NULL, + .help_str = "Delete port tm node private shaper profile", + .tokens = { + (void *)&cmd_del_port_tm_node_shaper_profile_del, + (void *)&cmd_del_port_tm_node_shaper_profile_port, + (void *)&cmd_del_port_tm_node_shaper_profile_tm, + (void *)&cmd_del_port_tm_node_shaper_profile_node, + (void *)&cmd_del_port_tm_node_shaper_profile_shaper, + (void *)&cmd_del_port_tm_node_shaper_profile_profile, + (void *)&cmd_del_port_tm_node_shaper_profile_port_id, + (void *)&cmd_del_port_tm_node_shaper_profile_shaper_id, + NULL, + }, +}; + +/* *** Add/Update Port TM shared Shaper *** */ +struct cmd_add_port_tm_node_shared_shaper_result { + cmdline_fixed_string_t cmd_type; + cmdline_fixed_string_t port; + cmdline_fixed_string_t tm; + cmdline_fixed_string_t node; + cmdline_fixed_string_t shared; + cmdline_fixed_string_t shaper; + uint16_t port_id; + uint32_t shared_shaper_id; + uint32_t shaper_profile_id; +}; + +cmdline_parse_token_string_t cmd_add_port_tm_node_shared_shaper_cmd_type = + TOKEN_STRING_INITIALIZER( + struct cmd_add_port_tm_node_shared_shaper_result, + cmd_type, "add#set"); +cmdline_parse_token_string_t cmd_add_port_tm_node_shared_shaper_port = + TOKEN_STRING_INITIALIZER( + struct cmd_add_port_tm_node_shared_shaper_result, port, "port"); +cmdline_parse_token_string_t cmd_add_port_tm_node_shared_shaper_tm = + TOKEN_STRING_INITIALIZER( + struct cmd_add_port_tm_node_shared_shaper_result, tm, "tm"); +cmdline_parse_token_string_t cmd_add_port_tm_node_shared_shaper_node = + TOKEN_STRING_INITIALIZER( + struct cmd_add_port_tm_node_shared_shaper_result, node, "node"); +cmdline_parse_token_string_t cmd_add_port_tm_node_shared_shaper_shared = + TOKEN_STRING_INITIALIZER( + struct cmd_add_port_tm_node_shared_shaper_result, + shared, "shared"); +cmdline_parse_token_string_t cmd_add_port_tm_node_shared_shaper_shaper = + TOKEN_STRING_INITIALIZER( + struct cmd_add_port_tm_node_shared_shaper_result, + shaper, "shaper"); +cmdline_parse_token_num_t cmd_add_port_tm_node_shared_shaper_port_id = + TOKEN_NUM_INITIALIZER( + struct cmd_add_port_tm_node_shared_shaper_result, + port_id, UINT16); +cmdline_parse_token_num_t cmd_add_port_tm_node_shared_shaper_shared_shaper_id = + TOKEN_NUM_INITIALIZER( + struct cmd_add_port_tm_node_shared_shaper_result, + shared_shaper_id, UINT32); +cmdline_parse_token_num_t cmd_add_port_tm_node_shared_shaper_shaper_profile_id = + TOKEN_NUM_INITIALIZER( + struct cmd_add_port_tm_node_shared_shaper_result, + shaper_profile_id, UINT32); + +static void cmd_add_port_tm_node_shared_shaper_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_add_port_tm_node_shared_shaper_result *res = parsed_result; + struct rte_tm_error error; + uint32_t shared_shaper_id = res->shared_shaper_id; + uint32_t shaper_profile_id = res->shaper_profile_id; + portid_t port_id = res->port_id; + int ret; + + if (port_id_is_invalid(port_id, ENABLED_WARN)) + return; + + /* Command type: add */ + if ((strcmp(res->cmd_type, "add") == 0) && + (port_is_started(port_id))) { + printf(" Port %u not stopped (error)\n", port_id); + return; + } + + /* Command type: set (update) */ + if ((strcmp(res->cmd_type, "set") == 0) && + (!port_is_started(port_id))) { + printf(" Port %u not started (error)\n", port_id); + return; + } + + ret = rte_tm_shared_shaper_add_update(port_id, shared_shaper_id, + shaper_profile_id, &error); + if (ret != 0) { + print_err_msg(&error); + return; + } +} + +cmdline_parse_inst_t cmd_add_port_tm_node_shared_shaper = { + .f = cmd_add_port_tm_node_shared_shaper_parsed, + .data = NULL, + .help_str = "add/update port tm node shared shaper", + .tokens = { + (void *)&cmd_add_port_tm_node_shared_shaper_cmd_type, + (void *)&cmd_add_port_tm_node_shared_shaper_port, + (void *)&cmd_add_port_tm_node_shared_shaper_tm, + (void *)&cmd_add_port_tm_node_shared_shaper_node, + (void *)&cmd_add_port_tm_node_shared_shaper_shared, + (void *)&cmd_add_port_tm_node_shared_shaper_shaper, + (void *)&cmd_add_port_tm_node_shared_shaper_port_id, + (void *)&cmd_add_port_tm_node_shared_shaper_shared_shaper_id, + (void *)&cmd_add_port_tm_node_shared_shaper_shaper_profile_id, + NULL, + }, +}; + +/* *** Delete Port TM shared Shaper *** */ +struct cmd_del_port_tm_node_shared_shaper_result { + cmdline_fixed_string_t del; + cmdline_fixed_string_t port; + cmdline_fixed_string_t tm; + cmdline_fixed_string_t node; + cmdline_fixed_string_t shared; + cmdline_fixed_string_t shaper; + uint16_t port_id; + uint32_t shared_shaper_id; +}; + +cmdline_parse_token_string_t cmd_del_port_tm_node_shared_shaper_del = + TOKEN_STRING_INITIALIZER( + struct cmd_del_port_tm_node_shared_shaper_result, del, "del"); +cmdline_parse_token_string_t cmd_del_port_tm_node_shared_shaper_port = + TOKEN_STRING_INITIALIZER( + struct cmd_del_port_tm_node_shared_shaper_result, port, "port"); +cmdline_parse_token_string_t cmd_del_port_tm_node_shared_shaper_tm = + TOKEN_STRING_INITIALIZER( + struct cmd_del_port_tm_node_shared_shaper_result, tm, "tm"); +cmdline_parse_token_string_t cmd_del_port_tm_node_shared_shaper_node = + TOKEN_STRING_INITIALIZER( + struct cmd_del_port_tm_node_shared_shaper_result, node, "node"); +cmdline_parse_token_string_t cmd_del_port_tm_node_shared_shaper_shared = + TOKEN_STRING_INITIALIZER( + struct cmd_del_port_tm_node_shared_shaper_result, + shared, "shared"); +cmdline_parse_token_string_t cmd_del_port_tm_node_shared_shaper_shaper = + TOKEN_STRING_INITIALIZER( + struct cmd_del_port_tm_node_shared_shaper_result, + shaper, "shaper"); +cmdline_parse_token_num_t cmd_del_port_tm_node_shared_shaper_port_id = + TOKEN_NUM_INITIALIZER( + struct cmd_del_port_tm_node_shared_shaper_result, + port_id, UINT16); +cmdline_parse_token_num_t cmd_del_port_tm_node_shared_shaper_shared_shaper_id = + TOKEN_NUM_INITIALIZER( + struct cmd_del_port_tm_node_shared_shaper_result, + shared_shaper_id, UINT32); + +static void cmd_del_port_tm_node_shared_shaper_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_del_port_tm_node_shared_shaper_result *res = parsed_result; + struct rte_tm_error error; + uint32_t shared_shaper_id = res->shared_shaper_id; + portid_t port_id = res->port_id; + int ret; + + if (port_id_is_invalid(port_id, ENABLED_WARN)) + return; + + ret = rte_tm_shared_shaper_delete(port_id, shared_shaper_id, &error); + if (ret != 0) { + print_err_msg(&error); + return; + } +} + +cmdline_parse_inst_t cmd_del_port_tm_node_shared_shaper = { + .f = cmd_del_port_tm_node_shared_shaper_parsed, + .data = NULL, + .help_str = "delete port tm node shared shaper", + .tokens = { + (void *)&cmd_del_port_tm_node_shared_shaper_del, + (void *)&cmd_del_port_tm_node_shared_shaper_port, + (void *)&cmd_del_port_tm_node_shared_shaper_tm, + (void *)&cmd_del_port_tm_node_shared_shaper_node, + (void *)&cmd_del_port_tm_node_shared_shaper_shared, + (void *)&cmd_del_port_tm_node_shared_shaper_shaper, + (void *)&cmd_del_port_tm_node_shared_shaper_port_id, + (void *)&cmd_del_port_tm_node_shared_shaper_shared_shaper_id, + NULL, + }, +}; + +/* *** Add Port TM Node WRED Profile *** */ +struct cmd_add_port_tm_node_wred_profile_result { + cmdline_fixed_string_t add; + cmdline_fixed_string_t port; + cmdline_fixed_string_t tm; + cmdline_fixed_string_t node; + cmdline_fixed_string_t wred; + cmdline_fixed_string_t profile; + uint16_t port_id; + uint32_t wred_profile_id; + cmdline_fixed_string_t color_g; + uint16_t min_th_g; + uint16_t max_th_g; + uint16_t maxp_inv_g; + uint16_t wq_log2_g; + cmdline_fixed_string_t color_y; + uint16_t min_th_y; + uint16_t max_th_y; + uint16_t maxp_inv_y; + uint16_t wq_log2_y; + cmdline_fixed_string_t color_r; + uint16_t min_th_r; + uint16_t max_th_r; + uint16_t maxp_inv_r; + uint16_t wq_log2_r; +}; + +cmdline_parse_token_string_t cmd_add_port_tm_node_wred_profile_add = + TOKEN_STRING_INITIALIZER( + struct cmd_add_port_tm_node_wred_profile_result, add, "add"); +cmdline_parse_token_string_t cmd_add_port_tm_node_wred_profile_port = + TOKEN_STRING_INITIALIZER( + struct cmd_add_port_tm_node_wred_profile_result, port, "port"); +cmdline_parse_token_string_t cmd_add_port_tm_node_wred_profile_tm = + TOKEN_STRING_INITIALIZER( + struct cmd_add_port_tm_node_wred_profile_result, tm, "tm"); +cmdline_parse_token_string_t cmd_add_port_tm_node_wred_profile_node = + TOKEN_STRING_INITIALIZER( + struct cmd_add_port_tm_node_wred_profile_result, node, "node"); +cmdline_parse_token_string_t cmd_add_port_tm_node_wred_profile_wred = + TOKEN_STRING_INITIALIZER( + struct cmd_add_port_tm_node_wred_profile_result, wred, "wred"); +cmdline_parse_token_string_t cmd_add_port_tm_node_wred_profile_profile = + TOKEN_STRING_INITIALIZER( + struct cmd_add_port_tm_node_wred_profile_result, + profile, "profile"); +cmdline_parse_token_num_t cmd_add_port_tm_node_wred_profile_port_id = + TOKEN_NUM_INITIALIZER( + struct cmd_add_port_tm_node_wred_profile_result, + port_id, UINT16); +cmdline_parse_token_num_t cmd_add_port_tm_node_wred_profile_wred_profile_id = + TOKEN_NUM_INITIALIZER( + struct cmd_add_port_tm_node_wred_profile_result, + wred_profile_id, UINT32); +cmdline_parse_token_string_t cmd_add_port_tm_node_wred_profile_color_g = + TOKEN_STRING_INITIALIZER( + struct cmd_add_port_tm_node_wred_profile_result, + color_g, "G#g"); +cmdline_parse_token_num_t cmd_add_port_tm_node_wred_profile_min_th_g = + TOKEN_NUM_INITIALIZER( + struct cmd_add_port_tm_node_wred_profile_result, + min_th_g, UINT16); +cmdline_parse_token_num_t cmd_add_port_tm_node_wred_profile_max_th_g = + TOKEN_NUM_INITIALIZER( + struct cmd_add_port_tm_node_wred_profile_result, + max_th_g, UINT16); +cmdline_parse_token_num_t cmd_add_port_tm_node_wred_profile_maxp_inv_g = + TOKEN_NUM_INITIALIZER( + struct cmd_add_port_tm_node_wred_profile_result, + maxp_inv_g, UINT16); +cmdline_parse_token_num_t cmd_add_port_tm_node_wred_profile_wq_log2_g = + TOKEN_NUM_INITIALIZER( + struct cmd_add_port_tm_node_wred_profile_result, + wq_log2_g, UINT16); +cmdline_parse_token_string_t cmd_add_port_tm_node_wred_profile_color_y = + TOKEN_STRING_INITIALIZER( + struct cmd_add_port_tm_node_wred_profile_result, + color_y, "Y#y"); +cmdline_parse_token_num_t cmd_add_port_tm_node_wred_profile_min_th_y = + TOKEN_NUM_INITIALIZER( + struct cmd_add_port_tm_node_wred_profile_result, + min_th_y, UINT16); +cmdline_parse_token_num_t cmd_add_port_tm_node_wred_profile_max_th_y = + TOKEN_NUM_INITIALIZER( + struct cmd_add_port_tm_node_wred_profile_result, + max_th_y, UINT16); +cmdline_parse_token_num_t cmd_add_port_tm_node_wred_profile_maxp_inv_y = + TOKEN_NUM_INITIALIZER( + struct cmd_add_port_tm_node_wred_profile_result, + maxp_inv_y, UINT16); +cmdline_parse_token_num_t cmd_add_port_tm_node_wred_profile_wq_log2_y = + TOKEN_NUM_INITIALIZER( + struct cmd_add_port_tm_node_wred_profile_result, + wq_log2_y, UINT16); +cmdline_parse_token_string_t cmd_add_port_tm_node_wred_profile_color_r = + TOKEN_STRING_INITIALIZER( + struct cmd_add_port_tm_node_wred_profile_result, + color_r, "R#r"); +cmdline_parse_token_num_t cmd_add_port_tm_node_wred_profile_min_th_r = + TOKEN_NUM_INITIALIZER( + struct cmd_add_port_tm_node_wred_profile_result, + min_th_r, UINT16); +cmdline_parse_token_num_t cmd_add_port_tm_node_wred_profile_max_th_r = + TOKEN_NUM_INITIALIZER( + struct cmd_add_port_tm_node_wred_profile_result, + max_th_r, UINT16); +cmdline_parse_token_num_t cmd_add_port_tm_node_wred_profile_maxp_inv_r = + TOKEN_NUM_INITIALIZER( + struct cmd_add_port_tm_node_wred_profile_result, + maxp_inv_r, UINT16); +cmdline_parse_token_num_t cmd_add_port_tm_node_wred_profile_wq_log2_r = + TOKEN_NUM_INITIALIZER( + struct cmd_add_port_tm_node_wred_profile_result, + wq_log2_r, UINT16); + + +static void cmd_add_port_tm_node_wred_profile_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_add_port_tm_node_wred_profile_result *res = parsed_result; + struct rte_tm_wred_params wp; + enum rte_tm_color color; + struct rte_tm_error error; + uint32_t wred_profile_id = res->wred_profile_id; + portid_t port_id = res->port_id; + int ret; + + if (port_id_is_invalid(port_id, ENABLED_WARN)) + return; + + memset(&wp, 0, sizeof(struct rte_tm_wred_params)); + + /* WRED Params (Green Color)*/ + color = RTE_TM_GREEN; + wp.red_params[color].min_th = res->min_th_g; + wp.red_params[color].max_th = res->max_th_g; + wp.red_params[color].maxp_inv = res->maxp_inv_g; + wp.red_params[color].wq_log2 = res->wq_log2_g; + + + /* WRED Params (Yellow Color)*/ + color = RTE_TM_YELLOW; + wp.red_params[color].min_th = res->min_th_y; + wp.red_params[color].max_th = res->max_th_y; + wp.red_params[color].maxp_inv = res->maxp_inv_y; + wp.red_params[color].wq_log2 = res->wq_log2_y; + + /* WRED Params (Red Color)*/ + color = RTE_TM_RED; + wp.red_params[color].min_th = res->min_th_r; + wp.red_params[color].max_th = res->max_th_r; + wp.red_params[color].maxp_inv = res->maxp_inv_r; + wp.red_params[color].wq_log2 = res->wq_log2_r; + + ret = rte_tm_wred_profile_add(port_id, wred_profile_id, &wp, &error); + if (ret != 0) { + print_err_msg(&error); + return; + } +} + +cmdline_parse_inst_t cmd_add_port_tm_node_wred_profile = { + .f = cmd_add_port_tm_node_wred_profile_parsed, + .data = NULL, + .help_str = "Add port tm node wred profile", + .tokens = { + (void *)&cmd_add_port_tm_node_wred_profile_add, + (void *)&cmd_add_port_tm_node_wred_profile_port, + (void *)&cmd_add_port_tm_node_wred_profile_tm, + (void *)&cmd_add_port_tm_node_wred_profile_node, + (void *)&cmd_add_port_tm_node_wred_profile_wred, + (void *)&cmd_add_port_tm_node_wred_profile_profile, + (void *)&cmd_add_port_tm_node_wred_profile_port_id, + (void *)&cmd_add_port_tm_node_wred_profile_wred_profile_id, + (void *)&cmd_add_port_tm_node_wred_profile_color_g, + (void *)&cmd_add_port_tm_node_wred_profile_min_th_g, + (void *)&cmd_add_port_tm_node_wred_profile_max_th_g, + (void *)&cmd_add_port_tm_node_wred_profile_maxp_inv_g, + (void *)&cmd_add_port_tm_node_wred_profile_wq_log2_g, + (void *)&cmd_add_port_tm_node_wred_profile_color_y, + (void *)&cmd_add_port_tm_node_wred_profile_min_th_y, + (void *)&cmd_add_port_tm_node_wred_profile_max_th_y, + (void *)&cmd_add_port_tm_node_wred_profile_maxp_inv_y, + (void *)&cmd_add_port_tm_node_wred_profile_wq_log2_y, + (void *)&cmd_add_port_tm_node_wred_profile_color_r, + (void *)&cmd_add_port_tm_node_wred_profile_min_th_r, + (void *)&cmd_add_port_tm_node_wred_profile_max_th_r, + (void *)&cmd_add_port_tm_node_wred_profile_maxp_inv_r, + (void *)&cmd_add_port_tm_node_wred_profile_wq_log2_r, + NULL, + }, +}; + +/* *** Delete Port TM node WRED Profile *** */ +struct cmd_del_port_tm_node_wred_profile_result { + cmdline_fixed_string_t del; + cmdline_fixed_string_t port; + cmdline_fixed_string_t tm; + cmdline_fixed_string_t node; + cmdline_fixed_string_t wred; + cmdline_fixed_string_t profile; + uint16_t port_id; + uint32_t wred_profile_id; +}; + +cmdline_parse_token_string_t cmd_del_port_tm_node_wred_profile_del = + TOKEN_STRING_INITIALIZER( + struct cmd_del_port_tm_node_wred_profile_result, del, "del"); +cmdline_parse_token_string_t cmd_del_port_tm_node_wred_profile_port = + TOKEN_STRING_INITIALIZER( + struct cmd_del_port_tm_node_wred_profile_result, port, "port"); +cmdline_parse_token_string_t cmd_del_port_tm_node_wred_profile_tm = + TOKEN_STRING_INITIALIZER( + struct cmd_del_port_tm_node_wred_profile_result, tm, "tm"); +cmdline_parse_token_string_t cmd_del_port_tm_node_wred_profile_node = + TOKEN_STRING_INITIALIZER( + struct cmd_del_port_tm_node_wred_profile_result, node, "node"); +cmdline_parse_token_string_t cmd_del_port_tm_node_wred_profile_wred = + TOKEN_STRING_INITIALIZER( + struct cmd_del_port_tm_node_wred_profile_result, wred, "wred"); +cmdline_parse_token_string_t cmd_del_port_tm_node_wred_profile_profile = + TOKEN_STRING_INITIALIZER( + struct cmd_del_port_tm_node_wred_profile_result, + profile, "profile"); +cmdline_parse_token_num_t cmd_del_port_tm_node_wred_profile_port_id = + TOKEN_NUM_INITIALIZER( + struct cmd_del_port_tm_node_wred_profile_result, + port_id, UINT16); +cmdline_parse_token_num_t cmd_del_port_tm_node_wred_profile_wred_profile_id = + TOKEN_NUM_INITIALIZER( + struct cmd_del_port_tm_node_wred_profile_result, + wred_profile_id, UINT32); + +static void cmd_del_port_tm_node_wred_profile_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_del_port_tm_node_wred_profile_result *res = parsed_result; + struct rte_tm_error error; + uint32_t wred_profile_id = res->wred_profile_id; + portid_t port_id = res->port_id; + int ret; + + if (port_id_is_invalid(port_id, ENABLED_WARN)) + return; + + ret = rte_tm_wred_profile_delete(port_id, wred_profile_id, &error); + if (ret != 0) { + print_err_msg(&error); + return; + } +} + +cmdline_parse_inst_t cmd_del_port_tm_node_wred_profile = { + .f = cmd_del_port_tm_node_wred_profile_parsed, + .data = NULL, + .help_str = "Delete port tm node wred profile", + .tokens = { + (void *)&cmd_del_port_tm_node_wred_profile_del, + (void *)&cmd_del_port_tm_node_wred_profile_port, + (void *)&cmd_del_port_tm_node_wred_profile_tm, + (void *)&cmd_del_port_tm_node_wred_profile_node, + (void *)&cmd_del_port_tm_node_wred_profile_wred, + (void *)&cmd_del_port_tm_node_wred_profile_profile, + (void *)&cmd_del_port_tm_node_wred_profile_port_id, + (void *)&cmd_del_port_tm_node_wred_profile_wred_profile_id, + NULL, + }, +}; + +/* *** Update Port TM Node Shaper profile *** */ +struct cmd_set_port_tm_node_shaper_profile_result { + cmdline_fixed_string_t set; + cmdline_fixed_string_t port; + cmdline_fixed_string_t tm; + cmdline_fixed_string_t node; + cmdline_fixed_string_t shaper; + cmdline_fixed_string_t profile; + uint16_t port_id; + uint32_t node_id; + uint32_t shaper_profile_id; +}; + +cmdline_parse_token_string_t cmd_set_port_tm_node_shaper_profile_set = + TOKEN_STRING_INITIALIZER( + struct cmd_set_port_tm_node_shaper_profile_result, set, "set"); +cmdline_parse_token_string_t cmd_set_port_tm_node_shaper_profile_port = + TOKEN_STRING_INITIALIZER( + struct cmd_set_port_tm_node_shaper_profile_result, + port, "port"); +cmdline_parse_token_string_t cmd_set_port_tm_node_shaper_profile_tm = + TOKEN_STRING_INITIALIZER( + struct cmd_set_port_tm_node_shaper_profile_result, tm, "tm"); +cmdline_parse_token_string_t cmd_set_port_tm_node_shaper_profile_node = + TOKEN_STRING_INITIALIZER( + struct cmd_set_port_tm_node_shaper_profile_result, + node, "node"); +cmdline_parse_token_string_t cmd_set_port_tm_node_shaper_profile_shaper = + TOKEN_STRING_INITIALIZER( + struct cmd_set_port_tm_node_shaper_profile_result, + shaper, "shaper"); +cmdline_parse_token_string_t cmd_set_port_tm_node_shaper_profile_profile = + TOKEN_STRING_INITIALIZER( + struct cmd_set_port_tm_node_shaper_profile_result, + profile, "profile"); +cmdline_parse_token_num_t cmd_set_port_tm_node_shaper_profile_port_id = + TOKEN_NUM_INITIALIZER( + struct cmd_set_port_tm_node_shaper_profile_result, + port_id, UINT16); +cmdline_parse_token_num_t cmd_set_port_tm_node_shaper_profile_node_id = + TOKEN_NUM_INITIALIZER(struct cmd_set_port_tm_node_shaper_profile_result, + node_id, UINT32); +cmdline_parse_token_num_t + cmd_set_port_tm_node_shaper_shaper_profile_profile_id = + TOKEN_NUM_INITIALIZER( + struct cmd_set_port_tm_node_shaper_profile_result, + shaper_profile_id, UINT32); + +static void cmd_set_port_tm_node_shaper_profile_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_set_port_tm_node_shaper_profile_result *res = parsed_result; + struct rte_tm_error error; + uint32_t node_id = res->node_id; + uint32_t shaper_profile_id = res->shaper_profile_id; + portid_t port_id = res->port_id; + int ret; + + if (port_id_is_invalid(port_id, ENABLED_WARN)) + return; + + /* Port status */ + if (!port_is_started(port_id)) { + printf(" Port %u not started (error)\n", port_id); + return; + } + + ret = rte_tm_node_shaper_update(port_id, node_id, + shaper_profile_id, &error); + if (ret != 0) { + print_err_msg(&error); + return; + } +} + +cmdline_parse_inst_t cmd_set_port_tm_node_shaper_profile = { + .f = cmd_set_port_tm_node_shaper_profile_parsed, + .data = NULL, + .help_str = "Set port tm node shaper profile", + .tokens = { + (void *)&cmd_set_port_tm_node_shaper_profile_set, + (void *)&cmd_set_port_tm_node_shaper_profile_port, + (void *)&cmd_set_port_tm_node_shaper_profile_tm, + (void *)&cmd_set_port_tm_node_shaper_profile_node, + (void *)&cmd_set_port_tm_node_shaper_profile_shaper, + (void *)&cmd_set_port_tm_node_shaper_profile_profile, + (void *)&cmd_set_port_tm_node_shaper_profile_port_id, + (void *)&cmd_set_port_tm_node_shaper_profile_node_id, + (void *)&cmd_set_port_tm_node_shaper_shaper_profile_profile_id, + NULL, + }, +}; + +/* *** Add Port TM nonleaf node *** */ +struct cmd_add_port_tm_nonleaf_node_result { + cmdline_fixed_string_t add; + cmdline_fixed_string_t port; + cmdline_fixed_string_t tm; + cmdline_fixed_string_t nonleaf; + cmdline_fixed_string_t node; + uint16_t port_id; + uint32_t node_id; + int32_t parent_node_id; + uint32_t priority; + uint32_t weight; + uint32_t level_id; + uint32_t shaper_profile_id; + uint32_t n_sp_priorities; + uint64_t stats_mask; + cmdline_multi_string_t multi_shared_shaper_id; +}; + +cmdline_parse_token_string_t cmd_add_port_tm_nonleaf_node_add = + TOKEN_STRING_INITIALIZER( + struct cmd_add_port_tm_nonleaf_node_result, add, "add"); +cmdline_parse_token_string_t cmd_add_port_tm_nonleaf_node_port = + TOKEN_STRING_INITIALIZER( + struct cmd_add_port_tm_nonleaf_node_result, port, "port"); +cmdline_parse_token_string_t cmd_add_port_tm_nonleaf_node_tm = + TOKEN_STRING_INITIALIZER( + struct cmd_add_port_tm_nonleaf_node_result, tm, "tm"); +cmdline_parse_token_string_t cmd_add_port_tm_nonleaf_node_nonleaf = + TOKEN_STRING_INITIALIZER( + struct cmd_add_port_tm_nonleaf_node_result, nonleaf, "nonleaf"); +cmdline_parse_token_string_t cmd_add_port_tm_nonleaf_node_node = + TOKEN_STRING_INITIALIZER( + struct cmd_add_port_tm_nonleaf_node_result, node, "node"); +cmdline_parse_token_num_t cmd_add_port_tm_nonleaf_node_port_id = + TOKEN_NUM_INITIALIZER( + struct cmd_add_port_tm_nonleaf_node_result, + port_id, UINT16); +cmdline_parse_token_num_t cmd_add_port_tm_nonleaf_node_node_id = + TOKEN_NUM_INITIALIZER(struct cmd_add_port_tm_nonleaf_node_result, + node_id, UINT32); +cmdline_parse_token_num_t cmd_add_port_tm_nonleaf_node_parent_node_id = + TOKEN_NUM_INITIALIZER(struct cmd_add_port_tm_nonleaf_node_result, + parent_node_id, INT32); +cmdline_parse_token_num_t cmd_add_port_tm_nonleaf_node_priority = + TOKEN_NUM_INITIALIZER(struct cmd_add_port_tm_nonleaf_node_result, + priority, UINT32); +cmdline_parse_token_num_t cmd_add_port_tm_nonleaf_node_weight = + TOKEN_NUM_INITIALIZER(struct cmd_add_port_tm_nonleaf_node_result, + weight, UINT32); +cmdline_parse_token_num_t cmd_add_port_tm_nonleaf_node_level_id = + TOKEN_NUM_INITIALIZER(struct cmd_add_port_tm_nonleaf_node_result, + level_id, UINT32); +cmdline_parse_token_num_t cmd_add_port_tm_nonleaf_node_shaper_profile_id = + TOKEN_NUM_INITIALIZER(struct cmd_add_port_tm_nonleaf_node_result, + shaper_profile_id, UINT32); +cmdline_parse_token_num_t cmd_add_port_tm_nonleaf_node_n_sp_priorities = + TOKEN_NUM_INITIALIZER(struct cmd_add_port_tm_nonleaf_node_result, + n_sp_priorities, UINT32); +cmdline_parse_token_num_t cmd_add_port_tm_nonleaf_node_stats_mask = + TOKEN_NUM_INITIALIZER(struct cmd_add_port_tm_nonleaf_node_result, + stats_mask, UINT64); +cmdline_parse_token_string_t + cmd_add_port_tm_nonleaf_node_multi_shared_shaper_id = + TOKEN_STRING_INITIALIZER(struct cmd_add_port_tm_nonleaf_node_result, + multi_shared_shaper_id, TOKEN_STRING_MULTI); + +static void cmd_add_port_tm_nonleaf_node_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_add_port_tm_nonleaf_node_result *res = parsed_result; + struct rte_tm_error error; + struct rte_tm_node_params np; + uint32_t *shared_shaper_id; + uint32_t parent_node_id, n_shared_shapers = 0; + char *s_str = res->multi_shared_shaper_id; + portid_t port_id = res->port_id; + int ret; + + if (port_id_is_invalid(port_id, ENABLED_WARN)) + return; + + memset(&np, 0, sizeof(struct rte_tm_node_params)); + + /* Node parameters */ + if (res->parent_node_id < 0) + parent_node_id = UINT32_MAX; + else + parent_node_id = res->parent_node_id; + + shared_shaper_id = (uint32_t *)malloc(MAX_NUM_SHARED_SHAPERS * + sizeof(uint32_t)); + if (shared_shaper_id == NULL) { + printf(" Memory not allocated for shared shapers (error)\n"); + return; + } + + /* Parse multi shared shaper id string */ + ret = parse_multi_ss_id_str(s_str, &n_shared_shapers, shared_shaper_id); + if (ret) { + printf(" Shared shapers params string parse error\n"); + free(shared_shaper_id); + return; + } + + np.shaper_profile_id = res->shaper_profile_id; + np.n_shared_shapers = n_shared_shapers; + if (np.n_shared_shapers) + np.shared_shaper_id = &shared_shaper_id[0]; + else + np.shared_shaper_id = NULL; + + np.nonleaf.n_sp_priorities = res->n_sp_priorities; + np.stats_mask = res->stats_mask; + np.nonleaf.wfq_weight_mode = NULL; + + ret = rte_tm_node_add(port_id, res->node_id, parent_node_id, + res->priority, res->weight, res->level_id, + &np, &error); + if (ret != 0) { + print_err_msg(&error); + free(shared_shaper_id); + return; + } +} + +cmdline_parse_inst_t cmd_add_port_tm_nonleaf_node = { + .f = cmd_add_port_tm_nonleaf_node_parsed, + .data = NULL, + .help_str = "Add port tm nonleaf node", + .tokens = { + (void *)&cmd_add_port_tm_nonleaf_node_add, + (void *)&cmd_add_port_tm_nonleaf_node_port, + (void *)&cmd_add_port_tm_nonleaf_node_tm, + (void *)&cmd_add_port_tm_nonleaf_node_nonleaf, + (void *)&cmd_add_port_tm_nonleaf_node_node, + (void *)&cmd_add_port_tm_nonleaf_node_port_id, + (void *)&cmd_add_port_tm_nonleaf_node_node_id, + (void *)&cmd_add_port_tm_nonleaf_node_parent_node_id, + (void *)&cmd_add_port_tm_nonleaf_node_priority, + (void *)&cmd_add_port_tm_nonleaf_node_weight, + (void *)&cmd_add_port_tm_nonleaf_node_level_id, + (void *)&cmd_add_port_tm_nonleaf_node_shaper_profile_id, + (void *)&cmd_add_port_tm_nonleaf_node_n_sp_priorities, + (void *)&cmd_add_port_tm_nonleaf_node_stats_mask, + (void *)&cmd_add_port_tm_nonleaf_node_multi_shared_shaper_id, + NULL, + }, +}; + +/* *** Add Port TM leaf node *** */ +struct cmd_add_port_tm_leaf_node_result { + cmdline_fixed_string_t add; + cmdline_fixed_string_t port; + cmdline_fixed_string_t tm; + cmdline_fixed_string_t leaf; + cmdline_fixed_string_t node; + uint16_t port_id; + uint32_t node_id; + int32_t parent_node_id; + uint32_t priority; + uint32_t weight; + uint32_t level_id; + uint32_t shaper_profile_id; + uint32_t cman_mode; + uint32_t wred_profile_id; + uint64_t stats_mask; + cmdline_multi_string_t multi_shared_shaper_id; +}; + +cmdline_parse_token_string_t cmd_add_port_tm_leaf_node_add = + TOKEN_STRING_INITIALIZER( + struct cmd_add_port_tm_leaf_node_result, add, "add"); +cmdline_parse_token_string_t cmd_add_port_tm_leaf_node_port = + TOKEN_STRING_INITIALIZER( + struct cmd_add_port_tm_leaf_node_result, port, "port"); +cmdline_parse_token_string_t cmd_add_port_tm_leaf_node_tm = + TOKEN_STRING_INITIALIZER( + struct cmd_add_port_tm_leaf_node_result, tm, "tm"); +cmdline_parse_token_string_t cmd_add_port_tm_leaf_node_nonleaf = + TOKEN_STRING_INITIALIZER( + struct cmd_add_port_tm_leaf_node_result, leaf, "leaf"); +cmdline_parse_token_string_t cmd_add_port_tm_leaf_node_node = + TOKEN_STRING_INITIALIZER( + struct cmd_add_port_tm_leaf_node_result, node, "node"); +cmdline_parse_token_num_t cmd_add_port_tm_leaf_node_port_id = + TOKEN_NUM_INITIALIZER(struct cmd_add_port_tm_leaf_node_result, + port_id, UINT16); +cmdline_parse_token_num_t cmd_add_port_tm_leaf_node_node_id = + TOKEN_NUM_INITIALIZER(struct cmd_add_port_tm_leaf_node_result, + node_id, UINT32); +cmdline_parse_token_num_t cmd_add_port_tm_leaf_node_parent_node_id = + TOKEN_NUM_INITIALIZER(struct cmd_add_port_tm_leaf_node_result, + parent_node_id, INT32); +cmdline_parse_token_num_t cmd_add_port_tm_leaf_node_priority = + TOKEN_NUM_INITIALIZER(struct cmd_add_port_tm_leaf_node_result, + priority, UINT32); +cmdline_parse_token_num_t cmd_add_port_tm_leaf_node_weight = + TOKEN_NUM_INITIALIZER(struct cmd_add_port_tm_leaf_node_result, + weight, UINT32); +cmdline_parse_token_num_t cmd_add_port_tm_leaf_node_level_id = + TOKEN_NUM_INITIALIZER(struct cmd_add_port_tm_leaf_node_result, + level_id, UINT32); +cmdline_parse_token_num_t cmd_add_port_tm_leaf_node_shaper_profile_id = + TOKEN_NUM_INITIALIZER(struct cmd_add_port_tm_leaf_node_result, + shaper_profile_id, UINT32); +cmdline_parse_token_num_t cmd_add_port_tm_leaf_node_cman_mode = + TOKEN_NUM_INITIALIZER(struct cmd_add_port_tm_leaf_node_result, + cman_mode, UINT32); +cmdline_parse_token_num_t cmd_add_port_tm_leaf_node_wred_profile_id = + TOKEN_NUM_INITIALIZER(struct cmd_add_port_tm_leaf_node_result, + wred_profile_id, UINT32); +cmdline_parse_token_num_t cmd_add_port_tm_leaf_node_stats_mask = + TOKEN_NUM_INITIALIZER(struct cmd_add_port_tm_leaf_node_result, + stats_mask, UINT64); +cmdline_parse_token_string_t + cmd_add_port_tm_leaf_node_multi_shared_shaper_id = + TOKEN_STRING_INITIALIZER(struct cmd_add_port_tm_leaf_node_result, + multi_shared_shaper_id, TOKEN_STRING_MULTI); + +static void cmd_add_port_tm_leaf_node_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_add_port_tm_leaf_node_result *res = parsed_result; + struct rte_tm_error error; + struct rte_tm_node_params np; + uint32_t *shared_shaper_id; + uint32_t parent_node_id, n_shared_shapers = 0; + portid_t port_id = res->port_id; + char *s_str = res->multi_shared_shaper_id; + int ret; + + if (port_id_is_invalid(port_id, ENABLED_WARN)) + return; + + memset(&np, 0, sizeof(struct rte_tm_node_params)); + + /* Node parameters */ + if (res->parent_node_id < 0) + parent_node_id = UINT32_MAX; + else + parent_node_id = res->parent_node_id; + + shared_shaper_id = (uint32_t *)malloc(MAX_NUM_SHARED_SHAPERS * + sizeof(uint32_t)); + if (shared_shaper_id == NULL) { + printf(" Memory not allocated for shared shapers (error)\n"); + return; + } + + /* Parse multi shared shaper id string */ + ret = parse_multi_ss_id_str(s_str, &n_shared_shapers, shared_shaper_id); + if (ret) { + printf(" Shared shapers params string parse error\n"); + free(shared_shaper_id); + return; + } + + np.shaper_profile_id = res->shaper_profile_id; + np.n_shared_shapers = n_shared_shapers; + + if (np.n_shared_shapers) + np.shared_shaper_id = &shared_shaper_id[0]; + else + np.shared_shaper_id = NULL; + + np.leaf.cman = res->cman_mode; + np.leaf.wred.wred_profile_id = res->wred_profile_id; + np.stats_mask = res->stats_mask; + + ret = rte_tm_node_add(port_id, res->node_id, parent_node_id, + res->priority, res->weight, res->level_id, + &np, &error); + if (ret != 0) { + print_err_msg(&error); + free(shared_shaper_id); + return; + } +} + +cmdline_parse_inst_t cmd_add_port_tm_leaf_node = { + .f = cmd_add_port_tm_leaf_node_parsed, + .data = NULL, + .help_str = "Add port tm leaf node", + .tokens = { + (void *)&cmd_add_port_tm_leaf_node_add, + (void *)&cmd_add_port_tm_leaf_node_port, + (void *)&cmd_add_port_tm_leaf_node_tm, + (void *)&cmd_add_port_tm_leaf_node_nonleaf, + (void *)&cmd_add_port_tm_leaf_node_node, + (void *)&cmd_add_port_tm_leaf_node_port_id, + (void *)&cmd_add_port_tm_leaf_node_node_id, + (void *)&cmd_add_port_tm_leaf_node_parent_node_id, + (void *)&cmd_add_port_tm_leaf_node_priority, + (void *)&cmd_add_port_tm_leaf_node_weight, + (void *)&cmd_add_port_tm_leaf_node_level_id, + (void *)&cmd_add_port_tm_leaf_node_shaper_profile_id, + (void *)&cmd_add_port_tm_leaf_node_cman_mode, + (void *)&cmd_add_port_tm_leaf_node_wred_profile_id, + (void *)&cmd_add_port_tm_leaf_node_stats_mask, + (void *)&cmd_add_port_tm_leaf_node_multi_shared_shaper_id, + NULL, + }, +}; + +/* *** Delete Port TM Node *** */ +struct cmd_del_port_tm_node_result { + cmdline_fixed_string_t del; + cmdline_fixed_string_t port; + cmdline_fixed_string_t tm; + cmdline_fixed_string_t node; + uint16_t port_id; + uint32_t node_id; +}; + +cmdline_parse_token_string_t cmd_del_port_tm_node_del = + TOKEN_STRING_INITIALIZER( + struct cmd_del_port_tm_node_result, del, "del"); +cmdline_parse_token_string_t cmd_del_port_tm_node_port = + TOKEN_STRING_INITIALIZER( + struct cmd_del_port_tm_node_result, port, "port"); +cmdline_parse_token_string_t cmd_del_port_tm_node_tm = + TOKEN_STRING_INITIALIZER( + struct cmd_del_port_tm_node_result, tm, "tm"); +cmdline_parse_token_string_t cmd_del_port_tm_node_node = + TOKEN_STRING_INITIALIZER( + struct cmd_del_port_tm_node_result, node, "node"); +cmdline_parse_token_num_t cmd_del_port_tm_node_port_id = + TOKEN_NUM_INITIALIZER(struct cmd_del_port_tm_node_result, + port_id, UINT16); +cmdline_parse_token_num_t cmd_del_port_tm_node_node_id = + TOKEN_NUM_INITIALIZER(struct cmd_del_port_tm_node_result, + node_id, UINT32); + +static void cmd_del_port_tm_node_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_del_port_tm_node_result *res = parsed_result; + struct rte_tm_error error; + uint32_t node_id = res->node_id; + portid_t port_id = res->port_id; + int ret; + + if (port_id_is_invalid(port_id, ENABLED_WARN)) + return; + + /* Port status */ + if (port_is_started(port_id)) { + printf(" Port %u not stopped (error)\n", port_id); + return; + } + + ret = rte_tm_node_delete(port_id, node_id, &error); + if (ret != 0) { + print_err_msg(&error); + return; + } +} + +cmdline_parse_inst_t cmd_del_port_tm_node = { + .f = cmd_del_port_tm_node_parsed, + .data = NULL, + .help_str = "Delete port tm node", + .tokens = { + (void *)&cmd_del_port_tm_node_del, + (void *)&cmd_del_port_tm_node_port, + (void *)&cmd_del_port_tm_node_tm, + (void *)&cmd_del_port_tm_node_node, + (void *)&cmd_del_port_tm_node_port_id, + (void *)&cmd_del_port_tm_node_node_id, + NULL, + }, +}; + +/* *** Update Port TM Node Parent *** */ +struct cmd_set_port_tm_node_parent_result { + cmdline_fixed_string_t set; + cmdline_fixed_string_t port; + cmdline_fixed_string_t tm; + cmdline_fixed_string_t node; + cmdline_fixed_string_t parent; + uint16_t port_id; + uint32_t node_id; + uint32_t parent_id; + uint32_t priority; + uint32_t weight; +}; + +cmdline_parse_token_string_t cmd_set_port_tm_node_parent_set = + TOKEN_STRING_INITIALIZER( + struct cmd_set_port_tm_node_parent_result, set, "set"); +cmdline_parse_token_string_t cmd_set_port_tm_node_parent_port = + TOKEN_STRING_INITIALIZER( + struct cmd_set_port_tm_node_parent_result, port, "port"); +cmdline_parse_token_string_t cmd_set_port_tm_node_parent_tm = + TOKEN_STRING_INITIALIZER( + struct cmd_set_port_tm_node_parent_result, tm, "tm"); +cmdline_parse_token_string_t cmd_set_port_tm_node_parent_node = + TOKEN_STRING_INITIALIZER( + struct cmd_set_port_tm_node_parent_result, node, "node"); +cmdline_parse_token_string_t cmd_set_port_tm_node_parent_parent = + TOKEN_STRING_INITIALIZER( + struct cmd_set_port_tm_node_parent_result, parent, "parent"); +cmdline_parse_token_num_t cmd_set_port_tm_node_parent_port_id = + TOKEN_NUM_INITIALIZER( + struct cmd_set_port_tm_node_parent_result, port_id, UINT16); +cmdline_parse_token_num_t cmd_set_port_tm_node_parent_node_id = + TOKEN_NUM_INITIALIZER( + struct cmd_set_port_tm_node_parent_result, node_id, UINT32); +cmdline_parse_token_num_t cmd_set_port_tm_node_parent_parent_id = + TOKEN_NUM_INITIALIZER(struct cmd_set_port_tm_node_parent_result, + parent_id, UINT32); +cmdline_parse_token_num_t cmd_set_port_tm_node_parent_priority = + TOKEN_NUM_INITIALIZER(struct cmd_set_port_tm_node_parent_result, + priority, UINT32); +cmdline_parse_token_num_t cmd_set_port_tm_node_parent_weight = + TOKEN_NUM_INITIALIZER(struct cmd_set_port_tm_node_parent_result, + weight, UINT32); + +static void cmd_set_port_tm_node_parent_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_set_port_tm_node_parent_result *res = parsed_result; + struct rte_tm_error error; + uint32_t node_id = res->node_id; + uint32_t parent_id = res->parent_id; + uint32_t priority = res->priority; + uint32_t weight = res->weight; + portid_t port_id = res->port_id; + int ret; + + if (port_id_is_invalid(port_id, ENABLED_WARN)) + return; + + /* Port status */ + if (!port_is_started(port_id)) { + printf(" Port %u not started (error)\n", port_id); + return; + } + + ret = rte_tm_node_parent_update(port_id, node_id, + parent_id, priority, weight, &error); + if (ret != 0) { + print_err_msg(&error); + return; + } +} + +cmdline_parse_inst_t cmd_set_port_tm_node_parent = { + .f = cmd_set_port_tm_node_parent_parsed, + .data = NULL, + .help_str = "Set port tm node parent", + .tokens = { + (void *)&cmd_set_port_tm_node_parent_set, + (void *)&cmd_set_port_tm_node_parent_port, + (void *)&cmd_set_port_tm_node_parent_tm, + (void *)&cmd_set_port_tm_node_parent_node, + (void *)&cmd_set_port_tm_node_parent_parent, + (void *)&cmd_set_port_tm_node_parent_port_id, + (void *)&cmd_set_port_tm_node_parent_node_id, + (void *)&cmd_set_port_tm_node_parent_parent_id, + (void *)&cmd_set_port_tm_node_parent_priority, + (void *)&cmd_set_port_tm_node_parent_weight, + NULL, + }, +}; + +/* *** Port TM Hierarchy Commit *** */ +struct cmd_port_tm_hierarchy_commit_result { + cmdline_fixed_string_t port; + cmdline_fixed_string_t tm; + cmdline_fixed_string_t hierarchy; + cmdline_fixed_string_t commit; + uint16_t port_id; + cmdline_fixed_string_t clean_on_fail; +}; + +cmdline_parse_token_string_t cmd_port_tm_hierarchy_commit_port = + TOKEN_STRING_INITIALIZER( + struct cmd_port_tm_hierarchy_commit_result, port, "port"); +cmdline_parse_token_string_t cmd_port_tm_hierarchy_commit_tm = + TOKEN_STRING_INITIALIZER( + struct cmd_port_tm_hierarchy_commit_result, tm, "tm"); +cmdline_parse_token_string_t cmd_port_tm_hierarchy_commit_hierarchy = + TOKEN_STRING_INITIALIZER( + struct cmd_port_tm_hierarchy_commit_result, + hierarchy, "hierarchy"); +cmdline_parse_token_string_t cmd_port_tm_hierarchy_commit_commit = + TOKEN_STRING_INITIALIZER( + struct cmd_port_tm_hierarchy_commit_result, commit, "commit"); +cmdline_parse_token_num_t cmd_port_tm_hierarchy_commit_port_id = + TOKEN_NUM_INITIALIZER( + struct cmd_port_tm_hierarchy_commit_result, + port_id, UINT16); +cmdline_parse_token_string_t cmd_port_tm_hierarchy_commit_clean_on_fail = + TOKEN_STRING_INITIALIZER(struct cmd_port_tm_hierarchy_commit_result, + clean_on_fail, "yes#no"); + +static void cmd_port_tm_hierarchy_commit_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_port_tm_hierarchy_commit_result *res = parsed_result; + struct rte_tm_error error; + uint32_t clean_on_fail; + portid_t port_id = res->port_id; + int ret; + + if (port_id_is_invalid(port_id, ENABLED_WARN)) + return; + + if (strcmp(res->clean_on_fail, "yes") == 0) + clean_on_fail = 1; + else + clean_on_fail = 0; + + ret = rte_tm_hierarchy_commit(port_id, clean_on_fail, &error); + if (ret != 0) { + print_err_msg(&error); + return; + } +} + +cmdline_parse_inst_t cmd_port_tm_hierarchy_commit = { + .f = cmd_port_tm_hierarchy_commit_parsed, + .data = NULL, + .help_str = "Set port tm node shaper profile", + .tokens = { + (void *)&cmd_port_tm_hierarchy_commit_port, + (void *)&cmd_port_tm_hierarchy_commit_tm, + (void *)&cmd_port_tm_hierarchy_commit_hierarchy, + (void *)&cmd_port_tm_hierarchy_commit_commit, + (void *)&cmd_port_tm_hierarchy_commit_port_id, + (void *)&cmd_port_tm_hierarchy_commit_clean_on_fail, + NULL, + }, +}; diff --git a/dpdk/app/test-pmd/cmdline_tm.h b/dpdk/app/test-pmd/cmdline_tm.h new file mode 100644 index 00000000..9d5fdf0a --- /dev/null +++ b/dpdk/app/test-pmd/cmdline_tm.h @@ -0,0 +1,56 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _CMDLINE_TM_H_ +#define _CMDLINE_TM_H_ + + /* Traffic Management CLI */ +extern cmdline_parse_inst_t cmd_show_port_tm_cap; +extern cmdline_parse_inst_t cmd_show_port_tm_level_cap; +extern cmdline_parse_inst_t cmd_show_port_tm_node_cap; +extern cmdline_parse_inst_t cmd_show_port_tm_node_type; +extern cmdline_parse_inst_t cmd_show_port_tm_node_stats; +extern cmdline_parse_inst_t cmd_add_port_tm_node_shaper_profile; +extern cmdline_parse_inst_t cmd_del_port_tm_node_shaper_profile; +extern cmdline_parse_inst_t cmd_add_port_tm_node_shared_shaper; +extern cmdline_parse_inst_t cmd_del_port_tm_node_shared_shaper; +extern cmdline_parse_inst_t cmd_add_port_tm_node_wred_profile; +extern cmdline_parse_inst_t cmd_del_port_tm_node_wred_profile; +extern cmdline_parse_inst_t cmd_set_port_tm_node_shaper_profile; +extern cmdline_parse_inst_t cmd_add_port_tm_nonleaf_node; +extern cmdline_parse_inst_t cmd_add_port_tm_leaf_node; +extern cmdline_parse_inst_t cmd_del_port_tm_node; +extern cmdline_parse_inst_t cmd_set_port_tm_node_parent; +extern cmdline_parse_inst_t cmd_port_tm_hierarchy_commit; + +#endif /* _CMDLINE_TM_H_ */ diff --git a/dpdk/app/test-pmd/config.c b/dpdk/app/test-pmd/config.c index 1457e4ad..a0f3c246 100644 --- a/dpdk/app/test-pmd/config.c +++ b/dpdk/app/test-pmd/config.c @@ -2,6 +2,7 @@ * BSD LICENSE * * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. + * Copyright 2013-2014 6WIND S.A. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -30,46 +31,19 @@ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -/* BSD LICENSE - * - * Copyright 2013-2014 6WIND S.A. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of 6WIND S.A. nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ #include #include #include #include -#include #include #include #include +#include +#include +#include +#include #include #include @@ -84,7 +58,6 @@ #include #include #include -#include #include #include #include @@ -93,6 +66,18 @@ #include #include #include +#include +#include +#ifdef RTE_LIBRTE_IXGBE_PMD +#include +#endif +#ifdef RTE_LIBRTE_I40E_PMD +#include +#endif +#ifdef RTE_LIBRTE_BNXT_PMD +#include +#endif +#include #include "testpmd.h" @@ -170,7 +155,7 @@ nic_stats_display(portid_t port_id) if (port_id_is_invalid(port_id, ENABLED_WARN)) { printf("Valid port range is [0"); - FOREACH_PORT(pid, ports) + RTE_ETH_FOREACH_DEV(pid) printf(", %d", pid); printf("]\n"); return; @@ -225,8 +210,10 @@ nic_stats_display(portid_t port_id) if (diff_cycles > 0) diff_cycles = prev_cycles[port_id] - diff_cycles; - diff_pkts_rx = stats.ipackets - prev_pkts_rx[port_id]; - diff_pkts_tx = stats.opackets - prev_pkts_tx[port_id]; + diff_pkts_rx = (stats.ipackets > prev_pkts_rx[port_id]) ? + (stats.ipackets - prev_pkts_rx[port_id]) : 0; + diff_pkts_tx = (stats.opackets > prev_pkts_tx[port_id]) ? + (stats.opackets - prev_pkts_tx[port_id]) : 0; prev_pkts_rx[port_id] = stats.ipackets; prev_pkts_tx[port_id] = stats.opackets; mpps_rx = diff_cycles > 0 ? @@ -248,7 +235,7 @@ nic_stats_clear(portid_t port_id) if (port_id_is_invalid(port_id, ENABLED_WARN)) { printf("Valid port range is [0"); - FOREACH_PORT(pid, ports) + RTE_ETH_FOREACH_DEV(pid) printf(", %d", pid); printf("]\n"); return; @@ -305,10 +292,13 @@ nic_xstats_display(portid_t port_id) } /* Display xstats */ - for (idx_xstat = 0; idx_xstat < cnt_xstats; idx_xstat++) + for (idx_xstat = 0; idx_xstat < cnt_xstats; idx_xstat++) { + if (xstats_hide_zero && !xstats[idx_xstat].value) + continue; printf("%s: %"PRIu64"\n", xstats_names[idx_xstat].name, xstats[idx_xstat].value); + } free(xstats_names); free(xstats); } @@ -330,7 +320,7 @@ nic_stats_mapping_display(portid_t port_id) if (port_id_is_invalid(port_id, ENABLED_WARN)) { printf("Valid port range is [0"); - FOREACH_PORT(pid, ports) + RTE_ETH_FOREACH_DEV(pid) printf(", %d", pid); printf("]\n"); return; @@ -380,7 +370,7 @@ rx_queue_infos_display(portid_t port_id, uint16_t queue_id) rc = rte_eth_rx_queue_info_get(port_id, queue_id, &qinfo); if (rc != 0) { - printf("Failed to retrieve information for port: %hhu, " + printf("Failed to retrieve information for port: %u, " "RX queue: %hu\nerror desc: %s(%d)\n", port_id, queue_id, strerror(-rc), rc); return; @@ -413,7 +403,7 @@ tx_queue_infos_display(portid_t port_id, uint16_t queue_id) rc = rte_eth_tx_queue_info_get(port_id, queue_id, &qinfo); if (rc != 0) { - printf("Failed to retrieve information for port: %hhu, " + printf("Failed to retrieve information for port: %u, " "TX queue: %hu\nerror desc: %s(%d)\n", port_id, queue_id, strerror(-rc), rc); return; @@ -445,20 +435,24 @@ port_infos_display(portid_t port_id) struct rte_mempool * mp; static const char *info_border = "*********************"; portid_t pid; + uint16_t mtu; if (port_id_is_invalid(port_id, ENABLED_WARN)) { printf("Valid port range is [0"); - FOREACH_PORT(pid, ports) + RTE_ETH_FOREACH_DEV(pid) printf(", %d", pid); printf("]\n"); return; } port = &ports[port_id]; rte_eth_link_get_nowait(port_id, &link); + memset(&dev_info, 0, sizeof(dev_info)); + rte_eth_dev_info_get(port_id, &dev_info); printf("\n%s Infos for port %-2d %s\n", info_border, port_id, info_border); rte_eth_macaddr_get(port_id, &mac_addr); print_ethaddr("MAC address: ", &mac_addr); + printf("\nDriver name: %s", dev_info.driver_name); printf("\nConnect to socket: %u", port->socket_id); if (port_numa[port_id] != NUMA_NO_CONFIG) { @@ -473,6 +467,10 @@ port_infos_display(portid_t port_id) printf("Link speed: %u Mbps\n", (unsigned) link.link_speed); printf("Link duplex: %s\n", (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? ("full-duplex") : ("half-duplex")); + + if (!rte_eth_dev_get_mtu(port_id, &mtu)) + printf("MTU: %u\n", mtu); + printf("Promiscuous mode: %s\n", rte_eth_promiscuous_get(port_id) ? "enabled" : "disabled"); printf("Allmulticast mode: %s\n", @@ -501,8 +499,6 @@ port_infos_display(portid_t port_id) printf(" qinq(extend) off \n"); } - memset(&dev_info, 0, sizeof(dev_info)); - rte_eth_dev_info_get(port_id, &dev_info); if (dev_info.hash_key_size > 0) printf("Hash key size in bytes: %u\n", dev_info.hash_key_size); if (dev_info.reta_size > 0) @@ -514,12 +510,15 @@ port_infos_display(portid_t port_id) char *p; printf("Supported flow types:\n"); - for (i = RTE_ETH_FLOW_UNKNOWN + 1; i < RTE_ETH_FLOW_MAX; - i++) { + for (i = RTE_ETH_FLOW_UNKNOWN + 1; + i < sizeof(dev_info.flow_type_rss_offloads) * CHAR_BIT; i++) { if (!(dev_info.flow_type_rss_offloads & (1ULL << i))) continue; p = flowtype_to_str(i); - printf(" %s\n", (p ? p : "unknown")); + if (p) + printf(" %s\n", p); + else + printf(" user defined %d\n", i); } } @@ -538,14 +537,201 @@ port_infos_display(portid_t port_id) printf("TXDs number alignment: %hu\n", dev_info.tx_desc_lim.nb_align); } +void +port_offload_cap_display(portid_t port_id) +{ + struct rte_eth_dev *dev; + struct rte_eth_dev_info dev_info; + static const char *info_border = "************"; + + if (port_id_is_invalid(port_id, ENABLED_WARN)) + return; + + dev = &rte_eth_devices[port_id]; + rte_eth_dev_info_get(port_id, &dev_info); + + printf("\n%s Port %d supported offload features: %s\n", + info_border, port_id, info_border); + + if (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_VLAN_STRIP) { + printf("VLAN stripped: "); + if (dev->data->dev_conf.rxmode.hw_vlan_strip) + printf("on\n"); + else + printf("off\n"); + } + + if (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_QINQ_STRIP) { + printf("Double VLANs stripped: "); + if (dev->data->dev_conf.rxmode.hw_vlan_extend) + printf("on\n"); + else + printf("off\n"); + } + + if (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_IPV4_CKSUM) { + printf("RX IPv4 checksum: "); + if (dev->data->dev_conf.rxmode.hw_ip_checksum) + printf("on\n"); + else + printf("off\n"); + } + + if (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_UDP_CKSUM) { + printf("RX UDP checksum: "); + if (dev->data->dev_conf.rxmode.hw_ip_checksum) + printf("on\n"); + else + printf("off\n"); + } + + if (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_TCP_CKSUM) { + printf("RX TCP checksum: "); + if (dev->data->dev_conf.rxmode.hw_ip_checksum) + printf("on\n"); + else + printf("off\n"); + } + + if (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM) + printf("RX Outer IPv4 checksum: on"); + + if (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_TCP_LRO) { + printf("Large receive offload: "); + if (dev->data->dev_conf.rxmode.enable_lro) + printf("on\n"); + else + printf("off\n"); + } + + if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_VLAN_INSERT) { + printf("VLAN insert: "); + if (ports[port_id].tx_ol_flags & + TESTPMD_TX_OFFLOAD_INSERT_VLAN) + printf("on\n"); + else + printf("off\n"); + } + + if (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_TIMESTAMP) { + printf("HW timestamp: "); + if (dev->data->dev_conf.rxmode.hw_timestamp) + printf("on\n"); + else + printf("off\n"); + } + + if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_QINQ_INSERT) { + printf("Double VLANs insert: "); + if (ports[port_id].tx_ol_flags & + TESTPMD_TX_OFFLOAD_INSERT_QINQ) + printf("on\n"); + else + printf("off\n"); + } + + if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_IPV4_CKSUM) { + printf("TX IPv4 checksum: "); + if (ports[port_id].tx_ol_flags & TESTPMD_TX_OFFLOAD_IP_CKSUM) + printf("on\n"); + else + printf("off\n"); + } + + if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_UDP_CKSUM) { + printf("TX UDP checksum: "); + if (ports[port_id].tx_ol_flags & TESTPMD_TX_OFFLOAD_UDP_CKSUM) + printf("on\n"); + else + printf("off\n"); + } + + if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_TCP_CKSUM) { + printf("TX TCP checksum: "); + if (ports[port_id].tx_ol_flags & TESTPMD_TX_OFFLOAD_TCP_CKSUM) + printf("on\n"); + else + printf("off\n"); + } + + if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_SCTP_CKSUM) { + printf("TX SCTP checksum: "); + if (ports[port_id].tx_ol_flags & TESTPMD_TX_OFFLOAD_SCTP_CKSUM) + printf("on\n"); + else + printf("off\n"); + } + + if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM) { + printf("TX Outer IPv4 checksum: "); + if (ports[port_id].tx_ol_flags & + TESTPMD_TX_OFFLOAD_OUTER_IP_CKSUM) + printf("on\n"); + else + printf("off\n"); + } + + if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_TCP_TSO) { + printf("TX TCP segmentation: "); + if (ports[port_id].tso_segsz != 0) + printf("on\n"); + else + printf("off\n"); + } + + if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_UDP_TSO) { + printf("TX UDP segmentation: "); + if (ports[port_id].tso_segsz != 0) + printf("on\n"); + else + printf("off\n"); + } + + if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_VXLAN_TNL_TSO) { + printf("TSO for VXLAN tunnel packet: "); + if (ports[port_id].tunnel_tso_segsz) + printf("on\n"); + else + printf("off\n"); + } + + if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_GRE_TNL_TSO) { + printf("TSO for GRE tunnel packet: "); + if (ports[port_id].tunnel_tso_segsz) + printf("on\n"); + else + printf("off\n"); + } + + if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_IPIP_TNL_TSO) { + printf("TSO for IPIP tunnel packet: "); + if (ports[port_id].tunnel_tso_segsz) + printf("on\n"); + else + printf("off\n"); + } + + if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_GENEVE_TNL_TSO) { + printf("TSO for GENEVE tunnel packet: "); + if (ports[port_id].tunnel_tso_segsz) + printf("on\n"); + else + printf("off\n"); + } + +} + int port_id_is_invalid(portid_t port_id, enum print_warning warning) { + uint16_t pid; + if (port_id == (portid_t)RTE_PORT_ALL) return 0; - if (port_id < RTE_MAX_ETHPORTS && ports[port_id].enabled) - return 0; + RTE_ETH_FOREACH_DEV(pid) + if (port_id == pid) + return 0; if (warning == ENABLED_WARN) printf("Invalid port %d\n", port_id); @@ -751,6 +937,530 @@ port_mtu_set(portid_t port_id, uint16_t mtu) printf("Set MTU failed. diag=%d\n", diag); } +/* Generic flow management functions. */ + +/** Generate flow_item[] entry. */ +#define MK_FLOW_ITEM(t, s) \ + [RTE_FLOW_ITEM_TYPE_ ## t] = { \ + .name = # t, \ + .size = s, \ + } + +/** Information about known flow pattern items. */ +static const struct { + const char *name; + size_t size; +} flow_item[] = { + MK_FLOW_ITEM(END, 0), + MK_FLOW_ITEM(VOID, 0), + MK_FLOW_ITEM(INVERT, 0), + MK_FLOW_ITEM(ANY, sizeof(struct rte_flow_item_any)), + MK_FLOW_ITEM(PF, 0), + MK_FLOW_ITEM(VF, sizeof(struct rte_flow_item_vf)), + MK_FLOW_ITEM(PORT, sizeof(struct rte_flow_item_port)), + MK_FLOW_ITEM(RAW, sizeof(struct rte_flow_item_raw)), /* +pattern[] */ + MK_FLOW_ITEM(ETH, sizeof(struct rte_flow_item_eth)), + MK_FLOW_ITEM(VLAN, sizeof(struct rte_flow_item_vlan)), + MK_FLOW_ITEM(IPV4, sizeof(struct rte_flow_item_ipv4)), + MK_FLOW_ITEM(IPV6, sizeof(struct rte_flow_item_ipv6)), + MK_FLOW_ITEM(ICMP, sizeof(struct rte_flow_item_icmp)), + MK_FLOW_ITEM(UDP, sizeof(struct rte_flow_item_udp)), + MK_FLOW_ITEM(TCP, sizeof(struct rte_flow_item_tcp)), + MK_FLOW_ITEM(SCTP, sizeof(struct rte_flow_item_sctp)), + MK_FLOW_ITEM(VXLAN, sizeof(struct rte_flow_item_vxlan)), + MK_FLOW_ITEM(E_TAG, sizeof(struct rte_flow_item_e_tag)), + MK_FLOW_ITEM(NVGRE, sizeof(struct rte_flow_item_nvgre)), + MK_FLOW_ITEM(MPLS, sizeof(struct rte_flow_item_mpls)), + MK_FLOW_ITEM(GRE, sizeof(struct rte_flow_item_gre)), + MK_FLOW_ITEM(FUZZY, sizeof(struct rte_flow_item_fuzzy)), + MK_FLOW_ITEM(GTP, sizeof(struct rte_flow_item_gtp)), + MK_FLOW_ITEM(GTPC, sizeof(struct rte_flow_item_gtp)), + MK_FLOW_ITEM(GTPU, sizeof(struct rte_flow_item_gtp)), +}; + +/** Compute storage space needed by item specification. */ +static void +flow_item_spec_size(const struct rte_flow_item *item, + size_t *size, size_t *pad) +{ + if (!item->spec) { + *size = 0; + goto empty; + } + switch (item->type) { + union { + const struct rte_flow_item_raw *raw; + } spec; + + case RTE_FLOW_ITEM_TYPE_RAW: + spec.raw = item->spec; + *size = offsetof(struct rte_flow_item_raw, pattern) + + spec.raw->length * sizeof(*spec.raw->pattern); + break; + default: + *size = flow_item[item->type].size; + break; + } +empty: + *pad = RTE_ALIGN_CEIL(*size, sizeof(double)) - *size; +} + +/** Generate flow_action[] entry. */ +#define MK_FLOW_ACTION(t, s) \ + [RTE_FLOW_ACTION_TYPE_ ## t] = { \ + .name = # t, \ + .size = s, \ + } + +/** Information about known flow actions. */ +static const struct { + const char *name; + size_t size; +} flow_action[] = { + MK_FLOW_ACTION(END, 0), + MK_FLOW_ACTION(VOID, 0), + MK_FLOW_ACTION(PASSTHRU, 0), + MK_FLOW_ACTION(MARK, sizeof(struct rte_flow_action_mark)), + MK_FLOW_ACTION(FLAG, 0), + MK_FLOW_ACTION(QUEUE, sizeof(struct rte_flow_action_queue)), + MK_FLOW_ACTION(DROP, 0), + MK_FLOW_ACTION(COUNT, 0), + MK_FLOW_ACTION(DUP, sizeof(struct rte_flow_action_dup)), + MK_FLOW_ACTION(RSS, sizeof(struct rte_flow_action_rss)), /* +queue[] */ + MK_FLOW_ACTION(PF, 0), + MK_FLOW_ACTION(VF, sizeof(struct rte_flow_action_vf)), +}; + +/** Compute storage space needed by action configuration. */ +static void +flow_action_conf_size(const struct rte_flow_action *action, + size_t *size, size_t *pad) +{ + if (!action->conf) { + *size = 0; + goto empty; + } + switch (action->type) { + union { + const struct rte_flow_action_rss *rss; + } conf; + + case RTE_FLOW_ACTION_TYPE_RSS: + conf.rss = action->conf; + *size = offsetof(struct rte_flow_action_rss, queue) + + conf.rss->num * sizeof(*conf.rss->queue); + break; + default: + *size = flow_action[action->type].size; + break; + } +empty: + *pad = RTE_ALIGN_CEIL(*size, sizeof(double)) - *size; +} + +/** Generate a port_flow entry from attributes/pattern/actions. */ +static struct port_flow * +port_flow_new(const struct rte_flow_attr *attr, + const struct rte_flow_item *pattern, + const struct rte_flow_action *actions) +{ + const struct rte_flow_item *item; + const struct rte_flow_action *action; + struct port_flow *pf = NULL; + size_t tmp; + size_t pad; + size_t off1 = 0; + size_t off2 = 0; + int err = ENOTSUP; + +store: + item = pattern; + if (pf) + pf->pattern = (void *)&pf->data[off1]; + do { + struct rte_flow_item *dst = NULL; + + if ((unsigned int)item->type >= RTE_DIM(flow_item) || + !flow_item[item->type].name) + goto notsup; + if (pf) + dst = memcpy(pf->data + off1, item, sizeof(*item)); + off1 += sizeof(*item); + flow_item_spec_size(item, &tmp, &pad); + if (item->spec) { + if (pf) + dst->spec = memcpy(pf->data + off2, + item->spec, tmp); + off2 += tmp + pad; + } + if (item->last) { + if (pf) + dst->last = memcpy(pf->data + off2, + item->last, tmp); + off2 += tmp + pad; + } + if (item->mask) { + if (pf) + dst->mask = memcpy(pf->data + off2, + item->mask, tmp); + off2 += tmp + pad; + } + off2 = RTE_ALIGN_CEIL(off2, sizeof(double)); + } while ((item++)->type != RTE_FLOW_ITEM_TYPE_END); + off1 = RTE_ALIGN_CEIL(off1, sizeof(double)); + action = actions; + if (pf) + pf->actions = (void *)&pf->data[off1]; + do { + struct rte_flow_action *dst = NULL; + + if ((unsigned int)action->type >= RTE_DIM(flow_action) || + !flow_action[action->type].name) + goto notsup; + if (pf) + dst = memcpy(pf->data + off1, action, sizeof(*action)); + off1 += sizeof(*action); + flow_action_conf_size(action, &tmp, &pad); + if (action->conf) { + if (pf) + dst->conf = memcpy(pf->data + off2, + action->conf, tmp); + off2 += tmp + pad; + } + off2 = RTE_ALIGN_CEIL(off2, sizeof(double)); + } while ((action++)->type != RTE_FLOW_ACTION_TYPE_END); + if (pf != NULL) + return pf; + off1 = RTE_ALIGN_CEIL(off1, sizeof(double)); + tmp = RTE_ALIGN_CEIL(offsetof(struct port_flow, data), sizeof(double)); + pf = calloc(1, tmp + off1 + off2); + if (pf == NULL) + err = errno; + else { + *pf = (const struct port_flow){ + .size = tmp + off1 + off2, + .attr = *attr, + }; + tmp -= offsetof(struct port_flow, data); + off2 = tmp + off1; + off1 = tmp; + goto store; + } +notsup: + rte_errno = err; + return NULL; +} + +/** Print a message out of a flow error. */ +static int +port_flow_complain(struct rte_flow_error *error) +{ + static const char *const errstrlist[] = { + [RTE_FLOW_ERROR_TYPE_NONE] = "no error", + [RTE_FLOW_ERROR_TYPE_UNSPECIFIED] = "cause unspecified", + [RTE_FLOW_ERROR_TYPE_HANDLE] = "flow rule (handle)", + [RTE_FLOW_ERROR_TYPE_ATTR_GROUP] = "group field", + [RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY] = "priority field", + [RTE_FLOW_ERROR_TYPE_ATTR_INGRESS] = "ingress field", + [RTE_FLOW_ERROR_TYPE_ATTR_EGRESS] = "egress field", + [RTE_FLOW_ERROR_TYPE_ATTR] = "attributes structure", + [RTE_FLOW_ERROR_TYPE_ITEM_NUM] = "pattern length", + [RTE_FLOW_ERROR_TYPE_ITEM] = "specific pattern item", + [RTE_FLOW_ERROR_TYPE_ACTION_NUM] = "number of actions", + [RTE_FLOW_ERROR_TYPE_ACTION] = "specific action", + }; + const char *errstr; + char buf[32]; + int err = rte_errno; + + if ((unsigned int)error->type >= RTE_DIM(errstrlist) || + !errstrlist[error->type]) + errstr = "unknown type"; + else + errstr = errstrlist[error->type]; + printf("Caught error type %d (%s): %s%s\n", + error->type, errstr, + error->cause ? (snprintf(buf, sizeof(buf), "cause: %p, ", + error->cause), buf) : "", + error->message ? error->message : "(no stated reason)"); + return -err; +} + +/** Validate flow rule. */ +int +port_flow_validate(portid_t port_id, + const struct rte_flow_attr *attr, + const struct rte_flow_item *pattern, + const struct rte_flow_action *actions) +{ + struct rte_flow_error error; + + /* Poisoning to make sure PMDs update it in case of error. */ + memset(&error, 0x11, sizeof(error)); + if (rte_flow_validate(port_id, attr, pattern, actions, &error)) + return port_flow_complain(&error); + printf("Flow rule validated\n"); + return 0; +} + +/** Create flow rule. */ +int +port_flow_create(portid_t port_id, + const struct rte_flow_attr *attr, + const struct rte_flow_item *pattern, + const struct rte_flow_action *actions) +{ + struct rte_flow *flow; + struct rte_port *port; + struct port_flow *pf; + uint32_t id; + struct rte_flow_error error; + + /* Poisoning to make sure PMDs update it in case of error. */ + memset(&error, 0x22, sizeof(error)); + flow = rte_flow_create(port_id, attr, pattern, actions, &error); + if (!flow) + return port_flow_complain(&error); + port = &ports[port_id]; + if (port->flow_list) { + if (port->flow_list->id == UINT32_MAX) { + printf("Highest rule ID is already assigned, delete" + " it first"); + rte_flow_destroy(port_id, flow, NULL); + return -ENOMEM; + } + id = port->flow_list->id + 1; + } else + id = 0; + pf = port_flow_new(attr, pattern, actions); + if (!pf) { + int err = rte_errno; + + printf("Cannot allocate flow: %s\n", rte_strerror(err)); + rte_flow_destroy(port_id, flow, NULL); + return -err; + } + pf->next = port->flow_list; + pf->id = id; + pf->flow = flow; + port->flow_list = pf; + printf("Flow rule #%u created\n", pf->id); + return 0; +} + +/** Destroy a number of flow rules. */ +int +port_flow_destroy(portid_t port_id, uint32_t n, const uint32_t *rule) +{ + struct rte_port *port; + struct port_flow **tmp; + uint32_t c = 0; + int ret = 0; + + if (port_id_is_invalid(port_id, ENABLED_WARN) || + port_id == (portid_t)RTE_PORT_ALL) + return -EINVAL; + port = &ports[port_id]; + tmp = &port->flow_list; + while (*tmp) { + uint32_t i; + + for (i = 0; i != n; ++i) { + struct rte_flow_error error; + struct port_flow *pf = *tmp; + + if (rule[i] != pf->id) + continue; + /* + * Poisoning to make sure PMDs update it in case + * of error. + */ + memset(&error, 0x33, sizeof(error)); + if (rte_flow_destroy(port_id, pf->flow, &error)) { + ret = port_flow_complain(&error); + continue; + } + printf("Flow rule #%u destroyed\n", pf->id); + *tmp = pf->next; + free(pf); + break; + } + if (i == n) + tmp = &(*tmp)->next; + ++c; + } + return ret; +} + +/** Remove all flow rules. */ +int +port_flow_flush(portid_t port_id) +{ + struct rte_flow_error error; + struct rte_port *port; + int ret = 0; + + /* Poisoning to make sure PMDs update it in case of error. */ + memset(&error, 0x44, sizeof(error)); + if (rte_flow_flush(port_id, &error)) { + ret = port_flow_complain(&error); + if (port_id_is_invalid(port_id, DISABLED_WARN) || + port_id == (portid_t)RTE_PORT_ALL) + return ret; + } + port = &ports[port_id]; + while (port->flow_list) { + struct port_flow *pf = port->flow_list->next; + + free(port->flow_list); + port->flow_list = pf; + } + return ret; +} + +/** Query a flow rule. */ +int +port_flow_query(portid_t port_id, uint32_t rule, + enum rte_flow_action_type action) +{ + struct rte_flow_error error; + struct rte_port *port; + struct port_flow *pf; + const char *name; + union { + struct rte_flow_query_count count; + } query; + + if (port_id_is_invalid(port_id, ENABLED_WARN) || + port_id == (portid_t)RTE_PORT_ALL) + return -EINVAL; + port = &ports[port_id]; + for (pf = port->flow_list; pf; pf = pf->next) + if (pf->id == rule) + break; + if (!pf) { + printf("Flow rule #%u not found\n", rule); + return -ENOENT; + } + if ((unsigned int)action >= RTE_DIM(flow_action) || + !flow_action[action].name) + name = "unknown"; + else + name = flow_action[action].name; + switch (action) { + case RTE_FLOW_ACTION_TYPE_COUNT: + break; + default: + printf("Cannot query action type %d (%s)\n", action, name); + return -ENOTSUP; + } + /* Poisoning to make sure PMDs update it in case of error. */ + memset(&error, 0x55, sizeof(error)); + memset(&query, 0, sizeof(query)); + if (rte_flow_query(port_id, pf->flow, action, &query, &error)) + return port_flow_complain(&error); + switch (action) { + case RTE_FLOW_ACTION_TYPE_COUNT: + printf("%s:\n" + " hits_set: %u\n" + " bytes_set: %u\n" + " hits: %" PRIu64 "\n" + " bytes: %" PRIu64 "\n", + name, + query.count.hits_set, + query.count.bytes_set, + query.count.hits, + query.count.bytes); + break; + default: + printf("Cannot display result for action type %d (%s)\n", + action, name); + break; + } + return 0; +} + +/** List flow rules. */ +void +port_flow_list(portid_t port_id, uint32_t n, const uint32_t group[n]) +{ + struct rte_port *port; + struct port_flow *pf; + struct port_flow *list = NULL; + uint32_t i; + + if (port_id_is_invalid(port_id, ENABLED_WARN) || + port_id == (portid_t)RTE_PORT_ALL) + return; + port = &ports[port_id]; + if (!port->flow_list) + return; + /* Sort flows by group, priority and ID. */ + for (pf = port->flow_list; pf != NULL; pf = pf->next) { + struct port_flow **tmp; + + if (n) { + /* Filter out unwanted groups. */ + for (i = 0; i != n; ++i) + if (pf->attr.group == group[i]) + break; + if (i == n) + continue; + } + tmp = &list; + while (*tmp && + (pf->attr.group > (*tmp)->attr.group || + (pf->attr.group == (*tmp)->attr.group && + pf->attr.priority > (*tmp)->attr.priority) || + (pf->attr.group == (*tmp)->attr.group && + pf->attr.priority == (*tmp)->attr.priority && + pf->id > (*tmp)->id))) + tmp = &(*tmp)->tmp; + pf->tmp = *tmp; + *tmp = pf; + } + printf("ID\tGroup\tPrio\tAttr\tRule\n"); + for (pf = list; pf != NULL; pf = pf->tmp) { + const struct rte_flow_item *item = pf->pattern; + const struct rte_flow_action *action = pf->actions; + + printf("%" PRIu32 "\t%" PRIu32 "\t%" PRIu32 "\t%c%c\t", + pf->id, + pf->attr.group, + pf->attr.priority, + pf->attr.ingress ? 'i' : '-', + pf->attr.egress ? 'e' : '-'); + while (item->type != RTE_FLOW_ITEM_TYPE_END) { + if (item->type != RTE_FLOW_ITEM_TYPE_VOID) + printf("%s ", flow_item[item->type].name); + ++item; + } + printf("=>"); + while (action->type != RTE_FLOW_ACTION_TYPE_END) { + if (action->type != RTE_FLOW_ACTION_TYPE_VOID) + printf(" %s", flow_action[action->type].name); + ++action; + } + printf("\n"); + } +} + +/** Restrict ingress traffic to the defined flow rules. */ +int +port_flow_isolate(portid_t port_id, int set) +{ + struct rte_flow_error error; + + /* Poisoning to make sure PMDs update it in case of error. */ + memset(&error, 0x66, sizeof(error)); + if (rte_flow_isolate(port_id, set, &error)) + return port_flow_complain(&error); + printf("Ingress traffic on port %u is %s to the defined flow rules\n", + port_id, + set ? "now restricted" : "not restricted anymore"); + return 0; +} + /* * RX/TX ring descriptors display functions. */ @@ -793,7 +1503,7 @@ tx_desc_id_is_invalid(uint16_t txdesc_id) } static const struct rte_memzone * -ring_dma_zone_lookup(const char *ring_name, uint8_t port_id, uint16_t q_id) +ring_dma_zone_lookup(const char *ring_name, portid_t port_id, uint16_t q_id) { char mz_name[RTE_MEMZONE_NAMESIZE]; const struct rte_memzone *mz; @@ -843,9 +1553,9 @@ ring_rxd_display_dword(union igb_ring_dword dword) static void ring_rx_descriptor_display(const struct rte_memzone *ring_mz, #ifndef RTE_LIBRTE_I40E_16BYTE_RX_DESC - uint8_t port_id, + portid_t port_id, #else - __rte_unused uint8_t port_id, + __rte_unused portid_t port_id, #endif uint16_t desc_id) { @@ -948,33 +1658,45 @@ fwd_lcores_config_display(void) void rxtx_config_display(void) { - printf(" %s packet forwarding%s - CRC stripping %s - " - "packets/burst=%d\n", cur_fwd_eng->fwd_mode_name, + portid_t pid; + + printf(" %s packet forwarding%s packets/burst=%d\n", + cur_fwd_eng->fwd_mode_name, retry_enabled == 0 ? "" : " with retry", - rx_mode.hw_strip_crc ? "enabled" : "disabled", nb_pkt_per_burst); - if (cur_fwd_eng == &tx_only_engine) + if (cur_fwd_eng == &tx_only_engine || cur_fwd_eng == &flow_gen_engine) printf(" packet len=%u - nb packet segments=%d\n", (unsigned)tx_pkt_length, (int) tx_pkt_nb_segs); - struct rte_eth_rxconf *rx_conf = &ports[0].rx_conf; - struct rte_eth_txconf *tx_conf = &ports[0].tx_conf; - printf(" nb forwarding cores=%d - nb forwarding ports=%d\n", nb_fwd_lcores, nb_fwd_ports); - printf(" RX queues=%d - RX desc=%d - RX free threshold=%d\n", - nb_rxq, nb_rxd, rx_conf->rx_free_thresh); - printf(" RX threshold registers: pthresh=%d hthresh=%d wthresh=%d\n", - rx_conf->rx_thresh.pthresh, rx_conf->rx_thresh.hthresh, - rx_conf->rx_thresh.wthresh); - printf(" TX queues=%d - TX desc=%d - TX free threshold=%d\n", - nb_txq, nb_txd, tx_conf->tx_free_thresh); - printf(" TX threshold registers: pthresh=%d hthresh=%d wthresh=%d\n", - tx_conf->tx_thresh.pthresh, tx_conf->tx_thresh.hthresh, - tx_conf->tx_thresh.wthresh); - printf(" TX RS bit threshold=%d - TXQ flags=0x%"PRIx32"\n", - tx_conf->tx_rs_thresh, tx_conf->txq_flags); + + RTE_ETH_FOREACH_DEV(pid) { + struct rte_eth_rxconf *rx_conf = &ports[pid].rx_conf; + struct rte_eth_txconf *tx_conf = &ports[pid].tx_conf; + + printf(" port %d:\n", (unsigned int)pid); + printf(" CRC stripping %s\n", + ports[pid].dev_conf.rxmode.hw_strip_crc ? + "enabled" : "disabled"); + printf(" RX queues=%d - RX desc=%d - RX free threshold=%d\n", + nb_rxq, nb_rxd, rx_conf->rx_free_thresh); + printf(" RX threshold registers: pthresh=%d hthresh=%d " + " wthresh=%d\n", + rx_conf->rx_thresh.pthresh, + rx_conf->rx_thresh.hthresh, + rx_conf->rx_thresh.wthresh); + printf(" TX queues=%d - TX desc=%d - TX free threshold=%d\n", + nb_txq, nb_txd, tx_conf->tx_free_thresh); + printf(" TX threshold registers: pthresh=%d hthresh=%d " + " wthresh=%d\n", + tx_conf->tx_thresh.pthresh, + tx_conf->tx_thresh.hthresh, + tx_conf->tx_thresh.wthresh); + printf(" TX RS bit threshold=%d - TXQ flags=0x%"PRIx32"\n", + tx_conf->tx_rs_thresh, tx_conf->txq_flags); + } } void @@ -1154,6 +1876,36 @@ setup_fwd_config_of_each_lcore(struct fwd_config *cfg) } } +static portid_t +fwd_topology_tx_port_get(portid_t rxp) +{ + static int warning_once = 1; + + RTE_ASSERT(rxp < cur_fwd_config.nb_fwd_ports); + + switch (port_topology) { + default: + case PORT_TOPOLOGY_PAIRED: + if ((rxp & 0x1) == 0) { + if (rxp + 1 < cur_fwd_config.nb_fwd_ports) + return rxp + 1; + if (warning_once) { + printf("\nWarning! port-topology=paired" + " and odd forward ports number," + " the last port will pair with" + " itself.\n\n"); + warning_once = 0; + } + return rxp; + } + return rxp - 1; + case PORT_TOPOLOGY_CHAINED: + return (rxp + 1) % cur_fwd_config.nb_fwd_ports; + case PORT_TOPOLOGY_LOOP: + return rxp; + } +} + static void simple_fwd_config_setup(void) { @@ -1198,7 +1950,7 @@ simple_fwd_config_setup(void) fwd_streams[i]->rx_queue = 0; fwd_streams[i]->tx_port = fwd_ports_ids[j]; fwd_streams[i]->tx_queue = 0; - fwd_streams[i]->peer_addr = j; + fwd_streams[i]->peer_addr = fwd_streams[i]->tx_port; fwd_streams[i]->retry_enabled = retry_enabled; if (port_topology == PORT_TOPOLOGY_PAIRED) { @@ -1206,7 +1958,7 @@ simple_fwd_config_setup(void) fwd_streams[j]->rx_queue = 0; fwd_streams[j]->tx_port = fwd_ports_ids[i]; fwd_streams[j]->tx_queue = 0; - fwd_streams[j]->peer_addr = i; + fwd_streams[j]->peer_addr = fwd_streams[j]->tx_port; fwd_streams[j]->retry_enabled = retry_enabled; } } @@ -1216,11 +1968,6 @@ simple_fwd_config_setup(void) * For the RSS forwarding test all streams distributed over lcores. Each stream * being composed of a RX queue to poll on a RX port for input messages, * associated with a TX queue of a TX port where to send forwarded packets. - * All packets received on the RX queue of index "RxQj" of the RX port "RxPi" - * are sent on the TX queue "TxQl" of the TX port "TxPk" according to the two - * following rules: - * - TxPk = (RxPi + 1) if RxPi is even, (RxPi - 1) if RxPi is odd - * - TxQl = RxQj */ static void rss_fwd_config_setup(void) @@ -1252,18 +1999,7 @@ rss_fwd_config_setup(void) struct fwd_stream *fs; fs = fwd_streams[sm_id]; - - if ((rxp & 0x1) == 0) - txp = (portid_t) (rxp + 1); - else - txp = (portid_t) (rxp - 1); - /* - * if we are in loopback, simply send stuff out through the - * ingress port - */ - if (port_topology == PORT_TOPOLOGY_LOOP) - txp = rxp; - + txp = fwd_topology_tx_port_get(rxp); fs->rx_port = fwd_ports_ids[rxp]; fs->rx_queue = rxq; fs->tx_port = fwd_ports_ids[txp]; @@ -1278,11 +2014,7 @@ rss_fwd_config_setup(void) * Restart from RX queue 0 on next RX port */ rxq = 0; - if (numa_support && (nb_fwd_ports <= (nb_ports >> 1))) - rxp = (portid_t) - (rxp + ((nb_ports >> 1) / nb_fwd_ports)); - else - rxp = (portid_t) (rxp + 1); + rxp++; } } @@ -1602,7 +2334,7 @@ set_fwd_ports_mask(uint64_t portmask) return; } nb_pt = 0; - for (i = 0; i < (unsigned)RTE_MIN(64, RTE_MAX_ETHPORTS); i++) { + RTE_ETH_FOREACH_DEV(i) { if (! ((uint64_t)(1ULL << i) & portmask)) continue; portlist[nb_pt++] = i; @@ -1738,6 +2470,113 @@ set_tx_pkt_segments(unsigned *seg_lengths, unsigned nb_segs) tx_pkt_nb_segs = (uint8_t) nb_segs; } +void +setup_gro(const char *onoff, portid_t port_id) +{ + if (!rte_eth_dev_is_valid_port(port_id)) { + printf("invalid port id %u\n", port_id); + return; + } + if (test_done == 0) { + printf("Before enable/disable GRO," + " please stop forwarding first\n"); + return; + } + if (strcmp(onoff, "on") == 0) { + if (gro_ports[port_id].enable != 0) { + printf("Port %u has enabled GRO. Please" + " disable GRO first\n", port_id); + return; + } + if (gro_flush_cycles == GRO_DEFAULT_FLUSH_CYCLES) { + gro_ports[port_id].param.gro_types = RTE_GRO_TCP_IPV4; + gro_ports[port_id].param.max_flow_num = + GRO_DEFAULT_FLOW_NUM; + gro_ports[port_id].param.max_item_per_flow = + GRO_DEFAULT_ITEM_NUM_PER_FLOW; + } + gro_ports[port_id].enable = 1; + } else { + if (gro_ports[port_id].enable == 0) { + printf("Port %u has disabled GRO\n", port_id); + return; + } + gro_ports[port_id].enable = 0; + } +} + +void +setup_gro_flush_cycles(uint8_t cycles) +{ + if (test_done == 0) { + printf("Before change flush interval for GRO," + " please stop forwarding first.\n"); + return; + } + + if (cycles > GRO_MAX_FLUSH_CYCLES || cycles < + GRO_DEFAULT_FLUSH_CYCLES) { + printf("The flushing cycle be in the range" + " of 1 to %u. Revert to the default" + " value %u.\n", + GRO_MAX_FLUSH_CYCLES, + GRO_DEFAULT_FLUSH_CYCLES); + cycles = GRO_DEFAULT_FLUSH_CYCLES; + } + + gro_flush_cycles = cycles; +} + +void +show_gro(portid_t port_id) +{ + struct rte_gro_param *param; + uint32_t max_pkts_num; + + param = &gro_ports[port_id].param; + + if (!rte_eth_dev_is_valid_port(port_id)) { + printf("Invalid port id %u.\n", port_id); + return; + } + if (gro_ports[port_id].enable) { + printf("GRO type: TCP/IPv4\n"); + if (gro_flush_cycles == GRO_DEFAULT_FLUSH_CYCLES) { + max_pkts_num = param->max_flow_num * + param->max_item_per_flow; + } else + max_pkts_num = MAX_PKT_BURST * GRO_MAX_FLUSH_CYCLES; + printf("Max number of packets to perform GRO: %u\n", + max_pkts_num); + printf("Flushing cycles: %u\n", gro_flush_cycles); + } else + printf("Port %u doesn't enable GRO.\n", port_id); +} + +void +setup_gso(const char *mode, portid_t port_id) +{ + if (!rte_eth_dev_is_valid_port(port_id)) { + printf("invalid port id %u\n", port_id); + return; + } + if (strcmp(mode, "on") == 0) { + if (test_done == 0) { + printf("before enabling GSO," + " please stop forwarding first\n"); + return; + } + gso_ports[port_id].enable = 1; + } else if (strcmp(mode, "off") == 0) { + if (test_done == 0) { + printf("before disabling GSO," + " please stop forwarding first\n"); + return; + } + gso_ports[port_id].enable = 0; + } +} + char* list_pkt_forwarding_modes(void) { @@ -2055,6 +2894,12 @@ set_qmap(portid_t port_id, uint8_t is_rx, uint16_t queue_id, uint8_t map_value) } } +void +set_xstats_hide_zero(uint8_t on_off) +{ + xstats_hide_zero = on_off; +} + static inline void print_fdir_mask(struct rte_eth_fdir_masks *mask) { @@ -2288,7 +3133,7 @@ fdir_set_flex_mask(portid_t port_id, struct rte_eth_fdir_flex_mask *cfg) return; } } - (void)rte_memcpy(&flex_conf->flex_mask[idx], + rte_memcpy(&flex_conf->flex_mask[idx], cfg, sizeof(struct rte_eth_fdir_flex_mask)); } @@ -2318,7 +3163,7 @@ fdir_set_flex_payload(portid_t port_id, struct rte_eth_flex_payload_cfg *cfg) return; } } - (void)rte_memcpy(&flex_conf->flex_set[idx], + rte_memcpy(&flex_conf->flex_set[idx], cfg, sizeof(struct rte_eth_flex_payload_cfg)); @@ -2327,39 +3172,24 @@ fdir_set_flex_payload(portid_t port_id, struct rte_eth_flex_payload_cfg *cfg) void set_vf_traffic(portid_t port_id, uint8_t is_rx, uint16_t vf, uint8_t on) { +#ifdef RTE_LIBRTE_IXGBE_PMD int diag; - if (port_id_is_invalid(port_id, ENABLED_WARN)) - return; if (is_rx) - diag = rte_eth_dev_set_vf_rx(port_id,vf,on); + diag = rte_pmd_ixgbe_set_vf_rx(port_id, vf, on); else - diag = rte_eth_dev_set_vf_tx(port_id,vf,on); + diag = rte_pmd_ixgbe_set_vf_tx(port_id, vf, on); + if (diag == 0) return; - if(is_rx) - printf("rte_eth_dev_set_vf_rx for port_id=%d failed " - "diag=%d\n", port_id, diag); - else - printf("rte_eth_dev_set_vf_tx for port_id=%d failed " - "diag=%d\n", port_id, diag); - -} - -void -set_vf_rx_vlan(portid_t port_id, uint16_t vlan_id, uint64_t vf_mask, uint8_t on) -{ - int diag; - - if (port_id_is_invalid(port_id, ENABLED_WARN)) - return; - if (vlan_id_is_invalid(vlan_id)) - return; - diag = rte_eth_dev_set_vf_vlan_filter(port_id, vlan_id, vf_mask, on); - if (diag == 0) - return; - printf("rte_eth_dev_set_vf_vlan_filter for port_id=%d failed " - "diag=%d\n", port_id, diag); + printf("rte_pmd_ixgbe_set_vf_%s for port_id=%d failed diag=%d\n", + is_rx ? "rx" : "tx", port_id, diag); + return; +#endif + printf("VF %s setting not supported for port %d\n", + is_rx ? "Rx" : "Tx", port_id); + RTE_SET_USED(vf); + RTE_SET_USED(on); } int @@ -2387,24 +3217,25 @@ set_queue_rate_limit(portid_t port_id, uint16_t queue_idx, uint16_t rate) int set_vf_rate_limit(portid_t port_id, uint16_t vf, uint16_t rate, uint64_t q_msk) { - int diag; - struct rte_eth_link link; + int diag = -ENOTSUP; - if (q_msk == 0) - return 0; + RTE_SET_USED(vf); + RTE_SET_USED(rate); + RTE_SET_USED(q_msk); - if (port_id_is_invalid(port_id, ENABLED_WARN)) - return 1; - rte_eth_link_get_nowait(port_id, &link); - if (rate > link.link_speed) { - printf("Invalid rate value:%u bigger than link speed: %u\n", - rate, link.link_speed); - return 1; - } - diag = rte_eth_set_vf_rate_limit(port_id, vf, rate, q_msk); +#ifdef RTE_LIBRTE_IXGBE_PMD + if (diag == -ENOTSUP) + diag = rte_pmd_ixgbe_set_vf_rate_limit(port_id, vf, rate, + q_msk); +#endif +#ifdef RTE_LIBRTE_BNXT_PMD + if (diag == -ENOTSUP) + diag = rte_pmd_bnxt_set_vf_rate_limit(port_id, vf, rate, q_msk); +#endif if (diag == 0) return diag; - printf("rte_eth_set_vf_rate_limit for port_id=%d failed diag=%d\n", + + printf("set_vf_rate_limit for port_id=%d failed diag=%d\n", port_id, diag); return diag; } @@ -2482,7 +3313,7 @@ mcast_addr_pool_remove(struct rte_port *port, uint32_t addr_idx) } static void -eth_port_multicast_addr_list_set(uint8_t port_id) +eth_port_multicast_addr_list_set(portid_t port_id) { struct rte_port *port; int diag; @@ -2497,7 +3328,7 @@ eth_port_multicast_addr_list_set(uint8_t port_id) } void -mcast_addr_add(uint8_t port_id, struct ether_addr *mc_addr) +mcast_addr_add(portid_t port_id, struct ether_addr *mc_addr) { struct rte_port *port; uint32_t i; @@ -2525,7 +3356,7 @@ mcast_addr_add(uint8_t port_id, struct ether_addr *mc_addr) } void -mcast_addr_remove(uint8_t port_id, struct ether_addr *mc_addr) +mcast_addr_remove(portid_t port_id, struct ether_addr *mc_addr) { struct rte_port *port; uint32_t i; @@ -2552,7 +3383,7 @@ mcast_addr_remove(uint8_t port_id, struct ether_addr *mc_addr) } void -port_dcb_info_display(uint8_t port_id) +port_dcb_info_display(portid_t port_id) { struct rte_eth_dcb_info dcb_info; uint16_t i; @@ -2593,3 +3424,131 @@ port_dcb_info_display(uint8_t port_id) printf("\t%4d", dcb_info.tc_queue.tc_txq[0][i].nb_queue); printf("\n"); } + +uint8_t * +open_ddp_package_file(const char *file_path, uint32_t *size) +{ + int fd = open(file_path, O_RDONLY); + off_t pkg_size; + uint8_t *buf = NULL; + int ret = 0; + struct stat st_buf; + + if (size) + *size = 0; + + if (fd == -1) { + printf("%s: Failed to open %s\n", __func__, file_path); + return buf; + } + + if ((fstat(fd, &st_buf) != 0) || (!S_ISREG(st_buf.st_mode))) { + close(fd); + printf("%s: File operations failed\n", __func__); + return buf; + } + + pkg_size = st_buf.st_size; + if (pkg_size < 0) { + close(fd); + printf("%s: File operations failed\n", __func__); + return buf; + } + + buf = (uint8_t *)malloc(pkg_size); + if (!buf) { + close(fd); + printf("%s: Failed to malloc memory\n", __func__); + return buf; + } + + ret = read(fd, buf, pkg_size); + if (ret < 0) { + close(fd); + printf("%s: File read operation failed\n", __func__); + close_ddp_package_file(buf); + return NULL; + } + + if (size) + *size = pkg_size; + + close(fd); + + return buf; +} + +int +save_ddp_package_file(const char *file_path, uint8_t *buf, uint32_t size) +{ + FILE *fh = fopen(file_path, "wb"); + + if (fh == NULL) { + printf("%s: Failed to open %s\n", __func__, file_path); + return -1; + } + + if (fwrite(buf, 1, size, fh) != size) { + fclose(fh); + printf("%s: File write operation failed\n", __func__); + return -1; + } + + fclose(fh); + + return 0; +} + +int +close_ddp_package_file(uint8_t *buf) +{ + if (buf) { + free((void *)buf); + return 0; + } + + return -1; +} + +void +port_queue_region_info_display(portid_t port_id, void *buf) +{ +#ifdef RTE_LIBRTE_I40E_PMD + uint16_t i, j; + struct rte_pmd_i40e_queue_regions *info = + (struct rte_pmd_i40e_queue_regions *)buf; + static const char *queue_region_info_stats_border = "-------"; + + if (!info->queue_region_number) + printf("there is no region has been set before"); + + printf("\n %s All queue region info for port=%2d %s", + queue_region_info_stats_border, port_id, + queue_region_info_stats_border); + printf("\n queue_region_number: %-14u \n", + info->queue_region_number); + + for (i = 0; i < info->queue_region_number; i++) { + printf("\n region_id: %-14u queue_number: %-14u " + "queue_start_index: %-14u \n", + info->region[i].region_id, + info->region[i].queue_num, + info->region[i].queue_start_index); + + printf(" user_priority_num is %-14u :", + info->region[i].user_priority_num); + for (j = 0; j < info->region[i].user_priority_num; j++) + printf(" %-14u ", info->region[i].user_priority[j]); + + printf("\n flowtype_num is %-14u :", + info->region[i].flowtype_num); + for (j = 0; j < info->region[i].flowtype_num; j++) + printf(" %-14u ", info->region[i].hw_flowtype[j]); + } +#else + RTE_SET_USED(port_id); + RTE_SET_USED(buf); +#endif + + printf("\n\n"); +} diff --git a/dpdk/app/test-pmd/csumonly.c b/dpdk/app/test-pmd/csumonly.c index ac4bd8f4..aa29f5fc 100644 --- a/dpdk/app/test-pmd/csumonly.c +++ b/dpdk/app/test-pmd/csumonly.c @@ -49,18 +49,14 @@ #include #include #include -#include #include #include #include #include #include #include -#include -#include #include #include -#include #include #include #include @@ -71,6 +67,10 @@ #include #include #include +#include +#include +#include + #include "testpmd.h" #define IP_DEFTTL 64 /* from RFC 1340. */ @@ -92,6 +92,7 @@ /* structure that caches offload info for the current packet */ struct testpmd_offload_info { uint16_t ethertype; + uint8_t gso_enable; uint16_t l2_len; uint16_t l3_len; uint16_t l4_len; @@ -102,6 +103,8 @@ struct testpmd_offload_info { uint16_t outer_l3_len; uint8_t outer_l4_proto; uint16_t tso_segsz; + uint16_t tunnel_tso_segsz; + uint32_t pkt_len; }; /* simplified GRE header */ @@ -110,15 +113,6 @@ struct simple_gre_hdr { uint16_t proto; } __attribute__((__packed__)); -static uint16_t -get_psd_sum(void *l3_hdr, uint16_t ethertype, uint64_t ol_flags) -{ - if (ethertype == _htons(ETHER_TYPE_IPv4)) - return rte_ipv4_phdr_cksum(l3_hdr, ol_flags); - else /* assume ethertype == ETHER_TYPE_IPv6 */ - return rte_ipv6_phdr_cksum(l3_hdr, ol_flags); -} - static uint16_t get_udptcp_checksum(void *l3_hdr, void *l4_hdr, uint16_t ethertype) { @@ -318,21 +312,6 @@ parse_encap_ip(void *encap_ip, struct testpmd_offload_info *info) info->l2_len = 0; } -/* modify the IPv4 or IPv4 source address of a packet */ -static void -change_ip_addresses(void *l3_hdr, uint16_t ethertype) -{ - struct ipv4_hdr *ipv4_hdr = l3_hdr; - struct ipv6_hdr *ipv6_hdr = l3_hdr; - - if (ethertype == _htons(ETHER_TYPE_IPv4)) { - ipv4_hdr->src_addr = - rte_cpu_to_be_32(rte_be_to_cpu_32(ipv4_hdr->src_addr) + 1); - } else if (ethertype == _htons(ETHER_TYPE_IPv6)) { - ipv6_hdr->src_addr[15] = ipv6_hdr->src_addr[15] + 1; - } -} - /* if possible, calculate the checksum of a packet in hw or sw, * depending on the testpmd command line configuration */ static uint64_t @@ -344,13 +323,28 @@ process_inner_cksums(void *l3_hdr, const struct testpmd_offload_info *info, struct tcp_hdr *tcp_hdr; struct sctp_hdr *sctp_hdr; uint64_t ol_flags = 0; + uint32_t max_pkt_len, tso_segsz = 0; + + /* ensure packet is large enough to require tso */ + if (!info->is_tunnel) { + max_pkt_len = info->l2_len + info->l3_len + info->l4_len + + info->tso_segsz; + if (info->tso_segsz != 0 && info->pkt_len > max_pkt_len) + tso_segsz = info->tso_segsz; + } else { + max_pkt_len = info->outer_l2_len + info->outer_l3_len + + info->l2_len + info->l3_len + info->l4_len + + info->tunnel_tso_segsz; + if (info->tunnel_tso_segsz != 0 && info->pkt_len > max_pkt_len) + tso_segsz = info->tunnel_tso_segsz; + } if (info->ethertype == _htons(ETHER_TYPE_IPv4)) { ipv4_hdr = l3_hdr; ipv4_hdr->hdr_checksum = 0; ol_flags |= PKT_TX_IPV4; - if (info->tso_segsz != 0 && info->l4_proto == IPPROTO_TCP) { + if (info->l4_proto == IPPROTO_TCP && tso_segsz) { ol_flags |= PKT_TX_IP_CKSUM; } else { if (testpmd_ol_flags & TESTPMD_TX_OFFLOAD_IP_CKSUM) @@ -369,11 +363,9 @@ process_inner_cksums(void *l3_hdr, const struct testpmd_offload_info *info, /* do not recalculate udp cksum if it was 0 */ if (udp_hdr->dgram_cksum != 0) { udp_hdr->dgram_cksum = 0; - if (testpmd_ol_flags & TESTPMD_TX_OFFLOAD_UDP_CKSUM) { + if (testpmd_ol_flags & TESTPMD_TX_OFFLOAD_UDP_CKSUM) ol_flags |= PKT_TX_UDP_CKSUM; - udp_hdr->dgram_cksum = get_psd_sum(l3_hdr, - info->ethertype, ol_flags); - } else { + else { udp_hdr->dgram_cksum = get_udptcp_checksum(l3_hdr, udp_hdr, info->ethertype); @@ -382,19 +374,17 @@ process_inner_cksums(void *l3_hdr, const struct testpmd_offload_info *info, } else if (info->l4_proto == IPPROTO_TCP) { tcp_hdr = (struct tcp_hdr *)((char *)l3_hdr + info->l3_len); tcp_hdr->cksum = 0; - if (info->tso_segsz != 0) { + if (tso_segsz) ol_flags |= PKT_TX_TCP_SEG; - tcp_hdr->cksum = get_psd_sum(l3_hdr, info->ethertype, - ol_flags); - } else if (testpmd_ol_flags & TESTPMD_TX_OFFLOAD_TCP_CKSUM) { + else if (testpmd_ol_flags & TESTPMD_TX_OFFLOAD_TCP_CKSUM) ol_flags |= PKT_TX_TCP_CKSUM; - tcp_hdr->cksum = get_psd_sum(l3_hdr, info->ethertype, - ol_flags); - } else { + else { tcp_hdr->cksum = get_udptcp_checksum(l3_hdr, tcp_hdr, info->ethertype); } + if (info->gso_enable) + ol_flags |= PKT_TX_TCP_SEG; } else if (info->l4_proto == IPPROTO_SCTP) { sctp_hdr = (struct sctp_hdr *)((char *)l3_hdr + info->l3_len); sctp_hdr->cksum = 0; @@ -412,12 +402,10 @@ process_inner_cksums(void *l3_hdr, const struct testpmd_offload_info *info, return ol_flags; } -/* Calculate the checksum of outer header (only vxlan is supported, - * meaning IP + UDP). The caller already checked that it's a vxlan - * packet */ +/* Calculate the checksum of outer header */ static uint64_t process_outer_cksums(void *outer_l3_hdr, struct testpmd_offload_info *info, - uint16_t testpmd_ol_flags) + uint16_t testpmd_ol_flags, int tso_enabled) { struct ipv4_hdr *ipv4_hdr = outer_l3_hdr; struct ipv6_hdr *ipv6_hdr = outer_l3_hdr; @@ -432,16 +420,26 @@ process_outer_cksums(void *outer_l3_hdr, struct testpmd_offload_info *info, ol_flags |= PKT_TX_OUTER_IP_CKSUM; else ipv4_hdr->hdr_checksum = rte_ipv4_cksum(ipv4_hdr); - } else if (testpmd_ol_flags & TESTPMD_TX_OFFLOAD_OUTER_IP_CKSUM) + } else ol_flags |= PKT_TX_OUTER_IPV6; if (info->outer_l4_proto != IPPROTO_UDP) return ol_flags; - /* outer UDP checksum is always done in software as we have no - * hardware supporting it today, and no API for it. */ - udp_hdr = (struct udp_hdr *)((char *)outer_l3_hdr + info->outer_l3_len); + + /* outer UDP checksum is done in software as we have no hardware + * supporting it today, and no API for it. In the other side, for + * UDP tunneling, like VXLAN or Geneve, outer UDP checksum can be + * set to zero. + * + * If a packet will be TSOed into small packets by NIC, we cannot + * set/calculate a non-zero checksum, because it will be a wrong + * value after the packet be split into several small packets. + */ + if (tso_enabled) + udp_hdr->dgram_cksum = 0; + /* do not recalculate udp cksum if it was 0 */ if (udp_hdr->dgram_cksum != 0) { udp_hdr->dgram_cksum = 0; @@ -588,7 +586,7 @@ pkt_copy_split(const struct rte_mbuf *pkt) rc = mbuf_copy_split(pkt, md, seglen, nb_seg); if (rc < 0) RTE_LOG(ERR, USER1, - "mbuf_copy_split for %p(len=%u, nb_seg=%hhu) " + "mbuf_copy_split for %p(len=%u, nb_seg=%u) " "into %u segments failed with error code: %d\n", pkt, pkt->pkt_len, pkt->nb_segs, nb_seg, rc); @@ -609,7 +607,6 @@ pkt_copy_split(const struct rte_mbuf *pkt) * Receive a burst of packets, and for each packet: * - parse packet, and try to recognize a supported packet type (1) * - if it's not a supported packet type, don't touch the packet, else: - * - modify the IPs in inner headers and in outer headers if any * - reprocess the checksum of all supported layers. This is done in SW * or HW, depending on testpmd command line configuration * - if TSO is enabled in testpmd command line, also flag the mbuf for TCP @@ -634,19 +631,28 @@ static void pkt_burst_checksum_forward(struct fwd_stream *fs) { struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; + struct rte_mbuf *gso_segments[GSO_MAX_PKT_BURST]; + struct rte_gso_ctx *gso_ctx; + struct rte_mbuf **tx_pkts_burst; struct rte_port *txp; struct rte_mbuf *m, *p; struct ether_hdr *eth_hdr; void *l3_hdr = NULL, *outer_l3_hdr = NULL; /* can be IPv4 or IPv6 */ + void **gro_ctx; + uint16_t gro_pkts_num; + uint8_t gro_enable; uint16_t nb_rx; uint16_t nb_tx; + uint16_t nb_prep; uint16_t i; - uint64_t ol_flags; + uint64_t rx_ol_flags, tx_ol_flags; uint16_t testpmd_ol_flags; uint32_t retry; uint32_t rx_bad_ip_csum; uint32_t rx_bad_l4_csum; struct testpmd_offload_info info; + uint16_t nb_segments = 0; + int ret; #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES uint64_t start_tsc; @@ -663,31 +669,38 @@ pkt_burst_checksum_forward(struct fwd_stream *fs) nb_pkt_per_burst); if (unlikely(nb_rx == 0)) return; - #ifdef RTE_TEST_PMD_RECORD_BURST_STATS fs->rx_burst_stats.pkt_burst_spread[nb_rx]++; #endif fs->rx_packets += nb_rx; rx_bad_ip_csum = 0; rx_bad_l4_csum = 0; + gro_enable = gro_ports[fs->rx_port].enable; txp = &ports[fs->tx_port]; testpmd_ol_flags = txp->tx_ol_flags; memset(&info, 0, sizeof(info)); info.tso_segsz = txp->tso_segsz; + info.tunnel_tso_segsz = txp->tunnel_tso_segsz; + if (gso_ports[fs->tx_port].enable) + info.gso_enable = 1; for (i = 0; i < nb_rx; i++) { if (likely(i < nb_rx - 1)) rte_prefetch0(rte_pktmbuf_mtod(pkts_burst[i + 1], void *)); - ol_flags = 0; - info.is_tunnel = 0; m = pkts_burst[i]; + info.is_tunnel = 0; + info.pkt_len = rte_pktmbuf_pkt_len(m); + tx_ol_flags = 0; + rx_ol_flags = m->ol_flags; /* Update the L3/L4 checksum error packet statistics */ - rx_bad_ip_csum += ((m->ol_flags & PKT_RX_IP_CKSUM_BAD) != 0); - rx_bad_l4_csum += ((m->ol_flags & PKT_RX_L4_CKSUM_BAD) != 0); + if ((rx_ol_flags & PKT_RX_IP_CKSUM_MASK) == PKT_RX_IP_CKSUM_BAD) + rx_bad_ip_csum += 1; + if ((rx_ol_flags & PKT_RX_L4_CKSUM_MASK) == PKT_RX_L4_CKSUM_BAD) + rx_bad_l4_csum += 1; /* step 1: dissect packet, parsing optional vlan, ip4/ip6, vxlan * and inner headers */ @@ -704,18 +717,27 @@ pkt_burst_checksum_forward(struct fwd_stream *fs) if (testpmd_ol_flags & TESTPMD_TX_OFFLOAD_PARSE_TUNNEL) { if (info.l4_proto == IPPROTO_UDP) { struct udp_hdr *udp_hdr; + udp_hdr = (struct udp_hdr *)((char *)l3_hdr + info.l3_len); parse_vxlan(udp_hdr, &info, m->packet_type); + if (info.is_tunnel) + tx_ol_flags |= PKT_TX_TUNNEL_VXLAN; } else if (info.l4_proto == IPPROTO_GRE) { struct simple_gre_hdr *gre_hdr; + gre_hdr = (struct simple_gre_hdr *) ((char *)l3_hdr + info.l3_len); parse_gre(gre_hdr, &info); + if (info.is_tunnel) + tx_ol_flags |= PKT_TX_TUNNEL_GRE; } else if (info.l4_proto == IPPROTO_IPIP) { void *encap_ip_hdr; + encap_ip_hdr = (char *)l3_hdr + info.l3_len; parse_encap_ip(encap_ip_hdr, &info); + if (info.is_tunnel) + tx_ol_flags |= PKT_TX_TUNNEL_IPIP; } } @@ -725,38 +747,37 @@ pkt_burst_checksum_forward(struct fwd_stream *fs) l3_hdr = (char *)l3_hdr + info.outer_l3_len + info.l2_len; } - /* step 2: change all source IPs (v4 or v6) so we need - * to recompute the chksums even if they were correct */ - - change_ip_addresses(l3_hdr, info.ethertype); - if (info.is_tunnel == 1) - change_ip_addresses(outer_l3_hdr, info.outer_ethertype); - - /* step 3: depending on user command line configuration, + /* step 2: depending on user command line configuration, * recompute checksum either in software or flag the * mbuf to offload the calculation to the NIC. If TSO * is configured, prepare the mbuf for TCP segmentation. */ /* process checksums of inner headers first */ - ol_flags |= process_inner_cksums(l3_hdr, &info, testpmd_ol_flags); + tx_ol_flags |= process_inner_cksums(l3_hdr, &info, + testpmd_ol_flags); /* Then process outer headers if any. Note that the software * checksum will be wrong if one of the inner checksums is * processed in hardware. */ if (info.is_tunnel == 1) { - ol_flags |= process_outer_cksums(outer_l3_hdr, &info, - testpmd_ol_flags); + tx_ol_flags |= process_outer_cksums(outer_l3_hdr, &info, + testpmd_ol_flags, + !!(tx_ol_flags & PKT_TX_TCP_SEG)); } - /* step 4: fill the mbuf meta data (flags and header lengths) */ + /* step 3: fill the mbuf meta data (flags and header lengths) */ if (info.is_tunnel == 1) { - if (testpmd_ol_flags & TESTPMD_TX_OFFLOAD_OUTER_IP_CKSUM) { + if (info.tunnel_tso_segsz || + (testpmd_ol_flags & + TESTPMD_TX_OFFLOAD_OUTER_IP_CKSUM) || + (tx_ol_flags & PKT_TX_OUTER_IPV6)) { m->outer_l2_len = info.outer_l2_len; m->outer_l3_len = info.outer_l3_len; m->l2_len = info.l2_len; m->l3_len = info.l3_len; m->l4_len = info.l4_len; + m->tso_segsz = info.tunnel_tso_segsz; } else { /* if there is a outer UDP cksum @@ -776,9 +797,9 @@ pkt_burst_checksum_forward(struct fwd_stream *fs) m->l2_len = info.l2_len; m->l3_len = info.l3_len; m->l4_len = info.l4_len; + m->tso_segsz = info.tso_segsz; } - m->tso_segsz = info.tso_segsz; - m->ol_flags = ol_flags; + m->ol_flags = tx_ol_flags; /* Do split & copy for the packet. */ if (tx_pkt_split != TX_PKT_SPLIT_OFF) { @@ -792,32 +813,19 @@ pkt_burst_checksum_forward(struct fwd_stream *fs) /* if verbose mode is enabled, dump debug info */ if (verbose_level > 0) { - struct { - uint64_t flag; - uint64_t mask; - } tx_flags[] = { - { PKT_TX_IP_CKSUM, PKT_TX_IP_CKSUM }, - { PKT_TX_UDP_CKSUM, PKT_TX_L4_MASK }, - { PKT_TX_TCP_CKSUM, PKT_TX_L4_MASK }, - { PKT_TX_SCTP_CKSUM, PKT_TX_L4_MASK }, - { PKT_TX_IPV4, PKT_TX_IPV4 }, - { PKT_TX_IPV6, PKT_TX_IPV6 }, - { PKT_TX_OUTER_IP_CKSUM, PKT_TX_OUTER_IP_CKSUM }, - { PKT_TX_OUTER_IPV4, PKT_TX_OUTER_IPV4 }, - { PKT_TX_OUTER_IPV6, PKT_TX_OUTER_IPV6 }, - { PKT_TX_TCP_SEG, PKT_TX_TCP_SEG }, - }; - unsigned j; - const char *name; + char buf[256]; printf("-----------------\n"); - printf("mbuf=%p, pkt_len=%u, nb_segs=%hhu:\n", - m, m->pkt_len, m->nb_segs); + printf("port=%u, mbuf=%p, pkt_len=%u, nb_segs=%u:\n", + fs->rx_port, m, m->pkt_len, m->nb_segs); /* dump rx parsed packet info */ + rte_get_rx_ol_flag_list(rx_ol_flags, buf, sizeof(buf)); printf("rx: l2_len=%d ethertype=%x l3_len=%d " - "l4_proto=%d l4_len=%d\n", + "l4_proto=%d l4_len=%d flags=%s\n", info.l2_len, rte_be_to_cpu_16(info.ethertype), - info.l3_len, info.l4_proto, info.l4_len); + info.l3_len, info.l4_proto, info.l4_len, buf); + if (rx_ol_flags & PKT_RX_LRO) + printf("rx: m->lro_segsz=%u\n", m->tso_segsz); if (info.is_tunnel == 1) printf("rx: outer_l2_len=%d outer_ethertype=%x " "outer_l3_len=%d\n", info.outer_l2_len, @@ -832,23 +840,80 @@ pkt_burst_checksum_forward(struct fwd_stream *fs) printf("tx: m->l2_len=%d m->l3_len=%d " "m->l4_len=%d\n", m->l2_len, m->l3_len, m->l4_len); - if ((info.is_tunnel == 1) && - (testpmd_ol_flags & TESTPMD_TX_OFFLOAD_OUTER_IP_CKSUM)) - printf("tx: m->outer_l2_len=%d m->outer_l3_len=%d\n", - m->outer_l2_len, m->outer_l3_len); - if (info.tso_segsz != 0) + if (info.is_tunnel == 1) { + if ((testpmd_ol_flags & + TESTPMD_TX_OFFLOAD_OUTER_IP_CKSUM) || + (tx_ol_flags & PKT_TX_OUTER_IPV6)) + printf("tx: m->outer_l2_len=%d " + "m->outer_l3_len=%d\n", + m->outer_l2_len, + m->outer_l3_len); + if (info.tunnel_tso_segsz != 0 && + (m->ol_flags & PKT_TX_TCP_SEG)) + printf("tx: m->tso_segsz=%d\n", + m->tso_segsz); + } else if (info.tso_segsz != 0 && + (m->ol_flags & PKT_TX_TCP_SEG)) printf("tx: m->tso_segsz=%d\n", m->tso_segsz); - printf("tx: flags="); - for (j = 0; j < sizeof(tx_flags)/sizeof(*tx_flags); j++) { - name = rte_get_tx_ol_flag_name(tx_flags[j].flag); - if ((m->ol_flags & tx_flags[j].mask) == - tx_flags[j].flag) - printf("%s ", name); - } + rte_get_tx_ol_flag_list(m->ol_flags, buf, sizeof(buf)); + printf("tx: flags=%s", buf); printf("\n"); } } - nb_tx = rte_eth_tx_burst(fs->tx_port, fs->tx_queue, pkts_burst, nb_rx); + + if (unlikely(gro_enable)) { + if (gro_flush_cycles == GRO_DEFAULT_FLUSH_CYCLES) { + nb_rx = rte_gro_reassemble_burst(pkts_burst, nb_rx, + &(gro_ports[fs->rx_port].param)); + } else { + gro_ctx = current_fwd_lcore()->gro_ctx; + nb_rx = rte_gro_reassemble(pkts_burst, nb_rx, gro_ctx); + + if (++fs->gro_times >= gro_flush_cycles) { + gro_pkts_num = rte_gro_get_pkt_count(gro_ctx); + if (gro_pkts_num > MAX_PKT_BURST - nb_rx) + gro_pkts_num = MAX_PKT_BURST - nb_rx; + + nb_rx += rte_gro_timeout_flush(gro_ctx, 0, + RTE_GRO_TCP_IPV4, + &pkts_burst[nb_rx], + gro_pkts_num); + fs->gro_times = 0; + } + } + } + + if (gso_ports[fs->tx_port].enable == 0) + tx_pkts_burst = pkts_burst; + else { + gso_ctx = &(current_fwd_lcore()->gso_ctx); + gso_ctx->gso_size = gso_max_segment_size; + for (i = 0; i < nb_rx; i++) { + ret = rte_gso_segment(pkts_burst[i], gso_ctx, + &gso_segments[nb_segments], + GSO_MAX_PKT_BURST - nb_segments); + if (ret >= 0) + nb_segments += ret; + else { + RTE_LOG(DEBUG, USER1, + "Unable to segment packet"); + rte_pktmbuf_free(pkts_burst[i]); + } + } + + tx_pkts_burst = gso_segments; + nb_rx = nb_segments; + } + + nb_prep = rte_eth_tx_prepare(fs->tx_port, fs->tx_queue, + tx_pkts_burst, nb_rx); + if (nb_prep != nb_rx) + printf("Preparing packet burst to transmit failed: %s\n", + rte_strerror(rte_errno)); + + nb_tx = rte_eth_tx_burst(fs->tx_port, fs->tx_queue, tx_pkts_burst, + nb_prep); + /* * Retry if necessary */ @@ -857,7 +922,7 @@ pkt_burst_checksum_forward(struct fwd_stream *fs) while (nb_tx < nb_rx && retry++ < burst_tx_retry_num) { rte_delay_us(burst_tx_delay_time); nb_tx += rte_eth_tx_burst(fs->tx_port, fs->tx_queue, - &pkts_burst[nb_tx], nb_rx - nb_tx); + &tx_pkts_burst[nb_tx], nb_rx - nb_tx); } } fs->tx_packets += nb_tx; @@ -870,9 +935,10 @@ pkt_burst_checksum_forward(struct fwd_stream *fs) if (unlikely(nb_tx < nb_rx)) { fs->fwd_dropped += (nb_rx - nb_tx); do { - rte_pktmbuf_free(pkts_burst[nb_tx]); + rte_pktmbuf_free(tx_pkts_burst[nb_tx]); } while (++nb_tx < nb_rx); } + #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES end_tsc = rte_rdtsc(); core_cycles = (end_tsc - start_tsc); diff --git a/dpdk/app/test-pmd/flowgen.c b/dpdk/app/test-pmd/flowgen.c index a6abe91e..46478fc3 100644 --- a/dpdk/app/test-pmd/flowgen.c +++ b/dpdk/app/test-pmd/flowgen.c @@ -50,15 +50,12 @@ #include #include #include -#include #include #include #include #include #include #include -#include -#include #include #include #include @@ -69,12 +66,12 @@ #include #include #include +#include #include "testpmd.h" /* hardcoded configuration (for now) */ static unsigned cfg_n_flows = 1024; -static unsigned cfg_pkt_size = 300; static uint32_t cfg_ip_src = IPv4(10, 254, 0, 0); static uint32_t cfg_ip_dst = IPv4(10, 253, 0, 0); static uint16_t cfg_udp_src = 1000; @@ -118,7 +115,7 @@ ip_sum(const unaligned_uint16_t *hdr, int hdr_len) static void pkt_burst_flow_gen(struct fwd_stream *fs) { - unsigned pkt_size = cfg_pkt_size - 4; /* Adjust FCS */ + unsigned pkt_size = tx_pkt_length - 4; /* Adjust FCS */ struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; struct rte_mempool *mbp; struct rte_mbuf *pkt; @@ -126,7 +123,7 @@ pkt_burst_flow_gen(struct fwd_stream *fs) struct ipv4_hdr *ip_hdr; struct udp_hdr *udp_hdr; uint16_t vlan_tci, vlan_tci_outer; - uint16_t ol_flags; + uint64_t ol_flags; uint16_t nb_rx; uint16_t nb_tx; uint16_t nb_pkt; @@ -154,7 +151,13 @@ pkt_burst_flow_gen(struct fwd_stream *fs) mbp = current_fwd_lcore()->mbp; vlan_tci = ports[fs->tx_port].tx_vlan_id; vlan_tci_outer = ports[fs->tx_port].tx_vlan_id_outer; - ol_flags = ports[fs->tx_port].tx_ol_flags; + + if (ports[fs->tx_port].tx_ol_flags & TESTPMD_TX_OFFLOAD_INSERT_VLAN) + ol_flags = PKT_TX_VLAN_PKT; + if (ports[fs->tx_port].tx_ol_flags & TESTPMD_TX_OFFLOAD_INSERT_QINQ) + ol_flags |= PKT_TX_QINQ_PKT; + if (ports[fs->tx_port].tx_ol_flags & TESTPMD_TX_OFFLOAD_MACSEC) + ol_flags |= PKT_TX_MACSEC; for (nb_pkt = 0; nb_pkt < nb_pkt_per_burst; nb_pkt++) { pkt = rte_mbuf_raw_alloc(mbp); diff --git a/dpdk/app/test-pmd/icmpecho.c b/dpdk/app/test-pmd/icmpecho.c index be308c9f..d4b4c9eb 100644 --- a/dpdk/app/test-pmd/icmpecho.c +++ b/dpdk/app/test-pmd/icmpecho.c @@ -52,7 +52,6 @@ #include #include #include -#include #include #include #include @@ -62,6 +61,7 @@ #include #include #include +#include #include "testpmd.h" @@ -200,7 +200,7 @@ ip_proto_name(uint16_t ip_proto) "OSPFIGP", /**< OSPFIGP */ "SRPC", /**< Strite RPC protocol */ - "LARP", /**< Locus Address Resoloution */ + "LARP", /**< Locus Address Resolution */ "MTP", /**< Multicast Transport */ "AX25", /**< AX.25 Frames */ "4IN4", /**< IP encapsulated in IP */ @@ -297,7 +297,7 @@ ipv4_hdr_cksum(struct ipv4_hdr *ip_h) (((rte_be_to_cpu_32((ipv4_addr)) >> 24) & 0x000000FF) == 0xE0) /* - * Receive a burst of packets, lookup for ICMP echo requets, and, if any, + * Receive a burst of packets, lookup for ICMP echo requests, and, if any, * send back ICMP echo replies. */ static void diff --git a/dpdk/app/test-pmd/ieee1588fwd.c b/dpdk/app/test-pmd/ieee1588fwd.c index 0d3b37a7..91ee7864 100644 --- a/dpdk/app/test-pmd/ieee1588fwd.c +++ b/dpdk/app/test-pmd/ieee1588fwd.c @@ -34,6 +34,7 @@ #include #include +#include #include "testpmd.h" @@ -85,12 +86,11 @@ port_ieee1588_rx_timestamp_check(portid_t pi, uint32_t index) struct timespec timestamp = {0, 0}; if (rte_eth_timesync_read_rx_timestamp(pi, ×tamp, index) < 0) { - printf("Port %u RX timestamp registers not valid\n", - (unsigned) pi); + printf("Port %u RX timestamp registers not valid\n", pi); return; } printf("Port %u RX timestamp value %lu s %lu ns\n", - (unsigned) pi, timestamp.tv_sec, timestamp.tv_nsec); + pi, timestamp.tv_sec, timestamp.tv_nsec); } #define MAX_TX_TMST_WAIT_MICROSECS 1000 /**< 1 milli-second */ @@ -109,12 +109,12 @@ port_ieee1588_tx_timestamp_check(portid_t pi) if (wait_us >= MAX_TX_TMST_WAIT_MICROSECS) { printf("Port %u TX timestamp registers not valid after " "%u micro-seconds\n", - (unsigned) pi, (unsigned) MAX_TX_TMST_WAIT_MICROSECS); + pi, MAX_TX_TMST_WAIT_MICROSECS); return; } printf("Port %u TX timestamp value %lu s %lu ns validated after " "%u micro-second%s\n", - (unsigned) pi, timestamp.tv_sec, timestamp.tv_nsec, wait_us, + pi, timestamp.tv_sec, timestamp.tv_nsec, wait_us, (wait_us == 1) ? "" : "s"); } @@ -147,11 +147,11 @@ ieee1588_packet_fwd(struct fwd_stream *fs) if (eth_type == ETHER_TYPE_1588) { printf("Port %u Received PTP packet not filtered" " by hardware\n", - (unsigned) fs->rx_port); + fs->rx_port); } else { printf("Port %u Received non PTP packet type=0x%4x " "len=%u\n", - (unsigned) fs->rx_port, eth_type, + fs->rx_port, eth_type, (unsigned) mb->pkt_len); } rte_pktmbuf_free(mb); @@ -160,7 +160,7 @@ ieee1588_packet_fwd(struct fwd_stream *fs) if (eth_type != ETHER_TYPE_1588) { printf("Port %u Received NON PTP packet incorrectly" " detected by hardware\n", - (unsigned) fs->rx_port); + fs->rx_port); rte_pktmbuf_free(mb); return; } @@ -174,19 +174,19 @@ ieee1588_packet_fwd(struct fwd_stream *fs) if (ptp_hdr->version != 0x02) { printf("Port %u Received PTP V2 Ethernet frame with wrong PTP" " protocol version 0x%x (should be 0x02)\n", - (unsigned) fs->rx_port, ptp_hdr->version); + fs->rx_port, ptp_hdr->version); rte_pktmbuf_free(mb); return; } if (ptp_hdr->msg_id != PTP_SYNC_MESSAGE) { printf("Port %u Received PTP V2 Ethernet frame with unexpected" " message ID 0x%x (expected 0x0 - PTP_SYNC_MESSAGE)\n", - (unsigned) fs->rx_port, ptp_hdr->msg_id); + fs->rx_port, ptp_hdr->msg_id); rte_pktmbuf_free(mb); return; } printf("Port %u IEEE1588 PTP V2 SYNC Message filtered by hardware\n", - (unsigned) fs->rx_port); + fs->rx_port); /* * Check that the received PTP packet has been timestamped by the @@ -195,7 +195,7 @@ ieee1588_packet_fwd(struct fwd_stream *fs) if (! (mb->ol_flags & PKT_RX_IEEE1588_TMST)) { printf("Port %u Received PTP packet not timestamped" " by hardware\n", - (unsigned) fs->rx_port); + fs->rx_port); rte_pktmbuf_free(mb); return; } @@ -215,8 +215,7 @@ ieee1588_packet_fwd(struct fwd_stream *fs) mb->ol_flags |= PKT_TX_IEEE1588_TMST; fs->tx_packets += 1; if (rte_eth_tx_burst(fs->rx_port, fs->tx_queue, &mb, 1) == 0) { - printf("Port %u sent PTP packet dropped\n", - (unsigned) fs->rx_port); + printf("Port %u sent PTP packet dropped\n", fs->rx_port); fs->fwd_dropped += 1; rte_pktmbuf_free(mb); return; diff --git a/dpdk/app/test-pmd/iofwd.c b/dpdk/app/test-pmd/iofwd.c index 7b6033a5..ff6de45c 100644 --- a/dpdk/app/test-pmd/iofwd.c +++ b/dpdk/app/test-pmd/iofwd.c @@ -48,15 +48,12 @@ #include #include #include -#include #include #include #include #include #include #include -#include -#include #include #include #include @@ -65,6 +62,7 @@ #include #include #include +#include #include "testpmd.h" diff --git a/dpdk/app/test-pmd/macfwd.c b/dpdk/app/test-pmd/macfwd.c index 5d1c1617..f4a4bf29 100644 --- a/dpdk/app/test-pmd/macfwd.c +++ b/dpdk/app/test-pmd/macfwd.c @@ -49,15 +49,12 @@ #include #include #include -#include #include #include #include #include #include #include -#include -#include #include #include #include @@ -66,6 +63,7 @@ #include #include #include +#include #include "testpmd.h" @@ -113,6 +111,8 @@ pkt_burst_mac_forward(struct fwd_stream *fs) ol_flags = PKT_TX_VLAN_PKT; if (txp->tx_ol_flags & TESTPMD_TX_OFFLOAD_INSERT_QINQ) ol_flags |= PKT_TX_QINQ_PKT; + if (txp->tx_ol_flags & TESTPMD_TX_OFFLOAD_MACSEC) + ol_flags |= PKT_TX_MACSEC; for (i = 0; i < nb_rx; i++) { if (likely(i < nb_rx - 1)) rte_prefetch0(rte_pktmbuf_mtod(pkts_burst[i + 1], diff --git a/dpdk/app/test-pmd/macswap.c b/dpdk/app/test-pmd/macswap.c index 4b0dbeb5..721865c9 100644 --- a/dpdk/app/test-pmd/macswap.c +++ b/dpdk/app/test-pmd/macswap.c @@ -49,15 +49,12 @@ #include #include #include -#include #include #include #include #include #include #include -#include -#include #include #include #include @@ -66,6 +63,7 @@ #include #include #include +#include #include "testpmd.h" @@ -113,6 +111,8 @@ pkt_burst_mac_swap(struct fwd_stream *fs) ol_flags = PKT_TX_VLAN_PKT; if (txp->tx_ol_flags & TESTPMD_TX_OFFLOAD_INSERT_QINQ) ol_flags |= PKT_TX_QINQ_PKT; + if (txp->tx_ol_flags & TESTPMD_TX_OFFLOAD_MACSEC) + ol_flags |= PKT_TX_MACSEC; for (i = 0; i < nb_rx; i++) { if (likely(i < nb_rx - 1)) rte_prefetch0(rte_pktmbuf_mtod(pkts_burst[i + 1], diff --git a/dpdk/app/test-pmd/parameters.c b/dpdk/app/test-pmd/parameters.c index 8792c2c6..8fbb5150 100644 --- a/dpdk/app/test-pmd/parameters.c +++ b/dpdk/app/test-pmd/parameters.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. + * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -41,7 +41,6 @@ #include #include #include -#include #include #include @@ -56,14 +55,12 @@ #include #include #include -#include #include #include #include #include #include #include -#include #include #include #include @@ -77,6 +74,7 @@ #ifdef RTE_LIBRTE_PMD_BOND #include #endif +#include #include "testpmd.h" @@ -86,8 +84,10 @@ usage(char* progname) printf("usage: %s " #ifdef RTE_LIBRTE_CMDLINE "[--interactive|-i] " + "[--cmdline-file=FILENAME] " #endif "[--help|-h] | [--auto-start|-a] | [" + "--tx-first | --stats-period=PERIOD | " "--coremask=COREMASK --portmask=PORTMASK --numa " "--mbuf-size= | --total-num-mbufs= | " "--nb-cores= | --nb-ports= | " @@ -103,10 +103,15 @@ usage(char* progname) progname); #ifdef RTE_LIBRTE_CMDLINE printf(" --interactive: run in interactive mode.\n"); + printf(" --cmdline-file: execute cli commands before startup.\n"); #endif printf(" --auto-start: start forwarding on init " "[always when non-interactive].\n"); printf(" --help: display this message and quit.\n"); + printf(" --tx-first: start forwarding sending a burst first " + "(only if interactive is disabled).\n"); + printf(" --stats-period=PERIOD: statistics will be shown " + "every PERIOD seconds (only if interactive is disabled).\n"); printf(" --nb-cores=N: set the number of forwarding cores " "(1 <= N <= %d).\n", nb_lcores); printf(" --nb-ports=N: set the number of forwarding ports " @@ -149,8 +154,14 @@ usage(char* progname) "the packet will be enqueued into the rx drop-queue. " "If the drop-queue doesn't exist, the packet is dropped. " "By default drop-queue=127.\n"); - printf(" --crc-strip: enable CRC stripping by hardware.\n"); +#ifdef RTE_LIBRTE_LATENCY_STATS + printf(" --latencystats=N: enable latency and jitter statistcs " + "monitoring on forwarding lcore id N.\n"); +#endif + printf(" --disable-crc-strip: disable CRC stripping by hardware.\n"); + printf(" --enable-lro: enable large receive offload.\n"); printf(" --enable-rx-cksum: enable rx hardware checksum offload.\n"); + printf(" --enable-rx-timestamp: enable rx hardware timestamp offload.\n"); printf(" --disable-hw-vlan: disable hardware vlan.\n"); printf(" --disable-hw-vlan-filter: disable hardware vlan filter.\n"); printf(" --disable-hw-vlan-strip: disable hardware vlan strip.\n"); @@ -191,9 +202,20 @@ usage(char* progname) "(0 <= mapping <= %d).\n", RTE_ETHDEV_QUEUE_STAT_CNTRS - 1); printf(" --no-flush-rx: Don't flush RX streams before forwarding." " Used mainly with PCAP drivers.\n"); - printf(" --txpkts=X[,Y]*: set TX segment sizes.\n"); + printf(" --txpkts=X[,Y]*: set TX segment sizes" + " or total packet length.\n"); printf(" --disable-link-check: disable check on link status when " "starting/stopping ports.\n"); + printf(" --no-lsc-interrupt: disable link status change interrupt.\n"); + printf(" --no-rmv-interrupt: disable device removal interrupt.\n"); + printf(" --bitrate-stats=N: set the logical core N to perform " + "bit-rate calculation.\n"); + printf(" --print-event : " + "enable print of designated event or all of them.\n"); + printf(" --mask-event : " + "disable print of designated event or all of them.\n"); + printf(" --flow-isolate-all: " + "requests flow API isolated mode on all ports at initialization time.\n"); } #ifdef RTE_LIBRTE_CMDLINE @@ -353,13 +375,26 @@ parse_queue_stats_mapping_config(const char *q_arg, int is_rx) return 0; } +static void +print_invalid_socket_id_error(void) +{ + unsigned int i = 0; + + printf("Invalid socket id, options are: "); + for (i = 0; i < num_sockets; i++) { + printf("%u%s", socket_ids[i], + (i == num_sockets - 1) ? "\n" : ","); + } +} + static int parse_portnuma_config(const char *q_arg) { char s[256]; const char *p, *p0 = q_arg; char *end; - uint8_t i,port_id,socket_id; + uint8_t i, socket_id; + portid_t port_id; unsigned size; enum fieldnames { FLD_PORT = 0, @@ -389,18 +424,18 @@ parse_portnuma_config(const char *q_arg) if (errno != 0 || end == str_fld[i] || int_fld[i] > 255) return -1; } - port_id = (uint8_t)int_fld[FLD_PORT]; - if (port_id_is_invalid(port_id, ENABLED_WARN)) { + port_id = (portid_t)int_fld[FLD_PORT]; + if (port_id_is_invalid(port_id, ENABLED_WARN) || + port_id == (portid_t)RTE_PORT_ALL) { printf("Valid port range is [0"); - FOREACH_PORT(pid, ports) + RTE_ETH_FOREACH_DEV(pid) printf(", %d", pid); printf("]\n"); return -1; } socket_id = (uint8_t)int_fld[FLD_SOCKET]; - if(socket_id >= max_socket) { - printf("Invalid socket id, range is [0, %d]\n", - max_socket - 1); + if (new_socket_id(socket_id)) { + print_invalid_socket_id_error(); return -1; } port_numa[port_id] = socket_id; @@ -415,7 +450,8 @@ parse_ringnuma_config(const char *q_arg) char s[256]; const char *p, *p0 = q_arg; char *end; - uint8_t i,port_id,ring_flag,socket_id; + uint8_t i, ring_flag, socket_id; + portid_t port_id; unsigned size; enum fieldnames { FLD_PORT = 0, @@ -449,18 +485,18 @@ parse_ringnuma_config(const char *q_arg) if (errno != 0 || end == str_fld[i] || int_fld[i] > 255) return -1; } - port_id = (uint8_t)int_fld[FLD_PORT]; - if (port_id_is_invalid(port_id, ENABLED_WARN)) { + port_id = (portid_t)int_fld[FLD_PORT]; + if (port_id_is_invalid(port_id, ENABLED_WARN) || + port_id == (portid_t)RTE_PORT_ALL) { printf("Valid port range is [0"); - FOREACH_PORT(pid, ports) + RTE_ETH_FOREACH_DEV(pid) printf(", %d", pid); printf("]\n"); return -1; } socket_id = (uint8_t)int_fld[FLD_SOCKET]; - if (socket_id >= max_socket) { - printf("Invalid socket id, range is [0, %d]\n", - max_socket - 1); + if (new_socket_id(socket_id)) { + print_invalid_socket_id_error(); return -1; } ring_flag = (uint8_t)int_fld[FLD_FLAG]; @@ -491,28 +527,65 @@ parse_ringnuma_config(const char *q_arg) return 0; } +static int +parse_event_printing_config(const char *optarg, int enable) +{ + uint32_t mask = 0; + + if (!strcmp(optarg, "unknown")) + mask = UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN; + else if (!strcmp(optarg, "intr_lsc")) + mask = UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC; + else if (!strcmp(optarg, "queue_state")) + mask = UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE; + else if (!strcmp(optarg, "intr_reset")) + mask = UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET; + else if (!strcmp(optarg, "vf_mbox")) + mask = UINT32_C(1) << RTE_ETH_EVENT_VF_MBOX; + else if (!strcmp(optarg, "macsec")) + mask = UINT32_C(1) << RTE_ETH_EVENT_MACSEC; + else if (!strcmp(optarg, "intr_rmv")) + mask = UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV; + else if (!strcmp(optarg, "all")) + mask = ~UINT32_C(0); + else { + fprintf(stderr, "Invalid event: %s\n", optarg); + return -1; + } + if (enable) + event_print_mask |= mask; + else + event_print_mask &= ~mask; + return 0; +} + void launch_args_parse(int argc, char** argv) { int n, opt; char **argvopt; int opt_idx; + portid_t pid; enum { TX, RX }; static struct option lgopts[] = { { "help", 0, 0, 0 }, #ifdef RTE_LIBRTE_CMDLINE { "interactive", 0, 0, 0 }, + { "cmdline-file", 1, 0, 0 }, { "auto-start", 0, 0, 0 }, { "eth-peers-configfile", 1, 0, 0 }, { "eth-peer", 1, 0, 0 }, #endif + { "tx-first", 0, 0, 0 }, + { "stats-period", 1, 0, 0 }, { "ports", 1, 0, 0 }, { "nb-cores", 1, 0, 0 }, { "nb-ports", 1, 0, 0 }, { "coremask", 1, 0, 0 }, { "portmask", 1, 0, 0 }, { "numa", 0, 0, 0 }, + { "no-numa", 0, 0, 0 }, { "mp-anon", 0, 0, 0 }, { "port-numa-config", 1, 0, 0 }, { "ring-numa-config", 1, 0, 0 }, @@ -524,8 +597,16 @@ launch_args_parse(int argc, char** argv) { "pkt-filter-report-hash", 1, 0, 0 }, { "pkt-filter-size", 1, 0, 0 }, { "pkt-filter-drop-queue", 1, 0, 0 }, - { "crc-strip", 0, 0, 0 }, +#ifdef RTE_LIBRTE_LATENCY_STATS + { "latencystats", 1, 0, 0 }, +#endif +#ifdef RTE_LIBRTE_BITRATE + { "bitrate-stats", 1, 0, 0 }, +#endif + { "disable-crc-strip", 0, 0, 0 }, + { "enable-lro", 0, 0, 0 }, { "enable-rx-cksum", 0, 0, 0 }, + { "enable-rx-timestamp", 0, 0, 0 }, { "enable-scatter", 0, 0, 0 }, { "disable-hw-vlan", 0, 0, 0 }, { "disable-hw-vlan-filter", 0, 0, 0 }, @@ -556,8 +637,13 @@ launch_args_parse(int argc, char** argv) { "tx-queue-stats-mapping", 1, 0, 0 }, { "rx-queue-stats-mapping", 1, 0, 0 }, { "no-flush-rx", 0, 0, 0 }, + { "flow-isolate-all", 0, 0, 0 }, { "txpkts", 1, 0, 0 }, { "disable-link-check", 0, 0, 0 }, + { "no-lsc-interrupt", 0, 0, 0 }, + { "no-rmv-interrupt", 0, 0, 0 }, + { "print-event", 1, 0, 0 }, + { "mask-event", 1, 0, 0 }, { 0, 0, 0, 0 }, }; @@ -592,10 +678,34 @@ launch_args_parse(int argc, char** argv) printf("Interactive-mode selected\n"); interactive = 1; } + if (!strcmp(lgopts[opt_idx].name, "cmdline-file")) { + printf("CLI commands to be read from %s\n", + optarg); + snprintf(cmdline_filename, + sizeof(cmdline_filename), "%s", + optarg); + } if (!strcmp(lgopts[opt_idx].name, "auto-start")) { printf("Auto-start selected\n"); auto_start = 1; } + if (!strcmp(lgopts[opt_idx].name, "tx-first")) { + printf("Ports to start sending a burst of " + "packets first\n"); + tx_first = 1; + } + if (!strcmp(lgopts[opt_idx].name, "stats-period")) { + char *end = NULL; + unsigned int n; + + n = strtoul(optarg, &end, 10); + if ((optarg[0] == '\0') || (end == NULL) || + (*end != '\0')) + break; + + stats_period = n; + break; + } if (!strcmp(lgopts[opt_idx].name, "eth-peers-configfile")) { if (init_peer_eth_addrs(optarg) != 0) @@ -630,7 +740,7 @@ launch_args_parse(int argc, char** argv) if (!strcmp(lgopts[opt_idx].name, "nb-ports")) { n = atoi(optarg); if (n > 0 && n <= nb_ports) - nb_fwd_ports = (uint8_t) n; + nb_fwd_ports = n; else rte_exit(EXIT_FAILURE, "Invalid port %d\n", n); @@ -648,12 +758,10 @@ launch_args_parse(int argc, char** argv) parse_fwd_coremask(optarg); if (!strcmp(lgopts[opt_idx].name, "portmask")) parse_fwd_portmask(optarg); - if (!strcmp(lgopts[opt_idx].name, "numa")) { + if (!strcmp(lgopts[opt_idx].name, "no-numa")) + numa_support = 0; + if (!strcmp(lgopts[opt_idx].name, "numa")) numa_support = 1; - memset(port_numa,NUMA_NO_CONFIG,RTE_MAX_ETHPORTS); - memset(rxring_numa,NUMA_NO_CONFIG,RTE_MAX_ETHPORTS); - memset(txring_numa,NUMA_NO_CONFIG,RTE_MAX_ETHPORTS); - } if (!strcmp(lgopts[opt_idx].name, "mp-anon")) { mp_anon = 1; } @@ -668,12 +776,13 @@ launch_args_parse(int argc, char** argv) "invalid ring-numa configuration\n"); if (!strcmp(lgopts[opt_idx].name, "socket-num")) { n = atoi(optarg); - if((uint8_t)n < max_socket) + if (!new_socket_id((uint8_t)n)) { socket_num = (uint8_t)n; - else + } else { + print_invalid_socket_id_error(); rte_exit(EXIT_FAILURE, - "The socket number should be < %d\n", - max_socket); + "Invalid socket id"); + } } if (!strcmp(lgopts[opt_idx].name, "mbuf-size")) { n = atoi(optarg); @@ -763,12 +872,42 @@ launch_args_parse(int argc, char** argv) "drop queue %d invalid - must" "be >= 0 \n", n); } - if (!strcmp(lgopts[opt_idx].name, "crc-strip")) - rx_mode.hw_strip_crc = 1; +#ifdef RTE_LIBRTE_LATENCY_STATS + if (!strcmp(lgopts[opt_idx].name, + "latencystats")) { + n = atoi(optarg); + if (n >= 0) { + latencystats_lcore_id = (lcoreid_t) n; + latencystats_enabled = 1; + } else + rte_exit(EXIT_FAILURE, + "invalid lcore id %d for latencystats" + " must be >= 0\n", n); + } +#endif +#ifdef RTE_LIBRTE_BITRATE + if (!strcmp(lgopts[opt_idx].name, "bitrate-stats")) { + n = atoi(optarg); + if (n >= 0) { + bitrate_lcore_id = (lcoreid_t) n; + bitrate_enabled = 1; + } else + rte_exit(EXIT_FAILURE, + "invalid lcore id %d for bitrate stats" + " must be >= 0\n", n); + } +#endif + if (!strcmp(lgopts[opt_idx].name, "disable-crc-strip")) + rx_mode.hw_strip_crc = 0; + if (!strcmp(lgopts[opt_idx].name, "enable-lro")) + rx_mode.enable_lro = 1; if (!strcmp(lgopts[opt_idx].name, "enable-scatter")) rx_mode.enable_scatter = 1; if (!strcmp(lgopts[opt_idx].name, "enable-rx-cksum")) rx_mode.hw_ip_checksum = 1; + if (!strcmp(lgopts[opt_idx].name, + "enable-rx-timestamp")) + rx_mode.hw_timestamp = 1; if (!strcmp(lgopts[opt_idx].name, "disable-hw-vlan")) { rx_mode.hw_vlan_filter = 0; @@ -802,7 +941,7 @@ launch_args_parse(int argc, char** argv) port_topology = PORT_TOPOLOGY_LOOP; else rte_exit(EXIT_FAILURE, "port-topology %s invalid -" - " must be: paired or chained \n", + " must be: paired, chained or loop\n", optarg); } if (!strcmp(lgopts[opt_idx].name, "forward-mode")) @@ -813,21 +952,21 @@ launch_args_parse(int argc, char** argv) rss_hf = ETH_RSS_UDP; if (!strcmp(lgopts[opt_idx].name, "rxq")) { n = atoi(optarg); - if (n >= 0 && n <= (int) MAX_QUEUE_ID) + if (n >= 0 && check_nb_rxq((queueid_t)n) == 0) nb_rxq = (queueid_t) n; else rte_exit(EXIT_FAILURE, "rxq %d invalid - must be" - " >= 0 && <= %d\n", n, - (int) MAX_QUEUE_ID); + " >= 0 && <= %u\n", n, + get_allowed_max_nb_rxq(&pid)); } if (!strcmp(lgopts[opt_idx].name, "txq")) { n = atoi(optarg); - if (n >= 0 && n <= (int) MAX_QUEUE_ID) + if (n >= 0 && check_nb_txq((queueid_t)n) == 0) nb_txq = (queueid_t) n; else rte_exit(EXIT_FAILURE, "txq %d invalid - must be" - " >= 0 && <= %d\n", n, - (int) MAX_QUEUE_ID); + " >= 0 && <= %u\n", n, + get_allowed_max_nb_txq(&pid)); } if (!nb_rxq && !nb_txq) { rte_exit(EXIT_FAILURE, "Either rx or tx queues should " @@ -973,6 +1112,22 @@ launch_args_parse(int argc, char** argv) no_flush_rx = 1; if (!strcmp(lgopts[opt_idx].name, "disable-link-check")) no_link_check = 1; + if (!strcmp(lgopts[opt_idx].name, "no-lsc-interrupt")) + lsc_interrupt = 0; + if (!strcmp(lgopts[opt_idx].name, "no-rmv-interrupt")) + rmv_interrupt = 0; + if (!strcmp(lgopts[opt_idx].name, "flow-isolate-all")) + flow_isolate_all = 1; + if (!strcmp(lgopts[opt_idx].name, "print-event")) + if (parse_event_printing_config(optarg, 1)) { + rte_exit(EXIT_FAILURE, + "invalid print-event argument\n"); + } + if (!strcmp(lgopts[opt_idx].name, "mask-event")) + if (parse_event_printing_config(optarg, 0)) { + rte_exit(EXIT_FAILURE, + "invalid mask-event argument\n"); + } break; case 'h': diff --git a/dpdk/app/test-pmd/rxonly.c b/dpdk/app/test-pmd/rxonly.c index fbf287dc..fb6e8e33 100644 --- a/dpdk/app/test-pmd/rxonly.c +++ b/dpdk/app/test-pmd/rxonly.c @@ -49,15 +49,12 @@ #include #include #include -#include #include #include #include #include #include #include -#include -#include #include #include #include @@ -67,6 +64,8 @@ #include #include #include +#include +#include #include "testpmd.h" @@ -92,14 +91,15 @@ pkt_burst_receive(struct fwd_stream *fs) uint16_t nb_rx; uint16_t i, packet_type; uint16_t is_encapsulation; + char buf[256]; + struct rte_net_hdr_lens hdr_lens; + uint32_t sw_packet_type; #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES uint64_t start_tsc; uint64_t end_tsc; uint64_t core_cycles; -#endif -#ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES start_tsc = rte_rdtsc(); #endif @@ -121,7 +121,7 @@ pkt_burst_receive(struct fwd_stream *fs) */ if (verbose_level > 0) printf("port %u/queue %u: received %u packets\n", - (unsigned) fs->rx_port, + fs->rx_port, (unsigned) fs->rx_queue, (unsigned) nb_rx); for (i = 0; i < nb_rx; i++) { @@ -144,7 +144,8 @@ pkt_burst_receive(struct fwd_stream *fs) if (ol_flags & PKT_RX_RSS_HASH) { printf(" - RSS hash=0x%x", (unsigned) mb->hash.rss); printf(" - RSS queue=0x%x",(unsigned) fs->rx_queue); - } else if (ol_flags & PKT_RX_FDIR) { + } + if (ol_flags & PKT_RX_FDIR) { printf(" - FDIR matched "); if (ol_flags & PKT_RX_FDIR_ID) printf("ID=0x%x", @@ -156,183 +157,35 @@ pkt_burst_receive(struct fwd_stream *fs) printf("hash=0x%x ID=0x%x ", mb->hash.fdir.hash, mb->hash.fdir.id); } + if (ol_flags & PKT_RX_TIMESTAMP) + printf(" - timestamp %"PRIu64" ", mb->timestamp); if (ol_flags & PKT_RX_VLAN_STRIPPED) printf(" - VLAN tci=0x%x", mb->vlan_tci); if (ol_flags & PKT_RX_QINQ_STRIPPED) printf(" - QinQ VLAN tci=0x%x, VLAN tci outer=0x%x", mb->vlan_tci, mb->vlan_tci_outer); if (mb->packet_type) { - uint32_t ptype; - - /* (outer) L2 packet type */ - ptype = mb->packet_type & RTE_PTYPE_L2_MASK; - switch (ptype) { - case RTE_PTYPE_L2_ETHER: - printf(" - (outer) L2 type: ETHER"); - break; - case RTE_PTYPE_L2_ETHER_TIMESYNC: - printf(" - (outer) L2 type: ETHER_Timesync"); - break; - case RTE_PTYPE_L2_ETHER_ARP: - printf(" - (outer) L2 type: ETHER_ARP"); - break; - case RTE_PTYPE_L2_ETHER_LLDP: - printf(" - (outer) L2 type: ETHER_LLDP"); - break; - case RTE_PTYPE_L2_ETHER_NSH: - printf(" - (outer) L2 type: ETHER_NSH"); - break; - default: - printf(" - (outer) L2 type: Unknown"); - break; - } - - /* (outer) L3 packet type */ - ptype = mb->packet_type & RTE_PTYPE_L3_MASK; - switch (ptype) { - case RTE_PTYPE_L3_IPV4: - printf(" - (outer) L3 type: IPV4"); - break; - case RTE_PTYPE_L3_IPV4_EXT: - printf(" - (outer) L3 type: IPV4_EXT"); - break; - case RTE_PTYPE_L3_IPV6: - printf(" - (outer) L3 type: IPV6"); - break; - case RTE_PTYPE_L3_IPV4_EXT_UNKNOWN: - printf(" - (outer) L3 type: IPV4_EXT_UNKNOWN"); - break; - case RTE_PTYPE_L3_IPV6_EXT: - printf(" - (outer) L3 type: IPV6_EXT"); - break; - case RTE_PTYPE_L3_IPV6_EXT_UNKNOWN: - printf(" - (outer) L3 type: IPV6_EXT_UNKNOWN"); - break; - default: - printf(" - (outer) L3 type: Unknown"); - break; - } - - /* (outer) L4 packet type */ - ptype = mb->packet_type & RTE_PTYPE_L4_MASK; - switch (ptype) { - case RTE_PTYPE_L4_TCP: - printf(" - (outer) L4 type: TCP"); - break; - case RTE_PTYPE_L4_UDP: - printf(" - (outer) L4 type: UDP"); - break; - case RTE_PTYPE_L4_FRAG: - printf(" - (outer) L4 type: L4_FRAG"); - break; - case RTE_PTYPE_L4_SCTP: - printf(" - (outer) L4 type: SCTP"); - break; - case RTE_PTYPE_L4_ICMP: - printf(" - (outer) L4 type: ICMP"); - break; - case RTE_PTYPE_L4_NONFRAG: - printf(" - (outer) L4 type: L4_NONFRAG"); - break; - default: - printf(" - (outer) L4 type: Unknown"); - break; - } - - /* packet tunnel type */ - ptype = mb->packet_type & RTE_PTYPE_TUNNEL_MASK; - switch (ptype) { - case RTE_PTYPE_TUNNEL_IP: - printf(" - Tunnel type: IP"); - break; - case RTE_PTYPE_TUNNEL_GRE: - printf(" - Tunnel type: GRE"); - break; - case RTE_PTYPE_TUNNEL_VXLAN: - printf(" - Tunnel type: VXLAN"); - break; - case RTE_PTYPE_TUNNEL_NVGRE: - printf(" - Tunnel type: NVGRE"); - break; - case RTE_PTYPE_TUNNEL_GENEVE: - printf(" - Tunnel type: GENEVE"); - break; - case RTE_PTYPE_TUNNEL_GRENAT: - printf(" - Tunnel type: GRENAT"); - break; - default: - printf(" - Tunnel type: Unknown"); - break; - } - - /* inner L2 packet type */ - ptype = mb->packet_type & RTE_PTYPE_INNER_L2_MASK; - switch (ptype) { - case RTE_PTYPE_INNER_L2_ETHER: - printf(" - Inner L2 type: ETHER"); - break; - case RTE_PTYPE_INNER_L2_ETHER_VLAN: - printf(" - Inner L2 type: ETHER_VLAN"); - break; - default: - printf(" - Inner L2 type: Unknown"); - break; - } - - /* inner L3 packet type */ - ptype = mb->packet_type & RTE_PTYPE_INNER_L3_MASK; - switch (ptype) { - case RTE_PTYPE_INNER_L3_IPV4: - printf(" - Inner L3 type: IPV4"); - break; - case RTE_PTYPE_INNER_L3_IPV4_EXT: - printf(" - Inner L3 type: IPV4_EXT"); - break; - case RTE_PTYPE_INNER_L3_IPV6: - printf(" - Inner L3 type: IPV6"); - break; - case RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN: - printf(" - Inner L3 type: IPV4_EXT_UNKNOWN"); - break; - case RTE_PTYPE_INNER_L3_IPV6_EXT: - printf(" - Inner L3 type: IPV6_EXT"); - break; - case RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN: - printf(" - Inner L3 type: IPV6_EXT_UNKNOWN"); - break; - default: - printf(" - Inner L3 type: Unknown"); - break; - } - - /* inner L4 packet type */ - ptype = mb->packet_type & RTE_PTYPE_INNER_L4_MASK; - switch (ptype) { - case RTE_PTYPE_INNER_L4_TCP: - printf(" - Inner L4 type: TCP"); - break; - case RTE_PTYPE_INNER_L4_UDP: - printf(" - Inner L4 type: UDP"); - break; - case RTE_PTYPE_INNER_L4_FRAG: - printf(" - Inner L4 type: L4_FRAG"); - break; - case RTE_PTYPE_INNER_L4_SCTP: - printf(" - Inner L4 type: SCTP"); - break; - case RTE_PTYPE_INNER_L4_ICMP: - printf(" - Inner L4 type: ICMP"); - break; - case RTE_PTYPE_INNER_L4_NONFRAG: - printf(" - Inner L4 type: L4_NONFRAG"); - break; - default: - printf(" - Inner L4 type: Unknown"); - break; - } - printf("\n"); - } else - printf("Unknown packet type\n"); + rte_get_ptype_name(mb->packet_type, buf, sizeof(buf)); + printf(" - hw ptype: %s", buf); + } + sw_packet_type = rte_net_get_ptype(mb, &hdr_lens, + RTE_PTYPE_ALL_MASK); + rte_get_ptype_name(sw_packet_type, buf, sizeof(buf)); + printf(" - sw ptype: %s", buf); + if (sw_packet_type & RTE_PTYPE_L2_MASK) + printf(" - l2_len=%d", hdr_lens.l2_len); + if (sw_packet_type & RTE_PTYPE_L3_MASK) + printf(" - l3_len=%d", hdr_lens.l3_len); + if (sw_packet_type & RTE_PTYPE_L4_MASK) + printf(" - l4_len=%d", hdr_lens.l4_len); + if (sw_packet_type & RTE_PTYPE_TUNNEL_MASK) + printf(" - tunnel_len=%d", hdr_lens.tunnel_len); + if (sw_packet_type & RTE_PTYPE_INNER_L2_MASK) + printf(" - inner_l2_len=%d", hdr_lens.inner_l2_len); + if (sw_packet_type & RTE_PTYPE_INNER_L3_MASK) + printf(" - inner_l3_len=%d", hdr_lens.inner_l3_len); + if (sw_packet_type & RTE_PTYPE_INNER_L4_MASK) + printf(" - inner_l4_len=%d", hdr_lens.inner_l4_len); if (is_encapsulation) { struct ipv4_hdr *ipv4_hdr; struct ipv6_hdr *ipv6_hdr; @@ -376,19 +229,8 @@ pkt_burst_receive(struct fwd_stream *fs) } printf(" - Receive queue=0x%x", (unsigned) fs->rx_queue); printf("\n"); - if (ol_flags != 0) { - unsigned rxf; - const char *name; - - for (rxf = 0; rxf < sizeof(mb->ol_flags) * 8; rxf++) { - if ((ol_flags & (1ULL << rxf)) == 0) - continue; - name = rte_get_rx_ol_flag_name(1ULL << rxf); - if (name == NULL) - continue; - printf(" %s\n", name); - } - } + rte_get_rx_ol_flag_list(mb->ol_flags, buf, sizeof(buf)); + printf(" ol_flags: %s\n", buf); rte_pktmbuf_free(mb); } diff --git a/dpdk/app/test-pmd/testpmd.c b/dpdk/app/test-pmd/testpmd.c index 8d0905eb..f66f4c64 100644 --- a/dpdk/app/test-pmd/testpmd.c +++ b/dpdk/app/test-pmd/testpmd.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. + * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -38,6 +38,7 @@ #include #include #include +#include #include #include @@ -56,14 +57,13 @@ #include #include #include -#include #include #include +#include #include #include #include #include -#include #include #include #include @@ -73,12 +73,20 @@ #include #include #include -#ifdef RTE_LIBRTE_PMD_XENVIRT -#include +#ifdef RTE_LIBRTE_IXGBE_PMD +#include #endif #ifdef RTE_LIBRTE_PDUMP #include #endif +#include +#include +#ifdef RTE_LIBRTE_BITRATE +#include +#endif +#ifdef RTE_LIBRTE_LATENCY_STATS +#include +#endif #include "testpmd.h" @@ -87,6 +95,8 @@ uint16_t verbose_level = 0; /**< Silent by default. */ /* use master core for command line ? */ uint8_t interactive = 0; uint8_t auto_start = 0; +uint8_t tx_first; +char cmdline_filename[PATH_MAX] = {0}; /* * NUMA support configuration. @@ -95,7 +105,7 @@ uint8_t auto_start = 0; * probed ports among the CPU sockets 0 and 1. * Otherwise, all memory is allocated from CPU socket 0. */ -uint8_t numa_support = 0; /**< No numa support by default */ +uint8_t numa_support = 1; /**< numa enabled by default */ /* * In UMA mode,all memory is allocated from socket 0 if --socket-num is @@ -111,7 +121,7 @@ uint8_t mp_anon = 0; /* * Record the Ethernet address of peer target ports to which packets are * forwarded. - * Must be instanciated with the ethernet addresses of peer traffic generator + * Must be instantiated with the ethernet addresses of peer traffic generator * ports. */ struct ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; @@ -153,6 +163,10 @@ struct fwd_engine * fwd_engines[] = { &tx_only_engine, &csum_fwd_engine, &icmp_echo_engine, +#if defined RTE_LIBRTE_PMD_SOFTNIC && defined RTE_LIBRTE_SCHED + &softnic_tm_engine, + &softnic_tm_bypass_engine, +#endif #ifdef RTE_LIBRTE_IEEE1588 &ieee1588_fwd_engine, #endif @@ -168,6 +182,13 @@ uint32_t burst_tx_retry_num = BURST_TX_RETRIES; uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */ uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if * specified on command-line. */ +uint16_t stats_period; /**< Period to show statistics (disabled by default) */ + +/* + * In container, it cannot terminate the process which running with 'stats-period' + * option. Set flag to exit stats period loop after received SIGINT/SIGTERM. + */ +uint8_t f_quit; /* * Configuration of packet segments used by the "txonly" processing engine. @@ -257,18 +278,58 @@ uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ */ uint8_t no_flush_rx = 0; /* flush by default */ +/* + * Flow API isolated mode. + */ +uint8_t flow_isolate_all; + /* * Avoids to check link status when starting/stopping a port. */ uint8_t no_link_check = 0; /* check by default */ +/* + * Enable link status change notification + */ +uint8_t lsc_interrupt = 1; /* enabled by default */ + +/* + * Enable device removal notification. + */ +uint8_t rmv_interrupt = 1; /* enabled by default */ + +/* + * Display or mask ether events + * Default to all events except VF_MBOX + */ +uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | + (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | + (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | + (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | + (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | + (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV); + /* * NIC bypass mode configuration options. */ -#ifdef RTE_NIC_BYPASS +#if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS /* The NIC bypass watchdog timeout. */ -uint32_t bypass_timeout = RTE_BYPASS_TMT_OFF; +uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF; +#endif + + +#ifdef RTE_LIBRTE_LATENCY_STATS + +/* + * Set when latency stats is enabled in the commandline + */ +uint8_t latencystats_enabled; + +/* + * Lcore ID to serive latency statistics. + */ +lcoreid_t latencystats_lcore_id = -1; #endif @@ -284,7 +345,8 @@ struct rte_eth_rxmode rx_mode = { .hw_vlan_strip = 1, /**< VLAN strip enabled. */ .hw_vlan_extend = 0, /**< Extended VLAN disabled. */ .jumbo_frame = 0, /**< Jumbo Frame Support disabled. */ - .hw_strip_crc = 0, /**< CRC stripping by hardware disabled. */ + .hw_strip_crc = 1, /**< CRC stripping by hardware enabled. */ + .hw_timestamp = 0, /**< HW timestamp enabled. */ }; struct rte_fdir_conf fdir_conf = { @@ -321,11 +383,31 @@ struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_a uint16_t nb_tx_queue_stats_mappings = 0; uint16_t nb_rx_queue_stats_mappings = 0; -unsigned max_socket = 0; +/* + * Display zero values by default for xstats + */ +uint8_t xstats_hide_zero; + +unsigned int num_sockets = 0; +unsigned int socket_ids[RTE_MAX_NUMA_NODES]; + +#ifdef RTE_LIBRTE_BITRATE +/* Bitrate statistics */ +struct rte_stats_bitrates *bitrate_data; +lcoreid_t bitrate_lcore_id; +uint8_t bitrate_enabled; +#endif + +struct gro_status gro_ports[RTE_MAX_ETHPORTS]; +uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES; /* Forward function declarations */ -static void map_port_queue_stats_mapping_registers(uint8_t pi, struct rte_port *port); +static void map_port_queue_stats_mapping_registers(portid_t pi, + struct rte_port *port); static void check_all_ports_link_status(uint32_t port_mask); +static int eth_event_callback(portid_t port_id, + enum rte_eth_event_type type, + void *param, void *ret_param); /* * Check if all the ports are started. @@ -333,18 +415,23 @@ static void check_all_ports_link_status(uint32_t port_mask); */ static int all_ports_started(void); -/* - * Find next enabled port - */ -portid_t -find_next_port(portid_t p, struct rte_port *ports, int size) -{ - if (ports == NULL) - rte_exit(-EINVAL, "failed to find a next port id\n"); +struct gso_status gso_ports[RTE_MAX_ETHPORTS]; +uint16_t gso_max_segment_size = ETHER_MAX_LEN - ETHER_CRC_LEN; - while ((p < size) && (ports[p].enabled == 0)) - p++; - return p; +/* + * Helper function to check if socket is already discovered. + * If yes, return positive value. If not, return zero. + */ +int +new_socket_id(unsigned int socket_id) +{ + unsigned int i; + + for (i = 0; i < num_sockets; i++) { + if (socket_ids[i] == socket_id) + return 0; + } + return 1; } /* @@ -359,11 +446,14 @@ set_default_fwd_lcores_config(void) nb_lc = 0; for (i = 0; i < RTE_MAX_LCORE; i++) { - sock_num = rte_lcore_to_socket_id(i) + 1; - if (sock_num > max_socket) { - if (sock_num > RTE_MAX_NUMA_NODES) - rte_exit(EXIT_FAILURE, "Total sockets greater than %u\n", RTE_MAX_NUMA_NODES); - max_socket = sock_num; + sock_num = rte_lcore_to_socket_id(i); + if (new_socket_id(sock_num)) { + if (num_sockets >= RTE_MAX_NUMA_NODES) { + rte_exit(EXIT_FAILURE, + "Total sockets greater than %u\n", + RTE_MAX_NUMA_NODES); + } + socket_ids[num_sockets++] = sock_num; } if (!rte_lcore_is_enabled(i)) continue; @@ -391,9 +481,10 @@ static void set_default_fwd_ports_config(void) { portid_t pt_id; + int i = 0; - for (pt_id = 0; pt_id < nb_ports; pt_id++) - fwd_ports_ids[pt_id] = pt_id; + RTE_ETH_FOREACH_DEV(pt_id) + fwd_ports_ids[i++] = pt_id; nb_cfg_ports = nb_ports; nb_fwd_ports = nb_ports; @@ -425,37 +516,25 @@ mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", pool_name, nb_mbuf, mbuf_seg_size, socket_id); -#ifdef RTE_LIBRTE_PMD_XENVIRT - rte_mp = rte_mempool_gntalloc_create(pool_name, nb_mbuf, mb_size, - (unsigned) mb_mempool_cache, - sizeof(struct rte_pktmbuf_pool_private), - rte_pktmbuf_pool_init, NULL, - rte_pktmbuf_init, NULL, - socket_id, 0); -#endif + if (mp_anon != 0) { + rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, + mb_size, (unsigned) mb_mempool_cache, + sizeof(struct rte_pktmbuf_pool_private), + socket_id, 0); + if (rte_mp == NULL) + goto err; - /* if the former XEN allocation failed fall back to normal allocation */ - if (rte_mp == NULL) { - if (mp_anon != 0) { - rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, - mb_size, (unsigned) mb_mempool_cache, - sizeof(struct rte_pktmbuf_pool_private), - socket_id, 0); - if (rte_mp == NULL) - goto err; - - if (rte_mempool_populate_anon(rte_mp) == 0) { - rte_mempool_free(rte_mp); - rte_mp = NULL; - goto err; - } - rte_pktmbuf_pool_init(rte_mp, NULL); - rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); - } else { - /* wrapper to rte_mempool_create() */ - rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, - mb_mempool_cache, 0, mbuf_seg_size, socket_id); + if (rte_mempool_populate_anon(rte_mp) == 0) { + rte_mempool_free(rte_mp); + rte_mp = NULL; + goto err; } + rte_pktmbuf_pool_init(rte_mp, NULL); + rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); + } else { + /* wrapper to rte_mempool_create() */ + rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, + mb_mempool_cache, 0, mbuf_seg_size, socket_id); } err: @@ -477,7 +556,7 @@ check_socket_id(const unsigned int socket_id) { static int warning_once = 0; - if (socket_id >= max_socket) { + if (new_socket_id(socket_id)) { if (!warning_once && numa_support) printf("Warning: NUMA should be configured manually by" " using --port-numa-config and" @@ -489,6 +568,98 @@ check_socket_id(const unsigned int socket_id) return 0; } +/* + * Get the allowed maximum number of RX queues. + * *pid return the port id which has minimal value of + * max_rx_queues in all ports. + */ +queueid_t +get_allowed_max_nb_rxq(portid_t *pid) +{ + queueid_t allowed_max_rxq = MAX_QUEUE_ID; + portid_t pi; + struct rte_eth_dev_info dev_info; + + RTE_ETH_FOREACH_DEV(pi) { + rte_eth_dev_info_get(pi, &dev_info); + if (dev_info.max_rx_queues < allowed_max_rxq) { + allowed_max_rxq = dev_info.max_rx_queues; + *pid = pi; + } + } + return allowed_max_rxq; +} + +/* + * Check input rxq is valid or not. + * If input rxq is not greater than any of maximum number + * of RX queues of all ports, it is valid. + * if valid, return 0, else return -1 + */ +int +check_nb_rxq(queueid_t rxq) +{ + queueid_t allowed_max_rxq; + portid_t pid = 0; + + allowed_max_rxq = get_allowed_max_nb_rxq(&pid); + if (rxq > allowed_max_rxq) { + printf("Fail: input rxq (%u) can't be greater " + "than max_rx_queues (%u) of port %u\n", + rxq, + allowed_max_rxq, + pid); + return -1; + } + return 0; +} + +/* + * Get the allowed maximum number of TX queues. + * *pid return the port id which has minimal value of + * max_tx_queues in all ports. + */ +queueid_t +get_allowed_max_nb_txq(portid_t *pid) +{ + queueid_t allowed_max_txq = MAX_QUEUE_ID; + portid_t pi; + struct rte_eth_dev_info dev_info; + + RTE_ETH_FOREACH_DEV(pi) { + rte_eth_dev_info_get(pi, &dev_info); + if (dev_info.max_tx_queues < allowed_max_txq) { + allowed_max_txq = dev_info.max_tx_queues; + *pid = pi; + } + } + return allowed_max_txq; +} + +/* + * Check input txq is valid or not. + * If input txq is not greater than any of maximum number + * of TX queues of all ports, it is valid. + * if valid, return 0, else return -1 + */ +int +check_nb_txq(queueid_t txq) +{ + queueid_t allowed_max_txq; + portid_t pid = 0; + + allowed_max_txq = get_allowed_max_nb_txq(&pid); + if (txq > allowed_max_txq) { + printf("Fail: input txq (%u) can't be greater " + "than max_tx_queues (%u) of port %u\n", + txq, + allowed_max_txq, + pid); + return -1; + } + return 0; +} + static void init_config(void) { @@ -498,8 +669,17 @@ init_config(void) unsigned int nb_mbuf_per_pool; lcoreid_t lc_id; uint8_t port_per_socket[RTE_MAX_NUMA_NODES]; + struct rte_gro_param gro_param; + uint32_t gso_types; memset(port_per_socket,0,RTE_MAX_NUMA_NODES); + + if (numa_support) { + memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); + memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); + memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); + } + /* Configuration of logical cores. */ fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", sizeof(struct fwd_lcore *) * nb_lcores, @@ -519,35 +699,7 @@ init_config(void) fwd_lcores[lc_id]->cpuid_idx = lc_id; } - /* - * Create pools of mbuf. - * If NUMA support is disabled, create a single pool of mbuf in - * socket 0 memory by default. - * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. - * - * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and - * nb_txd can be configured at run time. - */ - if (param_total_num_mbufs) - nb_mbuf_per_pool = param_total_num_mbufs; - else { - nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX + (nb_lcores * mb_mempool_cache) - + RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST; - - if (!numa_support) - nb_mbuf_per_pool = - (nb_mbuf_per_pool * RTE_MAX_ETHPORTS); - } - - if (!numa_support) { - if (socket_num == UMA_NO_CONFIG) - mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool, 0); - else - mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool, - socket_num); - } - - FOREACH_PORT(pid, ports) { + RTE_ETH_FOREACH_DEV(pid) { port = &ports[pid]; rte_eth_dev_info_get(pid, &port->dev_info); @@ -569,22 +721,42 @@ init_config(void) port->need_reconfig_queues = 1; } + /* + * Create pools of mbuf. + * If NUMA support is disabled, create a single pool of mbuf in + * socket 0 memory by default. + * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. + * + * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and + * nb_txd can be configured at run time. + */ + if (param_total_num_mbufs) + nb_mbuf_per_pool = param_total_num_mbufs; + else { + nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX + + (nb_lcores * mb_mempool_cache) + + RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST; + nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; + } + if (numa_support) { uint8_t i; - unsigned int nb_mbuf; - if (param_total_num_mbufs) - nb_mbuf_per_pool = nb_mbuf_per_pool/nb_ports; - - for (i = 0; i < max_socket; i++) { - nb_mbuf = (nb_mbuf_per_pool * RTE_MAX_ETHPORTS); - if (nb_mbuf) - mbuf_pool_create(mbuf_data_size, - nb_mbuf,i); - } + for (i = 0; i < num_sockets; i++) + mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool, + socket_ids[i]); + } else { + if (socket_num == UMA_NO_CONFIG) + mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool, 0); + else + mbuf_pool_create(mbuf_data_size, nb_mbuf_per_pool, + socket_num); } + init_port_config(); + gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO | + DEV_TX_OFFLOAD_GRE_TNL_TSO; /* * Records which Mbuf pool to use by each logical core, if needed. */ @@ -595,6 +767,13 @@ init_config(void) if (mbp == NULL) mbp = mbuf_pool_find(0); fwd_lcores[lc_id]->mbp = mbp; + /* initialize GSO context */ + fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp; + fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp; + fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types; + fwd_lcores[lc_id]->gso_ctx.gso_size = ETHER_MAX_LEN - + ETHER_CRC_LEN; + fwd_lcores[lc_id]->gso_ctx.flag = 0; } /* Configuration of packet forwarding streams. */ @@ -602,6 +781,20 @@ init_config(void) rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n"); fwd_config_setup(); + + /* create a gro context for each lcore */ + gro_param.gro_types = RTE_GRO_TCP_IPV4; + gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES; + gro_param.max_item_per_flow = MAX_PKT_BURST; + for (lc_id = 0; lc_id < nb_lcores; lc_id++) { + gro_param.socket_id = rte_lcore_to_socket_id( + fwd_lcores_cpuids[lc_id]); + fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param); + if (fwd_lcores[lc_id]->gro_ctx == NULL) { + rte_exit(EXIT_FAILURE, + "rte_gro_ctx_create() failed\n"); + } + } } @@ -632,7 +825,7 @@ init_fwd_streams(void) queueid_t q; /* set socket id according to numa or not */ - FOREACH_PORT(pid, ports) { + RTE_ETH_FOREACH_DEV(pid) { port = &ports[pid]; if (nb_rxq > port->dev_info.max_rx_queues) { printf("Fail: nb_rxq(%d) is greater than " @@ -922,12 +1115,42 @@ run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) struct fwd_stream **fsm; streamid_t nb_fs; streamid_t sm_id; +#ifdef RTE_LIBRTE_BITRATE + uint64_t tics_per_1sec; + uint64_t tics_datum; + uint64_t tics_current; + uint8_t idx_port, cnt_ports; + cnt_ports = rte_eth_dev_count(); + tics_datum = rte_rdtsc(); + tics_per_1sec = rte_get_timer_hz(); +#endif fsm = &fwd_streams[fc->stream_idx]; nb_fs = fc->stream_nb; do { for (sm_id = 0; sm_id < nb_fs; sm_id++) (*pkt_fwd)(fsm[sm_id]); +#ifdef RTE_LIBRTE_BITRATE + if (bitrate_enabled != 0 && + bitrate_lcore_id == rte_lcore_id()) { + tics_current = rte_rdtsc(); + if (tics_current - tics_datum >= tics_per_1sec) { + /* Periodic bitrate calculation */ + for (idx_port = 0; + idx_port < cnt_ports; + idx_port++) + rte_stats_bitrate_calc(bitrate_data, + idx_port); + tics_datum = tics_current; + } + } +#endif +#ifdef RTE_LIBRTE_LATENCY_STATS + if (latencystats_enabled != 0 && + latencystats_lcore_id == rte_lcore_id()) + rte_latencystats_update(); +#endif + } while (! fc->stopped); } @@ -1118,6 +1341,7 @@ stop_packet_forwarding(void) #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES uint64_t fwd_cycles; #endif + static const char *acc_stats_border = "+++++++++++++++"; if (test_done) { @@ -1208,6 +1432,7 @@ stop_packet_forwarding(void) fwd_port_stats_display(pt_id, &stats); } + printf("\n %s Accumulated forward statistics for all ports" "%s\n", acc_stats_border, acc_stats_border); @@ -1236,14 +1461,14 @@ stop_packet_forwarding(void) void dev_set_link_up(portid_t pid) { - if (rte_eth_dev_set_link_up((uint8_t)pid) < 0) + if (rte_eth_dev_set_link_up(pid) < 0) printf("\nSet link up fail.\n"); } void dev_set_link_down(portid_t pid) { - if (rte_eth_dev_set_link_down((uint8_t)pid) < 0) + if (rte_eth_dev_set_link_down(pid) < 0) printf("\nSet link down fail.\n"); } @@ -1253,7 +1478,7 @@ all_ports_started(void) portid_t pi; struct rte_port *port; - FOREACH_PORT(pi, ports) { + RTE_ETH_FOREACH_DEV(pi) { port = &ports[pi]; /* Check if there is a port which is not started */ if ((port->port_status != RTE_PORT_STARTED) && @@ -1271,7 +1496,7 @@ all_ports_stopped(void) portid_t pi; struct rte_port *port; - FOREACH_PORT(pi, ports) { + RTE_ETH_FOREACH_DEV(pi) { port = &ports[pi]; if ((port->port_status != RTE_PORT_STOPPED) && (port->slave_flag == 0)) @@ -1313,13 +1538,14 @@ start_port(portid_t pid) queueid_t qi; struct rte_port *port; struct ether_addr mac_addr; + enum rte_eth_event_type event_type; if (port_id_is_invalid(pid, ENABLED_WARN)) return 0; if(dcb_config) dcb_test = 1; - FOREACH_PORT(pi, ports) { + RTE_ETH_FOREACH_DEV(pi) { if (pid != pi && pid != (portid_t)RTE_PORT_ALL) continue; @@ -1334,6 +1560,15 @@ start_port(portid_t pid) if (port->need_reconfig > 0) { port->need_reconfig = 0; + if (flow_isolate_all) { + int ret = port_flow_isolate(pi, 1); + if (ret) { + printf("Failed to apply isolated" + " mode on port %d\n", pi); + return -1; + } + } + printf("Configuring Port %d (socket %u)\n", pi, port->socket_id); /* configure port */ @@ -1424,6 +1659,21 @@ start_port(portid_t pid) return -1; } } + + for (event_type = RTE_ETH_EVENT_UNKNOWN; + event_type < RTE_ETH_EVENT_MAX; + event_type++) { + diag = rte_eth_dev_callback_register(pi, + event_type, + eth_event_callback, + NULL); + if (diag) { + printf("Failed to setup even callback for event %d\n", + event_type); + return -1; + } + } + /* start port */ if (rte_eth_dev_start(pi) < 0) { printf("Fail to start port %d\n", pi); @@ -1476,7 +1726,7 @@ stop_port(portid_t pid) printf("Stopping ports...\n"); - FOREACH_PORT(pi, ports) { + RTE_ETH_FOREACH_DEV(pi) { if (pid != pi && pid != (portid_t)RTE_PORT_ALL) continue; @@ -1519,7 +1769,7 @@ close_port(portid_t pid) printf("Closing ports...\n"); - FOREACH_PORT(pi, ports) { + RTE_ETH_FOREACH_DEV(pi) { if (pid != pi && pid != (portid_t)RTE_PORT_ALL) continue; @@ -1546,6 +1796,8 @@ close_port(portid_t pid) continue; } + if (port->flow_list) + port_flow_flush(pi); rte_eth_dev_close(pi); if (rte_atomic16_cmpset(&(port->port_status), @@ -1556,6 +1808,47 @@ close_port(portid_t pid) printf("Done\n"); } +void +reset_port(portid_t pid) +{ + int diag; + portid_t pi; + struct rte_port *port; + + if (port_id_is_invalid(pid, ENABLED_WARN)) + return; + + printf("Resetting ports...\n"); + + RTE_ETH_FOREACH_DEV(pi) { + if (pid != pi && pid != (portid_t)RTE_PORT_ALL) + continue; + + if (port_is_forwarding(pi) != 0 && test_done == 0) { + printf("Please remove port %d from forwarding " + "configuration.\n", pi); + continue; + } + + if (port_is_bonding_slave(pi)) { + printf("Please remove port %d from bonded device.\n", + pi); + continue; + } + + diag = rte_eth_dev_reset(pi); + if (diag == 0) { + port = &ports[pi]; + port->need_reconfig = 1; + port->need_reconfig_queues = 1; + } else { + printf("Failed to reset port %d. diag=%d\n", pi, diag); + } + } + + printf("Done\n"); +} + void attach_port(char *identifier) { @@ -1572,7 +1865,6 @@ attach_port(char *identifier) if (rte_eth_dev_attach(identifier, &pi)) return; - ports[pi].enabled = 1; socket_id = (unsigned)rte_eth_dev_socket_id(pi); /* if socket_id is invalid, set to 0 */ if (check_socket_id(socket_id) < 0) @@ -1589,7 +1881,7 @@ attach_port(char *identifier) } void -detach_port(uint8_t port_id) +detach_port(portid_t port_id) { char name[RTE_ETH_NAME_MAX_LEN]; @@ -1600,10 +1892,14 @@ detach_port(uint8_t port_id) return; } - if (rte_eth_dev_detach(port_id, name)) - return; + if (ports[port_id].flow_list) + port_flow_flush(port_id); + + if (rte_eth_dev_detach(port_id, name)) { + RTE_LOG(ERR, USER1, "Failed to detach port '%s'\n", name); + return; + } - ports[port_id].enabled = 0; nb_ports = rte_eth_dev_count(); printf("Port '%s' is detached. Now total ports is %d\n", @@ -1622,7 +1918,7 @@ pmd_test_exit(void) if (ports != NULL) { no_link_check = 1; - FOREACH_PORT(pt_id, ports) { + RTE_ETH_FOREACH_DEV(pt_id) { printf("\nShutting down port %d...\n", pt_id); fflush(stdout); stop_port(pt_id); @@ -1646,14 +1942,15 @@ check_all_ports_link_status(uint32_t port_mask) { #define CHECK_INTERVAL 100 /* 100ms */ #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ - uint8_t portid, count, all_ports_up, print_flag = 0; + portid_t portid; + uint8_t count, all_ports_up, print_flag = 0; struct rte_eth_link link; printf("Checking link statuses...\n"); fflush(stdout); for (count = 0; count <= MAX_CHECK_TIME; count++) { all_ports_up = 1; - FOREACH_PORT(portid, ports) { + RTE_ETH_FOREACH_DEV(portid) { if ((port_mask & (1 << portid)) == 0) continue; memset(&link, 0, sizeof(link)); @@ -1661,14 +1958,13 @@ check_all_ports_link_status(uint32_t port_mask) /* print link status if flag set */ if (print_flag == 1) { if (link.link_status) - printf("Port %d Link Up - speed %u " - "Mbps - %s\n", (uint8_t)portid, - (unsigned)link.link_speed, + printf( + "Port%d Link Up. speed %u Mbps- %s\n", + portid, link.link_speed, (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? ("full-duplex") : ("half-duplex\n")); else - printf("Port %d Link Down\n", - (uint8_t)portid); + printf("Port %d Link Down\n", portid); continue; } /* clear all_ports_up flag if any link down */ @@ -1690,11 +1986,72 @@ check_all_ports_link_status(uint32_t port_mask) if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { print_flag = 1; } + + if (lsc_interrupt) + break; } } +static void +rmv_event_callback(void *arg) +{ + struct rte_eth_dev *dev; + portid_t port_id = (intptr_t)arg; + + RTE_ETH_VALID_PORTID_OR_RET(port_id); + dev = &rte_eth_devices[port_id]; + + stop_port(port_id); + close_port(port_id); + printf("removing device %s\n", dev->device->name); + if (rte_eal_dev_detach(dev->device)) + RTE_LOG(ERR, USER1, "Failed to detach device %s\n", + dev->device->name); +} + +/* This function is used by the interrupt thread */ static int -set_tx_queue_stats_mapping_registers(uint8_t port_id, struct rte_port *port) +eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param, + void *ret_param) +{ + static const char * const event_desc[] = { + [RTE_ETH_EVENT_UNKNOWN] = "Unknown", + [RTE_ETH_EVENT_INTR_LSC] = "LSC", + [RTE_ETH_EVENT_QUEUE_STATE] = "Queue state", + [RTE_ETH_EVENT_INTR_RESET] = "Interrupt reset", + [RTE_ETH_EVENT_VF_MBOX] = "VF Mbox", + [RTE_ETH_EVENT_MACSEC] = "MACsec", + [RTE_ETH_EVENT_INTR_RMV] = "device removal", + [RTE_ETH_EVENT_MAX] = NULL, + }; + + RTE_SET_USED(param); + RTE_SET_USED(ret_param); + + if (type >= RTE_ETH_EVENT_MAX) { + fprintf(stderr, "\nPort %" PRIu8 ": %s called upon invalid event %d\n", + port_id, __func__, type); + fflush(stderr); + } else if (event_print_mask & (UINT32_C(1) << type)) { + printf("\nPort %" PRIu8 ": %s event\n", port_id, + event_desc[type]); + fflush(stdout); + } + + switch (type) { + case RTE_ETH_EVENT_INTR_RMV: + if (rte_eal_alarm_set(100000, + rmv_event_callback, (void *)(intptr_t)port_id)) + fprintf(stderr, "Could not set up deferred device removal\n"); + break; + default: + break; + } + return 0; +} + +static int +set_tx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port) { uint16_t i; int diag; @@ -1717,7 +2074,7 @@ set_tx_queue_stats_mapping_registers(uint8_t port_id, struct rte_port *port) } static int -set_rx_queue_stats_mapping_registers(uint8_t port_id, struct rte_port *port) +set_rx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port) { uint16_t i; int diag; @@ -1740,7 +2097,7 @@ set_rx_queue_stats_mapping_registers(uint8_t port_id, struct rte_port *port) } static void -map_port_queue_stats_mapping_registers(uint8_t pi, struct rte_port *port) +map_port_queue_stats_mapping_registers(portid_t pi, struct rte_port *port) { int diag = 0; @@ -1818,7 +2175,7 @@ init_port_config(void) portid_t pid; struct rte_port *port; - FOREACH_PORT(pid, ports) { + RTE_ETH_FOREACH_DEV(pid) { port = &ports[pid]; port->dev_conf.rxmode = rx_mode; port->dev_conf.fdir_conf = fdir_conf; @@ -1830,31 +2187,40 @@ init_port_config(void) port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; } - if (port->dcb_flag == 0 && port->dev_info.max_vfs == 0) { + if (port->dcb_flag == 0) { if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_RSS; else port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE; } - if (port->dev_info.max_vfs != 0) { - if (port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) - port->dev_conf.rxmode.mq_mode = - ETH_MQ_RX_VMDQ_RSS; - else - port->dev_conf.rxmode.mq_mode = - ETH_MQ_RX_NONE; - - port->dev_conf.txmode.mq_mode = ETH_MQ_TX_NONE; - } - rxtx_port_config(port); rte_eth_macaddr_get(pid, &port->eth_addr); map_port_queue_stats_mapping_registers(pid, port); -#ifdef RTE_NIC_BYPASS - rte_eth_dev_bypass_init(pid); +#if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS + rte_pmd_ixgbe_bypass_init(pid); +#endif + + if (lsc_interrupt && + (rte_eth_devices[pid].data->dev_flags & + RTE_ETH_DEV_INTR_LSC)) + port->dev_conf.intr_conf.lsc = 1; + if (rmv_interrupt && + (rte_eth_devices[pid].data->dev_flags & + RTE_ETH_DEV_INTR_RMV)) + port->dev_conf.intr_conf.rmv = 1; + +#if defined RTE_LIBRTE_PMD_SOFTNIC && defined RTE_LIBRTE_SCHED + /* Detect softnic port */ + if (!strcmp(port->dev_info.driver_name, "net_softnic")) { + port->softnic_enable = 1; + memset(&port->softport, 0, sizeof(struct softnic_port)); + + if (!strcmp(cur_fwd_eng->fwd_mode_name, "tm")) + port->softport.tm_flag = 1; + } #endif } } @@ -1908,7 +2274,7 @@ get_eth_dcb_conf(struct rte_eth_conf *eth_conf, struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; - /* VMDQ+DCB RX and TX configrations */ + /* VMDQ+DCB RX and TX configurations */ vmdq_rx_conf->enable_default_pool = 0; vmdq_rx_conf->default_pool = 0; vmdq_rx_conf->nb_queue_pools = @@ -1923,8 +2289,8 @@ get_eth_dcb_conf(struct rte_eth_conf *eth_conf, 1 << (i % vmdq_rx_conf->nb_queue_pools); } for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { - vmdq_rx_conf->dcb_tc[i] = i; - vmdq_tx_conf->dcb_tc[i] = i; + vmdq_rx_conf->dcb_tc[i] = i % num_tcs; + vmdq_tx_conf->dcb_tc[i] = i % num_tcs; } /* set DCB mode of RX and TX of multiple queues */ @@ -1939,9 +2305,9 @@ get_eth_dcb_conf(struct rte_eth_conf *eth_conf, rx_conf->nb_tcs = num_tcs; tx_conf->nb_tcs = num_tcs; - for (i = 0; i < num_tcs; i++) { - rx_conf->dcb_tc[i] = i; - tx_conf->dcb_tc[i] = i; + for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { + rx_conf->dcb_tc[i] = i % num_tcs; + tx_conf->dcb_tc[i] = i % num_tcs; } eth_conf->rxmode.mq_mode = ETH_MQ_RX_DCB_RSS; eth_conf->rx_adv_conf.rss_conf.rss_hf = rss_hf; @@ -1985,7 +2351,7 @@ init_port_dcb_config(portid_t pid, * Set the numbers of RX & TX queues to 0, so * the RX & TX queues will not be setup. */ - (void)rte_eth_dev_configure(pid, 0, 0, &port_conf); + rte_eth_dev_configure(pid, 0, 0, &port_conf); rte_eth_dev_info_get(pid, &rte_port->dev_info); @@ -2042,8 +2408,6 @@ init_port_dcb_config(portid_t pid, static void init_port(void) { - portid_t pid; - /* Configuration of Ethernet ports. */ ports = rte_zmalloc("testpmd: ports", sizeof(struct rte_port) * RTE_MAX_ETHPORTS, @@ -2053,10 +2417,6 @@ init_port(void) "rte_zmalloc(%d struct rte_port) failed\n", RTE_MAX_ETHPORTS); } - - /* enabled allocated ports */ - for (pid = 0; pid < nb_ports; pid++) - ports[pid].enabled = 1; } static void @@ -2066,6 +2426,21 @@ force_quit(void) prompt_exit(); } +static void +print_stats(void) +{ + uint8_t i; + const char clr[] = { 27, '[', '2', 'J', '\0' }; + const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' }; + + /* Clear screen and move to top left */ + printf("%s%s", clr, top_left); + + printf("\nPort statistics ===================================="); + for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) + nic_stats_display(fwd_ports_ids[i]); +} + static void signal_handler(int signum) { @@ -2075,8 +2450,13 @@ signal_handler(int signum) #ifdef RTE_LIBRTE_PDUMP /* uninitialize packet capture framework */ rte_pdump_uninit(); +#endif +#ifdef RTE_LIBRTE_LATENCY_STATS + rte_latencystats_uninit(); #endif force_quit(); + /* Set flag to indicate the force termination. */ + f_quit = 1; /* exit with the expected status */ signal(signum, SIG_DFL); kill(getpid(), signum); @@ -2087,7 +2467,7 @@ int main(int argc, char** argv) { int diag; - uint8_t port_id; + portid_t port_id; signal(SIGINT, signal_handler); signal(SIGTERM, signal_handler); @@ -2096,6 +2476,11 @@ main(int argc, char** argv) if (diag < 0) rte_panic("Cannot init EAL\n"); + if (mlockall(MCL_CURRENT | MCL_FUTURE)) { + RTE_LOG(NOTICE, USER1, "mlockall() failed with error \"%s\"\n", + strerror(errno)); + } + #ifdef RTE_LIBRTE_PDUMP /* initialize packet capture framework */ rte_pdump_init(NULL); @@ -2113,11 +2498,29 @@ main(int argc, char** argv) rte_panic("Empty set of forwarding logical cores - check the " "core mask supplied in the command parameters\n"); + /* Bitrate/latency stats disabled by default */ +#ifdef RTE_LIBRTE_BITRATE + bitrate_enabled = 0; +#endif +#ifdef RTE_LIBRTE_LATENCY_STATS + latencystats_enabled = 0; +#endif + argc -= diag; argv += diag; if (argc > 1) launch_args_parse(argc, argv); + if (tx_first && interactive) + rte_exit(EXIT_FAILURE, "--tx-first cannot be used on " + "interactive mode.\n"); + + if (tx_first && lsc_interrupt) { + printf("Warning: lsc_interrupt needs to be off when " + " using tx_first. Disabling.\n"); + lsc_interrupt = 0; + } + if (!nb_rxq && !nb_txq) printf("Warning: Either rx or tx queues should be non-zero\n"); @@ -2131,24 +2534,77 @@ main(int argc, char** argv) rte_exit(EXIT_FAILURE, "Start ports failed\n"); /* set all ports to promiscuous mode by default */ - FOREACH_PORT(port_id, ports) + RTE_ETH_FOREACH_DEV(port_id) rte_eth_promiscuous_enable(port_id); + /* Init metrics library */ + rte_metrics_init(rte_socket_id()); + +#ifdef RTE_LIBRTE_LATENCY_STATS + if (latencystats_enabled != 0) { + int ret = rte_latencystats_init(1, NULL); + if (ret) + printf("Warning: latencystats init()" + " returned error %d\n", ret); + printf("Latencystats running on lcore %d\n", + latencystats_lcore_id); + } +#endif + + /* Setup bitrate stats */ +#ifdef RTE_LIBRTE_BITRATE + if (bitrate_enabled != 0) { + bitrate_data = rte_stats_bitrate_create(); + if (bitrate_data == NULL) + rte_exit(EXIT_FAILURE, + "Could not allocate bitrate data.\n"); + rte_stats_bitrate_reg(bitrate_data); + } +#endif + #ifdef RTE_LIBRTE_CMDLINE + if (strlen(cmdline_filename) != 0) + cmdline_read_from_file(cmdline_filename); + if (interactive == 1) { if (auto_start) { printf("Start automatic packet forwarding\n"); start_packet_forwarding(0); } prompt(); + pmd_test_exit(); } else #endif { char c; int rc; + f_quit = 0; + printf("No commandline core given, start packet forwarding\n"); - start_packet_forwarding(0); + start_packet_forwarding(tx_first); + if (stats_period != 0) { + uint64_t prev_time = 0, cur_time, diff_time = 0; + uint64_t timer_period; + + /* Convert to number of cycles */ + timer_period = stats_period * rte_get_timer_hz(); + + while (f_quit == 0) { + cur_time = rte_get_timer_cycles(); + diff_time += cur_time - prev_time; + + if (diff_time >= timer_period) { + print_stats(); + /* Reset the timer */ + diff_time = 0; + } + /* Sleep to avoid unnecessary checks */ + prev_time = cur_time; + sleep(1); + } + } + printf("Press enter to exit\n"); rc = read(0, &c, 1); pmd_test_exit(); diff --git a/dpdk/app/test-pmd/testpmd.h b/dpdk/app/test-pmd/testpmd.h index 74bf5cb4..92e16073 100644 --- a/dpdk/app/test-pmd/testpmd.h +++ b/dpdk/app/test-pmd/testpmd.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. + * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -34,6 +34,11 @@ #ifndef _TESTPMD_H_ #define _TESTPMD_H_ +#include +#include +#include +#include + #define RTE_PORT_ALL (~(portid_t)0x0) #define RTE_TEST_RX_DESC_MAX 2048 @@ -75,12 +80,18 @@ #define UMA_NO_CONFIG 0xFF typedef uint8_t lcoreid_t; -typedef uint8_t portid_t; +typedef uint16_t portid_t; typedef uint16_t queueid_t; typedef uint16_t streamid_t; #define MAX_QUEUE_ID ((1 << (sizeof(queueid_t) * 8)) - 1) +#if defined RTE_LIBRTE_PMD_SOFTNIC && defined RTE_LIBRTE_SCHED +#define TM_MODE 1 +#else +#define TM_MODE 0 +#endif + enum { PORT_TOPOLOGY_PAIRED, PORT_TOPOLOGY_CHAINED, @@ -117,6 +128,7 @@ struct fwd_stream { unsigned int fwd_dropped; /**< received packets not forwarded */ unsigned int rx_bad_ip_csum ; /**< received packets has bad ip checksum */ unsigned int rx_bad_l4_csum ; /**< received packets has bad l4 checksum */ + unsigned int gro_times; /**< GRO operation times */ #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES uint64_t core_cycles; /**< used for RX and TX processing */ #endif @@ -143,12 +155,58 @@ struct fwd_stream { #define TESTPMD_TX_OFFLOAD_INSERT_VLAN 0x0040 /** Insert double VLAN header in forward engine */ #define TESTPMD_TX_OFFLOAD_INSERT_QINQ 0x0080 +/** Offload MACsec in forward engine */ +#define TESTPMD_TX_OFFLOAD_MACSEC 0x0100 + +/** Descriptor for a single flow. */ +struct port_flow { + size_t size; /**< Allocated space including data[]. */ + struct port_flow *next; /**< Next flow in list. */ + struct port_flow *tmp; /**< Temporary linking. */ + uint32_t id; /**< Flow rule ID. */ + struct rte_flow *flow; /**< Opaque flow object returned by PMD. */ + struct rte_flow_attr attr; /**< Attributes. */ + struct rte_flow_item *pattern; /**< Pattern. */ + struct rte_flow_action *actions; /**< Actions. */ + uint8_t data[]; /**< Storage for pattern/actions. */ +}; + +#ifdef TM_MODE +/** + * Soft port tm related parameters + */ +struct softnic_port_tm { + uint32_t default_hierarchy_enable; /**< def hierarchy enable flag */ + uint32_t hierarchy_config; /**< set to 1 if hierarchy configured */ + + uint32_t n_subports_per_port; /**< Num of subport nodes per port */ + uint32_t n_pipes_per_subport; /**< Num of pipe nodes per subport */ + + uint64_t tm_pktfield0_slabpos; /**< Pkt field position for subport */ + uint64_t tm_pktfield0_slabmask; /**< Pkt field mask for the subport */ + uint64_t tm_pktfield0_slabshr; + uint64_t tm_pktfield1_slabpos; /**< Pkt field position for the pipe */ + uint64_t tm_pktfield1_slabmask; /**< Pkt field mask for the pipe */ + uint64_t tm_pktfield1_slabshr; + uint64_t tm_pktfield2_slabpos; /**< Pkt field position table index */ + uint64_t tm_pktfield2_slabmask; /**< Pkt field mask for tc table idx */ + uint64_t tm_pktfield2_slabshr; + uint64_t tm_tc_table[64]; /**< TC translation table */ +}; + +/** + * The data structure associate with softnic port + */ +struct softnic_port { + unsigned int tm_flag; /**< set to 1 if tm feature is enabled */ + struct softnic_port_tm tm; /**< softnic port tm parameters */ +}; +#endif /** * The data structure associated with each port. */ struct rte_port { - uint8_t enabled; /**< Port enabled or not */ struct rte_eth_dev_info dev_info; /**< PCI info + driver name */ struct rte_eth_conf dev_conf; /**< Port configuration. */ struct ether_addr eth_addr; /**< Port ethernet address */ @@ -158,7 +216,8 @@ struct rte_port { struct fwd_stream *tx_stream; /**< Port TX stream, if unique */ unsigned int socket_id; /**< For NUMA support */ uint16_t tx_ol_flags;/**< TX Offload Flags (TESTPMD_TX_OFFLOAD...). */ - uint16_t tso_segsz; /**< MSS for segmentation offload. */ + uint16_t tso_segsz; /**< Segmentation offload MSS for non-tunneled packets. */ + uint16_t tunnel_tso_segsz; /**< Segmentation offload MSS for tunneled pkts. */ uint16_t tx_vlan_id;/**< The tag ID */ uint16_t tx_vlan_id_outer;/**< The outer tag ID */ void *fwd_ctx; /**< Forwarding mode context */ @@ -176,16 +235,13 @@ struct rte_port { struct ether_addr *mc_addr_pool; /**< pool of multicast addrs */ uint32_t mc_addr_nb; /**< nb. of addr. in mc_addr_pool */ uint8_t slave_flag; /**< bonding slave port */ + struct port_flow *flow_list; /**< Associated flows. */ +#ifdef TM_MODE + unsigned int softnic_enable; /**< softnic flag */ + struct softnic_port softport; /**< softnic port params */ +#endif }; -extern portid_t __rte_unused -find_next_port(portid_t p, struct rte_port *ports, int size); - -#define FOREACH_PORT(p, ports) \ - for (p = find_next_port(0, ports, RTE_MAX_ETHPORTS); \ - p < RTE_MAX_ETHPORTS; \ - p = find_next_port(p + 1, ports, RTE_MAX_ETHPORTS)) - /** * The data structure associated with each forwarding logical core. * The logical cores are internally numbered by a core index from 0 to @@ -194,7 +250,9 @@ find_next_port(portid_t p, struct rte_port *ports, int size); * CPU id. configuration table. */ struct fwd_lcore { + struct rte_gso_ctx gso_ctx; /**< GSO context */ struct rte_mempool *mbp; /**< The mbuf pool to use by this core */ + void *gro_ctx; /**< GRO context */ streamid_t stream_idx; /**< index of 1st stream in "fwd_streams" */ streamid_t stream_nb; /**< number of streams in "fwd_streams" */ lcoreid_t cpuid_idx; /**< index of logical core in CPU id table */ @@ -242,6 +300,10 @@ extern struct fwd_engine rx_only_engine; extern struct fwd_engine tx_only_engine; extern struct fwd_engine csum_fwd_engine; extern struct fwd_engine icmp_echo_engine; +#ifdef TM_MODE +extern struct fwd_engine softnic_tm_engine; +extern struct fwd_engine softnic_tm_bypass_engine; +#endif #ifdef RTE_LIBRTE_IEEE1588 extern struct fwd_engine ieee1588_fwd_engine; #endif @@ -272,7 +334,7 @@ enum dcb_mode_enable #define MAX_RX_QUEUE_STATS_MAPPINGS 4096 /* MAX_PORT of 32 @ 128 rx_queues/port */ struct queue_stats_mappings { - uint8_t port_id; + portid_t port_id; uint16_t queue_id; uint8_t stats_counter_id; } __rte_cache_aligned; @@ -287,18 +349,27 @@ extern struct queue_stats_mappings *rx_queue_stats_mappings; extern uint16_t nb_tx_queue_stats_mappings; extern uint16_t nb_rx_queue_stats_mappings; +extern uint8_t xstats_hide_zero; /**< Hide zero values for xstats display */ + /* globals used for configuration */ extern uint16_t verbose_level; /**< Drives messages being displayed, if any. */ extern uint8_t interactive; extern uint8_t auto_start; +extern uint8_t tx_first; +extern char cmdline_filename[PATH_MAX]; /**< offline commands file */ extern uint8_t numa_support; /**< set by "--numa" parameter */ extern uint16_t port_topology; /**< set by "--port-topology" parameter */ extern uint8_t no_flush_rx; /** +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "testpmd.h" + +#define SUBPORT_NODES_PER_PORT 1 +#define PIPE_NODES_PER_SUBPORT 4096 +#define TC_NODES_PER_PIPE 4 +#define QUEUE_NODES_PER_TC 4 + +#define NUM_PIPE_NODES \ + (SUBPORT_NODES_PER_PORT * PIPE_NODES_PER_SUBPORT) + +#define NUM_TC_NODES \ + (NUM_PIPE_NODES * TC_NODES_PER_PIPE) + +#define ROOT_NODE_ID 1000000 +#define SUBPORT_NODES_START_ID 900000 +#define PIPE_NODES_START_ID 800000 +#define TC_NODES_START_ID 700000 + +#define STATS_MASK_DEFAULT \ + (RTE_TM_STATS_N_PKTS | \ + RTE_TM_STATS_N_BYTES | \ + RTE_TM_STATS_N_PKTS_GREEN_DROPPED | \ + RTE_TM_STATS_N_BYTES_GREEN_DROPPED) + +#define STATS_MASK_QUEUE \ + (STATS_MASK_DEFAULT | \ + RTE_TM_STATS_N_PKTS_QUEUED) + +#define BYTES_IN_MBPS (1000 * 1000 / 8) +#define TOKEN_BUCKET_SIZE 1000000 + +/* TM Hierarchy Levels */ +enum tm_hierarchy_level { + TM_NODE_LEVEL_PORT = 0, + TM_NODE_LEVEL_SUBPORT, + TM_NODE_LEVEL_PIPE, + TM_NODE_LEVEL_TC, + TM_NODE_LEVEL_QUEUE, + TM_NODE_LEVEL_MAX, +}; + +struct tm_hierarchy { + /* TM Nodes */ + uint32_t root_node_id; + uint32_t subport_node_id[SUBPORT_NODES_PER_PORT]; + uint32_t pipe_node_id[SUBPORT_NODES_PER_PORT][PIPE_NODES_PER_SUBPORT]; + uint32_t tc_node_id[NUM_PIPE_NODES][TC_NODES_PER_PIPE]; + uint32_t queue_node_id[NUM_TC_NODES][QUEUE_NODES_PER_TC]; + + /* TM Hierarchy Nodes Shaper Rates */ + uint32_t root_node_shaper_rate; + uint32_t subport_node_shaper_rate; + uint32_t pipe_node_shaper_rate; + uint32_t tc_node_shaper_rate; + uint32_t tc_node_shared_shaper_rate; + + uint32_t n_shapers; +}; + +#define BITFIELD(byte_array, slab_pos, slab_mask, slab_shr) \ +({ \ + uint64_t slab = *((uint64_t *) &byte_array[slab_pos]); \ + uint64_t val = \ + (rte_be_to_cpu_64(slab) & slab_mask) >> slab_shr; \ + val; \ +}) + +#define RTE_SCHED_PORT_HIERARCHY(subport, pipe, \ + traffic_class, queue, color) \ + ((((uint64_t) (queue)) & 0x3) | \ + ((((uint64_t) (traffic_class)) & 0x3) << 2) | \ + ((((uint64_t) (color)) & 0x3) << 4) | \ + ((((uint64_t) (subport)) & 0xFFFF) << 16) | \ + ((((uint64_t) (pipe)) & 0xFFFFFFFF) << 32)) + + +static void +pkt_metadata_set(struct rte_port *p, struct rte_mbuf **pkts, + uint32_t n_pkts) +{ + struct softnic_port_tm *tm = &p->softport.tm; + uint32_t i; + + for (i = 0; i < (n_pkts & (~0x3)); i += 4) { + struct rte_mbuf *pkt0 = pkts[i]; + struct rte_mbuf *pkt1 = pkts[i + 1]; + struct rte_mbuf *pkt2 = pkts[i + 2]; + struct rte_mbuf *pkt3 = pkts[i + 3]; + + uint8_t *pkt0_data = rte_pktmbuf_mtod(pkt0, uint8_t *); + uint8_t *pkt1_data = rte_pktmbuf_mtod(pkt1, uint8_t *); + uint8_t *pkt2_data = rte_pktmbuf_mtod(pkt2, uint8_t *); + uint8_t *pkt3_data = rte_pktmbuf_mtod(pkt3, uint8_t *); + + uint64_t pkt0_subport = BITFIELD(pkt0_data, + tm->tm_pktfield0_slabpos, + tm->tm_pktfield0_slabmask, + tm->tm_pktfield0_slabshr); + uint64_t pkt0_pipe = BITFIELD(pkt0_data, + tm->tm_pktfield1_slabpos, + tm->tm_pktfield1_slabmask, + tm->tm_pktfield1_slabshr); + uint64_t pkt0_dscp = BITFIELD(pkt0_data, + tm->tm_pktfield2_slabpos, + tm->tm_pktfield2_slabmask, + tm->tm_pktfield2_slabshr); + uint32_t pkt0_tc = tm->tm_tc_table[pkt0_dscp & 0x3F] >> 2; + uint32_t pkt0_tc_q = tm->tm_tc_table[pkt0_dscp & 0x3F] & 0x3; + uint64_t pkt1_subport = BITFIELD(pkt1_data, + tm->tm_pktfield0_slabpos, + tm->tm_pktfield0_slabmask, + tm->tm_pktfield0_slabshr); + uint64_t pkt1_pipe = BITFIELD(pkt1_data, + tm->tm_pktfield1_slabpos, + tm->tm_pktfield1_slabmask, + tm->tm_pktfield1_slabshr); + uint64_t pkt1_dscp = BITFIELD(pkt1_data, + tm->tm_pktfield2_slabpos, + tm->tm_pktfield2_slabmask, + tm->tm_pktfield2_slabshr); + uint32_t pkt1_tc = tm->tm_tc_table[pkt1_dscp & 0x3F] >> 2; + uint32_t pkt1_tc_q = tm->tm_tc_table[pkt1_dscp & 0x3F] & 0x3; + + uint64_t pkt2_subport = BITFIELD(pkt2_data, + tm->tm_pktfield0_slabpos, + tm->tm_pktfield0_slabmask, + tm->tm_pktfield0_slabshr); + uint64_t pkt2_pipe = BITFIELD(pkt2_data, + tm->tm_pktfield1_slabpos, + tm->tm_pktfield1_slabmask, + tm->tm_pktfield1_slabshr); + uint64_t pkt2_dscp = BITFIELD(pkt2_data, + tm->tm_pktfield2_slabpos, + tm->tm_pktfield2_slabmask, + tm->tm_pktfield2_slabshr); + uint32_t pkt2_tc = tm->tm_tc_table[pkt2_dscp & 0x3F] >> 2; + uint32_t pkt2_tc_q = tm->tm_tc_table[pkt2_dscp & 0x3F] & 0x3; + + uint64_t pkt3_subport = BITFIELD(pkt3_data, + tm->tm_pktfield0_slabpos, + tm->tm_pktfield0_slabmask, + tm->tm_pktfield0_slabshr); + uint64_t pkt3_pipe = BITFIELD(pkt3_data, + tm->tm_pktfield1_slabpos, + tm->tm_pktfield1_slabmask, + tm->tm_pktfield1_slabshr); + uint64_t pkt3_dscp = BITFIELD(pkt3_data, + tm->tm_pktfield2_slabpos, + tm->tm_pktfield2_slabmask, + tm->tm_pktfield2_slabshr); + uint32_t pkt3_tc = tm->tm_tc_table[pkt3_dscp & 0x3F] >> 2; + uint32_t pkt3_tc_q = tm->tm_tc_table[pkt3_dscp & 0x3F] & 0x3; + + uint64_t pkt0_sched = RTE_SCHED_PORT_HIERARCHY(pkt0_subport, + pkt0_pipe, + pkt0_tc, + pkt0_tc_q, + 0); + uint64_t pkt1_sched = RTE_SCHED_PORT_HIERARCHY(pkt1_subport, + pkt1_pipe, + pkt1_tc, + pkt1_tc_q, + 0); + uint64_t pkt2_sched = RTE_SCHED_PORT_HIERARCHY(pkt2_subport, + pkt2_pipe, + pkt2_tc, + pkt2_tc_q, + 0); + uint64_t pkt3_sched = RTE_SCHED_PORT_HIERARCHY(pkt3_subport, + pkt3_pipe, + pkt3_tc, + pkt3_tc_q, + 0); + + pkt0->hash.sched.lo = pkt0_sched & 0xFFFFFFFF; + pkt0->hash.sched.hi = pkt0_sched >> 32; + pkt1->hash.sched.lo = pkt1_sched & 0xFFFFFFFF; + pkt1->hash.sched.hi = pkt1_sched >> 32; + pkt2->hash.sched.lo = pkt2_sched & 0xFFFFFFFF; + pkt2->hash.sched.hi = pkt2_sched >> 32; + pkt3->hash.sched.lo = pkt3_sched & 0xFFFFFFFF; + pkt3->hash.sched.hi = pkt3_sched >> 32; + } + + for (; i < n_pkts; i++) { + struct rte_mbuf *pkt = pkts[i]; + + uint8_t *pkt_data = rte_pktmbuf_mtod(pkt, uint8_t *); + + uint64_t pkt_subport = BITFIELD(pkt_data, + tm->tm_pktfield0_slabpos, + tm->tm_pktfield0_slabmask, + tm->tm_pktfield0_slabshr); + uint64_t pkt_pipe = BITFIELD(pkt_data, + tm->tm_pktfield1_slabpos, + tm->tm_pktfield1_slabmask, + tm->tm_pktfield1_slabshr); + uint64_t pkt_dscp = BITFIELD(pkt_data, + tm->tm_pktfield2_slabpos, + tm->tm_pktfield2_slabmask, + tm->tm_pktfield2_slabshr); + uint32_t pkt_tc = tm->tm_tc_table[pkt_dscp & 0x3F] >> 2; + uint32_t pkt_tc_q = tm->tm_tc_table[pkt_dscp & 0x3F] & 0x3; + + uint64_t pkt_sched = RTE_SCHED_PORT_HIERARCHY(pkt_subport, + pkt_pipe, + pkt_tc, + pkt_tc_q, + 0); + + pkt->hash.sched.lo = pkt_sched & 0xFFFFFFFF; + pkt->hash.sched.hi = pkt_sched >> 32; + } +} + +/* + * Soft port packet forward + */ +static void +softport_packet_fwd(struct fwd_stream *fs) +{ + struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; + struct rte_port *rte_tx_port = &ports[fs->tx_port]; + uint16_t nb_rx; + uint16_t nb_tx; + uint32_t retry; + +#ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES + uint64_t start_tsc; + uint64_t end_tsc; + uint64_t core_cycles; +#endif + +#ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES + start_tsc = rte_rdtsc(); +#endif + + /* Packets Receive */ + nb_rx = rte_eth_rx_burst(fs->rx_port, fs->rx_queue, + pkts_burst, nb_pkt_per_burst); + fs->rx_packets += nb_rx; + +#ifdef RTE_TEST_PMD_RECORD_BURST_STATS + fs->rx_burst_stats.pkt_burst_spread[nb_rx]++; +#endif + + if (rte_tx_port->softnic_enable) { + /* Set packet metadata if tm flag enabled */ + if (rte_tx_port->softport.tm_flag) + pkt_metadata_set(rte_tx_port, pkts_burst, nb_rx); + + /* Softport run */ + rte_pmd_softnic_run(fs->tx_port); + } + nb_tx = rte_eth_tx_burst(fs->tx_port, fs->tx_queue, + pkts_burst, nb_rx); + + /* Retry if necessary */ + if (unlikely(nb_tx < nb_rx) && fs->retry_enabled) { + retry = 0; + while (nb_tx < nb_rx && retry++ < burst_tx_retry_num) { + rte_delay_us(burst_tx_delay_time); + nb_tx += rte_eth_tx_burst(fs->tx_port, fs->tx_queue, + &pkts_burst[nb_tx], nb_rx - nb_tx); + } + } + fs->tx_packets += nb_tx; + +#ifdef RTE_TEST_PMD_RECORD_BURST_STATS + fs->tx_burst_stats.pkt_burst_spread[nb_tx]++; +#endif + + if (unlikely(nb_tx < nb_rx)) { + fs->fwd_dropped += (nb_rx - nb_tx); + do { + rte_pktmbuf_free(pkts_burst[nb_tx]); + } while (++nb_tx < nb_rx); + } +#ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES + end_tsc = rte_rdtsc(); + core_cycles = (end_tsc - start_tsc); + fs->core_cycles = (uint64_t) (fs->core_cycles + core_cycles); +#endif +} + +static void +set_tm_hiearchy_nodes_shaper_rate(portid_t port_id, struct tm_hierarchy *h) +{ + struct rte_eth_link link_params; + uint64_t tm_port_rate; + + memset(&link_params, 0, sizeof(link_params)); + + rte_eth_link_get(port_id, &link_params); + tm_port_rate = (uint64_t)link_params.link_speed * BYTES_IN_MBPS; + + if (tm_port_rate > UINT32_MAX) + tm_port_rate = UINT32_MAX; + + /* Set tm hierarchy shapers rate */ + h->root_node_shaper_rate = tm_port_rate; + h->subport_node_shaper_rate = + tm_port_rate / SUBPORT_NODES_PER_PORT; + h->pipe_node_shaper_rate + = h->subport_node_shaper_rate / PIPE_NODES_PER_SUBPORT; + h->tc_node_shaper_rate = h->pipe_node_shaper_rate; + h->tc_node_shared_shaper_rate = h->subport_node_shaper_rate; +} + +static int +softport_tm_root_node_add(portid_t port_id, struct tm_hierarchy *h, + struct rte_tm_error *error) +{ + struct rte_tm_node_params rnp; + struct rte_tm_shaper_params rsp; + uint32_t priority, weight, level_id, shaper_profile_id; + + memset(&rsp, 0, sizeof(struct rte_tm_shaper_params)); + memset(&rnp, 0, sizeof(struct rte_tm_node_params)); + + /* Shaper profile Parameters */ + rsp.peak.rate = h->root_node_shaper_rate; + rsp.peak.size = TOKEN_BUCKET_SIZE; + rsp.pkt_length_adjust = RTE_TM_ETH_FRAMING_OVERHEAD_FCS; + shaper_profile_id = 0; + + if (rte_tm_shaper_profile_add(port_id, shaper_profile_id, + &rsp, error)) { + printf("%s ERROR(%d)-%s!(shaper_id %u)\n ", + __func__, error->type, error->message, + shaper_profile_id); + return -1; + } + + /* Root Node Parameters */ + h->root_node_id = ROOT_NODE_ID; + weight = 1; + priority = 0; + level_id = TM_NODE_LEVEL_PORT; + rnp.shaper_profile_id = shaper_profile_id; + rnp.nonleaf.n_sp_priorities = 1; + rnp.stats_mask = STATS_MASK_DEFAULT; + + /* Add Node to TM Hierarchy */ + if (rte_tm_node_add(port_id, h->root_node_id, RTE_TM_NODE_ID_NULL, + priority, weight, level_id, &rnp, error)) { + printf("%s ERROR(%d)-%s!(node_id %u, parent_id %u, level %u)\n", + __func__, error->type, error->message, + h->root_node_id, RTE_TM_NODE_ID_NULL, + level_id); + return -1; + } + /* Update */ + h->n_shapers++; + + printf(" Root node added (Start id %u, Count %u, level %u)\n", + h->root_node_id, 1, level_id); + + return 0; +} + +static int +softport_tm_subport_node_add(portid_t port_id, struct tm_hierarchy *h, + struct rte_tm_error *error) +{ + uint32_t subport_parent_node_id, subport_node_id = 0; + struct rte_tm_node_params snp; + struct rte_tm_shaper_params ssp; + uint32_t priority, weight, level_id, shaper_profile_id; + uint32_t i; + + memset(&ssp, 0, sizeof(struct rte_tm_shaper_params)); + memset(&snp, 0, sizeof(struct rte_tm_node_params)); + + shaper_profile_id = h->n_shapers; + + /* Add Shaper Profile to TM Hierarchy */ + for (i = 0; i < SUBPORT_NODES_PER_PORT; i++) { + ssp.peak.rate = h->subport_node_shaper_rate; + ssp.peak.size = TOKEN_BUCKET_SIZE; + ssp.pkt_length_adjust = RTE_TM_ETH_FRAMING_OVERHEAD_FCS; + + if (rte_tm_shaper_profile_add(port_id, shaper_profile_id, + &ssp, error)) { + printf("%s ERROR(%d)-%s!(shaper_id %u)\n ", + __func__, error->type, error->message, + shaper_profile_id); + return -1; + } + + /* Node Parameters */ + h->subport_node_id[i] = SUBPORT_NODES_START_ID + i; + subport_parent_node_id = h->root_node_id; + weight = 1; + priority = 0; + level_id = TM_NODE_LEVEL_SUBPORT; + snp.shaper_profile_id = shaper_profile_id; + snp.nonleaf.n_sp_priorities = 1; + snp.stats_mask = STATS_MASK_DEFAULT; + + /* Add Node to TM Hiearchy */ + if (rte_tm_node_add(port_id, + h->subport_node_id[i], + subport_parent_node_id, + priority, weight, + level_id, + &snp, + error)) { + printf("%s ERROR(%d)-%s!(node %u,parent %u,level %u)\n", + __func__, + error->type, + error->message, + h->subport_node_id[i], + subport_parent_node_id, + level_id); + return -1; + } + shaper_profile_id++; + subport_node_id++; + } + /* Update */ + h->n_shapers = shaper_profile_id; + + printf(" Subport nodes added (Start id %u, Count %u, level %u)\n", + h->subport_node_id[0], SUBPORT_NODES_PER_PORT, level_id); + + return 0; +} + +static int +softport_tm_pipe_node_add(portid_t port_id, struct tm_hierarchy *h, + struct rte_tm_error *error) +{ + uint32_t pipe_parent_node_id; + struct rte_tm_node_params pnp; + struct rte_tm_shaper_params psp; + uint32_t priority, weight, level_id, shaper_profile_id; + uint32_t i, j; + + memset(&psp, 0, sizeof(struct rte_tm_shaper_params)); + memset(&pnp, 0, sizeof(struct rte_tm_node_params)); + + shaper_profile_id = h->n_shapers; + + /* Shaper Profile Parameters */ + psp.peak.rate = h->pipe_node_shaper_rate; + psp.peak.size = TOKEN_BUCKET_SIZE; + psp.pkt_length_adjust = RTE_TM_ETH_FRAMING_OVERHEAD_FCS; + + /* Pipe Node Parameters */ + weight = 1; + priority = 0; + level_id = TM_NODE_LEVEL_PIPE; + pnp.nonleaf.n_sp_priorities = 4; + pnp.stats_mask = STATS_MASK_DEFAULT; + + /* Add Shaper Profiles and Nodes to TM Hierarchy */ + for (i = 0; i < SUBPORT_NODES_PER_PORT; i++) { + for (j = 0; j < PIPE_NODES_PER_SUBPORT; j++) { + if (rte_tm_shaper_profile_add(port_id, + shaper_profile_id, &psp, error)) { + printf("%s ERROR(%d)-%s!(shaper_id %u)\n ", + __func__, error->type, error->message, + shaper_profile_id); + return -1; + } + pnp.shaper_profile_id = shaper_profile_id; + pipe_parent_node_id = h->subport_node_id[i]; + h->pipe_node_id[i][j] = PIPE_NODES_START_ID + + (i * PIPE_NODES_PER_SUBPORT) + j; + + if (rte_tm_node_add(port_id, + h->pipe_node_id[i][j], + pipe_parent_node_id, + priority, weight, level_id, + &pnp, + error)) { + printf("%s ERROR(%d)-%s!(node %u,parent %u )\n", + __func__, + error->type, + error->message, + h->pipe_node_id[i][j], + pipe_parent_node_id); + + return -1; + } + shaper_profile_id++; + } + } + /* Update */ + h->n_shapers = shaper_profile_id; + + printf(" Pipe nodes added (Start id %u, Count %u, level %u)\n", + h->pipe_node_id[0][0], NUM_PIPE_NODES, level_id); + + return 0; +} + +static int +softport_tm_tc_node_add(portid_t port_id, struct tm_hierarchy *h, + struct rte_tm_error *error) +{ + uint32_t tc_parent_node_id; + struct rte_tm_node_params tnp; + struct rte_tm_shaper_params tsp, tssp; + uint32_t shared_shaper_profile_id[TC_NODES_PER_PIPE]; + uint32_t priority, weight, level_id, shaper_profile_id; + uint32_t pos, n_tc_nodes, i, j, k; + + memset(&tsp, 0, sizeof(struct rte_tm_shaper_params)); + memset(&tssp, 0, sizeof(struct rte_tm_shaper_params)); + memset(&tnp, 0, sizeof(struct rte_tm_node_params)); + + shaper_profile_id = h->n_shapers; + + /* Private Shaper Profile (TC) Parameters */ + tsp.peak.rate = h->tc_node_shaper_rate; + tsp.peak.size = TOKEN_BUCKET_SIZE; + tsp.pkt_length_adjust = RTE_TM_ETH_FRAMING_OVERHEAD_FCS; + + /* Shared Shaper Profile (TC) Parameters */ + tssp.peak.rate = h->tc_node_shared_shaper_rate; + tssp.peak.size = TOKEN_BUCKET_SIZE; + tssp.pkt_length_adjust = RTE_TM_ETH_FRAMING_OVERHEAD_FCS; + + /* TC Node Parameters */ + weight = 1; + level_id = TM_NODE_LEVEL_TC; + tnp.n_shared_shapers = 1; + tnp.nonleaf.n_sp_priorities = 1; + tnp.stats_mask = STATS_MASK_DEFAULT; + + /* Add Shared Shaper Profiles to TM Hierarchy */ + for (i = 0; i < TC_NODES_PER_PIPE; i++) { + shared_shaper_profile_id[i] = shaper_profile_id; + + if (rte_tm_shaper_profile_add(port_id, + shared_shaper_profile_id[i], &tssp, error)) { + printf("%s ERROR(%d)-%s!(Shared shaper profileid %u)\n", + __func__, error->type, error->message, + shared_shaper_profile_id[i]); + + return -1; + } + if (rte_tm_shared_shaper_add_update(port_id, i, + shared_shaper_profile_id[i], error)) { + printf("%s ERROR(%d)-%s!(Shared shaper id %u)\n", + __func__, error->type, error->message, i); + + return -1; + } + shaper_profile_id++; + } + + /* Add Shaper Profiles and Nodes to TM Hierarchy */ + n_tc_nodes = 0; + for (i = 0; i < SUBPORT_NODES_PER_PORT; i++) { + for (j = 0; j < PIPE_NODES_PER_SUBPORT; j++) { + for (k = 0; k < TC_NODES_PER_PIPE ; k++) { + priority = k; + tc_parent_node_id = h->pipe_node_id[i][j]; + tnp.shared_shaper_id = + (uint32_t *)calloc(1, sizeof(uint32_t)); + tnp.shared_shaper_id[0] = k; + pos = j + (i * PIPE_NODES_PER_SUBPORT); + h->tc_node_id[pos][k] = + TC_NODES_START_ID + n_tc_nodes; + + if (rte_tm_shaper_profile_add(port_id, + shaper_profile_id, &tsp, error)) { + printf("%s ERROR(%d)-%s!(shaper %u)\n", + __func__, error->type, + error->message, + shaper_profile_id); + + return -1; + } + tnp.shaper_profile_id = shaper_profile_id; + if (rte_tm_node_add(port_id, + h->tc_node_id[pos][k], + tc_parent_node_id, + priority, weight, + level_id, + &tnp, error)) { + printf("%s ERROR(%d)-%s!(node id %u)\n", + __func__, + error->type, + error->message, + h->tc_node_id[pos][k]); + + return -1; + } + shaper_profile_id++; + n_tc_nodes++; + } + } + } + /* Update */ + h->n_shapers = shaper_profile_id; + + printf(" TC nodes added (Start id %u, Count %u, level %u)\n", + h->tc_node_id[0][0], n_tc_nodes, level_id); + + return 0; +} + +static int +softport_tm_queue_node_add(portid_t port_id, struct tm_hierarchy *h, + struct rte_tm_error *error) +{ + uint32_t queue_parent_node_id; + struct rte_tm_node_params qnp; + uint32_t priority, weight, level_id, pos; + uint32_t n_queue_nodes, i, j, k; + + memset(&qnp, 0, sizeof(struct rte_tm_node_params)); + + /* Queue Node Parameters */ + priority = 0; + weight = 1; + level_id = TM_NODE_LEVEL_QUEUE; + qnp.shaper_profile_id = RTE_TM_SHAPER_PROFILE_ID_NONE; + qnp.leaf.cman = RTE_TM_CMAN_TAIL_DROP; + qnp.stats_mask = STATS_MASK_QUEUE; + + /* Add Queue Nodes to TM Hierarchy */ + n_queue_nodes = 0; + for (i = 0; i < NUM_PIPE_NODES; i++) { + for (j = 0; j < TC_NODES_PER_PIPE; j++) { + queue_parent_node_id = h->tc_node_id[i][j]; + for (k = 0; k < QUEUE_NODES_PER_TC; k++) { + pos = j + (i * TC_NODES_PER_PIPE); + h->queue_node_id[pos][k] = n_queue_nodes; + if (rte_tm_node_add(port_id, + h->queue_node_id[pos][k], + queue_parent_node_id, + priority, + weight, + level_id, + &qnp, error)) { + printf("%s ERROR(%d)-%s!(node %u)\n", + __func__, + error->type, + error->message, + h->queue_node_id[pos][k]); + + return -1; + } + n_queue_nodes++; + } + } + } + printf(" Queue nodes added (Start id %u, Count %u, level %u)\n", + h->queue_node_id[0][0], n_queue_nodes, level_id); + + return 0; +} + +/* + * TM Packet Field Setup + */ +static void +softport_tm_pktfield_setup(portid_t port_id) +{ + struct rte_port *p = &ports[port_id]; + uint64_t pktfield0_mask = 0; + uint64_t pktfield1_mask = 0x0000000FFF000000LLU; + uint64_t pktfield2_mask = 0x00000000000000FCLLU; + + p->softport.tm = (struct softnic_port_tm) { + .n_subports_per_port = SUBPORT_NODES_PER_PORT, + .n_pipes_per_subport = PIPE_NODES_PER_SUBPORT, + + /* Packet field to identify subport + * + * Default configuration assumes only one subport, thus + * the subport ID is hardcoded to 0 + */ + .tm_pktfield0_slabpos = 0, + .tm_pktfield0_slabmask = pktfield0_mask, + .tm_pktfield0_slabshr = + __builtin_ctzll(pktfield0_mask), + + /* Packet field to identify pipe. + * + * Default value assumes Ethernet/IPv4/UDP packets, + * UDP payload bits 12 .. 23 + */ + .tm_pktfield1_slabpos = 40, + .tm_pktfield1_slabmask = pktfield1_mask, + .tm_pktfield1_slabshr = + __builtin_ctzll(pktfield1_mask), + + /* Packet field used as index into TC translation table + * to identify the traffic class and queue. + * + * Default value assumes Ethernet/IPv4 packets, IPv4 + * DSCP field + */ + .tm_pktfield2_slabpos = 8, + .tm_pktfield2_slabmask = pktfield2_mask, + .tm_pktfield2_slabshr = + __builtin_ctzll(pktfield2_mask), + + .tm_tc_table = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + }, /**< TC translation table */ + }; +} + +static int +softport_tm_hierarchy_specify(portid_t port_id, struct rte_tm_error *error) +{ + + struct tm_hierarchy h; + int status; + + memset(&h, 0, sizeof(struct tm_hierarchy)); + + /* TM hierarchy shapers rate */ + set_tm_hiearchy_nodes_shaper_rate(port_id, &h); + + /* Add root node (level 0) */ + status = softport_tm_root_node_add(port_id, &h, error); + if (status) + return status; + + /* Add subport node (level 1) */ + status = softport_tm_subport_node_add(port_id, &h, error); + if (status) + return status; + + /* Add pipe nodes (level 2) */ + status = softport_tm_pipe_node_add(port_id, &h, error); + if (status) + return status; + + /* Add traffic class nodes (level 3) */ + status = softport_tm_tc_node_add(port_id, &h, error); + if (status) + return status; + + /* Add queue nodes (level 4) */ + status = softport_tm_queue_node_add(port_id, &h, error); + if (status) + return status; + + /* TM packet fields setup */ + softport_tm_pktfield_setup(port_id); + + return 0; +} + +/* + * Soft port Init + */ +static void +softport_tm_begin(portid_t pi) +{ + struct rte_port *port = &ports[pi]; + + /* Soft port TM flag */ + if (port->softport.tm_flag == 1) { + printf("\n\n TM feature available on port %u\n", pi); + + /* Soft port TM hierarchy configuration */ + if ((port->softport.tm.hierarchy_config == 0) && + (port->softport.tm.default_hierarchy_enable == 1)) { + struct rte_tm_error error; + int status; + + /* Stop port */ + rte_eth_dev_stop(pi); + + /* TM hierarchy specification */ + status = softport_tm_hierarchy_specify(pi, &error); + if (status) { + printf(" TM Hierarchy built error(%d) - %s\n", + error.type, error.message); + return; + } + printf("\n TM Hierarchy Specified!\n\v"); + + /* TM hierarchy commit */ + status = rte_tm_hierarchy_commit(pi, 0, &error); + if (status) { + printf(" Hierarchy commit error(%d) - %s\n", + error.type, error.message); + return; + } + printf(" Hierarchy Committed (port %u)!", pi); + port->softport.tm.hierarchy_config = 1; + + /* Start port */ + status = rte_eth_dev_start(pi); + if (status) { + printf("\n Port %u start error!\n", pi); + return; + } + printf("\n Port %u started!\n", pi); + return; + } + } + printf("\n TM feature not available on port %u", pi); +} + +struct fwd_engine softnic_tm_engine = { + .fwd_mode_name = "tm", + .port_fwd_begin = softport_tm_begin, + .port_fwd_end = NULL, + .packet_fwd = softport_packet_fwd, +}; + +struct fwd_engine softnic_tm_bypass_engine = { + .fwd_mode_name = "tm-bypass", + .port_fwd_begin = NULL, + .port_fwd_end = NULL, + .packet_fwd = softport_packet_fwd, +}; diff --git a/dpdk/app/test-pmd/txonly.c b/dpdk/app/test-pmd/txonly.c index 11fd681d..4ce4d61c 100644 --- a/dpdk/app/test-pmd/txonly.c +++ b/dpdk/app/test-pmd/txonly.c @@ -49,18 +49,14 @@ #include #include #include -#include #include #include #include #include #include #include -#include -#include #include #include -#include #include #include #include @@ -69,6 +65,7 @@ #include #include #include +#include #include "testpmd.h" @@ -107,6 +104,7 @@ copy_buf_to_pkt_segs(void* buf, unsigned len, struct rte_mbuf *pkt, buf = ((char*) buf + copy_len); seg = seg->next; seg_buf = rte_pktmbuf_mtod(seg, char *); + copy_len = seg->data_len; } rte_memcpy(seg_buf, buf, (size_t) len); } @@ -215,6 +213,8 @@ pkt_burst_transmit(struct fwd_stream *fs) ol_flags = PKT_TX_VLAN_PKT; if (txp->tx_ol_flags & TESTPMD_TX_OFFLOAD_INSERT_QINQ) ol_flags |= PKT_TX_QINQ_PKT; + if (txp->tx_ol_flags & TESTPMD_TX_OFFLOAD_MACSEC) + ol_flags |= PKT_TX_MACSEC; for (nb_pkt = 0; nb_pkt < nb_pkt_per_burst; nb_pkt++) { pkt = rte_mbuf_raw_alloc(mbp); if (pkt == NULL) { @@ -223,6 +223,14 @@ pkt_burst_transmit(struct fwd_stream *fs) return; break; } + + /* + * Using raw alloc is good to improve performance, + * but some consumers may use the headroom and so + * decrement data_off. We need to make sure it is + * reset to default value. + */ + rte_pktmbuf_reset_headroom(pkt); pkt->data_len = tx_pkt_seg_lengths[0]; pkt_seg = pkt; if (tx_pkt_split == TX_PKT_SPLIT_RND) diff --git a/dpdk/buildtools/auto-config-h.sh b/dpdk/buildtools/auto-config-h.sh new file mode 100644 index 00000000..cb8bce9b --- /dev/null +++ b/dpdk/buildtools/auto-config-h.sh @@ -0,0 +1,136 @@ +#!/bin/sh +# +# BSD LICENSE +# +# Copyright 2014-2015 6WIND S.A. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of 6WIND S.A. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Crude script to detect whether particular types, macros and functions are +# defined by trying to compile a file with a given header. Can be used to +# perform cross-platform checks since the resulting object file is not +# executed. +# +# Set VERBOSE=1 in the environment to display compiler output and errors. +# +# CC, CPPFLAGS, CFLAGS, EXTRA_CPPFLAGS and EXTRA_CFLAGS are taken from the +# environment. +# +# AUTO_CONFIG_CFLAGS may append additional CFLAGS without modifying the +# above variables. + +file=${1:?output file name required (config.h)} +macro=${2:?output macro name required (HAVE_*)} +include=${3:?include name required (foo.h)} +type=${4:?object type required (define, enum, type, field, func)} +name=${5:?define/type/function name required} + +: ${CC:=cc} + +temp=/tmp/${0##*/}.$$.c + +case $type in +define) + code="\ +#ifndef $name +#error $name not defined +#endif +" + ;; +enum) + code="\ +long test____ = $name; +" + ;; +type) + code="\ +$name test____; +" + ;; +field) + code="\ +void test____(void) +{ + ${name%%.*} test_____; + + (void)test_____.${name#*.}; +} +" + ;; +func) + code="\ +void (*test____)() = (void (*)())$name; +" + ;; +*) + unset error + : ${error:?unknown object type \"$type\"} + exit +esac + +if [ "${VERBOSE}" = 1 ] +then + err=2 + out=1 + eol=' +' +else + exec 3> /dev/null || + exit + err=3 + out=3 + eol=' ' +fi && +printf 'Looking for %s %s in %s.%s' \ + "${name}" "${type}" "${include}" "${eol}" && +printf "\ +#include <%s> + +%s +" "$include" "$code" > "${temp}" && +if ${CC} ${CPPFLAGS} ${EXTRA_CPPFLAGS} ${CFLAGS} ${EXTRA_CFLAGS} \ + ${AUTO_CONFIG_CFLAGS} \ + -c -o ${temp}.o "${temp}" 1>&${out} 2>&${err} +then + rm -f "${temp}" "${temp}.o" + printf "\ +#ifndef %s +#define %s 1 +#endif /* %s */ + +" "${macro}" "${macro}" "${macro}" >> "${file}" && + printf 'Defining %s.\n' "${macro}" +else + rm -f "${temp}" "${temp}.o" + printf "\ +/* %s is not defined. */ + +" "${macro}" >> "${file}" && + printf 'Not defining %s.\n' "${macro}" +fi + +exit diff --git a/dpdk/scripts/gen-build-mk.sh b/dpdk/buildtools/gen-build-mk.sh old mode 100755 new mode 100644 similarity index 100% rename from dpdk/scripts/gen-build-mk.sh rename to dpdk/buildtools/gen-build-mk.sh diff --git a/dpdk/scripts/gen-config-h.sh b/dpdk/buildtools/gen-config-h.sh old mode 100755 new mode 100644 similarity index 100% rename from dpdk/scripts/gen-config-h.sh rename to dpdk/buildtools/gen-config-h.sh diff --git a/dpdk/buildtools/pmdinfogen/Makefile b/dpdk/buildtools/pmdinfogen/Makefile index bd8f9005..bf07b6f2 100644 --- a/dpdk/buildtools/pmdinfogen/Makefile +++ b/dpdk/buildtools/pmdinfogen/Makefile @@ -44,6 +44,4 @@ SRCS-y += pmdinfogen.c HOST_CFLAGS += $(WERROR_FLAGS) -g HOST_CFLAGS += -I$(RTE_OUTPUT)/include -DEPDIRS-y += lib/librte_eal - include $(RTE_SDK)/mk/rte.hostapp.mk diff --git a/dpdk/buildtools/pmdinfogen/pmdinfogen.c b/dpdk/buildtools/pmdinfogen/pmdinfogen.c index 59ab9569..b07dbcf4 100644 --- a/dpdk/buildtools/pmdinfogen/pmdinfogen.c +++ b/dpdk/buildtools/pmdinfogen/pmdinfogen.c @@ -158,7 +158,8 @@ static int parse_elf(struct elf_info *info, const char *filename) * There are more than 64k sections, * read count from .sh_size. */ - info->num_sections = TO_NATIVE(endian, 32, sechdrs[0].sh_size); + info->num_sections = + TO_NATIVE(endian, ADDR_SIZE, sechdrs[0].sh_size); } else { info->num_sections = hdr->e_shnum; } @@ -181,7 +182,7 @@ static int parse_elf(struct elf_info *info, const char *filename) sechdrs[i].sh_offset = TO_NATIVE(endian, ADDR_SIZE, sechdrs[i].sh_offset); sechdrs[i].sh_size = - TO_NATIVE(endian, 32, sechdrs[i].sh_size); + TO_NATIVE(endian, ADDR_SIZE, sechdrs[i].sh_size); sechdrs[i].sh_link = TO_NATIVE(endian, 32, sechdrs[i].sh_link); sechdrs[i].sh_info = @@ -226,13 +227,14 @@ static int parse_elf(struct elf_info *info, const char *filename) } if (!info->symtab_start) fprintf(stderr, "%s has no symtab?\n", filename); - - /* Fix endianness in symbols */ - for (sym = info->symtab_start; sym < info->symtab_stop; sym++) { - sym->st_shndx = TO_NATIVE(endian, 16, sym->st_shndx); - sym->st_name = TO_NATIVE(endian, 32, sym->st_name); - sym->st_value = TO_NATIVE(endian, ADDR_SIZE, sym->st_value); - sym->st_size = TO_NATIVE(endian, ADDR_SIZE, sym->st_size); + else { + /* Fix endianness in symbols */ + for (sym = info->symtab_start; sym < info->symtab_stop; sym++) { + sym->st_shndx = TO_NATIVE(endian, 16, sym->st_shndx); + sym->st_name = TO_NATIVE(endian, 32, sym->st_name); + sym->st_value = TO_NATIVE(endian, ADDR_SIZE, sym->st_value); + sym->st_size = TO_NATIVE(endian, ADDR_SIZE, sym->st_size); + } } if (symtab_shndx_idx != ~0U) { @@ -269,6 +271,7 @@ struct opt_tag { static const struct opt_tag opt_tags[] = { {"_param_string_export", "params"}, + {"_kmod_dep_export", "kmod"}, }; static int complete_pmd_entry(struct elf_info *info, struct pmd_driver *drv) @@ -325,6 +328,10 @@ static int locate_pmd_entries(struct elf_info *info) do { new = calloc(sizeof(struct pmd_driver), 1); + if (new == NULL) { + fprintf(stderr, "Failed to calloc memory\n"); + return -1; + } new->name_sym = find_sym_in_symtab(info, "this_pmd_name", last); last = new->name_sym; if (!new->name_sym) @@ -395,7 +402,7 @@ static void output_pmd_info_string(struct elf_info *info, char *outfile) int main(int argc, char **argv) { - struct elf_info info; + struct elf_info info = {0}; int rc = 1; if (argc < 3) { @@ -406,7 +413,8 @@ int main(int argc, char **argv) } parse_elf(&info, argv[1]); - locate_pmd_entries(&info); + if (locate_pmd_entries(&info) < 0) + exit(1); if (info.drivers) { output_pmd_info_string(&info, argv[2]); diff --git a/dpdk/buildtools/pmdinfogen/pmdinfogen.h b/dpdk/buildtools/pmdinfogen/pmdinfogen.h index 1da2966f..27bab30e 100644 --- a/dpdk/buildtools/pmdinfogen/pmdinfogen.h +++ b/dpdk/buildtools/pmdinfogen/pmdinfogen.h @@ -16,12 +16,16 @@ #include #include #include +#ifdef __linux__ +#include +#else +#include +#endif #include #include #include #include #include -#include /* On BSD-alike OSes elf.h defines these according to host's word size */ #undef ELF_ST_BIND @@ -75,9 +79,9 @@ #define CONVERT_NATIVE(fend, width, x) ({ \ typeof(x) ___x; \ if ((fend) == ELFDATA2LSB) \ - ___x = rte_le_to_cpu_##width(x); \ + ___x = le##width##toh(x); \ else \ - ___x = rte_be_to_cpu_##width(x); \ + ___x = be##width##toh(x); \ ___x; \ }) @@ -85,6 +89,7 @@ else \ enum opt_params { PMD_PARAM_STRING = 0, + PMD_KMOD_DEP, PMD_OPT_MAX }; diff --git a/dpdk/buildtools/relpath.sh b/dpdk/buildtools/relpath.sh new file mode 100644 index 00000000..139b7813 --- /dev/null +++ b/dpdk/buildtools/relpath.sh @@ -0,0 +1,105 @@ +#!/bin/sh + +# BSD LICENSE +# +# Copyright(c) 2010-2014 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# +# print the relative path of $1 from $2 directory +# $1 and $2 MUST be absolute paths +# + +if [ $# -ne 2 ]; then + echo "Bad arguments" + echo "Usage:" + echo " $0 path1 path2" + exit 1 +fi + +# get the real absolute path, derefencing symlinks +ABS1="$(dirname $(readlink -f $1))/$(basename $1)" +ABS2=$(readlink -f $2) + +# remove leading slash +REL1=${ABS1#/} +REL2=${ABS2#/} + +left1=${REL1%%/*} +right1=${REL1#*/} +prev_right1=$REL1 +prev_left1= + +left2=${REL2%%/*} +right2=${REL2#*/} +prev_right2=$REL2 +prev_left2= + +prefix= + +while [ "${right1}" != "" -a "${right2}" != "" ]; do + + if [ "$left1" != "$left2" ]; then + break + fi + + prev_left1=$left1 + left1=$left1/${right1%%/*} + prev_right1=$right1 + right1=${prev_right1#*/} + if [ "$right1" = "$prev_right1" ]; then + right1="" + fi + + prev_left2=$left2 + left2=$left2/${right2%%/*} + prev_right2=$right2 + right2=${prev_right2#*/} + if [ "$right2" = "$prev_right2" ]; then + right2="" + fi +done + +if [ "${left1}" != "${left2}" ]; then + right2=${prev_right2} + right1=${prev_right1} +fi + +while [ "${right2}" != "" ]; do + prefix=${prefix}../ + prev_right2=$right2 + right2=${right2#*/} + if [ "$right2" = "$prev_right2" ]; then + right2="" + fi +done + +echo ${prefix}${right1} + +exit 0 diff --git a/dpdk/config/common_armv8a_linuxapp b/dpdk/config/common_armv8a_linuxapp new file mode 100644 index 00000000..6732d1e3 --- /dev/null +++ b/dpdk/config/common_armv8a_linuxapp @@ -0,0 +1,51 @@ +# BSD LICENSE +# +# Copyright (C) Cavium, Inc 2017. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Cavium, Inc nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +#include "common_linuxapp" + +CONFIG_RTE_MACHINE="armv8a" + +CONFIG_RTE_ARCH="arm64" +CONFIG_RTE_ARCH_ARM64=y +CONFIG_RTE_ARCH_64=y + +CONFIG_RTE_FORCE_INTRINSICS=y + +# Maximum available cache line size in arm64 implementations. +# Setting to maximum available cache line size in generic config +# to address minimum DMA alignment across all arm64 implementations. +CONFIG_RTE_CACHE_LINE_SIZE=128 + +CONFIG_RTE_LIBRTE_FM10K_PMD=n +CONFIG_RTE_LIBRTE_SFC_EFX_PMD=n +CONFIG_RTE_LIBRTE_AVP_PMD=n + +CONFIG_RTE_SCHED_VECTOR=n diff --git a/dpdk/config/common_base b/dpdk/config/common_base index 67ae6331..214d9a27 100644 --- a/dpdk/config/common_base +++ b/dpdk/config/common_base @@ -1,6 +1,6 @@ # BSD LICENSE # -# Copyright(c) 2010-2016 Intel Corporation. All rights reserved. +# Copyright(c) 2010-2017 Intel Corporation. All rights reserved. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -74,6 +74,11 @@ CONFIG_RTE_BUILD_SHARED_LIB=n # CONFIG_RTE_NEXT_ABI=y +# +# Major ABI to overwrite library specific LIBABIVER +# +CONFIG_RTE_MAJOR_ABI= + # # Machine's cache line size # @@ -88,14 +93,26 @@ CONFIG_RTE_MAX_NUMA_NODES=8 CONFIG_RTE_MAX_MEMSEG=256 CONFIG_RTE_MAX_MEMZONE=2560 CONFIG_RTE_MAX_TAILQ=32 +CONFIG_RTE_ENABLE_ASSERT=n CONFIG_RTE_LOG_LEVEL=RTE_LOG_INFO +CONFIG_RTE_LOG_DP_LEVEL=RTE_LOG_INFO CONFIG_RTE_LOG_HISTORY=256 +CONFIG_RTE_BACKTRACE=y CONFIG_RTE_LIBEAL_USE_HPET=n CONFIG_RTE_EAL_ALLOW_INV_SOCKET_ID=n CONFIG_RTE_EAL_ALWAYS_PANIC_ON_ERROR=n CONFIG_RTE_EAL_IGB_UIO=n CONFIG_RTE_EAL_VFIO=n CONFIG_RTE_MALLOC_DEBUG=n +CONFIG_RTE_EAL_NUMA_AWARE_HUGEPAGES=n + +# +# Recognize/ignore the AVX/AVX512 CPU flags for performance/power testing. +# AVX512 is marked as experimental for now, will enable it after enough +# field test and possible optimization. +# +CONFIG_RTE_ENABLE_AVX=y +CONFIG_RTE_ENABLE_AVX512=n # Default driver path (or "" to disable) CONFIG_RTE_EAL_PMD_PATH="" @@ -105,6 +122,11 @@ CONFIG_RTE_EAL_PMD_PATH="" # CONFIG_RTE_LIBRTE_EAL_VMWARE_TSC_MAP_SUPPORT=y +# +# Compile the PCI library +# +CONFIG_RTE_LIBRTE_PCI=y + # # Compile the argument parser library # @@ -120,11 +142,25 @@ CONFIG_RTE_MAX_QUEUES_PER_PORT=1024 CONFIG_RTE_LIBRTE_IEEE1588=n CONFIG_RTE_ETHDEV_QUEUE_STAT_CNTRS=16 CONFIG_RTE_ETHDEV_RXTX_CALLBACKS=y +CONFIG_RTE_ETHDEV_PROFILE_ITT_WASTED_RX_ITERATIONS=n # -# Support NIC bypass logic +# Turn off Tx preparation stage # -CONFIG_RTE_NIC_BYPASS=n +# Warning: rte_eth_tx_prepare() can be safely disabled only if using a +# driver which do not implement any Tx preparation. +# +CONFIG_RTE_ETHDEV_TX_PREPARE_NOOP=n + +# +# Compile PCI bus driver +# +CONFIG_RTE_LIBRTE_PCI_BUS=y + +# +# Compile the vdev bus +# +CONFIG_RTE_LIBRTE_VDEV_BUS=y # # Compile burst-oriented Amazon ENA PMD driver @@ -159,20 +195,17 @@ CONFIG_RTE_LIBRTE_IXGBE_DEBUG_TX_FREE=n CONFIG_RTE_LIBRTE_IXGBE_DEBUG_DRIVER=n CONFIG_RTE_LIBRTE_IXGBE_PF_DISABLE_STRIP_CRC=n CONFIG_RTE_IXGBE_INC_VECTOR=y -CONFIG_RTE_IXGBE_RX_OLFLAGS_ENABLE=y +CONFIG_RTE_LIBRTE_IXGBE_BYPASS=n # # Compile burst-oriented I40E PMD driver # CONFIG_RTE_LIBRTE_I40E_PMD=y -CONFIG_RTE_LIBRTE_I40E_DEBUG_INIT=n CONFIG_RTE_LIBRTE_I40E_DEBUG_RX=n CONFIG_RTE_LIBRTE_I40E_DEBUG_TX=n CONFIG_RTE_LIBRTE_I40E_DEBUG_TX_FREE=n -CONFIG_RTE_LIBRTE_I40E_DEBUG_DRIVER=n CONFIG_RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC=y CONFIG_RTE_LIBRTE_I40E_INC_VECTOR=y -CONFIG_RTE_LIBRTE_I40E_RX_OLFLAGS_ENABLE=y CONFIG_RTE_LIBRTE_I40E_16BYTE_RX_DESC=n CONFIG_RTE_LIBRTE_I40E_QUEUE_NUM_PER_PF=64 CONFIG_RTE_LIBRTE_I40E_QUEUE_NUM_PER_VF=4 @@ -197,13 +230,10 @@ CONFIG_RTE_LIBRTE_FM10K_INC_VECTOR=y # CONFIG_RTE_LIBRTE_MLX4_PMD=n CONFIG_RTE_LIBRTE_MLX4_DEBUG=n -CONFIG_RTE_LIBRTE_MLX4_SGE_WR_N=4 -CONFIG_RTE_LIBRTE_MLX4_MAX_INLINE=0 CONFIG_RTE_LIBRTE_MLX4_TX_MP_CACHE=8 -CONFIG_RTE_LIBRTE_MLX4_SOFT_COUNTERS=1 # -# Compile burst-oriented Mellanox ConnectX-4 (MLX5) PMD +# Compile burst-oriented Mellanox ConnectX-4 & ConnectX-5 (MLX5) PMD # CONFIG_RTE_LIBRTE_MLX5_PMD=n CONFIG_RTE_LIBRTE_MLX5_DEBUG=n @@ -221,7 +251,7 @@ CONFIG_RTE_LIBRTE_BNX2X_MF_SUPPORT=n CONFIG_RTE_LIBRTE_BNX2X_DEBUG_PERIODIC=n # -# Compile burst-oriented Chelsio Terminator 10GbE/40GbE (CXGBE) PMD +# Compile burst-oriented Chelsio Terminator (CXGBE) PMD # CONFIG_RTE_LIBRTE_CXGBE_PMD=y CONFIG_RTE_LIBRTE_CXGBE_DEBUG=n @@ -229,12 +259,14 @@ CONFIG_RTE_LIBRTE_CXGBE_DEBUG_REG=n CONFIG_RTE_LIBRTE_CXGBE_DEBUG_MBOX=n CONFIG_RTE_LIBRTE_CXGBE_DEBUG_TX=n CONFIG_RTE_LIBRTE_CXGBE_DEBUG_RX=n +CONFIG_RTE_LIBRTE_CXGBE_TPUT=y # # Compile burst-oriented Cisco ENIC PMD driver # CONFIG_RTE_LIBRTE_ENIC_PMD=y CONFIG_RTE_LIBRTE_ENIC_DEBUG=n +CONFIG_RTE_LIBRTE_ENIC_DEBUG_FLOW=n # # Compile burst-oriented Netronome NFP PMD driver @@ -242,35 +274,94 @@ CONFIG_RTE_LIBRTE_ENIC_DEBUG=n CONFIG_RTE_LIBRTE_NFP_PMD=n CONFIG_RTE_LIBRTE_NFP_DEBUG=n +# +# Compile Marvell PMD driver +# +CONFIG_RTE_LIBRTE_MRVL_PMD=n + # # Compile burst-oriented Broadcom BNXT PMD driver # CONFIG_RTE_LIBRTE_BNXT_PMD=y +# +# Compile burst-oriented Solarflare libefx-based PMD +# +CONFIG_RTE_LIBRTE_SFC_EFX_PMD=y +CONFIG_RTE_LIBRTE_SFC_EFX_DEBUG=n + +# +# Compile SOFTNIC PMD +# +CONFIG_RTE_LIBRTE_PMD_SOFTNIC=y + # # Compile software PMD backed by SZEDATA2 device # CONFIG_RTE_LIBRTE_PMD_SZEDATA2=n # # Defines firmware type address space. -# RTE_LIBRTE_PMD_SZEDATA2_AS can be: -# 0 - for firmwares: -# NIC_100G1_LR4 -# HANIC_100G1_LR4 -# HANIC_100G1_SR10 -# Other values raise compile time error +# See documentation for supported values. +# Other values raise compile time error. CONFIG_RTE_LIBRTE_PMD_SZEDATA2_AS=0 # # Compile burst-oriented Cavium Thunderx NICVF PMD driver # -CONFIG_RTE_LIBRTE_THUNDERX_NICVF_PMD=n +CONFIG_RTE_LIBRTE_THUNDERX_NICVF_PMD=y CONFIG_RTE_LIBRTE_THUNDERX_NICVF_DEBUG_INIT=n CONFIG_RTE_LIBRTE_THUNDERX_NICVF_DEBUG_RX=n CONFIG_RTE_LIBRTE_THUNDERX_NICVF_DEBUG_TX=n CONFIG_RTE_LIBRTE_THUNDERX_NICVF_DEBUG_DRIVER=n CONFIG_RTE_LIBRTE_THUNDERX_NICVF_DEBUG_MBOX=n +# +# Compile burst-oriented Cavium LiquidIO PMD driver +# +CONFIG_RTE_LIBRTE_LIO_PMD=y +CONFIG_RTE_LIBRTE_LIO_DEBUG_DRIVER=n +CONFIG_RTE_LIBRTE_LIO_DEBUG_INIT=n +CONFIG_RTE_LIBRTE_LIO_DEBUG_RX=n +CONFIG_RTE_LIBRTE_LIO_DEBUG_TX=n +CONFIG_RTE_LIBRTE_LIO_DEBUG_MBOX=n +CONFIG_RTE_LIBRTE_LIO_DEBUG_REGS=n + +# NXP DPAA Bus +CONFIG_RTE_LIBRTE_DPAA_BUS=n +CONFIG_RTE_LIBRTE_DPAA_MEMPOOL=n +CONFIG_RTE_LIBRTE_DPAA_PMD=n + +# +# Compile burst-oriented Cavium OCTEONTX network PMD driver +# +CONFIG_RTE_LIBRTE_OCTEONTX_PMD=y +CONFIG_RTE_LIBRTE_OCTEONTX_DEBUG_INIT=n +CONFIG_RTE_LIBRTE_OCTEONTX_DEBUG_RX=n +CONFIG_RTE_LIBRTE_OCTEONTX_DEBUG_TX=n +CONFIG_RTE_LIBRTE_OCTEONTX_DEBUG_DRIVER=n +CONFIG_RTE_LIBRTE_OCTEONTX_DEBUG_MBOX=n + +# +# Compile NXP DPAA2 FSL-MC Bus +# +CONFIG_RTE_LIBRTE_FSLMC_BUS=n + +# +# Compile Support Libraries for NXP DPAA2 +# +CONFIG_RTE_LIBRTE_DPAA2_MEMPOOL=n +CONFIG_RTE_LIBRTE_DPAA2_USE_PHYS_IOVA=y + +# +# Compile burst-oriented NXP DPAA2 PMD driver +# +CONFIG_RTE_LIBRTE_DPAA2_PMD=n +CONFIG_RTE_LIBRTE_DPAA2_DEBUG_INIT=n +CONFIG_RTE_LIBRTE_DPAA2_DEBUG_DRIVER=n +CONFIG_RTE_LIBRTE_DPAA2_DEBUG_RX=n +CONFIG_RTE_LIBRTE_DPAA2_DEBUG_TX=n +CONFIG_RTE_LIBRTE_DPAA2_DEBUG_TX_FREE=n + # # Compile burst-oriented VIRTIO PMD driver # @@ -315,9 +406,9 @@ CONFIG_RTE_LIBRTE_PMD_BOND=y CONFIG_RTE_LIBRTE_BOND_DEBUG_ALB=n CONFIG_RTE_LIBRTE_BOND_DEBUG_ALB_L1=n -# QLogic 25G/40G/100G PMD +# QLogic 10G/25G/40G/50G/100G PMD # -CONFIG_RTE_LIBRTE_QEDE_PMD=n +CONFIG_RTE_LIBRTE_QEDE_PMD=y CONFIG_RTE_LIBRTE_QEDE_DEBUG_INIT=n CONFIG_RTE_LIBRTE_QEDE_DEBUG_INFO=n CONFIG_RTE_LIBRTE_QEDE_DEBUG_DRIVER=n @@ -333,15 +424,40 @@ CONFIG_RTE_LIBRTE_QEDE_FW="" CONFIG_RTE_LIBRTE_PMD_AF_PACKET=n # -# Compile Xen PMD +# Compile ARK PMD # -CONFIG_RTE_LIBRTE_PMD_XENVIRT=n +CONFIG_RTE_LIBRTE_ARK_PMD=y +CONFIG_RTE_LIBRTE_ARK_PAD_TX=y +CONFIG_RTE_LIBRTE_ARK_DEBUG_RX=n +CONFIG_RTE_LIBRTE_ARK_DEBUG_TX=n +CONFIG_RTE_LIBRTE_ARK_DEBUG_STATS=n +CONFIG_RTE_LIBRTE_ARK_DEBUG_TRACE=n + +# +# Compile WRS accelerated virtual port (AVP) guest PMD driver +# +CONFIG_RTE_LIBRTE_AVP_PMD=n +CONFIG_RTE_LIBRTE_AVP_DEBUG_RX=n +CONFIG_RTE_LIBRTE_AVP_DEBUG_TX=n +CONFIG_RTE_LIBRTE_AVP_DEBUG_DRIVER=y +CONFIG_RTE_LIBRTE_AVP_DEBUG_BUFFERS=n + +# +# Compile the TAP PMD +# It is enabled by default for Linux only. +# +CONFIG_RTE_LIBRTE_PMD_TAP=n # # Compile null PMD # CONFIG_RTE_LIBRTE_PMD_NULL=y +# +# Compile fail-safe PMD +# +CONFIG_RTE_LIBRTE_PMD_FAILSAFE=y + # # Do prefetch of packet data within PMD driver receive function # @@ -355,6 +471,28 @@ CONFIG_RTE_LIBRTE_CRYPTODEV_DEBUG=n CONFIG_RTE_CRYPTO_MAX_DEVS=64 CONFIG_RTE_CRYPTODEV_NAME_LEN=64 +# +# Compile PMD for ARMv8 Crypto device +# +CONFIG_RTE_LIBRTE_PMD_ARMV8_CRYPTO=n +CONFIG_RTE_LIBRTE_PMD_ARMV8_CRYPTO_DEBUG=n + +# +# Compile NXP DPAA2 crypto sec driver for CAAM HW +# +CONFIG_RTE_LIBRTE_PMD_DPAA2_SEC=n +CONFIG_RTE_LIBRTE_DPAA2_SEC_DEBUG_INIT=n +CONFIG_RTE_LIBRTE_DPAA2_SEC_DEBUG_DRIVER=n +CONFIG_RTE_LIBRTE_DPAA2_SEC_DEBUG_RX=n + +# +# NXP DPAA caam - crypto driver +# +CONFIG_RTE_LIBRTE_PMD_DPAA_SEC=n +CONFIG_RTE_LIBRTE_DPAA_SEC_DEBUG_INIT=n +CONFIG_RTE_LIBRTE_DPAA_SEC_DEBUG_DRIVER=n +CONFIG_RTE_LIBRTE_DPAA_SEC_DEBUG_RX=n + # # Compile PMD for QuickAssist based devices # @@ -375,6 +513,12 @@ CONFIG_RTE_QAT_PMD_MAX_NB_SESSIONS=2048 CONFIG_RTE_LIBRTE_PMD_AESNI_MB=n CONFIG_RTE_LIBRTE_PMD_AESNI_MB_DEBUG=n +# +# Compile PMD for Software backed device +# +CONFIG_RTE_LIBRTE_PMD_OPENSSL=n +CONFIG_RTE_LIBRTE_PMD_OPENSSL_DEBUG=n + # # Compile PMD for AESNI GCM device # @@ -393,18 +537,64 @@ CONFIG_RTE_LIBRTE_PMD_SNOW3G_DEBUG=n CONFIG_RTE_LIBRTE_PMD_KASUMI=n CONFIG_RTE_LIBRTE_PMD_KASUMI_DEBUG=n +# +# Compile PMD for ZUC device +# +CONFIG_RTE_LIBRTE_PMD_ZUC=n +CONFIG_RTE_LIBRTE_PMD_ZUC_DEBUG=n + +# +# Compile PMD for Crypto Scheduler device +# +CONFIG_RTE_LIBRTE_PMD_CRYPTO_SCHEDULER=y +CONFIG_RTE_LIBRTE_PMD_CRYPTO_SCHEDULER_DEBUG=n + # # Compile PMD for NULL Crypto device # CONFIG_RTE_LIBRTE_PMD_NULL_CRYPTO=y +# +# Compile PMD for Marvell Crypto device +# +CONFIG_RTE_LIBRTE_PMD_MRVL_CRYPTO=n +CONFIG_RTE_LIBRTE_PMD_MRVL_CRYPTO_DEBUG=n + +# +# Compile generic security library +# +CONFIG_RTE_LIBRTE_SECURITY=y + +# +# Compile generic event device library +# +CONFIG_RTE_LIBRTE_EVENTDEV=y +CONFIG_RTE_LIBRTE_EVENTDEV_DEBUG=n +CONFIG_RTE_EVENT_MAX_DEVS=16 +CONFIG_RTE_EVENT_MAX_QUEUES_PER_DEV=64 + +# +# Compile PMD for skeleton event device +# +CONFIG_RTE_LIBRTE_PMD_SKELETON_EVENTDEV=y +CONFIG_RTE_LIBRTE_PMD_SKELETON_EVENTDEV_DEBUG=n + +# +# Compile PMD for software event device +# +CONFIG_RTE_LIBRTE_PMD_SW_EVENTDEV=y +CONFIG_RTE_LIBRTE_PMD_SW_EVENTDEV_DEBUG=n + +# +# Compile PMD for octeontx sso event device +# +CONFIG_RTE_LIBRTE_PMD_OCTEONTX_SSOVF=y +CONFIG_RTE_LIBRTE_PMD_OCTEONTX_SSOVF_DEBUG=n + # # Compile librte_ring # CONFIG_RTE_LIBRTE_RING=y -CONFIG_RTE_LIBRTE_RING_DEBUG=n -CONFIG_RTE_RING_SPLIT_PROD_CONS=n -CONFIG_RTE_RING_PAUSE_REP_COUNT=0 # # Compile librte_mempool @@ -413,6 +603,18 @@ CONFIG_RTE_LIBRTE_MEMPOOL=y CONFIG_RTE_MEMPOOL_CACHE_MAX_SIZE=512 CONFIG_RTE_LIBRTE_MEMPOOL_DEBUG=n +# +# Compile Mempool drivers +# +CONFIG_RTE_DRIVER_MEMPOOL_RING=y +CONFIG_RTE_DRIVER_MEMPOOL_STACK=y + +# +# Compile PMD for octeontx fpa mempool device +# +CONFIG_RTE_LIBRTE_OCTEONTX_MEMPOOL=y +CONFIG_RTE_LIBRTE_OCTEONTX_MEMPOOL_DEBUG=n + # # Compile librte_mbuf # @@ -445,11 +647,36 @@ CONFIG_RTE_LIBRTE_CMDLINE_DEBUG=n CONFIG_RTE_LIBRTE_HASH=y CONFIG_RTE_LIBRTE_HASH_DEBUG=n +# +# Compile librte_efd +# +CONFIG_RTE_LIBRTE_EFD=y + +# +# Compile librte_member +# +CONFIG_RTE_LIBRTE_MEMBER=y + # # Compile librte_jobstats # CONFIG_RTE_LIBRTE_JOBSTATS=y +# +# Compile the device metrics library +# +CONFIG_RTE_LIBRTE_METRICS=y + +# +# Compile the bitrate statistics library +# +CONFIG_RTE_LIBRTE_BITRATE=y + +# +# Compile the latency statistics library +# +CONFIG_RTE_LIBRTE_LATENCY_STATS=y + # # Compile librte_lpm # @@ -482,11 +709,26 @@ CONFIG_RTE_LIBRTE_IP_FRAG_DEBUG=n CONFIG_RTE_LIBRTE_IP_FRAG_MAX_FRAG=4 CONFIG_RTE_LIBRTE_IP_FRAG_TBL_STAT=n +# +# Compile GRO library +# +CONFIG_RTE_LIBRTE_GRO=y + +# +# Compile GSO library +# +CONFIG_RTE_LIBRTE_GSO=y + # # Compile librte_meter # CONFIG_RTE_LIBRTE_METER=y +# +# Compile librte_classify +# +CONFIG_RTE_LIBRTE_FLOW_CLASSIFY=y + # # Compile librte_sched # @@ -531,14 +773,10 @@ CONFIG_RTE_PIPELINE_STATS_COLLECT=n # Compile librte_kni # CONFIG_RTE_LIBRTE_KNI=n +CONFIG_RTE_LIBRTE_PMD_KNI=n CONFIG_RTE_KNI_KMOD=n +CONFIG_RTE_KNI_KMOD_ETHTOOL=n CONFIG_RTE_KNI_PREEMPT_DEFAULT=y -CONFIG_RTE_KNI_KO_DEBUG=n -CONFIG_RTE_KNI_VHOST=n -CONFIG_RTE_KNI_VHOST_MAX_CACHE_SIZE=1024 -CONFIG_RTE_KNI_VHOST_VNET_HDR_EN=n -CONFIG_RTE_KNI_VHOST_DEBUG_RX=n -CONFIG_RTE_KNI_VHOST_DEBUG_TX=n # # Compile the pdump library @@ -546,13 +784,9 @@ CONFIG_RTE_KNI_VHOST_DEBUG_TX=n CONFIG_RTE_LIBRTE_PDUMP=y # -# Compile vhost library -# fuse-devel is needed to run vhost-cuse. -# fuse-devel enables user space char driver development -# vhost-user is turned on by default. +# Compile vhost user library # CONFIG_RTE_LIBRTE_VHOST=n -CONFIG_RTE_LIBRTE_VHOST_USER=y CONFIG_RTE_LIBRTE_VHOST_NUMA=n CONFIG_RTE_LIBRTE_VHOST_DEBUG=n @@ -562,25 +796,30 @@ CONFIG_RTE_LIBRTE_VHOST_DEBUG=n # CONFIG_RTE_LIBRTE_PMD_VHOST=n -# -#Compile Xen domain0 support -# -CONFIG_RTE_LIBRTE_XEN_DOM0=n - -# -# Enable warning directives -# -CONFIG_RTE_INSECURE_FUNCTION_WARNING=n - # # Compile the test application # -CONFIG_RTE_APP_TEST=n +CONFIG_RTE_APP_TEST=y CONFIG_RTE_APP_TEST_RESOURCE_TAR=n +# +# Compile the procinfo application +# +CONFIG_RTE_PROC_INFO=n + # # Compile the PMD test application # CONFIG_RTE_TEST_PMD=y CONFIG_RTE_TEST_PMD_RECORD_CORE_CYCLES=n CONFIG_RTE_TEST_PMD_RECORD_BURST_STATS=n + +# +# Compile the crypto performance application +# +CONFIG_RTE_APP_CRYPTO_PERF=y + +# +# Compile the eventdev application +# +CONFIG_RTE_APP_EVENTDEV=y diff --git a/dpdk/config/common_linuxapp b/dpdk/config/common_linuxapp index 2483dfa5..15c69616 100644 --- a/dpdk/config/common_linuxapp +++ b/dpdk/config/common_linuxapp @@ -35,12 +35,19 @@ CONFIG_RTE_EXEC_ENV="linuxapp" CONFIG_RTE_EXEC_ENV_LINUXAPP=y +CONFIG_RTE_EAL_NUMA_AWARE_HUGEPAGES=y CONFIG_RTE_EAL_IGB_UIO=y CONFIG_RTE_EAL_VFIO=y CONFIG_RTE_KNI_KMOD=y CONFIG_RTE_LIBRTE_KNI=y +CONFIG_RTE_LIBRTE_PMD_KNI=y CONFIG_RTE_LIBRTE_VHOST=y +CONFIG_RTE_LIBRTE_VHOST_NUMA=y CONFIG_RTE_LIBRTE_PMD_VHOST=y CONFIG_RTE_LIBRTE_PMD_AF_PACKET=y +CONFIG_RTE_LIBRTE_PMD_TAP=y +CONFIG_RTE_LIBRTE_AVP_PMD=y +CONFIG_RTE_LIBRTE_NFP_PMD=y CONFIG_RTE_LIBRTE_POWER=y CONFIG_RTE_VIRTIO_USER=y +CONFIG_RTE_PROC_INFO=y diff --git a/dpdk/config/defconfig_arm-armv7a-linuxapp-gcc b/dpdk/config/defconfig_arm-armv7a-linuxapp-gcc index bde6acd9..a20b7a85 100644 --- a/dpdk/config/defconfig_arm-armv7a-linuxapp-gcc +++ b/dpdk/config/defconfig_arm-armv7a-linuxapp-gcc @@ -47,6 +47,10 @@ CONFIG_RTE_ARCH_STRICT_ALIGN=y CONFIG_RTE_TOOLCHAIN="gcc" CONFIG_RTE_TOOLCHAIN_GCC=y +# NUMA is not supported on ARM +CONFIG_RTE_EAL_NUMA_AWARE_HUGEPAGES=n +CONFIG_RTE_LIBRTE_VHOST_NUMA=n + # ARM doesn't have support for vmware TSC map CONFIG_RTE_LIBRTE_EAL_VMWARE_TSC_MAP_SUPPORT=n @@ -61,6 +65,7 @@ CONFIG_RTE_SCHED_VECTOR=n # cannot use those on ARM CONFIG_RTE_KNI_KMOD=n +CONFIG_RTE_LIBRTE_ARK_PMD=n CONFIG_RTE_LIBRTE_EM_PMD=n CONFIG_RTE_LIBRTE_IGB_PMD=n CONFIG_RTE_LIBRTE_CXGBE_PMD=n @@ -70,8 +75,8 @@ CONFIG_RTE_LIBRTE_FM10K_PMD=n CONFIG_RTE_LIBRTE_I40E_PMD=n CONFIG_RTE_LIBRTE_IXGBE_PMD=n CONFIG_RTE_LIBRTE_MLX4_PMD=n -CONFIG_RTE_LIBRTE_MPIPE_PMD=n CONFIG_RTE_LIBRTE_VMXNET3_PMD=n -CONFIG_RTE_LIBRTE_PMD_XENVIRT=n -CONFIG_RTE_LIBRTE_PMD_BNX2X=n +CONFIG_RTE_LIBRTE_BNX2X_PMD=n CONFIG_RTE_LIBRTE_QEDE_PMD=n +CONFIG_RTE_LIBRTE_SFC_EFX_PMD=n +CONFIG_RTE_LIBRTE_AVP_PMD=n diff --git a/dpdk/config/defconfig_arm64-armv8a-linuxapp-clang b/dpdk/config/defconfig_arm64-armv8a-linuxapp-clang new file mode 100644 index 00000000..91c0ec74 --- /dev/null +++ b/dpdk/config/defconfig_arm64-armv8a-linuxapp-clang @@ -0,0 +1,35 @@ +# BSD LICENSE +# +# Copyright (C) Cavium, Inc 2017. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Cavium, Inc nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +#include "common_armv8a_linuxapp" + +CONFIG_RTE_TOOLCHAIN="clang" +CONFIG_RTE_TOOLCHAIN_CLANG=y diff --git a/dpdk/config/defconfig_arm64-armv8a-linuxapp-gcc b/dpdk/config/defconfig_arm64-armv8a-linuxapp-gcc index 1a171266..9775ca11 100644 --- a/dpdk/config/defconfig_arm64-armv8a-linuxapp-gcc +++ b/dpdk/config/defconfig_arm64-armv8a-linuxapp-gcc @@ -1,6 +1,6 @@ # BSD LICENSE # -# Copyright (C) Cavium networks 2015. All rights reserved. +# Copyright (C) Cavium, Inc 2015. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -12,7 +12,7 @@ # notice, this list of conditions and the following disclaimer in # the documentation and/or other materials provided with the # distribution. -# * Neither the name of Cavium networks nor the names of its +# * Neither the name of Cavium, Inc nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # @@ -29,23 +29,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # -#include "common_linuxapp" - -CONFIG_RTE_MACHINE="armv8a" - -CONFIG_RTE_ARCH="arm64" -CONFIG_RTE_ARCH_ARM64=y -CONFIG_RTE_ARCH_64=y - -CONFIG_RTE_FORCE_INTRINSICS=y +#include "common_armv8a_linuxapp" CONFIG_RTE_TOOLCHAIN="gcc" CONFIG_RTE_TOOLCHAIN_GCC=y - -CONFIG_RTE_EAL_IGB_UIO=n - -CONFIG_RTE_LIBRTE_IVSHMEM=n -CONFIG_RTE_LIBRTE_FM10K_PMD=n -CONFIG_RTE_LIBRTE_I40E_PMD=n - -CONFIG_RTE_SCHED_VECTOR=n diff --git a/dpdk/config/defconfig_arm64-dpaa-linuxapp-gcc b/dpdk/config/defconfig_arm64-dpaa-linuxapp-gcc new file mode 100644 index 00000000..e577432f --- /dev/null +++ b/dpdk/config/defconfig_arm64-dpaa-linuxapp-gcc @@ -0,0 +1,76 @@ +# BSD LICENSE +# +# Copyright 2016 Freescale Semiconductor, Inc. +# Copyright 2017 NXP. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of NXP nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +#include "defconfig_arm64-armv8a-linuxapp-gcc" + +# NXP (Freescale) - Soc Architecture with FMAN, QMAN & BMAN support +CONFIG_RTE_MACHINE="dpaa" +CONFIG_RTE_ARCH_ARM_TUNE="cortex-a72" +CONFIG_RTE_LIBRTE_VHOST_NUMA=n +CONFIG_RTE_EAL_NUMA_AWARE_HUGEPAGES=n + +# +# Compile Environment Abstraction Layer +# +CONFIG_RTE_MAX_LCORE=4 +CONFIG_RTE_MAX_NUMA_NODES=1 +CONFIG_RTE_CACHE_LINE_SIZE=64 +CONFIG_RTE_PKTMBUF_HEADROOM=128 + +# NXP DPAA Bus +CONFIG_RTE_LIBRTE_DPAA_BUS=y +CONFIG_RTE_LIBRTE_DPAA_DEBUG_DRIVER=n +CONFIG_RTE_LIBRTE_DPAA_HWDEBUG=n + +# NXP DPAA Mempool +CONFIG_RTE_LIBRTE_DPAA_MEMPOOL=y +CONFIG_RTE_MBUF_DEFAULT_MEMPOOL_OPS="dpaa" + +# Compile software NXP DPAA PMD +CONFIG_RTE_LIBRTE_DPAA_PMD=y + +# +# FSL DPAA caam - crypto driver +# +CONFIG_RTE_LIBRTE_PMD_DPAA_SEC=y +CONFIG_RTE_LIBRTE_DPAA_SEC_DEBUG_INIT=n +CONFIG_RTE_LIBRTE_DPAA_SEC_DEBUG_DRIVER=n +CONFIG_RTE_LIBRTE_DPAA_SEC_DEBUG_RX=n + +# DPAA CAAM driver instances +CONFIG_RTE_LIBRTE_DPAA_MAX_CRYPTODEV=4 + +# +# Number of sessions to create in the session memory pool +# on a single DPAA SEC device. +# +CONFIG_RTE_DPAA_SEC_PMD_MAX_NB_SESSIONS=2048 diff --git a/dpdk/config/defconfig_arm64-dpaa2-linuxapp-gcc b/dpdk/config/defconfig_arm64-dpaa2-linuxapp-gcc index 66df54c5..91f4993d 100644 --- a/dpdk/config/defconfig_arm64-dpaa2-linuxapp-gcc +++ b/dpdk/config/defconfig_arm64-dpaa2-linuxapp-gcc @@ -1,6 +1,7 @@ # BSD LICENSE # -# Copyright(c) 2016 Freescale Semiconductor, Inc. All rights reserved. +# Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. +# Copyright 2016 NXP. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -33,10 +34,59 @@ # NXP (Freescale) - Soc Architecture with WRIOP and QBMAN support CONFIG_RTE_MACHINE="dpaa2" -CONFIG_RTE_ARCH_ARM_TUNE="cortex-a57+fp+simd" +CONFIG_RTE_ARCH_ARM_TUNE="cortex-a72" # # Compile Environment Abstraction Layer # -CONFIG_RTE_MAX_LCORE=8 +CONFIG_RTE_MAX_LCORE=16 CONFIG_RTE_MAX_NUMA_NODES=1 +CONFIG_RTE_CACHE_LINE_SIZE=64 + +CONFIG_RTE_PKTMBUF_HEADROOM=256 + +# Doesn't support NUMA +CONFIG_RTE_EAL_NUMA_AWARE_HUGEPAGES=n +CONFIG_RTE_LIBRTE_VHOST_NUMA=n + +# +# Compile Support Libraries for DPAA2 +# +CONFIG_RTE_LIBRTE_DPAA2_MEMPOOL=y +CONFIG_RTE_MBUF_DEFAULT_MEMPOOL_OPS="dpaa2" +CONFIG_RTE_LIBRTE_DPAA2_USE_PHYS_IOVA=n + +# +# Compile NXP DPAA2 FSL-MC Bus +# +CONFIG_RTE_LIBRTE_FSLMC_BUS=y + +# +# Compile burst-oriented NXP DPAA2 PMD driver +# +CONFIG_RTE_LIBRTE_DPAA2_PMD=y +CONFIG_RTE_LIBRTE_DPAA2_DEBUG_INIT=n +CONFIG_RTE_LIBRTE_DPAA2_DEBUG_DRIVER=n +CONFIG_RTE_LIBRTE_DPAA2_DEBUG_RX=n +CONFIG_RTE_LIBRTE_DPAA2_DEBUG_TX=n +CONFIG_RTE_LIBRTE_DPAA2_DEBUG_TX_FREE=n + +# +# Compile NXP DPAA2 crypto sec driver for CAAM HW +# +CONFIG_RTE_LIBRTE_PMD_DPAA2_SEC=y +CONFIG_RTE_LIBRTE_DPAA2_SEC_DEBUG_INIT=n +CONFIG_RTE_LIBRTE_DPAA2_SEC_DEBUG_DRIVER=n +CONFIG_RTE_LIBRTE_DPAA2_SEC_DEBUG_RX=n + +# +# Number of sessions to create in the session memory pool +# on a single DPAA2 SEC device. +# +CONFIG_RTE_DPAA2_SEC_PMD_MAX_NB_SESSIONS=2048 + +# +# Compile schedule-oriented NXP DPAA2 EVENTDEV driver +# +CONFIG_RTE_LIBRTE_PMD_DPAA2_EVENTDEV=y +CONFIG_RTE_LIBRTE_PMD_DPAA2_EVENTDEV_DEBUG=n diff --git a/dpdk/config/defconfig_arm64-thunderx-linuxapp-gcc b/dpdk/config/defconfig_arm64-thunderx-linuxapp-gcc index a5b1e24c..45038b11 100644 --- a/dpdk/config/defconfig_arm64-thunderx-linuxapp-gcc +++ b/dpdk/config/defconfig_arm64-thunderx-linuxapp-gcc @@ -1,6 +1,6 @@ # BSD LICENSE # -# Copyright (C) Cavium networks 2015. All rights reserved. +# Copyright (C) Cavium, Inc 2015. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -12,7 +12,7 @@ # notice, this list of conditions and the following disclaimer in # the documentation and/or other materials provided with the # distribution. -# * Neither the name of Cavium networks nor the names of its +# * Neither the name of Cavium, Inc nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # @@ -38,11 +38,7 @@ CONFIG_RTE_MAX_NUMA_NODES=2 CONFIG_RTE_MAX_LCORE=96 # -# Compile Cavium Thunderx NICVF PMD driver +# Compile PMD for octeontx sso event device # -CONFIG_RTE_LIBRTE_THUNDERX_NICVF_PMD=y -CONFIG_RTE_LIBRTE_THUNDERX_NICVF_DEBUG_INIT=n -CONFIG_RTE_LIBRTE_THUNDERX_NICVF_DEBUG_RX=n -CONFIG_RTE_LIBRTE_THUNDERX_NICVF_DEBUG_TX=n -CONFIG_RTE_LIBRTE_THUNDERX_NICVF_DEBUG_DRIVER=n -CONFIG_RTE_LIBRTE_THUNDERX_NICVF_DEBUG_MBOX=n +CONFIG_RTE_LIBRTE_PMD_OCTEONTX_SSOVF=y +CONFIG_RTE_LIBRTE_PMD_OCTEONTX_SSOVF_DEBUG=n diff --git a/dpdk/config/defconfig_arm64-xgene1-linuxapp-gcc b/dpdk/config/defconfig_arm64-xgene1-linuxapp-gcc index f096166b..19fe3517 100644 --- a/dpdk/config/defconfig_arm64-xgene1-linuxapp-gcc +++ b/dpdk/config/defconfig_arm64-xgene1-linuxapp-gcc @@ -1,6 +1,6 @@ # BSD LICENSE # -# Copyright (C) Cavium networks 2015. All rights reserved. +# Copyright (C) Cavium, Inc 2015. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -12,7 +12,7 @@ # notice, this list of conditions and the following disclaimer in # the documentation and/or other materials provided with the # distribution. -# * Neither the name of Cavium networks nor the names of its +# * Neither the name of Cavium, Inc nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # @@ -32,3 +32,4 @@ #include "defconfig_arm64-armv8a-linuxapp-gcc" CONFIG_RTE_MACHINE="xgene1" +CONFIG_RTE_CACHE_LINE_SIZE=64 diff --git a/dpdk/config/defconfig_i686-native-linuxapp-gcc b/dpdk/config/defconfig_i686-native-linuxapp-gcc index ba072592..60afe159 100644 --- a/dpdk/config/defconfig_i686-native-linuxapp-gcc +++ b/dpdk/config/defconfig_i686-native-linuxapp-gcc @@ -47,9 +47,9 @@ CONFIG_RTE_TOOLCHAIN_GCC=y CONFIG_RTE_LIBRTE_KNI=n # -# Vectorized PMD is not supported on 32-bit +# Solarflare PMD is not supported on 32-bit # -CONFIG_RTE_IXGBE_INC_VECTOR=n +CONFIG_RTE_LIBRTE_SFC_EFX_PMD=n # # AES-NI multi-buffer PMD is not supported on 32-bit @@ -65,3 +65,13 @@ CONFIG_RTE_LIBRTE_PMD_AESNI_GCM=n # KASUMI PMD is not supported on 32-bit # CONFIG_RTE_LIBRTE_PMD_KASUMI=n + +# +# ZUC PMD is not supported on 32-bit +# +CONFIG_RTE_LIBRTE_PMD_ZUC=n + +# +# AVP PMD is not supported on 32-bit +# +CONFIG_RTE_LIBRTE_AVP_PMD=n diff --git a/dpdk/config/defconfig_i686-native-linuxapp-icc b/dpdk/config/defconfig_i686-native-linuxapp-icc index 850e5364..269e88e9 100644 --- a/dpdk/config/defconfig_i686-native-linuxapp-icc +++ b/dpdk/config/defconfig_i686-native-linuxapp-icc @@ -51,6 +51,11 @@ CONFIG_RTE_LIBRTE_KNI=n # CONFIG_RTE_IXGBE_INC_VECTOR=n +# +# Solarflare PMD is not supported on 32-bit +# +CONFIG_RTE_LIBRTE_SFC_EFX_PMD=n + # # AES-NI multi-buffer PMD is not supported on 32-bit # @@ -65,3 +70,13 @@ CONFIG_RTE_LIBRTE_PMD_AESNI_GCM=n # KASUMI PMD is not supported on 32-bit # CONFIG_RTE_LIBRTE_PMD_KASUMI=n + +# +# ZUC PMD is not supported on 32-bit +# +CONFIG_RTE_LIBRTE_PMD_ZUC=n + +# +# AVP PMD is not supported on 32-bit +# +CONFIG_RTE_LIBRTE_AVP_PMD=n diff --git a/dpdk/config/defconfig_ppc_64-power8-linuxapp-gcc b/dpdk/config/defconfig_ppc_64-power8-linuxapp-gcc index bef8f49a..a52e22ef 100644 --- a/dpdk/config/defconfig_ppc_64-power8-linuxapp-gcc +++ b/dpdk/config/defconfig_ppc_64-power8-linuxapp-gcc @@ -49,17 +49,9 @@ CONFIG_RTE_LIBRTE_EAL_VMWARE_TSC_MAP_SUPPORT=n # Note: Initially, all of the PMD drivers compilation are turned off on Power # Will turn on them only after the successful testing on Power CONFIG_RTE_LIBRTE_IXGBE_PMD=n -CONFIG_RTE_LIBRTE_I40E_PMD=n CONFIG_RTE_LIBRTE_VIRTIO_PMD=y CONFIG_RTE_LIBRTE_VMXNET3_PMD=n -CONFIG_RTE_LIBRTE_PMD_BOND=n CONFIG_RTE_LIBRTE_ENIC_PMD=n CONFIG_RTE_LIBRTE_FM10K_PMD=n - -# This following libraries are not available on Power. So they're turned off. -CONFIG_RTE_LIBRTE_LPM=n -CONFIG_RTE_LIBRTE_ACL=n -CONFIG_RTE_LIBRTE_SCHED=n -CONFIG_RTE_LIBRTE_PORT=n -CONFIG_RTE_LIBRTE_TABLE=n -CONFIG_RTE_LIBRTE_PIPELINE=n +CONFIG_RTE_LIBRTE_SFC_EFX_PMD=n +CONFIG_RTE_LIBRTE_AVP_PMD=n diff --git a/dpdk/config/defconfig_tile-tilegx-linuxapp-gcc b/dpdk/config/defconfig_tile-tilegx-linuxapp-gcc deleted file mode 100644 index 5a50793d..00000000 --- a/dpdk/config/defconfig_tile-tilegx-linuxapp-gcc +++ /dev/null @@ -1,73 +0,0 @@ -# BSD LICENSE -# -# Copyright (C) EZchip Semiconductor 2015. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in -# the documentation and/or other materials provided with the -# distribution. -# * Neither the name of EZchip Semiconductor nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -#include "common_linuxapp" - -CONFIG_RTE_MACHINE="tilegx" - -CONFIG_RTE_ARCH="tile" -CONFIG_RTE_ARCH_TILE=y -CONFIG_RTE_ARCH_64=y -CONFIG_RTE_ARCH_STRICT_ALIGN=y -CONFIG_RTE_FORCE_INTRINSICS=y - -CONFIG_RTE_TOOLCHAIN="gcc" -CONFIG_RTE_TOOLCHAIN_GCC=y - -CONFIG_RTE_MEMPOOL_ALIGN=128 - -CONFIG_RTE_LIBRTE_MPIPE_PMD=y -CONFIG_RTE_LIBRTE_MPIPE_PMD_DEBUG=n - -# Disable things that we don't support or need -CONFIG_RTE_LIBRTE_EAL_VMWARE_TSC_MAP_SUPPORT=n -CONFIG_RTE_EAL_IGB_UIO=n -CONFIG_RTE_EAL_VFIO=n -CONFIG_RTE_LIBRTE_KNI=n -CONFIG_RTE_KNI_KMOD=n -CONFIG_RTE_LIBRTE_XEN_DOM0=n -CONFIG_RTE_LIBRTE_IGB_PMD=n -CONFIG_RTE_LIBRTE_EM_PMD=n -CONFIG_RTE_LIBRTE_IXGBE_PMD=n -CONFIG_RTE_LIBRTE_I40E_PMD=n -CONFIG_RTE_LIBRTE_FM10K_PMD=n -CONFIG_RTE_LIBRTE_VIRTIO_PMD=n -CONFIG_RTE_LIBRTE_VMXNET3_PMD=n -CONFIG_RTE_LIBRTE_ENIC_PMD=n - -# This following libraries are not available on the tile architecture. -# So they're turned off. -CONFIG_RTE_LIBRTE_LPM=n -CONFIG_RTE_LIBRTE_ACL=n -CONFIG_RTE_LIBRTE_SCHED=n -CONFIG_RTE_LIBRTE_PORT=n -CONFIG_RTE_LIBRTE_TABLE=n -CONFIG_RTE_LIBRTE_PIPELINE=n -CONFIG_RTE_LIBRTE_CXGBE_PMD=n diff --git a/dpdk/config/defconfig_x86_64-ivshmem-linuxapp-gcc b/dpdk/config/defconfig_x86_64-ivshmem-linuxapp-gcc deleted file mode 100644 index 41ac5c31..00000000 --- a/dpdk/config/defconfig_x86_64-ivshmem-linuxapp-gcc +++ /dev/null @@ -1,49 +0,0 @@ -# BSD LICENSE -# -# Copyright(c) 2010-2014 Intel Corporation. All rights reserved. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in -# the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Intel Corporation nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# - -# -# use default config -# - -#include "defconfig_x86_64-native-linuxapp-gcc" - -# -# Compile IVSHMEM library -# -CONFIG_RTE_LIBRTE_IVSHMEM=y -CONFIG_RTE_LIBRTE_IVSHMEM_DEBUG=n -CONFIG_RTE_LIBRTE_IVSHMEM_MAX_PCI_DEVS=4 -CONFIG_RTE_LIBRTE_IVSHMEM_MAX_ENTRIES=128 -CONFIG_RTE_LIBRTE_IVSHMEM_MAX_METADATA_FILES=32 - -# Set EAL to single file segments -CONFIG_RTE_EAL_SINGLE_FILE_SEGMENTS=y \ No newline at end of file diff --git a/dpdk/config/defconfig_x86_64-ivshmem-linuxapp-icc b/dpdk/config/defconfig_x86_64-ivshmem-linuxapp-icc deleted file mode 100644 index 77fec939..00000000 --- a/dpdk/config/defconfig_x86_64-ivshmem-linuxapp-icc +++ /dev/null @@ -1,49 +0,0 @@ -# BSD LICENSE -# -# Copyright(c) 2010-2014 Intel Corporation. All rights reserved. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in -# the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Intel Corporation nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# - -# -# use default config -# - -#include "defconfig_x86_64-native-linuxapp-icc" - -# -# Compile IVSHMEM library -# -CONFIG_RTE_LIBRTE_IVSHMEM=y -CONFIG_RTE_LIBRTE_IVSHMEM_DEBUG=n -CONFIG_RTE_LIBRTE_IVSHMEM_MAX_PCI_DEVS=4 -CONFIG_RTE_LIBRTE_IVSHMEM_MAX_ENTRIES=128 -CONFIG_RTE_LIBRTE_IVSHMEM_MAX_METADATA_FILES=32 - -# Set EAL to single file segments -CONFIG_RTE_EAL_SINGLE_FILE_SEGMENTS=y diff --git a/dpdk/config/defconfig_x86_64-native-linuxapp-icc b/dpdk/config/defconfig_x86_64-native-linuxapp-icc index bfa8a3da..872d1a1b 100644 --- a/dpdk/config/defconfig_x86_64-native-linuxapp-icc +++ b/dpdk/config/defconfig_x86_64-native-linuxapp-icc @@ -41,3 +41,8 @@ CONFIG_RTE_ARCH_64=y CONFIG_RTE_TOOLCHAIN="icc" CONFIG_RTE_TOOLCHAIN_ICC=y + +# +# Solarflare PMD build is not supported using icc toolchain +# +CONFIG_RTE_LIBRTE_SFC_EFX_PMD=n diff --git a/dpdk/config/defconfig_x86_x32-native-linuxapp-gcc b/dpdk/config/defconfig_x86_x32-native-linuxapp-gcc index 0c26857c..19573cb5 100644 --- a/dpdk/config/defconfig_x86_x32-native-linuxapp-gcc +++ b/dpdk/config/defconfig_x86_x32-native-linuxapp-gcc @@ -45,3 +45,13 @@ CONFIG_RTE_TOOLCHAIN_GCC=y # KNI is not supported on 32-bit # CONFIG_RTE_LIBRTE_KNI=n + +# +# Solarflare PMD is not supported on 32-bit +# +CONFIG_RTE_LIBRTE_SFC_EFX_PMD=n + +# +# AVP PMD is not supported on 32-bit +# +CONFIG_RTE_LIBRTE_AVP_PMD=n diff --git a/dpdk/devtools/build-tags.sh b/dpdk/devtools/build-tags.sh new file mode 100644 index 00000000..942da2ba --- /dev/null +++ b/dpdk/devtools/build-tags.sh @@ -0,0 +1,233 @@ +#!/bin/sh -e +# Generate tags or gtags or cscope or etags files +# +# BSD LICENSE +# +# Copyright 2017 Cavium, Inc +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Cavium, Inc nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +verbose=false +linux=true +bsd=true +x86_32=true +x86_64=true +ppc_64=true +arm_32=true +arm_64=true + +print_usage() +{ + echo "Usage: $(basename $0) [-h] [-v] tags|cscope|gtags|etags [config]" + echo "Valid configs are:" + make showconfigs | sed 's,^,\t,' +} + +# Move to the root of the git tree +cd $(dirname $0)/.. + +while getopts hv ARG ; do + case $ARG in + v ) verbose=true ;; + h ) print_usage; exit 0 ;; + ? ) print_usage; exit 1 ;; + esac +done +shift $(($OPTIND - 1)) + +#ignore version control files +ignore="( -name .svn -o -name CVS -o -name .hg -o -name .git ) -prune -o" + +source_dirs="test app buildtools drivers examples lib" + +skip_bsd="( -name bsdapp ) -prune -o" +skip_linux="( -name linuxapp ) -prune -o" +skip_arch="( -name arch ) -prune -o" +skip_sse="( -name *_sse*.[chS] ) -prune -o" +skip_avx="( -name *_avx*.[chS] ) -prune -o" +skip_neon="( -name *_neon*.[chS] ) -prune -o" +skip_altivec="( -name *_altivec*.[chS] ) -prune -o" +skip_arm64="( -name *arm64*.[chS] ) -prune -o" +skip_x86="( -name *x86*.[chS] ) -prune -o" +skip_32b_files="( -name *_32.h ) -prune -o" +skip_64b_files="( -name *_64.h ) -prune -o" + +skiplist="$skip_bsd $skip_linux $skip_arch $skip_sse $skip_avx \ + $skip_neon $skip_altivec $skip_x86 $skip_arm64" + +find_sources() +{ + find $1 $ignore $3 -name $2 -not -type l -print +} + +common_sources() +{ + find_sources "$source_dirs" '*.[chS]' "$skiplist" +} + +linux_sources() +{ + find_sources "lib/librte_eal/linuxapp" '*.[chS]' +} + +bsd_sources() +{ + find_sources "lib/librte_eal/bsdapp" '*.[chS]' +} + +arm_common() +{ + find_sources "lib/librte_eal/common/arch/arm" '*.[chS]' + find_sources "$source_dirs" '*neon*.[chS]' +} + +arm_32_sources() +{ + arm_common + find_sources "lib/librte_eal/common/include/arch/arm" '*.[chS]' \ + "$skip_64b_files" +} + +arm_64_sources() +{ + arm_common + find_sources "lib/librte_eal/common/include/arch/arm" '*.[chS]' \ + "$skip_32b_files" + find_sources "$source_dirs" '*arm64.[chS]' +} + +x86_common() +{ + find_sources "lib/librte_eal/common/arch/x86" '*.[chS]' + + find_sources "examples/performance-thread/common/arch/x86" '*.[chS]' + find_sources "$source_dirs" '*_sse*.[chS]' + find_sources "$source_dirs" '*_avx*.[chS]' + find_sources "$source_dirs" '*x86.[chS]' +} + +x86_32_sources() +{ + x86_common + find_sources "lib/librte_eal/common/include/arch/x86" '*.[chS]' \ + "$skip_64b_files" +} + +x86_64_sources() +{ + x86_common + find_sources "lib/librte_eal/common/include/arch/x86" '*.[chS]' \ + "$skip_32b_files" +} + +ppc_64_sources() +{ + find_sources "lib/librte_eal/common/arch/ppc_64" '*.[chS]' + find_sources "lib/librte_eal/common/include/arch/ppc_64" '*.[chS]' + find_sources "$source_dirs" '*altivec*.[chS]' +} + +check_valid_target() +{ + cfgfound=false + allconfigs=$(make showconfigs) + for cfg in $allconfigs ; do + if [ "$cfg" = "$1" ] ; then + cfgfound=true + fi + done + if ! $cfgfound ; then + echo "Invalid config: $1" + print_usage + exit 0 + fi +} + +if [ -n "$2" ]; then + check_valid_target $2 + + echo $2 | grep -q "linuxapp-" || linux=false + echo $2 | grep -q "bsdapp-" || bsd=false + echo $2 | grep -q "x86_64-" || x86_64=false + echo $2 | grep -q "arm-" || arm_32=false + echo $2 | grep -q "arm64-" || arm_64=false + echo $2 | grep -q "ppc_64-" || ppc_64=false + echo $2 | grep -q -e "i686-" -e "x32-" || x86_32=false +fi + +all_sources() +{ + common_sources + if $linux ; then linux_sources ; fi + if $bsd ; then bsd_sources ; fi + if $x86_64 ; then x86_64_sources ; fi + if $x86_32 ; then x86_32_sources ; fi + if $ppc_64 ; then ppc_64_sources ; fi + if $arm_32 ; then arm_32_sources ; fi + if $arm_64 ; then arm_64_sources ; fi +} + +show_flags() +{ + if $verbose ; then + echo "mode: $1" + echo "config: $2" + echo "linux: $linux" + echo "bsd: $bsd" + echo "x86_32: $x86_32" + echo "x86_64: $x86_64" + echo "ppc_64: $ppc_64" + echo "arm_32: $arm_32" + echo "arm_64: $arm_64" + fi +} + +case "$1" in + "cscope") + show_flags $1 $2 + all_sources > cscope.files + cscope -q -b -f cscope.out + ;; + "gtags") + show_flags $1 $2 + all_sources | gtags -i -f - + ;; + "tags") + show_flags $1 $2 + rm -f tags + all_sources | xargs ctags -a + ;; + "etags") + show_flags $1 $2 + rm -f TAGS + all_sources | xargs etags -a + ;; + *) + echo "Invalid mode: $1" + print_usage + ;; +esac diff --git a/dpdk/devtools/check-dup-includes.sh b/dpdk/devtools/check-dup-includes.sh new file mode 100644 index 00000000..10365e4a --- /dev/null +++ b/dpdk/devtools/check-dup-includes.sh @@ -0,0 +1,37 @@ +#! /bin/sh -e + +# Copyright 2017 Mellanox Technologies, Ltd. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +# Check C files in git repository for duplicated includes. +# Usage: devtools/check-dup-includes.sh [directory] + +dir=${1:-$(dirname $(readlink -m $0))/..} +cd $dir + +# speed up by ignoring Unicode details +export LC_ALL=C + +for file in $(git ls-files '*.[ch]') ; do + sed -rn 's,^[[:space:]]*#include[[:space:]]*[<"](.*)[>"].*,\1,p' $file | + sort | uniq -d | + sed "s,^,$file: duplicated include: ," +done diff --git a/dpdk/devtools/check-git-log.sh b/dpdk/devtools/check-git-log.sh new file mode 100644 index 00000000..910daba4 --- /dev/null +++ b/dpdk/devtools/check-git-log.sh @@ -0,0 +1,203 @@ +#! /bin/sh + +# BSD LICENSE +# +# Copyright 2016 6WIND S.A. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of 6WIND S.A. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Check commit logs (headlines and references) +# +# If any doubt about the formatting, please check in the most recent history: +# git log --format='%>|(15)%cr %s' --reverse | grep -i + +if [ "$1" = '-h' -o "$1" = '--help' ] ; then + cat <<- END_OF_HELP + usage: $(basename $0) [-h] [range] + + Check commit log formatting. + The git range can be specified as a "git log" option, + e.g. -1 to check only the latest commit. + The default range starts from origin/master to HEAD. + END_OF_HELP + exit +fi + +selfdir=$(dirname $(readlink -f $0)) +range=${1:-origin/master..} +# convert -N to HEAD~N.. in order to comply with git-log-fixes.sh getopts +if printf -- $range | grep -q '^-[0-9]\+' ; then + range="HEAD$(printf -- $range | sed 's,^-,~,').." +fi + +commits=$(git log --format='%h' --reverse $range) +headlines=$(git log --format='%s' --reverse $range) +bodylines=$(git log --format='%b' --reverse $range) +fixes=$(git log --format='%h %s' --reverse $range | grep -i ': *fix' | cut -d' ' -f1) +stablefixes=$($selfdir/git-log-fixes.sh $range | sed '/(N\/A)$/d' | cut -d' ' -f2) +tags=$(git log --format='%b' --reverse $range | grep -i -e 'by *:' -e 'fix.*:') +bytag='\(Reported\|Suggested\|Signed-off\|Acked\|Reviewed\|Tested\)-by:' + +# check headline format (spacing, no punctuation, no code) +bad=$(echo "$headlines" | grep --color=always \ + -e ' ' \ + -e '^ ' \ + -e ' $' \ + -e '\.$' \ + -e '[,;!?&|]' \ + -e ':.*_' \ + -e '^[^:]\+$' \ + -e ':[^ ]' \ + -e ' :' \ + | sed 's,^,\t,') +[ -z "$bad" ] || printf "Wrong headline format:\n$bad\n" + +# check headline prefix when touching only drivers, e.g. net/ +bad=$(for commit in $commits ; do + headline=$(git log --format='%s' -1 $commit) + files=$(git diff-tree --no-commit-id --name-only -r $commit) + [ -z "$(echo "$files" | grep -v '^\(drivers\|doc\|config\)/')" ] || + continue + drv=$(echo "$files" | grep '^drivers/' | cut -d "/" -f 2,3 | sort -u) + drvgrp=$(echo "$drv" | cut -d "/" -f 1 | uniq) + if [ $(echo "$drvgrp" | wc -l) -gt 1 ] ; then + echo "$headline" | grep -v '^drivers:' + elif [ $(echo "$drv" | wc -l) -gt 1 ] ; then + echo "$headline" | grep -v "^drivers/$drvgrp" + else + echo "$headline" | grep -v "^$drv" + fi +done | sed 's,^,\t,') +[ -z "$bad" ] || printf "Wrong headline prefix:\n$bad\n" + +# check headline label for common typos +bad=$(echo "$headlines" | grep --color=always \ + -e '^example[:/]' \ + -e '^apps/' \ + -e '^testpmd' \ + -e 'test-pmd' \ + -e '^bond:' \ + | sed 's,^,\t,') +[ -z "$bad" ] || printf "Wrong headline label:\n$bad\n" + +# check headline lowercase for first words +bad=$(echo "$headlines" | grep --color=always \ + -e '^.*[A-Z].*:' \ + -e ': *[A-Z]' \ + | sed 's,^,\t,') +[ -z "$bad" ] || printf "Wrong headline uppercase:\n$bad\n" + +# check headline uppercase (Rx/Tx, VF, L2, MAC, Linux, ARM...) +bad=$(echo "$headlines" | grep -E --color=always \ + -e ':.*\<(rx|tx|RX|TX)\>' \ + -e ':.*\<[pv]f\>' \ + -e ':.*\<[hsf]w\>' \ + -e ':.*\' \ + -e ':.*\' \ + -e ':.*\' \ + -e ':.*\' \ + -e ':.*\' \ + -e ':.*\' \ + -e ':.*\' \ + -e ':.*\' \ + -e ':.*\' \ + -e ':.*\' \ + -e ':.*\' \ + -e ':.*\' \ + -e ':.*\' \ + -e ':.*\' \ + -e ':.*\' \ + -e ':.*\' \ + -e ':.*\' \ + -e ':.*\' \ + -e ':.*\' \ + -e ':.*\' \ + -e ':.*\<[Vv]lan\>' \ + -e ':.*\' \ + | sed 's,^,\t,') +[ -z "$bad" ] || printf "Wrong headline lowercase:\n$bad\n" + +# special case check for VMDq to give good error message +bad=$(echo "$headlines" | grep -E --color=always \ + -e '\<(vmdq|VMDQ)\>' \ + | sed 's,^,\t,') +[ -z "$bad" ] || printf "Wrong headline capitalization, use 'VMDq':\n$bad\n" + +# check headline length (60 max) +bad=$(echo "$headlines" | + awk 'length>60 {print}' | + sed 's,^,\t,') +[ -z "$bad" ] || printf "Headline too long:\n$bad\n" + +# check body lines length (75 max) +bad=$(echo "$bodylines" | grep -v '^Fixes:' | + awk 'length>75 {print}' | + sed 's,^,\t,') +[ -z "$bad" ] || printf "Line too long:\n$bad\n" + +# check starting commit message with "It" +bad=$(for commit in $commits ; do + firstbodyline=$(git log --format='%b' -1 $commit | head -n1) + echo "$firstbodyline" | grep --color=always -ie '^It ' +done | sed 's,^,\t,') +[ -z "$bad" ] || printf "Wrong beginning of commit message:\n$bad\n" + +# check tags spelling +bad=$(echo "$tags" | + grep -v "^$bytag [^,]* <.*@.*>$" | + grep -v '^Fixes: [0-9a-f]\{7\}[0-9a-f]* (".*")$' | + sed 's,^.,\t&,') +[ -z "$bad" ] || printf "Wrong tag:\n$bad\n" + +# check missing Fixes: tag +bad=$(for fix in $fixes ; do + git log --format='%b' -1 $fix | grep -q '^Fixes: ' || + git log --format='\t%s' -1 $fix +done) +[ -z "$bad" ] || printf "Missing 'Fixes' tag:\n$bad\n" + +# check Fixes: reference +IFS=' +' +fixtags=$(echo "$tags" | grep '^Fixes: ') +bad=$(for fixtag in $fixtags ; do + hash=$(echo "$fixtag" | sed 's,^Fixes: \([0-9a-f]*\).*,\1,') + if git branch --contains $hash 2>&- | grep -q '^\*' ; then + good="Fixes: $hash "$(git log --format='("%s")' -1 $hash 2>&-) + else + good="reference not in current branch" + fi + printf "$fixtag" | grep -v "^$good$" +done | sed 's,^,\t,') +[ -z "$bad" ] || printf "Wrong 'Fixes' reference:\n$bad\n" + +# check Cc: stable@dpdk.org for fixes +bad=$(for fix in $stablefixes ; do + git log --format='%b' -1 $fix | grep -qi '^Cc: *stable@dpdk.org' || + git log --format='\t%s' -1 $fix +done) +[ -z "$bad" ] || printf "Is it candidate for Cc: stable@dpdk.org backport?\n$bad\n" diff --git a/dpdk/devtools/check-includes.sh b/dpdk/devtools/check-includes.sh new file mode 100644 index 00000000..c4ec73f1 --- /dev/null +++ b/dpdk/devtools/check-includes.sh @@ -0,0 +1,288 @@ +#!/bin/sh -e +# +# BSD LICENSE +# +# Copyright 2016 6WIND S.A. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of 6WIND S.A. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# This script checks that header files in a given directory do not miss +# dependencies when included on their own, do not conflict and accept being +# compiled with the strictest possible flags. +# +# Files are looked up in the directory provided as the first argument, +# otherwise build/include by default. +# +# Recognized environment variables: +# +# VERBOSE=1 is the same as -v. +# +# QUIET=1 is the same as -q. +# +# SUMMARY=1 is the same as -s. +# +# CC, CPPFLAGS, CFLAGS, EXTRA_CPPFLAGS, EXTRA_CFLAGS, CXX, CXXFLAGS and +# EXTRA_CXXFLAGS are taken into account. +# +# PEDANTIC_CFLAGS, PEDANTIC_CXXFLAGS and PEDANTIC_CPPFLAGS provide strict +# C/C++ compilation flags. +# +# IGNORE contains a list of shell patterns matching files (relative to the +# include directory) to avoid. It is set by default to known DPDK headers +# which must not be included on their own. +# +# IGNORE_CXX provides additional files for C++. + +while getopts hqvs arg; do + case $arg in + h) + cat < /dev/null + +[ "$VERBOSE" = 1 ] && +output () +{ + local CCV + local CXXV + + shift + CCV=$CC + CXXV=$CXX + CC="echo $CC" CXX="echo $CXX" "$@" + CC=$CCV + CXX=$CXXV + + "$@" +} || +output () +{ + + printf ' %s\n' "$1" + shift + "$@" +} + +trap 'rm -f "$temp_cc" "$temp_cxx"' EXIT + +compile_cc () +{ + ${CC} -I"$include_dir" \ + ${PEDANTIC_CPPFLAGS} ${CPPFLAGS} ${EXTRA_CPPFLAGS} \ + ${PEDANTIC_CFLAGS} ${CFLAGS} ${EXTRA_CFLAGS} \ + -c -o /dev/null "${temp_cc}" +} + +compile_cxx () +{ + ${CXX} -I"$include_dir" \ + ${PEDANTIC_CPPFLAGS} ${CPPFLAGS} ${EXTRA_CPPFLAGS} \ + ${PEDANTIC_CXXFLAGS} ${CXXFLAGS} ${EXTRA_CXXFLAGS} \ + -c -o /dev/null "${temp_cxx}" +} + +ignore () +{ + file="$1" + shift + while [ $# -ne 0 ]; do + case "$file" in + $1) + return 0 + ;; + esac + shift + done + return 1 +} + +# Check C/C++ compilation for each header file. + +while read -r path +do + file=${path#$include_dir} + file=${file##/} + if ignore "$file" $IGNORE; then + output "SKIP $file" : + continue + fi + if printf "\ +#include <%s> + +int main(void) +{ + return 0; +} +" "$file" > "$temp_cc" && + output "CC $file" compile_cc + then + pass_cc="$pass_cc $file" + else + failures_cc=$(($failures_cc + 1)) + fi + if ignore "$file" $IGNORE_CXX; then + output "SKIP CXX $file" : + continue + fi + if printf "\ +#include <%s> + +int main() +{ +} +" "$file" > "$temp_cxx" && + output "CXX $file" compile_cxx + then + pass_cxx="$pass_cxx $file" + else + failures_cxx=$(($failures_cxx + 1)) + fi +done < "$temp_cc" && +for file in $pass_cc; do + printf "\ +#include <%s> +" "$file" >> $temp_cc +done +if printf "\ +int main(void) +{ + return 0; +} +" >> "$temp_cc" && + output "CC (all includes that did not fail)" compile_cc +then + : +else + failures_cc=$(($failures_cc + 1)) +fi + +# Check C++ compilation with all includes. + +: > "$temp_cxx" && +for file in $pass_cxx; do + printf "\ +#include <%s> +" "$file" >> $temp_cxx +done +if printf "\ +int main() +{ +} +" >> "$temp_cxx" && + output "CXX (all includes that did not fail)" compile_cxx +then + : +else + failures_cxx=$(($failures_cxx + 1)) +fi + +# Report results. + +if [ "$SUMMARY" = 1 ]; then + printf "\ +Summary: + %u failure(s) for C using '%s'. + %u failure(s) for C++ using '%s'. +" $failures_cc "$CC" $failures_cxx "$CXX" 1>&2 +fi + +# Exit with nonzero status if there are failures. + +[ $failures_cc -eq 0 ] && +[ $failures_cxx -eq 0 ] diff --git a/dpdk/devtools/check-maintainers.sh b/dpdk/devtools/check-maintainers.sh new file mode 100644 index 00000000..ac5326b8 --- /dev/null +++ b/dpdk/devtools/check-maintainers.sh @@ -0,0 +1,160 @@ +#! /bin/sh + +# BSD LICENSE +# +# Copyright 2015 6WIND S.A. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of 6WIND S.A. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Do some basic checks in MAINTAINERS file + +cd $(dirname $0)/.. + +# speed up by ignoring Unicode details +export LC_ALL=C + +# Get files matching paths with wildcards and / meaning recursing +files () # [ ...] +{ + if [ -z "$1" ] ; then + return + fi + if [ -d .git ] ; then + git ls-files "$1" + else + find "$1" -type f | + sed 's,^\./,,' + fi | + # if not ended by / + if ! echo "$1" | grep -q '/[[:space:]]*$' ; then + # filter out deeper directories + sed "/\(\/[^/]*\)\{$(($(echo "$1" | grep -o / | wc -l) + 1))\}/d" + else + cat + fi + # next path + shift + files "$@" +} + +# Get all files matching F: and X: fields +parse_fx () # +{ + IFS=' +' + # parse each line excepted underlining + for line in $( (sed '/^-\+$/d' $1 ; echo) | sed 's,^$,§,') ; do + if echo "$line" | grep -q '^§$' ; then + # empty line delimit end of section + whitelist=$(files $flines) + blacklist=$(files $xlines) + match=$(aminusb "$whitelist" "$blacklist") + if [ -n "$whitelist" ] ; then + printf "# $title " + maintainers=$(echo "$maintainers" | sed -r 's,.*<(.*)>.*,\1,') + maintainers=$(printf "$maintainers" | sed -e 's,^,<,' -e 's,$,>,') + echo $maintainers + fi + if [ -n "$match" ] ; then + echo "$match" + fi + # flush section + unset maintainers + unset flines + unset xlines + elif echo "$line" | grep -q '^[A-Z]: ' ; then + # maintainer + maintainers=$(add_line_to_if "$line" "$maintainers" 'M: ') + # file matching pattern + flines=$(add_line_to_if "$line" "$flines" 'F: ') + # file exclusion pattern + xlines=$(add_line_to_if "$line" "$xlines" 'X: ') + else # assume it is a title + title="$line" + fi + done +} + +# Check patterns in F: and X: +check_fx () # +{ + IFS=' +' + for line in $(sed -n 's,^[FX]: ,,p' $1 | tr '*' '#') ; do + line=$(printf "$line" | tr '#' '*') + match=$(files "$line") + if [ -z "$match" ] ; then + echo "$line" + fi + done +} + +# Add a line to a set of lines if it begins with right pattern +add_line_to_if () # +{ + ( + echo "$2" + echo "$1" | sed -rn "s,^$3(.*),\1,p" + ) | + sed '/^$/d' +} + +# Subtract two sets of lines +aminusb () # +{ + printf "$1\n$2\n$2" | sort | uniq -u | sed '/^$/d' +} + +printf 'sections: ' +parsed=$(parse_fx MAINTAINERS) +echo "$parsed" | grep -c '^#' +printf 'with maintainer: ' +echo "$parsed" | grep -c '^#.*@' +printf 'maintainers: ' +grep '^M:.*<' MAINTAINERS | sort -u | wc -l + +echo +echo '##########' +echo '# orphan areas' +echo '##########' +echo "$parsed" | sed -rn 's,^#([^@]*)$,\1,p' | uniq + +echo +echo '##########' +echo '# files not listed' +echo '##########' +all=$(files ./) +listed=$(echo "$parsed" | sed '/^#/d' | sort -u) +aminusb "$all" "$listed" + +echo +echo '##########' +echo '# wrong patterns' +echo '##########' +check_fx MAINTAINERS + +# TODO: check overlaps diff --git a/dpdk/devtools/checkpatches.sh b/dpdk/devtools/checkpatches.sh new file mode 100644 index 00000000..a56c41a3 --- /dev/null +++ b/dpdk/devtools/checkpatches.sh @@ -0,0 +1,135 @@ +#! /bin/sh + +# BSD LICENSE +# +# Copyright 2015 6WIND S.A. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of 6WIND S.A. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Load config options: +# - DPDK_CHECKPATCH_PATH +# - DPDK_CHECKPATCH_LINE_LENGTH +. $(dirname $(readlink -e $0))/load-devel-config + +length=${DPDK_CHECKPATCH_LINE_LENGTH:-80} + +# override default Linux options +options="--no-tree" +options="$options --max-line-length=$length" +options="$options --show-types" +options="$options --ignore=LINUX_VERSION_CODE,FILE_PATH_CHANGES,\ +VOLATILE,PREFER_PACKED,PREFER_ALIGNED,PREFER_PRINTF,\ +PREFER_KERNEL_TYPES,BIT_MACRO,CONST_STRUCT,\ +SPLIT_STRING,LONG_LINE_STRING,\ +LINE_SPACING,PARENTHESIS_ALIGNMENT,NETWORKING_BLOCK_COMMENT_STYLE,\ +NEW_TYPEDEFS,COMPARISON_TO_NULL" + +print_usage () { + cat <<- END_OF_HELP + usage: $(basename $0) [-q] [-v] [-nX|patch1 [patch2] ...]] + + Run Linux kernel checkpatch.pl with DPDK options. + The environment variable DPDK_CHECKPATCH_PATH must be set. + + The patches to check can be from stdin, files specified on the command line, + or latest git commits limited with -n option (default limit: origin/master). + END_OF_HELP +} + +number=0 +quiet=false +verbose=false +while getopts hn:qv ARG ; do + case $ARG in + n ) number=$OPTARG ;; + q ) quiet=true ;; + v ) verbose=true ;; + h ) print_usage ; exit 0 ;; + ? ) print_usage ; exit 1 ;; + esac +done +shift $(($OPTIND - 1)) + +if [ ! -x "$DPDK_CHECKPATCH_PATH" ] ; then + print_usage >&2 + echo + echo 'Cannot execute DPDK_CHECKPATCH_PATH' >&2 + exit 1 +fi + +total=0 +status=0 + +check () { # + total=$(($total + 1)) + ! $verbose || printf '\n### %s\n\n' "$3" + if [ -n "$1" ] ; then + report=$($DPDK_CHECKPATCH_PATH $options "$1" 2>/dev/null) + elif [ -n "$2" ] ; then + report=$(git format-patch --find-renames --no-stat --stdout -1 $commit | + $DPDK_CHECKPATCH_PATH $options - 2>/dev/null) + else + report=$($DPDK_CHECKPATCH_PATH $options - 2>/dev/null) + fi + [ $? -ne 0 ] || return 0 + $verbose || printf '\n### %s\n\n' "$3" + printf '%s\n' "$report" | sed -n '1,/^total:.*lines checked$/p' + status=$(($status + 1)) +} + +if [ -n "$1" ] ; then + for patch in "$@" ; do + # Subject can be on 2 lines + subject=$(sed '/^Subject: */!d;s///;N;s,\n[[:space:]]\+, ,;s,\n.*,,;q' "$patch") + check "$patch" '' "$subject" + done +elif [ ! -t 0 ] ; then # stdin + subject=$(while read header value ; do + if [ "$header" = 'Subject:' ] ; then + IFS= read next + continuation=$(echo "$next" | sed -n 's,^[[:space:]]\+, ,p') + echo $value$continuation + break + fi + done) + check '' '' "$subject" +else + if [ $number -eq 0 ] ; then + commits=$(git rev-list --reverse origin/master..) + else + commits=$(git rev-list --reverse --max-count=$number HEAD) + fi + for commit in $commits ; do + subject=$(git log --format='%s' -1 $commit) + check '' $commit "$subject" + done +fi +pass=$(($total - $status)) +$quiet || printf '\n%d/%d valid patch' $pass $total +$quiet || [ $pass -le 1 ] || printf 'es' +$quiet || printf '\n' +exit $status diff --git a/dpdk/devtools/cocci.sh b/dpdk/devtools/cocci.sh new file mode 100644 index 00000000..4ca5025f --- /dev/null +++ b/dpdk/devtools/cocci.sh @@ -0,0 +1,64 @@ +#! /bin/sh + +# BSD LICENSE +# +# Copyright 2015 EZchip Semiconductor Ltd. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of EZchip Semiconductor nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Apply coccinelle transforms. + +SRCTREE=$(readlink -f $(dirname $0)/..) +COCCI=$SRCTREE/devtools/cocci +[ -n "$SPATCH" ] || SPATCH=$(which spatch) + +PATCH_LIST="$@" +[ -n "$PATCH_LIST" ] || PATCH_LIST=$(echo $COCCI/*.cocci) + +[ -x "$SPATCH" ] || ( + echo "Coccinelle tools not installed." + exit 1 +) + +tmp=$(mktemp) + +for c in $PATCH_LIST; do + while true; do + echo -n "Applying $c..." + $SPATCH --sp-file $c -c --linux-spacing --very-quiet \ + --include-headers --preprocess \ + --in-place --dir $SRCTREE > $tmp + if [ -s $tmp ]; then + echo " changes applied, retrying." + else + echo " no change." + break; + fi + done +done + +rm -f $tmp diff --git a/dpdk/devtools/cocci/mtod-offset.cocci b/dpdk/devtools/cocci/mtod-offset.cocci new file mode 100644 index 00000000..3f83f322 --- /dev/null +++ b/dpdk/devtools/cocci/mtod-offset.cocci @@ -0,0 +1,76 @@ +// +// Replace explicit packet offset computations with rte_pktmbuf_mtod_offset(). +// +@disable paren@ +typedef uint8_t; +expression M, O; +@@ +( +- rte_pktmbuf_mtod(M, char *) + O ++ rte_pktmbuf_mtod_offset(M, char *, O) +| +- rte_pktmbuf_mtod(M, char *) - O ++ rte_pktmbuf_mtod_offset(M, char *, -O) +| +- rte_pktmbuf_mtod(M, unsigned char *) + O ++ rte_pktmbuf_mtod_offset(M, unsigned char *, O) +| +- rte_pktmbuf_mtod(M, unsigned char *) - O ++ rte_pktmbuf_mtod_offset(M, unsigned char *, -O) +| +- rte_pktmbuf_mtod(M, uint8_t *) + O ++ rte_pktmbuf_mtod_offset(M, uint8_t *, O) +| +- rte_pktmbuf_mtod(M, uint8_t *) - O ++ rte_pktmbuf_mtod_offset(M, uint8_t *, -O) +) + + +// +// Fold subsequent offset terms into pre-existing offset used in +// rte_pktmbuf_mtod_offset(). +// +@disable paren@ +expression M, O1, O2; +@@ +( +- rte_pktmbuf_mtod_offset(M, char *, O1) + O2 ++ rte_pktmbuf_mtod_offset(M, char *, O1 + O2) +| +- rte_pktmbuf_mtod_offset(M, char *, O1) - O2 ++ rte_pktmbuf_mtod_offset(M, char *, O1 - O2) +| +- rte_pktmbuf_mtod_offset(M, unsigned char *, O1) + O2 ++ rte_pktmbuf_mtod_offset(M, unsigned char *, O1 + O2) +| +- rte_pktmbuf_mtod_offset(M, unsigned char *, O1) - O2 ++ rte_pktmbuf_mtod_offset(M, unsigned char *, O1 - O2) +| +- rte_pktmbuf_mtod_offset(M, uint8_t *, O1) + O2 ++ rte_pktmbuf_mtod_offset(M, uint8_t *, O1 + O2) +| +- rte_pktmbuf_mtod_offset(M, uint8_t *, O1) - O2 ++ rte_pktmbuf_mtod_offset(M, uint8_t *, O1 - O2) +) + + +// +// Cleanup rules. Fold in double casts, remove unnecessary parenthesis, etc. +// +@disable paren@ +expression M, O; +type C, T; +@@ +( +- (C)rte_pktmbuf_mtod_offset(M, T, O) ++ rte_pktmbuf_mtod_offset(M, C, O) +| +- (rte_pktmbuf_mtod_offset(M, T, O)) ++ rte_pktmbuf_mtod_offset(M, T, O) +| +- (C)rte_pktmbuf_mtod(M, T) ++ rte_pktmbuf_mtod(M, C) +| +- (rte_pktmbuf_mtod(M, T)) ++ rte_pktmbuf_mtod(M, T) +) diff --git a/dpdk/devtools/get-maintainer.sh b/dpdk/devtools/get-maintainer.sh new file mode 100644 index 00000000..5ae6b5ae --- /dev/null +++ b/dpdk/devtools/get-maintainer.sh @@ -0,0 +1,85 @@ +#!/bin/sh + +# BSD LICENSE +# +# Copyright(c) 2017 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +# Load config options: +# - DPDK_GETMAINTAINER_PATH +. $(dirname $(readlink -e $0))/load-devel-config + +options="--no-git-fallback" +options="$options --no-rolestats" + +print_usage () { + cat <<- END_OF_HELP + usage: $(basename $0) <patch> + END_OF_HELP +} + +# Requires DPDK_GETMAINTAINER_PATH devel config option set, +# please check devtools/load-devel-config. +# DPDK_GETMAINTAINER_PATH should be full path to the get_maintainer.pl script, +# like: +# DPDK_GETMAINTAINER_PATH=~/linux/scripts/get_maintainer.pl + +if [ ! -x "$DPDK_GETMAINTAINER_PATH" ] ; then + print_usage >&2 + echo + echo 'Cannot execute DPDK_GETMAINTAINER_PATH' >&2 + exit 1 +fi + +FILES="COPYING CREDITS Kbuild" +FOLDERS="Documentation arch include fs init ipc kernel scripts" + +# Kernel script checks for some files and folders to run +workaround () { + for f in $FILES; do + if [ ! -f $f ]; then touch $f; fi + done + + for d in $FOLDERS; do + if [ ! -d $d ]; then mkdir $d; fi + done +} + +fix_workaround () { + for f in $FILES; do if [ -f $f ]; then rm -f $f; fi; done + for d in $FOLDERS; do if [ -d $d ]; then rmdir $d; fi; done +} + +# clean workaround on exit +trap fix_workaround EXIT + +workaround +$DPDK_GETMAINTAINER_PATH $options $@ +# fix_workaround called on exit by trap diff --git a/dpdk/devtools/git-log-fixes.sh b/dpdk/devtools/git-log-fixes.sh new file mode 100644 index 00000000..cd5cf893 --- /dev/null +++ b/dpdk/devtools/git-log-fixes.sh @@ -0,0 +1,140 @@ +#! /bin/sh -e + +# BSD LICENSE +# +# Copyright 2016 6WIND S.A. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of 6WIND S.A. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +print_usage () +{ + echo "usage: $(basename $0) [-h] <git_range>" +} + +print_help () +{ + print_usage + cat <<- END_OF_HELP + + Find fixes to backport on previous versions. + It looks for the word "fix" in the headline or a tag "Fixes" or "Reverts". + The oldest bug origin is printed as well as partially fixed versions. + END_OF_HELP +} + +usage_error () # <message> +{ + echo "$*" >&2 + print_usage >&2 + exit 1 +} + +while getopts h ARG ; do + case $ARG in + h ) print_help ; exit 0 ;; + ? ) print_usage >&2 ; exit 1 ;; + esac +done +shift $(($OPTIND - 1)) +[ $# -ge 1 ] || usage_error 'range argument required' +range="$*" + +# get major release version of a commit +commit_version () # <hash> +{ + # use current branch as history reference + local refbranch=$(git rev-parse --abbrev-ref HEAD) + local tag=$( (git tag -l --contains $1 --merged $refbranch 2>&- || + # tag --merged option has been introduced in git 2.7.0 + # below is a fallback in case of old git version + for t in $(git tag -l --contains $1) ; do + git branch $refbranch --contains $t | + sed "s,.\+,$t," + done) | + head -n1) + if [ -z "$tag" ] ; then + # before -rc1 tag of release in progress + make showversion | cut -d'.' -f-2 + else + echo $tag | sed 's,^v,,' | sed 's,-rc.*,,' + fi +} + +# get bug origin hashes of a fix +origin_filter () # <hash> +{ + git log --format='%b' -1 $1 | + sed -n 's,^ *\([Ff]ixes\|[Rr]everts\): *\([0-9a-f]*\).*,\2,p' +} + +# get oldest major release version of bug origins +origin_version () # <origin_hash> ... +{ + for origin in $* ; do + # check hash is valid + git rev-parse -q --verify $1 >&- || continue + # get version of this bug origin + local origver=$(commit_version $origin) + local roothashes="$(origin_filter $origin)" + if [ -n "$roothashes" ] ; then + # look chained fix of fix recursively + local rootver="$(origin_version $roothashes)" + [ -n "$rootver" ] || continue + echo "$rootver (partially fixed in $origver)" + else + echo "$origver" + fi + # filter the oldest origin + done | sort -uV | head -n1 +} + +# print a marker for stable tag presence +stable_tag () # <hash> +{ + if git log --format='%b' -1 $1 | grep -qi '^Cc: *stable@dpdk.org' ; then + echo 'S' + else + echo '-' + fi +} + +git log --oneline --reverse $range | +while read id headline ; do + origins=$(origin_filter $id) + stable=$(stable_tag $id) + [ "$stable" = "S" ] || [ -n "$origins" ] || echo "$headline" | grep -q fix || continue + version=$(commit_version $id) + if [ -n "$origins" ] ; then + origver="$(origin_version $origins)" + [ -n "$origver" ] || continue + # ignore fix of bug introduced in the same release + ! echo "$origver" | grep -q "^$version" || continue + else + origver='N/A' + fi + printf '%s %7s %s %s (%s)\n' $version $id $stable "$headline" "$origver" +done diff --git a/dpdk/devtools/load-devel-config b/dpdk/devtools/load-devel-config new file mode 100644 index 00000000..4f43cb35 --- /dev/null +++ b/dpdk/devtools/load-devel-config @@ -0,0 +1,12 @@ +# Load DPDK devel config and allow override +# from system file +test ! -r /etc/dpdk/devel.config || + . /etc/dpdk/devel.config +# from user file +test ! -r ~/.config/dpdk/devel.config || + . ~/.config/dpdk/devel.config +# from local file +test ! -r $(dirname $(readlink -m $0))/../.develconfig || + . $(dirname $(readlink -m $0))/../.develconfig + +# The config files must export variables in the shell style diff --git a/dpdk/devtools/test-build.sh b/dpdk/devtools/test-build.sh new file mode 100644 index 00000000..092d3a7f --- /dev/null +++ b/dpdk/devtools/test-build.sh @@ -0,0 +1,259 @@ +#! /bin/sh -e + +# BSD LICENSE +# +# Copyright 2015 6WIND S.A. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of 6WIND S.A. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +default_path=$PATH + +# Load config options: +# - AESNI_MULTI_BUFFER_LIB_PATH +# - ARMV8_CRYPTO_LIB_PATH +# - DPDK_BUILD_TEST_CONFIGS (defconfig1+option1+option2 defconfig2) +# - DPDK_DEP_ARCHIVE +# - DPDK_DEP_CFLAGS +# - DPDK_DEP_LDFLAGS +# - DPDK_DEP_MLX (y/[n]) +# - DPDK_DEP_NUMA ([y]/n) +# - DPDK_DEP_PCAP (y/[n]) +# - DPDK_DEP_SSL (y/[n]) +# - DPDK_DEP_SZE (y/[n]) +# - DPDK_DEP_ZLIB (y/[n]) +# - DPDK_MAKE_JOBS (int) +# - DPDK_NOTIFY (notify-send) +# - LIBMUSDK_PATH +# - LIBSSO_SNOW3G_PATH +# - LIBSSO_KASUMI_PATH +# - LIBSSO_ZUC_PATH +. $(dirname $(readlink -e $0))/load-devel-config + +print_usage () { + echo "usage: $(basename $0) [-h] [-jX] [-s] [config1 [config2] ...]]" +} + +print_help () { + echo 'Test building several targets with different options' + echo + print_usage + cat <<- END_OF_HELP + + options: + -h this help + -jX use X parallel jobs in "make" + -s short test only first config without tests|examples|doc + -v verbose build + + config: defconfig[[~][+]option1[[~][+]option2...]] + Example: x86_64-native-linuxapp-gcc+debug~RXTX_CALLBACKS + The lowercase options are defined inside $(basename $0). + The uppercase options can be the end of a defconfig option + to enable if prefixed with '+' or to disable if prefixed with '~'. + Default is to automatically enable most of the options. + The external dependencies are setup with DPDK_DEP_* variables. + If no config on command line, DPDK_BUILD_TEST_CONFIGS is used. + END_OF_HELP +} + +J=$DPDK_MAKE_JOBS +short=false +unset verbose +maxerr=-Wfatal-errors +while getopts hj:sv ARG ; do + case $ARG in + j ) J=$OPTARG ;; + s ) short=true ;; + v ) verbose='V=1' ;; + h ) print_help ; exit 0 ;; + ? ) print_usage ; exit 1 ;; + esac +done +shift $(($OPTIND - 1)) +configs=${*:-$DPDK_BUILD_TEST_CONFIGS} + +success=false +on_exit () +{ + if $success ; then + [ "$DPDK_NOTIFY" != notify-send ] || \ + notify-send -u low --icon=dialog-information 'DPDK build' 'finished' + elif [ -z "$signal" ] ; then + [ -z "$dir" ] || echo "failed to build $dir" >&2 + [ "$DPDK_NOTIFY" != notify-send ] || \ + notify-send -u low --icon=dialog-error 'DPDK build' 'failed' + fi +} +# catch manual interrupt to ignore notification +trap "signal=INT ; trap - INT ; kill -INT $$" INT +# notify result on exit +trap on_exit EXIT + +cd $(dirname $(readlink -m $0))/.. + +reset_env () +{ + export PATH=$default_path + unset CROSS + unset DPDK_DEP_ARCHIVE + unset DPDK_DEP_CFLAGS + unset DPDK_DEP_LDFLAGS + unset DPDK_DEP_MLX + unset DPDK_DEP_NUMA + unset DPDK_DEP_PCAP + unset DPDK_DEP_SSL + unset DPDK_DEP_SZE + unset DPDK_DEP_ZLIB + unset AESNI_MULTI_BUFFER_LIB_PATH + unset ARMV8_CRYPTO_LIB_PATH + unset LIBMUSDK_PATH + unset LIBSSO_SNOW3G_PATH + unset LIBSSO_KASUMI_PATH + unset LIBSSO_ZUC_PATH + unset PQOS_INSTALL_PATH +} + +config () # <directory> <target> <options> +{ + reconfig=false + if git rev-parse 2>&- && [ -n "$(git diff HEAD~ -- config)" ] ; then + echo 'Default config may have changed' + reconfig=true + fi + if [ ! -e $1/.config ] || $reconfig ; then + echo "================== Configure $1" + make T=$2 O=$1 config + + echo 'Customize configuration' + # Built-in options (lowercase) + ! echo $3 | grep -q '+default' || \ + sed -ri 's,(RTE_MACHINE=")native,\1default,' $1/.config + echo $3 | grep -q '+next' || \ + sed -ri 's,(NEXT_ABI=)y,\1n,' $1/.config + ! echo $3 | grep -q '+shared' || \ + sed -ri 's,(SHARED_LIB=)n,\1y,' $1/.config + ! echo $3 | grep -q '+debug' || ( \ + sed -ri 's,(RTE_LOG_LEVEL=).*,\1RTE_LOG_DEBUG,' $1/.config + sed -ri 's,(_DEBUG.*=)n,\1y,' $1/.config + sed -ri 's,(_STAT.*=)n,\1y,' $1/.config + sed -ri 's,(TEST_PMD_RECORD_.*=)n,\1y,' $1/.config ) + + # Automatic configuration + test "$DPDK_DEP_NUMA" != n || \ + sed -ri 's,(NUMA.*=)y,\1n,' $1/.config + sed -ri 's,(LIBRTE_IEEE1588=)n,\1y,' $1/.config + sed -ri 's,(BYPASS=)n,\1y,' $1/.config + test "$DPDK_DEP_ARCHIVE" != y || \ + sed -ri 's,(RESOURCE_TAR=)n,\1y,' $1/.config + test "$DPDK_DEP_MLX" != y || \ + sed -ri 's,(MLX._PMD=)n,\1y,' $1/.config + test "$DPDK_DEP_SZE" != y || \ + sed -ri 's,(PMD_SZEDATA2=)n,\1y,' $1/.config + test "$DPDK_DEP_ZLIB" != y || \ + sed -ri 's,(BNX2X_PMD=)n,\1y,' $1/.config + sed -ri 's,(NFP_PMD=)n,\1y,' $1/.config + test "$DPDK_DEP_PCAP" != y || \ + sed -ri 's,(PCAP=)n,\1y,' $1/.config + test -z "$ARMV8_CRYPTO_LIB_PATH" || \ + sed -ri 's,(PMD_ARMV8_CRYPTO=)n,\1y,' $1/.config + test -z "$AESNI_MULTI_BUFFER_LIB_PATH" || \ + sed -ri 's,(PMD_AESNI_MB=)n,\1y,' $1/.config + test -z "$AESNI_MULTI_BUFFER_LIB_PATH" || \ + sed -ri 's,(PMD_AESNI_GCM=)n,\1y,' $1/.config + test -z "$LIBSSO_SNOW3G_PATH" || \ + sed -ri 's,(PMD_SNOW3G=)n,\1y,' $1/.config + test -z "$LIBSSO_KASUMI_PATH" || \ + sed -ri 's,(PMD_KASUMI=)n,\1y,' $1/.config + test -z "$LIBSSO_ZUC_PATH" || \ + sed -ri 's,(PMD_ZUC=)n,\1y,' $1/.config + test "$DPDK_DEP_SSL" != y || \ + sed -ri 's,(PMD_OPENSSL=)n,\1y,' $1/.config + test "$DPDK_DEP_SSL" != y || \ + sed -ri 's,(PMD_QAT=)n,\1y,' $1/.config + sed -ri 's,(SCHED_.*=)n,\1y,' $1/.config + test -z "$LIBMUSDK_PATH" || \ + sed -ri 's,(PMD_MRVL_CRYPTO=)n,\1y,' $1/.config + build_config_hook $1 $2 $3 + + # Explicit enabler/disabler (uppercase) + for option in $(echo $3 | sed 's,[~+], &,g') ; do + pattern=$(echo $option | cut -c2-) + if echo $option | grep -q '^~' ; then + sed -ri "s,($pattern=)y,\1n," $1/.config + elif echo $option | grep -q '^+' ; then + sed -ri "s,($pattern=)n,\1y," $1/.config + fi + done + fi +} + +# default empty hook to override in devel config +build_config_hook () # <directory> <target> <options> +{ + : +} + +for conf in $configs ; do + target=$(echo $conf | sed 's,[~+].*,,') + # reload config with DPDK_TARGET set + DPDK_TARGET=$target + reset_env + . $(dirname $(readlink -e $0))/load-devel-config + + options=$(echo $conf | sed 's,[^~+]*,,') + dir=$conf + config $dir $target $options + + echo "================== Build $dir" + make -j$J EXTRA_CFLAGS="$maxerr $DPDK_DEP_CFLAGS" \ + EXTRA_LDFLAGS="$DPDK_DEP_LDFLAGS" $verbose O=$dir + ! $short || break + echo "================== Build tests for $dir" + make test-build -j$J EXTRA_CFLAGS="$maxerr $DPDK_DEP_CFLAGS" \ + EXTRA_LDFLAGS="$DPDK_DEP_LDFLAGS" $verbose O=$dir + echo "================== Build examples for $dir" + export RTE_SDK=$(pwd) + export RTE_TARGET=$dir + make -j$J -sC examples \ + EXTRA_LDFLAGS="$DPDK_DEP_LDFLAGS" $verbose \ + O=$(readlink -m $dir/examples) + unset RTE_TARGET + echo "################## $dir done." + unset dir +done + +if ! $short ; then + mkdir -p .check + echo "================== Build doxygen HTML API" + make doc-api-html >/dev/null 2>.check/doc.txt + echo "================== Build sphinx HTML guides" + make doc-guides-html >/dev/null 2>>.check/doc.txt + echo "================== Check docs" + diff -u /dev/null .check/doc.txt +fi + +success=true diff --git a/dpdk/devtools/test-null.sh b/dpdk/devtools/test-null.sh new file mode 100644 index 00000000..30cd0b03 --- /dev/null +++ b/dpdk/devtools/test-null.sh @@ -0,0 +1,45 @@ +#! /bin/sh -e + +# BSD LICENSE +# +# Copyright 2015 6WIND S.A. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of 6WIND S.A. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Run a quick testpmd forwarding with null PMD without hugepage + +build=${1:-build} +coremask=${2:-3} # default using cores 0 and 1 + +if grep -q SHARED_LIB=y $build/.config; then + pmd='-d librte_pmd_null.so' +fi + +(sleep 1 && echo stop) | +$build/app/testpmd -c $coremask -n 1 --no-huge \ + $pmd --vdev net_null1 --vdev net_null2 -- \ + --total-num-mbufs=2048 -ia diff --git a/dpdk/devtools/validate-abi.sh b/dpdk/devtools/validate-abi.sh new file mode 100644 index 00000000..8caf43e8 --- /dev/null +++ b/dpdk/devtools/validate-abi.sh @@ -0,0 +1,275 @@ +#!/usr/bin/env bash +# BSD LICENSE +# +# Copyright(c) 2015 Neil Horman. All rights reserved. +# Copyright(c) 2017 6WIND S.A. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set -e + +abicheck=abi-compliance-checker +abidump=abi-dumper +default_dst=abi-check +default_target=x86_64-native-linuxapp-gcc + +# trap on error +err_report() { + echo "$0: error at line $1" +} +trap 'err_report $LINENO' ERR + +print_usage () { + cat <<- END_OF_HELP + $(basename $0) [options] <rev1> <rev2> + + This script compares the ABI of 2 git revisions of the current + workspace. The output is a html report and a compilation log. + + The objective is to make sure that applications built against + DSOs from the first revision can still run when executed using + the DSOs built from the second revision. + + <rev1> and <rev2> are git commit id or tags. + + Options: + -h show this help + -j <num> enable parallel compilation with <num> threads + -v show compilation logs on the console + -d <dir> change working directory (default is ${default_dst}) + -t <target> the dpdk target to use (default is ${default_target}) + -f overwrite existing files in destination directory + + The script returns 0 on success, or the value of last failing + call of ${abicheck} (incompatible abi or the tool has run with errors). + The errors returned by ${abidump} are ignored. + + END_OF_HELP +} + +# log in the file, and on stdout if verbose +# $1: level string +# $2: string to be logged +log() { + echo "$1: $2" + if [ "${verbose}" != "true" ]; then + echo "$1: $2" >&3 + fi +} + +# launch a command and log it, taking care of surrounding spaces with quotes +cmd() { + local i s whitespace ret + s="" + whitespace="[[:space:]]" + for i in "$@"; do + if [[ $i =~ $whitespace ]]; then + i=\"$i\" + fi + if [ -z "$s" ]; then + s="$i" + else + s="$s $i" + fi + done + + ret=0 + log "CMD" "$s" + "$@" || ret=$? + if [ "$ret" != "0" ]; then + log "CMD" "previous command returned $ret" + fi + + return $ret +} + +# redirect or copy stderr/stdout to a file +# the syntax is unfamiliar, but it makes the rest of the +# code easier to read, avoiding the use of pipes +set_log_file() { + # save original stdout and stderr in fd 3 and 4 + exec 3>&1 + exec 4>&2 + # create a new fd 5 that send to a file + exec 5> >(cat > $1) + # send stdout and stderr to fd 5 + if [ "${verbose}" = "true" ]; then + exec 1> >(tee /dev/fd/5 >&3) + exec 2> >(tee /dev/fd/5 >&4) + else + exec 1>&5 + exec 2>&5 + fi +} + +# Make sure we configure SHARED libraries +# Also turn off IGB and KNI as those require kernel headers to build +fixup_config() { + local conf=config/defconfig_$target + cmd sed -i -e"$ a\CONFIG_RTE_BUILD_SHARED_LIB=y" $conf + cmd sed -i -e"$ a\CONFIG_RTE_NEXT_ABI=n" $conf + cmd sed -i -e"$ a\CONFIG_RTE_EAL_IGB_UIO=n" $conf + cmd sed -i -e"$ a\CONFIG_RTE_LIBRTE_KNI=n" $conf + cmd sed -i -e"$ a\CONFIG_RTE_KNI_KMOD=n" $conf +} + +# build dpdk for the given tag and dump abi +# $1: hash of the revision +gen_abi() { + local i + + cmd git clone ${dpdkroot} ${dst}/${1} + cmd cd ${dst}/${1} + + log "INFO" "Checking out version ${1} of the dpdk" + # Move to the old version of the tree + cmd git checkout ${1} + + fixup_config + + # Now configure the build + log "INFO" "Configuring DPDK ${1}" + cmd make config T=$target O=$target + + # Checking abi compliance relies on using the dwarf information in + # the shared objects. Build with -g to include them. + log "INFO" "Building DPDK ${1}. This might take a moment" + cmd make -j$parallel O=$target V=1 EXTRA_CFLAGS="-g -Og -Wno-error" \ + EXTRA_LDFLAGS="-g" || log "INFO" "The build failed" + + # Move to the lib directory + cmd cd ${PWD}/$target/lib + log "INFO" "Collecting ABI information for ${1}" + for i in *.so; do + [ -e "$i" ] || break + cmd $abidump ${i} -o $dst/${1}/${i}.dump -lver ${1} || true + # hack to ignore empty SymbolsInfo section (no public ABI) + if grep -q "'SymbolInfo' => {}," $dst/${1}/${i}.dump \ + 2> /dev/null; then + log "INFO" "${i} has no public ABI, remove dump file" + cmd rm -f $dst/${1}/${i}.dump + fi + done +} + +verbose=false +parallel=1 +dst=${default_dst} +target=${default_target} +force=0 +while getopts j:vd:t:fh ARG ; do + case $ARG in + j ) parallel=$OPTARG ;; + v ) verbose=true ;; + d ) dst=$OPTARG ;; + t ) target=$OPTARG ;; + f ) force=1 ;; + h ) print_usage ; exit 0 ;; + ? ) print_usage ; exit 1 ;; + esac +done +shift $(($OPTIND - 1)) + +if [ $# != 2 ]; then + print_usage + exit 1 +fi + +tag1=$1 +tag2=$2 + +# convert path to absolute +case "${dst}" in + /*) ;; + *) dst=${PWD}/${dst} ;; +esac +dpdkroot=$(readlink -e $(dirname $0)/..) + +if [ -e "${dst}" -a "$force" = 0 ]; then + echo "The ${dst} directory is not empty. Remove it, use another" + echo "one (-d <dir>), or force overriding (-f)" + exit 1 +fi + +rm -rf ${dst} +mkdir -p ${dst} +set_log_file ${dst}/abi-check.log +log "INFO" "Logs available in ${dst}/abi-check.log" + +command -v ${abicheck} || log "INFO" "Can't find ${abicheck} utility" +command -v ${abidump} || log "INFO" "Can't find ${abidump} utility" + +hash1=$(git show -s --format=%h "$tag1" -- 2> /dev/null | tail -1) +hash2=$(git show -s --format=%h "$tag2" -- 2> /dev/null | tail -1) + +# Make hashes available in output for non-local reference +tag1="$tag1 ($hash1)" +tag2="$tag2 ($hash2)" + +if [ "$hash1" = "$hash2" ]; then + log "ERROR" "$tag1 and $tag2 are the same revisions" + exit 1 +fi + +cmd mkdir -p ${dst} + +# dump abi for each revision +gen_abi ${hash1} +gen_abi ${hash2} + +# compare the abi dumps +cmd cd ${dst} +ret=0 +list="" +for i in ${hash2}/*.dump; do + name=`basename $i` + libname=${name%.dump} + + if [ ! -f ${hash1}/$name ]; then + log "INFO" "$NAME does not exist in $tag1. skipping..." + continue + fi + + local_ret=0 + cmd $abicheck -l $libname \ + -old ${hash1}/$name -new ${hash2}/$name || local_ret=$? + if [ $local_ret != 0 ]; then + log "NOTICE" "$abicheck returned $local_ret" + ret=$local_ret + list="$list $libname" + fi +done + +if [ $ret != 0 ]; then + log "NOTICE" "ABI may be incompatible, check reports/logs for details." + log "NOTICE" "Incompatible list: $list" +else + log "NOTICE" "No error detected, ABI is compatible." +fi + +log "INFO" "Logs are in ${dst}/abi-check.log" +log "INFO" "HTML reports are in ${dst}/compat_reports directory" + +exit $ret diff --git a/dpdk/doc/api/doxy-api-index.md b/dpdk/doc/api/doxy-api-index.md index 2284a53b..34927023 100644 --- a/dpdk/doc/api/doxy-api-index.md +++ b/dpdk/doc/api/doxy-api-index.md @@ -4,7 +4,7 @@ API {#index} <!-- BSD LICENSE - Copyright 2013 6WIND S.A. + Copyright 2013-2017 6WIND S.A. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions @@ -33,18 +33,34 @@ API {#index} OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --> -There are many libraries, so their headers may be grouped by topics: +The public API headers are grouped by topics: - **device**: [dev] (@ref rte_dev.h), [ethdev] (@ref rte_ethdev.h), [ethctrl] (@ref rte_eth_ctrl.h), + [rte_flow] (@ref rte_flow.h), + [rte_tm] (@ref rte_tm.h), + [rte_mtr] (@ref rte_mtr.h), [cryptodev] (@ref rte_cryptodev.h), + [security] (@ref rte_security.h), + [eventdev] (@ref rte_eventdev.h), + [event_eth_rx_adapter] (@ref rte_event_eth_rx_adapter.h), + [metrics] (@ref rte_metrics.h), + [bitrate] (@ref rte_bitrate.h), + [latency] (@ref rte_latencystats.h), [devargs] (@ref rte_devargs.h), + [PCI] (@ref rte_pci.h) + +- **device specific**: + [softnic] (@ref rte_eth_softnic.h), [bond] (@ref rte_eth_bond.h), - [vhost] (@ref rte_virtio_net.h), + [vhost] (@ref rte_vhost.h), [KNI] (@ref rte_kni.h), - [PCI] (@ref rte_pci.h), + [ixgbe] (@ref rte_pmd_ixgbe.h), + [i40e] (@ref rte_pmd_i40e.h), + [bnxt] (@ref rte_pmd_bnxt.h), + [crypto_scheduler] (@ref rte_cryptodev_scheduler.h) - **memory**: [memseg] (@ref rte_memory.h), @@ -66,28 +82,35 @@ There are many libraries, so their headers may be grouped by topics: - **CPU arch**: [branch prediction] (@ref rte_branch_prediction.h), [cache prefetch] (@ref rte_prefetch.h), + [SIMD] (@ref rte_vect.h), [byte order] (@ref rte_byteorder.h), - [CPU flags] (@ref rte_cpuflags.h) + [CPU flags] (@ref rte_cpuflags.h), + [CPU pause] (@ref rte_pause.h), + [I/O access] (@ref rte_io.h) - **CPU multicore**: [interrupts] (@ref rte_interrupts.h), [launch] (@ref rte_launch.h), [lcore] (@ref rte_lcore.h), [per-lcore] (@ref rte_per_lcore.h), + [service cores] (@ref rte_service.h), + [keepalive] (@ref rte_keepalive.h), [power/freq] (@ref rte_power.h) - **layers**: [ethernet] (@ref rte_ether.h), [ARP] (@ref rte_arp.h), [ICMP] (@ref rte_icmp.h), + [ESP] (@ref rte_esp.h), [IP] (@ref rte_ip.h), [SCTP] (@ref rte_sctp.h), [TCP] (@ref rte_tcp.h), [UDP] (@ref rte_udp.h), + [GRO] (@ref rte_gro.h), + [GSO] (@ref rte_gso.h), [frag/reass] (@ref rte_ip_frag.h), [LPM IPv4 route] (@ref rte_lpm.h), - [LPM IPv6 route] (@ref rte_lpm6.h), - [ACL] (@ref rte_acl.h) + [LPM IPv6 route] (@ref rte_lpm6.h) - **QoS**: [metering] (@ref rte_meter.h), @@ -101,14 +124,19 @@ There are many libraries, so their headers may be grouped by topics: [FBK hash] (@ref rte_fbk_hash.h), [CRC hash] (@ref rte_hash_crc.h) +- **classification** + [reorder] (@ref rte_reorder.h), + [distributor] (@ref rte_distributor.h), + [EFD] (@ref rte_efd.h), + [ACL] (@ref rte_acl.h), + [member] (@ref rte_member.h), + [flow classify] (@ref rte_flow_classify.h) + - **containers**: [mbuf] (@ref rte_mbuf.h), [ring] (@ref rte_ring.h), - [distributor] (@ref rte_distributor.h), - [reorder] (@ref rte_reorder.h), [tailq] (@ref rte_tailq.h), - [bitmap] (@ref rte_bitmap.h), - [ivshmem] (@ref rte_ivshmem.h) + [bitmap] (@ref rte_bitmap.h) - **packet framework**: * [port] (@ref rte_port.h): @@ -137,15 +165,14 @@ There are many libraries, so their headers may be grouped by topics: - **debug**: [jobstats] (@ref rte_jobstats.h), + [pdump] (@ref rte_pdump.h), [hexdump] (@ref rte_hexdump.h), [debug] (@ref rte_debug.h), [log] (@ref rte_log.h), - [warnings] (@ref rte_warnings.h), [errno] (@ref rte_errno.h) - **misc**: [EAL config] (@ref rte_eal.h), [common] (@ref rte_common.h), [ABI compat] (@ref rte_compat.h), - [keepalive] (@ref rte_keepalive.h), [version] (@ref rte_version.h) diff --git a/dpdk/doc/api/doxy-api.conf b/dpdk/doc/api/doxy-api.conf index af5d6dd4..b2cbe940 100644 --- a/dpdk/doc/api/doxy-api.conf +++ b/dpdk/doc/api/doxy-api.conf @@ -1,6 +1,6 @@ # BSD LICENSE # -# Copyright 2013 6WIND S.A. +# Copyright 2013-2017 6WIND S.A. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -30,34 +30,49 @@ PROJECT_NAME = DPDK INPUT = doc/api/doxy-api-index.md \ - doc/api/examples.dox \ + drivers/crypto/scheduler \ + drivers/net/bnxt \ drivers/net/bonding \ + drivers/net/i40e \ + drivers/net/ixgbe \ + drivers/net/softnic \ lib/librte_eal/common/include \ lib/librte_eal/common/include/generic \ lib/librte_acl \ + lib/librte_bitratestats \ lib/librte_cfgfile \ lib/librte_cmdline \ lib/librte_compat \ lib/librte_cryptodev \ lib/librte_distributor \ + lib/librte_efd \ lib/librte_ether \ + lib/librte_eventdev \ + lib/librte_flow_classify \ + lib/librte_gro \ + lib/librte_gso \ lib/librte_hash \ lib/librte_ip_frag \ - lib/librte_ivshmem \ lib/librte_jobstats \ lib/librte_kni \ lib/librte_kvargs \ + lib/librte_latencystats \ lib/librte_lpm \ lib/librte_mbuf \ + lib/librte_member \ lib/librte_mempool \ lib/librte_meter \ + lib/librte_metrics \ lib/librte_net \ + lib/librte_pci \ + lib/librte_pdump \ lib/librte_pipeline \ lib/librte_port \ lib/librte_power \ lib/librte_reorder \ lib/librte_ring \ lib/librte_sched \ + lib/librte_security \ lib/librte_table \ lib/librte_timer \ lib/librte_vhost diff --git a/dpdk/doc/api/examples.dox b/dpdk/doc/api/examples.dox deleted file mode 100644 index 200af0b0..00000000 --- a/dpdk/doc/api/examples.dox +++ /dev/null @@ -1,113 +0,0 @@ -/** -@page examples DPDK Example Programs - -@example bond/main.c -@example cmdline/commands.c -@example cmdline/main.c -@example cmdline/parse_obj_list.c -@example distributor/main.c -@example dpdk_qat/crypto.c -@example dpdk_qat/main.c -@example ethtool/ethtool-app/ethapp.c -@example ethtool/ethtool-app/main.c -@example ethtool/lib/rte_ethtool.c -@example exception_path/main.c -@example helloworld/main.c -@example ip_fragmentation/main.c -@example ip_pipeline/config_check.c -@example ip_pipeline/config_parse.c -@example ip_pipeline/config_parse_tm.c -@example ip_pipeline/cpu_core_map.c -@example ip_pipeline/init.c -@example ip_pipeline/main.c -@example ip_pipeline/pipeline/pipeline_common_be.c -@example ip_pipeline/pipeline/pipeline_common_fe.c -@example ip_pipeline/pipeline/pipeline_firewall_be.c -@example ip_pipeline/pipeline/pipeline_firewall.c -@example ip_pipeline/pipeline/pipeline_flow_actions_be.c -@example ip_pipeline/pipeline/pipeline_flow_actions.c -@example ip_pipeline/pipeline/pipeline_flow_classification_be.c -@example ip_pipeline/pipeline/pipeline_flow_classification.c -@example ip_pipeline/pipeline/pipeline_master_be.c -@example ip_pipeline/pipeline/pipeline_master.c -@example ip_pipeline/pipeline/pipeline_passthrough_be.c -@example ip_pipeline/pipeline/pipeline_passthrough.c -@example ip_pipeline/pipeline/pipeline_routing_be.c -@example ip_pipeline/pipeline/pipeline_routing.c -@example ip_pipeline/thread.c -@example ip_pipeline/thread_fe.c -@example ip_reassembly/main.c -@example ipv4_multicast/main.c -@example kni/main.c -@example l2fwd-crypto/main.c -@example l2fwd-ivshmem/guest/guest.c -@example l2fwd-ivshmem/host/host.c -@example l2fwd-jobstats/main.c -@example l2fwd-keepalive/main.c -@example l2fwd/main.c -@example l3fwd-acl/main.c -@example l3fwd/main.c -@example l3fwd-power/main.c -@example l3fwd-vf/main.c -@example link_status_interrupt/main.c -@example load_balancer/config.c -@example load_balancer/init.c -@example load_balancer/main.c -@example load_balancer/runtime.c -@example multi_process/client_server_mp/mp_client/client.c -@example multi_process/client_server_mp/mp_server/args.c -@example multi_process/client_server_mp/mp_server/init.c -@example multi_process/client_server_mp/mp_server/main.c -@example multi_process/l2fwd_fork/flib.c -@example multi_process/l2fwd_fork/main.c -@example multi_process/simple_mp/main.c -@example multi_process/simple_mp/mp_commands.c -@example multi_process/symmetric_mp/main.c -@example netmap_compat/bridge/bridge.c -@example netmap_compat/lib/compat_netmap.c -@example packet_ordering/main.c -@example performance-thread/common/arch/x86/ctx.c -@example performance-thread/common/lthread.c -@example performance-thread/common/lthread_cond.c -@example performance-thread/common/lthread_diag.c -@example performance-thread/common/lthread_mutex.c -@example performance-thread/common/lthread_sched.c -@example performance-thread/common/lthread_tls.c -@example performance-thread/l3fwd-thread/main.c -@example performance-thread/pthread_shim/main.c -@example performance-thread/pthread_shim/pthread_shim.c -@example ptpclient/ptpclient.c -@example qos_meter/main.c -@example qos_meter/rte_policer.c -@example qos_sched/app_thread.c -@example qos_sched/args.c -@example qos_sched/cfg_file.c -@example qos_sched/cmdline.c -@example qos_sched/init.c -@example qos_sched/main.c -@example qos_sched/stats.c -@example quota_watermark/qw/args.c -@example quota_watermark/qwctl/commands.c -@example quota_watermark/qwctl/qwctl.c -@example quota_watermark/qw/init.c -@example quota_watermark/qw/main.c -@example rxtx_callbacks/main.c -@example skeleton/basicfwd.c -@example tep_termination/main.c -@example tep_termination/vxlan.c -@example tep_termination/vxlan_setup.c -@example timer/main.c -@example vhost/main.c -@example vhost_xen/main.c -@example vhost_xen/vhost_monitor.c -@example vhost_xen/xenstore_parse.c -@example vmdq_dcb/main.c -@example vmdq/main.c -@example vm_power_manager/channel_manager.c -@example vm_power_manager/channel_monitor.c -@example vm_power_manager/guest_cli/main.c -@example vm_power_manager/guest_cli/vm_power_cli_guest.c -@example vm_power_manager/main.c -@example vm_power_manager/power_manager.c -@example vm_power_manager/vm_power_cli.c -*/ diff --git a/dpdk/doc/build-sdk-quick.txt b/dpdk/doc/build-sdk-quick.txt index 967ff095..ff297806 100644 --- a/dpdk/doc/build-sdk-quick.txt +++ b/dpdk/doc/build-sdk-quick.txt @@ -1,7 +1,10 @@ Basic build + make defconfig && make + or make config T=x86_64-native-linuxapp-gcc && make Build commands config get configuration from target template (T=) + defconfig auto-select target template based on arch, OS, etc. all same as build (default rule) build build in a configured directory clean remove files but keep configuration @@ -9,6 +12,10 @@ Build commands install install optionally staged in DESTDIR examples build examples for given targets (T=) examples_clean clean examples for given targets (T=) + test compile tests and run basic unit tests + test-* run specific subset of unit tests + tags|etags|gtags generate tags database for given targets (T=) + cscope generate cscope database for given targets (T=) Build variables EXTRA_CPPFLAGS preprocessor options EXTRA_CFLAGS compiler options @@ -20,8 +27,8 @@ Build variables V verbose D debug dependencies O build directory (default: build/ - install T= default: ./) - DESTDIR staging install directory - prefix root install directory + DESTDIR staging install directory (default: empty) + prefix root install directory (default: /usr/local) T target template - used with config or install format: <arch-machine-execenv-toolchain> templates in config/defconfig_* diff --git a/dpdk/doc/guides/conf.py b/dpdk/doc/guides/conf.py index 2c5610f8..31f914a8 100644 --- a/dpdk/doc/guides/conf.py +++ b/dpdk/doc/guides/conf.py @@ -28,28 +28,51 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from __future__ import print_function import subprocess from docutils import nodes from distutils.version import LooseVersion from sphinx import __version__ as sphinx_version from sphinx.highlighting import PygmentsBridge from pygments.formatters.latex import LatexFormatter +from os import listdir +from os.path import basename +from os.path import dirname +from os.path import join as path_join + +try: + # Python 2. + import ConfigParser as configparser +except: + # Python 3. + import configparser + +try: + import sphinx_rtd_theme + + html_theme = "sphinx_rtd_theme" + html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] +except: + print('Install the sphinx ReadTheDocs theme for improved html documentation ' + 'layout: pip install sphinx_rtd_theme') + pass project = 'Data Plane Development Kit' - -if LooseVersion(sphinx_version) >= LooseVersion('1.3.1'): - html_theme = "sphinx_rtd_theme" html_logo = '../logo/DPDK_logo_vertical_rev_small.png' latex_logo = '../logo/DPDK_logo_horizontal_tag.png' html_add_permalinks = "" html_show_copyright = False highlight_language = 'none' -version = subprocess.check_output(['make', '-sRrC', '../../', 'showversion']).decode('utf-8').rstrip() +version = subprocess.check_output(['make', '-sRrC', '../../', 'showversion']) +version = version.decode('utf-8').rstrip() release = version master_doc = 'index' +# Maximum feature description string length +feature_str_len = 25 + # Figures, tables and code-blocks automatically numbered if they have caption numfig = True @@ -62,7 +85,7 @@ latex_documents = [ ] # Latex directives to be included directly in the latex/pdf docs. -latex_preamble = r""" +custom_latex_preamble = r""" \usepackage[utf8]{inputenc} \usepackage[T1]{fontenc} \usepackage{helvet} @@ -78,9 +101,10 @@ latex_elements = { 'classoptions': ',openany,oneside', 'babel': '\\usepackage[english]{babel}', # customize Latex formatting - 'preamble': latex_preamble + 'preamble': custom_latex_preamble } + # Override the default Latex formatter in order to modify the # code/verbatim blocks. class CustomLatexFormatter(LatexFormatter): @@ -92,12 +116,24 @@ class CustomLatexFormatter(LatexFormatter): # Replace the default latex formatter. PygmentsBridge.latex_formatter = CustomLatexFormatter -######## :numref: fallback ######## +# Configuration for man pages +man_pages = [("testpmd_app_ug/run_app", "testpmd", + "tests for dpdk pmds", "", 1), + ("tools/pdump", "dpdk-pdump", + "enable packet capture on dpdk ports", "", 1), + ("tools/proc_info", "dpdk-procinfo", + "access dpdk port stats and memory info", "", 1), + ("tools/pmdinfo", "dpdk-pmdinfo", + "dump a PMDs hardware support info", "", 1), + ("tools/devbind", "dpdk-devbind", + "check device status and bind/unbind them from drivers", "", 8)] + + +# ####### :numref: fallback ######## # The following hook functions add some simple handling for the :numref: # directive for Sphinx versions prior to 1.3.1. The functions replace the # :numref: reference with a link to the target (for all Sphinx doc types). # It doesn't try to label figures/tables. - def numref_role(reftype, rawtext, text, lineno, inliner): """ Add a Sphinx role to handle numref references. Note, we can't convert @@ -111,6 +147,7 @@ def numref_role(reftype, rawtext, text, lineno, inliner): internal=True) return [newnode], [] + def process_numref(app, doctree, from_docname): """ Process the numref nodes once the doctree has been built and prior to @@ -146,7 +183,234 @@ def process_numref(app, doctree, from_docname): internal=True) node.replace_self(newnode) + +def generate_overview_table(output_filename, table_id, section, table_name, title): + """ + Function to generate the Overview Table from the ini files that define + the features for each driver. + + The default features for the table and their order is defined by the + 'default.ini' file. + + """ + # Default warning string. + warning = 'Warning generate_overview_table()' + + # Get the default features and order from the 'default.ini' file. + ini_path = path_join(dirname(output_filename), 'features') + config = configparser.ConfigParser() + config.optionxform = str + config.read(path_join(ini_path, 'default.ini')) + default_features = config.items(section) + + # Create a dict of the valid features to validate the other ini files. + valid_features = {} + max_feature_length = 0 + for feature in default_features: + key = feature[0] + valid_features[key] = ' ' + max_feature_length = max(max_feature_length, len(key)) + + # Get a list of driver ini files, excluding 'default.ini'. + ini_files = [basename(file) for file in listdir(ini_path) + if file.endswith('.ini') and file != 'default.ini'] + ini_files.sort() + + # Build up a list of the table header names from the ini filenames. + header_names = [] + for ini_filename in ini_files: + name = ini_filename[:-4] + name = name.replace('_vf', 'vf') + + # Pad the table header names to match the existing format. + if '_vec' in name: + pmd, vec = name.split('_') + name = '{0:{fill}{align}7}vec'.format(pmd, fill='.', align='<') + else: + name = '{0:{fill}{align}10}'.format(name, fill=' ', align='<') + + header_names.append(name) + + # Create a dict of the defined features for each driver from the ini files. + ini_data = {} + for ini_filename in ini_files: + config = configparser.ConfigParser() + config.optionxform = str + config.read(path_join(ini_path, ini_filename)) + + # Initialize the dict with the default.ini value. + ini_data[ini_filename] = valid_features.copy() + + # Check for a valid ini section. + if not config.has_section(section): + print("{}: File '{}' has no [{}] secton".format(warning, + ini_filename, + section)) + continue + + # Check for valid features names. + for name, value in config.items(section): + if name not in valid_features: + print("{}: Unknown feature '{}' in '{}'".format(warning, + name, + ini_filename)) + continue + + if value is not '': + # Get the first letter only. + ini_data[ini_filename][name] = value[0] + + # Print out the RST Driver Overview table from the ini file data. + outfile = open(output_filename, 'w') + num_cols = len(header_names) + + print_table_css(outfile, table_id) + print('.. table:: ' + table_name + '\n', file=outfile) + print_table_header(outfile, num_cols, header_names, title) + print_table_body(outfile, num_cols, ini_files, ini_data, default_features) + + +def print_table_header(outfile, num_cols, header_names, title): + """ Print the RST table header. The header names are vertical. """ + print_table_divider(outfile, num_cols) + + line = '' + for name in header_names: + line += ' ' + name[0] + + print_table_row(outfile, title, line) + + for i in range(1, 10): + line = '' + for name in header_names: + line += ' ' + name[i] + + print_table_row(outfile, '', line) + + print_table_divider(outfile, num_cols) + + +def print_table_body(outfile, num_cols, ini_files, ini_data, default_features): + """ Print out the body of the table. Each row is a NIC feature. """ + + for feature, _ in default_features: + line = '' + + for ini_filename in ini_files: + line += ' ' + ini_data[ini_filename][feature] + + print_table_row(outfile, feature, line) + + print_table_divider(outfile, num_cols) + + +def print_table_row(outfile, feature, line): + """ Print a single row of the table with fixed formatting. """ + line = line.rstrip() + print(' {:<{}}{}'.format(feature, feature_str_len, line), file=outfile) + + +def print_table_divider(outfile, num_cols): + """ Print the table divider line. """ + line = ' ' + column_dividers = ['='] * num_cols + line += ' '.join(column_dividers) + + feature = '=' * feature_str_len + + print_table_row(outfile, feature, line) + + +def print_table_css(outfile, table_id): + template = """ +.. raw:: html + + <style> + .wy-nav-content { + opacity: .99; + } + table#idx { + cursor: default; + overflow: hidden; + } + table#idx th, table#idx td { + text-align: center; + } + table#idx th { + font-size: 80%; + white-space: pre-wrap; + vertical-align: top; + padding: 0.5em 0; + min-width: 0.9em; + width: 2em; + } + table#idx col:first-child { + width: 0; + } + table#idx th:first-child { + vertical-align: bottom; + } + table#idx td { + font-size: 70%; + padding: 1px; + } + table#idx td:first-child { + padding-left: 1em; + text-align: left; + } + table#idx tr:nth-child(2n-1) td { + background-color: rgba(210, 210, 210, 0.2); + } + table#idx th:not(:first-child):hover, + table#idx td:not(:first-child):hover { + position: relative; + } + table#idx th:not(:first-child):hover::after, + table#idx td:not(:first-child):hover::after { + content: ''; + height: 6000px; + top: -3000px; + width: 100%; + left: 0; + position: absolute; + z-index: -1; + background-color: #ffb; + } + table#idx tr:hover td { + background-color: #ffb; + } + </style> +""" + print(template.replace("idx", "id%d" % (table_id)), file=outfile) + + def setup(app): + table_file = dirname(__file__) + '/nics/overview_table.txt' + generate_overview_table(table_file, 1, + 'Features', + 'Features availability in networking drivers', + 'Feature') + table_file = dirname(__file__) + '/cryptodevs/overview_feature_table.txt' + generate_overview_table(table_file, 1, + 'Features', + 'Features availability in crypto drivers', + 'Feature') + table_file = dirname(__file__) + '/cryptodevs/overview_cipher_table.txt' + generate_overview_table(table_file, 2, + 'Cipher', + 'Cipher algorithms in crypto drivers', + 'Cipher algorithm') + table_file = dirname(__file__) + '/cryptodevs/overview_auth_table.txt' + generate_overview_table(table_file, 3, + 'Auth', + 'Authentication algorithms in crypto drivers', + 'Authentication algorithm') + table_file = dirname(__file__) + '/cryptodevs/overview_aead_table.txt' + generate_overview_table(table_file, 4, + 'AEAD', + 'AEAD algorithms in crypto drivers', + 'AEAD algorithm') + if LooseVersion(sphinx_version) < LooseVersion('1.3.1'): print('Upgrade sphinx to version >= 1.3.1 for ' 'improved Figure/Table number handling.') diff --git a/dpdk/doc/guides/contributing/coding_style.rst b/dpdk/doc/guides/contributing/coding_style.rst index 1eb67f34..d8e4a0f9 100644 --- a/dpdk/doc/guides/contributing/coding_style.rst +++ b/dpdk/doc/guides/contributing/coding_style.rst @@ -603,22 +603,30 @@ In the DPDK environment, use the logging interface provided: .. code-block:: c - #define RTE_LOGTYPE_TESTAPP1 RTE_LOGTYPE_USER1 - #define RTE_LOGTYPE_TESTAPP2 RTE_LOGTYPE_USER2 + /* register log types for this application */ + int my_logtype1 = rte_log_register("myapp.log1"); + int my_logtype2 = rte_log_register("myapp.log2"); - /* enable these logs type */ - rte_set_log_type(RTE_LOGTYPE_TESTAPP1, 1); - rte_set_log_type(RTE_LOGTYPE_TESTAPP2, 1); + /* set global log level to INFO */ + rte_log_set_global_level(RTE_LOG_INFO); + + /* only display messages higher than NOTICE for log2 (default + * is DEBUG) */ + rte_log_set_level(my_logtype2, RTE_LOG_NOTICE); + + /* enable all PMD logs (whose identifier string starts with "pmd") */ + rte_log_set_level_regexp("pmd.*", RTE_LOG_DEBUG); /* log in debug level */ - rte_set_log_level(RTE_LOG_DEBUG); - RTE_LOG(DEBUG, TESTAPP1, "this is is a debug level message\n"); - RTE_LOG(INFO, TESTAPP1, "this is is a info level message\n"); - RTE_LOG(WARNING, TESTAPP1, "this is is a warning level message\n"); + rte_log_set_global_level(RTE_LOG_DEBUG); + RTE_LOG(DEBUG, my_logtype1, "this is is a debug level message\n"); + RTE_LOG(INFO, my_logtype1, "this is is a info level message\n"); + RTE_LOG(WARNING, my_logtype1, "this is is a warning level message\n"); + RTE_LOG(WARNING, my_logtype2, "this is is a debug level message (not displayed)\n"); /* log in info level */ - rte_set_log_level(RTE_LOG_INFO); - RTE_LOG(DEBUG, TESTAPP2, "debug level message (not displayed)\n"); + rte_log_set_global_level(RTE_LOG_INFO); + RTE_LOG(DEBUG, my_logtype1, "debug level message (not displayed)\n"); Branch Prediction ~~~~~~~~~~~~~~~~~ @@ -690,6 +698,7 @@ Control Statements Python Code ----------- -All python code should be compliant with `PEP8 (Style Guide for Python Code) <https://www.python.org/dev/peps/pep-0008/>`_. +All Python code should work with Python 2.7+ and 3.2+ and be compliant with +`PEP8 (Style Guide for Python Code) <https://www.python.org/dev/peps/pep-0008/>`_. The ``pep8`` tool can be used for testing compliance with the guidelines. diff --git a/dpdk/doc/guides/contributing/design.rst b/dpdk/doc/guides/contributing/design.rst index bac3d1b4..88d3a433 100644 --- a/dpdk/doc/guides/contributing/design.rst +++ b/dpdk/doc/guides/contributing/design.rst @@ -158,3 +158,17 @@ cache bandwidth, memory bandwidth, etc) that depends on: branches are usually required. When processing a burst of packets that have been validated for header integrity, counting the number of bits set in a bitmask might be needed. + +PF and VF Considerations +------------------------ + +The primary goal of DPDK is to provide a userspace dataplane. Managing VFs from +a PF driver is a control plane feature and developers should generally rely on +the Linux Kernel for that. + +Developers should work with the Linux Kernel community to get the required +functionality upstream. PF functionality should only be added to DPDK for +testing and prototyping purposes while the kernel work is ongoing. It should +also be marked with an "EXPERIMENTAL" tag. If the functionality isn't +upstreamable then a case can be made to maintain the PF functionality in DPDK +without the EXPERIMENTAL tag. diff --git a/dpdk/doc/guides/contributing/documentation.rst b/dpdk/doc/guides/contributing/documentation.rst index b2cc903f..170dacdb 100644 --- a/dpdk/doc/guides/contributing/documentation.rst +++ b/dpdk/doc/guides/contributing/documentation.rst @@ -34,7 +34,6 @@ The main directories that contain files related to documentation are shown below |-- testpmd_app_ug |-- rel_notes |-- nics - |-- xen |-- ... @@ -282,33 +281,21 @@ The additional guidelines below reiterate or expand upon those guidelines. Line Length ~~~~~~~~~~~ -* The recommended style for the DPDK documentation is to put sentences on separate lines. - This allows for easier reviewing of patches. - Multiple sentences which are not separated by a blank line are joined automatically into paragraphs, for example:: +* Lines in sentences should be less than 80 characters and wrapped at + words. Multiple sentences which are not separated by a blank line are joined + automatically into paragraphs. - Here is an example sentence. - Long sentences over the limit shown below can be wrapped onto - a new line. - These three sentences will be joined into the same paragraph. +* Lines in literal blocks **must** be less than 80 characters since + they are not wrapped by the document formatters and can exceed the page width + in PDF documents. - This is a new paragraph, since it is separated from the - previous paragraph by a blank line. + Long literal command lines can be shown wrapped with backslashes. For + example:: - This would be rendered as follows: - - *Here is an example sentence. - Long sentences over the limit shown below can be wrapped onto - a new line. - These three sentences will be joined into the same paragraph.* - - *This is a new paragraph, since it is separated from the - previous paragraph by a blank line.* - - -* Long sentences should be wrapped at 120 characters +/- 10 characters. They should be wrapped at words. - -* Lines in literal blocks must by less than 80 characters since they aren't wrapped by the document formatters - and can exceed the page width in PDF documents. + testpmd -l 2-3 -n 4 \ + --vdev=virtio_user0,path=/dev/vhost-net,queues=2,queue_size=1024 \ + -- -i --txqflags=0x0 --disable-hw-vlan --enable-lro \ + --enable-rx-cksum --txq=2 --rxq=2 --rxd=1024 --txd=1024 Whitespace @@ -332,7 +319,7 @@ Whitespace Section Headers ~~~~~~~~~~~~~~~ -* Section headers should use the use the following underline formats:: +* Section headers should use the following underline formats:: Level 1 Heading =============== @@ -380,12 +367,11 @@ Lists #. Item one. - #. Item two is a long line that is wrapped and then indented - to match the start of the e first line. - #. Item two is a long line that is wrapped and then indented to match the start of the previous line. + #. Item three. + * Definition lists can be written with or without a bullet:: * Item one. @@ -458,8 +444,8 @@ Code and Literal block sections For long literal lines that exceed that limit try to wrap the text at sensible locations. For example a long command line could be documented like this and still work if copied directly from the docs:: - build/app/testpmd -c7 -n3 --vdev=eth_pcap0,iface=eth0 \ - --vdev=eth_pcap1,iface=eth1 \ + build/app/testpmd -l 0-2 -n3 --vdev=net_pcap0,iface=eth0 \ + --vdev=net_pcap1,iface=eth1 \ -- -i --nb-cores=2 --nb-ports=2 \ --total-num-mbufs=2048 @@ -477,7 +463,7 @@ Images * `Inkscape <http://inkscape.org>`_ is the recommended graphics editor for creating the images. Use some of the older images in ``doc/guides/prog_guide/img/`` as a template, for example ``mbuf1.svg`` - or ``ring-enqueue.svg``. + or ``ring-enqueue1.svg``. * The SVG images should include a copyright notice, as an XML comment. @@ -631,7 +617,7 @@ The following are some guidelines for use of Doxygen in the DPDK API documentati * @param devargs * A pointer to a strings array describing the new device * to be attached. The strings should be a pci address like - * `0000:01:00.0` or **virtual** device name like `eth_pcap0`. + * `0000:01:00.0` or **virtual** device name like `net_pcap0`. * @param port_id * A pointer to a port identifier actually attached. * @@ -643,7 +629,7 @@ The following are some guidelines for use of Doxygen in the DPDK API documentati * Doxygen supports Markdown style syntax such as bold, italics, fixed width text and lists. For example the second line in the ``devargs`` parameter in the previous example will be rendered as: - The strings should be a pci address like ``0000:01:00.0`` or **virtual** device name like ``eth_pcap0``. + The strings should be a pci address like ``0000:01:00.0`` or **virtual** device name like ``net_pcap0``. * Use ``-`` instead of ``*`` for lists within the Doxygen comment since the latter can get confused with the comment delimiter. diff --git a/dpdk/doc/guides/contributing/index.rst b/dpdk/doc/guides/contributing/index.rst index f6af317f..329b678a 100644 --- a/dpdk/doc/guides/contributing/index.rst +++ b/dpdk/doc/guides/contributing/index.rst @@ -10,4 +10,5 @@ Contributor's Guidelines versioning documentation patches + stable cheatsheet diff --git a/dpdk/doc/guides/contributing/patches.rst b/dpdk/doc/guides/contributing/patches.rst index 16a21a52..40983c15 100644 --- a/dpdk/doc/guides/contributing/patches.rst +++ b/dpdk/doc/guides/contributing/patches.rst @@ -7,12 +7,12 @@ This document outlines the guidelines for submitting code to DPDK. The DPDK development process is modelled (loosely) on the Linux Kernel development model so it is worth reading the Linux kernel guide on submitting patches: -`How to Get Your Change Into the Linux Kernel <http://www.kernel.org/doc/Documentation/SubmittingPatches>`_. +`How to Get Your Change Into the Linux Kernel <https://www.kernel.org/doc/html/latest/process/submitting-patches.html>`_. The rationale for many of the DPDK guidelines is explained in greater detail in the kernel guidelines. The DPDK Development Process ------------------------------ +---------------------------- The DPDK development process has the following features: @@ -20,25 +20,94 @@ The DPDK development process has the following features: * There is a mailing list where developers submit patches. * There are maintainers for hierarchical components. * Patches are reviewed publicly on the mailing list. -* Successfully reviewed patches are merged to the master branch of the repository. +* Successfully reviewed patches are merged to the repository. +* Patches should be sent to the target repository or sub-tree, see below. +* All sub-repositories are merged into main repository for ``-rc1`` and ``-rc2`` versions of the release. +* After the ``-rc2`` release all patches should target the main repository. The mailing list for DPDK development is `dev@dpdk.org <http://dpdk.org/ml/archives/dev/>`_. Contributors will need to `register for the mailing list <http://dpdk.org/ml/listinfo/dev>`_ in order to submit patches. -It is also worth registering for the DPDK `Patchwork <http://dpdk.org/dev/patchwxispork/project/dpdk/list/>`_ +It is also worth registering for the DPDK `Patchwork <http://dpdk.org/dev/patchwork/project/dpdk/list/>`_ The development process requires some familiarity with the ``git`` version control system. Refer to the `Pro Git Book <http://www.git-scm.com/book/>`_ for further information. +Maintainers and Sub-trees +------------------------- + +The DPDK maintenance hierarchy is divided into a main repository ``dpdk`` and sub-repositories ``dpdk-next-*``. + +There are maintainers for the trees and for components within the tree. + +Trees and maintainers are listed in the ``MAINTAINERS`` file. For example:: + + Crypto Drivers + -------------- + M: Some Name <some.name@email.com> + B: Another Name <another.name@email.com> + T: git://dpdk.org/next/dpdk-next-crypto + + Intel AES-NI GCM PMD + M: Some One <some.one@email.com> + F: drivers/crypto/aesni_gcm/ + F: doc/guides/cryptodevs/aesni_gcm.rst + +Where: + +* ``M`` is a tree or component maintainer. +* ``B`` is a tree backup maintainer. +* ``T`` is a repository tree. +* ``F`` is a maintained file or directory. + +Additional details are given in the ``MAINTAINERS`` file. + +The role of the component maintainers is to: + +* Review patches for the component or delegate the review. + The review should be done, ideally, within 1 week of submission to the mailing list. +* Add an ``acked-by`` to patches, or patchsets, that are ready for committing to a tree. +* Reply to questions asked about the component. + +Component maintainers can be added or removed by submitting a patch to the ``MAINTAINERS`` file. +Maintainers should have demonstrated a reasonable level of contributions or reviews to the component area. +The maintainer should be confirmed by an ``ack`` from an established contributor. +There can be more than one component maintainer if desired. + +The role of the tree maintainers is to: + +* Maintain the overall quality of their tree. + This can entail additional review, compilation checks or other tests deemed necessary by the maintainer. +* Commit patches that have been reviewed by component maintainers and/or other contributors. + The tree maintainer should determine if patches have been reviewed sufficiently. +* Ensure that patches are reviewed in a timely manner. +* Prepare the tree for integration. +* Ensure that there is a designated back-up maintainer and coordinate a handover for periods where the + tree maintainer can't perform their role. + +Tree maintainers can be added or removed by submitting a patch to the ``MAINTAINERS`` file. +The proposer should justify the need for a new sub-tree and should have demonstrated a sufficient level of contributions in the area or to a similar area. +The maintainer should be confirmed by an ``ack`` from an existing tree maintainer. +Disagreements on trees or maintainers can be brought to the Technical Board. + +The backup maintainer for the master tree should be selected from the existing sub-tree maintainers from the project. +The backup maintainer for a sub-tree should be selected from among the component maintainers within that sub-tree. + + Getting the Source Code ----------------------- -The source code can be cloned using either of the following:: +The source code can be cloned using either of the following: + +main repository:: git clone git://dpdk.org/dpdk - git clone http://dpdk.org/git/dpdk +sub-repositories (`list <http://dpdk.org/browse/next>`_):: + + git clone git://dpdk.org/next/dpdk-next-* + git clone http://dpdk.org/git/next/dpdk-next-* Make your Changes ----------------- @@ -124,7 +193,7 @@ Here are some guidelines for the body of a commit message: git commit --signoff # or -s The purpose of the signoff is explained in the - `Developer's Certificate of Origin <http://www.kernel.org/doc/Documentation/SubmittingPatches>`_ + `Developer's Certificate of Origin <https://www.kernel.org/doc/html/latest/process/submitting-patches.html#developer-s-certificate-of-origin-1-1>`_ section of the Linux kernel guidelines. .. Note:: @@ -138,18 +207,21 @@ Here are some guidelines for the body of a commit message: * The text of the commit message should be wrapped at 72 characters. -* When fixing a regression, it is a good idea to reference the id of the commit which introduced the bug. - You can generate the required text using the following git alias:: +* When fixing a regression, it is required to reference the id of the commit + which introduced the bug, and put the original author of that commit on CC. + You can generate the required lines using the following git alias, which prints + the commit SHA and the author of the original code:: - git config alias.fixline "log -1 --abbrev=12 --format='Fixes: %h (\"%s\")'" + git config alias.fixline "log -1 --abbrev=12 --format='Fixes: %h (\"%s\")%nCc: %ae'" - The ``Fixes:`` line can then be added to the commit message:: + The output of ``git fixline <SHA>`` must then be added to the commit message:: - doc: fix vhost sample parameter + doc: fix some parameter description - Update the docs to reflect removed dev-index. + Update the docs, fixing description of some parameter. - Fixes: 17b8320a3e11 ("vhost: remove index parameter") + Fixes: abcdefgh1234 ("doc: add some parameter") + Cc: author@example.com Signed-off-by: Alex Smith <alex.smith@example.com> @@ -157,13 +229,9 @@ Here are some guidelines for the body of a commit message: * Use correct capitalization, punctuation and spelling. -In addition to the ``Signed-off-by:`` name the commit messages can also have one or more of the following: - -* ``Reported-by:`` The reporter of the issue. -* ``Tested-by:`` The tester of the change. -* ``Reviewed-by:`` The reviewer of the change. -* ``Suggested-by:`` The person who suggested the change. -* ``Acked-by:`` When a previous version of the patch was acked and the ack is still relevant. +In addition to the ``Signed-off-by:`` name the commit messages can also have +tags for who reported, suggested, tested and reviewed the patch being +posted. Please refer to the `Tested, Acked and Reviewed by`_ section. Creating Patches @@ -230,7 +298,7 @@ For example:: Checking the Patches -------------------- -Patches should be checked for formatting and syntax issues using the ``checkpatches.sh`` script in the ``scripts`` +Patches should be checked for formatting and syntax issues using the ``checkpatches.sh`` script in the ``devtools`` directory of the DPDK repo. This uses the Linux kernel development tool ``checkpatch.pl`` which can be obtained by cloning, and periodically, updating the Linux kernel sources. @@ -245,7 +313,7 @@ files, in order of preference:: Once the environment variable the script can be run as follows:: - scripts/checkpatches.sh ~/patch/ + devtools/checkpatches.sh ~/patch/ The script usage is:: @@ -272,10 +340,10 @@ Where the range is a ``git log`` option. Checking Compilation -------------------- -Compilation of patches and changes should be tested using the the ``test-build.sh`` script in the ``scripts`` +Compilation of patches and changes should be tested using the the ``test-build.sh`` script in the ``devtools`` directory of the DPDK repo:: - scripts/test-build.sh x86_64-native-linuxapp-gcc+next+shared + devtools/test-build.sh x86_64-native-linuxapp-gcc+next+shared The script usage is:: @@ -294,12 +362,11 @@ Examples of configs are:: x86_64-native-linuxapp-gcc+next+shared x86_64-native-linuxapp-clang+shared -The builds can be modifies via the following environmental variables: +The builds can be modified via the following environmental variables: * ``DPDK_BUILD_TEST_CONFIGS`` (target1+option1+option2 target2) * ``DPDK_DEP_CFLAGS`` * ``DPDK_DEP_LDFLAGS`` -* ``DPDK_DEP_MOFED`` (y/[n]) * ``DPDK_DEP_PCAP`` (y/[n]) * ``DPDK_NOTIFY`` (notify-send) @@ -336,6 +403,10 @@ The appropriate maintainer can be found in the ``MAINTAINERS`` file:: git send-email --to maintainer@some.org --cc dev@dpdk.org 000*.patch +Script ``get-maintainer.sh`` can be used to select maintainers automatically:: + + git send-email --to-cmd ./devtools/get-maintainer.sh --cc dev@dpdk.org 000*.patch + New additions can be sent without a maintainer:: git send-email --to dev@dpdk.org 000*.patch @@ -359,9 +430,57 @@ The options ``--annotate`` and ``confirm = always`` are recommended for checking The Review Process ------------------ -The more work you put into the previous steps the easier it will be to get a patch accepted. +Patches are reviewed by the community, relying on the experience and +collaboration of the members to double-check each other's work. There are a +number of ways to indicate that you have checked a patch on the mailing list. -The general cycle for patch review and acceptance is: + +Tested, Acked and Reviewed by +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To indicate that you have interacted with a patch on the mailing list you +should respond to the patch in an email with one of the following tags: + + * Reviewed-by: + * Acked-by: + * Tested-by: + * Reported-by: + * Suggested-by: + +The tag should be on a separate line as follows:: + + tag-here: Name Surname <email@address.com> + +Each of these tags has a specific meaning. In general, the DPDK community +follows the kernel usage of the tags. A short summary of the meanings of each +tag is given here for reference: + +.. _statement: https://www.kernel.org/doc/html/latest/process/submitting-patches.html#reviewer-s-statement-of-oversight + +``Reviewed-by:`` is a strong statement_ that the patch is an appropriate state +for merging without any remaining serious technical issues. Reviews from +community members who are known to understand the subject area and to perform +thorough reviews will increase the likelihood of the patch getting merged. + +``Acked-by:`` is a record that the person named was not directly involved in +the preparation of the patch but wishes to signify and record their acceptance +and approval of it. + +``Tested-by:`` indicates that the patch has been successfully tested (in some +environment) by the person named. + +``Reported-by:`` is used to acknowledge person who found or reported the bug. + +``Suggested-by:`` indicates that the patch idea was suggested by the named +person. + + + +Steps to getting your patch merged +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The more work you put into the previous steps the easier it will be to get a +patch accepted. The general cycle for patch review and acceptance is: #. Submit the patch. @@ -392,4 +511,20 @@ The general cycle for patch review and acceptance is: #. In addition a patch will not be accepted if it doesn't address comments from a previous version with fixes or valid arguments. -#. Acked patches will be merged in the current or next merge window. +#. It is the responsibility of a maintainer to ensure that patches are reviewed and to provide an ``ack`` or + ``nack`` of those patches as appropriate. + +#. Once a patch has been acked by the relevant maintainer, reviewers may still comment on it for a further + two weeks. After that time, the patch should be merged into the relevant git tree for the next release. + Additional notes and restrictions: + + * Patches should be acked by a maintainer at least two days before the release merge + deadline, in order to make that release. + * For patches acked with less than two weeks to go to the merge deadline, all additional + comments should be made no later than two days before the merge deadline. + * After the appropriate time for additional feedback has passed, if the patch has not yet + been merged to the relevant tree by the committer, it should be treated as though it had, + in that any additional changes needed to it must be addressed by a follow-on patch, rather + than rework of the original. + * Trivial patches may be merged sooner than described above at the tree committer's + discretion. diff --git a/dpdk/doc/guides/contributing/stable.rst b/dpdk/doc/guides/contributing/stable.rst new file mode 100644 index 00000000..0f2f1f37 --- /dev/null +++ b/dpdk/doc/guides/contributing/stable.rst @@ -0,0 +1,99 @@ +.. stable_lts_releases: + +DPDK Stable Releases and Long Term Support +========================================== + +This section sets out the guidelines for the DPDK Stable Releases and the DPDK +Long Term Support releases (LTS). + + +Introduction +------------ + +The purpose of the DPDK Stable Releases is to maintain releases of DPDK with +backported fixes over an extended period of time. This provides downstream +consumers of DPDK with a stable target on which to base applications or +packages. + +The Long Term Support release (LTS) is a designation applied to a Stable +Release to indicate longer term support. + + +Stable Releases +--------------- + +Any major release of DPDK can be designated as a Stable Release if a +maintainer volunteers to maintain it. + +A Stable Release is used to backport fixes from an ``N`` release back to an +``N-1`` release, for example, from 16.11 to 16.07. + +The duration of a stable is one complete release cycle (3 months). It can be +longer, up to 1 year, if a maintainer continues to support the stable branch, +or if users supply backported fixes, however the explicit commitment should be +for one release cycle. + +The release cadence is determined by the maintainer based on the number of +bugfixes and the criticality of the bugs. Releases should be coordinated with +the validation engineers to ensure that a tagged release has been tested. + + +LTS Release +----------- + +A stable release can be designated as an LTS release based on community +agreement and a commitment from a maintainer. The current policy is that each +year's November release will be maintained as an LTS for 2 years. + +The current DPDK LTS releases are 16.11 and 17.11. + +It is anticipated that there will be at least 4 releases per year of the LTS +or approximately 1 every 3 months. However, the cadence can be shorter or +longer depending on the number and criticality of the backported +fixes. Releases should be coordinated with the validation engineers to ensure +that a tagged release has been tested. + + +What changes should be backported +--------------------------------- + +Backporting should be limited to bug fixes. + +Features should not be backported to stable releases. It may be acceptable, in +limited cases, to back port features for the LTS release where: + +* There is a justifiable use case (for example a new PMD). +* The change is non-invasive. +* The work of preparing the backport is done by the proposer. +* There is support within the community. + + +The Stable Mailing List +----------------------- + +The Stable and LTS release are coordinated on the stable@dpdk.org mailing +list. + +All fix patches to the master branch that are candidates for backporting +should also be CCed to the `stable@dpdk.org <http://dpdk.org/ml/listinfo/stable>`_ +mailing list. + + +Releasing +--------- + +A Stable Release will be released by: + +* Tagging the release with YY.MM.n (year, month, number). +* Uploading a tarball of the release to dpdk.org. +* Sending an announcement to the `announce@dpdk.org <http://dpdk.org/ml/listinfo/announce>`_ + list. + +Stable releases are available on the `dpdk.org download page <http://dpdk.org/download>`_. + + +ABI +--- + +The Stable Release should not be seen as a way of breaking or circumventing +the DPDK ABI policy. diff --git a/dpdk/doc/guides/contributing/versioning.rst b/dpdk/doc/guides/contributing/versioning.rst index 92b4d7ca..40009062 100644 --- a/dpdk/doc/guides/contributing/versioning.rst +++ b/dpdk/doc/guides/contributing/versioning.rst @@ -13,11 +13,15 @@ General Guidelines ------------------ #. Whenever possible, ABI should be preserved -#. The libraries marked in experimental state may change without constraint. +#. Libraries or APIs marked in ``experimental`` state may change without constraint. +#. New APIs will be marked as ``experimental`` for at least one release to allow + any issues found by users of the new API to be fixed quickly #. The addition of symbols is generally not problematic #. The modification of symbols can generally be managed with versioning #. The removal of symbols generally is an ABI break and requires bumping of the LIBABIVER macro +#. Updates to the minimum hardware requirements, which drop support for hardware which + was previously supported, should be treated as an ABI change. What is an ABI -------------- @@ -39,6 +43,13 @@ ABI versions are set at the time of major release labeling, and the ABI may change multiple times, without warning, between the last release label and the HEAD label of the git tree. +APIs marked as ``experimental`` are not considered part of the ABI and may +change without warning at any time. Since changes to APIs are most likely +immediately after their introduction, as users begin to take advantage of +those new APIs and start finding issues with them, new DPDK APIs will be +automatically marked as ``experimental`` to allow for a period of stabilization +before they become part of a tracked ABI. + ABI versions, once released, are available until such time as their deprecation has been noted in the Release Notes for at least one major release cycle. For example consider the case where the ABI for DPDK 2.0 has been @@ -56,6 +67,14 @@ being provided. The requirements for doing so are: #. At least 3 acknowledgments of the need to do so must be made on the dpdk.org mailing list. + - The acknowledgment of the maintainer of the component is mandatory, or if + no maintainer is available for the component, the tree/sub-tree maintainer + for that component must acknowledge the ABI change instead. + + - It is also recommended that acknowledgments from different "areas of + interest" be sought for each deprecation, for example: from NIC vendors, + CPU vendors, end-users, etc. + #. The changes (including an alternative map file) must be gated with the ``RTE_NEXT_ABI`` option, and provided with a deprecation notice at the same time. @@ -77,6 +96,13 @@ for significant reasons, such as performance enhancements. ABI breakage due to changes such as reorganizing public structure fields for aesthetic or readability purposes should be avoided. +.. note:: + + Updates to the minimum hardware requirements, which drop support for hardware + which was previously supported, should be treated as an ABI change, and + follow the relevant deprecation policy procedures as above: 3 acks and + announcement at least one release in advance. + Examples of Deprecation Notices ------------------------------- @@ -133,6 +159,31 @@ The macros exported are: fully qualified function ``p``, so that if a symbol becomes versioned, it can still be mapped back to the public symbol name. +Setting a Major ABI version +--------------------------- + +Downstreams might want to provide different DPDK releases at the same time to +support multiple consumers of DPDK linked against older and newer sonames. + +Also due to the interdependencies that DPDK libraries can have applications +might end up with an executable space in which multiple versions of a library +are mapped by ld.so. + +Think of LibA that got an ABI bump and LibB that did not get an ABI bump but is +depending on LibA. + +.. note:: + + Application + \-> LibA.old + \-> LibB.new -> LibA.new + +That is a conflict which can be avoided by setting ``CONFIG_RTE_MAJOR_ABI``. +If set, the value of ``CONFIG_RTE_MAJOR_ABI`` overwrites all - otherwise per +library - versions defined in the libraries ``LIBABIVER``. +An example might be ``CONFIG_RTE_MAJOR_ABI=16.11`` which will make all libraries +``librte<?>.so.16.11`` instead of ``librte<?>.so.<LIBABIVER>``. + Examples of ABI Macro use ------------------------- @@ -331,11 +382,12 @@ defined, we add this .. code-block:: c - struct rte_acl_create_v21(const struct rte_acl_param *param, int debug) + struct rte_acl_ctx * + rte_acl_create_v21(const struct rte_acl_param *param, int debug) { ... } - MAP_STATIC_SYMBOL(struct rte_acl_create(const struct rte_acl_param *param, int debug), rte_acl_create_v21); + MAP_STATIC_SYMBOL(struct rte_acl_ctx *rte_acl_create(const struct rte_acl_param *param, int debug), rte_acl_create_v21); That tells the compiler that, when building a static library, any calls to the symbol ``rte_acl_create`` should be linked to ``rte_acl_create_v21`` @@ -456,7 +508,7 @@ versions of the symbol. Running the ABI Validator ------------------------- -The ``scripts`` directory in the DPDK source tree contains a utility program, +The ``devtools`` directory in the DPDK source tree contains a utility program, ``validate-abi.sh``, for validating the DPDK ABI based on the Linux `ABI Compliance Checker <http://ispras.linuxbase.org/index.php/ABI_compliance_checker>`_. @@ -469,7 +521,7 @@ utilities which can be installed via a package manager. For example:: The syntax of the ``validate-abi.sh`` utility is:: - ./scripts/validate-abi.sh <REV1> <REV2> <TARGET> + ./devtools/validate-abi.sh <REV1> <REV2> <TARGET> Where ``REV1`` and ``REV2`` are valid gitrevisions(7) https://www.kernel.org/pub/software/scm/git/docs/gitrevisions.html @@ -478,13 +530,13 @@ on the local repo and target is the usual DPDK compilation target. For example:: # Check between the previous and latest commit: - ./scripts/validate-abi.sh HEAD~1 HEAD x86_64-native-linuxapp-gcc + ./devtools/validate-abi.sh HEAD~1 HEAD x86_64-native-linuxapp-gcc # Check between two tags: - ./scripts/validate-abi.sh v2.0.0 v2.1.0 x86_64-native-linuxapp-gcc + ./devtools/validate-abi.sh v2.0.0 v2.1.0 x86_64-native-linuxapp-gcc # Check between git master and local topic-branch "vhost-hacking": - ./scripts/validate-abi.sh master vhost-hacking x86_64-native-linuxapp-gcc + ./devtools/validate-abi.sh master vhost-hacking x86_64-native-linuxapp-gcc After the validation script completes (it can take a while since it need to compile both tags) it will create compatibility reports in the diff --git a/dpdk/doc/guides/cryptodevs/aesni_gcm.rst b/dpdk/doc/guides/cryptodevs/aesni_gcm.rst index 7ff1c6b3..a1f58483 100644 --- a/dpdk/doc/guides/cryptodevs/aesni_gcm.rst +++ b/dpdk/doc/guides/cryptodevs/aesni_gcm.rst @@ -1,5 +1,5 @@ .. BSD LICENSE - Copyright(c) 2016 Intel Corporation. All rights reserved. + Copyright(c) 2016-2017 Intel Corporation. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions @@ -35,20 +35,55 @@ The AES-NI GCM PMD (**librte_pmd_aesni_gcm**) provides poll mode crypto driver support for utilizing Intel multi buffer library (see AES-NI Multi-buffer PMD documentation to learn more about it, including installation). -The AES-NI GCM PMD has current only been tested on Fedora 21 64-bit with gcc. - Features -------- AESNI GCM PMD has support for: -Cipher algorithms: - -* RTE_CRYPTO_CIPHER_AES_GCM - Authentication algorithms: -* RTE_CRYPTO_AUTH_AES_GCM +* RTE_CRYPTO_AUTH_AES_GMAC + +AEAD algorithms: + +* RTE_CRYPTO_AEAD_AES_GCM + + +Limitations +----------- + +* Chained mbufs are supported but only out-of-place (destination mbuf must be contiguous). +* Cipher only is not supported. + + +Installation +------------ + +To build DPDK with the AESNI_GCM_PMD the user is required to download the multi-buffer +library from `here <https://github.com/01org/intel-ipsec-mb>`_ +and compile it on their user system before building DPDK. +The latest version of the library supported by this PMD is v0.47, which +can be downloaded in `<https://github.com/01org/intel-ipsec-mb/archive/v0.47.zip>`_. + +.. code-block:: console + + make + +As a reference, the following table shows a mapping between the past DPDK versions +and the external crypto libraries supported by them: + +.. _table_aesni_gcm_versions: + +.. table:: DPDK and external crypto library version compatibility + + ============= ================================ + DPDK version Crypto library version + ============= ================================ + 16.04 - 16.11 Multi-buffer library 0.43 - 0.44 + 17.02 - 17.05 ISA-L Crypto v2.18 + 17.08+ Multi-buffer library 0.46+ + ============= ================================ + Initialization -------------- @@ -58,15 +93,15 @@ In order to enable this virtual crypto PMD, user must: * Export the environmental variable AESNI_MULTI_BUFFER_LIB_PATH with the path where the library was extracted. -* Build the multi buffer library (go to Installation section in AES-NI MB PMD documentation). +* Build the multi buffer library (explained in Installation section). * Set CONFIG_RTE_LIBRTE_PMD_AESNI_GCM=y in config/common_base. To use the PMD in an application, user must: -* Call rte_eal_vdev_init("cryptodev_aesni_gcm_pmd") within the application. +* Call rte_vdev_init("crypto_aesni_gcm") within the application. -* Use --vdev="cryptodev_aesni_gcm_pmd" in the EAL options, which will call rte_eal_vdev_init() internally. +* Use --vdev="crypto_aesni_gcm" in the EAL options, which will call rte_vdev_init() internally. The following parameters (all optional) can be provided in the previous two calls: @@ -81,14 +116,5 @@ Example: .. code-block:: console - ./l2fwd-crypto -c 40 -n 4 --vdev="cryptodev_aesni_gcm_pmd,socket_id=1,max_nb_sessions=128" - -Limitations ------------ - -* Chained mbufs are not supported. -* Hash only is not supported. -* Cipher only is not supported. -* Only in-place is currently supported (destination address is the same as source address). -* Only supports session-oriented API implementation (session-less APIs are not supported). -* Not performance tuned. + ./l2fwd-crypto -l 1 -n 4 --vdev="crypto_aesni_gcm,socket_id=0,max_nb_sessions=128" \ + -- -p 1 --cdev SW --chain AEAD --aead_algo "aes-gcm" diff --git a/dpdk/doc/guides/cryptodevs/aesni_mb.rst b/dpdk/doc/guides/cryptodevs/aesni_mb.rst index 60a89142..79e722c6 100644 --- a/dpdk/doc/guides/cryptodevs/aesni_mb.rst +++ b/dpdk/doc/guides/cryptodevs/aesni_mb.rst @@ -1,5 +1,5 @@ .. BSD LICENSE - Copyright(c) 2015 Intel Corporation. All rights reserved. + Copyright(c) 2015-2017 Intel Corporation. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions @@ -27,7 +27,7 @@ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -AESN-NI Multi Buffer Crytpo Poll Mode Driver +AESN-NI Multi Buffer Crypto Poll Mode Driver ============================================ @@ -45,42 +45,62 @@ AESNI MB PMD has support for: Cipher algorithms: -* RTE_CRYPTO_SYM_CIPHER_AES128_CBC -* RTE_CRYPTO_SYM_CIPHER_AES192_CBC -* RTE_CRYPTO_SYM_CIPHER_AES256_CBC -* RTE_CRYPTO_SYM_CIPHER_AES128_CTR -* RTE_CRYPTO_SYM_CIPHER_AES192_CTR -* RTE_CRYPTO_SYM_CIPHER_AES256_CTR +* RTE_CRYPTO_CIPHER_AES128_CBC +* RTE_CRYPTO_CIPHER_AES192_CBC +* RTE_CRYPTO_CIPHER_AES256_CBC +* RTE_CRYPTO_CIPHER_AES128_CTR +* RTE_CRYPTO_CIPHER_AES192_CTR +* RTE_CRYPTO_CIPHER_AES256_CTR +* RTE_CRYPTO_CIPHER_AES_DOCSISBPI +* RTE_CRYPTO_CIPHER_DES_CBC +* RTE_CRYPTO_CIPHER_DES_DOCSISBPI Hash algorithms: -* RTE_CRYPTO_SYM_HASH_SHA1_HMAC -* RTE_CRYPTO_SYM_HASH_SHA256_HMAC -* RTE_CRYPTO_SYM_HASH_SHA512_HMAC +* RTE_CRYPTO_HASH_MD5_HMAC +* RTE_CRYPTO_HASH_SHA1_HMAC +* RTE_CRYPTO_HASH_SHA224_HMAC +* RTE_CRYPTO_HASH_SHA256_HMAC +* RTE_CRYPTO_HASH_SHA384_HMAC +* RTE_CRYPTO_HASH_SHA512_HMAC +* RTE_CRYPTO_HASH_AES_XCBC_HMAC Limitations ----------- * Chained mbufs are not supported. -* Hash only is not supported. -* Cipher only is not supported. * Only in-place is currently supported (destination address is the same as source address). -* Only supports session-oriented API implementation (session-less APIs are not supported). -* Not performance tuned. + Installation ------------ -To build DPDK with the AESNI_MB_PMD the user is required to download the mult- -buffer library from `here <https://downloadcenter.intel.com/download/22972>`_ -and compile it on their user system before building DPDK. When building the -multi-buffer library it is necessary to have YASM package installed and also -requires the overriding of YASM path when building, as a path is hard coded in -the Makefile of the release package. +To build DPDK with the AESNI_MB_PMD the user is required to download the multi-buffer +library from `here <https://github.com/01org/intel-ipsec-mb>`_ +and compile it on their user system before building DPDK. +The latest version of the library supported by this PMD is v0.47, which +can be downloaded from `<https://github.com/01org/intel-ipsec-mb/archive/v0.47.zip>`_. .. code-block:: console - make YASM=/usr/bin/yasm + make + +As a reference, the following table shows a mapping between the past DPDK versions +and the Multi-Buffer library version supported by them: + +.. _table_aesni_mb_versions: + +.. table:: DPDK and Multi-Buffer library version compatibility + + ============== ============================ + DPDK version Multi-buffer library version + ============== ============================ + 2.2 - 16.11 0.43 - 0.44 + 17.02 0.44 + 17.05 - 17.08 0.45 - 0.47 + 17.11+ 0.47 + ============== ============================ + Initialization -------------- @@ -96,9 +116,9 @@ In order to enable this virtual crypto PMD, user must: To use the PMD in an application, user must: -* Call rte_eal_vdev_init("cryptodev_aesni_mb_pmd") within the application. +* Call rte_vdev_init("crypto_aesni_mb") within the application. -* Use --vdev="cryptodev_aesni_mb_pmd" in the EAL options, which will call rte_eal_vdev_init() internally. +* Use --vdev="crypto_aesni_mb" in the EAL options, which will call rte_vdev_init() internally. The following parameters (all optional) can be provided in the previous two calls: @@ -113,4 +133,18 @@ Example: .. code-block:: console - ./l2fwd-crypto -c 40 -n 4 --vdev="cryptodev_aesni_mb_pmd,socket_id=1,max_nb_sessions=128" + ./l2fwd-crypto -l 1 -n 4 --vdev="crypto_aesni_mb,socket_id=0,max_nb_sessions=128" \ + -- -p 1 --cdev SW --chain CIPHER_HASH --cipher_algo "aes-cbc" --auth_algo "sha1-hmac" + +Extra notes +----------- + +For AES Counter mode (AES-CTR), the library supports two different sizes for Initialization +Vector (IV): + +* 12 bytes: used mainly for IPSec, as it requires 12 bytes from the user, which internally + are appended the counter block (4 bytes), which is set to 1 for the first block + (no padding required from the user) + +* 16 bytes: when passing 16 bytes, the library will take them and use the last 4 bytes + as the initial counter block for the first block. diff --git a/dpdk/doc/guides/cryptodevs/armv8.rst b/dpdk/doc/guides/cryptodevs/armv8.rst new file mode 100644 index 00000000..b1107114 --- /dev/null +++ b/dpdk/doc/guides/cryptodevs/armv8.rst @@ -0,0 +1,98 @@ +.. BSD LICENSE + Copyright (C) Cavium, Inc. 2017. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Cavium, Inc nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +ARMv8 Crypto Poll Mode Driver +============================= + +This code provides the initial implementation of the ARMv8 crypto PMD. +The driver uses ARMv8 cryptographic extensions to process chained crypto +operations in an optimized way. The core functionality is provided by +a low-level library, written in the assembly code. + +Features +-------- + +ARMv8 Crypto PMD has support for the following algorithm pairs: + +Supported cipher algorithms: + +* ``RTE_CRYPTO_CIPHER_AES_CBC`` + +Supported authentication algorithms: + +* ``RTE_CRYPTO_AUTH_SHA1_HMAC`` +* ``RTE_CRYPTO_AUTH_SHA256_HMAC`` + +Installation +------------ + +In order to enable this virtual crypto PMD, user must: + +* Download ARMv8 crypto library source code from + `here <https://github.com/caviumnetworks/armv8_crypto>`_ + +* Export the environmental variable ARMV8_CRYPTO_LIB_PATH with + the path where the ``armv8_crypto`` library was downloaded + or cloned. + +* Build the library by invoking: + +.. code-block:: console + + make -C $ARMV8_CRYPTO_LIB_PATH/ + +* Set CONFIG_RTE_LIBRTE_PMD_ARMV8_CRYPTO=y in + config/defconfig_arm64-armv8a-linuxapp-gcc + +The corresponding device can be created only if the following features +are supported by the CPU: + +* ``RTE_CPUFLAG_AES`` +* ``RTE_CPUFLAG_SHA1`` +* ``RTE_CPUFLAG_SHA2`` +* ``RTE_CPUFLAG_NEON`` + +Initialization +-------------- + +User can use app/test application to check how to use this PMD and to verify +crypto processing. + +Test name is cryptodev_sw_armv8_autotest. +For performance test cryptodev_sw_armv8_perftest can be used. + +Limitations +----------- + +* Maximum number of sessions is 2048. +* Only chained operations are supported. +* AES-128-CBC is the only supported cipher variant. +* Cipher input data has to be a multiple of 16 bytes. +* Digest input data has to be a multiple of 8 bytes. diff --git a/dpdk/doc/guides/cryptodevs/dpaa2_sec.rst b/dpdk/doc/guides/cryptodevs/dpaa2_sec.rst new file mode 100644 index 00000000..13474c18 --- /dev/null +++ b/dpdk/doc/guides/cryptodevs/dpaa2_sec.rst @@ -0,0 +1,239 @@ +.. BSD LICENSE + Copyright 2016 NXP. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of NXP nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +NXP DPAA2 CAAM (DPAA2_SEC) +========================== + +The DPAA2_SEC PMD provides poll mode crypto driver support for NXP DPAA2 CAAM +hardware accelerator. + +Architecture +------------ + +SEC is the SOC's security engine, which serves as NXP's latest cryptographic +acceleration and offloading hardware. It combines functions previously +implemented in separate modules to create a modular and scalable acceleration +and assurance engine. It also implements block encryption algorithms, stream +cipher algorithms, hashing algorithms, public key algorithms, run-time +integrity checking, and a hardware random number generator. SEC performs +higher-level cryptographic operations than previous NXP cryptographic +accelerators. This provides significant improvement to system level performance. + +DPAA2_SEC is one of the hardware resource in DPAA2 Architecture. More information +on DPAA2 Architecture is described in :ref:`dpaa2_overview`. + +DPAA2_SEC PMD is one of DPAA2 drivers which interacts with Management Complex (MC) +portal to access the hardware object - DPSECI. The MC provides access to create, +discover, connect, configure and destroy dpseci objects in DPAA2_SEC PMD. + +DPAA2_SEC PMD also uses some of the other hardware resources like buffer pools, +queues, queue portals to store and to enqueue/dequeue data to the hardware SEC. + +DPSECI objects are detected by PMD using a resource container called DPRC (like +in :ref:`dpaa2_overview`). + +For example: + +.. code-block:: console + + DPRC.1 (bus) + | + +--+--------+-------+-------+-------+---------+ + | | | | | | + DPMCP.1 DPIO.1 DPBP.1 DPNI.1 DPMAC.1 DPSECI.1 + DPMCP.2 DPIO.2 DPNI.2 DPMAC.2 DPSECI.2 + DPMCP.3 + +Implementation +-------------- + +SEC provides platform assurance by working with SecMon, which is a companion +logic block that tracks the security state of the SOC. SEC is programmed by +means of descriptors (not to be confused with frame descriptors (FDs)) that +indicate the operations to be performed and link to the message and +associated data. SEC incorporates two DMA engines to fetch the descriptors, +read the message data, and write the results of the operations. The DMA +engine provides a scatter/gather capability so that SEC can read and write +data scattered in memory. SEC may be configured by means of software for +dynamic changes in byte ordering. The default configuration for this version +of SEC is little-endian mode. + +A block diagram similar to dpaa2 NIC is shown below to show where DPAA2_SEC +fits in the DPAA2 Bus model + +.. code-block:: console + + + +----------------+ + | DPDK DPAA2_SEC | + | PMD | + +----------------+ +------------+ + | MC SEC object |.......| Mempool | + . . . . . . . . . | (DPSECI) | | (DPBP) | + . +---+---+--------+ +-----+------+ + . ^ | . + . | |<enqueue, . + . | | dequeue> . + . | | . + . +---+---V----+ . + . . . . . . . . . . .| DPIO driver| . + . . | (DPIO) | . + . . +-----+------+ . + . . | QBMAN | . + . . | Driver | . + +----+------+-------+ +-----+----- | . + | dpaa2 bus | | . + | VFIO fslmc-bus |....................|......................... + | | | + | /bus/fslmc | | + +-------------------+ | + | + ========================== HARDWARE =====|======================= + DPIO + | + DPSECI---DPBP + =========================================|======================== + + + +Features +-------- + +The DPAA2_SEC PMD has support for: + +Cipher algorithms: + +* ``RTE_CRYPTO_CIPHER_3DES_CBC`` +* ``RTE_CRYPTO_CIPHER_AES128_CBC`` +* ``RTE_CRYPTO_CIPHER_AES192_CBC`` +* ``RTE_CRYPTO_CIPHER_AES256_CBC`` +* ``RTE_CRYPTO_CIPHER_AES128_CTR`` +* ``RTE_CRYPTO_CIPHER_AES192_CTR`` +* ``RTE_CRYPTO_CIPHER_AES256_CTR`` + +Hash algorithms: + +* ``RTE_CRYPTO_AUTH_SHA1_HMAC`` +* ``RTE_CRYPTO_AUTH_SHA224_HMAC`` +* ``RTE_CRYPTO_AUTH_SHA256_HMAC`` +* ``RTE_CRYPTO_AUTH_SHA384_HMAC`` +* ``RTE_CRYPTO_AUTH_SHA512_HMAC`` +* ``RTE_CRYPTO_AUTH_MD5_HMAC`` + +AEAD algorithms: + +* ``RTE_CRYPTO_AEAD_AES_GCM`` + +Supported DPAA2 SoCs +-------------------- + +* LS2080A/LS2040A +* LS2084A/LS2044A +* LS2088A/LS2048A +* LS1088A/LS1048A + +Limitations +----------- + +* Chained mbufs are not supported. +* Hash followed by Cipher mode is not supported +* Only supports the session-oriented API implementation (session-less APIs are not supported). + +Prerequisites +------------- + +DPAA2_SEC driver has similar pre-requisites as described in :ref:`dpaa2_overview`. +The following dependencies are not part of DPDK and must be installed separately: + +* **NXP Linux SDK** + + NXP Linux software development kit (SDK) includes support for the family + of QorIQ® ARM-Architecture-based system on chip (SoC) processors + and corresponding boards. + + It includes the Linux board support packages (BSPs) for NXP SoCs, + a fully operational tool chain, kernel and board specific modules. + + SDK and related information can be obtained from: `NXP QorIQ SDK <http://www.nxp.com/products/software-and-tools/run-time-software/linux-sdk/linux-sdk-for-qoriq-processors:SDKLINUX>`_. + +* **DPDK Extra Scripts** + + DPAA2 based resources can be configured easily with the help of ready scripts + as provided in the DPDK helper repository. + + `DPDK Extra Scripts <https://github.com/qoriq-open-source/dpdk-extras>`_. + +Currently supported by DPDK: + +* NXP SDK **17.08+**. +* MC Firmware version **10.3.1** and higher. +* Supported architectures: **arm64 LE**. + +* Follow the DPDK :ref:`Getting Started Guide for Linux <linux_gsg>` to setup the basic DPDK environment. + +Pre-Installation Configuration +------------------------------ + +Config File Options +~~~~~~~~~~~~~~~~~~~ + +Basic DPAA2 config file options are described in :ref:`dpaa2_overview`. +In addition to those, the following options can be modified in the ``config`` file +to enable DPAA2_SEC PMD. + +Please note that enabling debugging options may affect system performance. + +* ``CONFIG_RTE_LIBRTE_PMD_DPAA2_SEC`` (default ``n``) + By default it is only enabled in defconfig_arm64-dpaa2-* config. + Toggle compilation of the ``librte_pmd_dpaa2_sec`` driver. + +* ``CONFIG_RTE_LIBRTE_DPAA2_SEC_DEBUG_INIT`` (default ``n``) + Toggle display of initialization related driver messages + +* ``CONFIG_RTE_LIBRTE_DPAA2_SEC_DEBUG_DRIVER`` (default ``n``) + Toggle display of driver runtime messages + +* ``CONFIG_RTE_LIBRTE_DPAA2_SEC_DEBUG_RX`` (default ``n``) + Toggle display of receive fast path run-time message + +* ``CONFIG_RTE_DPAA2_SEC_PMD_MAX_NB_SESSIONS`` + By default it is set as 2048 in defconfig_arm64-dpaa2-* config. + It indicates Number of sessions to create in the session memory pool + on a single DPAA2 SEC device. + +Installations +------------- +To compile the DPAA2_SEC PMD for Linux arm64 gcc target, run the +following ``make`` command: + +.. code-block:: console + + cd <DPDK-source-directory> + make config T=arm64-dpaa2-linuxapp-gcc install diff --git a/dpdk/doc/guides/cryptodevs/dpaa_sec.rst b/dpdk/doc/guides/cryptodevs/dpaa_sec.rst new file mode 100644 index 00000000..d3438cc3 --- /dev/null +++ b/dpdk/doc/guides/cryptodevs/dpaa_sec.rst @@ -0,0 +1,182 @@ +.. BSD LICENSE + Copyright 2017 NXP. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of NXP nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +NXP DPAA CAAM (DPAA_SEC) +======================== + +The DPAA_SEC PMD provides poll mode crypto driver support for NXP DPAA CAAM +hardware accelerator. + +Architecture +------------ + +SEC is the SOC's security engine, which serves as NXP's latest cryptographic +acceleration and offloading hardware. It combines functions previously +implemented in separate modules to create a modular and scalable acceleration +and assurance engine. It also implements block encryption algorithms, stream +cipher algorithms, hashing algorithms, public key algorithms, run-time +integrity checking, and a hardware random number generator. SEC performs +higher-level cryptographic operations than previous NXP cryptographic +accelerators. This provides significant improvement to system level performance. + +DPAA_SEC is one of the hardware resource in DPAA Architecture. More information +on DPAA Architecture is described in :ref:`dpaa_overview`. + +DPAA_SEC PMD is one of DPAA drivers which interacts with QBMAN to create, +configure and destroy the device instance using queue pair with CAAM portal. + +DPAA_SEC PMD also uses some of the other hardware resources like buffer pools, +queues, queue portals to store and to enqueue/dequeue data to the hardware SEC. + +Implementation +-------------- + +SEC provides platform assurance by working with SecMon, which is a companion +logic block that tracks the security state of the SOC. SEC is programmed by +means of descriptors (not to be confused with frame descriptors (FDs)) that +indicate the operations to be performed and link to the message and +associated data. SEC incorporates two DMA engines to fetch the descriptors, +read the message data, and write the results of the operations. The DMA +engine provides a scatter/gather capability so that SEC can read and write +data scattered in memory. SEC may be configured by means of software for +dynamic changes in byte ordering. The default configuration for this version +of SEC is little-endian mode. + +Features +-------- + +The DPAA PMD has support for: + +Cipher algorithms: + +* ``RTE_CRYPTO_CIPHER_3DES_CBC`` +* ``RTE_CRYPTO_CIPHER_AES128_CBC`` +* ``RTE_CRYPTO_CIPHER_AES192_CBC`` +* ``RTE_CRYPTO_CIPHER_AES256_CBC`` +* ``RTE_CRYPTO_CIPHER_AES128_CTR`` +* ``RTE_CRYPTO_CIPHER_AES192_CTR`` +* ``RTE_CRYPTO_CIPHER_AES256_CTR`` + +Hash algorithms: + +* ``RTE_CRYPTO_AUTH_SHA1_HMAC`` +* ``RTE_CRYPTO_AUTH_SHA224_HMAC`` +* ``RTE_CRYPTO_AUTH_SHA256_HMAC`` +* ``RTE_CRYPTO_AUTH_SHA384_HMAC`` +* ``RTE_CRYPTO_AUTH_SHA512_HMAC`` +* ``RTE_CRYPTO_AUTH_MD5_HMAC`` + +AEAD algorithms: + +* ``RTE_CRYPTO_AEAD_AES_GCM`` + +Supported DPAA SoCs +-------------------- + +* LS1046A/LS1026A +* LS1043A/LS1023A + +Limitations +----------- + +* Chained mbufs are not supported. +* Hash followed by Cipher mode is not supported +* Only supports the session-oriented API implementation (session-less APIs are not supported). + +Prerequisites +------------- + +DPAA_SEC driver has similar pre-requisites as described in :ref:`dpaa_overview`. +The following dependencies are not part of DPDK and must be installed separately: + +* **NXP Linux SDK** + + NXP Linux software development kit (SDK) includes support for the family + of QorIQ® ARM-Architecture-based system on chip (SoC) processors + and corresponding boards. + + It includes the Linux board support packages (BSPs) for NXP SoCs, + a fully operational tool chain, kernel and board specific modules. + + SDK and related information can be obtained from: `NXP QorIQ SDK <http://www.nxp.com/products/software-and-tools/run-time-software/linux-sdk/linux-sdk-for-qoriq-processors:SDKLINUX>`_. + +* **DPDK Extras Scripts** + + DPAA based resources can be configured easily with the help of ready scripts + as provided in the DPDK Extras repository. + + `DPDK Extras Scripts <https://github.com/qoriq-open-source/dpdk-extras>`_. + +Currently supported by DPDK: + +* NXP SDK **2.0+**. +* Supported architectures: **arm64 LE**. + +* Follow the DPDK :ref:`Getting Started Guide for Linux <linux_gsg>` to setup the basic DPDK environment. + +Pre-Installation Configuration +------------------------------ + +Config File Options +~~~~~~~~~~~~~~~~~~~ + +Basic DPAA config file options are described in :ref:`dpaa_overview`. +In addition to those, the following options can be modified in the ``config`` file +to enable DPAA_SEC PMD. + +Please note that enabling debugging options may affect system performance. + +* ``CONFIG_RTE_LIBRTE_PMD_DPAA_SEC`` (default ``n``) + By default it is only enabled in defconfig_arm64-dpaa-* config. + Toggle compilation of the ``librte_pmd_dpaa_sec`` driver. + +* ``CONFIG_RTE_LIBRTE_DPAA_SEC_DEBUG_INIT`` (default ``n``) + Toggle display of initialization related driver messages + +* ``CONFIG_RTE_LIBRTE_DPAA_SEC_DEBUG_DRIVER`` (default ``n``) + Toggle display of driver runtime messages + +* ``CONFIG_RTE_LIBRTE_DPAA_SEC_DEBUG_RX`` (default ``n``) + Toggle display of receive fast path run-time message + +* ``CONFIG_RTE_DPAA_SEC_PMD_MAX_NB_SESSIONS`` + By default it is set as 2048 in defconfig_arm64-dpaa-* config. + It indicates Number of sessions to create in the session memory pool + on a single DPAA SEC device. + +Installations +------------- +To compile the DPAA_SEC PMD for Linux arm64 gcc target, run the +following ``make`` command: + +.. code-block:: console + + cd <DPDK-source-directory> + make config T=arm64-dpaa-linuxapp-gcc install diff --git a/dpdk/doc/guides/cryptodevs/features/aesni_gcm.ini b/dpdk/doc/guides/cryptodevs/features/aesni_gcm.ini new file mode 100644 index 00000000..bacd94e3 --- /dev/null +++ b/dpdk/doc/guides/cryptodevs/features/aesni_gcm.ini @@ -0,0 +1,30 @@ +; +; Supported features of the 'aesni_gcm' crypto driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] +Symmetric crypto = Y +Sym operation chaining = Y +CPU AESNI = Y +CPU SSE = Y +CPU AVX = Y +CPU AVX2 = Y +; +; Supported crypto algorithms of the 'aesni_gcm' crypto driver. +; +[Cipher] + +; +; Supported authentication algorithms of the 'aesni_gcm' crypto driver. +; +[Auth] +AES GMAC = Y + +; +; Supported AEAD algorithms of the 'aesni_gcm' crypto driver. +; +[AEAD] +AES GCM (128) = Y +AES GCM (192) = Y +AES GCM (256) = Y diff --git a/dpdk/doc/guides/cryptodevs/features/aesni_mb.ini b/dpdk/doc/guides/cryptodevs/features/aesni_mb.ini new file mode 100644 index 00000000..fab07cb2 --- /dev/null +++ b/dpdk/doc/guides/cryptodevs/features/aesni_mb.ini @@ -0,0 +1,44 @@ +; +; Supported features of the 'aesni_mb' crypto driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] +Symmetric crypto = Y +Sym operation chaining = Y +CPU SSE = Y +CPU AVX = Y +CPU AVX2 = Y +CPU AVX512 = Y +CPU AESNI = Y + +; +; Supported crypto algorithms of the 'aesni_mb' crypto driver. +; +[Cipher] +AES CBC (128) = Y +AES CBC (192) = Y +AES CBC (256) = Y +AES CTR (128) = Y +AES CTR (192) = Y +AES CTR (256) = Y +AES DOCSIS BPI = Y +DES CBC = Y +DES DOCSIS BPI = Y + +; +; Supported authentication algorithms of the 'aesni_mb' crypto driver. +; +[Auth] +MD5 HMAC = Y +SHA1 HMAC = Y +SHA224 HMAC = Y +SHA256 HMAC = Y +SHA384 HMAC = Y +SHA512 HMAC = Y +AES XCBC MAC = Y + +; +; Supported AEAD algorithms of the 'aesni_mb' crypto driver. +; +[AEAD] diff --git a/dpdk/doc/guides/cryptodevs/features/armv8.ini b/dpdk/doc/guides/cryptodevs/features/armv8.ini new file mode 100644 index 00000000..1e104771 --- /dev/null +++ b/dpdk/doc/guides/cryptodevs/features/armv8.ini @@ -0,0 +1,28 @@ +; +; Supported features of the 'armv8' crypto driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] +Symmetric crypto = Y +Sym operation chaining = Y +CPU NEON = Y +CPU ARM CE = Y + +; +; Supported crypto algorithms of the 'armv8' crypto driver. +; +[Cipher] +AES CBC (128) = Y + +; +; Supported authentication algorithms of the 'armv8' crypto driver. +; +[Auth] +SHA1 HMAC = Y +SHA256 HMAC = Y + +; +; Supported AEAD algorithms of the 'armv8' crypto driver. +; +[AEAD] diff --git a/dpdk/doc/guides/cryptodevs/features/default.ini b/dpdk/doc/guides/cryptodevs/features/default.ini new file mode 100644 index 00000000..18d66cb9 --- /dev/null +++ b/dpdk/doc/guides/cryptodevs/features/default.ini @@ -0,0 +1,74 @@ +; +; Features of a default crypto driver. +; +; This file defines the features that are valid for inclusion in +; the other driver files and also the order that they appear in +; the features table in the documentation. +; +[Features] +Symmetric crypto = +Asymmetric crypto = +Sym operation chaining = +HW Accelerated = +Protocol offload = +CPU SSE = +CPU AVX = +CPU AVX2 = +CPU AVX512 = +CPU AESNI = +CPU NEON = +CPU ARM CE = + +; +; Supported crypto algorithms of a default crypto driver. +; +[Cipher] +NULL = +AES CBC (128) = +AES CBC (192) = +AES CBC (256) = +AES CTR (128) = +AES CTR (192) = +AES CTR (256) = +AES DOCSIS BPI = +3DES CBC = +3DES CTR = +DES CBC = +DES DOCSIS BPI = +SNOW3G UEA2 = +KASUMI F8 = +ZUC EEA3 = + +; +; Supported authentication algorithms of a default crypto driver. +; +[Auth] +NULL = +MD5 = +MD5 HMAC = +SHA1 = +SHA1 HMAC = +SHA224 = +SHA224 HMAC = +SHA256 = +SHA256 HMAC = +SHA384 = +SHA384 HMAC = +SHA512 = +SHA512 HMAC = +AES XCBC MAC = +AES GMAC = +SNOW3G UIA2 = +KASUMI F9 = +ZUC EIA3 = + +; +; Supported AEAD algorithms of a default crypto driver. +; +[AEAD] +AES GCM (128) = +AES GCM (192) = +AES GCM (256) = +AES CCM (128) = +AES CCM (192) = +AES CCM (256) = diff --git a/dpdk/doc/guides/cryptodevs/features/dpaa2_sec.ini b/dpdk/doc/guides/cryptodevs/features/dpaa2_sec.ini new file mode 100644 index 00000000..8fd07d6a --- /dev/null +++ b/dpdk/doc/guides/cryptodevs/features/dpaa2_sec.ini @@ -0,0 +1,41 @@ +; +; Supported features of the 'dpaa2_sec' crypto driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] +Symmetric crypto = Y +Sym operation chaining = Y +HW Accelerated = Y +Protocol offload = Y + +; +; Supported crypto algorithms of the 'dpaa2_sec' crypto driver. +; +[Cipher] +AES CBC (128) = Y +AES CBC (192) = Y +AES CBC (256) = Y +AES CTR (128) = Y +AES CTR (192) = Y +AES CTR (256) = Y +3DES CBC = Y + +; +; Supported authentication algorithms of the 'dpaa2_sec' crypto driver. +; +[Auth] +MD5 HMAC = Y +SHA1 HMAC = Y +SHA224 HMAC = Y +SHA256 HMAC = Y +SHA384 HMAC = Y +SHA512 HMAC = Y + +; +; Supported AEAD algorithms of the 'dpaa2_sec' crypto driver. +; +[AEAD] +AES GCM (128) = Y +AES GCM (192) = Y +AES GCM (256) = Y diff --git a/dpdk/doc/guides/cryptodevs/features/dpaa_sec.ini b/dpdk/doc/guides/cryptodevs/features/dpaa_sec.ini new file mode 100644 index 00000000..0e8f5b2a --- /dev/null +++ b/dpdk/doc/guides/cryptodevs/features/dpaa_sec.ini @@ -0,0 +1,40 @@ +; +; Supported features of the 'dpaa_sec' crypto driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] +Symmetric crypto = Y +Sym operation chaining = Y +HW Accelerated = Y + +; +; Supported crypto algorithms of the 'dpaa_sec' crypto driver. +; +[Cipher] +AES CBC (128) = Y +AES CBC (192) = Y +AES CBC (256) = Y +AES CTR (128) = Y +AES CTR (192) = Y +AES CTR (256) = Y +3DES CBC = Y + +; +; Supported authentication algorithms of the 'dpaa_sec' crypto driver. +; +[Auth] +MD5 HMAC = Y +SHA1 HMAC = Y +SHA224 HMAC = Y +SHA256 HMAC = Y +SHA384 HMAC = Y +SHA512 HMAC = Y + +; +; Supported AEAD algorithms of the 'dpaa_sec' crypto driver. +; +[AEAD] +AES GCM (128) = Y +AES GCM (192) = Y +AES GCM (256) = Y diff --git a/dpdk/doc/guides/cryptodevs/features/kasumi.ini b/dpdk/doc/guides/cryptodevs/features/kasumi.ini new file mode 100644 index 00000000..0e138f5a --- /dev/null +++ b/dpdk/doc/guides/cryptodevs/features/kasumi.ini @@ -0,0 +1,24 @@ +; +; Supported features of the 'kasumi' crypto driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] +Symmetric crypto = Y +Sym operation chaining = Y + +; +; Supported crypto algorithms of the 'kasumi' crypto driver. +; +[Cipher] +KASUMI F8 = Y +; +; Supported authentication algorithms of the 'kasumi' crypto driver. +; +[Auth] +KASUMI F9 = Y + +; +; Supported AEAD algorithms of the 'kasumi' crypto driver. +; +[AEAD] diff --git a/dpdk/doc/guides/cryptodevs/features/mrvl.ini b/dpdk/doc/guides/cryptodevs/features/mrvl.ini new file mode 100644 index 00000000..6d2fe6aa --- /dev/null +++ b/dpdk/doc/guides/cryptodevs/features/mrvl.ini @@ -0,0 +1,42 @@ +; Supported features of the 'mrvl' crypto driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] +Symmetric crypto = Y +Sym operation chaining = Y + +; +; Supported crypto algorithms of a default crypto driver. +; +[Cipher] +AES CBC (128) = Y +AES CBC (192) = Y +AES CBC (256) = Y +AES CTR (128) = Y +AES CTR (192) = Y +AES CTR (256) = Y +3DES CBC = Y +3DES CTR = Y + +; +; Supported authentication algorithms of a default crypto driver. +; +[Auth] +MD5 = Y +MD5 HMAC = Y +SHA1 = Y +SHA1 HMAC = Y +SHA256 = Y +SHA256 HMAC = Y +SHA384 = Y +SHA384 HMAC = Y +SHA512 = Y +SHA512 HMAC = Y +AES GMAC = Y + +; +; Supported AEAD algorithms of a default crypto driver. +; +[AEAD] +AES GCM (128) = Y diff --git a/dpdk/doc/guides/cryptodevs/features/null.ini b/dpdk/doc/guides/cryptodevs/features/null.ini new file mode 100644 index 00000000..523c453c --- /dev/null +++ b/dpdk/doc/guides/cryptodevs/features/null.ini @@ -0,0 +1,25 @@ +; +; Supported features of the 'null' crypto driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] +Symmetric crypto = Y +Sym operation chaining = Y + +; +; Supported crypto algorithms of the 'null' crypto driver. +; +[Cipher] +NULL = Y + +; +; Supported authentication algorithms of the 'null' crypto driver. +; +[Auth] +NULL = Y + +; +; Supported AEAD algorithms of the 'null' crypto driver. +; +[AEAD] diff --git a/dpdk/doc/guides/cryptodevs/features/openssl.ini b/dpdk/doc/guides/cryptodevs/features/openssl.ini new file mode 100644 index 00000000..385ec4e0 --- /dev/null +++ b/dpdk/doc/guides/cryptodevs/features/openssl.ini @@ -0,0 +1,50 @@ +; +; Supported features of the 'openssl' crypto driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] +Symmetric crypto = Y +Sym operation chaining = Y + +; +; Supported crypto algorithms of the 'openssl' crypto driver. +; +[Cipher] +AES CBC (128) = Y +AES CBC (192) = Y +AES CBC (256) = Y +AES CTR (128) = Y +AES CTR (192) = Y +AES CTR (256) = Y +3DES CBC = Y +3DES CTR = Y +DES DOCSIS BPI = Y +; +; Supported authentication algorithms of the 'openssl' crypto driver. +; +[Auth] +MD5 = Y +MD5 HMAC = Y +SHA1 = Y +SHA1 HMAC = Y +SHA224 = Y +SHA224 HMAC = Y +SHA256 = Y +SHA256 HMAC = Y +SHA384 = Y +SHA384 HMAC = Y +SHA512 = Y +SHA512 HMAC = Y +AES GMAC = Y + +; +; Supported AEAD algorithms of the 'openssl' crypto driver. +; +[AEAD] +AES GCM (128) = Y +AES GCM (192) = Y +AES GCM (256) = Y +AES CCM (128) = Y +AES CCM (192) = Y +AES CCM (256) = Y diff --git a/dpdk/doc/guides/cryptodevs/features/qat.ini b/dpdk/doc/guides/cryptodevs/features/qat.ini new file mode 100644 index 00000000..40da8985 --- /dev/null +++ b/dpdk/doc/guides/cryptodevs/features/qat.ini @@ -0,0 +1,53 @@ +; +; Supported features of the 'qat' crypto driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] +Symmetric crypto = Y +Sym operation chaining = Y +HW Accelerated = Y + +; +; Supported crypto algorithms of the 'qat' crypto driver. +; +[Cipher] +NULL = Y +AES CBC (128) = Y +AES CBC (192) = Y +AES CBC (256) = Y +AES CTR (128) = Y +AES CTR (192) = Y +AES CTR (256) = Y +3DES CBC = Y +3DES CTR = Y +DES CBC = Y +SNOW3G UEA2 = Y +KASUMI F8 = Y +AES DOCSIS BPI = Y +DES DOCSIS BPI = Y +ZUC EEA3 = Y +; +; Supported authentication algorithms of the 'qat' crypto driver. +; +[Auth] +NULL = Y +MD5 HMAC = Y +SHA1 HMAC = Y +SHA224 HMAC = Y +SHA256 HMAC = Y +SHA384 HMAC = Y +SHA512 HMAC = Y +AES GMAC = Y +SNOW3G UIA2 = Y +KASUMI F9 = Y +AES XCBC MAC = Y +ZUC EIA3 = Y + +; +; Supported AEAD algorithms of the 'qat' crypto driver. +; +[AEAD] +AES GCM (128) = Y +AES GCM (192) = Y +AES GCM (256) = Y diff --git a/dpdk/doc/guides/cryptodevs/features/snow3g.ini b/dpdk/doc/guides/cryptodevs/features/snow3g.ini new file mode 100644 index 00000000..27713617 --- /dev/null +++ b/dpdk/doc/guides/cryptodevs/features/snow3g.ini @@ -0,0 +1,24 @@ +; +; Supported features of the 'snow3g' crypto driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] +Symmetric crypto = Y +Sym operation chaining = Y + +; +; Supported crypto algorithms of the 'snow3g' crypto driver. +; +[Cipher] +SNOW3G UEA2 = Y +; +; Supported authentication algorithms of the 'snow3g' crypto driver. +; +[Auth] +SNOW3G UIA2 = Y + +; +; Supported AEAD algorithms of the 'snow3g' crypto driver. +; +[AEAD] diff --git a/dpdk/doc/guides/cryptodevs/features/zuc.ini b/dpdk/doc/guides/cryptodevs/features/zuc.ini new file mode 100644 index 00000000..5bb02afd --- /dev/null +++ b/dpdk/doc/guides/cryptodevs/features/zuc.ini @@ -0,0 +1,24 @@ +; +; Supported features of the 'zuc' crypto driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] +Symmetric crypto = Y +Sym operation chaining = Y + +; +; Supported crypto algorithms of the 'zuc' crypto driver. +; +[Cipher] +ZUC EEA3 = Y +; +; Supported authentication algorithms of the 'zuc' crypto driver. +; +[Auth] +ZUC EIA3 = Y + +; +; Supported AEAD algorithms of the 'zuc' crypto driver. +; +[AEAD] diff --git a/dpdk/doc/guides/cryptodevs/img/scheduler-overview.svg b/dpdk/doc/guides/cryptodevs/img/scheduler-overview.svg new file mode 100644 index 00000000..82bb775b --- /dev/null +++ b/dpdk/doc/guides/cryptodevs/img/scheduler-overview.svg @@ -0,0 +1,277 @@ +<?xml version="1.0" encoding="UTF-8" standalone="no"?> +<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"> +<!-- Generated by Microsoft Visio, SVG Export scheduler-fan.svg Page-1 --> +<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:ev="http://www.w3.org/2001/xml-events" + xmlns:v="http://schemas.microsoft.com/visio/2003/SVGExtensions/" width="6.81229in" height="3.40992in" + viewBox="0 0 490.485 245.514" xml:space="preserve" color-interpolation-filters="sRGB" class="st10"> + <v:documentProperties v:langID="1033" v:metric="true" v:viewMarkup="false"/> + + <style type="text/css"> + <![CDATA[ + .st1 {visibility:visible} + .st2 {fill:#fec000;fill-opacity:0.25;filter:url(#filter_2);stroke:#fec000;stroke-opacity:0.25} + .st3 {fill:#cc3399;stroke:#ff8c00;stroke-width:3} + .st4 {fill:#ffffff;font-family:Calibri;font-size:1.33333em} + .st5 {fill:#ff9900;stroke:#ff8c00;stroke-width:3} + .st6 {fill:#ffffff;font-family:Calibri;font-size:1.33333em;font-weight:bold} + .st7 {fill:#ffc000;stroke:#ffffff;stroke-linecap:round;stroke-linejoin:round;stroke-width:0.5} + .st8 {marker-end:url(#mrkr4-40);stroke:#ff0000;stroke-linecap:round;stroke-linejoin:round;stroke-width:1.5} + .st9 {fill:#ff0000;fill-opacity:1;stroke:#ff0000;stroke-opacity:1;stroke-width:0.37313432835821} + .st10 {fill:none;fill-rule:evenodd;font-size:12px;overflow:visible;stroke-linecap:square;stroke-miterlimit:3} + ]]> + </style> + + <defs id="Markers"> + <g id="lend4"> + <path d="M 2 1 L 0 0 L 2 -1 L 2 1 " style="stroke:none"/> + </g> + <marker id="mrkr4-40" class="st9" v:arrowType="4" v:arrowSize="2" v:setback="5.36" refX="-5.36" orient="auto" + markerUnits="strokeWidth" overflow="visible"> + <use xlink:href="#lend4" transform="scale(-2.68,-2.68) "/> + </marker> + </defs> + <defs id="Filters"> + <filter id="filter_2"> + <feGaussianBlur stdDeviation="2"/> + </filter> + </defs> + <g v:mID="0" v:index="1" v:groupContext="foregroundPage"> + <title>Page-1 + + + + Rounded Rectangle.55 + User Application + + + + + + + + + + + + + + + + + + + + + + + + + + User Application + + Rounded Rectangle.135 + Cryptodev + + + + + + + + + + + + + + + + + + + + + + + + + + Cryptodev + + Rounded Rectangle.136 + Cryptodev + + + + + + + + + + + + + + + + + + + + + + + + + + Cryptodev + + Rounded Rectangle.137 + Cryptodev + + + + + + + + + + + + + + + + + + + + + + + + + + Cryptodev + + Sheet.139 + + Rounded Rectangle.40 + Cryptodev Scheduler + + + + + + + + + + + + + + + + + + + + + + + + + + Cryptodev Scheduler + + Rounded Rectangle.138 + Crypto Op Distribution Mechanism + + + + + + + + + + + + + + + + + + + + + + + Crypto Op Distribution Mechanism + + + Dynamic connector.229 + + + + Dynamic connector.141 + + + + Dynamic connector.142 + + + + Dynamic connector.143 + + + + Dynamic connector.144 + + + + Dynamic connector.145 + + + + Dynamic connector.146 + + + + Dynamic connector.147 + + + + diff --git a/dpdk/doc/guides/cryptodevs/index.rst b/dpdk/doc/guides/cryptodevs/index.rst index 9616de1e..6d4e15b9 100644 --- a/dpdk/doc/guides/cryptodevs/index.rst +++ b/dpdk/doc/guides/cryptodevs/index.rst @@ -1,5 +1,5 @@ .. BSD LICENSE - Copyright(c) 2015 - 2016 Intel Corporation. All rights reserved. + Copyright(c) 2015 - 2017 Intel Corporation. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions @@ -38,7 +38,14 @@ Crypto Device Drivers overview aesni_mb aesni_gcm + armv8 + dpaa2_sec + dpaa_sec kasumi + openssl + mrvl null + scheduler snow3g qat + zuc diff --git a/dpdk/doc/guides/cryptodevs/kasumi.rst b/dpdk/doc/guides/cryptodevs/kasumi.rst index 7346b217..573312b4 100644 --- a/dpdk/doc/guides/cryptodevs/kasumi.rst +++ b/dpdk/doc/guides/cryptodevs/kasumi.rst @@ -41,17 +41,17 @@ KASUMI PMD has support for: Cipher algorithm: -* RTE_CRYPTO_SYM_CIPHER_KASUMI_F8 +* RTE_CRYPTO_CIPHER_KASUMI_F8 Authentication algorithm: -* RTE_CRYPTO_SYM_AUTH_KASUMI_F9 +* RTE_CRYPTO_AUTH_KASUMI_F9 Limitations ----------- * Chained mbufs are not supported. -* KASUMI(F9) supported only if hash offset field is byte-aligned. +* KASUMI(F9) supported only if hash offset and length field is byte-aligned. * In-place bit-level operations for KASUMI(F8) are not supported (if length and/or offset of data to be ciphered is not byte-aligned). @@ -68,7 +68,25 @@ and click on "Kasumi Bit Stream crypto library" link, to download the library. After downloading the library, the user needs to unpack and compile it on their system before building DPDK:: - make kasumi + make + +**Note**: When encrypting with KASUMI F8, by default the library +encrypts full blocks of 8 bytes, regardless the number of bytes to +be encrypted provided (which leads to a possible buffer overflow). +To avoid this situation, it is necessary not to pass +3GPP_SAFE_BUFFERS as a compilation flag. +Also, this is required when using chained operations +(cipher-then-auth/auth-then-cipher). +For this, in the Makefile of the library, make sure that this flag +is commented out:: + + #EXTRA_CFLAGS += -D_3GPP_SAFE_BUFFERS + +**Note**: To build the PMD as a shared library, the libsso_kasumi +library must be built as follows:: + + make KASUMI_CFLAGS=-DKASUMI_C + Initialization -------------- @@ -84,9 +102,9 @@ In order to enable this virtual crypto PMD, user must: To use the PMD in an application, user must: -* Call rte_eal_vdev_init("cryptodev_kasumi_pmd") within the application. +* Call rte_vdev_init("crypto_kasumi") within the application. -* Use --vdev="cryptodev_kasumi_pmd" in the EAL options, which will call rte_eal_vdev_init() internally. +* Use --vdev="crypto_kasumi" in the EAL options, which will call rte_vdev_init() internally. The following parameters (all optional) can be provided in the previous two calls: @@ -101,4 +119,22 @@ Example: .. code-block:: console - ./l2fwd-crypto -c 40 -n 4 --vdev="cryptodev_kasumi_pmd,socket_id=1,max_nb_sessions=128" + ./l2fwd-crypto -l 1 -n 4 --vdev="crypto_kasumi,socket_id=0,max_nb_sessions=128" \ + -- -p 1 --cdev SW --chain CIPHER_ONLY --cipher_algo "kasumi-f8" + +Extra notes on KASUMI F9 +------------------------ + +When using KASUMI F9 authentication algorithm, the input buffer must be +constructed according to the 3GPP KASUMI specifications (section 4.4, page 13): +``_. +Input buffer has to have COUNT (4 bytes), FRESH (4 bytes), MESSAGE and DIRECTION (1 bit) +concatenated. After the DIRECTION bit, a single '1' bit is appended, followed by +between 0 and 7 '0' bits, so that the total length of the buffer is multiple of 8 bits. +Note that the actual message can be any length, specified in bits. + +Once this buffer is passed this way, when creating the crypto operation, +length of data to authenticate (op.sym.auth.data.length) must be the length +of all the items described above, including the padding at the end. +Also, offset of data to authenticate (op.sym.auth.data.offset) +must be such that points at the start of the COUNT bytes. diff --git a/dpdk/doc/guides/cryptodevs/mrvl.rst b/dpdk/doc/guides/cryptodevs/mrvl.rst new file mode 100644 index 00000000..4e992fbb --- /dev/null +++ b/dpdk/doc/guides/cryptodevs/mrvl.rst @@ -0,0 +1,206 @@ +.. BSD LICENSE + Copyright(c) 2017 Marvell International Ltd. + Copyright(c) 2017 Semihalf. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +MRVL Crypto Poll Mode Driver +============================ + +The MRVL CRYPTO PMD (**librte_crypto_mrvl_pmd**) provides poll mode crypto driver +support by utilizing MUSDK library, which provides cryptographic operations +acceleration by using Security Acceleration Engine (EIP197) directly from +user-space with minimum overhead and high performance. + +Features +-------- + +MRVL CRYPTO PMD has support for: + +* Symmetric crypto +* Sym operation chaining +* AES CBC (128) +* AES CBC (192) +* AES CBC (256) +* AES CTR (128) +* AES CTR (192) +* AES CTR (256) +* 3DES CBC +* 3DES CTR +* MD5 +* MD5 HMAC +* SHA1 +* SHA1 HMAC +* SHA256 +* SHA256 HMAC +* SHA384 +* SHA384 HMAC +* SHA512 +* SHA512 HMAC +* AES GCM (128) + +Limitations +----------- + +* Hardware only supports scenarios where ICV (digest buffer) is placed just + after the authenticated data. Other placement will result in error. + +* Before running crypto test suite it is advised to increase limit of + opened files: + + .. code-block:: console + + ulimit -n 20000 + +Installation +------------ + +MRVL CRYPTO PMD driver compilation is disabled by default due to external dependencies. +Currently there are two driver specific compilation options in +``config/common_base`` available: + +- ``CONFIG_RTE_LIBRTE_MRVL_CRYPTO`` (default ``n``) + + Toggle compilation of the librte_pmd_mrvl driver. + +- ``CONFIG_RTE_LIBRTE_MRVL_CRYPTO_DEBUG`` (default ``n``) + + Toggle display of debugging messages. + +During compilation external MUSDK library, which provides direct access +to Marvell's EIP197 cryptographic engine, is necessary. Library sources are +available `here `__. + +Alternatively, prebuilt library can be downloaded from +`Marvell Extranet `_. Once approval has been +granted, library can be found by typing ``musdk`` in the search box. + +For MUSDK library build instructions please refer to ``doc/musdk_get_started.txt`` +in library sources directory. + +MUSDK requires out of tree kernel modules to work. Kernel tree needed to build +them is available +`here `__. + +Initialization +-------------- + +After successfully building MRVL CRYPTO PMD, the following modules need to be +loaded: + +.. code-block:: console + + insmod musdk_uio.ko + insmod mvpp2x_sysfs.ko + insmod mv_pp_uio.ko + insmod mv_sam_uio.ko + insmod crypto_safexcel.ko + +- `musdk_uio.ko`, `mv_pp2_uio.ko` and `mv_sam_uio.ko` are distributed together with MUSDK library. +- `crypto_safexcel.ko` is an in-kernel module. +- `mvpp2x_sysfs.ko` can be build from sources available `here `__. + +The following parameters (all optional) are exported by the driver: + +* max_nb_queue_pairs: maximum number of queue pairs in the device (8 by default). +* max_nb_sessions: maximum number of sessions that can be created (2048 by default). +* socket_id: socket on which to allocate the device resources on. + +l2fwd-crypto example application can be used to verify MRVL CRYPTO PMD +operation: + +.. code-block:: console + + ./l2fwd-crypto --vdev=net_mrvl,iface=eth0 --vdev=crypto_mrvl -- \ + --cipher_op ENCRYPT --cipher_algo aes-cbc \ + --cipher_key 00:01:02:03:04:05:06:07:08:09:0a:0b:0c:0d:0e:0f \ + --auth_op GENERATE --auth_algo sha1-hmac \ + --auth_key 10:11:12:13:14:15:16:17:18:19:1a:1b:1c:1d:1e:1f + +Example output: + +.. code-block:: console + + [...] + AAD: at [0x7f253ceb80], len= + P ID 0 configuration ---- + Port mode : KR + MAC status : disabled + Link status : link up + Port speed : 10G + Port duplex : full + Port: Egress enable tx_port_num=16 qmap=0x1 + PORT: Port0 - link + P ID 0 configuration ---- + Port mode : KR + MAC status : disabled + Link status : link down + Port speed : 10G + Port duplex : full + Port: Egress enable tx_port_num=16 qmap=0x1 + Port 0, MAC address: 00:50:43:02:21:20 + + + Checking link statusdone + Port 0 Link Up - speed 0 Mbps - full-duplex + Lcore 0: RX port 0 + Allocated session pool on socket 0 + eip197: 0:0 registers: paddr: 0xf2880000, vaddr: 0x0x7f56a80000 + DMA buffer (131136 bytes) for CDR #0 allocated: paddr = 0xb0585e00, vaddr = 0x7f09384e00 + DMA buffer (131136 bytes) for RDR #0 allocated: paddr = 0xb05a5f00, vaddr = 0x7f093a4f00 + DMA buffers allocated for 2049 operations. Tokens - 256 bytes + Lcore 0: cryptodev 0 + L2FWD: lcore 1 has nothing to do + L2FWD: lcore 2 has nothing to do + L2FWD: lcore 3 has nothing to do + L2FWD: entering main loop on lcore 0 + L2FWD: -- lcoreid=0 portid=0 + L2FWD: -- lcoreid=0 cryptoid=0 + Options:- + nportmask: ffffffff + ports per lcore: 1 + refresh period : 10000 + single lcore mode: disabled + stats_printing: enabled + sessionless crypto: disabled + + Crypto chain: Input --> Encrypt --> Auth generate --> Output + + ---- Cipher information --- + Algorithm: aes-cbc + Cipher key: at [0x7f56db4e80], len=16 + 00000000: 00 01 02 03 04 05 06 07 08 09 0A 0B 0C 0D 0E 0F | ................ + IV: at [0x7f56db4b80], len=16 + 00000000: 20 F0 63 0E 45 EB 2D 84 72 D4 13 6E 36 B5 AF FE | .c.E.-.r..n6... + + ---- Authentication information --- + Algorithm: sha1-hmac + Auth key: at [0x7f56db4d80], len=16 + 00000000: 10 11 12 13 14 15 16 17 18 19 1A 1B 1C 1D 1E 1F | ................ + IV: at [0x7f56db4a80], len=0 + AAD: at [0x7f253ceb80], len= diff --git a/dpdk/doc/guides/cryptodevs/null.rst b/dpdk/doc/guides/cryptodevs/null.rst index b68d4cd2..03a3ddc0 100644 --- a/dpdk/doc/guides/cryptodevs/null.rst +++ b/dpdk/doc/guides/cryptodevs/null.rst @@ -76,9 +76,9 @@ Initialization To use the PMD in an application, user must: -* Call rte_eal_vdev_init("cryptodev_null_pmd") within the application. +* Call rte_vdev_init("crypto_null") within the application. -* Use --vdev="cryptodev_null_pmd" in the EAL options, which will call rte_eal_vdev_init() internally. +* Use --vdev="crypto_null" in the EAL options, which will call rte_vdev_init() internally. The following parameters (all optional) can be provided in the previous two calls: @@ -93,4 +93,5 @@ Example: .. code-block:: console - ./l2fwd-crypto -c 40 -n 4 --vdev="cryptodev_null_pmd,socket_id=1,max_nb_sessions=128" + ./l2fwd-crypto -l 1 -n 4 --vdev="crypto_null,socket_id=0,max_nb_sessions=128" \ + -- -p 1 --cdev SW --chain CIPHER_ONLY --cipher_algo "null" diff --git a/dpdk/doc/guides/cryptodevs/openssl.rst b/dpdk/doc/guides/cryptodevs/openssl.rst new file mode 100644 index 00000000..61a4da35 --- /dev/null +++ b/dpdk/doc/guides/cryptodevs/openssl.rst @@ -0,0 +1,131 @@ +.. BSD LICENSE + Copyright(c) 2016 Intel Corporation. All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +OpenSSL Crypto Poll Mode Driver +=============================== + +This code provides the initial implementation of the openssl poll mode +driver. All cryptography operations are using Openssl library crypto API. +Each algorithm uses EVP interface from openssl API - which is recommended +by Openssl maintainers. + +For more details about openssl library please visit openssl webpage: +https://www.openssl.org/ + +Features +-------- + +OpenSSL PMD has support for: + +Supported cipher algorithms: + +* ``RTE_CRYPTO_CIPHER_3DES_CBC`` +* ``RTE_CRYPTO_CIPHER_AES_CBC`` +* ``RTE_CRYPTO_CIPHER_AES_CTR`` +* ``RTE_CRYPTO_CIPHER_3DES_CTR`` +* ``RTE_CRYPTO_CIPHER_DES_DOCSISBPI`` + +Supported authentication algorithms: + +* ``RTE_CRYPTO_AUTH_AES_GMAC`` +* ``RTE_CRYPTO_AUTH_MD5`` +* ``RTE_CRYPTO_AUTH_SHA1`` +* ``RTE_CRYPTO_AUTH_SHA224`` +* ``RTE_CRYPTO_AUTH_SHA256`` +* ``RTE_CRYPTO_AUTH_SHA384`` +* ``RTE_CRYPTO_AUTH_SHA512`` +* ``RTE_CRYPTO_AUTH_MD5_HMAC`` +* ``RTE_CRYPTO_AUTH_SHA1_HMAC`` +* ``RTE_CRYPTO_AUTH_SHA224_HMAC`` +* ``RTE_CRYPTO_AUTH_SHA256_HMAC`` +* ``RTE_CRYPTO_AUTH_SHA384_HMAC`` +* ``RTE_CRYPTO_AUTH_SHA512_HMAC`` + +Supported AEAD algorithms: + +* ``RTE_CRYPTO_AEAD_AES_GCM`` +* ``RTE_CRYPTO_AEAD_AES_CCM`` + + +Installation +------------ + +To compile openssl PMD, it has to be enabled in the config/common_base file +and appropriate openssl packages have to be installed in the build environment. + +The newest openssl library version is supported: + +* 1.0.2h-fips 3 May 2016. + +Older versions that were also verified: + +* 1.0.1f 6 Jan 2014 +* 1.0.1 14 Mar 2012 + +For Ubuntu 14.04 LTS these packages have to be installed in the build system: + +.. code-block:: console + + sudo apt-get install openssl + sudo apt-get install libc6-dev-i386 # for i686-native-linuxapp-gcc target + +This code was also verified on Fedora 24. +This code has NOT been verified on FreeBSD yet. + +Initialization +-------------- + +User can use app/test application to check how to use this pmd and to verify +crypto processing. + +Test name is cryptodev_openssl_autotest. +For performance test cryptodev_openssl_perftest can be used. + +To verify real traffic l2fwd-crypto example can be used with this command: + +.. code-block:: console + + sudo ./build/l2fwd-crypto -l 0-1 -n 4 --vdev "crypto_openssl" + --vdev "crypto_openssl"-- -p 0x3 --chain CIPHER_HASH + --cipher_op ENCRYPT --cipher_algo AES_CBC + --cipher_key 00:01:02:03:04:05:06:07:08:09:0a:0b:0c:0d:0e:0f + --iv 00:01:02:03:04:05:06:07:08:09:0a:0b:0c:0d:0e:ff + --auth_op GENERATE --auth_algo SHA1_HMAC + --auth_key 11:11:11:11:11:11:11:11:11:11:11:11:11:11:11:11:11:11:11:11:11:11 + :11:11:11:11:11:11:11:11:11:11:11:11:11:11:11:11:11:11:11:11:11:11:11:11:11 + :11:11:11:11:11:11:11:11:11:11:11:11:11:11:11:11:11 + +Limitations +----------- + +* Maximum number of sessions is 2048. +* Chained mbufs are supported only for source mbuf (destination must be + contiguous). +* Hash only is not supported for GCM and GMAC. +* Cipher only is not supported for GCM and GMAC. diff --git a/dpdk/doc/guides/cryptodevs/overview.rst b/dpdk/doc/guides/cryptodevs/overview.rst index d612f71b..6764d0d9 100644 --- a/dpdk/doc/guides/cryptodevs/overview.rst +++ b/dpdk/doc/guides/cryptodevs/overview.rst @@ -1,5 +1,5 @@ .. BSD LICENSE - Copyright(c) 2016 Intel Corporation. All rights reserved. + Copyright(c) 2016-2017 Intel Corporation. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions @@ -11,7 +11,7 @@ notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - * Neither the name of Intel Corporation nor the names of its + * Neither the name of 6WIND S.A. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -28,68 +28,32 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. Crypto Device Supported Functionality Matrices ----------------------------------------------- +============================================== Supported Feature Flags +----------------------- -.. csv-table:: - :header: "Feature Flags", "qat", "null", "aesni_mb", "aesni_gcm", "snow3g", "kasumi" - :stub-columns: 1 +.. _table_crypto_pmd_features: - "RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO",x,x,x,x,x,x - "RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO",,,,,, - "RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING",x,x,x,x,x,x - "RTE_CRYPTODEV_FF_CPU_SSE",,,x,x,x,x - "RTE_CRYPTODEV_FF_CPU_AVX",,,x,x,x,x - "RTE_CRYPTODEV_FF_CPU_AVX2",,,x,x,, - "RTE_CRYPTODEV_FF_CPU_AESNI",,,x,x,, - "RTE_CRYPTODEV_FF_HW_ACCELERATED",x,,,,, +.. include:: overview_feature_table.txt Supported Cipher Algorithms +--------------------------- -.. csv-table:: - :header: "Cipher Algorithms", "qat", "null", "aesni_mb", "aesni_gcm", "snow3g", "kasumi" - :stub-columns: 1 +.. _table_crypto_pmd_cipher_algos: - "NULL",,x,,,, - "AES_CBC_128",x,,x,,, - "AES_CBC_192",x,,x,,, - "AES_CBC_256",x,,x,,, - "AES_CTR_128",x,,x,,, - "AES_CTR_192",x,,x,,, - "AES_CTR_256",x,,x,,, - "SNOW3G_UEA2",x,,,,x, - "KASUMI_F8",,,,,,x +.. include:: overview_cipher_table.txt Supported Authentication Algorithms +----------------------------------- -.. csv-table:: - :header: "Cipher Algorithms", "qat", "null", "aesni_mb", "aesni_gcm", "snow3g", "kasumi" - :stub-columns: 1 +.. _table_crypto_pmd_auth_algos: - "NONE",,x,,,, - "MD5",,,,,, - "MD5_HMAC",,,x,,, - "SHA1",,,,,, - "SHA1_HMAC",x,,x,,, - "SHA224",,,,,, - "SHA224_HMAC",,,x,,, - "SHA256",,,,,, - "SHA256_HMAC",x,,x,,, - "SHA384",,,,,, - "SHA384_HMAC",,,x,,, - "SHA512",,,,,, - "SHA512_HMAC",x,,x,,, - "AES_XCBC",x,,x,,, - "SNOW3G_UIA2",x,,,,x, - "KASUMI_F9",,,,,,x +.. include:: overview_auth_table.txt Supported AEAD Algorithms +------------------------- -.. csv-table:: - :header: "AEAD Algorithms", "qat", "null", "aesni_mb", "aesni_gcm", "snow3g", "kasumi" - :stub-columns: 1 +.. _table_crypto_pmd_aead_algos: - "AES_GCM_128",x,,x,,, - "AES_GCM_192",x,,,,, - "AES_GCM_256",x,,,,, +.. include:: overview_aead_table.txt diff --git a/dpdk/doc/guides/cryptodevs/qat.rst b/dpdk/doc/guides/cryptodevs/qat.rst index cae19582..581cd2f7 100644 --- a/dpdk/doc/guides/cryptodevs/qat.rst +++ b/dpdk/doc/guides/cryptodevs/qat.rst @@ -27,11 +27,16 @@ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -Quick Assist Crypto Poll Mode Driver -==================================== +Intel(R) QuickAssist (QAT) Crypto Poll Mode Driver +================================================== -The QAT PMD provides poll mode crypto driver support for **Intel QuickAssist -Technology DH895xxC** hardware accelerator. +The QAT PMD provides poll mode crypto driver support for the following +hardware accelerator devices: + +* ``Intel QuickAssist Technology DH895xCC`` +* ``Intel QuickAssist Technology C62x`` +* ``Intel QuickAssist Technology C3xxx`` +* ``Intel QuickAssist Technology D15xx`` Features @@ -41,53 +46,178 @@ The QAT PMD has support for: Cipher algorithms: -* ``RTE_CRYPTO_SYM_CIPHER_AES128_CBC`` -* ``RTE_CRYPTO_SYM_CIPHER_AES192_CBC`` -* ``RTE_CRYPTO_SYM_CIPHER_AES256_CBC`` -* ``RTE_CRYPTO_SYM_CIPHER_AES128_CTR`` -* ``RTE_CRYPTO_SYM_CIPHER_AES192_CTR`` -* ``RTE_CRYPTO_SYM_CIPHER_AES256_CTR`` -* ``RTE_CRYPTO_SYM_CIPHER_SNOW3G_UEA2`` -* ``RTE_CRYPTO_CIPHER_AES_GCM`` +* ``RTE_CRYPTO_CIPHER_3DES_CBC`` +* ``RTE_CRYPTO_CIPHER_3DES_CTR`` +* ``RTE_CRYPTO_CIPHER_AES128_CBC`` +* ``RTE_CRYPTO_CIPHER_AES192_CBC`` +* ``RTE_CRYPTO_CIPHER_AES256_CBC`` +* ``RTE_CRYPTO_CIPHER_AES128_CTR`` +* ``RTE_CRYPTO_CIPHER_AES192_CTR`` +* ``RTE_CRYPTO_CIPHER_AES256_CTR`` +* ``RTE_CRYPTO_CIPHER_SNOW3G_UEA2`` +* ``RTE_CRYPTO_CIPHER_NULL`` +* ``RTE_CRYPTO_CIPHER_KASUMI_F8`` +* ``RTE_CRYPTO_CIPHER_DES_CBC`` +* ``RTE_CRYPTO_CIPHER_AES_DOCSISBPI`` +* ``RTE_CRYPTO_CIPHER_DES_DOCSISBPI`` +* ``RTE_CRYPTO_CIPHER_ZUC_EEA3`` Hash algorithms: * ``RTE_CRYPTO_AUTH_SHA1_HMAC`` +* ``RTE_CRYPTO_AUTH_SHA224_HMAC`` * ``RTE_CRYPTO_AUTH_SHA256_HMAC`` +* ``RTE_CRYPTO_AUTH_SHA384_HMAC`` * ``RTE_CRYPTO_AUTH_SHA512_HMAC`` * ``RTE_CRYPTO_AUTH_AES_XCBC_MAC`` * ``RTE_CRYPTO_AUTH_SNOW3G_UIA2`` +* ``RTE_CRYPTO_AUTH_MD5_HMAC`` +* ``RTE_CRYPTO_AUTH_NULL`` +* ``RTE_CRYPTO_AUTH_KASUMI_F9`` +* ``RTE_CRYPTO_AUTH_AES_GMAC`` +* ``RTE_CRYPTO_AUTH_ZUC_EIA3`` + +Supported AEAD algorithms: + +* ``RTE_CRYPTO_AEAD_AES_GCM`` Limitations ----------- -* Chained mbufs are not supported. -* Hash only is not supported except Snow3G UIA2. -* Cipher only is not supported except Snow3G UEA2. * Only supports the session-oriented API implementation (session-less APIs are not supported). -* Not performance tuned. -* Snow3g(UEA2) supported only if cipher length, cipher offset fields are byte-aligned. -* Snow3g(UIA2) supported only if hash length, hash offset fields are byte-aligned. +* SNOW 3G (UEA2), KASUMI (F8) and ZUC (EEA3) supported only if cipher length and offset fields are byte-multiple. +* SNOW 3G (UIA2) and ZUC (EIA3) supported only if hash length and offset fields are byte-multiple. * No BSD support as BSD QAT kernel driver not available. +* ZUC EEA3/EIA3 is not supported by dh895xcc devices +* Maximum additional authenticated data (AAD) for GCM is 240 bytes long. +* Queue pairs are not thread-safe (that is, within a single queue pair, RX and TX from different lcores is not supported). Installation ------------ -To use the DPDK QAT PMD an SRIOV-enabled QAT kernel driver is required. The -VF devices exposed by this driver will be used by QAT PMD. +To enable QAT in DPDK, follow the instructions for modifying the compile-time +configuration file as described `here `_. -If you are running on kernel 4.4 or greater, see instructions for -`Installation using kernel.org driver`_ below. If you are on a kernel earlier -than 4.4, see `Installation using 01.org QAT driver`_. +Quick instructions are as follows: + +.. code-block:: console + + cd to the top-level DPDK directory + make config T=x86_64-native-linuxapp-gcc + sed -i 's,\(CONFIG_RTE_LIBRTE_PMD_QAT\)=n,\1=y,' build/.config + make + +To use the DPDK QAT PMD an SRIOV-enabled QAT kernel driver is required. The VF +devices exposed by this driver will be used by the QAT PMD. The devices and +available kernel drivers and device ids are : + +.. _table_qat_pmds_drivers: + +.. table:: QAT device generations, devices and drivers + + +-----+----------+--------+---------------+------------+--------+------+--------+--------+ + | Gen | Device | Driver | Kernel Module | Pci Driver | PF Did | #PFs | Vf Did | VFs/PF | + +=====+==========+========+===============+============+========+======+========+========+ + | 1 | DH895xCC | 01.org | icp_qa_al | n/a | 435 | 1 | 443 | 32 | + +-----+----------+--------+---------------+------------+--------+------+--------+--------+ + | 1 | DH895xCC | 4.4+ | qat_dh895xcc | dh895xcc | 435 | 1 | 443 | 32 | + +-----+----------+--------+---------------+------------+--------+------+--------+--------+ + | 2 | C62x | 4.5+ | qat_c62x | c6xx | 37c8 | 3 | 37c9 | 16 | + +-----+----------+--------+---------------+------------+--------+------+--------+--------+ + | 2 | C3xxx | 4.5+ | qat_c3xxx | c3xxx | 19e2 | 1 | 19e3 | 16 | + +-----+----------+--------+---------------+------------+--------+------+--------+--------+ + | 2 | D15xx | p | qat_d15xx | d15xx | 6f54 | 1 | 6f55 | 16 | + +-----+----------+--------+---------------+------------+--------+------+--------+--------+ + + +The ``Driver`` column indicates either the Linux kernel version in which +support for this device was introduced or a driver available on Intel's 01.org +website. There are both linux and 01.org kernel drivers available for some +devices. p = release pending. + +If you are running on a kernel which includes a driver for your device, see +`Installation using kernel.org driver`_ below. Otherwise see +`Installation using 01.org QAT driver`_. + + +Installation using kernel.org driver +------------------------------------ + +The examples below are based on the C62x device, if you have a different device +use the corresponding values in the above table. + +In BIOS ensure that SRIOV is enabled and either: + +* Disable VT-d or +* Enable VT-d and set ``"intel_iommu=on iommu=pt"`` in the grub file. + +Check that the QAT driver is loaded on your system, by executing:: + + lsmod | grep qa + +You should see the kernel module for your device listed, e.g.:: + + qat_c62x 5626 0 + intel_qat 82336 1 qat_c62x + +Next, you need to expose the Virtual Functions (VFs) using the sysfs file system. + +First find the BDFs (Bus-Device-Function) of the physical functions (PFs) of +your device, e.g.:: + + lspci -d : 37c8 + +You should see output similar to:: + + 1a:00.0 Co-processor: Intel Corporation Device 37c8 + 3d:00.0 Co-processor: Intel Corporation Device 37c8 + 3f:00.0 Co-processor: Intel Corporation Device 37c8 + +Enable the VFs for each PF by echoing the number of VFs per PF to the pci driver:: + + echo 16 > /sys/bus/pci/drivers/c6xx/0000:1a:00.0/sriov_numvfs + echo 16 > /sys/bus/pci/drivers/c6xx/0000:3d:00.0/sriov_numvfs + echo 16 > /sys/bus/pci/drivers/c6xx/0000:3f:00.0/sriov_numvfs + +Check that the VFs are available for use. For example ``lspci -d:37c9`` should +list 48 VF devices available for a ``C62x`` device. + +To complete the installation follow the instructions in +`Binding the available VFs to the DPDK UIO driver`_. + +.. Note:: + + If the QAT kernel modules are not loaded and you see an error like ``Failed + to load MMP firmware qat_895xcc_mmp.bin`` in kernel logs, this may be as a + result of not using a distribution, but just updating the kernel directly. + + Download firmware from the `kernel firmware repo + `_. + + Copy qat binaries to ``/lib/firmware``:: + + cp qat_895xcc.bin /lib/firmware + cp qat_895xcc_mmp.bin /lib/firmware + + Change to your linux source root directory and start the qat kernel modules:: + + insmod ./drivers/crypto/qat/qat_common/intel_qat.ko + insmod ./drivers/crypto/qat/qat_dh895xcc/qat_dh895xcc.ko + + +.. Note:: + + If you see the following warning in ``/var/log/messages`` it can be ignored: + ``IOMMU should be enabled for SR-IOV to work correctly``. Installation using 01.org QAT driver ------------------------------------ Download the latest QuickAssist Technology Driver from `01.org -`_ +`_. Consult the *Getting Started Guide* at the same URL for further information. The steps below assume you are: @@ -108,136 +238,150 @@ Build and install the SRIOV-enabled QAT driver:: mkdir /QAT cd /QAT - # copy qatmux.l.2.3.0-34.tgz to this location + + # Copy qatmux.l.2.3.0-34.tgz to this location tar zxof qatmux.l.2.3.0-34.tgz export ICP_WITHOUT_IOMMU=1 ./installer.sh install QAT1.6 host You can use ``cat /proc/icp_dh895xcc_dev0/version`` to confirm the driver is correctly installed. -You can use ``lspci -d:443`` to confirm the bdf of the 32 VF devices are available per ``DH895xCC`` device. +You can use ``lspci -d:443`` to confirm the of the 32 VF devices available per ``DH895xCC`` device. To complete the installation - follow instructions in `Binding the available VFs to the DPDK UIO driver`_. -**Note**: If using a later kernel and the build fails with an error relating to ``strict_stroul`` not being available apply the following patch: +.. Note:: -.. code-block:: diff + If using a later kernel and the build fails with an error relating to + ``strict_stroul`` not being available apply the following patch: - /QAT/QAT1.6/quickassist/utilities/downloader/Target_CoreLibs/uclo/include/linux/uclo_platform.h - + #if LINUX_VERSION_CODE >= KERNEL_VERSION(3,18,5) - + #define STR_TO_64(str, base, num, endPtr) {endPtr=NULL; if (kstrtoul((str), (base), (num))) printk("Error strtoull convert %s\n", str); } - + #else - #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,38) - #define STR_TO_64(str, base, num, endPtr) {endPtr=NULL; if (strict_strtoull((str), (base), (num))) printk("Error strtoull convert %s\n", str); } - #else - #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,25) - #define STR_TO_64(str, base, num, endPtr) {endPtr=NULL; strict_strtoll((str), (base), (num));} - #else - #define STR_TO_64(str, base, num, endPtr) \ - do { \ - if (str[0] == '-') \ - { \ - *(num) = -(simple_strtoull((str+1), &(endPtr), (base))); \ - }else { \ - *(num) = simple_strtoull((str), &(endPtr), (base)); \ - } \ - } while(0) - + #endif - #endif - #endif + .. code-block:: diff + + /QAT/QAT1.6/quickassist/utilities/downloader/Target_CoreLibs/uclo/include/linux/uclo_platform.h + + #if LINUX_VERSION_CODE >= KERNEL_VERSION(3,18,5) + + #define STR_TO_64(str, base, num, endPtr) {endPtr=NULL; if (kstrtoul((str), (base), (num))) printk("Error strtoull convert %s\n", str); } + + #else + #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,38) + #define STR_TO_64(str, base, num, endPtr) {endPtr=NULL; if (strict_strtoull((str), (base), (num))) printk("Error strtoull convert %s\n", str); } + #else + #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,25) + #define STR_TO_64(str, base, num, endPtr) {endPtr=NULL; strict_strtoll((str), (base), (num));} + #else + #define STR_TO_64(str, base, num, endPtr) \ + do { \ + if (str[0] == '-') \ + { \ + *(num) = -(simple_strtoull((str+1), &(endPtr), (base))); \ + }else { \ + *(num) = simple_strtoull((str), &(endPtr), (base)); \ + } \ + } while(0) + + #endif + #endif + #endif -If the build fails due to missing header files you may need to do following: +.. Note:: -* ``sudo yum install zlib-devel`` -* ``sudo yum install openssl-devel`` + If the build fails due to missing header files you may need to do following:: -If the build or install fails due to mismatching kernel sources you may need to do the following: + sudo yum install zlib-devel + sudo yum install openssl-devel -* ``sudo yum install kernel-headers-`uname -r``` -* ``sudo yum install kernel-src-`uname -r``` -* ``sudo yum install kernel-devel-`uname -r``` +.. Note:: + If the build or install fails due to mismatching kernel sources you may need to do the following:: -Installation using kernel.org driver ------------------------------------- - -Assuming you are running on at least a 4.4 kernel, you can use the stock kernel.org QAT -driver to start the QAT hardware. - -The steps below assume you are: - -* Running DPDK on a platform with one ``DH895xCC`` device. -* On a kernel at least version 4.4. - -In BIOS ensure that SRIOV is enabled and VT-d is disabled. - -Ensure the QAT driver is loaded on your system, by executing:: - - lsmod | grep qat - -You should see the following output:: - - qat_dh895xcc 5626 0 - intel_qat 82336 1 qat_dh895xcc - -Next, you need to expose the VFs using the sysfs file system. - -First find the bdf of the DH895xCC device:: - - lspci -d : 435 - -You should see output similar to:: - - 03:00.0 Co-processor: Intel Corporation Coleto Creek PCIe Endpoint - -Using the sysfs, enable the VFs:: - - echo 32 > /sys/bus/pci/drivers/dh895xcc/0000\:03\:00.0/sriov_numvfs - -If you get an error, it's likely you're using a QAT kernel driver earlier than kernel 4.4. - -To verify that the VFs are available for use - use ``lspci -d:443`` to confirm -the bdf of the 32 VF devices are available per ``DH895xCC`` device. - -To complete the installation - follow instructions in `Binding the available VFs to the DPDK UIO driver`_. - -**Note**: If the QAT kernel modules are not loaded and you see an error like - ``Failed to load MMP firmware qat_895xcc_mmp.bin`` this may be as a - result of not using a distribution, but just updating the kernel directly. - -Download firmware from the kernel firmware repo at: -http://git.kernel.org/cgit/linux/kernel/git/firmware/linux-firmware.git/tree/ - -Copy qat binaries to /lib/firmware: -* ``cp qat_895xcc.bin /lib/firmware`` -* ``cp qat_895xcc_mmp.bin /lib/firmware`` - -cd to your linux source root directory and start the qat kernel modules: -* ``insmod ./drivers/crypto/qat/qat_common/intel_qat.ko`` -* ``insmod ./drivers/crypto/qat/qat_dh895xcc/qat_dh895xcc.ko`` - -**Note**:The following warning in /var/log/messages can be ignored: - ``IOMMU should be enabled for SR-IOV to work correctly`` - + sudo yum install kernel-headers-`uname -r` + sudo yum install kernel-src-`uname -r` + sudo yum install kernel-devel-`uname -r` Binding the available VFs to the DPDK UIO driver ------------------------------------------------ -The unbind command below assumes ``bdfs`` of ``03:01.00-03:04.07``, if yours are different adjust the unbind command below:: +Unbind the VFs from the stock driver so they can be bound to the uio driver. - cd $RTE_SDK - modprobe uio - insmod ./build/kmod/igb_uio.ko +For an Intel(R) QuickAssist Technology DH895xCC device +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - for device in $(seq 1 4); do \ - for fn in $(seq 0 7); do \ - echo -n 0000:03:0${device}.${fn} > \ - /sys/bus/pci/devices/0000\:03\:0${device}.${fn}/driver/unbind; \ - done; \ - done +The unbind command below assumes ``BDFs`` of ``03:01.00-03:04.07``, if your +VFs are different adjust the unbind command below:: - echo "8086 0443" > /sys/bus/pci/drivers/igb_uio/new_id + for device in $(seq 1 4); do \ + for fn in $(seq 0 7); do \ + echo -n 0000:03:0${device}.${fn} > \ + /sys/bus/pci/devices/0000\:03\:0${device}.${fn}/driver/unbind; \ + done; \ + done -You can use ``lspci -vvd:443`` to confirm that all devices are now in use by igb_uio kernel driver. +For an Intel(R) QuickAssist Technology C62x device +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The unbind command below assumes ``BDFs`` of ``1a:01.00-1a:02.07``, +``3d:01.00-3d:02.07`` and ``3f:01.00-3f:02.07``, if your VFs are different +adjust the unbind command below:: + + for device in $(seq 1 2); do \ + for fn in $(seq 0 7); do \ + echo -n 0000:1a:0${device}.${fn} > \ + /sys/bus/pci/devices/0000\:1a\:0${device}.${fn}/driver/unbind; \ + + echo -n 0000:3d:0${device}.${fn} > \ + /sys/bus/pci/devices/0000\:3d\:0${device}.${fn}/driver/unbind; \ + + echo -n 0000:3f:0${device}.${fn} > \ + /sys/bus/pci/devices/0000\:3f\:0${device}.${fn}/driver/unbind; \ + done; \ + done + +For Intel(R) QuickAssist Technology C3xxx or D15xx device +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The unbind command below assumes ``BDFs`` of ``01:01.00-01:02.07``, if your +VFs are different adjust the unbind command below:: + + for device in $(seq 1 2); do \ + for fn in $(seq 0 7); do \ + echo -n 0000:01:0${device}.${fn} > \ + /sys/bus/pci/devices/0000\:01\:0${device}.${fn}/driver/unbind; \ + done; \ + done + +Bind to the DPDK uio driver +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Install the DPDK igb_uio driver, bind the VF PCI Device id to it and use lspci +to confirm the VF devices are now in use by igb_uio kernel driver, +e.g. for the C62x device:: + + cd to the top-level DPDK directory + modprobe uio + insmod ./build/kmod/igb_uio.ko + echo "8086 37c9" > /sys/bus/pci/drivers/igb_uio/new_id + lspci -vvd:37c9 + + +Another way to bind the VFs to the DPDK UIO driver is by using the +``dpdk-devbind.py`` script:: + + cd to the top-level DPDK directory + ./usertools/dpdk-devbind.py -b igb_uio 0000:03:01.1 + + +Extra notes on KASUMI F9 +------------------------ + +When using KASUMI F9 authentication algorithm, the input buffer must be +constructed according to the 3GPP KASUMI specifications (section 4.4, page 13): +``_. +Input buffer has to have COUNT (4 bytes), FRESH (4 bytes), MESSAGE and DIRECTION (1 bit) +concatenated. After the DIRECTION bit, a single '1' bit is appended, followed by +between 0 and 7 '0' bits, so that the total length of the buffer is multiple of 8 bits. +Note that the actual message can be any length, specified in bits. + +Once this buffer is passed this way, when creating the crypto operation, +length of data to authenticate (op.sym.auth.data.length) must be the length +of all the items described above, including the padding at the end. +Also, offset of data to authenticate (op.sym.auth.data.offset) +must be such that points at the start of the COUNT bytes. diff --git a/dpdk/doc/guides/cryptodevs/scheduler.rst b/dpdk/doc/guides/cryptodevs/scheduler.rst new file mode 100644 index 00000000..e7e0247b --- /dev/null +++ b/dpdk/doc/guides/cryptodevs/scheduler.rst @@ -0,0 +1,199 @@ +.. BSD LICENSE + Copyright(c) 2017 Intel Corporation. All rights reserved. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Cryptodev Scheduler Poll Mode Driver Library +============================================ + +Scheduler PMD is a software crypto PMD, which has the capabilities of +attaching hardware and/or software cryptodevs, and distributes ingress +crypto ops among them in a certain manner. + +.. figure:: img/scheduler-overview.* + + Cryptodev Scheduler Overview + + +The Cryptodev Scheduler PMD library (**librte_pmd_crypto_scheduler**) acts as +a software crypto PMD and shares the same API provided by librte_cryptodev. +The PMD supports attaching multiple crypto PMDs, software or hardware, as +slaves, and distributes the crypto workload to them with certain behavior. +The behaviors are categorizes as different "modes". Basically, a scheduling +mode defines certain actions for scheduling crypto ops to its slaves. + +The librte_pmd_crypto_scheduler library exports a C API which provides an API +for attaching/detaching slaves, set/get scheduling modes, and enable/disable +crypto ops reordering. + +Limitations +----------- + +* Sessionless crypto operation is not supported +* OOP crypto operation is not supported when the crypto op reordering feature + is enabled. + + +Installation +------------ + +To build DPDK with CRYTPO_SCHEDULER_PMD the user is required to set +CONFIG_RTE_LIBRTE_PMD_CRYPTO_SCHEDULER=y in config/common_base, and +recompile DPDK + + +Initialization +-------------- + +To use the PMD in an application, user must: + +* Call rte_vdev_init("crypto_scheduler") within the application. + +* Use --vdev="crypto_scheduler" in the EAL options, which will call + rte_vdev_init() internally. + + +The following parameters (all optional) can be provided in the previous +two calls: + +* socket_id: Specify the socket where the memory for the device is going + to be allocated (by default, socket_id will be the socket where the core + that is creating the PMD is running on). + +* max_nb_sessions: Specify the maximum number of sessions that can be + created. This value may be overwritten internally if there are too + many devices are attached. + +* slave: If a cryptodev has been initialized with specific name, it can be + attached to the scheduler using this parameter, simply filling the name + here. Multiple cryptodevs can be attached initially by presenting this + parameter multiple times. + +* mode: Specify the scheduling mode of the PMD. The supported scheduling + mode parameter values are specified in the "Cryptodev Scheduler Modes + Overview" section. + +* ordering: Specify the status of the crypto operations ordering feature. + The value of this parameter can be "enable" or "disable". This feature + is disabled by default. + +Example: + +.. code-block:: console + + ... --vdev "crypto_aesni_mb0,name=aesni_mb_1" --vdev "crypto_aesni_mb1,name=aesni_mb_2" --vdev "crypto_scheduler,slave=aesni_mb_1,slave=aesni_mb_2" ... + +.. note:: + + * The scheduler cryptodev cannot be started unless the scheduling mode + is set and at least one slave is attached. Also, to configure the + scheduler in the run-time, like attach/detach slave(s), change + scheduling mode, or enable/disable crypto op ordering, one should stop + the scheduler first, otherwise an error will be returned. + + * The crypto op reordering feature requires using the userdata field of + every mbuf to be processed to store temporary data. By the end of + processing, the field is set to pointing to NULL, any previously + stored value of this field will be lost. + + +Cryptodev Scheduler Modes Overview +---------------------------------- + +Currently the Crypto Scheduler PMD library supports following modes of +operation: + +* **CDEV_SCHED_MODE_ROUNDROBIN:** + + *Initialization mode parameter*: **round-robin** + + Round-robin mode, which distributes the enqueued burst of crypto ops + among its slaves in a round-robin manner. This mode may help to fill + the throughput gap between the physical core and the existing cryptodevs + to increase the overall performance. + +* **CDEV_SCHED_MODE_PKT_SIZE_DISTR:** + + *Initialization mode parameter*: **packet-size-distr** + + Packet-size based distribution mode, which works with 2 slaves, the primary + slave and the secondary slave, and distributes the enqueued crypto + operations to them based on their data lengths. A crypto operation will be + distributed to the primary slave if its data length is equal to or bigger + than the designated threshold, otherwise it will be handled by the secondary + slave. + + A typical usecase in this mode is with the QAT cryptodev as the primary and + a software cryptodev as the secondary slave. This may help applications to + process additional crypto workload than what the QAT cryptodev can handle on + its own, by making use of the available CPU cycles to deal with smaller + crypto workloads. + + The threshold is set to 128 bytes by default. It can be updated by calling + function **rte_cryptodev_scheduler_option_set**. The parameter of + **option_type** must be **CDEV_SCHED_OPTION_THRESHOLD** and **option** should + point to a rte_cryptodev_scheduler_threshold_option structure filled with + appropriate threshold value. Please NOTE this threshold has be a power-of-2 + unsigned integer. + +* **CDEV_SCHED_MODE_FAILOVER:** + + *Initialization mode parameter*: **fail-over** + + Fail-over mode, which works with 2 slaves, the primary slave and the + secondary slave. In this mode, the scheduler will enqueue the incoming + crypto operation burst to the primary slave. When one or more crypto + operations fail to be enqueued, then they will be enqueued to the secondary + slave. + +* **CDEV_SCHED_MODE_MULTICORE:** + + *Initialization mode parameter*: **multi-core** + + Multi-core mode, which distributes the workload with several (up to eight) + worker cores. The enqueued bursts are distributed among the worker cores in a + round-robin manner. If scheduler cannot enqueue entire burst to the same worker, + it will enqueue the remaining operations to the next available worker. + For pure small packet size (64 bytes) traffic however the multi-core mode is not + an optimal solution, as it doesn't give significant per-core performance improvement. + For mixed traffic (IMIX) the optimal number of worker cores is around 2-3. + For large packets (1.5 Kbytes) scheduler shows linear scaling in performance + up to eight cores. + Each worker uses its own slave cryptodev. Only software cryptodevs + are supported. Only the same type of cryptodevs should be used concurrently. + + The multi-core mode uses one extra parameter: + + * corelist: Semicolon-separated list of logical cores to be used as workers. + The number of worker cores should be equal to the number of slave cryptodevs. + These cores should be present in EAL core list parameter and + should not be used by the application or any other process. + + Example: + ... --vdev "crypto_aesni_mb1,name=aesni_mb_1" --vdev "crypto_aesni_mb_pmd2,name=aesni_mb_2" \ + --vdev "crypto_scheduler,slave=aesni_mb_1,slave=aesni_mb_2,mode=multi-core,corelist=23;24" ... diff --git a/dpdk/doc/guides/cryptodevs/snow3g.rst b/dpdk/doc/guides/cryptodevs/snow3g.rst index 670a62a9..8914e29c 100644 --- a/dpdk/doc/guides/cryptodevs/snow3g.rst +++ b/dpdk/doc/guides/cryptodevs/snow3g.rst @@ -41,24 +41,24 @@ SNOW 3G PMD has support for: Cipher algorithm: -* RTE_CRYPTO_SYM_CIPHER_SNOW3G_UEA2 +* RTE_CRYPTO_CIPHER_SNOW3G_UEA2 Authentication algorithm: -* RTE_CRYPTO_SYM_AUTH_SNOW3G_UIA2 +* RTE_CRYPTO_AUTH_SNOW3G_UIA2 Limitations ----------- * Chained mbufs are not supported. -* Snow3g(UIA2) supported only if hash offset field is byte-aligned. -* In-place bit-level operations for Snow3g(UEA2) are not supported +* SNOW 3G (UIA2) supported only if hash offset field is byte-aligned. +* In-place bit-level operations for SNOW 3G (UEA2) are not supported (if length and/or offset of data to be ciphered is not byte-aligned). Installation ------------ -To build DPDK with the KASUMI_PMD the user is required to download +To build DPDK with the SNOW3G_PMD the user is required to download the export controlled ``libsso_snow3g`` library, by requesting it from ``_. Once approval has been granted, the user needs to log in @@ -69,6 +69,17 @@ on their system before building DPDK:: make snow3G +**Note**: When encrypting with SNOW3G UEA2, by default the library +encrypts blocks of 4 bytes, regardless the number of bytes to +be encrypted provided (which leads to a possible buffer overflow). +To avoid this situation, it is necessary not to pass +3GPP_SAFE_BUFFERS as a compilation flag. +For this, in the Makefile of the library, make sure that this flag +is commented out.:: + + #EXTRA_CFLAGS += -D_3GPP_SAFE_BUFFERS + + Initialization -------------- @@ -83,9 +94,9 @@ In order to enable this virtual crypto PMD, user must: To use the PMD in an application, user must: -* Call rte_eal_vdev_init("cryptodev_snow3g_pmd") within the application. +* Call rte_vdev_init("crypto_snow3g") within the application. -* Use --vdev="cryptodev_snow3g_pmd" in the EAL options, which will call rte_eal_vdev_init() internally. +* Use --vdev="crypto_snow3g" in the EAL options, which will call rte_vdev_init() internally. The following parameters (all optional) can be provided in the previous two calls: @@ -100,4 +111,5 @@ Example: .. code-block:: console - ./l2fwd-crypto -c 40 -n 4 --vdev="cryptodev_snow3g_pmd,socket_id=1,max_nb_sessions=128" + ./l2fwd-crypto -l 1 -n 4 --vdev="crypto_snow3g,socket_id=0,max_nb_sessions=128" \ + -- -p 1 --cdev SW --chain CIPHER_ONLY --cipher_algo "snow3g-uea2" diff --git a/dpdk/doc/guides/cryptodevs/zuc.rst b/dpdk/doc/guides/cryptodevs/zuc.rst new file mode 100644 index 00000000..7fcfc077 --- /dev/null +++ b/dpdk/doc/guides/cryptodevs/zuc.rst @@ -0,0 +1,115 @@ +.. BSD LICENSE + Copyright(c) 2016 Intel Corporation. All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +ZUC Crypto Poll Mode Driver +=========================== + +The ZUC PMD (**librte_pmd_zuc**) provides poll mode crypto driver +support for utilizing Intel Libsso library, which implements F8 and F9 functions +for ZUC EEA3 cipher and EIA3 hash algorithms. + +Features +-------- + +ZUC PMD has support for: + +Cipher algorithm: + +* RTE_CRYPTO_CIPHER_ZUC_EEA3 + +Authentication algorithm: + +* RTE_CRYPTO_AUTH_ZUC_EIA3 + +Limitations +----------- + +* Chained mbufs are not supported. +* ZUC (EIA3) supported only if hash offset field is byte-aligned. +* ZUC (EEA3) supported only if cipher length, cipher offset fields are byte-aligned. +* ZUC PMD cannot be built as a shared library, due to limitations in + in the underlying library. + + +Installation +------------ + +To build DPDK with the ZUC_PMD the user is required to download +the export controlled ``libsso_zuc`` library, by requesting it from +``_. +Once approval has been granted, the user needs to log in +``_ +and click on "ZUC Library" link, to download the library. +After downloading the library, the user needs to unpack and compile it +on their system before building DPDK:: + + make + +Initialization +-------------- + +In order to enable this virtual crypto PMD, user must: + +* Export the environmental variable LIBSSO_ZUC_PATH with the path where + the library was extracted (zuc folder). + +* Export the environmental variable LD_LIBRARY_PATH with the path + where the built libsso library is (LIBSSO_ZUC_PATH/build). + +* Build the LIBSSO_ZUC library (explained in Installation section). + +* Build DPDK as follows: + +.. code-block:: console + + make config T=x86_64-native-linuxapp-gcc + sed -i 's,\(CONFIG_RTE_LIBRTE_PMD_ZUC\)=n,\1=y,' build/.config + make + +To use the PMD in an application, user must: + +* Call rte_vdev_init("crypto_zuc") within the application. + +* Use --vdev="crypto_zuc" in the EAL options, which will call rte_vdev_init() internally. + +The following parameters (all optional) can be provided in the previous two calls: + +* socket_id: Specify the socket where the memory for the device is going to be allocated + (by default, socket_id will be the socket where the core that is creating the PMD is running on). + +* max_nb_queue_pairs: Specify the maximum number of queue pairs in the device (8 by default). + +* max_nb_sessions: Specify the maximum number of sessions that can be created (2048 by default). + +Example: + +.. code-block:: console + + ./l2fwd-crypto -l 1 -n 4 --vdev="crypto_zuc,socket_id=0,max_nb_sessions=128" \ + -- -p 1 --cdev SW --chain CIPHER_ONLY --cipher_algo "zuc-eea3" diff --git a/dpdk/doc/guides/eventdevs/dpaa2.rst b/dpdk/doc/guides/eventdevs/dpaa2.rst new file mode 100644 index 00000000..0970b266 --- /dev/null +++ b/dpdk/doc/guides/eventdevs/dpaa2.rst @@ -0,0 +1,175 @@ +.. BSD LICENSE + Copyright 2017 NXP. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of NXP nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +NXP DPAA2 Eventdev Driver +========================= + +The dpaa2 eventdev is an implementation of the eventdev API, that provides a +wide range of the eventdev features. The eventdev relies on a dpaa2 hw to +perform event scheduling. + +More information can be found at `NXP Official Website +`_. + +Features +-------- + +The DPAA2 EVENTDEV implements many features in the eventdev API; + +- Hardware based event scheduler +- 8 event ports +- 8 event queues +- Parallel flows +- Atomic flows + +Supported DPAA2 SoCs +-------------------- + +- LS2080A/LS2040A +- LS2084A/LS2044A +- LS2088A/LS2048A +- LS1088A/LS1048A + +Prerequisites +------------- + +There are three main pre-requisities for executing DPAA2 EVENTDEV on a DPAA2 +compatible board: + +1. **ARM 64 Tool Chain** + + For example, the `*aarch64* Linaro Toolchain `_. + +2. **Linux Kernel** + + It can be obtained from `NXP's Github hosting `_. + +3. **Rootfile system** + + Any *aarch64* supporting filesystem can be used. For example, + Ubuntu 15.10 (Wily) or 16.04 LTS (Xenial) userland which can be obtained + from `here `_. + +As an alternative method, DPAA2 EVENTDEV can also be executed using images provided +as part of SDK from NXP. The SDK includes all the above prerequisites necessary +to bring up a DPAA2 board. + +The following dependencies are not part of DPDK and must be installed +separately: + +- **NXP Linux SDK** + + NXP Linux software development kit (SDK) includes support for family + of QorIQ® ARM-Architecture-based system on chip (SoC) processors + and corresponding boards. + + It includes the Linux board support packages (BSPs) for NXP SoCs, + a fully operational tool chain, kernel and board specific modules. + + SDK and related information can be obtained from: `NXP QorIQ SDK `_. + +- **DPDK Extra Scripts** + + DPAA2 based resources can be configured easily with the help of ready scripts + as provided in the DPDK Extra repository. + + `DPDK Extras Scripts `_. + +Currently supported by DPDK: + +- NXP SDK **2.0+**. +- MC Firmware version **10.0.0** and higher. +- Supported architectures: **arm64 LE**. + +- Follow the DPDK :ref:`Getting Started Guide for Linux ` to setup the basic DPDK environment. + +.. note:: + + Some part of fslmc bus code (mc flib - object library) routines are + dual licensed (BSD & GPLv2). + +Pre-Installation Configuration +------------------------------ + +Config File Options +~~~~~~~~~~~~~~~~~~~ + +The following options can be modified in the ``config`` file. +Please note that enabling debugging options may affect system performance. + +- ``CONFIG_RTE_LIBRTE_PMD_DPAA2_EVENTDEV`` (default ``y``) + + Toggle compilation of the ``lrte_pmd_dpaa2_event`` driver. + +- ``CONFIG_RTE_LIBRTE_PMD_DPAA2_EVENTDEV_DEBUG`` (default ``n``) + + Toggle display of generic debugging messages + +Driver Compilation +~~~~~~~~~~~~~~~~~~ + +To compile the DPAA2 EVENTDEV PMD for Linux arm64 gcc target, run the +following ``make`` command: + +.. code-block:: console + + cd + make config T=arm64-dpaa2-linuxapp-gcc install + +Initialization +-------------- + +The dpaa2 eventdev is exposed as a vdev device which consists of a set of dpcon +devices and dpci devices. On EAL initialization, dpcon and dpci devices will be +probed and then vdev device can be created from the application code by + +* Invoking ``rte_vdev_init("event_dpaa2")`` from the application + +* Using ``--vdev="event_dpaa2"`` in the EAL options, which will call + rte_vdev_init() internally + +Example: + +.. code-block:: console + + ./your_eventdev_application --vdev="event_dpaa2" + +Limitations +----------- + +Platform Requirement +~~~~~~~~~~~~~~~~~~~~ + +DPAA2 drivers for DPDK can only work on NXP SoCs as listed in the +``Supported DPAA2 SoCs``. + +Port-core binding +~~~~~~~~~~~~~~~~~ + +DPAA2 EVENTDEV driver requires event port 'x' to be used on core 'x'. diff --git a/dpdk/doc/guides/eventdevs/index.rst b/dpdk/doc/guides/eventdevs/index.rst new file mode 100644 index 00000000..ba2048c7 --- /dev/null +++ b/dpdk/doc/guides/eventdevs/index.rst @@ -0,0 +1,42 @@ +.. BSD LICENSE + Copyright(c) 2017 Intel Corporation. All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Event Device Drivers +==================== + +The following are a list of event device PMDs, which can be used from an +application trough the eventdev API. + +.. toctree:: + :maxdepth: 2 + :numbered: + + dpaa2 + sw + octeontx diff --git a/dpdk/doc/guides/eventdevs/octeontx.rst b/dpdk/doc/guides/eventdevs/octeontx.rst new file mode 100644 index 00000000..cef004a2 --- /dev/null +++ b/dpdk/doc/guides/eventdevs/octeontx.rst @@ -0,0 +1,130 @@ +.. BSD LICENSE + Copyright (C) Cavium, Inc. 2017. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Cavium, Inc nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +OCTEONTX SSOVF Eventdev Driver +============================== + +The OCTEONTX SSOVF PMD (**librte_pmd_octeontx_ssovf**) provides poll mode +eventdev driver support for the inbuilt event device found in the **Cavium OCTEONTX** +SoC family as well as their virtual functions (VF) in SR-IOV context. + +More information can be found at `Cavium, Inc Official Website +`_. + +Features +-------- + +Features of the OCTEONTX SSOVF PMD are: + +- 64 Event queues +- 32 Event ports +- HW event scheduler +- Supports 1M flows per event queue +- Flow based event pipelining +- Flow pinning support in flow based event pipelining +- Queue based event pipelining +- Supports ATOMIC, ORDERED, PARALLEL schedule types per flow +- Event scheduling QoS based on event queue priority +- Open system with configurable amount of outstanding events +- HW accelerated dequeue timeout support to enable power management +- SR-IOV VF + +Supported OCTEONTX SoCs +----------------------- +- CN83xx + +Prerequisites +------------- + +See :doc: `../platform/octeontx` for setup information. + +Pre-Installation Configuration +------------------------------ + +Config File Options +~~~~~~~~~~~~~~~~~~~ + +The following options can be modified in the ``config`` file. +Please note that enabling debugging options may affect system performance. + +- ``CONFIG_RTE_LIBRTE_PMD_OCTEONTX_SSOVF`` (default ``y``) + + Toggle compilation of the ``librte_pmd_octeontx_ssovf`` driver. + +- ``CONFIG_RTE_LIBRTE_PMD_OCTEONTX_SSOVF_DEBUG`` (default ``n``) + + Toggle display of generic debugging messages + +Driver Compilation +~~~~~~~~~~~~~~~~~~ + +To compile the OCTEONTX SSOVF PMD for Linux arm64 gcc target, run the +following ``make`` command: + +.. code-block:: console + + cd + make config T=arm64-thunderx-linuxapp-gcc install + + +Initialization +-------------- + +The octeontx eventdev is exposed as a vdev device which consists of a set +of SSO group and work-slot PCIe VF devices. On EAL initialization, +SSO PCIe VF devices will be probed and then the vdev device can be created +from the application code, or from the EAL command line based on +the number of probed/bound SSO PCIe VF device to DPDK by + +* Invoking ``rte_vdev_init("event_octeontx")`` from the application + +* Using ``--vdev="event_octeontx"`` in the EAL options, which will call + rte_vdev_init() internally + +Example: + +.. code-block:: console + + ./your_eventdev_application --vdev="event_octeontx" + +Limitations +----------- + +Burst mode support +~~~~~~~~~~~~~~~~~~ + +Burst mode is not supported. Dequeue and Enqueue functions accepts only single +event at a time. + +Rx adapter support +~~~~~~~~~~~~~~~~~~ + +When eth_octeontx is used as Rx adapter event schedule type +``RTE_SCHED_TYPE_PARALLEL`` is not supported. diff --git a/dpdk/doc/guides/eventdevs/sw.rst b/dpdk/doc/guides/eventdevs/sw.rst new file mode 100644 index 00000000..ec49b3b5 --- /dev/null +++ b/dpdk/doc/guides/eventdevs/sw.rst @@ -0,0 +1,158 @@ +.. BSD LICENSE + Copyright(c) 2017 Intel Corporation. All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Software Eventdev Poll Mode Driver +================================== + +The software eventdev is an implementation of the eventdev API, that provides a +wide range of the eventdev features. The eventdev relies on a CPU core to +perform event scheduling. This PMD can use the service core library to run the +scheduling function, allowing an application to utilize the power of service +cores to multiplex other work on the same core if required. + + +Features +-------- + +The software eventdev implements many features in the eventdev API; + +Queues + * Atomic + * Ordered + * Parallel + * Single-Link + +Ports + * Load balanced (for Atomic, Ordered, Parallel queues) + * Single Link (for single-link queues) + +Event Priorities + * Each event has a priority, which can be used to provide basic QoS + + +Configuration and Options +------------------------- + +The software eventdev is a vdev device, and as such can be created from the +application code, or from the EAL command line: + +* Call ``rte_vdev_init("event_sw0")`` from the application + +* Use ``--vdev="event_sw0"`` in the EAL options, which will call + rte_vdev_init() internally + +Example: + +.. code-block:: console + + ./your_eventdev_application --vdev="event_sw0" + + +Scheduling Quanta +~~~~~~~~~~~~~~~~~ + +The scheduling quanta sets the number of events that the device attempts to +schedule in a single schedule call performed by the service core. Note that +is a *hint* only, and that fewer or more events may be scheduled in a given +iteration. + +The scheduling quanta can be set using a string argument to the vdev +create call: + +.. code-block:: console + + --vdev="event_sw0,sched_quanta=64" + + +Credit Quanta +~~~~~~~~~~~~~ + +The credit quanta is the number of credits that a port will fetch at a time from +the instance's credit pool. Higher numbers will cause less overhead in the +atomic credit fetch code, however it also reduces the overall number of credits +in the system faster. A balanced number (eg 32) ensures that only small numbers +of credits are pre-allocated at a time, while also mitigating performance impact +of the atomics. + +Experimentation with higher values may provide minor performance improvements, +at the cost of the whole system having less credits. On the other hand, +reducing the quanta may cause measurable performance impact but provide the +system with a higher number of credits at all times. + +A value of 32 seems a good balance however your specific application may +benefit from a higher or reduced quanta size, experimentation is required to +verify possible gains. + +.. code-block:: console + + --vdev="event_sw0,credit_quanta=64" + + +Limitations +----------- + +The software eventdev implementation has a few limitations. The reason for +these limitations is usually that the performance impact of supporting the +feature would be significant. + + +"All Types" Queues +~~~~~~~~~~~~~~~~~~ + +The software eventdev does not support creating queues that handle all types of +traffic. An eventdev with this capability allows enqueueing Atomic, Ordered and +Parallel traffic to the same queue, but scheduling each of them appropriately. + +The reason to not allow Atomic, Ordered and Parallel event types in the +same queue is that it causes excessive branching in the code to enqueue packets +to the queue, causing a significant performance impact. + +The ``RTE_EVENT_DEV_CAP_QUEUE_ALL_TYPES`` flag is not set in the +``event_dev_cap`` field of the ``rte_event_dev_info`` struct for the software +eventdev. + +Distributed Scheduler +~~~~~~~~~~~~~~~~~~~~~ + +The software eventdev is a centralized scheduler, requiring a service core to +perform the required event distribution. This is not really a limitation but +rather a design decision. + +The ``RTE_EVENT_DEV_CAP_DISTRIBUTED_SCHED`` flag is not set in the +``event_dev_cap`` field of the ``rte_event_dev_info`` struct for the software +eventdev. + +Dequeue Timeout +~~~~~~~~~~~~~~~ + +The eventdev API supports a timeout when dequeuing packets using the +``rte_event_dequeue_burst`` function. +This allows a core to wait for an event to arrive, or until ``timeout`` number +of ticks have passed. Timeout ticks is not supported by the software eventdev +for performance reasons. diff --git a/dpdk/doc/guides/faq/faq.rst b/dpdk/doc/guides/faq/faq.rst index 8d1ea6cc..dac80509 100644 --- a/dpdk/doc/guides/faq/faq.rst +++ b/dpdk/doc/guides/faq/faq.rst @@ -37,8 +37,8 @@ Alternatively, applications can also be run as regular user. For more information, please refer to :ref:`DPDK Getting Started Guide `. -If I want to change the number of TLB Hugepages allocated, how do I remove the original pages allocated? --------------------------------------------------------------------------------------------------------- +If I want to change the number of hugepages allocated, how do I remove the original pages allocated? +---------------------------------------------------------------------------------------------------- The number of pages allocated can be seen by executing the following command:: @@ -50,12 +50,12 @@ When you stop and restart the test application, it looks to see if the pages are If you look in the directory, you will see ``n`` number of 2M pages files. If you specified 1024, you will see 1024 page files. These are then placed in memory segments to get contiguous memory. -If you need to change the number of pages, it is easier to first remove the pages. The tools/dpdk-setup.sh script provides an option to do this. +If you need to change the number of pages, it is easier to first remove the pages. The usertools/dpdk-setup.sh script provides an option to do this. See the "Quick Start Setup Script" section in the :ref:`DPDK Getting Started Guide ` for more information. -If I execute "l2fwd -c f -m 64 -n 3 -- -p 3", I get the following output, indicating that there are no socket 0 hugepages to allocate the mbuf and ring structures to? ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- +If I execute "l2fwd -l 0-3 -m 64 -n 3 -- -p 3", I get the following output, indicating that there are no socket 0 hugepages to allocate the mbuf and ring structures to? +------------------------------------------------------------------------------------------------------------------------------------------------------------------------ I have set up a total of 1024 Hugepages (that is, allocated 512 2M pages to each NUMA node). @@ -66,19 +66,21 @@ To request memory to be reserved on a specific socket, please use the --socket-m I am running a 32-bit DPDK application on a NUMA system, and sometimes the application initializes fine but cannot allocate memory. Why is that happening? ---------------------------------------------------------------------------------------------------------------------------------------------------------- -32-bit applications have limitations in terms of how much virtual memory is available, hence the number of hugepages they are able to allocate is also limited (1 GB per page size). -If your system has a lot (>1 GB per page size) of hugepage memory, not all of it will be allocated. +32-bit applications have limitations in terms of how much virtual memory is available, hence the number of hugepages they are able to allocate is also limited (1 GB size). +If your system has a lot (>1 GB size) of hugepage memory, not all of it will be allocated. Due to hugepages typically being allocated on a local NUMA node, the hugepages allocation the application gets during the initialization depends on which NUMA node it is running on (the EAL does not affinitize cores until much later in the initialization process). Sometimes, the Linux OS runs the DPDK application on a core that is located on a different NUMA node from DPDK master core and therefore all the hugepages are allocated on the wrong socket. -To avoid this scenario, either lower the amount of hugepage memory available to 1 GB per page size (or less), or run the application with taskset +To avoid this scenario, either lower the amount of hugepage memory available to 1 GB size (or less), or run the application with taskset affinitizing the application to a would-be master core. For example, if your EAL coremask is 0xff0, the master core will usually be the first core in the coremask (0x10); this is what you have to supply to taskset:: - taskset 0x10 ./l2fwd -c 0xff0 -n 2 + taskset 0x10 ./l2fwd -l 4-11 -n 2 + +.. Note: Instead of '-c 0xff0' use the '-l 4-11' as a cleaner way to define lcores. In this way, the hugepages have a greater chance of being allocated to the correct socket. Additionally, a ``--socket-mem`` option could be used to ensure the availability of memory for each socket, so that if hugepages were allocated on @@ -101,7 +103,7 @@ Yes, the option ``--log-level=`` accepts one of these numbers: #define RTE_LOG_INFO 7U /* Informational. */ #define RTE_LOG_DEBUG 8U /* Debug-level messages. */ -It is also possible to change the maximum (and default level) at compile time +It is also possible to change the default level at compile time with ``CONFIG_RTE_LOG_LEVEL``. @@ -113,16 +115,16 @@ but the end-to-end latency of an average packet typically increases as a result. Similarly, the application can be tuned to have, on average, a low end-to-end latency at the cost of lower throughput. To achieve higher throughput, the DPDK attempts to aggregate the cost of processing each packet individually by processing packets in bursts. -Using the testpmd application as an example, the "burst" size can be set on the command line to a value of 16 (also the default value). -This allows the application to request 16 packets at a time from the PMD. -The testpmd application then immediately attempts to transmit all the packets that were received, in this case, all 16 packets. +Using the testpmd application as an example, the "burst" size can be set on the command line to a value of 32 (also the default value). +This allows the application to request 32 packets at a time from the PMD. +The testpmd application then immediately attempts to transmit all the packets that were received, in this case, all 32 packets. The packets are not transmitted until the tail pointer is updated on the corresponding TX queue of the network port. This behavior is desirable when tuning for high throughput because the cost of tail pointer updates to both the RX and TX queues -can be spread across 16 packets, effectively hiding the relatively slow MMIO cost of writing to the PCIe* device. +can be spread across 32 packets, effectively hiding the relatively slow MMIO cost of writing to the PCIe* device. -However, this is not very desirable when tuning for low latency, because the first packet that was received must also wait for the other 15 packets to be received. -It cannot be transmitted until the other 15 packets have also been processed because the NIC will not know to transmit the packets until the TX tail pointer has been updated, -which is not done until all 16 packets have been processed for transmission. +However, this is not very desirable when tuning for low latency, because the first packet that was received must also wait for the other 31 packets to be received. +It cannot be transmitted until the other 31 packets have also been processed because the NIC will not know to transmit the packets until the TX tail pointer has been updated, +which is not done until all 32 packets have been processed for transmission. To consistently achieve low latency even under heavy system load, the application developer should avoid processing packets in bunches. The testpmd application can be configured from the command line to use a burst value of 1. @@ -220,3 +222,10 @@ How can hugepage-backed memory be shared among multiple processes? ------------------------------------------------------------------ See the Primary and Secondary examples in the :ref:`multi-process sample application `. + + +Why can't my application receive packets on my system with UEFI Secure Boot enabled? +------------------------------------------------------------------------------------ + +If UEFI secure boot is enabled, the Linux kernel may disallow the use of UIO on the system. +Therefore, devices for use by DPDK should be bound to the ``vfio-pci`` kernel module rather than ``igb_uio`` or ``uio_pci_generic``. diff --git a/dpdk/doc/guides/freebsd_gsg/build_dpdk.rst b/dpdk/doc/guides/freebsd_gsg/build_dpdk.rst index 93c43661..8bd9b6a1 100644 --- a/dpdk/doc/guides/freebsd_gsg/build_dpdk.rst +++ b/dpdk/doc/guides/freebsd_gsg/build_dpdk.rst @@ -88,7 +88,7 @@ The ports required and their locations are as follows: For compiling and using the DPDK with gcc, the compiler must be installed from the ports collection: -* gcc: version 4.8 is recommended ``/usr/ports/lang/gcc48``. +* gcc: version 4.9 is recommended ``/usr/ports/lang/gcc49``. Ensure that ``CPU_OPTS`` is selected (default is OFF). When running the make config-recursive command, a dialog may be presented to the @@ -111,10 +111,6 @@ First, uncompress the archive and move to the DPDK source directory: unzip DPDK-.zip cd DPDK- - ls - app/ config/ examples/ lib/ LICENSE.GPL LICENSE.LGPL Makefile - mk/ scripts/ tools/ - The DPDK is composed of several directories: * lib: Source code of DPDK libraries @@ -123,7 +119,7 @@ The DPDK is composed of several directories: * examples: Source code of DPDK applications -* config, tools, scripts, mk: Framework-related makefiles, scripts and configuration +* config, buildtools, mk: Framework-related makefiles, scripts and configuration Installation of the DPDK Target Environments -------------------------------------------- @@ -168,7 +164,7 @@ For example to compile for FreeBSD use: If the compiler binary to be used does not correspond to that given in the TOOLCHAIN part of the target, the compiler command may need to be explicitly specified. For example, if compiling for gcc, where the gcc binary is called - gcc4.8, the command would need to be ``gmake install T= CC=gcc4.8``. + gcc4.9, the command would need to be ``gmake install T= CC=gcc4.9``. Browsing the Installed DPDK Environment Target ---------------------------------------------- @@ -177,14 +173,7 @@ Once a target is created, it contains all the libraries and header files for the DPDK environment that are required to build customer applications. In addition, the test and testpmd applications are built under the build/app directory, which may be used for testing. A kmod directory is also present that -contains the kernel modules to install: - -.. code-block:: console - - ls x86_64-native-bsdapp-gcc - - app build include kmod lib Makefile - +contains the kernel modules to install. .. _loading_contigmem: diff --git a/dpdk/doc/guides/freebsd_gsg/build_sample_apps.rst b/dpdk/doc/guides/freebsd_gsg/build_sample_apps.rst index 2662303d..d84f15b8 100644 --- a/dpdk/doc/guides/freebsd_gsg/build_sample_apps.rst +++ b/dpdk/doc/guides/freebsd_gsg/build_sample_apps.rst @@ -54,7 +54,7 @@ the following variables must be exported: The following is an example of creating the ``helloworld`` application, which runs in the DPDK FreeBSD environment. While the example demonstrates compiling -using gcc version 4.8, compiling with clang will be similar, except that the ``CC=`` +using gcc version 4.9, compiling with clang will be similar, except that the ``CC=`` parameter can probably be omitted. The ``helloworld`` example may be found in the ``${RTE_SDK}/examples`` directory. @@ -72,7 +72,7 @@ in the build directory. setenv RTE_SDK $HOME/DPDK setenv RTE_TARGET x86_64-native-bsdapp-gcc - gmake CC=gcc48 + gmake CC=gcc49 CC main.o LD helloworld INSTALL-APP helloworld @@ -96,7 +96,7 @@ in the build directory. cd my_rte_app/ setenv RTE_TARGET x86_64-native-bsdapp-gcc - gmake CC=gcc48 + gmake CC=gcc49 CC main.o LD helloworld INSTALL-APP helloworld @@ -119,7 +119,7 @@ The following is the list of options that can be given to the EAL: .. code-block:: console - ./rte-app -c COREMASK [-n NUM] [-b ] \ + ./rte-app -l CORELIST [-n NUM] [-b ] \ [-r NUM] [-v] [--proc-type ] .. note:: @@ -130,9 +130,10 @@ The following is the list of options that can be given to the EAL: The EAL options for FreeBSD are as follows: -* ``-c COREMASK``: +* ``-c COREMASK`` or ``-l CORELIST``: A hexadecimal bit mask of the cores to run on. Note that core numbering - can change between platforms and should be determined beforehand. + can change between platforms and should be determined beforehand. The corelist + is a list of cores to use instead of a core mask. * ``-n NUM``: Number of memory channels per processor socket. @@ -162,6 +163,9 @@ Other options, specific to Linux and are not supported under FreeBSD are as foll * ``--huge-dir``: The directory where hugetlbfs is mounted. +* ``mbuf-pool-ops-name``: + Pool ops name for mbuf to use. + * ``--file-prefix``: The prefix text used for hugepage filenames. @@ -169,13 +173,13 @@ Other options, specific to Linux and are not supported under FreeBSD are as foll Memory to allocate from hugepages, regardless of processor socket. It is recommended that ``--socket-mem`` be used instead of this option. -The ``-c`` option is mandatory; the others are optional. +The ``-c`` or ``-l`` option is mandatory; the others are optional. Copy the DPDK application binary to your target, then run the application as follows (assuming the platform has four memory channels, and that cores 0-3 are present and are to be used for running the application):: - ./helloworld -c f -n 4 + ./helloworld -l 0-3 -n 4 .. note:: diff --git a/dpdk/doc/guides/freebsd_gsg/install_from_ports.rst b/dpdk/doc/guides/freebsd_gsg/install_from_ports.rst index 81770291..67e63d27 100644 --- a/dpdk/doc/guides/freebsd_gsg/install_from_ports.rst +++ b/dpdk/doc/guides/freebsd_gsg/install_from_ports.rst @@ -111,7 +111,7 @@ compiled and run as below: INSTALL-APP helloworld INSTALL-MAP helloworld.map - sudo ./build/helloworld -c F -n 2 + sudo ./build/helloworld -l 0-3 -n 2 EAL: Contigmem driver has 2 buffers, each of size 1GB EAL: Sysctl reports 8 cpus diff --git a/dpdk/doc/guides/howto/flow_bifurcation.rst b/dpdk/doc/guides/howto/flow_bifurcation.rst index a1c6262b..61016a4f 100644 --- a/dpdk/doc/guides/howto/flow_bifurcation.rst +++ b/dpdk/doc/guides/howto/flow_bifurcation.rst @@ -119,14 +119,14 @@ The typical procedure to achieve this is as follows: .. code-block:: console modprobe vfio-pci - dpdk_nic_bind.py -b vfio-pci 01:10.0 - dpdk_nic_bind.py -b vfio-pci 01:10.1 + dpdk-devbind.py -b vfio-pci 01:10.0 + dpdk-devbind.py -b vfio-pci 01:10.1 #. Run a DPDK application on the VFs: .. code-block:: console - testpmd -c 0xff -n 4 -- -i -w 01:10.0 -w 01:10.1 --forward-mode=mac + testpmd -l 0-7 -n 4 -- -i -w 01:10.0 -w 01:10.1 --forward-mode=mac In this example, traffic matching the rules will go through the VF by matching the filter rule. All other traffic, not matching the rules, will go through @@ -279,14 +279,14 @@ The typical procedure to achieve this is as follows: .. code-block:: console modprobe vfio-pci - dpdk_nic_bind.py -b vfio-pci 01:10.0 - dpdk_nic_bind.py -b vfio-pci 01:10.1 + dpdk-devbind.py -b vfio-pci 01:10.0 + dpdk-devbind.py -b vfio-pci 01:10.1 #. run DPDK application on VFs: .. code-block:: console - testpmd -c 0xff -n 4 -- -i -w 01:10.0 -w 01:10.1 --forward-mode=mac + testpmd -l 0-7 -n 4 -- -i -w 01:10.0 -w 01:10.1 --forward-mode=mac .. note:: diff --git a/dpdk/doc/guides/howto/img/packet_capture_framework.svg b/dpdk/doc/guides/howto/img/packet_capture_framework.svg new file mode 100644 index 00000000..a76baf71 --- /dev/null +++ b/dpdk/doc/guides/howto/img/packet_capture_framework.svg @@ -0,0 +1,471 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + DPDK Primary Application + + dpdk-pdumptool + + PCAP PMD + + dpdk_port0 + + librte_pdump + + capture.pcap + + Traffic Generator + + + + + + + diff --git a/dpdk/doc/guides/howto/img/pvp_2nics.svg b/dpdk/doc/guides/howto/img/pvp_2nics.svg new file mode 100644 index 00000000..517a8008 --- /dev/null +++ b/dpdk/doc/guides/howto/img/pvp_2nics.svg @@ -0,0 +1,556 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + TE + + 10G NIC + Moongen + + DUT + + VM + + 10G NIC + + TestPMD(macswap) + + TestPMD (io) + + + + + + + + 10G NIC + + 10G NIC + + + + + + + + + + diff --git a/dpdk/doc/guides/howto/img/use_models_for_running_dpdk_in_containers.svg b/dpdk/doc/guides/howto/img/use_models_for_running_dpdk_in_containers.svg new file mode 100644 index 00000000..662c2266 --- /dev/null +++ b/dpdk/doc/guides/howto/img/use_models_for_running_dpdk_in_containers.svg @@ -0,0 +1,398 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + Page-1 + + + + Rectangle + Container + + + + + + + + + + Container + + Rectangle.3 + + + + + + + + + + Rectangle.4 + + + + + + + + + + Sheet.5 + Host kernel + + + + Host kernel + + Sheet.6 + NIC + + + + NIC + + Rectangle.7 + PF + + + + + + + + + + PF + + Rectangle.8 + VF + + + + + + + + + + VF + + Rectangle.10 + Hardware virtual switch + + + + + + + + + + Hardware virtual switch + + Rectangle.14 + Container + + + + + + + + + + Container + + Rectangle.15 + VF + + + + + + + + + + VF + + Dynamic connector.16 + + + + Ellipse + PF driver + + + + + + + + + + PF driver + + Dynamic connector.19 + + + + Ellipse.20 + DPDK + + + + + + + + + + DPDK + + Dynamic connector.21 + + + + Ellipse.22 + DPDK + + + + + + + + + + DPDK + + Rectangle.23 + Virtual Appliance + + + + + + + + + + Virtual Appliance + + Rectangle.25 + VM + Container + + + + + + + + + + VM + Container + + Rectangle.27 + Container + + + + + + + + + + Container + + Ellipse.28 + DPDK + + + + + + + + + + DPDK + + Rectangle.29 + + + + + + + + + + Sheet.30 + Host kernel + + + + Host kernel + + Rectangle.31 + vSwitch or vRouter + + + + + + + + + + vSwitchorvRouter + + Ellipse.32 + DPDK + + + + + + + + + + DPDK + + Dynamic connector + + + + Dynamic connector.35 + + + + Dynamic connector.36 + + + + Rectangle.37 + + + + + + + + + + Rectangle.38 + + + + + + + + + + Sheet.39 + NIC + + + + NIC + + Dynamic connector.41 + + + + Dynamic connector.42 + + + + Sheet.43 + (1) Slicing + + + + (1) Slicing + + Sheet.44 + (2) Aggregation + + + + (2) Aggregation + + diff --git a/dpdk/doc/guides/howto/img/vf_daemon_overview.svg b/dpdk/doc/guides/howto/img/vf_daemon_overview.svg new file mode 100644 index 00000000..d4a47234 --- /dev/null +++ b/dpdk/doc/guides/howto/img/vf_daemon_overview.svg @@ -0,0 +1,440 @@ + + + + + +image/svg+xmlSimple Double Arrow.14VM +VF Application +DPDK +Virtual ethdev +VF driver +Simple Double Arrow.14Simple Double Arrow.14Host +PF Application +DPDK +Ethdev +PF driver + diff --git a/dpdk/doc/guides/howto/img/virtio_user_as_exceptional_path.svg b/dpdk/doc/guides/howto/img/virtio_user_as_exceptional_path.svg new file mode 100644 index 00000000..b231b709 --- /dev/null +++ b/dpdk/doc/guides/howto/img/virtio_user_as_exceptional_path.svg @@ -0,0 +1,207 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + Page-1 + + + + Rectangle.23 + + + + + + + + + + Dynamic connector.42 + + + + Rectangle.45 + tap + + + + + + + + + + tap + + Rectangle.46 + vhost ko + + + + + + + + + + vhost ko + + Sheet.47 + Kernel space + + + + Kernel space + + Sheet.48 + User space + + + + User space + + Rectangle.49 + ETHDEV + + + + + + + + + + ETHDEV + + Rectangle.50 + virtio PMD + + + + + + + + + + virtio PMD + + Rectangle.51 + other PMDs + + + + + + + + + + other PMDs + + Rectangle.52 + virtio-user + + + + + + + + + + virtio-user + + Rectangle.53 + vhost adapter + + + + + + + + + + vhost adapter + + Dynamic connector + + + + Dynamic connector.55 + + + + Rectangle.38 + + + + + + + + + + Sheet.57 + NIC + + + + NIC + + Dynamic connector.41 + + + + diff --git a/dpdk/doc/guides/howto/img/virtio_user_for_container_networking.svg b/dpdk/doc/guides/howto/img/virtio_user_for_container_networking.svg new file mode 100644 index 00000000..de808066 --- /dev/null +++ b/dpdk/doc/guides/howto/img/virtio_user_for_container_networking.svg @@ -0,0 +1,685 @@ + +image/svg+xmlPage-1Rectangle.23Rectangle.49ETHDEVethdev +Rectangle.50virtio PMDvirtio PMD +Rectangle.52virtio-user (virtual device)virtio-user(virtual device) +Rectangle.53vhost-user adapterRectangle.38Sheet.57NICNIC +Rectangle.59virtio (PCI device)virtio(PCI device) +Rectangle.60vSwitch or vRoutervSwitchorvRouter +Sheet.61DPDKDPDK +Rectangle.62Sheet.63Contanier/AppContainer/App +Rectangle.64Rectangle.65Sheet.66virtiovirtio +Sheet.67vhostvhost +Dynamic connectorDynamic connector.70Dynamic connector.72Rectangle.71unix socket fileunix socket file +vhost-user +adapter + \ No newline at end of file diff --git a/dpdk/doc/guides/howto/index.rst b/dpdk/doc/guides/howto/index.rst index aa6d3e2b..b5e8ac68 100644 --- a/dpdk/doc/guides/howto/index.rst +++ b/dpdk/doc/guides/howto/index.rst @@ -28,8 +28,8 @@ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -How To User Guides -================== +HowTo Guides +============ .. toctree:: :maxdepth: 2 @@ -38,3 +38,9 @@ How To User Guides lm_bond_virtio_sriov lm_virtio_vhost_user flow_bifurcation + rte_flow + pvp_reference_benchmark + vfd + virtio_user_for_container_networking + virtio_user_as_exceptional_path + packet_capture_framework diff --git a/dpdk/doc/guides/howto/lm_bond_virtio_sriov.rst b/dpdk/doc/guides/howto/lm_bond_virtio_sriov.rst index 49666f13..40dd2cb2 100644 --- a/dpdk/doc/guides/howto/lm_bond_virtio_sriov.rst +++ b/dpdk/doc/guides/howto/lm_bond_virtio_sriov.rst @@ -142,7 +142,7 @@ Bonding is port 2 (P2). .. code-block:: console testpmd> create bonded device 1 0 - Created new bonded device eth_bond_testpmd_0 on (port 2). + Created new bonded device net_bond_testpmd_0 on (port 2). testpmd> add bonding slave 0 2 testpmd> add bonding slave 1 2 testpmd> show bonding config 2 @@ -613,17 +613,17 @@ Set up DPDK in the Virtual Machine cat /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages ifconfig -a - /root/dpdk/tools/dpdk_nic_bind.py --status + /root/dpdk/usertools/dpdk-devbind.py --status rmmod virtio-pci ixgbevf modprobe uio insmod /root/dpdk/x86_64-default-linuxapp-gcc/kmod/igb_uio.ko - /root/dpdk/tools/dpdk_nic_bind.py -b igb_uio 0000:00:03.0 - /root/dpdk/tools/dpdk_nic_bind.py -b igb_uio 0000:00:04.0 + /root/dpdk/usertools/dpdk-devbind.py -b igb_uio 0000:00:03.0 + /root/dpdk/usertools/dpdk-devbind.py -b igb_uio 0000:00:04.0 - /root/dpdk/tools/dpdk_nic_bind.py --status + /root/dpdk/usertools/dpdk-devbind.py --status run_testpmd_bonding_in_vm.sh ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -641,7 +641,7 @@ Run testpmd in the Virtual Machine. # use for bonding of virtio and vf tests in VM /root/dpdk/x86_64-default-linuxapp-gcc/app/testpmd \ - -c f -n 4 --socket-mem 350 -- --i --port-topology=chained + -l 0-3 -n 4 --socket-mem 350 -- --i --port-topology=chained .. _lm_bond_virtio_sriov_switch_conf: diff --git a/dpdk/doc/guides/howto/lm_virtio_vhost_user.rst b/dpdk/doc/guides/howto/lm_virtio_vhost_user.rst index fad1f2a8..9402ed8d 100644 --- a/dpdk/doc/guides/howto/lm_virtio_vhost_user.rst +++ b/dpdk/doc/guides/howto/lm_virtio_vhost_user.rst @@ -90,15 +90,15 @@ For Fortville NIC. .. code-block:: console - cd /root/dpdk/tools - ./dpdk_nic_bind.py -b igb_uio 0000:02:00.0 + cd /root/dpdk/usertools + ./dpdk-devbind.py -b igb_uio 0000:02:00.0 For Niantic NIC. .. code-block:: console - cd /root/dpdk/tools - ./dpdk_nic_bind.py -b igb_uio 0000:09:00.0 + cd /root/dpdk/usertools + ./dpdk-devbind.py -b igb_uio 0000:09:00.0 On host_server_1: Terminal 3 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -171,15 +171,15 @@ For Fortville NIC. .. code-block:: console - cd /root/dpdk/tools - ./dpdk_nic_bind.py -b igb_uio 0000:03:00.0 + cd /root/dpdk/usertools + ./dpdk-devbind.py -b igb_uio 0000:03:00.0 For Niantic NIC. .. code-block:: console - cd /root/dpdk/tools - ./dpdk_nic_bind.py -b igb_uio 0000:06:00.0 + cd /root/dpdk/usertools + ./dpdk-devbind.py -b igb_uio 0000:06:00.0 On host_server_2: Terminal 3 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -444,17 +444,17 @@ setup_dpdk_virtio_in_vm.sh cat /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages ifconfig -a - /root/dpdk/tools/dpdk_nic_bind.py --status + /root/dpdk/usertools/dpdk-devbind.py --status rmmod virtio-pci modprobe uio insmod /root/dpdk/x86_64-default-linuxapp-gcc/kmod/igb_uio.ko - /root/dpdk/tools/dpdk_nic_bind.py -b igb_uio 0000:00:03.0 - /root/dpdk/tools/dpdk_nic_bind.py -b igb_uio 0000:00:04.0 + /root/dpdk/usertools/dpdk-devbind.py -b igb_uio 0000:00:03.0 + /root/dpdk/usertools/dpdk-devbind.py -b igb_uio 0000:00:04.0 - /root/dpdk/tools/dpdk_nic_bind.py --status + /root/dpdk/usertools/dpdk-devbind.py --status run_testpmd_in_vm.sh ~~~~~~~~~~~~~~~~~~~~ @@ -466,4 +466,4 @@ run_testpmd_in_vm.sh # test system has 8 cpus (0-7), use cpus 2-7 for VM /root/dpdk/x86_64-default-linuxapp-gcc/app/testpmd \ - -c 3f -n 4 --socket-mem 350 -- --burst=64 --i --disable-hw-vlan-filter + -l 0-5 -n 4 --socket-mem 350 -- --burst=64 --i --disable-hw-vlan-filter diff --git a/dpdk/doc/guides/howto/packet_capture_framework.rst b/dpdk/doc/guides/howto/packet_capture_framework.rst new file mode 100644 index 00000000..4467442e --- /dev/null +++ b/dpdk/doc/guides/howto/packet_capture_framework.rst @@ -0,0 +1,140 @@ +.. BSD LICENSE + Copyright(c) 2017 Intel Corporation. All rights reserved. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + + +DPDK pdump Library and pdump Tool +================================= + +This document describes how the Data Plane Development Kit (DPDK) Packet +Capture Framework is used for capturing packets on DPDK ports. It is intended +for users of DPDK who want to know more about the Packet Capture feature and +for those who want to monitor traffic on DPDK-controlled devices. + +The DPDK packet capture framework was introduced in DPDK v16.07. The DPDK +packet capture framework consists of the DPDK pdump library and DPDK pdump +tool. + + +Introduction +------------ + +The :ref:`librte_pdump ` library provides the APIs required to +allow users to initialize the packet capture framework and to enable or +disable packet capture. The library works on a client/server model and its +usage is recommended for debugging purposes. + +The :ref:`dpdk-pdump ` tool is developed based on the +``librte_pdump`` library. It runs as a DPDK secondary process and is capable +of enabling or disabling packet capture on DPDK ports. The ``dpdk-pdump`` tool +provides command-line options with which users can request enabling or +disabling of the packet capture on DPDK ports. + +The application which initializes the packet capture framework will act as a +server and the application that enables or disables the packet capture will +act as a client. The server sends the Rx and Tx packets from the DPDK ports +to the client. + +In DPDK the ``testpmd`` application can be used to initialize the packet +capture framework and act as a server, and the ``dpdk-pdump`` tool acts as a +client. To view Rx or Tx packets of ``testpmd``, the application should be +launched first, and then the ``dpdk-pdump`` tool. Packets from ``testpmd`` +will be sent to the tool, which then sends them on to the Pcap PMD device and +that device writes them to the Pcap file or to an external interface depending +on the command-line option used. + +Some things to note: + +* The ``dpdk-pdump`` tool can only be used in conjunction with a primary + application which has the packet capture framework initialized already. In + dpdk, only ``testpmd`` is modified to initialize packet capture framework, + other applications remain untouched. So, if the ``dpdk-pdump`` tool has to + be used with any application other than the testpmd, the user needs to + explicitly modify that application to call the packet capture framework + initialization code. Refer to the ``app/test-pmd/testpmd.c`` code and look + for ``pdump`` keyword to see how this is done. + +* The ``dpdk-pdump`` tool depends on the libpcap based PMD which is disabled + by default in the build configuration files, owing to an external dependency + on the libpcap development files. Once the libpcap development files are + installed, the libpcap based PMD can be enabled by setting + ``CONFIG_RTE_LIBRTE_PMD_PCAP=y`` and recompiling the DPDK. + + +Test Environment +---------------- + +The overview of using the Packet Capture Framework and the ``dpdk-pdump`` tool +for packet capturing on the DPDK port in +:numref:`figure_packet_capture_framework`. + +.. _figure_packet_capture_framework: + +.. figure:: img/packet_capture_framework.* + + Packet capturing on a DPDK port using the dpdk-pdump tool. + + +Configuration +------------- + +Modify the DPDK primary application to initialize the packet capture framework +as mentioned in the above notes and enable the following config options and +build DPDK:: + + CONFIG_RTE_LIBRTE_PMD_PCAP=y + CONFIG_RTE_LIBRTE_PDUMP=y + + +Running the Application +----------------------- + +The following steps demonstrate how to run the ``dpdk-pdump`` tool to capture +Rx side packets on dpdk_port0 in :numref:`figure_packet_capture_framework` and +inspect them using ``tcpdump``. + +#. Launch testpmd as the primary application:: + + sudo ./app/testpmd -c 0xf0 -n 4 -- -i --port-topology=chained + +#. Launch the pdump tool as follows:: + + sudo ./build/app/dpdk-pdump -- \ + --pdump 'port=0,queue=*,rx-dev=/tmp/capture.pcap' + +#. Send traffic to dpdk_port0 from traffic generator. + Inspect packets captured in the file capture.pcap using a tool + that can interpret Pcap files, for example tcpdump:: + + $tcpdump -nr /tmp/capture.pcap + reading from file /tmp/capture.pcap, link-type EN10MB (Ethernet) + 11:11:36.891404 IP 4.4.4.4.whois++ > 3.3.3.3.whois++: UDP, length 18 + 11:11:36.891442 IP 4.4.4.4.whois++ > 3.3.3.3.whois++: UDP, length 18 + 11:11:36.891445 IP 4.4.4.4.whois++ > 3.3.3.3.whois++: UDP, length 18 diff --git a/dpdk/doc/guides/howto/pvp_reference_benchmark.rst b/dpdk/doc/guides/howto/pvp_reference_benchmark.rst new file mode 100644 index 00000000..228b4a25 --- /dev/null +++ b/dpdk/doc/guides/howto/pvp_reference_benchmark.rst @@ -0,0 +1,398 @@ +.. BSD LICENSE + Copyright(c) 2016 Red Hat, Inc. All rights reserved. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +PVP reference benchmark setup using testpmd +=========================================== + +This guide lists the steps required to setup a PVP benchmark using testpmd as +a simple forwarder between NICs and Vhost interfaces. The goal of this setup +is to have a reference PVP benchmark without using external vSwitches (OVS, +VPP, ...) to make it easier to obtain reproducible results and to facilitate +continuous integration testing. + +The guide covers two ways of launching the VM, either by directly calling the +QEMU command line, or by relying on libvirt. It has been tested with DPDK +v16.11 using RHEL7 for both host and guest. + + +Setup overview +-------------- + +.. _figure_pvp_2nics: + +.. figure:: img/pvp_2nics.* + + PVP setup using 2 NICs + +In this diagram, each red arrow represents one logical core. This use-case +requires 6 dedicated logical cores. A forwarding configuration with a single +NIC is also possible, requiring 3 logical cores. + + +Host setup +---------- + +In this setup, we isolate 6 cores (from CPU2 to CPU7) on the same NUMA +node. Two cores are assigned to the VM vCPUs running testpmd and four are +assigned to testpmd on the host. + + +Host tuning +~~~~~~~~~~~ + +#. On BIOS, disable turbo-boost and hyper-threads. + +#. Append these options to Kernel command line: + + .. code-block:: console + + intel_pstate=disable mce=ignore_ce default_hugepagesz=1G hugepagesz=1G hugepages=6 isolcpus=2-7 rcu_nocbs=2-7 nohz_full=2-7 iommu=pt intel_iommu=on + +#. Disable hyper-threads at runtime if necessary or if BIOS is not accessible: + + .. code-block:: console + + cat /sys/devices/system/cpu/cpu*[0-9]/topology/thread_siblings_list \ + | sort | uniq \ + | awk -F, '{system("echo 0 > /sys/devices/system/cpu/cpu"$2"/online")}' + +#. Disable NMIs: + + .. code-block:: console + + echo 0 > /proc/sys/kernel/nmi_watchdog + +#. Exclude isolated CPUs from the writeback cpumask: + + .. code-block:: console + + echo ffffff03 > /sys/bus/workqueue/devices/writeback/cpumask + +#. Isolate CPUs from IRQs: + + .. code-block:: console + + clear_mask=0xfc #Isolate CPU2 to CPU7 from IRQs + for i in /proc/irq/*/smp_affinity + do + echo "obase=16;$(( 0x$(cat $i) & ~$clear_mask ))" | bc > $i + done + + +Qemu build +~~~~~~~~~~ + +Build Qemu: + + .. code-block:: console + + git clone git://git.qemu.org/qemu.git + cd qemu + mkdir bin + cd bin + ../configure --target-list=x86_64-softmmu + make + + +DPDK build +~~~~~~~~~~ + +Build DPDK: + + .. code-block:: console + + git clone git://dpdk.org/dpdk + cd dpdk + export RTE_SDK=$PWD + make install T=x86_64-native-linuxapp-gcc DESTDIR=install + + +Testpmd launch +~~~~~~~~~~~~~~ + +#. Assign NICs to DPDK: + + .. code-block:: console + + modprobe vfio-pci + $RTE_SDK/install/sbin/dpdk-devbind -b vfio-pci 0000:11:00.0 0000:11:00.1 + + .. Note:: + + The Sandy Bridge family seems to have some IOMMU limitations giving poor + performance results. To achieve good performance on these machines + consider using UIO instead. + +#. Launch the testpmd application: + + .. code-block:: console + + $RTE_SDK/install/bin/testpmd -l 0,2,3,4,5 --socket-mem=1024 -n 4 \ + --vdev 'net_vhost0,iface=/tmp/vhost-user1' \ + --vdev 'net_vhost1,iface=/tmp/vhost-user2' -- \ + --portmask=f --disable-hw-vlan -i --rxq=1 --txq=1 \ + --nb-cores=4 --forward-mode=io + + With this command, isolated CPUs 2 to 5 will be used as lcores for PMD threads. + +#. In testpmd interactive mode, set the portlist to obtain the correct port + chaining: + + .. code-block:: console + + set portlist 0,2,1,3 + start + + +VM launch +~~~~~~~~~ + +The VM may be launched either by calling QEMU directly, or by using libvirt. + +Qemu way +^^^^^^^^ + +Launch QEMU with two Virtio-net devices paired to the vhost-user sockets +created by testpmd. Below example uses default Virtio-net options, but options +may be specified, for example to disable mergeable buffers or indirect +descriptors. + + .. code-block:: console + + /bin/x86_64-softmmu/qemu-system-x86_64 \ + -enable-kvm -cpu host -m 3072 -smp 3 \ + -chardev socket,id=char0,path=/tmp/vhost-user1 \ + -netdev type=vhost-user,id=mynet1,chardev=char0,vhostforce \ + -device virtio-net-pci,netdev=mynet1,mac=52:54:00:02:d9:01,addr=0x10 \ + -chardev socket,id=char1,path=/tmp/vhost-user2 \ + -netdev type=vhost-user,id=mynet2,chardev=char1,vhostforce \ + -device virtio-net-pci,netdev=mynet2,mac=52:54:00:02:d9:02,addr=0x11 \ + -object memory-backend-file,id=mem,size=3072M,mem-path=/dev/hugepages,share=on \ + -numa node,memdev=mem -mem-prealloc \ + -net user,hostfwd=tcp::1002$1-:22 -net nic \ + -qmp unix:/tmp/qmp.socket,server,nowait \ + -monitor stdio .qcow2 + +You can use this `qmp-vcpu-pin `_ +script to pin vCPUs. + +It can be used as follows, for example to pin 3 vCPUs to CPUs 1, 6 and 7, +where isolated CPUs 6 and 7 will be used as lcores for Virtio PMDs: + + .. code-block:: console + + export PYTHONPATH=$PYTHONPATH:/scripts/qmp + ./qmp-vcpu-pin -s /tmp/qmp.socket 1 6 7 + +Libvirt way +^^^^^^^^^^^ + +Some initial steps are required for libvirt to be able to connect to testpmd's +sockets. + +First, SELinux policy needs to be set to permissive, since testpmd is +generally run as root (note, as reboot is required): + + .. code-block:: console + + cat /etc/selinux/config + + # This file controls the state of SELinux on the system. + # SELINUX= can take one of these three values: + # enforcing - SELinux security policy is enforced. + # permissive - SELinux prints warnings instead of enforcing. + # disabled - No SELinux policy is loaded. + SELINUX=permissive + + # SELINUXTYPE= can take one of three two values: + # targeted - Targeted processes are protected, + # minimum - Modification of targeted policy. + # Only selected processes are protected. + # mls - Multi Level Security protection. + SELINUXTYPE=targeted + + +Also, Qemu needs to be run as root, which has to be specified in +``/etc/libvirt/qemu.conf``: + + .. code-block:: console + + user = "root" + +Once the domain created, the following snippet is an extract of he most +important information (hugepages, vCPU pinning, Virtio PCI devices): + + .. code-block:: xml + + + 3145728 + 3145728 + + + + + + + 3 + + + + + + + + + + + hvm + + + + + + + + + + + + + + +
+ + + + + + +
+ + + + + +Guest setup +----------- + + +Guest tuning +~~~~~~~~~~~~ + +#. Append these options to the Kernel command line: + + .. code-block:: console + + default_hugepagesz=1G hugepagesz=1G hugepages=1 intel_iommu=on iommu=pt isolcpus=1,2 rcu_nocbs=1,2 nohz_full=1,2 + +#. Disable NMIs: + + .. code-block:: console + + echo 0 > /proc/sys/kernel/nmi_watchdog + +#. Exclude isolated CPU1 and CPU2 from the writeback cpumask: + + .. code-block:: console + + echo 1 > /sys/bus/workqueue/devices/writeback/cpumask + +#. Isolate CPUs from IRQs: + + .. code-block:: console + + clear_mask=0x6 #Isolate CPU1 and CPU2 from IRQs + for i in /proc/irq/*/smp_affinity + do + echo "obase=16;$(( 0x$(cat $i) & ~$clear_mask ))" | bc > $i + done + + +DPDK build +~~~~~~~~~~ + +Build DPDK: + + .. code-block:: console + + git clone git://dpdk.org/dpdk + cd dpdk + export RTE_SDK=$PWD + make install T=x86_64-native-linuxapp-gcc DESTDIR=install + + +Testpmd launch +~~~~~~~~~~~~~~ + +Probe vfio module without iommu: + + .. code-block:: console + + modprobe -r vfio_iommu_type1 + modprobe -r vfio + modprobe vfio enable_unsafe_noiommu_mode=1 + cat /sys/module/vfio/parameters/enable_unsafe_noiommu_mode + modprobe vfio-pci + +Bind the virtio-net devices to DPDK: + + .. code-block:: console + + $RTE_SDK/usertools/dpdk-devbind.py -b vfio-pci 0000:00:10.0 0000:00:11.0 + +Start testpmd: + + .. code-block:: console + + $RTE_SDK/install/bin/testpmd -l 0,1,2 --socket-mem 1024 -n 4 \ + --proc-type auto --file-prefix pg -- \ + --portmask=3 --forward-mode=macswap --port-topology=chained \ + --disable-hw-vlan --disable-rss -i --rxq=1 --txq=1 \ + --rxd=256 --txd=256 --nb-cores=2 --auto-start + +Results template +---------------- + +Below template should be used when sharing results: + + .. code-block:: none + + Traffic Generator: + Acceptable Loss: % + Validation run time: min + Host DPDK version/commit: + Guest DPDK version/commit: + Patches applied: + QEMU version/commit: + Virtio features: + CPU: , + NIC: + Result: Mpps diff --git a/dpdk/doc/guides/howto/rte_flow.rst b/dpdk/doc/guides/howto/rte_flow.rst new file mode 100644 index 00000000..4a27c995 --- /dev/null +++ b/dpdk/doc/guides/howto/rte_flow.rst @@ -0,0 +1,333 @@ +.. BSD LICENSE + Copyright(c) 2017 Mellanox Corporation. All rights reserved. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Mellanox Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +Generic flow API - examples +=========================== + +This document demonstrates some concrete examples for programming flow rules +with the ``rte_flow`` APIs. + +* Detail of the rte_flow APIs can be found in the following link: + :ref:`Generic flow API ` . + +* Details of the TestPMD commands to set the flow rules can be found in the + following link: :ref:`TestPMD Flow rules ` + +Simple IPv4 drop +---------------- + +Description +~~~~~~~~~~~ + +In this example we will create a simple rule that drops packets whose IPv4 +destination equals 192.168.3.2. This code is equivalent to the following +testpmd command (wrapped for clarity):: + + tpmd> flow create 0 ingress pattern eth / vlan / + ipv4 dst is 192.168.3.2 / end actions drop / end + +Code +~~~~ + +.. code-block:: c + + /* create the attribute structure */ + struct rte_flow_attr attr = {.ingress = 1}; + struct rte_flow_item pattern[MAX_PATTERN_IN_FLOW]; + struct rte_flow_action actions[MAX_ACTIONS_IN_FLOW]; + struct rte_flow_item_etc eth; + struct rte_flow_item_vlan vlan; + struct rte_flow_item_ipv4 ipv4; + struct rte_flow *flow; + struct rte_flow_error error; + + /* setting the eth to pass all packets */ + pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH; + pattern[0].spec = ð + + /* set the vlan to pas all packets */ + pattern[1] = RTE_FLOW_ITEM_TYPE_VLAN; + pattern[1].spec = &vlan; + + /* set the dst ipv4 packet to the required value */ + ipv4.hdr.dst_addr = htonl(0xc0a80302); + pattern[2].type = RTE_FLOW_ITEM_TYPE_IPV4; + pattern[2].spec = &ipv4; + + /* end the pattern array */ + pattern[3].type = RTE_FLOW_ITEM)TYPE_END; + + /* create the drop action */ + actions[0].type = RTE_FLOW_ACTION_TYPE_DROP; + actions[1].type = RTE_FLOW_ACTION_TYPE_END; + + /* validate and create the flow rule */ + if (!rte_flow_validate(port_id, &attr, pattern, actions, &error) + flow = rte_flow_create(port_id, &attr, pattern, actions, &error) + +Output +~~~~~~ + +Terminal 1: running sample app with the flow rule disabled:: + + ./filter-program disable + [waiting for packets] + +Terminal 2: running scapy:: + + $scapy + welcome to Scapy + >> sendp(Ether()/Dot1Q()/IP(src='176.80.50.4', dst='192.168.3.1'), \ + iface='some interface', count=1) + >> sendp(Ether()/Dot1Q()/IP(src='176.80.50.5', dst='192.168.3.2'), \ + iface='some interface', count=1) + +Terminal 1: output log:: + + received packet with src ip = 176.80.50.4 + received packet with src ip = 176.80.50.5 + +Terminal 1: running sample the app flow rule enabled:: + + ./filter-program enabled + [waiting for packets] + +Terminal 2: running scapy:: + + $scapy + welcome to Scapy + >> sendp(Ether()/Dot1Q()/IP(src='176.80.50.4', dst='192.168.3.1'), \ + iface='some interface', count=1) + >> sendp(Ether()/Dot1Q()/IP(src='176.80.50.5', dst ='192.168.3.2'), \ + iface='some interface', count=1) + +Terminal 1: output log:: + + received packet with src ip = 176.80.50.4 + +Range IPv4 drop +---------------- + +Description +~~~~~~~~~~~ + +In this example we will create a simple rule that drops packets whose IPv4 +destination is in the range 192.168.3.0 to 192.168.3.255. This is done using +a mask. + +This code is equivalent to the following testpmd command (wrapped for +clarity):: + + tpmd> flow create 0 ingress pattern eth / vlan / + ipv4 dst spec 192.168.3.0 dst mask 255.255.255.0 / + end actions drop / end + +Code +~~~~ + +.. code-block:: c + + struct rte_flow_attr attr = {.ingress = 1}; + struct rte_flow_item pattern[MAX_PATTERN_IN_FLOW]; + struct rte_flow_action actions[MAX_ACTIONS_IN_FLOW]; + struct rte_flow_item_etc eth; + struct rte_flow_item_vlan vlan; + struct rte_flow_item_ipv4 ipv4; + struct rte_flow_item_ipv4 ipv4_mask; + struct rte_flow *flow; + struct rte_flow_error error; + + /* setting the eth to pass all packets */ + pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH; + pattern[0].spec = ð + + /* set the vlan to pas all packets */ + pattern[1] = RTE_FLOW_ITEM_TYPE_VLAN; + pattern[1].spec = &vlan; + + /* set the dst ipv4 packet to the required value */ + ipv4.hdr.dst_addr = htonl(0xc0a80300); + ipv4_mask.hdr.dst_addr = htonl(0xffffff00); + pattern[2].type = RTE_FLOW_ITEM_TYPE_IPV4; + pattern[2].spec = &ipv4; + pattern[2].mask = &ipv4_mask; + + /* end the pattern array */ + pattern[3].type = RTE_FLOW_ITEM)TYPE_END; + + /* create the drop action */ + actions[0].type = RTE_FLOW_ACTION_TYPE_DROP; + actions[1].type = RTE_FLOW_ACTION_TYPE_END; + + /* validate and create the flow rule */ + if (!rte_flow_validate(port_id, &attr, pattern, actions, &error) + flow = rte_flow_create(port_id, &attr, pattern, actions, &error) + +Output +~~~~~~ + +Terminal 1: running sample app flow rule disabled:: + + ./filter-program disable + [waiting for packets] + +Terminal 2: running scapy:: + + $scapy + welcome to Scapy + >> sendp(Ether()/Dot1Q()/IP(src='176.80.50.4', dst='192.168.3.1'), \ + iface='some interface', count=1) + >> sendp(Ether()/Dot1Q()/IP(src='176.80.50.5', dst='192.168.3.2'), \ + iface='some interface', count=1) + >> sendp(Ether()/Dot1Q()/IP(src='176.80.50.6', dst='192.168.5.2'), \ + iface='some interface', count=1) + +Terminal 1: output log:: + + received packet with src ip = 176.80.50.4 + received packet with src ip = 176.80.50.5 + received packet with src ip = 176.80.50.6 + +Terminal 1: running sample app flow rule enabled:: + + ./filter-program enabled + [waiting for packets] + +Terminal 2: running scapy:: + + $scapy + welcome to Scapy + >> sendp(Ether()/Dot1Q()/IP(src='176.80.50.4', dst='192.168.3.1'), \ + iface='some interface', count=1) + >> sendp(Ether()/Dot1Q()/IP(src='176.80.50.5', dst='192.168.3.2'), \ + iface='some interface', count=1) + >> sendp(Ether()/Dot1Q()/IP(src='176.80.50.6', dst='192.168.5.2'), \ + iface='some interface', count=1) + +Terminal 1: output log:: + + received packet with src ip = 176.80.50.6 + +Send vlan to queue +------------------ + +Description +~~~~~~~~~~~ + +In this example we will create a rule that routes all vlan id 123 to queue 3. + +This code is equivalent to the following testpmd command (wrapped for +clarity):: + + tpmd> flow create 0 ingress pattern eth / vlan vid spec 123 / + end actions queue index 3 / end + +Code +~~~~ + +.. code-block:: c + + struct rte_flow_attr attr = {.ingress = 1}; + struct rte_flow_item pattern[MAX_PATTERN_IN_FLOW]; + struct rte_flow_action actions[MAX_ACTIONS_IN_FLOW]; + struct rte_flow_item_etc eth; + struct rte_flow_item_vlan vlan; + struct rte_flow_action_queue queue = { .index = 3 }; + struct rte_flow *flow; + struct rte_flow_error error; + + /* setting the eth to pass all packets */ + pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH; + pattern[0].spec = ð + + /* set the vlan to pas all packets */ + vlan.vid = 123; + pattern[1] = RTE_FLOW_ITEM_TYPE_VLAN; + pattern[1].spec = &vlan; + + /* end the pattern array */ + pattern[2].type = RTE_FLOW_ITEM)TYPE_END; + + /* create the drop action */ + actions[0].type = RTE_FLOW_ACTION_TYPE_QUEUE; + actions[0].conf = &queue + actions[1].type = RTE_FLOW_ACTION_TYPE_END; + + /* validate and create the flow rule */ + if (!rte_flow_validate(port_id, &attr, pattern, actions, &error) + flow = rte_flow_create(port_id, &attr, pattern, actions, &error) + +Output +~~~~~~ + +Terminal 1: running sample app flow rule disabled:: + + ./filter-program disable + [waiting for packets] + +Terminal 2: running scapy:: + + $scapy + welcome to Scapy + >> sendp(Ether()/Dot1Q(vlan=123)/IP(src='176.80.50.4', dst='192.168.3.1'), \ + iface='some interface', count=1) + >> sendp(Ether()/Dot1Q(vlan=50)/IP(src='176.80.50.5', dst='192.168.3.2'), \ + iface='some interface', count=1) + >> sendp(Ether()/Dot1Q(vlan=123)/IP(src='176.80.50.6', dst='192.168.5.2'), \ + iface='some interface', count=1) + +Terminal 1: output log:: + + received packet with src ip = 176.80.50.4 sent to queue 2 + received packet with src ip = 176.80.50.5 sent to queue 1 + received packet with src ip = 176.80.50.6 sent to queue 0 + +Terminal 1: running sample app flow rule enabled:: + + ./filter-program enabled + [waiting for packets] + +Terminal 2: running scapy:: + + $scapy + welcome to Scapy + >> sendp(Ether()/Dot1Q(vlan=123)/IP(src='176.80.50.4', dst='192.168.3.1'), \ + iface='some interface', count=1) + >> sendp(Ether()/Dot1Q(vlan=50)/IP(src='176.80.50.5', dst='192.168.3.2'), \ + iface='some interface', count=1) + >> sendp(Ether()/Dot1Q(vlan=123)/IP(src='176.80.50.6', dst='192.168.5.2'), \ + iface='some interface', count=1) + +Terminal 1: output log:: + + received packet with src ip = 176.80.50.4 sent to queue 3 + received packet with src ip = 176.80.50.5 sent to queue 1 + received packet with src ip = 176.80.50.6 sent to queue 3 diff --git a/dpdk/doc/guides/howto/vfd.rst b/dpdk/doc/guides/howto/vfd.rst new file mode 100644 index 00000000..6f083b87 --- /dev/null +++ b/dpdk/doc/guides/howto/vfd.rst @@ -0,0 +1,407 @@ +.. BSD LICENSE + Copyright(c) 2017 Intel Corporation. All rights reserved. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +VF daemon (VFd) +=============== + +VFd (the VF daemon) is a mechanism which can be used to configure features on +a VF (SR-IOV Virtual Function) without direct access to the PF (SR-IOV +Physical Function). VFd is an *EXPERIMENTAL* feature which can only be used in +the scenario of DPDK PF with a DPDK VF. If the PF port is driven by the Linux +kernel driver then the VFd feature will not work. Currently VFd is only +supported by the ixgbe and i40e drivers. + +In general VF features cannot be configured directly by an end user +application since they are under the control of the PF. The normal approach to +configuring a feature on a VF is that an application would call the APIs +provided by the VF driver. If the required feature cannot be configured by the +VF directly (the most common case) the VF sends a message to the PF through +the mailbox on ixgbe and i40e. This means that the availability of the feature +depends on whether the appropriate mailbox messages are defined. + +DPDK leverages the mailbox interface defined by the Linux kernel driver so +that compatibility with the kernel driver can be guaranteed. The downside of +this approach is that the availability of messages supported by the kernel +become a limitation when the user wants to configure features on the VF. + +VFd is a new method of controlling the features on a VF. The VF driver doesn't +talk directly to the PF driver when configuring a feature on the VF. When a VF +application (i.e., an application using the VF ports) wants to enable a VF +feature, it can send a message to the PF application (i.e., the application +using the PF port, which can be the same as the VF application). The PF +application will configure the feature for the VF. Obviously, the PF +application can also configure the VF features without a request from the VF +application. + +.. _VF_daemon_overview: + +.. figure:: img/vf_daemon_overview.* + + VF daemon (VFd) Overview + +Compared with the traditional approach the VFd moves the negotiation between +VF and PF from the driver level to application level. So the application +should define how the negotiation between the VF and PF works, or even if the +control should be limited to the PF. + +It is the application's responsibility to use VFd. Consider for example a KVM +migration, the VF application may transfer from one VM to another. It is +recommended in this case that the PF control the VF features without +participation from the VF. Then the VF application has no capability to +configure the features. So the user doesn't need to define the interface +between the VF application and the PF application. The service provider should +take the control of all the features. + +The following sections describe the VFd functionality. + +.. Note:: + + Although VFd is supported by both ixgbe and i40e, please be aware that + since the hardware capability is different, the functions supported by + ixgbe and i40e are not the same. + + +Preparing +--------- + +VFd only can be used in the scenario of DPDK PF + DPDK VF. Users should bind +the PF port to ``igb_uio``, then create the VFs based on the DPDK PF host. + +The typical procedure to achieve this is as follows: + +#. Boot the system without iommu, or with ``iommu=pt``. + +#. Bind the PF port to ``igb_uio``, for example:: + + dpdk-devbind.py -b igb_uio 01:00.0 + +#. Create a Virtual Function:: + + echo 1 > /sys/bus/pci/devices/0000:01:00.0/max_vfs + +#. Start a VM with the new VF port bypassed to it. + +#. Run a DPDK application on the PF in the host:: + + testpmd -l 0-7 -n 4 -- -i --txqflags=0 + +#. Bind the VF port to ``igb_uio`` in the VM:: + + dpdk-devbind.py -b igb_uio 03:00.0 + +#. Run a DPDK application on the VF in the VM:: + + testpmd -l 0-7 -n 4 -- -i --txqflags=0 + + +Common functions of IXGBE and I40E +---------------------------------- + +The following sections show how to enable PF/VF functionality based on the +above testpmd setup. + + +TX loopback +~~~~~~~~~~~ + +Run a testpmd runtime command on the PF to set TX loopback:: + + set tx loopback 0 on|off + +This sets whether the PF port and all the VF ports that belong to it are +allowed to send the packets to other virtual ports. + +Although it is a VFd function, it is the global setting for the whole +physical port. When using this function, the PF and all the VFs TX loopback +will be enabled/disabled. + + +VF MAC address setting +~~~~~~~~~~~~~~~~~~~~~~ + +Run a testpmd runtime command on the PF to set the MAC address for a VF port:: + + set vf mac addr 0 0 A0:36:9F:7B:C3:51 + +This testpmd runtime command will change the MAC address of the VF port to +this new address. If any other addresses are set before, they will be +overwritten. + + +VF MAC anti-spoofing +~~~~~~~~~~~~~~~~~~~~ + +Run a testpmd runtime command on the PF to enable/disable the MAC +anti-spoofing for a VF port:: + + set vf mac antispoof 0 0 on|off + +When enabling the MAC anti-spoofing, the port will not forward packets whose +source MAC address is not the same as the port. + + +VF VLAN anti-spoofing +~~~~~~~~~~~~~~~~~~~~~ + +Run a testpmd runtime command on the PF to enable/disable the VLAN +anti-spoofing for a VF port:: + + set vf vlan antispoof 0 0 on|off + +When enabling the VLAN anti-spoofing, the port will not send packets whose +VLAN ID does not belong to VLAN IDs that this port can receive. + + +VF VLAN insertion +~~~~~~~~~~~~~~~~~ + +Run a testpmd runtime command on the PF to set the VLAN insertion for a VF +port:: + + set vf vlan insert 0 0 1 + +When using this testpmd runtime command, an assigned VLAN ID can be inserted +to the transmitted packets by the hardware. + +The assigned VLAN ID can be 0. It means disabling the VLAN insertion. + + +VF VLAN stripping +~~~~~~~~~~~~~~~~~ + +Run a testpmd runtime command on the PF to enable/disable the VLAN stripping +for a VF port:: + + set vf vlan stripq 0 0 on|off + +This testpmd runtime command is used to enable/disable the RX VLAN stripping +for a specific VF port. + + +VF VLAN filtering +~~~~~~~~~~~~~~~~~ + +Run a testpmd runtime command on the PF to set the VLAN filtering for a VF +port:: + + rx_vlan add 1 port 0 vf 1 + rx_vlan rm 1 port 0 vf 1 + +These two testpmd runtime commands can be used to add or remove the VLAN +filter for several VF ports. When the VLAN filters are added only the packets +that have the assigned VLAN IDs can be received. Other packets will be dropped +by hardware. + + +The IXGBE specific VFd functions +-------------------------------- + +The functions in this section are specific to the ixgbe driver. + + +All queues drop +~~~~~~~~~~~~~~~ + +Run a testpmd runtime command on the PF to enable/disable the all queues +drop:: + + set all queues drop on|off + +This is a global setting for the PF and all the VF ports of the physical port. + +Enabling the ``all queues drop`` feature means that when there is no available +descriptor for the received packets they are dropped. The ``all queues drop`` +feature should be enabled in SR-IOV mode to avoid one queue blocking others. + + +VF packet drop +~~~~~~~~~~~~~~ + +Run a testpmd runtime command on the PF to enable/disable the packet drop for +a specific VF:: + + set vf split drop 0 0 on|off + +This is a similar function as ``all queues drop``. The difference is that this +function is per VF setting and the previous function is a global setting. + + +VF rate limit +~~~~~~~~~~~~~ + +Run a testpmd runtime command on the PF to all queues' rate limit for a +specific VF:: + + set port 0 vf 0 rate 10 queue_mask 1 + +This is a function to set the rate limit for all the queues in the +``queue_mask`` bitmap. It is not used to set the summary of the rate +limit. The rate limit of every queue will be set equally to the assigned rate +limit. + + +VF RX enabling +~~~~~~~~~~~~~~ + +Run a testpmd runtime command on the PF to enable/disable packet receiving for +a specific VF:: + + set port 0 vf 0 rx on|off + +This function can be used to stop/start packet receiving on a VF. + + +VF TX enabling +~~~~~~~~~~~~~~ + +Run a testpmd runtime command on the PF to enable/disable packet transmitting +for a specific VF:: + + set port 0 vf 0 tx on|off + +This function can be used to stop/start packet transmitting on a VF. + + +VF RX mode setting +~~~~~~~~~~~~~~~~~~ + +Run a testpmd runtime command on the PF to set the RX mode for a specific VF:: + + set port 0 vf 0 rxmode AUPE|ROPE|BAM|MPE on|off + +This function can be used to enable/disable some RX modes on the VF, including: + +* If it accept untagged packets. +* If it accepts packets matching the MAC filters. +* If it accept MAC broadcast packets, +* If it enables MAC multicast promiscuous mode. + + +The I40E specific VFd functions +------------------------------- + +The functions in this section are specific to the i40e driver. + + +VF statistics +~~~~~~~~~~~~~ + +This provides an API to get the a specific VF's statistic from PF. + + +VF statistics resetting +~~~~~~~~~~~~~~~~~~~~~~~ + +This provides an API to rest the a specific VF's statistic from PF. + + +VF link status change notification +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This provide an API to let a specific VF know if the physical link status +changed. + +Normally if a VF received this notification, the driver should notify the +application to reset the VF port. + + +VF MAC broadcast setting +~~~~~~~~~~~~~~~~~~~~~~~~ + +Run a testpmd runtime command on the PF to enable/disable MAC broadcast packet +receiving for a specific VF:: + + set vf broadcast 0 0 on|off + + +VF MAC multicast promiscuous mode +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Run a testpmd runtime command on the PF to enable/disable MAC multicast +promiscuous mode for a specific VF:: + + set vf allmulti 0 0 on|off + + +VF MAC unicast promiscuous mode +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Run a testpmd runtime command on the PF to enable/disable MAC unicast +promiscuous mode for a specific VF:: + + set vf promisc 0 0 on|off + + +VF max bandwidth +~~~~~~~~~~~~~~~~ + +Run a testpmd runtime command on the PF to set the TX maximum bandwidth for a +specific VF:: + + set vf tx max-bandwidth 0 0 2000 + +The maximum bandwidth is an absolute value in Mbps. + + +VF TC bandwidth allocation +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Run a testpmd runtime command on the PF to set the TCs (traffic class) TX +bandwidth allocation for a specific VF:: + + set vf tc tx min-bandwidth 0 0 (20,20,20,40) + +The allocated bandwidth should be set for all the TCs. The allocated bandwidth +is a relative value as a percentage. The sum of all the bandwidth should +be 100. + + +VF TC max bandwidth +~~~~~~~~~~~~~~~~~~~ + +Run a testpmd runtime command on the PF to set the TCs TX maximum bandwidth +for a specific VF:: + + set vf tc tx max-bandwidth 0 0 0 10000 + +The maximum bandwidth is an absolute value in Mbps. + + +TC strict priority scheduling +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Run a testpmd runtime command on the PF to enable/disable several TCs TX +strict priority scheduling:: + + set tx strict-link-priority 0 0x3 + +The 0 in the TC bitmap means disabling the strict priority scheduling for this +TC. To enable use a value of 1. diff --git a/dpdk/doc/guides/howto/virtio_user_as_exceptional_path.rst b/dpdk/doc/guides/howto/virtio_user_as_exceptional_path.rst new file mode 100644 index 00000000..3f99fe82 --- /dev/null +++ b/dpdk/doc/guides/howto/virtio_user_as_exceptional_path.rst @@ -0,0 +1,142 @@ +.. BSD LICENSE + Copyright(c) 2016 Intel Corporation. All rights reserved. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +.. _virtio_user_as_excpetional_path: + +Virtio_user as Exceptional Path +=============================== + +The virtual device, virtio-user, was originally introduced with vhost-user +backend, as a high performance solution for IPC (Inter-Process Communication) +and user space container networking. + +Virtio_user with vhost-kernel backend is a solution for exceptional path, +such as KNI which exchanges packets with kernel networking stack. This +solution is very promising in: + +* Maintenance + + All kernel modules needed by this solution, vhost and vhost-net (kernel), + are upstreamed and extensively used kernel module. + +* Features + + vhost-net is born to be a networking solution, which has lots of networking + related featuers, like multi queue, tso, multi-seg mbuf, etc. + +* Performance + + similar to KNI, this solution would use one or more kthreads to + send/receive packets to/from user space DPDK applications, which has little + impact on user space polling thread (except that it might enter into kernel + space to wake up those kthreads if necessary). + +The overview of an application using virtio-user as exceptional path is shown +in :numref:`figure_virtio_user_as_exceptional_path`. + +.. _figure_virtio_user_as_exceptional_path: + +.. figure:: img/virtio_user_as_exceptional_path.* + + Overview of a DPDK app using virtio-user as excpetional path + + +Sample Usage +------------ + +As a prerequisite, the vhost/vhost-net kernel CONFIG should be chosen before +compiling the kernel and those kernel modules should be inserted. + +#. Compile DPDK and bind a physical NIC to igb_uio/uio_pci_generic/vfio-pci. + + This physical NIC is for communicating with outside. + +#. Run testpmd. + + .. code-block:: console + + $(testpmd) -l 2-3 -n 4 \ + --vdev=virtio_user0,path=/dev/vhost-net,queue_size=1024 \ + -- -i --txqflags=0x0 --disable-hw-vlan --enable-lro \ + --enable-rx-cksum --rxd=1024 --txd=1024 + + This command runs testpmd with two ports, one physical NIC to communicate + with outside, and one virtio-user to communicate with kernel. + +* ``--enable-lro`` + + This is used to negotiate VIRTIO_NET_F_GUEST_TSO4 and + VIRTIO_NET_F_GUEST_TSO6 feature so that large packets from kernel can be + transmitted to DPDK application and further TSOed by physical NIC. + +* ``--enable-rx-cksum`` + + This is used to negotiate VIRTIO_NET_F_GUEST_CSUM so that packets from + kernel can be deemed as valid Rx checksumed. + +* ``queue_size`` + + 256 by default. To avoid shortage of descriptors, we can increase it to 1024. + +* ``queues`` + + Number of multi-queues. Each qeueue will be served by a kthread. For example: + + .. code-block:: console + + $(testpmd) -l 2-3 -n 4 \ + --vdev=virtio_user0,path=/dev/vhost-net,queues=2,queue_size=1024 \ + -- -i --txqflags=0x0 --disable-hw-vlan --enable-lro \ + --enable-rx-cksum --txq=2 --rxq=2 --rxd=1024 \ + --txd=1024 + +#. Start testpmd: + + .. code-block:: console + + (testpmd) start + +#. Configure IP address and start tap: + + .. code-block:: console + + ifconfig tap0 1.1.1.1/24 up + +.. note:: + + The tap device will be named tap0, tap1, etc, by kernel. + +Then, all traffic from physical NIC can be forwarded into kernel stack, and all +traffic on the tap0 can be sent out from physical NIC. + +Limitations +----------- + +This solution is only available on Linux systems. diff --git a/dpdk/doc/guides/howto/virtio_user_for_container_networking.rst b/dpdk/doc/guides/howto/virtio_user_for_container_networking.rst new file mode 100644 index 00000000..f71d0718 --- /dev/null +++ b/dpdk/doc/guides/howto/virtio_user_for_container_networking.rst @@ -0,0 +1,144 @@ +.. BSD LICENSE + Copyright(c) 2016 Intel Corporation. All rights reserved. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +.. _virtio_user_for_container_networking: + +Virtio_user for Container Networking +==================================== + +Container becomes more and more popular for strengths, like low overhead, fast +boot-up time, and easy to deploy, etc. How to use DPDK to accelerate container +networking becomes a common question for users. There are two use models of +running DPDK inside containers, as shown in +:numref:`figure_use_models_for_running_dpdk_in_containers`. + +.. _figure_use_models_for_running_dpdk_in_containers: + +.. figure:: img/use_models_for_running_dpdk_in_containers.* + + Use models of running DPDK inside container + +This page will only cover aggregation model. + +Overview +-------- + +The virtual device, virtio-user, with unmodified vhost-user backend, is designed +for high performance user space container networking or inter-process +communication (IPC). + +The overview of accelerating container networking by virtio-user is shown +in :numref:`figure_virtio_user_for_container_networking`. + +.. _figure_virtio_user_for_container_networking: + +.. figure:: img/virtio_user_for_container_networking.* + + Overview of accelerating container networking by virtio-user + +Different virtio PCI devices we usually use as a para-virtualization I/O in the +context of QEMU/VM, the basic idea here is to present a kind of virtual devices, +which can be attached and initialized by DPDK. The device emulation layer by +QEMU in VM's context is saved by just registering a new kind of virtual device +in DPDK's ether layer. And to minimize the change, we reuse already-existing +virtio PMD code (driver/net/virtio/). + +Virtio, in essence, is a shm-based solution to transmit/receive packets. How is +memory shared? In VM's case, qemu always shares the whole physical layout of VM +to vhost backend. But it's not feasible for a container, as a process, to share +all virtual memory regions to backend. So only those virtual memory regions +(aka, hugepages initialized in DPDK) are sent to backend. It restricts that only +addresses in these areas can be used to transmit or receive packets. + +Sample Usage +------------ + +Here we use Docker as container engine. It also applies to LXC, Rocket with +some minor changes. + +#. Compile DPDK. + + .. code-block:: console + + make install RTE_SDK=`pwd` T=x86_64-native-linuxapp-gcc + +#. Write a Dockerfile like below. + + .. code-block:: console + + cat <> Dockerfile + FROM ubuntu:latest + WORKDIR /usr/src/dpdk + COPY . /usr/src/dpdk + ENV PATH "$PATH:/usr/src/dpdk/x86_64-native-linuxapp-gcc/app/" + EOT + +#. Build a Docker image. + + .. code-block:: console + + docker build -t dpdk-app-testpmd . + +#. Start a testpmd on the host with a vhost-user port. + + .. code-block:: console + + $(testpmd) -l 0-1 -n 4 --socket-mem 1024,1024 \ + --vdev 'eth_vhost0,iface=/tmp/sock0' \ + --file-prefix=host --no-pci -- -i + +#. Start a container instance with a virtio-user port. + + .. code-block:: console + + docker run -i -t -v /tmp/sock0:/var/run/usvhost \ + -v /dev/hugepages:/dev/hugepages \ + dpdk-app-testpmd testpmd -l 6-7 -n 4 -m 1024 --no-pci \ + --vdev=virtio_user0,path=/var/run/usvhost \ + --file-prefix=container \ + -- -i --txqflags=0xf00 --disable-hw-vlan + +Note: If we run all above setup on the host, it's a shm-based IPC. + +Limitations +----------- + +We have below limitations in this solution: + * Cannot work with --huge-unlink option. As we need to reopen the hugepage + file to share with vhost backend. + * Cannot work with --no-huge option. Currently, DPDK uses anonymous mapping + under this option which cannot be reopened to share with vhost backend. + * Cannot work when there are more than VHOST_MEMORY_MAX_NREGIONS(8) hugepages. + In another word, do not use 2MB hugepage so far. + * Applications should not use file name like HUGEFILE_FMT ("%smap_%d"). That + will bring confusion when sharing hugepage files with backend by name. + * Root privilege is a must. DPDK resolves physical addresses of hugepages + which seems not necessary, and some discussions are going on to remove this + restriction. diff --git a/dpdk/doc/guides/index.rst b/dpdk/doc/guides/index.rst index 04418599..f924a7cb 100644 --- a/dpdk/doc/guides/index.rst +++ b/dpdk/doc/guides/index.rst @@ -36,13 +36,16 @@ DPDK documentation linux_gsg/index freebsd_gsg/index - xen/index + sample_app_ug/index prog_guide/index + howto/index + tools/index + testpmd_app_ug/index nics/index cryptodevs/index - sample_app_ug/index - testpmd_app_ug/index - faq/index - howto/index - rel_notes/index + eventdevs/index + mempool/index + platform/index contributing/index + rel_notes/index + faq/index diff --git a/dpdk/doc/guides/linux_gsg/build_dpdk.rst b/dpdk/doc/guides/linux_gsg/build_dpdk.rst index f8007b31..5452041f 100644 --- a/dpdk/doc/guides/linux_gsg/build_dpdk.rst +++ b/dpdk/doc/guides/linux_gsg/build_dpdk.rst @@ -45,12 +45,8 @@ First, uncompress the archive and move to the uncompressed DPDK source directory .. code-block:: console - unzip DPDK-.zip - cd DPDK- - - ls - app/ config/ examples/ lib/ LICENSE.GPL LICENSE.LGPL Makefile - mk/ scripts/ tools/ + tar xJf dpdk-.tar.xz + cd dpdk- The DPDK is composed of several directories: @@ -62,7 +58,7 @@ The DPDK is composed of several directories: * examples: Source code of DPDK application examples -* config, tools, scripts, mk: Framework-related makefiles, scripts and configuration +* config, buildtools, mk: Framework-related makefiles, scripts and configuration Installation of DPDK Target Environments ---------------------------------------- @@ -73,9 +69,9 @@ The format of a DPDK target is:: where: -* ``ARCH`` can be: ``i686``, ``x86_64``, ``ppc_64`` +* ``ARCH`` can be: ``i686``, ``x86_64``, ``ppc_64``, ``arm64`` -* ``MACHINE`` can be: ``native``, ``ivshmem``, ``power8`` +* ``MACHINE`` can be: ``native``, ``power8``, ``armv8a`` * ``EXECENV`` can be: ``linuxapp``, ``bsdapp`` @@ -147,136 +143,3 @@ Browsing the Installed DPDK Environment Target Once a target is created it contains all libraries, including poll-mode drivers, and header files for the DPDK environment that are required to build customer applications. In addition, the test and testpmd applications are built under the build/app directory, which may be used for testing. A kmod directory is also present that contains kernel modules which may be loaded if needed. - -.. code-block:: console - - ls x86_64-native-linuxapp-gcc - - app build include kmod lib Makefile - -Loading Modules to Enable Userspace IO for DPDK ------------------------------------------------ - -To run any DPDK application, a suitable uio module can be loaded into the running kernel. -In many cases, the standard ``uio_pci_generic`` module included in the Linux kernel -can provide the uio capability. This module can be loaded using the command - -.. code-block:: console - - sudo modprobe uio_pci_generic - -As an alternative to the ``uio_pci_generic``, the DPDK also includes the igb_uio -module which can be found in the kmod subdirectory referred to above. It can -be loaded as shown below: - -.. code-block:: console - - sudo modprobe uio - sudo insmod kmod/igb_uio.ko - -.. note:: - - For some devices which lack support for legacy interrupts, e.g. virtual function - (VF) devices, the ``igb_uio`` module may be needed in place of ``uio_pci_generic``. - -Since DPDK release 1.7 onward provides VFIO support, use of UIO is optional -for platforms that support using VFIO. - -Loading VFIO Module -------------------- - -To run an DPDK application and make use of VFIO, the ``vfio-pci`` module must be loaded: - -.. code-block:: console - - sudo modprobe vfio-pci - -Note that in order to use VFIO, your kernel must support it. -VFIO kernel modules have been included in the Linux kernel since version 3.6.0 and are usually present by default, -however please consult your distributions documentation to make sure that is the case. - -Also, to use VFIO, both kernel and BIOS must support and be configured to use IO virtualization (such as Intel® VT-d). - -For proper operation of VFIO when running DPDK applications as a non-privileged user, correct permissions should also be set up. -This can be done by using the DPDK setup script (called dpdk-setup.sh and located in the tools directory). - -.. _linux_gsg_binding_kernel: - -Binding and Unbinding Network Ports to/from the Kernel Modules --------------------------------------------------------------- - -As of release 1.4, DPDK applications no longer automatically unbind all supported network ports from the kernel driver in use. -Instead, all ports that are to be used by an DPDK application must be bound to the -``uio_pci_generic``, ``igb_uio`` or ``vfio-pci`` module before the application is run. -Any network ports under Linux* control will be ignored by the DPDK poll-mode drivers and cannot be used by the application. - -.. warning:: - - The DPDK will, by default, no longer automatically unbind network ports from the kernel driver at startup. - Any ports to be used by an DPDK application must be unbound from Linux* control and - bound to the ``uio_pci_generic``, ``igb_uio`` or ``vfio-pci`` module before the application is run. - -To bind ports to the ``uio_pci_generic``, ``igb_uio`` or ``vfio-pci`` module for DPDK use, -and then subsequently return ports to Linux* control, -a utility script called dpdk_nic _bind.py is provided in the tools subdirectory. -This utility can be used to provide a view of the current state of the network ports on the system, -and to bind and unbind those ports from the different kernel modules, including the uio and vfio modules. -The following are some examples of how the script can be used. -A full description of the script and its parameters can be obtained by calling the script with the ``--help`` or ``--usage`` options. -Note that the uio or vfio kernel modules to be used, should be loaded into the kernel before -running the ``dpdk-devbind.py`` script. - -.. warning:: - - Due to the way VFIO works, there are certain limitations to which devices can be used with VFIO. - Mainly it comes down to how IOMMU groups work. - Any Virtual Function device can be used with VFIO on its own, but physical devices will require either all ports bound to VFIO, - or some of them bound to VFIO while others not being bound to anything at all. - - If your device is behind a PCI-to-PCI bridge, the bridge will then be part of the IOMMU group in which your device is in. - Therefore, the bridge driver should also be unbound from the bridge PCI device for VFIO to work with devices behind the bridge. - -.. warning:: - - While any user can run the dpdk-devbind.py script to view the status of the network ports, - binding or unbinding network ports requires root privileges. - -To see the status of all network ports on the system: - -.. code-block:: console - - ./tools/dpdk-devbind.py --status - - Network devices using DPDK-compatible driver - ============================================ - 0000:82:00.0 '82599EB 10-GbE NIC' drv=uio_pci_generic unused=ixgbe - 0000:82:00.1 '82599EB 10-GbE NIC' drv=uio_pci_generic unused=ixgbe - - Network devices using kernel driver - =================================== - 0000:04:00.0 'I350 1-GbE NIC' if=em0 drv=igb unused=uio_pci_generic *Active* - 0000:04:00.1 'I350 1-GbE NIC' if=eth1 drv=igb unused=uio_pci_generic - 0000:04:00.2 'I350 1-GbE NIC' if=eth2 drv=igb unused=uio_pci_generic - 0000:04:00.3 'I350 1-GbE NIC' if=eth3 drv=igb unused=uio_pci_generic - - Other network devices - ===================== - - -To bind device ``eth1``,``04:00.1``, to the ``uio_pci_generic`` driver: - -.. code-block:: console - - ./tools/dpdk-devbind.py --bind=uio_pci_generic 04:00.1 - -or, alternatively, - -.. code-block:: console - - ./tools/dpdk-devbind.py --bind=uio_pci_generic eth1 - -To restore device ``82:00.0`` to its original kernel binding: - -.. code-block:: console - - ./tools/dpdk-devbind.py --bind=ixgbe 82:00.0 diff --git a/dpdk/doc/guides/linux_gsg/build_sample_apps.rst b/dpdk/doc/guides/linux_gsg/build_sample_apps.rst index e53bd517..594cbc59 100644 --- a/dpdk/doc/guides/linux_gsg/build_sample_apps.rst +++ b/dpdk/doc/guides/linux_gsg/build_sample_apps.rst @@ -98,12 +98,14 @@ Running a Sample Application .. warning:: - The UIO drivers and hugepages must be setup prior to running an application. + Before running the application make sure: -.. warning:: + - Hugepages setup is done. + - Any kernel driver being used is loaded. + - In case needed, ports being used by the application should be + bound to the corresponding kernel driver. - Any ports to be used by the application must be already bound to an appropriate kernel - module, as described in :ref:`linux_gsg_binding_kernel`, prior to running the application. + refer to :ref:`linux_gsg_linux_drivers` for more details. The application is linked with the DPDK target environment's Environmental Abstraction Layer (EAL) library, which provides some options that are generic to every DPDK application. @@ -112,15 +114,16 @@ The following is the list of options that can be given to the EAL: .. code-block:: console - ./rte-app -c COREMASK [-n NUM] [-b ] \ - [--socket-mem=MB,...] [-m MB] [-r NUM] [-v] [--file-prefix] \ - [--proc-type ] [-- xen-dom0] + ./rte-app [-c COREMASK | -l CORELIST] [-n NUM] [-b ] \ + [--socket-mem=MB,...] [-d LIB.so|DIR] [-m MB] [-r NUM] [-v] [--file-prefix] \ + [--proc-type ] The EAL options are as follows: -* ``-c COREMASK``: +* ``-c COREMASK`` or ``-l CORELIST``: An hexadecimal bit mask of the cores to run on. Note that core numbering can - change between platforms and should be determined beforehand. + change between platforms and should be determined beforehand. The corelist is + a set of core numbers instead of a bitmap core mask. * ``-n NUM``: Number of memory channels per processor socket. @@ -136,6 +139,11 @@ The EAL options are as follows: * ``--socket-mem``: Memory to allocate from hugepages on specific sockets. +* ``-d``: + Add a driver or driver directory to be loaded. + The application should use this option to load the pmd drivers + that are built as shared libraries. + * ``-m MB``: Memory to allocate from hugepages, regardless of processor socket. It is recommended that ``--socket-mem`` be used instead of this option. @@ -149,15 +157,15 @@ The EAL options are as follows: * ``--huge-dir``: The directory where hugetlbfs is mounted. +* ``mbuf-pool-ops-name``: + Pool ops name for mbuf to use. + * ``--file-prefix``: The prefix text used for hugepage filenames. * ``--proc-type``: The type of process instance. -* ``--xen-dom0``: - Support application running on Xen Domain0 without hugetlbfs. - * ``--vmware-tsc-map``: Use VMware TSC map instead of native RDTSC. @@ -167,13 +175,13 @@ The EAL options are as follows: * ``--vfio-intr``: Specify interrupt type to be used by VFIO (has no effect if VFIO is not used). -The ``-c`` and option is mandatory; the others are optional. +The ``-c`` or ``-l`` and option is mandatory; the others are optional. Copy the DPDK application binary to your target, then run the application as follows (assuming the platform has four memory channels per processor socket, and that cores 0-3 are present and are to be used for running the application):: - ./helloworld -c f -n 4 + ./helloworld -l 0-3 -n 4 .. note:: @@ -185,10 +193,10 @@ and that cores 0-3 are present and are to be used for running the application):: Logical Core Use by Applications ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The coremask parameter is always mandatory for DPDK applications. -Each bit of the mask corresponds to the equivalent logical core number as reported by Linux. +The coremask (-c 0x0f) or corelist (-l 0-3) parameter is always mandatory for DPDK applications. +Each bit of the mask corresponds to the equivalent logical core number as reported by Linux. The preferred corelist option is a cleaner method to define cores to be used. Since these logical core numbers, and their mapping to specific cores on specific NUMA sockets, can vary from platform to platform, -it is recommended that the core layout for each platform be considered when choosing the coremask to use in each case. +it is recommended that the core layout for each platform be considered when choosing the coremask/corelist to use in each case. On initialization of the EAL layer by an DPDK application, the logical cores to be used and their socket location are displayed. This information can also be determined for all cores on the system by examining the ``/proc/cpuinfo`` file, for example, by running cat ``/proc/cpuinfo``. @@ -205,7 +213,7 @@ This can be useful when using other processors to understand the mapping of the .. warning:: - The logical core layout can change between different board layouts and should be checked before selecting an application coremask. + The logical core layout can change between different board layouts and should be checked before selecting an application coremask/corelist. Hugepage Memory Use by Applications ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/dpdk/doc/guides/linux_gsg/enable_func.rst b/dpdk/doc/guides/linux_gsg/enable_func.rst index 04e066c9..5ac0184a 100644 --- a/dpdk/doc/guides/linux_gsg/enable_func.rst +++ b/dpdk/doc/guides/linux_gsg/enable_func.rst @@ -35,8 +35,8 @@ Enabling Additional Functionality .. _High_Precision_Event_Timer: -High Precision Event Timer HPET) Functionality ----------------------------------------------- +High Precision Event Timer (HPET) Functionality +----------------------------------------------- BIOS Support ~~~~~~~~~~~~ @@ -87,6 +87,14 @@ The application can then determine what action to take, if any, if the HPET is n Running DPDK Applications Without Root Privileges -------------------------------------------------------- +.. note:: + + The instructions below will allow running DPDK as non-root with older + Linux kernel versions. However, since version 4.0, the kernel does not allow + unprivileged processes to read the physical address information from + the pagemaps file, making it impossible for those processes to use HW + devices which require physical addresses + Although applications using the DPDK use network ports and other hardware resources directly, with a number of small permission adjustments it is possible to run these applications as a user other than "root". To do so, the ownership, or permissions, on the following Linux file system objects should be adjusted to ensure that @@ -176,28 +184,3 @@ Also, if ``INTEL_IOMMU_DEFAULT_ON`` is not set in the kernel, the ``intel_iommu= This ensures that the Intel IOMMU is being initialized as expected. Please note that while using ``iommu=pt`` is compulsory for ``igb_uio driver``, the ``vfio-pci`` driver can actually work with both ``iommu=pt`` and ``iommu=on``. - -High Performance of Small Packets on 40G NIC --------------------------------------------- - -As there might be firmware fixes for performance enhancement in latest version -of firmware image, the firmware update might be needed for getting high performance. -Check with the local Intel's Network Division application engineers for firmware updates. -Users should consult the release notes specific to a DPDK release to identify -the validated firmware version for a NIC using the i40e driver. - -Use 16 Bytes RX Descriptor Size -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -As i40e PMD supports both 16 and 32 bytes RX descriptor sizes, and 16 bytes size can provide helps to high performance of small packets. -Configuration of ``CONFIG_RTE_LIBRTE_I40E_16BYTE_RX_DESC`` in config files can be changed to use 16 bytes size RX descriptors. - -High Performance and per Packet Latency Tradeoff -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Due to the hardware design, the interrupt signal inside NIC is needed for per -packet descriptor write-back. The minimum interval of interrupts could be set -at compile time by ``CONFIG_RTE_LIBRTE_I40E_ITR_INTERVAL`` in configuration files. -Though there is a default configuration, the interval could be tuned by the -users with that configuration item depends on what the user cares about more, -performance or per packet latency. diff --git a/dpdk/doc/guides/linux_gsg/index.rst b/dpdk/doc/guides/linux_gsg/index.rst index 3d3ada15..799559c2 100644 --- a/dpdk/doc/guides/linux_gsg/index.rst +++ b/dpdk/doc/guides/linux_gsg/index.rst @@ -40,6 +40,7 @@ Getting Started Guide for Linux intro sys_reqs build_dpdk + linux_drivers build_sample_apps enable_func quick_start diff --git a/dpdk/doc/guides/linux_gsg/linux_drivers.rst b/dpdk/doc/guides/linux_gsg/linux_drivers.rst new file mode 100644 index 00000000..08f7c9ba --- /dev/null +++ b/dpdk/doc/guides/linux_gsg/linux_drivers.rst @@ -0,0 +1,204 @@ +.. BSD LICENSE + Copyright(c) 2010-2015 Intel Corporation. + Copyright(c) 2017 Mellanox Corporation. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +.. _linux_gsg_linux_drivers: + +Linux Drivers +============= + +Different PMDs may require different kernel drivers in order to work properly. +Depends on the PMD being used, a corresponding kernel driver should be load +and bind to the network ports. + +UIO +--- + +A small kernel module to set up the device, map device memory to user-space and register interrupts. +In many cases, the standard ``uio_pci_generic`` module included in the Linux kernel +can provide the uio capability. This module can be loaded using the command: + +.. code-block:: console + + sudo modprobe uio_pci_generic + +.. note:: + + ``uio_pci_generic`` module doesn't support the creation of virtual functions. + +As an alternative to the ``uio_pci_generic``, the DPDK also includes the igb_uio +module which can be found in the kmod subdirectory referred to above. It can +be loaded as shown below: + +.. code-block:: console + + sudo modprobe uio + sudo insmod kmod/igb_uio.ko + +.. note:: + + For some devices which lack support for legacy interrupts, e.g. virtual function + (VF) devices, the ``igb_uio`` module may be needed in place of ``uio_pci_generic``. + +.. note:: + + If UEFI secure boot is enabled, the Linux kernel may disallow the use of + UIO on the system. Therefore, devices for use by DPDK should be bound to the + ``vfio-pci`` kernel module rather than ``igb_uio`` or ``uio_pci_generic``. + For more details see :ref:`linux_gsg_binding_kernel` below. + +Since DPDK release 1.7 onward provides VFIO support, use of UIO is optional +for platforms that support using VFIO. + +VFIO +---- + +A more robust and secure driver in compare to the ``UIO``, relying on IOMMU protection. +To make use of VFIO, the ``vfio-pci`` module must be loaded: + +.. code-block:: console + + sudo modprobe vfio-pci + +Note that in order to use VFIO, your kernel must support it. +VFIO kernel modules have been included in the Linux kernel since version 3.6.0 and are usually present by default, +however please consult your distributions documentation to make sure that is the case. + +Also, to use VFIO, both kernel and BIOS must support and be configured to use IO virtualization (such as Intel® VT-d). + +.. note:: + + ``vfio-pci`` module doesn't support the creation of virtual functions. + +For proper operation of VFIO when running DPDK applications as a non-privileged user, correct permissions should also be set up. +This can be done by using the DPDK setup script (called dpdk-setup.sh and located in the usertools directory). + +.. note:: + + VFIO can be used without IOMMU. While this is just as unsafe as using UIO, it does make it possible for the user to keep the degree of device access and programming that VFIO has, in situations where IOMMU is not available. + +Bifurcated Driver +----------------- + +PMDs which use the bifurcated driver co-exists with the device kernel driver. +On such model the NIC is controlled by the kernel, while the data +path is performed by the PMD directly on top of the device. + +Such model has the following benefits: + + - It is secure and robust, as the memory management and isolation + is done by the kernel. + - It enables the user to use legacy linux tools such as ``ethtool`` or + ``ifconfig`` while running DPDK application on the same network ports. + - It enables the DPDK application to filter only part of the traffic, + While the rest will be directed and handled by the kernel driver. + +More about the bifurcated driver can be found in +`Mellanox Bifurcated DPDK PMD +`__. + +.. _linux_gsg_binding_kernel: + +Binding and Unbinding Network Ports to/from the Kernel Modules +-------------------------------------------------------------- + +.. note:: + + PMDs Which use the bifurcated driver should not be unbind from their kernel drivers. this section is for PMDs which use the UIO or VFIO drivers. + +As of release 1.4, DPDK applications no longer automatically unbind all supported network ports from the kernel driver in use. +Instead, in case the PMD being used use the UIO or VFIO drivers, all ports that are to be used by an DPDK application must be bound to the +``uio_pci_generic``, ``igb_uio`` or ``vfio-pci`` module before the application is run. +For such PMDs, any network ports under Linux* control will be ignored and cannot be used by the application. + +To bind ports to the ``uio_pci_generic``, ``igb_uio`` or ``vfio-pci`` module for DPDK use, +and then subsequently return ports to Linux* control, +a utility script called dpdk-devbind.py is provided in the usertools subdirectory. +This utility can be used to provide a view of the current state of the network ports on the system, +and to bind and unbind those ports from the different kernel modules, including the uio and vfio modules. +The following are some examples of how the script can be used. +A full description of the script and its parameters can be obtained by calling the script with the ``--help`` or ``--usage`` options. +Note that the uio or vfio kernel modules to be used, should be loaded into the kernel before +running the ``dpdk-devbind.py`` script. + +.. warning:: + + Due to the way VFIO works, there are certain limitations to which devices can be used with VFIO. + Mainly it comes down to how IOMMU groups work. + Any Virtual Function device can be used with VFIO on its own, but physical devices will require either all ports bound to VFIO, + or some of them bound to VFIO while others not being bound to anything at all. + + If your device is behind a PCI-to-PCI bridge, the bridge will then be part of the IOMMU group in which your device is in. + Therefore, the bridge driver should also be unbound from the bridge PCI device for VFIO to work with devices behind the bridge. + +.. warning:: + + While any user can run the dpdk-devbind.py script to view the status of the network ports, + binding or unbinding network ports requires root privileges. + +To see the status of all network ports on the system: + +.. code-block:: console + + ./usertools/dpdk-devbind.py --status + + Network devices using DPDK-compatible driver + ============================================ + 0000:82:00.0 '82599EB 10-GbE NIC' drv=uio_pci_generic unused=ixgbe + 0000:82:00.1 '82599EB 10-GbE NIC' drv=uio_pci_generic unused=ixgbe + + Network devices using kernel driver + =================================== + 0000:04:00.0 'I350 1-GbE NIC' if=em0 drv=igb unused=uio_pci_generic *Active* + 0000:04:00.1 'I350 1-GbE NIC' if=eth1 drv=igb unused=uio_pci_generic + 0000:04:00.2 'I350 1-GbE NIC' if=eth2 drv=igb unused=uio_pci_generic + 0000:04:00.3 'I350 1-GbE NIC' if=eth3 drv=igb unused=uio_pci_generic + + Other network devices + ===================== + + +To bind device ``eth1``,``04:00.1``, to the ``uio_pci_generic`` driver: + +.. code-block:: console + + ./usertools/dpdk-devbind.py --bind=uio_pci_generic 04:00.1 + +or, alternatively, + +.. code-block:: console + + ./usertools/dpdk-devbind.py --bind=uio_pci_generic eth1 + +To restore device ``82:00.0`` to its original kernel binding: + +.. code-block:: console + + ./usertools/dpdk-devbind.py --bind=ixgbe 82:00.0 diff --git a/dpdk/doc/guides/linux_gsg/nic_perf_intel_platform.rst b/dpdk/doc/guides/linux_gsg/nic_perf_intel_platform.rst index d4a83624..febd7337 100644 --- a/dpdk/doc/guides/linux_gsg/nic_perf_intel_platform.rst +++ b/dpdk/doc/guides/linux_gsg/nic_perf_intel_platform.rst @@ -158,7 +158,7 @@ Configurations before running DPDK cd dpdk_folder - tools/cpu_layout.py + usertools/cpu_layout.py Or run ``lscpu`` to check the the cores on each socket. @@ -186,75 +186,5 @@ Configurations before running DPDK **Note**: To get the best performance, ensure that the core and NICs are in the same socket. In the example above ``85:00.0`` is on socket 1 and should be used by cores on socket 1 for the best performance. -4. Bind the test ports to DPDK compatible drivers, such as igb_uio. For example bind two ports to a DPDK compatible driver and check the status: - - .. code-block:: console - - - # Bind ports 82:00.0 and 85:00.0 to dpdk driver - ./dpdk_folder/tools/dpdk-devbind.py -b igb_uio 82:00.0 85:00.0 - - # Check the port driver status - ./dpdk_folder/tools/dpdk-devbind.py --status - - See ``dpdk-devbind.py --help`` for more details. - - -More details about DPDK setup and Linux kernel requirements see :ref:`linux_gsg_compiling_dpdk`. - - -Example of getting best performance for an Intel NIC ----------------------------------------------------- - -The following is an example of running the DPDK ``l3fwd`` sample application to get high performance with an -Intel server platform and Intel XL710 NICs. -For specific 40G NIC configuration please refer to the i40e NIC guide. - -The example scenario is to get best performance with two Intel XL710 40GbE ports. -See :numref:`figure_intel_perf_test_setup` for the performance test setup. - -.. _figure_intel_perf_test_setup: - -.. figure:: img/intel_perf_test_setup.* - - Performance Test Setup - - -1. Add two Intel XL710 NICs to the platform, and use one port per card to get best performance. - The reason for using two NICs is to overcome a PCIe Gen3's limitation since it cannot provide 80G bandwidth - for two 40G ports, but two different PCIe Gen3 x8 slot can. - Refer to the sample NICs output above, then we can select ``82:00.0`` and ``85:00.0`` as test ports:: - - 82:00.0 Ethernet [0200]: Intel XL710 for 40GbE QSFP+ [8086:1583] - 85:00.0 Ethernet [0200]: Intel XL710 for 40GbE QSFP+ [8086:1583] - -2. Connect the ports to the traffic generator. For high speed testing, it's best to use a hardware traffic generator. - -3. Check the PCI devices numa node (socket id) and get the cores number on the exact socket id. - In this case, ``82:00.0`` and ``85:00.0`` are both in socket 1, and the cores on socket 1 in the referenced platform - are 18-35 and 54-71. - Note: Don't use 2 logical cores on the same core (e.g core18 has 2 logical cores, core18 and core54), instead, use 2 logical - cores from different cores (e.g core18 and core19). - -4. Bind these two ports to igb_uio. - -5. As to XL710 40G port, we need at least two queue pairs to achieve best performance, then two queues per port - will be required, and each queue pair will need a dedicated CPU core for receiving/transmitting packets. - -6. The DPDK sample application ``l3fwd`` will be used for performance testing, with using two ports for bi-directional forwarding. - Compile the ``l3fwd sample`` with the default lpm mode. - -7. The command line of running l3fwd would be something like the followings:: - - ./l3fwd -c 0x3c0000 -n 4 -w 82:00.0 -w 85:00.0 \ - -- -p 0x3 --config '(0,0,18),(0,1,19),(1,0,20),(1,1,21)' - - This means that the application uses core 18 for port 0, queue pair 0 forwarding, core 19 for port 0, queue pair 1 forwarding, - core 20 for port 1, queue pair 0 forwarding, and core 21 for port 1, queue pair 1 forwarding. - - -8. Configure the traffic at a traffic generator. - - * Start creating a stream on packet generator. - - * Set the Ethernet II type to 0x0800. +4. Check which kernel drivers needs to be loaded and whether there is a need to unbind the network ports from their kernel drivers. +More details about DPDK setup and Linux kernel requirements see :ref:`linux_gsg_compiling_dpdk` and :ref:`linux_gsg_linux_drivers`. diff --git a/dpdk/doc/guides/linux_gsg/quick_start.rst b/dpdk/doc/guides/linux_gsg/quick_start.rst index 8789b588..39675db5 100644 --- a/dpdk/doc/guides/linux_gsg/quick_start.rst +++ b/dpdk/doc/guides/linux_gsg/quick_start.rst @@ -33,7 +33,7 @@ Quick Start Setup Script ======================== -The dpdk-setup.sh script, found in the tools subdirectory, allows the user to perform the following tasks: +The dpdk-setup.sh script, found in the usertools subdirectory, allows the user to perform the following tasks: * Build the DPDK libraries @@ -108,7 +108,7 @@ Some options in the script prompt the user for further data before proceeding. .. code-block:: console - source tools/dpdk-setup.sh + source usertools/dpdk-setup.sh ------------------------------------------------------------------------ @@ -126,19 +126,15 @@ Some options in the script prompt the user for further data before proceeding. [3] ppc_64-power8-linuxapp-gcc - [4] x86_64-ivshmem-linuxapp-gcc + [4] x86_64-native-bsdapp-clang - [5] x86_64-ivshmem-linuxapp-icc + [5] x86_64-native-bsdapp-gcc - [6] x86_64-native-bsdapp-clang + [6] x86_64-native-linuxapp-clang - [7] x86_64-native-bsdapp-gcc + [7] x86_64-native-linuxapp-gcc - [8] x86_64-native-linuxapp-clang - - [9] x86_64-native-linuxapp-gcc - - [10] x86_64-native-linuxapp-icc + [8] x86_64-native-linuxapp-icc ------------------------------------------------------------------------ @@ -286,7 +282,7 @@ the logical core layout of the platform should be determined when selecting a co INSTALL-APP helloworld INSTALL-MAP helloworld.map - sudo ./build/app/helloworld -c 0xf -n 3 + sudo ./build/app/helloworld -l 0-3 -n 3 [sudo] password for rte: EAL: coremask set to f diff --git a/dpdk/doc/guides/linux_gsg/sys_reqs.rst b/dpdk/doc/guides/linux_gsg/sys_reqs.rst index b3215448..3e7fe637 100644 --- a/dpdk/doc/guides/linux_gsg/sys_reqs.rst +++ b/dpdk/doc/guides/linux_gsg/sys_reqs.rst @@ -43,14 +43,21 @@ BIOS Setting Prerequisite on x86 For the majority of platforms, no special BIOS settings are needed to use basic DPDK functionality. However, for additional HPET timer and power management functionality, -and high performance of small packets on 40G NIC, BIOS setting changes may be needed. +and high performance of small packets, BIOS setting changes may be needed. Consult the section on :ref:`Enabling Additional Functionality ` for more information on the required changes. +.. note:: + + If UEFI secure boot is enabled, the Linux kernel may disallow the use of + UIO on the system. Therefore, devices for use by DPDK should be bound to the + ``vfio-pci`` kernel module rather than ``igb_uio`` or ``uio_pci_generic``. + For more details see :ref:`linux_gsg_binding_kernel`. + Compilation of the DPDK ----------------------- -**Required Tools:** +**Required Tools and Libraries:** .. note:: @@ -61,8 +68,8 @@ Compilation of the DPDK * coreutils: ``cmp``, ``sed``, ``grep``, ``arch``, etc. -* gcc: versions 4.5.x or later is recommended for ``i686/x86_64``. Versions 4.8.x or later is recommended - for ``ppc_64`` and ``x86_x32`` ABI. On some distributions, some specific compiler flags and linker flags are enabled by +* gcc: versions 4.9 or later is recommended for all platforms. + On some distributions, some specific compiler flags and linker flags are enabled by default and affect performance (``-fstack-protector``, for example). Please refer to the documentation of your distribution and to ``gcc -dumpspecs``. @@ -79,14 +86,14 @@ Compilation of the DPDK * glibc.ppc64, libgcc.ppc64, libstdc++.ppc64 and glibc-devel.ppc64 for IBM ppc_64; -.. note:: + .. note:: - x86_x32 ABI is currently supported with distribution packages only on Ubuntu - higher than 13.10 or recent Debian distribution. The only supported compiler is gcc 4.8+. + x86_x32 ABI is currently supported with distribution packages only on Ubuntu + higher than 13.10 or recent Debian distribution. The only supported compiler is gcc 4.9+. -.. note:: +* libnuma-devel - library for handling NUMA (Non Uniform Memory Access). - Python, version 2.6 or 2.7, to use various helper scripts included in the DPDK package. +* Python, version 2.7+ or 3.2+, to use various helper scripts included in the DPDK package. **Optional Tools:** @@ -131,8 +138,6 @@ System Software For other kernel builds, options which should be enabled for DPDK include: - * UIO support - * HUGETLBFS * PROC_PAGE_MONITOR support @@ -202,6 +207,12 @@ On a NUMA machine, pages should be allocated explicitly on separate nodes:: For 1G pages, it is not possible to reserve the hugepage memory after the system has booted. + On IBM POWER system, the nr_overcommit_hugepages should be set to the same value as nr_hugepages. + For example, if the required page number is 128, the following commands are used:: + + echo 128 > /sys/kernel/mm/hugepages/hugepages-16384kB/nr_hugepages + echo 128 > /sys/kernel/mm/hugepages/hugepages-16384kB/nr_overcommit_hugepages + Using Hugepages with the DPDK ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -217,56 +228,3 @@ The mount point can be made permanent across reboots, by adding the following li For 1GB pages, the page size must be specified as a mount option:: nodev /mnt/huge_1GB hugetlbfs pagesize=1GB 0 0 - -Xen Domain0 Support in the Linux Environment -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The existing memory management implementation is based on the Linux kernel hugepage mechanism. -On the Xen hypervisor, hugepage support for DomainU (DomU) Guests means that DPDK applications work as normal for guests. - -However, Domain0 (Dom0) does not support hugepages. -To work around this limitation, a new kernel module rte_dom0_mm is added to facilitate the allocation and mapping of memory via -**IOCTL** (allocation) and **MMAP** (mapping). - -Enabling Xen Dom0 Mode in the DPDK -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -By default, Xen Dom0 mode is disabled in the DPDK build configuration files. -To support Xen Dom0, the CONFIG_RTE_LIBRTE_XEN_DOM0 setting should be changed to “y”, which enables the Xen Dom0 mode at compile time. - -Furthermore, the CONFIG_RTE_EAL_ALLOW_INV_SOCKET_ID setting should also be changed to “y” in the case of the wrong socket ID being received. - -Loading the DPDK rte_dom0_mm Module -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -To run any DPDK application on Xen Dom0, the ``rte_dom0_mm`` module must be loaded into the running kernel with rsv_memsize option. -The module is found in the kmod sub-directory of the DPDK target directory. -This module should be loaded using the insmod command as shown below (assuming that the current directory is the DPDK target directory):: - - sudo insmod kmod/rte_dom0_mm.ko rsv_memsize=X - -The value X cannot be greater than 4096(MB). - -Configuring Memory for DPDK Use -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -After the rte_dom0_mm.ko kernel module has been loaded, the user must configure the memory size for DPDK usage. -This is done by echoing the memory size to a memsize file in the /sys/devices/ directory. -Use the following command (assuming that 2048 MB is required):: - - echo 2048 > /sys/kernel/mm/dom0-mm/memsize-mB/memsize - -The user can also check how much memory has already been used:: - - cat /sys/kernel/mm/dom0-mm/memsize-mB/memsize_rsvd - -Xen Domain0 does not support NUMA configuration, as a result the ``--socket-mem`` command line option is invalid for Xen Domain0. - -.. note:: - - The memsize value cannot be greater than the rsv_memsize value. - -Running the DPDK Application on Xen Domain0 -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -To run the DPDK application on Xen Domain0, an extra command line option ``--xen-dom0`` is required. diff --git a/dpdk/doc/guides/mempool/index.rst b/dpdk/doc/guides/mempool/index.rst new file mode 100644 index 00000000..b3c8e7f0 --- /dev/null +++ b/dpdk/doc/guides/mempool/index.rst @@ -0,0 +1,40 @@ +.. BSD LICENSE + Copyright(c) 2017 Cavium Inc. All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Cavium Inc nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Mempool Device Driver +===================== + +The following are a list of mempool PMDs, which can be used from an +application through the mempool API. + +.. toctree:: + :maxdepth: 2 + :numbered: + + octeontx diff --git a/dpdk/doc/guides/mempool/octeontx.rst b/dpdk/doc/guides/mempool/octeontx.rst new file mode 100644 index 00000000..b262c823 --- /dev/null +++ b/dpdk/doc/guides/mempool/octeontx.rst @@ -0,0 +1,104 @@ +.. BSD LICENSE + Copyright (C) Cavium, Inc. 2017. All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Cavium, Inc nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +OCTEONTX FPAVF Mempool Driver +============================= + +The OCTEONTX FPAVF PMD (**librte_mempool_octeontx**) is a mempool +driver for offload mempool device found in **Cavium OCTEONTX** SoC +family. + +More information can be found at `Cavium, Inc Official Website +`_. + +Features +-------- + +Features of the OCTEONTX FPAVF PMD are: + +- 32 SR-IOV Virtual functions +- 32 Pools +- HW mempool manager + +Supported OCTEONTX SoCs +----------------------- + +- CN83xx + +Prerequisites +------------- + +See :doc: `../platform/octeontx.rst` for setup information. + +Pre-Installation Configuration +------------------------------ + +Config File Options +~~~~~~~~~~~~~~~~~~~ + +The following options can be modified in the ``config`` file. +Please note that enabling debugging options may affect system performance. + +- ``CONFIG_RTE_MBUF_DEFAULT_MEMPOOL_OPS`` ( set to ``octeontx_fpavf``) + + Set default mempool ops to octeontx_fpavf. + +- ``CONFIG_RTE_LIBRTE_OCTEONTX_MEMPOOL`` (default ``y``) + + Toggle compilation of the ``librte_mempool_octeontx`` driver. + +- ``CONFIG_RTE_LIBRTE_OCTEONTX_MEMPOOL_DEBUG`` (default ``n``) + + Toggle display of generic debugging messages + +Driver Compilation +~~~~~~~~~~~~~~~~~~ + +To compile the OCTEONTX FPAVF MEMPOOL PMD for Linux arm64 gcc target, run the +following ``make`` command: + +.. code-block:: console + + cd + make config T=arm64-thunderx-linuxapp-gcc test-build + + +Initialization +-------------- + +The octeontx fpavf mempool initialization similar to other mempool +drivers like ring. However user need to pass --base-virtaddr as +command line input to application example test_mempool.c application. + +Example: + +.. code-block:: console + + ./build/app/test -c 0xf --base-virtaddr=0x100000000000 \ + --mbuf-pool-ops-name="octeontx_fpavf" diff --git a/dpdk/doc/guides/nics/ark.rst b/dpdk/doc/guides/nics/ark.rst new file mode 100644 index 00000000..6c135eeb --- /dev/null +++ b/dpdk/doc/guides/nics/ark.rst @@ -0,0 +1,258 @@ +.. BSD LICENSE + + Copyright (c) 2015-2017 Atomic Rules LLC + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Atomic Rules LLC nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +ARK Poll Mode Driver +==================== + +The ARK PMD is a DPDK poll-mode driver for the Atomic Rules Arkville +(ARK) family of devices. + +More information can be found at the `Atomic Rules website +`_. + +Overview +-------- + +The Atomic Rules Arkville product is DPDK and AXI compliant product +that marshals packets across a PCIe conduit between host DPDK mbufs and +FPGA AXI streams. + +The ARK PMD, and the spirit of the overall Arkville product, +has been to take the DPDK API/ABI as a fixed specification; +then implement much of the business logic in FPGA RTL circuits. +The approach of *working backwards* from the DPDK API/ABI and having +the GPP host software *dictate*, while the FPGA hardware *copes*, +results in significant performance gains over a naive implementation. + +While this document describes the ARK PMD software, it is helpful to +understand what the FPGA hardware is and is not. The Arkville RTL +component provides a single PCIe Physical Function (PF) supporting +some number of RX/Ingress and TX/Egress Queues. The ARK PMD controls +the Arkville core through a dedicated opaque Core BAR (CBAR). +To allow users full freedom for their own FPGA application IP, +an independent FPGA Application BAR (ABAR) is provided. + +One popular way to imagine Arkville's FPGA hardware aspect is as the +FPGA PCIe-facing side of a so-called Smart NIC. The Arkville core does +not contain any MACs, and is link-speed independent, as well as +agnostic to the number of physical ports the application chooses to +use. The ARK driver exposes the familiar PMD interface to allow packet +movement to and from mbufs across multiple queues. + +However FPGA RTL applications could contain a universe of added +functionality that an Arkville RTL core does not provide or can +not anticipate. To allow for this expectation of user-defined +innovation, the ARK PMD provides a dynamic mechanism of adding +capabilities without having to modify the ARK PMD. + +The ARK PMD is intended to support all instances of the Arkville +RTL Core, regardless of configuration, FPGA vendor, or target +board. While specific capabilities such as number of physical +hardware queue-pairs are negotiated; the driver is designed to +remain constant over a broad and extendable feature set. + +Intentionally, Arkville by itself DOES NOT provide common NIC +capabilities such as offload or receive-side scaling (RSS). +These capabilities would be viewed as a gate-level "tax" on +Green-box FPGA applications that do not require such function. +Instead, they can be added as needed with essentially no +overhead to the FPGA Application. + +The ARK PMD also supports optional user extensions, through dynamic linking. +The ARK PMD user extensions are a feature of Arkville’s DPDK +net/ark poll mode driver, allowing users to add their +own code to extend the net/ark functionality without +having to make source code changes to the driver. One motivation for +this capability is that while DPDK provides a rich set of functions +to interact with NIC-like capabilities (e.g. MAC addresses and statistics), +the Arkville RTL IP does not include a MAC. Users can supply their +own MAC or custom FPGA applications, which may require control from +the PMD. The user extension is the means providing the control +between the user's FPGA application and the existing DPDK features via +the PMD. + +Device Parameters +------------------- + +The ARK PMD supports device parameters that are used for packet +routing and for internal packet generation and packet checking. This +section describes the supported parameters. These features are +primarily used for diagnostics, testing, and performance verification +under the guidance of an Arkville specialist. The nominal use of +Arkville does not require any configuration using these parameters. + +"Pkt_dir" + +The Packet Director controls connectivity between Arkville's internal +hardware components. The features of the Pkt_dir are only used for +diagnostics and testing; it is not intended for nominal use. The full +set of features are not published at this level. + +Format: +Pkt_dir=0x00110F10 + +"Pkt_gen" + +The packet generator parameter takes a file as its argument. The file +contains configuration parameters used internally for regression +testing and are not intended to be published at this level. The +packet generator is an internal Arkville hardware component. + +Format: +Pkt_gen=./config/pg.conf + +"Pkt_chkr" + +The packet checker parameter takes a file as its argument. The file +contains configuration parameters used internally for regression +testing and are not intended to be published at this level. The +packet checker is an internal Arkville hardware component. + +Format: +Pkt_chkr=./config/pc.conf + + +Data Path Interface +------------------- + +Ingress RX and Egress TX operation is by the nominal DPDK API . +The driver supports single-port, multi-queue for both RX and TX. + +Configuration Information +------------------------- + +**DPDK Configuration Parameters** + + The following configuration options are available for the ARK PMD: + + * **CONFIG_RTE_LIBRTE_ARK_PMD** (default y): Enables or disables inclusion + of the ARK PMD driver in the DPDK compilation. + + * **CONFIG_RTE_LIBRTE_ARK_PAD_TX** (default y): When enabled TX + packets are padded to 60 bytes to support downstream MACS. + + * **CONFIG_RTE_LIBRTE_ARK_DEBUG_RX** (default n): Enables or disables debug + logging and internal checking of RX ingress logic within the ARK PMD driver. + + * **CONFIG_RTE_LIBRTE_ARK_DEBUG_TX** (default n): Enables or disables debug + logging and internal checking of TX egress logic within the ARK PMD driver. + + * **CONFIG_RTE_LIBRTE_ARK_DEBUG_STATS** (default n): Enables or disables debug + logging of detailed packet and performance statistics gathered in + the PMD and FPGA. + + * **CONFIG_RTE_LIBRTE_ARK_DEBUG_TRACE** (default n): Enables or disables debug + logging of detailed PMD events and status. + + +Building DPDK +------------- + +See the :ref:`DPDK Getting Started Guide for Linux ` for +instructions on how to build DPDK. + +By default the ARK PMD library will be built into the DPDK library. + +For configuring and using UIO and VFIO frameworks, please also refer :ref:`the +documentation that comes with DPDK suite `. + +Supported ARK RTL PCIe Instances +-------------------------------- + +ARK PMD supports the following Arkville RTL PCIe instances including: + +* ``1d6c:100d`` - AR-ARKA-FX0 [Arkville 32B DPDK Data Mover] +* ``1d6c:100e`` - AR-ARKA-FX1 [Arkville 64B DPDK Data Mover] + +Supported Operating Systems +--------------------------- + +Any Linux distribution fulfilling the conditions described in ``System Requirements`` +section of :ref:`the DPDK documentation ` or refer to *DPDK +Release Notes*. ARM and PowerPC architectures are not supported at this time. + + +Supported Features +------------------ + +* Dynamic ARK PMD extensions +* Multiple receive and transmit queues +* Jumbo frames up to 9K +* Hardware Statistics + +Unsupported Features +-------------------- + +Features that may be part of, or become part of, the Arkville RTL IP that are +not currently supported or exposed by the ARK PMD include: + +* PCIe SR-IOV Virtual Functions (VFs) +* Arkville's Packet Generator Control and Status +* Arkville's Packet Director Control and Status +* Arkville's Packet Checker Control and Status +* Arkville's Timebase Management + +Pre-Requisites +-------------- + +#. Prepare the system as recommended by DPDK suite. This includes environment + variables, hugepages configuration, tool-chains and configuration + +#. Insert igb_uio kernel module using the command 'modprobe igb_uio' + +#. Bind the intended ARK device to igb_uio module + +At this point the system should be ready to run DPDK applications. Once the +application runs to completion, the ARK PMD can be detached from igb_uio if necessary. + +Usage Example +------------- + +Follow instructions available in the document +:ref:`compiling and testing a PMD for a NIC ` to launch +**testpmd** with Atomic Rules ARK devices managed by librte_pmd_ark. + +Example output: + +.. code-block:: console + + [...] + EAL: PCI device 0000:01:00.0 on NUMA socket -1 + EAL: probe driver: 1d6c:100e rte_ark_pmd + EAL: PCI memory mapped at 0x7f9b6c400000 + PMD: eth_ark_dev_init(): Initializing 0:2:0.1 + ARKP PMD CommitID: 378f3a67 + Configuring Port 0 (socket 0) + Port 0: DC:3C:F6:00:00:01 + Checking link statuses... + Port 0 Link Up - speed 100000 Mbps - full-duplex + Done + testpmd> diff --git a/dpdk/doc/guides/nics/avp.rst b/dpdk/doc/guides/nics/avp.rst new file mode 100644 index 00000000..1fcba66c --- /dev/null +++ b/dpdk/doc/guides/nics/avp.rst @@ -0,0 +1,111 @@ +.. BSD LICENSE + Copyright(c) 2017 Wind River Systems, Inc. rights reserved. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +AVP Poll Mode Driver +================================================================= + +The Accelerated Virtual Port (AVP) device is a shared memory based device +only available on `virtualization platforms `_ +from Wind River Systems. The Wind River Systems virtualization platform +currently uses QEMU/KVM as its hypervisor and as such provides support for all +of the QEMU supported virtual and/or emulated devices (e.g., virtio, e1000, +etc.). The platform offers the virtio device type as the default device when +launching a virtual machine or creating a virtual machine port. The AVP device +is a specialized device available to customers that require increased +throughput and decreased latency to meet the demands of their performance +focused applications. + +The AVP driver binds to any AVP PCI devices that have been exported by the Wind +River Systems QEMU/KVM hypervisor. As a user of the DPDK driver API it +supports a subset of the full Ethernet device API to enable the application to +use the standard device configuration functions and packet receive/transmit +functions. + +These devices enable optimized packet throughput by bypassing QEMU and +delivering packets directly to the virtual switch via a shared memory +mechanism. This provides DPDK applications running in virtual machines with +significantly improved throughput and latency over other device types. + +The AVP device implementation is integrated with the QEMU/KVM live-migration +mechanism to allow applications to seamlessly migrate from one hypervisor node +to another with minimal packet loss. + + +Features and Limitations of the AVP PMD +--------------------------------------- + +The AVP PMD driver provides the following functionality. + +* Receive and transmit of both simple and chained mbuf packets, + +* Chained mbufs may include up to 5 chained segments, + +* Up to 8 receive and transmit queues per device, + +* Only a single MAC address is supported, + +* The MAC address cannot be modified, + +* The maximum receive packet length is 9238 bytes, + +* VLAN header stripping and inserting, + +* Promiscuous mode + +* VM live-migration + +* PCI hotplug insertion and removal + + +Prerequisites +------------- + +The following prerequisites apply: + +* A virtual machine running in a Wind River Systems virtualization + environment and configured with at least one neutron port defined with a + vif-model set to "avp". + + +Launching a VM with an AVP type network attachment +-------------------------------------------------- + +The following example will launch a VM with three network attachments. The +first attachment will have a default vif-model of "virtio". The next two +network attachments will have a vif-model of "avp" and may be used with a DPDK +application which is built to include the AVP PMD driver. + +.. code-block:: console + + nova boot --flavor small --image my-image \ + --nic net-id=${NETWORK1_UUID} \ + --nic net-id=${NETWORK2_UUID},vif-model=avp \ + --nic net-id=${NETWORK3_UUID},vif-model=avp \ + --security-group default my-instance1 diff --git a/dpdk/doc/guides/nics/bnx2x.rst b/dpdk/doc/guides/nics/bnx2x.rst index 6453168e..fbfc048e 100644 --- a/dpdk/doc/guides/nics/bnx2x.rst +++ b/dpdk/doc/guides/nics/bnx2x.rst @@ -96,9 +96,11 @@ Config File Options The following options can be modified in the ``.config`` file. Please note that enabling debugging options may affect system performance. -- ``CONFIG_RTE_LIBRTE_BNX2X_PMD`` (default **y**) +- ``CONFIG_RTE_LIBRTE_BNX2X_PMD`` (default **n**) - Toggle compilation of bnx2x driver. + Toggle compilation of bnx2x driver. To use bnx2x PMD set this config parameter + to 'y'. Also, in order for firmware binary to load user will need zlib devel + package installed. - ``CONFIG_RTE_LIBRTE_BNX2X_DEBUG`` (default **n**) @@ -123,143 +125,14 @@ enabling debugging options may affect system performance. .. _bnx2x_driver-compilation: -Driver Compilation -~~~~~~~~~~~~~~~~~~ +Driver compilation and testing +------------------------------ -BNX2X PMD for Linux x86_64 gcc target, run the following "make" -command:: - - cd - make config T=x86_64-native-linuxapp-gcc install - -To compile BNX2X PMD for Linux x86_64 clang target, run the following "make" -command:: - - cd - make config T=x86_64-native-linuxapp-clang install - -To compile BNX2X PMD for Linux i686 gcc target, run the following "make" -command:: - - cd - make config T=i686-native-linuxapp-gcc install - -To compile BNX2X PMD for Linux i686 gcc target, run the following "make" -command: - -.. code-block:: console - - cd - make config T=i686-native-linuxapp-gcc install - -To compile BNX2X PMD for FreeBSD x86_64 clang target, run the following "gmake" -command:: - - cd - gmake config T=x86_64-native-bsdapp-clang install - -To compile BNX2X PMD for FreeBSD x86_64 gcc target, run the following "gmake" -command:: - - cd - gmake config T=x86_64-native-bsdapp-gcc install -Wl,-rpath=/usr/local/lib/gcc48 CC=gcc48 - -To compile BNX2X PMD for FreeBSD x86_64 gcc target, run the following "gmake" -command: - -.. code-block:: console - - cd - gmake config T=x86_64-native-bsdapp-gcc install -Wl,-rpath=/usr/local/lib/gcc48 CC=gcc48 - -Linux ------ - -.. _bnx2x_Linux-installation: - -Linux Installation -~~~~~~~~~~~~~~~~~~ - -Sample Application Notes -~~~~~~~~~~~~~~~~~~~~~~~~ - -This section demonstrates how to launch ``testpmd`` with QLogic 578xx -devices managed by ``librte_pmd_bnx2x`` in Linux operating system. - -#. Request huge pages: - - .. code-block:: console - - echo 1024 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages/nr_hugepages - -#. Load ``igb_uio`` or ``vfio-pci`` driver: - - .. code-block:: console - - insmod ./x86_64-native-linuxapp-gcc/kmod/igb_uio.ko - - or - - .. code-block:: console - - modprobe vfio-pci - -#. Bind the QLogic adapters to ``igb_uio`` or ``vfio-pci`` loaded in the - previous step:: - - ./tools/dpdk-devbind.py --bind igb_uio 0000:84:00.0 0000:84:00.1 - - or - - Setup VFIO permissions for regular users and then bind to ``vfio-pci``: - - .. code-block:: console - - sudo chmod a+x /dev/vfio - - sudo chmod 0666 /dev/vfio/* - - ./tools/dpdk-devbind.py --bind vfio-pci 0000:84:00.0 0000:84:00.1 - -#. Start ``testpmd`` with basic parameters: - - .. code-block:: console - - ./x86_64-native-linuxapp-gcc/app/testpmd -c 0xf -n 4 -- -i - - Example output: - - .. code-block:: console - - [...] - EAL: PCI device 0000:84:00.0 on NUMA socket 1 - EAL: probe driver: 14e4:168e rte_bnx2x_pmd - EAL: PCI memory mapped at 0x7f14f6fe5000 - EAL: PCI memory mapped at 0x7f14f67e5000 - EAL: PCI memory mapped at 0x7f15fbd9b000 - EAL: PCI device 0000:84:00.1 on NUMA socket 1 - EAL: probe driver: 14e4:168e rte_bnx2x_pmd - EAL: PCI memory mapped at 0x7f14f5fe5000 - EAL: PCI memory mapped at 0x7f14f57e5000 - EAL: PCI memory mapped at 0x7f15fbd4f000 - Interactive-mode selected - Configuring Port 0 (socket 0) - PMD: bnx2x_dev_tx_queue_setup(): fp[00] req_bd=512, thresh=512, - usable_bd=1020, total_bd=1024, - tx_pages=4 - PMD: bnx2x_dev_rx_queue_setup(): fp[00] req_bd=128, thresh=0, - usable_bd=510, total_bd=512, - rx_pages=1, cq_pages=8 - PMD: bnx2x_print_adapter_info(): - [...] - Checking link statuses... - Port 0 Link Up - speed 10000 Mbps - full-duplex - Port 1 Link Up - speed 10000 Mbps - full-duplex - Done - testpmd> +Refer to the document :ref:`compiling and testing a PMD for a NIC ` +for details. SR-IOV: Prerequisites and sample Application Notes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +-------------------------------------------------- This section provides instructions to configure SR-IOV with Linux OS. @@ -311,7 +184,6 @@ This section provides instructions to configure SR-IOV with Linux OS. echo 2 > /sys/devices/pci0000:00/0000:00:03.0/0000:81:00.0/sriov_numvfs - #. Assign VF MAC address: Assign MAC address to the VF using iproute2 utility. The syntax is: @@ -323,9 +195,45 @@ This section provides instructions to configure SR-IOV with Linux OS. ip link set ens5f0 vf 0 mac 52:54:00:2f:9d:e8 - #. PCI Passthrough: The VF devices may be passed through to the guest VM using virt-manager or virsh etc. bnx2x PMD should be used to bind the VF devices in the guest VM using the instructions outlined in the Application notes below. + +#. Running testpmd: + + Follow instructions available in the document + :ref:`compiling and testing a PMD for a NIC ` + to run testpmd. + + Example output: + + .. code-block:: console + + [...] + EAL: PCI device 0000:84:00.0 on NUMA socket 1 + EAL: probe driver: 14e4:168e rte_bnx2x_pmd + EAL: PCI memory mapped at 0x7f14f6fe5000 + EAL: PCI memory mapped at 0x7f14f67e5000 + EAL: PCI memory mapped at 0x7f15fbd9b000 + EAL: PCI device 0000:84:00.1 on NUMA socket 1 + EAL: probe driver: 14e4:168e rte_bnx2x_pmd + EAL: PCI memory mapped at 0x7f14f5fe5000 + EAL: PCI memory mapped at 0x7f14f57e5000 + EAL: PCI memory mapped at 0x7f15fbd4f000 + Interactive-mode selected + Configuring Port 0 (socket 0) + PMD: bnx2x_dev_tx_queue_setup(): fp[00] req_bd=512, thresh=512, + usable_bd=1020, total_bd=1024, + tx_pages=4 + PMD: bnx2x_dev_rx_queue_setup(): fp[00] req_bd=128, thresh=0, + usable_bd=510, total_bd=512, + rx_pages=1, cq_pages=8 + PMD: bnx2x_print_adapter_info(): + [...] + Checking link statuses... + Port 0 Link Up - speed 10000 Mbps - full-duplex + Port 1 Link Up - speed 10000 Mbps - full-duplex + Done + testpmd> diff --git a/dpdk/doc/guides/nics/bnxt.rst b/dpdk/doc/guides/nics/bnxt.rst index 2669e984..9826b350 100644 --- a/dpdk/doc/guides/nics/bnxt.rst +++ b/dpdk/doc/guides/nics/bnxt.rst @@ -27,16 +27,33 @@ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -bnxt poll mode driver library -============================= +BNXT Poll Mode Driver +===================== -The bnxt poll mode library (**librte_pmd_bnxt**) implements support for -**Broadcom NetXtreme® C-Series**. These adapters support Standards- -compliant 10/25/50Gbps 30MPPS full-duplex throughput. +The bnxt poll mode library (**librte_pmd_bnxt**) implements support for: -Information about this family of adapters can be found in the -`NetXtreme® Brand section `_ -of the `Broadcom web site `_. + * **Broadcom NetXtreme-C®/NetXtreme-E® BCM5730X and BCM574XX family of + Ethernet Network Controllers** + + These adapters support Standards compliant 10/25/50/100Gbps 30MPPS + full-duplex throughput. + + Information about the NetXtreme family of adapters can be found in the + `NetXtreme® Brand section + `_ + of the `Broadcom website `_. + + * **Broadcom StrataGX® BCM5871X Series of Communucations Processors** + + These ARM based processors target a broad range of networking applications + including virtual CPE (vCPE) and NFV appliances, 10G service routers and + gateways, control plane processing for Ethernet switches and network + attached storage (NAS). + + Information about the StrataGX family of adapters can be found in the + `StrataGX® BCM5871X Series section + `_ + of the `Broadcom website `_. Limitations ----------- @@ -45,5 +62,3 @@ With the current driver, allocated mbufs must be large enough to hold the entire received frame. If the mbufs are not large enough, the packets will be dropped. This is most limiting when jumbo frames are used. - -SR-IOV is not supported. diff --git a/dpdk/doc/guides/nics/build_and_test.rst b/dpdk/doc/guides/nics/build_and_test.rst new file mode 100644 index 00000000..2d70af88 --- /dev/null +++ b/dpdk/doc/guides/nics/build_and_test.rst @@ -0,0 +1,179 @@ +.. BSD LICENSE + Copyright(c) 2017 Cavium, Inc. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Cavium, Inc. nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +.. _pmd_build_and_test: + +Compiling and testing a PMD for a NIC +===================================== + +This section demonstrates how to compile and run a Poll Mode Driver (PMD) for +the available Network Interface Cards in DPDK using TestPMD. + +TestPMD is one of the reference applications distributed with the DPDK. Its main +purpose is to forward packets between Ethernet ports on a network interface and +as such is the best way to test a PMD. + +Refer to the :ref:`testpmd application user guide ` for detailed +information on how to build and run testpmd. + +Driver Compilation +------------------ + +To compile a PMD for a platform, run make with appropriate target as shown below. +Use "make" command in Linux and "gmake" in FreeBSD. This will also build testpmd. + +To check available targets: + +.. code-block:: console + + cd + make showconfigs + +Example output: + +.. code-block:: console + + arm-armv7a-linuxapp-gcc + arm64-armv8a-linuxapp-gcc + arm64-dpaa2-linuxapp-gcc + arm64-thunderx-linuxapp-gcc + arm64-xgene1-linuxapp-gcc + i686-native-linuxapp-gcc + i686-native-linuxapp-icc + ppc_64-power8-linuxapp-gcc + x86_64-native-bsdapp-clang + x86_64-native-bsdapp-gcc + x86_64-native-linuxapp-clang + x86_64-native-linuxapp-gcc + x86_64-native-linuxapp-icc + x86_x32-native-linuxapp-gcc + +To compile a PMD for Linux x86_64 gcc target, run the following "make" command: + +.. code-block:: console + + make install T=x86_64-native-linuxapp-gcc + +Use ARM (ThunderX, DPAA, X-Gene) or PowerPC target for respective platform. + +For more information, refer to the :ref:`Getting Started Guide for Linux ` +or :ref:`Getting Started Guide for FreeBSD ` depending on your platform. + +Running testpmd in Linux +------------------------ + +This section demonstrates how to setup and run ``testpmd`` in Linux. + +#. Mount huge pages: + + .. code-block:: console + + mkdir /mnt/huge + mount -t hugetlbfs nodev /mnt/huge + +#. Request huge pages: + + Hugepage memory should be reserved as per application requirement. Check + hugepage size configured in the system and calculate the number of pages + required. + + To reserve 1024 pages of 2MB: + + .. code-block:: console + + echo 1024 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages + + .. note:: + + Check ``/proc/meminfo`` to find system hugepage size: + + .. code-block:: console + + grep "Hugepagesize:" /proc/meminfo + + Example output: + + .. code-block:: console + + Hugepagesize: 2048 kB + +#. Load ``igb_uio`` or ``vfio-pci`` driver: + + .. code-block:: console + + modprobe uio + insmod ./x86_64-native-linuxapp-gcc/kmod/igb_uio.ko + + or + + .. code-block:: console + + modprobe vfio-pci + +#. Setup VFIO permissions for regular users before binding to ``vfio-pci``: + + .. code-block:: console + + sudo chmod a+x /dev/vfio + + sudo chmod 0666 /dev/vfio/* + +#. Bind the adapters to ``igb_uio`` or ``vfio-pci`` loaded in the previous step: + + .. code-block:: console + + ./usertools/dpdk-devbind.py --bind igb_uio DEVICE1 DEVICE2 ... + + Or setup VFIO permissions for regular users and then bind to ``vfio-pci``: + + .. code-block:: console + + ./usertools/dpdk-devbind.py --bind vfio-pci DEVICE1 DEVICE2 ... + + .. note:: + + DEVICE1, DEVICE2 are specified via PCI "domain:bus:slot.func" syntax or + "bus:slot.func" syntax. + +#. Start ``testpmd`` with basic parameters: + + .. code-block:: console + + ./x86_64-native-linuxapp-gcc/app/testpmd -l 0-3 -n 4 -- -i + + Successful execution will show initialization messages from EAL, PMD and + testpmd application. A prompt will be displayed at the end for user commands + as interactive mode (``-i``) is on. + + .. code-block:: console + + testpmd> + + Refer to the :ref:`testpmd runtime functions ` for a list + of available commands. diff --git a/dpdk/doc/guides/nics/cxgbe.rst b/dpdk/doc/guides/nics/cxgbe.rst index d8236b08..8651a7be 100644 --- a/dpdk/doc/guides/nics/cxgbe.rst +++ b/dpdk/doc/guides/nics/cxgbe.rst @@ -1,5 +1,5 @@ .. BSD LICENSE - Copyright 2015 Chelsio Communications. + Copyright 2015-2017 Chelsio Communications. All rights reserved. Redistribution and use in source and binary forms, with or without @@ -32,8 +32,8 @@ CXGBE Poll Mode Driver ====================== The CXGBE PMD (**librte_pmd_cxgbe**) provides poll mode driver support -for **Chelsio T5** 10/40 Gbps family of adapters. CXGBE PMD has support -for the latest Linux and FreeBSD operating systems. +for **Chelsio Terminator** 10/25/40/100 Gbps family of adapters. CXGBE PMD +has support for the latest Linux and FreeBSD operating systems. More information can be found at `Chelsio Communications Official Website `_. @@ -55,9 +55,10 @@ CXGBE PMD has support for: Limitations ----------- -The Chelsio T5 devices provide two/four ports but expose a single PCI bus -address, thus, librte_pmd_cxgbe registers itself as a -PCI driver that allocates one Ethernet device per detected port. +The Chelsio Terminator series of devices provide two/four ports but +expose a single PCI bus address, thus, librte_pmd_cxgbe registers +itself as a PCI driver that allocates one Ethernet device per detected +port. For this reason, one cannot whitelist/blacklist a single port without whitelisting/blacklisting the other ports on the same device. @@ -70,10 +71,16 @@ Supported Chelsio T5 NICs - 40G NICs: T580-CR, T580-LP-CR, T580-SO-CR - Other T5 NICs: T522-CR +Supported Chelsio T6 NICs +------------------------- + +- 25G NICs: T6425-CR, T6225-CR, T6225-LL-CR, T6225-SO-CR +- 100G NICs: T62100-CR, T62100-LP-CR, T62100-SO-CR + Prerequisites ------------- -- Requires firmware version **1.13.32.0** and higher. Visit +- Requires firmware version **1.16.43.0** and higher. Visit `Chelsio Download Center `_ to get latest firmware bundled with the latest Chelsio Unified Wire package. @@ -123,26 +130,17 @@ enabling debugging options may affect system performance. Toggle display of receiving data path run-time check messages. +- ``CONFIG_RTE_LIBRTE_CXGBE_TPUT`` (default **y**) + + Toggle behaviour to prefer Throughput or Latency. + .. _driver-compilation: -Driver Compilation -~~~~~~~~~~~~~~~~~~ +Driver compilation and testing +------------------------------ -To compile CXGBE PMD for Linux x86_64 gcc target, run the following "make" -command: - -.. code-block:: console - - cd - make config T=x86_64-native-linuxapp-gcc install - -To compile CXGBE PMD for FreeBSD x86_64 clang target, run the following "gmake" -command: - -.. code-block:: console - - cd - gmake config T=x86_64-native-bsdapp-clang install +Refer to the document :ref:`compiling and testing a PMD for a NIC ` +for details. Linux ----- @@ -210,21 +208,14 @@ Unified Wire package for Linux operating system are as follows: .. code-block:: console - firmware-version: 1.13.32.0, TP 0.1.4.8 + firmware-version: 1.16.43.0, TP 0.1.4.9 Running testpmd ~~~~~~~~~~~~~~~ -This section demonstrates how to launch **testpmd** with Chelsio T5 +This section demonstrates how to launch **testpmd** with Chelsio devices managed by librte_pmd_cxgbe in Linux operating system. -#. Change to DPDK source directory where the target has been compiled in - section :ref:`driver-compilation`: - - .. code-block:: console - - cd - #. Load the kernel module: .. code-block:: console @@ -246,7 +237,7 @@ devices managed by librte_pmd_cxgbe in Linux operating system. .. note:: - Both the interfaces of a Chelsio T5 2-port adapter are bound to the + Both the interfaces of a Chelsio 2-port adapter are bound to the same PCI bus address. #. Unload the kernel module: @@ -255,59 +246,15 @@ devices managed by librte_pmd_cxgbe in Linux operating system. modprobe -ar cxgb4 csiostor -#. Request huge pages: +#. Running testpmd - .. code-block:: console - - echo 1024 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages/nr_hugepages - -#. Mount huge pages: - - .. code-block:: console - - mkdir /mnt/huge - mount -t hugetlbfs nodev /mnt/huge - -#. Load igb_uio or vfio-pci driver: - - .. code-block:: console - - insmod ./x86_64-native-linuxapp-gcc/kmod/igb_uio.ko - - or - - .. code-block:: console - - modprobe vfio-pci - -#. Bind the Chelsio T5 adapters to igb_uio or vfio-pci loaded in the previous - step: - - .. code-block:: console - - ./tools/dpdk-devbind.py --bind igb_uio 0000:02:00.4 - - or - - Setup VFIO permissions for regular users and then bind to vfio-pci: - - .. code-block:: console - - sudo chmod a+x /dev/vfio - - sudo chmod 0666 /dev/vfio/* - - ./tools/dpdk-devbind.py --bind vfio-pci 0000:02:00.4 + Follow instructions available in the document + :ref:`compiling and testing a PMD for a NIC ` + to run testpmd. .. note:: - Currently, CXGBE PMD only supports the binding of PF4 for Chelsio T5 NICs. - -#. Start testpmd with basic parameters: - - .. code-block:: console - - ./x86_64-native-linuxapp-gcc/app/testpmd -c 0xf -n 4 -w 0000:02:00.4 -- -i + Currently, CXGBE PMD only supports the binding of PF4 for Chelsio NICs. Example output: @@ -319,7 +266,7 @@ devices managed by librte_pmd_cxgbe in Linux operating system. EAL: PCI memory mapped at 0x7fd7c0200000 EAL: PCI memory mapped at 0x7fd77cdfd000 EAL: PCI memory mapped at 0x7fd7c10b7000 - PMD: rte_cxgbe_pmd: fw: 1.13.32.0, TP: 0.1.4.8 + PMD: rte_cxgbe_pmd: fw: 1.16.43.0, TP: 0.1.4.9 PMD: rte_cxgbe_pmd: Coming up as MASTER: Initializing adapter Interactive-mode selected Configuring Port 0 (socket 0) @@ -334,10 +281,10 @@ devices managed by librte_pmd_cxgbe in Linux operating system. Done testpmd> -.. note:: + .. note:: - Flow control pause TX/RX is disabled by default and can be enabled via - testpmd. Refer section :ref:`flow-control` for more details. + Flow control pause TX/RX is disabled by default and can be enabled via + testpmd. Refer section :ref:`flow-control` for more details. FreeBSD ------- @@ -403,12 +350,12 @@ Unified Wire package for FreeBSD operating system are as follows: .. code-block:: console - dev.t5nex.0.firmware_version: 1.13.32.0 + dev.t5nex.0.firmware_version: 1.16.43.0 Running testpmd ~~~~~~~~~~~~~~~ -This section demonstrates how to launch **testpmd** with Chelsio T5 +This section demonstrates how to launch **testpmd** with Chelsio devices managed by librte_pmd_cxgbe in FreeBSD operating system. #. Change to DPDK source directory where the target has been compiled in @@ -477,7 +424,7 @@ devices managed by librte_pmd_cxgbe in FreeBSD operating system. .. note:: - Both the interfaces of a Chelsio T5 2-port adapter are bound to the + Both the interfaces of a Chelsio 2-port adapter are bound to the same PCI bus address. #. Unload the kernel module: @@ -497,7 +444,7 @@ devices managed by librte_pmd_cxgbe in FreeBSD operating system. .. note:: - Currently, CXGBE PMD only supports the binding of PF4 for Chelsio T5 NICs. + Currently, CXGBE PMD only supports the binding of PF4 for Chelsio NICs. #. Load nic_uio kernel driver: @@ -509,7 +456,7 @@ devices managed by librte_pmd_cxgbe in FreeBSD operating system. .. code-block:: console - ./x86_64-native-bsdapp-clang/app/testpmd -c 0xf -n 4 -w 0000:02:00.4 -- -i + ./x86_64-native-bsdapp-clang/app/testpmd -l 0-3 -n 4 -w 0000:02:00.4 -- -i Example output: @@ -521,7 +468,7 @@ devices managed by librte_pmd_cxgbe in FreeBSD operating system. EAL: PCI memory mapped at 0x8007ec000 EAL: PCI memory mapped at 0x842800000 EAL: PCI memory mapped at 0x80086c000 - PMD: rte_cxgbe_pmd: fw: 1.13.32.0, TP: 0.1.4.8 + PMD: rte_cxgbe_pmd: fw: 1.16.43.0, TP: 0.1.4.9 PMD: rte_cxgbe_pmd: Coming up as MASTER: Initializing adapter Interactive-mode selected Configuring Port 0 (socket 0) diff --git a/dpdk/doc/guides/nics/dpaa.rst b/dpdk/doc/guides/nics/dpaa.rst new file mode 100644 index 00000000..d331c053 --- /dev/null +++ b/dpdk/doc/guides/nics/dpaa.rst @@ -0,0 +1,378 @@ +.. BSD LICENSE + Copyright 2017 NXP. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of NXP nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +DPAA Poll Mode Driver +===================== + +The DPAA NIC PMD (**librte_pmd_dpaa**) provides poll mode driver +support for the inbuilt NIC found in the **NXP DPAA** SoC family. + +More information can be found at `NXP Official Website +`_. + +NXP DPAA (Data Path Acceleration Architecture - Gen 1) +------------------------------------------------------ + +This section provides an overview of the NXP DPAA architecture +and how it is integrated into the DPDK. + +Contents summary + +- DPAA overview +- DPAA driver architecture overview + +.. _dpaa_overview: + +DPAA Overview +~~~~~~~~~~~~~ + +Reference: `FSL DPAA Architecture `_. + +The QorIQ Data Path Acceleration Architecture (DPAA) is a set of hardware +components on specific QorIQ series multicore processors. This architecture +provides the infrastructure to support simplified sharing of networking +interfaces and accelerators by multiple CPU cores, and the accelerators +themselves. + +DPAA includes: + +- Cores +- Network and packet I/O +- Hardware offload accelerators +- Infrastructure required to facilitate flow of packets between the components above + +Infrastructure components are: + +- The Queue Manager (QMan) is a hardware accelerator that manages frame queues. + It allows CPUs and other accelerators connected to the SoC datapath to + enqueue and dequeue ethernet frames, thus providing the infrastructure for + data exchange among CPUs and datapath accelerators. +- The Buffer Manager (BMan) is a hardware buffer pool management block that + allows software and accelerators on the datapath to acquire and release + buffers in order to build frames. + +Hardware accelerators are: + +- SEC - Cryptographic accelerator +- PME - Pattern matching engine + +The Network and packet I/O component: + +- The Frame Manager (FMan) is a key component in the DPAA and makes use of the + DPAA infrastructure (QMan and BMan). FMan is responsible for packet + distribution and policing. Each frame can be parsed, classified and results + may be attached to the frame. This meta data can be used to select + particular QMan queue, which the packet is forwarded to. + + +DPAA DPDK - Poll Mode Driver Overview +------------------------------------- + +This section provides an overview of the drivers for DPAA: + +* Bus driver and associated "DPAA infrastructure" drivers +* Functional object drivers (such as Ethernet). + +Brief description of each driver is provided in layout below as well as +in the following sections. + +.. code-block:: console + + +------------+ + | DPDK DPAA | + | PMD | + +-----+------+ + | + +-----+------+ +---------------+ + : Ethernet :.......| DPDK DPAA | + . . . . . . . . . : (FMAN) : | Mempool driver| + . +---+---+----+ | (BMAN) | + . ^ | +-----+---------+ + . | | . + . | | . + . +---+---V----+ . + . . . . . . . . . . .: Portal drv : . + . . : : . + . . +-----+------+ . + . . : QMAN : . + . . : Driver : . + +----+------+-------+ +-----+------+ . + | DPDK DPAA Bus | | . + | driver |....................|..................... + | /bus/dpaa | | + +-------------------+ | + | + ========================== HARDWARE =====|======================== + PHY + =========================================|======================== + +In the above representation, solid lines represent components which interface +with DPDK RTE Framework and dotted lines represent DPAA internal components. + +DPAA Bus driver +~~~~~~~~~~~~~~~ + +The DPAA bus driver is a ``rte_bus`` driver which scans the platform like bus. +Key functions include: + +- Scanning and parsing the various objects and adding them to their respective + device list. +- Performing probe for available drivers against each scanned device +- Creating necessary ethernet instance before passing control to the PMD + +DPAA NIC Driver (PMD) +~~~~~~~~~~~~~~~~~~~~~ + +DPAA PMD is traditional DPDK PMD which provides necessary interface between +RTE framework and DPAA internal components/drivers. + +- Once devices have been identified by DPAA Bus, each device is associated + with the PMD +- PMD is responsible for implementing necessary glue layer between RTE APIs + and lower level QMan and FMan blocks. + The Ethernet driver is bound to a FMAN port and implements the interfaces + needed to connect the DPAA network interface to the network stack. + Each FMAN Port corresponds to a DPDK network interface. + + +Features +^^^^^^^^ + + Features of the DPAA PMD are: + + - Multiple queues for TX and RX + - Receive Side Scaling (RSS) + - Packet type information + - Checksum offload + - Promiscuous mode + +DPAA Mempool Driver +~~~~~~~~~~~~~~~~~~~ + +DPAA has a hardware offloaded buffer pool manager, called BMan, or Buffer +Manager. + +- Using standard Mempools operations RTE API, the mempool driver interfaces + with RTE to service each mempool creation, deletion, buffer allocation and + deallocation requests. +- Each FMAN instance has a BMan pool attached to it during initialization. + Each Tx frame can be automatically released by hardware, if allocated from + this pool. + + +Supported DPAA SoCs +------------------- + +- LS1043A/LS1023A +- LS1046A/LS1026A + +Prerequisites +------------- + +There are three main pre-requisities for executing DPAA PMD on a DPAA +compatible board: + +1. **ARM 64 Tool Chain** + + For example, the `*aarch64* Linaro Toolchain `_. + +2. **Linux Kernel** + + It can be obtained from `NXP's Github hosting `_. + +3. **Rootfile system** + + Any *aarch64* supporting filesystem can be used. For example, + Ubuntu 15.10 (Wily) or 16.04 LTS (Xenial) userland which can be obtained + from `here `_. + +4. **FMC Tool** + + Before any DPDK application can be executed, the Frame Manager Configuration + Tool (FMC) need to be executed to set the configurations of the queues. This + includes the queue state, RSS and other policies. + This tool can be obtained from `NXP (Freescale) Public Git Repository `_. + + This tool needs configuration files which are available in the + :ref:`DPDK Extra Scripts `, described below for DPDK usages. + +As an alternative method, DPAA PMD can also be executed using images provided +as part of SDK from NXP. The SDK includes all the above prerequisites necessary +to bring up a DPAA board. + +The following dependencies are not part of DPDK and must be installed +separately: + +- **NXP Linux SDK** + + NXP Linux software development kit (SDK) includes support for family + of QorIQ® ARM-Architecture-based system on chip (SoC) processors + and corresponding boards. + + It includes the Linux board support packages (BSPs) for NXP SoCs, + a fully operational tool chain, kernel and board specific modules. + + SDK and related information can be obtained from: `NXP QorIQ SDK `_. + + +.. _extra_scripts: + +- **DPDK Extra Scripts** + + DPAA based resources can be configured easily with the help of ready scripts + as provided in the DPDK Extra repository. + + `DPDK Extras Scripts `_. + +Currently supported by DPDK: + +- NXP SDK **2.0+**. +- Supported architectures: **arm64 LE**. + +- Follow the DPDK :ref:`Getting Started Guide for Linux ` + to setup the basic DPDK environment. + +.. note:: + + Some part of dpaa bus code (qbman and fman - library) routines are + dual licensed (BSD & GPLv2), however they are used as BSD in DPDK in userspace. + +Pre-Installation Configuration +------------------------------ + +Config File Options +~~~~~~~~~~~~~~~~~~~ + +The following options can be modified in the ``config`` file. +Please note that enabling debugging options may affect system performance. + +- ``CONFIG_RTE_LIBRTE_DPAA_BUS`` (default ``n``) + + By default it is enabled only for defconfig_arm64-dpaa-* config. + Toggle compilation of the ``librte_bus_dpaa`` driver. + +- ``CONFIG_RTE_LIBRTE_DPAA_PMD`` (default ``n``) + + By default it is enabled only for defconfig_arm64-dpaa-* config. + Toggle compilation of the ``librte_pmd_dpaa`` driver. + +- ``CONFIG_RTE_LIBRTE_DPAA_DEBUG_DRIVER`` (default ``n``) + + Toggles display of bus configurations and enables a debugging queue + to fetch error (Rx/Tx) packets to driver. By default, packets with errors + (like wrong checksum) are dropped by the hardware. + +- ``CONFIG_RTE_LIBRTE_DPAA_HWDEBUG`` (default ``n``) + + Enables debugging of the Queue and Buffer Manager layer which interacts + with the DPAA hardware. + +- ``CONFIG_RTE_MBUF_DEFAULT_MEMPOOL_OPS`` (default ``dpaa``) + + This is not a DPAA specific configuration - it is a generic RTE config. + For optimal performance and hardware utilization, it is expected that DPAA + Mempool driver is used for mempools. For that, this configuration needs to + enabled. + +Environment Variables +~~~~~~~~~~~~~~~~~~~~~ + +DPAA drivers uses the following environment variables to configure its +state during application initialization: + +- ``DPAA_NUM_RX_QUEUES`` (default 1) + + This defines the number of Rx queues configured for an application, per + port. Hardware would distribute across these many number of queues on Rx + of packets. + In case the application is configured to use lesser number of queues than + configured above, it might result in packet loss (because of distribution). + + +Driver compilation and testing +------------------------------ + +Refer to the document :ref:`compiling and testing a PMD for a NIC ` +for details. + +#. Running testpmd: + + Follow instructions available in the document + :ref:`compiling and testing a PMD for a NIC ` + to run testpmd. + + Example output: + + .. code-block:: console + + ./arm64-dpaa-linuxapp-gcc/testpmd -c 0xff -n 1 \ + -- -i --portmask=0x3 --nb-cores=1 --no-flush-rx + + ..... + EAL: Registered [pci] bus. + EAL: Registered [dpaa] bus. + EAL: Detected 4 lcore(s) + ..... + EAL: dpaa: Bus scan completed + ..... + Configuring Port 0 (socket 0) + Port 0: 00:00:00:00:00:01 + Configuring Port 1 (socket 0) + Port 1: 00:00:00:00:00:02 + ..... + Checking link statuses... + Port 0 Link Up - speed 10000 Mbps - full-duplex + Port 1 Link Up - speed 10000 Mbps - full-duplex + Done + testpmd> + +Limitations +----------- + +Platform Requirement +~~~~~~~~~~~~~~~~~~~~ + +DPAA drivers for DPDK can only work on NXP SoCs as listed in the +``Supported DPAA SoCs``. + +Maximum packet length +~~~~~~~~~~~~~~~~~~~~~ + +The DPAA SoC family support a maximum of a 10240 jumbo frame. The value +is fixed and cannot be changed. So, even when the ``rxmode.max_rx_pkt_len`` +member of ``struct rte_eth_conf`` is set to a value lower than 10240, frames +up to 10240 bytes can still reach the host interface. + +Multiprocess Support +~~~~~~~~~~~~~~~~~~~~ + +Current version of DPAA driver doesn't support multi-process applications +where I/O is performed using secondary processes. This feature would be +implemented in subsequent versions. diff --git a/dpdk/doc/guides/nics/dpaa2.rst b/dpdk/doc/guides/nics/dpaa2.rst new file mode 100644 index 00000000..85bc0488 --- /dev/null +++ b/dpdk/doc/guides/nics/dpaa2.rst @@ -0,0 +1,606 @@ +.. BSD LICENSE + Copyright 2016 NXP. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of NXP nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +DPAA2 Poll Mode Driver +====================== + +The DPAA2 NIC PMD (**librte_pmd_dpaa2**) provides poll mode driver +support for the inbuilt NIC found in the **NXP DPAA2** SoC family. + +More information can be found at `NXP Official Website +`_. + +NXP DPAA2 (Data Path Acceleration Architecture Gen2) +---------------------------------------------------- + +This section provides an overview of the NXP DPAA2 architecture +and how it is integrated into the DPDK. + +Contents summary + +- DPAA2 overview +- Overview of DPAA2 objects +- DPAA2 driver architecture overview + +.. _dpaa2_overview: + +DPAA2 Overview +~~~~~~~~~~~~~~ + +Reference: `FSL MC BUS in Linux Kernel `_. + +DPAA2 is a hardware architecture designed for high-speed network +packet processing. DPAA2 consists of sophisticated mechanisms for +processing Ethernet packets, queue management, buffer management, +autonomous L2 switching, virtual Ethernet bridging, and accelerator +(e.g. crypto) sharing. + +A DPAA2 hardware component called the Management Complex (or MC) manages the +DPAA2 hardware resources. The MC provides an object-based abstraction for +software drivers to use the DPAA2 hardware. + +The MC uses DPAA2 hardware resources such as queues, buffer pools, and +network ports to create functional objects/devices such as network +interfaces, an L2 switch, or accelerator instances. + +The MC provides memory-mapped I/O command interfaces (MC portals) +which DPAA2 software drivers use to operate on DPAA2 objects: + +The diagram below shows an overview of the DPAA2 resource management +architecture: + +.. code-block:: console + + +--------------------------------------+ + | OS | + | DPAA2 drivers | + | | | + +-----------------------------|--------+ + | + | (create,discover,connect + | config,use,destroy) + | + DPAA2 | + +------------------------| mc portal |-+ + | | | + | +- - - - - - - - - - - - -V- - -+ | + | | | | + | | Management Complex (MC) | | + | | | | + | +- - - - - - - - - - - - - - - -+ | + | | + | Hardware Hardware | + | Resources Objects | + | --------- ------- | + | -queues -DPRC | + | -buffer pools -DPMCP | + | -Eth MACs/ports -DPIO | + | -network interface -DPNI | + | profiles -DPMAC | + | -queue portals -DPBP | + | -MC portals ... | + | ... | + | | + +--------------------------------------+ + +The MC mediates operations such as create, discover, +connect, configuration, and destroy. Fast-path operations +on data, such as packet transmit/receive, are not mediated by +the MC and are done directly using memory mapped regions in +DPIO objects. + +Overview of DPAA2 Objects +~~~~~~~~~~~~~~~~~~~~~~~~~ + +The section provides a brief overview of some key DPAA2 objects. +A simple scenario is described illustrating the objects involved +in creating a network interfaces. + +DPRC (Datapath Resource Container) + + A DPRC is a container object that holds all the other + types of DPAA2 objects. In the example diagram below there + are 8 objects of 5 types (DPMCP, DPIO, DPBP, DPNI, and DPMAC) + in the container. + +.. code-block:: console + + +---------------------------------------------------------+ + | DPRC | + | | + | +-------+ +-------+ +-------+ +-------+ +-------+ | + | | DPMCP | | DPIO | | DPBP | | DPNI | | DPMAC | | + | +-------+ +-------+ +-------+ +---+---+ +---+---+ | + | | DPMCP | | DPIO | | + | +-------+ +-------+ | + | | DPMCP | | + | +-------+ | + | | + +---------------------------------------------------------+ + +From the point of view of an OS, a DPRC behaves similar to a plug and +play bus, like PCI. DPRC commands can be used to enumerate the contents +of the DPRC, discover the hardware objects present (including mappable +regions and interrupts). + +.. code-block:: console + + DPRC.1 (bus) + | + +--+--------+-------+-------+-------+ + | | | | | + DPMCP.1 DPIO.1 DPBP.1 DPNI.1 DPMAC.1 + DPMCP.2 DPIO.2 + DPMCP.3 + +Hardware objects can be created and destroyed dynamically, providing +the ability to hot plug/unplug objects in and out of the DPRC. + +A DPRC has a mappable MMIO region (an MC portal) that can be used +to send MC commands. It has an interrupt for status events (like +hotplug). + +All objects in a container share the same hardware "isolation context". +This means that with respect to an IOMMU the isolation granularity +is at the DPRC (container) level, not at the individual object +level. + +DPRCs can be defined statically and populated with objects +via a config file passed to the MC when firmware starts +it. There is also a Linux user space tool called "restool" +that can be used to create/destroy containers and objects +dynamically. + +DPAA2 Objects for an Ethernet Network Interface +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +A typical Ethernet NIC is monolithic-- the NIC device contains TX/RX +queuing mechanisms, configuration mechanisms, buffer management, +physical ports, and interrupts. DPAA2 uses a more granular approach +utilizing multiple hardware objects. Each object provides specialized +functions. Groups of these objects are used by software to provide +Ethernet network interface functionality. This approach provides +efficient use of finite hardware resources, flexibility, and +performance advantages. + +The diagram below shows the objects needed for a simple +network interface configuration on a system with 2 CPUs. + +.. code-block:: console + + +---+---+ +---+---+ + CPU0 CPU1 + +---+---+ +---+---+ + | | + +---+---+ +---+---+ + DPIO DPIO + +---+---+ +---+---+ + \ / + \ / + \ / + +---+---+ + DPNI --- DPBP,DPMCP + +---+---+ + | + | + +---+---+ + DPMAC + +---+---+ + | + port/PHY + +Below the objects are described. For each object a brief description +is provided along with a summary of the kinds of operations the object +supports and a summary of key resources of the object (MMIO regions +and IRQs). + +DPMAC (Datapath Ethernet MAC): represents an Ethernet MAC, a +hardware device that connects to an Ethernet PHY and allows +physical transmission and reception of Ethernet frames. + +- MMIO regions: none +- IRQs: DPNI link change +- commands: set link up/down, link config, get stats, IRQ config, enable, reset + +DPNI (Datapath Network Interface): contains TX/RX queues, +network interface configuration, and RX buffer pool configuration +mechanisms. The TX/RX queues are in memory and are identified by +queue number. + +- MMIO regions: none +- IRQs: link state +- commands: port config, offload config, queue config, parse/classify config, IRQ config, enable, reset + +DPIO (Datapath I/O): provides interfaces to enqueue and dequeue +packets and do hardware buffer pool management operations. The DPAA2 +architecture separates the mechanism to access queues (the DPIO object) +from the queues themselves. The DPIO provides an MMIO interface to +enqueue/dequeue packets. To enqueue something a descriptor is written +to the DPIO MMIO region, which includes the target queue number. +There will typically be one DPIO assigned to each CPU. This allows all +CPUs to simultaneously perform enqueue/dequeued operations. DPIOs are +expected to be shared by different DPAA2 drivers. + +- MMIO regions: queue operations, buffer management +- IRQs: data availability, congestion notification, buffer pool depletion +- commands: IRQ config, enable, reset + +DPBP (Datapath Buffer Pool): represents a hardware buffer +pool. + +- MMIO regions: none +- IRQs: none +- commands: enable, reset + +DPMCP (Datapath MC Portal): provides an MC command portal. +Used by drivers to send commands to the MC to manage +objects. + +- MMIO regions: MC command portal +- IRQs: command completion +- commands: IRQ config, enable, reset + +Object Connections +~~~~~~~~~~~~~~~~~~ + +Some objects have explicit relationships that must +be configured: + +- DPNI <--> DPMAC +- DPNI <--> DPNI +- DPNI <--> L2-switch-port + +A DPNI must be connected to something such as a DPMAC, +another DPNI, or L2 switch port. The DPNI connection +is made via a DPRC command. + +.. code-block:: console + + +-------+ +-------+ + | DPNI | | DPMAC | + +---+---+ +---+---+ + | | + +==========+ + +- DPNI <--> DPBP + +A network interface requires a 'buffer pool' (DPBP object) which provides +a list of pointers to memory where received Ethernet data is to be copied. +The Ethernet driver configures the DPBPs associated with the network +interface. + +Interrupts +~~~~~~~~~~ + +All interrupts generated by DPAA2 objects are message +interrupts. At the hardware level message interrupts +generated by devices will normally have 3 components-- +1) a non-spoofable 'device-id' expressed on the hardware +bus, 2) an address, 3) a data value. + +In the case of DPAA2 devices/objects, all objects in the +same container/DPRC share the same 'device-id'. +For ARM-based SoC this is the same as the stream ID. + + +DPAA2 DPDK - Poll Mode Driver Overview +-------------------------------------- + +This section provides an overview of the drivers for +DPAA2-- 1) the bus driver and associated "DPAA2 infrastructure" +drivers and 2) functional object drivers (such as Ethernet). + +As described previously, a DPRC is a container that holds the other +types of DPAA2 objects. It is functionally similar to a plug-and-play +bus controller. + +Each object in the DPRC is a Linux "device" and is bound to a driver. +The diagram below shows the dpaa2 drivers involved in a networking +scenario and the objects bound to each driver. A brief description +of each driver follows. + +.. code-block: console + + + +------------+ + | DPDK DPAA2 | + | PMD | + +------------+ +------------+ + | Ethernet |.......| Mempool | + . . . . . . . . . | (DPNI) | | (DPBP) | + . +---+---+----+ +-----+------+ + . ^ | . + . | | . + . | | . + . +---+---V----+ . + . . . . . . . . . . .| DPIO driver| . + . . | (DPIO) | . + . . +-----+------+ . + . . | QBMAN | . + . . | Driver | . + +----+------+-------+ +-----+----- | . + | dpaa2 bus | | . + | VFIO fslmc-bus |....................|..................... + | | | + | /bus/fslmc | | + +-------------------+ | + | + ========================== HARDWARE =====|======================= + DPIO + | + DPNI---DPBP + | + DPMAC + | + PHY + =========================================|======================== + + +A brief description of each driver is provided below. + +DPAA2 bus driver +~~~~~~~~~~~~~~~~ + +The DPAA2 bus driver is a rte_bus driver which scans the fsl-mc bus. +Key functions include: + +- Reading the container and setting up vfio group +- Scanning and parsing the various MC objects and adding them to + their respective device list. + +Additionally, it also provides the object driver for generic MC objects. + +DPIO driver +~~~~~~~~~~~ + +The DPIO driver is bound to DPIO objects and provides services that allow +other drivers such as the Ethernet driver to enqueue and dequeue data for +their respective objects. +Key services include: + +- Data availability notifications +- Hardware queuing operations (enqueue and dequeue of data) +- Hardware buffer pool management + +To transmit a packet the Ethernet driver puts data on a queue and +invokes a DPIO API. For receive, the Ethernet driver registers +a data availability notification callback. To dequeue a packet +a DPIO API is used. + +There is typically one DPIO object per physical CPU for optimum +performance, allowing different CPUs to simultaneously enqueue +and dequeue data. + +The DPIO driver operates on behalf of all DPAA2 drivers +active -- Ethernet, crypto, compression, etc. + +DPBP based Mempool driver +~~~~~~~~~~~~~~~~~~~~~~~~~ + +The DPBP driver is bound to a DPBP objects and provides sevices to +create a hardware offloaded packet buffer mempool. + +DPAA2 NIC Driver +~~~~~~~~~~~~~~~~ +The Ethernet driver is bound to a DPNI and implements the kernel +interfaces needed to connect the DPAA2 network interface to +the network stack. + +Each DPNI corresponds to a DPDK network interface. + +Features +^^^^^^^^ + +Features of the DPAA2 PMD are: + +- Multiple queues for TX and RX +- Receive Side Scaling (RSS) +- MAC/VLAN filtering +- Packet type information +- Checksum offload +- Promiscuous mode +- Multicast mode +- Port hardware statistics +- Jumbo frames +- Link flow control +- Scattered and gather for TX and RX + +Supported DPAA2 SoCs +-------------------- + +- LS2080A/LS2040A +- LS2084A/LS2044A +- LS2088A/LS2048A +- LS1088A/LS1048A + +Prerequisites +------------- + +There are three main pre-requisities for executing DPAA2 PMD on a DPAA2 +compatible board: + +1. **ARM 64 Tool Chain** + + For example, the `*aarch64* Linaro Toolchain `_. + +2. **Linux Kernel** + + It can be obtained from `NXP's Github hosting `_. + +3. **Rootfile system** + + Any *aarch64* supporting filesystem can be used. For example, + Ubuntu 15.10 (Wily) or 16.04 LTS (Xenial) userland which can be obtained + from `here `_. + +As an alternative method, DPAA2 PMD can also be executed using images provided +as part of SDK from NXP. The SDK includes all the above prerequisites necessary +to bring up a DPAA2 board. + +The following dependencies are not part of DPDK and must be installed +separately: + +- **NXP Linux SDK** + + NXP Linux software development kit (SDK) includes support for family + of QorIQ® ARM-Architecture-based system on chip (SoC) processors + and corresponding boards. + + It includes the Linux board support packages (BSPs) for NXP SoCs, + a fully operational tool chain, kernel and board specific modules. + + SDK and related information can be obtained from: `NXP QorIQ SDK `_. + +- **DPDK Extra Scripts** + + DPAA2 based resources can be configured easily with the help of ready scripts + as provided in the DPDK Extra repository. + + `DPDK Extras Scripts `_. + +Currently supported by DPDK: + +- NXP SDK **17.08+**. +- MC Firmware version **10.3.1** and higher. +- Supported architectures: **arm64 LE**. + +- Follow the DPDK :ref:`Getting Started Guide for Linux ` to setup the basic DPDK environment. + +.. note:: + + Some part of fslmc bus code (mc flib - object library) routines are + dual licensed (BSD & GPLv2), however they are used as BSD in DPDK in userspace. + +Pre-Installation Configuration +------------------------------ + +Config File Options +~~~~~~~~~~~~~~~~~~~ + +The following options can be modified in the ``config`` file. +Please note that enabling debugging options may affect system performance. + +- ``CONFIG_RTE_LIBRTE_FSLMC_BUS`` (default ``n``) + + By default it is enabled only for defconfig_arm64-dpaa2-* config. + Toggle compilation of the ``librte_bus_fslmc`` driver. + +- ``CONFIG_RTE_LIBRTE_DPAA2_PMD`` (default ``n``) + + By default it is enabled only for defconfig_arm64-dpaa2-* config. + Toggle compilation of the ``librte_pmd_dpaa2`` driver. + +- ``CONFIG_RTE_LIBRTE_DPAA2_DEBUG_DRIVER`` (default ``n``) + + Toggle display of generic debugging messages + +- ``CONFIG_RTE_LIBRTE_DPAA2_USE_PHYS_IOVA`` (default ``y``) + + Toggle to use physical address vs virtual address for hardware accelerators. + +- ``CONFIG_RTE_LIBRTE_DPAA2_DEBUG_INIT`` (default ``n``) + + Toggle display of initialization related messages. + +- ``CONFIG_RTE_LIBRTE_DPAA2_DEBUG_RX`` (default ``n``) + + Toggle display of receive fast path run-time message + +- ``CONFIG_RTE_LIBRTE_DPAA2_DEBUG_TX`` (default ``n``) + + Toggle display of transmit fast path run-time message + +- ``CONFIG_RTE_LIBRTE_DPAA2_DEBUG_TX_FREE`` (default ``n``) + + Toggle display of transmit fast path buffer free run-time message + +Driver compilation and testing +------------------------------ + +Refer to the document :ref:`compiling and testing a PMD for a NIC ` +for details. + +#. Running testpmd: + + Follow instructions available in the document + :ref:`compiling and testing a PMD for a NIC ` + to run testpmd. + + Example output: + + .. code-block:: console + + ./arm64-dpaa2-linuxapp-gcc/testpmd -c 0xff -n 1 \ + -- -i --portmask=0x3 --nb-cores=1 --no-flush-rx + + ..... + EAL: Registered [pci] bus. + EAL: Registered [fslmc] bus. + EAL: Detected 8 lcore(s) + EAL: Probing VFIO support... + EAL: VFIO support initialized + ..... + PMD: DPAA2: Processing Container = dprc.2 + EAL: fslmc: DPRC contains = 51 devices + EAL: fslmc: Bus scan completed + ..... + Configuring Port 0 (socket 0) + Port 0: 00:00:00:00:00:01 + Configuring Port 1 (socket 0) + Port 1: 00:00:00:00:00:02 + ..... + Checking link statuses... + Port 0 Link Up - speed 10000 Mbps - full-duplex + Port 1 Link Up - speed 10000 Mbps - full-duplex + Done + testpmd> + +Limitations +----------- + +Platform Requirement +~~~~~~~~~~~~~~~~~~~~ +DPAA2 drivers for DPDK can only work on NXP SoCs as listed in the +``Supported DPAA2 SoCs``. + +Maximum packet length +~~~~~~~~~~~~~~~~~~~~~ + +The DPAA2 SoC family support a maximum of a 10240 jumbo frame. The value +is fixed and cannot be changed. So, even when the ``rxmode.max_rx_pkt_len`` +member of ``struct rte_eth_conf`` is set to a value lower than 10240, frames +up to 10240 bytes can still reach the host interface. + +Other Limitations +~~~~~~~~~~~~~~~~~ + +- RSS hash key cannot be modified. +- RSS RETA cannot be configured. +- Secondary process packet I/O is not supported. diff --git a/dpdk/doc/guides/nics/ena.rst b/dpdk/doc/guides/nics/ena.rst index 073b35ae..d19912e9 100644 --- a/dpdk/doc/guides/nics/ena.rst +++ b/dpdk/doc/guides/nics/ena.rst @@ -200,52 +200,23 @@ application runs to completion, the ENA can be detached from igb_uio if necessar Usage example ------------- -This section demonstrates how to launch **testpmd** with Amazon ENA -devices managed by librte_pmd_ena. +Follow instructions available in the document +:ref:`compiling and testing a PMD for a NIC ` to launch +**testpmd** with Amazon ENA devices managed by librte_pmd_ena. -#. Load the kernel modules: +Example output: - .. code-block:: console +.. code-block:: console - modprobe uio - insmod ./x86_64-native-linuxapp-gcc/kmod/igb_uio.ko - - .. note:: - - Currently Amazon ENA PMD driver depends on igb_uio user space I/O kernel module - -#. Mount and request huge pages: - - .. code-block:: console - - mount -t hugetlbfs nodev /mnt/hugepages - echo 1024 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages - -#. Bind UIO driver to ENA device (using provided by DPDK binding tool): - - .. code-block:: console - - ./tools/dpdk-devbind.py --bind=igb_uio 0000:02:00.1 - -#. Start testpmd with basic parameters: - - .. code-block:: console - - ./x86_64-native-linuxapp-gcc/app/testpmd -c 0xf -n 4 -- -i - - Example output: - - .. code-block:: console - - [...] - EAL: PCI device 0000:02:00.1 on NUMA socket -1 - EAL: probe driver: 1d0f:ec20 rte_ena_pmd - EAL: PCI memory mapped at 0x7f9b6c400000 - PMD: eth_ena_dev_init(): Initializing 0:2:0.1 - Interactive-mode selected - Configuring Port 0 (socket 0) - Port 0: 00:00:00:11:00:01 - Checking link statuses... - Port 0 Link Up - speed 10000 Mbps - full-duplex - Done - testpmd> + [...] + EAL: PCI device 0000:02:00.1 on NUMA socket -1 + EAL: probe driver: 1d0f:ec20 rte_ena_pmd + EAL: PCI memory mapped at 0x7f9b6c400000 + PMD: eth_ena_dev_init(): Initializing 0:2:0.1 + Interactive-mode selected + Configuring Port 0 (socket 0) + Port 0: 00:00:00:11:00:01 + Checking link statuses... + Port 0 Link Up - speed 10000 Mbps - full-duplex + Done + testpmd> diff --git a/dpdk/doc/guides/nics/enic.rst b/dpdk/doc/guides/nics/enic.rst index 26692022..cb5ae125 100644 --- a/dpdk/doc/guides/nics/enic.rst +++ b/dpdk/doc/guides/nics/enic.rst @@ -1,5 +1,5 @@ .. BSD LICENSE - Copyright (c) 2015, Cisco Systems, Inc. + Copyright (c) 2017, Cisco Systems, Inc. All rights reserved. Redistribution and use in source and binary forms, with or without @@ -71,9 +71,9 @@ Configuration information - The number of RQs configured in the vNIC should be greater or equal to *twice* the value of the expected nb_rx_q parameter in - the call to rte_eth_dev_configure(). With the addition of rx + the call to rte_eth_dev_configure(). With the addition of Rx scatter, a pair of RQs on the vnic is needed for each receive - queue used by DPDK, even if rx scatter is not being used. + queue used by DPDK, even if Rx scatter is not being used. Having a vNIC with only 1 RQ is not a valid configuration, and will fail with an error message. @@ -99,7 +99,7 @@ Configuration information gives the application the greatest amount of flexibility in its queue configuration. - - *Note*: Since the introduction of rx scatter, for performance + - *Note*: Since the introduction of Rx scatter, for performance reasons, this PMD uses two RQs on the vNIC per receive queue in DPDK. One RQ holds descriptors for the start of a packet the second RQ holds the descriptors for the rest of the fragments of @@ -119,7 +119,140 @@ Configuration information Only one interrupt per vNIC interface should be configured in the UCS manager regardless of the number receive/transmit queues. The ENIC PMD - uses this interrupt to get information about errors in the fast path. + uses this interrupt to get information about link status and errors + in the fast path. + +.. _enic-flow-director: + +Flow director support +--------------------- + +Advanced filtering support was added to 1300 series VIC firmware starting +with version 2.0.13 for C-series UCS servers and version 3.1.2 for UCSM +managed blade servers. In order to enable advanced filtering the 'Advanced +filter' radio button should be enabled via CIMC or UCSM followed by a reboot +of the server. + +With advanced filters, perfect matching of all fields of IPv4, IPv6 headers +as well as TCP, UDP and SCTP L4 headers is available through flow director. +Masking of these fields for partial match is also supported. + +Without advanced filter support, the flow director is limited to IPv4 +perfect filtering of the 5-tuple with no masking of fields supported. + +SR-IOV mode utilization +----------------------- + +UCS blade servers configured with dynamic vNIC connection policies in UCS +manager are capable of supporting assigned devices on virtual machines (VMs) +through a KVM hypervisor. Assigned devices, also known as 'passthrough' +devices, are SR-IOV virtual functions (VFs) on the host which are exposed +to VM instances. + +The Cisco Virtual Machine Fabric Extender (VM-FEX) gives the VM a dedicated +interface on the Fabric Interconnect (FI). Layer 2 switching is done at +the FI. This may eliminate the requirement for software switching on the +host to route intra-host VM traffic. + +Please refer to `Creating a Dynamic vNIC Connection Policy +`_ +for information on configuring SR-IOV Adapter policies using UCS manager. + +Once the policies are in place and the host OS is rebooted, VFs should be +visible on the host, E.g.: + +.. code-block:: console + + # lspci | grep Cisco | grep Ethernet + 0d:00.0 Ethernet controller: Cisco Systems Inc VIC Ethernet NIC (rev a2) + 0d:00.1 Ethernet controller: Cisco Systems Inc VIC SR-IOV VF (rev a2) + 0d:00.2 Ethernet controller: Cisco Systems Inc VIC SR-IOV VF (rev a2) + 0d:00.3 Ethernet controller: Cisco Systems Inc VIC SR-IOV VF (rev a2) + 0d:00.4 Ethernet controller: Cisco Systems Inc VIC SR-IOV VF (rev a2) + 0d:00.5 Ethernet controller: Cisco Systems Inc VIC SR-IOV VF (rev a2) + 0d:00.6 Ethernet controller: Cisco Systems Inc VIC SR-IOV VF (rev a2) + 0d:00.7 Ethernet controller: Cisco Systems Inc VIC SR-IOV VF (rev a2) + +Enable Intel IOMMU on the host and install KVM and libvirt. A VM instance should +be created with an assigned device. When using libvirt, this configuration can +be done within the domain (i.e. VM) config file. For example this entry maps +host VF 0d:00:01 into the VM. + +.. code-block:: console + + + + +
+ + +Alternatively, the configuration can be done in a separate file using the +``network`` keyword. These methods are described in the libvirt documentation for +`Network XML format `_. + +When the VM instance is started, the ENIC KVM driver will bind the host VF to +vfio, complete provisioning on the FI and bring up the link. + +.. note:: + + It is not possible to use a VF directly from the host because it is not + fully provisioned until the hypervisor brings up the VM that it is assigned + to. + +In the VM instance, the VF will now be visible. E.g., here the VF 00:04.0 is +seen on the VM instance and should be available for binding to a DPDK. + +.. code-block:: console + + # lspci | grep Ether + 00:04.0 Ethernet controller: Cisco Systems Inc VIC SR-IOV VF (rev a2) + +Follow the normal DPDK install procedure, binding the VF to either ``igb_uio`` +or ``vfio`` in non-IOMMU mode. + +Please see :ref:`Limitations ` for limitations in +the use of SR-IOV. + +.. _enic-genic-flow-api: + +Generic Flow API support +------------------------ + +Generic Flow API is supported. The baseline support is: + +- **1200 series VICs** + + 5-tuple exact Flow support for 1200 series adapters. This allows: + + - Attributes: ingress + - Items: ipv4, ipv6, udp, tcp (must exactly match src/dst IP + addresses and ports and all must be specified). + - Actions: queue and void + - Selectors: 'is' + +- **1300 series VICS with Advanced filters disabled** + + With advanced filters disabled, an IPv4 or IPv6 item must be specified + in the pattern. + + - Attributes: ingress + - Items: eth, ipv4, ipv6, udp, tcp, vxlan, inner eth, ipv4, ipv6, udp, tcp + - Actions: queue and void + - Selectors: 'is', 'spec' and 'mask'. 'last' is not supported + - In total, up to 64 bytes of mask is allowed across all haeders + +- **1300 series VICS with Advanced filters enabled** + + - Attributes: ingress + - Items: eth, ipv4, ipv6, udp, tcp, vxlan, inner eth, ipv4, ipv6, udp, tcp + - Actions: queue, mark, flag and void + - Selectors: 'is', 'spec' and 'mask'. 'last' is not supported + - In total, up to 64 bytes of mask is allowed across all haeders + +More features may be added in future firmware and new versions of the VIC. +Please refer to the release notes. + +.. _enic_limitations: Limitations ----------- @@ -144,12 +277,49 @@ Limitations vlan_offload |= ETH_VLAN_STRIP_OFFLOAD; rte_eth_dev_set_vlan_offload(port, vlan_offload); -How to build the suite? ------------------------ +- Limited flow director support on 1200 series and 1300 series Cisco VIC + adapters with old firmware. Please see :ref:`enic-flow-director`. + +- Flow director features are not supported on generation 1 Cisco VIC adapters + (M81KR and P81E) + +- **SR-IOV** + + - KVM hypervisor support only. VMware has not been tested. + - Requires VM-FEX, and so is only available on UCS managed servers connected + to Fabric Interconnects. It is not on standalone C-Series servers. + - VF devices are not usable directly from the host. They can only be used + as assigned devices on VM instances. + - Currently, unbind of the ENIC kernel mode driver 'enic.ko' on the VM + instance may hang. As a workaround, enic.ko should blacklisted or removed + from the boot process. + - pci_generic cannot be used as the uio module in the VM. igb_uio or + vfio in non-IOMMU mode can be used. + - The number of RQs in UCSM dynamic vNIC configurations must be at least 2. + - The number of SR-IOV devices is limited to 256. Components on target system + might limit this number to fewer than 256. + +- **Flow API** + + - The number of filters that can be specified with the Generic Flow API is + dependent on how many header fields are being masked. Use 'flow create' in + a loop to determine how many filters your VIC will support (not more than + 1000 for 1300 series VICs). Filter are checked for matching in the order they + were added. Since there currently is no grouping or priority support, + 'catch-all' filters should be added last. + +How to build the suite +---------------------- + The build instructions for the DPDK suite should be followed. By default the ENIC PMD library will be built into the DPDK library. -For configuring and using UIO and VFIO frameworks, please refer the +Refer to the document :ref:`compiling and testing a PMD for a NIC +` for details. + +By default the ENIC PMD library will be built into the DPDK library. + +For configuring and using UIO and VFIO frameworks, please refer to the documentation that comes with DPDK suite. Supported Cisco VIC adapters @@ -169,16 +339,15 @@ ENIC PMD supports all recent generations of Cisco VIC adapters including: - VIC 1385 - VIC 1387 -- Flow director features are not supported on generation 1 Cisco VIC adapters - (M81KR and P81E) - Supported Operating Systems --------------------------- + Any Linux distribution fulfilling the conditions described in Dependencies section of DPDK documentation. Supported features ------------------ + - Unicast, multicast and broadcast transmission and reception - Receive queue polling - Port Hardware Statistics @@ -186,8 +355,7 @@ Supported features - IP checksum offload - Receive side VLAN stripping - Multiple receive and transmit queues -- Flow Director ADD, UPDATE, DELETE, STATS operation support for IPV4 5-TUPLE - flows +- Flow Director ADD, UPDATE, DELETE, STATS operation support IPv4 and IPv6 - Promiscuous mode - Setting RX VLAN (supported via UCSM/CIMC only) - VLAN filtering (supported via UCSM/CIMC only) @@ -195,9 +363,12 @@ Supported features - IPV4, IPV6 and TCP RSS hashing - Scattered Rx - MTU update +- SR-IOV on UCS managed servers connected to Fabric Interconnects. +- Flow API -Known bugs and Unsupported features in this release +Known bugs and unsupported features in this release --------------------------------------------------- + - Signature or flex byte based flow direction - Drop feature of flow direction - VLAN based flow direction @@ -208,6 +379,7 @@ Known bugs and Unsupported features in this release Prerequisites ------------- + - Prepare the system as recommended by DPDK suite. This includes environment variables, hugepages configuration, tool-chains and configuration - Insert vfio-pci kernel module using the command 'modprobe vfio-pci' if the @@ -217,9 +389,8 @@ Prerequisites - DPDK suite should be configured based on the user's decision to use VFIO or UIO framework - If the vNIC device(s) to be used is bound to the kernel mode Ethernet driver - (enic), use 'ifconfig' to bring the interface down. The dpdk-devbind.py tool - can then be used to unbind the device's bus id from the enic kernel mode - driver. + use 'ifconfig' to bring the interface down. The dpdk-devbind.py tool can + then be used to unbind the device's bus id from the ENIC kernel mode driver. - Bind the intended vNIC to vfio-pci in case the user wants ENIC PMD to use VFIO framework using dpdk-devbind.py. - Bind the intended vNIC to igb_uio in case the user wants ENIC PMD to use @@ -250,10 +421,12 @@ libraries and the initialization time of the application. Additional Reference -------------------- + - http://www.cisco.com/c/en/us/products/servers-unified-computing Contact Information ------------------- + Any questions or bugs should be reported to DPDK community and to the ENIC PMD maintainers: diff --git a/dpdk/doc/guides/nics/fail_safe.rst b/dpdk/doc/guides/nics/fail_safe.rst new file mode 100644 index 00000000..c4e3d2e8 --- /dev/null +++ b/dpdk/doc/guides/nics/fail_safe.rst @@ -0,0 +1,221 @@ +.. BSD LICENSE + Copyright 2017 6WIND S.A. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of 6WIND S.A. nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Fail-safe poll mode driver library +================================== + +The Fail-safe poll mode driver library (**librte_pmd_failsafe**) is a virtual +device that allows using any device supporting hotplug (sudden device removal +and plugging on its bus), without modifying other components relying on such +device (application, other PMDs). + +Additionally to the Seamless Hotplug feature, the Fail-safe PMD offers the +ability to redirect operations to secondary devices when the primary has been +removed from the system. + +.. note:: + + The library is enabled by default. You can enable it or disable it manually + by setting the ``CONFIG_RTE_LIBRTE_PMD_FAILSAFE`` configuration option. + +Features +-------- + +The Fail-safe PMD only supports a limited set of features. If you plan to use a +device underneath the Fail-safe PMD with a specific feature, this feature must +be supported by the Fail-safe PMD to avoid throwing any error. + +A notable exception is the device removal feature. The fail-safe PMD being a +virtual device, it cannot currently be removed in the sense of a specific bus +hotplug, like for PCI for example. It will however enable this feature for its +sub-device automatically, detecting those that are capable and register the +relevant callback for such event. + +Check the feature matrix for the complete set of supported features. + +Compilation option +------------------ + +This option can be modified in the ``$RTE_TARGET/build/.config`` file. + +- ``CONFIG_RTE_LIBRTE_PMD_FAILSAFE`` (default **y**) + + Toggle compiling librte_pmd_failsafe. + +Using the Fail-safe PMD from the EAL command line +------------------------------------------------- + +The Fail-safe PMD can be used like most other DPDK virtual devices, by passing a +``--vdev`` parameter to the EAL when starting the application. The device name +must start with the *net_failsafe* prefix, followed by numbers or letters. This +name must be unique for each device. Each fail-safe instance must have at least one +sub-device, up to ``RTE_MAX_ETHPORTS-1``. + +A sub-device can be any legal DPDK device, including possibly another fail-safe +instance. + +Fail-safe command line parameters +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- **dev()** parameter + + This parameter allows the user to define a sub-device. The ```` part of + this parameter must be a valid device definition. It could be the argument + provided to any ``-w`` device specification or the argument that would be + given to a ``--vdev`` parameter (including a fail-safe). + Enclosing the device definition within parenthesis here allows using + additional sub-device parameters if need be. They will be passed on to the + sub-device. + +- **exec()** parameter + + This parameter allows the user to provide a command to the fail-safe PMD to + execute and define a sub-device. + It is done within a regular shell context. + The first line of its output is read by the fail-safe PMD and otherwise + interpreted as if passed by the regular **dev** parameter. + Any other line is discarded. + If the command fail or output an incorrect string, the sub-device is not + initialized. + All commas within the ``shell command`` are replaced by spaces before + executing the command. This helps using scripts to specify devices. + +- **mac** parameter [MAC address] + + This parameter allows the user to set a default MAC address to the fail-safe + and all of its sub-devices. + If no default mac address is provided, the fail-safe PMD will read the MAC + address of the first of its sub-device to be successfully probed and use it as + its default MAC address, trying to set it to all of its other sub-devices. + If no sub-device was successfully probed at initialization, then a random MAC + address is generated, that will be subsequently applied to all sub-device once + they are probed. + +- **hotplug_poll** parameter [UINT64] (default **2000**) + + This parameter allows the user to configure the amount of time in milliseconds + between two slave upkeep round. + +Usage example +~~~~~~~~~~~~~ + +This section shows some example of using **testpmd** with a fail-safe PMD. + +#. To build a PMD and configure DPDK, refer to the document + :ref:`compiling and testing a PMD for a NIC `. + +#. Start testpmd. The slave device should be blacklisted from normal EAL + operations to avoid probing it twice when in PCI blacklist mode. + + .. code-block:: console + + $RTE_TARGET/build/app/testpmd -c 0xff -n 4 \ + --vdev 'net_failsafe0,mac=de:ad:be:ef:01:02,dev(84:00.0),dev(net_ring0)' \ + -b 84:00.0 -b 00:04.0 -- -i + + If the slave device being used is not blacklisted, it will be probed by the + EAL first. When the fail-safe then tries to initialize it the probe operation + fails. + + Note that PCI blacklist mode is the default PCI operating mode. + +#. Alternatively, it can be used alongside any other device in whitelist mode. + + .. code-block:: console + + $RTE_TARGET/build/app/testpmd -c 0xff -n 4 \ + --vdev 'net_failsafe0,mac=de:ad:be:ef:01:02,dev(84:00.0),dev(net_ring0)' \ + -w 81:00.0 -- -i + +#. Start testpmd using a flexible device definition + + .. code-block:: console + + $RTE_TARGET/build/app/testpmd -c 0xff -n 4 --no-pci \ + --vdev='net_failsafe0,exec(echo 84:00.0)' -- -i + +Using the Fail-safe PMD from an application +------------------------------------------- + +This driver strives to be as seamless as possible to existing applications, in +order to propose the hotplug functionality in the easiest way possible. + +Care must be taken, however, to respect the **ether** API concerning device +access, and in particular, using the ``RTE_ETH_FOREACH_DEV`` macro to iterate +over ethernet devices, instead of directly accessing them or by writing one's +own device iterator. + +Plug-in feature +--------------- + +A sub-device can be defined without existing on the system when the fail-safe +PMD is initialized. Upon probing this device, the fail-safe PMD will detect its +absence and postpone its use. It will then register for a periodic check on any +missing sub-device. + +During this time, the fail-safe PMD can be used normally, configured and told to +emit and receive packets. It will store any applied configuration, and try to +apply it upon the probing of its missing sub-device. After this configuration +pass, the new sub-device will be synchronized with other sub-devices, i.e. be +started if the fail-safe PMD has been started by the user before. + +Plug-out feature +---------------- + +A sub-device supporting the device removal event can be removed from its bus at +any time. The fail-safe PMD will register a callback for such event and react +accordingly. It will try to safely stop, close and uninit the sub-device having +emitted this event, allowing it to free its eventual resources. + +Fail-safe glossary +------------------ + +Fallback device : Secondary device + The fail-safe will fail-over onto this device when the preferred device is + absent. + +Preferred device : Primary device + The first declared sub-device in the fail-safe parameters. + When this device is plugged, it is always used as emitting device. + It is the main sub-device and is used as target for configuration + operations if there is any ambiguity. + +Upkeep round + Periodical process when slaves are serviced. Each devices having a state + different to that of the fail-safe device itself, is synchronized with it. + Additionally, each slave having the remove flag set are cleaned-up. + +Slave + In the context of the fail-safe PMD, synonymous to sub-device. + +Sub-device + A device being utilized by the fail-safe PMD. + This is another PMD running underneath the fail-safe PMD. + Any sub-device can disappear at any time. The fail-safe will ensure + that the device removal happens gracefully. diff --git a/dpdk/doc/guides/nics/features.rst b/dpdk/doc/guides/nics/features.rst new file mode 100644 index 00000000..d5bf38a2 --- /dev/null +++ b/dpdk/doc/guides/nics/features.rst @@ -0,0 +1,942 @@ +.. BSD LICENSE + Copyright(c) 2017 Intel Corporation. All rights reserved. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Features Overview +================= + +This section explains the supported features that are listed in the +:doc:`overview`. + +As a guide to implementers it also shows the structs where the features are +defined and the APIs that can be use to get/set the values. + +Following tags used for feature details, these are from driver point of view: + +``[uses]`` : Driver uses some kind of input from the application. + +``[implements]`` : Driver implements a functionality. + +``[provides]`` : Driver provides some kind of data to the application. It is possible +to provide data by implementing some function, but "provides" is used +for cases where provided data can't be represented simply by a function. + +``[related]`` : Related API with that feature. + + +.. _nic_features_speed_capabilities: + +Speed capabilities +------------------ + +Supports getting the speed capabilities that the current device is capable of. + +* **[provides] rte_eth_dev_info**: ``speed_capa:ETH_LINK_SPEED_*``. +* **[related] API**: ``rte_eth_dev_info_get()``. + + +.. _nic_features_link_status: + +Link status +----------- + +Supports getting the link speed, duplex mode and link state (up/down). + +* **[implements] eth_dev_ops**: ``link_update``. +* **[implements] rte_eth_dev_data**: ``dev_link``. +* **[related] API**: ``rte_eth_link_get()``, ``rte_eth_link_get_nowait()``. + + +.. _nic_features_link_status_event: + +Link status event +----------------- + +Supports Link Status Change interrupts. + +* **[uses] user config**: ``dev_conf.intr_conf.lsc``. +* **[uses] rte_eth_dev_data**: ``dev_flags:RTE_ETH_DEV_INTR_LSC``. +* **[uses] rte_eth_event_type**: ``RTE_ETH_EVENT_INTR_LSC``. +* **[implements] rte_eth_dev_data**: ``dev_link``. +* **[provides] rte_pci_driver.drv_flags**: ``RTE_PCI_DRV_INTR_LSC``. +* **[related] API**: ``rte_eth_link_get()``, ``rte_eth_link_get_nowait()``. + + +.. _nic_features_removal_event: + +Removal event +------------- + +Supports device removal interrupts. + +* **[uses] user config**: ``dev_conf.intr_conf.rmv``. +* **[uses] rte_eth_dev_data**: ``dev_flags:RTE_ETH_DEV_INTR_RMV``. +* **[uses] rte_eth_event_type**: ``RTE_ETH_EVENT_INTR_RMV``. +* **[provides] rte_pci_driver.drv_flags**: ``RTE_PCI_DRV_INTR_RMV``. + + +.. _nic_features_queue_status_event: + +Queue status event +------------------ + +Supports queue enable/disable events. + +* **[uses] rte_eth_event_type**: ``RTE_ETH_EVENT_QUEUE_STATE``. + + +.. _nic_features_rx_interrupt: + +Rx interrupt +------------ + +Supports Rx interrupts. + +* **[uses] user config**: ``dev_conf.intr_conf.rxq``. +* **[implements] eth_dev_ops**: ``rx_queue_intr_enable``, ``rx_queue_intr_disable``. +* **[related] API**: ``rte_eth_dev_rx_intr_enable()``, ``rte_eth_dev_rx_intr_disable()``. + + +.. _nic_features_lock-free_tx_queue: + +Lock-free Tx queue +------------------ + +If a PMD advertises DEV_TX_OFFLOAD_MT_LOCKFREE capable, multiple threads can +invoke rte_eth_tx_burst() concurrently on the same Tx queue without SW lock. + +* **[uses] rte_eth_txconf,rte_eth_txmode**: ``offloads:DEV_TX_OFFLOAD_MT_LOCKFREE``. +* **[provides] rte_eth_dev_info**: ``tx_offload_capa,tx_queue_offload_capa:DEV_TX_OFFLOAD_MT_LOCKFREE``. +* **[related] API**: ``rte_eth_tx_burst()``. + + +.. _nic_features_fast_mbuf_free: + +Fast mbuf free +-------------- + +Supports optimization for fast release of mbufs following successful Tx. +Requires that per queue, all mbufs come from the same mempool and has refcnt = 1. + +* **[uses] rte_eth_txconf,rte_eth_txmode**: ``offloads:DEV_TX_OFFLOAD_MBUF_FAST_FREE``. +* **[provides] rte_eth_dev_info**: ``tx_offload_capa,tx_queue_offload_capa:DEV_TX_OFFLOAD_MBUF_FAST_FREE``. + + +.. _nic_features_free_tx_mbuf_on_demand: + +Free Tx mbuf on demand +---------------------- + +Supports freeing consumed buffers on a Tx ring. + +* **[implements] eth_dev_ops**: ``tx_done_cleanup``. +* **[related] API**: ``rte_eth_tx_done_cleanup()``. + + +.. _nic_features_queue_start_stop: + +Queue start/stop +---------------- + +Supports starting/stopping a specific Rx/Tx queue of a port. + +* **[implements] eth_dev_ops**: ``rx_queue_start``, ``rx_queue_stop``, ``tx_queue_start``, + ``tx_queue_stop``. +* **[related] API**: ``rte_eth_dev_rx_queue_start()``, ``rte_eth_dev_rx_queue_stop()``, + ``rte_eth_dev_tx_queue_start()``, ``rte_eth_dev_tx_queue_stop()``. + + +.. _nic_features_mtu_update: + +MTU update +---------- + +Supports updating port MTU. + +* **[implements] eth_dev_ops**: ``mtu_set``. +* **[implements] rte_eth_dev_data**: ``mtu``. +* **[provides] rte_eth_dev_info**: ``max_rx_pktlen``. +* **[related] API**: ``rte_eth_dev_set_mtu()``, ``rte_eth_dev_get_mtu()``. + + +.. _nic_features_jumbo_frame: + +Jumbo frame +----------- + +Supports Rx jumbo frames. + +* **[uses] rte_eth_rxconf,rte_eth_rxmode**: ``offloads:DEV_RX_OFFLOAD_JUMBO_FRAME``. + ``dev_conf.rxmode.max_rx_pkt_len``. +* **[related] rte_eth_dev_info**: ``max_rx_pktlen``. +* **[related] API**: ``rte_eth_dev_set_mtu()``. + + +.. _nic_features_scattered_rx: + +Scattered Rx +------------ + +Supports receiving segmented mbufs. + +* **[uses] rte_eth_rxconf,rte_eth_rxmode**: ``offloads:DEV_RX_OFFLOAD_SCATTER``. +* **[implements] datapath**: ``Scattered Rx function``. +* **[implements] rte_eth_dev_data**: ``scattered_rx``. +* **[provides] eth_dev_ops**: ``rxq_info_get:scattered_rx``. +* **[related] eth_dev_ops**: ``rx_pkt_burst``. + + +.. _nic_features_lro: + +LRO +--- + +Supports Large Receive Offload. + +* **[uses] rte_eth_rxconf,rte_eth_rxmode**: ``offloads:DEV_RX_OFFLOAD_TCP_LRO``. +* **[implements] datapath**: ``LRO functionality``. +* **[implements] rte_eth_dev_data**: ``lro``. +* **[provides] mbuf**: ``mbuf.ol_flags:PKT_RX_LRO``, ``mbuf.tso_segsz``. +* **[provides] rte_eth_dev_info**: ``rx_offload_capa,rx_queue_offload_capa:DEV_RX_OFFLOAD_TCP_LRO``. + + +.. _nic_features_tso: + +TSO +--- + +Supports TCP Segmentation Offloading. + +* **[uses] rte_eth_txconf,rte_eth_txmode**: ``offloads:DEV_TX_OFFLOAD_TCP_TSO``. +* **[uses] rte_eth_desc_lim**: ``nb_seg_max``, ``nb_mtu_seg_max``. +* **[uses] mbuf**: ``mbuf.ol_flags:PKT_TX_TCP_SEG``. +* **[uses] mbuf**: ``mbuf.tso_segsz``, ``mbuf.l2_len``, ``mbuf.l3_len``, ``mbuf.l4_len``. +* **[implements] datapath**: ``TSO functionality``. +* **[provides] rte_eth_dev_info**: ``tx_offload_capa,tx_queue_offload_capa:DEV_TX_OFFLOAD_TCP_TSO,DEV_TX_OFFLOAD_UDP_TSO``. + + +.. _nic_features_promiscuous_mode: + +Promiscuous mode +---------------- + +Supports enabling/disabling promiscuous mode for a port. + +* **[implements] eth_dev_ops**: ``promiscuous_enable``, ``promiscuous_disable``. +* **[implements] rte_eth_dev_data**: ``promiscuous``. +* **[related] API**: ``rte_eth_promiscuous_enable()``, ``rte_eth_promiscuous_disable()``, + ``rte_eth_promiscuous_get()``. + + +.. _nic_features_allmulticast_mode: + +Allmulticast mode +----------------- + +Supports enabling/disabling receiving multicast frames. + +* **[implements] eth_dev_ops**: ``allmulticast_enable``, ``allmulticast_disable``. +* **[implements] rte_eth_dev_data**: ``all_multicast``. +* **[related] API**: ``rte_eth_allmulticast_enable()``, + ``rte_eth_allmulticast_disable()``, ``rte_eth_allmulticast_get()``. + + +.. _nic_features_unicast_mac_filter: + +Unicast MAC filter +------------------ + +Supports adding MAC addresses to enable whitelist filtering to accept packets. + +* **[implements] eth_dev_ops**: ``mac_addr_set``, ``mac_addr_add``, ``mac_addr_remove``. +* **[implements] rte_eth_dev_data**: ``mac_addrs``. +* **[related] API**: ``rte_eth_dev_default_mac_addr_set()``, + ``rte_eth_dev_mac_addr_add()``, ``rte_eth_dev_mac_addr_remove()``, + ``rte_eth_macaddr_get()``. + + +.. _nic_features_multicast_mac_filter: + +Multicast MAC filter +-------------------- + +Supports setting multicast addresses to filter. + +* **[implements] eth_dev_ops**: ``set_mc_addr_list``. +* **[related] API**: ``rte_eth_dev_set_mc_addr_list()``. + + +.. _nic_features_rss_hash: + +RSS hash +-------- + +Supports RSS hashing on RX. + +* **[uses] user config**: ``dev_conf.rxmode.mq_mode`` = ``ETH_MQ_RX_RSS_FLAG``. +* **[uses] user config**: ``dev_conf.rx_adv_conf.rss_conf``. +* **[provides] rte_eth_dev_info**: ``flow_type_rss_offloads``. +* **[provides] mbuf**: ``mbuf.ol_flags:PKT_RX_RSS_HASH``, ``mbuf.rss``. + + +.. _nic_features_rss_key_update: + +RSS key update +-------------- + +Supports configuration of Receive Side Scaling (RSS) hash computation. Updating +Receive Side Scaling (RSS) hash key. + +* **[implements] eth_dev_ops**: ``rss_hash_update``, ``rss_hash_conf_get``. +* **[provides] rte_eth_dev_info**: ``hash_key_size``. +* **[related] API**: ``rte_eth_dev_rss_hash_update()``, + ``rte_eth_dev_rss_hash_conf_get()``. + + +.. _nic_features_rss_reta_update: + +RSS reta update +--------------- + +Supports updating Redirection Table of the Receive Side Scaling (RSS). + +* **[implements] eth_dev_ops**: ``reta_update``, ``reta_query``. +* **[provides] rte_eth_dev_info**: ``reta_size``. +* **[related] API**: ``rte_eth_dev_rss_reta_update()``, ``rte_eth_dev_rss_reta_query()``. + + +.. _nic_features_vmdq: + +VMDq +---- + +Supports Virtual Machine Device Queues (VMDq). + +* **[uses] user config**: ``dev_conf.rxmode.mq_mode`` = ``ETH_MQ_RX_VMDQ_FLAG``. +* **[uses] user config**: ``dev_conf.rx_adv_conf.vmdq_dcb_conf``. +* **[uses] user config**: ``dev_conf.rx_adv_conf.vmdq_rx_conf``. +* **[uses] user config**: ``dev_conf.tx_adv_conf.vmdq_dcb_tx_conf``. +* **[uses] user config**: ``dev_conf.tx_adv_conf.vmdq_tx_conf``. + + +.. _nic_features_sriov: + +SR-IOV +------ + +Driver supports creating Virtual Functions. + +* **[implements] rte_eth_dev_data**: ``sriov``. + +.. _nic_features_dcb: + +DCB +--- + +Supports Data Center Bridging (DCB). + +* **[uses] user config**: ``dev_conf.rxmode.mq_mode`` = ``ETH_MQ_RX_DCB_FLAG``. +* **[uses] user config**: ``dev_conf.rx_adv_conf.vmdq_dcb_conf``. +* **[uses] user config**: ``dev_conf.rx_adv_conf.dcb_rx_conf``. +* **[uses] user config**: ``dev_conf.tx_adv_conf.vmdq_dcb_tx_conf``. +* **[uses] user config**: ``dev_conf.tx_adv_conf.vmdq_tx_conf``. +* **[implements] eth_dev_ops**: ``get_dcb_info``. +* **[related] API**: ``rte_eth_dev_get_dcb_info()``. + + +.. _nic_features_vlan_filter: + +VLAN filter +----------- + +Supports filtering of a VLAN Tag identifier. + +* **[uses] rte_eth_rxconf,rte_eth_rxmode**: ``offloads:DEV_RX_OFFLOAD_VLAN_FILTER``. +* **[implements] eth_dev_ops**: ``vlan_filter_set``. +* **[related] API**: ``rte_eth_dev_vlan_filter()``. + + +.. _nic_features_ethertype_filter: + +Ethertype filter +---------------- + +Supports filtering on Ethernet type. + +* **[implements] eth_dev_ops**: ``filter_ctrl:RTE_ETH_FILTER_ETHERTYPE``. +* **[related] API**: ``rte_eth_dev_filter_ctrl()``, ``rte_eth_dev_filter_supported()``. + +.. _nic_features_ntuple_filter: + +N-tuple filter +-------------- + +Supports filtering on N-tuple values. + +* **[implements] eth_dev_ops**: ``filter_ctrl:RTE_ETH_FILTER_NTUPLE``. +* **[related] API**: ``rte_eth_dev_filter_ctrl()``, ``rte_eth_dev_filter_supported()``. + + +.. _nic_features_syn_filter: + +SYN filter +---------- + +Supports TCP syn filtering. + +* **[implements] eth_dev_ops**: ``filter_ctrl:RTE_ETH_FILTER_SYN``. +* **[related] API**: ``rte_eth_dev_filter_ctrl()``, ``rte_eth_dev_filter_supported()``. + + +.. _nic_features_tunnel_filter: + +Tunnel filter +------------- + +Supports tunnel filtering. + +* **[implements] eth_dev_ops**: ``filter_ctrl:RTE_ETH_FILTER_TUNNEL``. +* **[related] API**: ``rte_eth_dev_filter_ctrl()``, ``rte_eth_dev_filter_supported()``. + + +.. _nic_features_flexible_filter: + +Flexible filter +--------------- + +Supports a flexible (non-tuple or Ethertype) filter. + +* **[implements] eth_dev_ops**: ``filter_ctrl:RTE_ETH_FILTER_FLEXIBLE``. +* **[related] API**: ``rte_eth_dev_filter_ctrl()``, ``rte_eth_dev_filter_supported()``. + + +.. _nic_features_hash_filter: + +Hash filter +----------- + +Supports Hash filtering. + +* **[implements] eth_dev_ops**: ``filter_ctrl:RTE_ETH_FILTER_HASH``. +* **[related] API**: ``rte_eth_dev_filter_ctrl()``, ``rte_eth_dev_filter_supported()``. + + +.. _nic_features_flow_director: + +Flow director +------------- + +Supports Flow Director style filtering to queues. + +* **[implements] eth_dev_ops**: ``filter_ctrl:RTE_ETH_FILTER_FDIR``. +* **[provides] mbuf**: ``mbuf.ol_flags:`` ``PKT_RX_FDIR``, ``PKT_RX_FDIR_ID``, + ``PKT_RX_FDIR_FLX``. +* **[related] API**: ``rte_eth_dev_filter_ctrl()``, ``rte_eth_dev_filter_supported()``. + + +.. _nic_features_flow_control: + +Flow control +------------ + +Supports configuring link flow control. + +* **[implements] eth_dev_ops**: ``flow_ctrl_get``, ``flow_ctrl_set``, + ``priority_flow_ctrl_set``. +* **[related] API**: ``rte_eth_dev_flow_ctrl_get()``, ``rte_eth_dev_flow_ctrl_set()``, + ``rte_eth_dev_priority_flow_ctrl_set()``. + + +.. _nic_features_flow_api: + +Flow API +-------- + +Supports the DPDK Flow API for generic filtering. + +* **[implements] eth_dev_ops**: ``filter_ctrl:RTE_ETH_FILTER_GENERIC``. +* **[implements] rte_flow_ops**: ``All``. + + +.. _nic_features_rate_limitation: + +Rate limitation +--------------- + +Supports Tx rate limitation for a queue. + +* **[implements] eth_dev_ops**: ``set_queue_rate_limit``. +* **[related] API**: ``rte_eth_set_queue_rate_limit()``. + + +.. _nic_features_traffic_mirroring: + +Traffic mirroring +----------------- + +Supports adding traffic mirroring rules. + +* **[implements] eth_dev_ops**: ``mirror_rule_set``, ``mirror_rule_reset``. +* **[related] API**: ``rte_eth_mirror_rule_set()``, ``rte_eth_mirror_rule_reset()``. + + +.. _nic_features_inline_crypto_doc: + +Inline crypto +------------- + +Supports inline crypto processing (eg. inline IPsec). See Security library and PMD documentation for more details. + +* **[uses] rte_eth_rxconf,rte_eth_rxmode**: ``offloads:DEV_RX_OFFLOAD_SECURITY``, +* **[uses] rte_eth_txconf,rte_eth_txmode**: ``offloads:DEV_TX_OFFLOAD_SECURITY``. +* **[implements] rte_security_ops**: ``session_create``, ``session_update``, + ``session_stats_get``, ``session_destroy``, ``set_pkt_metadata``, ``capabilities_get``. +* **[provides] rte_eth_dev_info**: ``rx_offload_capa,rx_queue_offload_capa:DEV_RX_OFFLOAD_SECURITY``, + ``tx_offload_capa,tx_queue_offload_capa:DEV_TX_OFFLOAD_SECURITY``. +* **[provides] mbuf**: ``mbuf.ol_flags:PKT_RX_SEC_OFFLOAD``, + ``mbuf.ol_flags:PKT_TX_SEC_OFFLOAD``, ``mbuf.ol_flags:PKT_RX_SEC_OFFLOAD_FAILED``. + + +.. _nic_features_crc_offload: + +CRC offload +----------- + +Supports CRC stripping by hardware. + +* **[uses] rte_eth_rxconf,rte_eth_rxmode**: ``offloads:DEV_RX_OFFLOAD_CRC_STRIP``. + + +.. _nic_features_vlan_offload: + +VLAN offload +------------ + +Supports VLAN offload to hardware. + +* **[uses] rte_eth_rxconf,rte_eth_rxmode**: ``offloads:DEV_RX_OFFLOAD_VLAN_STRIP,DEV_RX_OFFLOAD_VLAN_FILTER,DEV_RX_OFFLOAD_VLAN_EXTEND``. +* **[uses] rte_eth_txconf,rte_eth_txmode**: ``offloads:DEV_TX_OFFLOAD_VLAN_INSERT``. +* **[implements] eth_dev_ops**: ``vlan_offload_set``. +* **[provides] mbuf**: ``mbuf.ol_flags:PKT_RX_VLAN_STRIPPED``, ``mbuf.vlan_tci``. +* **[provides] rte_eth_dev_info**: ``rx_offload_capa,rx_queue_offload_capa:DEV_RX_OFFLOAD_VLAN_STRIP``, + ``tx_offload_capa,tx_queue_offload_capa:DEV_TX_OFFLOAD_VLAN_INSERT``. +* **[related] API**: ``rte_eth_dev_set_vlan_offload()``, + ``rte_eth_dev_get_vlan_offload()``. + + +.. _nic_features_qinq_offload: + +QinQ offload +------------ + +Supports QinQ (queue in queue) offload. + +* **[uses] rte_eth_rxconf,rte_eth_rxmode**: ``offloads:DEV_RX_OFFLOAD_QINQ_STRIP``. +* **[uses] rte_eth_txconf,rte_eth_txmode**: ``offloads:DEV_TX_OFFLOAD_QINQ_INSERT``. +* **[uses] mbuf**: ``mbuf.ol_flags:PKT_TX_QINQ_PKT``. +* **[provides] mbuf**: ``mbuf.ol_flags:PKT_RX_QINQ_STRIPPED``, ``mbuf.vlan_tci``, + ``mbuf.vlan_tci_outer``. +* **[provides] rte_eth_dev_info**: ``rx_offload_capa,rx_queue_offload_capa:DEV_RX_OFFLOAD_QINQ_STRIP``, + ``tx_offload_capa,tx_queue_offload_capa:DEV_TX_OFFLOAD_QINQ_INSERT``. + + +.. _nic_features_l3_checksum_offload: + +L3 checksum offload +------------------- + +Supports L3 checksum offload. + +* **[uses] rte_eth_rxconf,rte_eth_rxmode**: ``offloads:DEV_RX_OFFLOAD_IPV4_CKSUM``. +* **[uses] rte_eth_txconf,rte_eth_txmode**: ``offloads:DEV_TX_OFFLOAD_IPV4_CKSUM``. +* **[uses] mbuf**: ``mbuf.ol_flags:PKT_TX_IP_CKSUM``, + ``mbuf.ol_flags:PKT_TX_IPV4`` | ``PKT_TX_IPV6``. +* **[provides] mbuf**: ``mbuf.ol_flags:PKT_RX_IP_CKSUM_UNKNOWN`` | + ``PKT_RX_IP_CKSUM_BAD`` | ``PKT_RX_IP_CKSUM_GOOD`` | + ``PKT_RX_IP_CKSUM_NONE``. +* **[provides] rte_eth_dev_info**: ``rx_offload_capa,rx_queue_offload_capa:DEV_RX_OFFLOAD_IPV4_CKSUM``, + ``tx_offload_capa,tx_queue_offload_capa:DEV_TX_OFFLOAD_IPV4_CKSUM``. + + +.. _nic_features_l4_checksum_offload: + +L4 checksum offload +------------------- + +Supports L4 checksum offload. + +* **[uses] rte_eth_rxconf,rte_eth_rxmode**: ``offloads:DEV_RX_OFFLOAD_UDP_CKSUM,DEV_RX_OFFLOAD_TCP_CKSUM``. +* **[uses] rte_eth_txconf,rte_eth_txmode**: ``offloads:DEV_TX_OFFLOAD_UDP_CKSUM,DEV_TX_OFFLOAD_TCP_CKSUM,DEV_TX_OFFLOAD_SCTP_CKSUM``. +* **[uses] user config**: ``dev_conf.rxmode.hw_ip_checksum``. +* **[uses] mbuf**: ``mbuf.ol_flags:PKT_TX_IPV4`` | ``PKT_TX_IPV6``, + ``mbuf.ol_flags:PKT_TX_L4_NO_CKSUM`` | ``PKT_TX_TCP_CKSUM`` | + ``PKT_TX_SCTP_CKSUM`` | ``PKT_TX_UDP_CKSUM``. +* **[provides] mbuf**: ``mbuf.ol_flags:PKT_RX_L4_CKSUM_UNKNOWN`` | + ``PKT_RX_L4_CKSUM_BAD`` | ``PKT_RX_L4_CKSUM_GOOD`` | + ``PKT_RX_L4_CKSUM_NONE``. +* **[provides] rte_eth_dev_info**: ``rx_offload_capa,rx_queue_offload_capa:DEV_RX_OFFLOAD_UDP_CKSUM,DEV_RX_OFFLOAD_TCP_CKSUM``, + ``tx_offload_capa,tx_queue_offload_capa:DEV_TX_OFFLOAD_UDP_CKSUM,DEV_TX_OFFLOAD_TCP_CKSUM,DEV_TX_OFFLOAD_SCTP_CKSUM``. + +.. _nic_features_hw_timestamp: + +Timestamp offload +----------------- + +Supports Timestamp. + +* **[uses] rte_eth_rxconf,rte_eth_rxmode**: ``offloads:DEV_RX_OFFLOAD_TIMESTAMP``. +* **[provides] mbuf**: ``mbuf.ol_flags:PKT_RX_TIMESTAMP``. +* **[provides] mbuf**: ``mbuf.timestamp``. +* **[provides] rte_eth_dev_info**: ``rx_offload_capa,rx_queue_offload_capa: DEV_RX_OFFLOAD_TIMESTAMP``. + +.. _nic_features_macsec_offload: + +MACsec offload +-------------- + +Supports MACsec. + +* **[uses] rte_eth_rxconf,rte_eth_rxmode**: ``offloads:DEV_RX_OFFLOAD_MACSEC_STRIP``. +* **[uses] rte_eth_txconf,rte_eth_txmode**: ``offloads:DEV_TX_OFFLOAD_MACSEC_INSERT``. +* **[uses] mbuf**: ``mbuf.ol_flags:PKT_TX_MACSEC``. +* **[provides] rte_eth_dev_info**: ``rx_offload_capa,rx_queue_offload_capa:DEV_RX_OFFLOAD_MACSEC_STRIP``, + ``tx_offload_capa,tx_queue_offload_capa:DEV_TX_OFFLOAD_MACSEC_INSERT``. + + +.. _nic_features_inner_l3_checksum: + +Inner L3 checksum +----------------- + +Supports inner packet L3 checksum. + +* **[uses] rte_eth_rxconf,rte_eth_rxmode**: ``offloads:DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM``. +* **[uses] rte_eth_txconf,rte_eth_txmode**: ``offloads:DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM``. +* **[uses] mbuf**: ``mbuf.ol_flags:PKT_TX_IP_CKSUM``, + ``mbuf.ol_flags:PKT_TX_IPV4`` | ``PKT_TX_IPV6``, + ``mbuf.ol_flags:PKT_TX_OUTER_IP_CKSUM``, + ``mbuf.ol_flags:PKT_TX_OUTER_IPV4`` | ``PKT_TX_OUTER_IPV6``. +* **[uses] mbuf**: ``mbuf.outer_l2_len``, ``mbuf.outer_l3_len``. +* **[provides] mbuf**: ``mbuf.ol_flags:PKT_RX_EIP_CKSUM_BAD``. +* **[provides] rte_eth_dev_info**: ``rx_offload_capa,rx_queue_offload_capa:DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM``, + ``tx_offload_capa,tx_queue_offload_capa:DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM``. + + +.. _nic_features_inner_l4_checksum: + +Inner L4 checksum +----------------- + +Supports inner packet L4 checksum. + + +.. _nic_features_packet_type_parsing: + +Packet type parsing +------------------- + +Supports packet type parsing and returns a list of supported types. + +* **[implements] eth_dev_ops**: ``dev_supported_ptypes_get``. +* **[related] API**: ``rte_eth_dev_get_supported_ptypes()``. + + +.. _nic_features_timesync: + +Timesync +-------- + +Supports IEEE1588/802.1AS timestamping. + +* **[implements] eth_dev_ops**: ``timesync_enable``, ``timesync_disable`` + ``timesync_read_rx_timestamp``, ``timesync_read_tx_timestamp``, + ``timesync_adjust_time``, ``timesync_read_time``, ``timesync_write_time``. +* **[related] API**: ``rte_eth_timesync_enable()``, ``rte_eth_timesync_disable()``, + ``rte_eth_timesync_read_rx_timestamp()``, + ``rte_eth_timesync_read_tx_timestamp``, ``rte_eth_timesync_adjust_time()``, + ``rte_eth_timesync_read_time()``, ``rte_eth_timesync_write_time()``. + + +.. _nic_features_rx_descriptor_status: + +Rx descriptor status +-------------------- + +Supports check the status of a Rx descriptor. When ``rx_descriptor_status`` is +used, status can be "Available", "Done" or "Unavailable". When +``rx_descriptor_done`` is used, status can be "DD bit is set" or "DD bit is +not set". + +* **[implements] eth_dev_ops**: ``rx_descriptor_status``. +* **[related] API**: ``rte_eth_rx_descriptor_status()``. +* **[implements] eth_dev_ops**: ``rx_descriptor_done``. +* **[related] API**: ``rte_eth_rx_descriptor_done()``. + + +.. _nic_features_tx_descriptor_status: + +Tx descriptor status +-------------------- + +Supports checking the status of a Tx descriptor. Status can be "Full", "Done" +or "Unavailable." + +* **[implements] eth_dev_ops**: ``tx_descriptor_status``. +* **[related] API**: ``rte_eth_tx_descriptor_status()``. + + +.. _nic_features_basic_stats: + +Basic stats +----------- + +Support basic statistics such as: ipackets, opackets, ibytes, obytes, +imissed, ierrors, oerrors, rx_nombuf. + +And per queue stats: q_ipackets, q_opackets, q_ibytes, q_obytes, q_errors. + +These apply to all drivers. + +* **[implements] eth_dev_ops**: ``stats_get``, ``stats_reset``. +* **[related] API**: ``rte_eth_stats_get``, ``rte_eth_stats_reset()``. + + +.. _nic_features_extended_stats: + +Extended stats +-------------- + +Supports Extended Statistics, changes from driver to driver. + +* **[implements] eth_dev_ops**: ``xstats_get``, ``xstats_reset``, ``xstats_get_names``. +* **[implements] eth_dev_ops**: ``xstats_get_by_id``, ``xstats_get_names_by_id``. +* **[related] API**: ``rte_eth_xstats_get()``, ``rte_eth_xstats_reset()``, + ``rte_eth_xstats_get_names``, ``rte_eth_xstats_get_by_id()``, + ``rte_eth_xstats_get_names_by_id()``, ``rte_eth_xstats_get_id_by_name()``. + + +.. _nic_features_stats_per_queue: + +Stats per queue +--------------- + +Supports configuring per-queue stat counter mapping. + +* **[implements] eth_dev_ops**: ``queue_stats_mapping_set``. +* **[related] API**: ``rte_eth_dev_set_rx_queue_stats_mapping()``, + ``rte_eth_dev_set_tx_queue_stats_mapping()``. + + +.. _nic_features_fw_version: + +FW version +---------- + +Supports getting device hardware firmware information. + +* **[implements] eth_dev_ops**: ``fw_version_get``. +* **[related] API**: ``rte_eth_dev_fw_version_get()``. + + +.. _nic_features_eeprom_dump: + +EEPROM dump +----------- + +Supports getting/setting device eeprom data. + +* **[implements] eth_dev_ops**: ``get_eeprom_length``, ``get_eeprom``, ``set_eeprom``. +* **[related] API**: ``rte_eth_dev_get_eeprom_length()``, ``rte_eth_dev_get_eeprom()``, + ``rte_eth_dev_set_eeprom()``. + + +.. _nic_features_register_dump: + +Registers dump +-------------- + +Supports retrieving device registers and registering attributes (number of +registers and register size). + +* **[implements] eth_dev_ops**: ``get_reg``. +* **[related] API**: ``rte_eth_dev_get_reg_info()``. + + +.. _nic_features_led: + +LED +--- + +Supports turning on/off a software controllable LED on a device. + +* **[implements] eth_dev_ops**: ``dev_led_on``, ``dev_led_off``. +* **[related] API**: ``rte_eth_led_on()``, ``rte_eth_led_off()``. + + +.. _nic_features_multiprocess_aware: + +Multiprocess aware +------------------ + +Driver can be used for primary-secondary process model. + + +.. _nic_features_bsd_nic_uio: + +BSD nic_uio +----------- + +BSD ``nic_uio`` module supported. + + +.. _nic_features_linux_uio: + +Linux UIO +--------- + +Works with ``igb_uio`` kernel module. + +* **[provides] RTE_PMD_REGISTER_KMOD_DEP**: ``igb_uio``. + +.. _nic_features_linux_vfio: + +Linux VFIO +---------- + +Works with ``vfio-pci`` kernel module. + +* **[provides] RTE_PMD_REGISTER_KMOD_DEP**: ``vfio-pci``. + +.. _nic_features_other_kdrv: + +Other kdrv +---------- + +Kernel module other than above ones supported. + + +.. _nic_features_armv7: + +ARMv7 +----- + +Support armv7 architecture. + +Use ``defconfig_arm-armv7a-*-*``. + + +.. _nic_features_armv8: + +ARMv8 +----- + +Support armv8a (64bit) architecture. + +Use ``defconfig_arm64-armv8a-*-*`` + + +.. _nic_features_power8: + +Power8 +------ + +Support PowerPC architecture. + +Use ``defconfig_ppc_64-power8-*-*`` + +.. _nic_features_x86-32: + +x86-32 +------ + +Support 32bits x86 architecture. + +Use ``defconfig_x86_x32-native-*-*`` and ``defconfig_i686-native-*-*``. + + +.. _nic_features_x86-64: + +x86-64 +------ + +Support 64bits x86 architecture. + +Use ``defconfig_x86_64-native-*-*``. + + +.. _nic_features_usage_doc: + +Usage doc +--------- + +Documentation describes usage. + +See ``doc/guides/nics/*.rst`` + + +.. _nic_features_design_doc: + +Design doc +---------- + +Documentation describes design. + +See ``doc/guides/nics/*.rst``. + + +.. _nic_features_perf_doc: + +Perf doc +-------- + +Documentation describes performance values. + +See ``dpdk.org/doc/perf/*``. + + + +.. _nic_features_other: + +Other dev ops not represented by a Feature +------------------------------------------ + +* ``rxq_info_get`` +* ``txq_info_get`` +* ``vlan_tpid_set`` +* ``vlan_strip_queue_set`` +* ``vlan_pvid_set`` +* ``rx_queue_count`` +* ``l2_tunnel_offload_set`` +* ``uc_hash_table_set`` +* ``uc_all_hash_table_set`` +* ``udp_tunnel_port_add`` +* ``udp_tunnel_port_del`` +* ``l2_tunnel_eth_type_conf`` +* ``l2_tunnel_offload_set`` +* ``tx_pkt_prepare`` diff --git a/dpdk/doc/guides/nics/features/afpacket.ini b/dpdk/doc/guides/nics/features/afpacket.ini new file mode 100644 index 00000000..99f87ab6 --- /dev/null +++ b/dpdk/doc/guides/nics/features/afpacket.ini @@ -0,0 +1,6 @@ +; +; Supported features of the 'afpacket' network poll mode driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] diff --git a/dpdk/doc/guides/nics/features/ark.ini b/dpdk/doc/guides/nics/features/ark.ini new file mode 100644 index 00000000..ec8a2b99 --- /dev/null +++ b/dpdk/doc/guides/nics/features/ark.ini @@ -0,0 +1,15 @@ +; +; Supported features of the 'ark' poll mode driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] +Speed capabilities = Y +Queue start/stop = Y +Jumbo frame = Y +Scattered Rx = Y +Basic stats = Y +Stats per queue = Y +Linux UIO = Y +x86-64 = Y +Usage doc = Y diff --git a/dpdk/doc/guides/nics/features/avp.ini b/dpdk/doc/guides/nics/features/avp.ini new file mode 100644 index 00000000..ceb69939 --- /dev/null +++ b/dpdk/doc/guides/nics/features/avp.ini @@ -0,0 +1,16 @@ +; +; Supported features of the 'AVP' network poll mode driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] +Link status = Y +Jumbo frame = Y +Scattered Rx = Y +Promiscuous mode = Y +Unicast MAC filter = Y +VLAN offload = Y +Basic stats = Y +Stats per queue = Y +Linux UIO = Y +x86-64 = Y diff --git a/dpdk/doc/guides/nics/features/bnx2x.ini b/dpdk/doc/guides/nics/features/bnx2x.ini new file mode 100644 index 00000000..3e33e9ab --- /dev/null +++ b/dpdk/doc/guides/nics/features/bnx2x.ini @@ -0,0 +1,17 @@ +; +; Supported features of the 'bnx2x' network poll mode driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] +Speed capabilities = P +Link status = Y +Link status event = Y +Promiscuous mode = Y +Unicast MAC filter = Y +Multicast MAC filter = Y +Basic stats = Y +Extended stats = Y +Linux UIO = Y +x86-64 = Y +Usage doc = Y diff --git a/dpdk/doc/guides/nics/features/bnx2x_vf.ini b/dpdk/doc/guides/nics/features/bnx2x_vf.ini new file mode 100644 index 00000000..c270902e --- /dev/null +++ b/dpdk/doc/guides/nics/features/bnx2x_vf.ini @@ -0,0 +1,18 @@ +; +; Supported features of the 'bnx2x_vf' network poll mode driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] +Speed capabilities = P +Link status = Y +Link status event = Y +Promiscuous mode = Y +Unicast MAC filter = Y +Multicast MAC filter = Y +SR-IOV = Y +Basic stats = Y +Extended stats = Y +Linux UIO = Y +x86-64 = Y +Usage doc = Y diff --git a/dpdk/doc/guides/nics/features/bnxt.ini b/dpdk/doc/guides/nics/features/bnxt.ini new file mode 100644 index 00000000..ef45dd76 --- /dev/null +++ b/dpdk/doc/guides/nics/features/bnxt.ini @@ -0,0 +1,43 @@ +; +; Supported features of the 'bnxt' network poll mode driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] +Link status = Y +Rx interrupt = Y +Queue start/stop = Y +MTU update = Y +Jumbo frame = Y +LRO = Y +TSO = Y +Promiscuous mode = Y +Allmulticast mode = Y +Unicast MAC filter = Y +Multicast MAC filter = Y +RSS hash = Y +RSS key update = Y +RSS reta update = Y +VMDq = Y +SR-IOV = Y +VLAN filter = Y +Ethertype filter = Y +N-tuple filter = Y +Flow control = Y +Flow API = Y +L3 checksum offload = Y +L4 checksum offload = Y +Inner L3 checksum = Y +Inner L4 checksum = Y +Packet type parsing = Y +VLAN offload = Y +Rx descriptor status = Y +Tx descriptor status = Y +Basic stats = Y +Extended stats = Y +FW version = Y +EEPROM dump = Y +LED = Y +Linux UIO = Y +Linux VFIO = Y +x86-64 = Y diff --git a/dpdk/doc/guides/nics/features/bonding.ini b/dpdk/doc/guides/nics/features/bonding.ini new file mode 100644 index 00000000..c1653051 --- /dev/null +++ b/dpdk/doc/guides/nics/features/bonding.ini @@ -0,0 +1,6 @@ +; +; Supported features of the 'bonding' network poll mode driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] diff --git a/dpdk/doc/guides/nics/features/cxgbe.ini b/dpdk/doc/guides/nics/features/cxgbe.ini new file mode 100644 index 00000000..3d0fde2f --- /dev/null +++ b/dpdk/doc/guides/nics/features/cxgbe.ini @@ -0,0 +1,32 @@ +; +; Supported features of the 'cxgbe' network poll mode driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] +Speed capabilities = Y +Link status = Y +Queue start/stop = Y +MTU update = Y +Jumbo frame = Y +Scattered Rx = Y +TSO = Y +Promiscuous mode = Y +Allmulticast mode = Y +RSS hash = Y +Flow control = Y +CRC offload = Y +VLAN offload = Y +L3 checksum offload = Y +L4 checksum offload = Y +Packet type parsing = Y +Basic stats = Y +Stats per queue = Y +EEPROM dump = Y +Registers dump = Y +BSD nic_uio = Y +Linux UIO = Y +Linux VFIO = Y +x86-32 = Y +x86-64 = Y +Usage doc = Y diff --git a/dpdk/doc/guides/nics/features/default.ini b/dpdk/doc/guides/nics/features/default.ini new file mode 100644 index 00000000..dae2ad77 --- /dev/null +++ b/dpdk/doc/guides/nics/features/default.ini @@ -0,0 +1,80 @@ +; +; Features of a default network driver. +; +; This file defines the features that are valid for inclusion in +; the other driver files and also the order that they appear in +; the features table in the documentation. The feature description +; string should not exceed feature_str_len defined in conf.py. +; +[Features] +Speed capabilities = +Link status = +Link status event = +Removal event = +Queue status event = +Rx interrupt = +Lock-free Tx queue = +Fast mbuf free = +Free Tx mbuf on demand = +Queue start/stop = +MTU update = +Jumbo frame = +Scattered Rx = +LRO = +TSO = +Promiscuous mode = +Allmulticast mode = +Unicast MAC filter = +Multicast MAC filter = +RSS hash = +RSS key update = +RSS reta update = +VMDq = +SR-IOV = +DCB = +VLAN filter = +Ethertype filter = +N-tuple filter = +SYN filter = +Tunnel filter = +Flexible filter = +Hash filter = +Flow director = +Flow control = +Flow API = +Rate limitation = +Traffic mirroring = +Inline crypto = +CRC offload = +VLAN offload = +QinQ offload = +L3 checksum offload = +L4 checksum offload = +Timestamp offload = +MACsec offload = +Inner L3 checksum = +Inner L4 checksum = +Packet type parsing = +Timesync = +Rx descriptor status = +Tx descriptor status = +Basic stats = +Extended stats = +Stats per queue = +FW version = +EEPROM dump = +Registers dump = +LED = +Multiprocess aware = +BSD nic_uio = +Linux UIO = +Linux VFIO = +Other kdrv = +ARMv7 = +ARMv8 = +Power8 = +x86-32 = +x86-64 = +Usage doc = +Design doc = +Perf doc = diff --git a/dpdk/doc/guides/nics/features/dpaa.ini b/dpdk/doc/guides/nics/features/dpaa.ini new file mode 100644 index 00000000..24cfd856 --- /dev/null +++ b/dpdk/doc/guides/nics/features/dpaa.ini @@ -0,0 +1,24 @@ +; +; Supported features of the 'dpaa' network poll mode driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] +Speed capabilities = P +Link status = Y +Jumbo frame = Y +MTU update = Y +Scattered Rx = Y +Promiscuous mode = Y +Allmulticast mode = Y +Unicast MAC filter = Y +RSS hash = Y +Flow control = Y +L3 checksum offload = Y +L4 checksum offload = Y +Packet type parsing = Y +Basic stats = Y +Extended stats = Y +FW version = Y +ARMv8 = Y +Usage doc = Y diff --git a/dpdk/doc/guides/nics/features/dpaa2.ini b/dpdk/doc/guides/nics/features/dpaa2.ini new file mode 100644 index 00000000..6ebbab4b --- /dev/null +++ b/dpdk/doc/guides/nics/features/dpaa2.ini @@ -0,0 +1,28 @@ +; +; Supported features of the 'dpaa2' network poll mode driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] +Speed capabilities = P +Link status = Y +Link status event = Y +Queue start/stop = Y +Jumbo frame = Y +MTU update = Y +Promiscuous mode = Y +Allmulticast mode = Y +Unicast MAC filter = Y +RSS hash = Y +VLAN filter = Y +Flow control = Y +VLAN offload = Y +L3 checksum offload = Y +L4 checksum offload = Y +Packet type parsing = Y +Basic stats = Y +Extended stats = Y +FW version = Y +Linux VFIO = Y +ARMv8 = Y +Usage doc = Y diff --git a/dpdk/doc/guides/nics/features/e1000.ini b/dpdk/doc/guides/nics/features/e1000.ini new file mode 100644 index 00000000..51ca580f --- /dev/null +++ b/dpdk/doc/guides/nics/features/e1000.ini @@ -0,0 +1,32 @@ +; +; Supported features of the 'e1000' network poll mode driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] +Speed capabilities = P +Link status = Y +Link status event = Y +Rx interrupt = Y +Free Tx mbuf on demand = Y +MTU update = Y +Jumbo frame = Y +Scattered Rx = Y +Promiscuous mode = Y +Allmulticast mode = Y +Unicast MAC filter = Y +VLAN filter = Y +Flow control = Y +CRC offload = Y +VLAN offload = Y +QinQ offload = Y +L3 checksum offload = Y +L4 checksum offload = Y +Rx descriptor status = Y +Tx descriptor status = Y +Basic stats = Y +BSD nic_uio = Y +Linux UIO = Y +Linux VFIO = Y +x86-32 = Y +x86-64 = Y diff --git a/dpdk/doc/guides/nics/features/ena.ini b/dpdk/doc/guides/nics/features/ena.ini new file mode 100644 index 00000000..691c1e3d --- /dev/null +++ b/dpdk/doc/guides/nics/features/ena.ini @@ -0,0 +1,27 @@ +; +; Supported features of the 'ena' network poll mode driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] +Speed capabilities = P +Queue start/stop = Y +MTU update = Y +Jumbo frame = Y +Scattered Rx = Y +TSO = Y +RSS hash = Y +RSS key update = Y +RSS reta update = Y +SR-IOV = Y +CRC offload = Y +VLAN offload = Y +L3 checksum offload = Y +L4 checksum offload = Y +Inner L3 checksum = Y +Inner L4 checksum = Y +Basic stats = Y +Extended stats = Y +Linux UIO = Y +x86-32 = Y +x86-64 = Y diff --git a/dpdk/doc/guides/nics/features/enic.ini b/dpdk/doc/guides/nics/features/enic.ini new file mode 100644 index 00000000..498341f0 --- /dev/null +++ b/dpdk/doc/guides/nics/features/enic.ini @@ -0,0 +1,34 @@ +; +; Supported features of the 'enic' network poll mode driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] +Link status = Y +Link status event = Y +Queue start/stop = Y +MTU update = Y +Jumbo frame = Y +Scattered Rx = Y +TSO = Y +Promiscuous mode = Y +Unicast MAC filter = Y +Multicast MAC filter = Y +RSS hash = Y +SR-IOV = Y +VLAN filter = Y +CRC offload = Y +VLAN offload = Y +Flow director = Y +Flow API = Y +L3 checksum offload = Y +L4 checksum offload = Y +Packet type parsing = Y +Basic stats = Y +Multiprocess aware = Y +BSD nic_uio = Y +Linux UIO = Y +Linux VFIO = Y +x86-32 = Y +x86-64 = Y +Usage doc = Y diff --git a/dpdk/doc/guides/nics/features/failsafe.ini b/dpdk/doc/guides/nics/features/failsafe.ini new file mode 100644 index 00000000..a42e344a --- /dev/null +++ b/dpdk/doc/guides/nics/features/failsafe.ini @@ -0,0 +1,26 @@ +; +; Supported features of the 'fail-safe' poll mode driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] +Link status = Y +Link status event = Y +MTU update = Y +Jumbo frame = Y +Promiscuous mode = Y +Allmulticast mode = Y +Unicast MAC filter = Y +Multicast MAC filter = Y +VLAN filter = Y +Flow control = Y +Flow API = Y +Packet type parsing = Y +Basic stats = Y +Stats per queue = Y +ARMv7 = Y +ARMv8 = Y +Power8 = Y +x86-32 = Y +x86-64 = Y +Usage doc = Y diff --git a/dpdk/doc/guides/nics/features/fm10k.ini b/dpdk/doc/guides/nics/features/fm10k.ini new file mode 100644 index 00000000..f0f61a7d --- /dev/null +++ b/dpdk/doc/guides/nics/features/fm10k.ini @@ -0,0 +1,35 @@ +; +; Supported features of the 'fm10k' network poll mode driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] +Speed capabilities = P +Rx interrupt = Y +Queue start/stop = Y +Jumbo frame = Y +Scattered Rx = Y +TSO = Y +Promiscuous mode = Y +Allmulticast mode = Y +Unicast MAC filter = Y +Multicast MAC filter = Y +RSS hash = Y +RSS key update = Y +RSS reta update = Y +VMDq = Y +VLAN filter = Y +CRC offload = Y +VLAN offload = Y +L3 checksum offload = Y +L4 checksum offload = Y +Packet type parsing = Y +Basic stats = Y +Extended stats = Y +Stats per queue = Y +Multiprocess aware = Y +BSD nic_uio = Y +Linux UIO = Y +Linux VFIO = Y +x86-32 = Y +x86-64 = Y diff --git a/dpdk/doc/guides/nics/features/fm10k_vf.ini b/dpdk/doc/guides/nics/features/fm10k_vf.ini new file mode 100644 index 00000000..32b93df4 --- /dev/null +++ b/dpdk/doc/guides/nics/features/fm10k_vf.ini @@ -0,0 +1,29 @@ +; +; Supported features of the 'fm10k_vf' network poll mode driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] +Speed capabilities = P +Rx interrupt = Y +Queue start/stop = Y +Jumbo frame = Y +Scattered Rx = Y +TSO = Y +RSS hash = Y +RSS key update = Y +RSS reta update = Y +CRC offload = Y +VLAN offload = Y +L3 checksum offload = Y +L4 checksum offload = Y +Packet type parsing = Y +Basic stats = Y +Extended stats = Y +Stats per queue = Y +Multiprocess aware = Y +BSD nic_uio = Y +Linux UIO = Y +Linux VFIO = Y +x86-32 = Y +x86-64 = Y diff --git a/dpdk/doc/guides/nics/features/i40e.ini b/dpdk/doc/guides/nics/features/i40e.ini new file mode 100644 index 00000000..e862712c --- /dev/null +++ b/dpdk/doc/guides/nics/features/i40e.ini @@ -0,0 +1,54 @@ +; +; Supported features of the 'i40e' network poll mode driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] +Speed capabilities = Y +Link status = Y +Link status event = Y +Rx interrupt = Y +Queue start/stop = Y +Jumbo frame = Y +Scattered Rx = Y +TSO = Y +Promiscuous mode = Y +Allmulticast mode = Y +Unicast MAC filter = Y +Multicast MAC filter = Y +RSS hash = Y +RSS key update = Y +RSS reta update = Y +VMDq = Y +SR-IOV = Y +DCB = Y +VLAN filter = Y +Ethertype filter = Y +Tunnel filter = Y +Hash filter = Y +Flow director = Y +Flow control = Y +Flow API = Y +Traffic mirroring = Y +CRC offload = Y +VLAN offload = Y +QinQ offload = Y +L3 checksum offload = Y +L4 checksum offload = Y +Inner L3 checksum = Y +Inner L4 checksum = Y +Packet type parsing = Y +Timesync = Y +Rx descriptor status = Y +Tx descriptor status = Y +Basic stats = Y +Extended stats = Y +FW version = Y +Multiprocess aware = Y +BSD nic_uio = Y +Linux UIO = Y +Linux VFIO = Y +x86-32 = Y +x86-64 = Y +ARMv8 = Y +Power8 = Y diff --git a/dpdk/doc/guides/nics/features/i40e_vec.ini b/dpdk/doc/guides/nics/features/i40e_vec.ini new file mode 100644 index 00000000..7d7b3a92 --- /dev/null +++ b/dpdk/doc/guides/nics/features/i40e_vec.ini @@ -0,0 +1,44 @@ +; +; Supported features of the 'i40e_vec' network poll mode driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] +Speed capabilities = Y +Link status = Y +Link status event = Y +Rx interrupt = Y +Queue start/stop = Y +Jumbo frame = Y +Scattered Rx = Y +TSO = Y +Promiscuous mode = Y +Allmulticast mode = Y +Unicast MAC filter = Y +Multicast MAC filter = Y +RSS hash = Y +RSS key update = Y +RSS reta update = Y +VMDq = Y +SR-IOV = Y +DCB = Y +VLAN filter = Y +Ethertype filter = Y +Tunnel filter = Y +Hash filter = Y +Flow director = Y +Flow control = Y +Traffic mirroring = Y +Timesync = Y +Rx descriptor status = Y +Tx descriptor status = Y +Basic stats = Y +Extended stats = Y +Multiprocess aware = Y +BSD nic_uio = Y +Linux UIO = Y +Linux VFIO = Y +x86-32 = Y +x86-64 = Y +ARMv8 = Y +Power8 = Y diff --git a/dpdk/doc/guides/nics/features/i40e_vf.ini b/dpdk/doc/guides/nics/features/i40e_vf.ini new file mode 100644 index 00000000..46e0d9fc --- /dev/null +++ b/dpdk/doc/guides/nics/features/i40e_vf.ini @@ -0,0 +1,38 @@ +; +; Supported features of the 'i40e_vf' network poll mode driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] +Rx interrupt = Y +Queue start/stop = Y +Jumbo frame = Y +Scattered Rx = Y +TSO = Y +Promiscuous mode = Y +Allmulticast mode = Y +Unicast MAC filter = Y +Multicast MAC filter = Y +RSS hash = Y +RSS key update = Y +RSS reta update = Y +VLAN filter = Y +Hash filter = Y +CRC offload = Y +VLAN offload = Y +QinQ offload = Y +L3 checksum offload = Y +L4 checksum offload = Y +Inner L3 checksum = Y +Inner L4 checksum = Y +Packet type parsing = Y +Rx descriptor status = Y +Tx descriptor status = Y +Basic stats = Y +Extended stats = Y +Multiprocess aware = Y +BSD nic_uio = Y +Linux UIO = Y +Linux VFIO = Y +x86-32 = Y +x86-64 = Y diff --git a/dpdk/doc/guides/nics/features/i40e_vf_vec.ini b/dpdk/doc/guides/nics/features/i40e_vf_vec.ini new file mode 100644 index 00000000..c2c6c19f --- /dev/null +++ b/dpdk/doc/guides/nics/features/i40e_vf_vec.ini @@ -0,0 +1,31 @@ +; +; Supported features of the 'i40evf_vec' network poll mode driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] +Rx interrupt = Y +Queue start/stop = Y +Jumbo frame = Y +Scattered Rx = Y +TSO = Y +Promiscuous mode = Y +Allmulticast mode = Y +Unicast MAC filter = Y +Multicast MAC filter = Y +RSS hash = Y +RSS key update = Y +RSS reta update = Y +VLAN filter = Y +Hash filter = Y +Rx descriptor status = Y +Tx descriptor status = Y +Basic stats = Y +Extended stats = Y +Multiprocess aware = Y +BSD nic_uio = Y +Linux UIO = Y +Linux VFIO = Y +x86-32 = Y +x86-64 = Y +ARMv8 = Y diff --git a/dpdk/doc/guides/nics/features/igb.ini b/dpdk/doc/guides/nics/features/igb.ini new file mode 100644 index 00000000..33d64d99 --- /dev/null +++ b/dpdk/doc/guides/nics/features/igb.ini @@ -0,0 +1,49 @@ +; +; Supported features of the 'igb' network poll mode driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] +Speed capabilities = P +Link status = Y +Link status event = Y +Rx interrupt = Y +MTU update = Y +Jumbo frame = Y +Scattered Rx = Y +TSO = Y +Promiscuous mode = Y +Allmulticast mode = Y +Unicast MAC filter = Y +RSS hash = Y +RSS key update = Y +RSS reta update = Y +VMDq = Y +SR-IOV = Y +DCB = Y +VLAN filter = Y +Ethertype filter = Y +N-tuple filter = Y +SYN filter = Y +Flexible filter = Y +Flow control = Y +Flow API = Y +CRC offload = Y +VLAN offload = Y +QinQ offload = Y +L3 checksum offload = Y +L4 checksum offload = Y +Packet type parsing = Y +Timesync = Y +Rx descriptor status = Y +Tx descriptor status = Y +Basic stats = Y +Extended stats = Y +FW version = Y +EEPROM dump = Y +Registers dump = Y +BSD nic_uio = Y +Linux UIO = Y +Linux VFIO = Y +x86-32 = Y +x86-64 = Y diff --git a/dpdk/doc/guides/nics/features/igb_vf.ini b/dpdk/doc/guides/nics/features/igb_vf.ini new file mode 100644 index 00000000..e641a2c9 --- /dev/null +++ b/dpdk/doc/guides/nics/features/igb_vf.ini @@ -0,0 +1,29 @@ +; +; Supported features of the 'igb_vf' network poll mode driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] +Rx interrupt = Y +Scattered Rx = Y +TSO = Y +Promiscuous mode = Y +Allmulticast mode = Y +Unicast MAC filter = Y +VLAN filter = Y +CRC offload = Y +VLAN offload = Y +QinQ offload = Y +L3 checksum offload = Y +L4 checksum offload = Y +Packet type parsing = Y +Rx descriptor status = Y +Tx descriptor status = Y +Basic stats = Y +Extended stats = Y +Registers dump = Y +BSD nic_uio = Y +Linux UIO = Y +Linux VFIO = Y +x86-32 = Y +x86-64 = Y diff --git a/dpdk/doc/guides/nics/features/ixgbe.ini b/dpdk/doc/guides/nics/features/ixgbe.ini new file mode 100644 index 00000000..1d68ee8e --- /dev/null +++ b/dpdk/doc/guides/nics/features/ixgbe.ini @@ -0,0 +1,61 @@ +; +; Supported features of the 'ixgbe' network poll mode driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] +Speed capabilities = Y +Link status = Y +Link status event = Y +Rx interrupt = Y +Queue start/stop = Y +MTU update = Y +Jumbo frame = Y +Scattered Rx = Y +LRO = Y +TSO = Y +Promiscuous mode = Y +Allmulticast mode = Y +Unicast MAC filter = Y +RSS hash = Y +RSS key update = Y +RSS reta update = Y +VMDq = Y +SR-IOV = Y +DCB = Y +VLAN filter = Y +Ethertype filter = Y +N-tuple filter = Y +SYN filter = Y +Tunnel filter = Y +Flow director = Y +Flow control = Y +Flow API = Y +Rate limitation = Y +Traffic mirroring = Y +Inline crypto = Y +CRC offload = Y +VLAN offload = Y +QinQ offload = Y +L3 checksum offload = Y +L4 checksum offload = Y +MACsec offload = Y +Inner L3 checksum = Y +Inner L4 checksum = Y +Packet type parsing = Y +Timesync = Y +Rx descriptor status = Y +Tx descriptor status = Y +Basic stats = Y +Extended stats = Y +Stats per queue = Y +FW version = Y +EEPROM dump = Y +Registers dump = Y +Multiprocess aware = Y +BSD nic_uio = Y +Linux UIO = Y +Linux VFIO = Y +ARMv8 = Y +x86-32 = Y +x86-64 = Y diff --git a/dpdk/doc/guides/nics/features/ixgbe_vec.ini b/dpdk/doc/guides/nics/features/ixgbe_vec.ini new file mode 100644 index 00000000..28bc0547 --- /dev/null +++ b/dpdk/doc/guides/nics/features/ixgbe_vec.ini @@ -0,0 +1,50 @@ +; +; Supported features of the 'ixgbe_vec' network poll mode driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] +Speed capabilities = Y +Link status = Y +Link status event = Y +Rx interrupt = Y +Queue start/stop = Y +MTU update = Y +Jumbo frame = Y +Scattered Rx = Y +LRO = Y +TSO = Y +Promiscuous mode = Y +Allmulticast mode = Y +Unicast MAC filter = Y +RSS hash = Y +RSS key update = Y +RSS reta update = Y +VMDq = Y +SR-IOV = Y +DCB = Y +VLAN filter = Y +Ethertype filter = Y +N-tuple filter = Y +SYN filter = Y +Tunnel filter = Y +Flow director = Y +Flow control = Y +Rate limitation = Y +Traffic mirroring = Y +Inline crypto = Y +Timesync = Y +Rx descriptor status = Y +Tx descriptor status = Y +Basic stats = Y +Extended stats = Y +Stats per queue = Y +EEPROM dump = Y +Registers dump = Y +Multiprocess aware = Y +BSD nic_uio = Y +Linux UIO = Y +Linux VFIO = Y +ARMv8 = Y +x86-32 = Y +x86-64 = Y diff --git a/dpdk/doc/guides/nics/features/ixgbe_vf.ini b/dpdk/doc/guides/nics/features/ixgbe_vf.ini new file mode 100644 index 00000000..0a15500b --- /dev/null +++ b/dpdk/doc/guides/nics/features/ixgbe_vf.ini @@ -0,0 +1,40 @@ +; +; Supported features of the 'ixgbe_vf' network poll mode driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] +Link status = Y +Rx interrupt = Y +MTU update = Y +Jumbo frame = Y +Scattered Rx = Y +LRO = Y +TSO = Y +Allmulticast mode = Y +Unicast MAC filter = Y +RSS hash = Y +RSS key update = Y +RSS reta update = Y +VLAN filter = Y +Inline crypto = Y +CRC offload = Y +VLAN offload = Y +QinQ offload = Y +L3 checksum offload = Y +L4 checksum offload = Y +Inner L3 checksum = Y +Inner L4 checksum = Y +Packet type parsing = Y +Rx descriptor status = Y +Tx descriptor status = Y +Basic stats = Y +Extended stats = Y +Registers dump = Y +Multiprocess aware = Y +BSD nic_uio = Y +Linux UIO = Y +Linux VFIO = Y +ARMv8 = Y +x86-32 = Y +x86-64 = Y diff --git a/dpdk/doc/guides/nics/features/ixgbe_vf_vec.ini b/dpdk/doc/guides/nics/features/ixgbe_vf_vec.ini new file mode 100644 index 00000000..80e7f3bd --- /dev/null +++ b/dpdk/doc/guides/nics/features/ixgbe_vf_vec.ini @@ -0,0 +1,32 @@ +; +; Supported features of the 'ixgbevf_vec' network poll mode driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] +Link status = Y +Rx interrupt = Y +MTU update = Y +Jumbo frame = Y +Scattered Rx = Y +LRO = Y +TSO = Y +Allmulticast mode = Y +Unicast MAC filter = Y +RSS hash = Y +RSS key update = Y +RSS reta update = Y +VLAN filter = Y +Inline crypto = Y +Rx descriptor status = Y +Tx descriptor status = Y +Basic stats = Y +Extended stats = Y +Registers dump = Y +Multiprocess aware = Y +BSD nic_uio = Y +Linux UIO = Y +Linux VFIO = Y +ARMv8 = Y +x86-32 = Y +x86-64 = Y diff --git a/dpdk/doc/guides/nics/features/kni.ini b/dpdk/doc/guides/nics/features/kni.ini new file mode 100644 index 00000000..6deb66ae --- /dev/null +++ b/dpdk/doc/guides/nics/features/kni.ini @@ -0,0 +1,7 @@ +; +; Supported features of the 'kni' network poll mode driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] +Usage doc = Y diff --git a/dpdk/doc/guides/nics/features/liquidio.ini b/dpdk/doc/guides/nics/features/liquidio.ini new file mode 100644 index 00000000..f628b764 --- /dev/null +++ b/dpdk/doc/guides/nics/features/liquidio.ini @@ -0,0 +1,31 @@ +; +; Supported features of the 'LiquidIO' network poll mode driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] +Speed capabilities = Y +Link status = Y +Link status event = Y +MTU update = Y +Jumbo frame = Y +Scattered Rx = Y +Promiscuous mode = Y +Allmulticast mode = Y +RSS hash = Y +RSS key update = Y +RSS reta update = Y +VLAN filter = Y +CRC offload = Y +VLAN offload = P +L3 checksum offload = Y +L4 checksum offload = Y +Inner L3 checksum = Y +Inner L4 checksum = Y +Basic stats = Y +Extended stats = Y +Multiprocess aware = Y +Linux UIO = Y +Linux VFIO = Y +x86-64 = Y +Usage doc = Y diff --git a/dpdk/doc/guides/nics/features/mlx4.ini b/dpdk/doc/guides/nics/features/mlx4.ini new file mode 100644 index 00000000..f6efd21d --- /dev/null +++ b/dpdk/doc/guides/nics/features/mlx4.ini @@ -0,0 +1,34 @@ +; +; Supported features of the 'mlx4' network poll mode driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] +Speed capabilities = P +Link status = Y +Link status event = Y +Removal event = Y +Rx interrupt = Y +Queue start/stop = Y +MTU update = Y +Jumbo frame = Y +Scattered Rx = Y +Promiscuous mode = Y +Allmulticast mode = Y +Unicast MAC filter = Y +Multicast MAC filter = Y +RSS hash = Y +SR-IOV = Y +VLAN filter = Y +L3 checksum offload = Y +L4 checksum offload = Y +Inner L3 checksum = Y +Inner L4 checksum = Y +Packet type parsing = Y +Basic stats = Y +Stats per queue = Y +Other kdrv = Y +Power8 = Y +x86-32 = Y +x86-64 = Y +Usage doc = Y diff --git a/dpdk/doc/guides/nics/features/mlx5.ini b/dpdk/doc/guides/nics/features/mlx5.ini new file mode 100644 index 00000000..c3636391 --- /dev/null +++ b/dpdk/doc/guides/nics/features/mlx5.ini @@ -0,0 +1,43 @@ +; +; Supported features of the 'mlx5' network poll mode driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] +Speed capabilities = Y +Link status = Y +Link status event = Y +Removal event = Y +Rx interrupt = Y +Queue start/stop = Y +MTU update = Y +Jumbo frame = Y +Scattered Rx = Y +TSO = Y +Promiscuous mode = Y +Allmulticast mode = Y +Unicast MAC filter = Y +Multicast MAC filter = Y +RSS hash = Y +RSS key update = Y +RSS reta update = Y +SR-IOV = Y +VLAN filter = Y +Flow director = Y +Flow API = Y +CRC offload = Y +VLAN offload = Y +L3 checksum offload = Y +L4 checksum offload = Y +Packet type parsing = Y +Rx descriptor status = Y +Tx descriptor status = Y +Basic stats = Y +Extended stats = Y +Stats per queue = Y +Multiprocess aware = Y +Other kdrv = Y +ARMv8 = Y +Power8 = Y +x86-64 = Y +Usage doc = Y diff --git a/dpdk/doc/guides/nics/features/mrvl.ini b/dpdk/doc/guides/nics/features/mrvl.ini new file mode 100644 index 00000000..00d96218 --- /dev/null +++ b/dpdk/doc/guides/nics/features/mrvl.ini @@ -0,0 +1,23 @@ +; +; Supported features of the 'mrvl' network poll mode driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] +Speed capabilities = Y +Link status = Y +MTU update = Y +Jumbo frame = Y +Promiscuous mode = Y +Allmulticast mode = Y +Unicast MAC filter = Y +Multicast MAC filter = Y +RSS hash = Y +VLAN filter = Y +CRC offload = Y +L3 checksum offload = Y +L4 checksum offload = Y +Packet type parsing = Y +Basic stats = Y +ARMv8 = Y +Usage doc = Y diff --git a/dpdk/doc/guides/nics/features/nfp_pf.ini b/dpdk/doc/guides/nics/features/nfp_pf.ini new file mode 100644 index 00000000..d2899e7f --- /dev/null +++ b/dpdk/doc/guides/nics/features/nfp_pf.ini @@ -0,0 +1,28 @@ +; +; Supported features of the 'nfp' network poll mode driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] +Speed capabilities = Y +Link status = Y +Link status event = Y +Rx interrupt = Y +Queue start/stop = Y +MTU update = Y +Jumbo frame = Y +Promiscuous mode = Y +TSO = Y +RSS hash = Y +RSS key update = Y +RSS reta update = Y +Flow control = Y +VLAN offload = Y +L3 checksum offload = Y +L4 checksum offload = Y +Basic stats = Y +Stats per queue = Y +Linux UIO = Y +Linux VFIO = Y +x86-64 = Y +Usage doc = Y diff --git a/dpdk/doc/guides/nics/features/nfp_vf.ini b/dpdk/doc/guides/nics/features/nfp_vf.ini new file mode 100644 index 00000000..d2899e7f --- /dev/null +++ b/dpdk/doc/guides/nics/features/nfp_vf.ini @@ -0,0 +1,28 @@ +; +; Supported features of the 'nfp' network poll mode driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] +Speed capabilities = Y +Link status = Y +Link status event = Y +Rx interrupt = Y +Queue start/stop = Y +MTU update = Y +Jumbo frame = Y +Promiscuous mode = Y +TSO = Y +RSS hash = Y +RSS key update = Y +RSS reta update = Y +Flow control = Y +VLAN offload = Y +L3 checksum offload = Y +L4 checksum offload = Y +Basic stats = Y +Stats per queue = Y +Linux UIO = Y +Linux VFIO = Y +x86-64 = Y +Usage doc = Y diff --git a/dpdk/doc/guides/nics/features/null.ini b/dpdk/doc/guides/nics/features/null.ini new file mode 100644 index 00000000..3957f7ca --- /dev/null +++ b/dpdk/doc/guides/nics/features/null.ini @@ -0,0 +1,6 @@ +; +; Supported features of the 'null' network poll mode driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] diff --git a/dpdk/doc/guides/nics/features/octeontx.ini b/dpdk/doc/guides/nics/features/octeontx.ini new file mode 100644 index 00000000..c92d5aa0 --- /dev/null +++ b/dpdk/doc/guides/nics/features/octeontx.ini @@ -0,0 +1,19 @@ +; +; Supported features of the 'octeontx' network poll mode driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] +Speed capabilities = Y +Link status = Y +Lock-free Tx queue = Y +Queue start/stop = P +Jumbo frame = Y +Promiscuous mode = Y +Unicast MAC filter = P +CRC offload = Y +Packet type parsing = Y +Basic stats = Y +Linux VFIO = Y +ARMv8 = Y +Usage doc = Y diff --git a/dpdk/doc/guides/nics/features/pcap.ini b/dpdk/doc/guides/nics/features/pcap.ini new file mode 100644 index 00000000..28e64880 --- /dev/null +++ b/dpdk/doc/guides/nics/features/pcap.ini @@ -0,0 +1,15 @@ +; +; Supported features of the 'pcap' network poll mode driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] +Jumbo frame = Y +Basic stats = Y +Multiprocess aware = Y +ARMv7 = Y +ARMv8 = Y +Power8 = Y +x86-32 = Y +x86-64 = Y +Usage doc = Y diff --git a/dpdk/doc/guides/nics/features/qede.ini b/dpdk/doc/guides/nics/features/qede.ini new file mode 100644 index 00000000..cbadc194 --- /dev/null +++ b/dpdk/doc/guides/nics/features/qede.ini @@ -0,0 +1,42 @@ +; +; Supported features of the 'qede' network poll mode driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] +Speed capabilities = Y +Link status = Y +Link status event = Y +MTU update = Y +Jumbo frame = Y +Scattered Rx = Y +Promiscuous mode = Y +Allmulticast mode = Y +Unicast MAC filter = Y +Multicast MAC filter = Y +RSS hash = Y +RSS key update = Y +RSS reta update = Y +VLAN filter = Y +Flow control = Y +CRC offload = Y +VLAN offload = Y +L3 checksum offload = Y +L4 checksum offload = Y +Tunnel filter = Y +Inner L3 checksum = Y +Inner L4 checksum = Y +Packet type parsing = Y +Basic stats = Y +Extended stats = Y +Stats per queue = Y +Multiprocess aware = Y +Linux UIO = Y +ARMv8 = Y +x86-32 = Y +x86-64 = Y +Usage doc = Y +N-tuple filter = Y +Flow director = Y +LRO = Y +TSO = Y diff --git a/dpdk/doc/guides/nics/features/qede_vf.ini b/dpdk/doc/guides/nics/features/qede_vf.ini new file mode 100644 index 00000000..18857b6e --- /dev/null +++ b/dpdk/doc/guides/nics/features/qede_vf.ini @@ -0,0 +1,38 @@ +; +; Supported features of the 'qede_vf' network poll mode driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] +Speed capabilities = Y +Link status = Y +Link status event = Y +MTU update = Y +Jumbo frame = Y +Scattered Rx = Y +Promiscuous mode = Y +Allmulticast mode = Y +Unicast MAC filter = Y +Multicast MAC filter = Y +RSS hash = Y +RSS key update = Y +RSS reta update = Y +SR-IOV = Y +VLAN filter = Y +Flow control = Y +CRC offload = Y +VLAN offload = Y +L3 checksum offload = Y +L4 checksum offload = Y +Packet type parsing = Y +Basic stats = Y +Extended stats = Y +Stats per queue = Y +Multiprocess aware = Y +Linux UIO = Y +ARMv8 = Y +x86-32 = Y +x86-64 = Y +LRO = Y +TSO = Y +Usage doc = Y diff --git a/dpdk/doc/guides/nics/features/ring.ini b/dpdk/doc/guides/nics/features/ring.ini new file mode 100644 index 00000000..ac207ba3 --- /dev/null +++ b/dpdk/doc/guides/nics/features/ring.ini @@ -0,0 +1,6 @@ +; +; Supported features of the 'ring' network poll mode driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] diff --git a/dpdk/doc/guides/nics/features/sfc_efx.ini b/dpdk/doc/guides/nics/features/sfc_efx.ini new file mode 100644 index 00000000..03890f30 --- /dev/null +++ b/dpdk/doc/guides/nics/features/sfc_efx.ini @@ -0,0 +1,37 @@ +; +; Supported features of the 'sfc_efx' network poll mode driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] +Speed capabilities = Y +Link status = Y +Link status event = Y +Queue start/stop = Y +MTU update = Y +Jumbo frame = Y +Scattered Rx = Y +TSO = Y +Promiscuous mode = Y +Allmulticast mode = Y +Multicast MAC filter = Y +RSS hash = Y +RSS key update = Y +RSS reta update = Y +SR-IOV = Y +Flow control = Y +Flow API = Y +VLAN offload = P +L3 checksum offload = Y +L4 checksum offload = Y +Packet type parsing = Y +Rx descriptor status = Y +Tx descriptor status = Y +Basic stats = Y +Extended stats = Y +FW version = Y +Multiprocess aware = Y +BSD nic_uio = Y +Linux UIO = Y +Linux VFIO = Y +x86-64 = Y diff --git a/dpdk/doc/guides/nics/features/szedata2.ini b/dpdk/doc/guides/nics/features/szedata2.ini new file mode 100644 index 00000000..a0e6f6e8 --- /dev/null +++ b/dpdk/doc/guides/nics/features/szedata2.ini @@ -0,0 +1,18 @@ +; +; Supported features of the 'szedata2' network poll mode driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] +Speed capabilities = P +Link status = Y +Queue start/stop = Y +Scattered Rx = Y +Promiscuous mode = Y +Allmulticast mode = Y +Basic stats = Y +Extended stats = Y +Stats per queue = Y +Other kdrv = Y +x86-64 = Y +Usage doc = Y diff --git a/dpdk/doc/guides/nics/features/tap.ini b/dpdk/doc/guides/nics/features/tap.ini new file mode 100644 index 00000000..f0e893d6 --- /dev/null +++ b/dpdk/doc/guides/nics/features/tap.ini @@ -0,0 +1,28 @@ +; +; Supported features of the 'tap' driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] +Speed capabilities = P +Link status = Y +Link status event = Y +Jumbo frame = Y +Promiscuous mode = Y +Allmulticast mode = Y +Basic stats = Y +Flow API = Y +L3 checksum offload = Y +L4 checksum offload = Y +MTU update = Y +Multicast MAC filter = Y +Unicast MAC filter = Y +Packet type parsing = Y +Flow control = Y +Other kdrv = Y +ARMv7 = Y +ARMv8 = Y +Power8 = Y +x86-32 = Y +x86-64 = Y +Usage doc = Y diff --git a/dpdk/doc/guides/nics/features/thunderx.ini b/dpdk/doc/guides/nics/features/thunderx.ini new file mode 100644 index 00000000..62685897 --- /dev/null +++ b/dpdk/doc/guides/nics/features/thunderx.ini @@ -0,0 +1,31 @@ +; +; Supported features of the 'thunderx' network poll mode driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] +Speed capabilities = Y +Link status = Y +Link status event = Y +Queue start/stop = Y +MTU update = Y +Jumbo frame = Y +Scattered Rx = Y +Promiscuous mode = Y +Allmulticast mode = Y +RSS hash = Y +RSS key update = Y +RSS reta update = Y +SR-IOV = Y +CRC offload = Y +VLAN offload = P +L3 checksum offload = Y +L4 checksum offload = Y +Packet type parsing = Y +Basic stats = Y +Stats per queue = Y +Registers dump = Y +Multiprocess aware = Y +Linux VFIO = Y +ARMv8 = Y +Usage doc = Y diff --git a/dpdk/doc/guides/nics/features/vhost.ini b/dpdk/doc/guides/nics/features/vhost.ini new file mode 100644 index 00000000..dffd1f49 --- /dev/null +++ b/dpdk/doc/guides/nics/features/vhost.ini @@ -0,0 +1,14 @@ +; +; Supported features of the 'vhost' network poll mode driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] +Link status = Y +Link status event = Y +Free Tx mbuf on demand = Y +Queue status event = Y +Basic stats = Y +Extended stats = Y +x86-32 = Y +x86-64 = Y diff --git a/dpdk/doc/guides/nics/features/virtio.ini b/dpdk/doc/guides/nics/features/virtio.ini new file mode 100644 index 00000000..16e577df --- /dev/null +++ b/dpdk/doc/guides/nics/features/virtio.ini @@ -0,0 +1,29 @@ +; +; Supported features of the 'virtio' network poll mode driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] +Speed capabilities = P +Link status = Y +Rx interrupt = Y +Queue start/stop = Y +Scattered Rx = Y +Promiscuous mode = Y +Allmulticast mode = Y +Unicast MAC filter = Y +Multicast MAC filter = Y +VLAN filter = Y +Basic stats = Y +Stats per queue = Y +Extended stats = Y +Multiprocess aware = Y +BSD nic_uio = Y +Linux UIO = Y +Linux VFIO = Y +ARMv7 = Y +ARMv8 = Y +x86-32 = Y +x86-64 = Y +Usage doc = Y +MTU update = Y diff --git a/dpdk/doc/guides/nics/features/virtio_vec.ini b/dpdk/doc/guides/nics/features/virtio_vec.ini new file mode 100644 index 00000000..c06c860d --- /dev/null +++ b/dpdk/doc/guides/nics/features/virtio_vec.ini @@ -0,0 +1,24 @@ +; +; Supported features of the 'virtio_vec' network poll mode driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] +Speed capabilities = P +Link status = Y +Rx interrupt = Y +Queue start/stop = Y +Promiscuous mode = Y +Allmulticast mode = Y +Unicast MAC filter = Y +Multicast MAC filter = Y +VLAN filter = Y +Basic stats = Y +Stats per queue = Y +BSD nic_uio = Y +Linux UIO = Y +Linux VFIO = Y +ARMv7 = Y +ARMv8 = Y +x86-32 = Y +x86-64 = Y diff --git a/dpdk/doc/guides/nics/features/vmxnet3.ini b/dpdk/doc/guides/nics/features/vmxnet3.ini new file mode 100644 index 00000000..9a115138 --- /dev/null +++ b/dpdk/doc/guides/nics/features/vmxnet3.ini @@ -0,0 +1,30 @@ +; +; Supported features of the 'vmxnet3' network poll mode driver. +; +; Refer to default.ini for the full list of available PMD features. +; +[Features] +Speed capabilities = P +Link status = Y +Link status event = Y +Queue start/stop = Y +MTU update = Y +Jumbo frame = Y +LRO = Y +TSO = Y +Promiscuous mode = Y +Allmulticast mode = Y +Unicast MAC filter = Y +RSS hash = Y +VLAN filter = Y +VLAN offload = Y +L4 checksum offload = Y +Packet type parsing = Y +Basic stats = Y +Extended stats = Y +Stats per queue = Y +Linux UIO = Y +Linux VFIO = Y +x86-32 = Y +x86-64 = Y +Usage doc = Y diff --git a/dpdk/doc/guides/nics/fm10k.rst b/dpdk/doc/guides/nics/fm10k.rst index 7fc48624..b47fc0db 100644 --- a/dpdk/doc/guides/nics/fm10k.rst +++ b/dpdk/doc/guides/nics/fm10k.rst @@ -161,6 +161,17 @@ FM10000 PMD driver. The switch driver can be acquired from Intel support. Only Testpoint is validated with DPDK, the latest version that has been validated with DPDK is 4.1.6. +Support for Switch Restart +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For FM10000 multi host based design a DPDK app running in the VM or host needs +to be aware of the switch's state since it may undergo a quit-restart. When +the switch goes down the DPDK app will receive a LSC event indicating link +status down, and the app should stop the worker threads that are polling on +the Rx/Tx queues. When switch comes up, a LSC event indicating ``LINK_UP`` is +sent to the app, which can then restart the FM10000 port to resume network +processing. + CRC striping ~~~~~~~~~~~~ diff --git a/dpdk/doc/guides/nics/i40e.rst b/dpdk/doc/guides/nics/i40e.rst index 5780268f..71d95611 100644 --- a/dpdk/doc/guides/nics/i40e.rst +++ b/dpdk/doc/guides/nics/i40e.rst @@ -64,6 +64,7 @@ Features of the I40E PMD are: - SR-IOV VF - Hot plug - IEEE1588/802.1AS timestamping +- VF Daemon (VFD) - EXPERIMENTAL Prerequisites @@ -77,6 +78,8 @@ Prerequisites - To get better performance on Intel platforms, please follow the "How to get best performance with NICs on Intel platforms" section of the :ref:`Getting Started Guide for Linux `. +- Upgrade the NVM/FW version following the `Intel® Ethernet NVM Update Tool Quick Usage Guide for Linux + `_ if needed. Pre-Installation Configuration ------------------------------ @@ -104,11 +107,6 @@ Please note that enabling debugging options may affect system performance. Toggle the use of Vector PMD instead of normal RX/TX path. To enable vPMD for RX, bulk allocation for Rx must be allowed. -- ``CONFIG_RTE_LIBRTE_I40E_RX_OLFLAGS_ENABLE`` (default ``y``) - - Toggle to enable RX ``olflags``. - This is only meaningful when Vector PMD is used. - - ``CONFIG_RTE_LIBRTE_I40E_16BYTE_RX_DESC`` (default ``n``) Toggle to use a 16-byte RX descriptor, by default the RX descriptor is 32 byte. @@ -130,82 +128,26 @@ Please note that enabling debugging options may affect system performance. Interrupt Throttling interval. -Driver Compilation -~~~~~~~~~~~~~~~~~~ +Driver compilation and testing +------------------------------ -To compile the I40E PMD see :ref:`Getting Started Guide for Linux ` or -:ref:`Getting Started Guide for FreeBSD ` depending on your platform. +Refer to the document :ref:`compiling and testing a PMD for a NIC ` +for details. +- ``Support multiple driver`` (default ``disable``) -Linux ------ + There was a multiple driver support issue during use of 700 series Ethernet + Adapter with both Linux kernel and DPDK PMD. To fix this issue, ``devargs`` + parameter ``support-multi-driver`` is introduced, for example:: + -w 84:00.0,support-multi-driver=1 -Running testpmd -~~~~~~~~~~~~~~~ - -This section demonstrates how to launch ``testpmd`` with Intel XL710/X710 -devices managed by ``librte_pmd_i40e`` in the Linux operating system. - -#. Load ``igb_uio`` or ``vfio-pci`` driver: - - .. code-block:: console - - modprobe uio - insmod ./x86_64-native-linuxapp-gcc/kmod/igb_uio.ko - - or - - .. code-block:: console - - modprobe vfio-pci - -#. Bind the XL710/X710 adapters to ``igb_uio`` or ``vfio-pci`` loaded in the previous step: - - .. code-block:: console - - ./tools/dpdk-devbind.py --bind igb_uio 0000:83:00.0 - - Or setup VFIO permissions for regular users and then bind to ``vfio-pci``: - - .. code-block:: console - - ./tools/dpdk-devbind.py --bind vfio-pci 0000:83:00.0 - -#. Start ``testpmd`` with basic parameters: - - .. code-block:: console - - ./x86_64-native-linuxapp-gcc/app/testpmd -c 0xf -n 4 -w 83:00.0 -- -i - - Example output: - - .. code-block:: console - - ... - EAL: PCI device 0000:83:00.0 on NUMA socket 1 - EAL: probe driver: 8086:1572 rte_i40e_pmd - EAL: PCI memory mapped at 0x7f7f80000000 - EAL: PCI memory mapped at 0x7f7f80800000 - PMD: eth_i40e_dev_init(): FW 5.0 API 1.5 NVM 05.00.02 eetrack 8000208a - Interactive-mode selected - Configuring Port 0 (socket 0) - ... - - PMD: i40e_dev_rx_queue_setup(): Rx Burst Bulk Alloc Preconditions are - satisfied.Rx Burst Bulk Alloc function will be used on port=0, queue=0. - - ... - Port 0: 68:05:CA:26:85:84 - Checking link statuses... - Port 0 Link Up - speed 10000 Mbps - full-duplex - Done - - testpmd> - + With the above configuration, DPDK PMD will not change global registers, and + will switch PF interrupt from IntN to Int0 to avoid interrupt conflict between + DPDK and Linux Kernel. SR-IOV: Prerequisites and sample Application Notes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +-------------------------------------------------- #. Load the kernel module: @@ -254,6 +196,37 @@ SR-IOV: Prerequisites and sample Application Notes #. Assign VF to VM, and bring up the VM. Please see the documentation for the *I40E/IXGBE/IGB Virtual Function Driver*. +#. Running testpmd: + + Follow instructions available in the document + :ref:`compiling and testing a PMD for a NIC ` + to run testpmd. + + Example output: + + .. code-block:: console + + ... + EAL: PCI device 0000:83:00.0 on NUMA socket 1 + EAL: probe driver: 8086:1572 rte_i40e_pmd + EAL: PCI memory mapped at 0x7f7f80000000 + EAL: PCI memory mapped at 0x7f7f80800000 + PMD: eth_i40e_dev_init(): FW 5.0 API 1.5 NVM 05.00.02 eetrack 8000208a + Interactive-mode selected + Configuring Port 0 (socket 0) + ... + + PMD: i40e_dev_rx_queue_setup(): Rx Burst Bulk Alloc Preconditions are + satisfied.Rx Burst Bulk Alloc function will be used on port=0, queue=0. + + ... + Port 0: 68:05:CA:26:85:84 + Checking link statuses... + Port 0 Link Up - speed 10000 Mbps - full-duplex + Done + + testpmd> + Sample Application Notes ------------------------ @@ -267,7 +240,7 @@ To start ``testpmd``, and add vlan 10 to port 0: .. code-block:: console - ./app/testpmd -c ffff -n 4 -- -i --forward-mode=mac + ./app/testpmd -l 0-15 -n 4 -- -i --forward-mode=mac ... testpmd> set promisc 0 off @@ -302,7 +275,7 @@ Start ``testpmd`` with ``--disable-rss`` and ``--pkt-filter-mode=perfect``: .. code-block:: console - ./app/testpmd -c ffff -n 4 -- -i --disable-rss --pkt-filter-mode=perfect \ + ./app/testpmd -l 0-15 -n 4 -- -i --disable-rss --pkt-filter-mode=perfect \ --rxq=8 --txq=8 --nb-cores=8 --nb-ports=1 Add a rule to direct ``ipv4-udp`` packet whose ``dst_ip=2.2.2.5, src_ip=2.2.2.3, src_port=32, dst_port=32`` to queue 1: @@ -436,26 +409,178 @@ used to classify MPLS packet by using a command in testpmd like: testpmd> ethertype_filter 0 add mac_ignr 00:00:00:00:00:00 ethertype \ 0x8847 fwd queue -16 Byte Descriptor cannot be used on DPDK VF -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +16 Byte RX Descriptor setting on DPDK VF +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -If the Linux i40e kernel driver is used as host driver, while DPDK i40e PMD -is used as the VF driver, DPDK cannot choose 16 byte receive descriptor. That -is to say, user should keep ``CONFIG_RTE_LIBRTE_I40E_16BYTE_RX_DESC=n`` in -config file. +Currently the VF's RX descriptor mode is decided by PF. There's no PF-VF +interface for VF to request the RX descriptor mode, also no interface to notify +VF its own RX descriptor mode. +For all available versions of the i40e driver, these drivers don't support 16 +byte RX descriptor. If the Linux i40e kernel driver is used as host driver, +while DPDK i40e PMD is used as the VF driver, DPDK cannot choose 16 byte receive +descriptor. The reason is that the RX descriptor is already set to 32 byte by +the i40e kernel driver. That is to say, user should keep +``CONFIG_RTE_LIBRTE_I40E_16BYTE_RX_DESC=n`` in config file. +In the future, if the Linux i40e driver supports 16 byte RX descriptor, user +should make sure the DPDK VF uses the same RX descriptor mode, 16 byte or 32 +byte, as the PF driver. -Link down with i40e kernel driver after DPDK application exist -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -After DPDK application quit, and the device is bound back to Linux i40e -kernel driver, the link cannot be up after ``ifconfig up``. -To work around this issue, ``ethtool -s autoneg on`` should be -set first and then the link can be brought up through ``ifconfig up``. - -NOTE: requires Linux kernel i40e driver version >= 1.4.X +The same rule for DPDK PF + DPDK VF. The PF and VF should use the same RX +descriptor mode. Or the VF RX will not work. Receive packets with Ethertype 0x88A8 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Due to the FW limitation, PF can receive packets with Ethertype 0x88A8 only when floating VEB is disabled. + +Incorrect Rx statistics when packet is oversize +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When a packet is over maximum frame size, the packet is dropped. +However the Rx statistics, when calling `rte_eth_stats_get` incorrectly +shows it as received. + +VF & TC max bandwidth setting +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The per VF max bandwidth and per TC max bandwidth cannot be enabled in parallel. +The dehavior is different when handling per VF and per TC max bandwidth setting. +When enabling per VF max bandwidth, SW will check if per TC max bandwidth is +enabled. If so, return failure. +When enabling per TC max bandwidth, SW will check if per VF max bandwidth +is enabled. If so, disable per VF max bandwidth and continue with per TC max +bandwidth setting. + +TC TX scheduling mode setting +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +There're 2 TX scheduling modes for TCs, round robin and strict priority mode. +If a TC is set to strict priority mode, it can consume unlimited bandwidth. +It means if APP has set the max bandwidth for that TC, it comes to no +effect. +It's suggested to set the strict priority mode for a TC that is latency +sensitive but no consuming much bandwidth. + +VF performance is impacted by PCI extended tag setting +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To reach maximum NIC performance in the VF the PCI extended tag must be +enabled. The DPDK I40E PF driver will set this feature during initialization, +but the kernel PF driver does not. So when running traffic on a VF which is +managed by the kernel PF driver, a significant NIC performance downgrade has +been observed (for 64 byte packets, there is about 25% linerate downgrade for +a 25G device and about 35% for a 40G device). + +For kernel version >= 4.11, the kernel's PCI driver will enable the extended +tag if it detects that the device supports it. So by default, this is not an +issue. For kernels <= 4.11 or when the PCI extended tag is disabled it can be +enabled using the steps below. + +#. Get the current value of the PCI configure register:: + + setpci -s a8.w + +#. Set bit 8:: + + value = value | 0x100 + +#. Set the PCI configure register with new value:: + + setpci -s a8.w= + +Vlan strip of VF +~~~~~~~~~~~~~~~~ + +The VF vlan strip function is only supported in the i40e kernel driver >= 2.1.26. + +Global configuration warning +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +I40E PMD will set some global registers to enable some function or set some +configure. Then when using different ports of the same NIC with Linux kernel +and DPDK, the port with Linux kernel will be impacted by the port with DPDK. +For example, register I40E_GL_SWT_L2TAGCTRL is used to control L2 tag, i40e +PMD uses I40E_GL_SWT_L2TAGCTRL to set vlan TPID. If setting TPID in port A +with DPDK, then the configuration will also impact port B in the NIC with +kernel driver, which don't want to use the TPID. +So PMD reports warning to clarify what is changed by writing global register. + +High Performance of Small Packets on 40G NIC +-------------------------------------------- + +As there might be firmware fixes for performance enhancement in latest version +of firmware image, the firmware update might be needed for getting high performance. +Check with the local Intel's Network Division application engineers for firmware updates. +Users should consult the release notes specific to a DPDK release to identify +the validated firmware version for a NIC using the i40e driver. + +Use 16 Bytes RX Descriptor Size +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +As i40e PMD supports both 16 and 32 bytes RX descriptor sizes, and 16 bytes size can provide helps to high performance of small packets. +Configuration of ``CONFIG_RTE_LIBRTE_I40E_16BYTE_RX_DESC`` in config files can be changed to use 16 bytes size RX descriptors. + +High Performance and per Packet Latency Tradeoff +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Due to the hardware design, the interrupt signal inside NIC is needed for per +packet descriptor write-back. The minimum interval of interrupts could be set +at compile time by ``CONFIG_RTE_LIBRTE_I40E_ITR_INTERVAL`` in configuration files. +Though there is a default configuration, the interval could be tuned by the +users with that configuration item depends on what the user cares about more, +performance or per packet latency. + +Example of getting best performance with l3fwd example +------------------------------------------------------ + +The following is an example of running the DPDK ``l3fwd`` sample application to get high performance with an +Intel server platform and Intel XL710 NICs. + +The example scenario is to get best performance with two Intel XL710 40GbE ports. +See :numref:`figure_intel_perf_test_setup` for the performance test setup. + +.. _figure_intel_perf_test_setup: + +.. figure:: img/intel_perf_test_setup.* + + Performance Test Setup + + +1. Add two Intel XL710 NICs to the platform, and use one port per card to get best performance. + The reason for using two NICs is to overcome a PCIe Gen3's limitation since it cannot provide 80G bandwidth + for two 40G ports, but two different PCIe Gen3 x8 slot can. + Refer to the sample NICs output above, then we can select ``82:00.0`` and ``85:00.0`` as test ports:: + + 82:00.0 Ethernet [0200]: Intel XL710 for 40GbE QSFP+ [8086:1583] + 85:00.0 Ethernet [0200]: Intel XL710 for 40GbE QSFP+ [8086:1583] + +2. Connect the ports to the traffic generator. For high speed testing, it's best to use a hardware traffic generator. + +3. Check the PCI devices numa node (socket id) and get the cores number on the exact socket id. + In this case, ``82:00.0`` and ``85:00.0`` are both in socket 1, and the cores on socket 1 in the referenced platform + are 18-35 and 54-71. + Note: Don't use 2 logical cores on the same core (e.g core18 has 2 logical cores, core18 and core54), instead, use 2 logical + cores from different cores (e.g core18 and core19). + +4. Bind these two ports to igb_uio. + +5. As to XL710 40G port, we need at least two queue pairs to achieve best performance, then two queues per port + will be required, and each queue pair will need a dedicated CPU core for receiving/transmitting packets. + +6. The DPDK sample application ``l3fwd`` will be used for performance testing, with using two ports for bi-directional forwarding. + Compile the ``l3fwd sample`` with the default lpm mode. + +7. The command line of running l3fwd would be something like the following:: + + ./l3fwd -l 18-21 -n 4 -w 82:00.0 -w 85:00.0 \ + -- -p 0x3 --config '(0,0,18),(0,1,19),(1,0,20),(1,1,21)' + + This means that the application uses core 18 for port 0, queue pair 0 forwarding, core 19 for port 0, queue pair 1 forwarding, + core 20 for port 1, queue pair 0 forwarding, and core 21 for port 1, queue pair 1 forwarding. + +8. Configure the traffic at a traffic generator. + + * Start creating a stream on packet generator. + + * Set the Ethernet II type to 0x0800. diff --git a/dpdk/doc/guides/linux_gsg/img/intel_perf_test_setup.svg b/dpdk/doc/guides/nics/img/intel_perf_test_setup.svg similarity index 100% rename from dpdk/doc/guides/linux_gsg/img/intel_perf_test_setup.svg rename to dpdk/doc/guides/nics/img/intel_perf_test_setup.svg diff --git a/dpdk/doc/guides/nics/index.rst b/dpdk/doc/guides/nics/index.rst index 92d56a59..23babe93 100644 --- a/dpdk/doc/guides/nics/index.rst +++ b/dpdk/doc/guides/nics/index.rst @@ -36,9 +36,15 @@ Network Interface Controller Drivers :numbered: overview + features + build_and_test + ark + avp bnx2x bnxt cxgbe + dpaa + dpaa2 e1000em ena enic @@ -46,16 +52,23 @@ Network Interface Controller Drivers i40e ixgbe intel_vf + kni + liquidio mlx4 mlx5 + mrvl nfp + octeontx qede + sfc_efx szedata2 + tap thunderx virtio vhost vmxnet3 pcap_ring + fail_safe **Figures** diff --git a/dpdk/doc/guides/nics/intel_vf.rst b/dpdk/doc/guides/nics/intel_vf.rst index 95a79b59..1e83bf6e 100644 --- a/dpdk/doc/guides/nics/intel_vf.rst +++ b/dpdk/doc/guides/nics/intel_vf.rst @@ -124,12 +124,12 @@ However: The above is an important consideration to take into account when targeting specific packets to a selected port. -Intel® Fortville 10/40 Gigabit Ethernet Controller VF Infrastructure -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Intel® X710/XL710 Gigabit Ethernet Controller VF Infrastructure +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ In a virtualized environment, the programmer can enable a maximum of *128 Virtual Functions (VF)* -globally per Intel® Fortville 10/40 Gigabit Ethernet Controller NIC device. -Each VF can have a maximum of 16 queue pairs. +globally per Intel® X710/XL710 Gigabit Ethernet Controller NIC device. +The number of queue pairs of each VF can be configured by ``CONFIG_RTE_LIBRTE_I40E_QUEUE_NUM_PER_VF`` in ``config`` file. The Physical Function in host could be either configured by the Linux* i40e driver (in the case of the Linux Kernel-based Virtual Machine [KVM]) or by DPDK PMD PF driver. When using both DPDK PMD PF/VF drivers, the whole NIC will be taken over by DPDK based application. @@ -156,44 +156,6 @@ For example, Launch the DPDK testpmd/example or your own host daemon application using the DPDK PMD library. -* Using the DPDK PMD PF ixgbe driver to enable VF RSS: - - Same steps as above to install the modules of uio, igb_uio, specify max_vfs for PCI device, and - launch the DPDK testpmd/example or your own host daemon application using the DPDK PMD library. - - The available queue number(at most 4) per VF depends on the total number of pool, which is - determined by the max number of VF at PF initialization stage and the number of queue specified - in config: - - * If the max number of VF is set in the range of 1 to 32: - - If the number of rxq is specified as 4(e.g. '--rxq 4' in testpmd), then there are totally 32 - pools(ETH_32_POOLS), and each VF could have 4 or less(e.g. 2) queues; - - If the number of rxq is specified as 2(e.g. '--rxq 2' in testpmd), then there are totally 32 - pools(ETH_32_POOLS), and each VF could have 2 queues; - - * If the max number of VF is in the range of 33 to 64: - - If the number of rxq is 4 ('--rxq 4' in testpmd), then error message is expected as rxq is not - correct at this case; - - If the number of rxq is 2 ('--rxq 2' in testpmd), then there is totally 64 pools(ETH_64_POOLS), - and each VF have 2 queues; - - On host, to enable VF RSS functionality, rx mq mode should be set as ETH_MQ_RX_VMDQ_RSS - or ETH_MQ_RX_RSS mode, and SRIOV mode should be activated(max_vfs >= 1). - It also needs config VF RSS information like hash function, RSS key, RSS key length. - - .. code-block:: console - - testpmd -c 0xffff -n 4 -- --coremask= --rxq=4 --txq=4 -i - - The limitation for VF RSS on Intel® 82599 10 Gigabit Ethernet Controller is: - The hash and key are shared among PF and all VF, the RETA table with 128 entries is also shared - among PF and all VF; So it could not to provide a method to query the hash and reta content per - VF on guest, while, if possible, please query them on host(PF) for the shared RETA information. - Virtual Function enumeration is performed in the following sequence by the Linux* pci driver for a dual-port NIC. When you enable the four Virtual Functions with the above command, the four enabled functions have a Function# represented by (Bus#, Device#, Function#) in sequence starting from 0 to 3. @@ -207,6 +169,9 @@ However: The above is an important consideration to take into account when targeting specific packets to a selected port. + For Intel® X710/XL710 Gigabit Ethernet Controller, queues are in pairs. One queue pair means one receive queue and + one transmit queue. The default number of queue pairs per VF is 4, and can be 16 in maximum. + Intel® 82599 10 Gigabit Ethernet Controller VF Infrastructure ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -241,6 +206,42 @@ For example, Launch the DPDK testpmd/example or your own host daemon application using the DPDK PMD library. +* Using the DPDK PMD PF ixgbe driver to enable VF RSS: + + Same steps as above to install the modules of uio, igb_uio, specify max_vfs for PCI device, and + launch the DPDK testpmd/example or your own host daemon application using the DPDK PMD library. + + The available queue number (at most 4) per VF depends on the total number of pool, which is + determined by the max number of VF at PF initialization stage and the number of queue specified + in config: + + * If the max number of VFs (max_vfs) is set in the range of 1 to 32: + + If the number of Rx queues is specified as 4 (``--rxq=4`` in testpmd), then there are totally 32 + pools (ETH_32_POOLS), and each VF could have 4 Rx queues; + + If the number of Rx queues is specified as 2 (``--rxq=2`` in testpmd), then there are totally 32 + pools (ETH_32_POOLS), and each VF could have 2 Rx queues; + + * If the max number of VFs (max_vfs) is in the range of 33 to 64: + + If the number of Rx queues in specified as 4 (``--rxq=4`` in testpmd), then error message is expected + as ``rxq`` is not correct at this case; + + If the number of rxq is 2 (``--rxq=2`` in testpmd), then there is totally 64 pools (ETH_64_POOLS), + and each VF have 2 Rx queues; + + On host, to enable VF RSS functionality, rx mq mode should be set as ETH_MQ_RX_VMDQ_RSS + or ETH_MQ_RX_RSS mode, and SRIOV mode should be activated (max_vfs >= 1). + It also needs config VF RSS information like hash function, RSS key, RSS key length. + +.. note:: + + The limitation for VF RSS on Intel® 82599 10 Gigabit Ethernet Controller is: + The hash and key are shared among PF and all VF, the RETA table with 128 entries is also shared + among PF and all VF; So it could not to provide a method to query the hash and reta content per + VF on guest, while, if possible, please query them on host for the shared RETA information. + Virtual Function enumeration is performed in the following sequence by the Linux* pci driver for a dual-port NIC. When you enable the four Virtual Functions with the above command, the four enabled functions have a Function# represented by (Bus#, Device#, Function#) in sequence starting from 0 to 3. @@ -486,7 +487,7 @@ The setup procedure is as follows: .. note:: - — The pci-assign,host=08:10.0 alue indicates that you want to attach a PCI device + — The pci-assign,host=08:10.0 value indicates that you want to attach a PCI device to a Virtual Machine and the respective (Bus:Device.Function) numbers should be passed for the Virtual Function to be attached. @@ -513,7 +514,7 @@ The setup procedure is as follows: .. code-block:: console make install T=x86_64-native-linuxapp-gcc - ./x86_64-native-linuxapp-gcc/app/testpmd -c f -n 4 -- -i + ./x86_64-native-linuxapp-gcc/app/testpmd -l 0-3 -n 4 -- -i #. Finally, access the Guest OS using vncviewer with the localhost:5900 port and check the lspci command output in the Guest OS. The virtual functions will be listed as available for use. diff --git a/dpdk/doc/guides/nics/ixgbe.rst b/dpdk/doc/guides/nics/ixgbe.rst index 3dc6b004..d477ea05 100644 --- a/dpdk/doc/guides/nics/ixgbe.rst +++ b/dpdk/doc/guides/nics/ixgbe.rst @@ -1,5 +1,5 @@ .. BSD LICENSE - Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + Copyright(c) 2010-2016 Intel Corporation. All rights reserved. All rights reserved. Redistribution and use in source and binary forms, with or without @@ -95,9 +95,6 @@ Other features are supported using optional MACRO configuration. They include: * HW extend dual VLAN -* Enabled by RX_OLFLAGS (RTE_IXGBE_RX_OLFLAGS_ENABLE=y) - - To guarantee the constraint, configuration flags in dev_conf.rxmode will be checked: * hw_vlan_strip @@ -129,7 +126,7 @@ The tx_rs_thresh value must be greater than or equal to RTE_PMD_IXGBE_TX_MAX_BUR but less or equal to RTE_IXGBE_TX_MAX_FREE_BUF_SZ. Consequently, by default the tx_rs_thresh value is in the range 32 to 64. -Feature not Supported by RX Vector PMD +Feature not Supported by TX Vector PMD ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ TX vPMD only works when txq_flags is set to IXGBE_SIMPLE_FLAGS. @@ -147,41 +144,34 @@ The following MACROs are used for these three features: * ETH_TXQ_FLAGS_NOXSUMTCP +Application Programming Interface +--------------------------------- + +In DPDK release v16.11 an API for ixgbe specific functions has been added to the ixgbe PMD. +The declarations for the API functions are in the header ``rte_pmd_ixgbe.h``. Sample Application Notes -~~~~~~~~~~~~~~~~~~~~~~~~ - -testpmd -^^^^^^^ - -By default, using CONFIG_RTE_IXGBE_RX_OLFLAGS_ENABLE=y: - -.. code-block:: console - - ./x86_64-native-linuxapp-gcc/app/testpmd -c 300 -n 4 -- -i --burst=32 --rxfreet=32 --mbcache=250 --txpt=32 --rxht=8 --rxwt=0 --txfreet=32 --txrst=32 --txqflags=0xf01 - -When CONFIG_RTE_IXGBE_RX_OLFLAGS_ENABLE=n, better performance can be achieved: - -.. code-block:: console - - ./x86_64-native-linuxapp-gcc/app/testpmd -c 300 -n 4 -- -i --burst=32 --rxfreet=32 --mbcache=250 --txpt=32 --rxht=8 --rxwt=0 --txfreet=32 --txrst=32 --txqflags=0xf01 --disable-hw-vlan +------------------------ l3fwd -^^^^^ +~~~~~ When running l3fwd with vPMD, there is one thing to note. In the configuration, ensure that port_conf.rxmode.hw_ip_checksum=0. Otherwise, by default, RX vPMD is disabled. load_balancer -^^^^^^^^^^^^^ +~~~~~~~~~~~~~ As in the case of l3fwd, set configure port_conf.rxmode.hw_ip_checksum=0 to enable vPMD. In addition, for improved performance, use -bsz "(32,32),(64,64),(32,32)" in load_balancer to avoid using the default burst size of 144. +Limitations or Known issues +--------------------------- + Malicious Driver Detection not Supported ----------------------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The Intel x550 series NICs support a feature called MDD (Malicious Driver Detection) which checks the behavior of the VF driver. @@ -195,14 +185,17 @@ There's significant performance impact to support MDD. DPDK should check if the advanced context descriptor should be set and set it. And DPDK has to ask the info about the header length from the upper layer, because parsing the packet itself is not acceptable. So, it's too expensive to support MDD. -When using kernel PF + DPDK VF on x550, please make sure using the kernel -driver that disables MDD or can disable MDD. (Some kernel driver can use -this CLI 'insmod ixgbe.ko MDD=0,0' to disable MDD. Some kernel driver disables -it by default.) +When using kernel PF + DPDK VF on x550, please make sure to use a kernel +PF driver that disables MDD or can disable MDD. + +Some kernel drivers already disable MDD by default while some kernels can use +the command ``insmod ixgbe.ko MDD=0,0`` to disable MDD. Each "0" in the +command refers to a port. For example, if there are 6 ixgbe ports, the command +should be changed to ``insmod ixgbe.ko MDD=0,0,0,0,0,0``. Statistics ----------- +~~~~~~~~~~ The statistics of ixgbe hardware must be polled regularly in order for it to remain consistent. Running a DPDK application without polling the statistics will @@ -224,3 +217,71 @@ be calculated as follows: max_read_interval = ~4 mins 48 sec. In order to ensure valid results, it is recommended to poll every 4 minutes. + +MTU setting +~~~~~~~~~~~ + +Although the user can set the MTU separately on PF and VF ports, the ixgbe NIC +only supports one global MTU per physical port. +So when the user sets different MTUs on PF and VF ports in one physical port, +the real MTU for all these PF and VF ports is the largest value set. +This behavior is based on the kernel driver behavior. + +VF MAC address setting +~~~~~~~~~~~~~~~~~~~~~~ + +On ixgbe, the concept of "pool" can be used for different things depending on +the mode. In VMDq mode, "pool" means a VMDq pool. In IOV mode, "pool" means a +VF. + +There is no RTE API to add a VF's MAC address from the PF. On ixgbe, the +``rte_eth_dev_mac_addr_add()`` function can be used to add a VF's MAC address, +as a workaround. + + +Inline crypto processing support +-------------------------------- + +Inline IPsec processing is supported for ``RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO`` +mode for ESP packets only: + +- ESP authentication only: AES-128-GMAC (128-bit key) +- ESP encryption and authentication: AES-128-GCM (128-bit key) + +IPsec Security Gateway Sample Application supports inline IPsec processing for +ixgbe PMD. + +For more details see the IPsec Security Gateway Sample Application and Security +library documentation. + + +Supported Chipsets and NICs +--------------------------- + +- Intel 82599EB 10 Gigabit Ethernet Controller +- Intel 82598EB 10 Gigabit Ethernet Controller +- Intel 82599ES 10 Gigabit Ethernet Controller +- Intel 82599EN 10 Gigabit Ethernet Controller +- Intel Ethernet Controller X540-AT2 +- Intel Ethernet Controller X550-BT2 +- Intel Ethernet Controller X550-AT2 +- Intel Ethernet Controller X550-AT +- Intel Ethernet Converged Network Adapter X520-SR1 +- Intel Ethernet Converged Network Adapter X520-SR2 +- Intel Ethernet Converged Network Adapter X520-LR1 +- Intel Ethernet Converged Network Adapter X520-DA1 +- Intel Ethernet Converged Network Adapter X520-DA2 +- Intel Ethernet Converged Network Adapter X520-DA4 +- Intel Ethernet Converged Network Adapter X520-QDA1 +- Intel Ethernet Converged Network Adapter X520-T2 +- Intel 10 Gigabit AF DA Dual Port Server Adapter +- Intel 10 Gigabit AT Server Adapter +- Intel 10 Gigabit AT2 Server Adapter +- Intel 10 Gigabit CX4 Dual Port Server Adapter +- Intel 10 Gigabit XF LR Server Adapter +- Intel 10 Gigabit XF SR Dual Port Server Adapter +- Intel 10 Gigabit XF SR Server Adapter +- Intel Ethernet Converged Network Adapter X540-T1 +- Intel Ethernet Converged Network Adapter X540-T2 +- Intel Ethernet Converged Network Adapter X550-T1 +- Intel Ethernet Converged Network Adapter X550-T2 diff --git a/dpdk/doc/guides/nics/kni.rst b/dpdk/doc/guides/nics/kni.rst new file mode 100644 index 00000000..77542b56 --- /dev/null +++ b/dpdk/doc/guides/nics/kni.rst @@ -0,0 +1,197 @@ +.. BSD LICENSE + Copyright(c) 2017 Intel Corporation. All rights reserved. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +KNI Poll Mode Driver +====================== + +KNI PMD is wrapper to the :ref:`librte_kni ` library. + +This PMD enables using KNI without having a KNI specific application, +any forwarding application can use PMD interface for KNI. + +Sending packets to any DPDK controlled interface or sending to the +Linux networking stack will be transparent to the DPDK application. + +To create a KNI device ``net_kni#`` device name should be used, and this +will create ``kni#`` Linux virtual network interface. + +There is no physical device backend for the virtual KNI device. + +Packets sent to the KNI Linux interface will be received by the DPDK +application, and DPDK application may forward packets to a physical NIC +or to a virtual device (like another KNI interface or PCAP interface). + +To forward any traffic from physical NIC to the Linux networking stack, +an application should control a physical port and create one virtual KNI port, +and forward between two. + +Using this PMD requires KNI kernel module be inserted. + + +Usage +----- + +EAL ``--vdev`` argument can be used to create KNI device instance, like:: + + testpmd --vdev=net_kni0 --vdev=net_kn1 -- -i + +Above command will create ``kni0`` and ``kni1`` Linux network interfaces, +those interfaces can be controlled by standard Linux tools. + +When testpmd forwarding starts, any packets sent to ``kni0`` interface +forwarded to the ``kni1`` interface and vice versa. + +There is no hard limit on number of interfaces that can be created. + + +Default interface configuration +------------------------------- + +``librte_kni`` can create Linux network interfaces with different features, +feature set controlled by a configuration struct, and KNI PMD uses a fixed +configuration: + + .. code-block:: console + + Interface name: kni# + force bind kernel thread to a core : NO + mbuf size: MAX_PACKET_SZ + +KNI control path is not supported with the PMD, since there is no physical +backend device by default. + + +PMD arguments +------------- + +``no_request_thread``, by default PMD creates a phtread for each KNI interface +to handle Linux network interface control commands, like ``ifconfig kni0 up`` + +With ``no_request_thread`` option, pthread is not created and control commands +not handled by PMD. + +By default request thread is enabled. And this argument should not be used +most of the time, unless this PMD used with customized DPDK application to handle +requests itself. + +Argument usage:: + + testpmd --vdev "net_kni0,no_request_thread=1" -- -i + + +PMD log messages +---------------- + +If KNI kernel module (rte_kni.ko) not inserted, following error log printed:: + + "KNI: KNI subsystem has not been initialized. Invoke rte_kni_init() first" + + +PMD testing +----------- + +It is possible to test PMD quickly using KNI kernel module loopback feature: + +* Insert KNI kernel module with loopback support: + + .. code-block:: console + + insmod build/kmod/rte_kni.ko lo_mode=lo_mode_fifo_skb + +* Start testpmd with no physical device but two KNI virtual devices: + + .. code-block:: console + + ./testpmd --vdev net_kni0 --vdev net_kni1 -- -i + + .. code-block:: console + + ... + Configuring Port 0 (socket 0) + KNI: pci: 00:00:00 c580:b8 + Port 0: 1A:4A:5B:7C:A2:8C + Configuring Port 1 (socket 0) + KNI: pci: 00:00:00 600:b9 + Port 1: AE:95:21:07:93:DD + Checking link statuses... + Port 0 Link Up - speed 10000 Mbps - full-duplex + Port 1 Link Up - speed 10000 Mbps - full-duplex + Done + testpmd> + +* Observe Linux interfaces + + .. code-block:: console + + $ ifconfig kni0 && ifconfig kni1 + kni0: flags=4098 mtu 1500 + ether ae:8e:79:8e:9b:c8 txqueuelen 1000 (Ethernet) + RX packets 0 bytes 0 (0.0 B) + RX errors 0 dropped 0 overruns 0 frame 0 + TX packets 0 bytes 0 (0.0 B) + TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0 + + kni1: flags=4098 mtu 1500 + ether 9e:76:43:53:3e:9b txqueuelen 1000 (Ethernet) + RX packets 0 bytes 0 (0.0 B) + RX errors 0 dropped 0 overruns 0 frame 0 + TX packets 0 bytes 0 (0.0 B) + TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0 + + +* Start forwarding with tx_first: + + .. code-block:: console + + testpmd> start tx_first + +* Quit and check forwarding stats: + + .. code-block:: console + + testpmd> quit + Telling cores to stop... + Waiting for lcores to finish... + + ---------------------- Forward statistics for port 0 ---------------------- + RX-packets: 35637905 RX-dropped: 0 RX-total: 35637905 + TX-packets: 35637947 TX-dropped: 0 TX-total: 35637947 + ---------------------------------------------------------------------------- + + ---------------------- Forward statistics for port 1 ---------------------- + RX-packets: 35637915 RX-dropped: 0 RX-total: 35637915 + TX-packets: 35637937 TX-dropped: 0 TX-total: 35637937 + ---------------------------------------------------------------------------- + + +++++++++++++++ Accumulated forward statistics for all ports+++++++++++++++ + RX-packets: 71275820 RX-dropped: 0 RX-total: 71275820 + TX-packets: 71275884 TX-dropped: 0 TX-total: 71275884 + ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + diff --git a/dpdk/doc/guides/nics/liquidio.rst b/dpdk/doc/guides/nics/liquidio.rst new file mode 100644 index 00000000..7bc16040 --- /dev/null +++ b/dpdk/doc/guides/nics/liquidio.rst @@ -0,0 +1,233 @@ +.. BSD LICENSE + Copyright(c) 2017 Cavium, Inc.. All rights reserved. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Cavium, Inc. nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +LiquidIO VF Poll Mode Driver +============================ + +The LiquidIO VF PMD library (librte_pmd_lio) provides poll mode driver support for +Cavium LiquidIO® II server adapter VFs. PF management and VF creation can be +done using kernel driver. + +More information can be found at `Cavium Official Website +`_. + +Supported LiquidIO Adapters +----------------------------- + +- LiquidIO II CN2350 210SV/225SV +- LiquidIO II CN2350 210SVPT +- LiquidIO II CN2360 210SV/225SV +- LiquidIO II CN2360 210SVPT + + +Pre-Installation Configuration +------------------------------ + +The following options can be modified in the ``config`` file. +Please note that enabling debugging options may affect system performance. + +- ``CONFIG_RTE_LIBRTE_LIO_PMD`` (default ``y``) + + Toggle compilation of LiquidIO PMD. + +- ``CONFIG_RTE_LIBRTE_LIO_DEBUG_DRIVER`` (default ``n``) + + Toggle display of generic debugging messages. + +- ``CONFIG_RTE_LIBRTE_LIO_DEBUG_INIT`` (default ``n``) + + Toggle display of initialization related messages. + +- ``CONFIG_RTE_LIBRTE_LIO_DEBUG_RX`` (default ``n``) + + Toggle display of receive fast path run-time messages. + +- ``CONFIG_RTE_LIBRTE_LIO_DEBUG_TX`` (default ``n``) + + Toggle display of transmit fast path run-time messages. + +- ``CONFIG_RTE_LIBRTE_LIO_DEBUG_MBOX`` (default ``n``) + + Toggle display of mailbox messages. + +- ``CONFIG_RTE_LIBRTE_LIO_DEBUG_REGS`` (default ``n``) + + Toggle display of register reads and writes. + + +SR-IOV: Prerequisites and Sample Application Notes +-------------------------------------------------- + +This section provides instructions to configure SR-IOV with Linux OS. + +#. Verify SR-IOV and ARI capabilities are enabled on the adapter using ``lspci``: + + .. code-block:: console + + lspci -s -vvv + + Example output: + + .. code-block:: console + + [...] + Capabilities: [148 v1] Alternative Routing-ID Interpretation (ARI) + [...] + Capabilities: [178 v1] Single Root I/O Virtualization (SR-IOV) + [...] + Kernel driver in use: LiquidIO + +#. Load the kernel module: + + .. code-block:: console + + modprobe liquidio + +#. Bring up the PF ports: + + .. code-block:: console + + ifconfig p4p1 up + ifconfig p4p2 up + +#. Change PF MTU if required: + + .. code-block:: console + + ifconfig p4p1 mtu 9000 + ifconfig p4p2 mtu 9000 + +#. Create VF device(s): + + Echo number of VFs to be created into ``"sriov_numvfs"`` sysfs entry + of the parent PF. + + .. code-block:: console + + echo 1 > /sys/bus/pci/devices/0000:03:00.0/sriov_numvfs + echo 1 > /sys/bus/pci/devices/0000:03:00.1/sriov_numvfs + +#. Assign VF MAC address: + + Assign MAC address to the VF using iproute2 utility. The syntax is:: + + ip link set vf mac + + Example output: + + .. code-block:: console + + ip link set p4p1 vf 0 mac F2:A8:1B:5E:B4:66 + +#. Assign VF(s) to VM. + + The VF devices may be passed through to the guest VM using qemu or + virt-manager or virsh etc. + + Example qemu guest launch command: + + .. code-block:: console + + ./qemu-system-x86_64 -name lio-vm -machine accel=kvm \ + -cpu host -m 4096 -smp 4 \ + -drive file=,if=none,id=disk1,format= \ + -device virtio-blk-pci,scsi=off,drive=disk1,id=virtio-disk1,bootindex=1 \ + -device vfio-pci,host=03:00.3 -device vfio-pci,host=03:08.3 + +#. Running testpmd + + Refer to the document + :ref:`compiling and testing a PMD for a NIC ` to run + ``testpmd`` application. + + .. note:: + + Use ``igb_uio`` instead of ``vfio-pci`` in VM. + + Example output: + + .. code-block:: console + + [...] + EAL: PCI device 0000:03:00.3 on NUMA socket 0 + EAL: probe driver: 177d:9712 net_liovf + EAL: using IOMMU type 1 (Type 1) + PMD: net_liovf[03:00.3]INFO: DEVICE : CN23XX VF + EAL: PCI device 0000:03:08.3 on NUMA socket 0 + EAL: probe driver: 177d:9712 net_liovf + PMD: net_liovf[03:08.3]INFO: DEVICE : CN23XX VF + Interactive-mode selected + USER1: create a new mbuf pool : n=171456, size=2176, socket=0 + Configuring Port 0 (socket 0) + PMD: net_liovf[03:00.3]INFO: Starting port 0 + Port 0: F2:A8:1B:5E:B4:66 + Configuring Port 1 (socket 0) + PMD: net_liovf[03:08.3]INFO: Starting port 1 + Port 1: 32:76:CC:EE:56:D7 + Checking link statuses... + Port 0 Link Up - speed 10000 Mbps - full-duplex + Port 1 Link Up - speed 10000 Mbps - full-duplex + Done + testpmd> + +#. Enabling VF promiscuous mode + + One VF per PF can be marked as trusted for promiscuous mode. + + .. code-block:: console + + ip link set dev vf trust on + + +Limitations +----------- + +VF MTU +~~~~~~ + +VF MTU is limited by PF MTU. Raise PF value before configuring VF for larger packet size. + +VLAN offload +~~~~~~~~~~~~ + +Tx VLAN insertion is not supported and consequently VLAN offload feature is +marked partial. + +Ring size +~~~~~~~~~ + +Number of descriptors for Rx/Tx ring should be in the range 128 to 512. + +CRC striping +~~~~~~~~~~~~ + +LiquidIO adapters strip ethernet FCS of every packet coming to the host +interface. So, CRC will be stripped even when the ``rxmode.hw_strip_crc`` +member is set to 0 in ``struct rte_eth_conf``. diff --git a/dpdk/doc/guides/nics/mlx4.rst b/dpdk/doc/guides/nics/mlx4.rst index 49f46263..cab45dfe 100644 --- a/dpdk/doc/guides/nics/mlx4.rst +++ b/dpdk/doc/guides/nics/mlx4.rst @@ -1,5 +1,6 @@ .. BSD LICENSE - Copyright 2012-2015 6WIND S.A. + Copyright 2012 6WIND S.A. + Copyright 2015 Mellanox Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions @@ -73,31 +74,6 @@ long as they share the same MAC address. Compiling librte_pmd_mlx4 causes DPDK to be linked against libibverbs. -Features --------- - -- RSS, also known as RCA, is supported. In this mode the number of - configured RX queues must be a power of two. -- VLAN filtering is supported. -- Link state information is provided. -- Promiscuous mode is supported. -- All multicast mode is supported. -- Multiple MAC addresses (unicast, multicast) can be configured. -- Scattered packets are supported for TX and RX. -- Inner L3/L4 (IP, TCP and UDP) TX/RX checksum offloading and validation. -- Outer L3 (IP) TX/RX checksum offloading and validation for VXLAN frames. -- Secondary process TX is supported. - -Limitations ------------ - -- RSS hash key cannot be modified. -- RSS RETA cannot be configured -- RSS always includes L3 (IPv4/IPv6) and L4 (UDP/TCP). They cannot be - dissociated. -- Hardware counters are not implemented (they are software counters). -- Secondary process RX is not supported. - Configuration ------------- @@ -116,18 +92,6 @@ These options can be modified in the ``.config`` file. adds additional run-time checks and debugging messages at the cost of lower performance. -- ``CONFIG_RTE_LIBRTE_MLX4_SGE_WR_N`` (default **4**) - - Number of scatter/gather elements (SGEs) per work request (WR). Lowering - this number improves performance but also limits the ability to receive - scattered packets (packets that do not fit a single mbuf). The default - value is a safe tradeoff. - -- ``CONFIG_RTE_LIBRTE_MLX4_MAX_INLINE`` (default **0**) - - Amount of data to be inlined during TX operations. Improves latency but - lowers throughput. - - ``CONFIG_RTE_LIBRTE_MLX4_TX_MP_CACHE`` (default **8**) Maximum number of cached memory pools (MPs) per TX queue. Each MP from @@ -136,32 +100,21 @@ These options can be modified in the ``.config`` file. This value is always 1 for RX queues since they use a single MP. -- ``CONFIG_RTE_LIBRTE_MLX4_SOFT_COUNTERS`` (default **1**) - - Toggle software counters. No counters are available if this option is - disabled since hardware counters are not supported. - -Environment variables -~~~~~~~~~~~~~~~~~~~~~ - -- ``MLX4_INLINE_RECV_SIZE`` - - A nonzero value enables inline receive for packets up to that size. May - significantly improve performance in some cases but lower it in - others. Requires careful testing. - Run-time configuration ~~~~~~~~~~~~~~~~~~~~~~ -- The only constraint when RSS mode is requested is to make sure the number - of RX queues is a power of two. This is a hardware requirement. - - librte_pmd_mlx4 brings kernel network interfaces up during initialization because it is affected by their state. Forcing them down prevents packets reception. - **ethtool** operations on related kernel interfaces also affect the PMD. +- ``port`` parameter [int] + + This parameter provides a physical port to probe and can be specified multiple + times for additional ports. All ports are probed by default if left + unspecified. + Kernel module parameters ~~~~~~~~~~~~~~~~~~~~~~~~ @@ -192,7 +145,7 @@ This driver relies on external libraries and kernel drivers for resources allocations and initialization. The following dependencies are not part of DPDK and must be installed separately: -- **libibverbs** +- **libibverbs** (provided by rdma-core package) User space verbs framework used by librte_pmd_mlx4. This library provides a generic interface between the kernel and low-level user space drivers @@ -202,7 +155,7 @@ DPDK and must be installed separately: resources allocations) to be managed by the kernel and fast operations to never leave user space. -- **libmlx4** +- **libmlx4** (provided by rdma-core package) Low-level user space driver library for Mellanox ConnectX-3 devices, it is automatically loaded by libibverbs. @@ -210,7 +163,7 @@ DPDK and must be installed separately: This library basically implements send/receive calls to the hardware queues. -- **Kernel modules** (mlnx-ofed-kernel) +- **Kernel modules** They provide the kernel-side verbs API and low level device drivers that manage actual hardware initialization and resources sharing with user @@ -236,25 +189,27 @@ DPDK and must be installed separately: Both libraries are BSD and GPL licensed. Linux kernel modules are GPL licensed. -Currently supported by DPDK: +Depending on system constraints and user preferences either RDMA core library +with a recent enough Linux kernel release (recommended) or Mellanox OFED, +which provides compatibility with older releases. -- Mellanox OFED **3.1**. -- Firmware version **2.35.5100** and higher. -- Supported architectures: **x86_64** and **POWER8**. +Current RDMA core package and Linux kernel (recommended) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Getting Mellanox OFED -~~~~~~~~~~~~~~~~~~~~~ +- Minimal Linux kernel version: 4.14. +- Minimal RDMA core version: v15 (see `RDMA core installation documentation`_). -While these libraries and kernel modules are available on OpenFabrics -Alliance's `website `_ and provided by package -managers on most distributions, this PMD requires Ethernet extensions that -may not be supported at the moment (this is a work in progress). +.. _`RDMA core installation documentation`: https://raw.githubusercontent.com/linux-rdma/rdma-core/master/README.md -`Mellanox OFED -`_ -includes the necessary support and should be used in the meantime. For DPDK, -only libibverbs, libmlx4, mlnx-ofed-kernel packages and firmware updates are -required from that distribution. +.. _Mellanox_OFED_as_a_fallback: + +Mellanox OFED as a fallback +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- `Mellanox OFED`_ version: **4.2**. +- firmware version: **2.42.5000** and above. + +.. _`Mellanox OFED`: http://www.mellanox.com/page/products_dyn?product_family=26&mtag=linux_sw_drivers .. note:: @@ -262,6 +217,155 @@ required from that distribution. this DPDK release was developed and tested against is strongly recommended. Please check the `prerequisites`_. +Installing Mellanox OFED +^^^^^^^^^^^^^^^^^^^^^^^^ + +1. Download latest Mellanox OFED. + +2. Install the required libraries and kernel modules either by installing + only the required set, or by installing the entire Mellanox OFED: + + For bare metal use: + + .. code-block:: console + + ./mlnxofedinstall --dpdk --upstream-libs + + For SR-IOV hypervisors use: + + .. code-block:: console + + ./mlnxofedinstall --dpdk --upstream-libs --enable-sriov --hypervisor + + For SR-IOV virtual machine use: + + .. code-block:: console + + ./mlnxofedinstall --dpdk --upstream-libs --guest + +3. Verify the firmware is the correct one: + + .. code-block:: console + + ibv_devinfo + +4. Set all ports links to Ethernet, follow instructions on the screen: + + .. code-block:: console + + connectx_port_config + +5. Continue with :ref:`section 2 of the Quick Start Guide `. + +Supported NICs +-------------- + +* Mellanox(R) ConnectX(R)-3 Pro 40G MCX354A-FCC_Ax (2*40G) + +.. _qsg: + +Quick Start Guide +----------------- + +1. Set all ports links to Ethernet + + .. code-block:: console + + PCI= + echo eth > "/sys/bus/pci/devices/$PCI/mlx4_port0" + echo eth > "/sys/bus/pci/devices/$PCI/mlx4_port1" + + .. note:: + + If using Mellanox OFED one can permanently set the port link + to Ethernet using connectx_port_config tool provided by it. + :ref:`Mellanox_OFED_as_a_fallback`: + +.. _QSG_2: + +2. In case of bare metal or hypervisor, configure optimized steering mode + by adding the following line to ``/etc/modprobe.d/mlx4_core.conf``: + + .. code-block:: console + + options mlx4_core log_num_mgm_entry_size=-7 + + .. note:: + + If VLAN filtering is used, set log_num_mgm_entry_size=-1. + Performance degradation can occur on this case. + +3. Restart the driver: + + .. code-block:: console + + /etc/init.d/openibd restart + + or: + + .. code-block:: console + + service openibd restart + +4. Compile DPDK and you are ready to go. See instructions on + :ref:`Development Kit Build System ` + +Performance tuning +------------------ + +1. Verify the optimized steering mode is configured: + + .. code-block:: console + + cat /sys/module/mlx4_core/parameters/log_num_mgm_entry_size + +2. Use the CPU near local NUMA node to which the PCIe adapter is connected, + for better performance. For VMs, verify that the right CPU + and NUMA node are pinned according to the above. Run: + + .. code-block:: console + + lstopo-no-graphics + + to identify the NUMA node to which the PCIe adapter is connected. + +3. If more than one adapter is used, and root complex capabilities allow + to put both adapters on the same NUMA node without PCI bandwidth degradation, + it is recommended to locate both adapters on the same NUMA node. + This in order to forward packets from one to the other without + NUMA performance penalty. + +4. Disable pause frames: + + .. code-block:: console + + ethtool -A rx off tx off + +5. Verify IO non-posted prefetch is disabled by default. This can be checked + via the BIOS configuration. Please contact you server provider for more + information about the settings. + +.. note:: + + On some machines, depends on the machine integrator, it is beneficial + to set the PCI max read request parameter to 1K. This can be + done in the following way: + + To query the read request size use: + + .. code-block:: console + + setpci -s 68.w + + If the output is different than 3XXX, set it by: + + .. code-block:: console + + setpci -s 68.w=3XXX + + The XXX can be different on different systems. Make sure to configure + according to the setpci output. + Usage example ------------- @@ -338,7 +442,7 @@ devices managed by librte_pmd_mlx4. .. code-block:: console - testpmd -c 0xff00 -n 4 -w 0000:83:00.0 -w 0000:84:00.0 -- --rxq=2 --txq=2 -i + testpmd -l 8-15 -n 4 -w 0000:83:00.0 -w 0000:84:00.0 -- --rxq=2 --txq=2 -i Example output: diff --git a/dpdk/doc/guides/nics/mlx5.rst b/dpdk/doc/guides/nics/mlx5.rst index 89231738..f9558da8 100644 --- a/dpdk/doc/guides/nics/mlx5.rst +++ b/dpdk/doc/guides/nics/mlx5.rst @@ -1,5 +1,6 @@ .. BSD LICENSE Copyright 2015 6WIND S.A. + Copyright 2015 Mellanox Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions @@ -30,10 +31,10 @@ MLX5 poll mode driver ===================== -The MLX5 poll mode driver library (**librte_pmd_mlx5**) provides support for -**Mellanox ConnectX-4** and **Mellanox ConnectX-4 Lx** families of -10/25/40/50/100 Gb/s adapters as well as their virtual functions (VF) in -SR-IOV context. +The MLX5 poll mode driver library (**librte_pmd_mlx5**) provides support +for **Mellanox ConnectX-4**, **Mellanox ConnectX-4 Lx** and **Mellanox +ConnectX-5** families of 10/25/40/50/100 Gb/s adapters as well as their +virtual functions (VF) in SR-IOV context. Information and documentation about these adapters can be found on the `Mellanox website `__. Help is also provided by the @@ -64,6 +65,9 @@ physical memory (or memory that does not belong to the current process). This capability allows the PMD to coexist with kernel network interfaces which remain functional, although they stop receiving unicast packets as long as they share the same MAC address. +This means legacy linux control tools (for example: ethtool, ifconfig and +more) can operate on the same network interfaces that owned by the DPDK +application. Enabling librte_pmd_mlx5 causes DPDK applications to be linked against libibverbs. @@ -71,6 +75,7 @@ libibverbs. Features -------- +- Multi arch support: x86_64, POWER8, ARMv8. - Multiple TX and RX queues. - Support for scattered TX and RX frames. - IPv4, IPv6, TCPv4, TCPv6, UDPv4 and UDPv6 RSS on any number of queues. @@ -86,16 +91,48 @@ Features - Hardware checksum offloads. - Flow director (RTE_FDIR_MODE_PERFECT, RTE_FDIR_MODE_PERFECT_MAC_VLAN and RTE_ETH_FDIR_REJECT). -- Secondary process TX is supported. +- Flow API. +- Multiple process. - KVM and VMware ESX SR-IOV modes are supported. +- RSS hash result is supported. +- Hardware TSO. +- Hardware checksum TX offload for VXLAN and GRE. +- RX interrupts. +- Statistics query including Basic, Extended and per queue. +- Rx HW timestamp. Limitations ----------- - Inner RSS for VXLAN frames is not supported yet. -- Port statistics through software counters only. -- Hardware checksum offloads for VXLAN inner header are not supported yet. -- Secondary process RX is not supported. +- Port statistics through software counters only. Flow statistics are + supported by hardware counters. +- Hardware checksum RX offloads for VXLAN inner header are not supported yet. +- Forked secondary process not supported. +- Flow pattern without any specific vlan will match for vlan packets as well: + + When VLAN spec is not specified in the pattern, the matching rule will be created with VLAN as a wild card. + Meaning, the flow rule:: + + flow create 0 ingress pattern eth / vlan vid is 3 / ipv4 / end ... + + Will only match vlan packets with vid=3. and the flow rules:: + + flow create 0 ingress pattern eth / ipv4 / end ... + + Or:: + + flow create 0 ingress pattern eth / vlan / ipv4 / end ... + + Will match any ipv4 packet (VLAN included). + +- A multi segment packet must have less than 6 segments in case the Tx burst function + is set to multi-packet send or Enhanced multi-packet send. Otherwise it must have + less than 50 segments. +- Count action for RTE flow is only supported in Mellanox OFED 4.2. +- Flows with a VXLAN Network Identifier equal (or ends to be equal) + to 0 are not supported. +- VXLAN TSO and checksum offloads are not supported on VM. Configuration ------------- @@ -140,6 +177,23 @@ Environment variables This is disabled by default since this can also decrease performance for unaligned packet sizes. +- ``MLX5_SHUT_UP_BF`` + + Configures HW Tx doorbell register as IO-mapped. + + By default, the HW Tx doorbell is configured as a write-combining register. + The register would be flushed to HW usually when the write-combining buffer + becomes full, but it depends on CPU design. + + Except for vectorized Tx burst routines, a write memory barrier is enforced + after updating the register so that the update can be immediately visible to + HW. + + When vectorized Tx burst is called, the barrier is set only if the burst size + is not aligned to MLX5_VPMD_TX_MAX_BURST. However, setting this environmental + variable will bring better latency even though the maximum throughput can + slightly decline. + Run-time configuration ~~~~~~~~~~~~~~~~~~~~~~ @@ -152,13 +206,12 @@ Run-time configuration - ``rxq_cqe_comp_en`` parameter [int] A nonzero value enables the compression of CQE on RX side. This feature - allows to save PCI bandwidth and improve performance at the cost of a - slightly higher CPU usage. Enabled by default. + allows to save PCI bandwidth and improve performance. Enabled by default. Supported on: - - x86_64 with ConnectX4 and ConnectX4 LX - - Power8 with ConnectX4 LX + - x86_64 with ConnectX-4, ConnectX-4 LX and ConnectX-5. + - POWER8 and ARMv8 with ConnectX-4 LX and ConnectX-5. - ``txq_inline`` parameter [int] @@ -166,8 +219,8 @@ Run-time configuration Can improve PPS performance when PCI back pressure is detected and may be useful for scenarios involving heavy traffic on many queues. - It is not enabled by default (set to 0) since the additional software - logic necessary to handle this mode can lower performance when back + Because additional software logic is necessary to handle this mode, this + option should be used with care, as it can lower performance when back pressure is not expected. - ``txqs_min_inline`` parameter [int] @@ -177,14 +230,69 @@ Run-time configuration This option should be used in combination with ``txq_inline`` above. + On ConnectX-4, ConnectX-4 LX and ConnectX-5 without Enhanced MPW: + + - Disabled by default. + - In case ``txq_inline`` is set recommendation is 4. + + On ConnectX-5 with Enhanced MPW: + + - Set to 8 by default. + - ``txq_mpw_en`` parameter [int] - A nonzero value enables multi-packet send. This feature allows the TX - burst function to pack up to five packets in two descriptors in order to - save PCI bandwidth and improve performance at the cost of a slightly - higher CPU usage. + A nonzero value enables multi-packet send (MPS) for ConnectX-4 Lx and + enhanced multi-packet send (Enhanced MPS) for ConnectX-5. MPS allows the + TX burst function to pack up multiple packets in a single descriptor + session in order to save PCI bandwidth and improve performance at the + cost of a slightly higher CPU usage. When ``txq_inline`` is set along + with ``txq_mpw_en``, TX burst function tries to copy entire packet data + on to TX descriptor instead of including pointer of packet only if there + is enough room remained in the descriptor. ``txq_inline`` sets + per-descriptor space for either pointers or inlined packets. In addition, + Enhanced MPS supports hybrid mode - mixing inlined packets and pointers + in the same descriptor. + + This option cannot be used in conjunction with ``tso`` below. When ``tso`` + is set, ``txq_mpw_en`` is disabled. + + It is currently only supported on the ConnectX-4 Lx and ConnectX-5 + families of adapters. Enabled by default. + +- ``txq_mpw_hdr_dseg_en`` parameter [int] + + A nonzero value enables including two pointers in the first block of TX + descriptor. This can be used to lessen CPU load for memory copy. + + Effective only when Enhanced MPS is supported. Disabled by default. + +- ``txq_max_inline_len`` parameter [int] + + Maximum size of packet to be inlined. This limits the size of packet to + be inlined. If the size of a packet is larger than configured value, the + packet isn't inlined even though there's enough space remained in the + descriptor. Instead, the packet is included with pointer. + + Effective only when Enhanced MPS is supported. The default value is 256. + +- ``tso`` parameter [int] + + A nonzero value enables hardware TSO. + When hardware TSO is enabled, packets marked with TCP segmentation + offload will be divided into segments by the hardware. Disabled by default. + +- ``tx_vec_en`` parameter [int] + + A nonzero value enables Tx vector on ConnectX-5 only NIC if the number of + global Tx queues on the port is lesser than MLX5_VPMD_MIN_TXQS. + + Enabled by default on ConnectX-5. + +- ``rx_vec_en`` parameter [int] + + A nonzero value enables Rx vector if the port is not configured in + multi-segment otherwise this parameter is ignored. - It is currently only supported on the ConnectX-4 Lx family of adapters. Enabled by default. Prerequisites @@ -206,13 +314,13 @@ DPDK and must be installed separately: - **libmlx5** - Low-level user space driver library for Mellanox ConnectX-4 devices, - it is automatically loaded by libibverbs. + Low-level user space driver library for Mellanox ConnectX-4/ConnectX-5 + devices, it is automatically loaded by libibverbs. This library basically implements send/receive calls to the hardware queues. -- **Kernel modules** (mlnx-ofed-kernel) +- **Kernel modules** They provide the kernel-side Verbs API and low level device drivers that manage actual hardware initialization and resources sharing with user @@ -221,14 +329,15 @@ DPDK and must be installed separately: Unlike most other PMDs, these modules must remain loaded and bound to their devices: - - mlx5_core: hardware driver managing Mellanox ConnectX-4 devices and - related Ethernet kernel network devices. + - mlx5_core: hardware driver managing Mellanox ConnectX-4/ConnectX-5 + devices and related Ethernet kernel network devices. - mlx5_ib: InifiniBand device driver. - ib_uverbs: user space driver for Verbs (entry point for libibverbs). - **Firmware update** - Mellanox OFED releases include firmware updates for ConnectX-4 adapters. + Mellanox OFED releases include firmware updates for ConnectX-4/ConnectX-5 + adapters. Because each release provides new features, these updates must be applied to match the kernel modules and libraries they come with. @@ -238,17 +347,33 @@ DPDK and must be installed separately: Both libraries are BSD and GPL licensed. Linux kernel modules are GPL licensed. -Currently supported by DPDK: +Installation +~~~~~~~~~~~~ -- Mellanox OFED **3.3-1.0.0.0** and **3.3-2.0.0.0**. +Either RDMA Core library with a recent enough Linux kernel release +(recommended) or Mellanox OFED, which provides compatibility with older +releases. -- Minimum firmware version: +RMDA Core with Linux Kernel +^^^^^^^^^^^^^^^^^^^^^^^^^^^ - - ConnectX-4: **12.16.1006** - - ConnectX-4 Lx: **14.16.1006** +- Minimal kernel version : v4.14 or the most recent 4.14-rc (see `Linux installation documentation`_) +- Minimal rdma-core version: v15+ commit 0c5f5765213a ("Merge pull request #227 from yishaih/tm") + (see `RDMA Core installation documentation`_) -Getting Mellanox OFED -~~~~~~~~~~~~~~~~~~~~~ +.. _`Linux installation documentation`: https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git/plain/Documentation/admin-guide/README.rst +.. _`RDMA Core installation documentation`: https://raw.githubusercontent.com/linux-rdma/rdma-core/master/README.md + +Mellanox OFED +^^^^^^^^^^^^^ + +- Mellanox OFED version: **4.2**. +- firmware version: + + - ConnectX-4: **12.21.1000** and above. + - ConnectX-4 Lx: **14.21.1000** and above. + - ConnectX-5: **16.21.1000** and above. + - ConnectX-5 Ex: **16.21.1000** and above. While these libraries and kernel modules are available on OpenFabrics Alliance's `website `__ and provided by package @@ -267,6 +392,178 @@ required from that distribution. this DPDK release was developed and tested against is strongly recommended. Please check the `prerequisites`_. +Supported NICs +-------------- + +* Mellanox(R) ConnectX(R)-4 10G MCX4111A-XCAT (1x10G) +* Mellanox(R) ConnectX(R)-4 10G MCX4121A-XCAT (2x10G) +* Mellanox(R) ConnectX(R)-4 25G MCX4111A-ACAT (1x25G) +* Mellanox(R) ConnectX(R)-4 25G MCX4121A-ACAT (2x25G) +* Mellanox(R) ConnectX(R)-4 40G MCX4131A-BCAT (1x40G) +* Mellanox(R) ConnectX(R)-4 40G MCX413A-BCAT (1x40G) +* Mellanox(R) ConnectX(R)-4 40G MCX415A-BCAT (1x40G) +* Mellanox(R) ConnectX(R)-4 50G MCX4131A-GCAT (1x50G) +* Mellanox(R) ConnectX(R)-4 50G MCX413A-GCAT (1x50G) +* Mellanox(R) ConnectX(R)-4 50G MCX414A-BCAT (2x50G) +* Mellanox(R) ConnectX(R)-4 50G MCX415A-GCAT (2x50G) +* Mellanox(R) ConnectX(R)-4 50G MCX416A-BCAT (2x50G) +* Mellanox(R) ConnectX(R)-4 50G MCX416A-GCAT (2x50G) +* Mellanox(R) ConnectX(R)-4 50G MCX415A-CCAT (1x100G) +* Mellanox(R) ConnectX(R)-4 100G MCX416A-CCAT (2x100G) +* Mellanox(R) ConnectX(R)-4 Lx 10G MCX4121A-XCAT (2x10G) +* Mellanox(R) ConnectX(R)-4 Lx 25G MCX4121A-ACAT (2x25G) +* Mellanox(R) ConnectX(R)-5 100G MCX556A-ECAT (2x100G) +* Mellanox(R) ConnectX(R)-5 Ex EN 100G MCX516A-CDAT (2x100G) + +Quick Start Guide on OFED +------------------------- + +1. Download latest Mellanox OFED. For more info check the `prerequisites`_. + + +2. Install the required libraries and kernel modules either by installing + only the required set, or by installing the entire Mellanox OFED: + + .. code-block:: console + + ./mlnxofedinstall --upstream-libs --dpdk + +3. Verify the firmware is the correct one: + + .. code-block:: console + + ibv_devinfo + +4. Verify all ports links are set to Ethernet: + + .. code-block:: console + + mlxconfig -d query | grep LINK_TYPE + LINK_TYPE_P1 ETH(2) + LINK_TYPE_P2 ETH(2) + + Link types may have to be configured to Ethernet: + + .. code-block:: console + + mlxconfig -d set LINK_TYPE_P1/2=1/2/3 + + * LINK_TYPE_P1=<1|2|3> , 1=Infiniband 2=Ethernet 3=VPI(auto-sense) + + For hypervisors verify SR-IOV is enabled on the NIC: + + .. code-block:: console + + mlxconfig -d query | grep SRIOV_EN + SRIOV_EN True(1) + + If needed, set enable the set the relevant fields: + + .. code-block:: console + + mlxconfig -d set SRIOV_EN=1 NUM_OF_VFS=16 + mlxfwreset -d reset + +5. Restart the driver: + + .. code-block:: console + + /etc/init.d/openibd restart + + or: + + .. code-block:: console + + service openibd restart + + If link type was changed, firmware must be reset as well: + + .. code-block:: console + + mlxfwreset -d reset + + For hypervisors, after reset write the sysfs number of virtual functions + needed for the PF. + + To dynamically instantiate a given number of virtual functions (VFs): + + .. code-block:: console + + echo [num_vfs] > /sys/class/infiniband/mlx5_0/device/sriov_numvfs + +6. Compile DPDK and you are ready to go. See instructions on + :ref:`Development Kit Build System ` + +Performance tuning +------------------ + +1. Configure aggressive CQE Zipping for maximum performance: + + .. code-block:: console + + mlxconfig -d s CQE_COMPRESSION=1 + + To set it back to the default CQE Zipping mode use: + + .. code-block:: console + + mlxconfig -d s CQE_COMPRESSION=0 + +2. In case of virtualization: + + - Make sure that hypervisor kernel is 3.16 or newer. + - Configure boot with ``iommu=pt``. + - Use 1G huge pages. + - Make sure to allocate a VM on huge pages. + - Make sure to set CPU pinning. + +3. Use the CPU near local NUMA node to which the PCIe adapter is connected, + for better performance. For VMs, verify that the right CPU + and NUMA node are pinned according to the above. Run: + + .. code-block:: console + + lstopo-no-graphics + + to identify the NUMA node to which the PCIe adapter is connected. + +4. If more than one adapter is used, and root complex capabilities allow + to put both adapters on the same NUMA node without PCI bandwidth degradation, + it is recommended to locate both adapters on the same NUMA node. + This in order to forward packets from one to the other without + NUMA performance penalty. + +5. Disable pause frames: + + .. code-block:: console + + ethtool -A rx off tx off + +6. Verify IO non-posted prefetch is disabled by default. This can be checked + via the BIOS configuration. Please contact you server provider for more + information about the settings. + +.. note:: + + On some machines, depends on the machine integrator, it is beneficial + to set the PCI max read request parameter to 1K. This can be + done in the following way: + + To query the read request size use: + + .. code-block:: console + + setpci -s 68.w + + If the output is different than 3XXX, set it by: + + .. code-block:: console + + setpci -s 68.w=3XXX + + The XXX can be different on different systems. Make sure to configure + according to the setpci output. + Notes for testpmd ----------------- @@ -287,8 +584,8 @@ behavior as librte_pmd_mlx4: Usage example ------------- -This section demonstrates how to launch **testpmd** with Mellanox ConnectX-4 -devices managed by librte_pmd_mlx5. +This section demonstrates how to launch **testpmd** with Mellanox +ConnectX-4/ConnectX-5 devices managed by librte_pmd_mlx5. #. Load the kernel modules: @@ -355,7 +652,7 @@ devices managed by librte_pmd_mlx5. .. code-block:: console - testpmd -c 0xff00 -n 4 -w 05:00.0 -w 05:00.1 -w 06:00.0 -w 06:00.1 -- --rxq=2 --txq=2 -i + testpmd -l 8-15 -n 4 -w 05:00.0 -w 05:00.1 -w 06:00.0 -w 06:00.1 -- --rxq=2 --txq=2 -i Example output: diff --git a/dpdk/doc/guides/nics/mrvl.rst b/dpdk/doc/guides/nics/mrvl.rst new file mode 100644 index 00000000..fbfdf478 --- /dev/null +++ b/dpdk/doc/guides/nics/mrvl.rst @@ -0,0 +1,253 @@ +.. BSD LICENSE + Copyright(c) 2017 Marvell International Ltd. + Copyright(c) 2017 Semihalf. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +MRVL Poll Mode Driver +====================== + +The MRVL PMD (librte_pmd_mrvl) provides poll mode driver support +for the Marvell PPv2 (Packet Processor v2) 1/10 Gbps adapter. + +Detailed information about SoCs that use PPv2 can be obtained here: + +* https://www.marvell.com/embedded-processors/armada-70xx/ +* https://www.marvell.com/embedded-processors/armada-80xx/ + +.. Note:: + + Due to external dependencies, this driver is disabled by default. It must + be enabled manually by setting relevant configuration option manually. + Please refer to `Config File Options`_ section for further details. + + +Features +-------- + +Features of the MRVL PMD are: + +- Speed capabilities +- Link status +- Queue start/stop +- MTU update +- Jumbo frame +- Promiscuous mode +- Allmulticast mode +- Unicast MAC filter +- Multicast MAC filter +- RSS hash +- VLAN filter +- CRC offload +- L3 checksum offload +- L4 checksum offload +- Packet type parsing +- Basic stats +- QoS + + +Limitations +----------- + +- Number of lcores is limited to 9 by MUSDK internal design. If more lcores + need to be allocated, locking will have to be considered. Number of available + lcores can be changed via ``MRVL_MUSDK_HIFS_RESERVED`` define in + ``mrvl_ethdev.c`` source file. + +- Flushing vlans added for filtering is not possible due to MUSDK missing + functionality. Current workaround is to reset board so that PPv2 has a + chance to start in a sane state. + + +Prerequisites +------------- + +- Custom Linux Kernel sources available + `here `__. + +- Out of tree `mvpp2x_sysfs` kernel module sources available + `here `__. + +- MUSDK (Marvell User-Space SDK) sources available + `here `__. + + MUSDK is a light-weight library that provides direct access to Marvell's + PPv2 (Packet Processor v2). Alternatively prebuilt MUSDK library can be + requested from `Marvell Extranet `_. Once + approval has been granted, library can be found by typing ``musdk`` in + the search box. + + MUSDK must be configured with the following features: + + .. code-block:: console + + --enable-bpool-dma=64 + +- DPDK environment + + Follow the DPDK :ref:`Getting Started Guide for Linux ` to setup + DPDK environment. + + +Config File Options +------------------- + +The following options can be modified in the ``config`` file. + +- ``CONFIG_RTE_LIBRTE_MRVL_PMD`` (default ``n``) + + Toggle compilation of the librte_pmd_mrvl driver. + +- ``CONFIG_RTE_MRVL_MUSDK_DMA_MEMSIZE`` (default ``41943040``) + + Size in bytes of the contiguous memory region that MUSDK will allocate + for run-time DMA-able data buffers. + + +QoS Configuration +----------------- + +QoS configuration is done through external configuration file. Path to the +file must be given as `cfg` in driver's vdev parameter list. + +Configuration syntax +~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: console + + [port default] + default_tc = + qos_mode = + + [port tc ] + rxq = + pcp = + dscp = + + [port tc ] + rxq = + pcp = + dscp = + +Where: + +- ````: DPDK Port number (0..n). + +- ````: Default traffic class (e.g. 0) + +- ````: QoS priority for mapping (`ip`, `vlan`, `ip/vlan` or `vlan/ip`). + +- ````: Traffic Class to be configured. + +- ````: List of DPDK RX queues (e.g. 0 1 3-4) + +- ````: List of PCP values to handle in particular TC (e.g. 0 1 3-4 7). + +- ````: List of DSCP values to handle in particular TC (e.g. 0-12 32-48 63). + +Setting PCP/DSCP values for the default TC is not required. All PCP/DSCP +values not assigned explicitly to particular TC will be handled by the +default TC. + +Configuration file example +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: console + + [port 0 default] + default_tc = 0 + qos_mode = ip + + [port 0 tc 0] + rxq = 0 1 + + [port 0 tc 1] + rxq = 2 + pcp = 5 6 7 + dscp = 26-38 + + [port 1 default] + default_tc = 0 + qos_mode = vlan/ip + + [port 1 tc 0] + rxq = 0 + + [port 1 tc 1] + rxq = 1 2 + pcp = 5 6 7 + dscp = 26-38 + +Usage example +^^^^^^^^^^^^^ + +.. code-block:: console + + ./testpmd --vdev=eth_mrvl,iface=eth0,iface=eth2,cfg=/home/user/mrvl.conf \ + -c 7 -- -i -a --disable-hw-vlan-strip --rxq=2 + + +Building DPDK +------------- + +Driver needs precompiled MUSDK library during compilation. Please consult +``doc/musdk_get_started.txt`` for the detailed build instructions. + +Before the DPDK build process the environmental variable ``LIBMUSDK_PATH`` with +the path to the MUSDK installation directory needs to be exported. + + +Usage Example +------------- + +MRVL PMD requires extra out of tree kernel modules to function properly. +`musdk_uio` and `mv_pp_uio` sources are part of the MUSDK. Please consult +``doc/musdk_get_started.txt`` for the detailed build instructions. +For `mvpp2x_sysfs` please consult ``Documentation/pp22_sysfs.txt`` for the +detailed build instructions. + +.. code-block:: console + + insmod musdk_uio.ko + insmod mv_pp_uio.ko + insmod mvpp2x_sysfs.ko + +Additionally interfaces used by DPDK application need to be put up: + +.. code-block:: console + + ip link set eth0 up + ip link set eth1 up + +In order to run testpmd example application following command can be used: + +.. code-block:: console + + ./testpmd --vdev=eth_mrvl,iface=eth0,iface=eth2 -c 7 -- \ + --burst=128 --txd=2048 --rxd=1024 --rxq=2 --txq=2 --nb-cores=2 \ + -i -a --disable-hw-vlan-strip --rss-udp diff --git a/dpdk/doc/guides/nics/nfp.rst b/dpdk/doc/guides/nics/nfp.rst index 4ef6e026..99a3b76e 100644 --- a/dpdk/doc/guides/nics/nfp.rst +++ b/dpdk/doc/guides/nics/nfp.rst @@ -1,5 +1,5 @@ .. BSD LICENSE - Copyright(c) 2015 Netronome Systems, Inc. All rights reserved. + Copyright(c) 2015-2017 Netronome Systems, Inc. All rights reserved. All rights reserved. Redistribution and use in source and binary forms, with or without @@ -38,40 +38,43 @@ up to 400 Gbps. This document explains how to use DPDK with the Netronome Poll Mode Driver (PMD) supporting Netronome's Network Flow Processor 6xxx -(NFP-6xxx). +(NFP-6xxx) and Netronome's Flow Processor 4xxx (NFP-4xxx). -Currently the driver supports virtual functions (VFs) only. +NFP is a SRIOV capable device and the PMD driver supports the physical +function (PF) and virtual functions (VFs). Dependencies ------------ -Before using the Netronome's DPDK PMD some NFP-6xxx configuration, +Before using the Netronome's DPDK PMD some NFP configuration, which is not related to DPDK, is required. The system requires -installation of **Netronome's BSP (Board Support Package)** which includes -Linux drivers, programs and libraries. +installation of **Netronome's BSP (Board Support Package)** along +with some specific NFP firmware application. Netronome's NSP ABI +version should be 0.20 or higher. -If you have a NFP-6xxx device you should already have the code and -documentation for doing this configuration. Contact +If you have a NFP device you should already have the code and +documentation for doing all this configuration. Contact **support@netronome.com** to obtain the latest available firmware. -The NFP Linux kernel drivers (including the required PF driver for the -NFP) are available on Github at +The NFP Linux netdev kernel driver for VFs is part of vanilla kernel +since kernel version 4.5, and support for the PF since kernel version +4.11. Support for older kernels can be obtained on Github at **https://github.com/Netronome/nfp-drv-kmods** along with build instructions. -DPDK runs in userspace and PMDs uses the Linux kernel UIO interface to -allow access to physical devices from userspace. The NFP PMD requires -the **igb_uio** UIO driver, available with DPDK, to perform correct -initialization. +NFP PMD needs to be used along with UIO ``igb_uio`` or VFIO (``vfio-pci``) +Linux kernel driver. Building the software --------------------- Netronome's PMD code is provided in the **drivers/net/nfp** directory. -Because Netronome´s BSP dependencies the driver is disabled by default -in DPDK build using **common_linuxapp configuration** file. Enabling the -driver or if you use another configuration file and want to have NFP -support, this variable is needed: +Although NFP PMD has Netronome´s BSP dependencies, it is possible to +compile it along with other DPDK PMDs even if no BSP was installed before. +Of course, a DPDK app will require such a BSP installed for using the +NFP PMD, along with a specific NFP firmware application. + +Default PMD configuration is at **common_linuxapp configuration** file: - **CONFIG_RTE_LIBRTE_NFP_PMD=y** @@ -79,90 +82,59 @@ Once DPDK is built all the DPDK apps and examples include support for the NFP PMD. +Driver compilation and testing +------------------------------ + +Refer to the document :ref:`compiling and testing a PMD for a NIC ` +for details. + +Using the PF +------------ + +NFP PMD has support for using the NFP PF as another DPDK port, but it does not +have any functionality for controlling VFs. In fact, it is not possible to use +the PMD with the VFs if the PF is being used by DPDK, that is, with the NFP PF +bound to ``igb_uio`` or ``vfio-pci`` kernel drivers. Future DPDK version will +have a PMD able to work with the PF and VFs at the same time and with the PF +implementing VF management along with other PF-only functionalities/offloads. + +The PMD PF has extra work to do which will delay the DPDK app initialization +like checking if a firmware is already available in the device, uploading the +firmware if necessary, and configure the Link state properly when starting or +stopping a PF port. Note that firmware upload is not always necessary which is +the main delay for NFP PF PMD initialization. + +Depending on the Netronome product installed in the system, firmware files +should be available under ``/lib/firmware/netronome``. DPDK PMD supporting the +PF requires a specific link, ``/lib/firmware/netronome/nic_dpdk_default.nffw``, +which should be created automatically with Netronome's Agilio products +installation. + +PF multiport support +-------------------- + +Some NFP cards support several physical ports with just one single PCI device. +DPDK core is designed with the 1:1 relationship between PCI devices and DPDK +ports, so NFP PMD PF support requires handling the multiport case specifically. +During NFP PF initialization, the PMD will extract the information about the +number of PF ports from the firmware and will create as many DPDK ports as +needed. + +Because the unusual relationship between a single PCI device and several DPDK +ports, there are some limitations when using more than one PF DPDK ports: there +is no support for RX interrupts and it is not possible either to use those PF +ports with the device hotplug functionality. + + System configuration -------------------- -Using the NFP PMD is not different to using other PMDs. Usual steps are: - -#. **Configure hugepages:** All major Linux distributions have the hugepages - functionality enabled by default. By default this allows the system uses for - working with transparent hugepages. But in this case some hugepages need to - be created/reserved for use with the DPDK through the hugetlbfs file system. - First the virtual file system need to be mounted: - - .. code-block:: console - - mount -t hugetlbfs none /mnt/hugetlbfs - - The command uses the common mount point for this file system and it needs to - be created if necessary. - - Configuring hugepages is performed via sysfs: - - .. code-block:: console - - /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages - - This sysfs file is used to specify the number of hugepages to reserve. - For example: - - .. code-block:: console - - echo 1024 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages - - This will reserve 2GB of memory using 1024 2MB hugepages. The file may be - read to see if the operation was performed correctly: - - .. code-block:: console - - cat /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages - - The number of unused hugepages may also be inspected. - - Before executing the DPDK app it should match the value of nr_hugepages. - - .. code-block:: console - - cat /sys/kernel/mm/hugepages/hugepages-2048kB/free_hugepages - - The hugepages reservation should be performed at system initialization and - it is usual to use a kernel parameter for configuration. If the reservation - is attempted on a busy system it will likely fail. Reserving memory for - hugepages may be done adding the following to the grub kernel command line: - - .. code-block:: console - - default_hugepagesz=1M hugepagesz=2M hugepages=1024 - - This will reserve 2GBytes of memory using 2Mbytes huge pages. - - Finally, for a NUMA system the allocation needs to be made on the correct - NUMA node. In a DPDK app there is a master core which will (usually) perform - memory allocation. It is important that some of the hugepages are reserved - on the NUMA memory node where the network device is attached. This is because - of a restriction in DPDK by which TX and RX descriptors rings must be created - on the master code. - - Per-node allocation of hugepages may be inspected and controlled using sysfs. - For example: - - .. code-block:: console - - cat /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages - - For a NUMA system there will be a specific hugepage directory per node - allowing control of hugepage reservation. A common problem may occur when - hugepages reservation is performed after the system has been working for - some time. Configuration using the global sysfs hugepage interface will - succeed but the per-node allocations may be unsatisfactory. - - The number of hugepages that need to be reserved depends on how the app uses - TX and RX descriptors, and packets mbufs. - -#. **Enable SR-IOV on the NFP-6xxx device:** The current NFP PMD works with - Virtual Functions (VFs) on a NFP device. Make sure that one of the Physical - Function (PF) drivers from the above Github repository is installed and - loaded. +#. **Enable SR-IOV on the NFP device:** The current NFP PMD supports the PF and + the VFs on a NFP device. However, it is not possible to work with both at the + same time because the VFs require the PF being bound to the NFP PF Linux + netdev driver. Make sure you are working with a kernel with NFP PF support or + get the drivers from the above Github repository and follow the instructions + for building and installing it. Virtual Functions need to be enabled before they can be used with the PMD. Before enabling the VFs it is useful to obtain information about the @@ -189,62 +161,3 @@ Using the NFP PMD is not different to using other PMDs. Usual steps are: -k option shows the device driver, if any, that devices are bound to. Depending on the modules loaded at this point the new PCI devices may be bound to nfp_netvf driver. - -#. **To install the uio kernel module (manually):** All major Linux - distributions have support for this kernel module so it is straightforward - to install it: - - .. code-block:: console - - modprobe uio - - The module should now be listed by the lsmod command. - -#. **To install the igb_uio kernel module (manually):** This module is part - of DPDK sources and configured by default (CONFIG_RTE_EAL_IGB_UIO=y). - - .. code-block:: console - - modprobe igb_uio.ko - - The module should now be listed by the lsmod command. - - Depending on which NFP modules are loaded, it could be necessary to - detach NFP devices from the nfp_netvf module. If this is the case the - device needs to be unbound, for example: - - .. code-block:: console - - echo 0000:03:08.0 > /sys/bus/pci/devices/0000:03:08.0/driver/unbind - - lspci -d19ee: -k - - The output of lspci should now show that 0000:03:08.0 is not bound to - any driver. - - The next step is to add the NFP PCI ID to the IGB UIO driver: - - .. code-block:: console - - echo 19ee 6003 > /sys/bus/pci/drivers/igb_uio/new_id - - And then to bind the device to the igb_uio driver: - - .. code-block:: console - - echo 0000:03:08.0 > /sys/bus/pci/drivers/igb_uio/bind - - lspci -d19ee: -k - - lspci should show that device bound to igb_uio driver. - -#. **Using scripts to install and bind modules:** DPDK provides scripts which are - useful for installing the UIO modules and for binding the right device to those - modules avoiding doing so manually: - - * **dpdk-setup.sh** - * **dpdk-devbind.py** - - Configuration may be performed by running dpdk-setup.sh which invokes - dpdk-devbind.py as needed. Executing dpdk-setup.sh will display a menu of - configuration options. diff --git a/dpdk/doc/guides/nics/octeontx.rst b/dpdk/doc/guides/nics/octeontx.rst new file mode 100644 index 00000000..90bb9e5d --- /dev/null +++ b/dpdk/doc/guides/nics/octeontx.rst @@ -0,0 +1,225 @@ +.. BSD LICENSE + Copyright (C) Cavium, Inc. 2017. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Cavium, Inc nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +OCTEONTX Poll Mode driver +========================= + +The OCTEONTX ETHDEV PMD (**librte_pmd_octeontx**) provides poll mode ethdev +driver support for the inbuilt network device found in the **Cavium OCTEONTX** +SoC family as well as their virtual functions (VF) in SR-IOV context. + +More information can be found at `Cavium, Inc Official Website +`_. + +Features +-------- + +Features of the OCTEONTX Ethdev PMD are: + +- Packet type information +- Promiscuous mode +- Port hardware statistics +- Jumbo frames +- Link state information +- SR-IOV VF +- Multiple queues for TX +- Lock-free Tx queue +- HW offloaded `ethdev Rx queue` to `eventdev event queue` packet injection + +Supported OCTEONTX SoCs +----------------------- + +- CN83xx + +Unsupported features +-------------------- + +The features supported by the device and not yet supported by this PMD include: + +- Receive Side Scaling (RSS) +- Scattered and gather for TX and RX +- Ingress classification support +- Egress hierarchical scheduling, traffic shaping, and marking + +Prerequisites +------------- + +See :doc:`../platform/octeontx` for setup information. + +Pre-Installation Configuration +------------------------------ + +Config File Options +~~~~~~~~~~~~~~~~~~~ + +The following options can be modified in the ``config`` file. +Please note that enabling debugging options may affect system performance. + +- ``CONFIG_RTE_LIBRTE_OCTEONTX_PMD`` (default ``y``) + + Toggle compilation of the ``librte_pmd_octeontx`` driver. + +- ``CONFIG_RTE_LIBRTE_OCTEONTX_DEBUG_DRIVER`` (default ``n``) + + Toggle display of generic debugging messages + +- ``CONFIG_RTE_LIBRTE_OCTEONTX_DEBUG_INIT`` (default ``n``) + + Toggle display of initialization related messages. + +- ``CONFIG_RTE_LIBRTE_OCTEONTX_DEBUG_RX`` (default ``n``) + + Toggle display of receive path message + +- ``CONFIG_RTE_LIBRTE_OCTEONTX_DEBUG_TX`` (default ``n``) + + Toggle display of transmit path message + +- ``CONFIG_RTE_LIBRTE_OCTEONTX_DEBUG_MBOX`` (default ``n``) + + Toggle display of mbox related message + + +Driver compilation and testing +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Refer to the document :ref:`compiling and testing a PMD for a NIC ` +for details. + +To compile the OCTEONTX PMD for Linux arm64 gcc target, run the +following ``make`` command: + +.. code-block:: console + + cd + make config T=arm64-thunderx-linuxapp-gcc install + +#. Running testpmd: + + Follow instructions available in the document + :ref:`compiling and testing a PMD for a NIC ` + to run testpmd. + + Example output: + + .. code-block:: console + + ./arm64-thunderx-linuxapp-gcc/app/testpmd -c 700 \ + --base-virtaddr=0x100000000000 \ + --mbuf-pool-ops-name="octeontx_fpavf" \ + --vdev='event_octeontx' \ + --vdev='eth_octeontx,nr_port=2' \ + -- --rxq=1 --txq=1 --nb-core=2 --total-num-mbufs=16384 \ + --disable-hw-vlan-filter -i + ..... + EAL: Detected 24 lcore(s) + EAL: Probing VFIO support... + EAL: VFIO support initialized + ..... + EAL: PCI device 0000:07:00.1 on NUMA socket 0 + EAL: probe driver: 177d:a04b octeontx_ssovf + ..... + EAL: PCI device 0001:02:00.7 on NUMA socket 0 + EAL: probe driver: 177d:a0dd octeontx_pkivf + ..... + EAL: PCI device 0001:03:01.0 on NUMA socket 0 + EAL: probe driver: 177d:a049 octeontx_pkovf + ..... + PMD: octeontx_probe(): created ethdev eth_octeontx for port 0 + PMD: octeontx_probe(): created ethdev eth_octeontx for port 1 + ..... + Configuring Port 0 (socket 0) + Port 0: 00:0F:B7:11:94:46 + Configuring Port 1 (socket 0) + Port 1: 00:0F:B7:11:94:47 + ..... + Checking link statuses... + Port 0 Link Up - speed 40000 Mbps - full-duplex + Port 1 Link Up - speed 40000 Mbps - full-duplex + Done + testpmd> + + +Initialization +-------------- + +The octeontx ethdev pmd is exposed as a vdev device which consists of a set +of PKI and PKO PCIe VF devices. On EAL initialization, +PKI/PKO PCIe VF devices will be probed and then the vdev device can be created +from the application code, or from the EAL command line based on +the number of probed/bound PKI/PKO PCIe VF device to DPDK by + +* Invoking ``rte_vdev_init("eth_octeontx")`` from the application + +* Using ``--vdev="eth_octeontx"`` in the EAL options, which will call + rte_vdev_init() internally + +Device arguments +~~~~~~~~~~~~~~~~ +Each ethdev port is mapped to a physical port(LMAC), Application can specify +the number of interesting ports with ``nr_ports`` argument. + +Dependency +~~~~~~~~~~ +``eth_octeontx`` pmd is depend on ``event_octeontx`` eventdev device and +``octeontx_fpavf`` external mempool handler. + +Example: + +.. code-block:: console + + ./your_dpdk_application --mbuf-pool-ops-name="octeontx_fpavf" \ + --vdev='event_octeontx' \ + --vdev="eth_octeontx,nr_port=2" + +Limitations +----------- + +``octeontx_fpavf`` external mempool handler dependency +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +The OCTEONTX SoC family NIC has inbuilt HW assisted external mempool manager. +This driver will only work with ``octeontx_fpavf`` external mempool handler +as it is the most performance effective way for packet allocation and Tx buffer +recycling on OCTEONTX SoC platform. + +CRC striping +~~~~~~~~~~~~ + +The OCTEONTX SoC family NICs strip the CRC for every packets coming into the +host interface. So, CRC will be stripped even when the +``rxmode.hw_strip_crc`` member is set to 0 in ``struct rte_eth_conf``. + +Maximum packet length +~~~~~~~~~~~~~~~~~~~~~ + +The OCTEONTX SoC family NICs support a maximum of a 32K jumbo frame. The value +is fixed and cannot be changed. So, even when the ``rxmode.max_rx_pkt_len`` +member of ``struct rte_eth_conf`` is set to a value lower than 32k, frames +up to 32k bytes can still reach the host interface. diff --git a/dpdk/doc/guides/nics/overview.rst b/dpdk/doc/guides/nics/overview.rst index 6abbae69..0df0ef81 100644 --- a/dpdk/doc/guides/nics/overview.rst +++ b/dpdk/doc/guides/nics/overview.rst @@ -48,105 +48,11 @@ There are more differences between drivers regarding some internal properties, portability or even documentation availability. Most of these differences are summarized below. +More details about features can be found in :doc:`features`. + .. _table_net_pmd_features: -.. raw:: html - - - -.. table:: Features availability in networking drivers - - ==================== = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = - Feature a b b b b c e e e i i i i i i i i i i f f f f m m m n n p q q r s t v v v v x - f n n n o x 1 n n 4 4 4 4 g g x x x x m m m m l l p f u c e e i z h h i i m e - p x x x n g 0 a i 0 0 0 0 b b g g g g 1 1 1 1 x x i p l a d d n e u o r r x n - a 2 2 t d b 0 c e e e e v b b b b 0 0 0 0 4 5 p l p e e g d n s t t n v - c x x i e 0 . v v f e e e e k k k k e v a d t i i e i - k v n . f f . v v . v v f t e o o t r - e f g . . . f f . f f a r . 3 t - t v v v v v v 2 x v - e e e e e e e - c c c c c c c - ==================== = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = - Speed capabilities - Link status Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y - Link status event Y Y Y Y Y Y Y Y Y Y Y Y Y Y - Queue status event Y - Rx interrupt Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y - Queue start/stop Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y - MTU update Y Y Y P Y Y Y Y Y Y Y Y Y Y - Jumbo frame Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y - Scattered Rx Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y - LRO Y Y Y Y - TSO Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y - Promiscuous mode Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y - Allmulticast mode Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y - Unicast MAC filter Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y - Multicast MAC filter Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y - RSS hash Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y - RSS key update Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y - RSS reta update Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y - VMDq Y Y Y Y Y Y Y - SR-IOV Y Y Y Y Y Y Y Y Y Y Y - DCB Y Y Y Y Y - VLAN filter Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y - Ethertype filter Y Y Y Y Y - N-tuple filter Y Y Y - SYN filter Y Y Y - Tunnel filter Y Y Y Y - Flexible filter Y - Hash filter Y Y Y Y - Flow director Y Y Y Y Y - Flow control Y Y Y Y Y Y Y Y Y - Rate limitation Y Y - Traffic mirroring Y Y Y Y - CRC offload Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y - VLAN offload Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y P - QinQ offload Y Y Y Y Y Y Y - L3 checksum offload Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y - L4 checksum offload Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y - Inner L3 checksum Y Y Y Y Y Y - Inner L4 checksum Y Y Y Y Y Y - Packet type parsing Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y - Timesync Y Y Y Y Y - Basic stats Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y - Extended stats Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y - Stats per queue Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y - EEPROM dump Y Y Y Y - Registers dump Y Y Y Y Y Y Y Y - Multiprocess aware Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y - BSD nic_uio Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y - Linux UIO Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y - Linux VFIO Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y - Other kdrv Y Y Y - ARMv7 Y Y Y - ARMv8 Y Y Y Y Y Y Y Y - Power8 Y Y Y - TILE-Gx Y - x86-32 Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y - x86-64 Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y - Usage doc Y Y Y Y Y Y Y Y Y Y Y Y - Design doc - Perf doc - ==================== = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = +.. include:: overview_table.txt .. Note:: diff --git a/dpdk/doc/guides/nics/pcap_ring.rst b/dpdk/doc/guides/nics/pcap_ring.rst index aa48d339..5e4f5f60 100644 --- a/dpdk/doc/guides/nics/pcap_ring.rst +++ b/dpdk/doc/guides/nics/pcap_ring.rst @@ -62,14 +62,16 @@ Libpcap-based PMD ~~~~~~~~~~~~~~~~~ Pcap-based devices can be created using the virtual device --vdev option. -The device name must start with the eth_pcap prefix followed by numbers or letters. +The device name must start with the net_pcap prefix followed by numbers or letters. The name is unique for each device. Each device can have multiple stream options and multiple devices can be used. Multiple device definitions can be arranged using multiple --vdev. Device name and stream options must be separated by commas as shown below: .. code-block:: console - $RTE_TARGET/app/testpmd -c f -n 4 --vdev 'eth_pcap0,stream_opt0=..,stream_opt1=..' --vdev='eth_pcap1,stream_opt0=..' + $RTE_TARGET/app/testpmd -l 0-3 -n 4 \ + --vdev 'net_pcap0,stream_opt0=..,stream_opt1=..' \ + --vdev='net_pcap1,stream_opt0=..' Device Streams ^^^^^^^^^^^^^^ @@ -120,25 +122,32 @@ Read packets from one pcap file and write them to another: .. code-block:: console - $RTE_TARGET/app/testpmd -c '0xf' -n 4 --vdev 'eth_pcap0,rx_pcap=/path/to/ file_rx.pcap,tx_pcap=/path/to/file_tx.pcap' -- --port-topology=chained + $RTE_TARGET/app/testpmd -l 0-3 -n 4 \ + --vdev 'net_pcap0,rx_pcap=file_rx.pcap,tx_pcap=file_tx.pcap' \ + -- --port-topology=chained Read packets from a network interface and write them to a pcap file: .. code-block:: console - $RTE_TARGET/app/testpmd -c '0xf' -n 4 --vdev 'eth_pcap0,rx_iface=eth0,tx_pcap=/path/to/file_tx.pcap' -- --port-topology=chained + $RTE_TARGET/app/testpmd -l 0-3 -n 4 \ + --vdev 'net_pcap0,rx_iface=eth0,tx_pcap=file_tx.pcap' \ + -- --port-topology=chained Read packets from a pcap file and write them to a network interface: .. code-block:: console - $RTE_TARGET/app/testpmd -c '0xf' -n 4 --vdev 'eth_pcap0,rx_pcap=/path/to/ file_rx.pcap,tx_iface=eth1' -- --port-topology=chained + $RTE_TARGET/app/testpmd -l 0-3 -n 4 \ + --vdev 'net_pcap0,rx_pcap=file_rx.pcap,tx_iface=eth1' \ + -- --port-topology=chained Forward packets through two network interfaces: .. code-block:: console - $RTE_TARGET/app/testpmd -c '0xf' -n 4 --vdev 'eth_pcap0,iface=eth0' --vdev='eth_pcap1;iface=eth1' + $RTE_TARGET/app/testpmd -l 0-3 -n 4 \ + --vdev 'net_pcap0,iface=eth0' --vdev='net_pcap1;iface=eth1' Using libpcap-based PMD with the testpmd Application ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -162,19 +171,21 @@ Otherwise, the first 512 packets from the input pcap file will be discarded by t .. code-block:: console - $RTE_TARGET/app/testpmd -c '0xf' -n 4 --vdev 'eth_pcap0,rx_pcap=/path/to/ file_rx.pcap,tx_pcap=/path/to/file_tx.pcap' -- --port-topology=chained --no-flush-rx + $RTE_TARGET/app/testpmd -l 0-3 -n 4 \ + --vdev 'net_pcap0,rx_pcap=file_rx.pcap,tx_pcap=file_tx.pcap' \ + -- --port-topology=chained --no-flush-rx Rings-based PMD ~~~~~~~~~~~~~~~ To run a DPDK application on a machine without any Ethernet devices, a pair of ring-based rte_ethdevs can be used as below. -The device names passed to the --vdev option must start with eth_ring and take no additional parameters. +The device names passed to the --vdev option must start with net_ring and take no additional parameters. Multiple devices may be specified, separated by commas. .. code-block:: console - ./testpmd -c E -n 4 --vdev=eth_ring0 --vdev=eth_ring1 -- -i + ./testpmd -l 1-3 -n 4 --vdev=net_ring0 --vdev=net_ring1 -- -i EAL: Detected lcore 1 as core 1 on socket 0 ... @@ -243,8 +254,8 @@ for reception on the same port (error handling omitted for clarity): /* create two ethdev's */ - port0 = rte_eth_from_rings("eth_ring0", ring, NUM_RINGS, ring, NUM_RINGS, SOCKET0); - port1 = rte_eth_from_rings("eth_ring1", ring, NUM_RINGS, ring, NUM_RINGS, SOCKET0); + port0 = rte_eth_from_rings("net_ring0", ring, NUM_RINGS, ring, NUM_RINGS, SOCKET0); + port1 = rte_eth_from_rings("net_ring1", ring, NUM_RINGS, ring, NUM_RINGS, SOCKET0); To create two pseudo-Ethernet ports where the traffic is switched between them, @@ -253,8 +264,8 @@ the final two lines could be changed as below: .. code-block:: c - port0 = rte_eth_from_rings("eth_ring0", &ring[0], 1, &ring[1], 1, SOCKET0); - port1 = rte_eth_from_rings("eth_ring1", &ring[1], 1, &ring[0], 1, SOCKET0); + port0 = rte_eth_from_rings("net_ring0", &ring[0], 1, &ring[1], 1, SOCKET0); + port1 = rte_eth_from_rings("net_ring1", &ring[1], 1, &ring[0], 1, SOCKET0); This type of configuration could be useful in a pipeline model, for example, where one may want to have inter-core communication using pseudo Ethernet devices rather than raw rings, diff --git a/dpdk/doc/guides/nics/qede.rst b/dpdk/doc/guides/nics/qede.rst index 53d749c9..84becc98 100644 --- a/dpdk/doc/guides/nics/qede.rst +++ b/dpdk/doc/guides/nics/qede.rst @@ -1,5 +1,6 @@ .. BSD LICENSE Copyright (c) 2016 QLogic Corporation + Copyright (c) 2017 Cavium Inc. All rights reserved. Redistribution and use in source and binary forms, with or without @@ -32,8 +33,7 @@ QEDE Poll Mode Driver ====================== The QEDE poll mode driver library (**librte_pmd_qede**) implements support -for **QLogic FastLinQ QL4xxxx 25G/40G CNA** family of adapters as well -as their virtual functions (VF) in SR-IOV context. It is supported on +for **QLogic FastLinQ QL4xxxx 10G/25G/40G/50G/100G Intelligent Ethernet Adapters (IEA) and Converged Network Adapters (CNA)** family of adapters as well as SR-IOV virtual functions (VF). It is supported on several standard Linux distros like RHEL7.x, SLES12.x and Ubuntu. It is compile-tested under FreeBSD OS. @@ -47,44 +47,57 @@ Supported Features - Promiscuous mode - Allmulti mode - Port hardware statistics -- Jumbo frames (using single buffer) -- VLAN offload - Filtering and stripping -- Stateless checksum offloads (IPv4/TCP/UDP) -- Multiple Rx/Tx queues (queue-pairs) -- RSS (with user configurable table/key) -- TSS +- Jumbo frames - Multiple MAC address +- MTU change - Default pause flow control -- SR-IOV VF for 25G/40G modes +- Multiprocess aware +- Scatter-Gather +- Multiple Rx/Tx queues +- RSS (with RETA/hash table/key) +- TSS +- Stateless checksum offloads (IPv4/IPv6/TCP/UDP) +- LRO/TSO +- VLAN offload - Filtering and stripping +- N-tuple filter and flow director (limited support) +- NPAR (NIC Partitioning) +- SR-IOV VF +- VXLAN tunneling offload +- MPLSoUDP Tx tunnel offload Non-supported Features ---------------------- -- Scatter-Gather Rx/Tx frames -- Unequal number of Rx/Tx queues -- MTU change (dynamic) - SR-IOV PF -- Tunneling offloads -- Reload of the PMD after a non-graceful termination +- GENEVE and NVGRE Tunneling offloads Supported QLogic Adapters ------------------------- -- QLogic FastLinQ QL4xxxx 25G/40G/100G CNAs. +- QLogic FastLinQ QL4xxxx 10G/25G/40G/50G/100G Intelligent Ethernet Adapters (IEA) and Converged Network Adapters (CNA) Prerequisites ------------- -- Requires firmware version **8.7.x.** and management firmware - version **8.7.x or higher**. Firmware may be available +- Requires storm firmware version **8.30.12.0**. Firmware may be available inbox in certain newer Linux distros under the standard directory - ``E.g. /lib/firmware/qed/qed_init_values_zipped-8.7.7.0.bin`` + ``E.g. /lib/firmware/qed/qed_init_values-8.30.12.0.bin`` + If the required firmware files are not available then download it from + `QLogic Driver Download Center `_. + For downloading firmware file, select adapter category, model and DPDK Poll Mode Driver. -- If the required firmware files are not available then visit - `QLogic Driver Download Center `_. +- Requires management firmware (MFW) version **8.30.x.x** or higher to be + flashed on to the adapter. If the required management firmware is not + available then download from + `QLogic Driver Download Center `_. + For downloading firmware upgrade utility, select adapter category, model and Linux distro. + To flash the management firmware refer to the instructions in the QLogic Firmware Upgrade Utility Readme document. + +- SR-IOV requires Linux PF driver version **8.20.x.x** or higher. + If the required PF driver is not available then download it from + `QLogic Driver Download Center `_. + For downloading PF driver, select adapter category, model and Linux distro. -- This driver relies on external zlib library (-lz) for uncompressing - the firmware file. Performance note ~~~~~~~~~~~~~~~~ @@ -117,133 +130,31 @@ enabling debugging options may affect system performance. Toggle display of receive fast path run-time messages. +- ``CONFIG_RTE_LIBRTE_QEDE_VF_TX_SWITCH`` (default **"y"**) + + A knob to control per-VF Tx switching feature. + - ``CONFIG_RTE_LIBRTE_QEDE_FW`` (default **""**) Gives absolute path of firmware file. - ``Eg: "/lib/firmware/qed/qed_init_values_zipped-8.7.7.0.bin"`` + ``Eg: "/lib/firmware/qed/qed_init_values-8.30.12.0.bin"`` Empty string indicates driver will pick up the firmware file - from the default location. + from the default location /lib/firmware/qed. + CAUTION this option is more for custom firmware, it is not + recommended for use under normal condition. -Driver Compilation -~~~~~~~~~~~~~~~~~~ - -To compile QEDE PMD for Linux x86_64 gcc target, run the following ``make`` -command:: - - cd - make config T=x86_64-native-linuxapp-gcc install - -To compile QEDE PMD for Linux x86_64 clang target, run the following ``make`` -command:: - - cd - make config T=x86_64-native-linuxapp-clang install - -To compile QEDE PMD for FreeBSD x86_64 clang target, run the following ``gmake`` -command:: - - cd - gmake config T=x86_64-native-bsdapp-clang install - -To compile QEDE PMD for FreeBSD x86_64 gcc target, run the following ``gmake`` -command:: - - cd - gmake config T=x86_64-native-bsdapp-gcc install -Wl,-rpath=\ - /usr/local/lib/gcc48 CC=gcc48 - - -Sample Application Notes -~~~~~~~~~~~~~~~~~~~~~~~~ - -This section demonstrates how to launch ``testpmd`` with QLogic 4xxxx -devices managed by ``librte_pmd_qede`` in Linux operating system. - -#. Request huge pages: - - .. code-block:: console - - echo 1024 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages/ \ - nr_hugepages - -#. Load ``igb_uio`` driver: - - .. code-block:: console - - insmod ./x86_64-native-linuxapp-gcc/kmod/igb_uio.ko - -#. Bind the QLogic 4xxxx adapters to ``igb_uio`` loaded in the - previous step: - - .. code-block:: console - - ./tools/dpdk-devbind.py --bind igb_uio 0000:84:00.0 0000:84:00.1 \ - 0000:84:00.2 0000:84:00.3 - -#. Start ``testpmd`` with basic parameters: - (Enable QEDE_DEBUG_INFO=y to view informational messages) - - .. code-block:: console - - testpmd -c 0xff1 -n 4 -- -i --nb-cores=8 --portmask=0xf --rxd=4096 \ - --txd=4096 --txfreet=4068 --enable-rx-cksum --rxq=4 --txq=4 \ - --rss-ip --rss-udp - - [...] - - EAL: PCI device 0000:84:00.0 on NUMA socket 1 - EAL: probe driver: 1077:1634 rte_qede_pmd - EAL: Not managed by a supported kernel driver, skipped - EAL: PCI device 0000:84:00.1 on NUMA socket 1 - EAL: probe driver: 1077:1634 rte_qede_pmd - EAL: Not managed by a supported kernel driver, skipped - EAL: PCI device 0000:88:00.0 on NUMA socket 1 - EAL: probe driver: 1077:1656 rte_qede_pmd - EAL: PCI memory mapped at 0x7f738b200000 - EAL: PCI memory mapped at 0x7f738b280000 - EAL: PCI memory mapped at 0x7f738b300000 - PMD: Chip details : BB1 - PMD: Driver version : QEDE PMD 8.7.9.0_1.0.0 - PMD: Firmware version : 8.7.7.0 - PMD: Management firmware version : 8.7.8.0 - PMD: Firmware file : /lib/firmware/qed/qed_init_values_zipped-8.7.7.0.bin - [QEDE PMD: (84:00.0:dpdk-port-0)]qede_common_dev_init:macaddr \ - 00:0e:1e:d2:09:9c - [...] - [QEDE PMD: (84:00.0:dpdk-port-0)]qede_tx_queue_setup:txq 0 num_desc 4096 \ - tx_free_thresh 4068 socket 0 - [QEDE PMD: (84:00.0:dpdk-port-0)]qede_tx_queue_setup:txq 1 num_desc 4096 \ - tx_free_thresh 4068 socket 0 - [QEDE PMD: (84:00.0:dpdk-port-0)]qede_tx_queue_setup:txq 2 num_desc 4096 \ - tx_free_thresh 4068 socket 0 - [QEDE PMD: (84:00.0:dpdk-port-0)]qede_tx_queue_setup:txq 3 num_desc 4096 \ - tx_free_thresh 4068 socket 0 - [QEDE PMD: (84:00.0:dpdk-port-0)]qede_rx_queue_setup:rxq 0 num_desc 4096 \ - rx_buf_size=2148 socket 0 - [QEDE PMD: (84:00.0:dpdk-port-0)]qede_rx_queue_setup:rxq 1 num_desc 4096 \ - rx_buf_size=2148 socket 0 - [QEDE PMD: (84:00.0:dpdk-port-0)]qede_rx_queue_setup:rxq 2 num_desc 4096 \ - rx_buf_size=2148 socket 0 - [QEDE PMD: (84:00.0:dpdk-port-0)]qede_rx_queue_setup:rxq 3 num_desc 4096 \ - rx_buf_size=2148 socket 0 - [QEDE PMD: (84:00.0:dpdk-port-0)]qede_dev_start:port 0 - [QEDE PMD: (84:00.0:dpdk-port-0)]qede_dev_start:link status: down - [...] - Checking link statuses... - Port 0 Link Up - speed 25000 Mbps - full-duplex - Port 1 Link Up - speed 25000 Mbps - full-duplex - Port 2 Link Up - speed 25000 Mbps - full-duplex - Port 3 Link Up - speed 25000 Mbps - full-duplex - Done - testpmd> +Driver compilation and testing +------------------------------ +Refer to the document :ref:`compiling and testing a PMD for a NIC ` +for details. SR-IOV: Prerequisites and Sample Application Notes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +-------------------------------------------------- This section provides instructions to configure SR-IOV with Linux OS. -**Note**: librte_pmd_qede will be used to bind to SR-IOV VF device and Linux native kernel driver (QEDE) will function as SR-IOV PF driver. +**Note**: librte_pmd_qede will be used to bind to SR-IOV VF device and Linux native kernel driver (qede) will function as SR-IOV PF driver. Requires PF driver to be 8.10.x.x or higher. #. Verify SR-IOV and ARI capability is enabled on the adapter using ``lspci``: @@ -311,4 +222,68 @@ This section provides instructions to configure SR-IOV with Linux OS. The VF devices may be passed through to the guest VM using ``virt-manager`` or ``virsh``. QEDE PMD should be used to bind the VF devices in the guest VM - using the instructions outlined in the Application notes above. + using the instructions from Driver compilation and testing section above. + + +#. Running testpmd + (Enable QEDE_DEBUG_INFO=y to view informational messages): + + Refer to the document + :ref:`compiling and testing a PMD for a NIC ` to run + ``testpmd`` application. + + Example output: + + .. code-block:: console + + testpmd -l 0,4-11 -n 4 -- -i --nb-cores=8 --portmask=0xf --rxd=4096 \ + --txd=4096 --txfreet=4068 --enable-rx-cksum --rxq=4 --txq=4 \ + --rss-ip --rss-udp + + [...] + + EAL: PCI device 0000:84:00.0 on NUMA socket 1 + EAL: probe driver: 1077:1634 rte_qede_pmd + EAL: Not managed by a supported kernel driver, skipped + EAL: PCI device 0000:84:00.1 on NUMA socket 1 + EAL: probe driver: 1077:1634 rte_qede_pmd + EAL: Not managed by a supported kernel driver, skipped + EAL: PCI device 0000:88:00.0 on NUMA socket 1 + EAL: probe driver: 1077:1656 rte_qede_pmd + EAL: PCI memory mapped at 0x7f738b200000 + EAL: PCI memory mapped at 0x7f738b280000 + EAL: PCI memory mapped at 0x7f738b300000 + PMD: Chip details : BB1 + PMD: Driver version : QEDE PMD 8.7.9.0_1.0.0 + PMD: Firmware version : 8.7.7.0 + PMD: Management firmware version : 8.7.8.0 + PMD: Firmware file : /lib/firmware/qed/qed_init_values_zipped-8.7.7.0.bin + [QEDE PMD: (84:00.0:dpdk-port-0)]qede_common_dev_init:macaddr \ + 00:0e:1e:d2:09:9c + [...] + [QEDE PMD: (84:00.0:dpdk-port-0)]qede_tx_queue_setup:txq 0 num_desc 4096 \ + tx_free_thresh 4068 socket 0 + [QEDE PMD: (84:00.0:dpdk-port-0)]qede_tx_queue_setup:txq 1 num_desc 4096 \ + tx_free_thresh 4068 socket 0 + [QEDE PMD: (84:00.0:dpdk-port-0)]qede_tx_queue_setup:txq 2 num_desc 4096 \ + tx_free_thresh 4068 socket 0 + [QEDE PMD: (84:00.0:dpdk-port-0)]qede_tx_queue_setup:txq 3 num_desc 4096 \ + tx_free_thresh 4068 socket 0 + [QEDE PMD: (84:00.0:dpdk-port-0)]qede_rx_queue_setup:rxq 0 num_desc 4096 \ + rx_buf_size=2148 socket 0 + [QEDE PMD: (84:00.0:dpdk-port-0)]qede_rx_queue_setup:rxq 1 num_desc 4096 \ + rx_buf_size=2148 socket 0 + [QEDE PMD: (84:00.0:dpdk-port-0)]qede_rx_queue_setup:rxq 2 num_desc 4096 \ + rx_buf_size=2148 socket 0 + [QEDE PMD: (84:00.0:dpdk-port-0)]qede_rx_queue_setup:rxq 3 num_desc 4096 \ + rx_buf_size=2148 socket 0 + [QEDE PMD: (84:00.0:dpdk-port-0)]qede_dev_start:port 0 + [QEDE PMD: (84:00.0:dpdk-port-0)]qede_dev_start:link status: down + [...] + Checking link statuses... + Port 0 Link Up - speed 25000 Mbps - full-duplex + Port 1 Link Up - speed 25000 Mbps - full-duplex + Port 2 Link Up - speed 25000 Mbps - full-duplex + Port 3 Link Up - speed 25000 Mbps - full-duplex + Done + testpmd> diff --git a/dpdk/doc/guides/nics/sfc_efx.rst b/dpdk/doc/guides/nics/sfc_efx.rst new file mode 100644 index 00000000..ae2b54a2 --- /dev/null +++ b/dpdk/doc/guides/nics/sfc_efx.rst @@ -0,0 +1,283 @@ +.. BSD LICENSE + Copyright (c) 2016 Solarflare Communications Inc. + All rights reserved. + + This software was jointly developed between OKTET Labs (under contract + for Solarflare) and Solarflare Communications, Inc. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Solarflare libefx-based Poll Mode Driver +======================================== + +The SFC EFX PMD (**librte_pmd_sfc_efx**) provides poll mode driver support +for **Solarflare SFN7xxx and SFN8xxx** family of 10/40 Gbps adapters. +SFC EFX PMD has support for the latest Linux and FreeBSD operating systems. + +More information can be found at `Solarflare Communications website +`_. + + +Features +-------- + +SFC EFX PMD has support for: + +- Multiple transmit and receive queues + +- Link state information including link status change interrupt + +- IPv4/IPv6 TCP/UDP transmit checksum offload + +- Port hardware statistics + +- Extended statistics (see Solarflare Server Adapter User's Guide for + the statistics description) + +- Basic flow control + +- MTU update + +- Jumbo frames up to 9K + +- Promiscuous mode + +- Allmulticast mode + +- TCP segmentation offload (TSO) + +- Multicast MAC filter + +- IPv4/IPv6 TCP/UDP receive checksum offload + +- Received packet type information + +- Receive side scaling (RSS) + +- RSS hash + +- Scattered Rx DMA for packet that are larger that a single Rx descriptor + +- Deferred receive and transmit queue start + +- Transmit VLAN insertion (if running firmware variant supports it) + +- Flow API + + +Non-supported Features +---------------------- + +The features not yet supported include: + +- Receive queue interupts + +- Priority-based flow control + +- Loopback + +- Configurable RX CRC stripping (always stripped) + +- Header split on receive + +- VLAN filtering + +- VLAN stripping + +- LRO + + +Limitations +----------- + +Due to requirements on receive buffer alignment and usage of the receive +buffer for the auxiliary packet information provided by the NIC up to +extra 269 (14 bytes prefix plus up to 255 bytes for end padding) bytes may be +required in the receive buffer. +It should be taken into account when mbuf pool for receive is created. + + +Flow API support +---------------- + +Supported attributes: + +- Ingress + +Supported pattern items: + +- VOID + +- ETH (exact match of source/destination addresses, individual/group match + of destination address, EtherType) + +- VLAN (exact match of VID, double-tagging is supported) + +- IPV4 (exact match of source/destination addresses, + IP transport protocol) + +- IPV6 (exact match of source/destination addresses, + IP transport protocol) + +- TCP (exact match of source/destination ports) + +- UDP (exact match of source/destination ports) + +Supported actions: + +- VOID + +- QUEUE + +- RSS + +Validating flow rules depends on the firmware variant. + +Ethernet destinaton individual/group match +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Ethernet item supports I/G matching, if only the corresponding bit is set +in the mask of destination address. If destinaton address in the spec is +multicast, it matches all multicast (and broadcast) packets, oherwise it +matches unicast packets that are not filtered by other flow rules. + + +Supported NICs +-------------- + +- Solarflare Flareon [Ultra] Server Adapters: + + - Solarflare SFN8522 Dual Port SFP+ Server Adapter + + - Solarflare SFN8542 Dual Port QSFP+ Server Adapter + + - Solarflare SFN7002F Dual Port SFP+ Server Adapter + + - Solarflare SFN7004F Quad Port SFP+ Server Adapter + + - Solarflare SFN7042Q Dual Port QSFP+ Server Adapter + + - Solarflare SFN7122F Dual Port SFP+ Server Adapter + + - Solarflare SFN7124F Quad Port SFP+ Server Adapter + + - Solarflare SFN7142Q Dual Port QSFP+ Server Adapter + + - Solarflare SFN7322F Precision Time Synchronization Server Adapter + + +Prerequisites +------------- + +- Requires firmware version: + + - SFN7xxx: **4.7.1.1001** or higher + + - SFN8xxx: **6.0.2.1004** or higher + +Visit `Solarflare Support Downloads `_ to get +Solarflare Utilities (either Linux or FreeBSD) with the latest firmware. +Follow instructions from Solarflare Server Adapter User's Guide to +update firmware and configure the adapter. + + +Pre-Installation Configuration +------------------------------ + + +Config File Options +~~~~~~~~~~~~~~~~~~~ + +The following options can be modified in the ``.config`` file. +Please note that enabling debugging options may affect system performance. + +- ``CONFIG_RTE_LIBRTE_SFC_EFX_PMD`` (default **y**) + + Enable compilation of Solarflare libefx-based poll-mode driver. + +- ``CONFIG_RTE_LIBRTE_SFC_EFX_DEBUG`` (default **n**) + + Enable compilation of the extra run-time consistency checks. + + +Per-Device Parameters +~~~~~~~~~~~~~~~~~~~~~ + +The following per-device parameters can be passed via EAL PCI device +whitelist option like "-w 02:00.0,arg1=value1,...". + +Case-insensitive 1/y/yes/on or 0/n/no/off may be used to specify +boolean parameters value. + +- ``rx_datapath`` [auto|efx|ef10] (default **auto**) + + Choose receive datapath implementation. + **auto** allows the driver itself to make a choice based on firmware + features available and required by the datapath implementation. + **efx** chooses libefx-based datapath which supports Rx scatter. + **ef10** chooses EF10 (SFN7xxx, SFN8xxx) native datapath which is + more efficient than libefx-based and provides richer packet type + classification, but lacks Rx scatter support. + +- ``tx_datapath`` [auto|efx|ef10|ef10_simple] (default **auto**) + + Choose transmit datapath implementation. + **auto** allows the driver itself to make a choice based on firmware + features available and required by the datapath implementation. + **efx** chooses libefx-based datapath which supports VLAN insertion + (full-feature firmware variant only), TSO and multi-segment mbufs. + Mbuf segments may come from different mempools, and mbuf reference + counters are treated responsibly. + **ef10** chooses EF10 (SFN7xxx, SFN8xxx) native datapath which is + more efficient than libefx-based but has no VLAN insertion and TSO + support yet. + Mbuf segments may come from different mempools, and mbuf reference + counters are treated responsibly. + **ef10_simple** chooses EF10 (SFN7xxx, SFN8xxx) native datapath which + is even more faster then **ef10** but does not support multi-segment + mbufs, disallows multiple mempools and neglects mbuf reference counters. + +- ``perf_profile`` [auto|throughput|low-latency] (default **throughput**) + + Choose hardware tunning to be optimized for either throughput or + low-latency. + **auto** allows NIC firmware to make a choice based on + installed licences and firmware variant configured using **sfboot**. + +- ``debug_init`` [bool] (default **n**) + + Enable extra logging during device initialization and startup. + +- ``mcdi_logging`` [bool] (default **n**) + + Enable extra logging of the communication with the NIC's management CPU. + The logging is done using RTE_LOG() with INFO level and PMD type. + The format is consumed by the Solarflare netlogdecode cross-platform tool. + +- ``stats_update_period_ms`` [long] (default **1000**) + + Adjust period in milliseconds to update port hardware statistics. + The accepted range is 0 to 65535. The value of **0** may be used + to disable periodic statistics update. One should note that it's + only possible to set an arbitrary value on SFN8xxx provided that + firmware version is 6.2.1.1033 or higher, otherwise any positive + value will select a fixed update period of **1000** milliseconds diff --git a/dpdk/doc/guides/nics/szedata2.rst b/dpdk/doc/guides/nics/szedata2.rst index 741b4008..1a5d4138 100644 --- a/dpdk/doc/guides/nics/szedata2.rst +++ b/dpdk/doc/guides/nics/szedata2.rst @@ -31,16 +31,16 @@ SZEDATA2 poll mode driver library ================================= -The SZEDATA2 poll mode driver library implements support for cards from COMBO -family (**COMBO-80G**, **COMBO-100G**). -The SZEDATA2 PMD uses interface provided by libsze2 library to communicate -with COMBO cards over sze2 layer. +The SZEDATA2 poll mode driver library implements support for the Netcope +FPGA Boards (**NFB-***), FPGA-based programmable NICs. +The SZEDATA2 PMD uses interface provided by the libsze2 library to communicate +with the NFB cards over the sze2 layer. -More information about family of -`COMBO cards `_ +More information about the +`NFB cards `_ and used technology -(`NetCOPE platform `_) can be -found on the `Liberouter website `_. +(`Netcope Development Kit `_) +can be found on the `Netcope Technologies website `_. .. note:: @@ -77,7 +77,7 @@ separately: sharing of resources for user space applications. Information about getting the dependencies can be found `here -`_. +`_. Configuration ------------- @@ -91,14 +91,34 @@ These configuration options can be modified before compilation in the * ``CONFIG_RTE_LIBRTE_PMD_SZEDATA2_AS`` default value: **0** - This option defines type of firmware address space. - Currently supported value is: + This option defines type of firmware address space and must be set + according to the used card and mode. + Currently supported values are: - * **0** for firmwares: + * **0** - for cards (modes): - * NIC_100G1_LR4 - * HANIC_100G1_LR4 - * HANIC_100G1_SR10 + * NFB-100G1 (100G1) + + * **1** - for cards (modes): + + * NFB-100G2Q (100G1) + + * **2** - for cards (modes): + + * NFB-40G2 (40G2) + * NFB-100G2C (100G2) + * NFB-100G2Q (40G2) + + * **3** - for cards (modes): + + * NFB-40G2 (10G8) + * NFB-100G2Q (10G8) + + * **4** - for cards (modes): + + * NFB-100G1 (10G10) + + * **5** - for experimental firmwares and future use Using the SZEDATA2 PMD ---------------------- @@ -117,7 +137,7 @@ transmit channel: .. code-block:: console - $RTE_TARGET/app/testpmd -c 0xf -n 2 \ + $RTE_TARGET/app/testpmd -l 0-3 -n 2 \ -- --port-topology=chained --rxq=2 --txq=2 --nb-cores=2 -i -a Example output: diff --git a/dpdk/doc/guides/nics/tap.rst b/dpdk/doc/guides/nics/tap.rst new file mode 100644 index 00000000..04086b11 --- /dev/null +++ b/dpdk/doc/guides/nics/tap.rst @@ -0,0 +1,215 @@ +.. BSD LICENSE + Copyright(c) 2016 Intel Corporation. All rights reserved. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Tun/Tap Poll Mode Driver +======================== + +The ``rte_eth_tap.c`` PMD creates a device using TUN/TAP interfaces on the +local host. The PMD allows for DPDK and the host to communicate using a raw +device interface on the host and in the DPDK application. + +The device created is a TAP device, which sends/receives packet in a raw +format with a L2 header. The usage for a TAP PMD is for connectivity to the +local host using a TAP interface. When the TAP PMD is initialized it will +create a number of tap devices in the host accessed via ``ifconfig -a`` or +``ip`` command. The commands can be used to assign and query the virtual like +device. + +These TAP interfaces can be used with Wireshark or tcpdump or Pktgen-DPDK +along with being able to be used as a network connection to the DPDK +application. The method enable one or more interfaces is to use the +``--vdev=net_tap0`` option on the DPDK application command line. Each +``--vdev=net_tap1`` option given will create an interface named dtap0, dtap1, +and so on. + +The interface name can be changed by adding the ``iface=foo0``, for example:: + + --vdev=net_tap0,iface=foo0 --vdev=net_tap1,iface=foo1, ... + +Also the speed of the interface can be changed from 10G to whatever number +needed, but the interface does not enforce that speed, for example:: + + --vdev=net_tap0,iface=foo0,speed=25000 + +Normally the PMD will generate a random MAC address, but when testing or with +a static configuration the developer may need a fixed MAC address style. +Using the option ``mac=fixed`` you can create a fixed known MAC address:: + + --vdev=net_tap0,mac=fixed + +The MAC address will have a fixed value with the last octet incrementing by one +for each interface string containing ``mac=fixed``. The MAC address is formatted +as 00:'d':'t':'a':'p':[00-FF]. Convert the characters to hex and you get the +actual MAC address: ``00:64:74:61:70:[00-FF]``. + +It is possible to specify a remote netdevice to capture packets from by adding +``remote=foo1``, for example:: + + --vdev=net_tap,iface=tap0,remote=foo1 + +If a ``remote`` is set, the tap MAC address will be set to match the remote one +just after netdevice creation. Using TC rules, traffic from the remote netdevice +will be redirected to the tap. If the tap is in promiscuous mode, then all +packets will be redirected. In allmulti mode, all multicast packets will be +redirected. + +Using the remote feature is especially useful for capturing traffic from a +netdevice that has no support in the DPDK. It is possible to add explicit +rte_flow rules on the tap PMD to capture specific traffic (see next section for +examples). + +After the DPDK application is started you can send and receive packets on the +interface using the standard rx_burst/tx_burst APIs in DPDK. From the host +point of view you can use any host tool like tcpdump, Wireshark, ping, Pktgen +and others to communicate with the DPDK application. The DPDK application may +not understand network protocols like IPv4/6, UDP or TCP unless the +application has been written to understand these protocols. + +If you need the interface as a real network interface meaning running and has +a valid IP address then you can do this with the following commands:: + + sudo ip link set dtap0 up; sudo ip addr add 192.168.0.250/24 dev dtap0 + sudo ip link set dtap1 up; sudo ip addr add 192.168.1.250/24 dev dtap1 + +Please change the IP addresses as you see fit. + +If routing is enabled on the host you can also communicate with the DPDK App +over the internet via a standard socket layer application as long as you +account for the protocol handing in the application. + +If you have a Network Stack in your DPDK application or something like it you +can utilize that stack to handle the network protocols. Plus you would be able +to address the interface using an IP address assigned to the internal +interface. + +Flow API support +---------------- + +The tap PMD supports major flow API pattern items and actions, when running on +linux kernels above 4.2 ("Flower" classifier required). +The kernel support can be checked with this command:: + + zcat /proc/config.gz | ( grep 'CLS_FLOWER=' || echo 'not supported' ) | + tee -a /dev/stderr | grep -q '=m' && + lsmod | ( grep cls_flower || echo 'try modprobe cls_flower' ) + +Supported items: + +- eth: src and dst (with variable masks), and eth_type (0xffff mask). +- vlan: vid, pcp, tpid, but not eid. (requires kernel 4.9) +- ipv4/6: src and dst (with variable masks), and ip_proto (0xffff mask). +- udp/tcp: src and dst port (0xffff) mask. + +Supported actions: + +- DROP +- QUEUE +- PASSTHRU + +It is generally not possible to provide a "last" item. However, if the "last" +item, once masked, is identical to the masked spec, then it is supported. + +Only IPv4/6 and MAC addresses can use a variable mask. All other items need a +full mask (exact match). + +As rules are translated to TC, it is possible to show them with something like:: + + tc -s filter show dev tap1 parent 1: + +Examples of testpmd flow rules +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Drop packets for destination IP 192.168.0.1:: + + testpmd> flow create 0 priority 1 ingress pattern eth / ipv4 dst is 1.1.1.1 \ + / end actions drop / end + +Ensure packets from a given MAC address are received on a queue 2:: + + testpmd> flow create 0 priority 2 ingress pattern eth src is 06:05:04:03:02:01 \ + / end actions queue index 2 / end + +Drop UDP packets in vlan 3:: + + testpmd> flow create 0 priority 3 ingress pattern eth / vlan vid is 3 / \ + ipv4 proto is 17 / end actions drop / end + +Example +------- + +The following is a simple example of using the TUN/TAP PMD with the Pktgen +packet generator. It requires that the ``socat`` utility is installed on the +test system. + +Build DPDK, then pull down Pktgen and build pktgen using the DPDK SDK/Target +used to build the dpdk you pulled down. + +Run pktgen from the pktgen directory in a terminal with a commandline like the +following:: + + sudo ./app/app/x86_64-native-linuxapp-gcc/app/pktgen -l 1-5 -n 4 \ + --proc-type auto --log-level 8 --socket-mem 512,512 --file-prefix pg \ + --vdev=net_tap0 --vdev=net_tap1 -b 05:00.0 -b 05:00.1 \ + -b 04:00.0 -b 04:00.1 -b 04:00.2 -b 04:00.3 \ + -b 81:00.0 -b 81:00.1 -b 81:00.2 -b 81:00.3 \ + -b 82:00.0 -b 83:00.0 -- -T -P -m [2:3].0 -m [4:5].1 \ + -f themes/black-yellow.theme + +.. Note: + + Change the ``-b`` options to blacklist all of your physical ports. The + following command line is all one line. + + Also, ``-f themes/black-yellow.theme`` is optional if the default colors + work on your system configuration. See the Pktgen docs for more + information. + +Verify with ``ifconfig -a`` command in a different xterm window, should have a +``dtap0`` and ``dtap1`` interfaces created. + +Next set the links for the two interfaces to up via the commands below:: + + sudo ip link set dtap0 up; sudo ip addr add 192.168.0.250/24 dev dtap0 + sudo ip link set dtap1 up; sudo ip addr add 192.168.1.250/24 dev dtap1 + +Then use socat to create a loopback for the two interfaces:: + + sudo socat interface:dtap0 interface:dtap1 + +Then on the Pktgen command line interface you can start sending packets using +the commands ``start 0`` and ``start 1`` or you can start both at the same +time with ``start all``. The command ``str`` is an alias for ``start all`` and +``stp`` is an alias for ``stop all``. + +While running you should see the 64 byte counters increasing to verify the +traffic is being looped back. You can use ``set all size XXX`` to change the +size of the packets after you stop the traffic. Use pktgen ``help`` +command to see a list of all commands. You can also use the ``-f`` option to +load commands at startup in command line or Lua script in pktgen. diff --git a/dpdk/doc/guides/nics/thunderx.rst b/dpdk/doc/guides/nics/thunderx.rst index 248b1af7..45bc690a 100644 --- a/dpdk/doc/guides/nics/thunderx.rst +++ b/dpdk/doc/guides/nics/thunderx.rst @@ -1,5 +1,5 @@ .. BSD LICENSE - Copyright (C) Cavium networks Ltd. 2016. + Copyright (C) Cavium, Inc. 2016. All rights reserved. Redistribution and use in source and binary forms, with or without @@ -12,7 +12,7 @@ notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - * Neither the name of Cavium networks nor the names of its + * Neither the name of Cavium, Inc nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -35,7 +35,7 @@ The ThunderX NICVF PMD (**librte_pmd_thunderx_nicvf**) provides poll mode driver support for the inbuilt NIC found in the **Cavium ThunderX** SoC family as well as their virtual functions (VF) in SR-IOV context. -More information can be found at `Cavium Networks Official Website +More information can be found at `Cavium, Inc Official Website `_. Features @@ -56,10 +56,13 @@ Features of the ThunderX PMD are: - VLAN stripping - SR-IOV VF - NUMA support +- Multi queue set support (up to 96 queues (12 queue sets)) per port Supported ThunderX SoCs ----------------------- - CN88xx +- CN81xx +- CN83xx Prerequisites ------------- @@ -74,9 +77,8 @@ Config File Options The following options can be modified in the ``config`` file. Please note that enabling debugging options may affect system performance. -- ``CONFIG_RTE_LIBRTE_THUNDERX_NICVF_PMD`` (default ``n``) +- ``CONFIG_RTE_LIBRTE_THUNDERX_NICVF_PMD`` (default ``y``) - By default it is enabled only for defconfig_arm64-thunderx-* config. Toggle compilation of the ``librte_pmd_thunderx_nicvf`` driver. - ``CONFIG_RTE_LIBRTE_THUNDERX_NICVF_DEBUG_INIT`` (default ``n``) @@ -99,35 +101,99 @@ Please note that enabling debugging options may affect system performance. Toggle display of PF mailbox related run-time check messages -Driver Compilation -~~~~~~~~~~~~~~~~~~ +Driver compilation and testing +------------------------------ -To compile the ThunderX NICVF PMD for Linux arm64 gcc target, run the -following “make” command: +Refer to the document :ref:`compiling and testing a PMD for a NIC ` +for details. -.. code-block:: console - - cd - make config T=arm64-thunderx-linuxapp-gcc install +To compile the ThunderX NICVF PMD for Linux arm64 gcc, +use arm64-thunderx-linuxapp-gcc as target. Linux ----- -.. _thunderx_testpmd_example: +SR-IOV: Prerequisites and sample Application Notes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Running testpmd -~~~~~~~~~~~~~~~ +Current ThunderX NIC PF/VF kernel modules maps each physical Ethernet port +automatically to virtual function (VF) and presented them as PCIe-like SR-IOV device. +This section provides instructions to configure SR-IOV with Linux OS. -This section demonstrates how to launch ``testpmd`` with ThunderX NIC VF device -managed by ``librte_pmd_thunderx_nicvf`` in the Linux operating system. - -#. Load ``vfio-pci`` driver: +#. Verify PF devices capabilities using ``lspci``: .. code-block:: console - modprobe vfio-pci + lspci -vvv - .. _thunderx_vfio_noiommu: + Example output: + + .. code-block:: console + + 0002:01:00.0 Ethernet controller: Cavium Networks Device a01e (rev 01) + ... + Capabilities: [100 v1] Alternative Routing-ID Interpretation (ARI) + ... + Capabilities: [180 v1] Single Root I/O Virtualization (SR-IOV) + ... + Kernel driver in use: thunder-nic + ... + + .. note:: + + Unless ``thunder-nic`` driver is in use make sure your kernel config includes ``CONFIG_THUNDER_NIC_PF`` setting. + +#. Verify VF devices capabilities and drivers using ``lspci``: + + .. code-block:: console + + lspci -vvv + + Example output: + + .. code-block:: console + + 0002:01:00.1 Ethernet controller: Cavium Networks Device 0011 (rev 01) + ... + Capabilities: [100 v1] Alternative Routing-ID Interpretation (ARI) + ... + Kernel driver in use: thunder-nicvf + ... + + 0002:01:00.2 Ethernet controller: Cavium Networks Device 0011 (rev 01) + ... + Capabilities: [100 v1] Alternative Routing-ID Interpretation (ARI) + ... + Kernel driver in use: thunder-nicvf + ... + + .. note:: + + Unless ``thunder-nicvf`` driver is in use make sure your kernel config includes ``CONFIG_THUNDER_NIC_VF`` setting. + +#. Pass VF device to VM context (PCIe Passthrough): + + The VF devices may be passed through to the guest VM using qemu or + virt-manager or virsh etc. + + Example qemu guest launch command: + + .. code-block:: console + + sudo qemu-system-aarch64 -name vm1 \ + -machine virt,gic_version=3,accel=kvm,usb=off \ + -cpu host -m 4096 \ + -smp 4,sockets=1,cores=8,threads=1 \ + -nographic -nodefaults \ + -kernel \ + -append "root=/dev/vda console=ttyAMA0 rw hugepagesz=512M hugepages=3" \ + -device vfio-pci,host=0002:01:00.1 \ + -drive file=,if=none,id=disk1,format=raw \ + -device virtio-blk-device,scsi=off,drive=disk1,id=virtio-disk1,bootindex=1 \ + -netdev tap,id=net0,ifname=tap0,script=/etc/qemu-ifup_thunder \ + -device virtio-net-device,netdev=net0 \ + -serial stdio \ + -mem-path /dev/huge #. Enable **VFIO-NOIOMMU** mode (optional): @@ -138,28 +204,21 @@ managed by ``librte_pmd_thunderx_nicvf`` in the Linux operating system. .. note:: **VFIO-NOIOMMU** is required only when running in VM context and should not be enabled otherwise. - See also :ref:`SR-IOV: Prerequisites and sample Application Notes `. -#. Bind the ThunderX NIC VF device to ``vfio-pci`` loaded in the previous step: +#. Running testpmd: - Setup VFIO permissions for regular users and then bind to ``vfio-pci``: - - .. code-block:: console - - ./tools/dpdk-devbind.py --bind vfio-pci 0002:01:00.2 - -#. Start ``testpmd`` with basic parameters: - - .. code-block:: console - - ./arm64-thunderx-linuxapp-gcc/app/testpmd -c 0xf -n 4 -w 0002:01:00.2 \ - -- -i --disable-hw-vlan-filter --crc-strip --no-flush-rx \ - --port-topology=loop + Follow instructions available in the document + :ref:`compiling and testing a PMD for a NIC ` + to run testpmd. Example output: .. code-block:: console + ./arm64-thunderx-linuxapp-gcc/app/testpmd -l 0-3 -n 4 -w 0002:01:00.2 \ + -- -i --disable-hw-vlan-filter --disable-crc-strip --no-flush-rx \ + --port-topology=loop + ... PMD: rte_nicvf_pmd_init(): librte_pmd_thunderx nicvf version 1.0 @@ -186,140 +245,111 @@ managed by ``librte_pmd_thunderx_nicvf`` in the Linux operating system. Done testpmd> -.. _thunderx_sriov_example: +Multiple Queue Set per DPDK port configuration +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -SR-IOV: Prerequisites and sample Application Notes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +There are two types of VFs: -Current ThunderX NIC PF/VF kernel modules maps each physical Ethernet port -automatically to virtual function (VF) and presented them as PCIe-like SR-IOV device. -This section provides instructions to configure SR-IOV with Linux OS. +- Primary VF +- Secondary VF -#. Verify PF devices capabilities using ``lspci``: +Each port consists of a primary VF and n secondary VF(s). Each VF provides 8 Tx/Rx queues to a port. +When a given port is configured to use more than 8 queues, it requires one (or more) secondary VF. +Each secondary VF adds 8 additional queues to the queue set. - .. code-block:: console +During PMD driver initialization, the primary VF's are enumerated by checking the +specific flag (see sqs message in DPDK boot log - sqs indicates secondary queue set). +They are at the beginning of VF list (the remain ones are secondary VF's). - lspci -vvv +The primary VFs are used as master queue sets. Secondary VFs provide +additional queue sets for primary ones. If a port is configured for more then +8 queues than it will request for additional queues from secondary VFs. - Example output: +Secondary VFs cannot be shared between primary VFs. - .. code-block:: console - - 0002:01:00.0 Ethernet controller: Cavium Networks Device a01e (rev 01) - ... - Capabilities: [100 v1] Alternative Routing-ID Interpretation (ARI) - ... - Capabilities: [180 v1] Single Root I/O Virtualization (SR-IOV) - ... - Kernel driver in use: thunder-nic - ... +Primary VFs are present on the beginning of the 'Network devices using kernel +driver' list, secondary VFs are on the remaining on the remaining part of the list. .. note:: - Unless ``thunder-nic`` driver is in use make sure your kernel config includes ``CONFIG_THUNDER_NIC_PF`` setting. - -#. Verify VF devices capabilities and drivers using ``lspci``: - - .. code-block:: console - - lspci -vvv - - Example output: - - .. code-block:: console - - 0002:01:00.1 Ethernet controller: Cavium Networks Device 0011 (rev 01) - ... - Capabilities: [100 v1] Alternative Routing-ID Interpretation (ARI) - ... - Kernel driver in use: thunder-nicvf - ... - - 0002:01:00.2 Ethernet controller: Cavium Networks Device 0011 (rev 01) - ... - Capabilities: [100 v1] Alternative Routing-ID Interpretation (ARI) - ... - Kernel driver in use: thunder-nicvf - ... + The VNIC driver in the multiqueue setup works differently than other drivers like `ixgbe`. + We need to bind separately each specific queue set device with the ``usertools/dpdk-devbind.py`` utility. .. note:: - Unless ``thunder-nicvf`` driver is in use make sure your kernel config includes ``CONFIG_THUNDER_NIC_VF`` setting. + Depending on the hardware used, the kernel driver sets a threshold ``vf_id``. VFs that try to attached with an id below or equal to + this boundary are considered primary VFs. VFs that try to attach with an id above this boundary are considered secondary VFs. -#. Verify PF/VF bind using ``dpdk-devbind.py``: + +Example device binding +~~~~~~~~~~~~~~~~~~~~~~ + +If a system has three interfaces, a total of 18 VF devices will be created +on a non-NUMA machine. + + .. note:: + + NUMA systems have 12 VFs per port and non-NUMA 6 VFs per port. .. code-block:: console - ./tools/dpdk-devbind.py --status + # usertools/dpdk-devbind.py --status - Example output: + Network devices using DPDK-compatible driver + ============================================ + + + Network devices using kernel driver + =================================== + 0000:01:10.0 'Device a026' if= drv=thunder-BGX unused=vfio-pci,uio_pci_generic + 0000:01:10.1 'Device a026' if= drv=thunder-BGX unused=vfio-pci,uio_pci_generic + 0002:01:00.0 'Device a01e' if= drv=thunder-nic unused=vfio-pci,uio_pci_generic + 0002:01:00.1 'Device 0011' if=eth0 drv=thunder-nicvf unused=vfio-pci,uio_pci_generic + 0002:01:00.2 'Device 0011' if=eth1 drv=thunder-nicvf unused=vfio-pci,uio_pci_generic + 0002:01:00.3 'Device 0011' if=eth2 drv=thunder-nicvf unused=vfio-pci,uio_pci_generic + 0002:01:00.4 'Device 0011' if= drv=thunder-nicvf unused=vfio-pci,uio_pci_generic + 0002:01:00.5 'Device 0011' if= drv=thunder-nicvf unused=vfio-pci,uio_pci_generic + 0002:01:00.6 'Device 0011' if= drv=thunder-nicvf unused=vfio-pci,uio_pci_generic + 0002:01:00.7 'Device 0011' if= drv=thunder-nicvf unused=vfio-pci,uio_pci_generic + 0002:01:01.0 'Device 0011' if= drv=thunder-nicvf unused=vfio-pci,uio_pci_generic + 0002:01:01.1 'Device 0011' if= drv=thunder-nicvf unused=vfio-pci,uio_pci_generic + 0002:01:01.2 'Device 0011' if= drv=thunder-nicvf unused=vfio-pci,uio_pci_generic + 0002:01:01.3 'Device 0011' if= drv=thunder-nicvf unused=vfio-pci,uio_pci_generic + 0002:01:01.4 'Device 0011' if= drv=thunder-nicvf unused=vfio-pci,uio_pci_generic + 0002:01:01.5 'Device 0011' if= drv=thunder-nicvf unused=vfio-pci,uio_pci_generic + 0002:01:01.6 'Device 0011' if= drv=thunder-nicvf unused=vfio-pci,uio_pci_generic + 0002:01:01.7 'Device 0011' if= drv=thunder-nicvf unused=vfio-pci,uio_pci_generic + 0002:01:02.0 'Device 0011' if= drv=thunder-nicvf unused=vfio-pci,uio_pci_generic + 0002:01:02.1 'Device 0011' if= drv=thunder-nicvf unused=vfio-pci,uio_pci_generic + 0002:01:02.2 'Device 0011' if= drv=thunder-nicvf unused=vfio-pci,uio_pci_generic + + Other network devices + ===================== + 0002:00:03.0 'Device a01f' unused=vfio-pci,uio_pci_generic + + +We want to bind two physical interfaces with 24 queues each device, we attach two primary VFs +and four secondary queues. In our example we choose two 10G interfaces eth1 (0002:01:00.2) and eth2 (0002:01:00.3). +We will choose four secondary queue sets from the ending of the list (0002:01:01.7-0002:01:02.2). + + +#. Bind two primary VFs to the ``vfio-pci`` driver: .. code-block:: console - ... - 0002:01:00.0 'Device a01e' if= drv=thunder-nic unused=vfio-pci - 0002:01:00.1 'Device 0011' if=eth0 drv=thunder-nicvf unused=vfio-pci - 0002:01:00.2 'Device 0011' if=eth1 drv=thunder-nicvf unused=vfio-pci - ... + usertools/dpdk-devbind.py -b vfio-pci 0002:01:00.2 + usertools/dpdk-devbind.py -b vfio-pci 0002:01:00.3 -#. Load ``vfio-pci`` driver: +#. Bind four primary VFs to the ``vfio-pci`` driver: .. code-block:: console - modprobe vfio-pci + usertools/dpdk-devbind.py -b vfio-pci 0002:01:01.7 + usertools/dpdk-devbind.py -b vfio-pci 0002:01:02.0 + usertools/dpdk-devbind.py -b vfio-pci 0002:01:02.1 + usertools/dpdk-devbind.py -b vfio-pci 0002:01:02.2 -#. Bind VF devices to ``vfio-pci`` using ``dpdk-devbind.py``: - - .. code-block:: console - - ./tools/dpdk-devbind.py --bind vfio-pci 0002:01:00.1 - ./tools/dpdk-devbind.py --bind vfio-pci 0002:01:00.2 - -#. Verify VF bind using ``dpdk-devbind.py``: - - .. code-block:: console - - ./tools/dpdk-devbind.py --status - - Example output: - - .. code-block:: console - - ... - 0002:01:00.1 'Device 0011' drv=vfio-pci unused= - 0002:01:00.2 'Device 0011' drv=vfio-pci unused= - ... - 0002:01:00.0 'Device a01e' if= drv=thunder-nic unused=vfio-pci - ... - -#. Pass VF device to VM context (PCIe Passthrough): - - The VF devices may be passed through to the guest VM using qemu or - virt-manager or virsh etc. - ``librte_pmd_thunderx_nicvf`` or ``thunder-nicvf`` should be used to bind - the VF devices in the guest VM in :ref:`VFIO-NOIOMMU ` mode. - - Example qemu guest launch command: - - .. code-block:: console - - sudo qemu-system-aarch64 -name vm1 \ - -machine virt,gic_version=3,accel=kvm,usb=off \ - -cpu host -m 4096 \ - -smp 4,sockets=1,cores=8,threads=1 \ - -nographic -nodefaults \ - -kernel \ - -append "root=/dev/vda console=ttyAMA0 rw hugepagesz=512M hugepages=3" \ - -device vfio-pci,host=0002:01:00.1 \ - -drive file=,if=none,id=disk1,format=raw \ - -device virtio-blk-device,scsi=off,drive=disk1,id=virtio-disk1,bootindex=1 \ - -netdev tap,id=net0,ifname=tap0,script=/etc/qemu-ifup_thunder \ - -device virtio-net-device,netdev=net0 \ - -serial stdio \ - -mem-path /dev/huge - -#. Refer to section :ref:`Running testpmd ` for instruction - how to launch ``testpmd`` application. +The nicvf thunderx driver will make use of attached secondary VFs automatically during the interface configuration stage. Limitations ----------- @@ -345,10 +375,3 @@ Maximum packet segments The ThunderX SoC family NICs support up to 12 segments per packet when working in scatter/gather mode. So, setting MTU will result with ``EINVAL`` when the frame size does not fit in the maximum number of segments. - -Limited VFs -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The ThunderX SoC family NICs has 128VFs and each VF has 8/8 queues -for RX/TX respectively. Current driver implementation has one to one mapping -between physical port and VF hence only limited VFs can be used. diff --git a/dpdk/doc/guides/nics/vhost.rst b/dpdk/doc/guides/nics/vhost.rst index 1e3f1ade..4f7ae899 100644 --- a/dpdk/doc/guides/nics/vhost.rst +++ b/dpdk/doc/guides/nics/vhost.rst @@ -66,6 +66,11 @@ The user can specify below arguments in `--vdev` option. It is used to specify the number of queues virtio-net device has. (Default: 1) +#. ``iommu-support``: + + It is used to enable iommu support in vhost library. + (Default: 0 (disabled)) + Vhost PMD event handling ------------------------ @@ -92,7 +97,7 @@ This section demonstrates vhost PMD with testpmd DPDK sample application. .. code-block:: console - ./testpmd -c f -n 4 --vdev 'eth_vhost0,iface=/tmp/sock0,queues=1' -- -i + ./testpmd -l 0-3 -n 4 --vdev 'net_vhost0,iface=/tmp/sock0,queues=1' -- -i Other basic DPDK preparations like hugepage enabling here. Please refer to the *DPDK Getting Started Guide* for detailed instructions. diff --git a/dpdk/doc/guides/nics/virtio.rst b/dpdk/doc/guides/nics/virtio.rst index 54310157..af82f86e 100644 --- a/dpdk/doc/guides/nics/virtio.rst +++ b/dpdk/doc/guides/nics/virtio.rst @@ -34,6 +34,7 @@ Poll Mode Driver for Emulated Virtio NIC Virtio is a para-virtualization framework initiated by IBM, and supported by KVM hypervisor. In the Data Plane Development Kit (DPDK), we provide a virtio Poll Mode Driver (PMD) as a software solution, comparing to SRIOV hardware solution, + for fast guest VM to guest VM communication and guest VM to host communication. Vhost is a kernel acceleration module for virtio qemu backend. @@ -41,9 +42,6 @@ The DPDK extends kni to support vhost raw socket interface, which enables vhost to directly read/ write packets from/to a physical port. With this enhancement, virtio could achieve quite promising performance. -In future release, we will also make enhancement to vhost backend, -releasing peak performance of virtio PMD driver. - For basic qemu-KVM installation and other Intel EM poll mode driver in guest VM, please refer to Chapter "Driver for VM Emulated Devices". @@ -73,20 +71,27 @@ In this release, the virtio PMD driver provides the basic functionality of packe * It supports multicast packets and promiscuous mode. -* The descriptor number for the Rx/Tx queue is hard-coded to be 256 by qemu. +* The descriptor number for the Rx/Tx queue is hard-coded to be 256 by qemu 2.7 and below. If given a different descriptor number by the upper application, the virtio PMD generates a warning and fall back to the hard-coded value. + Rx queue size can be configureable and up to 1024 since qemu 2.8 and above. Rx queue size is 256 + by default. Tx queue size is still hard-coded to be 256. * Features of mac/vlan filter are supported, negotiation with vhost/backend are needed to support them. When backend can't support vlan filter, virtio app on guest should disable vlan filter to make sure the virtio port is configured correctly. E.g. specify '--disable-hw-vlan' in testpmd command line. -* RTE_PKTMBUF_HEADROOM should be defined larger than sizeof(struct virtio_net_hdr), which is 10 bytes. +* "RTE_PKTMBUF_HEADROOM" should be defined + no less than "sizeof(struct virtio_net_hdr_mrg_rxbuf)", which is 12 bytes when mergeable or + "VIRTIO_F_VERSION_1" is set. + no less than "sizeof(struct virtio_net_hdr)", which is 10 bytes, when using non-mergeable. * Virtio does not support runtime configuration. * Virtio supports Link State interrupt. +* Virtio supports Rx interrupt (so far, only support 1:1 mapping for queue/interrupt). + * Virtio supports software vlan stripping and inserting. * Virtio supports using port IO to get PCI resource when uio/igb_uio module is not available. @@ -128,7 +133,7 @@ Host2VM communication example .. code-block:: console - examples/kni/build/app/kni -c 0xf -n 4 -- -p 0x1 -P --config="(0,1,3)" + examples/kni/build/app/kni -l 0-3 -n 4 -- -p 0x1 -P --config="(0,1,3)" This command generates one network device vEth0 for physical port. If specify more physical ports, the generated network device will be vEth1, vEth2, and so on. @@ -172,7 +177,7 @@ Host2VM communication example modprobe uio echo 512 > /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages modprobe uio_pci_generic - python tools/dpdk-devbind.py -b uio_pci_generic 00:03.0 + python usertools/dpdk-devbind.py -b uio_pci_generic 00:03.0 We use testpmd as the forwarding application in this example. @@ -273,4 +278,69 @@ The corresponding callbacks are: Example of using the vector version of the virtio poll mode driver in ``testpmd``:: - testpmd -c 0x7 -n 4 -- -i --txqflags=0xF01 --rxq=1 --txq=1 --nb-cores=1 + testpmd -l 0-2 -n 4 -- -i --txqflags=0xF01 --rxq=1 --txq=1 --nb-cores=1 + + +Interrupt mode +-------------- + +.. _virtio_interrupt_mode: + +There are three kinds of interrupts from a virtio device over PCI bus: config +interrupt, Rx interrupts, and Tx interrupts. Config interrupt is used for +notification of device configuration changes, especially link status (lsc). +Interrupt mode is translated into Rx interrupts in the context of DPDK. + +.. Note:: + + Virtio PMD already has support for receiving lsc from qemu when the link + status changes, especially when vhost user disconnects. However, it fails + to do that if the VM is created by qemu 2.6.2 or below, since the + capability to detect vhost user disconnection is introduced in qemu 2.7.0. + +Prerequisites for Rx interrupts +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To support Rx interrupts, +#. Check if guest kernel supports VFIO-NOIOMMU: + + Linux started to support VFIO-NOIOMMU since 4.8.0. Make sure the guest + kernel is compiled with: + + .. code-block:: console + + CONFIG_VFIO_NOIOMMU=y + +#. Properly set msix vectors when starting VM: + + Enable multi-queue when starting VM, and specify msix vectors in qemu + cmdline. (N+1) is the minimum, and (2N+2) is mostly recommended. + + .. code-block:: console + + $(QEMU) ... -device virtio-net-pci,mq=on,vectors=2N+2 ... + +#. In VM, insert vfio module in NOIOMMU mode: + + .. code-block:: console + + modprobe vfio enable_unsafe_noiommu_mode=1 + modprobe vfio-pci + +#. In VM, bind the virtio device with vfio-pci: + + .. code-block:: console + + python usertools/dpdk-devbind.py -b vfio-pci 00:03.0 + +Example +~~~~~~~ + +Here we use l3fwd-power as an example to show how to get started. + + Example: + + .. code-block:: console + + $ l3fwd-power -l 0-1 -- -p 1 -P --config="(0,0,1)" \ + --no-numa --parse-ptype diff --git a/dpdk/doc/guides/nics/vmxnet3.rst b/dpdk/doc/guides/nics/vmxnet3.rst index e919088d..bf845942 100644 --- a/dpdk/doc/guides/nics/vmxnet3.rst +++ b/dpdk/doc/guides/nics/vmxnet3.rst @@ -32,17 +32,11 @@ Poll Mode Driver for Paravirtual VMXNET3 NIC ============================================ The VMXNET3 adapter is the next generation of a paravirtualized NIC, introduced by VMware* ESXi. -It is designed for performance and is not related to VMXNET or VMXENET2. -It offers all the features available in VMXNET2, and adds several new features such as, +It is designed for performance, offers all the features available in VMXNET2, and adds several new features such as, multi-queue support (also known as Receive Side Scaling, RSS), IPv6 offloads, and MSI/MSI-X interrupt delivery. -Because operating system vendors do not provide built-in drivers for this card, -VMware Tools must be installed to have a driver for the VMXNET3 network adapter available. One can use the same device in a DPDK application with VMXNET3 PMD introduced in DPDK API. -Currently, the driver provides basic support for using the device in a DPDK application running on a guest OS. -Optimization is needed on the backend, that is, the VMware* ESXi vmkernel switch, to achieve optimal performance end-to-end. - In this chapter, two setups with the use of the VMXNET3 PMD are demonstrated: #. Vmxnet3 with a native NIC connected to a vSwitch @@ -59,8 +53,6 @@ For performance details, refer to the following link from VMware: `http://www.vmware.com/pdf/vsp_4_vmxnet3_perf.pdf `_ As a PMD, the VMXNET3 driver provides the packet reception and transmission callbacks, vmxnet3_recv_pkts and vmxnet3_xmit_pkts. -It does not support scattered packet reception as part of vmxnet3_recv_pkts and vmxnet3_xmit_pkts. -Also, it does not support scattered packet reception as part of the device operations supported. The VMXNET3 PMD handles all the packet buffer memory allocation and resides in guest address space and it is solely responsible to free that memory when not needed. @@ -79,7 +71,7 @@ This keeps performance up on the RX side, even though the device provides a noti In the transmit routine, the DPDK application fills packet buffer pointers in the descriptors of the command ring and notifies the hypervisor. -In response the hypervisor takes packets and passes them to the vSwitch. It writes into the completion descriptors ring. +In response the hypervisor takes packets and passes them to the vSwitch, It writes into the completion descriptors ring. The rings are read by the PMD in the next transmit routine call and the buffers and descriptors are freed from memory. Features and Limitations of VMXNET3 PMD diff --git a/dpdk/doc/guides/platform/index.rst b/dpdk/doc/guides/platform/index.rst new file mode 100644 index 00000000..69e560cd --- /dev/null +++ b/dpdk/doc/guides/platform/index.rst @@ -0,0 +1,39 @@ +.. BSD LICENSE + Copyright (C) Cavium, Inc. 2017. All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Cavium Inc nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Platform Specific Guides +======================== + +The following are platform specific guides and setup information. + +.. toctree:: + :maxdepth: 2 + :numbered: + + octeontx diff --git a/dpdk/doc/guides/platform/octeontx.rst b/dpdk/doc/guides/platform/octeontx.rst new file mode 100644 index 00000000..fb708caf --- /dev/null +++ b/dpdk/doc/guides/platform/octeontx.rst @@ -0,0 +1,81 @@ +.. BSD LICENSE + Copyright (C) Cavium, Inc. 2017. All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Cavium, Inc nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +OCTEONTX Board Support Package +============================== + +This doc has information about steps to setup octeontx platform +and information about common offload hw block drivers of +**Cavium OCTEONTX** SoC family. + + +More information about SoC can be found at `Cavium, Inc Official Website +`_. + +Common Offload HW Block Drivers +------------------------------- + +1. **Eventdev Driver** + See :doc: `../eventdevs/octeontx.rst` for octeontx ssovf eventdev driver + information. + +2. **Mempool Driver** + See :doc: `../mempool/octeontx.rst` for octeontx fpavf mempool driver + information. + +Steps To Setup Platform +----------------------- + +There are three main pre-prerequisites for setting up Platform drivers on +OCTEONTX compatible board: + +1. **OCTEONTX Linux kernel PF driver for Network acceleration HW blocks** + + The OCTEONTX Linux kernel drivers (includes the required PF driver for the + Platform drivers) are available on Github at `octeontx-kmod `_ + along with build, install and dpdk usage instructions. + +2. **ARM64 Tool Chain** + + For example, the *aarch64* Linaro Toolchain, which can be obtained from + `here `_. + +3. **Rootfile system** + + Any *aarch64* supporting filesystem can be used. For example, + Ubuntu 15.10 (Wily) or 16.04 LTS (Xenial) userland which can be obtained + from ``_. + + As an alternative method, Platform drivers can also be executed using images provided + as part of SDK from Cavium. The SDK includes all the above prerequisites necessary + to bring up a OCTEONTX board. + + SDK and related information can be obtained from: `Cavium support site `_. + +- Follow the DPDK :doc: `../linux_gsg/index.rst` to setup the basic DPDK environment. diff --git a/dpdk/doc/guides/prog_guide/cryptodev_lib.rst b/dpdk/doc/guides/prog_guide/cryptodev_lib.rst index ca3f551b..2b338b92 100644 --- a/dpdk/doc/guides/prog_guide/cryptodev_lib.rst +++ b/dpdk/doc/guides/prog_guide/cryptodev_lib.rst @@ -1,5 +1,5 @@ .. BSD LICENSE - Copyright(c) 2016 Intel Corporation. All rights reserved. + Copyright(c) 2016-2017 Intel Corporation. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions @@ -68,13 +68,13 @@ From the command line using the --vdev EAL option .. code-block:: console - --vdev 'cryptodev_aesni_mb_pmd0,max_nb_queue_pairs=2,max_nb_sessions=1024,socket_id=0' + --vdev 'crypto_aesni_mb0,max_nb_queue_pairs=2,max_nb_sessions=1024,socket_id=0' -Our using the rte_eal_vdev_init API within the application code. +Our using the rte_vdev_init API within the application code. .. code-block:: c - rte_eal_vdev_init("cryptodev_aesni_mb_pmd", + rte_vdev_init("crypto_aesni_mb", "max_nb_queue_pairs=2,max_nb_sessions=1024,socket_id=0") All virtual Crypto devices support the following initialization parameters: @@ -114,9 +114,8 @@ The rte_cryptodev_configure API is used to configure a Crypto device. int rte_cryptodev_configure(uint8_t dev_id, struct rte_cryptodev_config *config) -The ``rte_cryptodev_config`` structure is used to pass the configuration parameters. -In contains parameter for socket selection, number of queue pairs and the -session mempool configuration. +The ``rte_cryptodev_config`` structure is used to pass the configuration +parameters for socket selection and number of queue pairs. .. code-block:: c @@ -125,12 +124,6 @@ session mempool configuration. /**< Socket to allocate resources on */ uint16_t nb_queue_pairs; /**< Number of queue pairs to configure on device */ - - struct { - uint32_t nb_objs; - uint32_t cache_size; - } session_mp; - /**< Session mempool configuration */ }; @@ -188,8 +181,9 @@ the device having hardware acceleration or supporting symmetric Crypto operations, The capabilities mechanism defines the individual algorithms/functions which -the device supports, such as a specific symmetric Crypto cipher or -authentication operation. +the device supports, such as a specific symmetric Crypto cipher, +authentication operation or Authenticated Encryption with Associated Data +(AEAD) operation. Device Features @@ -245,7 +239,8 @@ algorithm AES_CBC. .max = 12, .increment = 0 }, - .aad_size = { 0 } + .aad_size = { 0 }, + .iv_size = { 0 } } } }, @@ -291,7 +286,7 @@ relevant information for the device. struct rte_cryptodev_info { const char *driver_name; - enum rte_cryptodev_type dev_type; + uint8_t driver_id; struct rte_pci_device *pci_dev; uint64_t feature_flags; @@ -359,11 +354,11 @@ Crypto operation to be processed on a particular Crypto device poll mode driver. .. figure:: img/crypto_op.* -The operation structure includes the operation type and the operation status, -a reference to the operation specific data, which can vary in size and content -depending on the operation being provisioned. It also contains the source -mempool for the operation, if it allocate from a mempool. Finally an -opaque pointer for user specific data is provided. +The operation structure includes the operation type, the operation status +and the session type (session-based/less), a reference to the operation +specific data, which can vary in size and content depending on the operation +being provisioned. It also contains the source mempool for the operation, +if it allocated from a mempool. If Crypto operations are allocated from a Crypto operation mempool, see next section, there is also the ability to allocate private memory with the @@ -431,7 +426,7 @@ operations, as well as also supporting AEAD operations. Session and Session Management ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Session are used in symmetric cryptographic processing to store the immutable +Sessions are used in symmetric cryptographic processing to store the immutable data defined in a cryptographic transform which is used in the operation processing of a packet flow. Sessions are used to manage information such as expand cipher keys and HMAC IPADs and OPADs, which need to be calculated for a @@ -442,29 +437,33 @@ Crypto workloads. .. figure:: img/cryptodev_sym_sess.* -The Crypto device framework provides a set of session pool management APIs for -the creation and freeing of the sessions, utilizing the Mempool Library. +The Crypto device framework provides APIs to allocate and initizalize sessions +for crypto devices, where sessions are mempool objects. +It is the application's responsibility to create and manage the session mempools. +This approach allows for different scenarios such as having a single session +mempool for all crypto devices (where the mempool object size is big +enough to hold the private session of any crypto device), as well as having +multiple session mempools of different sizes for better memory usage. -The framework also provides hooks so the PMDs can pass the amount of memory -required for that PMDs private session parameters, as well as initialization -functions for the configuration of the session parameters and freeing function -so the PMD can managed the memory on destruction of a session. +An application can use ``rte_cryptodev_get_private_session_size()`` to +get the private session size of given crypto device. This function would allow +an application to calculate the max device session size of all crypto devices +to create a single session mempool. +If instead an application creates multiple session mempools, the Crypto device +framework also provides ``rte_cryptodev_get_header_session_size`` to get +the size of an uninitialized session. -**Note**: Sessions created on a particular device can only be used on Crypto -devices of the same type, and if you try to use a session on a device different -to that on which it was created then the Crypto operation will fail. +Once the session mempools have been created, ``rte_cryptodev_sym_session_create()`` +is used to allocate an uninitialized session from the given mempool. +The session then must be initialized using ``rte_cryptodev_sym_session_init()`` +for each of the required crypto devices. A symmetric transform chain +is used to specify the operation and its parameters. See the section below for +details on transforms. -``rte_cryptodev_sym_session_create()`` is used to create a symmetric session on -Crypto device. A symmetric transform chain is used to specify the particular -operation and its parameters. See the section below for details on transforms. - -.. code-block:: c - - struct rte_cryptodev_sym_session * rte_cryptodev_sym_session_create( - uint8_t dev_id, struct rte_crypto_sym_xform *xform); - -**Note**: For AEAD operations the algorithm selected for authentication and -ciphering must aligned, eg AES_GCM. +When a session is no longer used, user must call ``rte_cryptodev_sym_session_clear()`` +for each of the crypto devices that are using the session, to free all driver +private session data. Once this is done, session should be freed using +``rte_cryptodev_sym_session_free`` which returns them to their mempool. Transforms and Transform Chaining @@ -476,9 +475,8 @@ operations such as cipher encrypt and authentication generate, the next pointer allows transform to be chained together. Crypto devices which support chaining must publish the chaining of symmetric Crypto operations feature flag. -Currently there are two transforms types cipher and authentication, to specify -an AEAD operation it is required to chain a cipher and an authentication -transform together. Also it is important to note that the order in which the +Currently there are three transforms types cipher, authentication and AEAD. +Also it is important to note that the order in which the transforms are passed indicates the order of the chaining. .. code-block:: c @@ -493,6 +491,8 @@ transforms are passed indicates the order of the chaining. /**< Authentication / hash xform */ struct rte_crypto_cipher_xform cipher; /**< Cipher xform */ + struct rte_crypto_aead_xform aead; + /**< AEAD xform */ }; }; @@ -512,9 +512,9 @@ buffer. It is used for either cipher, authentication, AEAD and chained operations. As a minimum the symmetric operation must have a source data buffer (``m_src``), -the session type (session-based/less), a valid session (or transform chain if in -session-less mode) and the minimum authentication/ cipher parameters required -depending on the type of operation specified in the session or the transform +a valid session (or transform chain if in session-less mode) and the minimum +authentication/ cipher/ AEAD parameters required depending on the type of operation +specified in the session or the transform chain. .. code-block:: c @@ -523,8 +523,6 @@ chain. struct rte_mbuf *m_src; struct rte_mbuf *m_dst; - enum rte_crypto_sym_op_sess_type type; - union { struct rte_cryptodev_sym_session *session; /**< Handle for the initialised session context */ @@ -532,39 +530,255 @@ chain. /**< Session-less API Crypto operation parameters */ }; - struct { + union { struct { - uint32_t offset; - uint32_t length; - } data; /**< Data offsets and length for ciphering */ + struct { + uint32_t offset; + uint32_t length; + } data; /**< Data offsets and length for AEAD */ + + struct { + uint8_t *data; + rte_iova_t phys_addr; + } digest; /**< Digest parameters */ + + struct { + uint8_t *data; + rte_iova_t phys_addr; + } aad; + /**< Additional authentication parameters */ + } aead; struct { - uint8_t *data; - phys_addr_t phys_addr; - uint16_t length; - } iv; /**< Initialisation vector parameters */ - } cipher; + struct { + struct { + uint32_t offset; + uint32_t length; + } data; /**< Data offsets and length for ciphering */ + } cipher; - struct { - struct { - uint32_t offset; - uint32_t length; - } data; /**< Data offsets and length for authentication */ + struct { + struct { + uint32_t offset; + uint32_t length; + } data; + /**< Data offsets and length for authentication */ - struct { - uint8_t *data; - phys_addr_t phys_addr; - uint16_t length; - } digest; /**< Digest parameters */ + struct { + uint8_t *data; + rte_iova_t phys_addr; + } digest; /**< Digest parameters */ + } auth; + }; + }; + }; - struct { - uint8_t *data; - phys_addr_t phys_addr; - uint16_t length; - } aad; /**< Additional authentication parameters */ - } auth; +Sample code +----------- + +There are various sample applications that show how to use the cryptodev library, +such as the L2fwd with Crypto sample application (L2fwd-crypto) and +the IPSec Security Gateway application (ipsec-secgw). + +While these applications demonstrate how an application can be created to perform +generic crypto operation, the required complexity hides the basic steps of +how to use the cryptodev APIs. + +The following sample code shows the basic steps to encrypt several buffers +with AES-CBC (although performing other crypto operations is similar), +using one of the crypto PMDs available in DPDK. + +.. code-block:: c + + /* + * Simple example to encrypt several buffers with AES-CBC using + * the Cryptodev APIs. + */ + + #define MAX_SESSIONS 1024 + #define NUM_MBUFS 1024 + #define POOL_CACHE_SIZE 128 + #define BURST_SIZE 32 + #define BUFFER_SIZE 1024 + #define AES_CBC_IV_LENGTH 16 + #define AES_CBC_KEY_LENGTH 16 + #define IV_OFFSET (sizeof(struct rte_crypto_op) + \ + sizeof(struct rte_crypto_sym_op)) + + struct rte_mempool *mbuf_pool, *crypto_op_pool, *session_pool; + unsigned int session_size; + int ret; + + /* Initialize EAL. */ + ret = rte_eal_init(argc, argv); + if (ret < 0) + rte_exit(EXIT_FAILURE, "Invalid EAL arguments\n"); + + uint8_t socket_id = rte_socket_id(); + + /* Create the mbuf pool. */ + mbuf_pool = rte_pktmbuf_pool_create("mbuf_pool", + NUM_MBUFS, + POOL_CACHE_SIZE, + 0, + RTE_MBUF_DEFAULT_BUF_SIZE, + socket_id); + if (mbuf_pool == NULL) + rte_exit(EXIT_FAILURE, "Cannot create mbuf pool\n"); + + /* + * The IV is always placed after the crypto operation, + * so some private data is required to be reserved. + */ + unsigned int crypto_op_private_data = AES_CBC_IV_LENGTH; + + /* Create crypto operation pool. */ + crypto_op_pool = rte_crypto_op_pool_create("crypto_op_pool", + RTE_CRYPTO_OP_TYPE_SYMMETRIC, + NUM_MBUFS, + POOL_CACHE_SIZE, + crypto_op_private_data, + socket_id); + if (crypto_op_pool == NULL) + rte_exit(EXIT_FAILURE, "Cannot create crypto op pool\n"); + + /* Create the virtual crypto device. */ + char args[128]; + const char *crypto_name = "crypto_aesni_mb0"; + snprintf(args, sizeof(args), "socket_id=%d", socket_id); + ret = rte_vdev_init(crypto_name, args); + if (ret != 0) + rte_exit(EXIT_FAILURE, "Cannot create virtual device"); + + uint8_t cdev_id = rte_cryptodev_get_dev_id(crypto_name); + + /* Get private session data size. */ + session_size = rte_cryptodev_get_private_session_size(cdev_id); + + /* + * Create session mempool, with two objects per session, + * one for the session header and another one for the + * private session data for the crypto device. + */ + session_pool = rte_mempool_create("session_pool", + MAX_SESSIONS * 2, + session_size, + POOL_CACHE_SIZE, + 0, NULL, NULL, NULL, + NULL, socket_id, + 0); + + /* Configure the crypto device. */ + struct rte_cryptodev_config conf = { + .nb_queue_pairs = 1, + .socket_id = socket_id + }; + struct rte_cryptodev_qp_conf qp_conf = { + .nb_descriptors = 2048 + }; + + if (rte_cryptodev_configure(cdev_id, &conf) < 0) + rte_exit(EXIT_FAILURE, "Failed to configure cryptodev %u", cdev_id); + + if (rte_cryptodev_queue_pair_setup(cdev_id, 0, &qp_conf, + socket_id, session_pool) < 0) + rte_exit(EXIT_FAILURE, "Failed to setup queue pair\n"); + + if (rte_cryptodev_start(cdev_id) < 0) + rte_exit(EXIT_FAILURE, "Failed to start device\n"); + + /* Create the crypto transform. */ + uint8_t cipher_key[16] = {0}; + struct rte_crypto_sym_xform cipher_xform = { + .next = NULL, + .type = RTE_CRYPTO_SYM_XFORM_CIPHER, + .cipher = { + .op = RTE_CRYPTO_CIPHER_OP_ENCRYPT, + .algo = RTE_CRYPTO_CIPHER_AES_CBC, + .key = { + .data = cipher_key, + .length = AES_CBC_KEY_LENGTH + }, + .iv = { + .offset = IV_OFFSET, + .length = AES_CBC_IV_LENGTH + } + } + }; + + /* Create crypto session and initialize it for the crypto device. */ + struct rte_cryptodev_sym_session *session; + session = rte_cryptodev_sym_session_create(session_pool); + if (session == NULL) + rte_exit(EXIT_FAILURE, "Session could not be created\n"); + + if (rte_cryptodev_sym_session_init(cdev_id, session, + &cipher_xform, session_pool) < 0) + rte_exit(EXIT_FAILURE, "Session could not be initialized " + "for the crypto device\n"); + + /* Get a burst of crypto operations. */ + struct rte_crypto_op *crypto_ops[BURST_SIZE]; + if (rte_crypto_op_bulk_alloc(crypto_op_pool, + RTE_CRYPTO_OP_TYPE_SYMMETRIC, + crypto_ops, BURST_SIZE) == 0) + rte_exit(EXIT_FAILURE, "Not enough crypto operations available\n"); + + /* Get a burst of mbufs. */ + struct rte_mbuf *mbufs[BURST_SIZE]; + if (rte_pktmbuf_alloc_bulk(mbuf_pool, mbufs, BURST_SIZE) < 0) + rte_exit(EXIT_FAILURE, "Not enough mbufs available"); + + /* Initialize the mbufs and append them to the crypto operations. */ + unsigned int i; + for (i = 0; i < BURST_SIZE; i++) { + if (rte_pktmbuf_append(mbufs[i], BUFFER_SIZE) == NULL) + rte_exit(EXIT_FAILURE, "Not enough room in the mbuf\n"); + crypto_ops[i]->sym->m_src = mbufs[i]; } + /* Set up the crypto operations. */ + for (i = 0; i < BURST_SIZE; i++) { + struct rte_crypto_op *op = crypto_ops[i]; + /* Modify bytes of the IV at the end of the crypto operation */ + uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, + IV_OFFSET); + + generate_random_bytes(iv_ptr, AES_CBC_IV_LENGTH); + + op->sym->cipher.data.offset = 0; + op->sym->cipher.data.length = BUFFER_SIZE; + + /* Attach the crypto session to the operation */ + rte_crypto_op_attach_sym_session(op, session); + } + + /* Enqueue the crypto operations in the crypto device. */ + uint16_t num_enqueued_ops = rte_cryptodev_enqueue_burst(cdev_id, 0, + crypto_ops, BURST_SIZE); + + /* + * Dequeue the crypto operations until all the operations + * are proccessed in the crypto device. + */ + uint16_t num_dequeued_ops, total_num_dequeued_ops = 0; + do { + struct rte_crypto_op *dequeued_ops[BURST_SIZE]; + num_dequeued_ops = rte_cryptodev_dequeue_burst(cdev_id, 0, + dequeued_ops, BURST_SIZE); + total_num_dequeued_ops += num_dequeued_ops; + + /* Check if operation was processed successfully */ + for (i = 0; i < num_dequeued_ops; i++) { + if (dequeued_ops[i]->status != RTE_CRYPTO_OP_STATUS_SUCCESS) + rte_exit(EXIT_FAILURE, + "Some operations were not processed correctly"); + } + + rte_mempool_put_bulk(crypto_op_pool, (void **)dequeued_ops, + num_dequeued_ops); + } while (total_num_dequeued_ops < num_enqueued_ops); + Asymmetric Cryptography ----------------------- diff --git a/dpdk/doc/guides/prog_guide/dev_kit_build_system.rst b/dpdk/doc/guides/prog_guide/dev_kit_build_system.rst index fa2411f7..ad032c5f 100644 --- a/dpdk/doc/guides/prog_guide/dev_kit_build_system.rst +++ b/dpdk/doc/guides/prog_guide/dev_kit_build_system.rst @@ -53,62 +53,7 @@ Build Directory Concept ~~~~~~~~~~~~~~~~~~~~~~~ After installation, a build directory structure is created. -Each build directory contains include files, libraries, and applications: - -.. code-block:: console - - ~/DPDK$ ls - app MAINTAINERS - config Makefile - COPYRIGHT mk - doc scripts - examples lib - tools x86_64-native-linuxapp-gcc - x86_64-native-linuxapp-icc i686-native-linuxapp-gcc - i686-native-linuxapp-icc - - ... - ~/DEV/DPDK$ ls i686-native-linuxapp-gcc - - app build buildtools include kmod lib Makefile - - - ~/DEV/DPDK$ ls i686-native-linuxapp-gcc/app/ - cmdline_test dump_cfg test testpmd - cmdline_test.map dump_cfg.map test.map - testpmd.map - - - ~/DEV/DPDK$ ls i686-native-linuxapp-gcc/lib/ - - libethdev.a librte_hash.a librte_mbuf.a librte_pmd_ixgbe.a - - librte_cmdline.a librte_lpm.a librte_mempool.a librte_ring.a - - librte_eal.a librte_pmd_e1000.a librte_timer.a - - - ~/DEV/DPDK$ ls i686-native-linuxapp-gcc/include/ - arch rte_cpuflags.h rte_memcpy.h - cmdline_cirbuf.h rte_cycles.h rte_memory.h - cmdline.h rte_debug.h rte_mempool.h - cmdline_parse_etheraddr.h rte_eal.h rte_memzone.h - cmdline_parse.h rte_errno.h rte_pci_dev_ids.h - cmdline_parse_ipaddr.h rte_ethdev.h rte_pci.h - cmdline_parse_num.h rte_ether.h rte_per_lcore.h - cmdline_parse_portlist.h rte_fbk_hash.h rte_prefetch.h - cmdline_parse_string.h rte_hash_crc.h rte_random.h - cmdline_rdline.h rte_hash.h rte_ring.h - cmdline_socket.h rte_interrupts.h rte_rwlock.h - cmdline_vt100.h rte_ip.h rte_sctp.h - exec-env rte_jhash.h rte_spinlock.h - rte_alarm.h rte_launch.h rte_string_fns.h - rte_atomic.h rte_lcore.h rte_tailq.h - rte_branch_prediction.h rte_log.h rte_tcp.h - rte_byteorder.h rte_lpm.h rte_timer.h - rte_common.h rte_malloc.h rte_udp.h - rte_config.h rte_mbuf.h - +Each build directory contains include files, libraries, and applications. A build directory is specific to a configuration that includes architecture + execution environment + toolchain. It is possible to have several build directories sharing the same sources with different configurations. @@ -319,7 +264,7 @@ instance the macro: .. code-block:: c - PMD_REGISTER_DRIVER(drv, name) + RTE_PMD_REGISTER_PCI(name, drv) Creates the following symbol: @@ -422,7 +367,7 @@ Variables that Can be Set/Overridden in a Makefile Only * POSTCLEAN: A list of actions to be taken after cleaning. The user should use += to append data in this variable. -* DEPDIR-y: Only used in the development kit framework to specify if the build of the current directory depends on build of another one. +* DEPDIRS-$(DIR): Only used in the development kit framework to specify if the build of the current directory depends on build of another one. This is needed to support parallel builds correctly. Variables that can be Set/Overridden by the User on the Command Line Only diff --git a/dpdk/doc/guides/prog_guide/dev_kit_root_make_help.rst b/dpdk/doc/guides/prog_guide/dev_kit_root_make_help.rst index fb3520e1..d7c41064 100644 --- a/dpdk/doc/guides/prog_guide/dev_kit_root_make_help.rst +++ b/dpdk/doc/guides/prog_guide/dev_kit_root_make_help.rst @@ -152,34 +152,6 @@ Documentation Targets Generate the guides documentation in pdf. - -Deps Targets ------------- - -* depdirs - - This target is implicitly called by make config. - Typically, there is no need for a user to call it, - except if DEPDIRS-y variables have been updated in Makefiles. - It will generate the file $(RTE_OUTPUT)/.depdirs. - - Example: - - .. code-block:: console - - make depdirs O=mybuild - -* depgraph - - This command generates a dot graph of dependencies. - It can be displayed to debug circular dependency issues, or just to understand the dependencies. - - Example: - - .. code-block:: console - - make depgraph O=mybuild > /tmp/graph.dot && dotty /tmp/ graph.dot - Misc Targets ------------ diff --git a/dpdk/doc/guides/prog_guide/efd_lib.rst b/dpdk/doc/guides/prog_guide/efd_lib.rst new file mode 100644 index 00000000..3f90fa9a --- /dev/null +++ b/dpdk/doc/guides/prog_guide/efd_lib.rst @@ -0,0 +1,455 @@ +.. BSD LICENSE + Copyright(c) 2016-2017 Intel Corporation. All rights reserved. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +.. _Efd_Library: + +Elastic Flow Distributor Library +================================ + +Introduction +------------ + +In Data Centers today, clustering and scheduling of distributed workloads +is a very common task. Many workloads require a deterministic +partitioning of a flat key space among a cluster of machines. When a +packet enters the cluster, the ingress node will direct the packet to +its handling node. For example, data-centers with disaggregated storage +use storage metadata tables to forward I/O requests to the correct back end +storage cluster, stateful packet inspection will use match incoming +flows to signatures in flow tables to send incoming packets to their +intended deep packet inspection (DPI) devices, and so on. + +EFD is a distributor library that uses perfect hashing to determine a +target/value for a given incoming flow key. It has the following +advantages: first, because it uses perfect hashing it does not store the +key itself and hence lookup performance is not dependent on the key +size. Second, the target/value can be any arbitrary value hence the +system designer and/or operator can better optimize service rates and +inter-cluster network traffic locating. Third, since the storage +requirement is much smaller than a hash-based flow table (i.e. better +fit for CPU cache), EFD can scale to millions of flow keys. Finally, +with the current optimized library implementation, performance is fully +scalable with any number of CPU cores. + +Flow Based Distribution +----------------------- + +Computation Based Schemes +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Flow distribution and/or load balancing can be simply done using a +stateless computation, for instance using round-robin or a simple +computation based on the flow key as an input. For example, a hash +function can be used to direct a certain flow to a target based on +the flow key (e.g. ``h(key) mod n``) where h(key) is the hash value of the +flow key and n is the number of possible targets. + +.. _figure_efd1: + +.. figure:: img/efd_i1.* + + Load Balancing Using Front End Node + +In this scheme (:numref:`figure_efd1`), the front end server/distributor/load balancer +extracts the flow key from the input packet and applies a computation to determine where +this flow should be directed. Intuitively, this scheme is very simple +and requires no state to be kept at the front end node, and hence, +storage requirements are minimum. + +.. _figure_efd2: + +.. figure:: img/efd_i2.* + + Consistent Hashing + +A widely used flow distributor that belongs to the same category of +computation-based schemes is ``consistent hashing``, shown in :numref:`figure_efd2`. +Target destinations (shown in red) are hashed into the same space as the flow +keys (shown in blue), and keys are mapped to the nearest target in a clockwise +fashion. Dynamically adding and removing targets with consistent hashing +requires only K/n keys to be remapped on average, where K is the number of +keys, and n is the number of targets. In contrast, in a traditional hash-based +scheme, a change in the number of targets causes nearly all keys to be +remapped. + +Although computation-based schemes are simple and need very little +storage requirement, they suffer from the drawback that the system +designer/operator can’t fully control the target to assign a specific +key, as this is dictated by the hash function. +Deterministically co-locating of keys together (for example, to minimize +inter-server traffic or to optimize for network traffic conditions, +target load, etc.) is simply not possible. + +Flow-Table Based Schemes +~~~~~~~~~~~~~~~~~~~~~~~~ + +When using a Flow-Table based scheme to handle flow distribution/load +balancing, in contrast with computation-based schemes, the system designer +has the flexibility of assigning a given flow to any given +target. The flow table (e.g. DPDK RTE Hash Library) will simply store +both the flow key and the target value. + +.. _figure_efd3: + +.. figure:: img/efd_i3.* + + Table Based Flow Distribution + +As shown in :numref:`figure_efd3`, when doing a lookup, the flow-table +is indexed with the hash of the flow key and the keys (more than one is possible, +because of hash collision) stored in this index and corresponding values +are retrieved. The retrieved key(s) is matched with the input flow key +and if there is a match the value (target id) is returned. + +The drawback of using a hash table for flow distribution/load balancing +is the storage requirement, since the flow table need to store keys, +signatures and target values. This doesn't allow this scheme to scale to +millions of flow keys. Large tables will usually not fit in +the CPU cache, and hence, the lookup performance is degraded because of +the latency to access the main memory. + +EFD Based Scheme +~~~~~~~~~~~~~~~~ + +EFD combines the advantages of both flow-table based and computation-based +schemes. It doesn't require the large storage necessary for +flow-table based schemes (because EFD doesn't store the key as explained +below), and it supports any arbitrary value for any given key. + +.. _figure_efd4: + +.. figure:: img/efd_i4.* + + Searching for Perfect Hash Function + +The basic idea of EFD is when a given key is to be inserted, a family of +hash functions is searched until the correct hash function that maps the +input key to the correct value is found, as shown in :numref:`figure_efd4`. +However, rather than explicitly storing all keys and their associated values, +EFD stores only indices of hash functions that map keys to values, and +thereby consumes much less space than conventional flow-based tables. +The lookup operation is very simple, similar to a computational-based +scheme: given an input key the lookup operation is reduced to hashing +that key with the correct hash function. + +.. _figure_efd5: + +.. figure:: img/efd_i5.* + + Divide and Conquer for Millions of Keys + +Intuitively, finding a hash function that maps each of a large number +(millions) of input keys to the correct output value is effectively +impossible, as a result EFD, as shown in :numref:`figure_efd5`, +breaks the problem into smaller pieces (divide and conquer). +EFD divides the entire input key set into many small groups. +Each group consists of approximately 20-28 keys (a configurable parameter +for the library), then, for each small group, a brute force search to find +a hash function that produces the correct outputs for each key in the group. + +It should be mentioned that, since the online lookup table for EFD +doesn't store the key itself, the size of the EFD table is independent +of the key size and hence EFD lookup performance which is almost +constant irrespective of the length of the key which is a highly +desirable feature especially for longer keys. + +In summary, EFD is a set separation data structure that supports millions of +keys. It is used to distribute a given key to an intended target. By itself +EFD is not a FIB data structure with an exact match the input flow key. + +.. _Efd_example: + +Example of EFD Library Usage +---------------------------- + +EFD can be used along the data path of many network functions and middleboxes. +As previously mentioned, it can used as an index table for + pairs, meta-data for objects, a flow-level load balancer, etc. +:numref:`figure_efd6` shows an example of using EFD as a flow-level load +balancer, where flows are received at a front end server before being forwarded +to the target back end server for processing. The system designer would +deterministically co-locate flows together in order to minimize cross-server +interaction. +(For example, flows requesting certain webpage objects are co-located +together, to minimize forwarding of common objects across servers). + +.. _figure_efd6: + +.. figure:: img/efd_i6.* + + EFD as a Flow-Level Load Balancer + +As shown in :numref:`figure_efd6`, the front end server will have an EFD table that +stores for each group what is the perfect hash index that satisfies the +correct output. Because the table size is small and fits in cache (since +keys are not stored), it sustains a large number of flows (N*X, where N +is the maximum number of flows served by each back end server of the X +possible targets). + +With an input flow key, the group id is computed (for example, using +last few bits of CRC hash) and then the EFD table is indexed with the +group id to retrieve the corresponding hash index to use. Once the index +is retrieved the key is hashed using this hash function and the result +will be the intended correct target where this flow is supposed to be +processed. + +It should be noted that as a result of EFD not matching the exact key but +rather distributing the flows to a target back end node based on the +perfect hash index, a key that has not been inserted before +will be distributed to a valid target. Hence, a local table which stores +the flows served at each node is used and is +exact matched with the input key to rule out new never seen before +flows. + +.. _Efd_api: + +Library API Overview +-------------------- + +The EFD library API is created with a very similar semantics of a +hash-index or a flow table. The application creates an EFD table for a +given maximum number of flows, a function is called to insert a flow key +with a specific target value, and another function is used to retrieve +target values for a given individual flow key or a bulk of keys. + +EFD Table Create +~~~~~~~~~~~~~~~~ + +The function ``rte_efd_create()`` is used to create and return a pointer +to an EFD table that is sized to hold up to num_flows key. +The online version of the EFD table (the one that does +not store the keys and is used for lookups) will be allocated and +created in the last level cache (LLC) of the socket defined by the +online_socket_bitmask, while the offline EFD table (the one that +stores the keys and is used for key inserts and for computing the +perfect hashing) is allocated and created in the LLC of the socket +defined by offline_socket_bitmask. It should be noted, that for +highest performance the socket id should match that where the thread is +running, i.e. the online EFD lookup table should be created on the same +socket as where the lookup thread is running. + +EFD Insert and Update +~~~~~~~~~~~~~~~~~~~~~ + +The EFD function to insert a key or update a key to a new value is +``rte_efd_update()``. This function will update an existing key to +a new value (target) if the key has already been inserted +before, or will insert the pair if this key has not been inserted +before. It will return 0 upon success. It will return +``EFD_UPDATE_WARN_GROUP_FULL (1)`` if the operation is insert, and the +last available space in the key's group was just used. It will return +``EFD_UPDATE_FAILED (2)`` when the insertion or update has failed (either it +failed to find a suitable perfect hash or the group was full). The function +will return ``EFD_UPDATE_NO_CHANGE (3)`` if there is no change to the EFD +table (i.e, same value already exists). + +.. Note:: + + This function is not multi-thread safe and should only be called + from one thread. + +EFD Lookup +~~~~~~~~~~ + +To lookup a certain key in an EFD table, the function ``rte_efd_lookup()`` +is used to return the value associated with single key. +As previously mentioned, if the key has been inserted, the correct value +inserted is returned, if the key has not been inserted before, +a ‘random’ value (based on hashing of the key) is returned. +For better performance and to decrease the overhead of +function calls per key, it is always recommended to use a bulk lookup +function (simultaneous lookup of multiple keys) instead of a single key +lookup function. ``rte_efd_lookup_bulk()`` is the bulk lookup function, +that looks up num_keys simultaneously stored in the key_list and the +corresponding return values will be returned in the value_list. + +.. Note:: + + This function is multi-thread safe, but there should not be other threads + writing in the EFD table, unless locks are used. + +EFD Delete +~~~~~~~~~~ + +To delete a certain key in an EFD table, the function +``rte_efd_delete()`` can be used. The function returns zero upon success +when the key has been found and deleted. Socket_id is the parameter to +use to lookup the existing value, which is ideally the caller's socket id. +The previous value associated with this key will be returned +in the prev_value argument. + +.. Note:: + + This function is not multi-thread safe and should only be called + from one thread. + +.. _Efd_internals: + +Library Internals +----------------- + +This section provides the brief high-level idea and an overview +of the library internals to accompany the RFC. The intent of this +section is to explain to readers the high-level implementation of +insert, lookup and group rebalancing in the EFD library. + +Insert Function Internals +~~~~~~~~~~~~~~~~~~~~~~~~~ + +As previously mentioned the EFD divides the whole set of keys into +groups of a manageable size (e.g. 28 keys) and then searches for the +perfect hash that satisfies the intended target value for each key. EFD +stores two version of the table: + +- Offline Version (in memory): Only used for the insertion/update + operation, which is less frequent than the lookup operation. In the + offline version the exact keys for each group is stored. When a new + key is added, the hash function is updated that will satisfy the + value for the new key together with the all old keys already inserted + in this group. + +- Online Version (in cache): Used for the frequent lookup operation. In + the online version, as previously mentioned, the keys are not stored + but rather only the hash index for each group. + +.. _figure_efd7: + +.. figure:: img/efd_i7.* + + Group Assignment + +:numref:`figure_efd7` depicts the group assignment for 7 flow keys as an example. +Given a flow key, a hash function (in our implementation CRC hash) is +used to get the group id. As shown in the figure, the groups can be +unbalanced. (We highlight group rebalancing further below). + +.. _figure_efd8: + +.. figure:: img/efd_i8.* + + Perfect Hash Search - Assigned Keys & Target Value + +Focusing on one group that has four keys, :numref:`figure_efd8` depicts the search +algorithm to find the perfect hash function. Assuming that the target +value bit for the keys is as shown in the figure, then the online EFD +table will store a 16 bit hash index and 16 bit lookup table per group +per value bit. + +.. _figure_efd9: + +.. figure:: img/efd_i9.* + + Perfect Hash Search - Satisfy Target Values + +For a given keyX, a hash function ``(h(keyX, seed1) + index * h(keyX, seed2))`` +is used to point to certain bit index in the 16bit lookup_table value, +as shown in :numref:`figure_efd9`. +The insert function will brute force search for all possible values for the +hash index until a non conflicting lookup_table is found. + +.. _figure_efd10: + +.. figure:: img/efd_i10.* + + Finding Hash Index for Conflict Free lookup_table + +For example, since both key3 and key7 have a target bit value of 1, it +is okay if the hash function of both keys point to the same bit in the +lookup table. A conflict will occur if a hash index is used that maps +both Key4 and Key7 to the same index in the lookup_table, +as shown in :numref:`figure_efd10`, since their target value bit are not the same. +Once a hash index is found that produces a lookup_table with no +contradictions, this index is stored for this group. This procedure is +repeated for each bit of target value. + +Lookup Function Internals +~~~~~~~~~~~~~~~~~~~~~~~~~ + +The design principle of EFD is that lookups are much more frequent than +inserts, and hence, EFD's design optimizes for the lookups which are +faster and much simpler than the slower insert procedure (inserts are +slow, because of perfect hash search as previously discussed). + +.. _figure_efd11: + +.. figure:: img/efd_i11.* + + EFD Lookup Operation + +:numref:`figure_efd11` depicts the lookup operation for EFD. Given an input key, +the group id is computed (using CRC hash) and then the hash index for this +group is retrieved from the EFD table. Using the retrieved hash index, +the hash function ``h(key, seed1) + index *h(key, seed2)`` is used which will +result in an index in the lookup_table, the bit corresponding to this +index will be the target value bit. This procedure is repeated for each +bit of the target value. + +Group Rebalancing Function Internals +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When discussing EFD inserts and lookups, the discussion is simplified by +assuming that a group id is simply a result of hash function. However, +since hashing in general is not perfect and will not always produce a +uniform output, this simplified assumption will lead to unbalanced +groups, i.e., some group will have more keys than other groups. +Typically, and to minimize insert time with an increasing number of keys, +it is preferable that all groups will have a balanced number of keys, so +the brute force search for the perfect hash terminates with a valid hash +index. In order to achieve this target, groups are rebalanced during +runtime inserts, and keys are moved around from a busy group to a less +crowded group as the more keys are inserted. + +.. _figure_efd12: + +.. figure:: img/efd_i12.* + + Runtime Group Rebalancing + +:numref:`figure_efd12` depicts the high level idea of group rebalancing, given an +input key the hash result is split into two parts a chunk id and 8-bit +bin id. A chunk contains 64 different groups and 256 bins (i.e. for any +given bin it can map to 4 distinct groups). When a key is inserted, the +bin id is computed, for example in :numref:`figure_efd12` bin_id=2, +and since each bin can be mapped to one of four different groups (2 bit storage), +the four possible mappings are evaluated and the one that will result in a +balanced key distribution across these four is selected the mapping result +is stored in these two bits. + + +.. _Efd_references: + +References +----------- + +1- EFD is based on collaborative research work between Intel and +Carnegie Mellon University (CMU), interested readers can refer to the paper +“Scaling Up Clustered Network Appliances with ScaleBricks;” Dong Zhou et al. +at SIGCOMM 2015 (`http://conferences.sigcomm.org/sigcomm/2015/pdf/papers/p241.pdf`) +for more information. diff --git a/dpdk/doc/guides/prog_guide/env_abstraction_layer.rst b/dpdk/doc/guides/prog_guide/env_abstraction_layer.rst index 10a10a88..34d871c9 100644 --- a/dpdk/doc/guides/prog_guide/env_abstraction_layer.rst +++ b/dpdk/doc/guides/prog_guide/env_abstraction_layer.rst @@ -117,17 +117,6 @@ The physical address of the reserved memory for that memory zone is also returne Memory reservations done using the APIs provided by rte_malloc are also backed by pages from the hugetlbfs filesystem. -Xen Dom0 support without hugetbls -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The existing memory management implementation is based on the Linux kernel hugepage mechanism. -However, Xen Dom0 does not support hugepages, so a new Linux kernel module rte_dom0_mm is added to workaround this limitation. - -The EAL uses IOCTL interface to notify the Linux kernel module rte_dom0_mm to allocate memory of specified size, -and get all memory segments information from the module, -and the EAL uses MMAP interface to map the allocated memory. -For each memory segment, the physical addresses are contiguous within it but actual hardware addresses are contiguous within 2MB. - PCI Access ~~~~~~~~~~ @@ -164,7 +153,7 @@ which can trigger the generation of a core file, readable by gdb. CPU Feature Identification ~~~~~~~~~~~~~~~~~~~~~~~~~~ -The EAL can query the CPU at runtime (using the rte_cpu_get_feature() function) to determine which CPU features are available. +The EAL can query the CPU at runtime (using the rte_cpu_get_features() function) to determine which CPU features are available. User Space Interrupt Event ~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -178,8 +167,8 @@ The EAL also allows timed callbacks to be used in the same way as for NIC interr .. note:: - In DPDK PMD, the only interrupts handled by the dedicated host thread are those for link status change, - i.e. link up and link down notification. + In DPDK PMD, the only interrupts handled by the dedicated host thread are those for link status change + (link up and link down notification) and for sudden device removal. + RX Interrupt Event @@ -207,6 +196,23 @@ The eth_dev driver takes responsibility to program the latter mapping. The RX interrupt are controlled/enabled/disabled by ethdev APIs - 'rte_eth_dev_rx_intr_*'. They return failure if the PMD hasn't support them yet. The intr_conf.rxq flag is used to turn on the capability of RX interrupt per device. ++ Device Removal Event + +This event is triggered by a device being removed at a bus level. Its +underlying resources may have been made unavailable (i.e. PCI mappings +unmapped). The PMD must make sure that on such occurrence, the application can +still safely use its callbacks. + +This event can be subscribed to in the same way one would subscribe to a link +status change event. The execution context is thus the same, i.e. it is the +dedicated interrupt host thread. + +Considering this, it is likely that an application would want to close a +device having emitted a Device Removal Event. In such case, calling +``rte_eth_dev_close()`` can trigger it to unregister its own Device Removal Event +callback. Care must be taken not to close the device from the interrupt handler +context. It is necessary to reschedule such closing operation. + Blacklisting ~~~~~~~~~~~~ @@ -303,7 +309,7 @@ All these impacts are mentioned in :ref:`known_issue_label` section. Public Thread API ~~~~~~~~~~~~~~~~~ -There are two public APIs ``rte_thread_set_affinity()`` and ``rte_pthread_get_affinity()`` introduced for threads. +There are two public APIs ``rte_thread_set_affinity()`` and ``rte_thread_get_affinity()`` introduced for threads. When they're used in any pthread context, the Thread Local Storage(TLS) will be set/get. Those TLS include *_cpuset* and *_socket_id*: @@ -352,11 +358,6 @@ Known Issues 3. It MUST not be used by multi-producer/consumer pthreads, whose scheduling policies are SCHED_FIFO or SCHED_RR. - ``RTE_RING_PAUSE_REP_COUNT`` is defined for rte_ring to reduce contention. It's mainly for case 2, a yield is issued after number of times pause repeat. - - It adds a sched_yield() syscall if the thread spins for too long while waiting on the other thread to finish its operations on the ring. - This gives the preempted thread a chance to proceed and finish with the ring enqueue/dequeue operation. - + rte_timer Running ``rte_timer_manager()`` on a non-EAL pthread is not allowed. However, resetting/stopping the timer from a non-EAL pthread is allowed. diff --git a/dpdk/doc/guides/prog_guide/event_ethernet_rx_adapter.rst b/dpdk/doc/guides/prog_guide/event_ethernet_rx_adapter.rst new file mode 100644 index 00000000..4e722219 --- /dev/null +++ b/dpdk/doc/guides/prog_guide/event_ethernet_rx_adapter.rst @@ -0,0 +1,168 @@ +.. BSD LICENSE + Copyright(c) 2017 Intel Corporation. All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Event Ethernet Rx Adapter Library +================================= + +The DPDK Eventdev API allows the application to use an event driven programming +model for packet processing. In this model, the application polls an event +device port for receiving events that reference packets instead of polling Rx +queues of ethdev ports. Packet transfer between ethdev and the event device can +be supported in hardware or require a software thread to receive packets from +the ethdev port using ethdev poll mode APIs and enqueue these as events to the +event device using the eventdev API. Both transfer mechanisms may be present on +the same platform depending on the particular combination of the ethdev and +the event device. + +The Event Ethernet Rx Adapter library is intended for the application code to +configure both transfer mechanisms using a common API. A capability API allows +the eventdev PMD to advertise features supported for a given ethdev and allows +the application to perform configuration as per supported features. + +API Walk-through +---------------- + +This section will introduce the reader to the adapter API. The +application has to first instantiate an adapter which is associated with +a single eventdev, next the adapter instance is configured with Rx queues +that are either polled by a SW thread or linked using hardware support. Finally +the adapter is started. + +For SW based packet transfers from ethdev to eventdev, the adapter uses a +DPDK service function and the application is also required to assign a core to +the service function. + +Creating an Adapter Instance +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +An adapter instance is created using ``rte_event_eth_rx_adapter_create()``. This +function is passed the event device to be associated with the adapter and port +configuration for the adapter to setup an event port if the adapter needs to use +a service function. + +.. code-block:: c + + int err; + uint8_t dev_id; + struct rte_event_dev_info dev_info; + struct rte_event_port_conf rx_p_conf; + + err = rte_event_dev_info_get(id, &dev_info); + + rx_p_conf.new_event_threshold = dev_info.max_num_events; + rx_p_conf.dequeue_depth = dev_info.max_event_port_dequeue_depth; + rx_p_conf.enqueue_depth = dev_info.max_event_port_enqueue_depth; + err = rte_event_eth_rx_adapter_create(id, dev_id, &rx_p_conf); + +If the application desires to have finer control of eventdev port allocation +and setup, it can use the ``rte_event_eth_rx_adapter_create_ext()`` function. +The ``rte_event_eth_rx_adapter_create_ext()`` function is passed a callback +function. The callback function is invoked if the adapter needs to use a +service function and needs to create an event port for it. The callback is +expected to fill the ``struct rte_event_eth_rx_adapter_conf structure`` +passed to it. + +Adding Rx Queues to the Adapter Instance +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Ethdev Rx queues are added to the instance using the +``rte_event_eth_rx_adapter_queue_add()`` function. Configuration for the Rx +queue is passed in using a ``struct rte_event_eth_rx_adapter_queue_conf`` +parameter. Event information for packets from this Rx queue is encoded in the +``ev`` field of ``struct rte_event_eth_rx_adapter_queue_conf``. The +servicing_weight member of the struct rte_event_eth_rx_adapter_queue_conf +is the relative polling frequency of the Rx queue and is applicable when the +adapter uses a service core function. + +.. code-block:: c + + ev.queue_id = 0; + ev.sched_type = RTE_SCHED_TYPE_ATOMIC; + ev.priority = 0; + + queue_config.rx_queue_flags = 0; + queue_config.ev = ev; + queue_config.servicing_weight = 1; + + err = rte_event_eth_rx_adapter_queue_add(id, + eth_dev_id, + 0, &queue_config); + +Querying Adapter Capabilities +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``rte_event_eth_rx_adapter_caps_get()`` function allows +the application to query the adapter capabilities for an eventdev and ethdev +combination. For e.g, if the ``RTE_EVENT_ETH_RX_ADAPTER_CAP_OVERRIDE_FLOW_ID`` +is set, the application can override the adapter generated flow ID in the event +using ``rx_queue_flags`` field in ``struct rte_event_eth_rx_adapter_queue_conf`` +which is passed as a parameter to the ``rte_event_eth_rx_adapter_queue_add()`` +function. + +.. code-block:: c + + err = rte_event_eth_rx_adapter_caps_get(dev_id, eth_dev_id, &cap); + + queue_config.rx_queue_flags = 0; + if (cap & RTE_EVENT_ETH_RX_ADAPTER_CAP_OVERRIDE_FLOW_ID) { + ev.flow_id = 1; + queue_config.rx_queue_flags = + RTE_EVENT_ETH_RX_ADAPTER_QUEUE_FLOW_ID_VALID; + } + +Configuring the Service Function +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If the adapter uses a service function, the application is required to assign +a service core to the service function as show below. + +.. code-block:: c + + uint32_t service_id; + + if (rte_event_eth_rx_adapter_service_id_get(0, &service_id) == 0) + rte_service_map_lcore_set(service_id, RX_CORE_ID); + +Starting the Adapter Instance +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The application calls ``rte_event_eth_rx_adapter_start()`` to start the adapter. +This function calls the start callbacks of the eventdev PMDs for hardware based +eventdev-ethdev connections and ``rte_service_run_state_set()`` to enable the +service function if one exists. + +Getting Adapter Statistics +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``rte_event_eth_rx_adapter_stats_get()`` function reports counters defined +in struct ``rte_event_eth_rx_adapter_stats``. The received packet and +enqueued event counts are a sum of the counts from the eventdev PMD callbacks +if the callback is supported, and the counts maintained by the service function, +if one exists. The service function also maintains a count of cycles for which +it was not able to enqueue to the event device. diff --git a/dpdk/doc/guides/prog_guide/eventdev.rst b/dpdk/doc/guides/prog_guide/eventdev.rst new file mode 100644 index 00000000..be9fccdd --- /dev/null +++ b/dpdk/doc/guides/prog_guide/eventdev.rst @@ -0,0 +1,394 @@ +.. BSD LICENSE + Copyright(c) 2017 Intel Corporation. All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Event Device Library +==================== + +The DPDK Event device library is an abstraction that provides the application +with features to schedule events. This is achieved using the PMD architecture +similar to the ethdev or cryptodev APIs, which may already be familiar to the +reader. + +The eventdev framework introduces the event driven programming model. In a +polling model, lcores poll ethdev ports and associated Rx queues directly +to look for a packet. By contrast in an event driven model, lcores call the +scheduler that selects packets for them based on programmer-specified criteria. +The Eventdev library adds support for an event driven programming model, which +offers applications automatic multicore scaling, dynamic load balancing, +pipelining, packet ingress order maintenance and synchronization services to +simplify application packet processing. + +By introducing an event driven programming model, DPDK can support both polling +and event driven programming models for packet processing, and applications are +free to choose whatever model (or combination of the two) best suits their +needs. + +Step-by-step instructions of the eventdev design is available in the `API +Walk-through`_ section later in this document. + +Event struct +------------ + +The eventdev API represents each event with a generic struct, which contains a +payload and metadata required for scheduling by an eventdev. The +``rte_event`` struct is a 16 byte C structure, defined in +``libs/librte_eventdev/rte_eventdev.h``. + +Event Metadata +~~~~~~~~~~~~~~ + +The rte_event structure contains the following metadata fields, which the +application fills in to have the event scheduled as required: + +* ``flow_id`` - The targeted flow identifier for the enq/deq operation. +* ``event_type`` - The source of this event, eg RTE_EVENT_TYPE_ETHDEV or CPU. +* ``sub_event_type`` - Distinguishes events inside the application, that have + the same event_type (see above) +* ``op`` - This field takes one of the RTE_EVENT_OP_* values, and tells the + eventdev about the status of the event - valid values are NEW, FORWARD or + RELEASE. +* ``sched_type`` - Represents the type of scheduling that should be performed + on this event, valid values are the RTE_SCHED_TYPE_ORDERED, ATOMIC and + PARALLEL. +* ``queue_id`` - The identifier for the event queue that the event is sent to. +* ``priority`` - The priority of this event, see RTE_EVENT_DEV_PRIORITY. + +Event Payload +~~~~~~~~~~~~~ + +The rte_event struct contains a union for payload, allowing flexibility in what +the actual event being scheduled is. The payload is a union of the following: + +* ``uint64_t u64`` +* ``void *event_ptr`` +* ``struct rte_mbuf *mbuf`` + +These three items in a union occupy the same 64 bits at the end of the rte_event +structure. The application can utilize the 64 bits directly by accessing the +u64 variable, while the event_ptr and mbuf are provided as convenience +variables. For example the mbuf pointer in the union can used to schedule a +DPDK packet. + +Queues +~~~~~~ + +An event queue is a queue containing events that are scheduled by the event +device. An event queue contains events of different flows associated with +scheduling types, such as atomic, ordered, or parallel. + +Queue All Types Capable +^^^^^^^^^^^^^^^^^^^^^^^ + +If RTE_EVENT_DEV_CAP_QUEUE_ALL_TYPES capability bit is set in the event device, +then events of any type may be sent to any queue. Otherwise, the queues only +support events of the type that it was created with. + +Queue All Types Incapable +^^^^^^^^^^^^^^^^^^^^^^^^^ + +In this case, each stage has a specified scheduling type. The application +configures each queue for a specific type of scheduling, and just enqueues all +events to the eventdev. An example of a PMD of this type is the eventdev +software PMD. + +The Eventdev API supports the following scheduling types per queue: + +* Atomic +* Ordered +* Parallel + +Atomic, Ordered and Parallel are load-balanced scheduling types: the output +of the queue can be spread out over multiple CPU cores. + +Atomic scheduling on a queue ensures that a single flow is not present on two +different CPU cores at the same time. Ordered allows sending all flows to any +core, but the scheduler must ensure that on egress the packets are returned to +ingress order on downstream queue enqueue. Parallel allows sending all flows +to all CPU cores, without any re-ordering guarantees. + +Single Link Flag +^^^^^^^^^^^^^^^^ + +There is a SINGLE_LINK flag which allows an application to indicate that only +one port will be connected to a queue. Queues configured with the single-link +flag follow a FIFO like structure, maintaining ordering but it is only capable +of being linked to a single port (see below for port and queue linking details). + + +Ports +~~~~~ + +Ports are the points of contact between worker cores and the eventdev. The +general use-case will see one CPU core using one port to enqueue and dequeue +events from an eventdev. Ports are linked to queues in order to retrieve events +from those queues (more details in `Linking Queues and Ports`_ below). + + +API Walk-through +---------------- + +This section will introduce the reader to the eventdev API, showing how to +create and configure an eventdev and use it for a two-stage atomic pipeline +with a single core for TX. The diagram below shows the final state of the +application after this walk-through: + +.. _figure_eventdev-usage1: + +.. figure:: img/eventdev_usage.* + + Sample eventdev usage, with RX, two atomic stages and a single-link to TX. + + +A high level overview of the setup steps are: + +* rte_event_dev_configure() +* rte_event_queue_setup() +* rte_event_port_setup() +* rte_event_port_link() +* rte_event_dev_start() + + +Init and Config +~~~~~~~~~~~~~~~ + +The eventdev library uses vdev options to add devices to the DPDK application. +The ``--vdev`` EAL option allows adding eventdev instances to your DPDK +application, using the name of the eventdev PMD as an argument. + +For example, to create an instance of the software eventdev scheduler, the +following vdev arguments should be provided to the application EAL command line: + +.. code-block:: console + + ./dpdk_application --vdev="event_sw0" + +In the following code, we configure eventdev instance with 3 queues +and 6 ports as follows. The 3 queues consist of 2 Atomic and 1 Single-Link, +while the 6 ports consist of 4 workers, 1 RX and 1 TX. + +.. code-block:: c + + const struct rte_event_dev_config config = { + .nb_event_queues = 3, + .nb_event_ports = 6, + .nb_events_limit = 4096, + .nb_event_queue_flows = 1024, + .nb_event_port_dequeue_depth = 128, + .nb_event_port_enqueue_depth = 128, + }; + int err = rte_event_dev_configure(dev_id, &config); + +The remainder of this walk-through assumes that dev_id is 0. + +Setting up Queues +~~~~~~~~~~~~~~~~~ + +Once the eventdev itself is configured, the next step is to configure queues. +This is done by setting the appropriate values in a queue_conf structure, and +calling the setup function. Repeat this step for each queue, starting from +0 and ending at ``nb_event_queues - 1`` from the event_dev config above. + +.. code-block:: c + + struct rte_event_queue_conf atomic_conf = { + .schedule_type = RTE_SCHED_TYPE_ATOMIC, + .priority = RTE_EVENT_DEV_PRIORITY_NORMAL, + .nb_atomic_flows = 1024, + .nb_atomic_order_sequences = 1024, + }; + int dev_id = 0; + int queue_id = 0; + int err = rte_event_queue_setup(dev_id, queue_id, &atomic_conf); + +The remainder of this walk-through assumes that the queues are configured as +follows: + + * id 0, atomic queue #1 + * id 1, atomic queue #2 + * id 2, single-link queue + +Setting up Ports +~~~~~~~~~~~~~~~~ + +Once queues are set up successfully, create the ports as required. Each port +should be set up with its corresponding port_conf type, worker for worker cores, +rx and tx for the RX and TX cores: + +.. code-block:: c + + struct rte_event_port_conf rx_conf = { + .dequeue_depth = 128, + .enqueue_depth = 128, + .new_event_threshold = 1024, + }; + struct rte_event_port_conf worker_conf = { + .dequeue_depth = 16, + .enqueue_depth = 64, + .new_event_threshold = 4096, + }; + struct rte_event_port_conf tx_conf = { + .dequeue_depth = 128, + .enqueue_depth = 128, + .new_event_threshold = 4096, + }; + int dev_id = 0; + int port_id = 0; + int err = rte_event_port_setup(dev_id, port_id, &CORE_FUNCTION_conf); + +It is now assumed that: + + * port 0: RX core + * ports 1,2,3,4: Workers + * port 5: TX core + +Linking Queues and Ports +~~~~~~~~~~~~~~~~~~~~~~~~ + +The final step is to "wire up" the ports to the queues. After this, the +eventdev is capable of scheduling events, and when cores request work to do, +the correct events are provided to that core. Note that the RX core takes input +from eg: a NIC so it is not linked to any eventdev queues. + +Linking all workers to atomic queues, and the TX core to the single-link queue +can be achieved like this: + +.. code-block:: c + + uint8_t port_id = 0; + uint8_t atomic_qs[] = {0, 1}; + uint8_t single_link_q = 2; + uint8_t tx_port_id = 5; + uin8t_t priority = RTE_EVENT_DEV_PRIORITY_NORMAL; + + for(int i = 0; i < 4; i++) { + int worker_port = i + 1; + int links_made = rte_event_port_link(dev_id, worker_port, atomic_qs, NULL, 2); + } + int links_made = rte_event_port_link(dev_id, tx_port_id, &single_link_q, &priority, 1); + +Starting the EventDev +~~~~~~~~~~~~~~~~~~~~~ + +A single function call tells the eventdev instance to start processing +events. Note that all queues must be linked to for the instance to start, as +if any queue is not linked to, enqueuing to that queue will cause the +application to backpressure and eventually stall due to no space in the +eventdev. + +.. code-block:: c + + int err = rte_event_dev_start(dev_id); + +Ingress of New Events +~~~~~~~~~~~~~~~~~~~~~ + +Now that the eventdev is set up, and ready to receive events, the RX core must +enqueue some events into the system for it to schedule. The events to be +scheduled are ordinary DPDK packets, received from an eth_rx_burst() as normal. +The following code shows how those packets can be enqueued into the eventdev: + +.. code-block:: c + + const uint16_t nb_rx = rte_eth_rx_burst(eth_port, 0, mbufs, BATCH_SIZE); + + for (i = 0; i < nb_rx; i++) { + ev[i].flow_id = mbufs[i]->hash.rss; + ev[i].op = RTE_EVENT_OP_NEW; + ev[i].sched_type = RTE_SCHED_TYPE_ATOMIC; + ev[i].queue_id = 0; + ev[i].event_type = RTE_EVENT_TYPE_ETHDEV; + ev[i].sub_event_type = 0; + ev[i].priority = RTE_EVENT_DEV_PRIORITY_NORMAL; + ev[i].mbuf = mbufs[i]; + } + + const int nb_tx = rte_event_enqueue_burst(dev_id, port_id, ev, nb_rx); + if (nb_tx != nb_rx) { + for(i = nb_tx; i < nb_rx; i++) + rte_pktmbuf_free(mbufs[i]); + } + +Forwarding of Events +~~~~~~~~~~~~~~~~~~~~ + +Now that the RX core has injected events, there is work to be done by the +workers. Note that each worker will dequeue as many events as it can in a burst, +process each one individually, and then burst the packets back into the +eventdev. + +The worker can lookup the events source from ``event.queue_id``, which should +indicate to the worker what workload needs to be performed on the event. +Once done, the worker can update the ``event.queue_id`` to a new value, to send +the event to the next stage in the pipeline. + +.. code-block:: c + + int timeout = 0; + struct rte_event events[BATCH_SIZE]; + uint16_t nb_rx = rte_event_dequeue_burst(dev_id, worker_port_id, events, BATCH_SIZE, timeout); + + for (i = 0; i < nb_rx; i++) { + /* process mbuf using events[i].queue_id as pipeline stage */ + struct rte_mbuf *mbuf = events[i].mbuf; + /* Send event to next stage in pipeline */ + events[i].queue_id++; + } + + uint16_t nb_tx = rte_event_enqueue_burst(dev_id, port_id, events, nb_rx); + + +Egress of Events +~~~~~~~~~~~~~~~~ + +Finally, when the packet is ready for egress or needs to be dropped, we need +to inform the eventdev that the packet is no longer being handled by the +application. This can be done by calling dequeue() or dequeue_burst(), which +indicates that the previous burst of packets is no longer in use by the +application. + +An event driven worker thread has following typical workflow on fastpath: + +.. code-block:: c + + while (1) { + rte_event_dequeue_burst(...); + (event processing) + rte_event_enqueue_burst(...); + } + + +Summary +------- + +The eventdev library allows an application to easily schedule events as it +requires, either using a run-to-completion or pipeline processing model. The +queues and ports abstract the logical functionality of an eventdev, providing +the application with a generic method to schedule events. With the flexible +PMD infrastructure applications benefit of improvements in existing eventdevs +and additions of new ones without modification. diff --git a/dpdk/doc/guides/prog_guide/extend_dpdk.rst b/dpdk/doc/guides/prog_guide/extend_dpdk.rst index 51f0b5c2..73d81999 100644 --- a/dpdk/doc/guides/prog_guide/extend_dpdk.rst +++ b/dpdk/doc/guides/prog_guide/extend_dpdk.rst @@ -39,14 +39,14 @@ Example: Adding a New Library libfoo To add a new library to the DPDK, proceed as follows: -#. Add a new configuration option: +#. Add a new configuration option: .. code-block:: bash for f in config/\*; do \ echo CONFIG_RTE_LIBFOO=y >> $f; done -#. Create a new directory with sources: +#. Create a new directory with sources: .. code-block:: console @@ -54,7 +54,7 @@ To add a new library to the DPDK, proceed as follows: touch ${RTE_SDK}/lib/libfoo/foo.c touch ${RTE_SDK}/lib/libfoo/foo.h -#. Add a foo() function in libfoo. +#. Add a foo() function in libfoo. Definition is in foo.c: @@ -71,7 +71,7 @@ To add a new library to the DPDK, proceed as follows: extern void foo(void); -#. Update lib/Makefile: +#. Update lib/Makefile: .. code-block:: console @@ -79,7 +79,7 @@ To add a new library to the DPDK, proceed as follows: # add: # DIRS-$(CONFIG_RTE_LIBFOO) += libfoo -#. Create a new Makefile for this library, for example, derived from mempool Makefile: +#. Create a new Makefile for this library, for example, derived from mempool Makefile: .. code-block:: console @@ -91,11 +91,11 @@ To add a new library to the DPDK, proceed as follows: # rte_mempool -> foo -#. Update mk/DPDK.app.mk, and add -lfoo in LDLIBS variable when the option is enabled. - This will automatically add this flag when linking a DPDK application. +#. Update mk/DPDK.app.mk, and add -lfoo in LDLIBS variable when the option is enabled. + This will automatically add this flag when linking a DPDK application. -#. Build the DPDK with the new library (we only show a specific target here): +#. Build the DPDK with the new library (we only show a specific target here): .. code-block:: console @@ -104,7 +104,7 @@ To add a new library to the DPDK, proceed as follows: make -#. Check that the library is installed: +#. Check that the library is installed: .. code-block:: console diff --git a/dpdk/doc/guides/prog_guide/flow_classify_lib.rst b/dpdk/doc/guides/prog_guide/flow_classify_lib.rst new file mode 100644 index 00000000..820dc72a --- /dev/null +++ b/dpdk/doc/guides/prog_guide/flow_classify_lib.rst @@ -0,0 +1,427 @@ +.. BSD LICENSE + Copyright(c) 2017 Intel Corporation. All rights reserved. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Flow Classification Library +=========================== + +DPDK provides a Flow Classification library that provides the ability +to classify an input packet by matching it against a set of Flow rules. + +The initial implementation supports counting of IPv4 5-tuple packets which match +a particular Flow rule only. + +Please refer to the +:doc:`./rte_flow` +for more information. + +The Flow Classification library uses the ``librte_table`` API for managing Flow +rules and matching packets against the Flow rules. +The library is table agnostic and can use the following tables: +``Access Control List``, ``Hash`` and ``Longest Prefix Match(LPM)``. +The ``Access Control List`` table is used in the initial implementation. + +Please refer to the +:doc:`./packet_framework` +for more information.on ``librte_table``. + +DPDK provides an Access Control List library that provides the ability to +classify an input packet based on a set of classification rules. + +Please refer to the +:doc:`./packet_classif_access_ctrl` +library for more information on ``librte_acl``. + +There is also a Flow Classify sample application which demonstrates the use of +the Flow Classification Library API's. + +Please refer to the +:doc:`../sample_app_ug/flow_classify` +for more information on the ``flow_classify`` sample application. + +Overview +-------- + +The library has the following API's + +.. code-block:: c + + /** + * Flow classifier create + * + * @param params + * Parameters for flow classifier creation + * @return + * Handle to flow classifier instance on success or NULL otherwise + */ + struct rte_flow_classifier * + rte_flow_classifier_create(struct rte_flow_classifier_params *params); + + /** + * Flow classifier free + * + * @param cls + * Handle to flow classifier instance + * @return + * 0 on success, error code otherwise + */ + int + rte_flow_classifier_free(struct rte_flow_classifier *cls); + + /** + * Flow classify table create + * + * @param cls + * Handle to flow classifier instance + * @param params + * Parameters for flow_classify table creation + * @param table_id + * Table ID. Valid only within the scope of table IDs of the current + * classifier. Only returned after a successful invocation. + * @return + * 0 on success, error code otherwise + */ + int + rte_flow_classify_table_create(struct rte_flow_classifier *cls, + struct rte_flow_classify_table_params *params, + uint32_t *table_id); + + /** + * Add a flow classify rule to the flow_classifier table. + * + * @param[in] cls + * Flow classifier handle + * @param[in] table_id + * id of table + * @param[in] attr + * Flow rule attributes + * @param[in] pattern + * Pattern specification (list terminated by the END pattern item). + * @param[in] actions + * Associated actions (list terminated by the END pattern item). + * @param[out] error + * Perform verbose error reporting if not NULL. Structure + * initialised in case of error only. + * @return + * A valid handle in case of success, NULL otherwise. + */ + struct rte_flow_classify_rule * + rte_flow_classify_table_entry_add(struct rte_flow_classifier *cls, + uint32_t table_id, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error); + + /** + * Delete a flow classify rule from the flow_classifier table. + * + * @param[in] cls + * Flow classifier handle + * @param[in] table_id + * id of table + * @param[in] rule + * Flow classify rule + * @return + * 0 on success, error code otherwise. + */ + int + rte_flow_classify_table_entry_delete(struct rte_flow_classifier *cls, + uint32_t table_id, + struct rte_flow_classify_rule *rule); + + /** + * Query flow classifier for given rule. + * + * @param[in] cls + * Flow classifier handle + * @param[in] table_id + * id of table + * @param[in] pkts + * Pointer to packets to process + * @param[in] nb_pkts + * Number of packets to process + * @param[in] rule + * Flow classify rule + * @param[in] stats + * Flow classify stats + * + * @return + * 0 on success, error code otherwise. + */ + int + rte_flow_classifier_query(struct rte_flow_classifier *cls, + uint32_t table_id, + struct rte_mbuf **pkts, + const uint16_t nb_pkts, + struct rte_flow_classify_rule *rule, + struct rte_flow_classify_stats *stats); + +Classifier creation +~~~~~~~~~~~~~~~~~~~ + +The application creates the ``Classifier`` using the +``rte_flow_classifier_create`` API. +The ``rte_flow_classify_params`` structure must be initialised by the +application before calling the API. + +.. code-block:: c + + struct rte_flow_classifier_params { + /** flow classifier name */ + const char *name; + + /** CPU socket ID where memory for the flow classifier and its */ + /** elements (tables) should be allocated */ + int socket_id; + + /** Table type */ + enum rte_flow_classify_table_type type; + }; + +The ``Classifier`` has the following internal structures: + +.. code-block:: c + + struct rte_table { + /* Input parameters */ + struct rte_table_ops ops; + uint32_t entry_size; + enum rte_flow_classify_table_type type; + + /* Handle to the low-level table object */ + void *h_table; + }; + + #define RTE_FLOW_CLASSIFIER_MAX_NAME_SZ 256 + + struct rte_flow_classifier { + /* Input parameters */ + char name[RTE_FLOW_CLASSIFIER_MAX_NAME_SZ]; + int socket_id; + enum rte_flow_classify_table_type type; + + /* Internal tables */ + struct rte_table tables[RTE_FLOW_CLASSIFY_TABLE_MAX]; + uint32_t num_tables; + uint16_t nb_pkts; + struct rte_flow_classify_table_entry + *entries[RTE_PORT_IN_BURST_SIZE_MAX]; + } __rte_cache_aligned; + +Adding a table to the Classifier +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The application adds a table to the ``Classifier`` using the +``rte_flow_classify_table_create`` API. +The ``rte_flow_classify_table_params`` structure must be initialised by the +application before calling the API. + +.. code-block:: c + + struct rte_flow_classify_table_params { + /** Table operations (specific to each table type) */ + struct rte_table_ops *ops; + + /** Opaque param to be passed to the table create operation */ + void *arg_create; + + /** Memory size to be reserved per classifier object entry for */ + /** storing meta data */ + uint32_t table_metadata_size; + }; + +To create an ACL table the ``rte_table_acl_params`` structure must be +initialised and assigned to ``arg_create`` in the +``rte_flow_classify_table_params`` structure. + +.. code-block:: c + + struct rte_table_acl_params { + /** Name */ + const char *name; + + /** Maximum number of ACL rules in the table */ + uint32_t n_rules; + + /** Number of fields in the ACL rule specification */ + uint32_t n_rule_fields; + + /** Format specification of the fields of the ACL rule */ + struct rte_acl_field_def field_format[RTE_ACL_MAX_FIELDS]; + }; + +The fields for the ACL rule must also be initialised by the application. + +An ACL table can be added to the ``Classifier`` for each ACL rule, for example +another table could be added for the IPv6 5-tuple rule. + +Flow Parsing +~~~~~~~~~~~~ + +The library currently supports three IPv4 5-tuple flow patterns, for UDP, TCP +and SCTP. + +.. code-block:: c + + /* Pattern for IPv4 5-tuple UDP filter */ + static enum rte_flow_item_type pattern_ntuple_1[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_END, + }; + + /* Pattern for IPv4 5-tuple TCP filter */ + static enum rte_flow_item_type pattern_ntuple_2[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_END, + }; + + /* Pattern for IPv4 5-tuple SCTP filter */ + static enum rte_flow_item_type pattern_ntuple_3[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_END, + }; + +The internal function ``flow_classify_parse_flow`` parses the +IPv4 5-tuple pattern, attributes and actions and returns the 5-tuple data in the +``rte_eth_ntuple_filter`` structure. + +.. code-block:: c + + static int + flow_classify_parse_flow( + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error) + +Adding Flow Rules +~~~~~~~~~~~~~~~~~ + +The ``rte_flow_classify_table_entry_add`` API creates an +``rte_flow_classify`` object which contains the flow_classify id and type, the +action, a union of add and delete keys and a union of rules. +It uses the ``flow_classify_parse_flow`` internal function for parsing the +flow parameters. +The 5-tuple ACL key data is obtained from the ``rte_eth_ntuple_filter`` +structure populated by the ``classify_parse_ntuple_filter`` function which +parses the Flow rule. + +.. code-block:: c + + struct acl_keys { + struct rte_table_acl_rule_add_params key_add; /* add key */ + struct rte_table_acl_rule_delete_params key_del; /* delete key */ + }; + + struct classify_rules { + enum rte_flow_classify_rule_type type; + union { + struct rte_flow_classify_ipv4_5tuple ipv4_5tuple; + } u; + }; + + struct rte_flow_classify { + uint32_t id; /* unique ID of classify object */ + struct rte_flow_action action; /* action when match found */ + struct classify_rules rules; /* union of rules */ + union { + struct acl_keys key; + } u; + int key_found; /* rule key found in table */ + void *entry; /* pointer to buffer to hold rule meta data */ + void *entry_ptr; /* handle to the table entry for rule meta data */ + }; + +It then calls the ``table[table_id].ops.f_add`` API to add the rule to the ACL +table. + +Deleting Flow Rules +~~~~~~~~~~~~~~~~~~~ + +The ``rte_flow_classify_table_entry_delete`` API calls the +``table[table_id].ops.f_delete`` API to delete a rule from the ACL table. + +Packet Matching +~~~~~~~~~~~~~~~ + +The ``rte_flow_classifier_query`` API is used to find packets which match a +given flow Flow rule in the table. +This API calls the flow_classify_run internal function which calls the +``table[table_id].ops.f_lookup`` API to see if any packets in a burst match any +of the Flow rules in the table. +The meta data for the highest priority rule matched for each packet is returned +in the entries array in the ``rte_flow_classify`` object. +The internal function ``action_apply`` implements the ``Count`` action which is +used to return data which matches a particular Flow rule. + +The rte_flow_classifier_query API uses the following structures to return data +to the application. + +.. code-block:: c + + /** IPv4 5-tuple data */ + struct rte_flow_classify_ipv4_5tuple { + uint32_t dst_ip; /**< Destination IP address in big endian. */ + uint32_t dst_ip_mask; /**< Mask of destination IP address. */ + uint32_t src_ip; /**< Source IP address in big endian. */ + uint32_t src_ip_mask; /**< Mask of destination IP address. */ + uint16_t dst_port; /**< Destination port in big endian. */ + uint16_t dst_port_mask; /**< Mask of destination port. */ + uint16_t src_port; /**< Source Port in big endian. */ + uint16_t src_port_mask; /**< Mask of source port. */ + uint8_t proto; /**< L4 protocol. */ + uint8_t proto_mask; /**< Mask of L4 protocol. */ + }; + + /** + * Flow stats + * + * For the count action, stats can be returned by the query API. + * + * Storage for stats is provided by the application. + * + * + */ + struct rte_flow_classify_stats { + void *stats; + }; + + struct rte_flow_classify_5tuple_stats { + /** count of packets that match IPv4 5tuple pattern */ + uint64_t counter1; + /** IPv4 5tuple data */ + struct rte_flow_classify_ipv4_5tuple ipv4_5tuple; + }; diff --git a/dpdk/doc/guides/prog_guide/generic_receive_offload_lib.rst b/dpdk/doc/guides/prog_guide/generic_receive_offload_lib.rst new file mode 100644 index 00000000..22e50ecb --- /dev/null +++ b/dpdk/doc/guides/prog_guide/generic_receive_offload_lib.rst @@ -0,0 +1,159 @@ +.. BSD LICENSE + Copyright(c) 2017 Intel Corporation. All rights reserved. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Generic Receive Offload Library +=============================== + +Generic Receive Offload (GRO) is a widely used SW-based offloading +technique to reduce per-packet processing overhead. It gains performance +by reassembling small packets into large ones. To enable more flexibility +to applications, DPDK implements GRO as a standalone library. Applications +explicitly use the GRO library to merge small packets into large ones. + +The GRO library assumes all input packets have correct checksums. In +addition, the GRO library doesn't re-calculate checksums for merged +packets. If input packets are IP fragmented, the GRO library assumes +they are complete packets (i.e. with L4 headers). + +Currently, the GRO library implements TCP/IPv4 packet reassembly. + +Reassembly Modes +---------------- + +The GRO library provides two reassembly modes: lightweight and +heavyweight mode. If applications want to merge packets in a simple way, +they can use the lightweight mode API. If applications want more +fine-grained controls, they can choose the heavyweight mode API. + +Lightweight Mode +~~~~~~~~~~~~~~~~ + +The ``rte_gro_reassemble_burst()`` function is used for reassembly in +lightweight mode. It tries to merge N input packets at a time, where +N should be less than or equal to ``RTE_GRO_MAX_BURST_ITEM_NUM``. + +In each invocation, ``rte_gro_reassemble_burst()`` allocates temporary +reassembly tables for the desired GRO types. Note that the reassembly +table is a table structure used to reassemble packets and different GRO +types (e.g. TCP/IPv4 GRO and TCP/IPv6 GRO) have different reassembly table +structures. The ``rte_gro_reassemble_burst()`` function uses the reassembly +tables to merge the N input packets. + +For applications, performing GRO in lightweight mode is simple. They +just need to invoke ``rte_gro_reassemble_burst()``. Applications can get +GROed packets as soon as ``rte_gro_reassemble_burst()`` returns. + +Heavyweight Mode +~~~~~~~~~~~~~~~~ + +The ``rte_gro_reassemble()`` function is used for reassembly in heavyweight +mode. Compared with the lightweight mode, performing GRO in heavyweight mode +is relatively complicated. + +Before performing GRO, applications need to create a GRO context object +by calling ``rte_gro_ctx_create()``. A GRO context object holds the +reassembly tables of desired GRO types. Note that all update/lookup +operations on the context object are not thread safe. So if different +processes or threads want to access the same context object simultaneously, +some external syncing mechanisms must be used. + +Once the GRO context is created, applications can then use the +``rte_gro_reassemble()`` function to merge packets. In each invocation, +``rte_gro_reassemble()`` tries to merge input packets with the packets +in the reassembly tables. If an input packet is an unsupported GRO type, +or other errors happen (e.g. SYN bit is set), ``rte_gro_reassemble()`` +returns the packet to applications. Otherwise, the input packet is either +merged or inserted into a reassembly table. + +When applications want to get GRO processed packets, they need to use +``rte_gro_timeout_flush()`` to flush them from the tables manually. + +TCP/IPv4 GRO +------------ + +TCP/IPv4 GRO supports merging small TCP/IPv4 packets into large ones, +using a table structure called the TCP/IPv4 reassembly table. + +TCP/IPv4 Reassembly Table +~~~~~~~~~~~~~~~~~~~~~~~~~ + +A TCP/IPv4 reassembly table includes a "key" array and an "item" array. +The key array keeps the criteria to merge packets and the item array +keeps the packet information. + +Each key in the key array points to an item group, which consists of +packets which have the same criteria values but can't be merged. A key +in the key array includes two parts: + +* ``criteria``: the criteria to merge packets. If two packets can be + merged, they must have the same criteria values. + +* ``start_index``: the item array index of the first packet in the item + group. + +Each element in the item array keeps the information of a packet. An item +in the item array mainly includes three parts: + +* ``firstseg``: the mbuf address of the first segment of the packet. + +* ``lastseg``: the mbuf address of the last segment of the packet. + +* ``next_pkt_index``: the item array index of the next packet in the same + item group. TCP/IPv4 GRO uses ``next_pkt_index`` to chain the packets + that have the same criteria value but can't be merged together. + +Procedure to Reassemble a Packet +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To reassemble an incoming packet needs three steps: + +#. Check if the packet should be processed. Packets with one of the + following properties aren't processed and are returned immediately: + + * FIN, SYN, RST, URG, PSH, ECE or CWR bit is set. + + * L4 payload length is 0. + +#. Traverse the key array to find a key which has the same criteria + value with the incoming packet. If found, go to the next step. + Otherwise, insert a new key and a new item for the packet. + +#. Locate the first packet in the item group via ``start_index``. Then + traverse all packets in the item group via ``next_pkt_index``. If a + packet is found which can be merged with the incoming one, merge them + together. If one isn't found, insert the packet into this item group. + Note that to merge two packets is to link them together via mbuf's + ``next`` field. + +When packets are flushed from the reassembly table, TCP/IPv4 GRO updates +packet header fields for the merged packets. Note that before reassembling +the packet, TCP/IPv4 GRO doesn't check if the checksums of packets are +correct. Also, TCP/IPv4 GRO doesn't re-calculate checksums for merged +packets. diff --git a/dpdk/doc/guides/prog_guide/generic_segmentation_offload_lib.rst b/dpdk/doc/guides/prog_guide/generic_segmentation_offload_lib.rst new file mode 100644 index 00000000..ef1de53d --- /dev/null +++ b/dpdk/doc/guides/prog_guide/generic_segmentation_offload_lib.rst @@ -0,0 +1,257 @@ +.. BSD LICENSE + Copyright(c) 2017 Intel Corporation. All rights reserved. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Generic Segmentation Offload Library +==================================== + +Overview +-------- +Generic Segmentation Offload (GSO) is a widely used software implementation of +TCP Segmentation Offload (TSO), which reduces per-packet processing overhead. +Much like TSO, GSO gains performance by enabling upper layer applications to +process a smaller number of large packets (e.g. MTU size of 64KB), instead of +processing higher numbers of small packets (e.g. MTU size of 1500B), thus +reducing per-packet overhead. + +For example, GSO allows guest kernel stacks to transmit over-sized TCP segments +that far exceed the kernel interface's MTU; this eliminates the need to segment +packets within the guest, and improves the data-to-overhead ratio of both the +guest-host link, and PCI bus. The expectation of the guest network stack in this +scenario is that segmentation of egress frames will take place either in the NIC +HW, or where that hardware capability is unavailable, either in the host +application, or network stack. + +Bearing that in mind, the GSO library enables DPDK applications to segment +packets in software. Note however, that GSO is implemented as a standalone +library, and not via a 'fallback' mechanism (i.e. for when TSO is unsupported +in the underlying hardware); that is, applications must explicitly invoke the +GSO library to segment packets. The size of GSO segments ``(segsz)`` is +configurable by the application. + +Limitations +----------- + +#. The GSO library doesn't check if input packets have correct checksums. + +#. In addition, the GSO library doesn't re-calculate checksums for segmented + packets (that task is left to the application). + +#. IP fragments are unsupported by the GSO library. + +#. The egress interface's driver must support multi-segment packets. + +#. Currently, the GSO library supports the following IPv4 packet types: + + - TCP + - VxLAN + - GRE + + See `Supported GSO Packet Types`_ for further details. + +Packet Segmentation +------------------- + +The ``rte_gso_segment()`` function is the GSO library's primary +segmentation API. + +Before performing segmentation, an application must create a GSO context object +``(struct rte_gso_ctx)``, which provides the library with some of the +information required to understand how the packet should be segmented. Refer to +`How to Segment a Packet`_ for additional details on same. Once the GSO context +has been created, and populated, the application can then use the +``rte_gso_segment()`` function to segment packets. + +The GSO library typically stores each segment that it creates in two parts: the +first part contains a copy of the original packet's headers, while the second +part contains a pointer to an offset within the original packet. This mechanism +is explained in more detail in `GSO Output Segment Format`_. + +The GSO library supports both single- and multi-segment input mbufs. + +GSO Output Segment Format +~~~~~~~~~~~~~~~~~~~~~~~~~ +To reduce the number of expensive memcpy operations required when segmenting a +packet, the GSO library typically stores each segment that it creates as a +two-part mbuf (technically, this is termed a 'two-segment' mbuf; however, since +the elements produced by the API are also called 'segments', for clarity the +term 'part' is used here instead). + +The first part of each output segment is a direct mbuf and contains a copy of +the original packet's headers, which must be prepended to each output segment. +These headers are copied from the original packet into each output segment. + +The second part of each output segment, represents a section of data from the +original packet, i.e. a data segment. Rather than copy the data directly from +the original packet into the output segment (which would impact performance +considerably), the second part of each output segment is an indirect mbuf, +which contains no actual data, but simply points to an offset within the +original packet. + +The combination of the 'header' segment and the 'data' segment constitutes a +single logical output GSO segment of the original packet. This is illustrated +in :numref:`figure_gso-output-segment-format`. + +.. _figure_gso-output-segment-format: + +.. figure:: img/gso-output-segment-format.* + :align: center + + Two-part GSO output segment + +In one situation, the output segment may contain additional 'data' segments. +This only occurs when: + +- the input packet on which GSO is to be performed is represented by a + multi-segment mbuf. + +- the output segment is required to contain data that spans the boundaries + between segments of the input multi-segment mbuf. + +The GSO library traverses each segment of the input packet, and produces +numerous output segments; for optimal performance, the number of output +segments is kept to a minimum. Consequently, the GSO library maximizes the +amount of data contained within each output segment; i.e. each output segment +``segsz`` bytes of data. The only exception to this is in the case of the very +final output segment; if ``pkt_len`` % ``segsz``, then the final segment is +smaller than the rest. + +In order for an output segment to meet its MSS, it may need to include data from +multiple input segments. Due to the nature of indirect mbufs (each indirect mbuf +can point to only one direct mbuf), the solution here is to add another indirect +mbuf to the output segment; this additional segment then points to the next +input segment. If necessary, this chaining process is repeated, until the sum of +all of the data 'contained' in the output segment reaches ``segsz``. This +ensures that the amount of data contained within each output segment is uniform, +with the possible exception of the last segment, as previously described. + +:numref:`figure_gso-three-seg-mbuf` illustrates an example of a three-part +output segment. In this example, the output segment needs to include data from +the end of one input segment, and the beginning of another. To achieve this, +an additional indirect mbuf is chained to the second part of the output segment, +and is attached to the next input segment (i.e. it points to the data in the +next input segment). + +.. _figure_gso-three-seg-mbuf: + +.. figure:: img/gso-three-seg-mbuf.* + :align: center + + Three-part GSO output segment + +Supported GSO Packet Types +-------------------------- + +TCP/IPv4 GSO +~~~~~~~~~~~~ +TCP/IPv4 GSO supports segmentation of suitably large TCP/IPv4 packets, which +may also contain an optional VLAN tag. + +VxLAN GSO +~~~~~~~~~ +VxLAN packets GSO supports segmentation of suitably large VxLAN packets, +which contain an outer IPv4 header, inner TCP/IPv4 headers, and optional +inner and/or outer VLAN tag(s). + +GRE GSO +~~~~~~~ +GRE GSO supports segmentation of suitably large GRE packets, which contain +an outer IPv4 header, inner TCP/IPv4 headers, and an optional VLAN tag. + +How to Segment a Packet +----------------------- + +To segment an outgoing packet, an application must: + +#. First create a GSO context ``(struct rte_gso_ctx)``; this contains: + + - a pointer to the mbuf pool for allocating the direct buffers, which are + used to store the GSO segments' packet headers. + + - a pointer to the mbuf pool for allocating indirect buffers, which are + used to locate GSO segments' packet payloads. + + .. note:: + + An application may use the same pool for both direct and indirect + buffers. However, since indirect mbufs simply store a pointer, the + application may reduce its memory consumption by creating a separate memory + pool, containing smaller elements, for the indirect pool. + + + - the size of each output segment, including packet headers and payload, + measured in bytes. + + - the bit mask of required GSO types. The GSO library uses the same macros as + those that describe a physical device's TX offloading capabilities (i.e. + ``DEV_TX_OFFLOAD_*_TSO``) for gso_types. For example, if an application + wants to segment TCP/IPv4 packets, it should set gso_types to + ``DEV_TX_OFFLOAD_TCP_TSO``. The only other supported values currently + supported for gso_types are ``DEV_TX_OFFLOAD_VXLAN_TNL_TSO``, and + ``DEV_TX_OFFLOAD_GRE_TNL_TSO``; a combination of these macros is also + allowed. + + - a flag, that indicates whether the IPv4 headers of output segments should + contain fixed or incremental ID values. + +2. Set the appropriate ol_flags in the mbuf. + + - The GSO library use the value of an mbuf's ``ol_flags`` attribute to + to determine how a packet should be segmented. It is the application's + responsibility to ensure that these flags are set. + + - For example, in order to segment TCP/IPv4 packets, the application should + add the ``PKT_TX_IPV4`` and ``PKT_TX_TCP_SEG`` flags to the mbuf's + ol_flags. + + - If checksum calculation in hardware is required, the application should + also add the ``PKT_TX_TCP_CKSUM`` and ``PKT_TX_IP_CKSUM`` flags. + +#. Check if the packet should be processed. Packets with one of the + following properties are not processed and are returned immediately: + + - Packet length is less than ``segsz`` (i.e. GSO is not required). + + - Packet type is not supported by GSO library (see + `Supported GSO Packet Types`_). + + - Application has not enabled GSO support for the packet type. + + - Packet's ol_flags have been incorrectly set. + +#. Allocate space in which to store the output GSO segments. If the amount of + space allocated by the application is insufficient, segmentation will fail. + +#. Invoke the GSO segmentation API, ``rte_gso_segment()``. + +#. If required, update the L3 and L4 checksums of the newly-created segments. + For tunneled packets, the outer IPv4 headers' checksums should also be + updated. Alternatively, the application may offload checksum calculation + to HW. + diff --git a/dpdk/doc/guides/prog_guide/img/crypto_xform_chain.svg b/dpdk/doc/guides/prog_guide/img/crypto_xform_chain.svg index 4670a07e..13681631 100644 --- a/dpdk/doc/guides/prog_guide/img/crypto_xform_chain.svg +++ b/dpdk/doc/guides/prog_guide/img/crypto_xform_chain.svg @@ -69,7 +69,9 @@ class="st3">auth_xform struct rte_crypto_cipher_xform + class="st3">xformstruct rte_crypto_aead_xform Rounded Rectangle.26 next transform (struct rte_crypto_sym_xform *) @@ -116,7 +118,9 @@ class="st3">auth_xform struct rte_crypto_cipher_xform + class="st3">xformstruct rte_crypto_aead_xform Rounded Rectangle.32 next transform (struct rte_crypto_sym_xform *) diff --git a/dpdk/doc/guides/prog_guide/img/cryptodev_sym_sess.svg b/dpdk/doc/guides/prog_guide/img/cryptodev_sym_sess.svg index e5f41ec5..a807ceba 100644 --- a/dpdk/doc/guides/prog_guide/img/cryptodev_sym_sess.svg +++ b/dpdk/doc/guides/prog_guide/img/cryptodev_sym_sess.svg @@ -1,10 +1,50 @@ - - - diff --git a/dpdk/doc/guides/prog_guide/img/efd_i1.svg b/dpdk/doc/guides/prog_guide/img/efd_i1.svg new file mode 100644 index 00000000..7f8fcb3b --- /dev/null +++ b/dpdk/doc/guides/prog_guide/img/efd_i1.svg @@ -0,0 +1,130 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Page-1 + + + Square + LB + + + + + + + + + + LB + + Sheet.3 + + + + Square.4 + Target 1 + + + + + + + + + + Target 1 + + Square.5 + Target 2 + + + + + + + + + + Target 2 + + Square.7 + Target N + + + + + + + + + + Target N + + Sheet.8 + + + + Sheet.9 + + + + Sheet.11 + + + + Sheet.12 + + + + diff --git a/dpdk/doc/guides/prog_guide/img/efd_i10.svg b/dpdk/doc/guides/prog_guide/img/efd_i10.svg new file mode 100644 index 00000000..d26ec61e --- /dev/null +++ b/dpdk/doc/guides/prog_guide/img/efd_i10.svg @@ -0,0 +1,384 @@ + + + + + + + + + + + + + + + + + + Page-1 + + + Sheet.3 + + + + Sheet.4 + + + + Sheet.5 + Key1: Value = 0 + + + + Key1: Value = 0 + + Sheet.6 + + + + Sheet.7 + + + + Sheet.8 + Key3: Value = 1 + + + + Key3: Value = 1 + + Sheet.9 + + + + Sheet.10 + + + + Sheet.11 + Key4: Value = 0 + + + + Key4: Value = 0 + + Sheet.12 + + + + Sheet.13 + + + + Sheet.14 + Key7: Value = 1 + + + + Key7: Value = 1 + + Sheet.15 + + + + Sheet.16 + + + + Sheet.17 + + + + Sheet.18 + F + + + + F + + Sheet.19 + (key, + + + + (key, + + Sheet.20 + hash_index = + + + + hash_index = + + Sheet.21 + i) + + + + i) + + Sheet.22 + + + + Sheet.23 + + + + Sheet.24 + Key1: Position 4 + + + + Key1: Position 4 + + Sheet.25 + + + + Sheet.26 + + + + Sheet.27 + Key3: Position 6 + + + + Key3: Position 6 + + Sheet.28 + + + + Sheet.29 + + + + Sheet.30 + Key4: Position 14 + + + + Key4: Position 14 + + Sheet.31 + + + + Sheet.32 + + + + Sheet.33 + Key7: Position 14 + + + + Key7: Position 14 + + Sheet.34 + + + + Sheet.35 + + + + Sheet.36 + + + + Sheet.37 + 0000 + + + + 0000 + + Sheet.38 + 0 + + + + 0 + + Sheet.39 + 0 + + + + 0 + + Sheet.40 + 1 + + + + 1 + + Sheet.41 + 0 0000 00 + + + + 0 0000 00 + + Sheet.42 + ? + + + + ? + + Sheet.43 + 0 + + + + 0 + + Sheet.44 + + + + Sheet.45 + + + + Sheet.46 + + + + Sheet.47 + + + + Sheet.48 + + + + Sheet.49 + Values + + + + Values + + Sheet.50 + + + + Sheet.51 + + + + Sheet.52 + + + + Sheet.53 + + + + Sheet.54 + + + + Sheet.55 + + + + Sheet.56 + Lookup_table + + + + Lookup_table + + Sheet.57 + (16 bits) + + + + (16 bits) + + diff --git a/dpdk/doc/guides/prog_guide/img/efd_i11.svg b/dpdk/doc/guides/prog_guide/img/efd_i11.svg new file mode 100644 index 00000000..f2cc656b --- /dev/null +++ b/dpdk/doc/guides/prog_guide/img/efd_i11.svg @@ -0,0 +1,319 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + Page-1 + + + Sheet.5 + + + + Sheet.6 + + + + Sheet.7 + Key + + + + Key + + Sheet.8 + + + + Sheet.9 + + + + Sheet.10 + + + + Sheet.11 + hash + + + + hash + + Sheet.12 + + + + Sheet.13 + + + + Sheet.14 + 0x0102ABCD + + + + 0x0102ABCD + + Sheet.15 + + + + Sheet.16 + + + + Sheet.17 + + + + Sheet.18 + + + + Sheet.19 + + + + Sheet.20 + + + + Sheet.21 + hash_index = + + + + hash_index = + + Sheet.22 + 38123 + + + + 38123 + + Sheet.23 + + + + Sheet.24 + + + + Sheet.25 + lookup_table = + + + + lookup_table = + + Sheet.26 + 0110 1100 0101 1101 + + + + 0110 1100 0101 1101 + + Sheet.27 + Group ID: 0x0102 + + + + Group ID: 0x0102 + + Sheet.28 + + + + Sheet.29 + + + + Sheet.30 + + + + Sheet.31 + + + + Sheet.35 + + + + Sheet.36 + + + + Sheet.37 + + + + Sheet.38 + Position = 6 + + + + Position = 6 + + Sheet.39 + + + + Sheet.41 + Apply the equation + + + + Apply the equation + + Sheet.42 + to retrieve the bit + + + + to retrieve the bit + + Sheet.43 + position in the + + + + position in the + + Sheet.44 + lookup_table + + + + lookup_table + + 1-D word balloon + Retrieve the value “0' from the specified location in the loo... + + + + + + + + + Retrieve the value “0' from the specified location in the lookup table + + Sheet.48 + F(Key, hash_index = 38123 + + + + F(Key, hash_index = 38123 + + 1-D word balloon.49 + Apply the equation to retrieve the bit position in the lookup... + + + + + + + + + Apply the equation to retrieve the bit position in the lookup_table + + Sheet.50 + + + + Sheet.51 + (Hash(key,seed1)+38123*hash(key,seed2))%16 + + + + (Hash(key,seed1)+38123*hash(key,seed2))%16 + + diff --git a/dpdk/doc/guides/prog_guide/img/efd_i12.svg b/dpdk/doc/guides/prog_guide/img/efd_i12.svg new file mode 100644 index 00000000..a309d582 --- /dev/null +++ b/dpdk/doc/guides/prog_guide/img/efd_i12.svg @@ -0,0 +1,1008 @@ + + + + + + + + + + + + + + + + + + Page-1 + + + Sheet.3 + + + + Sheet.4 + + + + Sheet.5 + Bins + + + + Bins + + Sheet.6 + Groups + + + + Groups + + Sheet.7 + + + + Sheet.8 + + + + Sheet.9 + 0 + + + + 0 + + Sheet.10 + + + + Sheet.11 + Chunks + + + + Chunks + + Sheet.12 + + + + Sheet.13 + + + + Sheet.14 + 1 + + + + 1 + + Sheet.15 + + + + Sheet.16 + + + + Sheet.17 + … + + + + … + + Sheet.18 + + + + Sheet.19 + + + + Sheet.20 + variable + + + + variable + + Sheet.21 + # of + + + + # of + + Sheet.22 + chunks + + + + chunks + + Sheet.23 + (power + + + + (power + + Sheet.24 + of 2) + + + + of 2) + + Sheet.25 + + + + Sheet.26 + + + + Sheet.27 + + + + Sheet.28 + + + + Sheet.29 + + + + Sheet.30 + + + + Sheet.31 + + + + Sheet.32 + + + + Sheet.33 + + + + Sheet.34 + + + + Sheet.35 + + + + Sheet.36 + + + + Sheet.37 + + + + Sheet.38 + + + + Sheet.39 + + + + Sheet.40 + + + + Sheet.41 + + + + Sheet.42 + + + + Sheet.43 + 0 + + + + 0 + + Sheet.44 + 4 + + + + 4 + + Sheet.45 + + + + Sheet.46 + + + + Sheet.47 + 1 + + + + 1 + + Sheet.48 + + + + Sheet.49 + + + + Sheet.50 + 2 + + + + 2 + + Sheet.51 + 3 + + + + 3 + + Sheet.52 + +1 + + + + +1 + + Sheet.53 + + + + Sheet.54 + + + + Sheet.55 + 3 + + + + 3 + + Sheet.56 + + + + Sheet.57 + + + + Sheet.58 + 4 + + + + 4 + + Sheet.59 + 0 + + + + 0 + + Sheet.60 + + + + Sheet.61 + + + + Sheet.62 + 5 + + + + 5 + + Sheet.63 + + + + Sheet.64 + + + + Sheet.65 + 6 + + + + 6 + + Sheet.66 + + + + Sheet.67 + + + + Sheet.68 + 7 + + + + 7 + + Sheet.69 + 2 + + + + 2 + + Sheet.70 + + + + Sheet.71 + + + + Sheet.72 + 8 + + + + 8 + + Sheet.73 + + + + Sheet.74 + + + + Sheet.75 + 9 + + + + 9 + + Sheet.76 + + + + Sheet.77 + + + + Sheet.78 + 10 + + + + 10 + + Sheet.79 + 1 + + + + 1 + + Sheet.80 + + + + Sheet.81 + + + + Sheet.82 + 11 + + + + 11 + + Sheet.83 + + + + Sheet.84 + + + + Sheet.85 + 12 + + + + 12 + + Sheet.86 + + + + Sheet.87 + + + + Sheet.88 + … + + + + … + + Sheet.89 + + + + Sheet.90 + + + + Sheet.91 + 255 + + + + 255 + + Sheet.92 + + + + Sheet.93 + + + + Sheet.94 + + + + Sheet.95 + + + + Sheet.96 + + + + Sheet.97 + + + + Sheet.98 + + + + Sheet.99 + … + + + + … + + Sheet.100 + + + + Sheet.101 + + + + Sheet.102 + 5 + + + + 5 + + Sheet.103 + + + + Sheet.104 + + + + Sheet.105 + 4 + + + + 4 + + Sheet.106 + 2 + + + + 2 + + Sheet.107 + 4 + + + + 4 + + Sheet.108 + 10 + + + + 10 + + Sheet.109 + 1 + + + + 1 + + Sheet.110 + +4 + + + + +4 + + Sheet.111 + + + + Sheet.112 + + + + Sheet.113 + 3 + + + + 3 + + Sheet.114 + + + + Sheet.115 + + + + Sheet.116 + 1 + + + + 1 + + Sheet.117 + 2 + + + + 2 + + Sheet.118 + + + + Sheet.119 + 7 + + + + 7 + + Sheet.120 + 5 + + + + 5 + + Sheet.121 + - + + + + - + + Sheet.122 + 3 + + + + 3 + + Sheet.123 + + + + Sheet.124 + + + + Sheet.125 + 0 + + + + 0 + + Sheet.126 + 0 + + + + 0 + + Sheet.127 + 4 + + + + 4 + + Sheet.128 + + + + Sheet.129 + + + + Sheet.130 + 64 + + + + 64 + + Sheet.131 + 96 + + + + 96 + + Sheet.132 + 7 + + + + 7 + + Sheet.133 + + + + Sheet.134 + + + + Sheet.135 + 6 + + + + 6 + + Sheet.136 + 98 + + + + 98 + + Sheet.137 + 5 + + + + 5 + + Sheet.138 + + + + Sheet.139 + + + + Sheet.140 + 2 + + + + 2 + + Sheet.141 + 99 + + + + 99 + + Sheet.142 + 9 + + + + 9 + + Sheet.143 + + + + Sheet.144 + + + + Sheet.145 + 7 + + + + 7 + + Sheet.146 + 97 + + + + 97 + + Sheet.147 + 6 + + + + 6 + + Sheet.148 + + + + Sheet.149 + + + + Sheet.150 + Insert key + + + + Insert key + + Sheet.151 + + + + Sheet.152 + + + + Sheet.153 + + + + Sheet.154 + hash + + + + hash + + Sheet.155 + + + + Sheet.156 + + + + Sheet.157 + 0x0102ABCD + + + + 0x0102ABCD + + Sheet.158 + + + + Sheet.159 + + + + Sheet.160 + + + + Sheet.161 + + + + Sheet.162 + chunk id + + + + chunk id + + Sheet.163 + + + + Sheet.164 + bin id + + + + bin id + + Sheet.165 + + + + Sheet.166 + + + + Sheet.167 + + + + Sheet.168 + + + + Sheet.169 + + + + Sheet.170 + Move bin from group 1 to 4 + + + + Move bin from group 1 to 4 + + diff --git a/dpdk/doc/guides/prog_guide/img/efd_i2.svg b/dpdk/doc/guides/prog_guide/img/efd_i2.svg new file mode 100644 index 00000000..a5f43f94 --- /dev/null +++ b/dpdk/doc/guides/prog_guide/img/efd_i2.svg @@ -0,0 +1,280 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Page-1 + + + Circle + + + + + + + + + + Circle.3 + + + + + + + + + + Circle.4 + + + + + + + + + + Circle.5 + + + + + + + + + + Circle.6 + + + + + + + + + + Circle.7 + + + + + + + + + + Circle.8 + + + + + + + + + + Circle.9 + + + + + + + + + + Circle.10 + + + + + + + + + + Circle.11 + + + + + + + + + + Circle.12 + + + + + + + + + + Circle.13 + + + + + + + + + + Circle.14 + + + + + + + + + + Circle.15 + + + + + + + + + + Circle.16 + + + + + + + + + + Circle.17 + + + + + + + + + + Ellipse + + + + + + + + + + Sheet.19 + + + + Sheet.20 + Target Hashed Value + + + + Target Hashed Value + + Sheet.21 + + + + Sheet.23 + Keys + + + + Keys + + Sheet.24 + + + + diff --git a/dpdk/doc/guides/prog_guide/img/efd_i3.svg b/dpdk/doc/guides/prog_guide/img/efd_i3.svg new file mode 100644 index 00000000..ae229037 --- /dev/null +++ b/dpdk/doc/guides/prog_guide/img/efd_i3.svg @@ -0,0 +1,634 @@ + + + + + + + + + + + + + + + + + + + + + + + Page-1 + + + + Rectangle + Packet Header + + + + + + + + + + Packet Header + + Rectangle.3 + Payload + + + + + + + + + + Payload + + Rectangle.4 + Flow Key + + + + + + + + + + Flow Key + + Sheet.5 + + + + Sheet.8 + Fields of the packet are used to form a flow Key + + + + Fields of the packet are used to form a flow Key + + Sheet.13 + + Trapezoid + + + + + + + + + + Sheet.12 + H(..) + + + + H(..) + + + Simple Arrow + + + + + + + + + + + Sheet.15 + Hash function is used to create a flow table index + + + + Hash function is used to create a flow table index + + Rectangle.58 + Key 1 + + + + + + + + + + Key 1 + + Rectangle.59 + Action 1 + + + + + + + + + + Action 1 + + Rectangle.60 + Key 2 + + + + + + + + + + Key 2 + + Rectangle.61 + Action 2 + + + + + + + + + + Action 2 + + Rectangle.62 + + + + + + + + + + Rectangle.63 + + + + + + + + + + Rectangle.64 + + + + + + + + + + Rectangle.65 + + + + + + + + + + Rectangle.66 + + + + + + + + + + Rectangle.67 + + + + + + + + + + Rectangle.68 + + + + + + + + + + Rectangle.69 + + + + + + + + + + Rectangle.70 + Key x + + + + + + + + + + Key x + + Rectangle.71 + Action x + + + + + + + + + + Action x + + Rectangle.72 + Key y + + + + + + + + + + Key y + + Rectangle.73 + Action y + + + + + + + + + + Action y + + Rectangle.74 + Key z + + + + + + + + + + Key z + + Rectangle.75 + Action z + + + + + + + + + + Action z + + Rectangle.76 + + + + + + + + + + Rectangle.77 + + + + + + + + + + Rectangle.78 + + + + + + + + + + Rectangle.79 + + + + + + + + + + Rectangle.80 + Key N + + + + + + + + + + Key N + + Rectangle.81 + Action N + + + + + + + + + + Action N + + Rectangle.82 + + + + + + + + + + Sheet.83 + + + + Sheet.84 + Load Balancing Flow Table + + + + Load Balancing Flow Table + + Rectangle.85 + + + + + + + + + + Sheet.86 + + + + Sheet.87 + Hash value used to index Flow table + + + + Hash value used to index Flow table + + Sheet.88 + + + + Sheet.89 + + + + Rectangle.90 + Key x + + + + + + + + + + Key x + + Rectangle.91 + Key z + + + + + + + + + + Key z + + Sheet.96 + + Trapezoid + + + + + + + + + + Sheet.98 + Match + + + + Match + + + Sheet.99 + + + + Sheet.100 + + + + Sheet.101 + + + + Rectangle.102 + Key y + + + + + + + + + + Key y + + Rectangle.103 + Flow Key + + + + + + + + + + Flow Key + + Sheet.104 + + + + Sheet.105 + Retrieved keys are matched with input key + + + + Retrieved keys are matched with input key + + Rectangle.106 + Action + + + + + + + + + + Action + + Simple Arrow.111 + + + + + + + + + + + diff --git a/dpdk/doc/guides/prog_guide/img/efd_i4.svg b/dpdk/doc/guides/prog_guide/img/efd_i4.svg new file mode 100644 index 00000000..5be5ccd7 --- /dev/null +++ b/dpdk/doc/guides/prog_guide/img/efd_i4.svg @@ -0,0 +1,203 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Page-1 + + + Sheet.2 + Key 1 Key 2 ... Key 28 + + + + Key 1Key 2...Key 28 + + Sheet.9 + Target Value + + + + Target Value + + Sheet.11 + 0 1 0 + + + + 010 + + Sheet.8 + + + + Sheet.10 + + + + Sheet.4 + + + + Sheet.5 + + + + Sheet.6 + + + + Sheet.7 + + + + Sheet.12 + 0 0 0 + + + + 000 + + Sheet.26 + H1(x) + + + + H1(x) + + Sheet.27 + 1 1 0 + + + + 110 + + Sheet.28 + H2(x) + + + + H2(x) + + Sheet.29 + 0 1 0 + + + + 010 + + Sheet.30 + Hm(x) + + + + Hm(x) + + Sheet.31 + ….. + + + + ….. + + Sheet.32 + Store m for this group of keys + + + + + + + Store m for this group of keys + + Sheet.36 + + + + Sheet.44 + + Sheet.42 + + + + Sheet.43 + + + + + Sheet.45 + + Sheet.46 + + + + Sheet.47 + + + + + diff --git a/dpdk/doc/guides/prog_guide/img/efd_i5.svg b/dpdk/doc/guides/prog_guide/img/efd_i5.svg new file mode 100644 index 00000000..b6540ba4 --- /dev/null +++ b/dpdk/doc/guides/prog_guide/img/efd_i5.svg @@ -0,0 +1,183 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Page-1 + + + Rectangle + All Keys + + + + + + + + + + All Keys + + Rectangle.3 + Group 1 + + + + + + + + + + Group 1 + + Rectangle.4 + Group 2 + + + + + + + + + + Group 2 + + Rectangle.5 + Group 3 + + + + + + + + + + Group 3 + + Rectangle.6 + Group X + + + + + + + + + + Group X + + Sheet.7 + + + + Sheet.8 + + + + Sheet.9 + + + + Sheet.10 + + + + Sheet.11 + + + + Sheet.12 + H7 + + + + H7 + + Sheet.13 + H267 + + + + H267 + + Sheet.14 + H46 + + + + H46 + + Sheet.15 + H132 + + + + H132 + + Sheet.16 + Store hash function index for each group of keys + + + + Store hash function index for each group of keys + + diff --git a/dpdk/doc/guides/prog_guide/img/efd_i6.svg b/dpdk/doc/guides/prog_guide/img/efd_i6.svg new file mode 100644 index 00000000..9aee30bc --- /dev/null +++ b/dpdk/doc/guides/prog_guide/img/efd_i6.svg @@ -0,0 +1,1254 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Page-1 + + + Rectangle.58 + Key 1 + + + + + + + + + + Key 1 + + Rectangle.59 + Action 1 + + + + + + + + + + Action 1 + + Rectangle.60 + Key 2 + + + + + + + + + + Key 2 + + Rectangle.61 + Action 2 + + + + + + + + + + Action 2 + + Rectangle.62 + + + + + + + + + + Rectangle.63 + + + + + + + + + + Rectangle.64 + + + + + + + + + + Rectangle.65 + + + + + + + + + + Rectangle.66 + + + + + + + + + + Rectangle.67 + + + + + + + + + + Rectangle.68 + + + + + + + + + + Rectangle.69 + + + + + + + + + + Rectangle.70 + Key x + + + + + + + + + + Key x + + Rectangle.71 + Action x + + + + + + + + + + Action x + + Rectangle.72 + Key y + + + + + + + + + + Key y + + Rectangle.73 + Action y + + + + + + + + + + Action y + + Rectangle.74 + Key z + + + + + + + + + + Key z + + Rectangle.75 + Action z + + + + + + + + + + Action z + + Rectangle.76 + + + + + + + + + + Rectangle.77 + + + + + + + + + + Rectangle.78 + + + + + + + + + + Rectangle.79 + + + + + + + + + + Rectangle.80 + Key N + + + + + + + + + + Key N + + Rectangle.81 + Action N + + + + + + + + + + Action N + + Rectangle.82 + + + + + + + + + + Sheet.28 + Local Table for N Specific Flows Serviced at Node 1 + + + + Local Table for N Specific Flows Serviced at Node 1 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Load balancer + + Sheet.35 + + + + + + + + + + + Sheet.36 + + + + + + + + Rectangle + + + + + + + + + + Sheet.38 + EFD Table + + + + EFD Table + + Rectangle.39 + Group_id + + + + + + + + + + Group_id + + Rectangle.40 + Hash index + + + + + + + + + + Hash index + + Rectangle.41 + + + + + + + + + + Rectangle.42 + + + + + + + + + + Rectangle.43 + + + + + + + + + + Rectangle.44 + + + + + + + + + + Rectangle.45 + + + + + + + + + + Rectangle.46 + + + + + + + + + + Sheet.47 + + + + Sheet.48 + Supports X*N Flows + + + + Supports X*N Flows + + Sheet.49 + Frontend Server or Load Balancer + + + + Frontend Serveror Load Balancer + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Server + + Sheet.52 + + + + + + + Sheet.53 + + + + + + + Sheet.54 + + + + + + + + + + + + Sheet.59 + + + + Sheet.60 + Backend Server 1 + + + + Backend Server 1 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Server.61 + + Sheet.62 + + + + + + + Sheet.63 + + + + + + + Sheet.64 + + + + + + + + + + + + Sheet.65 + Backend Server 2 + + + + Backend Server 2 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Server.66 + + Sheet.67 + + + + + + + Sheet.68 + + + + + + + Sheet.69 + + + + + + + + + + + + Sheet.70 + Backend Server X + + + + Backend Server X + + Sheet.71 + + + + Sheet.72 + + + + Rectangle.73 + Key 1 + + + + + + + + + + Key 1 + + Rectangle.74 + Action 1 + + + + + + + + + + Action 1 + + Rectangle.75 + Key 2 + + + + + + + + + + Key 2 + + Rectangle.76 + Action 2 + + + + + + + + + + Action 2 + + Rectangle.77 + + + + + + + + + + Rectangle.78 + + + + + + + + + + Rectangle.79 + + + + + + + + + + Rectangle.80 + + + + + + + + + + Rectangle.81 + + + + + + + + + + Rectangle.82 + + + + + + + + + + Rectangle.83 + + + + + + + + + + Rectangle.84 + + + + + + + + + + Rectangle.85 + Key x + + + + + + + + + + Key x + + Rectangle.86 + Action x + + + + + + + + + + Action x + + Rectangle.87 + Key y + + + + + + + + + + Key y + + Rectangle.88 + Action y + + + + + + + + + + Action y + + Rectangle.89 + Key z + + + + + + + + + + Key z + + Rectangle.90 + Action z + + + + + + + + + + Action z + + Rectangle.91 + + + + + + + + + + Rectangle.92 + + + + + + + + + + Rectangle.93 + + + + + + + + + + Rectangle.94 + + + + + + + + + + Rectangle.95 + Key N + + + + + + + + + + Key N + + Rectangle.96 + Action N + + + + + + + + + + Action N + + Rectangle.97 + + + + + + + + + + Sheet.98 + Local Table for N Specific Flows Serviced at Node X + + + + Local Table for N Specific Flows Serviced at Node X + + Sheet.99 + + + + Sheet.100 + + + + Sheet.101 + + + + Sheet.102 + Supports N Flows + + + + Supports N Flows + + diff --git a/dpdk/doc/guides/prog_guide/img/efd_i7.svg b/dpdk/doc/guides/prog_guide/img/efd_i7.svg new file mode 100644 index 00000000..98f80005 --- /dev/null +++ b/dpdk/doc/guides/prog_guide/img/efd_i7.svg @@ -0,0 +1,790 @@ + + + + + + + + + + + + + + + + + + + + + + + Page-1 + + + Sheet.3 + + + + Sheet.4 + + + + Sheet.5 + Key1 + + + + Key1 + + Sheet.6 + + + + Sheet.7 + + + + Sheet.8 + + + + Sheet.9 + hash + + + + hash + + Sheet.10 + + + + Sheet.11 + + + + Sheet.12 + 0x0102ABCD + + + + 0x0102ABCD + + Sheet.13 + + + + Sheet.14 + + + + Sheet.15 + + + + Sheet.16 + + + + Sheet.17 + Key2 + + + + Key2 + + Sheet.18 + + + + Sheet.19 + + + + Sheet.20 + + + + Sheet.21 + hash + + + + hash + + Sheet.22 + + + + Sheet.23 + + + + Sheet.24 + 0x0103CDAB + + + + 0x0103CDAB + + Sheet.25 + + + + Sheet.26 + + + + Sheet.27 + + + + Sheet.28 + + + + Sheet.29 + Key3 + + + + Key3 + + Sheet.30 + + + + Sheet.31 + + + + Sheet.32 + + + + Sheet.33 + hash + + + + hash + + Sheet.34 + + + + Sheet.35 + + + + Sheet.36 + 0x0102BAAD + + + + 0x0102BAAD + + Sheet.37 + + + + Sheet.38 + + + + Sheet.39 + + + + Sheet.40 + + + + Sheet.41 + Key4 + + + + Key4 + + Sheet.42 + + + + Sheet.43 + + + + Sheet.44 + + + + Sheet.45 + hash + + + + hash + + Sheet.46 + + + + Sheet.47 + + + + Sheet.48 + 0x0104BEEF + + + + 0x0104BEEF + + Sheet.49 + + + + Sheet.50 + + + + Sheet.51 + + + + Sheet.52 + + + + Sheet.53 + Key5 + + + + Key5 + + Sheet.54 + + + + Sheet.55 + + + + Sheet.56 + + + + Sheet.57 + hash + + + + hash + + Sheet.58 + + + + Sheet.59 + + + + Sheet.60 + 0x0103DABD + + + + 0x0103DABD + + Sheet.61 + + + + Sheet.62 + + + + Sheet.63 + + + + Sheet.64 + + + + Sheet.65 + Key6 + + + + Key6 + + Sheet.66 + + + + Sheet.67 + + + + Sheet.68 + + + + Sheet.69 + hash + + + + hash + + Sheet.70 + + + + Sheet.71 + + + + Sheet.72 + 0x0102ADCB + + + + 0x0102ADCB + + Sheet.73 + + + + Sheet.74 + + + + Sheet.75 + + + + Sheet.76 + + + + Sheet.77 + Key7 + + + + Key7 + + Sheet.78 + + + + Sheet.79 + + + + Sheet.80 + + + + Sheet.81 + hash + + + + hash + + Sheet.82 + + + + Sheet.83 + + + + Sheet.84 + 0x0104DBCA + + + + 0x0104DBCA + + Sheet.85 + + + + Sheet.86 + + + + Sheet.87 + + + + Sheet.88 + + + + Sheet.89 + 0x0102 + + + + 0x0102 + + Sheet.90 + 4 + + + + 4 + + Sheet.91 + + + + Sheet.92 + + + + Sheet.93 + 0x0103 + + + + 0x0103 + + Sheet.94 + 2 + + + + 2 + + Sheet.95 + + + + Sheet.96 + + + + Sheet.97 + 0x0104 + + + + 0x0104 + + Sheet.98 + 1 + + + + 1 + + Sheet.99 + + + + Sheet.100 + + + + Sheet.101 + + + + Sheet.102 + + + + Sheet.103 + + + + Sheet.104 + + + + Sheet.105 + + + + Sheet.106 + + + + Sheet.109 + + + + Sheet.110 + Groups + + + + Groups + + Sheet.111 + + + + Sheet.112 + group id + + + + group id + + Sheet.114 + - + + + + - + + Sheet.115 + Keys separated into + + + + Keys separated into + + Sheet.116 + groups based on + + + + groups based on + + Sheet.117 + some bits from hash + + + + some bits from hash + + Sheet.118 + - + + + + - + + Sheet.119 + Groups contain a + + + + Groups contain a + + Sheet.120 + small number of + + + + small number of + + Sheet.121 + keys (<28) + + + + keys (<28) + + Sheet.122 + + + + Sheet.123 + Group + + + + Group + + Sheet.124 + Identifier + + + + Identifier + + Sheet.125 + (simplified) + + + + (simplified) + + Sheet.127 + Keys separated into groups based on some bits from hash. Grou... + + + + + + + · Keys separated into groups based on some bits from hash.· Groups contain a small number of keys (<28) + + Sheet.129 + Total # of keys in group so far + + + + Total # of keys in group so far + + diff --git a/dpdk/doc/guides/prog_guide/img/efd_i8.svg b/dpdk/doc/guides/prog_guide/img/efd_i8.svg new file mode 100644 index 00000000..d0fd463a --- /dev/null +++ b/dpdk/doc/guides/prog_guide/img/efd_i8.svg @@ -0,0 +1,182 @@ + + + + + + + + + + + + + + + + + + Page-2 + + + Sheet.4 + + + + Sheet.5 + + + + Sheet.6 + + + + Sheet.7 + + + + Sheet.8 + hash_index + + + + hash_index + + Sheet.9 + (integer, 16 bits) + + + + (integer, 16 bits) + + Sheet.10 + + + + Sheet.11 + + + + Sheet.12 + lookup_table + + + + lookup_table + + Sheet.13 + (16 bits) + + + + (16 bits) + + Sheet.14 + Group ID: 0x0102 + + + + Group ID: 0x0102 + + Sheet.15 + + + + Sheet.16 + + + + Sheet.17 + Key1: Value = 0 + + + + Key1: Value = 0 + + Sheet.18 + + + + Sheet.19 + + + + Sheet.20 + Key3: Value = 1 + + + + Key3: Value = 1 + + Sheet.21 + + + + Sheet.22 + + + + Sheet.23 + Key4: Value = 0 + + + + Key4: Value = 0 + + Sheet.24 + + + + Sheet.25 + + + + Sheet.26 + Key7: Value = 1 + + + + Key7: Value = 1 + + Sheet.27 + + + + diff --git a/dpdk/doc/guides/prog_guide/img/efd_i9.svg b/dpdk/doc/guides/prog_guide/img/efd_i9.svg new file mode 100644 index 00000000..b2e385d1 --- /dev/null +++ b/dpdk/doc/guides/prog_guide/img/efd_i9.svg @@ -0,0 +1,390 @@ + + + + + + + + + + + + + + + + + + Page-1 + + + Sheet.68 + + + + Sheet.69 + + + + Sheet.70 + (hash(key, seed1) + hash_index * + + + + (hash(key, seed1) + hash_index * + + Sheet.71 + hash(key + + + + hash(key + + Sheet.72 + , seed2)) % 16 + + + + , seed2)) % 16 + + Sheet.73 + + + + Sheet.74 + + + + Sheet.75 + + + + Sheet.76 + lookup_table + + + + lookup_table + + Sheet.77 + bit + + + + bit + + Sheet.78 + index for key1 + + + + index for key1 + + Sheet.79 + + + + Sheet.80 + + + + Sheet.81 + lookup_table + + + + lookup_table + + Sheet.82 + bit + + + + bit + + Sheet.83 + index for key3 + + + + index for key3 + + Sheet.84 + + + + Sheet.85 + + + + Sheet.86 + lookup_table + + + + lookup_table + + Sheet.87 + bit + + + + bit + + Sheet.88 + index for key4 + + + + index for key4 + + Sheet.89 + + + + Sheet.90 + + + + Sheet.91 + lookup_table + + + + lookup_table + + Sheet.92 + bit + + + + bit + + Sheet.93 + index for key7 + + + + index for key7 + + Sheet.94 + + + + Sheet.95 + CRC32 (32 + + + + CRC32 (32 + + Sheet.96 + bit output) + + + + bit output) + + Sheet.97 + + + + Sheet.98 + Goal: Find a valid + + + + Goal: Find a valid + + Sheet.99 + hash_index + + + + hash_index + + Sheet.100 + + + + Sheet.101 + Lookup Table has + + + + Lookup Table has + + Sheet.102 + 16 bits + + + + 16 bits + + Sheet.103 + + + + Sheet.104 + CRC32 (32 + + + + CRC32 (32 + + Sheet.105 + bit output) + + + + bit output) + + Sheet.106 + + + + Sheet.107 + Goal is to find a hash_index that produces + + + + Goal is to find a hash_index that produces + + Sheet.108 + a lookup_table with no contradictions + + + + a lookup_table with no contradictions + + Sheet.109 + + + + Sheet.110 + + + + Sheet.111 + Key1: Value = 0 + + + + Key1: Value = 0 + + Sheet.112 + + + + Sheet.113 + + + + Sheet.114 + Key3: Value = 1 + + + + Key3: Value = 1 + + Sheet.115 + + + + Sheet.116 + + + + Sheet.117 + Key4: Value = 0 + + + + Key4: Value = 0 + + Sheet.118 + + + + Sheet.119 + + + + Sheet.120 + Key7: Value = 1 + + + + Key7: Value = 1 + + Sheet.121 + + + + diff --git a/dpdk/doc/guides/prog_guide/img/eventdev_usage.svg b/dpdk/doc/guides/prog_guide/img/eventdev_usage.svg new file mode 100644 index 00000000..7765649b --- /dev/null +++ b/dpdk/doc/guides/prog_guide/img/eventdev_usage.svg @@ -0,0 +1,994 @@ + + +image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + + Page-1 + + + + Square + Atomic Queue #1 + + + + + + + + + + + + + Square.3 + Atomic Queue #2 + + + + + + + + + + + + + Square.4 + Single Link Queue # 1 + + + + + + + + + + + + + Circle + RX + + + + + + + + + + RX + + + + Dynamic connector + + + + Circle.7 + W .. + + + + + + + + + + W .. + + + + Circle.9 + W N + + + + + + + + + + W N + + + + Circle.10 + W 1 + + + + + + + + + + W 1 + + + + Dynamic connector.11 + + + + Dynamic connector.12 + + + + Dynamic connector.13 + + + + Dynamic connector.14 + + + + Dynamic connector.15 + + + + Circle.19 + W .. + + + + + + + + + + W .. + + + + Circle.20 + W N + + + + + + + + + + W N + + + + Circle.21 + W 1 + + + + + + + + + + W 1 + + + + Dynamic connector.28 + + + + Dynamic connector.29 + + + + Dynamic connector.30 + + + + Dynamic connector.31 + + + + Dynamic connector.32 + + + + Dynamic connector.33 + + + + Dynamic connector.34 + + + Atomic #1 +Atomic #2 + +Single Link +CircleRX TX +Dynamic connector.28 + diff --git a/dpdk/doc/guides/prog_guide/img/gso-output-segment-format.svg b/dpdk/doc/guides/prog_guide/img/gso-output-segment-format.svg new file mode 100644 index 00000000..bdb5ec33 --- /dev/null +++ b/dpdk/doc/guides/prog_guide/img/gso-output-segment-format.svg @@ -0,0 +1,313 @@ + + + + + + + + + + Page-1 + + + Sheet.3 + + + + Sheet.4 + + + + Sheet.5 + + + + Sheet.6 + + + + Sheet.7 + + + + Sheet.10 + + + + Sheet.11 + + + + Sheet.12 + Payload 0 + + + + Payload 0 + + Sheet.13 + Payload 1 + + + + Payload 1 + + Sheet.14 + Payload 2 + + + + Payload 2 + + Sheet.15 + + + + Sheet.16 + + + + Sheet.17 + + + + Sheet.18 + Header + + + + Header + + Sheet.19 + + + + Sheet.20 + + + + Sheet.21 + Header + + + + Header + + Sheet.22 + + + + Sheet.23 + + + + Sheet.24 + Payload 1 + + + + Payload 1 + + Sheet.25 + + + + Sheet.26 + + + + Sheet.27 + Indirect mbuf + + + + Indirect mbuf + + Sheet.28 + (pointer to data) + + + + (pointer to data) + + Sheet.29 + + + + Sheet.30 + Memory copy + + + + Memory copy + + Sheet.31 + No Memory Copy + + + + No Memory Copy + + Sheet.32 + Logical output segment + + + + Logical output segment + + Sheet.36 + Two-part output segment + + + + Two-part output segment + + Sheet.37 + + + + Sheet.38 + + + + Sheet.39 + + + + Sheet.40 + + + + Sheet.41 + Direct mbuf + + + + Direct mbuf + + Sheet.42 + (copy of headers) + + + + (copy of headers) + + Sheet.43 + next + + + + next + + Sheet.44 + + + + Sheet.45 + segsz + + + + segsz + + Sheet.46 + Input packet + + + + Input packet + + diff --git a/dpdk/doc/guides/prog_guide/img/gso-three-seg-mbuf.svg b/dpdk/doc/guides/prog_guide/img/gso-three-seg-mbuf.svg new file mode 100644 index 00000000..0431012d --- /dev/null +++ b/dpdk/doc/guides/prog_guide/img/gso-three-seg-mbuf.svg @@ -0,0 +1,477 @@ + + + + + GSO three-part output segment + + + + + + Page-1 + + + + + Sheet.111 + + + + Sheet.110 + + + + Sheet.4 + + + + Sheet.5 + + + + Sheet.6 + + + + Sheet.7 + + + + Sheet.10 + + + + Sheet.11 + + + + Sheet.12 + Payload 0 + + + + Payload 0 + + Sheet.13 + Payload 1 + + + + Payload 1 + + Sheet.15 + + + + Sheet.16 + + + + Sheet.17 + Header + + + + Header + + Sheet.23 + + + + Sheet.24 + + + + Sheet.25 + Header + + + + Header + + Sheet.31 + + + + Sheet.32 + + + + Sheet.33 + Payload 1 + + + + Payload 1 + + Sheet.35 + Logical output segment + + + + Logical output segment + + Sheet.39 + + + + Sheet.40 + + + + Sheet.46 + + + + Sheet.47 + + + + Sheet.48 + Direct mbuf + + + + Direct mbuf + + Sheet.51 + (copy of headers) + + + + (copy of headers) + + Sheet.53 + next + + + + next + + Sheet.54 + + + + Sheet.55 + segsz + + + + segsz + + Sheet.56 + + + + Sheet.57 + + + + Sheet.58 + + + + Sheet.59 + Payload 1 + + + + Payload 1 + + Sheet.60 + Payload 2 + + + + Payload 2 + + Sheet.63 + Multi-segment input packet + + + + Multi-segment input packet + + Sheet.70 + + + + Sheet.71 + + + + Sheet.72 + Indirect mbuf + + + + Indirect mbuf + + Sheet.75 + (pointer to data) + + + + (pointer to data) + + Sheet.77 + next + + + + next + + Sheet.78 + + + + Sheet.79 + + + + Sheet.80 + + + + Sheet.81 + pkt_len + + + + pkt_len + + Sheet.82 + % segsz + + + + % segsz + + Sheet.34 + + + + Sheet.85 + + + + Sheet.87 + 1 + + + + 1 + + Sheet.88 + + + + Sheet.90 + 2 + + + + 2 + + Sheet.95 + next + + + + next + + Sheet.97 + + + + Sheet.100 + + + + Sheet.104 + + + + Sheet.105 + Indirect mbuf + + + + Indirect mbuf + + Sheet.106 + (pointer to data) + + + + (pointer to data) + + Sheet.107 + + + + Sheet.108 + 3 + + + + 3 + + Sheet.109 + (pointer to data) + + + + (pointer to data) + + Sheet.113 + Three-part output segment + + + + Three-part output segment + + diff --git a/dpdk/doc/guides/prog_guide/img/ivshmem.png b/dpdk/doc/guides/prog_guide/img/ivshmem.png deleted file mode 100644 index 2b34a2cfdf3d65025c1f942f5cc1f2a1ef05892c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 44920 zcmdSA1Cu4q^ESL=dj~tVZR3n>+qP})*x0eH9ox2TduGRa&;5J;U*bIxCpx;LJF7DD zs;e@qDkBx-#1UYzVSoJifgmX%qV(ej2oLbz3k?Z;vK2Xl2fTndD~SVs)J)@^0dK&~ zg=B?({HTwE`!Iw6-orRZXgdG+fjIczA0&ka5eNA2my4)|i?Y3$i@Twd=?_^;V<&rO z;2oj7nVF@rss9z59-XQ2h_o_cdMC%k)Gv~eGkfGy|=e;-lXD;d227iDgbslzcwH|k^`F1pRG@8$J zIiJL)5e0++e^~6;`F*{bTidP~e7-$ONJ!LmqX&)Tuv(%b67buk zSO9zFp25asErK`jAoNqK95W4qobn%jVu|tpP!#7KYxZC9v%{pj89L~-)Fz)zy0mI z@`PWGys){&k_#O-cF+g+7iK#Hk8|AZ4J3XieBsH#X_1$gS3j{LVP{vvTY>q?**G~l z(I&&j#{Pyd#rC@YdcQenoZ%nXY;#7z$G1zt2)Y0T1$9_bRWHAh=YK6C9-!B4Hy8** zc-jBWkWBqMI(o1$j=$n-`VSY{!^w;~xZ7EA8g)V2b^8TSC8$l0gP7K*b!!w{+^Q!` zG_=Cx!`?tJ^u2gWu^8;oX4{SZ4f`HZDJfXHu1B?|tHDULxk`10f0t`$Kpzb=^n4UR zhl6vPj3PJDYBqp3noTM7Sg+LJU3Wd}y}Z2azMkZY1Fc=}cBN&?yyLcEd%Dvj@b!8^ z;PY`e64l$53zUv${>Kp#5uZ1rT)DKOhqQgY$=djEG~xUvnM%3N``OO*lINyV*|x`Y z0z)Q?T{eNL=fUU7d2x1@lP$TtLA%YV*y~i%<^AbuAWh$|T*7Rt-PP)|LbdXz(CW)o z%T97dzEdIU;eh{wEC&_KV|B~6(mP?G+xa5QijFga)nZv-DxG$LP%z}_W*bYp+ZC+7 zzP`1+6Z7`ARHW@jGeVk<)0kpo>T0_yO)7(a4L{m%Oa`@(R{w9m&e!9ts27Qn&9L{) zChIifgX^sh8o=jqmVL57{Dp^H6;Iq;kt7=L*UJH?(_sX0tMfFMcmhF-_f^x1if@U0 zZp|XSD#H@P1%TN9{iod=8R=&HY5WBL*+8i^`%kFWIPoZ0s;b$ z8Klh)dw|Y|X&x)0vh&u}JS{jFn7-i{9Q1nCN;O%&OW7=TRjIf#_1YZb(etHBntAKF zf>0$@pj8IWN^@%_)o@60u9NtMQBfoZpROD`dV2*QGj`2W6`K?z6{(aA6J@kp>|%Pp zzqHHQmm^Cov$WM9DPPqQKE^D^#2Eip49D3ei-}*?jf5?u+ z;?0%F;xzepBl~^aiHqJ=2?KGs^rTX{X?C;SC3TeJ`?Plchn1D}3uip}@}(CDgxqr$ zbQHp%5C2`UuAQ1Ob)b$jjV$aV9kOWkE4&3DY zv(PpvBEtwH*AAtuw=C;7Mo1#(MGhUyK;IuPN=L)UDJIT%3dajPWg0W2VtUA4QAlOz zd66=2RPsofzQRna8&l2L4e(@>G|?9{hBK^O#&rQ$x0_CCQb$=#-P>6o0LUG_O&8~r*7IC2&YlvdYEKc zvX_Kp=)4WO&#zYwB4mmugf@qQdKLw(heJx?vJ!`TA)vN#Js| zwt9lVrvtd`?(D3B>-468Dp8N})|SHn!ntX^%^LHgHv&S;cNh>nZ?mgl6NsS`^c^x8 zo-SX{XA9!hJ^N8KKLhJOfc&83M~y07saCz%9**xNK}CX!BWsWdJ5kg9W+f`;vWs>Z zuzPcMK%B8k$duUol18{T4^Fx_M<9N`ufLp7!WbvZl5$~br8JeWdYTIytq+R;C*ofT zqmqfN{*ujANHYmmF^|L<^DZSp2mF4YkLs*;n;Nz=#&u{xBZd?RJdzd-u>=D6{-2L& z-1_(@#c6sN8b@Lot3kk47>W`_;?hmV-^ zy?#FLq+|e*7&06S^R_5qK^AD01RHBIgcySsI4B-EoTH!F{4pWT51cCt{RDv8-iOQG zk>+_d4k&s3JI(i!jf1G@68L%%M!`cjw(LdB6#b1+j&Ghh1BA!doZL^~7g2%aJO~H~ zae^6c3ja-IDry!D3$wGg(5;ivrZZ<__?djEdV1^vY5%mnHB`smn}n6dVR^%i16}DZ z4lFG!{%}+Oxx`xTO8~{Cm=N5b$KNJJVmLf}C8lA~wTSs8gKI3-Rdowb60AQt6OtwX z3r_(4!sO6ee2zs)NhtXW{XS+7+sX%)^op}A3O&xF!EZLvZ_EiT9czdMar-ze^caCn z-C11p)>s2iX-0SloO)Cxego%rYsEbiOgy|85HQHHRo2*a&Kz#1An*pGBdyD|dp>Sg zn^@YL*wg^D#Cr_pvF#s+*HA&;k_kOkNh#Z3wZXpHgwcBN?mE?;7h^U&w z_5uFZy|r9$CMZS^_eu8<_Z-O3yjzw*3GE!0pcAtEaJE1Soi{dhXtIwcwZrbl4O!xVt|`ge}DE}?6h5mCn$tDL18 z6z&x!EP-iU<87fyESLWT2Z)VkHR%0ZwrF7z1QL#6>bm$3fq{s&Va5iNTse~}#^}!h zc#;S6gB$;4ERG$Y%BBL7D$T}WkHZA<=+#w?N)chO9^v{_ zxwr$RPAecV@?c9vye<){5~M%w3PqdPLe*p5caL;`*b~rPNbWZpwMD^JJ@D>o@6eLW z_asJ5fm#U=^;ajUQjWgjNO-}NIVP2L!HKqcj!EyoL?s(Lz`Xmgv9jv?t~e4Z{YHq* zSJni$ILd<(t&B4-QvW<00fQpI@NNi=U+bb|{76>Sj%NVR;+IUfAV7XCusM}r%)Adp zUWX?HJe^o0ek+Zl7W_tYg75eR-7bQT{!^vws5U=e5wc9nq|^XK_Q`8Eg0jwv0~!T^ zWg?=JbGMF?`18{&~5~NeK+>V_?qAEC5>H)FhCyA;F26p~UJeEjA z{S#3!CQUOm=!0}kzTi<*Sh$cn?>qs|TQ}y)s6Y<6phGcYy8IMT`nAxgy&0nb#~K}# z-{|@}c_(cA31Gn=ix`~z4KwFU2Tm!X5pt>qJvY|8N6M`{mx0VIn@%}G&HWq_7AAHi zhR$LP7C6}>vIMiH($S4I7kP%wAr?`Sf}H)qhwVpPL2ujqk#e&nKCWMS_SS~H)dFyt zi9~ftugL(o+%b%nkVK*Cei0FIEB@wpJ>rzD0su#6MF4flfC(~ysf9dePAQJ+Nj zS#B})yy4)6TIy2};NUh;sitTq6kS}F1+aoug9`Ecx)dB@F5Dw&Y{*4=Exh zNW1y>sQJaeA+9dHjLD=c>GE%Gi%P`39B5e{1AG#N2>O3|RLCh3#7ncYWz1zpW>r0w zfc3mHFO5m5%o;Wc7sLfNy`S28@W>xOceQ6|AMVl7JW`Rj;1yk1bvLrI&{%aPW2NS} zWYnqK2aH31@_?ZrWtmc6ii9Tvd5BDfNjn-`xL_}r#7K=fwx*bpgNT=5!)2-|LQbLr zqg1hq{5+P)9*{h?5VF|a=%xERs|Mh**8)4o<*jQ>C%Pg!sv@I7k%4iF2n#f1NX1E` zF$KinpBNm|j2d_b$9QB(nw$7vdi%;-v?atv;|K8#w|Anl>Y^?Hzo7wXi}rgg5To|> zC^@0Ym7;PdO2rglJf#Y$Iukv$K~W@%_7|K$mp3xj(S-rn;{IvF%IG=x!s4TqF*>Y8 zJNc@ki`+sAKX+!0ipx+0y=ZJgN9E}#9MGjwaX^<^METK*EQ}I!SH4}+DwJIK5w)j< zp5i*Xcs7becA39D`dDd)234g1T;Lh<sE$ww7PCmIV7 zg)dt!v9VU5R$#0-E&@h}09wyunBpEV!lGrvR6Bnf;g#MYaG<$k(afIYD#56Kh!-Uk?{tkqbRxEJVu`&#!5UV>)XUqKbE@cf!2ai zoirag*XpXLj=k0-qROmE3OW5F+}mBZe<~8{*_13RDQ5&~PM_?w_T;fgHstFfE;6U6 zn0VS*nK5+vYU(5pv=?G4S+DZILqyEwly zYU+*JW9xl@$fBcMc!XJSIbyOYAeleT>;cYZ|Dn@FT$GHO?%9?oD^X^JK~_{*apaV)O`P0*)C%6 zIW)R}uE067hLk%t8hK&qm$-Qq%6_8lXJ*tq8p~4H{apP@ipUA0t}VfqVu3RyH}$|j z0@L@ofZ9THbs+!D7g{Cg`ZXY{No(fNin&bc0SdhJMk`m5>;8@jD#&vwXSM@OLSTWO zk1erwEJ*UjTrH^-?jnzpN_}sNP?s~02q!jWI z?iFGOTBSn@c92!I?G`bstEPgbz{3M3l72`f1_;V}-?Il9=0}uJjd;6iV8UBQ4P@Gp zz>n$pxWZpyxoG#)sa+v>Sh&CTSb9mB!>Sz)g~!3yQXO5$LemuHVDde~HEF=9jM_`4 zm?km+Q3?kRT%QI1>khiITL`*Lm5&R+jVz;vnjqU5I{gnhOq}IfMn?_woZ~-;Ncujd z|7hVAjcXW5@GA1YV_7Yq&^sx;-*Fl9e|%RV?xy1YD!}zKd%>kCJb=kNlEffgB&|bN zW>-iMSzKSkpbB?DI`-}KPd3WGxiLqq%8^NG_WKYSH2owb)muJbn%`HZB~Lq2$Q&Dc zp!?0ng($mX9(rnrS>SAfB~}%`fI|#z4YS4t8oAcD3sjO-_pDGFtw6cnYkCmGf2>>N zC=?M5)W!te3P?f_tLU1pIjWpe>TeAWrgKLP+Gs>Y#s@y zt!4Q42RKSnp!EJwZM^9at#)~H(6=3;Mq{(Vrs*DIjVZu(et+>;IbC&JJSWOUR1ciH^FdNx=5W!PgoX z9vmy>SfayX)hNYR#Maww?Yw+6$}`z!rnd*}r^S=D7!ARGyd1<~3Hb3^EupHG15K!; z!nuaW>!Faal|Uk{P^-sxc(P%v8;4|XZ;#9E0?ot(g@QFoWQ;f)R~i(;0S>M zoREhHfr~4{VzCSb002x%)^vfHDl3}H6_8v?dPc?h9({S@a*4UQl^|=_>bT5_k}dCy zj6eYsD&fcTrMpR%nLI7Fa0UfnEJIf2&@eGIHIzsuo!rl($r88G*bV_ff%(*TC7DXP z5)crO^yTYVhtH}U2esS?%)u?T+MBqW|1T~CuJz>a!Fn_wj>^~Z17q4&9^S0WeTpA2 zwUODC;r^EY?*S()@slB;8CLvLqiPxOgVV8aC*?hcj-abyY$Tf={5x|N7l^`;5O;3j_P50 zuRQekQv>?OjnF+#vj!Ljk?pOSjDY``hVlMWC%bVgBzTqfoIPowLEV{35e`Y+i|p@3E~;8#%t17q~a&{j5? zl`=(2jT8#NwiKZ0t?}X+C2dQ9Ek5q4MgEwdpGRqO#&Kev-@USRuS|u{X(GDF0`G~_ z63R6`>8_Eu_eF;2o|>gKfVT>z!(38_>tNOw)={DXe_=jasDCca-{Zy_O?$%C?7+`t zx2|ct#09exb@Q*)i0#Pz6U}+!MZ6wVbjVpM*1*!~AU5G!hL;d4nk6T$r3M0f5C7L1 zH(cKC0~g`E#Bfgl!Qm#_IzAKS2gA;RImPm<15Br{0y06cP_~uPJ)Ard>0*MX_5tN) z)Me`AH$<9zW_S^PVL+WivQ@LNJ=dp?K69>*8Etl13#k3(&X;iK+#Sl{p||0oz|8JI z8I;JGWy3;Q$ZZ;XhbIL6^gE1x7e7co3O9?JqytUEyYrrX>wPsD=c03GysqHc=rq4~ zX()9M-2M3Aq?|BQ-52j*5yw!dwcl__GP_l8XqZ?r=;suUx}n4R%8Sdsg{<31!27S~ zFh*6s8ru6dRydf1-;FZRj4T|`@?w0UtN*wykUOE|IXrQ@kxj)NPd1-4;GQLkU9US{ z94~J^ysNG>k2gzRjO;ITVw;2=zUdsH zbWyV z1AOoboEQgeEIgq4;mBFJJgd&oiMH+yCr=Tx`~}o>N2c!4e4wk7 zKL7QJRQGJ37q-Zx(M-oczvr62uXoQMUq?{3GtG;k5C6^qDlL%8J=zqxgYfZ?q60;> z@zf!4`7=+*now>0`$M4_3EXPzk!w!6v??z`Mi1IwCZYW*q5W@~mo(7@UgJ<_35wGd z>XTGgPqLRi7g(btkKNs;V#-MaHyV38LmoD-MGlt2{XxqB65%Fuog4LSes+y5J94@Q>gl zp`=H}iNn6M@G3QDLWR+3O{7PI@djN9x0=oY_Aird!!Ce?)R#F!%k-r(zDQ{|b)4$WZ-oyqb)TIJUzv+$q>T+Jx3L{hmY#>+ogbnE;QTLQ%qAOEuB*`!$Q$ z%w|H*49y@zEL1p2*OE~EOhHVSx5FEBzERjg>robKB4min%60z z2`rb|WdDLl{amR;ot<<@2#H9O8`^vw%t2WY-!NQZXP;L~GA49n+#`P-$|>RV%CxgY zLiZtGUMA(?HDJDqV}^_iB%Q*xxT);)xB&4ran1pTbu9(MXnHyZCuGLBMtx?34xwZ( zaUDSXBs|)~N@ICW>nGlvmb%^Ek#201m}p?tk{Cuna!6r@LjeEgVsU*NNEuHtz#u3? z$ixsF%4m6whnz&gD>IqTBWsj*6F0*p*+(8`9C%TF%^exE&OGyP%at&H)K4Nr6EF%V z66k_PwpJFJJYi2AA{;{LK^N3N6oTRG`!!L(sXX-4#=M*GrD2tt+HD3l@5Z;J4XCw_ z1gl7Bi&o`!;L;%k@uME$d6wTIMyX_1@mJOJlWT#}H|qr+VTvi9G9uWHJR=YJbcgVj zSS9z05YU}OR~#+9NDP<1!mA~1L(}H(3=UQlfqqj;gur3)HBIBSe)8z%0QXSezAZPp z7Gp<2jRxnIUfV1kVnuIs9EcOF?T|5dud6D2d%MNhh4a)X!%p=j?ZISzlNzl#Ff|JY zR}J)!_!vI7J=h}A2s zs7?5sS8!gf2!72Gv_B=v>GEV;OhE6}fIR0S`&r9aeuedsy$VLc?jKQvt7!jZ9UL;S zE&O)lG%xeOBm+G>VYMgzl)3kF=>oW~Gj?~*g7_N`LdAvES=c(OM>gI!zIAn?)W>gA z?P)4}Mt^cY+C+0uhC5SB&rNK8cVP6u5$Ely>>sG90N1j){aTJxLRYkk_a?!GaHvvF zbzlXv_Vn+Ns{!s&SLpcn!2{*JgpbuZFK^~|?%-d@MCeI#wQ0{edL>@P$b|AUS)7EIb2>u=Y}9$4O! z&?9{^XHTxQNX3I_8=*gVs)G7%nCz1lmo@M-i%l%<;_KOX-*tc2xx8w%`%MWMW`3Jb z{36IEATG1~d-8F!r;EVvSf8FZOcx19XXgGMGR|5*E|I^{?}-bQq8%SSRJoiZZQt~X zk3rg^fd>6g3!sp-4%%2(>Lf8T)*!fKY-~u9_~U}Mr7yhrVW=Vk~^&@?`%z){v zi97FY_0)7oA_kB}#po<`MQQ(9Sx0!T6p_u=Wwf!{Aud*F-amc2a;>HIzoq=mjneV> zmV)d4E43~zN|%}jDE!EqT0J1*Ly?e7`5aABNDO)auXDW>g0Dpc4@IZWJP6#wvV;+| z3)A7i?2Ur-64;3k8L=8K+T|Gpdg~=)$b1*--L#1j3r<{7HcL z>96KcBhrL+zp%t4@!kbjlq1fLb^m?|7WNJxum`hh1dC9r3=-5c9Z?iBHaIO6beJ8> z>d3s5f{=A13p7bfh#TDzH#l!wn5&Gtq@K_@vb;@&I-9r&m{Xh~a_>Db+q=_(S2Y0O za?7>BFFe%r_hqL!w>Xj31Ehf=;<|8nAA=4aDYjBfgwfo(;zogmPyf= zO@s4SQJJ)Wlqt$4>2m$2WZh`ZQ=<3nR$V#g3Fs^R)f@>C@7<7jFC7g9@pC6r2`U%L z8X3OWb6m|}91mt;ji%O~2F--&R=DxbBSA{yW%GC(J!np%SdxC9Ng8R)MX`gyEUc{C z5i};3N(JN#_UK+6K|M3C=a{MdzGTqNudhwjUCQ;OW18df3CB6&Z_8u`)-~WBi3j-q ze=gZSy7@nH%^CPRAt_)OG+&(614qp8eG>EG z`XusBG@*!@1G z8Q8n?VuuESM| zcM&Tf39+u>A^D8bv~Q`aDcJsr-p}LQZ=%_5m!LfqNOh)S=ss}N*qPVb+3XI2&_%h` zf}Y7i1;zxhZ;Jf*FL|?9g=t9~)i1mAo@zbV^aBO7A1mGlce6pk86aT+L%6nd{znNF<5Ej8 z`D1d0MuaBrEBZHurXO-e@8P>w{LjQBv@eIgfhh<={=Yr_BqC~1;@4DRSr=6Q+nl8I ze*uX>5+LK&Dk2nbJrPG+5X#u0W32GES-Nb~KT53r0rN0{U4Y1qhN z9Piuq6mLR`?eyOQ-`-?bXztt> z@Kt-10U337g7K6*B+#Rr;>mh-cIqj(Iiha3h(9Ue!UON@HIb0^mln0wzzKYq9-eQ9p`!rCvxf-}YP&)HhpVK4W1F3GU||73iP^;xZXF z@@UYoQnT;|UzU-x(}VKk)BB&m5EX;)V8Y#XAsim1OT`5*z;rjgu5~k$GWngb>)1FQ zdUZtS4rTW3AKhuBM-DGX7nX7`FSwB6ELar@a%J&p<(t+y>^0V@QBnfm3AK=LA_g# zYHX5y>HnUq^d+FkzXs@i?S0b^fKV>>DYqf5gP}8{sdI>G%dK*G_((T&S+Z%2y>qJB zjD3vVf}l-kU~?|&&M;f+j%nC}QX1I_{U>Tx?`Mj@RlDF`GhD~EQ&y}04=a|9u3haD zUjye$(TK_>^&Q*d_2Q5?b;Na%Qn^)N?XGX9Nd~)&b@B@-)Bf0ksKkUQYjjvJ&!)~8 z=C%|WE;0<(c8!&Yod`O!n&U++y9P zEmCOWS7;)^oplivk)!Vr4L-NobVx@yPQe>2=r9$A+?vIpHl(^jag4s906dgd(q)zS zwVrw~S9{iZr}5XmFPW6Y%TL_Ooii7+!8KCjKRr({DD`!({tj?r=^-h5ZyV~l(_aT5 zx4Cu8>}`(W5|>2QNcDD&#m2;2`XTbW{jICD@3^T=Kb;pihE}%0?4mIeuyp= z&Pj&tY8U>D5`ZsTo<3?XViRKLBg{o*jt4rW(kcx*Zk2Kx*?3Oynl=hs!y zwZpRx6*CR@PQpt8pJAlb!S^A8Gs{OvnK9QvU`1`DwrYDtW@e#d8B9!W?W$ch)KLn- zS_s9JNc}BIU682tgfbru?zKfHWd(P;ol;PTkCemiB0%S~zV2kaAO(=1~kcqaz$hK^fBkNEd zwTSsDnj&&5*y>Oh9vxySxlxa0XD50|L3O5stO>BUv+7E7moiCp%RWdUFp__k zkSu5_M&)gmK@CL&s0t-d=!*J9qKy2gX8%1Rq3k0ka^JWa#8Vbc=(CQMHC;Z~5%r1z zlp66-T*)`Q7tD@tOkP*O2q9DqHc53SM@bd+@(W-kVj=ol|K;8p-XpPHBqmvAF3KR> z&^)|5MyLEKz7S1U8Lm2gh0D}btNWPp#@%$1y{g-yxmGEzPVAAdv*9b~rcLn2t#~ zD;wg2F3NJk$KXK6V1)p{Ow~{w4?WRp$d;JuQ?-V=`btrE2*-9Nv$vFz;s?<9y@QJO z2BxST2P*_~$C?O5F0PT-&2K6-Q6l3l;v;~O&QNssPD-RkbP`q_=yn1p=}Y^FS&&IX zehpb}{%m-^Tl~8M&6&rlUeZRTP?6@aMQMbj;|K)30KNjUN+0s$uFbP1@IKo_{#NIk z=J03NJ*>k`VhalG$=^mbX`i)Et%vsHp)VRRpCTrR+SGSyhK$K+rjNSb{Yq8zz2&^emWR*?)mU*saxKRMB6rkjYb)yO-H<IhS$=H(&N+))@!8pqz_lDBjBDin;+XIbb&Ndf9*04PAy#RnuLKdQn-b{PLI%;UUG(Ok>?Bd$NFa;)4k@n5mSN`;bQ6 zb=%bA1o$3T241{myrc3fnNfW=MCZ2Y>nZXdZuW5J_;|V_J6ua>4i1{=M|Amrg}obP z_)=VYt-qge`yJnDV)~i!y`+s8ZneeczG~;7^*9q$1$QC&nRE5^Y}U|TcVBWaa=_`! z;KUMHn;UvxEuX&k65~eGh9e@Y2d>nIm7ROt?~ifI>R}h;J{79|6xo#_l$&#L3RE zFJU~ZesJ*SB)PnJd*j&6_&RFa8Lz}_#7k&Q&+yW>9C&>-jV1eEeLk@GPlUd^<3IEW z;x~5}oYQN62fvN~*bi>nj5zhk_Bj#*XGlVF>T8^icdjHADOI%cKwRT;tP20|KoVF% z*Vmta8fqcAo2}}Uk(pBSg&n&Wq3kB0kkWeAAOC_oT>MaWc6iRc>1!cG%DQM85jT&c zB=vUxQJZEDkNg+F4+UZgHvWWdr2@SfcZv1Yr>B^cp*QZHuSuKsRxl-SurDhxhPfN# zSBOlYxj-76=%=iTd0g{=)<0oQ9F?&rNM>^|ABsNDiN85+-eU5_(7L~*pWty^Xv!kl zJbc&pEG0zm_in)WcBK(<X>Vx7Ciw5w*)Vzj1@#q`BC_{I^cu29D+VQZaC0SgN1c-Bql;lHLnAAir( zya18DUBcmmRdq>zJN_)@XMzz6|Ej2$2Lqk$P7h$u4g(e7=BI%XSwbHaJxD|}^e~@q zgodd!tDbe}uO6l6;`)`vUL~&PMcvue^xUemcNF@usN#Yz0K&#$FYJ{@ukrjZX%rqI z5NQOo1I7oQz$fdOX8ZmXmF@FoodFf-^MuqomK zV;whbcvZ!Qt%?E++2!zA%zjv}NL?fxKim%3Rh~bY3>#Q7A6)c18k!JzIFm+;rG$2S zV^f5OfbU+lOwFjcoaEa3xd~UvtwrFKojk-#6mO5eVM>!~}e z^~8k5)2N?bLWjKBhJ70PCYqgSsv6kve{V{~rDN?r%={-13@S5s3mK+O^5trjARrBi zzIu%SWE;06Pch}cjOgFXG0uWI=1=fVe*qOM500Uov5Pty5?DW@{->x@yA&%G)Bh47 zyn@0&_Z>C?eHWD3+T7PQ**oJ8pL$c}&{Ho9v}M^mpd^=AKyKcQzK%Yu?Cb&?Q$Q`z30Gm(a~qtrODeM z91&ugf8*n_L}FaFFY55osKmQa{n5^-KTO-2wSpVyTwDSIU+~KfsKqjnO?QIbi#`$O`}!^(1q6J3BV#7`{Jb8y)A4I2Mw1kn`ZE zR2jenYJ0Hya)`FOE2O*>gKLT2&JcS@Zlt*Hf5{8(43`?zRG!-$6Ccy zpdyAr3s5~@-0=w@e=R`Z3GAvJ{AuNM0z;y=EPl@8o-A>WP%k-f071vd7nh zx&{8=^ovR7t!)3j;(P)MhnX4t@k{WMs3P`-!jsQv!9CwikndmIB6$^$a8h z_tM8k4W7e-ERZC`)r8pn%o(ck><2Z=2o%(PmmTCS8XjC|_2575f4^o>{UIiqn!ykF z)P$=3HTdBmS?zbvc&ex29!Ab159zI!5t@GIDzh_~Z-^fvkn|;F9C%bo~EY%6S5Yn@As?ql-OXKSCN@Z5CF<(n^}H7&4<~#4{L4w9Fp5Yka(|j zYq_Z1YWtw^=Xu5ztNqE24aV&;5AqamZeU$$U@L!YI$NkgS{m%@V}Xn4UC z@dLs|V;Il={RAA;3s2uzjsG_uwA=TuzsExi!$;P3!Zk!D$48{@mT#n-p5W(D%Okz( zCkLUaae2-DvesR|@j%1lF7DLun2+Ul1!~uU@N&UW+3TxFkw<ZVK=!$o>#%0U@;N8X}#|l9_pzw6VQ7Y@Aym%icqMKPVl#-oZw{0}W zzh?APjH7FTdeK202u_oiH&7%#led4#`=TJDzs7D{1 zFOyP9rrTB|{+B#%_Pc4Go89({nt>&C{pwWv?pLuCXMgt{-xiI1qbQ#EjEqPoOC^Ysu= zO!&u#k0?1Qp*g|w_t&$)k+2Uzd7-j%f^}b;T$8l4)>ekhZ%u5U;%<=g07w(e)rosjY_2&!az8Zy7Wlw zvB75si#c_FBg`_SKacO5%OlU%HcU!j`WIKu?{nA3ehs2_b1mLz!Bp#3KHz91rtkXs zawpC*A<4wDGBbnJ*VhLP2d7Lbc2C@fA|8v|S0bBrXuh%!EF2geAMbx$(T}sPX@hj% z_AI>|huhxXE>kQFThF<@yAztqWJ4KMz!U-&>k*FvyJHMIl~xf=80dPji~u~md>v=ybTpn~6!;s4espRIJ@74f-F=}%7G0BP4tRoMrZy8< zHyC{#)# zL%%UAeG?cZ-8MO3;?8aF569BkZLv|Ykm(GzydKX(fwev#PaF0Xmp47%YVT(&wL5dZ zoO7gg`Q^LOKG7%z-VH-ww{)ms1$)=CCMVZ*!tx<`DmNEs@`4)uv^Srn!s@#GLW^r z9{bbT{3u=)W@bkBYaJ0WvHgDw@&Xm3mse#~%qE*P@z}$;Q-WFBFrQ zn;Tm?jb2nmcnEXGLz%J$bXfCRyfoR$5UA}{@ZX2oq5ZTr*pz6i3o}4 zOS(W`r`%kx==%==Ys+dUm9u7nMRK(`cCA9M_xRqIOp7(T+SMedB%Q$W$Q1uLeqiAo z2X)fQ%1VpGNk~llSCfhbOmNtmVg8;c(00>=hA?{uZ!Yx*nHr%KVGH2Az~ZjCJRvaN z$3^9+qG|36+NPY63R|^?=t0q;V-<&x&#i5W@AdN-vx_8o{_=&nkiybo@Lc6YhNQ%a zsj^s?26_I`emsdVJoK%x9m_wvJt@1H-S}=v%qRTQ!CVe#R<2qbdgcVbiSUKrev?PE#h^aw%eVW z2_^@_F{(54V^zSmqM$i!Rs(?eq}m}Ei`}DFvDwTip%ONfNL7iT)B3BBO+srP$MET< zGlFlFxkyArBq1j^oWtk+UlpooIyN>tFqHUjk4k;q@r3Qm>(b84asI@LNV0DKoO0^p@MJ)rQe~O|o+CwBJ)rL(AN~CSEA| zNXdA1OR^u&HpNpI_r%#t+K$KAli#wh%ac)q`odFV3df%li|~*`C)~BFPBb}x?L^vk6K z7x-$!Xw0%1|5NGo!?vTJE{ZToH+@O3eg&+Gl%)%_y7-&7e#aCzTxrV z!%LyUflTl-+5>xy=^Zq9<-FWbkg`kFM(}OBJS{Qi40!Q1>h))35!6b90(Aia=kc@4)EkZ?mk~3yFZE9%*3edgR}ytUZVI zwOnd7Pq zba&DP4{Armu4cC(xqevV<2@abeM@92D&g;=sgNeYv~b1aRgNa?QL?Acak2>@yrUuI zNe3fxw&?4NHt@P{hqGBP=lQ-rRXl4HMg^I;)7oq_Pn|B7ClQTLQ&WYof*t^;4W<&A z45Omr;_?O{>#V19IhVXmGS_(T213CTi8V180Vkm_R;+^raeO8bI_!Hw9P?b?=D7k< zxC3B$AoJ7JMrred$Yid-PV3j(1%js~R)Nc+zFyB+uAf!W#Ph$NFXnlM)?SJ4Cv6fw zzAYUmr|5rt-qHUmYnbQtJ^CclE^5YESYrMinvnSZvmp6=+KBlV3du(hxg8k6_J{BRhZLQON>2^?k zm0RY8A1Pq;$6wF?pisW_mvNyUJHE#IUq@0hq3Q{V8O8?YhG_K`KLdDOzqI(K|+|xO6iUlWgoG_AK<8Zk0@?Cz#?BZMmz5oW_T#6w$v+b1!nL1l})Mj2Fukm$T=~q9~KK$pQv^R^Ny1e5qp~;4sQK_(xckd=(uTowB~E`alzS~lR0XalG~J3_Qb5DP4gby_p?n;173vduA%2DWNNPc;b;FBeXVb#v z>*;Y)Gq&o7mpOG1n6BM#1a#vn!%9;6yC&=l2^zKaVMm=ZnN{e2Grfvz2^!?V^Q}S3c?E; zjDnEdSx3GO5h}&D;c`0U!HVZGgybXQUfPDUdr%)jnD{aqjyBEccVc;bc{(G)lxVgP zJ&W5x&Q|?mv`SW`%v&4V)AoK!7aE%8GUTF9g*z#*FqR{dP@eMCdH43rtQFl0jZ{>g zBWvZmu3U9cgFyl~A1i*BTbz z!UN4q{lYEOq+lPfA@NZP%UY~2YNUS1|u^mxxf zEDxjICg)OFYxh2W6*c7Bn%N{{xDQj||3}kRhDFsyYYPyiLmH%U;G?^{Lt2pT&Y@E} zha3s%p+Qol8)oQ`PU#$wZs8v9eeOU09A=*#>+H4G8#S0drY0lAc^>xS1la@F^#)bV zG%YXDMl$-hirfftgPc@h@&y&r5Km761LHrO%Q~Bi6|dx3<@?oK->zADuV=QTy&fej zHuVkqP4ww~A#d#^%!c&S9?T%IhM-z52nhYlA?J2kj?hfecEnFL$)|yOS6$d)rN(;^ zv6l%)>954tjD>%n;YNLxflT*viNKE&Hj^n3+?*zOo>DNsb~)Xs^s{f?oD)q2x?yTKDNhp~Pxu)PuK!dKBwBUjPH zI16vRxZ7qyD8J4)*_{x>uTQYL-2bBzU3@#D%%tDj%m`8PMJrUxH?GcF zN8f5_#w9pLSNSjO{p8@V&|4eDt5rT2!}I<**B zgh1&-kv>Cv6`P5a#bZ_=#|UM1Ka}Wr6WKR-^ncwh`jh76LzJ%u@?{!-2evLUW_OV5 z-ZrOy(hAZ_Cha#tv~skCy_zeB-p=V*7^qF9QZsNHQVepP_7fHFEB5bl8)gW2pIHo$ z#bSM^nD8)>muEd=H{G>2o84K@*ukb==2I>cU4%X+Bdvx$E+n?%M8AthLM}$}IujUq9EU-ZZoz>C%LIHMad|e(D9> zi@D3=mCs4I^_m|~7fd2wL?6X*jeT6;$U5XVIt2s|v=;v3pZ~U&K497#kstK*SkCv= z#F+>!qdronKgE_HJpIYejNDW9<77OUHGB}4P7u_RPHggvj!U8h6C{X7p)xn6TK@(m z=*cnpcNDJGA|nxYR*9R^pjv+{+N!Q-JyE|j<`tIK^X^BZM{j4~gk_!a86}?;cZUO< zxNF}FZsGYGs|)v=y(^1j1Wbsbz_?G~>1;B7g#-P0-T1CP1e6TFv>$!SR1Av_akV}`r+pl;G{s3f9FkPq6AMO%$I)h52 zfBVCU!?e%f>8^m9JB5brW)OQyv#r zM9_U-YnY^rTmf0*CZT1E=+s$0jlj`r%p<{Yitc6x19pLjJlEEPeV`tuS1?2C5LS>6 zldThHTUdKFZ_NE(%r}712QbTGoNnEwmfJTW1r(I5vQXHy1Mc=?@0xs>lj2n<~fl)C5=N`YTgQ zkwV2uT{^-ezz#%}e;n@t)U7IzYw5GhRnLxF6RtY+WT&79BfN&dE|EqU>Eh*2$u+ta zDFkvGvb9fxD#=Zp4^Mp4h@xTNeAk%${nyB-8J=JwMSU+UTLNi{55LO@(q360b=xcc z%51MpPT00s#q19m>535=ZCp&Yf4JBC`U5pZKS3r`d`efDr))iE!a_IN1i}373(hy) zYNIDCw)}!`{IxdT=)39fC)L{p2v@|O2tb0?A2~xIB!0WEF&GsqJHR&`!0U&(H|csITQ?Mm6_r3wz`l>(0iKk(;_a&8}wZ@voYp zm66xX0BE|%1??#o{k%6@^NSmEU4(PeFb4p^j@M0DphZ4=5-`;WU!9eTd_02D4vuD8 z;^n98%~o4W*w?vnM_VP-hFFxg+M7XdF%9EKycPzlI1 z%{zO3?;oZXxZ|yaAPH|+j)lT*Q7Z%m@*`uk1H479m)#>o2qNlvTy2P~jjU_fo!LVp zGcKXjbpPaWHqDOQBBJNrW8Ff`Lhj8fDf~(8<|VqR2Q` z3PyFK&cMHuZdWI6#&Ny%)+M_f#Jh-vJ(5<$4|xpLp9$SKB#_57q^_&0tK4<}k9A>0 zZs485(KzdFA`cwbjn@ydyB)}1(3>N~BO+or8aPhvW`H=~cqptSdkG^kw)qo#&cPlp zu*<&k)A1V(E(B+tj$0}MLtXRnKfOBXqfmrN9(aJ-?>%)_q4XRcPaYupiSyCakixo8Ns z^JO2bti~Bp=c}G`eOLMS)oh33;na%4&Zw#9fiB>96}mPeWH@j${EWLr_WIAxmvldu z=kT2CXUAl|jnkKC!h(0|+{pP?9GLi%t=%Nv_f@*JPD{QlHy&R~YcNJ2e=%{XXHIWL zG{cB7M;>-QTUc6|N^Ecz-H5iWV}k8}6B-BpCEUQKfWt?QJ`s14;7BYekyuKJR5Bojcj+=MOD;2ZE(n%7%F^*#m zT+5G+fTCL$aY=3~P!x!0Z##vPMc^d5cB2px1NoDd<%8dnV4+p0R~>F@Q7^WD|aXA5ImISBXG3o@@258b4*$G`Vv z92^{QK$68BOtqYAU}JuJwjII!PKs1N$=v^JkP!}O7R3X)L?r_q=?x*fuIHnCL?6=N zy zzt12t=EfT4h%gFjS5MhqT|FSyU2zG;W!qL8xYMt5bkx(;3|wlrRx|%LeffWyFT&+G?!L2fnFc?K2T~cX>$WaWKt6LraTk>wQ34w>oy-BmwC@Pm$FsG@ule zqI|Up6fAWuuWor97pD``Qil?0EnF{_n`_$p;qdbNS3$(jc|zm#dF)AO|B&0#(KC=> zi`~>vV@Iu%9avA7YWEUPT>G0FUx;p=Cx~&C&&9ES#FX{6Xqv3D{JdE$_Qa{f`q}n} z)6Akcaa`iSM_6F?9b8URjMMB2K}>Yn`Vo3RlXytu;eFE)In6=YvMbTcxI8ZI;SAF` z(wB|=Nued=vk}cWeQ6@%(sg)t@@?_^cRiY-gf5jke(sp-H@ozyKoF#)T7guwF z9&c$7fM!*_o%4tr?N-@P^NB(D2nMw!3=rUw2?PpNV74G07GqYUIS8hX%+1!T?W#%2h^X*Yo7jG zXXLl2TV(UByGxM!4cCmp@Ox7MLeO}E^QVrCTUqKGAU8J0kZ07MjqXwZpaf*hAPJBSrnd@dT8NQcl_mjC{gZ4IrAJQ6DCH<&$ppL*w87O zaar#qH~vguYVmIlLF_l)!!T9sdN%G5dF3Cs17(1nvI+Ug3R&11Zzn8nY z*v226=ZZ0Nf{q2W#maGJp_f*M#009BqYgY*KJqY)FFV3$Ik5qhaT!|;X07X&+afeK z5muD^kJ}9eG#k#yD|j3q_VV?!Sri9EZ`x`ssnL3=cFDWNpSH1tWV&Y4hS;#(;MJzW zr}ts6BJ=DKte<}Vp^7uQz(AcrMy>)qKI4&PEDBXA{XhoUk$~6n0f(mY*|9iV6jAT~ z`<&>))=Nn}R4K z&lCBVUgVnpR4->{y}G*E17TrqH`m9l{b>>zPeH8R_&!F=L1A_LC9lW7`S&>93Q^Z+ zIywKqDN{_47(sW3}BHT(UNpNyqAu=E)YY6UOK@^9x1;6>haOB ziRWK$6D8Hg&qS8LVBr$)hQOF}+QVpRi9dTSId)kB+Bq-qR`w==POeok!SDs%=RrDI zY9YcVq}hDmP#i8BS{$qf6K-?eO$k}^YnhnOKdR*!Z)S*IlhYNG^5H?+h#M8ca}AO>f#ze zeuI0(jW9KD>>^eH^s$H^j@~<6OHFxJ)p&JjpJ-`gFrWJg-D$EY>HjCqhZ_Il@N-dD zwcKOZ(!FSZlht*ewX~_*`1k$R6aNdL0is`)S=kZy0Gq@T4Vqrf2DHAzrRS+Qlzs-HmU3l5z)mTGB@TBWF zE~c|wP=x4A%O)1uq$Poc=`uujow+mMXTOw8WbhGBWN`bhyILA)KDNLln(kVaKk$W_ zZ4?vJ&u}N@c>$8N8Ds+4f#DZ!YIV%(P#}YWJoAYXyA3LyacxNOkvDf13y~jxqeF5J2smLrpH~;{_xMIvxmv7bMy`l=3l?ht70uo5AJY7f6U9|h& zZ*dW|MSPY_G1PvB=+Y=6yq;WGP|A{)n}T)-WL7=chl&bXspc1K?!9Gh*RgyyRhQuC zOXQ0}m^|^blBtb02IAi4`IcGOxtta2pk#0;30I~*O&M{v)gTE5w0Q3JR@I9SSN4uh zCLNU$G|KW=D?B=Av_fo)xR>!6>4e}9?37~=C^fNC-zTd0H_!gsVvE}B)Yh+GObXPe zCGN?`Amf+@nF`jQF8J;`azQpW(Z%;I)$!2^|T#CZhKS!Jpc`Lvi3 zphYBTMeLbb<;*>=`N1b~3D|SuCY-QyGT-6g`4Q$L+Z_CUya#H(9G~ZCq8L4c^-iF1cAW{ zVEcB=f@FZ$6QFLy?9`$$$q^U6JvK?i8kUyEXG9bZpBJ?5s%cE!CD>Yzv&%lUIa-Rn z0X7yVYAatUHr{SqdwA=oy|P239@gVUJw=P)*91d;!8h9HO$Rxz@L}J zz=oFSp6q~Sweyqw!XCSDmLJRCU)e1=O&Pbq+*?5*2C`0%&yZfs>}Qz0P7kQ^#43Ie zX(ocAAmiH`y4ja;T*I|#gtK$w7HT^cUtYr37KzHyD;+1mhNkqZ;JdwflGD{kHumBL z3jPvb)A8f0xD{^$|}vP zazQMs23M>rwQVhc+;>|9CFG$GyM}4G05A*n4CT@{8E<2Y*%9QN=i#y+Ot0ZaE?>0Y z*8rMT*Db%U;U{=Ya=Ubb@Pnj1;xxoygQG>%;Lb)G78mYy>FnSg+-MFgTz{`psj|Xi zRq34Mf+N(n)vtrW_Td0nH8SRyD&0%TDG$FpCmyfLH*BsY}QoE}B`ICH8Mi?(*? zc-3>DP`5$^bR?CpH|UrGcO{d}%Sjmq&FjQGW!Z1(Q%E=iFsG^~er$8c0DuZj+t2cz z%I%UG(-Z_n&#IG3*I>X{fd~=}DQ1oy!(#iO3f79mp!39|7~NFlRPgLbvlV36E;4x- z8+?uD?gl;=O0?YSLM(mG_m%y@EiNk2O&N%8;GC~Q+i44?QWDD zrH}OOYPt`U>o1oQ6IW~=Yf~9#<|CNOmC8dT*Vlw{YoiF zdlc#vEOvA3uR0djJPOqkysuXZL|dvmQNOZ=4>&}P{CtCgT=T(lm)HC+816!6((KD3SaG$bx;{sqX)$SfDSy*EzQ0 z6Tn^0(6;qlsH$z_U;{f}MZ=S1h6+kfg?D%T+Nn23AN73-`4PM)fJ1>H<@Eu2ZEQ%{ zcjJId?U~roL+KtLqmi+bsMi;!cs*)jC>ol81RuA^b_+ARJGwS0)X2iZ^v}WZ3ZMu5 zjBmvP(or)^vU_uPD4sycE*&1kB+YMeVsddfr?$d}npBUKAokd#nByD5aF#1x;AeP{ zoSqovexDQjBMak|_bqkT994#ZUb>)1xgvTXT)>)!nO3uZ&h>Rhygy7_rFB5>!CgQg zL0cEp;jqA}ML@%NZaQ8Sj|?Gyb@!@E?s4HXAS^AHh}UT`Y^ztn`y?L?8noba4=6J0 zL^JyY0n-4)Vp76LKoB^(|2EnecY;Whd8EVk4wb0Bmxe3F8S_y0#hvmkXMWH) zgLp`1KB$mwoCJ_CvuMWoM%|fTXTFPPEDBaRVKZmQh|Wa(mt$VdB2o7b4ua6!b;w9W zIZLom(|ujPtFb~j(44fK^|98w&hXR?=$n5Y6&emwt(gj4m9T`z7=#{@o-YQqZ8%b}ny1z_ zLhJ@A)M!1@Q-sAXohlHf==VJt04b_TdgDZapaOXX0G zHa5Hls-eu13eHwE4!kWo(EhX3Sl#e*qq~M4i=*&tYXg+%B7i?1PGnSC6)ig>l~1eM zOP{B20wCeZ{b~(mz;uL}BXk5;HJXkbI9Uwb0o7G6psWGbS;+pfpCD3<&W-ksM7hVA zQeOZkVuoC5R1@e_;LTbXCCs}W@)1~&FdJ_dcZYD9bJ)I&ylH-Ov0qDhvC>>enI#g( zz@m`mG5NnwO0p037zh0=dRt&+s%$KW$=GR&uM;;}Mx+XtdBsIU_9_Ek8puk1G`qMT-O!Z zW3l`y_I1I8^XAHRhV8x$nKgPaI{>ZvRvan`m6q|E)w9+sB78eb4FW0B+n8X}258WK z!RFUyYV@%DrH#iC$x_!_D7^Wo`ik<~x&ZlEsd_`+U{y$WBEp^KT;$v&0hv6jc7H>0;6yeh1wzBE4EY$d7K8Yz^>1KqdqwG{ywL zWW`7uY_nQ^wJFCJZ2~{$zDAOp1PBa$=|)t&=ykL4=wJkwqDm6Ljk%5q>^5=lW#;XVZxIWw zR@q5h!*y`HGX(I0LkFvyHe$^028PIx>RCcwRh;?b{F@jRKforX7=OL0Fl5#}EJXIO zTr5RwX6sq~aZ{&jVd}TQi@)enQYK7HOw3D#R-{IY4B|$Ha`9#yP6sMhnfCPh3H1-P~}A~N|r)CK4p z9$3u09-R~PI4DhgE4XWjGMlZZNBn9uEwV!VtVGec(JA{)>}2T!&>k2w8K>Ue%9C1x z(kwzS&8>UOLSC*dly&DA=;pjd94DI);~aq!A4sSaWDsc?vfGgit!}SWSXJev3IE~5 zpqI@B5*lXSgJMq)p}dZZrPC@Uc5bdW!~p?xD!N!hyToP?GZVZ|y6|O=T?%c6`kAIP zUz=2{Oym$QhKh*+u^eL}oxyWFvQZJML@W63rw!%Zj(OT)BLl%5LzFu*K*YH$>fW0p zWE-+?eDNR0*HJ=jdguRy=!u(@wkcDlhew1kLK4X@W!$`1cjA{S^>;$4v3!nrFo{?WDx^Hf1afj5+**g4*D4IpK*tQ zNSr0bl$qNG=!rJ&^|TTrO69;YPq+6B49fnLT6Jd8oteiU3R)ut^%+4$o(fh|rFA}WZ5 zwpqgfQFWP_0F_gWEe%7G&Y})CeI|uVevd&ptms_v*hb}aXG3d)E{5HH(cRhbfEdAO zG9)3UINZiZIGDP@5H~;s`#>Ib8@iI_b1ctJ&Um0W50f@yVW zb~2U;ph*ke+dLsARa|nBF79nQ65VZ2T7XGH#R=Psru(N_SN43EVip@}aTm3>N1n-WH+ipotP5c5UP z$UuPYWi<+-tfi$l43u&mu^H)cP%`vV#gh@eoMtxabJ&ydjGMO^}mFLBScI*`g34N`2K-J2#~*3c3s5 z!`>jrpfk|zJ$`SJmBDO&yjNkqhp8#nEO?bL7mEI3@4%&KafS(4O^uN59_|@_wJiWdqQLFu;%yO2byR6)hO-oy&Ry=ap0cb7+ADRC%z)Nta%1!(l{uPg#goI+e z%XExk8#EDSwfnl)aF45!#e{3)?U#d?Aj28c9^aam(dFM_hMV-Dms7+=BcNw#g%?=lNgxx$e`{hNo zhGmP{>umfsR63)GH`?L%yJHjWwe-O;763GEzRibgOzg?u60doqyQp)p3egW3Hh^pV zEYYf|gwVZ54S+*EmvEWqb2Q|8;+wd4Q>yORaz?*4IHBAAD$yX_CtWYnRL12w=a7c2r=ApmOXYQrSzp2VXz{F90bDN3;yLuIt04C z0Xk~2+eS{6VQY)Sr^T|6%c9&NXfz(u9zID>Rf*Q!lDKDZBz0k7Q|BJMPLJdXEP_+n6ioIht$d+I$N zhL`CHJJ)+Ii!aa1s@u+D4xkOE$E|-hf}ZYkFTY=`ofI+V`fH#pzmwBLz&S*doSsD- zozz!|dkj(sYH3$t^)a0OqE{z@AN$hy=iq(H6?sSZ1*m1Ke8_#v+9cjD3tZgjNsc8^ zdVDA1ib>>u&jD5!KHS4^5PNlyzxUU=zne#q3-#J1Z=_fq>eIHltjDwzq1#zjr~gb6 z4sZQ-v#PZk{#7SLdw?>z>IVg(s7wk>q`Cz=7sU~hdZ#N0RZz3&0VsGL%wNRULY5(H z*yWla%^b908`Z^pVPGuZ-`vonTP(A#aQ{X9uC1ks<&$?iC%0>=GBBfzL>uvHk?2 zFEZ!UG_H+N9{9E@C~H}08N2U}X*VzZ!-n&`y9A(}>W?DujewJGG>~%m*a?EtPZ^gi zNX~83HyIs@5~Nm8n|;H5E(*XLW#X0}HxtBktcVnLzo1cmWP5=}jegXPM%DMvt|mX8 zgiDsbM3w?~5Z4PgLP}c{|Vy2!g=T+}b zOc_;-TOXh^I~^M(eR_9TU2(LpR@6dYcKT4Pnq59yf1Tw~q&9*jP`istEsPk*rku&G5U}HnhE3@J$OGTQ0Tis=#olBuz#?hZmpd`& zujH0K)uRswlFVvMYCiLSt`zt2e5*Eb6hb-y9Vb0+<8_S#v;%g0Ir_B%!d>DHt2>RL z=iZ^Hn4EZccyWMrM)yD0BS9)^aav*F<(oZNLR@Jn6AZ`-Bhq&c_#`#p%|Iq^VPQco z;;ilm2N($sucONY00)veoQ8_oQ-fPFY84o;WD)6S2gESB1m3WlnZP~^o>kw&{~Fry zCBgq1%bFX_x^&(-XCWFM>$kRA03&Yu|)e0<@G(G0WdJ7u@I-%8C!tb>yp=8?2z*RYwDmBB*#!iEyE^Ss-rC;!uMbR7c zl3@UW9S_7m^$a+7lI^30?OUG;bC=wQ|Fm&c%raMnSr1q1t<^5SG3IznS_Ie(}4B8yEL%E@rmqkivdPL_Ar^fckVkC%phTri1_SC%=5~1?|EbuRlBb@$e(& zp+fSutF7A`PwjR;RDNUmlsW9i&qe&Asg zU=3Bvr`)goyb^GC($B#*#56#-WgY?zf30HHxl?j!Ly#B%H3)E{dUY)^l})Fx;YIJA zb3yydL%d+Nl-QeY;WWQ^BhNUte8AI5ms=i8{=GY0P;`wR^Ndm8bR8U#bMkyg(yR?x zH=KF^dx|ZO2&37lI+zRqFagjz>cLY}Mu{ZweCzPJ7l^!^nGyY?j4Sp0w?VA%{i%`) z^CW|^V|lHIkrs8+j3mH9|zt9H8P(ap8*n{?}FR}m*%WB}gdniXmMMx`TnZcRy||Ipfa;YAxy>mOt>V{SA{Ii)3B!a zV{Kl=G8dCTq`wT_wFxv4I{E))H`(s|l_P@)C#!$^+V6;mtTg*hi(q)sGrb#!a* z(dZ}CHn&9KUIGIJaGc)ejXJz@Qs6jRTJB>%yG6Ld8-v@Nq8^K7%D+ujs4Aa4b!t=O zU=M4v7IN<4#mw3Szt^f>g?iuSwSt11>w>SpJcomso7^s*{q?MTD+EGLQUkX`R-%sd zBXOtrI=PH*waXNLspEnhyKS!O9Ol6hDP2wE*cg9InqI!N6xd1eFpm_s&DwuLL7r|3 zEakI*a63%X}+0sZ~gowl#ERD(2h9 zKrRSB?6KE??vRx!GP&28S2N;ECu{L%MRnj_h3xF!D8bGpMMt`~dHr!P*l^Q*B{9~>NcfdT_$tgz zA;nziS_(gf8ePNdHeZ`*&6$|8ixVw_`4csUsK%JTJ(&Oe*BL)~ME{{gB>zk?bJqk; zrV@G6Zpw5y=|0_igR|iJ57|uP2repPhmP zB_Bd@_l^>Nv|-pfhF_j0IZ6eDBqimB8^CJ*1F9TfcS(E(SeVF%?Ox>vw%;8H&L>di;`=$U{~ac^OyFb`63q4KmG^c~zQ3mGTKd!Wh57hR zXtmd?gEdZjC0-)qzI8?mnm3pnlB7{()6$L9gk%;g*fznRah{zY{h^;uPk4olTMWGw zW9vO<6D}Ew&3%C$f%yP%hI+ESR?8@zL_dP%J0H%5o32_8*QAoo?ap(9P-WvxGO^DQ zbb{0l{x{Sv`$Z!~j$bt~qd`6Qf24>Bh}pYX7@upHj-}_zyDQCA%`=3Z{d&t*IzyK+ zF7{-QM1Hsdnu@#(Ur4KjXgt9ZuQF1NXI-CZ9= zHp{)E2Dwu?IeRz!;B)4YZM6N%V8c~I``GBm>r^I&!Si=i^@TBYOW3l7^1>56YYZ|T zV(O^f+?dvAAS{3TH1WQM6@ar;3@sG=Z42NKI_4uEi0S=~8IiH{=Aq5Mmerlvs`81_ zy1xMtCFHEx{KDtBjgMRrk>yvC{6!y@`hEdQ-{+v!JmEmTm}_C+!!B z?33`V_}AZVPf+(Z9r5<2^9(;6zdCPfKZ`LXAozsxu2`8P4vmUCu8^*xt0x2{!u9Zb zM6G#btq7?6g!oZ4me6C7S+kgIHWz+0vgHxapz+EvL6X#^b)jjW&1M z6G(%I9Dd9a!(rHBcFVUT^5&9bpW(Xu?%#h&Dn|wMdUFKEBv}VEp1KaPY<8qu$9vN* zfH+RASM5CF8sNMYeSOat&N_O>Nvppfz+(h zzB+ghZltliWdHqsv+Kzpmn+lS9!%OK1DQh`mQATBR&`5aPgshD-@Rc zd8OzWO3on#LCZv`44v{C822eyK3h2Caieot^#Pdiv-%PmcrN!xSxwJq-m1*Ve)JK3 z60Y$VV0R*48_7I9YV10!etEt6_J>2Xvbq<_Z1WP6eeQNy$Swdl*9{qh#GbFFGWx&u zKbgs)I@##(a2d}Zs_9NTH@-=mo3@C&rKqwl80W0-BFV0fjPvzr3_ZQcg zu0chpOuBQ#PCspBM@jQkklz463}yW%%Hv8;OhH%!fHP58icYeplB23VZsJ7JSnOCm z4ri;Y&-J9JAGg;=OlajLW>Jj@phWfFpiFwBjdG|CR5fZV7A!G5QOY=Sc#v&aRisL% zv22@LQt?a>Y!>-Mbn$G5r8*yg@8@wv@lPBkGro+q1NCnKC;lmp09i5A9Y#w`BdXjPMs~U=D)-}(d8#XIA14`wKg9Y z0_WQ=Pm|@bG*`>)SPqsmnyq(RgIwyT$1mgGycgK)Cwn-naQ*Io7TdddPU7o*Qd;0m&$hUO zjv4m+{(vDg>rHLy7LT&e>}8kcrGIWlu_J>3JCKf^l+izO2`8#LA8M#Q$l!mnDx;1U zebdT0dneeWbn`=Ul&8q)CS^7asarT?qF5=Gwtst5x5UPN5Y%{2Xz1~yM^i&j$bXHX zLZ!fdm7Lhfh7!Q9Ov?55yul?K&wJ5~w<=CCRfB62J_$;|_ zzeOkNUl%@%{OwZDwQJ`(>@Qas@gJGS6A17M9F9UcspbZHyEF#6`|7t(;MjCqhQOAFy_zEk=FIbHeRvMtitaHf8R8g2aCn* z->HxAdAHrlXW<(0kNLRYscKHnw10{^cB|0Jlnvw*h1}{~p5?gR&+uEO3N75ee;xga zq4@Bd4Rztp#bgosXohHu(qvxxev6^9ZpvOxs^Ys$y!+G4N<(@}37tWOkocT1Yk+Lg zf}mov3rt=s-LvTtRGHJOf82&?<`vN1uCSD+ru=3XRfGKs<|f+14-SA+3xkH@rM&`H zn5P9hVz76~y9;Tl@*CbFOSzXMo`6}7aN0>>1Oc1Bfk z0z_K(CSoOCciZcmh4biipc*=|a4icpLB#5x^OK+S^XSCl!>glTL&hj}Ku?uW?s~tZ zi);MZm;-$lpm*cdi`hk=mi`>^(`~wx;582PC$A-bj@St+fXcuI5JYz0zr4YCWU)eh z(SPUtk9mI%a&Ru*Zy~9U+1D4#gJ{1Yl;h*sVHTF4GHd|a^-_zx!nG;Ihg~pHL7ESI zdKn1JzlR%>R$er6BUUR4>ZeRzb}5bboN zpYl-ENl9!cBHqIH^B47mkoe`-tG7aAjoidm(98j*0r95+QE$4lSr`CN*xMG-)JE$a z{*Ea4*$W0dsmv`~Rzr`ms3`mZeorr8|5a?S;q*qa4Yr%(_Q8~vP=k5`B`Yn@(o zYG#3i!@YQcMB<+z*TrNKi*p+;@s4*e*x8}zWn{3cMW6P}o6Ws{f+jj6CGz9L(j|6B zMTUf{je}+_nd|iCALnq5ICoE8_%neL*XgYzi028kZ|H@%AWq910boz>=qYeK-V_gt{lvSJzKNR9WVOlW;jU3$#}NQcRZWZz z7a+T!-xyirXuv`eu3Wv_sP zGLU8A)NKYok|S{?&AkWnf^PRpFJdCoCQ_q%h7~6p2=RHJU25G1p^oYZaTj`fN2dS? zqB^dSb6kO5C3!HN*xjDP6qMh8{^H5?>vf(cQh1VyZ;PZjfp6vwo2hl*JcE~l-+4$v z`;>qMBI6x+_5%35LZ{EW@fu%a9 zqT<^?v*?gxFCU8ltPwh)p#oq;mQh6~h6fmg(E}cK^Bp4Asa7;1|MW7S{o()XnKAGV z&v%fVWb)xTG(3#@16_~WuZ~_+4nxKU($CT;k2u#t#nJnB-si3TV2`l0!lS;h>=cel zCv~c9PQ%9bDB0(iC(^O^kgK+L#3f(3hr8nf>U(;jLx zsACcV339cCk$iNR0IT%-{^;(bktT{1i0Wpn`)_nv)8RiYPdY)oEDAZ@aifX@&S_k8gu4`ND#M|)zTovKI(+)CG0gz;N8B_G zop)~W!qY^Ay!lrHG{ww>E0K~Rbns)G(kAGi|5)J&n|VHh886_~(m9 z2sxWC2;9P`V*)?(vKC%vRsc{*OPf`ESlgqNRm?0zBk%HEKZC>1ldbYNs}wviGTsxO z6u{mx>nIl)Upr;yv_S*yJMYmCz}+|GPxwv)_%n_kfA~exZ8V483LL^x8*haifG}r_ zh95?Lq#EVVVQ3uZi!H=?U5b_PI@hk8m9$C%B`N#E!YKd0b(iYODy{&)FH{tp|7e2{ zrWWk*J$N*h@DxZy20yqQ63w??;f_ZkC!4wj``o$8!c1udgz{;oJdeWy>9jQKtDA$< z1n3(9S=JUEAr<}4>#=*60aWHr4>Zu|d6G5zh&2E28+&6e7HkX29h#RdM&BcX3E%#6 z23)8A@0ht@kKriL7*ox!Yz9{9!A)&r`faZClh*46pE5|KGNI>2tjjRi0}nW%z(lRn z178;^&D4E;dGPY@4?5qTcPdvLXZc4KA71P~T>(4Dvb-d@2LPLKhJ^S2=iM<5CZv1}SZ+|0Xq#=P`U_pJ}gEhv5}c{%7LDZOe)Zg<2e+0Ke>eIUyI z@Er&+ZuAD3AmzuUdX;w^xOWwtRYElI9!b1jxegYEfGI9!4b$lKy59*tjJZ+O$wI`$eyi zxNbvr^1A@k-XJ*OSm2*F-NH=LxyuMk@w)FvnQgM$^smBGfB|IvuW>r+V(&YSD_!M; zd#uf{9{?Y9#2*~W-X`y@z)89ASfBx>le6|7RN^+AD%+#fre@{sh|i96#v1R%ysT#D zs>v!~l--;LA}eB{uP3}FGrDdwOn?^%=)fjz2OLjLtL@(^GXcJhW+|kW>ezGgw+nWi zjF`#fLO94^({jF&{h_hfe!!}l%ww@pPZIJOF4PZG+t3NB_Na8`0j_4fO7mo`f{rOr z)25oana_*T2A;x{l^&Y7QywYV?S-uP&hIV(7Vu*-p^=+2?i6&LUT6=NVosnAt`*y? zfBDQ;1rp@kuUjwpRZzuu46&OUhIf z(PJb=oUYdd$e}pi05BHdpL)Ns<{&)t5D5Nmg-6jn{RfDf%_$Vo5a9H834Prctd%DR zPSfPXEvi{JFH3L!r@VLkj=Ou~h11x!)1=Wl0aS3^hW=X^m9igZ1Q&xgM6+ggJ;Nr3@fp# zrf*{*WWimMxWCMCg;p-H@bZKJ-#Z7Ue5^3u3&5ZO8Oh-(#3Py&8bMl&?eEw{Ip-~N zK|5c#%mRo0wCLC(rnyZ;icz%>N~f3_H@ZOzE_L^Z5kz<*p?-Nn_6{z$8e(Y8`5Kjt zK*L zoqZevVJe2%J4ZJk-su%=-c}gKtlfW8=aGHzxR2Paj|_?(T66NO~1xm>osAUvM1h}>CA%q1FSIiA!Rrl%h?lPR>UUH2_Cfft$fTPFlrI1312h!yahWJqwMjsURRA`(vZSe|3g zH|EEYnw{9=MxM0&hsX}A_`8;a4t>|7$fqVgnnJVzkp$yx*IoiL2W5afH~h|9HvdX9 z39E=yej)0vkG2_ii<(M^en-NVfmbj8odV&Bq*=K(cq}2kiSu`eL*Y{=gYwhU)5wJ| zFewR_RK$-^3r;^YsH-zBE9pnlFZH6L63VYaFjmB5F1{nlGIBB!06?&=shOFoP@$ld zMmLej=+NwKzce~E9S6yyR{N*PhBFElt+QQpGw+!%{9CV8pAYnd=icIyv(eac19M+z zo61#LuM?!SN`C&+BZO%G!TxhEhJ}Np`f&bS;(voI=;aXWyagKJLLUBgIP%JkPzL}S zNURf}e_|3Vq3f~AOt0ML+f z8}aH=*xv9XAoJovmqglYlbRC{nb;N734Le_QqGn^ zB_^l?f^aJ&R|sl?6kp_1n6Xd<5g_1({FgK2>!7V)HaG!)-@Vo$(v}rfKb-u_#e(-b zpQH?lJAEJjF`R!t-UdVHOc01n#a*>tb9F{cu+PQZZ7~`0R&D3Y*t^qOJ=zH3P4 z=jA$sBE4**a_sH5^hAS8W0f? z-CE*tg_+RPkxQ27yAHm|@hTUNe_Xm%W2&1R!w+;;&aWg9`DQV`eD2*k$J6xxI zngiK;*ikF5>e=~3J)CJRGgYZJK?Z(IcDR(pquN1~+&C|GtY1LnvlZ|eQxAPA_!T{F z(<~XLau7#ieU+0@5RP5V5aw*aO~KV^40;tyU}jX*n$ky8O1DE8ED6Klk?YK_SP<{z zs{iXmjy|Y`vhE8_8p@;rYf3>+G+qx;7#T0I2F!{SXv}z>v4x?}zrRW?cI=0ovgZJ+ zFnE%%Mx>$s{N)nC)R+@3fV2~M=6?x8`E3DFe1;0RYsrqwP>rZsg9JJ9sAW=4|VZX zaun|;Ck%Ywz{FRWZYo4N6?Hr$+PjX|%f2Z@63?e7hf_}butJWsw7kGTu9;_ViSiva z=dP8QrFBLp9UC7~oA}%ye28+6{}R|aLvE)?m1yUGM!+Y%*NHUDVuI-8$wft9S$+5m z!gJ~p`_f5&7aRAu&X@EQNJ@sp2?)m)a`CTVm_0mP^7uLvu)>xoa{Z@LN4F&Z;~poy zpBN7$3+Z-v+!7}qdxJaP3K$A9A~bf%iS179xY|W3>UAg$Ipz& zX`&QQI|#->Y4WHBJJ#r|#H@qy>of|>JVZzD4Fok65HT2_B=(%g2t)Mp{kcrm~{Nwn17908$uzD#8z)+q7>!BXxufgm-R zxhIrfm>R=QtvSyq)-76LsZ3Qs%Zf`+rdkHzo7Tc8n^1`0%9#&KQmY0|=TSNG@;USHJ3*vD>0mxAS2*W?y&t4P z1eYed=GysYn%N+DJR45CBos(uXS#i=S1V#u9G87xuXZ}6?orVn9Efszt-n^fK(h*& zoM|MSM+(n6bpb4f0n8;PrE&W8Wx{MvG4Z~oa!_Nl_!{^f8Vcwy!xWZEoy7(0{IIdc zrk_fZ=x*w?9w)dI-LwiP*tu@3^c$wKyy<%A5EZoIGqQuBcUm@ie7Cbk9IbIxC=_AN z@)WJ@eqAqQ%O@jgq@07o&d(toisBK-m>y|}A`(3S+7N3^5{)xVjDma28{4PO0qa-~ z%E_oJDklANGG0|ljgd(Senx+m@?N{nb#^z@0>%JTx+a1Qusf|E;Ae_{bbl5xu-X`p zi}mTS26c2i42%x^`o+)xSQ^iBPN0fK7oJBFSR+af4#eglAOQRA=DVZ}#fp%*(UFzEt;OIMT!m#~ z(v!wzU;^*E{|xRLwNr`8BTj>DiYk`BY|Ef|DzPiZy`f*APhmSthK~Y;L2rNH=b|MUd^98*sa?U*V1};u zfggCcl`Ilo-^afFS!uu)U!0Q2<_bD*pi+amXfD*jW16M9ih;3cO%rcvRvvOrm>*5vbDpdI<&N*>o^s-~#c}}S_MZ9uE5AvGa z!5D^ix4M%@Kw_MrUv|J2491VPl?%@yw=^Gd7}Qp!#Em+Y1AwwpC{Lx9+0>LhF1O&k zy|;awAF$YkT-GrXZAp_7~k9Ks3eTsOA5qq#3*X;3#<1fKqaH%mbBweP0tAq9u&bsifm_X`Y zp2jbF@po0^X8Xkhb*vF_)rM!pYgIK(ZY*M zsaSvU&Q*@~kK3s()JXgEvT2T75 zB&NQFQr@^3nrpF21%GsTpHMg$Sg=x;f4@0M@~+;(k4~$#-tgF(c#O|;lx`HK&F^4W zRc;Xlz_pq_kgM-|y?iVXwt3>iJLW`8&7SRBb@AhBVh99$6quV+Eb{!ud@_eN*&5wa zr)o#pcUYB3H24yB6i;Tt6BoMwqTThVM!5NGO~qJav25t>$crEV_i5SV$sKpUZg~DJ zN-wC0uEen`mu>g+nXyLV<(L;=i%_p(OxpTuuL;^mVr^mVPWsCdsq z)z;T}d@lR@7T+-q3OQ`eY3S$d5(r~I)(DIv)7tz!KY~hJ zeB%iTgIa0@PvHvil|OgKi2{KvVh=U|Z~uMhu~Dtx&vx}p5HAWW%@A3+@|c6!RH`iq z1mEx%(nf?5MBp*1yq?K$VwE)B+b?c|tZ8=~Cm$O()BsJ-^xOEyQ}wv<+uQ@d`~`uE za}V*fXeeLujrGr37pC~q)Bv1bfVi_}IHnPr`4b$qs7y712FK&@;6~nRDtA^7kJz;w z2M#gJ$hLi`U$+ z*T-8(GeE2O8lF4yTml(yRVblMoH#RxoVif*avz5>cZDyu6g3GfPo;yR0=B&5&Tro6@i8 z`Kv`_2kCy8w(8OF-al7n$dQ2kP*OKkvn6}$JTl$?PGYV5O7RPI16-{b2mqxu3ZXv& zO=OtdQX?MBdn2++s^hlePkHLvH2B_@FF7ZK94IyUPT{wL{qTm=_>%LP?byvXyjrZs z*^cY2e!fj0)a84`T~Oo+axL_ay=zF;Zi)~AJh}FT=fcjzkyaKUY%U}S5yu;|Z+sN3 zZ`B_LGkX~w&GzRvJE{()*udTrajQKsQ>KP|AHlK**8^}(oVGbxm7}X**O@RZ(U{4E%r=Y6?D$Xm z5RkYLWgNfK)qhcUDJJ>^M&s43d1-ERg{BijFvg7RS@V3yFifr-R zV8Mo$mUVRzLL23)aU?ZeR@~z8Jbc`0$8btV8mjClg05K!_B%YRlnXt|MhpF5W%Qtk zTli_(ul(=YU8W;$FQM*d`*yEjj@_Q$78yJrQT-N0Vj$uHW9JVc^_&eD>Zp$GAdT|d zUAg{QeWqhwVMj19U93=ybpMOlvGnTrqx)RI{52}d?iqoJXSu7x4JcR$j06WQrw&#( zC#a~ydXq{aPWWMSIXQ+?EomhDSWm`$arsI#0;3mq6qDqx=uQWhfL}U-XU8VMP1^0= ziHXA+h#zv99dmc}Zh*C(L#()yn;QZ`;4KsO#goGSi&te8w7i7`aE_pbDNp_9!HTZC z;zCjZsPM|Iz5hJXD`Y@+wBYjM@S=OWXh z4?@rj%V9om`&jaHFR;@g70%K0EDp9xBwIR?Jris}laWihDxDkqN;nZq@R@nBr4Tc; zEHj~+XOx;6PMeQJX5GqBi&R16ltr1Du4bo*=h;=crRJ|3sWg?)9&Xa)6uj;u&!cZg z8+qJAT($*X68?I5=U#$m0Q9x4>rm^<1rX|4Z`I&B?w>XGG_8>9+77?Eo-=|Vk1lAN zcypu%IiNre%ERUk)n3~G$0Z~H&UWcPk;%ZI{4WRvGdmkG&RaWVWvoW>t3S4X?goq# zXU#Y#GdtaPpx7*RS-EPpe`F)oLI zz7T5BH&LZSaAwPhDW2MPB99wjUZD zwBjk0K&gsAZ?quL?0>l_Ir zKKN&NL`+Qn>HpCMnynT2cTEf2Ie zn2l*>CO&{?6F2tvMDhEx6VY<^{{@0Ju$QMF^dp(X{6gw(b-Px+5*KB2kAAU>U)&=9RWRIPnF`bg{5n!EyBFxs(Cd@3G*(uwNWUl75r^*j<22nqz z^^K>UjaASw>EZBZ$z`g`iF(e5deXUong()RDPMvdvbuAhX)HI_QN^p(S_1G|~!YtKW-9JWFN!G5HWDe^@<& zJl^;x)s6`FbHo?lWbCSf|Gk!Ey+dkh?T0X95h1UX7_M~-F&iUKsTN9-)+o>)i8o|iM@!>zWR7XIgBXpWp*B0}+IuHeMAHmX5yW%MdFyHX z24T4rT#k4-K})sIm|l6uf|p0n3P)eXQeW|Nzu4ArN_QWTxp2SJ+)=V^;jUM_UzA@uA1r)6%8G#lj zKeXF57)46AJbt`1D<_ ziW?gyN(5q+erX|AP`0G5D$8@+L4ouqg+M-szGC&~eupG_@o3vB%VhV#Y=%#^o8*EO z(kR(F4O`0yb7{ur z1HMLiNYfh`q7-4)t2M%)r%+m4P!{m4{h@>dU9TO6VNyC?^bO@XR2rHLezfV9okp(T zX~QUKc&%VcVm3?~imERuf8pR0QtEWQL)zOxS4<|8x@W*=0>LOX{?Dhh9ER#3;U2%V zRRj8NEck1zhf3C0K}rlVSr=fLB&X?(zXnh3N4~4WDhBs7i%ulXw<#gbB$Z~n!6X8; zyX3nnh9$Qp*Ve^yz|-UC{VmLQeFlknaou|EgWEQA;Q$t@rNg6<8_C}J{qt-m)x{k(=pnM zR4HYI3?R)&xc#e63(IKanh~RI&B=;s69^7pDW3BT$y`A`$bRAie6Mn74mG*ote&U} zC=>b{w-@?JQ36ZNV8%}lU(Mi2(D`tZcBL((f|~IAY*}UpZbg)hpwYx*W5Ho4ec5)| zewnM8(ZPAZAZ^D6d%gP@vO9B@|DRXRRi*+gnI0?Jlf z+lOoZFUhDIVevQ2+1?5IWKK&PF5iXl@DutoF-*`cWdh~19`^AKxl~2O1T4qV6koDB}g6Y`+|9Wo!XyO~tcA7OqS&&Xk9^i>Bb}Wu!K4LL1 zv7FQz1RpJ{bH%N*Dwodfo{=6=xUaa0bNrdmKg#K_UKA?QA0Mli&gOJ($#FTXH@w+0!MRF-aRujrrr_ zuL&3)2W^Y-cYkpHO9`0CxD`+yTgk_vxA41((?`F;Pc*{ScxNqXRrb*6%&#I)t6)1F zuYt`yE(a_qe+(1ogn-ltR2|$|+=&<&yTUQ+*0~nfM2r*YpS^u#pJmpN;shJaVk&qX zOSR$WEKkQpKm_<6eRb5q=Nb|CC!s9f;WdMD`8$!VrTxqYRhxyZ7*g5Tn_2!0%?W&+Fi?fBRC zdQ>$S1O{8IOX~!CnCZ=b8>9;rA{0A-;kB*P*9d8h8ln{p&TXx|xn*lU>YKKId$YqK zGf8!$?mCfmo(lWg`M@hoE_xTVYjqis+o~inF)LYRsKMsdQ#X|zC@<*sx_SMwX7NUL z^-ZUL1^+31X+uXVz>+Nc-mKGmD>~Jt1vTURgGugt3+TS5F6(PE%fR`C4~g@l;-Tvl zqN-=gvrE;Mt+n+V9mLIZX670ZBB_8^ww)Cd9xl}iF~AHD50@7E|66+QAaey({Ve(q z199>Gi)|nCVF7sLtc~2;bc<5XxKJ(d>=`8gPA1P0_f&TxEx%r)G2XCuv&Kjj*8|o_ zLq~YwqYLe7>?|+rYDEkjPBdY)qeQM~Y9GP|j&mM4A$}YPz7q0+JQN(KZ+DAy{kkP& z*H&LyK77&8FvM@9$Nj&3&ihW?`sISF+s(w&7^GnVkTFEVHg;siq=4DS!v%TKv|3nX z9AuI7YCSaWni=odiG7bg&kW%q8yV$&OpQk`qRBAtzw!aJtg}7F5BMNgnUQ`iIgc1F znAvw~p&85^rnJE#bxLQwSNy@UUeifK%2~?#tJ_Uf8XYQ~=+FzYkFX>2uRB(U5@sNN z%LIp|L7HS5s`#I3L%51OXHc1`(A{0;C1|U{7?IYr2aQ1nmB8m37$JNl7xw$a^Eg@V za>Rx?pv(%0D9VTQP(<&UsP@Lnje@Bp(|oZC$JsfeOy2JASpfSHN9>_KoLbX8--=;C zzu97fSwdcb57E<}0T)A2AWvR=l^)CMeip8JSzUeip;h{xL#hnXU5})^LAo9|9T0%` zW&F+AHwC6~4Qiopk|T%s@>3IE*MR$CB2Hh!=6x18v2`{j0MXA~l>Oq<_V5j@56AnI ziK?6%ZJN*V3z$858+`jUSzrv}<+u5=3HRHMZeLUUqlw_NJ=K6uS9-$_ngSV(rag0^ zEam)nsJvKA;!*qVv`!zLp5Rehfoxy0-|Hx%KX9}nph1=7^QUdTm_wZ^AOF9 zphK8~#g-MKEaw7_tb334U0PP&7sZK1Yo$(l+v+=&=XYbO4wIgf`uCwN$uH3Ylj{2` z0)T||D?zGe_QYBNX@CTd+hs_tE^Y6XKJ)zrQt7EX5gP= zE)4_`dHVQalOzHw)HmxvD^apd`qS>;6tGVyY*P|v`p)o6_|*c$%cG_qq#Nk$?)D1;BLfWHm3;Pr>d1UUr?}QGo7XKQj@Zd^%tqs$ajxh2tQ3N zRKbspg%pBatmz!FWk+JxLmkZ!{H(ZfC$pI^Cih-V@N#cPL{2XcFqFK5KOoO+^NUxf59 zr9xL)Ax3?rfz^~&SkhBWe2D#aD6xX-cH$b8j^UK?cBrW8o{P9shkgT?qcA|}V6hMY zl>E7KFkD zEK9>{YN_7sg~^YwzwEg9X0*DjS%{jZEZVXT;}$fWO-{wPkO8XIM3vwtpJmHw1lo1l zstS!jZrrsASx3NZmPXyQ*KL=*WroiWoC`LDM+)p-l{>V?BuDa^r52)8!r=2|c5O@1 zKkMP4{a4IKR;;0&WPfkH)>Dj$2-@?pxMMCZWIX>1+vmqw81zD7kkz@_;%6~B;z05> z3cYLb^pRb|@iaiZ+-fBYv~nul5vg5EgeW5QJ&$<9KlLQK+b`1?C!#FH)3@-CKd-Ph z2TprpVNfWk|IFXHjHBbH zW20!^^b`{GI5)<##f?~t|Mxyh$(m->buf_physaddr is the(m->buf_iova is the + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Page-1 + + + + Sheet.165 + + Sheet.1 + + Circle + List 1 matching Criteria 1 + + + + + + + + + List 1 matching Criteria 1 + + List 1 matching Criteria 1 + + Circle.23 + List 2 + + + + + + + + + List 2 + + List 2 + + Circle.24 + List 1 matching Criteria 1 + + + + + + + + + List 1 matching Criteria 1 + + List 1 matching Criteria 1 + + Sheet.5 + + Triangle + + + + + + + + + + Sheet.7 + setsum + + + + setsum + + + Circle.29 + List 2 matching Criteria 2 + + + + + + + + + List 2 matching Criteria 2 + + List 2 matching Criteria 2 + + + Sheet.9 + + Sheet.10 + + Triangle + + + + + + + + + + Sheet.12 + Set Summary + + + + Set Summary + + + Sheet.13 + + + + + + + Sheet.14 + Flow Key + + + + Flow Key + + Sheet.15 + + + + + + + Sheet.16 + + + + + + + Sheet.17 + + + + + + + Sheet.18 + + + + + + + Sheet.19 + New Flow => New Assignment + + + + New Flow => New Assignment + + Sheet.20 + + + + + + + Sheet.21 + Old Flow => forward to specific thread + + + + Old Flow => forward to specific thread + + Sheet.22 + + + + + + + Sheet.23 + + + + + + + Sheet.24 + + + + + + + + Sheet.25 + + Circle + + + + + + + + + + Circle.156 + + + + + + + + + + Circle.157 + + + + + + + + + + Circle.158 + + + + + + + + + + Circle.159 + + + + + + + + + + Sheet.31 + + + + + + + Sheet.32 + + + + + + + Sheet.33 + + + + + + + Sheet.34 + + + + + + + Sheet.35 + + + + + + + Rectangle.167 + SUM + + + + + + + + + + SUM + + Rectangle.168 + Packet + + + + + + + + + + + Packet + + Rectangle.169 + SUM + + + + + + + + + + SUM + + Rectangle.170 + Packet + + + + + + + + + + + Packet + + Sheet.40 + + + + Sheet.41 + + + + + + + Sheet.42 + Encode ID + + + + Encode ID + + + Sheet.43 + + + + + + User + + Sheet.45 + + + + + + + + + + + + + + + + + + + Sheet.46 + + + + + + + Sheet.47 + + + + + + + Sheet.48 + + + + + + + + + + + + User.7 + + Sheet.50 + + + + + + + + + + + + + + + + + + + Sheet.51 + + + + + + + Sheet.52 + + + + + + + Sheet.53 + + + + + + + + + + + + User.12 + + Sheet.55 + + + + + + + + + + + + + + + + + + + Sheet.56 + + + + + + + Sheet.57 + + + + + + + Sheet.58 + + + + + + + + + + + + Data Center + + Sheet.60 + + + + + + + Sheet.61 + + + + + + + Sheet.62 + + + + + + + Sheet.63 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Load balancer + + Sheet.65 + + + + + + + + + + + Sheet.66 + + + + + + + + + + + + Directory server + + Sheet.68 + + + + + + + Sheet.69 + + + + Sheet.70 + + + + Sheet.71 + + + + Sheet.72 + + + + Sheet.73 + + + + Sheet.74 + + + + Sheet.75 + + + + Sheet.76 + + + + Sheet.77 + + + + Sheet.78 + + + + Sheet.79 + + + + Sheet.80 + + + + Sheet.81 + + + + Sheet.82 + + + + Sheet.83 + + + + Sheet.84 + + + + Sheet.85 + + + + + Sheet.86 + + + + + Sheet.87 + + + + + Sheet.88 + + + + + Sheet.89 + + + + + Sheet.90 + + + + Sheet.91 + + + + Sheet.92 + + + + Sheet.93 + + + + Sheet.94 + + + + Sheet.95 + + + + Sheet.96 + + + + Sheet.97 + + + + Sheet.98 + + + + Sheet.99 + + + + Sheet.100 + + + + Sheet.101 + + + + Sheet.102 + + + + Sheet.103 + + + + + + + + Sheet.104 + + + + Sheet.105 + + + + + + Sheet.106 + + + + + + + + + Directory server.104 + + Sheet.108 + + + + + + + Sheet.109 + + + + Sheet.110 + + + + Sheet.111 + + + + Sheet.112 + + + + Sheet.113 + + + + Sheet.114 + + + + Sheet.115 + + + + Sheet.116 + + + + Sheet.117 + + + + Sheet.118 + + + + Sheet.119 + + + + Sheet.120 + + + + Sheet.121 + + + + Sheet.122 + + + + Sheet.123 + + + + Sheet.124 + + + + Sheet.125 + + + + + Sheet.126 + + + + + Sheet.127 + + + + + Sheet.128 + + + + + Sheet.129 + + + + + Sheet.130 + + + + Sheet.131 + + + + Sheet.132 + + + + Sheet.133 + + + + Sheet.134 + + + + Sheet.135 + + + + Sheet.136 + + + + Sheet.137 + + + + Sheet.138 + + + + Sheet.139 + + + + Sheet.140 + + + + Sheet.141 + + + + Sheet.142 + + + + Sheet.143 + + + + + + + + Sheet.144 + + + + Sheet.145 + + + + + + Sheet.146 + + + + + Cloud + + + + + + + + + + + + + + Triangle + setsum + + + + + + + + + + setsum + + Sheet.149 + + + + + + + Sheet.150 + + + + + + + Sheet.151 + + + + + + + Sheet.152 + Clients + + + + Clients + + Sheet.153 + Distributed Cache + + + + Distributed Cache + + Sheet.154 + Web Servers + + + + Web Servers + + Simple Arrow + + + + + + + + + + + Simple Arrow.153 + + + + + + + + + + + + Rectangle + + + + + + + + + + Rectangle.158 + + + + + + + + + + Rectangle.159 + + + + + + + + + + Rectangle.160 + + + + + + + + + + Sheet.161 + (a) Distributed Web Cache + + + + + + + (a) Distributed Web Cache + + Sheet.162 + (b) Detecting Routing Loops + + + + + + + (b) Detecting Routing Loops + + Sheet.163 + (c) In-order Workload Scheduler + + + + + + + (c) In-order Workload Scheduler + + Sheet.164 + (d) Database Semi-join Operations + + + + + + + (d) Database Semi-join Operations + + + diff --git a/dpdk/doc/guides/prog_guide/img/member_i2.svg b/dpdk/doc/guides/prog_guide/img/member_i2.svg new file mode 100644 index 00000000..759c6545 --- /dev/null +++ b/dpdk/doc/guides/prog_guide/img/member_i2.svg @@ -0,0 +1,36 @@ + + + + + + + + + + Page-1 + + + Sheet.3 + False Positive Probability = (1-(1-1/m)kn)k ≃ (1-ekn/m)k + + + + False Positive Probability = (1-(1-1/m)kn)k ≃ (1-ekn/m)k + + diff --git a/dpdk/doc/guides/prog_guide/img/member_i3.svg b/dpdk/doc/guides/prog_guide/img/member_i3.svg new file mode 100644 index 00000000..41e92cb8 --- /dev/null +++ b/dpdk/doc/guides/prog_guide/img/member_i3.svg @@ -0,0 +1,148 @@ + + + + + + + + + + + + + + + + + + + + + + + + Page-1 + + Sheet.174 + + Circle + + + + + + + Circle.156 + + + + + + + Circle.157 + + + + + + + Circle.158 + + + + + + + Circle.159 + + + + + + + Sheet.160 + + + + Sheet.161 + + + + Sheet.162 + + + + Sheet.163 + + + + Sheet.164 + + + + Rectangle.167 + BF of IDs + + + + + BF of IDs + + Rectangle.168 + Packet + + + + + Packet + + Rectangle.169 + BF of IDs + + + + + BF of IDs + + Rectangle.170 + Packet + + + + + Packet + + Sheet.171 + + + + Sheet.172 + + + + Sheet.173 + Encode ID + + Encode ID + + + diff --git a/dpdk/doc/guides/prog_guide/img/member_i4.svg b/dpdk/doc/guides/prog_guide/img/member_i4.svg new file mode 100644 index 00000000..a2b6f2f5 --- /dev/null +++ b/dpdk/doc/guides/prog_guide/img/member_i4.svg @@ -0,0 +1,450 @@ + + + + + + + + + + + + + + + + + + + + + Page-1 + + + Sheet.47 + + Sheet.1 + Element + + + + Element + + Sheet.2 + + + + + + + Rectangle.54 + + + + + + + + + + Rectangle.55 + + + + + + + + + + Rectangle.56 + + + + + + + + + + Rectangle.57 + + + + + + + + + + Rectangle.58 + + + + + + + + + + Rectangle.59 + + + + + + + + + + Sheet.9 + BF-1 + + + + BF-1 + + Rectangle.74 + + + + + + + + + + Rectangle.75 + + + + + + + + + + Rectangle.76 + + + + + + + + + + Rectangle.77 + + + + + + + + + + Rectangle.78 + + + + + + + + + + Rectangle.79 + + + + + + + + + + Rectangle.81 + + + + + + + + + + Rectangle.82 + + + + + + + + + + Rectangle.83 + + + + + + + + + + Rectangle.84 + + + + + + + + + + Rectangle.85 + + + + + + + + + + Rectangle.86 + + + + + + + + + + Rectangle.88 + + + + + + + + + + Rectangle.89 + + + + + + + + + + Rectangle.90 + + + + + + + + + + Rectangle.91 + + + + + + + + + + Rectangle.92 + + + + + + + + + + Rectangle.93 + + + + + + + + + + Sheet.28 + + + + + + + Sheet.29 + + + + + + + Sheet.31 + + Block Arrow + + + + + + + + + + Sheet.33 + h1, h2 .. hk + + + + h1, h2 .. hk + + + Sheet.34 + + + + + + + Sheet.35 + + + + + + + Sheet.36 + + + + + + + Sheet.37 + + + + + + + Sheet.38 + BF-2 + + + + BF-2 + + Sheet.39 + BF-X + + + + BF-X + + Sheet.40 + BF-L + + + + BF-L + + Sheet.41 + Hashing for lookup/Insertion into a vector of BFs happens once + + + + Hashing for lookup/Insertion into a vector of BFs happens once + + Sheet.44 + + + + + + + Sheet.45 + Lookup/Insertion is done in the series of BFs, one by one or ... + + + + Lookup/Insertion is done in the series of BFs, one by one or can be optimized to do in parallel. + + Sheet.46 + + + + + + + + diff --git a/dpdk/doc/guides/prog_guide/img/member_i5.svg b/dpdk/doc/guides/prog_guide/img/member_i5.svg new file mode 100644 index 00000000..c1728cf2 --- /dev/null +++ b/dpdk/doc/guides/prog_guide/img/member_i5.svg @@ -0,0 +1,163 @@ + + + + + + + + + + + + + + + + + + + + + Page-1 + + + Sheet.1 + + Triangle + + + + + + + + + + Sheet.3 + vBF + + + + vBF + + + Sheet.4 + + + + + + + Sheet.5 + Flow Key + + + + Flow Key + + Sheet.6 + + + + + + + Sheet.7 + + + + + + + Sheet.8 + + + + + + + Sheet.9 + + + + + + + Sheet.10 + New Flow => New Assignment + + + + New Flow => New Assignment + + Sheet.11 + + + + + + + Sheet.12 + Old Flow => forward to specific thread + + + + Old Flow => forward to specific thread + + Sheet.13 + + + + + + + Sheet.14 + + + + + + + Sheet.15 + + + + + + + Sheet.17 + + + + Sheet.18 + A BF corresponding to each worker thread + + + + A BF corresponding to each worker thread + + diff --git a/dpdk/doc/guides/prog_guide/img/member_i6.svg b/dpdk/doc/guides/prog_guide/img/member_i6.svg new file mode 100644 index 00000000..265179f6 --- /dev/null +++ b/dpdk/doc/guides/prog_guide/img/member_i6.svg @@ -0,0 +1,332 @@ + + + + + + + + + + + + + + + + + + Page-1 + + + Sheet.121 + + Rectangle.2 + + + + + + + + + + Rectangle.4 + + + + + + + + + + Rectangle.10 + Signatures for target 1 + + + + + + + + + + Signatures fortarget 1 + + Sheet.53 + + + + + + + Sheet.54 + Match 1 + + + + Match 1 + + Sheet.55 + Packet Payload + + + + Packet Payload + + Sheet.56 + + + + + + + Sheet.96 + Attack Signature Length 1 + + + + Attack Signature Length 1 + + Sheet.114 + + Sheet.106 + + Rectangle.100 + + + + + + + + + + Rectangle.101 + + + + + + + + + + Rectangle.102 + + + + + + + + + + Rectangle.103 + + + + + + + + + + Rectangle.104 + + + + + + + + + + Rectangle.105 + + + + + + + + + + + Sheet.107 + + Rectangle.100 + + + + + + + + + + Rectangle.101 + + + + + + + + + + Rectangle.102 + + + + + + + + + + Rectangle.103 + + + + + + + + + + + Rectangle.104 + + + + + + + + + + Rectangle.105 + + + + + + + + + + + + + Sheet.89 + + + + + + + Rectangle.115 + Signatures for target 2 + + + + + + + + + + Signatures fortarget 2 + + Sheet.116 + Attack Signature Length 2 + + + + Attack Signature Length 2 + + Sheet.118 + Attack Signature Length L + + + + Attack Signature Length L + + Sheet.119 + + + + + + + Sheet.120 + Match 2 + + + + Match 2 + + Sheet.85 + + + + + + + Sheet.117 + Attack Signature Length X + + + + Attack Signature Length X + + + Sheet.122 + HTSS + + + + HTSS + + diff --git a/dpdk/doc/guides/prog_guide/img/member_i7.svg b/dpdk/doc/guides/prog_guide/img/member_i7.svg new file mode 100644 index 00000000..e23ae26c --- /dev/null +++ b/dpdk/doc/guides/prog_guide/img/member_i7.svg @@ -0,0 +1,399 @@ + + + + + + + + + + + + + + + + + + + + + + + + Page-1 + + + Sheet.121 + + Rectangle.2 + + + + + + + + + + Rectangle.4 + + + + + + + + + + Rectangle.10 + Flow Keys Matching Mask 1 + + + + + + + + + + Flow Keys Matching Mask 1 + + Sheet.53 + + + + + + + Sheet.54 + Match + + + + Match + + Sheet.55 + Flow ID1 + + + + Flow ID1 + + Sheet.96 + Flow Mask 1 + + + + Flow Mask 1 + + Sheet.114 + + Sheet.106 + + Rectangle.100 + + + + + + + + + + Rectangle.101 + + + + + + + + + + Rectangle.102 + + + + + + + + + + Rectangle.103 + + + + + + + + + + Rectangle.104 + + + + + + + + + + Rectangle.105 + + + + + + + + + + + Sheet.107 + + Rectangle.100 + + + + + + + + + + Rectangle.101 + + + + + + + + + + Rectangle.102 + + + + + + + + + + + Rectangle.103 + + + + + + + + + + + Rectangle.104 + + + + + + + + + + Rectangle.105 + + + + + + + + + + + + + Sheet.89 + + + + + + + Rectangle.115 + Flow Keys Matching Mask 2 + + + + + + + + + + Flow Keys Matching Mask 2 + + Sheet.85 + + + + + + + Sheet.56 + + + + + Sheet.122 + HTSS with False Negative (Cache) + + + + HTSS with False Negative (Cache) + + Sheet.123 + Active + + + + Active + + Sheet.124 + Target for Flow ID 1 + + + + Target for Flow ID 1 + + Sheet.125 + Flow ID2 + + + + + + + Flow ID2 + + Sheet.126 + New/Inactive + + + + New/Inactive + + Sheet.127 + + + + + + + Sheet.128 + Miss + + + + Miss + + Sheet.129 + Flow Mask 2 + + + + Flow Mask 2 + + Sheet.130 + + + + + + + Sheet.131 + + + + + + + Sheet.132 + + + + + + + Sheet.133 + Flow Mask X + + + + Flow Mask X + + Sheet.134 + Flow Mask L + + + + Flow Mask L + + diff --git a/dpdk/doc/guides/prog_guide/index.rst b/dpdk/doc/guides/prog_guide/index.rst index 07a4d354..c4beb346 100644 --- a/dpdk/doc/guides/prog_guide/index.rst +++ b/dpdk/doc/guides/prog_guide/index.rst @@ -1,5 +1,5 @@ .. BSD LICENSE - Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + Copyright(c) 2010-2017 Intel Corporation. All rights reserved. All rights reserved. Redistribution and use in source and binary forms, with or without @@ -38,29 +38,41 @@ Programmer's Guide intro overview env_abstraction_layer + service_cores ring_lib mempool_lib mbuf_lib poll_mode_drv + rte_flow + traffic_metering_and_policing + traffic_management cryptodev_lib - ivshmem_lib + rte_security link_bonding_poll_mode_drv_lib timer_lib hash_lib + efd_lib + member_lib lpm_lib lpm6_lib + flow_classify_lib packet_distrib_lib reorder_lib ip_fragment_reassembly_lib + generic_receive_offload_lib + generic_segmentation_offload_lib pdump_lib multi_proc_support kernel_nic_interface thread_safety_dpdk_functions + eventdev + event_ethernet_rx_adapter qos_framework power_man packet_classif_access_ctrl packet_framework vhost_lib + metrics_lib port_hotplug_framework source_org dev_kit_build_system @@ -126,10 +138,6 @@ Programmer's Guide :numref:`figure_pkt_flow_kni` :ref:`figure_pkt_flow_kni` -:numref:`figure_vhost_net_arch2` :ref:`figure_vhost_net_arch2` - -:numref:`figure_kni_traffic_flow` :ref:`figure_kni_traffic_flow` - :numref:`figure_pkt_proc_pipeline_qos` :ref:`figure_pkt_proc_pipeline_qos` @@ -167,6 +175,41 @@ Programmer's Guide :numref:`figure_figure39` :ref:`figure_figure39` +:numref:`figure_efd1` :ref:`figure_efd1` + +:numref:`figure_efd2` :ref:`figure_efd2` + +:numref:`figure_efd3` :ref:`figure_efd3` + +:numref:`figure_efd4` :ref:`figure_efd4` + +:numref:`figure_efd5` :ref:`figure_efd5` + +:numref:`figure_efd6` :ref:`figure_efd6` + +:numref:`figure_efd7` :ref:`figure_efd7` + +:numref:`figure_efd8` :ref:`figure_efd8` + +:numref:`figure_efd9` :ref:`figure_efd9` + +:numref:`figure_efd10` :ref:`figure_efd10` + +:numref:`figure_efd11` :ref:`figure_efd11` + +:numref:`figure_membership1` :ref:`figure_membership1` + +:numref:`figure_membership2` :ref:`figure_membership2` + +:numref:`figure_membership3` :ref:`figure_membership3` + +:numref:`figure_membership4` :ref:`figure_membership4` + +:numref:`figure_membership5` :ref:`figure_membership5` + +:numref:`figure_membership6` :ref:`figure_membership6` + +:numref:`figure_membership7` :ref:`figure_membership7` **Tables** diff --git a/dpdk/doc/guides/prog_guide/intro.rst b/dpdk/doc/guides/prog_guide/intro.rst index d6daab37..9fe1f0c3 100644 --- a/dpdk/doc/guides/prog_guide/intro.rst +++ b/dpdk/doc/guides/prog_guide/intro.rst @@ -44,7 +44,7 @@ Documentation Roadmap The following is a list of DPDK documents in the suggested reading order: -* **Release Notes** (this document): Provides release-specific information, including supported features, +* **Release Notes** : Provides release-specific information, including supported features, limitations, fixed issues, known issues and so on. Also, provides the answers to frequently asked questions in FAQ format. diff --git a/dpdk/doc/guides/prog_guide/ivshmem_lib.rst b/dpdk/doc/guides/prog_guide/ivshmem_lib.rst deleted file mode 100644 index b8a32e4c..00000000 --- a/dpdk/doc/guides/prog_guide/ivshmem_lib.rst +++ /dev/null @@ -1,160 +0,0 @@ -.. BSD LICENSE - Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in - the documentation and/or other materials provided with the - distribution. - * Neither the name of Intel Corporation nor the names of its - contributors may be used to endorse or promote products derived - from this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -IVSHMEM Library -=============== - -The DPDK IVSHMEM library facilitates fast zero-copy data sharing among virtual machines -(host-to-guest or guest-to-guest) by means of QEMU's IVSHMEM mechanism. - -The library works by providing a command line for QEMU to map several hugepages into a single IVSHMEM device. -For the guest to know what is inside any given IVSHMEM device -(and to distinguish between DPDK and non-DPDK IVSHMEM devices), -a metadata file is also mapped into the IVSHMEM segment. -No work needs to be done by the guest application to map IVSHMEM devices into memory; -they are automatically recognized by the DPDK Environment Abstraction Layer (EAL). - -A typical DPDK IVSHMEM use case looks like the following. - - -.. figure:: img/ivshmem.* - - Typical Ivshmem use case - - -The same could work with several virtual machines, providing host-to-VM or VM-to-VM communication. -The maximum number of metadata files is 32 (by default) and each metadata file can contain different (or even the same) hugepages. -The only constraint is that each VM has to have access to the memory it is sharing with other entities (be it host or another VM). -For example, if the user wants to share the same memzone across two VMs, each VM must have that memzone in its metadata file. - -IVHSHMEM Library API Overview ------------------------------ - -The following is a simple guide to using the IVSHMEM Library API: - -* Call rte_ivshmem_metadata_create() to create a new metadata file. - The metadata name is used to distinguish between multiple metadata files. - -* Populate each metadata file with DPDK data structures. - This can be done using the following API calls: - - * rte_ivhshmem_metadata_add_memzone() to add rte_memzone to metadata file - - * rte_ivshmem_metadata_add_ring() to add rte_ring to metadata file - - * rte_ivshmem_metadata_add_mempool() to add rte_mempool to metadata file - -* Finally, call rte_ivshmem_metadata_cmdline_generate() to generate the command line for QEMU. - Multiple metadata files (and thus multiple command lines) can be supplied to a single VM. - -.. note:: - - Only data structures fully residing in DPDK hugepage memory work correctly. - Supported data structures created by malloc(), mmap() - or otherwise using non-DPDK memory cause undefined behavior and even a segmentation fault. - Specifically, because the memzone field in an rte_ring refers to a memzone structure residing in local memory, - accessing the memzone field in a shared rte_ring will cause an immediate segmentation fault. - -IVSHMEM Environment Configuration ---------------------------------- - -The steps needed to successfully run IVSHMEM applications are the following: - -* Compile a special version of QEMU from sources. - - The source code can be found on the QEMU website (currently, version 1.4.x is supported, but version 1.5.x is known to work also), - however, the source code will need to be patched to support using regular files as the IVSHMEM memory backend. - The patch is not included in the DPDK package, - but is available on the `Intel®DPDK-vswitch project webpage `_ - (either separately or in a DPDK vSwitch package). - -* Enable IVSHMEM library in the DPDK build configuration. - - In the default configuration, IVSHMEM library is not compiled. To compile the IVSHMEM library, - one has to either use one of the provided IVSHMEM targets - (for example, x86_64-ivshmem-linuxapp-gcc), - or set CONFIG_RTE_LIBRTE_IVSHMEM to "y" in the build configuration. - -* Set up hugepage memory on the virtual machine. - - The guest applications run as regular DPDK (primary) processes and thus need their own hugepage memory set up inside the VM. - The process is identical to the one described in the *DPDK Getting Started Guide*. - -Best Practices for Writing IVSHMEM Applications ------------------------------------------------ - -When considering the use of IVSHMEM for sharing memory, security implications need to be carefully evaluated. -IVSHMEM is not suitable for untrusted guests, as IVSHMEM is essentially a window into the host process memory. -This also has implications for the multiple VM scenarios. -While the IVSHMEM library tries to share as little memory as possible, -it is quite probable that data designated for one VM might also be present in an IVSMHMEM device designated for another VM. -Consequently, any shared memory corruption will affect both host and all VMs sharing that particular memory. - -IVSHMEM applications essentially behave like multi-process applications, -so it is important to implement access serialization to data and thread safety. -DPDK ring structures are already thread-safe, however, -any custom data structures that the user might need would have to be thread-safe also. - -Similar to regular DPDK multi-process applications, -it is not recommended to use function pointers as functions might have different memory addresses in different processes. - -It is best to avoid freeing the rte_mbuf structure on a different machine from where it was allocated, -that is, if the mbuf was allocated on the host, the host should free it. -Consequently, any packet transmission and reception should also happen on the same machine (whether virtual or physical). -Failing to do so may lead to data corruption in the mempool cache. - -Despite the IVSHMEM mechanism being zero-copy and having good performance, -it is still desirable to do processing in batches and follow other procedures described in -:ref:`Performance Optimization `. - -Best Practices for Running IVSHMEM Applications ------------------------------------------------ - -For performance reasons, -it is best to pin host processes and QEMU processes to different cores so that they do not interfere with each other. -If NUMA support is enabled, it is also desirable to keep host process' hugepage memory and QEMU process on the same NUMA node. - -For the best performance across all NUMA nodes, each QEMU core should be pinned to host CPU core on the appropriate NUMA node. -QEMU's virtual NUMA nodes should also be set up to correspond to physical NUMA nodes. -More on how to set up DPDK and QEMU NUMA support can be found in *DPDK Getting Started Guide* and -`QEMU documentation `_ respectively. -A script called cpu_layout.py is provided with the DPDK package (in the tools directory) -that can be used to identify which CPU cores correspond to which NUMA node. - -The QEMU IVSHMEM command line creation should be considered the last step before starting the virtual machine. -Currently, there is no hot plug support for QEMU IVSHMEM devices, -so one cannot add additional memory to an IVSHMEM device once it has been created. -Therefore, the correct sequence to run an IVSHMEM application is to run host application first, -obtain the command lines for each IVSHMEM device and then run all QEMU instances with guest applications afterwards. - -It is important to note that once QEMU is started, it holds on to the hugepages it uses for IVSHMEM devices. -As a result, if the user wishes to shut down or restart the IVSHMEM host application, -it is not enough to simply shut the application down. -The virtual machine must also be shut down (if not, it will hold onto outdated host data). diff --git a/dpdk/doc/guides/prog_guide/kernel_nic_interface.rst b/dpdk/doc/guides/prog_guide/kernel_nic_interface.rst index fac1960b..4b2dd770 100644 --- a/dpdk/doc/guides/prog_guide/kernel_nic_interface.rst +++ b/dpdk/doc/guides/prog_guide/kernel_nic_interface.rst @@ -69,7 +69,7 @@ The KNI kernel loadable module provides support for two types of devices: (simulating the RX side of the net driver). * For multiple kernel thread mode, maintains a kernel thread context for each KNI instance - (simulating the RX side of the new driver). + (simulating the RX side of the net driver). * Net device: @@ -102,6 +102,9 @@ Refer to rte_kni_common.h in the DPDK source code for more details. The physical addresses will be re-mapped into the kernel address space and stored in separate KNI contexts. +The affinity of kernel RX thread (both single and multi-threaded modes) is controlled by force_bind and +core_id config parameters. + The KNI interfaces can be deleted by a DPDK application dynamically after being created. Furthermore, all those KNI interfaces not deleted will be deleted on the release operation of the miscellaneous device (when the DPDK application is closed). @@ -165,114 +168,3 @@ The application handlers can be registered upon interface creation or explicitly This provides flexibility in multiprocess scenarios (where the KNI is created in the primary process but the callbacks are handled in the secondary one). The constraint is that a single process can register and handle the requests. - -KNI Working as a Kernel vHost Backend -------------------------------------- - -vHost is a kernel module usually working as the backend of virtio (a para- virtualization driver framework) -to accelerate the traffic from the guest to the host. -The DPDK Kernel NIC interface provides the ability to hookup vHost traffic into userspace DPDK application. -Together with the DPDK PMD virtio, it significantly improves the throughput between guest and host. -In the scenario where DPDK is running as fast path in the host, kni-vhost is an efficient path for the traffic. - -Overview -~~~~~~~~ - -vHost-net has three kinds of real backend implementations. They are: 1) tap, 2) macvtap and 3) RAW socket. -The main idea behind kni-vhost is making the KNI work as a RAW socket, attaching it as the backend instance of vHost-net. -It is using the existing interface with vHost-net, so it does not require any kernel hacking, -and is fully-compatible with the kernel vhost module. -As vHost is still taking responsibility for communicating with the front-end virtio, -it naturally supports both legacy virtio -net and the DPDK PMD virtio. -There is a little penalty that comes from the non-polling mode of vhost. -However, it scales throughput well when using KNI in multi-thread mode. - -.. _figure_vhost_net_arch2: - -.. figure:: img/vhost_net_arch.* - - vHost-net Architecture Overview - - -Packet Flow -~~~~~~~~~~~ - -There is only a minor difference from the original KNI traffic flows. -On transmit side, vhost kthread calls the RAW socket's ops sendmsg and it puts the packets into the KNI transmit FIFO. -On the receive side, the kni kthread gets packets from the KNI receive FIFO, puts them into the queue of the raw socket, -and wakes up the task in vhost kthread to begin receiving. -All the packet copying, irrespective of whether it is on the transmit or receive side, -happens in the context of vhost kthread. -Every vhost-net device is exposed to a front end virtio device in the guest. - -.. _figure_kni_traffic_flow: - -.. figure:: img/kni_traffic_flow.* - - KNI Traffic Flow - - -Sample Usage -~~~~~~~~~~~~ - -Before starting to use KNI as the backend of vhost, the CONFIG_RTE_KNI_VHOST configuration option must be turned on. -Otherwise, by default, KNI will not enable its backend support capability. - -Of course, as a prerequisite, the vhost/vhost-net kernel CONFIG should be chosen before compiling the kernel. - -#. Compile the DPDK and insert uio_pci_generic/igb_uio kernel modules as normal. - -#. Insert the KNI kernel module: - - .. code-block:: console - - insmod ./rte_kni.ko - - If using KNI in multi-thread mode, use the following command line: - - .. code-block:: console - - insmod ./rte_kni.ko kthread_mode=multiple - -#. Running the KNI sample application: - - .. code-block:: console - - examples/kni/build/app/kni -c -0xf0 -n 4 -- -p 0x3 -P --config="(0,4,6),(1,5,7)" - - This command runs the kni sample application with two physical ports. - Each port pins two forwarding cores (ingress/egress) in user space. - -#. Assign a raw socket to vhost-net during qemu-kvm startup. - The DPDK does not provide a script to do this since it is easy for the user to customize. - The following shows the key steps to launch qemu-kvm with kni-vhost: - - .. code-block:: bash - - #!/bin/bash - echo 1 > /sys/class/net/vEth0/sock_en - fd=`cat /sys/class/net/vEth0/sock_fd` - qemu-kvm \ - -name vm1 -cpu host -m 2048 -smp 1 -hda /opt/vm-fc16.img \ - -netdev tap,fd=$fd,id=hostnet1,vhost=on \ - -device virti-net-pci,netdev=hostnet1,id=net1,bus=pci.0,addr=0x4 - -It is simple to enable raw socket using sysfs sock_en and get raw socket fd using sock_fd under the KNI device node. - -Then, using the qemu-kvm command with the -netdev option to assign such raw socket fd as vhost's backend. - -.. note:: - - The key word tap must exist as qemu-kvm now only supports vhost with a tap backend, so here we cheat qemu-kvm by an existing fd. - -Compatibility Configure Option -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -There is a CONFIG_RTE_KNI_VHOST_VNET_HDR_EN configuration option in DPDK configuration file. -By default, it set to n, which means do not turn on the virtio net header, -which is used to support additional features (such as, csum offload, vlan offload, generic-segmentation and so on), -since the kni-vhost does not yet support those features. - -Even if the option is turned on, kni-vhost will ignore the information that the header contains. -When working with legacy virtio on the guest, it is better to turn off unsupported offload features using ethtool -K. -Otherwise, there may be problems such as an incorrect L4 checksum error. diff --git a/dpdk/doc/guides/prog_guide/link_bonding_poll_mode_drv_lib.rst b/dpdk/doc/guides/prog_guide/link_bonding_poll_mode_drv_lib.rst index 01ddcb91..1ef231df 100644 --- a/dpdk/doc/guides/prog_guide/link_bonding_poll_mode_drv_lib.rst +++ b/dpdk/doc/guides/prog_guide/link_bonding_poll_mode_drv_lib.rst @@ -356,7 +356,7 @@ Using Link Bonding Devices from the EAL Command Line Link bonding devices can be created at application startup time using the ``--vdev`` EAL command line option. The device name must start with the -eth_bond prefix followed by numbers or letters. The name must be unique for +net_bonding prefix followed by numbers or letters. The name must be unique for each device. Each device can have multiple options arranged in a comma separated list. Multiple devices definitions can be arranged by calling the ``--vdev`` option multiple times. @@ -365,7 +365,7 @@ Device names and bonding options must be separated by commas as shown below: .. code-block:: console - $RTE_TARGET/app/testpmd -c f -n 4 --vdev 'eth_bond0,bond_opt0=..,bond opt1=..'--vdev 'eth_bond1,bond _opt0=..,bond_opt1=..' + $RTE_TARGET/app/testpmd -l 0-3 -n 4 --vdev 'net_bonding0,bond_opt0=..,bond opt1=..'--vdev 'net_bonding1,bond _opt0=..,bond_opt1=..' Link Bonding EAL Options ^^^^^^^^^^^^^^^^^^^^^^^^ @@ -373,7 +373,7 @@ Link Bonding EAL Options There are multiple ways of definitions that can be assessed and combined as long as the following two rules are respected: -* A unique device name, in the format of eth_bondX is provided, +* A unique device name, in the format of net_bondingX is provided, where X can be any combination of numbers and/or letters, and the name is no greater than 32 characters long. @@ -465,22 +465,22 @@ Create a bonded device in round robin mode with two slaves specified by their PC .. code-block:: console - $RTE_TARGET/app/testpmd -c '0xf' -n 4 --vdev 'eth_bond0,mode=0, slave=0000:00a:00.01,slave=0000:004:00.00' -- --port-topology=chained + $RTE_TARGET/app/testpmd -l 0-3 -n 4 --vdev 'net_bonding0,mode=0, slave=0000:00a:00.01,slave=0000:004:00.00' -- --port-topology=chained Create a bonded device in round robin mode with two slaves specified by their PCI address and an overriding MAC address: .. code-block:: console - $RTE_TARGET/app/testpmd -c '0xf' -n 4 --vdev 'eth_bond0,mode=0, slave=0000:00a:00.01,slave=0000:004:00.00,mac=00:1e:67:1d:fd:1d' -- --port-topology=chained + $RTE_TARGET/app/testpmd -l 0-3 -n 4 --vdev 'net_bonding0,mode=0, slave=0000:00a:00.01,slave=0000:004:00.00,mac=00:1e:67:1d:fd:1d' -- --port-topology=chained Create a bonded device in active backup mode with two slaves specified, and a primary slave specified by their PCI addresses: .. code-block:: console - $RTE_TARGET/app/testpmd -c '0xf' -n 4 --vdev 'eth_bond0,mode=1, slave=0000:00a:00.01,slave=0000:004:00.00,primary=0000:00a:00.01' -- --port-topology=chained + $RTE_TARGET/app/testpmd -l 0-3 -n 4 --vdev 'net_bonding0,mode=1, slave=0000:00a:00.01,slave=0000:004:00.00,primary=0000:00a:00.01' -- --port-topology=chained Create a bonded device in balance mode with two slaves specified by their PCI addresses, and a transmission policy of layer 3 + 4 forwarding: .. code-block:: console - $RTE_TARGET/app/testpmd -c '0xf' -n 4 --vdev 'eth_bond0,mode=2, slave=0000:00a:00.01,slave=0000:004:00.00,xmit_policy=l34' -- --port-topology=chained + $RTE_TARGET/app/testpmd -l 0-3 -n 4 --vdev 'net_bonding0,mode=2, slave=0000:00a:00.01,slave=0000:004:00.00,xmit_policy=l34' -- --port-topology=chained diff --git a/dpdk/doc/guides/prog_guide/lpm6_lib.rst b/dpdk/doc/guides/prog_guide/lpm6_lib.rst index 0aea5c5c..f7915073 100644 --- a/dpdk/doc/guides/prog_guide/lpm6_lib.rst +++ b/dpdk/doc/guides/prog_guide/lpm6_lib.rst @@ -53,7 +53,7 @@ several thousand IPv6 rules, but the number can vary depending on the case. An LPM prefix is represented by a pair of parameters (128-bit key, depth), with depth in the range of 1 to 128. An LPM rule is represented by an LPM prefix and some user data associated with the prefix. The prefix serves as the unique identifier for the LPM rule. -In this implementation, the user data is 1-byte long and is called "next hop", +In this implementation, the user data is 21-bits long and is called "next hop", which corresponds to its main use of storing the ID of the next hop in a routing table entry. The main methods exported for the LPM component are: diff --git a/dpdk/doc/guides/prog_guide/mbuf_lib.rst b/dpdk/doc/guides/prog_guide/mbuf_lib.rst index 8e616826..6e73fc5a 100644 --- a/dpdk/doc/guides/prog_guide/mbuf_lib.rst +++ b/dpdk/doc/guides/prog_guide/mbuf_lib.rst @@ -175,8 +175,8 @@ a vxlan-encapsulated tcp packet: set out_ip checksum to 0 in the packet set out_udp checksum to pseudo header using rte_ipv4_phdr_cksum() - This is supported on hardware advertising DEV_TX_OFFLOAD_IPV4_CKSUM - and DEV_TX_OFFLOAD_UDP_CKSUM. + This is supported on hardware advertising DEV_TX_OFFLOAD_IPV4_CKSUM + and DEV_TX_OFFLOAD_UDP_CKSUM. - calculate checksum of in_ip:: @@ -228,8 +228,8 @@ a vxlan-encapsulated tcp packet: set in_ip checksum to 0 in the packet set in_tcp checksum to pseudo header using rte_ipv4_phdr_cksum() - This is supported on hardware advertising DEV_TX_OFFLOAD_IPV4_CKSUM, - DEV_TX_OFFLOAD_UDP_CKSUM and DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM. + This is supported on hardware advertising DEV_TX_OFFLOAD_IPV4_CKSUM, + DEV_TX_OFFLOAD_UDP_CKSUM and DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM. The list of flags and their precise meaning is described in the mbuf API documentation (rte_mbuf.h). Also refer to the testpmd source code @@ -253,7 +253,8 @@ Similarly, whenever the indirect buffer is detached, the reference counter on th If the resulting reference counter is equal to 0, the direct buffer is freed since it is no longer in use. There are a few things to remember when dealing with indirect buffers. -First of all, it is not possible to attach an indirect buffer to another indirect buffer. +First of all, an indirect buffer is never attached to another indirect buffer. +Attempting to attach buffer A to indirect buffer B that is attached to C, makes rte_pktmbuf_attach() automatically attach A to C, effectively cloning B. Secondly, for a buffer to become indirect, its reference counter must be equal to 1, that is, it must not be already referenced by another indirect buffer. Finally, it is not possible to reattach an indirect buffer to the direct buffer (unless it is detached first). diff --git a/dpdk/doc/guides/prog_guide/member_lib.rst b/dpdk/doc/guides/prog_guide/member_lib.rst new file mode 100644 index 00000000..caca8dc2 --- /dev/null +++ b/dpdk/doc/guides/prog_guide/member_lib.rst @@ -0,0 +1,420 @@ +.. BSD LICENSE + Copyright(c) 2017 Intel Corporation. All rights reserved. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +.. _member_library: + +Membership Library +================== + +Introduction +------------ + +The DPDK Membership Library provides an API for DPDK applications to insert a +new member, delete an existing member, or query the existence of a member in a +given set, or a group of sets. For the case of a group of sets, the library +will return not only whether the element has been inserted before in one of +the sets but also which set it belongs to. The Membership Library is an +extension and generalization of a traditional filter structure (for example +Bloom Filter [Member-bloom]) that has multiple usages in a wide variety of +workloads and applications. In general, the Membership Library is a data +structure that provides a "set-summary" on whether a member belongs to a set, +and as discussed in detail later, there are two advantages of using such a +set-summary rather than operating on a "full-blown" complete list of elements: +first, it has a much smaller storage requirement than storing the whole list of +elements themselves, and secondly checking an element membership (or other +operations) in this set-summary is much faster than checking it for the +original full-blown complete list of elements. + +We use the term "Set-Summary" in this guide to refer to the space-efficient, +probabilistic membership data structure that is provided by the library. A +membership test for an element will return the set this element belongs to or +that the element is "not-found" with very high probability of accuracy. Set-summary +is a fundamental data aggregation component that can be used in many network +(and other) applications. It is a crucial structure to address performance and +scalability issues of diverse network applications including overlay networks, +data-centric networks, flow table summaries, network statistics and +traffic monitoring. A set-summary is useful for applications who need to +include a list of elements while a complete list requires too much space +and/or too much processing cost. In these situations, the set-summary works as +a lossy hash-based representation of a set of members. It can dramatically +reduce space requirement and significantly improve the performance of set +membership queries at the cost of introducing a very small membership test error +probability. + +.. _figure_membership1: +.. figure:: img/member_i1.* + + Example Usages of Membership Library + +There are various usages for a Membership Library in a very +large set of applications and workloads. Interested readers can refer to +[Member-survey] for a survey of possible networking usages. The above figure +provide a small set of examples of using the Membership Library: + +* Sub-figure (a) + depicts a distributed web cache architecture where a collection of proxies + attempt to share their web caches (cached from a set of back-end web servers) to + provide faster responses to clients, and the proxies use the Membership + Library to share summaries of what web pages/objects they are caching. With the + Membership Library, a proxy receiving an http request will inquire the + set-summary to find its location and quickly determine whether to retrieve the + requested web page from a nearby proxy or from a back-end web server. + +* Sub-figure (b) depicts another example for using the Membership Library to + prevent routing loops which is typically done using slow TTL countdown and + dropping packets when TTL expires. As shown in Sub-figure (b), an embedded + set-summary in the packet header itself can be used to summarize the set of + nodes a packet has gone through, and each node upon receiving a packet can check + whether its id is a member of the set of visited nodes, and if it is, then a + routing loop is detected. + +* Sub-Figure (c) presents another usage of the Membership + Library to load-balance flows to worker threads with in-order guarantee where a + set-summary is used to query if a packet belongs to an existing flow or a new + flow. Packets belonging to a new flow are forwarded to the current least loaded + worker thread, while those belonging to an existing flow are forwarded to the + pre-assigned thread to guarantee in-order processing. + +* Sub-figure (d) highlights + yet another usage example in the database domain where a set-summary is used to + determine joins between sets instead of creating a join by comparing each + element of a set against the other elements in a different set, a join is done + on the summaries since they can efficiently encode members of a given set. + +Membership Library is a configurable library that is optimized to cover set +membership functionality for both a single set and multi-set scenarios. Two set-summary +schemes are presented including (a) vector of Bloom Filters and (b) Hash-Table based +set-summary schemes with and without false negative probability. +This guide first briefly describes these different types of set-summaries, usage examples for each, +and then it highlights the Membership Library API. + +Vector of Bloom Filters +----------------------- + +Bloom Filter (BF) [Member-bloom] is a well-known space-efficient +probabilistic data structure that answers set membership queries (test whether +an element is a member of a set) with some probability of false positives and +zero false negatives; a query for an element returns either it is "possibly in +a set" (with very high probability) or "definitely not in a set". + +The BF is a method for representing a set of ``n`` elements (for example flow keys +in network applications domain) to support membership queries. The idea of BF is +to allocate a bit-vector ``v`` with ``m`` bits, which are initially all set to 0. Then +it chooses ``k`` independent hash functions ``h1``, ``h2``, ... ``hk`` with hash values range from +``0`` to ``m-1`` to perform hashing calculations on each element to be inserted. Every time when an +element ``X`` being inserted into the set, the bits at positions ``h1(X)``, ``h2(X)``, ... +``hk(X)`` in ``v`` are set to 1 (any particular bit might be set to 1 multiple times +for multiple different inserted elements). Given a query for any element ``Y``, the +bits at positions ``h1(Y)``, ``h2(Y)``, ... ``hk(Y)`` are checked. If any of them is 0, +then Y is definitely not in the set. Otherwise there is a high probability that +Y is a member of the set with certain false positive probability. As shown in +the next equation, the false positive probability can be made arbitrarily small +by changing the number of hash functions (``k``) and the vector length (``m``). + +.. _figure_membership2: +.. figure:: img/member_i2.* + + Bloom Filter False Positive Probability + +Without BF, an accurate membership testing could involve a costly hash table +lookup and full element comparison. The advantage of using a BF is to simplify +the membership test into a series of hash calculations and memory accesses for a +small bit-vector, which can be easily optimized. Hence the lookup throughput +(set membership test) can be significantly faster than a normal hash table +lookup with element comparison. + +.. _figure_membership3: +.. figure:: img/member_i3.* + + Detecting Routing Loops Using BF + +BF is used for applications that need only one set, and the +membership of elements is checked against the BF. The example discussed +in the above figure is one example of potential applications that uses only one +set to capture the node IDs that have been visited so far by the packet. Each +node will then check this embedded BF in the packet header for its own id, and +if the BF indicates that the current node is definitely not in the set then a +loop-free route is guaranteed. + + +.. _figure_membership4: +.. figure:: img/member_i4.* + + Vector Bloom Filter (vBF) Overview + +To support membership test for both multiple sets and a single set, +the library implements a Vector Bloom Filter (vBF) scheme. +vBF basically composes multiple bloom filters into a vector of bloom filers. +The membership test is conducted on all of the +bloom filters concurrently to determine which set(s) it belongs to or none of +them. The basic idea of vBF is shown in the above figure where an element is +used to address multiple bloom filters concurrently and the bloom filter +index(es) with a hit is returned. + +.. _figure_membership5: +.. figure:: img/member_i5.* + + vBF for Flow Scheduling to Worker Thread + +As previously mentioned, there are many usages of such structures. vBF is used +for applications that need to check membership against multiple sets +simultaneously. The example shown in the above figure uses a set to capture +all flows being assigned for processing at a given worker thread. Upon receiving +a packet the vBF is used to quickly figure out if this packet belongs to a new flow +so as to be forwarded to the current least loaded worker thread, or otherwise it +should be queued for an existing thread to guarantee in-order processing (i.e. +the property of vBF to indicate right away that a given flow is a new one or +not is critical to minimize response time latency). + +It should be noted that vBF can be implemented using a set of single bloom +filters with sequential lookup of each BF. However, being able to concurrently +search all set-summaries is a big throughput advantage. In the library, certain +parallelism is realized by the implementation of checking all bloom filters +together. + + +Hash-Table based Set-Summaries +------------------------------ + +Hash-table based set-summary (HTSS) is another scheme in the membership library. +Cuckoo filter [Member-cfilter] is an example of HTSS. +HTSS supports multi-set membership testing like +vBF does. However, while vBF is better for a small number of targets, HTSS is more suitable +and can easily outperform vBF when the number of sets is +large, since HTSS uses a single hash table for membership testing while vBF +requires testing a series of Bloom Filters each corresponding to one set. +As a result, generally speaking vBF is more adequate for the case of a small limited number of sets +while HTSS should be used with a larger number of sets. + +.. _figure_membership6: +.. figure:: img/member_i6.* + + Using HTSS for Attack Signature Matching + +As shown in the above figure, attack signature matching where each set +represents a certain signature length (for correctness of this example, an +attack signature should not be a subset of another one) in the payload is a good +example for using HTSS with 0% false negative (i.e., when an element returns not +found, it has a 100% certainty that it is not a member of any set). The packet +inspection application benefits from knowing right away that the current payload +does not match any attack signatures in the database to establish its +legitimacy, otherwise a deep inspection of the packet is needed. + +HTSS employs a similar but simpler data structure to a traditional hash table, +and the major difference is that HTSS stores only the signatures but not the +full keys/elements which can significantly reduce the footprint of the table. +Along with the signature, HTSS also stores a value to indicate the target set. +When looking up an element, the element is hashed and the HTSS is addressed +to retrieve the signature stored. If the signature matches then the value is +retrieved corresponding to the index of the target set which the element belongs +to. Because signatures can collide, HTSS can still has false positive +probability. Furthermore, if elements are allowed to be +overwritten or evicted when the hash table becomes full, it will also have a +false negative probability. We discuss this case in the next section. + +Set-Summaries with False Negative Probability +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +As previously mentioned, traditional set-summaries (e.g. Bloom Filters) do not +have a false negative probability, i.e., it is 100% certain when an element +returns "not to be present" for a given set. However, the Membership Library +also supports a set-summary probabilistic data structure based on HTSS which +allows for false negative probability. + +In HTSS, when the hash table becomes full, keys/elements will fail to be added +into the table and the hash table has to be resized to accommodate for these new +elements, which can be expensive. However, if we allow new elements to overwrite +or evict existing elements (as a cache typically does), then the resulting +set-summary will begin to have false negative probability. This is because the +element that was evicted from the set-summary may still be present in the target +set. For subsequent inquiries the set-summary will falsely report the element +not being in the set, hence having a false negative probability. + +The major usage of HTSS with false negative is to use it as a cache for +distributing elements to different target sets. By allowing HTSS to evict old +elements, the set-summary can keep track of the most recent elements +(i.e. active) as a cache typically does. Old inactive elements (infrequently +used elements) will automatically and eventually get evicted from the +set-summary. It is worth noting that the set-summary still has false positive +probability, which means the application either can tolerate certain false positive +or it has fall-back path when false positive happens. + +.. _figure_membership7: +.. figure:: img/member_i7.* + + Using HTSS with False Negatives for Wild Card Classification + +HTSS with false negative (i.e. a cache) also has its wide set of applications. +For example wild card flow classification (e.g. ACL rules) highlighted in the +above figure is an example of such application. In that case each target set +represents a sub-table with rules defined by a certain flow mask. The flow masks +are non-overlapping, and for flows matching more than one rule only the highest +priority one is inserted in the corresponding sub-table (interested readers can +refer to the Open vSwitch (OvS) design of Mega Flow Cache (MFC) [Member-OvS] +for further details). Typically the rules will have a large number of distinct +unique masks and hence, a large number of target sets each corresponding to one +mask. Because the active set of flows varies widely based on the network +traffic, HTSS with false negative will act as a cache for pair for the current active set of flows. When a miss occurs (as +shown in red in the above figure) the sub-tables will be searched sequentially +one by one for a possible match, and when found the flow key and target +sub-table will be inserted into the set-summary (i.e. cache insertion) so +subsequent packets from the same flow don’t incur the overhead of the +sequential search of sub-tables. + +Library API Overview +-------------------- + +The design goal of the Membership Library API is to be as generic as possible to +support all the different types of set-summaries we discussed in previous +sections and beyond. Fundamentally, the APIs need to include creation, +insertion, deletion, and lookup. + + +Set-summary Create +~~~~~~~~~~~~~~~~~~ + +The ``rte_member_create()`` function is used to create a set-summary structure, the input parameter +is a struct to pass in parameters that needed to initialize the set-summary, while the function returns the +pointer to the created set-summary or ``NULL`` if the creation failed. + +The general input arguments used when creating the set-summary should include ``name`` +which is the name of the created set-summary, *type* which is one of the types +supported by the library (e.g. ``RTE_MEMBER_TYPE_HT`` for HTSS or ``RTE_MEMBER_TYPE_VBF`` for vBF), and ``key_len`` +which is the length of the element/key. There are other parameters +are only used for certain type of set-summary, or which have a slightly different meaning for different types of set-summary. +For example, ``num_keys`` parameter means the maximum number of entries for Hash table based set-summary. +However, for bloom filter, this value means the expected number of keys that could be +inserted into the bloom filter(s). The value is used to calculate the size of each +bloom filter. + +We also pass two seeds: ``prim_hash_seed`` and +``sec_hash_seed`` for the primary and secondary hash functions to calculate two independent hash values. +``socket_id`` parameter is the NUMA socket ID for the memory used to create the +set-summary. For HTSS, another parameter ``is_cache`` is used to indicate +if this set-summary is a cache (i.e. with false negative probability) or not. +For vBF, extra parameters are needed. For example, ``num_set`` is the number of +sets needed to initialize the vector bloom filters. This number is equal to the +number of bloom filters will be created. +``false_pos_rate`` is the false positive rate. num_keys and false_pos_rate will be used to determine +the number of hash functions and the bloom filter size. + + +Set-summary Element Insertion +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``rte_member_add()`` function is used to insert an element/key into a set-summary structure. If it fails an +error is returned. For success the returned value is dependent on the +set-summary mode to provide extra information for the users. For vBF +mode, a return value of 0 means a successful insert. For HTSS mode without false negative, the insert +could fail with ``-ENOSPC`` if the table is full. With false negative (i.e. cache mode), +for insert that does not cause any eviction (i.e. no overwriting happens to an +existing entry) the return value is 0. For insertion that causes eviction, the return +value is 1 to indicate such situation, but it is not an error. + +The input arguments for the function should include the ``key`` which is a pointer to the element/key that needs to +be added to the set-summary, and ``set_id`` which is the set id associated +with the key that needs to be added. + + +Set-summary Element Lookup +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``rte_member_lookup()`` function looks up a single key/element in the set-summary structure. It +returns as soon as the first match is found. The return value is 1 if a +match is found and 0 otherwise. The arguments for the function include ``key`` which is a pointer to the +element/key that needs to be looked up, and ``set_id`` which is used to return the +first target set id where the key has matched, if any. + +The ``rte_member_lookup_bulk()`` function is used to look up a bulk of keys/elements in the +set-summary structure for their first match. Each key lookup returns as soon as the first match is found. The +return value is the number of keys that find a match. The arguments of the function include ``keys`` +which is a pointer to a bulk of keys that are to be looked up, +``num_keys`` is the number +of keys that will be looked up, and ``set_ids`` are the return target set +ids for the first match found for each of the input keys. ``set_ids`` is an array +needs to be sized according to the ``num_keys``. If there is no match, the set id +for that key will be set to RTE_MEMBER_NO_MATCH. + +The ``rte_member_lookup_multi()`` function looks up a single key/element in the +set-summary structure for multiple matches. It +returns ALL the matches (possibly more than one) found for this key when it +is matched against all target sets (it is worth noting that for cache mode HTSS, +the current implementation matches at most one target set). The return value is +the number of matches +that was found for this key (for cache mode HTSS the return value +should be at most 1). The arguments for the function include ``key`` which is a pointer to the +element/key that needs to be looked up, ``max_match_per_key`` which is to indicate the maximum number of matches +the user expects to find for each key, and ``set_id`` which is used to return all +target set ids where the key has matched, if any. The ``set_id`` array should be sized +according to ``max_match_per_key``. For vBF, the maximum number of matches per key is equal +to the number of sets. For HTSS, the maximum number of matches per key is equal to two time +entry count per bucket. ``max_match_per_key`` should be equal or smaller than the maximum number of +possible matches. + +The ``rte_membership_lookup_multi_bulk()`` function looks up a bulk of keys/elements in the +set-summary structure for multiple matches, each key lookup returns ALL the matches (possibly more +than one) found for this key when it is matched against all target sets (cache mode HTSS +matches at most one target set). The +return value is the number of keys that find one or more matches in the +set-summary structure. The arguments of the +function include ``keys`` which is +a pointer to a bulk of keys that are to be looked up, ``num_keys`` is the number +of keys that will be looked up, ``max_match_per_key`` is the possible +maximum number of matches for each key, ``match_count`` which is the returned number +of matches for each key, and ``set_ids`` are the returned target set +ids for all matches found for each keys. ``set_ids`` is 2-D array +containing a 1-D array for each key (the size of 1-D array per key should be set by the user according to ``max_match_per_key``). +``max_match_per_key`` should be equal or smaller than the maximum number of +possible matches, similar to ``rte_member_lookup_multi``. + + +Set-summary Element Delete +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``rte_membership_delete()`` function deletes an element/key from a set-summary structure, if it fails +an error is returned. The input arguments should include ``key`` which is a pointer to the +element/key that needs to be deleted from the set-summary, and ``set_id`` +which is the set id associated with the key to delete. It is worth noting that current +implementation of vBF does not support deletion [1]_. An error code ``-EINVAL`` will be returned. + +.. [1] Traditional bloom filter does not support proactive deletion. Supporting proactive deletion require additional implementation and performance overhead. + +References +----------- + +[Member-bloom] B H Bloom, "Space/Time Trade-offs in Hash Coding with Allowable Errors," Communications of the ACM, 1970. + +[Member-survey] A Broder and M Mitzenmacher, "Network Applications of Bloom Filters: A Survey," in Internet Mathematics, 2005. + +[Member-cfilter] B Fan, D G Andersen and M Kaminsky, "Cuckoo Filter: Practically Better Than Bloom," in Conference on emerging Networking Experiments and Technologies, 2014. + +[Member-OvS] B Pfaff, "The Design and Implementation of Open vSwitch," in NSDI, 2015. diff --git a/dpdk/doc/guides/prog_guide/metrics_lib.rst b/dpdk/doc/guides/prog_guide/metrics_lib.rst new file mode 100644 index 00000000..d52204fd --- /dev/null +++ b/dpdk/doc/guides/prog_guide/metrics_lib.rst @@ -0,0 +1,300 @@ +.. BSD LICENSE + Copyright(c) 2017 Intel Corporation. All rights reserved. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +.. _Metrics_Library: + +Metrics Library +=============== + +The Metrics library implements a mechanism by which *producers* can +publish numeric information for later querying by *consumers*. In +practice producers will typically be other libraries or primary +processes, whereas consumers will typically be applications. + +Metrics themselves are statistics that are not generated by PMDs. Metric +information is populated using a push model, where producers update the +values contained within the metric library by calling an update function +on the relevant metrics. Consumers receive metric information by querying +the central metric data, which is held in shared memory. + +For each metric, a separate value is maintained for each port id, and +when publishing metric values the producers need to specify which port is +being updated. In addition there is a special id ``RTE_METRICS_GLOBAL`` +that is intended for global statistics that are not associated with any +individual device. Since the metrics library is self-contained, the only +restriction on port numbers is that they are less than ``RTE_MAX_ETHPORTS`` +- there is no requirement for the ports to actually exist. + +Initialising the library +------------------------ + +Before the library can be used, it has to be initialized by calling +``rte_metrics_init()`` which sets up the metric store in shared memory. +This is where producers will publish metric information to, and where +consumers will query it from. + +.. code-block:: c + + rte_metrics_init(rte_socket_id()); + +This function **must** be called from a primary process, but otherwise +producers and consumers can be in either primary or secondary processes. + +Registering metrics +------------------- + +Metrics must first be *registered*, which is the way producers declare +the names of the metrics they will be publishing. Registration can either +be done individually, or a set of metrics can be registered as a group. +Individual registration is done using ``rte_metrics_reg_name()``: + +.. code-block:: c + + id_1 = rte_metrics_reg_name("mean_bits_in"); + id_2 = rte_metrics_reg_name("mean_bits_out"); + id_3 = rte_metrics_reg_name("peak_bits_in"); + id_4 = rte_metrics_reg_name("peak_bits_out"); + +or alternatively, a set of metrics can be registered together using +``rte_metrics_reg_names()``: + +.. code-block:: c + + const char * const names[] = { + "mean_bits_in", "mean_bits_out", + "peak_bits_in", "peak_bits_out", + }; + id_set = rte_metrics_reg_names(&names[0], 4); + +If the return value is negative, it means registration failed. Otherwise +the return value is the *key* for the metric, which is used when updating +values. A table mapping together these key values and the metrics' names +can be obtained using ``rte_metrics_get_names()``. + +Updating metric values +---------------------- + +Once registered, producers can update the metric for a given port using +the ``rte_metrics_update_value()`` function. This uses the metric key +that is returned when registering the metric, and can also be looked up +using ``rte_metrics_get_names()``. + +.. code-block:: c + + rte_metrics_update_value(port_id, id_1, values[0]); + rte_metrics_update_value(port_id, id_2, values[1]); + rte_metrics_update_value(port_id, id_3, values[2]); + rte_metrics_update_value(port_id, id_4, values[3]); + +if metrics were registered as a single set, they can either be updated +individually using ``rte_metrics_update_value()``, or updated together +using the ``rte_metrics_update_values()`` function: + +.. code-block:: c + + rte_metrics_update_value(port_id, id_set, values[0]); + rte_metrics_update_value(port_id, id_set + 1, values[1]); + rte_metrics_update_value(port_id, id_set + 2, values[2]); + rte_metrics_update_value(port_id, id_set + 3, values[3]); + + rte_metrics_update_values(port_id, id_set, values, 4); + +Note that ``rte_metrics_update_values()`` cannot be used to update +metric values from *multiple* *sets*, as there is no guarantee two +sets registered one after the other have contiguous id values. + +Querying metrics +---------------- + +Consumers can obtain metric values by querying the metrics library using +the ``rte_metrics_get_values()`` function that return an array of +``struct rte_metric_value``. Each entry within this array contains a metric +value and its associated key. A key-name mapping can be obtained using the +``rte_metrics_get_names()`` function that returns an array of +``struct rte_metric_name`` that is indexed by the key. The following will +print out all metrics for a given port: + +.. code-block:: c + + void print_metrics() { + struct rte_metric_value *metrics; + struct rte_metric_name *names; + int len; + + len = rte_metrics_get_names(NULL, 0); + if (len < 0) { + printf("Cannot get metrics count\n"); + return; + } + if (len == 0) { + printf("No metrics to display (none have been registered)\n"); + return; + } + metrics = malloc(sizeof(struct rte_metric_value) * len); + names = malloc(sizeof(struct rte_metric_name) * len); + if (metrics == NULL || names == NULL) { + printf("Cannot allocate memory\n"); + free(metrics); + free(names); + return; + } + ret = rte_metrics_get_values(port_id, metrics, len); + if (ret < 0 || ret > len) { + printf("Cannot get metrics values\n"); + free(metrics); + free(names); + return; + } + printf("Metrics for port %i:\n", port_id); + for (i = 0; i < len; i++) + printf(" %s: %"PRIu64"\n", + names[metrics[i].key].name, metrics[i].value); + free(metrics); + free(names); + } + + +Bit-rate statistics library +--------------------------- + +The bit-rate library calculates the exponentially-weighted moving +average and peak bit-rates for each active port (i.e. network device). +These statistics are reported via the metrics library using the +following names: + + - ``mean_bits_in``: Average inbound bit-rate + - ``mean_bits_out``: Average outbound bit-rate + - ``ewma_bits_in``: Average inbound bit-rate (EWMA smoothed) + - ``ewma_bits_out``: Average outbound bit-rate (EWMA smoothed) + - ``peak_bits_in``: Peak inbound bit-rate + - ``peak_bits_out``: Peak outbound bit-rate + +Once initialised and clocked at the appropriate frequency, these +statistics can be obtained by querying the metrics library. + +Initialization +~~~~~~~~~~~~~~ + +Before the library can be used, it has to be initialised by calling +``rte_stats_bitrate_create()``, which will return a bit-rate +calculation object. Since the bit-rate library uses the metrics library +to report the calculated statistics, the bit-rate library then needs to +register the calculated statistics with the metrics library. This is +done using the helper function ``rte_stats_bitrate_reg()``. + +.. code-block:: c + + struct rte_stats_bitrates *bitrate_data; + + bitrate_data = rte_stats_bitrate_create(); + if (bitrate_data == NULL) + rte_exit(EXIT_FAILURE, "Could not allocate bit-rate data.\n"); + rte_stats_bitrate_reg(bitrate_data); + +Controlling the sampling rate +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Since the library works by periodic sampling but does not use an +internal thread, the application has to periodically call +``rte_stats_bitrate_calc()``. The frequency at which this function +is called should be the intended sampling rate required for the +calculated statistics. For instance if per-second statistics are +desired, this function should be called once a second. + +.. code-block:: c + + tics_datum = rte_rdtsc(); + tics_per_1sec = rte_get_timer_hz(); + + while( 1 ) { + /* ... */ + tics_current = rte_rdtsc(); + if (tics_current - tics_datum >= tics_per_1sec) { + /* Periodic bitrate calculation */ + for (idx_port = 0; idx_port < cnt_ports; idx_port++) + rte_stats_bitrate_calc(bitrate_data, idx_port); + tics_datum = tics_current; + } + /* ... */ + } + + +Latency statistics library +-------------------------- + +The latency statistics library calculates the latency of packet +processing by a DPDK application, reporting the minimum, average, +and maximum nano-seconds that packet processing takes, as well as +the jitter in processing delay. These statistics are then reported +via the metrics library using the following names: + + - ``min_latency_ns``: Minimum processing latency (nano-seconds) + - ``avg_latency_ns``: Average processing latency (nano-seconds) + - ``mac_latency_ns``: Maximum processing latency (nano-seconds) + - ``jitter_ns``: Variance in processing latency (nano-seconds) + +Once initialised and clocked at the appropriate frequency, these +statistics can be obtained by querying the metrics library. + +Initialization +~~~~~~~~~~~~~~ + +Before the library can be used, it has to be initialised by calling +``rte_latencystats_init()``. + +.. code-block:: c + + lcoreid_t latencystats_lcore_id = -1; + + int ret = rte_latencystats_init(1, NULL); + if (ret) + rte_exit(EXIT_FAILURE, "Could not allocate latency data.\n"); + + +Triggering statistic updates +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``rte_latencystats_update()`` function needs to be called +periodically so that latency statistics can be updated. + +.. code-block:: c + + if (latencystats_lcore_id == rte_lcore_id()) + rte_latencystats_update(); + +Library shutdown +~~~~~~~~~~~~~~~~ + +When finished, ``rte_latencystats_uninit()`` needs to be called to +de-initialise the latency library. + +.. code-block:: c + + rte_latencystats_uninit(); diff --git a/dpdk/doc/guides/prog_guide/multi_proc_support.rst b/dpdk/doc/guides/prog_guide/multi_proc_support.rst index badd102e..9a9dca7f 100644 --- a/dpdk/doc/guides/prog_guide/multi_proc_support.rst +++ b/dpdk/doc/guides/prog_guide/multi_proc_support.rst @@ -35,7 +35,7 @@ Multi-process Support In the DPDK, multi-process support is designed to allow a group of DPDK processes to work together in a simple transparent manner to perform packet processing, -or other workloads, on Intel® architecture hardware. +or other workloads. To support this functionality, a number of additions have been made to the core DPDK Environment Abstraction Layer (EAL). @@ -173,7 +173,7 @@ Some of these are documented below: so it is recommended that it be disabled only when absolutely necessary, and only when the implications of this change have been understood. -* All DPDK processes running as a single application and using shared memory must have distinct coremask arguments. +* All DPDK processes running as a single application and using shared memory must have distinct coremask/corelist arguments. It is not possible to have a primary and secondary instance, or two secondary instances, using any of the same logical cores. Attempting to do so can cause corruption of memory pool caches, among other issues. diff --git a/dpdk/doc/guides/prog_guide/overview.rst b/dpdk/doc/guides/prog_guide/overview.rst index 68cc75cc..9986e3cb 100644 --- a/dpdk/doc/guides/prog_guide/overview.rst +++ b/dpdk/doc/guides/prog_guide/overview.rst @@ -157,7 +157,7 @@ The mbuf library provides the facility to create and destroy buffers that may be used by the DPDK application to store message buffers. The message buffers are created at startup time and stored in a mempool, using the DPDK mempool library. -This library provide an API to allocate/free mbufs, manipulate control message buffers (ctrlmbuf) which are generic message buffers, +This library provides an API to allocate/free mbufs, manipulate control message buffers (ctrlmbuf) which are generic message buffers, and packet buffers (pktmbuf) which are used to carry network packets. Network Packet Buffer Management is described in :ref:`Mbuf Library `. diff --git a/dpdk/doc/guides/prog_guide/packet_classif_access_ctrl.rst b/dpdk/doc/guides/prog_guide/packet_classif_access_ctrl.rst index 5fd3d348..a6bee9ba 100644 --- a/dpdk/doc/guides/prog_guide/packet_classif_access_ctrl.rst +++ b/dpdk/doc/guides/prog_guide/packet_classif_access_ctrl.rst @@ -329,8 +329,9 @@ When creating a set of rules, for each rule, additional information must be supp Each set could be assigned its own category and by combining them into a single database, one lookup returns a result for each of the four sets. -* **userdata**: A user-defined field that could be any value except zero. +* **userdata**: A user-defined value. For each category, a successful match returns the userdata field of the highest priority matched rule. + When no rules match, returned value is zero. .. note:: diff --git a/dpdk/doc/guides/prog_guide/packet_distrib_lib.rst b/dpdk/doc/guides/prog_guide/packet_distrib_lib.rst index b5bdabbf..ee2b1a66 100644 --- a/dpdk/doc/guides/prog_guide/packet_distrib_lib.rst +++ b/dpdk/doc/guides/prog_guide/packet_distrib_lib.rst @@ -42,6 +42,10 @@ The model of operation is shown in the diagram below. Packet Distributor mode of operation +There are two modes of operation of the API in the distributor library, +one which sends one packet at a time to workers using 32-bits for flow_id, +and an optimized mode which sends bursts of up to 8 packets at a time to workers, using 15 bits of flow_id. +The mode is selected by the type field in the ``rte_distributor_create()`` function. Distributor Core Operation -------------------------- diff --git a/dpdk/doc/guides/prog_guide/perf_opt_guidelines.rst b/dpdk/doc/guides/prog_guide/perf_opt_guidelines.rst index 7c24aa48..286bf983 100644 --- a/dpdk/doc/guides/prog_guide/perf_opt_guidelines.rst +++ b/dpdk/doc/guides/prog_guide/perf_opt_guidelines.rst @@ -38,7 +38,7 @@ Performance Optimization Guidelines Introduction ------------ -The following sections describe optimizations used in the DPDK and optimizations that should be considered for a new applications. +The following sections describe optimizations used in DPDK and optimizations that should be considered for new applications. They also highlight the performance-impacting coding techniques that should, and should not be, used when developing an application using the DPDK. diff --git a/dpdk/doc/guides/prog_guide/poll_mode_drv.rst b/dpdk/doc/guides/prog_guide/poll_mode_drv.rst index bf3ea9fd..6a0c9f99 100644 --- a/dpdk/doc/guides/prog_guide/poll_mode_drv.rst +++ b/dpdk/doc/guides/prog_guide/poll_mode_drv.rst @@ -84,7 +84,7 @@ Whenever needed and appropriate, asynchronous communication should be introduced Avoiding lock contention is a key issue in a multi-core environment. To address this issue, PMDs are designed to work with per-core private resources as much as possible. -For example, a PMD maintains a separate transmit queue per-core, per-port. +For example, a PMD maintains a separate transmit queue per-core, per-port, if the PMD is not ``DEV_TX_OFFLOAD_MT_LOCKFREE`` capable. In the same way, every receive queue of a port is assigned to and polled by a single logical core (lcore). To comply with Non-Uniform Memory Access (NUMA), memory management is designed to assign to each logical core @@ -146,6 +146,16 @@ This is also true for the pipe-line model provided all logical cores used are lo Multiple logical cores should never share receive or transmit queues for interfaces since this would require global locks and hinder performance. +If the PMD is ``DEV_TX_OFFLOAD_MT_LOCKFREE`` capable, multiple threads can invoke ``rte_eth_tx_burst()`` +concurrently on the same tx queue without SW lock. This PMD feature found in some NICs and useful in the following use cases: + +* Remove explicit spinlock in some applications where lcores are not mapped to Tx queues with 1:1 relation. + +* In the eventdev use case, avoid dedicating a separate TX core for transmitting and thus + enables more scaling as all workers can send the packets. + +See `Hardware Offload`_ for ``DEV_TX_OFFLOAD_MT_LOCKFREE`` capability probing details. + Device Identification and Configuration --------------------------------------- @@ -200,8 +210,8 @@ Ethernet* flow control (pause frame) can be configured on the individual port. Refer to the testpmd source code for details. Also, L4 (UDP/TCP/ SCTP) checksum offload by the NIC can be enabled for an individual packet as long as the packet mbuf is set up correctly. See `Hardware Offload`_ for details. -Configuration of Transmit and Receive Queues -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Configuration of Transmit Queues +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Each transmit queue is independently configured with the following information: @@ -249,12 +259,49 @@ One descriptor in the TX ring is used as a sentinel to avoid a hardware race con When configuring for DCB operation, at port initialization, both the number of transmit queues and the number of receive queues must be set to 128. +Free Tx mbuf on Demand +~~~~~~~~~~~~~~~~~~~~~~ + +Many of the drivers do not release the mbuf back to the mempool, or local cache, +immediately after the packet has been transmitted. +Instead, they leave the mbuf in their Tx ring and +either perform a bulk release when the ``tx_rs_thresh`` has been crossed +or free the mbuf when a slot in the Tx ring is needed. + +An application can request the driver to release used mbufs with the ``rte_eth_tx_done_cleanup()`` API. +This API requests the driver to release mbufs that are no longer in use, +independent of whether or not the ``tx_rs_thresh`` has been crossed. +There are two scenarios when an application may want the mbuf released immediately: + +* When a given packet needs to be sent to multiple destination interfaces + (either for Layer 2 flooding or Layer 3 multi-cast). + One option is to make a copy of the packet or a copy of the header portion that needs to be manipulated. + A second option is to transmit the packet and then poll the ``rte_eth_tx_done_cleanup()`` API + until the reference count on the packet is decremented. + Then the same packet can be transmitted to the next destination interface. + The application is still responsible for managing any packet manipulations needed + between the different destination interfaces, but a packet copy can be avoided. + This API is independent of whether the packet was transmitted or dropped, + only that the mbuf is no longer in use by the interface. + +* Some applications are designed to make multiple runs, like a packet generator. + For performance reasons and consistency between runs, + the application may want to reset back to an initial state + between each run, where all mbufs are returned to the mempool. + In this case, it can call the ``rte_eth_tx_done_cleanup()`` API + for each destination interface it has been using + to request it to release of all its used mbufs. + +To determine if a driver supports this API, check for the *Free Tx mbuf on demand* feature +in the *Network Interface Controller Drivers* document. + Hardware Offload ~~~~~~~~~~~~~~~~ Depending on driver capabilities advertised by ``rte_eth_dev_info_get()``, the PMD may support hardware offloading -feature like checksumming, TCP segmentation or VLAN insertion. +feature like checksumming, TCP segmentation, VLAN insertion or +lockfree multithreaded TX burst on the same TX queue. The support of these offload features implies the addition of dedicated status bit(s) and value field(s) into the rte_mbuf data structure, along @@ -263,6 +310,26 @@ exported by each PMD. The list of flags and their precise meaning is described in the mbuf API documentation and in the in :ref:`Mbuf Library `, section "Meta Information". +Per-Port and Per-Queue Offloads +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +In the DPDK offload API, offloads are divided into per-port and per-queue offloads. +The different offloads capabilities can be queried using ``rte_eth_dev_info_get()``. +Supported offloads can be either per-port or per-queue. + +Offloads are enabled using the existing ``DEV_TX_OFFLOAD_*`` or ``DEV_RX_OFFLOAD_*`` flags. +Per-port offload configuration is set using ``rte_eth_dev_configure``. +Per-queue offload configuration is set using ``rte_eth_rx_queue_setup`` and ``rte_eth_tx_queue_setup``. +To enable per-port offload, the offload should be set on both device configuration and queue setup. +In case of a mixed configuration the queue setup shall return with an error. +To enable per-queue offload, the offload can be set only on the queue setup. +Offloads which are not enabled are disabled by default. + +For an application to use the Tx offloads API it should set the ``ETH_TXQ_FLAGS_IGNORE`` flag in the ``txq_flags`` field located in ``rte_eth_txconf`` struct. +In such cases it is not required to set other flags in ``txq_flags``. +For an application to use the Rx offloads API it should set the ``ignore_offload_bitfield`` bit in the ``rte_eth_rxmode`` struct. +In such cases it is not required to set other bitfield offloads in the ``rxmode`` struct. + Poll Mode Driver API -------------------- @@ -298,24 +365,21 @@ The Ethernet device API exported by the Ethernet PMDs is described in the *DPDK Extended Statistics API ~~~~~~~~~~~~~~~~~~~~~~~ -The extended statistics API allows each individual PMD to expose a unique set -of statistics. Accessing these from application programs is done via two -functions: +The extended statistics API allows a PMD to expose all statistics that are +available to it, including statistics that are unique to the device. +Each statistic has three properties ``name``, ``id`` and ``value``: -* ``rte_eth_xstats_get``: Fills in an array of ``struct rte_eth_xstat`` - with extended statistics. -* ``rte_eth_xstats_get_names``: Fills in an array of - ``struct rte_eth_xstat_name`` with extended statistic name lookup - information. +* ``name``: A human readable string formatted by the scheme detailed below. +* ``id``: An integer that represents only that statistic. +* ``value``: A unsigned 64-bit integer that is the value of the statistic. -Each ``struct rte_eth_xstat`` contains an identifier and value pair, and -each ``struct rte_eth_xstat_name`` contains a string. Each identifier -within the ``struct rte_eth_xstat`` lookup array must have a corresponding -entry in the ``struct rte_eth_xstat_name`` lookup array. Within the latter -the index of the entry is the identifier the string is associated with. -These identifiers, as well as the number of extended statistic exposed, must -remain constant during runtime. Note that extended statistic identifiers are +Note that extended statistic identifiers are driver-specific, and hence might not be the same for different ports. +The API consists of various ``rte_eth_xstats_*()`` functions, and allows an +application to be flexible in how it retrieves statistics. + +Scheme for Human Readable Names +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ A naming scheme exists for the strings exposed to clients of the API. This is to allow scraping of the API for statistics of interest. The naming scheme uses @@ -356,3 +420,179 @@ Some additions in the metadata scheme are as follows: An example where queue numbers are used is as follows: ``tx_q7_bytes`` which indicates this statistic applies to queue number 7, and represents the number of transmitted bytes on that queue. + +API Design +^^^^^^^^^^ + +The xstats API uses the ``name``, ``id``, and ``value`` to allow performant +lookup of specific statistics. Performant lookup means two things; + +* No string comparisons with the ``name`` of the statistic in fast-path +* Allow requesting of only the statistics of interest + +The API ensures these requirements are met by mapping the ``name`` of the +statistic to a unique ``id``, which is used as a key for lookup in the fast-path. +The API allows applications to request an array of ``id`` values, so that the +PMD only performs the required calculations. Expected usage is that the +application scans the ``name`` of each statistic, and caches the ``id`` +if it has an interest in that statistic. On the fast-path, the integer can be used +to retrieve the actual ``value`` of the statistic that the ``id`` represents. + +API Functions +^^^^^^^^^^^^^ + +The API is built out of a small number of functions, which can be used to +retrieve the number of statistics and the names, IDs and values of those +statistics. + +* ``rte_eth_xstats_get_names_by_id()``: returns the names of the statistics. When given a + ``NULL`` parameter the function returns the number of statistics that are available. + +* ``rte_eth_xstats_get_id_by_name()``: Searches for the statistic ID that matches + ``xstat_name``. If found, the ``id`` integer is set. + +* ``rte_eth_xstats_get_by_id()``: Fills in an array of ``uint64_t`` values + with matching the provided ``ids`` array. If the ``ids`` array is NULL, it + returns all statistics that are available. + + +Application Usage +^^^^^^^^^^^^^^^^^ + +Imagine an application that wants to view the dropped packet count. If no +packets are dropped, the application does not read any other metrics for +performance reasons. If packets are dropped, the application has a particular +set of statistics that it requests. This "set" of statistics allows the app to +decide what next steps to perform. The following code-snippets show how the +xstats API can be used to achieve this goal. + +First step is to get all statistics names and list them: + +.. code-block:: c + + struct rte_eth_xstat_name *xstats_names; + uint64_t *values; + int len, i; + + /* Get number of stats */ + len = rte_eth_xstats_get_names_by_id(port_id, NULL, NULL, 0); + if (len < 0) { + printf("Cannot get xstats count\n"); + goto err; + } + + xstats_names = malloc(sizeof(struct rte_eth_xstat_name) * len); + if (xstats_names == NULL) { + printf("Cannot allocate memory for xstat names\n"); + goto err; + } + + /* Retrieve xstats names, passing NULL for IDs to return all statistics */ + if (len != rte_eth_xstats_get_names_by_id(port_id, xstats_names, NULL, len)) { + printf("Cannot get xstat names\n"); + goto err; + } + + values = malloc(sizeof(values) * len); + if (values == NULL) { + printf("Cannot allocate memory for xstats\n"); + goto err; + } + + /* Getting xstats values */ + if (len != rte_eth_xstats_get_by_id(port_id, NULL, values, len)) { + printf("Cannot get xstat values\n"); + goto err; + } + + /* Print all xstats names and values */ + for (i = 0; i < len; i++) { + printf("%s: %"PRIu64"\n", xstats_names[i].name, values[i]); + } + +The application has access to the names of all of the statistics that the PMD +exposes. The application can decide which statistics are of interest, cache the +ids of those statistics by looking up the name as follows: + +.. code-block:: c + + uint64_t id; + uint64_t value; + const char *xstat_name = "rx_errors"; + + if(!rte_eth_xstats_get_id_by_name(port_id, xstat_name, &id)) { + rte_eth_xstats_get_by_id(port_id, &id, &value, 1); + printf("%s: %"PRIu64"\n", xstat_name, value); + } + else { + printf("Cannot find xstats with a given name\n"); + goto err; + } + +The API provides flexibility to the application so that it can look up multiple +statistics using an array containing multiple ``id`` numbers. This reduces the +function call overhead of retrieving statistics, and makes lookup of multiple +statistics simpler for the application. + +.. code-block:: c + + #define APP_NUM_STATS 4 + /* application cached these ids previously; see above */ + uint64_t ids_array[APP_NUM_STATS] = {3,4,7,21}; + uint64_t value_array[APP_NUM_STATS]; + + /* Getting multiple xstats values from array of IDs */ + rte_eth_xstats_get_by_id(port_id, ids_array, value_array, APP_NUM_STATS); + + uint32_t i; + for(i = 0; i < APP_NUM_STATS; i++) { + printf("%d: %"PRIu64"\n", ids_array[i], value_array[i]); + } + + +This array lookup API for xstats allows the application create multiple +"groups" of statistics, and look up the values of those IDs using a single API +call. As an end result, the application is able to achieve its goal of +monitoring a single statistic ("rx_errors" in this case), and if that shows +packets being dropped, it can easily retrieve a "set" of statistics using the +IDs array parameter to ``rte_eth_xstats_get_by_id`` function. + +NIC Reset API +~~~~~~~~~~~~~ + +.. code-block:: c + + int rte_eth_dev_reset(uint16_t port_id); + +Sometimes a port has to be reset passively. For example when a PF is +reset, all its VFs should also be reset by the application to make them +consistent with the PF. A DPDK application also can call this function +to trigger a port reset. Normally, a DPDK application would invokes this +function when an RTE_ETH_EVENT_INTR_RESET event is detected. + +It is the duty of the PMD to trigger RTE_ETH_EVENT_INTR_RESET events and +the application should register a callback function to handle these +events. When a PMD needs to trigger a reset, it can trigger an +RTE_ETH_EVENT_INTR_RESET event. On receiving an RTE_ETH_EVENT_INTR_RESET +event, applications can handle it as follows: Stop working queues, stop +calling Rx and Tx functions, and then call rte_eth_dev_reset(). For +thread safety all these operations should be called from the same thread. + +For example when PF is reset, the PF sends a message to notify VFs of +this event and also trigger an interrupt to VFs. Then in the interrupt +service routine the VFs detects this notification message and calls +_rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RESET, NULL, +NULL). This means that a PF reset triggers an RTE_ETH_EVENT_INTR_RESET +event within VFs. The function _rte_eth_dev_callback_process() will +call the registered callback function. The callback function can trigger +the application to handle all operations the VF reset requires including +stopping Rx/Tx queues and calling rte_eth_dev_reset(). + +The rte_eth_dev_reset() itself is a generic function which only does +some hardware reset operations through calling dev_unint() and +dev_init(), and itself does not handle synchronization, which is handled +by application. + +The PMD itself should not call rte_eth_dev_reset(). The PMD can trigger +the application to handle reset event. It is duty of application to +handle all synchronization before it calls rte_eth_dev_reset(). diff --git a/dpdk/doc/guides/prog_guide/port_hotplug_framework.rst b/dpdk/doc/guides/prog_guide/port_hotplug_framework.rst index fe6d72a6..fb0efc18 100644 --- a/dpdk/doc/guides/prog_guide/port_hotplug_framework.rst +++ b/dpdk/doc/guides/prog_guide/port_hotplug_framework.rst @@ -80,7 +80,7 @@ Port Hotplug API overview returns the attached port number. Before calling the API, the device should be recognized by an userspace driver I/O framework. The API receives a pci address like "0000:01:00.0" or a virtual device name - like "eth_pcap0,iface=eth0". In the case of virtual device name, the + like "net_pcap0,iface=eth0". In the case of virtual device name, the format is the same as the general "--vdev" option of DPDK. * Detaching a port @@ -101,10 +101,6 @@ Limitations * The framework can only be enabled with Linux. BSD is not supported. -* To detach a port, the port should be backed by a device that igb_uio - manages. VFIO is not supported. - * Not all PMDs support detaching feature. - To know whether a PMD can support detaching, search for the - "RTE_PCI_DRV_DETACHABLE" flag in PMD implementation. If the flag is - defined in the PMD, detaching is supported. + The underlying bus must support hot-unplug. If not supported, + the function ``rte_eth_dev_detach()`` will return negative ENOTSUP. diff --git a/dpdk/doc/guides/prog_guide/power_man.rst b/dpdk/doc/guides/prog_guide/power_man.rst index 114d0b1c..c5d62a39 100644 --- a/dpdk/doc/guides/prog_guide/power_man.rst +++ b/dpdk/doc/guides/prog_guide/power_man.rst @@ -89,6 +89,14 @@ Core state can be altered by speculative sleeps whenever the specified lcore has In the DPDK, if no packet is received after polling, speculative sleeps can be triggered according the strategies defined by the user space application. +Per-core Turbo Boost +-------------------- + +Individual cores can be allowed to enter a Turbo Boost state on a per-core +basis. This is achieved by enabling Turbo Boost Technology in the BIOS, then +looping through the relevant cores and enabling/disabling Turbo Boost on each +core. + API Overview of the Power Library --------------------------------- @@ -108,6 +116,10 @@ The main methods exported by power library are for CPU frequency scaling and inc * **Freq set**: Prompt the kernel to set the frequency for the specific lcore. +* **Enable turbo**: Prompt the kernel to enable Turbo Boost for the specific lcore. + +* **Disable turbo**: Prompt the kernel to disable Turbo Boost for the specific lcore. + User Cases ---------- diff --git a/dpdk/doc/guides/prog_guide/profile_app.rst b/dpdk/doc/guides/prog_guide/profile_app.rst index 32261875..ca1c91f6 100644 --- a/dpdk/doc/guides/prog_guide/profile_app.rst +++ b/dpdk/doc/guides/prog_guide/profile_app.rst @@ -31,8 +31,16 @@ Profile Your Application ======================== +The following sections describe methods of profiling DPDK applications on +different architectures. + + +Profiling on x86 +---------------- + Intel processors provide performance counters to monitor events. -Some tools provided by Intel can be used to profile and benchmark an application. +Some tools provided by Intel, such as Intel® VTune™ Amplifier, can be used +to profile and benchmark an application. See the *VTune Performance Analyzer Essentials* publication from Intel Press for more information. For a DPDK application, this can be done in a Linux* application environment only. @@ -50,3 +58,92 @@ The main situations that should be monitored through event counters are: Refer to the `Intel Performance Analysis Guide `_ for details about application profiling. + + +Empty cycles tracing +~~~~~~~~~~~~~~~~~~~~ + +Iterations that yielded no RX packets (empty cycles, wasted iterations) can +be analyzed using VTune Amplifier. This profiling employs the +`Instrumentation and Tracing Technology (ITT) API +`_ +feature of VTune Amplifier and requires only reconfiguring the DPDK library, +no changes in a DPDK application are needed. + +To trace wasted iterations on RX queues, first reconfigure DPDK with +``CONFIG_RTE_ETHDEV_RXTX_CALLBACKS`` and +``CONFIG_RTE_ETHDEV_PROFILE_ITT_WASTED_RX_ITERATIONS`` enabled. + +Then rebuild DPDK, specifying paths to the ITT header and library, which can +be found in any VTune Amplifier distribution in the *include* and *lib* +directories respectively: + +.. code-block:: console + + make EXTRA_CFLAGS=-I \ + EXTRA_LDLIBS="-L -littnotify" + +Finally, to see wasted iterations in your performance analysis results, +select the *"Analyze user tasks, events, and counters"* checkbox in the +*"Analysis Type"* tab when configuring analysis via VTune Amplifier GUI. +Alternatively, when running VTune Amplifier via command line, specify +``-knob enable-user-tasks=true`` option. + +Collected regions of wasted iterations will be marked on VTune Amplifier's +timeline as ITT tasks. These ITT tasks have predefined names, containing +Ethernet device and RX queue identifiers. + + +Profiling on ARM64 +------------------ + +Using Linux perf +~~~~~~~~~~~~~~~~ + +The ARM64 architecture provide performance counters to monitor events. The +Linux ``perf`` tool can be used to profile and benchmark an application. In +addition to the standard events, ``perf`` can be used to profile arm64 +specific PMU (Performance Monitor Unit) events through raw events (``-e`` +``-rXX``). + +For more derails refer to the +`ARM64 specific PMU events enumeration `_. + + +High-resolution cycle counter +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The default ``cntvct_el0`` based ``rte_rdtsc()`` provides a portable means to +get a wall clock counter in user space. Typically it runs at <= 100MHz. + +The alternative method to enable ``rte_rdtsc()`` for a high resolution wall +clock counter is through the armv8 PMU subsystem. The PMU cycle counter runs +at CPU frequency. However, access to the PMU cycle counter from user space is +not enabled by default in the arm64 linux kernel. It is possible to enable +cycle counter for user space access by configuring the PMU from the privileged +mode (kernel space). + +By default the ``rte_rdtsc()`` implementation uses a portable ``cntvct_el0`` +scheme. Application can choose the PMU based implementation with +``CONFIG_RTE_ARM_EAL_RDTSC_USE_PMU``. + +The example below shows the steps to configure the PMU based cycle counter on +an armv8 machine. + +.. code-block:: console + + git clone https://github.com/jerinjacobk/armv8_pmu_cycle_counter_el0 + cd armv8_pmu_cycle_counter_el0 + make + sudo insmod pmu_el0_cycle_counter.ko + cd $DPDK_DIR + make config T=arm64-armv8a-linuxapp-gcc + echo "CONFIG_RTE_ARM_EAL_RDTSC_USE_PMU=y" >> build/.config + make + +.. warning:: + + The PMU based scheme is useful for high accuracy performance profiling with + ``rte_rdtsc()``. However, this method can not be used in conjunction with + Linux userspace profiling tools like ``perf`` as this scheme alters the PMU + registers state. diff --git a/dpdk/doc/guides/prog_guide/ring_lib.rst b/dpdk/doc/guides/prog_guide/ring_lib.rst index 9f697538..b31ab7a4 100644 --- a/dpdk/doc/guides/prog_guide/ring_lib.rst +++ b/dpdk/doc/guides/prog_guide/ring_lib.rst @@ -102,21 +102,6 @@ Name A ring is identified by a unique name. It is not possible to create two rings with the same name (rte_ring_create() returns NULL if this is attempted). -Water Marking -~~~~~~~~~~~~~ - -The ring can have a high water mark (threshold). -Once an enqueue operation reaches the high water mark, the producer is notified, if the water mark is configured. - -This mechanism can be used, for example, to exert a back pressure on I/O to inform the LAN to PAUSE. - -Debug -~~~~~ - -When debug is enabled (CONFIG_RTE_LIBRTE_RING_DEBUG is set), -the library stores some per-ring statistic counters about the number of enqueues/dequeues. -These statistics are per-core to avoid concurrent accesses or atomic operations. - Use Cases --------- diff --git a/dpdk/doc/guides/prog_guide/rte_flow.rst b/dpdk/doc/guides/prog_guide/rte_flow.rst new file mode 100644 index 00000000..d158be5e --- /dev/null +++ b/dpdk/doc/guides/prog_guide/rte_flow.rst @@ -0,0 +1,2354 @@ +.. BSD LICENSE + Copyright 2016 6WIND S.A. + Copyright 2016 Mellanox. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of 6WIND S.A. nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +.. _Generic_flow_API: + +Generic flow API (rte_flow) +=========================== + +Overview +-------- + +This API provides a generic means to configure hardware to match specific +ingress or egress traffic, alter its fate and query related counters +according to any number of user-defined rules. + +It is named *rte_flow* after the prefix used for all its symbols, and is +defined in ``rte_flow.h``. + +- Matching can be performed on packet data (protocol headers, payload) and + properties (e.g. associated physical port, virtual device function ID). + +- Possible operations include dropping traffic, diverting it to specific + queues, to virtual/physical device functions or ports, performing tunnel + offloads, adding marks and so on. + +It is slightly higher-level than the legacy filtering framework which it +encompasses and supersedes (including all functions and filter types) in +order to expose a single interface with an unambiguous behavior that is +common to all poll-mode drivers (PMDs). + +Several methods to migrate existing applications are described in `API +migration`_. + +Flow rule +--------- + +Description +~~~~~~~~~~~ + +A flow rule is the combination of attributes with a matching pattern and a +list of actions. Flow rules form the basis of this API. + +Flow rules can have several distinct actions (such as counting, +encapsulating, decapsulating before redirecting packets to a particular +queue, etc.), instead of relying on several rules to achieve this and having +applications deal with hardware implementation details regarding their +order. + +Support for different priority levels on a rule basis is provided, for +example in order to force a more specific rule to come before a more generic +one for packets matched by both. However hardware support for more than a +single priority level cannot be guaranteed. When supported, the number of +available priority levels is usually low, which is why they can also be +implemented in software by PMDs (e.g. missing priority levels may be +emulated by reordering rules). + +In order to remain as hardware-agnostic as possible, by default all rules +are considered to have the same priority, which means that the order between +overlapping rules (when a packet is matched by several filters) is +undefined. + +PMDs may refuse to create overlapping rules at a given priority level when +they can be detected (e.g. if a pattern matches an existing filter). + +Thus predictable results for a given priority level can only be achieved +with non-overlapping rules, using perfect matching on all protocol layers. + +Flow rules can also be grouped, the flow rule priority is specific to the +group they belong to. All flow rules in a given group are thus processed +either before or after another group. + +Support for multiple actions per rule may be implemented internally on top +of non-default hardware priorities, as a result both features may not be +simultaneously available to applications. + +Considering that allowed pattern/actions combinations cannot be known in +advance and would result in an impractically large number of capabilities to +expose, a method is provided to validate a given rule from the current +device configuration state. + +This enables applications to check if the rule types they need is supported +at initialization time, before starting their data path. This method can be +used anytime, its only requirement being that the resources needed by a rule +should exist (e.g. a target RX queue should be configured first). + +Each defined rule is associated with an opaque handle managed by the PMD, +applications are responsible for keeping it. These can be used for queries +and rules management, such as retrieving counters or other data and +destroying them. + +To avoid resource leaks on the PMD side, handles must be explicitly +destroyed by the application before releasing associated resources such as +queues and ports. + +The following sections cover: + +- **Attributes** (represented by ``struct rte_flow_attr``): properties of a + flow rule such as its direction (ingress or egress) and priority. + +- **Pattern item** (represented by ``struct rte_flow_item``): part of a + matching pattern that either matches specific packet data or traffic + properties. It can also describe properties of the pattern itself, such as + inverted matching. + +- **Matching pattern**: traffic properties to look for, a combination of any + number of items. + +- **Actions** (represented by ``struct rte_flow_action``): operations to + perform whenever a packet is matched by a pattern. + +Attributes +~~~~~~~~~~ + +Attribute: Group +^^^^^^^^^^^^^^^^ + +Flow rules can be grouped by assigning them a common group number. Lower +values have higher priority. Group 0 has the highest priority. + +Although optional, applications are encouraged to group similar rules as +much as possible to fully take advantage of hardware capabilities +(e.g. optimized matching) and work around limitations (e.g. a single pattern +type possibly allowed in a given group). + +Note that support for more than a single group is not guaranteed. + +Attribute: Priority +^^^^^^^^^^^^^^^^^^^ + +A priority level can be assigned to a flow rule. Like groups, lower values +denote higher priority, with 0 as the maximum. + +A rule with priority 0 in group 8 is always matched after a rule with +priority 8 in group 0. + +Group and priority levels are arbitrary and up to the application, they do +not need to be contiguous nor start from 0, however the maximum number +varies between devices and may be affected by existing flow rules. + +If a packet is matched by several rules of a given group for a given +priority level, the outcome is undefined. It can take any path, may be +duplicated or even cause unrecoverable errors. + +Note that support for more than a single priority level is not guaranteed. + +Attribute: Traffic direction +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Flow rules can apply to inbound and/or outbound traffic (ingress/egress). + +Several pattern items and actions are valid and can be used in both +directions. At least one direction must be specified. + +Specifying both directions at once for a given rule is not recommended but +may be valid in a few cases (e.g. shared counters). + +Pattern item +~~~~~~~~~~~~ + +Pattern items fall in two categories: + +- Matching protocol headers and packet data (ANY, RAW, ETH, VLAN, IPV4, + IPV6, ICMP, UDP, TCP, SCTP, VXLAN, MPLS, GRE, ESP and so on), usually + associated with a specification structure. + +- Matching meta-data or affecting pattern processing (END, VOID, INVERT, PF, + VF, PORT and so on), often without a specification structure. + +Item specification structures are used to match specific values among +protocol fields (or item properties). Documentation describes for each item +whether they are associated with one and their type name if so. + +Up to three structures of the same type can be set for a given item: + +- ``spec``: values to match (e.g. a given IPv4 address). + +- ``last``: upper bound for an inclusive range with corresponding fields in + ``spec``. + +- ``mask``: bit-mask applied to both ``spec`` and ``last`` whose purpose is + to distinguish the values to take into account and/or partially mask them + out (e.g. in order to match an IPv4 address prefix). + +Usage restrictions and expected behavior: + +- Setting either ``mask`` or ``last`` without ``spec`` is an error. + +- Field values in ``last`` which are either 0 or equal to the corresponding + values in ``spec`` are ignored; they do not generate a range. Nonzero + values lower than those in ``spec`` are not supported. + +- Setting ``spec`` and optionally ``last`` without ``mask`` causes the PMD + to use the default mask defined for that item (defined as + ``rte_flow_item_{name}_mask`` constants). + +- Not setting any of them (assuming item type allows it) is equivalent to + providing an empty (zeroed) ``mask`` for broad (nonspecific) matching. + +- ``mask`` is a simple bit-mask applied before interpreting the contents of + ``spec`` and ``last``, which may yield unexpected results if not used + carefully. For example, if for an IPv4 address field, ``spec`` provides + *10.1.2.3*, ``last`` provides *10.3.4.5* and ``mask`` provides + *255.255.0.0*, the effective range becomes *10.1.0.0* to *10.3.255.255*. + +Example of an item specification matching an Ethernet header: + +.. _table_rte_flow_pattern_item_example: + +.. table:: Ethernet item + + +----------+----------+--------------------+ + | Field | Subfield | Value | + +==========+==========+====================+ + | ``spec`` | ``src`` | ``00:01:02:03:04`` | + | +----------+--------------------+ + | | ``dst`` | ``00:2a:66:00:01`` | + | +----------+--------------------+ + | | ``type`` | ``0x22aa`` | + +----------+----------+--------------------+ + | ``last`` | unspecified | + +----------+----------+--------------------+ + | ``mask`` | ``src`` | ``00:ff:ff:ff:00`` | + | +----------+--------------------+ + | | ``dst`` | ``00:00:00:00:ff`` | + | +----------+--------------------+ + | | ``type`` | ``0x0000`` | + +----------+----------+--------------------+ + +Non-masked bits stand for any value (shown as ``?`` below), Ethernet headers +with the following properties are thus matched: + +- ``src``: ``??:01:02:03:??`` +- ``dst``: ``??:??:??:??:01`` +- ``type``: ``0x????`` + +Matching pattern +~~~~~~~~~~~~~~~~ + +A pattern is formed by stacking items starting from the lowest protocol +layer to match. This stacking restriction does not apply to meta items which +can be placed anywhere in the stack without affecting the meaning of the +resulting pattern. + +Patterns are terminated by END items. + +Examples: + +.. _table_rte_flow_tcpv4_as_l4: + +.. table:: TCPv4 as L4 + + +-------+----------+ + | Index | Item | + +=======+==========+ + | 0 | Ethernet | + +-------+----------+ + | 1 | IPv4 | + +-------+----------+ + | 2 | TCP | + +-------+----------+ + | 3 | END | + +-------+----------+ + +| + +.. _table_rte_flow_tcpv6_in_vxlan: + +.. table:: TCPv6 in VXLAN + + +-------+------------+ + | Index | Item | + +=======+============+ + | 0 | Ethernet | + +-------+------------+ + | 1 | IPv4 | + +-------+------------+ + | 2 | UDP | + +-------+------------+ + | 3 | VXLAN | + +-------+------------+ + | 4 | Ethernet | + +-------+------------+ + | 5 | IPv6 | + +-------+------------+ + | 6 | TCP | + +-------+------------+ + | 7 | END | + +-------+------------+ + +| + +.. _table_rte_flow_tcpv4_as_l4_meta: + +.. table:: TCPv4 as L4 with meta items + + +-------+----------+ + | Index | Item | + +=======+==========+ + | 0 | VOID | + +-------+----------+ + | 1 | Ethernet | + +-------+----------+ + | 2 | VOID | + +-------+----------+ + | 3 | IPv4 | + +-------+----------+ + | 4 | TCP | + +-------+----------+ + | 5 | VOID | + +-------+----------+ + | 6 | VOID | + +-------+----------+ + | 7 | END | + +-------+----------+ + +The above example shows how meta items do not affect packet data matching +items, as long as those remain stacked properly. The resulting matching +pattern is identical to "TCPv4 as L4". + +.. _table_rte_flow_udpv6_anywhere: + +.. table:: UDPv6 anywhere + + +-------+------+ + | Index | Item | + +=======+======+ + | 0 | IPv6 | + +-------+------+ + | 1 | UDP | + +-------+------+ + | 2 | END | + +-------+------+ + +If supported by the PMD, omitting one or several protocol layers at the +bottom of the stack as in the above example (missing an Ethernet +specification) enables looking up anywhere in packets. + +It is unspecified whether the payload of supported encapsulations +(e.g. VXLAN payload) is matched by such a pattern, which may apply to inner, +outer or both packets. + +.. _table_rte_flow_invalid_l3: + +.. table:: Invalid, missing L3 + + +-------+----------+ + | Index | Item | + +=======+==========+ + | 0 | Ethernet | + +-------+----------+ + | 1 | UDP | + +-------+----------+ + | 2 | END | + +-------+----------+ + +The above pattern is invalid due to a missing L3 specification between L2 +(Ethernet) and L4 (UDP). Doing so is only allowed at the bottom and at the +top of the stack. + +Meta item types +~~~~~~~~~~~~~~~ + +They match meta-data or affect pattern processing instead of matching packet +data directly, most of them do not need a specification structure. This +particularity allows them to be specified anywhere in the stack without +causing any side effect. + +Item: ``END`` +^^^^^^^^^^^^^ + +End marker for item lists. Prevents further processing of items, thereby +ending the pattern. + +- Its numeric value is 0 for convenience. +- PMD support is mandatory. +- ``spec``, ``last`` and ``mask`` are ignored. + +.. _table_rte_flow_item_end: + +.. table:: END + + +----------+---------+ + | Field | Value | + +==========+=========+ + | ``spec`` | ignored | + +----------+---------+ + | ``last`` | ignored | + +----------+---------+ + | ``mask`` | ignored | + +----------+---------+ + +Item: ``VOID`` +^^^^^^^^^^^^^^ + +Used as a placeholder for convenience. It is ignored and simply discarded by +PMDs. + +- PMD support is mandatory. +- ``spec``, ``last`` and ``mask`` are ignored. + +.. _table_rte_flow_item_void: + +.. table:: VOID + + +----------+---------+ + | Field | Value | + +==========+=========+ + | ``spec`` | ignored | + +----------+---------+ + | ``last`` | ignored | + +----------+---------+ + | ``mask`` | ignored | + +----------+---------+ + +One usage example for this type is generating rules that share a common +prefix quickly without reallocating memory, only by updating item types: + +.. _table_rte_flow_item_void_example: + +.. table:: TCP, UDP or ICMP as L4 + + +-------+--------------------+ + | Index | Item | + +=======+====================+ + | 0 | Ethernet | + +-------+--------------------+ + | 1 | IPv4 | + +-------+------+------+------+ + | 2 | UDP | VOID | VOID | + +-------+------+------+------+ + | 3 | VOID | TCP | VOID | + +-------+------+------+------+ + | 4 | VOID | VOID | ICMP | + +-------+------+------+------+ + | 5 | END | + +-------+--------------------+ + +Item: ``INVERT`` +^^^^^^^^^^^^^^^^ + +Inverted matching, i.e. process packets that do not match the pattern. + +- ``spec``, ``last`` and ``mask`` are ignored. + +.. _table_rte_flow_item_invert: + +.. table:: INVERT + + +----------+---------+ + | Field | Value | + +==========+=========+ + | ``spec`` | ignored | + +----------+---------+ + | ``last`` | ignored | + +----------+---------+ + | ``mask`` | ignored | + +----------+---------+ + +Usage example, matching non-TCPv4 packets only: + +.. _table_rte_flow_item_invert_example: + +.. table:: Anything but TCPv4 + + +-------+----------+ + | Index | Item | + +=======+==========+ + | 0 | INVERT | + +-------+----------+ + | 1 | Ethernet | + +-------+----------+ + | 2 | IPv4 | + +-------+----------+ + | 3 | TCP | + +-------+----------+ + | 4 | END | + +-------+----------+ + +Item: ``PF`` +^^^^^^^^^^^^ + +Matches packets addressed to the physical function of the device. + +If the underlying device function differs from the one that would normally +receive the matched traffic, specifying this item prevents it from reaching +that device unless the flow rule contains a `Action: PF`_. Packets are not +duplicated between device instances by default. + +- Likely to return an error or never match any traffic if applied to a VF + device. +- Can be combined with any number of `Item: VF`_ to match both PF and VF + traffic. +- ``spec``, ``last`` and ``mask`` must not be set. + +.. _table_rte_flow_item_pf: + +.. table:: PF + + +----------+-------+ + | Field | Value | + +==========+=======+ + | ``spec`` | unset | + +----------+-------+ + | ``last`` | unset | + +----------+-------+ + | ``mask`` | unset | + +----------+-------+ + +Item: ``VF`` +^^^^^^^^^^^^ + +Matches packets addressed to a virtual function ID of the device. + +If the underlying device function differs from the one that would normally +receive the matched traffic, specifying this item prevents it from reaching +that device unless the flow rule contains a `Action: VF`_. Packets are not +duplicated between device instances by default. + +- Likely to return an error or never match any traffic if this causes a VF + device to match traffic addressed to a different VF. +- Can be specified multiple times to match traffic addressed to several VF + IDs. +- Can be combined with a PF item to match both PF and VF traffic. +- Default ``mask`` matches any VF ID. + +.. _table_rte_flow_item_vf: + +.. table:: VF + + +----------+----------+---------------------------+ + | Field | Subfield | Value | + +==========+==========+===========================+ + | ``spec`` | ``id`` | destination VF ID | + +----------+----------+---------------------------+ + | ``last`` | ``id`` | upper range value | + +----------+----------+---------------------------+ + | ``mask`` | ``id`` | zeroed to match any VF ID | + +----------+----------+---------------------------+ + +Item: ``PORT`` +^^^^^^^^^^^^^^ + +Matches packets coming from the specified physical port of the underlying +device. + +The first PORT item overrides the physical port normally associated with the +specified DPDK input port (port_id). This item can be provided several times +to match additional physical ports. + +Note that physical ports are not necessarily tied to DPDK input ports +(port_id) when those are not under DPDK control. Possible values are +specific to each device, they are not necessarily indexed from zero and may +not be contiguous. + +As a device property, the list of allowed values as well as the value +associated with a port_id should be retrieved by other means. + +- Default ``mask`` matches any port index. + +.. _table_rte_flow_item_port: + +.. table:: PORT + + +----------+-----------+--------------------------------+ + | Field | Subfield | Value | + +==========+===========+================================+ + | ``spec`` | ``index`` | physical port index | + +----------+-----------+--------------------------------+ + | ``last`` | ``index`` | upper range value | + +----------+-----------+--------------------------------+ + | ``mask`` | ``index`` | zeroed to match any port index | + +----------+-----------+--------------------------------+ + +Data matching item types +~~~~~~~~~~~~~~~~~~~~~~~~ + +Most of these are basically protocol header definitions with associated +bit-masks. They must be specified (stacked) from lowest to highest protocol +layer to form a matching pattern. + +The following list is not exhaustive, new protocols will be added in the +future. + +Item: ``ANY`` +^^^^^^^^^^^^^ + +Matches any protocol in place of the current layer, a single ANY may also +stand for several protocol layers. + +This is usually specified as the first pattern item when looking for a +protocol anywhere in a packet. + +- Default ``mask`` stands for any number of layers. + +.. _table_rte_flow_item_any: + +.. table:: ANY + + +----------+----------+--------------------------------------+ + | Field | Subfield | Value | + +==========+==========+======================================+ + | ``spec`` | ``num`` | number of layers covered | + +----------+----------+--------------------------------------+ + | ``last`` | ``num`` | upper range value | + +----------+----------+--------------------------------------+ + | ``mask`` | ``num`` | zeroed to cover any number of layers | + +----------+----------+--------------------------------------+ + +Example for VXLAN TCP payload matching regardless of outer L3 (IPv4 or IPv6) +and L4 (UDP) both matched by the first ANY specification, and inner L3 (IPv4 +or IPv6) matched by the second ANY specification: + +.. _table_rte_flow_item_any_example: + +.. table:: TCP in VXLAN with wildcards + + +-------+------+----------+----------+-------+ + | Index | Item | Field | Subfield | Value | + +=======+======+==========+==========+=======+ + | 0 | Ethernet | + +-------+------+----------+----------+-------+ + | 1 | ANY | ``spec`` | ``num`` | 2 | + +-------+------+----------+----------+-------+ + | 2 | VXLAN | + +-------+------------------------------------+ + | 3 | Ethernet | + +-------+------+----------+----------+-------+ + | 4 | ANY | ``spec`` | ``num`` | 1 | + +-------+------+----------+----------+-------+ + | 5 | TCP | + +-------+------------------------------------+ + | 6 | END | + +-------+------------------------------------+ + +Item: ``RAW`` +^^^^^^^^^^^^^ + +Matches a byte string of a given length at a given offset. + +Offset is either absolute (using the start of the packet) or relative to the +end of the previous matched item in the stack, in which case negative values +are allowed. + +If search is enabled, offset is used as the starting point. The search area +can be delimited by setting limit to a nonzero value, which is the maximum +number of bytes after offset where the pattern may start. + +Matching a zero-length pattern is allowed, doing so resets the relative +offset for subsequent items. + +- This type does not support ranges (``last`` field). +- Default ``mask`` matches all fields exactly. + +.. _table_rte_flow_item_raw: + +.. table:: RAW + + +----------+--------------+-------------------------------------------------+ + | Field | Subfield | Value | + +==========+==============+=================================================+ + | ``spec`` | ``relative`` | look for pattern after the previous item | + | +--------------+-------------------------------------------------+ + | | ``search`` | search pattern from offset (see also ``limit``) | + | +--------------+-------------------------------------------------+ + | | ``reserved`` | reserved, must be set to zero | + | +--------------+-------------------------------------------------+ + | | ``offset`` | absolute or relative offset for ``pattern`` | + | +--------------+-------------------------------------------------+ + | | ``limit`` | search area limit for start of ``pattern`` | + | +--------------+-------------------------------------------------+ + | | ``length`` | ``pattern`` length | + | +--------------+-------------------------------------------------+ + | | ``pattern`` | byte string to look for | + +----------+--------------+-------------------------------------------------+ + | ``last`` | if specified, either all 0 or with the same values as ``spec`` | + +----------+----------------------------------------------------------------+ + | ``mask`` | bit-mask applied to ``spec`` values with usual behavior | + +----------+----------------------------------------------------------------+ + +Example pattern looking for several strings at various offsets of a UDP +payload, using combined RAW items: + +.. _table_rte_flow_item_raw_example: + +.. table:: UDP payload matching + + +-------+------+----------+--------------+-------+ + | Index | Item | Field | Subfield | Value | + +=======+======+==========+==============+=======+ + | 0 | Ethernet | + +-------+----------------------------------------+ + | 1 | IPv4 | + +-------+----------------------------------------+ + | 2 | UDP | + +-------+------+----------+--------------+-------+ + | 3 | RAW | ``spec`` | ``relative`` | 1 | + | | | +--------------+-------+ + | | | | ``search`` | 1 | + | | | +--------------+-------+ + | | | | ``offset`` | 10 | + | | | +--------------+-------+ + | | | | ``limit`` | 0 | + | | | +--------------+-------+ + | | | | ``length`` | 3 | + | | | +--------------+-------+ + | | | | ``pattern`` | "foo" | + +-------+------+----------+--------------+-------+ + | 4 | RAW | ``spec`` | ``relative`` | 1 | + | | | +--------------+-------+ + | | | | ``search`` | 0 | + | | | +--------------+-------+ + | | | | ``offset`` | 20 | + | | | +--------------+-------+ + | | | | ``limit`` | 0 | + | | | +--------------+-------+ + | | | | ``length`` | 3 | + | | | +--------------+-------+ + | | | | ``pattern`` | "bar" | + +-------+------+----------+--------------+-------+ + | 5 | RAW | ``spec`` | ``relative`` | 1 | + | | | +--------------+-------+ + | | | | ``search`` | 0 | + | | | +--------------+-------+ + | | | | ``offset`` | -29 | + | | | +--------------+-------+ + | | | | ``limit`` | 0 | + | | | +--------------+-------+ + | | | | ``length`` | 3 | + | | | +--------------+-------+ + | | | | ``pattern`` | "baz" | + +-------+------+----------+--------------+-------+ + | 6 | END | + +-------+----------------------------------------+ + +This translates to: + +- Locate "foo" at least 10 bytes deep inside UDP payload. +- Locate "bar" after "foo" plus 20 bytes. +- Locate "baz" after "bar" minus 29 bytes. + +Such a packet may be represented as follows (not to scale):: + + 0 >= 10 B == 20 B + | |<--------->| |<--------->| + | | | | | + |-----|------|-----|-----|-----|-----|-----------|-----|------| + | ETH | IPv4 | UDP | ... | baz | foo | ......... | bar | .... | + |-----|------|-----|-----|-----|-----|-----------|-----|------| + | | + |<--------------------------->| + == 29 B + +Note that matching subsequent pattern items would resume after "baz", not +"bar" since matching is always performed after the previous item of the +stack. + +Item: ``ETH`` +^^^^^^^^^^^^^ + +Matches an Ethernet header. + +- ``dst``: destination MAC. +- ``src``: source MAC. +- ``type``: EtherType. +- Default ``mask`` matches destination and source addresses only. + +Item: ``VLAN`` +^^^^^^^^^^^^^^ + +Matches an 802.1Q/ad VLAN tag. + +- ``tpid``: tag protocol identifier. +- ``tci``: tag control information. +- Default ``mask`` matches TCI only. + +Item: ``IPV4`` +^^^^^^^^^^^^^^ + +Matches an IPv4 header. + +Note: IPv4 options are handled by dedicated pattern items. + +- ``hdr``: IPv4 header definition (``rte_ip.h``). +- Default ``mask`` matches source and destination addresses only. + +Item: ``IPV6`` +^^^^^^^^^^^^^^ + +Matches an IPv6 header. + +Note: IPv6 options are handled by dedicated pattern items. + +- ``hdr``: IPv6 header definition (``rte_ip.h``). +- Default ``mask`` matches source and destination addresses only. + +Item: ``ICMP`` +^^^^^^^^^^^^^^ + +Matches an ICMP header. + +- ``hdr``: ICMP header definition (``rte_icmp.h``). +- Default ``mask`` matches ICMP type and code only. + +Item: ``UDP`` +^^^^^^^^^^^^^ + +Matches a UDP header. + +- ``hdr``: UDP header definition (``rte_udp.h``). +- Default ``mask`` matches source and destination ports only. + +Item: ``TCP`` +^^^^^^^^^^^^^ + +Matches a TCP header. + +- ``hdr``: TCP header definition (``rte_tcp.h``). +- Default ``mask`` matches source and destination ports only. + +Item: ``SCTP`` +^^^^^^^^^^^^^^ + +Matches a SCTP header. + +- ``hdr``: SCTP header definition (``rte_sctp.h``). +- Default ``mask`` matches source and destination ports only. + +Item: ``VXLAN`` +^^^^^^^^^^^^^^^ + +Matches a VXLAN header (RFC 7348). + +- ``flags``: normally 0x08 (I flag). +- ``rsvd0``: reserved, normally 0x000000. +- ``vni``: VXLAN network identifier. +- ``rsvd1``: reserved, normally 0x00. +- Default ``mask`` matches VNI only. + +Item: ``E_TAG`` +^^^^^^^^^^^^^^^ + +Matches an IEEE 802.1BR E-Tag header. + +- ``tpid``: tag protocol identifier (0x893F) +- ``epcp_edei_in_ecid_b``: E-Tag control information (E-TCI), E-PCP (3b), + E-DEI (1b), ingress E-CID base (12b). +- ``rsvd_grp_ecid_b``: reserved (2b), GRP (2b), E-CID base (12b). +- ``in_ecid_e``: ingress E-CID ext. +- ``ecid_e``: E-CID ext. +- Default ``mask`` simultaneously matches GRP and E-CID base. + +Item: ``NVGRE`` +^^^^^^^^^^^^^^^ + +Matches a NVGRE header (RFC 7637). + +- ``c_k_s_rsvd0_ver``: checksum (1b), undefined (1b), key bit (1b), + sequence number (1b), reserved 0 (9b), version (3b). This field must have + value 0x2000 according to RFC 7637. +- ``protocol``: protocol type (0x6558). +- ``tni``: virtual subnet ID. +- ``flow_id``: flow ID. +- Default ``mask`` matches TNI only. + +Item: ``MPLS`` +^^^^^^^^^^^^^^ + +Matches a MPLS header. + +- ``label_tc_s_ttl``: label, TC, Bottom of Stack and TTL. +- Default ``mask`` matches label only. + +Item: ``GRE`` +^^^^^^^^^^^^^ + +Matches a GRE header. + +- ``c_rsvd0_ver``: checksum, reserved 0 and version. +- ``protocol``: protocol type. +- Default ``mask`` matches protocol only. + +Item: ``FUZZY`` +^^^^^^^^^^^^^^^ + +Fuzzy pattern match, expect faster than default. + +This is for device that support fuzzy match option. Usually a fuzzy match is +fast but the cost is accuracy. i.e. Signature Match only match pattern's hash +value, but it is possible two different patterns have the same hash value. + +Matching accuracy level can be configured by threshold. Driver can divide the +range of threshold and map to different accuracy levels that device support. + +Threshold 0 means perfect match (no fuzziness), while threshold 0xffffffff +means fuzziest match. + +.. _table_rte_flow_item_fuzzy: + +.. table:: FUZZY + + +----------+---------------+--------------------------------------------------+ + | Field | Subfield | Value | + +==========+===============+==================================================+ + | ``spec`` | ``threshold`` | 0 as perfect match, 0xffffffff as fuzziest match | + +----------+---------------+--------------------------------------------------+ + | ``last`` | ``threshold`` | upper range value | + +----------+---------------+--------------------------------------------------+ + | ``mask`` | ``threshold`` | bit-mask apply to "spec" and "last" | + +----------+---------------+--------------------------------------------------+ + +Usage example, fuzzy match a TCPv4 packets: + +.. _table_rte_flow_item_fuzzy_example: + +.. table:: Fuzzy matching + + +-------+----------+ + | Index | Item | + +=======+==========+ + | 0 | FUZZY | + +-------+----------+ + | 1 | Ethernet | + +-------+----------+ + | 2 | IPv4 | + +-------+----------+ + | 3 | TCP | + +-------+----------+ + | 4 | END | + +-------+----------+ + +Item: ``GTP``, ``GTPC``, ``GTPU`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Matches a GTPv1 header. + +Note: GTP, GTPC and GTPU use the same structure. GTPC and GTPU item +are defined for a user-friendly API when creating GTP-C and GTP-U +flow rules. + +- ``v_pt_rsv_flags``: version (3b), protocol type (1b), reserved (1b), + extension header flag (1b), sequence number flag (1b), N-PDU number + flag (1b). +- ``msg_type``: message type. +- ``msg_len``: message length. +- ``teid``: tunnel endpoint identifier. +- Default ``mask`` matches teid only. + +Item: ``ESP`` +^^^^^^^^^^^^^ + +Matches an ESP header. + +- ``hdr``: ESP header definition (``rte_esp.h``). +- Default ``mask`` matches SPI only. + +Actions +~~~~~~~ + +Each possible action is represented by a type. Some have associated +configuration structures. Several actions combined in a list can be affected +to a flow rule. That list is not ordered. + +They fall in three categories: + +- Terminating actions (such as QUEUE, DROP, RSS, PF, VF) that prevent + processing matched packets by subsequent flow rules, unless overridden + with PASSTHRU. + +- Non-terminating actions (PASSTHRU, DUP) that leave matched packets up for + additional processing by subsequent flow rules. + +- Other non-terminating meta actions that do not affect the fate of packets + (END, VOID, MARK, FLAG, COUNT, SECURITY). + +When several actions are combined in a flow rule, they should all have +different types (e.g. dropping a packet twice is not possible). + +Only the last action of a given type is taken into account. PMDs still +perform error checking on the entire list. + +Like matching patterns, action lists are terminated by END items. + +*Note that PASSTHRU is the only action able to override a terminating rule.* + +Example of action that redirects packets to queue index 10: + +.. _table_rte_flow_action_example: + +.. table:: Queue action + + +-----------+-------+ + | Field | Value | + +===========+=======+ + | ``index`` | 10 | + +-----------+-------+ + +Action lists examples, their order is not significant, applications must +consider all actions to be performed simultaneously: + +.. _table_rte_flow_count_and_drop: + +.. table:: Count and drop + + +-------+--------+ + | Index | Action | + +=======+========+ + | 0 | COUNT | + +-------+--------+ + | 1 | DROP | + +-------+--------+ + | 2 | END | + +-------+--------+ + +| + +.. _table_rte_flow_mark_count_redirect: + +.. table:: Mark, count and redirect + + +-------+--------+-----------+-------+ + | Index | Action | Field | Value | + +=======+========+===========+=======+ + | 0 | MARK | ``mark`` | 0x2a | + +-------+--------+-----------+-------+ + | 1 | COUNT | + +-------+--------+-----------+-------+ + | 2 | QUEUE | ``queue`` | 10 | + +-------+--------+-----------+-------+ + | 3 | END | + +-------+----------------------------+ + +| + +.. _table_rte_flow_redirect_queue_5: + +.. table:: Redirect to queue 5 + + +-------+--------+-----------+-------+ + | Index | Action | Field | Value | + +=======+========+===========+=======+ + | 0 | DROP | + +-------+--------+-----------+-------+ + | 1 | QUEUE | ``queue`` | 5 | + +-------+--------+-----------+-------+ + | 2 | END | + +-------+----------------------------+ + +In the above example, considering both actions are performed simultaneously, +the end result is that only QUEUE has any effect. + +.. _table_rte_flow_redirect_queue_3: + +.. table:: Redirect to queue 3 + + +-------+--------+-----------+-------+ + | Index | Action | Field | Value | + +=======+========+===========+=======+ + | 0 | QUEUE | ``queue`` | 5 | + +-------+--------+-----------+-------+ + | 1 | VOID | + +-------+--------+-----------+-------+ + | 2 | QUEUE | ``queue`` | 3 | + +-------+--------+-----------+-------+ + | 3 | END | + +-------+----------------------------+ + +As previously described, only the last action of a given type found in the +list is taken into account. The above example also shows that VOID is +ignored. + +Action types +~~~~~~~~~~~~ + +Common action types are described in this section. Like pattern item types, +this list is not exhaustive as new actions will be added in the future. + +Action: ``END`` +^^^^^^^^^^^^^^^ + +End marker for action lists. Prevents further processing of actions, thereby +ending the list. + +- Its numeric value is 0 for convenience. +- PMD support is mandatory. +- No configurable properties. + +.. _table_rte_flow_action_end: + +.. table:: END + + +---------------+ + | Field | + +===============+ + | no properties | + +---------------+ + +Action: ``VOID`` +^^^^^^^^^^^^^^^^ + +Used as a placeholder for convenience. It is ignored and simply discarded by +PMDs. + +- PMD support is mandatory. +- No configurable properties. + +.. _table_rte_flow_action_void: + +.. table:: VOID + + +---------------+ + | Field | + +===============+ + | no properties | + +---------------+ + +Action: ``PASSTHRU`` +^^^^^^^^^^^^^^^^^^^^ + +Leaves packets up for additional processing by subsequent flow rules. This +is the default when a rule does not contain a terminating action, but can be +specified to force a rule to become non-terminating. + +- No configurable properties. + +.. _table_rte_flow_action_passthru: + +.. table:: PASSTHRU + + +---------------+ + | Field | + +===============+ + | no properties | + +---------------+ + +Example to copy a packet to a queue and continue processing by subsequent +flow rules: + +.. _table_rte_flow_action_passthru_example: + +.. table:: Copy to queue 8 + + +-------+--------+-----------+-------+ + | Index | Action | Field | Value | + +=======+========+===========+=======+ + | 0 | PASSTHRU | + +-------+--------+-----------+-------+ + | 1 | QUEUE | ``queue`` | 8 | + +-------+--------+-----------+-------+ + | 2 | END | + +-------+----------------------------+ + +Action: ``MARK`` +^^^^^^^^^^^^^^^^ + +Attaches an integer value to packets and sets ``PKT_RX_FDIR`` and +``PKT_RX_FDIR_ID`` mbuf flags. + +This value is arbitrary and application-defined. Maximum allowed value +depends on the underlying implementation. It is returned in the +``hash.fdir.hi`` mbuf field. + +.. _table_rte_flow_action_mark: + +.. table:: MARK + + +--------+--------------------------------------+ + | Field | Value | + +========+======================================+ + | ``id`` | integer value to return with packets | + +--------+--------------------------------------+ + +Action: ``FLAG`` +^^^^^^^^^^^^^^^^ + +Flags packets. Similar to `Action: MARK`_ without a specific value; only +sets the ``PKT_RX_FDIR`` mbuf flag. + +- No configurable properties. + +.. _table_rte_flow_action_flag: + +.. table:: FLAG + + +---------------+ + | Field | + +===============+ + | no properties | + +---------------+ + +Action: ``QUEUE`` +^^^^^^^^^^^^^^^^^ + +Assigns packets to a given queue index. + +- Terminating by default. + +.. _table_rte_flow_action_queue: + +.. table:: QUEUE + + +-----------+--------------------+ + | Field | Value | + +===========+====================+ + | ``index`` | queue index to use | + +-----------+--------------------+ + +Action: ``DROP`` +^^^^^^^^^^^^^^^^ + +Drop packets. + +- No configurable properties. +- Terminating by default. +- PASSTHRU overrides this action if both are specified. + +.. _table_rte_flow_action_drop: + +.. table:: DROP + + +---------------+ + | Field | + +===============+ + | no properties | + +---------------+ + +Action: ``COUNT`` +^^^^^^^^^^^^^^^^^ + +Enables counters for this rule. + +These counters can be retrieved and reset through ``rte_flow_query()``, see +``struct rte_flow_query_count``. + +- Counters can be retrieved with ``rte_flow_query()``. +- No configurable properties. + +.. _table_rte_flow_action_count: + +.. table:: COUNT + + +---------------+ + | Field | + +===============+ + | no properties | + +---------------+ + +Query structure to retrieve and reset flow rule counters: + +.. _table_rte_flow_query_count: + +.. table:: COUNT query + + +---------------+-----+-----------------------------------+ + | Field | I/O | Value | + +===============+=====+===================================+ + | ``reset`` | in | reset counter after query | + +---------------+-----+-----------------------------------+ + | ``hits_set`` | out | ``hits`` field is set | + +---------------+-----+-----------------------------------+ + | ``bytes_set`` | out | ``bytes`` field is set | + +---------------+-----+-----------------------------------+ + | ``hits`` | out | number of hits for this rule | + +---------------+-----+-----------------------------------+ + | ``bytes`` | out | number of bytes through this rule | + +---------------+-----+-----------------------------------+ + +Action: ``DUP`` +^^^^^^^^^^^^^^^ + +Duplicates packets to a given queue index. + +This is normally combined with QUEUE, however when used alone, it is +actually similar to QUEUE + PASSTHRU. + +- Non-terminating by default. + +.. _table_rte_flow_action_dup: + +.. table:: DUP + + +-----------+------------------------------------+ + | Field | Value | + +===========+====================================+ + | ``index`` | queue index to duplicate packet to | + +-----------+------------------------------------+ + +Action: ``RSS`` +^^^^^^^^^^^^^^^ + +Similar to QUEUE, except RSS is additionally performed on packets to spread +them among several queues according to the provided parameters. + +Note: RSS hash result is stored in the ``hash.rss`` mbuf field which +overlaps ``hash.fdir.lo``. Since `Action: MARK`_ sets the ``hash.fdir.hi`` +field only, both can be requested simultaneously. + +- Terminating by default. + +.. _table_rte_flow_action_rss: + +.. table:: RSS + + +--------------+------------------------------+ + | Field | Value | + +==============+==============================+ + | ``rss_conf`` | RSS parameters | + +--------------+------------------------------+ + | ``num`` | number of entries in queue[] | + +--------------+------------------------------+ + | ``queue[]`` | queue indices to use | + +--------------+------------------------------+ + +Action: ``PF`` +^^^^^^^^^^^^^^ + +Redirects packets to the physical function (PF) of the current device. + +- No configurable properties. +- Terminating by default. + +.. _table_rte_flow_action_pf: + +.. table:: PF + + +---------------+ + | Field | + +===============+ + | no properties | + +---------------+ + +Action: ``VF`` +^^^^^^^^^^^^^^ + +Redirects packets to a virtual function (VF) of the current device. + +Packets matched by a VF pattern item can be redirected to their original VF +ID instead of the specified one. This parameter may not be available and is +not guaranteed to work properly if the VF part is matched by a prior flow +rule or if packets are not addressed to a VF in the first place. + +- Terminating by default. + +.. _table_rte_flow_action_vf: + +.. table:: VF + + +--------------+--------------------------------+ + | Field | Value | + +==============+================================+ + | ``original`` | use original VF ID if possible | + +--------------+--------------------------------+ + | ``vf`` | VF ID to redirect packets to | + +--------------+--------------------------------+ + +Action: ``METER`` +^^^^^^^^^^^^^^^^^ + +Applies a stage of metering and policing. + +The metering and policing (MTR) object has to be first created using the +rte_mtr_create() API function. The ID of the MTR object is specified as +action parameter. More than one flow can use the same MTR object through +the meter action. The MTR object can be further updated or queried using +the rte_mtr* API. + +- Non-terminating by default. + +.. _table_rte_flow_action_meter: + +.. table:: METER + + +--------------+---------------+ + | Field | Value | + +==============+===============+ + | ``mtr_id`` | MTR object ID | + +--------------+---------------+ + +Action: ``SECURITY`` +^^^^^^^^^^^^^^^^^^^^ + +Perform the security action on flows matched by the pattern items +according to the configuration of the security session. + +This action modifies the payload of matched flows. For INLINE_CRYPTO, the +security protocol headers and IV are fully provided by the application as +specified in the flow pattern. The payload of matching packets is +encrypted on egress, and decrypted and authenticated on ingress. +For INLINE_PROTOCOL, the security protocol is fully offloaded to HW, +providing full encapsulation and decapsulation of packets in security +protocols. The flow pattern specifies both the outer security header fields +and the inner packet fields. The security session specified in the action +must match the pattern parameters. + +The security session specified in the action must be created on the same +port as the flow action that is being specified. + +The ingress/egress flow attribute should match that specified in the +security session if the security session supports the definition of the +direction. + +Multiple flows can be configured to use the same security session. + +- Non-terminating by default. + +.. _table_rte_flow_action_security: + +.. table:: SECURITY + + +----------------------+--------------------------------------+ + | Field | Value | + +======================+======================================+ + | ``security_session`` | security session to apply | + +----------------------+--------------------------------------+ + +The following is an example of configuring IPsec inline using the +INLINE_CRYPTO security session: + +The encryption algorithm, keys and salt are part of the opaque +``rte_security_session``. The SA is identified according to the IP and ESP +fields in the pattern items. + +.. _table_rte_flow_item_esp_inline_example: + +.. table:: IPsec inline crypto flow pattern items. + + +-------+----------+ + | Index | Item | + +=======+==========+ + | 0 | Ethernet | + +-------+----------+ + | 1 | IPv4 | + +-------+----------+ + | 2 | ESP | + +-------+----------+ + | 3 | END | + +-------+----------+ + +.. _table_rte_flow_action_esp_inline_example: + +.. table:: IPsec inline flow actions. + + +-------+----------+ + | Index | Action | + +=======+==========+ + | 0 | SECURITY | + +-------+----------+ + | 1 | END | + +-------+----------+ + +Negative types +~~~~~~~~~~~~~~ + +All specified pattern items (``enum rte_flow_item_type``) and actions +(``enum rte_flow_action_type``) use positive identifiers. + +The negative space is reserved for dynamic types generated by PMDs during +run-time. PMDs may encounter them as a result but must not accept negative +identifiers they are not aware of. + +A method to generate them remains to be defined. + +Planned types +~~~~~~~~~~~~~ + +Pattern item types will be added as new protocols are implemented. + +Variable headers support through dedicated pattern items, for example in +order to match specific IPv4 options and IPv6 extension headers would be +stacked after IPv4/IPv6 items. + +Other action types are planned but are not defined yet. These include the +ability to alter packet data in several ways, such as performing +encapsulation/decapsulation of tunnel headers. + +Rules management +---------------- + +A rather simple API with few functions is provided to fully manage flow +rules. + +Each created flow rule is associated with an opaque, PMD-specific handle +pointer. The application is responsible for keeping it until the rule is +destroyed. + +Flows rules are represented by ``struct rte_flow`` objects. + +Validation +~~~~~~~~~~ + +Given that expressing a definite set of device capabilities is not +practical, a dedicated function is provided to check if a flow rule is +supported and can be created. + +.. code-block:: c + + int + rte_flow_validate(uint16_t port_id, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error); + +The flow rule is validated for correctness and whether it could be accepted +by the device given sufficient resources. The rule is checked against the +current device mode and queue configuration. The flow rule may also +optionally be validated against existing flow rules and device resources. +This function has no effect on the target device. + +The returned value is guaranteed to remain valid only as long as no +successful calls to ``rte_flow_create()`` or ``rte_flow_destroy()`` are made +in the meantime and no device parameter affecting flow rules in any way are +modified, due to possible collisions or resource limitations (although in +such cases ``EINVAL`` should not be returned). + +Arguments: + +- ``port_id``: port identifier of Ethernet device. +- ``attr``: flow rule attributes. +- ``pattern``: pattern specification (list terminated by the END pattern + item). +- ``actions``: associated actions (list terminated by the END action). +- ``error``: perform verbose error reporting if not NULL. PMDs initialize + this structure in case of error only. + +Return values: + +- 0 if flow rule is valid and can be created. A negative errno value + otherwise (``rte_errno`` is also set), the following errors are defined. +- ``-ENOSYS``: underlying device does not support this functionality. +- ``-EINVAL``: unknown or invalid rule specification. +- ``-ENOTSUP``: valid but unsupported rule specification (e.g. partial + bit-masks are unsupported). +- ``EEXIST``: collision with an existing rule. Only returned if device + supports flow rule collision checking and there was a flow rule + collision. Not receiving this return code is no guarantee that creating + the rule will not fail due to a collision. +- ``ENOMEM``: not enough memory to execute the function, or if the device + supports resource validation, resource limitation on the device. +- ``-EBUSY``: action cannot be performed due to busy device resources, may + succeed if the affected queues or even the entire port are in a stopped + state (see ``rte_eth_dev_rx_queue_stop()`` and ``rte_eth_dev_stop()``). + +Creation +~~~~~~~~ + +Creating a flow rule is similar to validating one, except the rule is +actually created and a handle returned. + +.. code-block:: c + + struct rte_flow * + rte_flow_create(uint16_t port_id, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action *actions[], + struct rte_flow_error *error); + +Arguments: + +- ``port_id``: port identifier of Ethernet device. +- ``attr``: flow rule attributes. +- ``pattern``: pattern specification (list terminated by the END pattern + item). +- ``actions``: associated actions (list terminated by the END action). +- ``error``: perform verbose error reporting if not NULL. PMDs initialize + this structure in case of error only. + +Return values: + +A valid handle in case of success, NULL otherwise and ``rte_errno`` is set +to the positive version of one of the error codes defined for +``rte_flow_validate()``. + +Destruction +~~~~~~~~~~~ + +Flow rules destruction is not automatic, and a queue or a port should not be +released if any are still attached to them. Applications must take care of +performing this step before releasing resources. + +.. code-block:: c + + int + rte_flow_destroy(uint16_t port_id, + struct rte_flow *flow, + struct rte_flow_error *error); + + +Failure to destroy a flow rule handle may occur when other flow rules depend +on it, and destroying it would result in an inconsistent state. + +This function is only guaranteed to succeed if handles are destroyed in +reverse order of their creation. + +Arguments: + +- ``port_id``: port identifier of Ethernet device. +- ``flow``: flow rule handle to destroy. +- ``error``: perform verbose error reporting if not NULL. PMDs initialize + this structure in case of error only. + +Return values: + +- 0 on success, a negative errno value otherwise and ``rte_errno`` is set. + +Flush +~~~~~ + +Convenience function to destroy all flow rule handles associated with a +port. They are released as with successive calls to ``rte_flow_destroy()``. + +.. code-block:: c + + int + rte_flow_flush(uint16_t port_id, + struct rte_flow_error *error); + +In the unlikely event of failure, handles are still considered destroyed and +no longer valid but the port must be assumed to be in an inconsistent state. + +Arguments: + +- ``port_id``: port identifier of Ethernet device. +- ``error``: perform verbose error reporting if not NULL. PMDs initialize + this structure in case of error only. + +Return values: + +- 0 on success, a negative errno value otherwise and ``rte_errno`` is set. + +Query +~~~~~ + +Query an existing flow rule. + +This function allows retrieving flow-specific data such as counters. Data +is gathered by special actions which must be present in the flow rule +definition. + +.. code-block:: c + + int + rte_flow_query(uint16_t port_id, + struct rte_flow *flow, + enum rte_flow_action_type action, + void *data, + struct rte_flow_error *error); + +Arguments: + +- ``port_id``: port identifier of Ethernet device. +- ``flow``: flow rule handle to query. +- ``action``: action type to query. +- ``data``: pointer to storage for the associated query data type. +- ``error``: perform verbose error reporting if not NULL. PMDs initialize + this structure in case of error only. + +Return values: + +- 0 on success, a negative errno value otherwise and ``rte_errno`` is set. + +Isolated mode +------------- + +The general expectation for ingress traffic is that flow rules process it +first; the remaining unmatched or pass-through traffic usually ends up in a +queue (with or without RSS, locally or in some sub-device instance) +depending on the global configuration settings of a port. + +While fine from a compatibility standpoint, this approach makes drivers more +complex as they have to check for possible side effects outside of this API +when creating or destroying flow rules. It results in a more limited set of +available rule types due to the way device resources are assigned (e.g. no +support for the RSS action even on capable hardware). + +Given that nonspecific traffic can be handled by flow rules as well, +isolated mode is a means for applications to tell a driver that ingress on +the underlying port must be injected from the defined flow rules only; that +no default traffic is expected outside those rules. + +This has the following benefits: + +- Applications get finer-grained control over the kind of traffic they want + to receive (no traffic by default). + +- More importantly they control at what point nonspecific traffic is handled + relative to other flow rules, by adjusting priority levels. + +- Drivers can assign more hardware resources to flow rules and expand the + set of supported rule types. + +Because toggling isolated mode may cause profound changes to the ingress +processing path of a driver, it may not be possible to leave it once +entered. Likewise, existing flow rules or global configuration settings may +prevent a driver from entering isolated mode. + +Applications relying on this mode are therefore encouraged to toggle it as +soon as possible after device initialization, ideally before the first call +to ``rte_eth_dev_configure()`` to avoid possible failures due to conflicting +settings. + +Once effective, the following functionality has no effect on the underlying +port and may return errors such as ``ENOTSUP`` ("not supported"): + +- Toggling promiscuous mode. +- Toggling allmulticast mode. +- Configuring MAC addresses. +- Configuring multicast addresses. +- Configuring VLAN filters. +- Configuring Rx filters through the legacy API (e.g. FDIR). +- Configuring global RSS settings. + +.. code-block:: c + + int + rte_flow_isolate(uint16_t port_id, int set, struct rte_flow_error *error); + +Arguments: + +- ``port_id``: port identifier of Ethernet device. +- ``set``: nonzero to enter isolated mode, attempt to leave it otherwise. +- ``error``: perform verbose error reporting if not NULL. PMDs initialize + this structure in case of error only. + +Return values: + +- 0 on success, a negative errno value otherwise and ``rte_errno`` is set. + +Verbose error reporting +----------------------- + +The defined *errno* values may not be accurate enough for users or +application developers who want to investigate issues related to flow rules +management. A dedicated error object is defined for this purpose: + +.. code-block:: c + + enum rte_flow_error_type { + RTE_FLOW_ERROR_TYPE_NONE, /**< No error. */ + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, /**< Cause unspecified. */ + RTE_FLOW_ERROR_TYPE_HANDLE, /**< Flow rule (handle). */ + RTE_FLOW_ERROR_TYPE_ATTR_GROUP, /**< Group field. */ + RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, /**< Priority field. */ + RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, /**< Ingress field. */ + RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, /**< Egress field. */ + RTE_FLOW_ERROR_TYPE_ATTR, /**< Attributes structure. */ + RTE_FLOW_ERROR_TYPE_ITEM_NUM, /**< Pattern length. */ + RTE_FLOW_ERROR_TYPE_ITEM, /**< Specific pattern item. */ + RTE_FLOW_ERROR_TYPE_ACTION_NUM, /**< Number of actions. */ + RTE_FLOW_ERROR_TYPE_ACTION, /**< Specific action. */ + }; + + struct rte_flow_error { + enum rte_flow_error_type type; /**< Cause field and error types. */ + const void *cause; /**< Object responsible for the error. */ + const char *message; /**< Human-readable error message. */ + }; + +Error type ``RTE_FLOW_ERROR_TYPE_NONE`` stands for no error, in which case +remaining fields can be ignored. Other error types describe the type of the +object pointed by ``cause``. + +If non-NULL, ``cause`` points to the object responsible for the error. For a +flow rule, this may be a pattern item or an individual action. + +If non-NULL, ``message`` provides a human-readable error message. + +This object is normally allocated by applications and set by PMDs in case of +error, the message points to a constant string which does not need to be +freed by the application, however its pointer can be considered valid only +as long as its associated DPDK port remains configured. Closing the +underlying device or unloading the PMD invalidates it. + +Helpers +------- + +Error initializer +~~~~~~~~~~~~~~~~~ + +.. code-block:: c + + static inline int + rte_flow_error_set(struct rte_flow_error *error, + int code, + enum rte_flow_error_type type, + const void *cause, + const char *message); + +This function initializes ``error`` (if non-NULL) with the provided +parameters and sets ``rte_errno`` to ``code``. A negative error ``code`` is +then returned. + +Caveats +------- + +- DPDK does not keep track of flow rules definitions or flow rule objects + automatically. Applications may keep track of the former and must keep + track of the latter. PMDs may also do it for internal needs, however this + must not be relied on by applications. + +- Flow rules are not maintained between successive port initializations. An + application exiting without releasing them and restarting must re-create + them from scratch. + +- API operations are synchronous and blocking (``EAGAIN`` cannot be + returned). + +- There is no provision for reentrancy/multi-thread safety, although nothing + should prevent different devices from being configured at the same + time. PMDs may protect their control path functions accordingly. + +- Stopping the data path (TX/RX) should not be necessary when managing flow + rules. If this cannot be achieved naturally or with workarounds (such as + temporarily replacing the burst function pointers), an appropriate error + code must be returned (``EBUSY``). + +- PMDs, not applications, are responsible for maintaining flow rules + configuration when stopping and restarting a port or performing other + actions which may affect them. They can only be destroyed explicitly by + applications. + +For devices exposing multiple ports sharing global settings affected by flow +rules: + +- All ports under DPDK control must behave consistently, PMDs are + responsible for making sure that existing flow rules on a port are not + affected by other ports. + +- Ports not under DPDK control (unaffected or handled by other applications) + are user's responsibility. They may affect existing flow rules and cause + undefined behavior. PMDs aware of this may prevent flow rules creation + altogether in such cases. + +PMD interface +------------- + +The PMD interface is defined in ``rte_flow_driver.h``. It is not subject to +API/ABI versioning constraints as it is not exposed to applications and may +evolve independently. + +It is currently implemented on top of the legacy filtering framework through +filter type *RTE_ETH_FILTER_GENERIC* that accepts the single operation +*RTE_ETH_FILTER_GET* to return PMD-specific *rte_flow* callbacks wrapped +inside ``struct rte_flow_ops``. + +This overhead is temporarily necessary in order to keep compatibility with +the legacy filtering framework, which should eventually disappear. + +- PMD callbacks implement exactly the interface described in `Rules + management`_, except for the port ID argument which has already been + converted to a pointer to the underlying ``struct rte_eth_dev``. + +- Public API functions do not process flow rules definitions at all before + calling PMD functions (no basic error checking, no validation + whatsoever). They only make sure these callbacks are non-NULL or return + the ``ENOSYS`` (function not supported) error. + +This interface additionally defines the following helper function: + +- ``rte_flow_ops_get()``: get generic flow operations structure from a + port. + +More will be added over time. + +Device compatibility +-------------------- + +No known implementation supports all the described features. + +Unsupported features or combinations are not expected to be fully emulated +in software by PMDs for performance reasons. Partially supported features +may be completed in software as long as hardware performs most of the work +(such as queue redirection and packet recognition). + +However PMDs are expected to do their best to satisfy application requests +by working around hardware limitations as long as doing so does not affect +the behavior of existing flow rules. + +The following sections provide a few examples of such cases and describe how +PMDs should handle them, they are based on limitations built into the +previous APIs. + +Global bit-masks +~~~~~~~~~~~~~~~~ + +Each flow rule comes with its own, per-layer bit-masks, while hardware may +support only a single, device-wide bit-mask for a given layer type, so that +two IPv4 rules cannot use different bit-masks. + +The expected behavior in this case is that PMDs automatically configure +global bit-masks according to the needs of the first flow rule created. + +Subsequent rules are allowed only if their bit-masks match those, the +``EEXIST`` error code should be returned otherwise. + +Unsupported layer types +~~~~~~~~~~~~~~~~~~~~~~~ + +Many protocols can be simulated by crafting patterns with the `Item: RAW`_ +type. + +PMDs can rely on this capability to simulate support for protocols with +headers not directly recognized by hardware. + +``ANY`` pattern item +~~~~~~~~~~~~~~~~~~~~ + +This pattern item stands for anything, which can be difficult to translate +to something hardware would understand, particularly if followed by more +specific types. + +Consider the following pattern: + +.. _table_rte_flow_unsupported_any: + +.. table:: Pattern with ANY as L3 + + +-------+-----------------------+ + | Index | Item | + +=======+=======================+ + | 0 | ETHER | + +-------+-----+---------+-------+ + | 1 | ANY | ``num`` | ``1`` | + +-------+-----+---------+-------+ + | 2 | TCP | + +-------+-----------------------+ + | 3 | END | + +-------+-----------------------+ + +Knowing that TCP does not make sense with something other than IPv4 and IPv6 +as L3, such a pattern may be translated to two flow rules instead: + +.. _table_rte_flow_unsupported_any_ipv4: + +.. table:: ANY replaced with IPV4 + + +-------+--------------------+ + | Index | Item | + +=======+====================+ + | 0 | ETHER | + +-------+--------------------+ + | 1 | IPV4 (zeroed mask) | + +-------+--------------------+ + | 2 | TCP | + +-------+--------------------+ + | 3 | END | + +-------+--------------------+ + +| + +.. _table_rte_flow_unsupported_any_ipv6: + +.. table:: ANY replaced with IPV6 + + +-------+--------------------+ + | Index | Item | + +=======+====================+ + | 0 | ETHER | + +-------+--------------------+ + | 1 | IPV6 (zeroed mask) | + +-------+--------------------+ + | 2 | TCP | + +-------+--------------------+ + | 3 | END | + +-------+--------------------+ + +Note that as soon as a ANY rule covers several layers, this approach may +yield a large number of hidden flow rules. It is thus suggested to only +support the most common scenarios (anything as L2 and/or L3). + +Unsupported actions +~~~~~~~~~~~~~~~~~~~ + +- When combined with `Action: QUEUE`_, packet counting (`Action: COUNT`_) + and tagging (`Action: MARK`_ or `Action: FLAG`_) may be implemented in + software as long as the target queue is used by a single rule. + +- A rule specifying both `Action: DUP`_ + `Action: QUEUE`_ may be translated + to two hidden rules combining `Action: QUEUE`_ and `Action: PASSTHRU`_. + +- When a single target queue is provided, `Action: RSS`_ can also be + implemented through `Action: QUEUE`_. + +Flow rules priority +~~~~~~~~~~~~~~~~~~~ + +While it would naturally make sense, flow rules cannot be assumed to be +processed by hardware in the same order as their creation for several +reasons: + +- They may be managed internally as a tree or a hash table instead of a + list. +- Removing a flow rule before adding another one can either put the new rule + at the end of the list or reuse a freed entry. +- Duplication may occur when packets are matched by several rules. + +For overlapping rules (particularly in order to use `Action: PASSTHRU`_) +predictable behavior is only guaranteed by using different priority levels. + +Priority levels are not necessarily implemented in hardware, or may be +severely limited (e.g. a single priority bit). + +For these reasons, priority levels may be implemented purely in software by +PMDs. + +- For devices expecting flow rules to be added in the correct order, PMDs + may destroy and re-create existing rules after adding a new one with + a higher priority. + +- A configurable number of dummy or empty rules can be created at + initialization time to save high priority slots for later. + +- In order to save priority levels, PMDs may evaluate whether rules are + likely to collide and adjust their priority accordingly. + +Future evolutions +----------------- + +- A device profile selection function which could be used to force a + permanent profile instead of relying on its automatic configuration based + on existing flow rules. + +- A method to optimize *rte_flow* rules with specific pattern items and + action types generated on the fly by PMDs. DPDK should assign negative + numbers to these in order to not collide with the existing types. See + `Negative types`_. + +- Adding specific egress pattern items and actions as described in + `Attribute: Traffic direction`_. + +- Optional software fallback when PMDs are unable to handle requested flow + rules so applications do not have to implement their own. + +API migration +------------- + +Exhaustive list of deprecated filter types (normally prefixed with +*RTE_ETH_FILTER_*) found in ``rte_eth_ctrl.h`` and methods to convert them +to *rte_flow* rules. + +``MACVLAN`` to ``ETH`` → ``VF``, ``PF`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +*MACVLAN* can be translated to a basic `Item: ETH`_ flow rule with a +terminating `Action: VF`_ or `Action: PF`_. + +.. _table_rte_flow_migration_macvlan: + +.. table:: MACVLAN conversion + + +--------------------------+---------+ + | Pattern | Actions | + +===+=====+==========+=====+=========+ + | 0 | ETH | ``spec`` | any | VF, | + | | +----------+-----+ PF | + | | | ``last`` | N/A | | + | | +----------+-----+ | + | | | ``mask`` | any | | + +---+-----+----------+-----+---------+ + | 1 | END | END | + +---+----------------------+---------+ + +``ETHERTYPE`` to ``ETH`` → ``QUEUE``, ``DROP`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +*ETHERTYPE* is basically an `Item: ETH`_ flow rule with a terminating +`Action: QUEUE`_ or `Action: DROP`_. + +.. _table_rte_flow_migration_ethertype: + +.. table:: ETHERTYPE conversion + + +--------------------------+---------+ + | Pattern | Actions | + +===+=====+==========+=====+=========+ + | 0 | ETH | ``spec`` | any | QUEUE, | + | | +----------+-----+ DROP | + | | | ``last`` | N/A | | + | | +----------+-----+ | + | | | ``mask`` | any | | + +---+-----+----------+-----+---------+ + | 1 | END | END | + +---+----------------------+---------+ + +``FLEXIBLE`` to ``RAW`` → ``QUEUE`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +*FLEXIBLE* can be translated to one `Item: RAW`_ pattern with a terminating +`Action: QUEUE`_ and a defined priority level. + +.. _table_rte_flow_migration_flexible: + +.. table:: FLEXIBLE conversion + + +--------------------------+---------+ + | Pattern | Actions | + +===+=====+==========+=====+=========+ + | 0 | RAW | ``spec`` | any | QUEUE | + | | +----------+-----+ | + | | | ``last`` | N/A | | + | | +----------+-----+ | + | | | ``mask`` | any | | + +---+-----+----------+-----+---------+ + | 1 | END | END | + +---+----------------------+---------+ + +``SYN`` to ``TCP`` → ``QUEUE`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +*SYN* is a `Item: TCP`_ rule with only the ``syn`` bit enabled and masked, +and a terminating `Action: QUEUE`_. + +Priority level can be set to simulate the high priority bit. + +.. _table_rte_flow_migration_syn: + +.. table:: SYN conversion + + +-----------------------------------+---------+ + | Pattern | Actions | + +===+======+==========+=============+=========+ + | 0 | ETH | ``spec`` | unset | QUEUE | + | | +----------+-------------+ | + | | | ``last`` | unset | | + | | +----------+-------------+ | + | | | ``mask`` | unset | | + +---+------+----------+-------------+---------+ + | 1 | IPV4 | ``spec`` | unset | END | + | | +----------+-------------+ | + | | | ``mask`` | unset | | + | | +----------+-------------+ | + | | | ``mask`` | unset | | + +---+------+----------+---------+---+ | + | 2 | TCP | ``spec`` | ``syn`` | 1 | | + | | +----------+---------+---+ | + | | | ``mask`` | ``syn`` | 1 | | + +---+------+----------+---------+---+ | + | 3 | END | | + +---+-------------------------------+---------+ + +``NTUPLE`` to ``IPV4``, ``TCP``, ``UDP`` → ``QUEUE`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +*NTUPLE* is similar to specifying an empty L2, `Item: IPV4`_ as L3 with +`Item: TCP`_ or `Item: UDP`_ as L4 and a terminating `Action: QUEUE`_. + +A priority level can be specified as well. + +.. _table_rte_flow_migration_ntuple: + +.. table:: NTUPLE conversion + + +-----------------------------+---------+ + | Pattern | Actions | + +===+======+==========+=======+=========+ + | 0 | ETH | ``spec`` | unset | QUEUE | + | | +----------+-------+ | + | | | ``last`` | unset | | + | | +----------+-------+ | + | | | ``mask`` | unset | | + +---+------+----------+-------+---------+ + | 1 | IPV4 | ``spec`` | any | END | + | | +----------+-------+ | + | | | ``last`` | unset | | + | | +----------+-------+ | + | | | ``mask`` | any | | + +---+------+----------+-------+ | + | 2 | TCP, | ``spec`` | any | | + | | UDP +----------+-------+ | + | | | ``last`` | unset | | + | | +----------+-------+ | + | | | ``mask`` | any | | + +---+------+----------+-------+ | + | 3 | END | | + +---+-------------------------+---------+ + +``TUNNEL`` to ``ETH``, ``IPV4``, ``IPV6``, ``VXLAN`` (or other) → ``QUEUE`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +*TUNNEL* matches common IPv4 and IPv6 L3/L4-based tunnel types. + +In the following table, `Item: ANY`_ is used to cover the optional L4. + +.. _table_rte_flow_migration_tunnel: + +.. table:: TUNNEL conversion + + +-------------------------------------------------------+---------+ + | Pattern | Actions | + +===+==========================+==========+=============+=========+ + | 0 | ETH | ``spec`` | any | QUEUE | + | | +----------+-------------+ | + | | | ``last`` | unset | | + | | +----------+-------------+ | + | | | ``mask`` | any | | + +---+--------------------------+----------+-------------+---------+ + | 1 | IPV4, IPV6 | ``spec`` | any | END | + | | +----------+-------------+ | + | | | ``last`` | unset | | + | | +----------+-------------+ | + | | | ``mask`` | any | | + +---+--------------------------+----------+-------------+ | + | 2 | ANY | ``spec`` | any | | + | | +----------+-------------+ | + | | | ``last`` | unset | | + | | +----------+---------+---+ | + | | | ``mask`` | ``num`` | 0 | | + +---+--------------------------+----------+---------+---+ | + | 3 | VXLAN, GENEVE, TEREDO, | ``spec`` | any | | + | | NVGRE, GRE, ... +----------+-------------+ | + | | | ``last`` | unset | | + | | +----------+-------------+ | + | | | ``mask`` | any | | + +---+--------------------------+----------+-------------+ | + | 4 | END | | + +---+---------------------------------------------------+---------+ + +``FDIR`` to most item types → ``QUEUE``, ``DROP``, ``PASSTHRU`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +*FDIR* is more complex than any other type, there are several methods to +emulate its functionality. It is summarized for the most part in the table +below. + +A few features are intentionally not supported: + +- The ability to configure the matching input set and masks for the entire + device, PMDs should take care of it automatically according to the + requested flow rules. + + For example if a device supports only one bit-mask per protocol type, + source/address IPv4 bit-masks can be made immutable by the first created + rule. Subsequent IPv4 or TCPv4 rules can only be created if they are + compatible. + + Note that only protocol bit-masks affected by existing flow rules are + immutable, others can be changed later. They become mutable again after + the related flow rules are destroyed. + +- Returning four or eight bytes of matched data when using flex bytes + filtering. Although a specific action could implement it, it conflicts + with the much more useful 32 bits tagging on devices that support it. + +- Side effects on RSS processing of the entire device. Flow rules that + conflict with the current device configuration should not be + allowed. Similarly, device configuration should not be allowed when it + affects existing flow rules. + +- Device modes of operation. "none" is unsupported since filtering cannot be + disabled as long as a flow rule is present. + +- "MAC VLAN" or "tunnel" perfect matching modes should be automatically set + according to the created flow rules. + +- Signature mode of operation is not defined but could be handled through + "FUZZY" item. + +.. _table_rte_flow_migration_fdir: + +.. table:: FDIR conversion + + +----------------------------------------+-----------------------+ + | Pattern | Actions | + +===+===================+==========+=====+=======================+ + | 0 | ETH, RAW | ``spec`` | any | QUEUE, DROP, PASSTHRU | + | | +----------+-----+ | + | | | ``last`` | N/A | | + | | +----------+-----+ | + | | | ``mask`` | any | | + +---+-------------------+----------+-----+-----------------------+ + | 1 | IPV4, IPv6 | ``spec`` | any | MARK | + | | +----------+-----+ | + | | | ``last`` | N/A | | + | | +----------+-----+ | + | | | ``mask`` | any | | + +---+-------------------+----------+-----+-----------------------+ + | 2 | TCP, UDP, SCTP | ``spec`` | any | END | + | | +----------+-----+ | + | | | ``last`` | N/A | | + | | +----------+-----+ | + | | | ``mask`` | any | | + +---+-------------------+----------+-----+ | + | 3 | VF, PF, FUZZY | ``spec`` | any | | + | | (optional) +----------+-----+ | + | | | ``last`` | N/A | | + | | +----------+-----+ | + | | | ``mask`` | any | | + +---+-------------------+----------+-----+ | + | 4 | END | | + +---+------------------------------------+-----------------------+ + +``HASH`` +~~~~~~~~ + +There is no counterpart to this filter type because it translates to a +global device setting instead of a pattern item. Device settings are +automatically set according to the created flow rules. + +``L2_TUNNEL`` to ``VOID`` → ``VXLAN`` (or others) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +All packets are matched. This type alters incoming packets to encapsulate +them in a chosen tunnel type, optionally redirect them to a VF as well. + +The destination pool for tag based forwarding can be emulated with other +flow rules using `Action: DUP`_. + +.. _table_rte_flow_migration_l2tunnel: + +.. table:: L2_TUNNEL conversion + + +---------------------------+--------------------+ + | Pattern | Actions | + +===+======+==========+=====+====================+ + | 0 | VOID | ``spec`` | N/A | VXLAN, GENEVE, ... | + | | | | | | + | | | | | | + | | +----------+-----+ | + | | | ``last`` | N/A | | + | | +----------+-----+ | + | | | ``mask`` | N/A | | + | | | | | | + +---+------+----------+-----+--------------------+ + | 1 | END | VF (optional) | + +---+ +--------------------+ + | 2 | | END | + +---+-----------------------+--------------------+ diff --git a/dpdk/doc/guides/prog_guide/rte_security.rst b/dpdk/doc/guides/prog_guide/rte_security.rst new file mode 100644 index 00000000..71be036c --- /dev/null +++ b/dpdk/doc/guides/prog_guide/rte_security.rst @@ -0,0 +1,564 @@ +.. BSD LICENSE + Copyright 2017 NXP. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of NXP nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +Security Library +================ + +The security library provides a framework for management and provisioning +of security protocol operations offloaded to hardware based devices. The +library defines generic APIs to create and free security sessions which can +support full protocol offload as well as inline crypto operation with +NIC or crypto devices. The framework currently only supports the IPSec protocol +and associated operations, other protocols will be added in future. + +Design Principles +----------------- + +The security library provides an additional offload capability to an existing +crypto device and/or ethernet device. + +.. code-block:: console + + +---------------+ + | rte_security | + +---------------+ + \ / + +-----------+ +--------------+ + | NIC PMD | | CRYPTO PMD | + +-----------+ +--------------+ + +.. note:: + + Currently, the security library does not support the case of multi-process. + It will be updated in the future releases. + +The supported offload types are explained in the sections below. + +Inline Crypto +~~~~~~~~~~~~~ + +RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO: +The crypto processing for security protocol (e.g. IPSec) is processed +inline during receive and transmission on NIC port. The flow based +security action should be configured on the port. + +Ingress Data path - The packet is decrypted in RX path and relevant +crypto status is set in Rx descriptors. After the successful inline +crypto processing the packet is presented to host as a regular Rx packet +however all security protocol related headers are still attached to the +packet. e.g. In case of IPSec, the IPSec tunnel headers (if any), +ESP/AH headers will remain in the packet but the received packet +contains the decrypted data where the encrypted data was when the packet +arrived. The driver Rx path check the descriptors and and based on the +crypto status sets additional flags in the rte_mbuf.ol_flags field. + +.. note:: + + The underlying device may not support crypto processing for all ingress packet + matching to a particular flow (e.g. fragmented packets), such packets will + be passed as encrypted packets. It is the responsibility of application to + process such encrypted packets using other crypto driver instance. + +Egress Data path - The software prepares the egress packet by adding +relevant security protocol headers. Only the data will not be +encrypted by the software. The driver will accordingly configure the +tx descriptors. The hardware device will encrypt the data before sending the +the packet out. + +.. note:: + + The underlying device may support post encryption TSO. + +.. code-block:: console + + Egress Data Path + | + +--------|--------+ + | egress IPsec | + | | | + | +------V------+ | + | | SADB lookup | | + | +------|------+ | + | +------V------+ | + | | Tunnel | | <------ Add tunnel header to packet + | +------|------+ | + | +------V------+ | + | | ESP | | <------ Add ESP header without trailer to packet + | | | | <------ Mark packet to be offloaded, add trailer + | +------|------+ | meta-data to mbuf + +--------V--------+ + | + +--------V--------+ + | L2 Stack | + +--------|--------+ + | + +--------V--------+ + | | + | NIC PMD | <------ Set hw context for inline crypto offload + | | + +--------|--------+ + | + +--------|--------+ + | HW ACCELERATED | <------ Packet Encryption and + | NIC | Authentication happens inline + | | + +-----------------+ + + +Inline protocol offload +~~~~~~~~~~~~~~~~~~~~~~~ + +RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL: +The crypto and protocol processing for security protocol (e.g. IPSec) +is processed inline during receive and transmission. The flow based +security action should be configured on the port. + +Ingress Data path - The packet is decrypted in the RX path and relevant +crypto status is set in the Rx descriptors. After the successful inline +crypto processing the packet is presented to the host as a regular Rx packet +but all security protocol related headers are optionally removed from the +packet. e.g. in the case of IPSec, the IPSec tunnel headers (if any), +ESP/AH headers will be removed from the packet and the received packet +will contains the decrypted packet only. The driver Rx path checks the +descriptors and based on the crypto status sets additional flags in +``rte_mbuf.ol_flags`` field. + +.. note:: + + The underlying device in this case is stateful. It is expected that + the device shall support crypto processing for all kind of packets matching + to a given flow, this includes fragmented packets (post reassembly). + E.g. in case of IPSec the device may internally manage anti-replay etc. + It will provide a configuration option for anti-replay behavior i.e. to drop + the packets or pass them to driver with error flags set in the descriptor. + +Egress Data path - The software will send the plain packet without any +security protocol headers added to the packet. The driver will configure +the security index and other requirement in tx descriptors. +The hardware device will do security processing on the packet that includes +adding the relevant protocol headers and encrypting the data before sending +the packet out. The software should make sure that the buffer +has required head room and tail room for any protocol header addition. The +software may also do early fragmentation if the resultant packet is expected +to cross the MTU size. + + +.. note:: + + The underlying device will manage state information required for egress + processing. E.g. in case of IPSec, the seq number will be added to the + packet, however the device shall provide indication when the sequence number + is about to overflow. The underlying device may support post encryption TSO. + +.. code-block:: console + + Egress Data Path + | + +--------|--------+ + | egress IPsec | + | | | + | +------V------+ | + | | SADB lookup | | + | +------|------+ | + | +------V------+ | + | | Desc | | <------ Mark packet to be offloaded + | +------|------+ | + +--------V--------+ + | + +--------V--------+ + | L2 Stack | + +--------|--------+ + | + +--------V--------+ + | | + | NIC PMD | <------ Set hw context for inline crypto offload + | | + +--------|--------+ + | + +--------|--------+ + | HW ACCELERATED | <------ Add tunnel, ESP header etc header to + | NIC | packet. Packet Encryption and + | | Authentication happens inline. + +-----------------+ + + +Lookaside protocol offload +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL: +This extends librte_cryptodev to support the programming of IPsec +Security Association (SA) as part of a crypto session creation including +the definition. In addition to standard crypto processing, as defined by +the cryptodev, the security protocol processing is also offloaded to the +crypto device. + +Decryption: The packet is sent to the crypto device for security +protocol processing. The device will decrypt the packet and it will also +optionally remove additional security headers from the packet. +E.g. in case of IPSec, IPSec tunnel headers (if any), ESP/AH headers +will be removed from the packet and the decrypted packet may contain +plain data only. + +.. note:: + + In case of IPSec the device may internally manage anti-replay etc. + It will provide a configuration option for anti-replay behavior i.e. to drop + the packets or pass them to driver with error flags set in descriptor. + +Encryption: The software will submit the packet to cryptodev as usual +for encryption, the hardware device in this case will also add the relevant +security protocol header along with encrypting the packet. The software +should make sure that the buffer has required head room and tail room +for any protocol header addition. + +.. note:: + + In the case of IPSec, the seq number will be added to the packet, + It shall provide an indication when the sequence number is about to + overflow. + +.. code-block:: console + + Egress Data Path + | + +--------|--------+ + | egress IPsec | + | | | + | +------V------+ | + | | SADB lookup | | <------ SA maps to cryptodev session + | +------|------+ | + | +------|------+ | + | | \--------------------\ + | | Crypto | | | <- Crypto processing through + | | /----------------\ | inline crypto PMD + | +------|------+ | | | + +--------V--------+ | | + | | | + +--------V--------+ | | create <-- SA is added to hw + | L2 Stack | | | inline using existing create + +--------|--------+ | | session sym session APIs + | | | | + +--------V--------+ +---|---|----V---+ + | | | \---/ | | <--- Add tunnel, ESP header etc + | NIC PMD | | INLINE | | header to packet.Packet + | | | CRYPTO PMD | | Encryption/Decryption and + +--------|--------+ +----------------+ Authentication happens + | inline. + +--------|--------+ + | NIC | + +--------|--------+ + V + +Device Features and Capabilities +--------------------------------- + +Device Capabilities For Security Operations +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The device (crypto or ethernet) capabilities which support security operations, +are defined by the security action type, security protocol, protocol +capabilities and corresponding crypto capabilities for security. For the full +scope of the Security capability see definition of rte_security_capability +structure in the *DPDK API Reference*. + +.. code-block:: c + + struct rte_security_capability; + +Each driver (crypto or ethernet) defines its own private array of capabilities +for the operations it supports. Below is an example of the capabilities for a +PMD which supports the IPSec protocol. + +.. code-block:: c + + static const struct rte_security_capability pmd_security_capabilities[] = { + { /* IPsec Lookaside Protocol offload ESP Tunnel Egress */ + .action = RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL, + .protocol = RTE_SECURITY_PROTOCOL_IPSEC, + .ipsec = { + .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP, + .mode = RTE_SECURITY_IPSEC_SA_MODE_TUNNEL, + .direction = RTE_SECURITY_IPSEC_SA_DIR_EGRESS, + .options = { 0 } + }, + .crypto_capabilities = pmd_capabilities + }, + { /* IPsec Lookaside Protocol offload ESP Tunnel Ingress */ + .action = RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL, + .protocol = RTE_SECURITY_PROTOCOL_IPSEC, + .ipsec = { + .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP, + .mode = RTE_SECURITY_IPSEC_SA_MODE_TUNNEL, + .direction = RTE_SECURITY_IPSEC_SA_DIR_INGRESS, + .options = { 0 } + }, + .crypto_capabilities = pmd_capabilities + }, + { + .action = RTE_SECURITY_ACTION_TYPE_NONE + } + }; + static const struct rte_cryptodev_capabilities pmd_capabilities[] = { + { /* SHA1 HMAC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + .sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + .auth = { + .algo = RTE_CRYPTO_AUTH_SHA1_HMAC, + .block_size = 64, + .key_size = { + .min = 64, + .max = 64, + .increment = 0 + }, + .digest_size = { + .min = 12, + .max = 12, + .increment = 0 + }, + .aad_size = { 0 }, + .iv_size = { 0 } + } + } + }, + { /* AES CBC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + .sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, + .cipher = { + .algo = RTE_CRYPTO_CIPHER_AES_CBC, + .block_size = 16, + .key_size = { + .min = 16, + .max = 32, + .increment = 8 + }, + .iv_size = { + .min = 16, + .max = 16, + .increment = 0 + } + } + } + } + } + + +Capabilities Discovery +~~~~~~~~~~~~~~~~~~~~~~ + +Discovering the features and capabilities of a driver (crypto/ethernet) +is achieved through the ``rte_security_capabilities_get()`` function. + +.. code-block:: c + + const struct rte_security_capability *rte_security_capabilities_get(uint16_t id); + +This allows the user to query a specific driver and get all device +security capabilities. It returns an array of ``rte_security_capability`` structures +which contains all the capabilities for that device. + +Security Session Create/Free +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Security Sessions are created to store the immutable fields of a particular Security +Association for a particular protocol which is defined by a security session +configuration structure which is used in the operation processing of a packet flow. +Sessions are used to manage protocol specific information as well as crypto parameters. +Security sessions cache this immutable data in a optimal way for the underlying PMD +and this allows further acceleration of the offload of Crypto workloads. + +The Security framework provides APIs to create and free sessions for crypto/ethernet +devices, where sessions are mempool objects. It is the application's responsibility +to create and manage the session mempools. The mempool object size should be able to +accommodate the driver's private data of security session. + +Once the session mempools have been created, ``rte_security_session_create()`` +is used to allocate and initialize a session for the required crypto/ethernet device. + +Session APIs need a parameter ``rte_security_ctx`` to identify the crypto/ethernet +security ops. This parameter can be retrieved using the APIs +``rte_cryptodev_get_sec_ctx()`` (for crypto device) or ``rte_eth_dev_get_sec_ctx`` +(for ethernet port). + +Sessions already created can be updated with ``rte_security_session_update()``. + +When a session is no longer used, the user must call ``rte_security_session_destroy()`` +to free the driver private session data and return the memory back to the mempool. + +For look aside protocol offload to hardware crypto device, the ``rte_crypto_op`` +created by the application is attached to the security session by the API +``rte_security_attach_session()``. + +For Inline Crypto and Inline protocol offload, device specific defined metadata is +updated in the mbuf using ``rte_security_set_pkt_metadata()`` if +``DEV_TX_OFFLOAD_SEC_NEED_MDATA`` is set. + +Security session configuration +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Security Session configuration structure is defined as ``rte_security_session_conf`` + +.. code-block:: c + + struct rte_security_session_conf { + enum rte_security_session_action_type action_type; + /**< Type of action to be performed on the session */ + enum rte_security_session_protocol protocol; + /**< Security protocol to be configured */ + union { + struct rte_security_ipsec_xform ipsec; + struct rte_security_macsec_xform macsec; + }; + /**< Configuration parameters for security session */ + struct rte_crypto_sym_xform *crypto_xform; + /**< Security Session Crypto Transformations */ + }; + +The configuration structure reuses the ``rte_crypto_sym_xform`` struct for crypto related +configuration. The ``rte_security_session_action_type`` struct is used to specify whether the +session is configured for Lookaside Protocol offload or Inline Crypto or Inline Protocol +Offload. + +.. code-block:: c + + enum rte_security_session_action_type { + RTE_SECURITY_ACTION_TYPE_NONE, + /**< No security actions */ + RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO, + /**< Crypto processing for security protocol is processed inline + * during transmission */ + RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL, + /**< All security protocol processing is performed inline during + * transmission */ + RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL + /**< All security protocol processing including crypto is performed + * on a lookaside accelerator */ + }; + +The ``rte_security_session_protocol`` is defined as + +.. code-block:: c + + enum rte_security_session_protocol { + RTE_SECURITY_PROTOCOL_IPSEC, + /**< IPsec Protocol */ + RTE_SECURITY_PROTOCOL_MACSEC, + /**< MACSec Protocol */ + }; + +Currently the library defines configuration parameters for IPSec only. For other +protocols like MACSec, structures and enums are defined as place holders which +will be updated in the future. + +IPsec related configuration parameters are defined in ``rte_security_ipsec_xform`` + +.. code-block:: c + + struct rte_security_ipsec_xform { + uint32_t spi; + /**< SA security parameter index */ + uint32_t salt; + /**< SA salt */ + struct rte_security_ipsec_sa_options options; + /**< various SA options */ + enum rte_security_ipsec_sa_direction direction; + /**< IPSec SA Direction - Egress/Ingress */ + enum rte_security_ipsec_sa_protocol proto; + /**< IPsec SA Protocol - AH/ESP */ + enum rte_security_ipsec_sa_mode mode; + /**< IPsec SA Mode - transport/tunnel */ + struct rte_security_ipsec_tunnel_param tunnel; + /**< Tunnel parameters, NULL for transport mode */ + }; + + +Security API +~~~~~~~~~~~~ + +The rte_security Library API is described in the *DPDK API Reference* document. + +Flow based Security Session +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In the case of NIC based offloads, the security session specified in the +'rte_flow_action_security' must be created on the same port as the +flow action that is being specified. + +The ingress/egress flow attribute should match that specified in the security +session if the security session supports the definition of the direction. + +Multiple flows can be configured to use the same security session. For +example if the security session specifies an egress IPsec SA, then multiple +flows can be specified to that SA. In the case of an ingress IPsec SA then +it is only valid to have a single flow to map to that security session. + +.. code-block:: console + + Configuration Path + | + +--------|--------+ + | Add/Remove | + | IPsec SA | <------ Build security flow action of + | | | ipsec transform + |--------|--------| + | + +--------V--------+ + | Flow API | + +--------|--------+ + | + +--------V--------+ + | | + | NIC PMD | <------ Add/Remove SA to/from hw context + | | + +--------|--------+ + | + +--------|--------+ + | HW ACCELERATED | + | NIC | + | | + +--------|--------+ + +* Add/Delete SA flow: + To add a new inline SA construct a rte_flow_item for Ethernet + IP + ESP + using the SA selectors and the ``rte_crypto_ipsec_xform`` as the ``rte_flow_action``. + Note that any rte_flow_items may be empty, which means it is not checked. + +.. code-block:: console + + In its most basic form, IPsec flow specification is as follows: + +-------+ +----------+ +--------+ +-----+ + | Eth | -> | IP4/6 | -> | ESP | -> | END | + +-------+ +----------+ +--------+ +-----+ + + However, the API can represent, IPsec crypto offload with any encapsulation: + +-------+ +--------+ +-----+ + | Eth | -> ... -> | ESP | -> | END | + +-------+ +--------+ +-----+ diff --git a/dpdk/doc/guides/prog_guide/service_cores.rst b/dpdk/doc/guides/prog_guide/service_cores.rst new file mode 100644 index 00000000..3a029ba9 --- /dev/null +++ b/dpdk/doc/guides/prog_guide/service_cores.rst @@ -0,0 +1,81 @@ +.. BSD LICENSE + Copyright(c) 2017 Intel Corporation. All rights reserved. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Service Cores +============= + +DPDK has a concept known as service cores, which enables a dynamic way of +performing work on DPDK lcores. Service core support is built into the EAL, and +an API is provided to optionally allow applications to control how the service +cores are used at runtime. + +The service cores concept is built up out of services (components of DPDK that +require CPU cycles to operate) and service cores (DPDK lcores, tasked with +running services). The power of the service core concept is that the mapping +between service cores and services can be configured to abstract away the +difference between platforms and environments. + +For example, the Eventdev has hardware and software PMDs. Of these the software +PMD requires an lcore to perform the scheduling operations, while the hardware +PMD does not. With service cores, the application would not directly notice +that the scheduling is done in software. + +For detailed information about the service core API, please refer to the docs. + +Service Core Initialization +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +There are two methods to having service cores in a DPDK application, either by +using the service coremask, or by dynamically adding cores using the API. +The simpler of the two is to pass the `-s` coremask argument to EAL, which will +take any cores available in the main DPDK coremask, an if the bits are also set +in the service coremask the cores become service-cores instead of DPDK +application lcores. + +Enabling Services on Cores +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Each registered service can be individually mapped to a service core, or set of +service cores. Enabling a service on a particular core means that the lcore in +question will run the service. Disabling that core on the service stops the +lcore in question from running the service. + +Using this method, it is possible to assign specific workloads to each +service core, and map N workloads to M number of service cores. Each service +lcore loops over the services that are enabled for that core, and invokes the +function to run the service. + +Service Core Statistics +~~~~~~~~~~~~~~~~~~~~~~~ + +The service core library is capable of collecting runtime statistics like number +of calls to a specific service, and number of cycles used by the service. The +cycle count collection is dynamically configurable, allowing any application to +profile the services running on the system at any time. diff --git a/dpdk/doc/guides/prog_guide/source_org.rst b/dpdk/doc/guides/prog_guide/source_org.rst index 0c06d47b..7aab4b45 100644 --- a/dpdk/doc/guides/prog_guide/source_org.rst +++ b/dpdk/doc/guides/prog_guide/source_org.rst @@ -70,7 +70,6 @@ The lib directory contains:: +-- librte_ether # Generic interface to poll mode driver +-- librte_hash # Hash library +-- librte_ip_frag # IP fragmentation library - +-- librte_ivshmem # QEMU IVSHMEM library +-- librte_kni # Kernel NIC interface +-- librte_kvargs # Argument parsing library +-- librte_lpm # Longest prefix match library @@ -109,7 +108,6 @@ The drivers directory has a *net* subdirectory which contains:: +-- szedata2 # SZEDATA2 poll mode driver +-- virtio # Virtio poll mode driver +-- vmxnet3 # VMXNET3 poll mode driver - +-- xenvirt # Xen virtio poll mode driver .. note:: @@ -141,7 +139,6 @@ The examples directory contains sample applications that show how libraries can examples +-- cmdline # Example of using the cmdline library - +-- dpdk_qat # Sample integration with Intel QuickAssist +-- exception_path # Sending packets to and from Linux TAP device +-- helloworld # Basic Hello World example +-- ip_reassembly # Example showing IP reassembly diff --git a/dpdk/doc/guides/prog_guide/traffic_management.rst b/dpdk/doc/guides/prog_guide/traffic_management.rst new file mode 100644 index 00000000..c0dc235e --- /dev/null +++ b/dpdk/doc/guides/prog_guide/traffic_management.rst @@ -0,0 +1,251 @@ +.. BSD LICENSE + Copyright(c) 2017 Intel Corporation. All rights reserved. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +Traffic Management API +====================== + + +Overview +-------- + +This is the generic API for the Quality of Service (QoS) Traffic Management of +Ethernet devices, which includes the following main features: hierarchical +scheduling, traffic shaping, congestion management, packet marking. This API +is agnostic of the underlying HW, SW or mixed HW-SW implementation. + +Main features: + +* Part of DPDK rte_ethdev API +* Capability query API per port, per hierarchy level and per hierarchy node +* Scheduling algorithms: Strict Priority (SP), Weighed Fair Queuing (WFQ) +* Traffic shaping: single/dual rate, private (per node) and + shared (by multiple nodes) shapers +* Congestion management for hierarchy leaf nodes: algorithms of tail drop, head + drop, WRED, private (per node) and shared (by multiple nodes) WRED contexts +* Packet marking: IEEE 802.1q (VLAN DEI), IETF RFC 3168 (IPv4/IPv6 ECN for TCP + and SCTP), IETF RFC 2597 (IPv4 / IPv6 DSCP) + + +Capability API +-------------- + +The aim of these APIs is to advertise the capability information (i.e critical +parameter values) that the TM implementation (HW/SW) is able to support for the +application. The APIs supports the information disclosure at the TM level, at +any hierarchical level of the TM and at any node level of the specific +hierarchical level. Such information helps towards rapid understanding of +whether a specific implementation does meet the needs to the user application. + +At the TM level, users can get high level idea with the help of various +parameters such as maximum number of nodes, maximum number of hierarchical +levels, maximum number of shapers, maximum number of private shapers, type of +scheduling algorithm (Strict Priority, Weighted Fair Queueing , etc.), etc., +supported by the implementation. + +Likewise, users can query the capability of the TM at the hierarchical level to +have more granular knowledge about the specific level. The various parameters +such as maximum number of nodes at the level, maximum number of leaf/non-leaf +nodes at the level, type of the shaper(dual rate, single rate) supported at +the level if node is non-leaf type etc., are exposed as a result of +hierarchical level capability query. + +Finally, the node level capability API offers knowledge about the capability +supported by the node at any specific level. The information whether the +support is available for private shaper, dual rate shaper, maximum and minimum +shaper rate, etc. is exposed by node level capability API. + + +Scheduling Algorithms +--------------------- + +The fundamental scheduling algorithms that are supported are Strict Priority +(SP) and Weighted Fair Queuing (WFQ). The SP and WFQ algorithms are supported +at the level of each node of the scheduling hierarchy, regardless of the node +level/position in the tree. The SP algorithm is used to schedule between +sibling nodes with different priority, while WFQ is used to schedule between +groups of siblings that have the same priority. + +Algorithms such as Weighed Round Robin (WRR), byte-level WRR, Deficit WRR +(DWRR), etc are considered approximations of the ideal WFQ and are therefore +assimilated to WFQ, although an associated implementation-dependent accuracy, +performance and resource usage trade-off might exist. + + +Traffic Shaping +--------------- + +The TM API provides support for single rate and dual rate shapers (rate +limiters) for the hierarchy nodes, subject to the specific implementation +support being available. + +Each hierarchy node has zero or one private shaper (only one node using it) +and/or zero, one or several shared shapers (multiple nodes use the same shaper +instance). A private shaper is used to perform traffic shaping for a single +node, while a shared shaper is used to perform traffic shaping for a group of +nodes. + +The configuration of private and shared shapers is done through the definition +of shaper profiles. Any shaper profile (single rate or dual rate shaper) can be +used by one or several shaper instances (either private or shared). + +Single rate shapers use a single token bucket. Therefore, single rate shaper is +configured by setting the rate of the committed bucket to zero, which +effectively disables this bucket. The peak bucket is used to limit the rate +and the burst size for the single rate shaper. Dual rate shapers use both the +committed and the peak token buckets. The rate of the peak bucket has to be +bigger than zero, as well as greater than or equal to the rate of the committed +bucket. + + +Congestion Management +--------------------- + +Congestion management is used to control the admission of packets into a packet +queue or group of packet queues on congestion. The congestion management +algorithms that are supported are: Tail Drop, Head Drop and Weighted Random +Early Detection (WRED). They are made available for every leaf node in the +hierarchy, subject to the specific implementation supporting them. +On request of writing a new packet into the current queue while the queue is +full, the Tail Drop algorithm drops the new packet while leaving the queue +unmodified, as opposed to the Head Drop* algorithm, which drops the packet +at the head of the queue (the oldest packet waiting in the queue) and admits +the new packet at the tail of the queue. + +The Random Early Detection (RED) algorithm works by proactively dropping more +and more input packets as the queue occupancy builds up. When the queue is full +or almost full, RED effectively works as Tail Drop. The Weighted RED (WRED) +algorithm uses a separate set of RED thresholds for each packet color and uses +separate set of RED thresholds for each packet color. + +Each hierarchy leaf node with WRED enabled as its congestion management mode +has zero or one private WRED context (only one leaf node using it) and/or zero, +one or several shared WRED contexts (multiple leaf nodes use the same WRED +context). A private WRED context is used to perform congestion management for +a single leaf node, while a shared WRED context is used to perform congestion +management for a group of leaf nodes. + +The configuration of WRED private and shared contexts is done through the +definition of WRED profiles. Any WRED profile can be used by one or several +WRED contexts (either private or shared). + + +Packet Marking +-------------- +The TM APIs have been provided to support various types of packet marking such +as VLAN DEI packet marking (IEEE 802.1Q), IPv4/IPv6 ECN marking of TCP and SCTP +packets (IETF RFC 3168) and IPv4/IPv6 DSCP packet marking (IETF RFC 2597). +All VLAN frames of a given color get their DEI bit set if marking is enabled +for this color. In case, when marking for a given color is not enabled, the +DEI bit is left as is (either set or not). + +All IPv4/IPv6 packets of a given color with ECN set to 2’b01 or 2’b10 carrying +TCP or SCTP have their ECN set to 2’b11 if the marking feature is enabled for +the current color, otherwise the ECN field is left as is. + +All IPv4/IPv6 packets have their color marked into DSCP bits 3 and 4 as +follows: green mapped to Low Drop Precedence (2’b01), yellow to Medium (2’b10) +and red to High (2’b11). Marking needs to be explicitly enabled for each color; +when not enabled for a given color, the DSCP field of all packets with that +color is left as is. + + +Steps to Setup the Hierarchy +---------------------------- + +The TM hierarchical tree consists of leaf nodes and non-leaf nodes. Each leaf +node sits on top of a scheduling queue of the current Ethernet port. Therefore, +the leaf nodes have predefined IDs in the range of 0... (N-1), where N is the +number of scheduling queues of the current Ethernet port. The non-leaf nodes +have their IDs generated by the application outside of the above range, which +is reserved for leaf nodes. + +Each non-leaf node has multiple inputs (its children nodes) and single output +(which is input to its parent node). It arbitrates its inputs using Strict +Priority (SP) and Weighted Fair Queuing (WFQ) algorithms to schedule input +packets to its output while observing its shaping (rate limiting) constraints. + +The children nodes with different priorities are scheduled using the SP +algorithm based on their priority, with 0 as the highest priority. Children +with the same priority are scheduled using the WFQ algorithm according to their +weights. The WFQ weight of a given child node is relative to the sum of the +weights of all its sibling nodes that have the same priority, with 1 as the +lowest weight. For each SP priority, the WFQ weight mode can be set as either +byte-based or packet-based. + + +Initial Hierarchy Specification +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The hierarchy is specified by incrementally adding nodes to build up the +scheduling tree. The first node that is added to the hierarchy becomes the root +node and all the nodes that are subsequently added have to be added as +descendants of the root node. The parent of the root node has to be specified +as RTE_TM_NODE_ID_NULL and there can only be one node with this parent ID +(i.e. the root node). The unique ID that is assigned to each node when the node +is created is further used to update the node configuration or to connect +children nodes to it. + +During this phase, some limited checks on the hierarchy specification can be +conducted, usually limited in scope to the current node, its parent node and +its sibling nodes. At this time, since the hierarchy is not fully defined, +there is typically no real action performed by the underlying implementation. + + +Hierarchy Commit +~~~~~~~~~~~~~~~~ + +The hierarchy commit API is called during the port initialization phase (before +the Ethernet port is started) to freeze the start-up hierarchy. This function +typically performs the following steps: + +* It validates the start-up hierarchy that was previously defined for the + current port through successive node add API invocations. +* Assuming successful validation, it performs all the necessary implementation + specific operations to install the specified hierarchy on the current port, + with immediate effect once the port is started. + +This function fails when the currently configured hierarchy is not supported by +the Ethernet port, in which case the user can abort or try out another +hierarchy configuration (e.g. a hierarchy with less leaf nodes), which can be +built from scratch or by modifying the existing hierarchy configuration. Note +that this function can still fail due to other causes (e.g. not enough memory +available in the system, etc.), even though the specified hierarchy is +supported in principle by the current port. + + +Run-Time Hierarchy Updates +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The TM API provides support for on-the-fly changes to the scheduling hierarchy, +thus operations such as node add/delete, node suspend/resume, parent node +update, etc., can be invoked after the Ethernet port has been started, subject +to the specific implementation supporting them. The set of dynamic updates +supported by the implementation is advertised through the port capability set. diff --git a/dpdk/doc/guides/prog_guide/traffic_metering_and_policing.rst b/dpdk/doc/guides/prog_guide/traffic_metering_and_policing.rst new file mode 100644 index 00000000..89f0e68b --- /dev/null +++ b/dpdk/doc/guides/prog_guide/traffic_metering_and_policing.rst @@ -0,0 +1,102 @@ +.. BSD LICENSE + Copyright(c) 2017 Intel Corporation. All rights reserved. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +Traffic Metering and Policing API +================================= + + +Overview +-------- + +This is the generic API for the Quality of Service (QoS) Traffic Metering and +Policing (MTR) of Ethernet devices. This API is agnostic of the underlying HW, +SW or mixed HW-SW implementation. + +The main features are: + +* Part of DPDK rte_ethdev API +* Capability query API +* Metering algorithms: RFC 2697 Single Rate Three Color Marker (srTCM), RFC 2698 + and RFC 4115 Two Rate Three Color Marker (trTCM) +* Policer actions (per meter output color): recolor, drop +* Statistics (per policer output color) + +Configuration steps +------------------- + +The metering and policing stage typically sits on top of flow classification, +which is why the MTR objects are enabled through a special "meter" action. + +The MTR objects are created and updated in their own name space (``rte_mtr``) +within the ``librte_ether`` library. Whether an MTR object is private to a +flow or potentially shared by several flows has to be specified at its +creation time. + +Once successfully created, an MTR object is hooked into the RX processing path +of the Ethernet device by linking it to one or several flows through the +dedicated "meter" flow action. One or several "meter" actions can be registered +for the same flow. An MTR object can only be destroyed if there are no flows +using it. + +Run-time processing +------------------- + +Traffic metering determines the color for the current packet (green, yellow, +red) based on the previous history for this flow as maintained by the MTR +object. The policer can do nothing, override the color the packet or drop the +packet. Statistics counters are maintained for MTR object, as configured. + +The processing done for each input packet hitting an MTR object is: + +* Traffic metering: The packet is assigned a color (the meter output color) + based on the previous traffic history reflected in the current state of the + MTR object, according to the specific traffic metering algorithm. The + traffic metering algorithm can typically work in color aware mode, in which + case the input packet already has an initial color (the input color), or in + color blind mode, which is equivalent to considering all input packets + initially colored as green. + +* Policing: There is a separate policer action configured for each meter + output color, which can: + + * Drop the packet. + + * Keep the same packet color: the policer output color matches the meter + output color (essentially a no-op action). + + * Recolor the packet: the policer output color is set to a different color + than the meter output color. The policer output color is the output color + of the packet, which is set in the packet meta-data (i.e. struct + ``rte_mbuf::sched::color``). + +* Statistics: The set of counters maintained for each MTR object is + configurable and subject to the implementation support. This set includes + the number of packets and bytes dropped or passed for each output color. diff --git a/dpdk/doc/guides/prog_guide/vhost_lib.rst b/dpdk/doc/guides/prog_guide/vhost_lib.rst index 6b0c6b26..e71bdbd5 100644 --- a/dpdk/doc/guides/prog_guide/vhost_lib.rst +++ b/dpdk/doc/guides/prog_guide/vhost_lib.rst @@ -46,40 +46,21 @@ vhost library should be able to: * Know all the necessary information about the vring: Information such as where the available ring is stored. Vhost defines some - messages to tell the backend all the information it needs to know how to - manipulate the vring. - -Currently, there are two ways to pass these messages and as a result there are -two Vhost implementations in DPDK: *vhost-cuse* (where the character devices -are in user space) and *vhost-user*. - -Vhost-cuse creates a user space character device and hook to a function ioctl, -so that all ioctl commands that are sent from the frontend (QEMU) will be -captured and handled. - -Vhost-user creates a Unix domain socket file through which messages are -passed. - -.. Note:: - - Since DPDK v2.2, the majority of the development effort has gone into - enhancing vhost-user, such as multiple queue, live migration, and - reconnect. Thus, it is strongly advised to use vhost-user instead of - vhost-cuse. + messages (passed through a Unix domain socket file) to tell the backend all + the information it needs to know how to manipulate the vring. Vhost API Overview ------------------ -The following is an overview of the Vhost API functions: +The following is an overview of some key Vhost API functions: * ``rte_vhost_driver_register(path, flags)`` - This function registers a vhost driver into the system. For vhost-cuse, a - ``/dev/path`` character device file will be created. For vhost-user server - mode, a Unix domain socket file ``path`` will be created. + This function registers a vhost driver into the system. ``path`` specifies + the Unix domain socket file path. - Currently two flags are supported (these are valid for vhost-user only): + Currently supported flags are: - ``RTE_VHOST_USER_CLIENT`` @@ -97,13 +78,59 @@ The following is an overview of the Vhost API functions: This reconnect option is enabled by default. However, it can be turned off by setting this flag. -* ``rte_vhost_driver_session_start()`` + - ``RTE_VHOST_USER_DEQUEUE_ZERO_COPY`` - This function starts the vhost session loop to handle vhost messages. It - starts an infinite loop, therefore it should be called in a dedicated - thread. + Dequeue zero copy will be enabled when this flag is set. It is disabled by + default. -* ``rte_vhost_driver_callback_register(virtio_net_device_ops)`` + There are some truths (including limitations) you might want to know while + setting this flag: + + * zero copy is not good for small packets (typically for packet size below + 512). + + * zero copy is really good for VM2VM case. For iperf between two VMs, the + boost could be above 70% (when TSO is enableld). + + * for VM2NIC case, the ``nb_tx_desc`` has to be small enough: <= 64 if virtio + indirect feature is not enabled and <= 128 if it is enabled. + + This is because when dequeue zero copy is enabled, guest Tx used vring will + be updated only when corresponding mbuf is freed. Thus, the nb_tx_desc + has to be small enough so that the PMD driver will run out of available + Tx descriptors and free mbufs timely. Otherwise, guest Tx vring would be + starved. + + * Guest memory should be backended with huge pages to achieve better + performance. Using 1G page size is the best. + + When dequeue zero copy is enabled, the guest phys address and host phys + address mapping has to be established. Using non-huge pages means far + more page segments. To make it simple, DPDK vhost does a linear search + of those segments, thus the fewer the segments, the quicker we will get + the mapping. NOTE: we may speed it by using tree searching in future. + + - ``RTE_VHOST_USER_IOMMU_SUPPORT`` + + IOMMU support will be enabled when this flag is set. It is disabled by + default. + + Enabling this flag makes possible to use guest vIOMMU to protect vhost + from accessing memory the virtio device isn't allowed to, when the feature + is negotiated and an IOMMU device is declared. + + However, this feature enables vhost-user's reply-ack protocol feature, + which implementation is buggy in Qemu v2.7.0-v2.9.0 when doing multiqueue. + Enabling this flag with these Qemu version results in Qemu being blocked + when multiple queue pairs are declared. + +* ``rte_vhost_driver_set_features(path, features)`` + + This function sets the feature bits the vhost-user driver supports. The + vhost-user driver could be vhost-user net, yet it could be something else, + say, vhost-user SCSI. + +* ``rte_vhost_driver_callback_register(path, vhost_device_ops)`` This function registers a set of callbacks, to let DPDK applications take the appropriate action when some events happen. The following events are @@ -111,19 +138,47 @@ The following is an overview of the Vhost API functions: * ``new_device(int vid)`` - This callback is invoked when a virtio net device becomes ready. ``vid`` - is the virtio net device ID. + This callback is invoked when a virtio device becomes ready. ``vid`` + is the vhost device ID. * ``destroy_device(int vid)`` - This callback is invoked when a virtio net device shuts down (or when the - vhost connection is broken). + This callback is invoked when a virtio device is paused or shut down. * ``vring_state_changed(int vid, uint16_t queue_id, int enable)`` This callback is invoked when a specific queue's state is changed, for example to enabled or disabled. + * ``features_changed(int vid, uint64_t features)`` + + This callback is invoked when the features is changed. For example, + ``VHOST_F_LOG_ALL`` will be set/cleared at the start/end of live + migration, respectively. + + * ``new_connection(int vid)`` + + This callback is invoked on new vhost-user socket connection. If DPDK + acts as the server the device should not be deleted before + ``destroy_connection`` callback is received. + + * ``destroy_connection(int vid)`` + + This callback is invoked when vhost-user socket connection is closed. + It indicates that device with id ``vid`` is no longer in use and can be + safely deleted. + +* ``rte_vhost_driver_disable/enable_features(path, features))`` + + This function disables/enables some features. For example, it can be used to + disable mergeable buffers and TSO features, which both are enabled by + default. + +* ``rte_vhost_driver_start(path)`` + + This function triggers the vhost-user negotiation. It should be invoked at + the end of initializing a vhost-user driver. + * ``rte_vhost_enqueue_burst(vid, queue_id, pkts, count)`` Transmits (enqueues) ``count`` packets from host to guest. @@ -132,42 +187,8 @@ The following is an overview of the Vhost API functions: Receives (dequeues) ``count`` packets from guest, and stored them at ``pkts``. -* ``rte_vhost_feature_disable/rte_vhost_feature_enable(feature_mask)`` - - This function disables/enables some features. For example, it can be used to - disable mergeable buffers and TSO features, which both are enabled by - default. - - -Vhost Implementations ---------------------- - -Vhost-cuse implementation -~~~~~~~~~~~~~~~~~~~~~~~~~ - -When vSwitch registers the vhost driver, it will register a cuse device driver -into the system and creates a character device file. This cuse driver will -receive vhost open/release/IOCTL messages from the QEMU simulator. - -When the open call is received, the vhost driver will create a vhost device -for the virtio device in the guest. - -When the ``VHOST_SET_MEM_TABLE`` ioctl is received, vhost searches the memory -region to find the starting user space virtual address that maps the memory of -the guest virtual machine. Through this virtual address and the QEMU pid, -vhost can find the file QEMU uses to map the guest memory. Vhost maps this -file into its address space, in this way vhost can fully access the guest -physical memory, which means vhost could access the shared virtio ring and the -guest physical address specified in the entry of the ring. - -The guest virtual machine tells the vhost whether the virtio device is ready -for processing or is de-activated through the ``VHOST_NET_SET_BACKEND`` -message. The registered callback from vSwitch will be called. - -When the release call is made, vhost will destroy the device. - -Vhost-user implementation -~~~~~~~~~~~~~~~~~~~~~~~~~ +Vhost-user Implementations +-------------------------- Vhost-user uses Unix domain sockets for passing messages. This means the DPDK vhost-user implementation has two options: @@ -214,8 +235,6 @@ For ``VHOST_SET_MEM_TABLE`` message, QEMU will send information for each memory region and its file descriptor in the ancillary data of the message. The file descriptor is used to map that region. -There is no ``VHOST_NET_SET_BACKEND`` message as in vhost-cuse to signal -whether the virtio device is ready or stopped. Instead, ``VHOST_SET_VRING_KICK`` is used as the signal to put the vhost device into the data plane, and ``VHOST_GET_VRING_BASE`` is used as the signal to remove the vhost device from the data plane. diff --git a/dpdk/doc/guides/prog_guide/writing_efficient_code.rst b/dpdk/doc/guides/prog_guide/writing_efficient_code.rst index 78d2afa9..d7ac6778 100644 --- a/dpdk/doc/guides/prog_guide/writing_efficient_code.rst +++ b/dpdk/doc/guides/prog_guide/writing_efficient_code.rst @@ -105,6 +105,21 @@ meaning that if all memory access operations are done on the first channel only, By default, the :ref:`Mempool Library ` spreads the addresses of objects among memory channels. +Locking memory pages +~~~~~~~~~~~~~~~~~~~~ + +The underlying operating system is allowed to load/unload memory pages at its own discretion. +These page loads could impact the performance, as the process is on hold when the kernel fetches them. + +To avoid these you could pre-load, and lock them into memory with the ``mlockall()`` call. + +.. code-block:: c + + if (mlockall(MCL_CURRENT | MCL_FUTURE)) { + RTE_LOG(NOTICE, USER1, "mlockall() failed with error \"%s\"\n", + strerror(errno)); + } + Communication Between lcores ---------------------------- @@ -124,7 +139,7 @@ The code algorithm that dequeues messages may be something similar to the follow while (1) { /* Process as many elements as can be dequeued. */ - count = rte_ring_dequeue_burst(ring, obj_table, MAX_BULK); + count = rte_ring_dequeue_burst(ring, obj_table, MAX_BULK, NULL); if (unlikely(count == 0)) continue; diff --git a/dpdk/doc/guides/rel_notes/deprecation.rst b/dpdk/doc/guides/rel_notes/deprecation.rst index d2dc4a9a..13e85432 100644 --- a/dpdk/doc/guides/rel_notes/deprecation.rst +++ b/dpdk/doc/guides/rel_notes/deprecation.rst @@ -8,64 +8,48 @@ API and ABI deprecation notices are to be posted here. Deprecation Notices ------------------- -* The log history is deprecated. - It is voided in 16.07 and will be removed in release 16.11. +* eal: several API and ABI changes are planned for ``rte_devargs`` in v18.02. + The format of device command line parameters will change. The bus will need + to be explicitly stated in the device declaration. The enum ``rte_devtype`` + was used to identify a bus and will disappear. + The structure ``rte_devargs`` will change. + The ``rte_devargs_list`` will be made private. + The following functions are deprecated starting from 17.08 and will either be + modified or removed in 18.02: -* The ethdev library file will be renamed from libethdev.* to librte_ethdev.* - in release 16.11 in order to have a more consistent namespace. + - ``rte_eal_devargs_add`` + - ``rte_eal_devargs_type_count`` + - ``rte_eal_parse_devargs_str``, replaced by ``rte_eal_devargs_parse`` -* In 16.11 ABI changes are planned: the ``rte_eth_dev`` structure will be - extended with new function pointer ``tx_pkt_prep`` allowing verification - and processing of packet burst to meet HW specific requirements before - transmit. Also new fields will be added to the ``rte_eth_desc_lim`` structure: - ``nb_seg_max`` and ``nb_mtu_seg_max`` providing information about number of - segments limit to be transmitted by device for TSO/non-TSO packets. +* pci: Several exposed functions are misnamed. + The following functions are deprecated starting from v17.11 and are replaced: -* The ethdev hotplug API is going to be moved to EAL with a notification - mechanism added to crypto and ethdev libraries so that hotplug is now - available to both of them. This API will be stripped of the device arguments - so that it only cares about hotplugging. + - ``eal_parse_pci_BDF`` replaced by ``rte_pci_addr_parse`` + - ``eal_parse_pci_DomBDF`` replaced by ``rte_pci_addr_parse`` + - ``rte_eal_compare_pci_addr`` replaced by ``rte_pci_addr_cmp`` -* Structures embodying pci and vdev devices are going to be reworked to - integrate new common rte_device / rte_driver objects (see - http://dpdk.org/ml/archives/dev/2016-January/031390.html). - ethdev and crypto libraries will then only handle those objects so that they - do not need to care about the kind of devices that are being used, making it - easier to add new buses later. +* ethdev: a new Tx and Rx offload API was introduced on 17.11. + In the new API, offloads are divided into per-port and per-queue offloads. + Offloads are disabled by default and enabled per application request. + The old offloads API is target to be deprecated on 18.05. This includes: -* ABI changes are planned for 16.11 in the ``rte_mbuf`` structure: some fields - may be reordered to facilitate the writing of ``data_off``, ``refcnt``, and - ``nb_segs`` in one operation, because some platforms have an overhead if the - store address is not naturally aligned. Other mbuf fields, such as the - ``port`` field, may be moved or removed as part of this mbuf work. + - removal of ``ETH_TXQ_FLAGS_NO*`` flags. + - removal of ``txq_flags`` field from ``rte_eth_txconf`` struct. + - removal of the offloads bit-field from ``rte_eth_rxmode`` struct. -* The mbuf flags PKT_RX_VLAN_PKT and PKT_RX_QINQ_PKT are deprecated and - are respectively replaced by PKT_RX_VLAN_STRIPPED and - PKT_RX_QINQ_STRIPPED, that are better described. The old flags and - their behavior will be kept in 16.07 and will be removed in 16.11. +* ethdev: the legacy filter API, including + ``rte_eth_dev_filter_supported()``, ``rte_eth_dev_filter_ctrl()`` as well + as filter types MACVLAN, ETHERTYPE, FLEXIBLE, SYN, NTUPLE, TUNNEL, FDIR, + HASH and L2_TUNNEL, is superseded by the generic flow API (rte_flow) in + PMDs that implement the latter. + Target release for removal of the legacy API will be defined once most + PMDs have switched to rte_flow. -* The APIs rte_mempool_count and rte_mempool_free_count are being deprecated - on the basis that they are confusing to use - free_count actually returns - the number of allocated entries, not the number of free entries as expected. - They are being replaced by rte_mempool_avail_count and - rte_mempool_in_use_count respectively. +* i40e: The default flexible payload configuration which extracts the first 16 + bytes of the payload for RSS will be deprecated starting from 18.02. If + required the previous behavior can be configured using existing flow + director APIs. There is no ABI/API break. This change will just remove a + global configuration setting and require explicit configuration. -* The mempool functions for single/multi producer/consumer are deprecated and - will be removed in 16.11. - It is replaced by rte_mempool_generic_get/put functions. - -* The ``rte_ivshmem`` feature (including library and EAL code) will be removed - in 16.11 because it has some design issues which are not planned to be fixed. - -* The vhost-cuse will be removed in 16.11. Since v2.1, a large majority of - development effort has gone to vhost-user, such as multiple-queue, live - migration, reconnect etc. Therefore, vhost-user should be used instead. - -* Driver names are quite inconsistent among each others and they will be - renamed to something more consistent (net and crypto prefixes) in 16.11. - Some of these driver names are used publicly, to create virtual devices, - so a deprecation notice is necessary. - -* API will change for ``rte_port_source_params`` and ``rte_port_sink_params`` - structures. The member ``file_name`` data type will be changed from - ``char *`` to ``const char *``. This change targets release 16.11. +* librte_meter: The API will change to accommodate configuration profiles. + Most of the API functions will have an additional opaque parameter. diff --git a/dpdk/doc/guides/rel_notes/index.rst b/dpdk/doc/guides/rel_notes/index.rst index 52c63b40..35659d8a 100644 --- a/dpdk/doc/guides/rel_notes/index.rst +++ b/dpdk/doc/guides/rel_notes/index.rst @@ -36,12 +36,16 @@ Release Notes :numbered: rel_description + release_17_11 + release_17_08 + release_17_05 + release_17_02 + release_16_11 release_16_07 release_16_04 release_2_2 release_2_1 release_2_0 release_1_8 - supported_os known_issues deprecation diff --git a/dpdk/doc/guides/rel_notes/known_issues.rst b/dpdk/doc/guides/rel_notes/known_issues.rst index 3cd42376..3f6d8cb5 100644 --- a/dpdk/doc/guides/rel_notes/known_issues.rst +++ b/dpdk/doc/guides/rel_notes/known_issues.rst @@ -640,3 +640,105 @@ I40e VF may not receive packets in the promiscuous mode **Driver/Module**: Poll Mode Driver (PMD). + + +uio pci generic module bind failed in X710/XL710/XXV710 +------------------------------------------------------- + +**Description**: + The ``uio_pci_generic`` module is not supported by XL710, since the errata of XL710 + states that the Interrupt Status bit is not implemented. The errata is the item #71 + from the `xl710 controller spec + `_. + The hw limitation is the same as other X710/XXV710 NICs. + +**Implication**: + When use ``--bind=uio_pci_generic``, the ``uio_pci_generic`` module probes device and check the Interrupt + Status bit. Since it is not supported by X710/XL710/XXV710, it return a *failed* value. The statement + that these products don’t support INTx masking, is indicated in the related `linux kernel commit + `_. + +**Resolution/Workaround**: + Do not bind the ``uio_pci_generic`` module in X710/XL710/XXV710 NICs. + +**Affected Environment/Platform**: + All. + +**Driver/Module**: + Poll Mode Driver (PMD). + + +virtio tx_burst() function cannot do TSO on shared packets +---------------------------------------------------------- + +**Description**: + The standard TX function of virtio driver does not manage shared + packets properly when doing TSO. These packets should be read-only + but the driver modifies them. + + When doing TSO, the virtio standard expects that the L4 checksum is + set to the pseudo header checksum in the packet data, which is + different than the DPDK API. The driver patches the L4 checksum to + conform to the virtio standard, but this solution is invalid when + dealing with shared packets (clones), because the packet data should + not be modified. + +**Implication**: + In this situation, the shared data will be modified by the driver, + potentially causing race conditions with the other users of the mbuf + data. + +**Resolution/Workaround**: + The workaround in the application is to ensure that the network + headers in the packet data are not shared. + +**Affected Environment/Platform**: + Virtual machines running a virtio driver. + +**Driver/Module**: + Poll Mode Driver (PMD). + + +igb uio legacy mode can not be used in X710/XL710/XXV710 +-------------------------------------------------------- + +**Description**: + X710/XL710/XXV710 NICs lack support for indicating INTx is asserted via the interrupt + bit in the PCI status register. Linux delected them from INTx support table. The related + `commit `_. + +**Implication**: + When insmod ``igb_uio`` with ``intr_mode=legacy`` and test link status interrupt. Since + INTx interrupt is not supported by X710/XL710/XXV710, it will cause Input/Output error + when reading file descriptor. + +**Resolution/Workaround**: + Do not bind ``igb_uio`` with legacy mode in X710/XL710/XXV710 NICs, or do not use kernel + version >4.7 when you bind ``igb_uio`` with legacy mode. + +**Affected Environment/Platform**: + ALL. + +**Driver/Module**: + Poll Mode Driver (PMD). + + +igb_uio can not be used when running l3fwd-power +------------------------------------------------ + +**Description**: + Link Status Change(LSC) interrupt and packet receiving interrupt are all enabled in l3fwd-power + APP. Because of UIO only support one interrupt, so these two kinds of interrupt need to share + one, and the receiving interrupt have the higher priority, so can't get the right link status. + +**Implication**: + When insmod ``igb_uio`` and running l3fwd-power APP, link status getting doesn't work properly. + +**Resolution/Workaround**: + Use vfio-pci when LSC and packet receiving interrupt enabled. + +**Affected Environment/Platform**: + ALL. + +**Driver/Module**: + ``igb_uio`` module. diff --git a/dpdk/doc/guides/rel_notes/release_16_07.rst b/dpdk/doc/guides/rel_notes/release_16_07.rst index 4d6e0d50..a8a3fc11 100644 --- a/dpdk/doc/guides/rel_notes/release_16_07.rst +++ b/dpdk/doc/guides/rel_notes/release_16_07.rst @@ -548,144 +548,3 @@ Tested OSes - Ubuntu 16.04 LTS - Wind River Linux 8 -Fixes in Stable Release ------------------------ - -16.07.1 -~~~~~~~ - -The following fixes were applied in DPDK 16.07.01 Stable Release: - -* app/test: fix verification of digest for GCM -* app/testpmd: fix crash when mempool allocation fails -* app/testpmd: fix help of MTU set commmand -* app/testpmd: fix timeout in Rx queue flushing -* contigmem: zero all pages during mmap -* crypto/null: fix key size increment value -* crypto/qat: fix FreeBSD build -* crypto: fix build with icc -* examples/ip_pipeline: fix Python interpreter -* examples/ip_pipeline: fix lcore mapping for ppc64 -* hash: fix false zero signature key hit lookup -* hash: fix ring size -* mbuf: fix error handling on pool creation -* mem: fix build with -O1 -* mem: fix crash on hugepage mapping error -* mempool: fix corruption due to invalid handler -* net/e1000: fix returned number of available Rx descriptors -* net/enic: fix bad L4 checksum flag on ICMP packets -* net/enic: fix freeing memory for descriptor ring -* net/fm10k: fix MAC address removal from switch -* net/i40e/base: fix UDP packet header -* net/i40e: fix dropping packets with ethertype 0x88A8 -* net/i40e: fix mbuf leak during Rx queue release -* net/i40e: fix null pointer dereferences when using VMDq+RSS -* net/i40e: fix parsing QinQ packets type -* net/ixgbe/base: fix check for NACK -* net/ixgbe/base: fix pointer check -* net/ixgbe/base: fix possible corruption of shadow RAM -* net/ixgbe/base: fix skipping PHY config -* net/ixgbe: fix VF reset to apply to correct VF -* net/ixgbe: fix mbuf leak during Rx queue release -* net/mlx: fix debug build with gcc 6.1 -* net/nfp: fix copying MAC address -* net/pcap: fix memory leak in jumbo frames -* net/virtio: fix xstats name -* net/virtio_user: fix error management during init -* net/virtio_user: fix first queue pair without multiqueue -* net/virtio_user: fix wrong sequence of messages -* pci: fix memory leak when detaching device -* pmdinfogen: fix clang build -* sched: fix releasing enqueued packets -* table: fix symbol exports -* timer: fix lag delay -* tools: fix json output of pmdinfo -* tools: fix virtio interface name when binding - - -16.07.2 -~~~~~~~ - -* app/procinfo: free xstats memory upon failure -* app/test: fix hash multiwriter sequence -* app/testpmd: fix DCB configuration -* app/testpmd: fix DCB configuration -* app/testpmd: fix PF/VF check of flow director -* app/testpmd: fix RSS hash key size -* app/testpmd: fix flow director endianness -* app/testpmd: fix flow director mask -* doc: add limitations for i40e PMD -* eal/arm: fix file descriptor leak when getting CPU features -* eal/ppc: fix file descriptor leak when getting CPU features -* ethdev: fix vendor id in debug message -* ethdev: prevent duplicate event callback -* examples/ip_pipeline: fix plugin loading -* examples/ipsec-secgw: check SP only when setup -* examples/l2fwd-crypto: fix verify with decrypt in chain -* examples/qos_sched: fix dequeue from ring -* examples/tep_term: fix L4 length -* examples/tep_term: fix packet length with multi-segments -* hash: fix bucket size usage -* hash: fix unlimited cuckoo path -* kni: fix build with kernel 4.8 -* kni: fix build with kernel 4.9 -* lpm: fix freeing memory -* lpm: fix freeing unused sub-table on rule delete -* mempool: fix leak if populate fails -* mempool: fix search of maximum contiguous pages -* net/bnx2x: fix build with icc -* net/bnx2x: fix maximum PF queues -* net/bnx2x: fix socket id for slowpath memory -* net/bnxt: ensure entry length is unsigned -* net/bnxt: fix bit shift size -* net/bnxt: fix crash when closing -* net/bonding: validate speed after link up -* net/ena: improve safety of string handling -* net/enic: document how to configure vNIC parameters -* net/enic: fix Rx queue index when not using Rx scatter -* net/enic: fix crash on MTU update or Rx queue reconfigure -* net/enic: fix crash with removed flow director filters -* net/enic: fix flow director -* net/enic: fix max packet length check -* net/enic: fix multi-queue Rx performance -* net/enic: revert truncated packets counter fix -* net/fm10k: fix Rx checksum flags -* net/fm10k: fix VF Tx queue initialization -* net/fm10k: fix out of order Rx read -* net/i40e: do not use VSI before NULL check -* net/i40e: fix DCB configuration -* net/i40e: fix Rx hang when disable LLDP -* net/i40e: fix VF bonded device link down -* net/i40e: fix floating VEB -* net/i40e: fix hash filter on X722 -* net/i40e: fix link status change interrupt -* net/i40e: fix out of order Rx read -* net/i40e: fixed build error with icc -* net/ixgbe: fix VF registers -* net/ixgbe: fix flow director mask -* net/ixgbe: fix out of order Rx read -* net/mlx5: fix Rx VLAN offload capability report -* net/mlx5: fix Rx checksum macros -* net/mlx5: fix Rx function selection -* net/mlx5: fix flow director drop mode -* net/mlx5: fix handling of small mbuf sizes -* net/mlx5: fix hash key size retrieval -* net/mlx5: fix inconsistent return value in flow director -* net/mlx5: fix initialization in secondary process -* net/mlx5: fix inline logic -* net/mlx5: fix link speed capability information -* net/mlx5: fix link status report -* net/mlx5: fix possible NULL dereference in Rx path -* net/mlx5: fix removing VLAN filter -* net/mlx5: fix support for newer link speeds -* net/mlx5: re-factorize functions -* net/mlx5: refactor allocation of flow director queues -* net/mlx5: support Mellanox OFED 3.4 -* net/qede/base: fix 32-bit build -* net/ring: fix ring device creation via devargs -* net/thunderx: fix Tx checksum handling -* net/virtio: revert fix restart -* net/vmxnet3: fix mbuf release on reset/stop -* pci: fix probing error if no driver found -* pdump: fix created directory permissions -* vhost: fix Windows VM hang diff --git a/dpdk/doc/guides/rel_notes/release_16_11.rst b/dpdk/doc/guides/rel_notes/release_16_11.rst new file mode 100644 index 00000000..8c9ec65c --- /dev/null +++ b/dpdk/doc/guides/rel_notes/release_16_11.rst @@ -0,0 +1,600 @@ +DPDK Release 16.11 +================== + +.. **Read this first.** + + The text below explains how to update the release notes. + + Use proper spelling, capitalization and punctuation in all sections. + + Variable and config names should be quoted as fixed width text: ``LIKE_THIS``. + + Build the docs and view the output file to ensure the changes are correct:: + + make doc-guides-html + + firefox build/doc/html/guides/rel_notes/release_16_11.html + + +New Features +------------ + +.. This section should contain new features added in this release. Sample format: + + * **Add a title in the past tense with a full stop.** + + Add a short 1-2 sentence description in the past tense. The description + should be enough to allow someone scanning the release notes to understand + the new feature. + + If the feature adds a lot of sub-features you can use a bullet list like this. + + * Added feature foo to do something. + * Enhanced feature bar to do something else. + + Refer to the previous release notes for examples. + + This section is a comment. Make sure to start the actual text at the margin. + + +* **Added software parser for packet type.** + + * Added a new function ``rte_pktmbuf_read()`` to read the packet data from an + mbuf chain, linearizing if required. + * Added a new function ``rte_net_get_ptype()`` to parse an Ethernet packet + in an mbuf chain and retrieve its packet type from software. + * Added new functions ``rte_get_ptype_*()`` to dump a packet type as a string. + +* **Improved offloads support in mbuf.** + + * Added a new function ``rte_raw_cksum_mbuf()`` to process the checksum of + data embedded in an mbuf chain. + * Added new Rx checksum flags in mbufs to describe more states: unknown, + good, bad, or not present (useful for virtual drivers). This modification + was done for IP and L4. + * Added a new Rx LRO mbuf flag, used when packets are coalesced. This + flag indicates that the segment size of original packets is known. + +* **Added vhost-user dequeue zero copy support.** + + The copy in the dequeue path is avoided in order to improve the performance. + In the VM2VM case, the boost is quite impressive. The bigger the packet size, + the bigger performance boost you may get. However, for the VM2NIC case, there + are some limitations, so the boost is not as impressive as the VM2VM case. + It may even drop quite a bit for small packets. + + For that reason, this feature is disabled by default. It can be enabled when + the ``RTE_VHOST_USER_DEQUEUE_ZERO_COPY`` flag is set. Check the VHost section + of the Programming Guide for more information. + +* **Added vhost-user indirect descriptors support.** + + If the indirect descriptor feature is enabled, each packet sent by the guest + will take exactly one slot in the enqueue virtqueue. Without this feature, as in + the current version, even 64 bytes packets take two slots with Virtio PMD on guest + side. + + The main impact is better performance for 0% packet loss use-cases, as it + behaves as if the virtqueue size was enlarged, so more packets can be buffered + in the case of system perturbations. On the downside, small performance degradations + were measured when running micro-benchmarks. + +* **Added vhost PMD xstats.** + + Added extended statistics to vhost PMD from a per port perspective. + +* **Supported offloads with virtio.** + + Added support for the following offloads in virtio: + + * Rx/Tx checksums. + * LRO. + * TSO. + +* **Added virtio NEON support for ARM.** + + Added NEON support for ARM based virtio. + +* **Updated the ixgbe base driver.** + + Updated the ixgbe base driver, including the following changes: + + * Added X550em_a 10G PHY support. + * Added support for flow control auto negotiation for X550em_a 1G PHY. + * Added X550em_a FW ALEF support. + * Increased mailbox version to ``ixgbe_mbox_api_13``. + * Added two MAC operations for Hyper-V support. + +* **Added APIs for VF management to the ixgbe PMD.** + + Eight new APIs have been added to the ixgbe PMD for VF management from the PF. + The declarations for the API's can be found in ``rte_pmd_ixgbe.h``. + +* **Updated the enic driver.** + + * Added update to use interrupt for link status checking instead of polling. + * Added more flow director modes on UCS Blade with firmware version >= 2.0(13e). + * Added full support for MTU update. + * Added support for the ``rte_eth_rx_queue_count`` function. + +* **Updated the mlx5 driver.** + + * Added support for RSS hash results. + * Added several performance improvements. + * Added several bug fixes. + +* **Updated the QAT PMD.** + + The QAT PMD was updated with additional support for: + + * MD5_HMAC algorithm. + * SHA224-HMAC algorithm. + * SHA384-HMAC algorithm. + * GMAC algorithm. + * KASUMI (F8 and F9) algorithm. + * 3DES algorithm. + * NULL algorithm. + * C3XXX device. + * C62XX device. + +* **Added openssl PMD.** + + A new crypto PMD has been added, which provides several ciphering and hashing algorithms. + All cryptography operations use the Openssl library crypto API. + +* **Updated the IPsec example.** + + Updated the IPsec example with the following support: + + * Configuration file support. + * AES CBC IV generation with cipher forward function. + * AES GCM/CTR mode. + +* **Added support for new gcc -march option.** + + The GCC 4.9 ``-march`` option supports the Intel processor code names. + The config option ``RTE_MACHINE`` can be used to pass code names to the compiler via the ``-march`` flag. + + +Resolved Issues +--------------- + +.. This section should contain bug fixes added to the relevant sections. Sample format: + + * **code/section Fixed issue in the past tense with a full stop.** + + Add a short 1-2 sentence description of the resolved issue in the past tense. + The title should contain the code/lib section like a commit message. + Add the entries in alphabetic order in the relevant sections below. + + This section is a comment. Make sure to start the actual text at the margin. + + +Drivers +~~~~~~~ + +* **enic: Fixed several flow director issues.** + +* **enic: Fixed inadvertent setting of L4 checksum ptype on ICMP packets.** + +* **enic: Fixed high driver overhead when servicing Rx queues beyond the first.** + + + +Known Issues +------------ + +.. This section should contain new known issues in this release. Sample format: + + * **Add title in present tense with full stop.** + + Add a short 1-2 sentence description of the known issue in the present + tense. Add information on any known workarounds. + + This section is a comment. Make sure to start the actual text at the margin. + +* **L3fwd-power app does not work properly when Rx vector is enabled.** + + The L3fwd-power app doesn't work properly with some drivers in vector mode + since the queue monitoring works differently between scalar and vector modes + leading to incorrect frequency scaling. In addition, L3fwd-power application + requires the mbuf to have correct packet type set but in some drivers the + vector mode must be disabled for this. + + Therefore, in order to use L3fwd-power, vector mode should be disabled + via the config file. + +* **Digest address must be supplied for crypto auth operation on QAT PMD.** + + The cryptodev API specifies that if the rte_crypto_sym_op.digest.data field, + and by inference the digest.phys_addr field which points to the same location, + is not set for an auth operation the driver is to understand that the digest + result is located immediately following the region over which the digest is + computed. The QAT PMD doesn't correctly handle this case and reads and writes + to an incorrect location. + + Callers can workaround this by always supplying the digest virtual and + physical address fields in the rte_crypto_sym_op for an auth operation. + + +API Changes +----------- + +.. This section should contain API changes. Sample format: + + * Add a short 1-2 sentence description of the API change. Use fixed width + quotes for ``rte_function_names`` or ``rte_struct_names``. Use the past tense. + + This section is a comment. Make sure to start the actual text at the margin. + +* The driver naming convention has been changed to make them more + consistent. It especially impacts ``--vdev`` arguments. For example + ``eth_pcap`` becomes ``net_pcap`` and ``cryptodev_aesni_mb_pmd`` becomes + ``crypto_aesni_mb``. + + For backward compatibility an alias feature has been enabled to support the + original names. + +* The log history has been removed. + +* The ``rte_ivshmem`` feature (including library and EAL code) has been removed + in 16.11 because it had some design issues which were not planned to be fixed. + +* The ``file_name`` data type of ``struct rte_port_source_params`` and + ``struct rte_port_sink_params`` is changed from ``char *`` to ``const char *``. + +* **Improved device/driver hierarchy and generalized hotplugging.** + + The device and driver relationship has been restructured by introducing generic + classes. This paves the way for having PCI, VDEV and other device types as + instantiated objects rather than classes in themselves. Hotplugging has also + been generalized into EAL so that Ethernet or crypto devices can use the + common infrastructure. + + * Removed ``pmd_type`` as a way of segregation of devices. + * Moved ``numa_node`` and ``devargs`` into ``rte_driver`` from + ``rte_pci_driver``. These can now be used by any instantiated object of + ``rte_driver``. + * Added ``rte_device`` class and all PCI and VDEV devices inherit from it + * Renamed devinit/devuninit handlers to probe/remove to make it more + semantically correct with respect to the device <=> driver relationship. + * Moved hotplugging support to EAL. Hereafter, PCI and vdev can use the + APIs ``rte_eal_dev_attach`` and ``rte_eal_dev_detach``. + * Renamed helpers and support macros to make them more synonymous + with their device types + (e.g. ``PMD_REGISTER_DRIVER`` => ``RTE_PMD_REGISTER_PCI``). + * Device naming functions have been generalized from ethdev and cryptodev + to EAL. ``rte_eal_pci_device_name`` has been introduced for obtaining + unique device name from PCI Domain-BDF description. + * Virtual device registration APIs have been added: ``rte_eal_vdrv_register`` + and ``rte_eal_vdrv_unregister``. + + +ABI Changes +----------- + +.. This section should contain ABI changes. Sample format: + + * Add a short 1-2 sentence description of the ABI change that was announced in + the previous releases and made in this release. Use fixed width quotes for + ``rte_function_names`` or ``rte_struct_names``. Use the past tense. + + This section is a comment. Make sure to start the actual text at the margin. + + + +Shared Library Versions +----------------------- + +.. Update any library version updated in this release and prepend with a ``+`` + sign, like this: + + libethdev.so.4 + librte_acl.so.2 + + librte_cfgfile.so.2 + librte_cmdline.so.2 + + + +The libraries prepended with a plus sign were incremented in this version. + +.. code-block:: diff + + librte_acl.so.2 + librte_cfgfile.so.2 + librte_cmdline.so.2 + + librte_cryptodev.so.2 + librte_distributor.so.1 + + librte_eal.so.3 + + librte_ethdev.so.5 + librte_hash.so.2 + librte_ip_frag.so.1 + librte_jobstats.so.1 + librte_kni.so.2 + librte_kvargs.so.1 + librte_lpm.so.2 + librte_mbuf.so.2 + librte_mempool.so.2 + librte_meter.so.1 + librte_net.so.1 + librte_pdump.so.1 + librte_pipeline.so.3 + librte_pmd_bond.so.1 + librte_pmd_ring.so.2 + librte_port.so.3 + librte_power.so.1 + librte_reorder.so.1 + librte_ring.so.1 + librte_sched.so.1 + librte_table.so.2 + librte_timer.so.1 + librte_vhost.so.3 + + +Tested Platforms +---------------- + +.. This section should contain a list of platforms that were tested with this release. + + The format is: + + #. Platform name. + + * Platform details. + * Platform details. + + This section is a comment. Make sure to start the actual text at the margin. + +#. SuperMicro 1U + + - BIOS: 1.0c + - Processor: Intel(R) Atom(TM) CPU C2758 @ 2.40GHz + +#. SuperMicro 1U + + - BIOS: 1.0a + - Processor: Intel(R) Xeon(R) CPU D-1540 @ 2.00GHz + - Onboard NIC: Intel(R) X552/X557-AT (2x10G) + + - Firmware-version: 0x800001cf + - Device ID (PF/VF): 8086:15ad /8086:15a8 + + - kernel driver version: 4.2.5 (ixgbe) + +#. SuperMicro 2U + + - BIOS: 1.0a + - Processor: Intel(R) Xeon(R) CPU E5-4667 v3 @ 2.00GHz + +#. Intel(R) Server board S2600GZ + + - BIOS: SE5C600.86B.02.02.0002.122320131210 + - Processor: Intel(R) Xeon(R) CPU E5-2680 v2 @ 2.80GHz + +#. Intel(R) Server board W2600CR + + - BIOS: SE5C600.86B.02.01.0002.082220131453 + - Processor: Intel(R) Xeon(R) CPU E5-2680 v2 @ 2.80GHz + +#. Intel(R) Server board S2600CWT + + - BIOS: SE5C610.86B.01.01.0009.060120151350 + - Processor: Intel(R) Xeon(R) CPU E5-2699 v3 @ 2.30GHz + +#. Intel(R) Server board S2600WTT + + - BIOS: SE5C610.86B.01.01.0005.101720141054 + - Processor: Intel(R) Xeon(R) CPU E5-2699 v3 @ 2.30GHz + +#. Intel(R) Server board S2600WTT + + - BIOS: SE5C610.86B.11.01.0044.090120151156 + - Processor: Intel(R) Xeon(R) CPU E5-2695 v4 @ 2.10GHz + +#. Intel(R) Server board S2600WTT + + - Processor: Intel(R) Xeon(R) CPU E5-2697 v2 @ 2.70GHz + +#. Intel(R) Server + + - Intel(R) Xeon(R) CPU E5-2697 v3 @ 2.60GHz + +#. IBM(R) Power8(R) + + - Machine type-model: 8247-22L + - Firmware FW810.21 (SV810_108) + - Processor: POWER8E (raw), AltiVec supported + + +Tested NICs +----------- + +.. This section should contain a list of NICs that were tested with this release. + + The format is: + + #. NIC name. + + * NIC details. + * NIC details. + + This section is a comment. Make sure to start the actual text at the margin. + +#. Intel(R) Ethernet Controller X540-AT2 + + - Firmware version: 0x80000389 + - Device id (pf): 8086:1528 + - Driver version: 3.23.2 (ixgbe) + +#. Intel(R) 82599ES 10 Gigabit Ethernet Controller + + - Firmware version: 0x61bf0001 + - Device id (pf/vf): 8086:10fb / 8086:10ed + - Driver version: 4.0.1-k (ixgbe) + +#. Intel(R) Corporation Ethernet Connection X552/X557-AT 10GBASE-T + + - Firmware version: 0x800001cf + - Device id (pf/vf): 8086:15ad / 8086:15a8 + - Driver version: 4.2.5 (ixgbe) + +#. Intel(R) Ethernet Converged Network Adapter X710-DA4 (4x10G) + + - Firmware version: 5.05 + - Device id (pf/vf): 8086:1572 / 8086:154c + - Driver version: 1.5.23 (i40e) + +#. Intel(R) Ethernet Converged Network Adapter X710-DA2 (2x10G) + + - Firmware version: 5.05 + - Device id (pf/vf): 8086:1572 / 8086:154c + - Driver version: 1.5.23 (i40e) + +#. Intel(R) Ethernet Converged Network Adapter XL710-QDA1 (1x40G) + + - Firmware version: 5.05 + - Device id (pf/vf): 8086:1584 / 8086:154c + - Driver version: 1.5.23 (i40e) + +#. Intel(R) Ethernet Converged Network Adapter XL710-QDA2 (2X40G) + + - Firmware version: 5.05 + - Device id (pf/vf): 8086:1583 / 8086:154c + - Driver version: 1.5.23 (i40e) + +#. Intel(R) Corporation I350 Gigabit Network Connection + + - Firmware version: 1.48, 0x800006e7 + - Device id (pf/vf): 8086:1521 / 8086:1520 + - Driver version: 5.2.13-k (igb) + +#. Intel(R) Ethernet Multi-host Controller FM10000 + + - Firmware version: N/A + - Device id (pf/vf): 8086:15d0 + - Driver version: 0.17.0.9 (fm10k) + +#. Mellanox(R) ConnectX(R)-4 10G MCX4111A-XCAT (1x10G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1013 + * MLNX_OFED: 3.4-1.0.0.0 + * Firmware version: 12.17.1010 + +#. Mellanox(R) ConnectX(R)-4 10G MCX4121A-XCAT (2x10G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1013 + * MLNX_OFED: 3.4-1.0.0.0 + * Firmware version: 12.17.1010 + +#. Mellanox(R) ConnectX(R)-4 25G MCX4111A-ACAT (1x25G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1013 + * MLNX_OFED: 3.4-1.0.0.0 + * Firmware version: 12.17.1010 + +#. Mellanox(R) ConnectX(R)-4 25G MCX4121A-ACAT (2x25G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1013 + * MLNX_OFED: 3.4-1.0.0.0 + * Firmware version: 12.17.1010 + +#. Mellanox(R) ConnectX(R)-4 40G MCX4131A-BCAT/MCX413A-BCAT (1x40G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1013 + * MLNX_OFED: 3.4-1.0.0.0 + * Firmware version: 12.17.1010 + +#. Mellanox(R) ConnectX(R)-4 40G MCX415A-BCAT (1x40G) + + * Host interface: PCI Express 3.0 x16 + * Device ID: 15b3:1013 + * MLNX_OFED: 3.4-1.0.0.0 + * Firmware version: 12.17.1010 + +#. Mellanox(R) ConnectX(R)-4 50G MCX4131A-GCAT/MCX413A-GCAT (1x50G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1013 + * MLNX_OFED: 3.4-1.0.0.0 + * Firmware version: 12.17.1010 + +#. Mellanox(R) ConnectX(R)-4 50G MCX414A-BCAT (2x50G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1013 + * MLNX_OFED: 3.4-1.0.0.0 + * Firmware version: 12.17.1010 + +#. Mellanox(R) ConnectX(R)-4 50G MCX415A-GCAT/MCX416A-BCAT/MCX416A-GCAT (2x50G) + + * Host interface: PCI Express 3.0 x16 + * Device ID: 15b3:1013 + * MLNX_OFED: 3.4-1.0.0.0 + * Firmware version: 12.17.1010 + +#. Mellanox(R) ConnectX(R)-4 50G MCX415A-CCAT (1x100G) + + * Host interface: PCI Express 3.0 x16 + * Device ID: 15b3:1013 + * MLNX_OFED: 3.4-1.0.0.0 + * Firmware version: 12.17.1010 + +#. Mellanox(R) ConnectX(R)-4 100G MCX416A-CCAT (2x100G) + + * Host interface: PCI Express 3.0 x16 + * Device ID: 15b3:1013 + * MLNX_OFED: 3.4-1.0.0.0 + * Firmware version: 12.17.1010 + +#. Mellanox(R) ConnectX(R)-4 Lx 10G MCX4121A-XCAT (2x10G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1015 + * MLNX_OFED: 3.4-1.0.0.0 + * Firmware version: 14.17.1010 + +#. Mellanox(R) ConnectX(R)-4 Lx 25G MCX4121A-ACAT (2x25G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1015 + * MLNX_OFED: 3.4-1.0.0.0 + * Firmware version: 14.17.1010 + + +Tested OSes +----------- + +.. This section should contain a list of OSes that were tested with this release. + The format is as follows, in alphabetical order: + + * CentOS 7.0 + * Fedora 23 + * Fedora 24 + * FreeBSD 10.3 + * Red Hat Enterprise Linux 7.2 + * SUSE Enterprise Linux 12 + * Ubuntu 15.10 + * Ubuntu 16.04 LTS + * Wind River Linux 8 + + This section is a comment. Make sure to start the actual text at the margin. + +* CentOS 7.2 +* Fedora 23 +* Fedora 24 +* FreeBSD 10.3 +* FreeBSD 11 +* Red Hat Enterprise Linux Server release 6.7 (Santiago) +* Red Hat Enterprise Linux Server release 7.0 (Maipo) +* Red Hat Enterprise Linux Server release 7.2 (Maipo) +* SUSE Enterprise Linux 12 +* Wind River Linux 6.0.0.26 +* Wind River Linux 8 +* Ubuntu 14.04 +* Ubuntu 15.04 +* Ubuntu 16.04 diff --git a/dpdk/doc/guides/rel_notes/release_17_02.rst b/dpdk/doc/guides/rel_notes/release_17_02.rst new file mode 100644 index 00000000..357965ac --- /dev/null +++ b/dpdk/doc/guides/rel_notes/release_17_02.rst @@ -0,0 +1,692 @@ +DPDK Release 17.02 +================== + +.. **Read this first.** + + The text below explains how to update the release notes. + + Use proper spelling, capitalization and punctuation in all sections. + + Variable and config names should be quoted as fixed width text: ``LIKE_THIS``. + + Build the docs and view the output file to ensure the changes are correct:: + + make doc-guides-html + + firefox build/doc/html/guides/rel_notes/release_17_02.html + + +New Features +------------ + +.. This section should contain new features added in this release. Sample format: + + * **Add a title in the past tense with a full stop.** + + Add a short 1-2 sentence description in the past tense. The description + should be enough to allow someone scanning the release notes to understand + the new feature. + + If the feature adds a lot of sub-features you can use a bullet list like this. + + * Added feature foo to do something. + * Enhanced feature bar to do something else. + + Refer to the previous release notes for examples. + + This section is a comment. do not overwrite or remove it. + Also, make sure to start the actual text at the margin. + ========================================================= + +* **Added support for representing buses in EAL** + + The ``rte_bus`` structure was introduced into the EAL. This allows for + devices to be represented by buses they are connected to. A new bus can be + added to DPDK by extending the ``rte_bus`` structure and implementing the + scan and probe functions. Once a new bus is registered using the provided + APIs, new devices can be detected and initialized using bus scan and probe + callbacks. + + With this change, devices other than PCI or VDEV type can be represented + in the DPDK framework. + +* **Added generic EAL API for I/O device memory read/write operations.** + + This API introduces 8 bit, 16 bit, 32 bit and 64 bit I/O device + memory read/write operations along with "relaxed" versions. + + Weakly-ordered architectures like ARM need an additional I/O barrier for + device memory read/write access over PCI bus. By introducing the EAL + abstraction for I/O device memory read/write access, the drivers can access + I/O device memory in an architecture-agnostic manner. The relaxed version + does not have an additional I/O memory barrier, which is useful in accessing + the device registers of integrated controllers which is implicitly strongly + ordered with respect to memory access. + +* **Added generic flow API (rte_flow).** + + This API provides a generic means to configure hardware to match specific + ingress or egress traffic, alter its behavior and query related counters + according to any number of user-defined rules. + + In order to expose a single interface with an unambiguous behavior that is + common to all poll-mode drivers (PMDs) the ``rte_flow`` API is slightly + higher-level than the legacy filtering framework, which it encompasses and + supersedes (including all functions and filter types) . + + See the :ref:`Generic flow API ` documentation for more + information. + +* **Added firmware version get API.** + + Added a new function ``rte_eth_dev_fw_version_get()`` to fetch the firmware + version for a given device. + +* **Added APIs for MACsec offload support to the ixgbe PMD.** + + Six new APIs have been added to the ixgbe PMD for MACsec offload support. + The declarations for the APIs can be found in ``rte_pmd_ixgbe.h``. + +* **Added I219 NICs support.** + + Added support for I219 Intel 1GbE NICs. + +* **Added VF Daemon (VFD) for i40e. - EXPERIMENTAL** + + This is an EXPERIMENTAL feature to enhance the capability of the DPDK PF as + many VF management features are not currently supported by the kernel PF + driver. Some new private APIs are implemented directly in the PMD without an + abstraction layer. They can be used directly by some users who have the + need. + + The new APIs to control VFs directly from PF include: + + * Set VF MAC anti-spoofing. + * Set VF VLAN anti-spoofing. + * Set TX loopback. + * Set VF unicast promiscuous mode. + * Set VF multicast promiscuous mode. + * Set VF MTU. + * Get/reset VF stats. + * Set VF MAC address. + * Set VF VLAN stripping. + * Vf VLAN insertion. + * Set VF broadcast mode. + * Set VF VLAN tag. + * Set VF VLAN filter. + + VFD also includes VF to PF mailbox message management from an application. + When the PF receives mailbox messages from the VF the PF should call the + callback provided by the application to know if they're permitted to be + processed. + + As an EXPERIMENTAL feature, please be aware it can be changed or even + removed without prior notice. + +* **Updated the i40e base driver.** + + Updated the i40e base driver, including the following changes: + + * Replace existing legacy ``memcpy()`` calls with ``i40e_memcpy()`` calls. + * Use ``BIT()`` macro instead of bit fields. + * Add clear all WoL filters implementation. + * Add broadcast promiscuous control per VLAN. + * Remove unused ``X722_SUPPORT`` and ``I40E_NDIS_SUPPORT`` macros. + +* **Updated the enic driver.** + + * Set new Rx checksum flags in mbufs to indicate unknown, good or bad checksums. + * Fix set/remove of MAC addresses. Allow up to 64 addresses per device. + * Enable TSO on outer headers. + +* **Added Solarflare libefx-based network PMD.** + + Added a new network PMD which supports Solarflare SFN7xxx and SFN8xxx family + of 10/40 Gbps adapters. + +* **Updated the mlx4 driver.** + + * Addressed a few bugs. + +* **Added support for Mellanox ConnectX-5 adapters (mlx5).** + + Added support for Mellanox ConnectX-5 family of 10/25/40/50/100 Gbps + adapters to the existing mlx5 PMD. + +* **Updated the mlx5 driver.** + + * Improve Tx performance by using vector logic. + * Improve RSS balancing when number of queues is not a power of two. + * Generic flow API support for Ethernet, IPv4, IPv4, UDP, TCP, VLAN and + VXLAN pattern items with DROP and QUEUE actions. + * Support for extended statistics. + * Addressed several data path bugs. + * As of MLNX_OFED 4.0-1.0.1.0, the Toeplitz RSS hash function is not + symmetric anymore for consistency with other PMDs. + +* **virtio-user with vhost-kernel as another exceptional path.** + + Previously, we upstreamed a virtual device, virtio-user with vhost-user as + the backend as a way of enabling IPC (Inter-Process Communication) and user + space container networking. + + Virtio-user with vhost-kernel as the backend is a solution for the exception + path, such as KNI, which exchanges packets with the kernel networking stack. + This solution is very promising in: + + * Maintenance: vhost and vhost-net (kernel) is an upstreamed and extensively + used kernel module. + * Features: vhost-net is designed to be a networking solution, which has + lots of networking related features, like multi-queue, TSO, multi-seg + mbuf, etc. + * Performance: similar to KNI, this solution would use one or more + kthreads to send/receive packets from user space DPDK applications, + which has little impact on user space polling thread (except that + it might enter into kernel space to wake up those kthreads if + necessary). + +* **Added virtio Rx interrupt support.** + + Added a feature to enable Rx interrupt mode for virtio pci net devices as + bound to VFIO (noiommu mode) and driven by virtio PMD. + + With this feature, the virtio PMD can switch between polling mode and + interrupt mode, to achieve best performance, and at the same time save + power. It can work on both legacy and modern virtio devices. In this mode, + each ``rxq`` is mapped with an excluded MSIx interrupt. + + See the :ref:`Virtio Interrupt Mode ` documentation + for more information. + +* **Added ARMv8 crypto PMD.** + + A new crypto PMD has been added, which provides combined mode cryptographic + operations optimized for ARMv8 processors. The driver can be used to enhance + performance in processing chained operations such as cipher + HMAC. + +* **Updated the QAT PMD.** + + The QAT PMD has been updated with additional support for: + + * DES algorithm. + * Scatter-gather list (SGL) support. + +* **Updated the AESNI MB PMD.** + + * The Intel(R) Multi Buffer Crypto for IPsec library used in + AESNI MB PMD has been moved to a new repository, in GitHub. + * Support has been added for single operations (cipher only and + authentication only). + +* **Updated the AES-NI GCM PMD.** + + The AES-NI GCM PMD was migrated from the Multi Buffer library to the ISA-L + library. The migration entailed adding additional support for: + + * GMAC algorithm. + * 256-bit cipher key. + * Session-less mode. + * Out-of place processing + * Scatter-gather support for chained mbufs (only out-of place and destination + mbuf must be contiguous) + +* **Added crypto performance test application.** + + Added a new performance test application for measuring performance + parameters of PMDs available in the crypto tree. + +* **Added Elastic Flow Distributor library (rte_efd).** + + Added a new library which uses perfect hashing to determine a target/value + for a given incoming flow key. + + The library does not store the key itself for lookup operations, and + therefore, lookup performance is not dependent on the key size. Also, the + target/value can be any arbitrary value (8 bits by default). Finally, the + storage requirement is much smaller than a hash-based flow table and + therefore, it can better fit in CPU cache and scale to millions of flow + keys. + + See the :ref:`Elastic Flow Distributor Library ` documentation in + the Programmers Guide document, for more information. + + +Resolved Issues +--------------- + +.. This section should contain bug fixes added to the relevant sections. Sample format: + + * **code/section Fixed issue in the past tense with a full stop.** + + Add a short 1-2 sentence description of the resolved issue in the past tense. + The title should contain the code/lib section like a commit message. + Add the entries in alphabetic order in the relevant sections below. + + This section is a comment. do not overwrite or remove it. + Also, make sure to start the actual text at the margin. + ========================================================= + +Drivers +~~~~~~~ + +* **net/virtio: Fixed multiple process support.** + + Fixed a few regressions introduced in recent releases that break the virtio + multiple process support. + + +Examples +~~~~~~~~ + +* **examples/ethtool: Fixed crash with non-PCI devices.** + + Fixed issue where querying a non-PCI device was dereferencing non-existent + PCI data resulting in a segmentation fault. + + + +API Changes +----------- + +.. This section should contain API changes. Sample format: + + * Add a short 1-2 sentence description of the API change. Use fixed width + quotes for ``rte_function_names`` or ``rte_struct_names``. Use the past tense. + + This section is a comment. do not overwrite or remove it. + Also, make sure to start the actual text at the margin. + ========================================================= + +* **Moved five APIs for VF management from the ethdev to the ixgbe PMD.** + + The following five APIs for VF management from the PF have been removed from + the ethdev, renamed, and added to the ixgbe PMD:: + + rte_eth_dev_set_vf_rate_limit() + rte_eth_dev_set_vf_rx() + rte_eth_dev_set_vf_rxmode() + rte_eth_dev_set_vf_tx() + rte_eth_dev_set_vf_vlan_filter() + + The API's have been renamed to the following:: + + rte_pmd_ixgbe_set_vf_rate_limit() + rte_pmd_ixgbe_set_vf_rx() + rte_pmd_ixgbe_set_vf_rxmode() + rte_pmd_ixgbe_set_vf_tx() + rte_pmd_ixgbe_set_vf_vlan_filter() + + The declarations for the API’s can be found in ``rte_pmd_ixgbe.h``. + + +ABI Changes +----------- + +.. This section should contain ABI changes. Sample format: + + * Add a short 1-2 sentence description of the ABI change that was announced in + the previous releases and made in this release. Use fixed width quotes for + ``rte_function_names`` or ``rte_struct_names``. Use the past tense. + + This section is a comment. do not overwrite or remove it. + Also, make sure to start the actual text at the margin. + ========================================================= + + + +Shared Library Versions +----------------------- + +.. Update any library version updated in this release and prepend with a ``+`` + sign, like this: + + librte_acl.so.2 + + librte_cfgfile.so.2 + librte_cmdline.so.2 + + This section is a comment. do not overwrite or remove it. + ========================================================= + + +The libraries prepended with a plus sign were incremented in this version. + +.. code-block:: diff + + librte_acl.so.2 + librte_cfgfile.so.2 + librte_cmdline.so.2 + librte_cryptodev.so.2 + librte_distributor.so.1 + librte_eal.so.3 + + librte_ethdev.so.6 + librte_hash.so.2 + librte_ip_frag.so.1 + librte_jobstats.so.1 + librte_kni.so.2 + librte_kvargs.so.1 + librte_lpm.so.2 + librte_mbuf.so.2 + librte_mempool.so.2 + librte_meter.so.1 + librte_net.so.1 + librte_pdump.so.1 + librte_pipeline.so.3 + librte_pmd_bond.so.1 + librte_pmd_ring.so.2 + librte_port.so.3 + librte_power.so.1 + librte_reorder.so.1 + librte_ring.so.1 + librte_sched.so.1 + librte_table.so.2 + librte_timer.so.1 + librte_vhost.so.3 + + +Tested Platforms +---------------- + +.. This section should contain a list of platforms that were tested with this release. + + The format is: + + * platform with combinations + + * List of CPU + * List of OS + * List of devices + * Other relevant details... + + This section is a comment. do not overwrite or remove it. + Also, make sure to start the actual text at the margin. + ========================================================= + +This release has been tested with the below list of CPU/device/firmware/OS. +Each section describes a different set of combinations. + +* Intel(R) platforms with Mellanox(R) NICs combinations + + * Platform details + + * Intel(R) Xeon(R) CPU E5-2697 v2 @ 2.70GHz + * Intel(R) Xeon(R) CPU E5-2680 v2 @ 2.80GHz + * Intel(R) Xeon(R) CPU E5-2697 v3 @ 2.60GHz + + * OS: + + * CentOS 7.0 + * Fedora 23 + * Fedora 24 + * FreeBSD 10.3 + * Red Hat Enterprise Linux 7.2 + * SUSE Enterprise Linux 12 + * Ubuntu 14.04 LTS + * Ubuntu 15.10 + * Ubuntu 16.04 LTS + * Wind River Linux 8 + + * MLNX_OFED: 4.0-1.0.1.0 + + * NICs: + + * Mellanox(R) ConnectX(R)-3 Pro 40G MCX354A-FCC_Ax (2x40G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1007 + * Firmware version: 2.40.5030 + + * Mellanox(R) ConnectX(R)-4 10G MCX4111A-XCAT (1x10G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1013 + * Firmware version: 12.18.1000 + + * Mellanox(R) ConnectX(R)-4 10G MCX4121A-XCAT (2x10G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1013 + * Firmware version: 12.18.1000 + + * Mellanox(R) ConnectX(R)-4 25G MCX4111A-ACAT (1x25G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1013 + * Firmware version: 12.18.1000 + + * Mellanox(R) ConnectX(R)-4 25G MCX4121A-ACAT (2x25G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1013 + * Firmware version: 12.18.1000 + + * Mellanox(R) ConnectX(R)-4 40G MCX4131A-BCAT/MCX413A-BCAT (1x40G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1013 + * Firmware version: 12.18.1000 + + * Mellanox(R) ConnectX(R)-4 40G MCX415A-BCAT (1x40G) + + * Host interface: PCI Express 3.0 x16 + * Device ID: 15b3:1013 + * Firmware version: 12.18.1000 + + * Mellanox(R) ConnectX(R)-4 50G MCX4131A-GCAT/MCX413A-GCAT (1x50G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1013 + * Firmware version: 12.18.1000 + + * Mellanox(R) ConnectX(R)-4 50G MCX414A-BCAT (2x50G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1013 + * Firmware version: 12.18.1000 + + * Mellanox(R) ConnectX(R)-4 50G MCX415A-GCAT/MCX416A-BCAT/MCX416A-GCAT (2x50G) + + * Host interface: PCI Express 3.0 x16 + * Device ID: 15b3:1013 + * Firmware version: 12.18.1000 + + * Mellanox(R) ConnectX(R)-4 50G MCX415A-CCAT (1x100G) + + * Host interface: PCI Express 3.0 x16 + * Device ID: 15b3:1013 + * Firmware version: 12.18.1000 + + * Mellanox(R) ConnectX(R)-4 100G MCX416A-CCAT (2x100G) + + * Host interface: PCI Express 3.0 x16 + * Device ID: 15b3:1013 + * Firmware version: 12.18.1000 + + * Mellanox(R) ConnectX(R)-4 Lx 10G MCX4121A-XCAT (2x10G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1015 + * Firmware version: 14.18.1000 + + * Mellanox(R) ConnectX(R)-4 Lx 25G MCX4121A-ACAT (2x25G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1015 + * Firmware version: 14.18.1000 + + * Mellanox(R) ConnectX(R)-5 100G MCX556A-ECAT (2x100G) + + * Host interface: PCI Express 3.0 x16 + * Device ID: 15b3:1017 + * Firmware version: 16.18.1000 + + * Mellanox(R) ConnectX-5 Ex EN 100G MCX516A-CDAT (2x100G) + + * Host interface: PCI Express 4.0 x16 + * Device ID: 15b3:1019 + * Firmware version: 16.18.1000 + +* IBM(R) Power8(R) with Mellanox(R) NICs combinations + + * Machine: + + * Processor: POWER8E (raw), AltiVec supported + + * type-model: 8247-22L + * Firmware FW810.21 (SV810_108) + + * OS: Ubuntu 16.04 LTS PPC le + + * MLNX_OFED: 4.0-1.0.1.0 + + * NICs: + + * Mellanox(R) ConnectX(R)-4 10G MCX4111A-XCAT (1x10G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1013 + * Firmware version: 12.18.1000 + + * Mellanox(R) ConnectX(R)-4 10G MCX4121A-XCAT (2x10G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1013 + * Firmware version: 12.18.1000 + + * Mellanox(R) ConnectX(R)-4 25G MCX4111A-ACAT (1x25G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1013 + * Firmware version: 12.18.1000 + + * Mellanox(R) ConnectX(R)-4 25G MCX4121A-ACAT (2x25G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1013 + * Firmware version: 12.18.1000 + + * Mellanox(R) ConnectX(R)-4 40G MCX4131A-BCAT/MCX413A-BCAT (1x40G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1013 + * Firmware version: 12.18.1000 + + * Mellanox(R) ConnectX(R)-4 40G MCX415A-BCAT (1x40G) + + * Host interface: PCI Express 3.0 x16 + * Device ID: 15b3:1013 + * Firmware version: 12.18.1000 + + * Mellanox(R) ConnectX(R)-4 50G MCX4131A-GCAT/MCX413A-GCAT (1x50G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1013 + * Firmware version: 12.18.1000 + + * Mellanox(R) ConnectX(R)-4 50G MCX414A-BCAT (2x50G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1013 + * Firmware version: 12.18.1000 + + * Mellanox(R) ConnectX(R)-4 50G MCX415A-GCAT/MCX416A-BCAT/MCX416A-GCAT (2x50G) + + * Host interface: PCI Express 3.0 x16 + * Device ID: 15b3:1013 + * Firmware version: 12.18.1000 + + * Mellanox(R) ConnectX(R)-4 50G MCX415A-CCAT (1x100G) + + * Host interface: PCI Express 3.0 x16 + * Device ID: 15b3:1013 + * Firmware version: 12.18.1000 + + * Mellanox(R) ConnectX(R)-4 100G MCX416A-CCAT (2x100G) + + * Host interface: PCI Express 3.0 x16 + * Device ID: 15b3:1013 + * Firmware version: 12.18.1000 + + * Mellanox(R) ConnectX(R)-4 Lx 10G MCX4121A-XCAT (2x10G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1015 + * Firmware version: 14.18.1000 + + * Mellanox(R) ConnectX(R)-4 Lx 25G MCX4121A-ACAT (2x25G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1015 + * Firmware version: 14.18.1000 + + * Mellanox(R) ConnectX(R)-5 100G MCX556A-ECAT (2x100G) + + * Host interface: PCI Express 3.0 x16 + * Device ID: 15b3:1017 + * Firmware version: 16.18.1000 + +* Intel(R) platforms with Intel(R) NICs combinations + + * Platform details + + * Intel(R) Atom(TM) CPU C2758 @ 2.40GHz + * Intel(R) Xeon(R) CPU D-1540 @ 2.00GHz + * Intel(R) Xeon(R) CPU E5-4667 v3 @ 2.00GHz + * Intel(R) Xeon(R) CPU E5-2680 v2 @ 2.80GHz + * Intel(R) Xeon(R) CPU E5-2699 v3 @ 2.30GHz + * Intel(R) Xeon(R) CPU E5-2695 v4 @ 2.10GHz + * Intel(R) Xeon(R) CPU E5-2658 v2 @ 2.40GHz + + * OS: + + * CentOS 7.2 + * Fedora 25 + * FreeBSD 11 + * Red Hat Enterprise Linux Server release 7.3 + * SUSE Enterprise Linux 12 + * Wind River Linux 8 + * Ubuntu 16.04 + * Ubuntu 16.10 + + * NICs: + + * Intel(R) 82599ES 10 Gigabit Ethernet Controller + + * Firmware version: 0x61bf0001 + * Device id (pf/vf): 8086:10fb / 8086:10ed + * Driver version: 4.0.1-k (ixgbe) + + * Intel(R) Corporation Ethernet Connection X552/X557-AT 10GBASE-T + + * Firmware version: 0x800001cf + * Device id (pf/vf): 8086:15ad / 8086:15a8 + * Driver version: 4.2.5 (ixgbe) + + * Intel(R) Ethernet Converged Network Adapter X710-DA4 (4x10G) + + * Firmware version: 5.05 + * Device id (pf/vf): 8086:1572 / 8086:154c + * Driver version: 1.5.23 (i40e) + + * Intel(R) Ethernet Converged Network Adapter X710-DA2 (2x10G) + + * Firmware version: 5.05 + * Device id (pf/vf): 8086:1572 / 8086:154c + * Driver version: 1.5.23 (i40e) + + * Intel(R) Ethernet Converged Network Adapter XL710-QDA1 (1x40G) + + * Firmware version: 5.05 + * Device id (pf/vf): 8086:1584 / 8086:154c + * Driver version: 1.5.23 (i40e) + + * Intel(R) Ethernet Converged Network Adapter XL710-QDA2 (2X40G) + + * Firmware version: 5.05 + * Device id (pf/vf): 8086:1583 / 8086:154c + * Driver version: 1.5.23 (i40e) + + * Intel(R) Corporation I350 Gigabit Network Connection + + * Firmware version: 1.48, 0x800006e7 + * Device id (pf/vf): 8086:1521 / 8086:1520 + * Driver version: 5.2.13-k (igb) diff --git a/dpdk/doc/guides/rel_notes/release_17_05.rst b/dpdk/doc/guides/rel_notes/release_17_05.rst new file mode 100644 index 00000000..87b3f927 --- /dev/null +++ b/dpdk/doc/guides/rel_notes/release_17_05.rst @@ -0,0 +1,830 @@ +DPDK Release 17.05 +================== + +.. **Read this first.** + + The text in the sections below explains how to update the release notes. + + Use proper spelling, capitalization and punctuation in all sections. + + Variable and config names should be quoted as fixed width text: + ``LIKE_THIS``. + + Build the docs and view the output file to ensure the changes are correct:: + + make doc-guides-html + + xdg-open build/doc/html/guides/rel_notes/release_17_05.html + + +New Features +------------ + +.. This section should contain new features added in this release. Sample + format: + + * **Add a title in the past tense with a full stop.** + + Add a short 1-2 sentence description in the past tense. The description + should be enough to allow someone scanning the release notes to + understand the new feature. + + If the feature adds a lot of sub-features you can use a bullet list like + this: + + * Added feature foo to do something. + * Enhanced feature bar to do something else. + + Refer to the previous release notes for examples. + + This section is a comment. do not overwrite or remove it. + Also, make sure to start the actual text at the margin. + ========================================================= + +* **Reorganized mbuf structure.** + + The mbuf structure has been reorganized as follows: + + * Align fields to facilitate the writing of ``data_off``, ``refcnt``, and + ``nb_segs`` in one operation. + * Use 2 bytes for port and number of segments. + * Move the sequence number to the second cache line. + * Add a timestamp field. + * Set default value for ``refcnt``, ``next`` and ``nb_segs`` at mbuf free. + +* **Added mbuf raw free API.** + + Moved ``rte_mbuf_raw_free()`` and ``rte_pktmbuf_prefree_seg()`` functions to + the public API. + +* **Added free Tx mbuf on demand API.** + + Added a new function ``rte_eth_tx_done_cleanup()`` which allows an + application to request the driver to release mbufs that are no longer in use + from a Tx ring, independent of whether or not the ``tx_rs_thresh`` has been + crossed. + +* **Added device removal interrupt.** + + Added a new ethdev event ``RTE_ETH_DEV_INTR_RMV`` to signify + the sudden removal of a device. + This event can be advertised by PCI drivers and enabled accordingly. + +* **Added EAL dynamic log framework.** + + Added new APIs to dynamically register named log types, and control + the level of each type independently. + +* **Added descriptor status ethdev API.** + + Added a new API to get the status of a descriptor. + + For Rx, it is almost similar to the ``rx_descriptor_done`` API, except + it differentiates descriptors which are held by the driver and not + returned to the hardware. For Tx, it is a new API. + +* **Increased number of next hops for LPM IPv6 to 2^21.** + + The next_hop field has been extended from 8 bits to 21 bits for IPv6. + +* **Added VFIO hotplug support.** + + Added hotplug support for VFIO in addition to the existing UIO support. + +* **Added PowerPC support to pci probing for vfio-pci devices.** + + Enabled sPAPR IOMMU based pci probing for vfio-pci devices. + +* **Kept consistent PMD batching behavior.** + + Removed the limit of fm10k/i40e/ixgbe Tx burst size and vhost Rx/Tx burst size + in order to support the same policy of "make an best effort to Rx/Tx pkts" + for PMDs. + +* **Updated the ixgbe base driver.** + + Updated the ixgbe base driver, including the following changes: + + * Add link block check for KR. + * Complete HW initialization even if SFP is not present. + * Add VF xcast promiscuous mode. + +* **Added PowerPC support for i40e and its vector PMD.** + + Enabled i40e PMD and its vector PMD by default in PowerPC. + +* **Added VF max bandwidth setting in i40e.** + + Enabled capability to set the max bandwidth for a VF in i40e. + +* **Added VF TC min and max bandwidth setting in i40e.** + + Enabled capability to set the min and max allocated bandwidth for a TC on a + VF in i40. + +* **Added TC strict priority mode setting on i40e.** + + There are 2 Tx scheduling modes supported for TCs by i40e HW: round robin + mode and strict priority mode. By default the round robin mode is used. It + is now possible to change the Tx scheduling mode for a TC. This is a global + setting on a physical port. + +* **Added i40e dynamic device personalization support.** + + * Added dynamic device personalization processing to i40e firmware. + +* **Added Cloud Filter for QinQ steering to i40e.** + + * Added a QinQ cloud filter on the i40e PMD, for steering traffic to a VM + using both VLAN tags. Note, this feature is not supported in Vector Mode. + +* **Updated mlx5 PMD.** + + Updated the mlx5 driver, including the following changes: + + * Added Generic flow API support for classification according to ether type. + * Extended Generic flow API support for classification of IPv6 flow + according to Vtc flow, Protocol and Hop limit. + * Added Generic flow API support for FLAG action. + * Added Generic flow API support for RSS action. + * Added support for TSO for non-tunneled and VXLAN packets. + * Added support for hardware Tx checksum offloads for VXLAN packets. + * Added support for user space Rx interrupt mode. + * Improved ConnectX-5 single core and maximum performance. + +* **Updated mlx4 PMD.** + + Updated the mlx4 driver, including the following changes: + + * Added support for Generic flow API basic flow items and actions. + * Added support for device removal event. + +* **Updated the sfc_efx driver.** + + * Added Generic Flow API support for Ethernet, VLAN, IPv4, IPv6, UDP and TCP + pattern items with QUEUE action for ingress traffic. + + * Added support for virtual functions (VFs). + +* **Added LiquidIO network PMD.** + + Added poll mode driver support for Cavium LiquidIO II server adapter VFs. + +* **Added Atomic Rules Arkville PMD.** + + Added a new poll mode driver for the Arkville family of + devices from Atomic Rules. The net/ark PMD supports line-rate + agnostic, multi-queue data movement on Arkville core FPGA instances. + +* **Added support for NXP DPAA2 - FSLMC bus.** + + Added the new bus "fslmc" driver for NXP DPAA2 devices. See the + "Network Interface Controller Drivers" document for more details of this new + driver. + +* **Added support for NXP DPAA2 Network PMD.** + + Added the new "dpaa2" net driver for NXP DPAA2 devices. See the + "Network Interface Controller Drivers" document for more details of this new + driver. + +* **Added support for the Wind River Systems AVP PMD.** + + Added a new networking driver for the AVP device type. Theses devices are + specific to the Wind River Systems virtualization platforms. + +* **Added vmxnet3 version 3 support.** + + Added support for vmxnet3 version 3 which includes several + performance enhancements such as configurable Tx data ring, Receive + Data Ring, and the ability to register memory regions. + +* **Updated the TAP driver.** + + Updated the TAP PMD to: + + * Support MTU modification. + * Support packet type for Rx. + * Support segmented packets on Rx and Tx. + * Speed up Rx on TAP when no packets are available. + * Support capturing traffic from another netdevice. + * Dynamically change link status when the underlying interface state changes. + * Added Generic Flow API support for Ethernet, VLAN, IPv4, IPv6, UDP and + TCP pattern items with DROP, QUEUE and PASSTHRU actions for ingress + traffic. + +* **Added MTU feature support to Virtio and Vhost.** + + Implemented new Virtio MTU feature in Vhost and Virtio: + + * Add ``rte_vhost_mtu_get()`` API to Vhost library. + * Enable Vhost PMD's MTU get feature. + * Get max MTU value from host in Virtio PMD + +* **Added interrupt mode support for virtio-user.** + + Implemented Rxq interrupt mode and LSC support for virtio-user as a virtual + device. Supported cases: + + * Rxq interrupt for virtio-user + vhost-user as the backend. + * Rxq interrupt for virtio-user + vhost-kernel as the backend. + * LSC interrupt for virtio-user + vhost-user as the backend. + +* **Added event driven programming model library (rte_eventdev).** + + This API introduces an event driven programming model. + + In a polling model, lcores poll ethdev ports and associated + Rx queues directly to look for a packet. By contrast in an event + driven model, lcores call the scheduler that selects packets for + them based on programmer-specified criteria. The Eventdev library + adds support for an event driven programming model, which offers + applications automatic multicore scaling, dynamic load balancing, + pipelining, packet ingress order maintenance and + synchronization services to simplify application packet processing. + + By introducing an event driven programming model, DPDK can support + both polling and event driven programming models for packet processing, + and applications are free to choose whatever model + (or combination of the two) best suits their needs. + +* **Added Software Eventdev PMD.** + + Added support for the software eventdev PMD. The software eventdev is a + software based scheduler device that implements the eventdev API. This + PMD allows an application to configure a pipeline using the eventdev + library, and run the scheduling workload on a CPU core. + +* **Added Cavium OCTEONTX Eventdev PMD.** + + Added the new octeontx ssovf eventdev driver for OCTEONTX devices. See the + "Event Device Drivers" document for more details on this new driver. + +* **Added information metrics library.** + + Added a library that allows information metrics to be added and updated + by producers, typically other libraries, for later retrieval by + consumers such as applications. It is intended to provide a + reporting mechanism that is independent of other libraries such + as ethdev. + +* **Added bit-rate calculation library.** + + Added a library that can be used to calculate device bit-rates. Calculated + bitrates are reported using the metrics library. + +* **Added latency stats library.** + + Added a library that measures packet latency. The collected statistics are + jitter and latency. For latency the minimum, average, and maximum is + measured. + +* **Added NXP DPAA2 SEC crypto PMD.** + + A new "dpaa2_sec" hardware based crypto PMD for NXP DPAA2 devices has been + added. See the "Crypto Device Drivers" document for more details on this + driver. + +* **Updated the Cryptodev Scheduler PMD.** + + * Added a packet-size based distribution mode, which distributes the enqueued + crypto operations among two slaves, based on their data lengths. + * Added fail-over scheduling mode, which enqueues crypto operations to a + primary slave first. Then, any operation that cannot be enqueued is + enqueued to a secondary slave. + * Added mode specific option support, so each scheduling mode can + now be configured individually by the new API. + +* **Updated the QAT PMD.** + + The QAT PMD has been updated with additional support for: + + * AES DOCSIS BPI algorithm. + * DES DOCSIS BPI algorithm. + * ZUC EEA3/EIA3 algorithms. + +* **Updated the AESNI MB PMD.** + + The AESNI MB PMD has been updated with additional support for: + + * AES DOCSIS BPI algorithm. + +* **Updated the OpenSSL PMD.** + + The OpenSSL PMD has been updated with additional support for: + + * DES DOCSIS BPI algorithm. + + +Resolved Issues +--------------- + +.. This section should contain bug fixes added to the relevant + sections. Sample format: + + * **code/section Fixed issue in the past tense with a full stop.** + + Add a short 1-2 sentence description of the resolved issue in the past + tense. + + The title should contain the code/lib section like a commit message. + + Add the entries in alphabetic order in the relevant sections below. + + This section is a comment. do not overwrite or remove it. + Also, make sure to start the actual text at the margin. + ========================================================= + + +* **l2fwd-keepalive: Fixed unclean shutdowns.** + + Added clean shutdown to l2fwd-keepalive so that it can free up + stale resources used for inter-process communication. + + +Known Issues +------------ + +.. This section should contain new known issues in this release. Sample format: + + * **Add title in present tense with full stop.** + + Add a short 1-2 sentence description of the known issue in the present + tense. Add information on any known workarounds. + + This section is a comment. do not overwrite or remove it. + Also, make sure to start the actual text at the margin. + ========================================================= + +* **LSC interrupt doesn't work for virtio-user + vhost-kernel.** + + LSC interrupt cannot be detected when setting the backend, tap device, + up/down as we fail to find a way to monitor such event. + + +API Changes +----------- + +.. This section should contain API changes. Sample format: + + * Add a short 1-2 sentence description of the API change. Use fixed width + quotes for ``rte_function_names`` or ``rte_struct_names``. Use the past + tense. + + This section is a comment. do not overwrite or remove it. + Also, make sure to start the actual text at the margin. + ========================================================= + +* The LPM ``next_hop`` field is extended from 8 bits to 21 bits for IPv6 + while keeping ABI compatibility. + +* **Reworked rte_ring library.** + + The rte_ring library has been reworked and updated. The following changes + have been made to it: + + * Removed the build-time setting ``CONFIG_RTE_RING_SPLIT_PROD_CONS``. + * Removed the build-time setting ``CONFIG_RTE_LIBRTE_RING_DEBUG``. + * Removed the build-time setting ``CONFIG_RTE_RING_PAUSE_REP_COUNT``. + * Removed the function ``rte_ring_set_water_mark`` as part of a general + removal of watermarks support in the library. + * Added an extra parameter to the burst/bulk enqueue functions to + return the number of free spaces in the ring after enqueue. This can + be used by an application to implement its own watermark functionality. + * Added an extra parameter to the burst/bulk dequeue functions to return + the number elements remaining in the ring after dequeue. + * Changed the return value of the enqueue and dequeue bulk functions to + match that of the burst equivalents. In all cases, ring functions which + operate on multiple packets now return the number of elements enqueued + or dequeued, as appropriate. The updated functions are: + + - ``rte_ring_mp_enqueue_bulk`` + - ``rte_ring_sp_enqueue_bulk`` + - ``rte_ring_enqueue_bulk`` + - ``rte_ring_mc_dequeue_bulk`` + - ``rte_ring_sc_dequeue_bulk`` + - ``rte_ring_dequeue_bulk`` + + NOTE: the above functions all have different parameters as well as + different return values, due to the other listed changes above. This + means that all instances of the functions in existing code will be + flagged by the compiler. The return value usage should be checked + while fixing the compiler error due to the extra parameter. + +* **Reworked rte_vhost library.** + + The rte_vhost library has been reworked to make it generic enough so that + the user could build other vhost-user drivers on top of it. To achieve this + the following changes have been made: + + * The following vhost-pmd APIs are removed: + + * ``rte_eth_vhost_feature_disable`` + * ``rte_eth_vhost_feature_enable`` + * ``rte_eth_vhost_feature_get`` + + * The vhost API ``rte_vhost_driver_callback_register(ops)`` is reworked to + be per vhost-user socket file. Thus, it takes one more argument: + ``rte_vhost_driver_callback_register(path, ops)``. + + * The vhost API ``rte_vhost_get_queue_num`` is deprecated, instead, + ``rte_vhost_get_vring_num`` should be used. + + * The following macros are removed in ``rte_virtio_net.h`` + + * ``VIRTIO_RXQ`` + * ``VIRTIO_TXQ`` + * ``VIRTIO_QNUM`` + + * The following net specific header files are removed in ``rte_virtio_net.h`` + + * ``linux/virtio_net.h`` + * ``sys/socket.h`` + * ``linux/if.h`` + * ``rte_ether.h`` + + * The vhost struct ``virtio_net_device_ops`` is renamed to + ``vhost_device_ops`` + + * The vhost API ``rte_vhost_driver_session_start`` is removed. Instead, + ``rte_vhost_driver_start`` should be used, and there is no need to create + a thread to call it. + + * The vhost public header file ``rte_virtio_net.h`` is renamed to + ``rte_vhost.h`` + + +ABI Changes +----------- + +.. This section should contain ABI changes. Sample format: + + * Add a short 1-2 sentence description of the ABI change that was announced + in the previous releases and made in this release. Use fixed width quotes + for ``rte_function_names`` or ``rte_struct_names``. Use the past tense. + + This section is a comment. do not overwrite or remove it. + Also, make sure to start the actual text at the margin. + ========================================================= + +* **Reorganized the mbuf structure.** + + The order and size of the fields in the ``mbuf`` structure changed, + as described in the `New Features`_ section. + +* The ``rte_cryptodev_info.sym`` structure has a new field ``max_nb_sessions_per_qp`` + to support drivers which may support a limited number of sessions per queue_pair. + + +Removed Items +------------- + +.. This section should contain removed items in this release. Sample format: + + * Add a short 1-2 sentence description of the removed item in the past + tense. + + This section is a comment. do not overwrite or remove it. + Also, make sure to start the actual text at the margin. + ========================================================= + +* KNI vhost support has been removed. + +* The dpdk_qat sample application has been removed. + +Shared Library Versions +----------------------- + +.. Update any library version updated in this release and prepend with a ``+`` + sign, like this: + + librte_acl.so.2 + + librte_cfgfile.so.2 + librte_cmdline.so.2 + + This section is a comment. do not overwrite or remove it. + ========================================================= + + +The libraries prepended with a plus sign were incremented in this version. + +.. code-block:: diff + + librte_acl.so.2 + + librte_bitratestats.so.1 + librte_cfgfile.so.2 + librte_cmdline.so.2 + librte_cryptodev.so.2 + librte_distributor.so.1 + + librte_eal.so.4 + librte_ethdev.so.6 + + librte_eventdev.so.1 + librte_hash.so.2 + librte_ip_frag.so.1 + librte_jobstats.so.1 + librte_kni.so.2 + librte_kvargs.so.1 + + librte_latencystats.so.1 + librte_lpm.so.2 + + librte_mbuf.so.3 + librte_mempool.so.2 + librte_meter.so.1 + + librte_metrics.so.1 + librte_net.so.1 + librte_pdump.so.1 + librte_pipeline.so.3 + librte_pmd_bond.so.1 + librte_pmd_ring.so.2 + librte_port.so.3 + librte_power.so.1 + librte_reorder.so.1 + librte_ring.so.1 + librte_sched.so.1 + librte_table.so.2 + librte_timer.so.1 + librte_vhost.so.3 + + +Tested Platforms +---------------- + +.. This section should contain a list of platforms that were tested with this + release. + + The format is: + + * platform with combinations + + * List of CPU + * List of OS + * List of devices + * Other relevant details... + + This section is a comment. do not overwrite or remove it. + Also, make sure to start the actual text at the margin. + ========================================================= + +* Intel(R) platforms with Intel(R) NICs combinations + + * CPU + + * Intel(R) Atom(TM) CPU C2758 @ 2.40GHz + * Intel(R) Xeon(R) CPU D-1540 @ 2.00GHz + * Intel(R) Xeon(R) CPU E5-4667 v3 @ 2.00GHz + * Intel(R) Xeon(R) CPU E5-2680 v2 @ 2.80GHz + * Intel(R) Xeon(R) CPU E5-2699 v3 @ 2.30GHz + * Intel(R) Xeon(R) CPU E5-2695 v4 @ 2.10GHz + * Intel(R) Xeon(R) CPU E5-2658 v2 @ 2.40GHz + * Intel(R) Xeon(R) CPU E5-2658 v3 @ 2.20GHz + + * OS: + + * CentOS 7.2 + * Fedora 25 + * FreeBSD 11 + * Red Hat Enterprise Linux Server release 7.3 + * SUSE Enterprise Linux 12 + * Wind River Linux 8 + * Ubuntu 16.04 + * Ubuntu 16.10 + + * NICs: + + * Intel(R) 82599ES 10 Gigabit Ethernet Controller + + * Firmware version: 0x61bf0001 + * Device id (pf/vf): 8086:10fb / 8086:10ed + * Driver version: 4.0.1-k (ixgbe) + + * Intel(R) Corporation Ethernet Connection X552/X557-AT 10GBASE-T + + * Firmware version: 0x800001cf + * Device id (pf/vf): 8086:15ad / 8086:15a8 + * Driver version: 4.2.5 (ixgbe) + + * Intel(R) Ethernet Converged Network Adapter X710-DA4 (4x10G) + + * Firmware version: 5.05 + * Device id (pf/vf): 8086:1572 / 8086:154c + * Driver version: 1.5.23 (i40e) + + * Intel(R) Ethernet Converged Network Adapter X710-DA2 (2x10G) + + * Firmware version: 5.05 + * Device id (pf/vf): 8086:1572 / 8086:154c + * Driver version: 1.5.23 (i40e) + + * Intel(R) Ethernet Converged Network Adapter XL710-QDA1 (1x40G) + + * Firmware version: 5.05 + * Device id (pf/vf): 8086:1584 / 8086:154c + * Driver version: 1.5.23 (i40e) + + * Intel(R) Ethernet Converged Network Adapter XL710-QDA2 (2X40G) + + * Firmware version: 5.05 + * Device id (pf/vf): 8086:1583 / 8086:154c + * Driver version: 1.5.23 (i40e) + + * Intel(R) Corporation I350 Gigabit Network Connection + + * Firmware version: 1.48, 0x800006e7 + * Device id (pf/vf): 8086:1521 / 8086:1520 + * Driver version: 5.2.13-k (igb) + +* Intel(R) platforms with Mellanox(R) NICs combinations + + * Platform details: + + * Intel(R) Xeon(R) CPU E5-2697A v4 @ 2.60GHz + * Intel(R) Xeon(R) CPU E5-2697 v3 @ 2.60GHz + * Intel(R) Xeon(R) CPU E5-2680 v2 @ 2.80GHz + * Intel(R) Xeon(R) CPU E5-2640 @ 2.50GHz + + * OS: + + * Red Hat Enterprise Linux Server release 7.3 (Maipo) + * Red Hat Enterprise Linux Server release 7.2 (Maipo) + * Ubuntu 16.10 + * Ubuntu 16.04 + * Ubuntu 14.04 + + * MLNX_OFED: 4.0-2.0.0.0 + + * NICs: + + * Mellanox(R) ConnectX(R)-3 Pro 40G MCX354A-FCC_Ax (2x40G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1007 + * Firmware version: 2.40.5030 + + * Mellanox(R) ConnectX(R)-4 10G MCX4111A-XCAT (1x10G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1013 + * Firmware version: 12.18.2000 + + * Mellanox(R) ConnectX(R)-4 10G MCX4121A-XCAT (2x10G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1013 + * Firmware version: 12.18.2000 + + * Mellanox(R) ConnectX(R)-4 25G MCX4111A-ACAT (1x25G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1013 + * Firmware version: 12.18.2000 + + * Mellanox(R) ConnectX(R)-4 25G MCX4121A-ACAT (2x25G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1013 + * Firmware version: 12.18.2000 + + * Mellanox(R) ConnectX(R)-4 40G MCX4131A-BCAT/MCX413A-BCAT (1x40G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1013 + * Firmware version: 12.18.2000 + + * Mellanox(R) ConnectX(R)-4 40G MCX415A-BCAT (1x40G) + + * Host interface: PCI Express 3.0 x16 + * Device ID: 15b3:1013 + * Firmware version: 12.18.2000 + + * Mellanox(R) ConnectX(R)-4 50G MCX4131A-GCAT/MCX413A-GCAT (1x50G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1013 + * Firmware version: 12.18.2000 + + * Mellanox(R) ConnectX(R)-4 50G MCX414A-BCAT (2x50G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1013 + * Firmware version: 12.18.2000 + + * Mellanox(R) ConnectX(R)-4 50G MCX415A-GCAT/MCX416A-BCAT/MCX416A-GCAT (2x50G) + + * Host interface: PCI Express 3.0 x16 + * Device ID: 15b3:1013 + * Firmware version: 12.18.2000 + + * Mellanox(R) ConnectX(R)-4 50G MCX415A-CCAT (1x100G) + + * Host interface: PCI Express 3.0 x16 + * Device ID: 15b3:1013 + * Firmware version: 12.18.2000 + + * Mellanox(R) ConnectX(R)-4 100G MCX416A-CCAT (2x100G) + + * Host interface: PCI Express 3.0 x16 + * Device ID: 15b3:1013 + * Firmware version: 12.18.2000 + + * Mellanox(R) ConnectX(R)-4 Lx 10G MCX4121A-XCAT (2x10G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1015 + * Firmware version: 14.18.2000 + + * Mellanox(R) ConnectX(R)-4 Lx 25G MCX4121A-ACAT (2x25G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1015 + * Firmware version: 14.18.2000 + + * Mellanox(R) ConnectX(R)-5 100G MCX556A-ECAT (2x100G) + + * Host interface: PCI Express 3.0 x16 + * Device ID: 15b3:1017 + * Firmware version: 16.19.1200 + + * Mellanox(R) ConnectX-5 Ex EN 100G MCX516A-CDAT (2x100G) + + * Host interface: PCI Express 4.0 x16 + * Device ID: 15b3:1019 + * Firmware version: 16.19.1200 + +* IBM(R) Power8(R) with Mellanox(R) NICs combinations + + * Platform details: + + * Processor: POWER8E (raw), AltiVec supported + * type-model: 8247-22L + * Firmware FW810.21 (SV810_108) + + * OS: Ubuntu 16.04 LTS PPC le + + * MLNX_OFED: 4.0-2.0.0.0 + + * NICs: + + * Mellanox(R) ConnectX(R)-4 10G MCX4111A-XCAT (1x10G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1013 + * Firmware version: 12.18.2000 + + * Mellanox(R) ConnectX(R)-4 10G MCX4121A-XCAT (2x10G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1013 + * Firmware version: 12.18.2000 + + * Mellanox(R) ConnectX(R)-4 25G MCX4111A-ACAT (1x25G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1013 + * Firmware version: 12.18.2000 + + * Mellanox(R) ConnectX(R)-4 25G MCX4121A-ACAT (2x25G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1013 + * Firmware version: 12.18.2000 + + * Mellanox(R) ConnectX(R)-4 40G MCX4131A-BCAT/MCX413A-BCAT (1x40G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1013 + * Firmware version: 12.18.2000 + + * Mellanox(R) ConnectX(R)-4 40G MCX415A-BCAT (1x40G) + + * Host interface: PCI Express 3.0 x16 + * Device ID: 15b3:1013 + * Firmware version: 12.18.2000 + + * Mellanox(R) ConnectX(R)-4 50G MCX4131A-GCAT/MCX413A-GCAT (1x50G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1013 + * Firmware version: 12.18.2000 + + * Mellanox(R) ConnectX(R)-4 50G MCX414A-BCAT (2x50G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1013 + * Firmware version: 12.18.2000 + + * Mellanox(R) ConnectX(R)-4 50G MCX415A-GCAT/MCX416A-BCAT/MCX416A-GCAT (2x50G) + + * Host interface: PCI Express 3.0 x16 + * Device ID: 15b3:1013 + * Firmware version: 12.18.2000 + + * Mellanox(R) ConnectX(R)-4 50G MCX415A-CCAT (1x100G) + + * Host interface: PCI Express 3.0 x16 + * Device ID: 15b3:1013 + * Firmware version: 12.18.2000 + + * Mellanox(R) ConnectX(R)-4 100G MCX416A-CCAT (2x100G) + + * Host interface: PCI Express 3.0 x16 + * Device ID: 15b3:1013 + * Firmware version: 12.18.2000 diff --git a/dpdk/doc/guides/rel_notes/release_17_08.rst b/dpdk/doc/guides/rel_notes/release_17_08.rst new file mode 100644 index 00000000..0bcdfb7b --- /dev/null +++ b/dpdk/doc/guides/rel_notes/release_17_08.rst @@ -0,0 +1,614 @@ +DPDK Release 17.08 +================== + +.. **Read this first.** + + The text in the sections below explains how to update the release notes. + + Use proper spelling, capitalization and punctuation in all sections. + + Variable and config names should be quoted as fixed width text: + ``LIKE_THIS``. + + Build the docs and view the output file to ensure the changes are correct:: + + make doc-guides-html + + xdg-open build/doc/html/guides/rel_notes/release_17_08.html + + +New Features +------------ + +.. This section should contain new features added in this release. Sample + format: + + * **Add a title in the past tense with a full stop.** + + Add a short 1-2 sentence description in the past tense. The description + should be enough to allow someone scanning the release notes to + understand the new feature. + + If the feature adds a lot of sub-features you can use a bullet list like + this: + + * Added feature foo to do something. + * Enhanced feature bar to do something else. + + Refer to the previous release notes for examples. + + This section is a comment. do not overwrite or remove it. + Also, make sure to start the actual text at the margin. + ========================================================= + +* **Increase minimum x86 ISA version to SSE4.2.** + + Starting with version 17.08, DPDK requires SSE4.2 to run on x86. + Previous versions required SSE3. + +* **Added Service Core functionality.** + + The service core functionality added to EAL allows DPDK to run services such + as software PMDs on lcores without the application manually running them. The + service core infrastructure allows flexibility of running multiple services + on the same service lcore, and provides the application with powerful APIs to + configure the mapping from service lcores to services. + +* **Added Generic Receive Offload API.** + + Added Generic Receive Offload (GRO) API support to reassemble TCP/IPv4 + packets. The GRO API assumes all input packets have the correct + checksums. GRO API doesn't update checksums for merged packets. If + input packets are IP fragmented, the GRO API assumes they are complete + packets (i.e. with L4 headers). + +* **Added Fail-Safe PMD** + + Added the new Fail-Safe PMD. This virtual device allows applications to + support seamless hotplug of devices. + See the :doc:`/nics/fail_safe` guide for more details about this driver. + +* **Added support for generic flow API (rte_flow) on igb NICs.** + + This API provides a generic means of configuring hardware to match specific + ingress or egress traffic, altering its behavior and querying related counters + according to any number of user-defined rules. + + Added generic flow API support for Ethernet, IPv4, UDP, TCP and RAW pattern + items with QUEUE actions. There are four types of filter support for this + feature on igb. + +* **Added support for generic flow API (rte_flow) on enic.** + + Added flow API support for outer Ethernet, VLAN, IPv4, IPv6, UDP, TCP, SCTP, + VxLAN and inner Ethernet, VLAN, IPv4, IPv6, UDP and TCP pattern items with + QUEUE, MARK, FLAG and VOID actions for ingress traffic. + +* **Added support for Chelsio T6 family of adapters** + + The CXGBE PMD was updated to run Chelsio T6 family of adapters. + +* **Added latency and performance improvements for cxgbe** + + the Tx and Rx path in cxgbe were reworked to improve performance. In + addition the latency was reduced for slow traffic. + +* **Updated the bnxt PMD.** + + Updated the bnxt PMD. The major enhancements include: + + * Support MTU modification. + * Add support for LRO. + * Add support for VLAN filter and strip functionality. + * Additional enhancements to add support for more dev_ops. + * Added PMD specific APIs mainly to control VF from PF. + * Update HWRM version to 1.7.7 + +* **Added support for Rx interrupts on mlx4 driver.** + + Rx queues can be now be armed with an interrupt which will trigger on the + next packet arrival. + +* **Updated mlx5 driver.** + + Updated the mlx5 driver including the following changes: + + * Added vectorized Rx/Tx burst for x86. + * Added support for isolated mode from flow API. + * Reworked the flow drop action to implement in hardware classifier. + * Improved Rx interrupts management. + +* **Updated szedata2 PMD.** + + Added support for firmware with multiple Ethernet ports per physical port. + +* **Updated dpaa2 PMD.** + + Updated dpaa2 PMD. Major enhancements include: + + * Added support for MAC Filter configuration. + * Added support for Segmented Buffers. + * Added support for VLAN filter and strip functionality. + * Additional enhancements to add support for more dev_ops. + * Optimized the packet receive path + +* **Reorganized the symmetric crypto operation structure.** + + The crypto operation (``rte_crypto_sym_op``) has been reorganized as follows: + + * Removed the ``rte_crypto_sym_op_sess_type`` field. + * Replaced the pointer and physical address of IV with offset from the start + of the crypto operation. + * Moved length and offset of cipher IV to ``rte_crypto_cipher_xform``. + * Removed "Additional Authentication Data" (AAD) length. + * Removed digest length. + * Removed AAD pointer and physical address from ``auth`` structure. + * Added ``aead`` structure, containing parameters for AEAD algorithms. + +* **Reorganized the crypto operation structure.** + + The crypto operation (``rte_crypto_op``) has been reorganized as follows: + + * Added the ``rte_crypto_op_sess_type`` field. + * The enumerations ``rte_crypto_op_status`` and ``rte_crypto_op_type`` + have been modified to be ``uint8_t`` values. + * Removed the field ``opaque_data``. + * Pointer to ``rte_crypto_sym_op`` has been replaced with a zero length array. + +* **Reorganized the crypto symmetric session structure.** + + The crypto symmetric session structure (``rte_cryptodev_sym_session``) has + been reorganized as follows: + + * The ``dev_id`` field has been removed. + * The ``driver_id`` field has been removed. + * The mempool pointer ``mp`` has been removed. + * Replaced ``private`` marker with array of pointers to private data sessions + ``sess_private_data``. + +* **Updated cryptodev library.** + + * Added AEAD algorithm specific functions and structures, so it is not + necessary to use a combination of cipher and authentication + structures anymore. + * Added helper functions for crypto device driver identification. + * Added support for multi-device sessions, so a single session can be + used in multiple drivers. + * Added functions to initialize and free individual driver private data + with the same session. + +* **Updated dpaa2_sec crypto PMD.** + + Added support for AES-GCM and AES-CTR. + +* **Updated the AESNI MB PMD.** + + The AESNI MB PMD has been updated with additional support for: + + * 12-byte IV on AES Counter Mode, apart from the previous 16-byte IV. + +* **Updated the AES-NI GCM PMD.** + + The AES-NI GCM PMD was migrated from the ISA-L library to the Multi Buffer + library, as the latter library has Scatter Gather List support + now. The migration entailed adding additional support for 192-bit keys. + +* **Updated the Cryptodev Scheduler PMD.** + + Added a multicore based distribution mode, which distributes the enqueued + crypto operations among several slaves, running on different logical cores. + +* **Added NXP DPAA2 Eventdev PMD.** + + Added the new dpaa2 eventdev driver for NXP DPAA2 devices. See the + "Event Device Drivers" document for more details on this new driver. + +* **Added dpdk-test-eventdev test application.** + + The dpdk-test-eventdev tool is a Data Plane Development Kit (DPDK) application + that allows exercising various eventdev use cases. + This application has a generic framework to add new eventdev based test cases + to verify functionality and measure the performance parameters of DPDK + eventdev devices. + + +Known Issues +------------ + +.. This section should contain new known issues in this release. Sample format: + + * **Add title in present tense with full stop.** + + Add a short 1-2 sentence description of the known issue in the present + tense. Add information on any known workarounds. + + This section is a comment. do not overwrite or remove it. + Also, make sure to start the actual text at the margin. + ========================================================= + +* **Starting with version 17.08, libnuma is required to build DPDK.** + + +API Changes +----------- + +.. This section should contain API changes. Sample format: + + * Add a short 1-2 sentence description of the API change. Use fixed width + quotes for ``rte_function_names`` or ``rte_struct_names``. Use the past + tense. + + This section is a comment. do not overwrite or remove it. + Also, make sure to start the actual text at the margin. + ========================================================= + +* **Modified the _rte_eth_dev_callback_process function in the ethdev library.** + + The function ``_rte_eth_dev_callback_process()`` has been modified. The + return value has been changed from void to int and an extra parameter ``void + *ret_param`` has been added. + +* **Moved bypass functions from the rte_ethdev library to ixgbe PMD** + + * The following rte_ethdev library functions were removed: + + * ``rte_eth_dev_bypass_event_show()`` + * ``rte_eth_dev_bypass_event_store()`` + * ``rte_eth_dev_bypass_init()`` + * ``rte_eth_dev_bypass_state_set()`` + * ``rte_eth_dev_bypass_state_show()`` + * ``rte_eth_dev_bypass_ver_show()`` + * ``rte_eth_dev_bypass_wd_reset()`` + * ``rte_eth_dev_bypass_wd_timeout_show()`` + * ``rte_eth_dev_wd_timeout_store()`` + + * The following ixgbe PMD functions were added: + + * ``rte_pmd_ixgbe_bypass_event_show()`` + * ``rte_pmd_ixgbe_bypass_event_store()`` + * ``rte_pmd_ixgbe_bypass_init()`` + * ``rte_pmd_ixgbe_bypass_state_set()`` + * ``rte_pmd_ixgbe_bypass_state_show()`` + * ``rte_pmd_ixgbe_bypass_ver_show()`` + * ``rte_pmd_ixgbe_bypass_wd_reset()`` + * ``rte_pmd_ixgbe_bypass_wd_timeout_show()`` + * ``rte_pmd_ixgbe_bypass_wd_timeout_store()`` + +* **Reworked rte_cryptodev library.** + + The rte_cryptodev library has been reworked and updated. The following changes + have been made to it: + + * The crypto device type enumeration has been removed from cryptodev library. + * The function ``rte_crypto_count_devtype()`` has been removed, and replaced + by the new function ``rte_crypto_count_by_driver()``. + * Moved crypto device driver names definitions to the particular PMDs. + These names are not public anymore. + * The ``rte_cryptodev_configure()`` function does not create the session + mempool for the device anymore. + * The ``rte_cryptodev_queue_pair_attach_sym_session()`` and + ``rte_cryptodev_queue_pair_dettach_sym_session()`` functions require + the new parameter ``device id``. + * Parameters of ``rte_cryptodev_sym_session_create()`` were modified to + accept ``mempool``, instead of ``device id`` and ``rte_crypto_sym_xform``. + * Removed ``device id`` parameter from ``rte_cryptodev_sym_session_free()``. + * Added a new field ``session_pool`` to ``rte_cryptodev_queue_pair_setup()``. + * Removed ``aad_size`` parameter from + ``rte_cryptodev_sym_capability_check_auth()``. + * Added ``iv_size`` parameter to + ``rte_cryptodev_sym_capability_check_auth()``. + * Removed ``RTE_CRYPTO_OP_STATUS_ENQUEUED`` from enum + ``rte_crypto_op_status``. + + +ABI Changes +----------- + +.. This section should contain ABI changes. Sample format: + + * Add a short 1-2 sentence description of the ABI change that was announced + in the previous releases and made in this release. Use fixed width quotes + for ``rte_function_names`` or ``rte_struct_names``. Use the past tense. + + This section is a comment. do not overwrite or remove it. + Also, make sure to start the actual text at the margin. + ========================================================= + +* Changed type of ``domain`` field in ``rte_pci_addr`` to ``uint32_t`` + to follow the PCI standard. + +* Added new ``rte_bus`` experimental APIs available as operators within the + ``rte_bus`` structure. + +* Made ``rte_devargs`` structure internal device representation generic to + prepare for a bus-agnostic EAL. + +* **Reorganized the crypto operation structures.** + + Some fields have been modified in the ``rte_crypto_op`` and + ``rte_crypto_sym_op`` structures, as described in the `New Features`_ + section. + +* **Reorganized the crypto symmetric session structure.** + + Some fields have been modified in the ``rte_cryptodev_sym_session`` + structure, as described in the `New Features`_ section. + +* **Reorganized the rte_crypto_sym_cipher_xform structure.** + + * Added cipher IV length and offset parameters. + * Changed field size of key length from ``size_t`` to ``uint16_t``. + +* **Reorganized the rte_crypto_sym_auth_xform structure.** + + * Added authentication IV length and offset parameters. + * Changed field size of AAD length from ``uint32_t`` to ``uint16_t``. + * Changed field size of digest length from ``uint32_t`` to ``uint16_t``. + * Removed AAD length. + * Changed field size of key length from ``size_t`` to ``uint16_t``. + +* Replaced ``dev_type`` enumeration with ``uint8_t`` ``driver_id`` in + ``rte_cryptodev_info`` and ``rte_cryptodev`` structures. + +* Removed ``session_mp`` from ``rte_cryptodev_config``. + + +Shared Library Versions +----------------------- + +.. Update any library version updated in this release and prepend with a ``+`` + sign, like this: + + librte_acl.so.2 + + librte_cfgfile.so.2 + librte_cmdline.so.2 + + This section is a comment. do not overwrite or remove it. + ========================================================= + + +The libraries prepended with a plus sign were incremented in this version. + +.. code-block:: diff + + librte_acl.so.2 + librte_bitratestats.so.1 + librte_cfgfile.so.2 + librte_cmdline.so.2 + + librte_cryptodev.so.3 + librte_distributor.so.1 + + librte_eal.so.5 + + librte_ethdev.so.7 + + librte_eventdev.so.2 + + librte_gro.so.1 + librte_hash.so.2 + librte_ip_frag.so.1 + librte_jobstats.so.1 + librte_kni.so.2 + librte_kvargs.so.1 + librte_latencystats.so.1 + librte_lpm.so.2 + librte_mbuf.so.3 + librte_mempool.so.2 + librte_meter.so.1 + librte_metrics.so.1 + librte_net.so.1 + librte_pdump.so.1 + librte_pipeline.so.3 + librte_pmd_bond.so.1 + librte_pmd_ring.so.2 + librte_port.so.3 + librte_power.so.1 + librte_reorder.so.1 + librte_ring.so.1 + librte_sched.so.1 + librte_table.so.2 + librte_timer.so.1 + librte_vhost.so.3 + + +Tested Platforms +---------------- + +.. This section should contain a list of platforms that were tested with this + release. + + The format is: + + * platform with combinations + + * List of CPU + * List of OS + * List of devices + * Other relevant details... + + This section is a comment. do not overwrite or remove it. + Also, make sure to start the actual text at the margin. + ========================================================= + +* Intel(R) platforms with Mellanox(R) NICs combinations + + * Platform details: + + * Intel(R) Xeon(R) CPU E5-2697A v4 @ 2.60GHz + * Intel(R) Xeon(R) CPU E5-2697 v3 @ 2.60GHz + * Intel(R) Xeon(R) CPU E5-2680 v2 @ 2.80GHz + * Intel(R) Xeon(R) CPU E5-2640 @ 2.50GHz + + * OS: + + * Red Hat Enterprise Linux Server release 7.3 (Maipo) + * Red Hat Enterprise Linux Server release 7.2 (Maipo) + * Ubuntu 16.10 + * Ubuntu 16.04 + * Ubuntu 14.04 + + * MLNX_OFED: 4.1-1.0.2.0 + + * NICs: + + * Mellanox(R) ConnectX(R)-3 Pro 40G MCX354A-FCC_Ax (2x40G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1007 + * Firmware version: 2.40.5030 + + * Mellanox(R) ConnectX(R)-4 10G MCX4111A-XCAT (1x10G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1013 + * Firmware version: 12.18.2000 + + * Mellanox(R) ConnectX(R)-4 10G MCX4121A-XCAT (2x10G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1013 + * Firmware version: 12.18.2000 + + * Mellanox(R) ConnectX(R)-4 25G MCX4111A-ACAT (1x25G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1013 + * Firmware version: 12.18.2000 + + * Mellanox(R) ConnectX(R)-4 25G MCX4121A-ACAT (2x25G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1013 + * Firmware version: 12.18.2000 + + * Mellanox(R) ConnectX(R)-4 40G MCX4131A-BCAT/MCX413A-BCAT (1x40G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1013 + * Firmware version: 12.18.2000 + + * Mellanox(R) ConnectX(R)-4 40G MCX415A-BCAT (1x40G) + + * Host interface: PCI Express 3.0 x16 + * Device ID: 15b3:1013 + * Firmware version: 12.18.2000 + + * Mellanox(R) ConnectX(R)-4 50G MCX4131A-GCAT/MCX413A-GCAT (1x50G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1013 + * Firmware version: 12.18.2000 + + * Mellanox(R) ConnectX(R)-4 50G MCX414A-BCAT (2x50G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1013 + * Firmware version: 12.18.2000 + + * Mellanox(R) ConnectX(R)-4 50G MCX415A-GCAT/MCX416A-BCAT/MCX416A-GCAT + (2x50G) + + * Host interface: PCI Express 3.0 x16 + * Device ID: 15b3:1013 + * Firmware version: 12.18.2000 + + * Mellanox(R) ConnectX(R)-4 50G MCX415A-CCAT (1x100G) + + * Host interface: PCI Express 3.0 x16 + * Device ID: 15b3:1013 + * Firmware version: 12.18.2000 + + * Mellanox(R) ConnectX(R)-4 100G MCX416A-CCAT (2x100G) + + * Host interface: PCI Express 3.0 x16 + * Device ID: 15b3:1013 + * Firmware version: 12.18.2000 + + * Mellanox(R) ConnectX(R)-4 Lx 10G MCX4121A-XCAT (2x10G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1015 + * Firmware version: 14.18.2000 + + * Mellanox(R) ConnectX(R)-4 Lx 25G MCX4121A-ACAT (2x25G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1015 + * Firmware version: 14.18.2000 + + * Mellanox(R) ConnectX(R)-5 100G MCX556A-ECAT (2x100G) + + * Host interface: PCI Express 3.0 x16 + * Device ID: 15b3:1017 + * Firmware version: 16.19.1200 + + * Mellanox(R) ConnectX-5 Ex EN 100G MCX516A-CDAT (2x100G) + + * Host interface: PCI Express 4.0 x16 + * Device ID: 15b3:1019 + * Firmware version: 16.19.1200 + +* Intel(R) platforms with Intel(R) NICs combinations + + * CPU + + * Intel(R) Atom(TM) CPU C2758 @ 2.40GHz + * Intel(R) Xeon(R) CPU D-1540 @ 2.00GHz + * Intel(R) Xeon(R) CPU D-1541 @ 2.10GHz + * Intel(R) Xeon(R) CPU E5-4667 v3 @ 2.00GHz + * Intel(R) Xeon(R) CPU E5-2680 v2 @ 2.80GHz + * Intel(R) Xeon(R) CPU E5-2699 v3 @ 2.30GHz + * Intel(R) Xeon(R) CPU E5-2695 v4 @ 2.10GHz + * Intel(R) Xeon(R) CPU E5-2658 v2 @ 2.40GHz + * Intel(R) Xeon(R) CPU E5-2658 v3 @ 2.20GHz + + * OS: + + * CentOS 7.2 + * Fedora 25 + * FreeBSD 11 + * Red Hat Enterprise Linux Server release 7.3 + * SUSE Enterprise Linux 12 + * Wind River Linux 8 + * Ubuntu 16.04 + * Ubuntu 16.10 + + * NICs: + + * Intel(R) 82599ES 10 Gigabit Ethernet Controller + + * Firmware version: 0x61bf0001 + * Device id (pf/vf): 8086:10fb / 8086:10ed + * Driver version: 4.0.1-k (ixgbe) + + * Intel(R) Corporation Ethernet Connection X552/X557-AT 10GBASE-T + + * Firmware version: 0x800001cf + * Device id (pf/vf): 8086:15ad / 8086:15a8 + * Driver version: 4.2.5 (ixgbe) + + * Intel(R) Ethernet Converged Network Adapter X710-DA4 (4x10G) + + * Firmware version: 6.01 0x80003205 + * Device id (pf/vf): 8086:1572 / 8086:154c + * Driver version: 2.0.19 (i40e) + + * Intel(R) Ethernet Converged Network Adapter X710-DA2 (2x10G) + + * Firmware version: 6.01 0x80003204 + * Device id (pf/vf): 8086:1572 / 8086:154c + * Driver version: 2.0.19 (i40e) + + * Intel(R) Ethernet Converged Network Adapter XXV710-DA2 (2x25G) + + * Firmware version: 6.01 0x80003221 + * Device id (pf/vf): 8086:158b + * Driver version: 2.0.19 (i40e) + + * Intel(R) Ethernet Converged Network Adapter XL710-QDA2 (2X40G) + + * Firmware version: 6.01 0x8000321c + * Device id (pf/vf): 8086:1583 / 8086:154c + * Driver version: 2.0.19 (i40e) + + * Intel(R) Corporation I350 Gigabit Network Connection + + * Firmware version: 1.48, 0x800006e7 + * Device id (pf/vf): 8086:1521 / 8086:1520 + * Driver version: 5.2.13-k (igb) diff --git a/dpdk/doc/guides/rel_notes/release_17_11.rst b/dpdk/doc/guides/rel_notes/release_17_11.rst new file mode 100644 index 00000000..fad7a7eb --- /dev/null +++ b/dpdk/doc/guides/rel_notes/release_17_11.rst @@ -0,0 +1,1092 @@ +DPDK Release 17.11 +================== + +.. **Read this first.** + + The text in the sections below explains how to update the release notes. + + Use proper spelling, capitalization and punctuation in all sections. + + Variable and config names should be quoted as fixed width text: + ``LIKE_THIS``. + + Build the docs and view the output file to ensure the changes are correct:: + + make doc-guides-html + + xdg-open build/doc/html/guides/rel_notes/release_17_11.html + + +New Features +------------ + +.. This section should contain new features added in this release. Sample + format: + + * **Add a title in the past tense with a full stop.** + + Add a short 1-2 sentence description in the past tense. The description + should be enough to allow someone scanning the release notes to + understand the new feature. + + If the feature adds a lot of sub-features you can use a bullet list like + this: + + * Added feature foo to do something. + * Enhanced feature bar to do something else. + + Refer to the previous release notes for examples. + + This section is a comment. do not overwrite or remove it. + Also, make sure to start the actual text at the margin. + ========================================================= + +* **Extended port_id range from uint8_t to uint16_t.** + + Increased the ``port_id`` range from 8 bits to 16 bits in order to support + more than 256 ports in DPDK. All ethdev APIs which have ``port_id`` as + parameter have been changed. + +* **Modified the return type of rte_eth_stats_reset.** + + Changed return type of ``rte_eth_stats_reset`` from ``void`` to ``int`` so + that the caller can determine whether a device supports the operation or not + and if the operation was carried out. + +* **Added a new driver for Marvell Armada 7k/8k devices.** + + Added the new ``mrvl`` net driver for Marvell Armada 7k/8k devices. See the + :doc:`../nics/mrvl` NIC guide for more details on this new driver. + +* **Updated mlx4 driver.** + + Updated the mlx4 driver including the following changes: + + * Isolated mode (rte_flow) can now be enabled anytime, not only during + initial device configuration. + * Flow rules now support up to 4096 priority levels usable at will by + applications. + * Enhanced error message to help debugging invalid/unsupported flow rules. + * Flow rules matching all multicast and promiscuous traffic are now allowed. + * No more software restrictions on flow rules with the RSS action, their + configuration is much more flexible. + * Significantly reduced memory footprint for Rx and Tx queue objects. + * While supported, UDP RSS is temporarily disabled due to a remaining issue + with its support in the Linux kernel. + * The new RSS implementation does not automatically spread traffic according + to the inner packet of VXLAN frames anymore, only the outer one (like + other PMDs). + * Partial (Tx only) support for secondary processes was broken and had to be + removed. + * Refactored driver to get rid of dependency on the components provided by + Mellanox OFED and instead rely on the current and public rdma-core + package and Linux version from now on. + * Removed compile-time limitation on number of device instances the PMD + can support. + +* **Updated mlx5 driver.** + + Updated the mlx5 driver including the following changes: + + * Enabled the PMD to run on top of upstream Linux kernel and rdma-core + libs, removing the dependency on specific Mellanox OFED libraries. + * Improved PMD latency performance. + * Improved PMD memory footprint. + * Added support for vectorized Rx/Tx burst for ARMv8. + * Added support for secondary process. + * Added support for flow counters. + * Added support for Rx hardware timestamp offload. + * Added support for device removal event. + +* **Added SoftNIC PMD.** + + Added a new SoftNIC PMD. This virtual device provides applications with + software fallback support for traffic management. + +* **Added support for NXP DPAA Devices.** + + Added support for NXP's DPAA devices - LS104x series. This includes: + + * DPAA Bus driver + * DPAA Mempool driver for supporting offloaded packet memory pool + * DPAA PMD for DPAA devices + + See the :doc:`../nics/dpaa` document for more details of this new driver. + +* **Updated support for Cavium OCTEONTX Device.** + + Updated support for Cavium's OCTEONTX device (CN83xx). This includes: + + * OCTEONTX Mempool driver for supporting offloaded packet memory pool + * OCTEONTX Ethdev PMD + * OCTEONTX Eventdev-Ethdev Rx adapter + + See the :doc:`../nics/octeontx` document for more details of this new driver. + +* **Added PF support to the Netronome NFP PMD.** + + Added PF support to the Netronome NFP PMD. Previously the NFP PMD only + supported VFs. PF support is just as a basic DPDK port and has no VF + management yet. + + PF support comes with firmware upload support which allows the PMD to + independently work from kernel netdev NFP drivers. + + NFP 4000 devices are also now supported along with previous 6000 devices. + +* **Updated bnxt PMD.** + + Major enhancements include: + + * Support for Flow API + * Support for Tx and Rx descriptor status functions + +* **Added bus agnostic functions to cryptodev for PMD initialization** + + Added new PMD assist, bus independent, functions + ``rte_cryptodev_pmd_parse_input_args()``, ``rte_cryptodev_pmd_create()`` and + ``rte_cryptodev_pmd_destroy()`` for drivers to manage creation and + destruction of new device instances. + +* **Updated QAT crypto PMD.** + + Added several performance enhancements: + + * Removed atomics from the internal queue pair structure. + * Added coalesce writes to HEAD CSR on response processing. + * Added coalesce writes to TAIL CSR on request processing. + + In addition support was added for the AES CCM algorithm. + +* **Updated the AESNI MB PMD.** + + The AESNI MB PMD has been updated with additional support for: + + * The DES CBC algorithm. + * The DES DOCSIS BPI algorithm. + + This change requires version 0.47 of the IPSec Multi-buffer library. For + more details see the :doc:`../cryptodevs/aesni_mb` documentation. + +* **Updated the OpenSSL PMD.** + + The OpenSSL PMD has been updated with additional support for: + + * The DES CBC algorithm. + * The AES CCM algorithm. + +* **Added NXP DPAA SEC crypto PMD.** + + A new ``dpaa_sec`` hardware based crypto PMD for NXP DPAA devices has been + added. See the :doc:`../cryptodevs/dpaa_sec` document for more details. + +* **Added MRVL crypto PMD.** + + A new crypto PMD has been added, which provides several ciphering and hashing + algorithms. All cryptography operations use the MUSDK library crypto API. + See the :doc:`../cryptodevs/mrvl` document for more details. + +* **Add new benchmarking mode to dpdk-test-crypto-perf application.** + + Added a new "PMD cyclecount" benchmark mode to the ``dpdk-test-crypto-perf`` + application to display a detailed breakdown of CPU cycles used by hardware + acceleration. + +* **Added the Security Offload Library.** + + Added an experimental library - ``rte_security``. This provide security APIs + for protocols like IPSec using inline ipsec offload to ethernet devices or + full protocol offload with lookaside crypto devices. + + See the :doc:`../prog_guide/rte_security` section of the DPDK Programmers + Guide document for more information. + +* **Updated the DPAA2_SEC crypto driver to support rte_security.** + + Updated the ``dpaa2_sec`` crypto PMD to support ``rte_security`` lookaside + protocol offload for IPSec. + +* **Updated the IXGBE ethernet driver to support rte_security.** + + Updated ixgbe ethernet PMD to support ``rte_security`` inline IPSec offload. + +* **Updated ipsec-secgw application to support rte_security.** + + Updated the ``ipsec-secgw`` sample application to support ``rte_security`` + actions for ipsec inline and full protocol offload using lookaside crypto + offload. + +* **Added IOMMU support to libvhost-user** + + Implemented device IOTLB in the Vhost-user backend, and enabled Virtio's + IOMMU feature. The feature is disabled by default, and can be enabled by + setting ``RTE_VHOST_USER_IOMMU_SUPPORT`` flag at vhost device registration + time. + +* **Added the Event Ethernet Adapter Library.** + + Added the Event Ethernet Adapter library. This library provides APIs for + eventdev applications to configure the ethdev for eventdev packet flow. + +* **Updated DPAA2 Event PMD for the Event Ethernet Adapter.** + + Added support for the eventdev ethernet adapter for DPAA2. + +* **Added Membership library (rte_member).** + + Added a new data structure library called the Membership Library. + + The Membership Library is an extension and generalization of a traditional + filter (for example Bloom Filter) structure that has multiple usages in a + wide variety of workloads and applications. In general, the Membership + Library is a data structure that provides a "set-summary" and responds to + set-membership queries whether a certain member belongs to a set(s). + + The library provides APIs for DPDK applications to insert a new member, + delete an existing member, and query the existence of a member in a given + set, or a group of sets. For the case of a group of sets the library will + return not only whether the element has been inserted in one of the sets but + also which set it belongs to. + + See the :doc:`../prog_guide/member_lib` documentation in the Programmers + Guide, for more information. + +* **Added the Generic Segmentation Offload Library.** + + Added the Generic Segmentation Offload (GSO) library to enable + applications to split large packets (e.g. MTU is 64KB) into small + ones (e.g. MTU is 1500B). Supported packet types are: + + * TCP/IPv4 packets. + * VxLAN packets, which must have an outer IPv4 header, and contain + an inner TCP/IPv4 packet. + * GRE packets, which must contain an outer IPv4 header, and inner + TCP/IPv4 headers. + + The GSO library doesn't check if the input packets have correct + checksums, and doesn't update checksums for output packets. + Additionally, the GSO library doesn't process IP fragmented packets. + +* **Added the Flow Classification Library.** + + Added an experimental Flow Classification library to provide APIs for DPDK + applications to classify an input packet by matching it against a set of + flow rules. It uses the ``librte_table`` API to manage the flow rules. + + +Resolved Issues +--------------- + +.. This section should contain bug fixes added to the relevant + sections. Sample format: + + * **code/section Fixed issue in the past tense with a full stop.** + + Add a short 1-2 sentence description of the resolved issue in the past + tense. + + The title should contain the code/lib section like a commit message. + + Add the entries in alphabetic order in the relevant sections below. + + This section is a comment. do not overwrite or remove it. + Also, make sure to start the actual text at the margin. + ========================================================= + + +* **Service core fails to call service callback due to atomic lock** + + In a specific configuration of multi-thread unsafe services and service + cores, a service core previously did not correctly release the atomic lock + on the service. This would result in the cores polling the service, but it + looked like another thread was executing the service callback. The logic for + atomic locking of the services has been fixed and refactored for readability. + + +API Changes +----------- + +.. This section should contain API changes. Sample format: + + * Add a short 1-2 sentence description of the API change. Use fixed width + quotes for ``rte_function_names`` or ``rte_struct_names``. Use the past + tense. + + This section is a comment. do not overwrite or remove it. + Also, make sure to start the actual text at the margin. + ========================================================= + +* **Ethdev device name length increased.** + + The size of internal device name has been increased to 64 characters + to allow for storing longer bus specific names. + +* **Removed the Ethdev RTE_ETH_DEV_DETACHABLE flag.** + + Removed the Ethdev ``RTE_ETH_DEV_DETACHABLE`` flag. This flag is not + required anymore, with the new hotplug implementation. It has been removed + from the ether library. Its semantics are now expressed at the bus and PMD + level. + +* **Service cores API updated for usability** + + The service cores API has been changed, removing pointers from the API where + possible, and instead using integer IDs to identify each service. This + simplifies application code, aids debugging, and provides better + encapsulation. A summary of the main changes made is as follows: + + * Services identified by ID not by ``rte_service_spec`` pointer + * Reduced API surface by using ``set`` functions instead of enable/disable + * Reworked ``rte_service_register`` to provide the service ID to registrar + * Reworked start and stop APIs into ``rte_service_runstate_set`` + * Added API to set runstate of service implementation to indicate readiness + +* **The following changes have been made in the mempool library** + + * Moved ``flags`` datatype from ``int`` to ``unsigned int`` for + ``rte_mempool``. + * Removed ``__rte_unused int flag`` param from ``rte_mempool_generic_put`` + and ``rte_mempool_generic_get`` API. + * Added ``flags`` param in ``rte_mempool_xmem_size`` and + ``rte_mempool_xmem_usage``. + * ``rte_mem_phy2mch`` was used in Xen dom0 to obtain the physical address; + remove this API as Xen dom0 support was removed. + +* **Added IOVA aliases related to physical address handling.** + + Some data types, structure members and functions related to physical address + handling are deprecated and have new aliases with IOVA wording. For example: + + * ``phys_addr_t`` can be often replaced by ``rte_iova_t`` of same size. + * ``RTE_BAD_PHYS_ADDR`` is often replaced by ``RTE_BAD_IOVA`` of same value. + * ``rte_memseg.phys_addr`` is aliased with ``rte_memseg.iova_addr``. + * ``rte_mem_virt2phy()`` can often be replaced by ``rte_mem_virt2iova``. + * ``rte_malloc_virt2phy`` is aliased with ``rte_malloc_virt2iova``. + * ``rte_memzone.phys_addr`` is aliased with ``rte_memzone.iova``. + * ``rte_mempool_objhdr.physaddr`` is aliased with + ``rte_mempool_objhdr.iova``. + * ``rte_mempool_memhdr.phys_addr`` is aliased with + ``rte_mempool_memhdr.iova``. + * ``rte_mempool_virt2phy()`` can be replaced by ``rte_mempool_virt2iova()``. + * ``rte_mempool_populate_phys*()`` are aliased with + ``rte_mempool_populate_iova*()`` + * ``rte_mbuf.buf_physaddr`` is aliased with ``rte_mbuf.buf_iova``. + * ``rte_mbuf_data_dma_addr*()`` are aliased with ``rte_mbuf_data_iova*()``. + * ``rte_pktmbuf_mtophys*`` are aliased with ``rte_pktmbuf_iova*()``. + +* **PCI bus API moved outside of the EAL** + + The PCI bus previously implemented within the EAL has been moved. + A first part has been added as an RTE library providing PCI helpers to + parse device locations or other such utilities. + A second part consisting of the actual bus driver has been moved to its + proper subdirectory, without changing its functionalities. + + As such, several PCI-related functions are not exposed by the EAL anymore: + + * ``rte_pci_detach`` + * ``rte_pci_dump`` + * ``rte_pci_ioport_map`` + * ``rte_pci_ioport_read`` + * ``rte_pci_ioport_unmap`` + * ``rte_pci_ioport_write`` + * ``rte_pci_map_device`` + * ``rte_pci_probe`` + * ``rte_pci_probe_one`` + * ``rte_pci_read_config`` + * ``rte_pci_register`` + * ``rte_pci_scan`` + * ``rte_pci_unmap_device`` + * ``rte_pci_unregister`` + * ``rte_pci_write_config`` + + These functions are made available either as part of ``librte_pci`` or + ``librte_bus_pci``. + +* **Moved vdev bus APIs outside of the EAL** + + Moved the following APIs from ``librte_eal`` to ``librte_bus_vdev``: + + * ``rte_vdev_init`` + * ``rte_vdev_register`` + * ``rte_vdev_uninit`` + * ``rte_vdev_unregister`` + +* **Add return value to stats_get dev op API** + + The ``stats_get`` dev op API return value has been changed to be int. + In this way PMDs can return an error value in case of failure at stats + getting process time. + +* **Modified the rte_cryptodev_allocate_driver function.** + + Modified the ``rte_cryptodev_allocate_driver()`` function in the cryptodev + library. An extra parameter ``struct cryptodev_driver *crypto_drv`` has been + added. + +* **Removed virtual device bus specific functions from librte_cryptodev.** + + The functions ``rte_cryptodev_vdev_parse_init_params()`` and + ``rte_cryptodev_vdev_pmd_init()`` have been removed from librte_cryptodev + and have been replaced by non bus specific functions + ``rte_cryptodev_pmd_parse_input_args()`` and ``rte_cryptodev_pmd_create()``. + + The ``rte_cryptodev_create_vdev()`` function was removed to avoid the + dependency on vdev in librte_cryptodev; instead, users can call + ``rte_vdev_init()`` directly. + +* **Removed PCI device bus specific functions from librte_cryptodev.** + + The functions ``rte_cryptodev_pci_generic_probe()`` and + ``rte_cryptodev_pci_generic_remove()`` have been removed from librte_cryptodev + and have been replaced by non bus specific functions + ``rte_cryptodev_pmd_create()`` and ``rte_cryptodev_pmd_destroy()``. + +* **Removed deprecated functions to manage log level or type.** + + The functions ``rte_set_log_level()``, ``rte_get_log_level()``, + ``rte_set_log_type()`` and ``rte_get_log_type()`` have been removed. + + They are respectively replaced by ``rte_log_set_global_level()``, + ``rte_log_get_global_level()``, ``rte_log_set_level()`` and + ``rte_log_get_level()``. + +* **Removed mbuf flags PKT_RX_VLAN_PKT and PKT_RX_QINQ_PKT.** + + The ``mbuf`` flags ``PKT_RX_VLAN_PKT`` and ``PKT_RX_QINQ_PKT`` have + been removed since their behavior was not properly described. + +* **Added mbuf flags PKT_RX_VLAN and PKT_RX_QINQ.** + + Two ``mbuf`` flags have been added to indicate that the VLAN + identifier has been saved in in the ``mbuf`` structure. For instance: + + - If VLAN is not stripped and TCI is saved: ``PKT_RX_VLAN`` + - If VLAN is stripped and TCI is saved: ``PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED`` + +* **Modified the vlan_offload_set_t function prototype in the ethdev library.** + + Modified the ``vlan_offload_set_t`` function prototype in the ethdev + library. The return value has been changed from ``void`` to ``int`` so the + caller can determine whether the backing device supports the operation or if + the operation was successfully performed. + + +ABI Changes +----------- + +.. This section should contain ABI changes. Sample format: + + * Add a short 1-2 sentence description of the ABI change that was announced + in the previous releases and made in this release. Use fixed width quotes + for ``rte_function_names`` or ``rte_struct_names``. Use the past tense. + + This section is a comment. do not overwrite or remove it. + Also, make sure to start the actual text at the margin. + ========================================================= + +* **Extended port_id range.** + + The size of the field ``port_id`` in the ``rte_eth_dev_data`` structure + has changed, as described in the `New Features` section above. + +* **New parameter added to rte_eth_dev.** + + A new parameter ``security_ctx`` has been added to ``rte_eth_dev`` to + support security operations like IPSec inline. + +* **New parameter added to rte_cryptodev.** + + A new parameter ``security_ctx`` has been added to ``rte_cryptodev`` to + support security operations like lookaside crypto. + + +Removed Items +------------- + +.. This section should contain removed items in this release. Sample format: + + * Add a short 1-2 sentence description of the removed item in the past + tense. + + This section is a comment. do not overwrite or remove it. + Also, make sure to start the actual text at the margin. + ========================================================= + +* Xen dom0 in EAL has been removed, as well as the xenvirt PMD and vhost_xen. + +* The crypto performance unit tests have been removed, + replaced by the ``dpdk-test-crypto-perf`` application. + + +Shared Library Versions +----------------------- + +.. Update any library version updated in this release and prepend with a ``+`` + sign, like this: + + librte_acl.so.2 + + librte_cfgfile.so.2 + librte_cmdline.so.2 + + This section is a comment. do not overwrite or remove it. + ========================================================= + + +The libraries prepended with a plus sign were incremented in this version. + +.. code-block:: diff + + librte_acl.so.2 + + librte_bitratestats.so.2 + + librte_bus_dpaa.so.1 + + librte_bus_fslmc.so.1 + + librte_bus_pci.so.1 + + librte_bus_vdev.so.1 + librte_cfgfile.so.2 + librte_cmdline.so.2 + + librte_cryptodev.so.4 + librte_distributor.so.1 + + librte_eal.so.6 + + librte_ethdev.so.8 + + librte_eventdev.so.3 + + librte_flow_classify.so.1 + librte_gro.so.1 + + librte_gso.so.1 + librte_hash.so.2 + librte_ip_frag.so.1 + librte_jobstats.so.1 + librte_kni.so.2 + librte_kvargs.so.1 + librte_latencystats.so.1 + librte_lpm.so.2 + librte_mbuf.so.3 + + librte_mempool.so.3 + librte_meter.so.1 + librte_metrics.so.1 + librte_net.so.1 + + librte_pci.so.1 + + librte_pdump.so.2 + librte_pipeline.so.3 + + librte_pmd_bnxt.so.2 + + librte_pmd_bond.so.2 + + librte_pmd_i40e.so.2 + + librte_pmd_ixgbe.so.2 + librte_pmd_ring.so.2 + + librte_pmd_softnic.so.1 + + librte_pmd_vhost.so.2 + librte_port.so.3 + librte_power.so.1 + librte_reorder.so.1 + librte_ring.so.1 + librte_sched.so.1 + + librte_security.so.1 + + librte_table.so.3 + librte_timer.so.1 + librte_vhost.so.3 + + +Tested Platforms +---------------- + +.. This section should contain a list of platforms that were tested with this + release. + + The format is: + + * platform with combinations + + * List of CPU + * List of OS + * List of devices + * Other relevant details... + + This section is a comment. do not overwrite or remove it. + Also, make sure to start the actual text at the margin. + ========================================================= + +* Intel(R) platforms with Intel(R) NICs combinations + + * CPU + + * Intel(R) Atom(TM) CPU C2758 @ 2.40GHz + * Intel(R) Xeon(R) CPU D-1540 @ 2.00GHz + * Intel(R) Xeon(R) CPU D-1541 @ 2.10GHz + * Intel(R) Xeon(R) CPU E5-4667 v3 @ 2.00GHz + * Intel(R) Xeon(R) CPU E5-2680 v2 @ 2.80GHz + * Intel(R) Xeon(R) CPU E5-2699 v3 @ 2.30GHz + * Intel(R) Xeon(R) CPU E5-2695 v4 @ 2.10GHz + * Intel(R) Xeon(R) CPU E5-2658 v2 @ 2.40GHz + * Intel(R) Xeon(R) CPU E5-2658 v3 @ 2.20GHz + + * OS: + + * CentOS 7.2 + * Fedora 25 + * Fedora 26 + * FreeBSD 11 + * Red Hat Enterprise Linux Server release 7.3 + * SUSE Enterprise Linux 12 + * Wind River Linux 8 + * Ubuntu 16.04 + * Ubuntu 16.10 + + * NICs: + + * Intel(R) 82599ES 10 Gigabit Ethernet Controller + + * Firmware version: 0x61bf0001 + * Device id (pf/vf): 8086:10fb / 8086:10ed + * Driver version: 5.2.3 (ixgbe) + + * Intel(R) Corporation Ethernet Connection X552/X557-AT 10GBASE-T + + * Firmware version: 0x800003e7 + * Device id (pf/vf): 8086:15ad / 8086:15a8 + * Driver version: 4.4.6 (ixgbe) + + * Intel(R) Ethernet Converged Network Adapter X710-DA4 (4x10G) + + * Firmware version: 6.01 0x80003205 + * Device id (pf/vf): 8086:1572 / 8086:154c + * Driver version: 2.1.26 (i40e) + + * Intel(R) Ethernet Converged Network Adapter X710-DA2 (2x10G) + + * Firmware version: 6.01 0x80003204 + * Device id (pf/vf): 8086:1572 / 8086:154c + * Driver version: 2.1.26 (i40e) + + * Intel(R) Ethernet Converged Network Adapter XXV710-DA2 (2x25G) + + * Firmware version: 6.01 0x80003221 + * Device id (pf/vf): 8086:158b + * Driver version: 2.1.26 (i40e) + + * Intel(R) Ethernet Converged Network Adapter XL710-QDA2 (2X40G) + + * Firmware version: 6.01 0x8000321c + * Device id (pf/vf): 8086:1583 / 8086:154c + * Driver version: 2.1.26 (i40e) + + * Intel(R) Corporation I350 Gigabit Network Connection + + * Firmware version: 1.63, 0x80000dda + * Device id (pf/vf): 8086:1521 / 8086:1520 + * Driver version: 5.3.0-k (igb) + +* Intel(R) platforms with Mellanox(R) NICs combinations + + * Platform details: + + * Intel(R) Xeon(R) CPU E5-2697A v4 @ 2.60GHz + * Intel(R) Xeon(R) CPU E5-2697 v3 @ 2.60GHz + * Intel(R) Xeon(R) CPU E5-2680 v2 @ 2.80GHz + * Intel(R) Xeon(R) CPU E5-2650 v4 @ 2.20GHz + * Intel(R) Xeon(R) CPU E5-2640 @ 2.50GHz + * Intel(R) Xeon(R) CPU E5-2620 v4 @ 2.10GHz + + * OS: + + * Red Hat Enterprise Linux Server release 7.3 (Maipo) + * Red Hat Enterprise Linux Server release 7.2 (Maipo) + * Ubuntu 16.10 + * Ubuntu 16.04 + * Ubuntu 14.04 + + * MLNX_OFED: 4.2-1.0.0.0 + + * NICs: + + * Mellanox(R) ConnectX(R)-3 Pro 40G MCX354A-FCC_Ax (2x40G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1007 + * Firmware version: 2.42.5000 + + * Mellanox(R) ConnectX(R)-4 10G MCX4111A-XCAT (1x10G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1013 + * Firmware version: 12.21.1000 + + * Mellanox(R) ConnectX(R)-4 10G MCX4121A-XCAT (2x10G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1013 + * Firmware version: 12.21.1000 + + * Mellanox(R) ConnectX(R)-4 25G MCX4111A-ACAT (1x25G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1013 + * Firmware version: 12.21.1000 + + * Mellanox(R) ConnectX(R)-4 25G MCX4121A-ACAT (2x25G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1013 + * Firmware version: 12.21.1000 + + * Mellanox(R) ConnectX(R)-4 40G MCX4131A-BCAT/MCX413A-BCAT (1x40G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1013 + * Firmware version: 12.21.1000 + + * Mellanox(R) ConnectX(R)-4 40G MCX415A-BCAT (1x40G) + + * Host interface: PCI Express 3.0 x16 + * Device ID: 15b3:1013 + * Firmware version: 12.21.1000 + + * Mellanox(R) ConnectX(R)-4 50G MCX4131A-GCAT/MCX413A-GCAT (1x50G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1013 + * Firmware version: 12.21.1000 + + * Mellanox(R) ConnectX(R)-4 50G MCX414A-BCAT (2x50G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1013 + * Firmware version: 12.21.1000 + + * Mellanox(R) ConnectX(R)-4 50G MCX415A-GCAT/MCX416A-BCAT/MCX416A-GCAT + (2x50G) + + * Host interface: PCI Express 3.0 x16 + * Device ID: 15b3:1013 + * Firmware version: 12.21.1000 + + * Mellanox(R) ConnectX(R)-4 50G MCX415A-CCAT (1x100G) + + * Host interface: PCI Express 3.0 x16 + * Device ID: 15b3:1013 + * Firmware version: 12.21.1000 + + * Mellanox(R) ConnectX(R)-4 100G MCX416A-CCAT (2x100G) + + * Host interface: PCI Express 3.0 x16 + * Device ID: 15b3:1013 + * Firmware version: 12.21.1000 + + * Mellanox(R) ConnectX(R)-4 Lx 10G MCX4121A-XCAT (2x10G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1015 + * Firmware version: 14.21.1000 + + * Mellanox(R) ConnectX(R)-4 Lx 25G MCX4121A-ACAT (2x25G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1015 + * Firmware version: 14.21.1000 + + * Mellanox(R) ConnectX(R)-5 100G MCX556A-ECAT (2x100G) + + * Host interface: PCI Express 3.0 x16 + * Device ID: 15b3:1017 + * Firmware version: 16.21.1000 + + * Mellanox(R) ConnectX-5 Ex EN 100G MCX516A-CDAT (2x100G) + + * Host interface: PCI Express 4.0 x16 + * Device ID: 15b3:1019 + * Firmware version: 16.21.1000 + +* ARM platforms with Mellanox(R) NICs combinations + + * Platform details: + + * Qualcomm ARM 1.1 2500MHz + + * OS: + + * Ubuntu 16.04 + + * MLNX_OFED: 4.2-1.0.0.0 + + * NICs: + + * Mellanox(R) ConnectX(R)-4 Lx 25G MCX4121A-ACAT (2x25G) + + * Host interface: PCI Express 3.0 x8 + * Device ID: 15b3:1015 + * Firmware version: 14.21.1000 + + * Mellanox(R) ConnectX(R)-5 100G MCX556A-ECAT (2x100G) + + * Host interface: PCI Express 3.0 x16 + * Device ID: 15b3:1017 + * Firmware version: 16.21.1000 + +Fixes in 17.11 LTS Release +-------------------------- + +17.11.1 +~~~~~~~ + +* app/procinfo: add compilation option in config +* app/testpmd: fix crash of txonly with multiple segments +* app/testpmd: fix flow director filter +* app/testpmd: fix flowgen forwarding offload flags +* app/testpmd: fix invalid Rx queue number setting +* app/testpmd: fix invalid Tx queue number setting +* app/testpmd: fix port configuration print +* app/testpmd: fix port id allocation +* app/testpmd: fix port index in RSS forward config +* app/testpmd: fix port topology in RSS forward config +* app/testpmd: fix port validation +* app/testpmd: remove xenvirt again +* bus/dpaa: fix ARM big endian build +* bus/dpaa: fix build when assert enabled +* bus/dpaa: fix default IOVA mode +* bus/fslmc: fix build with latest glibc +* bus/fslmc: fix the cplusplus macro closure +* bus/pci: fix interrupt handler type +* bus/pci: forbid IOVA mode if IOMMU address width too small +* bus/vdev: continue probing after a device failure +* cmdline: avoid garbage in unused fields of parsed result +* cmdline: fix dynamic tokens parsing +* cryptodev: add missing CPU flag string +* cryptodev: fix function prototype +* cryptodev: fix session pointer cast +* crypto/dpaa2_sec: fix enum conversion for GCM +* crypto: fix pedantic compilation +* crypto/qat: fix allocation check and leak +* crypto/qat: fix null auth algo overwrite +* crypto/qat: fix out-of-bounds access +* crypto/qat: fix parameter type +* crypto/scheduler: fix strncpy +* doc: fix format in OpenSSL installation guide +* doc: fix lists of supported crypto algorithms +* drivers: change the deprecated memseg physaddr to IOVA +* eal/arm64: remove the braces in memory barrier macros +* eal/ppc64: revert arch-specific TSC freq query +* eal/ppc: remove the braces in memory barrier macros +* ethdev: fix link autonegotiation value +* ethdev: fix missing imissed counter in xstats +* ethdev: fix port data reset timing +* ethdev: fix port id allocation +* eventdev: fix doxygen comments +* eventdev: set error code in port link/unlink functions +* event/octeontx: fix Rx adapter port id mapping +* event/sw: fix debug logging config option +* event/sw: fix queue memory leak and multi-link bug +* examples/bond: check mbuf allocation +* examples/bond: fix vdev name +* examples/ip_pipeline: fix timer period unit +* examples/ipsec-secgw: fix corner case for SPI value +* examples/ipsec-secgw: fix missing ingress flow attribute +* examples/ipsec-secgw: fix SPI byte order in flow item +* examples/ipsec-secgw: fix usage of incorrect port +* examples/l3fwd-power: fix frequency detection +* examples/l3fwd-power: fix Rx without interrupt +* examples/vhost: fix sending ARP packet to self +* examples/vhost: fix startup check +* flow_classify: fix ISO C in exported header +* igb_uio: allow multi-process access +* keepalive: fix state alignment +* kni: fix build dependency +* kni: fix build with kernel 4.15 +* lib: fix missing includes in exported headers +* log: fix memory leak in regexp level set +* lpm: fix ARM big endian build +* malloc: fix end for bounded elements +* malloc: protect stats with lock +* mbuf: fix NULL freeing when debug enabled +* mbuf: fix performance of freeing with non atomic refcnt +* member: fix ISO C in exported header +* member: fix memory leak on error +* mempool: fix first memory area notification +* mempool: fix physical contiguous check +* mempool/octeontx: fix improper memory barrier +* mempool/octeontx: fix memory area registration +* mempool/octeontx: fix natural alignment being optimized out +* memzone: fix leak on allocation error +* mk: fix external build +* mk: remove TILE-Gx machine type +* mk: support renamed Makefile in external project +* net/bnxt: fix check for ether type +* net/bnxt: fix double increment of idx during Tx ring alloc +* net/bnxt: fix duplicate filter pattern creation error +* net/bnxt: fix duplicate pattern for 5tuple filter +* net/bnxt: fix group info usage +* net/bnxt: fix link speed setting with autoneg off +* net/bnxt: fix number of pools for RSS +* net/bnxt: fix return code in MAC address set +* net/bnxt: fix Rx checksum flags +* net/bnxt: fix size of Tx ring in HW +* net/bnxt: free the aggregation ring +* net/bnxt: parse checksum offload flags +* net/bonding: check error of MAC address setting +* net/bonding: fix activated slave in 8023ad mode +* net/bonding: fix bonding in 8023ad mode +* net/bonding: fix setting slave MAC addresses +* net/dpaa: fix FW version code +* net/dpaa: fix potential memory leak +* net/dpaa: fix the mbuf packet type if zero +* net/dpaa: fix uninitialized and unused variables +* net/e1000: fix null pointer check +* net/e1000: fix VF Rx interrupt enabling +* net/ena: do not set Tx L4 offloads in Rx path +* net/enic: fix crash due to static max number of queues +* net/enic: fix L4 Rx ptype comparison +* net/failsafe: fix invalid free +* net/failsafe: fix Rx safe check compiler hint +* net: fix ESP header byte ordering definition +* net/fm10k: fix logical port delete +* net/i40e: add debug logs when writing global registers +* net/i40e: add FDIR NVGRE parameter check +* net/i40e: check multi-driver option parsing +* net/i40e: exclude LLDP packet count +* net/i40e: fix ARM big endian build +* net/i40e: fix FDIR input set conflict +* net/i40e: fix FDIR rule confiliction issue +* net/i40e: fix flag for MAC address write +* net/i40e: fix flow director Rx resource defect +* net/i40e: fix interrupt conflict with multi-driver +* net/i40e: fix ISO C in exported header +* net/i40e: fix memory leak +* net/i40e: fix multiple DDP packages conflict +* net/i40e: fix multiple driver support +* net/i40e: fix packet type for X722 +* net/i40e: fix port segmentation fault when restart +* net/i40e: fix Rx interrupt +* net/i40e: fix setting MAC address of VF +* net/i40e: fix setting of MAC address on i40evf +* net/i40e: fix VF reset stats crash +* net/i40e: fix VF Rx interrupt enabling +* net/i40e: fix VLAN offload setting +* net/i40e: fix VLAN offload setting issue +* net/i40e: fix VSI MAC filter on primary address change +* net/i40e: warn when writing global registers +* net/igb: fix Tx queue number assignment +* net/ixgbe: fix ARM big endian build +* net/ixgbe: fix max queue number for VF +* net/ixgbe: fix parsing FDIR NVGRE issue +* net/ixgbe: fix reset error handling +* net/ixgbe: fix the failure of number of Tx queue check +* net/ixgbe: fix tunnel filter fail problem +* net/ixgbe: fix VF Rx interrupt enabling +* net/ixgbe: fix wrong PBA setting +* net/mlx4: fix drop flow resources leak +* net/mlx4: fix Rx offload non-fragmented indication +* net/mlx4: fix Tx packet drop application report +* net/mlx4: fix unnecessary include +* net/mlx4: revert workaround for broken Verbs +* net/mlx5: cleanup allocation of ethtool stats +* net/mlx5: fix calculation of flow ID flag +* net/mlx5: fix deadlock of link status alarm +* net/mlx5: fix flow item validation +* net/mlx5: fix flow priority on queue action +* net/mlx5: fix flow RSS configuration +* net/mlx5: fix handling link status event +* net/mlx5: fix HW checksum offload for outer IP +* net/mlx5: fix link state on device start +* net/mlx5: fix memory region boundary checks +* net/mlx5: fix memory region cache last index +* net/mlx5: fix memory region cache lookup +* net/mlx5: fix memory region lookup +* net/mlx5: fix Memory Region registration +* net/mlx5: fix missing attribute size for drop action +* net/mlx5: fix missing RSS capability +* net/mlx5: fix overflow of Memory Region cache +* net/mlx5: fix overwriting bit-fields in SW Rx queue +* net/mlx5: fix port stop by verify flows are still present +* net/mlx5: fix return value of start operation +* net/mlx5: fix RSS key configuration +* net/mlx5: fix secondary process verification +* net/mlx5: fix Tx checksum offloads +* net/mlx5: fix UAR remapping on non configured queues +* net/mlx5: fix un-supported RSS hash fields use +* net/mlx5: fix VLAN configuration after port stop +* net/mlx5: remove parser/flow drop queue +* net/mlx5: use PCI address as port name +* net/mrvl: fix HIF objects allocation +* net/mrvl: fix multiple probe +* net/mrvl: fix oversize bpool handling +* net/mrvl: fix shadow queue tail and size calculations +* net/mrvl: keep shadow Txqs inside PMD Txq +* net/nfp: fix CRC strip check behaviour +* net/nfp: fix jumbo settings +* net/nfp: fix MTU settings +* net/octeontx: add channel to port id mapping +* net/pcap: fix the NUMA id display in logs +* net/qede/base: fix VF LRO tunnel configuration +* net/qede: check tunnel L3 header +* net/qede: fix clearing of queue stats +* net/qede: fix few log messages +* net/qede: fix MTU set and max Rx length +* net/qede: fix to enable LRO over tunnels +* net/qede: fix to reject config with no Rx queue +* net/qede: fix tunnel header size in Tx BD configuration +* net/qede: replace config option with run-time arg +* net/sfc: do not hold management event queue lock while MCDI +* net/sfc: fix DMA memory leak after kvarg processing failure +* net/sfc: fix flow RSS check in error handling +* net/sfc: fix incorrect bitwise ORing of L3/L4 packet types +* net/sfc: fix initialization of flow structure +* net/sfc: fix label name to be consistent +* net/sfc: fix main MAC address handling +* net/sfc: fix multicast address list copy memory leak +* net/sfc: stop periodic DMA if MAC stats upload fails +* net/szedata2: fix check of mmap return value +* net/tap: fix cleanup on allocation failure +* net/tap: remove unused kernel version definitions +* net/thunderx: fix multi segment Tx function return +* net/virtio: fix incorrect cast +* net/virtio: fix memory leak when reinitializing device +* net/virtio: fix queue flushing with vector Rx enabled +* net/virtio: fix Rx and Tx handler selection for ARM32 +* net/virtio: fix typo in LRO support +* net/virtio: fix vector Rx flushing +* net/virtio-user: fix crash as features change +* pdump: fix error check when creating/canceling thread +* pmdinfogen: fix cross compilation for ARM big endian +* security: fix device operation type +* security: fix enum start value +* security: fix pedantic compilation +* service: fix lcore role after delete +* service: fix number mapped cores count +* service: fix possible mem leak on initialize +* service: fix service core launch +* test/bitmap: fix memory leak +* test/crypto: fix missing include +* test/eventdev: use CPU event type +* test/memzone: fix freeing test +* test/memzone: fix NULL freeing +* test/memzone: fix wrong test +* test: register test as failed if setup failed +* test/reorder: fix memory leak +* test/ring: fix memory leak +* test/ring_perf: fix memory leak +* test/table: fix memory leak +* test/table: fix uninitialized parameter +* test/timer_perf: fix memory leak +* timer: fix reset on service cores +* usertools/devbind: fix kernel module reporting +* vfio: fix enabled check on error +* vhost: fix crash +* vhost: fix dequeue zero copy with virtio1 +* vhost: fix error code check when creating thread +* vhost: fix IOTLB pool out-of-memory handling +* vhost: fix mbuf free +* vhost: protect active rings from async ring changes +* vhost: remove pending IOTLB entry if miss request failed + +17.11.2 +~~~~~~~ + +* examples/vhost: move to safe GPA translation API +* examples/vhost_scsi: move to safe GPA translation API +* vhost: add support for non-contiguous indirect descs tables (fixes CVE-2018-1059) +* vhost: check all range is mapped when translating GPAs (fixes CVE-2018-1059) +* vhost: deprecate unsafe GPA translation API (fixes CVE-2018-1059) +* vhost: ensure all range is mapped when translating QVAs (fixes CVE-2018-1059) +* vhost: fix indirect descriptors table translation size (fixes CVE-2018-1059) +* vhost: handle virtually non-contiguous buffers in Rx (fixes CVE-2018-1059) +* vhost: handle virtually non-contiguous buffers in Rx-mrg (fixes CVE-2018-1059) +* vhost: handle virtually non-contiguous buffers in Tx (fixes CVE-2018-1059) +* vhost: introduce safe API for GPA translation (fixes CVE-2018-1059) diff --git a/dpdk/doc/guides/rel_notes/supported_os.rst b/dpdk/doc/guides/rel_notes/supported_os.rst deleted file mode 100644 index 92218c18..00000000 --- a/dpdk/doc/guides/rel_notes/supported_os.rst +++ /dev/null @@ -1,49 +0,0 @@ -.. BSD LICENSE - Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in - the documentation and/or other materials provided with the - distribution. - * Neither the name of Intel Corporation nor the names of its - contributors may be used to endorse or promote products derived - from this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -Supported Operating Systems -=========================== - -The following Linux distributions were successfully used to compiler or run DPDK. - -* FreeBSD 10 - -* Fedora release 20 - -* Ubuntu 14.04 LTS - -* Wind River Linux 6 - -* Red Hat Enterprise Linux 6.5 - -* SUSE Enterprise Linux 11 SP3 - -These distributions may need additional packages that are not installed by default, or a specific kernel. -Refer to the :ref:`Linux guide ` and :ref:`FreeBSD guide ` for details. diff --git a/dpdk/doc/guides/sample_app_ug/cmd_line.rst b/dpdk/doc/guides/sample_app_ug/cmd_line.rst index 02a295ff..eabbac15 100644 --- a/dpdk/doc/guides/sample_app_ug/cmd_line.rst +++ b/dpdk/doc/guides/sample_app_ug/cmd_line.rst @@ -68,26 +68,9 @@ There are three simple commands: Compiling the Application ------------------------- -#. Go to example directory: +To compile the sample application see :doc:`compiling` - .. code-block:: console - - export RTE_SDK=/path/to/rte_sdk - cd ${RTE_SDK}/examples/cmdline - -#. Set the target (a default target is used if not specified). For example: - - .. code-block:: console - - export RTE_TARGET=x86_64-native-linuxapp-gcc - - Refer to the *DPDK Getting Started Guide* for possible RTE_TARGET values. - -#. Build the application: - - .. code-block:: console - - make +The application is located in the ``cmd_line`` sub-directory. Running the Application ----------------------- @@ -96,7 +79,7 @@ To run the application in linuxapp environment, issue the following command: .. code-block:: console - $ ./build/cmdline -c f -n 4 + $ ./build/cmdline -l 0-3 -n 4 Refer to the *DPDK Getting Started Guide* for general information on running applications and the Environment Abstraction Layer (EAL) options. diff --git a/dpdk/doc/guides/sample_app_ug/compiling.rst b/dpdk/doc/guides/sample_app_ug/compiling.rst new file mode 100644 index 00000000..8bedaa79 --- /dev/null +++ b/dpdk/doc/guides/sample_app_ug/compiling.rst @@ -0,0 +1,122 @@ + .. BSD LICENSE + Copyright(c) 2015 Intel Corporation. All rights reserved. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Compiling the Sample Applications +================================= + +This section explains how to compile the DPDK sample applications. + +To compile all the sample applications +-------------------------------------- + + +Set the path to DPDK source code if its not set: + + .. code-block:: console + + export RTE_SDK=/path/to/rte_sdk + +Go to DPDK source: + + .. code-block:: console + + cd $RTE_SDK + +Build DPDK: + + .. code-block:: console + + make defconfig + make + +Build the sample applications: + + .. code-block:: console + + export RTE_TARGET=build + make -C examples + +For other possible ``RTE_TARGET`` values and additional information on +compiling see +:ref:`Compiling DPDK on Linux ` or +:ref:`Compiling DPDK on FreeBSD `. +Applications are output to: ``$RTE_SDK/examples/app-dir/build`` or +``$RTE_SDK/examples/app-dir/$RTE_TARGET``. + + +In the example above the compiled application is written to the ``build`` subdirectory. +To have the applications written to a different location, +the ``O=/path/to/build/directory`` option may be specified in the make command. + + .. code-block:: console + + make O=/tmp + +To build the applications for debugging use the ``DEBUG`` option. +This option adds some extra flags, disables compiler optimizations and +sets verbose output. + + .. code-block:: console + + make DEBUG=1 + + +To compile a single application +------------------------------- + +Set the path to DPDK source code: + + .. code-block:: console + + export RTE_SDK=/path/to/rte_sdk + +Go to DPDK source: + + .. code-block:: console + + cd $RTE_SDK + +Build DPDK: + + .. code-block:: console + + make defconfig + make + +Go to the sample application directory. Unless otherwise specified the sample +applications are located in ``$RTE_SDK/examples/``. + + +Build the application: + + .. code-block:: console + + export RTE_TARGET=build + make diff --git a/dpdk/doc/guides/sample_app_ug/dist_app.rst b/dpdk/doc/guides/sample_app_ug/dist_app.rst index e242748f..45a51313 100644 --- a/dpdk/doc/guides/sample_app_ug/dist_app.rst +++ b/dpdk/doc/guides/sample_app_ug/dist_app.rst @@ -53,30 +53,12 @@ generator as shown in the figure below. Performance Benchmarking Setup (Basic Environment) - Compiling the Application ------------------------- -#. Go to the sample application directory: +To compile the sample application see :doc:`compiling`. - .. code-block:: console - - export RTE_SDK=/path/to/rte_sdk - cd ${RTE_SDK}/examples/distributor - -#. Set the target (a default target is used if not specified). For example: - - .. code-block:: console - - export RTE_TARGET=x86_64-native-linuxapp-gcc - - See the DPDK Getting Started Guide for possible RTE_TARGET values. - -#. Build the application: - - .. code-block:: console - - make +The application is located in the ``distributor`` sub-directory. Running the Application ----------------------- @@ -96,7 +78,7 @@ Running the Application .. code-block:: console - $ ./build/distributor_app -c 0x4003fe -n 4 -- -p f + $ ./build/distributor_app -l 1-9,22 -n 4 -- -p f #. Refer to the DPDK Getting Started Guide for general information on running applications and the Environment Abstraction Layer (EAL) options. @@ -104,33 +86,35 @@ Running the Application Explanation ----------- -The distributor application consists of three types of threads: a receive -thread (lcore_rx()), a set of worker threads(lcore_worker()) -and a transmit thread(lcore_tx()). How these threads work together is shown -in :numref:`figure_dist_app` below. The main() function launches threads of these three types. -Each thread has a while loop which will be doing processing and which is -terminated only upon SIGINT or ctrl+C. The receive and transmit threads -communicate using a software ring (rte_ring structure). +The distributor application consists of four types of threads: a receive +thread (``lcore_rx()``), a distributor thread (``lcore_dist()``), a set of +worker threads (``lcore_worker()``), and a transmit thread(``lcore_tx()``). +How these threads work together is shown in :numref:`figure_dist_app` below. +The ``main()`` function launches threads of these four types. Each thread +has a while loop which will be doing processing and which is terminated +only upon SIGINT or ctrl+C. -The receive thread receives the packets using rte_eth_rx_burst() and gives -them to the distributor (using rte_distributor_process() API) which will -be called in context of the receive thread itself. The distributor distributes -the packets to workers threads based on the tagging of the packet - -indicated by the hash field in the mbuf. For IP traffic, this field is -automatically filled by the NIC with the "usr" hash value for the packet, -which works as a per-flow tag. +The receive thread receives the packets using ``rte_eth_rx_burst()`` and will +enqueue them to an rte_ring. The distributor thread will dequeue the packets +from the ring and assign them to workers (using ``rte_distributor_process()`` API). +This assignment is based on the tag (or flow ID) of the packet - indicated by +the hash field in the mbuf. For IP traffic, this field is automatically filled +by the NIC with the "usr" hash value for the packet, which works as a per-flow +tag. The distributor thread communicates with the worker threads using a +cache-line swapping mechanism, passing up to 8 mbuf pointers at a time +(one cache line) to each worker. More than one worker thread can exist as part of the application, and these worker threads do simple packet processing by requesting packets from the distributor, doing a simple XOR operation on the input port mbuf field (to indicate the output port which will be used later for packet transmission) -and then finally returning the packets back to the distributor in the RX thread. +and then finally returning the packets back to the distributor thread. -Meanwhile, the receive thread will call the distributor api -rte_distributor_returned_pkts() to get the packets processed, and will enqueue -them to a ring for transfer to the TX thread for transmission on the output port. -The transmit thread will dequeue the packets from the ring and transmit them on -the output port specified in packet mbuf. +The distributor thread will then call the distributor api +``rte_distributor_returned_pkts()`` to get the processed packets, and will enqueue +them to another rte_ring for transfer to the TX thread for transmission on the +output port. The transmit thread will dequeue the packets from the ring and +transmit them on the output port specified in packet mbuf. Users who wish to terminate the running of the application have to press ctrl+C (or send SIGINT to the app). Upon this signal, a signal handler provided @@ -153,8 +137,10 @@ the line "#define DEBUG" defined in start of the application in main.c to enable Statistics ---------- -Upon SIGINT (or) ctrl+C, the print_stats() function displays the count of packets -processed at the different stages in the application. +The main function will print statistics on the console every second. These +statistics include the number of packets enqueued and dequeued at each stage +in the application, and also key statistics per worker, including how many +packets of each burst size (1-8) were sent to each worker thread. Application Initialization -------------------------- diff --git a/dpdk/doc/guides/sample_app_ug/ethtool.rst b/dpdk/doc/guides/sample_app_ug/ethtool.rst index 4d1697e8..6dd11dc9 100644 --- a/dpdk/doc/guides/sample_app_ug/ethtool.rst +++ b/dpdk/doc/guides/sample_app_ug/ethtool.rst @@ -40,28 +40,9 @@ is based upon a simple L2 frame reflector. Compiling the Application ------------------------- -To compile the application: +To compile the sample application see :doc:`compiling`. -#. Go to the sample application directory: - - .. code-block:: console - - export RTE_SDK=/path/to/rte_sdk - cd ${RTE_SD}/examples/ethtool - -#. Set the target (a default target is used if not specified). For example: - - .. code-block:: console - - export RTE_TARGET=x86_64-native-linuxapp-gcc - - See the *DPDK Getting Started Guide* for possible RTE_TARGET values. - -#. Build the application: - - .. code-block:: console - - make +The application is located in the ``ethtool`` sub-directory. Running the Application ----------------------- diff --git a/dpdk/doc/guides/sample_app_ug/eventdev_pipeline_sw_pmd.rst b/dpdk/doc/guides/sample_app_ug/eventdev_pipeline_sw_pmd.rst new file mode 100644 index 00000000..01a5f9b2 --- /dev/null +++ b/dpdk/doc/guides/sample_app_ug/eventdev_pipeline_sw_pmd.rst @@ -0,0 +1,173 @@ + +.. BSD LICENSE + Copyright(c) 2017 Intel Corporation. All rights reserved. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Eventdev Pipeline SW PMD Sample Application +=========================================== + +The eventdev pipeline sample application is a sample app that demonstrates +the usage of the eventdev API using the software PMD. It shows how an +application can configure a pipeline and assign a set of worker cores to +perform the processing required. + +The application has a range of command line arguments allowing it to be +configured for various numbers worker cores, stages,queue depths and cycles per +stage of work. This is useful for performance testing as well as quickly testing +a particular pipeline configuration. + + +Compiling the Application +------------------------- + +To compile the sample application see :doc:`compiling`. + +The application is located in the ``examples`` sub-directory. + + + +Running the Application +----------------------- + +The application has a lot of command line options. This allows specification of +the eventdev PMD to use, and a number of attributes of the processing pipeline +options. + +An example eventdev pipeline running with the software eventdev PMD using +these settings is shown below: + + * ``-r1``: core mask 0x1 for RX + * ``-t1``: core mask 0x1 for TX + * ``-e4``: core mask 0x4 for the software scheduler + * ``-w FF00``: core mask for worker cores, 8 cores from 8th to 16th + * ``-s4``: 4 atomic stages + * ``-n0``: process infinite packets (run forever) + * ``-c32``: worker dequeue depth of 32 + * ``-W1000``: do 1000 cycles of work per packet in each stage + * ``-D``: dump statistics on exit + +.. code-block:: console + + ./build/eventdev_pipeline_sw_pmd --vdev event_sw0 -- -r1 -t1 -e4 -w FF00 -s4 -n0 -c32 -W1000 -D + +The application has some sanity checking built-in, so if there is a function +(eg; the RX core) which doesn't have a cpu core mask assigned, the application +will print an error message: + +.. code-block:: console + + Core part of pipeline was not assigned any cores. This will stall the + pipeline, please check core masks (use -h for details on setting core masks): + rx: 0 + tx: 1 + +Configuration of the eventdev is covered in detail in the programmers guide, +see the Event Device Library section. + + +Observing the Application +------------------------- + +At runtime the eventdev pipeline application prints out a summary of the +configuration, and some runtime statistics like packets per second. On exit the +worker statistics are printed, along with a full dump of the PMD statistics if +required. The following sections show sample output for each of the output +types. + +Configuration +~~~~~~~~~~~~~ + +This provides an overview of the pipeline, +scheduling type at each stage, and parameters to options such as how many +flows to use and what eventdev PMD is in use. See the following sample output +for details: + +.. code-block:: console + + Config: + ports: 2 + workers: 8 + packets: 0 + priorities: 1 + Queue-prio: 0 + qid0 type: atomic + Cores available: 44 + Cores used: 10 + Eventdev 0: event_sw + Stages: + Stage 0, Type Atomic Priority = 128 + Stage 1, Type Atomic Priority = 128 + Stage 2, Type Atomic Priority = 128 + Stage 3, Type Atomic Priority = 128 + +Runtime +~~~~~~~ + +At runtime, the statistics of the consumer are printed, stating the number of +packets received, runtime in milliseconds, average mpps, and current mpps. + +.. code-block:: console + + # consumer RX= xxxxxxx, time yyyy ms, avg z.zzz mpps [current w.www mpps] + +Shutdown +~~~~~~~~ + +At shutdown, the application prints the number of packets received and +transmitted, and an overview of the distribution of work across worker cores. + +.. code-block:: console + + Signal 2 received, preparing to exit... + worker 12 thread done. RX=4966581 TX=4966581 + worker 13 thread done. RX=4963329 TX=4963329 + worker 14 thread done. RX=4953614 TX=4953614 + worker 0 thread done. RX=0 TX=0 + worker 11 thread done. RX=4970549 TX=4970549 + worker 10 thread done. RX=4986391 TX=4986391 + worker 9 thread done. RX=4970528 TX=4970528 + worker 15 thread done. RX=4974087 TX=4974087 + worker 8 thread done. RX=4979908 TX=4979908 + worker 2 thread done. RX=0 TX=0 + + Port Workload distribution: + worker 0 : 12.5 % (4979876 pkts) + worker 1 : 12.5 % (4970497 pkts) + worker 2 : 12.5 % (4986359 pkts) + worker 3 : 12.5 % (4970517 pkts) + worker 4 : 12.5 % (4966566 pkts) + worker 5 : 12.5 % (4963297 pkts) + worker 6 : 12.5 % (4953598 pkts) + worker 7 : 12.5 % (4974055 pkts) + +To get a full dump of the state of the eventdev PMD, pass the ``-D`` flag to +this application. When the app is terminated using ``Ctrl+C``, the +``rte_event_dev_dump()`` function is called, resulting in a dump of the +statistics that the PMD provides. The statistics provided depend on the PMD +used, see the Event Device Drivers section for a list of eventdev PMDs. diff --git a/dpdk/doc/guides/sample_app_ug/exception_path.rst b/dpdk/doc/guides/sample_app_ug/exception_path.rst index 161b6e0f..d7f45e6d 100644 --- a/dpdk/doc/guides/sample_app_ug/exception_path.rst +++ b/dpdk/doc/guides/sample_app_ug/exception_path.rst @@ -58,28 +58,9 @@ To make throughput measurements, kernel bridges must be setup to forward data be Compiling the Application ------------------------- -#. Go to example directory: +To compile the sample application see :doc:`compiling`. - .. code-block:: console - - export RTE_SDK=/path/to/rte_sdk - cd ${RTE_SDK}/examples/exception_path - -#. Set the target (a default target will be used if not specified). - For example: - - .. code-block:: console - - export RTE_TARGET=x86_64-native-linuxapp-gcc - -This application is intended as a linuxapp only. -See the *DPDK Getting Started Guide* for possible RTE_TARGET values. - -#. Build the application: - - .. code-block:: console - - make +The application is located in the ``exception_path`` sub-directory. Running the Application ----------------------- @@ -102,7 +83,7 @@ Refer to the *DPDK Getting Started Guide* for general information on running app and the Environment Abstraction Layer (EAL) options. The number of bits set in each bitmask must be the same. -The coremask -c parameter of the EAL options should include IN_CORES and OUT_CORES. +The coremask -c or the corelist -l parameter of the EAL options should include IN_CORES and OUT_CORES. The same bit must not be set in IN_CORES and OUT_CORES. The affinities between ports and cores are set beginning with the least significant bit of each mask, that is, the port represented by the lowest bit in PORTMASK is read from by the core represented by the lowest bit in IN_CORES, @@ -112,7 +93,7 @@ For example to run the application with two ports and four cores: .. code-block:: console - ./build/exception_path -c f -n 4 -- -p 3 -i 3 -o c + ./build/exception_path -l 0-3 -n 4 -- -p 3 -i 3 -o c Getting Statistics ~~~~~~~~~~~~~~~~~~ diff --git a/dpdk/doc/guides/sample_app_ug/flow_classify.rst b/dpdk/doc/guides/sample_app_ug/flow_classify.rst new file mode 100644 index 00000000..bc12b87d --- /dev/null +++ b/dpdk/doc/guides/sample_app_ug/flow_classify.rst @@ -0,0 +1,575 @@ +.. BSD LICENSE + Copyright(c) 2017 Intel Corporation. All rights reserved. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Flow Classify Sample Application +================================ + +The Flow Classify sample application is based on the simple *skeleton* example +of a forwarding application. + +It is intended as a demonstration of the basic components of a DPDK forwarding +application which uses the Flow Classify library API's. + +Please refer to the +:doc:`../prog_guide/flow_classify_lib` +for more information. + +Compiling the Application +------------------------- + +To compile the sample application see :doc:`compiling`. + +The application is located in the ``flow_classify`` sub-directory. + +Running the Application +----------------------- + +To run the example in a ``linuxapp`` environment: + +.. code-block:: console + + cd ~/dpdk/examples/flow_classify + ./build/flow_classify -c 4 -n 4 -- --rule_ipv4="../ipv4_rules_file.txt" + +Please refer to the *DPDK Getting Started Guide*, section +:doc:`../linux_gsg/build_sample_apps` +for general information on running applications and the Environment Abstraction +Layer (EAL) options. + + +Sample ipv4_rules_file.txt +-------------------------- + +.. code-block:: console + + #file format: + #src_ip/masklen dst_ip/masklen src_port : mask dst_port : mask proto/mask priority + # + 2.2.2.3/24 2.2.2.7/24 32 : 0xffff 33 : 0xffff 17/0xff 0 + 9.9.9.3/24 9.9.9.7/24 32 : 0xffff 33 : 0xffff 17/0xff 1 + 9.9.9.3/24 9.9.9.7/24 32 : 0xffff 33 : 0xffff 6/0xff 2 + 9.9.8.3/24 9.9.8.7/24 32 : 0xffff 33 : 0xffff 6/0xff 3 + 6.7.8.9/24 2.3.4.5/24 32 : 0x0000 33 : 0x0000 132/0xff 4 + +Explanation +----------- + +The following sections provide an explanation of the main components of the +code. + +All DPDK library functions used in the sample code are prefixed with ``rte_`` +and are explained in detail in the *DPDK API Documentation*. + +ACL field definitions for the IPv4 5 tuple rule +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The following field definitions are used when creating the ACL table during +initialisation of the ``Flow Classify`` application.. + +.. code-block:: c + + enum { + PROTO_FIELD_IPV4, + SRC_FIELD_IPV4, + DST_FIELD_IPV4, + SRCP_FIELD_IPV4, + DSTP_FIELD_IPV4, + NUM_FIELDS_IPV4 + }; + + enum { + PROTO_INPUT_IPV4, + SRC_INPUT_IPV4, + DST_INPUT_IPV4, + SRCP_DESTP_INPUT_IPV4 + }; + + static struct rte_acl_field_def ipv4_defs[NUM_FIELDS_IPV4] = { + /* first input field - always one byte long. */ + { + .type = RTE_ACL_FIELD_TYPE_BITMASK, + .size = sizeof(uint8_t), + .field_index = PROTO_FIELD_IPV4, + .input_index = PROTO_INPUT_IPV4, + .offset = sizeof(struct ether_hdr) + + offsetof(struct ipv4_hdr, next_proto_id), + }, + /* next input field (IPv4 source address) - 4 consecutive bytes. */ + { + /* rte_flow uses a bit mask for IPv4 addresses */ + .type = RTE_ACL_FIELD_TYPE_BITMASK, + .size = sizeof(uint32_t), + .field_index = SRC_FIELD_IPV4, + .input_index = SRC_INPUT_IPV4, + .offset = sizeof(struct ether_hdr) + + offsetof(struct ipv4_hdr, src_addr), + }, + /* next input field (IPv4 destination address) - 4 consecutive bytes. */ + { + /* rte_flow uses a bit mask for IPv4 addresses */ + .type = RTE_ACL_FIELD_TYPE_BITMASK, + .size = sizeof(uint32_t), + .field_index = DST_FIELD_IPV4, + .input_index = DST_INPUT_IPV4, + .offset = sizeof(struct ether_hdr) + + offsetof(struct ipv4_hdr, dst_addr), + }, + /* + * Next 2 fields (src & dst ports) form 4 consecutive bytes. + * They share the same input index. + */ + { + /* rte_flow uses a bit mask for protocol ports */ + .type = RTE_ACL_FIELD_TYPE_BITMASK, + .size = sizeof(uint16_t), + .field_index = SRCP_FIELD_IPV4, + .input_index = SRCP_DESTP_INPUT_IPV4, + .offset = sizeof(struct ether_hdr) + + sizeof(struct ipv4_hdr) + + offsetof(struct tcp_hdr, src_port), + }, + { + /* rte_flow uses a bit mask for protocol ports */ + .type = RTE_ACL_FIELD_TYPE_BITMASK, + .size = sizeof(uint16_t), + .field_index = DSTP_FIELD_IPV4, + .input_index = SRCP_DESTP_INPUT_IPV4, + .offset = sizeof(struct ether_hdr) + + sizeof(struct ipv4_hdr) + + offsetof(struct tcp_hdr, dst_port), + }, + }; + +The Main Function +~~~~~~~~~~~~~~~~~ + +The ``main()`` function performs the initialization and calls the execution +threads for each lcore. + +The first task is to initialize the Environment Abstraction Layer (EAL). +The ``argc`` and ``argv`` arguments are provided to the ``rte_eal_init()`` +function. The value returned is the number of parsed arguments: + +.. code-block:: c + + int ret = rte_eal_init(argc, argv); + if (ret < 0) + rte_exit(EXIT_FAILURE, "Error with EAL initialization\n"); + +It then parses the flow_classify application arguments + +.. code-block:: c + + ret = parse_args(argc, argv); + if (ret < 0) + rte_exit(EXIT_FAILURE, "Invalid flow_classify parameters\n"); + +The ``main()`` function also allocates a mempool to hold the mbufs +(Message Buffers) used by the application: + +.. code-block:: c + + mbuf_pool = rte_mempool_create("MBUF_POOL", + NUM_MBUFS * nb_ports, + MBUF_SIZE, + MBUF_CACHE_SIZE, + sizeof(struct rte_pktmbuf_pool_private), + rte_pktmbuf_pool_init, NULL, + rte_pktmbuf_init, NULL, + rte_socket_id(), + 0); + +mbufs are the packet buffer structure used by DPDK. They are explained in +detail in the "Mbuf Library" section of the *DPDK Programmer's Guide*. + +The ``main()`` function also initializes all the ports using the user defined +``port_init()`` function which is explained in the next section: + +.. code-block:: c + + for (portid = 0; portid < nb_ports; portid++) { + if (port_init(portid, mbuf_pool) != 0) { + rte_exit(EXIT_FAILURE, + "Cannot init port %" PRIu8 "\n", portid); + } + } + +The ``main()`` function creates the ``flow classifier object`` and adds an ``ACL +table`` to the flow classifier. + +.. code-block:: c + + struct flow_classifier { + struct rte_flow_classifier *cls; + uint32_t table_id[RTE_FLOW_CLASSIFY_TABLE_MAX]; + }; + + struct flow_classifier_acl { + struct flow_classifier cls; + } __rte_cache_aligned; + + /* Memory allocation */ + size = RTE_CACHE_LINE_ROUNDUP(sizeof(struct flow_classifier_acl)); + cls_app = rte_zmalloc(NULL, size, RTE_CACHE_LINE_SIZE); + if (cls_app == NULL) + rte_exit(EXIT_FAILURE, "Cannot allocate classifier memory\n"); + + cls_params.name = "flow_classifier"; + cls_params.socket_id = socket_id; + cls_params.type = RTE_FLOW_CLASSIFY_TABLE_TYPE_ACL; + + cls_app->cls = rte_flow_classifier_create(&cls_params); + if (cls_app->cls == NULL) { + rte_free(cls_app); + rte_exit(EXIT_FAILURE, "Cannot create classifier\n"); + } + + /* initialise ACL table params */ + table_acl_params.name = "table_acl_ipv4_5tuple"; + table_acl_params.n_rule_fields = RTE_DIM(ipv4_defs); + table_acl_params.n_rules = FLOW_CLASSIFY_MAX_RULE_NUM; + memcpy(table_acl_params.field_format, ipv4_defs, sizeof(ipv4_defs)); + + /* initialise table create params */ + cls_table_params.ops = &rte_table_acl_ops, + cls_table_params.arg_create = &table_acl_params, + cls_table_params.table_metadata_size = 0; + + ret = rte_flow_classify_table_create(cls_app->cls, &cls_table_params, + &cls->table_id[0]); + if (ret) { + rte_flow_classifier_free(cls_app->cls); + rte_free(cls); + rte_exit(EXIT_FAILURE, "Failed to create classifier table\n"); + } + +It then reads the ipv4_rules_file.txt file and initialises the parameters for +the ``rte_flow_classify_table_entry_add`` API. +This API adds a rule to the ACL table. + +.. code-block:: c + + if (add_rules(parm_config.rule_ipv4_name)) { + rte_flow_classifier_free(cls_app->cls); + rte_free(cls_app); + rte_exit(EXIT_FAILURE, "Failed to add rules\n"); + } + +Once the initialization is complete, the application is ready to launch a +function on an lcore. In this example ``lcore_main()`` is called on a single +lcore. + +.. code-block:: c + + lcore_main(cls_app); + +The ``lcore_main()`` function is explained below. + +The Port Initialization Function +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The main functional part of the port initialization used in the Basic +Forwarding application is shown below: + +.. code-block:: c + + static inline int + port_init(uint8_t port, struct rte_mempool *mbuf_pool) + { + struct rte_eth_conf port_conf = port_conf_default; + const uint16_t rx_rings = 1, tx_rings = 1; + struct ether_addr addr; + int retval; + uint16_t q; + + if (port >= rte_eth_dev_count()) + return -1; + + /* Configure the Ethernet device. */ + retval = rte_eth_dev_configure(port, rx_rings, tx_rings, &port_conf); + if (retval != 0) + return retval; + + /* Allocate and set up 1 RX queue per Ethernet port. */ + for (q = 0; q < rx_rings; q++) { + retval = rte_eth_rx_queue_setup(port, q, RX_RING_SIZE, + rte_eth_dev_socket_id(port), NULL, mbuf_pool); + if (retval < 0) + return retval; + } + + /* Allocate and set up 1 TX queue per Ethernet port. */ + for (q = 0; q < tx_rings; q++) { + retval = rte_eth_tx_queue_setup(port, q, TX_RING_SIZE, + rte_eth_dev_socket_id(port), NULL); + if (retval < 0) + return retval; + } + + /* Start the Ethernet port. */ + retval = rte_eth_dev_start(port); + if (retval < 0) + return retval; + + /* Display the port MAC address. */ + rte_eth_macaddr_get(port, &addr); + printf("Port %u MAC: %02" PRIx8 " %02" PRIx8 " %02" PRIx8 + " %02" PRIx8 " %02" PRIx8 " %02" PRIx8 "\n", + port, + addr.addr_bytes[0], addr.addr_bytes[1], + addr.addr_bytes[2], addr.addr_bytes[3], + addr.addr_bytes[4], addr.addr_bytes[5]); + + /* Enable RX in promiscuous mode for the Ethernet device. */ + rte_eth_promiscuous_enable(port); + + return 0; + } + +The Ethernet ports are configured with default settings using the +``rte_eth_dev_configure()`` function and the ``port_conf_default`` struct. + +.. code-block:: c + + static const struct rte_eth_conf port_conf_default = { + .rxmode = { .max_rx_pkt_len = ETHER_MAX_LEN } + }; + +For this example the ports are set up with 1 RX and 1 TX queue using the +``rte_eth_rx_queue_setup()`` and ``rte_eth_tx_queue_setup()`` functions. + +The Ethernet port is then started: + +.. code-block:: c + + retval = rte_eth_dev_start(port); + + +Finally the RX port is set in promiscuous mode: + +.. code-block:: c + + rte_eth_promiscuous_enable(port); + +The Add Rules function +~~~~~~~~~~~~~~~~~~~~~~ + +The ``add_rules`` function reads the ``ipv4_rules_file.txt`` file and calls the +``add_classify_rule`` function which calls the +``rte_flow_classify_table_entry_add`` API. + +.. code-block:: c + + static int + add_rules(const char *rule_path) + { + FILE *fh; + char buff[LINE_MAX]; + unsigned int i = 0; + unsigned int total_num = 0; + struct rte_eth_ntuple_filter ntuple_filter; + + fh = fopen(rule_path, "rb"); + if (fh == NULL) + rte_exit(EXIT_FAILURE, "%s: Open %s failed\n", __func__, + rule_path); + + fseek(fh, 0, SEEK_SET); + + i = 0; + while (fgets(buff, LINE_MAX, fh) != NULL) { + i++; + + if (is_bypass_line(buff)) + continue; + + if (total_num >= FLOW_CLASSIFY_MAX_RULE_NUM - 1) { + printf("\nINFO: classify rule capacity %d reached\n", + total_num); + break; + } + + if (parse_ipv4_5tuple_rule(buff, &ntuple_filter) != 0) + rte_exit(EXIT_FAILURE, + "%s Line %u: parse rules error\n", + rule_path, i); + + if (add_classify_rule(&ntuple_filter) != 0) + rte_exit(EXIT_FAILURE, "add rule error\n"); + + total_num++; + } + + fclose(fh); + return 0; + } + + +The Lcore Main function +~~~~~~~~~~~~~~~~~~~~~~~ + +As we saw above the ``main()`` function calls an application function on the +available lcores. +The ``lcore_main`` function calls the ``rte_flow_classifier_query`` API. +For the Basic Forwarding application the ``lcore_main`` function looks like the +following: + +.. code-block:: c + + /* flow classify data */ + static int num_classify_rules; + static struct rte_flow_classify_rule *rules[MAX_NUM_CLASSIFY]; + static struct rte_flow_classify_ipv4_5tuple_stats ntuple_stats; + static struct rte_flow_classify_stats classify_stats = { + .stats = (void *)&ntuple_stats + }; + + static __attribute__((noreturn)) void + lcore_main(cls_app) + { + const uint8_t nb_ports = rte_eth_dev_count(); + uint8_t port; + + /* + * Check that the port is on the same NUMA node as the polling thread + * for best performance. + */ + for (port = 0; port < nb_ports; port++) + if (rte_eth_dev_socket_id(port) > 0 && + rte_eth_dev_socket_id(port) != (int)rte_socket_id()) { + printf("\n\n"); + printf("WARNING: port %u is on remote NUMA node\n", + port); + printf("to polling thread.\n"); + printf("Performance will not be optimal.\n"); + + printf("\nCore %u forwarding packets. \n", + rte_lcore_id()); + printf("[Ctrl+C to quit]\n + } + + /* Run until the application is quit or killed. */ + for (;;) { + /* + * Receive packets on a port and forward them on the paired + * port. The mapping is 0 -> 1, 1 -> 0, 2 -> 3, 3 -> 2, etc. + */ + for (port = 0; port < nb_ports; port++) { + + /* Get burst of RX packets, from first port of pair. */ + struct rte_mbuf *bufs[BURST_SIZE]; + const uint16_t nb_rx = rte_eth_rx_burst(port, 0, + bufs, BURST_SIZE); + + if (unlikely(nb_rx == 0)) + continue; + + for (i = 0; i < MAX_NUM_CLASSIFY; i++) { + if (rules[i]) { + ret = rte_flow_classifier_query( + cls_app->cls, + cls_app->table_id[0], + bufs, nb_rx, rules[i], + &classify_stats); + if (ret) + printf( + "rule [%d] query failed ret [%d]\n\n", + i, ret); + else { + printf( + "rule[%d] count=%"PRIu64"\n", + i, ntuple_stats.counter1); + + printf("proto = %d\n", + ntuple_stats.ipv4_5tuple.proto); + } + } + } + + /* Send burst of TX packets, to second port of pair. */ + const uint16_t nb_tx = rte_eth_tx_burst(port ^ 1, 0, + bufs, nb_rx); + + /* Free any unsent packets. */ + if (unlikely(nb_tx < nb_rx)) { + uint16_t buf; + for (buf = nb_tx; buf < nb_rx; buf++) + rte_pktmbuf_free(bufs[buf]); + } + } + } + } + +The main work of the application is done within the loop: + +.. code-block:: c + + for (;;) { + for (port = 0; port < nb_ports; port++) { + + /* Get burst of RX packets, from first port of pair. */ + struct rte_mbuf *bufs[BURST_SIZE]; + const uint16_t nb_rx = rte_eth_rx_burst(port, 0, + bufs, BURST_SIZE); + + if (unlikely(nb_rx == 0)) + continue; + + /* Send burst of TX packets, to second port of pair. */ + const uint16_t nb_tx = rte_eth_tx_burst(port ^ 1, 0, + bufs, nb_rx); + + /* Free any unsent packets. */ + if (unlikely(nb_tx < nb_rx)) { + uint16_t buf; + for (buf = nb_tx; buf < nb_rx; buf++) + rte_pktmbuf_free(bufs[buf]); + } + } + } + +Packets are received in bursts on the RX ports and transmitted in bursts on +the TX ports. The ports are grouped in pairs with a simple mapping scheme +using the an XOR on the port number:: + + 0 -> 1 + 1 -> 0 + + 2 -> 3 + 3 -> 2 + + etc. + +The ``rte_eth_tx_burst()`` function frees the memory buffers of packets that +are transmitted. If packets fail to transmit, ``(nb_tx < nb_rx)``, then they +must be freed explicitly using ``rte_pktmbuf_free()``. + +The forwarding loop can be interrupted and the application closed using +``Ctrl-C``. diff --git a/dpdk/doc/guides/sample_app_ug/flow_filtering.rst b/dpdk/doc/guides/sample_app_ug/flow_filtering.rst new file mode 100644 index 00000000..725dcb44 --- /dev/null +++ b/dpdk/doc/guides/sample_app_ug/flow_filtering.rst @@ -0,0 +1,545 @@ +.. BSD LICENSE + Copyright(c) 2017 Mellanox Corporation. All rights reserved. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Mellanox Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +Basic RTE Flow Filtering Sample Application +=========================================== + +The Basic RTE flow filtering sample application is a simple example of a +creating a RTE flow rule. + +It is intended as a demonstration of the basic components RTE flow rules. + + +Compiling the Application +------------------------- + +To compile the application export the path to the DPDK source tree and go to +the example directory: + +.. code-block:: console + + export RTE_SDK=/path/to/rte_sdk + + cd ${RTE_SDK}/examples/flow_filtering + +Set the target, for example: + +.. code-block:: console + + export RTE_TARGET=x86_64-native-linuxapp-gcc + +See the *DPDK Getting Started* Guide for possible ``RTE_TARGET`` values. + +Build the application as follows: + +.. code-block:: console + + make + + +Running the Application +----------------------- + +To run the example in a ``linuxapp`` environment: + +.. code-block:: console + + ./build/flow -l 1 -n 1 + +Refer to *DPDK Getting Started Guide* for general information on running +applications and the Environment Abstraction Layer (EAL) options. + + +Explanation +----------- + +The example is build from 2 main files, +``main.c`` which holds the example logic and ``flow_blocks.c`` that holds the +implementation for building the flow rule. + +The following sections provide an explanation of the main components of the +code. + +All DPDK library functions used in the sample code are prefixed with ``rte_`` +and are explained in detail in the *DPDK API Documentation*. + + +The Main Function +~~~~~~~~~~~~~~~~~ + +The ``main()`` function located in ``main.c`` file performs the initialization +and runs the main loop function. + +The first task is to initialize the Environment Abstraction Layer (EAL). The +``argc`` and ``argv`` arguments are provided to the ``rte_eal_init()`` +function. The value returned is the number of parsed arguments: + +.. code-block:: c + + int ret = rte_eal_init(argc, argv); + if (ret < 0) + rte_exit(EXIT_FAILURE, "Error with EAL initialization\n"); + + +The ``main()`` also allocates a mempool to hold the mbufs (Message Buffers) +used by the application: + +.. code-block:: c + + mbuf_pool = rte_pktmbuf_pool_create("mbuf_pool", 4096, 128, 0, + RTE_MBUF_DEFAULT_BUF_SIZE, + rte_socket_id()); + +Mbufs are the packet buffer structure used by DPDK. They are explained in +detail in the "Mbuf Library" section of the *DPDK Programmer's Guide*. + +The ``main()`` function also initializes all the ports using the user defined +``init_port()`` function which is explained in the next section: + +.. code-block:: c + + init_port(); + +Once the initialization is complete, we set the flow rule using the +following code: + +.. code-block:: c + + /* create flow for send packet with */ + flow = generate_ipv4_flow(port_id, selected_queue, + SRC_IP, EMPTY_MASK, + DEST_IP, FULL_MASK, &error); + if (!flow) { + printf("Flow can't be created %d message: %s\n", + error.type, + error.message ? error.message : "(no stated reason)"); + rte_exit(EXIT_FAILURE, "error in creating flow"); + } + +In the last part the application is ready to launch the +``main_loop()`` function. Which is explained below. + + +.. code-block:: c + + main_loop(); + +The Port Initialization Function +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The main functional part of the port initialization used in the flow filtering +application is shown below: + +.. code-block:: c + + init_port(void) + { + int ret; + uint16_t i; + struct rte_eth_conf port_conf = { + .rxmode = { + .split_hdr_size = 0, + /**< Header Split disabled */ + .header_split = 0, + /**< IP checksum offload disabled */ + .hw_ip_checksum = 0, + /**< VLAN filtering disabled */ + .hw_vlan_filter = 0, + /**< Jumbo Frame Support disabled */ + .jumbo_frame = 0, + /**< CRC stripped by hardware */ + .hw_strip_crc = 1, + }, + }; + + printf(":: initializing port: %d\n", port_id); + ret = rte_eth_dev_configure(port_id, + nr_queues, nr_queues, &port_conf); + if (ret < 0) { + rte_exit(EXIT_FAILURE, + ":: cannot configure device: err=%d, port=%u\n", + ret, port_id); + } + + /* only set Rx queues: something we care only so far */ + for (i = 0; i < nr_queues; i++) { + ret = rte_eth_rx_queue_setup(port_id, i, 512, + rte_eth_dev_socket_id(port_id), + NULL, + mbuf_pool); + if (ret < 0) { + rte_exit(EXIT_FAILURE, + ":: Rx queue setup failed: err=%d, port=%u\n", + ret, port_id); + } + } + + + rte_eth_promiscuous_enable(port_id); + + ret = rte_eth_dev_start(port_id); + if (ret < 0) { + rte_exit(EXIT_FAILURE, + "rte_eth_dev_start:err=%d, port=%u\n", + ret, port_id); + } + + assert_link_status(); + + printf(":: initializing port: %d done\n", port_id); + } + +The Ethernet port is configured with default settings using the +``rte_eth_dev_configure()`` function and the ``port_conf_default`` struct: + +.. code-block:: c + + struct rte_eth_conf port_conf = { + .rxmode = { + .split_hdr_size = 0, + /**< Header Split disabled */ + .header_split = 0, + /**< IP checksum offload disabled */ + .hw_ip_checksum = 0, + /**< VLAN filtering disabled */ + .hw_vlan_filter = 0, + /**< Jumbo Frame Support disabled */ + .jumbo_frame = 0, + /**< CRC stripped by hardware */ + .hw_strip_crc = 1, + }, + }; + + ret = rte_eth_dev_configure(port_id, nr_queues, nr_queues, &port_conf); + if (ret < 0) { + rte_exit(EXIT_FAILURE, + ":: cannot configure device: err=%d, port=%u\n", + ret, port_id); + } + +For this example we are configuring number of rx queues that are connected to +a single port. + +.. code-block:: c + + for (i = 0; i < nr_queues; i++) { + ret = rte_eth_rx_queue_setup(port_id, i, 512, + rte_eth_dev_socket_id(port_id), + NULL, + mbuf_pool); + if (ret < 0) { + rte_exit(EXIT_FAILURE, + ":: Rx queue setup failed: err=%d, port=%u\n", + ret, port_id); + } + } + +In the next step we create and apply the flow rule. which is to send packets +with destination ip equals to 192.168.1.1 to queue number 1. The detail +explanation of the ``generate_ipv4_flow()`` appears later in this document: + +.. code-block:: c + + flow = generate_ipv4_flow(port_id, selected_queue, + SRC_IP, EMPTY_MASK, + DEST_IP, FULL_MASK, &error); + +We are setting the RX port to promiscuous mode: + +.. code-block:: c + + rte_eth_promiscuous_enable(port_id); + +The last step is to start the port. + +.. code-block:: c + + ret = rte_eth_dev_start(port_id); + if (ret < 0) { + rte_exit(EXIT_FAILURE, "rte_eth_dev_start:err%d, port=%u\n", + ret, port_id); + } + + +The main_loop function +~~~~~~~~~~~~~~~~~~~~~~ + +As we saw above the ``main()`` function calls an application function to handle +the main loop. For the flow filtering application the main_loop function +looks like the following: + +.. code-block:: c + + static void + main_loop(void) + { + struct rte_mbuf *mbufs[32]; + struct ether_hdr *eth_hdr; + uint16_t nb_rx; + uint16_t i; + uint16_t j; + + while (!force_quit) { + for (i = 0; i < nr_queues; i++) { + nb_rx = rte_eth_rx_burst(port_id, + i, mbufs, 32); + if (nb_rx) { + for (j = 0; j < nb_rx; j++) { + struct rte_mbuf *m = mbufs[j]; + + eth_hdr = rte_pktmbuf_mtod(m, + struct ether_hdr *); + print_ether_addr("src=", + ð_hdr->s_addr); + print_ether_addr(" - dst=", + ð_hdr->d_addr); + printf(" - queue=0x%x", + (unsigned int)i); + printf("\n"); + rte_pktmbuf_free(m); + } + } + } + } + /* closing and releasing resources */ + rte_flow_flush(port_id, &error); + rte_eth_dev_stop(port_id); + rte_eth_dev_close(port_id); + } + +The main work of the application is reading the packets from all +queues and printing for each packet the destination queue: + +.. code-block:: c + + while (!force_quit) { + for (i = 0; i < nr_queues; i++) { + nb_rx = rte_eth_rx_burst(port_id, i, mbufs, 32); + if (nb_rx) { + for (j = 0; j < nb_rx; j++) { + struct rte_mbuf *m = mbufs[j]; + eth_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *); + print_ether_addr("src=", ð_hdr->s_addr); + print_ether_addr(" - dst=", ð_hdr->d_addr); + printf(" - queue=0x%x", (unsigned int)i); + printf("\n"); + rte_pktmbuf_free(m); + } + } + } + } + + +The forwarding loop can be interrupted and the application closed using +``Ctrl-C``. Which results in closing the port and the device using +``rte_eth_dev_stop`` and ``rte_eth_dev_close`` + +The generate_ipv4_flow function +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The generate_ipv4_rule function is responsible for creating the flow rule. +This function is located in the ``flow_blocks.c`` file. + +.. code-block:: c + + static struct rte_flow * + generate_ipv4_flow(uint8_t port_id, uint16_t rx_q, + uint32_t src_ip, uint32_t src_mask, + uint32_t dest_ip, uint32_t dest_mask, + struct rte_flow_error *error) + { + struct rte_flow_attr attr; + struct rte_flow_item pattern[MAX_PATTERN_NUM]; + struct rte_flow_action action[MAX_PATTERN_NUM]; + struct rte_flow *flow = NULL; + struct rte_flow_action_queue queue = { .index = rx_q }; + struct rte_flow_item_eth eth_spec; + struct rte_flow_item_eth eth_mask; + struct rte_flow_item_vlan vlan_spec; + struct rte_flow_item_vlan vlan_mask; + struct rte_flow_item_ipv4 ip_spec; + struct rte_flow_item_ipv4 ip_mask; + + memset(pattern, 0, sizeof(pattern)); + memset(action, 0, sizeof(action)); + + /* + * set the rule attribute. + * in this case only ingress packets will be checked. + */ + memset(&attr, 0, sizeof(struct rte_flow_attr)); + attr.ingress = 1; + + /* + * create the action sequence. + * one action only, move packet to queue + */ + + action[0].type = RTE_FLOW_ACTION_TYPE_QUEUE; + action[0].conf = &queue; + action[1].type = RTE_FLOW_ACTION_TYPE_END; + + /* + * set the first level of the pattern (eth). + * since in this example we just want to get the + * ipv4 we set this level to allow all. + */ + memset(ð_spec, 0, sizeof(struct rte_flow_item_eth)); + memset(ð_mask, 0, sizeof(struct rte_flow_item_eth)); + eth_spec.type = 0; + eth_mask.type = 0; + pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH; + pattern[0].spec = ð_spec; + pattern[0].mask = ð_mask; + + /* + * setting the second level of the pattern (vlan). + * since in this example we just want to get the + * ipv4 we also set this level to allow all. + */ + memset(&vlan_spec, 0, sizeof(struct rte_flow_item_vlan)); + memset(&vlan_mask, 0, sizeof(struct rte_flow_item_vlan)); + pattern[1].type = RTE_FLOW_ITEM_TYPE_VLAN; + pattern[1].spec = &vlan_spec; + pattern[1].mask = &vlan_mask; + + /* + * setting the third level of the pattern (ip). + * in this example this is the level we care about + * so we set it according to the parameters. + */ + memset(&ip_spec, 0, sizeof(struct rte_flow_item_ipv4)); + memset(&ip_mask, 0, sizeof(struct rte_flow_item_ipv4)); + ip_spec.hdr.dst_addr = htonl(dest_ip); + ip_mask.hdr.dst_addr = dest_mask; + ip_spec.hdr.src_addr = htonl(src_ip); + ip_mask.hdr.src_addr = src_mask; + pattern[2].type = RTE_FLOW_ITEM_TYPE_IPV4; + pattern[2].spec = &ip_spec; + pattern[2].mask = &ip_mask; + + /* the final level must be always type end */ + pattern[3].type = RTE_FLOW_ITEM_TYPE_END; + + int res = rte_flow_validate(port_id, &attr, pattern, action, error); + if(!res) + flow = rte_flow_create(port_id, &attr, pattern, action, error); + + return flow; + } + +The first part of the function is declaring the structures that will be used. + +.. code-block:: c + + struct rte_flow_attr attr; + struct rte_flow_item pattern[MAX_PATTERN_NUM]; + struct rte_flow_action action[MAX_PATTERN_NUM]; + struct rte_flow *flow; + struct rte_flow_error error; + struct rte_flow_action_queue queue = { .index = rx_q }; + struct rte_flow_item_eth eth_spec; + struct rte_flow_item_eth eth_mask; + struct rte_flow_item_vlan vlan_spec; + struct rte_flow_item_vlan vlan_mask; + struct rte_flow_item_ipv4 ip_spec; + struct rte_flow_item_ipv4 ip_mask; + +The following part create the flow attributes, in our case ingress. + +.. code-block:: c + + memset(&attr, 0, sizeof(struct rte_flow_attr)); + attr.ingress = 1; + +The third part defines the action to be taken when a packet matches +the rule. In this case send the packet to queue. + +.. code-block:: c + + action[0].type = RTE_FLOW_ACTION_TYPE_QUEUE; + action[0].conf = &queue; + action[1].type = RTE_FLOW_ACTION_TYPE_END; + +The forth part is responsible for creating the pattern and is build from +number of step. In each step we build one level of the pattern starting with +the lowest one. + +Setting the first level of the pattern ETH: + +.. code-block:: c + + memset(ð_spec, 0, sizeof(struct rte_flow_item_eth)); + memset(ð_mask, 0, sizeof(struct rte_flow_item_eth)); + eth_spec.type = 0; + eth_mask.type = 0; + pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH; + pattern[0].spec = ð_spec; + pattern[0].mask = ð_mask; + +Setting the second level of the pattern VLAN: + +.. code-block:: c + + memset(&vlan_spec, 0, sizeof(struct rte_flow_item_vlan)); + memset(&vlan_mask, 0, sizeof(struct rte_flow_item_vlan)); + pattern[1].type = RTE_FLOW_ITEM_TYPE_VLAN; + pattern[1].spec = &vlan_spec; + pattern[1].mask = &vlan_mask; + +Setting the third level ip: + +.. code-block:: c + + memset(&ip_spec, 0, sizeof(struct rte_flow_item_ipv4)); + memset(&ip_mask, 0, sizeof(struct rte_flow_item_ipv4)); + ip_spec.hdr.dst_addr = htonl(dest_ip); + ip_mask.hdr.dst_addr = dest_mask; + ip_spec.hdr.src_addr = htonl(src_ip); + ip_mask.hdr.src_addr = src_mask; + pattern[2].type = RTE_FLOW_ITEM_TYPE_IPV4; + pattern[2].spec = &ip_spec; + pattern[2].mask = &ip_mask; + +Closing the pattern part. + +.. code-block:: c + + pattern[3].type = RTE_FLOW_ITEM_TYPE_END; + +The last part of the function is to validate the rule and create it. + +.. code-block:: c + + int res = rte_flow_validate(port_id, &attr, pattern, action, &error); + if (!res) + flow = rte_flow_create(port_id, &attr, pattern, action, &error); + diff --git a/dpdk/doc/guides/sample_app_ug/hello_world.rst b/dpdk/doc/guides/sample_app_ug/hello_world.rst index 72e1e115..3018d459 100644 --- a/dpdk/doc/guides/sample_app_ug/hello_world.rst +++ b/dpdk/doc/guides/sample_app_ug/hello_world.rst @@ -37,26 +37,9 @@ The application simply prints an "helloworld" message on every enabled lcore. Compiling the Application ------------------------- -#. Go to the example directory: +To compile the sample application see :doc:`compiling`. - .. code-block:: console - - export RTE_SDK=/path/to/rte_sdk - cd ${RTE_SDK}/examples/helloworld - -#. Set the target (a default target is used if not specified). For example: - - .. code-block:: console - - export RTE_TARGET=x86_64-native-linuxapp-gcc - - See the *DPDK Getting Started* Guide for possible RTE_TARGET values. - -#. Build the application: - - .. code-block:: console - - make +The application is located in the ``helloworld`` sub-directory. Running the Application ----------------------- @@ -65,7 +48,7 @@ To run the example in a linuxapp environment: .. code-block:: console - $ ./build/helloworld -c f -n 4 + $ ./build/helloworld -l 0-3 -n 4 Refer to *DPDK Getting Started Guide* for general information on running applications and the Environment Abstraction Layer (EAL) options. diff --git a/dpdk/doc/guides/sample_app_ug/img/dist_app.svg b/dpdk/doc/guides/sample_app_ug/img/dist_app.svg index 4714c7db..944f4377 100644 --- a/dpdk/doc/guides/sample_app_ug/img/dist_app.svg +++ b/dpdk/doc/guides/sample_app_ug/img/dist_app.svg @@ -1,8 +1,7 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + Host + + + Guest1 + Guest2 + + L2FWD + + + + + + + + diff --git a/dpdk/doc/guides/sample_app_ug/img/qemu_virtio_net.png b/dpdk/doc/guides/sample_app_ug/img/qemu_virtio_net.png deleted file mode 100644 index a852c1662fe978e1dc785f4771ff285172ae5e86..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 31557 zcmXtf1yoy2*EI$D;10!IQ@jMX;_gsfQz-7Ph2mP=p~VUA4lNc61h=5Yi@V#O_W$0y z)?KWGWHNW=$UbN9iBeaU!$K!RM?gTpQjnL{L_m1u0>9Hy|AW6HlVv6h|AXu%si2Jt zzx+`@M#5jCIm_$2As}G)|Gi(OvSO1XAkZKvNK0sY=N#txnnes|fths~T-5%P8)Cpp_Y2qtHdlY3r?RKZER-vvyX;Zwk)l zcwc4A!eM?V;EN}XS<#j0m4^2o!aOq<4{`6Ba@@cjJ5?>KI`RNUj=xs`{&t1Q=NR}k zlDCQOLQG7YE*8t$Z){{_bkDV@+l4@!o~N$c5~IH+O6g?7h;`KwDGSop!TH`)ocL2J zqQPg=x6?$t9rA^a0SQH?&S!HsB_$0X^S|=4viH;OS!84rB0?&0STnIC3M{w?w2{)i zjm?eqFGCtY-Wi_Sf}~vG?*0^})YLCFb`Ax;r(Ar2-FXECbc8BWTF912|J85DsHCK% z;K2v11WPL_C@EpIS5Gf53_TahPA@Kpg7-qDBtm{7@IEAzd=F$sLcy|lXm%)z9{wSb zioBRbs?D1bM?R%XkRSKJ$!ZZpY z^&Nr4?GU+dRjJlMMQEhP;cC;DbBQkt-43i&D}6trC6E^lSf~XeRml0d%-#!1YMlCX z{b`RlT58t(Aolq^&5y6&`%qE7AS1rg)?viOvJ%X{nGhs85x-rEDHJBs?_>dNU%;9BqY|iiHMJ4)X@d!+t*c?l_b(!xMqK9pR8>h z=}6GO+mU@Y`-}&l<-y32_fue2c9uHf&)TQ!Qpa7-|FkhJ4v4b>?Cga_jj{3q@-7Ob z*k+3Kz`hKX^l`Cvbc8^TP&XTYtc1jp(TU= zxf_E{zdn_%T`?9TyOBn{s)+ z*gl^=bdd6R+ONJag&r)p?188sNFf{U2Q2d<6R(EeJ|r65Db9SZ&eM6k;nbwIQ!$zy&0IR&2qRpvADLwX9R zxT|VYq;02p7w_%$XPU|&MFvThJy6z};~0Tl8Ng1CrZ@_>JpZfxTArIS0e~?YY%$m{ znrxKtbXP!A{A7%pl%Gh5UgR%}Pe9uk>l0c)2PKi5JmMBM@!Igk&rf4Gt1{dz4I-Xv z+Rk9MG`znlW9W&V@zL&zg`lP;qki})D$GzT zG^6(Uk2H9qvBd)yhW0(>x&yb_S@ChbG&a-0uw`dYz%GK@!F(;*M{_~VnhKg6b{r^cJeqvap)M-p{Wx&_{0R5Vbg{77Sw4EI zpOG@%``fLp9}8I0tci0){I{?H92g?Sw?N_^W72RJog95M^}!UEAMNdg(C)CM`YbP_dit;O zz%DMK`iiKIOYZv)5D(>u(T37ms~lNvdX~hG_8m;~&NeahlsZ#Jb!u}%q-J#|1^&ra zcY>(uhg0v)|67~b=fj23j~C?R0!MNEPR*pC1lfAtr)W=MPq}bi-(Xaw7b8gG1vrDp zNyH8H)8dkG`+Hoprm2|)y{|AM+T*FxufH0Pp2MqN>3YMc7on7@)0FJpfy5Y&YpL$> zW10MUf><7G%DjE8ThfYAJ?^-(vtKaY<=YYbTRU~_CUqae*9^Vcc75+jwXjDwfnWS% z{~XulJFYh_&DT0xlDq|HWRH7X(TC!OE|I}j)HCLZbt5}21(o}TOgsw{8t&qRR$s>p zl!x!Fh?a_>-?I66qEF-eAz7X49rk!H1JDwtSn3iqYEO`!6}f$S{y*E1dztBfWzg!N zK)D3iJP+G!t1sN~`~6wj@3s`0_FKGNsNB31;@xs_pY34}f#mp5QsJl0vKd%ADLwHc zn2d?E$6ijzJp^+(zHg!CdmxJjf4A|K*l|^wY`+BTcP|fm1)iEx2<*yQe4nfF74)IE z(?rYeT$#E4eL=lG9zL z1P~dwyk{3}aYaHca;G}YBq<7)3X+TlmkNQ5f<4btOL* zM4ATTU1vmf6CA0K%2U~S$<=n8Mt;;Kp9UWJEOu<^T`WKtg%C3VSB{#DZ=+Nc>zU$h zyi7EOV$r^_*xvKKLM#q|mP9p*-^`$BY;@F*@#95{sx@;fh^p>!scl^FCE^diwIdNe z8fBYjIP?DN+|Ynzp-4E;xTi3-*@)gZKUkclFqs}~BQ}v%Qu^#!zP8HF3~OR6^hmEo zt{$pmr$to|oKFiqGBAnoX(wwSzd$$wO_@KB4JuogHh)%HO9L~;6fGznSV*N^9Qjl# z5odx*D%okJmtG4OiIQlF)#QMZ>c`@j>PM5?u2Z-P>`v@QRd?{BB}K@#bbIb8e>Nwx za8$e>_FfdVCJA4HK?WF!jUz}DF{eL{N<_Gh&VZ~ViEJ%79uT9^&zv)o29&rrTlJ8n$4H4>sjSUJ;fz0xh(2FA}e>hgtaQ=ODe)tRF-*5Tepa(xrV4LIW%T^ zQM za(}n88vmp-I_=UO|KjQU=pVbH$_F)2h0N@!L#~pf<+SHE zt1%?l;HYPpBJ;(^p4EON;Tw@7J7Pq-_1e^aP`)Fh4gglre{`tD58shu8kT5D2K@DP zzhgMqF+BAaMWt~h`Jn8~7G8ZIkA^?}jKrq_WCtK`oC$yn7Mg^RqYH3j1}1>e#Zs`% zl0_X!d&!sD`Uh=4DxK*%cR?E{b8%`&eCn!GKZpgKrR<}tlWGkg%cdW6EdrJVzUeF6 zlLl!3Rgh@8u#j{#q9CBMthBVr4DVW+_c}xt@Lx&tqmt37+_Z3UhmSeSw$6y9EkjGi zpHz&o7*;5)sN~HG?s}us$F+e<3!jETM>Rp*pgQ4e6`jZdEO%pivbch`;3qR6p5#xi zMJ04p^WtGvNwW+l#NZM;K)xTvce5HnZZ$#~ma;5G5h}TP4~t0>NxW+>J9VCd@HTOQ z^r@+LqtdiQi4*-YSv16vISTLZe*BANp+EKp6IVRhY_i6}uMT|n#?>wwV#fP6&*&0K zT$EbL!Wmqrl}PX9Cd{Fg2{c+7=k-ZEv2O0y-t5 zetQ3#T}^{tbpuH$4MNO5jbJ`T}=xj7U`4<~o)6GIZOMJPfaP=ko5 zFLG%qrO}7*`vPcCgBO|yiC(a-c?AmKh_;=rPV`jlY%*=ysS~g97KtxFJ&>>@vCdLt zqmrXZnoqptK+taLqKdt`YnlJ1;7=SiF5q(nUxPPtEWO>$Gd}Vr5nwP$NidMHeO}28 zM9;tKgA+l)ms!3h60XjY(n|XM3xe9RUppk?_{{y_dCBl!9+iKb^xPbB%r?e1w2>rsPF|1{`gUJEObQljVVyyrtLUXivC|61u+ zx%tErDT|1Vo^tpPl*tq<=1ymnNHCBMSldX=Hiit`wqH6D8kd%rhjF$9DX@`AXxIm{ zBP?01PUg@RH%=dqU^EPdjBadf;2td|d;u-OZNjEiUz)>4J_^h-b7=adrz~=-VddS) zQalv`*s4@yOGA*m<1hnBc zr3fV{*0$!Dl>dBG{*Wt?BjKPLG%YmTUwB|+Xk>&yNXtUWg#e9LS2XLwL;v_js_Xb_ z^Q^r|%gCty8$-;aun^cnhC#12Gmm4BAs`=(mSbsixIvJQ{-+fgIXQYg-``lr-Cn_X zGwSI`!Zca7B-+bJprvSFU~tGa>j_*U*CjZ(mcIiRX(snee9NjN=4vx=9 z3n_<-Z87o!19edW)K(t$irPAm-3#K6QI0u(#|EevDMnQv+{TD_2Z%+-#ArMPP5@Iv zFMxJ&Wo$C_LUU%r{t%|j5SY$NxzSZlyJFzpUIry3k~||ZvPm_aK@xR%czA+W^e9hL zn<%rduDp!QcSfun^NuVb66aWpbqR$lB4Q7xgbD~n9z=E z#gl8W3nB3w4iI_Mw|TYb!B@G+QewVw<_j;EZ~aYr*CTZ>`l}z-Z|+dTC9T0(gvjmH z1W~e;J{upf@Bs`Z#t;Uogv~48NXUkDyG}ztP({RqbJ7|)e_T$0P(nlcoyW?-1wMM% z(J7xDmCAIabfGm^QN;0^AFv3Z;2x1M0J`lIvxUU()b=scf~uWpu+ZwfXTX_BTqg1d zVuYk>1_%?KCl1 zT4Z4dNT5nE;Q!n_o0wWu)VGN;{x%GDB}R`pd+7Ac#fKNkwiuuT^Y7(C!RojP)$pYN zoIhyrrkc@2!m~m?Guh8VzoezFW{U7?EYX@Ue?GfX3qMLasmaLQSWZi4F+{@O1ZUkD zxo0f_{I;>__1Qx}<7>o}G(x27t3N{D*&Q-)g~GMseX^2Ve=uP8?hmf^GSgpcs->afj7w zxi{Ovf0v9VUXYK`GjobXR*Chyvv>Z?`>r~YV6zF=#kQSO_rsPPXMtFAvZI?6<;G7V zuF=K=wPj^xG)nG;T1NL2@nIzB@SP<&^ImO#eZ_d_y2e3@D$<^pw?K~{ukw(Nolz2{ z`OR;5_Aee%733>!%!!HCW?#A%D+?>NXexaG>rRp%V~~QBS(Z!4Pc!(V z+y1S+jz?hCP=Mt7WqIrM>dT}RutEb49kN)>xTDbI&$`U*=0tpvZVSZ`MT~lpe?WzB zU_Dkv!Ea5PINe@efrVn&1SGb^&P0Qcj*CesOW*|sa0H+Oa^36|m+HzDX6vr5ESHIs z*cK)M12O|;2Eze#b)&5MC)`oTX9YxMqS=4zBf<94C@&Djt3>%wJvOVQFEA$l{RNZPz8ie{2T@4)!Qi$dn32&t=Xwp- zcdAAj^5OAEeXa3OwuQK)2u#W(6EzVSj9E$4rK8%s^F1Tx`E%Lr?Ph|{QR<7722p6J zH82)(z)Xhr{lEkl{zM@qevvZt*L=cXO*FVRTpFbnV^@KB%$ZLGf~B(ljW%L;$LV0VKT zV{q`=?kLK~Jz-XB9Js*(mSrQ8X%U?%u|Y_qm&T(Z9GQ0b3t+-+`ZaX0Xj8B+{(P1k zU40W5I_Z(1v%4V1Oci6WyO=AB-z*S38^EC9qc#hS^~Thh)=0Y0TMH{WK>d!NAUn}z zQjl2NrK0e_L8zmU1za>`ukxjXBdCOd3HMxYw&)afL2s7QT)z1?DP0&c?}7M?R0;a? zDOLTIBi(Vh8JhKWpC}XM8LBkux4pOC`chCX%I(ZZb! zInN9^CG?8^zJh>U#q-C*`RVSwP+S#skq+@9UL5)IMH~!hH5 z6?D?WBEKZ%m{`EG0ZZz5;AkMwPepNYkw~WLwdVsb)12cXqh9e$m7lH+@y47dk6_UG zq6!v(&{MQF<3(U6TN{gNf9Q$9(o6W-eY|e(%zGl$`(d|PZ4bOX<7wz$xcNqn5R8|x-K2FQkw_byY;QBX{a zUl$!6RTbkbBX`k=lEosIXjBPqEtyxRZWDDjWP9I192}ImlD{wR)QO6XBS*BSO{^9J zfHSgoc1jX+y0{}9cvyHCO}Gij=-jzVPVZ3}(1X+dD@ZH@^(khu{1Z@P+=>WDwk9aXF2p`$a+vvTce zL`zYD&1~6=JSe6g(oFA|I>gY>tGFbib#&UNqEB>T;eiiM2=h&%G;z(ERY~Zo(r@okx(CFF*j%!$1Wu`BIVcK};Fv{$2eowcLPen^z z6|;vOfAKe^irbhHXYn7aMNyXM5F8eE*7aoG{MwyD9G{ESben}Xq@Fc78wIrdfP26R zxyzixOVZrVn{b?Ji@}+4l zmHhHzqVrelQ9+267j!)uOmCSn`|_Ltp$d%5HS9KXSHaT4&iwI{VC?sgnt-_decR*i z2gf4Z9N+!&N~5+UEONfn870n(KRhF(T0A*-;TFs#cJo2ZQ}IxLRMl zyz19C)iONJ75ooM#m)cJnv+IAHe5PQIQzn}w>#W+^z71;62$L9Ayj^`Je*xQC;9&P z;$piff5fQQ<@+b?T=ur%5ET^+t~mAXa<8;WWd~`YO0-rb4z7gRq!?uY+XE}RT0b)5 z<>Fi0xQj$F>5970B4NxI7PtErqh7p?%kF^U$N27ywAK(62nFM6(`P*ad|DX`p?aaxyIf{k0w+j`lr^Kop zKB!wYL@H@RV;~36@hl;A658X%fg3>veA;hCqj!2Rqpa3r>K|G2dEY#?o$l7emxp^_ z+M^{>Q*#q$WduJf5HO$rDNJzf=jO?{=)2H+xs#2K!I&QCoVc`d8^+L8-bJqlIc&a( zjm@8L13?ALAYwz|BBSh~--=q1b*A5eNtmLIEprOXyq8OGIRdDK7lzmQ>+G{yTUv91 zx*L(_M1M7?iT_NLQ#o?YDJfxETwFv&7E+Vc(n)0{eok=j{9}Mb>UwHI-zL|U=LMuFP4jhEkJW{mu~C}z{0|M+)AH!d2wu z^3?wHAm=i%g>&mIIC7fHB>Xt>}M3d2st53+fks-xqhC(l&Pbeu%Yu%G~J-`r`` zc8f09M>h0q*h}rW3li_iI73kz_Jw<3SN~pV{+J5?inNc9AcQ(dWZpDL^k&%&$7|UI zMUKJ+Ih)^UXG)24d(p0DdxW!RRA{WJ=aEt7%iFe>pl3L?bh~Ufa^*U^VRb>5D62{=DG4lyx>?i(rgzfeX1xE_^z*Ex&eJXW{RPp* zNzdxQ-Wu8X;Yn_yi#->~)|{b}PtRrs9g%E-G&ysG?$_N%he=HMxF@k_5Wi zap9c&+!M=#mM=b9gD!P?v3AvM;rI8xX&lCi3JMD8ARIDaq!LG4Xdkw4?A^s~AqzTl zzQ5}tw^Rh?`fak>&NC$KeCeP@iIaCZVsCaQC6gZnG#T0em1+r}wqpz9Y{u&twu(?& z5t8fFDXMt^ONaV7DZiHN>BZ$&v#2k2jdVxiBGOZhm)9Kv&{T|A$d<$da`mTs2Ftjc zNFh%<{5|FuJ5V;m=1PbPphP*d7nC46=(C#(X?58YIQ`)+{IKVy_GuONsp0uzoK)2H zW6#8(Uky{|(?59gZkK;^H2_aeNOQM>B$yD6M|= z6c%OXb)jX51{YWhsR)agmQ0O=)&z-ewwC4xW^E>j=Oxn19nbp1UkRzGsB~V<8InFQ zspXA=dF>))>7IlrgT}oxF_?`Zp#)Y%4lD3YUs0Xf1GX`5e_A}@-@Z~T0-zxy!sx5& zMK6*2>1cb`)lKG8e^fCa*XyEw|F#c%H(hxmLWV8ycx4!N%e$iX6n;Z0UVxauX}k&R z^5drtxWrp~I`+EpE z8+?e)G8j@Z@H{!>Z;_rB`DJi*S3(0X2!CT_;7 z1kV<*&&&;_(Bq6Ryc~C&XzJ+5$b|903X+-CO1itn=mBNzVD#_c%B5DkuJ=!b7g2>f zDH?^DKT0cA(Eq5gXB1#g)@FSTc+GI5G@?rO(YlJ4)?d+ zXr2}#LA5B#nn4C&AGu3#F!XQMg;;`#r3r|8n-(wmLkkM_O~m}RX+ON|{kM@wLnm_~ z?=D>TASoO!cN0=zmc?Y%8kO}V_mSe_Cki#!$Dyx>NFw6mFmjjYi_^!){+H5v>ws%5 zTRS^$mE^Rv(Yz?|{Kx}!fOak6f&WG&$n8JfRa9MN#SGCBw*fy&EFpg%MkGwU-I@E zOc^Eq^2q+6P)Xwe(dc8f5l0{;ub1Qhv;de$D$}^VU>uInA6%I5hJIa`yX~L6SsSBz zyIpL$U@OmRLj|*WgE^;#{!=`1wEw=y1K^kW?pW?8Lj##qM-lfTRH(<A8Tw7TUMT{qudpNu!;Q1_khuHSCD znYh8F0J}ME*pjH)Iw}uHU5@K&4`H~RvAs=E0}$0uu$+0HLgrqtnY+$O@4(d$v!o?9 znk^Oa*1)7Q-cJrGvCG5`#*v=yi}TGLHZB#TXFI0Y2cZ+NWTTQqM~A?G@W!IykAT| zG1XsnBRUY6to8{ylHj^}NH6$tV_hCF>5=O!Fq*6;_RFrO3l1vVV$3Lmwk6nnsr0&Y zU^gp}RvE3#m&dc^beVX_ri%y_PR||Gi3Dh^?n*_A%)jDJ@P(C?)-En%UXSZqPGotT zStv2v)HfuChh6G5qf<}?IrYcIq62SXc!x%g`@B+AIDCc0r|0Rq??A~Z#opuk*Axrs z*C`e~W6BnddZmQKlHg-j%c+~dC?Hm||E0*(C?YBY~Q6dqdr)?JY6 zL|wXi3kXAw@{U(kbTg+BbiL#xo)>t(cZfvM^Z7Lbo7qpj-?tByM;NPcdvmKcXhNR` z-u+SwOsYF6CSnzEfAlU`H~J-JjP~d6)Q^pXA4t;D0^pw1Vn~|mmx}+gp;P8YJ+hp~ zh_$A1=)2M3c|?bgUCOJ-lo^)?ts~{si&qY&$iB-PD4xhTW0BMlbiW?ITMT+6Dazc5 zQxoFTD64R*o;Q<{LOQb!wXS`g0D-X9bSJuuZv9Fwuc%P6z4sd#PGu#9ujSx%)lxHF zoHPl))uryLw$H$9Q%YP_vHc-$1aR1Z*;^+;703M&oZv-day6a zJbFB?C*JLXf*BjJX5Q|>Qs{W(g9=6Gb3w?gDnNO#q$27@X%t@5-?;pUy#Q2Eh2dc^ z-=I~Zv_cNNfUZ)gtesp`Snswz7v6oeBcF8k{bMR4^E!gc*V@sc69sNNCV_X`m3-GN zYrfh0(ZPgwPlAorcobYVy!j6MA48zz`cmwmn;$9l=D+rue^+nMK-MEGOp7d^Y6O~h zr3A-Ev!}E8DBKNgL&kJECz9*_<^$&Z(I|c3ol8@MVADnp>Z`z>HE?RuZJwpymT|bC z3~F!io2;x{4Z8M%6F-nqwEva73(?Tt)A23r_=} z=V9gK{IFrX-sfVcE9>MJbAAyR9NmXA9xE4gP{l8J``N%>%v6>3IhTOx{PeD^adTep zw9542n3Si3)0YM?>Ypg=C{v%XLm7w zNXYV(2ol1VtR0}EtV1YazZOPs7&yA(Lj@v{4JoOY8V%#KWm6^zKGoZTFb>2GlU=uDD-IO?==L{q?7<$fqbzyf7}SRRRSE z!Oc=b#LmwGOc2wZ-e6v%4EYJ4Gccjs@L^8*bsLG>HP~$IOzW~Ff`Qf^ycn)+~#b%->=}YjY6`2IakVi zxiA$t89gCL`1B&>ndek%{bXP;*E0lt%!+DOD7HVQALB#u+}HePx~7m+o>zQzx6KfN z{lR$$H*_{X^JFOkA_N(AO-+k!z|ah|ygAj~y@tA{hYPD|C656pxLnyr{K!dU0TU#?; zA^kToGb0U;BqyC0*zgbblvP!2;{@Fz(aU~p;pWt<{R}Flg0@Z;CZj3A_C!=4;Xb&G zBz%p^VbYuyqcxZ>a6LcA!92>QYGYK`Jz!L#<##B=@tgZN=!7lLkQ@KAhn@Imin-#$ z&e1TW@%3j4zl8R>h$tz)&jl=d`6};JMPl0Y;C{)pe~=q9ydR6mtLW)*77*88MTz~I zLPh-8x;b?U-gF`j+6Z#lee!a?oSeR043**;6J8G`5IAn#^uC(a)o*iUqz=5LCFgf! zz(rNp)rFuj=luBn^Gh2XFq4S-x{uw}bUgcxz9$zKdSL5acP5UmC>G`1Z@rtfV-cnf zY)+3KYCJRT3a`Hig7()P_}JpmYv*{_8;p$-1c_%aHYHe8%Fign`ddf*$auuQ_>s3b zJ3*kFCSgM9j~}@E-^wh_Em{%(u&rU{Abee1aG43co_df-X>r5_#2MT zz6<3fH4 z{64d)XgCoG!`ld;ISIV4JTnrj?i&mjdLz~t(-i|;N%R~4Riz<_c`PThS?ofEF>zZiuX3y zY#_)hh~QgXet-G-DOlFA?Mb)udQP$1=W;T~XHFlJG|Q%N2FOlN$mU2=d3OuS^WQ+r z^SQU-5WGRj3EIhI0ZY;>5D+LnH|+wu?^q4G+jdKZ!BGihP~>(E*3J%wru)8DP%4o@ z>-tSYxp4|{H^2PDjY-LuVlEW{*RrC8Nmd&6n=8O#M9ZKT&4$w6<_}I3Km46^Tlp;XNlW-r=}zL|>4)E&rmp8r`)wlU)@T zXU*Y~F=HlU{3P*L1V;0>BV&>}AXXKXknv#v??;Ez_Pb1Jz55RsZmMrLXKYMNg6Nwx zbBoKz6^;;^9WhEpm-32vEf)W$+e<4i-^qIOX3bHQaB5UQpNgp*#8#ySb1}nG%hMjb zPCd{+m90kiHKoL7EBy)c*+sE3S9#1|87s8Zt!Gt&`>CTWHM7gysSpe%0fmH{F|P z@Gx@PZ*AU;ZR``#llCEU+zzbiqr{T(W}%(HUdA>L5u*y;uIS2!(#$MMD`|;O_WL=B zx^QaQ1-!>#73N8ukz2sVB;R_C-&-HcN<8N*4Mr=e&yu;Y^>+QxPJ=@2wiEa1VfZTN z+MdDZczbu2E9_l(jmlc!gU=;~aOZn7#GiArkN`SCD1h>VS=N8%--^rQ7U7LsDW-s2 z`TnBvy2QCOt7xUi<(5BQ+5=KX{;N1=50CH~=W_wByOg`C4+pu z-*MrJWK7Ct>IWQhMHdA$OU7cj(py$XXA$w0fuR8xMWEmBS-T`6l##))E$|}Bb-Ni6 z{n$1&&YbIPOAbK%pi*$)*LfYipbNH9-R*W`6(?JFd<%0zAxK1W_3N*fA_2Z5Q%#)Y z4~pH>-<#T4BAHfu=rFrxFcIf?KP?RL&&vhByRJaiRHimOXvh|JbvWTfrTtnGmDqFu z&6^fg5fRo@#O#9?#067eytr4gY|-W<*uv&oDm0}RGxvGnL~q{6@w#;QAEw5qBm`tG z1)jZf#@@&Gn=T?_WBx=%F3IpU2`=ve^6R(ZsW@a!yjNB z=h~{xH>X5t2_7UiaJ5BN!y2=uRmloQFkYM)VHw0q0!(tP>ZcgW-tkIg#@*rzm5Ujm z@3WWq0Zpxz{T2tTw)?_b;Lqv2)0)}}Ndv2CY|8J8`w0rRiB$3mXPpA%qkV$s!AHii z)*4+S-2frg-ij}LD2X;iVMO|UAQfl?I zwJ6Ho15VfFI+(HGw2piR9SsE}ncH4bs560{@wQ{2~kES#~_aK*p=U2;A7?+Bb~ z%SjW4$IB1XyWwl&CD+8%TMGxu=s_Ps-_ya55HM23G_7g%Uf+ELoH8@^gSeH3h(E*O z({Wsu0057XSH#D}0|mN>P0zp+{|KM$LVSYYg>6Qjuc9Sy%p1>z<`)0O(T*Q@a=N9` zkdFYoi4pl`#JYwR3UXaV;zw5RW0?}DEY(lYesCXs00FPFyx@`z@-}ZD{eHX_XBE0P{H&s`J>Lk zkc)ZO3njZY@6ciePO2;9m~NZQ_dA?|B`D_M^a5s6(%2c3y0PAZIf?(e2p*zr)WXqU zn5YLOAXi%yySS3yD=SAG(I=~OSfXuPK8*+siLUYj6v)`hb- z47%O5zT+?z$kR&ZUc*ZhYYZ>WjkOh}xen~OnpjA!+ZlD}!T>OSH_I;MZ%ElO>4E{k z(H7aNBG+WEzwHYdu!XM|sjqb=@+PbPK4)|F8eNDIrLadP=JcaG$jj4}9#xeftk&4Q z7>a9{rpPnqNgB~*8*gVm(ZQ^<==M5ZmT3ePew61^Jl0b)(h!A9l0|ApEvP;1J@e=D zjr(kr_7|RYIE&OKpK~j)`4x|=an-Co?EdZMaB6Sg+SbPW zp}7j!()jHydalq*?5}M>gzt#k4IPB zR2v&g)K{2xf(e#9IfQtiMgtgn*s50xD7g+gC?5(x%9!`8_nqE7?!_e?nPgFRpcW{c8Q@EuN(qWREBn|EH?$gAt5($Q>Poc$C&son4L@;-s`a}}BoD~`Y^ z&&}zC`PvVP&ActOKATLoaV@A&LoLbEN!CXq8*@V zU@m4AG%mYtFNwIwSi?w%H)6B4Lu*jtn~z9nA5`S#hS+CgQck1zC+#NW;!E2asP`5v zmQwgl{@OV8^C?Pbf%9A@yt2KrZMKK?M!;1!ZP`wi>NaTNVh?Ogl&+|gda?2hQy?u> zTf*TCAGL{hPHNR{*@u<@gHOL0z4_~I7hr5u2hOGUa57^HS_gpZ>rr)nSvU^I5}=&k z#{k~G9h(oCTG*w`=D7R5Qnx4P1;jSo(~iXW{>_>y*7~zwpqsEToLG8n82`B6Xp@p= z6>izw)UFbo$lIyz=To1J|hoz^u&3pr6815Td3)?d>-%{!>Cp($n{v8H|Frb>`L zHM$4Fiw-I9ySu!CXtcagC$RZMMo{}kZpMq#Rv@6Tf_Rps%!hveG;%!~U<=ql^MqiH zwb{SQuQrr(aZzim7&~0>OZo=-6i^cHc4a3@LC+sg{A1W0QvEc5q6}}2B-Y{co%vg2 zF?n?Cvq7w;WHTZO_>Ts0kK_So#OXKrz8j8(TLA<%YYSP&F3WZ0ESWV;lBhr@?h`iC z{@Mves=NFMQqkt-kuwph1YQ&D|8#ZTPDF1`ZwW?jCsV~ilOFS~x8EbC=GfzwVU9Pf z)p@^kDwyEGL{0DdQn%*&;Ul#MyqB|l8b=kEiUkL(s&m4>-pvi7Wb*BCjv9No1r}=A zLv~KHo=*R^L0VI^2yj{^7d12LDPB+Z8{>7K&>rT?3;b>c2^$hgsZAoRzKmE5p5m#A zwN)y;TAi8cZ}&J@fOD7sx`KP1W-!vy@$KDtqY>Vowl8Y)HKj#G-s4JaL4OOsL4xH) zdX-1sJU zw83}0^RySE(x54Na8S0{aV2{GKt}PG46$)Pywc7T@_5Uf>#jL>p(eH`ot2$^dzLD` znc(#lZw^-y982Er_LeU=GVi%PSJZ6^?00$ioE(C96;-vl?IFHjM(}eHz5NFWuZAw0 zcL`ZwrJO6I9PzZs?Indme-?mHcNL>a5#YR2@$-|M&*iZxxBtr_Esjz;$MzY7y0o$~ z_9W=#A(n(YBp3;0h%r9^jOYO4Xq8bs(*L{cA{q(PghjTeLCgx*F5VLCvSxT;9mDfU* z(V0rsuwjn2OiBTRilVUpsi`?BcBArElO9 zHH80#UMppGcGl&nX&z2Tx2=bh3*7$x(h%PP2jDJSiE^&%VZ`BF>;H9MY4zK46tk001|8TtNp=Y-90B33J-cB2J3 zm9^`lsk|n1>I3}=(`IiN>Da{QpU@`JP z8?b?-q@|rdHH`19%293Kw5)~{u=rH=B&n`~pni;|=31x#4crtQiq9o}@1BJ9SB*kf zSlY_#xZ;v92rU*qtxm$v1uSsp2hYjMtq$%V9JIYWKSasG1*5j}gu1Y+y^0(W5dN(G z;cxy`Z~Efm;*~eQ%VSA-!na4#-Qe_ogX{Ki>{G+1aVt3xL4Kuigq&Bs+oZ1C2~VlL z#aUe#8vSULaddNAd8Qt4@pwTQs5s_pr$lJXxe%a%o9zw3rAY6Nm1cRH)q*Ue;ebV| z`Ep+jCk7?Z!=l0F+S*3}ySrfU+ra0G{EHw?F&xF$l9Q1#i;+i+%GnFXN;oT3=EHoj zW~BVhoD~d7AdRso?lZhf)mg#u{IbFf~oUMVtmEYQiu0;l}MV=KSye3&EJ#?QuIR!la^&DRoVml9y^h_XKbeKtVx) zMNRch>t;M?Z>qTSe3*4KCo^w8U@K9}`*@EzdG9u0Z zOm1$siCQ@UCtMM+Dzkg4g0aZON~;6;&3Gf_QAKExsS+Zb1#;d5b=XWZ(Iv`j*0M&2 zn3Md{J62bsoHXO<=(#0kjwcTR7tgb<)mvg5%vEjPAJnWJ9Lxgav{VJd;5)=E6{(iX zm&*L&1#ob1AS61GS@!1q-eR)>BjLGzzxuo+YxYrMcUGH7M&ay)QPQ~KuVM`NIXYS* zOy5w#%4LtAq>jsSwlKrx7d3aR{DWu}oZf7Fw{SS){oT5;(~o$Bx;qozb@JnNVu2fW zt+{qeI+yb@=36<+!BQvA-3h@6ONxh8zY`5DEh!C+NfT)1@6$etTs6sNsE&4-MlPe3t0zh%)7etT_TGw!7j(QLM6VOPpe6z4eJ)7m~-V4nnrWNA5Fm zcP0wgwqb2}(jn}PO-(y)8Vbw48yuptiUi-^{#mtf&%Y|%4?K9G>glRN*0~|TY{W%7 z(U(+y3tt1}Y`!=~9MK;?qJyRTUP%`vg7-(cs&T|0zrv}1D^g3*m=g4k`W@{ORie=# zs=R#hRG9;}abqp5FSsH1_oLz9FJQ3V&7EvSjDG;{C7;_cyuA)GYKvbC-aS1bH*Fc2 zkKZ~c+B<@s$GvatF=mn%;A^1dWXOWVlw1)&N>cDq2(FEM(WZcjML~7%u3jV&&<7Zi zDA2gs0DyHkV_no=O3-sq17G&UG$BI9i_|$TOn5gX+cKq&&09-TI7n>SiyDG7K2FS* z&)&wlw9XgxHX=Xh*=Mjd8~|$%5WYmBU1fy=c$@~UI|X--CEc$R>CRw9{sP7Vd{}+X zTiqwva2Z$SWdWo^>E~BTGm>!ntNy1zad?16{vv@%kB$0RY-#Olg+gUKALYgR^N&y3 z1xWANKV5Ec^Q@8x6XgS@^o{rV^vAXzmwYOn^E8Z;dTfQfX7C0Ej0J{X!>*kGg)2cj zJ@CBtW*N}U%E9}!$f$T4YXuVlUnTKRU; zgVcU$P?Q2M0ykWkm$Wlc&5643pM&jJ>H(XP0IibK87;MKiuK!vHIXM6bMcQzId0d5 z(cD1~`}Qrg1K|V+oj}DZBar$pF@^Ktg=aI;$$yg^$j4V7(7{txwdQ}JA{$2*RwH!| z-#cNh4Q|<^o*;fGe;4_DPSR_D2BE~jH+1)eRiyS^iT`N<9JRkT%7HutWKg5PwxW*P zEv0ao-xRe8ud#1Df=D-=Tm=c!V4R#3rm!=aUAHyU8|3F$~^uZ)>n`rnk5Z!)6Y^t_Op3V01hC!ntw$TvEJ^A^YC8f)qebik1c%2 zg)RPg8tc*dw3DqYc5CMEsp6XcEWgn;etuN^c0LMoTRC9q9QNlmICn#c??U>U%aU$a zDCTO=#(oHt@yFQE<%Y+^PY@%4{!MZT;ExpcW>C6luFEo9Rdl<0(+vrEjbZKRxLVj| z;p#sTX4nw^ay#fQN3F@xg+og?jFM95r%92GFDxwR=hy7b zrp$#qM8SsB`qq_d;S5Gz*?L7NX5uog+m!<3>+0$JcA)@33txKZynHj_3cQx8T`+aIVVG4>&%pCJy~ed|9R&4^DiS;<@JZ+dwO!@I|G! zvqgWSnC7(nC&eBWeu5AZpX&*T+}cqkXJiFO6eJ~uR8Dx<3ysE)ul_z_#NgV%6AS!* zy?td=96{4A0TMhA+}+(>65J)Y2MDsk-6dFXf;$9v2<{MqY_KJ`+u{&papw;2cmLdT z?)}d9>mClAWu|AgyL!5+y6UNCV7Vp?f6%^N=2R*c?1V}cq9!ydWy)sPe~^mA$ozA1 z+>P|-qRYs60598Rm}xXeBn^q&D=3vcbFRj?BQ@{oqV(a|vu88c_jn`SB5p`<=R7vc z>%K`>=p<$;a=`pi_P7neKFLRxFrrg*Q;7HUbZOLUb4rxbvyTbZ!HX@g#Hb)C;H{&G z2TWi?k;r_VYo_kl;5myk8q~kz^!M45I!%*4U3cHc^w_>5F!7NYm}S)|yHYEH3iid{if`{O->$2W3OiG4mXIeo&lV{prSpQY zZRcvXfqAkxT5PggY>Wgzwpg*U89df+d4Q<%)xU*gw=?3-b&1*^MstFPzH2nN=t5q` z>|HoY^1hKuSfHKcI4~NJ{`0VAN>9bSdPlZU{1+LH3FVDJ^3=(vlWG^-ox!dzK_TL1gEAiwdIP*IM4x$i?+1%N`w`dwdV{f>X)6WS)d ze^x$Ety%9S=4l=p`>yHpL)p2z;sVI$!u07*cPGb}y|MY{L&7&A!5`V5IHx&IYyprm zpvdsT^83P}=eP1au4oBUbwn6V6N_OuD@@CaOzhrNRA>boL7T%lAVQ(gSXz7xZo7Z9 z!Iuo@b3JQAcVniU?Iq!^<6>zr9Qw~i5Zlbugyg!)nmEr@|1=4)OWbFprl?BxwumXK zF>2CCtbgB#Ta_4SNwYdU-tC{RWQ9+F5_R(xC)y>K={V$Sjd(``RPFpIDGBG`ca)vr)#<#U&hx(_ z67l%A9@g_13$qw>x}ick&w6f8JbPsc6f5Z{34v>ht~W9@T__P{0wgL@kLR@#`OlPH zqrke#&dw6fcJRpBQZxSYQ_ZpTH6oXHA`A4Esp{u;D~ZAmhZ~|R#5^QI5MB3^-0_tK zgDa?y)>wz!Ic-nKae#wlkotrh*}w`HDoK zYIi&3q~7^axT3OB|8mvWD|9Q`-IhqqPtZa(W?ew{SwJ&hkaT5}ju0JtUammh$W@WQ z3aO=7yL4xH>9Rj55cdd~3i&m0dF7J+cf1|r2)ROy-(%Ih_V9L~iz*emZ)8xCDcwlV%Xd|UBKhmeAUbXZx}RsU|PPb?Rr;u+I?r>j_cxS044 z+MPPUL$xz4Eua&*VXU|$1Ag0z#|zCe-%utBG^yh%Ykq9;Zz%MJUupTQhcp;87_Rds zIgRH$6z4lOkoN#emJ+ckdb;-UuaY#_?_u#8q3@^Mi;{t;&+mk%gvbSBBNb5+cuhsg ze5W%<$f_O66>DBva$-TfVg-_Q^V?A*Zvv>)x3|UIXZSG8XBh<$6A-ER$*w9XNc#Km zQp2<*8sOB>=`6vSaP^o9Kp!K|3xImL+ULN>?Z(L%gzqh+5b*Fq z{_^Gw4z47pV0@aZAp`O-ACQs5etWsamXydcztN^rL5QRzev2aj-7}$RFDW>9?&%VV zPpl9nczj-wiWU1iT1KeeONg!%>`XL)%q-oX#kk7yWz;zeog62j>M&BG(0Q1eYn9}^ z1$BsGOdGK(Hm4B$g!BO7v=^O^Hg6Do!92RyU#VE%o5#A!3gD-`GT(OIc-*?$+1YGpF?j34qya;kapucOWv$(ZvO3kzs{!8hsP6!ne_vOp*L=^PK?UNeV7%E~QxIFnWkr`hs!W1y-FNS@9y#&e__4_0*pkR6C&kx2P%P~!sVdL0B9X(+> ziQT2w=|P!#{eO6?D@-HRCz*t3(+cbz!4cq&ZYGASiP430_4iYq2-bK~RCk7EY|del zvI&Csfs5QwM1|6=V#XW4?#4LfJ@3NLBr4Z#Bq-Xf*!^LLxU<+TzV`Y{%?;(;kqRU; zt)E?lL6N)AP6jB8TR_nhH$yZ|HJJc!IukSa85>=Vp>0pmepu57MwuCCmyqm zS7n1Gd6!iMahZL(-Uqijha(BiMz>o?7VvEb5PBGhHVk;BvV^1r%c_1yJ^n} zTULCLX6WICF1V!PJum(MZ%oBfCoKWc+H5!7JnvwBv@5sOQooGOM?yi#s5PoV;cNnf zOSvAqO*Q^YK?Q1NW>b~TpwRX~?>vuU%0tr$i$eyX6+)x{d0J*R?-o12wV zEeusDHQ;zUNVOI0x3aP|tC%4%eKju1$yM!HC6Sg35bsIs6Il&hVt`Cb3`9bgv^PIm zm)pwoSzASHe1QIgQESYMZDx^r1(X%7?yhD06w=ddVaQLGk9FD9l-?Yfs<%psi>hg8 zD8-7;K!}d@Rt}&Ki3f*(hyl-Ja&_WsiEvUluL>O&^52Pt$hl1_TvQz7n+d)gJ<;%* z{YFm@bzQ?O;k9~J$qmjXKE8ga^B4G6z~>Us25c|AM8V#k&Hm`f3n&p>em}{IqWI$vlSu|@qUR0cszs)>In>gsVJc9jB8!GDu~E9TaWiNdeS_^B z`)rNLfY`G$T2m7r#ICb{4k}L@wYJ-*SjA?)zz>uDu7&qn`!Ev*q6@AmU`&tHapWZ) zIJy2>{Yjs`qU7D%Zzwm9s$^5cuv@=V4% zrjTAbh*&9D=~dj+Ht9;UFSNG>wKeW3T9H_*+frHBF4`_TA|F=R9=ZRhYQU?=+Ak{_ zH$zl$_jVOMGVFBmZHuL9hc5Phc{Z@uNS9^h@+rQdjX=}gUo?BzV~UbwB`#hGziRFy z4F9??8&_gW)HYs0{Yq46?o09a+Y3Rki(~S?%h#}>MqjFS&E#VeItn%t>(KC&FaAoD zIL5d#mZ$5zUY)Bon4Torj=pUIig9vM9x`DbSGIKZodE=B(sVW%p*-7roZ zDN9}SCYy1HRp&T1(cLlV?(QyeT0OXR#4f$|M|M|C&UTG_hM zJr*Hsl5mWl7Wk2_P8d?VHCzhtayu+=0OO$iuTk3$yPst&{8rW0H6WMKf5M(M2cBhC z+ZLCjG0fN|dc`c0K6l!H0Q-+jVsLOU5W*i?WZ`eZGi$k@zX=X0tEJFNBv!<#r6k#5 zVQlbO^0p>r2z;kk7k7ZZ8RlVkWse(O0_w`6t@s{W@E!n&Q?H~7$5IQ@PVJ6<|bNO>qan$~)7Ctpg1 zu#PZl5~wZZ`8a%Ve50il~C<-*?)2_#=CHbL!);0m2)6)%#@~hPqlh~Ck zk31lo8JmcF6VMYMgCZtLE6guP#)DeBHaAnJP@g1rjn${um~<7_oT#Nif3FqnjJB|x z*2;DKxf{7k@0?VGfc>*k-GO#Oa)99gC)2IFAbxz|%$pC_KZFk>gEPfDxI})Hj#VjF zW2N&)bKgIC$1gx)hB_&Py@#E3TA8b|joe3dRqIv7Qj%)_NsiPV)!efjB%~w}Pvsr1 ziKix=cu$E<_gggb^1X|d?Wi_S@5%q6Ug7A5-ERyM#VJYdc)LB7M?%K{k$xqaj}@r@ zY^|kDgCCD`dXExDbJIy4(~tW_WrmZDr-?hi!6B+4PC@jzN~s1!JYkcZnMs+Gul^-A z$%c1_REbs3QlCE@w0?aXAzc6oHb~u{W~g&1fY(cnt+DNgz7enerpFprQL)!9D~Mgp z@2mpo!%RIXI`7fLb8|!BO6a+Fq>RO0`%YC*8KdgFk}^dL&bG9LGkgHWgbk}koT?y{ z6*5DFx-ro)TwacXeSp59QpI9!0C9a6O=RQa;x1=BFt`Pt=}Oma!>=@_CcV-f~W3Wj6YjJm0A zrZW_~f5mQu-{ZY@Q-2am|0NhAu8?ZU{rL}8Fh-x(*7L)=AfOcCr&*t%*D2-YpJ7qNMHhb;FSP}-9V3y~%J0Eq>_?;*64ZZW?$FK=&5R#7s!IQhE{ zOa|^&yPo*ViQLDD@~sb@h&yzrg#OFnLBv1-Bcn}wgJeO&_tdXzyQPHYy{!#3kY;{5NgxNQFZknWa{1Q58@)zye05Y+I@ z1GgOjV{VyASfW5l_~Xcl1!z zxVtx{r>gK0mjHfcT-;kfaSEVT15c}qoI+MK9FzgHpPZZ=J>8%I{7cz6G2s87fzJVP z&&#lvetlBGXqPK(Dh|^CllJu@AeFzkfUE#W^6RG{z-yGd#uhLKXg7XOAOkaLLmOq8 zF%4w1PJexf6S~rSOC}5eN%KpZ!yogGc5k@AQ#T6%099o>xNjp>2k>H5HC=If-~ZJw zn6HnJv4|n|L@kUX3QN!nsSVGvVUYK#9-wfkTHMDkRc>sjiQ)WU-M z_-*Utv0D9Tw^(k_TD+w`q~(o@5?fG8hagGd5teKWs|vUT@b0)D*XmFp{TM29Ikgkz zEEnWp-xv)!010BE>Y<;arCMuwedd@yH>#cYubshgd;jdn0~2*Zoy6Y$+xhZp4{b!p zLNZDWhbNh0)v24}TFRGwbuPGh$j@mb+<~FAUCgsbrEtRb$+yk+OrB)JQX0ttC-b-k zx~wW)b6hJT&kdK6&D3n>>SP`E-9)aGK&LdtifyrHN8H0se(6rsoA)Z zD%|1xFG95U%~uM9(_vH7@9#~_A>XyE3%A@9seg%N#oz+oLt2|nTWnX`B7WCF0pnTf zYX+NVi}N+1^Xyt~zb$!)=x(KS;X9q@GlK0JgPA`qQB;Dlu^)@ii!3x-73=sFJ+fRm z(tFzoM&Uj^d>3x+G>sj>PD^VD4Q|kI@o85C)7L_1f;h-y*`$ufKzo}?3(+(5Qk zrqnZ(vz8O-4x*sJ@8f0r^5t;P#*s!(Q;BvtvY_IoGk`jQQ#>%prU*ynoK{>g+r2y1 zi&T5+r!qpGggh_yhq>(3G*Cav?CjLSM4SW`C3&>DK7Vhff$-82^7J$|hG%fa#^qgv zEDf+bDM-M|+>z19hJl+cEi#<7i~$;macXWOljY>)yS zD}E4rDSqaxqcj&6%XsJBFDPI{#z?xR4^i$g{KW z4*tEr-utxBpA=vE$Qcj_M}t4m+)GgfY>s~&qG3gc{&iD>wh3rx;nCpBT;BnlX0_vL zqWCIeKyX}E#a=9j%-*Yu-1_lqYZD|9Gfn}dH|p$P_ei{_C6&Cs%Wk)Y5O1o+G;>)v zyX6tk;Dafu^3PAH-co$|oDNQUT)>?7Lf}J$?=^owaiamYC5pO-3=$&|TrU60{m!ny z4+m|S8hm6^5u0cp=Ka)+QBjutb#<^iAPu6y+m)kNgrhw*;f9$C%HJn1*zgb~muR-n zKOl#LCa6<8^~3ILIb#J=i9Bm;iEo0bpmoR<1?I$D?|U~L`y%K@_TSHR*<{993RN=5 zN*#Imy|QD`h_eTZZT@xzlxDxjkYx?GCsGwkjpX>NAbOz8c)OyMmtV%*UW>tlFEg_g z5hemOv}Vk3I)8Ng~YbC4gN!401`Rdh*5Z{A8!aX7OEmBUnoaC7g zp3i;W!z$WP1J``xEl`bfx5>7~f&%@mjk-V2yQy&?m;I+d`q>&v;$%*Ca>3S)9hkkp zw&-b5SEBuHjRYp@TocjEw;3l6c3xTI~`wP#h5G7u!|tNN_01#Y~f zH@!mG&0HBCg3Mjjk;;>f_{7;|dFI^{;aIaNc33$N@B4_#8K~;anYOra6ST4wWq&R$ zdAGlJMJgh4k4QBA_dS#Ri-l7+%`Eto;aug^xp!HP3v&YTZ;x3?FOGb4pwq;CJr#iy zEvxh~H%9dxdy-g_WW=!f62O9Foi~pCJ+;*dPv!ha1#!jBhZ)f=@|F*|o5oj3&|%7|A4nZmV3zWZTs-i;i@>~1USG= zZry!|A28ZH>QJ7P>(+%iS)1lkPMi~AD zaMW?!9=*}`jo<>YTE8LurEt9*oc<`71Wt69r-1hyz|;Lqs;P~qt=)FqewfZ>A^Tz5 zajiQCiNf#u)z#HBA8Tc$E62T_THvoC;T)&mD9g@ZG9Z3Np!Lo4@j%yZ{wLMu;=*{K z4^dZ_YG#RZABKR_)9o&S))PP`G3tN4BFBhZ&`=l~|Fo9ncL(G9lm#M-7z#`zGmCWY zLM?}NMHQGW(etbG+~}!x*}Pj_r23RmUiUkaf$v=ztJauQ(UGS3@)?PNc)$`9#|%@N zpn&trg~iG^&CG|i&hdGkXQV=gvEMOrn&5_z5FVca@K-Vp*v8J*XE6GB3(BGtnSU{%rdkSvwJ=!7w;y{Rf>XOr zew6tfv?wY070cAopX zJ#&G;x-pDq*EQ#!Z%Eu~i9aM>ENtBLC)~U6uE zc1_wJcp5_S)AM9CnBAldaMPW8UlZUqx=#Sp_DfSuOh$AykW8z+hR1%9m%Za(y~%hi zcyk#bh4?pLw;3^P&|swk*#siO0N+U056pTX3-|NOOa%?IglP^?=e(QaNlswmWmdaO z65b0Ph+gRS_wAW^^`pGJe4X;?e(gCI;H6>R)bn-SoJi@hWwH3?dx}964kn<*K9JVP ze(ym1<}&2l#Oh*9PfhN)N9Tx~p-F$^od5}i{ZoV{u>nYY8Xykw9E@jSZn_NFx3L|d zK%ZQg4K8g4Ca-FH{b~HnotHL**rgX=jH-|Ah;fDyKq z(@lzJ!Aq1$Lo~EMH-U+k`sK5G0^Kl-P7}uuc)9%c!~iG5PS^D+;DcP0cN&3}r#=Jx zFWGLB3a?X6U|y}jjNk671xA&C%2OSDi1yjmFDwAu8#tKR|>nN;IeR9G(<_ z-X2)2+AInbaU5Q9TSKzN#gIy~T~PA^?m#Wqsx`t}zAA}efV>sJOk$4&dQfZ*3Y~;` zEzAFNJLyyfU{5x@vXtw)PKF}SYh)7ZLkxkjUUnj zg8YrfOH?@3(n4Q#q0WoccnW7c>vyUyhxJo%*2h|@JNK;2OhREdoA_FVJFcZ>0311* zE}moh05yvv;zRA@{K-n9qRBP3<(gt=#N2PL;ZWV+y_2z4vWoOFPIA%K^`4%K>taiN zKrc1y%^KcZXE{htX`%N?&$*-by`RhJY%nAA5+%1=+8IdWuUjt(1SmmK*ub*=VaZuf znf;s2C8kcVSact<0bUn?F@PM1lD40Vb3wht$N>qE$EQIcst!-bk^s~ojfed5>dty@ zq@ET2>-*kVA}4dGW8l3Zupw8+4Qq4VEtsFFcL|8xCN4B|NYP_(z8Jo2AHFZ={(~h& z73x&?Y6@GzYMzidOPt-4FBA(tNFj+HHXiuYu(yq#PFcf<>bY{`)mE9{@?1f}Tp_NW z;f^g=YgAj=ndjpH98yelNOX(Bl59;!bOvt6g#wjC-Y^Fu;aI;10FRt1G+F_RKoXfC ziIP2Z&uVwgas5f6>%D6m3$+j32P5Q(u~sfkv*)SK#h$~pNb2vG{b6O)BF_N`YJuOq z?aspDU12bN2J*mnrX+2VyVsq417I~){YU9F;I0LVyge=HrG_=o9TMyHe?qQ7bI)!f zji?XT+Yx z97eT1;XEfME6M=fl;*3X7>Mv|!4OWJ`1Q3Jfv)QN9^e^CMl55$dP<64P1Rt=Qx0y$ znz1|4X@)OU6OBFgZHWIo|7TKH;UX50LdC3z==HA3FRgB$FRMkU|JE4TV5mSSK7zUF z{6uLURk{1rfXE;EBBV0OAsDo9{7&JT3vj&vd1+W?t{MJQz7eu~H+P*Zt3d2+7w6Yj zhWpZ%-ZyZ7ruH~g6W@7GsB5f$c|8A-t~^B$_2nc!aPl#56_KVEx?_SM_;>cttDxS( zB07`a&x>7H+8c1SJxxQmCr_o`pJ%34AcX$2YTQ#S->zXOoZc~2IUk2!z8p{1X=jdN zcT{}YVwc?dW@EQ_v!OsiSOz(vPvqecVLf?>`{C#BmkD)COCMq?PabCY&m2O;RFU@d z6BWz7fbF5A8_#hzxUF(I1?zP}YU}^~(I~QT4f6H9w=xiUTO6>~c&QKfuu5qi;4Ir# zL1$D(0;Pq$tU!Z#dsuj zwO(_bp1Ykoe9mfF^*-5Sp*1>^Usb!VZU6c)bgp7FQh?t-W}&`ic?L_S2;(@fJlwJC zJhxLcb?{|9fp(&CGL_NGll4DZO#kNvdd?v?5Xbw8F5V6>G5-GdQQ-cd|?|AYsip!F*B+=m}v0oM{%aFtM z`Ro#FPZ=&93YE0`|78jX#v)A{NtaAarl~`*3Nllg=(=4O7C=&HmOkBBk{8+&A+0!N zu84aW=R|>rsf`GRt?r$OE>xj=(bPz=9MKjlw9$^Fu+7PM$4^gHc)vtvm-w6QEtU>5 z!1xaS>Lqu`RMPoTr2U>h?a`DoIK?;jAeIv5uQ@IE?cFdadLoo|U$M*%#f%l6VWYjF z7!M{GHpZlqW=f`F$hFDgi5KC?+KtKMEWVi8*X{7$vjrDoK+YTXid}SnVrFB8xCXR! zEaSbCy!i3wgC7uB{*}r%o}lc0V&uWr;_zz6Sn-M$LblYr?qJcjecP4J=F4UHG?>V< zUNsP?^w3S{nM#)ZRF(v( z8ui~KuNgy~WZCsgv2US+Ecjq9!)8QZ-4W}UTz_T`cd9BkVkKb9HI3&K4;a>LCcj3rFI72gImYW^zIj}l#Fep<5E%Cj^BkMqC!V`> zA|;xbuY5`OU1||x^4^P?5@OkH`-K*0wM|s&s>8*??ve<)rV~gt%rynGG0+A|#4FVv z2F&V5KMxl2VE1I7+8=km)u|ajAY8fBo$YiMR&r3-Sp>U`oU3sQoE$k#s-R_LJxFEO z1<_e|4rRu6@Idl-T<%Gv4XS{>pTnbXg43$R(WrJIqux= zr`9pq(oCd09$h3Iul9bEXV>QLcd6X5c1gH9dW+#Y_wPti$!KmV$D`q~%df@%*A=;8 zM`^=nTU~vK+MK}GB&z^17@oJlc(p&z`JrpO^QKTGxZa0dE_va=_V3O^c$;l|554ip z*R2F;W=oy82`wJ>&+I)La-jX5Sj?5aN+I}e##kt@U`JZ7X4!t>rXK&CasgsLhr1if z9kr>|3#!(VE)fqk;G2nqW9v>IaCM{~&*|?%)?r0mIH%vQM}=q}5ACTPbFQyQRd4uT zBy3CWlg#J%+28-Tj&;6u?_+6PSTfDlEoxYrCwG)p; z{pcP@^OlgefqFUV&4Dd8Q%r6B5x=#c&)iX<#gOmZcOe*~>(-x9eckhegSCsv%whIJ zFLo=l0@bh-n&Exv-OG`Q$lM&Aku*gGMaQlM65{9;abTPcE{Ln@RJi{xEZ_i6EXx1PwB-dB*&3;*OzHDHk@f_#XmyMh>{pwnjGVTpK@TX=%n;m6~%6#*n+@PQ9d)2T80J zYphah4HB|m?ko649&)|<`Plx@rF30dA=&aM*>6xS9p13HuCO%)yTE7c24#sL4>By2 z=23k3gN2U&HWS`p7Eoo)m1dV>tOrc585AYM2~jmHRog*KN)WFkne(ahO;yF$>s1Q` z7pwKT&Qz{cH@_|!&Hw3f|DFznGSOqjz7U<^OndEH z<(pYsYP)5S`XTiT<2UM-ey}qvo9QQ1t_zc5xh;mG8m`;x4DQc2G%9Hju2>GY!$uCo z3Ud>0XO~2x)s@IPUMVP$^ z0(3mWx$E{X--q(lOU+O$=pF$4_^%TW)Tz6AO-_%ZPR*c^rV6v;oWM{lNjC;e$J14s0|reb4mm-k$BrI5LP$zySRo$a$X zx~e9*?bEC*k^%gAU7t1<&cAdBH{t3)1nT?XA>e@ zx==!S+6V%LZa6?Lj|#zF@8sbKB$J~MVFC27jF2@HwHNC4uir$AZ|q#STH{~d6%D@8Mgq3XEtYDCkhuB_muCDFbk6gV3mkRf2Ecxk$mSoi8A5*92n~`~jN05r^+BFh>v#-Nf;(B9rlF5C!*d%El3_}O!7(d~TPGGdGeoKe6!K>%8-80OItqba}v zyAOHc;`a{7>xTjoD8J^qU9>Z7sY4m@P@u1!q~M-6S>m}|{_Lg;-zK`~4rt3la>x8x zYIlcuMvQT&;$n(4eKhu8Epu)@Yz%Ds1&pQ%9&I%!8Qx^_$;g)nTzQg5SnO*ixf17!^7yp!m@rbE7lkJgoN)lzRTrtw==SPV65w$9@ zSl#lBLWb8lUA747+nPX+pvI5iI3IgrUtOKL47S9^ds!XJZC4~?F=6++Iw`^Rtj9b< zoM}{5UPjBY#WPlAa8@$ZZK9Ll`2S^9t*d0kFYAzyxh@nGHK6~yzTU13lLLiy5c z{d3}Bo8QMXlj=?v#~OU+lkLlzuV#fogF0_7TuL;r`#|pY@++x4mx(qTo=gwUZh>+4 zn{}uHiIlm;lL%h6_f${k91^>SyQ}K@mm=qW3^rX&OlojyTSFvkI|*%u3z(6j=UT@d z8C9RAezkd;CZ==;C6q7KcslqNWoVtY-}?cKSZntmP1MApT`eWtW|N@Nj4qLJ3nhUF z(6#CK>7dF3=fuZ6uk%ZzfXz!I1zpPje1^~m2wo5LKPJXo@P9@;BaNqVw!0%4I=?%7 z>D&YpXc_C*^Ud(4k@%wso=#7lk@qH{4WRzd5Mv?*7=(pS6yH#m=_y|3lyWs?XZ^<1hKtH{nUQqO>Ox8zT%Q zWWLlat@IRf2&RG-^0S*Z+bP?i_z+-j9CUx2jDwk%<$}E=fk_=KiN781dB}Fm_C>&O z?;AGGLGQE98AJ>koNaucBD0UC1)!O)j}WNMs}W+l`prr ze1X^szG`jJlF5(KTos8I?G>SQUBgiRqnNNYH~J*|Ny~B9Qb{zf<;-8OBuBG;S4S(e z0u!1-{!8Lvi;Tag>#Gn?w`RgCQPA_EN+q=9rsv9T`{>BW3FFcZN|1*n>_mGy{wUL_-5wN9__O9MOhL2lY&1Um;fBG`+w0h&WGxUS zt+q3FMAMo8LEW(m@0KZDbGy?r<(Ox!T?Pf^T$raA@C z5&g{_rB-jw)SAyumvw54gi^9}J&FwgU6g7qt{_nwI$mAI7-hfY(Z(`Cfh)67Lqc%^ z%T~l(AMd&16?atEp>`|(ELY4LLkD+k>W)y1T&ta=T7Cid1lA?DP(R{)F$d!1QErNK zNKSn*=fKh3qRno-yW-DbSTzFchA&Um^C{Ft{JBj5`&Ss2+~@Z#{VYO3a^_S|z+uP( z^dibYQHQkc(RS{vuW7p`tc)L_na*VYRVVfwJb`5Ec#vJBt_z%_y+l8Si2X)SYugW)Q(5&t9)EE=zq%Uhd^$FLdH7xiCbN0xHoVwm3%h^Zu8|D~ z9!)E?O(Z|podOqK8Nk%HiAU%fz`aCe-g=U{Tz;&VM809pCIut0%Z|q)P^tJWC{rO|Ou_3z z0*Y>)Ah(tK(A%hz**AtL_CeU3thj3TU0dJ`#$&+Px4?Uf#{d7RSDXG01`vH^NTbfR3W8U7}` zoj&JdnLO?9mn<6Bw9Dnej3~ggkJqLMILox#7|=Aej?Qae%`tA87j47rS+ zGeB5KS6gCX_bU=PC3Ym-Pv2_?a{mhrkCRp7RYf4G^z}`^$!AR9g;4-=n;cKv(OOU5 zLW6ZFd=Ybn2 zdhR>9Zury-kI!!OXTHv2#F!0;PBmnI@B<3wLJQ5r9iG#r@}-Q?KIuEh)k@^{e(mM` zmUVkpXUgvGe?NeLDf$bbg|xtD)uZI$0+dIpK+f5ugsFLtV_?hcpW?R9qcc4GGzk%- zzi}ne3B5yZ(j=ZOuflVRZ49Z|%8PB{DcRChxAgm1fE~5p0LFlP=zv9W`7XHMb#c# zdJG39KJS?bDLSY95elx@`N&;lHo+ z!Y*B|S?aRtz02jgquhO#QhT8XNqA~(Y<0CK00fLS&}@G%e@qxD?FFzp+!bR_m8cx1 h9RlILrrYONJi5!PE+IE2zCkp diff --git a/dpdk/doc/guides/sample_app_ug/img/server_node_efd.svg b/dpdk/doc/guides/sample_app_ug/img/server_node_efd.svg new file mode 100644 index 00000000..9aee30bc --- /dev/null +++ b/dpdk/doc/guides/sample_app_ug/img/server_node_efd.svg @@ -0,0 +1,1254 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Page-1 + + + Rectangle.58 + Key 1 + + + + + + + + + + Key 1 + + Rectangle.59 + Action 1 + + + + + + + + + + Action 1 + + Rectangle.60 + Key 2 + + + + + + + + + + Key 2 + + Rectangle.61 + Action 2 + + + + + + + + + + Action 2 + + Rectangle.62 + + + + + + + + + + Rectangle.63 + + + + + + + + + + Rectangle.64 + + + + + + + + + + Rectangle.65 + + + + + + + + + + Rectangle.66 + + + + + + + + + + Rectangle.67 + + + + + + + + + + Rectangle.68 + + + + + + + + + + Rectangle.69 + + + + + + + + + + Rectangle.70 + Key x + + + + + + + + + + Key x + + Rectangle.71 + Action x + + + + + + + + + + Action x + + Rectangle.72 + Key y + + + + + + + + + + Key y + + Rectangle.73 + Action y + + + + + + + + + + Action y + + Rectangle.74 + Key z + + + + + + + + + + Key z + + Rectangle.75 + Action z + + + + + + + + + + Action z + + Rectangle.76 + + + + + + + + + + Rectangle.77 + + + + + + + + + + Rectangle.78 + + + + + + + + + + Rectangle.79 + + + + + + + + + + Rectangle.80 + Key N + + + + + + + + + + Key N + + Rectangle.81 + Action N + + + + + + + + + + Action N + + Rectangle.82 + + + + + + + + + + Sheet.28 + Local Table for N Specific Flows Serviced at Node 1 + + + + Local Table for N Specific Flows Serviced at Node 1 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Load balancer + + Sheet.35 + + + + + + + + + + + Sheet.36 + + + + + + + + Rectangle + + + + + + + + + + Sheet.38 + EFD Table + + + + EFD Table + + Rectangle.39 + Group_id + + + + + + + + + + Group_id + + Rectangle.40 + Hash index + + + + + + + + + + Hash index + + Rectangle.41 + + + + + + + + + + Rectangle.42 + + + + + + + + + + Rectangle.43 + + + + + + + + + + Rectangle.44 + + + + + + + + + + Rectangle.45 + + + + + + + + + + Rectangle.46 + + + + + + + + + + Sheet.47 + + + + Sheet.48 + Supports X*N Flows + + + + Supports X*N Flows + + Sheet.49 + Frontend Server or Load Balancer + + + + Frontend Serveror Load Balancer + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Server + + Sheet.52 + + + + + + + Sheet.53 + + + + + + + Sheet.54 + + + + + + + + + + + + Sheet.59 + + + + Sheet.60 + Backend Server 1 + + + + Backend Server 1 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Server.61 + + Sheet.62 + + + + + + + Sheet.63 + + + + + + + Sheet.64 + + + + + + + + + + + + Sheet.65 + Backend Server 2 + + + + Backend Server 2 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Server.66 + + Sheet.67 + + + + + + + Sheet.68 + + + + + + + Sheet.69 + + + + + + + + + + + + Sheet.70 + Backend Server X + + + + Backend Server X + + Sheet.71 + + + + Sheet.72 + + + + Rectangle.73 + Key 1 + + + + + + + + + + Key 1 + + Rectangle.74 + Action 1 + + + + + + + + + + Action 1 + + Rectangle.75 + Key 2 + + + + + + + + + + Key 2 + + Rectangle.76 + Action 2 + + + + + + + + + + Action 2 + + Rectangle.77 + + + + + + + + + + Rectangle.78 + + + + + + + + + + Rectangle.79 + + + + + + + + + + Rectangle.80 + + + + + + + + + + Rectangle.81 + + + + + + + + + + Rectangle.82 + + + + + + + + + + Rectangle.83 + + + + + + + + + + Rectangle.84 + + + + + + + + + + Rectangle.85 + Key x + + + + + + + + + + Key x + + Rectangle.86 + Action x + + + + + + + + + + Action x + + Rectangle.87 + Key y + + + + + + + + + + Key y + + Rectangle.88 + Action y + + + + + + + + + + Action y + + Rectangle.89 + Key z + + + + + + + + + + Key z + + Rectangle.90 + Action z + + + + + + + + + + Action z + + Rectangle.91 + + + + + + + + + + Rectangle.92 + + + + + + + + + + Rectangle.93 + + + + + + + + + + Rectangle.94 + + + + + + + + + + Rectangle.95 + Key N + + + + + + + + + + Key N + + Rectangle.96 + Action N + + + + + + + + + + Action N + + Rectangle.97 + + + + + + + + + + Sheet.98 + Local Table for N Specific Flows Serviced at Node X + + + + Local Table for N Specific Flows Serviced at Node X + + Sheet.99 + + + + Sheet.100 + + + + Sheet.101 + + + + Sheet.102 + Supports N Flows + + + + Supports N Flows + + diff --git a/dpdk/doc/guides/sample_app_ug/img/tx_dpdk_testpmd.png b/dpdk/doc/guides/sample_app_ug/img/tx_dpdk_testpmd.png deleted file mode 100644 index 656e17b88248d2dfb2cb7fdb038507c1f9f1a103..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 76019 zcmY(r2RPQ@{y(ls5|SiDN(f0RgeY0b$_^n(vR9H>*(6Dlgdz#?gdQU+AxRRlH%YRS zz5cI{&iQ@M|6JF(&Z&5w&*vWR_iNpEkh-eEHfk1X3JQvCXBA~NC@9voQc$etrP_?Y zIdCa_KmKQrh>AY&+ zF&o=GIXmX<-d!Ph>yU-)rPJGESPlkD4hdA=8)ykOwB^pR72L%W!5tx5F;lUJqoyuMwLTI^)%RmOt8PuPHLz<@@J-S9%G9as)%}V=OuXk!=8Mh6 z&Y0|4T^{A&lA0rrt0kwV2Gbk7&`;M$*U5didzLL$HMh6lYBgrqS)5UB{&fsjxo6dz z#%=VoxsLhD`G#G;%{wCgx)m{HH2c3cB1PTb*>`jwC%=?aYHC6%-|9fg(yscUV@Vok zhi@JXzEf)Q-b~nr%3tRBsU(f(`soSH!r<-QCib(3D>yU*= zI%$t<%o*iQlbRz`lSK>7MPicl)4RS+rCl6fsFV_nRZU*bZ?v+0l)&3oR4ih>Q?7K> zWvZZLvEbOTV;$dycglFUahX+IRn^x1JRlq-d#AJDMz&462#r78%E6ygH-)2Vwr<_} zu=vswiP9Cs(yDLOeFr4861cm8o)9Cu8O1}`ITRP~&P z)bHQFCtM|8rGEbKVSSa^14F^exRdUm2B)|p&YEAnI`sSZ2h&UolD3uIp}>q~32U}T zMN*HYN2WNYpN6@uOoth-PB%;E&g9EHe_xlV(3)ezT559t)Lse?AxHX<3fJNET;50} z(LxT+^5ktdfI^K=;%7P47{UK(w#=j18lSn_4OYzFIRpt&;H?f zf|gC}eR=uD9XobVjI?G`hE@7hnbo(&D02tNCq7o@_K@NbiP<(c*ps%YeR`ei=wS7L zFh4*43yp&xdWCe-v{|#3wWpGsSC$tlGvaGAPVz)lO)O2-_D{{XWL~C}a2cVH`ZcMh zqT=xP*ZJtJyZVmq<>b$~TV?ja;+Uxr^V|2)PoMs%*k$~Kw7Nh#;rN3pI6(;S*1a;? z{b#Xcb&*ZV<1v|heC4m&Dq~c&+|{c*At51!3%}FY#GL~NCj#)ChqBFwTc6wlY#nF>dX ze7QhNM>mBh-a)f@K5F%kvE}XS>2$-}n%siiisH2RDnbqye;R5?o>yBPlCEoPBy)a} z&KNcxO#G>HU3|~guTP(+r-$0K=Ul5j%1px?DzD5P@z=y}jz;lQ@S*FTbZ^vo@7I`b z*|>gb#%@)lJ=czhDP%I;&NHOh$USs=`Z_JUq+*gr@3-e?JO(AYFS)rr8TTA_XOs4N z8W+b-?YF+BriOSK@7eK{J$6OYOoxl-_K^zzP#e#E*oFDS4x!n+!F%Sa_s5_eJVMdR zOQjr3>x@V@&e*FcD^GN}k5BeP^F{~#A2$ie;~pb zTV?i2C$0YSPSdSJ-qV`CbJiCN7@94P^*QGM`u$r*TN|Uu%M`*J{NeLw>Rp0{J$Nzf zMiy39tfjW!o%%u|qM}%`sGk@hi?q8tf;RG~#gz{^*SEKabdNVPZLazHRaja&Of|V| z$hmK7{uPPD!$^JU7|V(4zZo}d*uWbsgWZG))l61(`19kepi$wcZ{O%ulOu9+L<|LY zUB7-k&rpz!meR(?=6hpfc=bS@;{fr-d7ZX+j0<{tJ#&-&ckkWHdiU;gie@q|BQ<`0 z$xB*#b-otkA;w1g*X`u;^0Kn3YS!Dg<*0R;t=7I}m)76DHB;?NZ#t}~rqdCgG446w zc%sMWreSuga*9?T-t%=vhACP9=2-rq>#yelzeYz7L{;1TDosgEwOMr#kJU=kPETf| z+W0sorvFRyQIFoO#y!6q(+Vd(Y%N?_7{(~AOwg^;2b7iO*tA<%S|*T4bRiWV7RO7} zL}IRZd9i1=z8*bmV_{)3-c_{JAicb!GK_-t^)KE^gWEgFZ7+W9Jp2$n~MGZ=dwaWY~OtRB`#tX#1qA zBvGSe6VJbV`BE{tL2|aki72kObd5cCX159~-uG*fVaV#cr?Ish?W3)YB_blC-1Nx3 zJreSEc83dGMn9m03r5|XD5acVXf9F3OnH@XpvIv|YVn0vnJIEH;M4DMNPnH58mtN*&ygq|8)v9HydNn86MQwUlN8fucpmWV3f zPTOQX0pXJ;(`~LiiH$uFUt4XTNpGMiqtkKzt~~0IpK0bN%v0^ih`{jhaLcHR%UH)* z3`lT-30WT_KGH@G85^^uz}CYDCO4eLgj!qQC2DE*)mrpOl$z&hX(6|7ufylL6vX0v zh*pVpZC55c2x6&f9Wt{&Lidr4ki{@1Tx&k71+N=r{`q%b-; zIpMpp4lrP@lB$AH`{QdPss}!q2{Y4d#&6LE_^hmWp*)c{=`}PoJe9aX8&OsM=Z|pw z>eApfYwL~QsIR`KEOJk4Oton^!c2oIjoOI~h1LHx?qv1v-+Z(IXZSlf8)JQ2#EEr) z^*-rVgn{$=6A^DB9F5vQ%qUO1$AgOnE=Mq8H4a84+vJlQw$lb&(AV!zkThUd$HRxYQSN z*m0!g2)3u%$!SJX(F{w}VdJ0pIPCMuX_EJzn3|9E^$e?XVXKOg?3MkK-#R-(YX=jP zlKfChiATg=sHA9pFwf4!=0A71$hXwQ#M3jjlHHvqzP1t-F6U~qnJ~)ggj-Qt4o`AJ z5Z1@w)OKpWmJ)AieB+ZGo|gVe6QI4$P3O{&() z=e>#0x0aT`Qj-?-l{YU%1PlbD&TH=sD7#%}@m;3PHuthf3^_8_t%$v#Qx)h{Bxdvs zDW!RsTz`+;mECFE?EgrG8|~_hm8P0nn9k+=nRn*dyGhMyektAK$z(E~MO8t8GCe(A zpOYq5HK5a$Xc@xM*f08ri*n?WXQV8{01b2Oy6+j0n*Ae5hyHT<&W)V?6f6;(z{?x# zcxL>sKE6J=VLm*p`7^uM!?p=4m2|1B+QEXM=Emk`ySWyNV;Z`;3KuVS`q%OYW7!t> z4qG`@k&PUA8BI}d7);)@>KX=6)rPBdYQm>S^toVlg~9$ta+343?!W;sE=r+ zHEyFMH8U}%q$;IoCFltN017eFWI5^vEN$EtkU_20rg3cHA>w>f5JI$>{uh(ll+poe6ckKs@_nRi}D<=up)1 z+@|@`($Y}5Sy))U4LO(hJ3NYtBA2bh9|35l25X&WMu_hn9Ua~1a~g|VQ&ST-%x2sc z&?s8jKR^G3>Qk}O+>&J*)Nd4E0@8p&yvfP=2o!=HfL#R61KOmYuRv-(C&tFj68fOV z96$=66I_vy6Ip3V#$TM}V)*j)E9l+iyvH#S5lguPzVtUJw_I@0)zpqO6jaz3{(f+3 zGQZaIUI4pWV99(;NpI|puHFFYB~GFuVdFl3&e?Tya1VA!XJ;o$A#--?7k~`c=|0A!#Nfvfxj0&Wf^ zMa=N`sw4&mz5P+5LfD@+E7Bx^UP5*q2JJ2@D_a^|U5cNt3-c{#rvwsh7hVqC+DiT=BJ< ze39s0SjNucu}zwXm_pjZZ(gm&lDBsm^C_Loc-v`o?Ei}PNV-=lChGi%!C1uP>4~mVpSEyV?lcXRnT7;L*$;s z!~oR`96lVOlyu8m&_R!v@vU1Wc@X_Wki~x|^gdDjeO3j492f#JjZc{qG44}V3hgIC1HfoTLihLmr za2N~h+xPFXf>HgQH@XK#bGv^^bkEZQSYVw?vM1}OTe=%qQBpsZ_Kq`N>f7~H{CaTr zQcd>@GC2}3Wt0pogP|mO4Dd_-`LrL!#N-$Y_77{@9|F8pRLG&))Jjg#U{I4TJ1TDs>2^!|T2NVgCU5`>7uM z0j3ZeG09F7p*Ijvmb^-GcI9fMSe#UmI}a)c%6R1X@d~uY3&zH)pM2=Fv>Ru7*iSh6 z0y{~v4-P~bb2HO`f#5lNN+~&(zKo^Q_eDgYSRR)u*4hf9@fk&L%A|=4D+MURb89^fSB?2-0_#SQ;W-U%U%l$3%0Jx=NX?oc>+4rHZgzL zYchnxEj^4{65O4Op>;wJLk*fIDw)f)(ws}e#xTikzjp0f{?HeV*3kP7DAyE!AO)YEl< z;IY2(LP!1>fcIPgsC1vbSkAj3nZSbk=*{Dy zVPR8E26mGK?i!4~8dsMpcYt9FM5xr%2g=+xSLO$?7QjtuIi!>UeX6Ug1q1}{SD8JI zkFOk`&2}18g_H%Rch%A|7~t<{t)x%$@baguV3|Po2;vJ42?z)sJ?fL1%7o&+5N z90r5^;|CW2AF+Wg$-z}-ruYEwMZ48Q{r&yKUvAwc2pmf=kK*FE9XzT+Qc|h{Oo{sG ze?DwIJn?CN*thofv_^3(pT!^E1$pCgiRYV|ngDPlmnOxxmaM>HUXJiaWLO~_AUV`GQhl7YIO5Ce5VfC*DQSQ~%U z&5Z@Du6m%J*61)Jb=#%&*se(B zuzLUg&e@`c1vkhD=+K3KKJycOU`=+>NbzkLf{H=#6KDz9+1V)0U}AvzDDR29sFmz7 zHU8b>l>?6F`68jZf?UZwr@wuQBDo>I(N33Ya`yw`C zx;6DYx#2l4HEJNl?T;Tn=17`NyPe$Zvkt6e`n6^XW1R)#To3)UK4%d2)R!+bnR+w< zWzTD}nW#4%%0sOSPOxbHg4W&dpp;b4R9N7xlcELP66~91v(NV{Ono8^&~(RUW!3{t7~|MOw_;gXHjy5mj#+RkruV@jz_+(y3pmi!Ef} z^2I#+?`A)Gj-ju5Hv7si!u+tj`c;Q7GTrtbeIG41Cx7`Nb#3i-R(@Z44mkjDmHHuP z=m1YulfPI|bJ{8qf2F~=33^k;#R3q!?%zKFVob996arPi+a4+%Ujy1l=e}gOfch`^}q)6hjCIz4l^4;M9+m0^NOy zk9^TQe1s(wh2W@=(5XzlrSfJi?VUwNVO3^*L&@=vp6%>o=OhLJd=)4fp!rR!b^M$g z^ji<9TBvM)wbySH56w7m_oG%6tzXnb~U1h#?o+X^sM>xe24A7aXgH*K{BWc%~wXl>8b ze9r$(DaIdyGtkdn`Tqa>`2C%x5kUGbGiij7({XnR$}Uhi1Nh5R$(s-0J^g3uPhQ{@ zNih`M5mtG^bK%$;IC_+B`@rSb;?j~N3>?%nE{0M|vZ+wV%IZ(_R@4j>ew3d0Dl1!T z=1b=0{y@w?>L*X0^p$zuF3gZwT>KT<9{e-d1%`sn$tx_!Q&SzsI!<8jQD!8kziY;~ z&Ki11{)xq7r8e^U$=n_oxKcm#S4T8pjfk2+KeRZO8hZ}12I0lAhYju@ymu~%0YB-O zf49@NWZ=Y4!t?m~6Jk+(+oX4MwK>t0ZmCc<13lwG!2sS^HD|Qzr|?HAIrM(mgi>mr zJ%2g(c&w_X029Pu4*n(nNNVT`AZQO1x$r?*Hx_BRuzNcGVQ<;<57TXl`U@@b-njSb z;!`n)J9t?Fq7xJgr5CTmF;B*4fTjZ$U?t>s_(w+WBot=u2oyep;;%IVhYk@~{Ktk(8dVQhWUPaSy-oJT-x+39vD7TJ2!8vg2f*93ER;dneG3N62o-P-!3)?VUGL*%LBQd%V%KFA7Ak{_#HtctuBo{>IyUwS)&zPvz`Kfy z%F@VHX*>))+@S2Mbka+`ba1&qA12Uw+W}*|%a~G?xyztn@TflB8&O5jHne#F-!+dg z-_g%8{|i)H)#M$?4O^a(p(JKP&J&L2Vx!G=pS8ue?BBm1c8k!76KBz1C2owp$j$}{ zAUvAOmoKB#Z2@vX&j)UVtlmBf*Vu9Tw>E*icBo0-HX>bx(sQHhCS`oB0x+uk?AQP% zNpf{*itv>rXId_M^s{!qAKpQdN;&|&CptcUaI%sEkXJKBOUz}28H)1GdT(w|Q$G){9k(#Q-7unuqwV21m#1vRzO3_Bw1JeMo1OjVJjr_iF6AO#L zp`mNokSJt?4Dnr4Exg9OZpgV!sGofDl_eP}+rFskUT5(O7cX9MaM+&PUW2BRl9tvG zXTTSUVh8wweqAw#zKzHj zV5?(ygH&9yu?YpPWt8KDIiId0x|FZ;T)hX}7HAT3J(N9Su8tlp(=A!hPQUdjz_S)> z3CtdU4W1<<$<7n2is_pR?Vj5PoDr$CiPT&TnK(671G)$7>+zylb|4zL^Kaen*Akrr zTz7j&1@sN5lo*L~=gvJn;j|e>gaFh2h$^s+x10m0wF$hy*Vv@_aLj@eo;43+Oho_M zPrDV(=#lk_MT;>9DgbINGOGRbi3%TtJqZ_pOhM}>qpYTOR7#4dY_YMiaM&OhLB9Zy zK)e7l^=05f;b!*`vpl^(1NCkJhU%w`NO-S_w z0MJOW?6tU{4!$j@*XTdm&*TBWPaN$BJ2)pss+;&l#DISI`+&nzH+1Wo1PlE{R85 ziG!^JZ4J67Y+FciFnz&~(Pf~0p)H`=J}WAQJ@73?ixF!fq50Wm>BLbvKHxEQJ7XXkm$3u8llsjWRKAyHjk?%VVcA^aT%1g2#=!en)o6|7+_OpZvW7gqHta1BCI?-#Ld5T3Zkzcim-%Y8^3$^4%-1z6(kMxg|^{~iybCHpCJ3SK)i!wj$u=c zlRPWLEDOXNk_sIiV>G;+Zz$+8+E(EW{c-!P@bK-Z$;@nQgsARXWQ0*!oE_(3rqSSw zbo~*oAX2zOyqV_WN73SS9zt#E1-oF=V_9H*6xY?=QV&0XG+hl217Q zqCZ?Xu^z^h`IRfs>r{XYqpCCW@)Yp}L@R+nLx^TraS(g)j#!uVP;)&!i*oiVB{y8k zZY44SSZb8)uD4K4CK}>b9`2EFK_KQPG7p#$?0(?4`I+33P%y)y*{;j@CUl?6Ld+J{ z)?5HUL_t$Z!bb62oSB^K51aZbF;2u&;OQe6l-9U#Z7I8mvniq&lzcnKK)JS_njhi6raGk(wJiI+XD$sJ}31-*_&%pf+K9{g|3(^*`S~ zDF$G%RI)m|>bf*p2~`;HDIEP}+msVoGbm%5F+gwgFh0djZ(5%(Qlv1h&Z3;yJk5yq z5bhb4gcyVae4xjCwQ-Z%XFr)<|3&q!@m2wvR%goD!vg*(ixsAmXT~q5FHe--NkHMM zvmo}AXtzg%?7q@6I_cNjPHbrFw&K{(X8#u53-%0!A^Ie&;`MUpq1M1`fV$K|w%Rpu zI_$f(NJQ-*79g}m2=|cFWcO`fSX?~ewIl|W+haP-xMFkTUxj8W_D=W#r-_EqLccHtY zA5Ooi=^a%25tjm{EbMoT0pQL}JUqg905uRBfB}*8(&i7idD5r!6n6v>r|Pc^J1Qy~ z&}#iMEp2N=)r;vlH~{GBAr*JvXs4uXfn2fea9XMo3{jUFpeqKT4q&Jn(e59{tHa(Z zTnv|7T^~b`fnEmZl=TjTXmoef)1Zc-7lwi;`60;ImKm4`PDRU;!!V)gn(%E-uqVggJD z9ELiEcSrodve$r@QC)l5pZX$X*kq_&@Z93uejmiZ!hnmF zxB)x%@slU@Znc<5tT43l(9lqr9>OP1Q~@Lb`ymX`>r#LnqpwzADCh(pf${@&#vsuA z7z_5_TQa#0O+b?UDoXj}qyx$zj5b0rKxc(KgA66q7>m&iEk1~401ogkfB=A=@MCSW zSu=K40F_W#?kjSEfxugig^);qtHCLR`$m^hU4WJGB=9z{R5LTT13C?CT^^Hntn_Vp zpIGv0|T4Zxr$Qk&Pxl&(OGfu?|mZ*&WgPdv}MB`=5X0ajmPq7Np^hMigJ zm-Z$fymna5?>T*)Gy*WA``@B-druZ|m5kfwlnjr*9&74F`p@)DN}ir4p#i`!P1n_p zDRV|*#KnbyW-~T6B3yV3?uac!-McZ?l6fYE$HvyH<53Cq>Jfq2V%B3Sw@%GFhsR+J;(es*{If>WT3ftn#RN8f?Pz^ z-|(yBM%@yres3!KTPNoAUwvQSSC%n#KCIjN_7fDWq@+zH-K)>jDr`jBx2?M|dcDSc zYTNkyg0W17=lEuyUjK9rGU-UPvVQp8Ct(Z4jsD>z&vgC!#^*@Qh)-vmR z&OO^9PB5X|ZKZyAnkbI7iF0%CRdbrBB037%y4u?A#I~Q+)=n49OyFe=K1ns1_Fo7A zAB!-Sy5EQ?baWuXf_JZCQwYI60!gSxGm2cfX`G2vP2hu#W2nWrN6 z_iWz08K}Q>c)Cm7)B%hQ@S&{tJu*l_%s@=a+=+X(54sef(6G9tHCk$?z0hhOgZS|_ z;ub0}Ha0X=kS>67NOf=mhLH24r$Sl=00Mh+bb8dKK<90VOW++a5>#Lmi_ts@)OEhd zeJr7{xq-j{(a|d1TG6-BRM=<(astHI!y>B8uwC1X^jg`S#vzKCMp#M*4*vXU`^tk$ z>RBou`X91!N6>7z7@lWlo^L<=q{e*r?%mJz1Q@9OENpFw3MMDz@9+Ogi;oD5(QKYX z97Sm}0K}j9LYJksjt(|(9BYwt$5`afGom2C=K!q_K%Ix94-*Xh9s>QPOSh5qz`Mb# z`-p9GvKu`u-F5=C3>gXtuQqObSwd~al9Ce>H-ORP8**d9BqhH9X~5|#n1NJ*X@L;d z*4BpSYNECZbdOqV)4YsClj2zE<#2_&4Y=Czo4eD&lBWnzB3v~Y5mfLpc!;&gaY68b zNdjsOtbj)0HF(1-fWX}lo`G(`QHU79slD&*mYeKs#MH<-hfV{k+@ckHjLO1%al%6*4w5;${e%$c|rD-a0_22|ps#ER#8d z)?PTbCtI)4O;nR{Q$gp~{b7=KQ2a78!xh+XLI}KtKV)Y|T#fg9gy9D5hT4E^Au*5u zIeYf(0sVw$i~+lP?OG5XG5zT!mC+x&(;7d+Mx9#?N*vJstxVgERIXH-~ zNz)~P785^(ih|+C&?$tmABa}2|Mu;a7~4<8K_Nr*IOcPijXMD2LUm~k(@6`|mQa57 z{P}IrH^_0QaGOgn>|yHnH5l9c0Ng6YlyxHd?D*UBs^ts~DUDT}09V2n^939Z|8U zYGUQcClY?xyLaHk0G3!ayE!Bhdb)^+(s@7X?+``9K`V=GqtoU ztVQhR@?0iPjhalh-QEnO4ARs)?RL$=B5$Z8sm8p4`cIV7fkR7tk-<8b-_1OUR+eAP z)8fm7{64A$)e&j4?l*Vxa-OB?31E9ePy0j(TRZC-kHaP3B-z4a_FJuY#kDVO-=p{LtG3&Xw3vC zM(sscLYoIeLrw?4`QpWkuodvn(D{(DJ!eoTA}$_`DZ7`lb>n(`r4OLYsOK?JQLC~- za`RU+VJsROi`Ky@pcH}y0Bi!9LsWuYgIM&pwl<|qJucXb;^L{XtNQI9_bXI&7Oa_D zZ3%I0(aFsSf|&{D_vX*;hwu+}3^0VMkB0&i3@qMqob<5{^Bu~qFO<;ApO5RK0 z75!p>69Pb|^1O`o!M8^`3!?C3m|54gy$+B@kA%Lcm#5%-I2P1f!2&1)R^9$;mi( z(3GJqA)V7ywsV#})-8MICYeB$G%dcur9aj1TVXW!25_uIQf`5sLqu;vO6In%AVG`d z$AOcqoMY>V4!VOcsv3aSab@WyAqrQ7>>|i9be2Kla7LX1$8u!zt8Gy$sgZAydk9%8 z_31iDQA5Z-&+lvxbAeda(b0q%0__zL8T4Wn>YlCg-J^&E35`SSz_ZjC?TWPuV~s4y7%rV%)&q>(~Y#|?Cnr_|Kw2+bct zdkgb4_%0#2(=!g&9y^cRu0cJ_6xS#>Z0z!R|m5NAVkf=mdQo0Oj)4TThfJG9BL z%KfDW%RLF7)c{NMErVNd0&FSQKUF*@?0YC+G^AC_KQqkpaidtF`QRN_)12;dGlacZ0viEt$Z@2%tLC z2xy()V{jj!kKR6Ik`}||n|B6|88PDa9F{m%=kGs@)*Qmq-w|BbT(8=qNkM){P! z&wOnU>B}Xb+Iy?Uy#1d+tZ^4hkuOpSpez69)7AtRsBQ4F(3`=r|Nj;wL=PG|I;8gg=dWKh3?Xm| zaTTNZ@2!toU*F)H*Z$9qoOFE(E(S6XsWh;|7_K{V9IzVzfG&^)%1;uhR$^u$-Vpv3 zjw*5ZSBijl(^C=gh0>O(tatPlLhz=YcVCMyfa*^=}Gy!xzOHe7=?jsOs{ zJF@sV_glpBf5A(Q!bF@BgJx9M)wPoHUC0Q*!_SfYPo+E;G`EZvoE|N%x+dRev^G$? zX*Q!kqo0EX;k2VxMq2YQ&NIYc=s~%IVl?0w4N%BxBnpgKI|yV8QSR?TSc$45Y#nfS zRAQtw@z98)L%D}20Y`i{GqcI;SSK36lGiG!@MUT$Ynv+u7R(T0mLk`m$?iGNS2?5B z2-6@_>*O@0BRW*fnHJ*L!Uc^DM`mO=pQ;WfyL4KTQ~gG+Zhs5xNceidus8@J@`XQH z;~;v>i5p{VkicW&oQxp7LW>7%gt9`w`Y&IAdZBv}zXekf))_2`!Cf4*AYwL=N|4|1+3;zH z>JJDS7k?@Bp&S|>E*o&{{HqV^1;<_nB6X2JMFb47X5#Q=RP}jX-PxeAvgK3S+8A8e zaC+%Y>9$`z2ga6$_LehYrC_wZAa&L^H&?=A%6)e>SN1$3%m{>Ya=X0{751qfKy2sh z(-UW)#iXR1jS;;F2?`ipw9C~82gdf&Kg-RHf*1?#PuLlRdTpB<1DzC43t}#J9<&DO zZb)!qaIhhKK~@JO12BIHV-D>Hvkk-Hl849BZg@(kaBx+TOAOQEJ$u7Dpse>h`*0uU?$*&uqy zDA$W59Ws-|!PCXjT%>aMBM*;U5+o?nwwkom&)^R@_1IxRoeY33v2VuuEp+=MWwIz%X*7L z800ixa9KHw=$&P&ywJ8_e$8j^wkPZ7ouyxlMltQ;)qf^eXTwi7%sI!74i;XWoR6Qq89%rRHi^SB z2}wx|kOzW>VIy9%v?O9qV2@~$uuw4R7*4=VBEMWcfcUD6@6@~``WmdigoYv5W@i%X zh(#G2JAfm1C{9?A_{C;p?~8%IA?V^%LR*_YHangZe3dwm2j+-B96Nq|U2{{8+EXzc zq`}#1oQ8l)2;~&QU#ST_>{t*ad=VfUnrGecG*+}@{!_)AhT-W#pbAuIG*F0hI9(My zsivw5MyZyhfdylaxDQ|#cnSJ?R_n-~gQ5{0_CKDymd#RmNXBuOmD^1_IJD44f#-0x zdQEh_3t8&p$IS?P;6Uo-%lj?KPhkDQaD`w_tW|6wfYD^tSO%HnM~<8ZC_w24chX3C zNZ$|d8Sv(xhlq#029$ujC5UEA3p2>LpWi0Dls+E#9J)Dr4Vg>^UTtV?4Z_YtQim_{ zo5e948i+v@2>cx(Kp<8kiH-xMI5LKP0>X8*Hg0F7F)yQ@fdSb2It0xT5Ww(S9Q%yN zBAK136c7>1nz zCsSQJ$ruL&h?fRa0{X|sKA+b3P$@~m^y$9&o=?%lRN&hz$t*=itLgz==%lD(ScEm^ z{W$A|rM@1q6+!$6OX%iwood8zAj3a-!a``W6qt9+%1?y1;lIJf{WvJSdegxO5t?Y_ z&3TW))9Wmd8bw+cr|w#E?KsfSAt{uYfOo=4L^la;v>X7aW$Zr|t&G$!4EGP9&0?+- zdy%hzjH9W!10@uZ%S_->R2fW&)8JPka3LQ16H)q|&xdOc%OQzY(jC@mOHgXm-tpzG zaRgl8_gulTAENR><0eqF+h1P91Re7Q%%?t|(2)6trG+gK=i0YL3atrr32-)1i|+{H z>@eNzRm8miCUQ6ch6CTBzHu@H;nW?n&@xnwmdD}AVYMMuI`6@ru5%6{hFH}^jTB~7 zRA?NawKx)hpS8BO2FWAdh?16#DDGJ2Mjn%LU(Ba2mgI7k$OONZb!6oYZRhyu^5ydr zE`$*&i32RCvSrpHQ=xVqK4>lgx%%l$AVGj!v8s0SHd67)FJJoGi_z@;Pyzc=otMi< z=AqejejrjznlP#6wK;leu&*FIyueP@2D47Kte# zW=OFp0+X$^UXPr`56NUejlPG0DPCwK5oR+Io5YFoakfa5b?$CK<~^bcc-exW3U9B=cG3e!)AB4<-&rt@J1 z->Z>=102{#gbF7Ta{$m63Mc{^08L;(nZ;-wxTrujx#5U}1V2Ma#^Dm-Xk|1L>=H<{ zXeMv<524_|HpkPhIdy?LX>xF(GxdD+boq8-EkV#Lv@o>mamaQ@A?llUBZA#X4FE0z z1PL^C9GIm&n8i?K^GqWJV7gt{!u6ZFEN;mlP6FWQ0I7K^=XrA*nCuW2ZEOx9lfkjn zL)mu#rxqb~DI>9s18GahItS4j0w#IMfBgYw{4fq2oHfVd$25ReL?IW$TfeZUs1X?Ifd)L6kdGB=3HU`BC<6DEgsHIl3}n_&|U08e+?A$HlOna|E3B zhz{s3rti7BqL@~zH4cu5fx#oK0v!lrO>~6T*N%JC5Z4flDziA|hyyQ~F04>YJr~|S z-Z|D)WKie|IzmYEAhG7zNLK+n?0apPqJ@HNaf}7T0xJ+58^JLcxk%Ben+ex``GTE? zIAi7TG;Rps-^i3vL<3DW`0g+S6e#BtU%swg>43LGzlW%5L`u%dc?1Rw1_;E668@~w z5}YT`Fr&cD?yQ&J(<&vdcG>6)jZIBb-$#EL3P$Pe-w3|aSBoczsN$mG{cd?2N&ytx z%V8(Er9@dmO_ytX?cqV-!@<}8<)VNwaA!tK4uh1o7j`>3sq^*gy;JkHLkU;N`h;@c zKZzqF6*CWaX91~MizGH$qS4{kI6vUik?F1oJe_|JH;tgnhy2+~)(>4H_bBz)gFBD9 zU)c=Flt^=-&Bdzn+|y0hAx_~bYqo~r;5*D_vorkdS$YCQWMk5`oS9_ouWoeb;VxVn z0PA5#??BC2z0bV2^A@lO zR2m}1Lx`Wk&&Ki{3o)$gJAuubW}idJ7Me{%9vJJ;}RHrL{5mg9InuW#FZF?Sa5m~YYKHa z_lhE-;>2N~Xk}dU17EvCcm=TvoKwM7H7GJ6N0iuA*WTpzA0co0t&2O$7I22;TUVEt z!9EGxF|ZcLf=LXCoiby$LPy3YXt~hQWHmJ-=)c1tIv?fEBP$ZEOq}^ISs5W!xOX%`xkvF8<#5d3T|xHFTav~ z0VEfj7y)FQo43tPth2zG672e{*4Hh%@b#ENi2HOQ1XtEH=K#4>)C?sh49&FkD{7K8SIsTu?dOXGXTi`r?U*jADuwHGau!7q#EN zyv!qvrKOb>RxjGFFfwh(#sC|_eYcr3>0)z(W@WmBY5zN@?Gaeki+gAMkrDalTDcxX zbnnv2^F~V;9}q|YJ2Ul^^$x>4!vVDN>stfLfL3v^4*p%J{22r%kq}W$MksX0=rsBy zDk+M`YyO6zV?aN80{%D!*K3b=05JmMq8h(EU|2meKDxj5F#s+~3>07BR_7|w<`E@$ zm68JgSnf3k1SPosNFDB?>-#5MfcgP46N7>GL<1neC(gWpec(9lDshT9OnOC(h`zyo zVf5c=e@NgYyjLJB=bKv2vGv=uB+Bw2~*aKZ%-oZ-tF2d}}G0kMGJ5&E~T zZWMhhuCu}%LlVSaQe2_|a785Bj%PSi7k6F^-DtcHtBbP|MTNF=qtiRk)Q5RF6^gsOg!<_IA}z{x|%8)^k6 z+;-yOqa`Y%q^4uAdPC9;2b zKde;|5fQPQf4GQ4Ys6hqzv>b_`gX*()Mu!qHNtB44Hz@9EAj(W-Bw}j+Peu`02t;2 z(;L7<;!qaE3ScwrQ_MK-%Yl)LyhpG43+q|j0|g}p#Q@ZxA0PXw(Gr%n>r{;>qzwY- z6T+dL_hLJQ7>EWKBSL2$ByLbbr2X@w7b)N#aVOn<3y^MP2DL+=hPaRg=fmL}%a}Dc zHObPo6M~P&VtY5V5)~?gV=NW8q|EQI2dE{FN1A9r5P}qtNA(2cAxZ&^d9TiT6EQn1 zm;i8Nn3CRv#ggyD1BnPveko*bm-pl@!eeP^`GkRjzn?I_Fk3u(5=a7E8^Kdh>6?SG zH$WEqa9>!0Tn`A+Zls6^@P+^o0^{#Nb+Jo`lXsF+Uyj1|08#^q=2-rvfGPu#vs}be ziUXq-zx0C#s1^egSlK`4QS9!jD=n@V9;s*>L3v>48cvqMa&;MLDV>-QLaGV^5VG*t zx8UUq3-9!#bqa5UR3yAv3Xx9)SHvM;7(~QXB<=0zV5fn1?7O@>!%q%r7(6?!J7K^v zs*aAQ7TcN;kHq}~3_(3E1rTO1qL99ZP`2?($f?tTK_+QzZ!bo!7jPHjAp~kC9&3)h z4d4lA2G)*4oV0tyWi$1DfV}2gbYy&?iln)OwvOw^MqQi$@d0pgCrG3caoi6r4JeP| z$dq!D2DTuE(|&Y7?iwvgV0*{!C;TqB+iH5Bf!?*_xzJajx?oe{;4nspklpkI9@GvZ z?)ly{^BWXMaH9&byYh)bQh4O&Sc-%s^ybZ8nOfj?vc3wgHDL)2j=EK2Lj|Q3X!q$= z;8?_RAb#wlrvef!FM=iqqJTtGP~3Y2?BQzVi+IF*`fL|u7Q_1}7_v=5$?s*@di2zO zKz>B@2?sZEtN^$b;u0Wjq|zmk7y`1bCH)cV!M?_eqFN)U0DH_fw|~-=uv-!n&-hY_ zlH~ZT(xLAGg94iQohaVd1IQFp!!4VdlcO_2LP!CtHs?Z$7UU7gwRLT6d@DG24`~}k z9;X68o^V+YkQjacyW;CH!soym!9I_m3?lNt%F6ov`*#>bJPeO}o_yl_xPS{SeD6fQ z5J+<7iKw!^;M4i^?@O~yyQnuoHUcLG^@fRvLl_ACqEO>jlD-A(#mq;x!qJ4q4I3T~ zalcDJ1#_gJA~qo6NQm=8;Q|JN*QCgj-9CyAgt{dd6{07=tv}J&*hr`+h$TiV2jBPR zi)4tsjS5iy1dtnC4m%P2*WgDd_dmFXtS)*c5;piAoB$bHJo(}fuS#9ys*Y?%TG~10 zPJXM;_+H|U2_KH7TTw=CK`4a9E7PP9skE43y`2v)#G1;GoXY@MQGknQi^d^j^So=~ z9UWwR4$)fX*+XGi269Ont@}(rD{Bs!A$(jeaDMxw!GS(F)_ArS?0w?KH%PeHIDc@E z1e25NoR9e5zHmkSfB5qXX^qHboyf^o%qGv<&PWh?)S*V+NHleo`q_ntjP`;Wikc!a%#21HzyhOh#W{WK~Jgdmoa!vrEghx;V>pUxmoZ)m5Fb=gDN)O5ygfU2+ z2-_4y3I6Oo+46%jBQ;EDjP!B#@ehN_(K z!n`@-BB*nZ{-`x%P^2i3X#Do=K7A=>9psa@8SClR;|kP590#hiBtu&Hm#_F|Umza| zGbmW<71Cm8Bf&3TLnO5pX@6us0rUp}Y-Xg@2WP^~gfqQYy|CZ0G*KzQu3$=0W*oRV zibL#Bmtn5pIxf?89JjuOd&JIfhTi^4Ck-&5(1`T-0^EF9ctG0w3dPY)piqLg?2FJn zShSmkh2e6RemX&>AIcuw-HPZ0x~6|X0Ceb`gYAP;nQ#FBYJsFrz1^v$Pl+2w@dijW z;0}`KI%x=Gnf}x_FyQ|h@t`N^bvXa|Ee9UHHjCn4jIyTtC;OQ7NKyIEC~dop7#K56 zF!xW8PiP7{X#=>zQ4C@%H&n>8EPxZ)tn-l3v7?~!eD(npM4*Ad7FftoYA}z<@m^2RFfSd<1O2G$2MKasT4PdvHq#mdo?@8qUkWvdDWsuHX-Cq1?Rrjh?`VIhmmX+^I;M-9Xqo zferi~DGb`gJvWLgl*wF8CJfxf9qNiuusakG~bKq8(4ZyHd>$?y%93;{SQ zMjA|nCjK_4IpGWwC;_G)8w5Xx_b3aX>vznT6>&mGz_AmY4iXj@=aV%as!NkSh!P$^ zyJvkydOJVo0Eihh@yzzo|7gr`rGAB2W$NK@FgPjX76Qaey4ttDWPM=u$qUk(2)u1z z{6EX%-tok^gn5o!zF}5SlCq2qp%A^e$|4vpS^nvUR*>u;F9HPR1escN<1HT;p$-2q z|8)EUdNVw4Oa(F>-g~n`4lq>tgn&?+#aNK_!GnQmXWe=6!|N2F4ri$_?pQ#_@o+uB zaTn*=A;BX_h_wyF_YuQWi3-3BSQ-yUI`|*@ZvprQGAnSjeh{!V|G#nqQtsyF^4~R- zq?n;Z!WhJuHL_Y$k8{EQK?X1@Gt)sg$D1D-74EFM;^N|BAmR)DDf4{qjv@Bb|0*#g z2V9c_*A_p8Z?!LfjnXNo& zba(pVU+bM!V&8)!Y5Q8agF{UOquQH%{)+mcTmir%RzyV5A-JVznf^;=!(cp2^G7tq z8mIK(M!<}u_PZlgh@>D$-kZEU-Y2xsfwpY@1So`ypU~4FAmV-}%pOo0YB*H;(jjMR zpWvEB%@I1}x&Uk7MY*}Tp$LOv5*|!G4q)sMR0beI5eDW(gTuB!{aojJO#vPP9tAmW zR3zXlkbVG9p+aP-(5hE{+O7Dbu%G}F89sohtkVno?-T|@pncQ&{-R?jJ`qHa*p@Lh z>v8ADS0G@lPat_vT1ILdf<>qdHKIq~*#ra$Fc~M$@FdW#VWT412Wbtj0jvz3E*|?5 zokEN6z=v9;t9z4P^Rdwa$*89_LY99L6GLo3yh)yUc|0| z!vz8OdhuAGX5z-a%Kl!M)wt0iTGo~CYw;U&DkIuNfFIis79Ai6E^|9F#^y+?qoX) zE=0n^5*%nXZadLi@xnkv`@#b)YI8C(aeXD!JVKO3@CZKCjz@Qb=_~skpmreg z54#gLZxID`Us~Wa$lE#WBammzHz*DECLr#11Qxg>0B@j5BIdbMs&q}Jx%u6a47lVV ziIq*<1PPSbK8oKXRxKPyaC|HyOc(eij(|~=SlZ%dtAVzW{l05rm->f?G48=D+_?Gz z^^r()5!ZzgW+5KF%VSD~2t$KY5VzTi;ba%BCw_2&7BdYZ4Y+QHxQZ!*IiUXFDx+J;Aj7gJD)tpoZ`!R?I$X|m;OJh-UP1abzT3@Oooz>VMPf^rda06P*H{yq9Ri= zRR|SLNT!e|$&@Lfi9*t3C>m5tR+3Um38g`l=KuStwa@GKKj-Y%*?Vp3`~7^L=N_)> zy07~c1BNsaPiZju2^(qRndiZp7zR@qLo&$fs`;k_t!jKMqbKwSjX!4@h~SN|Nn zb3wpcKs5TA&Cfs}wH&4v=QF0Y;O%cm4gIk`eLn6*BqJ&p7|=) zyvR!+V87Y&W#0r2_I33F3kpDDGleM-4maic3=ebDLU)Z=iPkad5l>nmj_JOkB!tVZooc@hBSKn-Yq%;!WA?h-%sAzO28h)=fPdctsDgII`DXhC~Tj1uwq2?~5Q;Vj! z2mM@6QwdZ%JJ^VlgO5ZTMaIOXmahwX2%=Y(1wZ4Spyhj-3gu3~6~&9G2)t39z*OPo zgD|DBxuQjb7T@$#Iz7SUi`t2EdrovMTchC_CP?9UefTeT=Uy*6R$YFBM!V3U)JUBq{gBOKp;fCC3&rHX>JmT zhNu=_N+N5ekgPgLL9BJ-5BU{U1oNm`H{Sd=`p-!hvKs>Mf(6|{M6V$uM?J$m5OZa1 z1Jwf78kjUX^(A)S1S3Jvlzj$F<@QiWQAuzxU`+H~>fg?+itj^=LRL+^Oeig|i4;x* zLNb6y8Iw|gV}}adGUmtdA93`;wpFpd2Je@Q{Cp?`VP1_UysN`v7`|=WFq3ejPv^yS zow04wz(}2j)0R3(ga4-m7`91NEc6ag!U%6suM%a%$UF)pngR6Z!orla-v&RHbQn3G z@;bTr=Um?<{zicezz75rMv6Pj!~!Z+v+)2q2U0EO2E6@PX^J=mVU`g0GA8)%BWyUy zg_QVtGA2f0-Zb;V>{w||0t!3SN0SDg+^kwr+W;I;m@bKm8hnPlN)<$J3hV>RA@S?i zFS`GEa4+lcuN^PPKjMm!-NZO9bI{kbBWX7aHUJ6(VsLJ*U#mySg@q*dfinkQ{P^9U zytU&Y-DxjxtQwmV|72cAQ9MuPuOriibm0S1CX2x^p|5T8wI{Do#3Gxb90Gvc{h~wj zK5iUCBZ@cw+(Q#XO`(`mV*$Kn+WgLPa{kW<>^sg)bU@@$Sy_25E-Fvoq?=~GWi${3 zUqT4E4I%>hE_Nw<>{=(|@_yPSUkE&yQYH4QRKp1Hl+`ZHphnlJ9y{v2BLDZ=QHa*?oK=m1hB{)S@_q~}Aaox{V8lBRm+ zuI8k|%~BUb_4F()l}WlhOGp$h!r^m|I48oVngfo%MG}a=kh|Pj2C+n7YBDKUCUZAr zO{iC{x>t`4$BL{KDxg{?8#r$LXgx*5V&?e(o52}oY4~AtVPNi~a?XW9cS`rxH)OD| zXGnVU1FCT>*YP#2@=bR`yvD(Lb9WtT6%un|0Y&YiQXSFMVN9z^(Cds;~*sl##7xfZZUxs1)V`TVSlGtw+W z+`Z>3ohkKfeGhFc2*aJ`*4E|DQu7o&XFhGM9loZ>Q{2QGHtJjMPz`4m!hr{XUS>b6=Bs&Yo8H3WRza z58u~Pj2xb&RkNlAbKE{t_voc{R+5s+{Ql6m4C|&!&E_tX{{DFOBZIPZAHqEaC~WPH z9hFzgR`81)SM|w0*>gwarwv0~@p;#f(p9Y6A6?b?(zFi8;?+++RX(Lpr9>qE7~Z6( zbmoZ1Hj>-HOi8nGj?LAzd*{m}by1QzFnYz+C65ClJL;XZL9Q`$O6of}p19{kyv3pZEPN^o{glD<#^ts1*w(!lL2 z?J`S$b6>0O=sQVCF-N8rMWo*Xbx?ytXu1dAgK0E<5XRgCXy z)wUCKW*!*A96(m4rBwTA?x|MDTuyq@FrnUJtPI$cl7U<@dO6;ECfZ)RO3tS6cC7Rz!Y&thiu69JbnHENHcE(Lt&(}!ibvQCkKil(i; zX_SrpY}zM+s*nSkm=p>ah>At7$Ki|3&Om__Qy@wpD{9Y*6h2Xm0C@o%bm^6@v6;q` z8LmHZD6mCwk@F+AKt6bx3Xt41aRMt+R_}2{#~=nX#NcA9ph}F(Bf-P+!20%t9&U5c z5FOX-;yRnO%Gb)daBQX?*P3>R=3mN9zZ`>vHmWbL;#%%xlzNm5PoGK<*!UoLg8gv} z|NJ)COnNv%s`8%;85Bdhmq6%)paUhP7!t0;@Huqry{%Ks)ex#9bA?R??IA(aIKji9 z2xWel;S`GpddGCGEty3<%k~es9-O#!c~{a)02sL$d@_1JbVZ<7Xhe(xBMvn}BS307 zYK)s6Or1uFvZEb_d_38L&V)C<6O6BqiYeh!v&0w_v>EkZ)-Y0MGC^R_p;~Vu4BCk*MaP zl#*&#Ii8}heg?(S4mSGydmoo>{ox&`m76IhgiM`Ty}CcIG{?$+N67{fwA9pe0kS%( z?WYeP-p9d!&IN1)02PtEFtND2*gZWbCx`VfAg5J$n{YyT`rHKxrrNb7z~eyy`AKv# z+|wJu!6!NL04u2PuCJ>FUgXF9H_k)>!Hee-EHgH#&bj`@jOaS1+(FT+p7aAsx z3$Tl}Si2VMQe8+&s>AiG(2DdovphSVY5_+f6p`j1%k=hfRil^ z0HpOP)Og26UXh-djA<9`tU^pZ|R=)(nZ4~)gT zM;$_cLri@MCk>XvRTf4$yyZVHG#lp%_aP0xlFEV?ga)w!1Nnkq{l5WBNYV+0VeEz^ zjwf@A#t0QHPzy1Jy9K~rS0LpwAFfzmqfUtvVb@?AHB$L6d*TDNsaKujN*YR91MuTM zca_e}JH*@4+iVn?MKA`&;@%G&IPlRScj{uX<_G~Q)%$&mK%+pRAm@v{&q7rL9RzmC5&gD%AYTZU1584K_kwH>kdc4HuR?LVcEbi&Ao6)S z=HC6EGM{>EeB%)|kzIeYnQJJTY@Y3?2Qo>+o~rjTh7bT#WYnR}2KA9+C|wD+5_7mA z4ewuZGx^h;lX<5Giv=wlNCByVhj0P09Y6rYQ(3=gBJ51-P6n>ISeZckN;%7*x|k9! zm@=qqN2*_dz6nCaObqWjijm^`r|=gLLF{u?B_>U~unr~^M!c`->#i*m($utOq zH1Zx@u=r!hpU#$@S;G5=lY^iS5|@^}i<1*|f(NM6Oz_l%c%sCRiCvO~U%wB+AUKxRoj1Noply z^mx^f^J0fcZtiP`ikKs8wSr%L{Alc@By$PDB$fU~QX_~=d~9D?PVSH;zniIbKI@zy zgAjn^o5RUZyyYLMUVm=@rQjz*aREicpS_PjE{P8gnrF%!)8}84Sw%CpYW47JrXm5w zv=!g|E_t|21y?39GR62l$TiuWP>KOTKM53QX#r>CmSaD}DX?0hyV&s1vH?D4y80_k z$Jz+^3AMDaD!@cjbdVSVk9ld1r`b=RH=Cms3`*CrT!?JJd`%&ElyHQN9hMV=N>41Z zQI>gUUfu3kQ9((F;2e4nh)jObVDZpR&t8>h$^wP^P-wlwmM7bJ1(~(NO$5R5)29P{ zU*j{ww;nvHI7`Ad>My@^yfjD39P4_Gsd7L)1x5yzD=k6>d%VTy<+Z`?YF9X%Jdm%y zt{)9SYVvKjb+=NLJ>JQYOCI^7p(SUAhE~7Djx6BHK4A&Uht7Vu%}34@h=2hf#caLa zalk{&%y4a~HK?R_ zA=HYEJw;iv&b&Bbu5O5-+Q}(16h<{T$R&GdC?}(N(9iGY0O zOJj!Tlb>E~VID9UI7+@@zc;N%f! zxjACOX!B-ieHvpt4GlPsuFFy&E?<(sVvs8K9h~xX|`scaw?6}|%%;Xvr z(EKi@NvJQU^wI*+dWVRWi;YGH6cr!~X$!zTx>QQ{=lYKr>;tayTx=X%|0 zzx0tZ{=%N)(}mk|CtnVF&S8iVMc+UF?AN)$wuEjhTEHiV8Ed59)W7U25&9PLBEV-m zbh+`?5xb}>dmFC!+B%mBe~p6vg>{6x)f67%j;UJqd5r!uFf-yTftn5TB_C(*TJroNXlB`p38qWpaTM&< zdwI<^@my{KEHyC=Sc{S^E}&&dj4YTqFaBnPxvhfnd6^-a#R`qr^s-r z^KmHv))~^%{8Fx zv&Fb}QyjjID%?5IC!n_B$M09yyOuVb{MM9~k{V_ZDmBB*rQFLY?|a0QU0X4HWz)`i z=m9lf7zZ{}?0@z7Sw(GrWya6()6|`ocvafz>!&aB=?vSgRq)0EjtFMv0$aaMU?@jb z47f#Xz@aTM!}(tXvrx8BlS4+09DPj7ri5NPdorUyG^Y704~SWh%Zo9|2U-(|(hm8{ zh1$o8*}?m2Qp&5 znt30lH7CZ=#2vnQXWjvTRu%8{jNsBVclj_@jfFJ9Mzn~W4aC!5f8AW=C&d6TQ!IG* zBL<0hJ$lKRYOnZsnm!!OKv8Iazhwm}b%fy>V-$EsitLKrAF$K*?45c)3$klIK8cOR z{Ht#-Jv*ox2DzVXiA?{U+daI)aQBtj5Mg45Ha52T>A9UGaai9fdq?-)2_NOR<2WlT z`El)-ZgI(Z&gWr2uJ24JEG$Ii-%~kjIwZz1Ei66|z0$d&MO!;wxnud_vQ5k=&sVZ} zTiP>jt1d%o@!M%jsoAc_I#jD)iyo{Jw+JJ`^z;^;TJKn~t$@UVwC??Fz4Bf;{jvqQ z@`b$uIG7e$pP1LW5>ptVvx(+=ma`pIFF!m>K96uqZv#n6>wvJCVn_5a{Ofy%8MKn7 zpTYUWl2S&Jn5Pqw)XM5dh~y6eJWDxOc0gRBtu^P{j~R?eUFbhENO|G}?P&Q)R9NKg z#^N>2!(HkFT$247ll)rOCeo0KP5||80wdK7{}0)!dECC4cA*1(ZvB{0NRMI$B^_&{ z@*xg|kuN(W>(cCE=W^RNI`8<=X@sn#$OpX6vq=K97HeZMr?pw2;A825Iu|^ATwJ4? z@VG+NaI-lxz?J<4iE8{$Bm&63>?Oeg_}H{!xGW$m zBJh1Ff_XYL;Nr!L+H%7#rds=Y15K|BqKy+q@<0JmArl+M&^6ZBQF$H10MWZI(-1OE)EDB=;xR+ z@b&ShSMaMKcOkc%>iB0om|>v_m#whs@s66Gt)oWdc>JKI8aF+W336~VeDeFM+gPmA zp$Vh0O6B&WLuc&`^qf_<5)@ICEGX>Jow|?clB|~a>(_(vCI&LOUX_LPgEVnbAIA<( zKz2xwKqTL~z^7ZfrT?)xt)ANFI5F!k;k{2aacI5EgL6f=@h#)n{$cIwN;OswcTfj&8&giKak>Gh{WZ1koQz?{35xF}BhhZhnx~g#9=0p_e zu)aro4mHRAx9A7^WVhkO!;Z_RU%sH==}gIZt|B0`uC6GA}j?mV&x&lXV2 zxN+fVsexLM%)=JXKj6iTL8~t66!Os3FJ<=+UWnV{y(dppChE_LXIV%%yZ>NOk#Bxx z@PSHlb=q#A{P*wQ&q8yYUIL;{_XjkI+Kji1o*w@LdTRji_qn;|yLK(D`J0f(Lt*Xj zgOC5R{4rzsX+3|sTrw6yjH0J8-~lk8fLOqapakIT2nvPG_``?8{h4+eL(}E@Orp_t zyvnK3^&+Ig)|22YI)J;~)Gk4L3tob3Z2XidBdtcjN20}HcpjHvv_cdaM64b!K_|g+ zw{*C`{)KEyEa9lt&4)}n>rz%|WCC0I7D4HlFYx@IIR+Ox8bq!cU4N+xw4-DnsP5Fq>s!Cy&8 z=(9PLCok#W(Epzn;ABss?`G|!e;L6V2i_BwlSj`(4bo&|!L}v_&PgUcz4a29v!`FT zw6aRN3T`c&)tGZ}`KjOBjW;a(3bDgLG{ejfy}6iW7bp#5yAUoEb|AmPRU|gnENvjK zoEt$r*oB)s-;&b2#ZIh`tor;4wOk|n5DO`8|9$sPZQ{ha1lfw(1#?}IHuBSuo{5== z9aB}8S_ALUE+xM7f}DcmAXxA`LCY90=3oJ1MS}1Saja}dxIue@AYS}2vBr?nfI|ZG z#c2UV#5~`wuc^1*aDGAS;QEWyi&KQO0EKxm3Pa9^*jhW{EZ2~kSq5U_D{8A#)-<0H zDUQFq_m{@P@>MtTM+3!02JK{5CVojF#wG|&X~P%zj7`d3iB2yj<}Zu`5y%QrK7gcn zm?z@#z~OO3|9)o*kj5=QDS%RHOZeB?1{TdbzknBE1<(~J8fvGImifyIRQ zcC2Z?ii`|NwipUxyIXe-7TzYHhHAcEch?@AucX*8;QkOt9F#R1!ZkPW|AohQaPXL3 zgQw1F6=YNy0L7_X@y?a^B7RG?UAt+nWh-8>HFj+-Hm`HU-96=~h`10f+%;rePkN1@JLsQSH z0lra{CU5A_;3a*0sJQjoSv@@UJezXyh568TCj zaKo33TmUIrUBQL$F-hWhlNrP|GqlM5r{ieAA3Zu(;anRfJTbDfld97-^;`D@i*MYz zs*MUYWKCQewJX(x*sF=W5=a=VN6`K>S|A5xleTdx7K_C`6N`(Gi;CF{uG~|r2`$`S zu{4$!%1*>}q*_P;?mqoD$(Q(#B`5>8APc~oLi+8Q(y@WVFLzuPSnAHg_)6fY@5l{xI+r-*vMZZ|6(+A zfQopwEj5t<&RtrZL};cY8r;!^=E!^EC-H0ngM^6}j$B|`CEPNqndXGJKmB3)f6{D- zg&?q6&8LVH^X*jEW~q_+a0o`y`=?s7Wml`@87%>IEUYQfg;5Maz9fn`iWUeWsGzRI zFaKfDjmmFz?}wTO_bJ;wUEuDhj|_q~Pa>6p#PDYRv-c)rfG#1p6LTxHmUM53u*e9I zaHE}c+(fI~HD@S6zO#cBCF?bIHWp}!$X6SI}`Q_w^ z;;xw%;q0m}J`wrECWEO|sZ5iN(2#sU z=v+0oU$|o1_#KrJ4UA`c$wrlm>RdHjP4XORZ7P%=?7mckjjclL#QTeDcN*TL@M+c0 zoZFJYMBlX~r>(-l2hp=K)4{CHV|6(Q6Sb$3i>&Wkj63Br?%0z1heuzz|Gl)2X$Rzr zBhKnh*XUNU0l-z=Jz?NZa8p=}ief|<C5h=f}A{-6d%L%Fn?Dk+LaX0k&xf>U;_1 z&?kIva6PmZ^suB*GG$mg&7VR)_oPoM<$E9Y-M~KuFe0EqML3&jNpx2GR!k59)W95-}(eww{XNT zx8Lms+$?|RX%w1$bhl!z^}RWjvFI;Sif{hWY0*I-#!19Z}}JLVa;(%loJhNzkY(#LDxV66uY78nvOS& zoj&}2=aJ1_UDnX!kM%k0w$N^Y|MBYn?%FtUmA!Ub7n3*;cL4mW(9gihhC2%j^LHZzxjcIMDuwQ&+ME5}1ZHgWkLs!7>N@i5In@una}L~H zH003mo9Cx}`(3~KRj{*sMX{Mu>;{8ZL)#zi8e8Za^e(4g`mNg&AoVGajg>}4U)J`I zQf}-$`}OB46tgJqB3GUJW(CwfvHf+O8}+_@n{I3_1^aN_X;k#eVv(w8K>Q>}8zblv!79%hBvagxJMuT*`Be#AUghog+E`eOO`p^at( zzf1a^BH3TB@=n>vpdBtO={P922g8l8_Y zo`s=B6is_Wn^9z`zbvzxmMdc=LW7U|1j93qxkwo3KoFmTT-$_RVK9Qwh6(~1@gLBH zgekFAJ&jLS=I%;Q8ahnpi{hAAdU(NRCyaBR&7Qi7+LXKV-uNEZQd{4Auwj8+c(=N|2>5+O`$E`3{YU1Oos+5u)t&E&Nyvv2M`tO2q7rJo+&cY zI8E*EfjwCw1No27_e+7_jX28_{#0d;nuVeWBD$_F-3492dc25f2;0T2j_a4*_l;5c zm!#1ag`5|HAkNBUgs^=^M+1*$2d*??@=qLfOy&R&TvzYB0tk=e6Aj0^)6_dq!$l`T zKlC0gb4-W}L{Utk05I2i?1cHUuKRl5Cen@dS&$}E@jA9^-~xlxF=5lz+m8O={pV8Q zOzYP}($*qg)m(b?#q0Psq zgI5qkxh;$rLB)Dod?i#1nG|yM&mz4VrQ_qJOU;hi4o#nmqc>eBGutN@SR%#Du$&)%Su!d*7jOUg&B(GSf1|bWCO=H4*q6}ZNb0)(^ zd=vBJcLh5z-+nY&x|{og`@>UNZu!uJ%9yD`Y4Wfk-50nB=Y5%}_@^o!8ArUo5%9uT*O$Y`uEWgq_HitG*=vQo`=#ltKqFE;|qi!fc z=3;YkHNYRnw2a~tr|AhPpu&KVSmg8Xou&L7<8UEBqP9g+LDS)T4Q0N-v%@Nu{;_l{ zTWHiVo8$MX=AA@%_Ulidu9iKZBvVxgmG9Pl=}(N4pvUQcRSKBKr8l6&0xF?Hppq)) zZHE;N!B?-OrqkAkRNTo{vnb(^(SZtVRMD|?B0ZpUG(L@>9(^5$3A=gXIVl6aF2pMw zw<~DM@p2@zj1#MO+wi_IGUOsbQP7=pD47uXsqla*U;U=xoua%MIM39k?wu*mXF;TB z<1RT1KGR#TIK(>~gF~8Qo;o)QVGCLo{xX#*Ei2z$__qV{;XGwI8JB#2->9y;IY-tz zw^~_whp&A6mol@9MJ5qnXOh8C!_fUDX2)@rjXUyenWbH|LB`Ku_t--Dm$Z)O>{n1CiW&B?d1GA(tzCH-@(!H+uM=dr;d{Uo`Au&omb)?O&~!xxe#_ zIM}YJOQ_A&X3kX2D%RGHuxGU$L+ssAn^EIig!>BOjygZGB!;aYStd7e<@#JT9^Mv) z^}s6gj(2;Tu16duwwYhM)(+G|EV2A(?c*xUR8pMs0QMNiO-WaIlzQ~Q*r7ho6nu2} zcx!~m#GEt?^Nc8pE_zs(PQUo(7^V;TXg zImM-J97jn^dm?@*Z`Mh8uzGgE6*1x}R+6TsE{d%>x&7X~d$SZhX9XKo`#{G^HT;Z~ zDN`?QngL*p1q2*!Y>CArNk1u&TkBUcdRh+Mpb|`iV>k5;KKR`16QUF`2LvLF?umMs z0QvUg$EJJZevE!`MJh~DokavSHwSq-6N<*&0%?!?(Ay=4TfAZZ#P8VVZp~$vkQ99DyxPW_lUOgJ9zMU zNy-(!Bxm`fG1hN3?_QBm_{soK5QU`gLOp+7y)2Rd8hx;pKsC?LHB2M&z0Y1z~^ z5uvGxdKeo9BWwTnfn5{)a4#4&=X;AL>bna&0TC)}KG#P}OD-?A*_%G!T17cG=}NaS z=t9sO!U9O_pH+J%y7m7mXhMb+g(F&n#zO1ut0mp$h%Gd9&xC|!88p&Smu6^;q4(qo zQT+Lrm-Y`<`h}}L7#~%|kTt0d(}1Rv5mBOj6cd^r@frKycLwP3T%D5^dzW}5Pz0vD zc_cMr$wN=+w&i5ztlD`Y3eO?_s=k|b|a4B)>m%PeOujp9W(~r*j z(fW=hh4t{>%nc&Yg%)_SVEf*2!yX_ei}JTmUV*XkYXG+?+w+k5rj*Q`8J?LQ|M7#| zy#)h*n;oB6+cwQ;%jP zZ0h|A3m>3CAa{+7?Z+NDZ))iGO2(;cqmh&D?VwRJpE!Q;wVXQmoMI=t^0_@awhv=C zdpgB~zkMA>K(fwaiG8pUneJu}6S|y<#&P-MHeIw^aDxe2bjE16nS*n<<&yBiZ{Z04 zX$RHUmuSHZ&B*;fi&JPW#dtCB)w8urzPlgqjH&3x4tEwSNOm8kAY*X(_?Uj~ z6wj>EEbd(OU1CJuu<7S0WhCzO9W!{zp~2V2dyZq?@SVY`x#Q)$)^~WWtt*!e$2=ei z%97F`0*CyDD~`9=6f|i;f0IPG01$$W(w)e(nHVgUM_U%47p;`_9mj0%N#vH@)2%1v+nPU@19eNio zxA3{uG{bdzp>H+|EihOD9Y^yw&87fA#}w6Us#pXrlGaDl$Fy&xcBH{3!c!%K;IF^0 z?42Vkzer=xX%Y&e$_bb((|5xN{@y1B&d6^W`pZvP1~Ufobx6}N?g+o2O?plQVtb_w zS)A)dMG9&w974bCSMa(-Pa=lo{inOnZ;0heQ%Oy8Z>Z0T8{0IYpytFVHfh%6Y+tZ9 z5ffMyZ|pu(K2?3WxSh0pv*?o;U}n84Z3-AA4gp;sNm{DPSwJkK}B?v1&9I1C0TEa-{zkp;x=S4NUL;&+F>f#BDB1=te z&eXv`UwT{OVosc{ibkVI!ED&~M<9Fi3iu0*fi6*&6_frMVAZfan>A6PgspDOt7wvkT4gCi(nf`Aol9yPrlA9%qoInP zt9R)5>zD51ViN($1wR9Se0LSgRm7=a-hYz1{r5Woj9p=@S_wjq4+R3i>!?1_iH};n z1*;HFob9x3qh^{oe&2nc5hQS?++5Y+-A9fb>A&DLiU18i{bV`GZsBmFkb*bM3OU^j zBxKAYvv-Jy>5s`PUCgiFzhU}Bh^|?_!y)hpRSX`dpd_z8dp3I%PTgTTJF%ixl+2=+ z*|Oys;N5`hmGe@IZD1RbFhECnp48vmgK=@{D13S)4{x`!Vphcslq)p<$)4_U`NA?> z{5hGE978&HW7}D%T7bm$ru{u(ddLV|s&F;n^HE+Cr)a~WgA6N-UZ*8A@G*;2_ks=K zHGy&%G6`1hFy!xyEdk0BC>MQgh8g z7sX{j?@qG7MTT3NhtKbptLz{Jf`)yfE>59s1X*-__LSg(;NAv7}-IC+C2DiS#;;^y|cYO1Tfs@>W?aWdQW;e-C=vK4G7-c zk#zHYXkb?97AiUV1TnqE*Z?^dJ|DF<_&4fEd|!k`*~1+KcgoM9)R10x1{BDl_~G*# zN<SZYfr!dsLZa~VVW`u|wo#Gcm*?K?y@lmS=>$l2h={6_QYbk)Jh9)?AI9hY zOhbxrGJ{?gd8}fLZ5;LK5`Yt=6Wg{exG;!DeJ9&D?@Um~9w#9S-9KAA38v^!xWM8A zE*wEVbNsPw%?8#htseH=6Va?bHt9c31hSJ0fo}lYX_s5MVCP=hX@AG`OE@svu- zzaW-ui|uyXvO>^NzvOQ+%rDgVOcLPw`jqEIb&PkFpd6i?n#T4tcbn&~uRfC<(|8AH zr_Cot%VzX5^A}1pxKExjfgUPqRg~>Xv2+)>o||lu@O<%NxnZWkMwb=`?B0a0@rbiY zWhw&+V1#4m`nP_`vdbRUuK)cIPE}mc>6bI&&`5?MhvD0S0Wxjjtw*-M;#6b&e^L?M zYdhAh8JX-=ulqkOfZ>xl(K)5R@q^<#)1NXZLu$nCr~3$S8S#3k+^uIwmqw5PI3-Ru z*Um*fjBo+U77CQQ8NI~J!{?oj>n$#j!tD=$A0T4B=@>@c;V&`1vtG^I6UE?`^+UB@ zG3`#eu<19{B8qAOfIzP|7hjwidQHFdTF(-#Adj9sddxp^!+vI3R7ET!nP*r1%_-CQPBY<5e;zC|e%Lc*=o|f7gn?WLMg$zfdPXt3n@wW z^U3oN=XmO|?7;cgthxEj@HI42GI1&rlRtm|{$xg=wDv};Jrc?0YBo+0+%ljZ5q7%U zcYJzO?)!Cn@My zE~aLr&b>Kj=^a@zkOLfS9t$#G-Zb(Fc+-`t*g>d&f7vul~P z4?gralC_4$6`Z4gDAD|+4tJZ96IueILBvMD94y08Jb|g!UC!@`!EDb0Y?-JxD9X{( zGl>%QI&z3x^7;dhH+f|I?PAt-^o*(#d3CwR4qV-_Z|jyV*AP!5pFm-~*i}_cUK2QA^xm^P&@IdI;oza#QQPt-5dom(17L_qL{CYk^^d-` zIN)p(sJD*4Ax-l%j!>^7uCTD!6DnKP%gU>3ydA#^?)I+Xo1=G|hPhQmnFL!<%&pLM zfCO0ns!*LUiW(ZaE82U@p;lG^xBO zxHr@Kx8>s33lpx+zO?4G9Os&!%pzWCe%VB9&~?(<$E(X;KJoYnM>}nELKUbM_X3$l zhlnv<)a5;Llchag`M(N&+@&@{uBOhB%!@RcVs;}FgLGxg32eOTyv*XBq}{xk)z_`u zE0zzn%;%r_k9=^u{BOA)(2M_o?IMNtoS%62$JSwcM1H1*C^B?Qd`jMfl^5>P?4$XY zsRz$((a(S4=gVwa(Yrt&wINegOl=^>Y~3ts z{={_Uw~xF`DkE?bdRofMEe<19M{A!vdixR^Gs4D<{_*)u@zO5FV1HO(PQ#Vzz$^&_ zo6OAc{f%-TK6n6FoSmCn1SQgvB%B*Lg8V3NOT8X05JSvi{zZQh*n&fA3{ejSUPp}(NDh^N@7+@Se>Ad6Z7t6xEM0JC!Dc} zB1i%#ICRJ9vT%cDJ9)s;Js(ye+s_v(EQ%4Q0U3g0Qjh|njglNI>jfznbjj$>GX4wR zIOrX5k$*#a@M|g)-5(SLi&=X>Yog^7B9xSIaVNOdh{Py;pzW#3>Q*LP7Qi90Lkd%u z>M1UGSNroP;53rAy)L%#Sar3psa@>%96O1+#-l?=pWCO^PcJYNlKo}(anVopUFw$- zS>akK4XAi^(e~1y2-7XcKLtnBI?b3m_3`L``2Mu)^=0JNAi9VKim`lQ)-q(vyX`MIo6 z;b@iiDSXzcXrave-zNTso|-Gx9C=?k_QbIShjK^1_b0}_TM-`#DLC_#^m*xtvELtD zeZOL4#`MBkG!iqf$eld?d(eF=?VxGyduDpt_Dhp1v@g_EY4{rCxNi!IZ?>$9R`X8M_x z>9XmbE9HCV_iVJ)2{yb#|7w#ocUgn`=VQJP?L%PmKj=PO%PyJ%p{9i45FWXxH|iX& z$4O_=d2F0PDFMX5w~Y?@zU_ATkwgF9whsm2>Ac%xyx+bNqo*%t$HTT|6}C~B-4`+# z196yisxh+U%Grc5nLTgRKf5zlF*5vW(dX%dtA-xGY8zU6?9FwP;O4{*4x9GrU23te zh}!jXxl`Krtb{mi1MT12);AdJ*&j2=u2;YE#>jdz=M43zoOHWpzo}`LYv*5@*JH`u zHBnhTmiZed$zEnD?ea;s5!?33N^haR`!a4-v&}f0h)us1Yq=;C^-CCPu;g;TzUi+& zo>I6nysGw<`VBj?6%Q@%+Q&R`G72!Sm>Kcd79BaeZ`f)>3}irH@Co-)=`7{WgEC#s zJlz6g%V#w7scN9?i>*=KXfLagylL#o=S(M zhdLVuH1A#Jo>zKF>Uuk;RRO=Y)ZILg95B1b`dHPY!;j3{dAf3Z-OWwE?e@l&?>XV& z@FV=n(xLBGYFov}sExOK^rKeI|nU~b-7A7gV#=vW+gA@l@46!pw z1$%^qkSGFzu|6WF_{wyJqho8bvVYiai5YolOh@2g)+l+OiGSDrzW#W2J%wi%$**^d zN&09J2IJ0wVvHoYG&F9pfR!luFqf>0|NPNv!9~#X8y*?%`L5*PBSV}M1@#Afc(~;H zMA=*aB6Ts0W1|M;jlcB1JIloJ%vdE_omoSNsw=zc;bSor&p))bmX=;M>m(ietdkxZ zeH2Vu2yWq85wBS1K32#KcJ3b_uaG(NibAJOZkRToj_JY=S4+g5@-6bDtWxj#y9l57 z6e9cs-;mPVe9QPZ(8Dw?JRs}}0aL}?4$>Txv~KPZFp88a00q}DXhPmNJh9wBYyZQX zA7J1@Nk+!2(-gwO&6A!iP}aPdit~zQ)%oWJdYku+hJBc}v1+b#Reu2ly`V&yHd3b^#Kj5LT z+sr%f63OeQ97NHm2seIA~Z?)0}w=bKHg<-H|Tc{N(vp<7f+1ZI)v1+D+zWiPu2w&JWl3tikq^ znjXWs=a^pbyJ_f@Ctgw)CO!S!tDbck@=u#1Kbj8k2dw$J;opau$hn%$nGHdGi3}#n zvupZDhkH`lxD&wPKUw|olgX@OoT^cB1Dt(U$fe4XhSzjOtBCXg&}A?%>o*99PSxyMVXJl4QP+fj7k6yeT#+*Ta>sn zj7UE;NkyT~YLbkGcTBrKG=X>GnJ*pdBg!&rF*RS|@JY^3zLKqf?EOGKDoIVM=sXBI z8>_c%e=tk=063?=-pbjtMF7*nwx6pfYBAHm8B1NzO-QKy!-4OB@BmF*RGtH_Ddh$iFPmwYjU+oU+ zkNE9A-4g;9%`VXB{p2kLlEK|rjzRe`wX-77b9<0`ip46r{RtDfeKw8=yt>1$>%}O9 ziuYGKzW2Ft_3FJN1_y%`|qblS_WpXZu0o2h}>0j^S-(M=JA|5@8cpHJOUCg*q{c>!!c z&OJ~pL1m=9@}wU_bMjprM*Nf6sjhKutWqbp{N9&)srLK&E%Nfox`XU*;P@fc(JTzR zNlILD^6vFzJ9u3E6_;Pq^T+3a0qrABx*Tc;S>A}U_#el`$_}<7&oau&LW}=_x&O~o zgRueL@kItUxR+*Gioud4?_Vg-HsV!bqCs32D*5E(bx8C#z$r(Z zrisrj_hV$7_H6iPw3*!0&67xWNQL)aeU-IUEX1SxGzreo>MJY1(W_q&XXL1FSxwDk z;BjmqyX

Q4$2tXAjK~aU|vq);2N_hi;q5%th68U1G;J__b8Kd=ZORKy|sSn*VGN zsp-D*3vntFWTa(n5rAM$fJ$`e^am?HM< z3Zr4tz1y6EVHL=V+K>^(W9sK$c!GKii5z1p-*(q;NXJwIJs;E)*?q{LO}+8Xf})`4 z0-A9wbN~=yZ5e4H{l^9875qp{uLMM_=l}l0hxLSPj$)e{fN5Jr&#TK+)XP$q+e;`< z*OxXTPYsdoF=>?5QX-)=kEbxC(J zyVaztsDwYw@ElHI(dmUT_%slk@2~7=U2!L*c`uq%cp$bXK)-+`Gww_$DFN#NA&g=R zfRZ+p|M@w!eUp3yI0B&Hfm=4fmR<4DhGmp{?H^? z)p(|ampG22r2on?ThyPmk<596Mgyket;2;Orvc$gfSij-zDJJ^x4K~A$3UEcs1eI- zSFfgcVuUh$^u1-msSorQ!U}2uXayV>i~+5zpaHAw2g5C}8?kMz?yN1T8XKUADl!0`e0VQ0?G3}L=9IWYl32A1w7(uw%PYjc_#9MHqy2sh6UparGk@F~YmUY64d<3XxKwkG%( z@CT|&$|N3SbzDnr3y=~Uc+hoYL4sns{Hhpv!$J-yxv(L> zMT2uEOe96@@=5XUnBC=R-r&QdmCG0u>4{hlT=CN4k{7X;q9|whPY_PXm$z71F*kw_ zHXRlcMFylW{|f-G;~Jqaxs1S^Ug7I%ss+3@!X4 zUVY|a!i3^(0>(*~M!JEqBmxS)2gt3B=^==K_wuX&RfKCCl2u@E5i45d=Mx5=c_Yh9fc&+1G z3ed}dEpR*f3N)Jt@%E-61ySvHR@;JxgEPQ?feEGs15R8R!YBuu&XQII8V|gguS2{d z`ob$C03e*n^g05Edp`Ax`bGb}|1LQ^N)#+BN9Lp~59XQER0A+WZ>s&JYVkL91WISn z>AgRbM&kAG`FGYDtfHVvy3GQ7q}NA}EYdAfQQN+s0i(EUP_Dv+h4L7ffz9hI4-)(! zRCn9lJK8q-bCM)HrN^7ih^%h3IyH+iG&&AzYt#)_JWB}3j0A~Bg^CG30C(vRDKZC=1x5~e*lb*+ zKSS4oB3L`%ZI#&XL|Vf|gppO}I8aT&pB*_u<)d~M70JeWv!K36*zlW}#sp>Ls#AO_ z#=BQt?CYm_HVJSAuw&W5vJU^%k>|aSdF7d1Og%1JNeSAaq&J$gGtxdVV ztM{eBpN1xCm!AogNL$`sssHRt?h+$stsOaX&ES$nz=@z0y z@=7UQtMfiH^Wm7PQFv@mniTKWz5i(?j3AOwh3W#4VbuU##P}&OthF4##m3yY*WmEo zt6%L|EvwDwdr#dnFersbG#l5AZwR*iX*0Whx?^ZWDf@-0$7p6u;@i)K3HeAF1hTbZE&zhVetGj@eCb2CKVoD zSINdM%!33$z%v*AfKowfO=TN3``Dx8V7$wNDfyu}A2hRqTUv}B#ac={TaZR;4zaZs zIU7{Kq(#F9_Zp@vgLsbmhI}LxU3ITFX^1s^#3?>F+mL6?oA;_ilT+jJc}Ih0-J5Hf zgA36_qRx#U0pEE#Y5d!iGxt>Dl2gH+_uZ2jB?gO98`#$Ts# z&i}LkF)`xNr)K)yZ#XvP)3LJua06mLy#l|dwP7-1W5`t7GPyHK_wkifmW6ij@*9JoDP zrkhv_xQpr(Qdw-V+~wCWm{XDCc!3YG#nQ6J!oHR{Us&KyawOb~N>Gf0i3uK#D;b@C z$Or5?=iI5!J1BVN_ZIc$*@<sJzDwWl5-L`s|^;$^%bk za5a?7n$jv7d+{mxMf}Eu{%g!xd!O?N^sC5yO`{?=MGVIw>=t!(uOS+Pys0-F;OshT3RSKF!QR-Q{>vLZ~CJ`8F3CZ@i~;1rFlN?pMgwvoJ{ z(&E;7NRMu&wc0By>tXRk$Y#(&6LAGf?v@>0W1GYQ-9-zLn2^ zG~aJ@8h%z6wUK;u5uXwL{9j%py291#xeylzG^`gw55i*agZrt5kSz%`~gy!tozThhlnESab4EJYVz5YoqTHagCp?Vz%Gf(plkqu&s~tC~ZD)*jc- zxp(@R+;fOuXf*VFC(};b;&yIpf7-)O_j`C|(>EhNn9(hpaf+Uat2LvtoXeNFe%oZd zz2}047U$A@r=1din9FDRs>|Budwq4!Sm*eSdm7Q)aQWq)&|%L6nIQeO}zs;$$Jf;#hWLC9{?XP#zplng7Ul; zsN(<_tz3q)2_P3JF)gb!^}TQPasu(GjRTPeMv6f%@M)>vff7(LWD6=#K}ln2#~N#N zaLd7TqRa+bhRSvAf1nDyWR?RP@pIXeE%4~iAoEaDQ~Q*H)@Fm%3Hv&<_!wzlf(c8{?G90@6My%J(e&$4S)TR{cj+qhh(UUwfUdd^z?l;=i=kzKgbF}!b`B$DmQa$wb za3TYVG*DBn(*Z1AOg%jX%p;chk&PEPp>X|R>Y;f{5pW^AWnfakR1vQa4{Uxe07@h<%fTxPoVxm|ti~>Cb;2D1 zo9^2FH{amwLMDRH=KUAa5Rh~A^d4yAVEYE4TjrVo;2&^R^phFj(?OgG^bQvZ@LL{- z{JpR|<+ux96YYHc*1z&VlLD0Z`(yw#6 zq5cFa54PsE56qLjSD)z70RcS6gBUhJN?JD%7JGNsU~5T*2KE#veyIrWlz{Om;+O+i=EW!jztPp}-q2~o z&oRd{yWzc+?zJc@A0H|S1+dL&DQAU9 z!cHW{+(ZK^EU5Xx?Ph*{UOyNc#-r5UydcVa{P^)Fk5#yN{$hzx8R6n`LFC|H8~3+B z4g`!4l*M2I1P*>jyi_-T;?R~~Y+>7)EQFmQ!6yWlX35QO&kbNba6sDzy>3cNOUnoU zF5LD*-at5h;gNu94H_8|J@1>t+?TGtmG}9fgh=Bc2=R*CFm$Kz2my5&v4e*y2`0{n z86!kMeFXF#y!Ef^DneBTl3;A8!vH!&$TDDD(gNxLC>NpZI|ZB6S5Po#hk?~UO2Y2F_qd^2x`NQ zm?R65HUOz%Qeo*M%ytPp6Gx=4<2Gjxr0|OaV(?Dp3itCa68~EadjUPItR0jvP^dvVZN zETU2daX3Q2fk`qDM&ZHVuhKi>8CrFo6DiDR{tE`dXaF`^IfCPkpQoNK5Rox@G+NyT zQ6jK01MVECTXU+pHoB$;6M)2Cc+%mCfkqp^P3SN-EJE+NbDUqP7j~315t1~P6wNZ( z0a;8U6?T3$1Mk;4m60MC4t#o@F~D0IjWi-= zS?6HPE|FjYRVS3$Ev#emKj8ksQHHvAz>Zzbd*N5-TDUbdJCrz3^nJpngfUhXe1oD?U%rr9RA`vo!PgYy|MB|@yAxq4rvEv+ZA-u{#j$MaXj zZ9QoU=O^?pFsy|18+x z^0d9N#uAk6{!%c%23cqHKRoBY_&E4)0IUM%UAR1;5#x~K!fOZWAnS^ET(>dAp2MN} z*YXtE7IN2mdSpH^5`$g{nt6Z_<{%tn3EaaFp9}!>kfjOV0z{JtD*~86mRd{?fB{>i z3WhvL26I^!`Kk$E}s$V7sb7+|MT0M6O-NCGef( zXwKXZOzTG!Yk>CyKM}6?&ZX(yrgZ~ztxJIUXq5r~gM$f1x! zj@`OtU$LeDi@J3Oo?d`)pa6in&3`-%z7lLy&~0Db0@uoGGVhN6cm_n_+>3+G*`K`< zck&W5bri5Z5h*Ze@2U!Oqi&w}SYs|_ zHV>G*KyMXLPm8oz7`~=O3;bS!Fyj61cP=&yDWM|a7rbc8!!ZJM=0aY!%l=ttanXS> zP#|X(l;jzN-F8>(kqAntP60pz?bOt0e}^3_l5`A!WkBK7^vhni>}{X;_V7e~mT%P} z(9CI3rE|+ouFZRY+v;aUSO`!!A;CZJcmjDFN|TK`9q^}w|E!+%O%cs3UFT~4e1J$o zpbX#bjYUbr48;*_+u+Ep(zH=Ky6NqBlM5-gBlvWuw;whx-OUNv==7q3Chh)lhZeH$q* z1hj^^;p5FgCkH1n43#-pc8><~^F1ejj?(1X-CM2RtA|l30R8}>eBd48csUo;N0Zw; zt(tD${5EbbF7$7`+i^kfBN4paHD` zOAe8?OA-nfNpheMg#rUC#DUfYJBzK;6jX05?qDYeE1n+0cOgC|BeyStPAG9804DzE{r+_(r!1_ofBD=+0P@BLT4SHyvnNrT>I0950S_aXJ?C2KGA zaEN!F@7bPTIua}agHb?nka7=1f=DO_z=W_h=2(=2Kp*B^uw_6O_v>3Y`!*@Hio(5~KzNd|8}M1Q?f4k9hy7NH+l*C=5k`d@Tc&c#!A?sOup>PJu88 zlwC-+6<9;{KyLsYCz84W#4_6Q^k*NF5yaK|?GbP|AR5)jW~UV0F?rYbS|uDGu-Dp5 z0*D85AL7?PqF%myR-_7h7Lu9-vpBe2F;bTe+?&T(9svLgKJ$6XXnK-^{?8{ia0-Jz zI%XZVMKU5Hf*J?B9uWBtpnvzUZ@*rJ+RpLx_#h(_@r6bl^595=U4;@j4;ZIC&?W=B zO@ERX+ANW4o={(cu)v+0BDe)cbWjf?(|@2%f~RQl8U#X$XMJI$bZ~>+!f=uo%D=x- zHrKID8=d-{v29)YX96V^n z;48_URUNc@lrDur$Z$|qzZ+hCgmTe=Y8du*ci=}I94`wGE(`75hhDfkv~+1v z;f-0EM=;>KP-?tECb+HZ(Ij)Izl1lCRS`UA8Laz%@^Yg++`IZv5_ zm-(3-y0HOq0X$O?pHR@sO=M7M15XFeK5O3awK~L2zTJTmI9bTan0~sNjv&G1tWE$% zgBnyndb6bEFaQcAD9@257@0IZJl$V_R){mH5B;gY?rRAYjN>1mW6>Eq8{Wi!$AlFn zbjSoHXl>E&PW1YR=rh&hv{LAd!R+gKmJcmxGT{p%EkXL(L3-TNT#w_ePDt?D7n@F0 zx3~M43TNYsvy5gXIb%4RAtWsw>P%!`2UpuGIYfK)OG|qP-mlG?PQT_FkV5+c8(OqG zN*W)aZz%PVs1HPBY9@RHh%Ukx1Kk8L#RJmbpYr^nLLZZxNXNccEkHbSlNmts2Iv5+ z)@X~L{5UWg6>@m}Un=JN(7`p}Bx~xEQH9{-@E?KNlYO*i;SweoIu%i{2C7 zIT}yS2{j;XK_}+`3jk^xpcun`onYed<4a!D8ZV~MmFHbpm+L>s`(7+jZqo}#_E5<7 zT7oeh621aCNU(LlS?3?q8j`u@`RvJ_WUj2lU1E@*)`7q%{X0RX|fCq|` zgX16f8{GhbeC*k@yMB4C*jKf9mVysI6u1YMwsxZkfPJ>BsizN68sa_)<02ImD2yHr zbAwb838fx$f$<;g?_5znFkcN7H%#d|B>MTNax967OljAbK3&V+*Sxmr` zfo>O27}x*|NQ3bvM8!;IOfLLq+z8q! z#3lo-vGDOc?tS2i!lSscu`#LZ3==5OQ%`1L1J4889Acp=1t17&p_c29V0HxRM`+4m z$GU(x96){&!Whv8tOWFKNI*Rx1i=R0-h9%(%5EK=V~^=R$a-2-Nq>w;wIKrnjYA~@0yMbGpzOz(XosOXU?zx^2!R+7!w682 zz%&ktvdYNITg}9|aNYO5?65Hm#K1@#dO%pdAPOsdd|fCBX`+QL!X*c|2xiq_Tl7J# z`A`K)p6dRN%NJW1Y0kl+Uln?rD#XpN|_7Ou(kA8}L=2Nq$|MKYzRP&v*{tE(2&_0&3inklT3I{@goX z@VBJ#%UQ3BVfjC@mp|Xv#A}2wF9+cM{rJxp;vU_qp+Ww~gO4sc+GFi#8_G_Al>hgg zVZ;bJc9@4*^VW~GL0x`CY9L7qx4gF;u>%2Y3%nVgJn7ZNmA-}dQwe!1_#(e<(zdq$ z_l5pmS_4ujj<*rB>3jFe{(B?jKH{8%rCl&0fO6sgYXv5O`@sxPKNytO{@)kGpfDvA zU{89VU-y50%(f+W2mbP$?fIVz#Z!k5VQ~KM{=(rzqAbWg;&_X9e*0l}L^L3<$H1V} zOArOEh5X<4qtk&yE)GggXyqXGrX&Of7xK>eg`9Jnk1-+BATD5oKQizlHxOpB69yASsz%`Y3e$)G`{93yb)aoFcb~!IEhPE( z$+D7&`3--5BgkF!8XQN1Y_61MB_VYGduMw&K};&4@W+$se;*L&BJ!D`FOx!*|M&KP zZx*ywtl_{F1r{WE#Kvwk@^>Wu{+&#sZh81hx_`gY@w;4@`Iu<@e|Ms{9B^-Bt_r8g z`mX)?hn!x!05}8%l*~}e4f>rr8EZf;mVQYHVKkW1r0&fuSMmA!vslg5)Vv^~^`Bc1 zjYXj_xhLk&4w}!{kvso%tF67R*eH2P7uSp;<-mm@hs1qt`{6{B5)+8R? z+DE;8p7kZ{K3mk8pK^W<@`~!3;$Gh;1Bs|mDQ;0Q0Xc@r%1!Z6m)bR+oi^(|m`pa| z<~40i|Ct|+8l;LmGnl+`G|uU((X^9zviWo0`-fgrudUtv;P5rig@o*sJ2xRQ`bEw&w;y>N0SE&zlFc9`aN;0KlO^plE||JF7h+N^9xRK9KGdBpmEEc zv#vOBjlX6?=o(2q-&xK2{7=s_+ew_rH6MmvH;)%}bBZJ(TCSUQIFXkkFNIy)F1qS? z5^{96-nlUtH*x_ppn6^I=6IO$?A4)Ei53z+iG*nSF@o1#dSs|aF&`YJQ!cLf+4E@& zTa6T5U+Q<->2#@HklCeC`&xc-_sG?u?9Ji0F{Vjxw)2HahteO-s>yWILb_uQXeKYH zZvXp`lqkoY4z2b;u~70DFo-8y0G#apenW-dI`R%Ul8`$ zYia8$3YT-^m1WJsQ_a@rU8rG0uPeSEf0fTm)Rt)~z>lu5$yuz+$ix2}Evm-*lJ1Fr zDL?(092~Lk{%t{c z|0Kkdp*SXz z0Sf`T6-j!04Q11glin9_^6=%PBb1dh4f*?u#-dF^jm0xrDCpG6nO1m%!h#&b9c$U% z8DdSm3(l8fUA)nwLQ(IFV|(%_at$Asg*@G$8|d>}SHwsz-d*+a}==+LR7iQc(&Kb=oB-Nj9=dkHA$A1Hk27vz?yY_zsn?48>_a=$m@l3wWP0wR2 zEt=3$uEQMM)(3gZKwY)s$x$m<;kl}EA^Y@l6;-0MdS*3!>l1k%0!5QohpgS@I-^e* zu2b-^4X$XX-BIGA0%M7XnKr%8`JGDr15-aA$_^j7HJVoFma9*2*YN5YSSy$*RhL;O zM*j%m)?uB@&b{CF_C)&Y5V^2BU+3;3R?d2c41Vz9(C7CKg^UAcQD*xGT?~n)GK9|Z zKW&LFcDg8_LQ5HT+B?g~BYTfMBm8`a_!~QUfp>-+BGhIE@0EQ|4WyG6A^qx* znR};iet`6G-M8P|Au*^sq;22J2Ufg;$}m!6?o+93nU0>iX^+uUtFBsv{nC;*?q=Y@ zzsc@rU^1+4th@ZpIdrch96N{m=?xUWQ_Th8$Ad+)cD=5FBsZ&kZ`_^CHWfIY_PZO+ zqDh()n7_doKOK2~UjVIxE)AK)Jf9r)nfpdZdsEefdR6|@8>|Z)0=Y&Lwmz=RJL8W6 zmLs`K^Co4Bl=pMf*(($(-;?s@(-}Qr>wY0)$J+`Y9BziAM0iB31UKvOn;}x`@a=;* z4c*4lf%*jrw!wrlDiW0#bJ_Qln-RllnD?r*jf(X3@8fj8mw9aT`u4O#lRa~9l}D6rxk=d4?Kn3EzhjaJICf%sk$)i>SvXY3E0? zs8@$(M=9gpRKuo_4xUGoo)js}q%d*TtF~1(I?Ogi&%A7|zu(s~Kq|JbkfT~KbI?75 z?fG7?$9h2Dvntpn#Knq4s$Ka|E31+(*u%-0Kq8@m!$(J-3o~Lc)%-gbNxw^601UDc8exr1WCtroWxN(Oa$xr&tLNWEJao=Kn&r(0xg|>6dweVf)gdW6+}9 zB~0Xp%6_qz*6$VftG!RcvBHQus3MQNj$V*pEtifsO}N$`)^FLOFJza+#HUviFg)3O zh;@F-2c`e6)Ef)>!%~hiP{*O54^NM<&Kk2OBoj@RHPT+tH+wbe{Nj?0LfV7arN+>U zO1&fNp-*FXC5h_Pz@7Sc|Yte zaz3s!L|Qx$=S~tA&%T%GSjqTQNpZnlo^mXRrm-3AlS(+I#g%~4)@DcD6+xkPj<>jd z?%uS_Y11;oAm+tp&T-P};^mL5~=C!gV6-UW!1Lk+ICclUXw1^1037vorL<(5vm@lLawM*+~P#y4Ei3r0zv#o?(&L zTja4xx_OR;d{4a?`jP_gFp#kH6^<~evWn(^)Yp17vve&M`^?87b)t%Vjh;Xwp{7ic zo=Q!D$j-D*ql{Fqur$w$9TmJ7lpp2Vx!Mz~&XTPChRtb9GBld`cJIu6LKa7to{l%3t0ZZ6fB$@XdHt%=!~3rc zVcnSw8!J^`a`@HHlfZYQgV>qvX(DDG=jT#czbDTlJ2*PM7JhR}m|MK(sJ6{G=zhfZ z%8ECxmxyhO$v{2rnq|H$Xa2MNiQ~giY^7}rGuJ#Ql z;a#^2#Ty~qHt9`~p4`Q)HeQ`s8yU<_^ZssqmbNBp71&|__~q&7wM}_q55MxRdLxW& zB9`%HREu_T@O5|;+EQn*8=jSQC(30RByCu~UQ&J$Hvem>O3o4E`gEy+ap-D1^Nn2> z-LW9Ykxb>hd(9i`6{Z6344J#%2bk1OMxGV8)q;os}R|3YH zThxE5zw!UOHZ)5)U!7(Sr)Ez(1-y@4Efs@lF%y;F)mK8%z@<7@WQ74_Tk-U2+X7~MT9q8y96UvOUK!$o6o%mS zjJz4MgoScV4p^l6BXJ2R_S6S8>8Fjf`!^;$o8LX?oZUOG(W$=i;{c!bFBl5k(@cp3 z^qF~>+D@%%sk>9up!Yo$Um*!j!#FgLE^V zPZb3Tn|5`DS9hipeFdc7BpZ3msM zc-92omk&1B$rQ}xLP*Uc@UM68l3cd?2fa2llyP(PdkT@Rd&Rc0GA*@2^mk7S`TiDH z?aBvC$*ge68h<=19RGM5i&joFhF*5k#Is7qO5l4@3~sKJgi0~H+H1X%vb@RNuICiS zk+Ys!?}~o(>TJx41t2Fmi94;#=O=$e7RFK%EO{4`BXXorZ?OziaxAU+&c-#9R%_S{ zR2Wh|!Fhy{p?5lSm`W@GI;&b@QKIDOXcM`t;ECBFX#XWZCLvWU0fo?r&eB z2vEPR+g;{IGhFbl6*XTjB#|>6v--!GJNUKk@N4Zn&)vFmak_8u>?(30+AYQ|!;fO< zr*6Ib`T3PHnPqrZ4}PYnVEI|nLf!9b@fVL@)g1&;yi)9J)aeb_W zBi-uIfXa64C(vj{^*Ct5m+e9+!xr%7r~J?tysY~BtzHWlaDpv@ZzH_9 z|NYv|$<;9S7deZis+CdHbP2^!FmG zlm@rPt`+b0Ua7_?o`-%6MPu5$E-C%xa&%dE`D5R%gsQoSy@1UiRv~Dyp}R|)tkIO< zI!l+9C2zInl`Fgf``xm9R?m#34U6>e)hcH{E93uk)5PE{G0H64kU4eY8sibZcG5_a zc3!C_H)a7R`nJsz0_u$y_of`|;s`t8J->2TG<-cPdVz0_+m_2XoFKQ$JCznPe@ZWPr zqk007d&VjA+$fQh3Rn)3ktF7?8crFEa*Y#b(ibVL%DeNvjIAa<2#=BT^)v~b7}3<3 z+Bh38zHXpGM;sd()HqRKFmm@wc1mU+>}mt@@-Zu<7UAjg^RzmFNSSua(4!7|Kb|}oOCYybr;%?p?nJ& zqHx5(00PMX0PqkP=8zw*G9qvN`fCA3{4UYlr3FLsq1*ee?BM75hlxBYS=IMNtu)1y7C#g_ytU5k;%It*^{Hfbe*YJl_?A&7XaFNS@D{=a#>YB8|GDG;sR=zlhyZ9Vv@xa-njXV2`jaTW~Q~B8M{gQ(H z^?M^bymdO0dYJ-vYx*CFn zhyjKRz;eT{*6R=u9rc|Qq*@I}8}-NSddC>RZ1@JY3=a=i-Xd;sK;UE|Jd}#8U;bhy zkxX|pis@B8FutPLAK5_6~F77v_+IqVMK!^d}@`?R4^sB@f|k4lr7ti7l-F` ziHl*CmA3ZeXr!kKOo}F>7bf_hvaNk7M0I+8^ImcfZm_LTK)=_(Nv6Z#YAI!i3CnxV zP1d>9<6u2TPp;ZPrtLEpyvkXtoayPS!Q^Fnv4$^wi*!tdW?L!p@Y;!h}Mqk!Ym4qY^ps-5X`zoEaBXwXlH(;*nEac49N zbC{b9SUJGI^gcV9eXOE_xE6xz3J@RWK_&zEPAOU##}+^Xq5Omq@C| zY1~zZV%7Eu`9*b>*V=md_6Bvb;ya@RCf!b`ul;3Io$ZxWo?3Lf*e0)ai=Mq7AK&;R ztFEyv+Cw8d@;*83pXT4r1kJkr(P4Lvzkj<@{2f!H$$AS{SO=)YI%C&KCf!H>MX#@G zDYm#$LDiX*XSSU7Q~r||eUE=6Y2NFepjG?=41!_TTCs~xKGqrCX^f6i@zrE!y-=GK zrO51V=haq`Lo1RwdX3R*;dnnwa-{U$-K%obzNKOFpC(96GwzDYJ3X%;z*riyTH#^$ z)W09OTuSdHOgm9k<>P7C}}JCYL~C;DC0AeAOv)U+y=yO7L2^^$?Zf= zBsOCMuLbn+K#lxycbt;7x0M4M=(a%L1o9q0>8BxjFi{2~`6oyZ1I~tqj%HTj8m_l2 zE`Eve$M#anL4i*g5`S_Y+1#LwbnU$F^WI;vy`b%lM#1QV57s1_wQ29)NnO5jyyrAy zJ8hC@>S+{hGDs+G~7pS*xI&l^~>lLWuC{>+_~obuUGRa^42T9cyQG6S~Kc}P&1Eyf?&`&)*d zNv)1Y+k36J`qYE{#*SE1N?rRm$%y!Xy;z6$w}UP2_Z6x-$ZiVM?w04NOBuYs+Y?jF zKW9R~zGwxg4qZOpXJddL)XU0#Twjk|GZbLUdo9C_TKJ~HOpe|wFN+d9^}BNpUyF~q zAeL`YzQ1p?_KFTyUaj*}QR!ZEgu$-uk=vLHFrk6oJ~s=zjz=(J2eul>QjEtQjrAT0 zXeq@!QZm6;p#OANBKE~|6pl;B{h1W~8o`e5j`4XWgBc(ojJf1hPqd)ZDLv#W--!MkFJ+{_t0Vcp)(^fi+1uI zr*ifdeXCb-C(=t}5x4r*E7k=?@~bO$f=W-R4JO!jYRRdR@Bjw^Ou|C;eIb!-j+!9B ziPz#apYibE^agrqzGn6(_(W|6zSCB0pT_X#&E5K}E|~MbveFv+SI0B?E#-RT@GL&9 z+HY&zF)Ri)yc>6Z^M*K3s*iIpDHk4kcoqF~q(x_pPY#EFEFZ^@VDj`X3swZ&rUD<# z8gTK3AjUHZU_oYQV@#rH#!q?V-hx+0@1oG}dPo@q{2nOef%*B^xXy1AslL~@!1m^-F=h9SH-7rz?K3CT zdfUk@xrSI#e;CbW%+-s-^Lw}%mA`V)M(!~ny@`H)mpuF89hF8+9f@tM7skydslwsK z1*sPIk2l>$Uj!LbQ%P+vGM2v_sqo%0sn88@_B#2Q+Obl@w(x|gF1hs~@9R0Gt4h{C zh3d zlG&2!KI17J>=KZb--*ay{q^JO+RK4eLS|me#Ti8U=i?0+Pd+^QSphchTOi4IIQ+eO^~W))mlNpi zNZzXVn$~RH`VMfd0a!u^suO1E2-6K*oj_Tu@h0`%;m*y%(9o;AX;%YDOtpKwu3&Y) z@8Jq*?7(W$Xw`*iaYv}W;8CB_Du}b!>PL=7;eD5XK`9x)~R!ZSvOHaJ) z>9S=SfHSpkZ#)!yb9ro0roDASBs44^x*Y~S)%=$OYrm^L)(B`V^;jnC5f_ZGfYl3EKBr!E*%YpCXW;{GA0}N6utv~SNq1S zDbqSa{LAf~nFIA#%)}}S0}-1v=fnF4;tGVohy}|dAeaJ$pL+iuXldYZ{|Bbaz=H!$ zEzlAn2v7`0KrjzQv|2z#00t0@AzKeQ!}6!Fw+72=Ry%TaGzE&w-;QveBfVg&BIY506+3jq{$fw#c+R;-h?b zsJV2a!xE5ISYf3yL(RfbZ^}`oJTEzSS}9nX-=!<}qbH|jP;&d_UDa_WjkKLFcaIY3 z;6&m!RJz)JntDTJ*?VPa>guKt-k==J$Hp( zIC$03rMS8IR41_JUBaCYn_2g+IBOyXPr zA;z?)mp7kDWx`izQ%fJTHx9%P(b(ORRJ$NUWG(*SVSqwt20%4 zR`$n-fgUcyb2ifq#54FZi4Ems)1{xSaB>zizO=RdxLd&@k)S@s0f>gO23?8Pq%0w~ zgyCCb#r&_kWc6dJgCsjgfKzU#9k>MU#!7H%GU%c6$bDl+U6nPmEbCZpNspHgTz)5u zwO88J`!)&PU6#X&znDy?@c!%gKtEoC+71>RTt{Q!@W{VgV3y0nEB}sZ-M@`rYMHR+ zK~XSdvnXd2E}+Ch#MZ$rz?qq`)w%ZrX*###IX4T z5#i_LEXYjvPE&T40V%HYT~`)(UJ+~3=F_$2CFI%xZy6QW9razE=ITsx=10>B!#0;d z?!**V@3dc*`Oa!0+m`orEEC%5Y)8fjv%MHD*kqg5Dr3i04fS7UXqI~;3>9+rai`u+ zVKT30|C8?u@hQ_5-h>jZjt^UQDqQVSJ!WvjzMG)NdWdVY$gZ8Gj zqTI2aEqaZUoxFT$eGsSGS0%OquW?Ou#5vHI>C9GoSNxttlT&LF%A?VriiB4tt+QM> zZci`G81vPGYpm`It~yN0th_zCWUuIE&wOU={=$Y?Ou+vECKc#0Xu9u#JP7zaAd-a9 zJ_wR%x*ddU2{lA#7)s`zd^)M&lE&|nGT;54p4w+^jET~y3#@piGrDP{cY0}dalJNrp%^TidQ zXBNEd*3L}3VxPbB-aq*1_P2CUQ2|{9s6s)3vU~f!Bp8-c;`kZUHK|Pz4owU5xRLw% zJe$1ePdaNwb6U8Uxab>MwSKgBFrE67uSTz)pZ1L3&C<+GZ}Kd5&$9W}`J^5O|B$c? zOtZc7@oYLWzek3-dWqqcK9Xv78ctje?ljrXXR%Vg_e%FjHs!}Qwbo{qn&@jDoFB2<2@@<@<*k*2=fuFa=)yH31Bvu7pFiO`I7RKtuPNub2WjE*+^|xu5 zMBhp5h1ehPKBUc1>*_QDuZwBSmV;01B_SSlZvY(ij!Iwgqb9dI})~i z=AguAQ+dI&8`t*-PmbqiWOj6!V416r-ONc4Ye7xyOB0RPJgK`AEXvn=)#D%19-P^2 z4g^frZHh(t4PKnnrzH-!+~Pwb9TQvSsqm9OVV%1yZt7heZxXK!*Lv!JPj8z~4f`G~ zm$cr-)asxNCoyj5m#=A{BT zvDVAOAGpK3d@Q1fZW?1`O2(pSdpUViY-SVU2{xipgd>m+dOIa7R-H2NJ0-PN40Tk* zLJ9ht<9&zPH||qx57PwLbZfebw5s_Xwo$zOBH&S#BzsSY%FuFEr*%g_ArD9E7spdO z0@uW=eM6ttba8J#aSKM@AQ)9$>Xx(>GbJmaUapaB zeku@0A3uN>#TZ&#bg#a0V5;YFpUyE)T?JJ^HC25HRbpp|LSN#=Vp2(B5mmhBxlt(T z*V#=ZTU~Uf3;~)iI9CRTgg!|sHHJ$zr_B7hm8{cdI{bq^R=a4LH-qelmY~Z2 z;R4VSwWn!i)hjU0S)kNsCH&N`@1{WpF{ODjRlO<1dSvAjV-JId012@$b$X{LeI?65 z_3b_4#M`pLLxnks8n~V6gJ=V{k++)GQjQi+k1QAh3UB=k(9r!&`%^4Cz9Pp$k(E#? z#LkHQ`L9XE2JSC|PK;5*QnGF37322S4sEsO+e4}Ej*>3duCTW6-tnYxdLzL9@s~b3BsI+0 zdst55T9L1`U+mr|-)uUgCF#tm@$?GLHpb8PxRzIZL8VN=b1`>sHtbq?z^#WWgd?JC z>5LUk;@&@egt+wQ+v7s`9e$Es>|cK%5Mq%GP0kSBFTuj(K>a6ITM}?|`CY;&_==t8 zhj@H{(b(N|p=nW_4eFc?Q0O}y2#%z04=5(lxIW#1KBRs8fYSR1*Cp?bo%A-tyLa`! zPKTNhhTn)1?jmX0r`PqMANV2jg68D+?0v5j!7P>wm7@#p4~4MbzOZfeb01GlLK5(uy7slPTf4Apmt`$+Eqk_`B2-JL>bXWSFt9bDwCzdmBv?v6zw5qgW|8jm7j5Ey;}KNy57 zUelBYS5bMde(ACkG8M}YEy>;+|Mo4LK#X!_t;MoxV_!SR!jSYze*R1@f8EQ1fr#n( z4~wNYBj{_nZ<<;=_{(NpC|F|JrK6F^XsjsVNeQDh6XniV&?UA0R%>?U+LaXhky`C1&2khiQ(aI~CjTPY}>B2qfCy7exqb0F=>9^ygt2kr)ka>a6iE)gadZ zj5N;EMZSIoiOM6Df{>e0VqV!Tga0SjSFz1mpRcI$b*O33m@h&k7byQ->@l3A!(OZn1F|)2Z z@AwbL8UDG>VllbV*hZ4YTN7hZju|Bs^=<8{7kG)o5rag~yMsy?^f=%c1o|EDM=Mli zsVj!?EM_Em&&1IYA8h17GkWe3o30(IL#_)f+4)+rHPSvugL$lkNV8|kEK*+0cKdE3 z=U?bP*uXgm4CPHL5QSlWK1D!LA=5bb19l;|=XnPz zqYGAr#q35mT;qQf3UzsBU+!W)_`+=5%FT2$P56%ln0n=#I0+i+j3It~Y;2AYa)qd; zC)XjGV|aEps&fI}S;ezF`yB|6uG*jto1pdE;`NQMgn9(y<;M#F2|+*jO9-=}!9 zbuMo85|#_6f=CMA@H-gx?j({lLh1`>#6cp4IPn1g9rPCm9ehn9a6e3FY?4sY z`W)gvg}{dj02Uy-BG%1taglH_kS;tMxf#&Z= zq>iXN5#J234tkpF)LB9=jSp&X1ds(bJt*;nk0u5YsRXEDRF>C6aB^UMk76FT#HpU|(j1$r`Y$iwyhbqU=3{&Q8S(JoFNdJd`T zkb?`p*Yjh|r#v91fmO2uv~y*PH?o1vX(c|36~~)?p6RmMna`HlPmVmmEI1r?>w4vx znOhHiQPMflgNjm*Eh@7~SJpF%s;YkUjBX#V9Uck*Q9B3t@zs+$PP&#VDlGmz+;Ul; zVJCPEp?Qe^n6bvpr{0POsg;*6J@hIK!S&m^?WlQk;?6qTlGMK!B-NKc&By(AyZnBS zZz}Ify*3q=HS}zA`AgZ`OCm`wF-`!@-z|rBN08MDk9t z+|KrL$W{#>d)FP@lr#`W&CeSkV%ZRZV?Qv}fvycUphZMnyqKsr^Tp`5`$IUN-ki?( zc2F&dQ@qVk(8`Gahw6KStXo!!Sya=9U!UglLv(8$gWy7de_5`)T6+3S5!kAO4WHl1 zn4cU7g~7TL`a-$R2<+W^^+GNE?HbM}D0xp^qkEaCYdqzeMR0;35_b?(mX7t`6avNj z_5n5_KL0CLLBZ+qs{Y10h-a#GT88v!&@Kwc{?q|(FStA)ym`I=w*deZzbpVoYnG3Itp?{z0@i`RjK#af^9xWUU?M zX!q|asu2B6F#y*C5E3-}aA@@LspRu;=*4tQ`kl%wC~q5pbzTI8Wr3}!ct>t$49XuB zH;5{Gt?`5E)6|eQkBB(r*nx3AoDXo{(6V%(-~fI;C~Tlwg|iLBQHaMETlL8gb=crx z!2*{al413nw@bLLGrJ-&W9mDddiCtG-SVC`;iJC5viZrXj=bWt=L>CPq~ z4K91D{pl$Fzs<$w>!mwC1y`R=f3dcts*im;Vs{(QdZscWyTJrRfXRr`ckcl^Y~675 zBgi`BZG+z~%qqYV;xS#!xa(F*j>Wl2Hy`Yy_WatrHRwl>J4wO$MTn8*QjlV_;HVpR zl#^3b1Bi|xpjap!3^Ip6dc3QgV*%n3B=QnD@8Da2>;%a-0p@lG*h8b`KpY28zw9u0 zajD$s!7-zvP9%sDqfgXiA(FvET9a~+8N->4nA5-tMruETP+5VXAUdI3B9$r|O_Tb% zIvIY-jz;>02atQgu?f z-6=|?(L;FlMZ>Q!qPaK)Z3Ow(Q%dL)dy!=gS`~C-COE}FP$wci97q=))H@l`+sgGj z%mtHRc#a>LnJuIHqfN+~XQO%->A>9W{(X&IhL6iG!xLY{*=}}xa!LFC3`H!2DH^fS zzpo4dS$cZ2x_ql_!ym;jYDl2@Wr*aze?@2gGy`0qK-E}-KBXW0=W0E6mU0%r1QLQt zk$vT>=8iO!o}l3UtBQVPq3a6*PKZNmH>{O+A;$oBToCzsde&KbiC(AO?taKSEE!3T z&;4qxIYocv3YQa?{v|i}g{CR6RLRN70k^*?*D~Gv((U~ze4t7GhG13peDUvl7r!bb za6$4IjMO3B3JkI2ZVQE9-+8CxVB8&QSeB_yMS2()^IEiQJmxNW&K(0fXuH1jabSb~ z3XZ190flu8>DYS_bV1euvaHY3o(QV&JbWTDDoRVnkQ$2L{g^r2XM+b;2VKmEg{>*K zE@5z@=r`RuIzhDM=d)|tco~)aVQUg&6OE2VM-^hBgOgLa&O)}LSoy`WEE)BYyIeYFF4r#KSlJ`vLDfC*HmN;mx-}J^JFB;(v+qyHT4U(+e*4X=UFU2u>;QT;rZCZ4 zcr}~$cAQBcA@By&r>YkL3orv4PoFR$fIxqVU1t=FZSg z=f?LBJ`G>@R^prb?7*}UlG|*ncfbL~XjlOz8&ILcgSdI*MjGA@-g_VyR|d7FTV33j zg6)@l95q!d&nwN~1 z2{{6gCbB5{&g(qG406rTiUV>9Cg4owI%}R}Knh}dLJ7#MQneZU+CzT-_T!GeFHI)O zSCp82sPE=uL`?=x);l~^9diL+XAn-L{P{7%0=m_wxh#B4U=0hwO%Rr0TTM~av3ItNh##UkvUU+XTgd)TL_PejLh+=hI9i6~#Qw>u+Ri}@^BIht;0f~B8* zcU{;y0Q(rp1QQs=6{HnZRS~suu6E6Z;h-mM$7fD7~w9jo!C6bW!M{uGoOhBRKNpFuUAKOk18%?7TEeIULA zjs8mA>8Hsu;5R_X4tyjCOzjux{70~atB#|`(t&8|OHcL^})TLSE{d9*Lv@8Rx{eF3I0Y5n2U;_ zL$~F)e;>BKnf`1Q9xx+8?C2+~5x=$BS-ssF_|RZ03a-RH<|QZm1W3K@Y^VAL(b4ta zM9=LUAZ8zsSaj}soe~*?LfpybcRk_Q{T}b&~8%U?q9FYvR# z6aqkm{(S#o%PLNQb07@yBe*Vr_!S!t1q#)%jW19TB5Ll(!iP7(IE?b4yM3?0=aEm$ zMg&F;K5I5ur=k|0hiw@R5;rr$NC^IAfX*`t2aT ztKfcDLhnQ>G)cHZ`~N_;4FstNPp%h#wF2-0(Mm&|50(SyBJbZQg{Bj5F7qm(puKh^ zeIAnG0FDEoU`|gqC^?ZzFoAZc+IE^Q^{El5r z$ev^=k+rdnEqi5Oi|jR+h(=n-R%96(Ju0%)WZy}$4OvEy$zDmeNQ>8aB$dZrW)MTj zd+zbR{`L7VbKlo>opXNYT;JdG^KLIVghMIwdK|^6;DV~k-zqd&R{(}jgJf^vv&kqp z!O`B*h8uKKB)x_*yTfyzxdSL`qW`$tsw3!dfQ=#i9K`d3kSGEoN-p^={mY1LjHc^$wcszy1L#&ZakpeV12@SOz0yzSIAB7@@WehG5zAiy`*|kch zs&JiVeM~QrCyy-?=rrh8>lL)6`B<8|`cdRsYD_!I@)c8_I@8^8i0dvaPg$m02{>$x z@Nx9c05=B}XSU1SVS0KR24C*NnpSq^x~zT>#;&k#Zk4RxzAC@adp9#j=Z7!qY5+g zQE3%iQNeWLYP&!bG!E!@#q?X;9^C}MiLbV+!X1FTFW_6ipFyl8fT?G$=CrMeZu4Ts z0jn5(i+iNN%nlJJ#L+AG7P}3L-*RyyKPSN`9f?zb2t9ge60XJpHO_CH;nF`pI5ZTC z6IN1IhO5@VN+^MF+Npz;$lJOXs?r8?{ad-gCq&vW>TT@k&08Afd}a~*GOO#LC#~a% zOiG5~ruzJGM!lUgMyTLjEF4+(-lS$~fSgva0Mif-!5ctw3EL1(>!<<6k&Nw?!049c z=ll_}&Qd%oKnMkZ;QoZWfc^ma87}M28Ua4SY|u3a79Py2LN?hRyBE%5N@P?)Fr>F8_j%gOlqRqE`|mdnDS%7* zul*@OP>v#Ad2q`frZIGRcTGIYo`{|t>7H7wRH~+@PzbLK-L}{#pDs=J;b(Fv>C!uz zu9fn*f62Hk)$)Q-McAuOf77iK_Vd$KSGHkx;Ac_-#wKAQmXVO<0+V!F2b{u^k*Y_l z2gsJ-LQxHb+2-G~6**4+T*NjY(t*?q|C`Cp0Z0I}1^}Q1xnRdk@ci*3P;tRfPTK~< zW3fLg^k~FSrJ`1cw>W{^)ctb$=fA{>X~^#pYr0iLZ^@!9BuM;;&X-uBPnB-vepD?e zDJo(~aJpIeA6ppjfnIqJoG*~kAUruhYr>O}c!T+ImHA{)mL%VZux#1@qiON;II`HG zKLN*AB=AgNPjN#*+&NxYRCLC^`JWy)3I9>jL|Op`#;!57C8

n)jgJ&Gg2M$XF* zOU_cN_=dT(|FZA^+tI<0_$vDY5d2^V!EQuuwWm7>7L$1kn>l=L-1wW{ZG7P<_|rfN z593iF38#rVARPp%)`I!szo4&y6bFQleqdSxf(NWF%T8qx}dE1Thd$;G!gm zZeY#>MkXM4Lh^?I#Q>5I9gBdrW$jwC#N}UjZ|X`urY*vv^x0Pu@0B}uuX3N<7x#M^ zZ_v7-2FWdS0T7!Y$j!e6LZYg;TMg7hD3+Gi5Z9MY0>U^2UUh!sX0@&iA|-Tb_Vu`9 z9VEYYL-)WgG&EG^8pv%V!Q1ksVwyH}LZN7XF+ z&u{w2xW=j}v8lGMN&~Sz~ z$WkrkSH9fF|JU-L?L&n&yZ-w(eSH(8s&D%oF*n)`TqqIiX1sx+3GWEF%wmUw-SWF# zZJ?d*SDmx`8cEfB&dhfZ77{!@&y{ezVo{tmPcBD=bl3vCa&A)Znwf_OUx?+C&HXF- z2OYBnDW(1&g|7(4VZ?VbzX4Q$R>c5dhx+`-jUZMWgQuokTV!Joq_Fp9h$oDgxpjk8 zH&FIE!7BusMtmXF9PSwz<0ptq9}-mGe^4T4==*xwBA-Kr7kW#wj%cp+Y_E(=*X;3v zt%(BcyUHc7$ICA|L7cx)moLw3E28BJ?J*c%9N!8>`ni}tubAGQLeX^}%rAmes}w8l zKu-j>`5QT$p@xk>AL%{6?@LXa+DzI$W@d`>kU8{jB}ZV#4h{uNGvhOScl?07?VuX( z_L;cd5s?n4mE_QsLe|4c4Cb@H4q(0#%Q6%=Ue&*z^)xO^B7wWDJ1pP_@1{neQi!Gt zGjsDZRi===G!5D8WBsNk7n1-}88NtR<3X6hYVm&{$w5?OJ~n7^gLAf&AlFZLXvSfQ z%uo*wAUt77 z*r>4Q0;a#Xt5}3wf|}nYzMFXfyf(lVUQ$1QD>doi8yPS~h>_vsRFc%P5>_d~@7Qf` zWOeE!C#QywC*ixKWzs`8{!2x1<=G?bXLozX2|HU<59kgken1=SLc|W0d%uEA`n(>J z`LDZ`sE$H2)S}KiEq19vFD^1#v_j{F@ytlW*|#aiwmnCOa(RYhYN$%0{;RL9;y_2L zSixA2<-MEHmT(>&=$@>D<01~SxmzF&+ZSDb;H>7R^vR4(pWFDUv$a2}LLy<%F??0daFesGdEpd6{P(d> zV`~#m#`PQqrNr^}&NV<8Tbr7;n>M->GK;&U$MUeZAc+oa8ikF4GjuSbOmD0#24MRt z{pFLzT@%#mj^8fNEH3>1^wMF}fK6VY-do1l9|M(YZgU6pUXSMBr1DRKm(H^nDVql# zb2~I`h4}Eh>6Y8 zZKI+svWvBLW|zDl8i_4^f7_>F9n;G1jw4)a#j8H&vP^!+hbb1{p64NvfW`&36q&1z z{5}_=D|?^)U9ScuHtu;?I3cSt>**nim}T<74-wPbrmL>nya(*;?1B#8VBy!jLS&6m znC&vKV!prBxo$3#r`E%$oL-=4>Ui-0YS4?E^+6&ekakH#cIJ$Ta(Urd8X@g+9X7t> z`-6TdJv=km0fiZ&6|}?NnDbwhMm9tzLltc3O{7K6S+};Hh3}qHM#_kR;kh8!dJJNxzOA*yM$w^|gPT z+t{?VQC(@MxHO{EQ4NB{lk1T@%&Y_buG1$wh7=!VVzNic{@H=Gb=syrt$Cf-y@0Z4 zFP(lK=U4daX=TlFV6EdnuQW7t+rct`5{S?3>KT;>;Q5;{Li4ks`dpF&RpYy;37`0_ zt_{@Z46~V>727HtXWW;3_Xme`$vAGi;A8Pu|FausWn7$2F99ioIsu6SI5-) zPbecRD^~W~zm`|te;IZU?vqnjCT~PJ)~9Z{oYsW4TU6;M4q(~NP_P-z2f6<0ZJ;2uz_$awI$t!+A#kkCp>7RFP;k!9ZJb(w<$eb)E z5iw4L!Ui$}v~!g^oRw)-BcoXoX*`^i(fgOij+S~T?T6bLm2yM0u}!rX(%3S_r=l`Z z4Xxsj-s%lJ!zT5b|27M84j3E1EJ1cYGAI9i&SUF1YaeLAk%pP?o%NNf!bwZHZYp> zW#Fxt0SAV2U~~ZllUf(w>YT%RASsT~4@dv_XvqxQ;p?{OG-xW(ZUo)wOvMSO`MFEK zFlc$5i%hzqEXzpfo>9)6E^=arC*n`jegtwVh>Hu@aOR(HvPWunc@wQ<)WYA@H}69N z416xz($lQNrK!y3U|P#VPi|t(T`*jBloy0HRt4r?Hp1C$-ICoUwq2^;i^G*vT|G7@ z%R$_k41X$dHyTiMDuYRg-QM+~b z$~k9ddb<0Be^rn|M!-jafPg@jkrw|30Rg!L99?iQz&EVU*kIrvw3Dce3LNm~4QCPt ze1^A|)^vh^KpA*HAd?wT2p}LxA!NiwRNS+UJH3*0)m`?UGObR06gp+;Wq$rL9c{KA zNA)N9K?1`3&0t_l!%?>uavEP0ibo7}VB&lbFZ&qd9ID{hKhct^FiQ!2c>t|1orlSN zk0Q4fDu>j2@4xel5l^$Z@~*w8oX_m6Y463js&h){XM%3Px7H%QT-|czr{e~Fvop@c ztgFm*EacDPJlS7WA;gQkPdbM|%FJhZfhgZ`IbOs1tc3Fs$lYB^L-w`e?k##U7p#6C z@^9o~i+U79Y+YO=r`RzN_YSEd2Z(TXXzJ8_2q|hFA!5Y7)?X-rFfynrb z!_1@m0}}gdFJ!vtL_i!AZ>t++F@b8JIu+aS59rcu?~r(V@36rmcb@O@wK4SOj-EC9 zm7)%q4DKX2cu+>4`*+WaUxaX@^*<)r?IeW84!!9ihTowWTOXlC@SEb3&9GNa@W^aTMG)7`jc+?MciFvYwya5M6d=&J-Bn7#w;}IP~yKO=9{-)k1d)^LmS}q0O zYtCkSdbG1n9B^Uz&~k}ScI->z^t~yrep{nQGm-Oj?{R>numZiB(wvIAdFM58lvFgV zs>(MGd@x}kEeMN0lEfuZf_pANO9D#;E405tN?L?;a3fnv1r551lvxi2l8rfPz2JvP z7HLkz6)|HjHQp9c#+{L6UPe9-S|AzlAiB}=X#2gOEU2gu63FcgLq<+&J-c4opJq5{ zW+;Q};QXP2?>K&Vvq-2r7eWg2hID->%$1}tH8^pOr(}jPpf|W~c1#>lm_ta;tTJyB5?>Mwl60mlMYl$g{Ky$m8Vd5fm1O&O zUX3d+ARQwqM!R%yO&iV@raUKJXcfddrSy%PSZpPPkRZx7{#Kdn{yVX_Dlr@Q?YW{@V)4#mm{-r^3+Kf+oEYr*jnhPd+`$lDf)_DN$Kb z29#_UQV`L9Iy+Fc>#J4-Mt9VQJhUQ`**`FXF0VK-TnD-0(0q|BloqL>bwS@%i{Ijt?X@5#=GJcp5V#=q&t}Y32$dOd2Y2k3kfd60vDTE#?zE;}0UPd$rapHRj zu6{LFh2}ue#n*WDOSQO$O18NKO396Hmja+N^2gNe3n8t_fZ=alBttw_Os#f5ptn++ zdMi{^>jlJ&;k3lhCqvDR2tReYX;=`Yvf&srGEZGd2C#9$+z?~yJL-6wxZrNrE`8o% zldJi~wUV#5?UuUHIUGqNKZ~;rk7iu582FKOh?SjPG{v+RLgDVEW-~-)HVF*QsijTG zipVse{u0c0kQbQyp_j?(1wJV&;@`|j*##MI_)^@$n7xUEgfOuD%%eo#Sc*TRR zrtlzojPdL|T~GMJ(0j}Bz!c#k_H%f~hY`Wo1o?<)Hv(;DQEVlY3-*WQdbd#~*8EA3 zWVlveqY%Pl*!^lVAFiH$;IA~3ke9rM=>AI&8BS0ZlNlM@82lkm8(Be~f3m~75I(N| z_FX|mkUWG*yV?Hw;Eyy(E|^fp2x`~c+&s+Xh#AJHlrkhpdzZh#>-&lbtP@Hd3})$E zsKN(yQW|wJF~N)-6E2$mjejA=O9yKq&l{D&dz4&*JZLrUX9o@w2BqS7BX^g#4Apx! z>dc0D1Q{QOKbfw-W_5SEQ9PkWZwnAk;F5jlzKKuf$N(;F7aY_GW?7PRR|2llKqp7fFQbgUqpF`{ycf{QBgTPh;^26H&YI8!SsolYDjttMN zME`C4E}rI0s3MWE<<|@U_^N<59;KvkOzE)a1@dAlR za&C{uJiVn~b|kpE&gi+X*Cm?WzCP)p$oVL<7kck&I&_bBfwG7ifrTfQ57!c`9C+6t z{4IGYh;dxHj?zdJq{f8vv%>a%cLgl3uRpej+k6#B;+>mZv1ot#V5QOEiB9s6dgS4~ zjnefNc*2H2>mICS`4D>ck~W;{iEfMpYaOivts2S6@FQ+ClF$TZe_~&Zc;l3|31FeI(FA7f6xJ-wOf*J}* zNNW9{O6BGOms$}r@>ecaw&zl< zAyhiCv9oid>XTMgm6g{)`50Zz$-ElL(x{$dWrGb{hQ3v2X+r_cUI>kHQNkBJllorc zdade3RZkIqzR-TMqV{1apVG=iu_e^(j8(kG~STyyx*giRxG09PRv8_RajJNh@r(- zwl_k^NU3RFHdz_i zmA7O}OJj&54Z(g$Rt?0JkZ0URw@=t&+tDkP;&NuZivB6kz>@*7i(6&0-bkna@x!-^ zS%hY+Gd1oaIHNud3r-3LJkH$V8&Qv#q0RNA0$LH@f;Qg%@Y^F3=?EuKT5i4LJO=p6 z|7c`3P+8!q?6Y(|nI8B?V1jKVJ9F!JW5-H0dg}-;Xf$eJXNY$0Ussnynb6;=MP<%! zn6i1))!IJYJl%_4cJ=T>CQsSc_0#B2X)ErJhb0~d`Jn~=f)Ag#HzzG#Ff8Ols3Zm` z>bB6ZqqHy*E-4%=rKfW%PFb`Hctmm1Ye_AY26hzuUMe54DCk7erZ+pq5*)0bh)xk~I)`6EL3!6&uo5 zMxOA24(v4-A|r2Tp!m&(bKKqt+4U%AI-!^fVy5w zG4bmd#Xm|Ey}`B38|DqX^lRryiS4g)91NfSh4tBEf%S9tOCPz0oa^*T{pxqy$W~`g zcRkOx9`m!x;*uDlKUFrF9*|94*^4Q%7EkilGiGj_gxK)!kJ-1f!R%l53^X<~TF}QV zw$^qv`3GxQx3@53PE}2Z1Y-)`FFZVqnen9|qM_&b+-i1RWuIZm(w;tT`O;Fy?)u?> z5f+a$vrO^&ToM*=h$zmh9tk3o56&U3{m|}uTBKJqYt1#9U6}6SdLpEd<7Ih+{{jUy zXUQ2wSIz(**epE_gF}`OIKJP^T1z1nYk#l-+19>U5Xo$Eu{F38|ByjB_DnV&$zr>^ z6px;u5ljH5nQt;YbT;l4dqu3znpb6$nFJdh(QwqB8O=kHoTJSx%a7TrC9BO@%|sS` z{(xy^yVBVbZc<_3EN1~t6Ox8#tX`Q(DYL?ceS|x7HDDvlt3=yOuwdoUuiCH9nKCcn z>L4)aS|JJkd&}x!wSGc&P@aFbaV+KVrjX?>?=4i$BIbcmB%>uHs|=qSOoT6P^+zI& zvJzJ?{@fndaB~#x%&if_@^v|W@9A^U>YRNOT7di|43x}}JqaNs*O_316*FagQt_dY zVsVr;c=)NN^wV7C%%U;x9>EWz%MCcBM={1*mhH&+wUJTc`kPyk>yvt@ZylyIorwE&-iIG5KU)Tg2T|%_A2AuV)edbm zib#fldSj`>8YqM8<_@gQT&x~m)YFel>*xoQrXeh>I@wc|qJ5!1RwXVr<`8Ga!*CT# zeNll8wH#@9`E>FyM6YLs<69u5!Duxq*p;)$t@+yEXkw%1?zQwZV4MS*@exEds=u<` zO-E$u5FA*&FxrQSpq;+h9)1{8@A{CnqZbojyCtOCVvcS#$saq;(q8%Eo0Slc9e|%u z6O=`bcO4`nP&wAuId-51JT4$8=xU?f zW|_X|B(q3%n(25gH`!$owa0GzxKG+3c@PuW)W4ncgvR!Fh~(FB*H^#LBI3;P@S!}# z#;}8b2VyxOnT6PUh85tV_?PEl@O>x?kwn74c4sIyTOi*#)UHwV{&n*uVor4Bh2JoN zRy=(yy&uoQpdu$HEY4K(tM)}zq`JFd;w~LcLJ0L9O+`(GUO^MrsaSu%NuQXK``q9y zel(0lMoNy;<|s}sYW~Rrlb#?9iI`v_k)q*q7GlFQhrFVOQ2KuQKCjHoW4Si5JwLqV zR@33eu>?b#5s&vhnj6@|OxvW8lPt28nXZ6*Y^Az0MBBDCc$~SNqxchHgwd`u2YmXn zxM2b%M<$-6=;LrJ;l5~T6RX@kX~7?J4rz?Yx?-fk6san*lOTAgr@u`M8m_Mhm5?;& zFC-@hPZbn9_!?De0q*(bGryCdU=3d}4$h#Ap-XbW#5}%O^^B!#!7q(Dy1}gK@oFaI zElr|1rj%SFQN0!h7*n2zUf<+eXo~TB+@VCb%!>SNc0WT&N=n{OtUphTF%sf1YK&Ct zwvT05alWBxT{j`=7>0G;@$8q*Tkan9v3i_YZ~=iAz}9TkvJz{$Sgl*B)fCbFJl?(2 zdFI1{h{I_9Ns9^9f&EMw507_DSy6Bz_jxoIz;jA_Q@K+m!Y>wTHF^NZU?~^zDH^#^ za~ca4VTpM;-|Sy?TlZOYU9quV@5*+Xa zWb3cZ)Ko07N02PDChluWTz$HuO(3mhMm)Bbbtfm=0=r&isa)l7&Ao22kGz>~6~Pf1 zdhdQGNPXB;KEWB18a@DB@UvS^+-|7MH?e;oSE3TJ&x}p9LDEm6F=_(o;}1_~C%gXg zq4tvwI><&$&32$|N{|_DwQ)n&52)r)8rPnQ$PIahc2I|J9EB7qiGQfx4+=rt1ezo~V3WF_vLzyl{dE-yMk63hu@33ms!sobjSl z_#te-=J8J12Oc=X2F|VLDXNGU{h^*bHiL2RZ+~@9usrv%u(Gm#&L}=!stYt7 zPx-U5LU1)%WrMxK(lOQ7p6$-8suJzU^QuRoo>USTSD9M5#s_?l( zJ-XKO=HE<@CrMj(T0@?g8lD73tQM8La)%R#lN3d0ThDm{RhqhF|IOu>s+4-8!EZB- zk`0GEz8x|trgisX0g?_fNh;;BYP?){=L(~lyu&d>`!vGKlXXqwrHft~0v%}{fq{fV zd}GZvPZk`aHddp*W58S&?d6gy9W^^_4A!QbP0GeZa;TFUP}-asv4Xdt?$@L4Np0{Y zPM%<8Mfve^L#TWv7v$HPMLc5S??An|zP%+U7r5UDtkmnoKNDo>erCtjv4@0$g6ep_ z951C86B9#1Whwae3w58c-R*2|*(}#|cPu$7CIr@IB7^h#`Z{S?ZO|VI_8m~a|5`Sc z+4N6=1fxdn53Q<&X7K5nm;2$*6Apaj>b8IMDU6x{<(DHAu}P?MDW=w|E%6r@4meC& z1FicxvYmP?u)~bwJf7FIuqcFif6VQHsc5!>*@Rr}^y2oqI~~*-PoRAz2>5&TDNDc#f2+8ouAb>Luy~5 z&-Ua+6lZO4iE5B+d)q&n9qoS4m@}UdyzYyYw}Jp*@oUl)>?jwTGw9l3}H5!iL0|EQ~Diy`FgD-pV(b3I)heUGK{QAOW` z-SO1fl@;yk28V6Q)7JeJM%}iki;D{`#rE?aZtJC<^|zPxPdq%p8(p%rKK`q~1A&Uh znol>5B@LxJ?`(KI(dn*knUb4sJ;l*`4a~pGV`L7*FexvUxvSuZ6({v8nie6AL(ca! z4l1!m4&pwmn>TeZc-J5HUYH1|K;Bai#U{3AQLho&xjNi~bXbx4uNcc>wzCRPQQ4D< zbviEPi*#AJ=m~{vrJZWEt_^QME+9g^Wbw9h`qHa8pHfrR&zQa1vbKz1I^*pKBrO;i z4DrnoW!TXS-UaHg8-w9~%3+TVsG7c0W8>>v1l$_5=BN`h?+(eFMQM)R=cY9PwrpXe zMER!%iN4x#iK_hdT*Bc_=ID9%~+&^q)9Yl7$}&-xhGxTqZBZfg0q!iKxG!?sud9 zaO&1QUKlH+1&{gUh!MAsNAhxT@*e%(&)Xy{nwj(7CFPFmK6ehljXrU3s5IYOq5n$C z=GWhCUw>iNf1F8eIqQ0IALa-5h9DET-!7ZU_IDZf!!%nj_YLCKZ2z^Yi4H7b64RH8 zCEm4r-LunZu_w!Y_-)p5(!2tkzJEW&-K80lfv_bGYsUTe>9cam5rip`Ss7h^QnnF7 zHYXLv3G_2k!zV@Dpy_JnA`gLZdFe{SqXj7MD?cF#iqOqL#Fo>xiqd&29sAyo9lK2G ztOA79f1IW|dwzA_*vvGl73)>j!4*eAV}XSm$sN*T|9x>Vih*0v^9&b*y_-$`td37^ zxUf_!NQaRcVxx!4T^;w$;7Z;0l&`57GwbV96tMz* zop-*WonCi!7PHi9)jFEC6*V=nKp|-`|3e-_#6J{_t2Z(|jmDr>xv?2a@*SAT&t9)i z?(S)?ufD+fqq%PnxwEsg{lIw|drtjrV{i8)euEnu5Qw~MBr7j3xSZ zjevY)rYQv~aYlDb5C$}XS4tb&g@`Dg#A=B;2|UxEz`-OklkjmBI9;V;c(8b10zMKK zYq(EhvAX1NFo_+J>1F^QZgusa+)5Mk+Q|G~TS8)xuv7}-6|S@4QHoC}Q7B66Ro`vDNk+)m`^w<( zf((hqUoBIY><61v&WPVIJy>~=gyvp`zOk@`$XS9WWcXZamBq@Wje;a1aW1;5*95rg=;PgZ_i-EYS_0a-Q9l$YQ2*kHZOXZNWy5&EEka)pO*haIAhL;IcNbx34`G zqa|n5tS_qXtkqLgMD6VC{24JTe!SY6=yleanv+AEKxqi97n0ZWP#w?1pFcw%w@Kd8 zHa895p(OAFV4+xFNe>SV-A}r8^HJI zU+qmWIK2!~vP?NO+jxaj)&W`n@#*Q{-zf@uic!7C7hYzuX@V~W>ZyS(q#sBF(Dh9i zDd>h%t-tlEGyspQ* zLmJ(MR(d3N5d8zcXcltaCEFob{gyJv5@Qm3g-tcX8ow5g69>XF$r{p1XH3T z8k;|Twtuhik4^h4J(Df@75V?-{LE|5X!i6fS=xHD*=EwZe|Ul%qL06uH8|GM8{~22 z2|t^}hC;-7ytcA4pS|7Dctx8&XOO{PB2PoDu=>T4$TXQV%ZPJ#oA zEEP-EoKHBB#~Aro+^(XcBGRg+Gv%jj>N${}OT70!z43QnGIW2SRTO^V=ze?Zo}v-D zfakLDOc3= zntUB%i0p2UB^_!=fRI(!^9Zi-y{TAr7$&OJY$!GkhE*wUko@8~MQKnz3 zeK(8xQvn|lzry`(Q+l;4cbx=5DNwi-Am;9x3h~Idjbuz1%wZ5{{b{Cm(<&8rElw;o zwqa~{Vmm%D#!GOEij4*G-_-f@_sA1sQ)_1|bnTRr8%n8`gT5>GS!}zlHY=E=&Xvm& z{Vp29pJyNRq0k-KNMYFE`;4IPw^GkU&|=bmhNY8G^R(7l%rfV!wj`1rjT%E%G=-e` zy?xAaKLA-I*bgv&v>qAuoDw7O6_?Kh8ISx7r$F8GJ`oYz(Zl0>p}b+vXwV!hiw;t) z8n=9*KW#TcY8WyIJLF{mWECVyq?A%D_CR3TD$7~u4u{e-z7@W=OaA^M;3x216Pm$wJ8Kp%jkdHFIqIu6e%Ma!7vX1GJZy7lDVJ#P?Q9$7zCd*DP|I zeP|j;z(7b3&-|2K0o$qPvA^ed#&P!ze^`yCe>x$dp%>t*^L|vP-DGvgKN7eO``K>n z?2btsRdoOm^4(SU4L+)FuUu9mLyxRyweJ{{#?2b_Y5t^$BnP~sDfjC=3nk9PQo5S%< zGbOvnEajX`U?Q3YtIz=^Wkz6Z#@US!(0va)2g`y$Eg+iNwZrEdhar58f`PoS^LL!g zk+})ew7@PPptGkA`QA>es#o|QcK^={pgY(wB{U9%9S9$vuXjAN^kA+U$E9G{I?50z zx;(g|=IS~QAAyO}Yt>VYx3IU5o5V%?uFhLu!-#~OkazVAJJ%iv3!Uutt<06w6$yxgvoEgpStSw|)0 z>7j^yj*}wsNs9G3E}WZ5THQTfYfsI6-jPUQ)*bt3wTO{eryK$SpCe9`5$j~~riXI8 z9FpJL;c<0P9wzIKmE7@D>UdT|yC{rf`~)vsh+%V{bc-#4Y^OBff;tQjA&Q1fWpyPg zTBLF?vezS)N+Qeb`hab9Q|TMJH`7O~LK*XQSOY?d@h%6+c=3ZV#23d{8nNhv5>o1< z?70raM3W<61w@w=L%^^w<6HkUw3<7=Hb=`;sxz1PL9N3fHUSMkGA;kzwJ&mGtJbQM zX^^rUhKn0!TTt?&6`(0^5rOIXQO|Zuxunm3pYfw&2YxCMvpL=YqS<91KPbJMZXFBe6?@zf;tNZRiRNV&(-$S*vwSD33FCcD;LpeS@ zg$Zj=8~htoU0vPGPHK#_{`T76aXE(azCr*zYx_R*0M;45%;+;`(P@c$I_u75@p~BG ztp_-Pi@I1J#<`*3s(RB~cfWetwjEN} zGj7N5pKjETGh1b{R_b@>h=roOqk*KGI$d2|8<^X+qv~z`_108a67RV8-0yzYdND-M z`mmSjk4Zy7l;uN9j>FP*PmU`5%(}L=X0yw)(rmLGFXcOMd%|)xVdtIb@9#S_H0*v| zH+=thv1N?!pYRWA7ymd{G}#lD@!U`LjDg?X;_<9*2$2We1c|HU=_Z`wvA@JGQLL8DuN_t z<$@>&)f8)x^lHv<&fdYFLLPZMokJ!5tL=1m)iprF(!akvs|7?B+s>PX*%{PZ2K@(w z2{u5O^UDVEYm8P#S4gpca~N_K&o(X0TfvLAy1M#`*KUsNg7K0PTV$pje%-zG{7Pop z8)MYtV)v_yZTkrqK<|w`&WJnLw@1w=u{I83cU)|~2cAd$@8CBP))U@>2U&OmejG%bfC?G)C2^PO8zUZMQ>RANr zfY2Sa#i-C?bvUFY1Fg#+vAlVTSMTF~MH82NxVd-dn#I?oe4z^*!ibn zJRU#g726=_j+}pa$}9ywH_w?_Q|&j~z8Bchj5q!f0K!KyP#Q30BP43C9Iim^9r45$ z8QE++kWxhL2(m(mYr`*wbC4|!RQ3N&zs0nb5clg2Px8ar&f9aJ`toxhP7L9=rrR2k z`^8QjeE%fLE2o%a7^?RMl=pSY-Sn(#ta=GmE9ll`tCPxxEOW8zZc+oanH4E`L$Hw& zm8Re1GU&VLLD3h|a{R@A`v`S!lyu)`PKH%L=SG>qY&95(<-BX7m$~yd)FJg^@DlkI zbTl2@7laWfs4SEUrjLs{!Tpmh<$?9p|D#Ps>y692`MxA>SKDF%<=Rfs88x zzHPP|d+Oy?1Q?aaU)|Z*3@K1b#7eYR#>)O^m(3WkqHHHkEkmFv6)OA+Sl#EDOGUw8 zpKjuSk!}gOgL`Gxb5%(6B6mrjG<)}(QaiyK43|tBtW(y>WX_A=zO2Ux9$~|$@TMJ) zxuBRSKfPnzf@`CIrj&~d>Tne=35JCLHZsLBuyJ%Rex&l4ON5F#x&z>7$Q)1QOyR|O zYqkx&S2xWMgYLy7asOw|vp$o4!Q9B1wV8hs#Z}t_PYmA4pKuj(VQ0Zqu+=Gtu^LVI z*>Tr-#$b2;&y1Sgmxft*xeebgZ0oC5_75$vo0Tq{N2z7^Qlfqd{dS9%u)u}4_6p)+ zWncpd(19AfJl?GO+^q@RE*bOeADo?;&6lg{_&)5u_Z6R8_UkK00Rg0kDz(yh`O?6k z0_t{7big5R@t;+!d8Ph!MD)<26{|*00IrBt(mZ*eDfJDzua>(5D4a71qYbtUAuQSp zk<6L`#svXPA$^ZjRZ`r-tboLP*@Hn+{tNh1o8n2|(#?nUZ4SY4Pvu&dDy!qr$XT1`2JmAx$2OH?ykIZck)NG@w za3Z$=TwPFIxUOxUglGA`P8>4hyrl9>9<`uli*XRO%(qGBXp6ITEy3;-G8^F#`|R?~ z%i9nb!*u7UW5OR{_5kHoZ@(!DAiRfTGvV=r1KM{?LGpG#+VY-nZ` z(H!rU1atdmA5&46A(3@UVSvVMIRvgO`tI*}!kda@jv@pfV`_g2gCA;7fuH0SVQ;evSryq66I!7OA*K=K6-QkF z#Yz*Wq>CeUTZYvyUDCU9QFt?335wMD{L$ zW}M@(u&6H0@P`_yyNjjgcLE+k5!h5R7+K?G^Z($Om_s8(qTt18P^o1CO2vEczUs%o zqWzr%PG}73EK!pk9lc;?4RJdATxhmxZIl?)v7-P1w*Yd)%|a%a-;PL0@i-m(+9rK` ziTxTe%o@4fUjh-nX@4UTn$J2)H}D0)T!PBFI1Y|@9;whJ>c}4;hrc&i>2MFZPzVh_ zMcI$vJ4$|(J5TF_awRVhxoC2#qIr5U86Pv|qzO?dsn1o`-M@)B8mK2FyLR+GG0r`H zR$K_NrXODJT}D*heSJFHzZlNln9Y54CeS<|c=luE^YQU{zDRuy`Y48Yf6n52KSyIy z{xKRFFoXsU8gv>E-kVcWB1rQ%P4X* zV+wP-$gM6ogK`ep^~m>Juu~LVk4r2M_|y0S==_?_;&UjAnG#}V12op}ZX6dZbqYThizhE3zDZQy6xL4h#ndWwGsG{3CQSc39XN*rb$-M%~8QRc$ZxNCkR87)PU4d3!R6}2HUwJn1+%iJbTl%0Mxzn5PSnu+{5 z`3{UD=H>XHB5N&v4WzrwsC>3g0MV!NKRq~@5$O^izI1}INm8bLsB;&f;IEL48D$ax zUFWN}d(Rw&krHyNq%mKh4=mAdtiLjj^YEG*IL~`_uhakswek8yLffC9xe!!ppF&Ni znG<~7&%wR-ZT@Rl!cxy_^eSXheWxB=`(z!P@n6L|xmI7M;j%RvxO{I}(jx3A=Y2wE zR9-t}3~|BVwcm8VUH@8d@FLOHbB{k+(GU5{K^=fSg283y>(6PmFwh|pcZ0!ZidY#V zt`9s5VR=IC{AIJ{Xj&83KcvtpXl5~{Af|0XfjXwHUX{S6`Ih7_!GkLPaC5UsI)cno zVmVAHLvV4iF+;vhIO&Fvcibq-a?ZIJ-y8gGjGL6^O7SzKA8e79htb^gFS7oQq{Zwo zf7j?O`~0ESdGuR#=dIT5*}9azzJA#ok1kpb5;Ag~=M6p%iylUL#b6X=+=?pAq};nD za5i|}y8K*+g0SsRy~5+c6r9!i+N zV^I?+=8xpbtjSIZKov2dOpt>Y#dL`7k?*Z{J<)d4v@e5m z|Kd-qZ&G|bx{9DYcjw%RI1+&A8A13)^Sl*XGdMDNT8qmfz&Ih52fq6WRtA(HGA5`! za-5ErE)qT&${lQ$n;#SzPc#xNewgH~Eyh^;_HJu4jujend)-$r{H8P#H;Fv3v!mm0 zxtnR@QCpztF+Z&|M*jX>#~|S<#r13NG7sZSK)&hDEumCpjgLq)U%^adb~#Q z%2vQ52Yr$iGJsqzKN*q9KNxU!8+`v71n}hL{>`SUsBQ-Kq!`Dd(Z4`|V}*sh$wJ1N ze&mz#Upp%L%0|qhB9O|wvZB=v^Dt+ohLwyePm2|e;`2Fdhs(=urS`8Fs(!?S=7v^_ zduLJF#j>CPo&#rV_}F8v~dBCItn$Js5`54=_;a!ME6MGG6vQnuzEx!Q z5c@(E6oqq&NMU|x?Zr=jT`q;bR4E-rB1iHs+8}Rw+4&;=?(07EFR%B0a9pU-pGs5| z(qzsp<-_TKP)ESY@_T)#)M&-1VItb9-MS{qM7wFIBCGny=5?lXXv(5l9|WM8IDp%y zn!h+)xv&mFkSmz*Z5AC=N3s^J=&4|>-Jr}@JKOBZFYI_8W7vG59Z`k7!f8_=3K%X7&T6rG_ub+LU@CKMFoveZVg;9Ei@}K& zxY5^#RxR9kp-jCn$GaABCn@jB3U19pfNa0QeE{y|#fK4;F?C)LtRd_g@gyXc|6%s` zLeAQ>8p{S3)X|ogs6yYSSm@!=3H%McP$|zxsXB78aRNgJIDBkRzl--z1dQxY~Dd2K2BDSV|rdmg;o-DT107QV@`EU^hsVIUZ7-I*Otr8 zvf^xGl}QO!;#V|G%cX3$A}+zQn7b)Y>jkZ(r4J~jKlAtZ@9*80dB-#eVCGSL>3=_x z-QA3IV;rle1kV*Vk#S^kHhjBVlkW&MLxZ0#=*?}e@Y%XmMAQHGzO4d_yifSKU-&%s z?LKz5-jZQhjX~!#VK=n_2vbi_&%au~v+*1I%q!jNXB2!z1JvyBm~V+WKZGoW z30{XYJ>LH&3OZ6}`bmE?Ag_dp-*=$A`nOWzt0TQ#7<^5n%ND<=oN3T0pGc$hc=tLj zjt%=Pvr`FH32<{%V!vmHuIryHP05F!D|C&d$?$ADE{Hnre9k1I?7hDlsxuVB*3iJus_iI z*3CEc?K$@qQ&I4WV^~UR1>aci*N!i3S*eOhM&6@Z7)+4`e*171cCpD9+Hckie^ZvE z9y&#OUFt?x4e_U3<^FO@mGH(3d6;D>uYY;Jlh0RG@XM*&e@rJ}Lw3}RcdNp$0UD}! ziqO`#*Qe#%#N6qXm3}~!R;nqnHW;tf^CsdvH7^vFL7QZ08vq%zDk*Y}u~f|d zQ9J0#5#QSP(7!eS%+DVYaUdFl!zzZF4(J2`()G_DrRLWiiAUH+kJ(4=Z0Z?ls$rb| zJgR<8QB)QEYboD;1XeI8OxnPfBA*F8sf2|*tq~{cZ289s1`IW4fwJ&8_vDDH=m7At zQc7@NR3zRIxg1o#Pd@b=5D{h#M{|SUEkN!TE3$BxZA@5xTBSG-OL5x28yFY3GkWY} z26772+-#qN5Vw5}5GWQ>bdAIVnm>Q5X_8Okb2nxygXI?_TYFElT!nunV0rRAJe#ab zK#idxIOq3;u};_Le=a-g(R;82KVmZ>m!@oPx`zEw4V?OS}R^mQn#xp=c9r<@WKiBG;qCGe@M2*1Mz;sM^`MXmX&$UzbF)~kGb%PH(#(h7G z{7ykBtF(UsbQ(rVjmE3;u6{*Uaq~=8ID-i3 zS4fXf#z!et9p}$iJJHm*Rh?nJHW1SwqW+2R+}8MYPaD2~VD@|I%^~YliwF}tVt@e# zo=#SE2Ww2obYZ_{yt*ct!iI_efrff3dUT$qA(fO>DklSl&pnWo{JrBMa$9r3MHRgM znz=Zu;E3?%eq*AU_;zQdzYX{%KKmDFhNT1*E`|;zNKghv()A7gFh5U|2&`iTMIjVQ zNx469+U-xI>QBm_dEyF2i;18s^h}_a4FH5;?E34;oAmY)+}}m1Z-@_SiO_-`HxWv1 z5n_+lU9R}2s0e)8f2ymSD*txX#3jSckxLAUr-uhv2%HR>)I=5;j{vLT)Ch3@2)y2J z6Y|*SRS8=+WyX+Jwwn;8>ihBn4jmy}eP=d7!A{lcvm_~W5mxX}LZ$#HsJzXga#M*E zchK^ywP=!xlcf;F4PHRF7xwSQ5HYicUbY=02kPF)-bj(Efl>MCIMkq$@L<`&U&^}W znBdGc^WCFf%ZiTi0ouse#HKA+tU}_#2qg(4;q7!u8u3+N9|6cBgWV1+00Rpvo1mPm ziDVG98$tk{1i(yWMRi+5e*VX~8IDTXIW|A}!E6MMxnKCISfRbiNlE9f719f3Qbw>u z&=@X7LNpg1<2@m8w^LE0a?{6R2Y&ws3x*0lCXViteq``2@ZQFZ? z0&fe}WxDrwfibXI#bA>W3hqCc~CBS zH%{Mhag#8%+_k${IT!5{Z3H))0*~sH2ZDM*AUA%ZPpa81_beNba9GXIwYrIT9fQnr zJVLlT59NR73da_3><0u`z*w`@VXwX1txo~#YW?!w zI2MXYB)o?Ck^Tw>_VDDimsiRtQ8|w{o3m37*cayV=f4Yu)&bYZ@z}%jy;1i|#%53p zq25>Wx_?)X0D%K!OkwlMxna4*f?`~q(QLCIcy2~3N%_a*B+crMFmmYYwgz!$e}5VQST03K*S|1UK- zVU>^N2)28?K1?7XAvJcYk5DiJUb0MK=T8t|{upAxKCyMLFhJ#`I*&sb3{yrMqdRfX zW>@=*>rgMmn@nX&_h*jjnV@igwpT~uzeSQIjuyHOq^V3U8zUoI4w-~fdAVhh1ua>9 zKVjkJTg(IwcyB$D_Oq;rcx4Hdm0OL{^HS+?s)cfUucuqY(~BZZF(=+KzU( zWE^h)TCTifttdfX2c%krJxPKH0E{V@N~0=#HO&a4l1+K<8r~IufF~lSmOd$)Sz80V zpCoxC4FH7A2hHv_-IH^&XzpEDS${-T9&HxTatAqv< z^u7nR7**9~^It=Y`-Kr;7U+jU73k^dfdH^XaY;$2Ruzs$8+lYLw>cL9kduBO6L8s& z{!Fd6UE_ncFuqba`CKrN4eg>)R-{C==ge8`+}SKk(|lZFz$zmD1+jJLi&Fq5%$|b9 zv|L`x*x!s@aW8uJnmYd3)xIdulBMy!k)*D49k%3xm!D|={`-c~IVB#1U3+zpp$JG+ z=rchE_39nK`09OZOtPJ`v$_u?1_-yQx~_QZ&wJbt&-=ntz#x7GTo-`YLnYR-Y@J|=+}7@Ozq&6p z?6Rmg+!u@}d#Oy;CX8K5@A0W5BmpGGBvT85jvddOJHpu~`X@jRk73OeEeW;nJ#rmk zkl;zTOsh8H6GK0D#Ho{#x+#4oD3ej}`Wuyd(lsd|-3UD~DN*3ehthjkEy(WJaFX`> zK#u~FVzN!`mRuX>%IW(o6U=kS-8=-IBi-KJIRQ-P<@43l)%C1m1K?X}wp*V(U26yI z$U}fbV0ddVQnS_Z512@I*I!rs#${PL2LSU`D_}5NYP7(7ColR*UQ+=Q z+7vK1WF#|8(RHe!lZ`j9tMJboGejtL!muGK!qI3Mo9{($3MEkt_`7grO)F{KoY$xeUdZD-3!@3)oD8a~S^lOCqR`rGPFQ?ZV>o#fBo1 zCiXWgnVB+tpqDrdD5sg~cT$bUSB3p`{+n*n0}%T0i1h{l57lV*>*CvEkPUY0Z_@J` z5F3-CYlJDEf}jzjLG1#mWGR7c#8xKqp37nY?!Q1ol{2)no?Vkx3gtn9$X_4VWDd)J zI_SpAd}v>)80L?ax2C1~7Zj$*FRJYnNZI5_ZdN(C&l|CoR8M6Tx#H^T$MVVO4|PZ= z&7_`qk(fR3XcnU5vYZ+WRCTpTnK@{5GhckU++*i2i78-ue_-)8voiACEFI7(Xb#e) zthv{eIvMs=^(sqXr|x&$C+42dG^e`nNX0rBtP9GN6X&>IinB}ex3`G;*5oQc$ za8_tGIZ|0IRt9Jg8|n6-xbK@4yrrt*UCaw30wFuXuPY1F=Nka7oxLevRG611hNv zOV>zqZbwhP7%Z>%J)H4yjNuRPDRJ9BmjnSZr6B_33|Q0|=NQ0Pvyowp#G*Cu>~V!| zOg~Ia1Lcdd5ZZwrH(hJE{`&In7Yl?%t@An)5>J*u#AOxe2ve-=;1dp6lNe`19TK*iLs4KPK06>tiUTX`kKDD)M!f)dE$aXAf z|KodujAG|xgldKR(aQRy>Y{Fc_oMo+M8F$_tN%=K+IGbJdeHrf{;rU>7` zmMs4kt-h-~5B<)q=Xpp)>|;bRCHTbno-^Og`wweVuP7cVY+%s#bZ_7)U}*!`DJmVu z(KhZ!do$=yIONRIFh)V|#zX;Lv^0Mgy0a|g1b8e z2@u@f-L-Ldf(F;d-GjSZaCd0jEkJPhQ~CaJF1TSpLsxa}z1CcFK3iyfQSB2(8Icw# zgW>43;U6vQig3?XhbgRcwW-SCcd2hV`HBMbr;QL!V}}hfbL*2A&W$87Pbfu*xqL^5 znj_YE(4vKLyoIBs7%n8t3jRn3gIT3<3dzyH@trEz#ux35^M}IpS`Fyl?g{p}<>^7E z^|*|FY(i4ik){y3Z!&pF$#;K;}|~g0JZ>#iYy)!B?fG zq=(jnPKxAxZqEp_@Ine{?DBs75BYI4qJ&5=sXaGt=RDr$_d^?pYkV{mGyx=XlVq^$ z&gWSwcRUu=o&PG&R$ED=Bx4vJFYrz{|8PLcV@SI|e;HP!R*fZ^D>5=oQJiw_+5!a$ z877*xWCJ+tdwfr{4wb5|_jRQUb7wI5&(GI?=BAWcYHBS0BRnO9mNtZjmWmjCH;EE@ z5bI`zA*Y0cn{;}VZwK2CPnJ2jCZ|lgw-*hc>2_*Nb)L>0cd4LKQMrWA{FN4)-9SvxnRnt69^)$Hx% zD_>rD)9}(iV_uI?>3VEvy9}L!N;*3|@-ErN&o|aXl@GsNz(+NIR222v|IHPK=|(lKF31yky{pYB`pfQHro4YeJ%?iGyh* z*!{p0939(i%7Csc)%Jh}V?{B;K_KWEw$6NBT+|u_7>$w!)xp?|C7T~h7K>z*W}(~^ zN@&{+%=t8<#5AKhc#|fsRT}v?I53C8L%nK6erhf#_2`{O2DG$aoU4>e-wPRa69)-T9RfXEEHgRhEBLVr%g)H5c(2f(uTxNTnIAT&B-G>W5vK(h zs3<8XVfVkOxHh`p5S{SZt5v!OCU$b8KptqIQikB!@$6dnvC}gSTVEu-oTR%K_mrtJkT(~%K|SxgOId;<@F*Vv4sEEBWS*idwR>sdNsTd*5pFSBDF&IO=m%*Y zArqulT_)EKh;|@}BsCvMdd3b)q3tE+$|}S(#wFfSO^q_dm{m+l75$b{sL4Hac2Psz z5&C;BREb5#9>ptW*6T#%R_73+E+iroL&}fz1v1lU%j#g3m@%l~?v^6Td z!#$`egKAm>NgAi9{&ZZD$y52VXHo|xawG>#4^#{fy42fe6)p{H^2xFM1eVQsM=>EG& zF!RBXGRN5dU z_}u-04Gg8g{%}$Ty13xwA2cyFkQU_+KFEcvl9WobJvB_{)CBY8YXS|l#jO&k(0NeM zgQ8|}n_yL3ju`a~i5V9gbpi5*qd$EURJz4#G ztF_FGjmq{RNdnpv4ypqCOe;ow4JwbjcPn7%VF|+M^Y?Wzaq45+mZXBw_Yau)VO?Fj z|Ms9vZojC+*Vn(B-8m6HSh0dP+?(y^d^RgJ;kPG?a|`QYX09PWX(=Bg5d4l({L_9C zdxhD*_X*bgDhL9+{p{6kY_0@bDhN_52mzv*_3C#eX||Kiu)R7)kj-7XL@~=~IRaZe zD7EZV7V)49nn|m27h1~eL-(tI&wg=&@7LXix5sJ>e~$9H^8TZQ9uSTP21pz3C$|+X zdkK*~#r9cxx@NFnHF$qb@urT*MWFQZ?Do}nMb>w4S=blDlS)U|@?+Zg?m4`85My9q zjE|2OM&nysTmLyeHU`YEwY83egM*pZVyL12O-x+>DlZp->#54(o3mtwgQBCOv$V9# zZEH)XiCF6PCuU~{##s*x3kwSkO-;r%K#R`8!690!DM}ZDnE*KC^1;S{pf2?yC)?uu zn`bRM+pesvoX7PTlf19*ttAWEz}A+rv9WP}VF8FIjP?Y<&~8sG&Q+hJ$}<3`n%dZC00!2h0^y!hN%^r6#Dg*8u!Z=GvF^t6a@MR z$XK}Zg$M`asfYl+ zM#2wi7uN4zqH6~eQ%0|5AWnG$(QD{S3SGjJ>48NxhMX9?1#2~C>8zMWbhYPRaAjItx7B{)MB{o&gfy6oCT`uw>B z_59_L@LwW?mv?lgpp!trXBadgEReE0D?^$yMGpNJ=&EU)T%(0hXMqnnRbpOoak?f* zA4n0#ZyN=YJWyZjT=`@n}&YHdG=S8q~sYz|FxW!aFERX)-~A4$$FfXl|Z^ zK%^^71_#BLn|w1OwLgu6vL)2iu#9jGqs%woI%VZ$l^L+npz4+h#!{GJ8HS!?ohzDC zwzu;z&noSIl~i`70op$bvA}q(NoT#JA~G_v{7(&O@lc6>fy5t?afeem3KkaBjg6eL za&nFr)1)zkCnAz!;;QIjbP)sjvF7}bV={=!*p~)0bJ$?X?~A5_@HXe^Fj29{&i5sP z{RvGl)GPtt=|W;6FwKjXT0uDXf)0YdzC(`#1}}#Ue~$Az8$(vo@dt-WxILYI6iCEg z?Gp3%?8XYpcd<>$HLq=@4^T@xK>nFY(=*Z^rYoE4Y=FsSrKbGs2NFGo;%>h1Zud0* zTbjhJhf54q4UPD*{aJ#}yV3~ZyfAP=L_~y^?hi0VoWtja;-;z?A2rT`f`Y%R?Xg*W z?iaxL2M4i2zcDlVgxTT*a<8PK=bZfGILivBM0ThjE{_G*ScPI1O&C9z4Fv`ROr~YX zRXPan8U;zWNtMc??00c(oLgQVOwzEWkvdvN zR~peC({taH!VQHZi6ASKhNDi87-cwb-)#a@5`>Z}M{pdrI4vNkS7{5OLJNFF*fQLD z3gxxycx-F<*4NWW#h+6&)~3~JE#*Z}3!za+h`2>s=eK>S7bA9!8BySpri~b|Vq5-k z_R03;g<5cD+fIx?jEKoyX=kF@jx=M!>nA8I0!=8l))J!I6tw_K4nz$W`f(SdASLE1 zN4dV9C91Sn02QB*gq)np_ zj%h0vpaZ6IXamTpv!*Bxl@N9AWD%;tdz%X59yJSLAIR&N7OglDM6iH_?D5#r2u6k7 zq@rH@^ymlBQmZrU=JSU8<6SfHRtm5cEY@6b@eVW#QThGVW$dOLfrO}1|41VBa<{*r|DA>ZB$fvE(fGlEdBY*B zsS^+JV_W_3!Ih=S@ZoiZL34KW$=8co@3ZaC?-%+NwEQO8LrhuD5OUr`hkJ!}rWhzU z^e6-&l0X+IsbSJ1Q(L|N;&N}mVl0m6^Pl_rlR zuZUU_Ng}-7Vp@)aGQGh@mwpo!zp}rsCbBwtg}KuMcV1S0YgNea#Bqk?v^2bI-!spr zCxtr8OXaGy4j#&%X-;%4A5@wYXoaMNd`R2azs7H*Kyc~kuhyKdk?l78PbMGoR|e41;}Ad`RFBo) z-bo4TRff_@Xp%_5*;_8wjH^4DL7lkRpv6xjQXyBILE0dMfk5||E_v0FhA2wM^vU4Q z#?t#(XWPz2Oh=i_3`T=Jor7OjlS{XPrWVF#3p4j(q8g9xf?Z}%H;!nNyo_qJhzTvG z9a$;+HEq#^;)Kn4|f8c9~@nO1_KPDV4M z1m#V18!%zCh?Sn!eqB^9q94XwM(vUKVy61wNS3d$E|Fe@Jfz6nHCpc(l2`5AO;eoR&|0hRwh3Jb8_ zDX7r4`ov65%2&1QAW%_J{nOlMEhMES29M8%QBmO%!Deq8P z@Yun!R4zQvOFhTWcW3*PeCMWP$>n;FF#$aRLVkyU-DII$quy>Ei-U8aT`a{d@@rHN zg8RHO4Mib3j?`x@QfSxP+ke42(>g7`L%}x*zt_b#EMT>+V$5!|+2}}Pe!H@G{0%Gc zU7*6v>hn2G!hm*u_N^UTvm_V@Px_-o9sp76d|*8F4|H(=#B6pCbnNn2IFpKPqN=SF zlv%YwEXjZ&_uKPL9}-gb&O*>nMLI(vh0oD)(QsQuwyQMQkw9V9syg;Xck5N9DRR$H z<|6sKF~3o-_J1%ob(*wO{7J9Kal#JHZlr%-24@y@c!eVy&EkL2>=R%h)fN;a78je- zU10$mpo*GW$S1ZmB;e}KLm*zo#U{w@TYbtJ8bif$8MCwS(7|bO5u*sGHO2#!m=R(w z6?40mc6Mar!EpcCp4!^1G_Y2xOdMVVzx$&`Rb6d%X88bNqtkk>!*jy)8li}Rv(b~w zMwbr}#(hXnl&-ICAJOvMvKX_Jp6<#s*yVAK9#ZKfyTryxgJ+4FY@=W2FyoVcuuvFC z$jBJIM~^uSR1*`b3#MslXD}xOra(2Kj6g@{G@i35#FXK9gq4F+URp{DhpDEP1pF^E z`*zGjb}}Se9z-nPuhAg9|5vj{RNy-kQ#jFb9oIeB!L! zVM~MGgUL-k;bJIZ{-+LxmwPa40VgCUW3Q~Pww}K|r)6i$$By8O485S}FTaS?+2?#bD_M^2m!TA_I%osmGe&2yQw!&NIv;RlyyTQ+QT~ck zo^3>GNT#puNO*lV*bk@9-th-KAoVsyJdxjdE;lu)!zduv zdf(d#+?Fb+=&6YboHtan2kiXgpG;-*!<~7QK$jev;pow!kf`VYxSe0qa5*6kq6^;vGhm4kEfKXjvBt@8sXEvHt7i4sD0vQykpQowqcTGZ#W z(=UZWz&*UQgo`axt7z)ulbP0wr2Qpgfb3uA0Dy0R?BeX~ic|6~@7TBn#_P7zxz1bb z?4em8gBFf2!j}R?8l<@b>e6b(1`gH4`A%`A{`kJ1RyH>IwWX~{mFS`aWbJO>B;%m|S4ldxeN4NWa0!yk} zzzKuJR0ttZKVu2}G z5{Dt9%;Uw<9-*@qBl@pvj(m&jP8hKI{NINn8h^#P}OxsdQE^6cBA6tQJ&1`I< z;%gnLN=imd+4k7&E|m>bCLSJe8?iilb(ydfYV=7N;Dpf~(Wv#wW?7k=jcX-QLnTSW z$Z$NVSp29mdj2HL7byc><#&M`cSZLE?CWT`LRHMjPk}5QTk=ENK?v)Yn*OrNZ6<2U zXjqZZ%LLjw$6xSUPVZJ*fiHfG@5NJf-~?c&%FeLsU^3gGF7mFmn317{H~mh9F}t+% z8{msbE4Y6}M!vNyI66Ps?H|D~L+07w-<}hLFdVO;II%Z z%{n}_71MfG2xrU)*ZZQF2E*>_`)bNYdV70&_BzL=T~$ANv?_QaKXp5(U~?C!UI5Q> zNUiSYcR;q|oa21tnu5qAg7U53BC4pgH70!p;df(RJ-5`LF4NG~7VTZvm0s7=kU>#i z7XbTM7SYq|lt93w?0IXk={B2k2!-O}l2iuyx|4mq z`9;V-a9_5^6})MxndufZ?ER_HcCS3$6&{5EuQWam?nM44#)@tqtXEu-)UkGaJFH>`V+P4mTtdP;fIZUp6Xl;#sO#aR zJDXS8#{`?hYINuiEt*789G z8+dMc3J0<+x8wH-4S7;yE~sZ(b>DI66vQ3lCAcyQ+S>3-F#*Zz<`$VdO$_95Ie?Rv zit8TDrMOhMpGo7M*_Evd=Fub`x56vY-kt>~u`N@SfN+V4EnFoSIvZlXGA)iw`K53cI?9iq`Wb&VE+m_e4s31M!J@t zA?Muk+S%~vyS;I7PfAiw>KiDWt{*5qWIu^W7Md8;^$@G)T-EVq9mDWJ7K+3`Lno(y=!xmIwJ3yyC$*m1 z4HKaGt=mc1m)GnSN!?GdwkMRi7CyTdrjDQV*FmXc7D^5et!m&3xgh{)Dk>>$6GSaq z#6*a(Ei+*3V*FFR0ZLcY!jE3%GsDg~TKAfkKWas274p?Z4CPqrWJk4#yCx>GQPD~y zrVbaFiHUQ7x)&ylX;Ku}^803q1yDpgYJLjAhtB}Fb=fDI56~#^Qp31cR>y!9E}20; z82i6x1ODDNO1+WH*bAf^nh4x)oRaUCK(2YPN84#M-aA5R=9)$f!5Mp(+bm4Se{ z-#o@}ZrgmJt!F$C!(e|dCAEC!hof{e-qo9f|L+AL;DZbSJJj;!tc=g@mre8awnhMj zg{<3V6vUk9TsX96$2b&R=r9`*fKXa7{!h56>Tps?FBqU&AT_(KzN<_}WUA6Lg#}so z!oMdae+H*29vO_g)t@6RZB;fe;&;z_OOMu}xpUUcv8!a_rG(+@l#E>-okF$ z^UO~qJGg}uv+01z!ts##)w@`3IQhOSMey-{>hfsnjsEPZ^>g9`_d-cf_1R(B#1~vb zJYy#xa9cVrn7J8H^-Idk8EvTyDI;V_!W}|#Cr-Pk}p8z}2qas_=-! z08vzFX5Tl7?8^a*W4}9*$B&OcV;z8s$L;Tuh^;N*{l@d^%T9@)n5!%2vJGUxb4aOe zNgeNyEx+_g)pC$9Cu6P%3@E{^Y>Y?qgN5DV!YzIyTdH%?_v(*YUG;x$@p~qhS9?Tz z)$6ir_B5V=&wP~|i`BMy77T!SynU=JY$$=wpsY)EzMD&PAi}zMHH;2AiBRgFpQnH{ z*U@7~$jXZTSqg6f9^=5+N`ooi%_nTa)6@Fz`O9X!@S;A+S=~fy%%*hhB|gYyzZW{c zPGt=3&2t2m$j{6`;2?4EADKE`D1(AZ{+tU6xvFtWKXv`qR~4ZRuL+dYi7PrFQKlSY zxwrR!hxdxTs{?XQ!0=|a4~y3vDDwsfVZOQ%spCMUy!WG}Tqu>D+^K5o(ohmA4NJ7K zKKQkb)Osj{l35z5(pAq~Osww=5x zsL(|NbqcR*wYMnp(NT^M(?-b*p5YkO@>jnHJpF58Jx69n(5^U)PMZJX3{g~35i>LUMCs^NatyB87^6C0cGDj( z$?1h&LKFxFin z=H{V*Ru*$ZWCkqn#z0l?Ypz%c)|+fD~v5^+RLt*m(Jy*sg_oE@ zOECf{4Ey#$%=gilne6lU<`L|{Lf^SEihgl|m~{po`k}Fsl3LzMs;q**3Bt9*^LvgP3J)XkvX5C*u>93S^JSgKojY`fxkI}j1h z!TUa)cqjcYMc~qw5as{Wv`SDRbL&B@q5rsww?zLMPJru{kz%kh;a%0rYpZUAio%Iz z7%h$vfFKeD$^7P+K%L_e7%?h+_BK2Y4Qcg4eHr#&^+UR~VWoWi{6%?#kK-mnMT-+6yJ-7ILnM-|GY!Os2;q_8S? zKnCwQFVz61CD#9i$lu1k*Qy)&G@o|~8r4l5g()+7lq2I#^$k2Fw;))VwYi>EQB|2D z+W=Yk1c5+!(YzQSKW0LO^M0uIikM|@TO$c4=a$u8wC`YB3+_3;A~hcE({?b`1|@$036*vqYZ|KDY5^j5dhDkFDAk_l3Fo6;?%AP-Zp zh5p*)3Lxhl(^oq^Ds5Ixw)KfT zZzKrL1yO`+oF1LZ%F9~bp1xS7Ey^-w2x+@F!x1l2?n1|nnE#v3Vq!gh^M9|juQz)2 zIzB%BRapvnD|!GzC4y)2`EpetzwM;Vp~6(O;sS3yOpbSnwr*I=&z%dRzdC&M_5k3j_I=ZBW|%+*BPzqfvq0LYDTS%q#Pj|UsXIcI&A_e9C& zoaFJxE0S&hx9ay((BI{DVp|4w_D^nq*9NYcJBF{B?RJy+y{CvwSC^w{vgCg{|0=>R zJi`D60|jkm3FTb&+_5J97Rv;p-z*Lr3TZio3x$t%f#zww{U^{b%TOg=vSgY9KMm9w zvr7##mBXK``do*&nWi}PffAPaC|m0g63r>Fgl^^gOBI0OUUD>u01_j=tCfw3bsLz4JRyaV!o z<#an=?n`;!Q?RnKrpSPoN$jc-9e^@;yGnWNr}o=md_nh@D-9RBY8QcnUSH=2tV5nW z)BRej4ap0bh|qMu|D>`Yso+gtz5O=Aon2TS3Ct$qvg?Wfra{x*H#GlT|A6kgzXGbt zD{0TTyK@7oBMj&s9o&i}J797AJyRgT@^=1;(!Akx{beQncOwbs z3Zt+efGl<%oA`Q}4XUDzekK^Pnml083ZW83T5Q~}vDeRnXFoawW`$w-5geaWMNNg` z_CS;OX?2!U6iL{^$hGK}-N;(l6foE>tisTv`$^wOWxfQ(X`v7P8nXrh}D6JNo{v&m)28&g_ zD;6}ats>syGbgoqO^fz++V8gP7JG+@?NMgk4W0IFVbH{0KMkOBYJx0*blLGSo#n_s zEd>{Rj1@g&l{WI}Yb@r28caUsF4{o)N*fGRd)Vgt*zx46{)+n^<| zQ7d7hJq?)D9Xocat#{%~jmLq3&)}cGfAj0>6QjR;z3N)s9J-jM1;*q{?e@fxI83t3 zlQ`?aCl)>n#f_N7)c?S305}(?1t3Ce)foP#ESc?h?1^slzF2>6?@OG)We@_vr=$e( zTw6ezEw_&Bkk>{{z3n-6IdZ_gZh~it$6K!6?42rt)5K)=tdZi}RHiTb1{s3u7G=oqiap@*#Ot0kgrZr z`QnpR@*$cMQtM1D)lh(u-1y}i$T&eaInD@v?Aufi8j;lR!*ldB(H}=gR^k%kwU#S~ zQ!-Kt%F>65V@;RE6SfvqrgI&Qkq;l~%B`0kuh?BU)mXdp0*hpDiv8(>b8GVg0PkG< zNH7xcG%8j@{sBsmq7~)ZeVjo5K*$-W~3zEH^q+FB2T{(51#3yBUS2p-owC z5GT;ppgtPPFRP1)@QEoK9c{2VtmFVO0uX(EKod7Q?|M~TR0WJaQYtDKz-%B&K|aUb z!FO+1sq6anXRWsP9wHL988Vy{u%P6;^h}w-E{qy@&7TIfq3^rWcQ1VHU5F)dNSkM zZG8JLIMa~;fX-oU9wB!3cVi%+(;JTvZ@Y6*|0I*FYffdERj5wU={66rd%$h_-qbeG zX?+QbA*(?3*~yN*$}X=aqZ^Nd7m;{LoUKix<OtLYuH~9ahJ9784qAF)FhkY6-byxxz!1gazctxWcsI zyL7~rlc-zikuqxAGU+53s@!3$Dhj}?+=Ltx#>lX&Zi~tXnRNLYKc(E_*blgS zeS#s}leYFuUe|~Xk)X_PJl}o&(zzpMRtgcL6G(K&QxTbsXrA2D12lmXw z!SDxlwnw1<>j1Ntr~pFCqJFG(XDV7+V$vHAT5lNy`nfr^Sgy;wV+5 z*ladMzh4v)+GsbYpHz2e7s+5hT-Q}4Wl|5|n)s*$QA}tXj}ie*T_TXEUN!Iu00a<$ zo4n6;xg|e!1yaPPthD5GZDOTMDG(_NX{29l>sZG)+OiRnkoXC)mOivM?#@rJ+?#;)Yj?84BS$xt| zddKSwEBnj7Jswll$+FiTe&EyhOQK096L;9zbHYzgxBOLJ(AXNTN~ByiPissXg8;M+IxLEbOfMN_kgS2k250ebQ*=3(&w?8=Q zV6>eCgj^oRiLA$PLXwP$SqLPVR~^K-;_-_iTGbXo1VOcJR^yNoUYD-zt7eo#^>D+l zP6l4E{GKqV@6LAhm(59TUS-x}R#qlHf!TZ4DnHj;Rm68UdX5u6BxVeA{ z`SkPicIBNo?e_JGem;$PVxs(=2@>GOw|H&4E%aJ&M!n4JTxCrAkZT7k?>p6~MDdX(W0$3@$xJ%JDkkJ5 zG57U=UAyaI*}%vJ`s~3%XW8Ra95|UFya`2{h?wwjW+-Rhd-IiO6$R9Fv+qANd-uZ! zl@;*UGy5NF;NpeTVrC*Fu_h<(Kohq<;!c&JOq>}^2Gy7csm4d0u;1v^$^CV$JNU)?o z&Cf4hZmc#t{9|Ig`@UxYKJd%mulLtep9%9zYr}oI0)bF$LgIh^&yC&N8UpY1QL#Cu z%XMbDbvAhZ*Ix}v%IXrnw>shLh<}@1DdGkz_jkS1bECy*f*Ra}-%56dv} zp{K|7&1by|RtBw()T!68sToC$F=C&`*vI#-OF4a?yISy8#SU!ttt6mZBk*IGx zJR?g(<65+QA9QM`?+fbv$n8yuan{ z*Cc({zajLocRbB=JsGTwy%ki~xxMa>pC22J(|D=t4UWnYW^%jyC|anL#Ti<<-9bc( zl^Dn5P>0^bx7GsSpBanQVrgO1nwWM;#LKeBG&0DgQ@dUdj;=N37OcD0Y#QAl;v&zNJqAha&*PuHNp0s#wfGU4aCjJ@{XV7rEZb~CIX>^m)y=N_6}=e~K=46cH>_x0qTQwwJlx5-Ic_;_kxAr^La z5qtaU3|;gFKvtq}|zmlb1bAdLd0UAsVciek75h zC%l|=VRIb-1eR*EY6-Jyy!-M%=XtNXPI4#Pj0;ww^Ejrz}LA8A^am5UP-AXzY{uwhN@% zy};?*J9SC)t#557mUdN6?X_FK^td_=rd-$dCl_&5j!)q$?*DN9t)zOB z5UK*0PRWB!_sf#z=jRJ~mP}tNCG9*)#9p>@9^-U0A6^MhPK&y$V}TB2Ga6xPPu|*s zxl2nBWwGl}TYPBnpR@u5uHvM5)tUyy1uieIt94EVH>UbN3-%Jf&1g(3?Gc>$es);* zuFld+zoxo6*|6g}G(7y9X^sx#=qbA6EH0H66SU{^6FDIPc6O>OO;WM$DP<$?j_09WcVk{h&8sq0sXS9 ztn5D<3X#u6JD?NCd!Iex{wvFm{I|)L_Le23mx95;Z~88$>))lsoK-M>T7w=2?cceY zH(U}y8$@DzY#4nVu$69M3K(B4k`pLM{2U+k880a&%4z*Bs?F3)ez zrQch$0O?1JNzW0O!Le}aZP~w{`Cse2-+iqW&fQddG5Wu!35aYVbsSXQ=A#6-e^2=S zDI0aC!Poim+p_M)J*_L*gA!&5B^*4@p3mJ$TYpPVqe(?!KXsb;Lh@3e>VHbC!gM2} zG8%>!9RG9D-xv80btIctFp$)f8fIbjh$u?qiWqwKJkZ(PuD*f3Za+}bn+q<* z-H!mJTqRAtveu%so@QrgHCUm2#F{g`p=DD>FNL2*3#j=uj>(p4s%v?yy{hGYZR>ON z{_-*6)32&>lfFg}aNz9_%i{zPo4{g>pvkMnrcaP`F#jWHl5b3)i!Z-Br2!4vl$B>(rPh8IHyh9GHvv+Z7uz30VwjhZ{9xQ|yE;S$P za~+6V@?H^4UQ0`gnifYES`ib^E3Ork%|v0?K(3bMyg6B1VliwgjDCq)?pnsC&Va2b z%?fNX*Q@Rqe}N-VcA8G3Vqz|x1)q%sUf78NRL-hv*bF73RROYO0JzGo0d*jTpa?4A zOhuXJ6-4<-h5Zwe@3Gb(6_q6xs<|-WY^l%M2gO`v?4ly0lz6vTf5gM`iD^szZBr86d~S~)uR(WnBQmX%yaXKW46sU zo^v?s_Fw&3zYCyB)AGM!x83h>gHs6~AJ4@QQhsxaJz z3ul(6JLsQ*4=_suL%WIvz=fEf&!!+&BpQ63F|%8!4LVk!XU_oS|_uBzso^y*9{b;Y5w0(~7#fTZh)7!pYJ zFBDMm(}zn2a)zwg0UIOY=pejrnG zuk>o)SNqn-!7nWbW#zi%9nHtu4~a7)B?ld;gkc4eV(Ry>T7D3DzoK!WzU#YxpXNGyN^estSh`LE!T&B zp~7ZrWvi>RY*X%ccqNTRAstHd8jDWu3a}<*IrDAP`2;QxBMR%$18m@Mnov}lBj+(d z_y_i<$Sld=Y*Qdf9NgPzQI1d#9PcK0a!eK=DOJ^GjS+NQxR|e||5YfL{Bvq&k22mO z0z+w!Y%tfFCM*!dq8y&xACZaDrv0S;rDW)ZOFB4fVzXq~+yqE03l52zvq6e@ff}_F z2!-Ne5ssxR+y6hB&VnnBwd=Nl;O_4365QRrAvnR^-QC^Y5;O#NcXw|B1czY3-EW=u zj_(IB2GG@2PwlacBcSwfYW3u5L)*gN0rmI&bPx7Bu@>FFHRyEH9?_E`Ck;emv;fUT_p%2nMG5 zaU@bf3}lgo!bOo$*f82uNSx}QHfvV5U>mOaG>4wmw)*H@RJVYA4DG8mOJf=pA;m+I zsYD_1#{~TjInKMm&u{hM%|k!RG$&RD`kEEQl)T1Sl9g|>H+}`PH{M5Q$A63Bamy}b zDcT=zNWi*1Hyi71&r&OOE(h}E^eoh+Cc&VJDbzr(3;K)Gqj}Rm-iJck!KdAbaS2F` zma`5({TPpcgTWE%2$LO=jv{cl8 z%Myvn*o zh`Ex0AIj4RZ5(?2lU+~~q&3t;o1ins0c@}vlCAaJkT#}i{0KpO62u6CXI>$He+%8# z2mWwCOo%a$;i#46f{mw3xvi~>aaE_17dQes&$N6@I;d4e3zYfuLq3dhkU&V7FHguPG{6Jl z0CW~1{>x(=LKTS9uMQ3I`jU!_k-qLk3jH!tBS8tUn-xwV5}p88qq>abeC-X33gipm z!}#$Dmp>Rg!>0K^w4Y9oA9S|j$l;j&J9P~Iw%k`;<+JpLDeAXJa9s~l(bxM=o7-Z# z7!+N&=Nj(1)cD{C2h3tI23Bd5wV`upSu&^Opma!6eYyD01O7LcC1oqT8+*J7>SNsW z^K%PZjHB~n@?v;0vh>yyOa!%Vl*BL3$7aQAjBu$x>Drg&pS(8s+xQDzkyl*X1?7K% z)$DvR?w81SKX_n|qZ3l~JAH95W?eDfRX**_L5E%;ZQMWW&SxkGoSfEy`OIr(D;4Ks zh9Edbk|q!ENLc=p^wH3a@Hy{70b3Ib9~&S;pXA^eXT6x~ak%Y+R=H&L?WU*_U#~6* zsJE6W!ob3`??hxStgI-Yg;jSwvN{13u5_(GeZU#_4=fG9)wyuCW0}65wqLc=Fcs z)YTB{H?C62FZqVWGVBwzoK}!DFY6l^!N+MyUc-Q?lFQKc;qrGi8bD8(&f zebE7CAyLVK2QI5(NuaV)YL#+D+Gy0m*Q@4>;%R8KaaQ|v5BnK73@5buI!NoPQ|UB3 z9VQwFqgU6IiRZsZGvm>(4%9WI7wQIbMQd1aD|KRWQg)_?pTfxjHIbx0Qw#+{I0Zn1 z0Dy{hR|AOZW_|Pkj?UqqLP4HMwzE@Zke~4Eso_J%ZJRNkNxa4TtM}_N?faCFXJfIjoJ_gB1#!RY`0gc8``^5%m$RZOc(K@}Zeh<8ekN|3sK zDJ_f-GZT!*-ZpxrSGgn4RI@{g!C_FGE_r`hMdCc6mt;z(NHA#-n}a#j31Jh(51m)W z)b@^2(N@^X|z_hvhR_|MZN z1pjknJK&22b}!;=v=}}yjQ0<^Wz55Qs^P9dfGTccv)h+l)Ac?4kgSvfNebbHa4!!H zJCCH+4|1K68XodNf0x$={o-=U`X4U$qh+wmR}J^1z_QbPfwVY74!0BB?I z7Ba*I+^=uF)|E-VnXT`JH>96#Z{v)UO)tyNqCjWb%Gylp8|bDCLL|}Kky8bbH(I-8 zkZ>~#p})6bgfUt=&*e=L!sogp2G7A&9HwE&ZUqnH#bS}m$sqdGVv`faEeGglTBaI7 zkI0wA)r0ZbZNKKM46bYh_piZ;!H{F@9S)C$i`t&Heh#)~R}vb^+3&(AJ0Q{#?qzFk zv6ky@?hUQh^jNqEZP$C%(B5xJEXw*EIZ~lvdCctDU}OZxj0IldA6c+IWkB*~m$2ii zXGSMf+FXh{X^bnxvwWyBaqLU;7Z0MeIwsS3&?U5g@N-Q5EhI92YH0jD z$A_AZ8m*LWq7QS5YeX$Nq)ibArc0N3)>@1?=7c@(IF_w)-;zUzNajHjmH!@MQ#uj7 z_tl~d_`UH-pH?-CgQ&a(ET_Cfs#X5|ag4KK?)_y?-ZCI%F@R8#-X{H*U0m{e%6sMc zYkKu}#duZD*9s^yaY)-fZ8D3CO&M}-MHNMm&=;|#s+-?egMw`j>={#rQ5X|)=nT*! zWPYolZ5ktl;&T7!J-YnCky-29zh$3Kn)y99d}+6y!L8OK>;?w;`cEN;jTUzd2v35L za2)uDN!|zaP*#qw;kC%Khc{gxo`EaXqyyDL^jbRdFh0J?;JWiTjz4^~@*1@9d^OhB zWCi`?L89{Sm!B{Cx4(xDMcYX$o!s${CXzZ=B&%Qrgn4h zsP%giVC6vFO|C~o+OC8oOnF{+<*Eq^`%8Nb@p(z z_3abbs=>Q|tbwSLbYuF=^Q?4&hqA&P9e^FCdgr#~0o|Dm_P*!LuK_jA&qi$RdcL>c|3?y$ql3ejsAoth-U~kUC`H%X_E3SB_lu zvNR(XM@w-($71HFH@iltt%XsHZguJJ9*<2Xj*1~8OYvaK2r)i%RjgsbI5neuJvKv1 zNf7b8#6){fD!nNBS9}1Jd{sP3@9Cz>gp=KCL^9DkouKxId$CGP+fyZF(y>=Lw~I&I z+0P_JhGb(nAzxs%(UUb>acqr4b8&G)3Lw7b6B8fjJL?WG51Oje4y8<6c&#ebdY;B# zcU!$Zh$HX|Ot8G|Az1q2A;u|p$g=6!dpUt?}9*OQxp#dz_p4r;@&V3mzM|MaaqskIa-Hwn7HsDrn552 zMTBxXqp22&omf^)7Cs^~_f&sfMD#!WBtufqPM^z-^qC-@L0p_BwSW8QV+1cusiQy} zjoq2ZHzfQvWMv_<`_ToF-hVat-a9FaZ7QXS;IjfpOpx@S$7=EuU5x@$al4O@NY(&!)O z4q${-@B<9wulxOlPUi=VIxRlrK>KumqxU9&p^-h5a`1Ic@7wrpX|dhgc=sQl(`vzY z+3@AcmFtMsu9)=77yU*D!)K#i9CjH!SkVq;KCzfKJkzLL^k&_7x`cUfiTym}e8eBY zP(+WD83Z2E>btQ7j}INg0h4$shKHI7GO`4Ui+z5VHPoJu&;K?1@6Yt2@0j?oI@bP! zB;(;bZ*K>9oc|fW_ie4Sm?i*M=?waA-gNH!&w}hZ*CZ}LK9*oggKWEu-`XaIJ{i0$ zVq{VA#xE`Q`!~?Cf#e5k1z&8(b) z9n-+;4`<*-Uzj_<2Is1^=Z-kw#{PnWP_%>??T6D&1jGmq%eGQ&Z*uT^fP>!wV<0&5 zp$f0OH2CY6&;---_EXx|Cm9h~n29&&89q<0+0I8lX_-hsV=WEs-UR;RKdLEwCG!DC z?&j%!{fg_bTNJx8zqlw@Oma1{;)f=_(C4}+Rp~Wd+a0sHNwQl8!<3rJ(RGYWy3Op^ z8{NoLRa)8Ex6*TsOhY^3y>V(7*^EG+SHIa~535GBxct?`ZU;_2eGaIxe})BYt^RQ{ zgr$v*nXO%bNA#@q+w5~==HhC;u?LOGX(H(^>-)b(zmvc>h_|;5D|@w~v2z0fZ>v$| zmiI?;K%@P;?aFRO@RX@4yHv9lm?}=A_Vi|x`rXJ7iEu+;Y`)KEEegyHM>@Py6`ijT z9BYA*^y{pN0Z^)vY&M>}%%&)RB3GImE)Ee>B^5Bz1~{qI`wiYfOLAjo`DMQlFO?AH z!^N`1^UA-(&2w6mjnm5!(bcr?l-(F zrfK+np1qy6Ai~BkoAr7|_o;?3logeo4$mH(#ybKutNC*|z=nO>1MDRJcDS`KXP$Mt zGrn!R3+YHGSQQ$Taes}=AOUpv56sc_-4ViUo~HP8MlIC}@XvoWP7mNt?^D<<<&VBk z{4wok0$Y|>3*-Z;&IjqNMz>|9M|i+@V4?v#^N~QRcF7ERgr8Q8LhNUgS&24Xk^LgR zp{W%o!RAjsy_^!+0`3Jb3_-rugcg6x6wVYJh^V*^*2sYa=5)5-RtWZ%^BwX@^%@DJ zJ0d;TfscO!{gSDR0SLAoK5NHgd|v03ep*oU-ss~N(23m-fjnKQIowrXG45cWOgLL# zy}KOj+KG4EAGn#>D~cyvt=qAy0uPNCHPaB%t>B5#pG=q0QoyGnM7b%14dP9%^j;x5 z0GNEb{Tr+p>H)t5ciWZgXpt9=1Y)5H=)=N`u;+R@k=p(rEa~#YG5EHBDY5NGMe%uny*=y~X4sIn#Lo~7+$Ry@`wAZKH3?oPpSNsWUg1!Yu>SPW z16RJykIgL_8*X^eJ_}!V;Tpb{{*@mvPR=xFbWgm*x9?Job&_}FFG^w+9K^l;M+|)D zxO)0JSL67+w((YaZ-p0%{RJc?|a zR*%%|Duw;|C4$%Mhz{4lNB9N@DRt5u5C3dG7`DX)@FJ?xnW+U2==S%&jsE7o1kP++-P04adNr$8GAB;e^d1^rFrcb;W-TC&&d4SDr|A)4#(}x;CJY9HPbv@?^ zJHQI0wHcv`pFkz$A3GoQShO1fT-*uS$MnE6Q_*ls>={Pk-}(b@qF`65c+XUL?hI`V z(W<4FS(FLi2N+xrf<3BN9Ed;aq3*WQkYu^y3FE z#~$EAEmTrrMWD^TzMk)QQnX6uBqI0~OA!w8z}VInpMZpBD!pPkcj@G7M8vZ-=P!*i z+X|4{mME&k;AcTBm?c{nTWDz+$!VA;{fN_Ve%J;Z22+OfGDM@dx3|FC+nB|dI2EKh zkvWz6VoV^H055Pmyka2UVi_gw$(rvk1U21_g$O;7Z|)Q6vwHV?Fin7e9M=^~!8ADYrnKi{3E_$^=Wi{bImU?@eY>j`6%Xm~iQTA1kzhz1&M7@UMK#Yn| zM@u}MaurAn81-32z`z27J9@&I-+h;JY@aDZk>Zhl1Sx)}%Tx&4Kl*oKV&)b`LYO>m zfk%KRx?;;|%XQxQAxu+F!{_}E)hW_4nZh|iS#WSe_YmdT1lg4ihO{?OYw+~9;!)N>vyIa>97z=kO>RM zBQH54J|(488a_hqh9$F5uW6%@@`;yznS6G)RiQ8zdRp~xcL&^W43R#~8>Bmjtr{uP z)96%%jC)O8AY$7PHbwK1Z}*j~#_!swjg5_d?}fq&L{h+Kqk2x@)NEZ4cX_S#M^wvk z>^}{M&_e(!J#=U&CE1QQu3=wzF8AEBftig>3IKh$nKEQcgf=aP5>M9PY6C@PWoKLa z_;m6Cl(2;cNLzz!Y2dv0pW0nbu8hF1h4{giHOok=Xi56zPZ}V0(--AY-gx=)R8BSt2p%Yb&Zl69Q);dQtCu_Ar4oKhsoB{(!Yf>%!YZn^YGCoa#eqmoE zYHALSMnxrMbUm>oi~jVoHp^`&WXOZ>IEy2B z&Dy}^yj>q9c4S+l==#ZTt#RexIWU9QOgu^9I$5}XMVga}oESbdo{;mjk<+&E7WK|$ zhR?SBIb6$m#AU^E89^a*OzIe}MB|8V8H-@3hMSnaXKV^X!KgehQg5HL?ByO+>h}F7jh-4d`v^| z%TwkB7C^y+;L{ppm%kTN4xVsLKK|wD9XZ{+WYDmRzU71>D><0^;cvIz6OQ}MpQ}BO z3d|K^_n8gub14YcIEy?6X^)oS_8v(szy6Q=C=N^eqqt8;C$Gy9)R560n(-GX`C>XU zCHxbz@M3$;6J(&B-G8!Nsng~}GMoUTkTESI+y7sZX!Vd~aDfW^HO4#?fdQ{_PE9r| zww^PpnFb9S&cx0tnj+H>WiW|1LDpqep8mm6Ea$^MnyW4RWGEW*Pr%EWzVCE1aF#uM zQ|PZR9*rsc2iX6jMhN1z}dwQt4TrrEKdE@ zz4dfqw1ZvC`iXty|FQsWNsY6*5i?=?1J*cISt|A|{nMn6Xz4m$XL{epj*bMwY`#~M zFQ7|Lg)j)z;C^k&Hs2e@1+q~A9LYiRQpI*1ui~0Wmi1pE(UWaIu`(gvD4l<-X=P`x z4Hj`J=rp|heS2^5T#TcPz`b=@)H9T&C%vAo4UBv_UmaMRoHZNIDb4%nLiBoI+>~j? z;Bed#+C4T`-OUce$O(d0W6*V|2Z;pgFw?$S;~!nP?oXHSi^>D~cJ1B(Vn^T8^){}@8r=7tFFd*88F;Gz(Kr6v!X4fyN&x}F&W zr8DQAmHn&z*%wuaivS&{Fp(0LECP)Dp`GoEJ3x@+P9)npR)r=@k!~LB*6?nq&8jz)t`y5K(i|9DJXh!DTafRr?}X%5~mX>z$xa+n)%5V z8ANw-M-XpOX^{W12BZ9GHaPX!^|eYp(LCs1BK6W zK~z#%84FT9Of7u<)QE+)3Q25W?h|7D#W-jp?8q-cDVJLfA=dAa@?sdp4LuEU|AtTa zJ^G>TbPV=(idCH@{c#)Cu}pPLqL+dW+N@DW_<0^R1ltTj#|QTTGc2R#9P&xOKs4j% z-_eiW-a@v^oObYc$nXTyu=D~v+Em)Hp+_p9em(p<3wS;OqJS~nKlN$yx}B|ATOz4! z1Atz3;JZB|yAAsH^WMV%O@LB7ee-REOG_m2M{rGsc36Tst&xjOUh${}&{Grk6eLg?=qL=?tS? z0-Fw!NbIt=`=%9sFo`av?U<|fZ%$=x5zuD)(GL-cqP?QUjh-jVk1Zx%=XN8IIeW9F z)f=-4)cr|>5VQf9JD=M&nU>%OCWuwB~-6jyoqi8y`^J zUQFf`#7PNy+O2t!f1EBB${%K6-X;MT=oU@pY@^kDQK7=G-BnIyh0nPtxv%7PU~j`G zzl1h?*W0;h&gbiF-}AOJhv&!!MJ*cbMh|?%kqBVT+;<>$Rb5II0uYSLqKbfzf=UjN zAU^6U8uMo#*M)HVR#tOPe}6zzg16rmL<9`3YUMIO=e_|M3nSFet*(lFh~gS=VrWsz zPnF=(h7DUy?u72)QNlqU@*$GX0}Y#8@zIRUO^6IuE#R#eAS+VJB(OOb%}C+a(rR7i z8(D22+IB;km_SK2Jg)I^y}+L~V&695Oo6%2@8v5%ElvXn?uR*c4~5{!5(!zkdJVUv z??GjQ)pZ5T1(?7D3l7s%k+{TU^_NtYo-d4;E5;84%n*HTGCB2o2PdTl%2hWe6`-n- zGVZ|}43E#Ie1 z9*Tm!B+k>SS=f$2Pt{x}&9q@@JF9ze?T0}S41=CJ?cV8YB>^xI!c9=fPmbHmJZA+Y z<<3#{Hyv$k5bX60s=zrv@yku%xcxmU^zJz0>_92t_wjMc9rRGyuX(RfXpmBvwqh%^ z$1_rS^=>4%g`POEhp3p3`S|)uM?*{hvC{VoIs;`(9kR22rH-3tXXo$s+po`$EU2HD zU!_n_ zGra*XkfAG&nJu)SZRcLYY$C0|mo(PqYx1 zdQq0t9RR&ZT{zpR-vp{Ler0umdpu+fXs4!2)i9Su!~5@%P$8=ZiG+;m4D6>%boU1+ zP1_awb|@p)(ZUXAc$tx_(1cm!7X+b>ek;W(;$g^!M`4!Y^d?92J{1PeiW=gw2Hp)< zee3>TZ>Nzf5SL0?f@rt^o$-xe$E6bIGyc&TDHW!?$C=L(z9#zjW^%aCXfU>9U?rcI z4txL%b+P$YrOz%=t!uv7r4|D9YXQcj?LESl&vDO&-xsMd6~E=x3ODy+Yk{K9OM*!% zC zvgI9S_1ZvPz7HW@tTl<$A)lZ>RpmUvEMakmTA*)#N zHlnZ6n0lviF->@wI}VM|mHAE~iUb=1dk%4|R~VnFi9Cy{*W+@m&XDYiH;rs;fLXhH zb&Z;6rQn%{CO^;TRKupfa;L$osV&MN1R8g!5{$4Gc_Q0i>acA_)1NXOv`4%vGlF1> ziH5A|=+ccpb}0Q$feaw)$3E*ZA-=g;Kb~cQJI;E`=5)U2#x!p4BXv;)e!yFE?@Vk> zcS8TpJm)jyWDz!FR`m)Wzwiuh?=|+fr^;HhjbG`?%M*{+KcE~Qtm0KudOsp(ua+GH zIPOY`KV68PZh39k^4Y#U-+EJtb4Q1!HdU+3Cr^jv`JT7`0S?ly^D-h4Fg4xR2sJ+a zH-LMml5Ri$hBmy!+T;;hY-rr|wU5*ZNLC9UTcjo9TaAtIwH!9>w0ZihL zrm@GuY9`~J4;$0HAu&Sv}_6h1xlT}Nn1oHJ7g8IoYM)XlFIeR+~BuJO%(b`3;8&PazsYxm_BAPRddS_Q|gRh)nr!jTy^U5B)6tipP|%7PS-Yr#bDxc4(~AEl|J_( zFbMZsBfULmzS-o03$pA*=%x3HB8{vq4X~WX-K$YaSt6vRWDh&cYYZlHtFp)TDmGX8 zMYR7rvwRyK5%u6S>$k2Mw^KU^by6zc;@1!PKW2Mn@kH=Gx?FzZ6eVOaN{9_j(F+;3 zUlN(KyLdfDMiIi{^u0sCHT8aG8*lX^WP882D8rM;G3NM_f7_y4i?)FXOr!m$eaYjGg94|4fOH28QuU4(e#~u)p*!{r8babjPJ7{FZnL9N# zbJ_FIod_V!G+7M#H-&ubg8LXAh?=e2o;9iBE8vA+R%frtn^e!sUD zmsKX(Jd=;>+WUTZi{@c2bAr`XZ)2Wex|doIcL=t$Wlj{nh7<5Scfm-9zr7~H2ZnUm z1143Ns#%O z#&OIF=CVzh*vE#jCZ$d0r}1Yr02iQqDGC7Pbe89w!PCi+tvEVjYA_i_0QN^WqcBOq z99R@r@zmNUWI1%$AyMdG?hgP#I~fvokL!8(1a|f1=qoTB<9XPfG zjZmHinrYuy?x@G5Zw#?3G${y@NM+GOZEO50eRTuwSn*+#os8(d5+~Y@U`WvPYPqOU zoalI6b9}Di3=>>58!t2kx>1$ZGj;$f#b>#KIhjLgDq=NGkX2p8@4b9YQ7gY_CDl>c z;AZ@=bh@+)wcNb0A)b+&L$kGvMx4BSK}tbAq3oB12P$gs2>V#~&YQo=-3AO-$MKp( zt}aVCND^4XOQ%XQOik!ugr*90jhxSso<%vdN8{%3Y(b%u)%CBB!LV%y8G71{cJzoN zI1OgwpfHzsoP|y^g9{Kjb+&l$WKCBHv_j@pE6}&kG<^31z$mhKPzCf%a05uS&&|1j zPk?r9Z7GR5qS@bppqM^W0jn#HUK^O7xvVl7zD;P29l8J)CQG{M(BQdT3^jI~6ORzH zx?I)(ys>Jt*pzj4!rSVRd*OsE-j#y`t*!WO%-(mq56kNsTDq?>1hR5OGIpB{=_>xu z9NM<p~3t{^F9*x z_>@rg_(NP4t)`|)d#cfF+=S#$OZrYSgeBW+57VO@v2iro0PCxGC>1qAcyshI;%`U)D_rig23M_`}vAyL!d#$)X7=X15@U zD4r@3E+xZ`l0jKUf@W*e`_z|Mem*{#BqzS#>Y@E>@VMX(uwzQaJFemfL4>&o$N(<{^bmUY`l8 z9@C&gK3$L3gzVS7Flv@=*e^xbBCK45T+KWEp1NRzyefn?N38dfOR}DrQ-SS#s zG->{ELj=D55Gl9(nQB|>OG3!n5#)Un0c_>^RplmeiK;Ag?`$DaVQo||Q460!a?$Z0 z?^X=@xmbPR0n#TPdwg>}2in3o-U+XCRFZD>$ln*9MxSLHjgeinujLqvcHH9@jR2a= zdFxHY!paFSM6v4b?C+&WCWb-~$63b-BwUsnq z4IgC2qKHq@-QrUV9VHFS6DOZN>OxO**)T7%SQN7X6HC0mE-HRBM_8Hj8I%|N?jHp& zEH49d*f=Rl$s(1!F2)5y-K(&CJ>_Mvursm!Nr;i#Pz!9}0B{2OuTiqthGY0$9f_n8~W$RXFI^z-3tc6NVZGo;D+mF@fB>e>*#fQ@+GWTj>%l~24ONUuWW!_mwijO6SFI1C z=x;lmTy=1;<^`_r4DYjk$>mwSskX(g~^;KOgi^{bp|9-_(TMs^uzu=Cm^JKrN8xKpgewKaK4MS9z(FwFLogJ!zCcQJdv_q zg!*8`MO$CKqNUUYVCl?w*NLH75s28?cndkBb)@$;qZCf z;;pwj0!&Oa>(C5J>UBN#DdK8n`kHcw>sGLT%lo${*%7<}(d#iv)SF9*^~aIwEi>JO zx+)FnQ-V>?tyvufT&JD(%IMb@TrL~I$G*?6N9Qlv^)_6N0gH}71(yZdL38+!$781A zyb^>v9Zz(*)ZP>;h7TC;SB1i}LLwcRfE_K9%T`mJ31_isru{H~w)OpOi_i3ozaiLH zx#n|b<95wUaxdZthL@!DXgw_#y$Gys1kJH1m*?v(5w1Np%&bH4>;6S~bA~qUF_p@{ z>#k3TON2@-o2}U&S1W~oOIdy4x862A0cbR(Y7CGRLFXd0`_q;j%Njrui<|KF%5tp3 z2Fx?oO&b5=<@0O{&7hqb>UWK%e-~Y~ll?Htq>bYBpn(rnmyM8t>DsIunG=1WV zf`Z21#1R|Bn?Qazq*h)+e*k&zK7vfl5R0{cR7HJwGI9){`pSDK7D_~ zL!>2FhSWc;qC!rw2XWC3O$MDdf%G8cX2$qQMY>HtaW2#Y>aL)g9YfdVP@H# ztqwknZz%ax^8Y|SHuj@Sm;dkXYD$}MQ|%a->wWWAjR|MN8-E%Yc&@hC1BYB97i!Fu z5~YdE6jS(FG@S2Px~-)MW&%yGw`67Z|@$iXFsl` z9NKoM|4TJi9Mgkq4OJS|ECC3nfJ1L60MBgdX8yPvwYD*079)E@G>I~S-H>$tC(z#3 zkjC5MiWB4k3b|WSQ80nKLeINp(ww!pQ;j?gq9n4N8zO6BoR%|QY$@fP|LP_~qqgCE zZ9H(mAT&?qZ4A_Vso58*{k2W~i|w@J_n$lDx1fK1U61@OC7e`ao@LtKktgfIgQ3$# zt3RgFA|%KAS7jAXqFLA0KflhmIEohxV7(Lw`}d;nUkVfMkyA+vCK#X0*7N*;tvFSo zRVVFldBLamlqhQZd}$xRVYD0lVE1uzs;~gZlH)o6cV_D05kVQ5SOTL-59@rwYvJRQ zW3j=dr$uoaLityOy>4!T15uXZcPf&dE#s?ZhD2ln8#}w$cdTUgZNfi^q#`4tfuyk` zBMWX4)u6+c_4Dm#tycn|p5^I_ju2a_&Q!u zju)sEXBBV8RRU579omAEcXrY&*?_cmjSQ)4iL5TtlVpo%;&_Hu0p`^Mq_X7AwiM5=r#ldzEAt)NBeVx@nESwtYY+8VQb2_Nc5I8Y))pW?g$<0=dcBKABoMjeiu-S_^Z(aQ#k<#fFXz>tpV% zPdxT^m}f|xw7|zzWfMwUpb>&|>-TgXf`1rYZo6!Omeoqhm*F)=vKnWs}=`<}VZ241k@96N9g!w(0Az{NWZEioVi8*ZTu z|1<@2qJn**2J=a+>zR-Ht=~@b?RoY0NC@GAA0CG)csg}KjD>;l&RE_$%8^Ls9QYml z{6`n3mcWN~H4YD~cGN%=B|w;R>h_kCI2=-pI}_ub41L>aB}tZ!rg?Mb>CW3n#)Kk)|>RZZ!9ENO?hb*3~FhD z;wqqgxM@?DB}$&c-I?UmW&6%8*Q4%(Q!q)|JOQ1eF3)e<=7QTw(qdkSQJg6nJ-`Jedeq`4 z%NBWO8*LCp#jH7bOSI7Gc};Q3s#x;Ddu$7Edk}zP<N3Y37R(_s>)e>$5@?Tf4%jb~%szZ19Q9bst1& z6DMeHeweT7Hwmr`^ELncc~C)xlHkgcs;l*2x5w*(uD1ZyC$3v_-!!VeVok$0_2UOQ zxKe-k!A(JpG!e8=G@EAqIZ55|@p;7&Eq0B;tYRAjiPHYyUZy zoENveDd16MY8fX)MWmA_rqRZ)v-9JCiu7-{U3QwyC+=78Vg#bL7eTsah`URbSC z1lsA3Yt6OBYyu!X^z^1%3HqGd5Lkcw@WtIv_vtL-3a7fab#OO&?*RFGQ%K+GsC0po zSy-ww6x~RGGfqaCOCf!#*BSF3UK4uGAICzfDmzyekoP1NZ3OI{XnY#v~kf-5AbkKl!4XDC6(Z*dIFI*84{KpX}dTsl15kZaMtaOq-%ho#7n@(VAtKWBUyrJM`y>;@%i7X#+iX3_E8nZo8RZ% zczeH($_u{mcmH>;@<*AmI=LI`6@)iUzon)(Yv3e6CZm~x`G%u{3G*Wp(Tz15oY1s0 z)VEQT-B(@Xp(zl>lT`g`*0A7BN%iV&UF^@he{z~O;RUxwEK`95=6Cj}P+XT(h{EJP zgU;1YN{arGTCt`9X>iOdFXUp2BsLe~h%ASjy5A$Fn|LB(i@h9+<)NGvLR0a8`R#e8 zgJ26lwqrHVpYs+i zI0rDVZJHUxzUOHaFU!^j+);G~q_7tJg!EvUN&e0|06DUfX3=N{3GCPR$L0r@Y~2sp zpdh;a{gIHB8%$q3iY$-|SII}5q0G=OuL%RO(;%$9HVD6|7(@(F?8MosI#8_B-4krE z|L2R`JfbV6EOsFA1^%uI>zHATRXY}Us%AO<6R4<9v-4<@Y8>vW80K=-T+aX8CNN=N z__(UH2CkD@&=sFjA(dD0i@3XA8!{spL-uV&y%SU=Ut6} zoC3Jt8`3U{OF2MUd+%##9FK3&S4KL0Yb2PG>wn|>5PE3bssFOWbfTQNe!@%e&|mdG zbT)BI(#H?Wv8>T0qCtQ{lGPnnxId*rpMKY+#vb8BgDfn!3oTVT>he5o5-~?JX15*> zVytHSZsJ%~a{5ZyKyuIocZ@&V)}d$%X?bJ@+354Q^lRxz0To}*mIz;< z3g#^XkI|U_RvQKUNsRH>*f__2hd{%Tx9#0ulsCkKb$wns@$vD?YFkku>{h$Nfo$hP z5v-2S%d>X937A>W%%dJtz z6F{7ee&_LU7*Ab>jHakSE@wXcml?5ir5HCgZhB5z4*4MBJECv$I(}1hXZO~-@B#l} zx`V%llH3ZnPN8V>=NCiDmXJ&wV_ppm4zBm|FC`jf4gGa)LdxVS{q7_`^4QSqri-7u zc%vvT?i9Br^3srES$~}jsBA>N4vL|QZdp5qeGIlf*_!qdo1Ut8%&*Q*k97rT2nV}S zgQ4D~mW5jvZ%IPz4h)k;$KL&ADu8^jDSmz-Z7!mLcY=Mr)5cS)_}u>MpDFhf-nF4D zZ8knG3CF;B%z_e6lsqxq3a_9(b|Wc*ZW=XP3f|U zAD%?J+5m68Nm#gawOljkVZR~Er6O%Jn{HvL5}5#d6a%;HujCuEc6D?P@-J*9B0?ti zFY43_1lAI6m;~ur&vs#HDHUzfJ(NWi%sA29s?|$fvcW?`A$H&E7wUaK0`E5kXHx$C zx_I66rQGe#R5s#&|7>r+wP`O9{;YT~Rrs_kKl^iaAh^XqbT%5n7{cj5ee zRzpBd?@YN{;5+k_zf}uY5FRcyt*b4QQ>r%BZQbV9@(ij4IOJ! z6CGL%^^s_yMEDOh9zO3AEFi>VezxX*+xmPGU>Ee%&iDNMGS;9tN!?AI0=F6anx`?d z-=*36w&sjxvm8=v+Z~(856>cG75R;}WB)DllFcyO!(^r%VT-@0cI>&BD2@Ks^2 zqOm;PHmdM@Z-ey>e^AZ|ldR}O0}QK1nkDwpC`{$ixB0a4FzwRQ9o%#}Qn0-Ndzusk zcya8@`$q3u@9*vx`GEb+qP}nw(YK3ZLF(p z+gP>Qsa7LxQ`>#={o(ls^2#guB=>z?=bZPkEaxs*CWt2bb%|EbZ~t^aEj-@W3oE-A zN9^~x@jFXzY;`sBp>chc$znhrS}kV2wK+xwDUU6!1)F~=PiPtv`C^x_eY+y`Zf37H zB}*P9Yr5$k-5+7c{{Uf!_1nF{G#pWXtvWZ zj@`ArVl-&D({u<1+^Q3W?ugH0IQk(ai-U(}2Cjf1+{lntFo@>1mRU?Ik)A_QqE zr|`%Z&+q*1iT8xBjA+U!279l-<;U?9vT1mAe9qq`RawBj(2nh!Wl>iWl0Aw7NRS4> zL-!Q2p^YkX)Op!k_1WCcR$#VxjGiJIUEOLEgKTPe;jSq_S+P=19xlyN7HpE1)CJIf zetZO}ZCKdY=$`o}cwQZM%qaUmJvP(P8GS=rNErkBMtMC?F<_AiOzdh;+qv*@wT%Ng zBasNr+-`8&E>=+9q-f9Blx|P1XCjcY+k&~sFbzp7V8Zxq#2Cz?8M?5q-xa*{#b%mMKmMyD zyvh|iIF2K!`!$4OeqCPI%i~a=G&KH;EC*dvs5@&ZLzs>^jkvzLDpMg%*8J{q+wb!+ z9MUvV+JIrxzYQs9-=Ygj1cX)-nyx5xIql(OHH>}>vd=aaM}l=f@=M>@bMO`SW={N( zg@JCll4ge00n&6B24Pf;il zUTaP|uhX99pW;0vYHD(#DOPEu(lH_=NoAE=*;15{g=UcI1@VBUoYO5l@#5sWk~Tz9 zISCD5I|E}OTn?|`*q&K-vX+CTc-K7X5~}XX@&iiA(i$dosmAeEv!ktcEv#!4j+hHv zpa_gr(xJNZ;YFl-Phi%{why`)0$6N%Tg-pP{W!0GhrJoeG2CcpZ}d`7Z`Fi#yV^;A z#oCCM1HGkk5(o!{c3*VG`uBMv_g~}p58fQ^NBIbDskmKkdf&Ej_y4xSgMT%b&}8)U z+_k5fyh3B(XIisgV-kb42aoGc-ZVF%TGit>nC5g-j3&8!Gc#s0m!Dg~qhl?x05m9$(<%D5Vyk3GPB?Wz|mf()h?$o+VUSbWp&q@(P%z~^i%q!G5Rzr z=uXTCizuS6y zD*yYKI>XKL*@G|Rs<9r4fp3xutE{Ajfq5~n{hI>hWnZ-B@jIY^eIPcSUI$kPZ%6?& zrZ=9Sp&tZ%n7@Dj&Vm1qfLwdl=BRV1B@EUZW(95;kpF$M*aHw7o!;fZ1qoQ{fNP(k znbN43S#3@m^zCYOyXEDI9K*N=`d&3VF#hvfb%Ic-{%2~}uXcJ069gT`Zb^@h|8X=Lv`t##m;Px!p=j>a;Z-d5^ z%@w1`adh^8z0~}Jq0hH+RCM1&@|@m3S{hWSPv!LypEt%IF-;7eA1@#<(S+AFYss9k z(BiW%(4g5|8VW)y@LX`*S$uZ;*I7!hj~6PR&cMKch_LS#*c_73+38EU`kmZFKXbyxNV-F9rI%# z$kpp?ec;rX^KuXBPA>JFnr6KE=PnJ7Hd6Z@ zhxDFhZ4TcY(&Oag6l~hM?>)(a!&^i*JwwNTzm);Hs~AFCGdg|W-^>$*W(f1&`LuhS zFnt>b9tbz~AWjYpOY`DclR3l|$2#}OtR?sc@NLS<51)hUA)|?g}3y+ z8s=gY5|rP~RWB}-5YC_snoX?@h0tgT5Sh((ec5K%FQOeS<-p%g*(z|Bh%TuzHY6UC z5LAGBgf^>h&Og(Q|dw80OUcL;7^ASn#ERwc);=p=&xt=J~PKB3v60nh(x6(SqrsqyXKvb-L zvHvu_h-Y|u8mhGoQ6KycdcOJ9H-X)E1>N^3*v>A%p%WALwqYP!^!=c3Y)z{~t#K4^Hz5wP+zm;x|f8Vm4oJt@AZb8OYPEN`3>c3ovTAj@=7h#_l z6N3|oUn@6o^KnQO*(RF8oKNN!=fFHPf#U#z~sZELl?LHo}MoNnLCN?%d zg5L&-k9I=H0}f~WpFX_%;=^7a!E`L5Hixah)9wGntWk;lO!5~5USI%J|Lza#=k4zQ z+h|`Ge?CR7wRL(eLyw+u=__i zS1Fur)~R!J$wU_A=#LTG_b)W@vXDk|D~? zzcIWzqJ9~8szIGd_JoB_z zK81(TBr6BXcWV*q!WONOT>XNQ=L>}xaCg=`*O)>hBO;)Ko+5+7k%OOx{!>$G^t-A2 zgra#(^)LI=Lcv1Z&3%1fSbR2@+kQOvRabSrXis9`cHYdNufcE9!NV@b!t~kk3nLtrn;b2B)PTG#*K*XG!pW7RB~ zH1O|C09oJ)B2nND19{8;2l(y$l+tg!_%>ZLe$`Nyk?Z~+aj(7SnK4F25l_Hp+vuie z3660A9o(akfJv{#A}l=nQJ(1jwrUGb;(LdMnTTx`1Oc%x_jd)AlS&cQvgeJ?T#lA- zgASy(jmp2!PmHPcNUf{o=8MPCWmc#q?Z)Q!|B{-#5hwP6eXmu`<8=vDe!B+x-0C~; zHcBI1jm!!U^uLwnN6`mQf?M)lk@EiN$g5%mfSF+#i|n@+wo=t=CAsao20A;n&i?EL z^O}2U+MQWHVv8!sH55@{Q#ee`%!)W3k zNKz2YyThYV+bAdAva-gsaQ>y2AsJUgF(dYZ`iiR^D;B=fdRVDcFIQ{&uUf{USgN75 zEu*{ohbdEAsm&d}`J)PIKu+ZM7fLUxUJC53Ur`a`d~U~s zUg0Z9g*wA=a8hv$95D9W0@G8PoX)`v06j1dKb0Wxf@Ul+f9H6^9Sv^u5DD6hzp+u1{KI2o?7|?P zxvd2v_#kRbh7sGDmnPmzm<>!_t}e4_QwyN!0ABd>x*s8c8Dy35sSHeQz7IRB2uS1; zg8yXjRe$_TvP1++AbVk_JnnvZF|y@V*R5HMU}hOvNJIn@`=a}GK?Q0N*`IOqa#}vh zjW)&kDZNvUp2FY)Q&FR0Q)!OPC z?766-9`(d#efO&SSrz=!LtMEWrjWQ;heiG6DhlIZGzke|j$2^V`tyQl%V*Nkz#Y6} zX}}Tx6cMh~iUkdMV))MVy}$ZAB}cBdKgz!7GcItL2i1wIrlnbq8wibo1%7bVh1KSI zE8byjUjKC-_`mwP*8jrv{B+qtM;JUP{8ns7tmq!?z-TjS`Zda=25WD}qPmN%E8O~% zaHDSb*T-Q$F!wTuAAJ&*c2ZL4_N0FopznN$=-l(0QGdl1+#DlxbK76q5}{VwTvUt# zDTuDmNxG)9cyDCAi$Xn9HS(t;4;GNwm0=TV3Oc$Ec(UG_yg5vj@qm$~^D(Xxhge{+ zbTCVrtk;t*&n%=2BSEsp^Q>h|BTB_#VJ&LrYqY1p8XNu%2ZdS_#<7S+3a?bm+i2i@ zp8mG;`uT-kD}HGJKO0KU_=k*wZ3a&EG1;&LWuA;^G|r9E;I%n>OIBD1L%d00XcLVc zth!{rrD~ETu4qXez1GP-N>_4SXg){0F1o$MkJxijRDK}GGJbM5DbxakIN4zdYx0|+ zSfWbU$P0!!qh%TDvIvbVR09@iG{rMY3JfXK1jU$+c(H;^=$1W`?y#O9lChj(dHMIK z)2*wLHeCImX`1rG6%J-;1T~-y%U@4}I?y&Ln$T-nL1O!eBrLDI`^yA*~Jv)~~X=`1*s#j{{^E z#WBD+zHKYPbjMr+pvlz^5(KE*9~gXpd;Q$sc6Ip_MY<6FtFA0Ky}C*P0w^|mg0#H0 zeGuEX{OI+$950xN;z%)|51^5SuSkNQU_YaSua$#;PG@u5%XPpSXfuY-dQbd!%hmBx zJ$uY1^Z|0G1N{ER_wTJIa)HnCZk94Xo=PwCy#@`XbL@G zWjN#lMVQ)RNfa81$8a^_ z)61=12K=0Y)Y%*1HxRM8g+~;9-FlpLM&!fUZWpM|+_dkY5H?TkeKs@eeL#6jO4Wxb z;Fz?}`dLx+m@o;&J~V5~srBWmFRpT}MXC6CA+;-Qk)0h(|IB^6dfh7Ll(|e+8I~pR z%rl|}aL9w~t>V=dG;LNWRKx3#LyA+ja2Dx&r4V421|2--);73Jg_?kNSW<}un{5`V(eeD8Y`!=K9 zP}$fR9krM*bK4zVI)U)u8jPZ;+$VGj8cH{T-sh7$w32UdjB`{|W#F?myZt1yK-mkU z{hFnC*6_KLdggmZ&tNPxzO39C? zm(kHk^OEW*{;5E=y{Y=ngs!sds;J#rNXuNf=hDk<2~~M`0cqd3VeCET(@fz@%Uj^LY;sqyPBGF# zWIPS=Bgpa<*%C`8YA6p2nz*7-DB7rSwaX>q=Oyf}1dok9Zt8sU=*Ni&pXs_%`KI-9 zt(l8j<$=c}SkB~H1Hf-bmk>MF-6BZXpaMj>it z-rA=!_X6g3jO%<6I0_9ojR!Rzj?6|2i{P#Hk`QH)0$o@Sp zUa1Mxq-Y|?)NS@1#9c$?XmjcbYD2`qm%*nU{WIr6yt3&*S=&crL$qjL+XutcI5jil zc9^c+NaxI9WXuk=w?eD8iOExQq5HLOgv9@)aKk*vwHZhm1CC1m)y z#2>&SqefC!{4FYCD9J>jBr)wtaQV)VcXJNfbbR3pkG&5QgU0E@_x$Ftm)>SjH7OWW z!7j|L+`akJa+zm*p&qMp#E;@bobt9DgYcS)r4HC63m#`0FgNn-s5J=-E0?1=tXiO0 z(E%uyb=&$y9(pOVa&yt%m=R9)tV03}66a)WVDW0P9eV8$?`&xgRDA_40r*vVp z2fh0NYV~1LOiChHGwi*bzd(XFFM&~Xj`j#|cf!8J>uN=`W0TC=gTYV8{V~>UMzZAK z51Fp6|Jv>4#kpwN?u%F+FyMe(7+d=)1m)<&{=U)FE;C-e+h+*hIRX9WU@0Y8-cDdn z3E6ETXTZ}Cak!iqvmr%zXcL3$=$u{aYF{9KciZLYq(Ekm5+)ZO7z?EjzgqNvFM#$< zVgAP)nuSG@?*z@_iUSwe^N0`jyS-+@44WP zW>$Sk9j-sI9c2d7_)jTEOI<)gr*^1cy%zyQFo@8lw%Ik z6}^Wj;lJ0;x5Z8Du1c^lCH->mpnJ#ZL8H)tTNqspm9qcQlLNUg+XQ9IWG0g~9Z%NQ zn&u-MX%7dm?G~e+Jxw#*LyKX9;5EZ$v4upHKY=WaN-c$>dN3m^T{xlS&E$%1PAjcc zj~n$Xk;?L`!9+_Xh2x@yCrvcLjl~(%qX~?bPcI+|7b)+J^5o+CUaO49Fj@jdY-1(v zu;~UJjV~c~#@Mv$Tfe-$QZ>B!tmb`x;7ll)XCEWn=MCKcAPfr&tHB{Qf6Yq&4_w3F z_o4XT5s98FboQ5m-}`LmxBq9t+KFuryFvFW+s8fgClzw=Eln-e+g$$#1iy1v92O%M z(fTa)#~ISa%W0Tiz4AO3;dgofN9S%?_47dDo6Izp#-_3zCR#1OlA)*%WNX&q_I6wd zg{t0!Qn`Bh&v;jg@r6rPz)gGbtMA9&V*Z|b{+ovZwXC-^rh8FrIf@8fEB{HJ)`-3h zW7G<J!j&{fFe{1hI;6myCNV*+tt!Wi+|kV$at`|E@IW$)w768iShP433i3GpjQx7V44 zi3tQqP$^uiSk#R4QLqA3NX~rgN2r3Rq%f!`y56IxJyJgPul_Pe&=2vzp#hV8+6hNuzDWGP5Mu zfhk!ewaG`J$+1uFhZD75F$P94&w^b=vWpe5zFhKi@@ByL*yX)7L5;91Ei5T%>EF4z zhn?iJjh=^4V~K>8Yda8w{#bmxOpj36YNKt54`ws%#Qt%d^pR^1b@rJ;o;fFrbm|222C$skag=4h8n}GuV9=ijcKkjOWOtQTE{>nH#TbGI{hM{k6BAA9Rx%xL zR9b!bAJim9`r>9;)z#fq131>|j;6^~6BL}}NE(7!c)1aOpTL%iJxJk)8JVa8X;X7@ zb4k-ko5QN0*(#AvX1*xwB&JmfY=}au6ZnN%UEK{zOBJY!$~S7jH%kK^sE}d09Z51^ z)0lbXlVjxuzXsypWs2`*cGNL5w8IYhtxRnLaJr{tg*|!9FnIAWliD>ZYB4ApOYeuB zXF$#qLxN>#GH^S4%zu9NVru2N>x3a`Yn^7d&IQnP4wW;1LY$Des|Ox@mR``^b@^(i z{r7ewZ+>OLXAWbYE0S5FfCNMSIufYajA{Bc-)E~A?W(PF^F-nQ+J(W%VZylhM8LKo z;W#}mhAl<*(#g8Jo*LP5DRNbw$1TWt=ZM>A{CHJmWw7W``T`P=p#uV?{7A9jAMziyDy&nKNnI&V@bY_rIv2yvv#j-W((T3In1|+&2YHGXfFaNr|+bZ|H8K-`# z?*EX#89{Pqw_d%y8d5+&Mh}2T)9Lmj{B^z#9%`~zIa;}3w&}!A$chZf#02BTQ2x7C z|NBp87@WjyVt|&Val;qIh`}gb;(|sUERDUl*sTiE!Ww@k3;*~#WlkF-nd5mc!Ppuj zt1=g31Bzr;Rg@)vnv16Qqin&hcO(@3S}A|*!Ke4=hE({)fz{G7`}YnFP0yI3xw{$(mw3<)nrBBuyJxNf0>$r<*OUeEy%tPZ z$eB{X`9N+ySFPf$m}cza&iT^4LaqcxTWMDsik7(d4u4n1a&S#1MOe z#Y2-Q#x8bNA;m%H{t0Kh`No2O(UpY;--ACeFV=2rE-DbW-Cy-P_B?;F2A*$Jncah+f$Av zjKl5wzBor>E;KqDKl*Vt&j+%Dh>0HDs=l$W_MSIzePm)TKBRPsI0_<uBEvEicy#xk?o267`@c(I67qV8U%2O!AqJ5PZs9s^!BF+@t zYFDFY2_8VGCfWS{ysZzysMeW@w{LSzwkUzbRe!0*aq3vh45QRQ6BwBZcebi}2xZWJ z-Gt+T8n#~og(b=apc{kSP2(PpPsK!Nik1qkT;(W%*x-gP8UIG7bh#>PR7$NzfYf2> z9JGllYoO$M&3|_ut8O?#hAg!Bz?4y_xcEvrdR0la)FA-p9&ABbd8{0lCWfs8#KHw6 z&ab|7^qPz|I&muy4YGQOgWti>-gN{C&#&WCVGOt|n7XKS&!XrSC&5z3=2wj_>o|A+Geka5I$NOC&kxRQ-M*mk6n25@zvKjhZds}9(AHBB z-(%fTJ&l$gNd`fyqeL@KX?Ffdn>p@8%zg=iOj}?xrVUrN{P2im*N?oaPu;6z5;0qL z@v9|4#l{Cdcejk+K&9&|a@cFY-u|EXkhENW?gk0U8VX}24y5JZ zM$AFkiYq8Sq>vb;g}u^0y1c^@<9ydR@Dyr-^219W_Z}XLx7N)~x$FwegLB4B@&J46 z(lEyFhrybhaiJ8b0W>avJ+w;OyR0=FUn7VR4o(j~Ldh+^#8%!>+L#WQ>&)W@i7U}! zE|8q2MI(G+T(bz2f5lKucw;4@>IuEI9@_B_3mi1d4_a(cnh7pP6_*++^?}J(D%U5r z*keP_3sR}}pHPKS=QJjcxT-@t23X{p@_TqKx4d!FvI`huVJIG_jeu%M786O{fG{Y& zTmdx^yYPiPs(Z=zPnB*UJIc_`+U%rzD-$_gBHv1w9f5dV( z-8aF1;~nMhbEAF?cqd5|^i1iTx=Iwj6OW>0Y17c?zQqyP@;0L?{yHwYNzIVpUhZ-3 z9>iLE>aBT5s-3RC-;au&qi2km1|?u?HJFiBsF(+Ad0{VIlt4X+%oiLEGY-uBd?M~I z(9Ap@vA_vBvKF3NeUf|0&Qe2~L9h8CJ7^Up8|^suV@E6j-36!QE>mNiq5(!$t#`Wt z3N7b8N|7dPu&~lbTo#Q1!25N*NjxsbeT9u3%bs5T6-7Kji(s&Zg zYy1sn36|s&CE7E;*E2dk{@n4LRvPs$9_x+%py~!u9r*IeB2$!(^d9R*la&BDET2VvPy6CPs=#ANkf{+gO;+^Q-{D(b!RfA zudnYfJ^}o(ao%L#FkNY`RIzB^DwQHSMFoq0@%#@InUrux{r2l?;SY1maJI3Iv!=;R zIdDR$l=ok!J@$+US*ZT;6@LG-GmkBIwTx9J@)ZOx+Q|f1GVMkiW(P#;1QiBC5 z17FIyy5%1^v5y%t8ovUer*Od4V_;4cj&uT%rOUKNZegP>0u&^az8w~s71xAXNNBAU z`qll#P0!9!rET2xKhG6nm5wi=su0U9$2|P{Ul*f}Z`IQ%%^c4*RmI@U5zIm}ly8bY z`o6_!WqF3FitutK-*qgtzR z#$IStt1VuWQ1y~<0ins^-PQRM2G2mtSD>=rSbIfsvrK)S8L+R`kI-e6Y&&n=g@U>bMbC|N@Qr(CK6hX z1y{iYAqXVm^om#iCJvp?S^1~jgyM8cs!B2yC29>qd=r@C0z=SZyMrc8=ZIztX_Cui zAi%O=CxQ}3P?XHm!>}jkxgHyjK_0WfrI1R2MC{gkBx?Z_+j}qQ9oo~_Z;Qx){6-$d zIuTDW@lq5mD=HAqR`qANPZ3oBb1iJwp{Feb2;YHVN%WWh3p`huz2|l1qmW5@7m6yK zcxX{5$ez#)@Ku~D!3|Yu777cSpyeyoe1y&s%-$|c+^tR{1y2r7oXxB5fVsl5NOg~ zdohc&8*%+EIHLmJl0p1dJ5}dPUC~!tq!pEN#%<)E^_+{h8#(tzkAb>uuOfn9ug&W< zcy}VmSMx#@S1=y{|N3IZgKzDxW zXkde(P+4BAsf=1B#EM0+AEz?e&(s#2(~AW-BWU!dhjOc?kHL+$u)_4*nLC8i0~>7U zdRvq>(pWrFs0Ly2KH|uxBnRk=y39G_Y2c)=8mXe!JNc$K4^4W4!F66RtVrZ?#?h$5 zQsF+5H}>TBZ2?7kl#E7*6>MFeSLO)JONP zl96w9lcHs%TI#8b;u%2&oVl2xU6)0DParJpMmCZV|2+iI_#F-hn%tw-6<|$vsrRq# z)l(B^4j)$2KH*|YrGx2D+k}s1f4(!R-g#+vpmK@VZu5%$1RXDP6oRTNMj%`=8j7MT zlkCJ@$=rT-IxmClUKOF4SbOOoS$>A z&=sh*i=c3FILoY1k<(>(YF%No@DYKrQNK3Ph7p0^o-OfBft?GfF%R-sCnTaGVkFIk zaFlzn$$v~7)txs*poDbM%w=+mn;Rh7jLFY?#LXaow;pUQ)AoHV(HT-Bffz_j_+my; zVtVMah?*h5pp%n?mei_nvZsRkiM|h^UVdY=HbiR-J*cE7W;_xoNis-rpjV&~|-Wk+`X zYV301N|6%rIMCvy!Hlj&GL(>IJg{6%RF4%ww_DMX< zt*lsBc;G^YDK_Ns_uqLF@cLCmGA(%oXOz(OPf}}gys89BwT6=4Tc__`)ROhW1l2_I zjcHQJRK2)YuyeL`IotVD(j_w28Y)=d8J0wGUcLwTI_NfSE^L>17dOjSyIsh=yg@$w z8S;B*Cj$l>e%2)%rg#U5vfoQoA~}OCkb?e1R=^p<7?}7^fIYBVnhvObPo)*tn||3a z3N{7j_LP{XD@aJLi7So(Q=s521v_O}Oc>Fe_m#>E7+P9Oc;~R0>qT7}w<>D|j8xo^ z_euz7a;a(-X>RzLm{Rfy*gK z%K(ZdCCZCA$zgUD#tp)XmJ}WQiF~ad(Zq?hP z!7$hc!h6fJLIgRpRd3sSQN%VKk?SRidN3tw1iBe2rhk}r#oWh9u<%TDtE<_mWXr>u z*{#8;j$1w$MBICFw(^4!w;9;~PG@9>E=slq8&jkauj)59=K4nQ;ao}2;;QeM4M%Dc zN;Ch1Ww+i2rG z93hgzDZ?(K9vS<-b-w`gr8XW~bI~J}UeFcbwfsHrkf0XM+f*|$F^-9V==6g{Zp)Tz zjck?RpFjuuOW>Xe+UcKK_$J4si<6Yc!f3uz?j_pw*k1e82dv-P!}y0#V4< zafyI<`;)`ny}65|RL();H;&{CY<{%UHY6o>-7amJ8aE-j<&v-d&o1 z7kIlO7w{aoK5Fkv2ET9)V>M*{=Re@{LAlb^oH#P8{~pD&StJSq@ms5RquU|Hh(RRt z+qLx*pgr%1J~GkecAC4{KH_k0@%3q#y6^9u*=T%yFyV?}d#XLN@%ZKE?wHTn$q+$M z-E{YsKR!S6#wZveEVIvpe-N6nNA~R>9TqlE&>4sIOpa+uzXUva(J089lMpwj**t60+r;I))$Fd3YX-OTIw>n|l8gE2nn(W!xg+2H&euBL&&ylFSNpNP zTX7xDb0|y<@0k&+&tso%e8%X~8Gm6JP2ojvDA5g`syzKK#l6GL?$$QU3{3$ZaS6<< z(Uv1i6y?aX3l0`(jv#`d^+o$)Q{{GGqek+k1r~VO8mn*Y3nprxO#~kZgz>kW~5bpZz@cTyo z$Bx=K|4aTZ=KXy8uCJM|l&K>$`e3No_kLj4V_yI@;PR}w!p7PcGi;d!j|Q6(Qo{O* z5v^r6x4*C-Kw3S(XPedJx=7{JcZfygdrV5S)*QKeSnROc5CTr*OsxEc%(29UgnUOp zx=Zb#na7dzsg`VLon24mtK_V7mS$ix-i2V$zaLe)ijnq+Ayf6AutxG7>umhFmtym6UC3RVZ*hViNjmE`i z1->Ybft@J6r|90PM~GSp@wYSi|E9_mv!4#YgfA{-+7KnQW1iWeM}g66YwHF{x#I1c zsb=a9+gq?w{D4SNjHtvsbF-wbe4RlYzztqE9$taHLOB9bLO@xEKdH(@CN>)s^wQL8 zfzx_iMy;{b`97iK(nzKg@J>AsFbNC{|9Nd)4=T%f-QRd3A|i05*DkCb0@AX#Zm1S^ zMY(D=2f)Q_zILCiFEB69k-|^!4#xcv^#RY53dH_MwCgoStuf<@-K#ye=*R*O6y-{@ zWL#fJ?XV(cFm3Y2#8-0*Krzo4)q`24ms#bntbdwrSpbgk5UDNBT@F0 z0n0JBvVvY5?ukbi0TD3-&b*j>PUG7j4#n;&S`pPWlQieHNjPZT;=$MLM zIkFW)D_EYag>G$w-p00vo@V>=_s0@%cc#7qbntY;1k~KB%IfLxJWKDp{+uz%;t=2z z@UqHgS*};>SM$#s{x;gN(79jKT2V^gKxJ9>Z)CCw7E9r|=mTX3T;Ce+v*!oNV2^~p z-1VgeKYUm2OV#?n7odMYe=CTbd@|6+Zqw=RuabJ|NGHIPg; zi=ShG1J-9M#pJaZBsbIBmTdQ(pf=E1Jb_W}R=y`q2)JbNML}ZJgxe9X{c>9T^>Koq zo%}$WZ_t?cLM>#QSQ&H$dNaWxkD0yb7(RncG(ELCCwCN z1JT91f|7m&27gX)cbh+f?O^qC|7Uic<$RlF9kYdwZG-)z*Mqqn7+y?t7i)Z`YT> z1SW%mWgv3$y`w)?uLNMIkCFP;q=>Hg73#B*o>X?i%kQPY1j*^Za!*0 zS5n)nv-b9)J$yxVm`&j=gX_jH6r>jO>?=}nMMzHK4B{BpP;E61OW6h`MstM8)8d>| z>AO5!mRA07kF3jnof{V@%R0qqGNOS${&GUVLIvPM-|9+b$8QI85j(~8#8&Z>;tRr7tKX|;# z6877Czf1FO^B!)blFKrWU0`AMicR{n32RzvV)Yo#H&69jNi9~jHUcZp>KGVP%FqVc zwq_O(kZDCfh^HJ(Z$9!wBmD}4lV>o#T;5JM`mUc`y;mM$QQ?sy7zlxV0zz>T$B za5t8@ta%jM_s`?_JJQ6HoY$&-7`E5zY%(6&+_c4h?p58>1AG-ADQ`w9BI~%j2DHH_ zlo%CHV4eV%cBQ;&Mb&+cIw7OFT~y&A3y;vw-E8vwIX1Vg#`$~OO56+m*2J|)Sl=_P z1m*Y|3yqD;4f(L`EOk>#XwiyBs~Z0!3;=VD_YA?#Rj_-r)wz=WE2)eIDO>AZE@t5D zD~%O#y~$xgY!L~zsf9&&@;qi*QwAAxgKV~#l39u* zZG6qpI70^q6!cngsX*S;vLWFYY3KddEqDI0MNb`j+$&hunaV$Ma~b5_4w;9ksU1Zo zErMhHPuRpeHtyJDVpfT}aaAX2q^1r51oi?{Ie&B~nLlr&Wm3|f*=TmVI9nER3}e7H zXyI3P;ql}n8~|SijVQPQoK9&RYapc|Djf1`wKG+~@Mo0EeUqvSyljd|?@3{1 zoPg5k9_TL@6I!euAttJ+jZGA%J=+T`%-weNEWX`^v3m&pE5qd9*idkB0XWSpgHM!g zH)}~PHU991d>P!TNy3K@%xYpdsY~`8EQk8OnK$=&q}Ty5a{&DG=C!`R`?Jx{r;x~; z>4Y4J_*9%-tNI7=T_RU=`5QK=jNqjPW!A6Bys^Oy4aEzklvNDn6-cRkJcvrTQ$i0- zO`WD(`R`RI#{CV7m2UBz_Q&*5c?teH}R;j2Wcl~0j?5>}rpN$(S&HIg-Xs;rKEL55b(1V#ulq{w#Be7I zqN_zDzBJNGUZtJHsm)loWt8EwOkNvZEgY5bhbR&bjf5ik0>(md=SE~ca*CP|)2JZ# z8}?2xcz;i#$OZ6^Wl%Cj-&@M2Q;^@aY{1clSs3)DV7n$!w7k5uj1u5h_$^w=U;2i) zQ(ZiHqsSS%-5i+qd2VYzg*bwfwsj7x)B#1!FSr6{yWDYBh?N&Sp+u*RZ+pI zc~ZGD^>ER%*nhajFb3%PGZx8zq||WBx+ASv?>3_D+Gqd}KSb<9AV=N3%@BEv{RI z65HA~%AF>Oiv5A|MxRTy-0%+F97ENR-rLdLZ~5k^!O=u8iPs(18`c)y3}rg+Y%*gb!`PI5d;w z;lAX!8N8mFHc{D~*d3gulw@1Ze-*>PTxm6rYMtJ37a?;2m!Fa@5CMQ+JWwx^o96Li zyb9~9<)Tp4tihRJHlJZU2?SX|#As zH8obX6tPfIxa+cuu^!BS^mHjmla#1q(+7xK#3OegZbV?47EpeYC~W-v`NL8_<$nNC zL9V_>jh0GPR?Tnt-Fwg3{l`A%-urv?Ug?$ms#KC{YdzI_zu^w&o^$rz-@W&D$L^d# zW4A~xJ}>E7AcpStQf%Kmr|fCjJqc~7@xDNu;>9Nb#$Zm3)@3IzenqYEc>=sbjFhD< z$v6X})6qpry2q|ZtoGxs35rLR>9pqb=`#sh%?%(2!SKy)G|4MCeCQB6#s>*es3`~v zZ4AU}JA3x-r7Rp)lC#zq&F1zn)WbTzuaRstr0;eft!i&jY~nC4C(*1}UMaFh9jtxc zU1C$t5lN0y&fNEf?+niuYv9&Zuo0n27NQrdbAyWIEaY{g5MtZGO)92@yFD!$3@A84 ztnt1=q(BkMYK(Ierc*~*j*^yJQ$5)g<7SAs^NZ0z2cG^J%z71IRDW&TVzq$<#7G$& zRaFtEjd|f7R>seLox}L@6iI~n*~7~0FjZA`t>hpnrlNx6>DhQE&(~QdatHxmxa2Vk zMLFvFo~kNkG44uJoTl>%EH^bH9~n7=q*n`%xPW)ULL3nr@q~ajG+GoEXs%_|L!jJKX>8 zf0v`j0ogJf5njznjItCJi0gNbUq?1;LA^vYw)AsXCH3D1i_eF^^72aGC!^30JVJo34RdHlr{^JOIhG;M2rOVzx#ef#$Dt>5~sRHMq` zu+YWC&6#xxkw_3?^0tbiAgY!AkjkWpk=)omcgaEN>TaW$GS{#~n<(MRvZM`>wrL^S z-wsnu#FPafRv(#@?b@7(;~lYSi6Nv$>;kFJs1PZO!fJ9;a-5^+AW2DZvn0baUu%pJ zpFC1uc%})GRV;Ri2xV2#BxTyMU!{&ZMd5KIj%F;D?OCK|0yv?z*{DGzOIDWsfzz?%@~gm=ID-Q4?zZ>MP@wF|TwXhX}t{l)*A zy$23Zm0Q^i*2L;yb1z#A5y0l8CfK;}9zt`Tqr8knyZ^0udA$eu5>cGzk+u7Kcb}a{ zYA%`g1(V4HjfzX*tQetf+76A<94fiQvOx^i;WT9ltKx8+K7EEKo_>6-#h&LQqE~*%P(~ z<}Pi-8ntV@Zn0la={>DlWKUFZnp`z?M8-z^UzCIpDP4(>`l-hL?o>;IX|cGE20M%M z9*HaIT#D0dn2an_JtQ2)IhvSr5Ij-6#TsM8iNhtx+zCh`5HQxP9v0TYej#e(hNky4 zcFpWFoNs^iU3y4x_h#mR!?SzWPWElzg%e@bHMH8&HZA+F+RbPpL|wtyng%f>Y5SjU zA*!KFYuD+yC+?H71`xA|SdZI~GKf*ar4FpUh_asATwlRH-Gxo8Y?nPRm6!kPnKB3W0#mWco&$1vx|+ZfNyK_SSzuB|ShFKm&g|&H85(HWWFX&2y6Hibc9rI4E zDA@HxL@0bg2yrIi$*A}en~b;O|Ln=NO@mWSJzfh!lUZk+m6x<_f?AWoQ+SH9WZDEH z0}-K)N{nXcl~STIA<)vaEmae{cuVUXZGcH`{vDu=#+Mv4B%@9!3rltg!TO4}u_a&z z!z4jSQH9mUvYbm-u)4BBF)k>)WkxSe`z%CN<6^E#M_G)RhCmZEm4r2U@GK!o6{Ru{ zJE%d*-V>Z6rFr~Kw3cHop+MT&{S@Bg%Mr`V%hXMaxRTsd+xx;4q$*8qnev(tfv64M zdBh1JTKm2X*8|oVo?Lt0-PM}kQ|uFPuCOjjOJ^y3!PM6T34{>YxqE~!rnK=4NR4F- ztA%!?vDc)g+W;im&-1Kf5r)*{U6?||S{bgcuClbW#M!0gE|K!(E@q%^#)I9A7Px5=1egP=HF?*>yL45!2rBKT+_K8msT7#w=;GTQl$X&0!fg?wb@MnMV2v0xrHA6*;$o~EN zx#pUyx%;l$Szcb|*Z<@FG)+SzFq+Jf z#+&ZAlN)Zhfv#t+it_A9q$glj$ui&wU@OyXd;8oXO z#~pXvK@$R>{NyKj=;23jzM?2g_Uzfiw|(2U;l1Ojr=H@^|Lo6r_St8X@-aU`DO8C=1*_-0Gb_*cWG{E+YSQQB zMDd+TP{yWz9naZm&GD60?zrO)GuVrwD4>gKbTQK0u5RDH zUA*T#?>25&%!rvR%rhAmc!}J5@0$s2izaWQEK3#_7ilGcgIiyB3)kOtEp3Q65yo?4 ziVC7M?A^bUxrNto!%a7I&u-`5Im#;HRk-@9eJp&-Te^i4fnB?H5xrw^&o1t~`>$~8 z%Ct+}AB{&$CKG}-6qVx}zvW)bf)?=JlHG6?)1Tiy$6M~b%RFpVoOkTqvkNH-q6n{h z?W+ZP`oTD67s0&1m z?A*1;U3cF_y;^tAR%Ock!W^+F7>`R!ILq79(3DRxy&k$xCaa1W!&t4U6brodo8QdM z*I&okrH1HtqZrx$t{bms-@bjQ6sV32+E#mBJz2XFoc1SI+bko|n#_wQQV=LmvHWRC zjoVX8dV&Umb6j=PO?>-ze>czGaZe(tNl^3^dD)4mbowVs=yE+mm-C@Gg%FjX!VNdx z$brMxq!6PJq_7`aL|SrB^Ghikx2s#L-v;l8HZWbOS)AL!u7zF9&(9kyRpfY8#1#c+ z&o1%Y>EkT3gfAR;Nkrj_x&%_WhD<7dUi~riUS6oFGyn{a0@f|((Hy4|qO&;fP8s2& zL&D{cFU>-EvxBse8r^s(d&ZyV`=$(bdLAmL$-<L+K#9kWqzj`l{ zj86uc+if9?{UQtFMXtZ@+O7kY#hhV0H|F5n0bE`O1fp9-V!?Q95PTv_>lN(U$Rus9POmT##S(3S=;|)vb($x% zDrG7mb>i8(Zy)pXhODV#p(}9)ElV6YDwj?+NOH<7P4uNXi4-ExvRp57`t&JoxaCG} zz4cc1?Ab*d4OR;pt(2iCdFY{scK`az%vIM7Z%{Nv{2U_~A$R`0xBTKK8LMAtjwI z4i)_H|NIYl`#ayt+_tSdZ?->TjHy0Za7Dq~+#I9Ph`p2jUA=LXdh1SnMJ^k0T@oTh z*K~XZwUkMN%nF->Z5iw`o{6ehFoRZaT@47E4%?E-335#e#9;@hg!BfA25( z+ykHK(&d}FwF#dP$RL5XZ2MlzsrPbb`k8T$ou~WadNLfCEz!fhwJv%&ckNSxJ0B6o zMuTx4-&VJ>q{3jDt&~~5)uC7Q!lbucb4Ky--MG&0>)cS6O)Z4Bu*8}umK_`|45$hk zbkMzK))SsNw)86?M%R;1MD1W`=Nwg8;JqMDQ_|4jyMx2dZXCAIvhP3qnH|Y=jRS8w z;cHm)bj7)rsK;qq*uCeq{I@^+89wrxA7*Yc=8gBopwLL6!Lqf**yOxgw;U0*M6@=ft5n1bI8BWtUKugi zSDHjTnSNI__GwwM=yPa;B#KrDZ6vqA@Zu?@z>C8vNU-w2OKqyI4-qG;wa>xIuca~q z=P8SlzxplT!s}l5Iu0Dzj~7P_(ModCasKkNf64EE;tzO6j&yZ|-C`ahMlYh{x^V;Q z%SB!a*LQixu(w81+()uCOy7)N`Wo5Tu_kSPRCEpPpG7ZCsVIgB<(km-uIXIa$~j}y zi{?c;=I1Aj$0H{5bL`l;$arpqbE&OfTGZMtZrcWjYiu*Yjtphba3@1|+}uaU6B&d^1yP}pf;YeUExhl2@8vbG zxi$3!1%t>;A>aSe`+4vSpWzvfbg)?;iy2zY7q%g7iRgxZybR>|?q{p-_vJQz=M(8{ zMCsSK&0xVgwas{hn?G(Yv&n{N*BC)BJp)Z2dk4|V>Iz+8+O9B~D`?t=#~*)+rViGf zQXOc4s@2`87Ux8|XaC*!oU<`uW)iX>$GX!dvne3{io7Ic5Vu#+HON*^JYkfhsnL9* z<|UODn&%rk)9+Gi*!8hd*N_vWYpl?`z!`$pfUG#&vPWBoQ;);xrbK0Ea58ITPI^qiAj-VoFIJ5;Uz<~dbv8^WUJ`Pe4Tb8%%IczE4J zg<073oTFsr)akwcnCBF*4}RI;Xyj%QZDJJaj;Fb8R^Akb2}Ai~85-{48;51whY-*h ztwkV8)3k{5lx1P%C~L24aJ=D*I#5h~Wcd7m(ZGkMOH2YJmiXps#&b_I-u)ye>nFJJ zHGBDk-~V&|*+2gvcfaYcz&)OYY7g3u07J53Yz@j&3n7SD0|n7805x;=+|VH>BMMfk zCraX}hM=t_s~|WUV+sTCmw%$-6?q{N5w-ZbN=vSOh>14WA_pGlzoEm0e+66Ml{12>Gp=BR52`UvE*o1!cz%YtA0#b4l~AN?@nxw%9#9#S)4Ks+Z- zoaD&yBk<;QZ<;#X!FT}-L!z4am(7!n7+Hsk z(6%jg{o-WG=1xkt-g+z7z3MRAw{K^%Fyg)6xr=Xj%UyivLw`VB*DNnDFP&$;T4=)Zg8*-Pl97a)RirA%E_1tKE*nR3!%K8d6 z)x$XvJY63S0Mxh0T)^)To_XeJPK1+)I##MxP^(+BdvdTJ^pL`cl$5{#DH__-Tj?ET zuXO-Pr9obEmxG_nTh1RRUYiiK-tTOBh|iw}u+DqB&bhuGSpx%|`;6ARzUP4-&kF&Z zJ7&7!`{|VgT^-oJ4@td@ZOSF@Z;)oNS;=fmmVIn0L%d^}I2x^|>!70*{rU2_Wah7p ze%5hA}=Eus8b#Mjtr)M4yn+w!J(#uq6#q@ z!BLuNm-On_SFpK97($UUuks!hU5!p!{2riG#y3~ULC4DWM15$L<~$irOQr&6C=ZD>M2E0TEA8GbjcBG@3#xHD7Xe=bV{^@lDSAz%fJ}}AeSzz4;&*c%gd)| zbWC89?y>10)sOtE@UKSP$Hm?x^qL$}&qwHkR+&t}bAP#=Si z5@w+F?3G;pq<>fBg^4hrI)a(?DV>soy6#70478|T+LEsI0qHKIX^?!B z5;%g7sDO7xqDxDCHABEx6|u!g``7FNIyk2D&e|@;zLEBZ#`P7sBA1aU*5yTj+wZu; zTFR+K1;u;Xx=kH#+@_2~RcJ$?C<|g@z{-0{+ghFN9e3QxbvIl^WnDEJEEggwvq_ltuHoE!+`(*AOJ~3 zK~(RRjpOAhJ)@Y2>y2woF<7vM8VtD7>6e{#X)7oSXq(0i<4YW_-p|}_$g)b6eyu~qwEyFsnT4vin zZT0wRY8IeiS+(K^K*wxW?J%jRD?L>hK70{JNCoK`h6VaoR@x0%`a(X_;E(jXpcGDM zng$4n-EJ<4yY+ zEu01&+4rSQu+?<%lj%(z8d}6#%-Q986=P*`UTqB2?#Usxp~isIM3<+wv8Kt=ZNgXN zWhmXeiHKwl`b3SHA_kJ?4}x{iN>r&Dtd?7_oZ9r2ODY?rRAsP9W?$;|gyBY!#dQi` zR`0C`G_Iotmov&`ag4lXl8$0M@RhcW8Tmq$oD#U>z@sgV-fLVrF zDfKm%bg_)2n4N9H{rA#m=*LX9ym&j;f%ekHG_&_2bO|on#Xa+VXr~0aY~C&$!R$D8 z3i*5q{5tq>I$p=m91|p*QJ6rN&9`A$&ADgEi5lXBgpl@dFG^V!jH;65How-x)obDL znaWVlgy!xbu$J(@0A#ttA+B7pO=`MvgRu3}F*M@=t63&KLzf+;3x< zkSxj98Ph%@$tX%gI!I^AJ);d6yWH3B6?tJYoZoR^OeT84TuzE6{)J?4rn>ZQf~XpC z;96C131nl4Sf$fX$(aR6{VJ2?>CKyxC`htKy9VbmfOAOUk=7PW$0a^U&gj?1MbNO4 zH)q>+imIShvyPqf-6njwFV9}F*bQ|a2-YXmIotGNLJV@oo%4G>GAE>y%aM%&crU9y5#^K4>NilW4OZ^Zr4 zX!JzHnhd0Is;aVk-#%l<${xA(qIKOlXXc)q^Gp9MVq{v^U3Z~;zH9`97!0~}ZqrR( zVt%&WwWQUM%7F@BP!=U@lG9$(b!Pj)bu_1e!j1DMHoe-6A338KwiiS-#KT2v0VvLG zK!$Pd)!DO1T+3}^;UW^!5Nf>lI2AOgjX_uTL}$1R>w;fQD#>`@c;G z?G5Eo6xz^Ilm#L}8(aI^7uH{Ox{CLymur#Y z18SxHp$#bpq8(r8mWT?%2n~XY^+XMA3j$SP9eiSp#5SgD!`jn{qx7TBXlm<}WP^Ri zi8mu5I$YwzR#lp&p)88j_clXM35x1G!2Vxxj?rivPK1X)_Xv00^#;(^AmJEMH-NJ{ zcq`vr55;ybsh!Syu{ctGQ8N4+X__f@y=o--2pQNNdWJ4{lu?Jo5sopdVT!iS=o{5h zP+^H4I)iY`>^ogeudPiL2+|+~N&+JA8z25t7UuVG?_2Ms*m088x+XS}q9~E*aatKg zV&+CgI-+(bLebU`8ng{mWkta}$b>eOG*Kz=cpRn6NlfcWqWhE<@k{G|;)=Y)MGMf> zwX4&hB)yT0VsG8A+F&vHs2**Fx=})7Xi4D((X&{KxXYjpb{Gln0J7u>f)d(>mE{$R zs=|x2M&^*>W|4*#bxOV3P_Hy_9cZP)maYSgHl5^RSfQQ8o5=wzfyCl&W|5a=wP7Oj zv8>ih{t^;eDrfXI$Y1}oYxUF+3raRbhvEPfbpB^)) z1I!p-TRMKK#9*;tvWJ-9o)`y^IHp)dFa{S*HP&`<%)*?~8B*T+$#r)ZQV!51XLP!}4+)4kPM9oPaPO8<0J`-H5uHCMnY&Ub6ZH<&ptuK2GNNo z2st|2Oi^MJapF687!63Ot?}NQa#6(<9*=oDMCrZ{eS-!I8(3nSIe1$g*bp=dz9>*a zf?#t(3&fz^h8|*T3d9wO7A;+&aBbAsPy@7n#Z|0+MvXG6&L3TQz6uX6QeR5UQvICYZjnrc3ufC@;m?K z0q*|~zs1p`5EFNVIM8|zKl)=o#yj414?A{0k$}8_%haeO8>H*t6bY<=uMoG$!p_4y z^~DJf{K;ed(VsrfQXQ#ABMNayYtKHz%R?^OPQfNP{ffLCrRN_O^__(krB}M1DN*VA z1#6m|MN#0@arp3I4j*_GtFZtSx0a>zBFnl8HdoF-?`w9}a_WkcVK>Y6Pq0{ATCgY0HXv}C-bwPi{Hpk<~k0W2EcvUhYpT+WknGi+b1C_?G&NOi{yOiOTV6EfOtge+ zaR_>09_;X3(tE-T?(xKl6FmIzBb=y@5OGB5TfgN~_@rP}g%C;#SFy6(@X$jKao6p; z@I}#m=X?n6dZID%u6Mnix7__^>N=u@BdSu@4gdPze1g6E_fu7yZ-MBYjgyFD@4mh4 zecM}+uxihZA|Wl{V=mvx&|nYnxj4>OcSXUzY$cJUUZQxMA1rheCe^2&`r27mR+d@Z zzR1GDP7duq$o~ENttVp2n$@U0@bN#g+Fuch!5~LbNIghjchehiA_;aDqQcj{_BDKs z7VQeZJ3~;31ZtdvQS?BOjJ8yw<5CI1dJB4Hp3i!-T{=Q`8?&dW1*!!aJ<}C~$0FH- z?Vo|~uFs5sK!jLuVBcYGy73kcAG)gBpq}5BoxQdwEjxGY;K24ns76}n2pCGT|LVV_ zoSzcpOg=k(YElM^`x{pvNX{4VNKrD`aTu?SkC;#n!IaLX-Y?S z;ZU{s<79501G{f}0SCV{A_7&)=#HT!G&S{!K;g%{`kL2p%~glF?Y7r&^UXKo#Ju(p zBcW}1_R;5P+7|CTMOhL;pbY_SQykMf-uKSGflp;G&IwHjeCbPH=JStyo=5mRkDUBG z6WzvDJ8z;5Rwg4|Qj68zZnAo0lQ$e`-7?~YBS)U*^r^GV@7d151N%9A_$qu6QwDfo zb#;Z4C(dyC)H3t)bA(jJ69o!iLbT%as(@>*x|ZF$_Ay;u#kq=;Cr-1v+;VK`6tRKI zjXALAdb@-U&g@~SzFkxx6lXx8epYc*plyOvhl+Jr7O+kN)u7?}u$I?Q7nfq?3R*EL z7uYtxhy8mGqEy6TW@TDBt}DAz#J6Y=yem1h|61<8>rLEn{S9U?3;?5bjoMt(+u{GF zV4J#y)s`4xD#2(C6?W`X+}tS|IYF*N6wMo%pt<>bcfU%=9I&PnVtP0_+ptLmM9m>0%F9q zTe@34udJ(JUE=!bI^#fZ$R`LaMrGHbz3kexi@EU_QKNMWttHebj^~~`&eO-9W(E<~ z#5F3Ch?(il?ya^&3sb6+Hb0^7M=K~-U3C@re#2Y&@gMy0Ek5zqitE^YY_NXg!@tgR z$Byy6-}}9Iu>_?UQ%tt1C{$^k;{zY~K==6CHsDeQty9;*=e@$T-@WtPeA7+beA7+b z_n!CgYrpnu?A?EWcfaR7y&)tPCrxx%M89Ss*PE*K#0>c z6b5&{_O-92C@RY+PQ>yeLR|-nqM+~)IZF%?=c+WNoz+e9@IHfmeFs?nK8!B2)&fDu7dT0g5KI&k47(oZ^03nj0*PbT;QRlBgl@H z(>pvX)v8gxq2}T`p}wjn)Kl0FS}Ee`NDi#fKV`yZNKN}e?e}u44|F*Z2c2!I;8J~E zDiz8_>i|v)G`6<<651zPph0nAgb!niFisouWh@;*yF$vq^v>dol9);?L0cdMM3o{T z19_-Qr$9hrv_u$k{4`pEj%I+*cTs^hmy1Zch8!j!ZSKx*IYMVJ&^#_q%!_kpQJNHs zF~av;eqw3)N}vIsTf`+WWn^UXt$;{N)PO6k23unfU&dGxX;29W8J^5;UcOtpq(4!? z>)6KHH7GTaR4%2~+f_8h5WAA1e%$i^vo)S0zftWDeNmulP(27S)HoL@3Xd>GaI8(n zvQOrx(AWNw*TQhUKwqlW-Cu@Y(~kh^O`WMwO2xN)kqPT z-#$vU&RYRb?L4&~p`wTcR035@wW?AOi*$r0>4V+I*yw$IC!@)yBDu&@2*EK8Agpg!JG$WKY zZjE>{9q1@1RcINvwLPy`(o_qCVw=_VCu`VilhCS^JJkKJe;OT&-6rl}gLZW`xgw@3 zMXYJEd4Wn*I2I;j>Uv5$KO*i3oSiJwb<}DVDed-AMMgrD7-u^f)|3 z`nyxSIaa3K@n*QENgF}J|1}M6R!{j$TRVD2rQoO#gs4;0wHAj{bDXoEOX^&sskwv> zYUS`{cV`|27n34YhWg2vuVVMfSs5I!yF5v89wEdAaSCz3jhvrV9JVk&cZ6m#ln2PN z#lbrh#@3!B*2>PKUgvl-f2DN(Ve*e`_+fg*I#_(&#s#QjVKPB>1-|;#lSBd`#k`(+ z{HshxV;07XxB^-cGdHA2XpxPfSLi#rV@9JfigI@8EaUN*#l=Oc(U>!5&T#z1 zaU%=?3h!Cmy_?Z!WSQyHDPMc;Yu&LYlL?c_gsQ3tA#m*2Y3jC3`<7Bw6$?ABW_3F4 zvhKZ0xdHaDhc?`KrJqZ$AW>P@`I)U|9Zcqde#tmxe&0u>h>@ym3g8G7)X{ajpgoJt zu#xqrd%nHzJl%9`64_FSNz>Y4opTgL2?fke#`tX|XXhKMF%+e317&4hb(*FD=ZLB7 z!yF$+NEW3_@z9QVeYja=^7Cw+tT(86%B^#!v27`;%GhYyDFoY^NgaT4H;&6)`g&xW zFsim${Qh} zo6s^t29dn~&P(6y)ukJ6+Lody&^C}M8$z3M6G-cs$gQ{D%HF+u*|~Ek%S+2V@x&9HIDUfZbczCp4jtmA z+Ya&QqmL#c6QL|T?Q{wxI=|#`XSjYgA$q-i?lN}FdjC=Eesf@`3JrU-;m2y;qPFQXbM=L>>b>n(B6I*@USs0hSh9!JEo zViB=;X%nDKD;G8DI*%nKrN+}ZL1I4(^KTgq8`?I`38Ah*jEYW5A%=iRWLhq>Qk`X` zy!=^Ux;2#)g18hYTbm-rd{q`jf_+TU7G;q(;|t+<5r~0TU4K0xM(f&PDa`o7GP3i% zZ03$Iwe@ffb3FbEL@C?>D=Llhdlx#R@k~w`vzrJvh67eGlf#6noPe${u@+P7QIZfs2O(}+lA&|Q z9d}r4PkN;~Jn!a2x_Gi1z_J;gaNe_b?>^?{7cS%qw@y^eNNxiw(<#1ij3;9xQFG4e zD7AlVQ7;`rHJMC!?|a|FiF?n`)=C@loW{W}f>>)I(OhyHB_gU6beIPWZx z5IfFjG>3+8&ioS*icCW(f=jS*y>f+|5uE@odAfCn*#-Hv2 z2M_M&x@)gNM2WG*)}r=C;18EG?QKIOpvGLm&oAk`S@dK&RPi zQ#56*&Ujo3cn=|+Ty@}%`k8RpdMJv^4sPs|pj>pFXBVR)!Km^=%F2B)+82HOB4nZr zmspGp?!JKUUx4h~xsz?%wm~+kW@p1iF;H^6Otohd>iletL@d>3Q}>+bTp}8=?B39A zav?^_s!Va!%}9<{3OC+(V`9)^;DN^OQ=7iu1S@CLarj;DJO>UOVA?bnaKx<@FV0xA zqUYhye4g=Qjq^5!BdOTEXE*cPCX|KsZ!H~^=MX4>~kP>-QC^SyHwYOHstj{UF`Dd{Hu9Dj`(&vMu1XY|b2zVM1XkjGtct(_~ zsaK@8*lXeK2DDK|b)>4Q1cwCbx~446+UwefrA;RPo+qFtPeziL;GCn?T+7s}|02Q0 z(-=G0v}&tlHMYcNmCn9Fm*2u^mZYVQChGu2&oR#IoHKsBWqK1vL7`3 z``lmG&L3Ix`!{{#H&Tu#7;BJpujgcq7-JW!-AeJ!<9$Jljd^$&u|eSr%N`zh0p}Gl zp62}d7($9E=kquY9z2-ZxNTAf0chtv5+g)wvhI8$9nVnUCM0j{IrbN{XIpaKbKP~< zBL2n6bax0v@Lae1dX6v7@qzFAU(q>w#>FxG=>PI#yz`xJVOxEa(yvk`!v4q<5)U%v zh!}!3e6Jc*JyBA|ZQ~If-VeBkoku8%v2V=L#@f795umj25|I%5CJw4G{Uijs8qw6X zv3Dh<8=}(2NYk{0RGK!eYb)yls%qRbl{s%Q_7s|_SB%ued7r{f8O0Go8}LOzSyj|4 zMn|eK8ZvH)CgA)aq@q-1=KEf6uAIAmuc+4-E)^{$H9{&=REH}naFgycsqe{vp1$|& z`ck-Z)92jSgN5#CW}XS>tfyp@G|5{yed;t)dYbvOmKp3crRUpmG${2FtjJ1lgsQSy ztkNZ#IG_+H6l$zrvt*AYUCexXEnbn4$-jKv_=kK z9^UNyaL$ZstAV$^zeO?)yHFE0otvh{xdKU4m_=1u-E1&p<|lsgCs|ot=4XHQpYX{~ zeu9tx(I4}hzxn_0kAC9Ex$CYwIC$_N^>k{=IkX-B_R7i%AOHBrdH1{Doq7%nWm$IF zzimPbY`=-AOPDG8B7Y7#bS&>txd*8Zj9-$H?zC#&!80C|n?!6h_*Z00C8JE1ROo4RWf7)ucY~(kH!G?S_#I97s>QGz37(S4narG%zv`IQHLJ_3 zjHOBn+Ee-h1e(-$G={*+bd}}RW#$*>X;&JYIOgWes!&yqKldEl_HL)DN>eOF_TzgG z)+Mi`O$t^NeCVw{Cy}U15JjCbt{1p@d^4}#bNdzz<;9h3u%OD0#YH4GmiUw5)3pw( zeQTW;H^1>KwxRgcIxfdzhlkmEwQk3B-dQ4v))~o3v9$Go)bk(Byh+zxe?~|$BUDvY zOy)P+@Sz`5Yb=lwZ=9sAK23N03#n_*wVL@HUF@7ANqPVPAOJ~3K~y`P@kQHfMTo(u zHg}9mUUU7mtZZ8$lxAd~WrTBrXN5pxzstyaE#tB3SO~h3ILsBV&f3srln(5<#z^6P z_qSDH^yyn)Mc=+MZvVGfx{rmRdQeR!~ zfM^C08dzOfMq_}ChNN>ss@KP!N!v_~phbG%!W3#u#zR)Hf-4dN5gOwJ(8#XcX1Mh} z%i2&23`s!N4E|O>%~!Z&6i7jbh>+{5Wcsk-FsbK-FTM>FY$6U9m(mCN2F*ScELMwL zx9d8-Qhbr!>}IZ+Bev#ByVUATyywJ;lRW$E5k{k;vl1s0Q=BU+D~!iunx^5zi4%n9 zYOcQi5XX-jck!M*MnWHBD|E z6w(qyWYrsdw>a6$oA-YMZ@ucRFL>}Sy_l!8ckf>2=c>NHZD)4r@RKv~rRR?%oV%Dc zMtoJ_d;uC6mNl)Hu(6d=mL+p@6JlJo`c_fAbF^K?8VggfBDC>D^1>a3dIFA+IvVBt45CuFqT{^q&L(#b zNle>SKocXGQWl=V2~R!w1S`u+96Yd>_r3Q$eDMGLAlv5Wc=Ma?rYa3e_JyOZYYQ`p zQkC8^NwiIfnJG&FT+w^f&2&oT$4SYjtvA8Yz9>_6x8c#wTSjmUHAOilw1LA{Ut>FH zOz}P^-J0o8Ixfx{x*}`E_lh-dBRR7@mu06q``McCZ1@fmWH%}2?FVbevj1X)=nU=H zdte{;yy+g^ao;y{_dWM;=c-?qToVy)+%_l#AATe0+*_ z!%=ml=mAfYelrWQtVj0Um%zO4LVgKWd-biaCbmcVV_CgexhQNpZ z&Yn=&vr z+LSgpAYE%c=TVni`YCN_6LCcg7r-AcBCISgclD{+qjzbcoC5=H@H5Zac)>TL*ZOQc4nMSOsklsGAfMB11T1A(ISyy+X>#3w)XCoJj? zuDRxVcJA6srurl`bPu6Lq_8B-HfNJ~&YW4IY0QPm6pE^#IliARHAX}7hOZ5Mnql^D z=kr9s6&AP6Yd6LI+SpnSfsm4}E^Pf=QLryuy5LTc5GG+p1DbM8dtvvgClS%kAcC{M zZ7dro*gpD`e;Xpv3)|*d*fvi;&4%yh-yJ)4EwW?RBCosc&Y8n!s!vFD|wXzjjg@OoZNWs=iKaTFCj=KgC>nAiT~}#_@!U^B_8_hXZYdw|KQ9s&WS}Jv?1_E z4?MuWgNHbH_4S=X&t(t8IN?@`r2F;0FvU0U3`4Y7y2WHK0EX>)5nIO{=Nv^@5E|uy z2OeO$8u+2_`-x2)DH%1YEH5uxVhcV&=z_LwUmCLMAgpvoXJY4L9N~6NEQeol^DeebqzQrqIhY$auv}rT`q_71hwZ_l$C{- zY(jC>)rT03OX~sLM4T^hzU-cN&RjA@hi^qK)Rh_&6LulRcD^$Mjp=a^~8bu(?9*w{G)&LlT_6R!BZr~3YMWA z&NGFF=XmMtR+q1IChu_H=u;{NpJK2l3LEWmn=zr+SFpZ6cb~93R9kk~FtGOe4j(oh zoo1D0O_TE^_P)S5v)n>xpzG}iO$^|%{6@f3y-7fY1-B}U2S(yY+h|MLt zzf46Kq#>plr7z0vlNZ8nl)Ykc&X{^tSzTFW>=$Xo>iPzy2NH!O#EFzu|3feH*vjd>z}$ zRjXsqjW?w146#+~MuCekr{-shXtK5wWbYjB`|E#=ojZ5py;6)yRK>WSQAj+-9)~X{ z#HK|xQY0>EO(i$tJ(`Tr5EbVOVr!JG4#A5HO(vZ7^ zOeED~O24Ph%DLBaInI!mibNu9?k_t~oPLI%G6iP4W*h^ABCD08Ya}(+IG|sl~vu1qHRnNq+DLzmv~Dagy<94qsS92J1f@ ztjuoad-?a-vFN&ll_UhH?cciqh(&5mNiGG+kp@q<^^nZ`#dmHaFY$R-0|x zn``oB*JfLrZQI^#PPTd8J?A$uYo0E`;WEZKx{X zVe%~l!Kc-Z3ZM6$9o_UIi~nBs2>yiiY-gn_RzPlbVE=Xv@#yK>q9W3MAzOucBjr~h zvfd%u_l(MI^9^Q|n5U|Q5m{07XUjgOTqgIJ@AJgw!P&*D84^)vI1m}J-R}U*T^;V` zLzIuCr$`f~{C<1qV>19DMr;)^lbAk(Z0^K#kAR2O>n08|L5%-QbNe=UoYy`#3Gh+Dnqd&l(gaCn_^ z3Wy`bK6HJ&vDu2I=5{*I^(D9*a`NHa`iT`7n;aGJ+G%wMJQt-{x7B#)8FWA3KEHrA z3_TnMcu$Z^xo*@x+ptf2I)A@3wN#cM`;1I*(D{ONJ?Yj-#ytG(W19PYzUx^Ns?ZV_ z?xJ<0UCgwi#Rv~Yoh4Jwas^W-)<{3xY2&$UbR}s?o3d<9RW6ffKxD^vctpJ=fqZV3 zKl?|TwC(DS*H6^nZR;9`hFXg<>Z@S>1ucMI0s=q(baTHC{~gBN>7zMi(&3y3K`+u) zFU9AYHsWZav#3%=`&dYxWG&&0Mn&=WCr0dlVYbvrT6WC^>rf=kaa3vOwVB`IhMzLY zxFnnjqX_Syw=6}v6A9aiXB3<01+N9)P<+8JAB}fOh<=5I zo=7f#N(K{gw8(~{J9rhV*h(f_%cb(}3D4Ce*#((rzt#}thn=})f_XI^&r?&7tbWO0j!gjG!NG%?q{SBy|xln?%$ z81dM3&StHeTX{TNs|h%ZSp6-c^rX6VTvoNC|32nBl3HipeAc)+&MS99lKdY%9o^;n z=G!`;qabtt9nfJOVYqYgMAR8li%a=D&b9P{P;ygZ*-LA$?I||wj!~X*6T%V*vlv5K zuoQ-%1GtKSyoWXl!)tY8H5kNn}(YT*#z$=u3-#sWR5}b^J>VJbO!n1yRDM@MhL*$Dyp$X|a ztmMsXm>&v=00wgKmoY02?o<*xj&=aMw7R+*m-$7hhi`f5%8QP!rn>l-Rfi|13a2T4 zbL56HtNG`DUK+ly-|EbeuEUepQFhjUsXf+S^XO0^QpP#dw#Bp*6eMI3$wV;OYuRI{ zA}l;HTpU`)i2jx4;L=G}FR@BG87H<2)BSF-+@_^ZrRoV|4q=$xb2Meh`pYV&4m|Sa z1@0u^t9_%Eio5{P^)YBmW7ehEbDqoi6T7dNH42yJ1}lB~_Y z50=?CF~MfEHxU~kV9Y7vrFuM6ep5z@WKc{0d${#9AykUOC$AHDb7g%CjD4Kqz~MzKiFM)Fw$ z>P*9piHL9vhoUpR&YhZWV!J+|x}M9LYTa)NvGbpRyshkVt(J^r-48Mg41}V;sM6^~ z(wAoW<~D4Eei{v>H4RhuZ^4w9^HE9&9JfI?0*4)Py9QY~#_3@eeoe3(_av?>^vYbB z6m>aG>TqsDOT;LlW}T$SGGL3&_mv?O(3i3gdlI93zRc}DzUOHUK!qWDCEepHEz7rSM>d`tz5r`P>*X9ef0 zCK?ipm!kcMfwP83I^P_t+5Rd;`@O4{3IxQ5F`L{rJP)fR9YAixvuo!$8*eXd8-r+D z7be|>p%PKaTkDnpuNFJDX(@!3L)G|DU6zK|xyXI+B+uQcofL=CHA@2zjg}$o*UU2B za0j-O3Zgl3Ve>*S7%e}odcF}Zz)k1`7f z>wpCl%@`Hz>+9VL6xQ1DP_?p$oNsC`K~uh2l&)txNxn|-*^ht0Oj>`f%Q1p1^S^J% zmiB?;eU=f+q812Gcigm7RH@Q%ifm7`;!C9z`O$rTRX9P-<>~RO4*3$qd=o z#mzT@g-YE%jeG->9Vm$yg55sfm&G;EZj{YRJz-(AI+p$_RYCd}@3VIJ*8cE3s>2aa z1wF+j{|?WG*`xZAYy3@yFw#>{iL|s((={L1&F2C6saW184V8I?_4%dw;CSM-7f_`S zBH~d={*wI{F-_!82ZzIfC3a!g!&MgI-9UpBuhm(G) z#agq*xd3O_g}M|%bYA36X4F>}YsizLY`Y|;g2E+o4}!KGo~gzy?CK4#?%$T817jM+M_7MG175|Yv9qL_WdVc zNv9-JXL;rvBxXK7O?<5(X^BNmX+KSb=nq#BUsTdKq$%kIf0&PZ0y}Uq0DlMEH&e2v zX|jIizGyGJeR#;F7MX1Cb0_E*FPK%_)``o%n~a4HJhiFYjX@-muC_OjSBPo0!Z?{+ z@I2qU57^<&ICFj8@NAX;L+!&GF-6u(Det);srhE)-XuQ^G&-Q7WT|=mZ=IrQc6V=uc<* zK{-`bF`e%|;Uavia*Fn^kFe2%eBC=@5gM%~uny{TLE^zTw1r&2$FKKz-p_n?f5rKM zk>!Bq7y$>#%@eEz8_?Q1ObrR8%tjv3jI;@x<%U0U1D+b1-r5`S3g4yamd?GeRW7&? z3w|i+5}Ysp;#*RvJ|co2`3ZGs?vpTAhc&(;aZFkqk?*z9_FQP#;vZsL4z_$_q>$qI z;vQ#((r;6ArG?dEyoW{+M9Upq^DW4_Xu6YNX=W|~ANAf(ND+6K`Ds1f!M~MBAx91z z=LThM*04k6YsnL$q%_cj>WJ~6!8%|3LKnTZwuGgu_e(S2r_N7f4qe*_2JPOK*1dDN z-1qo>S8owdS;hRa8Z!@mQEE4>tSJCn-L2aR;;RJ9FtB(AZJXAUc641dDhUr`_L82# zJ}3bnEwgzJU%%wYFja8i@+v}24LP5wwLj>1@9&?k)FsnvHyT95)yyc~pRV_DUUa5U zq%)enwX(y?D*VW@7@vR~u8rIuOAeU)^*i*U=dW|Y)IGwNcX1Kg@_6S`^g5*}AsIpX zcBA@8%)kL&ce~wS#PL-23k1f$#~R;mL`Fs3EE&fDIdm=z3CTC+9c7sjiCzA& zs#b9&cT*l!w?X>K5(u`942Qii(`^!L$8Q%rzFjPikr(TZn!t&%JWv4J{l=kQAWc zog{Olv!%&~F$Z~6!1)W*2pPgPzdxUVmdEr)Co_4e#^R9Q_CL1R1%1$r(TsK6=P`6Z z+Z6pE)e^3n?2d=}Rf2YoKkc6?`z@F>iEpVSiPF;W*fKjW*9E0~68s}EN7)rR!)Osw z&q+A>x5Rj+JjL8P)&8&-j#B%QMwY2ic`p2%``Pq~TdgpPe*z5Vvk;v*2hx`nsBA37 zCWbwolY&U^?zvi0a1<|L zahuYDCZD8(vY=^T>|m?jcrnX6QZ^yMl!#J1?R0(a;PDCqtYxCf_y?kD=*sN7k4Siw z#P%8S29iHf!TQJ%I%4@5$&|U#?SJ6{m;9jQ*mTUOa`jN}1^Bv-KT3~0XB_VGXo)z^ zW*SY~ECP?_AG$f}h_BmGF|2(!LsBL>}{m_s-@-gpe9G=KFc~^JGddB{`=hFNfCi66xKIOF2 zGw7!?Ka!v!#n$+rq9r9m@J77rK=HNnM*|y{p=hC^=HwH&%G}7=L4hJ%X5v3Z9?icA zv#))&MCK10-fNG?>X%s$uY|j3Xl&OR(hMuzkB?BRHx7Pv@JC{1S36@h09kpZ4V&fLFiz)uN z`tQQuhVs)&MKm5pOJLup%rIC$Pck*DMJ-ig91Q~LDRrrHTxceZaKUX}DZ<^-HJwzo zMVpn)H%!hiR;6ryhcg}W}@%Nmv-bBA#9jX+FZ9=XnZ!fZ44P2|zRrG(!%ynUu5Dnfz} z3h%n4frH9N&gF(bhwX)jr@8e=Ub@BzVWAyxySnl zUO~D0+Qo9+p&4TWLiasbSH4bUxEK}#kF8^CA5HWM8T3E}T?wf!Xg_BnQ1_B-Y5JR$jMOZjF^&|et&z};6sHV3)uW;L)gmL5wH%SQJKi5_2IM0aXNyubK ze^vYQeAKd)D%T||DBvz0!h?4fGCcwA?>5sMy!d#(fSgB?D`}O`Zs3ABS*gjC3o)zX z-Um;~r86)Wx@O8d)th?-6?6Sj6Q-HbZl@1J&UTUOfxf z2^HG+Q|^&rSN>s1<9^;ozI*I{f0bDv2i{Oe+ghxF5@)Fxk6y2GV6pQy09;e#eUT6WTCTXt5;zGX>8eCDakdkRcul5Z2u*UW7cEGHo4JP&-BG|GdB%>&nrN)DCEBG}D>pgYXRxG`U zO%qA%^uFOhqHz}7nz$P*gv>g}niTlMG~7hT#>TQ0)hos6pRc8o;LKS>wUZ=z<}oj_ zhY4&pxN)`cgS}woT*NXtmo#()9!D$CblOv5nO?p+Zxbo0(K2V zM$XRYq$My*KtR3kgPZTr6F7TH{(VpWI>h%e1c(FFtzWgRU#Qob@SP_tM?{;kAnE)e zP*&3ZZUL&7tLdlO={jvMY=1km19N_yb>%PHTwlfbB3$r4Y<@7w^VOvLz79=QpNw>D zdH+U9_zu!+-%`$gM%M@YDTPO#z7eq}yu-UUnAerq>zLD5LFa6jVSq5(Bi}QJw9qA zbK*W><{Zk%D7b!WfU})Fz}wS&5BYf1em$;w)9!o>@&#xS2gS1~pv{+!u_1x`nOnYJ zzjRdeoN#vbXS%Hl=bX6G{~Zgx9x*IR;>Z;M^+H`u>#+Ejv7W_Lfa@e&t^h!$Jl*Wd zE|gA&Vew0OzwjDsK*j%mE&!PK-`DX^$!W7(8&p%n417ZqK%zw!a1dQBLI z)4^oH%J6b;zEJ6uw$v zeOyVWHm~?!vwJ1?fxjAgilRfFF^Xx%5;2OHbAheOTbhODu`1*37UY6x!;H0(WXQr< zO+Q3jS29(qTs@5Cm0FG^vY%ZZ%fmOyLMk^B)feDB7fp(28d2IGQ#e#VNxPWjA>pWd z?|AVNYh;Pf2@!KzA*ORwm=78-K0;K~(YVbo;|Awo6XI--P`a`5@X%Pwz(pexncFxg z{0rI=gIzKC&dHo-bY(bDk5B!oFjXSZW($ta)NvCc} zWc=deCu=O^nmaYxENaUmbwx)XmPTh}DtQQ-| zC#cbRux!95$OxK-u7j$>7Xb1@K=XwMNZfF1r!KYG7AzK(}?J6?i>SvFimz5Cka8{bY(q6_ya&tY8t@pMk$KV}7Dw>Nsr^sn;y8cp%4 zJa&y_v@eDO-mJO}*sWW69#Omcznc<@DOsUk1Q_GQV-`KAdR>jg|V|R5|Dy}Xp z&$6S@axf~|$p>bP=P?+Mg$J>9j?kn%S7a3vx!}M*Oi?n++;!)8txQ;ln9qA3>X|Ky zM{Y6(>q)?MtL-5`KCEIwJuh^HT0|a|3F~ij3w8Ob^o6ej(h`maLxY3EM_(To08kGB zZj^Bm z2BTkA5)@jH_LweRrR!1gi!Lm6F}2f?FnQJ>4y2`V>+)^%KaK(_i^5h8Ik>5rd7O=APg>Z!r()wRTOK~RZQeZ zsRbHV5NOlya&PWqe=MnB=KgGiP~`04`)MrKMHpH>UxOXd9_Bg<*+E9at5kmi zaYKne_`Dq>$_#7)BdU;uxnEZ)0#hQGDY|Wiu%Pfu1Sj_Q-$X2nO!)K+DN{0rHHhUg z7ftIKnzBIQqM2{|RJ{xHbxB6pJW)kjbpWk_L~YfA{NNqQ{7@IC8qy>e$YfjqDOR^vmep=*Oo}?BvBx`d6(M~ z*Hm-1YcFi+E8dw{*zD?!#X(2TToy}ADh%wkJI~OSYj#1Efz|j2S#RB{JLSa=_qjJ6 zE91ALR-G+b^b~KD)B{!3t9n9DZr^dUnvFc>qlfw|2No`N?sdld32xSETJ=g3 zApS=lUy~aMX9ckU3GIcpv$D7py(}dKEm8-@G$9a z(xOkRz`Qw_fHbx5`Tb(>n~l(1x>jRN57v`4ny~Oza#8tCgRNOnyH%~81o!WAIgf2b zoxGkK4}rLZL@M*pFIUQ*%=LhFRI4u;jOFy7viWiknG4vE{?x?fZ{jzKRC*qWn-)R2 z4Yp)|VY})K8dvf*_n;>fv^}Ge&l=O<@nC0=LA@g^Dpmxt3Z52NYB%Tw{YeN<@y5 zzda%|D6Q`Si7fca^p8z#^2n9Wkumk$V0{Vd^cqS|VfDmhY~9|)TQ&+O1wmo7ylTj1Q=6ISt9%PN~dE}`uRnFn> zJlSjwDjQr^;ZNr{?{{f@0a;dL))RVQaWpHh#oH5`h$eOxwZ5l%6&jvR&u84vn^9CA zty{Y^O}O(&oM%XqSL$`7Nf5dR3O)u1eql-hNp(F&boGm2>3%Ppmx9Tolm2(_UJc?| zS8I+N{{0)Jmmpl`isX>%O>w#)wZV`z2UBFB9t-*b*-JKW7HK6EigA4SUE!=A-LT0j z(8eUQVpvaV)NG&`kdpJkpZ=|egij8E%Z{%p5n<^>7AHq{zBlZ}Wu_2QoIbVOsz5aT zL`J5;M}`OxQkf>`s3;p&Y?!2?4W*^a z=0&$x6Do;EM71KEviRmK3TrTRG>uqrh{UZ2S8cEPu#WgCA4(F2ZB)Dm)$Fbuz=l%H6?X0EKe_C*1QE`;uH(!BB6$SSt zlo(%igf$lGuc2YdxXoD}<(qbtELQ2n;Z~+LxF0Qw@VWk6vE z-^%a#N9?T8nG(;gJ49D+1Zr~arrDMR6>vM5vR{JQ!#9isxmgI?Sc@50T(%E}ka4WViw*&r( zsQN;Xl~+c)_-!zl@z5+l6!iMYqYm%?vb5$WqzmNF#t%_)*70T97o#1bvw*SOg zpH$0g60(Xb+I7Irtn>ZwX|!=?OC*iZ%ZJDCX;tQ?$y1mYfeT^`Abw#AxN&TE&B2hu zDX0H0U?D5-Fc3uei%$ZYq+x7JM^Z7#BF!)Mm^vJ_qF`_RyC;Nlq?66#7x^>(ZuL$_ z$`x$f))43$EGFM=K93g<=G?VRSZ{nY?3Evy-E!vm`vK8E+t4^5UaCIMo9^#5M8x_SX*JWn8C!=<@as%*roKqk}kVsS=jEEM${(`eqpxNCH zajo{9`Vx}m5eWH$JiECW2PfXx=MN2^!fZ@23hw5jO7Bd!GKTpC8?=3u`9c-$ihU8S zQ8Vf(gk`u!n}KWym7|N!Cd-NqJC#=W@)%3x&P)|{J9$oVF-BxCT~Vf-@PTfkcw2i94YJwg2w#8p~br=jRubfGX27Ch;U2Fn;uCjs1o)I<)8gso7$s zuFN7@jJ0KCJ~&E;+8~QQC<*EUN~;RZ7DsYcGlW^IyDuK`j*f4V_XD0&kB``& zkqJ$JPnb|>dbKkEo(zBnB%4T!I!=g0kKLa&fM23jD+gu-gkY;dJ6}h|{i?dhh0wUy?TmhXlhs{zq@5U7p&{2{FO|+dClH$8x zP}z=r7Q7U8JzX0a9)`_;U#$p@o4EEK@X3Xx zO|3!GGc-TSCN$@Go-(C^m^Rc2T~DQ%eT0QNgTH@Mvc*Ot?PhoYM0${)kuHH8)BpnH ziy_Zpm4K`)LZ2twuJOWTJ<^Db=&VFhG9wh=sR>b&OYxKVr1!%Wb!e2Kn7dFFi`hej zJq_xzK{b|D7L|e8XrhKb6M}wCdEGmCJ#@Zc6J_UU)3O(B^)C)@sthW>xYm4h>!Dvh zqS4ptd?(6i98YBpNM-$;%5D5*#wEJ!KQc@bn8cXnLxEmJ$Bi0n?etjDqEE= zNU{-kg<7U6?qIX~m7#eHLM@|7hL4MKYRs6HYE~NiXhx!m#mj~ZqxEt_5mOB%+c_oE#gR3&5Mk74ZrkKZoZuzW0H-p-_R8zQTm`BI6S1txtGA2zJ*P56!*Fa8zqq+x)TB ze)-XnzvpK?FiQ@uah<@`yBk#x*R!_8{caQt7q4P)OpL8Z2G4)L#F4euke>atoLw|y zVZSYw{#e4<5o@d6bhWflF`mZG`a9T*%fq{l1Okin3rZEOb5cxv@d_SLB{OuoTr@q6 zn+wGE>cQdsndx@rj~e+D_sCPePBGRDy0J>tI5#mahY0`mxdMR$H+Jhx?6Mw0@njRB z7?vb>d)xRQWaf!%^ED+iva?e_&uQ0ML^u$1{+~XyhS&UPq7c*Rr=%F~s@8D8ZWTUV z_~c4!55qVOX&d+D<(bO1drh|^i{*&*<>6x2TE8uX(Q7-xIb6e^j}(SoF`%A{T7f$Y zqOju^*-CNDp-hd}z7b1ioxvZ8=vV}s2G1Cn#uRMh)uc1$Q4T%r(cfA05L8kIO)kyMyiJ$tn~l*((4S!Xs?exP%W zEm}U|`LD`vhv*HSKO7mJd)Pehc+eAgS)QUWsi3k&Z6_9rdi*U-Fs2d&_I;cd2GFxX z#6^AN_+txeQ>)-lKyAO=VU3mjyhoXuVv#)~AiQ{>dx~0t4rts>&eGU{MJ@X)gLd-e zr+;DOQbJ&c@)CkCW6A!j#c`|EOztRxA==hi45}2{6Z&FT`*R|TKVH3kCsF4*Pr9^$ zd4g9{Y=>XN8aT^1v!DMKr+oQgm<0w$@1gU{86&u@yI?hK`tTCHUxq+8US*lp4FIk9 z&wnTL@jPI<@8R;Sr%TFw?oZKnD4m>z6=~eynfa4XY~GQ!f_~%%2I$iZnu?L*gCL*? zbzo`WBLbl@jDAh<8K0F#oo;$2pba%{C&~^vZ5TmVPb^E7lW@7JwsXjET|^bmx33mW;v|^25t2K57M0RWT7ITb1HRaacYNLTBF_`7Vww$ zHJ%^EXusZP6m^HE>33}JZ|CbZS>UNgsOZ-)?fGkX^$lBhVyy)}g|q%Y!9s0g3hAkAU{~gB*bdRrNazO(K+FwF)_eHyXo z-Fsp6FDcSXD39L&tkK?*5q)XMC%aeeJvYIYDc;7AshQp)Wm zr8Y?_F}RG^mJxFt5em_$WtKrFw_dDFv4)RcV8B!DY7v?B(u*={f31Rmcj@%}vb z;&|Ws8L#R`(5n^Vf@&}oUt7^6=t)|y=S1(r!{>BJPb?Yh)Y{QkN~)8;!A)@UJvq6? ze3MXoNIvi^6hyFsmtkGJl-_{Rr&oYw8gN+As}+>zr-oQh6X04MB3F!h@sJgk*N3>M zJF(p-@wf)QFu3ifYIEZs>DX*dPS*8B#N)8--^0nQVQ%64g5k2)njRHQjNC)}_fsJ^ z+LRs7*%blrbm*exZtxc(;@QVG8|?2a$|4eCmYPkjh!I!}C`msxbLxt24Uh;8O(MLIJ8rMyykudD?6X3PjwAWh<#i&2w@>h%1ZiBCu&iL+l{`E2DgRARxtV<@J1ZCMI;5ehuXCJ`E z%|IY*F32f%HsL&o37+4f&o3{joOW5=Of=5B=p^ig{g;G%Ucatt9C*$kV&ENq_}>d- zD{6&^F?%uhx(NKl7rFM-oCQXHTjQw+Bs7|TUIrOoHUAeeIHnMnmCt_WAfG?e{c@a_5U$|(mw6$FGEm0w8p^um z3MVouI^SzBJP2!~j*OC%Tl6OQ@a#<1UPESIVRU;8HJq7Z#g@h9vYyYqw&X4$z4#$A z0n-7Qz=U#_;%oN_eii!)cC8s>3VwU<3WTi@+$U#>L2lPmY)ZLEqj>uAtrm~hAMc7= zh=1GFzV0FOdSo&)P0Q=+3%+>=0HW+)WeH0Pwc3o*L+6zD%B`qXI?rwJo)GcM6{Vl* zu+}a8%m(18o>o7uu-dl|IIAuDp#M0M7HoWBnWKiUPXcAm1@8`p{5yjdKMtezGzUjb z!B>OYdSj~UHfBe&#n{};H0fh0swf?3 zp_oad17C&_do+LK;xkd?S5+ma1_ZEaR~Qg`H*A+y;Te+o8f^LiXgc+1EypeGcOen< zR@@t!4f!|F{ATSq6R@H^JYP_y^7?S;$%Sb_Yk*$okqcPKaweVP`3~oNOT}3p;ehde zhWE?8!`s~=tJ8tZ^;YVDSl;?iAWMK52=ASpbM%|U?pBLEkQ9b zQ!oW}fI^rtNzf=!N=**FEvn&Q#BG^MxPm^E$20j^$;Z&5ejG}-;BB%u>%Hz^)mbmb zuK~J2wrh%$#VYe%Xz*fNycHHncb!X8g*MS{n67-y#U+z_eh8l(5G`v;EiN;OMZz_9 z6>ALAN8Nv}r!HO3D&;_*J%&*$6x1{T!f+dTdrNP8p)REP^q!Yq`pY>i%2iET$%977 zz8&*3;IPNcl6G!i>h6*a#v$Va!Xy6ApvMnBKc(q8{DNf*imF;FDTmLxp}&YJaeYvNGX1C;#fN&xx!iVos@7zx;3pXQzM zURYtU)U6Xso*7lPV6ehQp+Dc6LPMT`VpJ8P@)~MpcN>Iz<_XoI$YBrVc-d>BK3Eu4 z1mmSJ7Y>eja6G&eUde=2V5pupMaY*#`AHW#y|_5#|8(Dm0a12DUs zEt%jtJvhRDZ;+Hf13H)4tdu_$80Oi+2uwVV{=5PoPYGnWb^q;f7yiBYmQT2)S|c{I zC5D9Y10kKG>@Q_?uz`2k!$6s6+&N!&6||?nH80T z`r+A$DG=kEtn;wrjEw4}+CWZIIWoUi$s6kJ?G13~ihY!h@eg`@0ACH`sVsh=1qx~E zysH4%7D;}7Qd`8R$p1#20muTwXHA=vz)w(|pDgktkg=3`+j7JbM!<0>aRJ@%Z;--L1GuG!2XEHbnI*dR zPnaJDq#7+uW=Lj*8`Z^<{kzA3jytzlvFO5h-`gtNuRO&}Zm!|SW;)VjT~{1nR~_K3 zc7@#sYyWhI#LDQ>dk(YE&W}wBn%5e8dsrXpTDw6!e}8T|I?9xkzxWSmJJ1UeXl&UQ zU}ZFt2(s4hxGIuJ#sa%%Y)CGf-ITGHOy=jTLyG(TL-xx;98v`vt=xE+|y(p-79M|);NnQPxkfA30er^dnV@y#| zr<%z*OKE}g!$+tn@c_O#Dn@%SMzY>;ySu36S58eBG4d}C*Z#U69V<|S%vc?`?$L1^ zs?KyVw#&Vl`tNWLUc1`PaDK8;u+(%{0y0vd-yq7bP%u<;Pgbz-R383J;=?`c*K3(h zCPkDu8WCbf6^0ovFp>+>F~R9f7SBS-TyUd#LMMbOp(UWvdcR=^DH6cy_s!Z5s5epu zmL6e+`acD(Q!J8DyZiNH<_z`V1iTKpy;E>SIFWBi!iB&MMUhZc?DW#Hn|D1t?xND3 ztk|e>&zhG-#YA5>`MxcR7{JBws|L@e$rb;g(p_i%aUyC%a%+4$@+T(xvzOrAY@> zsu%13a{PTHW0{|eGApc z{1g|4rW0)^O?yQ`&RtT7)~WiOW)P4y$Wy1g1Fmq>TrolE(VHA+~V0(Gn7jB zS55ug>x z;FgVeJA|#7XQ~{_>rTl4n488=;TUU=t?7I2u4h?&d|z~6VvuXji#e-7QErHF&IUkI zHUJ`Y70kl^k-HdV*|2?M&@4HXDOEDVP_%!#PUlJwV47yOxW zJy>0rU8cJgCOWvr1>*+A{}n0@{$@{iUdd$WI|^`1Zc0lyV_v@8<8GXjd@~Z6ENMtW zqm+Z1XCCopxbF(Pw1#T@GR$KwiZSi2+2V;a!|{bum7C0;Rrzcx~R&0@O32$|MLi4=x^VKu&6*8 zO9wRk93N}YA<2wH3cC{raQd>GtB`%y-E+Fq2u!h(DI2MkaV*WupRt+#DP1a`wb&~Z zt5J%tmuN&+#-=AOhi?p!f+C}0b^>SpIckW%pHdPvvwfP4-u>=k7kMnFtCgG0eem7;B~Wo`IF;WGW>lnzO)sD?0zqfNaPSnfLLk76 zTb{t0Dd(Wh-k!ZVL(elqao%7i!)H>zAvA)sQk*s{)%(3JdtcVhL%I1Ub#W2oC(_V~ zKxW(^G%x>{-9ef>M5&X&oc^Fq6I8Ut4d(o&5%-7%Mb7^)ZYxpCxhz%Wk5`{h;6y4b zt6I$fhp5k;_mIv53Dzw&*rci&e8a;_m?;XxJyzSOF!YB)0yJZBRlZMkw^U;1eJlrj zSCf1>9lt&K!rwk4*V?`yyjHBCO~oU)=5T0aU5VF@ zh2=V9e?13nzhoK-n&k#vAantgJcGk4XLp{u+LY1bx6Fo1UvG!fMvr^9BHUV)2S#9wWn%V(SIeTL~n4>>CM+_+g9DfZ|G3z`Zk#il5##c zJhGfVo!cu7$@2PdUpvw~T<30XZXEmc!fJpj1qfRV^B9S#(8mfUI9TMCz zENK<1jaQdsF>tU0nk=PxSioLon2l3v*nu~ELKYpNo*Agf8deJ;}``b?OQf$eiw3nZsOfsJo2nfy#fPE$dqeLU_CC4F|T1v zTnWYKiHUsndHEax8JG^{q{Do!NcJB{4OpcEO;RuPVf>a}VZrP`ESbboy`?fn`m6dH zDxZi>MBWx8342MLb0n{BA^JaXZME@Y>i{^6F{(Te2}hDWAA3GOJ}-7nUe*}fG;h7` z8G7VUjj0H&@&e*8-orD3-@-Us>=y()@czr@ycR*&xM3wwe_;JmzZ(_}8r=2XPg?x1 z5Bb$Vx(1G;a*yLit?Awb1<^~f{B^kcE4I{KcyEM~0o00Pkg>CKf?Tnv{8V=dgQZg0 zvj#aM-#KVka7T3zzlE+lN2GMR#KJprJxT1-_bUXyMnL;LV{_e)CJ$)dK5^)~9zCoB z33&&QPD=&vw4%#G7x+=_vAsfCVs(9(Hyng{Sf9#z(lc0e9=#iZiHJS$Vf`JUx#lrVe`)XsQz^Q0&?!fvhh9QhEvy=@4@;!Oe3*}4MHGco@J*Mi{4 zG6)ts<_#kq$pfix@KeFJK?OHrI0%K;IUUEO)Smmw?J_%g9Di_N195j7YVg%(P1G>v zSf-r*T_wPKS4%~lW-ym)*QI?X^7;*s=z70L!z_4Sw}(B%VHVVz0i!aYN|HBXm5Mc2 z!m5{HX6A>5D^)%C^6=G;#p87ZEs8)Gnh32@v|tila{(^P?1zIR#@q^bST@f5*m%zq z-|IAzxCES9g;tbOG{5J4wf9hCjxB$*Qm@XU76-goG4Y}-;p#K?%l#SHrK-byWeOXN zih%hN{hvN6i9;cIB9^+t!QLvvR25${PT@l<-eK zMb;esdGvg$T>`l{T8?NTm(MHLq^H?bmU0bR$qv(3e_-R|xaW`j!`ku+$Kn86r3VH~ zdaZt-O~oc8EUIb>G0Vrn^u8*npOJwX&l}3EFNmSCnzxc`T~_|^XvNI^Ss2J?DD$hs z+ToCfgKW_maF}yAoc=`UV?o>GdvNm0{N4XkFeZZIZp>zi(TAz!m8J)Oc7yy=m0}DJ9I-`35#ORy?Ng|nibLI@X4-f) zKnO4!Pa$!zpg~O@5h0|mO|9RkvAUxHn#?gj$QLmaa`b71$ypl?U``~KPVA7J1AR1> za-BzNP%$Zo?&KJcE~unIhML%(#O!TWo5olV*5WmXaRs74ftubwg0VB@-^B0b{Dr~u zK_KzHGQkL#yZ*cWO-4`|TJ}(r=*x`c;&3?bmItur8^^kMM5R@$B4@?PluI9SJ4N|3!E@dTw$^dH5~I^? z*XX=xgM8BcR)5KQXY^MVCN#qERaW+CM)@me;H45efrr)`$VF9fEXXj;JjS`z&8%Wk~>1*-+kIIRK^E!UsD+NBefo|NRi1%4rr<;68D&B zE0&w=h$N7Rwl8Dzb)4=J{oMIS`(kXiM`;-%Vz(>xjw)xbye#J#ScAe6LM7?6+M9Ru z3off(Jb&YzJK#ELR4qmQZ)Bsr`Ks?kJD-H%hX#y2R?Hi zS-dCHG#b&~sVq~gM$TMJl$2dSQFjy6L4e0VXtX<$Qu7+Is6i?+z^tgWn{DA&HJPmV zcXuzEac~VfJ9_jlNTY!{YIm0u?g5^OYo#Auvnyh}wk;y>_4|({P?Rb*89{Ub(KKsS zMaLVU)E%WPwT_~?MKWVR4U#S{xVnDSN+BM1K_K860lMbwH=Ktx+g@{hxt(^PGc$G* z>DQlT34Q4)mUMQQBuHwhmV!ThNc4fOghnySc=O@lv zfi+1wZddybGtGALrF$kf?IJt!J&jnAZJgzQ0)g#kg{_B{R(F33U?H%o7ANDbw4v}w z^vk`6csOre2_#5lqT^9<(%+$=Ou1!>bf*}s%5m>_)$g9{4m@@oyIcz$Ly40+Q_=

E$7B1rS5+4t|DHm&!;HY9PYTZJuR79?YPlG2X|(KuU`;A?3pgMexd0tb{z&!L#F zYdHx2bb;}9t3$;A%?=1CgdNyEFSp;nxmrvfEY|vX6P^Dz>e@kz%9mEd#?s-guC?+E z6$JB*WeETGrL|4O(}}!x^VP4BCiy`EF{uqnsmcCb(P3^kr_l9}kNJ(O!sQ0LXB4!! zgp+Vhf=bFiXJ^I85k}V^WWrkl-=>SB;Kx`aoD9iR^GR6Itvq46ZWevp-o`4m>YNgv zPKh_8)|i8_(vTK|G4&YTfZ-vd38O%Px?23g>Li?{Nv`3yAr$UsT&+ZK1P{;5R`k>; z_(`bO^^C%>?H7qvnMu)Ht4=s4!x5{U%Ivt+(bPUCu^%;sSd_OtZ!BtN6GXbj;}U3MP0D?Pf-_U&D~}BNOtI!}6or)%+8? zb}4bSih{$I5~P@PJho{bB(*|CwQK8MT+jb7dz`QIYIeV{p<7A_rMNql4#XZH+I*2 zQ@jmdxOr`7OUp4_hv=~c@nifJY&^E|-AUnbzP}j?e%fq($aa52_}H8N)aW8KN7?_8 z34lwlzEy^uLMBFA#cqhys^&=}Pu7gFG6W3ctTE6@8N@>xzr)myDgYKH@bfYNz2k6{ zvg^lJ;C|4*AAe~EG*(|_bzgHx4>p*SY)JX7pty49F^U@Q)iBh1+f(O56Pg?Z-cS-A zlw>(r^nb^Ucr)EWOP?TWCg5wKyIun?$l5`7o~E4sN%IrA_$~$kQ~>reKR=HGEpS#R zj_rkDX&BuG^*rgFN{+39+Q%hBE&Zd?-ptTyy4E93YanW|QnL|A*=TXx&fPD6W01hKIs z?~Z?owxL(ysvx1K_PZHyZ+aT(goT#*fmMrC)U?z_F!+lU42hVX^0&Vx+2xw= zcqaStwPfI1-jiqd$Lm`kV`oq_g8^f`GG+1POs0zWlx_=Fu~bRtf*}nPwgzMbVy#_#1C0huM zd=B#4iDF$30bDBfO8F}ofSkqQ?}c`KIA8+&=Bi$Xp$x{Z=d@j61^FA!^c{fc&- zHe|{R3d*n+&h|6O3qGKH6!H6wA&&t2^G=#v0$3Se^u#9jGSiIin+Ux5Qn6WI+W+j` z3innlj<>np4mo*~%k-{1fg{?lqTKnr+bgS{B#vn?qWCe;-D{=Q?89EEQN7o3JC%I-hpJHvmsF-6kOx9QLY4;l zPX}SRrjwrW;PBbfV(=M=&yCO)QG(d|06)h!WQiTNWb>1b#Ml!NBa!`4I}%uRk2# zZlf(S@A)JzINj$6Z%uQiSZyZpQp(*Zp8@RANsP8`yG z-}U)QB;m-KiOQNfl&f;N`{)W()zh&o+)mii;#4GG5)%9AS~bfPZw^A@9~4=R!EQ;c zRqm|4+UYPqp?IPa4djY_tSbm}_BEV}7$w9*@Dx-md+WPnF2t4vsztN5d82^`mV5}R zC=#iTu&$Wr-l3u{YT4_R4s^vfz~Gsb))9*9@T3gLlw%Z1ccwqy7`+!O4aNa~7SO+n zpXS&VoNj=^|DE|i24uHX2JJq;v^y&UCq(B{9SdDNAV+?UwR8BV)%J46VBiI$sRZ;T z_26WdAUd*UPmahjmSM~cB=4$6j`o@vu>=L@phnDS;~${7cG&OePxbnne!&yqn%dGQ zJ>PT>4ta&i-Qb=mL_FU=Gh7R6RQDdR8p&~&YogTp?LZHRODFGgYcbnp>>I!>DKuP| zKFqa0KF>S7v-Q-Ze56G@|MiEb&HCW-?gdVrmq^&Kff<&Hfhw&HjfM znu9|QAct88dwY9bT=6i6W^rdLF(GhhVIpW~Vq#Zy6=@rS%g*EM&cML)&gph^rCCC* z1JyH8DVq+NlePtgV-0J@0#5s&M4|9LxK+{}4aA5wAhDZ;(16myl&ESEzdZ9kt&t9jPCFF4LFiGCq z(3WV!;y<56v#*q4zC%9>68A&}&>HU02oo1)R5vB05`vRw!OF|Kv*$MNqNrpNRL~d8 z>{6U~FDxxJIBfC&H-+v}tnh;d?Nmm6FO_J(Zoml2dAp^$_nu^#A)QqT_>8tv%K z)Vt~=&0K@I4A(gc&ehcdZg(10_fRQp-FbULC{)#;1*UbBz&$s=aW(fO#(EV}$Z1bnY^@>x3AlA72OoW7?u(>YIkS8%Vp*c|-TNYP6FSZz)~J4B5E6FNvzSXlUnizH5jf|7Ez%OA77LFTnX?Bu<< zu2Cg(G;$O*q2Q!C?U?rT>{Ke0!l|w<$54|-Sl?prAZ9yDKPQq+I+ttPbC1u?nuLMf z{L{47Nd8ZfAjY11ASDG5tS@&N^bJkf4EHo0i)Jp+E*+~ZuQfe$&Bydmt*;D1-c%KJ zk==N%mbR0m=D581=!D-lJ#(zlY}Diz27^LI1C52BUocSTH@l&v=&g;NK~%Sobb>7h zD=FvPshN|zes|ll?SRRqyMTfrig16&!l6SH^L<>CK@vhh#cM!=t{)X;7d!fQ&to|i zBct}y1mUe#`c~7KGGaX3SHKgmfCSa32tC;1z)DKogXAL=BmJ5@&`&&A< zHgQ6M`BUsh*|vydv3Fz#DX;w*u)*^TJgJS~dj>>CW!ePWrS>2~-J{h*;NxmoiJE?E zZodCAKqI!knk7eJT7S=OZoZi$Q8JM=C%I2#-e{{exgw<`_2oNhQoer83E6?FWe3cRSMs;k*H%=pF&HHMX(gP<%dL7(Drn!C; zaiiLSbUTEp=OjEw!n)X4K2n_W)aCeIuNcS6i#iSbeO9_dF!$xk#60b~-EmXiUX(TZ7tc%=3#;$i z$enjawlj1+s~a@V&t~&+7+8Ot`FqdKGyM^w@o0ml)cdpLdMjhj$1P97*x37xfY%E> zj}vns9d3?{Y`n$&+cjX8?R#{!Ad`anEoCh_qcnXa;@TItLGd{hP9boYkC@09Dpo)2 zyZstKK}lvzm!LgK`LX+rjh!=L9A~wIDrRP_jyGpNg1AEoulfJ90AlSy?{hS2SMhlt zf_x7v{eF+#-TgtGg-4?;5v>n9sGQw-Q{3*-g9i}|<^kw3e4d87#=APl+Sv{cbFn|0 zrduoL#>U3Hk9oI8C-Xk|2AK}_qNhqLE63w>2E88eVnDfa6?Bqw-aA2%SOhpnhm73^ zp59|0Wqq-8T=h^V8LJMjb<0m#<$?lau{URX-yb z$#dD@c3(1W#>U6bO9ne7;b0CTJO+bT?FNJc-D@ldSSsIH(VKM zHLC%P$6>62Cb6G2jUjAZzjgt?RJ+a(;e1_fkYEYOwB(t~Ybx%nWW)?J`3y{O?}}2LHqH$h~xR+SJ5R zvl*S)C5|ir9IcfSdpA|BNR85vlapiO;dsG~zfdHekeCs9Gh)xtCqcJvKIXzaQKW#D z_8|Zi6y%x)3PtQd;FskmP<^}69{taSFc-~Ww6(!)#*$%u8GrC-6oW0ZuJrQV!yEU0sBumH*&nODUkc%UQI2*gTcT2W3J(t%LRNpj9r9FYn6I{#@c1G05 zFv{WT0Htbhs0K~B3Vf)Q`FH6be?sJXsNq`adNz9BS{Qu(TArJCO>fY{ zUeDW@{IjAprDoVmGQFecyk0<$3^t zz}Jq^ot-}WF)6l6KM*GO__`OjrnDGB3w#|T_w59$ zR$Ej!7t1{Y<0r(Mu3_t4Y0;1i`rkc@*3?p9&+kn%}=M>g-NbXeBK{ zt}{qf1tU!uFkxUMSFsDEIOx<-gCv7}j+~HZ74&r;Q98;K$Fcu)=yz;(xLocEq*m+w zaD=wbSGWM%%)!Mz9c+F0PZ@8w$wc{@bJYT!QYGU* z`6QAIK!VOpfMeM6w=~bf$tt0xg#*B}7@b}@yW=#NH#axSZS+PVX6y*uN3F1_+t0lf zJOH0)Dz34C)L?l+^sz6*eMF!v{dvYya^!ZF+8#XTM$hl9`RYN}?pZQ@GeP z0&5%_Px61Hr)TfI-A-qfSdoICL_lWFPjls*UhqDRrG4Uu1*KP3jy+%f5Euw&UH%7i zPIcd@95qc#Ln)mgtdF3+N02rZQ z1h9<31K@zUmsJ$GY?@X>ng}wguPRWNQ^?OMQh4E>5+P=_m0%Jp-~D2C|?a8iZ~ONxx(L#IlcMD z0dIwba{|5`$95}1k9}cpo4KM3OKj=hy34nNR$ivQS6wL(zV-b6)2$L<8ZD^CcZu?{ zLS&;WuDH6MRUDqMj}c#>LR7p|bp9BUm;TMg{Lxd!wMT~SoEvEwxvh)naWYrjU7PmO zA6~73R6<$)T_ZwRhZJi$YcbRSA%oV^!CF!NpcjS z7K07N=YWF_V7yOL8H_*Pyw%h-SeOMy4-V9P0?)%$Q=b}mmRb|QbVb%}a|BwA(O3mI zrl$0Nb9?yxx0|R;Dv^(_LR;grcxVPjI71!S%P&o$UgaVcB}M_xxIE;{+Q!QLa1fIm z8uu!R*6qTa`SE>Nf_-$%k3EXiRIYaNzYfOpY@^W_1DfelRJQTXa(vNQe?xzN{$}@k zI>RnJ!DR9VitR2-&s{4%NeY(&1d`?lm*^1OSLZy|Mcq)eTx+(qB8H>`*S)KblA4-S z&yIb3znyP{J88PDtZ0YN+kv;5+5Uf5)zs94_zwS2LV*v81Vu(dn}QKlOEfm)^C~N_ z!jb-cdUziahZNdMyVFh-LeQwyR*?0l1qSSo{EXlCd@KPRiA8VloJ^gbH3crkKjdrv zw#EDxWIVz{oB%|W>8Y;x8|vHB=c4)T8VVZT_`>TXd@y*{bkjfXJdZ^=ygv9zyiD+l zystIeV&mZ$`&9@5%lrNPJ%+XPI;N5QS=i9D zMPwnBI04;;9otXS=q4?1jcRH%7=RMLiFNb~`9bhO&YSC(2?fOzzi6J;6|A?UbxIB{ z|4PTjiZK0fnzNSnd)Nv@r8uY@!>Kxyo5GxIly{5-6`AoU5*oX$7YnrzifnBSA$AXx^%@-oTJ&gP4Zh|gV zn+oZzMh?d!OwA9D?JVZeW=&A=)AIWvZf#7y$jhOH!QpN9&S(ifQ^h-$_(zz^V-2O4 zv*~1cs}wTA87mWuW79B@6y0m6t3qWtkrOMUDeEWi_DVW_1%0)zNMXY1zm1`A>Y^Gs zazTn$r9w;Sk~%n5v&z6DA8tSOp1e#S^Zt{PTb)?QRlXjqoTFUXueIE{>|`1=wDr+(<#c39=PIC9mA+ zjyQy^$B$N5yD}kVO{B4vk#id;ar9^mdiF*9{)bY9GK!?!_;=a#FhX7Y?o;)&dXqtw zrf{j~0St%Fpm!*?v!$+)34+QKBv2hwYfFpvpO?D|3 z_&)`>EKhr{j!zvX$$kHRQvZ2H9B{W!c-;_fD8W%zgmDsdf9Y04r@5I9(`)ce-b62l z$w%Ez(v_~ss@`WCF=-Zi`Zl7adoEnB2Q`PDg@&pH|gD7uMiRk8-=Etg(h_%n=xOPhU(1({ByE zkW;T&P%j|r!ozJ!(cASUU)=7~D&@*fr%5UAV%l3pX?sEJv5(JP^n3WQ>xj!}TQ$;I zR@&Ukf*c1`h*!E{h90w{*0Lvig%}P!cxg312#DYSvv87H+kBk=?fDOe&&ud0L@^S#J$s-hxlc`qfjtDiN>RlFBC0Zr(IuiP5s*+4oe;Q4I_u+yVL-d4s zycYM&v)bV3v!|n`g1Rhfpy- zZ<|hLU!8|O7w{0cURgbP-Q3iq^ee~7K`ip2Dh!96iHpq+*(7KVEE4$ng2P`RmZ}Z& z$V{btfB!YD&O2oKIOyN7RG$k}ld1Q8bU3v+E30P4h@7MNW!dP)>vsV4iil@6o!f~9 z^ZoSHW+l)_5QJ{t)Y?3^F&CUba;@S)Cj2NQt6b4bKi}2bSt^Xg6!_s6(<2>vDP^a! z0)95tUl*bu4L(kU*|W2QbPs<&FYNKCfqc^VbY^A7dR>ak3jN@XK+l$FNb~?Brh*(r zC*$KaK*W~I(U`nO(W~p&36z7qeAmtWQ&{cGvHplBOxyOCp7W+5n$fIZyON&!G*n8U zbbdQ+qCT^%s_JJ)L7$rCQjfm1_}RJkhX3OxTD17os*?v;BzA~Pp#$nL3?md&{|roB z7AATz^hynyhy;1|Pq@f~>@Pva?;4uR&d=zYPW|woJrDYjSkb%mfgcuA8(t+rj^cLw z&fifGnbdzqeHXzgMO6iaoKZ@)evCvK%!9#3E-ZUnz5%_Whyqhblf2J)zdvJ%M&{<{ z`+t9)_rJX^3n-|2qH0BGf*wAaM92l&ejWTuebY_xqmy$HIL?lh&YPs)vJe}QT=O)zKGERhPYWC32C#`)EdE0hxXGDA3VY#Y0&AC(zw|9Rx z>9jl24c!rnU2h&YeHrp+%#z zva`itT?Omjl}$lbaFiy+z-D#CUh54n*m-GrcX#&>weHcODzLJ3R7^mrb{Clei`Yk^ zCjTe4VC+0F_Y(PO3L?|APmB}mJR=jPvM}-a-%JZN9*M)(&B)FMf~}_Z#|^#NKW`R4 zkABqv%S}fpqT#2xPUgB+iy5`L{B%oTD=a*gG9=tvxe17FVt;>MPEH|1e0>~g5Ls1K z^$w43U}uM@Q77}76EiO$6_nxoSm4bOr=h&*nO#B5@aINrB#`e5a0Q2;6!YrFq)GEB6^6tG zv^5!Po`Yp(PkF)nR9y#S<7F5PI>XT;NKc);)XG27cD;DW;ySYkLGyY3HL%$VKW|@r z3PITvHs9k2b z0Gd<9SuQK+`$+f@PhO$Shw8H_yhYHGz|aTne4dykhKEl6ocFM5F#=4Bp8w+6p9}6G zY|Ct64J<)O4v9-OY?>_YQ5VUr2ASu}cOV@{nww z{zpq3Ad$R;#N(dgq%bXzI*Owu-5htV3i@-km=w~RA#?Qdmo}ox;V*Fxe z-aW#PKa_7b_4ETW8PAILP;AZY>TaH%jaIXJMVbczQ@48T#u&}tVKI=v+pQ-|mdC|; z^B>q&ZRuyltd#2@%ad!x^A%-p@hqAr3$W12%V5%}#6oAyOiV33o~()@5P7OG>$xp5K2hn5S|R{`xrK#=k4#$L z3k1qVbBbi=JQ*Wb(lE<)urac&`b1=b_wwnR(`7wzOG(Mm2-Ld=yd%Mn{Yv0MdddEm zc}z_iAyqEhnCg{%_)cWF*5GYZGa&0_JeGU+rJ^W-pQKqFT(dpetg2$6yREV^WdA)R zCO&>2S{M)gM0TvjwY9O0Xp}jTzm!0p!7-6k?$Y~LD&o_URFuWj<1{Ooi9IscXmTl1t!7J-vDHrjBem4^a>s(W zcVdtxM!40S3s!Y#s6NQIpJ~r(n=9idUM&kGSOL_{b2^op+Opcpw)yJ}PeVeTbTIIw zMk#jNL!BaLVwuU0t*6y_s5U1A<6>iOuCoLNU*-+vG&M14hR}KNxaWrDIsK!~Y6Bf% zm~}IxSZhMe`JzM1?uW5F9Pc)|TJJKDO>n?=YN{E^UEqEB6bwg7?p2qn_KXx6*IO@z zU<%_n*!V`~%we_L>c%Jdehh}DQx>hJqxX|Y#^Dh+ficWyf&uYeXu+eSe7qj#2b+P< z-CE5yj90jUOnPS9w1z$Z0 z-A@Ttpq@RI1T>L+L+m0rWRaG^ubJT;ulWzS9MUG|Za_4!Dr{t~ebAHg}i_5UA z{#T|YmNug9lk147lKe%@PC_ASaE1#7nzxG<4xA8NIH_s!DX!haxk`axkZ^aSS!znt zNUJ#*g#}+BZ*}8UYp_fzzqwt(R#8`W{Cw4}V@IWAXgFhWW5BTEmB-*YN^@>$O$JZ= zsVlDuqHp9*e4ncXP!}-4TsRMaFUg6nITIDL1JfUf%=?sNNZTDrBn5o1S-u7A1LA<- zGWs$yRN8^YkhK*TBid+NR3tNzzBq;@Gmo|EfeTJGcyrXn(b7Z*IF&GoQq%@WB%ota zP{>^d)+C|D8)PiLrF!NKSvvg6h>S!I%@p&-F^_D#KG0jUn6^}?cD8f2?M_K@KT{xW z8Dmyh5R&4JQ=+N1M9stIt#dT9o+!>qG18*jlb#nogu)sH-5~mx?ym_NjCk~s`ZObdFz+tFO z=M%H{xXM#@J~wU zM0gN*19~u8Lu(O7+84QPrp--q5v=*oG`nCF4STzMO=>zD65%MbBI@!dx?Rb2`RH9F z(37|>nFh7Ew>CF`xF4^7nm7G$U4^r^Z!E8@Awex7c>&%OQ0JvGT3WWF&x8Yv98~Z1 zhp<4vydbift5;Vd9D9~NYl-VH{-?OPc>AUsc1t7f{7uLg^kKc5o3t#;zBM-VHMH;=6U^rG)%1WkGF8^>@`*;wU1D86FlTs;haz3uBuO6nu5j z9VsM>`pVi`PDeLQZ)4xz(CvL413>=&`?nCj_dAC;JY2Gj=r>%gZ0VYAOt)ZTvM8UY@kAp`XQyO=Qgr03M@ z?cvJuE~U3eh>cAiC4xg&UU48)rJLN+Du{3CJp3og;pEuJ=SJ32ZvOSi=gNZgn?m{c ziQf8jdaQDi+@xTheO{89_20wIYlSRC`gdJLUV_K4xk+V#yyK@HA|R!TIiUdw;C@tp zqDNzT)IW%qk}XsF@6KWG&*2s)>qp8yp`#yG8MaQ@XhwrCBDX+hJ?P6$_1f!I{gIWF z{ISW#WRF86?0xQ@Zj79l4WJx`KfNF>TiwJNwRf6Sh}d&4k;91VqX3SFG2l8nJK#kk zQy0g2%=-5$`>D*_?ZwCssS-&xoatNC8|0pzo-aT=HCF4E4=v)`o-)O)INFJP;HGWu z(H&;3IPWZ!HQZEvDCMzLvKOcVEpu_LG1~f90fi;s#lmC-3i#Ix;C^+xR%vC(ltx7w z;Yz&L3aziJ%fqH(&9Vv?lqXCbap9%h*U?nj)|B-nD9HMFI&b>T8@6pHIQA)5mIKVZ zn3&&IKecL^Jo2#pv=@d>4_o|vt1}Ch@z==OeDQ4)!jjufc)K2Un%<2I!P*_ur|a^j z--T`KH>WLcOV=%L-1a*VBPVRNJfFhy(xbmy6&K)GLEYb9P}AIh4sh|`d`=tFg3+Y+ zL8*>m>Z+V8LX$Q>tx)yEhXc=Qfh?hHbvrg9Qa?LAIQnTnC;u|VaY8w?myiV7h)8qV zdm*Wgn2gRcMd27-w>{16og3g|;u>Kj^iS#jI73vT3T+e7{j%J751-xguyD|2v&`uU zP0w9=g8u7wAddXm`KsTGYMzi+kb7Mcut3d^FDs8WGykA3YJaHDsZCW;(pR?^B&1aD zaFt?tq}x207#PL;|BD>a9jS;Q$pefOcUg4x6SI|-e-a5R}U-qDV)(Sk%`e}3M?j|IsaZY-spV7k5K9}^c>@3)#cSNH+i zojAUI1{>mF+GNV8C(Z-h!7UD(V?f-am7QHdad8IVnTbg2x+D2Gna>-r*9#@WN4@4t zwzJFQg%J%VGJ5>0e@7@dMO|7Y%?V!gQIGK85jnY zbiUD(Xpy83lNM3HT)C6*?-L;oa(|Z#cl1Bg1tLBpAOiww6J?sz&56E)KOGWQKk%;dzSRuA$p1kBR*xLHKcC$@!BEqCaq+UbwWi%CwSt$|I z#(#BMX7fisw0}RafmKGNLcOh@p@!!x%kp0Pv!POo|Jx1Ktl*VLs#`b4Qgit6GRBZc zkjhC5xGWz3I&0*yfbJmVo6U!;CcqpkeD5(qUYUck>3m`5o{+*tFl1~3u zN<}8mdVJ$3J``9yRn*mm-A`4wq`7a@NnyO&F&82<#o)Tiw8d8m((3o;#AUU$$Nz%% zj1N@BR8o?Zgyc;_utkwY$;lXEFMYX+sa(69=QQ2Y6lEgs_!?H}()0tkdBzg*qW(2P zY`+a5YPC5DY(^i=ZfGc~s-h~JNsAS|bE4A%T2bx*zyA)$bI&?iaWmy8k^z}%6?V`C z^)Kz!e|N$^W*H{?7moX3`4Mu@n0EQ!@X@#V!-FfxD= z#tQe!b>yUSmag@FIXB;d{116{xkdAg4J2E+8hRLA?dQ+<%pb5r`kzRC4`8ys_KQEQ zpuxexf#dQOsns=|MdE~|A+KDn$}cGy%H}=dv1M~c%c|IL5D-$*Ru;C^#g&(rA8P6g z8h4hWfRY30lbGzL^vWQ^dfLYxF+Xr~z6ApuZ%yZej)~mw!`&Z*KrP(6CWix8GQ4F? zX?69aTIoY*(PftD%vwS&UwjU4-rj<$BD+0_?|INw^6NR}Fgq%xDe#n!dD*1M6JZSi&Vq~? z8oUfZoyD#IZU-$IK%H!gEPYCK)@j15qZLnNm68VJP8AxDBTbP`FQ&`58jY%C@VDr$0L^7^R+xNl-*XxNEk+JAE!d@U&YK|z;4GG1r1 zNDYT5=lu4E2z6f-2yqP^9fgj==ao6@(9ZV!SbZ{|e|1p6aj|iUs;{3tp$y#XR$LJx zTh1@~Ob^JqUaT&!Ir^#c&wISSJ`O-zz@%?)z=OYsH*3&VUG-`}vu7J77cz<&-h?Gf zN)jQG-gQp1!)Lbfa|s3_VRv4?+S^yFsV|e^Hojg4CQMBs%)?GwK=(&PjzYIMtPYKV zXaP8jEJh^m0lzrVDWX0S+N~;GFpyz#a?sRUnHTYK~$_~2+r$j{#i2ank3daO`1b3jSy zwU29HIRUdlF>->f8!8MY8UG;vBkMB}fM#B=hcpFCwswSa-t#55j6Jb6`QB{!k z_?V013LOLkpdbJjn%mfrV~+V(nc99nrkU6nt7{`Uv>kaVSnF?6pTRuAB`% z?x;z4GO}uT*!Y(4&wgU!-U#TpSR1!7%35HFLpY&gf8cTxCTtu^?3Y9o^j#rbkKJ5e z?YH-F4PS|&v|8+-@1re{!;$ORV6!h@&CG_zJgm}&h9p>qwgeyrcYbC@NWEn)PSZFkQr%r9y^jav zCYQ@fToSk`5=UBC$k*jgYxvfO3A`VB{Fl#$Ze|TBAP_R`q(Ya1{3wXVa6e@)I%d9VNb^nEvUU0l?38Ktc)NLkze zn)iNQ+V*lvi-g4-&jbftGWXpW&kI^=II0QozmD9`N{>pE5YnEU{-uZ`Q9Utl&`KYm z@g0;?kYnQGgA|gBNLnb@# z8?pC9h-VX~jvD@fMMlDyXG^uErXPO)Opl3x!CDvSV6Pkn?_!UIL>8AezqDi#8!Qlx z@DntTcrAesF<0SjGA@bY{|8@v1d+f++igEkwQjh;R$tJ+J^$Xap{w1)%0~;jqh-A%<7@t^M=qs7Xm0F$#{f1uUQwr?Wd^{It5ZrNDnz_>X$gY z{9_<#mSB4@zB>?+ajPzSQ1+3PYN+#t2h0=pem(f}qVrk)3U-nM&N!+pTh14is+c58uyn+51X zJsi@(52d6Ggvuu0w;Rl1|8*9%zukh<)vWAZ` zl(4sW7%>8k>b)h{{0$T(<@Gm0YW|X&@40KwXs-wa2f4m~k0}M)e?&9zq>VEMa_4;< z43B^Zt(JLXD@Bb=t;$%x+ZLZlVn$gs(uIlnpeUvZy`-L94bzdZpmzKIJIVOjWN3K5 z*$QBYLE)1$%h2#BiVy4(?H6{+#XyRUQO;!y=yghA>%fpj5fR94*!pdA4iG$+facCy zax!C(=nR??&T9NsSulPg5A-2iXW7LpWrAo^DMUZC3LqcS^}FRy7P5Ix^jGV@C8zTf z$xLnD)q)ej>&bm(N^lhSX1{lbXXD^dx?@8JYM{cPb`Oz%^$|_Va; zmhVDDeqc|DOUNzKit)a}d{j=YF3m4bx@F|a0hLab=H~~kz3ABBoQ{BFJDE{lNVv1* zViFb0rchF>qVAdmg`{q>M;1`2>oh1?+@R(Ld9k+Rf!Xiu>Sc(d`~;|a;4%v)Xf&Q$ zZB^ZN6%0EGk{c1mc+e~#%uPl?Q=#B@Iq}Bha{6y8mq4S|KvdNgR-~suJi4r^LJl0w zBvu@E11puEa^js14fYNZ4k(KiHZ>)cpPaPPEhgmMiU&Kz!AWQL<#2Cq9_<={mIPuf zfVWWyK>ZFuAPZGAiKpCEDyxj09SgmQ#*dhqc<()_=nGa@IJ+*p6BM9rYVv6Pf*Z>v zom}0UfF(}5(I@QPKNlIio*ph}m5F0vx%A3TS_-xpmdw=D^zE)xu(%~Z5jbpX+8R>l z4gGK?C+XeXCWde6w4o*Wf4#t#WI?4A{U3xR=dxIG z3G)*q-9QyX+*W5d1>J;W>1LKt`uL!fLC^ZqEF_>R>aNcZ->iPH043CrPy#2l9;oWT z23EjjGTX-h$W)-_I#Q7&q&AHcREoxgMybG81}ETKu~a}On&qJzCU}xXi{YRWMx5UZ zs%!E0=X*IX`WCF_c)T2DQVLxrE+#1=X!Fahd82mU+11t6G|aE~vb37M_|1eLm`!ne zgdkg?iQpV3cfNYCI!=iIG|QjU+~Eq;Ce8z3|o^ws0Aq-)MwoKu5mhmln^pGY_AA4u&$#FH zcMWogy(o$;mkalf+v%FZE@3jVG?V=Q&`7YqM^yVHn3@VSpHcHon?9Ut(9l%&Q1mh> zNTb1k7EC36ZYkzV^JeDLvr)==97&5;#ReypN8mA!4>V!~Ta{2ydSCTIK2PC4CdIQm zLugXn5*j|?P7v9F0uS-B!xFK+KQ|pnq%=u@?L6lDI zLm79O*Hti)U44ss?x=2%f#183lBNJOrTC6s?`O#{b20W_(6Cx-uV+b#m)hz~uB=th zA_D+g1EpxUJ$knF1d-(jcdG2;05OO({pXq5H#Utyd*WYavw+XOr1nhnf8?^M0QJ(v z;SE7{we_fCbDfPm=udNqRO;-ZojQl z-p^n~`25{yf9k4FgAM+Vi-zLz>-OV}NYbR4cAe4J<7{8xMon*lMj1DH3=8z1=N;F6 zzW5(BV1u{BpGvbr;tm7%`4p8Z?aoZ9s23*_L-cg7oGTZFs(q#gAOfxpxE;X8WxQ+)vn4mzA*N*(Yici`>*iStx>~ zNAmyCbe2JJv|STU2*KUm-QC@tu(-PgcZU$%-Q9yM?(pEj-QC?K!2{p$R(-#zg(_xt z=01J8uP#sTQRU#~R#C~2h7-pNgrOB5xBBHol+ex)2*wO$!3z2L{&pW{Wu5UG09h}0 zT66zGI7awI64=m$1xvmHgDWF&LK=duuX(R*n}2EPInZunVd31-(J`?bp{SD!y{ZL; zmL+6|L4O-UM*Q+ft<`FU3Pe>4z~GqJSfxgFOtx=jZHJ;pu}hTFMI+~fa*dKRsw@4C zgOJrvLwcCF5noPIP*D*H6d~OtbpM0#lYZgxyhiz8`2gP1QnBRg zK;+Ii^-4_@Rn;JgnU$4@#mJciN9fZxX2taF>~6=~?<3}ObN;5l zqFuaE+_l}me8!kXE5PvI1`esz%C>&lPj~#s=g0exb$h;7vHV(f&R=t{s7)2C@whHL zOhst|bNi*!Hx@1QTz7|^p~oL!Yh4CXDII+JE|*<*D;Z$=m!rvKeLBMj?wMi zZH6=*m!|K=2ic4{qUIfz1-m~bT)a^Cu|=4q{HvRd_=&!W-5te!JwgOXY)?p^M<8Or zY{8$_hXeh7HGN7T&0+)|FIY-`Gv71W%+1JRQWI$n*Mw5Ug3%gfKhGVkBvVpZYwD<| z>dMcl1&XP#a;6+j7Z3r+2pKnZ-lnD|AfE=(R!?E@gn+G(m}@qh;f!j{_|HvH6nd>> ztX|2x{W#Hy_=VDZ%RNIcRC25%#t@8+79}8-Mn0?+KHlfuiYu=Ufyl2dl=ZFgC7Qh4 zitFDlY_+dCb%RAydfF1nHOegag%Q}vyG{(J5~74-woWTg)OmI)63YfelF11~R`3|vp~0K2H-TK{GNU@Y_9D)Up~&rP0tjw+Pkf70x<{=~Dnr7UDv zdt^hwl`z}NA5lz*yi9bz4+;2qcW;JT$U^p$7skrVD~c=X6GtoPJm`DG>Pwc8L1q*q zPqMcuz*trXKykbQpz3r%kbR9;EqV7lxJ`;q`vE;x=|z!SnKkGo?vWP*P!## zAM%mWrMY}AeuWNeY(0BO%G>OJb}IuJ7${*#Mn@wwxSeq=*646sZ(OG`_L2H_UC7f1$w zDqxv>NAw)r9k9T?oKszNZ9`v5JcP;@dEH>Xn9nZ-^97MiLEReT$9UEpDmQ|YEV0yEW1|~y9GvOAiV~&`RI(As(+!rK^J22pjKq{|>E_;d0(kc@1 zuv)c7fgz8!VvX!s5fb^}!umFX8lR0bo_Uw_|9l?m{a)OKp30|gBGU}V+m3Q43agbe z@$kUfL5=tbE|;R)b9_ErO`M#ve-luw8|!Z)OT90T)8%n6aq;lv5`74UB7S}&7$Omj zG?pQCfH4da%PlXbW$Q{~(e~Sd1|#08sEoKdyubr`+{xvs7%j*J(%k!gR$~# zy0*AeN6_zlYMl_jwPmqmbU!*=>bz2wlA3H~muH-2#7GJ_6pFDK zi%FQIWF^nNJ{vyvbFz}3pY|aR`M|vL1eKWt!?BdhkX4b}q}Q1|;(E@|x@h^FB6`aBQf6TM6m#R0qos># zg47`k7Msy5JU9&+Ei6RH)wE-%3C<9WQP1r`p5~*9gzxnH_k9tpJ5p)rLMtvV4w#o} zEvM|eo||fBR#!(~y+N z>BiciYxAvL#F$RPOd)Rmu2=PE?4%b*3&+h_Qf1ql7|hRTsm2m{{I%mj|FgrkWegN?1Q)ZwoNRV1V> zr6CHazBXW>w1rGj++pYW^Qm)kuDh>sYd1Mle)4$RRstACA}%iMxfri|L&D?3*D&3l z*P_6AfB$5o!|P94R@(_WvGGV^2+U_y^u0%a5tPSZl?2VMT{0!@28gXYg_^}?-k&|> z3{u+}raq55irMuRh<8UGI1jkl!TE}@qZ*EEm>x3db?p4u5zP&a3n`1Mza%2&qXi>P zAcXVF>mmSzke{B{?*oDBJLty-kFGQX$BQP1{XOD$858C-5|-Q*Tf!^h*u^^VHl$V> z)z;m~#YKS@LQ-AziY?$rD~#Z|!jOL=*<#rxa8#+DZRN1Y#6Pi74k_L^%h_f?pN zTxsMmuWCdyLfC-JAT4E04=4UgA|s=1;YuS@F+>x4Q%sRGf)YyFvgMBN^KiL0m-j|} z`rU?hHDgWL?{p3bO5Me{nG?+Tw5A32ica#}DF9b?#Uno~jCiFlhWkzU{&HjN)_iQ( z!sUFeB`@&J-{3a!?r9C*!DJ%S56~{VEEhKYOPK#EytV`PeL&B(-(12<-7TY&U0{LQoqAi35wCt9bLdk_ebV<=KQT@}BuW;Mb78^LF)~%axuO@#gEdDt2U*e3Er`$$NhJ80lq=;I zm!QbSS#TAwkGUzpv>4vf_T-gk`O;R77FzN&k$asuIwU&}`(adL=UsP+2HYgm*KGk$ ze0Tgh>FnJZlJFe}y;5+hG4gx;kzj$3;01G~r_zIhjb(5(x@_-$`*Qcjf7chCmKY&` z?cMp$vXi?&TT zb#)0q!Vd}xstw3JW^A#BEmUftLw?rDfV<6hnKV921q-y@(E@W8e}-^b%H^dJ0J%_h zbK^u7x<+}wFR4j?%fpyooOj&$dNjANU{~R{4X|-2yZ&A9kH1z#wq34rJFTv&0{+I5 z`ufqX+{~_<+PecO-tBYwn>qCGdJL&wB8+lNzc_)QB3^Gj+s zx!s=pE^8JI&Igf+n;lAYQke9>fVR@@Ae@w>+8oAoeSV-OchigPmt1!Po3va~cZ-it zY-whjoVV^rJd1oCygl~*R%6zX0om)A_4!NFOS7O+xMpIbFus|~++^DyhmzR|5$k}} zbLH{FU25ScSYW^bT6p6sD&&4DKuz8Ks;CfyG42d)IBZ+*4j={q?Z*9~4U7c}q=Iqx z_wKPof};TOFk&zKZEpy`9ThY*pl=~FJnvE-v5EIRZvZn&7YXUiD#h2iB05~K$gbIo zAZlR*kN+PS;g^*l+qL$Xzz7wC`W@TLfNj!Z&B;?$rGl#U$(zqGqLStzCsdS(R1Z`s zBlw~)y}dzUIQaNQ&84ZQnL49mmA|U1i~N*6YKi;Bf!|dB`xlIL%(a}K0%A54WqjF0 zm8|3scKx_HqSVCH61qg8it}X&s%G{7P|yx&g!}$%7GuaviF(K~vEq$r zjfmKBq0T0a=LyF}+(0lg##H&-d^ahfp=UPLkY@+{-J~$SJ>wI61;R%LNn;1o+8$TA z{bGf4!KY-}!<-|fy>?D;A!Koa_c*Fg0)k=Q$AZ4;DV3@9zE94F6FK5OXSOds@_zy^ zEF`Vzqm|oXsAqjCVHIcrhY2MZ46f4Z$_5%dqs4@jZO&}#GZb$17_@4Cj6}dfg@t+@ z?+j_xJAMWX9>?eBpQ2;00gvgZhSG{cD4;BZR8=RguUP<+0+JXlP-daBxA%x+Jz#mnr!uI1BEuNDl~PPTAAVfpU$Sm^R@1{AZX;7xixvO#e9^boR*(n z7Z=R|+4c`0P!ekD@zoV|edeH5&M?7uFfgeC3x!W&KKcGA-!zwgfHHsKkR?r2il@#( zMxB)xmIL<CHbHZl7>=p=v1D$OCsm!{QeV6w2D^&@R z7((Xe=JQKSKxjycNpo<205VC{0$l|-GY?mI=N1EBPM~r@=UC7PfVM_NIO*tUP?$lNmHpgfnc05sJ2bRxx7oX;eYzyu*8-+@R9Hg_hj(%QzTp zB7j_^`g24E#29njP0{o4E_}Z!Fd17U@bIqj*LgC-Z(-ajt!iV{PCGG%aZHM9aPT~f zm5Koh9fI`7g(E3FEH=6riHHn|2pUA2F*Nfsimk0F)>WmMY_t1FqDAAR^W@28MSFmn z`U7r;X2AWp&Huk>BP~L!HO?*@Y zS^HTPx#XgU_xXZ6hbwJ+QNLvl4RCmNt1-fzRq0gO6I65rVXM>a_jr1#iImCQ)IV| z_ESp&hZ>F59~D*jU7iDn$S6A~(#3kj#?w8iuu%-^Gl1XlJ&`xuJqMnb7pn_;{!iwA zkQJz6P3gJI*_SkJ&o(z}xWoGd8XORg2U-2_zEsU%5dwp((Lv-vcHMVO7}TCg@()#b@IrZMRph4>f+AaY_zlPO?TRswdO3hC zD67XL0Th)oFmXhK?i&cFL`pK)i)iAZ^Niu(mY2tzb>s*8C}m(oF?4`NG~fxcCk=o` zsyBR|9=8I*b90$ib#-BjP?F4_EPj)J>cYb3*}J2Vh1pI2g<;ATBGTdQ+k^eS*HE$$ z#`OPQfTu%{^#Mfph^%gx{wV&iJVcsZ)VkO=b6imjR?|U*(Sv$QKU&GmQVA-@b6<1a z3;{eRF%&=oqL7!JO9qvX7ZiwIcHSwq`?TYx3y8e;{J#(KIH?MKKPT+P z2EcDkK@QAhhawVWgb{(BaEGGxaXsOoz$wVqn*Ao*e4A{U zUI9@fl#u&N-mhn6(x%8+B^9`(CRzL>DK@BSIEY|W$Y4tWVW^a=v5caXoi_0WVt5V} zurQo@7@e7SsmhiZSoI%Kbo;{CmZp@#0zP`|*3*hgJ^bhm0?#MGppZo~9Kg;w>b9gm zgvh-)Y$^1@>+e1lB+zvu8a)GqG5&R))-T=c?;-DZU$N(M$if8u&;f^nBAwgj^ZC>f z;ESNW&J-~uv8WLCVU_rqf>fKEhW%6LlHLCrTy%QexuenMga=8=pDxfDqi7Lp?wOO9 zn3qqci&{!UuWl@Dc5o@G;tQl~MGwC^zFrlL$ZANPb#|{6L5RUL+ipg?(kT!U=I8;G zv*L`x_4V&HEv)l4ZiOO;Q-NuX2*K(b_1>%QbEym8DVq2OwfyW!AwNJDK;xFP7@?9d zPkeB}MD`tGR8pUswuMiAjOZwfG*M6UHqa+T8bC}-Sy1bo(xo=*$vDSB8E#@mGji&N z(KpJN3by?&t%+SZsrCEl?XQahQTjM-gR+x?MMoWcP<)fk(lu}b&ZCSglVlAIrCHb-#T>2_yR~K&x)duLo1&7EuO%F`P%)S+u=M8j-m{-b6{8I;6(E-_ zjrZl0W{rZS@=!s$kXq~NqlsqNk)Wps({v_E_f zZ|WxU$38Zfi>X{^)^sK2Eqp_0j1+S8eNx8$&}Xn;WHa-d3iQ4U?#N~4-5rqX$TsOn zW{LGpzXhasBh_YPgVuen{&RJ8onGUQDCjK+jTSa{d=Gqm@l_e(>%c)!7Px`z@9(cN z@Cyynjsgl|8sT~{M44f#I1CGZpCON7tc2Gst>FI*8dKA|sHDX>l5pM)JVMZPOS<3kjy=gSG4i`f@O=6WLskX^FNPBjifN z0^UFAZ)|WhHDTk3_e;siB_vG-zbF`<)WR8%y|+zyrhtGt_&5|XP<3*KkR9KKpkVTt zITCht;B)CV}<8NIkkMwaxb13Ap zMMdT5?z$L0;T4hX4?PGCEI0%(HCW!Gh0umC!y(`VX{<(mH5w7;S{^Y;h`;Y`dalD2 zMVXzS7gKg1mQQ6W)3e@fM-B+5VPdsA#qb$NoKBXmh@PQA{6L|UmoNM?{6hM*TV+}% zW0f5nHbWLxLPi3&2*GB`;44ln0Z|d3FE?}NBieK65F}%e-a8n&e8FTlUjbu5mfbuf zRqCKuk6&wU>EPDLf}#Y3zSewEY|zBlZE9!Ap`J;TBBz!~w#!p*hEAR>mh$e0&4dU; z4&b~C43yIars29x%C8;Ws_hSgKSz2Ibfe&cWfOz50Hb(X=RRd~ORe!F1G8zmMNHYVeLo+1SC#TfEwYz?(D=YPPPyXbLN{>pG)A zz!Q%ZHeY^mb6O6aj8)R~VCI4wHJ@ig5WzMrV^Asc1*$z6`n7fZ$?)H-Yl-Jfp+CHL{+wUHbLRk_-O_?sQd z)aaaarsTYRd~C58d-$ zz*O%B|MC^Ie<}!)*>%Qk9C?8-$I(VpC>6mt7YAbk2jY5{2Lmqq^9#egd z3U#1vSu&9dMx_s@e|XqQD)dW*sj0+)%FpSHm?H30=|wuC%`l3Rt%}G&5};r!3^f%hrS@ZNFJqxI9lC5+bD6);4K?B zp3LTjA?0xd>?SP^mV1Lkt#74Q^+uZlhf`TVG^4@V!A7$ENTlF)oFkHqO%uO*>18=P1bx<+Qn>*J8H1!#)`z`V%W*x&~R zLCZ$>+S`0p!CB!umxz4EkBRv75{(>`3S}<)cT#kS2UDn5i=ZGx-Kv5@UF4gfcQ!ym zze}L6-cBxoZ@XXhxY_A`!|Cw2pp;bwkojW(IeMh;tBZ9-u$pvip}Lr^4}WW91S%HY zBx2i_kqeK1@k87W#&4ZZzUSz~r2$4i0=xWIb#-hV3{bYgP0@igv8cdvhmF{4nyH>U z<)IfYI`#KijMN!xBQ6z!m>JiP#p#vxUU-AX>?!|CLmWO|NKz0MSZjFpAjzlLK_ zi_p}R9UT|aeh1M7*@*bj_dk3UGZi792e52LeQxPj2$0CUK?t4Y2$$J#C=@oXc6MHJ zVic3N5Fz2k*YVR>Y4g#)KXql=i;4jE8y4$XvMLXYXS%6$TrH4021>$+S=zeetX0d{ zOa~P#CbMOc;@TCZyI{wUG_W6VH7b{r8WyYd$3EPQ3*YCzbKV2te`JA262gm%D!sO! zK!y3Lj;^&C^ovHj2~m{IVS@oFWZ)VMrtZwETuetQRMHxzce?+=N#*hB`EEehyD$P! zgk(KitcT4ni)r|L%l)`+DZE1v2WO1dXnrRD`JbO(cEFogV`Jm=68DH50cXMIll=8)$HctXxmYC20DA=m8xqPy>n=}z30)cvb@Gklxe(pt(aBtv{g>+ z$qe|9>%+`EM$(B2m9ll|e+{s-^b9(!@fi)JsG+-%Vc3fQ8I7fUI2HV*+M!Bn5G#rpL_7*z(rvI89A0DOwPJS@*X^ah5JIRF*xQE}an27`~%xz=D3j*zxy@SFp zj}R->Fj^-Prm8}cVWg|>L?^MSLTK^>9i%%(SMs(V>7$RO-L+;0$wor+Rc#b=m48(>DfxTTX*AwBJ1d84IM8$ctz1nDz$wht zEJe6T^fO)_t~pF1P)QTKUL8$0bR2OBJP7FM>Zr&8Yvz%v4E7a452OQU^K64_p!}ot z*bAEl7&LWsbkDCkX8Y^Sqr4XA4dnoXIN;F*(kmwt#xl~<&>%VB#{Brn=m~uq_K;9Q z*V~{eS&z`%o@RAWsb>sIKEmHRy0B0vuYc+eCVQ8J>jCfoIJh|*jQc~fJM0CCU)KUC z;y=@=;tiIcJ%C0!qA!!=?)VFJDw*`=Z`N3qGw&YnQi)F^1v`^-+y z>9@_v^81_j{PJ6Ke)hhEF8&pO?6A72bFoE4P*?-u`*r(L zXGm_Fi{}vkW^cenTaq?(T&)quPMtKdsjcfc+*KGFImYO`>}fybEWEZ=51MW;LPS3s zx8DK&2XrXyzw;E|JN_0FUU$|(hix|`RNYT*v;ydR*Q~d>vmcM+Ft)8No zfif~aI9&lMnrL&dOxPWp-#sQi^qTnHJx+K32U>R5z3bbn!`shAjjMceQO)TU98+>y z&;SV>yVr%W4vFgGDA+*rqgTTGrhF_z@C>6=DQ9#XcP^q3O@-#<+^W#e8rnul*WzN! zEZmh*lN0(tq;?Qhu(j+P8@o-q6>&mfeaf6t~&|2^Ne zUbeq0;z%idS!EqBq2$nBUO2@svzBWs`HK6N8CT97j4amJOXvC-M;xTA_wr3b#Q0Mz z3Wb}G1C((%LN-1q5iL!QiAG1t6(zWO(ZO+1&MUMc1t@oqfGR=-)`hHe^0y=odu4Z1 zib(=5ASIwo%i9Nh?l!8%RlQLm#5YLlfczqo>K)blOXcGprq_QF}{vCoYzF2E;>Pq@*rss~wva>^QCYy)e>D3FM%V`4C;JCmOd{m44s zEPyB0TXX0qRLNrs)=r1*3*ju znaF}*C4uHLSP--my=c4Mn6lyXaFg1i7ZOjJF&kUE?S2KZCVaBS7>!wV4G``h6$aRS z8R{2?9XW!l0}W|Ambcx9nZ0kKcmw=$dc})gr<93|fl?7y4_8nu*(^;lxNK~(lO{P& zpmO0so@?>A@kYap3|pM$H){=Gdygk8)KK(%+zETS#da{7DQ`&ccEl3=_n)hqTLO+G zPN*3Ym%1FO+i>gVxFM^+*&MoFAIo@y4H$#1QM=QrcPI+=G3qh~(-~*Q&LDY_w?VZv zgatud4K(FvrtFQ=|Cif0&B6c7dkiNeyZm)!$`{QP0iRoNGOYRq8bA!h0cnM3jmVt@ zRR8C>yqmu@0Z~)__jAC}1&K&tI86A~X>hROY4CK_X6tw07wD(g?tl?4$~Kp$usHwl z$yj%vf388(LFB@mi{y8F#Z$+@6BG>`V^?h~FfNGpKy;$^Uxo_C%{n5!R_FVtI6L2H z2fr8Z(73qkLixO*aqGF`%ge(5iux_Uvrs7YuoO0(ejwoR>%|j@0Y{(gT|yBW#?4V;C+CoH^nY;wnxDG zsn;2=lh`&E&vSv}!TW0N>?(lxMBqHn1JTnHAMQ-x+zE3KHiXnSvozM){?=zHz3}Zk z&%s854b>oEjuq7?FNL(Hm03V=@O}{P{mU0I8W!3NkeZtaLFlng-=1#EG^Yz|1QIIF zB<~9XmEw2uNQA*+&!^vIs`?TJI&3&*C}@xgm`B{HG$6uCr}JTBD#g>18VAr@eEV<$ z^G~5Neq6I(2@BtET^jh0RIw+hQGSvS4SG7ivjEX3i_;zool#PsxR`Gboa)9$V*%NQ z7)k^H3e)f*3G}fciX|pAcf3XcN^J=Qc!EK0P|#t=S+i5mcT&%gc zPefw-c`L(cOs}p{KeTJj5rzMqY(z(^Z`i>7Qio&<+bEd<&W~+~ch>LCQCZJTJG@^6 zy1eMQ*FE7YCwXE(80f7eS2poML1Dmn*fk;Sf8MX8*B4(*t1Qr#Q{#8%E%dUY40MJw zHQQ}X&J+#(T0raQPUM2I+p3GI3Ap3061HJ9CeU7V>i5ehtXksWD91ykW}$UAa_L)SQDL($ zgrES_GQPKZ_HmwPf;lBrMC$^=hM>Rg8{jM;rx%NmM?GP%L#}rXp5O75<5X;WZFJ-){&izhSA`_-eBZa52K0 z(Q0#=ANQbMzRmz`+b(zj{ZDV& z9O}Gn^Di42RnmM|m=U#{7oyg*1PQ166-(9fVPDj1P6p$LzzJ(O9nfbeiUHIQH(&E>UZ?0`c5lmhToii&sn$E1xdGNB)dHzHo;) z8xg=CMy_wgKwlQU-+)T`b#}Sx4*;0QBigFRRPSsEv7mh%MZ*!g$Jn0t)x$EyUX#+e zlU?`z;fx=5S*;{=H}x*#;lW+P(;Vf$P2>grLJO9ioo^0xBsqCE9pr%g(O?0r^!G0FHYRj8pP4>w-j*)a zSxi*C{)?&0Z1-YrYlwin#bZW7nqZ6Cg)t5~uhM4EU^RsTDmP33A>#GG&~Rv}>lV@A zel`sygcJnR$3o^ZDYqF4k`XP1+f{p8e=J`As?dSW{+ns)D`$%NFb!EWxhwcm>keWk zx!h5Q$X!Unz-59hD+HJkqB_4Fu7KVDn{q1O2U;R|g1jM%o7&Y=;d926ag+vDwj6=G zZ0qn_WG%>|pAy^Uu-#=wJ0f8wn3~up+2h|3vh~?%GbF{1d)xAc2fJF=S?n(^`!uR89U?3l3%S2rV&XvAbJ|4II{9#Xnnq8teE6`omAb5FA#-q-^ zXE#!NI@(f=H7&B%Hj^SsRX6YrS2h?p0B3wKung&DSX|k0uO(nW`|PORYQ>dHuQRp0 zp=@?(DF{WfG89##TkAV>t7AQV_{i_&t=rTVEycAS@!rIN1zpBh+}jXl-^}me$$!Y} z${4?a*B?7DwBI5~TaWRSl-1OLek?v+0&<2quBK&9ou;LT`EUr;PmQ zxwy&vN_x_L>Ru#F7+EG0hnklP;+$OwQVuTpy3{1}Q^<6=?f^bpV!kbItFeTpE`0_< zgwFRrnwHBtR)XZ;pmxGlunLoYsU?*_(VA9Do=+3#*{Q(1H1PbqywuQO5Kgh#x>HJ| zC;+5a%rZvP=RYTn&00Q{Js02b^#vRihPvMTb#(b76~5r*lkY&(RIWuLP!7gRp`)Te zp{3xKPGk;c9^u5s3Jg@OG+5bOkBl^6h-+A z1rC|}7Aun&od>WkfVALA&io|2fh&`Is&ZLJSyXH+k`AA4`0gh=wyLPOKBJH=x6<&! zcxT?0Ij15gb=1ACSE^DFEkDh)e@%U#Job50I;#uRP#p5zg~QeE`rjj`QQETUm4%2! z0Nu?5$gjU2?BDD^&-`9|Q=qM}_zKv=n>LH6{oq%}y&oT0i2p59)1=WWWO1dG$YVJz zy5uGq^3&QvWmzvxMT+1Xvm23WH6E)4#{*jueT7_skj-2JKE74f$< zTa@O1rw0l->qFzX%@0ig4Oec(UMBeEoFtxi_S61f(CdxLpT~;v#J5-=q0iW|E@mlB z4SzUQR8x}&=uw3&Enz2fWvDz8rh2?k7qR0aDAM^&P0F^dQNF1sQxdH<=2zBby9dSC zA)ijT>cr&e1&kY|eNn~8zGR$n$(J=BG#G=`CPOvkIR8+tz0;rWN+K@v9X$#?CRrP1 z;dI;n{|Y7$t1ra}F;E3kO#3z}IuaX-QZ19s+clY7nb1^%Zm^zDI+jq9I`W&i6~fN} zQTfjx$X^OYm@;H*yW3bMH`v;FV?7d)scw*&DDT%c7UP-nK1UR}tHJ}oXHSlE#md{l)D~$u zB_)gx(@XlyGZq77U~;;-VWy!-BMnqd3D!VmKqxGp*JU-G(}VA9Do@G50pH+N6gDQ= zC^}?T;J>Ao<;F%X;R(D8>!Y>Dq)_jC{hrA4Id*5eo^Op!&50DS0E3AnQVc*15Vj2_ zrUSLo0|1P{1%?SmTrHDF)kXerXKnW6|Gfa2tfrwkw@Ksmf~)ECXFxGQ4oWbC4olu* zD%GTR%jDmS_BJkeV~_n?sBm1s*#RkbyF3^}Kul!p>Kg4;(gv*BI6@Kn2|d;iw8|5O zz0a~vdRlOwhDmO>x9czs%b*9pe$Kmi?%6xq>Kpp~_a^FXdK%?J3;*lji*Co*jLZcC z5qyK&;FJfz1#RzRG4*~ed~a6U79N`bb2`f;fWFmGpKmrrjU<)TdXAt0Gp(GVp&=<& zpt;1J7ebemgKbnQ>++_yitVWHnMzTblPSN3lEkb*MrAAfh2Y^~?_*?$NEx34=Pe71 zf286};{KN^r|ZUKHt9-%l@TH35VeyCF@b-a?C5D-r-}fdo?vnUTnQS{HLmhu!lC_N_;=cp;anNYAUq`e!w|E}!Snc12w7U<+4dDNNAM%Q?mV_cjxpXNR z)4B;{mA8AhEipOTY?_2Bu$xW zFycdegcrxNJ)p(-ubwk>34}Y{N+Ec|PPhQV=25R%J+G9VKUW0eA z&!bCSlwWJ@Ep8+T$jzbKOb@t^#jyOOfLDV@+*!ZxK%jN)k^;|?wHuWj9d;k>3<~w!v7*g_?YEffqiEo;N z#WJNHz7VC^aHZEiMJ{8h`wiLiPugMFY*r-kpa%}1N9|Qt@o!7B+>5i-iy<* zY@6HLXK>o`NBjWzWWs{*PNCek?(YEz_|+DpL#s%^4zL;&ELtWcW*rWs^=; zM_<6wDnz@bf?TMyw5RhL(x%r$-p$~WVXt|OFRvVY^>e=mP*tb5d2`=#_>CTsQ`WR9 z&z5P4ruJ7T7vwBS;{}~^iiMeH`qlcQK2V|T4~&q%5x1WBR&jED-tJm<lz`)#{G+ zmV2$vATWj!;FHOOseiB}<*J9fn&3Z+EJJ`jE|3%l4*3(+8h+bme6jR1ha)@P!U%j-&5;z zD(Wb#DNV%!;9wS_Qa%g(B?o30hJg077&YgC2rAaA0K9xVi$=_9R|orWRyr_Qb%M9U z{Q$9FrLx4<`&G8=nOx=+H;c=@NJo*jnBZbyB{lD7o3$=Jo@P^CfS(`zmv9bebxnuoh1Z4$z*q|k3R(fJ(;mp6p7w^Rfrrl3m-3Xoq0ETAbeei7 z4BoL7Kd3(W_ncbR5q8kipSxP_8bKxDcMN!c3X++H*ygw8Vhp#o`x{JqjMTqD6U`JdZ!f0qkV3dR-k3NCk6gJNc68CE$x^=97r@j zN5=yD)~XnE`*|nsju&!O-=njwoke2d;c8TI&Y55~lEg$E7wj6J{;YIR*g zwxV#L2|&jbWm*4jwj|2y{iWvMWj;#aCP~V&I*ew;PVZKzI zK-Z+Bz;G3$Z-@*YU}9 z8EYt@VDwGbB?2pwz2#z2q@|~XVqA}Pe`?DrzARjJOEE`F(&(!~d2(5)$WIqKc74mN z>gru^@1vwS-RHnzBSXNezJrTQvvWpA6}$it71A(G;!FX9z*s7S`BVMDhr`ly4>G!r z!Q{b5h2ZFtx=ZMUeOYSw1{-NdMswEAN8k@N+zP)bLXx+S=U;lmK7h>!+&ciAQQ&4j zDS3>wrtRF^3o64pYXk`q(dluTbkb`Q4>GE!}ofn$>#!l*ZzBs#eVJOMCHOt~?{3mAW;yQHtG50?2viEA~Fr zmByoJ9J=rDhKK*(lpLJyppSRPUO$OqC~#^ZO+m#QqnONb({t9f$?7-sJM4bKcMGOj znUH=zQ^{2R?}rOeK|H^z8cZb)6S6VJ9|MX*ffdCFnbaTbA{%2vnVwfGC~wQS(hCp- z3ZoAFu6&5)v{ig*DI0*g<{;u%lA$=loj)mKV7R1Y6i6AnmoJhiBz@uHFUTlRb?TUK zW=cG;61WF8_Ln#!Br%^8jLl9-eqkX1(22gJxUHd%Ehpjy47Sh7JAazS9`CxJ(G{|; zjiHL{Hk#T>VC_!bMN?3uMXQ4$A;6 z@-g}KO6vLiU2t`6EJ`lTwO*_eN*kXTUyONRGrtwF2bZSUR7Qx^7w0;_{U>2*PcY5m zOQ>NU$1ul|?pvEPV}b{rSrXKfr58CMf885B482%r48S1tmz{>KAjMzFr8Hr{Gd2)t>ty zTP?g=Dwg7|D8XSd6boR|@}%>M1W38i+1{rnyF64AppQoCo4x)&4~k^bVie6-NtCdG zbMXt+)pyfD$biK@Fg~O+-)UL{1rlZWr#l?dtnK6Czm2(#-wT@WtWB8(Ok(w*jByHA z*Csvxt|DnaKd5DJXbn#~6vgGm@~w-KF=p zjg>WHhk?k~&xjJtl|&o@0)4xxX?g2z>>*dxj-r)y7F5Cc<>h@J^U?YF_Hcq}Gy<~A znq5nRd1UFKMJj`Vn58DQ9T==O>SSP(B5HJ&$%JB60yxeNk7tUg-E2W{fV={zYOwOD z*>ZrY=Tx4YqIke{aG-eQq3OFc_W|o%H4EMOZPyPyA^&yf_0*{sqfoW+U zX?_)qI{OWm$2jKKD0M353=6;%+m;{b1VOs}J?b(ym5ll}9(^@(@B8?nNm*iX(_{%W z7&Qd5!*epj((&=w$=`SCzv$SQdXFih55;+PfnK}wQ7tQHux{heOg^MJ!^)HmhCDA) ziyJf<5OCaYO8YN*Zz$IJdRsKQ&W;J z+(0wTD-H|P3JC-&tN;5Tu(ezaMHiVWiXq1jCz*0BmQ7fNqeS5doD6qOS^kAiWNyT=DI2dfMI;%JiM6(|Ie2vPL4<-vR;gSlkBM;t z{#CIAyb9%Ng_R9CBT3ZgM}vo7wSNQa#ulO$YHnK_05Iv2!EV{C9x@t2nCpL`kykgm zlS^hf^z6cCci;@D^#pT9^2a-`8SJI3F3%kKGvbS4eX9$c7u38s9GZ%Ky?uK@xq_~} zRT+Tjm=T(aRTgK*{z8TU)@#fD5+sgPy)_DMd5fn21M6!OAPbWA|CB7}bmNccV!3|UXVrhmsQXc%Z0!H` zTTj6Z-2MLXf`}6L?01U}LPCt7>G&!?R~&1UI9~6U1%*OaP~snsq&|cb(%0}AzmtqG z5DM}@j(2Jx83cH(51e_o{?_oBV*2?dweH38d%nNC{p^>Hn_Jx+OmXl;{Lq?RlrrqT zWHfi2upqEnXEFonz27%QJk9?FilIqjF>&Fxv3R4%x<#(3W(fAO-;DJ))bH;Qz;Rfbo!RHV#Khv`UhgXBvq9E2!@DS5n2kY z)AlrBpubk$M}(?7th~ghDree-6_@4 zWG@ERI`dabY`v0GblYd08>7BHc0F6>I*4DWfa0DI|9_bIX*taJ1hsCk*X&eAJPHfH2d$zaRskdagHic9wGix~l>Akf9g4ue& z89M8oldm>;w#A0PST28-&B2Lk8zB6@Oa!7Si)X(#p|9ecl4t`GDiS`l zJAZ3BesceJgXr3ldbc&Bve3rkIGU@bnL!=I-4 z-!og*FUZ7FxuiK0H^(ZQk-GLSTpq1vjMQ>!*e35Bp4jr(r~K1oW3wL^SfEMeGwf`8 zz&BSnH^X_Cga)5f;##bbC(YlGuBf;!J^7Ausuh_G9GiNb+rn}%qIr+3*NJi&ZmWB#(mZM^wehP_pnf^9)$~|JizK#yU(=At5SBx}?0)61 z>Des_?T-?S02@(l-2j8Pp5n~@N$(4YL2|%QP@0<&vQh`8XB`m-@6>)ifXbH8GIeof z4`J+cb|^vwaVKB5K1%_1(dWb7#}B9kGTru=+A(J4Fb54OG44mQdc-!E2Wicl`EG8HAXT8 zu{#9^fl8@mielxmg|#gi{g4WE%g3>Gf6R?tOL3n^w^gb0>0XIzAXtoN`9r)+zBmiL zX)~>X3{1ztRvA>$_fbae+?7d;S|2)|ICxSzuvilD6J%qhjS57V)BB`^1jWZ1Js7gT z)>*_%+y{NHxh6*`r8l^y#1I8DW(w(YqcTwWhkIC6g@6TF$VbNKVt?Cei+KBhL@nB` zH#YqKjV9>|@-=F*E?{2pp5T9=QFMlg>0fY-I;ESAr^JPQi|I4LA%}4qz8;0wS4S5* z2G6bCj>7+v(7-t&e!RL&qZ>kOJ}TXbs^%!*UbcbqqLJo2XoS zkYr+Tn~JL_ax#mp+xg0rEXU?xzZhIPj~k+Lsa$bkA)0Yja&ppE_0RzhHDaK3uwga5 zb&C0S7`XPs;gy2EyDp{~_Cc!shB+CkSbYBf#NEplPd$9(VDO_jCr5wIebquLf0m)R zb#hc96&1uza$olP0LyN=x*9+-Y%1!i4q9!L?^7_|?(TvoJ-#q?w|xY;Sr9(SQ)VB(Wlr;TyEO`#>2G}QWE(?49A$o>rv)}buxGPCqq}#;gVW(gHotem-qn(>1D=P#LkNs z571(4WQP5f!|do!X04G%6S?1M zKUcwtV>Xo`^(iP|p5+L&-ePNfyPw(q?@$|#=ou{)z_G9kft`bbJqb7b{F?a&8sL@%jB{Zstx;7kx-{$Sp9!K}sA;p97 z$9*L&k1@`;oOWmxE6Ny(LzSUSFcV22whTP^kQW*W3rCOZwJnG?aS2|a{qx-K)LQ31MaqD(}k_9|Rc+&ZOpd9hZit~U?A<*@r+qv^KRS>)F*1~4qNr4MM zoqwP6>{`K=PDs@P(@vC|k05^a;{MOl{P^B9_~{Vi)Jxb+)*47xsJ05U4gx*&iD%)5 z$hq3+o5HH}sZ^=szg%5^e^1FmhYYYT;P{Vz5rF0Zya!_D=Fp`RIbwye6A^R-?V(jy z_sa!lUn%0{Z!)SvB4eNeOb1hberY_EG2bm`u>@TfmWmm#m^s4GeI3EjM5C z+G2Y9&tS7Eq8l&^Wh?;_^T;Ey2(z=R0ZVMd!{S^{zkn7@oOUI1Zn-l4AEcVEqL1TP z+|w&ieGBEvk=vZ#*pDMT`92E$6!=ILMDR7$nW}{h?(-u$WU@; z_b^*sg+@FFnms*b;~YtFH#WYcZ690z^I=19?(c~N`A}kH)$zXXJu-`J=h%)tW>jS` z!d$~E(kLnP#?O~d8@$}%Kn9pNIK=zBKGZvp$X&i%%O9VC%wO&-WipvVIO*GotF955 zIt&f58iGL`{{ut9&OQqMvzUr>JA; zv0Y~e>JY!}#ov_ohE7A0*sqGI0Hpf4fO|YJ-mgTJ`uBHm3?7G4?J^zOimfNF%{I0q z9?2_lJa+ZXWJO3d34Ar&T#0NP3%i;r08U7f{s9nG3l}TZFZ0!{TB+5di7s6X6Dznx z?bxg}MF12cneE)|(|LWelGYbGwQ_{{Y1lnH|K*7T17PXlqeVVtG!<6q1EjyImS8}6 zigRjRl5kR_)R4o(F>Y^^DHhe}hUnarDm3W%I$cqMYcH`%nDGyFM7Z`>hE4gkt*Kj= zb}6))6L9)>q(D5^-wi-YKwh6o%Q~)HH0n8K+ECzd64s`IP{3iCU9##@4Pu-y-KI5G zLJjv96d$tVFGKx{s?=1_0DnN7F1^}iECBMp4J(iB%V?B4vKP418&(P7CIJ_+)?s}% z_l#y7gCX+=DIUz6GSdx7Q;{Vn8(|m0NQ#Zx^BF^ueHy#lAOl(|wCE@}fZjp*M6)1U ze`(Kmcf7<$E)FXkuYiy;9iyShBmR#^qI{jmQ7g7y1=T=YHqJh1iGiBcdj zp+u4;y4~YM9%XclRvEsCJmQtn}j)*q&BhmKW z#LA7XqGmG-i@*1N>$p|sV@FlyHc;!cvwiJnOZ5i3r1U*+PIIil=XtcLrDfGzmgcAE zubba74!+2c3ujV}8&Awp^cC;je8-=QgYwazrT?8I?SYk%1^0P6qySv`6sF^i{;7w> z0Kq6>aakBRc4;cO5MpSg*N~l~CCV7beki?nsiQ$0!zyG?p!;uY3n&I+H)&rL^|mP$ zBNYINA>|@*;WDzx`{UY^F+@P?k49;^2X}~!3@uLUNgL#=L@(cA7oum#8M|>((1s5bu1$NiATks{rf;kBYc@pEFtZyOys25o zh84~}_@UJHa%#^WNw7z^pTVf>2&?G8p^}q&Uf}WL0G>DwDK^u$TpE zG%_VhesD-aF%$b0EU2hZVXVc*q8bKVN~Iy@QZoaLG>qz2Ovd_XGZLYbdK}^xvSh<| z#|!i^DEQXG2M|j8hrnw5x7VuP2Y)NnL(mv9P}L8KTKntu@G#E(qhzUVRZ#gX8&`8<3!MUSl) z-mM?`T}|{o;kTqTdNm4kk|Z)-095$!Tnv`GBk$=)#(s~WTTJf&&blo|mM$d76g#>)_X}LRUoA zYuyB@iYP~S)L_HEO#1$!(4R!7_p0^6T@A_xD?-ig*NUcl0vK$?RTmeU2T;v+$$y6k zW^@SXFGE%vM6EP171&N1CpEGd3X&eG6~aY5@*jRg?ii=Dzbztr4Jtj@qIJ7YaX?7d zZWp@Ja2~lIWj_MK_wRlq&DK{tXh%}+XODI4Nhg9Zn^RL$4tFcFk}u^# zffmAr%m+cc&s&e(Klzw>U(d19bGJu6-sfdxW$U?dhlLu$C)aet!7bvq8&U1v7Z+!< zb;ezgBX9>@CjZf-a=G4F{B0i|4RuyT-#1QLdeWevNf68WYf}-c*Rx&r?wzHSy%|~L zjv?H_T1-PKoS*-pZlt#$HhHn*66*C@oT_*D==X_485t-6ND0lGPu*M@$BoQk7jJ_& zLZ`im78<AoSeUeqhKMZht@k&)c}+b&B5D*m(E)ULwID zo5Ra%9D*5aktk{>%vdrl@cD-QJfj6yl9>Fs7F@*gK-dtl`6al!77?RN=1QTuHbJz= zvcJ6i`+&mFfU<#-P6cJR3L#HLU|AYjSQ0cVjk%;X2jqg;SuwRjV~$>e71X1Kr&)vL zWK~>+;g_g|0_za0anQn5BQDRnEYn7oC`l1($(lluYUmr9+9vj@Nj?`_dq(9y-WfQZ z9=><^uylG|ld7sZZm%|~%omDd$~)Xa_Xo7&hz4fFS=o>i=a0)NKC}T}6e=%1OuL%8 zwzl4X!ZnlO9sBP(TD_DA(cMv`utbJCDB}I10f+>}L%f>xKP<8)j{lk{&qCNaUwsvJh9}~A9K4fF48pLK?)-nYO8cH1t}rsY zDOyrt!{|-}#qc$xwr*CtdXklzJTy`_5$%3U@j*J8uB@`drThEndvRDBjg6p)r*tTH zIeBwGnDwo&;p)u|vuNK#6YoF$eJezC2DDF92ic`jiD74+(D$_!F}2oV{@*;19(inmQHoz!OQx7 z+1qb1+xXhW<;@HCT!TD@U2eFwF^@W3p2`3I{iDNMxe$Ymu}^)hnq-qfoZW!IzfdJaZcweOerbqZH~8?SPXgxr;A7$1TolmuM;w|^>uZ< z5yrRY+dXl5n!&^1!P?DuFJFcEYMIJ9$Y0}y2Z<=g5}4BeLb>kEgi3n7`;tXKEBxuW z@HR3^=xQM;OY)M1*CGy07;qKIVO~JsR^LbYxMBDVe1An^gGdm!FK7xP<|^CF=y!cV z9wP!r!?8UxPh3+ZwtMake!%t2mp)r;Ti6wCwp5z&JnZ8puik)wez*nN4Yw04@`|ct zn3$OE_y0KDuZOUnuL_a4d~Z09hOGd(o^b9R6)18k9+h13) z`oxiXDk|#isq8ls2%743*VPOPJnefwFWGLj-?3x;XUX@??SJPdiNKhx z5AWTK2cgG7f;etdg|&?cDSUG?6?AmbsBzo0nYnp-`sPRk8io7EnZCJ)M@VL?-SytI zA0VLmB=RsVcMTjE$HO~IpzeiUEMVNOs$wj%d-2oW`Qco14B#lW;V`#{cTv5F4woFR z5oE99@mILTg<%9F=C2mWr#6eR&cppl7TBr5w*t}K6Bn|7?~;Xz=U-b9%cUm%#85kv zUh~*SAaGmjw!XsuJ&Rz9Wv>n7e+A1pa% zth<(P15X|MDhd5_L?iQ>BhxP`E0j7PC-JN?s>u~KRIl$zwmbbSSb%zr0Up1cNm&OX z8gh91^i1s%SlA&VBa3_dt6cU$rT#N>KWPjYk1iGVy)wT1Gmxb67>_*Iq;t@8(Zk-9?y0k zIen1iRj=4nBgiMt??wJo^G=Oym$79}dAkABNc4vz0pT2x_$A8FLas~cM>$LguWp?j zll&Uh@+^7PaffDynU`MjdMlR07K`5E?_|T43 zSf%brZ!47G@A{<~!MtyOhT>};0k1p&?9w3gmbh!vV=-#*62WSx)4Oj_%K7J{5m(TE zlB|gS;MNXqINwbeL%0ov8HWco(Fsp4mY4HS(TvjsymM*kSa1Ye@;+LraBRA-#h3ls z8c9rF8A27_3s^wypXArOzeYumDN4OosIOZ6b(?+YDQWD#2kS+hbj(w)lGx ziQoA8Fq>3VFT~D^TRFH+U)2X+f_>-tpa0(-V`gEQzkBt;OjS=yPsf21XkBMQQ0AG^ z8j0qXL@$9^9>V=;|LOZMImOJw5_B8O>`|=PyImQmJQ%EV1C$kT$+nW*EHiVF9&6dX zqocb*Yz!#AAaGGFJm|h(Q4j1=-`?I@-{c$ad}Ba!q=Hw`HImx@R{Bv%W%8r?DR72_ z^;v~o=vH_3$|vGgly+U}+z19%g+f02;3)#_?a!XzjJGtg|3@HbG7*>ltg2d&va_PCH6_(%fP$#_wOYz@sXz%0ev9l`FVT+Mc9 zrHkZpFM5hq(^`?1j1i%PniEPU!^G5-)5n_!A^)d?1_S$Vc~hJBaw@B%3>6tl4kc+k zt<_?$X~{Cj>6OgCeV^ZiU>C}%?e#kUR@Qc-Iw{o;r9CG6fH!72{Qf(~DevNe^)O#a zw<^N0eaSysoJ{FNctw0A$C)H7-6>``$sKxY$P5hGsLTi(QT?iWSfgw+zQDY$nAA0R z1?4YHEe&UnjH1dN^iJ3A%e{8z+l@6g71j8crH3->;yS*_0(nhD>^pM>22N;~YR=LC zhkqfye%$A&NI}OAZMZO+QylHuAij4YBGu_ri@W+7iHfhTF52qw!uE{B|8Tsz^fZ>+ zOQeLk{mN4sFrrgS$XRr4L43nnkp(NB1NYzjn=!57qU)oL4th1B=*F>gY-gRI*6_P4 z&&cmdkjHBSubWL*n}(cgN@$~yrCUNfYRXR|*SI{pP+1oSYig=QKLUygLp*QLe_U^U zwUoVg%&f`Il2Q;rGcHqCCy{#^D-bl6T-OjGz>~#Bd!2SYl&+Z-oLY(cOLz{Mrwtxo zgm~5wD+W4CkJ<#WkuWadsk5@dD9j}7TZtH%*20T@IQ}VLHdhA|{EkSfy4j*$ILVbU z9TTin45L~Y ztoOOl?&9NsnpeIEbf)KMaxw`V9Nb2mZ+1n^^UkU2X-D_GoGdydZjk18ODUq4VsM0S ze?69yqFn|mg&4QlAJdSfd{D`w18gwKf!c@{>U1?*N_n6EKp0Zk(SX(q3yTF`Kicg; zHn+OPsC@XZu`q6`VpODL41 zMt_6ifzJ?Yte$#WWF3w@X-G>%G+g^NQL*Xmgjh$i&u&8JW*WfPKTWh(*jp1u*H1+fj zfmwmbj#*`RDCNxa(@)+$J3%;R&xU-&vvk&RMXM7 zglFO~Nxa|o9LV$JCoY}E!R~6O8V2VsX zrj0gL)>BzUQ<{J86&3wjQd*Lv;4`9Ar2E9nQZ3NRxP8T5QBHk`4nI3*qhW zGCvrHlFQI+G9q+&PE=$R6;ghP2YOrln%2q?K@W7%&?uV=<~iySIX!L6w}(}JJ@E& ztMW=*ZOFrdvpZy2+-edNT=>9d@{|${C0~YxfY5v5-{DP&zawEwD5;_t$BE#f+LATM zw3m~HSWo-7GWrq-!(9qxeHDqmj!cBtJiyb&K{q#%KI{dgn87jUJejiB2XFGQ*Bxq)qLWclx|6^I>mOMpW! zs;~~u3s(*w;*wBpCDa&WV$bKDRio4m?jR&#zfeNn=7<^0E~fmc7AoRkVjcXSDp4-@Q8W4X|t=5_82V;}_KN(^KkZ zXJAI@W)ooTY%G;$;^PYe10JByKu~vi!DSO$V>_L;jf@IrvrUdC?b2UygL~=gU16z_4ejnS1TsT>7jo+D?!90ATSO5LQGN4KJ0wJI~?rW zs0QzH{`4D$Shu#)($rjuetFO#)6&U=DmMkY-BeQv5TogNr>SC^xXdi3jFRKza32c4 z`uI04+wCFKMiSef!yVpR876K@!3x&qaSIhB)N_I3g=G$Ws`1E923(|EQhtZ?#7N7- zmxVE~ZCQ;um+rR?=gCqjCCx{f;H)-;@!?p=H8L3o0~9{I17l;-3ZsDr23_{2vaemf z@#OIB@n#JxXZLB@*$9!ci!da?zkgy`H8Fkd(nGR_AE%*tJD~6z7kIt+BzNy8*wdTA z&-wP97Ti3KcDxrewjBt1seo-}su?VflwgD1!JW)hN;v+PRDjr{!j?b%@HVBog`&Yaz2(}kwDJIDIkbEC$iW@;4h8f4Zg$wn&eKAv{OPMGCW!ij1j)nG{p z>Ln|(83Cq)lNT78m*>L}o}jF8MZm-A#T354-~M=L)gI+RjiAC6;H3`GwJA`y;J%y& zw4jUKQv`+uDF}kfU}taHjml18BZeDoyD#zL&~@BZPSi=7qZ|X;j+s!Ie54QGrd-4Yg3+yLQM7gy|ZWs5SyiYqpX9ck3v}ZY9 zpxr?}3dzSr$%)SrW9fbtFx&zH7hMU2%jm99U}+JgdAyc)#9Xo12@P5EEc2`+<-dDX zF=1i9b;Z}pgi_${N%|`d-y0W`WEBa0UwRlKU$8u28iw4*TZ*)qq?{wReU2AA=SS7v zS)GaP{%09t8lZ8Ct9_h6Ouc*Rz-w%40$FsA`a>zUD^6NTSG|eLXOpB6kNon(df8go z`?@iOVB+N;LCVeA<7TcbDcq*>03lP=x6Q2<@uGJ1L<30?u*f6k2)1AIi2RWb(Ycv_ z#a*v`TQK#*wykcqCNt<*wp?%aCxa!{@hw;U#n~w5<=i`ZHoPf(eqSbF_Z*AreV4mZ zveaa+@xn^)XOCFvzvTn~54%H5+AI&nh-mHCusqsZRfh&PNK0DvF|OVho^vu_!=JWy z_bOqHv)8t+6K=+E+u2gyr6Aw6e5YFXaw6z`awYaRd~OrQm6^ZI7OBqRWGjEMI$2tbcV6J3DtQUbOiXN0xI}pa;;UnCGOJSFs;+yFhV=}d z=o;-jU=GO((=z^Ao4=C#uDeJ|Si4m&B3-)rD{8JEUeF(Ebr4>OkC2>gmk{A=zteBo zvey8}Mk0vSpvk}g{jGd4f%sEeubO&hcV^r9wjbtO7)Gfp?^wP3WVtbz&9?=pyF|wY z;gjaTRIw@9ymj9;Ab_8(iS2)xXR8{qd*bTscD7@!o(;|5FxX_r)3H|ZG0)~n_|1>F z6kp$wE9JH-epbCH$KC|oH5dQ?yDO3_YW;BiZx^^FFH^kdh`IPQt&ZO{&MvKcLoSly z*4B!DUYD*7Fo^uvl68MY$dsppFNS7R0;MK$Dv>rn!lU$vLf4I43~nLPll`9nZI78y499We7G6 zWouID^@}9N_xUoBG8M?d2iuGCzW`-5_wLH!a8HvF%>X4o84i92`vI1p~zOrwrr2BENqKPO=l=;DY1D& zr*!iom%HXsuWp5r^zk1K1=>43)STZrA!$7!mWmOBp_-jP!^2&d)&wyY%9TD zw~p{FW=NGLtG1kGTK?nk>tJxE3=S&1C5QehLOJ=cH8Vdd{8fvP(D2uh5$EbnQ}%~5 z+P05b7taTnpd4s~Y!7s?t8l7}_f)eR%X`D>Hh9J#v@ z0}Sp`q;KS`5-@x9jt%=u_PoY!WX)(Y)p$I)9Fr?7`~)4s2fw(XV`3O~->hwr17Rxy z4Mxa0D;g-iw>1WAy26lRss?#NXnfpxZI5VRWh9)z=1oo}cQ?w>NpT0Yt=Vq$#K7Au z7Yv!XlA^ars4lykKOYa^;dA{$Ci_|@#CW|6X8ra4;!V}!H9J&}&r;!M2rn*Dvo?B> z;Zh~fm5SwDg>;4D{*a_4fohB}h#wJbFjk=Z`9aV9_z|~du5N=%LNP@aufwbj6#i5S z1qWU!Pcq2<{XTq6?(9f+93Cn>)zp!;A~7BVaK@ED>zax^;*;XjXU%E8DiRLdiE74WKsI?1TFj8 z$Z{AR@aHX`DXx*&HASmPE2bKv?&??+#S$|i(q5-E8L#M$1hF-ARF70}WAm^p{d zC(DB>{5p&y%_I5-F3!6|tGMz*r4)sLM5|HlFVp_*P>M`J{nLVx9HfZFdYKVpNPdOx zgOLE+??UAI2&;0Ylmgmv35?+<>}$LL+JL*>g%!O3R=KeQAjq=~Q(DgFIJsM3ecW^- zQe`8$MGllE;LG2%GBCE#5@xUg0|k*};7BZ(+a?yIyrs1QOIfv`)KIu`K{K8@SFn zTdui1)V60z%gA`ryw~g3INe3O+6wvTgvDa$4uRI|d4g(@Y+pWrHb;6$G;TZ%kYkn8 z3CB#(x}UtN)PJYuHK&jE7}jWty#NJj` zowrjiX3$|M9AFoQ=|*M_@iZXaLy(?(5Ym=D?rkRvIfNw`K%g5&G|O=0zH6`d@7>LA z*zA;?U1WOsAyJt9U(_IzSb?YZ1hNN+Ne1X(s-Iv$*SX^6BXtXQZ7Lo6wUPn03JqsG ze$>#_9r6MmxCeO}sXl`imCtPQYi7c}SBe?>1j^-zPBpv*ehe8}1 z@hq9d*BWk1(%Y%YKIQ-m+9yf5mj*y`pE#Il<_%o?i+|BzZj*@BZX z$=E2hb^PEee@rs%n~(GThBKMpU1Sf1FQ9YPsNXrHUE2k7Lz-;24N1J;7|@&osEx77 zuWa0DejFqDMK)uKQP~LTs^^~Z%kiqj+o{y(&U7cmGc%LREi?-j8Eh}$lz851qz|hG z=F1xp$g$@${`Yo|V;Adicb=v&Qs3P5zq&G^YPJ&|5iC{lz3a+1+r2_3z$LaD<^dqazM+Lc$i5CG@cM!_AX+p}N zJ}?lUi|&3_;quv#DNZhWK|Tzufy}I}Owh)D*NE51!X0{H!XAN!@o_Tr)zp5LQ5C{K zju}MeYyixiCzNr2P}M0Z-r^u04SFkf{XR#2jn{1VmfS0K6I7|8tZQ;^#}D{9N^EI( z7_xGcA3oIn1L!2VS{XuFobH2gTk5is8&%(kr~(R*ssj+7S8aLq1T%ul_CRYc`2eC~ zuyF5sIL{Frh1XWz&>Y#SlTy;xh8&Tdd0n=x=Dwno5>~oUgLg0g@94e!SIgm`s~S_p zQntVV)k61c3tP$TO7`Uj}?(QkWpUV`772K zhm|9yV$F>|f`lV`T?U4(5|+oi|h{I;E#4&orQ#1v7C zI3?vHKmY}LT6%fGjv_*mkw52?d{X<@*7wl*K&@!%spW8{go|Uvuf`?D0lX^{teb2K zD-EjXOjLEKpphn&A<_iAy7khoKUjtx+3-f@6SzFZ7dFe%sQDz3b4^qC)=L;)=MxTrht2)OM^lJKAlv+UtARHXrk-3BRG;<*|Gi zv|O`(@4no<70lXPyuTjdT(`ut-D|vkrD7tIk&tjb3anYI!L7ovJ7>$EkBn&^6|Kb! zS|VhNq?kSXNNc}8mC76LS);G#TKR}y$^c!e;I%EaNQDtmvW|7UT#&Rfh&=Bf1!v7< zUYRDrHkM=4mJLnStm}K(oSX~$yuJSGXpSm~?V#?GKCmhRvsw&o_UGuzB>9e)u@ld# zaNK{vBM#0V^wpqE6q_XN4e8kWGo}j#LZfO{Qpo=4(4kV@my1J|`Qe}yi}eerC47&H z^mzoe&8^i+yEf1X$8Ym31Z#WOD%79855wo4_6C1y#U4ThnAq>d|Gbx)zV9LQG@=~& z-NXZLnK!Rzmp!}63ExRS&E*PV*E%PPw`J?=R)zBnrI8wXK}}q|&(^2mX8BygCaLRt zvvvq2zRuXZeYtPjuVy=W))sJ4_4Co4`*jwhWJSFCAX*-2bNKZ~SY|6+W{h(kTS2j% zW7lR3KIzo|M)gRp&@=IttWpHF%y5ZpTgn{hJZe}SlepzE5P_s|KK3=9eWWoG`?_W>I0g~>-kL=>3e>7lt_7%R@= zN;hJ82B_nS;2VvN;b#l%MR4dmq784YN_)_p%8`!AkJ7GaZ(9MT!D^4tONI2A&dv3z zHy{NDw;mYE46$DhOu9esd4Y1#j>^(RPBeE5Mx zfI{~G`8sDKuh$8qV}q_ubqFtNser44Yra_(x=o~=oicUrz>HgJRjytmG+0Nw0_gaoE% zz#8B)KNgEJrcyc=6bYEiKV1171-Xsh*YzT2=q@xRDqAOOqL**2?)2ubo22Eqq7oAl z>apUT>;CV-;PF52lLP1Oho*zz@M%fVvm2Y2mdLg^GOBAZM0CC2x0J}14g!el5=!KDiHT1-M&Ll+pz1s?kNp zq!KYK%*+pvK-U3G!&-51=1~cQ2@0jp3kzN?iSka55j6z|FAz-}J~HpLAmHPzDwTbS z#Vmp6*%*Yj$f>;yB(+@u_@rC+$kEN=!7XxI(0;RN2eZ_t7`2g)`3qt)jCB-n0Jdw9=@cN}79SoY^Q!wpI95jZb-)CY=0~!&(1LNbm^WkI zFpQS}BU$Im&b4P*mJ&L9iv~+ts9~p%h34Ss_+=SKdgR~~GyI+wp)p+6miV_3DCoe_ zNvPugQe*&<1Q-@9W+e3{d3a|rze2;>aY|8@&>CSF8yVu1Y1qj2O9eW4aC+A*Fa@C2 zg)|blmnBg}Y;?87(H~pUQF2c4-)>?CJgX=%_K>Iza|pElZn^)-&hxjjXa%%;pT)dA z6qm9hv%k6j_$kQJ(KBSwXc41A8Xt2_F0a69bM)WtL{8U2@NFIp{##+N|5iAh6Wm#& z%=-AAT860zS5;$MLPp5&D;r!H)6UNY`TXwTS6FMfP6lKqo_##4SJJv5<;E|<@>=M1 z;q^~~5yM*JsXY(qaeO{du`IU;lG~X)-`oaF*|`PXa9C{uwS6B$Y9xDtv`-Ev?7hC^V;|C*nFy z+e^;=(ghwR#uU$008BH>^Q0Zw<7JPa1?=W8;cm8tzEEB#Er>5nP@-2A>1#Q_Hl_fa zIiw&Nw8ml>GeJTu0@};_7iVa5_k(Na34#ox>%({sy{^%@8`7P?OM53|0qc)=${joq z*F`I&OY{^Id0*stKTys&NBHlx>q5kQ43j}Or5%ER`d6+|w6JY<%5IG8aB+1XC3jf(V_+voO96@hAHu-$7cloE zEw$?*R2jN2gzCP7qs}93SII03a93fU4-%a&*L<{y&0qDB7ZYxXK9}CJczuv0V)2JD zg?t@dhOPxZFaVhYVt1cYFA@pq7{`Pcw zYlp(`+L;lfTLE1@E^th>?4{+4{C`IYk_^c-5iFTaMi)sd$eo_|_A4Mji9Sp%@b&@M z>2*Rq{<9Jrt(%3XF^(eV2^_$83?vl3I(Ce0e{9OP`@E9BQ;nx~-@u0*AsKj$^@+Tu z^*n-&T=ETex8neu%lcPSe~9Yp+Wz(xzH)kd$3E{vb2*)ZF#rt;P`4Y-QPO_tgY)iB z{hbfzM-u7{uNzJzfd?=oKF{ADdDF?`>K@O<>SY`8WE2nAGh~k>dy3cYir(k*^wKr{ zKoM<)<-b-ju_-woGvhVFSAFjXO<3)^hRz8j-zzk$qdFtx`m);9R=-g5o+nlc`G

V#S?!er%L=4}jixL>905DW2b*V6P-oOp zwNk6=0>A!IbD8fN(D(6mil4PN1t1p}*!S9gCsBhJc2jv4WSN0#1M0E~cC@9mPuVJ4^6qj7X?A;^#Gkz%PO%c}OKq83=k?2+EqS4*&=}st71G^p#Re!|SerG2lUO0Vt z_3?ZsB>b2A*1Fxl$ba5`jo)QXSYLk0mW&&aupn1Ux}EahB7X7so(UG-%!#DOi@i-w z-F5W>9F3a;zVpoN{k|~y>&sUP1?c6PXkiP@-X#Bi2h(J24$OE@SV%kHU)gS(E_OQF z1?O0@`q8=^_eIj(r)}MP^`AT2Sm1Lay(8i)zJ2@WSWu zB%+u`=Y1K}To`CZ+K#3>_XlqZ#PWNA^?N847*?YYmM5q_zDH=*KKy_NeKXv_jRq{ig*3{*+{Zm^@hv_aB5+($;ov=@_-(U#L*E@@>;YAwz8;INB=q{a)DuOXz7^H zQPF>G`d`5Kj{ z4m+K8qh;}WFaXe`vB#cIpAsa#zVyaNfb>8y=T$x@^WT8LoG$-2jj^A^$67>D1PM&k<-B z^#0D!=sxfK9+ZsT|4_!QbCF?oCiE)Mm<4P=a-xC^pU?$z*)PeAZ9Ok@<8Kf4pX}yG zzA*#NBk7qul8X2JYX$E6g_6Sl_bDYo(Ke+iHN;xt`MyYdX&LDS%Sf+IXG_&H#>q1r zu8haZn0&C>4Rv)gx?u%#%n(z+76wo%qi7bCh{Z+?81u|Mu_KgZ&4N9zn(Ka9Oa7mx zu05XV?vG=Xxy~(eA4uk$&V_j$k1=bYC`8o0df=yP}bv(#dD!XkZhUP6r5i;4aYWXt;UJkXv{q5% zc6Yw}yGW_}Q=bH!Pk^R57OAiN5wjKhMpB|rx@N1q(X4zhARHj!DF-0qBKmFDe`3_# zX2oESzDGd}2Vy@JrK-{r-zw66ul*TOI-o-*C1S9)^tl*YhRxsA ztcovEb-iE)UsL`lspDDmn2S#I@UW!zkJ=wCYrdk_h%cErr)O=_lP^`lW7)GM4oGf< zVcaZjIYtO}R7tp1`hxi9o(SqAnvJw^j&w$W^QGInRp^A?KVG20Y)>}4*%)gU95WW_ z$S`J(9VoZ|w+n1D06wwWzD08&o>zk!G{ErVkFkZH+D%%XZm$Fxxlq;h}DM)HdRU5!u)NpVN#m2Uw(uN_K9Q=!@^-kVqYDgV%9>Y;=5r)vLVybx3CPPgT0mb!1 z(r%sQg~CSgOWig=%A!ASAo0gdEmn1KbYd=Ncf{Kwf5DqJ0l)rfz?=LVo+1wD1lD{8 zvgub!b}kl`B3o&oX!mr0??G)BO66>BifJ(^XPxhz$K10MRo2N+6wh)w>if6$#|*etyR0XiL^V6iH&iu`fioT2|-gk1g?A41Y(%d2Vd(3y?qAGEr@9(`v} zxsRKzWxt6M<$S!xvdg6?<3dgp7N*8H%D6sQBAu`nTAgU6AgteiV0lTkCfMDW+d-hL zJ8Ebyy;#t4!%H@)wI~D`?6-K88mJ!vdM2D1eXmjtzHQD?aw#dfb{+GOC_w)Jp~>mS zHx6xj6VHj`Fp*e8*yOj9Z1;4-SyK37^erX9eu(#VzZ_RYE_b1dy7G84H^LBG#5OMR z4#*N=+1Po%o?fxb-wCdZ!aXBd(c1NjJolkl%l3STvqxzE{Id&;`t2pOC$6a`|9`X8 z7`1wL0}3_`R^IX)$^^8E<|4x7iN=#lBGCR+D#>JU51w`VgmNfY*LMndhV4 zcmhTRpJj=>lr@I9RfLTPFP6f~%t6e1{93}!j79`ag}tj&N7HK9b0MuJ)6FG6g&p)r5X;!McTv%Je*BRq?bJ~_ zI`mFIy%jL!$uZhw%jehSS+2W&=9f`!DQame=>*&8>_AHn19{7iz28aVxxL%!TocRPM_{5Th$6bzD9yG{)m5z$U42i|0qE&?}8y{-&rRbI4F zDu~u*-^*w2{XMcyorRFF^(megQYBov797_@{FQAfp+`xPU}60JX;LG~EM=ud>B@OP z1gZ+iZfwPCLcZyHqxk)!mR70Rb7F77=&sYtwl}m~iRbNMaPkB?5U4%cu;3eHFBJ&< zY3nY*DRy@Em*_TnG+o9$G(Un+HNgA-ecE?qwTd%ZUvbq9%0hI*OE`gFn;omA!s09L zsRw~B!s@+#T;or-ADAVr0{=DMdLK{$?qU z=`f*W$uUUT?21lPTU*bs0>>9at22hU>&Wq#?7+&}S#izf8!o3fLnISN9S; zVP*Wh^ebU&bzBJfy{ph-9B!3jiQHN?7V(jtL!eVYtQe(!= zZ=X65Fl8J#W$Nd_imlwlpVrH^i5WAlDvxK-;o&E%FNBr^%1O_zd^GM8M!JQCN+Kjw znDHPCnKRu_SokGuZeE@ZyW&yWo)a0kwBn0r%C7zilJ-dKFb|l@5&_Bu=SxoY4ye?> E0Ncuvf&c&j diff --git a/dpdk/doc/guides/sample_app_ug/img/vhost_net_sample_app.png b/dpdk/doc/guides/sample_app_ug/img/vhost_net_sample_app.png deleted file mode 100644 index c7a181b20d84b4b738f6385636974d5a3d73270d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 23800 zcmbTe1yoes|1}OOA|(RS-O}Bq()|d6G}7JOD$*t0AT8YtJxF(Vcf-&_^IlM&=lOlV zzyE)|>z%b&cbH-3o^$Rw`|Qu&`x5Y%oHz<19wH1342qe_*5Op5w#7kibZaym{}Ov^(eJ-gkO; zcQ~_Sj^^vr#;HKh=Q1)=TyG}ptI{2o(5l{-r{rEbf0%u$d?M@;dAbsExv7bvELG{V z#UPTeV%2jH%8rCK!@}0b*)rXKaa$UqgX4HM-M>H4v!*f3Q%oFo%xhFU2XzhOr|h$CT%C^YuZHRC`p5N}NoT zHdA`UaqncEedsq1Gu{K37cLAp8(C zEWg6piM`dVcC@J;_!eenSJ5)&Dc^>O{2v-{lMyTah&w789nuvy1)Hh=0x{ycCKChj6^pImdbGE01GE5nYA6{;hT$rP)2$Rt*KIgtEcmk^}te z_G;nPwYzVLzZ1@~9i~`~=BW#XINS)`QM6)VxS$S_B#p&bk2EPG>$!c<0zQ7p)Uib9 zcF@2#r;kG~+>uvi_jl^;9WBjT_6QU7(w@)Hb(AslKCyzS3(4e?nqSDZNGXoFprG_p zKDrY;2@EFZ7|`NB`Tl_d5<^5y9@n)q+->!S0VrtUg$&MzvGm+~k_g5jeHmw#stso_`{<=lq&WQ8fm z*Lg{h7-bIZjG<@I)srq!@rw%sCqr4)F2=g#`c-2wAB&ug+6-<2YROcERJg|*-@fVj zC7ss0K%{uv??PLano7;cK=Zuenp>s8`Y%hjycQ=p!i)CTl$2h-m7!%Yq*9Ek3yF}tQPd}MB)VyW4qDuO{)5?+A5&9*(G?R~DH(@fGjdSh?5~@rm zwfpFCuXb$r)=;p|WQ^D$mi#x3)$f~4LDd7Up{StMWW^rht$der(8hUB(|N>njII2z znFw|C*VC_eCG&d-5kYbhjhvq##IE`0DoIQ3kfnq8ptU{2mF|_3h7BzD0UrDhtCLI} zQ18btF)7)~*L23CLfJZJTI=5(lj;)Ighpn14f?J@`)P9I(8BZ5nrIa0*wfUlgSaqZ71 z*Ob={|T9ONK(qc*%dDp5*9P;2@`?=@60xKNFQN$lIQ_{q=~XYNkypti() zG`eq&z^c-(yt3yR+`#0uBn$~tXAZZaiE;|D*Rg%+DJBUju?eTk zWrLPlKy$ucP0*<}65TGPMx?q#A+}KlacK%ul`-GE;v=6`VLrf&$;SK$qAWdlj}3J| zR*mvVLD7)aEB3{l8j0^nXI*$~y`+~az22y~Yq=!ZV%NXy(^g#gykntoHLh+>n<^F4 zNQTSJci~sZ&z^@~dG*p!%hWn0cQX2ARL{&Ea$wZCCV9nb6>d#Bz8c~Si>24TmPnnt zLT9ox6{&k8T%B_(zsnOR)||May2x32-b~P8)f#$IpII2a6Rc;03rTZvb3N-Z>TB2T zj;C3C+tEm>Rd{3U#+*27rxr68q`iaP*H0!Me`BR<2r>L{n*zX~H_-ONcf*CZ+Y-Kw zYD4)WiGvK))HAwRrR*H=oum$1WV%(S{P_-}Z*P22Y&wFdQ0CG;Q!~_~mQdfYu79<+ zl8q%bX{A3eO0slCLg7zbyqOUk?uqC8!ORmFs$*3`#R31S>1+?NvGWeYLhWU^2Qz#E zdv;nDjQX%`Od*dp(uU}+R_87cfMEdo(bH_GXzU6^dGJ|0Bf@-C+`k$G8M8Z8 z`F;eWvWXakW=Q0v=#t_zrTIA;76@FOvQO@OQF~!;w&~bUkiMTxtp5h$Ed#X*)){Rx zQWHe&5b9}YwAh612HD>&YXRSR7;qhr<&mg8JjB;_WvE8rj6e*kJ?F3Rdm~N$j5jCi z{4FG{fcM;2hFWKjIc_A;*Z1>~^)uoFm|~%iYY9jMZg_AAtt67Ei|p63#9a3bEj-~nGY{epvr2?*BOOFkB$MBC z=_MNQ$Wd7QeT=Pr*q)=+xJB=G=lcQ6OCuK#W!7!=kH`-?9c*zR_P9DT*cmT0yz30` zkKE#w4maIASZZ-U84}-}tr^)>5J@tG%r`uNhc7o8AmVk{)_3F~TNVK}zA=?L$Gu+0 zHzFddfw|I}A|f(tOE#a?lLP&w|GIC0;@X;KJHub1nPTt$wCCSPMGmX^*@hg{bEF?+Z|f~-g$102-pg6pf}h+qpx@Z4{T(%&BjXn`$S z0L>Kep;*$UZ4*ZX%xo8KQH|8v(Z4$jd@WuJ&GO~1Gi)nh+jsyU10Kuo?~l5)4!J?q zf^c&8qjrx;XlN|a?Enw?pD!u-Jtw^HXOhfkMWC0J9#@X#(l5}c7609xKg0fi*DVF$ z%IrLRUJ)3&*3*B{rGMyCF1xSd@3;Q-^bfhp<>%-BSF569V~rdU$d>j0-N8Q{{d3f% z)nupmGaR+X68~Q(wr6rco5=Zf0UAbuF$MUOf1K2Z{~WE~qn1lGEa8u=JDpH5B5%;n z*2kOjwYQmW!F@;>3wlb?eTG@MZGF+t9G7_e}>pD7m{$6{*IYE;r{0>@sY zq_mmmHAEQ~=N-tq1mu*iFy2(5>pnzn3~{QMQ$GVN3h1WBzjwCn_6#r>@zv~VAEWCP zraV!399Q*`T=i`ECfSsQ@B$sd%}&~MzOy47_K8YiaL8rU9Cebbzlca4128LcVFS-S zc5Ja+2FL^{6I-LWo*nUub&E3{ZE*~gajrcxyMq`u;$J;iKiY*MnBROeXyhjWG&~z~ zN_1Ub|DvIHy3Y*PGO;He#l*;BgJ|0-Oh-(Hjf|6+)aICAx+t+bA`?TU<-@nQ2wB~3 z%k|m%P|)~(wm*FpeXxG%m5@r+3Vh0zLacXuKi^x9cwl~Oq}ENr`TJ`mz43tHZ`JR5 z#MoQ)Hr;-%sJ(nqht5@MAWVGOORKq$W*atR0&^~S{X!sNBKMq84aI1)VJ~;XF_=*x z@!-PBuNo4(m$Z7&PfVKI6YGzusUg^9x3&x_Wj_iz=UqqUr@=K$*xCFz&%5 zFTk*3Z$32Bz{b3b9^kU8sPjf^dz!{G73WrZK_94HpFo%7Gry&by?`Xj9T~Wj#h>N< zfY2tKSA90%Q*wN_M+)7xllHx236SBcJtAHC)iS%Q^gSh3gb-M*y0y_K`R8n=I**yZ2s#aivr z1;;0sJl*-$dp{!%Eu~{epqQaX!WbsViG8K2<0=wsD7=XgwQN{4n6`nVRVnTW*>^Wp5^pHetkl4`rhMo$(5aiic2W$1bsa3$uhw7iZHIF7OMxH)O&R8<6&{dE zf?)(0Pd40l=kiZM;F!;mo>?908JoLlBWbD+L%Lz<2unt$)VF+!?K>c&e=bu_hlZ9* zctYLqYajdtW5<6Go+WyH>DNg7YfF#~`h}3IWs_{n@kF!Yi`pb_G|zny{UM~Ooz z@hx5)IgqQ?9CDOboI-5e? zJLI++p!c-+YgPAbjiUU#h-7_Vdtmm;1DDr|6Y0v)a5{mVB=*Kki`ImN-bJMS+h3f_ zSQZ903(HtT%X>nt6#)R&c0_)>`rPIrJ#jyvzL@WzN1FwwnSF@qeh&Ny@#2yJM6X}# z^Q%tsG;*bY#MOc(wpbFvyn36o&=ufC1$&ah$Csfm>%3PC!iFe`!y+u?2>Or&_=BL@ zjy1y`8x|C|LdgL*tNp%VbGTRf%vv1RGDZ0Serlr#$p-k-%<{76_92SR&PKxy1;}je zAa#UAL{G(^S>UHu9=uZ!bd8x{gKT8PsppANIrxg|9(b&A5sUsx26U zIIBLk#v%lpVRlvz44S>AE5;+xJJa~3EfDD5aIqj8bA0Ae+|O$BG3}f6RzE-= za&4ayL#5-F=I#Tk%z+m{-{K)O+c8Ye{aVnGtYJ@)A78!dxHxP8q-G9WV(-lwQj?BMe0aNv{rQW#YA)AOO^}FPw$=4I12@l{$5CG5nVtB`q;&K*-cXHN3H zL=n#}q3&CUo~e>M{vC*Bw;9tK^GPD|`0a+prCYM+-YSGAiQCLYx_rE1MOxV-)i+t= ztnf05$!arnp%Qbmy>WGA=(=}B$K(rCmcf{l!q|iZVf`lYrpyN4;xdE#M8Uk zvCpoMUXLf5Kp$+RwsQn!Ipv)Fv!P&I&feh4<}@~Y5cU(zsKBDA>E5hh`c^X49OQ^U zp*wrI=q8Rs9^aRtdD|S^M{TmCbh{n1vroaUj~eO5nIk{>_IXv_*+09mh>@vTUIR6;@^7%XIX=Zxq3MpvpTmm^78Vmv_-`8tp5kPUt~iasL4 zmw6?D>*72xhuK!(f8Vh*imk*%G@|$>?D6^|wL^@)bG-ay=xxM8UeM5-?JvP`&oViD z+jf|cQ{y`i;tfBLMN2;HPXKYh=l_Kazv(9+Ge-ZyZ_VvrJGY2T7ULhD`ln#{bIcyb zk^7s%{{8gtcaIJZm>+rzOnfvJB`vL#`{^c^6Hw*IPmS=`Xwd}iw6BJ}Q`uh5DfBvA_JYqvue|G5|!7EBF~Qh3GVCB1j@EM*?z$W=OxR=6sKSWy~`!D zTyH0cc)(^|f{%7qyket&lJG;Toyw73)DT!u^EZ<$@V4EJy!uAH(QYFRK0F^#lXCwc zZx@6SE}d=?E~1gD#@*&KIeBpm8=At#&lcO8K^=If>g+Fok2hR;%Aus0vRdmf^AJ|jJ1iNANrL*HTJ7^o*-&*`7(Ne`te-GP&2fl_t?>PT(-T#h;>Inkh1rH zoZj-zrRp8}%a-L5o_v`HWuG}1R)jJYpniNVT~>oV%t?yg4Etn|JJ!{#7udMLIyg9D z;?47upZYux$68%}RqaIrzQ_Mo*pIK#I1ANK(wv6-c%ytmHR+K}4Ai10!))|HDCb`$ zgr;Nw?|_JOD-u=Gdc3hgnD*j^*l@1Gdc9hfqM`H6yUU$(#yk80hB-ghoQG40By}!h zHQ|oTYD}#zfu)>z)f980+;#a>P{%4ema zdTIwJBf|1brl9Z>F&x3QN|G20%@c%`5DIn4LB8H9j^gN1CBx)02$(tk75F1r#;w+D zx~V;d`K@E%9YtwmES|VNQ>ty1rdWSiLEs_f$r6#Xdeib;kiWlWuXc<~JEgm&Ld~xM z>`UYVkbR@9V&%5ek93`mb8Qlqk0-H_%-8dYlbJ$4A{>Q#ldVIbm(nh3o0oeAMOXY{?$sfI$TRj3YxtH+3&XC<2Ff@4PwJFCdb@AakZZCk z#^0JtXE+B@OEB90WxpsoYb4O&r+&m;?ojGK>(DTTcT1{{2wPRYzF`&CKU}{Q@HeIin6b=B(nzRt+I)oYK-#%829B z8|CT2+`o=-ad`U-JP`>0Tf^7qyo>N^pZ3{uoBRum5VAwJ{GR%f9pE@gg+qN${1j#j zwW|04%33xfp@pM5OzELyXz-EE}YF}$@|5qnLe z?x`3{Sn`p<1lOXyMma3 zDp}(nqk4FRuJ`Jp*;~+}=Zo^dvHTC3-}xH8;5iRBUpt^6$$C@bB}4x0D08JVoJWUc zD6uML7_~wNB)36Y8>h)mEVR{qsya*DQ}A}*CH3&s5KAZ-jHVJIu=>TgFbERFdcQzQ zrlc8>1VyF{1$A9mZ zat^O{Y5f=-UTroQN{MOuGsDj7?Ul<4AzpKGqeH*zT&(9nqXd*`KP&AWc4}v;$q?wI zx!CZDz$vZwFxBG9x>(D11LvDPqHDSgzlL`&6w$@rvF~Bc21i()vK5UoDOEjv%$52o zkjT?J=SxC#ooX(gFTm^KG`~u?Ah#^8UV3RCxjH;ft=>J&>j@J!HKo11z0ID`!+9D& zVHFu(noy@^KS|H0s45iu>|54-* zwpxHa1~5ap4|5PlGP3pBSr={xOSD{Epw3cwiOPaV1lA*GwqqQ#nj#L%LD_fSUBfFj{ImWAr`c{>E zYS?j@Y=%jRmua53MMysnVp&b$ZMB@w(ZbHf zWJk>zm8dGlXI=G+bl+5mx1BB>y5uD;yz%a}al<8Cz}LQ^S=0STcr_qtvls&e0nc#x zVwadon$(|XE;83%5bKrhLw1S^F;TIUHo{&<r6b&yY(zUwd%O?D(L#JmtTt8`g}8)e_7lib-FA?A`Z# zzuQYpSU5Qw8cxSs_fHvT`ChmBhl z+U?X4s+1otwL~!%I7`fVg!Eq~ovP`%7uUtqUR+H^yJnu#NJp}su9bPs*>dg{2lBS@ z!(}ogw~g=*$mheC*~$^9A4!3ePHRegEM8!Ad@zBVv{=Bn5``76tx)a>8%0D+QfMKh z(m6BR#TAI~+=BpDd*b-A6)Gv{5W&=n#f8^mECG^4*i9J?$-hsvu*-libEWgI43D%R z+AgIXl}{?b0obT#8x6QE#s)NYI&vWDbDCAi6OfhcMoE0|t7~fSnOKdImC@V~diG7h zK-?vl`uXGOh@h`eOPs~oEZypbOwb4sc8^HtEVAa#Ww!{<_?;lBt&3+MX1!xcwI4J+ zm1_k?XVneCM0V7BSHZfLiKy}1Rv2x2AS^sM(Gq(F#uc^kwmzdrCf-_Wxg?ssuPVLo zvp$B{q(yzq$mP;MO16(yzCFkT;H^^}KphBF3pPuK$EWTT}4dfMGO$+l6k?Mmy^TGNzMZ=PBk1q_4p)W2;1{z~Ji;jKzn)TDKUD2X7 zW_5Pw9mwh&HC-pa25Pat=ZA;XI3bOE-6rGs4D2xy42FkDA63A8^r~Qx^B3DI*jrHc zr@=-1bILqyF%<&NKy~ID_kHr+pP@LupKX-5sP^PODu8@(bZmCzLz@S1WRnUPM+@~) zTAB{-`91!I#)MLw%t`WN*f?JOXTI{>$BMRr+7AJr>fXy-3Q-h(J%TfngugU1sqTH$ z+&=mV4DJG?{8);zp66$p0rXSjk&BjVvxwtWNKjI`SGL^N*QEif7T$4{G}s-@-$pQE zH*TBi-;MC#)Fxb7; zvt8jUy`G0#6(|jsE&pQ7XNKX$y&gT)?n3o`p(?Dmn_rI1R`kB2_$_K=VwKssTk%_k zRj{(V(#a)C=89D(IHH0@*hd2Vwcgr`uqO+~y@rUbUAr3Nx5vFHWI7L83EM;HotX+N zxtdvdL+B1cCXLeE6>vVA&4QVOJ!qKr7J?$C@A(R3ViT--nLQtBkjCdPSsZfU*Uph= z0;jM_QNQ09{kmPqkyb0gaZkf4gOl~{JmSOeK;FLdvb~JiuCKO~7pk0zkg858s?|lq zq#5JI<&zG3mGt0)}3a65PMy1R7V|-K;2WFee_H`0wWH9>{TO74;#b<}j z)V72?Sp!`AKNY3L6}k6aXM@{kUydZzK)z}>3$!Joos5Fx-kE=ejoV1e#~ zg&=W&22Bg|gBi45dlaxBnpq2&$+=m|#aeygmKiMs&qdJJaG?o-)3=EMr&h};ykdZTu22y&H`C<#zK?%x1{&e?WxHj(w zX;8xwM7sAbVDg#5`qkv83rJi6+5TJZ)o@%}FIK;T(DlJ%pl-0=pN-$2tD|!de9sEF z%tp+}ExYKM3C_Ev9nO#jsK$9Z>BKL|Db~7&0mN*(DgiLHyzBI|yv$c5{rWh**))-M zO0gL77LBwOv_%TwL1OIU`XfXEx3kp-=)B$ZZQDg&5A?!a3cQi8T$rv6_KdBtG|D+l=BqY8+L z3YzRdbfLmyXXEt)1q3KDD=X{YUc8LnfWMLJ`Anw%ZPW6>7KKsfw-=aiKaE9xB0?*e zj8sqCdufr{H`rJ%Ns~9}Nmu0#t2z^Va3*19BS5ZAreoE%UGx2(pTd_o?||s{`98JH zglS%I0*N~wK&@nv#8TIt_gnfr8G9F$SqE9GC0@`u3UgxhI@TFytPf!rLeN+GC~ZlY zvO2;cuj{}kCUpYiD`u7TW$L(o*DaPAs6JHM-TN-1gj?lL*6W<;ln3BIZuE%-`%WXd zLy+CJz_1CW57nY*;oJB2Xz#m<)Z~1`>c$J=UCCMD5iykRq;K@2W^7+*ijgyE0OkaD z{C<~dvea%D%*67e!eRIpiLTk(h0T;-xUeN2@XUFfP|J)gS64beEh|fz8D$qr(#^Ns z`|>8TgfDC2JTA^y+YCzA%8k$X&t9g1T!9dytd(aHP4TSoO~YZ1x`12o`U){yiQ=&7 z^a^RVEHX*ceZXL03cOC)t|ov`@|4-)r&!sps`-WaxarTBA8!9)iri=`2{WdQr8v+>AqK2P|nVBy= zH=Gp4i&qYA`*x?j(66lS*78SZY8V!In-YmIVp+TtNFJ#D8p@aY&@uxZilDi>pJjwV ze8F5tN_%1X#^_cgW}&BGu0Cd|s)m00Bc0+bf-T!pj1Va6%~ZlM=-^^>q7#pxV>(Kh zK5_f!aw(*QvY4X4aQ`QRGs2Fd)(R6YF$;M%E?Q#Pw`!N?789}tlIZry++k>hYI-PM zv0WbOTNJ?;BV+;o$6FfIWMkqtXUK8!udhe;F820dk=-~`WZX2}>(0{Lma)#(^>we- zl-p%L6-PKlp7t1?AwOTv7PDNl8t{BFGvH8T_7O(QD0T05KH$TGRBVdu`5~w`$gyPx zb&q6gVt$s;Jd4QTpw{ArQz`t+l8~GQTc#5o{q@hKWJz<$gx(CzgcpNZsCJRh!=v4n zrNX}XON3+sd7R7yz&C#0f^)3qQRZ5(l9O|ZVLw+mJGK89UC6=6X({!5rckgzqakGP zT45Nmrt(^rCp7$crmN@2k}00+*xI5MJWnW77RCnmktg*Yde!i5>6$zHVJMrrUYa|RkM zn2BxRCnlF?MS?f1A2k(X-u^szh!EIe)qOHE_L}&pa=_T*3Al{5ZI^sV=+!Yys;3@! zjvZNF9H$gP`pJv?hji+SE0`#vn-LLsGT!UWyh*WOc1z1R@ykC!cdvUOWPpO$P zHgCC7+%9E(Lp9p!O9>HzVQS?+UP`B_?5Ou|hk^#bU%fK?Dm1_*P;9^o^Evp+9D$to zN5hB=FL6Lz3qbCkmj`{@Zed0~iDuURtp|8v`RV|f6bn-nTbD-#DcN3LjM*n{MELk_ zU7*;}n!KY;mB4ab-!#C4L1T%H2kP&KOzZ!@Yon2rTtBKL;^WZDj2x^L86e%qQuhjt zCgaRLcjFr#SG<<*XPxswvm-q7Sa!`=i9SGMXJkxG?h#s&n2deN<7qS^AjG;FU0kB+ z4nR~8tqj1im;zUugBVPxnRpW3t=Rf=TSugy_{^T&Al1iPSJQUCWJ!LM^V>Yw_{mIk z!ou9Vj)u(vlX}2CybG7v7Sh7RA6lVU_>{;#8K>`LzES0%DMwbCEeIP8TOp3Xr*ekE z5t1ST%D;Y=2#))qHTNuR#&51MtolH!?e6LKPbYUC1O0<9_D>4E+pai$XBGV>`>v{u zl`(B#6V5eFDP|^6aF1Ff1y2ee^XyF;dnAXA-@aJ~2zV$Y#Hz+i%Ha68-Qr~D;w2sn zm4e6UC`;15CUry^LP5(%$!?L_+L^8zPrh-J)G|QW-NO}e2P;!D@cFC~b!9ew(A$1& z%^_%UJ=I`N9B;pt4kdeWZ|G!+ner{6J-ryGxlI-O%+JFWs(u;L;We(XT<027?h7Qb zaSVyvzj(=-TL|bbG!DF zxbwNtummS339kl&b;p=s&}?cwuInMMj8a37*uo-En-GSfyO|apq-Ia19Rp6b!O3n&i)>vGdj+_qUxVpf}Zs-sWK`)AK$gMEfdoKK*tcrGIMiRHDdWj zfGjW|)xD?bgQ&B?sR%H+X#!bDfXVzTbNFA5(7(LPe-ecufBeQjFa5tBb^mP|-Zz*5#fy~eHC2~vkuEWk#!FO4JBL;vK@RowfLoKMr4Ld2X5l8Gh zz`Z<-7?@fH_%$bFv^>SD3mF0Bn2+vb;moW!zEjq|4UG#sb zesmTqazH6;+AG|zf3}e(zIQITv8FVSd!*ZYE2D9EjFI~cuD{IOipBEG)t0d$SilDk zHSFW(8Ij>2f|ptHS#4!bs+#Cx^0>ItwHf}isp8hwv+o5Z1#A#}R>KL#kib9m`)BhW z?vn`VAzf~g3>?O`oq;kP7&b3SPgVi0ZTc_E+a-{i{Pu$NMsNLa%B3YOIq9n?KWGm& zn|}n}3Q48j%ILaz;@KgimHijMqp*4koonHmVIyVosa9mjAVhG}d(6rAl1hAO!;V-7 z)v@*7*1{6`BTIBw6ugDqy1PnAP9vS&Qil(MS|HctJTFazCS%F~g>`72IS@3qdu1K7 zG;ax`N40vaVZZF6#x4_YRS~(6+MphZB#;w`%`7yjog)tiZa{}hNm$|-CR{gq% zdz!P|HRpVYJE_OCe_eSC)Ul<4*wpjv-dWzu;FaQ$}9SRh-^Bidq!;1vf_NbN$; z6G8p6x&^)RSgq1q-FTarjivbnMq!fm7Awi}nJ(#zFFq_2gU|tn{wvZXpf=sM(~F%U zp$aV?Ml6|Wq&^FJl*$xB)cT}s)asR3Dl2GtDt*_5*%6rf;Xsly0s-usI znt)*cK%gzr9L`NYo&78?9x)f-7SQY(FQm>SXwx5YkZeX0QV^vIS^)oO#th4lO><68ZVya zk4Z^LNGOd`0P{@Zc)hRhK0DsqP^I0q7ooYmiRgir%k!sT>}{|wSh$~uAWK3U8ykA{ zs@)_wz;e+{_bE4jado*S{EwSEc?Y|~fHKSrV z$e(W=AdncLn{~er=7GIvO^o>69^Y)Zw+=2)vHHYbaMxp4eyC1G+V|p@xjK?8=%1FP zv=Q>W<{l8b<1aVtt6lse?2DY69+7u{GTO!iBssbknxae}MI9v|-@iP|%ionlBOmi* z#>x0<3SG?J7zW5%Pw#H$VMAR{w-2}Ti;q?#<@0=w`Arp(a%z`FGxLDRc3j4%g1g3_ zqkeZ?Sf75fv<}q!UAlo6Z*n8gNw!G(o(|}?w=z$2P5oT1YyUQ#g_0P1)5>6{%$!vW z5rtZ_o+{20@g#*sBn;j~L*tySU{g~Yl4p)3o%tfVAcc>Mm-sKngZK}q@rf5OTW4=z z7&fmb=~Hm+Pz$%QBTiS|&J*zkM8`K=?C8nzNQzSv=XBd_)1JmKsIvm%Lky#4_wCJ9 zKuE}1X$KKoIVX^Wfdh8G-texffe*vF{qhvq`jOmrB(e|18l|&fLwIYHNBy)V$^iW} zQwPlCPeZ7V2xL+rtpvfEo5c@bYqE2CfM|KK98z{TNNi-$;T;vGwK`&!Vqnn?+|sqs z=gxITSVaJiM>8u6+L=DFx~9a4AIX+wHy-?+XNGZqzXCd{T18zhf3o&V~+g@l5%SRAA*iex{UI_ro$(mHDP{svlG{!zqpGnzPRnl!Nt0J2JwBYg%MYN#8c}U zB^e{Co~)v6Mx<8qnFW;6__eE%%NGG{rsk&Hquybpn7(m`hk74uYP<}^ragc0`@3Zo z>5{GPi0OpnDF7X`7jn){^?84;P~-7y!UV@n>Eb52ZzptME)!Y4M*T6U$z*=IpfpCk z6>nGUVSs*0gK+WA!C`H}%9Uful}OieU-xM29nykFy!ANd(b$8V=wD6M6EZ8HMclGOYse;Pbi6XZE`h`mYI9O}^)VW)*%f8qK0FyU5%EWL zB4r`j>~?Av(LL6qWjxmQOs7a}$igFJN{@4MbI-4?cFJGGDqjkKlT`n*FUqSyZXscz zoY`evWqksWww;ZBX5gVp;X8L|bIya%I@gpweO!q1A zPY)ifj2?GZb~*9%ed-gTvG4xwKRFNYA9bd5SIJy@?jacub~%v8%iSVy0Y^GkQlo5R zcF=;>xvY{R7` zASeV#L>Tp^4!ykIf8W#8pv_zd1AI1jh~Ls+Qa$Y8Y%=tT*z5A2<#8;B%6M^_A0DJ) zGC;oG9z0p^h(DTJkzuKeGL`z3l=?ei+WP*#6HUZtLJPJ7CB^IO19ddE?d;~=fe%wtvtVC;$E;< zM;|0>uUve&WR`^Zn@=EyX!hH3uT|LzuHjkuMsVks#OPh>rozwZ{3J;X>`62QZb=wi z7~Gx;1y(K%vF;EXdFdh}-D*FXx`Q&3^Rf#zm|ZblZ^U<~Of5MqJGI)#j*AHfs5Q0A zF~v+l3`#a4ozCTFq2HTnw#eS}pEW_Dkvd*Y%B+@A6f23HsSEe&Gd$I4-}{N4TE zjQ7i^AJ5?Spai~W1@4eYq8$V1a9q8e4n?e;AVd|KXv;D-2> zj0`h_(ZL4q-ncLSg2v@!@Y~vE<&X?@mx-h4MjZ95uss;l7Q|pE@t0+byVN0rEA%qWW1F>B4r6mEefel30w~n(s?dqc)XQ z)I?22rjVFCm-Sr3iO#9&>Ql%J@gfs=eT1$A!?R{j(iusq2?mDX%l-caxTN(dD4NRD zn)8`2+-?r1n<>8;d=7Y%vc@IRt|~vA?jzcQKt%gBNz2_dK9h8ptt6d9trwh~^amn> zZiMr@y*W1xC5KzcA@ps;6H3uT-&jO3$IPsBIntQeQAX3Oks)n^UJ-10lPvhm(k?k( zO@3yMNSP4>_R3)bKLNb{mR|OxZ&c_*M`fZm6qJ8Gi8dp#>?hWE3Xx#MayxbvB;s#v zm{==6({ekUUx27BtVMu&&BH%qz`!gb6M1esdY`m8?by4@e!3L3fA1IZ8IRSfob$>0 zcGD#FX!+QMA^k*{8z|R(H{&z~Iykn+`$MsxbS;{9Z0HKy_f4s-BYJKy;o8V#sdj

TZse;X*Grd)CFSVMAcmZbTp<9c(hT(5Tc{einENciDX<7BXn0cIQx)M5{1F8a-;Y5LpqQNmw{zb(e!)bOt(U&eI-`Bjv8G#sh@4|F&l2)dXfxPW2ZtghfBg;ZZ zA$bXj!PWo_CNr&2rJ^2Yg z2e{j0>n?$n*ItP$-wZYZ2e_yoX=3*jie(fv{t^;+X#=jse-o%uuVF*}lx8@EGKWq2 ztP^bvJRt|+Q$RXnQX1X#JgO?&FURp6$`mq=ua4yf#<6HFt5H$gwXwiTUjF?jKSiI3 z{V0oDT8sB7*~D?*TUi7&(+zTH;D@N~t7Pumjb~A9D8L~GOBAt6y09cqd%I$T$CeAy z1>GvAS!j}4K3ZRign?eC$(}#u-7s-TPN=4h4oglzW%%_2Uc` zbPaKx2om>iw`p7v^t&QPo{pb*C!CEEBI2HAE*Iaayn~o4&5f^3V(Q@3n*LS6%9g>g znMWcziK?o~GN{k(htULQV+%GjoAn_PqR~4fN7Qo3IV*B|@4-*emD?-}j@;AP^3G~> zz4UT3%_5gdi#=X?c!GNe`3rIfeqXm~ zxBeqM*C5i>WC||Dq8rHtrX09Z8t(&FzK&6TM{73U!{^x;Lqr5C>X8X49^Jx^U1(+(k%Bebm#Ll zlt>sM)#=osUDJ#c!_4cerdaHCBjB{jBcC@%(_o( zE*kjb#&Qb-Q3#4A3q|wPcuTc< zmJzZVFtWC4P)+5BOp*n!%lP$D$A(8&HdHUB3yFg;^&7R$>3UiNZ#g2F~K%@zGMQ zE>J0w9acU``Sol^@s)N&=IPEv{&vR)j$zQ4DE5fOnOJf`ULGH+-B#xJcI7AE1MZjf zM&6e4o#zFgnr+(t$uX_Md{=6}d*`tlN)m;~`5`!J&pvPRP@dtY#Jkx%nSw)KC3Ae+ z)#BSpnx?k?2Mt#bm;^!q=N9?%>c*}pDYt~Mx8;NqjlPS??Bv~Xb!*S*apson2gu>w z&ZfT9yohkEFwJu0qW>^)&V3`lu+2Flje5NgEKs%z-dc$fH|H@FiQbpGD2$g|17bO>6p8Z-aIkW7 zs|ErmB31&;Om}VCn2`ZJ5C-NW;{zV=+c7!7V7~<}no76De%ABFFZZY=J=Ba)E%fjQ z71odT!vM%DdC8FU8|y@os26YV{|ts1^B&D|88vHFG#|l4K42oR?W+G6B@8p}{q6;m zO$J;Sn5OqB%GkhfYN2^~wE5Dn$#0UNp;-S7r*gu-Ptlg_3j-AgS_SjzenQ`$sf7#{ zfZ3=GYdqhfO_zq|RO8|M6?fnAJeq+eePoFyJp0_o_a}QmE*va`e?j*5_)cYoUhBSv zZ#N5mN9Y_2oQjez(L!E_IUuu(%`?RE7p-&l+EsNK2$5`(!u+&Ib?HfgFxWu_(|4W*&6)IHT}$=-^Dgm&hl^I(OHeEnf^32w<# zXFkR+`;yu?k)2>5>#tf=Fr0o&pCjSN**h{)1@z3vO@i|uGySZfDj}VFBE@?UQ_UJ72jrPjvxL2VD9q&v~r$NO=ayK59o{nA~Fc$AmCUK zfdN!{Zvp}WQj$m&Mp}q;LJ3`mFp9u{6zK>FJ@gs~D2PB%AoLb`kzhaqp#%f>pw8>P z^S*bj`{8~&D`#czeRg)9=l}nmXYZE0XQ!=)-ft1YmLguUB!7v&+0G7*+VDQG`qT3D zyB3pjh}de;3G zEbg4Ze9oSR&(}t$#&eZ9SXTC0^e9uO_Fmc?mOpoVD0-Jr8^Jm$1r;?7jTiI6n(zrF z%yka7rpc`Io|i;e?CIU-&0WlF8A>HhllQ`H2=67=xxE~8V)!a>e`w1u`&aun%xRx8 zK`ZMUIwgIKyId4nmhsvT&GYeoQ2~*$;Oa0N_QSbIF7b9rDZLnMylZ83KPTp{k#0nl z)X+8x@=yuWv~ogP%oEM*(yS*R#hS1}$%i);U-w*JUj(V3zdanR5i?*%GQc z^!-!>u7<|?_II-Ij!V+?@V`3$*rO~U5B8@RjxCk-YcIm+;VgQ51J9HA)s7q1ADi}2 z=5imIxgp8T*J$zNB?Vi0G8*#obE}Cw!gnVsa(pXTkFAAuASslgSr*GMP&iAS!^ z3R-RriqR&n>n~idXf=!tF5)(>>&o9|T2{dZ2bKDg4k~2CtAyc1d%W`;ofNl!D(XLo zkSA$jw~`em^y`=Pl$#pWs8NYlPSG^*rojE%#mBhWZuU9O_3{3 zG5soZwN=zIF=u!9=3Dt|!k(-xrq zLPgue{x*ER&aV{MO`o2$zJ|8E(4s#kNf!>9(X>c5ors<`K6c5bpA?ycaPPFmH)E}= z8SjhU*ED=Czm4d~WpyZv8G49ETW7_N;3Bqh-cQ4mdJ5*=Ru*(zf3_4_Haln59^GO@ z)Fnv=&fko-KCEBbZz+6JaG(Io!n|fXmiS`#p#c;U;bk&Cvbke0Qps01ae2dnL7(UC zr`+tJRHmAKG3ou|AWCfgmrwoP_-(Hky*j_aAkSaXtuVCJNE;m9kq`eGTQ1C>;an|wvnKvo#xf zlx_1{flgpV0o1@`i7d3i`nZ`66F7_5_b*c0SEGl*qxM0nNr9sgGJBR9=1VIlwOw5CDh?c zc%B`Zgd4y86vp#br?xYqQa4^?aavT3P+pJbi8;Cl5TD<>pOiR__aji4?0sj?GI_o29RK0A*bx^vTXZLh4b)ACz!CEGIWDbBqLDs3 ztCHO0>2dO-J5UHD&vl{}1|WRoY{btc;>&#LZ2_LZPFbjzSEb{-7j_6?wOmfQa3p|~ zuQrlOrQyTC^WAx#>pm4dgcCa?;_S+Xl(x^~>EII7e&fi+xa_&{x%v6Mezv+<8y%<> zC(7=g^&kVq|1A(i*`g?f+?p&G5le?-V{T z?V z7`zaY@(t2tc!6!t zf^9;5lSBYZXdE6eR++E0U3xt>dU7E z+YQWKVDYV}_kH8XwAd|rRx;dI!inx(o;bTz*U>F*!gPa)yXx=7rOR;vdJ3cENO2Jf z<4xE3>Y)9W>Yyz-v1xPaW-<%tJzqFim=-nhU>4Zg>>;npcy6-ZY=mzPgnBnOYDU%Z z54^?Hi3A(R^Liy&6gT%e522jw;owUhIi5&cqObJzK%2idXY}454(SCb(J=A+I*I3o z4DREnfgW*_@mz!pUUl?+ff4nwLDFGSR>Lij^O1}!@udH?U0T*xV{N#iMqom0(F*<4 zsCUy=FAPSNJ%j2jii)|FWa^I!<)@46?I;-(xMujQ4UBhI&>u|RQMOIX*}JjxfjDaK z&~bX?`otZ{CtBqROR*cKn!WEc``A$N?n6rU%)+D9Tum@|mev zmZBxOQSZVLO)H-GU4k-I<;Ctaq}Og>tIJPk!OWm|r94tvRpgFcZHgX2u~hT?(hAHn zFk-A#DzrpsUpK)@Jw9=gxrcCYw^JL^!rmN3%O z#~r~(uV3=nza48BXbwNb-*<>d*OG&*WifdQK2{xzx2Mn#B5PrXFTLbLck7xv&lOp= zA$7@lcE<2(IRz^wqAs1Vk%!vPGNbpO6WjTW9sa2D@~jTdhL8U){-DX4z(L2$xJ?uY z%WqiJ3g+8*suo@|;tlf74kC|OrIv=iV?3s=*T8e<3x?mh7B3pL6mQL?mZ$ouVc(XT zRnBwJ7vs2*j5fp63apy&1doWvfED&mqDJl%@o{iqH_u-4(zCTL^?Q^0{LUhV80MSh5^>wr<&o)El|!oT_?pwQ@5Tf0oJZBouk6t12xO!JwVjat|YC4{~f*`)8=rWm^hA%|*{*D7RcjNyCJ z3hI93Ft3=5BFm*WamtG?O%-Zo!7GK{TOZU^BiQ*cyMws|3Hr)%vxeJ~z`|o|v37AF zDM{>5#k##R?S%Ux#{}BCmxP(cyAk<=RZIvXMh=jv0w?#CQbZOY^~RwCtPPCMQAHxg zvmA@jK-pc*=8nl4M}vrI*!ZA$>27_c!~@|BfqgU495=tH z)2>`vC;QPfMI<4s!Ls3if3P$o#E<}dBjeBxVV=yGq&c84XIO_!ZBP)!(ne(s%q+j4jU52 z8A=x~s&fe$HkAahw>fW5oXAeKJ9iW%OWNhX+0?H-@E^eL_j^~4h#ryotVo-R&gEyE zPxa?H&i74v5YHT0+ONip^{J_agJzyep1}={4~eHFXt;&ZC{T&oSKI3$44Ah{_NNd> z$Lb%gxzQ*0Ph;AnO?PfLR%UP$YK@?G(bi)Kzr+=E9NRD)HYi8itgZ9{ys!A0)bZrG zUw8#OAYC%%7At?66y&2@%~M?hHAQ7IO{M^ZmLzAw6qLwp^`kZU$OyO6_E4)S{*6Dz zrWDSM4(Xk8bw_A{#E%`_&E-Goc10&}hi_PS8IunfWrN69EF)$X+0QXUUHvI|h_{pe zM&m(Y;VKVYOJV2WM^Lu>OMQM|Wwa$M#^9D}BoA74->O0nZx%exk&qh5_We1;&;$$~ zO1#Fd#&vyllzD~#jKC+F-4kV3ptOhY&g>Qa{6xF=#4G7fi)TE%7`_(_@ zo!pO!`Wp(^76Rw*{fifXY(>7Puv1aTzYqry=#JKrr+?}x$MoL`fIc9xzEX$pSKs)9 zFiMMg^1^rA@Y^hfBKE{CE-WeR2wdEP!3u%CoiBaC0`zSP9Kiy*qkYtg%W_W!BpLL| z9=P?75r0R!?O3sYjYJ764qEHI{!F`fmHXh<^PWiWh_P`?YvxJ5Z@-uiuzVrOVcK(e zP1EcnX#9PezI9N37EoW`kjl3!S!lsCc6X7S#*1wqN;z-%I-FI+inD^kc#e#>X$nt% z_7PK-v7Za3Zy7~Ay)6m_6J1Ep>G3vuN)ubWOj3XMjxoz??~=J@k&;5>^#T8N>7!x- zkE|i-GKJs^xi+CSN8Zuq*4Mlahh!g6Dn>a%NI_sp2rV*dB_On-vp;vrP{*JhGbb@fA3El8vD z;g0$B^`qpUKrf!Dws^>AzOI8=eEKOG=lJLEG89O^WoTbGEY|r>CqMG)fd!tE7T)g? zsY2M(DAQ^#?!A9LcMW^$7t}=n!ScannSi-(FDNQ1s>Gd~3HqPLqO2i^??{<{=)34T z)npzX9!_}o&TORAIx#J+_|+>lewCnmi;IhhUZA|jP)kb-F>pH`kaVo9tP6{anATPe z+8W8Sp`n3FrN+m`#sZ%Y*1(|-4h}=g%gf6`x=u>6wy!{xDSYFaye@FsWI_zTVxCGc z{o3Wrn$6A4KpA5)(43y09(g!9F%e^!p-@;*P_TB?9?oHBWkAQ=+}x?8&Nod%L!&xy zXAMW8ZEP3=2RJ6Aq?p%z9H(Gg3@GC@0|00Z+1~c0jDxwkx#Q8pSW8h^StwAJsb^#a z1%5Uaz8eH{n{LJGXQk37r6GfYw3A^Bw?w?24k zsj7lfFMMT^A{KJQhcydFWN?xN$gST9<9CesZ~xP;;p1D-_znxd2ZR5V2L2zvl=v-Q zdAJ`L8I8vqp1>vs%M97-hj#;y1Jczpyj7}U`{=&_z`EUw diff --git a/dpdk/doc/guides/sample_app_ug/img/virtio_linux_vhost.png b/dpdk/doc/guides/sample_app_ug/img/virtio_linux_vhost.png deleted file mode 100644 index 0614269943b9c90ea93e3e255ee9117d70c7bd5f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 30290 zcmcG$Ra9Kjwk-<5r9yCm1%gX(*FXXUhv4oI+}$O(ySux)yGsh!;BLVw(F%U97G87aPNc_k5pHNWH8o(nT5gvFY3nh>g_yub(EUt(M z{CFT51_RHLY<{TOLqWaoe|tbD(7h*sf+B$u|1PBHl76!0?1AO@a(}rH|K;%a2c8^q zRr%0tKLpwV7y=hWr?1N)nLb4?d?M#EJ$}x_s%ThaB~cwM)a2S6B_; z>4y(J${%1!VP^{`zka(uTTA=ob)QpGM~dhtSEk+JR%5xs0hMoKK@vwE!YunbJG=A& zR;YOHA7rwCtv}T4t-=3k;d?bUtPmnJx&@rj2jTzS`2W4P5XX#cdk2=iBS_DOtE4kP zLLX!l^dYPtdCXnEPSN;Te%K{?NZ{xG{ol7l?KHcmXiCw6e*Qn}(tjuLe~+8|=&H|!&%l2J8>aD&hnB)n4uLgbVI=PSW8((|tCt}O}HL-NYZ+Of(L||b^ zN&G5E5QPzgb?FfOKD_(9i0nra;Dgxs|NAXi%0*)a-E6jKxD;saG~g0pn8~_rCR7AV z!;}2~eTD3Q%@yLWYAv>(!m&pX)`&S@Da{YCIUg}8{-`3sb5XtI#W1{cCAXlz!#K1Y96g-1kYmH5_jdC$s7oo^lh1I^mv+J_Sw741Rd}{g1-&o z{K4L&YwpAw@Gt;fUI)w;)^Vy}0HN2Z43tEzf^qQ=I-K6GDSm&`qcFn$nBJ#%)O zn|Jz^Gpf}zaLUTiFj6>Y*-&B9!BHFtx@#x=O&q)~z=p|e`tG?40!`@rzCt4&{O%N& zB64HqTQAvCImO{bRJ?0wx~+DMvP_?%sf|p`mqG=xFc`q?M7h#(cTM>Ad%+CsvGJ8fq#6_m1-Bw4pw^0BIB6{EHgH;&!eNKbl%P}?Vw#$~M_X7MlOh{N z3i=XyUS=l^ZDPI0aWK;`_|wpF#YOkWKvjR7YlT3N(vw8`x<9j|?s%*lmAXl$6H)ew zdKvt1s>DqKs@M3+uf?KzbPVGB5}lKY*&+eJNI2JrwX^gT{B$2OM`QFsyxyfE4gk{awab z-Eq4TBIbFyT!Xy&=Sj`JxHhVV8U70BJ1N8zlKG`ai8d)Cr`;`vk0xk{+O@)%;s%cy z>s7;Z1g(@;>RCa}?l!w*ohkO-uKs^oRa~;K&8yGNTe{#2T9v5vj5~VTDVU~oMx4xz z`w0BzD>fVcJpLIEbsT&-_4L$C_<5+p$2+9Lh!XGR>23LkiFlDGlFpo4C1PQ^*qt&! z*r(OY$mSMGwf-8D#blg>XjqdeGljsnB(Hc`TyJVJqPxXcT>X~-yr5{${vj$6OZYwa zZW?4t`tJ$Tg3hQm(nCqS}ZP9tkL>aj58L&I2NDxf*E|ned z=Ods&UAqSF)^&jFuV_!^J$o~x(pgiO56|5QQWCY@mKRzqU%A|hrYh=PxAb3ynjitr zY65Ly1?Go>?jUoz)ws>R!Jb&>%|N}~g{CvTEvgO;FS)Fs#Xa7_NA_^*MSe_SQ5I=1 z>Trs(>^J!PZlQ` zcch>0EeeRodh8)ICILpKc@gr+n)9N8A)eUBFKDX)<`Z zE}RtghvRwO^b!nsM3iFhJu)p~XBp>(5|$!nKos6VJhEAv1C!_bqI~clfE5aMh|!3M zjve(wsxZ|VvJ&GkVKL~SBUi(K&Djn8t7~4yI{mJ`i#o#y7ux;3?Q;-Z&tw7#;upmo z_!S*A+vlpA753BEv#0G`8TvM5SWTf1_>?}5jRYwv@~&`eF=0LttRuy>wN|@o-KkyW zH}0-ZXwfm<=czobK*B8OhFuA-Gd-*$^5`gos{TV$P-6cXQJ4!^ie2~-l_NL?NLq;O z7$^esVG(KY{V-PVjW`{(0RSTwv+`Bh`f}CTn(7foj3evsap7bn38EeiQxAtdGz_cp zPnEg_p9l9vIU}nTN_%6fL@jKcJ1>FUE+A`1puc5QlMG)R-R*JI{FJX`g(|4Y;DbjB zM89cOXsyw90YUdYG#?SNn8J`00G0$3l3Kqlfv=K2h(+kbXQ?>0=m-sBWK!J0*=Q`utj1rtMEuqf{v^3vDee&X+8pyplQq1;7Bf;89Dh z$#|lq%X0p)L$iyxdr-Qmh_%&0LL|NY^-=p}Wn46TcPCv~32O`N&gQ5LOS)6ep`l=s z)AJ6~LX5tj4Hte#QF>^=Tjx3DRd}g?D~~F@@0U+{+$!V+V$YcMb7GjN-|P9s7<&3ZER; z?7gSj1g3=EN!V|lXC|%prT(d$a&)=-;(b0+!(9G!VXzM2X*WhK|AD%2Go&&ARbao= zxcBLOV*yYk^LQPnYnuIi7u;YcmS|Jt<-PfS=I zrL(E}7(YJ-6cFegm3~w_H(>I4NRVh-rP{T>0Ah|3Suf-Rbd)1Q*c;B zfmWN7bVdR^%CtvgUh=4@aiq>y;|$)ay~7j&y>8V?)S_4$Ry}<9LOl}6Tu$K=M_TtV2UAX`Io??La=4f_ZENd; z%rgW%<`+zq3~GHQ1zjedBgmq5BKXla@P)ks( zZOg=4$yMhfQ55X%;!SH+CBE!}GZ~1W{ZcBSj&`N?+uoa_4S{t&!Kj)3?>UfwUES(D zKanU+VK(#PDq{_-a1mC%<0vNe9E3loMI|taB^6)SbD#5)pvqbL7I*i^Q-?Wrb~-6d zGuOn}od0-#Trb&Lcd{T$L(&5`e zUolFtEKzXz2Mu9eSA8#I9m8(VDv`B?k7U6nGg!}WAQuDoQ|Ge<438f@&}S)oYz6aW zy>5@&WaGM&fWfSd40KrtG17gRo%wtE(%+LAym{2fMf|2hMC8S{&`asFiCzExD&-*O zPnNvE{|;}KA3lG#Pa7yE24pDnMH5)^(Q&UH)AO$(dK*Bn}R>+Xy~;etNG2#vB#Li06{vs_wdVUrrCv zB`|jQw!bqH*BSCl##voBgu0Cg%;xI7D>3hJ!|wi`IM>{bCxgdv>mM|XxYb-}yfO_J z;d?uK`^7GA!IdWK{i}VQeg;X@gO#AP*9(N@{KB7GHrWU^-=oOIzl(`%+O?=XWpO@i z?q@yRe8oBBcZ*I<9Ss)pKm5}`C(HN3wPCg1YhKkgxjruCbh18CE4wf~8Z$U(U6;aJ z1gc?Xj^`qr<)3rIR|yYT|K@hl4XTo3D%g_TR^xIf?2z1le+Ce)y?Z*AuCe56XU|ui zoMRO&ia$Ml$y|q8UYVy5o8naM!Sk=DkDF+c|#V z@KuHiPWa1M%k=?ri;=91?s0OhZ06q(_p`mMx=;D`Ly^szHw4m!)GRYKwib)cAwyZ7 z?RDg@gE!-kq@y84E~LPE+FNw zlWf%EDehVMni%MY4>H6FgA<8$wOHp9Sg8&%2Uf{=B0c2uLYF5S14p7H0@~i69{&fo zKKRTG!K?}I^vO6h!S7A=lUUeEYzJ;q;C`#R%^Am+$5{oBt1HG;7FQPUbaew(I;$oE z@mQcsY1Vr$`T68tf*xF5p9Ion1tHPm{6E8wF$B z`*zzz;FinC!>3wzPTR}N+%e5$7U^oW3jNT5;Z2RA@WM1Nc_l0!$CKaz{SsE2#d2lj zuuvD>NlkS}ne7p(ERs>;`&X4ua#iLeKr@7BbA+Oa@6zwDB}TrLE)WIJ1j zhK3$_=Lpx4NMcwu4PNGnrSZ^#DE%xy>X1k?Jg-ez=$aI=@O+H4n6$24=SmeLJMM6qk`P@vxsIyHKfhxYEQ} z!am;UFgC%teR(jRdBisVl7UdUm@# z-?bjszr6RGUYnhdQ@rxHan89$_@cl$#yxiVw~UhWNW z7`3G~Iz3aE^!zrwA~6Ry1~d8FQl57d+FNx6QVZwQjv^yNHVX!-?AnWy(Kg&x&IO30 zf}Gb~gQf&u3AGluAt!a>l9C2zt1UZwd$CDLLt>vi`@{3!%o>zu1EzoA5II%uplnzoV)+#wb=FFMU~)F z%CU~%2)YmdgLG%8zh3Ij&d%lxzgJjeEV*8t`u({(-NDQiA4BIst{LQHsV*pnOw10r zT8+)7Fhk2uEIx1~MMdGgU#_xJc%2#UR_)PHaFsuO`qiM`;`z@|TdOs`w^!KOIs|`_ z9%tR{oF5+aQP|DxCT9hRf;){XRtDDxGkjO$teQ4Ga1HajnU3>nd@uLg<0-5`?&lpz zQvz3lN|ML%SXRV>nG~PA0>Hb-$BCyu$i%Y=;8VrS|HB?SSAJU54~;+wj%uO}ZjmyV zP6EOMD6@h^V~l<(Xh2%M9$dBLx$JhR>ALRyy7(@>HY`LD@eXDbm9v4FMRXXR+SsK=zDGCD=Pt~5h%3-_plZ7Y)Txl@t#PuN)I+=&TRLps&jALesVXx zZmRP3vEB8G>-FiXWj9d+Ki(^fnBV2`u*mhSWuMD->wCk3b^9e{=VCRwia1hj){~lI zLKUY~?&|=|!D6Rp!2)WZ?s~hcrt5A(INnUP>89Jw(SVlo+Q8Gp)$=Wn9M*4NfA|{x zn^&_ho+uOHJD%PF_I<|v@GE;5e6mU8Ba3-?W>4xB#kA&}7TL~O0(06cou0fJO;)=- z2%jtziQ&XPSw30 z52>uWo^M5Wt$PEIfPhm2yx3J0b^mzN}@G@`U7 zZT9f~D(iK8G=Yv=$w<2RY^`mkLW9$5y`fDV`!~5bu-*U07>O#G8nQ~qVbdwwTupEy z82TL$ShvRj5oPGy+|<-GeRxE#uW3W?4ip{iP@2njj7HqYR=slS07RHKm@#p82Cs38 zlz8rYi(3H6pl|RcPg>k1P@nGW%d@7} z!|sc0yPNuw!*8&%ds5on3vV=>SK|7?QXSRlO4B|NujaJJ4rcs2)DTLaFH1vvH_61J zu@eGl$tD48#%=Q#>KSr(7uO<%gG!&#=R^X7FrUfk0}TUDB94K8M!ch&P@s6iL9l;! z*cW3U7V~@^5X@Ff59GCz48hOA_IQ%90c*y*6;6JkLy24x!;|mhjH;i551f}5J{kWx zymb)OiJz=JinvYKP2X8ZJDy3#d8ZuqJ3j!nJVab|&8q#25k~(|Ok7-;VNGmuwA^7b zzzG3NKRjG)-tkZpjt@;Hn#?4<TrBf_m9Ccn`(13x{DyH*(iYv z!4{h|U719&I{QqWiL4i7*CaTMOs1qcGwobEaqr{k>&8Z7Ele3~Nhm~lS-I8vw?p>7 zhiA*aYa+YU!+YM&E#_L({e648eAIpL}@b+bPF>-5V-Y{}4ea4pCgsQL^(saH6|tyk(%-}Xia0n&`oSz zts)`H8MuND@>0BQk=pUyvG@VAiKHi=EE_5Ymlaex2>tS+W>WD}t zjzXBr%{_(>%)!JZ7<&AoLDG4tq$^F)>t7*zi)8$i=QieMA0HjxpMfUe2R`g(l0NAa zv{G8A`7VZTzXPk$^_rQ&YVs5KQD(2U-jr1)3Ar%9bI`$Z2zxO#JRI2=q$ zqg%iR`!+oe3S-ALhFBEwfPPK3DSm4uCJ|8tAwf`sIobsr!BU=HNMvBZX+Pu;Cow5$ zKO?V+SXs7=*ZS{i%4XFNtWU}D4_l<~M(MvRk5#2G*Yfhz6|tfa!bm{#O?frZh#jRz$PskjCHEe1-Nmtg< zjb{gimHMqjtQq&g0?nz#GNEFMe0jC`!%s}E(-4t6tSj-u3YVh{duh8T*G&-1aDu>s zB8d^LHoV;R$FBrGy*jE6T}fk~T+Z#ye&E*|FP8dC9_b0i8I*m}PBiG|vC z@j{K<)y5Fvg^O0a;GWNo(ZeyWTEPLYdE^~{^;J`oZ%q4v(;oqC@X#Ug+1L4OHRZe$ z@?IMJlw>6~-p%`Q6V*5$hRg#7*Vv-iHY5-Dq}}BJF00}22SYW{1xEm8($R&zOB$R; zpCdRV2d`zkQ2mb=!0>L~y3F$YPl05{HxIG+_l2se&g)QKPnTlLjh4rE>}hM0!{%bP zOBWjpsmA%LH|sx7n3++Daqiha+X39LpeU<5&$Z|W8>+vh28V*ju}h$~mM5FOB!!

Q*(}-zd$D{PY8ceFPhGq2%We2T&;6jK$c!1DJ#><;_ zwpLa(++YeecGH@k@_uP`D43qo^>;d5MkgQ`YuZp|yy582`4&V?P5svoh2iZ{e4RS9 zy*b9%mG1~*O%Wl~!4?x1)|ZpXS}euxU^o~`*}wkI>EM>`$ieS^N4i~pyNu{Pgm$Ho75{bT{eGv1wnK6r&)K7ESY3QOc6?cb1pBC1kP2fZORrHG-_{h)z&23rOcoZr2B9 z&n!(AOMd(una#9XQ?zC)+z|jzShap>wBDGM6=b(MEg}<(tadz}$KO|8Y_ue(9gtAO zVIX)pjprqtQ6X}E-A;M%ivk)xG*um!p4b>ivwp1fxh?xmzR^$B(&ry zc&NyFUOV?1ir`!ymgZdR&*XQPfviUH{X=YNX?X+GbK1Nf_e>j>Yob{w=ATIa6lge; zyJKlIT9n)_7t7}zfQ!(!2P1e-(O~|O+ z51u;e#o??|`s0?E$%5VjbtZ2>S?A>+E{DZpcdkg7%k#~`M;1fGBpomAnwlD;j8RDm z2`-0&53i7o*ZaLR>*l9_V<`fjtsB@foYw17R`SfOlrohnFMnE#z&GReE)Tm&DIAvS z4GXwm)cXMx>3Ud@q}uMn44})cUKB=v-Uco%s`3$|xw^WZtTaU#jV0P%AIOVF5l@hB zq#C@mzrH{K{Ae&8t@l#+F485EWU9E}m?w2_5Jyj|QzArn#V-Qy8Nql>AXA%8#cobX zMakXsZ@2ID@j_ITz+>=48fTp;nDXD%)ujs`K#(ms2xw?%Q1O_2N)(Eek6q=C4rcQB z9v2P5U%};ITIR@>8xG>O*C|6o!~2TYj-%2tJRS#X$wYcRV6{E|jZgus-gajgU6TQK zt<^E_ThRNrE4}eZn_J%UR*YGzGqjAUc>ipZT+nrD^#anJh^rIg1C(F z*5jp9`KRYB6qhH)yO`M6UJ#e6|AFi z&7RckQlm}D>9XysLDbd7#>?RA%UzfHof6jX@Nm?}+tV@E{R|b8?1{-q5O}6V$ z_@=WgS1Z{w?-L}eG$vo^G>*B}stMz_a|z63bb7iJfXb$gR~ZA_|(7d(Y+VTH6(yuc0L;aCi@cSE%1oA;c*257s*z! zM$a4VjwU=^>AY?YiYv5u^e05{EIfEJ{=MAS-YQVh1~gRfJwTVC%1QKUEi}zN=8-9i zJ2*Hnt8(WfPW+-V7>XvaGtc`1`!y?!!Ag->gAEhV<`)Em$g<^uI%&ybb#XBM5oy;+dOdbh`_Za$#LQZHBRRx47R`uLw&_Hr$~hVG%}G zdlQ5_4#6u{Z4r$`JX_Oz--ia6jsGZ4azR$J_p1hC+9kIg%5$2;vlsJnpPUH7u|6+~ zRtNNxTcw`>t2x9!-8S^)x{((;yy%HO8<|RGtBx(4caHZ z;hb_9dd+%7Y4D=+1=HO`hM3*)ymV|=;`aqdJ9|6-R8d=b^cHUJhH{)gwIMtj&`@gf z2h%l}uT5?gX`Si_-MJ?Fb*5{$(&?;>15E*ACu-ib8k26voM|oYe_W9N`9p4+23Un+ z60#STmh*ChBO{THcZUhS!D>KX$dEkm9?32?pGopKTzh3$WGn|#RDId&w+>3YDccOJSk5F>6j7#68m1sQN=99YfW!p~`seXNVbW&Up7& zkln*@*}ctD+p7DJVq*DY{6R7vUxKobCC|c|11l7Sm34+6-&yd2M2I0=1Rm2+ys$W* zt9XPkaMN z!FekA2J5>)x^Z@PVA)fEz5&f0cNls~l1ViB=@dw2bBa|cDAcUvrScGb3EClgM?^sI zBv2ho4h&DIc$jU%7$O#v7Uy3mvd88rw0Zo9eNuuZ85%Y+H5ImK28(X-yE?CB4Sgk- z2&=PFClGzrJvBaEs_pia>~;xQyq>B(%?RD7P+@s2a&jqVY?rU9iYQ3p;zH77f{am^ zLs6V7cS_YEu@!$+>?=j7pIhnPS^6YryRyAEHiBdSn6@V4AQcWc9p&Wh8I~w1DfQ*a z>^(f?eyr~kebTui?DhRDY1cpbNa0uU!3hXrtvO^fe1fRGk%yqhTEbPEzYbggp6w^b zPT?2-p7Gy{&{FqrU$U({{U#dWJywyowFx>N>>rFxjt`&&=0uJ`bq?{^cp%1hAdIch zY1+7n^VlrX#;`$E{P++1-~EdFb}bpIfq{W!eK~)UcogLkr;7y4&6U?7I&%VNVzopO z7&WLCc23CMDd^snPQY43KHBJ`WWawq`B%kXGSd8<67ckdtC>G_2{=%q-S|^v&0e_i ztCr))mx9Q_ZFmM!QWhO(YMgYFkaW5yzLRs9`rfNVawH{r+fcYqa5=W~6QN`VC+3bP za}kv$FxDR9t?-2g@?VB)>G6l(4RC5MC{JOS%y$Jbx$Y~JPtX|3%89~LManut-oCV|7yDP^%K@u`I71Si981IT^K!#eZTzQ2H9#g!>7wC#%q3Ah% zjv{Y8s^D3v1goW9(fc2}+iiT>YrK11$ks;UNCSlpdWIZl z2;}y^`QY&oT{<@3Gl%nqi#&c}+5X88d@Rc1(BRtEs*zh_1 z9#Z^a*c{Ct@a!HCIs|+hRbk?QT?y($HnYMfS`w!&u^yU{~14djrVa?T2sB^`%!()bc5v-I~CwKCd6W z#*^K@7Z&O8x)gnWePSA+%lDFQ@v2JC@#F*;v)`Q6(5W3eWs{yHEO0KfIigQD`WSrb zSfA=O7rtoaE~WNGPrYzMVq=z&hR1QhhYLYDXSIfWV8L_(gqQgqQaae)n<2_gn@OIhK5e zksl-t2I?(d6y)SKGAme`=sYFjBul4G0FeV2DjuCmJ8r!LU=5O&m*?}CLcKj(BdB+H z*Mot5-=%?W;s5LY1oUQK=WceS*kH_aaHB&vwq6>ULEK=Vx#b)ydp{~Ihh&8yKln6 z3#;j){Mf4wr#iWev(FC-%eW=1878p5pk6bpnmJuziPay8*Zvg;}i#5b@-!(MD!j6>k$j-?&| z*kRt9vL~)2AR>q=RR6U-cb${b?sl4cj({AWPzY;SJ|T$Evy zV=O?m)2R(8vd;a8{jRV7nIS1GHa50!Qf9GEy`tW1F4SNs60YHBSieTMX+>OG+VU2# zzEA*&Z*^6ZM9RP7gXUY^V0$QPf3`${oSdA^(W&%}t^)-WUZ7lMaa3N9x1ebcQZ7r_ z9ZN#^60z3i{H8$x1sLpw2n!u``ww*)BWYs{dUx%MI%pEMWLx`g>tLov2qD*&-UeXje(f9=Z(_i-8t~# z4ewX}rVTG%g!hCO?tsFe0A?%G>RLJ7d%Bw1pWxic2PFBO{rv^?E$te8I98L%{OaoJ zKkhk|kfSK@q2X+?JReY^0$A~UYgM6X`@d1Tr{{BTF>rmYIZuX(#$)T)?$%LRmGMlj z2wro*DFy^^9#VflE8DXB!GbBFkboo}(^=`z;N<@fh4R-F&VneJ!Y=i_D# zHar=t-mlM3!03dO=J2yj4$oUFw;gysa$0FNbjbsb7C^0S0aT2m@=hMtf8;=+ZWAc{ zcI(&7KCYLzRn*cavA?7iiUTeJpi;40HzhzuyVw^jodvLxR2LIO%WK)7_eA66<@9w7 z+}zy8fF|-T@$5IQ_AeY9ocWY9fWulZTQvFLi~~TmY`RRPiU%AK60$i;R|QDPC}|VQr=IFr4n*$}3QhIvmYTOh>5k|JVBRJS zkT@=GSFDrP_EXn~3IqWpyLftQMxm#*&~n;V4s2VbBP!2`Gmb~=Y zNd9T7BOc4O9nN|r#rmPQaz)IT?e_x*PhZ0t)gR*Gn}Dn% zBun)NfB3CZ3v5pxCRCs%e|fQ^k_Ir1<;d{~kLx+QRlyfHdWoy!!mGo*CIBu@eoy~hH1$|c%jbR(wp`h~Zqk4&=0e~+`Cu7x#gV9CudqemyRuFD zFL%0(8VHWujSTHqV{OljzUyK~`yNQ&t*M?&3o(!T~It-Mmsq zG|gWj-`V{>IPnpK`k0sob~b%UeJkCEC%zz>u{dd9)iB#m+IccS)4{l`WvUisugD@Vuk1j3Jp@!6cX`VPkJ9I@Lxq48!9w}uPpKj z!>GK#omv4zO)C+bY03(SgkZL#^%8U}H6bPXb!Zy&AFvvvCqL{=aX-sPe4wr^k52u0 zC*>KD3jge}sK7npU zj)WzWqhxoSU6@B(YVDdkoK7Ib7LBAgbm){=QB2R48Pc?(Vry$F2exg=i029&@M!e= zz<{N8E1mfnfN^?$fM>wsHk;C*ZaE|}bf6*xXrCIK?oQS}L;>d+So-7h2u5HIf@ig_ z)xj^~re7qOz}~TkQS%b21cpcgtd}%30}Zkn4-nrkU6`b5e)^LHz=fF!SrrzX*#HKq zSlCNjBq8Y`t2)e2X&*{Z*x&fVr>4EoMaOc#9=3_-@FQAUh8@K-aYc5Pm0f@s^+qD) zuBecttoxL~wM=sG&;|zFf9J~_PS=-z1N3{VUlY?8k<;dsIY=X>xnfT=aHxhc`{OyH z*%#h8wl19_ndHwpOF|fG>hYn977>W5j|&?SI3RcHFK4uAsu-}&wc;4XIM&}n=JrmN zfru_9ZR0lDqoli;{CKUc?6FhW z=Y-i~>E*LO?In2%+UzWnN@31hdPg-csy=2qa-a1U65VXiuqRf!W_AQQ$6G>7mItBQMXD{$6oz^fFFNT-+*xTzt6JPY(}( zv*Km>SZqr5mY*AARFrX{N z&^Te`4dWn({xe-GWd}snI<>=jiU^_<9a|RmxUDUHpz)EhDHt$02M2KxSO{1NDq!Gd z4M54hmLAinE-oWYI-VcCY40-!C5}&TuCRqN?|Tzwnu?#m^a;ndN7!0PlQlR3RZmwS zMrYJKSS`V+R5Tj z`=F4d`o7Dz9|8=d!?>^CtZvJJr)K5kS8AYX>Y{}*V11Bxa{pbgq-g~z8}J+2ra4Y6 zRQ*E8YZqRWJgt-o_#vEvef4sfLYzg5$7wS*?9o8h62#zqv#%TW8K5)4A(4~~ zj>!AhhY0FOcy0;qjC1ELDx*-lLn{_L#C-i)cLMz&@8-a=p$OjV1H7`Z;x79tLqTwkNvoJ^91W~8Ly+%Q$Oy!4#Ti^+bw z;x;6-7KkC(;{krXuv^>P_>S-8YeDSdw6CFLu;F4+uu75MHBJ))75~uErK}q=V!F|kK#~u$c3t!D5rF%8ssAl%}BoUdEm$J zL%KC9o_h1K@##{T8Slxrv+Fw#n`!2io4~&gUR|U5BaU3(U`sAu{bd{klm^Am=fZB++JvW-zudwv{s{^0jc8J| zGN0xCKEko{y@q323S6#zwD`}PueQMSp8`WlP82Bc_j%oI_1#^j=ijofu005UGdVdQ zF^u$;+%`l;-lml;@#W5_)#0o^y9b7ORp^(6LyP$FtS{~ha}qSJHdN=w^noMX|4 zabpmm;!S^6G`Bch3=)*~FF><~`^)7b*%k+GullqyiGpUm%e9WNn?Z)zvEr>dM8L7y2ZH4EW5*4r2pKqC1iy3)oQ^~>Y^A7(6c_;z_c zU_{PyhZd)hyv`u(v7pX8nbH&Z^*d7bmBHlmy5`pqhZ;)?x04=Si`rDt*Lx_2O%7u9 z-vJu^Wa6jAMgxyc?R4Q6Doy6455Hf02SsDavnJ}8&g;KBtXy1e1R(+tJ;tF4nFWzX zM7|kdf5Tm&nuv~_+1yy52rP7sxTB?K_}t9;a-a{=`(_bk3`}`Ig@Cor{;~?S7ozPc zqhK^-cNvU;w7BUJEtAO=yH{TWQqpbN_Cq)s{hY#Ne~10Xf`Z|xq@(Z0Ju)NOq2n%Uu5vtbOzF+AUUiJYppe{@0K#6+s2dw7Xh(YQ zbBgieyv6n}>v^Fj%M9nQmR1~;1BmL{kt)XEkxM>)k+hi+?LNQLKE)F_e3m!X#T0wv z$aG~_ELEa-ur$R4Jj(8lLq!z~ z;1cpWMEGv&^Oc;F)H6`Z5rOI?t2C-~&Zti06+{?&qBYN5u-~}fI}d+SQP*BbzHaG! zr}bShhSh_tDj2-q(9X_Yg8Tws%Slb5YA1WRHF9SGyxZM;Hp@fS4xPj=`JVNNETXW2?=c~R2v@TwrcO|r!udYDfjxw41&)za`T>s$r$h9ag}I) z2Q#v4Vc>}q?`%0y?9DfoeLAO@Tb)^*;r=%{*OYY)^+POG*2;4&Yu#H=5!n}<2nJEq zT!$0iKx(r5_0Y(CzVnjqIbD}|D_$uE9=8Afc3?ZGoqQFmriRa&=gZ= z)PA6$_9Gl|IwnZkPi$Rah=%h3~Y?vl#HN+5B^I` z79&NIieGN&j4wh)#bZ$QAR|cKTnm|GwrlxoSWRIba;DBL`>@ifl|oj?i$n{~bWkjs z0DmJPPdx{-NiA*q?vn>Ia)Fwr4(_=0b!BhR~j#o`zj%Y<8kw%DfkYMDy3X zNkkl8vUFSteofQKslU@muF!YCIFT`dSTKLy9r(WNwi8`5`y01nOGqxNkK$Avo!knnL4d-@R zrsp(_MkYZ`M{|h3gFpZDyn|x3`uI}u!t_j*6?9oC*% zW=hdsc^n=|o?lWH8h&C*Lg+J%j)`$<5i#I9qXwQPdB?`hN>Ij0CMi_pI44AJ8D*o= z2Hb~OM|<|Mu#rpCiTJJ;xi3jbNYu-uqd=u<4hVxd;@-}IRb*e|Oxb(4*-ftcKdi3QxCxhN=bhop!uk=t)c`T-XUR>iiM_oz5}U@>oD3sA}oj zKtvD4Sm3x^_e{-!vLtL`0oRE4tC?JsWwx^zu`hlJ0+m41kpg~t2F(nZT8gA*b%&>k z=a*(2w^^GngZ^UA_-g!Ysj6Gv`yv{AF5kaeXhGOs!bYr1pL0Q_ty`B)HwVsV&`!@9 z%j@IFzw%$q+YC`-P&LY=TX4AITxg1sZo!j;OtyEt`+e;=4n%|T`6h?(ZzI9q^PPwL zE=>eFsipLg^>^8r=nvIWNd7m=JEh+;f;dz>PpS8{L_iq8??+e zR@t#S_Zt*LUC{dEIFxmVj)8Q7=)eBrPK$P>R6qCyNKkQGUtuo}|GNgc8~Vbi&8Vj+ zSYld`ST21^pcONc;f~DRMKwe z;s$Byq@YxtQiM8+IFrZk=x?o(T1#BkF9_CstH~8Ybui{gPo$HPt!?Si_ zxzsi`Hwr#k1wMZYsJ9|7q-^ZR9Gzlsr& zi2KhkJ1Lcl7B@qo_tYcl6qc*tn1_$|(zw$ zqQmE*!H6JkLqH<(MFEn5Qe0g7XUgb;^SiwP7q&;!sQFqhD!NtE(iWEgS zK6c_r(OY8C_!pzK*41>S6&(i(*KL-==4h|t)Cc(2)FUTFd(dYk>f(=V7K>YKobU8? zge#EH08A@o`R>%8Ebk47q`ce^dfsc-k8o@+G}K;G=Oy0*KK7RMG-D`A;DZ>^_O!cz z!RQkoYHO5xx#=rtftyyH#{+#E+&=WwF30S@i&a4)6z+U@;uY#7s;?5Q5lfB>5vee) ze@D2|T`6fkYHVhDD{lfz+B2M7veE?yu6fw}AXq}Rzu0U+nIWPJ@fcjQZm#g7PXK#Q z)_<&uLZ))EWv-zoW?uym0=lG>&kw^{|8ztm00iz1*=Ql14Od7}#ZuyhN&6L&HfKbZ zhS>;vbDGdz$5^rqe?Nr+TY~5d*|wEoKhcaothhxGYvL9v9%j_3gNH+~LRol!+L>;E z7)!9ihn6^P&$G0D1~3%{3*6R;mWfhSaRB0yco#mamIS;e+A-$*bkMdm0%0X4zm{*l zCuV0cMptQ9p6dl~I2dsKpo5rbDalm(9Wpq+1QLJI+wAA4cMddq!ndf69)j98%N>JBKbXuwcX{OvbChLj6@RZ@$jC>I8EAb$nipjS)Wa^{970t7IiH7%t4XuQx+gG`{;ZIuL&BBe+Mv z@m-clD;I=|0B5*x`t@EB3%{a+uzgQ(uUdDou-jv8rS6LB_0$@R<9z-b@%@53=UH$K zT7i!`#HUozTy(-Yu!8gAVAWoU`rn}xa`2flXe(CIOenTY!DKZ1R&DV9$L>O{7g~)8 zebi^vHcy9w!Q^4sI;NRSic9ZPDBic`XyWfQLgq$G;h5iU{ZE(3-zc$ONqEik%~x4A zx3!IPG}sZ4Cn#s;(CegFZJoDvF8RKtScbNE*4wjI_z1j!X%MU!rE315c%RYQ4RnlU z$2H#74jUk=(K|!n&CO!Nl`8Z_!YyNxkzHGnJMnq=cuF+mfCi@OpX9q@PG@${U`PsGu_GY88K zv2b7YAT!m3g!zUAN698N<7=oZt;KSj(}x(NSGwz_)iHOK`9cExi^JgWvH7Ag4BF7F z>V?=iR2yIRP-MwcV*+wUK&E#qNJzi&SOg7)B$SjeX>I3{ZjSs~gvwU<0&`_Y1BAf4 zHE76DFC$;6gqZ(q+B;Zd*Gunl?j!X0O%F~oJU$3ZEb6lT%yJ*hsVOb|`Fr(#T*|Yq z82{@Wdk!*Hs$~uW6I)2P_y~9Yc&=Q94tGRlQ-by8C|sx55ZH6g^11Dt+il=qO^`>A zI{6FletW-Vm# zq!p|0&&I9Ybf29rUU3ZDL&ayRVCD$Y42{5`zYxaKxj zYayghkyB3AQ(8Xjwi$3IiS*DIuMVi3KVg(M^el^IR!`J*)(nn4Eg!PY0TPK7>$@=! ziy15JL_N-j(H-Th6L82u$UqF=wv)8_LPy0_FqlL&3Klh~b~fH?q(Q2};8CTp`S}lm zmf82+-JEg~X3dD>!6)F&O!D|H6-m&@)A!-UsPfh@Kxl6mNx!Lg5Ua(Oe6W+k;UhF7 z9bbkG5>vggRz2&ofS2#KwSh{%?#Y{h^?Se3cm8S+zS8~W;)Glnd>85pJ)%x;5D3_^ z@yfQ{tAHGi-r>E?eI2gMN#ML~$9kM@W+;+tj##;Y+lw21o&(3rRS2|*ZT_~fv=v7V z4GIsRT|thIKK-$qLI001#(H$iG_pVAV(6@Ma)y_Gb(ZgBKi{Y=VjT!qh3U9_CWYau zBgZ*Vq{>tN@VZyNa%*Lc#a{IB{HpKel+2?FDgGx)oG5{RJ&!nH0y+XQ&jpz_t8Ze= zKx~LAV<2vaLR2c?N4A5v7>@#KG-&cdn+(qEXuwu$Pwwv}QObbkbLoAvihkv|Lg{7yG7 zVo>FUDLebla8_Yy;U(5Jg&~xCJVQ)bBCVjHpd6XagB~3n)BKuFN_8-3=P>L;Uspwm zz{CFwoZ4b;_B`DivoxJF8InhC?@5q>ss9f;>bO1f!)DFODTtN2IcYsO)_#HSB*DQN zG3)9{^>ix1Uzjtn64YoU91R4mbQykAnGBp~wfnvQD8xT=-@(PV+LG5xWc*pltCMZf z_E7y7jzp1X^_q=tZWXEY%*C@G*^}2t|7UGgb@gPZs5>kb0=uW3%VFZ;ndt%&?QmRT zXhD<9U`W4aweG|ZeaDi)VNf{I5U%vc8A#UWVz&I#snbTad-1n=U)P(&)9)gF2l%v+ zwkNL!-4w`+hYa=h(sp4+qEN4oc;%hL7z_RxzUOUt<#dfD-P}=@x=hyT3f~+;@7`;> zqu4JEJ6MxUeAJAV9*hblEzRkTP@XzJLG2=(kH6c8OK-4ouoYKHnOo;QA+SpTjJekC zGAGG?&NQJ6c8@QAN^~^;B;$K0V2?vLpBf2kND3jp{#m`~0SY6|U)@oL1f`AqsvuR7 z8uAxCytwX*&}e?onp0)pKwXiOl@C=|C)eDu*dc|5$+;IxUySyFFk2%>J1WhkLdyO* zIx<{+A^x6`ytV?o_~2EcrY<^Ow{grfE4Ma3@RE!?nJdlg@t{`=7;&xRoj}))Xo4ig zZ$MYQfL-sn{{myjyil`IVKsl#lEBOI^Gn-1n@uE^6IQ&t%bL=OZNcmnWqK1 zV%qn}Cb}(@;LXs_;aQg!drWxoUy$EvDAXP#E@E1K8KYx0@)H|%Y^yWx1f+MR6IX}h z@BZVrZV#%Rf%)IJ+%%g$TbAuD{JSWG#oO+l#*f3ElcoF}i;%!ovP{5KJp=E~1a~@-EEBo$ z1*aOV#_Blg^pAm)`Fb%P8aBKp$e86!S+~uAo&CrYk@znZti+O|G8d0bmxIa|>5BmZfcI>~O-?bMMDC%(ryNj9ncZnpC;!wQX2-^#a{$whH)*KVYAjoMq$H}e> zk5VrXb#&PXkUfMOz36?Oy}+599F1c6XO4j zb@5vDwt4VJQf&$q0Sc&$5>(GvfF*T>9gD5QNDw%9-@%b}Vw+VJ4z89LUokMX$F%eLZ+iDnO z7Yt!P1lC(ThYX}pTKqa`5eo$8Heq0V3_WpB!vKi2l1*qVW!O7uK($rV(FrvT1^mQ? zg}5Z7Dr$PuX!)J*lxAvJ<)FhpdVXy7UQoe-_*broqj-*r3=x#=olCrjmw<+vpL*)* znmXduYwa8uDNlF|DB%)QP<45E1$jnhCVFgS)xe$i>6YQ0Ug9$d?Y?PSOpIOo*Um?p z-(u^~WhhfKZU^WP*hIx75;NE-W*QQiTFtM{%dUT%bWXdo6kBxdKRK^;eRMj^sP3r? zIcUvM{ZF3yIqb{jXLx{KPAn*@siC|EVKt3@FO90P)RzD{Y~)C#t`{1M4Ty?3N6_GD ze*-!KiMk%xk_92%a?8creso?V$aXosa`3dIq$CRHGF-RRC>SAs9b8tDs*@fzy4%Ac zanMRS%7ra4Sl5#d(y?#<0jEFP9+p`eOveUU200Y}{v8U)HocpHLpR0i6)7fJ*)B^F z`))O4&14GXc`?2G)_0D@u^+DQwO#GF>E2db+{c05!T+o;Tzlnw*ZwBD{<4PQXV8X~ z00ebs5#Fyp(l|UCi&>4H2abxd8%Vh-ptSaeX4IKq#QusX@je%tD1kJ$%e zKgGma-Y|Uw5vc6*tE!{WWc;IcjXO93;KE=e5Ys0H8xB(d3}xNwgvp0TP%peim~b0dhEWWC=_9m^r%?2Q^&1Oj4wPn4!}s?<(vs3XME>l&MjH zDl!dy|JET_aY^fXUB=kK&%j59hRN&&DuSzj8kMRI((+NJBui&s#;LFmT7^h-e+sx3 zhi+}nay2piKPz_4HYLHO&h*y9aM5ohc4;!biz%vF|ERD{r_9Ctqcj+nKgx9~kzs## z6t|;4#HR}UAT1CA&Warb(7GQNEaxb{bWZZNLJ~&m#(0=Xx}9?sKdj{?4%y_^QuBsl zKfnnlP4Ic=R#Kzb>E<>hdInz+OL#KnSK=+=IQ;Q<{=>jC^IM9?R($7_(eo1yaxvX0 zGORS};>1@uwYn_kJemKDh42=(;&c`&;dE?AD&$|;(=~BCt_lYBa^F*GFygm_)ANjs zK3TMpKB_DVFf*y8g8~W=hsb1-JG-wArKnL5zaOm$1f`mUs>keVn*nkGleLX3zMz>4 zqK5nIELThZz2XJTRB+fcPMj-Gj(ICRr8^6wOQOopLyQ}>zF%*in6}RM4>wy*S-W z?f)7SPgC+Q${p;2$z*%^sHtO67K$_!!bD%FmLEcWjsqCAlc^D;!@@>>yTec+7$q>2 zeEQGY$}7Gnpe`XhmMR}@t82&XS|$sLOFUYy#D&=Z1kUo*q1m6)(Sv_BG|UYq8UoyV!^-= zO#XLA08g18co;}?E1inFdCrt^ru>J54xIG`6)=&%r{a0m(#28na)-5?Esd6%K}b(q zCTi6qWavePJ@_QCfSrVbnbd!a`^IRo6Ds8%FG3aN!cbnn`h<6E|BIZs9{FcgaH%At zTlS}vId>9e$V4cYp$uM#sw_rg<7ec*YjDq100#JjxxV|I=-^k`ho0)Z0 zqF9C?hARVxa+a&kQW}m6FI3rgmP=OI2gTtRAN3Ze=XxN?^KXq88kFWLv1~=|LU${R zpJ<%hCHfZdAXfHy)CH^(#q5T^S_+cBKa0LoiaF0H1`!8E3R_XX%0z;9v*+)8Qowln zVmj(^%?!P}xE?yWv06o$*2P5=Rjd1-Y2OrupiBKfNFRCcyf=883sbggwjeRPhAs(YfQ#)#H_o*&D^tif7 z!~qR%J3|C93m4|~taRyt>O6WTcyMjyp3iCUO(xF#70W(dPe??8!XL*|-|}j~z%kD} zF5;nDOmtxKv$4h2Z^tc<7L5;DBF&!g&-a0SeILhrkD1(lH!-QAI*2p|U7yJY|4J+$ zT+cmqa3c->`t_Gv2QCDC+ud3@HxyzrQ_7obCp;eEyJqp=z0ZB)N>pN04FlkC)}tqI z(0?-qtw>wkcx$-DUM+cOyjampwBK-hoUhtCK0_;fcZp5}uLvDH?Fz)-y$fqLJi9nf zmnb+uAE`SnwK*^lF3grQ@8}<>eEXc#mK)P4kVdVdHj)yRolS~@0JT$3mY8~%y6B-H z?1LjYP7FlP%#=6q=O4SXr&-6rbaAMYwh%3jPL*dlp-S(ood>UyH7&%MhT7PF+BL{? z;BT$!_(c$1^|KF*^waiJe}75*9Ka!$Ub3b4e{XYWYPWIWVLQ#;2?+>*SkW}1&jkl? z^7d_l9Wp)~>F2!9m6)UaE|(LAVfyc;tz6R(5$1@tzLtojrL8_`iD1AY(W- zn}=QFfbG$_O@uio_5WJ!8jAce(nhd}(u@%wI7?eDGUFSHM}hbUlL@owomz6y=9n1) zK;+$TSgx=_G{)Lf;6K>v@$m5pXj@Wld_`jg(o02wJY^P9Kq4Qi&9%NvFTMNDj1ngS ztfD+W9pc3Sa$zP$S+?ht$u7g6o9;xasyoui2dlr{2P(kOSm552%VDBQ%K#z)j2 z7wz?t)T1QarwqT~`e%$iM_*BKc{x`F8k&hVGGbMUa@AZ+Ixz8`)V3PyOr-ueOAYLZ|rk+MQiHOEN|ZRQ^KAjR1{#%QhAV{!*{EX-2Qd6wR|GK`s>)& z`aELLQ2adECGAcvsUGkGb5x&sr8A|lH}-2w3?Xm2EFHT8w-hWLM;&Izn`YTaNaDfy z_N0q^Q!t|n4abI?;hGAIw_SpBsD7N&V+e9<;f#CW$`~auYWioiAZJ!kFj%70kTQmB zz`iP~y>R)Sem#>Hhl$}Q3PvA0heV`PUbyLoM@!KZYU>_qM82)8!m__ijBdG_i5wnE zvs)W^##s+~H7`Ns%;^A6Q{mauI zX<6O{cOh0C){JVHrr1(#Jme^14Q-%VXkUKs9`Dp4U9V67j8gBlV)LMb*{tJ zC!8vF@~+tHPwVHyFUkTvi9^h3+y|n+<#&9lkGB(9a(^Ten`WUIA%K(Qw;V>a@ZbmGYoEW!1s|-X&kBExD;BBaeTpx&#U~{E zLL-bHZ2*XHDbDXOT=IdslNGGek%gZ_rocFnl{6@CyByw}&JeAZyfj9yNJmvtPvC?~ z0HAzw$uen10>M~Zed)C#+YpIrqTBhOLzX};k|+{NSA62UnSwty2-28EKz%f+)1Tt7 z;S}T*6~=5-ci83a>w|qD?y2(v`tIfgEir)ik_ON9J)^oL0EuK1H9i(L3Zq5>dTy(k zTiGicNKBk+WA^G`Huzxo z^o4aP^%T&mqdI zmr!R*yf4P9uPEfLILx{?R{?~u;EQtuW3+IScG@BW|nIPn-(hLLP%W?`O+*yy#RRi%vgcU%ZQ zvD-Yd5#&|qC_a#c?O4^`S26dLRy%hk@Tr&VGq4Yw9<~bNLAAPfp6ONJo%prpAcOv8 zj2KRRO#812!W68k-2|k%<+faz7992BE{#61@dq5buy!HO5Nyo9r_0p`YWF>})(--$ z6CTnl5AuH%3ZV;+53S2jc6f%QVpir}69aTnAo%JR0l!#(BfmBjXuNAo!@Ba|B0sl9 z=(x>ly&LD%UU`cj4B6_11b^)lbLPwK=PuD#GVAuhp6lzhNe@K81=^KE@mFFlF=nyx zocxG1!QIbk1m2=i_>;S!7UbR>6H&jbYPXf4lW^ZbM~nN%Udy6E*X>DY zMP{R&-lsn- zdOV-*lvRlRkzm|!$ppueJl0kEZyH%!KmHk|vgd4?emu7EO~`AWzxmn6C(Y*=2peQ; z48Q(7g?@V#vps_c%FW--Sbx`;D*z`(e2XWyc`v=TV%DtBO2nqz(uDT=n0Y)$qaM!t*y>g4M6X zvAX7qt#>MZ=zf1MrNQy-(Zs_?koufTa`2p*5IGfxVhYV*yjf^S-X|J=O|Iwg&#L$A zDh<~WB3FACK}&f!-7uK;)??v5@7<|VCgcPP&J5=pXzVYwS5aOcp8dNh*(0ME4_ti7 zFJAF-mQZfWJ6iFQ&@69gUPYzGtDi=^IV~sXBTaTbjR*wVOZT|XUN38KE+lRAI9=(@ z1mi-uY1M3KQq%5EANn+rhW?Xa-iamlkd(WLr-$Z&^A5DPK0p5LZPGOMxTZ25VoMQK^-VK1ZQH}z({eRnj!6BT z>Q@RHU|w@jX&&s2q)6%MR2J4 z(S>C_$KXAO`7DB@xUr^%mppHZwtHwaC8Xf9?E4mF?VTU#2#zg7SP*~C-Ebo3*l-)9 z!|jMCKlP95CrhdI1$KxZYY-|%!aN zf;_#T<9y|#oeL%x`oMSA%v5_>dwO4V`+&r$BATh1dGePQhqv*g}vyj|S}_N$|IS zJtpcazngAG;eihRxm!G~dBE=0pVG#oRMI#gNVCB8atQkEIANkENj7Q@TM@i8pDsygI~t|?|GbCla6cNwQrkqxjVMS z<3f}Zbz`)rcu!#K0{ovW{exKJ`Hgp@Fm5?EBpf;ft!|)Uo`Ukk86VrJ0}>B%r(T@q z2*|O>@?hf|J;!YVn(efEA_FYa4MeKj6L;qPXA#cgjC|jv5CJ=7vv~?eX5QdQ=wVmo zLej%7!H;&2_}+zV`c%?Qo}K*3ABV?Y9cKFATZ~(DM!w0RJOYoRSBM&3rvD`a`0cLC z<3o<|=+1N1UbfbCGrf;n_7#J`+**hCU0P4*W#XO!%rS53pPLXp%(daBAQc)(1;+M`^%IJ**Q zx-$ZkB{4-v2S(Y+PtYznCMlsSv1xUoVss(8bsI?JEFh4mL^#IJ4!4ve$sxgMuUjR4 z7s?=1G3N2n1&a=B%0<`ES`}Q594-DEwrbDnLI8#AVxb%8FddyLnlx8}g=2b)QA#Jf-y$?3-UN z9q)6UnTan&(eo393n0?ME5*6V0Sr=+K&e^SdO;LM*Ro%h52F|hw05|TDisP;8M%UR z^}{3zhG69F2~fkR;RW{U+y5h&12EmWWmS@AzE97@wn$yu>>Gi`_vf~tqO*rz=m)M2 zdJt7c{H1=*+RBOwi>sng|UJ-ct;3ecY!R8Z!hX7t|9jeNsD!+&9{)naTFjE_29 zX>kQOs~y~g><;7M=s(TL6E+L^YsrM(^=xL8R^x<8B;hgUgVO&~k8W6}DXgFOFH~gE zVH^~FRa_A18c?MtXim+ZufQ2C_Oq7utDR}lJ*eP7aNeeW_zb@N)~L0A)|5)z-lQ0X zyeGd)_!z|ar>C6pe;KiWjBR&ujekk~GM1n8P*=mq?1cN`TggK!upP<$VoFUkTH6x2 zf5(~umC&7<3Ei@2JRW6qjOZh+7yhluy~pevOuVAviO7CbfBSg3^Cm0@k8ybRWnS&S zw%Sj#gZ0H$MxwUxPCj+Sn5l3+tDNrF4@@9u!ea7~^*I*vH;ka<#vc2#v9_Xy6+UZd z&y(C|d(`F$*imO_8;%%bz5LggwLiWQrUbnJQSszgFRcF-Hn*@a zxN1-%xncJ9l@VVgr6C$*Ys)leTqz0-es_2xz4srL;P2|{eSppJUFsw=ezf*9GdM0T z?k+Su>d>B!|95%0d4Dv)0~D-pG5Paz5UyUBglW;$(x*O2ED?Ny<+<{&+;+2!YXEGV z^FL6e4Cz00{|!d)=H4Jm?EPJOT2lTI4rGSFWF*b4JEestP7>*>^RKWUyDvhXWRu0TIV)3ZT|No;4&GA>F$D0|ICE&6G>pXWt z7=W0~6ri3!Cli=l3jaA~*8&vcOkZqhpTTbEvn-7jhHY4Oz3RDc36R|L@bXfX6nAv| z0PaZr^QRM-`_?#IN~l`;`U)TrTUuJ0rna_+SA_=4WLX-pG)Pw^^8phXXbsC71ydVR WXa+HwGH@v@jGVNxRF#BD$o~LKi=qtx diff --git a/dpdk/doc/guides/sample_app_ug/index.rst b/dpdk/doc/guides/sample_app_ug/index.rst index 96bb3179..db68ef76 100644 --- a/dpdk/doc/guides/sample_app_ug/index.rst +++ b/dpdk/doc/guides/sample_app_ug/index.rst @@ -28,20 +28,23 @@ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -Sample Applications User Guide -============================== +Sample Applications User Guides +=============================== .. toctree:: :maxdepth: 2 :numbered: intro + compiling cmd_line ethtool exception_path hello_world skeleton rxtx_callbacks + flow_classify + flow_filtering ip_frag ipv4_multicast ip_reassembly @@ -57,26 +60,27 @@ Sample Applications User Guide l3_forward_virtual link_status_intr load_balancer + server_node_efd + service_cores multi_process qos_metering qos_scheduler - intel_quickassist quota_watermark timer packet_ordering vmdq_dcb_forwarding vhost + vhost_scsi netmap_compatibility ip_pipeline test_pipeline + eventdev_pipeline_sw_pmd dist_app vm_power_management tep_termination - proc_info ptpclient performance_thread ipsec_secgw - pdump **Figures** @@ -110,8 +114,6 @@ Sample Applications User Guide :numref:`figure_qos_sched_app_arch` :ref:`figure_qos_sched_app_arch` -:numref:`figure_quickassist_block_diagram` :ref:`figure_quickassist_block_diagram` - :numref:`figure_pipeline_overview` :ref:`figure_pipeline_overview` :numref:`figure_ring_pipeline_perf_setup` :ref:`figure_ring_pipeline_perf_setup` @@ -120,16 +122,6 @@ Sample Applications User Guide :numref:`figure_vmdq_dcb_example` :ref:`figure_vmdq_dcb_example` -:numref:`figure_qemu_virtio_net` :ref:`figure_qemu_virtio_net` - -:numref:`figure_virtio_linux_vhost` :ref:`figure_virtio_linux_vhost` - -:numref:`figure_vhost_net_arch` :ref:`figure_vhost_net_arch` - -:numref:`figure_vhost_net_sample_app` :ref:`figure_vhost_net_sample_app` - -:numref:`figure_tx_dpdk_testpmd` :ref:`figure_tx_dpdk_testpmd` - :numref:`figure_test_pipeline_app` :ref:`figure_test_pipeline_app` :numref:`figure_dist_perf` :ref:`figure_dist_perf` @@ -144,6 +136,8 @@ Sample Applications User Guide :numref:`figure_ptpclient_highlevel` :ref:`figure_ptpclient_highlevel` +:numref:`figure_efd_sample_app_overview` :ref:`figure_efd_sample_app_overview` + **Tables** :numref:`table_qos_metering_1` :ref:`table_qos_metering_1` diff --git a/dpdk/doc/guides/sample_app_ug/intel_quickassist.rst b/dpdk/doc/guides/sample_app_ug/intel_quickassist.rst deleted file mode 100644 index 04d1593d..00000000 --- a/dpdk/doc/guides/sample_app_ug/intel_quickassist.rst +++ /dev/null @@ -1,221 +0,0 @@ -.. BSD LICENSE - Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in - the documentation and/or other materials provided with the - distribution. - * Neither the name of Intel Corporation nor the names of its - contributors may be used to endorse or promote products derived - from this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -Intel® QuickAssist Technology Sample Application -================================================ - -This sample application demonstrates the use of the cryptographic operations provided -by the Intel® QuickAssist Technology from within the DPDK environment. -Therefore, building and running this application requires having both the DPDK and -the QuickAssist Technology Software Library installed, as well as at least one -Intel® QuickAssist Technology hardware device present in the system. - -For this sample application, there is a dependency on either of: - -* Intel® Communications Chipset 8900 to 8920 Series Software for Linux* package - -* Intel® Communications Chipset 8925 to 8955 Series Software for Linux* package - -Overview --------- - -An overview of the application is provided in :numref:`figure_quickassist_block_diagram`. -For simplicity, only two NIC ports and one Intel® QuickAssist Technology device are shown in this diagram, -although the number of NIC ports and Intel® QuickAssist Technology devices can be different. - -.. _figure_quickassist_block_diagram: - -.. figure:: img/quickassist_block_diagram.* - - Intel® QuickAssist Technology Application Block Diagram - - -The application allows the configuration of the following items: - -* Number of NIC ports - -* Number of logical cores (lcores) - -* Mapping of NIC RX queues to logical cores - -Each lcore communicates with every cryptographic acceleration engine in the system through a pair of dedicated input - output queues. -Each lcore has a dedicated NIC TX queue with every NIC port in the system. -Therefore, each lcore reads packets from its NIC RX queues and cryptographic accelerator output queues and -writes packets to its NIC TX queues and cryptographic accelerator input queues. - -Each incoming packet that is read from a NIC RX queue is either directly forwarded to its destination NIC TX port (forwarding path) -or first sent to one of the Intel® QuickAssist Technology devices for either encryption or decryption -before being sent out on its destination NIC TX port (cryptographic path). - -The application supports IPv4 input packets only. -For each input packet, the decision between the forwarding path and -the cryptographic path is taken at the classification stage based on the value of the IP source address field read from the input packet. -Assuming that the IP source address is A.B.C.D, then if: - -* D = 0: the forwarding path is selected (the packet is forwarded out directly) - -* D = 1: the cryptographic path for encryption is selected (the packet is first encrypted and then forwarded out) - -* D = 2: the cryptographic path for decryption is selected (the packet is first decrypted and then forwarded out) - -For the cryptographic path cases (D = 1 or D = 2), byte C specifies the cipher algorithm and -byte B the cryptographic hash algorithm to be used for the current packet. -Byte A is not used and can be any value. -The cipher and cryptographic hash algorithms supported by this application are listed in the crypto.h header file. - -For each input packet, the destination NIC TX port is decided at the forwarding stage (executed after the cryptographic stage, -if enabled for the packet) by looking at the RX port index of the dst_ports[ ] array, -which was initialized at startup, being the outport the adjacent enabled port. -For example, if ports 1,3,5 and 6 are enabled, for input port 1, outport port will be 3 and vice versa, -and for input port 5, output port will be 6 and vice versa. - -For the cryptographic path, it is the payload of the IPv4 packet that is encrypted or decrypted. - -Setup -~~~~~ - -Building and running this application requires having both the DPDK package and -the QuickAssist Technology Software Library installed, -as well as at least one Intel® QuickAssist Technology hardware device present in the system. - -For more details on how to build and run DPDK and Intel® QuickAssist Technology applications, -please refer to the following documents: - -* *DPDK Getting Started Guide* - -* Intel® Communications Chipset 8900 to 8920 Series Software for Linux* Getting Started Guide (440005) - -* Intel® Communications Chipset 8925 to 8955 Series Software for Linux* Getting Started Guide (523128) - -For more details on the actual platforms used to validate this application, as well as performance numbers, -please refer to the Test Report, which is accessible by contacting your Intel representative. - -Building the Application ------------------------- - -Steps to build the application: - -#. Set up the following environment variables: - - .. code-block:: console - - export RTE_SDK= - export ICP_ROOT= - -#. Set the target (a default target is used if not specified). For example: - - .. code-block:: console - - export RTE_TARGET=x86_64-native-linuxapp-gcc - - Refer to the *DPDK Getting Started Guide* for possible RTE_TARGET values. - -#. Build the application: - - .. code-block:: console - - cd ${RTE_SDK}/examples/dpdk_qat - make - -Running the Application ------------------------ - -Intel® QuickAssist Technology Configuration Files -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The Intel® QuickAssist Technology configuration files used by the application are located in the config_files folder in the application folder. -There following sets of configuration files are included in the DPDK package: - -* Stargo CRB (single CPU socket): located in the stargo folder - - * dh89xxcc_qa_dev0.conf - -* Shumway CRB (dual CPU socket): located in the shumway folder - - * dh89xxcc_qa_dev0.conf - - * dh89xxcc_qa_dev1.conf - -* Coleto Creek: located in the coleto folder - - * dh895xcc_qa_dev0.conf - -The relevant configuration file(s) must be copied to the /etc/ directory. - -Please note that any change to these configuration files requires restarting the Intel® -QuickAssist Technology driver using the following command: - -.. code-block:: console - - # service qat_service restart - -Refer to the following documents for information on the Intel® QuickAssist Technology configuration files: - -* Intel® Communications Chipset 8900 to 8920 Series Software Programmer's Guide - -* Intel® Communications Chipset 8925 to 8955 Series Software Programmer's Guide - -* Intel® Communications Chipset 8900 to 8920 Series Software for Linux* Getting Started Guide. - -* Intel® Communications Chipset 8925 to 8955 Series Software for Linux* Getting Started Guide. - -Traffic Generator Setup and Application Startup -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The application has a number of command line options: - - dpdk_qat [EAL options] -- -p PORTMASK [--no-promisc] [--config '(port,queue,lcore)[,(port,queue,lcore)]'] - -where, - -* -p PORTMASK: Hexadecimal bitmask of ports to configure - -* --no-promisc: Disables promiscuous mode for all ports, - so that only packets with the Ethernet MAC destination address set to the Ethernet address of the port are accepted. - By default promiscuous mode is enabled so that packets are accepted regardless of the packet's Ethernet MAC destination address. - -* --config'(port,queue,lcore)[,(port,queue,lcore)]': determines which queues from which ports are mapped to which cores. - -Refer to the :doc:`l3_forward` for more detailed descriptions of the --config command line option. - -As an example, to run the application with two ports and two cores, -which are using different Intel® QuickAssist Technology execution engines, -performing AES-CBC-128 encryption with AES-XCBC-MAC-96 hash, the following settings can be used: - -* Traffic generator source IP address: 0.9.6.1 - -* Command line: - - .. code-block:: console - - ./build/dpdk_qat -c 0xff -n 2 -- -p 0x3 --config '(0,0,1),(1,0,2)' - -Refer to the *DPDK Test Report* for more examples of traffic generator setup and the application startup command lines. -If no errors are generated in response to the startup commands, the application is running correctly. diff --git a/dpdk/doc/guides/sample_app_ug/intro.rst b/dpdk/doc/guides/sample_app_ug/intro.rst index d3f261b3..ae12503b 100644 --- a/dpdk/doc/guides/sample_app_ug/intro.rst +++ b/dpdk/doc/guides/sample_app_ug/intro.rst @@ -1,5 +1,5 @@ .. BSD LICENSE - Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + Copyright(c) 2010-2017 Intel Corporation. All rights reserved. All rights reserved. Redistribution and use in source and binary forms, with or without @@ -28,42 +28,115 @@ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -Introduction -============ +Introduction to the DPDK Sample Applications +============================================ -This document describes the sample applications that are included in the Data Plane Development Kit (DPDK). -Each chapter describes a sample application that showcases specific functionality and -provides instructions on how to compile, run and use the sample application. +The DPDK Sample Applications are small standalone applications which +demonstrate various features of DPDK. They can be considered as a cookbook of +DPDK features. Users interested in getting started with DPDK can take the +applications, try out the features, and then extend them to fit their needs. -Documentation Roadmap ---------------------- -The following is a list of DPDK documents in suggested reading order: +The DPDK Sample Applications +---------------------------- -* **Release Notes** : Provides release-specific information, including supported features, - limitations, fixed issues, known issues and so on. - Also, provides the answers to frequently asked questions in FAQ format. +Table :numref:`table_sample_apps` shows a list of some of the main sample +applications that are available in the examples directory of DPDK: -* **Getting Started Guides** : Describes how to install and - configure the DPDK software for your operating system; - designed to get users up and running quickly with the software. + .. _table_sample_apps: -* **Programmer's Guide:** Describes: + .. table:: **Some of the DPDK Sample applications** - * The software architecture and how to use it (through examples), - specifically in a Linux* application (linuxapp) environment. + +---------------------------------------+--------------------------------------+ + | Bonding | Netmap Compatibility | + +---------------------------------------+--------------------------------------+ + | Command Line | Packet Ordering | + +---------------------------------------+--------------------------------------+ + | Distributor | Performance Thread | + +---------------------------------------+--------------------------------------+ + | Ethtool | Precision Time Protocol (PTP) Client | + +---------------------------------------+--------------------------------------+ + | Exception Path | Quality of Service (QoS) Metering | + +---------------------------------------+--------------------------------------+ + | Hello World | QoS Scheduler | + +---------------------------------------+--------------------------------------+ + | Internet Protocol (IP) Fragmentation | Quota and Watermark | + +---------------------------------------+--------------------------------------+ + | IP Pipeline | RX/TX Callbacks | + +---------------------------------------+--------------------------------------+ + | IP Reassembly | Server node EFD | + +---------------------------------------+--------------------------------------+ + | IPsec Security Gateway | Basic Forwarding/Skeleton App | + +---------------------------------------+--------------------------------------+ + | IPv4 multicast | Tunnel End Point (TEP) termination | + +---------------------------------------+--------------------------------------+ + | Kernel NIC Interface | Timer | + +---------------------------------------+--------------------------------------+ + | Network Layer 2 Forwarding + variants | Vhost | + +---------------------------------------+--------------------------------------+ + | Network Layer 3 Forwarding + variants | Vhost Xen | + +---------------------------------------+--------------------------------------+ + | Link Status Interrupt | VMDQ Forwarding | + +---------------------------------------+--------------------------------------+ + | Load Balancer | VMDQ and DCB Forwarding | + +---------------------------------------+--------------------------------------+ + | Multi-process | VM Power Management | + +---------------------------------------+--------------------------------------+ - * The content of the DPDK, the build system - (including the commands that can be used in the root DPDK Makefile to build the development kit and an application) - and guidelines for porting an application. +These examples range from simple to reasonably complex but most are designed +to demonstrate one particular feature of DPDK. Some of the more interesting +examples are highlighted below. - * Optimizations used in the software and those that should be considered for new development -A glossary of terms is also provided. +* :doc:`Hello World`: As with most introductions to a + programming framework a good place to start is with the Hello World + application. The Hello World example sets up the DPDK Environment Abstraction + Layer (EAL), and prints a simple "Hello World" message to each of the DPDK + enabled cores. This application doesn't do any packet forwarding but it is a + good way to test if the DPDK environment is compiled and set up properly. -* **API Reference** : Provides detailed information about DPDK functions, - data structures and other programming constructs. +* :doc:`Basic Forwarding/Skeleton Application`: The Basic + Forwarding/Skeleton contains the minimum amount of code required to enable + basic packet forwarding with DPDK. This allows you to test if your network + interfaces are working with DPDK. -* **Sample Applications User Guide** : Describes a set of sample applications. - Each chapter describes a sample application that showcases specific functionality and - provides instructions on how to compile, run and use the sample application. +* :doc:`Network Layer 2 forwarding`: The Network Layer 2 + forwarding, or ``l2fwd`` application does forwarding based on Ethernet MAC + addresses like a simple switch. + +* :doc:`Network Layer 3 forwarding`: The Network Layer3 + forwarding, or ``l3fwd`` application does forwarding based on Internet + Protocol, IPv4 or IPv6 like a simple router. + +* :doc:`Packet Distributor`: The Packet Distributor + demonstrates how to distribute packets arriving on an Rx port to different + cores for processing and transmission. + +* :doc:`Multi-Process Application`: The + multi-process application shows how two DPDK processes can work together using + queues and memory pools to share information. + +* :doc:`RX/TX callbacks Application`: The RX/TX + callbacks sample application is a packet forwarding application that + demonstrates the use of user defined callbacks on received and transmitted + packets. The application calculates the latency of a packet between RX + (packet arrival) and TX (packet transmission) by adding callbacks to the RX + and TX packet processing functions. + +* :doc:`IPSec Security Gateway`: The IPSec Security + Gateway application is minimal example of something closer to a real world + example. This is also a good example of an application using the DPDK + Cryptodev framework. + +* :doc:`Precision Time Protocol (PTP) client`: The PTP + client is another minimal implementation of a real world application. + In this case the application is a PTP client that communicates with a PTP + master clock to synchronize time on a Network Interface Card (NIC) using the + IEEE1588 protocol. + +* :doc:`Quality of Service (QoS) Scheduler`: The QoS + Scheduler application demonstrates the use of DPDK to provide QoS scheduling. + +There are many more examples shown in the following chapters. Each of the +documented sample applications show how to compile, configure and run the +application as well as explaining the main functionality of the code. diff --git a/dpdk/doc/guides/sample_app_ug/ip_frag.rst b/dpdk/doc/guides/sample_app_ug/ip_frag.rst index 0c8da194..f45adfe7 100644 --- a/dpdk/doc/guides/sample_app_ug/ip_frag.rst +++ b/dpdk/doc/guides/sample_app_ug/ip_frag.rst @@ -59,31 +59,12 @@ Any unmatched packets are forwarded to the originating port. By default, input frame sizes up to 9.5 KB are supported. Before forwarding, the input IP packet is fragmented to fit into the "standard" Ethernet* v2 MTU (1500 bytes). -Building the Application ------------------------- +Compiling the Application +------------------------- -To build the application: +To compile the sample application see :doc:`compiling`. -#. Go to the sample application directory: - - .. code-block:: console - - export RTE_SDK=/path/to/rte_sdk - cd ${RTE_SDK}/examples/ip_fragmentation - -#. Set the target (a default target is used if not specified). For example: - - .. code-block:: console - - export RTE_TARGET=x86_64-native-linuxapp-gcc - -See the *DPDK Getting Started Guide* for possible RTE_TARGET values. - -#. Build the application: - - .. code-block:: console - - make +The application is located in the ``ip_fragmentation`` sub-directory. Running the Application ----------------------- @@ -111,7 +92,7 @@ To run the example in linuxapp environment with 2 lcores (2,4) over 2 ports(0,2) .. code-block:: console - ./build/ip_fragmentation -c 0x14 -n 3 -- -p 5 + ./build/ip_fragmentation -l 2,4 -n 3 -- -p 5 EAL: coremask set to 14 EAL: Detected lcore 0 on socket 0 EAL: Detected lcore 1 on socket 1 @@ -140,7 +121,7 @@ To run the example in linuxapp environment with 1 lcore (4) over 2 ports(0,2) wi .. code-block:: console - ./build/ip_fragmentation -c 0x10 -n 3 -- -p 5 -q 2 + ./build/ip_fragmentation -l 4 -n 3 -- -p 5 -q 2 To test the application, flows should be set up in the flow generator that match the values in the l3fwd_ipv4_route_array and/or l3fwd_ipv6_route_array table. diff --git a/dpdk/doc/guides/sample_app_ug/ip_pipeline.rst b/dpdk/doc/guides/sample_app_ug/ip_pipeline.rst index 693d813c..e0aa1484 100644 --- a/dpdk/doc/guides/sample_app_ug/ip_pipeline.rst +++ b/dpdk/doc/guides/sample_app_ug/ip_pipeline.rst @@ -233,7 +233,7 @@ The application startup arguments are: * Default: Not present * Argument: Path to the CLI script file to be run by the master pipeline at application startup. - No CLI script file will be run at startup of this argument is not present. + No CLI script file will be run at startup if this argument is not present. ``-p PORT_MASK`` diff --git a/dpdk/doc/guides/sample_app_ug/ip_reassembly.rst b/dpdk/doc/guides/sample_app_ug/ip_reassembly.rst index 3c5cc708..7dc80d0b 100644 --- a/dpdk/doc/guides/sample_app_ug/ip_reassembly.rst +++ b/dpdk/doc/guides/sample_app_ug/ip_reassembly.rst @@ -53,28 +53,14 @@ There are two key differences from the L2 Forwarding sample application: The Longest Prefix Match (LPM for IPv4, LPM6 for IPv6) table is used to store/lookup an outgoing port number, associated with that IPv4 address. Any unmatched packets are forwarded to the originating port.Compiling the Application -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -To compile the application: -#. Go to the sample application directory: +Compiling the Application +------------------------- - .. code-block:: console +To compile the sample application see :doc:`compiling`. - export RTE_SDK=/path/to/rte_sdk - cd ${RTE_SDK}/examples/ip_reassembly +The application is located in the ``ip_reassembly`` sub-directory. -#. Set the target (a default target is used if not specified). For example: - - .. code-block:: console - - export RTE_TARGET=x86_64-native-linuxapp-gcc - -See the *DPDK Getting Started Guide* for possible RTE_TARGET values. - -#. Build the application: - - .. code-block:: console - - make Running the Application ----------------------- @@ -102,7 +88,7 @@ To run the example in linuxapp environment with 2 lcores (2,4) over 2 ports(0,2) .. code-block:: console - ./build/ip_reassembly -c 0x14 -n 3 -- -p 5 + ./build/ip_reassembly -l 2,4 -n 3 -- -p 5 EAL: coremask set to 14 EAL: Detected lcore 0 on socket 0 EAL: Detected lcore 1 on socket 1 @@ -133,7 +119,7 @@ To run the example in linuxapp environment with 1 lcore (4) over 2 ports(0,2) wi .. code-block:: console - ./build/ip_reassembly -c 0x10 -n 3 -- -p 5 -q 2 + ./build/ip_reassembly -l 4 -n 3 -- -p 5 -q 2 To test the application, flows should be set up in the flow generator that match the values in the l3fwd_ipv4_route_array and/or l3fwd_ipv6_route_array table. diff --git a/dpdk/doc/guides/sample_app_ug/ipsec_secgw.rst b/dpdk/doc/guides/sample_app_ug/ipsec_secgw.rst index fcb33c26..ae18acdd 100644 --- a/dpdk/doc/guides/sample_app_ug/ipsec_secgw.rst +++ b/dpdk/doc/guides/sample_app_ug/ipsec_secgw.rst @@ -1,5 +1,5 @@ .. BSD LICENSE - Copyright(c) 2016 Intel Corporation. All rights reserved. + Copyright(c) 2016-2017 Intel Corporation. All rights reserved. All rights reserved. Redistribution and use in source and binary forms, with or without @@ -52,13 +52,28 @@ The application classifies the ports as *Protected* and *Unprotected*. Thus, traffic received on an Unprotected or Protected port is consider Inbound or Outbound respectively. +The application also supports complete IPSec protocol offload to hardware +(Look aside crypto accelarator or using ethernet device). It also support +inline ipsec processing by the supported ethernet device during transmission. +These modes can be selected during the SA creation configuration. + +In case of complete protocol offload, the processing of headers(ESP and outer +IP header) is done by the hardware and the application does not need to +add/remove them during outbound/inbound processing. + +For inline offloaded outbound traffic, the application will not do the LPM +lookup for routing, as the port on which the packet has to be forwarded will be +part of the SA. Security parameters will be configured on that port only, and +sending the packet on other ports could result in unencrypted packets being +sent out. + The Path for IPsec Inbound traffic is: * Read packets from the port. * Classify packets between IPv4 and ESP. * Perform Inbound SA lookup for ESP packets based on their SPI. -* Perform Verification/Decryption. -* Remove ESP and outer IP header +* Perform Verification/Decryption (Not needed in case of inline ipsec). +* Remove ESP and outer IP header (Not needed in case of protocol offload). * Inbound SP check using ACL of decrypted packets and any other IPv4 packets. * Routing. * Write packet to port. @@ -68,8 +83,8 @@ The Path for the IPsec Outbound traffic is: * Read packets from the port. * Perform Outbound SP check using ACL of all IPv4 traffic. * Perform Outbound SA lookup for packets that need IPsec protection. -* Add ESP and outer IP header. -* Perform Encryption/Digest. +* Add ESP and outer IP header (Not needed in case protocol offload). +* Perform Encryption/Digest (Not needed in case of inline ipsec). * Routing. * Write packet to port. @@ -79,31 +94,16 @@ Constraints * No IPv6 options headers. * No AH mode. -* Currently only EAS-CBC, HMAC-SHA1 and NULL. +* Supported algorithms: AES-CBC, AES-CTR, AES-GCM, HMAC-SHA1 and NULL. * Each SA must be handle by a unique lcore (*1 RX queue per port*). * No chained mbufs. - Compiling the Application ------------------------- -To compile the application: +To compile the sample application see :doc:`compiling`. -#. Go to the sample application directory:: - - export RTE_SDK=/path/to/rte_sdk - cd ${RTE_SDK}/examples/ipsec-secgw - -#. Set the target (a default target is used if not specified). For example:: - - - export RTE_TARGET=x86_64-native-linuxapp-gcc - - See the *DPDK Getting Started Guide* for possible RTE_TARGET values. - -#. Build the application:: - - make +The application is located in the ``rpsec-secgw`` sub-directory. #. [Optional] Build the application for debugging: This option adds some extra flags, disables compiler optimizations and @@ -119,10 +119,10 @@ The application has a number of command line options:: ./build/ipsec-secgw [EAL options] -- - -p PORTMASK -P -u PORTMASK + -p PORTMASK -P -u PORTMASK -j FRAMESIZE --config (port,queue,lcore)[,(port,queue,lcore] --single-sa SAIDX - --ep0|--ep1 + -f CONFIG_FILE_PATH Where: @@ -135,6 +135,10 @@ Where: * ``-u PORTMASK``: hexadecimal bitmask of unprotected ports +* ``-j FRAMESIZE``: *optional*. Enables jumbo frames with the maximum size + specified as FRAMESIZE. If an invalid value is provided as FRAMESIZE + then the default value 9000 is used. + * ``--config (port,queue,lcore)[,(port,queue,lcore)]``: determines which queues from which ports are mapped to which cores. @@ -142,22 +146,20 @@ Where: on both Inbound and Outbound. This option is meant for debugging/performance purposes. -* ``--ep0``: configure the app as Endpoint 0. +* ``-f CONFIG_FILE_PATH``: the full path of text-based file containing all + configuration items for running the application (See Configuration file + syntax section below). ``-f CONFIG_FILE_PATH`` **must** be specified. + **ONLY** the UNIX format configuration file is accepted. -* ``--ep1``: configure the app as Endpoint 1. - -Either one of ``--ep0`` or ``--ep1`` **must** be specified. -The main purpose of these options is to easily configure two systems -back-to-back that would forward traffic through an IPsec tunnel (see -:ref:`figure_ipsec_endpoints`). The mapping of lcores to port/queues is similar to other l3fwd applications. For example, given the following command line:: ./build/ipsec-secgw -l 20,21 -n 4 --socket-mem 0,2048 \ - --vdev "cryptodev_null_pmd" -- -p 0xf -P -u 0x3 \ - --config="(0,0,20),(1,0,20),(2,0,21),(3,0,21)" --ep0 \ + --vdev "crypto_null" -- -p 0xf -P -u 0x3 \ + --config="(0,0,20),(1,0,20),(2,0,21),(3,0,21)" \ + -f /path/to/config_file \ where each options means: @@ -167,7 +169,7 @@ where each options means: * The ``--socket-mem`` to use 2GB on socket 1. -* The ``--vdev "cryptodev_null_pmd"`` option creates virtual NULL cryptodev PMD. +* The ``--vdev "crypto_null"`` option creates virtual NULL cryptodev PMD. * The ``-p`` option enables ports (detected) 0, 1, 2 and 3. @@ -194,8 +196,12 @@ where each options means: | | | | | +----------+-----------+-----------+---------------------------------------+ -* The ``--ep0`` options configures the app with a given set of SP, SA and Routing - entries as explained below in more detail. +* The ``-f /path/to/config_file`` option enables the application read and + parse the configuration file specified, and configures the application + with a given set of SP, SA and Routing entries accordingly. The syntax of + the configuration file will be explained below in more detail. Please + **note** the parser only accepts UNIX format text file. Other formats + such as DOS/MAC format will cause a parse error. Refer to the *DPDK Getting Started Guide* for general information on running applications and the Environment Abstraction Layer (EAL) options. @@ -216,499 +222,425 @@ For example, something like the following command line: ./build/ipsec-secgw -l 20,21 -n 4 --socket-mem 0,2048 \ -w 81:00.0 -w 81:00.1 -w 81:00.2 -w 81:00.3 \ - --vdev "cryptodev_aesni_mb_pmd" --vdev "cryptodev_null_pmd" \ + --vdev "crypto_aesni_mb" --vdev "crypto_null" \ -- \ -p 0xf -P -u 0x3 --config="(0,0,20),(1,0,20),(2,0,21),(3,0,21)" \ - --ep0 + -f sample.cfg Configurations -------------- -The following sections provide some details on the default values used to -initialize the SP, SA and Routing tables. -Currently all configuration information is hard coded into the application. - -The following image illustrate a few of the concepts regarding IPSec, such -as protected/unprotected and inbound/outbound traffic, from the point of -view of two back-to-back endpoints: - -.. _figure_ipsec_endpoints: - -.. figure:: img/ipsec_endpoints.* - - IPSec Inbound/Outbound traffic - -Note that the above image only displays unidirectional traffic per port -for illustration purposes. -The application supports bidirectional traffic on all ports, +The following sections provide the syntax of configurations to initialize +your SP, SA and Routing tables. +Configurations shall be specified in the configuration file to be passed to +the application. The file is then parsed by the application. The successful +parsing will result in the appropriate rules being applied to the tables +accordingly. -Security Policy Initialization +Configuration File Syntax ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ As mention in the overview, the Security Policies are ACL rules. -The application defines two ACLs, one each of Inbound and Outbound, and -it replicates them per socket in use. +The application parsers the rules specified in the configuration file and +passes them to the ACL table, and replicates them per socket in use. -Following are the default rules which show only the relevant information, -assuming ANY value is valid for the fields not mentioned (src ip, proto, -src/dst ports). +Following are the configuration file syntax. -.. _table_ipsec_endpoint_outbound_sp: +General rule syntax +^^^^^^^^^^^^^^^^^^^ -.. table:: Endpoint 0 Outbound Security Policies +The parse treats one line in the configuration file as one configuration +item (unless the line concatenation symbol exists). Every configuration +item shall follow the syntax of either SP, SA, or Routing rules specified +below. - +-----------------------------------+------------+ - | **Dst** | **SA idx** | - | | | - +-----------------------------------+------------+ - | 192.168.105.0/24 | 5 | - | | | - +-----------------------------------+------------+ - | 192.168.106.0/24 | 6 | - | | | - +-----------------------------------+------------+ - | 192.168.175.0/24 | 10 | - | | | - +-----------------------------------+------------+ - | 192.168.176.0/24 | 11 | - | | | - +-----------------------------------+------------+ - | 192.168.200.0/24 | 15 | - | | | - +-----------------------------------+------------+ - | 192.168.201.0/24 | 16 | - | | | - +-----------------------------------+------------+ - | 192.168.55.0/24 | 25 | - | | | - +-----------------------------------+------------+ - | 192.168.56.0/24 | 26 | - | | | - +-----------------------------------+------------+ - | 192.168.240.0/24 | BYPASS | - | | | - +-----------------------------------+------------+ - | 192.168.241.0/24 | BYPASS | - | | | - +-----------------------------------+------------+ - | 0:0:0:0:5555:5555:0:0/96 | 5 | - | | | - +-----------------------------------+------------+ - | 0:0:0:0:6666:6666:0:0/96 | 6 | - | | | - +-----------------------------------+------------+ - | 0:0:1111:1111:0:0:0:0/96 | 10 | - | | | - +-----------------------------------+------------+ - | 0:0:1111:1111:1111:1111:0:0/96 | 11 | - | | | - +-----------------------------------+------------+ - | 0:0:0:0:aaaa:aaaa:0:0/96 | 25 | - | | | - +-----------------------------------+------------+ - | 0:0:0:0:bbbb:bbbb:0:0/96 | 26 | - | | | - +-----------------------------------+------------+ +The configuration parser supports the following special symbols: -.. _table_ipsec_endpoint_inbound_sp: + * Comment symbol **#**. Any character from this symbol to the end of + line is treated as comment and will not be parsed. -.. table:: Endpoint 0 Inbound Security Policies - - +-----------------------------------+------------+ - | **Dst** | **SA idx** | - | | | - +-----------------------------------+------------+ - | 192.168.115.0/24 | 105 | - | | | - +-----------------------------------+------------+ - | 192.168.116.0/24 | 106 | - | | | - +-----------------------------------+------------+ - | 192.168.185.0/24 | 110 | - | | | - +-----------------------------------+------------+ - | 192.168.186.0/24 | 111 | - | | | - +-----------------------------------+------------+ - | 192.168.210.0/24 | 115 | - | | | - +-----------------------------------+------------+ - | 192.168.211.0/24 | 116 | - | | | - +-----------------------------------+------------+ - | 192.168.65.0/24 | 125 | - | | | - +-----------------------------------+------------+ - | 192.168.66.0/24 | 126 | - | | | - +-----------------------------------+------------+ - | 192.168.245.0/24 | BYPASS | - | | | - +-----------------------------------+------------+ - | 192.168.246.0/24 | BYPASS | - | | | - +-----------------------------------+------------+ - | ffff:0:0:0:5555:5555:0:0/96 | 105 | - | | | - +-----------------------------------+------------+ - | ffff:0:0:0:6666:6666:0:0/96 | 106 | - | | | - +-----------------------------------+------------+ - | ffff:0:1111:1111:0:0:0:0/96 | 110 | - | | | - +-----------------------------------+------------+ - | ffff:0:1111:1111:1111:1111:0:0/96 | 111 | - | | | - +-----------------------------------+------------+ - | ffff:0:0:0:aaaa:aaaa:0:0/96 | 125 | - | | | - +-----------------------------------+------------+ - | ffff:0:0:0:bbbb:bbbb:0:0/96 | 126 | - | | | - +-----------------------------------+------------+ - -For Endpoint 1, we use the same policies in reverse, meaning the Inbound SP -entries are set as Outbound and vice versa. + * Line concatenation symbol **\\**. This symbol shall be placed in the end + of the line to be concatenated to the line below. Multiple lines' + concatenation is supported. -Security Association Initialization -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +SP rule syntax +^^^^^^^^^^^^^^ -The SAs are kept in a array table. +The SP rule syntax is shown as follows: -For Inbound, the SPI is used as index modulo the table size. -This means that on a table for 100 SA, SPI 5 and 105 would use the same index -and that is not currently supported. +.. code-block:: console -Notice that it is not an issue for Outbound traffic as we store the index and -not the SPI in the Security Policy. - -All SAs configured with AES-CBC and HMAC-SHA1 share the same values for cipher -block size and key, and authentication digest size and key. - -The following are the default values: - -.. _table_ipsec_endpoint_outbound_sa: - -.. table:: Endpoint 0 Outbound Security Associations - - +---------+----------+------------+-----------+----------------+----------------+ - | **SPI** | **Mode** | **Cipher** | **Auth** | **Tunnel src** | **Tunnel dst** | - | | | | | | | - +---------+----------+------------+-----------+----------------+----------------+ - | 5 | Tunnel | AES-CBC | HMAC-SHA1 | 172.16.1.5 | 172.16.2.5 | - | | | | | | | - +---------+----------+------------+-----------+----------------+----------------+ - | 6 | Tunnel | AES-CBC | HMAC-SHA1 | 172.16.1.6 | 172.16.2.6 | - | | | | | | | - +---------+----------+------------+-----------+----------------+----------------+ - | 10 | Trans | AES-CBC | HMAC-SHA1 | N/A | N/A | - | | | | | | | - +---------+----------+------------+-----------+----------------+----------------+ - | 11 | Trans | AES-CBC | HMAC-SHA1 | N/A | N/A | - | | | | | | | - +---------+----------+------------+-----------+----------------+----------------+ - | 15 | Tunnel | NULL | NULL | 172.16.1.5 | 172.16.2.5 | - | | | | | | | - +---------+----------+------------+-----------+----------------+----------------+ - | 16 | Tunnel | NULL | NULL | 172.16.1.6 | 172.16.2.6 | - | | | | | | | - +---------+----------+------------+-----------+----------------+----------------+ - | 25 | Tunnel | AES-CBC | HMAC-SHA1 | 1111:1111: | 2222:2222: | - | | | | | 1111:1111: | 2222:2222: | - | | | | | 1111:1111: | 2222:2222: | - | | | | | 1111:5555 | 2222:5555 | - | | | | | | | - +---------+----------+------------+-----------+----------------+----------------+ - | 26 | Tunnel | AES-CBC | HMAC-SHA1 | 1111:1111: | 2222:2222: | - | | | | | 1111:1111: | 2222:2222: | - | | | | | 1111:1111: | 2222:2222: | - | | | | | 1111:6666 | 2222:6666 | - | | | | | | | - +---------+----------+------------+-----------+----------------+----------------+ - -.. _table_ipsec_endpoint_inbound_sa: - -.. table:: Endpoint 0 Inbound Security Associations - - +---------+----------+------------+-----------+----------------+----------------+ - | **SPI** | **Mode** | **Cipher** | **Auth** | **Tunnel src** | **Tunnel dst** | - | | | | | | | - +---------+----------+------------+-----------+----------------+----------------+ - | 105 | Tunnel | AES-CBC | HMAC-SHA1 | 172.16.2.5 | 172.16.1.5 | - | | | | | | | - +---------+----------+------------+-----------+----------------+----------------+ - | 106 | Tunnel | AES-CBC | HMAC-SHA1 | 172.16.2.6 | 172.16.1.6 | - | | | | | | | - +---------+----------+------------+-----------+----------------+----------------+ - | 110 | Trans | AES-CBC | HMAC-SHA1 | N/A | N/A | - | | | | | | | - +---------+----------+------------+-----------+----------------+----------------+ - | 111 | Trans | AES-CBC | HMAC-SHA1 | N/A | N/A | - | | | | | | | - +---------+----------+------------+-----------+----------------+----------------+ - | 115 | Tunnel | NULL | NULL | 172.16.2.5 | 172.16.1.5 | - | | | | | | | - +---------+----------+------------+-----------+----------------+----------------+ - | 116 | Tunnel | NULL | NULL | 172.16.2.6 | 172.16.1.6 | - | | | | | | | - +---------+----------+------------+-----------+----------------+----------------+ - | 125 | Tunnel | AES-CBC | HMAC-SHA1 | 2222:2222: | 1111:1111: | - | | | | | 2222:2222: | 1111:1111: | - | | | | | 2222:2222: | 1111:1111: | - | | | | | 2222:5555 | 1111:5555 | - | | | | | | | - +---------+----------+------------+-----------+----------------+----------------+ - | 126 | Tunnel | AES-CBC | HMAC-SHA1 | 2222:2222: | 1111:1111: | - | | | | | 2222:2222: | 1111:1111: | - | | | | | 2222:2222: | 1111:1111: | - | | | | | 2222:6666 | 1111:6666 | - | | | | | | | - +---------+----------+------------+-----------+----------------+----------------+ - -For Endpoint 1, we use the same policies in reverse, meaning the Inbound SP -entries are set as Outbound and vice versa. + sp

esp + -Routing Initialization -~~~~~~~~~~~~~~~~~~~~~~ +where each options means: -The Routing is implemented using an LPM table. +```` -Following default values: + * IP protocol version -.. _table_ipsec_endpoint_outbound_routing: + * Optional: No -.. table:: Endpoint 0 Routing Table + * Available options: - +------------------+----------+ - | **Dst addr** | **Port** | - | | | - +------------------+----------+ - | 172.16.2.5/32 | 0 | - | | | - +------------------+----------+ - | 172.16.2.6/32 | 1 | - | | | - +------------------+----------+ - | 192.168.175.0/24 | 0 | - | | | - +------------------+----------+ - | 192.168.176.0/24 | 1 | - | | | - +------------------+----------+ - | 192.168.240.0/24 | 0 | - | | | - +------------------+----------+ - | 192.168.241.0/24 | 1 | - | | | - +------------------+----------+ - | 192.168.115.0/24 | 2 | - | | | - +------------------+----------+ - | 192.168.116.0/24 | 3 | - | | | - +------------------+----------+ - | 192.168.65.0/24 | 2 | - | | | - +------------------+----------+ - | 192.168.66.0/24 | 3 | - | | | - +------------------+----------+ - | 192.168.185.0/24 | 2 | - | | | - +------------------+----------+ - | 192.168.186.0/24 | 3 | - | | | - +------------------+----------+ - | 192.168.210.0/24 | 2 | - | | | - +------------------+----------+ - | 192.168.211.0/24 | 3 | - | | | - +------------------+----------+ - | 192.168.245.0/24 | 2 | - | | | - +------------------+----------+ - | 192.168.246.0/24 | 3 | - | | | - +------------------+----------+ - | 2222:2222: | 0 | - | 2222:2222: | | - | 2222:2222: | | - | 2222:5555/116 | | - | | | - +------------------+----------+ - | 2222:2222: | 1 | - | 2222:2222: | | - | 2222:2222: | | - | 2222:6666/116 | | - | | | - +------------------+----------+ - | 0000:0000: | 0 | - | 1111:1111: | | - | 0000:0000: | | - | 0000:0000/116 | | - | | | - +------------------+----------+ - | 0000:0000: | 1 | - | 1111:1111: | | - | 1111:1111: | | - | 0000:0000/116 | | - | | | - +------------------+----------+ - | ffff:0000: | 2 | - | 0000:0000: | | - | aaaa:aaaa: | | - | 0000:0/116 | | - | | | - +------------------+----------+ - | ffff:0000: | 3 | - | 0000:0000: | | - | bbbb:bbbb: | | - | 0000:0/116 | | - | | | - +------------------+----------+ - | ffff:0000: | 2 | - | 0000:0000: | | - | 5555:5555: | | - | 0000:0/116 | | - | | | - +------------------+----------+ - | ffff:0000: | 3 | - | 0000:0000: | | - | 6666:6666: | | - | 0000:0/116 | | - | | | - +------------------+----------+ - | ffff:0000: | 2 | - | 1111:1111: | | - | 0000:0000: | | - | 0000:0000/116 | | - | | | - +------------------+----------+ - | ffff:0000: | 3 | - | 1111:1111: | | - | 1111:1111: | | - | 0000:0000/116 | | - | | | - +------------------+----------+ + * *ipv4*: IP protocol version 4 + * *ipv6*: IP protocol version 6 -.. _table_ipsec_endpoint_inbound_routing: +```` -.. table:: Endpoint 1 Routing Table + * The traffic direction - +------------------+----------+ - | **Dst addr** | **Port** | - | | | - +------------------+----------+ - | 172.16.1.5/32 | 0 | - | | | - +------------------+----------+ - | 172.16.1.6/32 | 1 | - | | | - +------------------+----------+ - | 192.168.185.0/24 | 0 | - | | | - +------------------+----------+ - | 192.168.186.0/24 | 1 | - | | | - +------------------+----------+ - | 192.168.245.0/24 | 0 | - | | | - +------------------+----------+ - | 192.168.246.0/24 | 1 | - | | | - +------------------+----------+ - | 192.168.105.0/24 | 2 | - | | | - +------------------+----------+ - | 192.168.106.0/24 | 3 | - | | | - +------------------+----------+ - | 192.168.55.0/24 | 2 | - | | | - +------------------+----------+ - | 192.168.56.0/24 | 3 | - | | | - +------------------+----------+ - | 192.168.175.0/24 | 2 | - | | | - +------------------+----------+ - | 192.168.176.0/24 | 3 | - | | | - +------------------+----------+ - | 192.168.200.0/24 | 2 | - | | | - +------------------+----------+ - | 192.168.201.0/24 | 3 | - | | | - +------------------+----------+ - | 192.168.240.0/24 | 2 | - | | | - +------------------+----------+ - | 192.168.241.0/24 | 3 | - | | | - +------------------+----------+ - | 1111:1111: | 0 | - | 1111:1111: | | - | 1111:1111: | | - | 1111:5555/116 | | - | | | - +------------------+----------+ - | 1111:1111: | 1 | - | 1111:1111: | | - | 1111:1111: | | - | 1111:6666/116 | | - | | | - +------------------+----------+ - | ffff:0000: | 0 | - | 1111:1111: | | - | 0000:0000: | | - | 0000:0000/116 | | - | | | - +------------------+----------+ - | ffff:0000: | 1 | - | 1111:1111: | | - | 1111:1111: | | - | 0000:0000/116 | | - | | | - +------------------+----------+ - | 0000:0000: | 2 | - | 0000:0000: | | - | aaaa:aaaa: | | - | 0000:0/116 | | - | | | - +------------------+----------+ - | 0000:0000: | 3 | - | 0000:0000: | | - | bbbb:bbbb: | | - | 0000:0/116 | | - | | | - +------------------+----------+ - | 0000:0000: | 2 | - | 0000:0000: | | - | 5555:5555: | | - | 0000:0/116 | | - | | | - +------------------+----------+ - | 0000:0000: | 3 | - | 0000:0000: | | - | 6666:6666: | | - | 0000:0/116 | | - | | | - +------------------+----------+ - | 0000:0000: | 2 | - | 1111:1111: | | - | 0000:0000: | | - | 0000:0000/116 | | - | | | - +------------------+----------+ - | 0000:0000: | 3 | - | 1111:1111: | | - | 1111:1111: | | - | 0000:0000/116 | | - | | | - +------------------+----------+ + * Optional: No + + * Available options: + + * *in*: inbound traffic + * *out*: outbound traffic + +```` + + * IPsec action + + * Optional: No + + * Available options: + + * *protect *: the specified traffic is protected by SA rule + with id SA_idx + * *bypass*: the specified traffic traffic is bypassed + * *discard*: the specified traffic is discarded + +```` + + * Rule priority + + * Optional: Yes, default priority 0 will be used + + * Syntax: *pri * + +```` + + * The source IP address and mask + + * Optional: Yes, default address 0.0.0.0 and mask of 0 will be used + + * Syntax: + + * *src X.X.X.X/Y* for IPv4 + * *src XXXX:XXXX:XXXX:XXXX:XXXX:XXXX:XXXX:XXXX/Y* for IPv6 + +```` + + * The destination IP address and mask + + * Optional: Yes, default address 0.0.0.0 and mask of 0 will be used + + * Syntax: + + * *dst X.X.X.X/Y* for IPv4 + * *dst XXXX:XXXX:XXXX:XXXX:XXXX:XXXX:XXXX:XXXX/Y* for IPv6 + +```` + + * The protocol start and end range + + * Optional: yes, default range of 0 to 0 will be used + + * Syntax: *proto X:Y* + +```` + + * The source port start and end range + + * Optional: yes, default range of 0 to 0 will be used + + * Syntax: *sport X:Y* + +```` + + * The destination port start and end range + + * Optional: yes, default range of 0 to 0 will be used + + * Syntax: *dport X:Y* + +Example SP rules: + +.. code-block:: console + + sp ipv4 out esp protect 105 pri 1 dst 192.168.115.0/24 sport 0:65535 \ + dport 0:65535 + + sp ipv6 in esp bypass pri 1 dst 0000:0000:0000:0000:5555:5555:\ + 0000:0000/96 sport 0:65535 dport 0:65535 + + +SA rule syntax +^^^^^^^^^^^^^^ + +The successfully parsed SA rules will be stored in an array table. + +The SA rule syntax is shown as follows: + +.. code-block:: console + + sa + + +where each options means: + +```` + + * The traffic direction + + * Optional: No + + * Available options: + + * *in*: inbound traffic + * *out*: outbound traffic + +```` + + * The SPI number + + * Optional: No + + * Syntax: unsigned integer number + +```` + + * Cipher algorithm + + * Optional: Yes, unless is not used + + * Available options: + + * *null*: NULL algorithm + * *aes-128-cbc*: AES-CBC 128-bit algorithm + * *aes-128-ctr*: AES-CTR 128-bit algorithm + + * Syntax: *cipher_algo * + +```` + + * Cipher key, NOT available when 'null' algorithm is used + + * Optional: Yes, unless is not used. + Must be followed by option + + * Syntax: Hexadecimal bytes (0x0-0xFF) concatenate by colon symbol ':'. + The number of bytes should be as same as the specified cipher algorithm + key size. + + For example: *cipher_key A1:B2:C3:D4:A1:B2:C3:D4:A1:B2:C3:D4: + A1:B2:C3:D4* + +```` + + * Authentication algorithm + + * Optional: Yes, unless is not used + + * Available options: + + * *null*: NULL algorithm + * *sha1-hmac*: HMAC SHA1 algorithm + +```` + + * Authentication key, NOT available when 'null' or 'aes-128-gcm' algorithm + is used. + + * Optional: Yes, unless is not used. + Must be followed by option + + * Syntax: Hexadecimal bytes (0x0-0xFF) concatenate by colon symbol ':'. + The number of bytes should be as same as the specified authentication + algorithm key size. + + For example: *auth_key A1:B2:C3:D4:A1:B2:C3:D4:A1:B2:C3:D4:A1:B2:C3:D4: + A1:B2:C3:D4* + +```` + + * AEAD algorithm + + * Optional: Yes, unless and are not used + + * Available options: + + * *aes-128-gcm*: AES-GCM 128-bit algorithm + + * Syntax: *cipher_algo * + +```` + + * Cipher key, NOT available when 'null' algorithm is used + + * Optional: Yes, unless and are not used. + Must be followed by option + + * Syntax: Hexadecimal bytes (0x0-0xFF) concatenate by colon symbol ':'. + The number of bytes should be as same as the specified AEAD algorithm + key size. + + For example: *aead_key A1:B2:C3:D4:A1:B2:C3:D4:A1:B2:C3:D4: + A1:B2:C3:D4* + +```` + + * The operation mode + + * Optional: No + + * Available options: + + * *ipv4-tunnel*: Tunnel mode for IPv4 packets + * *ipv6-tunnel*: Tunnel mode for IPv6 packets + * *transport*: transport mode + + * Syntax: mode XXX + +```` + + * The source IP address. This option is not available when + transport mode is used + + * Optional: Yes, default address 0.0.0.0 will be used + + * Syntax: + + * *src X.X.X.X* for IPv4 + * *src XXXX:XXXX:XXXX:XXXX:XXXX:XXXX:XXXX:XXXX* for IPv6 + +```` + + * The destination IP address. This option is not available when + transport mode is used + + * Optional: Yes, default address 0.0.0.0 will be used + + * Syntax: + + * *dst X.X.X.X* for IPv4 + * *dst XXXX:XXXX:XXXX:XXXX:XXXX:XXXX:XXXX:XXXX* for IPv6 + +```` + + * Action type to specify the security action. This option specify + the SA to be performed with look aside protocol offload to HW + accelerator or protocol offload on ethernet device or inline + crypto processing on the ethernet device during transmission. + + * Optional: Yes, default type *no-offload* + + * Available options: + + * *lookaside-protocol-offload*: look aside protocol offload to HW accelerator + * *inline-protocol-offload*: inline protocol offload on ethernet device + * *inline-crypto-offload*: inline crypto processing on ethernet device + * *no-offload*: no offloading to hardware + + ```` + + * Port/device ID of the ethernet/crypto accelerator for which the SA is + configured. For *inline-crypto-offload* and *inline-protocol-offload*, this + port will be used for routing. The routing table will not be referred in + this case. + + * Optional: No, if *type* is not *no-offload* + + * Syntax: + + * *port_id X* X is a valid device number in decimal + + +Example SA rules: + +.. code-block:: console + + sa out 5 cipher_algo null auth_algo null mode ipv4-tunnel \ + src 172.16.1.5 dst 172.16.2.5 + + sa out 25 cipher_algo aes-128-cbc \ + cipher_key c3:c3:c3:c3:c3:c3:c3:c3:c3:c3:c3:c3:c3:c3:c3:c3 \ + auth_algo sha1-hmac \ + auth_key c3:c3:c3:c3:c3:c3:c3:c3:c3:c3:c3:c3:c3:c3:c3:c3:c3:c3:c3:c3 \ + mode ipv6-tunnel \ + src 1111:1111:1111:1111:1111:1111:1111:5555 \ + dst 2222:2222:2222:2222:2222:2222:2222:5555 + + sa in 105 aead_algo aes-128-gcm \ + aead_key de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef:de:ad:be:ef \ + mode ipv4-tunnel src 172.16.2.5 dst 172.16.1.5 + + sa out 5 cipher_algo aes-128-cbc cipher_key 0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0 \ + auth_algo sha1-hmac auth_key 0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0 \ + mode ipv4-tunnel src 172.16.1.5 dst 172.16.2.5 \ + type lookaside-protocol-offload port_id 4 + +Routing rule syntax +^^^^^^^^^^^^^^^^^^^ + +The Routing rule syntax is shown as follows: + +.. code-block:: console + + rt + + +where each options means: + +```` + + * IP protocol version + + * Optional: No + + * Available options: + + * *ipv4*: IP protocol version 4 + * *ipv6*: IP protocol version 6 + +```` + + * The source IP address and mask + + * Optional: Yes, default address 0.0.0.0 and mask of 0 will be used + + * Syntax: + + * *src X.X.X.X/Y* for IPv4 + * *src XXXX:XXXX:XXXX:XXXX:XXXX:XXXX:XXXX:XXXX/Y* for IPv6 + +```` + + * The destination IP address and mask + + * Optional: Yes, default address 0.0.0.0 and mask of 0 will be used + + * Syntax: + + * *dst X.X.X.X/Y* for IPv4 + * *dst XXXX:XXXX:XXXX:XXXX:XXXX:XXXX:XXXX:XXXX/Y* for IPv6 + +```` + + * The traffic output port id + + * Optional: yes, default output port 0 will be used + + * Syntax: *port X* + +Example SP rules: + +.. code-block:: console + + rt ipv4 dst 172.16.1.5/32 port 0 + + rt ipv6 dst 1111:1111:1111:1111:1111:1111:1111:5555/116 port 0 diff --git a/dpdk/doc/guides/sample_app_ug/ipv4_multicast.rst b/dpdk/doc/guides/sample_app_ug/ipv4_multicast.rst index 72da8c42..7a8e7ebc 100644 --- a/dpdk/doc/guides/sample_app_ug/ipv4_multicast.rst +++ b/dpdk/doc/guides/sample_app_ug/ipv4_multicast.rst @@ -51,43 +51,24 @@ The lookup method is the Four-byte Key (FBK) hash-based method. The lookup table is composed of pairs of destination IPv4 address (the FBK) and a port mask associated with that IPv4 address. +.. note:: + + The max port mask supported in the given hash table is 0xf, so only first + four ports can be supported. + If using non-consecutive ports, use the destination IPv4 address accordingly. + For convenience and simplicity, this sample application does not take IANA-assigned multicast addresses into account, but instead equates the last four bytes of the multicast group (that is, the last four bytes of the destination IP address) with the mask of ports to multicast packets to. Also, the application does not consider the Ethernet addresses; it looks only at the IPv4 destination address for any given packet. -Building the Application ------------------------- +Compiling the Application +------------------------- -To compile the application: +To compile the sample application see :doc:`compiling`. -#. Go to the sample application directory: - - .. code-block:: console - - export RTE_SDK=/path/to/rte_sdk - cd ${RTE_SDK}/examples/ipv4_multicast - -#. Set the target (a default target is used if not specified). For example: - - .. code-block:: console - - export RTE_TARGET=x86_64-native-linuxapp-gcc - -See the *DPDK Getting Started Guide* for possible RTE_TARGET values. - -#. Build the application: - - .. code-block:: console - - make - -.. note:: - - The compiled application is written to the build subdirectory. - To have the application written to a different location, - the O=/path/to/build/directory option may be specified in the make command. +The application is located in the ``ipv4_multicast`` sub-directory. Running the Application ----------------------- @@ -113,11 +94,11 @@ Typically, to run the IPv4 Multicast sample application, issue the following com .. code-block:: console - ./build/ipv4_multicast -c 0x00f -n 3 -- -p 0x3 -q 1 + ./build/ipv4_multicast -l 0-3 -n 3 -- -p 0x3 -q 1 In this command: -* The -c option enables cores 0, 1, 2 and 3 +* The -l option enables cores 0, 1, 2 and 3 * The -n option specifies 3 memory channels @@ -145,12 +126,12 @@ Memory pools for indirect buffers are initialized differently from the memory po .. code-block:: c - packet_pool = rte_mempool_create("packet_pool", NB_PKT_MBUF, PKT_MBUF_SIZE, 32, sizeof(struct rte_pktmbuf_pool_private), - rte_pktmbuf_pool_init, NULL, rte_pktmbuf_init, NULL, rte_socket_id(), 0); - - header_pool = rte_mempool_create("header_pool", NB_HDR_MBUF, HDR_MBUF_SIZE, 32, 0, NULL, NULL, rte_pktmbuf_init, NULL, rte_socket_id(), 0); - clone_pool = rte_mempool_create("clone_pool", NB_CLONE_MBUF, - CLONE_MBUF_SIZE, 32, 0, NULL, NULL, rte_pktmbuf_init, NULL, rte_socket_id(), 0); + packet_pool = rte_pktmbuf_pool_create("packet_pool", NB_PKT_MBUF, 32, + 0, PKT_MBUF_DATA_SIZE, rte_socket_id()); + header_pool = rte_pktmbuf_pool_create("header_pool", NB_HDR_MBUF, 32, + 0, HDR_MBUF_DATA_SIZE, rte_socket_id()); + clone_pool = rte_pktmbuf_pool_create("clone_pool", NB_CLONE_MBUF, 32, + 0, 0, rte_socket_id()); The reason for this is because indirect buffers are not supposed to hold any packet data and therefore can be initialized with lower amount of reserved memory for each buffer. @@ -262,7 +243,7 @@ The actual packet transmission is done in the mcast_send_pkt() function: .. code-block:: c - static inline void mcast_send_pkt(struct rte_mbuf *pkt, struct ether_addr *dest_addr, struct lcore_queue_conf *qconf, uint8_t port) + static inline void mcast_send_pkt(struct rte_mbuf *pkt, struct ether_addr *dest_addr, struct lcore_queue_conf *qconf, uint16_t port) { struct ether_hdr *ethdr; uint16_t len; @@ -358,7 +339,7 @@ It is the mcast_out_pkt() function that performs the packet duplication (either /* update header's fields */ hdr->pkt.pkt_len = (uint16_t)(hdr->pkt.data_len + pkt->pkt.pkt_len); - hdr->pkt.nb_segs = (uint8_t)(pkt->pkt.nb_segs + 1); + hdr->pkt.nb_segs = pkt->pkt.nb_segs + 1; /* copy metadata from source packet */ diff --git a/dpdk/doc/guides/sample_app_ug/keep_alive.rst b/dpdk/doc/guides/sample_app_ug/keep_alive.rst index 38973779..6c131a86 100644 --- a/dpdk/doc/guides/sample_app_ug/keep_alive.rst +++ b/dpdk/doc/guides/sample_app_ug/keep_alive.rst @@ -66,27 +66,9 @@ of the L2 forwarding application. Compiling the Application ------------------------- -To compile the application: +To compile the sample application see :doc:`compiling`. -#. Go to the sample application directory: - - .. code-block:: console - - export RTE_SDK=/path/to/rte_sdk cd ${RTE_SDK}/examples/keep_alive - -#. Set the target (a default target is used if not specified). For example: - - .. code-block:: console - - export RTE_TARGET=x86_64-native-linuxapp-gcc - - See the *DPDK Getting Started Guide* for possible RTE_TARGET values. - -#. Build the application: - - .. code-block:: console - - make +The application is located in the ``l2fwd_keep_alive`` sub-directory. Running the Application ----------------------- @@ -114,7 +96,7 @@ To run the application in linuxapp environment with 4 lcores, 16 ports .. code-block:: console - ./build/l2fwd-keepalive -c f -n 4 -- -q 8 -p ffff -K 10 + ./build/l2fwd-keepalive -l 0-3 -n 4 -- -q 8 -p ffff -K 10 Refer to the *DPDK Getting Started Guide* for general information on running applications and the Environment Abstraction Layer (EAL) @@ -186,5 +168,5 @@ The rte_keepalive_mark_alive function simply sets the core state to alive. static inline void rte_keepalive_mark_alive(struct rte_keepalive *keepcfg) { - keepcfg->state_flags[rte_lcore_id()] = ALIVE; + keepcfg->live_data[rte_lcore_id()].core_state = RTE_KA_STATE_ALIVE; } diff --git a/dpdk/doc/guides/sample_app_ug/kernel_nic_interface.rst b/dpdk/doc/guides/sample_app_ug/kernel_nic_interface.rst index 2ae9b702..e1ac4153 100644 --- a/dpdk/doc/guides/sample_app_ug/kernel_nic_interface.rst +++ b/dpdk/doc/guides/sample_app_ug/kernel_nic_interface.rst @@ -77,35 +77,17 @@ The packet flow through the Kernel NIC Interface application is as shown in the Kernel NIC Application Packet Flow - Compiling the Application ------------------------- -Compile the application as follows: +To compile the sample application see :doc:`compiling`. -#. Go to the example directory: +The application is located in the ``kni`` sub-directory. - .. code-block:: console - - export RTE_SDK=/path/to/rte_sdk - cd ${RTE_SDK}/examples/kni - -#. Set the target (a default target is used if not specified) - - .. note:: +.. note:: This application is intended as a linuxapp only. - .. code-block:: console - - export RTE_TARGET=x86_64-native-linuxapp-gcc - -#. Build the application: - - .. code-block:: console - - make - Loading the Kernel Module ------------------------- @@ -180,7 +162,7 @@ Where: Refer to *DPDK Getting Started Guide* for general information on running applications and the Environment Abstraction Layer (EAL) options. -The -c coremask parameter of the EAL options should include the lcores indicated by the lcore_rx and lcore_tx, +The -c coremask or -l corelist parameter of the EAL options should include the lcores indicated by the lcore_rx and lcore_tx, but does not need to include lcores indicated by lcore_kthread as they are used to pin the kernel thread on. The -p PORTMASK parameter should include the ports indicated by the port in --config, neither more nor less. @@ -199,7 +181,7 @@ and one lcore of kernel thread for each port: .. code-block:: console - ./build/kni -c 0xf0 -n 4 -- -P -p 0x3 -config="(0,4,6,8),(1,5,7,9)" + ./build/kni -l 4-7 -n 4 -- -P -p 0x3 -config="(0,4,6,8),(1,5,7,9)" KNI Operations -------------- @@ -246,7 +228,7 @@ The code for allocating the kernel NIC interfaces for a specific port is as foll .. code-block:: c static int - kni_alloc(uint8_t port_id) + kni_alloc(uint16_t port_id) { uint8_t i; struct rte_kni *kni; @@ -335,7 +317,7 @@ The code is as follows: int i, j, nb_token; char *str_fld[_NUM_FLD]; unsigned long int_fld[_NUM_FLD]; - uint8_t port_id, nb_kni_port_params = 0; + uint16_t port_id, nb_kni_port_params = 0; memset(&kni_port_params_array, 0, sizeof(kni_port_params_array)); @@ -532,7 +514,7 @@ Currently, setting a new MTU and configuring the network interface (up/ down) ar /* Callback for request of changing MTU */ static int - kni_change_mtu(uint8_t port_id, unsigned new_mtu) + kni_change_mtu(uint16_t port_id, unsigned new_mtu) { int ret; struct rte_eth_conf conf; @@ -581,7 +563,7 @@ Currently, setting a new MTU and configuring the network interface (up/ down) ar /* Callback for request of configuring network interface up/down */ static int - kni_config_network_interface(uint8_t port_id, uint8_t if_up) + kni_config_network_interface(uint16_t port_id, uint8_t if_up) { int ret = 0; diff --git a/dpdk/doc/guides/sample_app_ug/l2_forward_cat.rst b/dpdk/doc/guides/sample_app_ug/l2_forward_cat.rst index 285c3c74..a10f23a1 100644 --- a/dpdk/doc/guides/sample_app_ug/l2_forward_cat.rst +++ b/dpdk/doc/guides/sample_app_ug/l2_forward_cat.rst @@ -67,38 +67,28 @@ White paper demonstrating example use case: Compiling the Application ------------------------- +.. note:: -Requires ``libpqos`` from Intel's -`intel-cmt-cat software package `_ -hosted on GitHub repository. For installation notes, please see ``README`` file. + Requires ``libpqos`` from Intel's + `intel-cmt-cat software package `_ + hosted on GitHub repository. For installation notes, please see ``README`` file. -GIT: + GIT: -* https://github.com/01org/intel-cmt-cat + * https://github.com/01org/intel-cmt-cat -To compile the application export the path to PQoS lib -and the DPDK source tree and go to the example directory: -.. code-block:: console +#. To compile the application export the path to PQoS lib + and the DPDK source tree and go to the example directory: - export PQOS_INSTALL_PATH=/path/to/libpqos - export RTE_SDK=/path/to/rte_sdk + .. code-block:: console - cd ${RTE_SDK}/examples/l2fwd-cat + export PQOS_INSTALL_PATH=/path/to/libpqos -Set the target, for example: -.. code-block:: console +To compile the sample application see :doc:`compiling`. - export RTE_TARGET=x86_64-native-linuxapp-gcc - -See the *DPDK Getting Started* Guide for possible ``RTE_TARGET`` values. - -Build the application as follows: - -.. code-block:: console - - make +The application is located in the ``l2fwd-cat`` sub-directory. Running the Application @@ -108,13 +98,13 @@ To run the example in a ``linuxapp`` environment and enable CAT on cpus 0-2: .. code-block:: console - ./build/l2fwd-cat -c 2 -n 4 -- --l3ca="0x3@(0-2)" + ./build/l2fwd-cat -l 1 -n 4 -- --l3ca="0x3@(0-2)" or to enable CAT and CDP on cpus 1,3: .. code-block:: console - ./build/l2fwd-cat -c 2 -n 4 -- --l3ca="(0x00C00,0x00300)@(1,3)" + ./build/l2fwd-cat -l 1 -n 4 -- --l3ca="(0x00C00,0x00300)@(1,3)" If CDP is not supported it will fail with following error message: @@ -242,4 +232,4 @@ relevant CPUs via ``pqos_l3ca_assoc_set(...)`` calls. ``atexit(...)`` is used to register ``cat_exit(...)`` to be called on a clean exit. ``cat_exit(...)`` performs a simple CAT clean-up, by associating -COS 0 to all involved CPUs via ``pqos_l3ca_assoc_set(...)`` calls. \ No newline at end of file +COS 0 to all involved CPUs via ``pqos_l3ca_assoc_set(...)`` calls. diff --git a/dpdk/doc/guides/sample_app_ug/l2_forward_crypto.rst b/dpdk/doc/guides/sample_app_ug/l2_forward_crypto.rst index 723376c5..1e85b4a3 100644 --- a/dpdk/doc/guides/sample_app_ug/l2_forward_crypto.rst +++ b/dpdk/doc/guides/sample_app_ug/l2_forward_crypto.rst @@ -1,5 +1,5 @@ .. BSD LICENSE - Copyright(c) 2016 Intel Corporation. All rights reserved. + Copyright(c) 2016-2017 Intel Corporation. All rights reserved. All rights reserved. Redistribution and use in source and binary forms, with or without @@ -46,7 +46,7 @@ for each packet that is received on a RX_PORT and performs L2 forwarding. The destination port is the adjacent port from the enabled portmask, that is, if the first four ports are enabled (portmask 0xf), ports 0 and 1 forward into each other, and ports 2 and 3 forward into each other. -Also, the MAC addresses are affected as follows: +Also, if MAC addresses updating is enabled, the MAC addresses are affected as follows: * The source MAC address is replaced by the TX_PORT MAC address @@ -55,26 +55,9 @@ Also, the MAC addresses are affected as follows: Compiling the Application ------------------------- -#. Go to the example directory: +To compile the sample application see :doc:`compiling`. - .. code-block:: console - - export RTE_SDK=/path/to/rte_sdk - cd ${RTE_SDK}/examples/l2fwd-crypto - -#. Set the target (a default target is used if not specified). For example: - - .. code-block:: console - - export RTE_TARGET=x86_64-native-linuxapp-gcc - - *See the DPDK Getting Started Guide* for possible RTE_TARGET values. - -#. Build the application: - - .. code-block:: console - - make +The application is located in the ``l2fwd-crypt`` sub-directory. Running the Application ----------------------- @@ -84,12 +67,16 @@ The application requires a number of command line options: .. code-block:: console ./build/l2fwd-crypto [EAL options] -- [-p PORTMASK] [-q NQ] [-s] [-T PERIOD] / - [--cdev_type HW/SW/ANY] [--chain HASH_CIPHER/CIPHER_HASH/CIPHER_ONLY/HASH_ONLY] / + [--cdev_type HW/SW/ANY] [--chain HASH_CIPHER/CIPHER_HASH/CIPHER_ONLY/HASH_ONLY/AEAD] / [--cipher_algo ALGO] [--cipher_op ENCRYPT/DECRYPT] [--cipher_key KEY] / - [--cipher_key_random_size SIZE] [--iv IV] [--iv_random_size SIZE] / + [--cipher_key_random_size SIZE] [--cipher_iv IV] [--cipher_iv_random_size SIZE] / [--auth_algo ALGO] [--auth_op GENERATE/VERIFY] [--auth_key KEY] / - [--auth_key_random_size SIZE] [--aad AAD] [--aad_random_size SIZE] / - [--digest size SIZE] [--sessionless] + [--auth_key_random_size SIZE] [--auth_iv IV] [--auth_iv_random_size SIZE] / + [--aead_algo ALGO] [--aead_op ENCRYPT/DECRYPT] [--aead_key KEY] / + [--aead_key_random_size SIZE] [--aead_iv] [--aead_iv_random_size SIZE] / + [--aad AAD] [--aad_random_size SIZE] / + [--digest size SIZE] [--sessionless] [--cryptodev_mask MASK] / + [--mac-updating] [--no-mac-updating] where, @@ -109,11 +96,13 @@ where, * chain: select the operation chaining to perform: Cipher->Hash (CIPHER_HASH), - Hash->Cipher (HASH_CIPHER), Cipher (CIPHER_ONLY), Hash(HASH_ONLY) + Hash->Cipher (HASH_CIPHER), Cipher (CIPHER_ONLY), Hash (HASH_ONLY) + + or AEAD (AEAD) (default is Cipher->Hash) -* cipher_algo: select the ciphering algorithm (default is AES CBC) +* cipher_algo: select the ciphering algorithm (default is aes-cbc) * cipher_op: select the ciphering operation to perform: ENCRYPT or DECRYPT @@ -127,15 +116,15 @@ where, Note that if --cipher_key is used, this will be ignored. -* iv: set the IV to be used. Bytes has to be separated with ":" +* cipher_iv: set the cipher IV to be used. Bytes has to be separated with ":" -* iv_random_size: set the size of the IV, which will be generated randomly. +* cipher_iv_random_size: set the size of the cipher IV, which will be generated randomly. - Note that if --iv is used, this will be ignored. + Note that if --cipher_iv is used, this will be ignored. -* auth_algo: select the authentication algorithm (default is SHA1-HMAC) +* auth_algo: select the authentication algorithm (default is sha1-hmac) -* cipher_op: select the authentication operation to perform: GENERATE or VERIFY +* auth_op: select the authentication operation to perform: GENERATE or VERIFY (default is GENERATE) @@ -147,6 +136,32 @@ where, Note that if --auth_key is used, this will be ignored. +* auth_iv: set the auth IV to be used. Bytes has to be separated with ":" + +* auth_iv_random_size: set the size of the auth IV, which will be generated randomly. + + Note that if --auth_iv is used, this will be ignored. + +* aead_algo: select the AEAD algorithm (default is aes-gcm) + +* aead_op: select the AEAD operation to perform: ENCRYPT or DECRYPT + + (default is ENCRYPT) + +* aead_key: set the AEAD key to be used. Bytes has to be separated with ":" + +* aead_key_random_size: set the size of the AEAD key, + + which will be generated randomly. + + Note that if --aead_key is used, this will be ignored. + +* aead_iv: set the AEAD IV to be used. Bytes has to be separated with ":" + +* aead_iv_random_size: set the size of the AEAD IV, which will be generated randomly. + + Note that if --aead_iv is used, this will be ignored. + * aad: set the AAD to be used. Bytes has to be separated with ":" * aad_random_size: set the size of the AAD, which will be generated randomly. @@ -157,6 +172,13 @@ where, * sessionless: no crypto session will be created. +* cryptodev_mask: A hexadecimal bitmask of the cryptodevs to be used by the + application. + + (default is all cryptodevs). + +* [no-]mac-updating: Enable or disable MAC addresses updating (enabled by default). + The application requires that crypto devices capable of performing the specified crypto operation are available on application initialization. @@ -167,11 +189,11 @@ To run the application in linuxapp environment with 2 lcores, 2 ports and 2 cryp .. code-block:: console - $ ./build/l2fwd-crypto -c 0x3 -n 4 --vdev "cryptodev_aesni_mb_pmd" \ - --vdev "cryptodev_aesni_mb_pmd" -- -p 0x3 --chain CIPHER_HASH \ - --cipher_op ENCRYPT --cipher_algo AES_CBC \ + $ ./build/l2fwd-crypto -l 0-1 -n 4 --vdev "crypto_aesni_mb0" \ + --vdev "crypto_aesni_mb1" -- -p 0x3 --chain CIPHER_HASH \ + --cipher_op ENCRYPT --cipher_algo aes-cbc \ --cipher_key 00:01:02:03:04:05:06:07:08:09:0a:0b:0c:0d:0e:0f \ - --auth_op GENERATE --auth_algo AES_XCBC_MAC \ + --auth_op GENERATE --auth_algo aes-xcbc-mac \ --auth_key 10:11:12:13:14:15:16:17:18:19:1a:1b:1c:1d:1e:1f Refer to the *DPDK Getting Started Guide* for general information on running applications @@ -326,8 +348,14 @@ This session is created and is later attached to the crypto operation: uint8_t cdev_id) { struct rte_crypto_sym_xform *first_xform; + struct rte_cryptodev_sym_session *session; + uint8_t socket_id = rte_cryptodev_socket_id(cdev_id); + struct rte_mempool *sess_mp = session_pool_socket[socket_id]; - if (options->xform_chain == L2FWD_CRYPTO_CIPHER_HASH) { + + if (options->xform_chain == L2FWD_CRYPTO_AEAD) { + first_xform = &options->aead_xform; + } else if (options->xform_chain == L2FWD_CRYPTO_CIPHER_HASH) { first_xform = &options->cipher_xform; first_xform->next = &options->auth_xform; } else if (options->xform_chain == L2FWD_CRYPTO_HASH_CIPHER) { @@ -339,8 +367,16 @@ This session is created and is later attached to the crypto operation: first_xform = &options->auth_xform; } - /* Setup Cipher Parameters */ - return rte_cryptodev_sym_session_create(cdev_id, first_xform); + session = rte_cryptodev_sym_session_create(sess_mp); + + if (session == NULL) + return NULL; + + if (rte_cryptodev_sym_session_init(cdev_id, session, + first_xform, sess_mp) < 0) + return NULL; + + return session; } ... diff --git a/dpdk/doc/guides/sample_app_ug/l2_forward_job_stats.rst b/dpdk/doc/guides/sample_app_ug/l2_forward_job_stats.rst index 2444e36e..54b970f8 100644 --- a/dpdk/doc/guides/sample_app_ug/l2_forward_job_stats.rst +++ b/dpdk/doc/guides/sample_app_ug/l2_forward_job_stats.rst @@ -97,26 +97,9 @@ in this case enabling a total of four Virtual Functions. Compiling the Application ------------------------- -#. Go to the example directory: +To compile the sample application see :doc:`compiling`. - .. code-block:: console - - export RTE_SDK=/path/to/rte_sdk - cd ${RTE_SDK}/examples/l2fwd-jobstats - -#. Set the target (a default target is used if not specified). For example: - - .. code-block:: console - - export RTE_TARGET=x86_64-native-linuxapp-gcc - - *See the DPDK Getting Started Guide* for possible RTE_TARGET values. - -#. Build the application: - - .. code-block:: console - - make +The application is located in the ``l2fwd-jobstats`` sub-directory. Running the Application ----------------------- @@ -140,7 +123,7 @@ thousands separator printing, issue the command: .. code-block:: console - $ ./build/l2fwd-jobstats -c f -n 4 -- -q 8 -p ffff -l + $ ./build/l2fwd-jobstats -l 0-3 -n 4 -- -q 8 -p ffff -l Refer to the *DPDK Getting Started Guide* for general information on running applications and the Environment Abstraction Layer (EAL) options. @@ -193,36 +176,25 @@ and the application to store network packet data: .. code-block:: c /* create the mbuf pool */ - l2fwd_pktmbuf_pool = - rte_mempool_create("mbuf_pool", NB_MBUF, - MBUF_SIZE, 32, - sizeof(struct rte_pktmbuf_pool_private), - rte_pktmbuf_pool_init, NULL, - rte_pktmbuf_init, NULL, - rte_socket_id(), 0); + l2fwd_pktmbuf_pool = rte_pktmbuf_pool_create("mbuf_pool", NB_MBUF, + MEMPOOL_CACHE_SIZE, 0, RTE_MBUF_DEFAULT_BUF_SIZE, + rte_socket_id()); if (l2fwd_pktmbuf_pool == NULL) rte_exit(EXIT_FAILURE, "Cannot init mbuf pool\n"); The rte_mempool is a generic structure used to handle pools of objects. -In this case, it is necessary to create a pool that will be used by the driver, -which expects to have some reserved space in the mempool structure, -sizeof(struct rte_pktmbuf_pool_private) bytes. -The number of allocated pkt mbufs is NB_MBUF, with a size of MBUF_SIZE each. -A per-lcore cache of 32 mbufs is kept. +In this case, it is necessary to create a pool that will be used by the driver. +The number of allocated pkt mbufs is NB_MBUF, with a data room size of +RTE_MBUF_DEFAULT_BUF_SIZE each. +A per-lcore cache of MEMPOOL_CACHE_SIZE mbufs is kept. The memory is allocated in rte_socket_id() socket, but it is possible to extend this code to allocate one mbuf pool per socket. -Two callback pointers are also given to the rte_mempool_create() function: - -* The first callback pointer is to rte_pktmbuf_pool_init() and is used - to initialize the private data of the mempool, which is needed by the driver. - This function is provided by the mbuf API, but can be copied and extended by the developer. - -* The second callback pointer given to rte_mempool_create() is the mbuf initializer. - The default is used, that is, rte_pktmbuf_init(), which is provided in the rte_mbuf library. - If a more complex application wants to extend the rte_pktmbuf structure for its own needs, - a new function derived from rte_pktmbuf_init( ) can be created. +The rte_pktmbuf_pool_create() function uses the default mbuf pool and mbuf +initializers, respectively rte_pktmbuf_pool_init() and rte_pktmbuf_init(). +An advanced application may want to use the mempool API to create the +mbuf pool with more control. Driver Initialization ~~~~~~~~~~~~~~~~~~~~~ @@ -569,7 +541,7 @@ If the table is full, the whole packets table is transmitted using the l2fwd_sen /* Send the packet on an output interface */ static int - l2fwd_send_packet(struct rte_mbuf *m, uint8_t port) + l2fwd_send_packet(struct rte_mbuf *m, uint16_t port) { unsigned lcore_id, len; struct lcore_queue_conf *qconf; @@ -604,7 +576,7 @@ however it improves performance: unsigned lcore_id; struct lcore_queue_conf *qconf; struct mbuf_table *m_table; - uint8_t portid; + uint16_t portid; lcore_id = rte_lcore_id(); qconf = &lcore_queue_conf[lcore_id]; diff --git a/dpdk/doc/guides/sample_app_ug/l2_forward_real_virtual.rst b/dpdk/doc/guides/sample_app_ug/l2_forward_real_virtual.rst index a1c10c04..4d9cae43 100644 --- a/dpdk/doc/guides/sample_app_ug/l2_forward_real_virtual.rst +++ b/dpdk/doc/guides/sample_app_ug/l2_forward_real_virtual.rst @@ -50,17 +50,14 @@ performs L2 forwarding for each packet that is received on an RX_PORT. The destination port is the adjacent port from the enabled portmask, that is, if the first four ports are enabled (portmask 0xf), ports 1 and 2 forward into each other, and ports 3 and 4 forward into each other. -Also, the MAC addresses are affected as follows: +Also, if MAC addresses updating is enabled, the MAC addresses are affected as follows: * The source MAC address is replaced by the TX_PORT MAC address * The destination MAC address is replaced by 02:00:00:00:00:TX_PORT_ID -This application can be used to benchmark performance using a traffic-generator, as shown in the :numref:`figure_l2_fwd_benchmark_setup`. - -The application can also be used in a virtualized environment as shown in :numref:`figure_l2_fwd_virtenv_benchmark_setup`. - -The L2 Forwarding application can also be used as a starting point for developing a new application based on the DPDK. +This application can be used to benchmark performance using a traffic-generator, as shown in the :numref:`figure_l2_fwd_benchmark_setup`, +or in a virtualized environment as shown in :numref:`figure_l2_fwd_virtenv_benchmark_setup`. .. _figure_l2_fwd_benchmark_setup: @@ -68,13 +65,23 @@ The L2 Forwarding application can also be used as a starting point for developin Performance Benchmark Setup (Basic Environment) - .. _figure_l2_fwd_virtenv_benchmark_setup: .. figure:: img/l2_fwd_virtenv_benchmark_setup.* Performance Benchmark Setup (Virtualized Environment) +This application may be used for basic VM to VM communication as shown in :numref:`figure_l2_fwd_vm2vm`, +when MAC addresses updating is disabled. + +.. _figure_l2_fwd_vm2vm: + +.. figure:: img/l2_fwd_vm2vm.* + + Virtual Machine to Virtual Machine communication. + +The L2 Forwarding application can also be used as a starting point for developing a new application based on the DPDK. + .. _l2_fwd_vf_setup: Virtual Function Setup Instructions @@ -100,26 +107,9 @@ in this case enabling a total of four Virtual Functions. Compiling the Application ------------------------- -#. Go to the example directory: +To compile the sample application see :doc:`compiling`. - .. code-block:: console - - export RTE_SDK=/path/to/rte_sdk - cd ${RTE_SDK}/examples/l2fwd - -#. Set the target (a default target is used if not specified). For example: - - .. code-block:: console - - export RTE_TARGET=x86_64-native-linuxapp-gcc - - *See the DPDK Getting Started Guide* for possible RTE_TARGET values. - -#. Build the application: - - .. code-block:: console - - make +The application is located in the ``l2fwd`` sub-directory. Running the Application ----------------------- @@ -128,7 +118,7 @@ The application requires a number of command line options: .. code-block:: console - ./build/l2fwd [EAL options] -- -p PORTMASK [-q NQ] + ./build/l2fwd [EAL options] -- -p PORTMASK [-q NQ] --[no-]mac-updating where, @@ -136,11 +126,14 @@ where, * q NQ: A number of queues (=ports) per lcore (default is 1) -To run the application in linuxapp environment with 4 lcores, 16 ports and 8 RX queues per lcore, issue the command: +* --[no-]mac-updating: Enable or disable MAC addresses updating (enabled by default). + +To run the application in linuxapp environment with 4 lcores, 16 ports and 8 RX queues per lcore and MAC address +updating enabled, issue the command: .. code-block:: console - $ ./build/l2fwd -c f -n 4 -- -q 8 -p ffff + $ ./build/l2fwd -l 0-3 -n 4 -- -q 8 -p ffff Refer to the *DPDK Getting Started Guide* for general information on running applications and the Environment Abstraction Layer (EAL) options. @@ -197,31 +190,25 @@ and the application to store network packet data: /* create the mbuf pool */ - l2fwd_pktmbuf_pool = rte_mempool_create("mbuf_pool", NB_MBUF, MBUF_SIZE, 32, sizeof(struct rte_pktmbuf_pool_private), - rte_pktmbuf_pool_init, NULL, rte_pktmbuf_init, NULL, SOCKET0, 0); + l2fwd_pktmbuf_pool = rte_pktmbuf_pool_create("mbuf_pool", NB_MBUF, + MEMPOOL_CACHE_SIZE, 0, RTE_MBUF_DEFAULT_BUF_SIZE, + rte_socket_id()); if (l2fwd_pktmbuf_pool == NULL) rte_panic("Cannot init mbuf pool\n"); The rte_mempool is a generic structure used to handle pools of objects. -In this case, it is necessary to create a pool that will be used by the driver, -which expects to have some reserved space in the mempool structure, -sizeof(struct rte_pktmbuf_pool_private) bytes. -The number of allocated pkt mbufs is NB_MBUF, with a size of MBUF_SIZE each. +In this case, it is necessary to create a pool that will be used by the driver. +The number of allocated pkt mbufs is NB_MBUF, with a data room size of +RTE_MBUF_DEFAULT_BUF_SIZE each. A per-lcore cache of 32 mbufs is kept. The memory is allocated in NUMA socket 0, but it is possible to extend this code to allocate one mbuf pool per socket. -Two callback pointers are also given to the rte_mempool_create() function: - -* The first callback pointer is to rte_pktmbuf_pool_init() and is used - to initialize the private data of the mempool, which is needed by the driver. - This function is provided by the mbuf API, but can be copied and extended by the developer. - -* The second callback pointer given to rte_mempool_create() is the mbuf initializer. - The default is used, that is, rte_pktmbuf_init(), which is provided in the rte_mbuf library. - If a more complex application wants to extend the rte_pktmbuf structure for its own needs, - a new function derived from rte_pktmbuf_init( ) can be created. +The rte_pktmbuf_pool_create() function uses the default mbuf pool and mbuf +initializers, respectively rte_pktmbuf_pool_init() and rte_pktmbuf_init(). +An advanced application may want to use the mempool API to create the +mbuf pool with more control. .. _l2_fwd_app_dvr_init: @@ -234,7 +221,7 @@ in the *DPDK Programmer's Guide* - Rel 1.4 EAR and the *DPDK API Reference*. .. code-block:: c - if (rte_eal_pci_probe() < 0) + if (rte_pci_probe() < 0) rte_exit(EXIT_FAILURE, "Cannot probe PCI\n"); nb_ports = rte_eth_dev_count(); @@ -275,7 +262,7 @@ Observe that: * rte_igb_pmd_init_all() simultaneously registers the driver as a PCI driver and as an Ethernet* Poll Mode Driver. -* rte_eal_pci_probe() parses the devices on the PCI bus and initializes recognized devices. +* rte_pci_probe() parses the devices on the PCI bus and initializes recognized devices. The next step is to configure the RX and TX queues. For each port, there is only one RX queue (only one lcore is able to poll a given port). @@ -415,7 +402,8 @@ Packets are read in a burst of size MAX_PKT_BURST. The rte_eth_rx_burst() function writes the mbuf pointers in a local table and returns the number of available mbufs in the table. Then, each mbuf in the table is processed by the l2fwd_simple_forward() function. -The processing is very simple: process the TX port from the RX port, then replace the source and destination MAC addresses. +The processing is very simple: process the TX port from the RX port, then replace the source and destination MAC addresses if MAC +addresses updating is enabled. .. note:: @@ -469,7 +457,7 @@ If the table is full, the whole packets table is transmitted using the l2fwd_sen /* Send the packet on an output interface */ static int - l2fwd_send_packet(struct rte_mbuf *m, uint8_t port) + l2fwd_send_packet(struct rte_mbuf *m, uint16_t port) { unsigned lcore_id, len; struct lcore_queue_conf *qconf; diff --git a/dpdk/doc/guides/sample_app_ug/l3_forward.rst b/dpdk/doc/guides/sample_app_ug/l3_forward.rst index e2e62236..19b91e27 100644 --- a/dpdk/doc/guides/sample_app_ug/l3_forward.rst +++ b/dpdk/doc/guides/sample_app_ug/l3_forward.rst @@ -42,7 +42,7 @@ The initialization and run-time paths are very similar to those of the :doc:`l2_ The main difference from the L2 Forwarding sample application is that the forwarding decision is made based on information read from the input packet. -The lookup method is either hash-based or LPM-based and is selected at compile time. When the selected lookup method is hash-based, +The lookup method is either hash-based or LPM-based and is selected at run time. When the selected lookup method is hash-based, a hash object is used to emulate the flow classification stage. The hash object is used in correlation with a flow table to map each input packet to its flow at runtime. @@ -62,28 +62,9 @@ In the sample application, hash-based forwarding supports IPv4 and IPv6. LPM-bas Compiling the Application ------------------------- -To compile the application: +To compile the sample application see :doc:`compiling`. -#. Go to the sample application directory: - - .. code-block:: console - - export RTE_SDK=/path/to/rte_sdk - cd ${RTE_SDK}/examples/l3fwd - -#. Set the target (a default target is used if not specified). For example: - - .. code-block:: console - - export RTE_TARGET=x86_64-native-linuxapp-gcc - - See the *DPDK Getting Started Guide* for possible RTE_TARGET values. - -#. Build the application: - - .. code-block:: console - - make +The application is located in the ``l3fwd`` sub-directory. Running the Application ----------------------- @@ -129,43 +110,33 @@ Where, * ``--parse-ptype:`` Optional, set to use software to analyze packet type. Without this option, hardware will check the packet type. -For example, consider a dual processor socket platform where cores 0-7 and 16-23 appear on socket 0, while cores 8-15 and 24-31 appear on socket 1. -Let's say that the programmer wants to use memory from both NUMA nodes, the platform has only two ports, one connected to each NUMA node, -and the programmer wants to use two cores from each processor socket to do the packet processing. +For example, consider a dual processor socket platform with 8 physical cores, where cores 0-7 and 16-23 appear on socket 0, +while cores 8-15 and 24-31 appear on socket 1. -To enable L3 forwarding between two ports, using two cores, cores 1 and 2, from each processor, -while also taking advantage of local memory access by optimizing around NUMA, the programmer must enable two queues from each port, -pin to the appropriate cores and allocate memory from the appropriate NUMA node. This is achieved using the following command: +To enable L3 forwarding between two ports, assuming that both ports are in the same socket, using two cores, cores 1 and 2, +(which are in the same socket too), use the following command: .. code-block:: console - ./build/l3fwd -c 606 -n 4 -- -p 0x3 --config="(0,0,1),(0,1,2),(1,0,9),(1,1,10)" + ./build/l3fwd -l 1,2 -n 4 -- -p 0x3 --config="(0,0,1),(1,0,2)" In this command: -* The -c option enables cores 0, 1, 2, 3 +* The -l option enables cores 1, 2 * The -p option enables ports 0 and 1 -* The --config option enables two queues on each port and maps each (port,queue) pair to a specific core. - Logic to enable multiple RX queues using RSS and to allocate memory from the correct NUMA nodes - is included in the application and is done transparently. +* The --config option enables one queue on each port and maps each (port,queue) pair to a specific core. The following table shows the mapping in this example: +----------+-----------+-----------+-------------------------------------+ | **Port** | **Queue** | **lcore** | **Description** | | | | | | +----------+-----------+-----------+-------------------------------------+ -| 0 | 0 | 0 | Map queue 0 from port 0 to lcore 0. | +| 0 | 0 | 1 | Map queue 0 from port 0 to lcore 1. | | | | | | +----------+-----------+-----------+-------------------------------------+ -| 0 | 1 | 2 | Map queue 1 from port 0 to lcore 2. | -| | | | | -+----------+-----------+-----------+-------------------------------------+ -| 1 | 0 | 1 | Map queue 0 from port 1 to lcore 1. | -| | | | | -+----------+-----------+-----------+-------------------------------------+ -| 1 | 1 | 3 | Map queue 1 from port 1 to lcore 3. | +| 1 | 0 | 2 | Map queue 0 from port 1 to lcore 2. | | | | | | +----------+-----------+-----------+-------------------------------------+ @@ -293,7 +264,7 @@ The get_ipv4_dst_port() function is shown below: .. code-block:: c static inline uint8_t - get_ipv4_dst_port(void *ipv4_hdr, uint8_t portid, lookup_struct_t *ipv4_l3fwd_lookup_struct) + get_ipv4_dst_port(void *ipv4_hdr, uint16_t portid, lookup_struct_t *ipv4_l3fwd_lookup_struct) { int ret = 0; union ipv4_5tuple_host key; @@ -322,7 +293,7 @@ The key code snippet of simple_ipv4_fwd_4pkts() is shown below: .. code-block:: c static inline void - simple_ipv4_fwd_4pkts(struct rte_mbuf* m[4], uint8_t portid, struct lcore_conf *qconf) + simple_ipv4_fwd_4pkts(struct rte_mbuf* m[4], uint16_t portid, struct lcore_conf *qconf) { // ... @@ -361,10 +332,10 @@ for LPM-based lookups is done by the get_ipv4_dst_port() function below: .. code-block:: c - static inline uint8_t - get_ipv4_dst_port(struct ipv4_hdr *ipv4_hdr, uint8_t portid, lookup_struct_t *ipv4_l3fwd_lookup_struct) + static inline uint16_t + get_ipv4_dst_port(struct ipv4_hdr *ipv4_hdr, uint16_t portid, lookup_struct_t *ipv4_l3fwd_lookup_struct) { uint8_t next_hop; - return (uint8_t) ((rte_lpm_lookup(ipv4_l3fwd_lookup_struct, rte_be_to_cpu_32(ipv4_hdr->dst_addr), &next_hop) == 0)? next_hop : portid); + return ((rte_lpm_lookup(ipv4_l3fwd_lookup_struct, rte_be_to_cpu_32(ipv4_hdr->dst_addr), &next_hop) == 0)? next_hop : portid); } diff --git a/dpdk/doc/guides/sample_app_ug/l3_forward_access_ctrl.rst b/dpdk/doc/guides/sample_app_ug/l3_forward_access_ctrl.rst index 4049e019..dfd372fb 100644 --- a/dpdk/doc/guides/sample_app_ug/l3_forward_access_ctrl.rst +++ b/dpdk/doc/guides/sample_app_ug/l3_forward_access_ctrl.rst @@ -252,28 +252,9 @@ Once the application starts, it transitions through three phases: Compiling the Application ------------------------- -To compile the application: +To compile the sample application see :doc:`compiling`. -#. Go to the sample application directory: - - .. code-block:: console - - export RTE_SDK=/path/to/rte_sdk - cd ${RTE_SDK}/examples/l3fwd-acl - -#. Set the target (a default target is used if not specified). For example: - - .. code-block:: console - - export RTE_TARGET=x86_64-native-linuxapp-gcc - - See the *DPDK IPL Getting Started Guide* for possible RTE_TARGET values. - -#. Build the application: - - .. code-block:: console - - make +The application is located in the ``l3fwd-acl`` sub-directory. Running the Application ----------------------- @@ -306,48 +287,35 @@ where, * --no-numa: optional, disables numa awareness -As an example, consider a dual processor socket platform where cores 0, 2, 4, 6, 8 and 10 appear on socket 0, -while cores 1, 3, 5, 7, 9 and 11 appear on socket 1. -Let's say that the user wants to use memory from both NUMA nodes, -the platform has only two ports and the user wants to use two cores from each processor socket to do the packet processing. +For example, consider a dual processor socket platform with 8 physical cores, where cores 0-7 and 16-23 appear on socket 0, +while cores 8-15 and 24-31 appear on socket 1. -To enable L3 forwarding between two ports, using two cores from each processor, -while also taking advantage of local memory access by optimizing around NUMA, -the user must enable two queues from each port, -pin to the appropriate cores and allocate memory from the appropriate NUMA node. -This is achieved using the following command: +To enable L3 forwarding between two ports, assuming that both ports are in the same socket, using two cores, cores 1 and 2, +(which are in the same socket too), use the following command: .. code-block:: console - ./build/l3fwd-acl -c f -n 4 -- -p 0x3 --config="(0,0,0),(0,1,2),(1,0,1),(1,1,3)" --rule_ipv4="./rule_ipv4.db" -- rule_ipv6="./rule_ipv6.db" --scalar + ./build/l3fwd-acl -l 1,2 -n 4 -- -p 0x3 --config="(0,0,1),(1,0,2)" --rule_ipv4="./rule_ipv4.db" -- rule_ipv6="./rule_ipv6.db" --scalar In this command: -* The -c option enables cores 0, 1, 2, 3 +* The -l option enables cores 1, 2 * The -p option enables ports 0 and 1 -* The --config option enables two queues on each port and maps each (port,queue) pair to a specific core. - Logic to enable multiple RX queues using RSS and to allocate memory from the correct NUMA nodes is included in the application - and is done transparently. +* The --config option enables one queue on each port and maps each (port,queue) pair to a specific core. The following table shows the mapping in this example: - +----------+------------+-----------+------------------------------------------------+ - | **Port** | **Queue** | **lcore** | **Description** | - | | | | | - +==========+============+===========+================================================+ - | 0 | 0 | 0 | Map queue 0 from port 0 to lcore 0. | - | | | | | - +----------+------------+-----------+------------------------------------------------+ - | 0 | 1 | 2 | Map queue 1 from port 0 to lcore 2. | - | | | | | - +----------+------------+-----------+------------------------------------------------+ - | 1 | 0 | 1 | Map queue 0 from port 1 to lcore 1. | - | | | | | - +----------+------------+-----------+------------------------------------------------+ - | 1 | 1 | 3 | Map queue 1 from port 1 to lcore 3. | - | | | | | - +----------+------------+-----------+------------------------------------------------+ + +----------+------------+-----------+-------------------------------------+ + | **Port** | **Queue** | **lcore** | **Description** | + | | | | | + +==========+============+===========+=====================================+ + | 0 | 0 | 1 | Map queue 0 from port 0 to lcore 1. | + | | | | | + +----------+------------+-----------+-------------------------------------+ + | 1 | 0 | 2 | Map queue 0 from port 1 to lcore 2. | + | | | | | + +----------+------------+-----------+-------------------------------------+ * The --rule_ipv4 option specifies the reading of IPv4 rules sets from the ./ rule_ipv4.db file. diff --git a/dpdk/doc/guides/sample_app_ug/l3_forward_power_man.rst b/dpdk/doc/guides/sample_app_ug/l3_forward_power_man.rst index ea9c404d..20fcc3c6 100644 --- a/dpdk/doc/guides/sample_app_ug/l3_forward_power_man.rst +++ b/dpdk/doc/guides/sample_app_ug/l3_forward_power_man.rst @@ -104,28 +104,9 @@ instead of always running to the C0 state waiting for packets. Compiling the Application ------------------------- -To compile the application: +To compile the sample application see :doc:`compiling`. -#. Go to the sample application directory: - - .. code-block:: console - - export RTE_SDK=/path/to/rte_sdk - cd ${RTE_SDK}/examples/l3fwd-power - -#. Set the target (a default target is used if not specified). For example: - - .. code-block:: console - - export RTE_TARGET=x86_64-native-linuxapp-gcc - - See the *DPDK Getting Started Guide* for possible RTE_TARGET values. - -#. Build the application: - - .. code-block:: console - - make +The application is located in the ``l3fwd-power`` sub-directory. Running the Application ----------------------- @@ -181,11 +162,11 @@ responsible for checking if it needs to scale down frequency at run time by chec struct lcore_conf *qconf; int ret; unsigned nb_ports; - uint16_t queueid; + uint16_t queueid, portid; unsigned lcore_id; uint64_t hz; uint32_t n_tx_queue, nb_lcores; - uint8_t portid, nb_rx_queue, queue, socketid; + uint8_t nb_rx_queue, queue, socketid; // ... diff --git a/dpdk/doc/guides/sample_app_ug/l3_forward_virtual.rst b/dpdk/doc/guides/sample_app_ug/l3_forward_virtual.rst index fa04722e..95e89651 100644 --- a/dpdk/doc/guides/sample_app_ug/l3_forward_virtual.rst +++ b/dpdk/doc/guides/sample_app_ug/l3_forward_virtual.rst @@ -64,34 +64,9 @@ The set of LPM rules used by the application is statically configured and loaded Compiling the Application ------------------------- -To compile the application: +To compile the sample application see :doc:`compiling`. -#. Go to the sample application directory: - - .. code-block:: console - - export RTE_SDK=/path/to/rte_sdk - cd ${RTE_SDK}/examples/l3fwd-vf - -#. Set the target (a default target is used if not specified). For example: - - .. code-block:: console - - export RTE_TARGET=x86_64-native-linuxapp-gcc - - See the *DPDK Getting Started Guide* for possible RTE_TARGET values. - -#. Build the application: - - .. code-block:: console - - make - -.. note:: - - The compiled application is written to the build subdirectory. - To have the application written to a different location, - the O=/path/to/build/directory option may be specified in the make command. +The application is located in the ``l3fwd-vf`` sub-directory. Running the Application ----------------------- @@ -110,40 +85,33 @@ where, * --no-numa: optional, disables numa awareness -For example, consider a dual processor socket platform where cores 0,2,4,6, 8, and 10 appear on socket 0, -while cores 1,3,5,7,9, and 11 appear on socket 1. -Let's say that the programmer wants to use memory from both NUMA nodes, -the platform has only two ports and the programmer wants to use one core from each processor socket to do the packet processing -since only one Rx/Tx queue pair can be used in virtualization mode. +For example, consider a dual processor socket platform with 8 physical cores, where cores 0-7 and 16-23 appear on socket 0, +while cores 8-15 and 24-31 appear on socket 1. -To enable L3 forwarding between two ports, using one core from each processor, -while also taking advantage of local memory accesses by optimizing around NUMA, -the programmer can pin to the appropriate cores and allocate memory from the appropriate NUMA node. -This is achieved using the following command: +To enable L3 forwarding between two ports, assuming that both ports are in the same socket, using two cores, cores 1 and 2, +(which are in the same socket too), use the following command: .. code-block:: console - ./build/l3fwd-vf -c 0x03 -n 3 -- -p 0x3 --config="(0,0,0),(1,0,1)" + ./build/l3fwd-vf -l 1,2 -n 4 -- -p 0x3 --config="(0,0,1),(1,0,2)" In this command: -* The -c option enables cores 0 and 1 +* The -l option enables cores 1 and 2 * The -p option enables ports 0 and 1 * The --config option enables one queue on each port and maps each (port,queue) pair to a specific core. - Logic to enable multiple RX queues using RSS and to allocate memory from the correct NUMA nodes - is included in the application and is done transparently. The following table shows the mapping in this example: +----------+-----------+-----------+------------------------------------+ | **Port** | **Queue** | **lcore** | **Description** | | | | | | +==========+===========+===========+====================================+ - | 0 | 0 | 0 | Map queue 0 from port 0 to lcore 0 | + | 0 | 0 | 1 | Map queue 0 from port 0 to lcore 1 | | | | | | +----------+-----------+-----------+------------------------------------+ - | 1 | 1 | 1 | Map queue 0 from port 1 to lcore 1 | + | 1 | 0 | 2 | Map queue 0 from port 1 to lcore 2 | | | | | | +----------+-----------+-----------+------------------------------------+ diff --git a/dpdk/doc/guides/sample_app_ug/link_status_intr.rst b/dpdk/doc/guides/sample_app_ug/link_status_intr.rst index 779df97d..358524c3 100644 --- a/dpdk/doc/guides/sample_app_ug/link_status_intr.rst +++ b/dpdk/doc/guides/sample_app_ug/link_status_intr.rst @@ -55,32 +55,9 @@ and the behavior of L2 forwarding each time the link status changes. Compiling the Application ------------------------- -#. Go to the example directory: +To compile the sample application see :doc:`compiling`. - .. code-block:: console - - export RTE_SDK=/path/to/rte_sdk - cd ${RTE_SDK}/examples/link_status_interrupt - -#. Set the target (a default target is used if not specified). For example: - - .. code-block:: console - - export RTE_TARGET=x86_64-native-linuxapp-gcc - - See the *DPDK Getting Started Guide* for possible RTE_TARGET values. - -#. Build the application: - - .. code-block:: console - - make - -.. note:: - - The compiled application is written to the build subdirectory. - To have the application written to a different location, - the O=/path/to/build/directory option may be specified on the make command line. +The application is located in the ``link_status_interrupt`` sub-directory. Running the Application ----------------------- @@ -104,7 +81,7 @@ issue the command: .. code-block:: console - $ ./build/link_status_interrupt -c f -n 4-- -q 8 -p ffff + $ ./build/link_status_interrupt -l 0-3 -n 4-- -q 8 -p ffff Refer to the *DPDK Getting Started Guide* for general information on running applications and the Environment Abstraction Layer (EAL) options. @@ -138,7 +115,7 @@ To fully understand this code, it is recommended to study the chapters that rela .. code-block:: c - if (rte_eal_pci_probe() < 0) + if (rte_pci_probe() < 0) rte_exit(EXIT_FAILURE, "Cannot probe PCI\n"); nb_ports = rte_eth_dev_count(); @@ -171,7 +148,7 @@ To fully understand this code, it is recommended to study the chapters that rela Observe that: -* rte_eal_pci_probe() parses the devices on the PCI bus and initializes recognized devices. +* rte_pci_probe() parses the devices on the PCI bus and initializes recognized devices. The next step is to configure the RX and TX queues. For each port, there is only one RX queue (only one lcore is able to poll a given port). @@ -219,7 +196,7 @@ An example callback function that has been written as indicated below. .. code-block:: c static void - lsi_event_callback(uint8_t port_id, enum rte_eth_event_type type, void *param) + lsi_event_callback(uint16_t port_id, enum rte_eth_event_type type, void *param) { struct rte_eth_link link; @@ -405,7 +382,7 @@ If the table is full, the whole packets table is transmitted using the lsi_send_ /* Send the packet on an output interface */ static int - lsi_send_packet(struct rte_mbuf *m, uint8_t port) + lsi_send_packet(struct rte_mbuf *m, uint16_t port) { unsigned lcore_id, len; struct lcore_queue_conf *qconf; diff --git a/dpdk/doc/guides/sample_app_ug/load_balancer.rst b/dpdk/doc/guides/sample_app_ug/load_balancer.rst index fdd8cbd3..18b29b24 100644 --- a/dpdk/doc/guides/sample_app_ug/load_balancer.rst +++ b/dpdk/doc/guides/sample_app_ug/load_balancer.rst @@ -86,24 +86,9 @@ The routing logic is LPM based, with all the worker threads sharing the same LPM Compiling the Application ------------------------- -The sequence of steps used to build the application is: +To compile the sample application see :doc:`compiling`. -#. Export the required environment variables: - - .. code-block:: console - - export RTE_SDK= - export RTE_TARGET=x86_64-native-linuxapp-gcc - -#. Build the application executable file: - - .. code-block:: console - - cd ${RTE_SDK}/examples/load_balancer - make - - For more details on how to build the DPDK libraries and sample applications, - please refer to the *DPDK Getting Started Guide.* +The application is located in the ``load_balancer`` sub-directory. Running the Application ----------------------- @@ -178,7 +163,7 @@ Example: .. code-block:: console - ./load_balancer -c 0xf8 -n 4 -- --rx "(0,0,3),(1,0,3)" --tx "(0,3),(1,3)" --w "4,5,6,7" --lpm "1.0.0.0/24=>0; 1.0.1.0/24=>1;" --pos-lb 29 + ./load_balancer -l 3-7 -n 4 -- --rx "(0,0,3),(1,0,3)" --tx "(0,3),(1,3)" --w "4,5,6,7" --lpm "1.0.0.0/24=>0; 1.0.1.0/24=>1;" --pos-lb 29 There is a single I/O lcore (lcore 3) that handles RX and TX for two NIC ports (ports 0 and 1) that handles packets to/from four worker lcores (lcores 4, 5, 6 and 7) that diff --git a/dpdk/doc/guides/sample_app_ug/multi_process.rst b/dpdk/doc/guides/sample_app_ug/multi_process.rst index 35714905..95a32432 100644 --- a/dpdk/doc/guides/sample_app_ug/multi_process.rst +++ b/dpdk/doc/guides/sample_app_ug/multi_process.rst @@ -40,31 +40,13 @@ Example Applications Building the Sample Applications ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - The multi-process example applications are built in the same way as other sample applications, and as documented in the *DPDK Getting Started Guide*. -To build all the example applications: -#. Set RTE_SDK and go to the example directory: - .. code-block:: console +To compile the sample application see :doc:`compiling`. - export RTE_SDK=/path/to/rte_sdk - cd ${RTE_SDK}/examples/multi_process - -#. Set the target (a default target will be used if not specified). For example: - - .. code-block:: console - - export RTE_TARGET=x86_64-native-linuxapp-gcc - - See the *DPDK Getting Started Guide* for possible RTE_TARGET values. - -#. Build the applications: - - .. code-block:: console - - make +The applications are located in the ``multi_process`` sub-directory. .. note:: @@ -82,11 +64,11 @@ Running the Application ^^^^^^^^^^^^^^^^^^^^^^^ To run the application, start one copy of the simple_mp binary in one terminal, -passing at least two cores in the coremask, as follows: +passing at least two cores in the coremask/corelist, as follows: .. code-block:: console - ./build/simple_mp -c 3 -n 4 --proc-type=primary + ./build/simple_mp -l 0-1 -n 4 --proc-type=primary For the first DPDK process run, the proc-type flag can be omitted or set to auto, since all DPDK processes will default to being a primary instance, @@ -95,7 +77,7 @@ The process should start successfully and display a command prompt as follows: .. code-block:: console - $ ./build/simple_mp -c 3 -n 4 --proc-type=primary + $ ./build/simple_mp -l 0-1 -n 4 --proc-type=primary EAL: coremask set to 3 EAL: Detected lcore 0 on socket 0 EAL: Detected lcore 1 on socket 0 @@ -119,11 +101,11 @@ The process should start successfully and display a command prompt as follows: simple_mp > To run the secondary process to communicate with the primary process, -again run the same binary setting at least two cores in the coremask: +again run the same binary setting at least two cores in the coremask/corelist: .. code-block:: console - ./build/simple_mp -c C -n 4 --proc-type=secondary + ./build/simple_mp -l 2-3 -n 4 --proc-type=secondary When running a secondary process such as that shown above, the proc-type parameter can again be specified as auto. However, omitting the parameter altogether will cause the process to try and start as a primary rather than secondary process. @@ -229,10 +211,10 @@ the following commands can be used (assuming run as root): .. code-block:: console - # ./build/symmetric_mp -c 2 -n 4 --proc-type=auto -- -p 3 --num-procs=4 --proc-id=0 - # ./build/symmetric_mp -c 4 -n 4 --proc-type=auto -- -p 3 --num-procs=4 --proc-id=1 - # ./build/symmetric_mp -c 8 -n 4 --proc-type=auto -- -p 3 --num-procs=4 --proc-id=2 - # ./build/symmetric_mp -c 10 -n 4 --proc-type=auto -- -p 3 --num-procs=4 --proc-id=3 + # ./build/symmetric_mp -l 1 -n 4 --proc-type=auto -- -p 3 --num-procs=4 --proc-id=0 + # ./build/symmetric_mp -l 2 -n 4 --proc-type=auto -- -p 3 --num-procs=4 --proc-id=1 + # ./build/symmetric_mp -l 3 -n 4 --proc-type=auto -- -p 3 --num-procs=4 --proc-id=2 + # ./build/symmetric_mp -l 4 -n 4 --proc-type=auto -- -p 3 --num-procs=4 --proc-id=3 .. note:: @@ -254,7 +236,7 @@ How the Application Works ^^^^^^^^^^^^^^^^^^^^^^^^^ The initialization calls in both the primary and secondary instances are the same for the most part, -calling the rte_eal_init(), 1 G and 10 G driver initialization and then rte_eal_pci_probe() functions. +calling the rte_eal_init(), 1 G and 10 G driver initialization and then rte_pci_probe() functions. Thereafter, the initialization done depends on whether the process is configured as a primary or secondary instance. In the primary instance, a memory pool is created for the packet mbufs and the network ports to be used are initialized - @@ -318,8 +300,8 @@ In addition to the EAL parameters, the application- specific parameters are: .. note:: - In the server process, a single thread, the master thread, that is, the lowest numbered lcore in the coremask, performs all packet I/O. - If a coremask is specified with more than a single lcore bit set in it, + In the server process, a single thread, the master thread, that is, the lowest numbered lcore in the coremask/corelist, performs all packet I/O. + If a coremask/corelist is specified with more than a single lcore bit set in it, an additional lcore will be used for a thread to periodically print packet count statistics. Since the server application stores configuration data in shared memory, including the network ports to be used, @@ -329,9 +311,9 @@ the following commands could be used: .. code-block:: console - # ./mp_server/build/mp_server -c 6 -n 4 -- -p 3 -n 2 - # ./mp_client/build/mp_client -c 8 -n 4 --proc-type=auto -- -n 0 - # ./mp_client/build/mp_client -c 10 -n 4 --proc-type=auto -- -n 1 + # ./mp_server/build/mp_server -l 1-2 -n 4 -- -p 3 -n 2 + # ./mp_client/build/mp_client -l 3 -n 4 --proc-type=auto -- -n 0 + # ./mp_client/build/mp_client -l 4 -n 4 --proc-type=auto -- -n 1 .. note:: @@ -524,7 +506,7 @@ The command is as follows: .. code-block:: console - #./build/l2fwd_fork -c 1c -n 4 -- -p 3 -f + #./build/l2fwd_fork -l 2-4 -n 4 -- -p 3 -f This example provides another -f option to specify the use of floating process. If not specified, the example will use a pinned process to perform the L2 forwarding task. diff --git a/dpdk/doc/guides/sample_app_ug/netmap_compatibility.rst b/dpdk/doc/guides/sample_app_ug/netmap_compatibility.rst index 41f05186..955f86b9 100644 --- a/dpdk/doc/guides/sample_app_ug/netmap_compatibility.rst +++ b/dpdk/doc/guides/sample_app_ug/netmap_compatibility.rst @@ -104,7 +104,7 @@ Porting Netmap applications typically involves two major steps: Since the ``compat_netmap`` functions have the same signature as the usual libc calls, the change is trivial in most cases. -The usual DPDK initialization code involving ``rte_eal_init()`` and ``rte_eal_pci_probe()`` +The usual DPDK initialization code involving ``rte_eal_init()`` and ``rte_pci_probe()`` has to be added to the Netmap application in the same way it is used in all other DPDK sample applications. Please refer to the *DPDK Programmer's Guide* and example source code for details about initialization. @@ -121,29 +121,12 @@ The bridge application is an example largely based on the bridge example shipped It shows how a minimal Netmap application with minimal and straightforward source code changes can be run on top of the DPDK. Please refer to ``$RTE_SDK/examples/netmap_compat/bridge/bridge.c`` for an example of a ported application. -Compiling the "bridge" Sample Application ------------------------------------------ +Compiling the Application +------------------------- -#. Go to the example directory: +To compile the sample application see :doc:`compiling`. - .. code-block:: console - - export RTE_SDK=/path/to/rte_sdk - cd ${RTE_SDK}/examples/netmap_compat - -#. Set the target (a default target is used if not specified). For example: - - .. code-block:: console - - export RTE_TARGET=x86_64-native-linuxapp-gcc - - See the *DPDK Getting Started Guide for Linux* for possible ``RTE_TARGET`` values. - -#. Build the application: - - .. code-block:: console - - make +The application is located in the ``netmap_compat`` sub-directory. Running the "bridge" Sample Application --------------------------------------- diff --git a/dpdk/doc/guides/sample_app_ug/packet_ordering.rst b/dpdk/doc/guides/sample_app_ug/packet_ordering.rst index ef851500..098cc89f 100644 --- a/dpdk/doc/guides/sample_app_ug/packet_ordering.rst +++ b/dpdk/doc/guides/sample_app_ug/packet_ordering.rst @@ -51,28 +51,11 @@ The application uses at least three CPU cores: from the reorder buffer and sends them to the NIC ports for transmission. Compiling the Application --------------------------- +------------------------- -#. Go to the example directory: +To compile the sample application see :doc:`compiling`. - .. code-block:: console - - export RTE_SDK=/path/to/rte_sdk - cd ${RTE_SDK}/examples/helloworld - -#. Set the target (a default target is used if not specified). For example: - - .. code-block:: console - - export RTE_TARGET=x86_64-native-linuxapp-gcc - - See the *DPDK Getting Started* Guide for possible RTE_TARGET values. - -#. Build the application: - - .. code-block:: console - - make +The application is located in the ``packet_ordering`` sub-directory. Running the Application ----------------------- diff --git a/dpdk/doc/guides/sample_app_ug/pdump.rst b/dpdk/doc/guides/sample_app_ug/pdump.rst deleted file mode 100644 index ac0e7c96..00000000 --- a/dpdk/doc/guides/sample_app_ug/pdump.rst +++ /dev/null @@ -1,144 +0,0 @@ - -.. BSD LICENSE - Copyright(c) 2016 Intel Corporation. All rights reserved. - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in - the documentation and/or other materials provided with the - distribution. - * Neither the name of Intel Corporation nor the names of its - contributors may be used to endorse or promote products derived - from this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -dpdk-pdump Application -====================== - -The ``dpdk-pdump`` tool is a Data Plane Development Kit (DPDK) tool that runs as -a DPDK secondary process and is capable of enabling packet capture on dpdk ports. - - .. Note:: - - * The ``dpdk-pdump`` tool depends on libpcap based PMD which is disabled - by default in the build configuration files, - owing to an external dependency on the libpcap development files - which must be installed on the board. - Once the libpcap development files are installed, the libpcap based PMD - can be enabled by setting CONFIG_RTE_LIBRTE_PMD_PCAP=y and recompiling the DPDK. - - -Running the Application ------------------------ - -The tool has a number of command line options: - -.. code-block:: console - - ./build/app/dpdk-pdump -- - --pdump '(port= | device_id=), - (queue=), - (rx-dev= | - tx-dev=), - [ring-size=], - [mbuf-size=], - [total-num-mbufs=]' - [--server-socket-path=] - [--client-socket-path=] - -The ``--pdump`` command line option is mandatory and it takes various sub arguments which are described in -below section. - - .. Note:: - - * Parameters inside the parentheses represents mandatory parameters. - - * Parameters inside the square brackets represents optional parameters. - - * Multiple instances of ``--pdump`` can be passed to capture packets on different port and queue combinations. - -The ``--server-socket-path`` command line option is optional. This represents the server socket directory. -If no value is passed default values are used i.e. ``/var/run/.dpdk/`` for root users and ``~/.dpdk/`` -for non root users. - -The ``--client-socket-path`` command line option is optional. This represents the client socket directory. -If no value is passed default values are used i.e. ``/var/run/.dpdk/`` for root users and ``~/.dpdk/`` -for non root users. - - -The ``--pdump`` parameters -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -``port``: -Port id of the eth device on which packets should be captured. - -``device_id``: -PCI address (or) name of the eth device on which packets should be captured. - - .. Note:: - - * As of now the ``dpdk-pdump`` tool cannot capture the packets of virtual devices - in the primary process due to a bug in the ethdev library. Due to this bug, in a multi process context, - when the primary and secondary have different ports set, then the secondary process - (here the ``dpdk-pdump`` tool) overwrites the ``rte_eth_devices[]`` entries of the primary process. - -``queue``: -Queue id of the eth device on which packets should be captured. The user can pass a queue value of ``*`` to enable -packet capture on all queues of the eth device. - -``rx-dev``: -Can be either a pcap file name or any Linux iface. - -``tx-dev``: -Can be either a pcap file name or any Linux iface. - - .. Note:: - - * To receive ingress packets only, ``rx-dev`` should be passed. - - * To receive egress packets only, ``tx-dev`` should be passed. - - * To receive ingress and egress packets separately ``rx-dev`` and ``tx-dev`` - should both be passed with the different file names or the Linux iface names. - - * To receive ingress and egress packets separately ``rx-dev`` and ``tx-dev`` - should both be passed with the same file names or the the Linux iface names. - -``ring-size``: -Size of the ring. This value is used internally for ring creation. The ring will be used to enqueue the packets from -the primary application to the secondary. This is an optional parameter with default size 16384. - -``mbuf-size``: -Size of the mbuf data. This is used internally for mempool creation. Ideally this value must be same as -the primary application's mempool's mbuf data size which is used for packet RX. This is an optional parameter with -default size 2176. - -``total-num-mbufs``: -Total number mbufs in mempool. This is used internally for mempool creation. This is an optional parameter with default -value 65535. - - -Example -------- - -.. code-block:: console - - $ sudo ./build/app/dpdk-pdump -- --pdump 'port=0,queue=*,rx-dev=/tmp/rx.pcap' diff --git a/dpdk/doc/guides/sample_app_ug/performance_thread.rst b/dpdk/doc/guides/sample_app_ug/performance_thread.rst index d7d9b084..57391caf 100644 --- a/dpdk/doc/guides/sample_app_ug/performance_thread.rst +++ b/dpdk/doc/guides/sample_app_ug/performance_thread.rst @@ -73,28 +73,10 @@ invalid. Compiling the Application ------------------------- -The application is located in the sample application folder in the -``performance-thread`` folder. -#. Go to the example applications folder - - .. code-block:: console - - export RTE_SDK=/path/to/rte_sdk - cd ${RTE_SDK}/examples/performance-thread/l3fwd-thread - -#. Set the target (a default target is used if not specified). For example: - - .. code-block:: console - - export RTE_TARGET=x86_64-native-linuxapp-gcc - - See the *DPDK Linux Getting Started Guide* for possible RTE_TARGET values. - -#. Build the application: - - make +To compile the sample application see :doc:`compiling`. +The application is located in the `performance-thread/l3fwd-thread` sub-directory. Running the Application ----------------------- @@ -107,6 +89,7 @@ The application has a number of command line options:: --tx(lcore,thread)[,(lcore,thread)] [--enable-jumbo] [--max-pkt-len PKTLEN]] [--no-numa] [--hash-entry-num] [--ipv6] [--no-lthreads] [--stat-lcore lcore] + [--parse-ptype] Where: @@ -142,6 +125,9 @@ Where: * ``--stat-lcore``: optional, run CPU load stats collector on the specified lcore. +* ``--parse-ptype:`` optional, set to use software to analyze packet type. + Without this option, hardware will check the packet type. + The parameters of the ``--rx`` and ``--tx`` options are: * ``--rx`` parameters @@ -183,14 +169,14 @@ in ``--rx/--tx`` are used to affinitize threads to the selected scheduler. For example, the following places every l-thread on different lcores:: - l3fwd-thread -c ff -n 2 -- -P -p 3 \ + l3fwd-thread -l 0-7 -n 2 -- -P -p 3 \ --rx="(0,0,0,0)(1,0,1,1)" \ --tx="(2,0)(3,1)" The following places RX l-threads on lcore 0 and TX l-threads on lcore 1 and 2 and so on:: - l3fwd-thread -c ff -n 2 -- -P -p 3 \ + l3fwd-thread -l 0-7 -n 2 -- -P -p 3 \ --rx="(0,0,0,0)(1,0,0,1)" \ --tx="(1,0)(2,1)" @@ -206,7 +192,7 @@ place every RX and TX thread on different lcores. For example, the following places every EAL thread on different lcores:: - l3fwd-thread -c ff -n 2 -- -P -p 3 \ + l3fwd-thread -l 0-7 -n 2 -- -P -p 3 \ --rx="(0,0,0,0)(1,0,1,1)" \ --tx="(2,0)(3,1)" \ --no-lthreads @@ -218,7 +204,7 @@ parameter is used. The following places RX EAL threads on lcore 0 and TX EAL threads on lcore 1 and 2 and so on:: - l3fwd-thread -c ff -n 2 --lcores="(0,1)@0,(2,3)@1" -- -P -p 3 \ + l3fwd-thread -l 0-7 -n 2 --lcores="(0,1)@0,(2,3)@1" -- -P -p 3 \ --rx="(0,0,0,0)(1,0,1,1)" \ --tx="(2,0)(3,1)" \ --no-lthreads @@ -232,13 +218,13 @@ and its corresponding EAL threads command line can be realized as follows: a) Start every thread on different scheduler (1:1):: - l3fwd-thread -c ff -n 2 -- -P -p 3 \ + l3fwd-thread -l 0-7 -n 2 -- -P -p 3 \ --rx="(0,0,0,0)(1,0,1,1)" \ --tx="(2,0)(3,1)" EAL thread equivalent:: - l3fwd-thread -c ff -n 2 -- -P -p 3 \ + l3fwd-thread -l 0-7 -n 2 -- -P -p 3 \ --rx="(0,0,0,0)(1,0,1,1)" \ --tx="(2,0)(3,1)" \ --no-lthreads @@ -247,13 +233,13 @@ b) Start all threads on one core (N:1). Start 4 L-threads on lcore 0:: - l3fwd-thread -c ff -n 2 -- -P -p 3 \ + l3fwd-thread -l 0-7 -n 2 -- -P -p 3 \ --rx="(0,0,0,0)(1,0,0,1)" \ --tx="(0,0)(0,1)" Start 4 EAL threads on cpu-set 0:: - l3fwd-thread -c ff -n 2 --lcores="(0-3)@0" -- -P -p 3 \ + l3fwd-thread -l 0-7 -n 2 --lcores="(0-3)@0" -- -P -p 3 \ --rx="(0,0,0,0)(1,0,0,1)" \ --tx="(2,0)(3,1)" \ --no-lthreads @@ -262,14 +248,14 @@ c) Start threads on different cores (N:M). Start 2 L-threads for RX on lcore 0, and 2 L-threads for TX on lcore 1:: - l3fwd-thread -c ff -n 2 -- -P -p 3 \ + l3fwd-thread -l 0-7 -n 2 -- -P -p 3 \ --rx="(0,0,0,0)(1,0,0,1)" \ --tx="(1,0)(1,1)" Start 2 EAL threads for RX on cpu-set 0, and 2 EAL threads for TX on cpu-set 1:: - l3fwd-thread -c ff -n 2 --lcores="(0-1)@0,(2-3)@1" -- -P -p 3 \ + l3fwd-thread -l 0-7 -n 2 --lcores="(0-1)@0,(2-3)@1" -- -P -p 3 \ --rx="(0,0,0,0)(1,0,1,1)" \ --tx="(2,0)(3,1)" \ --no-lthreads diff --git a/dpdk/doc/guides/sample_app_ug/proc_info.rst b/dpdk/doc/guides/sample_app_ug/proc_info.rst deleted file mode 100644 index 73f21958..00000000 --- a/dpdk/doc/guides/sample_app_ug/proc_info.rst +++ /dev/null @@ -1,71 +0,0 @@ - -.. BSD LICENSE - Copyright(c) 2015 Intel Corporation. All rights reserved. - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in - the documentation and/or other materials provided with the - distribution. - * Neither the name of Intel Corporation nor the names of its - contributors may be used to endorse or promote products derived - from this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -dpdk-procinfo Application -========================= - -The dpdk-procinfo application is a Data Plane Development Kit (DPDK) application -that runs as a DPDK secondary process and is capable of retrieving port -statistics, resetting port statistics and printing DPDK memory information. -This application extends the original functionality that was supported by -dump_cfg. - -Running the Application ------------------------ -The application has a number of command line options: - -.. code-block:: console - - ./$(RTE_TARGET)/app/dpdk-procinfo -- -m | [-p PORTMASK] [--stats | --xstats | - --stats-reset | --xstats-reset] - -Parameters -~~~~~~~~~~ -**-p PORTMASK**: Hexadecimal bitmask of ports to configure. - -**--stats** -The stats parameter controls the printing of generic port statistics. If no -port mask is specified stats are printed for all DPDK ports. - -**--xstats** -The stats parameter controls the printing of extended port statistics. If no -port mask is specified xstats are printed for all DPDK ports. - -**--stats-reset** -The stats-reset parameter controls the resetting of generic port statistics. If -no port mask is specified, the generic stats are reset for all DPDK ports. - -**--xstats-reset** -The xstats-reset parameter controls the resetting of extended port statistics. -If no port mask is specified xstats are reset for all DPDK ports. - -**-m**: Print DPDK memory information. diff --git a/dpdk/doc/guides/sample_app_ug/ptpclient.rst b/dpdk/doc/guides/sample_app_ug/ptpclient.rst index 6e425b79..9cbb6c2a 100644 --- a/dpdk/doc/guides/sample_app_ug/ptpclient.rst +++ b/dpdk/doc/guides/sample_app_ug/ptpclient.rst @@ -78,39 +78,20 @@ The adjustment for slave can be represented as: If the command line parameter ``-T 1`` is used the application also synchronizes the PTP PHC clock with the Linux kernel clock. - Compiling the Application ------------------------- -To compile the application, export the path to the DPDK source tree and edit -the ``config/common_linuxapp`` configuration file to enable IEEE1588: +To compile the sample application see :doc:`compiling`. -.. code-block:: console +The application is located in the ``ptpclient`` sub-directory. - export RTE_SDK=/path/to/rte_sdk +.. note:: + To compile the application edit the ``config/common_linuxapp`` configuration file to enable IEEE1588 + and then recompile DPDK: - # Edit common_linuxapp and set the following options: - CONFIG_RTE_LIBRTE_IEEE1588=y - -Set the target, for example: - -.. code-block:: console - - export RTE_TARGET=x86_64-native-linuxapp-gcc - -See the *DPDK Getting Started* Guide for possible ``RTE_TARGET`` values. - -Build the application as follows: - -.. code-block:: console - - # Recompile DPDK. - make install T=$RTE_TARGET - - # Compile the application. - cd ${RTE_SDK}/examples/ptpclient - make + .. code-block:: console + CONFIG_RTE_LIBRTE_IEEE1588=y Running the Application ----------------------- @@ -119,7 +100,7 @@ To run the example in a ``linuxapp`` environment: .. code-block:: console - ./build/ptpclient -c 2 -n 4 -- -p 0x1 -T 0 + ./build/ptpclient -l 1 -n 4 -- -p 0x1 -T 0 Refer to *DPDK Getting Started Guide* for general information on running applications and the Environment Abstraction Layer (EAL) options. @@ -171,15 +152,8 @@ used by the application: .. code-block:: c - mbuf_pool = rte_mempool_create("MBUF_POOL", - NUM_MBUFS * nb_ports, - MBUF_SIZE, - MBUF_CACHE_SIZE, - sizeof(struct rte_pktmbuf_pool_private), - rte_pktmbuf_pool_init, NULL, - rte_pktmbuf_init, NULL, - rte_socket_id(), - 0); + mbuf_pool = rte_pktmbuf_pool_create("MBUF_POOL", NUM_MBUFS * nb_ports, + MBUF_CACHE_SIZE, 0, RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id()); Mbufs are the packet buffer structure used by DPDK. They are explained in detail in the "Mbuf Library" section of the *DPDK Programmer's Guide*. @@ -264,7 +238,7 @@ PTP IEEE1588 L2 functionality. .. code-block:: c void - parse_ptp_frames(uint8_t portid, struct rte_mbuf *m) { + parse_ptp_frames(uint16_t portid, struct rte_mbuf *m) { struct ptp_header *ptp_hdr; struct ether_hdr *eth_hdr; uint16_t eth_type; diff --git a/dpdk/doc/guides/sample_app_ug/qos_metering.rst b/dpdk/doc/guides/sample_app_ug/qos_metering.rst index e1a6ac7a..067de9e1 100644 --- a/dpdk/doc/guides/sample_app_ug/qos_metering.rst +++ b/dpdk/doc/guides/sample_app_ug/qos_metering.rst @@ -64,29 +64,9 @@ all the incoming packets are colored as green. Compiling the Application ------------------------- -#. Go to the example directory: +To compile the sample application see :doc:`compiling`. - .. code-block:: console - - export RTE_SDK=/path/to/rte_sdk - cd ${RTE_SDK}/examples/qos_meter - -#. Set the target - (a default target is used if not specified): - - .. note:: - - This application is intended as a linuxapp only. - - .. code-block:: console - - export RTE_TARGET=x86_64-native-linuxapp-gcc - -#. Build the application: - - .. code-block:: console - - make +The application is located in the ``qos_meter`` sub-directory. Running the Application ----------------------- diff --git a/dpdk/doc/guides/sample_app_ug/qos_scheduler.rst b/dpdk/doc/guides/sample_app_ug/qos_scheduler.rst index 7f9e9259..1c38d191 100644 --- a/dpdk/doc/guides/sample_app_ug/qos_scheduler.rst +++ b/dpdk/doc/guides/sample_app_ug/qos_scheduler.rst @@ -58,31 +58,14 @@ The TX thread, if present, reads from the TX ring and write the packets to the T Compiling the Application ------------------------- -To compile the application: +To compile the sample application see :doc:`compiling`. -#. Go to the sample application directory: - - .. code-block:: console - - export RTE_SDK=/path/to/rte_sdk - cd ${RTE_SDK}/examples/qos_sched - -#. Set the target (a default target is used if not specified). For example: +The application is located in the ``qos_sched`` sub-directory. .. note:: This application is intended as a linuxapp only. - .. code-block:: console - - export RTE_TARGET=x86_64-native-linuxapp-gcc - -#. Build the application: - - .. code-block:: console - - make - .. note:: To get statistics on the sample app using the command line interface as described in the next section, @@ -289,7 +272,7 @@ The following is an example command with a single packet flow configuration: .. code-block:: console - ./qos_sched -c a2 -n 4 -- --pfc "3,2,5,7" --cfg ./profile.cfg + ./qos_sched -l 1,5,7 -n 4 -- --pfc "3,2,5,7" --cfg ./profile.cfg This example uses a single packet flow configuration which creates one RX thread on lcore 5 reading from port 3 and a worker thread on lcore 7 writing to port 2. @@ -298,12 +281,12 @@ Another example with 2 packet flow configurations using different ports but shar .. code-block:: console - ./qos_sched -c c6 -n 4 -- --pfc "3,2,2,6,7" --pfc "1,0,2,6,7" --cfg ./profile.cfg + ./qos_sched -l 1,2,6,7 -n 4 -- --pfc "3,2,2,6,7" --pfc "1,0,2,6,7" --cfg ./profile.cfg Note that independent cores for the packet flow configurations for each of the RX, WT and TX thread are also supported, providing flexibility to balance the work. -The EAL coremask is constrained to contain the default mastercore 1 and the RX, WT and TX cores only. +The EAL coremask/corelist is constrained to contain the default mastercore 1 and the RX, WT and TX cores only. Explanation ----------- diff --git a/dpdk/doc/guides/sample_app_ug/quota_watermark.rst b/dpdk/doc/guides/sample_app_ug/quota_watermark.rst index c56683aa..32bd78d7 100644 --- a/dpdk/doc/guides/sample_app_ug/quota_watermark.rst +++ b/dpdk/doc/guides/sample_app_ug/quota_watermark.rst @@ -1,5 +1,5 @@ .. BSD LICENSE - Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + Copyright(c) 2010-2017 Intel Corporation. All rights reserved. All rights reserved. Redistribution and use in source and binary forms, with or without @@ -31,11 +31,13 @@ Quota and Watermark Sample Application ====================================== -The Quota and Watermark sample application is a simple example of packet processing using Data Plane Development Kit (DPDK) that -showcases the use of a quota as the maximum number of packets enqueue/dequeue at a time and low and high watermarks -to signal low and high ring usage respectively. +The Quota and Watermark sample application is a simple example of packet +processing using Data Plane Development Kit (DPDK) that showcases the use +of a quota as the maximum number of packets enqueue/dequeue at a time and +low and high thresholds, or watermarks, to signal low and high ring usage +respectively. -Additionally, it shows how ring watermarks can be used to feedback congestion notifications to data producers by +Additionally, it shows how the thresholds can be used to feedback congestion notifications to data producers by temporarily stopping processing overloaded rings and sending Ethernet flow control frames. This sample application is split in two parts: @@ -64,7 +66,7 @@ each stage of which being connected by rings, as shown in :numref:`figure_pipeli An adjustable quota value controls how many packets are being moved through the pipeline per enqueue and dequeue. -Adjustable watermark values associated with the rings control a back-off mechanism that +Adjustable threshold values associated with the rings control a back-off mechanism that tries to prevent the pipeline from being overloaded by: * Stopping enqueuing on rings for which the usage has crossed the high watermark threshold @@ -86,30 +88,12 @@ as shown in :numref:`figure_ring_pipeline_perf_setup`. Ring-based Processing Pipeline Performance Setup - Compiling the Application ------------------------- -#. Go to the example directory: +To compile the sample application see :doc:`compiling`. - .. code-block:: console - - export RTE_SDK=/path/to/rte_sdk - cd ${RTE_SDK}/examples/quota_watermark - -#. Set the target (a default target is used if not specified). For example: - - .. code-block:: console - - export RTE_TARGET=x86_64-native-linuxapp-gcc - - See the *DPDK Getting Started Guide* for possible RTE_TARGET values. - -#. Build the application: - - .. code-block:: console - - make +The application is located in the ``quota_watermark`` sub-directory. Running the Application ----------------------- @@ -136,7 +120,7 @@ issue the following command: .. code-block:: console - ./qw/build/qw -c f -n 4 -- -p 5 + ./qw/build/qw -l 0-3 -n 4 -- -p 5 Refer to the *DPDK Getting Started Guide* for general information on running applications and the Environment Abstraction Layer (EAL) options. @@ -157,7 +141,7 @@ To run the application in a linuxapp environment on logical core 0, issue the fo .. code-block:: console - ./qwctl/build/qwctl -c 1 -n 4 --proc-type=secondary + ./qwctl/build/qwctl -l 0 -n 4 --proc-type=secondary Refer to the *DPDK Getting Started* Guide for general information on running applications and the Environment Abstraction Layer (EAL) options. @@ -202,9 +186,9 @@ Then, a call to init_dpdk(), defined in init.c, is made to initialize the poll m /* Bind the drivers to usable devices */ - ret = rte_eal_pci_probe(); + ret = rte_pci_probe(); if (ret < 0) - rte_exit(EXIT_FAILURE, "rte_eal_pci_probe(): error %d\n", ret); + rte_exit(EXIT_FAILURE, "rte_pci_probe(): error %d\n", ret); if (rte_eth_dev_count() < 2) rte_exit(EXIT_FAILURE, "Not enough Ethernet port available\n"); @@ -216,25 +200,26 @@ in the *DPDK Getting Started Guide* and the *DPDK API Reference*. Shared Variables Setup ^^^^^^^^^^^^^^^^^^^^^^ -The quota and low_watermark shared variables are put into an rte_memzone using a call to setup_shared_variables(): +The quota and high and low watermark shared variables are put into an rte_memzone using a call to setup_shared_variables(): .. code-block:: c void setup_shared_variables(void) { - const struct rte_memzone *qw_memzone; + const struct rte_memzone *qw_memzone; - qw_memzone = rte_memzone_reserve(QUOTA_WATERMARK_MEMZONE_NAME, 2 * sizeof(int), rte_socket_id(), RTE_MEMZONE_2MB); + qw_memzone = rte_memzone_reserve(QUOTA_WATERMARK_MEMZONE_NAME, + 3 * sizeof(int), rte_socket_id(), 0); + if (qw_memzone == NULL) + rte_exit(EXIT_FAILURE, "%s\n", rte_strerror(rte_errno)); - if (qw_memzone == NULL) - rte_exit(EXIT_FAILURE, "%s\n", rte_strerror(rte_errno)); + quota = qw_memzone->addr; + low_watermark = (unsigned int *) qw_memzone->addr + 1; + high_watermark = (unsigned int *) qw_memzone->addr + 2; + } - quota = qw_memzone->addr; - low_watermark = (unsigned int *) qw_memzone->addr + sizeof(int); - } - -These two variables are initialized to a default value in main() and +These three variables are initialized to a default value in main() and can be changed while qw is running using the qwctl control program. Application Arguments @@ -254,32 +239,24 @@ It contains a set of mbuf objects that are used by the driver and the applicatio .. code-block:: c /* Create a pool of mbuf to store packets */ - - mbuf_pool = rte_mempool_create("mbuf_pool", MBUF_PER_POOL, MBUF_SIZE, 32, sizeof(struct rte_pktmbuf_pool_private), - rte_pktmbuf_pool_init, NULL, rte_pktmbuf_init, NULL, rte_socket_id(), 0); + mbuf_pool = rte_pktmbuf_pool_create("mbuf_pool", MBUF_PER_POOL, 32, 0, + MBUF_DATA_SIZE, rte_socket_id()); if (mbuf_pool == NULL) rte_panic("%s\n", rte_strerror(rte_errno)); The rte_mempool is a generic structure used to handle pools of objects. -In this case, it is necessary to create a pool that will be used by the driver, -which expects to have some reserved space in the mempool structure, sizeof(struct rte_pktmbuf_pool_private) bytes. +In this case, it is necessary to create a pool that will be used by the driver. -The number of allocated pkt mbufs is MBUF_PER_POOL, with a size of MBUF_SIZE each. +The number of allocated pkt mbufs is MBUF_PER_POOL, with a data room size +of MBUF_DATA_SIZE each. A per-lcore cache of 32 mbufs is kept. The memory is allocated in on the master lcore's socket, but it is possible to extend this code to allocate one mbuf pool per socket. -Two callback pointers are also given to the rte_mempool_create() function: - -* The first callback pointer is to rte_pktmbuf_pool_init() and is used to initialize the private data of the mempool, - which is needed by the driver. - This function is provided by the mbuf API, but can be copied and extended by the developer. - -* The second callback pointer given to rte_mempool_create() is the mbuf initializer. - -The default is used, that is, rte_pktmbuf_init(), which is provided in the rte_mbuf library. -If a more complex application wants to extend the rte_pktmbuf structure for its own needs, -a new function derived from rte_pktmbuf_init() can be created. +The rte_pktmbuf_pool_create() function uses the default mbuf pool and mbuf +initializers, respectively rte_pktmbuf_pool_init() and rte_pktmbuf_init(). +An advanced application may want to use the mempool API to create the +mbuf pool with more control. Ports Configuration and Pairing ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -357,27 +334,37 @@ This is done using the following code: /* Process each port round robin style */ for (port_id = 0; port_id < RTE_MAX_ETHPORTS; port_id++) { - if (!is_bit_set(port_id, portmask)) - continue; + if (!is_bit_set(port_id, portmask)) + continue; - ring = rings[lcore_id][port_id]; + ring = rings[lcore_id][port_id]; - if (ring_state[port_id] != RING_READY) { - if (rte_ring_count(ring) > *low_watermark) - continue; - else - ring_state[port_id] = RING_READY; - } + if (ring_state[port_id] != RING_READY) { + if (rte_ring_count(ring) > *low_watermark) + continue; + else + ring_state[port_id] = RING_READY; + } - /* Enqueue received packets on the RX ring */ + /* Enqueue received packets on the RX ring */ + nb_rx_pkts = rte_eth_rx_burst(port_id, 0, pkts, + (uint16_t) *quota); + ret = rte_ring_enqueue_bulk(ring, (void *) pkts, + nb_rx_pkts, &free); + if (RING_SIZE - free > *high_watermark) { + ring_state[port_id] = RING_OVERLOADED; + send_pause_frame(port_id, 1337); + } - nb_rx_pkts = rte_eth_rx_burst(port_id, 0, pkts, *quota); + if (ret == 0) { - ret = rte_ring_enqueue_bulk(ring, (void *) pkts, nb_rx_pkts); - if (ret == -EDQUOT) { - ring_state[port_id] = RING_OVERLOADED; - send_pause_frame(port_id, 1337); - } + /* + * Return mbufs to the pool, + * effectively dropping packets + */ + for (i = 0; i < nb_rx_pkts; i++) + rte_pktmbuf_free(pkts[i]); + } } For each port in the port mask, the corresponding ring's pointer is fetched into ring and that ring's state is checked: @@ -398,30 +385,40 @@ This thread is running on most of the logical cores to create and arbitrarily lo previous_lcore_id = get_previous_lcore_id(lcore_id); for (port_id = 0; port_id < RTE_MAX_ETHPORTS; port_id++) { - if (!is_bit_set(port_id, portmask)) - continue; + if (!is_bit_set(port_id, portmask)) + continue; - tx = rings[lcore_id][port_id]; - rx = rings[previous_lcore_id][port_id]; - if (ring_state[port_id] != RING_READY) { - if (rte_ring_count(tx) > *low_watermark) - continue; - else - ring_state[port_id] = RING_READY; - } + tx = rings[lcore_id][port_id]; + rx = rings[previous_lcore_id][port_id]; - /* Dequeue up to quota mbuf from rx */ + if (ring_state[port_id] != RING_READY) { + if (rte_ring_count(tx) > *low_watermark) + continue; + else + ring_state[port_id] = RING_READY; + } - nb_dq_pkts = rte_ring_dequeue_burst(rx, pkts, *quota); + /* Dequeue up to quota mbuf from rx */ + nb_dq_pkts = rte_ring_dequeue_burst(rx, pkts, + *quota, NULL); + if (unlikely(nb_dq_pkts < 0)) + continue; - if (unlikely(nb_dq_pkts < 0)) - continue; + /* Enqueue them on tx */ + ret = rte_ring_enqueue_bulk(tx, pkts, + nb_dq_pkts, &free); + if (RING_SIZE - free > *high_watermark) + ring_state[port_id] = RING_OVERLOADED; - /* Enqueue them on tx */ + if (ret == 0) { - ret = rte_ring_enqueue_bulk(tx, pkts, nb_dq_pkts); - if (ret == -EDQUOT) - ring_state[port_id] = RING_OVERLOADED; + /* + * Return mbufs to the pool, + * effectively dropping packets + */ + for (i = 0; i < nb_dq_pkts; i++) + rte_pktmbuf_free(pkts[i]); + } } The thread's logic works mostly like receive_stage(), @@ -490,5 +487,6 @@ low_watermark from the rte_memzone previously created by qw. quota = qw_memzone->addr; - low_watermark = (unsigned int *) qw_memzone->addr + sizeof(int); + low_watermark = (unsigned int *) qw_memzone->addr + 1; + high_watermark = (unsigned int *) qw_memzone->addr + 2; } diff --git a/dpdk/doc/guides/sample_app_ug/rxtx_callbacks.rst b/dpdk/doc/guides/sample_app_ug/rxtx_callbacks.rst index 9df57ed5..928c9635 100644 --- a/dpdk/doc/guides/sample_app_ug/rxtx_callbacks.rst +++ b/dpdk/doc/guides/sample_app_ug/rxtx_callbacks.rst @@ -45,23 +45,9 @@ prior to transmission to calculate the elapsed time, in CPU cycles. Compiling the Application ------------------------- -To compile the application export the path to the DPDK source tree and go to -the example directory: +To compile the sample application see :doc:`compiling`. -.. code-block:: console - - export RTE_SDK=/path/to/rte_sdk - - cd ${RTE_SDK}/examples/rxtx_callbacks - - -Set the target, for example: - -.. code-block:: console - - export RTE_TARGET=x86_64-native-linuxapp-gcc - -See the *DPDK Getting Started* Guide for possible ``RTE_TARGET`` values. +The application is located in the ``rxtx_callbacks`` sub-directory. The callbacks feature requires that the ``CONFIG_RTE_ETHDEV_RXTX_CALLBACKS`` setting is on in the ``config/common_`` config file that applies to the @@ -71,13 +57,6 @@ target. This is generally on by default: CONFIG_RTE_ETHDEV_RXTX_CALLBACKS=y -Build the application as follows: - -.. code-block:: console - - make - - Running the Application ----------------------- @@ -85,7 +64,7 @@ To run the example in a ``linuxapp`` environment: .. code-block:: console - ./build/rxtx_callbacks -c 2 -n 4 + ./build/rxtx_callbacks -l 1 -n 4 Refer to *DPDK Getting Started Guide* for general information on running applications and the Environment Abstraction Layer (EAL) options. @@ -124,7 +103,7 @@ comments: .. code-block:: c static inline int - port_init(uint8_t port, struct rte_mempool *mbuf_pool) + port_init(uint16_t port, struct rte_mempool *mbuf_pool) { struct rte_eth_conf port_conf = port_conf_default; const uint16_t rx_rings = 1, tx_rings = 1; @@ -196,7 +175,7 @@ all packets received: .. code-block:: c static uint16_t - add_timestamps(uint8_t port __rte_unused, uint16_t qidx __rte_unused, + add_timestamps(uint16_t port __rte_unused, uint16_t qidx __rte_unused, struct rte_mbuf **pkts, uint16_t nb_pkts, void *_ __rte_unused) { unsigned i; @@ -222,7 +201,7 @@ packets prior to transmission: .. code-block:: c static uint16_t - calc_latency(uint8_t port __rte_unused, uint16_t qidx __rte_unused, + calc_latency(uint16_t port __rte_unused, uint16_t qidx __rte_unused, struct rte_mbuf **pkts, uint16_t nb_pkts, void *_ __rte_unused) { uint64_t cycles = 0; diff --git a/dpdk/doc/guides/sample_app_ug/server_node_efd.rst b/dpdk/doc/guides/sample_app_ug/server_node_efd.rst new file mode 100644 index 00000000..8891f3b3 --- /dev/null +++ b/dpdk/doc/guides/sample_app_ug/server_node_efd.rst @@ -0,0 +1,477 @@ +.. BSD LICENSE + Copyright(c) 2016-2017 Intel Corporation. All rights reserved. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Server-Node EFD Sample Application +================================== + +This sample application demonstrates the use of EFD library as a flow-level +load balancer, for more information about the EFD Library please refer to the +DPDK programmer's guide. + +This sample application is a variant of the +:ref:`client-server sample application ` +where a specific target node is specified for every and each flow +(not in a round-robin fashion as the original load balancing sample application). + +Overview +-------- + +The architecture of the EFD flow-based load balancer sample application is +presented in the following figure. + +.. _figure_efd_sample_app_overview: + +.. figure:: img/server_node_efd.* + + Using EFD as a Flow-Level Load Balancer + +As shown in :numref:`figure_efd_sample_app_overview`, +the sample application consists of a front-end node (server) +using the EFD library to create a load-balancing table for flows, +for each flow a target backend worker node is specified. The EFD table does not +store the flow key (unlike a regular hash table), and hence, it can +individually load-balance millions of flows (number of targets * maximum number +of flows fit in a flow table per target) while still fitting in CPU cache. + +It should be noted that although they are referred to as nodes, the frontend +server and worker nodes are processes running on the same platform. + +Front-end Server +~~~~~~~~~~~~~~~~ + +Upon initializing, the frontend server node (process) creates a flow +distributor table (based on the EFD library) which is populated with flow +information and its intended target node. + +The sample application assigns a specific target node_id (process) for each of +the IP destination addresses as follows: + +.. code-block:: c + + node_id = i % num_nodes; /* Target node id is generated */ + ip_dst = rte_cpu_to_be_32(i); /* Specific ip destination address is + assigned to this target node */ + +then the pair of is inserted into the flow distribution table. + +The main loop of the server process receives a burst of packets, then for +each packet, a flow key (IP destination address) is extracted. The flow +distributor table is looked up and the target node id is returned. Packets are +then enqueued to the specified target node id. + +It should be noted that flow distributor table is not a membership test table. +I.e. if the key has already been inserted the target node id will be correct, +but for new keys the flow distributor table will return a value (which can be +valid). + +Backend Worker Nodes +~~~~~~~~~~~~~~~~~~~~ + +Upon initializing, the worker node (process) creates a flow table (a regular +hash table that stores the key default size 1M flows) which is populated with +only the flow information that is serviced at this node. This flow key is +essential to point out new keys that have not been inserted before. + +The worker node's main loop is simply receiving packets then doing a hash table +lookup. If a match occurs then statistics are updated for flows serviced by +this node. If no match is found in the local hash table then this indicates +that this is a new flow, which is dropped. + + +Compiling the Application +------------------------- + +To compile the sample application see :doc:`compiling`. + +The application is located in the ``server_node_efd`` sub-directory. + +Running the Application +----------------------- + +The application has two binaries to be run: the front-end server +and the back-end node. + +The frontend server (server) has the following command line options:: + + ./server [EAL options] -- -p PORTMASK -n NUM_NODES -f NUM_FLOWS + +Where, + +* ``-p PORTMASK:`` Hexadecimal bitmask of ports to configure +* ``-n NUM_NODES:`` Number of back-end nodes that will be used +* ``-f NUM_FLOWS:`` Number of flows to be added in the EFD table (1 million, by default) + +The back-end node (node) has the following command line options:: + + ./node [EAL options] -- -n NODE_ID + +Where, + +* ``-n NODE_ID:`` Node ID, which cannot be equal or higher than NUM_MODES + + +First, the server app must be launched, with the number of nodes that will be run. +Once it has been started, the node instances can be run, with different NODE_ID. +These instances have to be run as secondary processes, with ``--proc-type=secondary`` +in the EAL options, which will attach to the primary process memory, and therefore, +they can access the queues created by the primary process to distribute packets. + +To successfully run the application, the command line used to start the +application has to be in sync with the traffic flows configured on the traffic +generator side. + +For examples of application command lines and traffic generator flows, please +refer to the DPDK Test Report. For more details on how to set up and run the +sample applications provided with DPDK package, please refer to the +:ref:`DPDK Getting Started Guide for Linux ` and +:ref:`DPDK Getting Started Guide for FreeBSD `. + + +Explanation +----------- + +As described in previous sections, there are two processes in this example. + +The first process, the front-end server, creates and populates the EFD table, +which is used to distribute packets to nodes, which the number of flows +specified in the command line (1 million, by default). + + +.. code-block:: c + + static void + create_efd_table(void) + { + uint8_t socket_id = rte_socket_id(); + + /* create table */ + efd_table = rte_efd_create("flow table", num_flows * 2, sizeof(uint32_t), + 1 << socket_id, socket_id); + + if (efd_table == NULL) + rte_exit(EXIT_FAILURE, "Problem creating the flow table\n"); + } + + static void + populate_efd_table(void) + { + unsigned int i; + int32_t ret; + uint32_t ip_dst; + uint8_t socket_id = rte_socket_id(); + uint64_t node_id; + + /* Add flows in table */ + for (i = 0; i < num_flows; i++) { + node_id = i % num_nodes; + + ip_dst = rte_cpu_to_be_32(i); + ret = rte_efd_update(efd_table, socket_id, + (void *)&ip_dst, (efd_value_t)node_id); + if (ret < 0) + rte_exit(EXIT_FAILURE, "Unable to add entry %u in " + "EFD table\n", i); + } + + printf("EFD table: Adding 0x%x keys\n", num_flows); + } + +After initialization, packets are received from the enabled ports, and the IPv4 +address from the packets is used as a key to look up in the EFD table, +which tells the node where the packet has to be distributed. + +.. code-block:: c + + static void + process_packets(uint32_t port_num __rte_unused, struct rte_mbuf *pkts[], + uint16_t rx_count, unsigned int socket_id) + { + uint16_t i; + uint8_t node; + efd_value_t data[EFD_BURST_MAX]; + const void *key_ptrs[EFD_BURST_MAX]; + + struct ipv4_hdr *ipv4_hdr; + uint32_t ipv4_dst_ip[EFD_BURST_MAX]; + + for (i = 0; i < rx_count; i++) { + /* Handle IPv4 header.*/ + ipv4_hdr = rte_pktmbuf_mtod_offset(pkts[i], struct ipv4_hdr *, + sizeof(struct ether_hdr)); + ipv4_dst_ip[i] = ipv4_hdr->dst_addr; + key_ptrs[i] = (void *)&ipv4_dst_ip[i]; + } + + rte_efd_lookup_bulk(efd_table, socket_id, rx_count, + (const void **) key_ptrs, data); + for (i = 0; i < rx_count; i++) { + node = (uint8_t) ((uintptr_t)data[i]); + + if (node >= num_nodes) { + /* + * Node is out of range, which means that + * flow has not been inserted + */ + flow_dist_stats.drop++; + rte_pktmbuf_free(pkts[i]); + } else { + flow_dist_stats.distributed++; + enqueue_rx_packet(node, pkts[i]); + } + } + + for (i = 0; i < num_nodes; i++) + flush_rx_queue(i); + } + +The burst of packets received is enqueued in temporary buffers (per node), +and enqueued in the shared ring between the server and the node. +After this, a new burst of packets is received and this process is +repeated infinitely. + +.. code-block:: c + + static void + flush_rx_queue(uint16_t node) + { + uint16_t j; + struct node *cl; + + if (cl_rx_buf[node].count == 0) + return; + + cl = &nodes[node]; + if (rte_ring_enqueue_bulk(cl->rx_q, (void **)cl_rx_buf[node].buffer, + cl_rx_buf[node].count, NULL) != cl_rx_buf[node].count){ + for (j = 0; j < cl_rx_buf[node].count; j++) + rte_pktmbuf_free(cl_rx_buf[node].buffer[j]); + cl->stats.rx_drop += cl_rx_buf[node].count; + } else + cl->stats.rx += cl_rx_buf[node].count; + + cl_rx_buf[node].count = 0; + } + +The second process, the back-end node, receives the packets from the shared +ring with the server and send them out, if they belong to the node. + +At initialization, it attaches to the server process memory, to have +access to the shared ring, parameters and statistics. + +.. code-block:: c + + rx_ring = rte_ring_lookup(get_rx_queue_name(node_id)); + if (rx_ring == NULL) + rte_exit(EXIT_FAILURE, "Cannot get RX ring - " + "is server process running?\n"); + + mp = rte_mempool_lookup(PKTMBUF_POOL_NAME); + if (mp == NULL) + rte_exit(EXIT_FAILURE, "Cannot get mempool for mbufs\n"); + + mz = rte_memzone_lookup(MZ_SHARED_INFO); + if (mz == NULL) + rte_exit(EXIT_FAILURE, "Cannot get port info structure\n"); + info = mz->addr; + tx_stats = &(info->tx_stats[node_id]); + filter_stats = &(info->filter_stats[node_id]); + +Then, the hash table that contains the flows that will be handled +by the node is created and populated. + +.. code-block:: c + + static struct rte_hash * + create_hash_table(const struct shared_info *info) + { + uint32_t num_flows_node = info->num_flows / info->num_nodes; + char name[RTE_HASH_NAMESIZE]; + struct rte_hash *h; + + /* create table */ + struct rte_hash_parameters hash_params = { + .entries = num_flows_node * 2, /* table load = 50% */ + .key_len = sizeof(uint32_t), /* Store IPv4 dest IP address */ + .socket_id = rte_socket_id(), + .hash_func_init_val = 0, + }; + + snprintf(name, sizeof(name), "hash_table_%d", node_id); + hash_params.name = name; + h = rte_hash_create(&hash_params); + + if (h == NULL) + rte_exit(EXIT_FAILURE, + "Problem creating the hash table for node %d\n", + node_id); + return h; + } + + static void + populate_hash_table(const struct rte_hash *h, const struct shared_info *info) + { + unsigned int i; + int32_t ret; + uint32_t ip_dst; + uint32_t num_flows_node = 0; + uint64_t target_node; + + /* Add flows in table */ + for (i = 0; i < info->num_flows; i++) { + target_node = i % info->num_nodes; + if (target_node != node_id) + continue; + + ip_dst = rte_cpu_to_be_32(i); + + ret = rte_hash_add_key(h, (void *) &ip_dst); + if (ret < 0) + rte_exit(EXIT_FAILURE, "Unable to add entry %u " + "in hash table\n", i); + else + num_flows_node++; + + } + + printf("Hash table: Adding 0x%x keys\n", num_flows_node); + } + +After initialization, packets are dequeued from the shared ring +(from the server) and, like in the server process, +the IPv4 address from the packets is used as a key to look up in the hash table. +If there is a hit, packet is stored in a buffer, to be eventually transmitted +in one of the enabled ports. If key is not there, packet is dropped, since the +flow is not handled by the node. + +.. code-block:: c + + static inline void + handle_packets(struct rte_hash *h, struct rte_mbuf **bufs, uint16_t num_packets) + { + struct ipv4_hdr *ipv4_hdr; + uint32_t ipv4_dst_ip[PKT_READ_SIZE]; + const void *key_ptrs[PKT_READ_SIZE]; + unsigned int i; + int32_t positions[PKT_READ_SIZE] = {0}; + + for (i = 0; i < num_packets; i++) { + /* Handle IPv4 header.*/ + ipv4_hdr = rte_pktmbuf_mtod_offset(bufs[i], struct ipv4_hdr *, + sizeof(struct ether_hdr)); + ipv4_dst_ip[i] = ipv4_hdr->dst_addr; + key_ptrs[i] = &ipv4_dst_ip[i]; + } + /* Check if packets belongs to any flows handled by this node */ + rte_hash_lookup_bulk(h, key_ptrs, num_packets, positions); + + for (i = 0; i < num_packets; i++) { + if (likely(positions[i] >= 0)) { + filter_stats->passed++; + transmit_packet(bufs[i]); + } else { + filter_stats->drop++; + /* Drop packet, as flow is not handled by this node */ + rte_pktmbuf_free(bufs[i]); + } + } + } + +Finally, note that both processes updates statistics, such as transmitted, received +and dropped packets, which are shown and refreshed by the server app. + +.. code-block:: c + + static void + do_stats_display(void) + { + unsigned int i, j; + const char clr[] = {27, '[', '2', 'J', '\0'}; + const char topLeft[] = {27, '[', '1', ';', '1', 'H', '\0'}; + uint64_t port_tx[RTE_MAX_ETHPORTS], port_tx_drop[RTE_MAX_ETHPORTS]; + uint64_t node_tx[MAX_NODES], node_tx_drop[MAX_NODES]; + + /* to get TX stats, we need to do some summing calculations */ + memset(port_tx, 0, sizeof(port_tx)); + memset(port_tx_drop, 0, sizeof(port_tx_drop)); + memset(node_tx, 0, sizeof(node_tx)); + memset(node_tx_drop, 0, sizeof(node_tx_drop)); + + for (i = 0; i < num_nodes; i++) { + const struct tx_stats *tx = &info->tx_stats[i]; + + for (j = 0; j < info->num_ports; j++) { + const uint64_t tx_val = tx->tx[info->id[j]]; + const uint64_t drop_val = tx->tx_drop[info->id[j]]; + + port_tx[j] += tx_val; + port_tx_drop[j] += drop_val; + node_tx[i] += tx_val; + node_tx_drop[i] += drop_val; + } + } + + /* Clear screen and move to top left */ + printf("%s%s", clr, topLeft); + + printf("PORTS\n"); + printf("-----\n"); + for (i = 0; i < info->num_ports; i++) + printf("Port %u: '%s'\t", (unsigned int)info->id[i], + get_printable_mac_addr(info->id[i])); + printf("\n\n"); + for (i = 0; i < info->num_ports; i++) { + printf("Port %u - rx: %9"PRIu64"\t" + "tx: %9"PRIu64"\n", + (unsigned int)info->id[i], info->rx_stats.rx[i], + port_tx[i]); + } + + printf("\nSERVER\n"); + printf("-----\n"); + printf("distributed: %9"PRIu64", drop: %9"PRIu64"\n", + flow_dist_stats.distributed, flow_dist_stats.drop); + + printf("\nNODES\n"); + printf("-------\n"); + for (i = 0; i < num_nodes; i++) { + const unsigned long long rx = nodes[i].stats.rx; + const unsigned long long rx_drop = nodes[i].stats.rx_drop; + const struct filter_stats *filter = &info->filter_stats[i]; + + printf("Node %2u - rx: %9llu, rx_drop: %9llu\n" + " tx: %9"PRIu64", tx_drop: %9"PRIu64"\n" + " filter_passed: %9"PRIu64", " + "filter_drop: %9"PRIu64"\n", + i, rx, rx_drop, node_tx[i], node_tx_drop[i], + filter->passed, filter->drop); + } + + printf("\n"); + } diff --git a/dpdk/doc/guides/sample_app_ug/service_cores.rst b/dpdk/doc/guides/sample_app_ug/service_cores.rst new file mode 100644 index 00000000..36c1b3c5 --- /dev/null +++ b/dpdk/doc/guides/sample_app_ug/service_cores.rst @@ -0,0 +1,172 @@ +.. BSD LICENSE + Copyright(c) 2017 Intel Corporation. All rights reserved. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Service Cores Sample Application +================================ + +The service cores sample application demonstrates the service cores capabilities +of DPDK. The service cores infrastructure is part of the DPDK EAL, and allows +any DPDK component to register a service. A service is a work item or task, that +requires CPU time to perform its duty. + +This sample application registers 5 dummy services. These 5 services are used +to show how the service_cores API can be used to orchestrate these services to +run on different service lcores. This orchestration is done by calling the +service cores APIs, however the sample application introduces a "profile" +concept to contain the service mapping details. Note that the profile concept +is application specific, and not a part of the service cores API. + + +Compiling the Application +------------------------- + +#. Go to the example directory: + + .. code-block:: console + + export RTE_SDK=/path/to/rte_sdk + cd ${RTE_SDK}/examples/service_cores + +#. Set the target (a default target is used if not specified). For example: + + .. code-block:: console + + export RTE_TARGET=x86_64-native-linuxapp-gcc + + See the *DPDK Getting Started* Guide for possible RTE_TARGET values. + +#. Build the application: + + .. code-block:: console + + make + +Running the Application +----------------------- + +To run the example, just execute the binary. Since the application dynamically +adds service cores in the application code itself, there is no requirement to +pass a service core-mask as an EAL argument at startup time. + +.. code-block:: console + + $ ./build/service_cores + + +Explanation +----------- + +The following sections provide some explanation of code focusing on +registering applications from an applications point of view, and modifying the +service core counts and mappings at runtime. + + +Registering a Service +~~~~~~~~~~~~~~~~~~~~~ + +The following code section shows how to register a service as an application. +Note that the service component header must be included by the application in +order to register services: ``rte_service_component.h``, in addition +to the ordinary service cores header ``rte_service.h`` which provides +the runtime functions to add, remove and remap service cores. + +.. code-block:: c + + struct rte_service_spec service = { + .name = "service_name", + }; + int ret = rte_service_component_register(services, &id); + if (ret) + return -1; + + /* set the service itself to be ready to run. In the case of + * ethdev, eventdev etc PMDs, this will be set when the + * appropriate configure or setup function is called. + */ + rte_service_component_runstate_set(id, 1); + + /* Collect statistics for the service */ + rte_service_set_stats_enable(id, 1); + + /* The application sets the service to running state. Note that this + * function enables the service to run - while the 'component' version + * of this function (as above) marks the service itself as ready */ + ret = rte_service_runstate_set(id, 1); + + +Controlling A Service Core +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This section demonstrates how to add a service core. The ``rte_service.h`` +header file provides the functions for dynamically adding and removing cores. +The APIs to add and remove cores use lcore IDs similar to existing DPDK +functions. + +These are the functions to start a service core, and have it run a service: + +.. code-block:: c + + /* the lcore ID to use as a service core */ + uint32_t service_core_id = 7; + ret = rte_service_lcore_add(service_core_id); + if(ret) + return -1; + + /* service cores are in "stopped" state when added, so start it */ + ret = rte_service_lcore_start(service_core_id); + if(ret) + return -1; + + /* map a service to the service core, causing it to run the service */ + uint32_t service_id; /* ID of a registered service */ + uint32_t enable = 1; /* 1 maps the service, 0 unmaps */ + ret = rte_service_map_lcore_set(service_id, service_core_id, enable); + if(ret) + return -1; + + +Removing A Service Core +~~~~~~~~~~~~~~~~~~~~~~~ + +To remove a service core, the steps are similar to adding but in reverse order. +Note that it is not allowed to remove a service core if the service is running, +and the service-core is the only core running that service (see documentation +for ``rte_service_lcore_stop`` function for details). + + +Conclusion +~~~~~~~~~~ + +The service cores infrastructure provides DPDK with two main features. The first +is to abstract away hardware differences: the service core can CPU cycles to +a software fallback implementation, allowing the application to be abstracted +from the difference in HW / SW availability. The second feature is a flexible +method of registering functions to be run, allowing the running of the +functions to be scaled across multiple CPUs. diff --git a/dpdk/doc/guides/sample_app_ug/skeleton.rst b/dpdk/doc/guides/sample_app_ug/skeleton.rst index e832c132..9c78f494 100644 --- a/dpdk/doc/guides/sample_app_ug/skeleton.rst +++ b/dpdk/doc/guides/sample_app_ug/skeleton.rst @@ -39,33 +39,12 @@ It is intended as a demonstration of the basic components of a DPDK forwarding application. For more detailed implementations see the L2 and L3 forwarding sample applications. - Compiling the Application ------------------------- -To compile the application export the path to the DPDK source tree and go to -the example directory: - -.. code-block:: console - - export RTE_SDK=/path/to/rte_sdk - - cd ${RTE_SDK}/examples/skeleton - -Set the target, for example: - -.. code-block:: console - - export RTE_TARGET=x86_64-native-linuxapp-gcc - -See the *DPDK Getting Started* Guide for possible ``RTE_TARGET`` values. - -Build the application as follows: - -.. code-block:: console - - make +To compile the sample application see :doc:`compiling`. +The application is located in the ``skeleton`` sub-directory. Running the Application ----------------------- @@ -74,7 +53,7 @@ To run the example in a ``linuxapp`` environment: .. code-block:: console - ./build/basicfwd -c 2 -n 4 + ./build/basicfwd -l 1 -n 4 Refer to *DPDK Getting Started Guide* for general information on running applications and the Environment Abstraction Layer (EAL) options. @@ -160,7 +139,7 @@ Forwarding application is shown below: .. code-block:: c static inline int - port_init(uint8_t port, struct rte_mempool *mbuf_pool) + port_init(uint16_t port, struct rte_mempool *mbuf_pool) { struct rte_eth_conf port_conf = port_conf_default; const uint16_t rx_rings = 1, tx_rings = 1; @@ -241,8 +220,8 @@ looks like the following: static __attribute__((noreturn)) void lcore_main(void) { - const uint8_t nb_ports = rte_eth_dev_count(); - uint8_t port; + const uint16_t nb_ports = rte_eth_dev_count(); + uint16_t port; /* * Check that the port is on the same NUMA node as the polling thread diff --git a/dpdk/doc/guides/sample_app_ug/tep_termination.rst b/dpdk/doc/guides/sample_app_ug/tep_termination.rst index c3d1e97c..04bf8a2c 100644 --- a/dpdk/doc/guides/sample_app_ug/tep_termination.rst +++ b/dpdk/doc/guides/sample_app_ug/tep_termination.rst @@ -99,7 +99,8 @@ The sample will support the followings: * TSO offload support for tunneling packet. -The following figure shows the framework of the TEP termination sample application based on vhost-cuse. +The following figure shows the framework of the TEP termination sample +application based on DPDK vhost lib. .. _figure_tep_termination_arch: @@ -118,89 +119,22 @@ The example in this section have been validated with the following distributions * Fedora* 20 -Prerequisites -------------- - -Refer to :ref:`vhost_app_prerequisites`. - Compiling the Sample Code ------------------------- -#. Compile vhost lib: - To enable vhost, turn on vhost library in the configure file config/common_linuxapp. +To enable vhost, turn on vhost library in the configure file +``config/common_linuxapp``. .. code-block:: console CONFIG_RTE_LIBRTE_VHOST=y - vhost user is turned on by default in the configure file config/common_linuxapp. - To enable vhost cuse, disable vhost user. - - .. code-block:: console - - CONFIG_RTE_LIBRTE_VHOST_USER=n - - After vhost is enabled and the implementation is selected, build the vhost library. - -#. Go to the examples directory: - - .. code-block:: console - - export RTE_SDK=/path/to/rte_sdk - cd ${RTE_SDK}/examples/tep_termination - -#. Set the target (a default target is used if not specified). For example: - - .. code-block:: console - - export RTE_TARGET=x86_64-native-linuxapp-gcc - - See the DPDK Getting Started Guide for possible RTE_TARGET values. - -#. Build the application: - - .. code-block:: console - - cd ${RTE_SDK} - make config ${RTE_TARGET} - make install ${RTE_TARGET} - cd ${RTE_SDK}/examples/tep_termination - make - -#. Go to the eventfd_link directory(vhost cuse required): - - .. code-block:: console - - cd ${RTE_SDK}/lib/librte_vhost/eventfd_link - -#. Build the eventfd_link kernel module(vhost cuse required): - - .. code-block:: console - - make +Then following the to compile the sample application shown in +:doc:`compiling`. Running the Sample Code ----------------------- -#. Install the cuse kernel module(vhost cuse required): - - .. code-block:: console - - modprobe cuse - -#. Go to the eventfd_link directory(vhost cuse required): - - .. code-block:: console - - export RTE_SDK=/path/to/rte_sdk - cd ${RTE_SDK}/lib/librte_vhost/eventfd_link - -#. Install the eventfd_link module(vhost cuse required): - - .. code-block:: console - - insmod ./eventfd_link.ko - #. Go to the examples directory: .. code-block:: console @@ -212,7 +146,7 @@ Running the Sample Code .. code-block:: console - user@target:~$ ./build/app/tep_termination -c f -n 4 --huge-dir /mnt/huge -- + user@target:~$ ./build/app/tep_termination -l 0-3 -n 4 --huge-dir /mnt/huge -- -p 0x1 --dev-basename tep-termination --nb-devices 4 --udp-port 4789 --filter-type 1 @@ -225,8 +159,7 @@ Parameters **The same parameters with the vhost sample.** -Refer to :ref:`vhost_app_parameters` for the meanings of 'Basename', -'Stats', 'RX Retry', 'RX Retry Number' and 'RX Retry Delay Time'. +Refer to :ref:`vhost_app_parameters` for detailed explanation. **Number of Devices.** @@ -235,7 +168,7 @@ The default value is 2. .. code-block:: console - user@target:~$ ./build/app/tep_termination -c f -n 4 --huge-dir /mnt/huge -- + user@target:~$ ./build/app/tep_termination -l 0-3 -n 4 --huge-dir /mnt/huge -- --nb-devices 2 **Tunneling UDP port.** @@ -245,7 +178,7 @@ The default value is 4789. .. code-block:: console - user@target:~$ ./build/app/tep_termination -c f -n 4 --huge-dir /mnt/huge -- + user@target:~$ ./build/app/tep_termination -l 0-3 -n 4 --huge-dir /mnt/huge -- --nb-devices 2 --udp-port 4789 **Filter Type.** @@ -256,7 +189,7 @@ The default value is 1, which means the filter type of inner MAC and tenant ID i .. code-block:: console - user@target:~$ ./build/app/tep_termination -c f -n 4 --huge-dir /mnt/huge -- + user@target:~$ ./build/app/tep_termination -l 0-3 -n 4 --huge-dir /mnt/huge -- --nb-devices 2 --udp-port 4789 --filter-type 1 **TX Checksum.** @@ -266,7 +199,7 @@ The default value is 0, which means the checksum offload is disabled. .. code-block:: console - user@target:~$ ./build/app/tep_termination -c f -n 4 --huge-dir /mnt/huge -- + user@target:~$ ./build/app/tep_termination -l 0-3 -n 4 --huge-dir /mnt/huge -- --nb-devices 2 --tx-checksum **TCP segment size.** @@ -276,7 +209,7 @@ The default value is 0, which means TSO offload is disabled. .. code-block:: console - user@target:~$ ./build/app/tep_termination -c f -n 4 --huge-dir /mnt/huge -- + user@target:~$ ./build/app/tep_termination -l 0-3 -n 4 --huge-dir /mnt/huge -- --tx-checksum --tso-segsz 800 **Decapsulation option.** @@ -286,7 +219,7 @@ The default value is 1. .. code-block:: console - user@target:~$ ./build/app/tep_termination -c f -n 4 --huge-dir /mnt/huge -- + user@target:~$ ./build/app/tep_termination -l 0-3 -n 4 --huge-dir /mnt/huge -- --nb-devices 4 --udp-port 4789 --decap 1 **Encapsulation option.** @@ -296,19 +229,19 @@ The default value is 1. .. code-block:: console - user@target:~$ ./build/app/tep_termination -c f -n 4 --huge-dir /mnt/huge -- + user@target:~$ ./build/app/tep_termination -l 0-3 -n 4 --huge-dir /mnt/huge -- --nb-devices 4 --udp-port 4789 --encap 1 Running the Virtual Machine (QEMU) ---------------------------------- -Refer to :ref:`vhost_app_running`. +Refer to :ref:`vhost_app_run_vm`. Running DPDK in the Virtual Machine ----------------------------------- -Refer to :ref:`vhost_app_running_dpdk`. +Refer to :ref:`vhost_app_run_dpdk_inside_guest`. Passing Traffic to the Virtual Machine Device --------------------------------------------- diff --git a/dpdk/doc/guides/sample_app_ug/test_pipeline.rst b/dpdk/doc/guides/sample_app_ug/test_pipeline.rst index fd288139..f538cd05 100644 --- a/dpdk/doc/guides/sample_app_ug/test_pipeline.rst +++ b/dpdk/doc/guides/sample_app_ug/test_pipeline.rst @@ -55,28 +55,12 @@ The application uses three CPU cores: Test Pipeline Application - Compiling the Application ------------------------- +To compile the sample application see :doc:`compiling` -#. Go to the app/test directory: +The application is located in the ``$RTE_SDK/test/test-pipline`` directory. - .. code-block:: console - - export RTE_SDK=/path/to/rte_sdk - cd ${RTE_SDK}/app/test/test-pipeline - -#. Set the target (a default target is used if not specified): - - .. code-block:: console - - export RTE_TARGET=x86_64-native-linuxapp-gcc - -#. Build the application: - - .. code-block:: console - - make Running the Application ----------------------- @@ -90,7 +74,7 @@ The application execution command line is: ./test-pipeline [EAL options] -- -p PORTMASK --TABLE_TYPE -The -c EAL CPU core mask option has to contain exactly 3 CPU cores. +The -c or -l EAL CPU coremask/corelist option has to contain exactly 3 CPU cores. The first CPU core in the core mask is assigned for core A, the second for core B and the third for core C. The PORTMASK parameter must contain 2 or 4 ports. diff --git a/dpdk/doc/guides/sample_app_ug/timer.rst b/dpdk/doc/guides/sample_app_ug/timer.rst index e4de359d..46a3a2fa 100644 --- a/dpdk/doc/guides/sample_app_ug/timer.rst +++ b/dpdk/doc/guides/sample_app_ug/timer.rst @@ -37,26 +37,9 @@ This application prints some messages from different lcores regularly, demonstra Compiling the Application ------------------------- -#. Go to the example directory: +To compile the sample application see :doc:`compiling`. - .. code-block:: console - - export RTE_SDK=/path/to/rte_sdk - cd ${RTE_SDK}/examples/timer - -#. Set the target (a default target is used if not specified). For example: - - .. code-block:: console - - export RTE_TARGET=x86_64-native-linuxapp-gcc - - See the *DPDK Getting Started Guide* for possible *RTE_TARGET* values. - -#. Build the application: - - .. code-block:: console - - make +The application is located in the ``timer`` sub-directory. Running the Application ----------------------- @@ -65,7 +48,7 @@ To run the example in linuxapp environment: .. code-block:: console - $ ./build/timer -c f -n 4 + $ ./build/timer -l 0-3 -n 4 Refer to the *DPDK Getting Started Guide* for general information on running applications and the Environment Abstraction Layer (EAL) options. diff --git a/dpdk/doc/guides/sample_app_ug/vhost.rst b/dpdk/doc/guides/sample_app_ug/vhost.rst index 2b7defc8..5735adbb 100644 --- a/dpdk/doc/guides/sample_app_ug/vhost.rst +++ b/dpdk/doc/guides/sample_app_ug/vhost.rst @@ -1,6 +1,6 @@ .. BSD LICENSE - Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + Copyright(c) 2010-2016 Intel Corporation. All rights reserved. All rights reserved. Redistribution and use in source and binary forms, with or without @@ -33,820 +33,186 @@ Vhost Sample Application ======================== -The vhost sample application demonstrates integration of the Data Plane Development Kit (DPDK) -with the Linux* KVM hypervisor by implementing the vhost-net offload API. -The sample application performs simple packet switching between virtual machines based on Media Access Control -(MAC) address or Virtual Local Area Network (VLAN) tag. -The splitting of Ethernet traffic from an external switch is performed in hardware by the Virtual Machine Device Queues -(VMDQ) and Data Center Bridging (DCB) features of the Intel® 82599 10 Gigabit Ethernet Controller. +The vhost sample application demonstrates integration of the Data Plane +Development Kit (DPDK) with the Linux* KVM hypervisor by implementing the +vhost-net offload API. The sample application performs simple packet +switching between virtual machines based on Media Access Control (MAC) +address or Virtual Local Area Network (VLAN) tag. The splitting of Ethernet +traffic from an external switch is performed in hardware by the Virtual +Machine Device Queues (VMDQ) and Data Center Bridging (DCB) features of +the Intel® 82599 10 Gigabit Ethernet Controller. -Background ----------- - -Virtio networking (virtio-net) was developed as the Linux* KVM para-virtualized method for communicating network packets -between host and guest. -It was found that virtio-net performance was poor due to context switching and packet copying between host, guest, and QEMU. -The following figure shows the system architecture for a virtio-based networking (virtio-net). - -.. _figure_qemu_virtio_net: - -.. figure:: img/qemu_virtio_net.* - - System Architecture for Virtio-based Networking (virtio-net). - - -The Linux* Kernel vhost-net module was developed as an offload mechanism for virtio-net. -The vhost-net module enables KVM (QEMU) to offload the servicing of virtio-net devices to the vhost-net kernel module, -reducing the context switching and packet copies in the virtual dataplane. - -This is achieved by QEMU sharing the following information with the vhost-net module through the vhost-net API: - -* The layout of the guest memory space, to enable the vhost-net module to translate addresses. - -* The locations of virtual queues in QEMU virtual address space, - to enable the vhost module to read/write directly to and from the virtqueues. - -* An event file descriptor (eventfd) configured in KVM to send interrupts to the virtio- net device driver in the guest. - This enables the vhost-net module to notify (call) the guest. - -* An eventfd configured in KVM to be triggered on writes to the virtio-net device's - Peripheral Component Interconnect (PCI) config space. - This enables the vhost-net module to receive notifications (kicks) from the guest. - -The following figure shows the system architecture for virtio-net networking with vhost-net offload. - -.. _figure_virtio_linux_vhost: - -.. figure:: img/virtio_linux_vhost.* - - Virtio with Linux - - -Sample Code Overview --------------------- - -The DPDK vhost-net sample code demonstrates KVM (QEMU) offloading the servicing of a Virtual Machine's (VM's) -virtio-net devices to a DPDK-based application in place of the kernel's vhost-net module. - -The DPDK vhost-net sample code is based on vhost library. Vhost library is developed for user space Ethernet switch to -easily integrate with vhost functionality. - -The vhost library implements the following features: - -* Management of virtio-net device creation/destruction events. - -* Mapping of the VM's physical memory into the DPDK vhost-net's address space. - -* Triggering/receiving notifications to/from VMs via eventfds. - -* A virtio-net back-end implementation providing a subset of virtio-net features. - -There are two vhost implementations in vhost library, vhost cuse and vhost user. In vhost cuse, a character device driver is implemented to -receive and process vhost requests through ioctl messages. In vhost user, a socket server is created to received vhost requests through -socket messages. Most of the messages share the same handler routine. - -.. note:: - **Any vhost cuse specific requirement in the following sections will be emphasized**. - -Two implementations are turned on and off statically through configure file. Only one implementation could be turned on. They don't co-exist in current implementation. - -The vhost sample code application is a simple packet switching application with the following feature: - -* Packet switching between virtio-net devices and the network interface card, - including using VMDQs to reduce the switching that needs to be performed in software. - -The following figure shows the architecture of the Vhost sample application based on vhost-cuse. - -.. _figure_vhost_net_arch: - -.. figure:: img/vhost_net_arch.* - - Vhost-net Architectural Overview - - -The following figure shows the flow of packets through the vhost-net sample application. - -.. _figure_vhost_net_sample_app: - -.. figure:: img/vhost_net_sample_app.* - - Packet Flow Through the vhost-net Sample Application - - -Supported Distributions ------------------------ - -The example in this section have been validated with the following distributions: - -* Fedora* 18 - -* Fedora* 19 - -* Fedora* 20 - -.. _vhost_app_prerequisites: - -Prerequisites +Testing steps ------------- -This section lists prerequisite packages that must be installed. +This section shows the steps how to test a typical PVP case with this +vhost-switch sample, whereas packets are received from the physical NIC +port first and enqueued to the VM's Rx queue. Through the guest testpmd's +default forwarding mode (io forward), those packets will be put into +the Tx queue. The vhost-switch example, in turn, gets the packets and +puts back to the same physical NIC port. -Installing Packages on the Host(vhost cuse required) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Build +~~~~~ -The vhost cuse code uses the following packages; fuse, fuse-devel, and kernel-modules-extra. -The vhost user code don't rely on those modules as eventfds are already installed into vhost process through -Unix domain socket. +To compile the sample application see :doc:`compiling`. -#. Install Fuse Development Libraries and headers: - - .. code-block:: console - - yum -y install fuse fuse-devel - -#. Install the Cuse Kernel Module: - - .. code-block:: console - - yum -y install kernel-modules-extra - -QEMU simulator -~~~~~~~~~~~~~~ - -For vhost user, qemu 2.2 is required. - -Setting up the Execution Environment -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The vhost sample code requires that QEMU allocates a VM's memory on the hugetlbfs file system. -As the vhost sample code requires hugepages, -the best practice is to partition the system into separate hugepage mount points for the VMs and the vhost sample code. +The application is located in the ``vhost`` sub-directory. .. note:: + In this example, you need build DPDK both on the host and inside guest. - This is best-practice only and is not mandatory. - For systems that only support 2 MB page sizes, - both QEMU and vhost sample code can use the same hugetlbfs mount point without issue. - -**QEMU** - -VMs with gigabytes of memory can benefit from having QEMU allocate their memory from 1 GB huge pages. -1 GB huge pages must be allocated at boot time by passing kernel parameters through the grub boot loader. - -#. Calculate the maximum memory usage of all VMs to be run on the system. - Then, round this value up to the nearest Gigabyte the execution environment will require. - -#. Edit the /etc/default/grub file, and add the following to the GRUB_CMDLINE_LINUX entry: - - .. code-block:: console - - GRUB_CMDLINE_LINUX="... hugepagesz=1G hugepages= default_hugepagesz=1G" - -#. Update the grub boot loader: - - .. code-block:: console - - grub2-mkconfig -o /boot/grub2/grub.cfg - -#. Reboot the system. - -#. The hugetlbfs mount point (/dev/hugepages) should now default to allocating gigabyte pages. - -.. note:: - - Making the above modification will change the system default hugepage size to 1 GB for all applications. - -**Vhost Sample Code** - -In this section, we create a second hugetlbs mount point to allocate hugepages for the DPDK vhost sample code. - -#. Allocate sufficient 2 MB pages for the DPDK vhost sample code: - - .. code-block:: console - - echo 256 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages - -#. Mount hugetlbs at a separate mount point for 2 MB pages: - - .. code-block:: console - - mount -t hugetlbfs nodev /mnt/huge -o pagesize=2M - -The above steps can be automated by doing the following: - -#. Edit /etc/fstab to add an entry to automatically mount the second hugetlbfs mount point: - - :: - - hugetlbfs /mnt/huge hugetlbfs defaults,pagesize=1G 0 0 - -#. Edit the /etc/default/grub file, and add the following to the GRUB_CMDLINE_LINUX entry: - - :: - - GRUB_CMDLINE_LINUX="... hugepagesz=2M hugepages=256 ... default_hugepagesz=1G" - -#. Update the grub bootloader: - - .. code-block:: console - - grub2-mkconfig -o /boot/grub2/grub.cfg - -#. Reboot the system. - -.. note:: - - Ensure that the default hugepage size after this setup is 1 GB. - -Setting up the Guest Execution Environment -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -It is recommended for testing purposes that the DPDK testpmd sample application is used in the guest to forward packets, -the reasons for this are discussed in `Running the Virtual Machine (QEMU)`_. - -The testpmd application forwards packets between pairs of Ethernet devices, -it requires an even number of Ethernet devices (virtio or otherwise) to execute. -It is therefore recommended to create multiples of two virtio-net devices for each Virtual Machine either through libvirt or -at the command line as follows. - -.. note:: - - Observe that in the example, "-device" and "-netdev" are repeated for two virtio-net devices. - -For vhost cuse: +Start the vswitch example +~~~~~~~~~~~~~~~~~~~~~~~~~ .. code-block:: console - qemu-system-x86_64 ... \ - -netdev tap,id=hostnet1,vhost=on,vhostfd= \ - -device virtio-net-pci, netdev=hostnet1,id=net1 \ - -netdev tap,id=hostnet2,vhost=on,vhostfd= \ - -device virtio-net-pci, netdev=hostnet2,id=net1 + ./vhost-switch -l 0-3 -n 4 --socket-mem 1024 \ + -- --socket-file /tmp/sock0 --client \ + ... -For vhost user: +Check the `Parameters`_ section for the explanations on what do those +parameters mean. + +.. _vhost_app_run_vm: + +Start the VM +~~~~~~~~~~~~ .. code-block:: console - qemu-system-x86_64 ... \ - -chardev socket,id=char1,path= \ - -netdev type=vhost-user,id=hostnet1,chardev=char1 \ - -device virtio-net-pci,netdev=hostnet1,id=net1 \ - -chardev socket,id=char2,path= \ - -netdev type=vhost-user,id=hostnet2,chardev=char2 \ - -device virtio-net-pci,netdev=hostnet2,id=net2 - -sock_path is the path for the socket file created by vhost. - -Compiling the Sample Code -------------------------- -#. Compile vhost lib: - - To enable vhost, turn on vhost library in the configure file config/common_linuxapp. - - .. code-block:: console - - CONFIG_RTE_LIBRTE_VHOST=n - - vhost user is turned on by default in the configure file config/common_linuxapp. - To enable vhost cuse, disable vhost user. - - .. code-block:: console - - CONFIG_RTE_LIBRTE_VHOST_USER=y - - After vhost is enabled and the implementation is selected, build the vhost library. - -#. Go to the examples directory: - - .. code-block:: console - - export RTE_SDK=/path/to/rte_sdk - cd ${RTE_SDK}/examples/vhost - -#. Set the target (a default target is used if not specified). For example: - - .. code-block:: console - - export RTE_TARGET=x86_64-native-linuxapp-gcc - - See the DPDK Getting Started Guide for possible RTE_TARGET values. - -#. Build the application: - - .. code-block:: console - - cd ${RTE_SDK} - make config ${RTE_TARGET} - make install ${RTE_TARGET} - cd ${RTE_SDK}/examples/vhost - make - -#. Go to the eventfd_link directory(vhost cuse required): - - .. code-block:: console - - cd ${RTE_SDK}/lib/librte_vhost/eventfd_link - -#. Build the eventfd_link kernel module(vhost cuse required): - - .. code-block:: console - - make - -Running the Sample Code ------------------------ - -#. Install the cuse kernel module(vhost cuse required): - - .. code-block:: console - - modprobe cuse - -#. Go to the eventfd_link directory(vhost cuse required): - - .. code-block:: console - - export RTE_SDK=/path/to/rte_sdk - cd ${RTE_SDK}/lib/librte_vhost/eventfd_link - -#. Install the eventfd_link module(vhost cuse required): - - .. code-block:: console - - insmod ./eventfd_link.ko - -#. Go to the examples directory: - - .. code-block:: console - - export RTE_SDK=/path/to/rte_sdk - cd ${RTE_SDK}/examples/vhost/build/app - -#. Run the vhost-switch sample code: - - vhost cuse: - - .. code-block:: console - - ./vhost-switch -c f -n 4 --socket-mem 1024 --huge-dir /mnt/huge \ - -- -p 0x1 --dev-basename usvhost - - vhost user: a socket file named usvhost will be created under current directory. Use its path as the socket path in guest's qemu commandline. - - .. code-block:: console - - ./vhost-switch -c f -n 4 --socket-mem 1024 --huge-dir /mnt/huge \ - -- -p 0x1 --dev-basename usvhost + qemu-system-x86_64 -machine accel=kvm -cpu host \ + -m $mem -object memory-backend-file,id=mem,size=$mem,mem-path=/dev/hugepages,share=on \ + -mem-prealloc -numa node,memdev=mem \ + \ + -chardev socket,id=char1,path=/tmp/sock0,server \ + -netdev type=vhost-user,id=hostnet1,chardev=char1 \ + -device virtio-net-pci,netdev=hostnet1,id=net1,mac=52:54:00:00:00:14 \ + ... .. note:: + For basic vhost-user support, QEMU 2.2 (or above) is required. For + some specific features, a higher version might be need. Such as + QEMU 2.7 (or above) for the reconnect feature. - Please note the huge-dir parameter instructs the DPDK to allocate its memory from the 2 MB page hugetlbfs. +.. _vhost_app_run_dpdk_inside_guest: -.. note:: +Run testpmd inside guest +~~~~~~~~~~~~~~~~~~~~~~~~ + +Make sure you have DPDK built inside the guest. Also make sure the +corresponding virtio-net PCI device is bond to a uio driver, which +could be done by: + +.. code-block:: console + + modprobe uio_pci_generic + $RTE_SDK/usertools/dpdk-devbind.py -b=uio_pci_generic 0000:00:04.0 + +Then start testpmd for packet forwarding testing. + +.. code-block:: console + + ./x86_64-native-gcc/app/testpmd -l 0-1 -- -i + > start tx_first + +Inject packets +-------------- + +While a virtio-net is connected to vhost-switch, a VLAN tag starts with +1000 is assigned to it. So make sure configure your packet generator +with the right MAC and VLAN tag, you should be able to see following +log from the vhost-switch console. It means you get it work:: + + VHOST_DATA: (0) mac 52:54:00:00:00:14 and vlan 1000 registered - The number used with the --socket-mem parameter may need to be more than 1024. - The number required depends on the number of mbufs allocated by vhost-switch. .. _vhost_app_parameters: Parameters -~~~~~~~~~~ +---------- -**Basename.** -vhost cuse uses a Linux* character device to communicate with QEMU. -The basename is used to generate the character devices name. +**--socket-file path** +Specifies the vhost-user socket file path. - /dev/ +**--client** +DPDK vhost-user will act as the client mode when such option is given. +In the client mode, QEMU will create the socket file. Otherwise, DPDK +will create it. Put simply, it's the server to create the socket file. -For compatibility with the QEMU wrapper script, a base name of "usvhost" should be used: -.. code-block:: console +**--vm2vm mode** +The vm2vm parameter sets the mode of packet switching between guests in +the host. - ./vhost-switch -c f -n 4 --socket-mem 1024 --huge-dir /mnt/huge \ - -- -p 0x1 --dev-basename usvhost +- 0 disables vm2vm, impling that VM's packets will always go to the NIC port. +- 1 means a normal mac lookup packet routing. +- 2 means hardware mode packet forwarding between guests, it allows packets + go to the NIC port, hardware L2 switch will determine which guest the + packet should forward to or need send to external, which bases on the + packet destination MAC address and VLAN tag. -**vm2vm.** -The vm2vm parameter disable/set mode of packet switching between guests in the host. -Value of "0" means disabling vm2vm implies that on virtual machine packet transmission will always go to the Ethernet port; -Value of "1" means software mode packet forwarding between guests, it needs packets copy in vHOST, -so valid only in one-copy implementation, and invalid for zero copy implementation; -value of "2" means hardware mode packet forwarding between guests, it allows packets go to the Ethernet port, -hardware L2 switch will determine which guest the packet should forward to or need send to external, -which bases on the packet destination MAC address and VLAN tag. +**--mergeable 0|1** +Set 0/1 to disable/enable the mergeable Rx feature. It's disabled by default. -.. code-block:: console - - ./vhost-switch -c f -n 4 --socket-mem 1024 --huge-dir /mnt/huge \ - -- --vm2vm [0,1,2] - -**Mergeable Buffers.** -The mergeable buffers parameter controls how virtio-net descriptors are used for virtio-net headers. -In a disabled state, one virtio-net header is used per packet buffer; -in an enabled state one virtio-net header is used for multiple packets. -The default value is 0 or disabled since recent kernels virtio-net drivers show performance degradation with this feature is enabled. - -.. code-block:: console - - ./vhost-switch -c f -n 4 --socket-mem 1024 --huge-dir /mnt/huge \ - -- --mergeable [0,1] - -**Stats.** +**--stats interval** The stats parameter controls the printing of virtio-net device statistics. -The parameter specifies an interval second to print statistics, with an interval of 0 seconds disabling statistics. - -.. code-block:: console - - ./vhost-switch -c f -n 4 --socket-mem 1024 --huge-dir /mnt/huge \ - -- --stats [0,n] - -**RX Retry.** -The rx-retry option enables/disables enqueue retries when the guests RX queue is full. -This feature resolves a packet loss that is observed at high data-rates, -by allowing it to delay and retry in the receive path. -This option is enabled by default. - -.. code-block:: console - - ./vhost-switch -c f -n 4 --socket-mem 1024 --huge-dir /mnt/huge \ - -- --rx-retry [0,1] - -**RX Retry Number.** -The rx-retry-num option specifies the number of retries on an RX burst, -it takes effect only when rx retry is enabled. -The default value is 4. - -.. code-block:: console - - ./vhost-switch -c f -n 4 --socket-mem 1024 --huge-dir /mnt/huge \ - -- --rx-retry 1 --rx-retry-num 5 - -**RX Retry Delay Time.** -The rx-retry-delay option specifies the timeout (in micro seconds) between retries on an RX burst, -it takes effect only when rx retry is enabled. -The default value is 15. - -.. code-block:: console - - ./vhost-switch -c f -n 4 --socket-mem 1024 --huge-dir /mnt/huge \ - -- --rx-retry 1 --rx-retry-delay 20 - -**Zero copy.** -Zero copy mode is removed, due to it has not been working for a while. And -due to the large and complex code, it's better to redesign it than fixing -it to make it work again. Hence, zero copy may be added back later. - -**VLAN strip.** -The VLAN strip option enable/disable the VLAN strip on host, if disabled, the guest will receive the packets with VLAN tag. -It is enabled by default. - -.. code-block:: console - - ./vhost-switch -c f -n 4 --socket-mem 1024 --huge-dir /mnt/huge \ - -- --vlan-strip [0, 1] - -.. _vhost_app_running: - -Running the Virtual Machine (QEMU) ----------------------------------- - -QEMU must be executed with specific parameters to: - -* Ensure the guest is configured to use virtio-net network adapters. - - .. code-block:: console - - qemu-system-x86_64 ... -device virtio-net-pci,netdev=hostnet1, \ - id=net1 ... - -* Ensure the guest's virtio-net network adapter is configured with offloads disabled. - - .. code-block:: console - - qemu-system-x86_64 ... -device virtio-net-pci,netdev=hostnet1, \ - id=net1, csum=off,gso=off,guest_tso4=off,guest_tso6=off,guest_ecn=off - -* Redirect QEMU to communicate with the DPDK vhost-net sample code in place of the vhost-net kernel module(vhost cuse). - - .. code-block:: console - - qemu-system-x86_64 ... -netdev tap,id=hostnet1,vhost=on, \ - vhostfd= ... - -* Enable the vhost-net sample code to map the VM's memory into its own process address space. - - .. code-block:: console - - qemu-system-x86_64 ... -mem-prealloc -mem-path /dev/hugepages ... - -.. note:: - - The QEMU wrapper (qemu-wrap.py) is a Python script designed to automate the QEMU configuration described above. - It also facilitates integration with libvirt, although the script may also be used standalone without libvirt. - -Redirecting QEMU to vhost-net Sample Code(vhost cuse) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -To redirect QEMU to the vhost-net sample code implementation of the vhost-net API, -an open file descriptor must be passed to QEMU running as a child process. - -.. code-block:: python - - #!/usr/bin/python - fd = os.open("/dev/usvhost-1", os.O_RDWR) - subprocess.call - ("qemu-system-x86_64 ... -netdev tap,id=vhostnet0,vhost=on,vhostfd=" - + fd +"...", shell=True) - -.. note:: - - This process is automated in the `QEMU Wrapper Script`_. - -Mapping the Virtual Machine's Memory -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -For the DPDK vhost-net sample code to be run correctly, QEMU must allocate the VM's memory on hugetlbfs. -This is done by specifying mem-prealloc and mem-path when executing QEMU. -The vhost-net sample code accesses the virtio-net device's virtual rings and packet buffers -by finding and mapping the VM's physical memory on hugetlbfs. -In this case, the path passed to the guest should be that of the 1 GB page hugetlbfs: - -.. code-block:: console - - qemu-system-x86_64 ... -mem-prealloc -mem-path /dev/hugepages ... - -.. note:: - - This process is automated in the `QEMU Wrapper Script`_. - The following two sections only applies to vhost cuse. - For vhost-user, please make corresponding changes to qemu-wrapper script and guest XML file. - -QEMU Wrapper Script -~~~~~~~~~~~~~~~~~~~ - -The QEMU wrapper script automatically detects and calls QEMU with the necessary parameters required -to integrate with the vhost sample code. -It performs the following actions: - -* Automatically detects the location of the hugetlbfs and inserts this into the command line parameters. - -* Automatically open file descriptors for each virtio-net device and inserts this into the command line parameters. - -* Disables offloads on each virtio-net device. - -* Calls Qemu passing both the command line parameters passed to the script itself and those it has auto-detected. - -The QEMU wrapper script will automatically configure calls to QEMU: - -.. code-block:: console - - qemu-wrap.py -machine pc-i440fx-1.4,accel=kvm,usb=off \ - -cpu SandyBridge -smp 4,sockets=4,cores=1,threads=1 \ - -netdev tap,id=hostnet1,vhost=on \ - -device virtio-net-pci,netdev=hostnet1,id=net1 \ - -hda -m 4096 - -which will become the following call to QEMU: - -.. code-block:: console - - qemu-system-x86_64 -machine pc-i440fx-1.4,accel=kvm,usb=off \ - -cpu SandyBridge -smp 4,sockets=4,cores=1,threads=1 \ - -netdev tap,id=hostnet1,vhost=on,vhostfd= \ - -device virtio-net-pci,netdev=hostnet1,id=net1, \ - csum=off,gso=off,guest_tso4=off,guest_tso6=off,guest_ecn=off \ - -hda -m 4096 -mem-path /dev/hugepages -mem-prealloc - -Libvirt Integration -~~~~~~~~~~~~~~~~~~~ - -The QEMU wrapper script (qemu-wrap.py) "wraps" libvirt calls to QEMU, -such that QEMU is called with the correct parameters described above. -To call the QEMU wrapper automatically from libvirt, the following configuration changes must be made: - -* Place the QEMU wrapper script in libvirt's binary search PATH ($PATH). - A good location is in the directory that contains the QEMU binary. - -* Ensure that the script has the same owner/group and file permissions as the QEMU binary. - -* Update the VM xml file using virsh edit : - - * Set the VM to use the launch script - - * Set the emulator path contained in the # tags For example, - replace /usr/bin/qemu-kvm with /usr/bin/qemu-wrap.py - - * Set the VM's virtio-net device's to use vhost-net offload: - - .. code-block:: xml - - - - - - - * Enable libvirt to access the DPDK Vhost sample code's character device file by adding it - to controllers cgroup for libvirtd using the following steps: - - .. code-block:: xml - - cgroup_controllers = [ ... "devices", ... ] clear_emulator_capabilities = 0 - user = "root" group = "root" - cgroup_device_acl = [ - "/dev/null", "/dev/full", "/dev/zero", - "/dev/random", "/dev/urandom", - "/dev/ptmx", "/dev/kvm", "/dev/kqemu", - "/dev/rtc", "/dev/hpet", "/dev/net/tun", - "/dev/-", - ] - -* Disable SELinux or set to permissive mode. - - -* Mount cgroup device controller: - - .. code-block:: console - - mkdir /dev/cgroup - mount -t cgroup none /dev/cgroup -o devices - -* Restart the libvirtd system process - - For example, on Fedora* "systemctl restart libvirtd.service" - -* Edit the configuration parameters section of the script: - - * Configure the "emul_path" variable to point to the QEMU emulator. - - .. code-block:: xml - - emul_path = "/usr/local/bin/qemu-system-x86_64" - - * Configure the "us_vhost_path" variable to point to the DPDK vhost-net sample code's character devices name. - DPDK vhost-net sample code's character device will be in the format "/dev/". - - .. code-block:: xml - - us_vhost_path = "/dev/usvhost" +The parameter specifies an interval (in unit of seconds) to print statistics, +with an interval of 0 seconds disabling statistics. + +**--rx-retry 0|1** +The rx-retry option enables/disables enqueue retries when the guests Rx queue +is full. This feature resolves a packet loss that is observed at high data +rates, by allowing it to delay and retry in the receive path. This option is +enabled by default. + +**--rx-retry-num num** +The rx-retry-num option specifies the number of retries on an Rx burst, it +takes effect only when rx retry is enabled. The default value is 4. + +**--rx-retry-delay msec** +The rx-retry-delay option specifies the timeout (in micro seconds) between +retries on an RX burst, it takes effect only when rx retry is enabled. The +default value is 15. + +**--dequeue-zero-copy** +Dequeue zero copy will be enabled when this option is given. + +**--vlan-strip 0|1** +VLAN strip option is removed, because different NICs have different behaviors +when disabling VLAN strip. Such feature, which heavily depends on hardware, +should be removed from this example to reduce confusion. Now, VLAN strip is +enabled and cannot be disabled. Common Issues -~~~~~~~~~~~~~ +------------- -* QEMU failing to allocate memory on hugetlbfs, with an error like the following:: +* QEMU fails to allocate memory on hugetlbfs, with an error like the + following:: - file_ram_alloc: can't mmap RAM pages: Cannot allocate memory + file_ram_alloc: can't mmap RAM pages: Cannot allocate memory - When running QEMU the above error indicates that it has failed to allocate memory for the Virtual Machine on - the hugetlbfs. This is typically due to insufficient hugepages being free to support the allocation request. - The number of free hugepages can be checked as follows: + When running QEMU the above error indicates that it has failed to allocate + memory for the Virtual Machine on the hugetlbfs. This is typically due to + insufficient hugepages being free to support the allocation request. The + number of free hugepages can be checked as follows: - .. code-block:: console + .. code-block:: console - cat /sys/kernel/mm/hugepages/hugepages-/nr_hugepages + cat /sys/kernel/mm/hugepages/hugepages-/nr_hugepages - The command above indicates how many hugepages are free to support QEMU's allocation request. + The command above indicates how many hugepages are free to support QEMU's + allocation request. -* User space VHOST when the guest has 2MB sized huge pages: +* vhost-user will not work with QEMU without the ``-mem-prealloc`` option - The guest may have 2MB or 1GB sized huge pages. The user space VHOST should work properly in both cases. + The current implementation works properly only when the guest memory is + pre-allocated. -* User space VHOST will not work with QEMU without the ``-mem-prealloc`` option: +* vhost-user will not work with a QEMU version without shared memory mapping: - The current implementation works properly only when the guest memory is pre-allocated, so it is required to - use a QEMU version (e.g. 1.6) which supports ``-mem-prealloc``. The ``-mem-prealloc`` option must be - specified explicitly in the QEMU command line. + Make sure ``share=on`` QEMU option is given. -* User space VHOST will not work with a QEMU version without shared memory mapping: +* Failed to build DPDK in VM - As shared memory mapping is mandatory for user space VHOST to work properly with the guest, user space VHOST - needs access to the shared memory from the guest to receive and transmit packets. It is important to make sure - the QEMU version supports shared memory mapping. - -* In an Ubuntu environment, QEMU fails to start a new guest normally with user space VHOST due to not being able - to allocate huge pages for the new guest: - - The solution for this issue is to add ``-boot c`` into the QEMU command line to make sure the huge pages are - allocated properly and then the guest should start normally. - - Use ``cat /proc/meminfo`` to check if there is any changes in the value of ``HugePages_Total`` and ``HugePages_Free`` - after the guest startup. - -* Log message: ``eventfd_link: module verification failed: signature and/or required key missing - tainting kernel``: - - This log message may be ignored. The message occurs due to the kernel module ``eventfd_link``, which is not a standard - Linux module but which is necessary for the user space VHOST current implementation (CUSE-based) to communicate with - the guest. - -.. _vhost_app_running_dpdk: - -Running DPDK in the Virtual Machine ------------------------------------ - -For the DPDK vhost-net sample code to switch packets into the VM, -the sample code must first learn the MAC address of the VM's virtio-net device. -The sample code detects the address from packets being transmitted from the VM, similar to a learning switch. - -This behavior requires no special action or configuration with the Linux* virtio-net driver in the VM -as the Linux* Kernel will automatically transmit packets during device initialization. -However, DPDK-based applications must be modified to automatically transmit packets during initialization -to facilitate the DPDK vhost- net sample code's MAC learning. - -The DPDK testpmd application can be configured to automatically transmit packets during initialization -and to act as an L2 forwarding switch. - -Testpmd MAC Forwarding -~~~~~~~~~~~~~~~~~~~~~~ - -At high packet rates, a minor packet loss may be observed. -To resolve this issue, a "wait and retry" mode is implemented in the testpmd and vhost sample code. -In the "wait and retry" mode if the virtqueue is found to be full, then testpmd waits for a period of time before retrying to enqueue packets. - -The "wait and retry" algorithm is implemented in DPDK testpmd as a forwarding method call "mac_retry". -The following sequence diagram describes the algorithm in detail. - -.. _figure_tx_dpdk_testpmd: - -.. figure:: img/tx_dpdk_testpmd.* - - Packet Flow on TX in DPDK-testpmd - - -Running Testpmd -~~~~~~~~~~~~~~~ - -The testpmd application is automatically built when DPDK is installed. -Run the testpmd application as follows: - -.. code-block:: console - - cd ${RTE_SDK}/x86_64-native-linuxapp-gcc/app - ./testpmd -c 0x3 -n 4 --socket-mem 512 \ - -- --burst=64 --i --disable-hw-vlan-filter - -The destination MAC address for packets transmitted on each port can be set at the command line: - -.. code-block:: console - - ./testpmd -c 0x3 -n 4 --socket-mem 512 \ - -- --burst=64 --i --disable-hw-vlan-filter \ - --eth-peer=0,aa:bb:cc:dd:ee:ff --eth-peer=1,ff:ee:dd:cc:bb:aa - -* Packets received on port 1 will be forwarded on port 0 to MAC address - - aa:bb:cc:dd:ee:ff - -* Packets received on port 0 will be forwarded on port 1 to MAC address - - ff:ee:dd:cc:bb:aa - -The testpmd application can then be configured to act as an L2 forwarding application: - -.. code-block:: console - - testpmd> set fwd mac_retry - -The testpmd can then be configured to start processing packets, -transmitting packets first so the DPDK vhost sample code on the host can learn the MAC address: - -.. code-block:: console - - testpmd> start tx_first - -.. note:: - - Please note "set fwd mac_retry" is used in place of "set fwd mac_fwd" to ensure the retry feature is activated. - -Passing Traffic to the Virtual Machine Device ---------------------------------------------- - -For a virtio-net device to receive traffic, -the traffic's Layer 2 header must include both the virtio-net device's MAC address and VLAN tag. -The DPDK sample code behaves in a similar manner to a learning switch in that -it learns the MAC address of the virtio-net devices from the first transmitted packet. -On learning the MAC address, -the DPDK vhost sample code prints a message with the MAC address and VLAN tag virtio-net device. -For example: - -.. code-block:: console - - DATA: (0) MAC_ADDRESS cc:bb:bb:bb:bb:bb and VLAN_TAG 1000 registered - -The above message indicates that device 0 has been registered with MAC address cc:bb:bb:bb:bb:bb and VLAN tag 1000. -Any packets received on the NIC with these values is placed on the devices receive queue. -When a virtio-net device transmits packets, the VLAN tag is added to the packet by the DPDK vhost sample code. - -Running virtio_user with vhost-switch -------------------------------------- - -We can also use virtio_user with vhost-switch now. -Virtio_user is a virtual device that can be run in a application (container) parallelly with vhost in the same OS, -aka, there is no need to start a VM. We just run it with a different --file-prefix to avoid startup failure. - -.. code-block:: console - - cd ${RTE_SDK}/x86_64-native-linuxapp-gcc/app - ./testpmd -c 0x3 -n 4 --socket-mem 1024 --no-pci --file-prefix=virtio_user-testpmd \ - --vdev=virtio_user0,mac=00:01:02:03:04:05,path=$path_vhost \ - -- -i --txqflags=0xf01 --disable-hw-vlan - -There is no difference on the vhost side. -Pleae note that there are some limitations (see release note for more information) in the usage of virtio_user. + Make sure "-cpu host" QEMU option is given. diff --git a/dpdk/doc/guides/sample_app_ug/vhost_scsi.rst b/dpdk/doc/guides/sample_app_ug/vhost_scsi.rst new file mode 100644 index 00000000..1b4dab32 --- /dev/null +++ b/dpdk/doc/guides/sample_app_ug/vhost_scsi.rst @@ -0,0 +1,106 @@ + +.. BSD LICENSE + Copyright(c) 2010-2017 Intel Corporation. All rights reserved. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +Vhost_scsi Sample Application +============================= + +The vhost_scsi sample application implemented a simple SCSI block device, +which used as the backend of Qemu vhost-user-scsi device. Users can extend +the exist example to use other type of block device(e.g. AIO) besides +memory based block device. Similar with vhost-user-net device, the sample +application used domain socket to communicate with Qemu, and the virtio +ring was processed by vhost_scsi sample application. + +The sample application reuse lots codes from SPDK(Storage Performance +Development Kit, https://github.com/spdk/spdk) vhost-user-scsi target, +for DPDK vhost library used in storage area, user can take SPDK as +reference as well. + +Testing steps +------------- + +This section shows the steps how to start a VM with the block device as +fast data path for critical application. + +Compiling the Application +------------------------- + +To compile the sample application see :doc:`compiling`. + +The application is located in the ``examples`` sub-directory. + +You will also need to build DPDK both on the host and inside the guest + +Start the vhost_scsi example +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: console + + ./vhost_scsi -m 1024 + +.. _vhost_scsi_app_run_vm: + +Start the VM +~~~~~~~~~~~~ + +.. code-block:: console + + qemu-system-x86_64 -machine accel=kvm \ + -m $mem -object memory-backend-file,id=mem,size=$mem,\ + mem-path=/dev/hugepages,share=on -numa node,memdev=mem \ + -drive file=os.img,if=none,id=disk \ + -device ide-hd,drive=disk,bootindex=0 \ + -chardev socket,id=char0,path=/tmp/vhost.socket \ + -device vhost-user-scsi-pci,chardev=char0,bootindex=2 \ + ... + +.. note:: + You must check whether your Qemu can support "vhost-user-scsi" or not, + Qemu v2.10 or newer version is required. + +Vhost_scsi Common Issues +------------------------ + +* vhost_scsi can not start with block size 512 Bytes: + + Currently DPDK vhost library was designed for NET device(althrough the APIs + are generic now), for 512 Bytes block device, Qemu BIOS(x86 BIOS Enhanced + Disk Device) will enumerate all block device and do some IOs to those block + devices with 512 Bytes sector size. DPDK vhost library can not process such + scenarios(both BIOS and OS will enumerate the block device), so as a + workaround, the vhost_scsi example application hardcoded the block size + with 4096 Bytes. + +* vhost_scsi can only support the block device as fast data disk(non OS image): + + Make sure ``bootindex=2`` Qemu option is given to vhost-user-scsi-pci device. + diff --git a/dpdk/doc/guides/sample_app_ug/vm_power_management.rst b/dpdk/doc/guides/sample_app_ug/vm_power_management.rst index aa02c156..fe53706e 100644 --- a/dpdk/doc/guides/sample_app_ug/vm_power_management.rst +++ b/dpdk/doc/guides/sample_app_ug/vm_power_management.rst @@ -201,9 +201,12 @@ Compiling and Running the Host Application Compiling ~~~~~~~~~ -#. export RTE_SDK=/path/to/rte_sdk -#. cd ${RTE_SDK}/examples/vm_power_manager -#. make +Compiling the Application +------------------------- + +To compile the sample application see :doc:`compiling`. + +The application is located in the ``vm_power_manager`` sub-directory. Running ~~~~~~~ @@ -220,7 +223,7 @@ on cores 0 & 1 on a system with 4 memory channels: .. code-block:: console - ./build/vm_power_mgr -c 0x3 -n 4 + ./build/vm_power_mgr -l 0-1 -n 4 After successful initialization the user is presented with VM Power Manager CLI: @@ -342,7 +345,7 @@ for example to run on cores 0,1,2,3 on a system with 4 memory channels: .. code-block:: console - ./build/guest_vm_power_mgr -c 0xf -n 4 + ./build/guest_vm_power_mgr -l 0-3 -n 4 After successful initialization the user is presented with VM Power Manager Guest CLI: diff --git a/dpdk/doc/guides/sample_app_ug/vmdq_dcb_forwarding.rst b/dpdk/doc/guides/sample_app_ug/vmdq_dcb_forwarding.rst index bf55fdad..4a6c70d2 100644 --- a/dpdk/doc/guides/sample_app_ug/vmdq_dcb_forwarding.rst +++ b/dpdk/doc/guides/sample_app_ug/vmdq_dcb_forwarding.rst @@ -86,26 +86,11 @@ No command-line options are taken by this application apart from the standard EA Compiling the Application ------------------------- -#. Go to the examples directory: - .. code-block:: console - export RTE_SDK=/path/to/rte_sdk - cd ${RTE_SDK}/examples/vmdq_dcb +To compile the sample application see :doc:`compiling`. -#. Set the target (a default target is used if not specified). For example: - - .. code-block:: console - - export RTE_TARGET=x86_64-native-linuxapp-gcc - - See the *DPDK Getting Started Guide* for possible RTE_TARGET values. - -#. Build the application: - - .. code-block:: console - - make +The application is located in the ``vmdq_dcb`` sub-directory. Running the Application ----------------------- @@ -114,7 +99,7 @@ To run the example in a linuxapp environment: .. code-block:: console - user@target:~$ ./build/vmdq_dcb -c f -n 4 -- -p 0x3 --nb-pools 32 --nb-tcs 4 + user@target:~$ ./build/vmdq_dcb -l 0-3 -n 4 -- -p 0x3 --nb-pools 32 --nb-tcs 4 Refer to the *DPDK Getting Started Guide* for general information on running applications and the Environment Abstraction Layer (EAL) options. diff --git a/dpdk/doc/guides/testpmd_app_ug/index.rst b/dpdk/doc/guides/testpmd_app_ug/index.rst index 1c39a177..61a91218 100644 --- a/dpdk/doc/guides/testpmd_app_ug/index.rst +++ b/dpdk/doc/guides/testpmd_app_ug/index.rst @@ -28,6 +28,8 @@ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +.. _testpmd_ug: + Testpmd Application User Guide ============================== diff --git a/dpdk/doc/guides/testpmd_app_ug/run_app.rst b/dpdk/doc/guides/testpmd_app_ug/run_app.rst index 7712bd24..4c0d2ced 100644 --- a/dpdk/doc/guides/testpmd_app_ug/run_app.rst +++ b/dpdk/doc/guides/testpmd_app_ug/run_app.rst @@ -72,7 +72,7 @@ See the DPDK Getting Started Guides for more information on these options. * ``-b, --pci-blacklist domain:bus:devid.func`` - Blacklist a PCI devise to prevent EAL from using it. Multiple -b options are allowed. + Blacklist a PCI device to prevent EAL from using it. Multiple -b options are allowed. * ``-d LIB.so`` @@ -94,10 +94,6 @@ See the DPDK Getting Started Guides for more information on these options. Display the version information on startup. -* ``--xen-dom0`` - - Support application running on Xen Domain0 without hugetlbfs. - * ``--syslog`` Set the syslog facility. @@ -110,6 +106,10 @@ See the DPDK Getting Started Guides for more information on these options. Specify the directory where the hugetlbfs is mounted. +* ``mbuf-pool-ops-name``: + + Pool ops name for mbuf to use. + * ``--proc-type`` Set the type of the current process. @@ -130,7 +130,7 @@ See the DPDK Getting Started Guides for more information on these options. For example:: - --vdev 'eth_pcap0,rx_pcap=input.pcap,tx_pcap=output.pcap' + --vdev 'net_pcap0,rx_pcap=input.pcap,tx_pcap=output.pcap' * ``--base-virtaddr`` @@ -165,7 +165,7 @@ They must be separated from the EAL options, shown in the previous section, with .. code-block:: console - sudo ./testpmd -c 0xF -n 4 -- -i --portmask=0x1 --nb-cores=2 + sudo ./testpmd -l 0-3 -n 4 -- -i --portmask=0x1 --nb-cores=2 The commandline options are: @@ -188,6 +188,19 @@ The commandline options are: Start forwarding on initialization. +* ``--tx-first`` + + Start forwarding, after sending a burst of packets first. + +.. Note:: + + This flag should be only used in non-interactive mode. + +* ``--stats-period PERIOD`` + + Display statistics every PERIOD seconds, if interactive mode is disabled. + The default value is 0, which means that the statistics will not be displayed. + * ``--nb-cores=N`` Set the number of forwarding cores, @@ -211,7 +224,12 @@ The commandline options are: * ``--numa`` - Enable NUMA-aware allocation of RX/TX rings and of RX memory buffers (mbufs). + Enable NUMA-aware allocation of RX/TX rings and of RX memory buffers + (mbufs). [Default setting] + +* ``--no-numa`` + + Disable NUMA-aware allocation of RX/TX rings and of RX memory buffers (mbufs). * ``--port-numa-config=(port,socket)[,(port,socket)]`` @@ -281,9 +299,13 @@ The commandline options are: In perfect filter mode, when a rule is added with queue = -1, the packet will be enqueued into the RX drop-queue. If the drop-queue does not exist, the packet is dropped. The default value is N=127. -* ``--crc-strip`` +* ``--disable-crc-strip`` - Enable hardware CRC stripping. + Disable hardware CRC stripping. + +* ``--enable-lro`` + + Enable large receive offload. * ``--enable-rx-cksum`` @@ -340,6 +362,7 @@ The commandline options are: csum icmpecho ieee1588 + tm * ``--rss-ip`` @@ -450,8 +473,39 @@ The commandline options are: * ``--txpkts=X[,Y]`` - Set TX segment sizes. + Set TX segment sizes or total packet length. Valid for ``tx-only`` + and ``flowgen`` forwarding modes. * ``--disable-link-check`` Disable check on link status when starting/stopping ports. + +* ``--no-lsc-interrupt`` + + Disable LSC interrupts for all ports, even those supporting it. + +* ``--no-rmv-interrupt`` + + Disable RMV interrupts for all ports, even those supporting it. + +* ``--bitrate-stats=N`` + + Set the logical core N to perform bitrate calculation. + +* ``--print-event `` + + Enable printing the occurrence of the designated event. Using all will + enable all of them. + +* ``--mask-event `` + + Disable printing the occurrence of the designated event. Using all will + disable all of them. + +* ``--flow-isolate-all`` + + Providing this parameter requests flow API isolated mode on all ports at + initialization time. It ensures all traffic is received through the + configured flow rules only (see flow command). + + Ports that do not support this mode are automatically discarded. diff --git a/dpdk/doc/guides/testpmd_app_ug/testpmd_funcs.rst b/dpdk/doc/guides/testpmd_app_ug/testpmd_funcs.rst index ed04532a..9789139a 100644 --- a/dpdk/doc/guides/testpmd_app_ug/testpmd_funcs.rst +++ b/dpdk/doc/guides/testpmd_app_ug/testpmd_funcs.rst @@ -1,5 +1,5 @@ .. BSD LICENSE - Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + Copyright(c) 2010-2016 Intel Corporation. All rights reserved. All rights reserved. Redistribution and use in source and binary forms, with or without @@ -35,7 +35,8 @@ Testpmd Runtime Functions Where the testpmd application is started in interactive mode, (``-i|--interactive``), it displays a prompt that can be used to start and stop forwarding, -configure the application, display statistics, set the Flow Director and other tasks:: +configure the application, display statistics (including the extended NIC +statistics aka xstats) , set the Flow Director and other tasks:: testpmd> @@ -50,10 +51,10 @@ If you type a partial command and hit ```` you get a list of the available testpmd> show port - info [Mul-choice STRING]: show|clear port info|stats|fdir|stat_qmap|dcb_tc X - info [Mul-choice STRING]: show|clear port info|stats|fdir|stat_qmap|dcb_tc all - stats [Mul-choice STRING]: show|clear port info|stats|fdir|stat_qmap|dcb_tc X - stats [Mul-choice STRING]: show|clear port info|stats|fdir|stat_qmap|dcb_tc all + info [Mul-choice STRING]: show|clear port info|stats|xstats|fdir|stat_qmap|dcb_tc|cap X + info [Mul-choice STRING]: show|clear port info|stats|xstats|fdir|stat_qmap|dcb_tc|cap all + stats [Mul-choice STRING]: show|clear port info|stats|xstats|fdir|stat_qmap|dcb_tc|cap X + stats [Mul-choice STRING]: show|clear port info|stats|xstats|fdir|stat_qmap|dcb_tc|cap all ... @@ -85,6 +86,61 @@ These are divided into sections and can be accessed using help, help section or help all : All of the above sections. +Command File Functions +---------------------- + +To facilitate loading large number of commands or to avoid cutting and pasting where not +practical or possible testpmd supports alternative methods for executing commands. + +* If started with the ``--cmdline-file=FILENAME`` command line argument testpmd + will execute all CLI commands contained within the file immediately before + starting packet forwarding or entering interactive mode. + +.. code-block:: console + + ./testpmd -n4 -r2 ... -- -i --cmdline-file=/home/ubuntu/flow-create-commands.txt + Interactive-mode selected + CLI commands to be read from /home/ubuntu/flow-create-commands.txt + Configuring Port 0 (socket 0) + Port 0: 7C:FE:90:CB:74:CE + Configuring Port 1 (socket 0) + Port 1: 7C:FE:90:CB:74:CA + Checking link statuses... + Port 0 Link Up - speed 10000 Mbps - full-duplex + Port 1 Link Up - speed 10000 Mbps - full-duplex + Done + Flow rule #0 created + Flow rule #1 created + ... + ... + Flow rule #498 created + Flow rule #499 created + Read all CLI commands from /home/ubuntu/flow-create-commands.txt + testpmd> + + +* At run-time additional commands can be loaded in bulk by invoking the ``load FILENAME`` + command. + +.. code-block:: console + + testpmd> load /home/ubuntu/flow-create-commands.txt + Flow rule #0 created + Flow rule #1 created + ... + ... + Flow rule #498 created + Flow rule #499 created + Read all CLI commands from /home/ubuntu/flow-create-commands.txt + testpmd> + + +In all cases output from any included command will be displayed as standard output. +Execution will continue until the end of the file is reached regardless of +whether any errors occur. The end user must examine the output to determine if +any failures occurred. + + Control Functions ----------------- @@ -130,7 +186,7 @@ show port Display information for a given port or all ports:: - testpmd> show port (info|stats|fdir|stat_qmap|dcb_tc) (port_id|all) + testpmd> show port (info|stats|xstats|fdir|stat_qmap|dcb_tc|cap) (port_id|all) The available information categories are: @@ -138,12 +194,16 @@ The available information categories are: * ``stats``: RX/TX statistics. +* ``xstats``: RX/TX extended NIC statistics. + * ``fdir``: Flow Director information and statistics. * ``stat_qmap``: Queue statistics mapping. * ``dcb_tc``: DCB information such as TC mapping. +* ``cap``: Supported offload capabilities. + For example: .. code-block:: console @@ -205,7 +265,7 @@ clear port Clear the port statistics for a given port or for all ports:: - testpmd> clear port (info|stats|fdir|stat_qmap) (port_id|all) + testpmd> clear port (info|stats|xstats|fdir|stat_qmap) (port_id|all) For example:: @@ -267,6 +327,9 @@ The available information categories are: This is the default mode. * ``mac``: Changes the source and the destination Ethernet addresses of packets before forwarding them. + Default application behaviour is to set source Ethernet address to that of the transmitting interface, and destination + address to a dummy value (set during init). The user may specify a target destination Ethernet address via the 'eth-peer' or + 'eth-peer-configfile' command-line options. It is not currently possible to specify a specific source Ethernet address. * ``macswap``: MAC swap forwarding mode. Swaps the source and the destination Ethernet addresses of packets before forwarding them. @@ -284,6 +347,13 @@ The available information categories are: * ``ieee1588``: Demonstrate L2 IEEE1588 V2 PTP timestamping for RX and TX. Requires ``CONFIG_RTE_LIBRTE_IEEE1588=y``. +* ``tm``: Traffic Management forwarding mode + Demonstrates the use of ethdev traffic management APIs and softnic PMD for + QoS traffic management. In this mode, 5-level hierarchical QoS scheduler is + available as an default option that can be enabled through CLI. The user can + also modify the default hierarchy or specify the new hierarchy through CLI for + implementing QoS scheduler. Requires ``CONFIG_RTE_LIBRTE_PMD_SOFTNIC=y`` ``CONFIG_RTE_LIBRTE_SCHED=y``. + Note: TX timestamping is only available in the "Full Featured" TX path. To force ``testpmd`` into this mode set ``--txqflags=0``. Example:: @@ -317,6 +387,41 @@ For example:: testpmd> read txd 0 0 4 0x00000001 - 0x24C3C440 / 0x000F0000 - 0x2330003C +ddp get list +~~~~~~~~~~~~ + +Get loaded dynamic device personalization (DDP) package info list:: + + testpmd> ddp get list (port_id) + +ddp get info +~~~~~~~~~~~~ + +Display information about dynamic device personalization (DDP) profile:: + + testpmd> ddp get info (profile_path) + +show vf stats +~~~~~~~~~~~~~ + +Display VF statistics:: + + testpmd> show vf stats (port_id) (vf_id) + +clear vf stats +~~~~~~~~~~~~~~ + +Reset VF statistics:: + + testpmd> clear vf stats (port_id) (vf_id) + +show port pctype mapping +~~~~~~~~~~~~~~~~~~~~~~~~ + +List all items from the pctype mapping table:: + + testpmd> show port (port_id) pctype mapping + Configuration Functions ----------------------- @@ -405,7 +510,7 @@ When retry is enabled, the transmit delay time and number of retries can also be set txpkts ~~~~~~~~~~ -Set the length of each segment of the TX-ONLY packets:: +Set the length of each segment of the TX-ONLY packets or length of packet for FLOWGEN mode:: testpmd> set txpkts (x[,y]*) @@ -473,6 +578,73 @@ For example, to change the port forwarding: RX P=1/Q=0 (socket 0) -> TX P=3/Q=0 (socket 0) peer=02:00:00:00:00:03 RX P=3/Q=0 (socket 0) -> TX P=1/Q=0 (socket 0) peer=02:00:00:00:00:02 +set tx loopback +~~~~~~~~~~~~~~~ + +Enable/disable tx loopback:: + + testpmd> set tx loopback (port_id) (on|off) + +set drop enable +~~~~~~~~~~~~~~~ + +set drop enable bit for all queues:: + + testpmd> set all queues drop (port_id) (on|off) + +set split drop enable (for VF) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +set split drop enable bit for VF from PF:: + + testpmd> set vf split drop (port_id) (vf_id) (on|off) + +set mac antispoof (for VF) +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Set mac antispoof for a VF from the PF:: + + testpmd> set vf mac antispoof (port_id) (vf_id) (on|off) + +set macsec offload +~~~~~~~~~~~~~~~~~~ + +Enable/disable MACsec offload:: + + testpmd> set macsec offload (port_id) on encrypt (on|off) replay-protect (on|off) + testpmd> set macsec offload (port_id) off + +set macsec sc +~~~~~~~~~~~~~ + +Configure MACsec secure connection (SC):: + + testpmd> set macsec sc (tx|rx) (port_id) (mac) (pi) + +.. note:: + + The pi argument is ignored for tx. + Check the NIC Datasheet for hardware limits. + +set macsec sa +~~~~~~~~~~~~~ + +Configure MACsec secure association (SA):: + + testpmd> set macsec sa (tx|rx) (port_id) (idx) (an) (pn) (key) + +.. note:: + + The IDX value must be 0 or 1. + Check the NIC Datasheet for hardware limits. + +set broadcast mode (for VF) +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Set broadcast mode for a VF from the PF:: + + testpmd> set vf broadcast (port_id) (vf_id) (on|off) + vlan set strip ~~~~~~~~~~~~~~ @@ -487,6 +659,34 @@ Set the VLAN strip for a queue on a port:: testpmd> vlan set stripq (on|off) (port_id,queue_id) +vlan set stripq (for VF) +~~~~~~~~~~~~~~~~~~~~~~~~ + +Set VLAN strip for all queues in a pool for a VF from the PF:: + + testpmd> set vf vlan stripq (port_id) (vf_id) (on|off) + +vlan set insert (for VF) +~~~~~~~~~~~~~~~~~~~~~~~~ + +Set VLAN insert for a VF from the PF:: + + testpmd> set vf vlan insert (port_id) (vf_id) (vlan_id) + +vlan set tag (for VF) +~~~~~~~~~~~~~~~~~~~~~ + +Set VLAN tag for a VF from the PF:: + + testpmd> set vf vlan tag (port_id) (vf_id) (on|off) + +vlan set antispoof (for VF) +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Set VLAN antispoof for a VF from the PF:: + + testpmd> set vf vlan antispoof (port_id) (vf_id) (on|off) + vlan set filter ~~~~~~~~~~~~~~~ @@ -657,6 +857,49 @@ Where: Check the NIC Datasheet for hardware limits. +RSS queue region +~~~~~~~~~~~~~~~~ + +Set RSS queue region span on a port:: + + testpmd> set port (port_id) queue-region region_id (value) \ + queue_start_index (value) queue_num (value) + +Set flowtype mapping on a RSS queue region on a port:: + + testpmd> set port (port_id) queue-region region_id (value) flowtype (value) + +where: + +* For the flowtype(pctype) of packet,the specific index for each type has + been defined in file i40e_type.h as enum i40e_filter_pctype. + +Set user priority mapping on a RSS queue region on a port:: + + testpmd> set port (port_id) queue-region UP (value) region_id (value) + +Flush all queue region related configuration on a port:: + + testpmd> set port (port_id) queue-region flush (on|off) + +where: + +* "on"is just an enable function which server for other configuration, + it is for all configuration about queue region from up layer, + at first will only keep in DPDK softwarestored in driver, + only after "flush on", it commit all configuration to HW. + "off" is just clean all configuration about queue region just now, + and restore all to DPDK i40e driver default config when start up. + +Show all queue region related configuration info on a port:: + + testpmd> show port (port_id) queue-region + +.. note:: + + Queue region only support on PF by now, so these command is + only for configuration of queue region on PF port. + csum parse-tunnel ~~~~~~~~~~~~~~~~~ @@ -713,6 +956,106 @@ Display the status of TCP Segmentation Offload:: testpmd> tso show (port_id) +set port - gro +~~~~~~~~~~~~~~ + +Enable or disable GRO in ``csum`` forwarding engine:: + + testpmd> set port gro on|off + +If enabled, the csum forwarding engine will perform GRO on the TCP/IPv4 +packets received from the given port. + +If disabled, packets received from the given port won't be performed +GRO. By default, GRO is disabled for all ports. + +.. note:: + + When enable GRO for a port, TCP/IPv4 packets received from the port + will be performed GRO. After GRO, all merged packets have bad + checksums, since the GRO library doesn't re-calculate checksums for + the merged packets. Therefore, if users want the merged packets to + have correct checksums, please select HW IP checksum calculation and + HW TCP checksum calculation for the port which the merged packets are + transmitted to. + +show port - gro +~~~~~~~~~~~~~~~ + +Display GRO configuration for a given port:: + + testpmd> show port gro + +set gro flush +~~~~~~~~~~~~~ + +Set the cycle to flush the GROed packets from reassembly tables:: + + testpmd> set gro flush + +When enable GRO, the csum forwarding engine performs GRO on received +packets, and the GROed packets are stored in reassembly tables. Users +can use this command to determine when the GROed packets are flushed +from the reassembly tables. + +The ``cycles`` is measured in GRO operation times. The csum forwarding +engine flushes the GROed packets from the tables every ``cycles`` GRO +operations. + +By default, the value of ``cycles`` is 1, which means flush GROed packets +from the reassembly tables as soon as one GRO operation finishes. The value +of ``cycles`` should be in the range of 1 to ``GRO_MAX_FLUSH_CYCLES``. + +Please note that the large value of ``cycles`` may cause the poor TCP/IP +stack performance. Because the GROed packets are delayed to arrive the +stack, thus causing more duplicated ACKs and TCP retransmissions. + +set port - gso +~~~~~~~~~~~~~~ + +Toggle per-port GSO support in ``csum`` forwarding engine:: + + testpmd> set port gso on|off + +If enabled, the csum forwarding engine will perform GSO on supported IPv4 +packets, transmitted on the given port. + +If disabled, packets transmitted on the given port will not undergo GSO. +By default, GSO is disabled for all ports. + +.. note:: + + When GSO is enabled on a port, supported IPv4 packets transmitted on that + port undergo GSO. Afterwards, the segmented packets are represented by + multi-segment mbufs; however, the csum forwarding engine doesn't calculation + of checksums for GSO'd segments in SW. As a result, if users want correct + checksums in GSO segments, they should enable HW checksum calculation for + GSO-enabled ports. + + For example, HW checksum calculation for VxLAN GSO'd packets may be enabled + by setting the following options in the csum forwarding engine: + + testpmd> csum set outer_ip hw + + testpmd> csum set ip hw + + testpmd> csum set tcp hw + +set gso segsz +~~~~~~~~~~~~~ + +Set the maximum GSO segment size (measured in bytes), which includes the +packet header and the packet payload for GSO-enabled ports (global):: + + testpmd> set gso segsz + +show port - gso +~~~~~~~~~~~~~~~ + +Display the status of Generic Segmentation Offload for a given port:: + + testpmd> show port gso + mac_addr add ~~~~~~~~~~~~ @@ -727,13 +1070,27 @@ Remove a MAC address from a port:: testpmd> mac_addr remove (port_id) (XX:XX:XX:XX:XX:XX) -mac_addr add(for VF) -~~~~~~~~~~~~~~~~~~~~ +mac_addr add (for VF) +~~~~~~~~~~~~~~~~~~~~~ Add an alternative MAC address for a VF to a port:: testpmd> mac_add add port (port_id) vf (vf_id) (XX:XX:XX:XX:XX:XX) +mac_addr set +~~~~~~~~~~~~ + +Set the default MAC address for a port:: + + testpmd> mac_addr set (port_id) (XX:XX:XX:XX:XX:XX) + +mac_addr set (for VF) +~~~~~~~~~~~~~~~~~~~~~ + +Set the MAC address for a VF from the PF:: + + testpmd> set vf mac addr (port_id) (vf_id) (XX:XX:XX:XX:XX:XX) + set port-uta ~~~~~~~~~~~~ @@ -758,6 +1115,59 @@ Set the allmulti mode for a port or for all ports:: Same as the ifconfig (8) option. Controls how multicast packets are handled. +set promisc (for VF) +~~~~~~~~~~~~~~~~~~~~ + +Set the unicast promiscuous mode for a VF from PF. +It's supported by Intel i40e NICs now. +In promiscuous mode packets are not dropped if they aren't for the specified MAC address:: + + testpmd> set vf promisc (port_id) (vf_id) (on|off) + +set allmulticast (for VF) +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Set the multicast promiscuous mode for a VF from PF. +It's supported by Intel i40e NICs now. +In promiscuous mode packets are not dropped if they aren't for the specified MAC address:: + + testpmd> set vf allmulti (port_id) (vf_id) (on|off) + +set tx max bandwidth (for VF) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Set TX max absolute bandwidth (Mbps) for a VF from PF:: + + testpmd> set vf tx max-bandwidth (port_id) (vf_id) (max_bandwidth) + +set tc tx min bandwidth (for VF) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Set all TCs' TX min relative bandwidth (%) for a VF from PF:: + + testpmd> set vf tc tx min-bandwidth (port_id) (vf_id) (bw1, bw2, ...) + +set tc tx max bandwidth (for VF) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Set a TC's TX max absolute bandwidth (Mbps) for a VF from PF:: + + testpmd> set vf tc tx max-bandwidth (port_id) (vf_id) (tc_no) (max_bandwidth) + +set tc strict link priority mode +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Set some TCs' strict link priority mode on a physical port:: + + testpmd> set tx strict-link-priority (port_id) (tc_bitmap) + +set tc tx min bandwidth +~~~~~~~~~~~~~~~~~~~~~~~ + +Set all TCs' TX min relative bandwidth (%) globally for all PF and VFs:: + + testpmd> set tc tx min-bandwidth (port_id) (bw1, bw2, ...) + set flow_ctrl rx ~~~~~~~~~~~~~~~~ @@ -779,7 +1189,7 @@ Where: * ``mac_ctrl_frame_fwd``: Enable receiving MAC control frames. -* ``autoneg``: Change the auto-negotiation para mete. +* ``autoneg``: Change the auto-negotiation parameter. set pfc_ctrl rx ~~~~~~~~~~~~~~~ @@ -810,6 +1220,17 @@ For example, to set rx queue 2 on port 0 to mapping 5:: testpmd>set stat_qmap rx 0 2 5 +set xstats-hide-zero +~~~~~~~~~~~~~~~~~~~~ + +Set the option to hide zero values for xstats display:: + + testpmd> set xstats-hide-zero on|off + +.. note:: + + By default, the zero values are displayed for xstats. + set port - rx/tx (for VF) ~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -973,6 +1394,56 @@ Add an E-tag forwarding filter on a port:: Delete an E-tag forwarding filter on a port:: testpmd> E-tag set filter del e-tag-id (value) port (port_id) +ddp add +~~~~~~~ + +Load a dynamic device personalization (DDP) package:: + + testpmd> ddp add (port_id) (package_path[,output_path]) + +ddp del +~~~~~~~ + +Delete a dynamic device personalization package:: + + testpmd> ddp del (port_id) (package_path) + +ptype mapping +~~~~~~~~~~~~~ + +List all items from the ptype mapping table:: + + testpmd> ptype mapping get (port_id) (valid_only) + +Where: + +* ``valid_only``: A flag indicates if only list valid items(=1) or all itemss(=0). + +Replace a specific or a group of software defined ptype with a new one:: + + testpmd> ptype mapping replace (port_id) (target) (mask) (pkt_type) + +where: + +* ``target``: A specific software ptype or a mask to represent a group of software ptypes. + +* ``mask``: A flag indicate if "target" is a specific software ptype(=0) or a ptype mask(=1). + +* ``pkt_type``: The new software ptype to replace the old ones. + +Update hardware defined ptype to software defined packet type mapping table:: + + testpmd> ptype mapping update (port_id) (hw_ptype) (sw_ptype) + +where: + +* ``hw_ptype``: hardware ptype as the index of the ptype mapping table. + +* ``sw_ptype``: software ptype as the value of the ptype mapping table. + +Reset ptype mapping table:: + + testpmd> ptype mapping reset (port_id) Port Functions -------------- @@ -999,7 +1470,7 @@ For example, to move a pci device using ixgbe under DPDK management: .. code-block:: console # Check the status of the available devices. - ./tools/dpdk-devbind.py --status + ./usertools/dpdk-devbind.py --status Network devices using DPDK-compatible driver ============================================ @@ -1011,11 +1482,11 @@ For example, to move a pci device using ixgbe under DPDK management: # Bind the device to igb_uio. - sudo ./tools/dpdk-devbind.py -b igb_uio 0000:0a:00.0 + sudo ./usertools/dpdk-devbind.py -b igb_uio 0000:0a:00.0 # Recheck the status of the devices. - ./tools/dpdk-devbind.py --status + ./usertools/dpdk-devbind.py --status Network devices using DPDK-compatible driver ============================================ 0000:0a:00.0 '82599ES 10-Gigabit' drv=igb_uio unused= @@ -1041,14 +1512,14 @@ For example, to attach a port created by pcap PMD. .. code-block:: console - testpmd> port attach eth_pcap0 + testpmd> port attach net_pcap0 Attaching a new port... - PMD: Initializing pmd_pcap for eth_pcap0 + PMD: Initializing pmd_pcap for net_pcap0 PMD: Creating pcap-backed ethdev on numa socket 0 Port 0 is attached. Now total ports is 1 Done -In this case, identifier is ``eth_pcap0``. +In this case, identifier is ``net_pcap0``. This identifier format is the same as ``--vdev`` format of DPDK applications. For example, to re-attach a bonded port which has been previously detached, @@ -1056,10 +1527,10 @@ the mode and slave parameters must be given. .. code-block:: console - testpmd> port attach eth_bond_0,mode=0,slave=1 + testpmd> port attach net_bond_0,mode=0,slave=1 Attaching a new port... - EAL: Initializing pmd_bond for eth_bond_0 - EAL: Create bonded device eth_bond_0 on port 0 in mode 0 on socket 0. + EAL: Initializing pmd_bond for net_bond_0 + EAL: Create bonded device net_bond_0 on port 0 in mode 0 on socket 0. Port 0 is attached. Now total ports is 1 Done @@ -1107,7 +1578,7 @@ For example, to detach a virtual device port 0. testpmd> port detach 0 Detaching a port... PMD: Closing pcap ethdev on numa socket 0 - Port 'eth_pcap0' is detached. Now total ports is 0 + Port 'net_pcap0' is detached. Now total ports is 0 Done To remove a pci device completely from the system, first detach the port from testpmd. @@ -1118,9 +1589,9 @@ For example, to move a pci device under kernel management: .. code-block:: console - sudo ./tools/dpdk-devbind.py -b ixgbe 0000:0a:00.0 + sudo ./usertools/dpdk-devbind.py -b ixgbe 0000:0a:00.0 - ./tools/dpdk-devbind.py --status + ./usertools/dpdk-devbind.py --status Network devices using DPDK-compatible driver ============================================ @@ -1167,7 +1638,7 @@ port config - speed Set the speed and duplex mode for all ports or a specific port:: - testpmd> port config (port_id|all) speed (10|100|1000|10000|40000|100000|auto) \ + testpmd> port config (port_id|all) speed (10|100|1000|10000|25000|40000|50000|100000|auto) \ duplex (half|full|auto) port config - queues/descriptors @@ -1195,9 +1666,9 @@ Set hardware CRC stripping on or off for all ports:: testpmd> port config all crc-strip (on|off) -CRC stripping is off by default. +CRC stripping is on by default. -The ``on`` option is equivalent to the ``--crc-strip`` command-line option. +The ``off`` option is equivalent to the ``--disable-crc-strip`` command-line option. port config - scatter ~~~~~~~~~~~~~~~~~~~~~~~ @@ -1361,6 +1832,23 @@ Enable/disable the E-tag support:: testpmd> port config (port_id|all) l2-tunnel E-tag (enable|disable) +port config pctype mapping +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Reset pctype mapping table:: + + testpmd> port config (port_id) pctype mapping reset + +Update hardware defined pctype to software defined flow type mapping table:: + + testpmd> port config (port_id) pctype mapping update (pctype_id_0[,pctype_id_1]*) (flow_type_id) + +where: + +* ``pctype_id_x``: hardware pctype id as index of bit in bitmask value of the pctype mapping table. + +* ``flow_type_id``: software flow type id as the index of the pctype mapping table. + Link Bonding Functions ---------------------- @@ -1464,6 +1952,23 @@ For example, to set the link status monitoring polling period of bonded device ( testpmd> set bonding mon_period 5 150 +set bonding lacp dedicated_queue +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Enable dedicated tx/rx queues on bonding devices slaves to handle LACP control plane traffic +when in mode 4 (link-aggregration-802.3ad):: + + testpmd> set bonding lacp dedicated_queues (port_id) (enable|disable) + + +set bonding agg_mode +~~~~~~~~~~~~~~~~~~~~ + +Enable one of the specific aggregators mode when in mode 4 (link-aggregration-802.3ad):: + + testpmd> set bonding agg_mode (port_id) (bandwidth|count|stable) + + show bonding config ~~~~~~~~~~~~~~~~~~~ @@ -1564,11 +2069,269 @@ For example, to set the high bit in the register from the example above:: port 0 PCI register at offset 0xEE00: 0x8000000A (2147483658) +Traffic Management +------------------ + +The following section shows functions for configuring traffic management on +on the ethernet device through the use of generic TM API. + +show port traffic management capability +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Show traffic management capability of the port:: + + testpmd> show port tm cap (port_id) + +show port traffic management capability (hierarchy level) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Show traffic management hierarchy level capability of the port:: + + testpmd> show port tm cap (port_id) (level_id) + +show port traffic management capability (hierarchy node level) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Show the traffic management hierarchy node capability of the port:: + + testpmd> show port tm cap (port_id) (node_id) + +show port traffic management hierarchy node type +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Show the port traffic management hierarchy node type:: + + testpmd> show port tm node type (port_id) (node_id) + +show port traffic management hierarchy node stats +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Show the port traffic management hierarchy node statistics:: + + testpmd> show port tm node stats (port_id) (node_id) (clear) + +where: + +* ``clear``: When this parameter has a non-zero value, the statistics counters + are cleared (i.e. set to zero) immediately after they have been read, + otherwise the statistics counters are left untouched. + +Add port traffic management private shaper profile +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Add the port traffic management private shaper profile:: + + testpmd> add port tm node shaper profile (port_id) (shaper_profile_id) \ + (tb_rate) (tb_size) (packet_length_adjust) + +where: + +* ``shaper_profile id``: Shaper profile ID for the new profile. +* ``tb_rate``: Token bucket rate (bytes per second). +* ``tb_size``: Token bucket size (bytes). +* ``packet_length_adjust``: The value (bytes) to be added to the length of + each packet for the purpose of shaping. This parameter value can be used to + correct the packet length with the framing overhead bytes that are consumed + on the wire. + +Delete port traffic management private shaper profile +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Delete the port traffic management private shaper:: + + testpmd> del port tm node shaper profile (port_id) (shaper_profile_id) + +where: + +* ``shaper_profile id``: Shaper profile ID that needs to be deleted. + +Add port traffic management shared shaper +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Create the port traffic management shared shaper:: + + testpmd> add port tm node shared shaper (port_id) (shared_shaper_id) \ + (shaper_profile_id) + +where: + +* ``shared_shaper_id``: Shared shaper ID to be created. +* ``shaper_profile id``: Shaper profile ID for shared shaper. + +Set port traffic management shared shaper +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Update the port traffic management shared shaper:: + + testpmd> set port tm node shared shaper (port_id) (shared_shaper_id) \ + (shaper_profile_id) + +where: + +* ``shared_shaper_id``: Shared shaper ID to be update. +* ``shaper_profile id``: Shaper profile ID for shared shaper. + +Delete port traffic management shared shaper +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Delete the port traffic management shared shaper:: + + testpmd> del port tm node shared shaper (port_id) (shared_shaper_id) + +where: + +* ``shared_shaper_id``: Shared shaper ID to be deleted. + +Set port traffic management hiearchy node private shaper +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +set the port traffic management hierarchy node private shaper:: + + testpmd> set port tm node shaper profile (port_id) (node_id) \ + (shaper_profile_id) + +where: + +* ``shaper_profile id``: Private shaper profile ID to be enabled on the + hierarchy node. + +Add port traffic management WRED profile +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Create a new WRED profile:: + + testpmd> add port tm node wred profile (port_id) (wred_profile_id) \ + (color_g) (min_th_g) (max_th_g) (maxp_inv_g) (wq_log2_g) \ + (color_y) (min_th_y) (max_th_y) (maxp_inv_y) (wq_log2_y) \ + (color_r) (min_th_r) (max_th_r) (maxp_inv_r) (wq_log2_r) + +where: + +* ``wred_profile id``: Identifier for the newly create WRED profile +* ``color_g``: Packet color (green) +* ``min_th_g``: Minimum queue threshold for packet with green color +* ``max_th_g``: Minimum queue threshold for packet with green color +* ``maxp_inv_g``: Inverse of packet marking probability maximum value (maxp) +* ``wq_log2_g``: Negated log2 of queue weight (wq) +* ``color_y``: Packet color (yellow) +* ``min_th_y``: Minimum queue threshold for packet with yellow color +* ``max_th_y``: Minimum queue threshold for packet with yellow color +* ``maxp_inv_y``: Inverse of packet marking probability maximum value (maxp) +* ``wq_log2_y``: Negated log2 of queue weight (wq) +* ``color_r``: Packet color (red) +* ``min_th_r``: Minimum queue threshold for packet with yellow color +* ``max_th_r``: Minimum queue threshold for packet with yellow color +* ``maxp_inv_r``: Inverse of packet marking probability maximum value (maxp) +* ``wq_log2_r``: Negated log2 of queue weight (wq) + +Delete port traffic management WRED profile +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Delete the WRED profile:: + + testpmd> del port tm node wred profile (port_id) (wred_profile_id) + +Add port traffic management hierarchy nonleaf node +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Add nonleaf node to port traffic management hiearchy:: + + testpmd> add port tm nonleaf node (port_id) (node_id) (parent_node_id) \ + (priority) (weight) (level_id) (shaper_profile_id) \ + (n_sp_priorities) (stats_mask) (n_shared_shapers) \ + [(shared_shaper_0) (shared_shaper_1) ...] \ + +where: + +* ``parent_node_id``: Node ID of the parent. +* ``priority``: Node priority (highest node priority is zero). This is used by + the SP algorithm running on the parent node for scheduling this node. +* ``weight``: Node weight (lowest weight is one). The node weight is relative + to the weight sum of all siblings that have the same priority. It is used by + the WFQ algorithm running on the parent node for scheduling this node. +* ``level_id``: Hiearchy level of the node. +* ``shaper_profile_id``: Shaper profile ID of the private shaper to be used by + the node. +* ``n_sp_priorities``: Number of strict priorities. +* ``stats_mask``: Mask of statistics counter types to be enabled for this node. +* ``n_shared_shapers``: Number of shared shapers. +* ``shared_shaper_id``: Shared shaper id. + +Add port traffic management hierarchy leaf node +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Add leaf node to port traffic management hiearchy:: + + testpmd> add port tm leaf node (port_id) (node_id) (parent_node_id) \ + (priority) (weight) (level_id) (shaper_profile_id) \ + (cman_mode) (wred_profile_id) (stats_mask) (n_shared_shapers) \ + [(shared_shaper_id) (shared_shaper_id) ...] \ + +where: + +* ``parent_node_id``: Node ID of the parent. +* ``priority``: Node priority (highest node priority is zero). This is used by + the SP algorithm running on the parent node for scheduling this node. +* ``weight``: Node weight (lowest weight is one). The node weight is relative + to the weight sum of all siblings that have the same priority. It is used by + the WFQ algorithm running on the parent node for scheduling this node. +* ``level_id``: Hiearchy level of the node. +* ``shaper_profile_id``: Shaper profile ID of the private shaper to be used by + the node. +* ``cman_mode``: Congestion management mode to be enabled for this node. +* ``wred_profile_id``: WRED profile id to be enabled for this node. +* ``stats_mask``: Mask of statistics counter types to be enabled for this node. +* ``n_shared_shapers``: Number of shared shapers. +* ``shared_shaper_id``: Shared shaper id. + +Delete port traffic management hierarchy node +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Delete node from port traffic management hiearchy:: + + testpmd> del port tm node (port_id) (node_id) + +Update port traffic management hierarchy parent node +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Update port traffic management hierarchy parent node:: + + testpmd> set port tm node parent (port_id) (node_id) (parent_node_id) \ + (priority) (weight) + +This function can only be called after the hierarchy commit invocation. Its +success depends on the port support for this operation, as advertised through +the port capability set. This function is valid for all nodes of the traffic +management hierarchy except root node. + +Commit port traffic management hierarchy +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Commit the traffic management hierarchy on the port:: + + testpmd> port tm hierarchy commit (port_id) (clean_on_fail) + +where: + +* ``clean_on_fail``: When set to non-zero, hierarchy is cleared on function + call failure. On the other hand, hierarchy is preserved when this parameter + is equal to zero. + +Set port traffic management default hierarchy (tm forwarding mode) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +set the traffic management default hierarchy on the port:: + + testpmd> set port tm hierarchy default (port_id) + Filter Functions ---------------- This section details the available filter functions that are available. +Note these functions interface the deprecated legacy filtering framework, +superseded by *rte_flow*. See `Flow rules management`_. + ethertype_filter ~~~~~~~~~~~~~~~~~~~~ @@ -1979,3 +2742,742 @@ Set different GRE key length for input set:: For example to set GRE key length for input set to 4 bytes on port 0:: testpmd> global_config 0 gre-key-len 4 + + +.. _testpmd_rte_flow: + +Flow rules management +--------------------- + +Control of the generic flow API (*rte_flow*) is fully exposed through the +``flow`` command (validation, creation, destruction, queries and operation +modes). + +Considering *rte_flow* overlaps with all `Filter Functions`_, using both +features simultaneously may cause undefined side-effects and is therefore +not recommended. + +``flow`` syntax +~~~~~~~~~~~~~~~ + +Because the ``flow`` command uses dynamic tokens to handle the large number +of possible flow rules combinations, its behavior differs slightly from +other commands, in particular: + +- Pressing *?* or the ** key displays contextual help for the current + token, not that of the entire command. + +- Optional and repeated parameters are supported (provided they are listed + in the contextual help). + +The first parameter stands for the operation mode. Possible operations and +their general syntax are described below. They are covered in detail in the +following sections. + +- Check whether a flow rule can be created:: + + flow validate {port_id} + [group {group_id}] [priority {level}] [ingress] [egress] + pattern {item} [/ {item} [...]] / end + actions {action} [/ {action} [...]] / end + +- Create a flow rule:: + + flow create {port_id} + [group {group_id}] [priority {level}] [ingress] [egress] + pattern {item} [/ {item} [...]] / end + actions {action} [/ {action} [...]] / end + +- Destroy specific flow rules:: + + flow destroy {port_id} rule {rule_id} [...] + +- Destroy all flow rules:: + + flow flush {port_id} + +- Query an existing flow rule:: + + flow query {port_id} {rule_id} {action} + +- List existing flow rules sorted by priority, filtered by group + identifiers:: + + flow list {port_id} [group {group_id}] [...] + +- Restrict ingress traffic to the defined flow rules:: + + flow isolate {port_id} {boolean} + +Validating flow rules +~~~~~~~~~~~~~~~~~~~~~ + +``flow validate`` reports whether a flow rule would be accepted by the +underlying device in its current state but stops short of creating it. It is +bound to ``rte_flow_validate()``:: + + flow validate {port_id} + [group {group_id}] [priority {level}] [ingress] [egress] + pattern {item} [/ {item} [...]] / end + actions {action} [/ {action} [...]] / end + +If successful, it will show:: + + Flow rule validated + +Otherwise it will show an error message of the form:: + + Caught error type [...] ([...]): [...] + +This command uses the same parameters as ``flow create``, their format is +described in `Creating flow rules`_. + +Check whether redirecting any Ethernet packet received on port 0 to RX queue +index 6 is supported:: + + testpmd> flow validate 0 ingress pattern eth / end + actions queue index 6 / end + Flow rule validated + testpmd> + +Port 0 does not support TCPv6 rules:: + + testpmd> flow validate 0 ingress pattern eth / ipv6 / tcp / end + actions drop / end + Caught error type 9 (specific pattern item): Invalid argument + testpmd> + +Creating flow rules +~~~~~~~~~~~~~~~~~~~ + +``flow create`` validates and creates the specified flow rule. It is bound +to ``rte_flow_create()``:: + + flow create {port_id} + [group {group_id}] [priority {level}] [ingress] [egress] + pattern {item} [/ {item} [...]] / end + actions {action} [/ {action} [...]] / end + +If successful, it will return a flow rule ID usable with other commands:: + + Flow rule #[...] created + +Otherwise it will show an error message of the form:: + + Caught error type [...] ([...]): [...] + +Parameters describe in the following order: + +- Attributes (*group*, *priority*, *ingress*, *egress* tokens). +- A matching pattern, starting with the *pattern* token and terminated by an + *end* pattern item. +- Actions, starting with the *actions* token and terminated by an *end* + action. + +These translate directly to *rte_flow* objects provided as-is to the +underlying functions. + +The shortest valid definition only comprises mandatory tokens:: + + testpmd> flow create 0 pattern end actions end + +Note that PMDs may refuse rules that essentially do nothing such as this +one. + +**All unspecified object values are automatically initialized to 0.** + +Attributes +^^^^^^^^^^ + +These tokens affect flow rule attributes (``struct rte_flow_attr``) and are +specified before the ``pattern`` token. + +- ``group {group id}``: priority group. +- ``priority {level}``: priority level within group. +- ``ingress``: rule applies to ingress traffic. +- ``egress``: rule applies to egress traffic. + +Each instance of an attribute specified several times overrides the previous +value as shown below (group 4 is used):: + + testpmd> flow create 0 group 42 group 24 group 4 [...] + +Note that once enabled, ``ingress`` and ``egress`` cannot be disabled. + +While not specifying a direction is an error, some rules may allow both +simultaneously. + +Most rules affect RX therefore contain the ``ingress`` token:: + + testpmd> flow create 0 ingress pattern [...] + +Matching pattern +^^^^^^^^^^^^^^^^ + +A matching pattern starts after the ``pattern`` token. It is made of pattern +items and is terminated by a mandatory ``end`` item. + +Items are named after their type (*RTE_FLOW_ITEM_TYPE_* from ``enum +rte_flow_item_type``). + +The ``/`` token is used as a separator between pattern items as shown +below:: + + testpmd> flow create 0 ingress pattern eth / ipv4 / udp / end [...] + +Note that protocol items like these must be stacked from lowest to highest +layer to make sense. For instance, the following rule is either invalid or +unlikely to match any packet:: + + testpmd> flow create 0 ingress pattern eth / udp / ipv4 / end [...] + +More information on these restrictions can be found in the *rte_flow* +documentation. + +Several items support additional specification structures, for example +``ipv4`` allows specifying source and destination addresses as follows:: + + testpmd> flow create 0 ingress pattern eth / ipv4 src is 10.1.1.1 + dst is 10.2.0.0 / end [...] + +This rule matches all IPv4 traffic with the specified properties. + +In this example, ``src`` and ``dst`` are field names of the underlying +``struct rte_flow_item_ipv4`` object. All item properties can be specified +in a similar fashion. + +The ``is`` token means that the subsequent value must be matched exactly, +and assigns ``spec`` and ``mask`` fields in ``struct rte_flow_item`` +accordingly. Possible assignment tokens are: + +- ``is``: match value perfectly (with full bit-mask). +- ``spec``: match value according to configured bit-mask. +- ``last``: specify upper bound to establish a range. +- ``mask``: specify bit-mask with relevant bits set to one. +- ``prefix``: generate bit-mask from a prefix length. + +These yield identical results:: + + ipv4 src is 10.1.1.1 + +:: + + ipv4 src spec 10.1.1.1 src mask 255.255.255.255 + +:: + + ipv4 src spec 10.1.1.1 src prefix 32 + +:: + + ipv4 src is 10.1.1.1 src last 10.1.1.1 # range with a single value + +:: + + ipv4 src is 10.1.1.1 src last 0 # 0 disables range + +Inclusive ranges can be defined with ``last``:: + + ipv4 src is 10.1.1.1 src last 10.2.3.4 # 10.1.1.1 to 10.2.3.4 + +Note that ``mask`` affects both ``spec`` and ``last``:: + + ipv4 src is 10.1.1.1 src last 10.2.3.4 src mask 255.255.0.0 + # matches 10.1.0.0 to 10.2.255.255 + +Properties can be modified multiple times:: + + ipv4 src is 10.1.1.1 src is 10.1.2.3 src is 10.2.3.4 # matches 10.2.3.4 + +:: + + ipv4 src is 10.1.1.1 src prefix 24 src prefix 16 # matches 10.1.0.0/16 + +Pattern items +^^^^^^^^^^^^^ + +This section lists supported pattern items and their attributes, if any. + +- ``end``: end list of pattern items. + +- ``void``: no-op pattern item. + +- ``invert``: perform actions when pattern does not match. + +- ``any``: match any protocol for the current layer. + + - ``num {unsigned}``: number of layers covered. + +- ``pf``: match packets addressed to the physical function. + +- ``vf``: match packets addressed to a virtual function ID. + + - ``id {unsigned}``: destination VF ID. + +- ``port``: device-specific physical port index to use. + + - ``index {unsigned}``: physical port index. + +- ``raw``: match an arbitrary byte string. + + - ``relative {boolean}``: look for pattern after the previous item. + - ``search {boolean}``: search pattern from offset (see also limit). + - ``offset {integer}``: absolute or relative offset for pattern. + - ``limit {unsigned}``: search area limit for start of pattern. + - ``pattern {string}``: byte string to look for. + +- ``eth``: match Ethernet header. + + - ``dst {MAC-48}``: destination MAC. + - ``src {MAC-48}``: source MAC. + - ``type {unsigned}``: EtherType. + +- ``vlan``: match 802.1Q/ad VLAN tag. + + - ``tpid {unsigned}``: tag protocol identifier. + - ``tci {unsigned}``: tag control information. + - ``pcp {unsigned}``: priority code point. + - ``dei {unsigned}``: drop eligible indicator. + - ``vid {unsigned}``: VLAN identifier. + +- ``ipv4``: match IPv4 header. + + - ``tos {unsigned}``: type of service. + - ``ttl {unsigned}``: time to live. + - ``proto {unsigned}``: next protocol ID. + - ``src {ipv4 address}``: source address. + - ``dst {ipv4 address}``: destination address. + +- ``ipv6``: match IPv6 header. + + - ``tc {unsigned}``: traffic class. + - ``flow {unsigned}``: flow label. + - ``proto {unsigned}``: protocol (next header). + - ``hop {unsigned}``: hop limit. + - ``src {ipv6 address}``: source address. + - ``dst {ipv6 address}``: destination address. + +- ``icmp``: match ICMP header. + + - ``type {unsigned}``: ICMP packet type. + - ``code {unsigned}``: ICMP packet code. + +- ``udp``: match UDP header. + + - ``src {unsigned}``: UDP source port. + - ``dst {unsigned}``: UDP destination port. + +- ``tcp``: match TCP header. + + - ``src {unsigned}``: TCP source port. + - ``dst {unsigned}``: TCP destination port. + +- ``sctp``: match SCTP header. + + - ``src {unsigned}``: SCTP source port. + - ``dst {unsigned}``: SCTP destination port. + - ``tag {unsigned}``: validation tag. + - ``cksum {unsigned}``: checksum. + +- ``vxlan``: match VXLAN header. + + - ``vni {unsigned}``: VXLAN identifier. + +- ``e_tag``: match IEEE 802.1BR E-Tag header. + + - ``grp_ecid_b {unsigned}``: GRP and E-CID base. + +- ``nvgre``: match NVGRE header. + + - ``tni {unsigned}``: virtual subnet ID. + +- ``mpls``: match MPLS header. + + - ``label {unsigned}``: MPLS label. + +- ``gre``: match GRE header. + + - ``protocol {unsigned}``: protocol type. + +- ``fuzzy``: fuzzy pattern match, expect faster than default. + + - ``thresh {unsigned}``: accuracy threshold. + +- ``gtp``, ``gtpc``, ``gtpu``: match GTPv1 header. + + - ``teid {unsigned}``: tunnel endpoint identifier. + +Actions list +^^^^^^^^^^^^ + +A list of actions starts after the ``actions`` token in the same fashion as +`Matching pattern`_; actions are separated by ``/`` tokens and the list is +terminated by a mandatory ``end`` action. + +Actions are named after their type (*RTE_FLOW_ACTION_TYPE_* from ``enum +rte_flow_action_type``). + +Dropping all incoming UDPv4 packets can be expressed as follows:: + + testpmd> flow create 0 ingress pattern eth / ipv4 / udp / end + actions drop / end + +Several actions have configurable properties which must be specified when +there is no valid default value. For example, ``queue`` requires a target +queue index. + +This rule redirects incoming UDPv4 traffic to queue index 6:: + + testpmd> flow create 0 ingress pattern eth / ipv4 / udp / end + actions queue index 6 / end + +While this one could be rejected by PMDs (unspecified queue index):: + + testpmd> flow create 0 ingress pattern eth / ipv4 / udp / end + actions queue / end + +As defined by *rte_flow*, the list is not ordered, all actions of a given +rule are performed simultaneously. These are equivalent:: + + queue index 6 / void / mark id 42 / end + +:: + + void / mark id 42 / queue index 6 / end + +All actions in a list should have different types, otherwise only the last +action of a given type is taken into account:: + + queue index 4 / queue index 5 / queue index 6 / end # will use queue 6 + +:: + + drop / drop / drop / end # drop is performed only once + +:: + + mark id 42 / queue index 3 / mark id 24 / end # mark will be 24 + +Considering they are performed simultaneously, opposite and overlapping +actions can sometimes be combined when the end result is unambiguous:: + + drop / queue index 6 / end # drop has no effect + +:: + + drop / dup index 6 / end # same as above + +:: + + queue index 6 / rss queues 6 7 8 / end # queue has no effect + +:: + + drop / passthru / end # drop has no effect + +Note that PMDs may still refuse such combinations. + +Actions +^^^^^^^ + +This section lists supported actions and their attributes, if any. + +- ``end``: end list of actions. + +- ``void``: no-op action. + +- ``passthru``: let subsequent rule process matched packets. + +- ``mark``: attach 32 bit value to packets. + + - ``id {unsigned}``: 32 bit value to return with packets. + +- ``flag``: flag packets. + +- ``queue``: assign packets to a given queue index. + + - ``index {unsigned}``: queue index to use. + +- ``drop``: drop packets (note: passthru has priority). + +- ``count``: enable counters for this rule. + +- ``dup``: duplicate packets to a given queue index. + + - ``index {unsigned}``: queue index to duplicate packets to. + +- ``rss``: spread packets among several queues. + + - ``queues [{unsigned} [...]] end``: queue indices to use. + +- ``pf``: redirect packets to physical device function. + +- ``vf``: redirect packets to virtual device function. + + - ``original {boolean}``: use original VF ID if possible. + - ``id {unsigned}``: VF ID to redirect packets to. + +Destroying flow rules +~~~~~~~~~~~~~~~~~~~~~ + +``flow destroy`` destroys one or more rules from their rule ID (as returned +by ``flow create``), this command calls ``rte_flow_destroy()`` as many +times as necessary:: + + flow destroy {port_id} rule {rule_id} [...] + +If successful, it will show:: + + Flow rule #[...] destroyed + +It does not report anything for rule IDs that do not exist. The usual error +message is shown when a rule cannot be destroyed:: + + Caught error type [...] ([...]): [...] + +``flow flush`` destroys all rules on a device and does not take extra +arguments. It is bound to ``rte_flow_flush()``:: + + flow flush {port_id} + +Any errors are reported as above. + +Creating several rules and destroying them:: + + testpmd> flow create 0 ingress pattern eth / ipv6 / end + actions queue index 2 / end + Flow rule #0 created + testpmd> flow create 0 ingress pattern eth / ipv4 / end + actions queue index 3 / end + Flow rule #1 created + testpmd> flow destroy 0 rule 0 rule 1 + Flow rule #1 destroyed + Flow rule #0 destroyed + testpmd> + +The same result can be achieved using ``flow flush``:: + + testpmd> flow create 0 ingress pattern eth / ipv6 / end + actions queue index 2 / end + Flow rule #0 created + testpmd> flow create 0 ingress pattern eth / ipv4 / end + actions queue index 3 / end + Flow rule #1 created + testpmd> flow flush 0 + testpmd> + +Non-existent rule IDs are ignored:: + + testpmd> flow create 0 ingress pattern eth / ipv6 / end + actions queue index 2 / end + Flow rule #0 created + testpmd> flow create 0 ingress pattern eth / ipv4 / end + actions queue index 3 / end + Flow rule #1 created + testpmd> flow destroy 0 rule 42 rule 10 rule 2 + testpmd> + testpmd> flow destroy 0 rule 0 + Flow rule #0 destroyed + testpmd> + +Querying flow rules +~~~~~~~~~~~~~~~~~~~ + +``flow query`` queries a specific action of a flow rule having that +ability. Such actions collect information that can be reported using this +command. It is bound to ``rte_flow_query()``:: + + flow query {port_id} {rule_id} {action} + +If successful, it will display either the retrieved data for known actions +or the following message:: + + Cannot display result for action type [...] ([...]) + +Otherwise, it will complain either that the rule does not exist or that some +error occurred:: + + Flow rule #[...] not found + +:: + + Caught error type [...] ([...]): [...] + +Currently only the ``count`` action is supported. This action reports the +number of packets that hit the flow rule and the total number of bytes. Its +output has the following format:: + + count: + hits_set: [...] # whether "hits" contains a valid value + bytes_set: [...] # whether "bytes" contains a valid value + hits: [...] # number of packets + bytes: [...] # number of bytes + +Querying counters for TCPv6 packets redirected to queue 6:: + + testpmd> flow create 0 ingress pattern eth / ipv6 / tcp / end + actions queue index 6 / count / end + Flow rule #4 created + testpmd> flow query 0 4 count + count: + hits_set: 1 + bytes_set: 0 + hits: 386446 + bytes: 0 + testpmd> + +Listing flow rules +~~~~~~~~~~~~~~~~~~ + +``flow list`` lists existing flow rules sorted by priority and optionally +filtered by group identifiers:: + + flow list {port_id} [group {group_id}] [...] + +This command only fails with the following message if the device does not +exist:: + + Invalid port [...] + +Output consists of a header line followed by a short description of each +flow rule, one per line. There is no output at all when no flow rules are +configured on the device:: + + ID Group Prio Attr Rule + [...] [...] [...] [...] [...] + +``Attr`` column flags: + +- ``i`` for ``ingress``. +- ``e`` for ``egress``. + +Creating several flow rules and listing them:: + + testpmd> flow create 0 ingress pattern eth / ipv4 / end + actions queue index 6 / end + Flow rule #0 created + testpmd> flow create 0 ingress pattern eth / ipv6 / end + actions queue index 2 / end + Flow rule #1 created + testpmd> flow create 0 priority 5 ingress pattern eth / ipv4 / udp / end + actions rss queues 6 7 8 end / end + Flow rule #2 created + testpmd> flow list 0 + ID Group Prio Attr Rule + 0 0 0 i- ETH IPV4 => QUEUE + 1 0 0 i- ETH IPV6 => QUEUE + 2 0 5 i- ETH IPV4 UDP => RSS + testpmd> + +Rules are sorted by priority (i.e. group ID first, then priority level):: + + testpmd> flow list 1 + ID Group Prio Attr Rule + 0 0 0 i- ETH => COUNT + 6 0 500 i- ETH IPV6 TCP => DROP COUNT + 5 0 1000 i- ETH IPV6 ICMP => QUEUE + 1 24 0 i- ETH IPV4 UDP => QUEUE + 4 24 10 i- ETH IPV4 TCP => DROP + 3 24 20 i- ETH IPV4 => DROP + 2 24 42 i- ETH IPV4 UDP => QUEUE + 7 63 0 i- ETH IPV6 UDP VXLAN => MARK QUEUE + testpmd> + +Output can be limited to specific groups:: + + testpmd> flow list 1 group 0 group 63 + ID Group Prio Attr Rule + 0 0 0 i- ETH => COUNT + 6 0 500 i- ETH IPV6 TCP => DROP COUNT + 5 0 1000 i- ETH IPV6 ICMP => QUEUE + 7 63 0 i- ETH IPV6 UDP VXLAN => MARK QUEUE + testpmd> + +Toggling isolated mode +~~~~~~~~~~~~~~~~~~~~~~ + +``flow isolate`` can be used to tell the underlying PMD that ingress traffic +must only be injected from the defined flow rules; that no default traffic +is expected outside those rules and the driver is free to assign more +resources to handle them. It is bound to ``rte_flow_isolate()``:: + + flow isolate {port_id} {boolean} + +If successful, enabling or disabling isolated mode shows either:: + + Ingress traffic on port [...] + is now restricted to the defined flow rules + +Or:: + + Ingress traffic on port [...] + is not restricted anymore to the defined flow rules + +Otherwise, in case of error:: + + Caught error type [...] ([...]): [...] + +Mainly due to its side effects, PMDs supporting this mode may not have the +ability to toggle it more than once without reinitializing affected ports +first (e.g. by exiting testpmd). + +Enabling isolated mode:: + + testpmd> flow isolate 0 true + Ingress traffic on port 0 is now restricted to the defined flow rules + testpmd> + +Disabling isolated mode:: + + testpmd> flow isolate 0 false + Ingress traffic on port 0 is not restricted anymore to the defined flow rules + testpmd> + +Sample QinQ flow rules +~~~~~~~~~~~~~~~~~~~~~~ + +Before creating QinQ rule(s) the following commands should be issued to enable QinQ:: + + testpmd> port stop 0 + testpmd> vlan set qinq on 0 + +The above command sets the inner and outer TPID's to 0x8100. + +To change the TPID's the following commands should be used:: + + testpmd> vlan set outer tpid 0xa100 0 + testpmd> vlan set inner tpid 0x9100 0 + testpmd> port start 0 + +Validate and create a QinQ rule on port 0 to steer traffic to a VF queue in a VM. + +:: + + testpmd> flow validate 0 ingress pattern eth / vlan tci is 123 / + vlan tci is 456 / end actions vf id 1 / queue index 0 / end + Flow rule #0 validated + + testpmd> flow create 0 ingress pattern eth / vlan tci is 4 / + vlan tci is 456 / end actions vf id 123 / queue index 0 / end + Flow rule #0 created + + testpmd> flow list 0 + ID Group Prio Attr Rule + 0 0 0 i- ETH VLAN VLAN=>VF QUEUE + +Validate and create a QinQ rule on port 0 to steer traffic to a queue on the host. + +:: + + testpmd> flow validate 0 ingress pattern eth / vlan tci is 321 / + vlan tci is 654 / end actions pf / queue index 0 / end + Flow rule #1 validated + + testpmd> flow create 0 ingress pattern eth / vlan tci is 321 / + vlan tci is 654 / end actions pf / queue index 1 / end + Flow rule #1 created + + testpmd> flow list 0 + ID Group Prio Attr Rule + 0 0 0 i- ETH VLAN VLAN=>VF QUEUE + 1 0 0 i- ETH VLAN VLAN=>PF QUEUE diff --git a/dpdk/doc/guides/tools/cryptoperf.rst b/dpdk/doc/guides/tools/cryptoperf.rst new file mode 100644 index 00000000..7e126770 --- /dev/null +++ b/dpdk/doc/guides/tools/cryptoperf.rst @@ -0,0 +1,472 @@ +.. BSD LICENSE + Copyright(c) 2016 Intel Corporation. All rights reserved. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +dpdk-test-crypto-perf Application +================================= + +The ``dpdk-test-crypto-perf`` tool is a Data Plane Development Kit (DPDK) +utility that allows measuring performance parameters of PMDs available in the +crypto tree. There are available two measurement types: throughput and latency. +User can use multiply cores to run tests on but only +one type of crypto PMD can be measured during single application +execution. Cipher parameters, type of device, type of operation and +chain mode have to be specified in the command line as application +parameters. These parameters are checked using device capabilities +structure. + +Limitations +----------- +On hardware devices the cycle-count doesn't always represent the actual offload +cost. The cycle-count only represents the offload cost when the hardware +accelerator is not fully loaded, when loaded the cpu cycles freed up by the +offload are still consumed by the test tool and included in the cycle-count. +These cycles are consumed by retries and inefficient API calls enqueuing and +dequeuing smaller bursts than specified by the cmdline parameter. This results +in a larger cycle-count measurement and should not be interpreted as an offload +cost measurement. Using "pmd-cyclecount" mode will give a better idea of +actual costs of hardware acceleration. + +On hardware devices the throughput measurement is not necessarily the maximum +possible for the device, e.g. it may be necessary to use multiple cores to keep +the hardware accelerator fully loaded and so measure maximum throughput. + +Compiling the Application +------------------------- + +**Step 1: PMD setting** + +The ``dpdk-test-crypto-perf`` tool depends on crypto device drivers PMD which +are disabled by default in the build configuration file ``common_base``. +The crypto device drivers PMD which should be tested can be enabled by setting:: + + CONFIG_RTE_LIBRTE_PMD_=y + +Setting example for open ssl PMD:: + + CONFIG_RTE_LIBRTE_PMD_OPENSSL=y + +**Step 2: Linearization setting** + +It is possible linearized input segmented packets just before crypto operation +for devices which doesn't support scatter-gather, and allows to measure +performance also for this use case. + +To set on the linearization options add below definition to the +``cperf_ops.h`` file:: + + #define CPERF_LINEARIZATION_ENABLE + +**Step 3: Build the application** + +Execute the ``dpdk-setup.sh`` script to build the DPDK library together with the +``dpdk-test-crypto-perf`` applcation. + +Initially, the user must select a DPDK target to choose the correct target type +and compiler options to use when building the libraries. +The user must have all libraries, modules, updates and compilers installed +in the system prior to this, +as described in the earlier chapters in this Getting Started Guide. + +Running the Application +----------------------- + +The tool application has a number of command line options: + +.. code-block:: console + + dpdk-test-crypto-perf [EAL Options] -- [Application Options] + +EAL Options +~~~~~~~~~~~ + +The following are the EAL command-line options that can be used in conjunction +with the ``dpdk-test-crypto-perf`` applcation. +See the DPDK Getting Started Guides for more information on these options. + +* ``-c `` or ``-l `` + + Set the hexadecimal bitmask of the cores to run on. The corelist is a + list cores to use. + +* ``-w `` + + Add a PCI device in white list. + +* ``--vdev `` + + Add a virtual device. + +Appication Options +~~~~~~~~~~~~~~~~~~ + +The following are the appication command-line options: + +* ``--ptest type`` + + Set test type, where ``type`` is one of the following:: + + throughput + latency + verify + pmd-cyclecount + +* ``--silent`` + + Disable options dump. + +* ``--pool-sz `` + + Set the number of mbufs to be allocated in the mbuf pool. + +* ``--total-ops `` + + Set the number of total operations performed. + +* ``--burst-sz `` + + Set the number of packets per burst. + + This can be set as: + * Single value (i.e. ``--burst-sz 16``) + * Range of values, using the following structure ``min:inc:max``, + where ``min`` is minimum size, ``inc`` is the increment size and ``max`` + is the maximum size (i.e. ``--burst-sz 16:2:32``) + * List of values, up to 32 values, separated in commas (i.e. ``--burst-sz 16,24,32``) + +* ``--buffer-sz `` + + Set the size of single packet (plaintext or ciphertext in it). + + This can be set as: + * Single value (i.e. ``--buffer-sz 16``) + * Range of values, using the following structure ``min:inc:max``, + where ``min`` is minimum size, ``inc`` is the increment size and ``max`` + is the maximum size (i.e. ``--buffer-sz 16:2:32``) + * List of values, up to 32 values, separated in commas (i.e. ``--buffer-sz 32,64,128``) + + +* ``--segment-sz `` + + Set the size of the segment to use, for Scatter Gather List testing. + By default, it is set to the size of the maximum buffer size, including the digest size, + so a single segment is created. + +* ``--devtype `` + + Set device type, where ``name`` is one of the following:: + + crypto_null + crypto_aesni_mb + crypto_aesni_gcm + crypto_openssl + crypto_qat + crypto_snow3g + crypto_kasumi + crypto_zuc + crypto_dpaa_sec + crypto_dpaa2_sec + crypto_armv8 + crypto_scheduler + crypto_mrvl + +* ``--optype `` + + Set operation type, where ``name`` is one of the following:: + + cipher-only + auth-only + cipher-then-auth + auth-then-cipher + aead + + For GCM/CCM algorithms you should use aead flag. + +* ``--sessionless`` + + Enable session-less crypto operations mode. + +* ``--out-of-place`` + + Enable out-of-place crypto operations mode. + +* ``--test-file `` + + Set test vector file path. See the Test Vector File chapter. + +* ``--test-name `` + + Set specific test name section in the test vector file. + +* ``--cipher-algo `` + + Set cipher algorithm name, where ``name`` is one of the following:: + + 3des-cbc + 3des-ecb + 3des-ctr + aes-cbc + aes-ctr + aes-ecb + aes-f8 + aes-xts + arc4 + null + kasumi-f8 + snow3g-uea2 + zuc-eea3 + +* ``--cipher-op `` + + Set cipher operation mode, where ``mode`` is one of the following:: + + encrypt + decrypt + +* ``--cipher-key-sz `` + + Set the size of cipher key. + +* ``--cipher-iv-sz `` + + Set the size of cipher iv. + +* ``--auth-algo `` + + Set authentication algorithm name, where ``name`` is one + of the following:: + + 3des-cbc + aes-cbc-mac + aes-cmac + aes-gmac + aes-xcbc-mac + md5 + md5-hmac + sha1 + sha1-hmac + sha2-224 + sha2-224-hmac + sha2-256 + sha2-256-hmac + sha2-384 + sha2-384-hmac + sha2-512 + sha2-512-hmac + kasumi-f9 + snow3g-uia2 + zuc-eia3 + +* ``--auth-op `` + + Set authentication operation mode, where ``mode`` is one of + the following:: + + verify + generate + +* ``--auth-key-sz `` + + Set the size of authentication key. + +* ``--auth-iv-sz `` + + Set the size of auth iv. + +* ``--aead-algo `` + + Set AEAD algorithm name, where ``name`` is one + of the following:: + + aes-ccm + aes-gcm + +* ``--aead-op `` + + Set AEAD operation mode, where ``mode`` is one of + the following:: + + encrypt + decrypt + +* ``--aead-key-sz `` + + Set the size of AEAD key. + +* ``--aead-iv-sz `` + + Set the size of AEAD iv. + +* ``--aead-aad-sz `` + + Set the size of AEAD aad. + +* ``--digest-sz `` + + Set the size of digest. + +* ``--desc-nb `` + + Set number of descriptors for each crypto device. + +* ``--pmd-cyclecount-delay-ms `` + + Add a delay (in milliseconds) between enqueue and dequeue in + pmd-cyclecount benchmarking mode (useful when benchmarking + hardware acceleration). + +* ``--csv-friendly`` + + Enable test result output CSV friendly rather than human friendly. + +Test Vector File +~~~~~~~~~~~~~~~~ + +The test vector file is a text file contain information about test vectors. +The file is made of the sections. The first section doesn't have header. +It contain global information used in each test variant vectors - +typically information about plaintext, ciphertext, cipher key, aut key, +initial vector. All other sections begin header. +The sections contain particular information typically digest. + +**Format of the file:** + +Each line beginig with sign '#' contain comment and it is ignored by parser:: + + # + +Header line is just name in square bracket:: + + [
] + +Data line contain information tocken then sign '=' and +a string of bytes in C byte array format:: + + = + +**Tockens list:** + +* ``plaintext`` + + Original plaintext to be crypted. + +* ``ciphertext`` + + Encrypted plaintext string. + +* ``cipher_key`` + + Key used in cipher operation. + +* ``auth_key`` + + Key used in auth operation. + +* ``cipher_iv`` + + Cipher Initial Vector. + +* ``auth_iv`` + + Auth Initial Vector. + +* ``aad`` + + Additional data. + +* ``digest`` + + Digest string. + +Examples +-------- + +Call application for performance throughput test of single Aesni MB PMD +for cipher encryption aes-cbc and auth generation sha1-hmac, +one million operations, burst size 32, packet size 64:: + + dpdk-test-crypto-perf -l 6-7 --vdev crypto_aesni_mb -w 0000:00:00.0 -- + --ptest throughput --devtype crypto_aesni_mb --optype cipher-then-auth + --cipher-algo aes-cbc --cipher-op encrypt --cipher-key-sz 16 --auth-algo + sha1-hmac --auth-op generate --auth-key-sz 64 --digest-sz 12 + --total-ops 10000000 --burst-sz 32 --buffer-sz 64 + +Call application for performance latency test of two Aesni MB PMD executed +on two cores for cipher encryption aes-cbc, ten operations in silent mode:: + + dpdk-test-crypto-perf -l 4-7 --vdev crypto_aesni_mb1 + --vdev crypto_aesni_mb2 -w 0000:00:00.0 -- --devtype crypto_aesni_mb + --cipher-algo aes-cbc --cipher-key-sz 16 --cipher-iv-sz 16 + --cipher-op encrypt --optype cipher-only --silent + --ptest latency --total-ops 10 + +Call application for verification test of single open ssl PMD +for cipher encryption aes-gcm and auth generation aes-gcm,ten operations +in silent mode, test vector provide in file "test_aes_gcm.data" +with packet verification:: + + dpdk-test-crypto-perf -l 4-7 --vdev crypto_openssl -w 0000:00:00.0 -- + --devtype crypto_openssl --aead-algo aes-gcm --aead-key-sz 16 + --aead-iv-sz 16 --aead-op encrypt --aead-aad-sz 16 --digest-sz 16 + --optype aead --silent --ptest verify --total-ops 10 + --test-file test_aes_gcm.data + +Test vector file for cipher algorithm aes cbc 256 with authorization sha:: + + # Global Section + plaintext = + 0xff, 0xca, 0xfb, 0xf1, 0x38, 0x20, 0x2f, 0x7b, 0x24, 0x98, 0x26, 0x7d, 0x1d, 0x9f, 0xb3, 0x93, + 0xd9, 0xef, 0xbd, 0xad, 0x4e, 0x40, 0xbd, 0x60, 0xe9, 0x48, 0x59, 0x90, 0x67, 0xd7, 0x2b, 0x7b, + 0x8a, 0xe0, 0x4d, 0xb0, 0x70, 0x38, 0xcc, 0x48, 0x61, 0x7d, 0xee, 0xd6, 0x35, 0x49, 0xae, 0xb4, + 0xaf, 0x6b, 0xdd, 0xe6, 0x21, 0xc0, 0x60, 0xce, 0x0a, 0xf4, 0x1c, 0x2e, 0x1c, 0x8d, 0xe8, 0x7b + ciphertext = + 0x77, 0xF9, 0xF7, 0x7A, 0xA3, 0xCB, 0x68, 0x1A, 0x11, 0x70, 0xD8, 0x7A, 0xB6, 0xE2, 0x37, 0x7E, + 0xD1, 0x57, 0x1C, 0x8E, 0x85, 0xD8, 0x08, 0xBF, 0x57, 0x1F, 0x21, 0x6C, 0xAD, 0xAD, 0x47, 0x1E, + 0x0D, 0x6B, 0x79, 0x39, 0x15, 0x4E, 0x5B, 0x59, 0x2D, 0x76, 0x87, 0xA6, 0xD6, 0x47, 0x8F, 0x82, + 0xB8, 0x51, 0x91, 0x32, 0x60, 0xCB, 0x97, 0xDE, 0xBE, 0xF0, 0xAD, 0xFC, 0x23, 0x2E, 0x22, 0x02 + cipher_key = + 0xE4, 0x23, 0x33, 0x8A, 0x35, 0x64, 0x61, 0xE2, 0x49, 0x03, 0xDD, 0xC6, 0xB8, 0xCA, 0x55, 0x7A, + 0xd0, 0xe7, 0x4b, 0xfb, 0x5d, 0xe5, 0x0c, 0xe7, 0x6f, 0x21, 0xb5, 0x52, 0x2a, 0xbb, 0xc7, 0xf7 + auth_key = + 0xaf, 0x96, 0x42, 0xf1, 0x8c, 0x50, 0xdc, 0x67, 0x1a, 0x43, 0x47, 0x62, 0xc7, 0x04, 0xab, 0x05, + 0xf5, 0x0c, 0xe7, 0xa2, 0xa6, 0x23, 0xd5, 0x3d, 0x95, 0xd8, 0xcd, 0x86, 0x79, 0xf5, 0x01, 0x47, + 0x4f, 0xf9, 0x1d, 0x9d, 0x36, 0xf7, 0x68, 0x1a, 0x64, 0x44, 0x58, 0x5d, 0xe5, 0x81, 0x15, 0x2a, + 0x41, 0xe4, 0x0e, 0xaa, 0x1f, 0x04, 0x21, 0xff, 0x2c, 0xf3, 0x73, 0x2b, 0x48, 0x1e, 0xd2, 0xf7 + cipher_iv = + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F + # Section sha 1 hmac buff 32 + [sha1_hmac_buff_32] + digest = + 0x36, 0xCA, 0x49, 0x6A, 0xE3, 0x54, 0xD8, 0x4F, 0x0B, 0x76, 0xD8, 0xAA, 0x78, 0xEB, 0x9D, 0x65, + 0x2C, 0xCA, 0x1F, 0x97 + # Section sha 256 hmac buff 32 + [sha256_hmac_buff_32] + digest = + 0x1C, 0xB2, 0x3D, 0xD1, 0xF9, 0xC7, 0x6C, 0x49, 0x2E, 0xDA, 0x94, 0x8B, 0xF1, 0xCF, 0x96, 0x43, + 0x67, 0x50, 0x39, 0x76, 0xB5, 0xA1, 0xCE, 0xA1, 0xD7, 0x77, 0x10, 0x07, 0x43, 0x37, 0x05, 0xB4 diff --git a/dpdk/doc/guides/tools/devbind.rst b/dpdk/doc/guides/tools/devbind.rst new file mode 100644 index 00000000..cabd99dd --- /dev/null +++ b/dpdk/doc/guides/tools/devbind.rst @@ -0,0 +1,143 @@ +.. BSD LICENSE + Copyright(c) 2016 Canonical Limited. All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +dpdk-devbind Application +======================== + +The ``dpdk-devbind`` tool is a Data Plane Development Kit (DPDK) utility +that helps binding and unbinding devices from specific drivers. +As well as checking their status in that regard. + + +Running the Application +----------------------- + +The tool has a number of command line options: + +.. code-block:: console + + dpdk-devbind [options] DEVICE1 DEVICE2 .... + +OPTIONS +------- + +* ``--help, --usage`` + + Display usage information and quit + +* ``-s, --status`` + + Print the current status of all known network interfaces. + For each device, it displays the PCI domain, bus, slot and function, + along with a text description of the device. Depending upon whether the + device is being used by a kernel driver, the ``igb_uio`` driver, or no + driver, other relevant information will be displayed: + - the Linux interface name e.g. ``if=eth0`` + - the driver being used e.g. ``drv=igb_uio`` + - any suitable drivers not currently using that device e.g. ``unused=igb_uio`` + NOTE: if this flag is passed along with a bind/unbind option, the + status display will always occur after the other operations have taken + place. + +* ``-b driver, --bind=driver`` + + Select the driver to use or "none" to unbind the device + +* ``-u, --unbind`` + + Unbind a device (Equivalent to ``-b none``) + +* ``--force`` + + By default, devices which are used by Linux - as indicated by having + routes in the routing table - cannot be modified. Using the ``--force`` + flag overrides this behavior, allowing active links to be forcibly + unbound. + WARNING: This can lead to loss of network connection and should be used + with caution. + + +.. warning:: + + Due to the way VFIO works, there are certain limitations to which devices can be used with VFIO. + Mainly it comes down to how IOMMU groups work. + Any Virtual Function device can be used with VFIO on its own, but physical devices will require either all ports bound to VFIO, + or some of them bound to VFIO while others not being bound to anything at all. + + If your device is behind a PCI-to-PCI bridge, the bridge will then be part of the IOMMU group in which your device is in. + Therefore, the bridge driver should also be unbound from the bridge PCI device for VFIO to work with devices behind the bridge. + +.. warning:: + + While any user can run the ``dpdk-devbind.py`` script to view the status of the network ports, + binding or unbinding network ports requires root privileges. + + +Examples +-------- + +To display current device status:: + + dpdk-devbind --status + +To bind eth1 from the current driver and move to use igb_uio:: + + dpdk-devbind --bind=igb_uio eth1 + +To unbind 0000:01:00.0 from using any driver:: + + dpdk-devbind -u 0000:01:00.0 + +To bind 0000:02:00.0 and 0000:02:00.1 to the ixgbe kernel driver:: + + dpdk-devbind -b ixgbe 02:00.0 02:00.1 + +To check status of all network ports, assign one to the igb_uio driver and check status again:: + + # Check the status of the available devices. + dpdk-devbind --status + Network devices using DPDK-compatible driver + ============================================ + + + Network devices using kernel driver + =================================== + 0000:0a:00.0 '82599ES 10-Gigabit' if=eth2 drv=ixgbe unused= + + + # Bind the device to igb_uio. + sudo dpdk-devbind -b igb_uio 0000:0a:00.0 + + + # Recheck the status of the devices. + dpdk-devbind --status + Network devices using DPDK-compatible driver + ============================================ + 0000:0a:00.0 '82599ES 10-Gigabit' drv=igb_uio unused= diff --git a/dpdk/doc/guides/tools/img/eventdev_order_atq_test.svg b/dpdk/doc/guides/tools/img/eventdev_order_atq_test.svg new file mode 100644 index 00000000..a36eafa7 --- /dev/null +++ b/dpdk/doc/guides/tools/img/eventdev_order_atq_test.svg @@ -0,0 +1,1576 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + test: order_atq(all types queue) +   + + + + + + + + + producer_flow_seq + + producer maintains per flow sequence number + + + flow 0 + + flow 1 + flow 2 + + flow n + + + + producer0 + all_types_queue0 + worker 0 + port n+1 + worker 1 + worker 2 + worker n + port 0 + port 1 + port 2 + port n + expected_flow_seq + per flow expected sequence number + + + flow 0 + + flow 1 + flow 2 + + flow n + + + + + + + + + + + + + + + + + + dequeue_ordered_flow(step 2) + enqueue ordered flow(step 1) + + produce ordered flows(step 0) + change to atomic flow and enqueue(step 3) + dequeue_atomic_flow (step 4) + + + + + + diff --git a/dpdk/doc/guides/tools/img/eventdev_order_queue_test.svg b/dpdk/doc/guides/tools/img/eventdev_order_queue_test.svg new file mode 100644 index 00000000..3e2113cd --- /dev/null +++ b/dpdk/doc/guides/tools/img/eventdev_order_queue_test.svg @@ -0,0 +1,1673 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + test: order_queue +   + + + + + + + + + + + + + + producer_flow_seq + + producer maintains per flow sequence number + + + flow 0 + + flow 1 + flow 2 + + flow n + + + + producer0 + ordered queue 0 + atomic queue 1 + worker 0 + port n+1 + worker 1 + worker 2 + worker n + port 0 + port 1 + port 2 + port n + expected_flow_seq + per flow expected sequence number + + + flow 0 + + flow 1 + flow 2 + + flow n + + + + + + enqueue ordered flow(step 1) + produce ordered flows(step 0) + + dequeue_ordered_flow(step 2) + + + + + + + + + + change to atomic flow and enqueue(step 3) + + dequeue_atomic_flow (step 4) + + + + diff --git a/dpdk/doc/guides/tools/img/eventdev_perf_atq_test.svg b/dpdk/doc/guides/tools/img/eventdev_perf_atq_test.svg new file mode 100644 index 00000000..404b572c --- /dev/null +++ b/dpdk/doc/guides/tools/img/eventdev_perf_atq_test.svg @@ -0,0 +1,3188 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + +   + + + + + worker 0 + worker 1 + worker n + port 0 + port 1 + port n + + producer 0 + + port n+1 + test: perf_atq(all types queues) + + producer 1 + + port n+2 + + producer m + + port n+m + + + + total queues = number of producers + All workers are linked to all queues + + + + + + + all types queue 0 + all types queue 1 + all types queue n + stage 0 + stage 1 + stage n + + + + + + + + + + + + + + + + diff --git a/dpdk/doc/guides/tools/img/eventdev_perf_queue_test.svg b/dpdk/doc/guides/tools/img/eventdev_perf_queue_test.svg new file mode 100644 index 00000000..f87ee367 --- /dev/null +++ b/dpdk/doc/guides/tools/img/eventdev_perf_queue_test.svg @@ -0,0 +1,2599 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + +   + + + + + + worker 0 + worker 1 + worker n + port 0 + port 1 + port n + + producer 0 + + + q0 + q1 + q2 + qs-1 + + + + port n+1 + + + + test: perf_queue + + + + producer 1 + + qs + qs+1 + qs+2 + q2s-1 + + + + port n+2 + + + + + + + + producer m + + q2s + q2s+1 + q2s+2 + q3s-1 + + + + port n+m + + + + + + + + + + + + + total queues = number of stages * number of producers + All workers are linked to all queues + + diff --git a/dpdk/doc/guides/tools/index.rst b/dpdk/doc/guides/tools/index.rst new file mode 100644 index 00000000..c9133ec8 --- /dev/null +++ b/dpdk/doc/guides/tools/index.rst @@ -0,0 +1,43 @@ +.. BSD LICENSE + Copyright(c) 2016 Canonical Limited. All rights reserved. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +DPDK Tools User Guides +====================== + +.. toctree:: + :maxdepth: 2 + :numbered: + + proc_info + pdump + pmdinfo + devbind + cryptoperf + testeventdev diff --git a/dpdk/doc/guides/tools/pdump.rst b/dpdk/doc/guides/tools/pdump.rst new file mode 100644 index 00000000..16502f7a --- /dev/null +++ b/dpdk/doc/guides/tools/pdump.rst @@ -0,0 +1,152 @@ +.. BSD LICENSE + Copyright(c) 2016 Intel Corporation. All rights reserved. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +.. _pdump_tool: + +dpdk-pdump Application +====================== + +The ``dpdk-pdump`` tool is a Data Plane Development Kit (DPDK) tool that runs as +a DPDK secondary process and is capable of enabling packet capture on dpdk ports. + + .. Note:: + * The ``dpdk-pdump`` tool can only be used in conjunction with a primary + application which has the packet capture framework initialized already. + In dpdk, only the ``testpmd`` is modified to initialize packet capture + framework, other applications remain untouched. So, if the ``dpdk-pdump`` + tool has to be used with any application other than the testpmd, user + needs to explicitly modify that application to call packet capture + framework initialization code. Refer ``app/test-pmd/testpmd.c`` + code to see how this is done. + + * The ``dpdk-pdump`` tool depends on libpcap based PMD which is disabled + by default in the build configuration files, + owing to an external dependency on the libpcap development files + which must be installed on the board. + Once the libpcap development files are installed, the libpcap based PMD + can be enabled by setting CONFIG_RTE_LIBRTE_PMD_PCAP=y and recompiling the DPDK. + + +Running the Application +----------------------- + +The tool has a number of command line options: + +.. code-block:: console + + ./build/app/dpdk-pdump -- + --pdump '(port= | device_id=), + (queue=), + (rx-dev= | + tx-dev=), + [ring-size=], + [mbuf-size=], + [total-num-mbufs=]' + [--server-socket-path=] + [--client-socket-path=] + +The ``--pdump`` command line option is mandatory and it takes various sub arguments which are described in +below section. + + .. Note:: + + * Parameters inside the parentheses represents mandatory parameters. + + * Parameters inside the square brackets represents optional parameters. + + * Multiple instances of ``--pdump`` can be passed to capture packets on different port and queue combinations. + +The ``--server-socket-path`` command line option is optional. This represents the server socket directory. +If no value is passed default values are used i.e. ``/var/run/.dpdk/`` for root users and ``~/.dpdk/`` +for non root users. + +The ``--client-socket-path`` command line option is optional. This represents the client socket directory. +If no value is passed default values are used i.e. ``/var/run/.dpdk/`` for root users and ``~/.dpdk/`` +for non root users. + + +The ``--pdump`` parameters +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``port``: +Port id of the eth device on which packets should be captured. + +``device_id``: +PCI address (or) name of the eth device on which packets should be captured. + + .. Note:: + + * As of now the ``dpdk-pdump`` tool cannot capture the packets of virtual devices + in the primary process due to a bug in the ethdev library. Due to this bug, in a multi process context, + when the primary and secondary have different ports set, then the secondary process + (here the ``dpdk-pdump`` tool) overwrites the ``rte_eth_devices[]`` entries of the primary process. + +``queue``: +Queue id of the eth device on which packets should be captured. The user can pass a queue value of ``*`` to enable +packet capture on all queues of the eth device. + +``rx-dev``: +Can be either a pcap file name or any Linux iface. + +``tx-dev``: +Can be either a pcap file name or any Linux iface. + + .. Note:: + + * To receive ingress packets only, ``rx-dev`` should be passed. + + * To receive egress packets only, ``tx-dev`` should be passed. + + * To receive ingress and egress packets separately ``rx-dev`` and ``tx-dev`` + should both be passed with the different file names or the Linux iface names. + + * To receive ingress and egress packets together, ``rx-dev`` and ``tx-dev`` + should both be passed with the same file name or the same Linux iface name. + +``ring-size``: +Size of the ring. This value is used internally for ring creation. The ring will be used to enqueue the packets from +the primary application to the secondary. This is an optional parameter with default size 16384. + +``mbuf-size``: +Size of the mbuf data. This is used internally for mempool creation. Ideally this value must be same as +the primary application's mempool's mbuf data size which is used for packet RX. This is an optional parameter with +default size 2176. + +``total-num-mbufs``: +Total number mbufs in mempool. This is used internally for mempool creation. This is an optional parameter with default +value 65535. + + +Example +------- + +.. code-block:: console + + $ sudo ./build/app/dpdk-pdump -- --pdump 'port=0,queue=*,rx-dev=/tmp/rx.pcap' diff --git a/dpdk/doc/guides/tools/pmdinfo.rst b/dpdk/doc/guides/tools/pmdinfo.rst new file mode 100644 index 00000000..efbf527f --- /dev/null +++ b/dpdk/doc/guides/tools/pmdinfo.rst @@ -0,0 +1,55 @@ +.. BSD LICENSE + Copyright(c) 2016 Canonical Limited. All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +dpdk-pmdinfo Application +======================== + +The ``dpdk-pmdinfo`` tool is a Data Plane Development Kit (DPDK) utility that +can dump a PMDs hardware support info. + + +Running the Application +----------------------- + +The tool has a number of command line options: + +.. code-block:: console + + dpdk-pmdinfo [-hrtp] [-d + + -h, --help Show a short help message and exit + -r, --raw Dump as raw json strings + -d FILE, --pcidb=FILE Specify a pci database to get vendor names from + -t, --table Output information on hw support as a hex table + -p, --plugindir Scan dpdk for autoload plugins + +.. Note:: + + * Parameters inside the square brackets represents optional parameters. diff --git a/dpdk/doc/guides/tools/proc_info.rst b/dpdk/doc/guides/tools/proc_info.rst new file mode 100644 index 00000000..fd17e278 --- /dev/null +++ b/dpdk/doc/guides/tools/proc_info.rst @@ -0,0 +1,70 @@ +.. BSD LICENSE + Copyright(c) 2015 Intel Corporation. All rights reserved. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +dpdk-procinfo Application +========================= + +The dpdk-procinfo application is a Data Plane Development Kit (DPDK) application +that runs as a DPDK secondary process and is capable of retrieving port +statistics, resetting port statistics and printing DPDK memory information. +This application extends the original functionality that was supported by +dump_cfg. + +Running the Application +----------------------- +The application has a number of command line options: + +.. code-block:: console + + ./$(RTE_TARGET)/app/dpdk-procinfo -- -m | [-p PORTMASK] [--stats | --xstats | + --stats-reset | --xstats-reset] + +Parameters +~~~~~~~~~~ +**-p PORTMASK**: Hexadecimal bitmask of ports to configure. + +**--stats** +The stats parameter controls the printing of generic port statistics. If no +port mask is specified stats are printed for all DPDK ports. + +**--xstats** +The xstats parameter controls the printing of extended port statistics. If no +port mask is specified xstats are printed for all DPDK ports. + +**--stats-reset** +The stats-reset parameter controls the resetting of generic port statistics. If +no port mask is specified, the generic stats are reset for all DPDK ports. + +**--xstats-reset** +The xstats-reset parameter controls the resetting of extended port statistics. +If no port mask is specified xstats are reset for all DPDK ports. + +**-m**: Print DPDK memory information. diff --git a/dpdk/doc/guides/tools/testeventdev.rst b/dpdk/doc/guides/tools/testeventdev.rst new file mode 100644 index 00000000..5aa2237b --- /dev/null +++ b/dpdk/doc/guides/tools/testeventdev.rst @@ -0,0 +1,455 @@ +.. BSD LICENSE + Copyright(c) 2017 Cavium, Inc. All rights reserved. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Cavium, Inc nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +dpdk-test-eventdev Application +============================== + +The ``dpdk-test-eventdev`` tool is a Data Plane Development Kit (DPDK) +application that allows exercising various eventdev use cases. +This application has a generic framework to add new eventdev based test cases to +verify functionality and measure the performance parameters of DPDK eventdev +devices. + +Compiling the Application +------------------------- + +**Build the application** + +Execute the ``dpdk-setup.sh`` script to build the DPDK library together with the +``dpdk-test-eventdev`` application. + +Initially, the user must select a DPDK target to choose the correct target type +and compiler options to use when building the libraries. +The user must have all libraries, modules, updates and compilers installed +in the system prior to this, +as described in the earlier chapters in this Getting Started Guide. + +Running the Application +----------------------- + +The application has a number of command line options: + +.. code-block:: console + + dpdk-test-eventdev [EAL Options] -- [application options] + +EAL Options +~~~~~~~~~~~ + +The following are the EAL command-line options that can be used in conjunction +with the ``dpdk-test-eventdev`` application. +See the DPDK Getting Started Guides for more information on these options. + +* ``-c `` or ``-l `` + + Set the hexadecimal bitmask of the cores to run on. The corelist is a + list of cores to use. + +* ``--vdev `` + + Add a virtual eventdev device. + +Application Options +~~~~~~~~~~~~~~~~~~~ + +The following are the application command-line options: + +* ``--verbose`` + + Set verbose level. Default is 1. Value > 1 displays more details. + +* ``--dev `` + + Set the device id of the event device. + +* ``--test `` + + Set test name, where ``name`` is one of the following:: + + order_queue + order_atq + perf_queue + perf_atq + +* ``--socket_id `` + + Set the socket id of the application resources. + +* ``--pool-sz `` + + Set the number of mbufs to be allocated from the mempool. + +* ``--plcores `` + + Set the list of cores to be used as producers. + +* ``--wlcores `` + + Set the list of cores to be used as workers. + +* ``--stlist `` + + Set the scheduled type of each stage where ``type_list`` size + determines the number of stages used in the test application. + Each type_list member can be one of the following:: + + P or p : Parallel schedule type + O or o : Ordered schedule type + A or a : Atomic schedule type + + Application expects the ``type_list`` in comma separated form (i.e. ``--stlist o,a,a,a``) + +* ``--nb_flows `` + + Set the number of flows to produce. + +* ``--nb_pkts `` + + Set the number of packets to produce. 0 implies no limit. + +* ``--worker_deq_depth `` + + Set the dequeue depth of the worker. + +* ``--fwd_latency`` + + Perform forward latency measurement. + +* ``--queue_priority`` + + Enable queue priority. + + +Eventdev Tests +-------------- + +ORDER_QUEUE Test +~~~~~~~~~~~~~~~~ + +This is a functional test case that aims at testing the following: + +#. Verify the ingress order maintenance. +#. Verify the exclusive(atomic) access to given atomic flow per eventdev port. + +.. _table_eventdev_order_queue_test: + +.. table:: Order queue test eventdev configuration. + + +---+--------------+----------------+------------------------+ + | # | Items | Value | Comments | + | | | | | + +===+==============+================+========================+ + | 1 | nb_queues | 2 | q0(ordered), q1(atomic)| + | | | | | + +---+--------------+----------------+------------------------+ + | 2 | nb_producers | 1 | | + | | | | | + +---+--------------+----------------+------------------------+ + | 3 | nb_workers | >= 1 | | + | | | | | + +---+--------------+----------------+------------------------+ + | 4 | nb_ports | nb_workers + | Workers use port 0 to | + | | | 1 | port n-1. Producer uses| + | | | | port n | + +---+--------------+----------------+------------------------+ + +.. _figure_eventdev_order_queue_test: + +.. figure:: img/eventdev_order_queue_test.* + + order queue test operation. + +The order queue test configures the eventdev with two queues and an event +producer to inject the events to q0(ordered) queue. Both q0(ordered) and +q1(atomic) are linked to all the workers. + +The event producer maintains a sequence number per flow and injects the events +to the ordered queue. The worker receives the events from ordered queue and +forwards to atomic queue. Since the events from an ordered queue can be +processed in parallel on the different workers, the ingress order of events +might have changed on the downstream atomic queue enqueue. On enqueue to the +atomic queue, the eventdev PMD driver reorders the event to the original +ingress order(i.e producer ingress order). + +When the event is dequeued from the atomic queue by the worker, this test +verifies the expected sequence number of associated event per flow by comparing +the free running expected sequence number per flow. + +Application options +^^^^^^^^^^^^^^^^^^^ + +Supported application command line options are following:: + + --verbose + --dev + --test + --socket_id + --pool_sz + --plcores + --wlcores + --nb_flows + --nb_pkts + --worker_deq_depth + +Example +^^^^^^^ + +Example command to run order queue test: + +.. code-block:: console + + sudo build/app/dpdk-test-eventdev --vdev=event_sw0 -- \ + --test=order_queue --plcores 1 --wlcores 2,3 + + +ORDER_ATQ Test +~~~~~~~~~~~~~~ + +This test verifies the same aspects of ``order_queue`` test, the difference is +the number of queues used, this test operates on a single ``all types queue(atq)`` +instead of two different queues for ordered and atomic. + +.. _table_eventdev_order_atq_test: + +.. table:: Order all types queue test eventdev configuration. + + +---+--------------+----------------+------------------------+ + | # | Items | Value | Comments | + | | | | | + +===+==============+================+========================+ + | 1 | nb_queues | 1 | q0(all types queue) | + | | | | | + +---+--------------+----------------+------------------------+ + | 2 | nb_producers | 1 | | + | | | | | + +---+--------------+----------------+------------------------+ + | 3 | nb_workers | >= 1 | | + | | | | | + +---+--------------+----------------+------------------------+ + | 4 | nb_ports | nb_workers + | Workers use port 0 to | + | | | 1 | port n-1.Producer uses | + | | | | port n. | + +---+--------------+----------------+------------------------+ + +.. _figure_eventdev_order_atq_test: + +.. figure:: img/eventdev_order_atq_test.* + + order all types queue test operation. + +Application options +^^^^^^^^^^^^^^^^^^^ + +Supported application command line options are following:: + + --verbose + --dev + --test + --socket_id + --pool_sz + --plcores + --wlcores + --nb_flows + --nb_pkts + --worker_deq_depth + +Example +^^^^^^^ + +Example command to run order ``all types queue`` test: + +.. code-block:: console + + sudo build/app/dpdk-test-eventdev --vdev=event_octeontx -- \ + --test=order_atq --plcores 1 --wlcores 2,3 + + +PERF_QUEUE Test +~~~~~~~~~~~~~~~ + +This is a performance test case that aims at testing the following: + +#. Measure the number of events can be processed in a second. +#. Measure the latency to forward an event. + +.. _table_eventdev_perf_queue_test: + +.. table:: Perf queue test eventdev configuration. + + +---+--------------+----------------+-----------------------------------------+ + | # | Items | Value | Comments | + | | | | | + +===+==============+================+=========================================+ + | 1 | nb_queues | nb_producers * | Queues will be configured based on the | + | | | nb_stages | user requested sched type list(--stlist)| + +---+--------------+----------------+-----------------------------------------+ + | 2 | nb_producers | >= 1 | Selected through --plcores command line | + | | | | argument. | + +---+--------------+----------------+-----------------------------------------+ + | 3 | nb_workers | >= 1 | Selected through --wlcores command line | + | | | | argument | + +---+--------------+----------------+-----------------------------------------+ + | 4 | nb_ports | nb_workers + | Workers use port 0 to port n-1. | + | | | nb_producers | Producers use port n to port p | + +---+--------------+----------------+-----------------------------------------+ + +.. _figure_eventdev_perf_queue_test: + +.. figure:: img/eventdev_perf_queue_test.* + + perf queue test operation. + +The perf queue test configures the eventdev with Q queues and P ports, where +Q and P is a function of the number of workers, the number of producers and +number of stages as mentioned in :numref:`table_eventdev_perf_queue_test`. + +The user can choose the number of workers, the number of producers and number of +stages through the ``--wlcores``, ``--plcores`` and the ``--stlist`` application +command line arguments respectively. + +The producer(s) injects the events to eventdev based the first stage sched type +list requested by the user through ``--stlist`` the command line argument. + +Based on the number of stages to process(selected through ``--stlist``), +The application forwards the event to next upstream queue and terminates when it +reaches the last stage in the pipeline. On event termination, application +increments the number events processed and print periodically in one second +to get the number of events processed in one second. + +When ``--fwd_latency`` command line option selected, the application inserts +the timestamp in the event on the first stage and then on termination, it +updates the number of cycles to forward a packet. The application uses this +value to compute the average latency to a forward packet. + +Application options +^^^^^^^^^^^^^^^^^^^ + +Supported application command line options are following:: + + --verbose + --dev + --test + --socket_id + --pool_sz + --plcores + --wlcores + --stlist + --nb_flows + --nb_pkts + --worker_deq_depth + --fwd_latency + --queue_priority + +Example +^^^^^^^ + +Example command to run perf queue test: + +.. code-block:: console + + sudo build/app/dpdk-test-eventdev -c 0xf -s 0x1 --vdev=event_sw0 -- \ + --test=perf_queue --plcores=2 --wlcore=3 --stlist=p --nb_pkts=0 + + +PERF_ATQ Test +~~~~~~~~~~~~~~~ + +This is a performance test case that aims at testing the following with +``all types queue`` eventdev scheme. + +#. Measure the number of events can be processed in a second. +#. Measure the latency to forward an event. + +.. _table_eventdev_perf_atq_test: + +.. table:: Perf all types queue test eventdev configuration. + + +---+--------------+----------------+-----------------------------------------+ + | # | Items | Value | Comments | + | | | | | + +===+==============+================+=========================================+ + | 1 | nb_queues | nb_producers | Queues will be configured based on the | + | | | | user requested sched type list(--stlist)| + +---+--------------+----------------+-----------------------------------------+ + | 2 | nb_producers | >= 1 | Selected through --plcores command line | + | | | | argument. | + +---+--------------+----------------+-----------------------------------------+ + | 3 | nb_workers | >= 1 | Selected through --wlcores command line | + | | | | argument | + +---+--------------+----------------+-----------------------------------------+ + | 4 | nb_ports | nb_workers + | Workers use port 0 to port n-1. | + | | | nb_producers | Producers use port n to port p | + +---+--------------+----------------+-----------------------------------------+ + +.. _figure_eventdev_perf_atq_test: + +.. figure:: img/eventdev_perf_atq_test.* + + perf all types queue test operation. + + +The ``all types queues(atq)`` perf test configures the eventdev with Q queues +and P ports, where Q and P is a function of the number of workers and number of +producers as mentioned in :numref:`table_eventdev_perf_atq_test`. + + +The atq queue test functions as same as ``perf_queue`` test. The difference +is, It uses, ``all type queue scheme`` instead of separate queues for each +stage and thus reduces the number of queues required to realize the use case +and enables flow pinning as the event does not move to the next queue. + + +Application options +^^^^^^^^^^^^^^^^^^^ + +Supported application command line options are following:: + + --verbose + --dev + --test + --socket_id + --pool_sz + --plcores + --wlcores + --stlist + --nb_flows + --nb_pkts + --worker_deq_depth + --fwd_latency + +Example +^^^^^^^ + +Example command to run perf ``all types queue`` test: + +.. code-block:: console + + sudo build/app/dpdk-test-eventdev --vdev=event_octeontx -- \ + --test=perf_atq --plcores=2 --wlcore=3 --stlist=p --nb_pkts=0 diff --git a/dpdk/doc/guides/xen/img/dpdk_xen_pkt_switch.png b/dpdk/doc/guides/xen/img/dpdk_xen_pkt_switch.png deleted file mode 100644 index 32a6d1618820e930b17231f8fe38aab5d5c5d370..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 163842 zcmV)2K+M01P)Px#32;bRa{vGi!~g&e!~vBn4jTXf02y>eSaefwW^{L9a%BKbX=8G4b8lvJAWvpy zX=7!7?KN=#0RNClL_t(|UhI7ZfE>q__FLu^qs6ji%K}TbC4hO zUGOdpm;b^!$8j7m#uUfQAls59Sqv7lR?7_k_q|uu(=*ev)3dvh<-{YecXp<`y6V+q zs=}{+^{aCJc^CEEc;oM-tFudb@_HmMKVS0l^2EP-dV2INKgq-QZ@k`sz#9;F0|J>L zKpyeW$0vGv@&k~NS7wI50f0@2U-yQYo|k9C-URpW{YtK+JMP8V9_nOFe!svofRl+a z-+{WM2Qs&-t4oUTo0!wx-6;hH1yaZdeDTX)l36on%2CJ6mbUhG2w?!d8&Jx_#eTgW zXovjzU&{&h3$q4lud|Hv27C?@p#O;<^Ln6z1Zx|pjmndPG=3Q=5Hs=AFxcW=gL`^% zAnCZ5zCLy^!_IN?aThvJo8QlIbjlFCR8G1fsEdp9WzX(CCZLOo3g!JDc)uKVUd`v#xQ$WLTmLmM@+goP>Iv=03w6s7VcT00yr-Oc;w6(RvqV8}4 zIv)Z$FDGqnpz;47W4zv-U;h|ud}9@FK;Y1ZK;}qYTv3-UDZ^pGTvs30^&B~_izvjK zc{~%#8)KQ|k~dNZI^s#8+nX%yO)h#XG*{Rk?CZlJ1_a)KKqLgx zY1n%j!hO4cB!i-kX5dFCmOg1+j-gk~8U{hpLndT&wodqYj8e-#XjAs;egKe+HYSZn>Kc}^}_OEH1ccflF`0FIaBO3gvg7>Y*VXiP~JT?vyu zo_|Dm*bMSHGW6IVDpEF|>0x1zsz8bRig{%eXd|-JO%%C{0~ru^E{D=}5XrQj1FeV} zif(`t&|r&!0!Cr(Fd~ntOjwmV#Dudh!k2(4f!QkJYH9~#i@d_%8B7n9oP(!C7}h5C zlU|XSL~;;-LpXT1jy>+sncG2F{u_^Y4I#i%QsW=k`4aj8y!@&}CF3c&&`IW=^ zdmB3qHl`(-TSwn2^~W?sa$dz7Hw-)kA{D*^z_SN>=3OcfL(hPXl%zSdbMlrL#3SiD^aX`fz;T>Hps-H4k)t7V%6;zcATJ3i5rKdXZ=V;)?z4fu z#_x~9G38lNp9M?Ty(MCSN8?8RQvt05h87j@O8Ug9xv+7G-?V-rL*H*tPz*Pw06zMX z;$kT*EY!4gowHkCVs}H>;0YuwYqCj#@IB#)hZ#_oF{tqPQZ)+5&W;Xg#}A1jW^aL* zAPX%FfM8~RG2w#}I?lmr+xsB=Qr4GN_K-?(QOr3Lj6wpLZ@ZBl(bm=`9i6BoNP78) z43MG_6Qci!mkqoJC1P_Gv9l3Zj?nF^=O--5=v~7&YyeSOT%IorQ4Ww*}%t(^H862%4*f1ziHl zLIK_l9aJtIK-h%~7s|?2D`nTNT|nXvwGH{b&N~(PlLU00pic|dkMRoHq^E;H`67j? zX05Z~fV4oc6*|OhM~t9QiV!H!#5n~U^g>3}?A#*{u;>CPy7O8AjV}Bv!`dP6yW0Vm9%+ZthMa%Q--Kyik0}DuO^Wtm z1`-DGjC(?5FBwbewj#XPL8q#!O2&>IBPX7Cio&O%sY6;@o9uq>iE5SH7mkGj{aoOa zOK2u1i=e+4-naV?ija+*S|)~{dU3RHj+%FDaL1DO{DrCn->2RUqw+1wAAfylvW7qyu2dJ%gr~IVvO}^G7>ldf7BwT2dm3ESAmBN1@CBX z>Xa9re_Gb9T_wAA?+$pHw|@iovuX;ApOWk5OA$cdRoDz(3|MqS0m^TaPC6Aj`MyY- zUlD#?50<~-b!*FjzezUY9H1|z>GfK4ZOGPlm)Gf_@H@kE($*a&HC zYL({JR_qqmSWt*@F}uDhczq+$zTc&?j%i~20nd&a$qQ-jxpysicS_%%+>bj+6hQKl zk|HTBES9_P{kz=r*L!5mrY+La+75)!_9{vfUCPkoP&1H$0!SL97%Z`pVsj9>rv8a3RdqDpB*9TwC*ke11bSAfe`%d zQA!MX8$K__I@`hQdh%OLK=)K)j-^u2*#a)pA#EUL5?emTQ%x0cDpPVv(o?Lz1TV%q-(bCXZuZ)vp{tgh zzBiihWmfID7j~8# zErjWiz=*K$SQFa`L0kk+80llg&9}*^)HkjlL&_*r)m7j&rt z<9MV~q|S2 zz@fBsLU{pmg#lEI`GJ>$9~8mTRai)o6!t)w!QXlLozhl_YoW}PNEJ9e1bE7`(!paw zEFVf2SDudx{`ALN<>V9Q$u(DBC8NfSl)A=7lq*50Qb+h}gp+X}>o?DZUd)nqG3WzD zEd3qV6C1U@(vo~>ZEKb<{QKwS@n>F;Bab{rX3jfXO3MeUc1G7RH>$IcqB9#*AWWd; zb(2t_%@sokcR@f?2ozIbbhSx)0rh4IHGWRF2_>=p_H=Ao8ME6>k)g$z`sBb1Kq80h zwhkKm-yVBG?z#VA`SO=OD;Jz|mNeEksphZoe~ib{1syjw06V0zybzjWgM9NF-7dXIkRimylxj^nQz^*@9T$N3I|Z6L;~_m92XKRM z?!spR?VYg6R}BS(0y^pH>QI+qmuCA@xp6TojHj?pYqA@5FG2+aGicbeQ|^28dAa|A zhvieB`P?uUtCixpcjr0{zVlvzXNom@oE=o)n}T>Oct?V&fqr% zcou~ue`cdpC;{K$ zMJ-aqIe-VT$3}h^6@Cm+2!31C428cKAS{6uu6BL$VgjCJA5$h2^pFp6ufeC#qPk|+ zPPy%#N92Kr9+S_1>2tE+q)E`K7=V2)vYmzY}u3`Eg*kudy4=kAaw9Q zygFK2q(t4WtxBhjl)V#lg8S5B{378e8(|4c2=@OTR3X01@fgYeN7g= zyjm`N)7znVR7qodr|d^JIKqw!h)zze+H8aj_!U6u0r4v_6qjS2UDDAG*Ha;w6Ex!j z#z%?=<98w8(+W$y1BVX1rt1{Rpb_Kcgt22~ z$E$$Pr@tuqh@qc<=4mRV8Gta@nAPxF*5tpILT%umJ+KN)F7wNZi`j5B<$I$W@C@P= zXqh<}tu3wcu}^$PcGfh?MQ?f=+|4DjzY(fuD?}y)pQle_k;yaa1`T#WK`$Vahk#ZL zgdpw$uP7|6gkp-Y5!Be$hB`Rc;8~+p4vVm`w9?$vZF|+@me*B<&-1}wIF9#X*|QQ^ z)Tz*bCyGK3)+VjM>e1D5;@GLOVdcwm{YO75Kl|Sw%G{Yp!DSBxUoFWL-V?z)C`45T zXg>?svFIsDFXZPw{RPZ0(=_V>R|Z~ky3gnAf%hI=KMlD6N0n=a3FlT+p$k=9T495tQ>A|Dh`YTXw)nN zW_asDjTjhNSPZ>z^jZuaccd&Bf4Hn$u~VW`j0OB8Z_$=AEZHN5d4S$f+Pd^#N z4l5pUbSy^i_A7|PI!j8+%@!GPMO9dosDplw1cr9jJ9YdRf1 ze2)KA6&);m?1({#N3}>31bqRl|6zbn8)DK$B_)zi;a>-V-PtTdtBR$pYNYJmU*|H# zsZg2QiLfEONO-BrhL#a`9=k?%kQj7f~(E-KHYn z;@l36`;BAa0df&81Nvao@KT1%gvW8PVqj;NX|&f4UQshL+)&HVVM=*fnf&=LcgTuW zYvrnUydSh)0Gx(Z3mhv1?@&up)0kZkg%n>ZEnV8a1;Ag}eR@^_R8qt`hsl2MheqU) z6hI)48&NK;@F-Oj7b4NIUiLR2Gy|bAa%i!XmJO91dm)HaKkb0HHBT8|#Q99Z2x6X5 zZ_P5LMcxDAV6bh<%;RL+wk>kQ4ZoM;fB4^Q0*M&7&wb9a@3dg`TC53UlCS;su6toQ zuaqm_{vK)SfF}+@nKY@MG39Zz6{uJwP*i7)&kEpaWClCGFVwg)!)zks#{oY3nh{gR zjsS*-4IeI@ogGq9+5`AENiE{URPINPfMHcsAvFz6@Pe|TCF70+R5I?H5|e-xg;xi8 zFn+p}r{!bb2)zLq(`L_??VC5r4gdFhnfv4K(&v%gC^e#m8J5ox11->KK-K1oI6zxpEOp!@q<6hzS=r@>sv3CBd1K3Z~p8K*|le% zloTSygruo1BvupxL78Ad4bwl@(>RgH;;d96NakQ0FfV)H4r^@bk{QRHEKl6~7kT`N zr{#jPPlc6O3k#iAE@Qeb*)0XN!m#lxBr4zVhd;`+8OKV+;L%VlTEQ%6Sn#O%{_1mtHY1603_kMhbY~QFr*8ddO=2_Y3M9{ zA)GqF*qYk9T65ES{U_M{PV)UL-DB2hwe;3+q=QzV`} z85R6Uwfs2iNTVT!M<&yXBWHI2bcN{s%3dC*R)_ni$)cEBQ)83da_j9fd)~>26PHK} zcu5P~&_(bNQQcGW(qt*4&ooXBI)uoPLc~#Ntuc1ojWI4d>m(#!HpriDeFQ>gh`jUa zbLD}@mda%p%~JvR!=K!Y7pf~PM=4tWt?qv?|H1;mr$uhL z-l1diU)b8fd%~&!9s@wdCE3xrMi;H+@mzE1`Lbr! zEAqg7%VgsCYPs&(3*~{oKO@&%brRg^d*nwyy&ZAyGWpPZE|w-(-9P&I-CAa%c|qiV zDLgz}9_sOsj`|)Z7La{VOMwTj17kF{bjq;{&X8vxxI-2#dRdN{bBr1qhxUt#QzFE| zeSBS5&;_uZwr<-lE7z}*c_*I*#*ITgxP6R`(2b=iYZ_b+gf?9`UJzv55E^(u)Fn$_ zk+)uZfs7t8R3?nBlxldi*RI(FVxBC6Dhg%xwDGcLBQ#cIP8Kss4Ug=?=I5aVA+H?n zZXja^EXV@(Fz0!m6pCI;AlWIurw!@IQ58U3$Lw%qLsyjHnZrlQgAYF{7oLAQC8|!D zS%*FzoDHd1O0{*{R#^j=`K)>806uNf3dT@~$r$($}N;{$XJGlMfUDv_HR;2IaR zyliUqMR47{{L)ME_UkT|vBL(*gi(W~3NTr}a-}SoKU+o*FO}(250edrEiz^D2sj`b zr4(@JY(~rp`%qX|3SwpGpiMPwW_oriBTT%!xDLElO8A&7_Li_LsTd?fMvVh}9+z{^ zIURe>Fag)XtD=*}G7IClk|geR={pl&5y-Zyy?@WWN_|Di-T3qA`LKeNeOt$XaBd5+e1)04_Kk8tKCSBD|rAc~}uRaLlDBB%V zY9RX>$)Z$ZJ0aj-TJ(xsbiqkdRaPuBj+~%w?v1am1MM6mBSsEGR`Do=gkVXI9wYTo z5{faeHVDzY5?F`j6%f))GBbf{ukd-1ID&mBNFwLb+N&F~VE$ zut8WCG4ux-ZG-Fj9d=#}d46toHge~YWAep4VKz|o9k&6+hXz6IiMldxH z2^!oI*L=e?E;?ia((4s6c@SXhx9^ntmRdP{%1D_!Zn&)5v{Uxf*QvW`_4;ixdGbia zojatqVL!4q;h2KIVf476QVb;D(gUuAxB_YSz5EUnWi-CMD(F zmMw`GJsb4nkQv$D7zsR8e3 zr)7e;b8EXb>QQWTZtys8B6vCqpl0JoDHk==E7xz93kuJWDTj}i!>1i4%T}(2V%#Rp zEzPoVQ>#p#GEVnt&py%%5>ZRKW%|)m;Qrn#+xAf)K|lvgA`=cP(P|Zisrav2zI8E(mAMm2u^)sj%_Zx%bLb z3F@ZD58*TFFr?RRByRP9=2x%VBIloTtV}s#l)~t_XI80oTi1Y)S#7gSpFSDoQG--@ zYHy{Vpi?TU$_2TuvL8GrpFE6JB=jUIUscn=!Hfo(=_G1vyCgnp939*&8cN{rWf*@u z#&2sk@1R+wnLP|~aN&Smql!p~pE)8rBwgG|K)^0A`?&xJ#AFF(4wE@^bC^Z#1r7xs ztuzYgLUrl({_3H72O93eWgF!D)8=5z4!QI07qo5;_4%@7#YQ>mh)HtB;q&B$7Z%GY zb7sKu?UK_@KNb~Ot#ac%&%@Qtnnxy#u|x<~V>rG`1d0?NmTAgZ3uDfOSXT>TC#}$m z%b+=O9+qnseRt}rs0FvRqTUe1RZviZxU<1WlO_WJMgpKfW3o%A%*qYM%M^i?2rTJ? zx_bu%>x)Yl%Yp@f4{E6Iyz5cO?jk9I)v#pQ3Yh|-J!$edd2!JqnK^43tnNJ2GndHR zBU z1Y#-waGnN?)d{7f2n+3m^^3w(@G~TyVf?zrX7pg#ELU8*Ksu3x_R8utQdU-tr2sxl zH^{tWW??@%Wyz8iGH2>|JiSRyJ!PJ3S-VbZUai4AN|mQ-XqL&PKIcT|WS&N!HLiUT z$VaGHV4ry+qbf9jUkmD?TY&>=`G+*ee&JXS^cL4rSLGpO$9dLxND62VOkbDCSY}~f zB@#Qpq4Ue^a1TXDBAaQKTAPTULKRM#<5k>Wy%Q>5LD3Ld{PL^v)^iug_C0%L)#@GS zzBO2?%gd33hFC^TvrHH}RMxEDEf=443M|25sjq96W1$faA6yQX5V8+dDKJ5z!dmB) zaOHY2>kTQ+RTCj-T|yb4M~NB96hf*2D>0g8m?_`d6Ldl&WHEj@2n-1q@GhuU31Mup zXA(ICF5SagtZkXqAxhTgs_%h9Q%_N$Z-OqpPI>92)pGSkr^?1{b+UTxF4U7&!mU?@ z8q9XtTeMe33?3zGSM8Nk7n~s7a3PLAd!iU^XmMzG?>gHH<%JhF z%hjjPmRs*%BsKfnWa7|5sYW9d@{##7rb}H-t?b#}E`yFPlo?Zp$@D3MW%6)jUBkQZ z>ZWZ{Qi^0$jcIFh1x$uo7yqN+W}_1JVzA5~CcvuN5cDl|9tRB59YlCb1W<)ST{bA8U@Tp08I52y`GLYUE*|k( z-^Wv}1G=0nW&^sRyv2JElH#nn$-*ziTh%)GND>|2(`!+LQnq}#{r(qZ&wjKSL~N?0 zuuPs>xKg(6Zj{|fnA@Apw<{ zATi9~p{h)1LPq#V;saq7!y}+`!roKyU_OWu8(YE{aD1|jUr#i{_j?~&bzIomL(*Rn zpQX;lCYXL<9KW#_!2(?U*<-*< zTx=%Y;F_o6g$#v?3T)?t2My%2nBZ z>{)a+#OKevxB~RkiR|PydGN`X!6qtD?uATl#AnOPE2T!uDoqILj-+fAKT|%yS!sEq z_1t7&uL}lBhie?7)*=&6mF|1T82>D#caDcPU!)`UFX#btXh()S4Hw3NHN<4pnKVc5 zv(!L=4#XN{>j+5hMLv=TK~0a`F7Sw(A6zD{EZ+(}qEwooz(4xTa%pPnmcKvxva~m& z?*S6$o?g5O33Oe6<4&17XO>hi-;0rUs<+h2%$y>lywn>%lml#}gzy86E#-IYM#lUk z6W|PkDz8-!1q~p_pCi~o0y}eI3~_Ke?s(qCZ7jp~=n&1~=CZ>bxM(HkLxb(;X?1Po zOdzwTZVQvHoO_wB6?MM^b9(Zo8(rOW!dQgLW{Mof|NzrZ7A*;6T z(VpM;Jhlv)dA>aHbd9=~is2T@gL|45qJfgF_X06tZ}e}M%w)J_E;9Vq^bhTE&JH|e z0Cc&Th3LxyJaI&!LBeI@EN*6(YIh&15Wg9T#_o=w<-JRLRd|bI3!y_ER`#vuu_Km4q1o`j3z~>JX8aUcx^lbhL#xg_#%+Tv=>)&i(RC}q8T7$yretfP^~-j{ z{EnT<^3jvP(mFZq?ECDeuFnO3(7I!-1BY`OYokR&IBV6}Uq%17S2ag&%5uz!iNJ7cuB*>Fz~LoYM4vBOnt(CVUlvRHOAwG1EpX zCgWzto%C1K1u*A*m>oQRVc ziFWN_5iHQSW)GhwPkg*IbUj#O}IT$znx%(I~ctBqXfs`jVvFpfut{Wh5g;@$*4XiFKDdKh&2K0z!t_NSp!HftjW>BYJvwOmB^R~5+A-R0le zb0;!AG(90GeV;QhmxxiA=j=0p-*U?Cwv@n%*Gt3+8vgDu+VagSe@tGUTswjgOu1g+#{3Rn* z&^`sU3VY9wQoupXW`d^*Q7(ye4Z5SdmznD74pW5zO_cDtTCXvLw8silbL}sdvU=sj zs8ZW|1vu&pQ!Gct#I)xb^we4G)M(N3M@#I&tjigeWPT76RcTbQjFOGlthBR-&0S#V za16EPQnQ<-77jqyXTa~$w&J+I8gcOHK(f+dv&y4RCDgsE_JTau4>HCsPg+feg2@sufg{dibS z8?4|iB&IQQnax0y__BDAlyf>J5XQ1^~ulO>mUHAcIYskBg^lM@aJ z>(8g~Cn2gwCX-_BKK!P;dB|{?G4D*(TB!td@-T~#fYfM&3O1)k^}7FVK5$d97qDsC zw0+xU#nR_w&yHQ@5g-g`EvTRksK`cEPeKxq^fODkanpCiZ>~Q>cGH?2Z1htXwNgdE z2-yCQ>(%XK0yg}zW$O(l-~pdf)Ob#tdphFT73SrFr4V#!1JhLPl0@7qTn6sT%r5Ed z&rplWgD~}SGd}qb;?e%Q#5IXMNS%X!ETNc^Hkffyks@D}GKV|vPLfrW zYd<@*a=RkR#htLAZLF|hp%9>eEP>V5(XdrkEP9g34v@oOgUkWZiQ6hIaKyNC@R9D7 z!qEw78yweCfT?>N`0w94!DZey3tg$L?GKXid?DC~&~DpJ%5v1}q;rvuY%AWxSwsX%d0D7+nS};-$B>w zCKI6mXfq-=b&q3SN$^U$K1|o)E;Nlv^HTT;d7uhVjs4N{V1|xnaq}L{uW@coH*-}M zX!-L8f|(6VSn@UV__L&Z$Vki^Ax9`Uq9^*VsTT=rJM%@ zYesGIzAdZdJs*4z>fI`^v4~IOIurq!tJQNx#~!Vo&MaR~SYZyIumH_nt#Hy(25708 zA!q=jac)|j!I4QCR<|9BA5(Y*b$7PM-Yvw>|K}IKku94x$gE>eb=I<-VVZa)9KZ2| z5D}PeDgMy?oR192U9xTCO8K`BykCY48mw&sxhQJd+6&k*R%HST8^_EYbD(QT-FF&) zvJPV^7@hVR+^>tX+DRM2X{xIA^9HwVO5DcC{8&n0j%g4=bhXj#|C?XmC_A@olp}|Y zRz_(DuQ>~WKF1&#FCKKFsVod{e?@u%o$-UdKgZzZ;bu(6EEj&xq#Tcu#G!!2C2&SX zoz0anPiwLey2X2Xtd(P;D;)P%U4Cp_qoEj&!^Y6nigEkyX~G49LOj)p{Qwm9w;mw~ zD=9Ir{wq#V3Q^Qf;bU1J@hnAoRi%;M zEk%4s=gu9hGRD)4;V5hL2|pXl?Jwzz`^``Fv^;YytEY~#$MF_Rj`Xx+z9K!QQk4qM zqYwuMRmz%`%jMQP{w|Y`o~wy#aOfH{_k1n@ic_3IK(gJoIs1}I#VhCHCjRvxZoF>! zV!8Ut^JUtRN9Y;L{us{GQl)?isz5Oz&e3@`vlc>h4cGiL= z>DhW1KH#B67Ftu`t@y9Es6wfwW@xd>$`pD9SzU7bZMVtlRZHdg1?Q?4l_%`^M3sT| z^bGWP9(D=N?{GB=$q&iT!Urv2r#hhdvK(s8>^Tt7RcOD51}1vK663}`eqv0Y*kAzl z49*c(Af!kJL$; z$V3x14dFb4b8kb;KOs1Hfw|Rm9ZFQDX=~ixF((RJKAVTh6V7DRovf==g~b??3IJwM zk|ij-IUX}d8DHYwI4xQ4$wRGkfsFj?-DuX&Q*fBr!5=TYJTPnx0msb&VZ9Qc@s@rF zArcp6(nIT=u9+Kl66#*A60Aw~;P zr}g;pj2{rmK-HL!w&-;dR22^=uN5C$Zi-fhr~+WTLV1HzaFlKw4r|Da!kX8G_;c8R z@=BOR_!+vxK_X~M^1^eA&}w)K#&)Hz^d>6ZVM^){O{jVO!NnZdP5eup(r&PO-Nm32 zE>2_l_=orc`#pE?AQwQttCNWathEIqbTvauCNY?$Xad7F-7`mOYZRmu$O;h0q`V-t zPYj_~17#fp`^UAgdeheS#2pA(C!Q(fJ5o;q1+03C32{4)*<2*X5@2K%% z@HK13ltoQgWb+!lI&n%HQG)KkZU|W>!Vq>mW>x2hFL_=%l}l9~`H#%ojiX~B+m03y z_mBHz`wqOoy!@Ogq;-DymJBlJz@%0-l6P<)pU2;(w3+qomAJ&&!Eqp)%ryNSse;Ah zVfGQtQuB1yOn`ggW8Mfa{8a14w`3UHO)9L&T5dxZGKBXS-)2Zl zaCl7*Po!`!KUS}>(na|3ZQjO1n8%}(i?a%F7!ewb@JI1d`4KO%P?avj&0X6rJE3CA zjfqfC|Em0fH(T`4yo=ftg}BBX9EbbHpRL5xypkTC3~#!7ozS+ioLGlJQ-mH19Xz?u zIJv^42e2`vm-I&C%?yWL={ZAgODa60a5Ou9X1>BWrYUEp?-;XtUAVYSskHNJ0E{OvP2hr78T?I6 zTakmAm=?WL!~(7t>e@_zvjYHtIY}WJbDy#9hQ*h zJ1H?{W*Oj^sPh1LEz?o10WHHT%{9l7vYOJp4w?3)C)n_~WoWd}j5E4x@KaS)X$dlD zpy3&uLImBEl-cpDajGF|nE|7bP*e!v-q291aH>EUrwf9#4enU>r{}r}H!BgcIAAL} z((&`k%1Z5Ptow`aO3?y#f6abpsL`UxJb1fTjL zkr^K=Wk@Ufx-)5wP<%|6pq>O0Xztec(0s2D2;bRi&z+fg`gleIZ}axyZ| zG70xX>43yc90A!;fQBRZQ4l&|Spxh z_j~T28*N8j9T%5lKgoMzQ&9DFA)vWP6F|<2b1_JsW!95V?H51!tr~$XhM*}ctBrBo>t&nyIrJ+NI;?$-JFvA8kA8C-0BS&csbZu>|3>rLG zI}_E^?9=!tTgMF^G(?)AdAH!GITg4XKXI{ZGAlfbwbQJXZdX^WS(^b8(*^DbgI08) z7?|qhkddx1Z?$IGW@SLETecXq9ehES18Yee@PPrTfGk7?c@gRo(SQVNW!EFtD>q=x z-C){fXwss!<$#x-K!tUY@eT$Is;hARHm+-KZkFN0he1%_j90v`s;*EganGJT3ZE`$ zz-4f)vp;YHj*z1;p_0%AIFq5NC6DPA<5)f(fysTbG+Ff{8ewWXK|1$Xn!%D4S4&RV zD1))MR<9aN2xMVm8gRmL&j}bUvsMAS>OocV?6c3v&wlz7*|vFuTyn*mH($1ZC{#E|%-~LU0cjIqy^3ID8tle_#+&S|3FMM8}dHNao{`bGH z7T#w*`x)82d$-(i`|T=#+kjWq)q~`-`22_?4wrlGxmRww=_bInNaoHxPOiJ|I-H_3 zNcZ=?`|gup{Nm>duMdCZ!*c1Rm&(hFUzYEE?|X91wb#n|=U{_Su7w~QY@9`kvx zTJC?m`H#T+8u{P{Kd7s`^_E+8zs^4UEL{6-dFMO-Rp!q>UKTEVK^}bYL9FjRIOn(2 z>QG%2H8z9Fa$&rE^RZ47puOE=WjauB=<;{oVa~c7N4K`2l`oNW2;j#yy6i_^1n)yR z@S5HSb^GF43KQb@h!Hrr0<^vheV@y4LQ^#k>8R)V0f5!eA>b9bz7~y1cnuY&3eZv= zPRZ+_LQGEt;H5qAL8DZvNP|V-A8l+P(v3qNVEwVr0uR5*!?Cz|Q05AGAPvi~s7Sle zSeUo>4lnb3=rsx_8U}X0y~~l)F?r^@HBGX(FO|$j^xfwXo+UVPwjQU;aUAkTu7kXe z>oB2@5F6H6RaK?u*M zQx0sCm={CEm9VqUgjD*WY~gFDuYe)jywW(Z9usyHuoupT6TQPGU6~tbYdIE8XiUdU z&)30A#i>r%{!!r3N+nXmN`R{;AHNHEfCggFRV7vG{i#G!1%+`*DNbE+NVOkl7MGQw zB^?l|xur?1s#>)<@&{LvRXCPivP9 zM$66)%)JDszR>+yQd*279UJi*?FB(V4Isv5SZq~LZ0h&z1q@4N(1_tmgyn!m<=`Re zCal|EBg3nQDoo4JFuSIv9t^1tr*)Oe5D0fNG#;i=42zNinrB~^;n~!n*_41pXI2~g zaMLnGc!I~>VX``ZG7B!p#r#wqVq9ApNA_1d=AM0=@D(m?#Vis3e=~_7K+EOcd+w6& zedoJ4X74R>Gz2jFG}CQ(-#vHZ_`UH`QGTlI+O-pc^)3k0Yvj>K9+VSLm?x*5dJ0Zq z8X_x~FPG1L`jhgJk6tgMaUv20#1V%dE|*_^rTp*z{uDi(XUe6QT!?YX)iPiF(jwWm zWsAJ;eeaV;AA4M$cRzLF+{&pvhM8vV8 zmt1_2yzllm<(saMBd1N4C5soSC*;(VPn5s> z@ei_Q#Vc~(-~J|z4Rtt5@Ne=jZ+|V|>Jq{mHt@#6Nz_EQt4dC7>!-wJE9SC9?1yqO_ zQeel#QYbl9RfA;bo}Cb82-mSMHJXPMbfMKKLlzLwXa}xmyXR{QP|JDA_ELAc8n%{1 ztIjr-f~g09BGvK~6o%U9h5W*0RAcb!(rWNk#KO@Pj5_k5(hB9Pq-zRpibT$x^v(m0 zhzkrM(9<(wF83%{n;MybNT z0$3;Qovm=WAl`v~(-jb0g^cf_Oll;aw{!PSJa?G7xEgT}nQ|L7UTtV%E<ZnhUb+ZMcxO$7Tb$b(l+YbaBFm?{Si@?_YFFg7 z#Dfk9%BE&`2}TbCeD>qO_Hxu67lJtI(D@4oc2p>tNdu0?s~%hhI5c3bglVhRYnOmo z6d;aD*s`NqIgZ_IXs(w^5SkGOdktH>n~G6B#w>sfYACF|8n{_$S<+3mZ{I#x8&!Hr z8S#-qlzZFB<`6xls>i@#4yI-bOd;b(goSoXb8yLGCB-I=951wpI5m1pXt7aQT_Nv!_j_JPTfN3Lebjzkqa>9udr4;)@ zE9K>v7s;iUUmwm%EAgyxr0g>G*Z-PR2IX+r^|W>wd*Hxcni_dDm4v zgyon~Be5@e@WO#kh7X01Xq2{=dbJdX52=!Rc%K-fE{8%(Z&?RVv1)FFM+&Wi0fP3XmjXLyoa!BY`_V1 zu<*OIpqkTTKXJ4Z1&uS=vWqp)bf>#Bsu^F+rif}t4^S!54q_H2p%BZ5k)W<-3*#gg7IXsqa?eKhJZ0*^vrKSw2fajudNeN)ZQw4_~Ho|xXXn`$KYik>% zJdag7JZk_U0Pszg#CBsN3$Ur(`K*10mrxTkO(C(D?Qzz~R^o;L{i4x09Z9pnzju5# z4}qJd^KMw*_}9hDqO@Vxrdic=fzrvLHFV7S1@gD!$TwgKkOf*Dw6DBo5*sXD5L5?< zh3=DT5YJv%&6)siK#{-v4H4LnI1Jhy zk%jK@%4!Ii+HN3GjSPh~-PF_s4HEGij5&1p2(?y7Xd0U)f@ma|NgcF!xg*>H#-Z62 z2(B8kdJv%^m+n{`E@!auD5JwNK}g{PLS`2quG9tLo8g+TtnSvimO_K48=IB^HDT5v zlTlDnsi~=>S55C5H*OrRfvXPIF|F}kyY~VH<$%Qy9LqTs*VZa5ckSG-t}6-?y1a%A z83osDC*ZT+;M3ZS=eDXfQd-7jF~*(UIw@t;7$6k9gNJJ%wCJ*TFjGRKBGL~|o&(m$ ziHULvw0?>pXj{RQ_U+w=GL<9c`RAUOPkrn|GHT2?`N{wOuYS*DE_nE%pn$<}&D1+^ z|Bjuzq_ljnOq#Mw&N=^l`R)JxPX6%6o8+^f{FI!u;57N}4}J{THEXT_-G&Wlo#@?^ zRQRYcwV>ZIJw!u?4ukt}pTc?=@Q?-w)x=@LhU?LtjlkuJlP1Z8$&=;Rzxg$+z81LM z7bp$T%AjRW%Zf=$_uhMteD-snH7^WljqYIxqObhtf69OS$A99u(?+R;J5!HJbtN{j z&^CuXh<&NUrBu(8-S2WeoFYZ*lcI!$e1rsXWw3-$5aVSI=+|Ju}>{FK=k<2vCJe)CF zqnlj{m-w#TwGiI5I&K5BN4jEIcA&BEZ97?a@1SjoX{Pb=6f7R+#{RnOoWqm27{kVs z>AC05SZ+$zmyfksCa}vYcY93zi}ipCBf?OvJrEWYhOImHXn*V7>Cu|S=*tbwc@tgu7@!5rie)5(VM{yV+@-GG z?K|sbFV1tO63~E#_#B&>GZ)y@3OMpK9<_w2R8Uc2;s_GW2mqWO#?mkqe%fo?vXfOn z5Ik{omkbc>pSBw{s3mvNm0t}7W!CIja?79oB47LZm*vAB|CIdZm%juPM)(DuhL_+f zzWBm3aE$3fFr_m6oNmEuuDMn&z4$^L)z=~0x9tEUDwj`x=F_U+J^92_P%@j<-Pa5T z*~xuif}b*0=29S53Rm;qecR;Or(cpat5?gr5fd&!_~u@?q0hVMVhzze_~65G{Ji-H zVU$9FZI!dmI#;g$=ttyjZ+)wVHC|cviY!~UT-L5#E5{ytthy{2KYa4ZCp0cvUx!0w z0QbFMmSe_GlJ~s-gYs!85EG_MRl%igVm-spioFAWf#!m^Iau~T@yZdBCgNDLBLYr8 zO=2tf4E$q1)@H=oBincHbBSDVR|8)=Kw zFne(Bb3N#Y$z{ZOeIowjIC(f&rWOZjbk;OTZ9NrbW+1~-N0vPSScEWln;r&1A!^yp z-LQ65p;LD#N29gN(mDK~Q=l#Fk&JG zksJu!+Q!lqTXJ)Y^=T#IwF7y2kfcCj%}3i%%e1sjZbg}r5r`eI&P-Zgiqh8HDYZ?t z${_O~=v(o(zI@q$)(2CY(+^Fs5Rpv^V|2RTfJDsTRKZPjV?u}{4sS*GB4msg`p7&H z(cDtj70Eb|@+VCivn+D0F$8nb(hoK;;n#?_H32@_Zj{Oe1UA1{_c<=<2GJ?95w?1) z6g6-NB*eMOn{ttXmmM?z=0(OENgM_kk&pA)dJo7Bsx_yVbV4r&GX_m~HMwTpnMb@u>)o??pF_v!G?@dZ) zi00BHt1y1y9Z=!KAR>_3hkBf2m%4L}$(!xSR+Ky!*@S7Q;xw*f>=&PPqkORiyhDW; z;?;VjB?NK@V8L8~9$ZiS)*b=4iYHhxX^C0J46SJy9N~kB?u0i=Ennbx8)$?3W95=) zVGMb#z4h6rnrz7kCBDaJo3=yvgQpjQFV#ZmYnlK$me zN<9T;U>RQYL%Yubb8~tQao3OS=J&b~&`DaMZXbJEyJRuC6*Y^GE)||<%9sr|8H=SL z(sCfJqBV<~drA%4b-JOuP-{zxUHGn3ldE(KZL<+eqn7FklxA)xH=5-n-X)>Jm)e_U z8*AO4mYcXf(mGcYLe!>dE%Vf7<~K7_l88mcLK8Uf9iwR8nV*UaY0OWpb==P{$Ov?k zjZIp1CkUTepZMY0=PuNNS)?NjAm_Ud`J}$Um9YyuUbov`z#yV zFw6+S69LC@GL;Mw*IS*mZUi?;_tRXp^gb{P)DGX^CA6g7POHmD%ZlilROaH@N$(Vs z6OqIqK7*0Nk-%Iu*r>pSyKch7iSolA|3sd7_E{CsM<0E(x<5}s2KSE7i%k9_2MSh8!75H?FDPdQxOs{{&#EvpXrH_Nf;9z5rq z^W~U1$10Pb0wGMxmw6Xd3cmU6@5-~!Jgu3q3*g2k&QG3txP0p05m$zl{*|wOL!UYA zs8)P`nku2^oOdA@EW9B#jmYZ$fM#=3LEwwoyx;oo?`X+ZeM6&s3>Nf0cuB}WwXd}* zsC{REecvyW&iZ7p-L)xkyNtGs#x#858fR$v5K*0Yt)T~_XEOf+mKU)%wF)yf zzhLQ*Wh|@J&eM_{y;!F#=krrMcEK$rUc+XM`D#71xypFPGfBvoTbbBKDG|<9Ry60i z0GLjh;dGO`)ov@jm64$`{EeEfi}?r@_>56*n+LEIb3p1a_jrhpunmn@!;ok8Q-@1Ot1C;*vAUeEZ`=T0k3d znaiq^wRS{yZUz}Vi3S8z1qM@r1OhwQ0hzL{UpMbFLQF8zKUW6yCuWrtvj zo=*5F_Gnv16OUomqaG)Ryw>c%2`0rj8{0E0A}E{mck`G^2rJc3-GLOU$?{4jH}D=J zZiJXl(gKxG$VYf03@vYaS^S#C`N+J43$+rNmhCJZ!5W$4?dv#=m`#8N!qu%?SEcbS z+XFim-6h~?W~@P>o7tWuTK}198UA~xxHxz#83qYb0a%>Sr!vJx{eTfH6CNMO_!kf5 z@LbheSlY{o)s&0_{fcCEP7`;tad0v?=EL3D1ffag=Y$haRwhgrKf@=rwT&_ul>+BN z08$XvBK}6JV)WRt*04cwhItE}h!IbbDM-?yrCbkT%d>j9+~Fwe(LKS&x#|;*0S`j_ zjxQ#SwYDJnkuG=S8x%s&9yjj<{jLdl1iY^lIRX`x)jFqgBH5>q@qZ*cgn9KfC%D2@8w_=nu^_jZW$D_+r1A2x~ZGJi`h2Ae-LSZ9LKA zT7u6w5vxj%wQJ^_T%EQKWju=`7tz?E7Z3PXUdDq@+@1ku`N;L4L1KqxxbjVH0%T#b zOl%{rn>sE&#;1WqMoj|04&f|r4*>pVDn3=Db|l{v6qPB`(`svIY>^c!R!a$lbP<>a zV|7ZJDlm*_f^ifjxWk4t+IWFNOeuzp(Onw|SbpX$Zv(RmM~7PB`RRA~VP<8r^mg~@ z;5IJ?fEsF%q&1Az2x2pBbuw~j83+eT2D=ccaAmTX5pAmY3CA?}o{YR28N$V^F@_M% z)A$&ZmEdYd!qAEpYf*Pu4q?F978#6b!wMfKm=nlm+NrM5wJTLZbCj5@nD~Yb?o-DU zJp(dW+aCDe>OxE!#KVQe?5t)8wY@cMGNhW>nK*H`6&d0~kU0()x_Kb5Hnn(+Q6|aB z_|$Mk!TkXyB7Pz?AScI2^E%x)BLSo z9=kbM2Zg2KzbT{^S9DfAEk9)9Q}Htk!rYPC5iS(w&>PK9=w7A}(1PUW&e>_Wu63xk zCR$OxZ}w)nZK<53A&XVybWq@fQVpLUHvFpl8eYkU__AmcNbCyuk4;@nB1JuZ&CaNXM>x z#J-^@ZC#cYwfJYFKrXTo8hfRCo7c!TE?FPh-ZNJ+ zN_2%9eV80`>Ll>VcGMSdk&|Z4Q=}pW(S6GpwZ_sjk7&vQ#-#-`dgKUL$;Bvt0DLBl z(B=x2Xr@wFTqU3VII?f)x@KG&NUQzC63cM&PC-h?O#IwE%q%g*6Iud^l7YdyT=J4} zw-B(*Bmukn`{XI-w9@uDWN@Z z$Q{LNBKTOK^oH=qZ-IY)0GEip=`YNqi7NyHzNKeX*9qYQg?`NFkx~iwbQ3%fMMzrA zV{#D~`*>s!KYH&i^6=lE!VT7nRl`|kUaGK7H?tC!mnk}auRX95bQFBzJx2T>_}VEk z&NZ$36;{PP|Dpk*iBGh`=~)9ExAe%&1?Q-1xVCPej6IA8$~dK4=iQ3{QUy0)bjmZw zj2ez39+B{cUIs;|$S6co;jmHTzaB0f)D)W^S^{BGY~DPNcK7>t_x!?t8z! zRKNr=YyXA2*MU8FraoPTQnJXPl2cp(Mu9T8&D++&CA(MV&6@{$VgeoPYIuVL8&1VEGj05;v<*^O#Bh8k414oHUcznq>M9A%mAkDtJOSIR2s^>&Aq? zS~BW52j{)@(qx6mEKMv!9-=!J5)I#XVptT#G#; z!NUBa5_Dl>hdHzfU)5WTIeER*4aFo|1)!beh3xfq^pfZ-2F&2msV;=4wQ_=n5qE7{ z3op-pnLB3&_8S!@sM8K`)f-F)7CmzsJpn3D4ayL)WAtcy4 zSnIA6M51g}|8vCP2*x~pZXKMzVvuEW>NhFOC(Ljx0mOqbtn>sF?8UtivfD8om+?_F zDQN3%l9!)+SS~vEESWKLhKViPY+SWW4m%gX{<@{2d&ZV*$(qku^m0`4<+h4<=t=;k9yjC0e3D|n8I zA*4|OtoaB_o_buaebc4r-*6c2w#;ek4(51J#0FR?s^1(c>j;a$9(d=w-XqU^#_jLm|;{* z#B^56T0}iDl$9YUf%*D3zKmMhcftsV=6EdN19u+#1@pzS2gbv!M^J*nst8c6)ctl4 z31XNYTPURYG{AxGH9JF?^3|V6V~z0j#2AH-QJU?8%T`T-PlDL}Ik{v6#YMQUZ4cmc zgUmQ$lDzfp|Dp*o1;r#X#%&G8UJcV-ix7TcdP(F(9aXJp=?y zsf*p}%txgmF~1%aa?bj5k1z_f%wtpF!#ipU6fwLL7AD`LWry+`EoTIO(qt*}B=@WY zc$`oQEyV=6;fv`S!w$ZI!>>^lLfsq`7j*rB`ZK1uf6goEn>>O({N(X27(VAIHnda2(yu&%3z~ zK?^!+)y!_KtFM>i=gvY4#qY}xfAULt^uF6=#Mp^af^I$R>f=ir$m|2qhj^MIixZmO z@-(?*S&R0yxk1<^i47zu#9Mge8eHoB=shhFaIHH%nOaPd2@g(aDL&a}UR&n6CM*+g2ozb-N$=cc~l=5Q9F{H zm_=!&S=$# zx~doVvJ*ndhhW@zMotBn(p5q%%Xa=7<-r?mb zS!oxn@?!R*$LV3j?GDgJ3z`D3w}M%oCXmfJz z#}+yJoRj6+w_T$ph-JlWsALC9WPdr&R@ip}0au4ox%nrbAm9A&ugb3pAGps)qRfw- z(9D9TD&wm)3{D1cy{wU)5*d8C+ptuk%;3>;E0O>0zDkZCi4MAXi-L}xL&+-v_#uW{2dW=R;& zs6J})I63hx*CRG^40`#ZPdHlqp~W>>iV~u}b93f;;*g!53dJ7enBxU-37>$N?ce_X zusrwNLfN}(pK7m~B}&kwx-#vFAq&m^w!=1@koLsKL=4QgeqWdg4~*zxZhvF5OsgdM-MQu+i>yMucqK7oKOaugp4Tj*J~U9t~aWh*7mkwI>4^nzzyu3UR6oVeUu|vPD^AU52uunNtsw z6W@BJ9DUR@DaGIAXc=p1vGJ@lv&Vg7Afs=7#!3*b@%)P}1aCc29(w3uS+WT5Y1o5W zcy_F_(m!>f2h7bYLIh1>!_78H=UQa7r9~w$wD!rPFD=xPyjio4LlWP32x^|T$8o5D zv#cz%56ZI)Tf=BY3%-lz3~)xcz8Ek%`f$MKy34iaKnW7uN&z2#QdulUdO)Z(0u~WW zJ*co#`gT@67JkSG=-!Bu6Q;MBoq8H@s6#cZw|883rCfLQ71(h0c5c&Fa&~yUWx|=j zOE5GKlZ4Q-FT+BRaVu(CU;o;F%Ev$SZV1uY^5g&a3wg`6ZoAsV`9zMV<725FHx-oI1#(LujXFiE@Vp$a%P&CxxC^sC) zK-gT3!y0PQ$*U29o5vVhhqRQ0C3W4WW2Pg&q%X9)5?tor{?^y!`uDv9iogl-y&wON zyy-1C%wf*lK#4J5ncoBedlwdw;WJuM2MPB$s(;?4)oOLz?`KO-s<-jt6S+ zZ+cwWD2nSMpjnpa!ry%&;DN@`Xz!aDF-hyg--nEV?j3T*+4xCoihSQ?hoTCz( zQil3Hc3|S}Xyu@05;!4gft-ZF;NNRJG@+L<4oiVaH_7ss7Ry_3X6~6d`tJ*$|FSGw zyhP4F>vXiHWB)z26m{_^H)%`jXMEPu`NJ>oLgPJ}DM|uE%!pbaYY&+eWi2d2l1IBO z7%{6%%;Ha_W=lf#youSF4G0M^>#`Iv@8W9Kj+-l4G1hCm3L39kB+VLYOf=!=RU}8A zalL2P4q35ysk{l+$SEhzm(P7}r@XQV@Hz7=X@rKX9>Y{7>5@~7DJ1s7mc>ua($mgL zcnfC;K4W(x*HTdjHI+qw6jZz?!RE5TC>~`dl)8A=AK0tk z952$<>$;nf`SWV;M_n70+v!U4~f6{=ku2>mgN+YC9HhwYXRP#U7_;B6-T-~{2 z5YD52X}1B!Ci*absiJDOOe_!E#i_Fi3fRemo|z3)=2s^2ININ$$M!4%x7N zt<0HwJlyIR}n<|)ixQ26ux#!W6rIkUW5}fR#A7x&kYemh`$7|GxYk!;93d<+&*!y)QLgaO z9Vbq?H%cBlb+)W!|9JL@H#<%7Oi>SSTzz49mEOhL+h(#K{gf$F$DCGqG%XoMdPQ{U_$IU+( z=VR{$e0V-QT0&{QWFGIi*bzg4}5BB6?gerL&VOpZ0y|(O zPoH(P{QbfEWFIu`(TIr-8$I0m#lrIC!31=L6M?7>?yTB?v?6FM6^>?An#)OJZoz1#Hn`k16MiYoI1^8&%PeG{D)O_7S1BmOnt=Y5z0WIE~(b4Wo5WNDcnuhx;rY*Lp)d-_b;^Bv}Bx79Y9d8u>XQ< z4;GOP45^0LW?u!V8zMFx#f%5U9xnnA{kb<2T>CdBFvR6fXdSb!%Sm~dSyjGh(DRIN zAFI#}*$h)$iU<~5ti^I9YOa^QvJ@TUT4mgnNm7mTojbvl*oT0Jk9Fx;&{+hu5wg`` z2d-q}Z`u=|Uu&OhzPMgqn-0x%G;_HiNmOLd(jnb%g+FN6W}zL$zlEt#`tQJc4{A^%W&zwH>RP%6@Xz(ah%U z_^O^UoosAGE+-v)iBI}LuQ5=^J;(M_(~uNarjxx+T-+h(@nCY(d)t_{4y~KNmV@Xr*jd3NkuLL~o=0yIqkxGBL(CPf?ICrYhB+)f<2kua zZA~~ox=_!5M!F<=;iAPTnGoKa%g_NRSOGuC&}l7ggZz%zyvzv~+V0n@WR6C}f9g;B zCyXON=UQlbI?Ko)owDaxVHV^HbBQosC-78-F3`ILbkT~8>pYYbH8nM$K?eF~BR$IjQJL}d6q?eN3!=}Bx6A6=!b43WyXtZ)-AW4+EBFH4g*NfeF$mw};X5PXp z;6%jfcP;4eP4eJ%9xbZEJ7SDI2_8yj$CDmltS zUaMDJW|hD(bC{SsIiLc1Ac-&0f-hOH1sQ`_kM&|ooT}vHQ-H(azqOv%`L1*W@po}C z&OY9`M?U}A&&z@ZC(D&rTp|DV{*TD*x89DDsCP(ngSn_#?Wh|bSn$+f*?Gt}n=qHL zHNJA<*B_6*J@5ID+;Pk8@_`S$4-BahadjSAf*UB`DOw$aVIL%aLQ5hB z?$#eH)-f(uyX{tEa}R3PdndGLSiN>6nr-iTTTNf@4elFYuGH80WW%)qWu1-&40n|r zdU=JF>t1&v<;&DTdo)35{n~Z%{cnFquDbeaIqtaQa_z3o5r6>9l0q zr#Cyz&-~)h;PMeWs+Ni4Crb+sQ{Rb$*egmaWpEYTFen9J$1cL?g4E-j=UqGZp<<&-s?h1M9p^{qBk7VBWDod<#+3sPU~g-r zmh4JF-CX@)5y^~$N_l2}8-9Y7PkQ(y*6PD`!7yMP-Y!nuABLotCcWjO+nu&gMN*B^ z)^ zGSNvoPDJ5Z?lLSJbz5T_=0K2V`(r0_vY=ILsH9IUBCRYzRROp7lo*)Z7oXXt8h1Ks z5F2sjJ0V)sGlFf3Nl6&V)~G`ow_)l^peq-@I^o*h^y(^k^JVADfBeT+4blsqsJ*rRjg9#T6scavsT;jTUFRJa@8aQR z_^NDzEJ*&#;?nGm38a0OLRW@^9;orw5C&S*v-w$HnNT*EWMfrP-^U%(JZe6>&_;?B z!;xTeS?|uX7I2D`z4OUsn(u+!lme7b)+466XWOfC>17wmH^2UMY5vvEDBJ7kr&Zq;HoPwlOO-^JM!~i z{7k-q_M$a=>*VQ$i)7yXW7Q3#bJ2Q4M=k^?cE?(d+B{NuXsV^`_t#G~SDQI~bHhm; zakRi*_Cb?uycTP|0@f9_w6w@Vz~_upPL$hjxfz+<_sFT|oF@(NE-YO5lC+@demUyA zHTgz61Q?9H>)$Ia*>N?(wuvB!++daq-t|61e~3W#b=VcDbP}l!qLi%`PU)JBpM;-FFg&h{;2TY7S{P4*#Vcb}mhyyIvjv6Y{4;C&#Tqg5PQonou)MpD?gaih_7a2y&Z)x+yHR8Bg62F?-a zl=ZJ}L8mu#YNYWK+tNKSC=$(z<1w1(d*cac6({ss1gKcj8tHXP=@e*Jp+^Pm5M^?D(?838`qwrrK({r-p;yt(>@&N@x$~m5V#QD3u}sDmJfNu zvrj!GT`djr0@{eKUbRYYzw@uMZu1T)LLLiCh*;aNy(cxnB00i)FyL^YU=fXH`ZVuE zlM82hUbV-;g4sagO>cDI zMT)neX#n<1K)>Dq6eK@pfD$d}v17+-2KTC!E98Y27RvF*ABO`W+U2pQo>$AcOXJ)b zTvRc4^7RHcle0JivaLggqHgg-bP%Ji+tk>k@5!Tf@2S<+FWqR^;1j=-!Qqse7#eaS zK`b#snjWejq^GgO+=6~x7T&*XUZvM^uV2e&_Y1%gH}}z5H6YW-1V{1%i%0YP=`L+l zG9Tvz)Ya~nT6E(&asE818oXZCZrJ8LZnPwcxD-P1Evq<-c&+w2Q%5x8k!C^DE@YeJ zacFY1V8H?;&y~osr7r_k$D?<7qdfTVbFiQxoN0AqvD%S1&D}q+@QMS|T(9Zh9wg(0 z4`uW!pcBr?^r`l5G6!zFAU{n>AbnFkp+^_?qn;NBHV~2a?AZ;iuT;jMr!G%fq6XN- z_IfxYHIGS8t}%*XQaJ9F8IcAWX3?5!$f4g?*Ey9CrvRH4Y%vkKsfy5~|1e}?_Q0h# zWbk0XXN*px6bFCM%~zCHn#6pxtc&;Ogza{9Z}8INMGv5W?(NzA70RA@C4nR)fg+(5 zcx#+1te1?G{@_`z_xkI5OC0s;Em@s$aVgyXc_nZuy>00uykt3kRhtz4;wZN zLa0LD@55OE8#e7mpL|Q0w5qA>Ysu#Q3ePh>Z86!?o}GQ25xg@aKS^JY&0mA6?v*c< z71c5hykpCjO(;t%(?%ubGSOba5ZN#bsN3FW>tOjtL)L&VH^NHf)fGAAStK*UC=x<$mPhM{wFoo7B|o zLn0Z35bQ+ud`+Y=g*BfTSk;duy{5MUTd1V$*|P^-#Qu&GxbkG_(j_utm;Q?sZ9mGCLq+UXi|wQSg_XQ9_$+Q(u6F3-fJ7CYyGBliLjrG&TeJTzSVWXFN)H5zX zzhS!iHgDafCwAc=8g*%z1@5aWZG&@Rod;|!S#a%OJU62SPU|WQl=^GEXA=s!l47>` z7#ifV^SREz;Z`4?(cii8ezpkaiE>9wJ3@{-_Bh$Tb+gp$+b74*J61~Jk$ZmeYIhI} z6HHl#nBKEL)2dV1d;jL^%6CE!%RFBH_AKEgGK_9Vt@C7r08Tz>zIJ1L@r4(_M;6F_ z#GrTXeMosow{}U(t~w-ySpVjj4s-7_&W?le_aWXKIUfJ%fhg*+wEJ=r(Ak(r_GP7C z5r3iaMo%5jH{!{9H{ih|8#iy>48i;qEa>%WEk5zY6KWyW?87mRX!qHnaVK}khPK1x z$eA~Gpx2kWj)5LO7%xrnL#H*KcZ=BCV~;(qXIQUVy+R&){4q@u+rD!*&aExdT153~ z_aPtCKl?!7*+2`XsPX!<7yVgtf1Z0F1ZaQfla*Ez96BeZFFfu*&XLC(^01!OIGO3O zCm)rSt5(X6o!jNfCmxek>o=pbG){SALK#n-i;AtKEO;R2HqdyQ&VLkxGH#hU&`{xw zZ1xSWu9Jr!eN;BATO<4S?FJutRGLttS&uWc3sC0Tp*@t8d8EAt9wyOq?xUU;8)dUz z7@ig;laqkXI{544f=mt`o{G}i+KPJLHfd~Zly)>0Xu)~0O-(IUV#OvL?27Lifh3lz zXo#N!pR^P|hBQ!{;bK;d_=3qW*@G=T%Rq1~?8&WXP4}pE&2yXqBs*W1ns7YjQ71a^ z1UOLZKS<-`0ILHfXb#d!`}CNo`iLly#kYZyr{VE0jz=JTrzVLhKj@KW&;w6=YerL! z=9VVVN~`jblByy*4GMIUdQ@$$!078d6FN;F5iQ4w2K0=_>7F1Y*4o~x`8;(E`=zY| zumPOv8*rp1)OzKuNHDcdkvTrB50}4|f;gJjkzu-|I$ghcAyq(YK-y7&9(~Aji<$V~ zF5)p@mg~d%))=Kk0f3_*Z6}W8`iiSD~#~JK}1Ea^;n8((#&ax&=(BQOfa|Ll&(j zNkr?THDmvb#Qx@JONO(Il3K;B{58%D+P})0@ z+;sB%6Xl}wF4T6O)KM?G_)?iXZG$}a%!|rfy7b_8gviK4BNiUOFa(p-mY}*{%2UE~ zir&RkWj;4Nx9)*qhjY4Mv3Pds78(^8f3>4AEsJ~B%$aiKWtYgfu_I(7`X*d+^|eyl z(2V0JZ$+g{3ob9Q+}pIa#o6~WxFW<{6Z0j^67aJ5M(uPo?u6GD2e-nrjYMz93d3V* zTPMts-WXO2k&L>)Kvh6@L+fK!9c%R}D=B~!kE8$xnisSdyPN!)VhHFA@Qk$n;-_(@ zkLonnFyh3ecg8zxkJ#y|TZ$1F>27b6O`A5sP4zSq8dk}+?OWuLM;?(T^wMo;XpoV^ zhl3cc!K4Xj!i4T_H?@e}3u}^Tc#(Qvsv;3FUK-KG=5X1OJa9291908|#-HhG0X`I= z6H;w$z5MNO4@e2JxtA?@MMjPs4cAl;l0J6GnaEwgF_b zOxX8R&y{47hfS^2!Q$l}cprph41x9I+nssqg0b=b`7@vmDp< zQ_=;|s&b4bRWn!K<1P-AztDPu{WXD?w85!kMUHJB28Zk5Pfct~+Cp|OOyASQ9uQ98 ztvD#773V!KS@^0v{`51l0>^Rg*s)6I&+OYKOV)aft;6WwcnPsCRBN;y%mCH*OE-JO^lSg20103Vi>l}J@Z zv9vUmnSiGD^4ts0N+XKwTbiNCz#qe&%IzIJNQ~l*7(nH+(}J@B$Sop`h#gYtf5%1a}A#hWL<`lj(z)eq_{!86eH(-+Hnr@v$ zDh&#v1hS=FD`Ibw1Y&mdOVDLHkLs0XjXYx9l;M(C z_hy+vAYgLTZ^D8tRmcT&sA!r#aFlB=u72qiMzzO0o<=d+NSK*IlFTpM-=v9@addxD z={N82lAnMj5dQ<3yIW zk##h@Jd9J((T$?BG9@=z$x0##?_F?s7YnuBd=f&$ct@ChgyEEmZoHP(Ab%IY_}RC6 zi&R!s%2ae|tlwWFRh8AsTkBy#FIl=m7B75Gy7C=-a!8^}Lvt0x2_!K1d7jyeWKLa- zF5JdadJ6z*Z+hjU$(v(S4C7^lOwjS+nPq>8veFVMMF?@vo?RxOKlH&5$bI+Shq8dD z-8i@!3uR-9Cd6a)&;*AHS?E5?*OBT(pN9-1bwZ;xxMmVQ{=0$z+$cg5)DH0fyr6|@c(}HbA?Z7d8MA`+NBNM`5bOPg_&uKfvon9hLG$I7E17j zHZ77t@cS&xGm2>Y49s3`J__CE-_jrOqi&5HsP>iApr`w-2qmwjY@3rw>y}!C!& zUNb^DV+`Koat6{#;YT@dzY4q<7Ib(T;k`(LmQxxt#c@;b%Skr~PFG7TG3x#tb}wgG z1>24{8Vck=GK3x0y}xF!eDs6wmQkZdDgPKc6gAPP^DTsM`o+(FA&Z}1C;-|X0{W`AT%|uzuU7TTxVW(*V-5@J?b7fDL@l~k4jm?y5W%X#lM{<|f&;#O^r);u zZ^UXIR6CRCRaIJag5>nEl5$^9O4N@26;1ZMB=*~weV}7hKoO8jk-tG%;a%de% zFmpUOX^%1~5z37mXd)2EMR5||&J28AL7R$Gy&+^s;(A>&6G{4^BL!mWGu%bNRyy)b z9osFtPy8ei=84OTq5zSN0F(Jj>XQ=5TBMD9S+j1v{ORVKptyF(i!UyeHEYo7^R~ap z+O_NDsyAIOZ+g?4wHm}q5Pf_%F>0jX)G5Z{D$#2hXay$jcwTArg0=vP%q}#3`t5Ii zB{zYeZpOI=>)?_6^G$z}y*2yMac#c5`v7S`mcPC4l6JUhb#MDAPt&j>4hZ9-(t)~; z{1QY=0bPWQcK^zn#a&vG^^yZM<%2#hHJ77~I#MpX>{7I%TPmwiE^+ZC7a?1@1w{9< zx*q*(8@< zda2ax-wzk}QpE8K^yiING6YomK7h_pH{bjRSU)A&D^|4_{8+oOkkvy=S|z;=lF3!c z@A`91Z)0al)D6!o&L-;;R9kcISG;dUylC3AX|i?eT-l8S#p~H_NaU)S~!uWA=p|(o{c*qU^ zcZ1w;!;KKo@Gd!ojpygf1(K-<*xyz5{eH3xFGtMXEjYodL5@5ASmmeWtqll?Z3lds zVEo*2+bweI9k*yLxyqd=Pf~a~!3M;~p17-t8v=+sUYzTE#ydkMY+~blEXqSwK!4{u z-<4~xyG~1tT3g$#*K{gCl*iCTlPl8nHbrCpX$(7zYmrPmh1of#pKJY`^~5!VU=mba zT|ExSctKXISfNQCP0dZRX7yU_AU1mR7!crDTCdlsHE9Vrm?tV;c}JWL3ttmFB~_^P z2BRo}kV!y9j+02lv7wL%1NgKyH*2EY(pO&5godqKx5CS?O1mVnhxxz$>m^czBs-1I zxp4_wPfzi1$1Sv1A@29zYlh2ROBH*Zn5_UhGZKo{Fk?NA5av0B!nb@0qtvvH`*Nm`DTJ+>7|{aGdyABr(n z*6&MFW6%tb?cKLmmLep!a^*_dxcOBL8?FHFsBfr8hq0-0_St8nHX3Q%jO$0v;9#xs zwZJpW;?DQ+EO6Xn;MDHDXfI7zl$Djssi&N#mJxcoYb_dkPUy?Y&&U{8zqj(h4bU5Z z=XaW`5qdlUv3p;=bVAr%e%Zyc4l$)AOBSOJ`YoEs@YlQVl{3ybMQ`#&0I8u86ZbKU z#1wfFB|CFrat8{EVdIqWX+R9R74?i)pq7y@9zS`}MfSeEy+;^+S(33mS zW%?h<*`3O3ef@{0dy0~6JN8I1l=q7-yg*)AFg z2NXE&rm%w7Uo4DzzEVh-1yS0f4w_xo(1>Ik>KC7R?s>qj2iC%BS+;DMPHyYA9a39| z&SEV%%?h_`zu}-%Av|6-5^-?tO~V=}Dm)b6p9JryjSd6Cv9Fw5+^H&KHu(3Bob);+ zXxjkj!#f)K$>{QlVx^w;lgH5+x`AwLP+N#O#OE#PVK^H*ncw8Xe@OD-?qTGI6g>Ou z8`PULal&{UBC`ywK6j!O?iyM4%5qJ9s%dJIX7E@Z$m0t@dQSkgN#>R$nh@Uf+ggi~GZuKc%iU6({4Y--mNez9s6r`vS+T&*%F5bKKI@8-vf7 zaL6RF9^`(t!h>r9+IRn6U6?wkSjaA$C4coxx@-*MgC%1pef z$CP#m*ACS7^1uhS_hjdv4!B&{J*&C7T~ppW(D0qsA+7A>GsPi{=;Bt*?zM1{^aWB` z?ny}r?}UIR0d;}du(q=ZS?}#=?9z%DQV-(zUC@Nt8Z{p=`gXu7%#P!hfCMx8M}c83 z+mcHhIxc*288gOQ;|>>hdL{^BJUA?=VqShgFY^Y8dq@w;Fk8{WvJL!!9kV)7+0cO; z4blTY?*!9mLpQ>1G+$^(>8fV;#94Q2;2g-+Mp*`l=Oyj6(+`vlx&@N2(x_<#Y&yXs zT5&QUbA7ttVQ59qjn)==aWGK!$e)5S& z)%8yWW)Etlsj%b~6zhpr{LHjW_k%M`g{}M!c>u)(cK}Cw!#>*`@TLq#x3F{0JVQTY zKuF7}1FP)iPj z#$8ZfuX9sj%ur8Zg3?NzefrTeWd_<=?%A_b>d-S__NX5>c0R&sBrJE-uh^fAc z^#8}7cv4>7^eS9dYvh549)MPi)2z_qvIyU^ttG8Ut)&PjiVawS_y8_(z=pDH{*vNS zbw}O*&?D$5)*|=adoSW`)kthFQrCAOTKci8m^N&QWg-mN1m76w4G0`m2!w@00eC?% z&Pspbxur6wxGk&nqa=U<59CQJ0_Nq^W%F0Rhk9m2dZ z;eTofBpQ_<0~glc*sZkSetO{MLVcSFZoWXXKo7&$Gjs zAk6=flIQvl%YV}2CJk^dU=l(o_SBs{>uCAZZ*GuD6Q;;rx8El3e9s5vuD?AZ-~RU3 z;O1`9+st*7Zh`lILk{f`xa6r*C(D0-^IPctbFzH@yWf!aU;kTQ@#QamQLef6t@5c)TrY3?*Z0Z&4?HN} zJ?|{pySEv_8Sek!D57kmzSiF#h+Rxy(?6{R9VKHKS8i(N*`!6XZ`W?w3=6uUzF9hY z_P~YPfD)oIw6;VdRGJI-;B3}wjgovm-Tg-%HCgZXr@P$P<)oP)vTfb4}FBmV| zch*Tc4%z8Kq8Q`)rotTR%KgFN`Tw*6S~WGr0}p6J4A%L6`m>*t4}IYM^81_qh=i~r zxM4nx8tMHq9G91&-DMAw85l=07p}YipTM6>__QObl^{7c^ceDKB;Gpk6Ci8z|Rfj$V!ZBz)?GJe=#l?B@$xnS$KK{{9%H8+gFOw!u zk@vm-1G0P1UKurF1deN^vLBbl zeW1x9u~~<*R%BWmt?06{GQ^{+m5?fN6+&3W5PYA?g z`b>uEmbTUoDDkDrE7(PQ!)@EnFyDdIBT!?zu93)*V~ z%Crxm%-)~?-9tlNPdrVe2<`dSOHJK=sUB2~W8`YIckGNAM}xRJL1bMx{}tlL)sDLs z3r^$v$8qa2UBlMq%E(+}7oP3Pn($V*`k5TljymWfoROW6R-iqoBd2?tXGI5f^EyUQ zaGnFozH#Xr5IFb{h%3O-XGZ>k;{l84g=^J%Xx2e9_F6>6l|3e)#52!@eH30BUT8h_ zUz?YC4cOGB87+f5U@3RN8%Ovs3*OF+XNeKZULD(cZ6@01rv}Yk`e^+}J@|kN=-1Yr z(+jLq(y)m<6h4IQ7r*oc88LjAyyt!IMx6B-`Q^|47unqZqr}zT+2$(Mh(sQfUMr;U zGcIMnF-e=V!Utw&cS897@CV;V*RWdj-hQV%i+IwVx85or`N)R>Iy5D4=eZv8A*AdO zc;kxKAq4s^U|rNRrkR*3=-8vby}en!`_)12eqQo(lTh21&BS^(laGK z`;L%FOx?@8)Ow~c=484?ALhUvb&XTf!k1}N4gBzQ%`bfZv+|Rl{6xO>jjv+Po$~g# zz7?^NPN^DHDg|(ThhLG19M%B)GDkk(K(5~}APPoq?!KF(&V$M=NgK4yI9fw}ogOPW z9bN_guHC;+sw#_NNt+T3US(}z-F*SPQuUeM_BI&>Q9zrkl0w**3*=*_`J4okANVzZ zURhBlElo|ba>a7lv1=FF+#&W73j7Bde!$pFXP3mqkG@=Iph`ZG%0rvq4Tfu*Zrktu z*Ei*gx4Z*gf%eNCcie$k(qL$RaPiVgWalvN{&hRWCLMiEZs^N84%9tiBoE^i_nc=y zA2WNl{P2fAly_cxwXEB`4d+9jB!BwjUm&C__0%i6|9zM(fxo%1h7dQ8A~Z+z2WtJ< z#|0$vojVQyVk^UNvZ2Ax{%>U?*lZ`8hRdH*eB7-qJ0b&qv~S7jRxX z#W_hHW;y(bW7`RF)|0!ych)1*b@%dqo4dKI&{sEYg2$GUc4+c$E1W-Z$x|4O#PdqL|x;b8+yJzA|#B1?F)V84Y0D3 ze3?74JUbdQ#_TL}fpj1g*a&3XZ+N zHW;(Fv32`M7|c1~Jbc?$dy0>Is1q_W5A8esS!SBeuYe*w zk@hW|Xf$C<^GZsN0y=~M4n8a}SJ?|@QCtQ$1}6D`zx$K??cN6wyWJxNw4xDbQ1jnV zF~CAjO^=-jwS)_vkcq&e2s&3HBrOnVA&d@s$0AXI?y8FNL2}9^?~}dFdGhlc?*OpD z3>BEl0@a$U$ zv9cA~@2IJBgcJ*$}Z%t3?4UK?)dvNnzKRvVdB{ln8GeG5#v8`i958G*U4KuIKz@l zM;|d)w$$vCp%adlzdroDmSmFW5?i&IOR9S&a*bhF4HOVkcvgmC4R@W1ssfs_0UQDS z#oN(gu^3+0(PKx+zx>N}a@JX=;4Ey!++91^0+$2e>%EPxNe_;eIIa`?q=TFUw4Gi; zj*(ah0ZRqF7@9*3+&Q27_bCtDsI|2vl^LPq{(Akl*+RM z0qov3jSRmcK1F<9Se-S(jgd5?N&m_9Yct%VC9g`V*q|QF`fxm*!_DLbeYaMMQy(e;0 z@P%-~^ZGE)_cRcYm8UTioQ7V03=^OxPgu1#aA=x#%4p>uo-?k_`+L=1xG9`RdecE(LlCm{VlZQbKk1ur-C) zuhD)V2ue#u+a%BjGl+WQc|o2gnsxz*onY81c0mse4-rlj*0i1nI$0~qn=sJN=k54Mf*Mc zbZT!DJY8_fQ`jJopbU*#P9FNiYMW15h}tvOBS&Hd#;_0mS_TqFdmGD|GZe$a3U=AX zOjPK8aHH~gQs^$Kc;(|{pTPX};LzV>u+Hl2)qB^ho@!?cK@ zgXQ-&MP<4wc&wJ*dkn&ss%gk(ZAN2+7s4jUc$MN>a4?Nox85sbrF|q|3Q$@b_Pqhf zS!S4%J=YFoaO9r=Aud&2*@HH)?rBx4{i1+XrJ08o(1Pa-beaA+Je|lC4jhFs90JP?$ND^+OJ|+Tjw7o681Ymdt z4jqjAvy-l|=Y_s9#X!AT>9Ea&_u*w` zyf5`3_UPUpLR81#>TLy_n$hftIZEB&9ptx-Eygz(Ce6<)GMNz!X@7*N$mVIZ!G$Kd z*xkLE=r?M*$eaCvQ?1l z#UbIq#ik&?U%w{?{X6wOxBp4kD7XWT9>64o^j`Q8fKdG2$Axur4(DxuOwi&-y{|?ODv(<`H{@tftJ!}+FK z`5vzWTDOZ;8PV7z1(5MQx-`vKXClUwF^|v>AFLPa9AO#@yOAaAZFnw-gLL&=k zBOUWn5HaUJorB)ZA^iL>I2FX~6_k;`DrymyrkmH@dRPQH`hT+-NZE;MRMyJvpEzJym0-n8(2#t(SbvAN>yhd4x>B1{M~jg={lxO;UyDbEiU zVbHcYMO|Z`Mfng?KTH1G?`1Mf!q+Ho!`F(^Ap^|uF}wpCkU?FtzGF(|4zU)Jf3Oz$z0PLdcpHO z*9+C>q_U2%&%=-AKMT`Jruz~zNRV7_>(_CTr>#jkGD(5L zkM3v)KJPi8JU50!pF@{st}Zxjo-BEJiL})>%8{Fvj4xT&rmSN6yN z4`6^bP#x+-?60gChsDe}RyJkZ+yP#y`!vbOw z%-rLTR!_>3Wv`<5bh)0BOg)8g;Qp&O4flg9*cj>~j~Pb{*K8#yGfd*I{d-kHluDdc zMRzI30$1R8osCapI~zIwv;zz#h(Ag-9#INo5i zlo&_3DrhB=S9*St`mAJTkqvUWf3L|Up?OBEmDFXS8>gQ;#-o)?YIK68I$8BlC~f#X zX2K*{vwA&Z?$+drMyug{m}7;es+N!aotuE>gbWi55Ag>D^v+#4pd4jYg9Z;lhZu5$ zVim}Y$8@!`c?GUVVn7;=kzu86Pwp`7(;cdv#mNj+{VwehUULcy8O?(IM}MUVs{j$g zpQlP*!Xzk5ivb{KX@{?jY!;9*R9_XG7&A(`M0FG}mUbjm6lq)$0!*#X78OdRWu^Lk z7jhNyQFfRQ+;2yBDKgR$oEuCb){1MZP*Yzuh(~mG%alo@<>b>&mY@IOLA{p(u$kGP zu$c4E@4UUERZc!(j%?kwMP6LC7TLm85CXJ9Q6+(6J}WEmOj^nmei)CXjoS6Dq)hUP z_?_iew6`^5&ah_C(TO<&dFY=mC;_3h&iL3FXUvwr-`_6#_S7L?qz$1d2x18D;$nCQ z(4w)ECOPKTi8@bO+{G{xHU0{2TTuq90klplsjCZ}=Frrqm07K@`16a)X)^_ii@MIK zyhL?;NZSx1SE2;We;tKpx$b40>%II7uBGrx)!LLRI{H^eXG}J6Fosh`0+|`94o6H! zAC`Ef(x0(od->m9Sq_huBz)Z$sakBDYj|s>`;vG-Xiib{$p(0TkJmlaTNNHN3J}=_ z&HjCy407`Md}^8mg`ZZ~Tf$J1A}73Q!0%y0N6Ov}%h5p%T^FGccY;`grBH|S+o5A6 z=Mv;3pe-qcql6585Lz3EPy^rUHe;~gVoln+nNuNf)#}h+p&d3vkBh3=!JESso`|yC zq8`^I!E3Wk05~!G*K930EIaO{@zCFVDakL-P@+bNxznjJzYLR(Ia=H$>vfTeKnjOu zsD|gIlpvuc9~NXsOCyMIkeqYr8ES1l{?tMoy;&pE4#z29+iO6Wt#a6?F>qy<$>vQP zL8JvTdFtWP+|n$|7Oj)~jvkqR%#m{Bgkdu4s7bP9$qE?(!F|s8XG=bW^WUF*S+;N9 zh2$fgiUfkc`0NF87!pt(fBGrev7<@a8eml~-+*ztWb)Y2$n|KGIkRTqgrIzR_^Fp< zZ(Wm~Wj+>`_Sp+gk(xdGWaZk;(C~|7$KE-3D@13C(V;76Gq633zz6w$9rpPW%H|hG&YUe2aM4nC!aV& zjyz(#Jon-&vV7HMDJ-oHXh}2717YPUDvHC0xK}E!9S8(S!4@^-BKj_;BNY<@-Takb zSi)T*xy>vleqvef*(fX|b;ANKqYyQPAb1hHDHe6woFhm#nLB5gW`_Hc#*a`cvaLAs z{;>24DVUC<-sfvv`_KG)^l`0*k8I_W_JjUgt5_wa-v ztG^f(FnoxM<~x_anW?GU%&~J4(1u`^@}S7)gsml}SZu4HvX8)NUT?d zkTV`r;yo!|h`=3}JRu8jWQVsgwK$%Q46q9mwCqk+xkmz)ACvR;b8%&)!jbh+cs`{c}%kC25gt(S$*zbt28 zd9qBHG**7}{a0l8;6ZZXIrHT4C!Upa&z>ju-Se;vMw5$5Sd^ox%Vb1#iM-{~)6roG z@nt;rx;LFBf4-?kIvZ+a-t5V;YV&R!PEsHvE;~tn^V`44*x?m&{`n`$&;R#kJo99@ zya%I6#%`H3W~dCVI#GW2m;0m=&%FBb1yaz~D1(Yd$f+mIl9!inl-urkR9)FcaL46A z0m$cp1xWB5URfe-ys*CyFlj6JzVx|-zt}%cRWfg zE96(d`HL(#egYU$h1_=QYB~SHGtfm0qjN#;{&V<&H#eI0TcCvAQBc~M)H z0Z-Iw=GvWL_WNm40v~E9hz2QgNEm8Tj~L~j<)YZO@_&hk`l+Z?2`G#-bff9e zbW<^=tc?t8!d*Pgh=j*E`m|3@+i91!resDPxT*#OiQFKOMC5>~dmXLMrK$_&!tlW4 z#&9iMoVWZv!MWca=6)LPF+P7I*(Gim5EYh6$OR`zf&`GzyuFgdIXPJqW&=rMyDyTS zpYmB4B0BvPCgP2rWEu$P6nLa{h#Epc8!w8Z*i#eCIl%l3 z(dHWt@wpt^#M<3-!sw*9gZsO1BTJr!j~*pQP9H13zu{R~zkZucn|`>|G<3)g1eB*A zF-|rwds!xq9E!tN%4F2w!E(gm!=%2kR`%|Lm0DdSP3C@F}q7djwr*VG*~< zprK{5ZRbAu?TwGi_%Va!)9<}VCXXH>)8-s44UP43!*3qJX-g&Y$q!s0L#m3U0~)?E z5Gd7!bU7EoJq;2*{q#9-ofpdg`{g~-*j6W3UU`9BeEtHt|F3t;f_YQr|899o)-78n zr=L7UF23ws=|uS*^a^Rh%@Zb%lw*&dDmVP@Z?bybRypmAlci$F7_4O{ZEHPPB@fL^ zv?Qz>8R-t!lCZ<=U%!#?b=-X`bxVeG^Qh@)EHV?Of4)q{D7%g zl)}jTQ?icDo$PInVZsmIdx{RNB~|7oFxM_}LQv~>BMj(NW@n&_;tJ`T6)Eqd_S7ZI z*i;j12(Ld-NW#x1ai@%D*U^26PFaZk_H>rXx^+9{?t7k+%Pv1hHRTm+Hp!}2_sJ_O zw#jAZPLpG1OqG3mx5%b_wK8-16d5;hm@Hbm1)e6r1lN;%+j*i7n);MN2#*~-NH(B_ zY6l3n7&#ty-}{)9w)M#QXHA2fdAGE86-zTRMd_(3h12z5pGLg89h*F1!Z?kw z@7z@@gNBThXJ1@}Id@3~lnxCKFy2j`#29!4j3c%XI(QOeHh>l_NlVS+Zb7v;mAA06Wl{Gf;%;Im2qvM zFM`P>Ruc%=NS^3sojOFskV&^^nsn+OxaDdpm=mgg+th3g6c*0cC5t*p1#WHSa8x9% zr+~>zUZ1X9<|LrQ8|Dk7n9Ul*UbW>JR^US{kpvWH+xv4(F4Wkc=ft}=7fn6hhzIAY zIP+FWM<9OKJs!{`-t->I8>oHDY^8@G7iWE$AnnPCHQx*=l|z6nUbJ3TEL$OS=1!OM z&OZqPv{W8?_CH;$!T!3>;0M?&Ot4EP2KM-X`w>Y3DB3Ss0bu;0N-2%Iz6trNjsg3|n50?j? zS}c1Tx@F>ZV`SyZ?fASFjSJ9FpQTKoorcCX88%|L9C6fSS^V-EIrse2poBjLttu~uYw|2xyaULr9=U0m~ot+cC})EVkFa z+hys>tup)gM#Q1#%PTLfk~0<@0bxzS3>E}oT~bsftJl0FXP$bToO$MaS%NtCITxG+ z#iLDDtVE(1EO<6Fp%OqpZhvqp?!EDM_8?suCl`a%l4@~!ArLjq2um+2<7wbw7DZjr z6gTkC?)!dOtLbSzu7q-R?#Llmxgll^5OU?|_(+0S>?po~;G>2nk~2d7P` zTs14)&n$NVodOe!zk@h=T8+zY(~XF?_P#^-0NojhL{az_wZ53vz=4>0e~>@+nh(Up zLt_r~S*pEFw}0>gapfZg5+-U{$q3o7VZS{7+zOdHZyE>^F79=kXdE?l%q_SUvYTXVa-uw)%RE0z_jHpuoJbyA4t zCmo>2=NGM!=GHFRv2}+$@Zj@u)U=5>#HL<0Zr_I*P4<=ta=WDCPORs}C7Y!Y4aiol z-z>!yOJw#jQ(zcRfp-9FTDTnZE|I(Ldq!p*J4wRQpwJu8Usnrv2q@0=iZcjDDY0c{UflN}Q&LF@{2r}dHrcsLs^_*8Ne)&ssX zvA0mSAs0oCB>Bxlp z*446af1}YUtZ+Vt*7}1_yofTe3MsA{2I0O=UVLdCtYVampys<9^B#%Pt257>CHMaw zC0-l1%UNe0B~y<+0x{-pc^+1JPeGYf4ITl7W0$N&!;^fR7v6!yv0@~q(c;%KYIVoc zVs{Cb1E|gS7oVc^Ej&u&Eipx!_~^m^}H32U74@NTfVN%`K^@K<=0w*;tCE z9^EJ}WF}`Ggi-52B7-T8QrYm3WF8>cI~2o^Atsi<7= z00Fw4^GgQl`0M~i*I8ld5RRz#6ci59``BufF)uwg2h5Rzo)Rag?qKH%_ugNa}~^!sKXhT?L#&DmSK2Mt-!kfamO-rvw7O|5Nm z-F25s{XTR*8(Jg}J-%2PVWpSgh*rAm+mKjRz?x@i(>as?2HHhTV{~5RSRyHVjHh=K zOW&Q?Wq$*xh$VB-WZcUIf3jn3^cA^1V}R*8f_G#NUBy@-EyUQ$MfzI$<=Jo^;#ldTC)DXJnGk$W;?_z;vDBB8U@`r{``cl@lPpW8gSJue9^V09 z^ne;AHiN+g}_t=lhqaJFwr*+@N@o`*zmzMYr{`;l{Y3n0R{w%dF^(P+)(&_F%5g9N>bJYPI3BaPp?EltnRn*dAvsiJXf6AO zBYX}I6p7gXc-tQjxevvwB3judrs?6)f2*P&>^4IJW+r7S`SZ-euOFKcTR}^IhRrT{ zj=qaUUuVPEN%{ZWX&M~v(UGILU{N~0#%jxG|DI*S>B5mm9%ghjJ-IjTpxHqy6Itg0g zZMKZKSg+V-Y^>2TT;%G%Q@)K#CW!l#NDVrNwInhU!vr1^W;#j2J8^vvYGicF-2(J| zE<7WAvyL1spd*!9 zdk8sLl5~6XBq!@o#`d@n(71+%aW@2KmnvoEwzeY*d5j_P&0N+=O>><3W}OM_j2y* zBBeb#poI`&@c2YKYL%Be1&fMSn1&)pv+ENtr!++@I1_$G^T%{TimGymf@@T}m)_&^ z(TY~;+WWeP22u}GV@#1%90v(c{86pslYZ|M*U^Nh0K{BUSt-kxyd+B(KQI6FrBBG9 z%5rHz%U{iuR62o^WDX`(iEV1m!h>{r4@#nS|&S@v2smpD@4J~!7s2@Q!qT8k;KEE@OH&vBx+j*s768L ziH;3~2rreoK{@uGG;Lhv|2p^$Lr80&f3joXQce7sr za2`scVH0sW2Bl{5(nJ9$h81Emh5sV~+4D(5s5ZoT&rMQ;u+N&6%h21tL%#pbugE#4 zpDYb^wHD@~;ln*uxx*XHv+R#`)$o`q_$q00?ZxeZk&qrt*k-J`6^SEU+lSuw4!Pj$ z)8wY#-z4|kcR!Ay+=X7gZR++SvTFM?E#P)2pzgX3zUFbKgT_ymawl-J*Iwvw7BM5^$AP$bCNpEB_WS$UYII+N+<7347lnFaf_}?*b&n}_5q(D6#G24pg7K1KG@EE^ zZ3Jv^auOUL9mqV?^VGwpLd3ij%|LX8h;tIg%KR~O2kTs0 zf=qOLKVh#3M+~O|$J2nDR1Aigabo@1){JFaQLs&U5E-iWSGH@tSEIayAq5Cnl%nJqDs5Ff#o}xg{;RkfMf_C#YB_r z8}Iiz3gBj7`FJdi08{KCu+R&D)jdp@>IVN;Udi`*JuLvDW4J?{gBQFr-QdIY?6AhZ zClBY0!#mZ|hGa`T4Sn~N*O+P-0|^xEV(e>7KqwjcYrUYQy3B;giei~Mb+Wwu%FE?V zZ@v<4?jcfN+n@>t`JpLmCS$@g3i7nvcoy8su@`sL{KBpfo7w?<=rWb$O7eo)-KG3Sw=cjVuC95GAubDJRih&^x?FvDrC|v ze`T56_UAuJMRk?D^&S5zLxzu*))v&4LbJ8gNgXjciK*Hv1X&@w&IH3d&6KF-n(6ntlvK%LoW)8oJ4`HEL3$?$lLh%J1kibRutz;!qU61f>ZSRxzkw zK}vaO5>7e%%`bi?n>MVIYyRb3GXLZ=q#l-}c86BjQ9<#Z1mcewJKYrG7^tkITwZ$i zX}R^LU&&F&%$2vl`(rBFdeC97P;(?03d+v+!tBH^ELvd4gO}&FUZ`>%Tt+GQ=y05B zH)YZUnK*GAWJZUsm$7EzwBdCxjK>>kp=tEN#htteeK4hd$OYQi*n}2<`8Yt}NE`q^ z(>!!$$*a4K-fjwYo{OsoPo%u!5@;lo@mj5S&5|fI1}-==w@99T>N&ab|DKT_{rLN` zWy=b1bU2wb~f{aS@eson@{KDgM+?-?3HZf1G!65=I zIP;orm<)U=eVPN!v~|95Bn}A(fMAmicrF<0EP-P6jjw(~Mh>f%+wb_Z{N#UrAvHU< z%EvzSL1}8XOygj0Nr>b+%5A+-+=GT3#qyIM?~rTWdbMoXv_zp$RI zz{nDyVrjhb=)}fK2}Jo&L0DMYnY_vH=?{+S^xSj9=o>Yrj~b^6)i5Nyz7yWX-i2_S zz0QCVk+>IcykPH>_9udEXNb=PF47R6iZY<%%_b9)dcT&cIsqNEIt6Dg%Z^Q7*)Q}$#{{&wxQk%*Ha9iMe}3hQ za^VGM$%!X?PJaEX8)f80m&(;|Iv)m2yD9yY9pUgIRTy^0W3m}vH#eGm zV|Tw$o<(o_6y^iEq4=-g0@@0*-Eot2aWcIe@!3weXS%vt;rd1|U)8V7xYup;Ov11= z{nZZvmfW4l##bc*nk#HzS+{D1Y+AooD$zKB{ft+vSdK1BTV%vA)QcjA59Q{`T@wRFl$FFb>@f2-9BXlU3kOP9PX4fW{73>Q@ZfIxr0C%ht_m{QB-Y+vbe zDn>V}I<=PDamMkko!ewe!xS9!01cP5@Oo-X!VQ+4?BiVFSZNx#sX3U2o)CC^pK_s6 znHxi`zbMw6JdXqc5t}V8o1_RNU2a!vjzegCdyE^&P07f>kM8aZ{A{G>7RcAz`v@eVq8>o!&D#HLC)FKw}FaP=u*#o!x%{SkK z0}Z<5BiCOqgNF{qXEituuu{*ohB$T`gZE@;0YjlOZ1Dp<{RcbH0S%YRh?cg|6%~J? z$yfIIMmJ0pvr-_3A47O687PCGrdN#4Rlokf-^eFF@jTsRH*(DKmX}G|-0GR{=T+tk9jm6<+ zj-`K-!2Q^qUn1GhmtCnpg4ZbH_6o8ZB#Cf@#`_JANq0Io7q0N0Ho6kICQaDUIa&Fb zCO6@Ig0DCL1sQ(acT@269)%Cbu;xUXI2Yl5PFF%X;dB>e6w{4LP>RN+^iQKm%bpP- z$Rj>Vn~z^R*9urQw>HZ+|LdFbl`nikZoKh#@-h_MkALQq(pZnO8o;xJiYosR2Q;Og zr>$Bwz1=NgDoI(aZ%HZhQ?qXtL2z8TSIDn*-%C@4tB@xWc2`Kx<7-b_$Y$r^d zDBu3xcjfmt{zf)!TrU%*Ou#X!SD<0eb{T@hQ}jyWh{_06@9;l6-HFnfFktEmW6vG% zP|9b{;u((Yt-~Uu$zB2m{@3X9b4`kF8w0 zoBA!F!w(Ov78QzyR>~$ErE^m(L)8;}v{dR3-l?S?XgeeZLw-XYXm?0v8a`qKj?zQp zerT$LhYZ89d6jU>;G9mqk&#-XQX7ZG=e(EXM1EosRWduh3ctHt@ZY>p{h56P0Sw?r zMhN^LlTd7J%s3r71ZNcw8I0b|-BLAZh?GL>FGZWm0>DJqsmz@jm`(r$b0U1H33EK| zGzEsiik+i0AD$hxywDq9&_oZHf>X-y;ypOZh$hac5Oi>0a**dq8iQ0p=m8rN%-({> zbkr))CL}f&J(UWb_vo3?FdQo(=*HyO98;Ni!sEp8z2>IT&=S%C zEob9{R<=gEKm^+0M4IAw+$a2KeLCP=Sb`33jvtY>d=*Lv)v|JiNUV=b3vo@2Bskh1 zTq}=09SZjhA2I0=C>{y}J6oOafMq(4rMy^N+B|Ul@kcZ| z9vx?%%D+P33K(Hl5N5c*lxFS^w}>*wE09*2=(5o9ov&hv^Q0)098dwBGzqU6v!oz! z2{?COz_7W%Oq3F5NfFmjyHUUyv*F9<`+3El=jayl)E;zkXahl&f*4rasR}aMU|`p8aF0%`~hkTd(9N5ur?SPbKI}-JrAl9r`T<^!= zcNB_L3M{NZW@T!NN#>jM7pZEeX$#S-55$Cj_kvN%{T+n7d{i+kgo{ zjqgiH7x-Ps`4D5&Yt*5icfrl*?<0P6vc)H^qZJ)}wrDD?XWE-|42ZHkyjLgr>If59 zw=j8UjiJWlJ((0i@28`<=|EdHT6*Qx<;u93AXy24RHR-cdaMkt24&+<)(nQHBDpzh z*vXV<{f?dqdZu)mP7!o_o{G!Kp^IK9DBhCelAyaT3`@SZ(mo~!)0!;CiES(qHInJYs#wq~B;x4yTeG?w>ipZXZlht?RQ6IJh3+HOD3Q?`(K-9H2#+K54EPeQtkZnWE=WK-pa@-y{g}0< z1V)&!%FJi?DPUFAO?nZ;)DpFshJNRWRblO2-tMNzt&aFGCQ5iuM|NQLof&Hk+2IN; z(Zw&lBsDerWeQ3uCQO*1IAr-2=tWJTh2IoVt#ErTUc5+M*)wL$kiGl%$=EStAlRFw z9$C60M~zbJmFF)LWM#;SShj4L3fP%5XCRRV0E88vdWOqw*w6c99M;QbU#>UHs^erR*jd1(Ug()Tb> zMeM5xoQN9=vyqaY-e0!kNg)yPF)PYCxLMdhl~c!GRPcaMI15_P{^hCR?6Xu*=ooQnCPJX~3l{od%TVXbTZ{qSVB%UmqFZ2I9^yUFMR6`Y{lhQB zsr`=E>h0>84WWZ-{V}LNoo!{msh2CveXSc}GC>84c?Ha(HgAaTRZQS|Lm-T%e9nOr z&;iqTGj@{=1hlQs3A=v5E9GX73?6B{24g#^jTXpo2?=Ib(y}0*Qr|3{FGLz6Bs-MA ztVZpP%=m&X$8~q|$fHO~A@4S8AeR$+tQe}KV!1~euGbKY0Q)yagpDq&5chy-H(@`A7iK~ml@7r zfhF*=tXRHW{&3UHI70DkRqPrY>Sg^p#MdCGD=I5w&FWPuaF06bC?p@gB7eQ}4w*E0 zl5F0*Ni$p5uC7F`!yXwoZX5*LL-LB4n2UYRsyvI_P+yLQQt!GolssY$BfL1ALgc8s?N zzqg^{f%z6EpLn9oJZg$O`Pic}a>PinQ||&`T%Ur{O95r1uPw-KvkY-Y6}%t zYhL2jc*k?yJ=U~`u;h8`Y)HcHbokMFHTJAkOA=2h2W7_e>1y#7A>W~xv1V9BZ@Km@^1%;(R3?m{An$zVJEXZ8RwpJx z3;X3Ii&6QYf5<9S6;KGDe%fiWefxH)*X(A7=>6HEZYa#p_RnRYAwnSFUJX#)o z^kF$>&RiKYb`0uP%jAwbZkLl#2mhPj{6;?Xp%2RAk3SBpxf>^79fp#Z3e34#GxNO? zvvA=;RR~VQT-UBy3zue_Jn%p*k_&QPCRwydKsobD}%5u^`g&>s!6Xa=L zmJr6;=R})F@gqvKg%i%03*d9^_}t*-c~u75PrBCK z6m8JOWFC?HL^nbOF$I&7I_RgYyg(`k4brxkMF^E)7iCaY1r$p(MIv8;w?OyUnVjb0 zHhf_N7bcxz>+wTg-L4^)va(8)qm`io2hY=^(Ht*}$01`~6{Q@e8V5=E=BL5LKPpoi z^6Cctw^@iuPrXyML#=Kb%W&JjEI8#c**4Yw&aso2*N!Y$?_InArprX-G>S_t=CBit zV#O;fWZRCNDBZ!iv-FpSFvh0G|2v85|Mau=h?q)!VO$DdT&Z>YfH5Of#BmF$T?}_c)j(i=m%~PO+1z{ZKlZ z!92Gh-M={W{V6fDwSQEH_FYo-zwE8?znIkJIcuR?%j>=i{#5+ z{(}7Y$3Ky!OP9!n7hNouUw)aoq^ZO;z#3k?a=Ct9j+37_gCdN{11E}0df+y0Q@8ls zIdkN~3(u3sAA17A?o?RAN653!J}Vnu-5{qdSOCG;gn89LF&K%l+vNQp{E$5J)bp}> z^=g?iWr{xM`GwEP(4m9nb7&CM*48DT_~gf61((Ucz4zVn^{;(X=FUA<`)G5G=bm?t zoN?xyN^6Ya@LGdh?W;PaTg)%{mA@kvZ7=#9J=bG!T z(S*xi{qk1|;WN%WLmF2#%D$Q!2>tP3P}FYi6st(ar;Lh8ggj0YPoPo+#>-AoJX@z4 z44!9JFJ8J-Hom$UO&{vj9d9RQIc?9;eXzxu=r4>Ax|`lB6v&9=Xj{NrRC1);o&2Yk zmVyRecxkcBnlVvo_HCED?z&H&S@<##smb+Nr{aO8KAqNg9lvz-gljh}!_u-sS@rTW z(t)E13yP~S)>6c5|AW?^&h+u{^z4*-;1zoI1)QJWRfmuY%Rk9*Q>=!FtqR1QTW=69NY>(9 z7{RfMu=0~Org;f18O-ttcsbqn!$3!qP^&%ts3T=KyNES5gO_k^Oq{jkF_SI^e^X49 zKpcmn*TH#2@H!L#Cfr`@{6DUR33lgIn)cIj0YLPJY&`e&kZKY zrc^GnD-5d?_&Nb%0@{I$(HsDPjXB^d>7u2Wr*+VCkDdx;5KbY3-3+yKCj~r(c3)(t_))-a8Y{r|KJFoF9nw zhv_ZT*a<SgEc=j-ofUtLsyX&NJN|)^3*CfxqunaYbjX3lOJwLrPbxbx5S6vd| z_6^%)*wAXgv{M$nvRYPc-UZ&YSI;o&U_u^7D)KR-6ktTHzgLdvzBB?&nGs!!smvJ3 zCfUKJ)Bz(g3S^%ov=lUS2K*upGw2`mIMS`ndV?Wi*~k z7}9Wf_|eBubK9s^^nGAj6r4jL%<=)}d;WGmT+gFr)abDgoEx5}+~Mk3P@Ggt*jc@)o9f(p zWX|3)Ce0O5fq|ud^B-@MKm7Stz^oZ(Uss|61BY}F57onAyBt=?!n`Gw-ZJrx@ECf3 z8Zz>My<;Qy4{Z@-%jVrO<%kLB#oQ)~m#vkam776F4VWw72O~iZ81&et<%p^wJ#*V! z<2c6~5`OoVS7iF6Vd_a)jou8+xTvjZe~{>>oKppx;dn0c)$@d)>L5Yc=}V3|HplFu zL8L#VbJlmW%_r6J5}bp)U#j!lWLQhPY~EQTk3F{-ispVSg6E%m&SIs9X7urdCzfSW ziKcy3EU#|bDbps7k=n*qdFka9sJz*X+UR|_8SSIV(^CX=!c}a0rug9u!%R34mc>z9 zon%4l#TI*3ybcLy^&%iwqXRsp5-ydi-h72zb=Bok1lnoiv2r>A-fM|&X^biY&2kjb zUTY041B>SY3sr-vWy{W8@~(e^9ckxO7F^P@01bs zEt#)NSzs)KZWo}AX3yrs)l^o@zSFR9V8P^r(5HeqO+Ioi8U(aJ7z1%|Z>N@0Ss{cr zw!y2Dox-?QC?HFpeh@A^xLbyglUc`~0u70aK>TQ+*OL3{v)n=LOasDN%AlFou%IcZ zUbwejipmE|)zHy$^qiAWyGaGb9d_XMDXFf{aR`>^SHeB|*aP?CejJQaQ77Y%o+Bg2 zO_uyFbo{}dx1)_Ay%NZ0O`BhoWfecRNX0K1=#g=0C}{pr2D;BQCZ--Bif;M;(2nx<(g1zfj7N81{CwupI{Pz~;?cWY+9s(R^kp?$4L;6DG+SXPpDDRtx^_ z0Ry>OX3d(7ICHZcapZLQ=*K@HM@>IUm4hi$r@^X5CNc!0Dl353puttJW*X(nE8iq* z5Z9)4cjg&q$~L&ymo8nVO)63aa;97X&FWW1m5A3C;xmRwKK+@`sdwvy)6SN;6Q`nN zsmyspZNJTe4vXh7YK4Sq2F*CroJ?_ZXcdz|(ekHTyA2JD3QGq|N!4(fHsd&{8a4_N z3?~=?*E;Y$hc519UxAQ-=A(dLENfh|tXc}%cB7gk4+jm6lNra(*H)jF17pDe=iMK0 z&d~P(k(j8lW+WTJy|4KvSYbh7sq8_slI^P>lftqp88Y%PnKkcpO`fFNTfKOiu5K2v zuFb{3^n5jZ$CB7KTu+#mmJgF7XC0586=Ok=E^s2W)`$tgzhbb3C^I{Cja%LU&`|@JTE`|!FN!?%Bl}4{pLwA4~I!<#^ucu zU||A_)EgS+B%sZum>ibpvF?1EufVdtNXw7UTfrktN+ovCa^e)Hk=w{>C;K#;sE9;zZPJ z?Zh1t4{-=$NL{%okEv@$GY`6CiqYt#M{3Zv4#o*CV)kl(CVmj^B_WY~l!)w9p)M{E z2-QJt9Sd*i=mAWqo${2ch@f%GH0iBqhLr~}uz?}Qfi;fuibBA(6G>3Ta9?)7_d^gd zzLxfiP+6nF81X4#p7P3^IOZQ@1Z`+%r*%vMV;Ul%DJMm@sa9j%^-iEC0S-6H{ zFp;XNxzO=_tv+mi<@xhhD&?q%%N&=UVOz*4kI=-nmL@XKEM})(VJ=EztgMdch5|51{&LHm8Ya2mimPSVn2Aykk3?si)uYog z*AlZR!K)C6m(>zG!7w9PT6Vubm9LZn8hn8ziXAWv_5%hT#W+BytIZg$n^rNd;mSMJ z`D03A9y*GZf$sKWKdb4Tz!(kK>%CC+m^mNrB4#Cz*wuR) zD>_(~hf^~4Hz4~TTh)fOw8Mz0Y3$NG6?!REL1rRV$Zg?koC(uHmiQHRN;7y#C-Aoe zaH|JQdrDZ&#>0M8u&6u=zao=PbKuL-$8ko%+j~o$o7?fqV(nN3`X7IkoO|gYdE)LH z<##vUB=3abT3fph-egRM-hmLVst3U>A~99_BqsrFut=UPiKP-9xc>B)JLRb-pO;In zej9{YF&K))eR<4e7h_TSa>6}PY)mXoBxZ$&P9?w$ny?NVTAdHLt0qMYTGbW0Dw&xK zL5r!7>FPbs#imUMz@$4|T!HnwOmHXQOjF_k6BaiilOS`~2KKbBDC#_+^2CxP?K^}& z={t~{k1FQ+17uBoo|~8Iw_KN63e?!_s6*9W83&>kr!GW0hB6Iv!s9wHb~D7mcBo`F zfL1%0G-1+>#EwoJGSUGCM5RX+XAhs0eGBhn6g@>HYV=@F66auNh zJ(ELV>3jCo%kaUYQZTiH18a>k7!U=nC3_ju z%u^+ecn4h&b)P)znSgGA^5^T2DIl4BJ%0Rnb(t6N%x3&<(G%a`vjC1UYrF+x`*Fh^ zyRNw6O5L}8Xah;5vjOg3%f4d`GVm1P7B-(zW!Nd1`FKVfVqwm{Kubvo4dAb~B2+l= zL^JFY%f)a%>vp+LKED|XhgJx&AqwElXKBIm@DDyPYvD?nBtO-GUBbVAxqeC zC=*4RNniSauE$xu(WSRJrI+)vvN;#4$K1$fl$X%+=HLPd>RwT_XiE$I96W^bMg@@_ zZL4LD_@)!WPotfzgJ&#LP3Y8h+uz{?*KF%g<(o-=I~YY?6Q*0PLC-@cJ<*7<{0{P2 z@^uPUtMLb{@V<-3^YVM8KSg6zaIEH=%!Ey!^ptYlTgGm+$q%1^zd2SL0HsZkd>LB` z+50@H@W)}6<3EoQx%!=sn>uK3N}tIz$_rNc(3Q>IAU_#gHB3%D?=tz-uYM=T&pT0$ zJYtIWC?_ph$k+BwZD*fDkn;H41az=t=F(6XAs$BJyydn#WZLYxh@Dj;o6{N?wBr~@ zLq9ISiB1&^pc()MeLFY9!bxkBw#zv-UnRhxd3kn(b z@TIOLV>YVg>pd*RV)FsMw`42cVjwpL9lZLojtpx{L!2DP=H+yuB(zS?qzn%EAU3+6 zRDf$;xG@QPrLeKkv-y_pYo60eSaKZCEzCW~1d2qsAPGKEYfW2dXg}O{9Mj^KDU8-l zH*(g>18jY^B;Q2GETKBMeMcTPNs3=tDtF%bSNZIxuGfYnrsI*xuwe)Lmp9|udKrYc zuXWY35DsCa@>vm%53?+73iNWsl_}6`_Ux8QoT63)SFk1ju$!3GH>=PLA+HG`QwBZE z+G#=J9yZ!BCdKLDeoUhUZuyqxTFeV^Ox(zFD<-0K>ivwPJAv%1H@cYfnT_R-MpQ6! z0cKg7wdLHaPB5t4a8@e2qggVoxvKRmr=Tb_vRQh{pn0R=9plb_z3W~iNtL2Ca;Axtqa`!2Xdt`C6^tn__r1GL#Xb266@O%=7HLMM(V%rQv^K)}6Pm8Q_Y+j)bI+ z&apl_1XwTRLLl)*mQE=QH`1!R*DLwfI3^i1UL=IC2^R16*;Ap$rMYfLD^#9`4gde- z{Re=ZXL;ri-_v{3Xw-Y}#j@lgS#me-#u%_MrU#M`NU{k@Hk)MgWwX1G~MmQ$rIjs_RvE=Bk%iRDx}LioUMFwGfGt5h^55=R76T!-lRU4 zf>jHYZc|m`Tj;f}tF5z(=3mS+;nkvCmHQktsFXhK{a%3f=}&&nJ&(f&_uDW+@)8Jz zREvw>PPf^orKkMd}>W{56^mWmn*R}j*m>Cy~MBQa!fdNY&-?(i)BoZEA9h2 zUBx^Uj0)$CNcqIYOhG>fTy^oA(iN`UBL^VHWJpW9-DqDYN_p-CS0n^(%98bbZy?8X z=eQ6<)cr<^DKJ$q@FNE%q|}1E3<^8!V*=E~TkE3l`RS8}PAsEBueA)=aGvnALCz;$ z-3&s}OE_ncb*V>&=j|Coi6cMZTdBt+Wkj#*IbTHU9z_8XIsgK1B$7OI=omY&d!MJt z6fskDBzVuUzR+LE0^^j88kmAkeo+q}Zb~7p<0p_UxIAJA4x4vbP*=v#c!N-hO+7{n?-Yv3>S)pR?J_Ctm){Gn}sqkdR?cDPHC=lLRqUwkQ?QzB88vgy9ziNBNN)bqW*p3PYzqL56ENq4=}?*)eHsW3wV zz(DTNbNV?_!=^q2Q^gi9Ug8L3Kl;JUKd@oz+QUcIoq&)Eu%uqP{cEA&FZy2kxhS(=Uyg&SoRZIrY^sHiM{^% z>lgvg*oQyzAw2FXn~P#!zWiA`L)`T4Vl|R6fmplA@C1#GUk9)UeOOlUtBOxUssUP!Lz~R>#=$Rx7c$P@KC(qij4A$r}B18b?IbEB!j{s$<>XDse7NNkMs80n21H< z2$WBFgl^@ub_r6qK%g_Bpg0ssO1=d8k2GpGUKx;rjtM2U5B(~-0fjL5HQ*8m)Ss3k zskBI0#ns;lLW@EH;ZC$c(K4lxu00*RUJjo#X7c+UHgN?kb*Z8fNQXY`E?1}_klY4mok!-6z6YU z%|}GoU7-x*+2UP0+w-UHY#_OyfwVlK2q6UI6QrPl=R9{7?Po!_gr4ufNXfiVyiZ}j zLs&9wZSatlzi!ch@-xi7rMRetLWpuU!kgA)MamoV(?|E?H>7-$Yxg8^Mhu+Lo2d_! zVB+Xa-AWp0vb2ttb6pVbF-C4gdcHp0e^=Weq6H7er$D)oQI0elfSLm~h>-T8g^O(E ztFL(Yxs6KQu_)D9vu4p>{4@7Xg@LU(<268GUUu1BTk}={ZB#&Q4yge~vk|ZON_z-(Lx8$5(o0D8<2`jM-E-E63DAS6D|z4s$Wj(VQ^jn`iH$X6}6 z;sQV!F=8ZeqciDyUKRFNI~(QH;sweyeG_P}Z&N9;tsK~)!ZW8L(LzWeIgL7^oAAe@ zGkMM~oHxk>;Tqlljwrv!$C5947H71JYBP0ezk|bYZ zIucgD^Py;0MPxyEi^gP()Do2$G!)D&`u3Xu{ApM29fjmIGG3G#crMgk1f{RF3s?`` zcjsJWOX`-zDyzSB^I}U$THT}XM&ty!at-%-#+gp*!h5t_3Xw0UfY?pbgR%E6b0dj` z3~dLCp}m=ro$8*UUF}ru5gS@u5s_nZq+{^?LLpsG{tvBNu@RLNDCr6eV=uNF>GmY! z{{Bu7q^_xF48}-!FhyErkVb~4NA-eQ4lS%PvHM$o-vd(6>UZn(6Dr3NZG;Pa(U5Fj ztUeK;68i3Qrt)b}PSIMXtU=OxNiN?x{uT>+c~=m*Ha%NCxb$U<2{ZvE8^Dn;1%h|@ zy-h6}4wN88rErXTrfDak^4PzxVg$bJz7<5W>g8u_>E-k6`fIPmqk6;MSn;GSzWy!> zfFM4;lMTX67ESbOy9z)W@~}hcec0PL-*S%6*vHqpDYfViSIMkJY~4C@WC6cwO+=;1 zlj^C?6o&q9?%fNXymd)!GVG;_fUx}3isZB;2IHslC;?ro(@H=b+qLcuYk_$#U;Giv zD=fC3KK!U%H@M0Q%7=JOLJkH}Cw+iw(>WtJRXl?%{?@fr^(k*#-m4d|$+Gr)?(s^a zdDG1jPrp3z*;HYAwsjYtm7*?D1%r47noxmT2dzj8FTTuv_P~#*%KSF@;1crBLwyKE zwHH|#UhgtO%WvbIHrQ2HU1Lu^`IP<7|MmAa4-_x8Q$7Fk3L83{5cbSj0Nku}kvvkmH_%)CTxjE}tn_s(2AWg!{2B^(2JK+q`1-%QXLidix1hkYZ2y6S4w@qP#EQxq z=$1hl;-;y7l$8;cUU!Qjq7 ze4yq?~J4ijj7XcX-BA|D-lL9YQ^!r`=bD&IE zTUB^>WjFA={4D^a$Y>Vkz`osE?7+^=_L-0VCZI_T_P_t?|5?T0u{LDF9QFO{tG_r=RhE<3M+5#g<@*~Q3B@v(G*?ka%VCrTho<&jI zUl`F@^d#oP@C9&F8`Zh3Al&2uSdtI%3)3?P0tuid0a{O7r^@7f@{k1(RB3)GrSj8k z&#ry;`pVZG)@klVa|z(UrVycKBX`hK?BxttNm3w5>{ac4+9yMWiDic)o0{DQ6j7V1cyroe&w}{)Lmv+6p-- zmo+Km8bbzE+dk;v$k7vajDI8XlEP32} z?zz{8QH+i9ylS0Vx@4JSHW!iSUb18fUU)n0R_%8E4cFUV-~o%tSIL{*vSpJaa%uVU z8PRveva4;zj2XU`trVaMmUGvx9X{Dg=dT3OOA1&(Ck=SnDn7gKQ=jr4bhBB`L zpL;M7fRlJ?AN$zH?ckw<0IXDiMPKIM6}0I`050*~?=)UVrrNLH>h0)6o|cNDY%vYE z84%=KSlftCx&_@by^*sVm#U1%8(}gAqr8Kf*;`UxbdCI|ijy>e? zi@U8UIwASZb5_vjbjQDhFLg*MBb>Srm<74==m8RGJZ4QTb?_M$6`z$*dH&|A*RjM( z?4m2DS^aS)_29+8Qy|vhQI-JN7==Z1@v>`=Y`1-c1GCy&VU2d%ylJDo`Nj&HO!3X> zrW1B_{i_g-dZb&({Z$bL9Yw`%4f!;A22Mhe{E3Mr(_G#;E*Jv9TW}C2QX*$iwBkWk zX_uf`JAk4)dFlw^MTL|Y_yIn7YiFZ9^zg$rV)8}S(s0Ten(Mf@V!fL7#{?WLCaGb zgS8|cE|^9MoF)vu@sp2RE&Y0z%+9`0Y@45FAgt+H;7)+JUMGRF?7<%f|8UCIW|41sIbsueNt5Wqh z73ft})mFv2>%D2yI5%r8<3Y5bHm^yOr~3C*aqhU|oU@GFwZ=!r#z%j9kf}26?>rak z^JFQ|+0W=YDUIyzsf2{L7vk(8m8T&M#TppWe%cyNz(XBW#iMMC>GjP)89xIL6%QQ` zfLMbaX*}h8rM0ZA7zHn&T-p_7!;iu#IC1Q-HIsfg!=98AXe%knv!6Zk5X!9F7A?L3 zgLfawdV?#xit5X=9_a^H=Ugl$PMSoC^eVmgLgm zJ;%~zj=*=j*f8V{sp1CXV}%>{%-y zjHphW7Epj3sDn`UadLxiyu1YgyNEF4M3@N++5te~obtB6Ko#U z$ru|)cdTcc(JCsHQIG{{v%wpHhSmMPZ3s&560T|g}D zL=SQi)ItMmN5KhRQ$j;{)WH}A$b^<9Cg%AFN_91Ka-6*9EcRg*#$9Ns3YNa3h^cnN zD>?(_wpyO@Ng2$cI$TQKK{~XTibQZefr!MGw*Y~!0Ye0+xeQEPG$x&0Fd zcH3KoU)!6Gu|};R=Lyob!-|CHM&3BLlmQ>BR@$lKhv7efDcs8}x2bcM*i2wo+gloK z)r#k=6xyg6rb%M*{#vOa7;3lNc00yy6^a*S-qK{*z!Gb-PVL`jTPft5ydOpzjWA!)vs7R#g7`WQcQi0Ca4L_XB5$(ywQ`an7QUB z5ATF1Pg*wXRabl3S}@S3EnI9vN6i8sV2^ED^@0rrNp9r0;W$^sKzXon=E z9!$uPH}C(t?=J=I0*ow%1Qhw=*PeaEj_%)%BB^jMW%Be%g!@Kexza*|msnC-W!v|k z@S3tJ1`FgMkTfl5J;atnaYlir9TlIk|Nei!M)ho!hpM!$VSoa}qRBdfc5`Kj-~nI3hxTo^4X-V?CFFF5Fma^> zrWr#RwT%L=1_U#)RFn;4TDqh*OYnQ$F+>=(E-2~q9di&rVw-f695FP% zK%vLd3ipmMWX2E~EX!FA3A-CHRTO&H4vlzttBPWj$TjC!0r zd9oL9$)lCR3~Qw5h-xbW$caZtcs~^Gg?sPr)dr(OFA38afy+lh>biVC^}E49QHmyf zp8EQg^OniI{h$ZcZURxxx?XqQa8#HW%q5bbCN zO)U$rZ~M-@0LGMI#F6Sjk+uR0{3i7(#e}_29wA&ib*z2)cfU+32*tx^>VKBe5AYtI zG2Jj7jILCFI*x@vkuMrJ_=cr zb#GFRU=MY*p{@)`pGL@E{Ye zDYbW2JZ<;fc{5P8A7>4Gwpqwmx(`W&iW^djK*0+|G4qg5t#}Uq__cqv*Irs~mt1$d zI}3`S2D;VC1SsIpuJ&%J>b@7&uPEJc9+txUOhHT313ZtZTX$qXKn>fhW+0Asm&G+yIQ6mlNcEV#^Avb(pujn$WBfA**Ujbfld zT^^f$hB8rW{*7fbEeyDvbh=&r~)$i=U!15=b`&2 zn7))!rh+nfQxfPQ9J_2xBd$!tZ&p&RHrp~gD-*cCjrl_JlDCppggmRTonVKafjHjtw_0XX; z_NhfM4Onh&quJH<9B1h6f3e&EVkEPS>aXHrKCriF?OnGg*wF?=fPlepE`NW8jva*H?3o6#T>$x zhw!$k)YZap^3t+04}o{#d1>H=6k~mTt-Zvo=0SK9qeqS)L=Bv1Y|QY}(Y_RSP#ccl zu#{@|2lfD;%cQ%lG3WMT|2fr zjO)uQU!lcqlu!9PeDJVGU`on}qOgx!)~*EXrrbXL@jtLZ%r>sAt>t+EqS12`_I6p~ z1*>3C^B3_N{-)-D8kf61e49P<^m2w!>;gNUl!f4Az358(mx)S^iQ;@$C0ZL@xvkNT z@7ZEkTrtN6l^0t*`1OO4c|%8zv97}hZ7{rE&-{X}qsJ+7tRgCqi}x(0O?vU7`HWP> zX~1(HKDd-AYPSMPL#l|`2r8h9+dRXlM#n0oUIRHrXkU$Zm85#^zV|MBVbz~-&XYtI zh6h+_CyP*N6=HPna!N6i<#g5yeCxN@YrFJ*RLz}ozW0}cb~;t1F8!2E%#I}|oR?ds zz91Hd>Pi@`uC9S1nUhTQ8Ae{ZnNV*&026bpf%cP5Ds|;fmy&}%ab&k|?uZdX>@*=M zsvEpmBZtrwLq$mNZ zen*ZNZ_hvdk}X>{(!nxn8xbsOx{nlCqL?S{g(L(U(4R1th*k_9rj{RuLy%giEUToj zL#RU9uTXs%I^W7J`XWazvGYdz!qK+y-Pt)@8(u*(H2`1G{r|Npw_O0ILwD#?_ zc)L@4hVe!~OMdX({~+R0Y2!$X6w-Ee_{b?{;r`e@bk~QxH@c27d}~Op$O9K}$UB-y zMK1CRdXvv>0+CWae%b&4|MW>jK~#3@ zYgJR}>{lVI{T}arH`4I6){JU?$oOu6mc6ytJ3{nkw$;`(kV^o5K>c@Zk%YG!I$1$8D)JBsXZUs{XP!kSmjiqbO3 z$uT2`Vf5GeK&4KK!Sa|tUkP09CPs5k7&qHn6zglrUsK8LZDp#`mf5euJVodmC)Rlj>7V({P@07 z&~bmD7NnwjFd;Qle1&pCo0Y?rf zPzR45vBAKz6_sS$<4?Y5U;6Du5S}I*$-234I=o^tk|99HRJR^=d+y zW2bwb)gv_-8n~Hg?{rL-0CYf$zwY!&k4WL?EE_gt1mWuKHs_*+czY;g2z)CA97T-x zbAtwJy@EN#Dnx2wq+$tqRZVB>CY}~{#D0~fU2ue7Zt-(@_ zXS%8uKwe%2>4AqHe#~kKNzJBMN&^6b(JyZynJ>!Z+28eJ5-mo_D|sI)_p4zDYDL@4 zY-eF>FJ$V~WtUyz&pu8RmK-^vqgsm6nT|Z4zZ1! zx7#(>Ot2CZuiE!EGH<*X54R45%ye&@o?YMHH28-?`D& zZQM>R%Oa+s$$?N?mhjhcaohlr%ust8QihLmsdFhV&$H!EJZ+6k$Qne;V>`7#?xBWe z>PR`q1*hk(pflmOzNjpzn1h0jBSJmzCRWbzc|sKFJld?zlkDG>uS-|RV{IalDxw)b z?IO>QpRAOV9)g5L3Byn>ffb14qi?>`#K2`+=b%ZcaoNBP8w7tmUefZ&L z2v^OqIrC?r%%xDgDF6#Dpauni^DvA(iyUpkkyiWqx4w_^%Cj+iuemKMS5S+U|BDs; zyRrn)Np&xw$j8I*vTCuwHSc%%!>jF;SKp-9b|#EXwJy(4+gKOcwYk4S6!P9Z2mSlG zjO1(}%CLUJX4`%ExZV7rkJ#(azi9vOpTBOu^&6jq=~R#*D>tCD#RyNKil+@i-9zZ{%uLFrJ1kw=Hsb|3&jZu-LAxB4 z;EDuN+Dp8LH?5uZ@v|7@&2%vgA2rp6jhbeM_mVyvTu$g&aHJ=#g~e{D(_T|JN)>Vz z^(ZHf9iSD8`WIBxks~Lp?M$&vnRgvt^)CA-dg&{udzlQ#jRRLG1)&f(#=abYZ1T_! zcqF!f7kBO6Yya@Iea(YpgpDdnZ2#=r*x5OQQftfgJ+_S-l9?Yp!zPIb)tHs2^0D9$HZM@KMo zZ#pxcxsWRulx`BRD{*dD)Ir+CGS5wm#CyrxeSdk-JaDIvOQ942FYuyEmf0W*HP)?u ziLlC1Cl2A4$f6TH3l_n9?sA%~*73u$trXCPWjEf8GMVmuzKfP++28!l|E5yA0n3l@ zigKi~SOHTA3R=sqYpTGk9mU*b6Q(S(Yp%N!i%uv&A`&Se0hlPfojD_&S2!vvzzzM& z{Gs*`Ld!SZb{B;iPgB(N8X?qtLU#(;D~zOqBc%tFa})F+9s_q?kAX3$W~^Ox%f~2w zskY0mz0KZv{W<&OC+;EFFNjx7V`42*!bwUoD0HVWhrkygc=BxaCD+)PDVO3Es_zl0 zqiaPmh>&{`C`GE}+y6Q_HhKoHfmy(TY23h&(2Uqf)y?WdK6-c`bFQ1nJ5G0@s=no^ z%tcl^)ASiLy?Rw%wgif1DR(l3X&3+_WhE7O6~Kftn(wlh%P19J#48< ztAz5-V4I^uGR-&kyj~wyh>U(2U@C?bpw5zOZ?!FNzHB?Ttg+*Vs7iLoSw+!waEXVU zrJz;xrOu8l3du%N|FY!9TP(kHgpHY2fe`AjKlsD1SZ-&t6;baYdeP`c{S#cSz)Z5U zsN+CktDveCu%!vp7TIN2-$M9Ud?MqxH58XcfENRiE&Z)$CbCD5+kn?Pm@4h7Z@JS} zKfl~w07E{{r&zHMj=fFy0ryVJh^NHViY6IM_qUCkwZP_Iwakv4K4X_$eY0(UYlZ#R zZ~hua2&qRR4#H^VM5 z*Z|BIg>%JF$yJv(4#Y{v>_7;MmJ$0$;(9zGch7eWzAj*xbw#~oGd?5|9iqvsid zoQtTyEq-mVJg=Z=oNyuR)8v)UP^mtcuyY~izl;iD9ZT7Hi5KBKp^ACwmq1#mO6{br zq2<(3o_{DItCO?~owSLQ$Jk&}2l5U>@moZYk=j)bN#R-LSQLfuNn=>sosRaFnIh_MEtB`vKTEm)?rRsVS>)Mi{Zt${oR*RM?)wz? z4Gf{?+Nw92F!cP37I|^k2*RJf)-1LrVJ%#ASul$^;{*m^sNY!C-}-zWaG^>8D7rBk za1$M`KF(ny#@eNeukgO)R%lUG@iMaX@_e}NEyJa`(26(RT4$_`qx;VXlM^ujfPVDO*N_j2*dPEinej2`7I6+u!|ie%w}82v)Y z6AGH(l)Eyk%x7)@~9||g6z134|FbOb{VbxTEOZm95FWC?f7(Gm@ zie$d4BG2P=R@5~y747&TE5cK4>!4*+%3O5lD12$?JI#`%qlt5oH}UR3&d7r%A`0qO z$fJ*bbQBC@0+|!7I!IK7%?e`3@EZy`*W1w=P97oRc)|)sR9GD{qW17^3e7aEW{}f* zSJc%YYyion&~cz~Hl!CyDEy_MOa*h*!zLoU$$`rW!Z<$DCFm&Su$i&$MqsL}&X6|x zIV{sPi?8eyEE2H>xR@+m5YkZ0u-= zhX0fkDWVOatp(*Rfk>NBD+2aR7ZuIqkOawh8Ze4CDg7^|@Sp?b(_DYhzW&eufHE5F zZ7{yBTI3!II(W%#=McMIt5)GLI^b`dE=nH~wr38Y`JR z?!N;CEGtfhU8j!kw`ZSv)Na1_4fo#I&5{8Ux)?|%9BZQZJ8 zZQhl)u>KUmh~Xlk>)`oXmM9XHdC5!fRYQ$;$L53`$#SmU9BsC;MRvi#(ftE4f2+hCw{8KiF)Q6N=IYSfv z?(hGVs2nQGd6b$E7suLUIT-c0dwvP znQs4!Fd(+Sv(|QPTx)lKup+u5PJ?c!ytPKHs@owRawMcN^+VtP(}(Tt zH(szyueu#CS~=@jA&c+g2DA&$IGON5(tFpM$Hr#7Q_(#+9R>s3-*H9=pXrnLoq`t9 zPZ67W^5`ClnKpxc4D(=_o^EWnix@^Qjb7-jTX)#7!8Hsp*zWL@H57zxr3gxGR7GV4 zHjE0)O>5uq7MkDv{r>`j&kzuCD2VcWq)(7`A)EU0VtyE^nd;`7Z@vzKQwZ#Bn|1M0 zm|91wbs`RA^k%0Ep_qF}HdhdF|E%tK!^ZWrna!{%GbTgyjr3_&Qaw4rM z`YnX4OG+y3%fI(U`{O_Pb1=K>C`zlrIUskY@Ry7i6@Q2%L6&||HK| B|4#=x}Zo+o_1hbF9&M0dCclF5UNd*_|K3RX?BN}m4Ou23A zDm=HNHWbWq?ORSxsm-6ggqoK^+p}}GrzW0S{hRYVw1)-&(}9R7(D$Jw#NN9g;mq{gM5 zmQz~8=#6d!aOY&>NuE8=7B5|7M^7BE)hnL{Dz)9NSaKEp;@NhZs9Qy47I*?x4$w1m z)Awy-w244|Pho_uu+9XBO__*`@K5{#A?@y4m;sGv}cXsJ>QA(k4tXFNdX>BVz< zrBlY4uVYZmft5ovO+4*vohUT-8}z-8T?0R!uf6XSw3MhsY$Gsx%68_-jP-md!vKhD z3)sFD@Qh+WM2t49B2av0&zWWW4jtuvP1lj9U1^7b@vE*HY~x0cBaBIy*S#NMdP~tJ zOXd@~1QjF@zE(Ip7q4^Z&?-B${-6_zSA9oTQKFsb{Pf7#Db1sD5-O^q4i}Q~P|vL0 zNz-Vvf)G>z8uThH9C&U5TFCV_EA!Cx88FHljw`CL6~0OYs&?Qyo62cq|rm2>UBt6v$S?19Ut3>+Fk@ zWSC6zjhQlLV5VS3MM~Nk3#?_@&R%_E6>TAT@4WMlEnK(|r5>iF$;VH%L}>Z>b#*hN)@qb(p483ef)N?OBKD#ATP z7Y%FZ-b$D0Go9OUnRp22cIMP?rP=FPIY+YW{I79-XGr*99)8aCXi$ej-RzNL0Av+< zB&ma*WXTzM!&yW`$|!UiNxI_bv3fR;kT?7&l&uq_+(u9=Ik=|U(;Av2r|NJOglV*? z@ah8vDgq<2ihw#m$to!=vfc2Y$iw#n={FXX<89y_4XHlmJ$!HV#f zK<}dmYnzCix*qf(l0iac96frh9mMEA(s0s^o&w}SMLCn+4jr)(gDVJa7ouP#Ka@IZ zr60P92oq5#0Yo8VN=gcC^r+Eb4AgpisB-m#X|u{DbgtP7C4|Ra{*bzii%mmB?EK;| znJeSI_8?Z$!FW$@RZ`A*`YQe)GW>P6_nm^)0%+hvdHG=SMOdv2A}B2`b8v`l+cz>} zcOyUn753yWwiEU(aP*lS+jn|@s<+b+I;tmBRSNcPHU&VdLC(4dTx9p`ls|UC;X&C* zj|8j73&t=23bGV#G)Rd?Qrpy_=djNgyK70O1(zI?SZegNF|sv7Chq zP{?F~K`?q}{U%o^f-)4Nmmox1P%_G`D#VmW9zKsq!k#@xfZ&^9J86rkqm7~h5RDcF zkOx5#n0wp860TjvvFq2rVaHFk+Dx#@+nAmuD=LKIQ@p}%#G9GG$>gYv8LUTw zE_Pmp@|8!jt(I|(Z>?Eni!YvM&(T6PdCC-5xEnzS(+ubx^e=CtxB1hb{xo4uylsTg zBac4pp7hN(-|RwNO8mtapC@W@+6P-m;J*zN>fytO?53M<^n$%de)1?-#rZy;StBde zx~1uH?-u1;NTGeU{YYEIEgFdGIzlyfhMFdmjlUqrg!+xxbQ@GNf_hIyJrw=Wl)N+c z$}2B{oJZ#feaf4+thapl^%zoS z3Ii*T>|Rt9T|rklALXLwicwz20S?-{bH7cRyTlcigCm8RSr_C<`#VW*K4KsUGE(iO z2M->#F{8#>F@{DhM!2v6%4l;vdKe=g*;<5wdywvg24U0#$dkuZ zu#p^lGlt7>YA=BEgy!WbYvDI-+T_DuB%jsdC^;pWr8U{Tbr&E%7(W$*NIOX}i=I5J z&`mM>h~aa5Wh_53KEF1$`=03?FLKj=!O!v6`%Xctin|pdr#7qUGcP7o{fcD|%BLWT z-d=_`v@~`Ca$oC(AI-TGD?#X|>W?^NUoM3n^#GY1Ida@Czv>n%WlGZze)4li!9A?d$ZLdAK!CrlF6(bL+IL;GJH^Ui#S=0A@@)sUo)1gMk zy)8m$oMLj*+BeqNzN0O+@cIwi)_2y}U;XXh*r)HihYERto~RWh6qxz7q8zGx0975! zli&ExTlSCt{4EQCkk%( zrnjg;I{+cfg9f&4UPq7n8p3a77UQ)_g6QhO^!qe( z#&RIdfxir8oO#R7+4rxPvd%DXd_L>b^SipkPzS&;2ymcBjF4alLoKow8Z5m08i%b^FZAA{`+dZs6l+dbkyml6TG%Uxjp74&C}&ETKs3~@zxht9 z9!Z~YZV8|Ox%Myr^bb}GQ;A5tr&#$!DV4|&3c68M+=Qs;@tZn*pIFngZ??Jh!irZC4< z|4;8C7O8rCYer9@z3O(9>j;~_u+Vm`ebRp8H-62!7*m_Y$UL=sC?p#gL4zDf1$di9 z2#;%^#GFMp*pTsytdUD~kZa3Pzo%MB9PlN;p9auSk>rK%Tg*gByTg*g9A4|i|Ec&( zZD^yI7*$QN;Mc$L4L~ODL`jztvOGc5p#&JlyXaj$0d!|61$a~JnP;BC&^uzE{p{y_ zAAb7NpE~0;Qi(fz&TK#(u11N!;bSAGQQ>{vb=TSb58Mx!LoI@?0K~RsHWbK26~=hF zBAT9cVT8teb(YMOL+`sgcV0Q)Ck;lYcnRJn@>58)@HIDlnAXjwZ3~9y;kHwJh(;ox zta-*hMitjr6PA?15$a$r-s4cBPfKpP1$_QtDE5)A^#A;S{C7gc$BFcj+5j~yN7cuw zdiG)}(i<|)$f>G`uoFnp3DXzbq9xZ`3q8^v+6HYb&q048Cm^GT&2$g?KTbT=LLPkh zgeergwb^S#hK?7M=X{+$yOh^i)?gDhkHedQt5-DE(Ek?zFl(F^;SGg9P`#m0d(dfQxNQ`YF7okn&R9i ztVkj{C{{eTGgw~A_edp#0;SMLsCEVQGUt&WbPN+L z*ACuqpqMJ3_cWVVVJZjVh!k@yHev-WBG9l52~$~rqLWbN9R5q_w*UVz%W6?UAxJG$unL-TCt#(al zl7iLD5TzD;kcdz3;azg7@&Fa$@|UVAwbl;H65u>g#i~snLAQsqZudp^z!9iw=j0xINt}+ zqsBsNx|G_qa)n04a>(l{0QyigTaPy}id@^aZQFqUJjN{OWu#5Qn8?&9#V_H}`@KZp z9{GP#iP+}x{nH2cJ`3Ct= z5VZ`Qq}}QagMCPBbDcsyS2v0HEaCy22*syL2Mu@H2zct@y}*khEZWWO>UY;J+yRbX zz*@+VEG3myHIx(pb9?nn>Zr-VfN-!E!3xp}Fqq+_m^Sb{I>Vmph*W&V0 zt74?4hJyrIp(P6;HRZAuxD-Fh@hPgP^3*`^IHTOXdeB|=Yf0jQPsx`H_sp4}DahXt zuMRg5vVQS?C2z{NwZ}caFN?(T2>VD0t3Pkm%g@>d!089!^$Z(6jv|eGyd4U3fRZ~6 z5>H-Uu~#r>XVKEa_p-5qg;)L5V?QCp-3X!OGs|?H{kK2)zD;97Ft>>w-s%OjiCNkslQlw{QZoVTMHMbE}|TkM;{bVP@6|97~C`Lum0-)kn^gv(FlrL zZmDz+TZMC_Lrd+_%NAqM6@e0VC;8(Vd;Qf{L6Z_v*g}e18d31meWIGI`det{bz0_7f|CJM&bzUlH1=;^%s~Gf6KOJGI#)dY#c;S2y zpvKuwig|_*%HFnRH&vD=fX=JJTiJ=m$af6yY;d_Zy#5kB!X@^5zxTTo>s&?Xk*aX@ zuHr?zMeO*nE;7SEl!XYgxT46u_}jl`+jdbfv}KjeT(Au5P@5BKg9D*t#!SJY%2>N3 zc}KSss30NK_4nGfZ&R#vy?x?S_k#ai>*v-X0ii0gHkDUW?Og~A7(6ERw09WM%?ReEUg2}CU(K23C$t}TCU0v;eYn)^}oI)PMq)AgK z6uZSe^ILAYm7MoHZ|!QO1upiS`|_ZFRTbQMZpeH7b%nO&Au6}5!lq4IUbTbAsW})v z0_8*TU{;AOn7@S4lX-Sv{~;R*53G27B_3~;^ZC(3$9(>L&5%mVV$9^m*Pmuc52%FY zxi&~cW1tQ23vKb`SKFGkZ`r&B3+=$ZgI;VnhI8%Ra{w>>j8!lfpm21F?b*EA#!m%h zZp37aV!o@io%4BSGUa?0w5K9hq_UGiLOZs+V}J37ziwAwak(8kS#NEMBoLB!+y__C z@@h4O3R#0u+eN2~==yhl`}4MM_wU%7FFtNN2+GHLkI77QCM|KIPDCTIDBZoEnTw6_JcUq(stU$ zjUDA!?M<~r^LT6@bBe@nq1U)`7;m7kIu%M*uf0dh(m#8Dd!D)${T$=HA#T!MzI}Mpqz<))%FU~&tP>e2#KxMzl{RDYSbO=6Cv5`L1&aYq zY8B`YIsNvQCa~#wPS&9qFTIQ0wWMen-8cn+ymU5Vv{Jo2WKbo1*~Pxp6CGtuDNbXz zqNNC)cDhv=&~x@ja(#ZT;;#_l_P8ATohvXS-Bs)hIBjd4!d?3Dl0*8ZDPZS(?O(w! z1_eXDLv2=-)n1h?MMOG~GXGq>WqgbH}q9`VWavE2+fbh}@ zdZ0_`fzB(fw6z< zA60dqWm5;Pn0uehRh*Pxq&#v_E*gQjaou{m4NPFIiDt*hy)GEUVxT+e z-{kKD%-(Koqum1V0gV9_#&8h9N;2u%D(;ANTV_IbV8Mk9&*@tlmyyS4;em@B6-=um zgU7cpL?FWd#pf4?h5bnAFaJAYz=Yo7Z~agE6CJIeZ4oM2a!12)euhj9lIY6=J^n02 z8J{sBNc@eSIymvS46gpYmXIQfl0GSUy0BXDLo19C-&y}gR;Ij@@!I(QC}Kw5lOE>^ zU^cTm)w|pUAQIRC+~$j<(n={5EG#UyUp)S-RacNN?#Mxz7BYl~DsGk{7mx7}oe)Dt zOr(>ffnvc{E3H;nhz4wgR?kXWL!W!*W%m>-%fTXmhpGThYGs&>>P3XW*G`3U3C9VA zu1mP@>XVkmKS0_@x7Qi3N%UnYkb)8bdvF3aZ{B26NZGX2f}@RtL_A5pkLQCOUtDdK zyzhfA=pE0;i8&3}$UEyc+eC`RP951!H_b_8a}5~VDvHy{O40;8_gcCgis+h9tCbq5 zI(gOtTG>jNW4~?J0hD(R1!Lu&9#f%SF!1#Xl}5iRugJVjZ!jd-DaG2ALjP5{`3rR) zYIkbILM;PvX8MAqwsG}~Ry|0g;h3PpjNL;jy{BqWnAqBI34-NJrYS~=RGCFBK4U{R)zD_Ah$Q5&FbCUNgCYdv5fxYnj<5=?lPL=sG?nBeLWZubZQD0^7 zUgVi}sKT8}QWbeT`TZB|*_YPXmA8G$Hm-Tio_y>PTgvQW^?Wzr9eb6yKsYo2L(2XP zg)g4>rs#=h-mwc5vgcRYq*+&44vM_(3~gOhTnaNc799KOXV>Vxfelp;Q9PQh>6COY)P!MzQNF82rq0PGVHli%Y?D;1i z#+$68cRJ4_Nqe?zh7~k`Cl~~R9$x;;^MrgkE1%bcJ#g@7gN>VWxmD8YB@i2>gM6O4 zM_`h#fy$J0$7ga58LS0`WhmIW_Ag(jZH%be%()kl9t((o`GUiR#{)mhi&kZ*Ot(C% z$I$=L4}M?=YjbQq!->|v{Fpuax&l?4=i8u_@Cpr(moagf~)VYRkfi@rVnRb+@L330oe1S7_IrAB9!4iG@}%Y^+`Vfe%{E*m)dT zXfrR#v48*OSIKD~b(lx5IF?Z#<#!WHFCv5>+p%mX&01uOZ@2@Cn*uQndy==vm%La) zUaf;p2p;mnD2@;^NeZwyq@r-Ws}tplkTR8nYHZ0>chJuBGy8{s_&ZdgUnEUL+z!)@ohFx;g$C;Hq!Y)}-WxL*b#D4Jof29RWI-U@)BpBE4j0R+Q zuvgzz#U)j&l?bv+ulk@38h@!Za&zr4POc{1$UJb5-M1-P_f!<{zlz;no{{#*IwK^J7!FtHJ&+4KNT3qdIrsT|Z&%g(F zkk6kn?+ROV^*vUH62ExK2W{`_=k3v-{tUR}V+!wim3%-X46-fMf`sMP%9hlOvWu5~ z5CA1=LdaWF5a95tn$U+U#P^gEiZB`mNCy|{@SXwoP#P3X+KP;QFAekmMXkFj7k^(V z=tzuu53c6!wlJ6V)TvW8ifKWE@Z6+`J2eKb6UFI$yI5#4>X-|y2~NU+?If<-48qMB z@+1Oe?_##}8OEgn+u=n*VhFv2EEO0+7|elq&Tz64ildaDY{#0+rYG9DTAxRR6_ja8 z2rt@814nZt-z(!Z2na zhYG>e?8CcjKr`*T$nmk`d|-QDrhtZX-G1CTcFG&YWcx&~_&VxctH(Z}If- z5WW40`VEa^)wtDGWm6zfs-8% ztML9oj@8<9pl~&!GY2<#)tWc$U%&mYHt*t_?47q(SqZR)x7~gZc;=Pl`pdjXL;xIW zLsE}#3)ZYxyd%w|jB9Ia?OA|3hEQxbW5JEq3{Xs$02n;#5T`#V=`jAU}HboZV9{CzJ> zoOW@rpE#9~b~(DgnJToBvMk+o(!7^$z5CjNUSE&Ku;R^13Dh#(?(J8ew0l1KLA&Yt z8*v245i5lxdN;g9@tqrZD$bN*SKoA}ZQr%e{{A2T&AI?KX>B@TE1zSs71#>-rBybR z7OT<78u8D*eFv!FD7OZRuTE1Km6wmS5`F>aSp6bHIQAc)&FTXf1u76x(IQIEEy?aT ze!juEv?Q8x()emcg&=yZd(GNU?6AN5|NWVbpEMI76#=va1)VKW6+f8&(;CUk<>W2Y z20i}B&+MsZe`*V^z1!Y=`4KC~s?kIRkmU=>5VW%r59f$-@OU+Slb~nE1b<(p4h<>zXpq~|9}dd7?) z_B+4%>-Jy2@+bE8ws&aRqA;X-kQeV9V}9u+ix)X=^6YstJU1dp$60e{*!CTJAtaE7 z^3MSLUIu0?)3mm|ZIj7^x44HYg$GZ;y{g{qiEMK=p*w77KShR$SBr-{)w*?W z+Z{J9wYxra8>m;OeUN%_k%~A37NB;g5O&TcADxdUAX_w__da;rO-!R%PcQWMX(t(E z*M8)K6sOP^d>X7Y7^kwl7z3xxW=xq3P{(E~sQ{4;2HUfDKWo}yV@K53wbxv1PdxSv zbEGfFXaQJXEUU1ghrVRBHdZ?lpuWd+$xG z%=NPubg7(;xHBR)E_DjGAbLlf`dr5}f-n6t_CY&6zApvnku;zu3n+TrxZ`b`vtX=! z@>8F%#*-8=3H?hEBX~$lXuIxf^7x2AeW!B9r&FSt~qLM19BTVO6$o^BSARY=Gin zQ!othDn+u&v-Ws}XYRcinaUEs|FgZ=7azsRv+UZo+5Ym6zhsv#RGU>Zlkiv*AW+?K zP!3*~N9_Y2yn!_?wN`NIOA2zR&}J~m zu~wTpe-7)g)ry#wRn2gk{rmR0VLytYK36TJ&FazRHg?K93dx4DCgh=sXapJJ@Ji7t zWL7Me>H4Hynm$M>4s!?k*EchhTRQUnr=VE`g}564mLJBPR#jJ?0=&JGFq4Nul;_04 zJ424d!`uqBN-0Qa;91wm#@ov>N5vr;g z0tPc>|8;eHv8WY_F^W9MgDdYHwTHM>ot9<#=7NC|ODZ6I5U_;Dc*ZbcM&a#d8#`ta z0s_WrR&kaPeIR9d^d5!}*(^ozb9LY3m*we{F&us3m~nRe;C|pP$5;WOI02K0&`twN zl7p+zNLhH#^DhV?w%#a3%)pO5P z8R%N{P&eI}A}O#_mGeMs>a)wP_|sSs#me?<3OCggjg@L&&)lElj&pWW(r%nZ%GMFl zdT6V@yy93X$De!N%-g+sc(y&xr)#BG`O>waT`YK;q3tAuFT;w3b}QG$y`@joB1w_ku#fDlYD$Q|3zAmb|yJ$u9Xq;M$d}JiE)sPe6KTl_2_o>9X`TVpMqE)-~a8W5uaSHu^>T_M&{ zz5YgczP+uP^jDWv7gpFr5DizXeZfYL9ExLw$Luqma{!EDSPz9{ZKM@Tz#GsSs$i^w z0$Oq1+u7bi0dN@#AD|?PRg25$J=2;bTJnPT9Lnx8SBgEK{QPLD`*q!B@hDmx z%66)c!=1@XM|}0ZQqb;q5hqpEawSuMc5d9rW*)U+Lq=Nzic|}cEoDruIXf4{EA%im z_*2y2twj*cpy~=+{pM@54mDZL_%U|m&_1u`_8}RxVjVto5CxoT8`iJ)#T*J+R2?&p z3#ryD%9pjxhA58>mlo5KA$27qK-;7cZeZTNMA$&ZN2(wZ2?k<_jRA$VA#O2a zR#Zf%g0TSW=dRPaFUA1Ad9KWtNq#R@jJ_w4*Vxys#U5qoT@TiNCf7(kUo1ew4C|x^ zq`$R?eRUux9GmW?_rp1ZKuPXRsvJq}+wXtj9hD7Q5vYqB41IP*^nb2%-MjwJoXS^$ zKO{H0NMDsQglGGvz0fWtT!pl8wz4+LU3mn65I0brxe*knQ!t7uk7+v_t+P`^PfjS%OW8GGB2TV%ym zLw(&eR3n5J(#w>dW@P zkA7q~UVDwVHng4ydE|W7K@kNhg1X~JyzMHVS==fLS^e5Ow*2`wZQ8|*Tm&hosjHsc znoMiu?1DnpdgwPn1ImbbH(%$%oRWc3C?)h?syXa57Q8_H-RO%v!`SoH$`>I}3+buX z_f%Y{f)_y*3bIhYOF_l^8!KfE&Je%skAkYv%)`5tMVZ{~l5c23PDo$6l-U zNv%5Wxu;p|Smvw$R`(i;b3({6wlDSUf{@iqo}ure4oOrSc0aN1FE(2IUO8BvRN{3O zH}G8c>_X7c8LVng6Vbabnw~Lz5nUmV*kAtHpV__l-siO~@(=^Azm^D-H^P(8R-qK{ zohB9bvxk3f|M8s%>1MbUo&xssq5G|ZK^aw*MQ#@F*tiyDq=*)VvGCndLeD+(6wxL| z*m9D(6DMrn;X0eL=xVB$M>^I*wuW;M+3Pdva3L1pEa%Yv<>Tngn01N$^FRKpwGjol z_|nC8Mpiu$fLw1|k*ZwG zUQWc3*Xpeazzldl1Y!Ed7G`K4vsss+T+2t=%!OCkYoICp{MnbuH8U(-xpk!ggqG&v z%kVHh-O4>L(gF-vA2x2bjRlaPMN@bj$Uu4c5Z(#f?C}o=A@KL4SE9%1D~ISk)2@(o zeR&c>cnhOH6?gP8fht0ag{Q947n4Mvc~Oa*3ZZw?`!_wbD`c-`TlLyYU@4!nspJ6# z)hUf6&7E3udf)&sjhreUB{vd?!r6@b+`w$x3J~oC5$ax~4{-_J*I2`ycwvf~3?ZuHC~YeA!vm3! zd-25=Z3^?NM~@jz{lRhDMUBRU2@}8#=X|Qx=Nzh2h`ST?w^AZ2R<5uG3l@-~7#zA7 zKu^<}$RJZe&aOSXeXU228S880h6n|kx)1mAqKH5~fA8D3*Nv1TM4>97omv1pE(5!~ zlj10UMH?Q|L@ca)2l*^HQ8Cbpr%ANO@t%sF94a;j1qz;Rg1ox%M-)~}xo)a-jOowK zT%7~7D$FVn!%A)H!lm}?%a7Xn9e+bklznutAKf*3~2*TgPuo0)XtrUY{Jxw ztc+2S`K>iJXYqBm4-D%=;Iro_swdv@is&%x$8A(nP}z8sXH;;7*Ete{f69W(xep;L zXgG&Mkf=w~_$E#|@K*svb)21V=2k5*`6`1qN2>cnCD z>NkFDW1oKmfHMg-$rbIH>L3)Qlz~U24c_l041O4F^3%XCFTDB=Z+xAGTvL(Gn{(LYF7nuORor9qULH=*m3sp4T9c zqet+HdGEqeJTQ*lbzh&%lyQ$WO%Pqu+XyKrv;(r zSlYpQ5929$9}oE=_4EOP--Mg^L6JD|=w2cX-IHe`>I@s%!^i1KFtK5fAdyJ2C0Y)g5D2T`-4C5O z!>%7Ug(~!eQAK+)2NL|)up%CQRxWvPkzI4?XoPVA-~x?!zU6ksZJz;x@GyW2wJNQP z+8M&t!#+xAhQXDpl%GZ3sD$FOCZABFf;qLViC6P3zaadagY3ldW1uLNvKRFzz}+4S{n>**v4s~ew0U5LzXfuf zl*nsDc?2c7=#oX?q?b|Xcz~2gIX@3N7tEM3-Jw}`?AYnbe#VR$7#(f)^2;yU%vm!* zirS12K526>GEP%jyG6x-tbx4nQd$c&j&=RI4K^O7Uo(6d=RM5mO>&B~^_5Uu)r`R2 zw`->p%c@nc+obUmZ2ytN3~ga}ismWDia<0Jc@mz=bef9oM}g-=+pqo6UP~E551r6} zclQO=7KE@9WE@54vDv9gshfg|MSFC7B$K=snsYajnkfaU_NqHS&#dnw4BkS4hq?sfCcyBZfAu?yp#Zr7HM@9Ey2J z3Ws}W08w!AqU&ws%**V=v7`Px(FFt7x!e;!s?AkVyIdPLyvzm<9qtBw8!cwJI75qW zz1O|w7UW8|E#cN;*y4m@MS)MPsIf9+bqh#_Vf~pC?FR_oLf6I9=1))K@xD(egr{|k z)>WAu&D(ao_1Hs->8sa@^je_|jtEX$=umNCA4oIh>0DmA5R_b$;T?`}isHw6uAozE zKir#jg{2!M{7g|qK&Ohf%kMw(K2vuvg_<-bF$`GeA6i!%JG6S3jTkZ_Fp!7zV&xvR zKu{`yp(WIn=TM=OI8U^ems`$k-fAZTy%LqHF%L&pg#y^}o&a({jlbj($&$}kJ~}LQ z%BF~b0z;%K+VqZHq|9Bx#u6+Ld*Zx)-+HcSq+@(fdbh_(yEHtVD1l}`j~gerKz&aX z8EeBp6&`LT9T1p`!We-r@IQs~)pwXpamz?xOml`3Y7=I6La4-qrF228>g6GJ(tg%y z#V|z=@z*Djr`}SmkRv*dh?jk$cEAl{-5&os3Ds*3xWcBPKU9aZC%)%BS-#IpCqHmf z3<5qDpm8ym_W4`i`j*dge&v-{Y|_Mu_MQLu5Bt*Z{EjQ=Idf)G$-mo9o;+qZ-LTAx zXc_s+SN_2M>`(u@4ab|VtE(eN&$J%KnQq^)!xkf`PovB?Zrb2V|ArfGu;q-ZETDyL z&6?E=ZTWqa>}x3B2kiFSZnrnyc!iY3d{FCtWY=-5iW{X|$B!S!Fo6SBJ87al`sgFJ zefu_7wkMfet$KjndvOnF>>mFq@RU_0`*Jv}(o@764#V2t9RaVw%Ur1AC?0r=YT;V*=;+Ki^*`6rofRglw~}l{%{Z&r4!4kO0O!diJl{c z=M)w2NR)0GdB8Q5N>=@fjw(R0I-g+(53L?eVGsWzSsPzENTo;y`ZlrFtVajRy_9ri zNH2-+llvF$Kyr|x;9QijB6q&DO6w6d(&%C3(>z@nP$F}EJ(AbyOG;#tqfZqi;rK{C zW*~avL*9D@9o(Ty;E^(@M>D>|d;Iu(E_{MSGJUHiMg`xg}5X1nVn zA7O;#&8~bkc2J5;;QvG^Jzrf;SXTvgngLyaa&E(DsTn%V7NNi{z4TI7$oJlRuea_! z`|NVZe_y<0u`APhz%fRS9AU%pPMc2GGrVLfqxzorb$Ij5Re0QEt!hvO#WF>Vw!GLb zy67VN8J_uXe(^VK*sx)E>RavhJ8pNQ;RN|i83O`=xRzRiu?#wR6(ggep@D+Rp*E6h zC_-=ok6DKD+i$HVYH*v4B0TU+tJP6NC?jstgz@ev zH)DWoSVw`Ojwf_403Vmm(Uhnlx01puZ7oJb51|b!QirZ56#8$aV3Ga{^j>4_3B1|^ zJ`_gu8Vcirw&{J|&}mwiqVfz?d*uEm`fZpd;c;-O$*Acfqd3jfH~P3AmvXo>H zr0pL~qk$5Sl5$>pjZ>>}ds0Z^`Ms?mwx(0aH8qhI-Lgk1Jj^5{uVn5jN!+pE65cJ0 z$MFt^AmY|jyOpUZ1s12KGRd6rbL-O-i(Y>@!cXv2d{0y{)02hLEkzpc>}BZq27~@a zb$%0y<*iq?;gyv!7gkul)eQ6yWND58bmm@FFRsmi-TbklA|~||s}O1=vz%|Z{s!BJ zx4CA`8ocNqcrJL{xN%;*5)AOjd*dEe@Uw5ue$IQ&ufF;kW^F%j zv+;_HiWyGAeb&=fxSD%Y*!`i09;RJqtnYzx+A8W2?pl)0I&9Zo zcb)hBOZn=UVZ%n)iWMsXr77oJ<7~sm4S3Y6Q1~CTqjX6OA2AG%`ZPsQ-*H1>@#U9u zKeO$--}w)_a@jI_ebp+PNQ9w;QLaY`YX^Hbq(B1Bb!;TZUT<=9V)95(j_W_AKm>QG zK749Nn0oZ)>u)eA?SzdUG03WU?n2fgcJ`u8NfCkiHDm*O@l*!%7~hNbI#kAoym7Z; ztw@W;p;a9g91Af<;F&OVCGd>bY$UE7$q&E7Kr50ji9B=6%c!~j(6?*5)@CIk_^imJ&PY7;>UrPlh=wBSP&xte%WD) z^*GU*^o+F1I2V7rgC$w+wH%@ApjQeIV?*OY+elBeQ&&-UV!yqy@f{TO5Sz7VDZIsq zOD@m}A18vIDTHyhx|Lo8l=?k?o}{vs(HLrC(!UZ#VksZLrwTfirZjTUl{1_D^g|EW zpsHf~wa?y*K&qyHHqU!+CD=mFAe}`V6K3eSORB3t_#qz}F#((CHOpR9l3Wcha$=L6 z9io(uh)zI$BBD4~llY2LW6(|BC8=C$ocNEFbF91KO?CqEwu9uxxdrZc)w z=*Kn+daE&{;@4?bqpy z_>b>=7Z3b;?td|g`WSiXZFtIK$V-D04Gip;f9Jp0P`vkrtWRZCh0i{gF(Y13aoa3% z>OzW=vQ=SU(%a~{Pm1US(uoWBMBoJ!Begc3u;(6qz~;=IZlC@5uUQqj`Fz6XS_h?a z1{R8Rp^%11A~_MUU5Hpkd=?!a=v)dg+-Pj5>8A&0c~ATz=fa0a`1?V*IGc*EQk1mI zlt`{bIz3^?cn&q;Ii`yHXgXaKc8_IB(&>=$)U8MNF8*}KC@A*$ew-) z6%+Rx@wsM7`s6a*hUzt3ChCqOX%t00T;4{CId-m+9C9YeeQgls^Il(lB-dNLmN@7%22!)wKL2Z}zSe6B5qP&-Ai*n-vU zOz1tvpd{?BkL!6*nyO212hDKw{>XP9HKMk8kN{g1SU zpFff%dSNvOJ<8^dS6u^QCEc^le`BgBbgWTI8 zcO&mw-l;4db|(@t!C(k0A3zGkDK2;wVTcuYk8SHo*UA$mUeWV*%YEtlIK1Sbq4Mz&;Nt~fVl61m1c`2{HO|ifemN|Q(kenbGtEC9> zYCBk$(SR&WAlKWfb}RlaC10zswkoovm{pu5150qQm3Z}y08Uf`?|9`kSEFc$5ngNc zeEFm)Q$uAxw<0fnaB->c$%Khhm=6tfBIj)3*iu@07B5-mVb2zTVkS+UMjKx{q5UCV zGEh+F0di<6xyV|cW8qz%}yq7T4oZe(GNI3opIQ_r9T_&MsbfDMe#DF#wigOk76cU@>D9N81?i-31-n+{&~v_+>JN zfZDV4?5c8i&(TAh?oJX>VVzhz0Wt>fp}WBx?~HS_YpkRtr!*Mfw$-n`Xcx_$W*`07 zhf&Hzd{{r7TLz7XX=CZyO-|^{QW?Ds#soE_7>#kHD(p~rCA`GgFm+9=`wwkyevETM zz?>INv?ITCiqLi>noNd~gR|f_ok+)tr*%YX#Lx?1-BV1;9yjWJGF`Z8d~Ls$KYx%} zIS$s8lt;ei&ikRzE>@7?6j295w+Lil=`bNLsvY149c_*03(8)GxMW@I%?A16_ZUUr zsJe`ltkxiSzxu30$b#%~XN2pY`TA(83rMJ&(^&3gk65)2RXP1n(ywj4%om;d#f(o6T5SEDHQZT{2=eC2OX>$g-B4& zZ1>$iZ?(KW=Zl3OYwkvBA`Qo{WV z-W)kb5q`o$jg%*aR00GERg4m5dgS1>=i#AAU_0|?hn5H_b1}EV#EzdVbtsQo#*#CV zL%oDGb*m#bI@AUt4<>Mn!1MjQ3DS#5(1|gAH&w($@1+oHbn@tU9UG=}>3-TUAe=kPV(vx=(B2(L(!q!@gX7xe?9^udIhLnzy z@f8dbeeS=pc5NW8iGG?11<#loU}B|IwSxLgf;S+4NlAy6s&>ZbWo0!x&Bg5{8pEeH zCecz8J?hGPwu9Lm(`D$-=Q?=P67OyFlj`MB8lB$%&*LoNcMeznuo&axmfP-dTzO$2 zd+Qj_*iPY>R$Be=${ptt#k%nWH+xRJv;wdil>cceljV7|(@qr7#sWns?;bx-OcyG! zN&u!ftpzm}zY^P>vMqkw@iDFtB`;Syry&Y zBs>|AiMoyM^cOCj`kt61DOO#8^1m1gRffKQBT5*FG&Y~p7oucTt7E({R02E{$>Qs6BZS&9xs)~}~6G`K_20-yZkS}DM^>iV(Q7j zXr->A)E4(q1;U?@%JfeKUoKNr5$}d|@UTUfK8rbue#q$b{HU%e`%VsmKaj>w;;Xh^I zQiGwi>2H4N^Y)kjdqI?Re-G2g=kKY4j`_fw(ea?Su3u|Q zmMp@%7-S9Plgbbj3VmsWn0f-;=#1rdDi3vI<~X;MWcpyJMt?hFM7FR_8+oD4kHNys{A#?QTsI;m1r zShEvPTlbI)!qB8C6F}9`q?;ByNbA$MaigfN-R^&j;qw^!-lZNe7yLSBz*p9}CE#0a7vn~tbzxo{|op9!TSndOkMh< zhaA*$Fcva;xxpM1wQQl*mIm9uce{Q1qqkEeh$`dx@(JH-LR7b{?u)$qy74`xA&!(# zXE%sbH_}7l=>+Nyj*?1fz!-Kg4Yd|}0i1H4qLbjA2Avs-FGUN&;@4XFUPGIqk2$Qd zWJep34~Z<*eYo-L6PvoIPsp=r( zo}}u3@BTwh2f@=1%0Hy-6misjQ$NJPt)!xq3KR&7`sFz&I5Hlk3v;(+RJz(uafF;c zQnx6vr&`|(>BmnH8B&Vbpl_t4%PI@(u8-bj|M<19+qBt4YjX*8JB=kn=f7+P9YXej zE8bfLEv3dnC|sp}!1m4-+As*IVfE)hr~*6ah?G;%&MB-M^lX}!!acDvPIpYKT%C4& zAv_!oGgYGCRA8gLr7SKHTrt(GVq@hOy(&~Wq)2IbOBJ4E5wLZgs_vyCPAy`R;n0S%3gT1W5$oQa>A0TtPKJXW>G5=>ry}qN^uc=ll9D-jXbCl;OrCv zRs2r(k{cmUDd(ta%Hb6AoB|Z&)X4^Nq+)_lg{^@Wr-hv@#V*er!LKP(MP&&4l0qL7 z2ZAsIG4^9Dl!nM3>t0MAQ97grv(ocB{4N19nBgJi48+*6W0y0oJbdL~)gLRM3n^|v z^ZC8Td?7TS1b{=2ZA)l@z(c7XReWS`5gJ+L2(5NFTQ*!y7RC&i;5C(l$$bu`AZf2n zVoe+#@El%ldbna0kO<0_zXk6op(;;6&nW8AKC1v!LN-PK({~h&P%D*F9RDX8ap9?R z@_PYvqt+Sk*T&$HV35+2%uvBa_NWm@oI8tfAWkmW)fkrwY^!im3?leRg^|N#w;)4eWVjeQwU~WZTBH-+QL0D^F&O?tO)xSA zkt;rHgs(l_B*T!pl?f9jx>vl1S^?2kKKq{w`j}Etq7Tu6qHp>vt}RfXy7o^6gifm+ z%^qFR`Kzd-P#d0#qXf~bdzIW#A)%roVSO01ixDSYR=tGoy_)Afd7{?Abn;{vx(AZw zjz)+KDdiKj$#TC#YO36k*uCd~tOPe)3kr(E=vx>^bd$q%+b}3Yj4p*PB10*Tc*Y-v zh^P#h{uT5Y9|k0&H$?m#nU(z#uXex(qlXTqJ*))0j1yKt=v(X6|5?vLpq#^3-dhFj zBop$QEeLm6UFBfAs`++!omCo5e{7gkxCw>y|ZFM5r%a5|$na%7WV@uD~xKoRJG zhgQ`C8$A4BL}B&;P?M7*FH*E2QtyxL0@H_A@Q8yk#(R!{)NVhbu%xN}v|V%E)wXf- zMmMgOEMDTYv3|pPN7#~)BBiwSilz4QikBI%a?&PEnnaP<6kEOeEkC=^xt@9EX`hun zlexkhH*au-yL9PN6l$(ld(W9Y+g^R`RjQ&dv$ceCw{P1*WU1EPe)Da+?)vNOO)9DP z?%l)pXV}wEKkfH7YxXQ#&HJ5%y)U6iOor3TFTa4|sIb|y=h#zEJ?W7a6}##C-~8q` z?AO2W>$E2wwI6=}dp2X*RJ)ui?B`y1j!4o8im&F92cMqscwE6oVe#M|hkv0RF(_*x zTt%6Uu#~An)PIkxYu}D2cPW~a8x3<)$YN8}hlH==p?n_yRtXgzDeS`nNsKFR5vsMZK%@S_UVD{z%6KJ{ zZnZyZC|QhHKDpYi#C^j;s5n46@|))oC1fPjwSlt6Ll5j=T12)3>|Qf?h+B|Fw8PYQ zH89y`jE9oTu)qd0W-=e_+=KLPAAuPZI?|c46Pc%#M1*&yc4gD#Pch|qa6eaS00wK=ZDBo=27fXzyuk2xw<#?;~qS8fROG?oLXB>&WRTM2`0-LloK zX|?>+*VEF}ews?(ZG_cEU{SZ*;e&@kmYSxr5EM95y_nF`N|BI+$F5y_+*9=byeK9V z%p~M*f(cTHRkGgqA?%O8;yv!^WkdbKQSTCjp9)_jyOkSvHyIbJ)#JqD9-bZDKa}7wkK^HNSZA<+c~a zUyCvJ(8G^-#6;G86(Pp?po|?qdfXoP>HR3@y&h_O`l+XE3>D@IcgiqO0oufglbMzK zG@i~$uioChb2lO3Jw!?N(i{CHMgry%1|Mb@alVNt59OAZUU7xJ{>lnFw0|Flq(a_9 z>CJu`tmT>BUmkCTLcXdm2>TNmyNUgY>O`E!x=XlZg>Rs9&mj13@kYx_4no(xoG2X? zz;jfgt0+v!R|afwK|v|k8!P-M=NwfThIeBlyqiQki>Veuv0jPiYik0pbo1Y3R1x>94bUn-j6bgFt1l4}ZwvnGQ{S)HUpxez5Ql(P=_ zQAWE!W`vB`g`6!TYIpeXG3e+d^svt(hpO(EY^&n;lPKXTrqL<&r6|C*9ozl7`8XL( zrw`lC9ea=$)i!?IXrg9=DIDb9aWFP++yvz7C>uelP_pX;;qN{B4k6$1${Sjs#S;#$ zbDG|G6@v{%M%5#)TJh))V65b^7IKifDX&y}t%m{cayz1-R8bMAE8amUhk+gXyAz`7 z^sW=hjWA+Zx%M;2;WJs>6TnA7i8~~n8|MMC_#9u#)kn%WHv4^QpYAQHb<&w7pF)>H z_lNO3|2bazf_zOxt#F&i``r|kH60YTa+m?$BCv26A=YE55?qrI9Vz#v^CZ8MWU;X0 zv90c^Lky|^F|fH4<=mxN(S#_61KM7V6?ha#$098B2Byk1Q@Aq*Yf*XLmJV`ogxeb1 z0CwL~Yi)#uMvtkqGo%=EA%I-vI@AlyW{es(9LA|-hBA=?6n6_DZhg0mah3vvP>4H^ zaF_6S3-N5nP8b6;`5kN94;Ta!durA}@sB7|0=yDw9_LKG8je3RN$q;&fi1$A41D zgR%Z6OdM@3kF{WRm$E05Jjx(7Fb)(iA$m=sMd=g-!8ATlj4x!h z5eRQ7R-uWh()qH>m$;!M_kul%l-n{InV5YOKL^|@>z#m^Fh9LZTqrjFQ zp1Nz{_aNj0lgBwRg-hC~LLE+VO&Q@pH=AUt^Gl$9gbn7R_eP4$4qw_|a;SFIeQz&X4wNP`M;u~-QM z7F7^Vufi~(`d5{%>UPmMJI*0k%+oN}Fg-KET|!5r-u53lN=SW#O`!cpq4$$8tioBj z1*{pKRwIhOW>}F8rzcjHxSp+uAp&Rc3gKn$Lsh7SJbUO__i2V+BojPc-^LKF3_Q_; z;ZpZ3^>ZB-|E-}Tq5UfeWv`6o_ncs*c<#O+ubg{Ov0uRAPJN|QCOwhYLYOod6>4W% zxbPBt@W(&1zxv<*!mx}<^aF3SfBeUPA{0HB5Zx(mgw#Z3jpy?etr$6KETAG!+c&=P zO;1s%AWs$AL-241gV!Fc^DMjR=37A~+vG)O>UnNruclDxJqWtE_L^($;h#TblfmPz zt!r?PD;w`fz1Ig2$U^^;=P?0=ykf-)yqd9uDT#0hJOw2`ciu&A(5cmGI0~czx{~lz z6@Lk-8S&g3AeW7yNKkqFV&1#z>X{A!T1WBIIR17+IR(kDKiQPp>CoS?nSCyes2UBK ziJk!?WmIe@p|y7EK62>*QRK}F^&*45$7TnvPa&p0BP#C6M3J0?v`94;-2d`(Ntuox zH-U5tH36p>{s3Pq<)uk^&G2&@`&owjoDC9ee zyv&+D&2z*JoUb6i$$3wFFN1yZ#7QW&)5xPX5EiLVK_;~Gdn?Fw`S4&<6N6Ler7uE0 zNQS6>p@#YPVz}B`WEwQ;XnV*qF7F6hXVj&FOgKndZrr#sE32TgoBP&1o}sWyT_07F zafICUw+c#qHZ_J~?Wrb=z*<@it0;J#K^VN)BSEDYq&lxAbZJ(6C16w4H6v{QL1ZAG z6-$Y?o#gga6e9-=1*tZ$h@VsUFNpEv9`q#>x_u|rdGlc;>OT)S6>|LquHrZV+@HD- zJ34Z$G(U8}#FvWO5i_Odv<7Oh_`8u~F#aX~BAq60I=qo+&yBzEo+{`NNeVYxOenXQ z!hy|z9gIdfcsaguSuP-Bw(DtCXU5*?@=!WcTwtTK1 z4B`x_dwmPn-s&WZ_83=kcp!8Mfu?LS@ zc`N;{Yf50$V)f=x8=#!|j-9(ax7`>S6$KQwbYdW=7){{yKBU5nSjB+B0!~_y(GolD zcHh&?b~LuALoUw)eeyuKfb!4E^(l-$>oO3rFdD+p_JLf76M1em5)7=VB0V)siTd=X zKVuE2sGc1;3{2~Jo^w~Y(hI=&!dL{ho=No!@|yNDAmffZKZxNs-05v9`FZWDa>MSG zQxtaj;-z@v)BH)4w^W*-k^v)s@~4099@+F6vz=as3>of*P8F#HPYw|Lv0;AExe zMPEk`9dn~EAMEL)hmU)4YZ1JuYaFMh01;*_`xunNz3d_CqHw&@Gj*I-a%Ly_W6f&r z(o{0!pYBh?CJHIQbJ@dUXlP2Y3dcC6L4~h8n`&z+$}6a9Kjv$pc0fH#0miCw-#S(X z)DhBw8jj)xT3Cc9kb&9RGBOo~3iSx-Dznc~OgK+Dys=~ePK&p9DKSBqQ<2TQJY?ne z1MqR1O#@r~w6t|Bp3ZX7%;eTtq)rMe`6I`4CbCQ6v-eg(yJZei7ylGO8B9*}_1E9D zYpz~K@3C%KLNDB~m;LvU4Qa(i8abSPF{K@%kcY0J@C~g+f{s)LqPGi>*J-3>MFK&& zm@KTuQs53xoJuTd}=Vjm{NpTmR=7q*%~%J+-wJVxX0c&{QG3 z8|n~7j3^7E8lzm5yxF*iSW%Bpe)@CX2_Qp9-t%D6CVEMLJ7@>@tWak)IZG+{cJCby zYv2GlzQ*}3JabXr4zJ4ls$AFK!x+2~uGeXU@XAG3F630OBa}6+SwoSK_DU7wXQGy? z03H$eu83%gyfmFx20^=~C!xGYl+ov`s$9Qj0N3crUYU9qz1DglZ@3qtXqr=~(^D}~ zt}-G+Z@smSJo+r2O}ru00PS5^yKa@gy)|vxJ4lrk@$Wr*0KEMO=58iUB|dck4o?MT zV`MA*9K5DNjM);30QZs8mtu=scaEMo=6tI)Z4-EjS`fvDGCuI^H1SHO=xWh*A_P7? z6ZF!E9B9A@4y{aTn-u;6JV;nQv^mK?r2Pr`;Pi{EIZX<#v9;0fPyY#fK~X;c>HHb; z>Ein~j_n{sE{`(@`K0i6Hs?QZbbnaSJd8@tbs8_;TgPB*#0fXvPO9#R(~oyP5R%vZ zMOiLtV@~}{K98{u{Q1rz595#CQw8mU z1genmYK2%P5+Zqd*LHjSu_x^A58pw*uEIAE9#5in_XMN3b?cI|w~SQ`HTB`OKhe|r z(e6!!mqL2}@;5?JOFRuFWO8|VR3sFGC}+Hzq78=w)o^vMP>|wwkr0tqC}#0n^EN#c zoH7Z7i5%-kd1fY{O4)mCKoy-)*w>#gIG!Hn4+1gdWc+vCyFa=-#vm}=?}dT9BUzTB z8$EiIQ2gf8cl^23CKUQ3^*rhuRS4Y~)R)DzawOg&57mpT^m`zt-~~s~jBvQGQkMfY z1c4EAmctq9JN|v$i3C(QPrL!WcQ3IS$%CRTN(kHd`?7?2hCDB}(D=}7LUV(Gw%kHR z^9#0U*$o&jVG2-UN>FsJluhsUPI#J#7ToO*w-H}kYvgT5YNNv|QTR~ag0FQ%jy~f# zQoMSMB*5V3EWw7VrTyYn%LSHiGLNB`;NxCWZwd#FWx-ne@ zAeB@h5iE0rtlTC&g-#*b4ZG!rr?&wmI4izx=u_{GVV$)$G1_6E zCtiNN@VT6lkgt_Y34Q+^x}MHSyONZ3NKI;vxV%y+doRNb9>4cLa~|OZ=n-^I$t~nh zysi$CXanOLO)y;ie&FY@X@0#p zTpr$w1&}K~eVTH3nVUKpj{m9lc;7f1;o;N^f~mkni4~02 zD!zfl2i=89^o*hJG_)l7o%s#+^jCubOGSf9oHb<&!+T)^VqYzJTipd&FmDhc(dJ4B zKJnCY`}xm*VRM&UXM;yhqk5gFK$eiD^va-6%%yVtFZxrw(i2Uilc(y_>4I`syJ!F_Ns7l1Pf*G&-s9BWjek$R7mOA8+wwl+ z!V%&FICve+xRK>nV8J!Sn<$v8fhKdNAw zg{mhbSNk0%j@6T`VbIYK;Mk^@6{meTa?G@YBslf9Ax0x>t{A z#766TTqG7Wry%Jx?TBzJO1iKdtm0elvKO9y%wGBZ|4zZm6*iAn9?cq6WpT{FG4x^2 z#XQ^eC*Q+pQ{xn)cs^DqJw==|g8qxBCH3>oUGR=9PZ3sNz+1JSO5zNN|6)+*Ap_>p55DR&Fa_f zx@)fTHI{cF9PWB@`jWgFps{xCTD#-++db4Lx^1Yh^>eA9PB5&e$XBYe_QyZ|G4FTU z6|^{=!2l>gk*jxAh+bucm{qLUK%P;c5(=X#%${yST3=*zVU%p_iC&nq1yV?Qnu(q1 ziy557qpU`zgAc`Qg0;K$7G}Xn^bKR_O5%eWOy%zB@W(6e4Gq#wSOu!?8z4tvXnE0 zyzpE}@gU@i(tHZR_ecWtv5(#Fr>CTduN^N`Pk!y8Ia6$*?p9Gx4XX&Bu=F*PTWDro zATm17q^`Kob6PEJ<*jfi2$3{9g5xe*gZTFjQa^71UJ-`W~is%GqKYR~X-! zm~BFgS8}KZO@zl2i4tQG8BlT^4bww;aW;!&7u;j4*LaFPVR7$TnCn zv^hl|rwAbbRu4YY->IjJ7cQO0$ZR(i^3RP9p^RKwcCeE0L7@@E9 zz)@sND2yS%>#>c@Ub={cQM~R^3lC>d&hXID1NPX@pRjtW<2Bd#?X|1zja6@WK3c6# zYu{c=rT0PysaW;KYc?IvdEf3mfYEGlAPN<4Em^X}Z)N4ml~e$a^L}SRELIOvPiVc5 zV|?eGb>8cGG3R>v>1X`2Lh{%j{pj5h=5pVPGCfc9_w?z{cW&qE`&#M4g0Z6R?;hjO zyl5drsT9s_prEd>Y`9%}+sEzTuC4awhHbWf%a%aRBHhRv=~cDuQJ$2PPQ+>HbW4mU zAMP-wjd;ys;h}6&Qo$vkCQ1rLf+lnxMD;X16JQJ&V{-JSrKO&%cBFc2TgZ zQ#Em&fc;#2#oNos>|`x zF9icMy^T(Zt8Ti7;>p<<_*r&<_E&YMq>7pTsPWuCPgOi;IZLEy-ctpwN=2a%Ds-oO zPJpguN|KaxZn2G?I>$y$Z1-bINahP^NrEeQV%6;PjW7sXTm^7nsb*WYl9edMDbvakHXA6mt*$#(r6 z_tB~prq9HlL<$dv5LSZMlY%6bmy*OShSC>QFhZ+^v>*jr5B}u)w(#QlwqfmhyY|jc zaz|h+yAhFQQDRz7J`BI5Dz%g~^+(qA#O%G#)kDoESkRA*t($cGlya)x!P?%}aj__+ zAMG24F@-UBrYc79etN>gDhKe&A_=<)SmUgm6%=&HPsFqk!l3y?Lf&}e4cemy+vh&< zDf{Pt`UiXNx#hG^jrJBb<-kXPiLO56@j$rVc*6}gnZEUvFTLcbcuN*9c27l^*CXhQ zmT|fR0Ej75Cb3ToC{iiI>wb&t=Gbq4`OEeaU?xvJ`6R_qCHAp<@3!xK_j^>&9;0F$ zym)WT>HbP56yx`L=XHA;EWM4|ft|09MNQC(6MByu?qT;yF?2g3!&=k(@bo9^|-BW1uKDkMLvmFEI;B@ zgL6Yvo6#3Tq+Y`1hTiC|%)2Lrg3=EqQ{m-HKw(o+0YFIGx7ynE|7M^3wci9>Xo&sc zfBOp?JL_tjJ{JR3Kta?jL<4pGJgKiBD|2R6Fn6yBQ?B zS8UppDYTg$un*kvF)$#wo|rJ!Zt*FS`Y}Zw$M=%5p#HshExIe=ATH7gtY}SoMihfU z?LmwbrgbU`fS$?0SXWIB|Ftm};lOdaW{56kWdk52=SF+u(6xPEvkywo=xq;cCvGow z|HGS!tH!d2_ug9tE$b#KPKtrn<0nQ>hQVMfNpG1qC zN-r^6oNf6-aaHeEg5F}Hj*d$*R`e~mdX*beAq!R5B?6{DY}@goG^DC2Ylv;#*Wgo; z;`<4m}>){2k1IQaU!7m_C_~ya!^t#zS~9krXA3ZnBaTn zcY5c0q?CeX9LpRxQDVcv6+Q3xjr11ou|h}+qHw>_(slcpB+QguNz>BPo9&VrJjeg_ z!5vuocn==Bbj)-;fbK87Fz&g_!E@Jos*p~FJ)w(0(FygVYp(L<^_%UrRj*?V)B;K~ z+D|ck`gB*^YQy^0x4z|GvVd$(96KJ?h|S68z7!%C%G*N^tzsVZs4#+&*A5CJ$BqH= z5ldfd_7-rU`)P$4M*pfp-1@FZm=vS)LV`ez{dp>VSD%JT==jy&c@pH7D!3w|(KS4j z%tCqLqD!F=htNI*)Gf)pkk%26hb{vhD?0N4ZuCq#uD9R{R;5CzsXP|CBs4uqx7DsX z0I#v22l;fD)ty!gVUy=-W4eziteY{OLYIL^!%sbqO%M3=$qXl zI>|js{ca?~BEFZ$ovj(|S=2A!tsJkTCMT~7u#g?}LZ3wOlmOXVY&-WLSG-U#=$OF2 z!9WSRPK9g9eeaduO=VDjO5Z46IUtke0{z-3WA5;@U^a>tG2c<1i zOfFfHXVPtTAEEs~wc-D}u~1BBoGb7|plLl;0A^8v@q77-6pP<@yrl|D#LRa zy&zS->Pe!oN{UgO>KSId$SKbruXpe7>XY8Pk?tCy#e=d6<;tOc0mLu4;U?Si%H#G9 zC{pF+l{Rh8#q=&KFWJ3BBot*NrDix&`fz#+oaTUDWU1(B*Ua1uOJm+ ziU`lL=Y6kTPZb%Tr1$%MDd_a;7@sG3SFy8``ke6uvD8~n)Dw6meKf^L%E7lR6x0md z7>*T$Jnx_ceFiTIdj9-*^g!3!PI^9so^!C!m&LV5;k0762kKYV3GGM)uS1KN=95sd(bgdx7MYS6z$8T*Tkc0Uk5lZoK(s zhg%h;EL^n6u>wMYZvV=;uPx408K?bzn(&JkU_~yb1fTwvC!*FFzgE;U9BeL*b^BVM zd-8KA{>v{bsq}-F99`#g=!5db06ZRiLg(t+t8#H1Sm zNd+4?&yA_GDv@yS;aIh%hzEk~&=fq$-Ykucr5#lj=E5S=B)>R1k>s^BYwL=qFB#H( z%@Xn>^=}UqM6gFca*8#=yQjA37gcoh=g?Hghe}) zI8TpdiKD9G`}bBs_fN}-BZ9@6*dmV=bSnt!K@*uQcIsF9K0Z^X4PHYqH9|X6m_w>Q zLA)m_9=&D3#}cMjy{Y#w5GAT(eP#G19`_a$ev;v03od{;^!>_Y-FmQ7KlGZsUZ&?x z$Hr&7?`}_&^2flHb6CVUHyJ#?u#TG0Bk+;iZn*`8OKxA|HYwP;<3o4(WFm#YCn6}r zgr_dM?SprPo@Mr0pe=$-RL}E};W>=8n;rE-%A4%64}bJt7o1^!w%Mn~Pjn@w^Iv@F zW!}m)|KddsY|$2Z-2-5qvmSb03R`q1!5I;nE8occQ!$oH&`uBQ2Q<9dGk)7^b8EDW0=ZfM$6k+tIdW?fi zQMZQLl{8M$+m_T9?_+N1m-msS>6Kw$JlInfhCL1ENUTb9Q2efFHH?uxyG+qJ`clnj zlL_7Vxb>56?bspe|5neL2-7=v=X^R}wf__fTEyZtRVj-bX51zsQoU9o zSoWFi0D?e$zsA{*3l*&0lxLdrIzjvU$vkfyN(*gB>Q_`~EO>T(lkC|yG+kb?haBoA zG|z`T-iz^vU! ze3f$QrDc%F()ZTBND6y3vHEj0f0%-z)9MwES``H);_+h-y@oHuVR13Wx!zk^uPeII zo;;JOTsV&{VQz$s2D7uy9u}`8M+nisO3*eI@yVl z`I980Qf|eBFG?Av#;~1^!(N5Gi&S=We7ZgKBNC_UPcEnXJa*1fXePtQic^P~?0Y81 zgf+?V;_=!!>n88rV@6R_AZ^V}w66at6m+_!aHSmytklmlUf>MZ*A<5LBcXs(EJ|PW zh**8kBJdGk{0IR^o8Yr*)gv2}bKDD=dJE|kZ@ODdtCXC1B)PkPBSZTn!NF`O#PHCH)bbHO7) zrm&i;ZgnAOyCQ`aj7^1ynLiGevxA>-ocFRqd>VG*bEO0EL`prtO=^uibQyzFMBj^r zXWWjMstDrWd(vF0U`;5zNcei#*t?bjodpCcO)&%2N|2U9j`Mmv3X13+Pz32T*v&`w z9if9^h%<_mvwDK1RMh&H2=Syw#s&49?HxG|#eZucDaBd!kbCe4CfmvBGj zNe7@1&z(kH9!>+I6-8aVcb+DKqL%ckg|N5Bs9+ zaILV--9uStE1g6=~evgU6K{Ym;Rj83Hb z9--O@4*I^`I~iEOFaiKd9P9)|_oMH9$3A-ZN39y@P*reeyLY5qZ%hZFS66f;5R|Yk z5mU+E83f@a8V`{RT~qkddQYCPSGM|Jp&U@NiG@9_5O!Hvi9Pbj zLq42i@fAy%;-t~31@`>&PgC?Z!d5f4IzPY2Zf6pga_&cHA*&fS1aOy9t0=1^{2jO< zx`pUIBJVLG*f=`Wm*9-4B@}gq$VG3~NbY~8w2*pSIyy^tp1y%cy5CXn@21aPyCJWA z_3*a;;!L8wbXZWQ?ciaD{J-ca$Qw^7WI*;Atw$oV5hx+S^N|ObnKv1N_Gz}~Y%Cr~ zBa;U?+gUso9~}^pN1yc6S9G-ByX@nDM3fHEB+63SEdu!a+4CitK70I(=kk2#@|{c;DbsroqWImqi%jk`gJXIzX9zpX zJCRaS3)K_9_=U}yHpecyXp(*VrRQmdddz<3x4&q+cWkp&udTGZ?!4Vc^{svDP5b?? z`~m1-wJ1%7CxD^+`s=Ux;QPs-h3(w2(-zU^Tv=6Nk3IGnXk*izBmerZ|JwfG5B|VM zLO%5HLw3ocOPES_%x;h)_HiliC5IDautU8MNhPWD z`uUtrJ5ZzfoGud7Puj2cDaaGvcFNayFSSM^1UW9?V)VGfo+eNxK4A!v{|fMbKev#n z(VwZ&mRP#zx&yT}eq|K&KnOe?fh1am26CbceyoJKXXGMF$vGuq4Gr{ieGz^p4AILz zlaUZxyaT$1d zQG*8u0V=HIB6z!(hyqxwdiRdY=Wnaq|{Nh?dy{KlzD= z*M|avQwnlc0iN|6uf1wV4jr_iH8mctq=BR;7gL7Hy@SPB8Nh+PQ8vz4d z?{xQdAX(?y@652@O*v`4K(?IWAx1DN*>$;u+y+B6nx!li%W|Rwk4_ca@)#4J; zn#`qzQ*L+!;attv{h$Bqf7>l;gW9$g*w436UR!PA#0i9T*VxplQ&EaX9h3VMbD}jP zSXJCLC}Jt$cbLz-^y+2yZ~yYIAd0=>Oza90B|B--Bs|heAFy#Fq485pPScFtE3aIJ zLVtk=9nvO@nx4WkGsvDeBuc^jsfwRPkqu}edSeG zjy24qzU-2Pws`{+pys2f3D55#q&|KEbE8qfgQ^F)5u&MOvr)j)rcJZI{oB8#?vAWTEJ{@s0@51sT>`sZ`5?%zF&au(vB%kfBVB)|{!W@>v^kTDK3 z9Gxz(_qj@$e)d7K)^91CKa-TgKz24>pG@m<&ZV_>Aomj=dp>*K{d#9Bg6zI$@fy7Q z+;J`|@+4Z~;51!`o z9+^WRsUlg$P(rg)`1rZ!pS3x2=i3(Y(CgN3v|Dbw-IgxB#J=^-@7oZ1n2!Qda`(sX zwXqY%+kg4J-?RLp691<*seCU41E-y7I+_Q)Y2!v8q9M=nrI%Owb*dR1`RUJm);;zr zpguL*yA9>J7(}vHUwy@%dg>{=@~SIs)20pf&bqbCnV#Z6C~MwYZLhxeip`rh&*tOB z|IU|x+d)xoz4cZUJYzVi3NK;=qMk{i=gTj@$sT|FaXWPIpxt%XU66>zCT7_;zxgdf z=0oh0pZuhI-k<&4XFxrhM!x+w>|5Xdrak(^BX;x6H!(hQup1NK2B77}8*jBiV1^69 zOL=S!`|$QMl!}V!q_p+#+u#1S{m~zeu)!#P&7yuUUVVAnXQQ*sX|TU1I!nKFjF8gp z6e)&&FAp{xD778t3iNpOcq6lsuj%gp+?9R^Z=c<4NL||tx6b|TL+{To?6tc~(x4gQ z>h11T+`hU|tw=&t^Y?qApfj0v^z0VExy{*3PEyYznA`#7DXf5(mPL*HF{8)J*fScD zo_I4=ewiMZc~M9}+v)!5v&5{gq+ZI`wKW08Md^Sj0OIfQAKNv0^jI5wW3|;(4YNP^ z!#}3*s~n|ti?_ZEp%|>aBcF*%H+zfAtXXqy>Wo?RJrBc!EU7Uf*ETl$H4972Jd|C?HA|UsJAA|linhiuj&qPzR8`xDKKv1@sHnuK zxWf_9vYBh$#C-4a>Kf*H(|609>C2WbLwR4uz0I}}W5?OhQDZ1x8*8KSwm*9BeRhyg z_kxQq@~DEwZ;qchjcb)zZC$-xbNvUb6c2gac!u0Ctfj8LnL#nrY{aOsRt9uwMOlr9 z{=*#Y2)R!JJf%3sfA_6O(rUa_XBVRB=q&Op`_(ahd^hpESRDwe9M{(cmZDPrGn%C=-ispcUi9iX|6z=#<9JIZ={vk36f`|= zcqjQ9h%!v4Gcx^tzkS-NawH9nq@#+=_CM$dZ?Tos_xgb#n==V==o-It1)Yg12i29y|56!uJ zNQP#%`fRvVgTOU=lXS51!@m+x(R zr#W8>UiRp56K&G8>DJuD_xR4$*WTdw*-ZPF6r>8ZhL0TM;rF4#%kf|VL0PcStI2hr zbq!5+^))xR!JwkEdZ0;*87>g+rPXS|JD)h2@b`2E4XC}Wpwu^Ra5dBD_*^;ZVMGFS zzh&jsu7qW96ql4+Nm+$MMP7FK5;q96rlFTNvPq+T3Z6#lehh>*`c}~0rDEm=W70fM zQP2aqr5?v7cdvETu5c8 z?a=EpxK|qOxHk<8a^m;*9D_`sMFvUg`e&+K6Vf)rG-Kb}a+dw}$B#cxD?!e2w}b5H z{z~^=->3hjE9eUy%1Aw5e?=lriy3Tmw|kV=E7pmKI834mmQgTEBj`+i?LzTNQL0eP zN6)E+MvMa!wFpyOxlCpTceG0-_PzA6SK-R}Y!h#yeH9}KQAh$$SbrT|ErcD(4RZbZ zrdkG(w=fpDGRzm|vd(zoBOghZ47UYD?{LXfPc)M?J4XsmeC?NUje*`{)VLNEdIXq6 zDIp1?R`RT}wOaTY)0NtiB*5bSbMUSvX5jUSEr^|&;^SiUW1R>uO+L)BE_EtjrWNlrj zu-3V`clF3BRIdAM*M0E*>4wwZQl-#zKIb1cbePN1dc0w$MXb*{&ZBeY0g};*B5#eN zRucGmjL2+agr=TR$G0(#P?-500%gIwmLM1H={)+pmdbDa%+l=gpffUloeP~@2)S@B z*FEb)IbX-hKuT3|L)ueXGxWr3zVsPniXb&LwN2RY0=@|&3c_@{4#1Cqh0%B({?-K%!_Ke8Lb42579X6Us!2UWj-Y_o^?DCv8~;6b^#%FWO;*U2WY- z_7>WAd8m_z0qsT)2r7P(8Aua^L5A+=U9q)A2LxBhXISl)DJXgq0BNY677{$k#%i_g5%|poM zD&)=I9Y9QF<*~}Fa5kt>xm3F55U?!jhdX0HmNpkD@bIWN^IOEm84a zy!(~1{&?LCj=mWwR9z+*)gujxdV~~$ypvoMR35@g=wO;)CB-HQCFP_ZVjUxe<#U_) zegXZ=Cyt-+ko3%%v%G*xLR897LRdmn3RKnFEhu2cU;@8(*S#-(owly33CCXxH#mx`CM1Ia(S%+vCv9J+{5}U!Fs9-+@3qFT zE2dvr?O}%xANE2uJ$EVNFK-z?(@c&^Zw9<)3ObWpzEH=U z|7K|MVvkuuT;7fBT09CEsSA&#nf8~C<_0E$hZcx4VlV__Gs3Ik*g-q82hX|Tgf$%5 zX=m`9vYio=uaL)y{*I_Fln8WJ(5cLoINTWk_(&{jF>K?}y_Q|pX>~{T%{)>0~V^p$l(tX~Sj;w>Eo^vJ_=q^D&gz zVSD)Dhsht8`SFD)=i|qZxU#AyBwk-vOR-oZ`QRE4bxLtbk?+~Jo7vJ#LZip}hU;!5 zZ#>8j9@qy!2>VBy(uotbpmf!^QK{)&hv;#Z;xDI7>(Ie{7=`&^<-!c$ZtbEzg!}Ra@7hka9!v}NiA~y(*9X-i&Roc;`$L&pq#9V*f4TQom802k5 z;3RJ7YHcOB^}h_#5IyN8lkdcP=nANh=vk^!mgzbBqoaO#$MsblnUg9DsBy|g=jOu8 zIT&*7P1Z~lqYDE{6QdM`@W=~5H%&)(TN|`x4SCR6p*0-b3UXV$_uRXu?9nlIA6y+P z&mGKEq8#pQL`G`}wD>C#>2|?_!sqLld)A7g?rJ_`^~d)TZK0;%(0XJ~YIDN*;~9kgtzEj$yYEXR`Md@wy7!bwU?%79cYiWD zNyf*dF&;vJh3`$$G>X#^PPKma|4%6BjMp*Efx+$%ezX(2uyC2hMwUU|<_S*6%tnSz$s8^m(%o(~!5L5wKDuQs7 zu!hn6y~sNg<0ar5qx%ifMy}Qc^U15vX|scE_4cQ${s;j*(yE8%*^AHIZ?ynUDEFwa z`;d|2Y(7OtqsC3~(UcSwbKZa;9P@vC=Ra)h=n=Fwtp`M8HI>*) z@u2tF|N38l=?ZVzvSs$wul~Isd+)vX+VbVgy%PHKU-*L63?{q`oz=IVq6KO<*EnJK z|KxrfHe|Sc-~%^t4|~|ta(m+O$LL3Hw9kI#bF@i~w`Z3>Yv1_BHvk{G*juQ6a{o`< zqyFq?KWjgsU;46)7KEVK$sGu|H+SkLVkR*ed$YI;_niB;;~0~ri1qJkAED{la2NlfAQxa zhTXw;OYEmV{V}anOp+e7?JfZ_r_x@Z2 zcxKU{Se7Ks>mXr-;SP#hKC&gZ)pl%q+cqLnGPa;wbWb6*A}VJe2>_ z(A}dCJ!mx}#@mUb`=HTXR)6Xk#sqa17!D&RO|^LouAq34Itx-7EqK&j%3te3eGKsE zy-#Q7*ARt?TySh;fw<6hVaYFCXWYPuMBDv&mMz@Qk(99x1 z2SC<_s(3^=$G}FCk3NI7-PCxBYgKwxWp+NcGoBAgO)ntYx%VLbyD!`B%^PjuQpOw> zlvBa0naUb77lKsXdoKH4wf=}Y;i2@i@GdBvP?LG_vwdY?bdU*~hN~qC^P~)1sHhh@ z$MdkmhYzJ7GK>t){Ny3}wBPX1w9vt>zUoTbjS=wu?|;{>z3v)PB!%8<-G;E8I(3SD^rIgp zEI$q7<7f7FfA=>an>}oAzOl-q151}Kaic|JG-phoO3T&FZfM+i(~aKit-kB|^DnZG zee7d)Ijv!yvyY0C;s{0NJJOB&qd)qiy5x7xMklGCrBkr}d4`wDlpZ5>C~4xf6{=cV zFnn_CbnRh#<<+P8z00n<`f?jJloSVEX;zkIg@?i`pIgsL(4Eph>K;xLof(AU zEwBQv#a$4I>#!CgO6#`lus2_L(yB*Jw%H4>a9Pymk&}24Gg-{DQ$_;nIPGCkrF3_N zkx+Wcx4Yj*CU-R;y^%OL_LO6#tF`VS-x3#(en}|l|3u>^&uww~4r}#1Ul1C-cihPu z0#mz`hqUpG^H7{GKlL!c58w=c{!>=Uj9(FZ2Z~#T8b0`&TyqB&WL|ceS2s#QYc6w% zsYg`sh8*jjau}YG8iMt{TmDI~x^>M#DjL#Dzo%kxJCk`D8HVb_1EwA03_o)yOe@2K z>1bE$1I&f)&JqqYtr}U?RW@bnWV__jMIcN)V=JEcxh=Wz4unFvwMhslw+eS>rY6t6 zGnopl3#W+Bl~Tg@biX|d7zv$>%5Y65OI@-1>t(4ZW8p8YXDjo+4)hm0sp(2jevo?{xETs3^xl1}h9^QbrDaM$Nzc}zS z^mobBPtC_cwcr@cx_y7LTx zFz|R*wv2J*`n6w##nqvqVlMUTE57Vbg@%@dItib3;zL?gP#O&TkiH3nR^n$DO-R-_ zIFEbol6sschx_Qv_po;b=WlHX&z)n-vRW-4dD!I`;6oU>I>nA&R%<_h;1PS{1)k;V zTPa!%wL2MGI%iqj{^UuzIR!zk_oSEeq@Rp5x|DH>r|kAnsKW>?g~8nlKO^2qScBo7 z-xCiy!D5LD!t-H=U&43gfoc*FJ)*@WxwdA-D^@_k*F7J<+d2u`32Z@mTnZ|^dRrc} zyc|Ipw!=_)IaFZ6MD6m3VWMW3PRds~%DD1XPay^p5evpqo7mFKc{ynaX@{Sxh;3@3 znm3yHsu|P4^A^K3bN*m0%c50VMmxjR^C?PeBZlo>v3uL(q{&07fa#<6e8|4}@87l6 zue@lBufHAT60GX;B;SOxJD=U@j~U;!bEiCf>{V&6YHDf-0neuYxzy%fbg_FjS6+Rs zD_e!KXU|<=(`U}MU3>OX@!CNv(k#N$585(Nz%IRP3BuGcjHcV9$&*3 zz(d-g%bRbxjj;BQQTlV((|hf~`yceahOu+uM($7_Q;o+c6w!Kt3C8d-oqA1b!z5#Dne`&zpBKIp$Loh0V1`84fah zSQjDnMF{;uJm!Ut5cVo5pP94gdf#|yxx)N!0daawqM$8VYmte__K-><U0@IobNIv;pMS$Wh}`@$ zghDT6>d_&vl(%ute7kbVVz2Jqv17M0MFk$mjF~gt>Qy_BU>&tnqwu_hhdh4#c%OKr z4cxeCqYpt)E6m zbGoJA$Xen~3Ggr|1g6Atd=o*~z$P{*@7QbF{XnVS+8_+|Y+ddBH zTXA!T|BRk1{RyyC30D^*_;zhsN3O7dK>%a%o+<1Q2Ss5ddCdNTDGCs9!wG?o9#P{^ zkdr1%wPAyY*ong@FlpPo*R-Z`5Fzty+rN9C_dqV7xaY{x6Si&pE*I{tt&Lu6R9qxR zr#id-I9B!wJfUSadEzL%dBUZqX`AV2#7i$nXja*}b(@@*Cr=#hJ+Rf~CAM?pZiGOA z8_a6sIYX<35rS8bV=LD51oELHMh~`KJ2u$Ni?6a)!e*gAwR?~|8P$or8h_Y#A$-(h z6VeO1l!M~Hz<_qYuY=>K3=Ho1I|O5#=L=pyXvdPr>+Mw-JetfT7|nL-4GIV&dutXa zC_>Rn$vLQlzR5moWMmG|mt_@%DAf{$Vpphg=BzoMn$YLsEqzA{H3u(O9dezX#|t18 zCB@~g@TIur0VUsJE{WiR$Qc%u>NmItqihF<@RsBJmheO$QIdI%(1z~#Sx^+EO zVEg&_P}lKdL+(+(s4}~hsE(dj&nHhQJZG_o#ND35V+;3Z&})fDbo5z1Mq!r_seHC% z1i%p~EiX!n!h5kXBvFz|bxB|#7eL8P^QCwrf9&*bwrjGM5qGe@O+;@jZVnlnfpXZ2Z&_PxfbVyoe z;oRA_a>Z&pb@ZrJ44=et1B7+$cYEWVNIfvT*Uc{K_dMsjpJ)5NbYpjliS96kp1aGO zAX3A7nJZ{#FSP=Zqfkzi%}}MXUV1O|(86vKXnhp#m? zz8&kdiriue7Hc)$G6g=IPZsf{iQ}+xUw0VB;S}05!Yp}I$L`$ymJ4eMpKOZCs!-ym zY8kkI&>uZ|s4ZE%#HwMs?b~+R1gdxEfd@Svh{*DaV#jOlz+l+Gq@^WG<}o>FjJ@*m zIy@p&E%%$1g(t7;dm)>{ueh+)4B=rzD{arAV-7E%aF~Af^lOr26lG~6WtA$i&bCLH zRzCJTM_-;(O!xiKMdkukD-zNSiDEo&Lz6rW{h|)nhaywNqpY-;bOYXwD{*QQpbshB ze7usjHiaHhc&fIg`n8?nogiofqEmi8ZFfEsTEHe;!1)K@3H2>|4FGrIsBe+K8Rx9H zJ@0YlB_lu>*YaLvNa$y5%;?;~U?NT<^?$jgoaF@)yT&M=FZ2eJ{$ogOtJ z3Q&OJMKL?*NVGCV4E1P501G88JX6wBbX%RaKCOv(qn&WM?$M16fw)NMc%OLalh)Fz z4JV*YMG=dz94@EiVFtY)_Y6lSp=AN0#h`4|J_PNK#NR6Ec%e9;)nh zsP;J^`-T2w8Tswrrdb0DoM15}RXbRaY{7qu{!Y?>@@C0&Wlta4~mKWPT zTJQ2w8wtMobI(0*bHQRSEobgF2AZH&4<9;;e95;~3?-!_%9zK!2ghK1s*(38o+&OMTsqj{ z9aVL!2FT5um{NowS3c?txoUZBIe1(0l=kj92)1!KA#|<*iqEEv+wH`$I-54Nz)q2? z+DD#HeXTXah7g7au?l#|#a2o_EAw$;YwJLt3nOa8I}9h@v8o=1P^=c*fD$;Fc5j^ z>_sjUs-^r!aaRksrM$gO#(>($|hrQ^!4&xVL^`svR!O?*H=65@8@X ziK(zlMj$*$szKI@_CGtyL(1XtHYq7?$r+5m<5b`m;o?r4HW}cWV^#&7b<(3RoOa2T z2@^)su2)GBBT%a#(J2?5tpZjVVVpzh6B*!b7*DzCG~t{LwN1?N-h@|8`zQBu_{a%l zNUcK?JM;*9f0Dl&>DjL%Os?9S!UAe#BuaRUY=<9|V-fbf6L0?vKuy_2xzJf5B=16k zQwToo2-_k;hjq342m`Zx-X;|+UcoFPdM$_do$pjhomFVN_w2PBuD{w|1l_BeP?3c5 zBwAPw(zCb;0E&D7Am+`TYuV)NPShPCw=>Ol?b_=Y#chP2b&Wa{{TRHK15^aR_U4Yi!ND2)FS|z zy!65=2)kUz-Oj?hevSU8A$OD88+07J}Kq{%yd?@^>`Fx z7fSB_`|l@j{9${OD9q{84c@;zefl(xdDH6>MkFSapGhv z;3r%0o<)Jw3+e*vcQI4NGXBM8=VSn~nvt~I+%FdlTt%~pH`v3AT|Kg)GFS%?n zJQW#2nQhSf;3zR&91U=n4uA*Tbiq>>78$dbg5yJ*^%)1e14rzkam&*lTv?Ro7CPee@$devd_7Ih1 za9qJ1d(PoF?^_SCl|+o@eW{=`*^rEn@%7~Y@Cp=C9LgYoRa@60gqtYpBG*D#Iaf$K z$`yJshtme$vlyU_JaDbO{>Ey4o^tObn=p7DLT=NhJ$4E~nMIqz_U#}*(c<*>n#~B= z={9-7be|6_;oL%=yPYcC6DR82vMyn&k@w1SPy6>Dq?M-C%0UG~Gt>9F-$RRnVXUXX zqpOYIbJZ535Tkcc@*j#_*<~za!Sg9M*^K9PaL*o_H17(E-Wpv=IcWb`lxb?%`Ikc6 zDfe_B#|rIs@uV!@PKVLV_o6)sMk(p_o7OW7qLh8y#NHftFSdi3$T@`h zHdDaY02s(&l+O!uxZ>p(Z41}kw0V=a zP;J<_9%XyPXF4lasF3o54?ajs)>WPxK7RZ-tyQO}Sy*pZUP(XnuAKmk{M5bvRj(e z>4=a9y-QlWsI&)mvv~eA{CbSSz*gjfhNSG>caW4r7IfH5*!lob3NR41A4D#du-Ccv z#_Q|7b70cM!9KrR=y5XQgekv|=v)g*w+OkSYiKaWzI^~G@o6bxaMd>K-**_fQs6l5 zLWw)wgxq4^a&zc$mP}FgEa7XHD@u1FTZ&272??*ImccUBqxm>pxIgHtb0vVZUb9Ha zLtdHF)WGG*Kpc`Nrk(liL<=JNINQXA0i4j+IOeheEX)8d@g5!;lxl|n05E(9RS$uX z5rFS(wCk>2W=$Qn4hSGz<}mhAS?Da=f8wYMHC3~2LHHFB+T2erClscskRlgvL;wU? zcrFJVJR*m}kacS}dN{BXCYvyMB85HbVa~&YIqV8sRkkszn5wNNwEiI>Q$Ierh zQ^48DAvTgHYB)t$lYCQ%WUH53%9-j~C|f?Vn5+yVP*eV?A@COmR^I6@?wG+B7q0pt7Fv7;XD{>-QDvhRKOXAbE(XU;sXIm0ox zZ@TGbd->%R0EIk*=R4SD(>)+$vU;K-cQB{=0LOp-2R|SLTIrSIy4T?p8p#N|a@i7$ z%~PHyRHR_qjH!+tuAH?)OR~LV@Suj_%?nmkg+_uQ9XWCY9Rl<009^<(W={8lMHMd! zMpWz4hT*F4xx(z5F>J2A_F6XvWN@eweF5!a9;pxw#quLIhTlVOfA=pP1aDB$0n=ad zW7OD*wsqYal-dDCjE(|Y69b$4v`@a%G5Qad+)M{aq57ja!zm&p*(mevc+8qvcjO3N z7_4_8X%fkbEt_|ESApcvn$>IFkx=?<2K%pcjJ#k)Hbi3%T%tRIFSb!TBSXIgxpIm$ z$4TfxmF6PI0|!vn$ir-k!+d%cN?J{&O8Jq!^BDt5Zxm4Rzk2oCDESITj22r1-6Cm4 z$_2H~f`RR?3wqvkRFUoiJ>LuQ>`Wj||EZuaJg+=w6h6185tJGSxN*xSd-RcC*t|=x zg4s(6V@Y@jyx%K7y1ilrm|BRJ;r8{tx*41x(>;y*2jkn0zZYBD-6=ASf@ZNLn3O|v zoMb5vSJq*QmD=1SsM-lJ%bVTI+DSpnd(^cwh~k-NmWLV72*Ec=M{GhlO&CAXakP&! z=t5Q6wY30u5Jgd5QK9h$MmQEARNGKqTQ+Z|HETYY+XrmLE3eQe{Q*MJBgxqo+yD5> zuX@4O(#2QUlTSWnFTMB*tuj@7ANs@dx%867KJ22M_B!=XAD}it`AT_DG7tnx^3n@0 zxUvmi^a-lHOY9~3`h|ctdh}>-@lwBYuILA$t>=@_7Xq5R@k*4uo=*mV+H+og^;JJ! zw5fKojT<)-h98f&Hii4>vgsHgs*z}H6try)9S~Kv3*eMr5SqP#ah!?-C}baCDMR>A z#cAElqZE1KshRW8qGzS0*t0!{^JhwTI1uZV7d0`lX} zy|CQgSoe-C{=kRm+Gg*e`%w9J_T9^L9eZw32yJH;);_OmoRv1xc&EB8-|AqD_LYN< z=|Ork92Y{}7e+yIt^nhA;i})xfk)nzujsKf7g(FxsUZd^uxfB<=U%$Ze*W+ic&$6= zSH0Bnh6@O9ih#SM7`dB#q`L;e!GY*yP=E+8tL|GlVWIuR`I3Y}iP_ejxWK>yKoJuK zv@yxABD zw28AXv5^z!+i5^bvIJV7)-k@<6YBF^XcfD;JFdp>rbZAN*I zqI!1>?-x-opeW1ix$_*IY!Ni5K4l@79X)!4>fJ#AmfYh-Yw~pd%kO=~!`Jhs_S-&4f!q{FF59_wJ-6tH7#jhI#mQ%Ji9zLgwvWt`tR| zYBvf3+*{O~rd?e1yKn!w4$#vkqfUB$$%+=V zL7$+f1<_EetU{D0>!kuLhvM|M$*5^GB6FwsD72ZR{McuY6=hU`?+cy$qLc--*e;DU zi#pU~g#C~rcK1NuGv0j;En7JtBewfwvRpd|4ZZZ-^Y#uAq}f;BXaz&Yk)om4Nk&)y zEO?Gv87b#(=MOxu4gr_n=Rrs3s?cM7C>`Q8+%hiva9+cM_gEP1hwr~$}9E8w$Q zv2yuWgwj@1Qx{Bhx`XdS)Y-CX-Rg{(DC}g^+LZDb4zuSI)?+>F>zZ6!=Rju~SLX73 z<01`CFHboqhxx)n9XeBHE$w*njHEn#;;_B_`a0XY>!@9G$L%(1@@%VX}4pOL-}6;pdd{@NtmMUSK1j5~Ut02h~J* zR60B{!bamc#t@2%sEnHsz(f=h7BZ7&_zt2-O28du}Te zZ1}Jd{(E(Gk@Kn)u1H61e2T)f(PA@%V}_3$?M8<3o9c7UN5LuzGj-Yw2Wt^xRRzYV zp0f(HF}>P|E@<7SqImTZvR-w~5E}w+x$ZZ#jj^8O@kLWZxmO{YX|}KoPOk#yn7Axa z`w$bK?}_&Uzbowh9-K5JQsP(elN1Gm7Fd88ixNeR^5CVaRRZzsM_f zLqd4me5QyVw3Lf6-$v-B3lOGuKHqns)>gf=+Iiv1yFOxh)#E5|4dYogBLGh&6FWKc zJ(2U`bP=8-`3lZTI-C4lWJ=~Y)cZ*Bg|}fwt&piZ((P#9h-l+dZhSxrI@J=>$#p*E zIs{B026NyBO6zr8+(&vo5BIC^UF5QpXy|8^5E1F-F-8#ZhMLiK=O=%iaO6?jK_SJ~ z9s5Gd1O!YV6v0kjc+=bT1JVzQDK`d1*ePOF;35V!i5}9crz2tuoECcz3j4(eMMhAs z_NQPyJDQJw6(xhBZf-nITf|N)sVZjTOg0|T0uRGym*&}ssf%pUb<-@Hp$n%lI6BFH z=V658fM%rLM&akC%hUcU?8({gA#(wZj@3VYS9GNDG#)kz@(aK5o8JGd zAx+fpz z;u)48Ol7#7p8?H?VA8CAuYnbjF~h8{uMZk zuUxdCxC&Y#P=27arm~zPtr1Yx}LFf26Pnq@d%I;-jwoJ-V~H4?%~*;4R&JJN*gk91bwgdw&c1` zSz{}$6#`YE4XhpfTrs#|oUoO{jNgzWPvmQ5L@3MQag}zw+{G?>a(J-1zLNd{_4v#8)CHoRAv;RzRu0(M=|3bl^Om zUggIUGKDXbHe8~*KkRoZ1Bbs8kz(Ce?xwxt0j4!4uPoj#q7{xmFJp>af#T z;;9%6@e)b{VlDEvKgt+4m#d=>E``o}a9#K~&I8luWfLq|ghvN2;Kg;U_0F=)2u)ya z$yQknk%tn>RD`mai0Km}mCiLp;fymkc2&;Zv-MvdzHEjCsg7=9@d8p?q8 zx_|c;AW{uOVLEc(q@~6j{yLj_$*NYx(%sBdu zzmMvXoF5&9z4%_;Yk`-sIBNRlYG(2TA?tJ>L&#Fd>A4*cd~j!{bOLAszqNHY$WdV9 z<`<&2Fu6sJ9M~=G8N;O$rB8~Y1UI+NfS;Rq7J~7(z!>9efeM^Vr7s@MW zeWl`FxyTky0+rX&)d~SX0ZuEM7Bb(ba|{GWA<-sh8Qu>!PRvXs`k-tk@?#>VWKUui zQ}Fl`7RJV9Y?DP^@d->_ih3*$v)~W{qx*rGxcUU{nl7SAL8{i1TMIs)p5u?R@7adbGVfJP>+!Z`PB6QnL_hoQ zdA>tKa*x9y#IB#ahqC_Yt?lu+*WB~Ee<_x94b0~XmuaTVm^wrLaMK+)ej&cXffVS< zC#b#HvnZWv_aqZC!B=1Pwn17~1()>VzN8`)WI~Y~4<|rvx7fk9{Tw~=8I+dFu7PLR+DS6(#7=cEu~CPT_ zSbBW+5V%M5fZ)FB>Ak_#gBqi|fwc&>$_KT80Zx^JdiLGhH_92OFP02I@wOa2Y~{G> zm-Qh`ODOj3Ksg|tDa~Ozj+ZAtdkoXO`A1jaJ1)Lh{_R`em#LFxE9zBK9qMUyU$D=i zw;m#=*j>3e(5yU(#ae7p!c%2HxoCK>{tF`=-pICqTR3;PS@3=_V~HuaZTwZ-72Me_ zR$~L!h1dV-?V@9{k`XcuOg62jv>Z7D8H_JI^e1@+S+d-HQdiYYN}?np8Czgd06RQr^#JHYy4`Ibl=(2}#6HdqSZp?uNRO?MW9*#w*#cOM))}wQ^ z68WeI*{8cUuZG*WK~6hknZ}M484JgzBMGb-4F&GL_iprQ-i4&Bc~GXK4n z>62SIOCT^je~RvlUd|6F#NlU-VRpJGq$soQ*>UZ(<)_IX|8yI0;a`_wzyoMG2E1qR z5Vg*9%ewmlyd#!|?0e$?33gCW3DW!8EAk|M%f|>Y8>8F)(PODt>Ghm-roRLVYQh(H zpG`oad6GWlL1Fr_a%J3XvueG=pCoE=3qi(HJM3S37~;4m&G#CEV0%t9khA}Ieb zCB)V@;>8L)BL`?!B))VxFM16%>fGRaEK+1x7=zJug5{1peZ1$3Pm5G+Jby@Ve;toK zo88y;hdv4a=|YZ?Jryu3a^%47)zaQ@6pfNj*QQD84bo?G#;OOciOhF&Pjan}LqLa? zRS~JU5l}1-1e`0bxI~_PdX@a;_CLy{*L*|@fwIh!e=Fl> z9oRhSfWk{=BLtk2kx__YA)}TG2{mRfV0d5SJW(T0m(K}UeIDDe>SzrtgWsH1(`jg%;D|pUFs&%YjAOO}_vGQpM?^gNn2j4Hh`R#9{4&9|LhGNRhX`+fPSTGk2FQ$Sp z(;>78nP`?_|6Z{()`N|Pglyt#1})xR?%d1uB3_Othqr0&hfq&=#}LivwEILvT8NmZ zW+dSe`tJLp56m{l5cfy zMVM@4bHxnXENN%)bLWL><|Eg3_i#Cyj#2j{8DqxAZEQHvcW!e>Pm0+F(WABSOf7bX z)pCF&SO5zaan0-^gr@+p%1RM0*NA2qT08sfg+lBYdsv9PjzR(ZjBRi*d}P)(D+h=V zPRL^-4zh_5UmCt)eu0}~qgWHZ>izIaPJl#%GB{GzseI0_QGosf^Wx<|#6j!@oac9NJ_xp$E;e&dycqE$DAQAG&fgPxn`LlfDhQE`^ z6UU+)5;zr}Dd~NkG8yrtaGbGx;5YGT%kO3G zl5-G8$p&f`>wa|!BQ9l&C{bwRnOvrjHF87&H2vHE=eR-z_ueFOss#;YitF zP6S=2P^vVyg%ro}gxsdODWjIGE}ffDzt>Di$1*}4p+2LhDksbnJ)Ao>Dv)@-yGG!g z;B6u<^L|YjnZ98ouLrL)G^!^8J`Ph{B8=4px(~gDm*zMGv|ThZIZngW zz6)&70BFlHv^)IY-~LQ~^PAtxLytTTmlfh7++3{Vqe6x&L>Rv55UYEO>ev$VkO-Sb#3ST7R84Vjy-6U4*jYno{b>gjH2l*Yq|J-1c?k!Odz zwCW+aIqDI}VKuZPpwPw(PeD-lMB>$6Vz_m7`8Z0*YRzM(5)h;A$Q~Jz*(|M1xpHv# zPI>W}$E5MtK{^z5KK(X{&#crgk6n-uQ;WnE-C@D8N8Sf75tk?SJ-4zv`NhHkZZ8ut3s9Qbs1g4BXFo3QxcoenH8raK->q5urkt^v#6pxl{pGfs zU5k!HudH4Ti~2A?VDej$dAjg+>zAj0yrC3m5H< zZWc$2Hs;_p6OlL`czE^QrMfaVfjFt~@2 zQ^I}75>3q~px{OAWILLTbTrF*yox59JeD+~vYd5$xWSGw}dVN-(DPb0-ap%~% zjCyN~AH`MJ6HHcOFyfy3EO_0>2%_7fwMiFk*87ZTXL%y+N*j z&(+8r-i+Ef;6(!=sRhXe9mHeBidf#5vCgpYGsj3Af!hB-o=K^Q%ERF%Ca}H<#SvZb zdU{FT!=0^DK>Qxm3Bv8f#0U*X1F!gzyAMkaQtT_rbI~diO$DH=q-a~ffs84%0d-D+ zCFIb{QZQF(>6yUU-6Y%AuLKHJs*D;@4xHq02|DZiIl<#1 z6&%Jl`uB>%>EoRmcT0qbj`OTNbwhXdpd^GL9EY}@6DCanxJRDUgZUX~hC#L}G93Qp z{>_09zwUkSL-|@G1Tb8S*u(FB_j@%#%#fOJ9QHa9xVXPrc0_Owt#|hO-ftPlz_i{P z=lWjC5kcc>u?>RJzE$hiSvB;6SokQw)OeT*W!|4^2^j!s40Aw*F_6`5$vpI^(4$}$ zrwW_M+y`ci-}9N&f~BBm`5C+g^cqlrL@ZRx(2ik-z0sbMkkPx(D%R^mNVmh?ebGhd zBNt#fTB~l79eZ|59TXt#^2KB*S509PWB;*LCQIvGu?%j7xXNk&c4_1&r$5>Kb(5R2 z5%;A(^)Kv!6kNStC*toEk`!@lPSIR^&U45L>?3Ow(ul+Eb>>hFCi4L;A45{Zxl_rCiC5>So$xo_BjF55TM{Bpb zW`=ng2GBOmypF}V)w#_~YSOf>jiUjjaTkhkyZJcg$&JYQ5272=X%J*e(Bj(Kj%f^f z5g-7U5oHN(L2vZwzitof)%cvDd5W`NOsMx~r2U-p{v;Q;;l*9Y@NIZKE&xDvY@Dcg zqSmy1^-Cx{E0B?ZQM~TT%cQ&nP>JLr{Tye19$6P0li0>uT`VpdS|R40gf2Ck+J5Y z+rV=-cO6SA>EGnU=YjFg^=_q@$5BhVi$Dtyk}Q9ytE*8jZWr;7NigaqGS?uJhKZ9+ zWUnkV%jiJQV-7we1NEnj$>BHd+jKx3YWWb}v)0*ShqmccwG_%WGN*tHmBN{M9CQOx zuz3NeNKs18rhM^bR*3zn2gLjF)B?sGsEDYmokMe3+?oW3lU=w=GsD?zh2y|`#eG*d zm>uJp1Cth)!L|ca zu*h85bad3)wDuMGpC50QqTDQ*GI^X_bJdlSjZhC;R#GXj&Q`XMNKD2bNJFa1^|c-J z_jt2r*W{1EU;8Yqw(G@x<{;wDjUST1pla9^Y&N3@7S73&8(rbY7 z1G+Z4fREfnIMzHcK0=9Naq{%&h}02J6uTfv#DCLa>K$ z3oO5Stc&(;bsBX-kA*UX&@1@-!8r(xQ3bR=HE>SUI`eZFb+h}+Srusi2-yUFcvr`M zHqoekyWAtYd{PQ0=09M+P~b56h+zq{22ChixPbR?LQVy!H7-!RXxxgOgZQ#Iode?F zs7(F=k`o>k{M(H7e>{iib1isJ!Iz&4JZ|2@d6SyprsP%F)7j`5@hspmr=ukxo2i(j zg)N0xS3_e11QrxAc&r!?SHTauBI=kW@BpnWKg%&q%qf0vM&cTqYY+^CP@k>1Hm&CZ zbmeNon5_Jdz(X8Yw_=RbD#!u@*4l0;-^ANylMf~b5f7S}-Sm(V&CINQC-4(lkTF|| z7P}5xUtaO>N&?q}Enb<^p#_diep0^S|MaB5i#&peReis7Ds_$eCNQ51(&tsZ6~XK3 z1Njg@?3wYlZM$driIJaO3wb#I+jH!#M+5H}fA@=A)cQy*5EQ zvNscum$`gh+@9k9Y+%`+WUS38; zn7(KD;b-*g5Gh}X?mZouyP!^_V+#HAka&yEtw3Ug{9Js2M5GSH zrnyj>6A3gP1+))l_Xe&Lvz=@@joRZKb z=-#Q1UBUYI;)A%lWT(OM!Km$10BO<@XE0E+Z^GH?-l_B%gE1wHv1t|L9yo-88=pP1 z_qFtySsCaN{UBmc?Pv^f0XmG;%PlwGBy;D@LC@rw^2{?&qv^%{$V+IJbI{0Q*|O!j z-VZ%=Fa90`#ABXZdF54TU%5fnpz%NsgnJX({|y^9Qm(xgZ64u@dGNvenZl z89#QMEL(mC1T2!%;1Rgz?%QO=lTVLv6Ce&oRiF`kRj^=dj=L++QK z|Lh0aTX-p2Me_BRJ8qSn++2+5Y(+u3>#p0RsCQBYcAQ>CZ0HQEI}H9d1PBnQ;&D?FaKAScThpHc))-pgWmhBe0eU zG1Ls9sa_ww9~Bt0R(*M3%sk*hlr2Q~Z*<;cpU_s1J{$^IohyEZGC)hvI$2TarC>tE z3Rj);ML-{RZPE4dbL(fLe+HJ$TtbW2#vVP!%1R@zpiBtx7BvHcEJf)!-lE#vh4(~^ zJ3F|yx!*b0CcJcRnKZ3yMrd0o{1&=$e=#Lh9JLSe1_vIy68+_H(V2-nn45ULsY^gm z?aZYHKvJli3lCr}FBMmj>)0aj8&}JwKSjwpIDNV~Yz}Om-F(MBihWXX8hT#6mS0%+ zUa#jDtq1i4AJ++JuUW;LOC1|FCrhYOp}~_7U^Tbn**jEVw>d~f#x&`EVo5^JIFv_f zi$0f|h2TBo+~4Pz8zK>(-;@DMT4A>vcC}*D4Qw7N3;yMl! z!)p~~Pc&|9bO;)A=Gh{ZZHky}5EgZ*8 zcpjOIpB)9*$8QEqV_I>EEu>9H2eknKo@kaU57yn7#`+*Oj zJ>_1x;S(Q~zapV+`;IO0Ks+eTICx6%5kkpQaJLdM46*S}5(y9E+Hf{CZp8b$wQvuq zii*j<9tu%J(f5<&8NHt%o)Z?RaV9kQRgiC3-GAiKZ3@&o5amxQ3N(K1yo-UwiRnwg z$DpS`ACLbvUgbkiRI2p4m2ZR&PvUUPGzDqGSqmGngi3;`# z3K)N5|5VDkbaU!7_yBa^oG9}T4l6LlZua~)I;ZaP!S~@!9%SH=^|j$dUXis5ZiwK8 zY{uOtHl-tKf(KVFy25^~=#o<0TIKw_2CQ1tMEi}&MhjMl98t9cTA^c5Zhjg6?8__U zW5v8;nsZk$v5DBdY5_8F1dLvVp1O32bmJb}{S(8#bSeFQ2+U?Y4ioxLeJgQ})Adq; z*Yij7fcb&gocCn#QDV{&MK0sSY2b0<@M0Y9_Np>=cP%}~0WK;7uxEAb%hpb}F3+)8 znvrh26vimSi=i2Y8<2Io5A7D8eQE{zF4sdyw`q&VKmGA0nL2HTeCIpgg$o-k9pRk1 z1p@eTSj==Ojz=%>1?W)4&QnZ6n?Gm1yuNOOeE35jmGKj%!qtdmHg+q+S}$6(SibR% zugL}HohSd_XFekj!K%Lfw%g_JK6-XP=P#k}4z>qOUmqo_Oq089ri|dcan_@Em%cpA8|Msc!D>4)lQ@HcI~bm)qs+ z(@&RDC>eXuznD!~nz+w2gPQC+vDFz@dDfY?b$97%I{CT}oTn+02_Ew~mXq7s_hfTn z-fp*x7QjVzd6;c$u%xeOdjQL4*Vtl0^(Wstj2*kWcS>?xdr?@=+J3wu#GN<0gr~#iY3H$F#%Gi(#<)s zDs+8tP(-9e>B_<|apuqSdX*VQV^QzCKPPr^;_=0JykEbwi$;_ZJ2!f?e`kEhT+;sf zZVCcN^AUVw7iC_KZ@4*d^J*?`Ya$hSnbxAG*j|lr8?(CIqq>FC&&OdwC)j{-Hk%PZ z><>NJ8V47T>&L%f+{q80-v7(FNli5PIN!&+cJMpAx0D52w4jDWQVK4T395i48`-x; zg3--G%dM|?tV4_f198i5-UcN`wd)RFFb+A+a&Ikge4hXq9!DGRA{vqa^F86)R*m+|Zx>|Nc?_aN{53mp}W3T>swp%hgxEOB+-$ zBb@z#J5&*4a~hXXP3sl;4%4U4kYaS0TEBL^96feSCQO)w+ym6f!qOfwVw7xpeG@Eq z^e{(@%KZES`H%njPmJdv`baNQWGG(a)?05y7p~D7fBy5We};fQ$1GNO!wB;@WBM#v zxMYcZ^IPAP@#80H8%a%eLzsa0<0>rpFLz!q>ye1^B1&8jO)JEpIi`MZ&-e!(*YJ9H znWAJGw(k16ZlYnw7cff5#@AKOP|0QwZBsRww5Yt*DG>Ygr)d9YN75dy*x#uxtqNQ5$m*nZEpOJT6a~a~qZHPU;2*8BPRLCNT&*h7B9RzX3?m(j*68T_-zt?nV^`LL}I<9r*rO zeWQHnqo0s<8@I@BfB7T1_Wd8z(22SSwbX{mcuuIC!UgG1a#y0;59HkNHBaWX;tG~H zvVd+_b_=88l*Z)2e!f&F1ekxK)6`mzxMNK{dZ^1W(Qos2?TlYF1FbHE!pqi_y&(Uu^I5$#~zS^!Za22 z=$R0%Y~Prcnt`nDJZ(#P^)=VZ^DCc|kACFuBnNKnwQFCMuYC2ZvTpqb`2*bFd3jl~ zcHMfp_|nTQj)mA`4v@*19At4xBjp?b@IU|Lvxqg{AX~R=llQ#ugED2x48*+OEC2D` z@5(1W@fkS=WU~w;kzIJ{Wk?*m2f|)pZO=lo*(jL{q^@URxu-&CuXtjG%$vVJ{{4IZ zp;_nu^x4nK0}nkcnc2D8<2)7CKFi3OQQ!S{AN#mG^W+MA&2CF(0no+G(z29(BtV_m z4BJ@)VXy-^cOYXvo^vBgUV@XzI($i6Fvi`NH(NJR#)kAlfx?`Qar{O1EYZKOK z$l~|)^vCmj@s9n!SeH|MkCS_Sv!(|H^gwsk37oI7wn#F9CoAX2Qf<{1hw*F=lNV80 zLoQi6%K-_Tz*q)PIe9Y>j6aisvaIQv>#kS#@iCOfP>uT1*T0GOtYuPCIz;~C2S1Xh zo_Y$$HOq%T@-eBd9tBjg!?F|&BMu!pBLDK`i!{@^2AR@X$g2LwfBBMBBAdDu!jrMS zdbpuK{NWGFo^#JrAwFWnNEtn9G~&}uvS`Ut`S0qH$P|AD8QwXveED+81~$PGWQxz2 zIZLXBj*wsd&+nxiowq6hrOD)$R*d%?WOrvG35|bW{L0s~L(;UFvyqU7e&MjPhmRZs z4Dx3E_g6RG3fFrp#$(;xR8eHsbqK8FeN8E8C#b|ND3c%Hx=t3or@ zUE7~Ks}&q5z4B$8;d${2pr(VtYr^<*G9 zsc0;IAnTIo+z2i7wUJN7%>&o$xo-x@{51)ze=n{@8fxG9Vm-a5d*cE<`HlCo@pxY$ zVVUqzUg7U-qo5GxZCltoI_rco&jVtBoPwZVW9NWzb8JiqhiN4!1W0Y;Zz9Y z;4PH3<3DehfIhJ)e|wyzy+1v~A~(Gj{<~r|O}Te1DHURm%-Bh9<9~(%<-S*8OoS@9 zQMJPl2*>EFzvTY*nA11oQ;-R^vVX3=v{+SIAV5_aWBn)uIH4>X-R_%DNp&6I1 zR+$@LQPxBvtf|) zb$p(Wp7~f-vc~HN%hGt&laad>E_LgxPXDA&=imixhv>}k0~iJThE)Ufui#$-q3E7{lku5THJ(#--RN3)>UZ?-6rYHI zwuScYGh3wy(1`tDe<%F%=N`v3oUxHeON)OcY9Ffp#f;SB4A_@uT~L$M_^@)B*hn!K z#9Ybl*o4pBlc6dwfSb>lu~R4`wwT7g&^WCDNjZm)9+pE#>ZA@aJ+_O}hUgfN3Sd3i zu0>Z9F_wMyCc!I+sZ7v|lA=r*f`%A*$Pgx>wF2jx!aqW|-e}Ux1+BB-v-S0yi6+3G z7l8n=xpfa7Sp_}JuT5ZD^?>k7#uHNENWMQNdhUR>=LzrY{-5X{HJJs%;YW_lgWvSM zT$cp2Cyhv(4Mu8mbxP}>7I1K*Y`X8wEbsYF~S(GvSB=w;o>J(nT? zR^z;t4%*C}?gtgTiyMEOwQIj}xJf$T>P;4v@od%J6)-g$+)8RQosUBT5hM10T*kILEHNU)ax= z?b4WVqcLA>QPmWYL?da^iei6m_I4!OFc!#RP$;trKxPK4QuUcAfH0IrL{?sr0VM$H9L6k&OP$_Yiz2JCl$j-ATzgA za$sqvqyoe~1wayj?BlT0v*5l$ISru=*sf4Wc*LW|1E6REacoOUv%U+6J!&}HHjXJg zwjvfYwHGR6&CQOv;gvd(*ru4<+M&svM_5z6ALdxE?PRsPh(}5C1Q&rG?Uf{WGRGDm zi~drGdlK72Hpz(?MWP{?YcYBuy-M&R?tRZ8x%u>`M)po%8~J%j`23y*16S8IaD&jH z1?VJjmI4jjpBbPU7%XDvdu=bonutk0omILUvzp}hGc-(9@iOb5;W+&{Z<^?pLm)BAJ1sy!CYvU6cI4=v3cl2)Ku7I~^5+hpdrsbfc z6=Xwqi_$#(dJ65V{nQ!4Gd{q}kvEdJu>{e2O|uJ@tNf)^t}wdnR2HR8K8R0V)36^E z3Y?p!F>?GukQ_q7YDkb{hUL#&;x9!LZMANuKRYkx7{71&>!IkcmEH3=SGmKidF-bw z^gYi8z)?M9kV+eN*ZI}2ex)tlT99;Nh>=7sLT4QIa8}%63Lf{=!AphG?5w(d*L~(YCl;sslTQ?fFw)CxHoUx2 zjvlR%?7~9Xv44l$dD~6ei<;R=w#jiVjYn~9J+_(acmqs7DsGH-X>8jGBvoQa6ai(3 zRz08G8XGw44A)C02Dm#`Te0=vtWIw0Y~wugJWy33-e(#1Aq7wxHt;J0y(z%cM?x29E#C zfsD!ci{VdOgB$W6?s0?j)=bvLkWq z$qX>&EOZk~2a_fPOHSp=f;TW@x)pWrS*hS9+vV_)eNsMjsBGK3Umm*mE(pX%8ynVx z<#4=yXGang`33;hfX6`)=b?LAChF+f8j{L@(yjg(Ber?6>}?_#npkvkY5YRLv1!xm z(uDH9bW|H0I(A6@diU+e245Xrt++Qkuwe$tW8K#EO_WTz zH{Qmsa^!8yOH5`@a}_&INT`5kC4s?B>#C^Y?=sEVKRv>uY+cj?zyvquXL`$kg9>|s zg1-f}SCq_p(Kus{?+f)YF{etV!pEWER`iT!09yCRk%K0nzx>57$Y1WcTV8&76|^aI zD#4`a0;EUJBQo4MO*1^^v zIUtAj??l7!I%z>`LZVZ&*49WX2sk54?+;I)<4L?}RAss_!CbQXMq!Uwe&C2`Juy1S!MYS8cb7@%4Y9YuTC=1!@qs*vGBDORoh|j!Sbq?%YtWz4WIE8_4gl8mb@YKRk`cp);XLvs2+7t)2y3(z z1%Fa6m=k)Q8Y5Jv0-Ba5EmZPP%F$6l0 zS3{S63Vx@;k)DqEg)yTRKKm|%pAy@igH*pAn|o-EY_utl9t)n_Kd)Sn0XV^9%%&gv z(3GS8yu9EK@(%WBP@HxAubJzj_k_2B2c3D=S^B5o-dbPlgmQoG_GUXqt!Q&+Y~6KG zRBTt!ZU6r2D#p6kx1&wq%TfRfcHGzux%`rIrF;kon0=z5 zNw>AZ1x;5lEpAO7vg9!~*s$G9r6?Ba(-W1B(GeI`a(x~!h%UIa)6+8~H*=<>cht+i zotvcu3JJIHCqDXjG8Br5wWGuzEo0R{J&riy;?Q1q?K>>H_a1=8PX&W+IgZIRVBS;b z40p`VhS){>%Ri*inx;~u8?_C#>(dH+vFQx{hAbmt{2UlD_{PXe}Q!j+X3f2_uMP8Yv*41 z@OB_NESm^|mVGnwG{%C+%ev$WT#gQ!JLEuMLA7Y)}a$!{b8ql*3Kl&J~ z_aSod$Wi(D2d|b9RaI!60c9QhYSZ?8vTnmxDFZK9ykNEzv&o9`0zY@=jSDKs4fXA^ z4g6-$kz>*fcRcI++2+*8x5;lfW_E{5Pd5f#R$44$M-7uA#NZ>NSH6Z%zAQDevayCv z{}#NT(l+K|{So95=@J@7d>4Eq2A#t0N7}y65(eJfudoRp9WCAQP55}{Umk-6StpC; zPnGFY#~^lM$FyOiHnkpsaHJMS3(+(??;bL`iEf}2_VuJNqlH?UAww%>$YHp2)~?$j zKl$}-a_-XkvT*iTJ6{Ra{vLRICA8S>h%4vI(5gzQtJxwg4K;EEv77W1XqKHVI?pPy zbnjZM##%<2WoYg7V%;VYp-kbdf)YcctBnwg@ECMJV@=0zn>TKkZ++`4XleM5GH32$ zKtFz1zVmP2kstimH&uaQeBN&rj$uztt32@Vi*f*-hO**Z8CH=m!_XEnCp!&pH@M*u zuj2k7;dFID!GLz#0F38WxVw%(tG|EkDml{B3NVdoIdkzuDbSroBB!$9*T=c5B6sHJ zrz3O!NU zx;4CHNmnL_qYT}6u8A_wkJ3_dWai}YX!BVu`w${~e&t44_sUM0Jh@7ekxuO`}&Ce5!ZmKm>GrnWk*i zHb-pAylFhaC8I^>w~#$(_>#fl`Dr?lV}jTA8z+(sq=8CW$sB;?i>e!*@eO>nyQW$0 zd-QotV7TO>r7~t%iH^0stq~v*EwBm!_CQURRyH#AmLKjkybU5%V~C5<@#RL>FCC?SdG^%31ERZGPLB(%p9q2YJ?jZ zRxf^MylL*d*|HKX18cFVbKzD`f;<0-XI9I?g);mm_S!gC~Rf?HZjF{WyiR6|H_*$plD&|$gd&K1yZ zt7PWXQBstNEyhQ1NGuvH@jxrO2J}#9HkwmRbk_SxN&P*ppO+ixVPW5u5!Z=dHQx0G zdd(9b&EMpi>Mq5Ts~Q=hMGOB?GIMobS+`A|S-D!KOc*80mdz6MEdh88P~BQ;0T$B) zcQE%jk~X^;8#ed08bZ33rrK*CXIiz6KGajyRuB40k-VHV$sJuG#rcJ@=H<=u2>9FK zV+WxO6lepJqevEA@#J$-QCTJzpTAhWca2S55M+&@-4^f}xQ`ijVuGw10#qqn95>?T z274m^W3PG*OR)=DCY1gR0OvGHBarBd0Z%)3{yg042`Sz9y0$JIi9DL^dti2Cr$dRy zHDLfH0~=@IoLQRGa@W02pbhFW8Cg-R`v-QBytICYtlqRsTH%=+Hm*u0j2J3aCAm_V ziR4{`71`y9pmRQ%I8#q>BBHHn*`*?zj^mTdGMiCGHKFyS+a1fE>r{JqODC0SZoMa zxK7fNA*hnFRgh6Qgn3~2GrHU&X^ECD`zA1Alpew)0uZnYfTyHiSv%jFXvL+77VWXywL3P5aUpM92Gapk32 zSJ(lo)O#UXZg4ke>{m-scQ(^(@k?rhsFM z2qigV<>3{t$TsZt3(j4HFi*LPz53=R#A+Ln=t;;_3t`bB?P6XD$w}`Td86kjbf)VP z<%MKEyjO<~?~&jB@J3m+W<6kBnM777TeohNg9o=l@n4PjY>qs-0=x&pZ2Y)Ua^F2S z%Vn2bE=!jz28kj5%gku!gky}_`kP1hq=y0a6Jejb5i`%1SJ$tX-`#it7$0 zvrk)y%<)ut`nfgWf!Q)^>R7nO3nV)WFt`nerKxVeG#s^3QfA%i9C56Xp_&q-(*_2Y zlA0uy#VInpVjK)FE>$4kKf1)ez0AVd&5xj?KV~c?CZo56kIu*PO@! zQ~~{-;J^0SYKA=5_1E=-9RP1RksYHXF-W3?J9sVFVPXi#DWcQA6Vq$RmSjvTF*fBC|f+v7=G%8e~?)-=E%oB@i8exzhCy} zWws+b6p`3lA*kt#YXdZ62CVH3P#}Kv)1RV#x={Y~yWb#tob}j^vVSL%$vRQvsLBH8 z7=++7qhQH_x!$>L7?*BQFqQh|G$g#$;x?U9443%nGpbPAT`AAJxK7q@-6dDQ;~acX zE!{SkLJ8_U20|JAJO7Q{#shi2L==BPId%X5f96RtRgvf^a!%L zH_9Ei|4Bag&;Kl+`osaBql?<;F+(Id zbE-UyxysKilvz_I$h0v-r5)kE#v^;=2$D}VVHL^)EqwAeUoNY2uP#IC^W=NV1Tmig zae_;~12Oup4;SdqF{ff*oBG$& zD0UWfxD+z7esKi#8^#)9B($GMy0y(RTk-VEGP-)K3g<2e0&;;bCj<54_ug}#Y+C=CoPNfcQc#?)0(`=R zk&tz55Q@ii!__w+BOEn@sB?lqMDqhAtrZmI%bj=q1uX?Hfg9*tWM{%+)u#7+Dwa8! z>Klz!*(7*~KV~=n>86{cY)Bd6Lf_D1ckSLU`}XbAW(LKjr6?)Om$I@VS-*a*l$MrA zRYjFdfrnx8_;K>7Pk&sVeg1J7KXxp%*d}CPTQUpdRGLwUiPbxL3tB@1ybCP>{({61 zXuDb(PPaS$(wNcOgjiAIQMkSGWZBX=a_=K6<-P}>mMbq6T9`6Ry9wo*t;iT3hfMM``Pol@Dd(MY4zjpMz*uNe6hayqj6W;)RL_|47FD+0 zed&2?1wO;CfBkE@;QaID)1UmfR5*?E$aUm&gE!6T8FTQGmJlopnOR~4X4Y!9kh>NK$~xf?k- zn3oP@>~gNU@TcWcj_)Z4(vKW0eHk`yYCceCBk?T>)_qGu9r)r=sYZjBjO6LE^63|0 ztmMnINh6&?6`Joia8LETwHC9vr;7d6A&CVug>3<&OpO|n`=mdd><8_OeY)Rcv<9<$ zeLlcTYd69D(<1M<@1b08^!?R z6^ui*-E`GlnK^mSefL7x7QjVZg!;roNYp5mX;Y@i;R6S7uROFCERlO)-A_cH-edKL zWzWu?NK$K-0@MwbqV?cSH~mpgM>{|TLII(4!GB~j(cDpHMcbyO^>1GO`EpJgo#t-fd?LeyMMCGm_AjWdh7{V0=HLTehKEMO(sql zEsGbQA;0ysAQE6sb5QL)&61bOH|7-DF zpTTT=RjYL4>|F{;dc4w+D?;1^hNeb=PU~8ZK)G*(P#hx*=Pr}i)~|!|odXYH9e|2< zA=hJpY(@C$P{RS(4&5?x#5nCWe)}Ew$^ZL@f6$PZTA-{6R=_6XDSR1Kxr9{QpMreL z=B-i-{DhBu=p%@ux5L|4huHTTjcE_9ESEcPze7$x^K5Nj`t+ku$^{pkCnb3Wa-i;r zT>OrA$<4RiC->g_D6Hsr!X82SW_!I9WP)!ZS#8%2xXWSSa8T_S3x#qUb8vKiQQ<;{ z3Qwgr6!PQLNwd$EQR_0$wc-eT3;c&T7v$AP5AT({qFkAK+H|>R#k10c@Zo}SLn$r# zxzqcyqWjuF{k%>1umrr!3FzRi?1kz1WBjN_Jc%O2tzWf^c-OiI-BrY`}si_%Reyrmq5wQMEldc$u!38zz zQj72-c5$CiTGXLH<^pc2mbq=5FI0+ey>cxEl5#5MC!dQo0c zDYdA*o;-Q7y4!Z*exuN9n69Z#xZyf6{#G1M4LlicxcXY60olf5y`yD^tG~^NSYyxX zgj0%+l!)lyXz^QXL?s>-?2PRj(en~*sJ)yyW3oK)+^h1)%2(yIh0~-CL{(qkgnCEB zx*h2ex-d5@2au)dKtl8VodN{mH*rvA+X!uKXdT!PO$pH8oBtH+QtWfl2M>7b&p;di z3YHmJ4X#r@X-Miex4qud)&;Ay79~W0-$e*Z%ZzZHGzhMIBwh}yDhIB@K9osz%7U2_ z)IHmb&=spg$a9QV4XnsqwC>va*35$FU+ffn@DK`5#Tr1IwF4Gn2N1-P!B=QGzq0yO z&@@AL$FyvUo;)hYxjDI7cTE~s4;c9$Dn@ESds;>uv6Q`m<#>oH*-<)Ssj26U?cf7U zXXUv=%7@4yfGjnEpG-lI^8I*Uf$zt``<8;78YZZB!*kHu&M~9Y8Tdg1+M8;bD$Asp zaSnLWNK9ERc$c`Qo&$VjX_aadOlI>9fsD3;UPJ@yVMP_k#;kOft{NHzaVQ$$KF`T1 zM48!mdGxVoWk~*cGNL?BYLGZrSJ$9WrR1$tR+ybmxMWmt33Lec$a)d&^R|q)*dNQ} zBHaZ8tS7RPUVH}Sd|0N3kLg9^S{8S|OI=Q`#bUw-3ks{2+LMA{sKDuxZGf-bvgeR& z-c=_@8zKCX*xJrj<$=i3kW`VEy%VnLcEH@B=5SbrE@B6SGX)GsYRMKVQEdK{X!zHE zX3gU}0?9fGA;&l|EJZ7m>Xs#o7RfEQ+$tl{UpEIyTNK0_HlPH6tBx0&-e)3)bL8ND zMe)hbLUtrH?N)#g95`@9-h2IgY-nh&h^3@Zd!&M`PIiU6OIYAKx1qCvCbhX@k z^UZSF>C2J1T__VK zxT#Z)pzD|`rRI+Eidn*Bs|Z;QzInUMK!CfNfOHC)123X9rZTN9U9pEW!xyhZHBE3QACPUk z4gm+?h_rymbRf?}xg+@ok}oqL{EG6oOEWCRc}S|vg@xP(p+d-BBU-1#Qb`XoXg9a9 zU-dk|E8WZ?DK0LQnKP%#t+(DT^Gg;2xeImH&5iQ%+7~rOtlgyW6gk;NNE+L$#sJa0 zQUEl?3KBm1#3_?BSt}RxM8BUY!Va1@;6_%8a-Z7}eyai~N(z+aTkg1BZus~|q^krj zYJ}tJ>#MbjqY$3Lef#zz4xS^!Mvhc#I1L6!F>n#?fAm?*;SRa_nyaM&AS$G1#ad9s zD3*trBxK0Yg%B}D#-Y8!9W?}^a}_L2qftTw2smv~uK@CPFdBq6o23$7&HZ_4a_>Vc z;FhlhFRTYp3o2b4jsvg764Gppy)ZvVD$5I`v^WRZ@_ABToM#?d8XSZ}=D!R%s?p>F z9|5ZxLwL559wd+=x^HzVROn~)ZVa>sO2_7dw$2I zm&&S_UXnZR{EJjE2IA`X8k@W#kOZr~!LktXqNUgh0^5zO^NTOJNER<$4DGlX)+4n? z!}2zv=#sI5F&S=VO%{WO3K+i+eBcA}jc@*|eCqH2A7a^;N)E8RwKEou*|Ke$3NaFZ z$Hng0z7r6qhvW}`_>)YVHcj1hgdog9nU=M&q!Zu31R`j`J&2i4(KFN$Ca54WbvM^Z zIT%P9$|{)TQ86T6GDemu`VnU(1=K*rrW45~t<4S6P**FfccNY$R(>jK9fy=>!7V>h zMq;fQ=QhEQ|3IP9`=}81#9<^7fdbVeom0Lm25<0YodHKPR_aIQYWsr<{6QpZKDX*M zIdHfcl@3`la%6=pSvp$^QOKVP3y*CU+n_uf1aOiZJye5jRxP83Rcbh`kzpKMmi$YT z!3OZ+KIWLHd=p9QA3psl;P!8m_3K_mNmT`)7+4p~5FStoK{w=|jC-5ef&W42CyL;~ z!$;-AAN`0b?*u4e4h}<4G4!a)zx{cB;@{KT_vuf4TE6?A|Ai#8qq1!IX)5?t&Zw~6 zq@_-~X$VolVbUsi$E$1C$)9iivs`ic<>1}r%A-4>I4HCl2CEs`?$ty1D!;SedlSsw zI8g4$@TD4U(GY@Og9;Ut9P-==W2@z8O$}qmVLvbl*5)a`%Y0xvpy%d!Lj^Re)sm13pO&00 z`)eBIo(G;s=5ewtI&G?q8Cs?(=IBC-*azwf8=)ytFf+c%?=eYPZ~^626e4EZjE&T& z%WHeU5U+w8Z~%1E<5X?}5{SpL<@@ja`*&mn;O+kP-~JUYV?f_=L1_hB>Y@43bZnM& znO{^YAN%Mh<*F;M(oK>DB$aOX#_57q_5qDwwMq3b#co_)h1YIIZDJu%%D(gMZ_8J{ z{w=xwJ?}AhJ}|Hqeo)H`+_`kkxFKXAUUdES@00V-I|stB5lJQ}J5!V)CVDaHjxMPv z!32q>zwg7x!M}xgR4cOH0eq5~t>{LKNp-e3>l@lNlRjugXTS?lT$m~&Mv_@2L+fpa z_S+!$-?s|ZLni9GC(3BlS!4N?&@|IrwOkX@F|Ncx;e0Z;{#)`|YVW8%Y3!5AE0OhH zTDt=|9P4BZ_R>2pn2e5ahGfu!N`@9fg1%HngM`xp7$?{8E*0(D<2oG}apHmCv360Vr&TYSXo zF=OTPpZ@}|4i-Up10x%rD2|^XDb|M8_=*4XL-wD^SL%?zGh@aK`R3QZE?@oH*X4&l z_%ZGQ1%y>4oC)n%LGP6bIWiXJuv7t7^kXO`oiuTR8XV~e3D6rrv_!St15SgWvXF?U zgcXYLaaudBBeJ-isW$FWb| zSOKkxLg`3E5V`qJ56iI8!{x$rXKSH7H(MPxSSzyq6qsJIoG}4xenDJWOaL2*w@?Vt zO@c-ke5Go#k|n)`H^FhtOTQ201|_)pbI&~|H{N(7wAL|-flC5Yt=83;TVfXyswjlu z0y~D7^Bt&@o;+!iEM2+?gpP6p;>vP^X{?+IfVrf4Y)lXll>%;l{`=rVkIHk;KPTs1 zaK02FK}|g|IG?TMEMWyOdCt+7T6~86_HqV{D3)xeLsCC$mEqfpjBYq~)U+a8_EDj2)gYlO~K% zsMX)zc)y%>+EO`t*#xzwskT{et=)&%--aULsAM3Vv)?;m@W40M4A`Rk{1J;!LgB-H zN`tp%^_D|&*Ml#}HJ6_yldFsL*e(DG)oYm)la>%q2A!rr*=2(q+rrps*k;;^33SJt z6NTti!RIq^Piz)Q2i|)2kh;XazUihv$U_f4pb4IQ8Z|)NAL+w~@z{Qr0(0k{y_$47 zal%BY2Fz$H1QN5x%|*-d%c$WJk6yik!0vtzKJb97df^4Q+e@@$vkibN>_<-_P7B&6 z-?E0g4NQlE{n?c(aqsamq@)DiOxA%jsg;I@dAZ{85I})~P z3AF|}1^AaO)X0vtR@fP&bl$R@fdKTh{@4rIG3HYVS!#Wmt@Fr+4IAW(U-*KYamE=k zYtCGai>g*j++FrgRW~k_33Li$mLyA!sKLL0&Hvy3{hx59d{;)HhOY&vKePt9am?zD zTCx6qelybz#;!MyKmM5f?svbH|M|@?q4my`nz|+=oYmm37RW-CiZVt3|AWXV&pbHRs+uXUkj3Ano-X#4Tq#^NU>aqvXj>~Zj)ON?F2x?QYi+a znT4diaCSsPn9;%4-|!4XtsL(?Vz;H;NCJEM$!F!Amn@b^5YA-Htq8#o(1=2eiJscF zQOlwz(2Py0l|});t>z$twQPy5Y0s|*Fx#YY54yxS^KG}?F2DHsFXa=T_$0hZ)lj0r z1*vGmYG)$~cC;fOpmI%1Fa`9@B)5C-yX$%2)Ir&eBaJsQWSkg1?*B!q=KuD332~9jv9Q@G4+F*dkc_ z8{l^;Xr}16WH_#yMil9)XD$>3c{|n~%1B+qKHyel%c41><@ zZV0!(G#IWn`2bcw{;6-UfEH-7yX0Wa5qaOcE|g46YFh_fOAs=ht?We&w=XqT?eED9 zZ*xspAl}qSYOwl8Xd5m`L~x(HxeB_={Z;)&p;i35-~A39XNJn@r=KnR(87{Jdo;49 zo3VN6mLk#Z*s&9!B|sX&Zm2nSM8|#x8kFosec^9@{TuoE*S=z#i(0vEAZqQFfE5&e zL01@w^Ojp~k&nZzv~}j^N%Y zfJLQiz7qfo?QC~SFGMFw=D7}#psKoKz(<@(=iKDAz8uB6dYJ#9?P)n2h4p`7s zBXM9dD;g9@P(`wPT+8#dI5vreS9a3*1o^Y4E3fQQp=bE&x4-?JoOAYhGH%=?IdJe0 z&S;lLr{q zWwN~f`bNDjjgpBICIA=WZaHJwQh+C!Afwli7IgUD!Tl5FabZ3i{;cEIE1DcI4~AbO z16Z7Jq~~?(R4MrDQLX$ns?hpo?F8X$Nuuy5*8H zrz>wtMO$9-8OudH!41$0A-$mjnp3zDoms|=AB7&t=2~q7SVuQpOvI_CzD13PMB^9N zYbqLQRRL{_#2Pql1a}H1$uD*=ZsX(2tPb1S4 zASKRvwk%4k-}G*y-mK%^gtl&cH})tQm@wwcdc%@&dRIOf9`iCuaLk)gY75KoA&n%<%6ezU4kka5!Z)|HS=0p0P3Up+T1pdV|OXy7GtZr8&$Y_o@1685wE-=;2E$k( z4=G22*$(gngwAN0lW({l_CY@8`H81fj)l1VOnznCNd8M}w6+djIT%uh4;}zQ*J23h z?U;rR2=S3X@7keNBhX|}NkXjbRM`QnhZ)nSIl=5Q$4ToUUgl@2F999@JSNoo)iz!e z71ECPup$L(Xoa=PkoX^WTU$R9_4=dhAq|fN6*O&&2|f)z+|+(pQUM4vW%6hkHXG0o zq(Ei>1Vs}AowDHCIz$)$B$Oq;p#r)YF5vpwCRu=l1io5t1Mc4ngWPWKI5-L~mS?`Lw>mMz<)qGA{Xj=5^oL!j*=1KHfp>h#t#$4Kp! z*_H&fU%h&bOow&cgoLkQAn5e8OxcS#^RlH&qz1~s!2^3yhddoMl7t9t(j+ZzytU}x zI;5-;*Fq@40skv{7zoe0XvHc9$7?}{g|lg>QMu`9i+7=pYwHKg8BB(2Oae)!D_oZt z)40&z3g}I zJp`9+CQznwbkEh)Bkl-2IGr`jjJGM86-unY)H`V%;s0tR>7H^{AceK&ANdf97tklR z>C{E8iC@e=f!7Ew+9JU4=A(>q)8@^x2xXk<9qG~l7yJ$g!!#s7?cBXfYqQf)%GnHF z&;(&l7k6e>uEGHCKX?%QXtCD)661iyhZJT^1Pu!4Ft6ji>0Zx+#dzrOVJRywM~I1a z-6^si##BB^ke+&Kg(aNA!$97WiFR6Tt#q-YIt5-st^pAY8_JJ7 z%Nt_9>kpr?>*#-Dcy`bvHVI*kH+21J87qhI_xBuEAu_PM@Y2C<=5@7v6Baao=!AjD zYR$~0-x~1&n@(#aZlquukf#3YV$j<58V$eqi`ak86i*oY?}Bj}UXd=2IUn6x9O{p`f^FPk%iYJS_^emG^D6nl_kVaMgTCjBFvw zn3`pJf{0LZK?R~7wSZs$+V^D#2<{V~{CmAOlOpH}q;<gKwc`6$F@Mb z1sF@@=KWvu!6V-;GmuC;N1u-ZNE52?RXY^v4rFR4K|56(XobTxm7j(|ip$GLf|6*Y z;9{t5BYR`OP_MkPaC~#=X@tDQzc@xJ-yiz$hvW-i_#zDFB$+yWrYa9DD2Zg%2l+}H z=pq$fw#6UP5{VndJI` z54>N#@vVOY?RDW=mHK@9_U%#rMMHypnYi2=)~;1Ug$Zr-5Xfs@T`T7SB6K_w#n^kG z03aZonT|j>t2YD4^SeYmB>218zP1>4O_$3ZV_=a_H~=hi7pH6~_i@{WrjedYo}+PS zf}&EIgZrq?hEOXTcu?7>Z;;%~T*UfYQ15+MW{$5?qaollx9gg@u6xFr49B-I+|h5S zfX<+1he>FJYbL2Q8=HZkAb2w73vuf4eEN7g!JKHgj*;}##kxdiTX#d?Br!W0?Ie{@ zGElRI&EAbti6p|mp#;lXxj_>qLJd)`T+VOM9TMCSiG;M@)WLsg$qpYrTt5Bj8|1s+ z`!95``m>_$Fd2gofhH(jdpBz1c+X^weftl<$~_<-_~1vS6p35>&etQFaJDjr;D!te zY7#L8@0{7Q<)Vu&lz;!uw~=xAJjw+MEy>Iw2YKUc>CmlMvQ~!@@dd>`Fv#mQ6Cdhl^^ly4JLu)m;K>0+x)%;3JZqdxF+A zJu4(2C?Ci?%S7C*0RS~b2_(3Hwwd=?K_^5OgD1UpXTUSWsGv12!X2Ip;ZuN`*B!=^ zT@qwFJvPjoWa$$3A2yujyuh9!L)SWAT#p8WCMEG~DzW4z?5$1$viuF*ecF^lnV8bC z6@~`(yeETzB^l-|bS<9_@S*G9`)(v`{YiH2+$C8+3-jIkG?SXKaWV`e#eg8 znw`61p$>h*!G#8(r(?qBp?St=r_TyYGQQk0v_2Gx?YT z1L4{5f5z%*MROb!g3UlooIh`#Tzv6GifNDo-a;jags&!4$TJz{ZT@qAt!ulc6UsKi z7|YzefF_(xURdZ%$K$|@)LPbGEUB!UJS7=n0Zmk8f-+=?>odTRMshbw$C7{<0pp+! zXpOq6P?n5~MaSZpo9>5!yiSpT_U3);;DY)sw>ZVth*3qdcHMfJG^Sh|es?A@lbM@c zby%G!^pV1*$dY3nDf0aD=w}SrNTN+KCb{WoC)z!}AXlC}8=#eK^85RqlZ8O}8IIbw z2cLRH&YNBZ%`#UvZALeRHQX@h2+j?a3he11VFWiwJU37I2(>VaeU%lp4`Ngk(p;|37+6~GM=eZ0+8xO(=a6wUVx%S%k$e1x`EP}W- zwO=A7k=R*=p^Y+B+l;!Tf}5;2LJJaI_`SbIPeJaO1`gmIeO!sKdU_3|g6NjF(rz9J zOCl*33Y#_Hh>{0?GZd)7*WcOJBuMqAw0VSyY`&N)mBTr;&{&0cf$)v^Zgo&*3PSP1Ep^uaPG(yOYn&VxvF zv7tkU0e|~5$j_)yOE)V6;~~~Ha!*Jf@ye_?mS7?*mHz^G{N8c#1+wcs}6xsb?w?OV_vu`ygm?DxR^^DflS>Ria_X5CPR2Ruj>~=smY)N|?umLpj+T z=M)KOa~XtR@diI<>Lj`4wTI<}HE>Z*8iNK9HQJ%1o$-KVxLq+B85G!)woUQQ)ZfLb~Y0cwg zmOD|PY6%$rh$ME)EjOcw^EUbFm%gM3HCoiqT1#5bma&phKr75B2!L7O{|UsOrBD#Q z@|CYBjHf2PFh-3Sx8KBCjseb)ig?ihKmq>Z7r&O@|L%X~V}SB&W3Oz?i3&LWU-zFS zZ<&q|Rjqons>8%|{Ga`OdhV2-LYVb0u~Hh271^irkN-5 z?3M$v5&g}tTsV%}tt4gU%U_<|By~voeE%6UK56Xu>^g%-~Wa$`Vz>0AlbqNP)*;~>4j%{q8d+vGp$&Y_5 zQ-QmE!G#w=-(~5Uc?4-4vsBazM@8g9waWhe2j#c_^FLC1#pQA#Fd15KZ;p@POMDTV zA#7~)C8)!Ew>Mst=vv`WT_U~&HN%t`gk33!lK?N>(OpOG4cM@3bdHQhKjwqYU2@x#+a$fP zN}7QT(+;BR0%93NvkJZxg3RNR+8;HJb=uc={GG(XO-+G#-+lMVV~;%sB3iE2F*B?e zGyVf&*}HeI3g>zd*-^xYR6b)QS%}R{Mk2*iPdqM9oVH9ZJpUYNM0Tl4M;xQ>DD})m zHAFuy3TA4oj1~R)&wrL`ltX;=D_=swOQY=By;p_)?p?dJE}F?21R8kY!3W_=FOj+P z=gKg|slSDD-t(UK%8M_)0A#M&+LDeT0=3&!C`So7yrKIeu*1X>k0=rmir40YyQqhN z#zb~@qxlDvj>e{Xl(3C=@n_B#l^f?`biRm_uiG9ksSO_ge=`FP=9L1${R!yb89kc# zH527ktI^En?uS>()fX-W`$b|Ecn%foc39I|2|{HL9qabgb<65Kjq*;qyHm2I1}@G= z*YB01hy`DH+GN=Uq`Tj(SSQsJP>GV(Et~f@%2If-GN2@Y3qoN1NU0htMab@8Cr3SwL1W0@zc0ai;udz#T^H|O zIu3Q(+ln#gi6IQ)dTYOGylZWPrvzcyCU{Gvs~Nc61$uD7(NVvk?Ao z1axGMqAm%2FuHQEsV0mqmvhgUEB8P2w2U7!4wm#dxN2E|4|lA(gn;Hl;elfhwIs=_ z4G_3Fd9r%rMrk~HP=*aDk==W?%lchI(3zrDt~!5-+^^B1k|~0tgte3)!56_Nzf> zDAqylK=wG}+cRd(k|!Qt0le+0k_k(?rlwXV0=0}uAnX-ghWnm%_F0PjGkxYXHP-dn zU5qL0QHUoj`y#3!@rU30>xBhxPfIyy!ABGSG^zzx)mnWZkQbvs&Zpe$Q%!mV0;#bjo z1nK$PwQCo=QnhF!c@ZFXk7+l)L+CqAoc}$0_Mm)fx@<;vITOFCf%m-+MhG*#>yQyY zZu~^aTk({1X~)RpfeB|``_u^Npreyd-BX9WN#M&}gAib^x%1i9xxD_movs$pb4l$)*x& zsZH|!3+Bq~nd9Zb73l!7KO$4Bie!3zx|E(VSsr<+QSt#BnFg1Hxpp9ySqH4(eGK?V zOAh>E_E$iA;^KKk2czr9XNb}T#H0=@nLs0V5EyF}c`Qq^_V3-JT5d8?(sWW%6y!b~ z?l&fuY0aa?Kr9X%-pqcCO5B)73SA0qy2#mfE*l7A+qZ2;8Nf&t;#)Rv(Hd;NFGGvY zRCKQDgp017WhGD)Si`9P3PpvnAe<(t1zd5x5zmPdRwAJtkM(oEXk6WtRE_l=Io1dO zhoiD!-c%W0Rcfc{e(T|HdXq|%!C{^B83-@E7B9OmHFj#b2{{l%SDd?0UfH-)o_b+D z>V}8Nf;lh@z?=zCp~+$tq@b~5@Pw_~-yo&wZSuk^Yow}pC_3on$a8CVOVyY{nN^f5 zi>J+!Fa8Xc<0MoPXn4wUNf8_ zoxgMMQ9RdpnGWSHlQB5D4xMY;UyFota!6(1#5Q<^|BD%L8eBw4qs!WX@34A9At244 z(uB1C(qgnSMaXO`EYQXC$E($(VPpJGdMPO=k;6>~!b$K#Y3}b-_NLo@z+75-Vd@XcA zVF5bL&6b<~bhFHvQ>>x1dMFAnz5KEYMe-PW(TJS)$}6vcETD7%qB9LvHp6PHW|%s4 zn$mwJV(IEZ&=RXyuc4=9pT*7V4uP^#TvDu_!OJeY9BnR!hG189uCe)B%r4VO$8w5~$LP~LUba;Y9Q1kVo37jSkd0I|;i)etEQ zVIDM-VM9a;5hRnjOll&DF;?ArEM2-(F2C$DdFZjnf&5dYB@bF(sF{`)uV!X9pPvGlxVf!B^V)-F zUVhr?Af&0-%pCww$gmicG6!{psYVyve7@y3#X7}9Yl#kLLN|*j@2GTtiKzzmg8jZtSRU+Q)J0?>Yn$? zjnAzd6wvYL@$LIvzuDy5=$2a^{d!`QcTwq#nw1R~q;em`o|G>?CAHzjMhHFmHGOlhfod+TNZwf1)Hq zD0Md0D&2QtzcS&B|I%`1?3rarRt^+Zx`X`2(+(})Yp=Z)b;YkLWN+orp-Kbfqjg9U zW9D=R3>Dw7g3m{Oa|o{S{opI_eCO5bk<^-MAP5>jNmL<76u$G_vEF^%wer)S{#@?8 z@2@gp;$%&bQW?Y?9+#Zv1uttdggnepW6-*=H_tY%D2_lW!23i3IWS2Ro-=ps>gPdtalY}6SRxlYhU#QU;dF(0Tu|ML0I%TIpxbGZuH zlXQ3S-oB@FL|8jlm1b)qtNrS$uaVQx=as+*$zU+bIFyMv5D;Z1u>!g;0*gm1@o>!J z5m^jTad9GB$JI$=zK;P8wF!z%L2)?%Sw_hCvBPEbF!Xpuv1+$A+3Z(s&AgWFXfaM7(3x@920wr!`p^x}GaN|#Esn4CH9bSce2!YPhm_oEhcvg$w$ zK9dhMLmD)Yo7y79sh~-?Z<{segqYSiCM8GmAgnW!8a1)35%u^5@bGm5fh-vZG48DR z(+cK*^C_(GnFbmS1&^^G_S$&`mzHl8DiePFt6$0&zW62iGr&hu;a;YPON||UGJ8=F zvbz>(o4DLJpqKO9xpR?hi&nA_lq}uU3K6<#14?+1=7Ri>$*q$BAN-Xse+lh#zb$tI zTbm_bRPw#^R?!Y|K_=E|=_nMFO3>toKJ;OD@Wx0BcvD7BCe3y&^D+a?h&t?@4Dy3` z$AtMsU)FM80y=D2d$KS97OoHC&v>?$cHojDjFys-FC#_|Rikmz*b!1uQfRzUBZRgJ zs)y+8xOGv}+g*78yyw&lXdRpv&YA=#TlOB5Tkd#3W=x$dmtMk5W2UvE*%uO>YTN1{ zSX;2U3A>h|jI|T-#%}F=gl&(Fkxn5DE0vB23KVszV^XQDGMS7nY{r?@%2utwv{(L) z>P5KCSq1FnZqWvrYTmwYle_BfihFR`K(O*jATcibT_Jpa6g1Q-Wm zlMcGRwDf{h$tyCBM8*Mz(bCc`k39T|rG+tiUz2S0q+x;z6H{Nr z;5UCfA`BB=ywBVKEV82lQHTAJky|7)X3v*#V=JT-*tqbKV1uw#rga`Nd2EV}wxG0< z#X$ibT!^>zdrJheE~6`p;c-!6qyMwM*iH^dgz(9=22P29HWD!h zZDibdgRqTk+jmIreeg+{KWDZqTQ~&_mgR*_sPk_{Pf=`m)qW8tWdaq5f(Slcbmu7O zIg{R7K`d}}Jn+R zVL`>?EJ-6%W6M85>qf>dhC=AkqGnPUwMx37Y1Ok1k!~xxV3qzc7n@P%cr*IFPMb1G zD#}VhOwhoc8_Xc-4A>+*Mo(N+3>bg9`R1E}om&V@?+VrYsd>}wa_G=u-Kz>{f&G7vIgJ34j8({9H;4K^L=kH*^{0UVFu`V!z4d#3DVQ+43` zfUr#?sZu&^tTG%W360%b?Amk^uha6!x3xqXe7L|2M3*}1y?f$?b+Ucue!24E zvt`@}xP7rV8tdz%4fIGz#}0-5BqAXdd?t_bTqvw?+3}t74%pdgY@VR4D3Mcw`q(rm zwe6tS_GB6uxQ0q)dcTbKSumRkU&OX&n9X2{&E4BG2{V|WnvGnRN6~?f`6Z)9jX{}H zxzcogVUfD6X<-xRn(rz6cxMV$()ot<8|C-E{a^Xgm%peqPvNO;CiVY4W6*y3{Qb$( z?)~dM^1%HMAWM4;=njGjoZ-c+_FCBR;6uu1*X6x6lhsAxDeDHnm zv&?NBBaMzIqFGp}Pb6&PeV>Sj^kr2C`BV6Pkbv0KNIjV3*(mKTD$YRZ>OOh-#dIlH zFh#NnbfdE!4LB>m;oMTcQ)YAWg!Cy9&>TMp+m4YVY=gz4I|OyhHH~uDqc6zvg|lVW z#A6s$#N8N~EibF~u;f##XRQ=Y~4tLias-k+fzV$jg^FV=bd)~ zP$5g*OVnU#HIEoI7C;3DbPU88e&(5H;GU_L!-o&TOHrd>`{U5iea)+@n!?LinkN=kz=`fo&zE6|VaUkaxuU)d&WUf(0vUv;(= z=4K#lhVU7h9(2O8R1X(;O^T~)Fj#Mul!*RrECh1VY zjS3;ow)IozUbfxmle84$Ab|z#c?vDq-~Ij%NIV^;$)|@=i(EBqn9P|sU(2AF5Y_@A zO$B}A$dR)Dz&`Cfl?K?`X=wAf>g83S?MLPO^Uno{2ow>zZ(YJ_n7;+-k_!G2bjkYj zZFe96bqpGIlql3^IXvuvYwQ>`vI!9oadpNBd^qR+4&G;f;*&p;sv zU^kMs>R6J%>_I9I6f9I&0@|(-d{nWZl}>_W%SmQyD}*P(+*#+07!upnRaC%|h9S^~ z@{CgShvux-)nQB|RfGug;4a>agpw907hbcEWLxS^CF42%e-jZ#Ji-aP+Uywbfr$W~ znv%w{HGn_R9a&zSk3O_J;za5UE&%tV@-CbA|%M){JkllnXS!|uJZC6b#-gINm$MGFaW6d zF9E9APT(zYKE{n5kM9ntMb9u~77|^V!b=Pa_MvCVDBbYZ#?~n+2@=No!)p>*%3-TK z9!V!EzNBjE7eej3MAXi@v?g3O{Ps~V)t)Y z*!snZCxrQ1UxGS#e)v5#SQh3F??w$IV#0*?y!XC)A++K8!gC!wxL=9^Fu?j~wwxbf?Ha9%EG{YrQ!>HE7nu7 z&i1ig=%rp(TCUY0^c-qcxAQ=T4G-l!>IP@E2YNaq@sxs=ee26$XlN^3@T6_qwkkT< zv(G-QV`b@R0~8+CV$(IuL^ARv@~*+W!3Dzv1_Z zev1+86s4?{cd|ttCCq?+a}I*2M$K}t4n##o0BlbLXX(}GS%99kDf-FXJ2prHf3OiwGizl4+M{P6ll z3B|09Cwi9;e1=;4}>mSk4PcP$~M2gRqlQ0S-IqbFKPG?vRbIZ$kg~ znQ%$Z0fcXpgUdO!`ELbSR(BZEGf}Q1?+K<3hh4jY)j+emJ8y`s|@cqKZ2;l@C zLA|~TXpaX$9_&>MVxK#Ci+N0gpuYFMMW&_gQ zWR^>PV}}$JmFo9h*hJL8w02J?#e{%wJ`s;%!kQ^R$Hp&XUDxr5DUooFckM*qacsdj zG&eL+e7^C`Z^~D{`gM5`8JE;7wai^zZMH-_heW4>9qUewb5!+c`N$_e32n1bH*gZ; zS(@Q#veLI`i^nKz zxta3POKZW9rpuV2m`gsTr;HGA$=gN}4L(TR8PJdz7Qdzm?q7I!%fLkWVhD&%suaz> zG+yBWOZuA1ctY3j`|utP(`uMLf;=c5?eIHaLb-U+LhzI?%QwIIuP6^gM*&Pcv%@KP zznF){$_;Pj>2s)+$^&|dXD$!yYrE+oOABE zpsfzX!EaQfAq|SX@+2*-GR21FvK^Su6pX7Kv3)`tFI%=82Fu0pa?w-493g80WQkR= z9?$R``nii2_Ug~PhWSgp&+!z5$Z;GW`#w7>+6)wqnJqQpxkE`R!-zy>Y(NNL#Z#+f zLiI2pK!VTP6i0;mNim;sxl1aLw5(jK9FuS60_%ZLa99^(aDkO2IG z2DR4Uci{zzM+}LCba1ourSS%T=UA9YS&0nl`Sa%^d1AK_f)b3&XJ;}QYbe&7BO)6^I_YZlG)BeNm0js7n{RJ?|FhwHLi9~U0!MBh8o8qsf+%90Y;~!GY^XTQrVk3}u$J1}=P(h(Tf^Rn zCpMT9wWckx#xYj~rYAfp%GvOcZKy-th)!n|&{Qp~Hr{254OETwgB|?$%|8RqWnxg8 z8aq=xPm%hlFguT)@t@eo2`bZJe&9d*Gvaybf_u6*rN1vjfr3&)IiT%{Namb9dyYK( z$RqkZw3^8`m8K$zW>D^xzqG+MUMw@AsOIF7wo&NLVixO!jmx;tMPD69dZ9Q+&>vc2NS_4h?!|V1GxU_>8Pw>=RPJ<5#3R(XPceg z6qI;;BOad#u0dP|GJGGRP--Gue>5lmTd-fiYy$6p; zb$L<1#qBrM$zeLDL_nKZ^V&EF!v>}!-V4-ts8-OM&bpv`vprwjCd& zcQ>Y^!oh#uCNn_2jm;wb_EooC+*eSMj3-2c^dAhdB@)K{ zi6hqHF4r3^msl;G5|G7{zYJtHr7?+ zzA4%}naOKbmb44-!FSwoo1B9tJd2ksMxIfl_JgKQp=DmqXb8_&ke7$~dQaZa(+Urt zAMbO#>-cAmH6WZE^2QD4iq@sZnDGbCCwqjare@2rx;Etd98v)tO+vM>pqG0CC96{+ zpiMcoawyA?6d7AxA#3k{Mru1ILx?!oPu3Oo@&xe+COE0_2qXM^cwu^5%-%HHcpHZV z$P8_^W@C2a$3FQfxR?N#0u6X1V$l>d%&2C@D6Lx)*oN3uil!AKMvjr6{o}>Tz(-FX`s|Hc>bGz28eHfQDy^lp&WvF)0Q1F zX$)|a!BmmHsj+94x@z)Lob3^h2gDO7@%Tz_yd}&(;u#b^|AzEN0r;E$`K?-|mtB4Z zl<6ADMimHYSKZ0j>*)~6hroA|h+{ynBxtuBzYhg0=&yeDEAW9yGJV=)l(E8!bwsU( zcSt}hb-5&6>u=wJUiG&qmiOtWpCvWO>MkxRQ}7cS1iU{v#3A&YW`Fq>DEO&JJf$MR zq}99bx=UN&78Mn!C!iDVZZpJ;+sAwM-h}q?D*nA01hK-{#u8jht&;Nlf)=w=D%ATx zuN?Uf8=)Y{x1&8BNxmo%=F0)&80S2BM!_i&(3Xh|_&hu| zfpbAfg#aqy`<(1tjY;o9_nFG7VQS&-+p|miFX!eJq9OVWxR5qT^Eqd0W_1T$+CE9l z6-6gf99VwArWf-UF4m+w!s`*woSBse&^&$As8LF6&4>+ESC4_RPzTNWu(Uwq<`~D1 zn<$$%ZIP(}O6k@n0k}7TT-5fAb*qQxAw2K>C89863k!JL?Ut-&?xvs$XD$VQjs^?x zyy}dH2Y|^x8mCrFUFJ6M&%zA~F*f*mo6mqLZ)_Dw2h_|dW99aHpOsxl>Sahlw$wEp zMqJbyW+YAkSnK3Y%>( z;Y(XuLO2s(DK)JQ!nYI^DTfhGo2t!4=0xl8WO zb54IfIhyx!5~=Le2xzJx&O>W&VeP?`K4)|^PdbHs|X%MvA5RnP{&264HMCG=}EZ(p>O-j^O&* z^I8&MR7ktC%o;GCfF|4#+n95mi(1zVme6Qiz(K$UF}Y77;S65t%{c>FvgzWp!(bFl znKnu8xc@16@1@HmA6Nx75Ol=(Bu#l33ENlRnu_ur{Vwu-Ps{Ik6#g*pigzxL;kZ1E zFdu`-q!u1!hIPBv7xVYShY#z&bUkNh zacTrKcqC)0?DK3PP)kv$D9V>BFFFIrTX#v#+!pk4o&u{5?pK0)Fj7J&R6^!a16ITy z;DW)OKsR|OztcsIwd_vyXHPW#->`7v=Vi;6Yv-3K(`HI3GS!(NaS#?P-OYA9+Oa`E zvqXZ46WO_0DzFb6u0c)VO1a|7E8!Z)eBnWp@YjXKH;!SODV|q=1j9OLCWk%s)KfAB z)^I+oVz!oKZ8b4-sfBa?v$Jw!*AA3s;NH1;SwJ2`zv%6IWaDd_gmiO?J^|L0=P<#hPi8H=zgCQR>mZC0f8YfzTM1_XXu7+62 z!`Zr&SnJJbqyfE1hEx?8tAv-r=U(Z7iBlbpYr~%!0qv|JEm7AwuvKZnW7hETs!ytj0TRPHd%IO0p zkPVT^x2WR=vXTk~3l~KgidD32%NZXPnvJS1LUL}NlS;gpOG=H@cn+MLPKkgvdn`5qOf57_0(hy+o;(&^pUUOQ zr`O0M4?ZUqXg4`xWGO(|vjNDSrCKm23}t89Vu%nO=s?AZ^ioidI3#sku}(i8Qt)y8?;{Wr$7IxWo?s|GzrFpJjZ8o zH@cj+)a!{T~_@^K!)%XIOk!TS-#D zq)0BB=Dh!dAJQK64X9_PF~F>A1x%vFAIAe3Tvpznu$_}8jgxo3=NkFN&wh!duR9fp zgzQLpiB@8oOPqA7J*km2HD=5N^r5~2iE8EA_=ieAQO;EODF^ReK=)?Z_a?N%d??7n zbzWV`l#L-8F16#V&jURr9pHl$%E@%8__n+BJS4G<3}HZaA_GEOJNRT?ex@v4G*=42 zw-eQh1Z?yZrkqnEplviRlZ`^cflKIm;voqPt9W6iGfqX`g5-r70 zz1Uy@!yEm0-BLlqiN1F+wTM>BXSDvd7CWY5f^kF3CDbWLLuaL0(T+Q*%MorSb}=K= z-jqpW<@D1QN;YDPMikt8#yre}GiW}aFo$>xpXgVe+Xna(GP!z$j0Jk$_Fa3?dUBg= zeRdC&JaonbQdUt`9(Yoz3?E8d>skmL?0n+G_#$qrGfM=RteUEb*6hrfM@QNQ^7b26m35ElSB&7ZK;p4)BC;T~)l^u3ZaYHcUa_hEja^HRTL(BaE zgyiXpaZT6v5p_3hQ<|y(7XTPN9A*${~Q1TroxL*tOTT+ni-es z(v2!3u^Y}56@u!oTenW0e(Gtt{(bL<7h;48b!KQ=%&1lcHsjTdYo#HHBMpR~gue$K zc~~BO_(2&rdIY|s=B_Pyl<4BM!t2dS=xsiuYe42;R;_vmoQKKYsKj-foo;?4n`k_o z8m$Aa$No=d=tChqVLe03?L>0QD2#vU()pOvTodxn8uxJ^tL)|;Jb9%LCz%EZFLzRB zAa^6KU-vb7#@){azJx@3_|A6EvKFyhqC;`91l zUjjN2Kk5&E@xt2+Y6{8w?|)P_Y}lwJPot1jTZhSe-znJ|`}ldt&Q z6cpe2_P4c_>|`{npbNZ>l@1y{idDRWYCvMGSndoPFlmYB^^@3FrXd zW}kcS=59uvGZD5lF%7(Y4LlQnL3h1sc(R6}g{)bP2P(|{r90U#+%wMnots(z{%{y8qPvUbV%QAF@x#Eh;wU+s#AAUc!k#j4n zR%h}utaW=AH@_!QJ@Ld7a@JXABeA0j_0GpMHZ^P*+RvfGRu$URwL&lxaKEgy45b(i zQrl3g+1_WIemX!6_9IE{89Dom)1(E6Vt(CxB8i{xu=d&)ql%|6Su=DJf@$2{>h9-f z`%|{631j@1+>{7DOfDgU5k7(uG{a(U>qOF5D!PI}i=94tq%532MaH7el?_S_A0@y$O2J|q~ls5$D2!c4jBoF!;kxWS}njV;}jbtVUbNhaY}eF1X-)D6`0@R)7<8_{Vz=j-C93 zJnLTMvkXI-*7BvNL--%lDv8OHCd2E8m@*PfsXT1nvRN@8n2>i20-Xsith;7szsDYX zRNnLM>-1jwe;+Q>8e;Q=g~#A`Q6ga*Z!Z0O2mkfQMZaURC_!476NFHPdgx4TQ)7s~ zsR7CVGB3fvk6M%3wOb-;Ly$X_B}r+SGOV&39>;MqfA&PFEX{ZF*tvz3dGaFUJ+{+M zhP*x{0@`~Q6OnE-hHe|56Qv0R&Iewci3Es>A}K`xzOXP2n+`$wgQkV~S3H+T@P3JY z@6D*2y~h6|mgRD=;_ zeRU;-^^wDeWFOmmvhOzH;;d_|t*rqw;ysO!NI<$Of!cFVG(mTmCxHICq!_2_gbC9H zH|pkPjuK8M+}tE;Ped=v57X_o1x9smvt9j&_vo}BJDWdCh~%c=g@sk0r~RFugE(t zy-JcBo3%`V0`#@lUek<5TCgnsAI=UhERVorFdH)y0Q9R@uaQEugXFUji(BNXCsUk`a|TGJ0egh$lz)3bkPC8iv0fd+g*AOdmXY z@aH$>47e~#aBScwrtuS_o@@||8Z^RO4EFaS7QMz#nUCR^abFg6JYJNDCSquVJC|76 zpgkXlwgx)g^6IOv%9U4LC5H|k0gaDQ+N8q2Y}s;Tc&~;f-Hezo5x$Nnf>;JxKe8lx zJxVCYjva$u#&%8X4`A@a@pziQH^;#on!OP2X;2iBaD8^LqQ%ZcG1fq{4JCOGc?c8A zi2c9}?;N<Z}uP!@!hqv%?pJ}5a`S`ju& z2hWAgf^c7p;yy$yd9|jw1}Lm!vS4bFWTq7&Nv}+*QMx&*DoaLJ79p{47^V=4lH!|E zZUux~M8=(50yuR7I))hd!Q2WxUzEr`c1RY>2XB`=B#9-1nb!jEG&7spqn#N~1MTfQ zf4_K^vwdeodOU*LZzBH7*zWn~ohLs79`>&Qoq5qk7fDfZKD6*$8IJ@D3euwD>5;K3 zu|OKpdj9z@evTUEXXL6Yuh1T@neY_YIG9ZhW}YANg$8~UiE40V!(wiw7XUpYS!bCCi@69H z=fx=5C@(G4#KO|DT+FQrW|Qg$jG*_?|N2mht-MbgJGo>tc(u3I3`CsUk$s}x78eGw zUps*QS0I~!fJJ4N`=8#ONXe{v`9d_|LQqjY66H@pz7ym=U!-fgnlHHU zd~~V%0WyYjW$xVh=zk6xD4C;{AZ6mRrAxGCn*xw@#Vqb4HMR1SAOB2yI8U2351zSp zK_a)&fk!3`@KH~n6wJIYv&R=LS_By2U&~WZu7H8DM6l@I}xT*qbo39VV>g05#%L(8CSt`;tPC+&%jeq z%iNWoC!@h{%2DQ9hmyPIhI%bi?Na1MxW2(BQ;@Hd1w)`DFIURIV}_IzB0Nc+QwV`= zJVtA})d=vOIVab8h&C@<2A2DBiUf2}V~q|l=p<4ZfIo_32IYNQO8W+}5NJE>%(81o zJz(pG;ybue0xptx3)z>&7heO`D}@AfETO6ECj`6cOAIKWskwgO10Rq--F&P3@sEGd zW*O?nc5Wq(TkD`bQDRfKz!jdFmMvFa`A)cv%9Q{NK+IY^SkeI#EZn_0SI(mL2_1|t zGY-W#_Jc?`d;EzS(B5S}EsW{0;>zu`XIX%E}N_o9?CS zO<&4Pcx`y`E@Z`abhp9fg{V$mnM|ENNfs`djRZB6bu>316oB7Qk-^&36c9`*GO{x= zlA)=y=>dRVps}aGs@9iU`j#N0lT4F`bM1)4d@_l9v^q)~{O50E2E5`Hd|cf<+&a!S zAk|e?lmgFvgVY|Yhj%L-jJ?r>sE2xV7w93Vxg~oMK0Sm{q8KdcgtUk2(K20vgtcS{ z#BYE5Ulpy3rG-{cZ0@ckB)n-UBKZR872dM?W^~p={P^-~t^=$oEMQi0P^hQ{Od99}BR=1r3cqbs#n z0tGai91%^CsA~)dvNoQbxUw>mU@Rr;XKz$m9|bZNO?xvNCPY7;$DT~GIz%e5U;rbb{ zipP%~C6gwMlp%S52jnHQ;r4bLUN@wn7P`c1yPm`Lv8}?NqQM{iF8GQF?jFAke(|?G z10gfVu^!#)*i}V&DTu44A^~bIT0D+JVj{||NH=t8s=|wX?C3#_3wpz#9C{0yI0g~k z6EQ!GW}Jx8-&wY!ST$v2Ccjf8pyLlL{H@!Xs=??>53qO; zH{9eYmGbiXR}rUuUFJ+cSp>Ao4hjZricW2lWmh!InwljwN^W`zSpE?E$91z*`{23G z^wJFgLG7~eRIc$}cw?iZtA-^%a0RKB$Md*3OsF!cBMx9Y%hojV} zLegTq$f`trT`Lm2x*;%gbBM_e7q8QDwIWTQi`$VK+jfs_TAL^je9t3!?*ahSpLNFB zn!G^^*{pEGsn**o0q|e;l5UkYB(!y+uc?8QIL4wtCZ^z+i(QB9p4F~rV=+SV31voD z=9;&Or@~}8)kc&=kxP+`95e@+JV}-*obYs-w=$79_EnuZH1;ijdThfvyFYye8<`m%R*UNBJWnBN>2?$LY zNQP!1tKxuoISyj=hbAWDJ*~nv2uK<@si=$rz=Dpz^SpSVD?SO-ylz%$xx6f<6~xqS&|t4obc7(_?+f7&j$_>B@pu=}hyR=+0X+bk)(K>s19iT5#`Dfvh#2dAu(rCj zyx?eKy%GxrX)F0xtE@ep%@I$J6>ZknRQP?%9WR60>NtZpkdp1?xG zyQ;yKqX7&ENdHV8d!oS~xCP`qq|nI&p293$#X#965| zGp0j`Po*(c6r50Zo7;A$2|CL>q@~I^@WLOiB!cRrRLY?0?4C;O)YjqRqEf(Nbg@hc zuXJ_W8Nn#AInRR!6X;YTq9tw_=}K~XKl3;++tHqh$shh$YO+2RNFCj%Jg^c%GYXQ4 zVg-Igd=ysVGS=CFjPhqxE`Sq^whJPkyIG4%?aTG{^rBSj zb2~G!i5Tm|7U^!r^_q~;2O)zJhgK*8M7T<$mDtI?%>>lveu29L*Gr+2pj*bYP$Ot+ z=W4qeA(~x_bI?SC8Oyq9DManh#_a#R>x+WS+}ZZK7nFA0=sA4Pdsu>txuMM!?VH%A zsCI5BV~B9 z-jyQ-WkY4!lrd=kH%dm2s6;j;l_;Ah^J$rOqgBj>4!ZB1m7{{%>Q+?qdx(g2tkNzkV|EhS*s?s4DHI^Q;3m%7*;aH z-0n%%Ldr#! zJzK^OtCaau(d&M4HNc{Z)i6?a^A-}$T$eY8fR5}C6Rc__a&rMSW_Xc!;2NtHe?@z_Ch+Uo$B!@O)_hm!rb+B=5wJ$jVifW?NnwgK4Y3Z z&*RXXm9GS3u!brjb)11XkaNYUhe8>E0m)OF*Ah9LmJDXnM%NOG`&(eC=M@&oc;Fq+ zm@r;r&clb6AUm6ikdLV2f!$4zgT`BY}3UMRB(OaU)gOmPdo!P%3Z;cfV6=Jh9j$rtMSx`LJkeFncsmsd8qw{v z_SJFbuC2nCb3|T*cN%Vkm!i|EcD(AY!rdQZ>W%$RlY-ziguzqanI+xnD2+nN>%@^m zWah+C$aNWxtoI@}2ezA{b7G#3H~Z^NCZKgYDUlekfF)6onP0SEvQ#3*`@*VCvJ>SH z?*Z7VKNN3q9L2&cheRN|OLP^V& z`r2;Ux$PKHz@bm9w%{W2Lr=asm~;IF2qeH!@~_GDKrqy1N#q7T;+- zAU8W*cYXt_r41&e?r%P)OJ)hUn1pTl)HQD|?smH3;C<*wK|9ObVi{2}6fmO0WJ>iA z88>{0Cb4+}QQ;i0oZsw3GC01s;|v(>sDO5ofINrZGtx}~+Ol5zuJh!fhhLOOS8PU2 z@ldHmYr&3ARFeP(QoU7bsFLa}6r@7MRX?ck*LbZ}6sUmIZSSpNdYKIVHYpW61?TVH zTO$V#H{z%a#G30Q`&fsJ99kghnQ-GGw#+*vGkiy@75vHsDPeI*{`jm(W`-%@W6nx* z7NCv@+%TVsb#F(H(;mnXDao6dcTJ5D%SwZB#My#7UaL^}Jg%aGiVXWE^DvB=Frg6; zCg;6~cm-XSybsB#r_Xu!bd@wNO}9A(z8MfyjM_2x?OpI{08gQ)v;qi<;5Va(g6C8L zYP3wT%e^AjTTrOuT0kOd8w8{ZdkiSj7Y? z0fu&yy#D$w0AUo!?5PvcimpXAYzNltmZLJPqD-oabKs`M`G|cejpAB_-cZFt1)r5F zc_JYbh-`Fa2o`s?m)BC=yBG`FG`{GcY5o4?Eo03}B-kb5=jsW-vv;9}yb}<0?My7e zL#3oaGet=g#3*vYqA@9rqp8yElo!39sT?{+y0NH%tHtaHR+SEml~M4sLs?4-LIZh2 zq@<`!CX6bRNh8sr40AOS7{c17jw`9Og|*cLzu=<3nMq`DY;VsQFeN%-rbYInMO;bI z87L{bcaIcP}Q+yu4eKv@8Gsj94&5o z)bJ~KM+ZDzsi_6ZD;k^FIXR;pd3j+yL0bZLc=_uJO0iT&MafFPj$U!+(C18y!ZCPnR(6jj#e!bxWx`(47 zBCVr>$mqx{--_Z4`Q+bSATO@nBCo!_Up8&-lJep#z_DdW0gC67nYrjPye+HP;PJE^ zi^5+w2(!Qe@Q(NeElAeIHPo~L%bK7WbxHtZM-M?_K@I@X56kHCk+OWrOxX&2@F!Qj zCNFI{gp9xx8H%xFLF1;&N;4c?as%>|r~#N|?TVSx~_G_u4&5w{+~Uwv#yzQNbLo z6Li`?PO_|Wdv1*sgZLZKHjl8H85#MC&eniZuI{cTJP-PA)3Km8Q%Z3H*%%GKquaVY zDceE-6!;*xPM7py?sjf^CWm0$={b^HR4UZ~XBl2ufQ-{psY1KC3P1v~=-*UsD-DUh zV%IjDfBhDwr@^7Wg=QeJlT(3aRUpvvTRL~ROd5Zgta|yS0!>4MNH%CTSJ%zY4!Z1WNqWGOA*wG_L*SQIAy3HAiY{xy?)peexF7%AT zTtGmVpkr65j2#NJuc~6HEGIN{F3`_R$m5PWAF#1iVez#%Z=vm}Lw&;qbY#LD&$cAU z5b3Bip;pK(iA+Ic1}~d2UKZkS>sAWsUGn;-oyfNAkbK~z<^YS9O)S{%QTvz%;xVja zW(&wx5K=AR>l%S`+lI}=M24KK6u`OVgAgEeh$w^l!*ZnQ=j7z+FvnMy%V_+q#XVPn zP&e#2CC;?X0Wd(7fh z(fKkLom#%IP}}J4LL3HN=zwn4ij4P6^kpps$V&xcStWQ*bO+meNDdx7EVcD@=t@Hu zb|lTr6d-kHu(lKB8X9A>dTOumq#`!Q7;_i{?i9;#B=ZdgfP4-n zA{7vUs-Gxx;@Vlm_ULwt&l=wvQK7dQi4Tr9-V8YV*?!WHi+@9Jl%*@`pVDlnROV;N zC1)>{ntFKGz;Cwh*e$yN-^;jbPJRLSMHWJC0JlJ8Mk)js@vjXv5IhT5_iZ4|7Ic4V zLW1itbk1r6id8BgZ3}<}Fluy>6zAtjKCV$#QUb$(xf&ctlAO12nkDrC2 z%ie8`Qd|g^ZUN;cpefZWMGMyAw3c|uP@7rQ#!I|{AGMVIZ$UvF!-q79%Nz ztXa23HUX@?4sF>;7-{Kn4N-H|zNBnZ2>VzutAQ8;fQ-uWGPtdWAv+w+El{3NhywPZ z6+^K2r9hW~1*nEFsel3yh2Bg~68WYkQJ|hA5?0jbDf)5auk6(L-nYsU+yV+f<9fZ;VEw%~Joml`3 z7&~zSP?4Bqk^#)>Ds)~c#~IKA0fOHE4Za0UHktvfuL2*t zJPX!1LHN_5;1Jf23QGO+RHNahP5Xfx1_a2mkz!PU{%zAXyYk`r(CO!OXSIh`1 z7Eo{iWW0<{Is|l+96nMH z_feA)4uv&00IlgX9GeRbFB_Uy2B0Lf@Ry&L2fz@>ZOG zG_dY2XZopG%lMvgVZASZCxVA&PQ$%}doxMB0~@-v6}|*EV5i$2kbb$a@~c4@Ln}*R z^+RwLmq}Ssq2%D%Y~Mnmfx2B;1f`^CxXc(=1%ccKB5Q?U#~%oWL_;71K71;(Dftd= zEp`*j%7UQIgcMw-OBvj`b}$ z1Kv3B`z;Ww^c&gz+eN&0XD|4lcyv*Ax-6JFN@h+%uVx4xD!)gL)`OlJk?{@e1SVi& zD_cK!D7z3oIXEXTy$*;YoQs=L%j?jf~yewtr(IAeCs@^piu(@r4W2H9Rk+MN(Fp2S#l8@ z9yMeXg!%}~OB=ihO_-x*Ouhkm$6ZbWG9N$yZ?fIgoxeS&u z|HhzYW%y;TAiH0mLW9O1jx^u^FMVu-`&SMu@#)C!=3hhGRM~~LlH1_=-nFMz4nT8k zMduVkgQlh;R-N7jOWoSr`8q)L{8H0)acgScss&?^giL9pPzGTTh=hMkmPiE=W@P6{ zF|yhV;byLY20pAb2kxIj1=c5`S@;2Q$jm3^pW`W%6k%Q+zcqBWd(WqCjdI}uZRku zdMY!-CXrd|8x^}Y47cQ@S02-$T4 zDKHJze+I)*5cFwq6SIa}jT)`9wj>lApJC)HxCj(~y5{M6Zh^3DcLJHBF9Vs0(8q`cdHokcle9`J2vAWHg^^jlcIb!yBXk18IFN2k%d~w zf(zv;cl_nw!ou0_fC0|bTkCYLR}8GA8@_k@#4lv;pNPqY);DQPwf;3h)2wZ1f<=7{ zP0SCvM&^B>z0^0ZfIu*P1R9y3|wWkJ0`&ONSQ@eK~>ZZOqk$d$# zC&H&V<+x@Eg3M7Bd%?>$r_hmX}s0~%V6THhka8tOD*q#1bJ?Bh(2H{DI<%3*gEXsvKFQSfG?^=1xYI!u70 zn>h_Jr*!9N_u*FT3T8L1_o?wY%(%33JMA-DXG`5UkeHf%E`%badEM*u&}bSGBGhp zN=kBbbMx`>ad2>miHWhXv5AU`%E-v@@bE}TNN8wiC@U+!d-qODOUu;MR837yMMXtb zRn^?w+{DC0Q&Ur4U;q94_XY+AdU|?>hKAbO+Q!Dl>gwudW@bi4Mjt+WP*6|+fk3*t zx^LdRvA4Ikwzjsgu&}eU`}FCPpPwHX3=Rwobai$0_Vx}63UYRK4h{~sw6yf}^mK4= z`26{EKtO<#lar&Pqpz>;$B!SatgKvIT)e!z+}zxJe0<#9-TnRjJv=;YZEbCAY!VX_ zlai7mA|fIqBQrBIv$M0ava<5>@{*I2^Yil)5)#tW(^FGZzkdCil9Cb=6O)mV5gQwu zlamu39v%`B5*-~~P*4yW8X6ZDmz$d#A0Pka%a?E8zNMw5MMXu0g@rXWHC0wtLZQ%- zl9KP=zatQcqN1XfmX^Z8!m_e5I2>MDTH4UiP+nf%+}!;0=T9UO+1S_!gTa3P{#{#J zTTxL_UteEWS65tI41qvuYHC_rTYvocQB_s->({U9>gt7s1r!Q3Iy&0h+uPRGHZU+S zJw4sm*VoqJUl!&I5<8&J~A>gKR-V< zHikx{XJ%%WmX_Mv+q=5D{`~nfJ3HIo-+z01dwP1hv$Jz^b8~lhcYl9>d3pKw@89L+ z<>TYy>+9>))z#hI-NVDffB*iSpP#R+tel;lZEbBG92{(HY+PJiY;SMx@9&?SoUE;_ zU0q#mZf>rxuOA&9?d|P7JUo=dS_M3&DTasMTYZ51=B(YLg72zk;tl|ibp6*c(s)Ve zAC0sg%Eli0E_NQ?mTtBHZ3k;N7xzbvNypC4!P++XK;$z3K-I1OM!~>m?jY|AY&lvmw1l6Rzo3-R;{h+!NCnQy1n&P?keLt)-6U&j$Cda*{zlh; z>{YXlJb5$eks=0_t^(?&@`0_egc{C*lPc;M7z|kSTfC_H@m+TExY_0RAv+nBmThaQ zTWQe4?bv|6dC9pVB#(}(>j z$eK>NW$nZoYGU{mJ7rM zQW$!cGLAfg7$K>9^DM!hDO|f&#A%|;8Bz6(x{9RKuT9M+&E2Ex%X)fG-)r=_iCE(Q zw;7iaX?RoHuOOv^SMcci4Lkd@Y^ZFLtl?WPMYNl!`|PQXo{O7_*mWVeDV;0#K6F4w zu%c0mv!A-3`Ux%%kEf5BSWzTrP87`6C1ZeR0(buEuD|~^;#OjuC*X=~%7R@?9(~eK z>O_fcEi%Ky?NtOW0^`tTo8bOtrUu>;>LS{v0OG&tZq4(rHiz%%5z?Zh+mVuUtsKKI z!zSSEE*D}y9)o+92a%(o3ztW0WJ0n_GG4v0ak|{zfOCN=JIeSZZijpz56c(tcURm7 zMkEr$v0Jy7wy}IU^3L)gVG0-J(?PMWqzPCnChPH)&7@iCc z3jE8o0b{()y*oVontKzQ(21$v_;7+7J-LZ3gG?Mw(?wWKg=K3!$8O(YO}N9E_|`uJ zvrjZ-2!IL4yh%tr{ZaAgkCW_-b-d;kk(XHiG|rSFxyheh$(O2@J;w|@!aXKpl65R=zQ2Tv!R-d}9lFEM3(9`+ zeIb>yJh~*B76zht`lnIJu|uOZ%6&piiPyPe{Mqrz*1(?S{EWoM932xWh%OVGFWHzY z*H-biE%Pb1HGy{P690qO+v_4ICQ9SqIYfO+S|IJoK~Sr3JXOe)i-vhFilh&7eF#gb zXJgFo4D&LU*My}8r9#|msw}^K?84%8B4sSsVRXO+w%~|ljqkoeOBQ59ElP>o29aED zou2e$koG}|m@O)uLU5iOijsS&u=@yXuNIx>X_mz1C_ZiM8Fe$ArHH#2 zr@QoBK-*~0_fI!{lPYLuh%0MzH39Q7ouQJE(ds~#$AxSq$~l2~E$ptT++RO+yxhPx zD@?=e9yeIStl*Ao-{ZQa*wlC%SQ(6>WhJt8`oLaOvClKb$(`ZSDMxt4H7M-6_^OOL ze-IC@7BalKRlQilxH9#slMnXR;LG1dpK86%sh_%t;bNZZjc{+5+mn2X^>*2?oc-qs z`;9${s7exNoeOhljg0pBj_rolC%HkEA-%qz#mhSbF5lH!O6NaufnJwFt}mzUlLpqa zA}D5UF(W-?!h)%6mt#GCej8>mYl^*LJ~$ZR*#W(dYe>)7RWDfSqkTAaM4RlcN5UF| zW!nj=IZY>W+>M292}inXYO!4Yx-gAf_AmuZXc9^x%8R^v&3`IWZvq2T7Cwn)t9%%= z_AUbFar0ODuHOB9Re$~&T@+QEvqvaUJFbx+vCR-RhK@(&rrnQlf!eKs-8}K|<6+iB zWbH>@vEeZ5otiH%QX%j{1%i^7f78=u!^&62sd_+e6=5hjth<6&;-J$h8b{1SLDFX& zb_l-?$)QdxFt|I+sfhk)`OT(3EGgHl^^dv_}wVx`YLk4@M$Ed=*4;blESj(=js zUE@Tb)?ly~>5tfua8}*OeK0_zS39s-hsi`=;ft|6>!aY6D`VFOrHBH91<>!SDEd{O zdg|%r`zWs`#QJTs5(rOlf%=M>Wh!aV<>PXZ%M8!z8?nSQb@2FuV_ zE%ilguf1Z5jb>ET7Aq`&<)TQDB#XYeU+!njlD)>>U9G-RO`V&Nsoa@^3|3zxHP|c$ z0r)-W6L+rtt1PSiT#1v_?v(8K{7?assv7}yX|GJ8wvWZxnDyngO;pqrGjP?n5BCFg zLW@T9M7xB~Fea7p$&+b(utlmezhal$qIbl{VOkT46e3oYv87RLupK-(G5JH!k1Z_Z zZ1v|mZhRfNNHJ)fDYmI5^B?t#-yGc?z}Y)$;$Mdb=ekev z=Wq&DFx6)}0R2%dKHF4^+R8Y(`yC`c=Im?=&y#F0(9~*H#uy@H3R0s#lvHc~;A8bZ zd4cq%dN#4Bc=%)0YM|yE8s0Ek7XV>xPpaQhLTK^dE%<{Q8^f31tg#rwgBN!lX@y7j zVN*2HwfrW)_Wiq4p!-O(MBM=J-L@yBjoP^3(WmYV2k3Gb1(;6!y0R6woN$Olz3aZ7~w@FIi8d;6Y$2P)9t zS@e}T&9dW_E)TE_WIW8%t)HFj&D-3?(wgh=FT@yat^VFhj|?XnqQ$-axE|nVG^nLc-_1XUYt=ks2_b$%xvh zH_%CnFE70#GE-21grt^H!DrkatiiTK)*a&FWwyUoz3hH`Yw4LTZ?ptk7C_F&n5zQ| z_?Kosb`D~IHtVAVZe&68#6rXfEQBIQk1XgfZY}rO{XY_kBSW)1#G<_8WD7{Sru@ThK+ep*e$4d3EuHQbv{|l4FYXrp zy-o|V?o2G6Te3$pgS9X`IK4_HzJ-?{)@8ajX``(a-sIr{{P-XkJWp*5rg`;Yfwfm| zR>Y?*-M7YN6h1Pom=siV6Tn zF!5`NF)Yd4x6uRSw~FX5A6AYbM>KDY19IH`)H&aZ^~ae0c4F62wV`M;N;4lboj3LjI4hT|Mu zR`A{-mYvplC3bd7D3_Z!Ckq#^V*D!U3yyfRyrF;*8R9bYRuTcTn{17hiUpjO3-Qce zA9|~z#rxKUKW!HR?X^8FirN>!jMo8Jm)?3ugcrDz9;YwaK3@UZ2+#Xv)K*t+bSL~j zC{TQ8!BekWBnWi7$Mo65SI)lYo-8_|a^&-4{G<~BiRE8Qsa^7y*$@PjHsaX60M8rh zJQM~QEGgU`^>4kxkl%n-)gI`YSu1_o5&M805@u1{M^N98B^k~b?t&FuB|A6ykK@S5 zS6gAiI8&Weefx&RWL4<=5c+)=*>>t(Yg#gOMnvo(a(*GNRp(0}P z1cHzf7xknE49G5(v5uFeFSAO+&tg`gB)_bBRg;ACo5OR_yao7pieSDhqYWj-(SbxTAuMf%xn{YT27o@pkX6TTn8 zRTa5iOjO3eIgSpddS;58wjD_KOH)8}(A!k@qKT#^~QUCuPKngAeMlLZP;BuCdLm`K~uI zRW1tiAqi>#6c3T0~Yu?*4t@* zAr-NM~0YAU+3}Z&E`8qOWDRdNPMs5u`U1DniwOFC_@`XA`N%Av8Y4%O&oAI9j zJrofPUdocCN4%fdW%o{%x`I<6D$Ew8&-P?S?C#V%(m}OpqwuSoUGS1_GDCo|npnz(r5H$b(SAhhGq_r$!ghG#JFN(urK^E!n zNkh%NgnRSZzbTT#G2YgXO3IU~T>GOD#}(zqDHc08RM%YC_1AeoI{1L-dm1Kgtl5;1 z6i#gfk{^$^YiquIxz?sSy^JOXVqTU_BUV~2+=M3RgA1~-3skcIp5MxaTwz$_M21p5 zBN}PTr?Nr@XYjgfd5dwds6O2@wf3_cBzHD5JLzn^S=t}8Ji(4Us1Wi;JHR*Q!2Q4{ zK5U5o1_HU5=7m(HHGxxlE2L9CSI20Mb<4Btwl3M~qUu6&Y2NG$V8w)Ss#VHO>5Ru0 z@Ta7N-j77t#?$)=w(w0{PZ|buDtG^VKMYmcku#nzK69U?Z|{Y$jMSvG&#m$cr(KKT zG`R#I{XH4FFJwv@ujq0T9ELlZT6p-9ifPYL&HbVrwc=3qOi6;(xh|YrfKPXW1@N?< zuN>jO5)W&XoMu6Ntsnf7e@-K2IZU~(Q9~P($LlJUp}HSi3z84njB{7wt`JVaht2I9 ze!*_IzE6p%&wmagyq_kJq-9Me@$Qj^oR}I>@%I`AJaDz9>LaM^%U%dXF*40j$c2+p zu*@-3TPFAIuViuC+~s3~{;3C=(c!W(t&>1|7I&{OkFKw+^i{3xkjs<_fI z&e2`u*vk$CA18vHKr;-E)+XUs zza1p2`bDv7)tIY5gZb1TcLlFfBW?uGv`DVnaLCT$(i8DhiSig`>rGmdpX05?}BY)eh`))!NG$xWTv05jX>r8cM-9u$SK-~wQsneujKsGW} zRKCm9m*wZ&69&Gm8s#YMvL&1#VcY?uuaIh5*R#GQ%(jQ17;DSdil$v!MvU#B&yPt% zxALv9e_9I#<`MKvWwc)H1e?XB@2rT3L!qWm>x=Dv1p^IoYroa-biO}!Se zf-<~NRZ0hR_6INmT2e3vW~MYVplwp^O{xUN<}O0uI#X>@-eO$fVcWsBAiW5Qeu?~k zKT`_xrpCMQ%xbh$OcP8wk9)vmofNfuC?m2j;` z@>os+wZJH*RQ~^?(%D~0xncrtF08R%tEa@>lolNuW19LIRhUU8ZLDv~I%9uJ}~nQ^p^#!dqTLO)74# R9${aAy7JpMP(_Qd{{c{~dJX^p diff --git a/dpdk/doc/guides/xen/img/grant_table.png b/dpdk/doc/guides/xen/img/grant_table.png deleted file mode 100644 index c23e5fa73e660c5be9c7cd01f55ff34e2917632b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 96762 zcmV*OKw-a$P)Px#32;bRa{vGi!~g&e!~vBn4jTXf02y>eSaefwW^{L9a%BKbX=8G4b8lvJAWvpy zX=7!7?KN=#0RNClL_t(|UhMq`fE?F#B@ExrJvq-H=M0b_NP?Ln6{tkfqLpPymgTsP zyX*9M?f>_#z23EK+48q*OAaebt6)o3q$~!BIe{4fk#i1%!Q|X|zH{z<)zdQ=U>ZPU z2B8WZ&UANm)vNca?z!>Y@Spt2pNPNtGvLpFKLh>@_%m=JGZ6mcKmOy93;Ck|ov|}8 zGLpj^{Nb;#a8gG?B7s~|f_yI+#s31j$EofQpAAj>_;8M-N2=2^uASj_=K<2hI%d+`1W%Z&4SunFk=FM!8%1FSS8h3Mc z2={PE(%Gm4@tu)eR>F}S?(-4J;%*J$-VS8-7RP>}Z#b2lmOc4-ZoPkr^}U z<>s4K$USelMV8KKlt`#TvQU&F0Gu4|&rm3Ydmi^^URB!ce=eurSMO-<)^C4L|dx<>w_V>$-+DeJyVaozI`0$P- z62fN>uOtIa4-PGTP;(;*SR+a2AMiYmg_`8}_X@p|1#SqvzVWXodasyc7qic~j$OVqGXGbxkd@WaeBMSbMYVJ90ucZP+fa zZrCgzeDJOEGw;7!%Aoj+WN9aJ3_Pm(ZS@!iA-+XY0BLPF9gAbzGFL)0}gnKLq zP0=1wA4F`Kd=M!mh?xDrAG)RBlu|1i7Dfu6B$HAJOXJqX^JVSqX4$xBul(CL9+vGp zcFTYIjgQOx#%lXEjc2t$g80WfnMd?Nqy%E+iaCT2BBfX=jiSQ>35E3$`pS=LwI@v9 z7#GpsjEqbq^e$q+{#X4OD3}4YL}>LYgfJFIcN9$vI%c8P@E#0AB>|8K!!nuQ*d!l& z-%rW+o_S3E@V|di{^E~+U6#*8oP-uLT(|)Zw)@GR*V;|ygUAK0I3E^*f(m@~H;3eh z&u*5leEHwy-8a5X?pU@=(u3^?TP7rktg9@vI_?#OA1grgVC+PrDBh-slF!H*p@!&V z$OW#B|FvRfz?2vS$#E~z63*#8%^Nz1+tYr6Ig4=wuunEMAhpr3eCWRS$ajD86Zx~h z{EB@3PkvLHs^b6=DnWGR`ZsMcbAL7MER+GWRjfYOjKYZ!ia?|P+a{HQPk{#?r z#tWh(hY3@nahR_fzb^5EU?kncS51Nr+eJtY6nZ+{G_zx0Li2o@jl zw`*qxd=R;SwH`IuEjwip^&tQJrN^ZqS}S+1TOs|qHp!t-E0_&S1Oa#Ym$TWlDrx~d zY$H?wT^5$Q4){{hg!Lw-(b-$h)pY@5^uOZIz~nPPuqd-z5BDhDSrW+5(W>`kmRW=@ zS|>pizT$Qa!yPNbNtqQdllR_vkNn`-CuGentL3Nex=vkU0R<*lCjNHK%zzIf=UD1d z9gTLsC-~`>(g-BJnZW}SlSiN4CI`3emJht=K8d0-J)Im@V}tb~BRPNvpHaR+Ux8-u zIwzL?VkVY5yuLKO(9UPTVw;CFJU2gPdYA3%!YJ3wP3vE1>1H4o^b{4uL$u_aKc0n5 zCl~K_OWq9wW`eM)Ga>Ag?w94WT4eQt#q!N>{aDtoTPbrYz{-F+Cx5%PX21uLqSl+K zj7+OiTfseKM1!E5M&TDik4+zG7 zPPYIb>Vj=d+|n!|XT}tDCH_wS4CH4(Zyzr*$KU2kwctAcqH~H$j>YE~A5qGaAgY?+ zx*9>z;Z4_FFFPN9M4o!)Rr$I1-H9*x1+v%L9p;0`IaYjPfp6NGJ#1ll`L&JGcCt<0 zb^m)|fsiN5k(GGDK5tR*M0`lkH+#idKsrwIVUI$R{blGzOUGSIq4DCr=Dsv~wP)MM zJLE*)D;)aS&ULQu6;7ysep`$2^Eg%W@10!ZaPHoubvm)maLxSp^Rldvo_2Yf5$qgjP?(MCxw4GVr4YF$Uzw@zYXXRt+}ZD6+6pT6}1aqy;ET7w~q+;=_$|hfx`v z8qCVElg9+HO{JPB%<%Gp^qv7V{4QiSGUaO8#Rrjd zu4cZX0}qXtxnhu#0@G=4|FD$DD2*2B9J6g`25U)~VLzdqMKDo@-n&vu-1?Q`{BW(Fd)p)sY zs^A*L6~L4?>WSVPe?{Z>@#duvtqb=l^vmaApb#S63G#O4Ii~PjfDl=LtTGx0b-tlBec|r>!k94M$Nfn+zQ7D!g|Z1@mDZx z&B545!zj_gW~WK&B_Eh#qJ~|Mmo7d+rfD+#5?AzHx^ZDyx-)SA~WObNrvFj{J)tqUL@9R21sLqonEN_hjmkbD#paW%~%p!uz4-NQ~ zB8IdxfgJj{=u#4R_(w!%kQQVH0|=rUCD)j}!&nCt0L4dW1Uygx2XdZe0x7N3rcSQs z^Xe?JbOizWaGne@@HW}_mm=DkM$(+l!EEG4*#svH3L3~OxMg`L+3S%0m4$jfx7I-#qNP(TyllzYwb zpN$QmT|;+kr9rVgRpG z5|5Uv&?hkE@H$ckjsYAuouS1MMo;^M#L8oql#oE`v|1+C8-~_FIyWe(OhW4F2?7yG z4KsbB;E@54(UE4>j@4Bp5`$7%)gaZ?wdgRLmj2!zSXi`1090u3WUe_cNrR>pfzC?_ zzswiXAD%x06U=}QBK~c0$!9?02C99hWiZh#%Ws}5@A>Gh5{`yY!WNTMIwc+LUGnDE z-LmDCqcS`Yme@#zR5ixrgTHW(RJBBZ+@dc#)C~kL{Ev9@(g4GiMx8M4d+Zo30{_ulSok z16Sw__#kqHF5HzgY@>mpu(UdZ8FLzB_Of~z?n}s#y{EL+X7w!#Wc3{j_pFi56TPzW^<7d`UIo`2viH!KjKTjP<5Lwo zfTi=HU$|Fpf7c2*ey~-Z{NZ-Eri>aUk%CJLv5E&i@E!$_*PqxdKY93NDT~a6VCOz? zN+EW$Xu%Q$=OgmO4_}n~?!QeE48W-S3fJNaJ{dbeqt1?QnLls{mq|&GiSgD z5&y2ZfeIRC`>9tv; z>gJlHtK+2XLzO;3q^hzKAWA+bqwNTw@G=}iJS82F=bw2)=C5d!yWV?)tX;Q8p8Lrz zoY(3X28I)I`(5kgmV1`T@f|(#!*6Xts%;G{2KE?7L2gn<*?tmqelvS%%NLO!%4EFZJ!a>?H zuGMCNZY+WZEgG8>Up!y(ThKqjC7b~tL@wdv`iFKiz}gAcAZTYJ68ZrgC`L7pH!NkL zO331<9NLRG2SB5uE+$pAF=R&JdrZc{Ze`^ovT#Y0+>BPN+>RaY0r+HKEzliAF^oC| z1jb|GsAM8lvUg{z-1Y7oq@lS+V(4+f3@*y=+4E+|;$?FXJITqnzV@7Sw&$e2vI?K0 zmIPU2?A#XwP^1P>rGfx_S-4)0pVrJWe5NsQroyr&g_Z(`UFEl1{`4@`%_pv_%=b8Z@MEv{X($4^0Hm<%v z-8U}7Nn<6*dSX^o7%95hBtQnrJ!>>HdJSM;CRX$8O8Kqdzh7#qt0i6^l;ium<>{Yn zgyn%?wezuP-NHx`c0o)9b~0h$5)2tXluCbM7KDEi;8a;zE;Hwr$+7(aG!zQ5x`g#C zQFawa+=4(t8^bG}3!~^Ui@{{^ZiF>MNDo#}(0_=GM7&yysBCz8x71eF$j$FsBp>|b zJ#zHGN$EL>j5L=%Y%vfo#g_qk-%)hwZ$keJmud!l5V=${d(p@BG}K}Q%;g=J)dDqu zg}Q-8P&)zZgIfeK2MX>ag3hTdy5u#&MTOlGgUI3q|r5UNDl1mlcPu4 z<&Bs3O7E!zQhb?`Yu7%4pf`*N?2FW09Iv^hMnR*edk8TO>;tQUTic0FdF|Ehh^^c& z4}Sb^Ike}1bhQp49k>qXA`>%N0W9^Ff%HHKmJ@3wk;fZ?%gjnj0XW%m8Ai4kYfe}V zuV{zu*~d1@qu<>L5UG}iIpwl&^$cm5RVSy89tY@EsKvy8T4V33)i3&1x#$CU_ZsJD z+cZM$<(0q zCQm`1uaTODnB02r3cZknyE>(>AK6*aMpzOe?I#E1i+}YHGO*&XT;Tpfjb9wW>YUQ= z+qEZhL+EAEhnkb9)Hj#Q&37%-_xJBPBFRBSVw9y8MT_Awd1J#NS+}NDuD^Sxyze9T z%0K_zlgQeNN?eT<(4Qb*I1S+HOY|UyVg-s!#~m6TkOVL=4EI|kS}iRz;?g_VDgE65 zy*`V%)FZVwhq0u*YIa$1KxI*-fgh{!E#-F_6#=Dkan_=jeylDx4>JXjh|39UuY) zKbMFiiz_OL01Eo>dxVmnoLiv5PYt319o_U6E?gjs7Q*_#N2#HdY<=dCJo?D1($$uh zSQHizeO>sBd}n3RxEwjyhM@N*dH<(wmv_DQcG zzGM(&k3^&Lz^Cq&Q+tPG+na}_3~}^eWl(DCtI!UhUbb&JEieCgpWJreQu)Q-eNZpH zzbzr(|Ci@vpc^$Pk$Tq=kf7&T9+$ld>EFiw3`{x$K8Q>@Mg9&KI|KBJxZR(0GwUf zxYSfs$}7+BMm!{eOe$DO0HP3@ivOo1OeEsC{ZWt<6%A@r?0LIo$du8T~W=W@#>PkCxvI})EwLl_e3e$2+-I$rh-7_Om-tl;i(wh)mwrxJ3v?%2j z;Nl|fJmMEoRIU#rCbRR+4h_`9c2U~)Aj~41Qeu&E8SFtl%1`!56o3W%poxx@97V#P$!41Xnv+qP0?xB2t4r=FGdFRqy*?5ce?f6## zzY+EK5oSy=C79&<+EI`1rB$ehAoQ8}W$|y9^+Zh{0q!8_+ez~tKzeYbL5l?0(}X4q znT5ju0U!=dB^?4mi05P4aYdvCr7bLUEXV1{^b6lwfZ1h81IGEP%NunJGBT*+AyAgV zI>MKY-UP86UXpT$xs+bBKRnJXZ>=u}bA)TaIpf?atz`bLvU|)+VV;sl!9U3rGy^_} zTn39pfFO9V4j}~1JO?2afJ&al?+7ZT$$+c5Gx=Kvs+EtiyX9n4MZtm;S zd6_o_Yr!RADci60I^pq|d4)`{d_Qthz;pZ*viH<>VKvxM)B-X4{LP<%D{%&V5ShkRD|B74$j#gwCT!^T(KfBn5Hk!I zvuP9EIhU zJ0Pl1HXsThsI8=ka5WjZAya}G%iwN2K955R6;zn(alBZB7u+@)k`Z<#WUF3+5-bUI zLh)QYr$UOgyv+vDzeIt74lIHQU146MDeTYhu%MC!i#k#RMh$^@^%yoW_P@(2)rhIw$fI7L!s)=-37R;Y5jZHPOb<18Q0c1@@0@*zk6*0Nt zhGnvE_c7`0>PMQX%T98l$hg^)stm|Mhi|?Q_A-T6= z!AhmBr(4nb^rCSV#+gX4-7exEks$=@V~D9#XrMoZ@zAx#V;TL5fqoVlX$hnYM_{WJ(vzLaeHh zGQ(^B8kfMe#Q=M=yQ!a0EEQ8St= zrMs8%G4W%Zil z!8qz$c$JtJ)R@dhL1lkemkjp|qeJ2hnK5%V43w;N_MVjEokswO(Q?#!Gv ze>ZZrxzMIT`82;6uNxqnaO2ggXsbhWus;M zKw}ha=9G}dGh5{T58N(a{=3Jdx2s3%Cz@y1$;UtQR{8QjJS@vrpx4J|-X>2yv_&5N z-iuO>Or~Ta4dGla|KWGu3jrUJzxl7W z-6-$<^zHJ<*I$yKeD4+c=%?4p;uSNb`&ggE8kb5}&wzaY>n}@RJ3tFrVi|xN!9!h_ zILA5DzDsi7da)YM+VBGFo~MmIK6js)>f&$y44g3oK8Q^5O<^?P&~iZv1&@^y3N|)Z zL$H-gE64#e8DwZFgDv!PXy+7){rxPogI4d+W~ip<1)CX1nc|CZr?c2MwP#YJBi$Cp z)FEj0vFJUra@7KP?%6k>#)o9Zb@O3449Urp;HFw@=b&!?o z)-8~D6>JWW>c*ludFIIt^6P)_DOt3#3AGqunKi#r9{Sfu;YNa+stm~n6*&m}Tjb0C z_z>bP=)+!xzU-Rsn~|EjxIFv_Iv+l{UFyo4;MO8YkH`xzKP$J~y+ZDN+uicF|Ld!$ zLy5p5sg#}Dj$r@iWjHe|zyC*{kQ>&okYk5l27ea&iy)SQ7)Ll3hf69Z|MHD*g2eH( z{L*hefM!*<%lH265u{!ZLe_@lmOEF=L*IH4#Eo&#*+V4@g5%PHfNSM~Ejx75bVcZ5>U!%zY z({q3MXT36Jzz2~jT_*;XQTG5h3hLS*OtZ4-_5JGVsYO4EojdobS;2>xZkKQ<1*Ti@ zHpLnRB)8@UW5j(NXk2<>bCyZc>?90_K79zC2zrzE^&~(6vLF0abL5%lHo-j=kmalA z%cf0RWheovcHD8>x9>*}j~TN1hI#VDqg$l5CN9_Cv>ZbIb?ke+gyUg3j1Ga%J@J}6 z_{q0Hz-HyCAHM`Tls>g0lG%hj{_yK^$Bj3GGiwq0q4!G?;G0CeVFUs63}Pg4^c1PC zXp~A&k|Yv?nm(OQp+aS(S~5e(uu7wc3AF4CD#;J*KZ5vsPG-)jR}0|i@e^>cp#mPR z#u3I_l*kux5xAU&K+i&#+R>v&WWmB45IYIWV1H8fY~Ld*S1ysNhOl&XcFWuEyanJt zC|z6IPzg`H0pml=CkOW5RI*%798Um%o|E7G-H*zGMJ=*_*J-%MNGi$tm#NbV>|Cx+ zXAF?g#4R*Grfy7A`bhdGyXYD4LFA&R@H`K|R9)v7V3VV=@^a~f&`G5dQWghq6X?a` z@p8F-=`#4_GjjOAA#}&H>Sxw2nEOgol2q7OwflLFp+vjW`k+;%8A=K$OnWUug1}+< z%&)9~i>geb@e#GOc73-EfpTjVJ2W_g22?xXM!HcRd*n^CQ~(0IAaJj{3F*v1d=JdN z&piHyyzRl8WN=_eUV3^nn0pZm!8u|Owg>i|mWsfBbRE21zW;+KjES8w1yuKCVa~i0 zjiGK`zeX}WLHYK#o|W!i1j8K_JL6E+^F0QEKB9>Y@A|;?G7BV*?MHhR@IqLN7#d+Q zQ;gOTfhiEq$%-{I zJTnk6vPd65O7zHFm51>`WJ;Hn4<@_(c_&3D3S=|Qo8KbKmd}?BFTMr}*`%zyakVs} z2So}haD82syz;_!#5oBTR=7qDnsEpVtb4T)Q+gqKG8Wn?g*Yq`Pt-)im2wDu&Dk1u z+3GoHJ3B0`r%%a=Bgig80GU=x3?R96^BWoiTDqc6)`E3+*Y@Kw)CX@Ax!)m8({ORE zTDAb3TBy;0t7yT(xpLr51g|ffjXPw;_4iA1qz@K|S&&-&4ZzJ|91*0RMWb+QW{{PpqMPffk?yGz z+xc1?t5`Q>z=HbfR%0rNQ(mV24)f1=CCq>iB2&If9#@i4f(HNsO^ppQFwhU;z)rZf z=1R-VdU+8P=KcLW@ZaAeP0bB*;^=9_9H{XZ6pH8r&${U8DIQUcRqHdTtg_o5;t~)z z1A}nyY&#|EZ(9NlJ|Vld97OzqnOJ1gC2?X%T2Hnib@{k_{8Mj-Cu~Ii{X09P25|!I zCzu+LvYHXO|EJ$3kA44fsjaP+_kZ+0`P;qU00xi(K7dr*3UzO7dTEEe^IhvDgP6-m zj+DPq_8ydD2ij%Z#uK0>83ZpD_^pruHwtLWggSm}y-7+2(FCh)OPh2Lv`KS) zxxDM0cf(!w8Up_>z#SJvR@s1*g9<*E49dTK^+{CXaAWgX44;1Z7}oKxQ{IO7n~vvTDt8 z=|)Y+>C^29A|rDtiT)dKs%y*GLI5ck_f-1O$4-|AUapu!D(_ov1iEoI~GeV zx>r?IRLTDRN981f;l{*^Gh+w3;F1_yQ**t%?VWdk_4kwvrTgXHw_h*YHth$oBVrhR zOw>hZL;zuRRfRUgVmC&z2gl;*<<6Q9Tr=M*LygFxy~m`ZqgB@2zDU0NwGEhu5Nbrw z>`H59*k4`b`n3zuVm>Ex7B|c4bxY*Y@4p}&rw6f5^{DngWib(FKPqCFBTED8g?7-C zjb*AIME`uHngJg~N;Q*39>)m5%^Ad>5CratBd286tY*0v!Ym6P`71BM*Nzy$tXZ?A zy0TXK`;J;3uLB*cZg&}4>i#J5hNe-+2IYybZ$=<~H#B>)`zGXtCpRNN53L=Y27d%gAe^g~;oh`7 z{Qc)-X5Cx>M1^d7?Wpv$cSse;8HajO@~tmFEi2$gN`o7b;PmuUugKiRHL`L|3lbRO z(%sV|N3}5&>Q|5^y!o{~s%l5U?mLW{kIt@ES-W%*FCwIt^{T%?s4g~e9Wc#K)aCITO z4uSiPuk6H_7RajAi>152Q@;F9-<88VhomA7Vn}ppq%uJ6G!^GlBD}}PJ3J5>1(Z_X zGXC+dz8UaAWQx~R={Pj^-ddzGyN3;#UWb6GuC0{5-hQ})P&pl|)Z)D@n|8}UKiFv5 zoex3Rtfpg9&>#+>{7?j!=?Z=lKp4Xyjxc=;!Do^ml4%#d|3`lC0)__79j+hl!_I`; zAXGsv0?oTp(ut7#`@e03;6!zPd8HQX{rh*H!L?V(4}Z8p-CS`9bE40t7xyEx>nJjv z%H*j>Uc)hQJhZ6|M=E6d=A*KC^ByUWR^eDx0HXbpeF}4kZ?Zs>)bQ~b=-&ZwhYqx% z&&Nr`BFb$(k;w4mqi-UJzelUvvzb90qYOaCHR^-$hilAK$Uu)En+onNq#vuC2grqS zZK=VGJoeCQuo@!(jTYIt8E&-AM^IT0hG2~K$;USW_(l*TiK93CetCJrKENEv8MA4$ zozF^Tq!F$zw$CMUFs+-(2~(vNc$r!1yRre~t&qaMeHPy-786(0>3k5G^5wF?I3HFm zD#Kr-MQ1u_0<;{Jl@&^NGMq?BZ|{J#YgKl&j*@Ar6lzAzZT@PF(@t9kjmKf-$soFk zI6hFNV@y$GYMBd;bS>y4f?#qKpvlx(IE!E}xZKE(bVtej!i6F#GZOs5t6SENH3LC~DcRsr@l=UgdSFbEqV z_=9i{8l61HA3!!-HQC)2kYi|KMN5x!%SSqR?`jwDc{0%id; zXQ$;52F(extFmy(W$=C$`$kY1A3()^2HzpjkQbO#Enx=LafT2mVi=Kg$TXlJ-~-UogAl4gAV4Ia{!9VSqyuJKS+-CB4WCFS5>|35 zSUjxi2gpPy@R7Nug3hHGGu1dp@W_|*6O0VIN3@2g)DP=rIiv6%gIY(tr$`y1@!B;H>ng<}v>KO1J!AQ{iU6Z~& zg2O4ZLTnoH?nv*Y@$pQbi}ZXRC4b+`Vg`H=nc`)z6i=Si+y&WeN}8K%W%b%6a^S#z zXsZYYBM!oTvZhL0kF(4Ie_WF z_@Pz6x)-{(Oz0|Z;2KdKPK43-ag}U^i=RN2mIc8D+CPMPPHRG7Stvz}1hJx$rFjQ-*|z7;hRs8CXjkJKtAp z2uDXtE2XE2U<^@e@OUSS^K#5A_6`F;f-D>~h*BWoNC`I#{%2vSWWdLjg;f^DLA5}9 z$~Wh-%3D6;g6dN~9{+?_;tcp8GS#c*`ek@7f)QxpB<1U9?~ylN*(0k~twhk746#uZ z*?|%y1^y@WCC}x$l*N}Z5f!nnnbV}Lb*K9JCODtEew@Y*hZXiP4c~fsnBYwlUsT+B zLKQH8+71eA?i?Z=yQ_)GB0dyAYHkEz5{LASA+8ZYrc@kph6qaggQ&ZJ=?0KMTiryT zL@|f#To}ZAApma(Zl^F@Yhk*jSTjPQ4(lKdupnS%LQzSBx+I0)Dg4eL0H4DTDTPyP zwTpmxB2+F(kS%73D)fcHDj^tZcf>4Md+~lIi0XO8eCu~YpbCx2gyJMzey$>~h}Cm* zT?V(1KZMJ827C~iw#6~GigHh}W`Tk=Kmp63D=H6?8PrFVA=4>_!aKTC7!x6ZBYiS_ z<^7(v3n==;Y~n;sJPd1s@e#%Va)@29SdoHUy-tjL05pQ^;{joufrd_4p^zug1OXyp z#vkx@EP+6Es!z(2{ZcX1BUOW)Ql03L@?@_>Ko?^@X`$uAniz;yNHPQf(0UfLYDVz> z2th*QD3+vuim|3qao<~BE1k9Va=NZrIx3rF z5JCF{vcNKEsjJ>L95I7q(PA>s87&g7HEknWL|i*wD-CccJ z-K>JhG5iKtfvo{xS|!us3n_Z$=n6{1O%zi%7OK(7BZcBT0!RRT*CSx=WtJK>b_S=@ zr~pr(Mkec;a+RS@qc+tc&HbmOxvO1jkiiy%bwPnV1Z!a^Rw1Y3O_HoZnlHHIM=)1G zrcmL$Nn~baV7V}TcLc8q8a%E#yzqC9?=viqv4qtsA#gEBPJjcTAen&{%m{vH-7t?h zI@Bo*-K}!NseMQQh)a9TOgYiKRE}5AmR=+V!2Uo20bOJO6##!2=b*ho$`-n@avGOm z1wH$ZaBS1%Mw-~2rT3E;Y_H*qS`0;i|D35q{U`;77O17f3ivW-75i2RFt77*vPq z4j~$aP(_+E3icQbkKwu)xro6UXiT1#IekZ@q3eW%QIMHN+Ex~z(GOm$uIfffBMmxH zUM2lyRS@1t!$nq966v`)9E)y3E4bFsciwo@Sc}591@E001z-@s%(ayN373 zxzr^MjWt@G&c;bS=*N*tutua3u@EmS%fy_wssB1Jh>9P_RPGtlhD!)N)5YUbN8=g9 z4Ki?Vfhs;EGM{Sd28x}}qrbp%u9G3;H7;8H?vpP>pb>j&+**s59*3OrHbU&oq3_&h+ zr_ss@!s79|D`xc`FxTCbZ{FpfC9v)T$&|0i6?sA*M5cV%On6x)r)kR+BjpIzW&s?> zkF{&{c9yOb1OOu7f)6s234%W*?FKFo6bCl+Fw~#{eKX z)N-T{iwFdVq$$%cb>NX|>_qkcslC!P+#zYuxF4E%qa0=fr*hPCXc{ilYRPmB-KTcn!@x>O)BJclfovrw=02u>M{}atu_GQ+GXJ- z4w=Q6>E_dsnD1@s7H@(xje(7V7o+Put;6$AewogI40b;!rQ6 zSx$pns}Fre0yrM>xg?6~)fDa;5zE~7I&hJhTf#B%v8kVm1UMJ(QBoLfw8P=G{vZV_&_)ZR}EPeB= zTF*;1gQMx#cnN6*2XqV!BV!3c@*LbXzzB6XsWS+gx=KTimvtH}62zwi0F?5~fUNF5 zAPbIdk`UN*J6aaYftKZRqI?d7D+DX+Kmb6Ry#z~S1l8_dy*@$2xVlWBdeh;}s>wsf zOD39`0wJ8Q5}#I83AXVVH<2ryi7Y^&6@ACk30 zFG0My z1{ysbN0&gbEsgg%C?-t@l#Ix5G$|PXNDx*+C|8DSq1y=oboAJu#SzZ1qaTGM(s|*g z8^LF35EGUm{;{;{fUG(6Dw3yCvUA}rvb*IvfJ>z++AKH})XXS&IO-JiuT`sy0dr2A z)>@p^0z1>8Ht{mMd7E(hCYw`EeBn{YzGfvnonL#ArPaRl4q0<*rYt+WL6#kSL1cbf zj@2)d1c4PmN!MPIwJYS>QhUv)U?Le5o#Gh7F{CrA#X>%@$;Nd)H?(Jkc*iWf2K&t4 z{27>R27C~iY=%nyx#yPP14@Ab>hyR>s%xtBp-sczA3}<11~mk8=QT?is?~?k9+u6X zv|&-6_QVVtb3{yoKa1j>F03>y4Z59>oD0ETMt>aYKA6|UY^)3zdTCJvQ`!MUD$oXa z*{LnE;@CD+gjdPdCHKmqh89U77#_<|Sh69xnHMI)sP(I9VXi#AV!Eu%rFZU)ZYCG= zIP0#QL~ox`3Qdj|Za54u38i2gvb8!vthwg!i?Z|}n2K?&hwImohElB&YmY^X6W@S^ zlOxT%S_Z5q0lP4OnQ74>mxf&O+p^$*hErp@id@Rm`NzJ(XTS%MDPBPHe%nR>$%0p^ zW#$|N)|W#Cx4!A>v{0wG$i7`irK7V4bmkPUqZ)QTgJ#++5!9Y**#{^>ASBhv0^t`# z+yZ4Fnr&oB0Iar0j08ymO{fCD?&#|>qhp_(M%-iPy!Fyr-7F~-@s(kJvsoZ4Ritv` zQJFx%>$bqR_5Y?7=ln>XKgk19m&TjdqE#TOe@KDoj4e+)Qd-TMZl`E~ofe@G034FY zvfEcT3&3!XtU2-qERxrx3`rBaTGmN7S%)=ijTvPoyqUF?j^O)n6NWNz%re|sta&ME z0PJj*$aEMpPOaa|l)5 zF=Su~3kgTiMiv5a&frnG?#O1T={hZ^nik8>xohP_WfeY$D-2X3!>KaF2$(!DWY&ht zH-M0<=GIx!+6|tEfa21lnVrQ7bXJxtO}+;PHXr&u70jegvH9{gqmZ%JHiadOH(=aZ z#I{KDl3`;j{L5m!57*3=$o!xzKDtSoTDNN@|E^ifr88I~73A`gUhGdY6j*mIX<`Ic zUMz)dJT0*oF4l-=9Kt zdZ*-&(wk=NfPrZQ&_l>>ilMfGRnHj$%o&e0A9n8orWQn0-r$GYafYn9q%0M=^?nW7$*VPAlM#Ydb7$fWNv8& zl=ZgoIyv(Cb?#`fDT}cIg!Zb>yZgp2s9H7n%mBoJu+AB)$X%hPy2E=*fZ@F@_ zn*g$8E@o9comTs*TBLvRowDl4Mx+66#JPuLchiliIjMxDMg?ritIZ-`8>RJgKF=Yz%g{@_v1x_| zpef|&66tPlm!W>TiM)D{QT@=A#_4(D=~YQWpk)CV5%}quMWqz$%rsJ(_eyC*&+^3` zn`M6MCJ6ngY+v$r!~qr}br+Tjsz1P11wddc+?>Ql`n86HsjL*2?9su@HPw?9E!6hW zDss~2XIO7Sjx2(QR)DkFm222lph6RX7)!F=B0L&w0=(vTNoc9xxdius(!m4xNhC?J z&lnER6h%kG6qfRMdA$VZ-5~4t_sNPQ+W;^X^7^csV29L7oJ_;c;!5K{DP*+;)!n5> zx&)&g54>iLT#9+Q(;DgANPr12dGKd4Ly=k?&uTGvks)B9yycx3U zhQ(-?JB*kJ>M%eF-`_P;HaxS%`NB=(9{+StM};v3Oft-Nxag;mLfwP9KVV6X#FXZv zDLWv`T6fFh6I&!*UL(8a-X@1D=SVNSa-?-fb8;Mk-~+Nk#R$-~mr+O}3k%ntN&+Zg z#n9cwWP-AIjfzh;@Ct=8($r8deSQ6yH}Z+W>d~Gd47NKH$xV`HeUNs=2rpTm0-ysk zFTY2Nfw=(rtho~159+=`9gM|uaLr6Qz&sHoLl{FgJt(K6aoM!+cDZ@q3o@f^yUZ!C zlWfgWM1^8%o#Y5W+7U7XAYt&`>TpWC6_P(Xca{$yrJdrd{AfOiOz8sifwe02O7z^W zsjrY7Tlc9;D~Oa<_7Cr9>ywSIL-3+U#_ZWMWbNAJQdw0FCSTNT$H?teuxXAkUr)Wx z$hbui;*P~}45a5ms1>2+JyIoAEF$dLEOSr22|~gK*+2JYIZ-he0c7~~q2ZQi87M|3 zlDcb<#f2*LB0`Lu$%VjC(=fF;>26TJjL3cOe7iKyipp^RfPChcJ}Ce8&7a6m9(zvSa`VkHXJMm!{?EQF$pLm3 zWa==$6(EsJB1jK9k_a^X3@ny(CLu9ca|xssr_i)&2yDpJjd8RfW-}>HBgTa6DF*6? zQv;fktCjfZxgG{6CBWrH7T`>p^(v(3M^AWUnWb^89Dt8NL28;%7<^bU(%F;a%-nId zS$QNY`01X@FgAkqV?TL|Ao2Zx9I9C$6)gj@^w{gN=)@~Bu&7cxqtzOS=jYnT9miE~ zmQuQudh?FyK+(%^Q{Ck9YnoN+tM`w0aWmk9$i>ayd3{@-NmZ1QO@!1YhYa8>SB`z-#FD1bG}fZ%Tm#gc3S z)spOx`E9$U65gpd7v3nxD`rt@YSCN>DXp4WMd8aaE1@FXN+g@a>+CRHHi(pf)bJ;d zyd?d7r=_R6O@8%vJ|gRHSqlqohpbz_MgsAyELyfu+D{HiJQkP32Ty8&;;OX^)RJp$ z?Llm*PG-%UC0n=cQH$^9JJw5I4_ssA!!np;oQ3m8cNPGkVno*8vlcyEn&jxw6UtSV zLt5>UrSq}xX<5E}twdk}z4r1(ZBsmZ#!OK5!*WDP$I3NpWc$v&vU}%liGhZmwJ=sK z&xTZv=S+(yt=@%nXC{`d;)U~(#2602jTK8KWe#d+7PRe^;aRJsJwi(fY2YC=#7d(# z30zsmf?V3oIM}eMK0@o3<8@Y@CId{LqEgx`^^bNbX21uLOEIfwJtjeeO^U!bH8`MT ziwhRal-FJb-FaDDPMkQZg@!pi#2I`xd)9mvlH}GRd#`2~jXSm$-c3QUb)rk6Qf$R< z77q!aP;d@+uf_>rKfgi1{p1)RAxxl-?T`UuW6QJN89B758NiT z^_B9&?>-~%eBgt!XlaeS0q|;`TPthVeMtVtfBQ$7J$HtD`d8m2dw1=Z?!I36#sBbr z`RYFhWa}IIq$0++)i8uV`i5w!;bfXT<^u7K?wH8I<~dR}JSYoJ9+5#X7wV z>m5R}aM3(jjn0OrPIgIO?*QJ5y9}c-qBx)CI)#^!Ka~*&S|4;~X4Ry+q^08sKr$@* zAOO4Iri*FD79d88`U-`T#S2UUc6Ilx(nc(Zag0zjCj+RFx$&lTvUt%_`QZ=#1vL4m zYHpq>Yp!1-4}I@RE&hCS%U=1r z|MfkYIjd3r>@WYbEL$}juEvz~qj&o^zW$Kx-gQiV?+<@f?z(ray!qOGO+g+}o-oq3 zc-MUH)ya@k3>IhWNP|>2EtUCQXmEvIDy>!ZAaVo+zN^%y#daxjrbdtHkX;L2z0KMd zl(xumI__5goR?|_d=R-*GdtmNEnPJXKS(B%lKp!QNd;mfH{7yZ7B55K8Jcncl;_n| zl~U7GE}d-yvhB@XDwra8KgWk#6p=iHqQW;LeFJ^6aN$h(2ijvsB8 zI#SHInh|Dc&08`@?*FN`$l!257A>ih)2BMY@reCVr*rrSx+;b%5l<00i5OK)13D)n ze%000EA1VFQeNH&2Ie+dcjIDYjUjfzHKGd$1%kpJ>qR6z&?Oi|ab^a^nJ21hf;g>%o4?pr4K;%iOs%*BRL8iHq0TMzkL3G{e%4K zul~F&UbqaOv*A@vX3VaUME0aKR?U&7rbg-N=)+j*q^70_OKMO9$B+_fQ^@|Y_N**CMADCTIHc8L zg?83QguS|rlETt^tq!8(>SJ1h%Q zq$^#AxsYrlD%v%f3kceH7}|Iet}Fn=Cw}$=GI!x@nX!F={Pu7Eru6iruAWdJ^?1nQ}=C=j66~Za}71NM_BM zC8thw%I zY-h$-E3CM-x@NQpMq{h)6SA+qTn2G#aIG1%FZDJCF7yS@L&_Pw?Z#Rqr9QcTyeobN zd=Q!9CCq1NxfD-Qz{k+xj{+it>?H=?=}(V3S`^b{rz>RU_l60YUab z85}?c8!{61?l>Y{r+ZPb2tF~mo07u``NrQr0g>H@61{TSxp}vI^|Jw3a>tO$TP6Ly zL-M7+`;Odt+r83-?vT$v{Y_BGM3Isk*DCNZ>T~|}E04>)Z@Uw7UMFAuKMy1Qw?d-e zvMP&K%8O6ECdd_#u2U!E`Dfk$LqA+;=|O~RE|-lX*? z+QAX7EiHa5a;?9F|9L=pd(LR}sCsVwV^g22(7D+{h_pI!i(gLQQ1N=q@lE{6cC#3A^~QIpY# zjI7>pjr5c^XnWcu;v^wx&7}UYo*)Hd%~zFMTqg@xSTq)sojZ2x@uCpkZ@z&vTU7N6 z=s$NS{p{`K210#RKLHzMab7J-$3z&R3iB{`IphraWaS`bXN&e0WzT9bGjT_<~U z^6jrb4Hs4ws^%l8>A`oxG5l1@hG%!k(?dIvSqE;bc%@p2A%IejEpr*e(6yd?jg<18 zjK3He`-lW#nRJ%Tk^XqQ%;*F-CA%ezEH!I|d?_}MPGv0U6^2DF#aaDhUL7;wgUA%G zV?HOJV35~T&DzMb%u)bBJNJ6A^U;Q4Z#GGxdx7t%S7>rUm3JelD{WNQw68kh;T=ao z%uS5OF1&`mUL4=i)x&B)C|oLrN2CcU%YD$6yW-`DGoUIT>Auw{oMdP{3z`$F+IE6f zvq&w)mfTg$mdYTW5RV~f&Hd2Kstm<;IzQ}H0(Ti}X>uq+jEBMe%k*5N(b8o!g8mzo zl@-_z6e9@8lM%S0yh7_MMj)73d`Nh%svw^iCo8V~PH-@#Orf_af+EjwSq1h*W>|on zUd$59!U}mqI=kAW0krk8D1+uC#Ei?K{m0~)$I<|*7{&vui&Xf?tW!E079X;<2c1GJ zBr+jwrlVQ0bzr8Lx&=8>#8NT{%J(BaGFXXtOm|x91`&s;Xp(*uHPZEG)Gk~WmrbQh z)? z!h(yTD1a2ETUtK9ve_1+q2>?h&`8h9ScQr)9f20fvG|c)^tc;+{94n<_>BRARjHGq zmwzLs0JhoqFa#X}+ieXssCPgmKI0oa2J1sebpp#If;a}jPLoHeZSy#q-DOlTc7CJ^ zdoF)11jO6Qw2P5KP%CS&V?fTqg{C$ctKRir(&kXTN&Ji4TWmkes&>6pONYjoDD(}} ztZs+#7|qtA3yyOPPz%Eywc+`VY9W-x>Tq5HCi=Z2_HqKL#)l5>m-4bGib453#(FrB zHeVoE!E5{g3`iJBi;45<)$}L^c*;NL9P`7uWK%D?|1xlK(!JRSVCw-OwhWv?|CO2G z9YbeBo`-~&N`FtrWLravli?@pI%I15;3BpH{UZ+^F6lw<%kyx_6>&Cym#c6Fd=Q!9 zMLpX?Icg9XfaI+UrM*xRbmtzS=p>sh`;1dOQHU0^sff|zSiu^zhgc%Wia`i`CObjX zhoc9sAqb`vL>}@K}UL%kuZcT>saW#Csh>!$D}bS8yN(xd!GzdmP;SpJW<4U zqR56)UM#qb0{C7g#XjM1TS0rzqRU|xR!W%Xg*Bq_0A=oF7e5jzCJaCn=a5^g0^j|Q zxRpVgpgzjR_s%-Q+-|HQCK^qPNC>l3&+4XuM8leK*1JZ#*8(dJ_Bnhgtd&LBATrLIt&^i;GUE&>LC^kNBAHNt=z#vGwCo-D_3@nr~7 z>IH=b1b_?yL`XZ&zU-q(4ClB%@_~A*s8`%}|5u#D{YP*kANF~3*6h`?WX>9JLNN`QX}p?P zYaoN5v`R#BK|$b`fy^+N8BhYnKG1gLs4dezV*eAe>%!%<@8Bjmab$;{ch=llvV75P z*r&$g26_qPKjGhA_reLTslbx5KgV7FClW>2$UVJ}b1&&bSW>_nT;r3#`R6 z#^x@I@hk}Xp+vuI+VQdsC)>4i;Vcvsu3UPP#%Q#%o!nT+!XvTq2rQsX2E7nrxebMC zq#tgqSjSe$^tK^Gt3fKu0Zt^81lZ8p3L!?5K-K-uJ)0#pFetZPe-{9}#-Z-{-P{vj z<@wxW=XXAT@2g`5d=R+;*U^-6uOO^g5GfSA_BUzDgNR-94kSU}k&@ePxLq^E-q`lK zHbh#wa4n$*+AnCy0SZyXDbm_MJtaYaPn7iP1D%Kq^vcn6r|dWm{;1j4gT0oN=QsoI zu9F>y0U`%v{i-_wFd2Du^Al2qET;K0*D27b?yj{Y9*|st3v#w0a}JQkJ(N5p9la-I z?dqH95R+{?U#EXgR*es_U2^=;PPm^^($E5qsm1G&*)$?sc0UKp1n!el$7Eg;oMN$@>_Ix};A_vz z+LgDU5D&rba8{0<+$r^S4KjZ=G;b999Xz%j736I)e+Dff7Z}DE6d0YmXi6vWlAVlK zgg6RFDoJM(j%Gn$zeJiFW=UJeQ903eSgybREpiYIsg4~uDt+l5she3L>lWM&>#$k2 z?Rym##-PMb9+x?xZfUFtN~&|0y!>jltXy`JG}VF;K%0SuWYyB^QElHPZ)|@}*^%iM zoTxB$qDvhuh;+WJya_kmxqp{zVf+J^cm{kBDe;6($>9n>gK<_bi|d@{_6ip1nnDah z1M*~+)n-i~gUt2HaJo&7oNNW1N4uxwKy$X#-Lwo0_efJCD%cU^W^lTqx>5#uNh-)r zh#Wlrrd0$VegHvi7AHdz~Tj3Qp)Y#13YO> zpm@1nVu1>Dv>cEf2R6Y))eCDcCUcjqk$DT(NU}974PEWB6dtPH#zxF#87xDtClVfT z`I!_+D>=Ppyd-7AyL&KOI?ZZF-uj*JDl$det)|NI)A0Ut!JpoKRY3M5B z^&Kd_L!+lC*a-)bNi;;#K^&en6No2UB|1`K6W{{l-%Pz79!#K!uL1#QWNRTXuZ}Er zYcZ|$>hZoS6%AZj5wH<6o)n2x%Gx#SrKX|*F`+6=)g@zb1U=@NX_ro*X9(ganuiB+ zPNoJ&kvW!5X5@6FUX}o4W`Yi<4=%(J&A$ftGQC){>j2dxj6{t~BsQeMD3W51pUtyb ztls^jx0H8Z>AmI0L#`n4Fl|HiL1fyd{^CxY_h_9*OoaXP2t3HE!pWH`OP@NV25T?i zMyf*#+clHhBRgqWI=WBFnq{}E%F9{^Qll`%mGPN; z8t=tT&1Ld!rZy8O$%|!x-B3LPMU*uF4Azi@q@{7DY~Hm21e^z@wyI7J9^E4?*e43w zmo!{|uwS%xWdH<>5m3q0V4gEzH%58Cgy1qvvGEnwau^K7!--yiQojrhb%2%ytjYm0 z6_16HiDGA(O0Mp!QB_3i84G4{)Iz*Wrq&?V*d8qi- z+${US3u?tRW$@iYm}gU{@1Qk-zvsd=)!BbqUfJ|KoKh$%ET17O7T<<_a#9hk1F*y; zlI}oVNKzV@l{Iu&UVGyuxo*{sQeV@cwIB=Uu9B|)PI>;7$F-JY_N>J+d*)IIJOUV< zG0e&$h*0w-Vd14{|61IOw>!TL&pkvSf$f`CWHRU#{MnioiS&oN}f zy`=k8L@E%Ax)pH^)ZRqcqPGq_lc?Bdp(A>oB>MYh^VXMS#qzr`|BdP6!r_L?)Se zeYgsqFi&6e%zkq~Uurju^L4wij@J@3K0RK0r1%)~KI0*bQD}vnrqzY1hlyD;+c>lI@rEYu~DLA#$u0^(9#Z%xB77 z=!=p;XTGuu7UH4M5b3}X2*X>hdp82VT}qQ)Rb7o^!$pSPA=WRRu@HvVX(hQU%hRun zyI@b^VZXvDw`=D*aIHqOiaZVaQm>9-v=>aEtC3a}`@WYWUXcbZOSRm#?rzL4DgE&o zK!QC{MnJ8ycKJOre_k6vI}IS80Z6TpRH9qfFTVo^jmsd42g_hJ4#HAwteqimX}M0) z;1Em0T}8KI7=W~T*)0GX2Jy3!!o0J+Fw=&|Z^{Cl4|>>*?IKm0oS~KX?q=62e!EfZ zM=SE(cihB|&aWLMA5fVtY384IT4ulpk#kPF_Yj?hep{&~C_g*_R=PDb3JQB@3%RjW zAS(3-{NZ>3S*+S37c%Sw2j9!8FeTL>sWcpEBm3F%eKloR>282R8jyhUGSR(+_uw|6 zfX`Bp;HRd1rpie2LXDu&EVnp({4u2c22u2uLIGkHfDr_DR&@nH236G=W?kW!0gy9> zaEykUS`}d0bAzrlHf&-Qe16K<0g?Py6Ud^`og7dA3cc8nyFP{qc&)rPUc2#Yjapo# zTN#e8@P43x<&V*(5FBU$5EMs9eFBS)ogPEr+{$HmpEx>v70TEbwT5+Qg_}dsUxypB2$tWz?YmhQ58sUAwn5+q0L%T-7+iwvv~KE*lXa4$x*9WQJ-#H}oi zn}m*q6UUbZED9Jbgha8Uau{IU=}z>r;&|Wip0gMV*Trh_?Y^-JcJh5G6M%xZ6{PZf zg>JEu&bfcE%XS8Q5GitLJSF%{ZOhZbiagK-?>0^#c*mhX8$T4BSvEv6G2y_?1RKRR zwc{LICi$b})pJ&tdlrDngfSbwK(~|~*1oN=708b`-CrmmOvIyT_=KRjj)mM*6yl7! z*x&TUBZlk_&RfozP1^C$Ub+EKFn}{BjDQhA4|;0H?A=b<55`uqr0DQwfUzG>V;(O| zFL({SFDmP$H`OMb9q2n{-CTkcs=P+*J2U{oNag49ZG`5y)KGJxg{c$jFog;fckq6So=AVkt6&Cv5GiI6o#K3kmR`(vOT3S! z_c~eYQJ(|}B+86yAVLAnbXg@whX6}EYTU58-<$ySE+#Kv3xs+Brc5+^9)*OAVYIO& zNDM;o^h5Ja()=K#sTPjZVI@^$0G$@g_<;F%&?;WoQ&bJ&I<;R3-7=(?@#`Cqd`ICkQqk2k@Nz$eh%_sGsI2jqpPUX^Wc?nie? zQ0Jq6igI)z%Pt(}IvNROe#?V?;X!>6xtO)MQorki$RsN_Mv0zaUwv!}6|zfuC!8q^ zslle$YFB{;f*XQ7O%gw7yeRsR0MbP@-T|=C=0|Sxu^OVmZfl;Z3-K|ay>ZD`{u6Gt70-ghWFQ(zMg zuto(1kgQXQ5=hBJtQGil?@{Pk4q9owS)QO&Q^dxr?jQC6AqzlS2^Sk%YUC5Y{Ghz$ zT{q}eCGpLw*y6m7FSZ|n<@naBI6I^*lCFnq@(B*Nvi!c3%(7KZp9>@fWHn-q+m83vro<)WgsY*D!tO}~hw|qfj z;dQy|<1Zc$P8l4qNBYrbHqkU!24>8ap>T!PH&8{+!cWiOcaR{W+*oHxPB>#Yo~h;L z)TFI#PHrj8zy(wWAPJ**G1%E9DNwZpQ^T-6@E!ubO74huy6hifb={@foNG`I=S>jf z5a!hy819uh^Bd(q{oyajqSdpctMizA=iC1(FTeD(oH~6B$qA^bK~|Mri?z_jU1Z0i z62PvIh>3*oet8N`J7i@I!8aKkNyyJ0ACRSkDf!1g{4F^FD*6}#_O!ARYJ9@A$sj(v zDslzPcX`f>4&&a4M9XDH%N)7ujF>=dzdqA4JY)8B~cg5lh!a3|fAIbrpP=O0DotG_Haw zV)bp)r_N+525zGOiC3^=P2$xJQBmOIHcYi1a)$CE(?zYRoS z?J#QxwGKgKW%WnVR~@t{!|>-bm`;!Z=P!QPGaejSs%aN{t5I7XH^4JwAp|Bg?_mV> zhil5z^%4gmVfy$9;94gbz1TAaaUuW-`*)Pq5@B5n<#WNH)P06ENcKx@V}<8Br) z-~YY;CO6!Chy3bqeM%nw!IP2%gL5!~Abx&H@~I1^m_Uh&M?hnrVk52`K#A95 zG%5zPxfBD;mW63Sc=oc$_jvcDLZOzg!jaa4=cpMyXW{OQ-x7|>QF$E)h^5izJgdZs zOc^!JKZn;LtxK^qQ2Sl5O?3YAn8N_lWpl_V>cKgBA$05OYo!j?o;w@enQGI z=vPL$p-OiPGu~vw=R{Epr`$%3MtV?w`Xe8er8mrxEt_7J|N3A5i}d!iOI=M3_A&R` zs5GZAwhSs}&Y2d%?vJ!WpwTnF1Q201L>OQhMBPb0sA7`AgfyeC%Sn7Q4GWYbVV_3E ze=?d}fb~N_o~Z|U5IM`O8R9~->m&-`qBA}_MP7drmsclL_%zV-8 zm`k}ifx|p~wvC0WW{4bCGiFL`#%yr$vQZQX4*?*^4Mo2wO}Wi6`-icD!nnkE!Qx)q zFUxH55~%Dg`@~#Hym|t~?<^08fu@x6W zs6m)Q0z&r2qz8<_BdAfSnl~GB0J;zm7NTpqox7BQb!z!qOvuR{P3;{T9*{*#=E^%B zxJ?EJy5)=i@LB2UIW09PuB69P+xxPMMjI#g>2fK#>B57DGJan})VA*qd{$nLCu6VG>k0Vf;x70b=n(l7m&1hu*>oGty(%Hdto7OPn> z##)G$z(Ek97R(xsv9$wdAC0*z)FN>*k8TySw`6!#!BB|kisY2kLV<5Pd zbYsdiU4&{y@Yom+xWsO_d6kq^MdgtneOtEg*a!f@+PH3w){fXL7n=%iOk5Yzz7RYp>A4Hs|oE;{y56eY)RA7g?z0@Mo06GKwlgC6fe!&NC)ERmx`6N<210x?>2U zt01-SXgvUf+d5AE&`)j8RBmzKT@OD)!NlSfy8~VeiotvkqlNYfQ z?+15ZP>9l6fHeg~EExW5CWz=4H?EMPeS3uej9E2Y{Gi4<`U zU0u8QAabttbPu~Hiai;i?gt3fc_{pvw{V8+I(%5x&6%S`V=V0R6i(Jq5J*heQMW9f z6vC4;v<9?=tO_d=ntZ3cA}r<24RGmHfiA<`TzYhU$O|foO+1-NQ9Op|w?^Grg~#-$ zULe4bR6s~ff~$$(#X1xX>@p^Tb9r7Qp0@Z?7Xcy!Cj~PKUAkn9ofvkBx{QV)l*?fS zR8a? zd2J3BhS4rJ0byPN6lY$*tdiTxpfBx~?a(W;kb*RnAkt+&9F2vp%)GEnPt6H164#v7N&VF)Fmn>u? zF|D4T06?m0q-+?J;hofm9S>5z7T_$$EkZtf-=y{7oqO!j=+~!(f!r@pkkcQZg;1gP zY&Ghp2af`WIo2+LN6DSZU6ivH$p^(c75HbQ9yIjLF(%L!Cs2!h^yw<1aMzw8ICPR! z_#=Z-0awt#V6XJ|!$Je=ZmG8v)|bo#F<5&>9^=6cEhDpV&tr+tI=m3-jQUW2lS(E* zpb1)k#NGnziDK#jz?<|aR7?Qk7|6~ehorh?hTOPvVTotD)HCQG?^4V_>F@MQG0Mx} zm?nn!;CLyc3>J&red{Xu-gnF8)tx)!-FM$9p}quMI}|VsjM9YxP9yF6fD~M#2*lq- z(48w$q8Sz{l&O8I*2(kY0|@Rkd|j!0){pt8OSM@n#91Kv%ba*R3$!{QK+2GA8pgGz zkiN=-M*<414|-HI@dxe?7CDx1SOu5onTc+m@CoTW!VrMetkZ}aL}KMqU5hS&%1&Ft zjq9q_`DTfvKz5i!wwPDY#~^j7>m_*qCGoT-aN0P|6?Hd<*MTsfQCBLw)VPdMr0|}N zIK_$gAB!Wowh?q+>;P@bw*9;0!$0$0X{rRPGm}fy9p`V?$_)4*QshdHJ{C^LCIc^kxab0C33J8Fc(G?!(ih9fJ;PmPEVhhEWf`+z z34IASl3XTnUb?bkbZ0Sut~LqXSXQ57kbQBmZSfnu+eYi6SoMSC1+8D_=gb)kSIMiJ zHcLZexxD`!x40`UVF7gwS?AqULBCk z5=1mZ3F9F4!__U)A4{;dy5GprCzVJ_`tmv#dEqg|32 z@IhqKX>}6F6GW=?@*x`~)2Kz(%y0bShvX0c%iqY0+uxMC*4~JC0b84*0FA;G^b!H6 z#^@-avZn}VmX2yv4w=h_o3Y|ja5%TtsCb|BN{ZP*Ari;WX=<)%`fB}f0Imu;h?ql& z{8*&AKYi`e2%{1$j*jQQ($VpxRUvB7>zp8#}J0(iX`1VlMjv_8Cn@9*a?) zHE;okoa8H{kx9}Dg^3p1mb%^x@XZUxeJA+k3aW> z+;RIFc{d0qN0p4cje$urRibU3dH zQRm`fI=w(o{_qyi2TQ2`Jf+3YJZ=NFk#&zap{Hc?y+Z{LM$#JfAssmDKl!>3RxpkR(PpJj=oqa^%`8x$MaXasMT@8 zTlY0l?meo830B%iqQPJK=sWQsMde?=`KWaD_RHIESR>`EU4W2HqH|mTd`@Q)b)Pv1 zvpPnlUp`w(1*2DYqNa%RSr>I7MeU@*nEq{6I;vncL9JYd&Z`w*P#>?;NTszVP?!T! zv2%u)C8dDkEGC_=5=(M--Fj3AE-mcLyVATMg%>X=>as8Uo#;~J{0`%(Wzk<)Yeg&} ziX9%+9>L#UhTMf-kk?#GzfYwAmUI=k;*Q{c{@#llBwiknKmCJWla=$E5SK%IjBCq* zp{`b&pIzvQ{qJ3oGvI?r(aYuu!BLOA2`~pJ8nmzn>K8w9uQb-y$~V69uhMt=sJvzU z?NZY?Q(0HTh=vD12d*w41%%PSHD4;_g%iM(2;@#;j|vvIu^5bT2!1h3eki2(vLe2; z$|w4cATfXuj!=etPUu&m17-e|rFOC^Jqq3|WfBg>5ZujRpAfQ!NRQ9@5@hmdp{19d zRO&QeA}q&4*s!!|myl|nJti>#$QhwwCaxBJmdKsvxe`rWTADYxxu}hMU2+*u;vAU2 zu?X-iG6+vz<}4B^2JusY_Q0e+vA!(~c9Yu*bHcfIoMEH4c0n?SDklZ;94ZI;SpKnr4=V5iT88-HfmjAXG#6F_&7g z3p4o$X&q(&)&sC;n^6cD2bgD|n9#))QJzbp0bO~P_Urtt?#lDFvwQ8`#qOrfUyL_i zST(rUdWMGN;isOIAy}IC|MXksQy+P!)WnfYf%`7Zc#P+s)ht0j8dAdB?20**4YCM$T~`_$uCN+l_J0fZY5AMDsb?h zj)3R$06?t?!gyi5$k}cTw@6$c1w&7Pf%@XH^0ov9h89gt4j+zSkn!XXnxnhrkCRV-Yf_s7qAA zOAJu7Ft4bSpU+N;8ah!1RGfd~v!bOy56iy&d!+q1+;YM~@%mOm|4TYU-Mn&)_aFrQsc!18EvB2^xbeE}%%@iKJ z0?waWP~Q1*4y~;rJ$zag%$*^>^&6j&=6X=b=Szb5w*}(&Ezdxi$QKr*sB< z5Gl!Y^8wTlr+Yq4Q&vIB^h-Q-brlhL@7r&dd+>9#wO5WDZI#1^j!RqHDe3O&22BTB z6uMs4D#%exs1LkRnTO#d(}xu9nKay5q}oWJ@l-Yd0HQD$C#}XS)81tkimC|}ZBN{L z0sDOVu%-H18YlaI6GQ-+>@%TMFqYb@E*u8sbzi6Ls`XOJ&A`bkhO`$no@g1Fn~4?U zSlS%GvSn6~5-*+ELQM7So+G0P!f25YjY&FcsvtQg+|5 zr#$-vjE{*JBA&7FZ5}&`^DwB+AQ zxP{|zLhqdQGKGRv0iZyt@-rQlxz>E(k9vgMRo;GFOC~vxW`^8luoVnQ zQMaL#FOcjZd^SP?N^i|L7JO#KqX0QCo}*{={CxJerh2&e9(%NqzD4`OAc-S{Vnf^1 z4AJA9xt#_5N;v=ip)S7}@Ij+Big_KSbg`MYeF~Bpd@aE87kIrhr_#NOX z-H1+`rTEe!;;EFn9GOaLl;C)57CWJ8U$V6)ml>+~Vg2xmBTRW_mA|uB3QxoNrcmcb zYe|zb0EWu7h2XTsoD4+VG-=F7aTD*6)RXl&tkYrQgV!lBi{j4!5;hdXapCNIsmFF! zh2DZXADAP8j%RInKr)K_dR~svj5@A8_lKy^c$X1tKtdzF@A1JNAHoNfb)FPUU;}rRE~-}eXeml4HbI{SkCdx;C)6Zx!uFgjuoooXX@GGtqz{VOFJFaa!pfJ&6F+*&+S1?7RP7Npr#+Q(q08O1G|>)8`19| z(65<1rkzrdPYHK|1M1p$Eb(1(=U?!mOS8^{bHpdYM7 zUPT6s#lwh|p^z0BX+x;tiA8uUWTug285veQCg-rcygYAl+T0kRc)+L%j#3uz;%Iid z767(%6K|?}aAHc!T6~tX`EOrlGvI?r$ri)}mO+8HjrWUNfuU-~&dL~#ffh(xeI`9* zFwNTz%cs>`R}_;J(yC=nsMU|Xuyk7aUL}f7boss)5MCL$QTQH(Ru zv5TMr$ijt-1cTeJWHeij^piYD17r;Z<$*48%6->@uNv_#gdUZX(r`Wvsy z;NXC{r0CKjP`C^L#*y+6nx7Efv93S7cN;pz#rDmcA$p|~f%4cp(SLQd%|Ov0;(4!K zh;__7IDJpw%NxLjcU1l})Ec9hL#t84YN2n(UT#Xs-cus|!KE zhC^9tsjHNpo*}8P2uT_4hPIx5Nkq^u9>Olo%$ziYP+eSxQvj31AS#vXC40O>f)Gw4 z$zf@(jY?B(R=RoyWf6FkdWRE|fMy(q5UN5AOnFTeg3i5K$v>wqDwWmMz$SnYER3VA zT~gazCbd=7l7c`bD3r(19T49?e4<0j0VMO6)Tx{2$dOKf3L1Mc8;n=XIE`8%qw}V` zWH`0~2xr;58!e__Gw5y3`f)2a(F~4%uke;O(P_uv`VKvIL%e9pEuXa&1Rt(Sk+tuKV8!Fb_)t*2}teH_9VF`mTKb zbDx*@Klpz6#3w!}`}Xby5J#o$^hw#eWvj$7&m(F9(bDuXv^+qtNgd6p^RA&FhLy+1 zJhJ3x*8k($nt`H0#G970X&c_Nthb@}{7m!|j-TIRpWey7SE}Q9QlapClO0<>zX(%a zpV2b`r$9ayh(52gHZ@5LdRqQgfp~PsqlQm+NE$)dTW^>z*Ug$EfA)HE(z7t$iQ&FtexK=|MM5`ke2|NKxV(T zb;_5%^Q8RJ$M2KF&wzaKnKo%G56XkT@}P8VJ0@HEhULHh^1I}C=aBr>m!6QsK)>9& zW~sdG?YGEZ{q+x|zN%9G;1}K^Nx(@8+IAXgM*sTIi_okWqbPHhG{q1A$GA`TkI4Sk zPI>5&Zu!82>!qc!21S`+IeF-$eEGYtOAms{@Y&!Rm~+KG-?Tj7>(|60rldl&Ys8&X zIzBytpi?-{4hqgweYRl!Sgl7Le~fQXv1f_ASA41kfY{f?ZgQ~pf{1@GrTe|_dr%G? zIVHdLYyVN+`qsPUbD#e!S-4=1?AoPv0mc4uT%LhL2LHWqXKPun;*1u@sWLtZ; zeEv`VSbo0kv}}2EP$F}hWy6cl$t$m*syro9+c;Z3|2LnP*I(ZxGiJ|`PyFm>+*T>x2czzV@W^|u+YvA9y}-?e(PN_50&!!S~F;|nv^?lm?b>} z1JXOtCsow|7O=y<<&K5&^3D?yNXDfc?vJ;vn+x*2kkpsMqUcS?i!W`LU%q#(yz!;5 zG&VHKY4B^kv9VQZn@~|7P0DpED&?-5=gC8lZIbF(NNTFkWe_wdfy$VaRmJ6R{_cBn z;B-G+Q&qUA3fcF>W=a3>dHJ1>-6`deQGfoeS0oj#k-B&_3Nug0p8Ze3ZBr+I_UrGK z6-!UZle@c7Y-!hTjy^9Q!%1=~j=JLtZdXl^o?t6@^K8n|xu5;XzU3uRjH(Ds#Wil8nSEo_k76aA4;RJG)U~i*(I7HT@p4aS!K12z+ez-M>ScnyO{lj2ijUk6wq1hyinvXI|PW3zsgGrSt0L&HZp)&25n7vn%E47k5dl zc>&yDF?s66z4E}ydC;71m1j1*E-yXuvh*YZQXego*0x^RwrQ8#ch4$${-uL(QNekS zJb)~+q%zrrlHgAaaCulduT z{*+oJue|~$-gwNck8457`PXu$?JC2hQAP=GfDF5}S) z_3rEou~yD?>?_F<@dE1%ZYyB8Drt8O%#zBHF_)If8MnyTcf!zU2L_X}Y5NJeeeDwY z?n4`7<)SqTRJ#uxhXqlNZgo*P)Y>XR0L~pZuaZ}`Jt;Tce7&@u>_U8^N0!V%?E_*Q zj8{DR=uy&G&VU)@9FC37MELkS+xP89#_jV`<(7Quxh!ikC z|1*+AWBaxgUj;{4^BaNZyT^H>)pUK-g2V+(0z`z-n*Amxhz_ z@t^sWl*QxnJOAm|<+j^zm0$nO-;&RM_J1J$b40C<-~avJLtLaozW0OgOI1xZ*5bJQ zPsB_X4GzoC9>7|D`V;bc$1!>S>8F)?<-Yg4LucZJ=bx8~inv73>*d(dgYrNB{Ik;6 z*DHVYhrcf$`tXP0`r4q@kW%b0^R9@*j7uto{DWW7Gf*^$Sh8n<*DC*jdVW$b=;j{j zLOE6{E2Eg>7|xB|iuqot_9?~rF%}z+IU5KfA(unzy`~6d&J{Cp z>l;9M0ZknRZQOIw+-8aG+b#9z;8=}-d%#s<0T z&SkRxj+^AE*E^v_SE^56;Zx~WEIUi%6#1;S@nKTT-28 z@}=E>|9j*sU;dgr4wmQ5o8ORo@BOCSh@kkM-FxJ>fBSdPA+u8ce}D8}WN@fo>g($u z_z{aj2Hap`Sdxr)p#^gc=U{4Y1?J>E?|ndCcN>5J>+Kz6{;Rbs@Cf7lP3mb{{gtT5+`Aew5VnB%qvH&qd9id}y0zfyVoilYiW88kY18%RX2xkl&I4xi%V{oQp|#28LUWuiu|`@fiqvH! z7wp}$OS7jUNU>fpf1bMYHg0-dKK8MX%Ev$X8F}ct-;$sCnV$jEb69@zlSk!-b?ao_ z;w2K;dr0Pf=f|@7h392Y7w!g99Zla79OA;we&%e4;e(x9FAusIdk!|}2 z5HG1gT%rfo2n2AbO4|BFHg7*9zw^^K$)A4nRcV7|5hTknvZsR3Za?^;dyze~Pxf!x zEz8!gl)LX)E7$GomgErP8l+vvenUtH{=t(6B} zlx$JsK~50j7vBq$cZ!wS1u+peYtq^pEt<4?4sVXk3h}64N`2p2!J)c~^#%8squq59i8&IzxZ>+L_Q!37tceQ z@E7HkmtKGs6vH@-a>bNCU+uft=k;3tevu%;n~hp6)jk*C9TPnF9zJudl=@M;CKt#F zAJ3&VT7W}K#H)-vg2e<+5@dp)q-rSfuKD7GB{iXA(fG>`&;FozPDZCWmO;zER&7_k;gY4la>Wb#V8Ro(t$Ty+j^vo z=LZlW2@C|46tqd{K_*zZyh7gh!0o8OPRqBS*dkB9d{9F1TDW7F&x>PdKX2+bmIe<@9auoO@9OJz;D#89D5 zOXcO4Ho!#~l2|3HhNxVGwdUr?p@Z^QpZlyeGJ_RVS%LZxUN5u6nB6y8?OKw{_78IH z%|MYL!Y9CE@w7=A!b8HQdiG$9hTHYA8LtXod-HnHSL^{X=`&oLCAhB;SHPSjUaE7R z3{;}ONF0h6a1$_e$cxBQKzNW7uyQ`;mmJGp_^{pShDiH|dScGm!c;DWv3UNlLGc+c^t?#F9`bz%xow0zqGi7?7Gqh$<=dIsgN~C03^gVNDn*Cf#!^ z&;*bmDpijCgAr8nlfV)0kbA2P1*8GEEOUV&1qkUqQNi#>2~znt#ul>Py5^*Gj7?@{4z zHc;|0(}vS|K7S-EnF%xy0wlRV8<#1=uAYw;2CW)v zPvxa$XfY}kK2qmv!i8xedfsBL<>GH={|Hy{44}6SH52N0%)5clnD;gXl6!Do-*;4g z^wbvFyyXp~3ARckj6$_I{JMy=hSAs=kG;`M=p@kx%mS;iTt(h}S_m%V-z&VA$3LqR zJ-cVyrx4ICWFLF1ZM%>$mg4+22F91nhtD+Lkdh%{t;qa}zf%ZO53*PLNTU-xHeRE4 zpWu#FISXOJOe#(5G|CHz9ThODtK)mLN&;}3(1qf)?WUH@-XKcI2#b_e=j>+zogYU; z#Rm2dBg=|o&fwn30Hg?dW#~Dgu{35t;qY0E9g!0gXn~k@P7IkuOJ=S^Cy%Jro3BGfN8~Nf3`<9nu(g4)=w6I0IF^UPzrdiy+^^yJs$@8QqD*=7Kq0kXG}g9#6$Qb_O~3rZV`p1%66N9FP7o`3-9 zk{NTtGJ7%Fy_P{B1fiWmlmws=TE784@vc})EfE8iD^-YJ79gU9gj^4}G~kK|k#CAK z3oVj!n}Q{Bw7XaOJGxYJSChr_BNn1{!ODA?TiO65FIMdjkB7?0A+3PXNFLo`op3Ww zLvnkl#8a7Mr*nTO5SbN4{wT#W{q4l1r*JssoMnUF2=v}7s9DhC>M5M_st_GH5J<{V zntOhiph61^_a%Ne!@6jwM$$ltH0=h=8bu<<6~Hi)hP8x3$DW@3^4gm_WaI14%AL2} zDWCe8_sIfqjcIofy-fB_LA*p$=6Hk>#!m7J;UWJ->J*0ELy z9sJCP-j3myh4_F_${7eV%ZDzgECa8`@+aG2ef;G=d`S+p?v*8rqcXD*8e*&qZk1sa zHIbg0cOuzWp&6=H%ZH6wFU)|ULTFUpo+_WeX$zi@@$ZefvAySIoHQ?HpD_=~S&nng zeVjbJ(0xjBejS6B80x&Lpwz}h1|sSRfmR$C8GvF80V#}ExuEeyN{IXtsTJfZ!8H@g zq?dt$Sqps2!xGbZS!?0}l7Wog2A<35oOkY7kgwo7qk9-&DJbgPsEuKARKb_yh06ePIxD5 z9q4|eRW*8Rm-1Ph^jO~Y^S2rmTy+P=`pdoN$0R46N`HqE&H%jFW_E|!1qa)6uWpyW z{`>zU!@<+CZheJR0zAUGK4@)&EGV`XtxT?gFk>?ux*+VPwMWaeM`!PkmkSfgiDKTH z%6)YF<7RlO&#yzEfOHlfy&arg`pLC95v)hhTm%>=(YaQ!QW_02f33VTM*TLnu1?#f zrJ?7f>(ZsMs^R1^EefX3!U_bS65h3X?ol#8FQr_yb)XoPA_k}fNE_C40(IY5-xGD& zs*Obif-5>ne)q~vI*Q`dydL~{hxm)|M8bUEf2h96~0E0CNPa=#+CAFF63D3 zwXRQHVORr~AfDZKI9wJbG~T2;H%60kt6lZ7g? zCqljA#^YI#jE#7%EVCp4QW|$CZ1}{KV*c2%%_U};KeR}++lhWoAzj@0i`~nJC)ssT zPo#A)R+nk#%t6y(MpFQB15E`s6PZY-AWWB=SIo(wWf7Z6WkFewET+0eMcK?`V%$XqKgFzB%=R7qx`OV=2&@ehZ3a{QoP>fASl$q^X`vl9u7*>U7++Pw6jf4~(>BtVsmpV3@`hAM0@T1j{KM zqkpn%YX*=lsu}3=58wES9BkVqH{U%3xM-6Q8r-N$9&r#)$MHI!nTF}wW_liCFB8QR z90l_@)!v}-=erWmZ~e+CkQryfj5FO+g)^-|_~uo^#~Q&xWICux`HR1Kb*@|tWR!H% zX3%0YbfdB9lEoe@f5=%XG?BDhp4(r)MEP!6d!^h=iD3ZW;x!t$_e>%$z^r4oLc_d^ z)te;s%Mt+L@={Q?#3uYOPd^tXQrMUgmQmOkB5UP3k4?Wu@l9E+^1Af*ga=>l1a3~eSds^N zmT_n^$Un?#wqD}iDAk0)VUVs43onSKT^Z1bM9RQkymAiue(aE!Hy)6;uU~9Ii0ML+ zf(Jp}Sp<>NPwTb#xcS$75i<~GZqLJyJ&VfLF0iG-g#}lI;2|18)(^E_$y5VpP%ccG z3D4TVLuej~LiN}21F7nH5hL=yS}+6Z8**Zr>C{%HU|CZ9$M+}@N$Fi6s4rsv%|+x1 zN_}DDQ6%tw__-&&D14pup}X4qb{uypd{JUcL{GzUvYvy=i@K$FdKZH+e9*bGL}h;D z;!6(V?0&W%W*KsfN=XkFE~)lOQD0XNhTN(_dGyg|<&GN{OC0Ieqh{X}OK?xGzz7H; zybSB*?msmC3{2k)g!h~}E!+0IDvRevC5{Sa66R$|#|<|C8war}fhA)jBcN4Krz`JT z*F$>T-K$!sS0y%m*K}2!S!JXJ6D%u8tq&(KUCc?{Ml3G#0_!SEC*sdiQ$XQp!a~Ht z8KV_JYeXGY_zwIeo?YQaqtcq0VtPy!W=!`rw~}eOrd&ih>c7q0Xf2yZ>mEyE)xq^2GM z1~niWuq9^?q(cyZVq(Ao zcesHWX;bWsg#hEdGujKCJK!2aO!;M1J*w`R$(0(G+8T6*9O;lZckM-I$U+cNSa&fQ zD|9qtjB8drEnTtmp*kCYnS5&eo76ZwRTlR_2p7ruXRrB;G8`0rR5ul)ZfQFYpLxdK z$l2vUI&WGQdu)HdYheb$+jh1|v>c6t;&4%Ez9`jM0*PslO8o%$mgzRltuaGE`*6}cC0 z6BJID*>rkSjE>puU`2=a%wG6&$i+0bo?GdHYnc4LDaF$I$GGxlAbjM|eyOT1$H;s2 zfl>lD6-M5zI8CW2SKeG+k`a6Bz$_zb-`Y-UZ@@yL_q-dF#p!DED-&4Wnz3oAk*(jH4vm+BQcztGd<&asWRVC0&Ly44BmoYDD^xC{MVYSd%?1;F^#nkFdHe6j@8iE8Yw-&bk z_RR}>VjZ9|>44S#>Y}8J&&}~^KdpK|)9lE;D=!WHnxv{th<3lcvSIqJ3 ziu8M?$D1q69aOJ_{Wy{tB%UyPYxEO08TN}-NF0rz%0V-qJAHnlyy#)NkrrlWd7~d4 z*ZCYRe=+$P;5@|QC@u!KSek7qdH?tTasiWiZG1)hiD)&n_LZO%pgVxKjpsi%0Z~#R zaBa-(MPB8o5K>RK7VJ?oG$Bnss1OQ4Lr8uIjFhV%UssLIhBPL@hu?va>IOv?{OMXd zU~n-MD=u}ucD)rmi-y2q)b7esfPGXEae-LvILIKi^Wk6QgcN+8V^{k>Kb}|UI;2&$MHJr#3M1#aKNKbL|2v_LoExv9}Gw=h;5&nChy+! zGQ0ApDCIrGkr1F9siKRX(`6TAA_91@;t?<$gFlM|p%eJq&b)AsdOu(H?TyzW3_2(0 zkfm%E)sOZG6_vAO&GqYK!^<0-GDf|OidiYI9~oHA3*{9XDu{SXT=WfQ07;P3i^G5H zE}Q|NtXp>z#$5sT-D%LkjO}V#^x^#-uZbCOEz#@-8YNDkV4xtOcFUil^98SoNt=p` zH09mgE6k8%9fS2&XGOK;A%%mT=%d|$Otx znU|e;_Z7_%{m5Ka4pgl6!qbVp0~Pw*OEhzOKBI#%dKfP*LyI6I<}J(#21I>b&m)%0VGHG}|xT0(To;&nwu3cW&(NIVMmX^j*W(TdQ_G>YYd1#>YHW4crT zFMw3HpFI=N2ooRB%nil6GiNT4pZmF=m%sjtKbGO4VJWXbjKm??MFWT%<{2>&bw$~2 za%Q5##P^`-DdY}gw@1W$l62uk$o=kr=SrRduQ~JR z9jFSV4;GihqM9!6I)z!YT0HC&Xc3J7#G`0+o9L2?G6a0nQMglv@X%M9z{*#CI_cuv z6~$avK|EigCw6BT`vaA@6_i6OvAL%m4ZNzeDH5Ua73CbnzLt zHXa1!F})aY_bfj%1a%!eASb}-+Bi`67N~Hc3NL~9ugMUMY2{>aD2=3m8u^Xi_>c0A zcl|WLuU}5LotC-t<|Ag}gUE&6J=enb&{xXXVHt$fFen@OsC$5Q2^1V4rBf~R@&^|7 zBDW+WCphu97JgA%_S_RHdtAC|5i4xef@3$&P8j=cvGY?XEYAL&(Z2Ou6{G0zGNjlp zmtk|%R3qKA3&Ls(is4p>BwM6!7#cCE;R4EvMr(k|rU~1uS0y)Q;N!=@eHs5C;SC$a zCIp5L?hClkGVqP(K#7w@W|%Tkv8Kqhbr%V7nPP<}$cwvC$;XeqY4A=z(X@vpPPWF*0;^bW@mBf;MxC;$+c%)ZfrL&tHYMoC_Xn{dvD0G#Ea727;L1u29LL>Bz- zh{G@|Oj}#Jmb2Qg-N0L|124L(u?o+Mu`H@WH_PbxYz_)#UVdmiCO^~hIYFozG_4M6 zR2xWPOaOt#hAcGcV~7v!khR z6_qH=9e&gn(M+&&-~XKF*|TTCEtHVWo8OeBOP9*cx4d0$Tz|Lh*uF*f?%5&Th?|g; zE8p+ln{S;jcR#rMqVTmnrWU&hutMf(3*k~H1x%-m@XuZP z8MvzKR6JgiOM^5I4|5j%wX+%UOEGxpgfL}*iZv87YT*S8(Fo z{IL%-_JL5Stw6$SYMTu7AD6PK>m?9hf{d+lNgx=X1`s7k(1`uQ+BwD~l_w4-5{v37 z()PPJK4UQGXc>$^_<*1@gR1aU4%UxL-OXi_NC{_yBOibV? za!iM278~nHU}c41RfWqmj>OkzI_V}`;nB}6%@;04)K@H8v`7veIxO3_?*O>f%iOth z<+|%`!q58R4>-x5L$AIe8(!Qb_k8?UL4ffu@#$OQLB;Egs++Nq|7li1DaFGjV|R5?-Cdg{{nl z8tH^e>e)5dLgQm%uyO92a}1G!i!~7e{7_)F_a)_2*ABFK?Sv(=K>GSyWN5e=uc>^} z&EYifNiJIn6A3!b?U$t`0($fS0EC5(&6Vi$9*Q6fDk!sC!qTV33J~f56JATkT~OF% zYvnzmD$(7^$Gfu}vMEST;Nr6CX|*lY0;z2;;xM?$DG>BtCtB*$LCUo%&S9p?BOTQS)m?{w{SYSa!x9yQsO62s$?b2S+B!eXN zqAMCe&UMy%bo_iPpqf={`Nw2{PDvs$jQb^xSWY#pr{&UwKJx?+wf0lUd*!IU|KD17*^k`iOog;%;oF3#2GdBJT2_8m`V z7LMgyN3$#;`$slibrXS6Hv>1vAbjMfPWMVz*NA3p4JA8~_n(4Q0D{TIJBy^28D5Jp)iz7c_gN5tY87xj5(*+SS^9q zv_^Ql0|3=C7L=bn^Rn#Pyib~%TNFstwOcqhMcy1rzUY9{)7+!csLY%-LuxU;!NEZU z*SitB7*t`M^W0gkNYN_oi`nRwgL4j`J(3>6HKYObDHJ&-WH<#Ylli`QO#+IEI!Sug zk~c;M%wy4*G*{PPoV8L@Qw#7w&5<^(F~IR%S;ef&wY|?(10wnmLb#;xfF@BZaH{*X zY(Kaie%%%+i$(2$(0(Y?1da7|{B|+g*4c&M*s~P7-`Q?U4}_E_SG7sHk3tZvv>xE1 zi3X*%c|4fW$--|G%7en%TeLFr~qhg;ZeQV-C(pfIuSt1&Y2K8hVFabWR^705= zNMT6^2PBytw6SZIxmBBwkaLVe8dj957@V(kPEMR`lY_?(NdosuhW+H#5-6>5LBpgH z6e#>@YHEaw)u@H+9vC&Ztg+bSF%hZ`3h7vb-oYM;qSCN>@jSWi#_Ir1I5yS@zDK>o zl?rE)jdR45@er)CQ4q8kRN;0p&&6b8@ISxuXW*&<5iW(+Xw#j-hcptD&Ve2oI@*t} zjd)P_MALUqJ4E|WFbmBre$~1PRKfA@ozFgn$2!|_&b1FeoAmsuN%M#C9cB?xU_jt& z@Ed@_V6t8p*;x5`nPw@OAU(IvYhcU?0U*&5fzTy2MuvfDoFj{1eLq!QXuXMWR)zzE zG8BTokFhZ5&b}sA2*->+Czf@*XP$Q=;WW{Hp2*Gb&#XZJS6~QZN;tX~MJh`GVL)6! z2gBfhI8))0M22{ zF4C9J%%U*0PVgBGATFYM83V->B&_PrOKLY4`n=nNJgE>r$37}Z@~$B}_LwXi)$*Ne z(GLsG?=8#t_s+Ht&XZ-P5`-bMuQY=RK%thFYmB77LOpm7f>p17l1(q(gs}=F6|789 zzaye@y!o$I(vS9eA;@v>wh*;34Vh5HnquTY>A zYJkP=!|!OCYYveS3T(Xy*dywa0#K?)$1;&9BBQSI`kZn~2W4Cu7SDk7;#hnsxB<(W zR5A$9;IVhyk^o5I#d2WgTJsQmBAwv;r^hnl3R{eF70lUZb5Yz@ z==Li7j!tNChw%5hrf1-)0g=4^O!f<<=Ae7RX=Pd)!ooR9SBN})J|_1WUz$534-xM0 zu0ZlUPG)WfJC0tr5-#$E2`nxB(~KlntC3|BCAK={KIfK_{FrTYwH8C$>hd?+B*T9W zR~aE>1PQiT6dh(1um~%(0F@3=ef#-BH(Gp7$)4o2lUzLo@R{S*_&>nRv;hL%Xk>$~ zJmZ0}j6C*Up>?M9GK!IO@Pz<|xGpuHsARYaEeVh~0qoVH@CwH*z3t&LsV1EuZ_O>o zP6;*vjsdR^t||_;d8Im&-BNkLDMWtfejD>$&zPF*fd1##pk=E2hu`OZcKlopKtZ1%+*TCGWC+gSbFBeU zflW|?Ei!_FK8~8&y-G2rjdGke9!oW~_b+anKr42|nE9YR$F1%ho}Yh_PmHPq$HDdI zEU{xB=hx^yM&Nr>7boWH%v6hu8*y=IyTzU(4-vLj9KX^LjJG&En%wWI0TC5stnOAfj1!tlz6U{~Svq`=5*QjPW#B~%wF|jloS;|fZlItn zz3T38{%-Y4QOI$e{F`nmE%Z>~VHSo}9g7NZW>0y`$m3A+=I7Sud=yxC4A0eJfNP&| zf6{>H1nT?h2+YOoSnx?P#>9EmDg^*m7%9WzfQRIV3X3?T_F*T8aGISJr0K|7r*p9& za<=)N*rK#FXg8-^Zzc$^^PvJR=Z3|QwAcWQ7$1`oE*|K{k5MOL(#=+-h#kjR1TLej zApT)Tv6OGU)yJLr>}M$A+>>5~zr(dU16K`*6fU4D;LKc7fjItQa@u||5ACJ6EAr`` z&|an>!eiKH5ccex=-hw$9W=gT3;|{Es?`hPFVxz>rR@NA(;mtZ_L6{_8ONmL;0C>lc`_VV7)R9iRWYFSNsCVvNi2K*VAUld7AGH_y6iO0iRu=nV(KOZ}>@y63NZ{aS3?}oK1PISL<2oENcA%32z(6gHy2toZkLkJ1%ss-|5E~UO z8&)opmx=pOy+yGb2oJTtIjj*R`8v(skpohv*HbelWmbr};EE0OUW!Gq+6=Gvh`QS} zK;-7v{C)-NaVf^&AJdrbzce$=943ujh(#WPmOP zqM)Es-N^qmyGZ@rv^bbu#S~Qtd|nTQpkfS{kAfdkTT^o=yWsh70T?6DUhk!z^4| zOtEEWB+|n>05l7PDYaf-3$s2nxIV2L(Sq?LyB9dm>e8A{<7isv!#}w{1ErgRs}4kz z?t&L(N`>P~l9gfyH!8v4y}-S4c&S z6+Z)49f%aJ2nB_u-<#kV zjh;@G$e6%|qcM*Hx0bq(Sb5AI>CCpGn~B03t|1lB5Y%udg>p@@dhxqt{;Zodb1Ib` zkZp$^l|!ws3m7o)Jp{nPM4Mx|IOFWdM9P8AJUu)#1n#LEs9)Bux>uU2XUb1r z`zO%l4@z}ag96WR0x=f4q?lrircW6FnI_FLPJ-SX8PKbj2b*NZ%vE2YOz_r3b@mex44&<2^uU5vm!>cKuBrf`$ym$yv!-D*tJ4^2mra^nwh(Wu*`7_|pz^~i`4V*d zYs4}3Yaa?`Xk}ADnN_!58sfKu$gf|P&DkqMi5_Y1-;ZgGp*P10nb~+fU~fo{cW;&M z{sXX>ShEhR1RVuy>z2rZMGwfH-otXT`yl#yfGQs8!=x(LS{Jy%cqLwBa7G2 z7x>jHG$CT{EKguu`emL-&Y3{tIQfej`-r471F~S|GMU%BQXYBkYj{1VF^Wa=ub0KM z-!4CV>TjfL;E*hyu}T)tzgv!;*d>8Trvhd+6O>i+??CqN40-C+uY#(*U*^qSDfh1a zd3kQ@!*bx{Gw^c7W!=*IVVT6_kz@Pi=H>5{=9+o(*lT|)i{{-dmElH7hvTwl$-Q#4 zZHJsZeN-0AT_-oK{D34gQGiB57R+BNPjCDNDCKEsfV?nPB~{fCQ0=di*^yi2)g3R$ z{-ZBT5IsIdvP3E3Hv{r?r5V_b-zGFBCcuV#E=1u0ij?=-4p!Hk?lpo2E7ciPCzyEb z0n?SOcj@mz{|x>NT+=%@pibiR!z~9F< zOCmc2%OWaeC>~5pk37Hi8`5y&DjCl9%8z&azoqo>Lla_h?X%Bl9F^8B`MNf3Q2 z=FFOp>4e(~RuUat;j*wCYI{v~KK(^mcl(EB&4TrE?9`iR#oGZ_9J9X=e_@3@ixnxv zCoudbO8cGIzL@CUVo=1u+Iq*Z+Mc(&S*Qjgh4%yhF2=gK_`AYCfcJQf$c+o$HKL@E{uNXzG-<3{F7cqGjLUb$f#JcT7WDhqOqK+ z^swJ%rr6(8zPYR=#W|uJ2SAAntv%Z#`;WgOs}|fL@ko=*p1E8Gh6m+z*8yC^5aJC8 za-&n>mOcM2hY#(SJJx)sUk47At~DZYq<3Mwr{SaY1#5IFfO@^g%ILgTx43tnca-s%BqM@k7V#5G~?XqM4(=vbNC#1H#1wr<`a-@BaEN;0S-m)3^ z&LFIg?Xo8EHkn^{yEN7}$dMC!B$e)m0uEObg}gHkvRPOnh@(g<36M%+Bxo<3?bce4 zDEe<_034BEg`@^8{Wt*Lt^f)j0|=t?8nPK{1VskltXI9Ku0*;Jy&4fO7B%$~&t=cY zUi6wRm|j>X;?#?&ADn+ie+I6C8Mul-!~+VhiWQdS$eWcHA`U=`aiw3vg&nPLV0ISb zBLq97_Xc4xb@iN<*0$5K{<;SMGzoca`_phCC8Z+TD61CUDu)m6k)ED|uoxO;@w^++ zOMXDQ`%c4s6p^Ft2jsf>56Y?~3uXI}?@Qlsha5!@@tc>t6YMD6^4h+~)%qFHLPA;! zv?MrBIZ3AbBpRs#7E}W~64Kst3<30{EM2@-j=c67fFdrd7u_nSk)4&wkSh!0W@}kk zDC`KB)t(+m1lDXUtHto*3m)Y@nSrNe<25>{He<+c01d9ICFP}hPm`rIEtBG()SrRU z&A?RzB9spht1tWv`XiD?a((6kwluG^-i`lW*(k4!L2)DtX6EKPx8yQo9el zBKuG5kd?^1%4R-|{cB~;?8Wkvm%a`VK_z_*B#cN$&XM+n1)WG$4n=6u07%lLZ8WNN ztqga(w z=@1w^CqaV+hp-4pS)w#2OaK5y3F<3$AA3T9<8=1{WKy-E@Gm0+iT(1-8()(pGjBsZ zNj(DKN083@EYf#hmtYj`B_!bv4|O9B@&josZ-gsrpTtnceyV%Byt4o6(%aXHUg#ax z9vFiD=&4Q8*@s$?U=LD-2W9(#ryxFurFllPWO@fcsF;#hxBO5#dJf3UmiZWAue`A7 zYtq)WS;}kbW$(5pB{SHCv9T?00PmlaE&E@9MT3fW&4{z)25tRhW%v4O`Zl-LYL+^8 zmRhNvV&PG;Op~=VeG}uK*`I-no`I_iMAUU<8l_dNyWZ+~OT_uIrp+2rN0Rf)Q*aST z*o6=jB_v;jknb5hC@*a}t&F>oSO~%GIJEo`X@$G$p_+S`A4#bZjFc9}w$tRH27a@t!>{8+Se= z+1;2sWOK3l9btIcy1!Mn?gJ$XxVu8ZUI~?Dr5{;Y8#g~9OA{T)2cV-Uh|IDR(%O0q zL7NQRTL{E6K;oK!4$z1pppEyzW}-bkG8(uBtqCeVqI6TCo+GQV2?WULW{M(KLEbE|{f^|y z0HZ)$zp;Ok-O&LY2!@l`5w5H(g*#7)G~zG-AjyVT7?++;=MEGs_zt(etZ z|E&HDTop5L^?`^k3u7S`YoHfJ8^wBH;sMDt_`hi*8S^@W+GOTs^)ze(2zpj`kDzWO zgY;a}{4MyKWh+{CBE*mN_~7?J|31P_gLnt2wyevbYRzawhHSb3sQ|R@ECf1Z7eUl@ zWa+*EK+sjCnN&oMYz23@L@W|jG zY{AZcEm>w_VSE{!GswQ{c?xA-gVBxXdqr)u&^+_Nf&Q59d4cd^kLP{axiwu3%6f@- z$G5nMk0wPekiU~Z17l|3DgzPgFAb4t`8di;h0G@kZLL% zpHD8-bT`HfI1)V{0EB;JZ5h}MiteEhL^`YfDC9z5$<+jb3at$3&FDofx~5a$(HrMR^?boZY(-(y}A zIo&O(u@oYv33{t;_t=3w>tS+Vl=XM#ec^~8mG%TKJ7lQ^G4DZ5vv##aX2$v=!k+1Ok~72-G1fnpp8<&v%JdSU6X{V+`%S~O-^*<8?uQLuDPuk>Ok1rfTJ98HSO zj|IW)uY&p2_xO6UIVYGp0MepJ4$eR%&&}zMZLl=CB4DHOH-85F8SrM{DgqH|)269Y zAX2B#!tKJ?fU@PPLhA&xNhMD@oq0DDD5GXQOmM>#zNdoPBlGhFT;Ufzh?qd=jX$=6 z+!=44hZnf!5gIHVYgnhdse*8ow7mQbDVg?f9-TBjZ21#VBeC&Prtopn7GcY;>R}qLf;hmSMaNRJ5pem>Kg*8ps3AbWu0*cXfy{R0l z!=4QU&cju5J{e?1W85lnUR~d*fccC1pLS7_ULfcvD^;P2qiz_m34R}qL%lQgxME_}9zHJ&7AeR$p) z_1LH*qo|cGUc5}^hH@}7k%gf`ckDJ{>Z$H`=7_9QstW%i~Q6Xndk?gunGp0l=4n!!-Cy>w++(yM{ z+Gn#xCODjz05D$8j<-a39TN`Gfk?slHBy0klq6c`X28Z936^U++$b1&!$|E_VXxL2 zv#a<5DZBiIe__T*fDGVolyU$_JlZYN(I5YSBrGHYfg}qOb{f<;5ZiFC<5a17LYr)VhQJlU_?Mc4GgVU zb&>g-KLh>@obL==Wguea277_4>jTAl*(^KjQSjg`%{=mA0L6jGM1s$N=)$)NfoY<{ z73%^z3y64!}A#vn{In{#S(o5=BJ zd$L|-PGjna>RwNum4Sd2Ck(UxXM6;9tzX|Kwu0HW#rU}UUi7Gd!SRZ&9^=#7~qy$ z-CX0AUf-KYd!9&V_55Y$zI_4^=4YHn4>#_Z6DsYaDX(J%@&xpTt~y(fOZOh6dMXi- zn{T*N-u<=*Wns%Y93qUK>VwjEx=-2%I;}ucgX~C1V9yW;RjplNl{|ztEs+$cSSkY` zVQfbcBDyH{>^>y}jVOM^{wZko(GWCzWNVQ=OY@#oKwBNkq#%N|356q4Q(Y;uYgfzs zo9>W%Z|av#yPuaI{rFpQX#Y;BtYvFOw&3NhTZGWp*hyji2@5@GePm^*A65FO#Akq= zz@jTKkyr9sk_i5_`ZI8zGjJXt;t4tLDLbpwJhfj3K&$)t&J%SJ?Dt^tWraMQ8dHxvu`vDSYSO25$~dKOP6 zg9yfOSRQ!i&&p>$@~aXH)ycN)ugW7o`ghs2W4jFYcS~Zp4+jDLin;{Dk^#NEW-X}= z0bz?G#1LBE4uZ$4BH1G!KRSTVWAe~{{v|n31$zVTunfSPJYB2}p*ze##4eziZp5_k zB3PQT@@jN;M2p_r-y-jR@B8G&757V1%>w!RfBs8(bH__kR*9Xoxt5({JOq!iasY_7 zG_1JivNO4Wz!=to$-0J0n_wNL@6amk)94@VT9|s3Ktw7emp)W@DL-T2G)pYjp#g}a-Er2tdU1BJ^_bS9L0wzw8Kq}RO5C)Cqo>AwTHBDjO7HN;23?g zn(I5+h8DdD+z+D>)~U8ud2`Dv^5|pVmS6dm|0r*N=Lh9?fAhb~U;OVsloRMJA1UYD z(RJw+lwt-ax_$M&<-}Vph($hZ-XoE)lvf(Zmnjvz&dJ6&jh~OxOpLZ=+(lo`zYY%#EhXZh^^yfgu}9|7=aEU)i(^vkQ~SwInD?X z2revwRRFwP29}s_O|{01dVdrJB9!A8H-i5WP~ext;%I9-EPwuIe++kAQ0{y0Ps?XM z`HS+UZ+#9q6X?ds_=u+cI_u1<^(Yji4%iFPA)pyr2Dl+)3glf`m*Aefq=)3~bV*O( zAKagT$!DNQcNQN~0)F`+$$clMt1XYz)0K$}f;qN{#OP4Q^6^n0` zwJUFxje8!KiVBcN!t^n>omm$)$GjLffU~-&tnS5VX9z~#eNgN~`1|=Y;LiYOphytm zGwBJ^!qiGn+xA3Ap}?N%33dT==MW<3sjlg%YAVRQu@gWjI4oV%Qhv{TPYdSq$XF-| z&yET~1GK!=6*FYb^7Yc+e@g!CJKvNjTzv!(yY}(@G$?=zfpYYO-ZVPc)gkdy&%p39 zk`*l7w4cA0OaP(#iAo7AMXq65%FCkCe)_n4_dEY8zxxM&CO53UQ#S8;8m=_59n*Ej zUM!>MwE=jObLM@Rr+|0hZ!h93Pl4lkC@PeIVBr^6GXgZQ_nTWaYJZ%ah<%?VV`?F!pC0QIE%z-++%$UE ziE~DQ$nTB%=yj-Sff(S>g+&T?B}r5O83!U0PNyd<4WM{@RmK4#8Gy)~g-fL}GDn`> z@RS@oa#+gHjgSDM8A@7wXMHuejkhMwxq2Q_6znu!2v4xfxo-#;D^dy)w+?hYkqNCV z9+b^*ye8e9t+H_5Dygk$knW+Q5<*NTZ%q^)T|s){`_W4{$agm6_s-M{ruydf&*#s; z6*&WE1rZHe!UvvJjg=eY2-M!$uqdYQGxZapQ1yRXK-y7$tA=YLrRbUncNR}e^k#GX z>oIb|3Bb{PENGqgCwXo+#vVw<2({UGlt=;2xB&$&wSJD%K#||i!VN@1c6CRQ%@(f} zn)G}oZTPqHa_QN;+68)GoZwMqRhif z^EG<>!mZ$%&g#-S%l<_UcUpFH7pYx4xqhsa^f!M7rf~+&3L>6%r;ny8xpc`O=@agQ zOd1~dFq$Xj&|Xz#FRc+0+R@L9Z-f|_wYzw)_7GfGL-A@qqAq0s@BSJS+06}XDv=<{F%#tY- zp%%ioa>0stjm~;+Pq(vrbX811pHVL}DuxT?exVH3IaSNcY_x(%S$=iL*=a?ry0?p( zP=-KiiFrt7nOCV)0&clJyieSVS;H83p18tRFx< zLPokYI z+uCwVr_rY)kF*5`&Bs7YAbT==qSJc7;(f)lIMi@ZU~&0qo)uCI297zA2MXqP#UxtvBDGvMXU*jYTxjx)f=eEV&bXklc8h*ol*KaXg7`c5GjL7Kzyu&t=nk?6 zGafIK`o_7^-rfosjMPf9xZ)X)AoHiWX}0wAorZuN)V|!-Nsfs@CP{+Y&}h`}KH2P= z0WXp&FbA2;n&Df?zRZf*lDXGP7kK-b+Yl_X$vX zm>>%gh`qXn|F6CG0FbP@&V*0ps?I%mdh#%N$T@>R0wh3y0?M*+j_X~A^@e}f>s@Dg z3d|_ildY zJNb;cV2-paWZHG0O~qs`)_Nd($C@hIy!wCX+6+k>UZOizhsD|{=bfd?ZJI55jgP-) z0{~8qL1GOU-i@_q$z7^dxz8-!Xi`0v4AkQ^QLt-`J(t=QwH2tX!0D~Pa1gQTAnII$ zh_~n)?3b2FljX68J}0ld{v+AE<7J7&P&#NZCyVA^FL&Pje)-yW{t@-O!RZDWj79)K z)HouT`J#uA%iYA!Y}Wyc+7)PT&=>ur*vVQQ=(~w$3KKbaU<;fp>Q}*oLi#9{@=m$S52M!&O*WY+mZolz5B`Xj#@e&}$_$LGP6rcJWh-@fG12Ym`M4RL| z(uef*CuG@zYh>frS5(c+iguSTxly`%29Y+SU!q9jmp~q-*47Ci=2__{>U4*F^+*BUkK9*DNDmXFp|w#4 z2jd|A$Du}I@LE)}u4bS%n$XxHiBzv7lLNRnER$N>U`)&kiA)1c-ggb&6PLGSHJNOa-?I2%$YY?29gQsIDS$N9B!8n z-1cettF8ZCuD|hinTS*zPe1o9RY`sOsi*YcFb2i-V&SGhARAP*-C*b<_wWZ${}SGItE4{UQj(|;%rzm9hF7!*5uM+GDdiQvBo+GHx+YP zG6VQeb0cCbnp|GnupV#~C#<9->Ad~yy{evLdVgc-(Q>;CfTWFXn0ii8#v{l&K&<4s zH}J=ZeeR{oK#*Xmxy=S_-4QP+R?_5|yrvZI@YxFeH7q=n>t1Vn*b2Z+$*8QTizi3V z{70IA4=K1|mc?)I#LD%_#tp0GuG{W|N@u$C^c{xR&vcnSX}0|E<);z67L^A+@G*%G zq-DnR=|~e^2ba>_5 z%iI~$<%OR-C42Ypm4_etv`naP20+xw`mGyf#e2_*FN~DpunqXY{nIQGA51I2_(J_4 z{z%m!O_<}L=4)zdte5?V`()37qY?yBr#a%4*I$sI{rK<7eee6QEMIc9ta$r%jD3&P z!87Rn_kKvWZQm$ccC3I@WYl=-aK*e)kgnp%UPv|oh0BXWewC=CLBg=zKT( zN84!Er9&ZsHTkdr;?hF-n^&lzb4+1oMC?=&H?aBI^nJpiG z><0PRhkpY(wUV;;$QA(OO4)b#J=wHviG1pjCuHf8 ztK^M$pOcf_#~>Z}q`hOWyt(p4`RIMWC>!>!kkxD6(p*zioy?eaxg0)x1b`7oekeE` zFfb0wmqq~Gy|=$lZn^GiIXQ4hUSIJdq^K@ks{rbDq%$zqhx`ECet>qpdV48xCP5^Y z8;Wm${hYq&wreO*s6T6%TT4JragM^ z(QB>%Se3U~yRxq%fTZXW$TW0a{aaM zgL*0^Teq!~tFE|7CN?cZEsMpnuX8(8Uei(IVwTLEvru-UhD2X~90Y!wOrAJfF1u_d zY%>gut3$w9v!?Ct9UPFCUVBRR9eNMO@7VyhWspX8%N0wm0asrqb-@UT{3L)wLe*Az zjcK8GR6p82CSj<&(isr(5OQXL8b0%*ujBqba@Wm|%KXRwqkRAQf0W$^SF6`l66!K? zx0y8STp*!flpHVY8kjG=vDA1EqdL97QprJi>IeS-+ei>;jOSyTMkaXPt+FINi3% zC_o~F8Y_!EqXg;9_`?(VSEc(}BefN{U{@eGsc9l?6i{CQC>UGdgBr`Hxv;{9z(J=T zfEb^Knq}MW4f4L*9+kPXuaJq8rpwDec~jLv44TM5^bdq{5{{vGP&6-zGzfGcDba>B z2s;h#P?zENjuQtEb$n75q9{_^q#5$gs+VQ?!s`G^x5$Ks>GIyjSKua_mOE~JP_Dh| zMySAc!a$9B6?8L(k(|UjoxuYh?1kDX35fwo6rn<4FxuqF69712S+)Lo*}m@$iPXW9 z40ZT8p9s`kM-Ly8mw)`U1Y@wrAf*z#j&T5U7pX$EdGJ$hh{(>l0GW`N1;jS*#t~ zGn$M>mk)h^uH~j`EtJLu84Es|1oHu-2yG;EhR`aIltgQ<+6vTGps)hLX%kxHP&*Y@ zB#cR`*H#U;a>nZUf`3?5ZvVk;ny={o4}3)8@m_c;?J#i^RP%$V`cQq2Tr@qbf>7CL zj0XRsDgXv!CVuVh>4Cv|uiSjgEijn(%FZ`d%T%}t-+t>oh&eeSd-iRW2G}QVz5X5m z%qn^M`LD=r@Bo|JwnP&(F)?3|g^$o?7zXQsIE?n$xa>W&Uw-t`_u!5klu!(5GV0^- zsA2#lF149t{jg2=A$hUx9)l25APg;yGt{%JD|tc#^18d*rEO*_z6;5nw>$!XP00^l z{AUo6gxqt-gR*_k8r&DhT$!;J>!tq3oG0dtI2s?oGyz9|aHs+EnyJBY1MqNbh)qD0 z^aGCV$kQHGiBbIwA7pPGdbKv-95B9*sw^7Q zDM_h3MUXl7cz1fMp`hV16R-veJqJ-A1?QTepF};Dw$@2-{buIpbN|v{Lsb=2VmCYY zQ zufPKf9#af71QiKK>Bmp^OV;5=jS8rsR9ypbqU}B$g?<^{OmzL-3eTl`Zg{_J*tbvG zPaKrJ2e!)luKkd_v3VWTV;BQK;>fW>a>bR)Q8(ffvIJgXK}a!BE~yJj2nht${0d$5 zAUwU^S^K(Xs+}^qQIe2=n&Hv)_R5z4R{J0|#Pq&Is#}&`ex-coGe3j2e3HoY$jh%j zD>J9hmb-8KfE+k}0P40n#BEHIXI}h<%%8giu^>0ex1RbZ5cWL!cuB6h@+P_Kjt|O< zKmG>n8dQJG;kXTvE zxCIKH4>dO9bC6zKn)YPKeo`>*a*NKv_)o1%qf#_L0xvpETLaHK?nACl26}ul_sY3a zM=y-?;6)}#V~$M9h}x^R0_SW6g4bLzRlf7RCb;3IrKJff7O9p}I2prTPcE<@p!CC7lS;iy$!Zc;`WQSp6#qcOO6|43DSH z(mT)zcUy4VcsP%D?vZc3{I|%3bP|$71jK%~em-<$r$&nVp+b26m2b***W9YVzxdks zAqnB3M%+bDe>>thz5?(-)j}Bjg8;0x8{bfXTYzMP5dgu?ox3p>M3f^KG7CFKBpQ&l zn^s6qPaj;7>lAzvsN>CaCENGBh5leDpRpK(;+U*h{VW^=cELMrsXY7gzrgUm8ETAX zN#_RS`!9SIiT_#wLP2@y)u*Mi=b(w=2hkxzc=+fp*|Mz>63BFn=eWGP`pdHKz;?vd zM5G-q!#{zIsBeI-&Zt!mH{u>t6kM_LWtGyRbtpZH;1t?d=n8C9TIZ4koBoG9f@mkR z(fJNUc!e{LoUO5laH%%=8*b z5adNC_aSy@N;Oy$y#@?*Zx=#AzJwItbgwUJ!8D5dbRbY>&LV_7w@k03JmI zIAcBDTJ<~t0|kjfb?7f8gHUa~{Q9#Ry-nK+V=HL?$io|I&BhmCYk@2RyMm@@38iGm zzIWlVwF;6`1mFU5I>zTmN|QWdUznt>{X{!#BCjcD$>`?*fHD*T!N9feZTvCD`v%l- zi2HzK6|DMDqwiwco0 zy3nevih>0d)71R$6h9ZuqhLJFu%A|~rSU4d{!MbSBL)YK33AO9mt8dTtxcr10@Yf9 zU=xV??YG|||NNbG;J6|(0UU7#L_D95nUo=vLdX%*U!K`v!)d5}sN%q9jXtN{0W}{` zXPZ&uel6_4&)}$`^Pp;rWG5PfgyBzVl0haYjK)}1j|3f`5hqdKL={d3zsEtC2bGw{ z;Av%wOwrg(BQ!bm6zm-!+)&JGZYgDjvye7G=xx3!7H@@P%V@M$AX+O}i$AG1_Dj zMFbnhAa!Ihf5hVHSZS0&&^`dzcyMvNbU6+{r54q?9&FmAC!t-*tf?)obuK_;gp=j< z!ZO+|BPF@-&!x;%t%t$*q;WsFPm~IMU<@L1;%HDFdGr>U3Q64)&|j@?Ywgrl;DTNO z${X^)-8VpG^pYGpxJ4S5fqPe@Iw^%!YAZKy-cwhf2N>i*XlYc|s&JZHir~OXbIfMT zW=$X%Q=piHfF!y!Y}4hI(Z=}AX1HbYMw1N}_s|eU6N_kD> zw52Sj&IYNe(XpF{mXy%)vq*npUiPuqAV@aJ&nBKi0A%{FUSbJcp=>X|p~0WO5j4!m zvfOt12bE$ohoX$>#AsH zCbyHpWe&s|L^ZnG0=5Qs(sw|yy)b{dRQ9Duli+O+*SG8*IwFcE*NbR27yqeo)$?|A?Z)5F_i(& zeD#~(M*IX^Une3}M*!Jpef>%xnDYt72SfZI1q|uy`M^=lgg+RFl^fO6DW3Hycc4qZ zD|<}|(W|5#M_|3%Q@GF5UTJ2*O4=^lPMLEsY;I*dt;U*vh2vpO6Q<`d^poICRT$BZ zUPcURwAp)yD0>A5*Kza1LB~XgI3jGf*#g?xrCX-4yz`h-0G=)>~xJy#gDD$l((dv<49Zo~2@OQ1}Z zO?2u(?h5v|c1djos<#3`dSV$4)h|E&u@6c+@<%=Q>Ps^J>SmcX2kMw~90q52L8;n? zU}0Q$i=HDZ4KU?8Rbk5Q*Is931(*<#Y8XdLO@&1BM`?nAa!NCuJ90&%a<0pUKC8ON ziar`!X<9eOz|n6G!Twd$#c5q+85Lz_$OLhaWNr!~kdesrKkJIEpQ$o5NfvcYRG8*E zX`M#gj`rZ(aP-Iw*}2Xy@4NN=^0S|L7-6@H%|nTEGWiu#bmVzWxDsY=oGVznytV?>SOJvYFb@b_Qpt@q#6t4# zo_J8^%$_1&`NmVSZsk5{o6~?iR!wlpWmGyiJJ<~JKGfyVG~?z0HO6Rjv<)cLUT0$k za&%^(-HNH;3#t}ln5TWpI2`aNoB6haK@Dm8oLbAsAzDpXfr`O~23V}Wg%O0D2C0HR zPWuw1`41vPLI5gMqrT%LJz!$rW?6z*4%*Kw=OY6Y)^M zJ(VvC_B35TcJB>{*q$ZddiKZi<~!@<(B9+H)SQ#X7PzJ&3OpEr%We<`X!WP(om49z zQk_mMG_2Ro_E|Y6IS@EHe!-bq)ip*OVNovxRQ4dpSen6*1OTh@8e>$3zcm!N01S>c z(=MRuASi)sHA&i_HNBQfEmoss>3Jc>q^c`nfGw4%M;)7H%)8OpE1VRjV^pY6rU(%{ zW?XQhP_IHAEY>Cwk->p_>FbUlYji*w11)m*4cEv=AG`+{5~ksP%IL`Z#)4;>D#xy2 zRp&C!#ysjQlTny+sY(6gT*cbuRbByD&jB7FM5-9T1BUd5`j9aWWI@|B`R!l)v^@Oq zL$YEGQl@X;jP&Qnr2S+Zg`mgI3wLx$u&y- zMrgZ;Q@sfKS7QZhnIJ9&OX~+$E!ldr9sYh9KuY@)cd#Z5+#XN_Ommo3a)9hQsZE9L zT2OE|Q=R2na*CY-&R7kLN-egpYLYo)0zlyB;pwtrJ93gWV1C+{1(Kr9=(T4{HYNHZxdRFV2 zlFie#sDVheR=3v9DXsvc3C*qg@bf>aKMaS%*al=4k>zboaz)!cl6>eML|zZd@%EEQ z9>_pCtH#m>3`lAEfN%50k81#>7N0Wi)d^IWoIQQ2&MrWAYAa(@1SBPP zExiYX*hQkFab^~k<&^pQ%%h)>k35K=Ph>=JWvtAIieHQjGT&0Q zM*uO&-0P4@VB(}EnLH^5S;f835}m%3IGeN0HWfrw!mDiM&dkisV_5)lcxP#R6y{%h z)mGqKu0W8UyXJARwJdCTQT3Ili>Ou~qW&ADZ9zC@G{x{Y^IVR()^~*~kb(|B-N%dr z*k5e)cu8P+3Bt7Ee)G2JG-jpV(-jW4qRU*rSIwV@IyOxe-81Gzbcw?#j!JURLD!QO1~S=HRUK=S&CQ1{&g{Id!;%Y@At~#+G|V4 zbh`6b<1q*z`f$ETeFLan56-$?(}$GIwU)0@0OCZvLC)QdYxTU{^57lr@}?I zmueQr)BkAKrctRf9Zn(N7_$QyRaTK9&W_KyBs=Ic=hFmzGG&iBn~bwz92D&zC|-t( z2)R_{8D*ArYSvE5$vf4W)fnwqY0r(qWKkG8Nat{5XK4Xy-JI1GD5X{Uh)pd?8}O2)!h83olxCHqVU>J$Hm@n6MLNwf`X`}c-d93# zNWFkaSWT>qKH?;@O0}r+D~X{wow>RFRKl&BR!DZjUXWi>2I0kI_I`YBsWikpV?57- zM#d9{AemByl>`^en()Tw3pv&F8k{5#=olwk_ZlC7jrFiNwVFyllo;GO_1@@!&NBvs zHTHV%-dYd}U`B0)iX|I4` z7FnE-$H-=%DDWwyUkqlZW|vgK&@y_#550yxBle5pf9wNQyx1e%TJ~b^!#f&{xj;aO z!(?3zD-4*2*p8jSex}?ozi`zGO$wHx*QtPs=~D4Z<24M9m^cXZ7lh-SON!yVs49I# z*p7EdLtvmMNDqYoYvJ`6{?*HrtYo1J|@}N4)xB(XhGPIL!6P7om^i9aiUbFI@O7v`}@pUGUH$@RwT>9 zafV6IGR^#1&qEbBhI2pN+6d0k3TS|`Wk&osXAH8getPQCIw+Y)mF{~?3nU?bQZE{^nIw?u6CI*UQSLb z4axD|IRqj`jl~QF*|N1y&YUEYK=Ct{9N+;qap-1UjSrepwW;#Zf$UflzW{#coVec& z+;fy3$e=FwihxmXR{4xw083LFVzpMls>SJ4!ff4m%1}AOhw|Sk2}bz| zXOPd7K5OPx1CgpMKJ5Y&LriLWE^0cJMO1Z0Vkj59@o0y9XXADVB=98CG#ukIa*sAd zHE_}X0gfR9V+h-3bv|x#ysJYVSbCYA-y#W%$srRey7Y_U!ql=RkAmviRh5s-H z`+9>OL@{(WyIIZd9G+b5WRH`ucS`!z>3ec-E&3Hda$}AM-`}_^pNfREr-gr_b zMo>r)51L_uheVMYh*V*3)f9yXTWhK4%M|NMXG{&2lZftLdGsWhMnqQ=M7#`oLG9uh zNa=SYU8CZGDGr64{*uyqCN#3sF%@OFI*-Zqv!`prCjH>plE|olWxBK88HV52!c^7- z)JsOps zzwx$xI`jATb*gi#^*`nrRZml#p+vS%`cVEn$>gI@W!V_tA(g#mH!-goh*XKpQKdpx zHxr9NE|}7Z(N_nN9wroHo+s3tSJxVAJkf{2KE0*-BYkM>sUPVYOFT)$L)>L!yVq@H zivL^^N-orX=PRp9kIFz~w7_iFf&&R>Tqj7G*l;gGR%vm|G!qR+BpOCpWH_k@=x%IG zeuaar=}zou42YD7C1b5^CF@`An$hk%t+YE7FVvQLQATprX%DSz$KbGDA;dRVbZUk zo_RQpni#dD<@0dljAD?Y3HkRB5Y#0MCL4|)zD_+fdD_PhivXpu>$PpW_-6DG} z{>?ze5mduE!9_zw#|0z>&SRuCB509}l_1WjvPe0p>J;|C)RyNTIVV>fc1RCn=tVhQ z)hTo~)-B37*xm|+%6(&bHeN$U&oe47@Kz)vQW&A{zQ)l5BC(~KoSu>PRuWTBKnf&A z{2FWIWq)`3=Uu&GI66z~dP<|O-E>+jP|%=Ibqbn=hnu$66#htOmMC^tHTa>X>WvIe@AOQjb3Ku)nSdG}(bjT4 znh07mA2J=uFuuI(YB3$X7RF-GBf(oKpYv!hCZDH9N=Mryt|i^3%{68kHt3Sv8au>T zxYy+9wEnyjdk}rB8L#Ixv}1*^94VX~S3GRgUNs&jEcHxBgtP&JL97?N{~~mDR_riU z8}s_D%J75`U_)h01j3>&DBQ%SX7%~>;#tLpdhoy zcD2Xqw)Xq^yaJZ7P!5TVA&-I4wD4$Av1N98=zP^c&aGt~IkJ0szY(ubC9kInKCBmE* z<`eRvdSWsw2R84P!J}Og2B7c}RCTAWt&yA-OyS^7@>!2V63$>!C`oa=88eY&a3kqu3LvBh#_0;Ynn+~;eekjK$+X6_ zG#}oBwtJW4qMA7B!Tg;6>=D+Az|StU>_1d>>Rh&42zt~n@F9DPv5 z7ley$E}xc21Zp4J3#w}}I%B&+;DG5t38_jStd@ZFpdb*zA4PxakNF*`akxG#u_mPC zM;(wX)I%9cb@ZU4hM6_rtph}XP8%#TFDr@o0M217Vl$+HSO`Iu=(Mq}8GkK!Pq3!* zhP4%`lxIP$oFu~U6!1*OR^F{}&Ml=R!f|ov@X+v|IH{groQv)|?I0>S*5r^gcEB#6 zhHD^l0d3Zq7^))%?q8;Ax%!$Li6E0AHDFrs-PZv=n=X3=3Bu<2j#`@ zKO+PG_I;#viO6Fg`J7C>;Ywwpv(W#4txlUNgn{a62?eT*Kv-#fOd}qL zG_@WotN795(tY5ts+R~Vq5ghI5D7Wb+pji-Y(ZsYjMKCo(I7t7%Q!UzB$G&>HvCCQ z1{pbeXrCO;M%Bfal87_%)1#)Mm)fG#fMe`-#2o}PwN7Z2I;6RvYdVW^83}R-(~7{Z zqK;}8U92lm1Cfh$XDghS0}rZ_QmK^8nl)Q~_1At=UVQOoIeO$EfZ*pO4Pbfaop#mn|>o>^cw%PJqzx|(N+q+vpEDy?kAAVFepExY5SFMqHq()%MmvlNS`}XdWI(()? za?9@Igs)rKK`TkPJ32P8nNrZL- z^;}~RpieqGIwf(WQzo>usx%R;Z&9g0(`u8uw?c;U{!|^7^MWY52q5?X@E}Sc6P>b1 zs_8>&ia;1W7eK_gK3Do+=imfE8#-%85XLe9`<~t|X+Zg4e+2#H=;`M?=d8-k+Q>Y% z)$@JJYU4YDD^TfKJA*^0HGQsDKn>+KpFsfJXBKLbDO0COBvzP>nSKBOdEQAxK~yi# zJoCKl+p|MH@%TrjZF-xWJb6OC@P#i(XLk>XOP$<&&9$;{=^|OVf0rav1M=*%Ps>wJ zJ}DC#TV>LuRzy~3A$5EU+OCYG(mA;S63ojly&`?!#y|S_C*;ZRJuP4QtG@=Zia{C) zNlQzk#=59&;XHq-=bGAhjOR)i@DZW92O!uC^4$lkd9c1!W=vZELQfJ5Up19J4174u zaPA3o#a4607>4y6{PvUBr>R)e`Xdu1-P10UX3WChYz;tURIw`yK=s(DhIOmX$_FTE zKpMgxDTqM2>Koh#V1t^``iwD+%7x+ZaGoEWi*$H35V=SEDWej7P(>l zdRg`EyYdHr{D*S<=pi|F^oTt5{qMpqg6MbHE@E{NX`3^L$zx^L4kVR?`o@QRV4wNy z&&e$}-zHnuZ;?0Od<}-)9PA%WGT7fIciefq%$_?}zVO9Am+8}|%jC&ZWk%a9`NE(6 zIk@wTJcITR9o#43s8&NcuU`cT1a^%yxON?JKa^Df$3QA0{eYA-Kf?{Tr>h6VI9MoI zY<=AQ{`7o6dB^r^ui6TfTY(ygT;k_K9i=Ro_&^x@`s1>5=T5od zrkiEvjOnsr?K@CU!O%`y00>Mro04DpsCw0j?FR!d7HlS<-Zb{GfQUAm?6LZ z8^0nAjRAT32j7)C*h4_}jJNB^A!*;XM_T+*>9RJD9CSn-U5AWkl-fH?P#)_u zs5}Bf?+oiEX-x(2iTl%#0JK0$zj#8)q@;Sfq#jNW11FA2U!WQ5%wRBUV_`OD7XdiR z`Bn%6$feU6X=-SM#5n`4r1j4SaQB00ps^lq02b&j@rR{0{jw`i1CdMSU>L5~ORMOT ztC80vZqT*&^mNPn-+!Nc;@;l!Gh`b@&w{4dH{lzcH45(#xY~3OuWDY&Q6&b1Bjd#hUwmCAh zZ9d38<4xF~32HU5hNG{fW*9(wCyyS6I*S>A4G1y+K!8#4>axt39#cKiy$VSM>~XTr zVu2FlC_{0|yX{SDE0J~F7a*XVT41&f#*{+Pb(#*%b+;3jIFRZOS5AL(7JlQTi z0c}Ad!;};jdQ3`Ymxe;gbJohZoTDZdFSo7Wa*Mc zaP0**f1*o)?ED6h;b(w!*3aaYQ0e(#D}rdE1SqUh!EiEO`km60N`)_U<_B1hlc}T(GwDfy8K&X3orZIR5@!%Z8vz1B%?fuW=Epbz44;pY zvCQ{QI`i6n3j0ZQm@VUMElRk9Fkt{x;ZH#jcM~`ipp4Uj63aq73&EElVzNL4P;AXa zQB!Sb+R!ttl+p~!jGpgJ`=zhaGv|ma-O>rn21M;lHL)fC$Nur2<6~kQCCJ500|`Fw z(%;;eIBT96+o@YWH(c-7c!@VY+_++^AY35+ZtcrVtL#^9ZrWc7pg1OHyw6|_L@HhV zi|{&fn#Qu!t0{^swcXYIT43%~yB(t%iohK6Y{qz}qBzVS6Z{UgEQ%*pf14V7**v} zOtQ+&tO6xgVS=P*SrD--z~4DU&=G+Q09}VoVKhUr3?Dr9T58Z>1@@E5SJ{$9FELj_ z>JUiDs!~@atoT2dW}MO)xqgPsXd+SN`VG;|gX;QuPB|-GGzM+Lro((rB=(HWq3W%W zX~9Y!cHopk3`kiDY`-r6V95jxv89w`z>GQ0v{&?*vY@XZM)f2;^SB2(E;~L8I+`Su z-*TLV`I@#IC=_*F*s@|kQb4g%jJk4LKvCeXaC3~_!Sw3|lY)e7e>2(bDu2~Lr1G`8 z=&p1|brN_IrAQQ-~YY;F3T^!LPC%bzPe*4Y#9gOaWzT)<8S?Y35FroqlN|R z4BQU&lONt%FaOKGh%CKA`XP;MS-%!)tZo2G9Rdpb#xU~l)`{7wj*JvT%4P1*^!1*b>8#*XX-o&h-$;5%t2JoT0gm2dV_8`H8`O^p zE$QCO++_q1wKZuVq=8VQC^Kq0Crx?BZN09`N+^^>3?o1QGW{tr=XV4of*l6T*-~w0 zq#vV(qvS&1G18eM%O)1bjA}@YE0yOw>d%M2Rh6SrTa_3X@VsFFZFdixniknjcI* zgIl+RtB|r@2`vdSlXoih)$4VJR%kIvVc3R49~DWIfHU71FC=*;h%m#fx+>!{B^0W* zG#4e;#Tp&lce*zv>*LOxBg-5KRcIvORH+n4vqupv=vz*r@7dy zAwO$lSSW$yz*v}qN0vBa4na5?sEUr&IW;3RfyH{BQ360aL6iv2;6FiyOBV4_W*>cN zUNW~57+whdGFt+3l^J6_)A_J1!|6K`ZsE$hG~<&^U88y3e1~6z-^>FlUW3VUHTtgX~O;=nDV}Dfn!ai@B?3AM;(C|4_X9kuzR?Ck84XZ6s z`B7=k@>0FY zaXpnbcos~#)AMBk!TrPgj%ir}h&f>n;5-Q>0YX!&Fy~qd-$8YWSQH;)Qs}J(1aRhj za`FMN?rCuFB*t{1hVO<^39&v6)9E89b|NI?JOHEv?#`_2Xgtke!%!*2FLfm7B>Me> z$ODRd8$tZR_|TA_fk8Y7)v3vlU;=L4ni8~BHK}>rc&6#~m67Hk0!pFE?Mf!#W=%;O zsctNtX0Eg_n5n=Z0&61}Yy%t>=syPHp8*iEaGYi)P*5<>m9a&c=fyd;us4oJpF?%b zz9>D1JiVwp7yT4X>JJlGz=)`>zD zp4=LURJc%=&Si#cvBGW9w7o_=zE@XP{L7578kddJr+4dx%;jY2oEN0+TEJRG$7tM! z?h*_!0fOVC1Qm^J+=|md*6ziN&+Jq!ZnIi;HU}_ad`GO3L zOeIy#=|h1nALE4d5ScTmxyLY-tOh}n%%L9yb>4Hs)e-_QeEqezAQ>aNUTqTek~8LG zKU|g>bARK4xzZ13kJTrRp#OUFJEaKLtH*n0X$T^6Hb+-Ns=#bk0rgfS14P1$WTDqD zn=h9ym@Qv_EzlEqj1TmV}6CMw$kmxuJq6$4I)m4({mt|8YOWULg^7gh}G61fhwhJ)!sK)~0iM(yh z44s3OnrUO$0=<(af*yw;qQvkyj`k5P0%8kE1HT2C9T@?#OQ%hg&)mL1PJnjYhU{2K)X_c&afSpMfM8Td=ENI&jSgDJ-HZuL00EB?G|(R80RTxv)lV5nG_krm zSuk^=?1KAx=4h|Xo;pRYUOZb?@7ymZkYOMI_iq;aQnNap0<8MVa9QleFko5%6TgQr z=M3d|KZ?6$k}_}7WSKLuRo>gVS5D^4JXqC~RWmsTR?y`A8Qe>7RPfMdZ4ogm`4GhO zDuEdQWv{Bn)<@Yr?c+N(6r%J(#CkGyKa^LPB*Xw*lSZIy5PGZ- z?1KXcG7W?wk<^7`N8btgyQfb=eU+fRBlGGT)sw5lVGgU;*rcR0_;5}c(* z-STn9^v>a$QeuNO7It7h?0xtF1k2xQYtgf=H9XhyNvK`mus~wK06MO`)JptA@%an;(=Z$Y5|iVucRw-7A;PZj-sQX2`K)?efZob#erB`POAi<=UB( zqy<3mi94>9J&CluxMiQj`@7`k>67J-g^R@3+$aZ+ACtE>?T|xAtIzmwVvBlW!ML49 z>_2u) zZoK9yIo6AQfAXG8jAi7`>z2us$&=xt9+%ZScFTtShgB66#C?}fZ}hFOtImrk6MGmHo)2cEyx>sKzdn z`iYaIzpGQ;+ptsi90$lW)=Orv4`4S@u3I!$CN?(9vCb3n#`c{uY34M!Y2GxM8qUin zZ@W@&U|@e+dklNm+=P zEC%~N^X^(X+1?>{VNaG_HV>s7f^q=qh+f;eLppGtf;8A`4(ReK?Q#dI1|nlVW*5r^ z*4ANFQpK|>ChkglMxgh`=3)Mnd%_#0m!NS|>x^5`B?PK7sI}giih_4JZ`&anzGIa zXd(>5(CWzZFKq)6g}E834B8i{+}8L6D9dyGS;Y3V$f(rwWk zUi5$>YM9bgBqItz1Q_*>;;XnVe&=e8IJJrM0#Ydz#^j_wC9T1=~5Q0Ej3Oe0Cr2mBBO&<*1@~I{@eLTP~N4TldMqU0da* z%NNN4fXqL?v`S8(W=#^REG-wTaU*c21qZLOA&@fskPw#5nu-)Od0Bm^Uk0#9pSo;@ z%tD6got-4M8JUjsI-grUTlUPKDm!4u80^W*RH&prd-oEl$NH|=v{t4}n;>^zb(y-a z=g`KChKSsI!!p^ld!OX{Lh|`*k!=A|#F{M@Zq%br6= zW%rgnvT(sXxoyE>*|2NBWcyA+3Y&t|Lfyy`4PeK4nA;SQyB186jZm5OK*g2MN93wW z^|EMslf1mHA2C^V^8N)=rDgF9+0oGp@IC^`Z$R$5ZlT=1bg`^hvs3begP7AIX>Ezh z-@k*nFFGmc38?&et%1ncw4ipy2rJ;!x%}D58t&49-R=@zJ@u}!0OC}B3JBm#Qy@yW z+;O?ASanQJ03ql?di|BNWpZ;~ezK}Vwzm(;4U1dlhRf&4SAMWg;%N{}aOqLhZ5MFF zfDI5#r2I(sO9(3Rd=e@U7Z*cGF%*2?7Fx-H+k5Nbr8gntm zivSeRwDFAU4$EMm4jxPPWkzdKOpnlrpj>u3sq$S1{r3ecwYybKt|G$ zG-}@@r3sw+^|PkP;p1I$tgj1N)dA@qTqxJiZjpVEJd#Ki6Q>l1alO0spnPxb7U>5N zedy{dq-{cz{GaFFkyYKD5<7BIrq>;pI1*e!--CPn^4&F?WCcK@0UZ2XBq+Ul;VgLu zk_vhTa{zFIu4)~UVm2Sh$z_l#p5Jv)*7d@kf<^uK^2O5Ihl~y{t(Ok;w`=bS`IE=) zmnCy2$cB^q<@U?xOIKgNd}ZYhX~&$K4{ec#STiJtPFdCp@35w+^5mAIatQmA@z-Hp zBXB??HwVWAY>*#}NC+xAs`wb8AEzt}pgr8xC4c|=D(Oz;WL|Tf+=)~}KVGp~ez5MO zGB`ci0eSd+x60z!VcFA(#$)4S`f4CD){{~DeZ&>e=whm_3X;hXAtFUmPW%#GKTTt3 zJ$;xwj~;6@7-vEF!>FIKW$O{S@5cGEWL}HBzW#``H2CGRS@p8=Jp^I4_e=apw@j^1 z$#UrOH*MZ8{UBnfunTK4=oxE&1nMQxmz2dbn&i%p-Y$tjq~6bU!nh5cI$dHbc(S=a zv@ewIlQIYN@S-~{y{k09Z|sdUfqA?cX9D^rKcp_^;!5JH9Z+HQLT{gdE#L^Gk0jJy zJ{V8(Sc^Pt0*N39Flxtx9?As7Vobz!EzJ|;*Y3L)#_TMrUP9G}VT2&Dq?lnCvRDX7 z85vWO#9RmAvP%mAU5Tf(*2&ySSl5l)q@|UmhebL%d*yN%zMt6|5w)+wYibZcwC+HM z3`Qq$QKfZqowRp$$&v23)P-m2!9ExVQHCp21J-*&QO_PWrIr@#c}995VTEE1n0P(ZcDM)gr%Syg5SUB*59XZ0*+_wL2!qj} zTQMcxU^m=y1$ztnJsBss2*kwbqUnLfl3Z;(>Y%c zBv1tZj=kN5D=Ii@<7lmR=_RuQsrL7+%# zc3pgCYLQZ_{s@iJEK6lb1V9&~Q5po6FJBmjVzvhfg@l%hGrGq{nVpw*3|N3eG-yLY z3xm6+hf(`LuN>}9YLF}awN{_lCR>kng2KX>%+=NS655bxOb5Zvh9R}2R4q6SX>_Op zTfTg$1_4eB1*91^h1sY!dF&|cDu`zxP-cKk={Uy1`t_j7S$+@eyjFZZ z8LdDGVjW8Z_-ORTg5eH~1iB@4XSa))%1d230uL)lhXBz4?xP)t85>SQowOG!t`KXt zfS7OCv|TnMi*h~Y2&m8?#x#;j`mooub5UiN$pKk$E!TwcB7V%9)-pf9l>SvI>>Ct% zY8!(40meg5voKZ$o@Y8nfLaiVI_W5Y^TfSA)YB{NnLbH{5Fk95ls6A{${v7w2>YZ` zZUsz(I|4Nj8S6=_{XT33lsw`881zKK!yG{--Gkw3jl1A@Omi3KBI|lsLe0g|6d52^ z40vO~r*#|l%f}yD3WIo1md=}ia>Si-0{O>~m<*gZ$T7TZvT$1^d14&H#HL2M5iY9> z@V>489qHP8MEXJOUWA=sPiIQnTH>;3@iGu=aGkdAu~ybWje=eQNX%HBLjdhAMnUF0 zvGg5hX8(1)FiU|EKmAnW!ApuDLhmMiHo43pQGj?yF~Be)1N|UsxC%z#u++gq?HkNW z0z@(nVtyRnS>Jnez2xdAsD}|l_`>z@D1yp}V@i`1_kmQ>ufq>DABc7%Yy&sIwRXpm zc6nj#PRWF!_QCh9pw5rqx5R6qKgF>kswMN%-dACV{s z`xLBse;^4pRZ0@*leT+06bER70HJ5idwMvfAlZ?A-w^dFsd}{WI-?`Oi9lql1cm@vCG>KUjBGFjewcQd%aIsga_@Z`Wr z0Kn(ccn%5`Wq@1;dTiJYlIbp_l!4s=M*1x3DbnS05S%}`+Z6OSK~23r`igpi!+{-L zUHt&5001n+06=_JZ!7-MlFm|yE*jcJS95muJAxjinFNp{rBT%&P9&`WHr~o=%zhC3 zAUWP(3aTid>_=4j`x?V?JBa_pfgAt;wvt_k<$nU zNDJ_CgUU<6PNTxXktYYBqN$U~FfKp4;-IWL48FV${gBZ~oIn|a$#U&wv*fuo5d`ZY zu0+QT)fRvxkNsG^2VP6_X3HmUy;0uUx*x{oCYdvLw*2EuZ)&}gKnQ9c#D--3gIXKM zhoHXz2y-xn?~(E+f;~{%5mZ$jAj~0noZYr~mI#dbMfx8)d|1zPL6m&WC@H7}F5i0LJ%$kFPCtf3Om$|ksHK61~Ma;VcM8}}xWhYT?i z;KHd^@gvU#H9ct%ut)$&Fgpk2xmWhd6g=p6?d*_AFo0e$wGFnE9?9mUU*KvlODhTv!5Dw{y8$1Srb z%Qdqns(s-AVjECP0Md-65Pe)%UzJ|DSqkvw|U<Em0KGZ9DAUewu6DW7~3_vJC!ci!qG6|n-Kh`U~;Pitqg6!<=mS;BYftT1M zxfAZZ&z;yQEB19r;BCL$10a|^b%OK)h|;jRtOp~Ihl*|mT!xvl?yk$);AJ&Yj$^)S zHXjf_q^ZpvJ+cxi%d6&2#k@1J{lsxO2%Fe<-`*l0fb+%uh~VFLv|XNBwNY9pPGre0 zc#vbvN4sFmPeM8ZV*nr@494WU@9mKLmd}?DTsB8lpIqCuU2yIggc^*JmW`;d6iI3z za`vdoMc0$2C!*?t%&7&ABJi+)a?wq^f(fa)i1q-e320!aQJscrKUn9m)uiXcFf zxl(TJP65mZ<@kvnO+&-PhGV_BcZGN$yN6m=665ad>6C78)1aEto9>5dW@>@Bs3MC} zhD+SXsWQ+I0CpS%_g{Xz77oTi7;GEl*g#hP@%7D6$xuBL0hb$+Z6{93h80I4JwS~G ziKGVv?(3_zNGsGaamLQTh@8Q-FTr!_ojrS{J_MB%R9$@oaU?(t<8vciaCfLmFvTKT zh%}4>tdMeA*6ceXYxlx94XeE`3fRu|10Z0BK`DaGCs6wNwRNUleUQ*+Iwo_PJnI{Dz$ z3H3y{US4>0wH${xmmeJQ!0`_KE`S_Vr3JCdExmeWK@>45&&?}C0s`qL8=;pGqs1XT zOq?cl6Pq>rt74<$R7-T2luk(4nFKL_Ig+i-L24L8N0g9a@El|DrZ^1TaU@}^%TPT8 z%LLV5zL*?lGn1SBW4 z&qR@RAghqZ2C$|e&+1A{p9S?^1AxGPKvhND-$BfQF;jFX$OB9U!c9;yqJ|Eo9Y|am z*idry09rT`w1aRbp>iqx`?Qqkj6cGXl$6$n^5j_iH-yaLg0E2j$ z{hD*fd@}%&ss_rn!gGZ;gv3qaM#lrc%zX9fT|`x6Xm z#ri}{N#uH*p$x{wti{GHq_S5HL@HaS+Ery%z#Ki3g;kR;THC3Kk1%CS^+6wxtH%Yh z(nFd%Lw?IEknB~8u%-f@W^?892})!#vPR@6jN-m9_0Vw7rBa=Q395%a*g?{)O92is z&)_=R7eJJw;7Y^kllTn67i^Hd9i(ajBtM)4wF-!zs` zll-vVXuv8B(M;f|u`ed~6>^Vg`2ia5!x#^Q43bQZP-j|_90;ankA~Wh2QLHLQ9vX6 zQ9c-8K;!)&{>{SumHy7$0>+jwg}QRtY||Qre7H9YL$t}P5X8N-aZt?0IO!>dG185g z(icHTYoeg77;~+dSpksf;e|25j>P;_P!6GeND+*;!BtrhaWVm#gON-Hq#>qeNdm-~ zCo4_q26Lq09En)~x(a_9@1Y)a=+f0R2{C$}<|!E12~J#1{G)laFna|uO*4bOvXNx~G@F{zLsfl{7(y|_ zh-Pq;89cz`nV9oOQ)$p$5)U%{_{lvxci@X*5!J~^pE8pI-{f00oSSm@1=SuNQoW8Y zq9AVo34Tt|dy3=^1dxVZKZ|(b6#y%$p0xN;3p9ap&;YHjqgZR&ps0%{siRbr_9NXo z)EZ%^WN1gxQmxFTr0!8>r3>{&_??c0gwO7<&V!%<=>d{MU;LzG;YpN1KwusOo^{(Z z;ARIwB?44;=-6ypce{}VjU@-J%z}{+a^U*;nF$0b=Y&YmIZRu&o!O$a>IfYVOj=pJ zhMrVph6UWcHpDeyt7KB>3&mxoq{D26Tx*;;dN}EN(IuC|fG1U46An_+07tGH0}m%V zb)CI}9o?;&Uy5r(Gc?o5xYKWvMe?VFWsm~&4DeWiGOkd%I`A^Agwhwirc5fsAU|`z zRcv5p2O7BSzF^%+uAL0XoL`Rl=d@T$FPD{u-cV3;@~RGZq~1=4g1 ztzE#kjHbsMHZ4P-oE^du5SJ>(arOl}g;EttboOBm76>llDSK(gi z^~EXa3r#4&a)fF~m8N&Nb5vMYbDdo|b63e&3T~2gzK zUNPH3nfVXPdRyAZu+Kwlr_6?DDZTSDl|+W->ALn_Z@x3*wmV!HsGEB63`^I!ctS&M zdx5gDy{>^sW$RSCs>}+gbb-@HPRFG^885A=Ei6HWt5*Ku9rAxNBufqSR9C1Pg!UD_ zH{?3aOtDH)Si}h5S!!bG`4A!I50>&o=)wy-LmiUywJn_oeTq*KxY^ejg$;lK7_@I# zMxnM@wG}9{0yPk+9g9<50d)_>-5Tu0_a0VBNg~0egyTBp&Mt`?N9O{K7o2g4LmZg_ z5LHi5H3J=n76~&zLc5E)t`;~}mUOl89!zVl(j!$_$mIpEF1nc3PisZQGxY2LBL@7j z3@qZARmW~k>X*bW)FwH;6{vy8_|~g-wOauYDn<=E5Mh2IFNo9@Y8)$|*kKV^b@k=$ zH9?E4nZaUUluNk&G+Q9t4u<7Ly zB{xikiX}3R2C5yetggBGV>y>C8Vn$!Vm;Jlv{?VlvAKxu^5e7tH}*%h3s`R_tDVw? zq?AD~q%=IWG_43+Icb!pK|PHafdpzwB;o)M07e?=K-9GwUO>Jm3OUumz#K*F1l(pd zq7CjmApk=R{j{R>Xls*tK2iEm+ltz{jO~JM1LH2)doDMm*7sRmff|UM)#bjpI-^@X zU-T?dVmFEr^5U9#@obdPYTa-a;X@!B%fJ#@)J@bN>u5ySTtatSE}*F^X9N=_!lNW$ zJeXLsK19%fS5XMLq_RkRe&p~$*@wV4&EkaEhgeG^QieB6J<`I4QBNX-M1+32{NlGP zE$MKvVp2bsZI@nlK>!CmznFNBP7>;CWObFw2b7hJF6JGr&22m@Py>0Xv^%pbeN?z$;_(`1uE4481NvqHt`FJ&FWSPI|d~Q z=7(YcTOuI|WdCjNIV3&Z$h(EW!ek7Ad8m39?++1jPVxinz+Gyx#~G(ZCS?tKDw>uLffy}c&obtH4v%pdR|hkxx7oxfL=6t zs!}c?9lFUS#UO1EeFojZ_K>9%Vf9hU(T@Nss{A6+CUD?-Q;<-t9wYZpOwq53kzHsoaPj z=$lc^nlO=oVvO?yGjDFKOKCqZsR#KqCtCxN(_BDzyAk=Rw|2ZZb;{5Nk~dM8S*^;fynJYiJ}6#+49U7@(-&VWZBRWLbz|ss0w8sYjvKZk1NRWT18t zJ?8m#Qnyz&t$|3D7Ti5!L!n%BZ#7Bq7zoCi<1|DUQ zMp(WyRYR3ujF{E5=5Q}H^X7a_8VZwmkX&`;Ie5l1JWbYNal`3ehE~^b^oH}VHuYoM zrh*kTmSlHeqsdLGy@cc0=-RbseFexkaLfJHh1|J`YsLh9 z&S9G?P@7uBdUZC4Y79rgbOVf=rjr6R4TutUrhs4-x9SmeC6m}?<~A}xVT@ch+n$C0 zXGrt=@Ic^-D)y_uq!H_E+X97c-_V>PZF+SUxwsV{=Yy0gM(IMtFn&aiK=OC}-b!WF z90QYOC}6W&<|IOP*2&HD0US(MfqkZ1ZG2GceJ3GVBtZ02uxap>L1tCvo#M@=S|CmY zv)e|RQBNEHBHz?qzhr`(_m9nlj23uQ46nEsl!Bon}=^`Ff7B$>UwO8XN7pdqDh*UjA}wV zC$(*;00ilei5Y2h*Ljm;acjc8FqH|N6jnM{?cD;#c;;Wp86B86(ZtU7Q;D>Gd*kR!)Tqh!EYUX_X3tX?9=?5R~|FI_oRSEt)csE1Td)mq@< zb_kp~CJE%6d9-z&B}twNhN7|8yPS|V>^rWjJ~W1~6ByJMp&M-Wo*8d5%y6sHDQjKx z$#c>Y>=LRra==G5k_q}#m#Km-)G!+?5Tza1J{VlcTv;jRG{JpZXprSku?{rKXZWv5 zq#Vmo7T-$dR;_E;^H}HhOkQ&)+pc7dF8uzM?in`L5Oc=V;e6=$>d~)(NHu62 zA7*AJ)wXE9w4Awe!o&@7#Y{&{eRv6>JI(%;b}}r%iUG{jNqY#17=zL>0h2b=NktxJ z{Iq&@BwG`pq9v2Rsj8Z~(Z@$3s^xZ=bdVlJCglgs$L2-$og%j?{7!4G-kZ7J>e9@8 z<=mZAqTktmh-s{}r$%k=z4_hb^}!nTrZ zkyJBVKT8mq5~$FXtInQ;)&9cT3LtFN6bJ}R3_{Dm;HtfA6+LnXB(kQy17Cb^_*<_c z<kldd>}HKJ31h z%(q}MCrS6zS+VP)f`+Q#xE@19UTtXB2}9S4YiNOrABG(MP^*Gxi+ikQ6@21((tS4I zMlm3OIp>kuy!NWCz<5`H=P%824yHt31N>#>8p0 zDij6P@aqIYZU<}Gk+@K$RXzKJdVm3(RF_WP927^nSQ1riydM&fz5r-L8DUR<(mEGQ zOeuv>^_DWe^=}Mu> ziFtaFjba%Czw~E3k?IOvGA13o_kO6Y0+2W*j<^tH-pxrwQUg6mtcW~WbnjLB5US)s z3Ncmi%&T;+r!#-o=9w_;j@7nzy=(oCe{aWfY%s^yxa9pM=3 z8vUIT1_ceI@V0KeYQ_bab$9p3yvZ#(M?@hgX{{DduBv{!dgTo3j!?Pls@AXV^tSmk zWby1aiPbk6sYBP0HL#6k!l{TT3Al*xqREf7w%$GQ4xYem0n8VO1ZlXU_}s1>GktN)c8%$g)p4H!}O z-#j&Bim6pdYtR=^9O<P#>lzs$azljS{RB{sa%Oxw2q^6>(mH5Y8{cBOq*0Mv#wdB z{EL&h_?~NYrq0@_txD~#3vC5VX+2M^&8?I1ak!ns^&}Iyrmvj!hd8dj~^vB&y%3rxIN8BlXsXO)89)~rnpM5=s76hu_5;+N*e zCh6|%mO&tSKA%*Oss5|H9Kyc(d%7g4>MU{&4IDV^&n*C1Dv@a>>-U!Ez*Fp0ybb_*qmx0Ql)M75-?YQr~%3N7B60; zo58;<9(1~+R`*p(j!o+dB7HrQ&KkpdOAEk+(gSpzOb}?sWP>~GfQAgPr2K`bhr)2Z zFrX?JfJYW;vYe{QC@oNTN|Sal;Na4gRK5;Kd9gMugQ=D0N88cnI^JGI@$jzwUa8L#kv=5Fge&1Lma$ zBGrAuF16MiW2vt5WQd_v357;hkk&xt44x2fLO2XJxpVZ&0zfkTw2X*MZ)=0<3Q_{4 z2d2mXr$JAjvZ@}X_+v{=jF!)VGp5U{Nqb`MGw!v<@@~L`@5gRWTTm&)IvxVf=?0aZ*KroK*rM4!TU*7?Wzj==S3% z`+CLIOQdc3Z0YM8)H1YQscxhLUg36K7?)Yw4w6jWKbcA#4OAlOS@em&wiJVm-{&yepZ=NHAO{ zk3RZw=xAfodHASAV~Ezas=?AjKUO~%!|!w|UBAhU4Ug1MyHHL!UMsAtLWAiIHZ9m3OdvRZXQXc>0C*{t& z?w9yLLI(QdGHG&Kp_#D``B*c#;C{FEk(#`5!R_2dIp7+ITy)!N4d%2<08404KXIZ{ z_V3>>bLY;N-}|3`CV%or|5Mhidk;qGkVH`c$?pv9bns!$B^t`TSzzrN72%4e)nMuo zaKr!(dIM4Ql%+wPNg(0oYf`S5+5`?nN`1+^pDDc<$psLEAA0-Zcps9FKmIxSQBOP;z36tX;t2uhDPb~7zxJxFz`0(58i<_h9XJF1t3<(?BB06Pa73m~ zohEyB@78jybLK6VKl|d}%6Gs0HF@c!A4>bNBa#BA&$JmVa%6})Jgt~IrJ!dwgDj41 z@JSNa^(C^BfGR6F2)hMhJmMiHXrsqg26htO#bkgchJ?oVky9{s57-nCsiI*v(SZ=dl4ryH>Kuo^2) zBoD*=Wt+E3=E?8N2S4@|ljs zHtdYs-mPWaL0bR`Aty~~mAZy29hsa_=4q{HDq|g&i~jc-h*W8d%RV=1{WOWnnS9acU@GB@$2X`7p4p-XL8GOm zsZN?2jfWG{hA>wSV@EU{N`>7Af>ET~K)z3K#0}G@$h<2T%ZxPjDQP5PMCEX%j|R_d zK*R(+j{Y)5rRW&oM~sVe5q6$pE{x6@)5kmS3a2&ZWoNrz(;tF4kDri3|Lyl>YR3_o zjHva@x1N-}-+V@Vk9|xU@4g+^^&FD0C=yod?3GZjQEdN2sD8hwtpjbTqV-Ex_) z1pgu@mA)MNHJ-L1jRbvl;!BukFdSmOXh)b^}gw_tme&_N=DZ}KEy4`~?E*eqhL1ca_{o0Mi`%sTI zMsHh`7?2J=A2;dRojG4qqF_z@cvAJDEb}myE`gea39?oiQSU2Um!sLI zFy*l)=*~{**&gY$r^RyXc}B)&11a<9kfxRp`~nb+@*oa9m_u!rwyr{mwS^RhQaYxR z&uYF6GwliTaM>JPruGLwQzf!Ari&xS%(CS`%!7i3F&5%e44CM;cvNiq(Q=T(SIkH$ z#e`^x%N%>FjVf<6+Gl(=-x&Bu1Klnj8@vr|TiK(qJ z^QV7K;4PtNif4gmE}*IT%Z;%dIj)S=HmXGL>Xcv2vS-I~D(>u>fO|Sy6tA^+Z2^Qf zd&#{c9a=H(&YU{@Y4U2W6Hh9c6b9Kk?*!Xy9xm$Ud2m0BeR?j{D{3H8g`GDUpUhD& zBuwT(WxWobL#?N(sSa&v)shFd+_POUAzSg<(x_o{-Er!kd9XXL?Zx%xEsm?#Sc1%d zT_9jd*^g0IN9&nG71jxmh@ifOA+M%${=X~2Rb;7jE#d+i%iYx7cnV~tXibWXWEFXK zc@TptLw@wmZA9{1j5L=1n~H4?@J3oA@1l6yck5fa{>7eX=8i(!uCY?|D|06Fp1OKz z`Ro(2>-*o8)(<@@;oI-ltN}qtyBQPcBcX!mW*fI{6{UwMJAZq985A{ZYb4j9v}!!p zis%;XM?NmzRBYz`@U<=l6?ZrauH1z+DM268Tv;`--r}9B9uQdBfE-l$`CX1T@_^PX z!f`MMoJ~bK>N8OLu~w9-DrZ2YkHYh6Pcv8?a^=h~WsbM8l=x7%soJk$gwlR73~Dm`;BW$nS1714DarEn^QEt56??!5}>^wae zd)K6BNk$-4MdR_@Fk(C9t|?vnavpNF!Pxumk?1e~qO^YGvtX1_2#oiF0NPrR!`Hjq zLX3Ji?2JmTaUX| z7TJ3-&l_*L9C!I;@W(Hpwg|s5USA=JvluVdB&aHuqE{*R%8B8-thI>@U2A{Pr`-UO z(%UkedU?;};TeR>B`7YdQ5@lH==}xyS=bSi&{79sijb98g_PTu;T@U#i?2n-EOpZk zqJYc7+wPRqz<> zRna=x5e`A^>%t1uK;#S^2J;YvWc~MkAc3{(LUKcdP!!JQh#A*CY@c&! zHKMsW-Gwu*U_-KBeA->A_oC^QF-#06Rhlxf%#&BrJ|8sr@G^&QK1i+(V39Vi1axRt z@{mWsAXQd{oLocAx@wQ!jxamt}~tabqsNe=MRktZ4`8 zcC3cD4iP_NLl{oTl{qMRS=cH5cGzY30($xk-LZWDX=ESJ;$n2E#vWv`2N^$Na{%mq zdTD8HF}r7@u2o4#9?ULt2y{=1q>2(EJ5o{J;boYyLYP2V8g18cG_WcK*bymynYjlT z!xW6k@xD%Zucuoc`{O^7`dN*Iq6OnV3ulOiTqw=fK%@#g<@Fx&+hAdvghFziCndRe zo8%`pNGRF_j+_rqarCx_tu>=@LV-gI&x26*fa{Z3LJZ1siDf2R<|!`wV3(uWE*60S zA?%;3JMGt$)@eb{oq9Mp<#G43w7}x4LALTgLfej(oBXK~w!P!g9juafHvjtYC;T-;G3M%Vq#6 z+xO5Iho&|B!jaSRmjSDS4|th1HXKv2SK}C)UG6a`x6P<9Ajx!(bnf0H-2mPc`htVB zu~Tt!JX@oJ>6oYPlDePx#O3v4_wCN}WZ(HbPy*a^7BEFT(;% zjc{_yGArgw@3<;o+TjiQ%Sv3u*YQ&Z*Y)*vOK;an@lBp4nPh4RYwQ5{PkHrdrHAa& zxvmBx)!u8IQaym?>5x3UmIB~*GDs`Z*V_wK1Ai+45jTd4K%@+TuT-^D?YWj|ZzS+5 z$Dx)ZYUOz5((;^|>i#t5JQBBDZakHayWG;2vsIM^zXAvhwQ^cxv=BQA*fW@K7B2Y#m7Is**KLDWEtI{)JH4uWVhzGQ?{H2E4%MDmI^?Iaj)l zm0st<{)&j-h!d=gTvgOJ85zsWs9a~?mj$}A`mD+xipoYjc3ahIr(!Uolxx+!w5^#8 z$U`p&BGnq#*eh^hUr_^*GrE5!7ZvMaGbVyu4}JO7k;{qA_M{dUd{SsI*XZx&q5|hg zLTcDd@6W|g!O(5ydtB?K7sJ%#-Y3vKJ*8qn$u&B>l6h#ip6@iuybKX5?ZP|U!n_Op z4?WJqY$9bv7rPWA%%j}6hp&Gndo+T%p)h|l@xm}XRH?8=3W#>Rp1F)Q#-T>-_Il2d zvC*;b%E;<4RpS`r(c@&W$>ZTsIA$ZB?ZWx9YvNRvg+@krCft3a?U5Uk*Q-#$)3#|J zm%=mTSMN3BB;Z#&Q?Zt_1|&n|WUO~J5E*NQPUm;`VCw^CpE~tc8hW_PMsk<1K<@e> zP_cmEo(u;nrL@xvP<)`uuvUc!)HBc0??$`E-8Nd=Y1awTWffkA0srQ;s9CI+vKdx={|``&HA5JtOa(E(qvBvQ#5)vg&@ff|UM z;nTo_t4^3;iOBnC@VZyGQ;u0I{qAgDqfc?kqMUC3l`+J{ofaWVnN6zp7>lS@mFHie zsuj$m8tZ>5>wPiJe~f3Enk*NG7UCMv1Cwt-aubGaiX69e98=FqTtNOZdW_9jHS_6?Y zu=d_VsL4(=Zx*Bx{CkQThBGkA3Y#45iSp?1%eLkK)eEd%TIpSk(zZPM@gm*3>!|cz zkJ_j(mU3hFj>$Xd(v9(sXQX+S&ShBJLy0H7HmHL7$~zaYkz}N;@y@aIy%&cnSl6uj{bF^`m_>JLkd>T1# z{DYt}rkY?~6&4xMr17}kq?X{f?8CIp6!Mxc7H!ZO!Iw4MVxCnJc|QWdd>EihoX6{o zqXPa#nP11$bL5(aAG$;C1pCTz444bSoVrE!m17EZn|0CbBWHo=d*~et@hwgVnKOm5 z7$MGI8Dq(R`K{#BI!2wZe$ST=i4pl-9+?UZbqZ>fwn6F=GnZlhvYswt!bir%% z#Hjin=~VP~+<^<3o+E(0P-hLMT4AD3BoxfkWtD5DY2ncx*OZ0E?0VBK$L}~uwb8MB zF?Ya%pChQc94=TD`&BSbt0xopfGdmLKoh1?ju|cc)vhW_3|xgL?^K%LB6EI}M$9=Z za=SJ^j~1VKDxF@`_tZe7n%k>Svw4cl!%t#E5J5g$e1YT?IGiRw&-wb|cvDDD$Lv=m zya~9IWfA9s>{k*(b;5KeA)#Od4*#=Kc(+kiD5v5nRp22Ae56BQ|9A5WCBs`Bjq2Ia1QI6i^6V}mJHHB2el+5#vTOV zVw?oK0Ek`??{O*m@k;};9w->0zc9{J0KcKeBvugB4RL=E7bbj3oI3{6F}HLWz~Yxw zAc=V)Mgr|+aShw>GmSj{ve-X%9fmhy5Nn6XSgey_c{qz8=A$6OL7?3<{z)OpW(c1H zco$X_G~jw(zX;eEX9nZtG_~lN!F>HQ$$N^uwa)ALoa4dOK&0v$;FaYKH8F-fGKt_2 zD|EZ8`mn~)U<8Cc4T7IWeu#Xu0fd^=A};ADh*o5RB)U@o6q5ovn+72dH%hi4DuH;f za?gSKW(5ubnAC%N_kk!S0y*iaYr?f4!pQ;T5d+weeCq;aRUG&6*ipOHk}M2v{8mYZN(=av_3cD>SWe2R}ks| zvgGE;y=%@YB$;H+4CJl?c?NKz3^2$xnSx%_sUHML`KC+&SOg_JfIMlvogni4(u5lE zOx2D>08oShh6FpLY|jQHnvW`B&H)4hlq`}q2#&`HAj=FKDuJnZ0}xE26fu%ql3Ss> zcoz2zKGYMS%dCuIIz_sIIc`)lc2`8ulT)?sZRwE&Ko3MvgEwCuwV%7i=a)O)f7 zF%Zl^Iu3#cfY`D_HvawJOLh?4G)`_>6P4vvYEm^@2HY^Km@tbiG6XOYVFH~0{0BZP zb3Xo%9Q^SL+&c)6^2>FfeoS(=T`R|4T_Jt|z|w~wmdUr>DDVBwf0RVmAc!F(E9NK$ zp-%%$f{+~202Zpm2rxl_i$9x0&Z?~R4uaTUew8eLx4rvNf3_185P?nNSEK?7no>ZxUm61X|^r<=~qn6@vp$aPiP$!TQx&r`8 zN;xFjhB=?wHed&{gp>!49ruzE$YF2+e^}CB9w?tu8YH-63EDbdQ;Flee%Sy+O~hHL z^TT;Pp>sR98i<_X^*3CNiHbDz$LF#W>Dl32VecAmN|19$$snqu;qoB=k z5LI&Vlt4~G71naYt#ZwO{w;|gACxz~|Fq;FfduPYWa%Au$dWtmlDD^Q0T3~GltdU~ zf`o>dO9O~>3>-d4JtQ=r#FLOH0Eq;RL_97bj5P|CAgxHWpRoE~3Uf-N2LQML(o9(D z1E{vH5!gOSurO5|(h&reIDoef*N334>x0ynPD7mswIA0mS_kqE;@OEbJQF`~zbtw5 zL3#6c|C{6vw~N0erfMweq!K7!(1*Q?LR}T1R0Hi~4id-!+D{JTqyd1}kG<=ML>2^a zH3XrO!@ik7zVrBHpG7iv=l|1q99F(bkAS)aG^*eI+pZ+hI|L{Zd=KuSOJpA#GNdK|@vitkbf&fO8TL*f8Sb|J~ z0H-0=bb#xgf5VN^3!=66-@Yx^-F%beXU>-7p}kTMt{uFw23W^H1QQ##i>%rL;)Q#! zxLog#|L|q;A=T<&eOTr!T`b3s9Fe+Nvt;4)8FKLL_oOGDl$I;6l4by2yr)-AY}zjA zW5*@X)Fe|DER_1Gvt_XJge<-F4#}N3DcOayMCvEV)RvjjaP#ewIDAy%C;C94Q?QSx zKr~TnBGE6w1+!$q@+Fc>vtluo%dv}3m$Dra0%B;mp zq~j=5VD)~v?25%Q(9tdJTXstwq^VrQFFi@XR;XFpu3sYIc@w2;*CFXzw+(xQGQ?CR z&6xr4OG$jNSLWUP0U6wJP!4V1Ax*RAO55T^;;%!W+jq%{-Mb|MHCN-jg)(9J72=yU zLmFI{&sQ);pXBB!)$W8Y-X=8zj9B@vq0 zsnR;%O?G*86{3FtAGf59ZHYh^pO@pCw#d%!eNW~=s#qR~NXznN^7dbTL82hA;8*c* zqFNk{+AuWJgD3?Kd7z;|+HSc`I<{?)y+8cETzBU!GWCX=<>S@{RQE56Rd7;6|D{Gg;^G>Q*EKQxe*##n>&B}AH+T_ekW{ih_3nlfqF zduD#@aas4(ugI~NUzhr5z0CdD&q*^>h@l361?*3=C$~xLjWx3U@BR*|ri2Vmm?qc! z3@REfnj`(4?K1Ph8FK9Hm9pWVz9yl*q%6JneyP9mG6|;oqz-dF@caq@=v2Ax^Pj=> z{pwjZ^V6S}E#LaCY;Z=6>{J65P34y8rqs0HK5|{M^T6&WG=k zzK#xcKA2k{l1+d2RXP6Sw-qny~cQ)b_Jha7n3Daj^TiJhwhyMwyF`oNXvWKh@2tXmS&5EM!96oX;9 zWtvR7Y>~YEr+*~9TeqpyAt*Wcq@Sm&qEp+2$SQ_%iLKDCDW6Zo*jFkhyox^n=2DJv;|K-0*@1BD){q~#W-v9EO za%AgTIr4)yC4v2!b?tSs`@8=lo4@!)smIz*g2cD}Z~j*HZrmbCNJ6*#_Akk@2kwyl zuf8A~pZS&yG}g(L4?Q9){_Ous{k}bLju?L}H0P3x5_|#PNW25BZVHAYzqOr3> z>R|t&J8>5Fig>S4;i!g|cPEN}0Em_2}VZ3>#0R zZJxBW&6Yho-xGf*Bs*StQx5##HJJ?qb^D5Sl6&lv(mHjT9F6XlS=ZelyP!gvG-;~L zpE^@c;CcpmG(qjQ3pR@!jOw|qyJg!~z9rEUFqG%(rDuPqbisY9W#J4E{sz>x84!PC zy@cyxk~?@4U0o_ zITJ_1!^)^ZooP=P=%$_^=eg05BV^$`5X+PX8PfL&@dd$za-j6sTK3^bq;c{T+5FNE zW!dewOZ%%ok^Ze4G;A;pcTpNg%}`6x^)(Da=|^oC+I&(5&ftxkkp6+=;%@~9KB+q_dC)L z)!z00^Yu^0SV7qAoD-}3EA`GSLOA8 z{kqKg;-AZspZ>h8{4f7aLipXJH3yvFUdAu9F1$=)b7srlZJQ+1)B@=MpfK1i-MhBQ zyjyOTW8eO^3?}=aN@Cz0;un;ORfU@Bt!)N<5XcNvN$_g&V+{GY1}Da;Zo7QGEc?yR z$hK#G0$a_i(gkVau|NDR2|~S=RTtp2+H@G}!l1@b4#r;%z@vdY47C{uJP2SKb_&&v zLwrhi4@v;iO1wS-ZC(-%0{|I#MFn7(4nZmkFiryN5{CQl@wFRd+2?*r=6>LVvS!mx z3B&Wsk3LfPEe|f9frDZA^aeoKflTt=KmCJrty&}XE#TT2_k!;mz`^?`0<6(J1OKuB zyTLTPuSl5TQI#M#Kr#U;$aWa8*Z;*|OB+;dgY?-&q-V1vWC9j02GwH@l0-HPppIjG z@b3-3@o@=HpDHW<2FBsgW~t^;d*gX-SSKCTv>G6SAfP}7yoW>p z&b1Sq^lhL1C@5W0_CNhYnSAtwZ2Hc3<>vqVUuEk1Z4ieV6unysi8jYhkG(y zbR#K-HWI``Ae|tCstH0%B*bdjXPgD37^r}Hw{FL$ep&SJeX`}tkTmKVW!|mVO8chu zlHPX^Dm6+ah><{$A+!#fPn9j z?8$b_`~C9H&b^Rwk}{#8Ui^!jWZ=LdfEA<<&J%%>^!kdz11v^$90*+~5QBGFw;bKM zMV8!qmmGfX70GPFIO-x&KWmPJdV1i=bpmG(z=JvYAbn9H$Uvg0gU43q!47GEZllXg;g6@}z`O6t&A;`lGVOi0%AuFukkI4?x&9*$ zNczZLIk{;M0#KVVr$GpfjEHBP2qi$k2c#hloOH*0tY@nn(3}n8tpSEOpLU!Y+;Lx7`XMw@srZrsEJAt!x53L_cloK_z{_l_KvQGDhyn(sWL~vC4fQg#OP{pYKdM$9h-Mc=Q|tZwtx5Y(tXc8($w65coIl9NqChpz>eub05)~3 zqYY5h%vBU|D1boqMoya%_2kG&z?o_!zI%^s`^Gor>Q6l`^XDx^-$?B-*e6F;t;D#X zhJqc$2au)uW+J%mHGlK>0EIre`jHRI!n^N=D{!wgwN8Q!A|^ZE{gFgKT+>I|<(gsoxI zH=dH4Ce4ut{_yvtbL&=#BBrAmSpeSsi~lG6h^S$M-4>Aw_fAB+MY&=&h%g{sJO=0%2~ad>A!*?z9|sX5RA*)zWJ0yzp~q^$npU&_?^)1(35kUD%=R{!B2OZ?z2i9pZ)?q7dd zdJncs6mbz6HN!}07=Yt2%)j*KUy^2U-!T9L+_)5MnFlKa@T$Xi+rRfL2r<%*KVNwKX})2ZG|iYOy-=HVZrdf{gGZzhQpwI2e+*B#RRCK6 z64sG9y>>tQf*jv}0OMLLeMj4+6H)(dlcypYaX=ceCyB#{!r%4iB#Z zX~24{`kQ~0{VU#)Df8#cF0A{>^;;!<@Tg2cKhSr{JAd+*l0S9=26i$W%-ZFmLwM*b zBg18D+Mk8fZd6$&ZB8kWoQ3W$x-M!Uaz?k*iOJ@{BA0!>OCdx$lQA|ArBPn`cI-g@ zDri8U5AQj0Sh5I?t3%*n3;>cqv~mLoUh2>xq@3CfFD9nsNJ+=4w?T|SXzP5^4%J8) zDMUg5A-W1P(`^bwn1P5V)@_jlq>5(f)HO6kb3uWdg~}=h6;pisZg_-c5kmn^7lR5z zy6AWt?Lj3*k1l54BC!oZ3g~_6N77A4V1P@n_Jhc57~1JR+_7OJfDlGv5Ih1@Ffk}6 z-hB`1sDOSSM*k@g<1FkZb+D!Qz+Thos(0N+=~=Z2Y9#%Fb?yYXCOo{{3G zqV)4f*H|@Nl4b&M+7YTL=*a{RrX184zyz?=00edb6%aG#=0LJ?U1*13Vnof`jm|;on!C$EBE}J!Sip1W&y-LDrX0{s!C|itHx>|az?iU91rO&xok~(2jb#tY}k`q zpzofOT>9O3u2GI^gtm)tYP9}IkU(_&NU}#FL2s>)rXr`(g6^CoG_)bm!a?Fs?4Yzj z(Bai|$0b1p!O{66x?BBMjU1K+QKUs7gxpNz*a<}B=rzl+iX*0m2rU>|AA@ocC@a%; zYZSe@OtVuaLNwlF158t{12Jlx90@(|Rp}4F0AjCekMF69@-cg`Izm7S!F&jSG}=8NoFi&7F~iC z#>i3wu!V3A^mQXh@_n3=A@(cFJ;59_%QDmT;~LIctBzoS2_78(AOQ+3gs?^dNF@Xq z&9|&)7^9=@!g?FhB|OdVG@qD}C>@TJJC8-**|qlOK|GV^-$~i(mqjLsBQ8OannW@_ zqF4iw@vT>dS1UWF*Sd#;G*AB=lV(qnf;RU=L9vTgAxuYrLm*%zAy-*BjRk+Pa=yp(`W~Q^LP>1 z2y~`gu>ux8Ge4I~7NnzO_3r}&VxOEhYn!MSnb}+|Ex`IG6s&+oN*GlL zq~t-MrJF6JAKH;H)}Y;Z?kgX71y&aDvP^*judcmmo1n!0Ie$|WjGj!U1__P-ly=nH zkXwO1`A)M7kWbKkCqppuIUendp6T#P-GX8ODFP3EC;=CpaP2j01!^Et^+TZ) zLJ5RE5X!w7&fV~PmY9|t&~i>GrQcO@jrSpRQ>)}Yca8a%^*YGOYCyCSDWf_fkua6| z_?}cqf2WFv*HhKOH1f)=n%|urNXv7Pc-e=Oph51HpUneJ8fo$vk;t+wa{DCEj8^A` zN~koSmmQS-hw0G$v|C`n3~fn52gX}S1R2Q4Xn2B;pFiS{uEJEi;hnk_o0^KI5K-2a z(^wr!X86pp)8=3lO9rTUe~Q{N5`Pf-2o{G#9OXtdZ9B{*`Y(Yt zMAaCj1$-s|m?n%W%oJoYaVn7}-Ny_kJ_F=Vy=qjfrD#!jdW%rSJF?n)?+VmFyn}J}GwKrPuK5k?!*XSGj8}{!xvhjISm5%2ngNk_WBdYWfdo zbv=TMF<=|5KPd_?CUv3SomC4Bcq(%a!b3vD8&haltLkavxDFgNfrW9;T92ND8U!$> zWoJoR4KXunF$4%wU1ilT{Y-xjb&bg#X$m&c7DJ+_QV@Hmx<%^~llYs=!2}f_=8{>8N)5ROR{E@tcK-Pca{p}>36(~?YJf7suC&`r^|~5}RCVVqr6_5W-;xtof=N#HR3EUh zK*05TsvtySXgM_khpJ~t!}#Y^DdfWOIjj-wh7ggvB{!W#+7i~1NYOipU_gC70f5RP zb3N-nCO_%E*`7!g}IKY{`0Mm3a5b{rnYL10u0 zZud!fXuj8Wb7~jiwTLCu>k=^3TdkTQ{3_{CS~7E{qXIuWNr`yHd}E4n+*bmI0wnD< z29S6?e^=EN-WyeIK7IPI6_M@QUCfx`RQd!3!@xF69`&c8X|Y?2a(E{QIN`CWDGs_x=Bu*6!`aAvM9 z38&gUXwcUm_`N0WD&-lQ1ixd+S?ZUP1V>|hQb|x>TqjEm^`o|dHl0*qf72*5!wz`dkFA6VDIQ)QRuHkN+}sc z?ITv81|n5B29CyRwp*(P;&JBV7=j7}W=?as$$=}W^Snx*4fm6K2))0L@ZMXtPy<4#7pbNR9y868P`d&0UiV(9sBOaR5To*kUI2ZI8E!0WbLa{) z#=?4gmR;9Obvp*kOh0xUTC3c4YsX@K=D5xH_*qFMZ^38vFvI6THy*vOurqdT-D=pW z*lw6jSsS$Lp%`s$*QF?Y)=SJ@*`JN$f|ovfH_q*eZLV;1++Mpz3X?5N+n{X8`O|L7 zo&S<^Zi)gFpYb8uZq>tfaz$5-aQ+$B5<=`tP(1`_uNpdg3hcTMR$6pSgl8(Xx5X<^ z1Cc762t!8Id$T|{42c4O;HmVHL&skRDw71*LF%%{dKGz&i~$BCf=Edx<)Fq8Uh|6~ zgkz0t8p8Q`6~QYY4>HuX>1(D4V0UXnQi&e7RFj-_7wpi$S=olD}I`2#OYv_~}-BZ~R%b$me?xj6! zw3FpIvwYa7Vl3G^?`OqVq}9p6EiMbKfFJfeBBwr*{1I=co-+lT&0o3ot!U_%$VD{} zsnSM!A3Hse`cV2&sQ8s-4cbt*=lr~zJpJz`ho!{A^ zu2f}Om$DI7#K>nvIwt{+zI1AxLm+7%0#tjMgH1~V*k{&r>)21}S=plVzLOe=RAGyZ z_Fzbbp@}i9-aIq0MLH)nNrE|=)Z4|nX^zov*$3HCAPW$V0F!FH^2}D9C!KENMVxO%)O4qFe0vtNHMklk$d?N+S;qRpyD(cZwPz&7j@P}juYHAE*;%cz-qt1k}2#^0& zf+0gBs?13*&rOaF1fyce_=h`mhv&KWHZy`iQejW_Zm+JQ7*R%PXC%^43P0l^A;Y1o z+EpCI>&ytd~|t3I-CG9 z1Jt1geHfXaX|-e87zGYhMcT)!k|9=ODK6kPY9LaD%^Tu43u;wuH3)s{^tp4T3AGv$ zNOeJ9SsxB=k*>T7{hy5{$^pz+9HMtz!z2;@=+VPF0&z~C*|G+ZQ|_>*@{AFJy96^21Y>n5d)#O|1^Kde z%~KfYC3{N^M5?v*N->PS!kt5ACJN(#*2n5K=aZ_K^o)-pa@(JESR;w$qg^xFectD5 zoNbRtN=w^0L;V-6#~GS`rJ(5XMHs$E9&oIb?;i#trP5V7X=|iwMp^%g=T}OptJHrH zeXr#F>0GQ$q>{x{zw{#CNAn1I`%CiL9T%vojd#2aojOPErTJs^S_!QftBq6ty;71<*(fV* z?bOFR-a9KDQDyTRZ||pe{c!1Gq;@nG$Vn8#9>k%$=yYGS5>_JSKVF%K*LXd?Pi_1c z|6Mf@sm31j^mz`VWn@fB3+7BJz=q*%E}qhxl@cYpK4Y!*IGP))scQXRPL1b`t`+s+ zrN^U5f)zDZ{fka}e$^k+skf+WU82q^CrjWG2Nntv8d(YHgf>fU!DtGc(%{qwbZOlA Z{{xM~vTICa&vpO+002ovPDHLkV1nHvhjah{ diff --git a/dpdk/doc/guides/xen/index.rst b/dpdk/doc/guides/xen/index.rst deleted file mode 100644 index cb43cd2f..00000000 --- a/dpdk/doc/guides/xen/index.rst +++ /dev/null @@ -1,38 +0,0 @@ -.. BSD LICENSE - Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in - the documentation and/or other materials provided with the - distribution. - * Neither the name of Intel Corporation nor the names of its - contributors may be used to endorse or promote products derived - from this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -Xen Guide -========= - -.. toctree:: - :maxdepth: 2 - :numbered: - - pkt_switch diff --git a/dpdk/doc/guides/xen/pkt_switch.rst b/dpdk/doc/guides/xen/pkt_switch.rst deleted file mode 100644 index 00a8f0c1..00000000 --- a/dpdk/doc/guides/xen/pkt_switch.rst +++ /dev/null @@ -1,470 +0,0 @@ -.. BSD LICENSE - Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in - the documentation and/or other materials provided with the - distribution. - * Neither the name of Intel Corporation nor the names of its - contributors may be used to endorse or promote products derived - from this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -DPDK Xen Based Packet-Switching Solution -======================================== - -Introduction ------------- - -DPDK provides a para-virtualization packet switching solution, based on the Xen hypervisor's Grant Table, Note 1, -which provides simple and fast packet switching capability between guest domains and host domain based on MAC address or VLAN tag. - -This solution is comprised of two components; -a Poll Mode Driver (PMD) as the front end in the guest domain and a switching back end in the host domain. -XenStore is used to exchange configure information between the PMD front end and switching back end, -including grant reference IDs for shared Virtio RX/TX rings, -MAC address, device state, and so on. XenStore is an information storage space shared between domains, -see further information on XenStore below. - -The front end PMD can be found in the DPDK directory lib/ librte_pmd_xenvirt and back end example in examples/vhost_xen. - -The PMD front end and switching back end use shared Virtio RX/TX rings as para- virtualized interface. -The Virtio ring is created by the front end, and Grant table references for the ring are passed to host. -The switching back end maps those grant table references and creates shared rings in a mapped address space. - -The following diagram describes the functionality of the DPDK Xen Packet- Switching Solution. - - -.. _figure_dpdk_xen_pkt_switch: - -.. figure:: img/dpdk_xen_pkt_switch.* - - Functionality of the DPDK Xen Packet Switching Solution. - - -Note 1 The Xen hypervisor uses a mechanism called a Grant Table to share memory between domains -(`http://wiki.xen.org/wiki/Grant Table `_). - -A diagram of the design is shown below, where "gva" is the Guest Virtual Address, -which is the data pointer of the mbuf, and "hva" is the Host Virtual Address: - - -.. _figure_grant_table: - -.. figure:: img/grant_table.* - - DPDK Xen Layout - - -In this design, a Virtio ring is used as a para-virtualized interface for better performance over a Xen private ring -when packet switching to and from a VM. -The additional performance is gained by avoiding a system call and memory map in each memory copy with a XEN private ring. - -Device Creation ---------------- - -Poll Mode Driver Front End -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -* Mbuf pool allocation: - - To use a Xen switching solution, the DPDK application should use rte_mempool_gntalloc_create() - to reserve mbuf pools during initialization. - rte_mempool_gntalloc_create() creates a mempool with objects from memory allocated and managed via gntalloc/gntdev. - - The DPDK now supports construction of mempools from allocated virtual memory through the rte_mempool_xmem_create() API. - - This front end constructs mempools based on memory allocated through the xen_gntalloc driver. - rte_mempool_gntalloc_create() allocates Grant pages, maps them to continuous virtual address space, - and calls rte_mempool_xmem_create() to build mempools. - The Grant IDs for all Grant pages are passed to the host through XenStore. - -* Virtio Ring Creation: - - The Virtio queue size is defined as 256 by default in the VQ_DESC_NUM macro. - Using the queue setup function, - Grant pages are allocated based on ring size and are mapped to continuous virtual address space to form the Virtio ring. - Normally, one ring is comprised of several pages. - Their Grant IDs are passed to the host through XenStore. - - There is no requirement that this memory be physically continuous. - -* Interrupt and Kick: - - There are no interrupts in DPDK Xen Switching as both front and back ends work in polling mode. - There is no requirement for notification. - -* Feature Negotiation: - - Currently, feature negotiation through XenStore is not supported. - -* Packet Reception & Transmission: - - With mempools and Virtio rings created, the front end can operate Virtio devices, - as it does in Virtio PMD for KVM Virtio devices with the exception that the host - does not require notifications or deal with interrupts. - -XenStore is a database that stores guest and host information in the form of (key, value) pairs. -The following is an example of the information generated during the startup of the front end PMD in a guest VM (domain ID 1): - -.. code-block:: console - - xenstore -ls /local/domain/1/control/dpdk - 0_mempool_gref="3042,3043,3044,3045" - 0_mempool_va="0x7fcbc6881000" - 0_tx_vring_gref="3049" - 0_rx_vring_gref="3053" - 0_ether_addr="4e:0b:d0:4e:aa:f1" - 0_vring_flag="3054" - ... - -Multiple mempools and multiple Virtios may exist in the guest domain, the first number is the index, starting from zero. - -The idx#_mempool_va stores the guest virtual address for mempool idx#. - -The idx#_ether_adder stores the MAC address of the guest Virtio device. - -For idx#_rx_ring_gref, idx#_tx_ring_gref, and idx#_mempool_gref, the value is a list of Grant references. -Take idx#_mempool_gref node for example, the host maps those Grant references to a continuous virtual address space. -The real Grant reference information is stored in this virtual address space, -where (gref, pfn) pairs follow each other with -1 as the terminator. - - -.. _figure_grant_refs: - -.. figure:: img/grant_refs.* - - Mapping Grant references to a continuous virtual address space - - -After all gref# IDs are retrieved, the host maps them to a continuous virtual address space. -With the guest mempool virtual address, the host establishes 1:1 address mapping. -With multiple guest mempools, the host establishes multiple address translation regions. - -Switching Back End -~~~~~~~~~~~~~~~~~~ - -The switching back end monitors changes in XenStore. -When the back end detects that a new Virtio device has been created in a guest domain, it will: - -#. Retrieve Grant and configuration information from XenStore. - -#. Map and create a Virtio ring. - -#. Map mempools in the host and establish address translation between the guest address and host address. - -#. Select a free VMDQ pool, set its affinity with the Virtio device, and set the MAC/ VLAN filter. - -Packet Reception -~~~~~~~~~~~~~~~~ - -When packets arrive from an external network, the MAC?VLAN filter classifies packets into queues in one VMDQ pool. -As each pool is bonded to a Virtio device in some guest domain, the switching back end will: - -#. Fetch an available entry from the Virtio RX ring. - -#. Get gva, and translate it to hva. - -#. Copy the contents of the packet to the memory buffer pointed to by gva. - -The DPDK application in the guest domain, based on the PMD front end, -is polling the shared Virtio RX ring for available packets and receives them on arrival. - -Packet Transmission -~~~~~~~~~~~~~~~~~~~ - -When a Virtio device in one guest domain is to transmit a packet, -it puts the virtual address of the packet's data area into the shared Virtio TX ring. - -The packet switching back end is continuously polling the Virtio TX ring. -When new packets are available for transmission from a guest, it will: - -#. Fetch an available entry from the Virtio TX ring. - -#. Get gva, and translate it to hva. - -#. Copy the packet from hva to the host mbuf's data area. - -#. Compare the destination MAC address with all the MAC addresses of the Virtio devices it manages. - If a match exists, it directly copies the packet to the matched VIrtio RX ring. - Otherwise, it sends the packet out through hardware. - -.. note:: - - The packet switching back end is for demonstration purposes only. - The user could implement their switching logic based on this example. - In this example, only one physical port on the host is supported. - Multiple segments are not supported. The biggest mbuf supported is 4KB. - When the back end is restarted, all front ends must also be restarted. - -Running the Application ------------------------ - -The following describes the steps required to run the application. - -Validated Environment -~~~~~~~~~~~~~~~~~~~~~ - -Host: - - Xen-hypervisor: 4.2.2 - - Distribution: Fedora release 18 - - Kernel: 3.10.0 - - Xen development package (including Xen, Xen-libs, xen-devel): 4.2.3 - -Guest: - - Distribution: Fedora 16 and 18 - - Kernel: 3.6.11 - -Xen Host Prerequisites -~~~~~~~~~~~~~~~~~~~~~~ - -Note that the following commands might not be the same on different Linux* distributions. - -* Install xen-devel package: - - .. code-block:: console - - yum install xen-devel.x86_64 - -* Start xend if not already started: - - .. code-block:: console - - /etc/init.d/xend start - -* Mount xenfs if not already mounted: - - .. code-block:: console - - mount -t xenfs none /proc/xen - -* Enlarge the limit for xen_gntdev driver: - - .. code-block:: console - - modprobe -r xen_gntdev - modprobe xen_gntdev limit=1000000 - -.. note:: - - The default limit for earlier versions of the xen_gntdev driver is 1024. - That is insufficient to support the mapping of multiple Virtio devices into multiple VMs, - so it is necessary to enlarge the limit by reloading this module. - The default limit of recent versions of xen_gntdev is 1048576. - The rough calculation of this limit is: - - limit=nb_mbuf# * VM#. - - In DPDK examples, nb_mbuf# is normally 8192. - -Building and Running the Switching Backend -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -#. Edit config/common_linuxapp, and change the default configuration value for the following two items: - - .. code-block:: console - - CONFIG_RTE_LIBRTE_XEN_DOM0=y - CONFIG RTE_LIBRTE_PMD_XENVIRT=n - -#. Build the target: - - .. code-block:: console - - make install T=x86_64-native-linuxapp-gcc - -#. Ensure that RTE_SDK and RTE_TARGET are correctly set. Build the switching example: - - .. code-block:: console - - make -C examples/vhost_xen/ - -#. Load the Xen DPDK memory management module and preallocate memory: - - .. code-block:: console - - insmod ./x86_64-native-linuxapp-gcc/build/lib/librte_eal/linuxapp/xen_dom0/rte_dom0_mm.ko - echo 2048> /sys/kernel/mm/dom0-mm/memsize-mB/memsize - - .. note:: - - On Xen Dom0, there is no hugepage support. - Under Xen Dom0, the DPDK uses a special memory management kernel module - to allocate chunks of physically continuous memory. - Refer to the *DPDK Getting Started Guide* for more information on memory management in the DPDK. - In the above command, 4 GB memory is reserved (2048 of 2 MB pages) for DPDK. - -#. Load uio_pci_generic and bind one Intel NIC controller to it: - - .. code-block:: console - - modprobe uio_pci_generic - python tools/dpdk-devbind.py -b uio_pci_generic 0000:09:00:00.0 - - In this case, 0000:09:00.0 is the PCI address for the NIC controller. - -#. Run the switching back end example: - - .. code-block:: console - - examples/vhost_xen/build/vhost-switch -c f -n 3 --xen-dom0 -- -p1 - -.. note:: - - The -xen-dom0 option instructs the DPDK to use the Xen kernel module to allocate memory. - -Other Parameters: - -* -vm2vm - - The vm2vm parameter enables/disables packet switching in software. - Disabling vm2vm implies that on a VM packet transmission will always go to the Ethernet port - and will not be switched to another VM - -* -Stats - - The Stats parameter controls the printing of Virtio-net device statistics. - The parameter specifies the interval (in seconds) at which to print statistics, - an interval of 0 seconds will disable printing statistics. - -Xen PMD Frontend Prerequisites -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -#. Install xen-devel package for accessing XenStore: - - .. code-block:: console - - yum install xen-devel.x86_64 - -#. Mount xenfs, if it is not already mounted: - - .. code-block:: console - - mount -t xenfs none /proc/xen - -#. Enlarge the default limit for xen_gntalloc driver: - - .. code-block:: console - - modprobe -r xen_gntalloc - modprobe xen_gntalloc limit=6000 - -.. note:: - - Before the Linux kernel version 3.8-rc5, Jan 15th 2013, - a critical defect occurs when a guest is heavily allocating Grant pages. - The Grant driver allocates fewer pages than expected which causes kernel memory corruption. - This happens, for example, when a guest uses the v1 format of a Grant table entry and allocates - more than 8192 Grant pages (this number might be different on different hypervisor versions). - To work around this issue, set the limit for gntalloc driver to 6000. - (The kernel normally allocates hundreds of Grant pages with one Xen front end per virtualized device). - If the kernel allocates a lot of Grant pages, for example, if the user uses multiple net front devices, - it is best to upgrade the Grant alloc driver. - This defect has been fixed in kernel version 3.8-rc5 and later. - -Building and Running the Front End -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -#. Edit config/common_linuxapp, and change the default configuration value: - - .. code-block:: console - - CONFIG_RTE_LIBRTE_XEN_DOM0=n - CONFIG_RTE_LIBRTE_PMD_XENVIRT=y - -#. Build the package: - - .. code-block:: console - - make install T=x86_64-native-linuxapp-gcc - -#. Enable hugepages. Refer to the *DPDK Getting Started Guide* for instructions on - how to use hugepages in the DPDK. - -#. Run TestPMD. Refer to *DPDK TestPMD Application User Guide* for detailed parameter usage. - - .. code-block:: console - - ./x86_64-native-linuxapp-gcc/app/testpmd -c f -n 4 --vdev="eth_xenvirt0,mac=00:00:00:00:00:11" - testpmd>set fwd mac - testpmd>start - - As an example to run two TestPMD instances over 2 Xen Virtio devices: - - .. code-block:: console - - --vdev="eth_xenvirt0,mac=00:00:00:00:00:11" --vdev="eth_xenvirt1;mac=00:00:00:00:00:22" - - -Usage Examples: Injecting a Packet Stream Using a Packet Generator -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Loopback Mode -^^^^^^^^^^^^^ - -Run TestPMD in a guest VM: - -.. code-block:: console - - ./x86_64-native-linuxapp-gcc/app/testpmd -c f -n 4 --vdev="eth_xenvirt0,mac=00:00:00:00:00:11" -- -i --eth-peer=0,00:00:00:00:00:22 - testpmd> set fwd mac - testpmd> start - -Example output of the vhost_switch would be: - -.. code-block:: console - - DATA:(0) MAC_ADDRESS 00:00:00:00:00:11 and VLAN_TAG 1000 registered. - -The above message indicates that device 0 has been registered with MAC address 00:00:00:00:00:11 and VLAN tag 1000. -Any packets received on the NIC with these values is placed on the device's receive queue. - -Configure a packet stream in the packet generator, set the destination MAC address to 00:00:00:00:00:11, and VLAN to 1000, -the guest Virtio receives these packets and sends them out with destination MAC address 00:00:00:00:00:22. - -Inter-VM Mode -^^^^^^^^^^^^^ - -Run TestPMD in guest VM1: - -.. code-block:: console - - ./x86_64-native-linuxapp-gcc/app/testpmd -c f -n 4 --vdev="eth_xenvirt0,mac=00:00:00:00:00:11" -- -i --eth-peer=0,00:00:00:00:00:22 -- -i - -Run TestPMD in guest VM2: - -.. code-block:: console - - ./x86_64-native-linuxapp-gcc/app/testpmd -c f -n 4 --vdev="eth_xenvirt0,mac=00:00:00:00:00:22" -- -i --eth-peer=0,00:00:00:00:00:33 - -Configure a packet stream in the packet generator, and set the destination MAC address to 00:00:00:00:00:11 and VLAN to 1000. -The packets received in Virtio in guest VM1 will be forwarded to Virtio in guest VM2 and -then sent out through hardware with destination MAC address 00:00:00:00:00:33. - -The packet flow is: - -packet generator->Virtio in guest VM1->switching backend->Virtio in guest VM2->switching backend->wire diff --git a/dpdk/drivers/Makefile b/dpdk/drivers/Makefile index 81c03a85..db0cd76e 100644 --- a/dpdk/drivers/Makefile +++ b/dpdk/drivers/Makefile @@ -31,7 +31,14 @@ include $(RTE_SDK)/mk/rte.vars.mk +DIRS-y += bus +DIRS-y += mempool +DEPDIRS-mempool := bus DIRS-y += net +DEPDIRS-net := bus mempool DIRS-$(CONFIG_RTE_LIBRTE_CRYPTODEV) += crypto +DEPDIRS-crypto := bus mempool +DIRS-$(CONFIG_RTE_LIBRTE_EVENTDEV) += event +DEPDIRS-event := bus mempool net include $(RTE_SDK)/mk/rte.subdir.mk diff --git a/dpdk/drivers/bus/Makefile b/dpdk/drivers/bus/Makefile new file mode 100644 index 00000000..c20beb43 --- /dev/null +++ b/dpdk/drivers/bus/Makefile @@ -0,0 +1,38 @@ +# BSD LICENSE +# +# Copyright 2016 NXP. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of NXP nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +include $(RTE_SDK)/mk/rte.vars.mk + +DIRS-$(CONFIG_RTE_LIBRTE_DPAA_BUS) += dpaa +DIRS-$(CONFIG_RTE_LIBRTE_FSLMC_BUS) += fslmc +DIRS-$(CONFIG_RTE_LIBRTE_PCI_BUS) += pci +DIRS-$(CONFIG_RTE_LIBRTE_VDEV_BUS) += vdev + +include $(RTE_SDK)/mk/rte.subdir.mk diff --git a/dpdk/drivers/bus/dpaa/Makefile b/dpdk/drivers/bus/dpaa/Makefile new file mode 100644 index 00000000..f672f540 --- /dev/null +++ b/dpdk/drivers/bus/dpaa/Makefile @@ -0,0 +1,78 @@ +# BSD LICENSE +# +# Copyright 2016 NXP. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of NXP nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +include $(RTE_SDK)/mk/rte.vars.mk +RTE_BUS_DPAA=$(RTE_SDK)/drivers/bus/dpaa + +# +# library name +# +LIB = librte_bus_dpaa.a + +CFLAGS := -I$(SRCDIR) $(CFLAGS) +CFLAGS += -O3 $(WERROR_FLAGS) +CFLAGS += -Wno-pointer-arith +CFLAGS += -Wno-cast-qual +CFLAGS += -D _GNU_SOURCE +CFLAGS += -I$(RTE_BUS_DPAA)/ +CFLAGS += -I$(RTE_BUS_DPAA)/include +CFLAGS += -I$(RTE_BUS_DPAA)/base/qbman +CFLAGS += -I$(RTE_SDK)/lib/librte_eal/linuxapp/eal +CFLAGS += -I$(RTE_SDK)/lib/librte_eal/common/include + +# versioning export map +EXPORT_MAP := rte_bus_dpaa_version.map + +LIBABIVER := 1 + +# all source are stored in SRCS-y +# +SRCS-$(CONFIG_RTE_LIBRTE_DPAA_BUS) += \ + dpaa_bus.c + +SRCS-$(CONFIG_RTE_LIBRTE_DPAA_BUS) += \ + base/fman/fman.c \ + base/fman/fman_hw.c \ + base/fman/of.c \ + base/fman/netcfg_layer.c \ + base/qbman/process.c \ + base/qbman/bman.c \ + base/qbman/bman_driver.c \ + base/qbman/qman.c \ + base/qbman/qman_driver.c \ + base/qbman/dpaa_alloc.c \ + base/qbman/dpaa_sys.c + +# Link Pthread +LDLIBS += -lpthread +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_ethdev + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/bus/dpaa/base/fman/fman.c b/dpdk/drivers/bus/dpaa/base/fman/fman.c new file mode 100644 index 00000000..3816dba5 --- /dev/null +++ b/dpdk/drivers/bus/dpaa/base/fman/fman.c @@ -0,0 +1,612 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2010-2016 Freescale Semiconductor Inc. + * Copyright 2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include + +/* This header declares the driver interface we implement */ +#include +#include +#include + +#define QMI_PORT_REGS_OFFSET 0x400 + +/* CCSR map address to access ccsr based register */ +void *fman_ccsr_map; +/* fman version info */ +u16 fman_ip_rev; +static int get_once; +u32 fman_dealloc_bufs_mask_hi; +u32 fman_dealloc_bufs_mask_lo; + +int fman_ccsr_map_fd = -1; +static COMPAT_LIST_HEAD(__ifs); + +/* This is the (const) global variable that callers have read-only access to. + * Internally, we have read-write access directly to __ifs. + */ +const struct list_head *fman_if_list = &__ifs; + +static void +if_destructor(struct __fman_if *__if) +{ + struct fman_if_bpool *bp, *tmpbp; + + if (!__if) + return; + + if (__if->__if.mac_type == fman_offline) + goto cleanup; + + list_for_each_entry_safe(bp, tmpbp, &__if->__if.bpool_list, node) { + list_del(&bp->node); + free(bp); + } +cleanup: + free(__if); +} + +static int +fman_get_ip_rev(const struct device_node *fman_node) +{ + const uint32_t *fman_addr; + uint64_t phys_addr; + uint64_t regs_size; + uint32_t ip_rev_1; + int _errno; + + fman_addr = of_get_address(fman_node, 0, ®s_size, NULL); + if (!fman_addr) { + pr_err("of_get_address cannot return fman address\n"); + return -EINVAL; + } + phys_addr = of_translate_address(fman_node, fman_addr); + if (!phys_addr) { + pr_err("of_translate_address failed\n"); + return -EINVAL; + } + fman_ccsr_map = mmap(NULL, regs_size, PROT_READ | PROT_WRITE, + MAP_SHARED, fman_ccsr_map_fd, phys_addr); + if (fman_ccsr_map == MAP_FAILED) { + pr_err("Can not map FMan ccsr base"); + return -EINVAL; + } + + ip_rev_1 = in_be32(fman_ccsr_map + FMAN_IP_REV_1); + fman_ip_rev = (ip_rev_1 & FMAN_IP_REV_1_MAJOR_MASK) >> + FMAN_IP_REV_1_MAJOR_SHIFT; + + _errno = munmap(fman_ccsr_map, regs_size); + if (_errno) + pr_err("munmap() of FMan ccsr failed"); + + return 0; +} + +static int +fman_get_mac_index(uint64_t regs_addr_host, uint8_t *mac_idx) +{ + int ret = 0; + + /* + * MAC1 : E_0000h + * MAC2 : E_2000h + * MAC3 : E_4000h + * MAC4 : E_6000h + * MAC5 : E_8000h + * MAC6 : E_A000h + * MAC7 : E_C000h + * MAC8 : E_E000h + * MAC9 : F_0000h + * MAC10: F_2000h + */ + switch (regs_addr_host) { + case 0xE0000: + *mac_idx = 1; + break; + case 0xE2000: + *mac_idx = 2; + break; + case 0xE4000: + *mac_idx = 3; + break; + case 0xE6000: + *mac_idx = 4; + break; + case 0xE8000: + *mac_idx = 5; + break; + case 0xEA000: + *mac_idx = 6; + break; + case 0xEC000: + *mac_idx = 7; + break; + case 0xEE000: + *mac_idx = 8; + break; + case 0xF0000: + *mac_idx = 9; + break; + case 0xF2000: + *mac_idx = 10; + break; + default: + ret = -EINVAL; + } + + return ret; +} + +static int +fman_if_init(const struct device_node *dpa_node) +{ + const char *rprop, *mprop; + uint64_t phys_addr; + struct __fman_if *__if; + struct fman_if_bpool *bpool; + + const phandle *mac_phandle, *ports_phandle, *pools_phandle; + const phandle *tx_channel_id = NULL, *mac_addr, *cell_idx; + const phandle *rx_phandle, *tx_phandle; + uint64_t tx_phandle_host[4] = {0}; + uint64_t rx_phandle_host[4] = {0}; + uint64_t regs_addr_host = 0; + uint64_t cell_idx_host = 0; + + const struct device_node *mac_node = NULL, *tx_node; + const struct device_node *pool_node, *fman_node, *rx_node; + const uint32_t *regs_addr = NULL; + const char *mname, *fname; + const char *dname = dpa_node->full_name; + size_t lenp; + int _errno; + const char *char_prop; + uint32_t na; + + if (of_device_is_available(dpa_node) == false) + return 0; + + rprop = "fsl,qman-frame-queues-rx"; + mprop = "fsl,fman-mac"; + + /* Allocate an object for this network interface */ + __if = malloc(sizeof(*__if)); + if (!__if) { + FMAN_ERR(-ENOMEM, "malloc(%zu)\n", sizeof(*__if)); + goto err; + } + memset(__if, 0, sizeof(*__if)); + INIT_LIST_HEAD(&__if->__if.bpool_list); + strncpy(__if->node_path, dpa_node->full_name, PATH_MAX - 1); + __if->node_path[PATH_MAX - 1] = '\0'; + + /* Obtain the MAC node used by this interface except macless */ + mac_phandle = of_get_property(dpa_node, mprop, &lenp); + if (!mac_phandle) { + FMAN_ERR(-EINVAL, "%s: no %s\n", dname, mprop); + goto err; + } + assert(lenp == sizeof(phandle)); + mac_node = of_find_node_by_phandle(*mac_phandle); + if (!mac_node) { + FMAN_ERR(-ENXIO, "%s: bad 'fsl,fman-mac\n", dname); + goto err; + } + mname = mac_node->full_name; + + /* Map the CCSR regs for the MAC node */ + regs_addr = of_get_address(mac_node, 0, &__if->regs_size, NULL); + if (!regs_addr) { + FMAN_ERR(-EINVAL, "of_get_address(%s)\n", mname); + goto err; + } + phys_addr = of_translate_address(mac_node, regs_addr); + if (!phys_addr) { + FMAN_ERR(-EINVAL, "of_translate_address(%s, %p)\n", + mname, regs_addr); + goto err; + } + __if->ccsr_map = mmap(NULL, __if->regs_size, + PROT_READ | PROT_WRITE, MAP_SHARED, + fman_ccsr_map_fd, phys_addr); + if (__if->ccsr_map == MAP_FAILED) { + FMAN_ERR(-errno, "mmap(0x%"PRIx64")\n", phys_addr); + goto err; + } + na = of_n_addr_cells(mac_node); + /* Get rid of endianness (issues). Convert to host byte order */ + regs_addr_host = of_read_number(regs_addr, na); + + + /* Get the index of the Fman this i/f belongs to */ + fman_node = of_get_parent(mac_node); + na = of_n_addr_cells(mac_node); + if (!fman_node) { + FMAN_ERR(-ENXIO, "of_get_parent(%s)\n", mname); + goto err; + } + fname = fman_node->full_name; + cell_idx = of_get_property(fman_node, "cell-index", &lenp); + if (!cell_idx) { + FMAN_ERR(-ENXIO, "%s: no cell-index)\n", fname); + goto err; + } + assert(lenp == sizeof(*cell_idx)); + cell_idx_host = of_read_number(cell_idx, lenp / sizeof(phandle)); + __if->__if.fman_idx = cell_idx_host; + if (!get_once) { + _errno = fman_get_ip_rev(fman_node); + if (_errno) { + FMAN_ERR(-ENXIO, "%s: ip_rev is not available\n", + fname); + goto err; + } + } + + if (fman_ip_rev >= FMAN_V3) { + /* + * Set A2V, OVOM, EBD bits in contextA to allow external + * buffer deallocation by fman. + */ + fman_dealloc_bufs_mask_hi = FMAN_V3_CONTEXTA_EN_A2V | + FMAN_V3_CONTEXTA_EN_OVOM; + fman_dealloc_bufs_mask_lo = FMAN_V3_CONTEXTA_EN_EBD; + } else { + fman_dealloc_bufs_mask_hi = 0; + fman_dealloc_bufs_mask_lo = 0; + } + /* Is the MAC node 1G, 10G? */ + __if->__if.is_memac = 0; + + if (of_device_is_compatible(mac_node, "fsl,fman-1g-mac")) + __if->__if.mac_type = fman_mac_1g; + else if (of_device_is_compatible(mac_node, "fsl,fman-10g-mac")) + __if->__if.mac_type = fman_mac_10g; + else if (of_device_is_compatible(mac_node, "fsl,fman-memac")) { + __if->__if.is_memac = 1; + char_prop = of_get_property(mac_node, "phy-connection-type", + NULL); + if (!char_prop) { + printf("memac: unknown MII type assuming 1G\n"); + /* Right now forcing memac to 1g in case of error*/ + __if->__if.mac_type = fman_mac_1g; + } else { + if (strstr(char_prop, "sgmii")) + __if->__if.mac_type = fman_mac_1g; + else if (strstr(char_prop, "rgmii")) { + __if->__if.mac_type = fman_mac_1g; + __if->__if.is_rgmii = 1; + } else if (strstr(char_prop, "xgmii")) + __if->__if.mac_type = fman_mac_10g; + } + } else { + FMAN_ERR(-EINVAL, "%s: unknown MAC type\n", mname); + goto err; + } + + /* + * For MAC ports, we cannot rely on cell-index. In + * T2080, two of the 10G ports on single FMAN have same + * duplicate cell-indexes as the other two 10G ports on + * same FMAN. Hence, we now rely upon addresses of the + * ports from device tree to deduce the index. + */ + + _errno = fman_get_mac_index(regs_addr_host, &__if->__if.mac_idx); + if (_errno) { + FMAN_ERR(-EINVAL, "Invalid register address: %lu", + regs_addr_host); + goto err; + } + + /* Extract the MAC address for private and shared interfaces */ + mac_addr = of_get_property(mac_node, "local-mac-address", + &lenp); + if (!mac_addr) { + FMAN_ERR(-EINVAL, "%s: no local-mac-address\n", + mname); + goto err; + } + memcpy(&__if->__if.mac_addr, mac_addr, ETHER_ADDR_LEN); + + /* Extract the Tx port (it's the second of the two port handles) + * and get its channel ID + */ + ports_phandle = of_get_property(mac_node, "fsl,port-handles", + &lenp); + if (!ports_phandle) + ports_phandle = of_get_property(mac_node, "fsl,fman-ports", + &lenp); + if (!ports_phandle) { + FMAN_ERR(-EINVAL, "%s: no fsl,port-handles\n", + mname); + goto err; + } + assert(lenp == (2 * sizeof(phandle))); + tx_node = of_find_node_by_phandle(ports_phandle[1]); + if (!tx_node) { + FMAN_ERR(-ENXIO, "%s: bad fsl,port-handle[1]\n", mname); + goto err; + } + /* Extract the channel ID (from tx-port-handle) */ + tx_channel_id = of_get_property(tx_node, "fsl,qman-channel-id", + &lenp); + if (!tx_channel_id) { + FMAN_ERR(-EINVAL, "%s: no fsl-qman-channel-id\n", + tx_node->full_name); + goto err; + } + + rx_node = of_find_node_by_phandle(ports_phandle[0]); + if (!rx_node) { + FMAN_ERR(-ENXIO, "%s: bad fsl,port-handle[0]\n", mname); + goto err; + } + regs_addr = of_get_address(rx_node, 0, &__if->regs_size, NULL); + if (!regs_addr) { + FMAN_ERR(-EINVAL, "of_get_address(%s)\n", mname); + goto err; + } + phys_addr = of_translate_address(rx_node, regs_addr); + if (!phys_addr) { + FMAN_ERR(-EINVAL, "of_translate_address(%s, %p)\n", + mname, regs_addr); + goto err; + } + __if->bmi_map = mmap(NULL, __if->regs_size, + PROT_READ | PROT_WRITE, MAP_SHARED, + fman_ccsr_map_fd, phys_addr); + if (__if->bmi_map == MAP_FAILED) { + FMAN_ERR(-errno, "mmap(0x%"PRIx64")\n", phys_addr); + goto err; + } + + /* No channel ID for MAC-less */ + assert(lenp == sizeof(*tx_channel_id)); + na = of_n_addr_cells(mac_node); + __if->__if.tx_channel_id = of_read_number(tx_channel_id, na); + + /* Extract the Rx FQIDs. (Note, the device representation is silly, + * there are "counts" that must always be 1.) + */ + rx_phandle = of_get_property(dpa_node, rprop, &lenp); + if (!rx_phandle) { + FMAN_ERR(-EINVAL, "%s: no fsl,qman-frame-queues-rx\n", dname); + goto err; + } + + assert(lenp == (4 * sizeof(phandle))); + + na = of_n_addr_cells(mac_node); + /* Get rid of endianness (issues). Convert to host byte order */ + rx_phandle_host[0] = of_read_number(&rx_phandle[0], na); + rx_phandle_host[1] = of_read_number(&rx_phandle[1], na); + rx_phandle_host[2] = of_read_number(&rx_phandle[2], na); + rx_phandle_host[3] = of_read_number(&rx_phandle[3], na); + + assert((rx_phandle_host[1] == 1) && (rx_phandle_host[3] == 1)); + __if->__if.fqid_rx_err = rx_phandle_host[0]; + __if->__if.fqid_rx_def = rx_phandle_host[2]; + + /* Extract the Tx FQIDs */ + tx_phandle = of_get_property(dpa_node, + "fsl,qman-frame-queues-tx", &lenp); + if (!tx_phandle) { + FMAN_ERR(-EINVAL, "%s: no fsl,qman-frame-queues-tx\n", dname); + goto err; + } + + assert(lenp == (4 * sizeof(phandle))); + /*TODO: Fix for other cases also */ + na = of_n_addr_cells(mac_node); + /* Get rid of endianness (issues). Convert to host byte order */ + tx_phandle_host[0] = of_read_number(&tx_phandle[0], na); + tx_phandle_host[1] = of_read_number(&tx_phandle[1], na); + tx_phandle_host[2] = of_read_number(&tx_phandle[2], na); + tx_phandle_host[3] = of_read_number(&tx_phandle[3], na); + assert((tx_phandle_host[1] == 1) && (tx_phandle_host[3] == 1)); + __if->__if.fqid_tx_err = tx_phandle_host[0]; + __if->__if.fqid_tx_confirm = tx_phandle_host[2]; + + /* Obtain the buffer pool nodes used by this interface */ + pools_phandle = of_get_property(dpa_node, "fsl,bman-buffer-pools", + &lenp); + if (!pools_phandle) { + FMAN_ERR(-EINVAL, "%s: no fsl,bman-buffer-pools\n", dname); + goto err; + } + /* For each pool, parse the corresponding node and add a pool object + * to the interface's "bpool_list" + */ + assert(lenp && !(lenp % sizeof(phandle))); + while (lenp) { + size_t proplen; + const phandle *prop; + uint64_t bpid_host = 0; + uint64_t bpool_host[6] = {0}; + const char *pname; + /* Allocate an object for the pool */ + bpool = malloc(sizeof(*bpool)); + if (!bpool) { + FMAN_ERR(-ENOMEM, "malloc(%zu)\n", sizeof(*bpool)); + goto err; + } + /* Find the pool node */ + pool_node = of_find_node_by_phandle(*pools_phandle); + if (!pool_node) { + FMAN_ERR(-ENXIO, "%s: bad fsl,bman-buffer-pools\n", + dname); + goto err; + } + pname = pool_node->full_name; + /* Extract the BPID property */ + prop = of_get_property(pool_node, "fsl,bpid", &proplen); + if (!prop) { + FMAN_ERR(-EINVAL, "%s: no fsl,bpid\n", pname); + goto err; + } + assert(proplen == sizeof(*prop)); + na = of_n_addr_cells(mac_node); + /* Get rid of endianness (issues). + * Convert to host byte-order + */ + bpid_host = of_read_number(prop, na); + bpool->bpid = bpid_host; + /* Extract the cfg property (count/size/addr). "fsl,bpool-cfg" + * indicates for the Bman driver to seed the pool. + * "fsl,bpool-ethernet-cfg" is used by the network driver. The + * two are mutually exclusive, so check for either of them. + */ + prop = of_get_property(pool_node, "fsl,bpool-cfg", + &proplen); + if (!prop) + prop = of_get_property(pool_node, + "fsl,bpool-ethernet-cfg", + &proplen); + if (!prop) { + /* It's OK for there to be no bpool-cfg */ + bpool->count = bpool->size = bpool->addr = 0; + } else { + assert(proplen == (6 * sizeof(*prop))); + na = of_n_addr_cells(mac_node); + /* Get rid of endianness (issues). + * Convert to host byte order + */ + bpool_host[0] = of_read_number(&prop[0], na); + bpool_host[1] = of_read_number(&prop[1], na); + bpool_host[2] = of_read_number(&prop[2], na); + bpool_host[3] = of_read_number(&prop[3], na); + bpool_host[4] = of_read_number(&prop[4], na); + bpool_host[5] = of_read_number(&prop[5], na); + + bpool->count = ((uint64_t)bpool_host[0] << 32) | + bpool_host[1]; + bpool->size = ((uint64_t)bpool_host[2] << 32) | + bpool_host[3]; + bpool->addr = ((uint64_t)bpool_host[4] << 32) | + bpool_host[5]; + } + /* Parsing of the pool is complete, add it to the interface + * list. + */ + list_add_tail(&bpool->node, &__if->__if.bpool_list); + lenp -= sizeof(phandle); + pools_phandle++; + } + + /* Parsing of the network interface is complete, add it to the list */ + DPAA_BUS_LOG(DEBUG, "Found %s, Tx Channel = %x, FMAN = %x," + "Port ID = %x\n", + dname, __if->__if.tx_channel_id, __if->__if.fman_idx, + __if->__if.mac_idx); + + list_add_tail(&__if->__if.node, &__ifs); + return 0; +err: + if_destructor(__if); + return _errno; +} + +int +fman_init(void) +{ + const struct device_node *dpa_node; + int _errno; + + /* If multiple dependencies try to initialise the Fman driver, don't + * panic. + */ + if (fman_ccsr_map_fd != -1) + return 0; + + fman_ccsr_map_fd = open(FMAN_DEVICE_PATH, O_RDWR); + if (unlikely(fman_ccsr_map_fd < 0)) { + DPAA_BUS_LOG(ERR, "Unable to open (/dev/mem)"); + return fman_ccsr_map_fd; + } + + for_each_compatible_node(dpa_node, NULL, "fsl,dpa-ethernet-init") { + _errno = fman_if_init(dpa_node); + if (_errno) { + FMAN_ERR(_errno, "if_init(%s)\n", dpa_node->full_name); + goto err; + } + } + + return 0; +err: + fman_finish(); + return _errno; +} + +void +fman_finish(void) +{ + struct __fman_if *__if, *tmpif; + + assert(fman_ccsr_map_fd != -1); + + list_for_each_entry_safe(__if, tmpif, &__ifs, __if.node) { + int _errno; + + /* disable Rx and Tx */ + if ((__if->__if.mac_type == fman_mac_1g) && + (!__if->__if.is_memac)) + out_be32(__if->ccsr_map + 0x100, + in_be32(__if->ccsr_map + 0x100) & ~(u32)0x5); + else + out_be32(__if->ccsr_map + 8, + in_be32(__if->ccsr_map + 8) & ~(u32)3); + /* release the mapping */ + _errno = munmap(__if->ccsr_map, __if->regs_size); + if (unlikely(_errno < 0)) + fprintf(stderr, "%s:%hu:%s(): munmap() = %d (%s)\n", + __FILE__, __LINE__, __func__, + -errno, strerror(errno)); + printf("Tearing down %s\n", __if->node_path); + list_del(&__if->__if.node); + free(__if); + } + + close(fman_ccsr_map_fd); + fman_ccsr_map_fd = -1; +} diff --git a/dpdk/drivers/bus/dpaa/base/fman/fman_hw.c b/dpdk/drivers/bus/dpaa/base/fman/fman_hw.c new file mode 100644 index 00000000..077c17c0 --- /dev/null +++ b/dpdk/drivers/bus/dpaa/base/fman/fman_hw.c @@ -0,0 +1,590 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +/* This header declares things about Fman hardware itself (the format of status + * words and an inline implementation of CRC64). We include it only in order to + * instantiate the one global variable it depends on. + */ +#include +#include +#include + +/* Instantiate the global variable that the inline CRC64 implementation (in + * ) depends on. + */ +DECLARE_FMAN_CRC64_TABLE(); + +#define ETH_ADDR_TO_UINT64(eth_addr) \ + (uint64_t)(((uint64_t)(eth_addr)[0] << 40) | \ + ((uint64_t)(eth_addr)[1] << 32) | \ + ((uint64_t)(eth_addr)[2] << 24) | \ + ((uint64_t)(eth_addr)[3] << 16) | \ + ((uint64_t)(eth_addr)[4] << 8) | \ + ((uint64_t)(eth_addr)[5])) + +void +fman_if_set_mcast_filter_table(struct fman_if *p) +{ + struct __fman_if *__if = container_of(p, struct __fman_if, __if); + void *hashtable_ctrl; + uint32_t i; + + hashtable_ctrl = &((struct memac_regs *)__if->ccsr_map)->hashtable_ctrl; + for (i = 0; i < 64; i++) + out_be32(hashtable_ctrl, i|HASH_CTRL_MCAST_EN); +} + +void +fman_if_reset_mcast_filter_table(struct fman_if *p) +{ + struct __fman_if *__if = container_of(p, struct __fman_if, __if); + void *hashtable_ctrl; + uint32_t i; + + hashtable_ctrl = &((struct memac_regs *)__if->ccsr_map)->hashtable_ctrl; + for (i = 0; i < 64; i++) + out_be32(hashtable_ctrl, i & ~HASH_CTRL_MCAST_EN); +} + +static +uint32_t get_mac_hash_code(uint64_t eth_addr) +{ + uint64_t mask1, mask2; + uint32_t xorVal = 0; + uint8_t i, j; + + for (i = 0; i < 6; i++) { + mask1 = eth_addr & (uint64_t)0x01; + eth_addr >>= 1; + + for (j = 0; j < 7; j++) { + mask2 = eth_addr & (uint64_t)0x01; + mask1 ^= mask2; + eth_addr >>= 1; + } + + xorVal |= (mask1 << (5 - i)); + } + + return xorVal; +} + +int +fman_if_add_hash_mac_addr(struct fman_if *p, uint8_t *eth) +{ + uint64_t eth_addr; + void *hashtable_ctrl; + uint32_t hash; + + struct __fman_if *__if = container_of(p, struct __fman_if, __if); + + eth_addr = ETH_ADDR_TO_UINT64(eth); + + if (!(eth_addr & GROUP_ADDRESS)) + return -1; + + hash = get_mac_hash_code(eth_addr) & HASH_CTRL_ADDR_MASK; + hash = hash | HASH_CTRL_MCAST_EN; + + hashtable_ctrl = &((struct memac_regs *)__if->ccsr_map)->hashtable_ctrl; + out_be32(hashtable_ctrl, hash); + + return 0; +} + +int +fman_if_get_primary_mac_addr(struct fman_if *p, uint8_t *eth) +{ + struct __fman_if *__if = container_of(p, struct __fman_if, __if); + void *mac_reg = + &((struct memac_regs *)__if->ccsr_map)->mac_addr0.mac_addr_l; + u32 val = in_be32(mac_reg); + + eth[0] = (val & 0x000000ff) >> 0; + eth[1] = (val & 0x0000ff00) >> 8; + eth[2] = (val & 0x00ff0000) >> 16; + eth[3] = (val & 0xff000000) >> 24; + + mac_reg = &((struct memac_regs *)__if->ccsr_map)->mac_addr0.mac_addr_u; + val = in_be32(mac_reg); + + eth[4] = (val & 0x000000ff) >> 0; + eth[5] = (val & 0x0000ff00) >> 8; + + return 0; +} + +void +fman_if_clear_mac_addr(struct fman_if *p, uint8_t addr_num) +{ + struct __fman_if *m = container_of(p, struct __fman_if, __if); + void *reg; + + if (addr_num) { + reg = &((struct memac_regs *)m->ccsr_map)-> + mac_addr[addr_num-1].mac_addr_l; + out_be32(reg, 0x0); + reg = &((struct memac_regs *)m->ccsr_map)-> + mac_addr[addr_num-1].mac_addr_u; + out_be32(reg, 0x0); + } else { + reg = &((struct memac_regs *)m->ccsr_map)->mac_addr0.mac_addr_l; + out_be32(reg, 0x0); + reg = &((struct memac_regs *)m->ccsr_map)->mac_addr0.mac_addr_u; + out_be32(reg, 0x0); + } +} + +int +fman_if_add_mac_addr(struct fman_if *p, uint8_t *eth, uint8_t addr_num) +{ + struct __fman_if *m = container_of(p, struct __fman_if, __if); + + void *reg; + u32 val; + + memcpy(&m->__if.mac_addr, eth, ETHER_ADDR_LEN); + + if (addr_num) + reg = &((struct memac_regs *)m->ccsr_map)-> + mac_addr[addr_num-1].mac_addr_l; + else + reg = &((struct memac_regs *)m->ccsr_map)->mac_addr0.mac_addr_l; + + val = (m->__if.mac_addr.addr_bytes[0] | + (m->__if.mac_addr.addr_bytes[1] << 8) | + (m->__if.mac_addr.addr_bytes[2] << 16) | + (m->__if.mac_addr.addr_bytes[3] << 24)); + out_be32(reg, val); + + if (addr_num) + reg = &((struct memac_regs *)m->ccsr_map)-> + mac_addr[addr_num-1].mac_addr_u; + else + reg = &((struct memac_regs *)m->ccsr_map)->mac_addr0.mac_addr_u; + + val = ((m->__if.mac_addr.addr_bytes[4] << 0) | + (m->__if.mac_addr.addr_bytes[5] << 8)); + out_be32(reg, val); + + return 0; +} + +void +fman_if_set_rx_ignore_pause_frames(struct fman_if *p, bool enable) +{ + struct __fman_if *__if = container_of(p, struct __fman_if, __if); + u32 value = 0; + void *cmdcfg; + + assert(fman_ccsr_map_fd != -1); + + /* Set Rx Ignore Pause Frames */ + cmdcfg = &((struct memac_regs *)__if->ccsr_map)->command_config; + if (enable) + value = in_be32(cmdcfg) | CMD_CFG_PAUSE_IGNORE; + else + value = in_be32(cmdcfg) & ~CMD_CFG_PAUSE_IGNORE; + + out_be32(cmdcfg, value); +} + +void +fman_if_conf_max_frame_len(struct fman_if *p, unsigned int max_frame_len) +{ + struct __fman_if *__if = container_of(p, struct __fman_if, __if); + unsigned int *maxfrm; + + assert(fman_ccsr_map_fd != -1); + + /* Set Max frame length */ + maxfrm = &((struct memac_regs *)__if->ccsr_map)->maxfrm; + out_be32(maxfrm, (MAXFRM_RX_MASK & max_frame_len)); +} + +void +fman_if_stats_get(struct fman_if *p, struct rte_eth_stats *stats) +{ + struct __fman_if *m = container_of(p, struct __fman_if, __if); + struct memac_regs *regs = m->ccsr_map; + + /* read recved packet count */ + stats->ipackets = ((u64)in_be32(®s->rfrm_u)) << 32 | + in_be32(®s->rfrm_l); + stats->ibytes = ((u64)in_be32(®s->roct_u)) << 32 | + in_be32(®s->roct_l); + stats->ierrors = ((u64)in_be32(®s->rerr_u)) << 32 | + in_be32(®s->rerr_l); + + /* read xmited packet count */ + stats->opackets = ((u64)in_be32(®s->tfrm_u)) << 32 | + in_be32(®s->tfrm_l); + stats->obytes = ((u64)in_be32(®s->toct_u)) << 32 | + in_be32(®s->toct_l); + stats->oerrors = ((u64)in_be32(®s->terr_u)) << 32 | + in_be32(®s->terr_l); +} + +void +fman_if_stats_get_all(struct fman_if *p, uint64_t *value, int n) +{ + struct __fman_if *m = container_of(p, struct __fman_if, __if); + struct memac_regs *regs = m->ccsr_map; + int i; + uint64_t base_offset = offsetof(struct memac_regs, reoct_l); + + for (i = 0; i < n; i++) + value[i] = ((u64)in_be32((char *)regs + + base_offset + 8 * i + 4)) << 32 | + ((u64)in_be32((char *)regs + + base_offset + 8 * i)); +} + +void +fman_if_stats_reset(struct fman_if *p) +{ + struct __fman_if *m = container_of(p, struct __fman_if, __if); + struct memac_regs *regs = m->ccsr_map; + uint32_t tmp; + + tmp = in_be32(®s->statn_config); + + tmp |= STATS_CFG_CLR; + + out_be32(®s->statn_config, tmp); + + while (in_be32(®s->statn_config) & STATS_CFG_CLR) + ; +} + +void +fman_if_promiscuous_enable(struct fman_if *p) +{ + struct __fman_if *__if = container_of(p, struct __fman_if, __if); + void *cmdcfg; + + assert(fman_ccsr_map_fd != -1); + + /* Enable Rx promiscuous mode */ + cmdcfg = &((struct memac_regs *)__if->ccsr_map)->command_config; + out_be32(cmdcfg, in_be32(cmdcfg) | CMD_CFG_PROMIS_EN); +} + +void +fman_if_promiscuous_disable(struct fman_if *p) +{ + struct __fman_if *__if = container_of(p, struct __fman_if, __if); + void *cmdcfg; + + assert(fman_ccsr_map_fd != -1); + + /* Disable Rx promiscuous mode */ + cmdcfg = &((struct memac_regs *)__if->ccsr_map)->command_config; + out_be32(cmdcfg, in_be32(cmdcfg) & (~CMD_CFG_PROMIS_EN)); +} + +void +fman_if_enable_rx(struct fman_if *p) +{ + struct __fman_if *__if = container_of(p, struct __fman_if, __if); + + assert(fman_ccsr_map_fd != -1); + + /* enable Rx and Tx */ + out_be32(__if->ccsr_map + 8, in_be32(__if->ccsr_map + 8) | 3); +} + +void +fman_if_disable_rx(struct fman_if *p) +{ + struct __fman_if *__if = container_of(p, struct __fman_if, __if); + + assert(fman_ccsr_map_fd != -1); + + /* only disable Rx, not Tx */ + out_be32(__if->ccsr_map + 8, in_be32(__if->ccsr_map + 8) & ~(u32)2); +} + +void +fman_if_loopback_enable(struct fman_if *p) +{ + struct __fman_if *__if = container_of(p, struct __fman_if, __if); + + assert(fman_ccsr_map_fd != -1); + + /* Enable loopback mode */ + if ((__if->__if.is_memac) && (__if->__if.is_rgmii)) { + unsigned int *ifmode = + &((struct memac_regs *)__if->ccsr_map)->if_mode; + out_be32(ifmode, in_be32(ifmode) | IF_MODE_RLP); + } else{ + unsigned int *cmdcfg = + &((struct memac_regs *)__if->ccsr_map)->command_config; + out_be32(cmdcfg, in_be32(cmdcfg) | CMD_CFG_LOOPBACK_EN); + } +} + +void +fman_if_loopback_disable(struct fman_if *p) +{ + struct __fman_if *__if = container_of(p, struct __fman_if, __if); + + assert(fman_ccsr_map_fd != -1); + /* Disable loopback mode */ + if ((__if->__if.is_memac) && (__if->__if.is_rgmii)) { + unsigned int *ifmode = + &((struct memac_regs *)__if->ccsr_map)->if_mode; + out_be32(ifmode, in_be32(ifmode) & ~IF_MODE_RLP); + } else { + unsigned int *cmdcfg = + &((struct memac_regs *)__if->ccsr_map)->command_config; + out_be32(cmdcfg, in_be32(cmdcfg) & ~CMD_CFG_LOOPBACK_EN); + } +} + +void +fman_if_set_bp(struct fman_if *fm_if, unsigned num __always_unused, + int bpid, size_t bufsize) +{ + u32 fmbm_ebmpi; + u32 ebmpi_val_ace = 0xc0000000; + u32 ebmpi_mask = 0xffc00000; + + struct __fman_if *__if = container_of(fm_if, struct __fman_if, __if); + + assert(fman_ccsr_map_fd != -1); + + fmbm_ebmpi = + in_be32(&((struct rx_bmi_regs *)__if->bmi_map)->fmbm_ebmpi[0]); + fmbm_ebmpi = ebmpi_val_ace | (fmbm_ebmpi & ebmpi_mask) | (bpid << 16) | + (bufsize); + + out_be32(&((struct rx_bmi_regs *)__if->bmi_map)->fmbm_ebmpi[0], + fmbm_ebmpi); +} + +int +fman_if_get_fc_threshold(struct fman_if *fm_if) +{ + struct __fman_if *__if = container_of(fm_if, struct __fman_if, __if); + unsigned int *fmbm_mpd; + + assert(fman_ccsr_map_fd != -1); + + fmbm_mpd = &((struct rx_bmi_regs *)__if->bmi_map)->fmbm_mpd; + return in_be32(fmbm_mpd); +} + +int +fman_if_set_fc_threshold(struct fman_if *fm_if, u32 high_water, + u32 low_water, u32 bpid) +{ + struct __fman_if *__if = container_of(fm_if, struct __fman_if, __if); + unsigned int *fmbm_mpd; + + assert(fman_ccsr_map_fd != -1); + + fmbm_mpd = &((struct rx_bmi_regs *)__if->bmi_map)->fmbm_mpd; + out_be32(fmbm_mpd, FMAN_ENABLE_BPOOL_DEPLETION); + return bm_pool_set_hw_threshold(bpid, low_water, high_water); + +} + +int +fman_if_get_fc_quanta(struct fman_if *fm_if) +{ + struct __fman_if *__if = container_of(fm_if, struct __fman_if, __if); + + assert(fman_ccsr_map_fd != -1); + + return in_be32(&((struct memac_regs *)__if->ccsr_map)->pause_quanta[0]); +} + +int +fman_if_set_fc_quanta(struct fman_if *fm_if, u16 pause_quanta) +{ + struct __fman_if *__if = container_of(fm_if, struct __fman_if, __if); + + assert(fman_ccsr_map_fd != -1); + + out_be32(&((struct memac_regs *)__if->ccsr_map)->pause_quanta[0], + pause_quanta); + return 0; +} + +int +fman_if_get_fdoff(struct fman_if *fm_if) +{ + u32 fmbm_ricp; + int fdoff; + int iceof_mask = 0x001f0000; + int icsz_mask = 0x0000001f; + + struct __fman_if *__if = container_of(fm_if, struct __fman_if, __if); + + assert(fman_ccsr_map_fd != -1); + + fmbm_ricp = + in_be32(&((struct rx_bmi_regs *)__if->bmi_map)->fmbm_ricp); + /*iceof + icsz*/ + fdoff = ((fmbm_ricp & iceof_mask) >> 16) * 16 + + (fmbm_ricp & icsz_mask) * 16; + + return fdoff; +} + +void +fman_if_set_err_fqid(struct fman_if *fm_if, uint32_t err_fqid) +{ + struct __fman_if *__if = container_of(fm_if, struct __fman_if, __if); + + assert(fman_ccsr_map_fd != -1); + + unsigned int *fmbm_refqid = + &((struct rx_bmi_regs *)__if->bmi_map)->fmbm_refqid; + out_be32(fmbm_refqid, err_fqid); +} + +int +fman_if_get_ic_params(struct fman_if *fm_if, struct fman_if_ic_params *icp) +{ + struct __fman_if *__if = container_of(fm_if, struct __fman_if, __if); + int val = 0; + int iceof_mask = 0x001f0000; + int icsz_mask = 0x0000001f; + int iciof_mask = 0x00000f00; + + assert(fman_ccsr_map_fd != -1); + + unsigned int *fmbm_ricp = + &((struct rx_bmi_regs *)__if->bmi_map)->fmbm_ricp; + val = in_be32(fmbm_ricp); + + icp->iceof = (val & iceof_mask) >> 12; + icp->iciof = (val & iciof_mask) >> 4; + icp->icsz = (val & icsz_mask) << 4; + + return 0; +} + +int +fman_if_set_ic_params(struct fman_if *fm_if, + const struct fman_if_ic_params *icp) +{ + struct __fman_if *__if = container_of(fm_if, struct __fman_if, __if); + int val = 0; + int iceof_mask = 0x001f0000; + int icsz_mask = 0x0000001f; + int iciof_mask = 0x00000f00; + + assert(fman_ccsr_map_fd != -1); + + val |= (icp->iceof << 12) & iceof_mask; + val |= (icp->iciof << 4) & iciof_mask; + val |= (icp->icsz >> 4) & icsz_mask; + + unsigned int *fmbm_ricp = + &((struct rx_bmi_regs *)__if->bmi_map)->fmbm_ricp; + out_be32(fmbm_ricp, val); + + return 0; +} + +void +fman_if_set_fdoff(struct fman_if *fm_if, uint32_t fd_offset) +{ + struct __fman_if *__if = container_of(fm_if, struct __fman_if, __if); + unsigned int *fmbm_rebm; + + assert(fman_ccsr_map_fd != -1); + + fmbm_rebm = &((struct rx_bmi_regs *)__if->bmi_map)->fmbm_rebm; + + out_be32(fmbm_rebm, in_be32(fmbm_rebm) | (fd_offset << 16)); +} + +void +fman_if_set_maxfrm(struct fman_if *fm_if, uint16_t max_frm) +{ + struct __fman_if *__if = container_of(fm_if, struct __fman_if, __if); + unsigned int *reg_maxfrm; + + assert(fman_ccsr_map_fd != -1); + + reg_maxfrm = &((struct memac_regs *)__if->ccsr_map)->maxfrm; + + out_be32(reg_maxfrm, (in_be32(reg_maxfrm) & 0xFFFF0000) | max_frm); +} + +uint16_t +fman_if_get_maxfrm(struct fman_if *fm_if) +{ + struct __fman_if *__if = container_of(fm_if, struct __fman_if, __if); + unsigned int *reg_maxfrm; + + assert(fman_ccsr_map_fd != -1); + + reg_maxfrm = &((struct memac_regs *)__if->ccsr_map)->maxfrm; + + return (in_be32(reg_maxfrm) | 0x0000FFFF); +} + +void +fman_if_set_dnia(struct fman_if *fm_if, uint32_t nia) +{ + struct __fman_if *__if = container_of(fm_if, struct __fman_if, __if); + unsigned int *fmqm_pndn; + + assert(fman_ccsr_map_fd != -1); + + fmqm_pndn = &((struct fman_port_qmi_regs *)__if->qmi_map)->fmqm_pndn; + + out_be32(fmqm_pndn, nia); +} + +void +fman_if_discard_rx_errors(struct fman_if *fm_if) +{ + struct __fman_if *__if = container_of(fm_if, struct __fman_if, __if); + unsigned int *fmbm_rfsdm, *fmbm_rfsem; + + fmbm_rfsem = &((struct rx_bmi_regs *)__if->bmi_map)->fmbm_rfsem; + out_be32(fmbm_rfsem, 0); + + /* Configure the discard mask to discard the error packets which have + * DMA errors, Frame size error, Header error etc. The mask 0x010CE3F0 + * is to configured discard all the errors which come in the FD[STATUS] + */ + fmbm_rfsdm = &((struct rx_bmi_regs *)__if->bmi_map)->fmbm_rfsdm; + out_be32(fmbm_rfsdm, 0x010CE3F0); +} diff --git a/dpdk/drivers/bus/dpaa/base/fman/netcfg_layer.c b/dpdk/drivers/bus/dpaa/base/fman/netcfg_layer.c new file mode 100644 index 00000000..26cff84a --- /dev/null +++ b/dpdk/drivers/bus/dpaa/base/fman/netcfg_layer.c @@ -0,0 +1,214 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2010-2016 Freescale Semiconductor Inc. + * Copyright 2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +/* Structure contains information about all the interfaces given by user + * on command line. + */ +struct netcfg_interface *netcfg_interface; + +/* This data structure contaings all configurations information + * related to usages of DPA devices. + */ +struct netcfg_info *netcfg; +/* fd to open a socket for making ioctl request to disable/enable shared + * interfaces. + */ +static int skfd = -1; + +#ifdef RTE_LIBRTE_DPAA_DEBUG_DRIVER +void +dump_netcfg(struct netcfg_info *cfg_ptr) +{ + int i; + + printf(".......... DPAA Configuration ..........\n\n"); + + /* Network interfaces */ + printf("Network interfaces: %d\n", cfg_ptr->num_ethports); + for (i = 0; i < cfg_ptr->num_ethports; i++) { + struct fman_if_bpool *bpool; + struct fm_eth_port_cfg *p_cfg = &cfg_ptr->port_cfg[i]; + struct fman_if *__if = p_cfg->fman_if; + + printf("\n+ Fman %d, MAC %d (%s);\n", + __if->fman_idx, __if->mac_idx, + (__if->mac_type == fman_mac_1g) ? "1G" : "10G"); + + printf("\tmac_addr: %02x:%02x:%02x:%02x:%02x:%02x\n", + (&__if->mac_addr)->addr_bytes[0], + (&__if->mac_addr)->addr_bytes[1], + (&__if->mac_addr)->addr_bytes[2], + (&__if->mac_addr)->addr_bytes[3], + (&__if->mac_addr)->addr_bytes[4], + (&__if->mac_addr)->addr_bytes[5]); + + printf("\ttx_channel_id: 0x%02x\n", + __if->tx_channel_id); + + printf("\tfqid_rx_def: 0x%x\n", p_cfg->rx_def); + printf("\tfqid_rx_err: 0x%x\n", __if->fqid_rx_err); + + printf("\tfqid_tx_err: 0x%x\n", __if->fqid_tx_err); + printf("\tfqid_tx_confirm: 0x%x\n", __if->fqid_tx_confirm); + fman_if_for_each_bpool(bpool, __if) + printf("\tbuffer pool: (bpid=%d, count=%"PRId64 + " size=%"PRId64", addr=0x%"PRIx64")\n", + bpool->bpid, bpool->count, bpool->size, + bpool->addr); + } +} +#endif /* RTE_LIBRTE_DPAA_DEBUG_DRIVER */ + +static inline int +get_num_netcfg_interfaces(char *str) +{ + char *pch; + uint8_t count = 0; + + if (str == NULL) + return -EINVAL; + pch = strtok(str, ","); + while (pch != NULL) { + count++; + pch = strtok(NULL, ","); + } + return count; +} + +struct netcfg_info * +netcfg_acquire(void) +{ + struct fman_if *__if; + int _errno, idx = 0; + uint8_t num_ports = 0; + uint8_t num_cfg_ports = 0; + size_t size; + + /* Extract dpa configuration from fman driver and FMC configuration + * for command-line interfaces. + */ + + /* Open a basic socket to enable/disable shared + * interfaces. + */ + skfd = socket(AF_PACKET, SOCK_RAW, 0); + if (unlikely(skfd < 0)) { + error(0, errno, "%s(): open(SOCK_RAW)", __func__); + return NULL; + } + + /* Initialise the Fman driver */ + _errno = fman_init(); + if (_errno) { + DPAA_BUS_LOG(ERR, "FMAN driver init failed (%d)", errno); + close(skfd); + skfd = -1; + return NULL; + } + + /* Number of MAC ports */ + list_for_each_entry(__if, fman_if_list, node) + num_ports++; + + if (!num_ports) { + DPAA_BUS_LOG(ERR, "FMAN ports not available"); + return NULL; + } + /* Allocate space for all enabled mac ports */ + size = sizeof(*netcfg) + + (num_ports * sizeof(struct fm_eth_port_cfg)); + + netcfg = calloc(1, size); + if (unlikely(netcfg == NULL)) { + DPAA_BUS_LOG(ERR, "Unable to allocat mem for netcfg"); + goto error; + } + + netcfg->num_ethports = num_ports; + + list_for_each_entry(__if, fman_if_list, node) { + struct fm_eth_port_cfg *cfg = &netcfg->port_cfg[idx]; + /* Hook in the fman driver interface */ + cfg->fman_if = __if; + cfg->rx_def = __if->fqid_rx_def; + num_cfg_ports++; + idx++; + } + + if (!num_cfg_ports) { + DPAA_BUS_LOG(ERR, "No FMAN ports found"); + goto error; + } else if (num_ports != num_cfg_ports) + netcfg->num_ethports = num_cfg_ports; + + return netcfg; + +error: + if (netcfg) { + free(netcfg); + netcfg = NULL; + } + + return NULL; +} + +void +netcfg_release(struct netcfg_info *cfg_ptr) +{ + free(cfg_ptr); + /* Close socket for shared interfaces */ + if (skfd >= 0) { + close(skfd); + skfd = -1; + } +} diff --git a/dpdk/drivers/bus/dpaa/base/fman/of.c b/dpdk/drivers/bus/dpaa/base/fman/of.c new file mode 100644 index 00000000..b2d7c020 --- /dev/null +++ b/dpdk/drivers/bus/dpaa/base/fman/of.c @@ -0,0 +1,576 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2010-2016 Freescale Semiconductor Inc. + * Copyright 2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include + +static int alive; +static struct dt_dir root_dir; +static const char *base_dir; +static COMPAT_LIST_HEAD(linear); + +static int +of_open_dir(const char *relative_path, struct dirent ***d) +{ + int ret; + char full_path[PATH_MAX]; + + snprintf(full_path, PATH_MAX, "%s/%s", base_dir, relative_path); + ret = scandir(full_path, d, 0, versionsort); + if (ret < 0) + DPAA_BUS_LOG(ERR, "Failed to open directory %s", + full_path); + return ret; +} + +static void +of_close_dir(struct dirent **d, int num) +{ + while (num--) + free(d[num]); + free(d); +} + +static int +of_open_file(const char *relative_path) +{ + int ret; + char full_path[PATH_MAX]; + + snprintf(full_path, PATH_MAX, "%s/%s", base_dir, relative_path); + ret = open(full_path, O_RDONLY); + if (ret < 0) + DPAA_BUS_LOG(ERR, "Failed to open directory %s", + full_path); + return ret; +} + +static void +process_file(struct dirent *dent, struct dt_dir *parent) +{ + int fd; + struct dt_file *f = malloc(sizeof(*f)); + + if (!f) { + DPAA_BUS_LOG(DEBUG, "Unable to allocate memory for file node"); + return; + } + f->node.is_file = 1; + snprintf(f->node.node.name, NAME_MAX, "%s", dent->d_name); + snprintf(f->node.node.full_name, PATH_MAX, "%s/%s", + parent->node.node.full_name, dent->d_name); + f->parent = parent; + fd = of_open_file(f->node.node.full_name); + if (fd < 0) { + DPAA_BUS_LOG(DEBUG, "Unable to open file node"); + free(f); + return; + } + f->len = read(fd, f->buf, OF_FILE_BUF_MAX); + close(fd); + if (f->len < 0) { + DPAA_BUS_LOG(DEBUG, "Unable to read file node"); + free(f); + return; + } + list_add_tail(&f->node.list, &parent->files); +} + +static const struct dt_dir * +node2dir(const struct device_node *n) +{ + struct dt_node *dn = container_of((struct device_node *)n, + struct dt_node, node); + const struct dt_dir *d = container_of(dn, struct dt_dir, node); + + assert(!dn->is_file); + return d; +} + +/* process_dir() calls iterate_dir(), but the latter will also call the former + * when recursing into sub-directories, so a predeclaration is needed. + */ +static int process_dir(const char *relative_path, struct dt_dir *dt); + +static int +iterate_dir(struct dirent **d, int num, struct dt_dir *dt) +{ + int loop; + /* Iterate the directory contents */ + for (loop = 0; loop < num; loop++) { + struct dt_dir *subdir; + int ret; + /* Ignore dot files of all types (especially "..") */ + if (d[loop]->d_name[0] == '.') + continue; + switch (d[loop]->d_type) { + case DT_REG: + process_file(d[loop], dt); + break; + case DT_DIR: + subdir = malloc(sizeof(*subdir)); + if (!subdir) { + perror("malloc"); + return -ENOMEM; + } + snprintf(subdir->node.node.name, NAME_MAX, "%s", + d[loop]->d_name); + snprintf(subdir->node.node.full_name, PATH_MAX, + "%s/%s", dt->node.node.full_name, + d[loop]->d_name); + subdir->parent = dt; + ret = process_dir(subdir->node.node.full_name, subdir); + if (ret) + return ret; + list_add_tail(&subdir->node.list, &dt->subdirs); + break; + default: + DPAA_BUS_LOG(DEBUG, "Ignoring invalid dt entry %s/%s", + dt->node.node.full_name, d[loop]->d_name); + } + } + return 0; +} + +static int +process_dir(const char *relative_path, struct dt_dir *dt) +{ + struct dirent **d; + int ret, num; + + dt->node.is_file = 0; + INIT_LIST_HEAD(&dt->subdirs); + INIT_LIST_HEAD(&dt->files); + ret = of_open_dir(relative_path, &d); + if (ret < 0) + return ret; + num = ret; + ret = iterate_dir(d, num, dt); + of_close_dir(d, num); + return (ret < 0) ? ret : 0; +} + +static void +linear_dir(struct dt_dir *d) +{ + struct dt_file *f; + struct dt_dir *dd; + + d->compatible = NULL; + d->status = NULL; + d->lphandle = NULL; + d->a_cells = NULL; + d->s_cells = NULL; + d->reg = NULL; + list_for_each_entry(f, &d->files, node.list) { + if (!strcmp(f->node.node.name, "compatible")) { + if (d->compatible) + DPAA_BUS_LOG(DEBUG, "Duplicate compatible in" + " %s", d->node.node.full_name); + d->compatible = f; + } else if (!strcmp(f->node.node.name, "status")) { + if (d->status) + DPAA_BUS_LOG(DEBUG, "Duplicate status in %s", + d->node.node.full_name); + d->status = f; + } else if (!strcmp(f->node.node.name, "linux,phandle")) { + if (d->lphandle) + DPAA_BUS_LOG(DEBUG, "Duplicate lphandle in %s", + d->node.node.full_name); + d->lphandle = f; + } else if (!strcmp(f->node.node.name, "#address-cells")) { + if (d->a_cells) + DPAA_BUS_LOG(DEBUG, "Duplicate a_cells in %s", + d->node.node.full_name); + d->a_cells = f; + } else if (!strcmp(f->node.node.name, "#size-cells")) { + if (d->s_cells) + DPAA_BUS_LOG(DEBUG, "Duplicate s_cells in %s", + d->node.node.full_name); + d->s_cells = f; + } else if (!strcmp(f->node.node.name, "reg")) { + if (d->reg) + DPAA_BUS_LOG(DEBUG, "Duplicate reg in %s", + d->node.node.full_name); + d->reg = f; + } + } + + list_for_each_entry(dd, &d->subdirs, node.list) { + list_add_tail(&dd->linear, &linear); + linear_dir(dd); + } +} + +int +of_init_path(const char *dt_path) +{ + int ret; + + base_dir = dt_path; + + /* This needs to be singleton initialization */ + DPAA_BUS_HWWARN(alive, "Double-init of device-tree driver!"); + + /* Prepare root node (the remaining fields are set in process_dir()) */ + root_dir.node.node.name[0] = '\0'; + root_dir.node.node.full_name[0] = '\0'; + INIT_LIST_HEAD(&root_dir.node.list); + root_dir.parent = NULL; + + /* Kick things off... */ + ret = process_dir("", &root_dir); + if (ret) { + DPAA_BUS_LOG(ERR, "Unable to parse device tree"); + return ret; + } + + /* Now make a flat, linear list of directories */ + linear_dir(&root_dir); + alive = 1; + return 0; +} + +static void +destroy_dir(struct dt_dir *d) +{ + struct dt_file *f, *tmpf; + struct dt_dir *dd, *tmpd; + + list_for_each_entry_safe(f, tmpf, &d->files, node.list) { + list_del(&f->node.list); + free(f); + } + list_for_each_entry_safe(dd, tmpd, &d->subdirs, node.list) { + destroy_dir(dd); + list_del(&dd->node.list); + free(dd); + } +} + +void +of_finish(void) +{ + DPAA_BUS_HWWARN(!alive, "Double-finish of device-tree driver!"); + + destroy_dir(&root_dir); + INIT_LIST_HEAD(&linear); + alive = 0; +} + +static const struct dt_dir * +next_linear(const struct dt_dir *f) +{ + if (f->linear.next == &linear) + return NULL; + return list_entry(f->linear.next, struct dt_dir, linear); +} + +static int +check_compatible(const struct dt_file *f, const char *compatible) +{ + const char *c = (char *)f->buf; + unsigned int len, remains = f->len; + + while (remains) { + len = strlen(c); + if (!strcmp(c, compatible)) + return 1; + + if (remains < len + 1) + break; + + c += (len + 1); + remains -= (len + 1); + } + return 0; +} + +const struct device_node * +of_find_compatible_node(const struct device_node *from, + const char *type __always_unused, + const char *compatible) +{ + const struct dt_dir *d; + + DPAA_BUS_HWWARN(!alive, "Device-tree driver not initialised!"); + + if (list_empty(&linear)) + return NULL; + if (!from) + d = list_entry(linear.next, struct dt_dir, linear); + else + d = node2dir(from); + for (d = next_linear(d); d && (!d->compatible || + !check_compatible(d->compatible, + compatible)); + d = next_linear(d)) + ; + if (d) + return &d->node.node; + return NULL; +} + +const void * +of_get_property(const struct device_node *from, const char *name, + size_t *lenp) +{ + const struct dt_dir *d; + const struct dt_file *f; + + DPAA_BUS_HWWARN(!alive, "Device-tree driver not initialised!"); + + d = node2dir(from); + list_for_each_entry(f, &d->files, node.list) + if (!strcmp(f->node.node.name, name)) { + if (lenp) + *lenp = f->len; + return f->buf; + } + return NULL; +} + +bool +of_device_is_available(const struct device_node *dev_node) +{ + const struct dt_dir *d; + + DPAA_BUS_HWWARN(!alive, "Device-tree driver not initialised!"); + d = node2dir(dev_node); + if (!d->status) + return true; + if (!strcmp((char *)d->status->buf, "okay")) + return true; + if (!strcmp((char *)d->status->buf, "ok")) + return true; + return false; +} + +const struct device_node * +of_find_node_by_phandle(phandle ph) +{ + const struct dt_dir *d; + + DPAA_BUS_HWWARN(!alive, "Device-tree driver not initialised!"); + list_for_each_entry(d, &linear, linear) + if (d->lphandle && (d->lphandle->len == 4) && + !memcmp(d->lphandle->buf, &ph, 4)) + return &d->node.node; + return NULL; +} + +const struct device_node * +of_get_parent(const struct device_node *dev_node) +{ + const struct dt_dir *d; + + DPAA_BUS_HWWARN(!alive, "Device-tree driver not initialised!"); + + if (!dev_node) + return NULL; + d = node2dir(dev_node); + if (!d->parent) + return NULL; + return &d->parent->node.node; +} + +const struct device_node * +of_get_next_child(const struct device_node *dev_node, + const struct device_node *prev) +{ + const struct dt_dir *p, *c; + + DPAA_BUS_HWWARN(!alive, "Device-tree driver not initialised!"); + + if (!dev_node) + return NULL; + p = node2dir(dev_node); + if (prev) { + c = node2dir(prev); + DPAA_BUS_HWWARN((c->parent != p), "Parent/child mismatch"); + if (c->parent != p) + return NULL; + if (c->node.list.next == &p->subdirs) + /* prev was the last child */ + return NULL; + c = list_entry(c->node.list.next, struct dt_dir, node.list); + return &c->node.node; + } + /* Return first child */ + if (list_empty(&p->subdirs)) + return NULL; + c = list_entry(p->subdirs.next, struct dt_dir, node.list); + return &c->node.node; +} + +uint32_t +of_n_addr_cells(const struct device_node *dev_node) +{ + const struct dt_dir *d; + + DPAA_BUS_HWWARN(!alive, "Device-tree driver not initialised"); + if (!dev_node) + return OF_DEFAULT_NA; + d = node2dir(dev_node); + while ((d = d->parent)) + if (d->a_cells) { + unsigned char *buf = + (unsigned char *)&d->a_cells->buf[0]; + assert(d->a_cells->len == 4); + return ((uint32_t)buf[0] << 24) | + ((uint32_t)buf[1] << 16) | + ((uint32_t)buf[2] << 8) | + (uint32_t)buf[3]; + } + return OF_DEFAULT_NA; +} + +uint32_t +of_n_size_cells(const struct device_node *dev_node) +{ + const struct dt_dir *d; + + DPAA_BUS_HWWARN(!alive, "Device-tree driver not initialised!"); + if (!dev_node) + return OF_DEFAULT_NA; + d = node2dir(dev_node); + while ((d = d->parent)) + if (d->s_cells) { + unsigned char *buf = + (unsigned char *)&d->s_cells->buf[0]; + assert(d->s_cells->len == 4); + return ((uint32_t)buf[0] << 24) | + ((uint32_t)buf[1] << 16) | + ((uint32_t)buf[2] << 8) | + (uint32_t)buf[3]; + } + return OF_DEFAULT_NS; +} + +const uint32_t * +of_get_address(const struct device_node *dev_node, size_t idx, + uint64_t *size, uint32_t *flags __rte_unused) +{ + const struct dt_dir *d; + const unsigned char *buf; + uint32_t na = of_n_addr_cells(dev_node); + uint32_t ns = of_n_size_cells(dev_node); + + if (!dev_node) + d = &root_dir; + else + d = node2dir(dev_node); + if (!d->reg) + return NULL; + assert(d->reg->len % ((na + ns) * 4) == 0); + assert(d->reg->len / ((na + ns) * 4) > (unsigned int) idx); + buf = (const unsigned char *)&d->reg->buf[0]; + buf += (na + ns) * idx * 4; + if (size) + for (*size = 0; ns > 0; ns--, na++) + *size = (*size << 32) + + (((uint32_t)buf[4 * na] << 24) | + ((uint32_t)buf[4 * na + 1] << 16) | + ((uint32_t)buf[4 * na + 2] << 8) | + (uint32_t)buf[4 * na + 3]); + return (const uint32_t *)buf; +} + +uint64_t +of_translate_address(const struct device_node *dev_node, + const uint32_t *addr) +{ + uint64_t phys_addr, tmp_addr; + const struct device_node *parent; + const uint32_t *ranges; + size_t rlen; + uint32_t na, pna; + + DPAA_BUS_HWWARN(!alive, "Device-tree driver not initialised!"); + assert(dev_node != NULL); + + na = of_n_addr_cells(dev_node); + phys_addr = of_read_number(addr, na); + + dev_node = of_get_parent(dev_node); + if (!dev_node) + return 0; + else if (node2dir(dev_node) == &root_dir) + return phys_addr; + + do { + pna = of_n_addr_cells(dev_node); + parent = of_get_parent(dev_node); + if (!parent) + return 0; + + ranges = of_get_property(dev_node, "ranges", &rlen); + /* "ranges" property is missing. Translation breaks */ + if (!ranges) + return 0; + /* "ranges" property is empty. Do 1:1 translation */ + else if (rlen == 0) + continue; + else + tmp_addr = of_read_number(ranges + na, pna); + + na = pna; + dev_node = parent; + phys_addr += tmp_addr; + } while (node2dir(parent) != &root_dir); + + return phys_addr; +} + +bool +of_device_is_compatible(const struct device_node *dev_node, + const char *compatible) +{ + const struct dt_dir *d; + + DPAA_BUS_HWWARN(!alive, "Device-tree driver not initialised!"); + if (!dev_node) + d = &root_dir; + else + d = node2dir(dev_node); + if (d->compatible && check_compatible(d->compatible, compatible)) + return true; + return false; +} diff --git a/dpdk/drivers/bus/dpaa/base/qbman/bman.c b/dpdk/drivers/bus/dpaa/base/qbman/bman.c new file mode 100644 index 00000000..0480caa9 --- /dev/null +++ b/dpdk/drivers/bus/dpaa/base/qbman/bman.c @@ -0,0 +1,394 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2008-2016 Freescale Semiconductor Inc. + * Copyright 2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "bman.h" +#include + +/* Compilation constants */ +#define RCR_THRESH 2 /* reread h/w CI when running out of space */ +#define IRQNAME "BMan portal %d" +#define MAX_IRQNAME 16 /* big enough for "BMan portal %d" */ + +struct bman_portal { + struct bm_portal p; + /* 2-element array. pools[0] is mask, pools[1] is snapshot. */ + struct bman_depletion *pools; + int thresh_set; + unsigned long irq_sources; + u32 slowpoll; /* only used when interrupts are off */ + /* When the cpu-affine portal is activated, this is non-NULL */ + const struct bm_portal_config *config; + char irqname[MAX_IRQNAME]; +}; + +static cpumask_t affine_mask; +static DEFINE_SPINLOCK(affine_mask_lock); +static RTE_DEFINE_PER_LCORE(struct bman_portal, bman_affine_portal); + +static inline struct bman_portal *get_affine_portal(void) +{ + return &RTE_PER_LCORE(bman_affine_portal); +} + +/* + * This object type refers to a pool, it isn't *the* pool. There may be + * more than one such object per BMan buffer pool, eg. if different users of + * the pool are operating via different portals. + */ +struct bman_pool { + struct bman_pool_params params; + /* Used for hash-table admin when using depletion notifications. */ + struct bman_portal *portal; + struct bman_pool *next; +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + atomic_t in_use; +#endif +}; + +static inline +struct bman_portal *bman_create_portal(struct bman_portal *portal, + const struct bm_portal_config *c) +{ + struct bm_portal *p; + const struct bman_depletion *pools = &c->mask; + int ret; + u8 bpid = 0; + + p = &portal->p; + /* + * prep the low-level portal struct with the mapped addresses from the + * config, everything that follows depends on it and "config" is more + * for (de)reference... + */ + p->addr.ce = c->addr_virt[DPAA_PORTAL_CE]; + p->addr.ci = c->addr_virt[DPAA_PORTAL_CI]; + if (bm_rcr_init(p, bm_rcr_pvb, bm_rcr_cce)) { + pr_err("Bman RCR initialisation failed\n"); + return NULL; + } + if (bm_mc_init(p)) { + pr_err("Bman MC initialisation failed\n"); + goto fail_mc; + } + portal->pools = kmalloc(2 * sizeof(*pools), GFP_KERNEL); + if (!portal->pools) + goto fail_pools; + portal->pools[0] = *pools; + bman_depletion_init(portal->pools + 1); + while (bpid < bman_pool_max) { + /* + * Default to all BPIDs disabled, we enable as required at + * run-time. + */ + bm_isr_bscn_mask(p, bpid, 0); + bpid++; + } + portal->slowpoll = 0; + /* Write-to-clear any stale interrupt status bits */ + bm_isr_disable_write(p, 0xffffffff); + portal->irq_sources = 0; + bm_isr_enable_write(p, portal->irq_sources); + bm_isr_status_clear(p, 0xffffffff); + snprintf(portal->irqname, MAX_IRQNAME, IRQNAME, c->cpu); + if (request_irq(c->irq, NULL, 0, portal->irqname, + portal)) { + pr_err("request_irq() failed\n"); + goto fail_irq; + } + + /* Need RCR to be empty before continuing */ + ret = bm_rcr_get_fill(p); + if (ret) { + pr_err("Bman RCR unclean\n"); + goto fail_rcr_empty; + } + /* Success */ + portal->config = c; + + bm_isr_disable_write(p, 0); + bm_isr_uninhibit(p); + return portal; +fail_rcr_empty: + free_irq(c->irq, portal); +fail_irq: + kfree(portal->pools); +fail_pools: + bm_mc_finish(p); +fail_mc: + bm_rcr_finish(p); + return NULL; +} + +struct bman_portal * +bman_create_affine_portal(const struct bm_portal_config *c) +{ + struct bman_portal *portal = get_affine_portal(); + + /*This function is called from the context which is already affine to + *CPU or in other words this in non-migratable to other CPUs. + */ + portal = bman_create_portal(portal, c); + if (portal) { + spin_lock(&affine_mask_lock); + CPU_SET(c->cpu, &affine_mask); + spin_unlock(&affine_mask_lock); + } + return portal; +} + +static inline +void bman_destroy_portal(struct bman_portal *bm) +{ + const struct bm_portal_config *pcfg; + + pcfg = bm->config; + bm_rcr_cce_update(&bm->p); + bm_rcr_cce_update(&bm->p); + + free_irq(pcfg->irq, bm); + + kfree(bm->pools); + bm_mc_finish(&bm->p); + bm_rcr_finish(&bm->p); + bm->config = NULL; +} + +const struct +bm_portal_config *bman_destroy_affine_portal(void) +{ + struct bman_portal *bm = get_affine_portal(); + const struct bm_portal_config *pcfg; + + pcfg = bm->config; + bman_destroy_portal(bm); + spin_lock(&affine_mask_lock); + CPU_CLR(pcfg->cpu, &affine_mask); + spin_unlock(&affine_mask_lock); + return pcfg; +} + +int +bman_get_portal_index(void) +{ + struct bman_portal *p = get_affine_portal(); + return p->config->index; +} + +static const u32 zero_thresholds[4] = {0, 0, 0, 0}; + +struct bman_pool *bman_new_pool(const struct bman_pool_params *params) +{ + struct bman_pool *pool = NULL; + u32 bpid; + + if (params->flags & BMAN_POOL_FLAG_DYNAMIC_BPID) { + int ret = bman_alloc_bpid(&bpid); + + if (ret) + return NULL; + } else { + if (params->bpid >= bman_pool_max) + return NULL; + bpid = params->bpid; + } + if (params->flags & BMAN_POOL_FLAG_THRESH) { + int ret = bm_pool_set(bpid, params->thresholds); + + if (ret) + goto err; + } + + pool = kmalloc(sizeof(*pool), GFP_KERNEL); + if (!pool) + goto err; + pool->params = *params; +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + atomic_set(&pool->in_use, 1); +#endif + if (params->flags & BMAN_POOL_FLAG_DYNAMIC_BPID) + pool->params.bpid = bpid; + + return pool; +err: + if (params->flags & BMAN_POOL_FLAG_THRESH) + bm_pool_set(bpid, zero_thresholds); + + if (params->flags & BMAN_POOL_FLAG_DYNAMIC_BPID) + bman_release_bpid(bpid); + kfree(pool); + + return NULL; +} + +void bman_free_pool(struct bman_pool *pool) +{ + if (pool->params.flags & BMAN_POOL_FLAG_THRESH) + bm_pool_set(pool->params.bpid, zero_thresholds); + if (pool->params.flags & BMAN_POOL_FLAG_DYNAMIC_BPID) + bman_release_bpid(pool->params.bpid); + kfree(pool); +} + +const struct bman_pool_params *bman_get_params(const struct bman_pool *pool) +{ + return &pool->params; +} + +static void update_rcr_ci(struct bman_portal *p, int avail) +{ + if (avail) + bm_rcr_cce_prefetch(&p->p); + else + bm_rcr_cce_update(&p->p); +} + +#define BMAN_BUF_MASK 0x0000fffffffffffful +int bman_release(struct bman_pool *pool, const struct bm_buffer *bufs, u8 num, + u32 flags __maybe_unused) +{ + struct bman_portal *p; + struct bm_rcr_entry *r; + u32 i = num - 1; + u8 avail; + +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + if (!num || (num > 8)) + return -EINVAL; + if (pool->params.flags & BMAN_POOL_FLAG_NO_RELEASE) + return -EINVAL; +#endif + + p = get_affine_portal(); + avail = bm_rcr_get_avail(&p->p); + if (avail < 2) + update_rcr_ci(p, avail); + r = bm_rcr_start(&p->p); + if (unlikely(!r)) + return -EBUSY; + + /* + * we can copy all but the first entry, as this can trigger badness + * with the valid-bit + */ + r->bufs[0].opaque = + cpu_to_be64(((u64)pool->params.bpid << 48) | + (bufs[0].opaque & BMAN_BUF_MASK)); + if (i) { + for (i = 1; i < num; i++) + r->bufs[i].opaque = + cpu_to_be64(bufs[i].opaque & BMAN_BUF_MASK); + } + + bm_rcr_pvb_commit(&p->p, BM_RCR_VERB_CMD_BPID_SINGLE | + (num & BM_RCR_VERB_BUFCOUNT_MASK)); + + return 0; +} + +int bman_acquire(struct bman_pool *pool, struct bm_buffer *bufs, u8 num, + u32 flags __maybe_unused) +{ + struct bman_portal *p = get_affine_portal(); + struct bm_mc_command *mcc; + struct bm_mc_result *mcr; + int ret, i; + +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + if (!num || (num > 8)) + return -EINVAL; + if (pool->params.flags & BMAN_POOL_FLAG_ONLY_RELEASE) + return -EINVAL; +#endif + + mcc = bm_mc_start(&p->p); + mcc->acquire.bpid = pool->params.bpid; + bm_mc_commit(&p->p, BM_MCC_VERB_CMD_ACQUIRE | + (num & BM_MCC_VERB_ACQUIRE_BUFCOUNT)); + while (!(mcr = bm_mc_result(&p->p))) + cpu_relax(); + ret = mcr->verb & BM_MCR_VERB_ACQUIRE_BUFCOUNT; + if (bufs) { + for (i = 0; i < num; i++) + bufs[i].opaque = + be64_to_cpu(mcr->acquire.bufs[i].opaque); + } + if (ret != num) + ret = -ENOMEM; + return ret; +} + +int bman_query_pools(struct bm_pool_state *state) +{ + struct bman_portal *p = get_affine_portal(); + struct bm_mc_result *mcr; + + bm_mc_start(&p->p); + bm_mc_commit(&p->p, BM_MCC_VERB_CMD_QUERY); + while (!(mcr = bm_mc_result(&p->p))) + cpu_relax(); + DPAA_ASSERT((mcr->verb & BM_MCR_VERB_CMD_MASK) == + BM_MCR_VERB_CMD_QUERY); + *state = mcr->query; + state->as.state.state[0] = be32_to_cpu(state->as.state.state[0]); + state->as.state.state[1] = be32_to_cpu(state->as.state.state[1]); + state->ds.state.state[0] = be32_to_cpu(state->ds.state.state[0]); + state->ds.state.state[1] = be32_to_cpu(state->ds.state.state[1]); + return 0; +} + +u32 bman_query_free_buffers(struct bman_pool *pool) +{ + return bm_pool_free_buffers(pool->params.bpid); +} + +int bman_update_pool_thresholds(struct bman_pool *pool, const u32 *thresholds) +{ + u32 bpid; + + bpid = bman_get_params(pool)->bpid; + + return bm_pool_set(bpid, thresholds); +} + +int bman_shutdown_pool(u32 bpid) +{ + struct bman_portal *p = get_affine_portal(); + return bm_shutdown_pool(&p->p, bpid); +} diff --git a/dpdk/drivers/bus/dpaa/base/qbman/bman.h b/dpdk/drivers/bus/dpaa/base/qbman/bman.h new file mode 100644 index 00000000..ef0a896e --- /dev/null +++ b/dpdk/drivers/bus/dpaa/base/qbman/bman.h @@ -0,0 +1,574 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2010-2016 Freescale Semiconductor Inc. + * Copyright 2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __BMAN_H +#define __BMAN_H + +#include "bman_priv.h" + +/* Cache-inhibited register offsets */ +#define BM_REG_RCR_PI_CINH 0x3000 +#define BM_REG_RCR_CI_CINH 0x3100 +#define BM_REG_RCR_ITR 0x3200 +#define BM_REG_CFG 0x3300 +#define BM_REG_SCN(n) (0x3400 + ((n) << 6)) +#define BM_REG_ISR 0x3e00 +#define BM_REG_IIR 0x3ec0 + +/* Cache-enabled register offsets */ +#define BM_CL_CR 0x0000 +#define BM_CL_RR0 0x0100 +#define BM_CL_RR1 0x0140 +#define BM_CL_RCR 0x1000 +#define BM_CL_RCR_PI_CENA 0x3000 +#define BM_CL_RCR_CI_CENA 0x3100 + +/* BTW, the drivers (and h/w programming model) already obtain the required + * synchronisation for portal accesses via lwsync(), hwsync(), and + * data-dependencies. Use of barrier()s or other order-preserving primitives + * simply degrade performance. Hence the use of the __raw_*() interfaces, which + * simply ensure that the compiler treats the portal registers as volatile (ie. + * non-coherent). + */ + +/* Cache-inhibited register access. */ +#define __bm_in(bm, o) be32_to_cpu(__raw_readl((bm)->ci + (o))) +#define __bm_out(bm, o, val) __raw_writel(cpu_to_be32(val), \ + (bm)->ci + (o)) +#define bm_in(reg) __bm_in(&portal->addr, BM_REG_##reg) +#define bm_out(reg, val) __bm_out(&portal->addr, BM_REG_##reg, val) + +/* Cache-enabled (index) register access */ +#define __bm_cl_touch_ro(bm, o) dcbt_ro((bm)->ce + (o)) +#define __bm_cl_touch_rw(bm, o) dcbt_rw((bm)->ce + (o)) +#define __bm_cl_in(bm, o) be32_to_cpu(__raw_readl((bm)->ce + (o))) +#define __bm_cl_out(bm, o, val) \ + do { \ + u32 *__tmpclout = (bm)->ce + (o); \ + __raw_writel(cpu_to_be32(val), __tmpclout); \ + dcbf(__tmpclout); \ + } while (0) +#define __bm_cl_invalidate(bm, o) dccivac((bm)->ce + (o)) +#define bm_cl_touch_ro(reg) __bm_cl_touch_ro(&portal->addr, BM_CL_##reg##_CENA) +#define bm_cl_touch_rw(reg) __bm_cl_touch_rw(&portal->addr, BM_CL_##reg##_CENA) +#define bm_cl_in(reg) __bm_cl_in(&portal->addr, BM_CL_##reg##_CENA) +#define bm_cl_out(reg, val) __bm_cl_out(&portal->addr, BM_CL_##reg##_CENA, val) +#define bm_cl_invalidate(reg)\ + __bm_cl_invalidate(&portal->addr, BM_CL_##reg##_CENA) + +/* Cyclic helper for rings. FIXME: once we are able to do fine-grain perf + * analysis, look at using the "extra" bit in the ring index registers to avoid + * cyclic issues. + */ +static inline u8 bm_cyc_diff(u8 ringsize, u8 first, u8 last) +{ + /* 'first' is included, 'last' is excluded */ + if (first <= last) + return last - first; + return ringsize + last - first; +} + +/* Portal modes. + * Enum types; + * pmode == production mode + * cmode == consumption mode, + * Enum values use 3 letter codes. First letter matches the portal mode, + * remaining two letters indicate; + * ci == cache-inhibited portal register + * ce == cache-enabled portal register + * vb == in-band valid-bit (cache-enabled) + */ +enum bm_rcr_pmode { /* matches BCSP_CFG::RPM */ + bm_rcr_pci = 0, /* PI index, cache-inhibited */ + bm_rcr_pce = 1, /* PI index, cache-enabled */ + bm_rcr_pvb = 2 /* valid-bit */ +}; + +enum bm_rcr_cmode { /* s/w-only */ + bm_rcr_cci, /* CI index, cache-inhibited */ + bm_rcr_cce /* CI index, cache-enabled */ +}; + +/* --- Portal structures --- */ + +#define BM_RCR_SIZE 8 + +struct bm_rcr { + struct bm_rcr_entry *ring, *cursor; + u8 ci, available, ithresh, vbit; +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + u32 busy; + enum bm_rcr_pmode pmode; + enum bm_rcr_cmode cmode; +#endif +}; + +struct bm_mc { + struct bm_mc_command *cr; + struct bm_mc_result *rr; + u8 rridx, vbit; +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + enum { + /* Can only be _mc_start()ed */ + mc_idle, + /* Can only be _mc_commit()ed or _mc_abort()ed */ + mc_user, + /* Can only be _mc_retry()ed */ + mc_hw + } state; +#endif +}; + +struct bm_addr { + void __iomem *ce; /* cache-enabled */ + void __iomem *ci; /* cache-inhibited */ +}; + +struct bm_portal { + struct bm_addr addr; + struct bm_rcr rcr; + struct bm_mc mc; + struct bm_portal_config config; +} ____cacheline_aligned; + +/* Bit-wise logic to wrap a ring pointer by clearing the "carry bit" */ +#define RCR_CARRYCLEAR(p) \ + (void *)((unsigned long)(p) & (~(unsigned long)(BM_RCR_SIZE << 6))) + +/* Bit-wise logic to convert a ring pointer to a ring index */ +static inline u8 RCR_PTR2IDX(struct bm_rcr_entry *e) +{ + return ((uintptr_t)e >> 6) & (BM_RCR_SIZE - 1); +} + +/* Increment the 'cursor' ring pointer, taking 'vbit' into account */ +static inline void RCR_INC(struct bm_rcr *rcr) +{ + /* NB: this is odd-looking, but experiments show that it generates + * fast code with essentially no branching overheads. We increment to + * the next RCR pointer and handle overflow and 'vbit'. + */ + struct bm_rcr_entry *partial = rcr->cursor + 1; + + rcr->cursor = RCR_CARRYCLEAR(partial); + if (partial != rcr->cursor) + rcr->vbit ^= BM_RCR_VERB_VBIT; +} + +static inline int bm_rcr_init(struct bm_portal *portal, enum bm_rcr_pmode pmode, + __maybe_unused enum bm_rcr_cmode cmode) +{ + /* This use of 'register', as well as all other occurrences, is because + * it has been observed to generate much faster code with gcc than is + * otherwise the case. + */ + register struct bm_rcr *rcr = &portal->rcr; + u32 cfg; + u8 pi; + + rcr->ring = portal->addr.ce + BM_CL_RCR; + rcr->ci = bm_in(RCR_CI_CINH) & (BM_RCR_SIZE - 1); + + pi = bm_in(RCR_PI_CINH) & (BM_RCR_SIZE - 1); + rcr->cursor = rcr->ring + pi; + rcr->vbit = (bm_in(RCR_PI_CINH) & BM_RCR_SIZE) ? BM_RCR_VERB_VBIT : 0; + rcr->available = BM_RCR_SIZE - 1 + - bm_cyc_diff(BM_RCR_SIZE, rcr->ci, pi); + rcr->ithresh = bm_in(RCR_ITR); +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + rcr->busy = 0; + rcr->pmode = pmode; + rcr->cmode = cmode; +#endif + cfg = (bm_in(CFG) & 0xffffffe0) | (pmode & 0x3); /* BCSP_CFG::RPM */ + bm_out(CFG, cfg); + return 0; +} + +static inline void bm_rcr_finish(struct bm_portal *portal) +{ + register struct bm_rcr *rcr = &portal->rcr; + u8 pi = bm_in(RCR_PI_CINH) & (BM_RCR_SIZE - 1); + u8 ci = bm_in(RCR_CI_CINH) & (BM_RCR_SIZE - 1); + +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + DPAA_ASSERT(!rcr->busy); +#endif + if (pi != RCR_PTR2IDX(rcr->cursor)) + pr_crit("losing uncommitted RCR entries\n"); + if (ci != rcr->ci) + pr_crit("missing existing RCR completions\n"); + if (rcr->ci != RCR_PTR2IDX(rcr->cursor)) + pr_crit("RCR destroyed unquiesced\n"); +} + +static inline struct bm_rcr_entry *bm_rcr_start(struct bm_portal *portal) +{ + register struct bm_rcr *rcr = &portal->rcr; + +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + DPAA_ASSERT(!rcr->busy); +#endif + if (!rcr->available) + return NULL; +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + rcr->busy = 1; +#endif + dcbz_64(rcr->cursor); + return rcr->cursor; +} + +static inline void bm_rcr_abort(struct bm_portal *portal) +{ + __maybe_unused register struct bm_rcr *rcr = &portal->rcr; + +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + DPAA_ASSERT(rcr->busy); + rcr->busy = 0; +#endif +} + +static inline struct bm_rcr_entry *bm_rcr_pend_and_next( + struct bm_portal *portal, u8 myverb) +{ + register struct bm_rcr *rcr = &portal->rcr; + +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + DPAA_ASSERT(rcr->busy); + DPAA_ASSERT(rcr->pmode != bm_rcr_pvb); +#endif + if (rcr->available == 1) + return NULL; + rcr->cursor->__dont_write_directly__verb = myverb | rcr->vbit; + dcbf_64(rcr->cursor); + RCR_INC(rcr); + rcr->available--; + dcbz_64(rcr->cursor); + return rcr->cursor; +} + +static inline void bm_rcr_pci_commit(struct bm_portal *portal, u8 myverb) +{ + register struct bm_rcr *rcr = &portal->rcr; + +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + DPAA_ASSERT(rcr->busy); + DPAA_ASSERT(rcr->pmode == bm_rcr_pci); +#endif + rcr->cursor->__dont_write_directly__verb = myverb | rcr->vbit; + RCR_INC(rcr); + rcr->available--; + hwsync(); + bm_out(RCR_PI_CINH, RCR_PTR2IDX(rcr->cursor)); +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + rcr->busy = 0; +#endif +} + +static inline void bm_rcr_pce_prefetch(struct bm_portal *portal) +{ + __maybe_unused register struct bm_rcr *rcr = &portal->rcr; + +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + DPAA_ASSERT(rcr->pmode == bm_rcr_pce); +#endif + bm_cl_invalidate(RCR_PI); + bm_cl_touch_rw(RCR_PI); +} + +static inline void bm_rcr_pce_commit(struct bm_portal *portal, u8 myverb) +{ + register struct bm_rcr *rcr = &portal->rcr; + +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + DPAA_ASSERT(rcr->busy); + DPAA_ASSERT(rcr->pmode == bm_rcr_pce); +#endif + rcr->cursor->__dont_write_directly__verb = myverb | rcr->vbit; + RCR_INC(rcr); + rcr->available--; + lwsync(); + bm_cl_out(RCR_PI, RCR_PTR2IDX(rcr->cursor)); +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + rcr->busy = 0; +#endif +} + +static inline void bm_rcr_pvb_commit(struct bm_portal *portal, u8 myverb) +{ + register struct bm_rcr *rcr = &portal->rcr; + struct bm_rcr_entry *rcursor; + +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + DPAA_ASSERT(rcr->busy); + DPAA_ASSERT(rcr->pmode == bm_rcr_pvb); +#endif + lwsync(); + rcursor = rcr->cursor; + rcursor->__dont_write_directly__verb = myverb | rcr->vbit; + dcbf_64(rcursor); + RCR_INC(rcr); + rcr->available--; +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + rcr->busy = 0; +#endif +} + +static inline u8 bm_rcr_cci_update(struct bm_portal *portal) +{ + register struct bm_rcr *rcr = &portal->rcr; + u8 diff, old_ci = rcr->ci; + +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + DPAA_ASSERT(rcr->cmode == bm_rcr_cci); +#endif + rcr->ci = bm_in(RCR_CI_CINH) & (BM_RCR_SIZE - 1); + diff = bm_cyc_diff(BM_RCR_SIZE, old_ci, rcr->ci); + rcr->available += diff; + return diff; +} + +static inline void bm_rcr_cce_prefetch(struct bm_portal *portal) +{ + __maybe_unused register struct bm_rcr *rcr = &portal->rcr; + +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + DPAA_ASSERT(rcr->cmode == bm_rcr_cce); +#endif + bm_cl_touch_ro(RCR_CI); +} + +static inline u8 bm_rcr_cce_update(struct bm_portal *portal) +{ + register struct bm_rcr *rcr = &portal->rcr; + u8 diff, old_ci = rcr->ci; + +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + DPAA_ASSERT(rcr->cmode == bm_rcr_cce); +#endif + rcr->ci = bm_cl_in(RCR_CI) & (BM_RCR_SIZE - 1); + bm_cl_invalidate(RCR_CI); + diff = bm_cyc_diff(BM_RCR_SIZE, old_ci, rcr->ci); + rcr->available += diff; + return diff; +} + +static inline u8 bm_rcr_get_ithresh(struct bm_portal *portal) +{ + register struct bm_rcr *rcr = &portal->rcr; + + return rcr->ithresh; +} + +static inline void bm_rcr_set_ithresh(struct bm_portal *portal, u8 ithresh) +{ + register struct bm_rcr *rcr = &portal->rcr; + + rcr->ithresh = ithresh; + bm_out(RCR_ITR, ithresh); +} + +static inline u8 bm_rcr_get_avail(struct bm_portal *portal) +{ + register struct bm_rcr *rcr = &portal->rcr; + + return rcr->available; +} + +static inline u8 bm_rcr_get_fill(struct bm_portal *portal) +{ + register struct bm_rcr *rcr = &portal->rcr; + + return BM_RCR_SIZE - 1 - rcr->available; +} + +/* --- Management command API --- */ + +static inline int bm_mc_init(struct bm_portal *portal) +{ + register struct bm_mc *mc = &portal->mc; + + mc->cr = portal->addr.ce + BM_CL_CR; + mc->rr = portal->addr.ce + BM_CL_RR0; + mc->rridx = (__raw_readb(&mc->cr->__dont_write_directly__verb) & + BM_MCC_VERB_VBIT) ? 0 : 1; + mc->vbit = mc->rridx ? BM_MCC_VERB_VBIT : 0; +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + mc->state = mc_idle; +#endif + return 0; +} + +static inline void bm_mc_finish(struct bm_portal *portal) +{ + __maybe_unused register struct bm_mc *mc = &portal->mc; + +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + DPAA_ASSERT(mc->state == mc_idle); + if (mc->state != mc_idle) + pr_crit("Losing incomplete MC command\n"); +#endif +} + +static inline struct bm_mc_command *bm_mc_start(struct bm_portal *portal) +{ + register struct bm_mc *mc = &portal->mc; + +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + DPAA_ASSERT(mc->state == mc_idle); + mc->state = mc_user; +#endif + dcbz_64(mc->cr); + return mc->cr; +} + +static inline void bm_mc_abort(struct bm_portal *portal) +{ + __maybe_unused register struct bm_mc *mc = &portal->mc; + +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + DPAA_ASSERT(mc->state == mc_user); + mc->state = mc_idle; +#endif +} + +static inline void bm_mc_commit(struct bm_portal *portal, u8 myverb) +{ + register struct bm_mc *mc = &portal->mc; + struct bm_mc_result *rr = mc->rr + mc->rridx; + +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + DPAA_ASSERT(mc->state == mc_user); +#endif + lwsync(); + mc->cr->__dont_write_directly__verb = myverb | mc->vbit; + dcbf(mc->cr); + dcbit_ro(rr); +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + mc->state = mc_hw; +#endif +} + +static inline struct bm_mc_result *bm_mc_result(struct bm_portal *portal) +{ + register struct bm_mc *mc = &portal->mc; + struct bm_mc_result *rr = mc->rr + mc->rridx; + +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + DPAA_ASSERT(mc->state == mc_hw); +#endif + /* The inactive response register's verb byte always returns zero until + * its command is submitted and completed. This includes the valid-bit, + * in case you were wondering. + */ + if (!__raw_readb(&rr->verb)) { + dcbit_ro(rr); + return NULL; + } + mc->rridx ^= 1; + mc->vbit ^= BM_MCC_VERB_VBIT; +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + mc->state = mc_idle; +#endif + return rr; +} + +#define SCN_REG(bpid) BM_REG_SCN((bpid) / 32) +#define SCN_BIT(bpid) (0x80000000 >> (bpid & 31)) +static inline void bm_isr_bscn_mask(struct bm_portal *portal, u8 bpid, + int enable) +{ + u32 val; + + DPAA_ASSERT(bpid < bman_pool_max); + /* REG_SCN for bpid=0..31, REG_SCN+4 for bpid=32..63 */ + val = __bm_in(&portal->addr, SCN_REG(bpid)); + if (enable) + val |= SCN_BIT(bpid); + else + val &= ~SCN_BIT(bpid); + __bm_out(&portal->addr, SCN_REG(bpid), val); +} + +static inline u32 __bm_isr_read(struct bm_portal *portal, enum bm_isr_reg n) +{ +#if defined(RTE_ARCH_ARM64) + return __bm_in(&portal->addr, BM_REG_ISR + (n << 6)); +#else + return __bm_in(&portal->addr, BM_REG_ISR + (n << 2)); +#endif +} + +static inline void __bm_isr_write(struct bm_portal *portal, enum bm_isr_reg n, + u32 val) +{ +#if defined(RTE_ARCH_ARM64) + __bm_out(&portal->addr, BM_REG_ISR + (n << 6), val); +#else + __bm_out(&portal->addr, BM_REG_ISR + (n << 2), val); +#endif +} + +/* Buffer Pool Cleanup */ +static inline int bm_shutdown_pool(struct bm_portal *p, u32 bpid) +{ + struct bm_mc_command *bm_cmd; + struct bm_mc_result *bm_res; + + int aq_count = 0; + bool stop = false; + + while (!stop) { + /* Acquire buffers until empty */ + bm_cmd = bm_mc_start(p); + bm_cmd->acquire.bpid = bpid; + bm_mc_commit(p, BM_MCC_VERB_CMD_ACQUIRE | 1); + while (!(bm_res = bm_mc_result(p))) + cpu_relax(); + if (!(bm_res->verb & BM_MCR_VERB_ACQUIRE_BUFCOUNT)) { + /* Pool is empty */ + stop = true; + } else + ++aq_count; + }; + return 0; +} + +#endif /* __BMAN_H */ diff --git a/dpdk/drivers/bus/dpaa/base/qbman/bman_driver.c b/dpdk/drivers/bus/dpaa/base/qbman/bman_driver.c new file mode 100644 index 00000000..5c13a803 --- /dev/null +++ b/dpdk/drivers/bus/dpaa/base/qbman/bman_driver.c @@ -0,0 +1,323 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2008-2016 Freescale Semiconductor Inc. + * Copyright 2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +#include +#include +#include "bman_priv.h" +#include + +/* + * Global variables of the max portal/pool number this bman version supported + */ +u16 bman_ip_rev; +u16 bman_pool_max; +void *bman_ccsr_map; + +/*****************/ +/* Portal driver */ +/*****************/ + +static __thread int fd = -1; +static __thread struct bm_portal_config pcfg; +static __thread struct dpaa_ioctl_portal_map map = { + .type = dpaa_portal_bman +}; + +static int fsl_bman_portal_init(uint32_t idx, int is_shared) +{ + cpu_set_t cpuset; + struct bman_portal *portal; + int loop, ret; + struct dpaa_ioctl_irq_map irq_map; + + /* Verify the thread's cpu-affinity */ + ret = pthread_getaffinity_np(pthread_self(), sizeof(cpu_set_t), + &cpuset); + if (ret) { + error(0, ret, "pthread_getaffinity_np()"); + return ret; + } + pcfg.cpu = -1; + for (loop = 0; loop < CPU_SETSIZE; loop++) + if (CPU_ISSET(loop, &cpuset)) { + if (pcfg.cpu != -1) { + pr_err("Thread is not affine to 1 cpu"); + return -EINVAL; + } + pcfg.cpu = loop; + } + if (pcfg.cpu == -1) { + pr_err("Bug in getaffinity handling!"); + return -EINVAL; + } + /* Allocate and map a bman portal */ + map.index = idx; + ret = process_portal_map(&map); + if (ret) { + error(0, ret, "process_portal_map()"); + return ret; + } + /* Make the portal's cache-[enabled|inhibited] regions */ + pcfg.addr_virt[DPAA_PORTAL_CE] = map.addr.cena; + pcfg.addr_virt[DPAA_PORTAL_CI] = map.addr.cinh; + pcfg.is_shared = is_shared; + pcfg.index = map.index; + bman_depletion_fill(&pcfg.mask); + + fd = open(BMAN_PORTAL_IRQ_PATH, O_RDONLY); + if (fd == -1) { + pr_err("BMan irq init failed"); + process_portal_unmap(&map.addr); + return -EBUSY; + } + /* Use the IRQ FD as a unique IRQ number */ + pcfg.irq = fd; + + portal = bman_create_affine_portal(&pcfg); + if (!portal) { + pr_err("Bman portal initialisation failed (%d)", + pcfg.cpu); + process_portal_unmap(&map.addr); + return -EBUSY; + } + + /* Set the IRQ number */ + irq_map.type = dpaa_portal_bman; + irq_map.portal_cinh = map.addr.cinh; + process_portal_irq_map(fd, &irq_map); + return 0; +} + +static int fsl_bman_portal_finish(void) +{ + __maybe_unused const struct bm_portal_config *cfg; + int ret; + + process_portal_irq_unmap(fd); + + cfg = bman_destroy_affine_portal(); + DPAA_BUG_ON(cfg != &pcfg); + ret = process_portal_unmap(&map.addr); + if (ret) + error(0, ret, "process_portal_unmap()"); + return ret; +} + +int bman_thread_init(void) +{ + /* Convert from contiguous/virtual cpu numbering to real cpu when + * calling into the code that is dependent on the device naming. + */ + return fsl_bman_portal_init(QBMAN_ANY_PORTAL_IDX, 0); +} + +int bman_thread_finish(void) +{ + return fsl_bman_portal_finish(); +} + +void bman_thread_irq(void) +{ + qbman_invoke_irq(pcfg.irq); + /* Now we need to uninhibit interrupts. This is the only code outside + * the regular portal driver that manipulates any portal register, so + * rather than breaking that encapsulation I am simply hard-coding the + * offset to the inhibit register here. + */ + out_be32(pcfg.addr_virt[DPAA_PORTAL_CI] + 0xe0c, 0); +} + +int bman_init_ccsr(const struct device_node *node) +{ + static int ccsr_map_fd; + uint64_t phys_addr; + const uint32_t *bman_addr; + uint64_t regs_size; + + bman_addr = of_get_address(node, 0, ®s_size, NULL); + if (!bman_addr) { + pr_err("of_get_address cannot return BMan address"); + return -EINVAL; + } + phys_addr = of_translate_address(node, bman_addr); + if (!phys_addr) { + pr_err("of_translate_address failed"); + return -EINVAL; + } + + ccsr_map_fd = open(BMAN_CCSR_MAP, O_RDWR); + if (unlikely(ccsr_map_fd < 0)) { + pr_err("Can not open /dev/mem for BMan CCSR map"); + return ccsr_map_fd; + } + + bman_ccsr_map = mmap(NULL, regs_size, PROT_READ | + PROT_WRITE, MAP_SHARED, ccsr_map_fd, phys_addr); + if (bman_ccsr_map == MAP_FAILED) { + pr_err("Can not map BMan CCSR base Bman: " + "0x%x Phys: 0x%lx size 0x%lx", + *bman_addr, phys_addr, regs_size); + return -EINVAL; + } + + return 0; +} + +int bman_global_init(void) +{ + const struct device_node *dt_node; + static int done; + + if (done) + return -EBUSY; + /* Use the device-tree to determine IP revision until something better + * is devised. + */ + dt_node = of_find_compatible_node(NULL, NULL, "fsl,bman-portal"); + if (!dt_node) { + pr_err("No bman portals available for any CPU\n"); + return -ENODEV; + } + if (of_device_is_compatible(dt_node, "fsl,bman-portal-1.0") || + of_device_is_compatible(dt_node, "fsl,bman-portal-1.0.0")) { + bman_ip_rev = BMAN_REV10; + bman_pool_max = 64; + } else if (of_device_is_compatible(dt_node, "fsl,bman-portal-2.0") || + of_device_is_compatible(dt_node, "fsl,bman-portal-2.0.8")) { + bman_ip_rev = BMAN_REV20; + bman_pool_max = 8; + } else if (of_device_is_compatible(dt_node, "fsl,bman-portal-2.1.0") || + of_device_is_compatible(dt_node, "fsl,bman-portal-2.1.1") || + of_device_is_compatible(dt_node, "fsl,bman-portal-2.1.2") || + of_device_is_compatible(dt_node, "fsl,bman-portal-2.1.3")) { + bman_ip_rev = BMAN_REV21; + bman_pool_max = 64; + } else { + pr_warn("unknown BMan version in portal node,default " + "to rev1.0"); + bman_ip_rev = BMAN_REV10; + bman_pool_max = 64; + } + + if (!bman_ip_rev) { + pr_err("Unknown bman portal version\n"); + return -ENODEV; + } + { + const struct device_node *dn = of_find_compatible_node(NULL, + NULL, "fsl,bman"); + if (!dn) + pr_err("No bman device node available"); + + if (bman_init_ccsr(dn)) + pr_err("BMan CCSR map failed."); + } + + done = 1; + return 0; +} + +#define BMAN_POOL_CONTENT(n) (0x0600 + ((n) * 0x04)) +u32 bm_pool_free_buffers(u32 bpid) +{ + return in_be32(bman_ccsr_map + BMAN_POOL_CONTENT(bpid)); +} + +static u32 __generate_thresh(u32 val, int roundup) +{ + u32 e = 0; /* co-efficient, exponent */ + int oddbit = 0; + + while (val > 0xff) { + oddbit = val & 1; + val >>= 1; + e++; + if (roundup && oddbit) + val++; + } + DPAA_ASSERT(e < 0x10); + return (val | (e << 8)); +} + +#define POOL_SWDET(n) (0x0000 + ((n) * 0x04)) +#define POOL_HWDET(n) (0x0100 + ((n) * 0x04)) +#define POOL_SWDXT(n) (0x0200 + ((n) * 0x04)) +#define POOL_HWDXT(n) (0x0300 + ((n) * 0x04)) +int bm_pool_set(u32 bpid, const u32 *thresholds) +{ + if (!bman_ccsr_map) + return -ENODEV; + if (bpid >= bman_pool_max) + return -EINVAL; + out_be32(bman_ccsr_map + POOL_SWDET(bpid), + __generate_thresh(thresholds[0], 0)); + out_be32(bman_ccsr_map + POOL_SWDXT(bpid), + __generate_thresh(thresholds[1], 1)); + out_be32(bman_ccsr_map + POOL_HWDET(bpid), + __generate_thresh(thresholds[2], 0)); + out_be32(bman_ccsr_map + POOL_HWDXT(bpid), + __generate_thresh(thresholds[3], 1)); + return 0; +} + +#define BMAN_LOW_DEFAULT_THRESH 0x40 +#define BMAN_HIGH_DEFAULT_THRESH 0x80 +int bm_pool_set_hw_threshold(u32 bpid, const u32 low_thresh, + const u32 high_thresh) +{ + if (!bman_ccsr_map) + return -ENODEV; + if (bpid >= bman_pool_max) + return -EINVAL; + if (low_thresh && high_thresh) { + out_be32(bman_ccsr_map + POOL_HWDET(bpid), + __generate_thresh(low_thresh, 0)); + out_be32(bman_ccsr_map + POOL_HWDXT(bpid), + __generate_thresh(high_thresh, 1)); + } else { + out_be32(bman_ccsr_map + POOL_HWDET(bpid), + __generate_thresh(BMAN_LOW_DEFAULT_THRESH, 0)); + out_be32(bman_ccsr_map + POOL_HWDXT(bpid), + __generate_thresh(BMAN_HIGH_DEFAULT_THRESH, 1)); + } + return 0; +} diff --git a/dpdk/drivers/bus/dpaa/base/qbman/bman_priv.h b/dpdk/drivers/bus/dpaa/base/qbman/bman_priv.h new file mode 100644 index 00000000..07d9cec6 --- /dev/null +++ b/dpdk/drivers/bus/dpaa/base/qbman/bman_priv.h @@ -0,0 +1,125 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2008-2016 Freescale Semiconductor Inc. + * Copyright 2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __BMAN_PRIV_H +#define __BMAN_PRIV_H + +#include "dpaa_sys.h" +#include + +/* Revision info (for errata and feature handling) */ +#define BMAN_REV10 0x0100 +#define BMAN_REV20 0x0200 +#define BMAN_REV21 0x0201 + +#define BMAN_PORTAL_IRQ_PATH "/dev/fsl-usdpaa-irq" +#define BMAN_CCSR_MAP "/dev/mem" + +/* This mask contains all the "irqsource" bits visible to API users */ +#define BM_PIRQ_VISIBLE (BM_PIRQ_RCRI | BM_PIRQ_BSCN) + +/* These are bm__(). So for example, bm_disable_write() means "write + * the disable register" rather than "disable the ability to write". + */ +#define bm_isr_status_read(bm) __bm_isr_read(bm, bm_isr_status) +#define bm_isr_status_clear(bm, m) __bm_isr_write(bm, bm_isr_status, m) +#define bm_isr_enable_read(bm) __bm_isr_read(bm, bm_isr_enable) +#define bm_isr_enable_write(bm, v) __bm_isr_write(bm, bm_isr_enable, v) +#define bm_isr_disable_read(bm) __bm_isr_read(bm, bm_isr_disable) +#define bm_isr_disable_write(bm, v) __bm_isr_write(bm, bm_isr_disable, v) +#define bm_isr_inhibit(bm) __bm_isr_write(bm, bm_isr_inhibit, 1) +#define bm_isr_uninhibit(bm) __bm_isr_write(bm, bm_isr_inhibit, 0) + +/* + * Global variables of the max portal/pool number this bman version supported + */ +extern u16 bman_pool_max; + +/* used by CCSR and portal interrupt code */ +enum bm_isr_reg { + bm_isr_status = 0, + bm_isr_enable = 1, + bm_isr_disable = 2, + bm_isr_inhibit = 3 +}; + +struct bm_portal_config { + /* + * Corenet portal addresses; + * [0]==cache-enabled, [1]==cache-inhibited. + */ + void __iomem *addr_virt[2]; + /* Allow these to be joined in lists */ + struct list_head list; + /* User-visible portal configuration settings */ + /* This is used for any "core-affine" portals, ie. default portals + * associated to the corresponding cpu. -1 implies that there is no + * core affinity configured. + */ + int cpu; + /* portal interrupt line */ + int irq; + /* the unique index of this portal */ + u32 index; + /* Is this portal shared? (If so, it has coarser locking and demuxes + * processing on behalf of other CPUs.). + */ + int is_shared; + /* These are the buffer pool IDs that may be used via this portal. */ + struct bman_depletion mask; + +}; + +int bman_init_ccsr(const struct device_node *node); + +struct bman_portal *bman_create_affine_portal( + const struct bm_portal_config *config); +const struct bm_portal_config *bman_destroy_affine_portal(void); + +/* Set depletion thresholds associated with a buffer pool. Requires that the + * operating system have access to Bman CCSR (ie. compiled in support and + * run-time access courtesy of the device-tree). + */ +int bm_pool_set(u32 bpid, const u32 *thresholds); + +/* Read the free buffer count for a given buffer */ +u32 bm_pool_free_buffers(u32 bpid); + +#endif /* __BMAN_PRIV_H */ diff --git a/dpdk/drivers/bus/dpaa/base/qbman/dpaa_alloc.c b/dpdk/drivers/bus/dpaa/base/qbman/dpaa_alloc.c new file mode 100644 index 00000000..35dba7f7 --- /dev/null +++ b/dpdk/drivers/bus/dpaa/base/qbman/dpaa_alloc.c @@ -0,0 +1,104 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2009-2016 Freescale Semiconductor Inc. + * Copyright 2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "dpaa_sys.h" +#include +#include +#include + +int bman_alloc_bpid_range(u32 *result, u32 count, u32 align, int partial) +{ + return process_alloc(dpaa_id_bpid, result, count, align, partial); +} + +void bman_release_bpid_range(u32 bpid, u32 count) +{ + process_release(dpaa_id_bpid, bpid, count); +} + +int bman_reserve_bpid_range(u32 bpid, u32 count) +{ + return process_reserve(dpaa_id_bpid, bpid, count); +} + +int qman_alloc_fqid_range(u32 *result, u32 count, u32 align, int partial) +{ + return process_alloc(dpaa_id_fqid, result, count, align, partial); +} + +void qman_release_fqid_range(u32 fqid, u32 count) +{ + process_release(dpaa_id_fqid, fqid, count); +} + +int qman_reserve_fqid_range(u32 fqid, unsigned int count) +{ + return process_reserve(dpaa_id_fqid, fqid, count); +} + +int qman_alloc_pool_range(u32 *result, u32 count, u32 align, int partial) +{ + return process_alloc(dpaa_id_qpool, result, count, align, partial); +} + +void qman_release_pool_range(u32 pool, u32 count) +{ + process_release(dpaa_id_qpool, pool, count); +} + +int qman_reserve_pool_range(u32 pool, u32 count) +{ + return process_reserve(dpaa_id_qpool, pool, count); +} + +int qman_alloc_cgrid_range(u32 *result, u32 count, u32 align, int partial) +{ + return process_alloc(dpaa_id_cgrid, result, count, align, partial); +} + +void qman_release_cgrid_range(u32 cgrid, u32 count) +{ + process_release(dpaa_id_cgrid, cgrid, count); +} + +int qman_reserve_cgrid_range(u32 cgrid, u32 count) +{ + return process_reserve(dpaa_id_cgrid, cgrid, count); +} diff --git a/dpdk/drivers/bus/dpaa/base/qbman/dpaa_sys.c b/dpdk/drivers/bus/dpaa/base/qbman/dpaa_sys.c new file mode 100644 index 00000000..0017da52 --- /dev/null +++ b/dpdk/drivers/bus/dpaa/base/qbman/dpaa_sys.c @@ -0,0 +1,136 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2013-2016 Freescale Semiconductor Inc. + * Copyright 2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include "dpaa_sys.h" + +struct process_interrupt { + int irq; + irqreturn_t (*isr)(int irq, void *arg); + unsigned long flags; + const char *name; + void *arg; + struct list_head node; +}; + +static COMPAT_LIST_HEAD(process_irq_list); +static pthread_mutex_t process_irq_lock = PTHREAD_MUTEX_INITIALIZER; + +static void process_interrupt_install(struct process_interrupt *irq) +{ + int ret; + /* Add the irq to the end of the list */ + ret = pthread_mutex_lock(&process_irq_lock); + assert(!ret); + list_add_tail(&irq->node, &process_irq_list); + ret = pthread_mutex_unlock(&process_irq_lock); + assert(!ret); +} + +static void process_interrupt_remove(struct process_interrupt *irq) +{ + int ret; + + ret = pthread_mutex_lock(&process_irq_lock); + assert(!ret); + list_del(&irq->node); + ret = pthread_mutex_unlock(&process_irq_lock); + assert(!ret); +} + +static struct process_interrupt *process_interrupt_find(int irq_num) +{ + int ret; + struct process_interrupt *i = NULL; + + ret = pthread_mutex_lock(&process_irq_lock); + assert(!ret); + list_for_each_entry(i, &process_irq_list, node) { + if (i->irq == irq_num) + goto done; + } +done: + ret = pthread_mutex_unlock(&process_irq_lock); + assert(!ret); + return i; +} + +/* This is the interface from the platform-agnostic driver code to (de)register + * interrupt handlers. We simply create/destroy corresponding structs. + */ +int qbman_request_irq(int irq, irqreturn_t (*isr)(int irq, void *arg), + unsigned long flags, const char *name, + void *arg __maybe_unused) +{ + struct process_interrupt *irq_node = + kmalloc(sizeof(*irq_node), GFP_KERNEL); + + if (!irq_node) + return -ENOMEM; + irq_node->irq = irq; + irq_node->isr = isr; + irq_node->flags = flags; + irq_node->name = name; + irq_node->arg = arg; + process_interrupt_install(irq_node); + return 0; +} + +int qbman_free_irq(int irq, __maybe_unused void *arg) +{ + struct process_interrupt *irq_node = process_interrupt_find(irq); + + if (!irq_node) + return -EINVAL; + process_interrupt_remove(irq_node); + kfree(irq_node); + return 0; +} + +/* This is the interface from the platform-specific driver code to obtain + * interrupt handlers that have been registered. + */ +void qbman_invoke_irq(int irq) +{ + struct process_interrupt *irq_node = process_interrupt_find(irq); + + if (irq_node) + irq_node->isr(irq, irq_node->arg); +} diff --git a/dpdk/drivers/bus/dpaa/base/qbman/dpaa_sys.h b/dpdk/drivers/bus/dpaa/base/qbman/dpaa_sys.h new file mode 100644 index 00000000..bee9fe57 --- /dev/null +++ b/dpdk/drivers/bus/dpaa/base/qbman/dpaa_sys.h @@ -0,0 +1,61 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2008-2016 Freescale Semiconductor Inc. + * Copyright 2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __DPAA_SYS_H +#define __DPAA_SYS_H + +#include + +/* For 2-element tables related to cache-inhibited and cache-enabled mappings */ +#define DPAA_PORTAL_CE 0 +#define DPAA_PORTAL_CI 1 + +#define DPAA_ASSERT(x) RTE_ASSERT(x) + +/* This is the interface from the platform-agnostic driver code to (de)register + * interrupt handlers. We simply create/destroy corresponding structs. + */ +int qbman_request_irq(int irq, irqreturn_t (*isr)(int irq, void *arg), + unsigned long flags, const char *name, void *arg); +int qbman_free_irq(int irq, void *arg); + +void qbman_invoke_irq(int irq); + +#endif /* __DPAA_SYS_H */ diff --git a/dpdk/drivers/bus/dpaa/base/qbman/process.c b/dpdk/drivers/bus/dpaa/base/qbman/process.c new file mode 100644 index 00000000..b8ec5398 --- /dev/null +++ b/dpdk/drivers/bus/dpaa/base/qbman/process.c @@ -0,0 +1,331 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2011-2016 Freescale Semiconductor Inc. + * Copyright 2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#include +#include +#include +#include + +#include "process.h" + +#include + +/* As higher-level drivers will be built on top of this (dma_mem, qbman, ...), + * it's preferable that the process driver itself not provide any exported API. + * As such, combined with the fact that none of these operations are + * performance critical, it is justified to use lazy initialisation, so that's + * what the lock is for. + */ +static int fd = -1; +static pthread_mutex_t fd_init_lock = PTHREAD_MUTEX_INITIALIZER; + +static int check_fd(void) +{ + int ret; + + if (fd >= 0) + return 0; + ret = pthread_mutex_lock(&fd_init_lock); + assert(!ret); + /* check again with the lock held */ + if (fd < 0) + fd = open(PROCESS_PATH, O_RDWR); + ret = pthread_mutex_unlock(&fd_init_lock); + assert(!ret); + return (fd >= 0) ? 0 : -ENODEV; +} + +#define DPAA_IOCTL_MAGIC 'u' +struct dpaa_ioctl_id_alloc { + uint32_t base; /* Return value, the start of the allocated range */ + enum dpaa_id_type id_type; /* what kind of resource(s) to allocate */ + uint32_t num; /* how many IDs to allocate (and return value) */ + uint32_t align; /* must be a power of 2, 0 is treated like 1 */ + int partial; /* whether to allow less than 'num' */ +}; + +struct dpaa_ioctl_id_release { + /* Input; */ + enum dpaa_id_type id_type; + uint32_t base; + uint32_t num; +}; + +struct dpaa_ioctl_id_reserve { + enum dpaa_id_type id_type; + uint32_t base; + uint32_t num; +}; + +#define DPAA_IOCTL_ID_ALLOC \ + _IOWR(DPAA_IOCTL_MAGIC, 0x01, struct dpaa_ioctl_id_alloc) +#define DPAA_IOCTL_ID_RELEASE \ + _IOW(DPAA_IOCTL_MAGIC, 0x02, struct dpaa_ioctl_id_release) +#define DPAA_IOCTL_ID_RESERVE \ + _IOW(DPAA_IOCTL_MAGIC, 0x0A, struct dpaa_ioctl_id_reserve) + +int process_alloc(enum dpaa_id_type id_type, uint32_t *base, uint32_t num, + uint32_t align, int partial) +{ + struct dpaa_ioctl_id_alloc id = { + .id_type = id_type, + .num = num, + .align = align, + .partial = partial + }; + int ret = check_fd(); + + if (ret) + return ret; + ret = ioctl(fd, DPAA_IOCTL_ID_ALLOC, &id); + if (ret) + return ret; + for (ret = 0; ret < (int)id.num; ret++) + base[ret] = id.base + ret; + return id.num; +} + +void process_release(enum dpaa_id_type id_type, uint32_t base, uint32_t num) +{ + struct dpaa_ioctl_id_release id = { + .id_type = id_type, + .base = base, + .num = num + }; + int ret = check_fd(); + + if (ret) { + fprintf(stderr, "Process FD failure\n"); + return; + } + ret = ioctl(fd, DPAA_IOCTL_ID_RELEASE, &id); + if (ret) + fprintf(stderr, "Process FD ioctl failure type %d base 0x%x num %d\n", + id_type, base, num); +} + +int process_reserve(enum dpaa_id_type id_type, uint32_t base, uint32_t num) +{ + struct dpaa_ioctl_id_reserve id = { + .id_type = id_type, + .base = base, + .num = num + }; + int ret = check_fd(); + + if (ret) + return ret; + return ioctl(fd, DPAA_IOCTL_ID_RESERVE, &id); +} + +/***************************************/ +/* Mapping and using QMan/BMan portals */ +/***************************************/ + +#define DPAA_IOCTL_PORTAL_MAP \ + _IOWR(DPAA_IOCTL_MAGIC, 0x07, struct dpaa_ioctl_portal_map) +#define DPAA_IOCTL_PORTAL_UNMAP \ + _IOW(DPAA_IOCTL_MAGIC, 0x08, struct dpaa_portal_map) + +int process_portal_map(struct dpaa_ioctl_portal_map *params) +{ + int ret = check_fd(); + + if (ret) + return ret; + + ret = ioctl(fd, DPAA_IOCTL_PORTAL_MAP, params); + if (ret) { + perror("ioctl(DPAA_IOCTL_PORTAL_MAP)"); + return ret; + } + return 0; +} + +int process_portal_unmap(struct dpaa_portal_map *map) +{ + int ret = check_fd(); + + if (ret) + return ret; + + ret = ioctl(fd, DPAA_IOCTL_PORTAL_UNMAP, map); + if (ret) { + perror("ioctl(DPAA_IOCTL_PORTAL_UNMAP)"); + return ret; + } + return 0; +} + +#define DPAA_IOCTL_PORTAL_IRQ_MAP \ + _IOW(DPAA_IOCTL_MAGIC, 0x09, struct dpaa_ioctl_irq_map) + +int process_portal_irq_map(int ifd, struct dpaa_ioctl_irq_map *map) +{ + map->fd = fd; + return ioctl(ifd, DPAA_IOCTL_PORTAL_IRQ_MAP, map); +} + +int process_portal_irq_unmap(int ifd) +{ + return close(ifd); +} + +struct dpaa_ioctl_raw_portal { + /* inputs */ + enum dpaa_portal_type type; /* Type of portal to allocate */ + + uint8_t enable_stash; /* set to non zero to turn on stashing */ + /* Stashing attributes for the portal */ + uint32_t cpu; + uint32_t cache; + uint32_t window; + /* Specifies the stash request queue this portal should use */ + uint8_t sdest; + + /* Specifes a specific portal index to map or QBMAN_ANY_PORTAL_IDX + * for don't care. The portal index will be populated by the + * driver when the ioctl() successfully completes. + */ + uint32_t index; + + /* outputs */ + uint64_t cinh; + uint64_t cena; +}; + +#define DPAA_IOCTL_ALLOC_RAW_PORTAL \ + _IOWR(DPAA_IOCTL_MAGIC, 0x0C, struct dpaa_ioctl_raw_portal) + +#define DPAA_IOCTL_FREE_RAW_PORTAL \ + _IOR(DPAA_IOCTL_MAGIC, 0x0D, struct dpaa_ioctl_raw_portal) + +static int process_portal_allocate(struct dpaa_ioctl_raw_portal *portal) +{ + int ret = check_fd(); + + if (ret) + return ret; + + ret = ioctl(fd, DPAA_IOCTL_ALLOC_RAW_PORTAL, portal); + if (ret) { + perror("ioctl(DPAA_IOCTL_ALLOC_RAW_PORTAL)"); + return ret; + } + return 0; +} + +static int process_portal_free(struct dpaa_ioctl_raw_portal *portal) +{ + int ret = check_fd(); + + if (ret) + return ret; + + ret = ioctl(fd, DPAA_IOCTL_FREE_RAW_PORTAL, portal); + if (ret) { + perror("ioctl(DPAA_IOCTL_FREE_RAW_PORTAL)"); + return ret; + } + return 0; +} + +int qman_allocate_raw_portal(struct dpaa_raw_portal *portal) +{ + struct dpaa_ioctl_raw_portal input; + int ret; + + input.type = dpaa_portal_qman; + input.index = portal->index; + input.enable_stash = portal->enable_stash; + input.cpu = portal->cpu; + input.cache = portal->cache; + input.window = portal->window; + input.sdest = portal->sdest; + + ret = process_portal_allocate(&input); + if (ret) + return ret; + portal->index = input.index; + portal->cinh = input.cinh; + portal->cena = input.cena; + return 0; +} + +int qman_free_raw_portal(struct dpaa_raw_portal *portal) +{ + struct dpaa_ioctl_raw_portal input; + + input.type = dpaa_portal_qman; + input.index = portal->index; + input.cinh = portal->cinh; + input.cena = portal->cena; + + return process_portal_free(&input); +} + +int bman_allocate_raw_portal(struct dpaa_raw_portal *portal) +{ + struct dpaa_ioctl_raw_portal input; + int ret; + + input.type = dpaa_portal_bman; + input.index = portal->index; + input.enable_stash = 0; + + ret = process_portal_allocate(&input); + if (ret) + return ret; + portal->index = input.index; + portal->cinh = input.cinh; + portal->cena = input.cena; + return 0; +} + +int bman_free_raw_portal(struct dpaa_raw_portal *portal) +{ + struct dpaa_ioctl_raw_portal input; + + input.type = dpaa_portal_bman; + input.index = portal->index; + input.cinh = portal->cinh; + input.cena = portal->cena; + + return process_portal_free(&input); +} diff --git a/dpdk/drivers/bus/dpaa/base/qbman/qman.c b/dpdk/drivers/bus/dpaa/base/qbman/qman.c new file mode 100644 index 00000000..b8511103 --- /dev/null +++ b/dpdk/drivers/bus/dpaa/base/qbman/qman.c @@ -0,0 +1,2502 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2008-2016 Freescale Semiconductor Inc. + * Copyright 2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "qman.h" +#include + +/* Compilation constants */ +#define DQRR_MAXFILL 15 +#define EQCR_ITHRESH 4 /* if EQCR congests, interrupt threshold */ +#define IRQNAME "QMan portal %d" +#define MAX_IRQNAME 16 /* big enough for "QMan portal %d" */ +/* maximum number of DQRR entries to process in qman_poll() */ +#define FSL_QMAN_POLL_LIMIT 8 + +/* Lock/unlock frame queues, subject to the "LOCKED" flag. This is about + * inter-processor locking only. Note, FQLOCK() is always called either under a + * local_irq_save() or from interrupt context - hence there's no need for irq + * protection (and indeed, attempting to nest irq-protection doesn't work, as + * the "irq en/disable" machinery isn't recursive...). + */ +#define FQLOCK(fq) \ + do { \ + struct qman_fq *__fq478 = (fq); \ + if (fq_isset(__fq478, QMAN_FQ_FLAG_LOCKED)) \ + spin_lock(&__fq478->fqlock); \ + } while (0) +#define FQUNLOCK(fq) \ + do { \ + struct qman_fq *__fq478 = (fq); \ + if (fq_isset(__fq478, QMAN_FQ_FLAG_LOCKED)) \ + spin_unlock(&__fq478->fqlock); \ + } while (0) + +static inline void fq_set(struct qman_fq *fq, u32 mask) +{ + dpaa_set_bits(mask, &fq->flags); +} + +static inline void fq_clear(struct qman_fq *fq, u32 mask) +{ + dpaa_clear_bits(mask, &fq->flags); +} + +static inline int fq_isset(struct qman_fq *fq, u32 mask) +{ + return fq->flags & mask; +} + +static inline int fq_isclear(struct qman_fq *fq, u32 mask) +{ + return !(fq->flags & mask); +} + +struct qman_portal { + struct qm_portal p; + /* PORTAL_BITS_*** - dynamic, strictly internal */ + unsigned long bits; + /* interrupt sources processed by portal_isr(), configurable */ + unsigned long irq_sources; + u32 use_eqcr_ci_stashing; + u32 slowpoll; /* only used when interrupts are off */ + /* only 1 volatile dequeue at a time */ + struct qman_fq *vdqcr_owned; + u32 sdqcr; + int dqrr_disable_ref; + /* A portal-specific handler for DCP ERNs. If this is NULL, the global + * handler is called instead. + */ + qman_cb_dc_ern cb_dc_ern; + /* When the cpu-affine portal is activated, this is non-NULL */ + const struct qm_portal_config *config; + struct dpa_rbtree retire_table; + char irqname[MAX_IRQNAME]; + /* 2-element array. cgrs[0] is mask, cgrs[1] is snapshot. */ + struct qman_cgrs *cgrs; + /* linked-list of CSCN handlers. */ + struct list_head cgr_cbs; + /* list lock */ + spinlock_t cgr_lock; + /* track if memory was allocated by the driver */ +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + /* Keep a shadow copy of the DQRR on LE systems as the SW needs to + * do byte swaps of DQRR read only memory. First entry must be aligned + * to 2 ** 10 to ensure DQRR index calculations based shadow copy + * address (6 bits for address shift + 4 bits for the DQRR size). + */ + struct qm_dqrr_entry shadow_dqrr[QM_DQRR_SIZE] + __attribute__((aligned(1024))); +#endif +}; + +/* Global handler for DCP ERNs. Used when the portal receiving the message does + * not have a portal-specific handler. + */ +static qman_cb_dc_ern cb_dc_ern; + +static cpumask_t affine_mask; +static DEFINE_SPINLOCK(affine_mask_lock); +static u16 affine_channels[NR_CPUS]; +static RTE_DEFINE_PER_LCORE(struct qman_portal, qman_affine_portal); + +static inline struct qman_portal *get_affine_portal(void) +{ + return &RTE_PER_LCORE(qman_affine_portal); +} + +/* This gives a FQID->FQ lookup to cover the fact that we can't directly demux + * retirement notifications (the fact they are sometimes h/w-consumed means that + * contextB isn't always a s/w demux - and as we can't know which case it is + * when looking at the notification, we have to use the slow lookup for all of + * them). NB, it's possible to have multiple FQ objects refer to the same FQID + * (though at most one of them should be the consumer), so this table isn't for + * all FQs - FQs are added when retirement commands are issued, and removed when + * they complete, which also massively reduces the size of this table. + */ +IMPLEMENT_DPAA_RBTREE(fqtree, struct qman_fq, node, fqid); +/* + * This is what everything can wait on, even if it migrates to a different cpu + * to the one whose affine portal it is waiting on. + */ +static DECLARE_WAIT_QUEUE_HEAD(affine_queue); + +static inline int table_push_fq(struct qman_portal *p, struct qman_fq *fq) +{ + int ret = fqtree_push(&p->retire_table, fq); + + if (ret) + pr_err("ERROR: double FQ-retirement %d\n", fq->fqid); + return ret; +} + +static inline void table_del_fq(struct qman_portal *p, struct qman_fq *fq) +{ + fqtree_del(&p->retire_table, fq); +} + +static inline struct qman_fq *table_find_fq(struct qman_portal *p, u32 fqid) +{ + return fqtree_find(&p->retire_table, fqid); +} + +#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP +static void **qman_fq_lookup_table; +static size_t qman_fq_lookup_table_size; + +int qman_setup_fq_lookup_table(size_t num_entries) +{ + num_entries++; + /* Allocate 1 more entry since the first entry is not used */ + qman_fq_lookup_table = vmalloc((num_entries * sizeof(void *))); + if (!qman_fq_lookup_table) { + pr_err("QMan: Could not allocate fq lookup table\n"); + return -ENOMEM; + } + memset(qman_fq_lookup_table, 0, num_entries * sizeof(void *)); + qman_fq_lookup_table_size = num_entries; + pr_debug("QMan: Allocated lookup table at %p, entry count %lu\n", + qman_fq_lookup_table, + (unsigned long)qman_fq_lookup_table_size); + return 0; +} + +/* global structure that maintains fq object mapping */ +static DEFINE_SPINLOCK(fq_hash_table_lock); + +static int find_empty_fq_table_entry(u32 *entry, struct qman_fq *fq) +{ + u32 i; + + spin_lock(&fq_hash_table_lock); + /* Can't use index zero because this has special meaning + * in context_b field. + */ + for (i = 1; i < qman_fq_lookup_table_size; i++) { + if (qman_fq_lookup_table[i] == NULL) { + *entry = i; + qman_fq_lookup_table[i] = fq; + spin_unlock(&fq_hash_table_lock); + return 0; + } + } + spin_unlock(&fq_hash_table_lock); + return -ENOMEM; +} + +static void clear_fq_table_entry(u32 entry) +{ + spin_lock(&fq_hash_table_lock); + DPAA_BUG_ON(entry >= qman_fq_lookup_table_size); + qman_fq_lookup_table[entry] = NULL; + spin_unlock(&fq_hash_table_lock); +} + +static inline struct qman_fq *get_fq_table_entry(u32 entry) +{ + DPAA_BUG_ON(entry >= qman_fq_lookup_table_size); + return qman_fq_lookup_table[entry]; +} +#endif + +static inline void cpu_to_hw_fqd(struct qm_fqd *fqd) +{ + /* Byteswap the FQD to HW format */ + fqd->fq_ctrl = cpu_to_be16(fqd->fq_ctrl); + fqd->dest_wq = cpu_to_be16(fqd->dest_wq); + fqd->ics_cred = cpu_to_be16(fqd->ics_cred); + fqd->context_b = cpu_to_be32(fqd->context_b); + fqd->context_a.opaque = cpu_to_be64(fqd->context_a.opaque); + fqd->opaque_td = cpu_to_be16(fqd->opaque_td); +} + +static inline void hw_fqd_to_cpu(struct qm_fqd *fqd) +{ + /* Byteswap the FQD to CPU format */ + fqd->fq_ctrl = be16_to_cpu(fqd->fq_ctrl); + fqd->dest_wq = be16_to_cpu(fqd->dest_wq); + fqd->ics_cred = be16_to_cpu(fqd->ics_cred); + fqd->context_b = be32_to_cpu(fqd->context_b); + fqd->context_a.opaque = be64_to_cpu(fqd->context_a.opaque); +} + +static inline void cpu_to_hw_fd(struct qm_fd *fd) +{ + fd->addr = cpu_to_be40(fd->addr); + fd->status = cpu_to_be32(fd->status); + fd->opaque = cpu_to_be32(fd->opaque); +} + +static inline void hw_fd_to_cpu(struct qm_fd *fd) +{ + fd->addr = be40_to_cpu(fd->addr); + fd->status = be32_to_cpu(fd->status); + fd->opaque = be32_to_cpu(fd->opaque); +} + +/* In the case that slow- and fast-path handling are both done by qman_poll() + * (ie. because there is no interrupt handling), we ought to balance how often + * we do the fast-path poll versus the slow-path poll. We'll use two decrementer + * sources, so we call the fast poll 'n' times before calling the slow poll + * once. The idle decrementer constant is used when the last slow-poll detected + * no work to do, and the busy decrementer constant when the last slow-poll had + * work to do. + */ +#define SLOW_POLL_IDLE 1000 +#define SLOW_POLL_BUSY 10 +static u32 __poll_portal_slow(struct qman_portal *p, u32 is); +static inline unsigned int __poll_portal_fast(struct qman_portal *p, + unsigned int poll_limit); + +/* Portal interrupt handler */ +static irqreturn_t portal_isr(__always_unused int irq, void *ptr) +{ + struct qman_portal *p = ptr; + /* + * The CSCI/CCSCI source is cleared inside __poll_portal_slow(), because + * it could race against a Query Congestion State command also given + * as part of the handling of this interrupt source. We mustn't + * clear it a second time in this top-level function. + */ + u32 clear = QM_DQAVAIL_MASK | (p->irq_sources & + ~(QM_PIRQ_CSCI | QM_PIRQ_CCSCI)); + u32 is = qm_isr_status_read(&p->p) & p->irq_sources; + /* DQRR-handling if it's interrupt-driven */ + if (is & QM_PIRQ_DQRI) + __poll_portal_fast(p, FSL_QMAN_POLL_LIMIT); + /* Handling of anything else that's interrupt-driven */ + clear |= __poll_portal_slow(p, is); + qm_isr_status_clear(&p->p, clear); + return IRQ_HANDLED; +} + +/* This inner version is used privately by qman_create_affine_portal(), as well + * as by the exported qman_stop_dequeues(). + */ +static inline void qman_stop_dequeues_ex(struct qman_portal *p) +{ + if (!(p->dqrr_disable_ref++)) + qm_dqrr_set_maxfill(&p->p, 0); +} + +static int drain_mr_fqrni(struct qm_portal *p) +{ + const struct qm_mr_entry *msg; +loop: + msg = qm_mr_current(p); + if (!msg) { + /* + * if MR was full and h/w had other FQRNI entries to produce, we + * need to allow it time to produce those entries once the + * existing entries are consumed. A worst-case situation + * (fully-loaded system) means h/w sequencers may have to do 3-4 + * other things before servicing the portal's MR pump, each of + * which (if slow) may take ~50 qman cycles (which is ~200 + * processor cycles). So rounding up and then multiplying this + * worst-case estimate by a factor of 10, just to be + * ultra-paranoid, goes as high as 10,000 cycles. NB, we consume + * one entry at a time, so h/w has an opportunity to produce new + * entries well before the ring has been fully consumed, so + * we're being *really* paranoid here. + */ + u64 now, then = mfatb(); + + do { + now = mfatb(); + } while ((then + 10000) > now); + msg = qm_mr_current(p); + if (!msg) + return 0; + } + if ((msg->verb & QM_MR_VERB_TYPE_MASK) != QM_MR_VERB_FQRNI) { + /* We aren't draining anything but FQRNIs */ + pr_err("Found verb 0x%x in MR\n", msg->verb); + return -1; + } + qm_mr_next(p); + qm_mr_cci_consume(p, 1); + goto loop; +} + +static inline int qm_eqcr_init(struct qm_portal *portal, + enum qm_eqcr_pmode pmode, + unsigned int eq_stash_thresh, + int eq_stash_prio) +{ + /* This use of 'register', as well as all other occurrences, is because + * it has been observed to generate much faster code with gcc than is + * otherwise the case. + */ + register struct qm_eqcr *eqcr = &portal->eqcr; + u32 cfg; + u8 pi; + + eqcr->ring = portal->addr.ce + QM_CL_EQCR; + eqcr->ci = qm_in(EQCR_CI_CINH) & (QM_EQCR_SIZE - 1); + qm_cl_invalidate(EQCR_CI); + pi = qm_in(EQCR_PI_CINH) & (QM_EQCR_SIZE - 1); + eqcr->cursor = eqcr->ring + pi; + eqcr->vbit = (qm_in(EQCR_PI_CINH) & QM_EQCR_SIZE) ? + QM_EQCR_VERB_VBIT : 0; + eqcr->available = QM_EQCR_SIZE - 1 - + qm_cyc_diff(QM_EQCR_SIZE, eqcr->ci, pi); + eqcr->ithresh = qm_in(EQCR_ITR); +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + eqcr->busy = 0; + eqcr->pmode = pmode; +#endif + cfg = (qm_in(CFG) & 0x00ffffff) | + (eq_stash_thresh << 28) | /* QCSP_CFG: EST */ + (eq_stash_prio << 26) | /* QCSP_CFG: EP */ + ((pmode & 0x3) << 24); /* QCSP_CFG::EPM */ + qm_out(CFG, cfg); + return 0; +} + +static inline void qm_eqcr_finish(struct qm_portal *portal) +{ + register struct qm_eqcr *eqcr = &portal->eqcr; + u8 pi, ci; + u32 cfg; + + /* + * Disable EQCI stashing because the QMan only + * presents the value it previously stashed to + * maintain coherency. Setting the stash threshold + * to 1 then 0 ensures that QMan has resyncronized + * its internal copy so that the portal is clean + * when it is reinitialized in the future + */ + cfg = (qm_in(CFG) & 0x0fffffff) | + (1 << 28); /* QCSP_CFG: EST */ + qm_out(CFG, cfg); + cfg &= 0x0fffffff; /* stash threshold = 0 */ + qm_out(CFG, cfg); + + pi = qm_in(EQCR_PI_CINH) & (QM_EQCR_SIZE - 1); + ci = qm_in(EQCR_CI_CINH) & (QM_EQCR_SIZE - 1); + + /* Refresh EQCR CI cache value */ + qm_cl_invalidate(EQCR_CI); + eqcr->ci = qm_cl_in(EQCR_CI) & (QM_EQCR_SIZE - 1); + +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + DPAA_ASSERT(!eqcr->busy); +#endif + if (pi != EQCR_PTR2IDX(eqcr->cursor)) + pr_crit("losing uncommitted EQCR entries\n"); + if (ci != eqcr->ci) + pr_crit("missing existing EQCR completions\n"); + if (eqcr->ci != EQCR_PTR2IDX(eqcr->cursor)) + pr_crit("EQCR destroyed unquiesced\n"); +} + +static inline int qm_dqrr_init(struct qm_portal *portal, + __maybe_unused const struct qm_portal_config *config, + enum qm_dqrr_dmode dmode, + __maybe_unused enum qm_dqrr_pmode pmode, + enum qm_dqrr_cmode cmode, u8 max_fill) +{ + register struct qm_dqrr *dqrr = &portal->dqrr; + u32 cfg; + + /* Make sure the DQRR will be idle when we enable */ + qm_out(DQRR_SDQCR, 0); + qm_out(DQRR_VDQCR, 0); + qm_out(DQRR_PDQCR, 0); + dqrr->ring = portal->addr.ce + QM_CL_DQRR; + dqrr->pi = qm_in(DQRR_PI_CINH) & (QM_DQRR_SIZE - 1); + dqrr->ci = qm_in(DQRR_CI_CINH) & (QM_DQRR_SIZE - 1); + dqrr->cursor = dqrr->ring + dqrr->ci; + dqrr->fill = qm_cyc_diff(QM_DQRR_SIZE, dqrr->ci, dqrr->pi); + dqrr->vbit = (qm_in(DQRR_PI_CINH) & QM_DQRR_SIZE) ? + QM_DQRR_VERB_VBIT : 0; + dqrr->ithresh = qm_in(DQRR_ITR); +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + dqrr->dmode = dmode; + dqrr->pmode = pmode; + dqrr->cmode = cmode; +#endif + /* Invalidate every ring entry before beginning */ + for (cfg = 0; cfg < QM_DQRR_SIZE; cfg++) + dccivac(qm_cl(dqrr->ring, cfg)); + cfg = (qm_in(CFG) & 0xff000f00) | + ((max_fill & (QM_DQRR_SIZE - 1)) << 20) | /* DQRR_MF */ + ((dmode & 1) << 18) | /* DP */ + ((cmode & 3) << 16) | /* DCM */ + 0xa0 | /* RE+SE */ + (0 ? 0x40 : 0) | /* Ignore RP */ + (0 ? 0x10 : 0); /* Ignore SP */ + qm_out(CFG, cfg); + qm_dqrr_set_maxfill(portal, max_fill); + return 0; +} + +static inline void qm_dqrr_finish(struct qm_portal *portal) +{ + __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr; +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + if ((dqrr->cmode != qm_dqrr_cdc) && + (dqrr->ci != DQRR_PTR2IDX(dqrr->cursor))) + pr_crit("Ignoring completed DQRR entries\n"); +#endif +} + +static inline int qm_mr_init(struct qm_portal *portal, + __maybe_unused enum qm_mr_pmode pmode, + enum qm_mr_cmode cmode) +{ + register struct qm_mr *mr = &portal->mr; + u32 cfg; + + mr->ring = portal->addr.ce + QM_CL_MR; + mr->pi = qm_in(MR_PI_CINH) & (QM_MR_SIZE - 1); + mr->ci = qm_in(MR_CI_CINH) & (QM_MR_SIZE - 1); + mr->cursor = mr->ring + mr->ci; + mr->fill = qm_cyc_diff(QM_MR_SIZE, mr->ci, mr->pi); + mr->vbit = (qm_in(MR_PI_CINH) & QM_MR_SIZE) ? QM_MR_VERB_VBIT : 0; + mr->ithresh = qm_in(MR_ITR); +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + mr->pmode = pmode; + mr->cmode = cmode; +#endif + cfg = (qm_in(CFG) & 0xfffff0ff) | + ((cmode & 1) << 8); /* QCSP_CFG:MM */ + qm_out(CFG, cfg); + return 0; +} + +static inline void qm_mr_pvb_update(struct qm_portal *portal) +{ + register struct qm_mr *mr = &portal->mr; + const struct qm_mr_entry *res = qm_cl(mr->ring, mr->pi); + +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + DPAA_ASSERT(mr->pmode == qm_mr_pvb); +#endif + /* when accessing 'verb', use __raw_readb() to ensure that compiler + * inlining doesn't try to optimise out "excess reads". + */ + if ((__raw_readb(&res->verb) & QM_MR_VERB_VBIT) == mr->vbit) { + mr->pi = (mr->pi + 1) & (QM_MR_SIZE - 1); + if (!mr->pi) + mr->vbit ^= QM_MR_VERB_VBIT; + mr->fill++; + res = MR_INC(res); + } + dcbit_ro(res); +} + +static inline +struct qman_portal *qman_create_portal( + struct qman_portal *portal, + const struct qm_portal_config *c, + const struct qman_cgrs *cgrs) +{ + struct qm_portal *p; + char buf[16]; + int ret; + u32 isdr; + + p = &portal->p; + + portal->use_eqcr_ci_stashing = ((qman_ip_rev >= QMAN_REV30) ? 1 : 0); + /* + * prep the low-level portal struct with the mapped addresses from the + * config, everything that follows depends on it and "config" is more + * for (de)reference + */ + p->addr.ce = c->addr_virt[DPAA_PORTAL_CE]; + p->addr.ci = c->addr_virt[DPAA_PORTAL_CI]; + /* + * If CI-stashing is used, the current defaults use a threshold of 3, + * and stash with high-than-DQRR priority. + */ + if (qm_eqcr_init(p, qm_eqcr_pvb, + portal->use_eqcr_ci_stashing ? 3 : 0, 1)) { + pr_err("Qman EQCR initialisation failed\n"); + goto fail_eqcr; + } + if (qm_dqrr_init(p, c, qm_dqrr_dpush, qm_dqrr_pvb, + qm_dqrr_cdc, DQRR_MAXFILL)) { + pr_err("Qman DQRR initialisation failed\n"); + goto fail_dqrr; + } + if (qm_mr_init(p, qm_mr_pvb, qm_mr_cci)) { + pr_err("Qman MR initialisation failed\n"); + goto fail_mr; + } + if (qm_mc_init(p)) { + pr_err("Qman MC initialisation failed\n"); + goto fail_mc; + } + + /* static interrupt-gating controls */ + qm_dqrr_set_ithresh(p, 0); + qm_mr_set_ithresh(p, 0); + qm_isr_set_iperiod(p, 0); + portal->cgrs = kmalloc(2 * sizeof(*cgrs), GFP_KERNEL); + if (!portal->cgrs) + goto fail_cgrs; + /* initial snapshot is no-depletion */ + qman_cgrs_init(&portal->cgrs[1]); + if (cgrs) + portal->cgrs[0] = *cgrs; + else + /* if the given mask is NULL, assume all CGRs can be seen */ + qman_cgrs_fill(&portal->cgrs[0]); + INIT_LIST_HEAD(&portal->cgr_cbs); + spin_lock_init(&portal->cgr_lock); + portal->bits = 0; + portal->slowpoll = 0; + portal->sdqcr = QM_SDQCR_SOURCE_CHANNELS | QM_SDQCR_COUNT_UPTO3 | + QM_SDQCR_DEDICATED_PRECEDENCE | QM_SDQCR_TYPE_PRIO_QOS | + QM_SDQCR_TOKEN_SET(0xab) | QM_SDQCR_CHANNELS_DEDICATED; + portal->dqrr_disable_ref = 0; + portal->cb_dc_ern = NULL; + sprintf(buf, "qportal-%d", c->channel); + dpa_rbtree_init(&portal->retire_table); + isdr = 0xffffffff; + qm_isr_disable_write(p, isdr); + portal->irq_sources = 0; + qm_isr_enable_write(p, portal->irq_sources); + qm_isr_status_clear(p, 0xffffffff); + snprintf(portal->irqname, MAX_IRQNAME, IRQNAME, c->cpu); + if (request_irq(c->irq, portal_isr, 0, portal->irqname, + portal)) { + pr_err("request_irq() failed\n"); + goto fail_irq; + } + + /* Need EQCR to be empty before continuing */ + isdr &= ~QM_PIRQ_EQCI; + qm_isr_disable_write(p, isdr); + ret = qm_eqcr_get_fill(p); + if (ret) { + pr_err("Qman EQCR unclean\n"); + goto fail_eqcr_empty; + } + isdr &= ~(QM_PIRQ_DQRI | QM_PIRQ_MRI); + qm_isr_disable_write(p, isdr); + if (qm_dqrr_current(p)) { + pr_err("Qman DQRR unclean\n"); + qm_dqrr_cdc_consume_n(p, 0xffff); + } + if (qm_mr_current(p) && drain_mr_fqrni(p)) { + /* special handling, drain just in case it's a few FQRNIs */ + if (drain_mr_fqrni(p)) + goto fail_dqrr_mr_empty; + } + /* Success */ + portal->config = c; + qm_isr_disable_write(p, 0); + qm_isr_uninhibit(p); + /* Write a sane SDQCR */ + qm_dqrr_sdqcr_set(p, portal->sdqcr); + return portal; +fail_dqrr_mr_empty: +fail_eqcr_empty: + free_irq(c->irq, portal); +fail_irq: + kfree(portal->cgrs); + spin_lock_destroy(&portal->cgr_lock); +fail_cgrs: + qm_mc_finish(p); +fail_mc: + qm_mr_finish(p); +fail_mr: + qm_dqrr_finish(p); +fail_dqrr: + qm_eqcr_finish(p); +fail_eqcr: + return NULL; +} + +struct qman_portal *qman_create_affine_portal(const struct qm_portal_config *c, + const struct qman_cgrs *cgrs) +{ + struct qman_portal *res; + struct qman_portal *portal = get_affine_portal(); + /* A criteria for calling this function (from qman_driver.c) is that + * we're already affine to the cpu and won't schedule onto another cpu. + */ + + res = qman_create_portal(portal, c, cgrs); + if (res) { + spin_lock(&affine_mask_lock); + CPU_SET(c->cpu, &affine_mask); + affine_channels[c->cpu] = + c->channel; + spin_unlock(&affine_mask_lock); + } + return res; +} + +static inline +void qman_destroy_portal(struct qman_portal *qm) +{ + const struct qm_portal_config *pcfg; + + /* Stop dequeues on the portal */ + qm_dqrr_sdqcr_set(&qm->p, 0); + + /* + * NB we do this to "quiesce" EQCR. If we add enqueue-completions or + * something related to QM_PIRQ_EQCI, this may need fixing. + * Also, due to the prefetching model used for CI updates in the enqueue + * path, this update will only invalidate the CI cacheline *after* + * working on it, so we need to call this twice to ensure a full update + * irrespective of where the enqueue processing was at when the teardown + * began. + */ + qm_eqcr_cce_update(&qm->p); + qm_eqcr_cce_update(&qm->p); + pcfg = qm->config; + + free_irq(pcfg->irq, qm); + + kfree(qm->cgrs); + qm_mc_finish(&qm->p); + qm_mr_finish(&qm->p); + qm_dqrr_finish(&qm->p); + qm_eqcr_finish(&qm->p); + + qm->config = NULL; + + spin_lock_destroy(&qm->cgr_lock); +} + +const struct qm_portal_config *qman_destroy_affine_portal(void) +{ + /* We don't want to redirect if we're a slave, use "raw" */ + struct qman_portal *qm = get_affine_portal(); + const struct qm_portal_config *pcfg; + int cpu; + + pcfg = qm->config; + cpu = pcfg->cpu; + + qman_destroy_portal(qm); + + spin_lock(&affine_mask_lock); + CPU_CLR(cpu, &affine_mask); + spin_unlock(&affine_mask_lock); + return pcfg; +} + +int qman_get_portal_index(void) +{ + struct qman_portal *p = get_affine_portal(); + return p->config->index; +} + +/* Inline helper to reduce nesting in __poll_portal_slow() */ +static inline void fq_state_change(struct qman_portal *p, struct qman_fq *fq, + const struct qm_mr_entry *msg, u8 verb) +{ + FQLOCK(fq); + switch (verb) { + case QM_MR_VERB_FQRL: + DPAA_ASSERT(fq_isset(fq, QMAN_FQ_STATE_ORL)); + fq_clear(fq, QMAN_FQ_STATE_ORL); + table_del_fq(p, fq); + break; + case QM_MR_VERB_FQRN: + DPAA_ASSERT((fq->state == qman_fq_state_parked) || + (fq->state == qman_fq_state_sched)); + DPAA_ASSERT(fq_isset(fq, QMAN_FQ_STATE_CHANGING)); + fq_clear(fq, QMAN_FQ_STATE_CHANGING); + if (msg->fq.fqs & QM_MR_FQS_NOTEMPTY) + fq_set(fq, QMAN_FQ_STATE_NE); + if (msg->fq.fqs & QM_MR_FQS_ORLPRESENT) + fq_set(fq, QMAN_FQ_STATE_ORL); + else + table_del_fq(p, fq); + fq->state = qman_fq_state_retired; + break; + case QM_MR_VERB_FQPN: + DPAA_ASSERT(fq->state == qman_fq_state_sched); + DPAA_ASSERT(fq_isclear(fq, QMAN_FQ_STATE_CHANGING)); + fq->state = qman_fq_state_parked; + } + FQUNLOCK(fq); +} + +static u32 __poll_portal_slow(struct qman_portal *p, u32 is) +{ + const struct qm_mr_entry *msg; + struct qm_mr_entry swapped_msg; + + if (is & QM_PIRQ_CSCI) { + struct qman_cgrs rr, c; + struct qm_mc_result *mcr; + struct qman_cgr *cgr; + + spin_lock(&p->cgr_lock); + /* + * The CSCI bit must be cleared _before_ issuing the + * Query Congestion State command, to ensure that a long + * CGR State Change callback cannot miss an intervening + * state change. + */ + qm_isr_status_clear(&p->p, QM_PIRQ_CSCI); + qm_mc_start(&p->p); + qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCONGESTION); + while (!(mcr = qm_mc_result(&p->p))) + cpu_relax(); + /* mask out the ones I'm not interested in */ + qman_cgrs_and(&rr, (const struct qman_cgrs *) + &mcr->querycongestion.state, &p->cgrs[0]); + /* check previous snapshot for delta, enter/exit congestion */ + qman_cgrs_xor(&c, &rr, &p->cgrs[1]); + /* update snapshot */ + qman_cgrs_cp(&p->cgrs[1], &rr); + /* Invoke callback */ + list_for_each_entry(cgr, &p->cgr_cbs, node) + if (cgr->cb && qman_cgrs_get(&c, cgr->cgrid)) + cgr->cb(p, cgr, qman_cgrs_get(&rr, cgr->cgrid)); + spin_unlock(&p->cgr_lock); + } + + if (is & QM_PIRQ_EQRI) { + qm_eqcr_cce_update(&p->p); + qm_eqcr_set_ithresh(&p->p, 0); + wake_up(&affine_queue); + } + + if (is & QM_PIRQ_MRI) { + struct qman_fq *fq; + u8 verb, num = 0; +mr_loop: + qm_mr_pvb_update(&p->p); + msg = qm_mr_current(&p->p); + if (!msg) + goto mr_done; + swapped_msg = *msg; + hw_fd_to_cpu(&swapped_msg.ern.fd); + verb = msg->verb & QM_MR_VERB_TYPE_MASK; + /* The message is a software ERN iff the 0x20 bit is set */ + if (verb & 0x20) { + switch (verb) { + case QM_MR_VERB_FQRNI: + /* nada, we drop FQRNIs on the floor */ + break; + case QM_MR_VERB_FQRN: + case QM_MR_VERB_FQRL: + /* Lookup in the retirement table */ + fq = table_find_fq(p, + be32_to_cpu(msg->fq.fqid)); + DPAA_BUG_ON(!fq); + fq_state_change(p, fq, &swapped_msg, verb); + if (fq->cb.fqs) + fq->cb.fqs(p, fq, &swapped_msg); + break; + case QM_MR_VERB_FQPN: + /* Parked */ +#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP + fq = get_fq_table_entry( + be32_to_cpu(msg->fq.contextB)); +#else + fq = (void *)(uintptr_t) + be32_to_cpu(msg->fq.contextB); +#endif + fq_state_change(p, fq, msg, verb); + if (fq->cb.fqs) + fq->cb.fqs(p, fq, &swapped_msg); + break; + case QM_MR_VERB_DC_ERN: + /* DCP ERN */ + if (p->cb_dc_ern) + p->cb_dc_ern(p, msg); + else if (cb_dc_ern) + cb_dc_ern(p, msg); + else { + static int warn_once; + + if (!warn_once) { + pr_crit("Leaking DCP ERNs!\n"); + warn_once = 1; + } + } + break; + default: + pr_crit("Invalid MR verb 0x%02x\n", verb); + } + } else { + /* Its a software ERN */ +#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP + fq = get_fq_table_entry(be32_to_cpu(msg->ern.tag)); +#else + fq = (void *)(uintptr_t)be32_to_cpu(msg->ern.tag); +#endif + fq->cb.ern(p, fq, &swapped_msg); + } + num++; + qm_mr_next(&p->p); + goto mr_loop; +mr_done: + qm_mr_cci_consume(&p->p, num); + } + /* + * QM_PIRQ_CSCI/CCSCI has already been cleared, as part of its specific + * processing. If that interrupt source has meanwhile been re-asserted, + * we mustn't clear it here (or in the top-level interrupt handler). + */ + return is & (QM_PIRQ_EQCI | QM_PIRQ_EQRI | QM_PIRQ_MRI); +} + +/* + * remove some slowish-path stuff from the "fast path" and make sure it isn't + * inlined. + */ +static noinline void clear_vdqcr(struct qman_portal *p, struct qman_fq *fq) +{ + p->vdqcr_owned = NULL; + FQLOCK(fq); + fq_clear(fq, QMAN_FQ_STATE_VDQCR); + FQUNLOCK(fq); + wake_up(&affine_queue); +} + +/* + * The only states that would conflict with other things if they ran at the + * same time on the same cpu are: + * + * (i) setting/clearing vdqcr_owned, and + * (ii) clearing the NE (Not Empty) flag. + * + * Both are safe. Because; + * + * (i) this clearing can only occur after qman_set_vdq() has set the + * vdqcr_owned field (which it does before setting VDQCR), and + * qman_volatile_dequeue() blocks interrupts and preemption while this is + * done so that we can't interfere. + * (ii) the NE flag is only cleared after qman_retire_fq() has set it, and as + * with (i) that API prevents us from interfering until it's safe. + * + * The good thing is that qman_set_vdq() and qman_retire_fq() run far + * less frequently (ie. per-FQ) than __poll_portal_fast() does, so the nett + * advantage comes from this function not having to "lock" anything at all. + * + * Note also that the callbacks are invoked at points which are safe against the + * above potential conflicts, but that this function itself is not re-entrant + * (this is because the function tracks one end of each FIFO in the portal and + * we do *not* want to lock that). So the consequence is that it is safe for + * user callbacks to call into any QMan API. + */ +static inline unsigned int __poll_portal_fast(struct qman_portal *p, + unsigned int poll_limit) +{ + const struct qm_dqrr_entry *dq; + struct qman_fq *fq; + enum qman_cb_dqrr_result res; + unsigned int limit = 0; +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + struct qm_dqrr_entry *shadow; +#endif + do { + qm_dqrr_pvb_update(&p->p); + dq = qm_dqrr_current(&p->p); + if (!dq) + break; +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + /* If running on an LE system the fields of the + * dequeue entry must be swapper. Because the + * QMan HW will ignore writes the DQRR entry is + * copied and the index stored within the copy + */ + shadow = &p->shadow_dqrr[DQRR_PTR2IDX(dq)]; + *shadow = *dq; + dq = shadow; + shadow->fqid = be32_to_cpu(shadow->fqid); + shadow->contextB = be32_to_cpu(shadow->contextB); + shadow->seqnum = be16_to_cpu(shadow->seqnum); + hw_fd_to_cpu(&shadow->fd); +#endif + + if (dq->stat & QM_DQRR_STAT_UNSCHEDULED) { + /* + * VDQCR: don't trust context_b as the FQ may have + * been configured for h/w consumption and we're + * draining it post-retirement. + */ + fq = p->vdqcr_owned; + /* + * We only set QMAN_FQ_STATE_NE when retiring, so we + * only need to check for clearing it when doing + * volatile dequeues. It's one less thing to check + * in the critical path (SDQCR). + */ + if (dq->stat & QM_DQRR_STAT_FQ_EMPTY) + fq_clear(fq, QMAN_FQ_STATE_NE); + /* + * This is duplicated from the SDQCR code, but we + * have stuff to do before *and* after this callback, + * and we don't want multiple if()s in the critical + * path (SDQCR). + */ + res = fq->cb.dqrr(p, fq, dq); + if (res == qman_cb_dqrr_stop) + break; + /* Check for VDQCR completion */ + if (dq->stat & QM_DQRR_STAT_DQCR_EXPIRED) + clear_vdqcr(p, fq); + } else { + /* SDQCR: context_b points to the FQ */ +#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP + fq = get_fq_table_entry(dq->contextB); +#else + fq = (void *)(uintptr_t)dq->contextB; +#endif + /* Now let the callback do its stuff */ + res = fq->cb.dqrr(p, fq, dq); + /* + * The callback can request that we exit without + * consuming this entry nor advancing; + */ + if (res == qman_cb_dqrr_stop) + break; + } + /* Interpret 'dq' from a driver perspective. */ + /* + * Parking isn't possible unless HELDACTIVE was set. NB, + * FORCEELIGIBLE implies HELDACTIVE, so we only need to + * check for HELDACTIVE to cover both. + */ + DPAA_ASSERT((dq->stat & QM_DQRR_STAT_FQ_HELDACTIVE) || + (res != qman_cb_dqrr_park)); + /* just means "skip it, I'll consume it myself later on" */ + if (res != qman_cb_dqrr_defer) + qm_dqrr_cdc_consume_1ptr(&p->p, dq, + res == qman_cb_dqrr_park); + /* Move forward */ + qm_dqrr_next(&p->p); + /* + * Entry processed and consumed, increment our counter. The + * callback can request that we exit after consuming the + * entry, and we also exit if we reach our processing limit, + * so loop back only if neither of these conditions is met. + */ + } while (++limit < poll_limit && res != qman_cb_dqrr_consume_stop); + + return limit; +} + +u16 qman_affine_channel(int cpu) +{ + if (cpu < 0) { + struct qman_portal *portal = get_affine_portal(); + + cpu = portal->config->cpu; + } + DPAA_BUG_ON(!CPU_ISSET(cpu, &affine_mask)); + return affine_channels[cpu]; +} + +struct qm_dqrr_entry *qman_dequeue(struct qman_fq *fq) +{ + struct qman_portal *p = get_affine_portal(); + const struct qm_dqrr_entry *dq; +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + struct qm_dqrr_entry *shadow; +#endif + + qm_dqrr_pvb_update(&p->p); + dq = qm_dqrr_current(&p->p); + if (!dq) + return NULL; + + if (!(dq->stat & QM_DQRR_STAT_FD_VALID)) { + /* Invalid DQRR - put the portal and consume the DQRR. + * Return NULL to user as no packet is seen. + */ + qman_dqrr_consume(fq, (struct qm_dqrr_entry *)dq); + return NULL; + } + +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + shadow = &p->shadow_dqrr[DQRR_PTR2IDX(dq)]; + *shadow = *dq; + dq = shadow; + shadow->fqid = be32_to_cpu(shadow->fqid); + shadow->contextB = be32_to_cpu(shadow->contextB); + shadow->seqnum = be16_to_cpu(shadow->seqnum); + hw_fd_to_cpu(&shadow->fd); +#endif + + if (dq->stat & QM_DQRR_STAT_FQ_EMPTY) + fq_clear(fq, QMAN_FQ_STATE_NE); + + return (struct qm_dqrr_entry *)dq; +} + +void qman_dqrr_consume(struct qman_fq *fq, + struct qm_dqrr_entry *dq) +{ + struct qman_portal *p = get_affine_portal(); + + if (dq->stat & QM_DQRR_STAT_DQCR_EXPIRED) + clear_vdqcr(p, fq); + + qm_dqrr_cdc_consume_1ptr(&p->p, dq, 0); + qm_dqrr_next(&p->p); +} + +int qman_poll_dqrr(unsigned int limit) +{ + struct qman_portal *p = get_affine_portal(); + int ret; + + ret = __poll_portal_fast(p, limit); + return ret; +} + +void qman_poll(void) +{ + struct qman_portal *p = get_affine_portal(); + + if ((~p->irq_sources) & QM_PIRQ_SLOW) { + if (!(p->slowpoll--)) { + u32 is = qm_isr_status_read(&p->p) & ~p->irq_sources; + u32 active = __poll_portal_slow(p, is); + + if (active) { + qm_isr_status_clear(&p->p, active); + p->slowpoll = SLOW_POLL_BUSY; + } else + p->slowpoll = SLOW_POLL_IDLE; + } + } + if ((~p->irq_sources) & QM_PIRQ_DQRI) + __poll_portal_fast(p, FSL_QMAN_POLL_LIMIT); +} + +void qman_stop_dequeues(void) +{ + struct qman_portal *p = get_affine_portal(); + + qman_stop_dequeues_ex(p); +} + +void qman_start_dequeues(void) +{ + struct qman_portal *p = get_affine_portal(); + + DPAA_ASSERT(p->dqrr_disable_ref > 0); + if (!(--p->dqrr_disable_ref)) + qm_dqrr_set_maxfill(&p->p, DQRR_MAXFILL); +} + +void qman_static_dequeue_add(u32 pools) +{ + struct qman_portal *p = get_affine_portal(); + + pools &= p->config->pools; + p->sdqcr |= pools; + qm_dqrr_sdqcr_set(&p->p, p->sdqcr); +} + +void qman_static_dequeue_del(u32 pools) +{ + struct qman_portal *p = get_affine_portal(); + + pools &= p->config->pools; + p->sdqcr &= ~pools; + qm_dqrr_sdqcr_set(&p->p, p->sdqcr); +} + +u32 qman_static_dequeue_get(void) +{ + struct qman_portal *p = get_affine_portal(); + return p->sdqcr; +} + +void qman_dca(struct qm_dqrr_entry *dq, int park_request) +{ + struct qman_portal *p = get_affine_portal(); + + qm_dqrr_cdc_consume_1ptr(&p->p, dq, park_request); +} + +/* Frame queue API */ +static const char *mcr_result_str(u8 result) +{ + switch (result) { + case QM_MCR_RESULT_NULL: + return "QM_MCR_RESULT_NULL"; + case QM_MCR_RESULT_OK: + return "QM_MCR_RESULT_OK"; + case QM_MCR_RESULT_ERR_FQID: + return "QM_MCR_RESULT_ERR_FQID"; + case QM_MCR_RESULT_ERR_FQSTATE: + return "QM_MCR_RESULT_ERR_FQSTATE"; + case QM_MCR_RESULT_ERR_NOTEMPTY: + return "QM_MCR_RESULT_ERR_NOTEMPTY"; + case QM_MCR_RESULT_PENDING: + return "QM_MCR_RESULT_PENDING"; + case QM_MCR_RESULT_ERR_BADCOMMAND: + return "QM_MCR_RESULT_ERR_BADCOMMAND"; + } + return ""; +} + +int qman_create_fq(u32 fqid, u32 flags, struct qman_fq *fq) +{ + struct qm_fqd fqd; + struct qm_mcr_queryfq_np np; + struct qm_mc_command *mcc; + struct qm_mc_result *mcr; + struct qman_portal *p; + + if (flags & QMAN_FQ_FLAG_DYNAMIC_FQID) { + int ret = qman_alloc_fqid(&fqid); + + if (ret) + return ret; + } + spin_lock_init(&fq->fqlock); + fq->fqid = fqid; + fq->flags = flags; + fq->state = qman_fq_state_oos; + fq->cgr_groupid = 0; +#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP + if (unlikely(find_empty_fq_table_entry(&fq->key, fq))) { + pr_info("Find empty table entry failed\n"); + return -ENOMEM; + } +#endif + if (!(flags & QMAN_FQ_FLAG_AS_IS) || (flags & QMAN_FQ_FLAG_NO_MODIFY)) + return 0; + /* Everything else is AS_IS support */ + p = get_affine_portal(); + mcc = qm_mc_start(&p->p); + mcc->queryfq.fqid = cpu_to_be32(fqid); + qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ); + while (!(mcr = qm_mc_result(&p->p))) + cpu_relax(); + DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYFQ); + if (mcr->result != QM_MCR_RESULT_OK) { + pr_err("QUERYFQ failed: %s\n", mcr_result_str(mcr->result)); + goto err; + } + fqd = mcr->queryfq.fqd; + hw_fqd_to_cpu(&fqd); + mcc = qm_mc_start(&p->p); + mcc->queryfq_np.fqid = cpu_to_be32(fqid); + qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP); + while (!(mcr = qm_mc_result(&p->p))) + cpu_relax(); + DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYFQ_NP); + if (mcr->result != QM_MCR_RESULT_OK) { + pr_err("QUERYFQ_NP failed: %s\n", mcr_result_str(mcr->result)); + goto err; + } + np = mcr->queryfq_np; + /* Phew, have queryfq and queryfq_np results, stitch together + * the FQ object from those. + */ + fq->cgr_groupid = fqd.cgid; + switch (np.state & QM_MCR_NP_STATE_MASK) { + case QM_MCR_NP_STATE_OOS: + break; + case QM_MCR_NP_STATE_RETIRED: + fq->state = qman_fq_state_retired; + if (np.frm_cnt) + fq_set(fq, QMAN_FQ_STATE_NE); + break; + case QM_MCR_NP_STATE_TEN_SCHED: + case QM_MCR_NP_STATE_TRU_SCHED: + case QM_MCR_NP_STATE_ACTIVE: + fq->state = qman_fq_state_sched; + if (np.state & QM_MCR_NP_STATE_R) + fq_set(fq, QMAN_FQ_STATE_CHANGING); + break; + case QM_MCR_NP_STATE_PARKED: + fq->state = qman_fq_state_parked; + break; + default: + DPAA_ASSERT(NULL == "invalid FQ state"); + } + if (fqd.fq_ctrl & QM_FQCTRL_CGE) + fq->state |= QMAN_FQ_STATE_CGR_EN; + return 0; +err: + if (flags & QMAN_FQ_FLAG_DYNAMIC_FQID) + qman_release_fqid(fqid); + return -EIO; +} + +void qman_destroy_fq(struct qman_fq *fq, u32 flags __maybe_unused) +{ + /* + * We don't need to lock the FQ as it is a pre-condition that the FQ be + * quiesced. Instead, run some checks. + */ + switch (fq->state) { + case qman_fq_state_parked: + DPAA_ASSERT(flags & QMAN_FQ_DESTROY_PARKED); + /* Fallthrough */ + case qman_fq_state_oos: + if (fq_isset(fq, QMAN_FQ_FLAG_DYNAMIC_FQID)) + qman_release_fqid(fq->fqid); +#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP + clear_fq_table_entry(fq->key); +#endif + return; + default: + break; + } + DPAA_ASSERT(NULL == "qman_free_fq() on unquiesced FQ!"); +} + +u32 qman_fq_fqid(struct qman_fq *fq) +{ + return fq->fqid; +} + +void qman_fq_state(struct qman_fq *fq, enum qman_fq_state *state, u32 *flags) +{ + if (state) + *state = fq->state; + if (flags) + *flags = fq->flags; +} + +int qman_init_fq(struct qman_fq *fq, u32 flags, struct qm_mcc_initfq *opts) +{ + struct qm_mc_command *mcc; + struct qm_mc_result *mcr; + struct qman_portal *p; + + u8 res, myverb = (flags & QMAN_INITFQ_FLAG_SCHED) ? + QM_MCC_VERB_INITFQ_SCHED : QM_MCC_VERB_INITFQ_PARKED; + + if ((fq->state != qman_fq_state_oos) && + (fq->state != qman_fq_state_parked)) + return -EINVAL; +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))) + return -EINVAL; +#endif + if (opts && (opts->we_mask & QM_INITFQ_WE_OAC)) { + /* And can't be set at the same time as TDTHRESH */ + if (opts->we_mask & QM_INITFQ_WE_TDTHRESH) + return -EINVAL; + } + /* Issue an INITFQ_[PARKED|SCHED] management command */ + p = get_affine_portal(); + FQLOCK(fq); + if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) || + ((fq->state != qman_fq_state_oos) && + (fq->state != qman_fq_state_parked)))) { + FQUNLOCK(fq); + return -EBUSY; + } + mcc = qm_mc_start(&p->p); + if (opts) + mcc->initfq = *opts; + mcc->initfq.fqid = cpu_to_be32(fq->fqid); + mcc->initfq.count = 0; + /* + * If the FQ does *not* have the TO_DCPORTAL flag, context_b is set as a + * demux pointer. Otherwise, the caller-provided value is allowed to + * stand, don't overwrite it. + */ + if (fq_isclear(fq, QMAN_FQ_FLAG_TO_DCPORTAL)) { + dma_addr_t phys_fq; + + mcc->initfq.we_mask |= QM_INITFQ_WE_CONTEXTB; +#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP + mcc->initfq.fqd.context_b = fq->key; +#else + mcc->initfq.fqd.context_b = (u32)(uintptr_t)fq; +#endif + /* + * and the physical address - NB, if the user wasn't trying to + * set CONTEXTA, clear the stashing settings. + */ + if (!(mcc->initfq.we_mask & QM_INITFQ_WE_CONTEXTA)) { + mcc->initfq.we_mask |= QM_INITFQ_WE_CONTEXTA; + memset(&mcc->initfq.fqd.context_a, 0, + sizeof(mcc->initfq.fqd.context_a)); + } else { + phys_fq = rte_mem_virt2iova(fq); + qm_fqd_stashing_set64(&mcc->initfq.fqd, phys_fq); + } + } + if (flags & QMAN_INITFQ_FLAG_LOCAL) { + mcc->initfq.fqd.dest.channel = p->config->channel; + if (!(mcc->initfq.we_mask & QM_INITFQ_WE_DESTWQ)) { + mcc->initfq.we_mask |= QM_INITFQ_WE_DESTWQ; + mcc->initfq.fqd.dest.wq = 4; + } + } + mcc->initfq.we_mask = cpu_to_be16(mcc->initfq.we_mask); + cpu_to_hw_fqd(&mcc->initfq.fqd); + qm_mc_commit(&p->p, myverb); + while (!(mcr = qm_mc_result(&p->p))) + cpu_relax(); + DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb); + res = mcr->result; + if (res != QM_MCR_RESULT_OK) { + FQUNLOCK(fq); + return -EIO; + } + if (opts) { + if (opts->we_mask & QM_INITFQ_WE_FQCTRL) { + if (opts->fqd.fq_ctrl & QM_FQCTRL_CGE) + fq_set(fq, QMAN_FQ_STATE_CGR_EN); + else + fq_clear(fq, QMAN_FQ_STATE_CGR_EN); + } + if (opts->we_mask & QM_INITFQ_WE_CGID) + fq->cgr_groupid = opts->fqd.cgid; + } + fq->state = (flags & QMAN_INITFQ_FLAG_SCHED) ? + qman_fq_state_sched : qman_fq_state_parked; + FQUNLOCK(fq); + return 0; +} + +int qman_schedule_fq(struct qman_fq *fq) +{ + struct qm_mc_command *mcc; + struct qm_mc_result *mcr; + struct qman_portal *p; + + int ret = 0; + u8 res; + + if (fq->state != qman_fq_state_parked) + return -EINVAL; +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))) + return -EINVAL; +#endif + /* Issue a ALTERFQ_SCHED management command */ + p = get_affine_portal(); + + FQLOCK(fq); + if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) || + (fq->state != qman_fq_state_parked))) { + ret = -EBUSY; + goto out; + } + mcc = qm_mc_start(&p->p); + mcc->alterfq.fqid = cpu_to_be32(fq->fqid); + qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_SCHED); + while (!(mcr = qm_mc_result(&p->p))) + cpu_relax(); + DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_SCHED); + res = mcr->result; + if (res != QM_MCR_RESULT_OK) { + ret = -EIO; + goto out; + } + fq->state = qman_fq_state_sched; +out: + FQUNLOCK(fq); + + return ret; +} + +int qman_retire_fq(struct qman_fq *fq, u32 *flags) +{ + struct qm_mc_command *mcc; + struct qm_mc_result *mcr; + struct qman_portal *p; + + int rval; + u8 res; + + if ((fq->state != qman_fq_state_parked) && + (fq->state != qman_fq_state_sched)) + return -EINVAL; +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))) + return -EINVAL; +#endif + p = get_affine_portal(); + + FQLOCK(fq); + if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) || + (fq->state == qman_fq_state_retired) || + (fq->state == qman_fq_state_oos))) { + rval = -EBUSY; + goto out; + } + rval = table_push_fq(p, fq); + if (rval) + goto out; + mcc = qm_mc_start(&p->p); + mcc->alterfq.fqid = cpu_to_be32(fq->fqid); + qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_RETIRE); + while (!(mcr = qm_mc_result(&p->p))) + cpu_relax(); + DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_RETIRE); + res = mcr->result; + /* + * "Elegant" would be to treat OK/PENDING the same way; set CHANGING, + * and defer the flags until FQRNI or FQRN (respectively) show up. But + * "Friendly" is to process OK immediately, and not set CHANGING. We do + * friendly, otherwise the caller doesn't necessarily have a fully + * "retired" FQ on return even if the retirement was immediate. However + * this does mean some code duplication between here and + * fq_state_change(). + */ + if (likely(res == QM_MCR_RESULT_OK)) { + rval = 0; + /* Process 'fq' right away, we'll ignore FQRNI */ + if (mcr->alterfq.fqs & QM_MCR_FQS_NOTEMPTY) + fq_set(fq, QMAN_FQ_STATE_NE); + if (mcr->alterfq.fqs & QM_MCR_FQS_ORLPRESENT) + fq_set(fq, QMAN_FQ_STATE_ORL); + else + table_del_fq(p, fq); + if (flags) + *flags = fq->flags; + fq->state = qman_fq_state_retired; + if (fq->cb.fqs) { + /* + * Another issue with supporting "immediate" retirement + * is that we're forced to drop FQRNIs, because by the + * time they're seen it may already be "too late" (the + * fq may have been OOS'd and free()'d already). But if + * the upper layer wants a callback whether it's + * immediate or not, we have to fake a "MR" entry to + * look like an FQRNI... + */ + struct qm_mr_entry msg; + + msg.verb = QM_MR_VERB_FQRNI; + msg.fq.fqs = mcr->alterfq.fqs; + msg.fq.fqid = fq->fqid; +#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP + msg.fq.contextB = fq->key; +#else + msg.fq.contextB = (u32)(uintptr_t)fq; +#endif + fq->cb.fqs(p, fq, &msg); + } + } else if (res == QM_MCR_RESULT_PENDING) { + rval = 1; + fq_set(fq, QMAN_FQ_STATE_CHANGING); + } else { + rval = -EIO; + table_del_fq(p, fq); + } +out: + FQUNLOCK(fq); + return rval; +} + +int qman_oos_fq(struct qman_fq *fq) +{ + struct qm_mc_command *mcc; + struct qm_mc_result *mcr; + struct qman_portal *p; + + int ret = 0; + u8 res; + + if (fq->state != qman_fq_state_retired) + return -EINVAL; +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))) + return -EINVAL; +#endif + p = get_affine_portal(); + FQLOCK(fq); + if (unlikely((fq_isset(fq, QMAN_FQ_STATE_BLOCKOOS)) || + (fq->state != qman_fq_state_retired))) { + ret = -EBUSY; + goto out; + } + mcc = qm_mc_start(&p->p); + mcc->alterfq.fqid = cpu_to_be32(fq->fqid); + qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_OOS); + while (!(mcr = qm_mc_result(&p->p))) + cpu_relax(); + DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_OOS); + res = mcr->result; + if (res != QM_MCR_RESULT_OK) { + ret = -EIO; + goto out; + } + fq->state = qman_fq_state_oos; +out: + FQUNLOCK(fq); + return ret; +} + +int qman_fq_flow_control(struct qman_fq *fq, int xon) +{ + struct qm_mc_command *mcc; + struct qm_mc_result *mcr; + struct qman_portal *p; + + int ret = 0; + u8 res; + u8 myverb; + + if ((fq->state == qman_fq_state_oos) || + (fq->state == qman_fq_state_retired) || + (fq->state == qman_fq_state_parked)) + return -EINVAL; + +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))) + return -EINVAL; +#endif + /* Issue a ALTER_FQXON or ALTER_FQXOFF management command */ + p = get_affine_portal(); + FQLOCK(fq); + if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) || + (fq->state == qman_fq_state_parked) || + (fq->state == qman_fq_state_oos) || + (fq->state == qman_fq_state_retired))) { + ret = -EBUSY; + goto out; + } + mcc = qm_mc_start(&p->p); + mcc->alterfq.fqid = fq->fqid; + mcc->alterfq.count = 0; + myverb = xon ? QM_MCC_VERB_ALTER_FQXON : QM_MCC_VERB_ALTER_FQXOFF; + + qm_mc_commit(&p->p, myverb); + while (!(mcr = qm_mc_result(&p->p))) + cpu_relax(); + DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb); + + res = mcr->result; + if (res != QM_MCR_RESULT_OK) { + ret = -EIO; + goto out; + } +out: + FQUNLOCK(fq); + return ret; +} + +int qman_query_fq(struct qman_fq *fq, struct qm_fqd *fqd) +{ + struct qm_mc_command *mcc; + struct qm_mc_result *mcr; + struct qman_portal *p = get_affine_portal(); + + u8 res; + + mcc = qm_mc_start(&p->p); + mcc->queryfq.fqid = cpu_to_be32(fq->fqid); + qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ); + while (!(mcr = qm_mc_result(&p->p))) + cpu_relax(); + DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ); + res = mcr->result; + if (res == QM_MCR_RESULT_OK) + *fqd = mcr->queryfq.fqd; + hw_fqd_to_cpu(fqd); + if (res != QM_MCR_RESULT_OK) + return -EIO; + return 0; +} + +int qman_query_fq_has_pkts(struct qman_fq *fq) +{ + struct qm_mc_command *mcc; + struct qm_mc_result *mcr; + struct qman_portal *p = get_affine_portal(); + + int ret = 0; + u8 res; + + mcc = qm_mc_start(&p->p); + mcc->queryfq.fqid = cpu_to_be32(fq->fqid); + qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP); + while (!(mcr = qm_mc_result(&p->p))) + cpu_relax(); + res = mcr->result; + if (res == QM_MCR_RESULT_OK) + ret = !!mcr->queryfq_np.frm_cnt; + return ret; +} + +int qman_query_fq_np(struct qman_fq *fq, struct qm_mcr_queryfq_np *np) +{ + struct qm_mc_command *mcc; + struct qm_mc_result *mcr; + struct qman_portal *p = get_affine_portal(); + + u8 res; + + mcc = qm_mc_start(&p->p); + mcc->queryfq.fqid = cpu_to_be32(fq->fqid); + qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP); + while (!(mcr = qm_mc_result(&p->p))) + cpu_relax(); + DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ_NP); + res = mcr->result; + if (res == QM_MCR_RESULT_OK) { + *np = mcr->queryfq_np; + np->fqd_link = be24_to_cpu(np->fqd_link); + np->odp_seq = be16_to_cpu(np->odp_seq); + np->orp_nesn = be16_to_cpu(np->orp_nesn); + np->orp_ea_hseq = be16_to_cpu(np->orp_ea_hseq); + np->orp_ea_tseq = be16_to_cpu(np->orp_ea_tseq); + np->orp_ea_hptr = be24_to_cpu(np->orp_ea_hptr); + np->orp_ea_tptr = be24_to_cpu(np->orp_ea_tptr); + np->pfdr_hptr = be24_to_cpu(np->pfdr_hptr); + np->pfdr_tptr = be24_to_cpu(np->pfdr_tptr); + np->ics_surp = be16_to_cpu(np->ics_surp); + np->byte_cnt = be32_to_cpu(np->byte_cnt); + np->frm_cnt = be24_to_cpu(np->frm_cnt); + np->ra1_sfdr = be16_to_cpu(np->ra1_sfdr); + np->ra2_sfdr = be16_to_cpu(np->ra2_sfdr); + np->od1_sfdr = be16_to_cpu(np->od1_sfdr); + np->od2_sfdr = be16_to_cpu(np->od2_sfdr); + np->od3_sfdr = be16_to_cpu(np->od3_sfdr); + } + if (res == QM_MCR_RESULT_ERR_FQID) + return -ERANGE; + else if (res != QM_MCR_RESULT_OK) + return -EIO; + return 0; +} + +int qman_query_wq(u8 query_dedicated, struct qm_mcr_querywq *wq) +{ + struct qm_mc_command *mcc; + struct qm_mc_result *mcr; + struct qman_portal *p = get_affine_portal(); + + u8 res, myverb; + + myverb = (query_dedicated) ? QM_MCR_VERB_QUERYWQ_DEDICATED : + QM_MCR_VERB_QUERYWQ; + mcc = qm_mc_start(&p->p); + mcc->querywq.channel.id = cpu_to_be16(wq->channel.id); + qm_mc_commit(&p->p, myverb); + while (!(mcr = qm_mc_result(&p->p))) + cpu_relax(); + DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb); + res = mcr->result; + if (res == QM_MCR_RESULT_OK) { + int i, array_len; + + wq->channel.id = be16_to_cpu(mcr->querywq.channel.id); + array_len = ARRAY_SIZE(mcr->querywq.wq_len); + for (i = 0; i < array_len; i++) + wq->wq_len[i] = be32_to_cpu(mcr->querywq.wq_len[i]); + } + if (res != QM_MCR_RESULT_OK) { + pr_err("QUERYWQ failed: %s\n", mcr_result_str(res)); + return -EIO; + } + return 0; +} + +int qman_testwrite_cgr(struct qman_cgr *cgr, u64 i_bcnt, + struct qm_mcr_cgrtestwrite *result) +{ + struct qm_mc_command *mcc; + struct qm_mc_result *mcr; + struct qman_portal *p = get_affine_portal(); + + u8 res; + + mcc = qm_mc_start(&p->p); + mcc->cgrtestwrite.cgid = cgr->cgrid; + mcc->cgrtestwrite.i_bcnt_hi = (u8)(i_bcnt >> 32); + mcc->cgrtestwrite.i_bcnt_lo = (u32)i_bcnt; + qm_mc_commit(&p->p, QM_MCC_VERB_CGRTESTWRITE); + while (!(mcr = qm_mc_result(&p->p))) + cpu_relax(); + DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_CGRTESTWRITE); + res = mcr->result; + if (res == QM_MCR_RESULT_OK) + *result = mcr->cgrtestwrite; + if (res != QM_MCR_RESULT_OK) { + pr_err("CGR TEST WRITE failed: %s\n", mcr_result_str(res)); + return -EIO; + } + return 0; +} + +int qman_query_cgr(struct qman_cgr *cgr, struct qm_mcr_querycgr *cgrd) +{ + struct qm_mc_command *mcc; + struct qm_mc_result *mcr; + struct qman_portal *p = get_affine_portal(); + u8 res; + unsigned int i; + + mcc = qm_mc_start(&p->p); + mcc->querycgr.cgid = cgr->cgrid; + qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCGR); + while (!(mcr = qm_mc_result(&p->p))) + cpu_relax(); + DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYCGR); + res = mcr->result; + if (res == QM_MCR_RESULT_OK) + *cgrd = mcr->querycgr; + if (res != QM_MCR_RESULT_OK) { + pr_err("QUERY_CGR failed: %s\n", mcr_result_str(res)); + return -EIO; + } + cgrd->cgr.wr_parm_g.word = + be32_to_cpu(cgrd->cgr.wr_parm_g.word); + cgrd->cgr.wr_parm_y.word = + be32_to_cpu(cgrd->cgr.wr_parm_y.word); + cgrd->cgr.wr_parm_r.word = + be32_to_cpu(cgrd->cgr.wr_parm_r.word); + cgrd->cgr.cscn_targ = be32_to_cpu(cgrd->cgr.cscn_targ); + cgrd->cgr.__cs_thres = be16_to_cpu(cgrd->cgr.__cs_thres); + for (i = 0; i < ARRAY_SIZE(cgrd->cscn_targ_swp); i++) + cgrd->cscn_targ_swp[i] = + be32_to_cpu(cgrd->cscn_targ_swp[i]); + return 0; +} + +int qman_query_congestion(struct qm_mcr_querycongestion *congestion) +{ + struct qm_mc_result *mcr; + struct qman_portal *p = get_affine_portal(); + u8 res; + unsigned int i; + + qm_mc_start(&p->p); + qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCONGESTION); + while (!(mcr = qm_mc_result(&p->p))) + cpu_relax(); + DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == + QM_MCC_VERB_QUERYCONGESTION); + res = mcr->result; + if (res == QM_MCR_RESULT_OK) + *congestion = mcr->querycongestion; + if (res != QM_MCR_RESULT_OK) { + pr_err("QUERY_CONGESTION failed: %s\n", mcr_result_str(res)); + return -EIO; + } + for (i = 0; i < ARRAY_SIZE(congestion->state.state); i++) + congestion->state.state[i] = + be32_to_cpu(congestion->state.state[i]); + return 0; +} + +int qman_set_vdq(struct qman_fq *fq, u16 num) +{ + struct qman_portal *p = get_affine_portal(); + uint32_t vdqcr; + int ret = -EBUSY; + + vdqcr = QM_VDQCR_EXACT; + vdqcr |= QM_VDQCR_NUMFRAMES_SET(num); + + if ((fq->state != qman_fq_state_parked) && + (fq->state != qman_fq_state_retired)) { + ret = -EINVAL; + goto out; + } + if (fq_isset(fq, QMAN_FQ_STATE_VDQCR)) { + ret = -EBUSY; + goto out; + } + vdqcr = (vdqcr & ~QM_VDQCR_FQID_MASK) | fq->fqid; + + if (!p->vdqcr_owned) { + FQLOCK(fq); + if (fq_isset(fq, QMAN_FQ_STATE_VDQCR)) + goto escape; + fq_set(fq, QMAN_FQ_STATE_VDQCR); + FQUNLOCK(fq); + p->vdqcr_owned = fq; + ret = 0; + } +escape: + if (!ret) + qm_dqrr_vdqcr_set(&p->p, vdqcr); + +out: + return ret; +} + +int qman_volatile_dequeue(struct qman_fq *fq, u32 flags __maybe_unused, + u32 vdqcr) +{ + struct qman_portal *p; + int ret = -EBUSY; + + if ((fq->state != qman_fq_state_parked) && + (fq->state != qman_fq_state_retired)) + return -EINVAL; + if (vdqcr & QM_VDQCR_FQID_MASK) + return -EINVAL; + if (fq_isset(fq, QMAN_FQ_STATE_VDQCR)) + return -EBUSY; + vdqcr = (vdqcr & ~QM_VDQCR_FQID_MASK) | fq->fqid; + + p = get_affine_portal(); + + if (!p->vdqcr_owned) { + FQLOCK(fq); + if (fq_isset(fq, QMAN_FQ_STATE_VDQCR)) + goto escape; + fq_set(fq, QMAN_FQ_STATE_VDQCR); + FQUNLOCK(fq); + p->vdqcr_owned = fq; + ret = 0; + } +escape: + if (ret) + return ret; + + /* VDQCR is set */ + qm_dqrr_vdqcr_set(&p->p, vdqcr); + return 0; +} + +static noinline void update_eqcr_ci(struct qman_portal *p, u8 avail) +{ + if (avail) + qm_eqcr_cce_prefetch(&p->p); + else + qm_eqcr_cce_update(&p->p); +} + +int qman_eqcr_is_empty(void) +{ + struct qman_portal *p = get_affine_portal(); + u8 avail; + + update_eqcr_ci(p, 0); + avail = qm_eqcr_get_fill(&p->p); + return (avail == 0); +} + +void qman_set_dc_ern(qman_cb_dc_ern handler, int affine) +{ + if (affine) { + struct qman_portal *p = get_affine_portal(); + + p->cb_dc_ern = handler; + } else + cb_dc_ern = handler; +} + +static inline struct qm_eqcr_entry *try_p_eq_start(struct qman_portal *p, + struct qman_fq *fq, + const struct qm_fd *fd, + u32 flags) +{ + struct qm_eqcr_entry *eq; + u8 avail; + + if (p->use_eqcr_ci_stashing) { + /* + * The stashing case is easy, only update if we need to in + * order to try and liberate ring entries. + */ + eq = qm_eqcr_start_stash(&p->p); + } else { + /* + * The non-stashing case is harder, need to prefetch ahead of + * time. + */ + avail = qm_eqcr_get_avail(&p->p); + if (avail < 2) + update_eqcr_ci(p, avail); + eq = qm_eqcr_start_no_stash(&p->p); + } + + if (unlikely(!eq)) + return NULL; + + if (flags & QMAN_ENQUEUE_FLAG_DCA) + eq->dca = QM_EQCR_DCA_ENABLE | + ((flags & QMAN_ENQUEUE_FLAG_DCA_PARK) ? + QM_EQCR_DCA_PARK : 0) | + ((flags >> 8) & QM_EQCR_DCA_IDXMASK); + eq->fqid = cpu_to_be32(fq->fqid); +#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP + eq->tag = cpu_to_be32(fq->key); +#else + eq->tag = cpu_to_be32((u32)(uintptr_t)fq); +#endif + eq->fd = *fd; + cpu_to_hw_fd(&eq->fd); + return eq; +} + +int qman_enqueue(struct qman_fq *fq, const struct qm_fd *fd, u32 flags) +{ + struct qman_portal *p = get_affine_portal(); + struct qm_eqcr_entry *eq; + + eq = try_p_eq_start(p, fq, fd, flags); + if (!eq) + return -EBUSY; + /* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */ + qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_CMD_ENQUEUE | + (flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT))); + /* Factor the below out, it's used from qman_enqueue_orp() too */ + return 0; +} + +int qman_enqueue_multi(struct qman_fq *fq, + const struct qm_fd *fd, + int frames_to_send) +{ + struct qman_portal *p = get_affine_portal(); + struct qm_portal *portal = &p->p; + + register struct qm_eqcr *eqcr = &portal->eqcr; + struct qm_eqcr_entry *eq = eqcr->cursor, *prev_eq; + + u8 i, diff, old_ci, sent = 0; + + /* Update the available entries if no entry is free */ + if (!eqcr->available) { + old_ci = eqcr->ci; + eqcr->ci = qm_cl_in(EQCR_CI) & (QM_EQCR_SIZE - 1); + diff = qm_cyc_diff(QM_EQCR_SIZE, old_ci, eqcr->ci); + eqcr->available += diff; + if (!diff) + return 0; + } + + /* try to send as many frames as possible */ + while (eqcr->available && frames_to_send--) { + eq->fqid = cpu_to_be32(fq->fqid); +#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP + eq->tag = cpu_to_be32(fq->key); +#else + eq->tag = cpu_to_be32((u32)(uintptr_t)fq); +#endif + eq->fd.opaque_addr = fd->opaque_addr; + eq->fd.addr = cpu_to_be40(fd->addr); + eq->fd.status = cpu_to_be32(fd->status); + eq->fd.opaque = cpu_to_be32(fd->opaque); + + eq = (void *)((unsigned long)(eq + 1) & + (~(unsigned long)(QM_EQCR_SIZE << 6))); + eqcr->available--; + sent++; + fd++; + } + lwsync(); + + /* In order for flushes to complete faster, all lines are recorded in + * 32 bit word. + */ + eq = eqcr->cursor; + for (i = 0; i < sent; i++) { + eq->__dont_write_directly__verb = + QM_EQCR_VERB_CMD_ENQUEUE | eqcr->vbit; + prev_eq = eq; + eq = (void *)((unsigned long)(eq + 1) & + (~(unsigned long)(QM_EQCR_SIZE << 6))); + if (unlikely((prev_eq + 1) != eq)) + eqcr->vbit ^= QM_EQCR_VERB_VBIT; + } + + /* We need to flush all the lines but without load/store operations + * between them + */ + eq = eqcr->cursor; + for (i = 0; i < sent; i++) { + dcbf(eq); + eq = (void *)((unsigned long)(eq + 1) & + (~(unsigned long)(QM_EQCR_SIZE << 6))); + } + /* Update cursor for the next call */ + eqcr->cursor = eq; + return sent; +} + +int qman_enqueue_orp(struct qman_fq *fq, const struct qm_fd *fd, u32 flags, + struct qman_fq *orp, u16 orp_seqnum) +{ + struct qman_portal *p = get_affine_portal(); + struct qm_eqcr_entry *eq; + + eq = try_p_eq_start(p, fq, fd, flags); + if (!eq) + return -EBUSY; + /* Process ORP-specifics here */ + if (flags & QMAN_ENQUEUE_FLAG_NLIS) + orp_seqnum |= QM_EQCR_SEQNUM_NLIS; + else { + orp_seqnum &= ~QM_EQCR_SEQNUM_NLIS; + if (flags & QMAN_ENQUEUE_FLAG_NESN) + orp_seqnum |= QM_EQCR_SEQNUM_NESN; + else + /* No need to check 4 QMAN_ENQUEUE_FLAG_HOLE */ + orp_seqnum &= ~QM_EQCR_SEQNUM_NESN; + } + eq->seqnum = cpu_to_be16(orp_seqnum); + eq->orp = cpu_to_be32(orp->fqid); + /* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */ + qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_ORP | + ((flags & (QMAN_ENQUEUE_FLAG_HOLE | QMAN_ENQUEUE_FLAG_NESN)) ? + 0 : QM_EQCR_VERB_CMD_ENQUEUE) | + (flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT))); + + return 0; +} + +int qman_modify_cgr(struct qman_cgr *cgr, u32 flags, + struct qm_mcc_initcgr *opts) +{ + struct qm_mc_command *mcc; + struct qm_mc_result *mcr; + struct qman_portal *p = get_affine_portal(); + + u8 res; + u8 verb = QM_MCC_VERB_MODIFYCGR; + + mcc = qm_mc_start(&p->p); + if (opts) + mcc->initcgr = *opts; + mcc->initcgr.we_mask = cpu_to_be16(mcc->initcgr.we_mask); + mcc->initcgr.cgr.wr_parm_g.word = + cpu_to_be32(mcc->initcgr.cgr.wr_parm_g.word); + mcc->initcgr.cgr.wr_parm_y.word = + cpu_to_be32(mcc->initcgr.cgr.wr_parm_y.word); + mcc->initcgr.cgr.wr_parm_r.word = + cpu_to_be32(mcc->initcgr.cgr.wr_parm_r.word); + mcc->initcgr.cgr.cscn_targ = cpu_to_be32(mcc->initcgr.cgr.cscn_targ); + mcc->initcgr.cgr.__cs_thres = cpu_to_be16(mcc->initcgr.cgr.__cs_thres); + + mcc->initcgr.cgid = cgr->cgrid; + if (flags & QMAN_CGR_FLAG_USE_INIT) + verb = QM_MCC_VERB_INITCGR; + qm_mc_commit(&p->p, verb); + while (!(mcr = qm_mc_result(&p->p))) + cpu_relax(); + + DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == verb); + res = mcr->result; + return (res == QM_MCR_RESULT_OK) ? 0 : -EIO; +} + +#define TARG_MASK(n) (0x80000000 >> (n->config->channel - \ + QM_CHANNEL_SWPORTAL0)) +#define TARG_DCP_MASK(n) (0x80000000 >> (10 + n)) +#define PORTAL_IDX(n) (n->config->channel - QM_CHANNEL_SWPORTAL0) + +int qman_create_cgr(struct qman_cgr *cgr, u32 flags, + struct qm_mcc_initcgr *opts) +{ + struct qm_mcr_querycgr cgr_state; + struct qm_mcc_initcgr local_opts; + int ret; + struct qman_portal *p; + + /* We have to check that the provided CGRID is within the limits of the + * data-structures, for obvious reasons. However we'll let h/w take + * care of determining whether it's within the limits of what exists on + * the SoC. + */ + if (cgr->cgrid >= __CGR_NUM) + return -EINVAL; + + p = get_affine_portal(); + + memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr)); + cgr->chan = p->config->channel; + spin_lock(&p->cgr_lock); + + /* if no opts specified, just add it to the list */ + if (!opts) + goto add_list; + + ret = qman_query_cgr(cgr, &cgr_state); + if (ret) + goto release_lock; + if (opts) + local_opts = *opts; + if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) + local_opts.cgr.cscn_targ_upd_ctrl = + QM_CGR_TARG_UDP_CTRL_WRITE_BIT | PORTAL_IDX(p); + else + /* Overwrite TARG */ + local_opts.cgr.cscn_targ = cgr_state.cgr.cscn_targ | + TARG_MASK(p); + local_opts.we_mask |= QM_CGR_WE_CSCN_TARG; + + /* send init if flags indicate so */ + if (opts && (flags & QMAN_CGR_FLAG_USE_INIT)) + ret = qman_modify_cgr(cgr, QMAN_CGR_FLAG_USE_INIT, &local_opts); + else + ret = qman_modify_cgr(cgr, 0, &local_opts); + if (ret) + goto release_lock; +add_list: + list_add(&cgr->node, &p->cgr_cbs); + + /* Determine if newly added object requires its callback to be called */ + ret = qman_query_cgr(cgr, &cgr_state); + if (ret) { + /* we can't go back, so proceed and return success, but screen + * and wail to the log file. + */ + pr_crit("CGR HW state partially modified\n"); + ret = 0; + goto release_lock; + } + if (cgr->cb && cgr_state.cgr.cscn_en && qman_cgrs_get(&p->cgrs[1], + cgr->cgrid)) + cgr->cb(p, cgr, 1); +release_lock: + spin_unlock(&p->cgr_lock); + return ret; +} + +int qman_create_cgr_to_dcp(struct qman_cgr *cgr, u32 flags, u16 dcp_portal, + struct qm_mcc_initcgr *opts) +{ + struct qm_mcc_initcgr local_opts; + struct qm_mcr_querycgr cgr_state; + int ret; + + if ((qman_ip_rev & 0xFF00) < QMAN_REV30) { + pr_warn("QMan version doesn't support CSCN => DCP portal\n"); + return -EINVAL; + } + /* We have to check that the provided CGRID is within the limits of the + * data-structures, for obvious reasons. However we'll let h/w take + * care of determining whether it's within the limits of what exists on + * the SoC. + */ + if (cgr->cgrid >= __CGR_NUM) + return -EINVAL; + + ret = qman_query_cgr(cgr, &cgr_state); + if (ret) + return ret; + + memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr)); + if (opts) + local_opts = *opts; + + if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) + local_opts.cgr.cscn_targ_upd_ctrl = + QM_CGR_TARG_UDP_CTRL_WRITE_BIT | + QM_CGR_TARG_UDP_CTRL_DCP | dcp_portal; + else + local_opts.cgr.cscn_targ = cgr_state.cgr.cscn_targ | + TARG_DCP_MASK(dcp_portal); + local_opts.we_mask |= QM_CGR_WE_CSCN_TARG; + + /* send init if flags indicate so */ + if (opts && (flags & QMAN_CGR_FLAG_USE_INIT)) + ret = qman_modify_cgr(cgr, QMAN_CGR_FLAG_USE_INIT, + &local_opts); + else + ret = qman_modify_cgr(cgr, 0, &local_opts); + + return ret; +} + +int qman_delete_cgr(struct qman_cgr *cgr) +{ + struct qm_mcr_querycgr cgr_state; + struct qm_mcc_initcgr local_opts; + int ret = 0; + struct qman_cgr *i; + struct qman_portal *p = get_affine_portal(); + + if (cgr->chan != p->config->channel) { + pr_crit("Attempting to delete cgr from different portal than" + " it was create: create 0x%x, delete 0x%x\n", + cgr->chan, p->config->channel); + ret = -EINVAL; + goto put_portal; + } + memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr)); + spin_lock(&p->cgr_lock); + list_del(&cgr->node); + /* + * If there are no other CGR objects for this CGRID in the list, + * update CSCN_TARG accordingly + */ + list_for_each_entry(i, &p->cgr_cbs, node) + if ((i->cgrid == cgr->cgrid) && i->cb) + goto release_lock; + ret = qman_query_cgr(cgr, &cgr_state); + if (ret) { + /* add back to the list */ + list_add(&cgr->node, &p->cgr_cbs); + goto release_lock; + } + /* Overwrite TARG */ + local_opts.we_mask = QM_CGR_WE_CSCN_TARG; + if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) + local_opts.cgr.cscn_targ_upd_ctrl = PORTAL_IDX(p); + else + local_opts.cgr.cscn_targ = cgr_state.cgr.cscn_targ & + ~(TARG_MASK(p)); + ret = qman_modify_cgr(cgr, 0, &local_opts); + if (ret) + /* add back to the list */ + list_add(&cgr->node, &p->cgr_cbs); +release_lock: + spin_unlock(&p->cgr_lock); +put_portal: + return ret; +} + +int qman_shutdown_fq(u32 fqid) +{ + struct qman_portal *p; + struct qm_portal *low_p; + struct qm_mc_command *mcc; + struct qm_mc_result *mcr; + u8 state; + int orl_empty, fq_empty, drain = 0; + u32 result; + u32 channel, wq; + u16 dest_wq; + + p = get_affine_portal(); + low_p = &p->p; + + /* Determine the state of the FQID */ + mcc = qm_mc_start(low_p); + mcc->queryfq_np.fqid = cpu_to_be32(fqid); + qm_mc_commit(low_p, QM_MCC_VERB_QUERYFQ_NP); + while (!(mcr = qm_mc_result(low_p))) + cpu_relax(); + DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ_NP); + state = mcr->queryfq_np.state & QM_MCR_NP_STATE_MASK; + if (state == QM_MCR_NP_STATE_OOS) + return 0; /* Already OOS, no need to do anymore checks */ + + /* Query which channel the FQ is using */ + mcc = qm_mc_start(low_p); + mcc->queryfq.fqid = cpu_to_be32(fqid); + qm_mc_commit(low_p, QM_MCC_VERB_QUERYFQ); + while (!(mcr = qm_mc_result(low_p))) + cpu_relax(); + DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ); + + /* Need to store these since the MCR gets reused */ + dest_wq = be16_to_cpu(mcr->queryfq.fqd.dest_wq); + channel = dest_wq & 0x7; + wq = dest_wq >> 3; + + switch (state) { + case QM_MCR_NP_STATE_TEN_SCHED: + case QM_MCR_NP_STATE_TRU_SCHED: + case QM_MCR_NP_STATE_ACTIVE: + case QM_MCR_NP_STATE_PARKED: + orl_empty = 0; + mcc = qm_mc_start(low_p); + mcc->alterfq.fqid = cpu_to_be32(fqid); + qm_mc_commit(low_p, QM_MCC_VERB_ALTER_RETIRE); + while (!(mcr = qm_mc_result(low_p))) + cpu_relax(); + DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == + QM_MCR_VERB_ALTER_RETIRE); + result = mcr->result; /* Make a copy as we reuse MCR below */ + + if (result == QM_MCR_RESULT_PENDING) { + /* Need to wait for the FQRN in the message ring, which + * will only occur once the FQ has been drained. In + * order for the FQ to drain the portal needs to be set + * to dequeue from the channel the FQ is scheduled on + */ + const struct qm_mr_entry *msg; + const struct qm_dqrr_entry *dqrr = NULL; + int found_fqrn = 0; + __maybe_unused u16 dequeue_wq = 0; + + /* Flag that we need to drain FQ */ + drain = 1; + + if (channel >= qm_channel_pool1 && + channel < (u16)(qm_channel_pool1 + 15)) { + /* Pool channel, enable the bit in the portal */ + dequeue_wq = (channel - + qm_channel_pool1 + 1) << 4 | wq; + } else if (channel < qm_channel_pool1) { + /* Dedicated channel */ + dequeue_wq = wq; + } else { + pr_info("Cannot recover FQ 0x%x," + " it is scheduled on channel 0x%x", + fqid, channel); + return -EBUSY; + } + /* Set the sdqcr to drain this channel */ + if (channel < qm_channel_pool1) + qm_dqrr_sdqcr_set(low_p, + QM_SDQCR_TYPE_ACTIVE | + QM_SDQCR_CHANNELS_DEDICATED); + else + qm_dqrr_sdqcr_set(low_p, + QM_SDQCR_TYPE_ACTIVE | + QM_SDQCR_CHANNELS_POOL_CONV + (channel)); + while (!found_fqrn) { + /* Keep draining DQRR while checking the MR*/ + qm_dqrr_pvb_update(low_p); + dqrr = qm_dqrr_current(low_p); + while (dqrr) { + qm_dqrr_cdc_consume_1ptr( + low_p, dqrr, 0); + qm_dqrr_pvb_update(low_p); + qm_dqrr_next(low_p); + dqrr = qm_dqrr_current(low_p); + } + /* Process message ring too */ + qm_mr_pvb_update(low_p); + msg = qm_mr_current(low_p); + while (msg) { + if ((msg->verb & + QM_MR_VERB_TYPE_MASK) + == QM_MR_VERB_FQRN) + found_fqrn = 1; + qm_mr_next(low_p); + qm_mr_cci_consume_to_current(low_p); + qm_mr_pvb_update(low_p); + msg = qm_mr_current(low_p); + } + cpu_relax(); + } + } + if (result != QM_MCR_RESULT_OK && + result != QM_MCR_RESULT_PENDING) { + /* error */ + pr_err("qman_retire_fq failed on FQ 0x%x," + " result=0x%x\n", fqid, result); + return -1; + } + if (!(mcr->alterfq.fqs & QM_MCR_FQS_ORLPRESENT)) { + /* ORL had no entries, no need to wait until the + * ERNs come in. + */ + orl_empty = 1; + } + /* Retirement succeeded, check to see if FQ needs + * to be drained. + */ + if (drain || mcr->alterfq.fqs & QM_MCR_FQS_NOTEMPTY) { + /* FQ is Not Empty, drain using volatile DQ commands */ + fq_empty = 0; + do { + const struct qm_dqrr_entry *dqrr = NULL; + u32 vdqcr = fqid | QM_VDQCR_NUMFRAMES_SET(3); + + qm_dqrr_vdqcr_set(low_p, vdqcr); + + /* Wait for a dequeue to occur */ + while (dqrr == NULL) { + qm_dqrr_pvb_update(low_p); + dqrr = qm_dqrr_current(low_p); + if (!dqrr) + cpu_relax(); + } + /* Process the dequeues, making sure to + * empty the ring completely. + */ + while (dqrr) { + if (dqrr->fqid == fqid && + dqrr->stat & QM_DQRR_STAT_FQ_EMPTY) + fq_empty = 1; + qm_dqrr_cdc_consume_1ptr(low_p, + dqrr, 0); + qm_dqrr_pvb_update(low_p); + qm_dqrr_next(low_p); + dqrr = qm_dqrr_current(low_p); + } + } while (fq_empty == 0); + } + qm_dqrr_sdqcr_set(low_p, 0); + + /* Wait for the ORL to have been completely drained */ + while (orl_empty == 0) { + const struct qm_mr_entry *msg; + + qm_mr_pvb_update(low_p); + msg = qm_mr_current(low_p); + while (msg) { + if ((msg->verb & QM_MR_VERB_TYPE_MASK) == + QM_MR_VERB_FQRL) + orl_empty = 1; + qm_mr_next(low_p); + qm_mr_cci_consume_to_current(low_p); + qm_mr_pvb_update(low_p); + msg = qm_mr_current(low_p); + } + cpu_relax(); + } + mcc = qm_mc_start(low_p); + mcc->alterfq.fqid = cpu_to_be32(fqid); + qm_mc_commit(low_p, QM_MCC_VERB_ALTER_OOS); + while (!(mcr = qm_mc_result(low_p))) + cpu_relax(); + DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == + QM_MCR_VERB_ALTER_OOS); + if (mcr->result != QM_MCR_RESULT_OK) { + pr_err( + "OOS after drain Failed on FQID 0x%x, result 0x%x\n", + fqid, mcr->result); + return -1; + } + return 0; + + case QM_MCR_NP_STATE_RETIRED: + /* Send OOS Command */ + mcc = qm_mc_start(low_p); + mcc->alterfq.fqid = cpu_to_be32(fqid); + qm_mc_commit(low_p, QM_MCC_VERB_ALTER_OOS); + while (!(mcr = qm_mc_result(low_p))) + cpu_relax(); + DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == + QM_MCR_VERB_ALTER_OOS); + if (mcr->result) { + pr_err("OOS Failed on FQID 0x%x\n", fqid); + return -1; + } + return 0; + + } + return -1; +} diff --git a/dpdk/drivers/bus/dpaa/base/qbman/qman.h b/dpdk/drivers/bus/dpaa/base/qbman/qman.h new file mode 100644 index 00000000..283cd31f --- /dev/null +++ b/dpdk/drivers/bus/dpaa/base/qbman/qman.h @@ -0,0 +1,946 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2008-2016 Freescale Semiconductor Inc. + * Copyright 2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "qman_priv.h" + +/***************************/ +/* Portal register assists */ +/***************************/ +#define QM_REG_EQCR_PI_CINH 0x3000 +#define QM_REG_EQCR_CI_CINH 0x3040 +#define QM_REG_EQCR_ITR 0x3080 +#define QM_REG_DQRR_PI_CINH 0x3100 +#define QM_REG_DQRR_CI_CINH 0x3140 +#define QM_REG_DQRR_ITR 0x3180 +#define QM_REG_DQRR_DCAP 0x31C0 +#define QM_REG_DQRR_SDQCR 0x3200 +#define QM_REG_DQRR_VDQCR 0x3240 +#define QM_REG_DQRR_PDQCR 0x3280 +#define QM_REG_MR_PI_CINH 0x3300 +#define QM_REG_MR_CI_CINH 0x3340 +#define QM_REG_MR_ITR 0x3380 +#define QM_REG_CFG 0x3500 +#define QM_REG_ISR 0x3600 +#define QM_REG_IIR 0x36C0 +#define QM_REG_ITPR 0x3740 + +/* Cache-enabled register offsets */ +#define QM_CL_EQCR 0x0000 +#define QM_CL_DQRR 0x1000 +#define QM_CL_MR 0x2000 +#define QM_CL_EQCR_PI_CENA 0x3000 +#define QM_CL_EQCR_CI_CENA 0x3040 +#define QM_CL_DQRR_PI_CENA 0x3100 +#define QM_CL_DQRR_CI_CENA 0x3140 +#define QM_CL_MR_PI_CENA 0x3300 +#define QM_CL_MR_CI_CENA 0x3340 +#define QM_CL_CR 0x3800 +#define QM_CL_RR0 0x3900 +#define QM_CL_RR1 0x3940 + +/* BTW, the drivers (and h/w programming model) already obtain the required + * synchronisation for portal accesses via lwsync(), hwsync(), and + * data-dependencies. Use of barrier()s or other order-preserving primitives + * simply degrade performance. Hence the use of the __raw_*() interfaces, which + * simply ensure that the compiler treats the portal registers as volatile (ie. + * non-coherent). + */ + +/* Cache-inhibited register access. */ +#define __qm_in(qm, o) be32_to_cpu(__raw_readl((qm)->ci + (o))) +#define __qm_out(qm, o, val) __raw_writel((cpu_to_be32(val)), \ + (qm)->ci + (o)) +#define qm_in(reg) __qm_in(&portal->addr, QM_REG_##reg) +#define qm_out(reg, val) __qm_out(&portal->addr, QM_REG_##reg, val) + +/* Cache-enabled (index) register access */ +#define __qm_cl_touch_ro(qm, o) dcbt_ro((qm)->ce + (o)) +#define __qm_cl_touch_rw(qm, o) dcbt_rw((qm)->ce + (o)) +#define __qm_cl_in(qm, o) be32_to_cpu(__raw_readl((qm)->ce + (o))) +#define __qm_cl_out(qm, o, val) \ + do { \ + u32 *__tmpclout = (qm)->ce + (o); \ + __raw_writel(cpu_to_be32(val), __tmpclout); \ + dcbf(__tmpclout); \ + } while (0) +#define __qm_cl_invalidate(qm, o) dccivac((qm)->ce + (o)) +#define qm_cl_touch_ro(reg) __qm_cl_touch_ro(&portal->addr, QM_CL_##reg##_CENA) +#define qm_cl_touch_rw(reg) __qm_cl_touch_rw(&portal->addr, QM_CL_##reg##_CENA) +#define qm_cl_in(reg) __qm_cl_in(&portal->addr, QM_CL_##reg##_CENA) +#define qm_cl_out(reg, val) __qm_cl_out(&portal->addr, QM_CL_##reg##_CENA, val) +#define qm_cl_invalidate(reg)\ + __qm_cl_invalidate(&portal->addr, QM_CL_##reg##_CENA) + +/* Cache-enabled ring access */ +#define qm_cl(base, idx) ((void *)base + ((idx) << 6)) + +/* Cyclic helper for rings. FIXME: once we are able to do fine-grain perf + * analysis, look at using the "extra" bit in the ring index registers to avoid + * cyclic issues. + */ +static inline u8 qm_cyc_diff(u8 ringsize, u8 first, u8 last) +{ + /* 'first' is included, 'last' is excluded */ + if (first <= last) + return last - first; + return ringsize + last - first; +} + +/* Portal modes. + * Enum types; + * pmode == production mode + * cmode == consumption mode, + * dmode == h/w dequeue mode. + * Enum values use 3 letter codes. First letter matches the portal mode, + * remaining two letters indicate; + * ci == cache-inhibited portal register + * ce == cache-enabled portal register + * vb == in-band valid-bit (cache-enabled) + * dc == DCA (Discrete Consumption Acknowledgment), DQRR-only + * As for "enum qm_dqrr_dmode", it should be self-explanatory. + */ +enum qm_eqcr_pmode { /* matches QCSP_CFG::EPM */ + qm_eqcr_pci = 0, /* PI index, cache-inhibited */ + qm_eqcr_pce = 1, /* PI index, cache-enabled */ + qm_eqcr_pvb = 2 /* valid-bit */ +}; + +enum qm_dqrr_dmode { /* matches QCSP_CFG::DP */ + qm_dqrr_dpush = 0, /* SDQCR + VDQCR */ + qm_dqrr_dpull = 1 /* PDQCR */ +}; + +enum qm_dqrr_pmode { /* s/w-only */ + qm_dqrr_pci, /* reads DQRR_PI_CINH */ + qm_dqrr_pce, /* reads DQRR_PI_CENA */ + qm_dqrr_pvb /* reads valid-bit */ +}; + +enum qm_dqrr_cmode { /* matches QCSP_CFG::DCM */ + qm_dqrr_cci = 0, /* CI index, cache-inhibited */ + qm_dqrr_cce = 1, /* CI index, cache-enabled */ + qm_dqrr_cdc = 2 /* Discrete Consumption Acknowledgment */ +}; + +enum qm_mr_pmode { /* s/w-only */ + qm_mr_pci, /* reads MR_PI_CINH */ + qm_mr_pce, /* reads MR_PI_CENA */ + qm_mr_pvb /* reads valid-bit */ +}; + +enum qm_mr_cmode { /* matches QCSP_CFG::MM */ + qm_mr_cci = 0, /* CI index, cache-inhibited */ + qm_mr_cce = 1 /* CI index, cache-enabled */ +}; + +/* ------------------------- */ +/* --- Portal structures --- */ + +#define QM_EQCR_SIZE 8 +#define QM_DQRR_SIZE 16 +#define QM_MR_SIZE 8 + +struct qm_eqcr { + struct qm_eqcr_entry *ring, *cursor; + u8 ci, available, ithresh, vbit; +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + u32 busy; + enum qm_eqcr_pmode pmode; +#endif +}; + +struct qm_dqrr { + const struct qm_dqrr_entry *ring, *cursor; + u8 pi, ci, fill, ithresh, vbit; +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + enum qm_dqrr_dmode dmode; + enum qm_dqrr_pmode pmode; + enum qm_dqrr_cmode cmode; +#endif +}; + +struct qm_mr { + const struct qm_mr_entry *ring, *cursor; + u8 pi, ci, fill, ithresh, vbit; +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + enum qm_mr_pmode pmode; + enum qm_mr_cmode cmode; +#endif +}; + +struct qm_mc { + struct qm_mc_command *cr; + struct qm_mc_result *rr; + u8 rridx, vbit; +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + enum { + /* Can be _mc_start()ed */ + qman_mc_idle, + /* Can be _mc_commit()ed or _mc_abort()ed */ + qman_mc_user, + /* Can only be _mc_retry()ed */ + qman_mc_hw + } state; +#endif +}; + +#define QM_PORTAL_ALIGNMENT ____cacheline_aligned + +struct qm_addr { + void __iomem *ce; /* cache-enabled */ + void __iomem *ci; /* cache-inhibited */ +}; + +struct qm_portal { + struct qm_addr addr; + struct qm_eqcr eqcr; + struct qm_dqrr dqrr; + struct qm_mr mr; + struct qm_mc mc; +} QM_PORTAL_ALIGNMENT; + +/* Bit-wise logic to wrap a ring pointer by clearing the "carry bit" */ +#define EQCR_CARRYCLEAR(p) \ + (void *)((unsigned long)(p) & (~(unsigned long)(QM_EQCR_SIZE << 6))) + +extern dma_addr_t rte_mem_virt2iova(const void *addr); + +/* Bit-wise logic to convert a ring pointer to a ring index */ +static inline u8 EQCR_PTR2IDX(struct qm_eqcr_entry *e) +{ + return ((uintptr_t)e >> 6) & (QM_EQCR_SIZE - 1); +} + +/* Increment the 'cursor' ring pointer, taking 'vbit' into account */ +static inline void EQCR_INC(struct qm_eqcr *eqcr) +{ + /* NB: this is odd-looking, but experiments show that it generates fast + * code with essentially no branching overheads. We increment to the + * next EQCR pointer and handle overflow and 'vbit'. + */ + struct qm_eqcr_entry *partial = eqcr->cursor + 1; + + eqcr->cursor = EQCR_CARRYCLEAR(partial); + if (partial != eqcr->cursor) + eqcr->vbit ^= QM_EQCR_VERB_VBIT; +} + +static inline struct qm_eqcr_entry *qm_eqcr_start_no_stash(struct qm_portal + *portal) +{ + register struct qm_eqcr *eqcr = &portal->eqcr; + +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + DPAA_ASSERT(!eqcr->busy); +#endif + if (!eqcr->available) + return NULL; + +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + eqcr->busy = 1; +#endif + + return eqcr->cursor; +} + +static inline struct qm_eqcr_entry *qm_eqcr_start_stash(struct qm_portal + *portal) +{ + register struct qm_eqcr *eqcr = &portal->eqcr; + u8 diff, old_ci; + +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + DPAA_ASSERT(!eqcr->busy); +#endif + if (!eqcr->available) { + old_ci = eqcr->ci; + eqcr->ci = qm_cl_in(EQCR_CI) & (QM_EQCR_SIZE - 1); + diff = qm_cyc_diff(QM_EQCR_SIZE, old_ci, eqcr->ci); + eqcr->available += diff; + if (!diff) + return NULL; + } +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + eqcr->busy = 1; +#endif + return eqcr->cursor; +} + +static inline void qm_eqcr_abort(struct qm_portal *portal) +{ + __maybe_unused register struct qm_eqcr *eqcr = &portal->eqcr; + +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + DPAA_ASSERT(eqcr->busy); + eqcr->busy = 0; +#endif +} + +static inline struct qm_eqcr_entry *qm_eqcr_pend_and_next( + struct qm_portal *portal, u8 myverb) +{ + register struct qm_eqcr *eqcr = &portal->eqcr; + +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + DPAA_ASSERT(eqcr->busy); + DPAA_ASSERT(eqcr->pmode != qm_eqcr_pvb); +#endif + if (eqcr->available == 1) + return NULL; + eqcr->cursor->__dont_write_directly__verb = myverb | eqcr->vbit; + dcbf(eqcr->cursor); + EQCR_INC(eqcr); + eqcr->available--; + return eqcr->cursor; +} + +#define EQCR_COMMIT_CHECKS(eqcr) \ +do { \ + DPAA_ASSERT(eqcr->busy); \ + DPAA_ASSERT(eqcr->cursor->orp == (eqcr->cursor->orp & 0x00ffffff)); \ + DPAA_ASSERT(eqcr->cursor->fqid == (eqcr->cursor->fqid & 0x00ffffff)); \ +} while (0) + +static inline void qm_eqcr_pci_commit(struct qm_portal *portal, u8 myverb) +{ + register struct qm_eqcr *eqcr = &portal->eqcr; + +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + EQCR_COMMIT_CHECKS(eqcr); + DPAA_ASSERT(eqcr->pmode == qm_eqcr_pci); +#endif + eqcr->cursor->__dont_write_directly__verb = myverb | eqcr->vbit; + EQCR_INC(eqcr); + eqcr->available--; + dcbf(eqcr->cursor); + hwsync(); + qm_out(EQCR_PI_CINH, EQCR_PTR2IDX(eqcr->cursor)); +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + eqcr->busy = 0; +#endif +} + +static inline void qm_eqcr_pce_prefetch(struct qm_portal *portal) +{ + __maybe_unused register struct qm_eqcr *eqcr = &portal->eqcr; + +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + DPAA_ASSERT(eqcr->pmode == qm_eqcr_pce); +#endif + qm_cl_invalidate(EQCR_PI); + qm_cl_touch_rw(EQCR_PI); +} + +static inline void qm_eqcr_pce_commit(struct qm_portal *portal, u8 myverb) +{ + register struct qm_eqcr *eqcr = &portal->eqcr; + +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + EQCR_COMMIT_CHECKS(eqcr); + DPAA_ASSERT(eqcr->pmode == qm_eqcr_pce); +#endif + eqcr->cursor->__dont_write_directly__verb = myverb | eqcr->vbit; + EQCR_INC(eqcr); + eqcr->available--; + dcbf(eqcr->cursor); + lwsync(); + qm_cl_out(EQCR_PI, EQCR_PTR2IDX(eqcr->cursor)); +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + eqcr->busy = 0; +#endif +} + +static inline void qm_eqcr_pvb_commit(struct qm_portal *portal, u8 myverb) +{ + register struct qm_eqcr *eqcr = &portal->eqcr; + struct qm_eqcr_entry *eqcursor; + +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + EQCR_COMMIT_CHECKS(eqcr); + DPAA_ASSERT(eqcr->pmode == qm_eqcr_pvb); +#endif + lwsync(); + eqcursor = eqcr->cursor; + eqcursor->__dont_write_directly__verb = myverb | eqcr->vbit; + dcbf(eqcursor); + EQCR_INC(eqcr); + eqcr->available--; +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + eqcr->busy = 0; +#endif +} + +static inline u8 qm_eqcr_cci_update(struct qm_portal *portal) +{ + register struct qm_eqcr *eqcr = &portal->eqcr; + u8 diff, old_ci = eqcr->ci; + + eqcr->ci = qm_in(EQCR_CI_CINH) & (QM_EQCR_SIZE - 1); + diff = qm_cyc_diff(QM_EQCR_SIZE, old_ci, eqcr->ci); + eqcr->available += diff; + return diff; +} + +static inline void qm_eqcr_cce_prefetch(struct qm_portal *portal) +{ + __maybe_unused register struct qm_eqcr *eqcr = &portal->eqcr; + + qm_cl_touch_ro(EQCR_CI); +} + +static inline u8 qm_eqcr_cce_update(struct qm_portal *portal) +{ + register struct qm_eqcr *eqcr = &portal->eqcr; + u8 diff, old_ci = eqcr->ci; + + eqcr->ci = qm_cl_in(EQCR_CI) & (QM_EQCR_SIZE - 1); + qm_cl_invalidate(EQCR_CI); + diff = qm_cyc_diff(QM_EQCR_SIZE, old_ci, eqcr->ci); + eqcr->available += diff; + return diff; +} + +static inline u8 qm_eqcr_get_ithresh(struct qm_portal *portal) +{ + register struct qm_eqcr *eqcr = &portal->eqcr; + + return eqcr->ithresh; +} + +static inline void qm_eqcr_set_ithresh(struct qm_portal *portal, u8 ithresh) +{ + register struct qm_eqcr *eqcr = &portal->eqcr; + + eqcr->ithresh = ithresh; + qm_out(EQCR_ITR, ithresh); +} + +static inline u8 qm_eqcr_get_avail(struct qm_portal *portal) +{ + register struct qm_eqcr *eqcr = &portal->eqcr; + + return eqcr->available; +} + +static inline u8 qm_eqcr_get_fill(struct qm_portal *portal) +{ + register struct qm_eqcr *eqcr = &portal->eqcr; + + return QM_EQCR_SIZE - 1 - eqcr->available; +} + +#define DQRR_CARRYCLEAR(p) \ + (void *)((unsigned long)(p) & (~(unsigned long)(QM_DQRR_SIZE << 6))) + +static inline u8 DQRR_PTR2IDX(const struct qm_dqrr_entry *e) +{ + return ((uintptr_t)e >> 6) & (QM_DQRR_SIZE - 1); +} + +static inline const struct qm_dqrr_entry *DQRR_INC( + const struct qm_dqrr_entry *e) +{ + return DQRR_CARRYCLEAR(e + 1); +} + +static inline void qm_dqrr_set_maxfill(struct qm_portal *portal, u8 mf) +{ + qm_out(CFG, (qm_in(CFG) & 0xff0fffff) | + ((mf & (QM_DQRR_SIZE - 1)) << 20)); +} + +static inline const struct qm_dqrr_entry *qm_dqrr_current( + struct qm_portal *portal) +{ + register struct qm_dqrr *dqrr = &portal->dqrr; + + if (!dqrr->fill) + return NULL; + return dqrr->cursor; +} + +static inline u8 qm_dqrr_cursor(struct qm_portal *portal) +{ + register struct qm_dqrr *dqrr = &portal->dqrr; + + return DQRR_PTR2IDX(dqrr->cursor); +} + +static inline u8 qm_dqrr_next(struct qm_portal *portal) +{ + register struct qm_dqrr *dqrr = &portal->dqrr; + + DPAA_ASSERT(dqrr->fill); + dqrr->cursor = DQRR_INC(dqrr->cursor); + return --dqrr->fill; +} + +static inline u8 qm_dqrr_pci_update(struct qm_portal *portal) +{ + register struct qm_dqrr *dqrr = &portal->dqrr; + u8 diff, old_pi = dqrr->pi; + +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + DPAA_ASSERT(dqrr->pmode == qm_dqrr_pci); +#endif + dqrr->pi = qm_in(DQRR_PI_CINH) & (QM_DQRR_SIZE - 1); + diff = qm_cyc_diff(QM_DQRR_SIZE, old_pi, dqrr->pi); + dqrr->fill += diff; + return diff; +} + +static inline void qm_dqrr_pce_prefetch(struct qm_portal *portal) +{ + __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr; + +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + DPAA_ASSERT(dqrr->pmode == qm_dqrr_pce); +#endif + qm_cl_invalidate(DQRR_PI); + qm_cl_touch_ro(DQRR_PI); +} + +static inline u8 qm_dqrr_pce_update(struct qm_portal *portal) +{ + register struct qm_dqrr *dqrr = &portal->dqrr; + u8 diff, old_pi = dqrr->pi; + +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + DPAA_ASSERT(dqrr->pmode == qm_dqrr_pce); +#endif + dqrr->pi = qm_cl_in(DQRR_PI) & (QM_DQRR_SIZE - 1); + diff = qm_cyc_diff(QM_DQRR_SIZE, old_pi, dqrr->pi); + dqrr->fill += diff; + return diff; +} + +static inline void qm_dqrr_pvb_update(struct qm_portal *portal) +{ + register struct qm_dqrr *dqrr = &portal->dqrr; + const struct qm_dqrr_entry *res = qm_cl(dqrr->ring, dqrr->pi); + +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + DPAA_ASSERT(dqrr->pmode == qm_dqrr_pvb); +#endif + /* when accessing 'verb', use __raw_readb() to ensure that compiler + * inlining doesn't try to optimise out "excess reads". + */ + if ((__raw_readb(&res->verb) & QM_DQRR_VERB_VBIT) == dqrr->vbit) { + dqrr->pi = (dqrr->pi + 1) & (QM_DQRR_SIZE - 1); + if (!dqrr->pi) + dqrr->vbit ^= QM_DQRR_VERB_VBIT; + dqrr->fill++; + } +} + +static inline void qm_dqrr_cci_consume(struct qm_portal *portal, u8 num) +{ + register struct qm_dqrr *dqrr = &portal->dqrr; + +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + DPAA_ASSERT(dqrr->cmode == qm_dqrr_cci); +#endif + dqrr->ci = (dqrr->ci + num) & (QM_DQRR_SIZE - 1); + qm_out(DQRR_CI_CINH, dqrr->ci); +} + +static inline void qm_dqrr_cci_consume_to_current(struct qm_portal *portal) +{ + register struct qm_dqrr *dqrr = &portal->dqrr; + +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + DPAA_ASSERT(dqrr->cmode == qm_dqrr_cci); +#endif + dqrr->ci = DQRR_PTR2IDX(dqrr->cursor); + qm_out(DQRR_CI_CINH, dqrr->ci); +} + +static inline void qm_dqrr_cce_prefetch(struct qm_portal *portal) +{ + __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr; + +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + DPAA_ASSERT(dqrr->cmode == qm_dqrr_cce); +#endif + qm_cl_invalidate(DQRR_CI); + qm_cl_touch_rw(DQRR_CI); +} + +static inline void qm_dqrr_cce_consume(struct qm_portal *portal, u8 num) +{ + register struct qm_dqrr *dqrr = &portal->dqrr; + +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + DPAA_ASSERT(dqrr->cmode == qm_dqrr_cce); +#endif + dqrr->ci = (dqrr->ci + num) & (QM_DQRR_SIZE - 1); + qm_cl_out(DQRR_CI, dqrr->ci); +} + +static inline void qm_dqrr_cce_consume_to_current(struct qm_portal *portal) +{ + register struct qm_dqrr *dqrr = &portal->dqrr; + +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + DPAA_ASSERT(dqrr->cmode == qm_dqrr_cce); +#endif + dqrr->ci = DQRR_PTR2IDX(dqrr->cursor); + qm_cl_out(DQRR_CI, dqrr->ci); +} + +static inline void qm_dqrr_cdc_consume_1(struct qm_portal *portal, u8 idx, + int park) +{ + __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr; + +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + DPAA_ASSERT(dqrr->cmode == qm_dqrr_cdc); +#endif + DPAA_ASSERT(idx < QM_DQRR_SIZE); + qm_out(DQRR_DCAP, (0 << 8) | /* S */ + ((park ? 1 : 0) << 6) | /* PK */ + idx); /* DCAP_CI */ +} + +static inline void qm_dqrr_cdc_consume_1ptr(struct qm_portal *portal, + const struct qm_dqrr_entry *dq, + int park) +{ + __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr; + u8 idx = DQRR_PTR2IDX(dq); + +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + DPAA_ASSERT(dqrr->cmode == qm_dqrr_cdc); +#endif + DPAA_ASSERT(idx < QM_DQRR_SIZE); + qm_out(DQRR_DCAP, (0 << 8) | /* DQRR_DCAP::S */ + ((park ? 1 : 0) << 6) | /* DQRR_DCAP::PK */ + idx); /* DQRR_DCAP::DCAP_CI */ +} + +static inline void qm_dqrr_cdc_consume_n(struct qm_portal *portal, u16 bitmask) +{ + __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr; + +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + DPAA_ASSERT(dqrr->cmode == qm_dqrr_cdc); +#endif + qm_out(DQRR_DCAP, (1 << 8) | /* DQRR_DCAP::S */ + ((u32)bitmask << 16)); /* DQRR_DCAP::DCAP_CI */ + dqrr->ci = qm_in(DQRR_CI_CINH) & (QM_DQRR_SIZE - 1); + dqrr->fill = qm_cyc_diff(QM_DQRR_SIZE, dqrr->ci, dqrr->pi); +} + +static inline u8 qm_dqrr_cdc_cci(struct qm_portal *portal) +{ + __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr; + +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + DPAA_ASSERT(dqrr->cmode == qm_dqrr_cdc); +#endif + return qm_in(DQRR_CI_CINH) & (QM_DQRR_SIZE - 1); +} + +static inline void qm_dqrr_cdc_cce_prefetch(struct qm_portal *portal) +{ + __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr; + +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + DPAA_ASSERT(dqrr->cmode == qm_dqrr_cdc); +#endif + qm_cl_invalidate(DQRR_CI); + qm_cl_touch_ro(DQRR_CI); +} + +static inline u8 qm_dqrr_cdc_cce(struct qm_portal *portal) +{ + __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr; + +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + DPAA_ASSERT(dqrr->cmode == qm_dqrr_cdc); +#endif + return qm_cl_in(DQRR_CI) & (QM_DQRR_SIZE - 1); +} + +static inline u8 qm_dqrr_get_ci(struct qm_portal *portal) +{ + register struct qm_dqrr *dqrr = &portal->dqrr; + +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + DPAA_ASSERT(dqrr->cmode != qm_dqrr_cdc); +#endif + return dqrr->ci; +} + +static inline void qm_dqrr_park(struct qm_portal *portal, u8 idx) +{ + __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr; + +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + DPAA_ASSERT(dqrr->cmode != qm_dqrr_cdc); +#endif + qm_out(DQRR_DCAP, (0 << 8) | /* S */ + (1 << 6) | /* PK */ + (idx & (QM_DQRR_SIZE - 1))); /* DCAP_CI */ +} + +static inline void qm_dqrr_park_current(struct qm_portal *portal) +{ + register struct qm_dqrr *dqrr = &portal->dqrr; + +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + DPAA_ASSERT(dqrr->cmode != qm_dqrr_cdc); +#endif + qm_out(DQRR_DCAP, (0 << 8) | /* S */ + (1 << 6) | /* PK */ + DQRR_PTR2IDX(dqrr->cursor)); /* DCAP_CI */ +} + +static inline void qm_dqrr_sdqcr_set(struct qm_portal *portal, u32 sdqcr) +{ + qm_out(DQRR_SDQCR, sdqcr); +} + +static inline u32 qm_dqrr_sdqcr_get(struct qm_portal *portal) +{ + return qm_in(DQRR_SDQCR); +} + +static inline void qm_dqrr_vdqcr_set(struct qm_portal *portal, u32 vdqcr) +{ + qm_out(DQRR_VDQCR, vdqcr); +} + +static inline u32 qm_dqrr_vdqcr_get(struct qm_portal *portal) +{ + return qm_in(DQRR_VDQCR); +} + +static inline u8 qm_dqrr_get_ithresh(struct qm_portal *portal) +{ + register struct qm_dqrr *dqrr = &portal->dqrr; + + return dqrr->ithresh; +} + +static inline void qm_dqrr_set_ithresh(struct qm_portal *portal, u8 ithresh) +{ + qm_out(DQRR_ITR, ithresh); +} + +static inline u8 qm_dqrr_get_maxfill(struct qm_portal *portal) +{ + return (qm_in(CFG) & 0x00f00000) >> 20; +} + +/* -------------- */ +/* --- MR API --- */ + +#define MR_CARRYCLEAR(p) \ + (void *)((unsigned long)(p) & (~(unsigned long)(QM_MR_SIZE << 6))) + +static inline u8 MR_PTR2IDX(const struct qm_mr_entry *e) +{ + return ((uintptr_t)e >> 6) & (QM_MR_SIZE - 1); +} + +static inline const struct qm_mr_entry *MR_INC(const struct qm_mr_entry *e) +{ + return MR_CARRYCLEAR(e + 1); +} + +static inline void qm_mr_finish(struct qm_portal *portal) +{ + register struct qm_mr *mr = &portal->mr; + + if (mr->ci != MR_PTR2IDX(mr->cursor)) + pr_crit("Ignoring completed MR entries\n"); +} + +static inline const struct qm_mr_entry *qm_mr_current(struct qm_portal *portal) +{ + register struct qm_mr *mr = &portal->mr; + + if (!mr->fill) + return NULL; + return mr->cursor; +} + +static inline u8 qm_mr_next(struct qm_portal *portal) +{ + register struct qm_mr *mr = &portal->mr; + + DPAA_ASSERT(mr->fill); + mr->cursor = MR_INC(mr->cursor); + return --mr->fill; +} + +static inline void qm_mr_cci_consume(struct qm_portal *portal, u8 num) +{ + register struct qm_mr *mr = &portal->mr; + +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + DPAA_ASSERT(mr->cmode == qm_mr_cci); +#endif + mr->ci = (mr->ci + num) & (QM_MR_SIZE - 1); + qm_out(MR_CI_CINH, mr->ci); +} + +static inline void qm_mr_cci_consume_to_current(struct qm_portal *portal) +{ + register struct qm_mr *mr = &portal->mr; + +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + DPAA_ASSERT(mr->cmode == qm_mr_cci); +#endif + mr->ci = MR_PTR2IDX(mr->cursor); + qm_out(MR_CI_CINH, mr->ci); +} + +static inline void qm_mr_set_ithresh(struct qm_portal *portal, u8 ithresh) +{ + qm_out(MR_ITR, ithresh); +} + +/* ------------------------------ */ +/* --- Management command API --- */ +static inline int qm_mc_init(struct qm_portal *portal) +{ + register struct qm_mc *mc = &portal->mc; + + mc->cr = portal->addr.ce + QM_CL_CR; + mc->rr = portal->addr.ce + QM_CL_RR0; + mc->rridx = (__raw_readb(&mc->cr->__dont_write_directly__verb) & + QM_MCC_VERB_VBIT) ? 0 : 1; + mc->vbit = mc->rridx ? QM_MCC_VERB_VBIT : 0; +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + mc->state = qman_mc_idle; +#endif + return 0; +} + +static inline void qm_mc_finish(struct qm_portal *portal) +{ + __maybe_unused register struct qm_mc *mc = &portal->mc; + +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + DPAA_ASSERT(mc->state == qman_mc_idle); + if (mc->state != qman_mc_idle) + pr_crit("Losing incomplete MC command\n"); +#endif +} + +static inline struct qm_mc_command *qm_mc_start(struct qm_portal *portal) +{ + register struct qm_mc *mc = &portal->mc; + +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + DPAA_ASSERT(mc->state == qman_mc_idle); + mc->state = qman_mc_user; +#endif + dcbz_64(mc->cr); + return mc->cr; +} + +static inline void qm_mc_commit(struct qm_portal *portal, u8 myverb) +{ + register struct qm_mc *mc = &portal->mc; + struct qm_mc_result *rr = mc->rr + mc->rridx; + +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + DPAA_ASSERT(mc->state == qman_mc_user); +#endif + lwsync(); + mc->cr->__dont_write_directly__verb = myverb | mc->vbit; + dcbf(mc->cr); + dcbit_ro(rr); +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + mc->state = qman_mc_hw; +#endif +} + +static inline struct qm_mc_result *qm_mc_result(struct qm_portal *portal) +{ + register struct qm_mc *mc = &portal->mc; + struct qm_mc_result *rr = mc->rr + mc->rridx; + +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + DPAA_ASSERT(mc->state == qman_mc_hw); +#endif + /* The inactive response register's verb byte always returns zero until + * its command is submitted and completed. This includes the valid-bit, + * in case you were wondering. + */ + if (!__raw_readb(&rr->verb)) { + dcbit_ro(rr); + return NULL; + } + mc->rridx ^= 1; + mc->vbit ^= QM_MCC_VERB_VBIT; +#ifdef RTE_LIBRTE_DPAA_HWDEBUG + mc->state = qman_mc_idle; +#endif + return rr; +} + +/* Portal interrupt register API */ +static inline void qm_isr_set_iperiod(struct qm_portal *portal, u16 iperiod) +{ + qm_out(ITPR, iperiod); +} + +static inline u32 __qm_isr_read(struct qm_portal *portal, enum qm_isr_reg n) +{ +#if defined(RTE_ARCH_ARM64) + return __qm_in(&portal->addr, QM_REG_ISR + (n << 6)); +#else + return __qm_in(&portal->addr, QM_REG_ISR + (n << 2)); +#endif +} + +static inline void __qm_isr_write(struct qm_portal *portal, enum qm_isr_reg n, + u32 val) +{ +#if defined(RTE_ARCH_ARM64) + __qm_out(&portal->addr, QM_REG_ISR + (n << 6), val); +#else + __qm_out(&portal->addr, QM_REG_ISR + (n << 2), val); +#endif +} diff --git a/dpdk/drivers/bus/dpaa/base/qbman/qman_driver.c b/dpdk/drivers/bus/dpaa/base/qbman/qman_driver.c new file mode 100644 index 00000000..7a688967 --- /dev/null +++ b/dpdk/drivers/bus/dpaa/base/qbman/qman_driver.c @@ -0,0 +1,288 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2008-2016 Freescale Semiconductor Inc. + * Copyright 2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include "qman_priv.h" +#include +#include + +/* Global variable containing revision id (even on non-control plane systems + * where CCSR isn't available). + */ +u16 qman_ip_rev; +u16 qm_channel_pool1 = QMAN_CHANNEL_POOL1; +u16 qm_channel_caam = QMAN_CHANNEL_CAAM; +u16 qm_channel_pme = QMAN_CHANNEL_PME; + +/* Ccsr map address to access ccsrbased register */ +void *qman_ccsr_map; +/* The qman clock frequency */ +u32 qman_clk; + +static __thread int fd = -1; +static __thread struct qm_portal_config pcfg; +static __thread struct dpaa_ioctl_portal_map map = { + .type = dpaa_portal_qman +}; + +static int fsl_qman_portal_init(uint32_t index, int is_shared) +{ + cpu_set_t cpuset; + struct qman_portal *portal; + int loop, ret; + struct dpaa_ioctl_irq_map irq_map; + + /* Verify the thread's cpu-affinity */ + ret = pthread_getaffinity_np(pthread_self(), sizeof(cpu_set_t), + &cpuset); + if (ret) { + error(0, ret, "pthread_getaffinity_np()"); + return ret; + } + pcfg.cpu = -1; + for (loop = 0; loop < CPU_SETSIZE; loop++) + if (CPU_ISSET(loop, &cpuset)) { + if (pcfg.cpu != -1) { + pr_err("Thread is not affine to 1 cpu\n"); + return -EINVAL; + } + pcfg.cpu = loop; + } + if (pcfg.cpu == -1) { + pr_err("Bug in getaffinity handling!\n"); + return -EINVAL; + } + + /* Allocate and map a qman portal */ + map.index = index; + ret = process_portal_map(&map); + if (ret) { + error(0, ret, "process_portal_map()"); + return ret; + } + pcfg.channel = map.channel; + pcfg.pools = map.pools; + pcfg.index = map.index; + + /* Make the portal's cache-[enabled|inhibited] regions */ + pcfg.addr_virt[DPAA_PORTAL_CE] = map.addr.cena; + pcfg.addr_virt[DPAA_PORTAL_CI] = map.addr.cinh; + + fd = open(QMAN_PORTAL_IRQ_PATH, O_RDONLY); + if (fd == -1) { + pr_err("QMan irq init failed\n"); + process_portal_unmap(&map.addr); + return -EBUSY; + } + + pcfg.is_shared = is_shared; + pcfg.node = NULL; + pcfg.irq = fd; + + portal = qman_create_affine_portal(&pcfg, NULL); + if (!portal) { + pr_err("Qman portal initialisation failed (%d)\n", + pcfg.cpu); + process_portal_unmap(&map.addr); + return -EBUSY; + } + + irq_map.type = dpaa_portal_qman; + irq_map.portal_cinh = map.addr.cinh; + process_portal_irq_map(fd, &irq_map); + return 0; +} + +static int fsl_qman_portal_finish(void) +{ + __maybe_unused const struct qm_portal_config *cfg; + int ret; + + process_portal_irq_unmap(fd); + + cfg = qman_destroy_affine_portal(); + DPAA_BUG_ON(cfg != &pcfg); + ret = process_portal_unmap(&map.addr); + if (ret) + error(0, ret, "process_portal_unmap()"); + return ret; +} + +int qman_thread_init(void) +{ + /* Convert from contiguous/virtual cpu numbering to real cpu when + * calling into the code that is dependent on the device naming. + */ + return fsl_qman_portal_init(QBMAN_ANY_PORTAL_IDX, 0); +} + +int qman_thread_finish(void) +{ + return fsl_qman_portal_finish(); +} + +void qman_thread_irq(void) +{ + qbman_invoke_irq(pcfg.irq); + + /* Now we need to uninhibit interrupts. This is the only code outside + * the regular portal driver that manipulates any portal register, so + * rather than breaking that encapsulation I am simply hard-coding the + * offset to the inhibit register here. + */ + out_be32(pcfg.addr_virt[DPAA_PORTAL_CI] + 0xe0c, 0); +} + +int qman_global_init(void) +{ + const struct device_node *dt_node; + int ret = 0; + size_t lenp; + const u32 *chanid; + static int ccsr_map_fd; + const uint32_t *qman_addr; + uint64_t phys_addr; + uint64_t regs_size; + const u32 *clk; + + static int done; + + if (done) + return -EBUSY; + + /* Use the device-tree to determine IP revision until something better + * is devised. + */ + dt_node = of_find_compatible_node(NULL, NULL, "fsl,qman-portal"); + if (!dt_node) { + pr_err("No qman portals available for any CPU\n"); + return -ENODEV; + } + if (of_device_is_compatible(dt_node, "fsl,qman-portal-1.0") || + of_device_is_compatible(dt_node, "fsl,qman-portal-1.0.0")) + pr_err("QMan rev1.0 on P4080 rev1 is not supported!\n"); + else if (of_device_is_compatible(dt_node, "fsl,qman-portal-1.1") || + of_device_is_compatible(dt_node, "fsl,qman-portal-1.1.0")) + qman_ip_rev = QMAN_REV11; + else if (of_device_is_compatible(dt_node, "fsl,qman-portal-1.2") || + of_device_is_compatible(dt_node, "fsl,qman-portal-1.2.0")) + qman_ip_rev = QMAN_REV12; + else if (of_device_is_compatible(dt_node, "fsl,qman-portal-2.0") || + of_device_is_compatible(dt_node, "fsl,qman-portal-2.0.0")) + qman_ip_rev = QMAN_REV20; + else if (of_device_is_compatible(dt_node, "fsl,qman-portal-3.0.0") || + of_device_is_compatible(dt_node, "fsl,qman-portal-3.0.1")) + qman_ip_rev = QMAN_REV30; + else if (of_device_is_compatible(dt_node, "fsl,qman-portal-3.1.0") || + of_device_is_compatible(dt_node, "fsl,qman-portal-3.1.1") || + of_device_is_compatible(dt_node, "fsl,qman-portal-3.1.2") || + of_device_is_compatible(dt_node, "fsl,qman-portal-3.1.3")) + qman_ip_rev = QMAN_REV31; + else if (of_device_is_compatible(dt_node, "fsl,qman-portal-3.2.0") || + of_device_is_compatible(dt_node, "fsl,qman-portal-3.2.1")) + qman_ip_rev = QMAN_REV32; + else + qman_ip_rev = QMAN_REV11; + + if (!qman_ip_rev) { + pr_err("Unknown qman portal version\n"); + return -ENODEV; + } + if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) { + qm_channel_pool1 = QMAN_CHANNEL_POOL1_REV3; + qm_channel_caam = QMAN_CHANNEL_CAAM_REV3; + qm_channel_pme = QMAN_CHANNEL_PME_REV3; + } + + dt_node = of_find_compatible_node(NULL, NULL, "fsl,pool-channel-range"); + if (!dt_node) { + pr_err("No qman pool channel range available\n"); + return -ENODEV; + } + chanid = of_get_property(dt_node, "fsl,pool-channel-range", &lenp); + if (!chanid) { + pr_err("Can not get pool-channel-range property\n"); + return -EINVAL; + } + + /* get ccsr base */ + dt_node = of_find_compatible_node(NULL, NULL, "fsl,qman"); + if (!dt_node) { + pr_err("No qman device node available\n"); + return -ENODEV; + } + qman_addr = of_get_address(dt_node, 0, ®s_size, NULL); + if (!qman_addr) { + pr_err("of_get_address cannot return qman address\n"); + return -EINVAL; + } + phys_addr = of_translate_address(dt_node, qman_addr); + if (!phys_addr) { + pr_err("of_translate_address failed\n"); + return -EINVAL; + } + + ccsr_map_fd = open("/dev/mem", O_RDWR); + if (unlikely(ccsr_map_fd < 0)) { + pr_err("Can not open /dev/mem for qman ccsr map\n"); + return ccsr_map_fd; + } + + qman_ccsr_map = mmap(NULL, regs_size, PROT_READ | PROT_WRITE, + MAP_SHARED, ccsr_map_fd, phys_addr); + if (qman_ccsr_map == MAP_FAILED) { + pr_err("Can not map qman ccsr base\n"); + return -EINVAL; + } + + clk = of_get_property(dt_node, "clock-frequency", NULL); + if (!clk) + pr_warn("Can't find Qman clock frequency\n"); + else + qman_clk = be32_to_cpu(*clk); + +#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP + ret = qman_setup_fq_lookup_table(CONFIG_FSL_QMAN_FQ_LOOKUP_MAX); + if (ret) + return ret; +#endif + return 0; +} diff --git a/dpdk/drivers/bus/dpaa/base/qbman/qman_priv.h b/dpdk/drivers/bus/dpaa/base/qbman/qman_priv.h new file mode 100644 index 00000000..3e1d7f92 --- /dev/null +++ b/dpdk/drivers/bus/dpaa/base/qbman/qman_priv.h @@ -0,0 +1,310 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2008-2016 Freescale Semiconductor Inc. + * Copyright 2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __QMAN_PRIV_H +#define __QMAN_PRIV_H + +#include "dpaa_sys.h" +#include + +/* Congestion Groups */ +/* + * This wrapper represents a bit-array for the state of the 256 QMan congestion + * groups. Is also used as a *mask* for congestion groups, eg. so we ignore + * those that don't concern us. We harness the structure and accessor details + * already used in the management command to query congestion groups. + */ +struct qman_cgrs { + struct __qm_mcr_querycongestion q; +}; + +static inline void qman_cgrs_init(struct qman_cgrs *c) +{ + memset(c, 0, sizeof(*c)); +} + +static inline void qman_cgrs_fill(struct qman_cgrs *c) +{ + memset(c, 0xff, sizeof(*c)); +} + +static inline int qman_cgrs_get(struct qman_cgrs *c, int num) +{ + return QM_MCR_QUERYCONGESTION(&c->q, num); +} + +static inline void qman_cgrs_set(struct qman_cgrs *c, int num) +{ + c->q.state[__CGR_WORD(num)] |= (0x80000000 >> __CGR_SHIFT(num)); +} + +static inline void qman_cgrs_unset(struct qman_cgrs *c, int num) +{ + c->q.state[__CGR_WORD(num)] &= ~(0x80000000 >> __CGR_SHIFT(num)); +} + +static inline int qman_cgrs_next(struct qman_cgrs *c, int num) +{ + while ((++num < (int)__CGR_NUM) && !qman_cgrs_get(c, num)) + ; + return num; +} + +static inline void qman_cgrs_cp(struct qman_cgrs *dest, + const struct qman_cgrs *src) +{ + memcpy(dest, src, sizeof(*dest)); +} + +static inline void qman_cgrs_and(struct qman_cgrs *dest, + const struct qman_cgrs *a, + const struct qman_cgrs *b) +{ + int ret; + u32 *_d = dest->q.state; + const u32 *_a = a->q.state; + const u32 *_b = b->q.state; + + for (ret = 0; ret < 8; ret++) + *(_d++) = *(_a++) & *(_b++); +} + +static inline void qman_cgrs_xor(struct qman_cgrs *dest, + const struct qman_cgrs *a, + const struct qman_cgrs *b) +{ + int ret; + u32 *_d = dest->q.state; + const u32 *_a = a->q.state; + const u32 *_b = b->q.state; + + for (ret = 0; ret < 8; ret++) + *(_d++) = *(_a++) ^ *(_b++); +} + +/* used by CCSR and portal interrupt code */ +enum qm_isr_reg { + qm_isr_status = 0, + qm_isr_enable = 1, + qm_isr_disable = 2, + qm_isr_inhibit = 3 +}; + +struct qm_portal_config { + /* + * Corenet portal addresses; + * [0]==cache-enabled, [1]==cache-inhibited. + */ + void __iomem *addr_virt[2]; + struct device_node *node; + /* Allow these to be joined in lists */ + struct list_head list; + /* User-visible portal configuration settings */ + /* If the caller enables DQRR stashing (and thus wishes to operate the + * portal from only one cpu), this is the logical CPU that the portal + * will stash to. Whether stashing is enabled or not, this setting is + * also used for any "core-affine" portals, ie. default portals + * associated to the corresponding cpu. -1 implies that there is no + * core affinity configured. + */ + int cpu; + /* portal interrupt line */ + int irq; + /* the unique index of this portal */ + u32 index; + /* Is this portal shared? (If so, it has coarser locking and demuxes + * processing on behalf of other CPUs.). + */ + int is_shared; + /* The portal's dedicated channel id, use this value for initialising + * frame queues to target this portal when scheduled. + */ + u16 channel; + /* A mask of which pool channels this portal has dequeue access to + * (using QM_SDQCR_CHANNELS_POOL(n) for the bitmask). + */ + u32 pools; + +}; + +/* Revision info (for errata and feature handling) */ +#define QMAN_REV11 0x0101 +#define QMAN_REV12 0x0102 +#define QMAN_REV20 0x0200 +#define QMAN_REV30 0x0300 +#define QMAN_REV31 0x0301 +#define QMAN_REV32 0x0302 +extern u16 qman_ip_rev; /* 0 if uninitialised, otherwise QMAN_REVx */ +extern u32 qman_clk; + +int qm_set_wpm(int wpm); +int qm_get_wpm(int *wpm); + +struct qman_portal *qman_create_affine_portal( + const struct qm_portal_config *config, + const struct qman_cgrs *cgrs); +const struct qm_portal_config *qman_destroy_affine_portal(void); + +struct qm_portal_config *qm_get_unused_portal(void); +struct qm_portal_config *qm_get_unused_portal_idx(uint32_t idx); + +void qm_put_unused_portal(struct qm_portal_config *pcfg); +void qm_set_liodns(struct qm_portal_config *pcfg); + +/* This CGR feature is supported by h/w and required by unit-tests and the + * debugfs hooks, so is implemented in the driver. However it allows an explicit + * corruption of h/w fields by s/w that are usually incorruptible (because the + * counters are usually maintained entirely within h/w). As such, we declare + * this API internally. + */ +int qman_testwrite_cgr(struct qman_cgr *cgr, u64 i_bcnt, + struct qm_mcr_cgrtestwrite *result); + +#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP +/* If the fq object pointer is greater than the size of context_b field, + * than a lookup table is required. + */ +int qman_setup_fq_lookup_table(size_t num_entries); +#endif + +/* QMan s/w corenet portal, low-level i/face */ + +/* + * For Choose one SOURCE. Choose one COUNT. Choose one + * dequeue TYPE. Choose TOKEN (8-bit). + * If SOURCE == CHANNELS, + * Choose CHANNELS_DEDICATED and/or CHANNELS_POOL(n). + * You can choose DEDICATED_PRECEDENCE if the portal channel should have + * priority. + * If SOURCE == SPECIFICWQ, + * Either select the work-queue ID with SPECIFICWQ_WQ(), or select the + * channel (SPECIFICWQ_DEDICATED or SPECIFICWQ_POOL()) and specify the + * work-queue priority (0-7) with SPECIFICWQ_WQ() - either way, you get the + * same value. + */ +#define QM_SDQCR_SOURCE_CHANNELS 0x0 +#define QM_SDQCR_SOURCE_SPECIFICWQ 0x40000000 +#define QM_SDQCR_COUNT_EXACT1 0x0 +#define QM_SDQCR_COUNT_UPTO3 0x20000000 +#define QM_SDQCR_DEDICATED_PRECEDENCE 0x10000000 +#define QM_SDQCR_TYPE_MASK 0x03000000 +#define QM_SDQCR_TYPE_NULL 0x0 +#define QM_SDQCR_TYPE_PRIO_QOS 0x01000000 +#define QM_SDQCR_TYPE_ACTIVE_QOS 0x02000000 +#define QM_SDQCR_TYPE_ACTIVE 0x03000000 +#define QM_SDQCR_TOKEN_MASK 0x00ff0000 +#define QM_SDQCR_TOKEN_SET(v) (((v) & 0xff) << 16) +#define QM_SDQCR_TOKEN_GET(v) (((v) >> 16) & 0xff) +#define QM_SDQCR_CHANNELS_DEDICATED 0x00008000 +#define QM_SDQCR_SPECIFICWQ_MASK 0x000000f7 +#define QM_SDQCR_SPECIFICWQ_DEDICATED 0x00000000 +#define QM_SDQCR_SPECIFICWQ_POOL(n) ((n) << 4) +#define QM_SDQCR_SPECIFICWQ_WQ(n) (n) + +#define QM_VDQCR_FQID_MASK 0x00ffffff +#define QM_VDQCR_FQID(n) ((n) & QM_VDQCR_FQID_MASK) + +#define QM_EQCR_VERB_VBIT 0x80 +#define QM_EQCR_VERB_CMD_MASK 0x61 /* but only one value; */ +#define QM_EQCR_VERB_CMD_ENQUEUE 0x01 +#define QM_EQCR_VERB_COLOUR_MASK 0x18 /* 4 possible values; */ +#define QM_EQCR_VERB_COLOUR_GREEN 0x00 +#define QM_EQCR_VERB_COLOUR_YELLOW 0x08 +#define QM_EQCR_VERB_COLOUR_RED 0x10 +#define QM_EQCR_VERB_COLOUR_OVERRIDE 0x18 +#define QM_EQCR_VERB_INTERRUPT 0x04 /* on command consumption */ +#define QM_EQCR_VERB_ORP 0x02 /* enable order restoration */ +#define QM_EQCR_DCA_ENABLE 0x80 +#define QM_EQCR_DCA_PARK 0x40 +#define QM_EQCR_DCA_IDXMASK 0x0f /* "DQRR::idx" goes here */ +#define QM_EQCR_SEQNUM_NESN 0x8000 /* Advance NESN */ +#define QM_EQCR_SEQNUM_NLIS 0x4000 /* More fragments to come */ +#define QM_EQCR_SEQNUM_SEQMASK 0x3fff /* sequence number goes here */ +#define QM_EQCR_FQID_NULL 0 /* eg. for an ORP seqnum hole */ + +#define QM_MCC_VERB_VBIT 0x80 +#define QM_MCC_VERB_MASK 0x7f /* where the verb contains; */ +#define QM_MCC_VERB_INITFQ_PARKED 0x40 +#define QM_MCC_VERB_INITFQ_SCHED 0x41 +#define QM_MCC_VERB_QUERYFQ 0x44 +#define QM_MCC_VERB_QUERYFQ_NP 0x45 /* "non-programmable" fields */ +#define QM_MCC_VERB_QUERYWQ 0x46 +#define QM_MCC_VERB_QUERYWQ_DEDICATED 0x47 +#define QM_MCC_VERB_ALTER_SCHED 0x48 /* Schedule FQ */ +#define QM_MCC_VERB_ALTER_FE 0x49 /* Force Eligible FQ */ +#define QM_MCC_VERB_ALTER_RETIRE 0x4a /* Retire FQ */ +#define QM_MCC_VERB_ALTER_OOS 0x4b /* Take FQ out of service */ +#define QM_MCC_VERB_ALTER_FQXON 0x4d /* FQ XON */ +#define QM_MCC_VERB_ALTER_FQXOFF 0x4e /* FQ XOFF */ +#define QM_MCC_VERB_INITCGR 0x50 +#define QM_MCC_VERB_MODIFYCGR 0x51 +#define QM_MCC_VERB_CGRTESTWRITE 0x52 +#define QM_MCC_VERB_QUERYCGR 0x58 +#define QM_MCC_VERB_QUERYCONGESTION 0x59 + +/* + * Used by all portal interrupt registers except 'inhibit' + * Channels with frame availability + */ +#define QM_PIRQ_DQAVAIL 0x0000ffff + +/* The DQAVAIL interrupt fields break down into these bits; */ +#define QM_DQAVAIL_PORTAL 0x8000 /* Portal channel */ +#define QM_DQAVAIL_POOL(n) (0x8000 >> (n)) /* Pool channel, n==[1..15] */ +#define QM_DQAVAIL_MASK 0xffff +/* This mask contains all the "irqsource" bits visible to API users */ +#define QM_PIRQ_VISIBLE (QM_PIRQ_SLOW | QM_PIRQ_DQRI) + +/* These are qm__(). So for example, qm_disable_write() means "write + * the disable register" rather than "disable the ability to write". + */ +#define qm_isr_status_read(qm) __qm_isr_read(qm, qm_isr_status) +#define qm_isr_status_clear(qm, m) __qm_isr_write(qm, qm_isr_status, m) +#define qm_isr_enable_read(qm) __qm_isr_read(qm, qm_isr_enable) +#define qm_isr_enable_write(qm, v) __qm_isr_write(qm, qm_isr_enable, v) +#define qm_isr_disable_read(qm) __qm_isr_read(qm, qm_isr_disable) +#define qm_isr_disable_write(qm, v) __qm_isr_write(qm, qm_isr_disable, v) +/* TODO: unfortunate name-clash here, reword? */ +#define qm_isr_inhibit(qm) __qm_isr_write(qm, qm_isr_inhibit, 1) +#define qm_isr_uninhibit(qm) __qm_isr_write(qm, qm_isr_inhibit, 0) + +#define QMAN_PORTAL_IRQ_PATH "/dev/fsl-usdpaa-irq" + +#endif /* _QMAN_PRIV_H */ diff --git a/dpdk/drivers/bus/dpaa/dpaa_bus.c b/dpdk/drivers/bus/dpaa/dpaa_bus.c new file mode 100644 index 00000000..3a0b2553 --- /dev/null +++ b/dpdk/drivers/bus/dpaa/dpaa_bus.c @@ -0,0 +1,529 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of NXP nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +/* System headers */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include + +int dpaa_logtype_bus; +int dpaa_logtype_mempool; +int dpaa_logtype_pmd; + +struct rte_dpaa_bus rte_dpaa_bus; +struct netcfg_info *dpaa_netcfg; + +/* define a variable to hold the portal_key, once created.*/ +pthread_key_t dpaa_portal_key; + +RTE_DEFINE_PER_LCORE(bool, _dpaa_io); + +static inline void +dpaa_add_to_device_list(struct rte_dpaa_device *dev) +{ + TAILQ_INSERT_TAIL(&rte_dpaa_bus.device_list, dev, next); +} + +static inline void +dpaa_remove_from_device_list(struct rte_dpaa_device *dev) +{ + TAILQ_INSERT_TAIL(&rte_dpaa_bus.device_list, dev, next); +} + +/* + * Reads the SEC device from DTS + * Returns -1 if SEC devices not available, 0 otherwise + */ +static inline int +dpaa_sec_available(void) +{ + const struct device_node *caam_node; + + for_each_compatible_node(caam_node, NULL, "fsl,sec-v4.0") { + return 0; + } + + return -1; +} + +static void dpaa_clean_device_list(void); + +static int +dpaa_create_device_list(void) +{ + int i; + int ret; + struct rte_dpaa_device *dev; + struct fm_eth_port_cfg *cfg; + struct fman_if *fman_intf; + + /* Creating Ethernet Devices */ + for (i = 0; i < dpaa_netcfg->num_ethports; i++) { + dev = calloc(1, sizeof(struct rte_dpaa_device)); + if (!dev) { + DPAA_BUS_LOG(ERR, "Failed to allocate ETH devices"); + ret = -ENOMEM; + goto cleanup; + } + + cfg = &dpaa_netcfg->port_cfg[i]; + fman_intf = cfg->fman_if; + + /* Device identifiers */ + dev->id.fman_id = fman_intf->fman_idx + 1; + dev->id.mac_id = fman_intf->mac_idx; + dev->device_type = FSL_DPAA_ETH; + dev->id.dev_id = i; + + /* Create device name */ + memset(dev->name, 0, RTE_ETH_NAME_MAX_LEN); + sprintf(dev->name, "fm%d-mac%d", (fman_intf->fman_idx + 1), + fman_intf->mac_idx); + DPAA_BUS_LOG(DEBUG, "Device added: %s", dev->name); + dev->device.name = dev->name; + + dpaa_add_to_device_list(dev); + } + + rte_dpaa_bus.device_count = i; + + /* Unlike case of ETH, RTE_LIBRTE_DPAA_MAX_CRYPTODEV SEC devices are + * constantly created only if "sec" property is found in the device + * tree. Logically there is no limit for number of devices (QI + * interfaces) that can be created. + */ + + if (dpaa_sec_available()) { + DPAA_BUS_LOG(INFO, "DPAA SEC devices are not available"); + return 0; + } + + /* Creating SEC Devices */ + for (i = 0; i < RTE_LIBRTE_DPAA_MAX_CRYPTODEV; i++) { + dev = calloc(1, sizeof(struct rte_dpaa_device)); + if (!dev) { + DPAA_BUS_LOG(ERR, "Failed to allocate SEC devices"); + ret = -1; + goto cleanup; + } + + dev->device_type = FSL_DPAA_CRYPTO; + dev->id.dev_id = rte_dpaa_bus.device_count + i; + + /* Even though RTE_CRYPTODEV_NAME_MAX_LEN is valid length of + * crypto PMD, using RTE_ETH_NAME_MAX_LEN as that is the size + * allocated for dev->name/ + */ + memset(dev->name, 0, RTE_ETH_NAME_MAX_LEN); + sprintf(dev->name, "dpaa-sec%d", i); + DPAA_BUS_LOG(DEBUG, "Device added: %s", dev->name); + + dpaa_add_to_device_list(dev); + } + + rte_dpaa_bus.device_count += i; + + return 0; + +cleanup: + dpaa_clean_device_list(); + return ret; +} + +static void +dpaa_clean_device_list(void) +{ + struct rte_dpaa_device *dev = NULL; + struct rte_dpaa_device *tdev = NULL; + + TAILQ_FOREACH_SAFE(dev, &rte_dpaa_bus.device_list, next, tdev) { + TAILQ_REMOVE(&rte_dpaa_bus.device_list, dev, next); + free(dev); + dev = NULL; + } +} + +/** XXX move this function into a separate file */ +static int +_dpaa_portal_init(void *arg) +{ + cpu_set_t cpuset; + pthread_t id; + uint32_t cpu = rte_lcore_id(); + int ret; + struct dpaa_portal *dpaa_io_portal; + + BUS_INIT_FUNC_TRACE(); + + if ((uint64_t)arg == 1 || cpu == LCORE_ID_ANY) + cpu = rte_get_master_lcore(); + /* if the core id is not supported */ + else + if (cpu >= RTE_MAX_LCORE) + return -1; + + /* Set CPU affinity for this thread */ + CPU_ZERO(&cpuset); + CPU_SET(cpu, &cpuset); + id = pthread_self(); + ret = pthread_setaffinity_np(id, sizeof(cpu_set_t), &cpuset); + if (ret) { + DPAA_BUS_LOG(ERR, "pthread_setaffinity_np failed on " + "core :%d with ret: %d", cpu, ret); + return ret; + } + + /* Initialise bman thread portals */ + ret = bman_thread_init(); + if (ret) { + DPAA_BUS_LOG(ERR, "bman_thread_init failed on " + "core %d with ret: %d", cpu, ret); + return ret; + } + + DPAA_BUS_LOG(DEBUG, "BMAN thread initialized"); + + /* Initialise qman thread portals */ + ret = qman_thread_init(); + if (ret) { + DPAA_BUS_LOG(ERR, "bman_thread_init failed on " + "core %d with ret: %d", cpu, ret); + bman_thread_finish(); + return ret; + } + + DPAA_BUS_LOG(DEBUG, "QMAN thread initialized"); + + dpaa_io_portal = rte_malloc(NULL, sizeof(struct dpaa_portal), + RTE_CACHE_LINE_SIZE); + if (!dpaa_io_portal) { + DPAA_BUS_LOG(ERR, "Unable to allocate memory"); + bman_thread_finish(); + qman_thread_finish(); + return -ENOMEM; + } + + dpaa_io_portal->qman_idx = qman_get_portal_index(); + dpaa_io_portal->bman_idx = bman_get_portal_index(); + dpaa_io_portal->tid = syscall(SYS_gettid); + + ret = pthread_setspecific(dpaa_portal_key, (void *)dpaa_io_portal); + if (ret) { + DPAA_BUS_LOG(ERR, "pthread_setspecific failed on " + "core %d with ret: %d", cpu, ret); + dpaa_portal_finish(NULL); + + return ret; + } + + RTE_PER_LCORE(_dpaa_io) = true; + + DPAA_BUS_LOG(DEBUG, "QMAN thread initialized"); + + return 0; +} + +/* + * rte_dpaa_portal_init - Wrapper over _dpaa_portal_init with thread level check + * XXX Complete this + */ +int +rte_dpaa_portal_init(void *arg) +{ + if (unlikely(!RTE_PER_LCORE(_dpaa_io))) + return _dpaa_portal_init(arg); + + return 0; +} + +void +dpaa_portal_finish(void *arg) +{ + struct dpaa_portal *dpaa_io_portal = (struct dpaa_portal *)arg; + + if (!dpaa_io_portal) { + DPAA_BUS_LOG(DEBUG, "Portal already cleaned"); + return; + } + + bman_thread_finish(); + qman_thread_finish(); + + pthread_setspecific(dpaa_portal_key, NULL); + + rte_free(dpaa_io_portal); + dpaa_io_portal = NULL; + + RTE_PER_LCORE(_dpaa_io) = false; +} + +#define DPAA_DEV_PATH1 "/sys/devices/platform/soc/soc:fsl,dpaa" +#define DPAA_DEV_PATH2 "/sys/devices/platform/fsl,dpaa" + +static int +rte_dpaa_bus_scan(void) +{ + int ret; + + BUS_INIT_FUNC_TRACE(); + + if ((access(DPAA_DEV_PATH1, F_OK) != 0) && + (access(DPAA_DEV_PATH2, F_OK) != 0)) { + RTE_LOG(DEBUG, EAL, "DPAA Bus not present. Skipping.\n"); + return 0; + } + + /* Load the device-tree driver */ + ret = of_init(); + if (ret) { + DPAA_BUS_LOG(ERR, "of_init failed with ret: %d", ret); + return -1; + } + + /* Get the interface configurations from device-tree */ + dpaa_netcfg = netcfg_acquire(); + if (!dpaa_netcfg) { + DPAA_BUS_LOG(ERR, "netcfg_acquire failed"); + return -EINVAL; + } + + RTE_LOG(NOTICE, EAL, "DPAA Bus Detected\n"); + + if (!dpaa_netcfg->num_ethports) { + DPAA_BUS_LOG(INFO, "no network interfaces available"); + /* This is not an error */ + return 0; + } + + DPAA_BUS_LOG(DEBUG, "Bus: Address of netcfg=%p, Ethports=%d", + dpaa_netcfg, dpaa_netcfg->num_ethports); + +#ifdef RTE_LIBRTE_DPAA_DEBUG_DRIVER + dump_netcfg(dpaa_netcfg); +#endif + + DPAA_BUS_LOG(DEBUG, "Number of devices = %d\n", + dpaa_netcfg->num_ethports); + ret = dpaa_create_device_list(); + if (ret) { + DPAA_BUS_LOG(ERR, "Unable to create device list. (%d)", ret); + return ret; + } + + /* create the key, supplying a function that'll be invoked + * when a portal affined thread will be deleted. + */ + ret = pthread_key_create(&dpaa_portal_key, dpaa_portal_finish); + if (ret) { + DPAA_BUS_LOG(DEBUG, "Unable to create pthread key. (%d)", ret); + dpaa_clean_device_list(); + return ret; + } + + DPAA_BUS_LOG(DEBUG, "dpaa_portal_key=%u, ret=%d\n", + (unsigned int)dpaa_portal_key, ret); + + return 0; +} + +/* register a dpaa bus based dpaa driver */ +void +rte_dpaa_driver_register(struct rte_dpaa_driver *driver) +{ + RTE_VERIFY(driver); + + BUS_INIT_FUNC_TRACE(); + + TAILQ_INSERT_TAIL(&rte_dpaa_bus.driver_list, driver, next); + /* Update Bus references */ + driver->dpaa_bus = &rte_dpaa_bus; +} + +/* un-register a dpaa bus based dpaa driver */ +void +rte_dpaa_driver_unregister(struct rte_dpaa_driver *driver) +{ + struct rte_dpaa_bus *dpaa_bus; + + BUS_INIT_FUNC_TRACE(); + + dpaa_bus = driver->dpaa_bus; + + TAILQ_REMOVE(&dpaa_bus->driver_list, driver, next); + /* Update Bus references */ + driver->dpaa_bus = NULL; +} + +static int +rte_dpaa_device_match(struct rte_dpaa_driver *drv, + struct rte_dpaa_device *dev) +{ + int ret = -1; + + BUS_INIT_FUNC_TRACE(); + + if (!drv || !dev) { + DPAA_BUS_DEBUG("Invalid drv or dev received."); + return ret; + } + + if (drv->drv_type == dev->device_type) { + DPAA_BUS_INFO("Device: %s matches for driver: %s", + dev->name, drv->driver.name); + ret = 0; /* Found a match */ + } + + return ret; +} + +static int +rte_dpaa_bus_probe(void) +{ + int ret = -1; + struct rte_dpaa_device *dev; + struct rte_dpaa_driver *drv; + + BUS_INIT_FUNC_TRACE(); + + /* For each registered driver, and device, call the driver->probe */ + TAILQ_FOREACH(dev, &rte_dpaa_bus.device_list, next) { + TAILQ_FOREACH(drv, &rte_dpaa_bus.driver_list, next) { + ret = rte_dpaa_device_match(drv, dev); + if (ret) + continue; + + if (!drv->probe) + continue; + + ret = drv->probe(drv, dev); + if (ret) + DPAA_BUS_ERR("Unable to probe.\n"); + break; + } + } + return 0; +} + +static struct rte_device * +rte_dpaa_find_device(const struct rte_device *start, rte_dev_cmp_t cmp, + const void *data) +{ + struct rte_dpaa_device *dev; + + TAILQ_FOREACH(dev, &rte_dpaa_bus.device_list, next) { + if (start && &dev->device == start) { + start = NULL; /* starting point found */ + continue; + } + + if (cmp(&dev->device, data) == 0) + return &dev->device; + } + + return NULL; +} + +/* + * Get iommu class of DPAA2 devices on the bus. + */ +static enum rte_iova_mode +rte_dpaa_get_iommu_class(void) +{ + if ((access(DPAA_DEV_PATH1, F_OK) != 0) && + (access(DPAA_DEV_PATH2, F_OK) != 0)) { + return RTE_IOVA_DC; + } + return RTE_IOVA_PA; +} + +struct rte_dpaa_bus rte_dpaa_bus = { + .bus = { + .scan = rte_dpaa_bus_scan, + .probe = rte_dpaa_bus_probe, + .find_device = rte_dpaa_find_device, + .get_iommu_class = rte_dpaa_get_iommu_class, + }, + .device_list = TAILQ_HEAD_INITIALIZER(rte_dpaa_bus.device_list), + .driver_list = TAILQ_HEAD_INITIALIZER(rte_dpaa_bus.driver_list), + .device_count = 0, +}; + +RTE_REGISTER_BUS(FSL_DPAA_BUS_NAME, rte_dpaa_bus.bus); + +RTE_INIT(dpaa_init_log); +static void +dpaa_init_log(void) +{ + dpaa_logtype_bus = rte_log_register("bus.dpaa"); + if (dpaa_logtype_bus >= 0) + rte_log_set_level(dpaa_logtype_bus, RTE_LOG_NOTICE); + + dpaa_logtype_mempool = rte_log_register("mempool.dpaa"); + if (dpaa_logtype_mempool >= 0) + rte_log_set_level(dpaa_logtype_mempool, RTE_LOG_NOTICE); + + dpaa_logtype_pmd = rte_log_register("pmd.dpaa"); + if (dpaa_logtype_pmd >= 0) + rte_log_set_level(dpaa_logtype_pmd, RTE_LOG_NOTICE); +} diff --git a/dpdk/drivers/bus/dpaa/include/compat.h b/dpdk/drivers/bus/dpaa/include/compat.h new file mode 100644 index 00000000..42733aeb --- /dev/null +++ b/dpdk/drivers/bus/dpaa/include/compat.h @@ -0,0 +1,385 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2011 Freescale Semiconductor, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __COMPAT_H +#define __COMPAT_H + +#include + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* The following definitions are primarily to allow the single-source driver + * interfaces to be included by arbitrary program code. Ie. for interfaces that + * are also available in kernel-space, these definitions provide compatibility + * with certain attributes and types used in those interfaces. + */ + +/* Required compiler attributes */ +#define __maybe_unused __rte_unused +#define __always_unused __rte_unused +#define __packed __rte_packed +#define noinline __attribute__((noinline)) + +#define L1_CACHE_BYTES 64 +#define ____cacheline_aligned __attribute__((aligned(L1_CACHE_BYTES))) +#define __stringify_1(x) #x +#define __stringify(x) __stringify_1(x) + +#ifdef ARRAY_SIZE +#undef ARRAY_SIZE +#endif +#define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0])) + +/* Debugging */ +#define prflush(fmt, args...) \ + do { \ + printf(fmt, ##args); \ + fflush(stdout); \ + } while (0) + +#define pr_crit(fmt, args...) prflush("CRIT:" fmt, ##args) +#define pr_err(fmt, args...) prflush("ERR:" fmt, ##args) +#define pr_warn(fmt, args...) prflush("WARN:" fmt, ##args) +#define pr_info(fmt, args...) prflush(fmt, ##args) + +#ifdef RTE_LIBRTE_DPAA_DEBUG_BUS +#ifdef pr_debug +#undef pr_debug +#endif +#define pr_debug(fmt, args...) printf(fmt, ##args) +#else +#define pr_debug(fmt, args...) {} +#endif + +#define DPAA_BUG_ON(x) RTE_ASSERT(x) + +/* Required types */ +typedef uint8_t u8; +typedef uint16_t u16; +typedef uint32_t u32; +typedef uint64_t u64; +typedef uint64_t dma_addr_t; +typedef cpu_set_t cpumask_t; +typedef uint32_t phandle; +typedef uint32_t gfp_t; +typedef uint32_t irqreturn_t; + +#define IRQ_HANDLED 0 +#define request_irq qbman_request_irq +#define free_irq qbman_free_irq + +#define __iomem +#define GFP_KERNEL 0 +#define __raw_readb(p) (*(const volatile unsigned char *)(p)) +#define __raw_readl(p) (*(const volatile unsigned int *)(p)) +#define __raw_writel(v, p) {*(volatile unsigned int *)(p) = (v); } + +/* to be used as an upper-limit only */ +#define NR_CPUS 64 + +/* Waitqueue stuff */ +typedef struct { } wait_queue_head_t; +#define DECLARE_WAIT_QUEUE_HEAD(x) int dummy_##x __always_unused +#define wake_up(x) do { } while (0) + +/* I/O operations */ +static inline u32 in_be32(volatile void *__p) +{ + volatile u32 *p = __p; + return rte_be_to_cpu_32(*p); +} + +static inline void out_be32(volatile void *__p, u32 val) +{ + volatile u32 *p = __p; + *p = rte_cpu_to_be_32(val); +} + +#define dcbt_ro(p) __builtin_prefetch(p, 0) +#define dcbt_rw(p) __builtin_prefetch(p, 1) + +#define dcbz(p) { asm volatile("dc zva, %0" : : "r" (p) : "memory"); } +#define dcbz_64(p) dcbz(p) +#define hwsync() rte_rmb() +#define lwsync() rte_wmb() +#define dcbf(p) { asm volatile("dc cvac, %0" : : "r"(p) : "memory"); } +#define dcbf_64(p) dcbf(p) +#define dccivac(p) { asm volatile("dc civac, %0" : : "r"(p) : "memory"); } + +#define dcbit_ro(p) \ + do { \ + dccivac(p); \ + asm volatile("prfm pldl1keep, [%0, #64]" : : "r" (p)); \ + } while (0) + +#define barrier() { asm volatile ("" : : : "memory"); } +#define cpu_relax barrier + +static inline uint64_t mfatb(void) +{ + uint64_t ret, ret_new, timeout = 200; + + asm volatile ("mrs %0, cntvct_el0" : "=r" (ret)); + asm volatile ("mrs %0, cntvct_el0" : "=r" (ret_new)); + while (ret != ret_new && timeout--) { + ret = ret_new; + asm volatile ("mrs %0, cntvct_el0" : "=r" (ret_new)); + } + DPAA_BUG_ON(!timeout && (ret != ret_new)); + return ret * 64; +} + +/* Spin for a few cycles without bothering the bus */ +static inline void cpu_spin(int cycles) +{ + uint64_t now = mfatb(); + + while (mfatb() < (now + cycles)) + ; +} + +/* Qman/Bman API inlines and macros; */ +#ifdef lower_32_bits +#undef lower_32_bits +#endif +#define lower_32_bits(x) ((u32)(x)) + +#ifdef upper_32_bits +#undef upper_32_bits +#endif +#define upper_32_bits(x) ((u32)(((x) >> 16) >> 16)) + +/* + * Swap bytes of a 48-bit value. + */ +static inline uint64_t +__bswap_48(uint64_t x) +{ + return ((x & 0x0000000000ffULL) << 40) | + ((x & 0x00000000ff00ULL) << 24) | + ((x & 0x000000ff0000ULL) << 8) | + ((x & 0x0000ff000000ULL) >> 8) | + ((x & 0x00ff00000000ULL) >> 24) | + ((x & 0xff0000000000ULL) >> 40); +} + +/* + * Swap bytes of a 40-bit value. + */ +static inline uint64_t +__bswap_40(uint64_t x) +{ + return ((x & 0x00000000ffULL) << 32) | + ((x & 0x000000ff00ULL) << 16) | + ((x & 0x0000ff0000ULL)) | + ((x & 0x00ff000000ULL) >> 16) | + ((x & 0xff00000000ULL) >> 32); +} + +/* + * Swap bytes of a 24-bit value. + */ +static inline uint32_t +__bswap_24(uint32_t x) +{ + return ((x & 0x0000ffULL) << 16) | + ((x & 0x00ff00ULL)) | + ((x & 0xff0000ULL) >> 16); +} + +#define be64_to_cpu(x) rte_be_to_cpu_64(x) +#define be32_to_cpu(x) rte_be_to_cpu_32(x) +#define be16_to_cpu(x) rte_be_to_cpu_16(x) + +#define cpu_to_be64(x) rte_cpu_to_be_64(x) +#define cpu_to_be32(x) rte_cpu_to_be_32(x) +#define cpu_to_be16(x) rte_cpu_to_be_16(x) + +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + +#define cpu_to_be48(x) __bswap_48(x) +#define be48_to_cpu(x) __bswap_48(x) + +#define cpu_to_be40(x) __bswap_40(x) +#define be40_to_cpu(x) __bswap_40(x) + +#define cpu_to_be24(x) __bswap_24(x) +#define be24_to_cpu(x) __bswap_24(x) + +#else /* RTE_BIG_ENDIAN */ + +#define cpu_to_be48(x) (x) +#define be48_to_cpu(x) (x) + +#define cpu_to_be40(x) (x) +#define be40_to_cpu(x) (x) + +#define cpu_to_be24(x) (x) +#define be24_to_cpu(x) (x) + +#endif /* RTE_BIG_ENDIAN */ + +/* When copying aligned words or shorts, try to avoid memcpy() */ +/* memcpy() stuff - when you know alignments in advance */ +#define CONFIG_TRY_BETTER_MEMCPY + +#ifdef CONFIG_TRY_BETTER_MEMCPY +static inline void copy_words(void *dest, const void *src, size_t sz) +{ + u32 *__dest = dest; + const u32 *__src = src; + size_t __sz = sz >> 2; + + DPAA_BUG_ON((unsigned long)dest & 0x3); + DPAA_BUG_ON((unsigned long)src & 0x3); + DPAA_BUG_ON(sz & 0x3); + while (__sz--) + *(__dest++) = *(__src++); +} + +static inline void copy_shorts(void *dest, const void *src, size_t sz) +{ + u16 *__dest = dest; + const u16 *__src = src; + size_t __sz = sz >> 1; + + DPAA_BUG_ON((unsigned long)dest & 0x1); + DPAA_BUG_ON((unsigned long)src & 0x1); + DPAA_BUG_ON(sz & 0x1); + while (__sz--) + *(__dest++) = *(__src++); +} + +static inline void copy_bytes(void *dest, const void *src, size_t sz) +{ + u8 *__dest = dest; + const u8 *__src = src; + + while (sz--) + *(__dest++) = *(__src++); +} +#else +#define copy_words memcpy +#define copy_shorts memcpy +#define copy_bytes memcpy +#endif + +/* Allocator stuff */ +#define kmalloc(sz, t) malloc(sz) +#define vmalloc(sz) malloc(sz) +#define kfree(p) { if (p) free(p); } +static inline void *kzalloc(size_t sz, gfp_t __foo __rte_unused) +{ + void *ptr = malloc(sz); + + if (ptr) + memset(ptr, 0, sz); + return ptr; +} + +static inline unsigned long get_zeroed_page(gfp_t __foo __rte_unused) +{ + void *p; + + if (posix_memalign(&p, 4096, 4096)) + return 0; + memset(p, 0, 4096); + return (unsigned long)p; +} + +/* Spinlock stuff */ +#define spinlock_t rte_spinlock_t +#define __SPIN_LOCK_UNLOCKED(x) RTE_SPINLOCK_INITIALIZER +#define DEFINE_SPINLOCK(x) spinlock_t x = __SPIN_LOCK_UNLOCKED(x) +#define spin_lock_init(x) rte_spinlock_init(x) +#define spin_lock_destroy(x) +#define spin_lock(x) rte_spinlock_lock(x) +#define spin_unlock(x) rte_spinlock_unlock(x) +#define spin_lock_irq(x) spin_lock(x) +#define spin_unlock_irq(x) spin_unlock(x) +#define spin_lock_irqsave(x, f) spin_lock_irq(x) +#define spin_unlock_irqrestore(x, f) spin_unlock_irq(x) + +#define atomic_t rte_atomic32_t +#define atomic_read(v) rte_atomic32_read(v) +#define atomic_set(v, i) rte_atomic32_set(v, i) + +#define atomic_inc(v) rte_atomic32_add(v, 1) +#define atomic_dec(v) rte_atomic32_sub(v, 1) + +#define atomic_inc_and_test(v) rte_atomic32_inc_and_test(v) +#define atomic_dec_and_test(v) rte_atomic32_dec_and_test(v) + +#define atomic_inc_return(v) rte_atomic32_add_return(v, 1) +#define atomic_dec_return(v) rte_atomic32_sub_return(v, 1) +#define atomic_sub_and_test(i, v) (rte_atomic32_sub_return(v, i) == 0) + +#include +#include + +#endif /* __COMPAT_H */ diff --git a/dpdk/drivers/bus/dpaa/include/dpaa_bits.h b/dpdk/drivers/bus/dpaa/include/dpaa_bits.h new file mode 100644 index 00000000..71f2d809 --- /dev/null +++ b/dpdk/drivers/bus/dpaa/include/dpaa_bits.h @@ -0,0 +1,65 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of NXP nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __DPAA_BITS_H +#define __DPAA_BITS_H + +/* Bitfield stuff. */ +#define BITS_PER_ULONG (sizeof(unsigned long) << 3) +#define SHIFT_PER_ULONG (((1 << 5) == BITS_PER_ULONG) ? 5 : 6) +#define BITS_MASK(idx) (1UL << ((idx) & (BITS_PER_ULONG - 1))) +#define BITS_IDX(idx) ((idx) >> SHIFT_PER_ULONG) + +static inline void dpaa_set_bits(unsigned long mask, + volatile unsigned long *p) +{ + *p |= mask; +} + +static inline void dpaa_set_bit(int idx, volatile unsigned long *bits) +{ + dpaa_set_bits(BITS_MASK(idx), bits + BITS_IDX(idx)); +} + +static inline void dpaa_clear_bits(unsigned long mask, + volatile unsigned long *p) +{ + *p &= ~mask; +} + +static inline void dpaa_clear_bit(int idx, + volatile unsigned long *bits) +{ + dpaa_clear_bits(BITS_MASK(idx), bits + BITS_IDX(idx)); +} + +#endif /* __DPAA_BITS_H */ diff --git a/dpdk/drivers/bus/dpaa/include/dpaa_list.h b/dpdk/drivers/bus/dpaa/include/dpaa_list.h new file mode 100644 index 00000000..871e6121 --- /dev/null +++ b/dpdk/drivers/bus/dpaa/include/dpaa_list.h @@ -0,0 +1,101 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of NXP nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __DPAA_LIST_H +#define __DPAA_LIST_H + +/****************/ +/* Linked-lists */ +/****************/ + +struct list_head { + struct list_head *prev; + struct list_head *next; +}; + +#define COMPAT_LIST_HEAD(n) \ +struct list_head n = { \ + .prev = &n, \ + .next = &n \ +} + +#define INIT_LIST_HEAD(p) \ +do { \ + struct list_head *__p298 = (p); \ + __p298->next = __p298; \ + __p298->prev = __p298->next; \ +} while (0) +#define list_entry(node, type, member) \ + (type *)((void *)node - offsetof(type, member)) +#define list_empty(p) \ +({ \ + const struct list_head *__p298 = (p); \ + ((__p298->next == __p298) && (__p298->prev == __p298)); \ +}) +#define list_add(p, l) \ +do { \ + struct list_head *__p298 = (p); \ + struct list_head *__l298 = (l); \ + __p298->next = __l298->next; \ + __p298->prev = __l298; \ + __l298->next->prev = __p298; \ + __l298->next = __p298; \ +} while (0) +#define list_add_tail(p, l) \ +do { \ + struct list_head *__p298 = (p); \ + struct list_head *__l298 = (l); \ + __p298->prev = __l298->prev; \ + __p298->next = __l298; \ + __l298->prev->next = __p298; \ + __l298->prev = __p298; \ +} while (0) +#define list_for_each(i, l) \ + for (i = (l)->next; i != (l); i = i->next) +#define list_for_each_safe(i, j, l) \ + for (i = (l)->next, j = i->next; i != (l); \ + i = j, j = i->next) +#define list_for_each_entry(i, l, name) \ + for (i = list_entry((l)->next, typeof(*i), name); &i->name != (l); \ + i = list_entry(i->name.next, typeof(*i), name)) +#define list_for_each_entry_safe(i, j, l, name) \ + for (i = list_entry((l)->next, typeof(*i), name), \ + j = list_entry(i->name.next, typeof(*j), name); \ + &i->name != (l); \ + i = j, j = list_entry(j->name.next, typeof(*j), name)) +#define list_del(i) \ +do { \ + (i)->next->prev = (i)->prev; \ + (i)->prev->next = (i)->next; \ +} while (0) + +#endif /* __DPAA_LIST_H */ diff --git a/dpdk/drivers/bus/dpaa/include/dpaa_rbtree.h b/dpdk/drivers/bus/dpaa/include/dpaa_rbtree.h new file mode 100644 index 00000000..f8c9b593 --- /dev/null +++ b/dpdk/drivers/bus/dpaa/include/dpaa_rbtree.h @@ -0,0 +1,143 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of NXP nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __DPAA_RBTREE_H +#define __DPAA_RBTREE_H + +#include +/************/ +/* RB-trees */ +/************/ + +/* Linux has a good RB-tree implementation, that we can't use (GPL). It also has + * a flat/hooked-in interface that virtually requires license-contamination in + * order to write a caller-compatible implementation. Instead, I've created an + * RB-tree encapsulation on top of linux's primitives (it does some of the work + * the client logic would normally do), and this gives us something we can + * reimplement on LWE. Unfortunately there's no good+free RB-tree + * implementations out there that are license-compatible and "flat" (ie. no + * dynamic allocation). I did find a malloc-based one that I could convert, but + * that will be a task for later on. For now, LWE's RB-tree is implemented using + * an ordered linked-list. + * + * Note, the only linux-esque type is "struct rb_node", because it's used + * statically in the exported header, so it can't be opaque. Our version doesn't + * include a "rb_parent_color" field because we're doing linked-list instead of + * a true rb-tree. + */ + +struct rb_node { + struct rb_node *prev, *next; +}; + +struct dpa_rbtree { + struct rb_node *head, *tail; +}; + +#define DPAA_RBTREE { NULL, NULL } +static inline void dpa_rbtree_init(struct dpa_rbtree *tree) +{ + tree->head = tree->tail = NULL; +} + +#define QMAN_NODE2OBJ(ptr, type, node_field) \ + (type *)((char *)ptr - offsetof(type, node_field)) + +#define IMPLEMENT_DPAA_RBTREE(name, type, node_field, val_field) \ +static inline int name##_push(struct dpa_rbtree *tree, type *obj) \ +{ \ + struct rb_node *node = tree->head; \ + if (!node) { \ + tree->head = tree->tail = &obj->node_field; \ + obj->node_field.prev = obj->node_field.next = NULL; \ + return 0; \ + } \ + while (node) { \ + type *item = QMAN_NODE2OBJ(node, type, node_field); \ + if (obj->val_field == item->val_field) \ + return -EBUSY; \ + if (obj->val_field < item->val_field) { \ + if (tree->head == node) \ + tree->head = &obj->node_field; \ + else \ + node->prev->next = &obj->node_field; \ + obj->node_field.prev = node->prev; \ + obj->node_field.next = node; \ + node->prev = &obj->node_field; \ + return 0; \ + } \ + node = node->next; \ + } \ + obj->node_field.prev = tree->tail; \ + obj->node_field.next = NULL; \ + tree->tail->next = &obj->node_field; \ + tree->tail = &obj->node_field; \ + return 0; \ +} \ +static inline void name##_del(struct dpa_rbtree *tree, type *obj) \ +{ \ + if (tree->head == &obj->node_field) { \ + if (tree->tail == &obj->node_field) \ + /* Only item in the list */ \ + tree->head = tree->tail = NULL; \ + else { \ + /* Is the head, next != NULL */ \ + tree->head = tree->head->next; \ + tree->head->prev = NULL; \ + } \ + } else { \ + if (tree->tail == &obj->node_field) { \ + /* Is the tail, prev != NULL */ \ + tree->tail = tree->tail->prev; \ + tree->tail->next = NULL; \ + } else { \ + /* Is neither the head nor the tail */ \ + obj->node_field.prev->next = obj->node_field.next; \ + obj->node_field.next->prev = obj->node_field.prev; \ + } \ + } \ +} \ +static inline type *name##_find(struct dpa_rbtree *tree, u32 val) \ +{ \ + struct rb_node *node = tree->head; \ + while (node) { \ + type *item = QMAN_NODE2OBJ(node, type, node_field); \ + if (val == item->val_field) \ + return item; \ + if (val < item->val_field) \ + return NULL; \ + node = node->next; \ + } \ + return NULL; \ +} + +#endif /* __DPAA_RBTREE_H */ diff --git a/dpdk/drivers/bus/dpaa/include/fman.h b/dpdk/drivers/bus/dpaa/include/fman.h new file mode 100644 index 00000000..9890e09c --- /dev/null +++ b/dpdk/drivers/bus/dpaa/include/fman.h @@ -0,0 +1,458 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2010-2012 Freescale Semiconductor, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __FMAN_H +#define __FMAN_H + +#include +#include + +#include +#include + +#include + +#ifndef FMAN_DEVICE_PATH +#define FMAN_DEVICE_PATH "/dev/mem" +#endif + +#define MEMAC_NUM_OF_PADDRS 7 /* Num of additional exact match MAC adr regs */ + +/* Control and Configuration Register (COMMAND_CONFIG) for MEMAC */ +#define CMD_CFG_LOOPBACK_EN 0x00000400 +/**< 21 XGMII/GMII loopback enable */ +#define CMD_CFG_PROMIS_EN 0x00000010 +/**< 27 Promiscuous operation enable */ +#define CMD_CFG_PAUSE_IGNORE 0x00000100 +/**< 23 Ignore Pause frame quanta */ + +/* Statistics Configuration Register (STATN_CONFIG) */ +#define STATS_CFG_CLR 0x00000004 +/**< 29 Reset all counters */ +#define STATS_CFG_CLR_ON_RD 0x00000002 +/**< 30 Clear on read */ +#define STATS_CFG_SATURATE 0x00000001 +/**< 31 Saturate at the maximum val */ + +/**< Max receive frame length mask */ +#define MAXFRM_SIZE_MEMAC 0x00007fe0 +#define MAXFRM_RX_MASK 0x0000ffff + +/**< Interface Mode Register Register for MEMAC */ +#define IF_MODE_RLP 0x00000820 + +/**< Pool Limits */ +#define FMAN_PORT_MAX_EXT_POOLS_NUM 8 +#define FMAN_PORT_OBS_EXT_POOLS_NUM 2 + +#define FMAN_PORT_CG_MAP_NUM 8 +#define FMAN_PORT_PRS_RESULT_WORDS_NUM 8 +#define FMAN_PORT_BMI_FIFO_UNITS 0x100 +#define FMAN_PORT_IC_OFFSET_UNITS 0x10 + +#define FMAN_ENABLE_BPOOL_DEPLETION 0xF00000F0 + +#define HASH_CTRL_MCAST_EN 0x00000100 +#define GROUP_ADDRESS 0x0000010000000000LL +#define HASH_CTRL_ADDR_MASK 0x0000003F + +/* Pre definitions of FMAN interface and Bpool structures */ +struct __fman_if; +struct fman_if_bpool; +/* Lists of fman interfaces and bpools */ +TAILQ_HEAD(rte_fman_if_list, __fman_if); + +/* Represents the different flavour of network interface */ +enum fman_mac_type { + fman_offline = 0, + fman_mac_1g, + fman_mac_10g, +}; + +struct mac_addr { + uint32_t mac_addr_l; /**< Lower 32 bits of 48-bit MAC address */ + uint32_t mac_addr_u; /**< Upper 16 bits of 48-bit MAC address */ +}; + +struct memac_regs { + /* General Control and Status */ + uint32_t res0000[2]; + uint32_t command_config; /**< 0x008 Ctrl and cfg */ + struct mac_addr mac_addr0; /**< 0x00C-0x010 MAC_ADDR_0...1 */ + uint32_t maxfrm; /**< 0x014 Max frame length */ + uint32_t res0018[5]; + uint32_t hashtable_ctrl; /**< 0x02C Hash table control */ + uint32_t res0030[4]; + uint32_t ievent; /**< 0x040 Interrupt event */ + uint32_t tx_ipg_length; + /**< 0x044 Transmitter inter-packet-gap */ + uint32_t res0048; + uint32_t imask; /**< 0x04C Interrupt mask */ + uint32_t res0050; + uint32_t pause_quanta[4]; /**< 0x054 Pause quanta */ + uint32_t pause_thresh[4]; /**< 0x064 Pause quanta threshold */ + uint32_t rx_pause_status; /**< 0x074 Receive pause status */ + uint32_t res0078[2]; + struct mac_addr mac_addr[MEMAC_NUM_OF_PADDRS]; + /**< 0x80-0x0B4 mac padr */ + uint32_t lpwake_timer; + /**< 0x0B8 Low Power Wakeup Timer */ + uint32_t sleep_timer; + /**< 0x0BC Transmit EEE Low Power Timer */ + uint32_t res00c0[8]; + uint32_t statn_config; + /**< 0x0E0 Statistics configuration */ + uint32_t res00e4[7]; + /* Rx Statistics Counter */ + uint32_t reoct_l; /**bpid); + * [...] + * } + */ +#define fman_if_for_each_bpool(bp, __if) \ + list_for_each_entry(bp, &(__if)->bpool_list, node) + +#define FMAN_ERR(rc, fmt, args...) \ + do { \ + _errno = (rc); \ + DPAA_BUS_LOG(ERR, fmt "(%d)", ##args, errno); \ + } while (0) + +#define FMAN_IP_REV_1 0xC30C4 +#define FMAN_IP_REV_1_MAJOR_MASK 0x0000FF00 +#define FMAN_IP_REV_1_MAJOR_SHIFT 8 +#define FMAN_V3 0x06 +#define FMAN_V3_CONTEXTA_EN_A2V 0x10000000 +#define FMAN_V3_CONTEXTA_EN_OVOM 0x02000000 +#define FMAN_V3_CONTEXTA_EN_EBD 0x80000000 +#define FMAN_CONTEXTA_DIS_CHECKSUM 0x7ull +#define FMAN_CONTEXTA_SET_OPCODE11 0x2000000b00000000 +extern u16 fman_ip_rev; +extern u32 fman_dealloc_bufs_mask_hi; +extern u32 fman_dealloc_bufs_mask_lo; + +/** + * Initialize the FMAN driver + * + * @args void + * @return + * 0 for success; error OTHERWISE + */ +int fman_init(void); + +/** + * Teardown the FMAN driver + * + * @args void + * @return void + */ +void fman_finish(void); + +#endif /* __FMAN_H */ diff --git a/dpdk/drivers/bus/dpaa/include/fsl_bman.h b/dpdk/drivers/bus/dpaa/include/fsl_bman.h new file mode 100644 index 00000000..383106b1 --- /dev/null +++ b/dpdk/drivers/bus/dpaa/include/fsl_bman.h @@ -0,0 +1,375 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2008-2012 Freescale Semiconductor, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __FSL_BMAN_H +#define __FSL_BMAN_H + +#ifdef __cplusplus +extern "C" { +#endif + +/* This wrapper represents a bit-array for the depletion state of the 64 Bman + * buffer pools. + */ +struct bman_depletion { + u32 state[2]; +}; + +static inline void bman_depletion_init(struct bman_depletion *c) +{ + c->state[0] = c->state[1] = 0; +} + +static inline void bman_depletion_fill(struct bman_depletion *c) +{ + c->state[0] = c->state[1] = ~0; +} + +/* --- Bman data structures (and associated constants) --- */ + +/* Represents s/w corenet portal mapped data structures */ +struct bm_rcr_entry; /* RCR (Release Command Ring) entries */ +struct bm_mc_command; /* MC (Management Command) command */ +struct bm_mc_result; /* MC result */ + +/* Code-reduction, define a wrapper for 48-bit buffers. In cases where a buffer + * pool id specific to this buffer is needed (BM_RCR_VERB_CMD_BPID_MULTI, + * BM_MCC_VERB_ACQUIRE), the 'bpid' field is used. + */ +struct bm_buffer { + union { + struct { +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + u8 __reserved1; + u8 bpid; + u16 hi; /* High 16-bits of 48-bit address */ + u32 lo; /* Low 32-bits of 48-bit address */ +#else + u32 lo; + u16 hi; + u8 bpid; + u8 __reserved; +#endif + }; + struct { +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + u64 __notaddress:16; + u64 addr:48; +#else + u64 addr:48; + u64 __notaddress:16; +#endif + }; + u64 opaque; + }; +} __attribute__((aligned(8))); +static inline u64 bm_buffer_get64(const struct bm_buffer *buf) +{ + return buf->addr; +} + +static inline dma_addr_t bm_buf_addr(const struct bm_buffer *buf) +{ + return (dma_addr_t)buf->addr; +} + +#define bm_buffer_set64(buf, v) \ + do { \ + struct bm_buffer *__buf931 = (buf); \ + __buf931->hi = upper_32_bits(v); \ + __buf931->lo = lower_32_bits(v); \ + } while (0) + +/* See 1.5.3.5.4: "Release Command" */ +struct bm_rcr_entry { + union { + struct { + u8 __dont_write_directly__verb; + u8 bpid; /* used with BM_RCR_VERB_CMD_BPID_SINGLE */ + u8 __reserved1[62]; + }; + struct bm_buffer bufs[8]; + }; +} __packed; +#define BM_RCR_VERB_VBIT 0x80 +#define BM_RCR_VERB_CMD_MASK 0x70 /* one of two values; */ +#define BM_RCR_VERB_CMD_BPID_SINGLE 0x20 +#define BM_RCR_VERB_CMD_BPID_MULTI 0x30 +#define BM_RCR_VERB_BUFCOUNT_MASK 0x0f /* values 1..8 */ + +/* See 1.5.3.1: "Acquire Command" */ +/* See 1.5.3.2: "Query Command" */ +struct bm_mcc_acquire { + u8 bpid; + u8 __reserved1[62]; +} __packed; +struct bm_mcc_query { + u8 __reserved2[63]; +} __packed; +struct bm_mc_command { + u8 __dont_write_directly__verb; + union { + struct bm_mcc_acquire acquire; + struct bm_mcc_query query; + }; +} __packed; +#define BM_MCC_VERB_VBIT 0x80 +#define BM_MCC_VERB_CMD_MASK 0x70 /* where the verb contains; */ +#define BM_MCC_VERB_CMD_ACQUIRE 0x10 +#define BM_MCC_VERB_CMD_QUERY 0x40 +#define BM_MCC_VERB_ACQUIRE_BUFCOUNT 0x0f /* values 1..8 go here */ + +/* See 1.5.3.3: "Acquire Response" */ +/* See 1.5.3.4: "Query Response" */ +struct bm_pool_state { + u8 __reserved1[32]; + /* "availability state" and "depletion state" */ + struct { + u8 __reserved1[8]; + /* Access using bman_depletion_***() */ + struct bman_depletion state; + } as, ds; +}; + +struct bm_mc_result { + union { + struct { + u8 verb; + u8 __reserved1[63]; + }; + union { + struct { + u8 __reserved1; + u8 bpid; + u8 __reserved2[62]; + }; + struct bm_buffer bufs[8]; + } acquire; + struct bm_pool_state query; + }; +} __packed; +#define BM_MCR_VERB_VBIT 0x80 +#define BM_MCR_VERB_CMD_MASK BM_MCC_VERB_CMD_MASK +#define BM_MCR_VERB_CMD_ACQUIRE BM_MCC_VERB_CMD_ACQUIRE +#define BM_MCR_VERB_CMD_QUERY BM_MCC_VERB_CMD_QUERY +#define BM_MCR_VERB_CMD_ERR_INVALID 0x60 +#define BM_MCR_VERB_CMD_ERR_ECC 0x70 +#define BM_MCR_VERB_ACQUIRE_BUFCOUNT BM_MCC_VERB_ACQUIRE_BUFCOUNT /* 0..8 */ + +/* Portal and Buffer Pools */ +/* Represents a managed portal */ +struct bman_portal; + +/* This object type represents Bman buffer pools. */ +struct bman_pool; + +/* This struct specifies parameters for a bman_pool object. */ +struct bman_pool_params { + /* index of the buffer pool to encapsulate (0-63), ignored if + * BMAN_POOL_FLAG_DYNAMIC_BPID is set. + */ + u32 bpid; + /* bit-mask of BMAN_POOL_FLAG_*** options */ + u32 flags; + /* depletion-entry/exit thresholds, if BMAN_POOL_FLAG_THRESH is set. NB: + * this is only allowed if BMAN_POOL_FLAG_DYNAMIC_BPID is used *and* + * when run in the control plane (which controls Bman CCSR). This array + * matches the definition of bm_pool_set(). + */ + u32 thresholds[4]; +}; + +/* Flags to bman_new_pool() */ +#define BMAN_POOL_FLAG_NO_RELEASE 0x00000001 /* can't release to pool */ +#define BMAN_POOL_FLAG_ONLY_RELEASE 0x00000002 /* can only release to pool */ +#define BMAN_POOL_FLAG_DYNAMIC_BPID 0x00000008 /* (de)allocate bpid */ +#define BMAN_POOL_FLAG_THRESH 0x00000010 /* set depletion thresholds */ + +/* Flags to bman_release() */ +#define BMAN_RELEASE_FLAG_NOW 0x00000008 /* issue immediate release */ + + +/** + * bman_get_portal_index - get portal configuration index + */ +int bman_get_portal_index(void); + +/** + * bman_rcr_is_empty - Determine if portal's RCR is empty + * + * For use in situations where a cpu-affine caller needs to determine when all + * releases for the local portal have been processed by Bman but can't use the + * BMAN_RELEASE_FLAG_WAIT_SYNC flag to do this from the final bman_release(). + * The function forces tracking of RCR consumption (which normally doesn't + * happen until release processing needs to find space to put new release + * commands), and returns zero if the ring still has unprocessed entries, + * non-zero if it is empty. + */ +int bman_rcr_is_empty(void); + +/** + * bman_alloc_bpid_range - Allocate a contiguous range of BPIDs + * @result: is set by the API to the base BPID of the allocated range + * @count: the number of BPIDs required + * @align: required alignment of the allocated range + * @partial: non-zero if the API can return fewer than @count BPIDs + * + * Returns the number of buffer pools allocated, or a negative error code. If + * @partial is non zero, the allocation request may return a smaller range of + * BPs than requested (though alignment will be as requested). If @partial is + * zero, the return value will either be 'count' or negative. + */ +int bman_alloc_bpid_range(u32 *result, u32 count, u32 align, int partial); +static inline int bman_alloc_bpid(u32 *result) +{ + int ret = bman_alloc_bpid_range(result, 1, 0, 0); + + return (ret > 0) ? 0 : ret; +} + +/** + * bman_release_bpid_range - Release the specified range of buffer pool IDs + * @bpid: the base BPID of the range to deallocate + * @count: the number of BPIDs in the range + * + * This function can also be used to seed the allocator with ranges of BPIDs + * that it can subsequently allocate from. + */ +void bman_release_bpid_range(u32 bpid, unsigned int count); +static inline void bman_release_bpid(u32 bpid) +{ + bman_release_bpid_range(bpid, 1); +} + +int bman_reserve_bpid_range(u32 bpid, unsigned int count); +static inline int bman_reserve_bpid(u32 bpid) +{ + return bman_reserve_bpid_range(bpid, 1); +} + +void bman_seed_bpid_range(u32 bpid, unsigned int count); + +int bman_shutdown_pool(u32 bpid); + +/** + * bman_new_pool - Allocates a Buffer Pool object + * @params: parameters specifying the buffer pool ID and behaviour + * + * Creates a pool object for the given @params. A portal and the depletion + * callback field of @params are only used if the BMAN_POOL_FLAG_DEPLETION flag + * is set. NB, the fields from @params are copied into the new pool object, so + * the structure provided by the caller can be released or reused after the + * function returns. + */ +struct bman_pool *bman_new_pool(const struct bman_pool_params *params); + +/** + * bman_free_pool - Deallocates a Buffer Pool object + * @pool: the pool object to release + */ +void bman_free_pool(struct bman_pool *pool); + +/** + * bman_get_params - Returns a pool object's parameters. + * @pool: the pool object + * + * The returned pointer refers to state within the pool object so must not be + * modified and can no longer be read once the pool object is destroyed. + */ +const struct bman_pool_params *bman_get_params(const struct bman_pool *pool); + +/** + * bman_release - Release buffer(s) to the buffer pool + * @pool: the buffer pool object to release to + * @bufs: an array of buffers to release + * @num: the number of buffers in @bufs (1-8) + * @flags: bit-mask of BMAN_RELEASE_FLAG_*** options + * + */ +int bman_release(struct bman_pool *pool, const struct bm_buffer *bufs, u8 num, + u32 flags); + +/** + * bman_acquire - Acquire buffer(s) from a buffer pool + * @pool: the buffer pool object to acquire from + * @bufs: array for storing the acquired buffers + * @num: the number of buffers desired (@bufs is at least this big) + * + * Issues an "Acquire" command via the portal's management command interface. + * The return value will be the number of buffers obtained from the pool, or a + * negative error code if a h/w error or pool starvation was encountered. + */ +int bman_acquire(struct bman_pool *pool, struct bm_buffer *bufs, u8 num, + u32 flags); + +/** + * bman_query_pools - Query all buffer pool states + * @state: storage for the queried availability and depletion states + */ +int bman_query_pools(struct bm_pool_state *state); + +/** + * bman_query_free_buffers - Query how many free buffers are in buffer pool + * @pool: the buffer pool object to query + * + * Return the number of the free buffers + */ +u32 bman_query_free_buffers(struct bman_pool *pool); + +/** + * bman_update_pool_thresholds - Change the buffer pool's depletion thresholds + * @pool: the buffer pool object to which the thresholds will be set + * @thresholds: the new thresholds + */ +int bman_update_pool_thresholds(struct bman_pool *pool, const u32 *thresholds); + +/** + * bm_pool_set_hw_threshold - Change the buffer pool's thresholds + * @pool: Pool id + * @low_thresh: low threshold + * @high_thresh: high threshold + */ +int bm_pool_set_hw_threshold(u32 bpid, const u32 low_thresh, + const u32 high_thresh); + +#ifdef __cplusplus +} +#endif + +#endif /* __FSL_BMAN_H */ diff --git a/dpdk/drivers/bus/dpaa/include/fsl_fman.h b/dpdk/drivers/bus/dpaa/include/fsl_fman.h new file mode 100644 index 00000000..95aee67a --- /dev/null +++ b/dpdk/drivers/bus/dpaa/include/fsl_fman.h @@ -0,0 +1,181 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __FSL_FMAN_H +#define __FSL_FMAN_H + +#ifdef __cplusplus +extern "C" { +#endif + +/* Status field in FD is updated on Rx side by FMAN with following information. + * Refer to field description in FM BG. + */ +struct fm_status_t { + unsigned int reserved0:3; + unsigned int dcl4c:1; /* Don't Check L4 Checksum */ + unsigned int reserved1:1; + unsigned int ufd:1; /* Unsupported Format */ + unsigned int lge:1; /* Length Error */ + unsigned int dme:1; /* DMA Error */ + + unsigned int reserved2:4; + unsigned int fpe:1; /* Frame physical Error */ + unsigned int fse:1; /* Frame Size Error */ + unsigned int dis:1; /* Discard by Classification */ + unsigned int reserved3:1; + + unsigned int eof:1; /* Key Extraction goes out of frame */ + unsigned int nss:1; /* No Scheme selected */ + unsigned int kso:1; /* Key Size Overflow */ + unsigned int reserved4:1; + unsigned int fcl:2; /* Frame Color */ + unsigned int ipp:1; /* Illegal Policer Profile Selected */ + unsigned int flm:1; /* Frame Length Mismatch */ + unsigned int pte:1; /* Parser Timeout */ + unsigned int isp:1; /* Invalid Soft Parser Instruction */ + unsigned int phe:1; /* Header Error during parsing */ + unsigned int frdr:1; /* Frame Dropped by disabled port */ + unsigned int reserved5:4; +} __attribute__ ((__packed__)); + +/* Set MAC address for a particular interface */ +int fman_if_add_mac_addr(struct fman_if *p, uint8_t *eth, uint8_t addr_num); + +/* Remove a MAC address for a particular interface */ +void fman_if_clear_mac_addr(struct fman_if *p, uint8_t addr_num); + +/* Get the FMAN statistics */ +void fman_if_stats_get(struct fman_if *p, struct rte_eth_stats *stats); + +/* Reset the FMAN statistics */ +void fman_if_stats_reset(struct fman_if *p); + +/* Get all of the FMAN statistics */ +void fman_if_stats_get_all(struct fman_if *p, uint64_t *value, int n); + +/* Set ignore pause option for a specific interface */ +void fman_if_set_rx_ignore_pause_frames(struct fman_if *p, bool enable); + +/* Set max frame length */ +void fman_if_conf_max_frame_len(struct fman_if *p, unsigned int max_frame_len); + +/* Enable/disable Rx promiscuous mode on specified interface */ +void fman_if_promiscuous_enable(struct fman_if *p); +void fman_if_promiscuous_disable(struct fman_if *p); + +/* Enable/disable Rx on specific interfaces */ +void fman_if_enable_rx(struct fman_if *p); +void fman_if_disable_rx(struct fman_if *p); + +/* Enable/disable loopback on specific interfaces */ +void fman_if_loopback_enable(struct fman_if *p); +void fman_if_loopback_disable(struct fman_if *p); + +/* Set buffer pool on specific interface */ +void fman_if_set_bp(struct fman_if *fm_if, unsigned int num, int bpid, + size_t bufsize); + +/* Get Flow Control threshold parameters on specific interface */ +int fman_if_get_fc_threshold(struct fman_if *fm_if); + +/* Enable and Set Flow Control threshold parameters on specific interface */ +int fman_if_set_fc_threshold(struct fman_if *fm_if, + u32 high_water, u32 low_water, u32 bpid); + +/* Get Flow Control pause quanta on specific interface */ +int fman_if_get_fc_quanta(struct fman_if *fm_if); + +/* Set Flow Control pause quanta on specific interface */ +int fman_if_set_fc_quanta(struct fman_if *fm_if, u16 pause_quanta); + +/* Set default error fqid on specific interface */ +void fman_if_set_err_fqid(struct fman_if *fm_if, uint32_t err_fqid); + +/* Get IC transfer params */ +int fman_if_get_ic_params(struct fman_if *fm_if, struct fman_if_ic_params *icp); + +/* Set IC transfer params */ +int fman_if_set_ic_params(struct fman_if *fm_if, + const struct fman_if_ic_params *icp); + +/* Get interface fd->offset value */ +int fman_if_get_fdoff(struct fman_if *fm_if); + +/* Set interface fd->offset value */ +void fman_if_set_fdoff(struct fman_if *fm_if, uint32_t fd_offset); + +/* Get interface Max Frame length (MTU) */ +uint16_t fman_if_get_maxfrm(struct fman_if *fm_if); + +/* Set interface Max Frame length (MTU) */ +void fman_if_set_maxfrm(struct fman_if *fm_if, uint16_t max_frm); + +/* Set interface next invoked action for dequeue operation */ +void fman_if_set_dnia(struct fman_if *fm_if, uint32_t nia); + +/* discard error packets on rx */ +void fman_if_discard_rx_errors(struct fman_if *fm_if); + +void fman_if_set_mcast_filter_table(struct fman_if *p); + +void fman_if_reset_mcast_filter_table(struct fman_if *p); + +int fman_if_add_hash_mac_addr(struct fman_if *p, uint8_t *eth); + +int fman_if_get_primary_mac_addr(struct fman_if *p, uint8_t *eth); + + +/* Enable/disable Rx on all interfaces */ +static inline void fman_if_enable_all_rx(void) +{ + struct fman_if *__if; + + list_for_each_entry(__if, fman_if_list, node) + fman_if_enable_rx(__if); +} + +static inline void fman_if_disable_all_rx(void) +{ + struct fman_if *__if; + + list_for_each_entry(__if, fman_if_list, node) + fman_if_disable_rx(__if); +} +#endif /* __FSL_FMAN_H */ diff --git a/dpdk/drivers/bus/dpaa/include/fsl_fman_crc64.h b/dpdk/drivers/bus/dpaa/include/fsl_fman_crc64.h new file mode 100644 index 00000000..af5803f3 --- /dev/null +++ b/dpdk/drivers/bus/dpaa/include/fsl_fman_crc64.h @@ -0,0 +1,263 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2011 Freescale Semiconductor, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __FSL_FMAN_CRC64_H +#define __FSL_FMAN_CRC64_H + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * This following definitions provide a software implementation of the CRC64 + * algorithm implemented within Fman. + * + * The following example shows how to compute a CRC64 hash value based on + * SRC_IP, DST_IP and ESP_SPI values + * + * #define compute_hash(saddr,daddr,spi) \ + * do { \ + * uint64_t result; \ + * result = fman_crc64_init(); \ + * result = fman_crc64_compute_32bit(saddr, result); \ + * result = fman_crc64_compute_32bit(daddr, result); \ + * result = fman_crc64_compute_32bit(spi, result); \ + * return (uint32_t) result & RC_HASH_MASK; \ + * } while (0); + * + * If hashing over a different number of fields (or of different types) is + * required, this can be implemented using the following primitives. + */ + +/* The following table provides the constants used by the Fman CRC64 + * implementation. The table is instantiated within the DPAA fman driver. + * However if the application is not going to be linked against the DPAA fman + * driver but will use this Fman CRC64 implementation, then it will need to + * instantiate this table by using the DECLARE_FMAN_CRC64_TABLE() macro. + */ +struct fman_crc64_t { + uint64_t initial; + uint64_t table[1 << 8]; +}; +extern struct fman_crc64_t FMAN_CRC64_ECMA_182; +#define DECLARE_FMAN_CRC64_TABLE() \ +struct fman_crc64_t FMAN_CRC64_ECMA_182 = { \ + 0xFFFFFFFFFFFFFFFFULL, \ + { \ + 0x0000000000000000ULL, 0xb32e4cbe03a75f6fULL, \ + 0xf4843657a840a05bULL, 0x47aa7ae9abe7ff34ULL, \ + 0x7bd0c384ff8f5e33ULL, 0xc8fe8f3afc28015cULL, \ + 0x8f54f5d357cffe68ULL, 0x3c7ab96d5468a107ULL, \ + 0xf7a18709ff1ebc66ULL, 0x448fcbb7fcb9e309ULL, \ + 0x0325b15e575e1c3dULL, 0xb00bfde054f94352ULL, \ + 0x8c71448d0091e255ULL, 0x3f5f08330336bd3aULL, \ + 0x78f572daa8d1420eULL, 0xcbdb3e64ab761d61ULL, \ + 0x7d9ba13851336649ULL, 0xceb5ed8652943926ULL, \ + 0x891f976ff973c612ULL, 0x3a31dbd1fad4997dULL, \ + 0x064b62bcaebc387aULL, 0xb5652e02ad1b6715ULL, \ + 0xf2cf54eb06fc9821ULL, 0x41e11855055bc74eULL, \ + 0x8a3a2631ae2dda2fULL, 0x39146a8fad8a8540ULL, \ + 0x7ebe1066066d7a74ULL, 0xcd905cd805ca251bULL, \ + 0xf1eae5b551a2841cULL, 0x42c4a90b5205db73ULL, \ + 0x056ed3e2f9e22447ULL, 0xb6409f5cfa457b28ULL, \ + 0xfb374270a266cc92ULL, 0x48190ecea1c193fdULL, \ + 0x0fb374270a266cc9ULL, 0xbc9d3899098133a6ULL, \ + 0x80e781f45de992a1ULL, 0x33c9cd4a5e4ecdceULL, \ + 0x7463b7a3f5a932faULL, 0xc74dfb1df60e6d95ULL, \ + 0x0c96c5795d7870f4ULL, 0xbfb889c75edf2f9bULL, \ + 0xf812f32ef538d0afULL, 0x4b3cbf90f69f8fc0ULL, \ + 0x774606fda2f72ec7ULL, 0xc4684a43a15071a8ULL, \ + 0x83c230aa0ab78e9cULL, 0x30ec7c140910d1f3ULL, \ + 0x86ace348f355aadbULL, 0x3582aff6f0f2f5b4ULL, \ + 0x7228d51f5b150a80ULL, 0xc10699a158b255efULL, \ + 0xfd7c20cc0cdaf4e8ULL, 0x4e526c720f7dab87ULL, \ + 0x09f8169ba49a54b3ULL, 0xbad65a25a73d0bdcULL, \ + 0x710d64410c4b16bdULL, 0xc22328ff0fec49d2ULL, \ + 0x85895216a40bb6e6ULL, 0x36a71ea8a7ace989ULL, \ + 0x0adda7c5f3c4488eULL, 0xb9f3eb7bf06317e1ULL, \ + 0xfe5991925b84e8d5ULL, 0x4d77dd2c5823b7baULL, \ + 0x64b62bcaebc387a1ULL, 0xd7986774e864d8ceULL, \ + 0x90321d9d438327faULL, 0x231c512340247895ULL, \ + 0x1f66e84e144cd992ULL, 0xac48a4f017eb86fdULL, \ + 0xebe2de19bc0c79c9ULL, 0x58cc92a7bfab26a6ULL, \ + 0x9317acc314dd3bc7ULL, 0x2039e07d177a64a8ULL, \ + 0x67939a94bc9d9b9cULL, 0xd4bdd62abf3ac4f3ULL, \ + 0xe8c76f47eb5265f4ULL, 0x5be923f9e8f53a9bULL, \ + 0x1c4359104312c5afULL, 0xaf6d15ae40b59ac0ULL, \ + 0x192d8af2baf0e1e8ULL, 0xaa03c64cb957be87ULL, \ + 0xeda9bca512b041b3ULL, 0x5e87f01b11171edcULL, \ + 0x62fd4976457fbfdbULL, 0xd1d305c846d8e0b4ULL, \ + 0x96797f21ed3f1f80ULL, 0x2557339fee9840efULL, \ + 0xee8c0dfb45ee5d8eULL, 0x5da24145464902e1ULL, \ + 0x1a083bacedaefdd5ULL, 0xa9267712ee09a2baULL, \ + 0x955cce7fba6103bdULL, 0x267282c1b9c65cd2ULL, \ + 0x61d8f8281221a3e6ULL, 0xd2f6b4961186fc89ULL, \ + 0x9f8169ba49a54b33ULL, 0x2caf25044a02145cULL, \ + 0x6b055fede1e5eb68ULL, 0xd82b1353e242b407ULL, \ + 0xe451aa3eb62a1500ULL, 0x577fe680b58d4a6fULL, \ + 0x10d59c691e6ab55bULL, 0xa3fbd0d71dcdea34ULL, \ + 0x6820eeb3b6bbf755ULL, 0xdb0ea20db51ca83aULL, \ + 0x9ca4d8e41efb570eULL, 0x2f8a945a1d5c0861ULL, \ + 0x13f02d374934a966ULL, 0xa0de61894a93f609ULL, \ + 0xe7741b60e174093dULL, 0x545a57dee2d35652ULL, \ + 0xe21ac88218962d7aULL, 0x5134843c1b317215ULL, \ + 0x169efed5b0d68d21ULL, 0xa5b0b26bb371d24eULL, \ + 0x99ca0b06e7197349ULL, 0x2ae447b8e4be2c26ULL, \ + 0x6d4e3d514f59d312ULL, 0xde6071ef4cfe8c7dULL, \ + 0x15bb4f8be788911cULL, 0xa6950335e42fce73ULL, \ + 0xe13f79dc4fc83147ULL, 0x521135624c6f6e28ULL, \ + 0x6e6b8c0f1807cf2fULL, 0xdd45c0b11ba09040ULL, \ + 0x9aefba58b0476f74ULL, 0x29c1f6e6b3e0301bULL, \ + 0xc96c5795d7870f42ULL, 0x7a421b2bd420502dULL, \ + 0x3de861c27fc7af19ULL, 0x8ec62d7c7c60f076ULL, \ + 0xb2bc941128085171ULL, 0x0192d8af2baf0e1eULL, \ + 0x4638a2468048f12aULL, 0xf516eef883efae45ULL, \ + 0x3ecdd09c2899b324ULL, 0x8de39c222b3eec4bULL, \ + 0xca49e6cb80d9137fULL, 0x7967aa75837e4c10ULL, \ + 0x451d1318d716ed17ULL, 0xf6335fa6d4b1b278ULL, \ + 0xb199254f7f564d4cULL, 0x02b769f17cf11223ULL, \ + 0xb4f7f6ad86b4690bULL, 0x07d9ba1385133664ULL, \ + 0x4073c0fa2ef4c950ULL, 0xf35d8c442d53963fULL, \ + 0xcf273529793b3738ULL, 0x7c0979977a9c6857ULL, \ + 0x3ba3037ed17b9763ULL, 0x888d4fc0d2dcc80cULL, \ + 0x435671a479aad56dULL, 0xf0783d1a7a0d8a02ULL, \ + 0xb7d247f3d1ea7536ULL, 0x04fc0b4dd24d2a59ULL, \ + 0x3886b22086258b5eULL, 0x8ba8fe9e8582d431ULL, \ + 0xcc0284772e652b05ULL, 0x7f2cc8c92dc2746aULL, \ + 0x325b15e575e1c3d0ULL, 0x8175595b76469cbfULL, \ + 0xc6df23b2dda1638bULL, 0x75f16f0cde063ce4ULL, \ + 0x498bd6618a6e9de3ULL, 0xfaa59adf89c9c28cULL, \ + 0xbd0fe036222e3db8ULL, 0x0e21ac88218962d7ULL, \ + 0xc5fa92ec8aff7fb6ULL, 0x76d4de52895820d9ULL, \ + 0x317ea4bb22bfdfedULL, 0x8250e80521188082ULL, \ + 0xbe2a516875702185ULL, 0x0d041dd676d77eeaULL, \ + 0x4aae673fdd3081deULL, 0xf9802b81de97deb1ULL, \ + 0x4fc0b4dd24d2a599ULL, 0xfceef8632775faf6ULL, \ + 0xbb44828a8c9205c2ULL, 0x086ace348f355aadULL, \ + 0x34107759db5dfbaaULL, 0x873e3be7d8faa4c5ULL, \ + 0xc094410e731d5bf1ULL, 0x73ba0db070ba049eULL, \ + 0xb86133d4dbcc19ffULL, 0x0b4f7f6ad86b4690ULL, \ + 0x4ce50583738cb9a4ULL, 0xffcb493d702be6cbULL, \ + 0xc3b1f050244347ccULL, 0x709fbcee27e418a3ULL, \ + 0x3735c6078c03e797ULL, 0x841b8ab98fa4b8f8ULL, \ + 0xadda7c5f3c4488e3ULL, 0x1ef430e13fe3d78cULL, \ + 0x595e4a08940428b8ULL, 0xea7006b697a377d7ULL, \ + 0xd60abfdbc3cbd6d0ULL, 0x6524f365c06c89bfULL, \ + 0x228e898c6b8b768bULL, 0x91a0c532682c29e4ULL, \ + 0x5a7bfb56c35a3485ULL, 0xe955b7e8c0fd6beaULL, \ + 0xaeffcd016b1a94deULL, 0x1dd181bf68bdcbb1ULL, \ + 0x21ab38d23cd56ab6ULL, 0x9285746c3f7235d9ULL, \ + 0xd52f0e859495caedULL, 0x6601423b97329582ULL, \ + 0xd041dd676d77eeaaULL, 0x636f91d96ed0b1c5ULL, \ + 0x24c5eb30c5374ef1ULL, 0x97eba78ec690119eULL, \ + 0xab911ee392f8b099ULL, 0x18bf525d915feff6ULL, \ + 0x5f1528b43ab810c2ULL, 0xec3b640a391f4fadULL, \ + 0x27e05a6e926952ccULL, 0x94ce16d091ce0da3ULL, \ + 0xd3646c393a29f297ULL, 0x604a2087398eadf8ULL, \ + 0x5c3099ea6de60cffULL, 0xef1ed5546e415390ULL, \ + 0xa8b4afbdc5a6aca4ULL, 0x1b9ae303c601f3cbULL, \ + 0x56ed3e2f9e224471ULL, 0xe5c372919d851b1eULL, \ + 0xa26908783662e42aULL, 0x114744c635c5bb45ULL, \ + 0x2d3dfdab61ad1a42ULL, 0x9e13b115620a452dULL, \ + 0xd9b9cbfcc9edba19ULL, 0x6a978742ca4ae576ULL, \ + 0xa14cb926613cf817ULL, 0x1262f598629ba778ULL, \ + 0x55c88f71c97c584cULL, 0xe6e6c3cfcadb0723ULL, \ + 0xda9c7aa29eb3a624ULL, 0x69b2361c9d14f94bULL, \ + 0x2e184cf536f3067fULL, 0x9d36004b35545910ULL, \ + 0x2b769f17cf112238ULL, 0x9858d3a9ccb67d57ULL, \ + 0xdff2a94067518263ULL, 0x6cdce5fe64f6dd0cULL, \ + 0x50a65c93309e7c0bULL, 0xe388102d33392364ULL, \ + 0xa4226ac498dedc50ULL, 0x170c267a9b79833fULL, \ + 0xdcd7181e300f9e5eULL, 0x6ff954a033a8c131ULL, \ + 0x28532e49984f3e05ULL, 0x9b7d62f79be8616aULL, \ + 0xa707db9acf80c06dULL, 0x14299724cc279f02ULL, \ + 0x5383edcd67c06036ULL, 0xe0ada17364673f59ULL} \ +} + +/* + * Return the initial CRC seed. Use the value returned from this API as the + * "crc" parameter to the first call to add data. + */ +static inline uint64_t fman_crc64_init(void) +{ + return FMAN_CRC64_ECMA_182.initial; +} + +/* Updates the CRC with arbitrary data */ +static inline uint64_t fman_crc64_update(uint64_t crc, + void *data, unsigned int len) +{ + uint8_t *p = data; + while (len--) + crc = FMAN_CRC64_ECMA_182.table[(crc ^ *(p++)) & 0xff] ^ + (crc >> 8); + return crc; +} + +/* Shorthands for updating the CRC with 8/16/32 bits of data. + * IMPORTANT NOTE: the typed "data" arguments should not be mistaken for + * host-endian numerical values, the assumption is that these values contain + * big-endian (ie. network byte order) data. + */ +static inline uint64_t fman_crc64_compute_32bit(uint32_t data, uint64_t crc) +{ + return fman_crc64_update(crc, &data, sizeof(data)); +} +static inline uint64_t fman_crc64_compute_16bit(uint16_t data, uint64_t crc) +{ + return fman_crc64_update(crc, &data, sizeof(data)); +} +static inline uint64_t fman_crc64_compute_8bit(uint8_t data, uint64_t crc) +{ + return fman_crc64_update(crc, &data, sizeof(data)); +} + +/* + * Finalise the CRC (using 2's complement) + */ +static inline uint64_t fman_crc64_finish(uint64_t seed) +{ + return ~seed; +} + +#ifdef __cplusplus +} +#endif + +#endif /* __FSL_FMAN_CRC64_H */ diff --git a/dpdk/drivers/bus/dpaa/include/fsl_qman.h b/dpdk/drivers/bus/dpaa/include/fsl_qman.h new file mode 100644 index 00000000..72556dc1 --- /dev/null +++ b/dpdk/drivers/bus/dpaa/include/fsl_qman.h @@ -0,0 +1,2021 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2008-2012 Freescale Semiconductor, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __FSL_QMAN_H +#define __FSL_QMAN_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +/* FQ lookups (turn this on for 64bit user-space) */ +#if (__WORDSIZE == 64) +#define CONFIG_FSL_QMAN_FQ_LOOKUP +/* if FQ lookups are supported, this controls the number of initialised, + * s/w-consumed FQs that can be supported at any one time. + */ +#define CONFIG_FSL_QMAN_FQ_LOOKUP_MAX (32 * 1024) +#endif + +/* Last updated for v00.800 of the BG */ + +/* Hardware constants */ +#define QM_CHANNEL_SWPORTAL0 0 +#define QMAN_CHANNEL_POOL1 0x21 +#define QMAN_CHANNEL_CAAM 0x80 +#define QMAN_CHANNEL_PME 0xa0 +#define QMAN_CHANNEL_POOL1_REV3 0x401 +#define QMAN_CHANNEL_CAAM_REV3 0x840 +#define QMAN_CHANNEL_PME_REV3 0x860 +extern u16 qm_channel_pool1; +extern u16 qm_channel_caam; +extern u16 qm_channel_pme; +enum qm_dc_portal { + qm_dc_portal_fman0 = 0, + qm_dc_portal_fman1 = 1, + qm_dc_portal_caam = 2, + qm_dc_portal_pme = 3 +}; + +/* Portal processing (interrupt) sources */ +#define QM_PIRQ_CCSCI 0x00200000 /* CEETM Congestion State Change */ +#define QM_PIRQ_CSCI 0x00100000 /* Congestion State Change */ +#define QM_PIRQ_EQCI 0x00080000 /* Enqueue Command Committed */ +#define QM_PIRQ_EQRI 0x00040000 /* EQCR Ring (below threshold) */ +#define QM_PIRQ_DQRI 0x00020000 /* DQRR Ring (non-empty) */ +#define QM_PIRQ_MRI 0x00010000 /* MR Ring (non-empty) */ +/* + * This mask contains all the interrupt sources that need handling except DQRI, + * ie. that if present should trigger slow-path processing. + */ +#define QM_PIRQ_SLOW (QM_PIRQ_CSCI | QM_PIRQ_EQCI | QM_PIRQ_EQRI | \ + QM_PIRQ_MRI | QM_PIRQ_CCSCI) + +/* For qman_static_dequeue_*** APIs */ +#define QM_SDQCR_CHANNELS_POOL_MASK 0x00007fff +/* for n in [1,15] */ +#define QM_SDQCR_CHANNELS_POOL(n) (0x00008000 >> (n)) +/* for conversion from n of qm_channel */ +static inline u32 QM_SDQCR_CHANNELS_POOL_CONV(u16 channel) +{ + return QM_SDQCR_CHANNELS_POOL(channel + 1 - qm_channel_pool1); +} + +/* For qman_volatile_dequeue(); Choose one PRECEDENCE. EXACT is optional. Use + * NUMFRAMES(n) (6-bit) or NUMFRAMES_TILLEMPTY to fill in the frame-count. Use + * FQID(n) to fill in the frame queue ID. + */ +#define QM_VDQCR_PRECEDENCE_VDQCR 0x0 +#define QM_VDQCR_PRECEDENCE_SDQCR 0x80000000 +#define QM_VDQCR_EXACT 0x40000000 +#define QM_VDQCR_NUMFRAMES_MASK 0x3f000000 +#define QM_VDQCR_NUMFRAMES_SET(n) (((n) & 0x3f) << 24) +#define QM_VDQCR_NUMFRAMES_GET(n) (((n) >> 24) & 0x3f) +#define QM_VDQCR_NUMFRAMES_TILLEMPTY QM_VDQCR_NUMFRAMES_SET(0) + +/* --- QMan data structures (and associated constants) --- */ + +/* Represents s/w corenet portal mapped data structures */ +struct qm_eqcr_entry; /* EQCR (EnQueue Command Ring) entries */ +struct qm_dqrr_entry; /* DQRR (DeQueue Response Ring) entries */ +struct qm_mr_entry; /* MR (Message Ring) entries */ +struct qm_mc_command; /* MC (Management Command) command */ +struct qm_mc_result; /* MC result */ + +#define QM_FD_FORMAT_SG 0x4 +#define QM_FD_FORMAT_LONG 0x2 +#define QM_FD_FORMAT_COMPOUND 0x1 +enum qm_fd_format { + /* + * 'contig' implies a contiguous buffer, whereas 'sg' implies a + * scatter-gather table. 'big' implies a 29-bit length with no offset + * field, otherwise length is 20-bit and offset is 9-bit. 'compound' + * implies a s/g-like table, where each entry itself represents a frame + * (contiguous or scatter-gather) and the 29-bit "length" is + * interpreted purely for congestion calculations, ie. a "congestion + * weight". + */ + qm_fd_contig = 0, + qm_fd_contig_big = QM_FD_FORMAT_LONG, + qm_fd_sg = QM_FD_FORMAT_SG, + qm_fd_sg_big = QM_FD_FORMAT_SG | QM_FD_FORMAT_LONG, + qm_fd_compound = QM_FD_FORMAT_COMPOUND +}; + +/* Capitalised versions are un-typed but can be used in static expressions */ +#define QM_FD_CONTIG 0 +#define QM_FD_CONTIG_BIG QM_FD_FORMAT_LONG +#define QM_FD_SG QM_FD_FORMAT_SG +#define QM_FD_SG_BIG (QM_FD_FORMAT_SG | QM_FD_FORMAT_LONG) +#define QM_FD_COMPOUND QM_FD_FORMAT_COMPOUND + +/* "Frame Descriptor (FD)" */ +struct qm_fd { + union { + struct { +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + u8 dd:2; /* dynamic debug */ + u8 liodn_offset:6; + u8 bpid:8; /* Buffer Pool ID */ + u8 eliodn_offset:4; + u8 __reserved:4; + u8 addr_hi; /* high 8-bits of 40-bit address */ + u32 addr_lo; /* low 32-bits of 40-bit address */ +#else + u8 liodn_offset:6; + u8 dd:2; /* dynamic debug */ + u8 bpid:8; /* Buffer Pool ID */ + u8 __reserved:4; + u8 eliodn_offset:4; + u8 addr_hi; /* high 8-bits of 40-bit address */ + u32 addr_lo; /* low 32-bits of 40-bit address */ +#endif + }; + struct { + u64 __notaddress:24; + /* More efficient address accessor */ + u64 addr:40; + }; + u64 opaque_addr; + }; + /* The 'format' field indicates the interpretation of the remaining 29 + * bits of the 32-bit word. For packing reasons, it is duplicated in the + * other union elements. Note, union'd structs are difficult to use with + * static initialisation under gcc, in which case use the "opaque" form + * with one of the macros. + */ + union { + /* For easier/faster copying of this part of the fd (eg. from a + * DQRR entry to an EQCR entry) copy 'opaque' + */ + u32 opaque; + /* If 'format' is _contig or _sg, 20b length and 9b offset */ + struct { +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + enum qm_fd_format format:3; + u16 offset:9; + u32 length20:20; +#else + u32 length20:20; + u16 offset:9; + enum qm_fd_format format:3; +#endif + }; + /* If 'format' is _contig_big or _sg_big, 29b length */ + struct { +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + enum qm_fd_format _format1:3; + u32 length29:29; +#else + u32 length29:29; + enum qm_fd_format _format1:3; +#endif + }; + /* If 'format' is _compound, 29b "congestion weight" */ + struct { +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + enum qm_fd_format _format2:3; + u32 cong_weight:29; +#else + u32 cong_weight:29; + enum qm_fd_format _format2:3; +#endif + }; + }; + union { + u32 cmd; + u32 status; + }; +} __attribute__((aligned(8))); +#define QM_FD_DD_NULL 0x00 +#define QM_FD_PID_MASK 0x3f +static inline u64 qm_fd_addr_get64(const struct qm_fd *fd) +{ + return fd->addr; +} + +static inline dma_addr_t qm_fd_addr(const struct qm_fd *fd) +{ + return (dma_addr_t)fd->addr; +} + +/* Macro, so we compile better if 'v' isn't always 64-bit */ +#define qm_fd_addr_set64(fd, v) \ + do { \ + struct qm_fd *__fd931 = (fd); \ + __fd931->addr = v; \ + } while (0) + +/* Scatter/Gather table entry */ +struct qm_sg_entry { + union { + struct { +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + u8 __reserved1[3]; + u8 addr_hi; /* high 8-bits of 40-bit address */ + u32 addr_lo; /* low 32-bits of 40-bit address */ +#else + u32 addr_lo; /* low 32-bits of 40-bit address */ + u8 addr_hi; /* high 8-bits of 40-bit address */ + u8 __reserved1[3]; +#endif + }; + struct { +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + u64 __notaddress:24; + u64 addr:40; +#else + u64 addr:40; + u64 __notaddress:24; +#endif + }; + u64 opaque; + }; + union { + struct { +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + u32 extension:1; /* Extension bit */ + u32 final:1; /* Final bit */ + u32 length:30; +#else + u32 length:30; + u32 final:1; /* Final bit */ + u32 extension:1; /* Extension bit */ +#endif + }; + u32 val; + }; + u8 __reserved2; + u8 bpid; + union { + struct { +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + u16 __reserved3:3; + u16 offset:13; +#else + u16 offset:13; + u16 __reserved3:3; +#endif + }; + u16 val_off; + }; +} __packed; +static inline u64 qm_sg_entry_get64(const struct qm_sg_entry *sg) +{ + return sg->addr; +} + +static inline dma_addr_t qm_sg_addr(const struct qm_sg_entry *sg) +{ + return (dma_addr_t)sg->addr; +} + +/* Macro, so we compile better if 'v' isn't always 64-bit */ +#define qm_sg_entry_set64(sg, v) \ + do { \ + struct qm_sg_entry *__sg931 = (sg); \ + __sg931->addr = v; \ + } while (0) + +/* See 1.5.8.1: "Enqueue Command" */ +struct qm_eqcr_entry { + u8 __dont_write_directly__verb; + u8 dca; + u16 seqnum; + u32 orp; /* 24-bit */ + u32 fqid; /* 24-bit */ + u32 tag; + struct qm_fd fd; + u8 __reserved3[32]; +} __packed; + + +/* "Frame Dequeue Response" */ +struct qm_dqrr_entry { + u8 verb; + u8 stat; + u16 seqnum; /* 15-bit */ + u8 tok; + u8 __reserved2[3]; + u32 fqid; /* 24-bit */ + u32 contextB; + struct qm_fd fd; + u8 __reserved4[32]; +}; + +#define QM_DQRR_VERB_VBIT 0x80 +#define QM_DQRR_VERB_MASK 0x7f /* where the verb contains; */ +#define QM_DQRR_VERB_FRAME_DEQUEUE 0x60 /* "this format" */ +#define QM_DQRR_STAT_FQ_EMPTY 0x80 /* FQ empty */ +#define QM_DQRR_STAT_FQ_HELDACTIVE 0x40 /* FQ held active */ +#define QM_DQRR_STAT_FQ_FORCEELIGIBLE 0x20 /* FQ was force-eligible'd */ +#define QM_DQRR_STAT_FD_VALID 0x10 /* has a non-NULL FD */ +#define QM_DQRR_STAT_UNSCHEDULED 0x02 /* Unscheduled dequeue */ +#define QM_DQRR_STAT_DQCR_EXPIRED 0x01 /* VDQCR or PDQCR expired*/ + + +/* "ERN Message Response" */ +/* "FQ State Change Notification" */ +struct qm_mr_entry { + u8 verb; + union { + struct { + u8 dca; + u16 seqnum; + u8 rc; /* Rejection Code */ + u32 orp:24; + u32 fqid; /* 24-bit */ + u32 tag; + struct qm_fd fd; + } __packed ern; + struct { +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + u8 colour:2; /* See QM_MR_DCERN_COLOUR_* */ + u8 __reserved1:4; + enum qm_dc_portal portal:2; +#else + enum qm_dc_portal portal:3; + u8 __reserved1:3; + u8 colour:2; /* See QM_MR_DCERN_COLOUR_* */ +#endif + u16 __reserved2; + u8 rc; /* Rejection Code */ + u32 __reserved3:24; + u32 fqid; /* 24-bit */ + u32 tag; + struct qm_fd fd; + } __packed dcern; + struct { + u8 fqs; /* Frame Queue Status */ + u8 __reserved1[6]; + u32 fqid; /* 24-bit */ + u32 contextB; + u8 __reserved2[16]; + } __packed fq; /* FQRN/FQRNI/FQRL/FQPN */ + }; + u8 __reserved2[32]; +} __packed; +#define QM_MR_VERB_VBIT 0x80 +/* + * ERNs originating from direct-connect portals ("dcern") use 0x20 as a verb + * which would be invalid as a s/w enqueue verb. A s/w ERN can be distinguished + * from the other MR types by noting if the 0x20 bit is unset. + */ +#define QM_MR_VERB_TYPE_MASK 0x27 +#define QM_MR_VERB_DC_ERN 0x20 +#define QM_MR_VERB_FQRN 0x21 +#define QM_MR_VERB_FQRNI 0x22 +#define QM_MR_VERB_FQRL 0x23 +#define QM_MR_VERB_FQPN 0x24 +#define QM_MR_RC_MASK 0xf0 /* contains one of; */ +#define QM_MR_RC_CGR_TAILDROP 0x00 +#define QM_MR_RC_WRED 0x10 +#define QM_MR_RC_ERROR 0x20 +#define QM_MR_RC_ORPWINDOW_EARLY 0x30 +#define QM_MR_RC_ORPWINDOW_LATE 0x40 +#define QM_MR_RC_FQ_TAILDROP 0x50 +#define QM_MR_RC_ORPWINDOW_RETIRED 0x60 +#define QM_MR_RC_ORP_ZERO 0x70 +#define QM_MR_FQS_ORLPRESENT 0x02 /* ORL fragments to come */ +#define QM_MR_FQS_NOTEMPTY 0x01 /* FQ has enqueued frames */ +#define QM_MR_DCERN_COLOUR_GREEN 0x00 +#define QM_MR_DCERN_COLOUR_YELLOW 0x01 +#define QM_MR_DCERN_COLOUR_RED 0x02 +#define QM_MR_DCERN_COLOUR_OVERRIDE 0x03 +/* + * An identical structure of FQD fields is present in the "Init FQ" command and + * the "Query FQ" result, it's suctioned out into the "struct qm_fqd" type. + * Within that, the 'stashing' and 'taildrop' pieces are also factored out, the + * latter has two inlines to assist with converting to/from the mant+exp + * representation. + */ +struct qm_fqd_stashing { + /* See QM_STASHING_EXCL_<...> */ +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + u8 exclusive; + u8 __reserved1:2; + /* Numbers of cachelines */ + u8 annotation_cl:2; + u8 data_cl:2; + u8 context_cl:2; +#else + u8 context_cl:2; + u8 data_cl:2; + u8 annotation_cl:2; + u8 __reserved1:2; + u8 exclusive; +#endif +} __packed; +struct qm_fqd_taildrop { +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + u16 __reserved1:3; + u16 mant:8; + u16 exp:5; +#else + u16 exp:5; + u16 mant:8; + u16 __reserved1:3; +#endif +} __packed; +struct qm_fqd_oac { + /* "Overhead Accounting Control", see QM_OAC_<...> */ +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + u8 oac:2; /* "Overhead Accounting Control" */ + u8 __reserved1:6; +#else + u8 __reserved1:6; + u8 oac:2; /* "Overhead Accounting Control" */ +#endif + /* Two's-complement value (-128 to +127) */ + signed char oal; /* "Overhead Accounting Length" */ +} __packed; +struct qm_fqd { + union { + u8 orpc; + struct { +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + u8 __reserved1:2; + u8 orprws:3; + u8 oa:1; + u8 olws:2; +#else + u8 olws:2; + u8 oa:1; + u8 orprws:3; + u8 __reserved1:2; +#endif + } __packed; + }; + u8 cgid; + u16 fq_ctrl; /* See QM_FQCTRL_<...> */ + union { + u16 dest_wq; + struct { +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + u16 channel:13; /* qm_channel */ + u16 wq:3; +#else + u16 wq:3; + u16 channel:13; /* qm_channel */ +#endif + } __packed dest; + }; +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + u16 __reserved2:1; + u16 ics_cred:15; +#else + u16 __reserved2:1; + u16 ics_cred:15; +#endif + /* + * For "Initialize Frame Queue" commands, the write-enable mask + * determines whether 'td' or 'oac_init' is observed. For query + * commands, this field is always 'td', and 'oac_query' (below) reflects + * the Overhead ACcounting values. + */ + union { + uint16_t opaque_td; + struct qm_fqd_taildrop td; + struct qm_fqd_oac oac_init; + }; + u32 context_b; + union { + /* Treat it as 64-bit opaque */ + u64 opaque; + struct { +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + u32 hi; + u32 lo; +#else + u32 lo; + u32 hi; +#endif + }; + /* Treat it as s/w portal stashing config */ + /* see "FQD Context_A field used for [...]" */ + struct { +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + struct qm_fqd_stashing stashing; + /* + * 48-bit address of FQ context to + * stash, must be cacheline-aligned + */ + u16 context_hi; + u32 context_lo; +#else + u32 context_lo; + u16 context_hi; + struct qm_fqd_stashing stashing; +#endif + } __packed; + } context_a; + struct qm_fqd_oac oac_query; +} __packed; +/* 64-bit converters for context_hi/lo */ +static inline u64 qm_fqd_stashing_get64(const struct qm_fqd *fqd) +{ + return ((u64)fqd->context_a.context_hi << 32) | + (u64)fqd->context_a.context_lo; +} + +static inline dma_addr_t qm_fqd_stashing_addr(const struct qm_fqd *fqd) +{ + return (dma_addr_t)qm_fqd_stashing_get64(fqd); +} + +static inline u64 qm_fqd_context_a_get64(const struct qm_fqd *fqd) +{ + return ((u64)fqd->context_a.hi << 32) | + (u64)fqd->context_a.lo; +} + +static inline void qm_fqd_stashing_set64(struct qm_fqd *fqd, u64 addr) +{ + fqd->context_a.context_hi = upper_32_bits(addr); + fqd->context_a.context_lo = lower_32_bits(addr); +} + +static inline void qm_fqd_context_a_set64(struct qm_fqd *fqd, u64 addr) +{ + fqd->context_a.hi = upper_32_bits(addr); + fqd->context_a.lo = lower_32_bits(addr); +} + +/* convert a threshold value into mant+exp representation */ +static inline int qm_fqd_taildrop_set(struct qm_fqd_taildrop *td, u32 val, + int roundup) +{ + u32 e = 0; + int oddbit = 0; + + if (val > 0xe0000000) + return -ERANGE; + while (val > 0xff) { + oddbit = val & 1; + val >>= 1; + e++; + if (roundup && oddbit) + val++; + } + td->exp = e; + td->mant = val; + return 0; +} + +/* and the other direction */ +static inline u32 qm_fqd_taildrop_get(const struct qm_fqd_taildrop *td) +{ + return (u32)td->mant << td->exp; +} + + +/* See "Frame Queue Descriptor (FQD)" */ +/* Frame Queue Descriptor (FQD) field 'fq_ctrl' uses these constants */ +#define QM_FQCTRL_MASK 0x07ff /* 'fq_ctrl' flags; */ +#define QM_FQCTRL_CGE 0x0400 /* Congestion Group Enable */ +#define QM_FQCTRL_TDE 0x0200 /* Tail-Drop Enable */ +#define QM_FQCTRL_ORP 0x0100 /* ORP Enable */ +#define QM_FQCTRL_CTXASTASHING 0x0080 /* Context-A stashing */ +#define QM_FQCTRL_CPCSTASH 0x0040 /* CPC Stash Enable */ +#define QM_FQCTRL_FORCESFDR 0x0008 /* High-priority SFDRs */ +#define QM_FQCTRL_AVOIDBLOCK 0x0004 /* Don't block active */ +#define QM_FQCTRL_HOLDACTIVE 0x0002 /* Hold active in portal */ +#define QM_FQCTRL_PREFERINCACHE 0x0001 /* Aggressively cache FQD */ +#define QM_FQCTRL_LOCKINCACHE QM_FQCTRL_PREFERINCACHE /* older naming */ + +/* See "FQD Context_A field used for [...] */ +/* Frame Queue Descriptor (FQD) field 'CONTEXT_A' uses these constants */ +#define QM_STASHING_EXCL_ANNOTATION 0x04 +#define QM_STASHING_EXCL_DATA 0x02 +#define QM_STASHING_EXCL_CTX 0x01 + +/* See "Intra Class Scheduling" */ +/* FQD field 'OAC' (Overhead ACcounting) uses these constants */ +#define QM_OAC_ICS 0x2 /* Accounting for Intra-Class Scheduling */ +#define QM_OAC_CG 0x1 /* Accounting for Congestion Groups */ + +/* + * This struct represents the 32-bit "WR_PARM_[GYR]" parameters in CGR fields + * and associated commands/responses. The WRED parameters are calculated from + * these fields as follows; + * MaxTH = MA * (2 ^ Mn) + * Slope = SA / (2 ^ Sn) + * MaxP = 4 * (Pn + 1) + */ +struct qm_cgr_wr_parm { + union { + u32 word; + struct { +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + u32 MA:8; + u32 Mn:5; + u32 SA:7; /* must be between 64-127 */ + u32 Sn:6; + u32 Pn:6; +#else + u32 Pn:6; + u32 Sn:6; + u32 SA:7; /* must be between 64-127 */ + u32 Mn:5; + u32 MA:8; +#endif + } __packed; + }; +} __packed; +/* + * This struct represents the 13-bit "CS_THRES" CGR field. In the corresponding + * management commands, this is padded to a 16-bit structure field, so that's + * how we represent it here. The congestion state threshold is calculated from + * these fields as follows; + * CS threshold = TA * (2 ^ Tn) + */ +struct qm_cgr_cs_thres { + union { + u16 hword; + struct { +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + u16 __reserved:3; + u16 TA:8; + u16 Tn:5; +#else + u16 Tn:5; + u16 TA:8; + u16 __reserved:3; +#endif + } __packed; + }; +} __packed; +/* + * This identical structure of CGR fields is present in the "Init/Modify CGR" + * commands and the "Query CGR" result. It's suctioned out here into its own + * struct. + */ +struct __qm_mc_cgr { + struct qm_cgr_wr_parm wr_parm_g; + struct qm_cgr_wr_parm wr_parm_y; + struct qm_cgr_wr_parm wr_parm_r; + u8 wr_en_g; /* boolean, use QM_CGR_EN */ + u8 wr_en_y; /* boolean, use QM_CGR_EN */ + u8 wr_en_r; /* boolean, use QM_CGR_EN */ + u8 cscn_en; /* boolean, use QM_CGR_EN */ + union { + struct { +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + u16 cscn_targ_upd_ctrl; /* use QM_CSCN_TARG_UDP_ */ + u16 cscn_targ_dcp_low; /* CSCN_TARG_DCP low-16bits */ +#else + u16 cscn_targ_dcp_low; /* CSCN_TARG_DCP low-16bits */ + u16 cscn_targ_upd_ctrl; /* use QM_CSCN_TARG_UDP_ */ +#endif + }; + u32 cscn_targ; /* use QM_CGR_TARG_* */ + }; + u8 cstd_en; /* boolean, use QM_CGR_EN */ + u8 cs; /* boolean, only used in query response */ + union { + struct qm_cgr_cs_thres cs_thres; + /* use qm_cgr_cs_thres_set64() */ + u16 __cs_thres; + }; + u8 mode; /* QMAN_CGR_MODE_FRAME not supported in rev1.0 */ +} __packed; +#define QM_CGR_EN 0x01 /* For wr_en_*, cscn_en, cstd_en */ +#define QM_CGR_TARG_UDP_CTRL_WRITE_BIT 0x8000 /* value written to portal bit*/ +#define QM_CGR_TARG_UDP_CTRL_DCP 0x4000 /* 0: SWP, 1: DCP */ +#define QM_CGR_TARG_PORTAL(n) (0x80000000 >> (n)) /* s/w portal, 0-9 */ +#define QM_CGR_TARG_FMAN0 0x00200000 /* direct-connect portal: fman0 */ +#define QM_CGR_TARG_FMAN1 0x00100000 /* : fman1 */ +/* Convert CGR thresholds to/from "cs_thres" format */ +static inline u64 qm_cgr_cs_thres_get64(const struct qm_cgr_cs_thres *th) +{ + return (u64)th->TA << th->Tn; +} + +static inline int qm_cgr_cs_thres_set64(struct qm_cgr_cs_thres *th, u64 val, + int roundup) +{ + u32 e = 0; + int oddbit = 0; + + while (val > 0xff) { + oddbit = val & 1; + val >>= 1; + e++; + if (roundup && oddbit) + val++; + } + th->Tn = e; + th->TA = val; + return 0; +} + +/* See 1.5.8.5.1: "Initialize FQ" */ +/* See 1.5.8.5.2: "Query FQ" */ +/* See 1.5.8.5.3: "Query FQ Non-Programmable Fields" */ +/* See 1.5.8.5.4: "Alter FQ State Commands " */ +/* See 1.5.8.6.1: "Initialize/Modify CGR" */ +/* See 1.5.8.6.2: "CGR Test Write" */ +/* See 1.5.8.6.3: "Query CGR" */ +/* See 1.5.8.6.4: "Query Congestion Group State" */ +struct qm_mcc_initfq { + u8 __reserved1; + u16 we_mask; /* Write Enable Mask */ + u32 fqid; /* 24-bit */ + u16 count; /* Initialises 'count+1' FQDs */ + struct qm_fqd fqd; /* the FQD fields go here */ + u8 __reserved3[30]; +} __packed; +struct qm_mcc_queryfq { + u8 __reserved1[3]; + u32 fqid; /* 24-bit */ + u8 __reserved2[56]; +} __packed; +struct qm_mcc_queryfq_np { + u8 __reserved1[3]; + u32 fqid; /* 24-bit */ + u8 __reserved2[56]; +} __packed; +struct qm_mcc_alterfq { + u8 __reserved1[3]; + u32 fqid; /* 24-bit */ + u8 __reserved2; + u8 count; /* number of consecutive FQID */ + u8 __reserved3[10]; + u32 context_b; /* frame queue context b */ + u8 __reserved4[40]; +} __packed; +struct qm_mcc_initcgr { + u8 __reserved1; + u16 we_mask; /* Write Enable Mask */ + struct __qm_mc_cgr cgr; /* CGR fields */ + u8 __reserved2[2]; + u8 cgid; + u8 __reserved4[32]; +} __packed; +struct qm_mcc_cgrtestwrite { + u8 __reserved1[2]; + u8 i_bcnt_hi:8;/* high 8-bits of 40-bit "Instant" */ + u32 i_bcnt_lo; /* low 32-bits of 40-bit */ + u8 __reserved2[23]; + u8 cgid; + u8 __reserved3[32]; +} __packed; +struct qm_mcc_querycgr { + u8 __reserved1[30]; + u8 cgid; + u8 __reserved2[32]; +} __packed; +struct qm_mcc_querycongestion { + u8 __reserved[63]; +} __packed; +struct qm_mcc_querywq { + u8 __reserved; + /* select channel if verb != QUERYWQ_DEDICATED */ + union { + u16 channel_wq; /* ignores wq (3 lsbits) */ + struct { +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + u16 id:13; /* qm_channel */ + u16 __reserved1:3; +#else + u16 __reserved1:3; + u16 id:13; /* qm_channel */ +#endif + } __packed channel; + }; + u8 __reserved2[60]; +} __packed; + +struct qm_mc_command { + u8 __dont_write_directly__verb; + union { + struct qm_mcc_initfq initfq; + struct qm_mcc_queryfq queryfq; + struct qm_mcc_queryfq_np queryfq_np; + struct qm_mcc_alterfq alterfq; + struct qm_mcc_initcgr initcgr; + struct qm_mcc_cgrtestwrite cgrtestwrite; + struct qm_mcc_querycgr querycgr; + struct qm_mcc_querycongestion querycongestion; + struct qm_mcc_querywq querywq; + }; +} __packed; + +/* INITFQ-specific flags */ +#define QM_INITFQ_WE_MASK 0x01ff /* 'Write Enable' flags; */ +#define QM_INITFQ_WE_OAC 0x0100 +#define QM_INITFQ_WE_ORPC 0x0080 +#define QM_INITFQ_WE_CGID 0x0040 +#define QM_INITFQ_WE_FQCTRL 0x0020 +#define QM_INITFQ_WE_DESTWQ 0x0010 +#define QM_INITFQ_WE_ICSCRED 0x0008 +#define QM_INITFQ_WE_TDTHRESH 0x0004 +#define QM_INITFQ_WE_CONTEXTB 0x0002 +#define QM_INITFQ_WE_CONTEXTA 0x0001 +/* INITCGR/MODIFYCGR-specific flags */ +#define QM_CGR_WE_MASK 0x07ff /* 'Write Enable Mask'; */ +#define QM_CGR_WE_WR_PARM_G 0x0400 +#define QM_CGR_WE_WR_PARM_Y 0x0200 +#define QM_CGR_WE_WR_PARM_R 0x0100 +#define QM_CGR_WE_WR_EN_G 0x0080 +#define QM_CGR_WE_WR_EN_Y 0x0040 +#define QM_CGR_WE_WR_EN_R 0x0020 +#define QM_CGR_WE_CSCN_EN 0x0010 +#define QM_CGR_WE_CSCN_TARG 0x0008 +#define QM_CGR_WE_CSTD_EN 0x0004 +#define QM_CGR_WE_CS_THRES 0x0002 +#define QM_CGR_WE_MODE 0x0001 + +struct qm_mcr_initfq { + u8 __reserved1[62]; +} __packed; +struct qm_mcr_queryfq { + u8 __reserved1[8]; + struct qm_fqd fqd; /* the FQD fields are here */ + u8 __reserved2[30]; +} __packed; +struct qm_mcr_queryfq_np { + u8 __reserved1; + u8 state; /* QM_MCR_NP_STATE_*** */ +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + u8 __reserved2; + u32 fqd_link:24; + u16 __reserved3:2; + u16 odp_seq:14; + u16 __reserved4:2; + u16 orp_nesn:14; + u16 __reserved5:1; + u16 orp_ea_hseq:15; + u16 __reserved6:1; + u16 orp_ea_tseq:15; + u8 __reserved7; + u32 orp_ea_hptr:24; + u8 __reserved8; + u32 orp_ea_tptr:24; + u8 __reserved9; + u32 pfdr_hptr:24; + u8 __reserved10; + u32 pfdr_tptr:24; + u8 __reserved11[5]; + u8 __reserved12:7; + u8 is:1; + u16 ics_surp; + u32 byte_cnt; + u8 __reserved13; + u32 frm_cnt:24; + u32 __reserved14; + u16 ra1_sfdr; /* QM_MCR_NP_RA1_*** */ + u16 ra2_sfdr; /* QM_MCR_NP_RA2_*** */ + u16 __reserved15; + u16 od1_sfdr; /* QM_MCR_NP_OD1_*** */ + u16 od2_sfdr; /* QM_MCR_NP_OD2_*** */ + u16 od3_sfdr; /* QM_MCR_NP_OD3_*** */ +#else + u8 __reserved2; + u32 fqd_link:24; + + u16 odp_seq:14; + u16 __reserved3:2; + + u16 orp_nesn:14; + u16 __reserved4:2; + + u16 orp_ea_hseq:15; + u16 __reserved5:1; + + u16 orp_ea_tseq:15; + u16 __reserved6:1; + + u8 __reserved7; + u32 orp_ea_hptr:24; + + u8 __reserved8; + u32 orp_ea_tptr:24; + + u8 __reserved9; + u32 pfdr_hptr:24; + + u8 __reserved10; + u32 pfdr_tptr:24; + + u8 __reserved11[5]; + u8 is:1; + u8 __reserved12:7; + u16 ics_surp; + u32 byte_cnt; + u8 __reserved13; + u32 frm_cnt:24; + u32 __reserved14; + u16 ra1_sfdr; /* QM_MCR_NP_RA1_*** */ + u16 ra2_sfdr; /* QM_MCR_NP_RA2_*** */ + u16 __reserved15; + u16 od1_sfdr; /* QM_MCR_NP_OD1_*** */ + u16 od2_sfdr; /* QM_MCR_NP_OD2_*** */ + u16 od3_sfdr; /* QM_MCR_NP_OD3_*** */ +#endif +} __packed; + +struct qm_mcr_alterfq { + u8 fqs; /* Frame Queue Status */ + u8 __reserved1[61]; +} __packed; +struct qm_mcr_initcgr { + u8 __reserved1[62]; +} __packed; +struct qm_mcr_cgrtestwrite { + u16 __reserved1; + struct __qm_mc_cgr cgr; /* CGR fields */ + u8 __reserved2[3]; + u32 __reserved3:24; + u32 i_bcnt_hi:8;/* high 8-bits of 40-bit "Instant" */ + u32 i_bcnt_lo; /* low 32-bits of 40-bit */ + u32 __reserved4:24; + u32 a_bcnt_hi:8;/* high 8-bits of 40-bit "Average" */ + u32 a_bcnt_lo; /* low 32-bits of 40-bit */ + u16 lgt; /* Last Group Tick */ + u16 wr_prob_g; + u16 wr_prob_y; + u16 wr_prob_r; + u8 __reserved5[8]; +} __packed; +struct qm_mcr_querycgr { + u16 __reserved1; + struct __qm_mc_cgr cgr; /* CGR fields */ + u8 __reserved2[3]; + union { + struct { +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + u32 __reserved3:24; + u32 i_bcnt_hi:8;/* high 8-bits of 40-bit "Instant" */ + u32 i_bcnt_lo; /* low 32-bits of 40-bit */ +#else + u32 i_bcnt_lo; /* low 32-bits of 40-bit */ + u32 i_bcnt_hi:8;/* high 8-bits of 40-bit "Instant" */ + u32 __reserved3:24; +#endif + }; + u64 i_bcnt; + }; + union { + struct { +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + u32 __reserved4:24; + u32 a_bcnt_hi:8;/* high 8-bits of 40-bit "Average" */ + u32 a_bcnt_lo; /* low 32-bits of 40-bit */ +#else + u32 a_bcnt_lo; /* low 32-bits of 40-bit */ + u32 a_bcnt_hi:8;/* high 8-bits of 40-bit "Average" */ + u32 __reserved4:24; +#endif + }; + u64 a_bcnt; + }; + union { + u32 cscn_targ_swp[4]; + u8 __reserved5[16]; + }; +} __packed; + +struct __qm_mcr_querycongestion { + u32 state[8]; +}; + +struct qm_mcr_querycongestion { + u8 __reserved[30]; + /* Access this struct using QM_MCR_QUERYCONGESTION() */ + struct __qm_mcr_querycongestion state; +} __packed; +struct qm_mcr_querywq { + union { + u16 channel_wq; /* ignores wq (3 lsbits) */ + struct { +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + u16 id:13; /* qm_channel */ + u16 __reserved:3; +#else + u16 __reserved:3; + u16 id:13; /* qm_channel */ +#endif + } __packed channel; + }; + u8 __reserved[28]; + u32 wq_len[8]; +} __packed; + +struct qm_mc_result { + u8 verb; + u8 result; + union { + struct qm_mcr_initfq initfq; + struct qm_mcr_queryfq queryfq; + struct qm_mcr_queryfq_np queryfq_np; + struct qm_mcr_alterfq alterfq; + struct qm_mcr_initcgr initcgr; + struct qm_mcr_cgrtestwrite cgrtestwrite; + struct qm_mcr_querycgr querycgr; + struct qm_mcr_querycongestion querycongestion; + struct qm_mcr_querywq querywq; + }; +} __packed; + +#define QM_MCR_VERB_RRID 0x80 +#define QM_MCR_VERB_MASK QM_MCC_VERB_MASK +#define QM_MCR_VERB_INITFQ_PARKED QM_MCC_VERB_INITFQ_PARKED +#define QM_MCR_VERB_INITFQ_SCHED QM_MCC_VERB_INITFQ_SCHED +#define QM_MCR_VERB_QUERYFQ QM_MCC_VERB_QUERYFQ +#define QM_MCR_VERB_QUERYFQ_NP QM_MCC_VERB_QUERYFQ_NP +#define QM_MCR_VERB_QUERYWQ QM_MCC_VERB_QUERYWQ +#define QM_MCR_VERB_QUERYWQ_DEDICATED QM_MCC_VERB_QUERYWQ_DEDICATED +#define QM_MCR_VERB_ALTER_SCHED QM_MCC_VERB_ALTER_SCHED +#define QM_MCR_VERB_ALTER_FE QM_MCC_VERB_ALTER_FE +#define QM_MCR_VERB_ALTER_RETIRE QM_MCC_VERB_ALTER_RETIRE +#define QM_MCR_VERB_ALTER_OOS QM_MCC_VERB_ALTER_OOS +#define QM_MCR_RESULT_NULL 0x00 +#define QM_MCR_RESULT_OK 0xf0 +#define QM_MCR_RESULT_ERR_FQID 0xf1 +#define QM_MCR_RESULT_ERR_FQSTATE 0xf2 +#define QM_MCR_RESULT_ERR_NOTEMPTY 0xf3 /* OOS fails if FQ is !empty */ +#define QM_MCR_RESULT_ERR_BADCHANNEL 0xf4 +#define QM_MCR_RESULT_PENDING 0xf8 +#define QM_MCR_RESULT_ERR_BADCOMMAND 0xff +#define QM_MCR_NP_STATE_FE 0x10 +#define QM_MCR_NP_STATE_R 0x08 +#define QM_MCR_NP_STATE_MASK 0x07 /* Reads FQD::STATE; */ +#define QM_MCR_NP_STATE_OOS 0x00 +#define QM_MCR_NP_STATE_RETIRED 0x01 +#define QM_MCR_NP_STATE_TEN_SCHED 0x02 +#define QM_MCR_NP_STATE_TRU_SCHED 0x03 +#define QM_MCR_NP_STATE_PARKED 0x04 +#define QM_MCR_NP_STATE_ACTIVE 0x05 +#define QM_MCR_NP_PTR_MASK 0x07ff /* for RA[12] & OD[123] */ +#define QM_MCR_NP_RA1_NRA(v) (((v) >> 14) & 0x3) /* FQD::NRA */ +#define QM_MCR_NP_RA2_IT(v) (((v) >> 14) & 0x1) /* FQD::IT */ +#define QM_MCR_NP_OD1_NOD(v) (((v) >> 14) & 0x3) /* FQD::NOD */ +#define QM_MCR_NP_OD3_NPC(v) (((v) >> 14) & 0x3) /* FQD::NPC */ +#define QM_MCR_FQS_ORLPRESENT 0x02 /* ORL fragments to come */ +#define QM_MCR_FQS_NOTEMPTY 0x01 /* FQ has enqueued frames */ +/* This extracts the state for congestion group 'n' from a query response. + * Eg. + * u8 cgr = [...]; + * struct qm_mc_result *res = [...]; + * printf("congestion group %d congestion state: %d\n", cgr, + * QM_MCR_QUERYCONGESTION(&res->querycongestion.state, cgr)); + */ +#define __CGR_WORD(num) (num >> 5) +#define __CGR_SHIFT(num) (num & 0x1f) +#define __CGR_NUM (sizeof(struct __qm_mcr_querycongestion) << 3) +static inline int QM_MCR_QUERYCONGESTION(struct __qm_mcr_querycongestion *p, + u8 cgr) +{ + return p->state[__CGR_WORD(cgr)] & (0x80000000 >> __CGR_SHIFT(cgr)); +} + + /* Portal and Frame Queues */ +/* Represents a managed portal */ +struct qman_portal; + +/* + * This object type represents QMan frame queue descriptors (FQD), it is + * cacheline-aligned, and initialised by qman_create_fq(). The structure is + * defined further down. + */ +struct qman_fq; + +/* + * This object type represents a QMan congestion group, it is defined further + * down. + */ +struct qman_cgr; + +/* + * This enum, and the callback type that returns it, are used when handling + * dequeued frames via DQRR. Note that for "null" callbacks registered with the + * portal object (for handling dequeues that do not demux because context_b is + * NULL), the return value *MUST* be qman_cb_dqrr_consume. + */ +enum qman_cb_dqrr_result { + /* DQRR entry can be consumed */ + qman_cb_dqrr_consume, + /* Like _consume, but requests parking - FQ must be held-active */ + qman_cb_dqrr_park, + /* Does not consume, for DCA mode only. This allows out-of-order + * consumes by explicit calls to qman_dca() and/or the use of implicit + * DCA via EQCR entries. + */ + qman_cb_dqrr_defer, + /* + * Stop processing without consuming this ring entry. Exits the current + * qman_p_poll_dqrr() or interrupt-handling, as appropriate. If within + * an interrupt handler, the callback would typically call + * qman_irqsource_remove(QM_PIRQ_DQRI) before returning this value, + * otherwise the interrupt will reassert immediately. + */ + qman_cb_dqrr_stop, + /* Like qman_cb_dqrr_stop, but consumes the current entry. */ + qman_cb_dqrr_consume_stop +}; + +typedef enum qman_cb_dqrr_result (*qman_cb_dqrr)(struct qman_portal *qm, + struct qman_fq *fq, + const struct qm_dqrr_entry *dqrr); + +/* + * This callback type is used when handling ERNs, FQRNs and FQRLs via MR. They + * are always consumed after the callback returns. + */ +typedef void (*qman_cb_mr)(struct qman_portal *qm, struct qman_fq *fq, + const struct qm_mr_entry *msg); + +/* This callback type is used when handling DCP ERNs */ +typedef void (*qman_cb_dc_ern)(struct qman_portal *qm, + const struct qm_mr_entry *msg); +/* + * s/w-visible states. Ie. tentatively scheduled + truly scheduled + active + + * held-active + held-suspended are just "sched". Things like "retired" will not + * be assumed until it is complete (ie. QMAN_FQ_STATE_CHANGING is set until + * then, to indicate it's completing and to gate attempts to retry the retire + * command). Note, park commands do not set QMAN_FQ_STATE_CHANGING because it's + * technically impossible in the case of enqueue DCAs (which refer to DQRR ring + * index rather than the FQ that ring entry corresponds to), so repeated park + * commands are allowed (if you're silly enough to try) but won't change FQ + * state, and the resulting park notifications move FQs from "sched" to + * "parked". + */ +enum qman_fq_state { + qman_fq_state_oos, + qman_fq_state_parked, + qman_fq_state_sched, + qman_fq_state_retired +}; + + +/* + * Frame queue objects (struct qman_fq) are stored within memory passed to + * qman_create_fq(), as this allows stashing of caller-provided demux callback + * pointers at no extra cost to stashing of (driver-internal) FQ state. If the + * caller wishes to add per-FQ state and have it benefit from dequeue-stashing, + * they should; + * + * (a) extend the qman_fq structure with their state; eg. + * + * // myfq is allocated and driver_fq callbacks filled in; + * struct my_fq { + * struct qman_fq base; + * int an_extra_field; + * [ ... add other fields to be associated with each FQ ...] + * } *myfq = some_my_fq_allocator(); + * struct qman_fq *fq = qman_create_fq(fqid, flags, &myfq->base); + * + * // in a dequeue callback, access extra fields from 'fq' via a cast; + * struct my_fq *myfq = (struct my_fq *)fq; + * do_something_with(myfq->an_extra_field); + * [...] + * + * (b) when and if configuring the FQ for context stashing, specify how ever + * many cachelines are required to stash 'struct my_fq', to accelerate not + * only the QMan driver but the callback as well. + */ + +struct qman_fq_cb { + qman_cb_dqrr dqrr; /* for dequeued frames */ + qman_cb_mr ern; /* for s/w ERNs */ + qman_cb_mr fqs; /* frame-queue state changes*/ +}; + +struct qman_fq { + /* Caller of qman_create_fq() provides these demux callbacks */ + struct qman_fq_cb cb; + /* + * These are internal to the driver, don't touch. In particular, they + * may change, be removed, or extended (so you shouldn't rely on + * sizeof(qman_fq) being a constant). + */ + spinlock_t fqlock; + u32 fqid; + /* DPDK Interface */ + void *dpaa_intf; + + volatile unsigned long flags; + enum qman_fq_state state; + int cgr_groupid; + struct rb_node node; +#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP + u32 key; +#endif +}; + +/* + * This callback type is used when handling congestion group entry/exit. + * 'congested' is non-zero on congestion-entry, and zero on congestion-exit. + */ +typedef void (*qman_cb_cgr)(struct qman_portal *qm, + struct qman_cgr *cgr, int congested); + +struct qman_cgr { + /* Set these prior to qman_create_cgr() */ + u32 cgrid; /* 0..255, but u32 to allow specials like -1, 256, etc.*/ + qman_cb_cgr cb; + /* These are private to the driver */ + u16 chan; /* portal channel this object is created on */ + struct list_head node; +}; + +/* Flags to qman_create_fq() */ +#define QMAN_FQ_FLAG_NO_ENQUEUE 0x00000001 /* can't enqueue */ +#define QMAN_FQ_FLAG_NO_MODIFY 0x00000002 /* can only enqueue */ +#define QMAN_FQ_FLAG_TO_DCPORTAL 0x00000004 /* consumed by CAAM/PME/Fman */ +#define QMAN_FQ_FLAG_LOCKED 0x00000008 /* multi-core locking */ +#define QMAN_FQ_FLAG_AS_IS 0x00000010 /* query h/w state */ +#define QMAN_FQ_FLAG_DYNAMIC_FQID 0x00000020 /* (de)allocate fqid */ + +/* Flags to qman_destroy_fq() */ +#define QMAN_FQ_DESTROY_PARKED 0x00000001 /* FQ can be parked or OOS */ + +/* Flags from qman_fq_state() */ +#define QMAN_FQ_STATE_CHANGING 0x80000000 /* 'state' is changing */ +#define QMAN_FQ_STATE_NE 0x40000000 /* retired FQ isn't empty */ +#define QMAN_FQ_STATE_ORL 0x20000000 /* retired FQ has ORL */ +#define QMAN_FQ_STATE_BLOCKOOS 0xe0000000 /* if any are set, no OOS */ +#define QMAN_FQ_STATE_CGR_EN 0x10000000 /* CGR enabled */ +#define QMAN_FQ_STATE_VDQCR 0x08000000 /* being volatile dequeued */ + +/* Flags to qman_init_fq() */ +#define QMAN_INITFQ_FLAG_SCHED 0x00000001 /* schedule rather than park */ +#define QMAN_INITFQ_FLAG_LOCAL 0x00000004 /* set dest portal */ + +/* Flags to qman_enqueue(). NB, the strange numbering is to align with hardware, + * bit-wise. (NB: the PME API is sensitive to these precise numberings too, so + * any change here should be audited in PME.) + */ +#define QMAN_ENQUEUE_FLAG_WATCH_CGR 0x00080000 /* watch congestion state */ +#define QMAN_ENQUEUE_FLAG_DCA 0x00008000 /* perform enqueue-DCA */ +#define QMAN_ENQUEUE_FLAG_DCA_PARK 0x00004000 /* If DCA, requests park */ +#define QMAN_ENQUEUE_FLAG_DCA_PTR(p) /* If DCA, p is DQRR entry */ \ + (((u32)(p) << 2) & 0x00000f00) +#define QMAN_ENQUEUE_FLAG_C_GREEN 0x00000000 /* choose one C_*** flag */ +#define QMAN_ENQUEUE_FLAG_C_YELLOW 0x00000008 +#define QMAN_ENQUEUE_FLAG_C_RED 0x00000010 +#define QMAN_ENQUEUE_FLAG_C_OVERRIDE 0x00000018 +/* For the ORP-specific qman_enqueue_orp() variant; + * - this flag indicates "Not Last In Sequence", ie. all but the final fragment + * of a frame. + */ +#define QMAN_ENQUEUE_FLAG_NLIS 0x01000000 +/* - this flag performs no enqueue but fills in an ORP sequence number that + * would otherwise block it (eg. if a frame has been dropped). + */ +#define QMAN_ENQUEUE_FLAG_HOLE 0x02000000 +/* - this flag performs no enqueue but advances NESN to the given sequence + * number. + */ +#define QMAN_ENQUEUE_FLAG_NESN 0x04000000 + +/* Flags to qman_modify_cgr() */ +#define QMAN_CGR_FLAG_USE_INIT 0x00000001 +#define QMAN_CGR_MODE_FRAME 0x00000001 + +/** + * qman_get_portal_index - get portal configuration index + */ +int qman_get_portal_index(void); + +/** + * qman_affine_channel - return the channel ID of an portal + * @cpu: the cpu whose affine portal is the subject of the query + * + * If @cpu is -1, the affine portal for the current CPU will be used. It is a + * bug to call this function for any value of @cpu (other than -1) that is not a + * member of the cpu mask. + */ +u16 qman_affine_channel(int cpu); + +/** + * qman_set_vdq - Issue a volatile dequeue command + * @fq: Frame Queue on which the volatile dequeue command is issued + * @num: Number of Frames requested for volatile dequeue + * + * This function will issue a volatile dequeue command to the QMAN. + */ +int qman_set_vdq(struct qman_fq *fq, u16 num); + +/** + * qman_dequeue - Get the DQRR entry after volatile dequeue command + * @fq: Frame Queue on which the volatile dequeue command is issued + * + * This function will return the DQRR entry after a volatile dequeue command + * is issued. It will keep returning NULL until there is no packet available on + * the DQRR. + */ +struct qm_dqrr_entry *qman_dequeue(struct qman_fq *fq); + +/** + * qman_dqrr_consume - Consume the DQRR entriy after volatile dequeue + * @fq: Frame Queue on which the volatile dequeue command is issued + * @dq: DQRR entry to consume. This is the one which is provided by the + * 'qbman_dequeue' command. + * + * This will consume the DQRR enrey and make it available for next volatile + * dequeue. + */ +void qman_dqrr_consume(struct qman_fq *fq, + struct qm_dqrr_entry *dq); + +/** + * qman_poll_dqrr - process DQRR (fast-path) entries + * @limit: the maximum number of DQRR entries to process + * + * Use of this function requires that DQRR processing not be interrupt-driven. + * Ie. the value returned by qman_irqsource_get() should not include + * QM_PIRQ_DQRI. If the current CPU is sharing a portal hosted on another CPU, + * this function will return -EINVAL, otherwise the return value is >=0 and + * represents the number of DQRR entries processed. + */ +int qman_poll_dqrr(unsigned int limit); + +/** + * qman_poll + * + * Dispatcher logic on a cpu can use this to trigger any maintenance of the + * affine portal. There are two classes of portal processing in question; + * fast-path (which involves demuxing dequeue ring (DQRR) entries and tracking + * enqueue ring (EQCR) consumption), and slow-path (which involves EQCR + * thresholds, congestion state changes, etc). This function does whatever + * processing is not triggered by interrupts. + * + * Note, if DQRR and some slow-path processing are poll-driven (rather than + * interrupt-driven) then this function uses a heuristic to determine how often + * to run slow-path processing - as slow-path processing introduces at least a + * minimum latency each time it is run, whereas fast-path (DQRR) processing is + * close to zero-cost if there is no work to be done. + */ +void qman_poll(void); + +/** + * qman_stop_dequeues - Stop h/w dequeuing to the s/w portal + * + * Disables DQRR processing of the portal. This is reference-counted, so + * qman_start_dequeues() must be called as many times as qman_stop_dequeues() to + * truly re-enable dequeuing. + */ +void qman_stop_dequeues(void); + +/** + * qman_start_dequeues - (Re)start h/w dequeuing to the s/w portal + * + * Enables DQRR processing of the portal. This is reference-counted, so + * qman_start_dequeues() must be called as many times as qman_stop_dequeues() to + * truly re-enable dequeuing. + */ +void qman_start_dequeues(void); + +/** + * qman_static_dequeue_add - Add pool channels to the portal SDQCR + * @pools: bit-mask of pool channels, using QM_SDQCR_CHANNELS_POOL(n) + * + * Adds a set of pool channels to the portal's static dequeue command register + * (SDQCR). The requested pools are limited to those the portal has dequeue + * access to. + */ +void qman_static_dequeue_add(u32 pools); + +/** + * qman_static_dequeue_del - Remove pool channels from the portal SDQCR + * @pools: bit-mask of pool channels, using QM_SDQCR_CHANNELS_POOL(n) + * + * Removes a set of pool channels from the portal's static dequeue command + * register (SDQCR). The requested pools are limited to those the portal has + * dequeue access to. + */ +void qman_static_dequeue_del(u32 pools); + +/** + * qman_static_dequeue_get - return the portal's current SDQCR + * + * Returns the portal's current static dequeue command register (SDQCR). The + * entire register is returned, so if only the currently-enabled pool channels + * are desired, mask the return value with QM_SDQCR_CHANNELS_POOL_MASK. + */ +u32 qman_static_dequeue_get(void); + +/** + * qman_dca - Perform a Discrete Consumption Acknowledgment + * @dq: the DQRR entry to be consumed + * @park_request: indicates whether the held-active @fq should be parked + * + * Only allowed in DCA-mode portals, for DQRR entries whose handler callback had + * previously returned 'qman_cb_dqrr_defer'. NB, as with the other APIs, this + * does not take a 'portal' argument but implies the core affine portal from the + * cpu that is currently executing the function. For reasons of locking, this + * function must be called from the same CPU as that which processed the DQRR + * entry in the first place. + */ +void qman_dca(struct qm_dqrr_entry *dq, int park_request); + +/** + * qman_eqcr_is_empty - Determine if portal's EQCR is empty + * + * For use in situations where a cpu-affine caller needs to determine when all + * enqueues for the local portal have been processed by Qman but can't use the + * QMAN_ENQUEUE_FLAG_WAIT_SYNC flag to do this from the final qman_enqueue(). + * The function forces tracking of EQCR consumption (which normally doesn't + * happen until enqueue processing needs to find space to put new enqueue + * commands), and returns zero if the ring still has unprocessed entries, + * non-zero if it is empty. + */ +int qman_eqcr_is_empty(void); + +/** + * qman_set_dc_ern - Set the handler for DCP enqueue rejection notifications + * @handler: callback for processing DCP ERNs + * @affine: whether this handler is specific to the locally affine portal + * + * If a hardware block's interface to Qman (ie. its direct-connect portal, or + * DCP) is configured not to receive enqueue rejections, then any enqueues + * through that DCP that are rejected will be sent to a given software portal. + * If @affine is non-zero, then this handler will only be used for DCP ERNs + * received on the portal affine to the current CPU. If multiple CPUs share a + * portal and they all call this function, they will be setting the handler for + * the same portal! If @affine is zero, then this handler will be global to all + * portals handled by this instance of the driver. Only those portals that do + * not have their own affine handler will use the global handler. + */ +void qman_set_dc_ern(qman_cb_dc_ern handler, int affine); + + /* FQ management */ + /* ------------- */ +/** + * qman_create_fq - Allocates a FQ + * @fqid: the index of the FQD to encapsulate, must be "Out of Service" + * @flags: bit-mask of QMAN_FQ_FLAG_*** options + * @fq: memory for storing the 'fq', with callbacks filled in + * + * Creates a frame queue object for the given @fqid, unless the + * QMAN_FQ_FLAG_DYNAMIC_FQID flag is set in @flags, in which case a FQID is + * dynamically allocated (or the function fails if none are available). Once + * created, the caller should not touch the memory at 'fq' except as extended to + * adjacent memory for user-defined fields (see the definition of "struct + * qman_fq" for more info). NO_MODIFY is only intended for enqueuing to + * pre-existing frame-queues that aren't to be otherwise interfered with, it + * prevents all other modifications to the frame queue. The TO_DCPORTAL flag + * causes the driver to honour any contextB modifications requested in the + * qm_init_fq() API, as this indicates the frame queue will be consumed by a + * direct-connect portal (PME, CAAM, or Fman). When frame queues are consumed by + * software portals, the contextB field is controlled by the driver and can't be + * modified by the caller. If the AS_IS flag is specified, management commands + * will be used on portal @p to query state for frame queue @fqid and construct + * a frame queue object based on that, rather than assuming/requiring that it be + * Out of Service. + */ +int qman_create_fq(u32 fqid, u32 flags, struct qman_fq *fq); + +/** + * qman_destroy_fq - Deallocates a FQ + * @fq: the frame queue object to release + * @flags: bit-mask of QMAN_FQ_FREE_*** options + * + * The memory for this frame queue object ('fq' provided in qman_create_fq()) is + * not deallocated but the caller regains ownership, to do with as desired. The + * FQ must be in the 'out-of-service' state unless the QMAN_FQ_FREE_PARKED flag + * is specified, in which case it may also be in the 'parked' state. + */ +void qman_destroy_fq(struct qman_fq *fq, u32 flags); + +/** + * qman_fq_fqid - Queries the frame queue ID of a FQ object + * @fq: the frame queue object to query + */ +u32 qman_fq_fqid(struct qman_fq *fq); + +/** + * qman_fq_state - Queries the state of a FQ object + * @fq: the frame queue object to query + * @state: pointer to state enum to return the FQ scheduling state + * @flags: pointer to state flags to receive QMAN_FQ_STATE_*** bitmask + * + * Queries the state of the FQ object, without performing any h/w commands. + * This captures the state, as seen by the driver, at the time the function + * executes. + */ +void qman_fq_state(struct qman_fq *fq, enum qman_fq_state *state, u32 *flags); + +/** + * qman_init_fq - Initialises FQ fields, leaves the FQ "parked" or "scheduled" + * @fq: the frame queue object to modify, must be 'parked' or new. + * @flags: bit-mask of QMAN_INITFQ_FLAG_*** options + * @opts: the FQ-modification settings, as defined in the low-level API + * + * The @opts parameter comes from the low-level portal API. Select + * QMAN_INITFQ_FLAG_SCHED in @flags to cause the frame queue to be scheduled + * rather than parked. NB, @opts can be NULL. + * + * Note that some fields and options within @opts may be ignored or overwritten + * by the driver; + * 1. the 'count' and 'fqid' fields are always ignored (this operation only + * affects one frame queue: @fq). + * 2. the QM_INITFQ_WE_CONTEXTB option of the 'we_mask' field and the associated + * 'fqd' structure's 'context_b' field are sometimes overwritten; + * - if @fq was not created with QMAN_FQ_FLAG_TO_DCPORTAL, then context_b is + * initialised to a value used by the driver for demux. + * - if context_b is initialised for demux, so is context_a in case stashing + * is requested (see item 4). + * (So caller control of context_b is only possible for TO_DCPORTAL frame queue + * objects.) + * 3. if @flags contains QMAN_INITFQ_FLAG_LOCAL, the 'fqd' structure's + * 'dest::channel' field will be overwritten to match the portal used to issue + * the command. If the WE_DESTWQ write-enable bit had already been set by the + * caller, the channel workqueue will be left as-is, otherwise the write-enable + * bit is set and the workqueue is set to a default of 4. If the "LOCAL" flag + * isn't set, the destination channel/workqueue fields and the write-enable bit + * are left as-is. + * 4. if the driver overwrites context_a/b for demux, then if + * QM_INITFQ_WE_CONTEXTA is set, the driver will only overwrite + * context_a.address fields and will leave the stashing fields provided by the + * user alone, otherwise it will zero out the context_a.stashing fields. + */ +int qman_init_fq(struct qman_fq *fq, u32 flags, struct qm_mcc_initfq *opts); + +/** + * qman_schedule_fq - Schedules a FQ + * @fq: the frame queue object to schedule, must be 'parked' + * + * Schedules the frame queue, which must be Parked, which takes it to + * Tentatively-Scheduled or Truly-Scheduled depending on its fill-level. + */ +int qman_schedule_fq(struct qman_fq *fq); + +/** + * qman_retire_fq - Retires a FQ + * @fq: the frame queue object to retire + * @flags: FQ flags (as per qman_fq_state) if retirement completes immediately + * + * Retires the frame queue. This returns zero if it succeeds immediately, +1 if + * the retirement was started asynchronously, otherwise it returns negative for + * failure. When this function returns zero, @flags is set to indicate whether + * the retired FQ is empty and/or whether it has any ORL fragments (to show up + * as ERNs). Otherwise the corresponding flags will be known when a subsequent + * FQRN message shows up on the portal's message ring. + * + * NB, if the retirement is asynchronous (the FQ was in the Truly Scheduled or + * Active state), the completion will be via the message ring as a FQRN - but + * the corresponding callback may occur before this function returns!! Ie. the + * caller should be prepared to accept the callback as the function is called, + * not only once it has returned. + */ +int qman_retire_fq(struct qman_fq *fq, u32 *flags); + +/** + * qman_oos_fq - Puts a FQ "out of service" + * @fq: the frame queue object to be put out-of-service, must be 'retired' + * + * The frame queue must be retired and empty, and if any order restoration list + * was released as ERNs at the time of retirement, they must all be consumed. + */ +int qman_oos_fq(struct qman_fq *fq); + +/** + * qman_fq_flow_control - Set the XON/XOFF state of a FQ + * @fq: the frame queue object to be set to XON/XOFF state, must not be 'oos', + * or 'retired' or 'parked' state + * @xon: boolean to set fq in XON or XOFF state + * + * The frame should be in Tentatively Scheduled state or Truly Schedule sate, + * otherwise the IFSI interrupt will be asserted. + */ +int qman_fq_flow_control(struct qman_fq *fq, int xon); + +/** + * qman_query_fq - Queries FQD fields (via h/w query command) + * @fq: the frame queue object to be queried + * @fqd: storage for the queried FQD fields + */ +int qman_query_fq(struct qman_fq *fq, struct qm_fqd *fqd); + +/** + * qman_query_fq_has_pkts - Queries non-programmable FQD fields and returns '1' + * if packets are in the frame queue. If there are no packets on frame + * queue '0' is returned. + * @fq: the frame queue object to be queried + */ +int qman_query_fq_has_pkts(struct qman_fq *fq); + +/** + * qman_query_fq_np - Queries non-programmable FQD fields + * @fq: the frame queue object to be queried + * @np: storage for the queried FQD fields + */ +int qman_query_fq_np(struct qman_fq *fq, struct qm_mcr_queryfq_np *np); + +/** + * qman_query_wq - Queries work queue lengths + * @query_dedicated: If non-zero, query length of WQs in the channel dedicated + * to this software portal. Otherwise, query length of WQs in a + * channel specified in wq. + * @wq: storage for the queried WQs lengths. Also specified the channel to + * to query if query_dedicated is zero. + */ +int qman_query_wq(u8 query_dedicated, struct qm_mcr_querywq *wq); + +/** + * qman_volatile_dequeue - Issue a volatile dequeue command + * @fq: the frame queue object to dequeue from + * @flags: a bit-mask of QMAN_VOLATILE_FLAG_*** options + * @vdqcr: bit mask of QM_VDQCR_*** options, as per qm_dqrr_vdqcr_set() + * + * Attempts to lock access to the portal's VDQCR volatile dequeue functionality. + * The function will block and sleep if QMAN_VOLATILE_FLAG_WAIT is specified and + * the VDQCR is already in use, otherwise returns non-zero for failure. If + * QMAN_VOLATILE_FLAG_FINISH is specified, the function will only return once + * the VDQCR command has finished executing (ie. once the callback for the last + * DQRR entry resulting from the VDQCR command has been called). If not using + * the FINISH flag, completion can be determined either by detecting the + * presence of the QM_DQRR_STAT_UNSCHEDULED and QM_DQRR_STAT_DQCR_EXPIRED bits + * in the "stat" field of the "struct qm_dqrr_entry" passed to the FQ's dequeue + * callback, or by waiting for the QMAN_FQ_STATE_VDQCR bit to disappear from the + * "flags" retrieved from qman_fq_state(). + */ +int qman_volatile_dequeue(struct qman_fq *fq, u32 flags, u32 vdqcr); + +/** + * qman_enqueue - Enqueue a frame to a frame queue + * @fq: the frame queue object to enqueue to + * @fd: a descriptor of the frame to be enqueued + * @flags: bit-mask of QMAN_ENQUEUE_FLAG_*** options + * + * Fills an entry in the EQCR of portal @qm to enqueue the frame described by + * @fd. The descriptor details are copied from @fd to the EQCR entry, the 'pid' + * field is ignored. The return value is non-zero on error, such as ring full + * (and FLAG_WAIT not specified), congestion avoidance (FLAG_WATCH_CGR + * specified), etc. If the ring is full and FLAG_WAIT is specified, this + * function will block. If FLAG_INTERRUPT is set, the EQCI bit of the portal + * interrupt will assert when Qman consumes the EQCR entry (subject to "status + * disable", "enable", and "inhibit" registers). If FLAG_DCA is set, Qman will + * perform an implied "discrete consumption acknowledgment" on the dequeue + * ring's (DQRR) entry, at the ring index specified by the FLAG_DCA_IDX(x) + * macro. (As an alternative to issuing explicit DCA actions on DQRR entries, + * this implicit DCA can delay the release of a "held active" frame queue + * corresponding to a DQRR entry until Qman consumes the EQCR entry - providing + * order-preservation semantics in packet-forwarding scenarios.) If FLAG_DCA is + * set, then FLAG_DCA_PARK can also be set to imply that the DQRR consumption + * acknowledgment should "park request" the "held active" frame queue. Ie. + * when the portal eventually releases that frame queue, it will be left in the + * Parked state rather than Tentatively Scheduled or Truly Scheduled. If the + * portal is watching congestion groups, the QMAN_ENQUEUE_FLAG_WATCH_CGR flag + * is requested, and the FQ is a member of a congestion group, then this + * function returns -EAGAIN if the congestion group is currently congested. + * Note, this does not eliminate ERNs, as the async interface means we can be + * sending enqueue commands to an un-congested FQ that becomes congested before + * the enqueue commands are processed, but it does minimise needless thrashing + * of an already busy hardware resource by throttling many of the to-be-dropped + * enqueues "at the source". + */ +int qman_enqueue(struct qman_fq *fq, const struct qm_fd *fd, u32 flags); + +int qman_enqueue_multi(struct qman_fq *fq, + const struct qm_fd *fd, + int frames_to_send); + +typedef int (*qman_cb_precommit) (void *arg); + +/** + * qman_enqueue_orp - Enqueue a frame to a frame queue using an ORP + * @fq: the frame queue object to enqueue to + * @fd: a descriptor of the frame to be enqueued + * @flags: bit-mask of QMAN_ENQUEUE_FLAG_*** options + * @orp: the frame queue object used as an order restoration point. + * @orp_seqnum: the sequence number of this frame in the order restoration path + * + * Similar to qman_enqueue(), but with the addition of an Order Restoration + * Point (@orp) and corresponding sequence number (@orp_seqnum) for this + * enqueue operation to employ order restoration. Each frame queue object acts + * as an Order Definition Point (ODP) by providing each frame dequeued from it + * with an incrementing sequence number, this value is generally ignored unless + * that sequence of dequeued frames will need order restoration later. Each + * frame queue object also encapsulates an Order Restoration Point (ORP), which + * is a re-assembly context for re-ordering frames relative to their sequence + * numbers as they are enqueued. The ORP does not have to be within the frame + * queue that receives the enqueued frame, in fact it is usually the frame + * queue from which the frames were originally dequeued. For the purposes of + * order restoration, multiple frames (or "fragments") can be enqueued for a + * single sequence number by setting the QMAN_ENQUEUE_FLAG_NLIS flag for all + * enqueues except the final fragment of a given sequence number. Ordering + * between sequence numbers is guaranteed, even if fragments of different + * sequence numbers are interlaced with one another. Fragments of the same + * sequence number will retain the order in which they are enqueued. If no + * enqueue is to performed, QMAN_ENQUEUE_FLAG_HOLE indicates that the given + * sequence number is to be "skipped" by the ORP logic (eg. if a frame has been + * dropped from a sequence), or QMAN_ENQUEUE_FLAG_NESN indicates that the given + * sequence number should become the ORP's "Next Expected Sequence Number". + * + * Side note: a frame queue object can be used purely as an ORP, without + * carrying any frames at all. Care should be taken not to deallocate a frame + * queue object that is being actively used as an ORP, as a future allocation + * of the frame queue object may start using the internal ORP before the + * previous use has finished. + */ +int qman_enqueue_orp(struct qman_fq *fq, const struct qm_fd *fd, u32 flags, + struct qman_fq *orp, u16 orp_seqnum); + +/** + * qman_alloc_fqid_range - Allocate a contiguous range of FQIDs + * @result: is set by the API to the base FQID of the allocated range + * @count: the number of FQIDs required + * @align: required alignment of the allocated range + * @partial: non-zero if the API can return fewer than @count FQIDs + * + * Returns the number of frame queues allocated, or a negative error code. If + * @partial is non zero, the allocation request may return a smaller range of + * FQs than requested (though alignment will be as requested). If @partial is + * zero, the return value will either be 'count' or negative. + */ +int qman_alloc_fqid_range(u32 *result, u32 count, u32 align, int partial); +static inline int qman_alloc_fqid(u32 *result) +{ + int ret = qman_alloc_fqid_range(result, 1, 0, 0); + + return (ret > 0) ? 0 : ret; +} + +/** + * qman_release_fqid_range - Release the specified range of frame queue IDs + * @fqid: the base FQID of the range to deallocate + * @count: the number of FQIDs in the range + * + * This function can also be used to seed the allocator with ranges of FQIDs + * that it can subsequently allocate from. + */ +void qman_release_fqid_range(u32 fqid, unsigned int count); +static inline void qman_release_fqid(u32 fqid) +{ + qman_release_fqid_range(fqid, 1); +} + +void qman_seed_fqid_range(u32 fqid, unsigned int count); + +int qman_shutdown_fq(u32 fqid); + +/** + * qman_reserve_fqid_range - Reserve the specified range of frame queue IDs + * @fqid: the base FQID of the range to deallocate + * @count: the number of FQIDs in the range + */ +int qman_reserve_fqid_range(u32 fqid, unsigned int count); +static inline int qman_reserve_fqid(u32 fqid) +{ + return qman_reserve_fqid_range(fqid, 1); +} + +/* Pool-channel management */ +/** + * qman_alloc_pool_range - Allocate a contiguous range of pool-channel IDs + * @result: is set by the API to the base pool-channel ID of the allocated range + * @count: the number of pool-channel IDs required + * @align: required alignment of the allocated range + * @partial: non-zero if the API can return fewer than @count + * + * Returns the number of pool-channel IDs allocated, or a negative error code. + * If @partial is non zero, the allocation request may return a smaller range of + * than requested (though alignment will be as requested). If @partial is zero, + * the return value will either be 'count' or negative. + */ +int qman_alloc_pool_range(u32 *result, u32 count, u32 align, int partial); +static inline int qman_alloc_pool(u32 *result) +{ + int ret = qman_alloc_pool_range(result, 1, 0, 0); + + return (ret > 0) ? 0 : ret; +} + +/** + * qman_release_pool_range - Release the specified range of pool-channel IDs + * @id: the base pool-channel ID of the range to deallocate + * @count: the number of pool-channel IDs in the range + */ +void qman_release_pool_range(u32 id, unsigned int count); +static inline void qman_release_pool(u32 id) +{ + qman_release_pool_range(id, 1); +} + +/** + * qman_reserve_pool_range - Reserve the specified range of pool-channel IDs + * @id: the base pool-channel ID of the range to reserve + * @count: the number of pool-channel IDs in the range + */ +int qman_reserve_pool_range(u32 id, unsigned int count); +static inline int qman_reserve_pool(u32 id) +{ + return qman_reserve_pool_range(id, 1); +} + +void qman_seed_pool_range(u32 id, unsigned int count); + + /* CGR management */ + /* -------------- */ +/** + * qman_create_cgr - Register a congestion group object + * @cgr: the 'cgr' object, with fields filled in + * @flags: QMAN_CGR_FLAG_* values + * @opts: optional state of CGR settings + * + * Registers this object to receiving congestion entry/exit callbacks on the + * portal affine to the cpu portal on which this API is executed. If opts is + * NULL then only the callback (cgr->cb) function is registered. If @flags + * contains QMAN_CGR_FLAG_USE_INIT, then an init hw command (which will reset + * any unspecified parameters) will be used rather than a modify hw hardware + * (which only modifies the specified parameters). + */ +int qman_create_cgr(struct qman_cgr *cgr, u32 flags, + struct qm_mcc_initcgr *opts); + +/** + * qman_create_cgr_to_dcp - Register a congestion group object to DCP portal + * @cgr: the 'cgr' object, with fields filled in + * @flags: QMAN_CGR_FLAG_* values + * @dcp_portal: the DCP portal to which the cgr object is registered. + * @opts: optional state of CGR settings + * + */ +int qman_create_cgr_to_dcp(struct qman_cgr *cgr, u32 flags, u16 dcp_portal, + struct qm_mcc_initcgr *opts); + +/** + * qman_delete_cgr - Deregisters a congestion group object + * @cgr: the 'cgr' object to deregister + * + * "Unplugs" this CGR object from the portal affine to the cpu on which this API + * is executed. This must be excuted on the same affine portal on which it was + * created. + */ +int qman_delete_cgr(struct qman_cgr *cgr); + +/** + * qman_modify_cgr - Modify CGR fields + * @cgr: the 'cgr' object to modify + * @flags: QMAN_CGR_FLAG_* values + * @opts: the CGR-modification settings + * + * The @opts parameter comes from the low-level portal API, and can be NULL. + * Note that some fields and options within @opts may be ignored or overwritten + * by the driver, in particular the 'cgrid' field is ignored (this operation + * only affects the given CGR object). If @flags contains + * QMAN_CGR_FLAG_USE_INIT, then an init hw command (which will reset any + * unspecified parameters) will be used rather than a modify hw hardware (which + * only modifies the specified parameters). + */ +int qman_modify_cgr(struct qman_cgr *cgr, u32 flags, + struct qm_mcc_initcgr *opts); + +/** + * qman_query_cgr - Queries CGR fields + * @cgr: the 'cgr' object to query + * @result: storage for the queried congestion group record + */ +int qman_query_cgr(struct qman_cgr *cgr, struct qm_mcr_querycgr *result); + +/** + * qman_query_congestion - Queries the state of all congestion groups + * @congestion: storage for the queried state of all congestion groups + */ +int qman_query_congestion(struct qm_mcr_querycongestion *congestion); + +/** + * qman_alloc_cgrid_range - Allocate a contiguous range of CGR IDs + * @result: is set by the API to the base CGR ID of the allocated range + * @count: the number of CGR IDs required + * @align: required alignment of the allocated range + * @partial: non-zero if the API can return fewer than @count + * + * Returns the number of CGR IDs allocated, or a negative error code. + * If @partial is non zero, the allocation request may return a smaller range of + * than requested (though alignment will be as requested). If @partial is zero, + * the return value will either be 'count' or negative. + */ +int qman_alloc_cgrid_range(u32 *result, u32 count, u32 align, int partial); +static inline int qman_alloc_cgrid(u32 *result) +{ + int ret = qman_alloc_cgrid_range(result, 1, 0, 0); + + return (ret > 0) ? 0 : ret; +} + +/** + * qman_release_cgrid_range - Release the specified range of CGR IDs + * @id: the base CGR ID of the range to deallocate + * @count: the number of CGR IDs in the range + */ +void qman_release_cgrid_range(u32 id, unsigned int count); +static inline void qman_release_cgrid(u32 id) +{ + qman_release_cgrid_range(id, 1); +} + +/** + * qman_reserve_cgrid_range - Reserve the specified range of CGR ID + * @id: the base CGR ID of the range to reserve + * @count: the number of CGR IDs in the range + */ +int qman_reserve_cgrid_range(u32 id, unsigned int count); +static inline int qman_reserve_cgrid(u32 id) +{ + return qman_reserve_cgrid_range(id, 1); +} + +void qman_seed_cgrid_range(u32 id, unsigned int count); + + /* Helpers */ + /* ------- */ +/** + * qman_poll_fq_for_init - Check if an FQ has been initialised from OOS + * @fqid: the FQID that will be initialised by other s/w + * + * In many situations, a FQID is provided for communication between s/w + * entities, and whilst the consumer is responsible for initialising and + * scheduling the FQ, the producer(s) generally create a wrapper FQ object using + * and only call qman_enqueue() (no FQ initialisation, scheduling, etc). Ie; + * qman_create_fq(..., QMAN_FQ_FLAG_NO_MODIFY, ...); + * However, data can not be enqueued to the FQ until it is initialised out of + * the OOS state - this function polls for that condition. It is particularly + * useful for users of IPC functions - each endpoint's Rx FQ is the other + * endpoint's Tx FQ, so each side can initialise and schedule their Rx FQ object + * and then use this API on the (NO_MODIFY) Tx FQ object in order to + * synchronise. The function returns zero for success, +1 if the FQ is still in + * the OOS state, or negative if there was an error. + */ +static inline int qman_poll_fq_for_init(struct qman_fq *fq) +{ + struct qm_mcr_queryfq_np np; + int err; + + err = qman_query_fq_np(fq, &np); + if (err) + return err; + if ((np.state & QM_MCR_NP_STATE_MASK) == QM_MCR_NP_STATE_OOS) + return 1; + return 0; +} + +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +#define cpu_to_hw_sg(x) +#define hw_sg_to_cpu(x) +#else +#define cpu_to_hw_sg(x) __cpu_to_hw_sg(x) +#define hw_sg_to_cpu(x) __hw_sg_to_cpu(x) + +static inline void __cpu_to_hw_sg(struct qm_sg_entry *sgentry) +{ + sgentry->opaque = cpu_to_be64(sgentry->opaque); + sgentry->val = cpu_to_be32(sgentry->val); + sgentry->val_off = cpu_to_be16(sgentry->val_off); +} + +static inline void __hw_sg_to_cpu(struct qm_sg_entry *sgentry) +{ + sgentry->opaque = be64_to_cpu(sgentry->opaque); + sgentry->val = be32_to_cpu(sgentry->val); + sgentry->val_off = be16_to_cpu(sgentry->val_off); +} +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* __FSL_QMAN_H */ diff --git a/dpdk/drivers/bus/dpaa/include/fsl_usd.h b/dpdk/drivers/bus/dpaa/include/fsl_usd.h new file mode 100644 index 00000000..a3243aff --- /dev/null +++ b/dpdk/drivers/bus/dpaa/include/fsl_usd.h @@ -0,0 +1,107 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2010-2011 Freescale Semiconductor, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __FSL_USD_H +#define __FSL_USD_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* Thread-entry/exit hooks; */ +int qman_thread_init(void); +int bman_thread_init(void); +int qman_thread_finish(void); +int bman_thread_finish(void); + +#define QBMAN_ANY_PORTAL_IDX 0xffffffff + +/* Obtain and free raw (unitialized) portals */ + +struct dpaa_raw_portal { + /* inputs */ + + /* set to non zero to turn on stashing */ + uint8_t enable_stash; + /* Stashing attributes for the portal */ + uint32_t cpu; + uint32_t cache; + uint32_t window; + + /* Specifies the stash request queue this portal should use */ + uint8_t sdest; + + /* Specifes a specific portal index to map or QBMAN_ANY_PORTAL_IDX + * for don't care. The portal index will be populated by the + * driver when the ioctl() successfully completes. + */ + uint32_t index; + + /* outputs */ + uint64_t cinh; + uint64_t cena; +}; + +int qman_allocate_raw_portal(struct dpaa_raw_portal *portal); +int qman_free_raw_portal(struct dpaa_raw_portal *portal); + +int bman_allocate_raw_portal(struct dpaa_raw_portal *portal); +int bman_free_raw_portal(struct dpaa_raw_portal *portal); + +/* Post-process interrupts. NB, the kernel IRQ handler disables the interrupt + * line before notifying us, and this post-processing re-enables it once + * processing is complete. As such, it is essential to call this before going + * into another blocking read/select/poll. + */ +void qman_thread_irq(void); +void bman_thread_irq(void); + +/* Global setup */ +int qman_global_init(void); +int bman_global_init(void); + +#ifdef __cplusplus +} +#endif + +#endif /* __FSL_USD_H */ diff --git a/dpdk/drivers/bus/dpaa/include/netcfg.h b/dpdk/drivers/bus/dpaa/include/netcfg.h new file mode 100644 index 00000000..b77a6787 --- /dev/null +++ b/dpdk/drivers/bus/dpaa/include/netcfg.h @@ -0,0 +1,96 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2010-2012 Freescale Semiconductor, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __NETCFG_H +#define __NETCFG_H + +#include +#include + +/* Configuration information related to a specific ethernet port */ +struct fm_eth_port_cfg { + /**< A list of PCD FQ ranges, obtained from FMC configuration */ + struct list_head *list; + /**< The "Rx default" FQID, obtained from FMC configuration */ + uint32_t rx_def; + /**< Other interface details are in the fman driver interface */ + struct fman_if *fman_if; +}; + +struct netcfg_info { + uint8_t num_ethports; + /**< Number of ports */ + struct fm_eth_port_cfg port_cfg[0]; + /**< Variable structure array of size num_ethports */ +}; + +struct interface_info { + char *name; + struct ether_addr mac_addr; + struct ether_addr peer_mac; + int mac_present; + int fman_enabled_mac_interface; +}; + +struct netcfg_interface { + uint8_t numof_netcfg_interface; + uint8_t numof_fman_enabled_macless; + struct interface_info interface_info[0]; +}; + +/* pcd_file: FMC netpcd XML ("policy") file, that contains PCD information. + * cfg_file: FMC config XML file + * Returns the configuration information in newly allocated memory. + */ +struct netcfg_info *netcfg_acquire(void); + +/* cfg_ptr: configuration information pointer. + * Frees the resources allocated by the configuration layer. + */ +void netcfg_release(struct netcfg_info *cfg_ptr); + +#ifdef RTE_LIBRTE_DPAA_DEBUG_DRIVER +/* cfg_ptr: configuration information pointer. + * This function dumps configuration data to stdout. + */ +void dump_netcfg(struct netcfg_info *cfg_ptr); +#endif + +#endif /* __NETCFG_H */ diff --git a/dpdk/drivers/bus/dpaa/include/of.h b/dpdk/drivers/bus/dpaa/include/of.h new file mode 100644 index 00000000..2984b1e1 --- /dev/null +++ b/dpdk/drivers/bus/dpaa/include/of.h @@ -0,0 +1,190 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2010-2016 Freescale Semiconductor, Inc. + * Copyright 2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __OF_H +#define __OF_H + +#include + +#ifndef OF_INIT_DEFAULT_PATH +#define OF_INIT_DEFAULT_PATH "/proc/device-tree" +#endif + +#define OF_DEFAULT_NA 1 +#define OF_DEFAULT_NS 1 + +#define OF_FILE_BUF_MAX 256 + +/** + * Layout of Device Tree: + * dt_dir + * |- dt_dir + * | |- dt_dir + * | | |- dt_dir + * | | | |- dt_file + * | | | ``- dt_file + * | | ``- dt_file + * | `-dt_file` + * ``- dt_file + * + * +------------------+ + * |dt_dir | + * |+----------------+| + * ||dt_node || + * ||+--------------+|| + * |||device_node ||| + * ||+--------------+|| + * || list_dt_nodes || + * |+----------------+| + * | list of subdir | + * | list of files | + * +------------------+ + */ + +/** + * Device description on of a device node in device tree. + */ +struct device_node { + char name[NAME_MAX]; + char full_name[PATH_MAX]; +}; + +/** + * List of device nodes available in a device tree layout + */ +struct dt_node { + struct device_node node; /**< Property of node */ + int is_file; /**< FALSE==dir, TRUE==file */ + struct list_head list; /**< Nodes within a parent subdir */ +}; + +/** + * Types we use to represent directories and files + */ +struct dt_file; +struct dt_dir { + struct dt_node node; + struct list_head subdirs; + struct list_head files; + struct list_head linear; + struct dt_dir *parent; + struct dt_file *compatible; + struct dt_file *status; + struct dt_file *lphandle; + struct dt_file *a_cells; + struct dt_file *s_cells; + struct dt_file *reg; +}; + +struct dt_file { + struct dt_node node; + struct dt_dir *parent; + ssize_t len; + uint64_t buf[OF_FILE_BUF_MAX >> 3]; +}; + +const struct device_node *of_find_compatible_node( + const struct device_node *from, + const char *type __always_unused, + const char *compatible) + __attribute__((nonnull(3))); + +#define for_each_compatible_node(dev_node, type, compatible) \ + for (dev_node = of_find_compatible_node(NULL, type, compatible); \ + dev_node != NULL; \ + dev_node = of_find_compatible_node(dev_node, type, compatible)) + +const void *of_get_property(const struct device_node *from, const char *name, + size_t *lenp) __attribute__((nonnull(2))); +bool of_device_is_available(const struct device_node *dev_node); + +const struct device_node *of_find_node_by_phandle(phandle ph); + +const struct device_node *of_get_parent(const struct device_node *dev_node); + +const struct device_node *of_get_next_child(const struct device_node *dev_node, + const struct device_node *prev); + +#define for_each_child_node(parent, child) \ + for (child = of_get_next_child(parent, NULL); child != NULL; \ + child = of_get_next_child(parent, child)) + +uint32_t of_n_addr_cells(const struct device_node *dev_node); +uint32_t of_n_size_cells(const struct device_node *dev_node); + +const uint32_t *of_get_address(const struct device_node *dev_node, size_t idx, + uint64_t *size, uint32_t *flags); + +uint64_t of_translate_address(const struct device_node *dev_node, + const u32 *addr) __attribute__((nonnull)); + +bool of_device_is_compatible(const struct device_node *dev_node, + const char *compatible); + +/* of_init() must be called prior to initialisation or use of any driver + * subsystem that is device-tree-dependent. Eg. Qman/Bman, config layers, etc. + * The path should usually be "/proc/device-tree". + */ +int of_init_path(const char *dt_path); + +/* of_finish() allows a controlled tear-down of the device-tree layer, eg. if a + * full reload is desired without a process exit. + */ +void of_finish(void); + +/* Use of this wrapper is recommended. */ +static inline int of_init(void) +{ + return of_init_path(OF_INIT_DEFAULT_PATH); +} + +/* Read a numeric property according to its size and return it as a 64-bit + * value. + */ +static inline uint64_t of_read_number(const __be32 *cell, int size) +{ + uint64_t r = 0; + + while (size--) + r = (r << 32) | be32toh(*(cell++)); + return r; +} + +#endif /* __OF_H */ diff --git a/dpdk/drivers/bus/dpaa/include/process.h b/dpdk/drivers/bus/dpaa/include/process.h new file mode 100644 index 00000000..989ddcd5 --- /dev/null +++ b/dpdk/drivers/bus/dpaa/include/process.h @@ -0,0 +1,107 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2010-2011 Freescale Semiconductor, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __PROCESS_H +#define __PROCESS_H + +#include + +/* The process device underlies process-wide user/kernel interactions, such as + * mapping dma_mem memory and providing accompanying ioctl()s. (This isn't used + * for portals, which use one UIO device each.). + */ +#define PROCESS_PATH "/dev/fsl-usdpaa" + +/* Allocation of resource IDs uses a generic interface. This enum is used to + * distinguish between the type of underlying object being manipulated. + */ +enum dpaa_id_type { + dpaa_id_fqid, + dpaa_id_bpid, + dpaa_id_qpool, + dpaa_id_cgrid, + dpaa_id_max /* <-- not a valid type, represents the number of types */ +}; + +int process_alloc(enum dpaa_id_type id_type, uint32_t *base, uint32_t num, + uint32_t align, int partial); +void process_release(enum dpaa_id_type id_type, uint32_t base, uint32_t num); + +int process_reserve(enum dpaa_id_type id_type, uint32_t base, uint32_t num); + +/* Mapping and using QMan/BMan portals */ +enum dpaa_portal_type { + dpaa_portal_qman, + dpaa_portal_bman, +}; + +struct dpaa_ioctl_portal_map { + /* Input parameter, is a qman or bman portal required. */ + enum dpaa_portal_type type; + /* Specifes a specific portal index to map or 0xffffffff + * for don't care. + */ + uint32_t index; + + /* Return value if the map succeeds, this gives the mapped + * cache-inhibited (cinh) and cache-enabled (cena) addresses. + */ + struct dpaa_portal_map { + void *cinh; + void *cena; + } addr; + /* Qman-specific return values */ + u16 channel; + uint32_t pools; +}; + +int process_portal_map(struct dpaa_ioctl_portal_map *params); +int process_portal_unmap(struct dpaa_portal_map *map); + +struct dpaa_ioctl_irq_map { + enum dpaa_portal_type type; /* Type of portal to map */ + int fd; /* File descriptor that contains the portal */ + void *portal_cinh; /* Cache inhibited area to identify the portal */ +}; + +int process_portal_irq_map(int fd, struct dpaa_ioctl_irq_map *irq); +int process_portal_irq_unmap(int fd); + +#endif /* __PROCESS_H */ diff --git a/dpdk/drivers/bus/dpaa/rte_bus_dpaa_version.map b/dpdk/drivers/bus/dpaa/rte_bus_dpaa_version.map new file mode 100644 index 00000000..fb9d5323 --- /dev/null +++ b/dpdk/drivers/bus/dpaa/rte_bus_dpaa_version.map @@ -0,0 +1,66 @@ +DPDK_17.11 { + global: + + bman_acquire; + bman_free_pool; + bman_get_params; + bman_global_init; + bman_new_pool; + bman_query_free_buffers; + bman_release; + dpaa_logtype_mempool; + dpaa_logtype_pmd; + dpaa_netcfg; + fman_ccsr_map_fd; + fman_dealloc_bufs_mask_hi; + fman_dealloc_bufs_mask_lo; + fman_if_add_mac_addr; + fman_if_clear_mac_addr; + fman_if_disable_rx; + fman_if_enable_rx; + fman_if_discard_rx_errors; + fman_if_get_fc_threshold; + fman_if_get_fc_quanta; + fman_if_get_fdoff; + fman_if_loopback_disable; + fman_if_loopback_enable; + fman_if_promiscuous_disable; + fman_if_promiscuous_enable; + fman_if_reset_mcast_filter_table; + fman_if_set_bp; + fman_if_set_fc_threshold; + fman_if_set_fc_quanta; + fman_if_set_fdoff; + fman_if_set_ic_params; + fman_if_set_maxfrm; + fman_if_set_mcast_filter_table; + fman_if_stats_get; + fman_if_stats_get_all; + fman_if_stats_reset; + fman_ip_rev; + netcfg_acquire; + netcfg_release; + of_find_compatible_node; + of_get_property; + qm_channel_caam; + qman_create_fq; + qman_dequeue; + qman_dqrr_consume; + qman_enqueue; + qman_enqueue_multi; + qman_fq_fqid; + qman_fq_state; + qman_global_init; + qman_init_fq; + qman_poll_dqrr; + qman_query_fq_np; + qman_set_vdq; + qman_reserve_fqid_range; + qman_volatile_dequeue; + rte_dpaa_driver_register; + rte_dpaa_driver_unregister; + rte_dpaa_mem_ptov; + rte_dpaa_portal_init; + + local: *; +}; diff --git a/dpdk/drivers/bus/dpaa/rte_dpaa_bus.h b/dpdk/drivers/bus/dpaa/rte_dpaa_bus.h new file mode 100644 index 00000000..bc933afd --- /dev/null +++ b/dpdk/drivers/bus/dpaa/rte_dpaa_bus.h @@ -0,0 +1,173 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of NXP nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef __RTE_DPAA_BUS_H__ +#define __RTE_DPAA_BUS_H__ + +#include +#include + +#include +#include +#include +#include +#include + +#define FSL_DPAA_BUS_NAME "FSL_DPAA_BUS" + +#define DEV_TO_DPAA_DEVICE(ptr) \ + container_of(ptr, struct rte_dpaa_device, device) + +struct rte_dpaa_device; +struct rte_dpaa_driver; + +/* DPAA Device and Driver lists for DPAA bus */ +TAILQ_HEAD(rte_dpaa_device_list, rte_dpaa_device); +TAILQ_HEAD(rte_dpaa_driver_list, rte_dpaa_driver); + +/* Configuration variables exported from DPAA bus */ +extern struct netcfg_info *dpaa_netcfg; + +enum rte_dpaa_type { + FSL_DPAA_ETH = 1, + FSL_DPAA_CRYPTO, +}; + +struct rte_dpaa_bus { + struct rte_bus bus; + struct rte_dpaa_device_list device_list; + struct rte_dpaa_driver_list driver_list; + int device_count; +}; + +struct dpaa_device_id { + uint8_t fman_id; /**< Fman interface ID, for ETH type device */ + uint8_t mac_id; /**< Fman MAC interface ID, for ETH type device */ + uint16_t dev_id; /**< Device Identifier from DPDK */ +}; + +struct rte_dpaa_device { + TAILQ_ENTRY(rte_dpaa_device) next; + struct rte_device device; + union { + struct rte_eth_dev *eth_dev; + struct rte_cryptodev *crypto_dev; + }; + struct rte_dpaa_driver *driver; + struct dpaa_device_id id; + enum rte_dpaa_type device_type; /**< Ethernet or crypto type device */ + char name[RTE_ETH_NAME_MAX_LEN]; +}; + +typedef int (*rte_dpaa_probe_t)(struct rte_dpaa_driver *dpaa_drv, + struct rte_dpaa_device *dpaa_dev); +typedef int (*rte_dpaa_remove_t)(struct rte_dpaa_device *dpaa_dev); + +struct rte_dpaa_driver { + TAILQ_ENTRY(rte_dpaa_driver) next; + struct rte_driver driver; + struct rte_dpaa_bus *dpaa_bus; + enum rte_dpaa_type drv_type; + rte_dpaa_probe_t probe; + rte_dpaa_remove_t remove; +}; + +struct dpaa_portal { + uint32_t bman_idx; /**< BMAN Portal ID*/ + uint32_t qman_idx; /**< QMAN Portal ID*/ + uint64_t tid;/**< Parent Thread id for this portal */ +}; + +/* TODO - this is costly, need to write a fast coversion routine */ +static inline void *rte_dpaa_mem_ptov(phys_addr_t paddr) +{ + const struct rte_memseg *memseg = rte_eal_get_physmem_layout(); + int i; + + for (i = 0; i < RTE_MAX_MEMSEG && memseg[i].addr != NULL; i++) { + if (paddr >= memseg[i].iova && paddr < + memseg[i].iova + memseg[i].len) + return (uint8_t *)(memseg[i].addr) + + (paddr - memseg[i].iova); + } + + return NULL; +} + +/** + * Register a DPAA driver. + * + * @param driver + * A pointer to a rte_dpaa_driver structure describing the driver + * to be registered. + */ +void rte_dpaa_driver_register(struct rte_dpaa_driver *driver); + +/** + * Unregister a DPAA driver. + * + * @param driver + * A pointer to a rte_dpaa_driver structure describing the driver + * to be unregistered. + */ +void rte_dpaa_driver_unregister(struct rte_dpaa_driver *driver); + +/** + * Initialize a DPAA portal + * + * @param arg + * Per thread ID + * + * @return + * 0 in case of success, error otherwise + */ +int rte_dpaa_portal_init(void *arg); + +/** + * Cleanup a DPAA Portal + */ +void dpaa_portal_finish(void *arg); + +/** Helper for DPAA device registration from driver (eth, crypto) instance */ +#define RTE_PMD_REGISTER_DPAA(nm, dpaa_drv) \ +RTE_INIT(dpaainitfn_ ##nm); \ +static void dpaainitfn_ ##nm(void) \ +{\ + (dpaa_drv).driver.name = RTE_STR(nm);\ + rte_dpaa_driver_register(&dpaa_drv); \ +} \ +RTE_PMD_EXPORT_NAME(nm, __COUNTER__) + +#ifdef __cplusplus +} +#endif + +#endif /* __RTE_DPAA_BUS_H__ */ diff --git a/dpdk/drivers/bus/dpaa/rte_dpaa_logs.h b/dpdk/drivers/bus/dpaa/rte_dpaa_logs.h new file mode 100644 index 00000000..037c96b0 --- /dev/null +++ b/dpdk/drivers/bus/dpaa/rte_dpaa_logs.h @@ -0,0 +1,107 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of NXP nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _DPAA_LOGS_H_ +#define _DPAA_LOGS_H_ + +#include + +extern int dpaa_logtype_bus; +extern int dpaa_logtype_mempool; +extern int dpaa_logtype_pmd; + +#define DPAA_BUS_LOG(level, fmt, args...) \ + rte_log(RTE_LOG_ ## level, dpaa_logtype_bus, "%s(): " fmt "\n", \ + __func__, ##args) + +#define BUS_INIT_FUNC_TRACE() DPAA_BUS_LOG(DEBUG, " >>") + +#ifdef RTE_LIBRTE_DPAA_DEBUG_BUS +#define DPAA_BUS_HWWARN(cond, fmt, args...) \ + do {\ + if (cond) \ + DPAA_BUS_LOG(DEBUG, "WARN: " fmt, ##args); \ + } while (0) +#else +#define DPAA_BUS_HWWARN(cond, fmt, args...) do { } while (0) +#endif + +#define DPAA_BUS_DEBUG(fmt, args...) \ + DPAA_BUS_LOG(DEBUG, fmt, ## args) +#define DPAA_BUS_INFO(fmt, args...) \ + DPAA_BUS_LOG(INFO, fmt, ## args) +#define DPAA_BUS_ERR(fmt, args...) \ + DPAA_BUS_LOG(ERR, fmt, ## args) +#define DPAA_BUS_WARN(fmt, args...) \ + DPAA_BUS_LOG(WARNING, fmt, ## args) + +/* Mempool related logs */ + +#define DPAA_MEMPOOL_LOG(level, fmt, args...) \ + rte_log(RTE_LOG_ ## level, dpaa_logtype_mempool, "%s(): " fmt "\n", \ + __func__, ##args) + +#define MEMPOOL_INIT_FUNC_TRACE() DPAA_MEMPOOL_LOG(DEBUG, " >>") + +#define DPAA_MEMPOOL_DPDEBUG(fmt, args...) \ + RTE_LOG_DP(DEBUG, PMD, fmt, ## args) +#define DPAA_MEMPOOL_DEBUG(fmt, args...) \ + DPAA_MEMPOOL_LOG(DEBUG, fmt, ## args) +#define DPAA_MEMPOOL_ERR(fmt, args...) \ + DPAA_MEMPOOL_LOG(ERR, fmt, ## args) +#define DPAA_MEMPOOL_INFO(fmt, args...) \ + DPAA_MEMPOOL_LOG(INFO, fmt, ## args) +#define DPAA_MEMPOOL_WARN(fmt, args...) \ + DPAA_MEMPOOL_LOG(WARNING, fmt, ## args) + +/* PMD related logs */ + +#define DPAA_PMD_LOG(level, fmt, args...) \ + rte_log(RTE_LOG_ ## level, dpaa_logtype_pmd, "%s(): " fmt "\n", \ + __func__, ##args) + +#define PMD_INIT_FUNC_TRACE() DPAA_PMD_LOG(DEBUG, " >>") + +#define DPAA_PMD_DEBUG(fmt, args...) \ + DPAA_PMD_LOG(DEBUG, fmt, ## args) +#define DPAA_PMD_ERR(fmt, args...) \ + DPAA_PMD_LOG(ERR, fmt, ## args) +#define DPAA_PMD_INFO(fmt, args...) \ + DPAA_PMD_LOG(INFO, fmt, ## args) +#define DPAA_PMD_WARN(fmt, args...) \ + DPAA_PMD_LOG(WARNING, fmt, ## args) + +/* DP Logs, toggled out at compile time if level lower than current level */ +#define DPAA_DP_LOG(level, fmt, args...) \ + RTE_LOG_DP(level, PMD, fmt, ## args) + +#endif /* _DPAA_LOGS_H_ */ diff --git a/dpdk/drivers/bus/fslmc/Makefile b/dpdk/drivers/bus/fslmc/Makefile new file mode 100644 index 00000000..c08b2af9 --- /dev/null +++ b/dpdk/drivers/bus/fslmc/Makefile @@ -0,0 +1,81 @@ +# BSD LICENSE +# +# Copyright 2016 NXP. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of NXP nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +include $(RTE_SDK)/mk/rte.vars.mk + +# +# library name +# +LIB = librte_bus_fslmc.a + +ifeq ($(CONFIG_RTE_LIBRTE_DPAA2_PMD),y) +CONFIG_RTE_LIBRTE_FSLMC_BUS = $(CONFIG_RTE_LIBRTE_DPAA2_PMD) +endif + +ifeq ($(CONFIG_RTE_LIBRTE_DPAA2_DEBUG_INIT),y) +CFLAGS += -O0 -g +CFLAGS += "-Wno-error" +else +CFLAGS += -O3 +CFLAGS += $(WERROR_FLAGS) +endif + +CFLAGS += -I$(RTE_SDK)/drivers/bus/fslmc +CFLAGS += -I$(RTE_SDK)/drivers/bus/fslmc/mc +CFLAGS += -I$(RTE_SDK)/drivers/bus/fslmc/qbman/include +CFLAGS += -I$(RTE_SDK)/lib/librte_eal/linuxapp/eal +CFLAGS += -I$(RTE_SDK)/lib/librte_eal/common +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_ethdev + +# versioning export map +EXPORT_MAP := rte_bus_fslmc_version.map + +# library version +LIBABIVER := 1 + +SRCS-$(CONFIG_RTE_LIBRTE_FSLMC_BUS) += \ + qbman/qbman_portal.c + +SRCS-$(CONFIG_RTE_LIBRTE_FSLMC_BUS) += \ + mc/dpmng.c \ + mc/dpbp.c \ + mc/dpio.c \ + mc/mc_sys.c \ + mc/dpcon.c \ + mc/dpci.c + +SRCS-$(CONFIG_RTE_LIBRTE_FSLMC_BUS) += portal/dpaa2_hw_dpio.c +SRCS-$(CONFIG_RTE_LIBRTE_FSLMC_BUS) += portal/dpaa2_hw_dpbp.c +SRCS-$(CONFIG_RTE_LIBRTE_FSLMC_BUS) += portal/dpaa2_hw_dpci.c +SRCS-$(CONFIG_RTE_LIBRTE_FSLMC_BUS) += fslmc_vfio.c +SRCS-$(CONFIG_RTE_LIBRTE_FSLMC_BUS) += fslmc_bus.c + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/bus/fslmc/fslmc_bus.c b/dpdk/drivers/bus/fslmc/fslmc_bus.c new file mode 100644 index 00000000..480857e5 --- /dev/null +++ b/dpdk/drivers/bus/fslmc/fslmc_bus.c @@ -0,0 +1,369 @@ +/*- + * BSD LICENSE + * + * Copyright 2016 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of NXP nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#define FSLMC_BUS_LOG(level, fmt, args...) \ + RTE_LOG(level, EAL, fmt "\n", ##args) + +#define VFIO_IOMMU_GROUP_PATH "/sys/kernel/iommu_groups" + +struct rte_fslmc_bus rte_fslmc_bus; + +static void +cleanup_fslmc_device_list(void) +{ + struct rte_dpaa2_device *dev; + struct rte_dpaa2_device *t_dev; + + TAILQ_FOREACH_SAFE(dev, &rte_fslmc_bus.device_list, next, t_dev) { + TAILQ_REMOVE(&rte_fslmc_bus.device_list, dev, next); + free(dev); + dev = NULL; + } +} + +static int +compare_dpaa2_devname(struct rte_dpaa2_device *dev1, + struct rte_dpaa2_device *dev2) +{ + int comp; + + if (dev1->dev_type > dev2->dev_type) { + comp = 1; + } else if (dev1->dev_type < dev2->dev_type) { + comp = -1; + } else { + /* Check the ID as types match */ + if (dev1->object_id > dev2->object_id) + comp = 1; + else if (dev1->object_id < dev2->object_id) + comp = -1; + else + comp = 0; /* Duplicate device name */ + } + + return comp; +} + +static void +insert_in_device_list(struct rte_dpaa2_device *newdev) +{ + int comp, inserted = 0; + struct rte_dpaa2_device *dev = NULL; + struct rte_dpaa2_device *tdev = NULL; + + TAILQ_FOREACH_SAFE(dev, &rte_fslmc_bus.device_list, next, tdev) { + comp = compare_dpaa2_devname(newdev, dev); + if (comp < 0) { + TAILQ_INSERT_BEFORE(dev, newdev, next); + inserted = 1; + break; + } + } + + if (!inserted) + TAILQ_INSERT_TAIL(&rte_fslmc_bus.device_list, newdev, next); +} + +static int +scan_one_fslmc_device(char *dev_name) +{ + char *dup_dev_name, *t_ptr; + struct rte_dpaa2_device *dev; + + if (!dev_name) + return -1; + + /* Ignore the Container name itself */ + if (!strncmp("dprc", dev_name, 4)) + return 0; + + /* Creating a temporary copy to perform cut-parse over string */ + dup_dev_name = strdup(dev_name); + if (!dup_dev_name) { + FSLMC_BUS_LOG(ERR, "Out of memory."); + return -ENOMEM; + } + + /* For all other devices, we allocate rte_dpaa2_device. + * For those devices where there is no driver, probe would release + * the memory associated with the rte_dpaa2_device after necessary + * initialization. + */ + dev = calloc(1, sizeof(struct rte_dpaa2_device)); + if (!dev) { + FSLMC_BUS_LOG(ERR, "Out of memory."); + free(dup_dev_name); + return -ENOMEM; + } + + /* Parse the device name and ID */ + t_ptr = strtok(dup_dev_name, "."); + if (!t_ptr) { + FSLMC_BUS_LOG(ERR, "Incorrect device string observed."); + goto cleanup; + } + if (!strncmp("dpni", t_ptr, 4)) + dev->dev_type = DPAA2_ETH; + else if (!strncmp("dpseci", t_ptr, 6)) + dev->dev_type = DPAA2_CRYPTO; + else if (!strncmp("dpcon", t_ptr, 5)) + dev->dev_type = DPAA2_CON; + else if (!strncmp("dpbp", t_ptr, 4)) + dev->dev_type = DPAA2_BPOOL; + else if (!strncmp("dpio", t_ptr, 4)) + dev->dev_type = DPAA2_IO; + else if (!strncmp("dpci", t_ptr, 5)) + dev->dev_type = DPAA2_CI; + else if (!strncmp("dpmcp", t_ptr, 5)) + dev->dev_type = DPAA2_MPORTAL; + else + dev->dev_type = DPAA2_UNKNOWN; + + t_ptr = strtok(NULL, "."); + if (!t_ptr) { + FSLMC_BUS_LOG(ERR, "Incorrect device string observed (%s).", + t_ptr); + goto cleanup; + } + + sscanf(t_ptr, "%hu", &dev->object_id); + dev->device.name = strdup(dev_name); + if (!dev->device.name) { + FSLMC_BUS_LOG(ERR, "Out of memory."); + goto cleanup; + } + + /* Add device in the fslmc device list */ + insert_in_device_list(dev); + + /* Don't need the duplicated device filesystem entry anymore */ + if (dup_dev_name) + free(dup_dev_name); + + return 0; +cleanup: + if (dup_dev_name) + free(dup_dev_name); + if (dev) + free(dev); + return -1; +} + +static int +rte_fslmc_scan(void) +{ + int ret; + int device_count = 0; + char fslmc_dirpath[PATH_MAX]; + DIR *dir; + struct dirent *entry; + static int process_once; + int groupid; + + if (process_once) { + FSLMC_BUS_LOG(DEBUG, + "Fslmc bus already scanned. Not rescanning"); + return 0; + } + process_once = 1; + + ret = fslmc_get_container_group(&groupid); + if (ret != 0) + goto scan_fail; + + /* Scan devices on the group */ + sprintf(fslmc_dirpath, "%s/%d/devices", VFIO_IOMMU_GROUP_PATH, + groupid); + dir = opendir(fslmc_dirpath); + if (!dir) { + FSLMC_BUS_LOG(ERR, "Unable to open VFIO group dir."); + goto scan_fail; + } + + while ((entry = readdir(dir)) != NULL) { + if (entry->d_name[0] == '.' || entry->d_type != DT_LNK) + continue; + + ret = scan_one_fslmc_device(entry->d_name); + if (ret != 0) { + /* Error in parsing directory - exit gracefully */ + goto scan_fail_cleanup; + } + device_count += 1; + } + + FSLMC_BUS_LOG(INFO, "fslmc: Bus scan completed"); + + closedir(dir); + return 0; + +scan_fail_cleanup: + closedir(dir); + + /* Remove all devices in the list */ + cleanup_fslmc_device_list(); +scan_fail: + FSLMC_BUS_LOG(DEBUG, "FSLMC Bus Not Available. Skipping."); + /* Irrespective of failure, scan only return success */ + return 0; +} + +static int +rte_fslmc_match(struct rte_dpaa2_driver *dpaa2_drv, + struct rte_dpaa2_device *dpaa2_dev) +{ + if (dpaa2_drv->drv_type == dpaa2_dev->dev_type) + return 0; + + return 1; +} + +static int +rte_fslmc_probe(void) +{ + int ret = 0; + struct rte_dpaa2_device *dev; + struct rte_dpaa2_driver *drv; + + if (TAILQ_EMPTY(&rte_fslmc_bus.device_list)) + return 0; + + ret = fslmc_vfio_setup_group(); + if (ret) { + FSLMC_BUS_LOG(ERR, "Unable to setup VFIO %d", ret); + return 0; + } + + ret = fslmc_vfio_process_group(); + if (ret) { + FSLMC_BUS_LOG(ERR, "Unable to setup devices %d", ret); + return 0; + } + + TAILQ_FOREACH(dev, &rte_fslmc_bus.device_list, next) { + TAILQ_FOREACH(drv, &rte_fslmc_bus.driver_list, next) { + ret = rte_fslmc_match(drv, dev); + if (ret) + continue; + + if (!drv->probe) + continue; + + ret = drv->probe(drv, dev); + if (ret) + FSLMC_BUS_LOG(ERR, "Unable to probe.\n"); + break; + } + } + + return 0; +} + +static struct rte_device * +rte_fslmc_find_device(const struct rte_device *start, rte_dev_cmp_t cmp, + const void *data) +{ + struct rte_dpaa2_device *dev; + + TAILQ_FOREACH(dev, &rte_fslmc_bus.device_list, next) { + if (start && &dev->device == start) { + start = NULL; /* starting point found */ + continue; + } + + if (cmp(&dev->device, data) == 0) + return &dev->device; + } + + return NULL; +} + +/*register a fslmc bus based dpaa2 driver */ +void +rte_fslmc_driver_register(struct rte_dpaa2_driver *driver) +{ + RTE_VERIFY(driver); + + TAILQ_INSERT_TAIL(&rte_fslmc_bus.driver_list, driver, next); + /* Update Bus references */ + driver->fslmc_bus = &rte_fslmc_bus; +} + +/*un-register a fslmc bus based dpaa2 driver */ +void +rte_fslmc_driver_unregister(struct rte_dpaa2_driver *driver) +{ + struct rte_fslmc_bus *fslmc_bus; + + fslmc_bus = driver->fslmc_bus; + + TAILQ_REMOVE(&fslmc_bus->driver_list, driver, next); + /* Update Bus references */ + driver->fslmc_bus = NULL; +} + +/* + * Get iommu class of DPAA2 devices on the bus. + */ +static enum rte_iova_mode +rte_dpaa2_get_iommu_class(void) +{ + return RTE_IOVA_PA; +} + +struct rte_fslmc_bus rte_fslmc_bus = { + .bus = { + .scan = rte_fslmc_scan, + .probe = rte_fslmc_probe, + .find_device = rte_fslmc_find_device, + .get_iommu_class = rte_dpaa2_get_iommu_class, + }, + .device_list = TAILQ_HEAD_INITIALIZER(rte_fslmc_bus.device_list), + .driver_list = TAILQ_HEAD_INITIALIZER(rte_fslmc_bus.driver_list), +}; + +RTE_REGISTER_BUS(fslmc, rte_fslmc_bus.bus); diff --git a/dpdk/drivers/bus/fslmc/fslmc_logs.h b/dpdk/drivers/bus/fslmc/fslmc_logs.h new file mode 100644 index 00000000..1f7c24b3 --- /dev/null +++ b/dpdk/drivers/bus/fslmc/fslmc_logs.h @@ -0,0 +1,76 @@ +/*- + * BSD LICENSE + * + * Copyright 2016 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of NXP nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _FSLMC_LOGS_H_ +#define _FSLMC_LOGS_H_ + +#define PMD_INIT_LOG(level, fmt, args...) \ + RTE_LOG(level, PMD, "%s(): " fmt "\n", __func__, ##args) + +#ifdef RTE_LIBRTE_DPAA2_DEBUG_INIT +#define PMD_INIT_FUNC_TRACE() PMD_INIT_LOG(DEBUG, " >>") +#else +#define PMD_INIT_FUNC_TRACE() do { } while (0) +#endif + +#ifdef RTE_LIBRTE_DPAA2_DEBUG_RX +#define PMD_RX_LOG(level, fmt, args...) \ + RTE_LOG(level, PMD, "%s(): " fmt "\n", __func__, ## args) +#else +#define PMD_RX_LOG(level, fmt, args...) do { } while (0) +#endif + +#ifdef RTE_LIBRTE_DPAA2_DEBUG_TX +#define PMD_TX_LOG(level, fmt, args...) \ + RTE_LOG(level, PMD, "%s(): " fmt "\n", __func__, ## args) +#else +#define PMD_TX_LOG(level, fmt, args...) do { } while (0) +#endif + +#ifdef RTE_LIBRTE_DPAA2_DEBUG_TX_FREE +#define PMD_TX_FREE_LOG(level, fmt, args...) \ + RTE_LOG(level, PMD, "%s(): " fmt "\n", __func__, ## args) +#else +#define PMD_TX_FREE_LOG(level, fmt, args...) do { } while (0) +#endif + +#ifdef RTE_LIBRTE_DPAA2_DEBUG_DRIVER +#define PMD_DRV_LOG_RAW(level, fmt, args...) \ + RTE_LOG(level, PMD, "%s(): " fmt, __func__, ## args) +#else +#define PMD_DRV_LOG_RAW(level, fmt, args...) do { } while (0) +#endif + +#define PMD_DRV_LOG(level, fmt, args...) \ + PMD_DRV_LOG_RAW(level, fmt "\n", ## args) + +#endif /* _FSLMC_LOGS_H_ */ diff --git a/dpdk/drivers/bus/fslmc/fslmc_vfio.c b/dpdk/drivers/bus/fslmc/fslmc_vfio.c new file mode 100644 index 00000000..a9363218 --- /dev/null +++ b/dpdk/drivers/bus/fslmc/fslmc_vfio.c @@ -0,0 +1,700 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2015-2016 Freescale Semiconductor, Inc. All rights reserved. + * Copyright 2016 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Freescale Semiconductor, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "rte_fslmc.h" +#include "fslmc_vfio.h" +#include + +#include "portal/dpaa2_hw_pvt.h" +#include "portal/dpaa2_hw_dpio.h" + +#define FSLMC_VFIO_LOG(level, fmt, args...) \ + RTE_LOG(level, EAL, fmt "\n", ##args) + +/** Pathname of FSL-MC devices directory. */ +#define SYSFS_FSL_MC_DEVICES "/sys/bus/fsl-mc/devices" + +#define FSLMC_CONTAINER_MAX_LEN 8 /**< Of the format dprc.XX */ + +/* Number of VFIO containers & groups with in */ +static struct fslmc_vfio_group vfio_group; +static struct fslmc_vfio_container vfio_container; +static int container_device_fd; +static char *g_container; +static uint32_t *msi_intr_vaddr; +void *(*rte_mcp_ptr_list); +static int is_dma_done; + +static struct rte_dpaa2_object_list dpaa2_obj_list = + TAILQ_HEAD_INITIALIZER(dpaa2_obj_list); + +/*register a fslmc bus based dpaa2 driver */ +void +rte_fslmc_object_register(struct rte_dpaa2_object *object) +{ + RTE_VERIFY(object); + + TAILQ_INSERT_TAIL(&dpaa2_obj_list, object, next); +} + +int +fslmc_get_container_group(int *groupid) +{ + int ret; + char *container; + + if (!g_container) { + container = getenv("DPRC"); + if (container == NULL) { + RTE_LOG(WARNING, EAL, "DPAA2: DPRC not available\n"); + return -EINVAL; + } + + if (strlen(container) >= FSLMC_CONTAINER_MAX_LEN) { + FSLMC_VFIO_LOG(ERR, "Invalid container name: %s\n", + container); + return -1; + } + + g_container = strdup(container); + if (!g_container) { + FSLMC_VFIO_LOG(ERR, "Out of memory."); + return -ENOMEM; + } + } + + /* get group number */ + ret = vfio_get_group_no(SYSFS_FSL_MC_DEVICES, g_container, groupid); + if (ret <= 0) { + FSLMC_VFIO_LOG(ERR, "Unable to find %s IOMMU group", + g_container); + return -1; + } + + FSLMC_VFIO_LOG(DEBUG, "Container: %s has VFIO iommu group id = %d", + g_container, *groupid); + + return 0; +} + +static int +vfio_connect_container(void) +{ + int fd, ret; + + if (vfio_container.used) { + FSLMC_VFIO_LOG(DEBUG, "No container available."); + return -1; + } + + /* Try connecting to vfio container if already created */ + if (!ioctl(vfio_group.fd, VFIO_GROUP_SET_CONTAINER, + &vfio_container.fd)) { + FSLMC_VFIO_LOG(INFO, + "Container pre-exists with FD[0x%x] for this group", + vfio_container.fd); + vfio_group.container = &vfio_container; + return 0; + } + + /* Opens main vfio file descriptor which represents the "container" */ + fd = vfio_get_container_fd(); + if (fd < 0) { + FSLMC_VFIO_LOG(ERR, "Failed to open VFIO container"); + return -errno; + } + + /* Check whether support for SMMU type IOMMU present or not */ + if (ioctl(fd, VFIO_CHECK_EXTENSION, VFIO_TYPE1_IOMMU)) { + /* Connect group to container */ + ret = ioctl(vfio_group.fd, VFIO_GROUP_SET_CONTAINER, &fd); + if (ret) { + FSLMC_VFIO_LOG(ERR, "Failed to setup group container"); + close(fd); + return -errno; + } + + ret = ioctl(fd, VFIO_SET_IOMMU, VFIO_TYPE1_IOMMU); + if (ret) { + FSLMC_VFIO_LOG(ERR, "Failed to setup VFIO iommu"); + close(fd); + return -errno; + } + } else { + FSLMC_VFIO_LOG(ERR, "No supported IOMMU available"); + close(fd); + return -EINVAL; + } + + vfio_container.used = 1; + vfio_container.fd = fd; + vfio_container.group = &vfio_group; + vfio_group.container = &vfio_container; + + return 0; +} + +static int vfio_map_irq_region(struct fslmc_vfio_group *group) +{ + int ret; + unsigned long *vaddr = NULL; + struct vfio_iommu_type1_dma_map map = { + .argsz = sizeof(map), + .flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE, + .vaddr = 0x6030000, + .iova = 0x6030000, + .size = 0x1000, + }; + + vaddr = (unsigned long *)mmap(NULL, 0x1000, PROT_WRITE | + PROT_READ, MAP_SHARED, container_device_fd, 0x6030000); + if (vaddr == MAP_FAILED) { + FSLMC_VFIO_LOG(ERR, "Unable to map region (errno = %d)", errno); + return -errno; + } + + msi_intr_vaddr = (uint32_t *)((char *)(vaddr) + 64); + map.vaddr = (unsigned long)vaddr; + ret = ioctl(group->container->fd, VFIO_IOMMU_MAP_DMA, &map); + if (ret == 0) + return 0; + + FSLMC_VFIO_LOG(ERR, "VFIO_IOMMU_MAP_DMA fails (errno = %d)", errno); + return -errno; +} + +int rte_fslmc_vfio_dmamap(void) +{ + int ret; + struct fslmc_vfio_group *group; + struct vfio_iommu_type1_dma_map dma_map = { + .argsz = sizeof(struct vfio_iommu_type1_dma_map), + .flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE, + }; + + int i; + const struct rte_memseg *memseg; + + if (is_dma_done) + return 0; + + memseg = rte_eal_get_physmem_layout(); + if (memseg == NULL) { + FSLMC_VFIO_LOG(ERR, "Cannot get physical layout."); + return -ENODEV; + } + + for (i = 0; i < RTE_MAX_MEMSEG; i++) { + if (memseg[i].addr == NULL && memseg[i].len == 0) { + FSLMC_VFIO_LOG(DEBUG, "Total %d segments found.", i); + break; + } + + dma_map.size = memseg[i].len; + dma_map.vaddr = memseg[i].addr_64; +#ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA + dma_map.iova = memseg[i].iova; +#else + dma_map.iova = dma_map.vaddr; +#endif + + /* SET DMA MAP for IOMMU */ + group = &vfio_group; + + if (!group->container) { + FSLMC_VFIO_LOG(ERR, "Container is not connected "); + return -1; + } + + FSLMC_VFIO_LOG(DEBUG, "-->Initial SHM Virtual ADDR %llX", + dma_map.vaddr); + FSLMC_VFIO_LOG(DEBUG, "-----> DMA size 0x%llX", dma_map.size); + ret = ioctl(group->container->fd, VFIO_IOMMU_MAP_DMA, + &dma_map); + if (ret) { + FSLMC_VFIO_LOG(ERR, "VFIO_IOMMU_MAP_DMA API(errno = %d)", + errno); + return ret; + } + } + + /* Verifying that at least single segment is available */ + if (i <= 0) { + FSLMC_VFIO_LOG(ERR, "No Segments found for VFIO Mapping"); + return -1; + } + + /* TODO - This is a W.A. as VFIO currently does not add the mapping of + * the interrupt region to SMMU. This should be removed once the + * support is added in the Kernel. + */ + vfio_map_irq_region(group); + + is_dma_done = 1; + + return 0; +} + +static int64_t vfio_map_mcp_obj(struct fslmc_vfio_group *group, char *mcp_obj) +{ + int64_t v_addr = (int64_t)MAP_FAILED; + int32_t ret, mc_fd; + + struct vfio_device_info d_info = { .argsz = sizeof(d_info) }; + struct vfio_region_info reg_info = { .argsz = sizeof(reg_info) }; + + /* getting the mcp object's fd*/ + mc_fd = ioctl(group->fd, VFIO_GROUP_GET_DEVICE_FD, mcp_obj); + if (mc_fd < 0) { + FSLMC_VFIO_LOG(ERR, "error in VFIO get dev %s fd from group %d", + mcp_obj, group->fd); + return v_addr; + } + + /* getting device info*/ + ret = ioctl(mc_fd, VFIO_DEVICE_GET_INFO, &d_info); + if (ret < 0) { + FSLMC_VFIO_LOG(ERR, "error in VFIO getting DEVICE_INFO"); + goto MC_FAILURE; + } + + /* getting device region info*/ + ret = ioctl(mc_fd, VFIO_DEVICE_GET_REGION_INFO, ®_info); + if (ret < 0) { + FSLMC_VFIO_LOG(ERR, "error in VFIO getting REGION_INFO"); + goto MC_FAILURE; + } + + FSLMC_VFIO_LOG(DEBUG, "region offset = %llx , region size = %llx", + reg_info.offset, reg_info.size); + + v_addr = (uint64_t)mmap(NULL, reg_info.size, + PROT_WRITE | PROT_READ, MAP_SHARED, + mc_fd, reg_info.offset); + +MC_FAILURE: + close(mc_fd); + + return v_addr; +} + +#define IRQ_SET_BUF_LEN (sizeof(struct vfio_irq_set) + sizeof(int)) + +int rte_dpaa2_intr_enable(struct rte_intr_handle *intr_handle, int index) +{ + int len, ret; + char irq_set_buf[IRQ_SET_BUF_LEN]; + struct vfio_irq_set *irq_set; + int *fd_ptr; + + len = sizeof(irq_set_buf); + + irq_set = (struct vfio_irq_set *)irq_set_buf; + irq_set->argsz = len; + irq_set->count = 1; + irq_set->flags = + VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER; + irq_set->index = index; + irq_set->start = 0; + fd_ptr = (int *)&irq_set->data; + *fd_ptr = intr_handle->fd; + + ret = ioctl(intr_handle->vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set); + if (ret) { + RTE_LOG(ERR, EAL, "Error:dpaa2 SET IRQs fd=%d, err = %d(%s)\n", + intr_handle->fd, errno, strerror(errno)); + return ret; + } + + return ret; +} + +int rte_dpaa2_intr_disable(struct rte_intr_handle *intr_handle, int index) +{ + struct vfio_irq_set *irq_set; + char irq_set_buf[IRQ_SET_BUF_LEN]; + int len, ret; + + len = sizeof(struct vfio_irq_set); + + irq_set = (struct vfio_irq_set *)irq_set_buf; + irq_set->argsz = len; + irq_set->flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_TRIGGER; + irq_set->index = index; + irq_set->start = 0; + irq_set->count = 0; + + ret = ioctl(intr_handle->vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set); + if (ret) + RTE_LOG(ERR, EAL, + "Error disabling dpaa2 interrupts for fd %d\n", + intr_handle->fd); + + return ret; +} + +/* set up interrupt support (but not enable interrupts) */ +int +rte_dpaa2_vfio_setup_intr(struct rte_intr_handle *intr_handle, + int vfio_dev_fd, + int num_irqs) +{ + int i, ret; + + /* start from MSI-X interrupt type */ + for (i = 0; i < num_irqs; i++) { + struct vfio_irq_info irq_info = { .argsz = sizeof(irq_info) }; + int fd = -1; + + irq_info.index = i; + + ret = ioctl(vfio_dev_fd, VFIO_DEVICE_GET_IRQ_INFO, &irq_info); + if (ret < 0) { + FSLMC_VFIO_LOG(ERR, + "cannot get IRQ(%d) info, error %i (%s)", + i, errno, strerror(errno)); + return -1; + } + + /* if this vector cannot be used with eventfd, + * fail if we explicitly + * specified interrupt type, otherwise continue + */ + if ((irq_info.flags & VFIO_IRQ_INFO_EVENTFD) == 0) + continue; + + /* set up an eventfd for interrupts */ + fd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC); + if (fd < 0) { + FSLMC_VFIO_LOG(ERR, + "cannot set up eventfd, error %i (%s)\n", + errno, strerror(errno)); + return -1; + } + + intr_handle->fd = fd; + intr_handle->type = RTE_INTR_HANDLE_VFIO_MSI; + intr_handle->vfio_dev_fd = vfio_dev_fd; + + return 0; + } + + /* if we're here, we haven't found a suitable interrupt vector */ + return -1; +} + +/* + * fslmc_process_iodevices for processing only IO (ETH, CRYPTO, and possibly + * EVENT) devices. + */ +static int +fslmc_process_iodevices(struct rte_dpaa2_device *dev) +{ + int dev_fd; + struct vfio_device_info device_info = { .argsz = sizeof(device_info) }; + struct rte_dpaa2_object *object = NULL; + + dev_fd = ioctl(vfio_group.fd, VFIO_GROUP_GET_DEVICE_FD, + dev->device.name); + if (dev_fd <= 0) { + FSLMC_VFIO_LOG(ERR, "Unable to obtain device FD for device:%s", + dev->device.name); + return -1; + } + + if (ioctl(dev_fd, VFIO_DEVICE_GET_INFO, &device_info)) { + FSLMC_VFIO_LOG(ERR, "DPAA2 VFIO_DEVICE_GET_INFO fail"); + return -1; + } + + switch (dev->dev_type) { + case DPAA2_ETH: + rte_dpaa2_vfio_setup_intr(&dev->intr_handle, dev_fd, + device_info.num_irqs); + break; + case DPAA2_CON: + case DPAA2_IO: + case DPAA2_CI: + case DPAA2_BPOOL: + TAILQ_FOREACH(object, &dpaa2_obj_list, next) { + if (dev->dev_type == object->dev_type) + object->create(dev_fd, &device_info, + dev->object_id); + else + continue; + } + break; + default: + break; + } + + FSLMC_VFIO_LOG(DEBUG, "Device (%s) abstracted from VFIO", + dev->device.name); + return 0; +} + +static int +fslmc_process_mcp(struct rte_dpaa2_device *dev) +{ + int64_t v_addr; + char *dev_name; + struct fsl_mc_io dpmng = {0}; + struct mc_version mc_ver_info = {0}; + + rte_mcp_ptr_list = malloc(sizeof(void *) * 1); + if (!rte_mcp_ptr_list) { + FSLMC_VFIO_LOG(ERR, "Out of memory"); + return -ENOMEM; + } + + dev_name = strdup(dev->device.name); + if (!dev_name) { + FSLMC_VFIO_LOG(ERR, "Out of memory."); + free(rte_mcp_ptr_list); + rte_mcp_ptr_list = NULL; + return -ENOMEM; + } + + v_addr = vfio_map_mcp_obj(&vfio_group, dev_name); + if (v_addr == (int64_t)MAP_FAILED) { + FSLMC_VFIO_LOG(ERR, "Error mapping region (errno = %d)", + errno); + free(rte_mcp_ptr_list); + rte_mcp_ptr_list = NULL; + return -1; + } + + /* check the MC version compatibility */ + dpmng.regs = (void *)v_addr; + if (mc_get_version(&dpmng, CMD_PRI_LOW, &mc_ver_info)) + RTE_LOG(WARNING, PMD, "\tmc_get_version failed\n"); + + if ((mc_ver_info.major != MC_VER_MAJOR) || + (mc_ver_info.minor < MC_VER_MINOR)) { + RTE_LOG(ERR, PMD, "DPAA2 MC version not compatible!" + " Expected %d.%d.x, Detected %d.%d.%d\n", + MC_VER_MAJOR, MC_VER_MINOR, + mc_ver_info.major, mc_ver_info.minor, + mc_ver_info.revision); + free(rte_mcp_ptr_list); + rte_mcp_ptr_list = NULL; + return -1; + } + rte_mcp_ptr_list[0] = (void *)v_addr; + + return 0; +} + +int +fslmc_vfio_process_group(void) +{ + int ret; + int found_mportal = 0; + struct rte_dpaa2_device *dev, *dev_temp; + + /* Search the MCP as that should be initialized first. */ + TAILQ_FOREACH_SAFE(dev, &rte_fslmc_bus.device_list, next, dev_temp) { + if (dev->dev_type == DPAA2_MPORTAL) { + ret = fslmc_process_mcp(dev); + if (ret) { + FSLMC_VFIO_LOG(DEBUG, "Unable to map Portal."); + return -1; + } + if (!found_mportal) + found_mportal = 1; + + TAILQ_REMOVE(&rte_fslmc_bus.device_list, dev, next); + free(dev); + dev = NULL; + /* Ideally there is only a single dpmcp, but in case + * multiple exists, looping on remaining devices. + */ + } + } + + /* Cannot continue if there is not even a single mportal */ + if (!found_mportal) { + FSLMC_VFIO_LOG(DEBUG, + "No MC Portal device found. Not continuing."); + return -1; + } + + TAILQ_FOREACH_SAFE(dev, &rte_fslmc_bus.device_list, next, dev_temp) { + if (!dev) + break; + + switch (dev->dev_type) { + case DPAA2_ETH: + case DPAA2_CRYPTO: + ret = fslmc_process_iodevices(dev); + if (ret) { + FSLMC_VFIO_LOG(DEBUG, + "Dev (%s) init failed.", + dev->device.name); + return ret; + } + break; + case DPAA2_CON: + case DPAA2_IO: + case DPAA2_CI: + case DPAA2_BPOOL: + /* Call the object creation routine and remove the + * device entry from device list + */ + ret = fslmc_process_iodevices(dev); + if (ret) { + FSLMC_VFIO_LOG(DEBUG, + "Dev (%s) init failed.", + dev->device.name); + return -1; + } + + /* This device is not required to be in the DPDK + * exposed device list. + */ + TAILQ_REMOVE(&rte_fslmc_bus.device_list, dev, next); + free(dev); + dev = NULL; + break; + case DPAA2_UNKNOWN: + default: + /* Unknown - ignore */ + FSLMC_VFIO_LOG(DEBUG, "Found unknown device (%s).", + dev->device.name); + TAILQ_REMOVE(&rte_fslmc_bus.device_list, dev, next); + free(dev); + dev = NULL; + } + } + + return 0; +} + +int +fslmc_vfio_setup_group(void) +{ + int groupid; + int ret; + struct vfio_group_status status = { .argsz = sizeof(status) }; + + /* if already done once */ + if (container_device_fd) + return 0; + + ret = fslmc_get_container_group(&groupid); + if (ret) + return ret; + + /* In case this group was already opened, continue without any + * processing. + */ + if (vfio_group.groupid == groupid) { + FSLMC_VFIO_LOG(ERR, "groupid already exists %d", groupid); + return 0; + } + + /* Get the actual group fd */ + ret = vfio_get_group_fd(groupid); + if (ret < 0) + return ret; + vfio_group.fd = ret; + + /* Check group viability */ + ret = ioctl(vfio_group.fd, VFIO_GROUP_GET_STATUS, &status); + if (ret) { + FSLMC_VFIO_LOG(ERR, "VFIO error getting group status"); + close(vfio_group.fd); + return ret; + } + + if (!(status.flags & VFIO_GROUP_FLAGS_VIABLE)) { + FSLMC_VFIO_LOG(ERR, "VFIO group not viable"); + close(vfio_group.fd); + return -EPERM; + } + /* Since Group is VIABLE, Store the groupid */ + vfio_group.groupid = groupid; + + /* check if group does not have a container yet */ + if (!(status.flags & VFIO_GROUP_FLAGS_CONTAINER_SET)) { + /* Now connect this IOMMU group to given container */ + ret = vfio_connect_container(); + if (ret) { + FSLMC_VFIO_LOG(ERR, + "Error connecting container with groupid %d", + groupid); + close(vfio_group.fd); + return ret; + } + } + + /* Get Device information */ + ret = ioctl(vfio_group.fd, VFIO_GROUP_GET_DEVICE_FD, g_container); + if (ret < 0) { + FSLMC_VFIO_LOG(ERR, "Error getting device %s fd from group %d", + g_container, vfio_group.groupid); + close(vfio_group.fd); + return ret; + } + container_device_fd = ret; + FSLMC_VFIO_LOG(DEBUG, "VFIO Container FD is [0x%X]", + container_device_fd); + + return 0; +} diff --git a/dpdk/drivers/bus/fslmc/fslmc_vfio.h b/dpdk/drivers/bus/fslmc/fslmc_vfio.h new file mode 100644 index 00000000..b442dc08 --- /dev/null +++ b/dpdk/drivers/bus/fslmc/fslmc_vfio.h @@ -0,0 +1,82 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2015-2016 Freescale Semiconductor, Inc. All rights reserved. + * Copyright 2016 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Freescale Semiconductor, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _FSLMC_VFIO_H_ +#define _FSLMC_VFIO_H_ + +#include + +#include "eal_vfio.h" + +#define DPAA2_MC_DPNI_DEVID 7 +#define DPAA2_MC_DPSECI_DEVID 3 +#define DPAA2_MC_DPCON_DEVID 5 +#define DPAA2_MC_DPIO_DEVID 9 +#define DPAA2_MC_DPBP_DEVID 10 +#define DPAA2_MC_DPCI_DEVID 11 + +typedef struct fslmc_vfio_device { + int fd; /* fslmc root container device ?? */ + int index; /*index of child object */ + struct fslmc_vfio_device *child; /* Child object */ +} fslmc_vfio_device; + +typedef struct fslmc_vfio_group { + int fd; /* /dev/vfio/"groupid" */ + int groupid; + struct fslmc_vfio_container *container; + int object_index; + struct fslmc_vfio_device *vfio_device; +} fslmc_vfio_group; + +typedef struct fslmc_vfio_container { + int fd; /* /dev/vfio/vfio */ + int used; + int index; /* index in group list */ + struct fslmc_vfio_group *group; +} fslmc_vfio_container; + +int rte_dpaa2_intr_enable(struct rte_intr_handle *intr_handle, int index); +int rte_dpaa2_intr_disable(struct rte_intr_handle *intr_handle, int index); + +int rte_dpaa2_vfio_setup_intr(struct rte_intr_handle *intr_handle, + int vfio_dev_fd, + int num_irqs); + +int fslmc_vfio_setup_group(void); +int fslmc_vfio_process_group(void); +char *fslmc_get_container(void); +int fslmc_get_container_group(int *gropuid); +int rte_fslmc_vfio_dmamap(void); + +#endif /* _FSLMC_VFIO_H_ */ diff --git a/dpdk/drivers/bus/fslmc/mc/dpbp.c b/dpdk/drivers/bus/fslmc/mc/dpbp.c new file mode 100644 index 00000000..a846245a --- /dev/null +++ b/dpdk/drivers/bus/fslmc/mc/dpbp.c @@ -0,0 +1,379 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2013-2016 Freescale Semiconductor Inc. + * Copyright 2016-2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#include +#include +#include +#include + +/** + * dpbp_open() - Open a control session for the specified object. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @dpbp_id: DPBP unique ID + * @token: Returned token; use in subsequent API calls + * + * This function can be used to open a control session for an + * already created object; an object may have been declared in + * the DPL or by calling the dpbp_create function. + * This function returns a unique authentication token, + * associated with the specific object ID and the specific MC + * portal; this token must be used in all subsequent commands for + * this specific object + * + * Return: '0' on Success; Error code otherwise. + */ +int dpbp_open(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + int dpbp_id, + uint16_t *token) +{ + struct dpbp_cmd_open *cmd_params; + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPBP_CMDID_OPEN, + cmd_flags, 0); + cmd_params = (struct dpbp_cmd_open *)cmd.params; + cmd_params->dpbp_id = cpu_to_le32(dpbp_id); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + *token = mc_cmd_hdr_read_token(&cmd); + + return err; +} + +/** + * dpbp_close() - Close the control session of the object + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPBP object + * + * After this function is called, no further operations are + * allowed on the object without opening a new control session. + * + * Return: '0' on Success; Error code otherwise. + */ +int dpbp_close(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token) +{ + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPBP_CMDID_CLOSE, cmd_flags, + token); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpbp_create() - Create the DPBP object. + * @mc_io: Pointer to MC portal's I/O object + * @dprc_token: Parent container token; '0' for default container + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @cfg: Configuration structure + * @obj_id: Returned object id; use in subsequent API calls + * + * Create the DPBP object, allocate required resources and + * perform required initialization. + * + * This function accepts an authentication token of a parent + * container that this object should be assigned to and returns + * an object id. This object_id will be used in all subsequent calls to + * this specific object. + * + * Return: '0' on Success; Error code otherwise. + */ +int dpbp_create(struct fsl_mc_io *mc_io, + uint16_t dprc_token, + uint32_t cmd_flags, + const struct dpbp_cfg *cfg, + uint32_t *obj_id) +{ + struct mc_command cmd = { 0 }; + int err; + + (void)(cfg); /* unused */ + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPBP_CMDID_CREATE, + cmd_flags, dprc_token); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + *obj_id = mc_cmd_read_object_id(&cmd); + + return 0; +} + +/** + * dpbp_destroy() - Destroy the DPBP object and release all its resources. + * @mc_io: Pointer to MC portal's I/O object + * @dprc_token: Parent container token; '0' for default container + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @obj_id: ID of DPBP object + * + * Return: '0' on Success; error code otherwise. + */ +int dpbp_destroy(struct fsl_mc_io *mc_io, + uint16_t dprc_token, + uint32_t cmd_flags, + uint32_t obj_id) +{ + struct dpbp_cmd_destroy *cmd_params; + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPBP_CMDID_DESTROY, + cmd_flags, dprc_token); + + cmd_params = (struct dpbp_cmd_destroy *)cmd.params; + cmd_params->object_id = cpu_to_le32(obj_id); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpbp_enable() - Enable the DPBP. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPBP object + * + * Return: '0' on Success; Error code otherwise. + */ +int dpbp_enable(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token) +{ + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPBP_CMDID_ENABLE, cmd_flags, + token); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpbp_disable() - Disable the DPBP. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPBP object + * + * Return: '0' on Success; Error code otherwise. + */ +int dpbp_disable(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token) +{ + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPBP_CMDID_DISABLE, + cmd_flags, token); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpbp_is_enabled() - Check if the DPBP is enabled. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPBP object + * @en: Returns '1' if object is enabled; '0' otherwise + * + * Return: '0' on Success; Error code otherwise. + */ +int dpbp_is_enabled(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + int *en) +{ + struct dpbp_rsp_is_enabled *rsp_params; + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPBP_CMDID_IS_ENABLED, cmd_flags, + token); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + rsp_params = (struct dpbp_rsp_is_enabled *)cmd.params; + *en = rsp_params->enabled & DPBP_ENABLE; + + return 0; +} + +/** + * dpbp_reset() - Reset the DPBP, returns the object to initial state. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPBP object + * + * Return: '0' on Success; Error code otherwise. + */ +int dpbp_reset(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token) +{ + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPBP_CMDID_RESET, + cmd_flags, token); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} +int dpbp_get_attributes(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + struct dpbp_attr *attr) +{ + struct dpbp_rsp_get_attributes *rsp_params; + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPBP_CMDID_GET_ATTR, + cmd_flags, token); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + rsp_params = (struct dpbp_rsp_get_attributes *)cmd.params; + attr->bpid = le16_to_cpu(rsp_params->bpid); + attr->id = le32_to_cpu(rsp_params->id); + + return 0; +} + +/** + * dpbp_get_api_version - Get Data Path Buffer Pool API version + * @mc_io: Pointer to Mc portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @major_ver: Major version of Buffer Pool API + * @minor_ver: Minor version of Buffer Pool API + * + * Return: '0' on Success; Error code otherwise. + */ +int dpbp_get_api_version(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t *major_ver, + uint16_t *minor_ver) +{ + struct dpbp_rsp_get_api_version *rsp_params; + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPBP_CMDID_GET_API_VERSION, + cmd_flags, 0); + + /* send command to mc */ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + rsp_params = (struct dpbp_rsp_get_api_version *)cmd.params; + *major_ver = le16_to_cpu(rsp_params->major); + *minor_ver = le16_to_cpu(rsp_params->minor); + + return 0; +} + +/** + * dpbp_get_num_free_bufs() - Get number of free buffers in the buffer pool + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPBP object + * @num_free_bufs: Number of free buffers + * + * Return: '0' on Success; Error code otherwise. + */ + +int dpbp_get_num_free_bufs(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint32_t *num_free_bufs) +{ + struct dpbp_rsp_get_num_free_bufs *rsp_params; + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPBP_CMDID_GET_FREE_BUFFERS_NUM, + cmd_flags, + token); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + rsp_params = (struct dpbp_rsp_get_num_free_bufs *)cmd.params; + *num_free_bufs = le32_to_cpu(rsp_params->num_free_bufs); + + return 0; +} diff --git a/dpdk/drivers/bus/fslmc/mc/dpci.c b/dpdk/drivers/bus/fslmc/mc/dpci.c new file mode 100644 index 00000000..5471024c --- /dev/null +++ b/dpdk/drivers/bus/fslmc/mc/dpci.c @@ -0,0 +1,473 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2013-2016 Freescale Semiconductor Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#include +#include +#include +#include + +/** + * dpci_open() - Open a control session for the specified object + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @dpci_id: DPCI unique ID + * @token: Returned token; use in subsequent API calls + * + * This function can be used to open a control session for an + * already created object; an object may have been declared in + * the DPL or by calling the dpci_create() function. + * This function returns a unique authentication token, + * associated with the specific object ID and the specific MC + * portal; this token must be used in all subsequent commands for + * this specific object. + * + * Return: '0' on Success; Error code otherwise. + */ +int dpci_open(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + int dpci_id, + uint16_t *token) +{ + struct dpci_cmd_open *cmd_params; + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPCI_CMDID_OPEN, + cmd_flags, + 0); + cmd_params = (struct dpci_cmd_open *)cmd.params; + cmd_params->dpci_id = cpu_to_le32(dpci_id); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + *token = mc_cmd_hdr_read_token(&cmd); + + return 0; +} + +/** + * dpci_close() - Close the control session of the object + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPCI object + * + * After this function is called, no further operations are + * allowed on the object without opening a new control session. + * + * Return: '0' on Success; Error code otherwise. + */ +int dpci_close(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token) +{ + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPCI_CMDID_CLOSE, + cmd_flags, + token); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpci_create() - Create the DPCI object. + * @mc_io: Pointer to MC portal's I/O object + * @dprc_token: Parent container token; '0' for default container + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @cfg: Configuration structure + * @obj_id: Returned object id + * + * Create the DPCI object, allocate required resources and perform required + * initialization. + * + * The object can be created either by declaring it in the + * DPL file, or by calling this function. + * + * The function accepts an authentication token of a parent + * container that this object should be assigned to. The token + * can be '0' so the object will be assigned to the default container. + * The newly created object can be opened with the returned + * object id and using the container's associated tokens and MC portals. + * + * Return: '0' on Success; Error code otherwise. + */ +int dpci_create(struct fsl_mc_io *mc_io, + uint16_t dprc_token, + uint32_t cmd_flags, + const struct dpci_cfg *cfg, + uint32_t *obj_id) +{ + struct dpci_cmd_create *cmd_params; + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPCI_CMDID_CREATE, + cmd_flags, + dprc_token); + cmd_params = (struct dpci_cmd_create *)cmd.params; + cmd_params->num_of_priorities = cfg->num_of_priorities; + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + *obj_id = mc_cmd_read_object_id(&cmd); + + return 0; +} + +/** + * dpci_destroy() - Destroy the DPCI object and release all its resources. + * @mc_io: Pointer to MC portal's I/O object + * @dprc_token: Parent container token; '0' for default container + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @object_id: The object id; it must be a valid id within the container that + * created this object; + * + * The function accepts the authentication token of the parent container that + * created the object (not the one that currently owns the object). The object + * is searched within parent using the provided 'object_id'. + * All tokens to the object must be closed before calling destroy. + * + * Return: '0' on Success; error code otherwise. + */ +int dpci_destroy(struct fsl_mc_io *mc_io, + uint16_t dprc_token, + uint32_t cmd_flags, + uint32_t object_id) +{ + struct dpci_cmd_destroy *cmd_params; + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPCI_CMDID_DESTROY, + cmd_flags, + dprc_token); + cmd_params = (struct dpci_cmd_destroy *)cmd.params; + cmd_params->dpci_id = cpu_to_le32(object_id); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpci_enable() - Enable the DPCI, allow sending and receiving frames. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPCI object + * + * Return: '0' on Success; Error code otherwise. + */ +int dpci_enable(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token) +{ + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPCI_CMDID_ENABLE, + cmd_flags, + token); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpci_disable() - Disable the DPCI, stop sending and receiving frames. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPCI object + * + * Return: '0' on Success; Error code otherwise. + */ +int dpci_disable(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token) +{ + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPCI_CMDID_DISABLE, + cmd_flags, + token); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpci_is_enabled() - Check if the DPCI is enabled. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPCI object + * @en: Returns '1' if object is enabled; '0' otherwise + * + * Return: '0' on Success; Error code otherwise. + */ +int dpci_is_enabled(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + int *en) +{ + struct dpci_rsp_is_enabled *rsp_params; + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPCI_CMDID_IS_ENABLED, cmd_flags, + token); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + rsp_params = (struct dpci_rsp_is_enabled *)cmd.params; + *en = dpci_get_field(rsp_params->en, ENABLE); + + return 0; +} + +/** + * dpci_reset() - Reset the DPCI, returns the object to initial state. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPCI object + * + * Return: '0' on Success; Error code otherwise. + */ +int dpci_reset(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token) +{ + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPCI_CMDID_RESET, + cmd_flags, + token); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +int dpci_get_attributes(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + struct dpci_attr *attr) +{ + struct dpci_rsp_get_attr *rsp_params; + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPCI_CMDID_GET_ATTR, + cmd_flags, + token); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + rsp_params = (struct dpci_rsp_get_attr *)cmd.params; + attr->id = le32_to_cpu(rsp_params->id); + attr->num_of_priorities = rsp_params->num_of_priorities; + + return 0; +} + +int dpci_set_rx_queue(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t priority, + const struct dpci_rx_queue_cfg *cfg) +{ + struct dpci_cmd_set_rx_queue *cmd_params; + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPCI_CMDID_SET_RX_QUEUE, + cmd_flags, + token); + cmd_params = (struct dpci_cmd_set_rx_queue *)cmd.params; + cmd_params->dest_id = cpu_to_le32(cfg->dest_cfg.dest_id); + cmd_params->dest_priority = cfg->dest_cfg.priority; + cmd_params->priority = priority; + cmd_params->user_ctx = cpu_to_le64(cfg->user_ctx); + cmd_params->options = cpu_to_le32(cfg->options); + dpci_set_field(cmd_params->dest_type, + DEST_TYPE, + cfg->dest_cfg.dest_type); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpci_get_rx_queue() - Retrieve Rx queue attributes. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPCI object + * @priority: Select the queue relative to number of + * priorities configured at DPCI creation + * @attr: Returned Rx queue attributes + * + * Return: '0' on Success; Error code otherwise. + */ +int dpci_get_rx_queue(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t priority, + struct dpci_rx_queue_attr *attr) +{ + struct dpci_cmd_get_queue *cmd_params; + struct dpci_rsp_get_rx_queue *rsp_params; + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPCI_CMDID_GET_RX_QUEUE, + cmd_flags, + token); + cmd_params = (struct dpci_cmd_get_queue *)cmd.params; + cmd_params->priority = priority; + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + rsp_params = (struct dpci_rsp_get_rx_queue *)cmd.params; + attr->user_ctx = le64_to_cpu(rsp_params->user_ctx); + attr->fqid = le32_to_cpu(rsp_params->fqid); + attr->dest_cfg.dest_id = le32_to_cpu(rsp_params->dest_id); + attr->dest_cfg.priority = rsp_params->dest_priority; + attr->dest_cfg.dest_type = dpci_get_field(rsp_params->dest_type, + DEST_TYPE); + + return 0; +} + +/** + * dpci_get_tx_queue() - Retrieve Tx queue attributes. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPCI object + * @priority: Select the queue relative to number of + * priorities of the peer DPCI object + * @attr: Returned Tx queue attributes + * + * Return: '0' on Success; Error code otherwise. + */ +int dpci_get_tx_queue(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t priority, + struct dpci_tx_queue_attr *attr) +{ + struct dpci_cmd_get_queue *cmd_params; + struct dpci_rsp_get_tx_queue *rsp_params; + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPCI_CMDID_GET_TX_QUEUE, + cmd_flags, + token); + cmd_params = (struct dpci_cmd_get_queue *)cmd.params; + cmd_params->priority = priority; + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + rsp_params = (struct dpci_rsp_get_tx_queue *)cmd.params; + attr->fqid = le32_to_cpu(rsp_params->fqid); + + return 0; +} + +/** + * dpci_get_api_version() - Get communication interface API version + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @major_ver: Major version of data path communication interface API + * @minor_ver: Minor version of data path communication interface API + * + * Return: '0' on Success; Error code otherwise. + */ +int dpci_get_api_version(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t *major_ver, + uint16_t *minor_ver) +{ + struct dpci_rsp_get_api_version *rsp_params; + struct mc_command cmd = { 0 }; + int err; + + cmd.header = mc_encode_cmd_header(DPCI_CMDID_GET_API_VERSION, + cmd_flags, + 0); + + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + rsp_params = (struct dpci_rsp_get_api_version *)cmd.params; + *major_ver = le16_to_cpu(rsp_params->major); + *minor_ver = le16_to_cpu(rsp_params->minor); + + return 0; +} diff --git a/dpdk/drivers/bus/fslmc/mc/dpcon.c b/dpdk/drivers/bus/fslmc/mc/dpcon.c new file mode 100644 index 00000000..477ee46e --- /dev/null +++ b/dpdk/drivers/bus/fslmc/mc/dpcon.c @@ -0,0 +1,357 @@ +/* Copyright 2013-2016 Freescale Semiconductor Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#include +#include +#include +#include + +/** + * dpcon_open() - Open a control session for the specified object + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @dpcon_id: DPCON unique ID + * @token: Returned token; use in subsequent API calls + * + * This function can be used to open a control session for an + * already created object; an object may have been declared in + * the DPL or by calling the dpcon_create() function. + * This function returns a unique authentication token, + * associated with the specific object ID and the specific MC + * portal; this token must be used in all subsequent commands for + * this specific object. + * + * Return: '0' on Success; Error code otherwise. + */ +int dpcon_open(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + int dpcon_id, + uint16_t *token) +{ + struct mc_command cmd = { 0 }; + struct dpcon_cmd_open *dpcon_cmd; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPCON_CMDID_OPEN, + cmd_flags, + 0); + dpcon_cmd = (struct dpcon_cmd_open *)cmd.params; + dpcon_cmd->dpcon_id = cpu_to_le32(dpcon_id); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + *token = mc_cmd_hdr_read_token(&cmd); + + return 0; +} + +/** + * dpcon_close() - Close the control session of the object + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPCON object + * + * After this function is called, no further operations are + * allowed on the object without opening a new control session. + * + * Return: '0' on Success; Error code otherwise. + */ +int dpcon_close(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token) +{ + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPCON_CMDID_CLOSE, + cmd_flags, + token); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpcon_create() - Create the DPCON object. + * @mc_io: Pointer to MC portal's I/O object + * @dprc_token: Parent container token; '0' for default container + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @cfg: Configuration structure + * @obj_id: Returned object id; use in subsequent API calls + * + * Create the DPCON object, allocate required resources and + * perform required initialization. + * + * The object can be created either by declaring it in the + * DPL file, or by calling this function. + * + * This function accepts an authentication token of a parent + * container that this object should be assigned to and returns + * an object id. This object_id will be used in all subsequent calls to + * this specific object. + * + * Return: '0' on Success; Error code otherwise. + */ +int dpcon_create(struct fsl_mc_io *mc_io, + uint16_t dprc_token, + uint32_t cmd_flags, + const struct dpcon_cfg *cfg, + uint32_t *obj_id) +{ + struct dpcon_cmd_create *dpcon_cmd; + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPCON_CMDID_CREATE, + cmd_flags, + dprc_token); + dpcon_cmd = (struct dpcon_cmd_create *)cmd.params; + dpcon_cmd->num_priorities = cfg->num_priorities; + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + *obj_id = mc_cmd_read_object_id(&cmd); + + return 0; +} + +/** + * dpcon_destroy() - Destroy the DPCON object and release all its resources. + * @mc_io: Pointer to MC portal's I/O object + * @dprc_token: Parent container token; '0' for default container + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @obj_id: ID of DPCON object + * + * Return: '0' on Success; error code otherwise. + */ +int dpcon_destroy(struct fsl_mc_io *mc_io, + uint16_t dprc_token, + uint32_t cmd_flags, + uint32_t obj_id) +{ + struct dpcon_cmd_destroy *cmd_params; + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPCON_CMDID_DESTROY, + cmd_flags, + dprc_token); + cmd_params = (struct dpcon_cmd_destroy *)cmd.params; + cmd_params->object_id = cpu_to_le32(obj_id); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpcon_enable() - Enable the DPCON + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPCON object + * + * Return: '0' on Success; Error code otherwise + */ +int dpcon_enable(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token) +{ + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPCON_CMDID_ENABLE, + cmd_flags, + token); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpcon_disable() - Disable the DPCON + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPCON object + * + * Return: '0' on Success; Error code otherwise + */ +int dpcon_disable(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token) +{ + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPCON_CMDID_DISABLE, + cmd_flags, + token); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpcon_is_enabled() - Check if the DPCON is enabled. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPCON object + * @en: Returns '1' if object is enabled; '0' otherwise + * + * Return: '0' on Success; Error code otherwise. + */ +int dpcon_is_enabled(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + int *en) +{ + struct dpcon_rsp_is_enabled *dpcon_rsp; + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPCON_CMDID_IS_ENABLED, + cmd_flags, + token); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + dpcon_rsp = (struct dpcon_rsp_is_enabled *)cmd.params; + *en = dpcon_rsp->enabled & DPCON_ENABLE; + + return 0; +} + +/** + * dpcon_reset() - Reset the DPCON, returns the object to initial state. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPCON object + * + * Return: '0' on Success; Error code otherwise. + */ +int dpcon_reset(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token) +{ + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPCON_CMDID_RESET, + cmd_flags, token); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpcon_get_attributes() - Retrieve DPCON attributes. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPCON object + * @attr: Object's attributes + * + * Return: '0' on Success; Error code otherwise. + */ +int dpcon_get_attributes(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + struct dpcon_attr *attr) +{ + struct dpcon_rsp_get_attr *dpcon_rsp; + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPCON_CMDID_GET_ATTR, + cmd_flags, + token); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + dpcon_rsp = (struct dpcon_rsp_get_attr *)cmd.params; + attr->id = le32_to_cpu(dpcon_rsp->id); + attr->qbman_ch_id = le16_to_cpu(dpcon_rsp->qbman_ch_id); + attr->num_priorities = dpcon_rsp->num_priorities; + + return 0; +} + +/** + * dpcon_get_api_version - Get Data Path Concentrator API version + * @mc_io: Pointer to MC portal's DPCON object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @major_ver: Major version of DPCON API + * @minor_ver: Minor version of DPCON API + * + * Return: '0' on Success; Error code otherwise + */ +int dpcon_get_api_version(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t *major_ver, + uint16_t *minor_ver) +{ + struct dpcon_rsp_get_api_version *rsp_params; + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPCON_CMDID_GET_API_VERSION, + cmd_flags, 0); + + /* send command to mc */ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + rsp_params = (struct dpcon_rsp_get_api_version *)cmd.params; + *major_ver = le16_to_cpu(rsp_params->major); + *minor_ver = le16_to_cpu(rsp_params->minor); + + return 0; +} diff --git a/dpdk/drivers/bus/fslmc/mc/dpio.c b/dpdk/drivers/bus/fslmc/mc/dpio.c new file mode 100644 index 00000000..76c5d7b7 --- /dev/null +++ b/dpdk/drivers/bus/fslmc/mc/dpio.c @@ -0,0 +1,503 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2013-2016 Freescale Semiconductor Inc. + * Copyright 2016-2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#include +#include +#include +#include + +/** + * dpio_open() - Open a control session for the specified object + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @dpio_id: DPIO unique ID + * @token: Returned token; use in subsequent API calls + * + * This function can be used to open a control session for an + * already created object; an object may have been declared in + * the DPL or by calling the dpio_create() function. + * This function returns a unique authentication token, + * associated with the specific object ID and any MC portals + * assigned to the parent container; this token must be used in + * all subsequent commands for this specific object. + * + * Return: '0' on Success; Error code otherwise. + */ +int dpio_open(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + int dpio_id, + uint16_t *token) +{ + struct dpio_cmd_open *cmd_params; + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPIO_CMDID_OPEN, + cmd_flags, + 0); + cmd_params = (struct dpio_cmd_open *)cmd.params; + cmd_params->dpio_id = cpu_to_le32(dpio_id); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + *token = mc_cmd_hdr_read_token(&cmd); + + return 0; +} + +/** + * dpio_close() - Close the control session of the object + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPIO object + * + * Return: '0' on Success; Error code otherwise. + */ +int dpio_close(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token) +{ + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPIO_CMDID_CLOSE, + cmd_flags, + token); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpio_create() - Create the DPIO object. + * @mc_io: Pointer to MC portal's I/O object + * @dprc_token: Parent container token; '0' for default container + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @cfg: Configuration structure + * @obj_id: Returned object id + * + * Create the DPIO object, allocate required resources and + * perform required initialization. + * + * The object can be created either by declaring it in the + * DPL file, or by calling this function. + * + * The function accepts an authentication token of a parent + * container that this object should be assigned to. The token + * can be '0' so the object will be assigned to the default container. + * The newly created object can be opened with the returned + * object id and using the container's associated tokens and MC portals. + * + * Return: '0' on Success; Error code otherwise. + */ +int dpio_create(struct fsl_mc_io *mc_io, + uint16_t dprc_token, + uint32_t cmd_flags, + const struct dpio_cfg *cfg, + uint32_t *obj_id) +{ + struct dpio_cmd_create *cmd_params; + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPIO_CMDID_CREATE, + cmd_flags, + dprc_token); + cmd_params = (struct dpio_cmd_create *)cmd.params; + cmd_params->num_priorities = cfg->num_priorities; + dpio_set_field(cmd_params->channel_mode, + CHANNEL_MODE, + cfg->channel_mode); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + *obj_id = mc_cmd_read_object_id(&cmd); + + return 0; +} + +/** + * dpio_destroy() - Destroy the DPIO object and release all its resources. + * @mc_io: Pointer to MC portal's I/O object + * @dprc_token: Parent container token; '0' for default container + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @object_id: The object id; it must be a valid id within the container that + * created this object; + * + * The function accepts the authentication token of the parent container that + * created the object (not the one that currently owns the object). The object + * is searched within parent using the provided 'object_id'. + * All tokens to the object must be closed before calling destroy. + * + * Return: '0' on Success; Error code otherwise + */ +int dpio_destroy(struct fsl_mc_io *mc_io, + uint16_t dprc_token, + uint32_t cmd_flags, + uint32_t object_id) +{ + struct dpio_cmd_destroy *cmd_params; + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPIO_CMDID_DESTROY, + cmd_flags, + dprc_token); + + /* set object id to destroy */ + cmd_params = (struct dpio_cmd_destroy *)cmd.params; + cmd_params->dpio_id = cpu_to_le32(object_id); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpio_enable() - Enable the DPIO, allow I/O portal operations. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPIO object + * + * Return: '0' on Success; Error code otherwise + */ +int dpio_enable(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token) +{ + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPIO_CMDID_ENABLE, + cmd_flags, + token); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpio_disable() - Disable the DPIO, stop any I/O portal operation. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPIO object + * + * Return: '0' on Success; Error code otherwise + */ +int dpio_disable(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token) +{ + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPIO_CMDID_DISABLE, + cmd_flags, + token); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpio_is_enabled() - Check if the DPIO is enabled. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPIO object + * @en: Returns '1' if object is enabled; '0' otherwise + * + * Return: '0' on Success; Error code otherwise. + */ +int dpio_is_enabled(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + int *en) +{ + struct dpio_rsp_is_enabled *rsp_params; + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPIO_CMDID_IS_ENABLED, cmd_flags, + token); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + rsp_params = (struct dpio_rsp_is_enabled *)cmd.params; + *en = dpio_get_field(rsp_params->en, ENABLE); + + return 0; +} + +/** + * dpio_reset() - Reset the DPIO, returns the object to initial state. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPIO object + * + * Return: '0' on Success; Error code otherwise. + */ +int dpio_reset(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token) +{ + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPIO_CMDID_RESET, + cmd_flags, + token); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +int dpio_get_attributes(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + struct dpio_attr *attr) +{ + struct dpio_rsp_get_attr *rsp_params; + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPIO_CMDID_GET_ATTR, + cmd_flags, + token); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + rsp_params = (struct dpio_rsp_get_attr *)cmd.params; + attr->id = le32_to_cpu(rsp_params->id); + attr->qbman_portal_id = le16_to_cpu(rsp_params->qbman_portal_id); + attr->num_priorities = rsp_params->num_priorities; + attr->qbman_portal_ce_offset = + le64_to_cpu(rsp_params->qbman_portal_ce_offset); + attr->qbman_portal_ci_offset = + le64_to_cpu(rsp_params->qbman_portal_ci_offset); + attr->qbman_version = le32_to_cpu(rsp_params->qbman_version); + attr->clk = le32_to_cpu(rsp_params->clk); + attr->channel_mode = dpio_get_field(rsp_params->channel_mode, + ATTR_CHANNEL_MODE); + + return 0; +} + +/** + * dpio_set_stashing_destination() - Set the stashing destination. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPIO object + * @sdest: Stashing destination value + * + * Return: '0' on Success; Error code otherwise. + */ +int dpio_set_stashing_destination(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t sdest) +{ + struct dpio_stashing_dest *cmd_params; + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPIO_CMDID_SET_STASHING_DEST, + cmd_flags, + token); + cmd_params = (struct dpio_stashing_dest *)cmd.params; + cmd_params->sdest = sdest; + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpio_get_stashing_destination() - Get the stashing destination.. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPIO object + * @sdest: Returns the stashing destination value + * + * Return: '0' on Success; Error code otherwise. + */ +int dpio_get_stashing_destination(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t *sdest) +{ + struct dpio_stashing_dest *rsp_params; + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPIO_CMDID_GET_STASHING_DEST, + cmd_flags, + token); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + rsp_params = (struct dpio_stashing_dest *)cmd.params; + *sdest = rsp_params->sdest; + + return 0; +} + +/** + * dpio_add_static_dequeue_channel() - Add a static dequeue channel. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPIO object + * @dpcon_id: DPCON object ID + * @channel_index: Returned channel index to be used in qbman API + * + * Return: '0' on Success; Error code otherwise. + */ +int dpio_add_static_dequeue_channel(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + int dpcon_id, + uint8_t *channel_index) +{ + struct dpio_rsp_add_static_dequeue_channel *rsp_params; + struct dpio_cmd_static_dequeue_channel *cmd_params; + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPIO_CMDID_ADD_STATIC_DEQUEUE_CHANNEL, + cmd_flags, + token); + cmd_params = (struct dpio_cmd_static_dequeue_channel *)cmd.params; + cmd_params->dpcon_id = cpu_to_le32(dpcon_id); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + rsp_params = (struct dpio_rsp_add_static_dequeue_channel *)cmd.params; + *channel_index = rsp_params->channel_index; + + return 0; +} + +/** + * dpio_remove_static_dequeue_channel() - Remove a static dequeue channel. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPIO object + * @dpcon_id: DPCON object ID + * + * Return: '0' on Success; Error code otherwise. + */ +int dpio_remove_static_dequeue_channel(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + int dpcon_id) +{ + struct dpio_cmd_static_dequeue_channel *cmd_params; + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header( + DPIO_CMDID_REMOVE_STATIC_DEQUEUE_CHANNEL, + cmd_flags, + token); + cmd_params = (struct dpio_cmd_static_dequeue_channel *)cmd.params; + cmd_params->dpcon_id = cpu_to_le32(dpcon_id); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpio_get_api_version() - Get Data Path I/O API version + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @major_ver: Major version of data path i/o API + * @minor_ver: Minor version of data path i/o API + * + * Return: '0' on Success; Error code otherwise. + */ +int dpio_get_api_version(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t *major_ver, + uint16_t *minor_ver) +{ + struct dpio_rsp_get_api_version *rsp_params; + struct mc_command cmd = { 0 }; + int err; + + cmd.header = mc_encode_cmd_header(DPIO_CMDID_GET_API_VERSION, + cmd_flags, + 0); + + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + rsp_params = (struct dpio_rsp_get_api_version *)cmd.params; + *major_ver = le16_to_cpu(rsp_params->major); + *minor_ver = le16_to_cpu(rsp_params->minor); + + return 0; +} diff --git a/dpdk/drivers/bus/fslmc/mc/dpmng.c b/dpdk/drivers/bus/fslmc/mc/dpmng.c new file mode 100644 index 00000000..f9946e8c --- /dev/null +++ b/dpdk/drivers/bus/fslmc/mc/dpmng.c @@ -0,0 +1,117 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2013-2015 Freescale Semiconductor Inc. + * Copyright 2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#include +#include +#include +#include + +/** + * mc_get_version() - Retrieves the Management Complex firmware + * version information + * @mc_io: Pointer to opaque I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @mc_ver_info: Returned version information structure + * + * Return: '0' on Success; Error code otherwise. + */ +int mc_get_version(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + struct mc_version *mc_ver_info) +{ + struct mc_command cmd = { 0 }; + struct dpmng_rsp_get_version *rsp_params; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPMNG_CMDID_GET_VERSION, + cmd_flags, + 0); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + rsp_params = (struct dpmng_rsp_get_version *)cmd.params; + mc_ver_info->revision = le32_to_cpu(rsp_params->revision); + mc_ver_info->major = le32_to_cpu(rsp_params->version_major); + mc_ver_info->minor = le32_to_cpu(rsp_params->version_minor); + + return 0; +} + +/** + * mc_get_soc_version() - Retrieves the Management Complex firmware + * version information + * @mc_io Pointer to opaque I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @mc_platform_info: Returned version information structure. The structure + * contains the values of SVR and PVR registers. + * Please consult platform specific reference manual + * for detailed information. + * + * Return: '0' on Success; Error code otherwise. + */ +int mc_get_soc_version(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + struct mc_soc_version *mc_platform_info) +{ + struct dpmng_rsp_get_soc_version *rsp_params; + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPMNG_CMDID_GET_SOC_VERSION, + cmd_flags, + 0); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + rsp_params = (struct dpmng_rsp_get_soc_version *)cmd.params; + mc_platform_info->svr = le32_to_cpu(rsp_params->svr); + mc_platform_info->pvr = le32_to_cpu(rsp_params->pvr); + + return 0; +} diff --git a/dpdk/drivers/bus/fslmc/mc/fsl_dpbp.h b/dpdk/drivers/bus/fslmc/mc/fsl_dpbp.h new file mode 100644 index 00000000..77ea6f27 --- /dev/null +++ b/dpdk/drivers/bus/fslmc/mc/fsl_dpbp.h @@ -0,0 +1,128 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2013-2016 Freescale Semiconductor Inc. + * Copyright 2016-2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef __FSL_DPBP_H +#define __FSL_DPBP_H + +/* + * Data Path Buffer Pool API + * Contains initialization APIs and runtime control APIs for DPBP + */ + +struct fsl_mc_io; + +int dpbp_open(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + int dpbp_id, + uint16_t *token); + +int dpbp_close(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token); + +/** + * struct dpbp_cfg - Structure representing DPBP configuration + * @options: place holder + */ +struct dpbp_cfg { + uint32_t options; +}; + +int dpbp_create(struct fsl_mc_io *mc_io, + uint16_t dprc_token, + uint32_t cmd_flags, + const struct dpbp_cfg *cfg, + uint32_t *obj_id); + +int dpbp_destroy(struct fsl_mc_io *mc_io, + uint16_t dprc_token, + uint32_t cmd_flags, + uint32_t obj_id); + +int dpbp_enable(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token); + +int dpbp_disable(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token); + +int dpbp_is_enabled(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + int *en); + +int dpbp_reset(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token); + +/** + * struct dpbp_attr - Structure representing DPBP attributes + * @id: DPBP object ID + * @bpid: Hardware buffer pool ID; should be used as an argument in + * acquire/release operations on buffers + */ +struct dpbp_attr { + int id; + uint16_t bpid; +}; + +int dpbp_get_attributes(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + struct dpbp_attr *attr); + +/** + * DPBP notifications options + */ + +/** + * BPSCN write will attempt to allocate into a cache (coherent write) + */ +int dpbp_get_api_version(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t *major_ver, + uint16_t *minor_ver); + +int dpbp_get_num_free_bufs(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint32_t *num_free_bufs); + +#endif /* __FSL_DPBP_H */ diff --git a/dpdk/drivers/bus/fslmc/mc/fsl_dpbp_cmd.h b/dpdk/drivers/bus/fslmc/mc/fsl_dpbp_cmd.h new file mode 100644 index 00000000..ce38c79b --- /dev/null +++ b/dpdk/drivers/bus/fslmc/mc/fsl_dpbp_cmd.h @@ -0,0 +1,131 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2013-2016 Freescale Semiconductor Inc. + * Copyright 2016-2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef _FSL_DPBP_CMD_H +#define _FSL_DPBP_CMD_H + +/* DPBP Version */ +#define DPBP_VER_MAJOR 3 +#define DPBP_VER_MINOR 3 + +/* Command versioning */ +#define DPBP_CMD_BASE_VERSION 1 +#define DPBP_CMD_ID_OFFSET 4 + +#define DPBP_CMD(id) ((id << DPBP_CMD_ID_OFFSET) | DPBP_CMD_BASE_VERSION) + +/* Command IDs */ +#define DPBP_CMDID_CLOSE DPBP_CMD(0x800) +#define DPBP_CMDID_OPEN DPBP_CMD(0x804) +#define DPBP_CMDID_CREATE DPBP_CMD(0x904) +#define DPBP_CMDID_DESTROY DPBP_CMD(0x984) +#define DPBP_CMDID_GET_API_VERSION DPBP_CMD(0xa04) + +#define DPBP_CMDID_ENABLE DPBP_CMD(0x002) +#define DPBP_CMDID_DISABLE DPBP_CMD(0x003) +#define DPBP_CMDID_GET_ATTR DPBP_CMD(0x004) +#define DPBP_CMDID_RESET DPBP_CMD(0x005) +#define DPBP_CMDID_IS_ENABLED DPBP_CMD(0x006) + +#define DPBP_CMDID_SET_IRQ_ENABLE DPBP_CMD(0x012) +#define DPBP_CMDID_GET_IRQ_ENABLE DPBP_CMD(0x013) +#define DPBP_CMDID_SET_IRQ_MASK DPBP_CMD(0x014) +#define DPBP_CMDID_GET_IRQ_MASK DPBP_CMD(0x015) +#define DPBP_CMDID_GET_IRQ_STATUS DPBP_CMD(0x016) +#define DPBP_CMDID_CLEAR_IRQ_STATUS DPBP_CMD(0x017) + +#define DPBP_CMDID_SET_NOTIFICATIONS DPBP_CMD(0x1b0) +#define DPBP_CMDID_GET_NOTIFICATIONS DPBP_CMD(0x1b1) + +#define DPBP_CMDID_GET_FREE_BUFFERS_NUM DPBP_CMD(0x1b2) + +#pragma pack(push, 1) +struct dpbp_cmd_open { + uint32_t dpbp_id; +}; + +struct dpbp_cmd_destroy { + uint32_t object_id; +}; + +#define DPBP_ENABLE 0x1 + +struct dpbp_rsp_is_enabled { + uint8_t enabled; +}; + +struct dpbp_rsp_get_attributes { + uint16_t pad; + uint16_t bpid; + uint32_t id; +}; + +struct dpbp_cmd_set_notifications { + uint32_t depletion_entry; + uint32_t depletion_exit; + uint32_t surplus_entry; + uint32_t surplus_exit; + uint16_t options; + uint16_t pad[3]; + uint64_t message_ctx; + uint64_t message_iova; +}; + +struct dpbp_rsp_get_notifications { + uint32_t depletion_entry; + uint32_t depletion_exit; + uint32_t surplus_entry; + uint32_t surplus_exit; + uint16_t options; + uint16_t pad[3]; + uint64_t message_ctx; + uint64_t message_iova; +}; + +struct dpbp_rsp_get_api_version { + uint16_t major; + uint16_t minor; +}; + +struct dpbp_rsp_get_num_free_bufs { + uint32_t num_free_bufs; +}; + +#pragma pack(pop) +#endif /* _FSL_DPBP_CMD_H */ diff --git a/dpdk/drivers/bus/fslmc/mc/fsl_dpci.h b/dpdk/drivers/bus/fslmc/mc/fsl_dpci.h new file mode 100644 index 00000000..f4aa6e57 --- /dev/null +++ b/dpdk/drivers/bus/fslmc/mc/fsl_dpci.h @@ -0,0 +1,253 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2013-2016 Freescale Semiconductor Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef __FSL_DPCI_H +#define __FSL_DPCI_H + +/* Data Path Communication Interface API + * Contains initialization APIs and runtime control APIs for DPCI + */ + +struct fsl_mc_io; + +/** General DPCI macros */ + +/** + * Maximum number of Tx/Rx priorities per DPCI object + */ +#define DPCI_PRIO_NUM 2 + +/** + * Indicates an invalid frame queue + */ +#define DPCI_FQID_NOT_VALID (uint32_t)(-1) + +/** + * All queues considered; see dpci_set_rx_queue() + */ +#define DPCI_ALL_QUEUES (uint8_t)(-1) + +int dpci_open(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + int dpci_id, + uint16_t *token); + +int dpci_close(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token); + +/** + * Enable the Order Restoration support + */ +#define DPCI_OPT_HAS_OPR 0x000040 + +/** + * Order Point Records are shared for the entire DPCI + */ +#define DPCI_OPT_OPR_SHARED 0x000080 + +/** + * struct dpci_cfg - Structure representing DPCI configuration + * @options: Any combination of the following options: + * DPCI_OPT_HAS_OPR + * DPCI_OPT_OPR_SHARED + * @num_of_priorities: Number of receive priorities (queues) for the DPCI; + * note, that the number of transmit priorities (queues) + * is determined by the number of receive priorities of + * the peer DPCI object + */ +struct dpci_cfg { + uint32_t options; + uint8_t num_of_priorities; +}; + +int dpci_create(struct fsl_mc_io *mc_io, + uint16_t dprc_token, + uint32_t cmd_flags, + const struct dpci_cfg *cfg, + uint32_t *obj_id); + +int dpci_destroy(struct fsl_mc_io *mc_io, + uint16_t dprc_token, + uint32_t cmd_flags, + uint32_t object_id); + +int dpci_enable(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token); + +int dpci_disable(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token); + +int dpci_is_enabled(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + int *en); + +int dpci_reset(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token); + +/** + * struct dpci_attr - Structure representing DPCI attributes + * @id: DPCI object ID + * @num_of_priorities: Number of receive priorities + */ +struct dpci_attr { + int id; + uint8_t num_of_priorities; +}; + +int dpci_get_attributes(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + struct dpci_attr *attr); + +/** + * enum dpci_dest - DPCI destination types + * @DPCI_DEST_NONE: Unassigned destination; The queue is set in parked mode + * and does not generate FQDAN notifications; user is + * expected to dequeue from the queue based on polling or + * other user-defined method + * @DPCI_DEST_DPIO: The queue is set in schedule mode and generates FQDAN + * notifications to the specified DPIO; user is expected + * to dequeue from the queue only after notification is + * received + * @DPCI_DEST_DPCON: The queue is set in schedule mode and does not generate + * FQDAN notifications, but is connected to the specified + * DPCON object; + * user is expected to dequeue from the DPCON channel + */ +enum dpci_dest { + DPCI_DEST_NONE = 0, + DPCI_DEST_DPIO = 1, + DPCI_DEST_DPCON = 2 +}; + +/** + * struct dpci_dest_cfg - Structure representing DPCI destination configuration + * @dest_type: Destination type + * @dest_id: Either DPIO ID or DPCON ID, depending on the destination type + * @priority: Priority selection within the DPIO or DPCON channel; valid + * values are 0-1 or 0-7, depending on the number of priorities + * in that channel; not relevant for 'DPCI_DEST_NONE' option + */ +struct dpci_dest_cfg { + enum dpci_dest dest_type; + int dest_id; + uint8_t priority; +}; + +/** DPCI queue modification options */ + +/** + * Select to modify the user's context associated with the queue + */ +#define DPCI_QUEUE_OPT_USER_CTX 0x00000001 + +/** + * Select to modify the queue's destination + */ +#define DPCI_QUEUE_OPT_DEST 0x00000002 + +/** + * struct dpci_rx_queue_cfg - Structure representing RX queue configuration + * @options: Flags representing the suggested modifications to the queue; + * Use any combination of 'DPCI_QUEUE_OPT_' flags + * @user_ctx: User context value provided in the frame descriptor of each + * dequeued frame; + * valid only if 'DPCI_QUEUE_OPT_USER_CTX' is contained in + * 'options' + * @dest_cfg: Queue destination parameters; + * valid only if 'DPCI_QUEUE_OPT_DEST' is contained in 'options' + */ +struct dpci_rx_queue_cfg { + uint32_t options; + uint64_t user_ctx; + struct dpci_dest_cfg dest_cfg; +}; + +int dpci_set_rx_queue(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t priority, + const struct dpci_rx_queue_cfg *cfg); + +/** + * struct dpci_rx_queue_attr - Structure representing Rx queue attributes + * @user_ctx: User context value provided in the frame descriptor of each + * dequeued frame + * @dest_cfg: Queue destination configuration + * @fqid: Virtual FQID value to be used for dequeue operations + */ +struct dpci_rx_queue_attr { + uint64_t user_ctx; + struct dpci_dest_cfg dest_cfg; + uint32_t fqid; +}; + +int dpci_get_rx_queue(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t priority, + struct dpci_rx_queue_attr *attr); + +/** + * struct dpci_tx_queue_attr - Structure representing attributes of Tx queues + * @fqid: Virtual FQID to be used for sending frames to peer DPCI; + * returns 'DPCI_FQID_NOT_VALID' if a no peer is connected or if + * the selected priority exceeds the number of priorities of the + * peer DPCI object + */ +struct dpci_tx_queue_attr { + uint32_t fqid; +}; + +int dpci_get_tx_queue(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t priority, + struct dpci_tx_queue_attr *attr); + +int dpci_get_api_version(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t *major_ver, + uint16_t *minor_ver); + +#endif /* __FSL_DPCI_H */ diff --git a/dpdk/drivers/bus/fslmc/mc/fsl_dpci_cmd.h b/dpdk/drivers/bus/fslmc/mc/fsl_dpci_cmd.h new file mode 100644 index 00000000..73a551a5 --- /dev/null +++ b/dpdk/drivers/bus/fslmc/mc/fsl_dpci_cmd.h @@ -0,0 +1,165 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2013-2016 Freescale Semiconductor Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef _FSL_DPCI_CMD_H +#define _FSL_DPCI_CMD_H + +/* DPCI Version */ +#define DPCI_VER_MAJOR 3 +#define DPCI_VER_MINOR 3 + +#define DPCI_CMD_BASE_VERSION 1 +#define DPCI_CMD_BASE_VERSION_V2 2 +#define DPCI_CMD_ID_OFFSET 4 + +#define DPCI_CMD_V1(id) ((id << DPCI_CMD_ID_OFFSET) | DPCI_CMD_BASE_VERSION) +#define DPCI_CMD_V2(id) ((id << DPCI_CMD_ID_OFFSET) | DPCI_CMD_BASE_VERSION_V2) + +/* Command IDs */ +#define DPCI_CMDID_CLOSE DPCI_CMD_V1(0x800) +#define DPCI_CMDID_OPEN DPCI_CMD_V1(0x807) +#define DPCI_CMDID_CREATE DPCI_CMD_V2(0x907) +#define DPCI_CMDID_DESTROY DPCI_CMD_V1(0x987) +#define DPCI_CMDID_GET_API_VERSION DPCI_CMD_V1(0xa07) + +#define DPCI_CMDID_ENABLE DPCI_CMD_V1(0x002) +#define DPCI_CMDID_DISABLE DPCI_CMD_V1(0x003) +#define DPCI_CMDID_GET_ATTR DPCI_CMD_V1(0x004) +#define DPCI_CMDID_RESET DPCI_CMD_V1(0x005) +#define DPCI_CMDID_IS_ENABLED DPCI_CMD_V1(0x006) + +#define DPCI_CMDID_SET_RX_QUEUE DPCI_CMD_V1(0x0e0) +#define DPCI_CMDID_GET_LINK_STATE DPCI_CMD_V1(0x0e1) +#define DPCI_CMDID_GET_PEER_ATTR DPCI_CMD_V1(0x0e2) +#define DPCI_CMDID_GET_RX_QUEUE DPCI_CMD_V1(0x0e3) +#define DPCI_CMDID_GET_TX_QUEUE DPCI_CMD_V1(0x0e4) + +/* Macros for accessing command fields smaller than 1byte */ +#define DPCI_MASK(field) \ + GENMASK(DPCI_##field##_SHIFT + DPCI_##field##_SIZE - 1, \ + DPCI_##field##_SHIFT) +#define dpci_set_field(var, field, val) \ + ((var) |= (((val) << DPCI_##field##_SHIFT) & DPCI_MASK(field))) +#define dpci_get_field(var, field) \ + (((var) & DPCI_MASK(field)) >> DPCI_##field##_SHIFT) + +#pragma pack(push, 1) +struct dpci_cmd_open { + uint32_t dpci_id; +}; + +struct dpci_cmd_create { + uint8_t num_of_priorities; + uint8_t pad[15]; + uint32_t options; +}; + +struct dpci_cmd_destroy { + uint32_t dpci_id; +}; + +#define DPCI_ENABLE_SHIFT 0 +#define DPCI_ENABLE_SIZE 1 + +struct dpci_rsp_is_enabled { + /* only the LSB bit */ + uint8_t en; +}; + +struct dpci_rsp_get_attr { + uint32_t id; + uint16_t pad; + uint8_t num_of_priorities; +}; + +struct dpci_rsp_get_peer_attr { + uint32_t id; + uint32_t pad; + uint8_t num_of_priorities; +}; + +#define DPCI_UP_SHIFT 0 +#define DPCI_UP_SIZE 1 + +struct dpci_rsp_get_link_state { + /* only the LSB bit */ + uint8_t up; +}; + +#define DPCI_DEST_TYPE_SHIFT 0 +#define DPCI_DEST_TYPE_SIZE 4 + +struct dpci_cmd_set_rx_queue { + uint32_t dest_id; + uint8_t dest_priority; + uint8_t priority; + /* from LSB: dest_type:4 */ + uint8_t dest_type; + uint8_t pad; + uint64_t user_ctx; + uint32_t options; +}; + +struct dpci_cmd_get_queue { + uint8_t pad[5]; + uint8_t priority; +}; + +struct dpci_rsp_get_rx_queue { + uint32_t dest_id; + uint8_t dest_priority; + uint8_t pad; + /* from LSB: dest_type:4 */ + uint8_t dest_type; + uint8_t pad1; + uint64_t user_ctx; + uint32_t fqid; +}; + +struct dpci_rsp_get_tx_queue { + uint32_t pad; + uint32_t fqid; +}; + +struct dpci_rsp_get_api_version { + uint16_t major; + uint16_t minor; +}; + +#pragma pack(pop) +#endif /* _FSL_DPCI_CMD_H */ diff --git a/dpdk/drivers/bus/fslmc/mc/fsl_dpcon.h b/dpdk/drivers/bus/fslmc/mc/fsl_dpcon.h new file mode 100644 index 00000000..1da807f0 --- /dev/null +++ b/dpdk/drivers/bus/fslmc/mc/fsl_dpcon.h @@ -0,0 +1,122 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2013-2016 Freescale Semiconductor Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef __FSL_DPCON_H +#define __FSL_DPCON_H + +/* Data Path Concentrator API + * Contains initialization APIs and runtime control APIs for DPCON + */ + +struct fsl_mc_io; + +/** General DPCON macros */ + +/** + * Use it to disable notifications; see dpcon_set_notification() + */ +#define DPCON_INVALID_DPIO_ID (int)(-1) + +int dpcon_open(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + int dpcon_id, + uint16_t *token); + +int dpcon_close(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token); + +/** + * struct dpcon_cfg - Structure representing DPCON configuration + * @num_priorities: Number of priorities for the DPCON channel (1-8) + */ +struct dpcon_cfg { + uint8_t num_priorities; +}; + +int dpcon_create(struct fsl_mc_io *mc_io, + uint16_t dprc_token, + uint32_t cmd_flags, + const struct dpcon_cfg *cfg, + uint32_t *obj_id); + +int dpcon_destroy(struct fsl_mc_io *mc_io, + uint16_t dprc_token, + uint32_t cmd_flags, + uint32_t obj_id); + +int dpcon_enable(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token); + +int dpcon_disable(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token); + +int dpcon_is_enabled(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + int *en); + +int dpcon_reset(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token); + +/** + * struct dpcon_attr - Structure representing DPCON attributes + * @id: DPCON object ID + * @qbman_ch_id: Channel ID to be used by dequeue operation + * @num_priorities: Number of priorities for the DPCON channel (1-8) + */ +struct dpcon_attr { + int id; + uint16_t qbman_ch_id; + uint8_t num_priorities; +}; + +int dpcon_get_attributes(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + struct dpcon_attr *attr); + +int dpcon_get_api_version(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t *major_ver, + uint16_t *minor_ver); + +#endif /* __FSL_DPCON_H */ diff --git a/dpdk/drivers/bus/fslmc/mc/fsl_dpcon_cmd.h b/dpdk/drivers/bus/fslmc/mc/fsl_dpcon_cmd.h new file mode 100644 index 00000000..4d0522c2 --- /dev/null +++ b/dpdk/drivers/bus/fslmc/mc/fsl_dpcon_cmd.h @@ -0,0 +1,108 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2013-2016 Freescale Semiconductor Inc. + * Copyright 2016-2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef _FSL_DPCON_CMD_H +#define _FSL_DPCON_CMD_H + +/* DPCON Version */ +#define DPCON_VER_MAJOR 3 +#define DPCON_VER_MINOR 3 + + +/* Command versioning */ +#define DPCON_CMD_BASE_VERSION 1 +#define DPCON_CMD_ID_OFFSET 4 + +#define DPCON_CMD(id) ((id << DPCON_CMD_ID_OFFSET) | DPCON_CMD_BASE_VERSION) + +/* Command IDs */ +#define DPCON_CMDID_CLOSE DPCON_CMD(0x800) +#define DPCON_CMDID_OPEN DPCON_CMD(0x808) +#define DPCON_CMDID_CREATE DPCON_CMD(0x908) +#define DPCON_CMDID_DESTROY DPCON_CMD(0x988) +#define DPCON_CMDID_GET_API_VERSION DPCON_CMD(0xa08) + +#define DPCON_CMDID_ENABLE DPCON_CMD(0x002) +#define DPCON_CMDID_DISABLE DPCON_CMD(0x003) +#define DPCON_CMDID_GET_ATTR DPCON_CMD(0x004) +#define DPCON_CMDID_RESET DPCON_CMD(0x005) +#define DPCON_CMDID_IS_ENABLED DPCON_CMD(0x006) + +#define DPCON_CMDID_SET_NOTIFICATION DPCON_CMD(0x100) + +#pragma pack(push, 1) +struct dpcon_cmd_open { + uint32_t dpcon_id; +}; + +struct dpcon_cmd_create { + uint8_t num_priorities; +}; + +struct dpcon_cmd_destroy { + uint32_t object_id; +}; + +#define DPCON_ENABLE 1 + +struct dpcon_rsp_is_enabled { + uint8_t enabled; +}; + +struct dpcon_rsp_get_attr { + uint32_t id; + uint16_t qbman_ch_id; + uint8_t num_priorities; + uint8_t pad; +}; + +struct dpcon_cmd_set_notification { + uint32_t dpio_id; + uint8_t priority; + uint8_t pad[3]; + uint64_t user_ctx; +}; + +struct dpcon_rsp_get_api_version { + uint16_t major; + uint16_t minor; +}; + +#pragma pack(pop) +#endif /* _FSL_DPCON_CMD_H */ diff --git a/dpdk/drivers/bus/fslmc/mc/fsl_dpio.h b/dpdk/drivers/bus/fslmc/mc/fsl_dpio.h new file mode 100644 index 00000000..3d96adfc --- /dev/null +++ b/dpdk/drivers/bus/fslmc/mc/fsl_dpio.h @@ -0,0 +1,165 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2013-2016 Freescale Semiconductor Inc. + * Copyright 2016-2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef __FSL_DPIO_H +#define __FSL_DPIO_H + +/* Data Path I/O Portal API + * Contains initialization APIs and runtime control APIs for DPIO + */ + +struct fsl_mc_io; + +int dpio_open(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + int dpio_id, + uint16_t *token); + +int dpio_close(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token); + +/** + * enum dpio_channel_mode - DPIO notification channel mode + * @DPIO_NO_CHANNEL: No support for notification channel + * @DPIO_LOCAL_CHANNEL: Notifications on data availability can be received by a + * dedicated channel in the DPIO; user should point the queue's + * destination in the relevant interface to this DPIO + */ +enum dpio_channel_mode { + DPIO_NO_CHANNEL = 0, + DPIO_LOCAL_CHANNEL = 1, +}; + +/** + * struct dpio_cfg - Structure representing DPIO configuration + * @channel_mode: Notification channel mode + * @num_priorities: Number of priorities for the notification channel (1-8); + * relevant only if 'channel_mode = DPIO_LOCAL_CHANNEL' + */ +struct dpio_cfg { + enum dpio_channel_mode channel_mode; + uint8_t num_priorities; +}; + + +int dpio_create(struct fsl_mc_io *mc_io, + uint16_t dprc_token, + uint32_t cmd_flags, + const struct dpio_cfg *cfg, + uint32_t *obj_id); + +int dpio_destroy(struct fsl_mc_io *mc_io, + uint16_t dprc_token, + uint32_t cmd_flags, + uint32_t object_id); + +int dpio_enable(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token); + +int dpio_disable(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token); + +int dpio_is_enabled(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + int *en); + +int dpio_reset(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token); + +int dpio_set_stashing_destination(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t sdest); + +int dpio_get_stashing_destination(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t *sdest); + +int dpio_add_static_dequeue_channel(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + int dpcon_id, + uint8_t *channel_index); + +int dpio_remove_static_dequeue_channel(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + int dpcon_id); + +/** + * struct dpio_attr - Structure representing DPIO attributes + * @id: DPIO object ID + * @qbman_portal_ce_offset: Offset of the software portal cache-enabled area + * @qbman_portal_ci_offset: Offset of the software portal + * cache-inhibited area + * @qbman_portal_id: Software portal ID + * @channel_mode: Notification channel mode + * @num_priorities: Number of priorities for the notification + * channel (1-8); relevant only if + * 'channel_mode = DPIO_LOCAL_CHANNEL' + * @qbman_version: QBMAN version + */ +struct dpio_attr { + int id; + uint64_t qbman_portal_ce_offset; + uint64_t qbman_portal_ci_offset; + uint16_t qbman_portal_id; + enum dpio_channel_mode channel_mode; + uint8_t num_priorities; + uint32_t qbman_version; + uint32_t clk; +}; + +int dpio_get_attributes(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + struct dpio_attr *attr); + +int dpio_get_api_version(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t *major_ver, + uint16_t *minor_ver); + +#endif /* __FSL_DPIO_H */ diff --git a/dpdk/drivers/bus/fslmc/mc/fsl_dpio_cmd.h b/dpdk/drivers/bus/fslmc/mc/fsl_dpio_cmd.h new file mode 100644 index 00000000..3e9e1f68 --- /dev/null +++ b/dpdk/drivers/bus/fslmc/mc/fsl_dpio_cmd.h @@ -0,0 +1,148 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2013-2016 Freescale Semiconductor Inc. + * Copyright 2016-2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef _FSL_DPIO_CMD_H +#define _FSL_DPIO_CMD_H + +/* DPIO Version */ +#define DPIO_VER_MAJOR 4 +#define DPIO_VER_MINOR 2 + +#define DPIO_CMD_BASE_VERSION 1 +#define DPIO_CMD_ID_OFFSET 4 + +#define DPIO_CMD(id) (((id) << DPIO_CMD_ID_OFFSET) | DPIO_CMD_BASE_VERSION) + +/* Command IDs */ +#define DPIO_CMDID_CLOSE DPIO_CMD(0x800) +#define DPIO_CMDID_OPEN DPIO_CMD(0x803) +#define DPIO_CMDID_CREATE DPIO_CMD(0x903) +#define DPIO_CMDID_DESTROY DPIO_CMD(0x983) +#define DPIO_CMDID_GET_API_VERSION DPIO_CMD(0xa03) + +#define DPIO_CMDID_ENABLE DPIO_CMD(0x002) +#define DPIO_CMDID_DISABLE DPIO_CMD(0x003) +#define DPIO_CMDID_GET_ATTR DPIO_CMD(0x004) +#define DPIO_CMDID_RESET DPIO_CMD(0x005) +#define DPIO_CMDID_IS_ENABLED DPIO_CMD(0x006) + +#define DPIO_CMDID_SET_IRQ_ENABLE DPIO_CMD(0x012) +#define DPIO_CMDID_GET_IRQ_ENABLE DPIO_CMD(0x013) +#define DPIO_CMDID_SET_IRQ_MASK DPIO_CMD(0x014) +#define DPIO_CMDID_GET_IRQ_MASK DPIO_CMD(0x015) +#define DPIO_CMDID_GET_IRQ_STATUS DPIO_CMD(0x016) +#define DPIO_CMDID_CLEAR_IRQ_STATUS DPIO_CMD(0x017) + +#define DPIO_CMDID_SET_STASHING_DEST DPIO_CMD(0x120) +#define DPIO_CMDID_GET_STASHING_DEST DPIO_CMD(0x121) +#define DPIO_CMDID_ADD_STATIC_DEQUEUE_CHANNEL DPIO_CMD(0x122) +#define DPIO_CMDID_REMOVE_STATIC_DEQUEUE_CHANNEL DPIO_CMD(0x123) + +/* Macros for accessing command fields smaller than 1byte */ +#define DPIO_MASK(field) \ + GENMASK(DPIO_##field##_SHIFT + DPIO_##field##_SIZE - 1, \ + DPIO_##field##_SHIFT) +#define dpio_set_field(var, field, val) \ + ((var) |= (((val) << DPIO_##field##_SHIFT) & DPIO_MASK(field))) +#define dpio_get_field(var, field) \ + (((var) & DPIO_MASK(field)) >> DPIO_##field##_SHIFT) + +#pragma pack(push, 1) +struct dpio_cmd_open { + uint32_t dpio_id; +}; + +#define DPIO_CHANNEL_MODE_SHIFT 0 +#define DPIO_CHANNEL_MODE_SIZE 2 + +struct dpio_cmd_create { + uint16_t pad1; + /* from LSB: channel_mode:2 */ + uint8_t channel_mode; + uint8_t pad2; + uint8_t num_priorities; +}; + +struct dpio_cmd_destroy { + uint32_t dpio_id; +}; + +#define DPIO_ENABLE_SHIFT 0 +#define DPIO_ENABLE_SIZE 1 + +struct dpio_rsp_is_enabled { + /* only the LSB */ + uint8_t en; +}; + +#define DPIO_ATTR_CHANNEL_MODE_SHIFT 0 +#define DPIO_ATTR_CHANNEL_MODE_SIZE 4 + +struct dpio_rsp_get_attr { + uint32_t id; + uint16_t qbman_portal_id; + uint8_t num_priorities; + /* from LSB: channel_mode:4 */ + uint8_t channel_mode; + uint64_t qbman_portal_ce_offset; + uint64_t qbman_portal_ci_offset; + uint32_t qbman_version; + uint32_t pad; + uint32_t clk; +}; + +struct dpio_stashing_dest { + uint8_t sdest; +}; + +struct dpio_cmd_static_dequeue_channel { + uint32_t dpcon_id; +}; + +struct dpio_rsp_add_static_dequeue_channel { + uint8_t channel_index; +}; + +struct dpio_rsp_get_api_version { + uint16_t major; + uint16_t minor; +}; + +#pragma pack(pop) +#endif /* _FSL_DPIO_CMD_H */ diff --git a/dpdk/drivers/bus/fslmc/mc/fsl_dpmng.h b/dpdk/drivers/bus/fslmc/mc/fsl_dpmng.h new file mode 100644 index 00000000..97030e4c --- /dev/null +++ b/dpdk/drivers/bus/fslmc/mc/fsl_dpmng.h @@ -0,0 +1,87 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2013-2015 Freescale Semiconductor Inc. + * Copyright 2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef __FSL_DPMNG_H +#define __FSL_DPMNG_H + +/* + * Management Complex General API + * Contains general API for the Management Complex firmware + */ + +struct fsl_mc_io; + +/** + * Management Complex firmware version information + */ +#define MC_VER_MAJOR 10 +#define MC_VER_MINOR 3 + +/** + * struct mc_version + * @major: Major version number: incremented on API compatibility changes + * @minor: Minor version number: incremented on API additions (that are + * backward compatible); reset when major version is incremented + * @revision: Internal revision number: incremented on implementation changes + * and/or bug fixes that have no impact on API + */ +struct mc_version { + uint32_t major; + uint32_t minor; + uint32_t revision; +}; + +int mc_get_version(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + struct mc_version *mc_ver_info); + +/** + * struct mc_platform + * @svr: System version (content of platform SVR register) + * @pvr: Processor version (content of platform PVR register) + */ +struct mc_soc_version { + uint32_t svr; + uint32_t pvr; +}; + +int mc_get_soc_version(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + struct mc_soc_version *mc_platform_info); +#endif /* __FSL_DPMNG_H */ diff --git a/dpdk/drivers/bus/fslmc/mc/fsl_dpmng_cmd.h b/dpdk/drivers/bus/fslmc/mc/fsl_dpmng_cmd.h new file mode 100644 index 00000000..4c0a6297 --- /dev/null +++ b/dpdk/drivers/bus/fslmc/mc/fsl_dpmng_cmd.h @@ -0,0 +1,68 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2013-2016 Freescale Semiconductor Inc. + * Copyright 2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __FSL_DPMNG_CMD_H +#define __FSL_DPMNG_CMD_H + +/* Command versioning */ +#define DPMNG_CMD_BASE_VERSION 1 +#define DPMNG_CMD_ID_OFFSET 4 + +#define DPMNG_CMD(id) ((id << DPMNG_CMD_ID_OFFSET) | DPMNG_CMD_BASE_VERSION) + +/* Command IDs */ +#define DPMNG_CMDID_GET_VERSION DPMNG_CMD(0x831) +#define DPMNG_CMDID_GET_SOC_VERSION DPMNG_CMD(0x832) + +#pragma pack(push, 1) +struct dpmng_rsp_get_version { + uint32_t revision; + uint32_t version_major; + uint32_t version_minor; +}; + +struct dpmng_rsp_get_soc_version { + uint32_t svr; + uint32_t pvr; +}; + +#pragma pack(pop) + +#endif /* __FSL_DPMNG_CMD_H */ diff --git a/dpdk/drivers/bus/fslmc/mc/fsl_mc_cmd.h b/dpdk/drivers/bus/fslmc/mc/fsl_mc_cmd.h new file mode 100644 index 00000000..2cec29ea --- /dev/null +++ b/dpdk/drivers/bus/fslmc/mc/fsl_mc_cmd.h @@ -0,0 +1,208 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2013-2016 Freescale Semiconductor Inc. + * Copyright 2016-2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef __FSL_MC_CMD_H +#define __FSL_MC_CMD_H + +#include +#include + +#define MC_CMD_NUM_OF_PARAMS 7 + +#define phys_addr_t uint64_t + +#define u64 uint64_t +#define u32 uint32_t +#define u16 uint16_t +#define u8 uint8_t + +#define cpu_to_le64 rte_cpu_to_le_64 +#define cpu_to_le32 rte_cpu_to_le_32 +#define cpu_to_le16 rte_cpu_to_le_16 + +#define le64_to_cpu rte_le_to_cpu_64 +#define le32_to_cpu rte_le_to_cpu_32 +#define le16_to_cpu rte_le_to_cpu_16 + +#define BITS_PER_LONG 64 +#define GENMASK(h, l) \ + (((~0UL) << (l)) & (~0UL >> (BITS_PER_LONG - 1 - (h)))) + +struct mc_cmd_header { + union { + struct { + uint8_t src_id; + uint8_t flags_hw; + uint8_t status; + uint8_t flags_sw; + uint16_t token; + uint16_t cmd_id; + }; + uint32_t word[2]; + }; +}; + +struct mc_command { + uint64_t header; + uint64_t params[MC_CMD_NUM_OF_PARAMS]; +}; + +struct mc_rsp_create { + uint32_t object_id; +}; + +enum mc_cmd_status { + MC_CMD_STATUS_OK = 0x0, /* Completed successfully */ + MC_CMD_STATUS_READY = 0x1, /* Ready to be processed */ + MC_CMD_STATUS_AUTH_ERR = 0x3, /* Authentication error */ + MC_CMD_STATUS_NO_PRIVILEGE = 0x4, /* No privilege */ + MC_CMD_STATUS_DMA_ERR = 0x5, /* DMA or I/O error */ + MC_CMD_STATUS_CONFIG_ERR = 0x6, /* Configuration error */ + MC_CMD_STATUS_TIMEOUT = 0x7, /* Operation timed out */ + MC_CMD_STATUS_NO_RESOURCE = 0x8, /* No resources */ + MC_CMD_STATUS_NO_MEMORY = 0x9, /* No memory available */ + MC_CMD_STATUS_BUSY = 0xA, /* Device is busy */ + MC_CMD_STATUS_UNSUPPORTED_OP = 0xB, /* Unsupported operation */ + MC_CMD_STATUS_INVALID_STATE = 0xC /* Invalid state */ +}; + +/* + * MC command flags + */ + +/* High priority flag */ +#define MC_CMD_FLAG_PRI 0x80 +/* Command completion flag */ +#define MC_CMD_FLAG_INTR_DIS 0x01 + +#define MC_CMD_HDR_FLAGS_MASK 0xFF00FF00 + +int mc_send_command(struct fsl_mc_io *mc_io, struct mc_command *cmd); + +static inline uint64_t mc_encode_cmd_header(uint16_t cmd_id, + uint32_t cmd_flags, + uint16_t token) +{ + uint64_t header = 0; + struct mc_cmd_header *hdr = (struct mc_cmd_header *)&header; + + hdr->cmd_id = cpu_to_le16(cmd_id); + hdr->token = cpu_to_le16(token); + hdr->status = MC_CMD_STATUS_READY; + hdr->word[0] |= cpu_to_le32(cmd_flags & MC_CMD_HDR_FLAGS_MASK); + + return header; +} + +static inline uint16_t mc_cmd_hdr_read_token(struct mc_command *cmd) +{ + struct mc_cmd_header *hdr = (struct mc_cmd_header *)&cmd->header; + uint16_t token = le16_to_cpu(hdr->token); + + return token; +} + +static inline uint32_t mc_cmd_read_object_id(struct mc_command *cmd) +{ + struct mc_rsp_create *rsp_params; + + rsp_params = (struct mc_rsp_create *)cmd->params; + return le32_to_cpu(rsp_params->object_id); +} + +static inline enum mc_cmd_status mc_cmd_read_status(struct mc_command *cmd) +{ + struct mc_cmd_header *hdr = (struct mc_cmd_header *)&cmd->header; + uint8_t status = hdr->status; + + return (enum mc_cmd_status)status; +} + +/** + * mc_write_command - writes a command to a Management Complex (MC) portal + * + * @portal: pointer to an MC portal + * @cmd: pointer to a filled command + */ +static inline void mc_write_command(struct mc_command __iomem *portal, + struct mc_command *cmd) +{ + struct mc_cmd_header *cmd_header = (struct mc_cmd_header *)&cmd->header; + char *header = (char *)&portal->header; + int i; + + /* copy command parameters into the portal */ + for (i = 0; i < MC_CMD_NUM_OF_PARAMS; i++) + iowrite64(cmd->params[i], &portal->params[i]); + + /* submit the command by writing the header */ + iowrite32(le32_to_cpu(cmd_header->word[1]), (((uint32_t *)header) + 1)); + iowrite32(le32_to_cpu(cmd_header->word[0]), (uint32_t *)header); +} + +/** + * mc_read_response - reads the response for the last MC command from a + * Management Complex (MC) portal + * + * @portal: pointer to an MC portal + * @resp: pointer to command response buffer + * + * Returns MC_CMD_STATUS_OK on Success; Error code otherwise. + */ +static inline enum mc_cmd_status mc_read_response( + struct mc_command __iomem *portal, + struct mc_command *resp) +{ + int i; + enum mc_cmd_status status; + + /* Copy command response header from MC portal: */ + resp->header = ioread64(&portal->header); + status = mc_cmd_read_status(resp); + if (status != MC_CMD_STATUS_OK) + return status; + + /* Copy command response data from MC portal: */ + for (i = 0; i < MC_CMD_NUM_OF_PARAMS; i++) + resp->params[i] = ioread64(&portal->params[i]); + + return status; +} + +#endif /* __FSL_MC_CMD_H */ diff --git a/dpdk/drivers/bus/fslmc/mc/fsl_mc_sys.h b/dpdk/drivers/bus/fslmc/mc/fsl_mc_sys.h new file mode 100644 index 00000000..e4b2404e --- /dev/null +++ b/dpdk/drivers/bus/fslmc/mc/fsl_mc_sys.h @@ -0,0 +1,96 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2013-2015 Freescale Semiconductor Inc. + * Copyright 2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef _FSL_MC_SYS_H +#define _FSL_MC_SYS_H + +#ifdef __linux_driver__ + +#include +#include +#include + +struct fsl_mc_io { + void *regs; +}; + +#ifndef ENOTSUP +#define ENOTSUP 95 +#endif + +#define ioread64(_p) readq(_p) +#define iowrite64(_v, _p) writeq(_v, _p) + +#else /* __linux_driver__ */ + +#include +#include +#include +#include +#include + +#ifndef dmb +#define dmb() {__asm__ __volatile__("" : : : "memory"); } +#endif +#define __iormb() dmb() +#define __iowmb() dmb() +#define __arch_getq(a) (*(volatile uint64_t *)(a)) +#define __arch_putq(v, a) (*(volatile uint64_t *)(a) = (v)) +#define __arch_putq32(v, a) (*(volatile uint32_t *)(a) = (v)) +#define readq(c) \ + ({ uint64_t __v = __arch_getq(c); __iormb(); __v; }) +#define writeq(v, c) \ + ({ uint64_t __v = v; __iowmb(); __arch_putq(__v, c); __v; }) +#define writeq32(v, c) \ + ({ uint32_t __v = v; __iowmb(); __arch_putq32(__v, c); __v; }) +#define ioread64(_p) readq(_p) +#define iowrite64(_v, _p) writeq(_v, _p) +#define iowrite32(_v, _p) writeq32(_v, _p) +#define __iomem + +/*GPP is supposed to use MC commands with low priority*/ +#define CMD_PRI_LOW 0 /*!< Low Priority command indication */ + +struct fsl_mc_io { + void *regs; +}; + +#endif /* __linux_driver__ */ + +#endif /* _FSL_MC_SYS_H */ diff --git a/dpdk/drivers/bus/fslmc/mc/mc_sys.c b/dpdk/drivers/bus/fslmc/mc/mc_sys.c new file mode 100644 index 00000000..f0f9a268 --- /dev/null +++ b/dpdk/drivers/bus/fslmc/mc/mc_sys.c @@ -0,0 +1,117 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2013-2015 Freescale Semiconductor Inc. + * Copyright 2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#include +#include + +#include + +/** User space framework uses MC Portal in shared mode. Following change + * introduces lock in MC FLIB + */ + +/** + * A static spinlock initializer. + */ +static rte_spinlock_t mc_portal_lock = RTE_SPINLOCK_INITIALIZER; + +static int mc_status_to_error(enum mc_cmd_status status) +{ + switch (status) { + case MC_CMD_STATUS_OK: + return 0; + case MC_CMD_STATUS_AUTH_ERR: + return -EACCES; /* Token error */ + case MC_CMD_STATUS_NO_PRIVILEGE: + return -EPERM; /* Permission denied */ + case MC_CMD_STATUS_DMA_ERR: + return -EIO; /* Input/Output error */ + case MC_CMD_STATUS_CONFIG_ERR: + return -EINVAL; /* Device not configured */ + case MC_CMD_STATUS_TIMEOUT: + return -ETIMEDOUT; /* Operation timed out */ + case MC_CMD_STATUS_NO_RESOURCE: + return -ENAVAIL; /* Resource temporarily unavailable */ + case MC_CMD_STATUS_NO_MEMORY: + return -ENOMEM; /* Cannot allocate memory */ + case MC_CMD_STATUS_BUSY: + return -EBUSY; /* Device busy */ + case MC_CMD_STATUS_UNSUPPORTED_OP: + return -ENOTSUP; /* Operation not supported by device */ + case MC_CMD_STATUS_INVALID_STATE: + return -ENODEV; /* Invalid device state */ + default: + break; + } + + /* Not expected to reach here */ + return -EINVAL; +} + +int mc_send_command(struct fsl_mc_io *mc_io, struct mc_command *cmd) +{ + enum mc_cmd_status status; + uint64_t response; + + if (!mc_io || !mc_io->regs) + return -EACCES; + + /* --- Call lock function here in case portal is shared --- */ + rte_spinlock_lock(&mc_portal_lock); + + mc_write_command(mc_io->regs, cmd); + + /* Spin until status changes */ + do { + response = ioread64(mc_io->regs); + status = mc_cmd_read_status((struct mc_command *)&response); + + /* --- Call wait function here to prevent blocking --- + * Change the loop condition accordingly to exit on timeout. + */ + } while (status == MC_CMD_STATUS_READY); + + /* Read the response back into the command buffer */ + mc_read_response(mc_io->regs, cmd); + + /* --- Call unlock function here in case portal is shared --- */ + rte_spinlock_unlock(&mc_portal_lock); + + return mc_status_to_error(status); +} diff --git a/dpdk/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c b/dpdk/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c new file mode 100644 index 00000000..334e1f5a --- /dev/null +++ b/dpdk/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c @@ -0,0 +1,144 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. + * Copyright 2016 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Freescale Semiconductor, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include "portal/dpaa2_hw_pvt.h" +#include "portal/dpaa2_hw_dpio.h" + +TAILQ_HEAD(dpbp_dev_list, dpaa2_dpbp_dev); +static struct dpbp_dev_list dpbp_dev_list + = TAILQ_HEAD_INITIALIZER(dpbp_dev_list); /*!< DPBP device list */ + +static int +dpaa2_create_dpbp_device(int vdev_fd __rte_unused, + struct vfio_device_info *obj_info __rte_unused, + int dpbp_id) +{ + struct dpaa2_dpbp_dev *dpbp_node; + int ret; + + /* Allocate DPAA2 dpbp handle */ + dpbp_node = rte_malloc(NULL, sizeof(struct dpaa2_dpbp_dev), 0); + if (!dpbp_node) { + PMD_INIT_LOG(ERR, "Memory allocation failed for DPBP Device"); + return -1; + } + + /* Open the dpbp object */ + dpbp_node->dpbp.regs = rte_mcp_ptr_list[MC_PORTAL_INDEX]; + ret = dpbp_open(&dpbp_node->dpbp, + CMD_PRI_LOW, dpbp_id, &dpbp_node->token); + if (ret) { + PMD_INIT_LOG(ERR, "Resource alloc failure with err code: %d", + ret); + rte_free(dpbp_node); + return -1; + } + + /* Clean the device first */ + ret = dpbp_reset(&dpbp_node->dpbp, CMD_PRI_LOW, dpbp_node->token); + if (ret) { + PMD_INIT_LOG(ERR, "Failure cleaning dpbp device with" + " error code %d\n", ret); + dpbp_close(&dpbp_node->dpbp, CMD_PRI_LOW, dpbp_node->token); + rte_free(dpbp_node); + return -1; + } + + dpbp_node->dpbp_id = dpbp_id; + rte_atomic16_init(&dpbp_node->in_use); + + TAILQ_INSERT_TAIL(&dpbp_dev_list, dpbp_node, next); + + RTE_LOG(DEBUG, PMD, "DPAA2: Added [dpbp.%d]\n", dpbp_id); + + return 0; +} + +struct dpaa2_dpbp_dev *dpaa2_alloc_dpbp_dev(void) +{ + struct dpaa2_dpbp_dev *dpbp_dev = NULL; + + /* Get DPBP dev handle from list using index */ + TAILQ_FOREACH(dpbp_dev, &dpbp_dev_list, next) { + if (dpbp_dev && rte_atomic16_test_and_set(&dpbp_dev->in_use)) + break; + } + + return dpbp_dev; +} + +void dpaa2_free_dpbp_dev(struct dpaa2_dpbp_dev *dpbp) +{ + struct dpaa2_dpbp_dev *dpbp_dev = NULL; + + /* Match DPBP handle and mark it free */ + TAILQ_FOREACH(dpbp_dev, &dpbp_dev_list, next) { + if (dpbp_dev == dpbp) { + rte_atomic16_dec(&dpbp_dev->in_use); + return; + } + } +} + +int dpaa2_dpbp_supported(void) +{ + if (TAILQ_EMPTY(&dpbp_dev_list)) + return -1; + return 0; +} + +static struct rte_dpaa2_object rte_dpaa2_dpbp_obj = { + .dev_type = DPAA2_BPOOL, + .create = dpaa2_create_dpbp_device, +}; + +RTE_PMD_REGISTER_DPAA2_OBJECT(dpbp, rte_dpaa2_dpbp_obj); diff --git a/dpdk/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c b/dpdk/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c new file mode 100644 index 00000000..ae189c72 --- /dev/null +++ b/dpdk/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c @@ -0,0 +1,179 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Freescale Semiconductor, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include "portal/dpaa2_hw_pvt.h" +#include "portal/dpaa2_hw_dpio.h" + +TAILQ_HEAD(dpci_dev_list, dpaa2_dpci_dev); +static struct dpci_dev_list dpci_dev_list + = TAILQ_HEAD_INITIALIZER(dpci_dev_list); /*!< DPCI device list */ + +static int +rte_dpaa2_create_dpci_device(int vdev_fd __rte_unused, + struct vfio_device_info *obj_info __rte_unused, + int dpci_id) +{ + struct dpaa2_dpci_dev *dpci_node; + struct dpci_attr attr; + struct dpci_rx_queue_cfg rx_queue_cfg; + struct dpci_rx_queue_attr rx_attr; + int ret, i; + + /* Allocate DPAA2 dpci handle */ + dpci_node = rte_malloc(NULL, sizeof(struct dpaa2_dpci_dev), 0); + if (!dpci_node) { + PMD_INIT_LOG(ERR, "Memory allocation failed for DPCI Device"); + return -1; + } + + /* Open the dpci object */ + dpci_node->dpci.regs = rte_mcp_ptr_list[MC_PORTAL_INDEX]; + ret = dpci_open(&dpci_node->dpci, + CMD_PRI_LOW, dpci_id, &dpci_node->token); + if (ret) { + PMD_INIT_LOG(ERR, "Resource alloc failure with err code: %d", + ret); + rte_free(dpci_node); + return -1; + } + + /* Get the device attributes */ + ret = dpci_get_attributes(&dpci_node->dpci, + CMD_PRI_LOW, dpci_node->token, &attr); + if (ret != 0) { + PMD_INIT_LOG(ERR, "Reading device failed with err code: %d", + ret); + rte_free(dpci_node); + return -1; + } + + /* Set up the Rx Queue */ + memset(&rx_queue_cfg, 0, sizeof(struct dpci_rx_queue_cfg)); + ret = dpci_set_rx_queue(&dpci_node->dpci, + CMD_PRI_LOW, + dpci_node->token, + 0, &rx_queue_cfg); + if (ret) { + PMD_INIT_LOG(ERR, "Setting Rx queue failed with err code: %d", + ret); + rte_free(dpci_node); + return -1; + } + + /* Enable the device */ + ret = dpci_enable(&dpci_node->dpci, + CMD_PRI_LOW, dpci_node->token); + if (ret != 0) { + PMD_INIT_LOG(ERR, "Enabling device failed with err code: %d", + ret); + rte_free(dpci_node); + return -1; + } + + for (i = 0; i < DPAA2_DPCI_MAX_QUEUES; i++) { + /* Get the Rx FQID's */ + ret = dpci_get_rx_queue(&dpci_node->dpci, + CMD_PRI_LOW, + dpci_node->token, i, + &rx_attr); + if (ret != 0) { + PMD_INIT_LOG(ERR, + "Reading device failed with err code: %d", + ret); + rte_free(dpci_node); + return -1; + } + + dpci_node->queue[i].fqid = rx_attr.fqid; + } + + dpci_node->dpci_id = dpci_id; + rte_atomic16_init(&dpci_node->in_use); + + TAILQ_INSERT_TAIL(&dpci_dev_list, dpci_node, next); + + RTE_LOG(DEBUG, PMD, "DPAA2: Added [dpci.%d]\n", dpci_id); + + return 0; +} + +struct dpaa2_dpci_dev *rte_dpaa2_alloc_dpci_dev(void) +{ + struct dpaa2_dpci_dev *dpci_dev = NULL; + + /* Get DPCI dev handle from list using index */ + TAILQ_FOREACH(dpci_dev, &dpci_dev_list, next) { + if (dpci_dev && rte_atomic16_test_and_set(&dpci_dev->in_use)) + break; + } + + return dpci_dev; +} + +void rte_dpaa2_free_dpci_dev(struct dpaa2_dpci_dev *dpci) +{ + struct dpaa2_dpci_dev *dpci_dev = NULL; + + /* Match DPCI handle and mark it free */ + TAILQ_FOREACH(dpci_dev, &dpci_dev_list, next) { + if (dpci_dev == dpci) { + rte_atomic16_dec(&dpci_dev->in_use); + return; + } + } +} + +static struct rte_dpaa2_object rte_dpaa2_dpci_obj = { + .dev_type = DPAA2_CI, + .create = rte_dpaa2_create_dpci_device, +}; + +RTE_PMD_REGISTER_DPAA2_OBJECT(dpci, rte_dpaa2_dpci_obj); diff --git a/dpdk/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c b/dpdk/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c new file mode 100644 index 00000000..f00070f3 --- /dev/null +++ b/dpdk/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c @@ -0,0 +1,544 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. + * Copyright 2016 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Freescale Semiconductor, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include "dpaa2_hw_pvt.h" +#include "dpaa2_hw_dpio.h" +#include + +#define NUM_HOST_CPUS RTE_MAX_LCORE + +struct dpaa2_io_portal_t dpaa2_io_portal[RTE_MAX_LCORE]; +RTE_DEFINE_PER_LCORE(struct dpaa2_io_portal_t, _dpaa2_io); + +struct swp_active_dqs rte_global_active_dqs_list[NUM_MAX_SWP]; + +TAILQ_HEAD(dpio_dev_list, dpaa2_dpio_dev); +static struct dpio_dev_list dpio_dev_list + = TAILQ_HEAD_INITIALIZER(dpio_dev_list); /*!< DPIO device list */ +static uint32_t io_space_count; + +/*Stashing Macros default for LS208x*/ +static int dpaa2_core_cluster_base = 0x04; +static int dpaa2_cluster_sz = 2; + +/* For LS208X platform There are four clusters with following mapping: + * Cluster 1 (ID = x04) : CPU0, CPU1; + * Cluster 2 (ID = x05) : CPU2, CPU3; + * Cluster 3 (ID = x06) : CPU4, CPU5; + * Cluster 4 (ID = x07) : CPU6, CPU7; + */ +/* For LS108X platform There are two clusters with following mapping: + * Cluster 1 (ID = x02) : CPU0, CPU1, CPU2, CPU3; + * Cluster 2 (ID = x03) : CPU4, CPU5, CPU6, CPU7; + */ +/* For LX2160 platform There are four clusters with following mapping: + * Cluster 1 (ID = x00) : CPU0, CPU1; + * Cluster 2 (ID = x01) : CPU2, CPU3; + * Cluster 3 (ID = x02) : CPU4, CPU5; + * Cluster 4 (ID = x03) : CPU6, CPU7; + * Cluster 1 (ID = x04) : CPU8, CPU9; + * Cluster 2 (ID = x05) : CPU10, CP11; + * Cluster 3 (ID = x06) : CPU12, CPU13; + * Cluster 4 (ID = x07) : CPU14, CPU15; + */ + +static int +dpaa2_core_cluster_sdest(int cpu_id) +{ + int x = cpu_id / dpaa2_cluster_sz; + + return dpaa2_core_cluster_base + x; +} + +static void dpaa2_affine_dpio_intr_to_respective_core(int32_t dpio_id) +{ +#define STRING_LEN 28 +#define COMMAND_LEN 50 + uint32_t cpu_mask = 1; + int ret; + size_t len = 0; + char *temp = NULL, *token = NULL; + char string[STRING_LEN], command[COMMAND_LEN]; + FILE *file; + + snprintf(string, STRING_LEN, "dpio.%d", dpio_id); + file = fopen("/proc/interrupts", "r"); + if (!file) { + PMD_DRV_LOG(WARNING, "Failed to open /proc/interrupts file\n"); + return; + } + while (getline(&temp, &len, file) != -1) { + if ((strstr(temp, string)) != NULL) { + token = strtok(temp, ":"); + break; + } + } + + if (!token) { + PMD_DRV_LOG(WARNING, "Failed to get interrupt id for dpio.%d\n", + dpio_id); + if (temp) + free(temp); + fclose(file); + return; + } + + cpu_mask = cpu_mask << rte_lcore_id(); + snprintf(command, COMMAND_LEN, "echo %X > /proc/irq/%s/smp_affinity", + cpu_mask, token); + ret = system(command); + if (ret < 0) + PMD_DRV_LOG(WARNING, + "Failed to affine interrupts on respective core\n"); + else + PMD_DRV_LOG(WARNING, " %s command is executed\n", command); + + free(temp); + fclose(file); +} + +static int dpaa2_dpio_intr_init(struct dpaa2_dpio_dev *dpio_dev) +{ + struct epoll_event epoll_ev; + int eventfd, dpio_epoll_fd, ret; + int threshold = 0x3, timeout = 0xFF; + + dpio_epoll_fd = epoll_create(1); + ret = rte_dpaa2_intr_enable(&dpio_dev->intr_handle, 0); + if (ret) { + PMD_DRV_LOG(ERR, "Interrupt registeration failed\n"); + return -1; + } + + if (getenv("DPAA2_PORTAL_INTR_THRESHOLD")) + threshold = atoi(getenv("DPAA2_PORTAL_INTR_THRESHOLD")); + + if (getenv("DPAA2_PORTAL_INTR_TIMEOUT")) + sscanf(getenv("DPAA2_PORTAL_INTR_TIMEOUT"), "%x", &timeout); + + qbman_swp_interrupt_set_trigger(dpio_dev->sw_portal, + QBMAN_SWP_INTERRUPT_DQRI); + qbman_swp_interrupt_clear_status(dpio_dev->sw_portal, 0xffffffff); + qbman_swp_interrupt_set_inhibit(dpio_dev->sw_portal, 0); + qbman_swp_dqrr_thrshld_write(dpio_dev->sw_portal, threshold); + qbman_swp_intr_timeout_write(dpio_dev->sw_portal, timeout); + + eventfd = dpio_dev->intr_handle.fd; + epoll_ev.events = EPOLLIN | EPOLLPRI | EPOLLET; + epoll_ev.data.fd = eventfd; + + ret = epoll_ctl(dpio_epoll_fd, EPOLL_CTL_ADD, eventfd, &epoll_ev); + if (ret < 0) { + PMD_DRV_LOG(ERR, "epoll_ctl failed\n"); + return -1; + } + dpio_dev->epoll_fd = dpio_epoll_fd; + + dpaa2_affine_dpio_intr_to_respective_core(dpio_dev->hw_id); + + return 0; +} + +static int +configure_dpio_qbman_swp(struct dpaa2_dpio_dev *dpio_dev) +{ + struct qbman_swp_desc p_des; + struct dpio_attr attr; + + dpio_dev->dpio = malloc(sizeof(struct fsl_mc_io)); + if (!dpio_dev->dpio) { + PMD_INIT_LOG(ERR, "Memory allocation failure\n"); + return -1; + } + + PMD_DRV_LOG(DEBUG, "Allocated DPIO Portal[%p]", dpio_dev->dpio); + dpio_dev->dpio->regs = dpio_dev->mc_portal; + if (dpio_open(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->hw_id, + &dpio_dev->token)) { + PMD_INIT_LOG(ERR, "Failed to allocate IO space\n"); + free(dpio_dev->dpio); + return -1; + } + + if (dpio_reset(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->token)) { + PMD_INIT_LOG(ERR, "Failed to reset dpio\n"); + dpio_close(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->token); + free(dpio_dev->dpio); + return -1; + } + + if (dpio_enable(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->token)) { + PMD_INIT_LOG(ERR, "Failed to Enable dpio\n"); + dpio_close(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->token); + free(dpio_dev->dpio); + return -1; + } + + if (dpio_get_attributes(dpio_dev->dpio, CMD_PRI_LOW, + dpio_dev->token, &attr)) { + PMD_INIT_LOG(ERR, "DPIO Get attribute failed\n"); + dpio_disable(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->token); + dpio_close(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->token); + free(dpio_dev->dpio); + return -1; + } + + /* Configure & setup SW portal */ + p_des.block = NULL; + p_des.idx = attr.qbman_portal_id; + p_des.cena_bar = (void *)(dpio_dev->qbman_portal_ce_paddr); + p_des.cinh_bar = (void *)(dpio_dev->qbman_portal_ci_paddr); + p_des.irq = -1; + p_des.qman_version = attr.qbman_version; + + dpio_dev->sw_portal = qbman_swp_init(&p_des); + if (dpio_dev->sw_portal == NULL) { + PMD_DRV_LOG(ERR, " QBMan SW Portal Init failed\n"); + dpio_close(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->token); + free(dpio_dev->dpio); + return -1; + } + + return 0; +} + +static int +dpaa2_configure_stashing(struct dpaa2_dpio_dev *dpio_dev, int cpu_id) +{ + int sdest, ret; + static int first_time; + + /* find the SoC type for the first time */ + if (!first_time) { + struct mc_soc_version mc_plat_info = {0}; + + if (mc_get_soc_version(dpio_dev->dpio, + CMD_PRI_LOW, &mc_plat_info)) { + PMD_INIT_LOG(ERR, "\tmc_get_soc_version failed\n"); + } else if ((mc_plat_info.svr & 0xffff0000) == SVR_LS1080A) { + dpaa2_core_cluster_base = 0x02; + dpaa2_cluster_sz = 4; + PMD_INIT_LOG(DEBUG, "\tLS108x (A53) Platform Detected"); + } else if ((mc_plat_info.svr & 0xffff0000) == SVR_LX2160A) { + dpaa2_core_cluster_base = 0x00; + dpaa2_cluster_sz = 2; + PMD_INIT_LOG(DEBUG, "\tLX2160 Platform Detected"); + } + first_time = 1; + } + + /* Set the Stashing Destination */ + if (cpu_id < 0) { + cpu_id = rte_get_master_lcore(); + if (cpu_id < 0) { + RTE_LOG(ERR, PMD, "\tGetting CPU Index failed\n"); + return -1; + } + } + /* Set the STASH Destination depending on Current CPU ID. + * Valid values of SDEST are 4,5,6,7. Where, + */ + + sdest = dpaa2_core_cluster_sdest(cpu_id); + PMD_DRV_LOG(DEBUG, "Portal= %d CPU= %u SDEST= %d", + dpio_dev->index, cpu_id, sdest); + + ret = dpio_set_stashing_destination(dpio_dev->dpio, CMD_PRI_LOW, + dpio_dev->token, sdest); + if (ret) { + PMD_DRV_LOG(ERR, "%d ERROR in SDEST\n", ret); + return -1; + } + + if (dpaa2_dpio_intr_init(dpio_dev)) { + PMD_DRV_LOG(ERR, "Interrupt registration failed for dpio\n"); + return -1; + } + + return 0; +} + +struct dpaa2_dpio_dev *dpaa2_get_qbman_swp(int cpu_id) +{ + struct dpaa2_dpio_dev *dpio_dev = NULL; + int ret; + + /* Get DPIO dev handle from list using index */ + TAILQ_FOREACH(dpio_dev, &dpio_dev_list, next) { + if (dpio_dev && rte_atomic16_test_and_set(&dpio_dev->ref_count)) + break; + } + if (!dpio_dev) + return NULL; + + PMD_DRV_LOG(DEBUG, "New Portal=0x%x (%d) affined thread - %lu", + dpio_dev, dpio_dev->index, syscall(SYS_gettid)); + + ret = dpaa2_configure_stashing(dpio_dev, cpu_id); + if (ret) + PMD_DRV_LOG(ERR, "dpaa2_configure_stashing failed"); + + return dpio_dev; +} + +int +dpaa2_affine_qbman_swp(void) +{ + unsigned int lcore_id = rte_lcore_id(); + uint64_t tid = syscall(SYS_gettid); + + if (lcore_id == LCORE_ID_ANY) + lcore_id = rte_get_master_lcore(); + /* if the core id is not supported */ + else if (lcore_id >= RTE_MAX_LCORE) + return -1; + + if (dpaa2_io_portal[lcore_id].dpio_dev) { + PMD_DRV_LOG(INFO, "DPAA Portal=0x%x (%d) is being shared" + " between thread %lu and current %lu", + dpaa2_io_portal[lcore_id].dpio_dev, + dpaa2_io_portal[lcore_id].dpio_dev->index, + dpaa2_io_portal[lcore_id].net_tid, + tid); + RTE_PER_LCORE(_dpaa2_io).dpio_dev + = dpaa2_io_portal[lcore_id].dpio_dev; + rte_atomic16_inc(&dpaa2_io_portal + [lcore_id].dpio_dev->ref_count); + dpaa2_io_portal[lcore_id].net_tid = tid; + + PMD_DRV_LOG(DEBUG, "Old Portal=0x%x (%d) affined thread - %lu", + dpaa2_io_portal[lcore_id].dpio_dev, + dpaa2_io_portal[lcore_id].dpio_dev->index, + tid); + return 0; + } + + /* Populate the dpaa2_io_portal structure */ + dpaa2_io_portal[lcore_id].dpio_dev = dpaa2_get_qbman_swp(lcore_id); + + if (dpaa2_io_portal[lcore_id].dpio_dev) { + RTE_PER_LCORE(_dpaa2_io).dpio_dev + = dpaa2_io_portal[lcore_id].dpio_dev; + dpaa2_io_portal[lcore_id].net_tid = tid; + + return 0; + } else { + return -1; + } +} + +int +dpaa2_affine_qbman_swp_sec(void) +{ + unsigned int lcore_id = rte_lcore_id(); + uint64_t tid = syscall(SYS_gettid); + + if (lcore_id == LCORE_ID_ANY) + lcore_id = rte_get_master_lcore(); + /* if the core id is not supported */ + else if (lcore_id >= RTE_MAX_LCORE) + return -1; + + if (dpaa2_io_portal[lcore_id].sec_dpio_dev) { + PMD_DRV_LOG(INFO, "DPAA Portal=0x%x (%d) is being shared" + " between thread %lu and current %lu", + dpaa2_io_portal[lcore_id].sec_dpio_dev, + dpaa2_io_portal[lcore_id].sec_dpio_dev->index, + dpaa2_io_portal[lcore_id].sec_tid, + tid); + RTE_PER_LCORE(_dpaa2_io).sec_dpio_dev + = dpaa2_io_portal[lcore_id].sec_dpio_dev; + rte_atomic16_inc(&dpaa2_io_portal + [lcore_id].sec_dpio_dev->ref_count); + dpaa2_io_portal[lcore_id].sec_tid = tid; + + PMD_DRV_LOG(DEBUG, "Old Portal=0x%x (%d) affined thread - %lu", + dpaa2_io_portal[lcore_id].sec_dpio_dev, + dpaa2_io_portal[lcore_id].sec_dpio_dev->index, + tid); + return 0; + } + + /* Populate the dpaa2_io_portal structure */ + dpaa2_io_portal[lcore_id].sec_dpio_dev = dpaa2_get_qbman_swp(lcore_id); + + if (dpaa2_io_portal[lcore_id].sec_dpio_dev) { + RTE_PER_LCORE(_dpaa2_io).sec_dpio_dev + = dpaa2_io_portal[lcore_id].sec_dpio_dev; + dpaa2_io_portal[lcore_id].sec_tid = tid; + return 0; + } else { + return -1; + } +} + +static int +dpaa2_create_dpio_device(int vdev_fd, + struct vfio_device_info *obj_info, + int object_id) +{ + struct dpaa2_dpio_dev *dpio_dev; + struct vfio_region_info reg_info = { .argsz = sizeof(reg_info)}; + + if (obj_info->num_regions < NUM_DPIO_REGIONS) { + PMD_INIT_LOG(ERR, "ERROR, Not sufficient number " + "of DPIO regions.\n"); + return -1; + } + + dpio_dev = rte_malloc(NULL, sizeof(struct dpaa2_dpio_dev), + RTE_CACHE_LINE_SIZE); + if (!dpio_dev) { + PMD_INIT_LOG(ERR, "Memory allocation failed for DPIO Device\n"); + return -1; + } + + dpio_dev->dpio = NULL; + dpio_dev->hw_id = object_id; + rte_atomic16_init(&dpio_dev->ref_count); + /* Using single portal for all devices */ + dpio_dev->mc_portal = rte_mcp_ptr_list[MC_PORTAL_INDEX]; + + reg_info.index = 0; + if (ioctl(vdev_fd, VFIO_DEVICE_GET_REGION_INFO, ®_info)) { + PMD_INIT_LOG(ERR, "vfio: error getting region info\n"); + rte_free(dpio_dev); + return -1; + } + + dpio_dev->ce_size = reg_info.size; + dpio_dev->qbman_portal_ce_paddr = (uint64_t)mmap(NULL, reg_info.size, + PROT_WRITE | PROT_READ, MAP_SHARED, + vdev_fd, reg_info.offset); + + reg_info.index = 1; + if (ioctl(vdev_fd, VFIO_DEVICE_GET_REGION_INFO, ®_info)) { + PMD_INIT_LOG(ERR, "vfio: error getting region info\n"); + rte_free(dpio_dev); + return -1; + } + + dpio_dev->ci_size = reg_info.size; + dpio_dev->qbman_portal_ci_paddr = (uint64_t)mmap(NULL, reg_info.size, + PROT_WRITE | PROT_READ, MAP_SHARED, + vdev_fd, reg_info.offset); + + if (configure_dpio_qbman_swp(dpio_dev)) { + PMD_INIT_LOG(ERR, + "Fail to configure the dpio qbman portal for %d\n", + dpio_dev->hw_id); + rte_free(dpio_dev); + return -1; + } + + io_space_count++; + dpio_dev->index = io_space_count; + + if (rte_dpaa2_vfio_setup_intr(&dpio_dev->intr_handle, vdev_fd, 1)) { + PMD_INIT_LOG(ERR, "Fail to setup interrupt for %d\n", + dpio_dev->hw_id); + rte_free(dpio_dev); + } + + TAILQ_INSERT_TAIL(&dpio_dev_list, dpio_dev, next); + RTE_LOG(DEBUG, PMD, "DPAA2: Added [dpio.%d]\n", object_id); + + return 0; +} + +void +dpaa2_free_dq_storage(struct queue_storage_info_t *q_storage) +{ + int i = 0; + + for (i = 0; i < NUM_DQS_PER_QUEUE; i++) { + if (q_storage->dq_storage[i]) + rte_free(q_storage->dq_storage[i]); + } +} + +int +dpaa2_alloc_dq_storage(struct queue_storage_info_t *q_storage) +{ + int i = 0; + + for (i = 0; i < NUM_DQS_PER_QUEUE; i++) { + q_storage->dq_storage[i] = rte_malloc(NULL, + DPAA2_DQRR_RING_SIZE * sizeof(struct qbman_result), + RTE_CACHE_LINE_SIZE); + if (!q_storage->dq_storage[i]) + goto fail; + } + return 0; +fail: + while (--i >= 0) + rte_free(q_storage->dq_storage[i]); + + return -1; +} + +static struct rte_dpaa2_object rte_dpaa2_dpio_obj = { + .dev_type = DPAA2_IO, + .create = dpaa2_create_dpio_device, +}; + +RTE_PMD_REGISTER_DPAA2_OBJECT(dpio, rte_dpaa2_dpio_obj); diff --git a/dpdk/drivers/bus/fslmc/portal/dpaa2_hw_dpio.h b/dpdk/drivers/bus/fslmc/portal/dpaa2_hw_dpio.h new file mode 100644 index 00000000..e845340c --- /dev/null +++ b/dpdk/drivers/bus/fslmc/portal/dpaa2_hw_dpio.h @@ -0,0 +1,75 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. + * Copyright 2016 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Freescale Semiconductor, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _DPAA2_HW_DPIO_H_ +#define _DPAA2_HW_DPIO_H_ + +#include +#include + +struct dpaa2_io_portal_t { + struct dpaa2_dpio_dev *dpio_dev; + struct dpaa2_dpio_dev *sec_dpio_dev; + uint64_t net_tid; + uint64_t sec_tid; + void *eventdev; +}; + +/*! Global per thread DPIO portal */ +RTE_DECLARE_PER_LCORE(struct dpaa2_io_portal_t, _dpaa2_io); + +#define DPAA2_PER_LCORE_DPIO RTE_PER_LCORE(_dpaa2_io).dpio_dev +#define DPAA2_PER_LCORE_PORTAL DPAA2_PER_LCORE_DPIO->sw_portal + +#define DPAA2_PER_LCORE_SEC_DPIO RTE_PER_LCORE(_dpaa2_io).sec_dpio_dev +#define DPAA2_PER_LCORE_SEC_PORTAL DPAA2_PER_LCORE_SEC_DPIO->sw_portal + +extern struct dpaa2_io_portal_t dpaa2_io_portal[RTE_MAX_LCORE]; + +struct dpaa2_dpio_dev *dpaa2_get_qbman_swp(int cpu_id); + +/* Affine a DPIO portal to current processing thread */ +int dpaa2_affine_qbman_swp(void); + +/* Affine additional DPIO portal to current crypto processing thread */ +int dpaa2_affine_qbman_swp_sec(void); + +/* allocate memory for FQ - dq storage */ +int +dpaa2_alloc_dq_storage(struct queue_storage_info_t *q_storage); + +/* free memory for FQ- dq storage */ +void +dpaa2_free_dq_storage(struct queue_storage_info_t *q_storage); + +#endif /* _DPAA2_HW_DPIO_H_ */ diff --git a/dpdk/drivers/bus/fslmc/portal/dpaa2_hw_pvt.h b/dpdk/drivers/bus/fslmc/portal/dpaa2_hw_pvt.h new file mode 100644 index 00000000..ece1a7d4 --- /dev/null +++ b/dpdk/drivers/bus/fslmc/portal/dpaa2_hw_pvt.h @@ -0,0 +1,378 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. + * Copyright 2016 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Freescale Semiconductor, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _DPAA2_HW_PVT_H_ +#define _DPAA2_HW_PVT_H_ + +#include + +#include +#include + +#ifndef false +#define false 0 +#endif +#ifndef true +#define true 1 +#endif +#define lower_32_bits(x) ((uint32_t)(x)) +#define upper_32_bits(x) ((uint32_t)(((x) >> 16) >> 16)) + +#define SVR_LS1080A 0x87030000 +#define SVR_LS2080A 0x87010000 +#define SVR_LS2088A 0x87090000 +#define SVR_LX2160A 0x87360000 + +#ifndef ETH_VLAN_HLEN +#define ETH_VLAN_HLEN 4 /** < Vlan Header Length */ +#endif + +#define MAX_TX_RING_SLOTS 8 + /** RTE_PKTMBUF_HEADROOM +#error "Annotation requirement is more than RTE_PKTMBUF_HEADROOM" +#endif + +/* we will re-use the HEADROOM for annotation in RX */ +#define DPAA2_HW_BUF_RESERVE 0 +#define DPAA2_PACKET_LAYOUT_ALIGN 64 /*changing from 256 */ + +#define DPAA2_DPCI_MAX_QUEUES 2 + +struct dpaa2_dpio_dev { + TAILQ_ENTRY(dpaa2_dpio_dev) next; + /**< Pointer to Next device instance */ + uint16_t index; /**< Index of a instance in the list */ + rte_atomic16_t ref_count; + /**< How many thread contexts are sharing this.*/ + struct fsl_mc_io *dpio; /** handle to DPIO portal object */ + uint16_t token; + struct qbman_swp *sw_portal; /** SW portal object */ + const struct qbman_result *dqrr[4]; + /**< DQRR Entry for this SW portal */ + void *mc_portal; /**< MC Portal for configuring this device */ + uintptr_t qbman_portal_ce_paddr; + /**< Physical address of Cache Enabled Area */ + uintptr_t ce_size; /**< Size of the CE region */ + uintptr_t qbman_portal_ci_paddr; + /**< Physical address of Cache Inhibit Area */ + uintptr_t ci_size; /**< Size of the CI region */ + struct rte_intr_handle intr_handle; /* Interrupt related info */ + int32_t epoll_fd; /**< File descriptor created for interrupt polling */ + int32_t hw_id; /**< An unique ID of this DPIO device instance */ + uint64_t dqrr_held; + uint8_t dqrr_size; +}; + +struct dpaa2_dpbp_dev { + TAILQ_ENTRY(dpaa2_dpbp_dev) next; + /**< Pointer to Next device instance */ + struct fsl_mc_io dpbp; /** handle to DPBP portal object */ + uint16_t token; + rte_atomic16_t in_use; + uint32_t dpbp_id; /*HW ID for DPBP object */ +}; + +struct queue_storage_info_t { + struct qbman_result *dq_storage[NUM_DQS_PER_QUEUE]; + struct qbman_result *active_dqs; + int active_dpio_id; + int toggle; +}; + +struct dpaa2_queue; + +typedef void (dpaa2_queue_cb_dqrr_t)(struct qbman_swp *swp, + const struct qbman_fd *fd, + const struct qbman_result *dq, + struct dpaa2_queue *rxq, + struct rte_event *ev); + +struct dpaa2_queue { + struct rte_mempool *mb_pool; /**< mbuf pool to populate RX ring. */ + void *dev; + int32_t eventfd; /*!< Event Fd of this queue */ + uint32_t fqid; /*!< Unique ID of this queue */ + uint8_t tc_index; /*!< traffic class identifier */ + uint16_t flow_id; /*!< To be used by DPAA2 frmework */ + uint64_t rx_pkts; + uint64_t tx_pkts; + uint64_t err_pkts; + union { + struct queue_storage_info_t *q_storage; + struct qbman_result *cscn; + }; + struct rte_event ev; + dpaa2_queue_cb_dqrr_t *cb; +}; + +struct swp_active_dqs { + struct qbman_result *global_active_dqs; + uint64_t reserved[7]; +}; + +#define NUM_MAX_SWP 64 + +extern struct swp_active_dqs rte_global_active_dqs_list[NUM_MAX_SWP]; + +struct dpaa2_dpci_dev { + TAILQ_ENTRY(dpaa2_dpci_dev) next; + /**< Pointer to Next device instance */ + struct fsl_mc_io dpci; /** handle to DPCI portal object */ + uint16_t token; + rte_atomic16_t in_use; + uint32_t dpci_id; /*HW ID for DPCI object */ + struct dpaa2_queue queue[DPAA2_DPCI_MAX_QUEUES]; +}; + +/*! Global MCP list */ +extern void *(*rte_mcp_ptr_list); + +/* Refer to Table 7-3 in SEC BG */ +struct qbman_fle { + uint32_t addr_lo; + uint32_t addr_hi; + uint32_t length; + /* FMT must be 00, MSB is final bit */ + uint32_t fin_bpid_offset; + uint32_t frc; + uint32_t reserved[3]; /* Not used currently */ +}; + +struct qbman_sge { + uint32_t addr_lo; + uint32_t addr_hi; + uint32_t length; + uint32_t fin_bpid_offset; +}; + +/* There are three types of frames: Single, Scatter Gather and Frame Lists */ +enum qbman_fd_format { + qbman_fd_single = 0, + qbman_fd_list, + qbman_fd_sg +}; +/*Macros to define operations on FD*/ +#define DPAA2_SET_FD_ADDR(fd, addr) do { \ + fd->simple.addr_lo = lower_32_bits((uint64_t)(addr)); \ + fd->simple.addr_hi = upper_32_bits((uint64_t)(addr)); \ +} while (0) +#define DPAA2_SET_FD_LEN(fd, length) (fd)->simple.len = length +#define DPAA2_SET_FD_BPID(fd, bpid) ((fd)->simple.bpid_offset |= bpid) +#define DPAA2_SET_FD_IVP(fd) ((fd->simple.bpid_offset |= 0x00004000)) +#define DPAA2_SET_FD_OFFSET(fd, offset) \ + ((fd->simple.bpid_offset |= (uint32_t)(offset) << 16)) +#define DPAA2_SET_FD_INTERNAL_JD(fd, len) fd->simple.frc = (0x80000000 | (len)) +#define DPAA2_SET_FD_FRC(fd, frc) fd->simple.frc = frc +#define DPAA2_RESET_FD_CTRL(fd) (fd)->simple.ctrl = 0 + +#define DPAA2_SET_FD_ASAL(fd, asal) ((fd)->simple.ctrl |= (asal << 16)) +#define DPAA2_SET_FD_FLC(fd, addr) do { \ + fd->simple.flc_lo = lower_32_bits((uint64_t)(addr)); \ + fd->simple.flc_hi = upper_32_bits((uint64_t)(addr)); \ +} while (0) +#define DPAA2_SET_FLE_INTERNAL_JD(fle, len) (fle->frc = (0x80000000 | (len))) +#define DPAA2_GET_FLE_ADDR(fle) \ + (uint64_t)((((uint64_t)(fle->addr_hi)) << 32) + fle->addr_lo) +#define DPAA2_SET_FLE_ADDR(fle, addr) do { \ + fle->addr_lo = lower_32_bits((uint64_t)addr); \ + fle->addr_hi = upper_32_bits((uint64_t)addr); \ +} while (0) +#define DPAA2_GET_FLE_CTXT(fle) \ + (uint64_t)((((uint64_t)((fle)->reserved[1])) << 32) + \ + (fle)->reserved[0]) +#define DPAA2_FLE_SAVE_CTXT(fle, addr) do { \ + fle->reserved[0] = lower_32_bits((uint64_t)addr); \ + fle->reserved[1] = upper_32_bits((uint64_t)addr); \ +} while (0) +#define DPAA2_SET_FLE_OFFSET(fle, offset) \ + ((fle)->fin_bpid_offset |= (uint32_t)(offset) << 16) +#define DPAA2_SET_FLE_BPID(fle, bpid) ((fle)->fin_bpid_offset |= (uint64_t)bpid) +#define DPAA2_GET_FLE_BPID(fle) ((fle)->fin_bpid_offset & 0x000000ff) +#define DPAA2_SET_FLE_FIN(fle) (fle->fin_bpid_offset |= (uint64_t)1 << 31) +#define DPAA2_SET_FLE_IVP(fle) (((fle)->fin_bpid_offset |= 0x00004000)) +#define DPAA2_SET_FD_COMPOUND_FMT(fd) \ + (fd->simple.bpid_offset |= (uint32_t)1 << 28) +#define DPAA2_GET_FD_ADDR(fd) \ +((uint64_t)((((uint64_t)((fd)->simple.addr_hi)) << 32) + (fd)->simple.addr_lo)) + +#define DPAA2_GET_FD_LEN(fd) ((fd)->simple.len) +#define DPAA2_GET_FD_BPID(fd) (((fd)->simple.bpid_offset & 0x00003FFF)) +#define DPAA2_GET_FD_IVP(fd) ((fd->simple.bpid_offset & 0x00004000) >> 14) +#define DPAA2_GET_FD_OFFSET(fd) (((fd)->simple.bpid_offset & 0x0FFF0000) >> 16) +#define DPAA2_GET_FLE_OFFSET(fle) (((fle)->fin_bpid_offset & 0x0FFF0000) >> 16) +#define DPAA2_SET_FLE_SG_EXT(fle) (fle->fin_bpid_offset |= (uint64_t)1 << 29) +#define DPAA2_IS_SET_FLE_SG_EXT(fle) \ + ((fle->fin_bpid_offset & ((uint64_t)1 << 29)) ? 1 : 0) + +#define DPAA2_INLINE_MBUF_FROM_BUF(buf, meta_data_size) \ + ((struct rte_mbuf *)((uint64_t)(buf) - (meta_data_size))) + +#define DPAA2_ASAL_VAL (DPAA2_MBUF_HW_ANNOTATION / 64) + +#define DPAA2_FD_SET_FORMAT(fd, format) do { \ + (fd)->simple.bpid_offset &= 0xCFFFFFFF; \ + (fd)->simple.bpid_offset |= (uint32_t)format << 28; \ +} while (0) +#define DPAA2_FD_GET_FORMAT(fd) (((fd)->simple.bpid_offset >> 28) & 0x3) + +#define DPAA2_SG_SET_FINAL(sg, fin) do { \ + (sg)->fin_bpid_offset &= 0x7FFFFFFF; \ + (sg)->fin_bpid_offset |= (uint32_t)fin << 31; \ +} while (0) +#define DPAA2_SG_IS_FINAL(sg) (!!((sg)->fin_bpid_offset >> 31)) +/* Only Enqueue Error responses will be + * pushed on FQID_ERR of Enqueue FQ + */ +#define DPAA2_EQ_RESP_ERR_FQ 0 +/* All Enqueue responses will be pushed on address + * set with qbman_eq_desc_set_response + */ +#define DPAA2_EQ_RESP_ALWAYS 1 + +#ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA +static void *dpaa2_mem_ptov(phys_addr_t paddr) __attribute__((unused)); +/* todo - this is costly, need to write a fast coversion routine */ +static void *dpaa2_mem_ptov(phys_addr_t paddr) +{ + const struct rte_memseg *memseg = rte_eal_get_physmem_layout(); + int i; + + for (i = 0; i < RTE_MAX_MEMSEG && memseg[i].addr_64 != 0; i++) { + if (paddr >= memseg[i].iova && + (char *)paddr < (char *)memseg[i].iova + memseg[i].len) + return (void *)(memseg[i].addr_64 + + (paddr - memseg[i].iova)); + } + return NULL; +} + +static phys_addr_t dpaa2_mem_vtop(uint64_t vaddr) __attribute__((unused)); +static phys_addr_t dpaa2_mem_vtop(uint64_t vaddr) +{ + const struct rte_memseg *memseg = rte_eal_get_physmem_layout(); + int i; + + for (i = 0; i < RTE_MAX_MEMSEG && memseg[i].addr_64 != 0; i++) { + if (vaddr >= memseg[i].addr_64 && + vaddr < memseg[i].addr_64 + memseg[i].len) + return memseg[i].iova + + (vaddr - memseg[i].addr_64); + } + return (phys_addr_t)(NULL); +} + +/** + * When we are using Physical addresses as IO Virtual Addresses, + * Need to call conversion routines dpaa2_mem_vtop & dpaa2_mem_ptov + * wherever required. + * These routines are called with help of below MACRO's + */ + +#define DPAA2_MBUF_VADDR_TO_IOVA(mbuf) ((mbuf)->buf_iova) +#define DPAA2_OP_VADDR_TO_IOVA(op) (op->phys_addr) + +/** + * macro to convert Virtual address to IOVA + */ +#define DPAA2_VADDR_TO_IOVA(_vaddr) dpaa2_mem_vtop((uint64_t)(_vaddr)) + +/** + * macro to convert IOVA to Virtual address + */ +#define DPAA2_IOVA_TO_VADDR(_iova) dpaa2_mem_ptov((phys_addr_t)(_iova)) + +/** + * macro to convert modify the memory containing IOVA to Virtual address + */ +#define DPAA2_MODIFY_IOVA_TO_VADDR(_mem, _type) \ + {_mem = (_type)(dpaa2_mem_ptov((phys_addr_t)(_mem))); } + +#else /* RTE_LIBRTE_DPAA2_USE_PHYS_IOVA */ + +#define DPAA2_MBUF_VADDR_TO_IOVA(mbuf) ((mbuf)->buf_addr) +#define DPAA2_OP_VADDR_TO_IOVA(op) (op) +#define DPAA2_VADDR_TO_IOVA(_vaddr) (_vaddr) +#define DPAA2_IOVA_TO_VADDR(_iova) (_iova) +#define DPAA2_MODIFY_IOVA_TO_VADDR(_mem, _type) + +#endif /* RTE_LIBRTE_DPAA2_USE_PHYS_IOVA */ + +static inline +int check_swp_active_dqs(uint16_t dpio_index) +{ + if (rte_global_active_dqs_list[dpio_index].global_active_dqs != NULL) + return 1; + return 0; +} + +static inline +void clear_swp_active_dqs(uint16_t dpio_index) +{ + rte_global_active_dqs_list[dpio_index].global_active_dqs = NULL; +} + +static inline +struct qbman_result *get_swp_active_dqs(uint16_t dpio_index) +{ + return rte_global_active_dqs_list[dpio_index].global_active_dqs; +} + +static inline +void set_swp_active_dqs(uint16_t dpio_index, struct qbman_result *dqs) +{ + rte_global_active_dqs_list[dpio_index].global_active_dqs = dqs; +} +struct dpaa2_dpbp_dev *dpaa2_alloc_dpbp_dev(void); +void dpaa2_free_dpbp_dev(struct dpaa2_dpbp_dev *dpbp); +int dpaa2_dpbp_supported(void); + +struct dpaa2_dpci_dev *rte_dpaa2_alloc_dpci_dev(void); +void rte_dpaa2_free_dpci_dev(struct dpaa2_dpci_dev *dpci); + +#endif diff --git a/dpdk/drivers/bus/fslmc/qbman/include/compat.h b/dpdk/drivers/bus/fslmc/qbman/include/compat.h new file mode 100644 index 00000000..423087cb --- /dev/null +++ b/dpdk/drivers/bus/fslmc/qbman/include/compat.h @@ -0,0 +1,124 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2008-2016 Freescale Semiconductor, Inc. + * Copyright 2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Freescale Semiconductor nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef HEADER_COMPAT_H +#define HEADER_COMPAT_H + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* The following definitions are primarily to allow the single-source driver + * interfaces to be included by arbitrary program code. Ie. for interfaces that + * are also available in kernel-space, these definitions provide compatibility + * with certain attributes and types used in those interfaces. + */ + +/* Required compiler attributes */ +#define likely(x) __builtin_expect(!!(x), 1) +#define unlikely(x) __builtin_expect(!!(x), 0) + +/* Required types */ +typedef uint64_t dma_addr_t; + +/* Debugging */ +#define prflush(fmt, args...) \ + do { \ + printf(fmt, ##args); \ + fflush(stdout); \ + } while (0) +#define pr_crit(fmt, args...) prflush("CRIT:" fmt, ##args) +#define pr_err(fmt, args...) prflush("ERR:" fmt, ##args) +#define pr_warn(fmt, args...) prflush("WARN:" fmt, ##args) +#define pr_info(fmt, args...) prflush(fmt, ##args) + +#ifdef RTE_LIBRTE_DPAA2_DEBUG_BUS + +/* Trace the 3 different classes of read/write access to QBMan. #undef as + * required. + */ +#define QBMAN_CCSR_TRACE +#define QBMAN_CINH_TRACE +#define QBMAN_CENA_TRACE + +#define QBMAN_CHECKING + +#ifdef pr_debug +#undef pr_debug +#endif +#define pr_debug(fmt, args...) printf(fmt, ##args) +#define QBMAN_BUG_ON(c) \ +do { \ + static int warned_##__LINE__; \ + if ((c) && !warned_##__LINE__) { \ + pr_warn("(%s:%d)\n", __FILE__, __LINE__); \ + warned_##__LINE__ = 1; \ + } \ +} while (0) +#else +#define QBMAN_BUG_ON(c) {} +#define pr_debug(fmt, args...) {} +#endif + +/* Other miscellaneous interfaces our APIs depend on; */ + +#define lower_32_bits(x) ((uint32_t)(x)) +#define upper_32_bits(x) ((uint32_t)(((x) >> 16) >> 16)) + + +#define __iomem + +#define __raw_readb(p) (*(const volatile unsigned char *)(p)) +#define __raw_readl(p) (*(const volatile unsigned int *)(p)) +#define __raw_writel(v, p) {*(volatile unsigned int *)(p) = (v); } + +#define atomic_t rte_atomic32_t +#define atomic_read(v) rte_atomic32_read(v) +#define atomic_set(v, i) rte_atomic32_set(v, i) + +#define atomic_inc(v) rte_atomic32_add(v, 1) +#define atomic_dec(v) rte_atomic32_sub(v, 1) + +#define atomic_inc_and_test(v) rte_atomic32_inc_and_test(v) +#define atomic_dec_and_test(v) rte_atomic32_dec_and_test(v) + +#define atomic_inc_return(v) rte_atomic32_add_return(v, 1) +#define atomic_dec_return(v) rte_atomic32_sub_return(v, 1) +#define atomic_sub_and_test(i, v) (rte_atomic32_sub_return(v, i) == 0) + +#endif /* HEADER_COMPAT_H */ diff --git a/dpdk/drivers/bus/fslmc/qbman/include/fsl_qbman_base.h b/dpdk/drivers/bus/fslmc/qbman/include/fsl_qbman_base.h new file mode 100644 index 00000000..14159609 --- /dev/null +++ b/dpdk/drivers/bus/fslmc/qbman/include/fsl_qbman_base.h @@ -0,0 +1,156 @@ +/*- + * BSD LICENSE + * + * Copyright (C) 2014 Freescale Semiconductor, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Freescale Semiconductor nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef _FSL_QBMAN_BASE_H +#define _FSL_QBMAN_BASE_H + +typedef uint64_t dma_addr_t; + +/** + * DOC: QBMan basic structures + * + * The QBMan block descriptor, software portal descriptor and Frame descriptor + * are defined here. + * + */ + +/** + * struct qbman_block_desc - qbman block descriptor structure + * @ccsr_reg_bar: CCSR register map. + * @irq_rerr: Recoverable error interrupt line. + * @irq_nrerr: Non-recoverable error interrupt line + * + * Descriptor for a QBMan instance on the SoC. On partitions/targets that do not + * control this QBMan instance, these values may simply be place-holders. The + * idea is simply that we be able to distinguish between them, eg. so that SWP + * descriptors can identify which QBMan instance they belong to. + */ +struct qbman_block_desc { + void *ccsr_reg_bar; + int irq_rerr; + int irq_nrerr; +}; + +enum qbman_eqcr_mode { + qman_eqcr_vb_ring = 2, /* Valid bit, with eqcr in ring mode */ + qman_eqcr_vb_array, /* Valid bit, with eqcr in array mode */ +}; + +/** + * struct qbman_swp_desc - qbman software portal descriptor structure + * @block: The QBMan instance. + * @cena_bar: Cache-enabled portal register map. + * @cinh_bar: Cache-inhibited portal register map. + * @irq: -1 if unused (or unassigned) + * @idx: SWPs within a QBMan are indexed. -1 if opaque to the user. + * @qman_version: the qman version. + * @eqcr_mode: Select the eqcr mode, currently only valid bit ring mode and + * valid bit array mode are supported. + * + * Descriptor for a QBMan software portal, expressed in terms that make sense to + * the user context. Ie. on MC, this information is likely to be true-physical, + * and instantiated statically at compile-time. On GPP, this information is + * likely to be obtained via "discovery" over a partition's "MC bus" + * (ie. in response to a MC portal command), and would take into account any + * virtualisation of the GPP user's address space and/or interrupt numbering. + */ +struct qbman_swp_desc { + const struct qbman_block_desc *block; + uint8_t *cena_bar; + uint8_t *cinh_bar; + int irq; + int idx; + uint32_t qman_version; + enum qbman_eqcr_mode eqcr_mode; +}; + +/* Driver object for managing a QBMan portal */ +struct qbman_swp; + +/** + * struct qbman_fd - basci structure for qbman frame descriptor + * @words: for easier/faster copying the whole FD structure. + * @addr_lo: the lower 32 bits of the address in FD. + * @addr_hi: the upper 32 bits of the address in FD. + * @len: the length field in FD. + * @bpid_offset: represent the bpid and offset fields in FD. offset in + * the MS 16 bits, BPID in the LS 16 bits. + * @frc: frame context + * @ctrl: the 32bit control bits including dd, sc,... va, err. + * @flc_lo: the lower 32bit of flow context. + * @flc_hi: the upper 32bits of flow context. + * + * Place-holder for FDs, we represent it via the simplest form that we need for + * now. Different overlays may be needed to support different options, etc. (It + * is impractical to define One True Struct, because the resulting encoding + * routines (lots of read-modify-writes) would be worst-case performance whether + * or not circumstances required them.) + * + * Note, as with all data-structures exchanged between software and hardware (be + * they located in the portal register map or DMA'd to and from main-memory), + * the driver ensures that the caller of the driver API sees the data-structures + * in host-endianness. "struct qbman_fd" is no exception. The 32-bit words + * contained within this structure are represented in host-endianness, even if + * hardware always treats them as little-endian. As such, if any of these fields + * are interpreted in a binary (rather than numerical) fashion by hardware + * blocks (eg. accelerators), then the user should be careful. We illustrate + * with an example; + * + * Suppose the desired behaviour of an accelerator is controlled by the "frc" + * field of the FDs that are sent to it. Suppose also that the behaviour desired + * by the user corresponds to an "frc" value which is expressed as the literal + * sequence of bytes 0xfe, 0xed, 0xab, and 0xba. So "frc" should be the 32-bit + * value in which 0xfe is the first byte and 0xba is the last byte, and as + * hardware is little-endian, this amounts to a 32-bit "value" of 0xbaabedfe. If + * the software is little-endian also, this can simply be achieved by setting + * frc=0xbaabedfe. On the other hand, if software is big-endian, it should set + * frc=0xfeedabba! The best away of avoiding trouble with this sort of thing is + * to treat the 32-bit words as numerical values, in which the offset of a field + * from the beginning of the first byte (as required or generated by hardware) + * is numerically encoded by a left-shift (ie. by raising the field to a + * corresponding power of 2). Ie. in the current example, software could set + * "frc" in the following way, and it would work correctly on both little-endian + * and big-endian operation; + * fd.frc = (0xfe << 0) | (0xed << 8) | (0xab << 16) | (0xba << 24); + */ +struct qbman_fd { + union { + uint32_t words[8]; + struct qbman_fd_simple { + uint32_t addr_lo; + uint32_t addr_hi; + uint32_t len; + uint32_t bpid_offset; + uint32_t frc; + uint32_t ctrl; + uint32_t flc_lo; + uint32_t flc_hi; + } simple; + }; +}; + +#endif /* !_FSL_QBMAN_BASE_H */ diff --git a/dpdk/drivers/bus/fslmc/qbman/include/fsl_qbman_portal.h b/dpdk/drivers/bus/fslmc/qbman/include/fsl_qbman_portal.h new file mode 100644 index 00000000..1e656602 --- /dev/null +++ b/dpdk/drivers/bus/fslmc/qbman/include/fsl_qbman_portal.h @@ -0,0 +1,1190 @@ +/*- + * BSD LICENSE + * + * Copyright (C) 2014 Freescale Semiconductor, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Freescale Semiconductor nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef _FSL_QBMAN_PORTAL_H +#define _FSL_QBMAN_PORTAL_H + +#include + +/** + * DOC - QBMan portal APIs to implement the following functions: + * - Initialize and destroy Software portal object. + * - Read and write Software portal interrupt registers. + * - Enqueue, including setting the enqueue descriptor, and issuing enqueue + * command etc. + * - Dequeue, including setting the dequeue descriptor, issuing dequeue command, + * parsing the dequeue response in DQRR and memeory, parsing the state change + * notifications etc. + * - Release, including setting the release descriptor, and issuing the buffer + * release command. + * - Acquire, acquire the buffer from the given buffer pool. + * - FQ management. + * - Channel management, enable/disable CDAN with or without context. + */ + +/** + * qbman_swp_init() - Create a functional object representing the given + * QBMan portal descriptor. + * @d: the given qbman swp descriptor + * + * Return qbman_swp portal object for success, NULL if the object cannot + * be created. + */ +struct qbman_swp *qbman_swp_init(const struct qbman_swp_desc *d); + +/** + * qbman_swp_finish() - Create and destroy a functional object representing + * the given QBMan portal descriptor. + * @p: the qbman_swp object to be destroyed. + * + */ +void qbman_swp_finish(struct qbman_swp *p); + +/** + * qbman_swp_get_desc() - Get the descriptor of the given portal object. + * @p: the given portal object. + * + * Return the descriptor for this portal. + */ +const struct qbman_swp_desc *qbman_swp_get_desc(struct qbman_swp *p); + + /**************/ + /* Interrupts */ + /**************/ + +/* EQCR ring interrupt */ +#define QBMAN_SWP_INTERRUPT_EQRI ((uint32_t)0x00000001) +/* Enqueue command dispatched interrupt */ +#define QBMAN_SWP_INTERRUPT_EQDI ((uint32_t)0x00000002) +/* DQRR non-empty interrupt */ +#define QBMAN_SWP_INTERRUPT_DQRI ((uint32_t)0x00000004) +/* RCR ring interrupt */ +#define QBMAN_SWP_INTERRUPT_RCRI ((uint32_t)0x00000008) +/* Release command dispatched interrupt */ +#define QBMAN_SWP_INTERRUPT_RCDI ((uint32_t)0x00000010) +/* Volatile dequeue command interrupt */ +#define QBMAN_SWP_INTERRUPT_VDCI ((uint32_t)0x00000020) + +/** + * qbman_swp_interrupt_get_vanish() - Get the data in software portal + * interrupt status disable register. + * @p: the given software portal object. + * + * Return the settings in SWP_ISDR register. + */ +uint32_t qbman_swp_interrupt_get_vanish(struct qbman_swp *p); + +/** + * qbman_swp_interrupt_set_vanish() - Set the data in software portal + * interrupt status disable register. + * @p: the given software portal object. + * @mask: The value to set in SWP_IDSR register. + */ +void qbman_swp_interrupt_set_vanish(struct qbman_swp *p, uint32_t mask); + +/** + * qbman_swp_interrupt_read_status() - Get the data in software portal + * interrupt status register. + * @p: the given software portal object. + * + * Return the settings in SWP_ISR register. + */ +uint32_t qbman_swp_interrupt_read_status(struct qbman_swp *p); + +/** + * qbman_swp_interrupt_clear_status() - Set the data in software portal + * interrupt status register. + * @p: the given software portal object. + * @mask: The value to set in SWP_ISR register. + */ +void qbman_swp_interrupt_clear_status(struct qbman_swp *p, uint32_t mask); + +/** + * qbman_swp_dqrr_thrshld_read_status() - Get the data in software portal + * DQRR interrupt threshold register. + * @p: the given software portal object. + */ +uint32_t qbman_swp_dqrr_thrshld_read_status(struct qbman_swp *p); + +/** + * qbman_swp_dqrr_thrshld_write() - Set the data in software portal + * DQRR interrupt threshold register. + * @p: the given software portal object. + * @mask: The value to set in SWP_DQRR_ITR register. + */ +void qbman_swp_dqrr_thrshld_write(struct qbman_swp *p, uint32_t mask); + +/** + * qbman_swp_intr_timeout_read_status() - Get the data in software portal + * Interrupt Time-Out period register. + * @p: the given software portal object. + */ +uint32_t qbman_swp_intr_timeout_read_status(struct qbman_swp *p); + +/** + * qbman_swp_intr_timeout_write() - Set the data in software portal + * Interrupt Time-Out period register. + * @p: the given software portal object. + * @mask: The value to set in SWP_ITPR register. + */ +void qbman_swp_intr_timeout_write(struct qbman_swp *p, uint32_t mask); + +/** + * qbman_swp_interrupt_get_trigger() - Get the data in software portal + * interrupt enable register. + * @p: the given software portal object. + * + * Return the settings in SWP_IER register. + */ +uint32_t qbman_swp_interrupt_get_trigger(struct qbman_swp *p); + +/** + * qbman_swp_interrupt_set_trigger() - Set the data in software portal + * interrupt enable register. + * @p: the given software portal object. + * @mask: The value to set in SWP_IER register. + */ +void qbman_swp_interrupt_set_trigger(struct qbman_swp *p, uint32_t mask); + +/** + * qbman_swp_interrupt_get_inhibit() - Get the data in software portal + * interrupt inhibit register. + * @p: the given software portal object. + * + * Return the settings in SWP_IIR register. + */ +int qbman_swp_interrupt_get_inhibit(struct qbman_swp *p); + +/** + * qbman_swp_interrupt_set_inhibit() - Set the data in software portal + * interrupt inhibit register. + * @p: the given software portal object. + * @mask: The value to set in SWP_IIR register. + */ +void qbman_swp_interrupt_set_inhibit(struct qbman_swp *p, int inhibit); + + /************/ + /* Dequeues */ + /************/ + +/** + * struct qbman_result - structure for qbman dequeue response and/or + * notification. + * @donot_manipulate_directly: the 16 32bit data to represent the whole + * possible qbman dequeue result. + */ +struct qbman_result { + union { + struct common { + uint8_t verb; + uint8_t reserved[63]; + } common; + struct dq { + uint8_t verb; + uint8_t stat; + __le16 seqnum; + __le16 oprid; + uint8_t reserved; + uint8_t tok; + __le32 fqid; + uint32_t reserved2; + __le32 fq_byte_cnt; + __le32 fq_frm_cnt; + __le64 fqd_ctx; + uint8_t fd[32]; + } dq; + struct scn { + uint8_t verb; + uint8_t stat; + uint8_t state; + uint8_t reserved; + __le32 rid_tok; + __le64 ctx; + } scn; + }; +}; + +/* TODO: + *A DQRI interrupt can be generated when there are dequeue results on the + * portal's DQRR (this mechanism does not deal with "pull" dequeues to + * user-supplied 'storage' addresses). There are two parameters to this + * interrupt source, one is a threshold and the other is a timeout. The + * interrupt will fire if either the fill-level of the ring exceeds 'thresh', or + * if the ring has been non-empty for been longer than 'timeout' nanoseconds. + * For timeout, an approximation to the desired nanosecond-granularity value is + * made, so there are get and set APIs to allow the user to see what actual + * timeout is set (compared to the timeout that was requested). + */ +int qbman_swp_dequeue_thresh(struct qbman_swp *s, unsigned int thresh); +int qbman_swp_dequeue_set_timeout(struct qbman_swp *s, unsigned int timeout); +int qbman_swp_dequeue_get_timeout(struct qbman_swp *s, unsigned int *timeout); + +/* ------------------- */ +/* Push-mode dequeuing */ +/* ------------------- */ + +/* The user of a portal can enable and disable push-mode dequeuing of up to 16 + * channels independently. It does not specify this toggling by channel IDs, but + * rather by specifying the index (from 0 to 15) that has been mapped to the + * desired channel. + */ + +/** + * qbman_swp_push_get() - Get the push dequeue setup. + * @s: the software portal object. + * @channel_idx: the channel index to query. + * @enabled: returned boolean to show whether the push dequeue is enabled for + * the given channel. + */ +void qbman_swp_push_get(struct qbman_swp *s, uint8_t channel_idx, int *enabled); + +/** + * qbman_swp_push_set() - Enable or disable push dequeue. + * @s: the software portal object. + * @channel_idx: the channel index.. + * @enable: enable or disable push dequeue. + * + * The user of a portal can enable and disable push-mode dequeuing of up to 16 + * channels independently. It does not specify this toggling by channel IDs, but + * rather by specifying the index (from 0 to 15) that has been mapped to the + * desired channel. + */ +void qbman_swp_push_set(struct qbman_swp *s, uint8_t channel_idx, int enable); + +/* ------------------- */ +/* Pull-mode dequeuing */ +/* ------------------- */ + +/** + * struct qbman_pull_desc - the structure for pull dequeue descriptor + */ +struct qbman_pull_desc { + union { + uint32_t donot_manipulate_directly[16]; + struct pull { + uint8_t verb; + uint8_t numf; + uint8_t tok; + uint8_t reserved; + uint32_t dq_src; + uint64_t rsp_addr; + uint64_t rsp_addr_virt; + uint8_t padding[40]; + } pull; + }; +}; + +enum qbman_pull_type_e { + /* dequeue with priority precedence, respect intra-class scheduling */ + qbman_pull_type_prio = 1, + /* dequeue with active FQ precedence, respect ICS */ + qbman_pull_type_active, + /* dequeue with active FQ precedence, no ICS */ + qbman_pull_type_active_noics +}; + +/** + * qbman_pull_desc_clear() - Clear the contents of a descriptor to + * default/starting state. + * @d: the pull dequeue descriptor to be cleared. + */ +void qbman_pull_desc_clear(struct qbman_pull_desc *d); + +/** + * qbman_pull_desc_set_storage()- Set the pull dequeue storage + * @d: the pull dequeue descriptor to be set. + * @storage: the pointer of the memory to store the dequeue result. + * @storage_phys: the physical address of the storage memory. + * @stash: to indicate whether write allocate is enabled. + * + * If not called, or if called with 'storage' as NULL, the result pull dequeues + * will produce results to DQRR. If 'storage' is non-NULL, then results are + * produced to the given memory location (using the physical/DMA address which + * the caller provides in 'storage_phys'), and 'stash' controls whether or not + * those writes to main-memory express a cache-warming attribute. + */ +void qbman_pull_desc_set_storage(struct qbman_pull_desc *d, + struct qbman_result *storage, + uint64_t storage_phys, + int stash); +/** + * qbman_pull_desc_set_numframes() - Set the number of frames to be dequeued. + * @d: the pull dequeue descriptor to be set. + * @numframes: number of frames to be set, must be between 1 and 16, inclusive. + */ +void qbman_pull_desc_set_numframes(struct qbman_pull_desc *d, + uint8_t numframes); +/** + * qbman_pull_desc_set_token() - Set dequeue token for pull command + * @d: the dequeue descriptor + * @token: the token to be set + * + * token is the value that shows up in the dequeue response that can be used to + * detect when the results have been published. The easiest technique is to zero + * result "storage" before issuing a dequeue, and use any non-zero 'token' value + */ +void qbman_pull_desc_set_token(struct qbman_pull_desc *d, uint8_t token); + +/* Exactly one of the following descriptor "actions" should be set. (Calling any + * one of these will replace the effect of any prior call to one of these.) + * - pull dequeue from the given frame queue (FQ) + * - pull dequeue from any FQ in the given work queue (WQ) + * - pull dequeue from any FQ in any WQ in the given channel + */ +/** + * qbman_pull_desc_set_fq() - Set fqid from which the dequeue command dequeues. + * @fqid: the frame queue index of the given FQ. + */ +void qbman_pull_desc_set_fq(struct qbman_pull_desc *d, uint32_t fqid); + +/** + * qbman_pull_desc_set_wq() - Set wqid from which the dequeue command dequeues. + * @wqid: composed of channel id and wqid within the channel. + * @dct: the dequeue command type. + */ +void qbman_pull_desc_set_wq(struct qbman_pull_desc *d, uint32_t wqid, + enum qbman_pull_type_e dct); + +/* qbman_pull_desc_set_channel() - Set channelid from which the dequeue command + * dequeues. + * @chid: the channel id to be dequeued. + * @dct: the dequeue command type. + */ +void qbman_pull_desc_set_channel(struct qbman_pull_desc *d, uint32_t chid, + enum qbman_pull_type_e dct); + +/** + * qbman_swp_pull() - Issue the pull dequeue command + * @s: the software portal object. + * @d: the software portal descriptor which has been configured with + * the set of qbman_pull_desc_set_*() calls. + * + * Return 0 for success, and -EBUSY if the software portal is not ready + * to do pull dequeue. + */ +int qbman_swp_pull(struct qbman_swp *s, struct qbman_pull_desc *d); + +/* -------------------------------- */ +/* Polling DQRR for dequeue results */ +/* -------------------------------- */ + +/** + * qbman_swp_dqrr_next() - Get an valid DQRR entry. + * @s: the software portal object. + * + * Return NULL if there are no unconsumed DQRR entries. Return a DQRR entry + * only once, so repeated calls can return a sequence of DQRR entries, without + * requiring they be consumed immediately or in any particular order. + */ +const struct qbman_result *qbman_swp_dqrr_next(struct qbman_swp *p); + +/** + * qbman_swp_dqrr_consume() - Consume DQRR entries previously returned from + * qbman_swp_dqrr_next(). + * @s: the software portal object. + * @dq: the DQRR entry to be consumed. + */ +void qbman_swp_dqrr_consume(struct qbman_swp *s, const struct qbman_result *dq); + +/** + * qbman_get_dqrr_idx() - Get dqrr index from the given dqrr + * @dqrr: the given dqrr object. + * + * Return dqrr index. + */ +uint8_t qbman_get_dqrr_idx(const struct qbman_result *dqrr); + +/** + * qbman_get_dqrr_from_idx() - Use index to get the dqrr entry from the + * given portal + * @s: the given portal. + * @idx: the dqrr index. + * + * Return dqrr entry object. + */ +struct qbman_result *qbman_get_dqrr_from_idx(struct qbman_swp *s, uint8_t idx); + +/* ------------------------------------------------- */ +/* Polling user-provided storage for dequeue results */ +/* ------------------------------------------------- */ + +/** + * qbman_result_has_new_result() - Check and get the dequeue response from the + * dq storage memory set in pull dequeue command + * @s: the software portal object. + * @dq: the dequeue result read from the memory. + * + * Only used for user-provided storage of dequeue results, not DQRR. For + * efficiency purposes, the driver will perform any required endianness + * conversion to ensure that the user's dequeue result storage is in host-endian + * format (whether or not that is the same as the little-endian format that + * hardware DMA'd to the user's storage). As such, once the user has called + * qbman_result_has_new_result() and been returned a valid dequeue result, + * they should not call it again on the same memory location (except of course + * if another dequeue command has been executed to produce a new result to that + * location). + * + * Return 1 for getting a valid dequeue result, or 0 for not getting a valid + * dequeue result. + */ +int qbman_result_has_new_result(struct qbman_swp *s, + struct qbman_result *dq); + +/** + * qbman_check_command_complete() - Check if the previous issued dq commnd + * is completed and results are available in memory. + * @s: the software portal object. + * @dq: the dequeue result read from the memory. + * + * Return 1 for getting a valid dequeue result, or 0 for not getting a valid + * dequeue result. + */ +int qbman_check_command_complete(struct qbman_result *dq); + +int qbman_check_new_result(struct qbman_result *dq); + +/* -------------------------------------------------------- */ +/* Parsing dequeue entries (DQRR and user-provided storage) */ +/* -------------------------------------------------------- */ + +/** + * qbman_result_is_DQ() - check the dequeue result is a dequeue response or not + * @dq: the dequeue result to be checked. + * + * DQRR entries may contain non-dequeue results, ie. notifications + */ +int qbman_result_is_DQ(const struct qbman_result *dq); + +/** + * qbman_result_is_SCN() - Check the dequeue result is notification or not + * @dq: the dequeue result to be checked. + * + * All the non-dequeue results (FQDAN/CDAN/CSCN/...) are "state change + * notifications" of one type or another. Some APIs apply to all of them, of the + * form qbman_result_SCN_***(). + */ +static inline int qbman_result_is_SCN(const struct qbman_result *dq) +{ + return !qbman_result_is_DQ(dq); +} + +/* Recognise different notification types, only required if the user allows for + * these to occur, and cares about them when they do. + */ + +/** + * qbman_result_is_FQDAN() - Check for FQ Data Availability + * @dq: the qbman_result object. + * + * Return 1 if this is FQDAN. + */ +int qbman_result_is_FQDAN(const struct qbman_result *dq); + +/** + * qbman_result_is_CDAN() - Check for Channel Data Availability + * @dq: the qbman_result object to check. + * + * Return 1 if this is CDAN. + */ +int qbman_result_is_CDAN(const struct qbman_result *dq); + +/** + * qbman_result_is_CSCN() - Check for Congestion State Change + * @dq: the qbman_result object to check. + * + * Return 1 if this is CSCN. + */ +int qbman_result_is_CSCN(const struct qbman_result *dq); + +/** + * qbman_result_is_BPSCN() - Check for Buffer Pool State Change. + * @dq: the qbman_result object to check. + * + * Return 1 if this is BPSCN. + */ +int qbman_result_is_BPSCN(const struct qbman_result *dq); + +/** + * qbman_result_is_CGCU() - Check for Congestion Group Count Update. + * @dq: the qbman_result object to check. + * + * Return 1 if this is CGCU. + */ +int qbman_result_is_CGCU(const struct qbman_result *dq); + +/* Frame queue state change notifications; (FQDAN in theory counts too as it + * leaves a FQ parked, but it is primarily a data availability notification) + */ + +/** + * qbman_result_is_FQRN() - Check for FQ Retirement Notification. + * @dq: the qbman_result object to check. + * + * Return 1 if this is FQRN. + */ +int qbman_result_is_FQRN(const struct qbman_result *dq); + +/** + * qbman_result_is_FQRNI() - Check for FQ Retirement Immediate + * @dq: the qbman_result object to check. + * + * Return 1 if this is FQRNI. + */ +int qbman_result_is_FQRNI(const struct qbman_result *dq); + +/** + * qbman_result_is_FQPN() - Check for FQ Park Notification + * @dq: the qbman_result object to check. + * + * Return 1 if this is FQPN. + */ +int qbman_result_is_FQPN(const struct qbman_result *dq); + +/* Parsing frame dequeue results (qbman_result_is_DQ() must be TRUE) + */ +/* FQ empty */ +#define QBMAN_DQ_STAT_FQEMPTY 0x80 +/* FQ held active */ +#define QBMAN_DQ_STAT_HELDACTIVE 0x40 +/* FQ force eligible */ +#define QBMAN_DQ_STAT_FORCEELIGIBLE 0x20 +/* Valid frame */ +#define QBMAN_DQ_STAT_VALIDFRAME 0x10 +/* FQ ODP enable */ +#define QBMAN_DQ_STAT_ODPVALID 0x04 +/* Volatile dequeue */ +#define QBMAN_DQ_STAT_VOLATILE 0x02 +/* volatile dequeue command is expired */ +#define QBMAN_DQ_STAT_EXPIRED 0x01 + +/** + * qbman_result_DQ_flags() - Get the STAT field of dequeue response + * @dq: the dequeue result. + * + * Return the state field. + */ +uint8_t qbman_result_DQ_flags(const struct qbman_result *dq); + +/** + * qbman_result_DQ_is_pull() - Check whether the dq response is from a pull + * command. + * @dq: the dequeue result. + * + * Return 1 for volatile(pull) dequeue, 0 for static dequeue. + */ +static inline int qbman_result_DQ_is_pull(const struct qbman_result *dq) +{ + return (int)(qbman_result_DQ_flags(dq) & QBMAN_DQ_STAT_VOLATILE); +} + +/** + * qbman_result_DQ_is_pull_complete() - Check whether the pull command is + * completed. + * @dq: the dequeue result. + * + * Return boolean. + */ +static inline int qbman_result_DQ_is_pull_complete( + const struct qbman_result *dq) +{ + return (int)(qbman_result_DQ_flags(dq) & QBMAN_DQ_STAT_EXPIRED); +} + +/** + * qbman_result_DQ_seqnum() - Get the seqnum field in dequeue response + * seqnum is valid only if VALIDFRAME flag is TRUE + * @dq: the dequeue result. + * + * Return seqnum. + */ +uint16_t qbman_result_DQ_seqnum(const struct qbman_result *dq); + +/** + * qbman_result_DQ_odpid() - Get the seqnum field in dequeue response + * odpid is valid only if ODPVAILD flag is TRUE. + * @dq: the dequeue result. + * + * Return odpid. + */ +uint16_t qbman_result_DQ_odpid(const struct qbman_result *dq); + +/** + * qbman_result_DQ_fqid() - Get the fqid in dequeue response + * @dq: the dequeue result. + * + * Return fqid. + */ +uint32_t qbman_result_DQ_fqid(const struct qbman_result *dq); + +/** + * qbman_result_DQ_byte_count() - Get the byte count in dequeue response + * @dq: the dequeue result. + * + * Return the byte count remaining in the FQ. + */ +uint32_t qbman_result_DQ_byte_count(const struct qbman_result *dq); + +/** + * qbman_result_DQ_frame_count - Get the frame count in dequeue response + * @dq: the dequeue result. + * + * Return the frame count remaining in the FQ. + */ +uint32_t qbman_result_DQ_frame_count(const struct qbman_result *dq); + +/** + * qbman_result_DQ_fqd_ctx() - Get the frame queue context in dequeue response + * @dq: the dequeue result. + * + * Return the frame queue context. + */ +uint64_t qbman_result_DQ_fqd_ctx(const struct qbman_result *dq); + +/** + * qbman_result_DQ_fd() - Get the frame descriptor in dequeue response + * @dq: the dequeue result. + * + * Return the frame descriptor. + */ +const struct qbman_fd *qbman_result_DQ_fd(const struct qbman_result *dq); + +/* State-change notifications (FQDAN/CDAN/CSCN/...). */ + +/** + * qbman_result_SCN_state() - Get the state field in State-change notification + * @scn: the state change notification. + * + * Return the state in the notifiation. + */ +uint8_t qbman_result_SCN_state(const struct qbman_result *scn); + +/** + * qbman_result_SCN_rid() - Get the resource id from the notification + * @scn: the state change notification. + * + * Return the resource id. + */ +uint32_t qbman_result_SCN_rid(const struct qbman_result *scn); + +/** + * qbman_result_SCN_ctx() - get the context from the notification + * @scn: the state change notification. + * + * Return the context. + */ +uint64_t qbman_result_SCN_ctx(const struct qbman_result *scn); + +/* Type-specific "resource IDs". Mainly for illustration purposes, though it + * also gives the appropriate type widths. + */ +/* Get the FQID from the FQDAN */ +#define qbman_result_FQDAN_fqid(dq) qbman_result_SCN_rid(dq) +/* Get the FQID from the FQRN */ +#define qbman_result_FQRN_fqid(dq) qbman_result_SCN_rid(dq) +/* Get the FQID from the FQRNI */ +#define qbman_result_FQRNI_fqid(dq) qbman_result_SCN_rid(dq) +/* Get the FQID from the FQPN */ +#define qbman_result_FQPN_fqid(dq) qbman_result_SCN_rid(dq) +/* Get the channel ID from the CDAN */ +#define qbman_result_CDAN_cid(dq) ((uint16_t)qbman_result_SCN_rid(dq)) +/* Get the CGID from the CSCN */ +#define qbman_result_CSCN_cgid(dq) ((uint16_t)qbman_result_SCN_rid(dq)) + +/** + * qbman_result_bpscn_bpid() - Get the bpid from BPSCN + * @scn: the state change notification. + * + * Return the buffer pool id. + */ +uint16_t qbman_result_bpscn_bpid(const struct qbman_result *scn); + +/** + * qbman_result_bpscn_has_free_bufs() - Check whether there are free + * buffers in the pool from BPSCN. + * @scn: the state change notification. + * + * Return the number of free buffers. + */ +int qbman_result_bpscn_has_free_bufs(const struct qbman_result *scn); + +/** + * qbman_result_bpscn_is_depleted() - Check BPSCN to see whether the + * buffer pool is depleted. + * @scn: the state change notification. + * + * Return the status of buffer pool depletion. + */ +int qbman_result_bpscn_is_depleted(const struct qbman_result *scn); + +/** + * qbman_result_bpscn_is_surplus() - Check BPSCN to see whether the buffer + * pool is surplus or not. + * @scn: the state change notification. + * + * Return the status of buffer pool surplus. + */ +int qbman_result_bpscn_is_surplus(const struct qbman_result *scn); + +/** + * qbman_result_bpscn_ctx() - Get the BPSCN CTX from BPSCN message + * @scn: the state change notification. + * + * Return the BPSCN context. + */ +uint64_t qbman_result_bpscn_ctx(const struct qbman_result *scn); + +/* Parsing CGCU */ +/** + * qbman_result_cgcu_cgid() - Check CGCU resouce id, i.e. cgid + * @scn: the state change notification. + * + * Return the CGCU resource id. + */ +uint16_t qbman_result_cgcu_cgid(const struct qbman_result *scn); + +/** + * qbman_result_cgcu_icnt() - Get the I_CNT from CGCU + * @scn: the state change notification. + * + * Return instantaneous count in the CGCU notification. + */ +uint64_t qbman_result_cgcu_icnt(const struct qbman_result *scn); + + /************/ + /* Enqueues */ + /************/ + +/* struct qbman_eq_desc - structure of enqueue descriptor */ +struct qbman_eq_desc { + union { + uint32_t donot_manipulate_directly[8]; + struct eq { + uint8_t verb; + uint8_t dca; + uint16_t seqnum; + uint16_t orpid; + uint16_t reserved1; + uint32_t tgtid; + uint32_t tag; + uint16_t qdbin; + uint8_t qpri; + uint8_t reserved[3]; + uint8_t wae; + uint8_t rspid; + uint64_t rsp_addr; + } eq; + }; +}; + +/** + * struct qbman_eq_response - structure of enqueue response + * @donot_manipulate_directly: the 16 32bit data to represent the whole + * enqueue response. + */ +struct qbman_eq_response { + uint32_t donot_manipulate_directly[16]; +}; + +/** + * qbman_eq_desc_clear() - Clear the contents of a descriptor to + * default/starting state. + * @d: the given enqueue descriptor. + */ +void qbman_eq_desc_clear(struct qbman_eq_desc *d); + +/* Exactly one of the following descriptor "actions" should be set. (Calling + * any one of these will replace the effect of any prior call to one of these.) + * - enqueue without order-restoration + * - enqueue with order-restoration + * - fill a hole in the order-restoration sequence, without any enqueue + * - advance NESN (Next Expected Sequence Number), without any enqueue + * 'respond_success' indicates whether an enqueue response should be DMA'd + * after success (otherwise a response is DMA'd only after failure). + * 'incomplete' indicates that other fragments of the same 'seqnum' are yet to + * be enqueued. + */ + +/** + * qbman_eq_desc_set_no_orp() - Set enqueue descriptor without orp + * @d: the enqueue descriptor. + * @response_success: 1 = enqueue with response always; 0 = enqueue with + * rejections returned on a FQ. + */ +void qbman_eq_desc_set_no_orp(struct qbman_eq_desc *d, int respond_success); +/** + * qbman_eq_desc_set_orp() - Set order-resotration in the enqueue descriptor + * @d: the enqueue descriptor. + * @response_success: 1 = enqueue with response always; 0 = enqueue with + * rejections returned on a FQ. + * @opr_id: the order point record id. + * @seqnum: the order restoration sequence number. + * @incomplete: indiates whether this is the last fragments using the same + * sequeue number. + */ +void qbman_eq_desc_set_orp(struct qbman_eq_desc *d, int respond_success, + uint16_t opr_id, uint16_t seqnum, int incomplete); + +/** + * qbman_eq_desc_set_orp_hole() - fill a hole in the order-restoration sequence + * without any enqueue + * @d: the enqueue descriptor. + * @opr_id: the order point record id. + * @seqnum: the order restoration sequence number. + */ +void qbman_eq_desc_set_orp_hole(struct qbman_eq_desc *d, uint16_t opr_id, + uint16_t seqnum); + +/** + * qbman_eq_desc_set_orp_nesn() - advance NESN (Next Expected Sequence Number) + * without any enqueue + * @d: the enqueue descriptor. + * @opr_id: the order point record id. + * @seqnum: the order restoration sequence number. + */ +void qbman_eq_desc_set_orp_nesn(struct qbman_eq_desc *d, uint16_t opr_id, + uint16_t seqnum); +/** + * qbman_eq_desc_set_response() - Set the enqueue response info. + * @d: the enqueue descriptor + * @storage_phys: the physical address of the enqueue response in memory. + * @stash: indicate that the write allocation enabled or not. + * + * In the case where an enqueue response is DMA'd, this determines where that + * response should go. (The physical/DMA address is given for hardware's + * benefit, but software should interpret it as a "struct qbman_eq_response" + * data structure.) 'stash' controls whether or not the write to main-memory + * expresses a cache-warming attribute. + */ +void qbman_eq_desc_set_response(struct qbman_eq_desc *d, + uint64_t storage_phys, + int stash); + +/** + * qbman_eq_desc_set_token() - Set token for the enqueue command + * @d: the enqueue descriptor + * @token: the token to be set. + * + * token is the value that shows up in an enqueue response that can be used to + * detect when the results have been published. The easiest technique is to zero + * result "storage" before issuing an enqueue, and use any non-zero 'token' + * value. + */ +void qbman_eq_desc_set_token(struct qbman_eq_desc *d, uint8_t token); + +/** + * Exactly one of the following descriptor "targets" should be set. (Calling any + * one of these will replace the effect of any prior call to one of these.) + * - enqueue to a frame queue + * - enqueue to a queuing destination + * Note, that none of these will have any affect if the "action" type has been + * set to "orp_hole" or "orp_nesn". + */ +/** + * qbman_eq_desc_set_fq() - Set Frame Queue id for the enqueue command + * @d: the enqueue descriptor + * @fqid: the id of the frame queue to be enqueued. + */ +void qbman_eq_desc_set_fq(struct qbman_eq_desc *d, uint32_t fqid); + +/** + * qbman_eq_desc_set_qd() - Set Queuing Destination for the enqueue command. + * @d: the enqueue descriptor + * @qdid: the id of the queuing destination to be enqueued. + * @qd_bin: the queuing destination bin + * @qd_prio: the queuing destination priority. + */ +void qbman_eq_desc_set_qd(struct qbman_eq_desc *d, uint32_t qdid, + uint16_t qd_bin, uint8_t qd_prio); + +/** + * qbman_eq_desc_set_eqdi() - enable/disable EQDI interrupt + * @d: the enqueue descriptor + * @enable: boolean to enable/disable EQDI + * + * Determines whether or not the portal's EQDI interrupt source should be + * asserted after the enqueue command is completed. + */ +void qbman_eq_desc_set_eqdi(struct qbman_eq_desc *d, int enable); + +/** + * qbman_eq_desc_set_dca() - Set DCA mode in the enqueue command. + * @d: the enqueue descriptor. + * @enable: enabled/disable DCA mode. + * @dqrr_idx: DCAP_CI, the DCAP consumer index. + * @park: determine the whether park the FQ or not + * + * Determines whether or not a portal DQRR entry should be consumed once the + * enqueue command is completed. (And if so, and the DQRR entry corresponds to a + * held-active (order-preserving) FQ, whether the FQ should be parked instead of + * being rescheduled.) + */ +void qbman_eq_desc_set_dca(struct qbman_eq_desc *d, int enable, + uint8_t dqrr_idx, int park); + +/** + * qbman_swp_enqueue() - Issue an enqueue command. + * @s: the software portal used for enqueue. + * @d: the enqueue descriptor. + * @fd: the frame descriptor to be enqueued. + * + * Please note that 'fd' should only be NULL if the "action" of the + * descriptor is "orp_hole" or "orp_nesn". + * + * Return 0 for a successful enqueue, -EBUSY if the EQCR is not ready. + */ +int qbman_swp_enqueue(struct qbman_swp *s, const struct qbman_eq_desc *d, + const struct qbman_fd *fd); +/** + * qbman_swp_enqueue_multiple() - Enqueue multiple frames with same + eq descriptor + * @s: the software portal used for enqueue. + * @d: the enqueue descriptor. + * @fd: the frame descriptor to be enqueued. + * @num_frames: the number of the frames to be enqueued. + * + * Return the number of enqueued frames, -EBUSY if the EQCR is not ready. + */ +int qbman_swp_enqueue_multiple(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd, + int num_frames); +/** + * qbman_swp_enqueue_multiple_desc() - Enqueue multiple frames with + * individual eq descriptor. + * @s: the software portal used for enqueue. + * @d: the enqueue descriptor. + * @fd: the frame descriptor to be enqueued. + * @num_frames: the number of the frames to be enqueued. + * + * Return the number of enqueued frames, -EBUSY if the EQCR is not ready. + */ +int qbman_swp_enqueue_multiple_desc(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd, + int num_frames); + +/* TODO: + * qbman_swp_enqueue_thresh() - Set threshold for EQRI interrupt. + * @s: the software portal. + * @thresh: the threshold to trigger the EQRI interrupt. + * + * An EQRI interrupt can be generated when the fill-level of EQCR falls below + * the 'thresh' value set here. Setting thresh==0 (the default) disables. + */ +int qbman_swp_enqueue_thresh(struct qbman_swp *s, unsigned int thresh); + + /*******************/ + /* Buffer releases */ + /*******************/ +/** + * struct qbman_release_desc - The structure for buffer release descriptor + * @donot_manipulate_directly: the 32bit data to represent the whole + * possible settings of qbman release descriptor. + */ +struct qbman_release_desc { + union { + uint32_t donot_manipulate_directly[16]; + struct br { + uint8_t verb; + uint8_t reserved; + uint16_t bpid; + uint32_t reserved2; + uint64_t buf[7]; + } br; + }; +}; + +/** + * qbman_release_desc_clear() - Clear the contents of a descriptor to + * default/starting state. + * @d: the qbman release descriptor. + */ +void qbman_release_desc_clear(struct qbman_release_desc *d); + +/** + * qbman_release_desc_set_bpid() - Set the ID of the buffer pool to release to + * @d: the qbman release descriptor. + */ +void qbman_release_desc_set_bpid(struct qbman_release_desc *d, uint16_t bpid); + +/** + * qbman_release_desc_set_rcdi() - Determines whether or not the portal's RCDI + * interrupt source should be asserted after the release command is completed. + * @d: the qbman release descriptor. + */ +void qbman_release_desc_set_rcdi(struct qbman_release_desc *d, int enable); + +/** + * qbman_swp_release() - Issue a buffer release command. + * @s: the software portal object. + * @d: the release descriptor. + * @buffers: a pointer pointing to the buffer address to be released. + * @num_buffers: number of buffers to be released, must be less than 8. + * + * Return 0 for success, -EBUSY if the release command ring is not ready. + */ +int qbman_swp_release(struct qbman_swp *s, const struct qbman_release_desc *d, + const uint64_t *buffers, unsigned int num_buffers); + +/* TODO: + * qbman_swp_release_thresh() - Set threshold for RCRI interrupt + * @s: the software portal. + * @thresh: the threshold. + * An RCRI interrupt can be generated when the fill-level of RCR falls below + * the 'thresh' value set here. Setting thresh==0 (the default) disables. + */ +int qbman_swp_release_thresh(struct qbman_swp *s, unsigned int thresh); + + /*******************/ + /* Buffer acquires */ + /*******************/ +/** + * qbman_swp_acquire() - Issue a buffer acquire command. + * @s: the software portal object. + * @bpid: the buffer pool index. + * @buffers: a pointer pointing to the acquired buffer address|es. + * @num_buffers: number of buffers to be acquired, must be less than 8. + * + * Return 0 for success, or negative error code if the acquire command + * fails. + */ +int qbman_swp_acquire(struct qbman_swp *s, uint16_t bpid, uint64_t *buffers, + unsigned int num_buffers); + + /*****************/ + /* FQ management */ + /*****************/ +/** + * qbman_swp_fq_schedule() - Move the fq to the scheduled state. + * @s: the software portal object. + * @fqid: the index of frame queue to be scheduled. + * + * There are a couple of different ways that a FQ can end up parked state, + * This schedules it. + * + * Return 0 for success, or negative error code for failure. + */ +int qbman_swp_fq_schedule(struct qbman_swp *s, uint32_t fqid); + +/** + * qbman_swp_fq_force() - Force the FQ to fully scheduled state. + * @s: the software portal object. + * @fqid: the index of frame queue to be forced. + * + * Force eligible will force a tentatively-scheduled FQ to be fully-scheduled + * and thus be available for selection by any channel-dequeuing behaviour (push + * or pull). If the FQ is subsequently "dequeued" from the channel and is still + * empty at the time this happens, the resulting dq_entry will have no FD. + * (qbman_result_DQ_fd() will return NULL.) + * + * Return 0 for success, or negative error code for failure. + */ +int qbman_swp_fq_force(struct qbman_swp *s, uint32_t fqid); + +/** + * These functions change the FQ flow-control stuff between XON/XOFF. (The + * default is XON.) This setting doesn't affect enqueues to the FQ, just + * dequeues. XOFF FQs will remain in the tenatively-scheduled state, even when + * non-empty, meaning they won't be selected for scheduled dequeuing. If a FQ is + * changed to XOFF after it had already become truly-scheduled to a channel, and + * a pull dequeue of that channel occurs that selects that FQ for dequeuing, + * then the resulting dq_entry will have no FD. (qbman_result_DQ_fd() will + * return NULL.) + */ +/** + * qbman_swp_fq_xon() - XON the frame queue. + * @s: the software portal object. + * @fqid: the index of frame queue. + * + * Return 0 for success, or negative error code for failure. + */ +int qbman_swp_fq_xon(struct qbman_swp *s, uint32_t fqid); +/** + * qbman_swp_fq_xoff() - XOFF the frame queue. + * @s: the software portal object. + * @fqid: the index of frame queue. + * + * Return 0 for success, or negative error code for failure. + */ +int qbman_swp_fq_xoff(struct qbman_swp *s, uint32_t fqid); + + /**********************/ + /* Channel management */ + /**********************/ + +/** + * If the user has been allocated a channel object that is going to generate + * CDANs to another channel, then these functions will be necessary. + * CDAN-enabled channels only generate a single CDAN notification, after which + * it they need to be reenabled before they'll generate another. (The idea is + * that pull dequeuing will occur in reaction to the CDAN, followed by a + * reenable step.) Each function generates a distinct command to hardware, so a + * combination function is provided if the user wishes to modify the "context" + * (which shows up in each CDAN message) each time they reenable, as a single + * command to hardware. + */ + +/** + * qbman_swp_CDAN_set_context() - Set CDAN context + * @s: the software portal object. + * @channelid: the channel index. + * @ctx: the context to be set in CDAN. + * + * Return 0 for success, or negative error code for failure. + */ +int qbman_swp_CDAN_set_context(struct qbman_swp *s, uint16_t channelid, + uint64_t ctx); + +/** + * qbman_swp_CDAN_enable() - Enable CDAN for the channel. + * @s: the software portal object. + * @channelid: the index of the channel to generate CDAN. + * + * Return 0 for success, or negative error code for failure. + */ +int qbman_swp_CDAN_enable(struct qbman_swp *s, uint16_t channelid); + +/** + * qbman_swp_CDAN_disable() - disable CDAN for the channel. + * @s: the software portal object. + * @channelid: the index of the channel to generate CDAN. + * + * Return 0 for success, or negative error code for failure. + */ +int qbman_swp_CDAN_disable(struct qbman_swp *s, uint16_t channelid); + +/** + * qbman_swp_CDAN_set_context_enable() - Set CDAN contest and enable CDAN + * @s: the software portal object. + * @channelid: the index of the channel to generate CDAN. + * @ctx: the context set in CDAN. + * + * Return 0 for success, or negative error code for failure. + */ +int qbman_swp_CDAN_set_context_enable(struct qbman_swp *s, uint16_t channelid, + uint64_t ctx); +#endif /* !_FSL_QBMAN_PORTAL_H */ diff --git a/dpdk/drivers/bus/fslmc/qbman/qbman_portal.c b/dpdk/drivers/bus/fslmc/qbman/qbman_portal.c new file mode 100644 index 00000000..809770c7 --- /dev/null +++ b/dpdk/drivers/bus/fslmc/qbman/qbman_portal.c @@ -0,0 +1,1425 @@ +/*- + * BSD LICENSE + * + * Copyright (C) 2014-2016 Freescale Semiconductor, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Freescale Semiconductor nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "qbman_portal.h" + +/* QBMan portal management command codes */ +#define QBMAN_MC_ACQUIRE 0x30 +#define QBMAN_WQCHAN_CONFIGURE 0x46 + +/* CINH register offsets */ +#define QBMAN_CINH_SWP_EQCR_PI 0x800 +#define QBMAN_CINH_SWP_EQCR_CI 0x840 +#define QBMAN_CINH_SWP_EQAR 0x8c0 +#define QBMAN_CINH_SWP_DQPI 0xa00 +#define QBMAN_CINH_SWP_DCAP 0xac0 +#define QBMAN_CINH_SWP_SDQCR 0xb00 +#define QBMAN_CINH_SWP_RAR 0xcc0 +#define QBMAN_CINH_SWP_ISR 0xe00 +#define QBMAN_CINH_SWP_IER 0xe40 +#define QBMAN_CINH_SWP_ISDR 0xe80 +#define QBMAN_CINH_SWP_IIR 0xec0 +#define QBMAN_CINH_SWP_DQRR_ITR 0xa80 +#define QBMAN_CINH_SWP_ITPR 0xf40 + +/* CENA register offsets */ +#define QBMAN_CENA_SWP_EQCR(n) (0x000 + ((uint32_t)(n) << 6)) +#define QBMAN_CENA_SWP_DQRR(n) (0x200 + ((uint32_t)(n) << 6)) +#define QBMAN_CENA_SWP_RCR(n) (0x400 + ((uint32_t)(n) << 6)) +#define QBMAN_CENA_SWP_CR 0x600 +#define QBMAN_CENA_SWP_RR(vb) (0x700 + ((uint32_t)(vb) >> 1)) +#define QBMAN_CENA_SWP_VDQCR 0x780 +#define QBMAN_CENA_SWP_EQCR_CI 0x840 + +/* Reverse mapping of QBMAN_CENA_SWP_DQRR() */ +#define QBMAN_IDX_FROM_DQRR(p) (((unsigned long)p & 0x1ff) >> 6) + +/* QBMan FQ management command codes */ +#define QBMAN_FQ_SCHEDULE 0x48 +#define QBMAN_FQ_FORCE 0x49 +#define QBMAN_FQ_XON 0x4d +#define QBMAN_FQ_XOFF 0x4e + +/*******************************/ +/* Pre-defined attribute codes */ +/*******************************/ + +#define QBMAN_RESPONSE_VERB_MASK 0x7f + +/*************************/ +/* SDQCR attribute codes */ +/*************************/ +#define QB_SDQCR_FC_SHIFT 29 +#define QB_SDQCR_FC_MASK 0x1 +#define QB_SDQCR_DCT_SHIFT 24 +#define QB_SDQCR_DCT_MASK 0x3 +#define QB_SDQCR_TOK_SHIFT 16 +#define QB_SDQCR_TOK_MASK 0xff +#define QB_SDQCR_SRC_SHIFT 0 +#define QB_SDQCR_SRC_MASK 0xffff + +/* opaque token for static dequeues */ +#define QMAN_SDQCR_TOKEN 0xbb + +enum qbman_sdqcr_dct { + qbman_sdqcr_dct_null = 0, + qbman_sdqcr_dct_prio_ics, + qbman_sdqcr_dct_active_ics, + qbman_sdqcr_dct_active +}; + +enum qbman_sdqcr_fc { + qbman_sdqcr_fc_one = 0, + qbman_sdqcr_fc_up_to_3 = 1 +}; + +/* We need to keep track of which SWP triggered a pull command + * so keep an array of portal IDs and use the token field to + * be able to find the proper portal + */ +#define MAX_QBMAN_PORTALS 64 +static struct qbman_swp *portal_idx_map[MAX_QBMAN_PORTALS]; + +/*********************************/ +/* Portal constructor/destructor */ +/*********************************/ + +/* Software portals should always be in the power-on state when we initialise, + * due to the CCSR-based portal reset functionality that MC has. + * + * Erk! Turns out that QMan versions prior to 4.1 do not correctly reset DQRR + * valid-bits, so we need to support a workaround where we don't trust + * valid-bits when detecting new entries until any stale ring entries have been + * overwritten at least once. The idea is that we read PI for the first few + * entries, then switch to valid-bit after that. The trick is to clear the + * bug-work-around boolean once the PI wraps around the ring for the first time. + * + * Note: this still carries a slight additional cost once the decrementer hits + * zero. + */ +struct qbman_swp *qbman_swp_init(const struct qbman_swp_desc *d) +{ + int ret; + uint32_t eqcr_pi; + struct qbman_swp *p = malloc(sizeof(*p)); + + if (!p) + return NULL; + p->desc = *d; +#ifdef QBMAN_CHECKING + p->mc.check = swp_mc_can_start; +#endif + p->mc.valid_bit = QB_VALID_BIT; + p->sdq = 0; + p->sdq |= qbman_sdqcr_dct_prio_ics << QB_SDQCR_DCT_SHIFT; + p->sdq |= qbman_sdqcr_fc_up_to_3 << QB_SDQCR_FC_SHIFT; + p->sdq |= QMAN_SDQCR_TOKEN << QB_SDQCR_TOK_SHIFT; + + atomic_set(&p->vdq.busy, 1); + p->vdq.valid_bit = QB_VALID_BIT; + p->dqrr.next_idx = 0; + p->dqrr.valid_bit = QB_VALID_BIT; + qman_version = p->desc.qman_version; + if ((qman_version & 0xFFFF0000) < QMAN_REV_4100) { + p->dqrr.dqrr_size = 4; + p->dqrr.reset_bug = 1; + } else { + p->dqrr.dqrr_size = 8; + p->dqrr.reset_bug = 0; + } + + ret = qbman_swp_sys_init(&p->sys, d, p->dqrr.dqrr_size); + if (ret) { + free(p); + pr_err("qbman_swp_sys_init() failed %d\n", ret); + return NULL; + } + /* SDQCR needs to be initialized to 0 when no channels are + * being dequeued from or else the QMan HW will indicate an + * error. The values that were calculated above will be + * applied when dequeues from a specific channel are enabled. + */ + qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_SDQCR, 0); + eqcr_pi = qbman_cinh_read(&p->sys, QBMAN_CINH_SWP_EQCR_PI); + p->eqcr.pi = eqcr_pi & 0xF; + p->eqcr.pi_vb = eqcr_pi & QB_VALID_BIT; + p->eqcr.ci = qbman_cinh_read(&p->sys, QBMAN_CINH_SWP_EQCR_CI) & 0xF; + p->eqcr.available = QBMAN_EQCR_SIZE - qm_cyc_diff(QBMAN_EQCR_SIZE, + p->eqcr.ci, p->eqcr.pi); + + portal_idx_map[p->desc.idx] = p; + return p; +} + +void qbman_swp_finish(struct qbman_swp *p) +{ +#ifdef QBMAN_CHECKING + QBMAN_BUG_ON(p->mc.check != swp_mc_can_start); +#endif + qbman_swp_sys_finish(&p->sys); + portal_idx_map[p->desc.idx] = NULL; + free(p); +} + +const struct qbman_swp_desc *qbman_swp_get_desc(struct qbman_swp *p) +{ + return &p->desc; +} + +/**************/ +/* Interrupts */ +/**************/ + +uint32_t qbman_swp_interrupt_get_vanish(struct qbman_swp *p) +{ + return qbman_cinh_read(&p->sys, QBMAN_CINH_SWP_ISDR); +} + +void qbman_swp_interrupt_set_vanish(struct qbman_swp *p, uint32_t mask) +{ + qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_ISDR, mask); +} + +uint32_t qbman_swp_interrupt_read_status(struct qbman_swp *p) +{ + return qbman_cinh_read(&p->sys, QBMAN_CINH_SWP_ISR); +} + +void qbman_swp_interrupt_clear_status(struct qbman_swp *p, uint32_t mask) +{ + qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_ISR, mask); +} + +uint32_t qbman_swp_dqrr_thrshld_read_status(struct qbman_swp *p) +{ + return qbman_cinh_read(&p->sys, QBMAN_CINH_SWP_DQRR_ITR); +} + +void qbman_swp_dqrr_thrshld_write(struct qbman_swp *p, uint32_t mask) +{ + qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_DQRR_ITR, mask); +} + +uint32_t qbman_swp_intr_timeout_read_status(struct qbman_swp *p) +{ + return qbman_cinh_read(&p->sys, QBMAN_CINH_SWP_ITPR); +} + +void qbman_swp_intr_timeout_write(struct qbman_swp *p, uint32_t mask) +{ + qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_ITPR, mask); +} + +uint32_t qbman_swp_interrupt_get_trigger(struct qbman_swp *p) +{ + return qbman_cinh_read(&p->sys, QBMAN_CINH_SWP_IER); +} + +void qbman_swp_interrupt_set_trigger(struct qbman_swp *p, uint32_t mask) +{ + qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_IER, mask); +} + +int qbman_swp_interrupt_get_inhibit(struct qbman_swp *p) +{ + return qbman_cinh_read(&p->sys, QBMAN_CINH_SWP_IIR); +} + +void qbman_swp_interrupt_set_inhibit(struct qbman_swp *p, int inhibit) +{ + qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_IIR, inhibit ? 0xffffffff : 0); +} + +/***********************/ +/* Management commands */ +/***********************/ + +/* + * Internal code common to all types of management commands. + */ + +void *qbman_swp_mc_start(struct qbman_swp *p) +{ + void *ret; +#ifdef QBMAN_CHECKING + QBMAN_BUG_ON(p->mc.check != swp_mc_can_start); +#endif + ret = qbman_cena_write_start(&p->sys, QBMAN_CENA_SWP_CR); +#ifdef QBMAN_CHECKING + if (!ret) + p->mc.check = swp_mc_can_submit; +#endif + return ret; +} + +void qbman_swp_mc_submit(struct qbman_swp *p, void *cmd, uint8_t cmd_verb) +{ + uint8_t *v = cmd; +#ifdef QBMAN_CHECKING + QBMAN_BUG_ON(!(p->mc.check != swp_mc_can_submit)); +#endif + /* TBD: "|=" is going to hurt performance. Need to move as many fields + * out of word zero, and for those that remain, the "OR" needs to occur + * at the caller side. This debug check helps to catch cases where the + * caller wants to OR but has forgotten to do so. + */ + QBMAN_BUG_ON((*v & cmd_verb) != *v); + *v = cmd_verb | p->mc.valid_bit; + qbman_cena_write_complete(&p->sys, QBMAN_CENA_SWP_CR, cmd); +#ifdef QBMAN_CHECKING + p->mc.check = swp_mc_can_poll; +#endif +} + +void *qbman_swp_mc_result(struct qbman_swp *p) +{ + uint32_t *ret, verb; +#ifdef QBMAN_CHECKING + QBMAN_BUG_ON(p->mc.check != swp_mc_can_poll); +#endif + qbman_cena_invalidate_prefetch(&p->sys, + QBMAN_CENA_SWP_RR(p->mc.valid_bit)); + ret = qbman_cena_read(&p->sys, QBMAN_CENA_SWP_RR(p->mc.valid_bit)); + /* Remove the valid-bit - command completed if the rest is non-zero */ + verb = ret[0] & ~QB_VALID_BIT; + if (!verb) + return NULL; +#ifdef QBMAN_CHECKING + p->mc.check = swp_mc_can_start; +#endif + p->mc.valid_bit ^= QB_VALID_BIT; + return ret; +} + +/***********/ +/* Enqueue */ +/***********/ + +#define QB_ENQUEUE_CMD_OPTIONS_SHIFT 0 +enum qb_enqueue_commands { + enqueue_empty = 0, + enqueue_response_always = 1, + enqueue_rejects_to_fq = 2 +}; + +#define QB_ENQUEUE_CMD_EC_OPTION_MASK 0x3 +#define QB_ENQUEUE_CMD_ORP_ENABLE_SHIFT 2 +#define QB_ENQUEUE_CMD_IRQ_ON_DISPATCH_SHIFT 3 +#define QB_ENQUEUE_CMD_TARGET_TYPE_SHIFT 4 +#define QB_ENQUEUE_CMD_DCA_PK_SHIFT 6 +#define QB_ENQUEUE_CMD_DCA_EN_SHIFT 7 +#define QB_ENQUEUE_CMD_NLIS_SHIFT 14 +#define QB_ENQUEUE_CMD_IS_NESN_SHIFT 15 + +void qbman_eq_desc_clear(struct qbman_eq_desc *d) +{ + memset(d, 0, sizeof(*d)); +} + +void qbman_eq_desc_set_no_orp(struct qbman_eq_desc *d, int respond_success) +{ + d->eq.verb &= ~(1 << QB_ENQUEUE_CMD_ORP_ENABLE_SHIFT); + if (respond_success) + d->eq.verb |= enqueue_response_always; + else + d->eq.verb |= enqueue_rejects_to_fq; +} + +void qbman_eq_desc_set_orp(struct qbman_eq_desc *d, int respond_success, + uint16_t opr_id, uint16_t seqnum, int incomplete) +{ + d->eq.verb |= 1 << QB_ENQUEUE_CMD_ORP_ENABLE_SHIFT; + if (respond_success) + d->eq.verb |= enqueue_response_always; + else + d->eq.verb |= enqueue_rejects_to_fq; + + d->eq.orpid = opr_id; + d->eq.seqnum = seqnum; + if (incomplete) + d->eq.seqnum |= 1 << QB_ENQUEUE_CMD_NLIS_SHIFT; + else + d->eq.seqnum &= ~(1 << QB_ENQUEUE_CMD_NLIS_SHIFT); +} + +void qbman_eq_desc_set_orp_hole(struct qbman_eq_desc *d, uint16_t opr_id, + uint16_t seqnum) +{ + d->eq.verb |= 1 << QB_ENQUEUE_CMD_ORP_ENABLE_SHIFT; + d->eq.verb &= ~QB_ENQUEUE_CMD_EC_OPTION_MASK; + d->eq.orpid = opr_id; + d->eq.seqnum = seqnum; + d->eq.seqnum &= ~(1 << QB_ENQUEUE_CMD_NLIS_SHIFT); + d->eq.seqnum &= ~(1 << QB_ENQUEUE_CMD_IS_NESN_SHIFT); +} + +void qbman_eq_desc_set_orp_nesn(struct qbman_eq_desc *d, uint16_t opr_id, + uint16_t seqnum) +{ + d->eq.verb |= 1 << QB_ENQUEUE_CMD_ORP_ENABLE_SHIFT; + d->eq.verb &= ~QB_ENQUEUE_CMD_EC_OPTION_MASK; + d->eq.orpid = opr_id; + d->eq.seqnum = seqnum; + d->eq.seqnum &= ~(1 << QB_ENQUEUE_CMD_NLIS_SHIFT); + d->eq.seqnum |= 1 << QB_ENQUEUE_CMD_IS_NESN_SHIFT; +} + +void qbman_eq_desc_set_response(struct qbman_eq_desc *d, + dma_addr_t storage_phys, + int stash) +{ + d->eq.rsp_addr = storage_phys; + d->eq.wae = stash; +} + +void qbman_eq_desc_set_token(struct qbman_eq_desc *d, uint8_t token) +{ + d->eq.rspid = token; +} + +void qbman_eq_desc_set_fq(struct qbman_eq_desc *d, uint32_t fqid) +{ + d->eq.verb &= ~(1 << QB_ENQUEUE_CMD_TARGET_TYPE_SHIFT); + d->eq.tgtid = fqid; +} + +void qbman_eq_desc_set_qd(struct qbman_eq_desc *d, uint32_t qdid, + uint16_t qd_bin, uint8_t qd_prio) +{ + d->eq.verb |= 1 << QB_ENQUEUE_CMD_TARGET_TYPE_SHIFT; + d->eq.tgtid = qdid; + d->eq.qdbin = qd_bin; + d->eq.qpri = qd_prio; +} + +void qbman_eq_desc_set_eqdi(struct qbman_eq_desc *d, int enable) +{ + if (enable) + d->eq.verb |= 1 << QB_ENQUEUE_CMD_IRQ_ON_DISPATCH_SHIFT; + else + d->eq.verb &= ~(1 << QB_ENQUEUE_CMD_IRQ_ON_DISPATCH_SHIFT); +} + +void qbman_eq_desc_set_dca(struct qbman_eq_desc *d, int enable, + uint8_t dqrr_idx, int park) +{ + if (enable) { + d->eq.dca = dqrr_idx; + if (park) + d->eq.dca |= 1 << QB_ENQUEUE_CMD_DCA_PK_SHIFT; + else + d->eq.dca &= ~(1 << QB_ENQUEUE_CMD_DCA_PK_SHIFT); + d->eq.dca |= 1 << QB_ENQUEUE_CMD_DCA_EN_SHIFT; + } else { + d->eq.dca &= ~(1 << QB_ENQUEUE_CMD_DCA_EN_SHIFT); + } +} + +#define EQAR_IDX(eqar) ((eqar) & 0x7) +#define EQAR_VB(eqar) ((eqar) & 0x80) +#define EQAR_SUCCESS(eqar) ((eqar) & 0x100) + +static int qbman_swp_enqueue_array_mode(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd) +{ + uint32_t *p; + const uint32_t *cl = qb_cl(d); + uint32_t eqar = qbman_cinh_read(&s->sys, QBMAN_CINH_SWP_EQAR); + + pr_debug("EQAR=%08x\n", eqar); + if (!EQAR_SUCCESS(eqar)) + return -EBUSY; + p = qbman_cena_write_start_wo_shadow(&s->sys, + QBMAN_CENA_SWP_EQCR(EQAR_IDX(eqar))); + memcpy(&p[1], &cl[1], 28); + memcpy(&p[8], fd, sizeof(*fd)); + /* Set the verb byte, have to substitute in the valid-bit */ + lwsync(); + p[0] = cl[0] | EQAR_VB(eqar); + qbman_cena_write_complete_wo_shadow(&s->sys, + QBMAN_CENA_SWP_EQCR(EQAR_IDX(eqar))); + return 0; +} + +static int qbman_swp_enqueue_ring_mode(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd) +{ + uint32_t *p; + const uint32_t *cl = qb_cl(d); + uint32_t eqcr_ci; + uint8_t diff; + + if (!s->eqcr.available) { + eqcr_ci = s->eqcr.ci; + s->eqcr.ci = qbman_cena_read_reg(&s->sys, + QBMAN_CENA_SWP_EQCR_CI) & 0xF; + diff = qm_cyc_diff(QBMAN_EQCR_SIZE, + eqcr_ci, s->eqcr.ci); + s->eqcr.available += diff; + if (!diff) + return -EBUSY; + } + + p = qbman_cena_write_start_wo_shadow(&s->sys, + QBMAN_CENA_SWP_EQCR(s->eqcr.pi & 7)); + memcpy(&p[1], &cl[1], 28); + memcpy(&p[8], fd, sizeof(*fd)); + lwsync(); + + /* Set the verb byte, have to substitute in the valid-bit */ + p[0] = cl[0] | s->eqcr.pi_vb; + qbman_cena_write_complete_wo_shadow(&s->sys, + QBMAN_CENA_SWP_EQCR(s->eqcr.pi & 7)); + s->eqcr.pi++; + s->eqcr.pi &= 0xF; + s->eqcr.available--; + if (!(s->eqcr.pi & 7)) + s->eqcr.pi_vb ^= QB_VALID_BIT; + + return 0; +} + +int qbman_swp_enqueue(struct qbman_swp *s, const struct qbman_eq_desc *d, + const struct qbman_fd *fd) +{ + if (s->sys.eqcr_mode == qman_eqcr_vb_array) + return qbman_swp_enqueue_array_mode(s, d, fd); + else /* Use ring mode by default */ + return qbman_swp_enqueue_ring_mode(s, d, fd); +} + +int qbman_swp_enqueue_multiple(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd, + int num_frames) +{ + uint32_t *p; + const uint32_t *cl = qb_cl(d); + uint32_t eqcr_ci, eqcr_pi; + uint8_t diff; + int i, num_enqueued = 0; + uint64_t addr_cena; + + if (!s->eqcr.available) { + eqcr_ci = s->eqcr.ci; + s->eqcr.ci = qbman_cena_read_reg(&s->sys, + QBMAN_CENA_SWP_EQCR_CI) & 0xF; + diff = qm_cyc_diff(QBMAN_EQCR_SIZE, + eqcr_ci, s->eqcr.ci); + s->eqcr.available += diff; + if (!diff) + return 0; + } + + eqcr_pi = s->eqcr.pi; + num_enqueued = (s->eqcr.available < num_frames) ? + s->eqcr.available : num_frames; + s->eqcr.available -= num_enqueued; + /* Fill in the EQCR ring */ + for (i = 0; i < num_enqueued; i++) { + p = qbman_cena_write_start_wo_shadow(&s->sys, + QBMAN_CENA_SWP_EQCR(eqcr_pi & 7)); + memcpy(&p[1], &cl[1], 28); + memcpy(&p[8], &fd[i], sizeof(*fd)); + eqcr_pi++; + eqcr_pi &= 0xF; + } + + lwsync(); + + /* Set the verb byte, have to substitute in the valid-bit */ + eqcr_pi = s->eqcr.pi; + for (i = 0; i < num_enqueued; i++) { + p = qbman_cena_write_start_wo_shadow(&s->sys, + QBMAN_CENA_SWP_EQCR(eqcr_pi & 7)); + p[0] = cl[0] | s->eqcr.pi_vb; + eqcr_pi++; + eqcr_pi &= 0xF; + if (!(eqcr_pi & 7)) + s->eqcr.pi_vb ^= QB_VALID_BIT; + } + + /* Flush all the cacheline without load/store in between */ + eqcr_pi = s->eqcr.pi; + addr_cena = (uint64_t)s->sys.addr_cena; + for (i = 0; i < num_enqueued; i++) { + dcbf((uint64_t *)(addr_cena + + QBMAN_CENA_SWP_EQCR(eqcr_pi & 7))); + eqcr_pi++; + eqcr_pi &= 0xF; + } + s->eqcr.pi = eqcr_pi; + + return num_enqueued; +} + +int qbman_swp_enqueue_multiple_desc(struct qbman_swp *s, + const struct qbman_eq_desc *d, + const struct qbman_fd *fd, + int num_frames) +{ + uint32_t *p; + const uint32_t *cl; + uint32_t eqcr_ci, eqcr_pi; + uint8_t diff; + int i, num_enqueued = 0; + uint64_t addr_cena; + + if (!s->eqcr.available) { + eqcr_ci = s->eqcr.ci; + s->eqcr.ci = qbman_cena_read_reg(&s->sys, + QBMAN_CENA_SWP_EQCR_CI) & 0xF; + diff = qm_cyc_diff(QBMAN_EQCR_SIZE, + eqcr_ci, s->eqcr.ci); + s->eqcr.available += diff; + if (!diff) + return 0; + } + + eqcr_pi = s->eqcr.pi; + num_enqueued = (s->eqcr.available < num_frames) ? + s->eqcr.available : num_frames; + s->eqcr.available -= num_enqueued; + /* Fill in the EQCR ring */ + for (i = 0; i < num_enqueued; i++) { + p = qbman_cena_write_start_wo_shadow(&s->sys, + QBMAN_CENA_SWP_EQCR(eqcr_pi & 7)); + cl = qb_cl(&d[i]); + memcpy(&p[1], &cl[1], 28); + memcpy(&p[8], &fd[i], sizeof(*fd)); + eqcr_pi++; + eqcr_pi &= 0xF; + } + + lwsync(); + + /* Set the verb byte, have to substitute in the valid-bit */ + eqcr_pi = s->eqcr.pi; + for (i = 0; i < num_enqueued; i++) { + p = qbman_cena_write_start_wo_shadow(&s->sys, + QBMAN_CENA_SWP_EQCR(eqcr_pi & 7)); + cl = qb_cl(&d[i]); + p[0] = cl[0] | s->eqcr.pi_vb; + eqcr_pi++; + eqcr_pi &= 0xF; + if (!(eqcr_pi & 7)) + s->eqcr.pi_vb ^= QB_VALID_BIT; + } + + /* Flush all the cacheline without load/store in between */ + eqcr_pi = s->eqcr.pi; + addr_cena = (uint64_t)s->sys.addr_cena; + for (i = 0; i < num_enqueued; i++) { + dcbf((uint64_t *)(addr_cena + + QBMAN_CENA_SWP_EQCR(eqcr_pi & 7))); + eqcr_pi++; + eqcr_pi &= 0xF; + } + s->eqcr.pi = eqcr_pi; + + return num_enqueued; +} + +/*************************/ +/* Static (push) dequeue */ +/*************************/ + +void qbman_swp_push_get(struct qbman_swp *s, uint8_t channel_idx, int *enabled) +{ + uint16_t src = (s->sdq >> QB_SDQCR_SRC_SHIFT) & QB_SDQCR_SRC_MASK; + + QBMAN_BUG_ON(channel_idx > 15); + *enabled = src | (1 << channel_idx); +} + +void qbman_swp_push_set(struct qbman_swp *s, uint8_t channel_idx, int enable) +{ + uint16_t dqsrc; + + QBMAN_BUG_ON(channel_idx > 15); + if (enable) + s->sdq |= 1 << channel_idx; + else + s->sdq &= ~(1 << channel_idx); + + /* Read make the complete src map. If no channels are enabled + * the SDQCR must be 0 or else QMan will assert errors + */ + dqsrc = (s->sdq >> QB_SDQCR_SRC_SHIFT) & QB_SDQCR_SRC_MASK; + if (dqsrc != 0) + qbman_cinh_write(&s->sys, QBMAN_CINH_SWP_SDQCR, s->sdq); + else + qbman_cinh_write(&s->sys, QBMAN_CINH_SWP_SDQCR, 0); +} + +/***************************/ +/* Volatile (pull) dequeue */ +/***************************/ + +/* These should be const, eventually */ +#define QB_VDQCR_VERB_DCT_SHIFT 0 +#define QB_VDQCR_VERB_DT_SHIFT 2 +#define QB_VDQCR_VERB_RLS_SHIFT 4 +#define QB_VDQCR_VERB_WAE_SHIFT 5 + +enum qb_pull_dt_e { + qb_pull_dt_channel, + qb_pull_dt_workqueue, + qb_pull_dt_framequeue +}; + +void qbman_pull_desc_clear(struct qbman_pull_desc *d) +{ + memset(d, 0, sizeof(*d)); +} + +void qbman_pull_desc_set_storage(struct qbman_pull_desc *d, + struct qbman_result *storage, + dma_addr_t storage_phys, + int stash) +{ + d->pull.rsp_addr_virt = (uint64_t)storage; + + if (!storage) { + d->pull.verb &= ~(1 << QB_VDQCR_VERB_RLS_SHIFT); + return; + } + d->pull.verb |= 1 << QB_VDQCR_VERB_RLS_SHIFT; + if (stash) + d->pull.verb |= 1 << QB_VDQCR_VERB_WAE_SHIFT; + else + d->pull.verb &= ~(1 << QB_VDQCR_VERB_WAE_SHIFT); + + d->pull.rsp_addr = storage_phys; +} + +void qbman_pull_desc_set_numframes(struct qbman_pull_desc *d, uint8_t numframes) +{ + d->pull.numf = numframes - 1; +} + +void qbman_pull_desc_set_token(struct qbman_pull_desc *d, uint8_t token) +{ + d->pull.tok = token; +} + +void qbman_pull_desc_set_fq(struct qbman_pull_desc *d, uint32_t fqid) +{ + d->pull.verb |= 1 << QB_VDQCR_VERB_DCT_SHIFT; + d->pull.verb |= qb_pull_dt_framequeue << QB_VDQCR_VERB_DT_SHIFT; + d->pull.dq_src = fqid; +} + +void qbman_pull_desc_set_wq(struct qbman_pull_desc *d, uint32_t wqid, + enum qbman_pull_type_e dct) +{ + d->pull.verb |= dct << QB_VDQCR_VERB_DCT_SHIFT; + d->pull.verb |= qb_pull_dt_workqueue << QB_VDQCR_VERB_DT_SHIFT; + d->pull.dq_src = wqid; +} + +void qbman_pull_desc_set_channel(struct qbman_pull_desc *d, uint32_t chid, + enum qbman_pull_type_e dct) +{ + d->pull.verb |= dct << QB_VDQCR_VERB_DCT_SHIFT; + d->pull.verb |= qb_pull_dt_channel << QB_VDQCR_VERB_DT_SHIFT; + d->pull.dq_src = chid; +} + +int qbman_swp_pull(struct qbman_swp *s, struct qbman_pull_desc *d) +{ + uint32_t *p; + uint32_t *cl = qb_cl(d); + + if (!atomic_dec_and_test(&s->vdq.busy)) { + atomic_inc(&s->vdq.busy); + return -EBUSY; + } + + d->pull.tok = s->sys.idx + 1; + s->vdq.storage = (void *)d->pull.rsp_addr_virt; + p = qbman_cena_write_start_wo_shadow(&s->sys, QBMAN_CENA_SWP_VDQCR); + memcpy(&p[1], &cl[1], 12); + + /* Set the verb byte, have to substitute in the valid-bit */ + lwsync(); + p[0] = cl[0] | s->vdq.valid_bit; + s->vdq.valid_bit ^= QB_VALID_BIT; + qbman_cena_write_complete_wo_shadow(&s->sys, QBMAN_CENA_SWP_VDQCR); + + return 0; +} + +/****************/ +/* Polling DQRR */ +/****************/ + +#define QMAN_DQRR_PI_MASK 0xf + +#define QBMAN_RESULT_DQ 0x60 +#define QBMAN_RESULT_FQRN 0x21 +#define QBMAN_RESULT_FQRNI 0x22 +#define QBMAN_RESULT_FQPN 0x24 +#define QBMAN_RESULT_FQDAN 0x25 +#define QBMAN_RESULT_CDAN 0x26 +#define QBMAN_RESULT_CSCN_MEM 0x27 +#define QBMAN_RESULT_CGCU 0x28 +#define QBMAN_RESULT_BPSCN 0x29 +#define QBMAN_RESULT_CSCN_WQ 0x2a + +/* NULL return if there are no unconsumed DQRR entries. Returns a DQRR entry + * only once, so repeated calls can return a sequence of DQRR entries, without + * requiring they be consumed immediately or in any particular order. + */ +const struct qbman_result *qbman_swp_dqrr_next(struct qbman_swp *s) +{ + uint32_t verb; + uint32_t response_verb; + uint32_t flags; + const struct qbman_result *p; + + /* Before using valid-bit to detect if something is there, we have to + * handle the case of the DQRR reset bug... + */ + if (unlikely(s->dqrr.reset_bug)) { + /* We pick up new entries by cache-inhibited producer index, + * which means that a non-coherent mapping would require us to + * invalidate and read *only* once that PI has indicated that + * there's an entry here. The first trip around the DQRR ring + * will be much less efficient than all subsequent trips around + * it... + */ + uint8_t pi = qbman_cinh_read(&s->sys, QBMAN_CINH_SWP_DQPI) & + QMAN_DQRR_PI_MASK; + + /* there are new entries if pi != next_idx */ + if (pi == s->dqrr.next_idx) + return NULL; + + /* if next_idx is/was the last ring index, and 'pi' is + * different, we can disable the workaround as all the ring + * entries have now been DMA'd to so valid-bit checking is + * repaired. Note: this logic needs to be based on next_idx + * (which increments one at a time), rather than on pi (which + * can burst and wrap-around between our snapshots of it). + */ + QBMAN_BUG_ON((s->dqrr.dqrr_size - 1) < 0); + if (s->dqrr.next_idx == (s->dqrr.dqrr_size - 1u)) { + pr_debug("DEBUG: next_idx=%d, pi=%d, clear reset bug\n", + s->dqrr.next_idx, pi); + s->dqrr.reset_bug = 0; + } + qbman_cena_invalidate_prefetch(&s->sys, + QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx)); + } + p = qbman_cena_read_wo_shadow(&s->sys, + QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx)); + verb = p->dq.verb; + + /* If the valid-bit isn't of the expected polarity, nothing there. Note, + * in the DQRR reset bug workaround, we shouldn't need to skip these + * check, because we've already determined that a new entry is available + * and we've invalidated the cacheline before reading it, so the + * valid-bit behaviour is repaired and should tell us what we already + * knew from reading PI. + */ + if ((verb & QB_VALID_BIT) != s->dqrr.valid_bit) + return NULL; + + /* There's something there. Move "next_idx" attention to the next ring + * entry (and prefetch it) before returning what we found. + */ + s->dqrr.next_idx++; + if (s->dqrr.next_idx == s->dqrr.dqrr_size) { + s->dqrr.next_idx = 0; + s->dqrr.valid_bit ^= QB_VALID_BIT; + } + /* If this is the final response to a volatile dequeue command + * indicate that the vdq is no longer busy + */ + flags = p->dq.stat; + response_verb = verb & QBMAN_RESPONSE_VERB_MASK; + if ((response_verb == QBMAN_RESULT_DQ) && + (flags & QBMAN_DQ_STAT_VOLATILE) && + (flags & QBMAN_DQ_STAT_EXPIRED)) + atomic_inc(&s->vdq.busy); + + return p; +} + +/* Consume DQRR entries previously returned from qbman_swp_dqrr_next(). */ +void qbman_swp_dqrr_consume(struct qbman_swp *s, + const struct qbman_result *dq) +{ + qbman_cinh_write(&s->sys, QBMAN_CINH_SWP_DCAP, QBMAN_IDX_FROM_DQRR(dq)); +} + +/*********************************/ +/* Polling user-provided storage */ +/*********************************/ +int qbman_result_has_new_result(struct qbman_swp *s, + struct qbman_result *dq) +{ + if (dq->dq.tok == 0) + return 0; + + /* + * Set token to be 0 so we will detect change back to 1 + * next time the looping is traversed. Const is cast away here + * as we want users to treat the dequeue responses as read only. + */ + ((struct qbman_result *)dq)->dq.tok = 0; + + /* + * VDQCR "no longer busy" hook - not quite the same as DQRR, because the + * fact "VDQCR" shows busy doesn't mean that we hold the result that + * makes it available. Eg. we may be looking at our 10th dequeue result, + * having released VDQCR after the 1st result and it is now busy due to + * some other command! + */ + if (s->vdq.storage == dq) { + s->vdq.storage = NULL; + atomic_inc(&s->vdq.busy); + } + + return 1; +} + +int qbman_check_new_result(struct qbman_result *dq) +{ + if (dq->dq.tok == 0) + return 0; + + /* + * Set token to be 0 so we will detect change back to 1 + * next time the looping is traversed. Const is cast away here + * as we want users to treat the dequeue responses as read only. + */ + ((struct qbman_result *)dq)->dq.tok = 0; + + return 1; +} + +int qbman_check_command_complete(struct qbman_result *dq) +{ + struct qbman_swp *s; + + if (dq->dq.tok == 0) + return 0; + + s = portal_idx_map[dq->dq.tok - 1]; + /* + * VDQCR "no longer busy" hook - not quite the same as DQRR, because the + * fact "VDQCR" shows busy doesn't mean that we hold the result that + * makes it available. Eg. we may be looking at our 10th dequeue result, + * having released VDQCR after the 1st result and it is now busy due to + * some other command! + */ + if (s->vdq.storage == dq) { + s->vdq.storage = NULL; + atomic_inc(&s->vdq.busy); + } + + return 1; +} + +/********************************/ +/* Categorising qbman results */ +/********************************/ + +static inline int __qbman_result_is_x(const struct qbman_result *dq, + uint8_t x) +{ + uint8_t response_verb = dq->dq.verb & QBMAN_RESPONSE_VERB_MASK; + + return (response_verb == x); +} + +int qbman_result_is_DQ(const struct qbman_result *dq) +{ + return __qbman_result_is_x(dq, QBMAN_RESULT_DQ); +} + +int qbman_result_is_FQDAN(const struct qbman_result *dq) +{ + return __qbman_result_is_x(dq, QBMAN_RESULT_FQDAN); +} + +int qbman_result_is_CDAN(const struct qbman_result *dq) +{ + return __qbman_result_is_x(dq, QBMAN_RESULT_CDAN); +} + +int qbman_result_is_CSCN(const struct qbman_result *dq) +{ + return __qbman_result_is_x(dq, QBMAN_RESULT_CSCN_MEM) || + __qbman_result_is_x(dq, QBMAN_RESULT_CSCN_WQ); +} + +int qbman_result_is_BPSCN(const struct qbman_result *dq) +{ + return __qbman_result_is_x(dq, QBMAN_RESULT_BPSCN); +} + +int qbman_result_is_CGCU(const struct qbman_result *dq) +{ + return __qbman_result_is_x(dq, QBMAN_RESULT_CGCU); +} + +int qbman_result_is_FQRN(const struct qbman_result *dq) +{ + return __qbman_result_is_x(dq, QBMAN_RESULT_FQRN); +} + +int qbman_result_is_FQRNI(const struct qbman_result *dq) +{ + return __qbman_result_is_x(dq, QBMAN_RESULT_FQRNI); +} + +int qbman_result_is_FQPN(const struct qbman_result *dq) +{ + return __qbman_result_is_x(dq, QBMAN_RESULT_FQPN); +} + +/*********************************/ +/* Parsing frame dequeue results */ +/*********************************/ + +/* These APIs assume qbman_result_is_DQ() is TRUE */ + +uint8_t qbman_result_DQ_flags(const struct qbman_result *dq) +{ + return dq->dq.stat; +} + +uint16_t qbman_result_DQ_seqnum(const struct qbman_result *dq) +{ + return dq->dq.seqnum; +} + +uint16_t qbman_result_DQ_odpid(const struct qbman_result *dq) +{ + return dq->dq.oprid; +} + +uint32_t qbman_result_DQ_fqid(const struct qbman_result *dq) +{ + return dq->dq.fqid; +} + +uint32_t qbman_result_DQ_byte_count(const struct qbman_result *dq) +{ + return dq->dq.fq_byte_cnt; +} + +uint32_t qbman_result_DQ_frame_count(const struct qbman_result *dq) +{ + return dq->dq.fq_frm_cnt; +} + +uint64_t qbman_result_DQ_fqd_ctx(const struct qbman_result *dq) +{ + return dq->dq.fqd_ctx; +} + +const struct qbman_fd *qbman_result_DQ_fd(const struct qbman_result *dq) +{ + return (const struct qbman_fd *)&dq->dq.fd[0]; +} + +/**************************************/ +/* Parsing state-change notifications */ +/**************************************/ +uint8_t qbman_result_SCN_state(const struct qbman_result *scn) +{ + return scn->scn.state; +} + +uint32_t qbman_result_SCN_rid(const struct qbman_result *scn) +{ + return scn->scn.rid_tok; +} + +uint64_t qbman_result_SCN_ctx(const struct qbman_result *scn) +{ + return scn->scn.ctx; +} + +/*****************/ +/* Parsing BPSCN */ +/*****************/ +uint16_t qbman_result_bpscn_bpid(const struct qbman_result *scn) +{ + return (uint16_t)qbman_result_SCN_rid(scn) & 0x3FFF; +} + +int qbman_result_bpscn_has_free_bufs(const struct qbman_result *scn) +{ + return !(int)(qbman_result_SCN_state(scn) & 0x1); +} + +int qbman_result_bpscn_is_depleted(const struct qbman_result *scn) +{ + return (int)(qbman_result_SCN_state(scn) & 0x2); +} + +int qbman_result_bpscn_is_surplus(const struct qbman_result *scn) +{ + return (int)(qbman_result_SCN_state(scn) & 0x4); +} + +uint64_t qbman_result_bpscn_ctx(const struct qbman_result *scn) +{ + return qbman_result_SCN_ctx(scn); +} + +/*****************/ +/* Parsing CGCU */ +/*****************/ +uint16_t qbman_result_cgcu_cgid(const struct qbman_result *scn) +{ + return (uint16_t)qbman_result_SCN_rid(scn) & 0xFFFF; +} + +uint64_t qbman_result_cgcu_icnt(const struct qbman_result *scn) +{ + return qbman_result_SCN_ctx(scn); +} + +/******************/ +/* Buffer release */ +/******************/ +#define QB_BR_RC_VALID_SHIFT 5 +#define QB_BR_RCDI_SHIFT 6 + +void qbman_release_desc_clear(struct qbman_release_desc *d) +{ + memset(d, 0, sizeof(*d)); + d->br.verb = 1 << QB_BR_RC_VALID_SHIFT; +} + +void qbman_release_desc_set_bpid(struct qbman_release_desc *d, uint16_t bpid) +{ + d->br.bpid = bpid; +} + +void qbman_release_desc_set_rcdi(struct qbman_release_desc *d, int enable) +{ + if (enable) + d->br.verb |= 1 << QB_BR_RCDI_SHIFT; + else + d->br.verb &= ~(1 << QB_BR_RCDI_SHIFT); +} + +#define RAR_IDX(rar) ((rar) & 0x7) +#define RAR_VB(rar) ((rar) & 0x80) +#define RAR_SUCCESS(rar) ((rar) & 0x100) + +int qbman_swp_release(struct qbman_swp *s, const struct qbman_release_desc *d, + const uint64_t *buffers, unsigned int num_buffers) +{ + uint32_t *p; + const uint32_t *cl = qb_cl(d); + uint32_t rar = qbman_cinh_read(&s->sys, QBMAN_CINH_SWP_RAR); + + pr_debug("RAR=%08x\n", rar); + if (!RAR_SUCCESS(rar)) + return -EBUSY; + + QBMAN_BUG_ON(!num_buffers || (num_buffers > 7)); + + /* Start the release command */ + p = qbman_cena_write_start_wo_shadow(&s->sys, + QBMAN_CENA_SWP_RCR(RAR_IDX(rar))); + + /* Copy the caller's buffer pointers to the command */ + u64_to_le32_copy(&p[2], buffers, num_buffers); + + /* Set the verb byte, have to substitute in the valid-bit and the number + * of buffers. + */ + lwsync(); + p[0] = cl[0] | RAR_VB(rar) | num_buffers; + qbman_cena_write_complete_wo_shadow(&s->sys, + QBMAN_CENA_SWP_RCR(RAR_IDX(rar))); + + return 0; +} + +/*******************/ +/* Buffer acquires */ +/*******************/ +struct qbman_acquire_desc { + uint8_t verb; + uint8_t reserved; + uint16_t bpid; + uint8_t num; + uint8_t reserved2[59]; +}; + +struct qbman_acquire_rslt { + uint8_t verb; + uint8_t rslt; + uint16_t reserved; + uint8_t num; + uint8_t reserved2[3]; + uint64_t buf[7]; +}; + +int qbman_swp_acquire(struct qbman_swp *s, uint16_t bpid, uint64_t *buffers, + unsigned int num_buffers) +{ + struct qbman_acquire_desc *p; + struct qbman_acquire_rslt *r; + + if (!num_buffers || (num_buffers > 7)) + return -EINVAL; + + /* Start the management command */ + p = qbman_swp_mc_start(s); + + if (!p) + return -EBUSY; + + /* Encode the caller-provided attributes */ + p->bpid = bpid; + p->num = num_buffers; + + /* Complete the management command */ + r = qbman_swp_mc_complete(s, p, QBMAN_MC_ACQUIRE); + if (unlikely(!r)) { + pr_err("qbman: acquire from BPID %d failed, no response\n", + bpid); + return -EIO; + } + + /* Decode the outcome */ + QBMAN_BUG_ON((r->verb & QBMAN_RESPONSE_VERB_MASK) != QBMAN_MC_ACQUIRE); + + /* Determine success or failure */ + if (unlikely(r->rslt != QBMAN_MC_RSLT_OK)) { + pr_err("Acquire buffers from BPID 0x%x failed, code=0x%02x\n", + bpid, r->rslt); + return -EIO; + } + + QBMAN_BUG_ON(r->num > num_buffers); + + /* Copy the acquired buffers to the caller's array */ + u64_from_le32_copy(buffers, &r->buf[0], r->num); + + return (int)r->num; +} + +/*****************/ +/* FQ management */ +/*****************/ +struct qbman_alt_fq_state_desc { + uint8_t verb; + uint8_t reserved[3]; + uint32_t fqid; + uint8_t reserved2[56]; +}; + +struct qbman_alt_fq_state_rslt { + uint8_t verb; + uint8_t rslt; + uint8_t reserved[62]; +}; + +#define ALT_FQ_FQID_MASK 0x00FFFFFF + +static int qbman_swp_alt_fq_state(struct qbman_swp *s, uint32_t fqid, + uint8_t alt_fq_verb) +{ + struct qbman_alt_fq_state_desc *p; + struct qbman_alt_fq_state_rslt *r; + + /* Start the management command */ + p = qbman_swp_mc_start(s); + if (!p) + return -EBUSY; + + p->fqid = fqid & ALT_FQ_FQID_MASK; + + /* Complete the management command */ + r = qbman_swp_mc_complete(s, p, alt_fq_verb); + if (unlikely(!r)) { + pr_err("qbman: mgmt cmd failed, no response (verb=0x%x)\n", + alt_fq_verb); + return -EIO; + } + + /* Decode the outcome */ + QBMAN_BUG_ON((r->verb & QBMAN_RESPONSE_VERB_MASK) != alt_fq_verb); + + /* Determine success or failure */ + if (unlikely(r->rslt != QBMAN_MC_RSLT_OK)) { + pr_err("ALT FQID %d failed: verb = 0x%08x, code = 0x%02x\n", + fqid, alt_fq_verb, r->rslt); + return -EIO; + } + + return 0; +} + +int qbman_swp_fq_schedule(struct qbman_swp *s, uint32_t fqid) +{ + return qbman_swp_alt_fq_state(s, fqid, QBMAN_FQ_SCHEDULE); +} + +int qbman_swp_fq_force(struct qbman_swp *s, uint32_t fqid) +{ + return qbman_swp_alt_fq_state(s, fqid, QBMAN_FQ_FORCE); +} + +int qbman_swp_fq_xon(struct qbman_swp *s, uint32_t fqid) +{ + return qbman_swp_alt_fq_state(s, fqid, QBMAN_FQ_XON); +} + +int qbman_swp_fq_xoff(struct qbman_swp *s, uint32_t fqid) +{ + return qbman_swp_alt_fq_state(s, fqid, QBMAN_FQ_XOFF); +} + +/**********************/ +/* Channel management */ +/**********************/ + +struct qbman_cdan_ctrl_desc { + uint8_t verb; + uint8_t reserved; + uint16_t ch; + uint8_t we; + uint8_t ctrl; + uint16_t reserved2; + uint64_t cdan_ctx; + uint8_t reserved3[48]; + +}; + +struct qbman_cdan_ctrl_rslt { + uint8_t verb; + uint8_t rslt; + uint16_t ch; + uint8_t reserved[60]; +}; + +/* Hide "ICD" for now as we don't use it, don't set it, and don't test it, so it + * would be irresponsible to expose it. + */ +#define CODE_CDAN_WE_EN 0x1 +#define CODE_CDAN_WE_CTX 0x4 + +static int qbman_swp_CDAN_set(struct qbman_swp *s, uint16_t channelid, + uint8_t we_mask, uint8_t cdan_en, + uint64_t ctx) +{ + struct qbman_cdan_ctrl_desc *p; + struct qbman_cdan_ctrl_rslt *r; + + /* Start the management command */ + p = qbman_swp_mc_start(s); + if (!p) + return -EBUSY; + + /* Encode the caller-provided attributes */ + p->ch = channelid; + p->we = we_mask; + if (cdan_en) + p->ctrl = 1; + else + p->ctrl = 0; + p->cdan_ctx = ctx; + + /* Complete the management command */ + r = qbman_swp_mc_complete(s, p, QBMAN_WQCHAN_CONFIGURE); + if (unlikely(!r)) { + pr_err("qbman: wqchan config failed, no response\n"); + return -EIO; + } + + /* Decode the outcome */ + QBMAN_BUG_ON((r->verb & QBMAN_RESPONSE_VERB_MASK) + != QBMAN_WQCHAN_CONFIGURE); + + /* Determine success or failure */ + if (unlikely(r->rslt != QBMAN_MC_RSLT_OK)) { + pr_err("CDAN cQID %d failed: code = 0x%02x\n", + channelid, r->rslt); + return -EIO; + } + + return 0; +} + +int qbman_swp_CDAN_set_context(struct qbman_swp *s, uint16_t channelid, + uint64_t ctx) +{ + return qbman_swp_CDAN_set(s, channelid, + CODE_CDAN_WE_CTX, + 0, ctx); +} + +int qbman_swp_CDAN_enable(struct qbman_swp *s, uint16_t channelid) +{ + return qbman_swp_CDAN_set(s, channelid, + CODE_CDAN_WE_EN, + 1, 0); +} + +int qbman_swp_CDAN_disable(struct qbman_swp *s, uint16_t channelid) +{ + return qbman_swp_CDAN_set(s, channelid, + CODE_CDAN_WE_EN, + 0, 0); +} + +int qbman_swp_CDAN_set_context_enable(struct qbman_swp *s, uint16_t channelid, + uint64_t ctx) +{ + return qbman_swp_CDAN_set(s, channelid, + CODE_CDAN_WE_EN | CODE_CDAN_WE_CTX, + 1, ctx); +} + +uint8_t qbman_get_dqrr_idx(const struct qbman_result *dqrr) +{ + return QBMAN_IDX_FROM_DQRR(dqrr); +} + +struct qbman_result *qbman_get_dqrr_from_idx(struct qbman_swp *s, uint8_t idx) +{ + struct qbman_result *dq; + + dq = qbman_cena_read(&s->sys, QBMAN_CENA_SWP_DQRR(idx)); + return dq; +} diff --git a/dpdk/drivers/bus/fslmc/qbman/qbman_portal.h b/dpdk/drivers/bus/fslmc/qbman/qbman_portal.h new file mode 100644 index 00000000..d9f3ed7e --- /dev/null +++ b/dpdk/drivers/bus/fslmc/qbman/qbman_portal.h @@ -0,0 +1,167 @@ +/*- + * BSD LICENSE + * + * Copyright (C) 2014-2016 Freescale Semiconductor, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Freescale Semiconductor nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "qbman_sys.h" +#include + +uint32_t qman_version; +#define QMAN_REV_4000 0x04000000 +#define QMAN_REV_4100 0x04010000 +#define QMAN_REV_4101 0x04010001 + +/* All QBMan command and result structures use this "valid bit" encoding */ +#define QB_VALID_BIT ((uint32_t)0x80) + +/* Management command result codes */ +#define QBMAN_MC_RSLT_OK 0xf0 + +/* QBMan DQRR size is set at runtime in qbman_portal.c */ + +#define QBMAN_EQCR_SIZE 8 + +static inline uint8_t qm_cyc_diff(uint8_t ringsize, uint8_t first, + uint8_t last) +{ + /* 'first' is included, 'last' is excluded */ + if (first <= last) + return last - first; + return (2 * ringsize) + last - first; +} + +/* --------------------- */ +/* portal data structure */ +/* --------------------- */ + +struct qbman_swp { + struct qbman_swp_desc desc; + /* The qbman_sys (ie. arch/OS-specific) support code can put anything it + * needs in here. + */ + struct qbman_swp_sys sys; + /* Management commands */ + struct { +#ifdef QBMAN_CHECKING + enum swp_mc_check { + swp_mc_can_start, /* call __qbman_swp_mc_start() */ + swp_mc_can_submit, /* call __qbman_swp_mc_submit() */ + swp_mc_can_poll, /* call __qbman_swp_mc_result() */ + } check; +#endif + uint32_t valid_bit; /* 0x00 or 0x80 */ + } mc; + /* Push dequeues */ + uint32_t sdq; + /* Volatile dequeues */ + struct { + /* VDQCR supports a "1 deep pipeline", meaning that if you know + * the last-submitted command is already executing in the + * hardware (as evidenced by at least 1 valid dequeue result), + * you can write another dequeue command to the register, the + * hardware will start executing it as soon as the + * already-executing command terminates. (This minimises latency + * and stalls.) With that in mind, this "busy" variable refers + * to whether or not a command can be submitted, not whether or + * not a previously-submitted command is still executing. In + * other words, once proof is seen that the previously-submitted + * command is executing, "vdq" is no longer "busy". + */ + atomic_t busy; + uint32_t valid_bit; /* 0x00 or 0x80 */ + /* We need to determine when vdq is no longer busy. This depends + * on whether the "busy" (last-submitted) dequeue command is + * targeting DQRR or main-memory, and detected is based on the + * presence of the dequeue command's "token" showing up in + * dequeue entries in DQRR or main-memory (respectively). + */ + struct qbman_result *storage; /* NULL if DQRR */ + } vdq; + /* DQRR */ + struct { + uint32_t next_idx; + uint32_t valid_bit; + uint8_t dqrr_size; + int reset_bug; + } dqrr; + struct { + uint32_t pi; + uint32_t pi_vb; + uint32_t ci; + int available; + } eqcr; +}; + +/* -------------------------- */ +/* portal management commands */ +/* -------------------------- */ + +/* Different management commands all use this common base layer of code to issue + * commands and poll for results. The first function returns a pointer to where + * the caller should fill in their MC command (though they should ignore the + * verb byte), the second function commits merges in the caller-supplied command + * verb (which should not include the valid-bit) and submits the command to + * hardware, and the third function checks for a completed response (returns + * non-NULL if only if the response is complete). + */ +void *qbman_swp_mc_start(struct qbman_swp *p); +void qbman_swp_mc_submit(struct qbman_swp *p, void *cmd, uint8_t cmd_verb); +void *qbman_swp_mc_result(struct qbman_swp *p); + +/* Wraps up submit + poll-for-result */ +static inline void *qbman_swp_mc_complete(struct qbman_swp *swp, void *cmd, + uint8_t cmd_verb) +{ + int loopvar = 1000; + + qbman_swp_mc_submit(swp, cmd, cmd_verb); + do { + cmd = qbman_swp_mc_result(swp); + } while (!cmd && loopvar--); + QBMAN_BUG_ON(!loopvar); + + return cmd; +} + +/* ---------------------- */ +/* Descriptors/cachelines */ +/* ---------------------- */ + +/* To avoid needless dynamic allocation, the driver API often gives the caller + * a "descriptor" type that the caller can instantiate however they like. + * Ultimately though, it is just a cacheline of binary storage (or something + * smaller when it is known that the descriptor doesn't need all 64 bytes) for + * holding pre-formatted pieces of hardware commands. The performance-critical + * code can then copy these descriptors directly into hardware command + * registers more efficiently than trying to construct/format commands + * on-the-fly. The API user sees the descriptor as an array of 32-bit words in + * order for the compiler to know its size, but the internal details are not + * exposed. The following macro is used within the driver for converting *any* + * descriptor pointer to a usable array pointer. The use of a macro (instead of + * an inline) is necessary to work with different descriptor types and to work + * correctly with const and non-const inputs (and similarly-qualified outputs). + */ +#define qb_cl(d) (&(d)->donot_manipulate_directly[0]) diff --git a/dpdk/drivers/bus/fslmc/qbman/qbman_sys.h b/dpdk/drivers/bus/fslmc/qbman/qbman_sys.h new file mode 100644 index 00000000..c216e9cf --- /dev/null +++ b/dpdk/drivers/bus/fslmc/qbman/qbman_sys.h @@ -0,0 +1,376 @@ +/*- + * BSD LICENSE + * + * Copyright (C) 2014-2016 Freescale Semiconductor, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Freescale Semiconductor nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +/* qbman_sys_decl.h and qbman_sys.h are the two platform-specific files in the + * driver. They are only included via qbman_private.h, which is itself a + * platform-independent file and is included by all the other driver source. + * + * qbman_sys_decl.h is included prior to all other declarations and logic, and + * it exists to provide compatibility with any linux interfaces our + * single-source driver code is dependent on (eg. kmalloc). Ie. this file + * provides linux compatibility. + * + * This qbman_sys.h header, on the other hand, is included *after* any common + * and platform-neutral declarations and logic in qbman_private.h, and exists to + * implement any platform-specific logic of the qbman driver itself. Ie. it is + * *not* to provide linux compatibility. + */ + +#include "qbman_sys_decl.h" + +/* Debugging assists */ +static inline void __hexdump(unsigned long start, unsigned long end, + unsigned long p, size_t sz, const unsigned char *c) +{ + while (start < end) { + unsigned int pos = 0; + char buf[64]; + int nl = 0; + + pos += sprintf(buf + pos, "%08lx: ", start); + do { + if ((start < p) || (start >= (p + sz))) + pos += sprintf(buf + pos, ".."); + else + pos += sprintf(buf + pos, "%02x", *(c++)); + if (!(++start & 15)) { + buf[pos++] = '\n'; + nl = 1; + } else { + nl = 0; + if (!(start & 1)) + buf[pos++] = ' '; + if (!(start & 3)) + buf[pos++] = ' '; + } + } while (start & 15); + if (!nl) + buf[pos++] = '\n'; + buf[pos] = '\0'; + pr_info("%s", buf); + } +} + +static inline void hexdump(const void *ptr, size_t sz) +{ + unsigned long p = (unsigned long)ptr; + unsigned long start = p & ~15; + unsigned long end = (p + sz + 15) & ~15; + const unsigned char *c = ptr; + + __hexdump(start, end, p, sz, c); +} + +/* Currently, the CENA support code expects each 32-bit word to be written in + * host order, and these are converted to hardware (little-endian) order on + * command submission. However, 64-bit quantities are must be written (and read) + * as two 32-bit words with the least-significant word first, irrespective of + * host endianness. + */ +static inline void u64_to_le32_copy(void *d, const uint64_t *s, + unsigned int cnt) +{ + uint32_t *dd = d; + const uint32_t *ss = (const uint32_t *)s; + + while (cnt--) { + /* TBD: the toolchain was choking on the use of 64-bit types up + * until recently so this works entirely with 32-bit variables. + * When 64-bit types become usable again, investigate better + * ways of doing this. + */ +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + *(dd++) = ss[1]; + *(dd++) = ss[0]; + ss += 2; +#else + *(dd++) = *(ss++); + *(dd++) = *(ss++); +#endif + } +} + +static inline void u64_from_le32_copy(uint64_t *d, const void *s, + unsigned int cnt) +{ + const uint32_t *ss = s; + uint32_t *dd = (uint32_t *)d; + + while (cnt--) { +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + dd[1] = *(ss++); + dd[0] = *(ss++); + dd += 2; +#else + *(dd++) = *(ss++); + *(dd++) = *(ss++); +#endif + } +} + + /******************/ + /* Portal access */ + /******************/ +struct qbman_swp_sys { + /* On GPP, the sys support for qbman_swp is here. The CENA region isi + * not an mmap() of the real portal registers, but an allocated + * place-holder, because the actual writes/reads to/from the portal are + * marshalled from these allocated areas using QBMan's "MC access + * registers". CINH accesses are atomic so there's no need for a + * place-holder. + */ + uint8_t *cena; + uint8_t __iomem *addr_cena; + uint8_t __iomem *addr_cinh; + uint32_t idx; + enum qbman_eqcr_mode eqcr_mode; +}; + +/* P_OFFSET is (ACCESS_CMD,0,12) - offset within the portal + * C is (ACCESS_CMD,12,1) - is inhibited? (0==CENA, 1==CINH) + * SWP_IDX is (ACCESS_CMD,16,10) - Software portal index + * P is (ACCESS_CMD,28,1) - (0==special portal, 1==any portal) + * T is (ACCESS_CMD,29,1) - Command type (0==READ, 1==WRITE) + * E is (ACCESS_CMD,31,1) - Command execute (1 to issue, poll for 0==complete) + */ + +static inline void qbman_cinh_write(struct qbman_swp_sys *s, uint32_t offset, + uint32_t val) +{ + __raw_writel(val, s->addr_cinh + offset); +#ifdef QBMAN_CINH_TRACE + pr_info("qbman_cinh_write(%p:%d:0x%03x) 0x%08x\n", + s->addr_cinh, s->idx, offset, val); +#endif +} + +static inline uint32_t qbman_cinh_read(struct qbman_swp_sys *s, uint32_t offset) +{ + uint32_t reg = __raw_readl(s->addr_cinh + offset); +#ifdef QBMAN_CINH_TRACE + pr_info("qbman_cinh_read(%p:%d:0x%03x) 0x%08x\n", + s->addr_cinh, s->idx, offset, reg); +#endif + return reg; +} + +static inline void *qbman_cena_write_start(struct qbman_swp_sys *s, + uint32_t offset) +{ + void *shadow = s->cena + offset; + +#ifdef QBMAN_CENA_TRACE + pr_info("qbman_cena_write_start(%p:%d:0x%03x) %p\n", + s->addr_cena, s->idx, offset, shadow); +#endif + QBMAN_BUG_ON(offset & 63); + dcbz(shadow); + return shadow; +} + +static inline void *qbman_cena_write_start_wo_shadow(struct qbman_swp_sys *s, + uint32_t offset) +{ +#ifdef QBMAN_CENA_TRACE + pr_info("qbman_cena_write_start(%p:%d:0x%03x)\n", + s->addr_cena, s->idx, offset); +#endif + QBMAN_BUG_ON(offset & 63); + return (s->addr_cena + offset); +} + +static inline void qbman_cena_write_complete(struct qbman_swp_sys *s, + uint32_t offset, void *cmd) +{ + const uint32_t *shadow = cmd; + int loop; +#ifdef QBMAN_CENA_TRACE + pr_info("qbman_cena_write_complete(%p:%d:0x%03x) %p\n", + s->addr_cena, s->idx, offset, shadow); + hexdump(cmd, 64); +#endif + for (loop = 15; loop >= 1; loop--) + __raw_writel(shadow[loop], s->addr_cena + + offset + loop * 4); + lwsync(); + __raw_writel(shadow[0], s->addr_cena + offset); + dcbf(s->addr_cena + offset); +} + +static inline void qbman_cena_write_complete_wo_shadow(struct qbman_swp_sys *s, + uint32_t offset) +{ +#ifdef QBMAN_CENA_TRACE + pr_info("qbman_cena_write_complete(%p:%d:0x%03x)\n", + s->addr_cena, s->idx, offset); +#endif + dcbf(s->addr_cena + offset); +} + +static inline uint32_t qbman_cena_read_reg(struct qbman_swp_sys *s, + uint32_t offset) +{ + return __raw_readl(s->addr_cena + offset); +} + +static inline void *qbman_cena_read(struct qbman_swp_sys *s, uint32_t offset) +{ + uint32_t *shadow = (uint32_t *)(s->cena + offset); + unsigned int loop; +#ifdef QBMAN_CENA_TRACE + pr_info("qbman_cena_read(%p:%d:0x%03x) %p\n", + s->addr_cena, s->idx, offset, shadow); +#endif + + for (loop = 0; loop < 16; loop++) + shadow[loop] = __raw_readl(s->addr_cena + offset + + loop * 4); +#ifdef QBMAN_CENA_TRACE + hexdump(shadow, 64); +#endif + return shadow; +} + +static inline void *qbman_cena_read_wo_shadow(struct qbman_swp_sys *s, + uint32_t offset) +{ +#ifdef QBMAN_CENA_TRACE + pr_info("qbman_cena_read(%p:%d:0x%03x)\n", + s->addr_cena, s->idx, offset); +#endif + return s->addr_cena + offset; +} + +static inline void qbman_cena_invalidate(struct qbman_swp_sys *s, + uint32_t offset) +{ + dccivac(s->addr_cena + offset); +} + +static inline void qbman_cena_invalidate_prefetch(struct qbman_swp_sys *s, + uint32_t offset) +{ + dccivac(s->addr_cena + offset); + prefetch_for_load(s->addr_cena + offset); +} + +static inline void qbman_cena_prefetch(struct qbman_swp_sys *s, + uint32_t offset) +{ + prefetch_for_load(s->addr_cena + offset); +} + + /******************/ + /* Portal support */ + /******************/ + +/* The SWP_CFG portal register is special, in that it is used by the + * platform-specific code rather than the platform-independent code in + * qbman_portal.c. So use of it is declared locally here. + */ +#define QBMAN_CINH_SWP_CFG 0xd00 +#define QBMAN_CINH_SWP_CFG 0xd00 +#define SWP_CFG_DQRR_MF_SHIFT 20 +#define SWP_CFG_EST_SHIFT 16 +#define SWP_CFG_WN_SHIFT 14 +#define SWP_CFG_RPM_SHIFT 12 +#define SWP_CFG_DCM_SHIFT 10 +#define SWP_CFG_EPM_SHIFT 8 +#define SWP_CFG_SD_SHIFT 5 +#define SWP_CFG_SP_SHIFT 4 +#define SWP_CFG_SE_SHIFT 3 +#define SWP_CFG_DP_SHIFT 2 +#define SWP_CFG_DE_SHIFT 1 +#define SWP_CFG_EP_SHIFT 0 + +static inline uint32_t qbman_set_swp_cfg(uint8_t max_fill, uint8_t wn, + uint8_t est, uint8_t rpm, uint8_t dcm, + uint8_t epm, int sd, int sp, int se, + int dp, int de, int ep) +{ + uint32_t reg; + + reg = (max_fill << SWP_CFG_DQRR_MF_SHIFT | + est << SWP_CFG_EST_SHIFT | + wn << SWP_CFG_WN_SHIFT | + rpm << SWP_CFG_RPM_SHIFT | + dcm << SWP_CFG_DCM_SHIFT | + epm << SWP_CFG_EPM_SHIFT | + sd << SWP_CFG_SD_SHIFT | + sp << SWP_CFG_SP_SHIFT | + se << SWP_CFG_SE_SHIFT | + dp << SWP_CFG_DP_SHIFT | + de << SWP_CFG_DE_SHIFT | + ep << SWP_CFG_EP_SHIFT); + + return reg; +} + +static inline int qbman_swp_sys_init(struct qbman_swp_sys *s, + const struct qbman_swp_desc *d, + uint8_t dqrr_size) +{ + uint32_t reg; + + s->addr_cena = d->cena_bar; + s->addr_cinh = d->cinh_bar; + s->idx = (uint32_t)d->idx; + s->cena = malloc(4096); + if (!s->cena) { + pr_err("Could not allocate page for cena shadow\n"); + return -1; + } + s->eqcr_mode = d->eqcr_mode; + QBMAN_BUG_ON(d->idx < 0); +#ifdef QBMAN_CHECKING + /* We should never be asked to initialise for a portal that isn't in + * the power-on state. (Ie. don't forget to reset portals when they are + * decommissioned!) + */ + reg = qbman_cinh_read(s, QBMAN_CINH_SWP_CFG); + QBMAN_BUG_ON(reg); +#endif + if (s->eqcr_mode == qman_eqcr_vb_array) + reg = qbman_set_swp_cfg(dqrr_size, 0, 0, 3, 2, 3, 1, 1, 1, 1, + 1, 1); + else + reg = qbman_set_swp_cfg(dqrr_size, 0, 2, 3, 2, 2, 1, 1, 1, 1, + 1, 1); + qbman_cinh_write(s, QBMAN_CINH_SWP_CFG, reg); + reg = qbman_cinh_read(s, QBMAN_CINH_SWP_CFG); + if (!reg) { + pr_err("The portal %d is not enabled!\n", s->idx); + free(s->cena); + return -1; + } + return 0; +} + +static inline void qbman_swp_sys_finish(struct qbman_swp_sys *s) +{ + free(s->cena); +} diff --git a/dpdk/drivers/bus/fslmc/qbman/qbman_sys_decl.h b/dpdk/drivers/bus/fslmc/qbman/qbman_sys_decl.h new file mode 100644 index 00000000..e1125cfe --- /dev/null +++ b/dpdk/drivers/bus/fslmc/qbman/qbman_sys_decl.h @@ -0,0 +1,52 @@ +/*- + * BSD LICENSE + * + * Copyright (C) 2014-2016 Freescale Semiconductor, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Freescale Semiconductor nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include +#include + +/* Sanity check */ +#if (__BYTE_ORDER__ != __ORDER_BIG_ENDIAN__) && \ + (__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__) +#error "Unknown endianness!" +#endif + + /****************/ + /* arch assists */ + /****************/ +#define dcbz(p) { asm volatile("dc zva, %0" : : "r" (p) : "memory"); } +#define lwsync() { asm volatile("dmb st" : : : "memory"); } +#define dcbf(p) { asm volatile("dc cvac, %0" : : "r"(p) : "memory"); } +#define dccivac(p) { asm volatile("dc civac, %0" : : "r"(p) : "memory"); } +static inline void prefetch_for_load(void *p) +{ + asm volatile("prfm pldl1keep, [%0, #0]" : : "r" (p)); +} + +static inline void prefetch_for_store(void *p) +{ + asm volatile("prfm pstl1keep, [%0, #0]" : : "r" (p)); +} diff --git a/dpdk/drivers/bus/fslmc/rte_bus_fslmc_version.map b/dpdk/drivers/bus/fslmc/rte_bus_fslmc_version.map new file mode 100644 index 00000000..51a2ac69 --- /dev/null +++ b/dpdk/drivers/bus/fslmc/rte_bus_fslmc_version.map @@ -0,0 +1,91 @@ +DPDK_17.05 { + global: + + dpaa2_affine_qbman_swp; + dpaa2_affine_qbman_swp_sec; + dpaa2_alloc_dpbp_dev; + dpaa2_alloc_dq_storage; + dpaa2_free_dpbp_dev; + dpaa2_free_dq_storage; + dpbp_disable; + dpbp_enable; + dpbp_get_attributes; + dpbp_get_num_free_bufs; + dpbp_open; + dpbp_reset; + dpio_close; + dpio_disable; + dpio_enable; + dpio_get_attributes; + dpio_open; + dpio_reset; + dpio_set_stashing_destination; + mc_send_command; + per_lcore__dpaa2_io; + qbman_check_command_complete; + qbman_eq_desc_clear; + qbman_eq_desc_set_fq; + qbman_eq_desc_set_no_orp; + qbman_eq_desc_set_qd; + qbman_eq_desc_set_response; + qbman_get_version; + qbman_pull_desc_clear; + qbman_pull_desc_set_fq; + qbman_pull_desc_set_numframes; + qbman_pull_desc_set_storage; + qbman_release_desc_clear; + qbman_release_desc_set_bpid; + qbman_result_DQ_fd; + qbman_result_DQ_flags; + qbman_result_has_new_result; + qbman_swp_acquire; + qbman_swp_pull; + qbman_swp_release; + qbman_swp_send_multiple; + rte_fslmc_driver_register; + rte_fslmc_driver_unregister; + rte_fslmc_vfio_dmamap; + rte_mcp_ptr_list; + + local: *; +}; + +DPDK_17.08 { + global: + + dpaa2_io_portal; + dpaa2_get_qbman_swp; + dpci_set_rx_queue; + dpcon_open; + dpcon_get_attributes; + dpio_add_static_dequeue_channel; + dpio_remove_static_dequeue_channel; + mc_get_soc_version; + mc_get_version; + qbman_check_new_result; + qbman_eq_desc_set_dca; + qbman_get_dqrr_from_idx; + qbman_get_dqrr_idx; + qbman_result_DQ_fqd_ctx; + qbman_result_SCN_state; + qbman_swp_dqrr_consume; + qbman_swp_dqrr_next; + qbman_swp_enqueue_multiple; + qbman_swp_enqueue_multiple_desc; + qbman_swp_interrupt_clear_status; + qbman_swp_push_set; + rte_dpaa2_alloc_dpci_dev; + rte_fslmc_object_register; + rte_global_active_dqs_list; + +} DPDK_17.05; + +DPDK_17.11 { + global: + + dpaa2_dpbp_supported; + rte_dpaa2_dev_type; + rte_dpaa2_intr_disable; + rte_dpaa2_intr_enable; + +} DPDK_17.08; diff --git a/dpdk/drivers/bus/fslmc/rte_fslmc.h b/dpdk/drivers/bus/fslmc/rte_fslmc.h new file mode 100644 index 00000000..0814e69b --- /dev/null +++ b/dpdk/drivers/bus/fslmc/rte_fslmc.h @@ -0,0 +1,201 @@ +/*- + * BSD LICENSE + * + * Copyright 2016 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of NXP nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_FSLMC_H_ +#define _RTE_FSLMC_H_ + +/** + * @file + * + * RTE FSLMC Bus Interface + */ + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include + +#define FSLMC_OBJECT_MAX_LEN 32 /**< Length of each device on bus */ + +struct rte_dpaa2_driver; + +/* DPAA2 Device and Driver lists for FSLMC bus */ +TAILQ_HEAD(rte_fslmc_device_list, rte_dpaa2_device); +TAILQ_HEAD(rte_fslmc_driver_list, rte_dpaa2_driver); + +extern struct rte_fslmc_bus rte_fslmc_bus; + +enum rte_dpaa2_dev_type { + /* Devices backed by DPDK driver */ + DPAA2_ETH, /**< DPNI type device*/ + DPAA2_CRYPTO, /**< DPSECI type device */ + DPAA2_CON, /**< DPCONC type device */ + /* Devices not backed by a DPDK driver: DPIO, DPBP, DPCI, DPMCP */ + DPAA2_BPOOL, /**< DPBP type device */ + DPAA2_IO, /**< DPIO type device */ + DPAA2_CI, /**< DPCI type device */ + DPAA2_MPORTAL, /**< DPMCP type device */ + /* Unknown device placeholder */ + DPAA2_UNKNOWN +}; + +TAILQ_HEAD(rte_dpaa2_object_list, rte_dpaa2_object); + +typedef int (*rte_dpaa2_obj_create_t)(int vdev_fd, + struct vfio_device_info *obj_info, + int object_id); + +/** + * A structure describing a DPAA2 object. + */ +struct rte_dpaa2_object { + TAILQ_ENTRY(rte_dpaa2_object) next; /**< Next in list. */ + const char *name; /**< Name of Object. */ + enum rte_dpaa2_dev_type dev_type; /**< Type of device */ + rte_dpaa2_obj_create_t create; +}; + +/** + * A structure describing a DPAA2 device. + */ +struct rte_dpaa2_device { + TAILQ_ENTRY(rte_dpaa2_device) next; /**< Next probed DPAA2 device. */ + struct rte_device device; /**< Inherit core device */ + union { + struct rte_eth_dev *eth_dev; /**< ethernet device */ + struct rte_cryptodev *cryptodev; /**< Crypto Device */ + }; + enum rte_dpaa2_dev_type dev_type; /**< Device Type */ + uint16_t object_id; /**< DPAA2 Object ID */ + struct rte_intr_handle intr_handle; /**< Interrupt handle */ + struct rte_dpaa2_driver *driver; /**< Associated driver */ + char name[FSLMC_OBJECT_MAX_LEN]; /**< DPAA2 Object name*/ +}; + +typedef int (*rte_dpaa2_probe_t)(struct rte_dpaa2_driver *dpaa2_drv, + struct rte_dpaa2_device *dpaa2_dev); +typedef int (*rte_dpaa2_remove_t)(struct rte_dpaa2_device *dpaa2_dev); + +/** + * A structure describing a DPAA2 driver. + */ +struct rte_dpaa2_driver { + TAILQ_ENTRY(rte_dpaa2_driver) next; /**< Next in list. */ + struct rte_driver driver; /**< Inherit core driver. */ + struct rte_fslmc_bus *fslmc_bus; /**< FSLMC bus reference */ + uint32_t drv_flags; /**< Flags for controlling device.*/ + enum rte_dpaa2_dev_type drv_type; /**< Driver Type */ + rte_dpaa2_probe_t probe; + rte_dpaa2_remove_t remove; +}; + +/* + * FSLMC bus + */ +struct rte_fslmc_bus { + struct rte_bus bus; /**< Generic Bus object */ + struct rte_fslmc_device_list device_list; + /**< FSLMC DPAA2 Device list */ + struct rte_fslmc_driver_list driver_list; + /**< FSLMC DPAA2 Driver list */ + int device_count; + /**< Optional: Count of devices on bus */ +}; + +/** + * Register a DPAA2 driver. + * + * @param driver + * A pointer to a rte_dpaa2_driver structure describing the driver + * to be registered. + */ +void rte_fslmc_driver_register(struct rte_dpaa2_driver *driver); + +/** + * Unregister a DPAA2 driver. + * + * @param driver + * A pointer to a rte_dpaa2_driver structure describing the driver + * to be unregistered. + */ +void rte_fslmc_driver_unregister(struct rte_dpaa2_driver *driver); + +/** Helper for DPAA2 device registration from driver (eth, crypto) instance */ +#define RTE_PMD_REGISTER_DPAA2(nm, dpaa2_drv) \ +RTE_INIT(dpaa2initfn_ ##nm); \ +static void dpaa2initfn_ ##nm(void) \ +{\ + (dpaa2_drv).driver.name = RTE_STR(nm);\ + rte_fslmc_driver_register(&dpaa2_drv); \ +} \ +RTE_PMD_EXPORT_NAME(nm, __COUNTER__) + +/** + * Register a DPAA2 MC Object driver. + * + * @param mc_object + * A pointer to a rte_dpaa_object structure describing the mc object + * to be registered. + */ +void rte_fslmc_object_register(struct rte_dpaa2_object *object); + +/** Helper for DPAA2 object registration */ +#define RTE_PMD_REGISTER_DPAA2_OBJECT(nm, dpaa2_obj) \ +RTE_INIT(dpaa2objinitfn_ ##nm); \ +static void dpaa2objinitfn_ ##nm(void) \ +{\ + (dpaa2_obj).name = RTE_STR(nm);\ + rte_fslmc_object_register(&dpaa2_obj); \ +} \ +RTE_PMD_EXPORT_NAME(nm, __COUNTER__) + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_FSLMC_H_ */ diff --git a/dpdk/drivers/bus/pci/Makefile b/dpdk/drivers/bus/pci/Makefile new file mode 100644 index 00000000..f3df1c4c --- /dev/null +++ b/dpdk/drivers/bus/pci/Makefile @@ -0,0 +1,62 @@ +# BSD LICENSE +# +# Copyright(c) 2017 6WIND S.A. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of 6WIND nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +include $(RTE_SDK)/mk/rte.vars.mk + +LIB = librte_bus_pci.a +LIBABIVER := 1 +EXPORT_MAP := rte_bus_pci_version.map + +CFLAGS := -I$(SRCDIR) $(CFLAGS) +CFLAGS += -O3 $(WERROR_FLAGS) + +ifneq ($(CONFIG_RTE_EXEC_ENV_LINUXAPP),) +SYSTEM := linux +endif +ifneq ($(CONFIG_RTE_EXEC_ENV_BSDAPP),) +SYSTEM := bsd +endif + +CFLAGS += -I$(RTE_SDK)/drivers/bus/pci/$(SYSTEM) +CFLAGS += -I$(RTE_SDK)/lib/librte_eal/common +CFLAGS += -I$(RTE_SDK)/lib/librte_eal/$(SYSTEM)app/eal + +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_ethdev -lrte_pci + +include $(RTE_SDK)/drivers/bus/pci/$(SYSTEM)/Makefile +SRCS-$(CONFIG_RTE_LIBRTE_PCI_BUS) := $(addprefix $(SYSTEM)/,$(SRCS)) +SRCS-$(CONFIG_RTE_LIBRTE_PCI_BUS) += pci_common.c +SRCS-$(CONFIG_RTE_LIBRTE_PCI_BUS) += pci_common_uio.c + +SYMLINK-$(CONFIG_RTE_LIBRTE_PCI_BUS)-include += rte_bus_pci.h + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/bus/pci/bsd/Makefile b/dpdk/drivers/bus/pci/bsd/Makefile new file mode 100644 index 00000000..4450913e --- /dev/null +++ b/dpdk/drivers/bus/pci/bsd/Makefile @@ -0,0 +1,32 @@ +# BSD LICENSE +# +# Copyright(c) 2017 6WIND S.A. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of 6WIND nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +SRCS += pci.c diff --git a/dpdk/drivers/bus/pci/bsd/pci.c b/dpdk/drivers/bus/pci/bsd/pci.c new file mode 100644 index 00000000..b8e21784 --- /dev/null +++ b/dpdk/drivers/bus/pci/bsd/pci.c @@ -0,0 +1,680 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(RTE_ARCH_X86) +#include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "eal_filesystem.h" +#include "private.h" + +/** + * @file + * PCI probing under BSD + * + * This code is used to simulate a PCI probe by parsing information in + * sysfs. Moreover, when a registered driver matches a device, the + * kernel driver currently using it is unloaded and replaced by + * igb_uio module, which is a very minimal userland driver for Intel + * network card, only providing access to PCI BAR to applications, and + * enabling bus master. + */ + +extern struct rte_pci_bus rte_pci_bus; + +/* Map pci device */ +int +rte_pci_map_device(struct rte_pci_device *dev) +{ + int ret = -1; + + /* try mapping the NIC resources */ + switch (dev->kdrv) { + case RTE_KDRV_NIC_UIO: + /* map resources for devices that use uio */ + ret = pci_uio_map_resource(dev); + break; + default: + RTE_LOG(DEBUG, EAL, + " Not managed by a supported kernel driver, skipped\n"); + ret = 1; + break; + } + + return ret; +} + +/* Unmap pci device */ +void +rte_pci_unmap_device(struct rte_pci_device *dev) +{ + /* try unmapping the NIC resources */ + switch (dev->kdrv) { + case RTE_KDRV_NIC_UIO: + /* unmap resources for devices that use uio */ + pci_uio_unmap_resource(dev); + break; + default: + RTE_LOG(DEBUG, EAL, + " Not managed by a supported kernel driver, skipped\n"); + break; + } +} + +void +pci_uio_free_resource(struct rte_pci_device *dev, + struct mapped_pci_resource *uio_res) +{ + rte_free(uio_res); + + if (dev->intr_handle.fd) { + close(dev->intr_handle.fd); + dev->intr_handle.fd = -1; + dev->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN; + } +} + +int +pci_uio_alloc_resource(struct rte_pci_device *dev, + struct mapped_pci_resource **uio_res) +{ + char devname[PATH_MAX]; /* contains the /dev/uioX */ + struct rte_pci_addr *loc; + + loc = &dev->addr; + + snprintf(devname, sizeof(devname), "/dev/uio@pci:%u:%u:%u", + dev->addr.bus, dev->addr.devid, dev->addr.function); + + if (access(devname, O_RDWR) < 0) { + RTE_LOG(WARNING, EAL, " "PCI_PRI_FMT" not managed by UIO driver, " + "skipping\n", loc->domain, loc->bus, loc->devid, loc->function); + return 1; + } + + /* save fd if in primary process */ + dev->intr_handle.fd = open(devname, O_RDWR); + if (dev->intr_handle.fd < 0) { + RTE_LOG(ERR, EAL, "Cannot open %s: %s\n", + devname, strerror(errno)); + goto error; + } + dev->intr_handle.type = RTE_INTR_HANDLE_UIO; + + /* allocate the mapping details for secondary processes*/ + *uio_res = rte_zmalloc("UIO_RES", sizeof(**uio_res), 0); + if (*uio_res == NULL) { + RTE_LOG(ERR, EAL, + "%s(): cannot store uio mmap details\n", __func__); + goto error; + } + + snprintf((*uio_res)->path, sizeof((*uio_res)->path), "%s", devname); + memcpy(&(*uio_res)->pci_addr, &dev->addr, sizeof((*uio_res)->pci_addr)); + + return 0; + +error: + pci_uio_free_resource(dev, *uio_res); + return -1; +} + +int +pci_uio_map_resource_by_index(struct rte_pci_device *dev, int res_idx, + struct mapped_pci_resource *uio_res, int map_idx) +{ + int fd; + char *devname; + void *mapaddr; + uint64_t offset; + uint64_t pagesz; + struct pci_map *maps; + + maps = uio_res->maps; + devname = uio_res->path; + pagesz = sysconf(_SC_PAGESIZE); + + /* allocate memory to keep path */ + maps[map_idx].path = rte_malloc(NULL, strlen(devname) + 1, 0); + if (maps[map_idx].path == NULL) { + RTE_LOG(ERR, EAL, "Cannot allocate memory for path: %s\n", + strerror(errno)); + return -1; + } + + /* + * open resource file, to mmap it + */ + fd = open(devname, O_RDWR); + if (fd < 0) { + RTE_LOG(ERR, EAL, "Cannot open %s: %s\n", + devname, strerror(errno)); + goto error; + } + + /* if matching map is found, then use it */ + offset = res_idx * pagesz; + mapaddr = pci_map_resource(NULL, fd, (off_t)offset, + (size_t)dev->mem_resource[res_idx].len, 0); + close(fd); + if (mapaddr == MAP_FAILED) + goto error; + + maps[map_idx].phaddr = dev->mem_resource[res_idx].phys_addr; + maps[map_idx].size = dev->mem_resource[res_idx].len; + maps[map_idx].addr = mapaddr; + maps[map_idx].offset = offset; + strcpy(maps[map_idx].path, devname); + dev->mem_resource[res_idx].addr = mapaddr; + + return 0; + +error: + rte_free(maps[map_idx].path); + return -1; +} + +static int +pci_scan_one(int dev_pci_fd, struct pci_conf *conf) +{ + struct rte_pci_device *dev; + struct pci_bar_io bar; + unsigned i, max; + + dev = malloc(sizeof(*dev)); + if (dev == NULL) { + return -1; + } + + memset(dev, 0, sizeof(*dev)); + dev->addr.domain = conf->pc_sel.pc_domain; + dev->addr.bus = conf->pc_sel.pc_bus; + dev->addr.devid = conf->pc_sel.pc_dev; + dev->addr.function = conf->pc_sel.pc_func; + + /* get vendor id */ + dev->id.vendor_id = conf->pc_vendor; + + /* get device id */ + dev->id.device_id = conf->pc_device; + + /* get subsystem_vendor id */ + dev->id.subsystem_vendor_id = conf->pc_subvendor; + + /* get subsystem_device id */ + dev->id.subsystem_device_id = conf->pc_subdevice; + + /* get class id */ + dev->id.class_id = (conf->pc_class << 16) | + (conf->pc_subclass << 8) | + (conf->pc_progif); + + /* TODO: get max_vfs */ + dev->max_vfs = 0; + + /* FreeBSD has no NUMA support (yet) */ + dev->device.numa_node = 0; + + pci_name_set(dev); + + /* FreeBSD has only one pass through driver */ + dev->kdrv = RTE_KDRV_NIC_UIO; + + /* parse resources */ + switch (conf->pc_hdr & PCIM_HDRTYPE) { + case PCIM_HDRTYPE_NORMAL: + max = PCIR_MAX_BAR_0; + break; + case PCIM_HDRTYPE_BRIDGE: + max = PCIR_MAX_BAR_1; + break; + case PCIM_HDRTYPE_CARDBUS: + max = PCIR_MAX_BAR_2; + break; + default: + goto skipdev; + } + + for (i = 0; i <= max; i++) { + bar.pbi_sel = conf->pc_sel; + bar.pbi_reg = PCIR_BAR(i); + if (ioctl(dev_pci_fd, PCIOCGETBAR, &bar) < 0) + continue; + + dev->mem_resource[i].len = bar.pbi_length; + if (PCI_BAR_IO(bar.pbi_base)) { + dev->mem_resource[i].addr = (void *)(bar.pbi_base & ~((uint64_t)0xf)); + continue; + } + dev->mem_resource[i].phys_addr = bar.pbi_base & ~((uint64_t)0xf); + } + + /* device is valid, add in list (sorted) */ + if (TAILQ_EMPTY(&rte_pci_bus.device_list)) { + rte_pci_add_device(dev); + } + else { + struct rte_pci_device *dev2 = NULL; + int ret; + + TAILQ_FOREACH(dev2, &rte_pci_bus.device_list, next) { + ret = rte_pci_addr_cmp(&dev->addr, &dev2->addr); + if (ret > 0) + continue; + else if (ret < 0) { + rte_pci_insert_device(dev2, dev); + } else { /* already registered */ + dev2->kdrv = dev->kdrv; + dev2->max_vfs = dev->max_vfs; + pci_name_set(dev2); + memmove(dev2->mem_resource, + dev->mem_resource, + sizeof(dev->mem_resource)); + free(dev); + } + return 0; + } + rte_pci_add_device(dev); + } + + return 0; + +skipdev: + free(dev); + return 0; +} + +/* + * Scan the content of the PCI bus, and add the devices in the devices + * list. Call pci_scan_one() for each pci entry found. + */ +int +rte_pci_scan(void) +{ + int fd; + unsigned dev_count = 0; + struct pci_conf matches[16]; + struct pci_conf_io conf_io = { + .pat_buf_len = 0, + .num_patterns = 0, + .patterns = NULL, + .match_buf_len = sizeof(matches), + .matches = &matches[0], + }; + + /* for debug purposes, PCI can be disabled */ + if (!rte_eal_has_pci()) + return 0; + + fd = open("/dev/pci", O_RDONLY); + if (fd < 0) { + RTE_LOG(ERR, EAL, "%s(): error opening /dev/pci\n", __func__); + goto error; + } + + do { + unsigned i; + if (ioctl(fd, PCIOCGETCONF, &conf_io) < 0) { + RTE_LOG(ERR, EAL, "%s(): error with ioctl on /dev/pci: %s\n", + __func__, strerror(errno)); + goto error; + } + + for (i = 0; i < conf_io.num_matches; i++) + if (pci_scan_one(fd, &matches[i]) < 0) + goto error; + + dev_count += conf_io.num_matches; + } while(conf_io.status == PCI_GETCONF_MORE_DEVS); + + close(fd); + + RTE_LOG(DEBUG, EAL, "PCI scan found %u devices\n", dev_count); + return 0; + +error: + if (fd >= 0) + close(fd); + return -1; +} + +/* + * Get iommu class of PCI devices on the bus. + */ +enum rte_iova_mode +rte_pci_get_iommu_class(void) +{ + /* Supports only RTE_KDRV_NIC_UIO */ + return RTE_IOVA_PA; +} + +int +pci_update_device(const struct rte_pci_addr *addr) +{ + int fd; + struct pci_conf matches[2]; + struct pci_match_conf match = { + .pc_sel = { + .pc_domain = addr->domain, + .pc_bus = addr->bus, + .pc_dev = addr->devid, + .pc_func = addr->function, + }, + }; + struct pci_conf_io conf_io = { + .pat_buf_len = 0, + .num_patterns = 1, + .patterns = &match, + .match_buf_len = sizeof(matches), + .matches = &matches[0], + }; + + fd = open("/dev/pci", O_RDONLY); + if (fd < 0) { + RTE_LOG(ERR, EAL, "%s(): error opening /dev/pci\n", __func__); + goto error; + } + + if (ioctl(fd, PCIOCGETCONF, &conf_io) < 0) { + RTE_LOG(ERR, EAL, "%s(): error with ioctl on /dev/pci: %s\n", + __func__, strerror(errno)); + goto error; + } + + if (conf_io.num_matches != 1) + goto error; + + if (pci_scan_one(fd, &matches[0]) < 0) + goto error; + + close(fd); + + return 0; + +error: + if (fd >= 0) + close(fd); + return -1; +} + +/* Read PCI config space. */ +int rte_pci_read_config(const struct rte_pci_device *dev, + void *buf, size_t len, off_t offset) +{ + int fd = -1; + int size; + struct pci_io pi = { + .pi_sel = { + .pc_domain = dev->addr.domain, + .pc_bus = dev->addr.bus, + .pc_dev = dev->addr.devid, + .pc_func = dev->addr.function, + }, + .pi_reg = offset, + }; + + fd = open("/dev/pci", O_RDWR); + if (fd < 0) { + RTE_LOG(ERR, EAL, "%s(): error opening /dev/pci\n", __func__); + goto error; + } + + while (len > 0) { + size = (len >= 4) ? 4 : ((len >= 2) ? 2 : 1); + pi.pi_width = size; + + if (ioctl(fd, PCIOCREAD, &pi) < 0) + goto error; + memcpy(buf, &pi.pi_data, size); + + buf = (char *)buf + size; + pi.pi_reg += size; + len -= size; + } + close(fd); + + return 0; + + error: + if (fd >= 0) + close(fd); + return -1; +} + +/* Write PCI config space. */ +int rte_pci_write_config(const struct rte_pci_device *dev, + const void *buf, size_t len, off_t offset) +{ + int fd = -1; + + struct pci_io pi = { + .pi_sel = { + .pc_domain = dev->addr.domain, + .pc_bus = dev->addr.bus, + .pc_dev = dev->addr.devid, + .pc_func = dev->addr.function, + }, + .pi_reg = offset, + .pi_data = *(const uint32_t *)buf, + .pi_width = len, + }; + + if (len == 3 || len > sizeof(pi.pi_data)) { + RTE_LOG(ERR, EAL, "%s(): invalid pci read length\n", __func__); + goto error; + } + + memcpy(&pi.pi_data, buf, len); + + fd = open("/dev/pci", O_RDWR); + if (fd < 0) { + RTE_LOG(ERR, EAL, "%s(): error opening /dev/pci\n", __func__); + goto error; + } + + if (ioctl(fd, PCIOCWRITE, &pi) < 0) + goto error; + + close(fd); + return 0; + + error: + if (fd >= 0) + close(fd); + return -1; +} + +int +rte_pci_ioport_map(struct rte_pci_device *dev, int bar, + struct rte_pci_ioport *p) +{ + int ret; + + switch (dev->kdrv) { +#if defined(RTE_ARCH_X86) + case RTE_KDRV_NIC_UIO: + if ((uintptr_t) dev->mem_resource[bar].addr <= UINT16_MAX) { + p->base = (uintptr_t)dev->mem_resource[bar].addr; + ret = 0; + } else + ret = -1; + break; +#endif + default: + ret = -1; + break; + } + + if (!ret) + p->dev = dev; + + return ret; +} + +static void +pci_uio_ioport_read(struct rte_pci_ioport *p, + void *data, size_t len, off_t offset) +{ +#if defined(RTE_ARCH_X86) + uint8_t *d; + int size; + unsigned short reg = p->base + offset; + + for (d = data; len > 0; d += size, reg += size, len -= size) { + if (len >= 4) { + size = 4; + *(uint32_t *)d = inl(reg); + } else if (len >= 2) { + size = 2; + *(uint16_t *)d = inw(reg); + } else { + size = 1; + *d = inb(reg); + } + } +#else + RTE_SET_USED(p); + RTE_SET_USED(data); + RTE_SET_USED(len); + RTE_SET_USED(offset); +#endif +} + +void +rte_pci_ioport_read(struct rte_pci_ioport *p, + void *data, size_t len, off_t offset) +{ + switch (p->dev->kdrv) { + case RTE_KDRV_NIC_UIO: + pci_uio_ioport_read(p, data, len, offset); + break; + default: + break; + } +} + +static void +pci_uio_ioport_write(struct rte_pci_ioport *p, + const void *data, size_t len, off_t offset) +{ +#if defined(RTE_ARCH_X86) + const uint8_t *s; + int size; + unsigned short reg = p->base + offset; + + for (s = data; len > 0; s += size, reg += size, len -= size) { + if (len >= 4) { + size = 4; + outl(reg, *(const uint32_t *)s); + } else if (len >= 2) { + size = 2; + outw(reg, *(const uint16_t *)s); + } else { + size = 1; + outb(reg, *s); + } + } +#else + RTE_SET_USED(p); + RTE_SET_USED(data); + RTE_SET_USED(len); + RTE_SET_USED(offset); +#endif +} + +void +rte_pci_ioport_write(struct rte_pci_ioport *p, + const void *data, size_t len, off_t offset) +{ + switch (p->dev->kdrv) { + case RTE_KDRV_NIC_UIO: + pci_uio_ioport_write(p, data, len, offset); + break; + default: + break; + } +} + +int +rte_pci_ioport_unmap(struct rte_pci_ioport *p) +{ + int ret; + + switch (p->dev->kdrv) { +#if defined(RTE_ARCH_X86) + case RTE_KDRV_NIC_UIO: + ret = 0; + break; +#endif + default: + ret = -1; + break; + } + + return ret; +} diff --git a/dpdk/drivers/bus/pci/linux/Makefile b/dpdk/drivers/bus/pci/linux/Makefile new file mode 100644 index 00000000..77c5f970 --- /dev/null +++ b/dpdk/drivers/bus/pci/linux/Makefile @@ -0,0 +1,36 @@ +# BSD LICENSE +# +# Copyright(c) 2017 6WIND S.A. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of 6WIND nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +SRCS += pci.c +SRCS += pci_uio.c +SRCS += pci_vfio.c + +CFLAGS += -D_GNU_SOURCE diff --git a/dpdk/drivers/bus/pci/linux/pci.c b/dpdk/drivers/bus/pci/linux/pci.c new file mode 100644 index 00000000..74deef3a --- /dev/null +++ b/dpdk/drivers/bus/pci/linux/pci.c @@ -0,0 +1,927 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "eal_private.h" +#include "eal_filesystem.h" + +#include "private.h" +#include "pci_init.h" + +/** + * @file + * PCI probing under linux + * + * This code is used to simulate a PCI probe by parsing information in sysfs. + * When a registered device matches a driver, it is then initialized with + * IGB_UIO driver (or doesn't initialize, if the device wasn't bound to it). + */ + +extern struct rte_pci_bus rte_pci_bus; + +static int +pci_get_kernel_driver_by_path(const char *filename, char *dri_name) +{ + int count; + char path[PATH_MAX]; + char *name; + + if (!filename || !dri_name) + return -1; + + count = readlink(filename, path, PATH_MAX); + if (count >= PATH_MAX) + return -1; + + /* For device does not have a driver */ + if (count < 0) + return 1; + + path[count] = '\0'; + + name = strrchr(path, '/'); + if (name) { + strncpy(dri_name, name + 1, strlen(name + 1) + 1); + return 0; + } + + return -1; +} + +/* Map pci device */ +int +rte_pci_map_device(struct rte_pci_device *dev) +{ + int ret = -1; + + /* try mapping the NIC resources using VFIO if it exists */ + switch (dev->kdrv) { + case RTE_KDRV_VFIO: +#ifdef VFIO_PRESENT + if (pci_vfio_is_enabled()) + ret = pci_vfio_map_resource(dev); +#endif + break; + case RTE_KDRV_IGB_UIO: + case RTE_KDRV_UIO_GENERIC: + if (rte_eal_using_phys_addrs()) { + /* map resources for devices that use uio */ + ret = pci_uio_map_resource(dev); + } + break; + default: + RTE_LOG(DEBUG, EAL, + " Not managed by a supported kernel driver, skipped\n"); + ret = 1; + break; + } + + return ret; +} + +/* Unmap pci device */ +void +rte_pci_unmap_device(struct rte_pci_device *dev) +{ + /* try unmapping the NIC resources using VFIO if it exists */ + switch (dev->kdrv) { + case RTE_KDRV_VFIO: +#ifdef VFIO_PRESENT + if (pci_vfio_is_enabled()) + pci_vfio_unmap_resource(dev); +#endif + break; + case RTE_KDRV_IGB_UIO: + case RTE_KDRV_UIO_GENERIC: + /* unmap resources for devices that use uio */ + pci_uio_unmap_resource(dev); + break; + default: + RTE_LOG(DEBUG, EAL, + " Not managed by a supported kernel driver, skipped\n"); + break; + } +} + +void * +pci_find_max_end_va(void) +{ + const struct rte_memseg *seg = rte_eal_get_physmem_layout(); + const struct rte_memseg *last = seg; + unsigned i = 0; + + for (i = 0; i < RTE_MAX_MEMSEG; i++, seg++) { + if (seg->addr == NULL) + break; + + if (seg->addr > last->addr) + last = seg; + + } + return RTE_PTR_ADD(last->addr, last->len); +} + +/* parse one line of the "resource" sysfs file (note that the 'line' + * string is modified) + */ +int +pci_parse_one_sysfs_resource(char *line, size_t len, uint64_t *phys_addr, + uint64_t *end_addr, uint64_t *flags) +{ + union pci_resource_info { + struct { + char *phys_addr; + char *end_addr; + char *flags; + }; + char *ptrs[PCI_RESOURCE_FMT_NVAL]; + } res_info; + + if (rte_strsplit(line, len, res_info.ptrs, 3, ' ') != 3) { + RTE_LOG(ERR, EAL, + "%s(): bad resource format\n", __func__); + return -1; + } + errno = 0; + *phys_addr = strtoull(res_info.phys_addr, NULL, 16); + *end_addr = strtoull(res_info.end_addr, NULL, 16); + *flags = strtoull(res_info.flags, NULL, 16); + if (errno != 0) { + RTE_LOG(ERR, EAL, + "%s(): bad resource format\n", __func__); + return -1; + } + + return 0; +} + +/* parse the "resource" sysfs file */ +static int +pci_parse_sysfs_resource(const char *filename, struct rte_pci_device *dev) +{ + FILE *f; + char buf[BUFSIZ]; + int i; + uint64_t phys_addr, end_addr, flags; + + f = fopen(filename, "r"); + if (f == NULL) { + RTE_LOG(ERR, EAL, "Cannot open sysfs resource\n"); + return -1; + } + + for (i = 0; imem_resource[i].phys_addr = phys_addr; + dev->mem_resource[i].len = end_addr - phys_addr + 1; + /* not mapped for now */ + dev->mem_resource[i].addr = NULL; + } + } + fclose(f); + return 0; + +error: + fclose(f); + return -1; +} + +/* Scan one pci sysfs entry, and fill the devices list from it. */ +static int +pci_scan_one(const char *dirname, const struct rte_pci_addr *addr) +{ + char filename[PATH_MAX]; + unsigned long tmp; + struct rte_pci_device *dev; + char driver[PATH_MAX]; + int ret; + + dev = malloc(sizeof(*dev)); + if (dev == NULL) + return -1; + + memset(dev, 0, sizeof(*dev)); + dev->addr = *addr; + + /* get vendor id */ + snprintf(filename, sizeof(filename), "%s/vendor", dirname); + if (eal_parse_sysfs_value(filename, &tmp) < 0) { + free(dev); + return -1; + } + dev->id.vendor_id = (uint16_t)tmp; + + /* get device id */ + snprintf(filename, sizeof(filename), "%s/device", dirname); + if (eal_parse_sysfs_value(filename, &tmp) < 0) { + free(dev); + return -1; + } + dev->id.device_id = (uint16_t)tmp; + + /* get subsystem_vendor id */ + snprintf(filename, sizeof(filename), "%s/subsystem_vendor", + dirname); + if (eal_parse_sysfs_value(filename, &tmp) < 0) { + free(dev); + return -1; + } + dev->id.subsystem_vendor_id = (uint16_t)tmp; + + /* get subsystem_device id */ + snprintf(filename, sizeof(filename), "%s/subsystem_device", + dirname); + if (eal_parse_sysfs_value(filename, &tmp) < 0) { + free(dev); + return -1; + } + dev->id.subsystem_device_id = (uint16_t)tmp; + + /* get class_id */ + snprintf(filename, sizeof(filename), "%s/class", + dirname); + if (eal_parse_sysfs_value(filename, &tmp) < 0) { + free(dev); + return -1; + } + /* the least 24 bits are valid: class, subclass, program interface */ + dev->id.class_id = (uint32_t)tmp & RTE_CLASS_ANY_ID; + + /* get max_vfs */ + dev->max_vfs = 0; + snprintf(filename, sizeof(filename), "%s/max_vfs", dirname); + if (!access(filename, F_OK) && + eal_parse_sysfs_value(filename, &tmp) == 0) + dev->max_vfs = (uint16_t)tmp; + else { + /* for non igb_uio driver, need kernel version >= 3.8 */ + snprintf(filename, sizeof(filename), + "%s/sriov_numvfs", dirname); + if (!access(filename, F_OK) && + eal_parse_sysfs_value(filename, &tmp) == 0) + dev->max_vfs = (uint16_t)tmp; + } + + /* get numa node, default to 0 if not present */ + snprintf(filename, sizeof(filename), "%s/numa_node", + dirname); + + if (access(filename, F_OK) != -1) { + if (eal_parse_sysfs_value(filename, &tmp) == 0) + dev->device.numa_node = tmp; + else + dev->device.numa_node = -1; + } else { + dev->device.numa_node = 0; + } + + pci_name_set(dev); + + /* parse resources */ + snprintf(filename, sizeof(filename), "%s/resource", dirname); + if (pci_parse_sysfs_resource(filename, dev) < 0) { + RTE_LOG(ERR, EAL, "%s(): cannot parse resource\n", __func__); + free(dev); + return -1; + } + + /* parse driver */ + snprintf(filename, sizeof(filename), "%s/driver", dirname); + ret = pci_get_kernel_driver_by_path(filename, driver); + if (ret < 0) { + RTE_LOG(ERR, EAL, "Fail to get kernel driver\n"); + free(dev); + return -1; + } + + if (!ret) { + if (!strcmp(driver, "vfio-pci")) + dev->kdrv = RTE_KDRV_VFIO; + else if (!strcmp(driver, "igb_uio")) + dev->kdrv = RTE_KDRV_IGB_UIO; + else if (!strcmp(driver, "uio_pci_generic")) + dev->kdrv = RTE_KDRV_UIO_GENERIC; + else + dev->kdrv = RTE_KDRV_UNKNOWN; + } else + dev->kdrv = RTE_KDRV_NONE; + + /* device is valid, add in list (sorted) */ + if (TAILQ_EMPTY(&rte_pci_bus.device_list)) { + rte_pci_add_device(dev); + } else { + struct rte_pci_device *dev2; + int ret; + + TAILQ_FOREACH(dev2, &rte_pci_bus.device_list, next) { + ret = rte_pci_addr_cmp(&dev->addr, &dev2->addr); + if (ret > 0) + continue; + + if (ret < 0) { + rte_pci_insert_device(dev2, dev); + } else { /* already registered */ + dev2->kdrv = dev->kdrv; + dev2->max_vfs = dev->max_vfs; + pci_name_set(dev2); + memmove(dev2->mem_resource, dev->mem_resource, + sizeof(dev->mem_resource)); + free(dev); + } + return 0; + } + + rte_pci_add_device(dev); + } + + return 0; +} + +int +pci_update_device(const struct rte_pci_addr *addr) +{ + char filename[PATH_MAX]; + + snprintf(filename, sizeof(filename), "%s/" PCI_PRI_FMT, + rte_pci_get_sysfs_path(), addr->domain, addr->bus, addr->devid, + addr->function); + + return pci_scan_one(filename, addr); +} + +/* + * split up a pci address into its constituent parts. + */ +static int +parse_pci_addr_format(const char *buf, int bufsize, struct rte_pci_addr *addr) +{ + /* first split on ':' */ + union splitaddr { + struct { + char *domain; + char *bus; + char *devid; + char *function; + }; + char *str[PCI_FMT_NVAL]; /* last element-separator is "." not ":" */ + } splitaddr; + + char *buf_copy = strndup(buf, bufsize); + if (buf_copy == NULL) + return -1; + + if (rte_strsplit(buf_copy, bufsize, splitaddr.str, PCI_FMT_NVAL, ':') + != PCI_FMT_NVAL - 1) + goto error; + /* final split is on '.' between devid and function */ + splitaddr.function = strchr(splitaddr.devid,'.'); + if (splitaddr.function == NULL) + goto error; + *splitaddr.function++ = '\0'; + + /* now convert to int values */ + errno = 0; + addr->domain = strtoul(splitaddr.domain, NULL, 16); + addr->bus = strtoul(splitaddr.bus, NULL, 16); + addr->devid = strtoul(splitaddr.devid, NULL, 16); + addr->function = strtoul(splitaddr.function, NULL, 10); + if (errno != 0) + goto error; + + free(buf_copy); /* free the copy made with strdup */ + return 0; +error: + free(buf_copy); + return -1; +} + +/* + * Scan the content of the PCI bus, and the devices in the devices + * list + */ +int +rte_pci_scan(void) +{ + struct dirent *e; + DIR *dir; + char dirname[PATH_MAX]; + struct rte_pci_addr addr; + + /* for debug purposes, PCI can be disabled */ + if (!rte_eal_has_pci()) + return 0; + +#ifdef VFIO_PRESENT + if (!pci_vfio_is_enabled()) + RTE_LOG(DEBUG, EAL, "VFIO PCI modules not loaded\n"); +#endif + + dir = opendir(rte_pci_get_sysfs_path()); + if (dir == NULL) { + RTE_LOG(ERR, EAL, "%s(): opendir failed: %s\n", + __func__, strerror(errno)); + return -1; + } + + while ((e = readdir(dir)) != NULL) { + if (e->d_name[0] == '.') + continue; + + if (parse_pci_addr_format(e->d_name, sizeof(e->d_name), &addr) != 0) + continue; + + snprintf(dirname, sizeof(dirname), "%s/%s", + rte_pci_get_sysfs_path(), e->d_name); + + if (pci_scan_one(dirname, &addr) < 0) + goto error; + } + closedir(dir); + return 0; + +error: + closedir(dir); + return -1; +} + +/* + * Is pci device bound to any kdrv + */ +static inline int +pci_one_device_is_bound(void) +{ + struct rte_pci_device *dev = NULL; + int ret = 0; + + FOREACH_DEVICE_ON_PCIBUS(dev) { + if (dev->kdrv == RTE_KDRV_UNKNOWN || + dev->kdrv == RTE_KDRV_NONE) { + continue; + } else { + ret = 1; + break; + } + } + return ret; +} + +/* + * Any one of the device bound to uio + */ +static inline int +pci_one_device_bound_uio(void) +{ + struct rte_pci_device *dev = NULL; + struct rte_devargs *devargs; + int need_check; + + FOREACH_DEVICE_ON_PCIBUS(dev) { + devargs = dev->device.devargs; + + need_check = 0; + switch (rte_pci_bus.bus.conf.scan_mode) { + case RTE_BUS_SCAN_WHITELIST: + if (devargs && devargs->policy == RTE_DEV_WHITELISTED) + need_check = 1; + break; + case RTE_BUS_SCAN_UNDEFINED: + case RTE_BUS_SCAN_BLACKLIST: + if (devargs == NULL || + devargs->policy != RTE_DEV_BLACKLISTED) + need_check = 1; + break; + } + + if (!need_check) + continue; + + if (dev->kdrv == RTE_KDRV_IGB_UIO || + dev->kdrv == RTE_KDRV_UIO_GENERIC) { + return 1; + } + } + return 0; +} + +/* + * Any one of the device has iova as va + */ +static inline int +pci_one_device_has_iova_va(void) +{ + struct rte_pci_device *dev = NULL; + struct rte_pci_driver *drv = NULL; + + FOREACH_DRIVER_ON_PCIBUS(drv) { + if (drv && drv->drv_flags & RTE_PCI_DRV_IOVA_AS_VA) { + FOREACH_DEVICE_ON_PCIBUS(dev) { + if (dev->kdrv == RTE_KDRV_VFIO && + rte_pci_match(drv, dev)) + return 1; + } + } + } + return 0; +} + +#if defined(RTE_ARCH_X86) +static bool +pci_one_device_iommu_support_va(struct rte_pci_device *dev) +{ +#define VTD_CAP_MGAW_SHIFT 16 +#define VTD_CAP_MGAW_MASK (0x3fULL << VTD_CAP_MGAW_SHIFT) +#define X86_VA_WIDTH 47 /* From Documentation/x86/x86_64/mm.txt */ + struct rte_pci_addr *addr = &dev->addr; + char filename[PATH_MAX]; + FILE *fp; + uint64_t mgaw, vtd_cap_reg = 0; + + snprintf(filename, sizeof(filename), + "%s/" PCI_PRI_FMT "/iommu/intel-iommu/cap", + rte_pci_get_sysfs_path(), addr->domain, addr->bus, addr->devid, + addr->function); + if (access(filename, F_OK) == -1) { + /* We don't have an Intel IOMMU, assume VA supported*/ + return true; + } + + /* We have an intel IOMMU */ + fp = fopen(filename, "r"); + if (fp == NULL) { + RTE_LOG(ERR, EAL, "%s(): can't open %s\n", __func__, filename); + return false; + } + + if (fscanf(fp, "%" PRIx64, &vtd_cap_reg) != 1) { + RTE_LOG(ERR, EAL, "%s(): can't read %s\n", __func__, filename); + fclose(fp); + return false; + } + + fclose(fp); + + mgaw = ((vtd_cap_reg & VTD_CAP_MGAW_MASK) >> VTD_CAP_MGAW_SHIFT) + 1; + if (mgaw < X86_VA_WIDTH) + return false; + + return true; +} +#elif defined(RTE_ARCH_PPC_64) +static bool +pci_one_device_iommu_support_va(__rte_unused struct rte_pci_device *dev) +{ + return false; +} +#else +static bool +pci_one_device_iommu_support_va(__rte_unused struct rte_pci_device *dev) +{ + return true; +} +#endif + +/* + * All devices IOMMUs support VA as IOVA + */ +static bool +pci_devices_iommu_support_va(void) +{ + struct rte_pci_device *dev = NULL; + struct rte_pci_driver *drv = NULL; + + FOREACH_DRIVER_ON_PCIBUS(drv) { + FOREACH_DEVICE_ON_PCIBUS(dev) { + if (!rte_pci_match(drv, dev)) + continue; + if (!pci_one_device_iommu_support_va(dev)) + return false; + } + } + return true; +} + +/* + * Get iommu class of PCI devices on the bus. + */ +enum rte_iova_mode +rte_pci_get_iommu_class(void) +{ + bool is_bound; + bool is_vfio_noiommu_enabled = true; + bool has_iova_va; + bool is_bound_uio; + bool iommu_no_va; + + is_bound = pci_one_device_is_bound(); + if (!is_bound) + return RTE_IOVA_DC; + + has_iova_va = pci_one_device_has_iova_va(); + is_bound_uio = pci_one_device_bound_uio(); + iommu_no_va = !pci_devices_iommu_support_va(); +#ifdef VFIO_PRESENT + is_vfio_noiommu_enabled = rte_vfio_noiommu_is_enabled() == true ? + true : false; +#endif + + if (has_iova_va && !is_bound_uio && !is_vfio_noiommu_enabled && + !iommu_no_va) + return RTE_IOVA_VA; + + if (has_iova_va) { + RTE_LOG(WARNING, EAL, "Some devices want iova as va but pa will be used because.. "); + if (is_vfio_noiommu_enabled) + RTE_LOG(WARNING, EAL, "vfio-noiommu mode configured\n"); + if (is_bound_uio) + RTE_LOG(WARNING, EAL, "few device bound to UIO\n"); + if (iommu_no_va) + RTE_LOG(WARNING, EAL, "IOMMU does not support IOVA as VA\n"); + } + + return RTE_IOVA_PA; +} + +/* Read PCI config space. */ +int rte_pci_read_config(const struct rte_pci_device *device, + void *buf, size_t len, off_t offset) +{ + const struct rte_intr_handle *intr_handle = &device->intr_handle; + + switch (intr_handle->type) { + case RTE_INTR_HANDLE_UIO: + case RTE_INTR_HANDLE_UIO_INTX: + return pci_uio_read_config(intr_handle, buf, len, offset); + +#ifdef VFIO_PRESENT + case RTE_INTR_HANDLE_VFIO_MSIX: + case RTE_INTR_HANDLE_VFIO_MSI: + case RTE_INTR_HANDLE_VFIO_LEGACY: + return pci_vfio_read_config(intr_handle, buf, len, offset); +#endif + default: + RTE_LOG(ERR, EAL, + "Unknown handle type of fd %d\n", + intr_handle->fd); + return -1; + } +} + +/* Write PCI config space. */ +int rte_pci_write_config(const struct rte_pci_device *device, + const void *buf, size_t len, off_t offset) +{ + const struct rte_intr_handle *intr_handle = &device->intr_handle; + + switch (intr_handle->type) { + case RTE_INTR_HANDLE_UIO: + case RTE_INTR_HANDLE_UIO_INTX: + return pci_uio_write_config(intr_handle, buf, len, offset); + +#ifdef VFIO_PRESENT + case RTE_INTR_HANDLE_VFIO_MSIX: + case RTE_INTR_HANDLE_VFIO_MSI: + case RTE_INTR_HANDLE_VFIO_LEGACY: + return pci_vfio_write_config(intr_handle, buf, len, offset); +#endif + default: + RTE_LOG(ERR, EAL, + "Unknown handle type of fd %d\n", + intr_handle->fd); + return -1; + } +} + +#if defined(RTE_ARCH_X86) +static int +pci_ioport_map(struct rte_pci_device *dev, int bar __rte_unused, + struct rte_pci_ioport *p) +{ + uint16_t start, end; + FILE *fp; + char *line = NULL; + char pci_id[16]; + int found = 0; + size_t linesz; + + snprintf(pci_id, sizeof(pci_id), PCI_PRI_FMT, + dev->addr.domain, dev->addr.bus, + dev->addr.devid, dev->addr.function); + + fp = fopen("/proc/ioports", "r"); + if (fp == NULL) { + RTE_LOG(ERR, EAL, "%s(): can't open ioports\n", __func__); + return -1; + } + + while (getdelim(&line, &linesz, '\n', fp) > 0) { + char *ptr = line; + char *left; + int n; + + n = strcspn(ptr, ":"); + ptr[n] = 0; + left = &ptr[n + 1]; + + while (*left && isspace(*left)) + left++; + + if (!strncmp(left, pci_id, strlen(pci_id))) { + found = 1; + + while (*ptr && isspace(*ptr)) + ptr++; + + sscanf(ptr, "%04hx-%04hx", &start, &end); + + break; + } + } + + free(line); + fclose(fp); + + if (!found) + return -1; + + p->base = start; + RTE_LOG(DEBUG, EAL, "PCI Port IO found start=0x%x\n", start); + + return 0; +} +#endif + +int +rte_pci_ioport_map(struct rte_pci_device *dev, int bar, + struct rte_pci_ioport *p) +{ + int ret = -1; + + switch (dev->kdrv) { +#ifdef VFIO_PRESENT + case RTE_KDRV_VFIO: + if (pci_vfio_is_enabled()) + ret = pci_vfio_ioport_map(dev, bar, p); + break; +#endif + case RTE_KDRV_IGB_UIO: + ret = pci_uio_ioport_map(dev, bar, p); + break; + case RTE_KDRV_UIO_GENERIC: +#if defined(RTE_ARCH_X86) + ret = pci_ioport_map(dev, bar, p); +#else + ret = pci_uio_ioport_map(dev, bar, p); +#endif + break; + case RTE_KDRV_NONE: +#if defined(RTE_ARCH_X86) + ret = pci_ioport_map(dev, bar, p); +#endif + break; + default: + break; + } + + if (!ret) + p->dev = dev; + + return ret; +} + +void +rte_pci_ioport_read(struct rte_pci_ioport *p, + void *data, size_t len, off_t offset) +{ + switch (p->dev->kdrv) { +#ifdef VFIO_PRESENT + case RTE_KDRV_VFIO: + pci_vfio_ioport_read(p, data, len, offset); + break; +#endif + case RTE_KDRV_IGB_UIO: + pci_uio_ioport_read(p, data, len, offset); + break; + case RTE_KDRV_UIO_GENERIC: + pci_uio_ioport_read(p, data, len, offset); + break; + case RTE_KDRV_NONE: +#if defined(RTE_ARCH_X86) + pci_uio_ioport_read(p, data, len, offset); +#endif + break; + default: + break; + } +} + +void +rte_pci_ioport_write(struct rte_pci_ioport *p, + const void *data, size_t len, off_t offset) +{ + switch (p->dev->kdrv) { +#ifdef VFIO_PRESENT + case RTE_KDRV_VFIO: + pci_vfio_ioport_write(p, data, len, offset); + break; +#endif + case RTE_KDRV_IGB_UIO: + pci_uio_ioport_write(p, data, len, offset); + break; + case RTE_KDRV_UIO_GENERIC: + pci_uio_ioport_write(p, data, len, offset); + break; + case RTE_KDRV_NONE: +#if defined(RTE_ARCH_X86) + pci_uio_ioport_write(p, data, len, offset); +#endif + break; + default: + break; + } +} + +int +rte_pci_ioport_unmap(struct rte_pci_ioport *p) +{ + int ret = -1; + + switch (p->dev->kdrv) { +#ifdef VFIO_PRESENT + case RTE_KDRV_VFIO: + if (pci_vfio_is_enabled()) + ret = pci_vfio_ioport_unmap(p); + break; +#endif + case RTE_KDRV_IGB_UIO: + ret = pci_uio_ioport_unmap(p); + break; + case RTE_KDRV_UIO_GENERIC: +#if defined(RTE_ARCH_X86) + ret = 0; +#else + ret = pci_uio_ioport_unmap(p); +#endif + break; + case RTE_KDRV_NONE: +#if defined(RTE_ARCH_X86) + ret = 0; +#endif + break; + default: + break; + } + + return ret; +} diff --git a/dpdk/drivers/bus/pci/linux/pci_init.h b/dpdk/drivers/bus/pci/linux/pci_init.h new file mode 100644 index 00000000..f342c47d --- /dev/null +++ b/dpdk/drivers/bus/pci/linux/pci_init.h @@ -0,0 +1,111 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef EAL_PCI_INIT_H_ +#define EAL_PCI_INIT_H_ + +#include + +#include + +/** IO resource type: */ +#define IORESOURCE_IO 0x00000100 +#define IORESOURCE_MEM 0x00000200 + +/* + * Helper function to map PCI resources right after hugepages in virtual memory + */ +extern void *pci_map_addr; +void *pci_find_max_end_va(void); + +/* parse one line of the "resource" sysfs file (note that the 'line' + * string is modified) + */ +int pci_parse_one_sysfs_resource(char *line, size_t len, uint64_t *phys_addr, + uint64_t *end_addr, uint64_t *flags); + +int pci_uio_alloc_resource(struct rte_pci_device *dev, + struct mapped_pci_resource **uio_res); +void pci_uio_free_resource(struct rte_pci_device *dev, + struct mapped_pci_resource *uio_res); +int pci_uio_map_resource_by_index(struct rte_pci_device *dev, int res_idx, + struct mapped_pci_resource *uio_res, int map_idx); + +int pci_uio_read_config(const struct rte_intr_handle *intr_handle, + void *buf, size_t len, off_t offs); +int pci_uio_write_config(const struct rte_intr_handle *intr_handle, + const void *buf, size_t len, off_t offs); + +int pci_uio_ioport_map(struct rte_pci_device *dev, int bar, + struct rte_pci_ioport *p); +void pci_uio_ioport_read(struct rte_pci_ioport *p, + void *data, size_t len, off_t offset); +void pci_uio_ioport_write(struct rte_pci_ioport *p, + const void *data, size_t len, off_t offset); +int pci_uio_ioport_unmap(struct rte_pci_ioport *p); + +#ifdef VFIO_PRESENT + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 10, 0) +#define RTE_PCI_MSIX_TABLE_BIR 0x7 +#define RTE_PCI_MSIX_TABLE_OFFSET 0xfffffff8 +#define RTE_PCI_MSIX_FLAGS_QSIZE 0x07ff +#else +#define RTE_PCI_MSIX_TABLE_BIR PCI_MSIX_TABLE_BIR +#define RTE_PCI_MSIX_TABLE_OFFSET PCI_MSIX_TABLE_OFFSET +#define RTE_PCI_MSIX_FLAGS_QSIZE PCI_MSIX_FLAGS_QSIZE +#endif + +/* access config space */ +int pci_vfio_read_config(const struct rte_intr_handle *intr_handle, + void *buf, size_t len, off_t offs); +int pci_vfio_write_config(const struct rte_intr_handle *intr_handle, + const void *buf, size_t len, off_t offs); + +int pci_vfio_ioport_map(struct rte_pci_device *dev, int bar, + struct rte_pci_ioport *p); +void pci_vfio_ioport_read(struct rte_pci_ioport *p, + void *data, size_t len, off_t offset); +void pci_vfio_ioport_write(struct rte_pci_ioport *p, + const void *data, size_t len, off_t offset); +int pci_vfio_ioport_unmap(struct rte_pci_ioport *p); + +/* map/unmap VFIO resource prototype */ +int pci_vfio_map_resource(struct rte_pci_device *dev); +int pci_vfio_unmap_resource(struct rte_pci_device *dev); + +int pci_vfio_is_enabled(void); + +#endif + +#endif /* EAL_PCI_INIT_H_ */ diff --git a/dpdk/drivers/bus/pci/linux/pci_uio.c b/dpdk/drivers/bus/pci/linux/pci_uio.c new file mode 100644 index 00000000..92b7f027 --- /dev/null +++ b/dpdk/drivers/bus/pci/linux/pci_uio.c @@ -0,0 +1,568 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(RTE_ARCH_X86) +#include +#endif + +#include +#include +#include +#include +#include +#include + +#include "eal_filesystem.h" +#include "pci_init.h" + +void *pci_map_addr = NULL; + +#define OFF_MAX ((uint64_t)(off_t)-1) + +int +pci_uio_read_config(const struct rte_intr_handle *intr_handle, + void *buf, size_t len, off_t offset) +{ + return pread(intr_handle->uio_cfg_fd, buf, len, offset); +} + +int +pci_uio_write_config(const struct rte_intr_handle *intr_handle, + const void *buf, size_t len, off_t offset) +{ + return pwrite(intr_handle->uio_cfg_fd, buf, len, offset); +} + +static int +pci_uio_set_bus_master(int dev_fd) +{ + uint16_t reg; + int ret; + + ret = pread(dev_fd, ®, sizeof(reg), PCI_COMMAND); + if (ret != sizeof(reg)) { + RTE_LOG(ERR, EAL, + "Cannot read command from PCI config space!\n"); + return -1; + } + + /* return if bus mastering is already on */ + if (reg & PCI_COMMAND_MASTER) + return 0; + + reg |= PCI_COMMAND_MASTER; + + ret = pwrite(dev_fd, ®, sizeof(reg), PCI_COMMAND); + if (ret != sizeof(reg)) { + RTE_LOG(ERR, EAL, + "Cannot write command to PCI config space!\n"); + return -1; + } + + return 0; +} + +static int +pci_mknod_uio_dev(const char *sysfs_uio_path, unsigned uio_num) +{ + FILE *f; + char filename[PATH_MAX]; + int ret; + unsigned major, minor; + dev_t dev; + + /* get the name of the sysfs file that contains the major and minor + * of the uio device and read its content */ + snprintf(filename, sizeof(filename), "%s/dev", sysfs_uio_path); + + f = fopen(filename, "r"); + if (f == NULL) { + RTE_LOG(ERR, EAL, "%s(): cannot open sysfs to get major:minor\n", + __func__); + return -1; + } + + ret = fscanf(f, "%u:%u", &major, &minor); + if (ret != 2) { + RTE_LOG(ERR, EAL, "%s(): cannot parse sysfs to get major:minor\n", + __func__); + fclose(f); + return -1; + } + fclose(f); + + /* create the char device "mknod /dev/uioX c major minor" */ + snprintf(filename, sizeof(filename), "/dev/uio%u", uio_num); + dev = makedev(major, minor); + ret = mknod(filename, S_IFCHR | S_IRUSR | S_IWUSR, dev); + if (ret != 0) { + RTE_LOG(ERR, EAL, "%s(): mknod() failed %s\n", + __func__, strerror(errno)); + return -1; + } + + return ret; +} + +/* + * Return the uioX char device used for a pci device. On success, return + * the UIO number and fill dstbuf string with the path of the device in + * sysfs. On error, return a negative value. In this case dstbuf is + * invalid. + */ +static int +pci_get_uio_dev(struct rte_pci_device *dev, char *dstbuf, + unsigned int buflen, int create) +{ + struct rte_pci_addr *loc = &dev->addr; + int uio_num = -1; + struct dirent *e; + DIR *dir; + char dirname[PATH_MAX]; + + /* depending on kernel version, uio can be located in uio/uioX + * or uio:uioX */ + + snprintf(dirname, sizeof(dirname), + "%s/" PCI_PRI_FMT "/uio", rte_pci_get_sysfs_path(), + loc->domain, loc->bus, loc->devid, loc->function); + + dir = opendir(dirname); + if (dir == NULL) { + /* retry with the parent directory */ + snprintf(dirname, sizeof(dirname), + "%s/" PCI_PRI_FMT, rte_pci_get_sysfs_path(), + loc->domain, loc->bus, loc->devid, loc->function); + dir = opendir(dirname); + + if (dir == NULL) { + RTE_LOG(ERR, EAL, "Cannot opendir %s\n", dirname); + return -1; + } + } + + /* take the first file starting with "uio" */ + while ((e = readdir(dir)) != NULL) { + /* format could be uio%d ...*/ + int shortprefix_len = sizeof("uio") - 1; + /* ... or uio:uio%d */ + int longprefix_len = sizeof("uio:uio") - 1; + char *endptr; + + if (strncmp(e->d_name, "uio", 3) != 0) + continue; + + /* first try uio%d */ + errno = 0; + uio_num = strtoull(e->d_name + shortprefix_len, &endptr, 10); + if (errno == 0 && endptr != (e->d_name + shortprefix_len)) { + snprintf(dstbuf, buflen, "%s/uio%u", dirname, uio_num); + break; + } + + /* then try uio:uio%d */ + errno = 0; + uio_num = strtoull(e->d_name + longprefix_len, &endptr, 10); + if (errno == 0 && endptr != (e->d_name + longprefix_len)) { + snprintf(dstbuf, buflen, "%s/uio:uio%u", dirname, uio_num); + break; + } + } + closedir(dir); + + /* No uio resource found */ + if (e == NULL) + return -1; + + /* create uio device if we've been asked to */ + if (rte_eal_create_uio_dev() && create && + pci_mknod_uio_dev(dstbuf, uio_num) < 0) + RTE_LOG(WARNING, EAL, "Cannot create /dev/uio%u\n", uio_num); + + return uio_num; +} + +void +pci_uio_free_resource(struct rte_pci_device *dev, + struct mapped_pci_resource *uio_res) +{ + rte_free(uio_res); + + if (dev->intr_handle.uio_cfg_fd >= 0) { + close(dev->intr_handle.uio_cfg_fd); + dev->intr_handle.uio_cfg_fd = -1; + } + if (dev->intr_handle.fd >= 0) { + close(dev->intr_handle.fd); + dev->intr_handle.fd = -1; + dev->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN; + } +} + +int +pci_uio_alloc_resource(struct rte_pci_device *dev, + struct mapped_pci_resource **uio_res) +{ + char dirname[PATH_MAX]; + char cfgname[PATH_MAX]; + char devname[PATH_MAX]; /* contains the /dev/uioX */ + int uio_num; + struct rte_pci_addr *loc; + + loc = &dev->addr; + + /* find uio resource */ + uio_num = pci_get_uio_dev(dev, dirname, sizeof(dirname), 1); + if (uio_num < 0) { + RTE_LOG(WARNING, EAL, " "PCI_PRI_FMT" not managed by UIO driver, " + "skipping\n", loc->domain, loc->bus, loc->devid, loc->function); + return 1; + } + snprintf(devname, sizeof(devname), "/dev/uio%u", uio_num); + + /* save fd if in primary process */ + dev->intr_handle.fd = open(devname, O_RDWR); + if (dev->intr_handle.fd < 0) { + RTE_LOG(ERR, EAL, "Cannot open %s: %s\n", + devname, strerror(errno)); + goto error; + } + + snprintf(cfgname, sizeof(cfgname), + "/sys/class/uio/uio%u/device/config", uio_num); + dev->intr_handle.uio_cfg_fd = open(cfgname, O_RDWR); + if (dev->intr_handle.uio_cfg_fd < 0) { + RTE_LOG(ERR, EAL, "Cannot open %s: %s\n", + cfgname, strerror(errno)); + goto error; + } + + if (dev->kdrv == RTE_KDRV_IGB_UIO) + dev->intr_handle.type = RTE_INTR_HANDLE_UIO; + else { + dev->intr_handle.type = RTE_INTR_HANDLE_UIO_INTX; + + /* set bus master that is not done by uio_pci_generic */ + if (pci_uio_set_bus_master(dev->intr_handle.uio_cfg_fd)) { + RTE_LOG(ERR, EAL, "Cannot set up bus mastering!\n"); + goto error; + } + } + + /* allocate the mapping details for secondary processes*/ + *uio_res = rte_zmalloc("UIO_RES", sizeof(**uio_res), 0); + if (*uio_res == NULL) { + RTE_LOG(ERR, EAL, + "%s(): cannot store uio mmap details\n", __func__); + goto error; + } + + snprintf((*uio_res)->path, sizeof((*uio_res)->path), "%s", devname); + memcpy(&(*uio_res)->pci_addr, &dev->addr, sizeof((*uio_res)->pci_addr)); + + return 0; + +error: + pci_uio_free_resource(dev, *uio_res); + return -1; +} + +int +pci_uio_map_resource_by_index(struct rte_pci_device *dev, int res_idx, + struct mapped_pci_resource *uio_res, int map_idx) +{ + int fd; + char devname[PATH_MAX]; + void *mapaddr; + struct rte_pci_addr *loc; + struct pci_map *maps; + + loc = &dev->addr; + maps = uio_res->maps; + + /* update devname for mmap */ + snprintf(devname, sizeof(devname), + "%s/" PCI_PRI_FMT "/resource%d", + rte_pci_get_sysfs_path(), + loc->domain, loc->bus, loc->devid, + loc->function, res_idx); + + /* allocate memory to keep path */ + maps[map_idx].path = rte_malloc(NULL, strlen(devname) + 1, 0); + if (maps[map_idx].path == NULL) { + RTE_LOG(ERR, EAL, "Cannot allocate memory for path: %s\n", + strerror(errno)); + return -1; + } + + /* + * open resource file, to mmap it + */ + fd = open(devname, O_RDWR); + if (fd < 0) { + RTE_LOG(ERR, EAL, "Cannot open %s: %s\n", + devname, strerror(errno)); + goto error; + } + + /* try mapping somewhere close to the end of hugepages */ + if (pci_map_addr == NULL) + pci_map_addr = pci_find_max_end_va(); + + mapaddr = pci_map_resource(pci_map_addr, fd, 0, + (size_t)dev->mem_resource[res_idx].len, 0); + close(fd); + if (mapaddr == MAP_FAILED) + goto error; + + pci_map_addr = RTE_PTR_ADD(mapaddr, + (size_t)dev->mem_resource[res_idx].len); + + maps[map_idx].phaddr = dev->mem_resource[res_idx].phys_addr; + maps[map_idx].size = dev->mem_resource[res_idx].len; + maps[map_idx].addr = mapaddr; + maps[map_idx].offset = 0; + strcpy(maps[map_idx].path, devname); + dev->mem_resource[res_idx].addr = mapaddr; + + return 0; + +error: + rte_free(maps[map_idx].path); + return -1; +} + +#if defined(RTE_ARCH_X86) +int +pci_uio_ioport_map(struct rte_pci_device *dev, int bar, + struct rte_pci_ioport *p) +{ + char dirname[PATH_MAX]; + char filename[PATH_MAX]; + int uio_num; + unsigned long start; + + uio_num = pci_get_uio_dev(dev, dirname, sizeof(dirname), 0); + if (uio_num < 0) + return -1; + + /* get portio start */ + snprintf(filename, sizeof(filename), + "%s/portio/port%d/start", dirname, bar); + if (eal_parse_sysfs_value(filename, &start) < 0) { + RTE_LOG(ERR, EAL, "%s(): cannot parse portio start\n", + __func__); + return -1; + } + /* ensure we don't get anything funny here, read/write will cast to + * uin16_t */ + if (start > UINT16_MAX) + return -1; + + /* FIXME only for primary process ? */ + if (dev->intr_handle.type == RTE_INTR_HANDLE_UNKNOWN) { + + snprintf(filename, sizeof(filename), "/dev/uio%u", uio_num); + dev->intr_handle.fd = open(filename, O_RDWR); + if (dev->intr_handle.fd < 0) { + RTE_LOG(ERR, EAL, "Cannot open %s: %s\n", + filename, strerror(errno)); + return -1; + } + dev->intr_handle.type = RTE_INTR_HANDLE_UIO; + } + + RTE_LOG(DEBUG, EAL, "PCI Port IO found start=0x%lx\n", start); + + p->base = start; + p->len = 0; + return 0; +} +#else +int +pci_uio_ioport_map(struct rte_pci_device *dev, int bar, + struct rte_pci_ioport *p) +{ + FILE *f; + char buf[BUFSIZ]; + char filename[PATH_MAX]; + uint64_t phys_addr, end_addr, flags; + int fd, i; + void *addr; + + /* open and read addresses of the corresponding resource in sysfs */ + snprintf(filename, sizeof(filename), "%s/" PCI_PRI_FMT "/resource", + rte_pci_get_sysfs_path(), dev->addr.domain, dev->addr.bus, + dev->addr.devid, dev->addr.function); + f = fopen(filename, "r"); + if (f == NULL) { + RTE_LOG(ERR, EAL, "Cannot open sysfs resource: %s\n", + strerror(errno)); + return -1; + } + for (i = 0; i < bar + 1; i++) { + if (fgets(buf, sizeof(buf), f) == NULL) { + RTE_LOG(ERR, EAL, "Cannot read sysfs resource\n"); + goto error; + } + } + if (pci_parse_one_sysfs_resource(buf, sizeof(buf), &phys_addr, + &end_addr, &flags) < 0) + goto error; + if ((flags & IORESOURCE_IO) == 0) { + RTE_LOG(ERR, EAL, "BAR %d is not an IO resource\n", bar); + goto error; + } + snprintf(filename, sizeof(filename), "%s/" PCI_PRI_FMT "/resource%d", + rte_pci_get_sysfs_path(), dev->addr.domain, dev->addr.bus, + dev->addr.devid, dev->addr.function, bar); + + /* mmap the pci resource */ + fd = open(filename, O_RDWR); + if (fd < 0) { + RTE_LOG(ERR, EAL, "Cannot open %s: %s\n", filename, + strerror(errno)); + goto error; + } + addr = mmap(NULL, end_addr + 1, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, 0); + close(fd); + if (addr == MAP_FAILED) { + RTE_LOG(ERR, EAL, "Cannot mmap IO port resource: %s\n", + strerror(errno)); + goto error; + } + + /* strangely, the base address is mmap addr + phys_addr */ + p->base = (uintptr_t)addr + phys_addr; + p->len = end_addr + 1; + RTE_LOG(DEBUG, EAL, "PCI Port IO found start=0x%"PRIx64"\n", p->base); + fclose(f); + + return 0; + +error: + fclose(f); + return -1; +} +#endif + +void +pci_uio_ioport_read(struct rte_pci_ioport *p, + void *data, size_t len, off_t offset) +{ + uint8_t *d; + int size; + uintptr_t reg = p->base + offset; + + for (d = data; len > 0; d += size, reg += size, len -= size) { + if (len >= 4) { + size = 4; +#if defined(RTE_ARCH_X86) + *(uint32_t *)d = inl(reg); +#else + *(uint32_t *)d = *(volatile uint32_t *)reg; +#endif + } else if (len >= 2) { + size = 2; +#if defined(RTE_ARCH_X86) + *(uint16_t *)d = inw(reg); +#else + *(uint16_t *)d = *(volatile uint16_t *)reg; +#endif + } else { + size = 1; +#if defined(RTE_ARCH_X86) + *d = inb(reg); +#else + *d = *(volatile uint8_t *)reg; +#endif + } + } +} + +void +pci_uio_ioport_write(struct rte_pci_ioport *p, + const void *data, size_t len, off_t offset) +{ + const uint8_t *s; + int size; + uintptr_t reg = p->base + offset; + + for (s = data; len > 0; s += size, reg += size, len -= size) { + if (len >= 4) { + size = 4; +#if defined(RTE_ARCH_X86) + outl_p(*(const uint32_t *)s, reg); +#else + *(volatile uint32_t *)reg = *(const uint32_t *)s; +#endif + } else if (len >= 2) { + size = 2; +#if defined(RTE_ARCH_X86) + outw_p(*(const uint16_t *)s, reg); +#else + *(volatile uint16_t *)reg = *(const uint16_t *)s; +#endif + } else { + size = 1; +#if defined(RTE_ARCH_X86) + outb_p(*s, reg); +#else + *(volatile uint8_t *)reg = *s; +#endif + } + } +} + +int +pci_uio_ioport_unmap(struct rte_pci_ioport *p) +{ +#if defined(RTE_ARCH_X86) + RTE_SET_USED(p); + /* FIXME close intr fd ? */ + return 0; +#else + return munmap((void *)(uintptr_t)p->base, p->len); +#endif +} diff --git a/dpdk/drivers/bus/pci/linux/pci_vfio.c b/dpdk/drivers/bus/pci/linux/pci_vfio.c new file mode 100644 index 00000000..e7d7f5dc --- /dev/null +++ b/dpdk/drivers/bus/pci/linux/pci_vfio.c @@ -0,0 +1,761 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "eal_filesystem.h" + +#include "pci_init.h" +#include "private.h" + +/** + * @file + * PCI probing under linux (VFIO version) + * + * This code tries to determine if the PCI device is bound to VFIO driver, + * and initialize it (map BARs, set up interrupts) if that's the case. + * + * This file is only compiled if CONFIG_RTE_EAL_VFIO is set to "y". + */ + +#ifdef VFIO_PRESENT + +#define PAGE_SIZE (sysconf(_SC_PAGESIZE)) +#define PAGE_MASK (~(PAGE_SIZE - 1)) + +static struct rte_tailq_elem rte_vfio_tailq = { + .name = "VFIO_RESOURCE_LIST", +}; +EAL_REGISTER_TAILQ(rte_vfio_tailq) + +int +pci_vfio_read_config(const struct rte_intr_handle *intr_handle, + void *buf, size_t len, off_t offs) +{ + return pread64(intr_handle->vfio_dev_fd, buf, len, + VFIO_GET_REGION_ADDR(VFIO_PCI_CONFIG_REGION_INDEX) + offs); +} + +int +pci_vfio_write_config(const struct rte_intr_handle *intr_handle, + const void *buf, size_t len, off_t offs) +{ + return pwrite64(intr_handle->vfio_dev_fd, buf, len, + VFIO_GET_REGION_ADDR(VFIO_PCI_CONFIG_REGION_INDEX) + offs); +} + +/* get PCI BAR number where MSI-X interrupts are */ +static int +pci_vfio_get_msix_bar(int fd, struct pci_msix_table *msix_table) +{ + int ret; + uint32_t reg; + uint16_t flags; + uint8_t cap_id, cap_offset; + + /* read PCI capability pointer from config space */ + ret = pread64(fd, ®, sizeof(reg), + VFIO_GET_REGION_ADDR(VFIO_PCI_CONFIG_REGION_INDEX) + + PCI_CAPABILITY_LIST); + if (ret != sizeof(reg)) { + RTE_LOG(ERR, EAL, "Cannot read capability pointer from PCI " + "config space!\n"); + return -1; + } + + /* we need first byte */ + cap_offset = reg & 0xFF; + + while (cap_offset) { + + /* read PCI capability ID */ + ret = pread64(fd, ®, sizeof(reg), + VFIO_GET_REGION_ADDR(VFIO_PCI_CONFIG_REGION_INDEX) + + cap_offset); + if (ret != sizeof(reg)) { + RTE_LOG(ERR, EAL, "Cannot read capability ID from PCI " + "config space!\n"); + return -1; + } + + /* we need first byte */ + cap_id = reg & 0xFF; + + /* if we haven't reached MSI-X, check next capability */ + if (cap_id != PCI_CAP_ID_MSIX) { + ret = pread64(fd, ®, sizeof(reg), + VFIO_GET_REGION_ADDR(VFIO_PCI_CONFIG_REGION_INDEX) + + cap_offset); + if (ret != sizeof(reg)) { + RTE_LOG(ERR, EAL, "Cannot read capability pointer from PCI " + "config space!\n"); + return -1; + } + + /* we need second byte */ + cap_offset = (reg & 0xFF00) >> 8; + + continue; + } + /* else, read table offset */ + else { + /* table offset resides in the next 4 bytes */ + ret = pread64(fd, ®, sizeof(reg), + VFIO_GET_REGION_ADDR(VFIO_PCI_CONFIG_REGION_INDEX) + + cap_offset + 4); + if (ret != sizeof(reg)) { + RTE_LOG(ERR, EAL, "Cannot read table offset from PCI config " + "space!\n"); + return -1; + } + + ret = pread64(fd, &flags, sizeof(flags), + VFIO_GET_REGION_ADDR(VFIO_PCI_CONFIG_REGION_INDEX) + + cap_offset + 2); + if (ret != sizeof(flags)) { + RTE_LOG(ERR, EAL, "Cannot read table flags from PCI config " + "space!\n"); + return -1; + } + + msix_table->bar_index = reg & RTE_PCI_MSIX_TABLE_BIR; + msix_table->offset = reg & RTE_PCI_MSIX_TABLE_OFFSET; + msix_table->size = + 16 * (1 + (flags & RTE_PCI_MSIX_FLAGS_QSIZE)); + + return 0; + } + } + return 0; +} + +/* set PCI bus mastering */ +static int +pci_vfio_set_bus_master(int dev_fd, bool op) +{ + uint16_t reg; + int ret; + + ret = pread64(dev_fd, ®, sizeof(reg), + VFIO_GET_REGION_ADDR(VFIO_PCI_CONFIG_REGION_INDEX) + + PCI_COMMAND); + if (ret != sizeof(reg)) { + RTE_LOG(ERR, EAL, "Cannot read command from PCI config space!\n"); + return -1; + } + + if (op) + /* set the master bit */ + reg |= PCI_COMMAND_MASTER; + else + reg &= ~(PCI_COMMAND_MASTER); + + ret = pwrite64(dev_fd, ®, sizeof(reg), + VFIO_GET_REGION_ADDR(VFIO_PCI_CONFIG_REGION_INDEX) + + PCI_COMMAND); + + if (ret != sizeof(reg)) { + RTE_LOG(ERR, EAL, "Cannot write command to PCI config space!\n"); + return -1; + } + + return 0; +} + +/* set up interrupt support (but not enable interrupts) */ +static int +pci_vfio_setup_interrupts(struct rte_pci_device *dev, int vfio_dev_fd) +{ + int i, ret, intr_idx; + enum rte_intr_mode intr_mode; + + /* default to invalid index */ + intr_idx = VFIO_PCI_NUM_IRQS; + + /* Get default / configured intr_mode */ + intr_mode = rte_eal_vfio_intr_mode(); + + /* get interrupt type from internal config (MSI-X by default, can be + * overridden from the command line + */ + switch (intr_mode) { + case RTE_INTR_MODE_MSIX: + intr_idx = VFIO_PCI_MSIX_IRQ_INDEX; + break; + case RTE_INTR_MODE_MSI: + intr_idx = VFIO_PCI_MSI_IRQ_INDEX; + break; + case RTE_INTR_MODE_LEGACY: + intr_idx = VFIO_PCI_INTX_IRQ_INDEX; + break; + /* don't do anything if we want to automatically determine interrupt type */ + case RTE_INTR_MODE_NONE: + break; + default: + RTE_LOG(ERR, EAL, " unknown default interrupt type!\n"); + return -1; + } + + /* start from MSI-X interrupt type */ + for (i = VFIO_PCI_MSIX_IRQ_INDEX; i >= 0; i--) { + struct vfio_irq_info irq = { .argsz = sizeof(irq) }; + int fd = -1; + + /* skip interrupt modes we don't want */ + if (intr_mode != RTE_INTR_MODE_NONE && + i != intr_idx) + continue; + + irq.index = i; + + ret = ioctl(vfio_dev_fd, VFIO_DEVICE_GET_IRQ_INFO, &irq); + if (ret < 0) { + RTE_LOG(ERR, EAL, " cannot get IRQ info, " + "error %i (%s)\n", errno, strerror(errno)); + return -1; + } + + /* if this vector cannot be used with eventfd, fail if we explicitly + * specified interrupt type, otherwise continue */ + if ((irq.flags & VFIO_IRQ_INFO_EVENTFD) == 0) { + if (intr_mode != RTE_INTR_MODE_NONE) { + RTE_LOG(ERR, EAL, + " interrupt vector does not support eventfd!\n"); + return -1; + } else + continue; + } + + /* set up an eventfd for interrupts */ + fd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC); + if (fd < 0) { + RTE_LOG(ERR, EAL, " cannot set up eventfd, " + "error %i (%s)\n", errno, strerror(errno)); + return -1; + } + + dev->intr_handle.fd = fd; + dev->intr_handle.vfio_dev_fd = vfio_dev_fd; + + switch (i) { + case VFIO_PCI_MSIX_IRQ_INDEX: + intr_mode = RTE_INTR_MODE_MSIX; + dev->intr_handle.type = RTE_INTR_HANDLE_VFIO_MSIX; + break; + case VFIO_PCI_MSI_IRQ_INDEX: + intr_mode = RTE_INTR_MODE_MSI; + dev->intr_handle.type = RTE_INTR_HANDLE_VFIO_MSI; + break; + case VFIO_PCI_INTX_IRQ_INDEX: + intr_mode = RTE_INTR_MODE_LEGACY; + dev->intr_handle.type = RTE_INTR_HANDLE_VFIO_LEGACY; + break; + default: + RTE_LOG(ERR, EAL, " unknown interrupt type!\n"); + return -1; + } + + return 0; + } + + /* if we're here, we haven't found a suitable interrupt vector */ + return -1; +} + +static int +pci_vfio_is_ioport_bar(int vfio_dev_fd, int bar_index) +{ + uint32_t ioport_bar; + int ret; + + ret = pread64(vfio_dev_fd, &ioport_bar, sizeof(ioport_bar), + VFIO_GET_REGION_ADDR(VFIO_PCI_CONFIG_REGION_INDEX) + + PCI_BASE_ADDRESS_0 + bar_index*4); + if (ret != sizeof(ioport_bar)) { + RTE_LOG(ERR, EAL, "Cannot read command (%x) from config space!\n", + PCI_BASE_ADDRESS_0 + bar_index*4); + return -1; + } + + return (ioport_bar & PCI_BASE_ADDRESS_SPACE_IO) != 0; +} + +static int +pci_rte_vfio_setup_device(struct rte_pci_device *dev, int vfio_dev_fd) +{ + if (pci_vfio_setup_interrupts(dev, vfio_dev_fd) != 0) { + RTE_LOG(ERR, EAL, "Error setting up interrupts!\n"); + return -1; + } + + /* set bus mastering for the device */ + if (pci_vfio_set_bus_master(vfio_dev_fd, true)) { + RTE_LOG(ERR, EAL, "Cannot set up bus mastering!\n"); + return -1; + } + + /* + * Reset the device. If the device is not capable of resetting, + * then it updates errno as EINVAL. + */ + if (ioctl(vfio_dev_fd, VFIO_DEVICE_RESET) && errno != EINVAL) { + RTE_LOG(ERR, EAL, "Unable to reset device! Error: %d (%s)\n", + errno, strerror(errno)); + return -1; + } + + return 0; +} + +static int +pci_vfio_mmap_bar(int vfio_dev_fd, struct mapped_pci_resource *vfio_res, + int bar_index, int additional_flags) +{ + struct memreg { + unsigned long offset, size; + } memreg[2] = {}; + void *bar_addr; + struct pci_msix_table *msix_table = &vfio_res->msix_table; + struct pci_map *bar = &vfio_res->maps[bar_index]; + + if (bar->size == 0) + /* Skip this BAR */ + return 0; + + if (msix_table->bar_index == bar_index) { + /* + * VFIO will not let us map the MSI-X table, + * but we can map around it. + */ + uint32_t table_start = msix_table->offset; + uint32_t table_end = table_start + msix_table->size; + table_end = (table_end + ~PAGE_MASK) & PAGE_MASK; + table_start &= PAGE_MASK; + + if (table_start == 0 && table_end >= bar->size) { + /* Cannot map this BAR */ + RTE_LOG(DEBUG, EAL, "Skipping BAR%d\n", bar_index); + bar->size = 0; + bar->addr = 0; + return 0; + } + + memreg[0].offset = bar->offset; + memreg[0].size = table_start; + memreg[1].offset = bar->offset + table_end; + memreg[1].size = bar->size - table_end; + + RTE_LOG(DEBUG, EAL, + "Trying to map BAR%d that contains the MSI-X " + "table. Trying offsets: " + "0x%04lx:0x%04lx, 0x%04lx:0x%04lx\n", bar_index, + memreg[0].offset, memreg[0].size, + memreg[1].offset, memreg[1].size); + } else { + memreg[0].offset = bar->offset; + memreg[0].size = bar->size; + } + + /* reserve the address using an inaccessible mapping */ + bar_addr = mmap(bar->addr, bar->size, 0, MAP_PRIVATE | + MAP_ANONYMOUS | additional_flags, -1, 0); + if (bar_addr != MAP_FAILED) { + void *map_addr = NULL; + if (memreg[0].size) { + /* actual map of first part */ + map_addr = pci_map_resource(bar_addr, vfio_dev_fd, + memreg[0].offset, + memreg[0].size, + MAP_FIXED); + } + + /* if there's a second part, try to map it */ + if (map_addr != MAP_FAILED + && memreg[1].offset && memreg[1].size) { + void *second_addr = RTE_PTR_ADD(bar_addr, + memreg[1].offset - + (uintptr_t)bar->offset); + map_addr = pci_map_resource(second_addr, + vfio_dev_fd, + memreg[1].offset, + memreg[1].size, + MAP_FIXED); + } + + if (map_addr == MAP_FAILED || !map_addr) { + munmap(bar_addr, bar->size); + bar_addr = MAP_FAILED; + RTE_LOG(ERR, EAL, "Failed to map pci BAR%d\n", + bar_index); + return -1; + } + } else { + RTE_LOG(ERR, EAL, + "Failed to create inaccessible mapping for BAR%d\n", + bar_index); + return -1; + } + + bar->addr = bar_addr; + return 0; +} + +static int +pci_vfio_map_resource_primary(struct rte_pci_device *dev) +{ + struct vfio_device_info device_info = { .argsz = sizeof(device_info) }; + char pci_addr[PATH_MAX] = {0}; + int vfio_dev_fd; + struct rte_pci_addr *loc = &dev->addr; + int i, ret; + struct mapped_pci_resource *vfio_res = NULL; + struct mapped_pci_res_list *vfio_res_list = + RTE_TAILQ_CAST(rte_vfio_tailq.head, mapped_pci_res_list); + + struct pci_map *maps; + + dev->intr_handle.fd = -1; + + /* store PCI address string */ + snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT, + loc->domain, loc->bus, loc->devid, loc->function); + + ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr, + &vfio_dev_fd, &device_info); + if (ret) + return ret; + + /* allocate vfio_res and get region info */ + vfio_res = rte_zmalloc("VFIO_RES", sizeof(*vfio_res), 0); + if (vfio_res == NULL) { + RTE_LOG(ERR, EAL, + "%s(): cannot store uio mmap details\n", __func__); + goto err_vfio_dev_fd; + } + memcpy(&vfio_res->pci_addr, &dev->addr, sizeof(vfio_res->pci_addr)); + + /* get number of registers (up to BAR5) */ + vfio_res->nb_maps = RTE_MIN((int) device_info.num_regions, + VFIO_PCI_BAR5_REGION_INDEX + 1); + + /* map BARs */ + maps = vfio_res->maps; + + vfio_res->msix_table.bar_index = -1; + /* get MSI-X BAR, if any (we have to know where it is because we can't + * easily mmap it when using VFIO) + */ + ret = pci_vfio_get_msix_bar(vfio_dev_fd, &vfio_res->msix_table); + if (ret < 0) { + RTE_LOG(ERR, EAL, " %s cannot get MSI-X BAR number!\n", + pci_addr); + goto err_vfio_dev_fd; + } + + for (i = 0; i < (int) vfio_res->nb_maps; i++) { + struct vfio_region_info reg = { .argsz = sizeof(reg) }; + void *bar_addr; + + reg.index = i; + + ret = ioctl(vfio_dev_fd, VFIO_DEVICE_GET_REGION_INFO, ®); + if (ret) { + RTE_LOG(ERR, EAL, " %s cannot get device region info " + "error %i (%s)\n", pci_addr, errno, strerror(errno)); + goto err_vfio_res; + } + + /* chk for io port region */ + ret = pci_vfio_is_ioport_bar(vfio_dev_fd, i); + if (ret < 0) + goto err_vfio_res; + else if (ret) { + RTE_LOG(INFO, EAL, "Ignore mapping IO port bar(%d)\n", + i); + continue; + } + + /* skip non-mmapable BARs */ + if ((reg.flags & VFIO_REGION_INFO_FLAG_MMAP) == 0) + continue; + + /* try mapping somewhere close to the end of hugepages */ + if (pci_map_addr == NULL) + pci_map_addr = pci_find_max_end_va(); + + bar_addr = pci_map_addr; + pci_map_addr = RTE_PTR_ADD(bar_addr, (size_t) reg.size); + + maps[i].addr = bar_addr; + maps[i].offset = reg.offset; + maps[i].size = reg.size; + maps[i].path = NULL; /* vfio doesn't have per-resource paths */ + + ret = pci_vfio_mmap_bar(vfio_dev_fd, vfio_res, i, 0); + if (ret < 0) { + RTE_LOG(ERR, EAL, " %s mapping BAR%i failed: %s\n", + pci_addr, i, strerror(errno)); + goto err_vfio_res; + } + + dev->mem_resource[i].addr = maps[i].addr; + } + + if (pci_rte_vfio_setup_device(dev, vfio_dev_fd) < 0) { + RTE_LOG(ERR, EAL, " %s setup device failed\n", pci_addr); + goto err_vfio_res; + } + + TAILQ_INSERT_TAIL(vfio_res_list, vfio_res, next); + + return 0; +err_vfio_res: + rte_free(vfio_res); +err_vfio_dev_fd: + close(vfio_dev_fd); + return -1; +} + +static int +pci_vfio_map_resource_secondary(struct rte_pci_device *dev) +{ + struct vfio_device_info device_info = { .argsz = sizeof(device_info) }; + char pci_addr[PATH_MAX] = {0}; + int vfio_dev_fd; + struct rte_pci_addr *loc = &dev->addr; + int i, ret; + struct mapped_pci_resource *vfio_res = NULL; + struct mapped_pci_res_list *vfio_res_list = + RTE_TAILQ_CAST(rte_vfio_tailq.head, mapped_pci_res_list); + + struct pci_map *maps; + + dev->intr_handle.fd = -1; + + /* store PCI address string */ + snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT, + loc->domain, loc->bus, loc->devid, loc->function); + + ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr, + &vfio_dev_fd, &device_info); + if (ret) + return ret; + + /* if we're in a secondary process, just find our tailq entry */ + TAILQ_FOREACH(vfio_res, vfio_res_list, next) { + if (rte_pci_addr_cmp(&vfio_res->pci_addr, + &dev->addr)) + continue; + break; + } + /* if we haven't found our tailq entry, something's wrong */ + if (vfio_res == NULL) { + RTE_LOG(ERR, EAL, " %s cannot find TAILQ entry for PCI device!\n", + pci_addr); + goto err_vfio_dev_fd; + } + + /* map BARs */ + maps = vfio_res->maps; + + for (i = 0; i < (int) vfio_res->nb_maps; i++) { + ret = pci_vfio_mmap_bar(vfio_dev_fd, vfio_res, i, MAP_FIXED); + if (ret < 0) { + RTE_LOG(ERR, EAL, " %s mapping BAR%i failed: %s\n", + pci_addr, i, strerror(errno)); + goto err_vfio_dev_fd; + } + + dev->mem_resource[i].addr = maps[i].addr; + } + + return 0; +err_vfio_dev_fd: + close(vfio_dev_fd); + return -1; +} + +/* + * map the PCI resources of a PCI device in virtual memory (VFIO version). + * primary and secondary processes follow almost exactly the same path + */ +int +pci_vfio_map_resource(struct rte_pci_device *dev) +{ + if (rte_eal_process_type() == RTE_PROC_PRIMARY) + return pci_vfio_map_resource_primary(dev); + else + return pci_vfio_map_resource_secondary(dev); +} + +int +pci_vfio_unmap_resource(struct rte_pci_device *dev) +{ + char pci_addr[PATH_MAX] = {0}; + struct rte_pci_addr *loc = &dev->addr; + int i, ret; + struct mapped_pci_resource *vfio_res = NULL; + struct mapped_pci_res_list *vfio_res_list; + + struct pci_map *maps; + + /* store PCI address string */ + snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT, + loc->domain, loc->bus, loc->devid, loc->function); + + + if (close(dev->intr_handle.fd) < 0) { + RTE_LOG(INFO, EAL, "Error when closing eventfd file descriptor for %s\n", + pci_addr); + return -1; + } + + if (pci_vfio_set_bus_master(dev->intr_handle.vfio_dev_fd, false)) { + RTE_LOG(ERR, EAL, " %s cannot unset bus mastering for PCI device!\n", + pci_addr); + return -1; + } + + ret = rte_vfio_release_device(rte_pci_get_sysfs_path(), pci_addr, + dev->intr_handle.vfio_dev_fd); + if (ret < 0) { + RTE_LOG(ERR, EAL, + "%s(): cannot release device\n", __func__); + return ret; + } + + vfio_res_list = RTE_TAILQ_CAST(rte_vfio_tailq.head, mapped_pci_res_list); + /* Get vfio_res */ + TAILQ_FOREACH(vfio_res, vfio_res_list, next) { + if (memcmp(&vfio_res->pci_addr, &dev->addr, sizeof(dev->addr))) + continue; + break; + } + /* if we haven't found our tailq entry, something's wrong */ + if (vfio_res == NULL) { + RTE_LOG(ERR, EAL, " %s cannot find TAILQ entry for PCI device!\n", + pci_addr); + return -1; + } + + /* unmap BARs */ + maps = vfio_res->maps; + + RTE_LOG(INFO, EAL, "Releasing pci mapped resource for %s\n", + pci_addr); + for (i = 0; i < (int) vfio_res->nb_maps; i++) { + + /* + * We do not need to be aware of MSI-X table BAR mappings as + * when mapping. Just using current maps array is enough + */ + if (maps[i].addr) { + RTE_LOG(INFO, EAL, "Calling pci_unmap_resource for %s at %p\n", + pci_addr, maps[i].addr); + pci_unmap_resource(maps[i].addr, maps[i].size); + } + } + + TAILQ_REMOVE(vfio_res_list, vfio_res, next); + + return 0; +} + +int +pci_vfio_ioport_map(struct rte_pci_device *dev, int bar, + struct rte_pci_ioport *p) +{ + if (bar < VFIO_PCI_BAR0_REGION_INDEX || + bar > VFIO_PCI_BAR5_REGION_INDEX) { + RTE_LOG(ERR, EAL, "invalid bar (%d)!\n", bar); + return -1; + } + + p->dev = dev; + p->base = VFIO_GET_REGION_ADDR(bar); + return 0; +} + +void +pci_vfio_ioport_read(struct rte_pci_ioport *p, + void *data, size_t len, off_t offset) +{ + const struct rte_intr_handle *intr_handle = &p->dev->intr_handle; + + if (pread64(intr_handle->vfio_dev_fd, data, + len, p->base + offset) <= 0) + RTE_LOG(ERR, EAL, + "Can't read from PCI bar (%" PRIu64 ") : offset (%x)\n", + VFIO_GET_REGION_IDX(p->base), (int)offset); +} + +void +pci_vfio_ioport_write(struct rte_pci_ioport *p, + const void *data, size_t len, off_t offset) +{ + const struct rte_intr_handle *intr_handle = &p->dev->intr_handle; + + if (pwrite64(intr_handle->vfio_dev_fd, data, + len, p->base + offset) <= 0) + RTE_LOG(ERR, EAL, + "Can't write to PCI bar (%" PRIu64 ") : offset (%x)\n", + VFIO_GET_REGION_IDX(p->base), (int)offset); +} + +int +pci_vfio_ioport_unmap(struct rte_pci_ioport *p) +{ + RTE_SET_USED(p); + return -1; +} + +int +pci_vfio_is_enabled(void) +{ + return rte_vfio_is_enabled("vfio_pci"); +} +#endif diff --git a/dpdk/drivers/bus/pci/pci_common.c b/dpdk/drivers/bus/pci/pci_common.c new file mode 100644 index 00000000..104fdf90 --- /dev/null +++ b/dpdk/drivers/bus/pci/pci_common.c @@ -0,0 +1,540 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * Copyright 2013-2014 6WIND S.A. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "private.h" + +extern struct rte_pci_bus rte_pci_bus; + +#define SYSFS_PCI_DEVICES "/sys/bus/pci/devices" + +const char *rte_pci_get_sysfs_path(void) +{ + const char *path = NULL; + + path = getenv("SYSFS_PCI_DEVICES"); + if (path == NULL) + return SYSFS_PCI_DEVICES; + + return path; +} + +static struct rte_devargs *pci_devargs_lookup(struct rte_pci_device *dev) +{ + struct rte_devargs *devargs; + struct rte_pci_addr addr; + struct rte_bus *pbus; + + pbus = rte_bus_find_by_name("pci"); + TAILQ_FOREACH(devargs, &devargs_list, next) { + if (devargs->bus != pbus) + continue; + devargs->bus->parse(devargs->name, &addr); + if (!rte_pci_addr_cmp(&dev->addr, &addr)) + return devargs; + } + return NULL; +} + +void +pci_name_set(struct rte_pci_device *dev) +{ + struct rte_devargs *devargs; + + /* Each device has its internal, canonical name set. */ + rte_pci_device_name(&dev->addr, + dev->name, sizeof(dev->name)); + devargs = pci_devargs_lookup(dev); + dev->device.devargs = devargs; + /* In blacklist mode, if the device is not blacklisted, no + * rte_devargs exists for it. + */ + if (devargs != NULL) + /* If an rte_devargs exists, the generic rte_device uses the + * given name as its namea + */ + dev->device.name = dev->device.devargs->name; + else + /* Otherwise, it uses the internal, canonical form. */ + dev->device.name = dev->name; +} + +/* + * Match the PCI Driver and Device using the ID Table + */ +int +rte_pci_match(const struct rte_pci_driver *pci_drv, + const struct rte_pci_device *pci_dev) +{ + const struct rte_pci_id *id_table; + + for (id_table = pci_drv->id_table; id_table->vendor_id != 0; + id_table++) { + /* check if device's identifiers match the driver's ones */ + if (id_table->vendor_id != pci_dev->id.vendor_id && + id_table->vendor_id != PCI_ANY_ID) + continue; + if (id_table->device_id != pci_dev->id.device_id && + id_table->device_id != PCI_ANY_ID) + continue; + if (id_table->subsystem_vendor_id != + pci_dev->id.subsystem_vendor_id && + id_table->subsystem_vendor_id != PCI_ANY_ID) + continue; + if (id_table->subsystem_device_id != + pci_dev->id.subsystem_device_id && + id_table->subsystem_device_id != PCI_ANY_ID) + continue; + if (id_table->class_id != pci_dev->id.class_id && + id_table->class_id != RTE_CLASS_ANY_ID) + continue; + + return 1; + } + + return 0; +} + +/* + * If vendor/device ID match, call the probe() function of the + * driver. + */ +static int +rte_pci_probe_one_driver(struct rte_pci_driver *dr, + struct rte_pci_device *dev) +{ + int ret; + struct rte_pci_addr *loc; + + if ((dr == NULL) || (dev == NULL)) + return -EINVAL; + + loc = &dev->addr; + + /* The device is not blacklisted; Check if driver supports it */ + if (!rte_pci_match(dr, dev)) + /* Match of device and driver failed */ + return 1; + + RTE_LOG(INFO, EAL, "PCI device "PCI_PRI_FMT" on NUMA socket %i\n", + loc->domain, loc->bus, loc->devid, loc->function, + dev->device.numa_node); + + /* no initialization when blacklisted, return without error */ + if (dev->device.devargs != NULL && + dev->device.devargs->policy == + RTE_DEV_BLACKLISTED) { + RTE_LOG(INFO, EAL, " Device is blacklisted, not" + " initializing\n"); + return 1; + } + + if (dev->device.numa_node < 0) { + RTE_LOG(WARNING, EAL, " Invalid NUMA socket, default to 0\n"); + dev->device.numa_node = 0; + } + + RTE_LOG(INFO, EAL, " probe driver: %x:%x %s\n", dev->id.vendor_id, + dev->id.device_id, dr->driver.name); + + if (dr->drv_flags & RTE_PCI_DRV_NEED_MAPPING) { + /* map resources for devices that use igb_uio */ + ret = rte_pci_map_device(dev); + if (ret != 0) + return ret; + } + + /* reference driver structure */ + dev->driver = dr; + dev->device.driver = &dr->driver; + + /* call the driver probe() function */ + ret = dr->probe(dr, dev); + if (ret) { + dev->driver = NULL; + dev->device.driver = NULL; + if ((dr->drv_flags & RTE_PCI_DRV_NEED_MAPPING) && + /* Don't unmap if device is unsupported and + * driver needs mapped resources. + */ + !(ret > 0 && + (dr->drv_flags & RTE_PCI_DRV_KEEP_MAPPED_RES))) + rte_pci_unmap_device(dev); + } + + return ret; +} + +/* + * If vendor/device ID match, call the remove() function of the + * driver. + */ +static int +rte_pci_detach_dev(struct rte_pci_device *dev) +{ + struct rte_pci_addr *loc; + struct rte_pci_driver *dr; + int ret = 0; + + if (dev == NULL) + return -EINVAL; + + dr = dev->driver; + loc = &dev->addr; + + RTE_LOG(DEBUG, EAL, "PCI device "PCI_PRI_FMT" on NUMA socket %i\n", + loc->domain, loc->bus, loc->devid, + loc->function, dev->device.numa_node); + + RTE_LOG(DEBUG, EAL, " remove driver: %x:%x %s\n", dev->id.vendor_id, + dev->id.device_id, dr->driver.name); + + if (dr->remove) { + ret = dr->remove(dev); + if (ret < 0) + return ret; + } + + /* clear driver structure */ + dev->driver = NULL; + + if (dr->drv_flags & RTE_PCI_DRV_NEED_MAPPING) + /* unmap resources for devices that use igb_uio */ + rte_pci_unmap_device(dev); + + return 0; +} + +/* + * If vendor/device ID match, call the probe() function of all + * registered driver for the given device. Return -1 if initialization + * failed, return 1 if no driver is found for this device. + */ +static int +pci_probe_all_drivers(struct rte_pci_device *dev) +{ + struct rte_pci_driver *dr = NULL; + int rc = 0; + + if (dev == NULL) + return -1; + + /* Check if a driver is already loaded */ + if (dev->driver != NULL) + return 0; + + FOREACH_DRIVER_ON_PCIBUS(dr) { + rc = rte_pci_probe_one_driver(dr, dev); + if (rc < 0) + /* negative value is an error */ + return -1; + if (rc > 0) + /* positive value means driver doesn't support it */ + continue; + return 0; + } + return 1; +} + +/* + * Find the pci device specified by pci address, then invoke probe function of + * the driver of the device. + */ +int +rte_pci_probe_one(const struct rte_pci_addr *addr) +{ + struct rte_pci_device *dev = NULL; + + int ret = 0; + + if (addr == NULL) + return -1; + + /* update current pci device in global list, kernel bindings might have + * changed since last time we looked at it. + */ + if (pci_update_device(addr) < 0) + goto err_return; + + FOREACH_DEVICE_ON_PCIBUS(dev) { + if (rte_pci_addr_cmp(&dev->addr, addr)) + continue; + + ret = pci_probe_all_drivers(dev); + if (ret) + goto err_return; + return 0; + } + return -1; + +err_return: + RTE_LOG(WARNING, EAL, + "Requested device " PCI_PRI_FMT " cannot be used\n", + addr->domain, addr->bus, addr->devid, addr->function); + return -1; +} + +/* + * Detach device specified by its pci address. + */ +int +rte_pci_detach(const struct rte_pci_addr *addr) +{ + struct rte_pci_device *dev = NULL; + int ret = 0; + + if (addr == NULL) + return -1; + + FOREACH_DEVICE_ON_PCIBUS(dev) { + if (rte_pci_addr_cmp(&dev->addr, addr)) + continue; + + ret = rte_pci_detach_dev(dev); + if (ret < 0) + /* negative value is an error */ + goto err_return; + if (ret > 0) + /* positive value means driver doesn't support it */ + continue; + + rte_pci_remove_device(dev); + free(dev); + return 0; + } + return -1; + +err_return: + RTE_LOG(WARNING, EAL, "Requested device " PCI_PRI_FMT + " cannot be used\n", dev->addr.domain, dev->addr.bus, + dev->addr.devid, dev->addr.function); + return -1; +} + +/* + * Scan the content of the PCI bus, and call the probe() function for + * all registered drivers that have a matching entry in its id_table + * for discovered devices. + */ +int +rte_pci_probe(void) +{ + struct rte_pci_device *dev = NULL; + size_t probed = 0, failed = 0; + struct rte_devargs *devargs; + int probe_all = 0; + int ret = 0; + + if (rte_pci_bus.bus.conf.scan_mode != RTE_BUS_SCAN_WHITELIST) + probe_all = 1; + + FOREACH_DEVICE_ON_PCIBUS(dev) { + probed++; + + devargs = dev->device.devargs; + /* probe all or only whitelisted devices */ + if (probe_all) + ret = pci_probe_all_drivers(dev); + else if (devargs != NULL && + devargs->policy == RTE_DEV_WHITELISTED) + ret = pci_probe_all_drivers(dev); + if (ret < 0) { + RTE_LOG(ERR, EAL, "Requested device " PCI_PRI_FMT + " cannot be used\n", dev->addr.domain, dev->addr.bus, + dev->addr.devid, dev->addr.function); + rte_errno = errno; + failed++; + ret = 0; + } + } + + return (probed && probed == failed) ? -1 : 0; +} + +/* dump one device */ +static int +pci_dump_one_device(FILE *f, struct rte_pci_device *dev) +{ + int i; + + fprintf(f, PCI_PRI_FMT, dev->addr.domain, dev->addr.bus, + dev->addr.devid, dev->addr.function); + fprintf(f, " - vendor:%x device:%x\n", dev->id.vendor_id, + dev->id.device_id); + + for (i = 0; i != sizeof(dev->mem_resource) / + sizeof(dev->mem_resource[0]); i++) { + fprintf(f, " %16.16"PRIx64" %16.16"PRIx64"\n", + dev->mem_resource[i].phys_addr, + dev->mem_resource[i].len); + } + return 0; +} + +/* dump devices on the bus */ +void +rte_pci_dump(FILE *f) +{ + struct rte_pci_device *dev = NULL; + + FOREACH_DEVICE_ON_PCIBUS(dev) { + pci_dump_one_device(f, dev); + } +} + +static int +pci_parse(const char *name, void *addr) +{ + struct rte_pci_addr *out = addr; + struct rte_pci_addr pci_addr; + bool parse; + + parse = (rte_pci_addr_parse(name, &pci_addr) == 0); + if (parse && addr != NULL) + *out = pci_addr; + return parse == false; +} + +/* register a driver */ +void +rte_pci_register(struct rte_pci_driver *driver) +{ + TAILQ_INSERT_TAIL(&rte_pci_bus.driver_list, driver, next); + driver->bus = &rte_pci_bus; +} + +/* unregister a driver */ +void +rte_pci_unregister(struct rte_pci_driver *driver) +{ + TAILQ_REMOVE(&rte_pci_bus.driver_list, driver, next); + driver->bus = NULL; +} + +/* Add a device to PCI bus */ +void +rte_pci_add_device(struct rte_pci_device *pci_dev) +{ + TAILQ_INSERT_TAIL(&rte_pci_bus.device_list, pci_dev, next); +} + +/* Insert a device into a predefined position in PCI bus */ +void +rte_pci_insert_device(struct rte_pci_device *exist_pci_dev, + struct rte_pci_device *new_pci_dev) +{ + TAILQ_INSERT_BEFORE(exist_pci_dev, new_pci_dev, next); +} + +/* Remove a device from PCI bus */ +void +rte_pci_remove_device(struct rte_pci_device *pci_dev) +{ + TAILQ_REMOVE(&rte_pci_bus.device_list, pci_dev, next); +} + +static struct rte_device * +pci_find_device(const struct rte_device *start, rte_dev_cmp_t cmp, + const void *data) +{ + struct rte_pci_device *dev; + + FOREACH_DEVICE_ON_PCIBUS(dev) { + if (start && &dev->device == start) { + start = NULL; /* starting point found */ + continue; + } + if (cmp(&dev->device, data) == 0) + return &dev->device; + } + + return NULL; +} + +static int +pci_plug(struct rte_device *dev) +{ + return pci_probe_all_drivers(RTE_DEV_TO_PCI(dev)); +} + +static int +pci_unplug(struct rte_device *dev) +{ + struct rte_pci_device *pdev; + int ret; + + pdev = RTE_DEV_TO_PCI(dev); + ret = rte_pci_detach_dev(pdev); + if (ret == 0) { + rte_pci_remove_device(pdev); + free(pdev); + } + return ret; +} + +struct rte_pci_bus rte_pci_bus = { + .bus = { + .scan = rte_pci_scan, + .probe = rte_pci_probe, + .find_device = pci_find_device, + .plug = pci_plug, + .unplug = pci_unplug, + .parse = pci_parse, + .get_iommu_class = rte_pci_get_iommu_class, + }, + .device_list = TAILQ_HEAD_INITIALIZER(rte_pci_bus.device_list), + .driver_list = TAILQ_HEAD_INITIALIZER(rte_pci_bus.driver_list), +}; + +RTE_REGISTER_BUS(pci, rte_pci_bus.bus); diff --git a/dpdk/drivers/bus/pci/pci_common_uio.c b/dpdk/drivers/bus/pci/pci_common_uio.c new file mode 100644 index 00000000..b75988de --- /dev/null +++ b/dpdk/drivers/bus/pci/pci_common_uio.c @@ -0,0 +1,234 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "private.h" + +static struct rte_tailq_elem rte_uio_tailq = { + .name = "UIO_RESOURCE_LIST", +}; +EAL_REGISTER_TAILQ(rte_uio_tailq) + +static int +pci_uio_map_secondary(struct rte_pci_device *dev) +{ + int fd, i, j; + struct mapped_pci_resource *uio_res; + struct mapped_pci_res_list *uio_res_list = + RTE_TAILQ_CAST(rte_uio_tailq.head, mapped_pci_res_list); + + TAILQ_FOREACH(uio_res, uio_res_list, next) { + + /* skip this element if it doesn't match our PCI address */ + if (rte_pci_addr_cmp(&uio_res->pci_addr, &dev->addr)) + continue; + + for (i = 0; i != uio_res->nb_maps; i++) { + /* + * open devname, to mmap it + */ + fd = open(uio_res->maps[i].path, O_RDWR); + if (fd < 0) { + RTE_LOG(ERR, EAL, "Cannot open %s: %s\n", + uio_res->maps[i].path, strerror(errno)); + return -1; + } + + void *mapaddr = pci_map_resource(uio_res->maps[i].addr, + fd, (off_t)uio_res->maps[i].offset, + (size_t)uio_res->maps[i].size, 0); + /* fd is not needed in slave process, close it */ + close(fd); + if (mapaddr != uio_res->maps[i].addr) { + RTE_LOG(ERR, EAL, + "Cannot mmap device resource file %s to address: %p\n", + uio_res->maps[i].path, + uio_res->maps[i].addr); + if (mapaddr != MAP_FAILED) { + /* unmap addrs correctly mapped */ + for (j = 0; j < i; j++) + pci_unmap_resource( + uio_res->maps[j].addr, + (size_t)uio_res->maps[j].size); + /* unmap addr wrongly mapped */ + pci_unmap_resource(mapaddr, + (size_t)uio_res->maps[i].size); + } + return -1; + } + } + return 0; + } + + RTE_LOG(ERR, EAL, "Cannot find resource for device\n"); + return 1; +} + +/* map the PCI resource of a PCI device in virtual memory */ +int +pci_uio_map_resource(struct rte_pci_device *dev) +{ + int i, map_idx = 0, ret; + uint64_t phaddr; + struct mapped_pci_resource *uio_res = NULL; + struct mapped_pci_res_list *uio_res_list = + RTE_TAILQ_CAST(rte_uio_tailq.head, mapped_pci_res_list); + + dev->intr_handle.fd = -1; + dev->intr_handle.uio_cfg_fd = -1; + + /* secondary processes - use already recorded details */ + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return pci_uio_map_secondary(dev); + + /* allocate uio resource */ + ret = pci_uio_alloc_resource(dev, &uio_res); + if (ret) + return ret; + + /* Map all BARs */ + for (i = 0; i != PCI_MAX_RESOURCE; i++) { + /* skip empty BAR */ + phaddr = dev->mem_resource[i].phys_addr; + if (phaddr == 0) + continue; + + ret = pci_uio_map_resource_by_index(dev, i, + uio_res, map_idx); + if (ret) + goto error; + + map_idx++; + } + + uio_res->nb_maps = map_idx; + + TAILQ_INSERT_TAIL(uio_res_list, uio_res, next); + + return 0; +error: + for (i = 0; i < map_idx; i++) { + pci_unmap_resource(uio_res->maps[i].addr, + (size_t)uio_res->maps[i].size); + rte_free(uio_res->maps[i].path); + } + pci_uio_free_resource(dev, uio_res); + return -1; +} + +static void +pci_uio_unmap(struct mapped_pci_resource *uio_res) +{ + int i; + + if (uio_res == NULL) + return; + + for (i = 0; i != uio_res->nb_maps; i++) { + pci_unmap_resource(uio_res->maps[i].addr, + (size_t)uio_res->maps[i].size); + if (rte_eal_process_type() == RTE_PROC_PRIMARY) + rte_free(uio_res->maps[i].path); + } +} + +static struct mapped_pci_resource * +pci_uio_find_resource(struct rte_pci_device *dev) +{ + struct mapped_pci_resource *uio_res; + struct mapped_pci_res_list *uio_res_list = + RTE_TAILQ_CAST(rte_uio_tailq.head, mapped_pci_res_list); + + if (dev == NULL) + return NULL; + + TAILQ_FOREACH(uio_res, uio_res_list, next) { + + /* skip this element if it doesn't match our PCI address */ + if (!rte_pci_addr_cmp(&uio_res->pci_addr, &dev->addr)) + return uio_res; + } + return NULL; +} + +/* unmap the PCI resource of a PCI device in virtual memory */ +void +pci_uio_unmap_resource(struct rte_pci_device *dev) +{ + struct mapped_pci_resource *uio_res; + struct mapped_pci_res_list *uio_res_list = + RTE_TAILQ_CAST(rte_uio_tailq.head, mapped_pci_res_list); + + if (dev == NULL) + return; + + /* find an entry for the device */ + uio_res = pci_uio_find_resource(dev); + if (uio_res == NULL) + return; + + /* secondary processes - just free maps */ + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return pci_uio_unmap(uio_res); + + TAILQ_REMOVE(uio_res_list, uio_res, next); + + /* unmap all resources */ + pci_uio_unmap(uio_res); + + /* free uio resource */ + rte_free(uio_res); + + /* close fd if in primary process */ + close(dev->intr_handle.fd); + if (dev->intr_handle.uio_cfg_fd >= 0) { + close(dev->intr_handle.uio_cfg_fd); + dev->intr_handle.uio_cfg_fd = -1; + } + + dev->intr_handle.fd = -1; + dev->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN; +} diff --git a/dpdk/drivers/bus/pci/private.h b/dpdk/drivers/bus/pci/private.h new file mode 100644 index 00000000..2283f093 --- /dev/null +++ b/dpdk/drivers/bus/pci/private.h @@ -0,0 +1,248 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 6WIND. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of 6WIND nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _PCI_PRIVATE_H_ +#define _PCI_PRIVATE_H_ + +#include +#include +#include +#include + +struct rte_pci_driver; +struct rte_pci_device; + +/** + * Probe the PCI bus + * + * @return + * - 0 on success. + * - !0 on error. + */ +int +rte_pci_probe(void); + +/** + * Scan the content of the PCI bus, and the devices in the devices + * list + * + * @return + * 0 on success, negative on error + */ +int rte_pci_scan(void); + +/** + * Probe the single PCI device. + * + * Scan the content of the PCI bus, and find the pci device specified by pci + * address, then call the probe() function for registered driver that has a + * matching entry in its id_table for discovered device. + * + * @param addr + * The PCI Bus-Device-Function address to probe. + * @return + * - 0 on success. + * - Negative on error. + */ +int rte_pci_probe_one(const struct rte_pci_addr *addr); + +/** + * Close the single PCI device. + * + * Scan the content of the PCI bus, and find the pci device specified by pci + * address, then call the remove() function for registered driver that has a + * matching entry in its id_table for discovered device. + * + * @param addr + * The PCI Bus-Device-Function address to close. + * @return + * - 0 on success. + * - Negative on error. + */ +int rte_pci_detach(const struct rte_pci_addr *addr); + +/** + * Find the name of a PCI device. + */ +void +pci_name_set(struct rte_pci_device *dev); + +/** + * Add a PCI device to the PCI Bus (append to PCI Device list). This function + * also updates the bus references of the PCI Device (and the generic device + * object embedded within. + * + * @param pci_dev + * PCI device to add + * @return void + */ +void rte_pci_add_device(struct rte_pci_device *pci_dev); + +/** + * Insert a PCI device in the PCI Bus at a particular location in the device + * list. It also updates the PCI Bus reference of the new devices to be + * inserted. + * + * @param exist_pci_dev + * Existing PCI device in PCI Bus + * @param new_pci_dev + * PCI device to be added before exist_pci_dev + * @return void + */ +void rte_pci_insert_device(struct rte_pci_device *exist_pci_dev, + struct rte_pci_device *new_pci_dev); + +/** + * Remove a PCI device from the PCI Bus. This sets to NULL the bus references + * in the PCI device object as well as the generic device object. + * + * @param pci_device + * PCI device to be removed from PCI Bus + * @return void + */ +void rte_pci_remove_device(struct rte_pci_device *pci_device); + +/** + * Update a pci device object by asking the kernel for the latest information. + * + * This function is private to EAL. + * + * @param addr + * The PCI Bus-Device-Function address to look for + * @return + * - 0 on success. + * - negative on error. + */ +int pci_update_device(const struct rte_pci_addr *addr); + +/** + * Unbind kernel driver for this device + * + * This function is private to EAL. + * + * @return + * 0 on success, negative on error + */ +int pci_unbind_kernel_driver(struct rte_pci_device *dev); + +/** + * Map the PCI resource of a PCI device in virtual memory + * + * This function is private to EAL. + * + * @return + * 0 on success, negative on error + */ +int pci_uio_map_resource(struct rte_pci_device *dev); + +/** + * Unmap the PCI resource of a PCI device + * + * This function is private to EAL. + */ +void pci_uio_unmap_resource(struct rte_pci_device *dev); + +/** + * Allocate uio resource for PCI device + * + * This function is private to EAL. + * + * @param dev + * PCI device to allocate uio resource + * @param uio_res + * Pointer to uio resource. + * If the function returns 0, the pointer will be filled. + * @return + * 0 on success, negative on error + */ +int pci_uio_alloc_resource(struct rte_pci_device *dev, + struct mapped_pci_resource **uio_res); + +/** + * Free uio resource for PCI device + * + * This function is private to EAL. + * + * @param dev + * PCI device to free uio resource + * @param uio_res + * Pointer to uio resource. + */ +void pci_uio_free_resource(struct rte_pci_device *dev, + struct mapped_pci_resource *uio_res); + +/** + * Map device memory to uio resource + * + * This function is private to EAL. + * + * @param dev + * PCI device that has memory information. + * @param res_idx + * Memory resource index of the PCI device. + * @param uio_res + * uio resource that will keep mapping information. + * @param map_idx + * Mapping information index of the uio resource. + * @return + * 0 on success, negative on error + */ +int pci_uio_map_resource_by_index(struct rte_pci_device *dev, int res_idx, + struct mapped_pci_resource *uio_res, int map_idx); + +/* + * Match the PCI Driver and Device using the ID Table + * + * @param pci_drv + * PCI driver from which ID table would be extracted + * @param pci_dev + * PCI device to match against the driver + * @return + * 1 for successful match + * 0 for unsuccessful match + */ +int +rte_pci_match(const struct rte_pci_driver *pci_drv, + const struct rte_pci_device *pci_dev); + +/** + * Get iommu class of PCI devices on the bus. + * And return their preferred iova mapping mode. + * + * @return + * - enum rte_iova_mode. + */ +enum rte_iova_mode +rte_pci_get_iommu_class(void); + +#endif /* _PCI_PRIVATE_H_ */ diff --git a/dpdk/drivers/bus/pci/rte_bus_pci.h b/dpdk/drivers/bus/pci/rte_bus_pci.h new file mode 100644 index 00000000..d4a29964 --- /dev/null +++ b/dpdk/drivers/bus/pci/rte_bus_pci.h @@ -0,0 +1,340 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * Copyright 2013-2014 6WIND S.A. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_BUS_PCI_H_ +#define _RTE_BUS_PCI_H_ + +/** + * @file + * + * RTE PCI Bus Interface + */ + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +/** Pathname of PCI devices directory. */ +const char *rte_pci_get_sysfs_path(void); + +/* Forward declarations */ +struct rte_pci_device; +struct rte_pci_driver; + +/** List of PCI devices */ +TAILQ_HEAD(rte_pci_device_list, rte_pci_device); +/** List of PCI drivers */ +TAILQ_HEAD(rte_pci_driver_list, rte_pci_driver); + +/* PCI Bus iterators */ +#define FOREACH_DEVICE_ON_PCIBUS(p) \ + TAILQ_FOREACH(p, &(rte_pci_bus.device_list), next) + +#define FOREACH_DRIVER_ON_PCIBUS(p) \ + TAILQ_FOREACH(p, &(rte_pci_bus.driver_list), next) + +struct rte_devargs; + +/** + * A structure describing a PCI device. + */ +struct rte_pci_device { + TAILQ_ENTRY(rte_pci_device) next; /**< Next probed PCI device. */ + struct rte_device device; /**< Inherit core device */ + struct rte_pci_addr addr; /**< PCI location. */ + struct rte_pci_id id; /**< PCI ID. */ + struct rte_mem_resource mem_resource[PCI_MAX_RESOURCE]; + /**< PCI Memory Resource */ + struct rte_intr_handle intr_handle; /**< Interrupt handle */ + struct rte_pci_driver *driver; /**< Associated driver */ + uint16_t max_vfs; /**< sriov enable if not zero */ + enum rte_kernel_driver kdrv; /**< Kernel driver passthrough */ + char name[PCI_PRI_STR_SIZE+1]; /**< PCI location (ASCII) */ +}; + +/** + * @internal + * Helper macro for drivers that need to convert to struct rte_pci_device. + */ +#define RTE_DEV_TO_PCI(ptr) container_of(ptr, struct rte_pci_device, device) + +#define RTE_ETH_DEV_TO_PCI(eth_dev) RTE_DEV_TO_PCI((eth_dev)->device) + +/** Any PCI device identifier (vendor, device, ...) */ +#define PCI_ANY_ID (0xffff) +#define RTE_CLASS_ANY_ID (0xffffff) + +#ifdef __cplusplus +/** C++ macro used to help building up tables of device IDs */ +#define RTE_PCI_DEVICE(vend, dev) \ + RTE_CLASS_ANY_ID, \ + (vend), \ + (dev), \ + PCI_ANY_ID, \ + PCI_ANY_ID +#else +/** Macro used to help building up tables of device IDs */ +#define RTE_PCI_DEVICE(vend, dev) \ + .class_id = RTE_CLASS_ANY_ID, \ + .vendor_id = (vend), \ + .device_id = (dev), \ + .subsystem_vendor_id = PCI_ANY_ID, \ + .subsystem_device_id = PCI_ANY_ID +#endif + +/** + * Initialisation function for the driver called during PCI probing. + */ +typedef int (pci_probe_t)(struct rte_pci_driver *, struct rte_pci_device *); + +/** + * Uninitialisation function for the driver called during hotplugging. + */ +typedef int (pci_remove_t)(struct rte_pci_device *); + +/** + * A structure describing a PCI driver. + */ +struct rte_pci_driver { + TAILQ_ENTRY(rte_pci_driver) next; /**< Next in list. */ + struct rte_driver driver; /**< Inherit core driver. */ + struct rte_pci_bus *bus; /**< PCI bus reference. */ + pci_probe_t *probe; /**< Device Probe function. */ + pci_remove_t *remove; /**< Device Remove function. */ + const struct rte_pci_id *id_table; /**< ID table, NULL terminated. */ + uint32_t drv_flags; /**< Flags contolling handling of device. */ +}; + +/** + * Structure describing the PCI bus + */ +struct rte_pci_bus { + struct rte_bus bus; /**< Inherit the generic class */ + struct rte_pci_device_list device_list; /**< List of PCI devices */ + struct rte_pci_driver_list driver_list; /**< List of PCI drivers */ +}; + +/** Device needs PCI BAR mapping (done with either IGB_UIO or VFIO) */ +#define RTE_PCI_DRV_NEED_MAPPING 0x0001 +/** Device driver supports link state interrupt */ +#define RTE_PCI_DRV_INTR_LSC 0x0008 +/** Device driver supports device removal interrupt */ +#define RTE_PCI_DRV_INTR_RMV 0x0010 +/** Device driver needs to keep mapped resources if unsupported dev detected */ +#define RTE_PCI_DRV_KEEP_MAPPED_RES 0x0020 +/** Device driver supports IOVA as VA */ +#define RTE_PCI_DRV_IOVA_AS_VA 0X0040 + +/** + * Map the PCI device resources in user space virtual memory address + * + * Note that driver should not call this function when flag + * RTE_PCI_DRV_NEED_MAPPING is set, as EAL will do that for + * you when it's on. + * + * @param dev + * A pointer to a rte_pci_device structure describing the device + * to use + * + * @return + * 0 on success, negative on error and positive if no driver + * is found for the device. + */ +int rte_pci_map_device(struct rte_pci_device *dev); + +/** + * Unmap this device + * + * @param dev + * A pointer to a rte_pci_device structure describing the device + * to use + */ +void rte_pci_unmap_device(struct rte_pci_device *dev); + +/** + * Dump the content of the PCI bus. + * + * @param f + * A pointer to a file for output + */ +void rte_pci_dump(FILE *f); + +/** + * Register a PCI driver. + * + * @param driver + * A pointer to a rte_pci_driver structure describing the driver + * to be registered. + */ +void rte_pci_register(struct rte_pci_driver *driver); + +/** Helper for PCI device registration from driver (eth, crypto) instance */ +#define RTE_PMD_REGISTER_PCI(nm, pci_drv) \ +RTE_INIT(pciinitfn_ ##nm); \ +static void pciinitfn_ ##nm(void) \ +{\ + (pci_drv).driver.name = RTE_STR(nm);\ + rte_pci_register(&pci_drv); \ +} \ +RTE_PMD_EXPORT_NAME(nm, __COUNTER__) + +/** + * Unregister a PCI driver. + * + * @param driver + * A pointer to a rte_pci_driver structure describing the driver + * to be unregistered. + */ +void rte_pci_unregister(struct rte_pci_driver *driver); + +/** + * Read PCI config space. + * + * @param device + * A pointer to a rte_pci_device structure describing the device + * to use + * @param buf + * A data buffer where the bytes should be read into + * @param len + * The length of the data buffer. + * @param offset + * The offset into PCI config space + */ +int rte_pci_read_config(const struct rte_pci_device *device, + void *buf, size_t len, off_t offset); + +/** + * Write PCI config space. + * + * @param device + * A pointer to a rte_pci_device structure describing the device + * to use + * @param buf + * A data buffer containing the bytes should be written + * @param len + * The length of the data buffer. + * @param offset + * The offset into PCI config space + */ +int rte_pci_write_config(const struct rte_pci_device *device, + const void *buf, size_t len, off_t offset); + +/** + * A structure used to access io resources for a pci device. + * rte_pci_ioport is arch, os, driver specific, and should not be used outside + * of pci ioport api. + */ +struct rte_pci_ioport { + struct rte_pci_device *dev; + uint64_t base; + uint64_t len; /* only filled for memory mapped ports */ +}; + +/** + * Initialize a rte_pci_ioport object for a pci device io resource. + * + * This object is then used to gain access to those io resources (see below). + * + * @param dev + * A pointer to a rte_pci_device structure describing the device + * to use. + * @param bar + * Index of the io pci resource we want to access. + * @param p + * The rte_pci_ioport object to be initialized. + * @return + * 0 on success, negative on error. + */ +int rte_pci_ioport_map(struct rte_pci_device *dev, int bar, + struct rte_pci_ioport *p); + +/** + * Release any resources used in a rte_pci_ioport object. + * + * @param p + * The rte_pci_ioport object to be uninitialized. + * @return + * 0 on success, negative on error. + */ +int rte_pci_ioport_unmap(struct rte_pci_ioport *p); + +/** + * Read from a io pci resource. + * + * @param p + * The rte_pci_ioport object from which we want to read. + * @param data + * A data buffer where the bytes should be read into + * @param len + * The length of the data buffer. + * @param offset + * The offset into the pci io resource. + */ +void rte_pci_ioport_read(struct rte_pci_ioport *p, + void *data, size_t len, off_t offset); + +/** + * Write to a io pci resource. + * + * @param p + * The rte_pci_ioport object to which we want to write. + * @param data + * A data buffer where the bytes should be read into + * @param len + * The length of the data buffer. + * @param offset + * The offset into the pci io resource. + */ +void rte_pci_ioport_write(struct rte_pci_ioport *p, + const void *data, size_t len, off_t offset); + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_BUS_PCI_H_ */ diff --git a/dpdk/drivers/bus/pci/rte_bus_pci_version.map b/dpdk/drivers/bus/pci/rte_bus_pci_version.map new file mode 100644 index 00000000..27e9c4f1 --- /dev/null +++ b/dpdk/drivers/bus/pci/rte_bus_pci_version.map @@ -0,0 +1,18 @@ +DPDK_17.11 { + global: + + rte_pci_dump; + rte_pci_get_sysfs_path; + rte_pci_ioport_map; + rte_pci_ioport_read; + rte_pci_ioport_unmap; + rte_pci_ioport_write; + rte_pci_map_device; + rte_pci_read_config; + rte_pci_register; + rte_pci_unmap_device; + rte_pci_unregister; + rte_pci_write_config; + + local: *; +}; diff --git a/dpdk/drivers/bus/vdev/Makefile b/dpdk/drivers/bus/vdev/Makefile new file mode 100644 index 00000000..84bd7244 --- /dev/null +++ b/dpdk/drivers/bus/vdev/Makefile @@ -0,0 +1,57 @@ +# BSD LICENSE +# +# Copyright(c) 2017 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +include $(RTE_SDK)/mk/rte.vars.mk + +# +# library name +# +LIB = librte_bus_vdev.a + +CFLAGS += -O3 +CFLAGS += $(WERROR_FLAGS) + +# versioning export map +EXPORT_MAP := rte_bus_vdev_version.map + +# library version +LIBABIVER := 1 + +SRCS-y += vdev.c + +LDLIBS += -lrte_eal + +# +# Export include files +# +SYMLINK-y-include += rte_bus_vdev.h + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/bus/vdev/rte_bus_vdev.h b/dpdk/drivers/bus/vdev/rte_bus_vdev.h new file mode 100644 index 00000000..41762b85 --- /dev/null +++ b/dpdk/drivers/bus/vdev/rte_bus_vdev.h @@ -0,0 +1,153 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016 RehiveTech. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of RehiveTech nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef RTE_VDEV_H +#define RTE_VDEV_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +struct rte_vdev_device { + TAILQ_ENTRY(rte_vdev_device) next; /**< Next attached vdev */ + struct rte_device device; /**< Inherit core device */ +}; + +/** + * @internal + * Helper macro for drivers that need to convert to struct rte_vdev_device. + */ +#define RTE_DEV_TO_VDEV(ptr) \ + container_of(ptr, struct rte_vdev_device, device) + +static inline const char * +rte_vdev_device_name(const struct rte_vdev_device *dev) +{ + if (dev && dev->device.name) + return dev->device.name; + return NULL; +} + +static inline const char * +rte_vdev_device_args(const struct rte_vdev_device *dev) +{ + if (dev && dev->device.devargs) + return dev->device.devargs->args; + return ""; +} + +/** Double linked list of virtual device drivers. */ +TAILQ_HEAD(vdev_driver_list, rte_vdev_driver); + +/** + * Probe function called for each virtual device driver once. + */ +typedef int (rte_vdev_probe_t)(struct rte_vdev_device *dev); + +/** + * Remove function called for each virtual device driver once. + */ +typedef int (rte_vdev_remove_t)(struct rte_vdev_device *dev); + +/** + * A virtual device driver abstraction. + */ +struct rte_vdev_driver { + TAILQ_ENTRY(rte_vdev_driver) next; /**< Next in list. */ + struct rte_driver driver; /**< Inherited general driver. */ + rte_vdev_probe_t *probe; /**< Virtual device probe function. */ + rte_vdev_remove_t *remove; /**< Virtual device remove function. */ +}; + +/** + * Register a virtual device driver. + * + * @param driver + * A pointer to a rte_vdev_driver structure describing the driver + * to be registered. + */ +void rte_vdev_register(struct rte_vdev_driver *driver); + +/** + * Unregister a virtual device driver. + * + * @param driver + * A pointer to a rte_vdev_driver structure describing the driver + * to be unregistered. + */ +void rte_vdev_unregister(struct rte_vdev_driver *driver); + +#define RTE_PMD_REGISTER_VDEV(nm, vdrv)\ +RTE_INIT(vdrvinitfn_ ##vdrv);\ +static const char *vdrvinit_ ## nm ## _alias;\ +static void vdrvinitfn_ ##vdrv(void)\ +{\ + (vdrv).driver.name = RTE_STR(nm);\ + (vdrv).driver.alias = vdrvinit_ ## nm ## _alias;\ + rte_vdev_register(&vdrv);\ +} \ +RTE_PMD_EXPORT_NAME(nm, __COUNTER__) + +#define RTE_PMD_REGISTER_ALIAS(nm, alias)\ +static const char *vdrvinit_ ## nm ## _alias = RTE_STR(alias) + +/** + * Initialize a driver specified by name. + * + * @param name + * The pointer to a driver name to be initialized. + * @param args + * The pointer to arguments used by driver initialization. + * @return + * 0 on success, negative on error + */ +int rte_vdev_init(const char *name, const char *args); + +/** + * Uninitalize a driver specified by name. + * + * @param name + * The pointer to a driver name to be initialized. + * @return + * 0 on success, negative on error + */ +int rte_vdev_uninit(const char *name); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/dpdk/drivers/bus/vdev/rte_bus_vdev_version.map b/dpdk/drivers/bus/vdev/rte_bus_vdev_version.map new file mode 100644 index 00000000..707b870c --- /dev/null +++ b/dpdk/drivers/bus/vdev/rte_bus_vdev_version.map @@ -0,0 +1,10 @@ +DPDK_17.11 { + global: + + rte_vdev_init; + rte_vdev_register; + rte_vdev_uninit; + rte_vdev_unregister; + + local: *; +}; diff --git a/dpdk/drivers/bus/vdev/vdev.c b/dpdk/drivers/bus/vdev/vdev.c new file mode 100644 index 00000000..ba0ed7a2 --- /dev/null +++ b/dpdk/drivers/bus/vdev/vdev.c @@ -0,0 +1,355 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016 RehiveTech. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of RehiveTech nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "rte_bus_vdev.h" +#include "vdev_logs.h" + +int vdev_logtype_bus; + +/* Forward declare to access virtual bus name */ +static struct rte_bus rte_vdev_bus; + +/** Double linked list of virtual device drivers. */ +TAILQ_HEAD(vdev_device_list, rte_vdev_device); + +static struct vdev_device_list vdev_device_list = + TAILQ_HEAD_INITIALIZER(vdev_device_list); +struct vdev_driver_list vdev_driver_list = + TAILQ_HEAD_INITIALIZER(vdev_driver_list); + +/* register a driver */ +void +rte_vdev_register(struct rte_vdev_driver *driver) +{ + TAILQ_INSERT_TAIL(&vdev_driver_list, driver, next); +} + +/* unregister a driver */ +void +rte_vdev_unregister(struct rte_vdev_driver *driver) +{ + TAILQ_REMOVE(&vdev_driver_list, driver, next); +} + +static int +vdev_parse(const char *name, void *addr) +{ + struct rte_vdev_driver **out = addr; + struct rte_vdev_driver *driver = NULL; + + TAILQ_FOREACH(driver, &vdev_driver_list, next) { + if (strncmp(driver->driver.name, name, + strlen(driver->driver.name)) == 0) + break; + if (driver->driver.alias && + strncmp(driver->driver.alias, name, + strlen(driver->driver.alias)) == 0) + break; + } + if (driver != NULL && + addr != NULL) + *out = driver; + return driver == NULL; +} + +static int +vdev_probe_all_drivers(struct rte_vdev_device *dev) +{ + const char *name; + struct rte_vdev_driver *driver; + int ret; + + name = rte_vdev_device_name(dev); + + VDEV_LOG(DEBUG, "Search driver %s to probe device %s\n", name, + rte_vdev_device_name(dev)); + + if (vdev_parse(name, &driver)) + return -1; + dev->device.driver = &driver->driver; + ret = driver->probe(dev); + if (ret) + dev->device.driver = NULL; + return ret; +} + +static struct rte_vdev_device * +find_vdev(const char *name) +{ + struct rte_vdev_device *dev; + + if (!name) + return NULL; + + TAILQ_FOREACH(dev, &vdev_device_list, next) { + const char *devname = rte_vdev_device_name(dev); + + if (!strncmp(devname, name, strlen(name))) + return dev; + } + + return NULL; +} + +static struct rte_devargs * +alloc_devargs(const char *name, const char *args) +{ + struct rte_devargs *devargs; + int ret; + + devargs = calloc(1, sizeof(*devargs)); + if (!devargs) + return NULL; + + devargs->bus = &rte_vdev_bus; + if (args) + devargs->args = strdup(args); + else + devargs->args = strdup(""); + + ret = snprintf(devargs->name, sizeof(devargs->name), "%s", name); + if (ret < 0 || ret >= (int)sizeof(devargs->name)) { + free(devargs->args); + free(devargs); + return NULL; + } + + return devargs; +} + +int +rte_vdev_init(const char *name, const char *args) +{ + struct rte_vdev_device *dev; + struct rte_devargs *devargs; + int ret; + + if (name == NULL) + return -EINVAL; + + dev = find_vdev(name); + if (dev) + return -EEXIST; + + devargs = alloc_devargs(name, args); + if (!devargs) + return -ENOMEM; + + dev = calloc(1, sizeof(*dev)); + if (!dev) { + ret = -ENOMEM; + goto fail; + } + + dev->device.devargs = devargs; + dev->device.numa_node = SOCKET_ID_ANY; + dev->device.name = devargs->name; + + ret = vdev_probe_all_drivers(dev); + if (ret) { + if (ret > 0) + VDEV_LOG(ERR, "no driver found for %s\n", name); + goto fail; + } + + TAILQ_INSERT_TAIL(&devargs_list, devargs, next); + + TAILQ_INSERT_TAIL(&vdev_device_list, dev, next); + return 0; + +fail: + free(devargs->args); + free(devargs); + free(dev); + return ret; +} + +static int +vdev_remove_driver(struct rte_vdev_device *dev) +{ + const char *name = rte_vdev_device_name(dev); + const struct rte_vdev_driver *driver; + + if (!dev->device.driver) { + VDEV_LOG(DEBUG, "no driver attach to device %s\n", name); + return 1; + } + + driver = container_of(dev->device.driver, const struct rte_vdev_driver, + driver); + return driver->remove(dev); +} + +int +rte_vdev_uninit(const char *name) +{ + struct rte_vdev_device *dev; + struct rte_devargs *devargs; + int ret; + + if (name == NULL) + return -EINVAL; + + dev = find_vdev(name); + if (!dev) + return -ENOENT; + + devargs = dev->device.devargs; + + ret = vdev_remove_driver(dev); + if (ret) + return ret; + + TAILQ_REMOVE(&vdev_device_list, dev, next); + + TAILQ_REMOVE(&devargs_list, devargs, next); + + free(devargs->args); + free(devargs); + free(dev); + return 0; +} + +static int +vdev_scan(void) +{ + struct rte_vdev_device *dev; + struct rte_devargs *devargs; + + /* for virtual devices we scan the devargs_list populated via cmdline */ + TAILQ_FOREACH(devargs, &devargs_list, next) { + + if (devargs->bus != &rte_vdev_bus) + continue; + + dev = find_vdev(devargs->name); + if (dev) + continue; + + dev = calloc(1, sizeof(*dev)); + if (!dev) + return -1; + + dev->device.devargs = devargs; + dev->device.numa_node = SOCKET_ID_ANY; + dev->device.name = devargs->name; + + TAILQ_INSERT_TAIL(&vdev_device_list, dev, next); + } + + return 0; +} + +static int +vdev_probe(void) +{ + struct rte_vdev_device *dev; + int ret = 0; + + /* call the init function for each virtual device */ + TAILQ_FOREACH(dev, &vdev_device_list, next) { + + if (dev->device.driver) + continue; + + if (vdev_probe_all_drivers(dev)) { + VDEV_LOG(ERR, "failed to initialize %s device\n", + rte_vdev_device_name(dev)); + ret = -1; + } + } + + return ret; +} + +static struct rte_device * +vdev_find_device(const struct rte_device *start, rte_dev_cmp_t cmp, + const void *data) +{ + struct rte_vdev_device *dev; + + TAILQ_FOREACH(dev, &vdev_device_list, next) { + if (start && &dev->device == start) { + start = NULL; + continue; + } + if (cmp(&dev->device, data) == 0) + return &dev->device; + } + return NULL; +} + +static int +vdev_plug(struct rte_device *dev) +{ + return vdev_probe_all_drivers(RTE_DEV_TO_VDEV(dev)); +} + +static int +vdev_unplug(struct rte_device *dev) +{ + return rte_vdev_uninit(dev->name); +} + +static struct rte_bus rte_vdev_bus = { + .scan = vdev_scan, + .probe = vdev_probe, + .find_device = vdev_find_device, + .plug = vdev_plug, + .unplug = vdev_unplug, + .parse = vdev_parse, +}; + +RTE_REGISTER_BUS(vdev, rte_vdev_bus); + +RTE_INIT(vdev_init_log) +{ + vdev_logtype_bus = rte_log_register("bus.vdev"); + if (vdev_logtype_bus >= 0) + rte_log_set_level(vdev_logtype_bus, RTE_LOG_NOTICE); +} diff --git a/dpdk/drivers/bus/vdev/vdev_logs.h b/dpdk/drivers/bus/vdev/vdev_logs.h new file mode 100644 index 00000000..43b0ab8b --- /dev/null +++ b/dpdk/drivers/bus/vdev/vdev_logs.h @@ -0,0 +1,45 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _VDEV_LOGS_H_ +#define _VDEV_LOGS_H_ + +#include + +extern int vdev_logtype_bus; + +#define VDEV_LOG(level, fmt, args...) \ + rte_log(RTE_LOG_ ## level, vdev_logtype_bus, "%s(): " fmt "\n", \ + __func__, ##args) + +#endif /* _VDEV_LOGS_H_ */ diff --git a/dpdk/drivers/crypto/Makefile b/dpdk/drivers/crypto/Makefile index dc4ef7f9..645b6967 100644 --- a/dpdk/drivers/crypto/Makefile +++ b/dpdk/drivers/crypto/Makefile @@ -1,6 +1,6 @@ # BSD LICENSE # -# Copyright(c) 2010-2016 Intel Corporation. All rights reserved. +# Copyright(c) 2010-2017 Intel Corporation. All rights reserved. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -33,9 +33,16 @@ include $(RTE_SDK)/mk/rte.vars.mk DIRS-$(CONFIG_RTE_LIBRTE_PMD_AESNI_GCM) += aesni_gcm DIRS-$(CONFIG_RTE_LIBRTE_PMD_AESNI_MB) += aesni_mb +DIRS-$(CONFIG_RTE_LIBRTE_PMD_ARMV8_CRYPTO) += armv8 +DIRS-$(CONFIG_RTE_LIBRTE_PMD_OPENSSL) += openssl DIRS-$(CONFIG_RTE_LIBRTE_PMD_QAT) += qat +DIRS-$(CONFIG_RTE_LIBRTE_PMD_CRYPTO_SCHEDULER) += scheduler DIRS-$(CONFIG_RTE_LIBRTE_PMD_SNOW3G) += snow3g DIRS-$(CONFIG_RTE_LIBRTE_PMD_KASUMI) += kasumi +DIRS-$(CONFIG_RTE_LIBRTE_PMD_ZUC) += zuc +DIRS-$(CONFIG_RTE_LIBRTE_PMD_MRVL_CRYPTO) += mrvl DIRS-$(CONFIG_RTE_LIBRTE_PMD_NULL_CRYPTO) += null +DIRS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_SEC) += dpaa2_sec +DIRS-$(CONFIG_RTE_LIBRTE_PMD_DPAA_SEC) += dpaa_sec include $(RTE_SDK)/mk/rte.subdir.mk diff --git a/dpdk/drivers/crypto/aesni_gcm/Makefile b/dpdk/drivers/crypto/aesni_gcm/Makefile index 5898cae1..ddfec4c6 100644 --- a/dpdk/drivers/crypto/aesni_gcm/Makefile +++ b/dpdk/drivers/crypto/aesni_gcm/Makefile @@ -1,6 +1,6 @@ # BSD LICENSE # -# Copyright(c) 2016 Intel Corporation. All rights reserved. +# Copyright(c) 2016-2017 Intel Corporation. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -53,7 +53,9 @@ EXPORT_MAP := rte_pmd_aesni_gcm_version.map CFLAGS += -I$(AESNI_MULTI_BUFFER_LIB_PATH) CFLAGS += -I$(AESNI_MULTI_BUFFER_LIB_PATH)/include LDLIBS += -L$(AESNI_MULTI_BUFFER_LIB_PATH) -lIPSec_MB -LDLIBS += -lcrypto +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_cryptodev +LDLIBS += -lrte_bus_vdev # library source files SRCS-$(CONFIG_RTE_LIBRTE_PMD_AESNI_GCM) += aesni_gcm_pmd.c @@ -62,11 +64,4 @@ SRCS-$(CONFIG_RTE_LIBRTE_PMD_AESNI_GCM) += aesni_gcm_pmd_ops.c # export include files SYMLINK-y-include += -# library dependencies -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_AESNI_GCM) += lib/librte_eal -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_AESNI_GCM) += lib/librte_mbuf -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_AESNI_GCM) += lib/librte_mempool -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_AESNI_GCM) += lib/librte_ring -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_AESNI_GCM) += lib/librte_cryptodev - include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/crypto/aesni_gcm/aesni_gcm_ops.h b/dpdk/drivers/crypto/aesni_gcm/aesni_gcm_ops.h index c399068c..d6a18efc 100644 --- a/dpdk/drivers/crypto/aesni_gcm/aesni_gcm_ops.h +++ b/dpdk/drivers/crypto/aesni_gcm/aesni_gcm_ops.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -45,83 +45,101 @@ enum aesni_gcm_vector_mode { RTE_AESNI_GCM_NOT_SUPPORTED = 0, RTE_AESNI_GCM_SSE, RTE_AESNI_GCM_AVX, - RTE_AESNI_GCM_AVX2 + RTE_AESNI_GCM_AVX2, + RTE_AESNI_GCM_VECTOR_NUM }; -typedef void (*aes_keyexp_128_enc_t)(void *key, void *enc_exp_keys); +enum aesni_gcm_key { + AESNI_GCM_KEY_128, + AESNI_GCM_KEY_192, + AESNI_GCM_KEY_256, + AESNI_GCM_KEY_NUM +}; -typedef void (*aesni_gcm_t)(gcm_data *my_ctx_data, u8 *out, const u8 *in, - u64 plaintext_len, u8 *iv, const u8 *aad, u64 aad_len, - u8 *auth_tag, u64 auth_tag_len); -typedef void (*aesni_gcm_precomp_t)(gcm_data *my_ctx_data, u8 *hash_subkey); +typedef void (*aesni_gcm_t)(const struct gcm_key_data *gcm_key_data, + struct gcm_context_data *gcm_ctx_data, uint8_t *out, + const uint8_t *in, uint64_t plaintext_len, const uint8_t *iv, + const uint8_t *aad, uint64_t aad_len, + uint8_t *auth_tag, uint64_t auth_tag_len); + +typedef void (*aesni_gcm_precomp_t)(const void *key, struct gcm_key_data *gcm_data); + +typedef void (*aesni_gcm_init_t)(const struct gcm_key_data *gcm_key_data, + struct gcm_context_data *gcm_ctx_data, + const uint8_t *iv, + uint8_t const *aad, + uint64_t aad_len); + +typedef void (*aesni_gcm_update_t)(const struct gcm_key_data *gcm_key_data, + struct gcm_context_data *gcm_ctx_data, + uint8_t *out, + const uint8_t *in, + uint64_t plaintext_len); + +typedef void (*aesni_gcm_finalize_t)(const struct gcm_key_data *gcm_key_data, + struct gcm_context_data *gcm_ctx_data, + uint8_t *auth_tag, + uint64_t auth_tag_len); /** GCM library function pointer table */ struct aesni_gcm_ops { - struct { - struct { - aes_keyexp_128_enc_t aes128_enc; - /**< AES128 enc key expansion */ - } keyexp; - /**< Key expansion functions */ - } aux; /**< Auxiliary functions */ - - struct { - aesni_gcm_t enc; /**< GCM encode function pointer */ - aesni_gcm_t dec; /**< GCM decode function pointer */ - aesni_gcm_precomp_t precomp; /**< GCM pre-compute */ - } gcm; /**< GCM functions */ + aesni_gcm_t enc; /**< GCM encode function pointer */ + aesni_gcm_t dec; /**< GCM decode function pointer */ + aesni_gcm_precomp_t precomp; /**< GCM pre-compute */ + aesni_gcm_init_t init; + aesni_gcm_update_t update_enc; + aesni_gcm_update_t update_dec; + aesni_gcm_finalize_t finalize; }; +#define AES_GCM_FN(keylen, arch) \ +aes_gcm_enc_##keylen##_##arch,\ +aes_gcm_dec_##keylen##_##arch,\ +aes_gcm_pre_##keylen##_##arch,\ +aes_gcm_init_##keylen##_##arch,\ +aes_gcm_enc_##keylen##_update_##arch,\ +aes_gcm_dec_##keylen##_update_##arch,\ +aes_gcm_enc_##keylen##_finalize_##arch, -static const struct aesni_gcm_ops gcm_ops[] = { +static const struct aesni_gcm_ops gcm_ops[RTE_AESNI_GCM_VECTOR_NUM][AESNI_GCM_KEY_NUM] = { [RTE_AESNI_GCM_NOT_SUPPORTED] = { - .aux = { - .keyexp = { - NULL - } - }, - .gcm = { - NULL - } + [AESNI_GCM_KEY_128] = {NULL}, + [AESNI_GCM_KEY_192] = {NULL}, + [AESNI_GCM_KEY_256] = {NULL} }, [RTE_AESNI_GCM_SSE] = { - .aux = { - .keyexp = { - aes_keyexp_128_enc_sse - } + [AESNI_GCM_KEY_128] = { + AES_GCM_FN(128, sse) }, - .gcm = { - aesni_gcm_enc_sse, - aesni_gcm_dec_sse, - aesni_gcm_precomp_sse + [AESNI_GCM_KEY_192] = { + AES_GCM_FN(192, sse) + }, + [AESNI_GCM_KEY_256] = { + AES_GCM_FN(256, sse) } }, [RTE_AESNI_GCM_AVX] = { - .aux = { - .keyexp = { - aes_keyexp_128_enc_avx, - } + [AESNI_GCM_KEY_128] = { + AES_GCM_FN(128, avx_gen2) }, - .gcm = { - aesni_gcm_enc_avx_gen2, - aesni_gcm_dec_avx_gen2, - aesni_gcm_precomp_avx_gen2 + [AESNI_GCM_KEY_192] = { + AES_GCM_FN(192, avx_gen2) + }, + [AESNI_GCM_KEY_256] = { + AES_GCM_FN(256, avx_gen2) } }, [RTE_AESNI_GCM_AVX2] = { - .aux = { - .keyexp = { - aes_keyexp_128_enc_avx2, - } + [AESNI_GCM_KEY_128] = { + AES_GCM_FN(128, avx_gen4) }, - .gcm = { - aesni_gcm_enc_avx_gen4, - aesni_gcm_dec_avx_gen4, - aesni_gcm_precomp_avx_gen4 + [AESNI_GCM_KEY_192] = { + AES_GCM_FN(192, avx_gen4) + }, + [AESNI_GCM_KEY_256] = { + AES_GCM_FN(256, avx_gen4) } } }; - - #endif /* _AESNI_GCM_OPS_H_ */ diff --git a/dpdk/drivers/crypto/aesni_gcm/aesni_gcm_pmd.c b/dpdk/drivers/crypto/aesni_gcm/aesni_gcm_pmd.c index dc0b0337..08dcacce 100644 --- a/dpdk/drivers/crypto/aesni_gcm/aesni_gcm_pmd.c +++ b/dpdk/drivers/crypto/aesni_gcm/aesni_gcm_pmd.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -30,84 +30,18 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#include - #include -#include #include #include #include -#include +#include #include #include +#include #include "aesni_gcm_pmd_private.h" -/** - * Global static parameter used to create a unique name for each AES-NI multi - * buffer crypto device. - */ -static unsigned unique_name_id; - -static inline int -create_unique_device_name(char *name, size_t size) -{ - int ret; - - if (name == NULL) - return -EINVAL; - - ret = snprintf(name, size, "%s_%u", RTE_STR(CRYPTODEV_NAME_AESNI_GCM_PMD), - unique_name_id++); - if (ret < 0) - return ret; - return 0; -} - -static int -aesni_gcm_calculate_hash_sub_key(uint8_t *hsubkey, unsigned hsubkey_length, - uint8_t *aeskey, unsigned aeskey_length) -{ - uint8_t key[aeskey_length] __rte_aligned(16); - AES_KEY enc_key; - - if (hsubkey_length % 16 != 0 && aeskey_length % 16 != 0) - return -EFAULT; - - memcpy(key, aeskey, aeskey_length); - - if (AES_set_encrypt_key(key, aeskey_length << 3, &enc_key) != 0) - return -EFAULT; - - AES_encrypt(hsubkey, hsubkey, &enc_key); - - return 0; -} - -/** Get xform chain order */ -static int -aesni_gcm_get_mode(const struct rte_crypto_sym_xform *xform) -{ - /* - * GCM only supports authenticated encryption or authenticated - * decryption, all other options are invalid, so we must have exactly - * 2 xform structs chained together - */ - if (xform->next == NULL || xform->next->next != NULL) - return -1; - - if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && - xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) { - return AESNI_GCM_OP_AUTHENTICATED_ENCRYPTION; - } - - if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH && - xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { - return AESNI_GCM_OP_AUTHENTICATED_DECRYPTION; - } - - return -1; -} +static uint8_t cryptodev_driver_id; /** Parse crypto xform chain and set private session parameters */ int @@ -115,98 +49,147 @@ aesni_gcm_set_session_parameters(const struct aesni_gcm_ops *gcm_ops, struct aesni_gcm_session *sess, const struct rte_crypto_sym_xform *xform) { - const struct rte_crypto_sym_xform *auth_xform = NULL; - const struct rte_crypto_sym_xform *cipher_xform = NULL; - - uint8_t hsubkey[16] __rte_aligned(16) = { 0 }; - - /* Select Crypto operation - hash then cipher / cipher then hash */ - switch (aesni_gcm_get_mode(xform)) { - case AESNI_GCM_OP_AUTHENTICATED_ENCRYPTION: - sess->op = AESNI_GCM_OP_AUTHENTICATED_ENCRYPTION; - - cipher_xform = xform; - auth_xform = xform->next; - break; - case AESNI_GCM_OP_AUTHENTICATED_DECRYPTION: - sess->op = AESNI_GCM_OP_AUTHENTICATED_DECRYPTION; + const struct rte_crypto_sym_xform *auth_xform; + const struct rte_crypto_sym_xform *aead_xform; + uint16_t digest_length; + uint8_t key_length; + uint8_t *key; + /* AES-GMAC */ + if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) { auth_xform = xform; - cipher_xform = xform->next; + if (auth_xform->auth.algo != RTE_CRYPTO_AUTH_AES_GMAC) { + GCM_LOG_ERR("Only AES GMAC is supported as an " + "authentication only algorithm"); + return -ENOTSUP; + } + /* Set IV parameters */ + sess->iv.offset = auth_xform->auth.iv.offset; + sess->iv.length = auth_xform->auth.iv.length; + + /* Select Crypto operation */ + if (auth_xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) + sess->op = AESNI_GMAC_OP_GENERATE; + else + sess->op = AESNI_GMAC_OP_VERIFY; + + key_length = auth_xform->auth.key.length; + key = auth_xform->auth.key.data; + digest_length = auth_xform->auth.digest_length; + + /* AES-GCM */ + } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) { + aead_xform = xform; + + if (aead_xform->aead.algo != RTE_CRYPTO_AEAD_AES_GCM) { + GCM_LOG_ERR("The only combined operation " + "supported is AES GCM"); + return -ENOTSUP; + } + + /* Set IV parameters */ + sess->iv.offset = aead_xform->aead.iv.offset; + sess->iv.length = aead_xform->aead.iv.length; + + /* Select Crypto operation */ + if (aead_xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) + sess->op = AESNI_GCM_OP_AUTHENTICATED_ENCRYPTION; + else + sess->op = AESNI_GCM_OP_AUTHENTICATED_DECRYPTION; + + key_length = aead_xform->aead.key.length; + key = aead_xform->aead.key.data; + + sess->aad_length = aead_xform->aead.aad_length; + digest_length = aead_xform->aead.digest_length; + } else { + GCM_LOG_ERR("Wrong xform type, has to be AEAD or authentication"); + return -ENOTSUP; + } + + + /* IV check */ + if (sess->iv.length != 16 && sess->iv.length != 12 && + sess->iv.length != 0) { + GCM_LOG_ERR("Wrong IV length"); + return -EINVAL; + } + + /* Check key length and calculate GCM pre-compute. */ + switch (key_length) { + case 16: + sess->key = AESNI_GCM_KEY_128; + break; + case 24: + sess->key = AESNI_GCM_KEY_192; + break; + case 32: + sess->key = AESNI_GCM_KEY_256; break; default: - GCM_LOG_ERR("Unsupported operation chain order parameter"); + GCM_LOG_ERR("Invalid key length"); return -EINVAL; } - /* We only support AES GCM */ - if (cipher_xform->cipher.algo != RTE_CRYPTO_CIPHER_AES_GCM && - auth_xform->auth.algo != RTE_CRYPTO_AUTH_AES_GCM) - return -EINVAL; + gcm_ops[sess->key].precomp(key, &sess->gdata_key); - /* Select cipher direction */ - if (sess->op == AESNI_GCM_OP_AUTHENTICATED_ENCRYPTION && - cipher_xform->cipher.op != - RTE_CRYPTO_CIPHER_OP_ENCRYPT) { - GCM_LOG_ERR("xform chain (CIPHER/AUTH) and cipher operation " - "(DECRYPT) specified are an invalid selection"); - return -EINVAL; - } else if (sess->op == AESNI_GCM_OP_AUTHENTICATED_DECRYPTION && - cipher_xform->cipher.op != - RTE_CRYPTO_CIPHER_OP_DECRYPT) { - GCM_LOG_ERR("xform chain (AUTH/CIPHER) and cipher operation " - "(ENCRYPT) specified are an invalid selection"); + /* Digest check */ + if (digest_length != 16 && + digest_length != 12 && + digest_length != 8) { + GCM_LOG_ERR("digest"); return -EINVAL; } - - /* Expand GCM AES128 key */ - (*gcm_ops->aux.keyexp.aes128_enc)(cipher_xform->cipher.key.data, - sess->gdata.expanded_keys); - - /* Calculate hash sub key here */ - aesni_gcm_calculate_hash_sub_key(hsubkey, sizeof(hsubkey), - cipher_xform->cipher.key.data, - cipher_xform->cipher.key.length); - - /* Calculate GCM pre-compute */ - (*gcm_ops->gcm.precomp)(&sess->gdata, hsubkey); + sess->digest_length = digest_length; return 0; } /** Get gcm session */ static struct aesni_gcm_session * -aesni_gcm_get_session(struct aesni_gcm_qp *qp, struct rte_crypto_sym_op *op) +aesni_gcm_get_session(struct aesni_gcm_qp *qp, struct rte_crypto_op *op) { struct aesni_gcm_session *sess = NULL; + struct rte_crypto_sym_op *sym_op = op->sym; - if (op->sess_type == RTE_CRYPTO_SYM_OP_WITH_SESSION) { - if (unlikely(op->session->dev_type - != RTE_CRYPTODEV_AESNI_GCM_PMD)) - return sess; - - sess = (struct aesni_gcm_session *)op->session->_private; + if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { + if (likely(sym_op->session != NULL)) + sess = (struct aesni_gcm_session *) + get_session_private_data( + sym_op->session, + cryptodev_driver_id); } else { void *_sess; + void *_sess_private_data = NULL; - if (rte_mempool_get(qp->sess_mp, &_sess)) - return sess; + if (rte_mempool_get(qp->sess_mp, (void **)&_sess)) + return NULL; - sess = (struct aesni_gcm_session *) - ((struct rte_cryptodev_session *)_sess)->_private; + if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data)) + return NULL; + + sess = (struct aesni_gcm_session *)_sess_private_data; if (unlikely(aesni_gcm_set_session_parameters(qp->ops, - sess, op->xform) != 0)) { + sess, sym_op->xform) != 0)) { rte_mempool_put(qp->sess_mp, _sess); + rte_mempool_put(qp->sess_mp, _sess_private_data); sess = NULL; } + sym_op->session = (struct rte_cryptodev_sym_session *)_sess; + set_session_private_data(sym_op->session, cryptodev_driver_id, + _sess_private_data); } + + if (unlikely(sess == NULL)) + op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; + return sess; } /** - * Process a crypto operation and complete a JOB_AES_HMAC job structure for - * submission to the multi buffer library for processing. + * Process a crypto operation, calling + * the GCM API from the multi buffer library. * * @param qp queue pair * @param op symmetric crypto operation @@ -216,67 +199,160 @@ aesni_gcm_get_session(struct aesni_gcm_qp *qp, struct rte_crypto_sym_op *op) * */ static int -process_gcm_crypto_op(struct aesni_gcm_qp *qp, struct rte_crypto_sym_op *op, +process_gcm_crypto_op(struct aesni_gcm_qp *qp, struct rte_crypto_op *op, struct aesni_gcm_session *session) { uint8_t *src, *dst; - struct rte_mbuf *m = op->m_src; + uint8_t *iv_ptr; + struct rte_crypto_sym_op *sym_op = op->sym; + struct rte_mbuf *m_src = sym_op->m_src; + uint32_t offset, data_offset, data_length; + uint32_t part_len, total_len, data_len; - src = rte_pktmbuf_mtod(m, uint8_t *) + op->cipher.data.offset; - dst = op->m_dst ? - rte_pktmbuf_mtod_offset(op->m_dst, uint8_t *, - op->cipher.data.offset) : - rte_pktmbuf_mtod_offset(m, uint8_t *, - op->cipher.data.offset); - - /* sanity checks */ - if (op->cipher.iv.length != 16 && op->cipher.iv.length != 0) { - GCM_LOG_ERR("iv"); - return -1; + if (session->op == AESNI_GCM_OP_AUTHENTICATED_ENCRYPTION || + session->op == AESNI_GCM_OP_AUTHENTICATED_DECRYPTION) { + offset = sym_op->aead.data.offset; + data_offset = offset; + data_length = sym_op->aead.data.length; + } else { + offset = sym_op->auth.data.offset; + data_offset = offset; + data_length = sym_op->auth.data.length; } - if (op->auth.aad.length != 12 && op->auth.aad.length != 8 && - op->auth.aad.length != 0) { - GCM_LOG_ERR("iv"); - return -1; + RTE_ASSERT(m_src != NULL); + + while (offset >= m_src->data_len && data_length != 0) { + offset -= m_src->data_len; + m_src = m_src->next; + + RTE_ASSERT(m_src != NULL); } - if (op->auth.digest.length != 16 && - op->auth.digest.length != 12 && - op->auth.digest.length != 8 && - op->auth.digest.length != 0) { - GCM_LOG_ERR("iv"); - return -1; + data_len = m_src->data_len - offset; + part_len = (data_len < data_length) ? data_len : + data_length; + + /* Destination buffer is required when segmented source buffer */ + RTE_ASSERT((part_len == data_length) || + ((part_len != data_length) && + (sym_op->m_dst != NULL))); + /* Segmented destination buffer is not supported */ + RTE_ASSERT((sym_op->m_dst == NULL) || + ((sym_op->m_dst != NULL) && + rte_pktmbuf_is_contiguous(sym_op->m_dst))); + + + dst = sym_op->m_dst ? + rte_pktmbuf_mtod_offset(sym_op->m_dst, uint8_t *, + data_offset) : + rte_pktmbuf_mtod_offset(sym_op->m_src, uint8_t *, + data_offset); + + src = rte_pktmbuf_mtod_offset(m_src, uint8_t *, offset); + + iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, + session->iv.offset); + /* + * GCM working in 12B IV mode => 16B pre-counter block we need + * to set BE LSB to 1, driver expects that 16B is allocated + */ + if (session->iv.length == 12) { + uint32_t *iv_padd = (uint32_t *)&(iv_ptr[12]); + *iv_padd = rte_bswap32(1); } if (session->op == AESNI_GCM_OP_AUTHENTICATED_ENCRYPTION) { - (*qp->ops->gcm.enc)(&session->gdata, dst, src, - (uint64_t)op->cipher.data.length, - op->cipher.iv.data, - op->auth.aad.data, - (uint64_t)op->auth.aad.length, - op->auth.digest.data, - (uint64_t)op->auth.digest.length); - } else if (session->op == AESNI_GCM_OP_AUTHENTICATED_DECRYPTION) { - uint8_t *auth_tag = (uint8_t *)rte_pktmbuf_append(m, - op->auth.digest.length); + qp->ops[session->key].init(&session->gdata_key, + &qp->gdata_ctx, + iv_ptr, + sym_op->aead.aad.data, + (uint64_t)session->aad_length); - if (!auth_tag) { - GCM_LOG_ERR("iv"); - return -1; + qp->ops[session->key].update_enc(&session->gdata_key, + &qp->gdata_ctx, dst, src, + (uint64_t)part_len); + total_len = data_length - part_len; + + while (total_len) { + dst += part_len; + m_src = m_src->next; + + RTE_ASSERT(m_src != NULL); + + src = rte_pktmbuf_mtod(m_src, uint8_t *); + part_len = (m_src->data_len < total_len) ? + m_src->data_len : total_len; + + qp->ops[session->key].update_enc(&session->gdata_key, + &qp->gdata_ctx, dst, src, + (uint64_t)part_len); + total_len -= part_len; } - (*qp->ops->gcm.dec)(&session->gdata, dst, src, - (uint64_t)op->cipher.data.length, - op->cipher.iv.data, - op->auth.aad.data, - (uint64_t)op->auth.aad.length, + qp->ops[session->key].finalize(&session->gdata_key, + &qp->gdata_ctx, + sym_op->aead.digest.data, + (uint64_t)session->digest_length); + } else if (session->op == AESNI_GCM_OP_AUTHENTICATED_DECRYPTION) { + uint8_t *auth_tag = qp->temp_digest; + + qp->ops[session->key].init(&session->gdata_key, + &qp->gdata_ctx, + iv_ptr, + sym_op->aead.aad.data, + (uint64_t)session->aad_length); + + qp->ops[session->key].update_dec(&session->gdata_key, + &qp->gdata_ctx, dst, src, + (uint64_t)part_len); + total_len = data_length - part_len; + + while (total_len) { + dst += part_len; + m_src = m_src->next; + + RTE_ASSERT(m_src != NULL); + + src = rte_pktmbuf_mtod(m_src, uint8_t *); + part_len = (m_src->data_len < total_len) ? + m_src->data_len : total_len; + + qp->ops[session->key].update_dec(&session->gdata_key, + &qp->gdata_ctx, + dst, src, + (uint64_t)part_len); + total_len -= part_len; + } + + qp->ops[session->key].finalize(&session->gdata_key, + &qp->gdata_ctx, auth_tag, - (uint64_t)op->auth.digest.length); - } else { - GCM_LOG_ERR("iv"); - return -1; + (uint64_t)session->digest_length); + } else if (session->op == AESNI_GMAC_OP_GENERATE) { + qp->ops[session->key].init(&session->gdata_key, + &qp->gdata_ctx, + iv_ptr, + src, + (uint64_t)data_length); + qp->ops[session->key].finalize(&session->gdata_key, + &qp->gdata_ctx, + sym_op->auth.digest.data, + (uint64_t)session->digest_length); + } else { /* AESNI_GMAC_OP_VERIFY */ + uint8_t *auth_tag = qp->temp_digest; + + qp->ops[session->key].init(&session->gdata_key, + &qp->gdata_ctx, + iv_ptr, + src, + (uint64_t)data_length); + + qp->ops[session->key].finalize(&session->gdata_key, + &qp->gdata_ctx, + auth_tag, + (uint64_t)session->digest_length); } return 0; @@ -293,34 +369,33 @@ process_gcm_crypto_op(struct aesni_gcm_qp *qp, struct rte_crypto_sym_op *op, * - Returns NULL on invalid job */ static void -post_process_gcm_crypto_op(struct rte_crypto_op *op) +post_process_gcm_crypto_op(struct aesni_gcm_qp *qp, + struct rte_crypto_op *op, + struct aesni_gcm_session *session) { - struct rte_mbuf *m = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src; - - struct aesni_gcm_session *session = - (struct aesni_gcm_session *)op->sym->session->_private; - op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; /* Verify digest if required */ - if (session->op == AESNI_GCM_OP_AUTHENTICATED_DECRYPTION) { + if (session->op == AESNI_GCM_OP_AUTHENTICATED_DECRYPTION || + session->op == AESNI_GMAC_OP_VERIFY) { + uint8_t *digest; - uint8_t *tag = rte_pktmbuf_mtod_offset(m, uint8_t *, - m->data_len - op->sym->auth.digest.length); + uint8_t *tag = (uint8_t *)&qp->temp_digest; + + if (session->op == AESNI_GMAC_OP_VERIFY) + digest = op->sym->auth.digest.data; + else + digest = op->sym->aead.digest.data; #ifdef RTE_LIBRTE_PMD_AESNI_GCM_DEBUG rte_hexdump(stdout, "auth tag (orig):", - op->sym->auth.digest.data, op->sym->auth.digest.length); + digest, session->digest_length); rte_hexdump(stdout, "auth tag (calc):", - tag, op->sym->auth.digest.length); + tag, session->digest_length); #endif - if (memcmp(tag, op->sym->auth.digest.data, - op->sym->auth.digest.length) != 0) + if (memcmp(tag, digest, session->digest_length) != 0) op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; - - /* trim area used for digest from mbuf */ - rte_pktmbuf_trim(m, op->sym->auth.digest.length); } } @@ -328,6 +403,7 @@ post_process_gcm_crypto_op(struct rte_crypto_op *op) * Process a completed GCM request * * @param qp Queue Pair to process + * @param op Crypto operation * @param job JOB_AES_HMAC job * * @return @@ -335,74 +411,82 @@ post_process_gcm_crypto_op(struct rte_crypto_op *op) */ static void handle_completed_gcm_crypto_op(struct aesni_gcm_qp *qp, - struct rte_crypto_op *op) + struct rte_crypto_op *op, + struct aesni_gcm_session *sess) { - post_process_gcm_crypto_op(op); + post_process_gcm_crypto_op(qp, op, sess); /* Free session if a session-less crypto op */ - if (op->sym->sess_type == RTE_CRYPTO_SYM_OP_SESSIONLESS) { + if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { + memset(sess, 0, sizeof(struct aesni_gcm_session)); + memset(op->sym->session, 0, + rte_cryptodev_get_header_session_size()); + rte_mempool_put(qp->sess_mp, sess); rte_mempool_put(qp->sess_mp, op->sym->session); op->sym->session = NULL; } - - rte_ring_enqueue(qp->processed_pkts, (void *)op); -} - -static uint16_t -aesni_gcm_pmd_enqueue_burst(void *queue_pair, - struct rte_crypto_op **ops, uint16_t nb_ops) -{ - struct aesni_gcm_session *sess; - struct aesni_gcm_qp *qp = queue_pair; - - int i, retval = 0; - - for (i = 0; i < nb_ops; i++) { - - sess = aesni_gcm_get_session(qp, ops[i]->sym); - if (unlikely(sess == NULL)) { - ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; - qp->qp_stats.enqueue_err_count++; - break; - } - - retval = process_gcm_crypto_op(qp, ops[i]->sym, sess); - if (retval < 0) { - ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; - qp->qp_stats.enqueue_err_count++; - break; - } - - handle_completed_gcm_crypto_op(qp, ops[i]); - - qp->qp_stats.enqueued_count++; - } - return i; } static uint16_t aesni_gcm_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops, uint16_t nb_ops) { + struct aesni_gcm_session *sess; struct aesni_gcm_qp *qp = queue_pair; - unsigned nb_dequeued; + int retval = 0; + unsigned int i, nb_dequeued; nb_dequeued = rte_ring_dequeue_burst(qp->processed_pkts, - (void **)ops, nb_ops); - qp->qp_stats.dequeued_count += nb_dequeued; + (void **)ops, nb_ops, NULL); - return nb_dequeued; + for (i = 0; i < nb_dequeued; i++) { + + sess = aesni_gcm_get_session(qp, ops[i]); + if (unlikely(sess == NULL)) { + ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; + qp->qp_stats.dequeue_err_count++; + break; + } + + retval = process_gcm_crypto_op(qp, ops[i], sess); + if (retval < 0) { + ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; + qp->qp_stats.dequeue_err_count++; + break; + } + + handle_completed_gcm_crypto_op(qp, ops[i], sess); + } + + qp->qp_stats.dequeued_count += i; + + return i; } -static int aesni_gcm_uninit(const char *name); +static uint16_t +aesni_gcm_pmd_enqueue_burst(void *queue_pair, + struct rte_crypto_op **ops, uint16_t nb_ops) +{ + struct aesni_gcm_qp *qp = queue_pair; + + unsigned int nb_enqueued; + + nb_enqueued = rte_ring_enqueue_burst(qp->processed_pkts, + (void **)ops, nb_ops, NULL); + qp->qp_stats.enqueued_count += nb_enqueued; + + return nb_enqueued; +} + +static int aesni_gcm_remove(struct rte_vdev_device *vdev); static int aesni_gcm_create(const char *name, - struct rte_crypto_vdev_init_params *init_params) + struct rte_vdev_device *vdev, + struct rte_cryptodev_pmd_init_params *init_params) { struct rte_cryptodev *dev; - char crypto_dev_name[RTE_CRYPTODEV_NAME_MAX_LEN]; struct aesni_gcm_private *internals; enum aesni_gcm_vector_mode vector_mode; @@ -412,34 +496,21 @@ aesni_gcm_create(const char *name, return -EFAULT; } + dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params); + if (dev == NULL) { + GCM_LOG_ERR("driver %s: create failed", init_params->name); + return -ENODEV; + } + /* Check CPU for supported vector instruction set */ if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2)) vector_mode = RTE_AESNI_GCM_AVX2; else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX)) vector_mode = RTE_AESNI_GCM_AVX; - else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_SSE4_1)) + else vector_mode = RTE_AESNI_GCM_SSE; - else { - GCM_LOG_ERR("Vector instructions are not supported by CPU"); - return -EFAULT; - } - /* create a unique device name */ - if (create_unique_device_name(crypto_dev_name, - RTE_CRYPTODEV_NAME_MAX_LEN) != 0) { - GCM_LOG_ERR("failed to create unique cryptodev name"); - return -EINVAL; - } - - - dev = rte_cryptodev_pmd_virtual_dev_init(crypto_dev_name, - sizeof(struct aesni_gcm_private), init_params->socket_id); - if (dev == NULL) { - GCM_LOG_ERR("failed to create cryptodev vdev"); - goto init_error; - } - - dev->dev_type = RTE_CRYPTODEV_AESNI_GCM_PMD; + dev->driver_id = cryptodev_driver_id; dev->dev_ops = rte_aesni_gcm_pmd_ops; /* register rx/tx burst functions for data path */ @@ -448,7 +519,8 @@ aesni_gcm_create(const char *name, dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | - RTE_CRYPTODEV_FF_CPU_AESNI; + RTE_CRYPTODEV_FF_CPU_AESNI | + RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER; switch (vector_mode) { case RTE_AESNI_GCM_SSE: @@ -464,7 +536,6 @@ aesni_gcm_create(const char *name, break; } - /* Set vector instructions mode supported */ internals = dev->data->dev_private; internals->vector_mode = vector_mode; @@ -473,55 +544,59 @@ aesni_gcm_create(const char *name, internals->max_nb_sessions = init_params->max_nb_sessions; return 0; - -init_error: - GCM_LOG_ERR("driver %s: create failed", name); - - aesni_gcm_uninit(crypto_dev_name); - return -EFAULT; } static int -aesni_gcm_init(const char *name, const char *input_args) +aesni_gcm_probe(struct rte_vdev_device *vdev) { - struct rte_crypto_vdev_init_params init_params = { - RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_QUEUE_PAIRS, - RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_SESSIONS, - rte_socket_id() + struct rte_cryptodev_pmd_init_params init_params = { + "", + sizeof(struct aesni_gcm_private), + rte_socket_id(), + RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS, + RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_SESSIONS }; + const char *name; + const char *input_args; - rte_cryptodev_parse_vdev_init_params(&init_params, input_args); + name = rte_vdev_device_name(vdev); + if (name == NULL) + return -EINVAL; + input_args = rte_vdev_device_args(vdev); + rte_cryptodev_pmd_parse_input_args(&init_params, input_args); - RTE_LOG(INFO, PMD, "Initialising %s on NUMA node %d\n", name, - init_params.socket_id); - RTE_LOG(INFO, PMD, " Max number of queue pairs = %d\n", - init_params.max_nb_queue_pairs); - RTE_LOG(INFO, PMD, " Max number of sessions = %d\n", - init_params.max_nb_sessions); - - return aesni_gcm_create(name, &init_params); + return aesni_gcm_create(name, vdev, &init_params); } static int -aesni_gcm_uninit(const char *name) +aesni_gcm_remove(struct rte_vdev_device *vdev) { + struct rte_cryptodev *cryptodev; + const char *name; + + name = rte_vdev_device_name(vdev); if (name == NULL) return -EINVAL; - GCM_LOG_INFO("Closing AESNI crypto device %s on numa socket %u\n", - name, rte_socket_id()); + cryptodev = rte_cryptodev_pmd_get_named_dev(name); + if (cryptodev == NULL) + return -ENODEV; - return 0; + return rte_cryptodev_pmd_destroy(cryptodev); } -static struct rte_driver aesni_gcm_pmd_drv = { - .type = PMD_VDEV, - .init = aesni_gcm_init, - .uninit = aesni_gcm_uninit +static struct rte_vdev_driver aesni_gcm_pmd_drv = { + .probe = aesni_gcm_probe, + .remove = aesni_gcm_remove }; -PMD_REGISTER_DRIVER(aesni_gcm_pmd_drv, CRYPTODEV_NAME_AESNI_GCM_PMD); -DRIVER_REGISTER_PARAM_STRING(CRYPTODEV_NAME_AESNI_GCM_PMD, +static struct cryptodev_driver aesni_gcm_crypto_drv; + +RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_AESNI_GCM_PMD, aesni_gcm_pmd_drv); +RTE_PMD_REGISTER_ALIAS(CRYPTODEV_NAME_AESNI_GCM_PMD, cryptodev_aesni_gcm_pmd); +RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_AESNI_GCM_PMD, "max_nb_queue_pairs= " "max_nb_sessions= " "socket_id="); +RTE_PMD_REGISTER_CRYPTO_DRIVER(aesni_gcm_crypto_drv, aesni_gcm_pmd_drv, + cryptodev_driver_id); diff --git a/dpdk/drivers/crypto/aesni_gcm/aesni_gcm_pmd_ops.c b/dpdk/drivers/crypto/aesni_gcm/aesni_gcm_pmd_ops.c index e824d4b3..0f315f00 100644 --- a/dpdk/drivers/crypto/aesni_gcm/aesni_gcm_pmd_ops.c +++ b/dpdk/drivers/crypto/aesni_gcm/aesni_gcm_pmd_ops.c @@ -39,17 +39,42 @@ #include "aesni_gcm_pmd_private.h" static const struct rte_cryptodev_capabilities aesni_gcm_pmd_capabilities[] = { - { /* AES GCM (AUTH) */ + { /* AES GMAC (AUTH) */ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, {.sym = { .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, {.auth = { - .algo = RTE_CRYPTO_AUTH_AES_GCM, + .algo = RTE_CRYPTO_AUTH_AES_GMAC, .block_size = 16, .key_size = { .min = 16, + .max = 32, + .increment = 8 + }, + .digest_size = { + .min = 8, .max = 16, + .increment = 4 + }, + .iv_size = { + .min = 12, + .max = 12, .increment = 0 + } + }, } + }, } + }, + { /* AES GCM */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD, + {.aead = { + .algo = RTE_CRYPTO_AEAD_AES_GCM, + .block_size = 16, + .key_size = { + .min = 16, + .max = 32, + .increment = 8 }, .digest_size = { .min = 8, @@ -57,28 +82,13 @@ static const struct rte_cryptodev_capabilities aesni_gcm_pmd_capabilities[] = { .increment = 4 }, .aad_size = { - .min = 8, - .max = 12, - .increment = 4 - } - }, } - }, } - }, - { /* AES GCM (CIPHER) */ - .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, - {.sym = { - .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, - {.cipher = { - .algo = RTE_CRYPTO_CIPHER_AES_GCM, - .block_size = 16, - .key_size = { - .min = 16, - .max = 16, - .increment = 0 + .min = 0, + .max = 65535, + .increment = 1 }, .iv_size = { - .min = 16, - .max = 16, + .min = 12, + .max = 12, .increment = 0 } }, } @@ -89,7 +99,8 @@ static const struct rte_cryptodev_capabilities aesni_gcm_pmd_capabilities[] = { /** Configure device */ static int -aesni_gcm_pmd_config(__rte_unused struct rte_cryptodev *dev) +aesni_gcm_pmd_config(__rte_unused struct rte_cryptodev *dev, + __rte_unused struct rte_cryptodev_config *config) { return 0; } @@ -155,9 +166,9 @@ aesni_gcm_pmd_info_get(struct rte_cryptodev *dev, struct aesni_gcm_private *internals = dev->data->dev_private; if (dev_info != NULL) { - dev_info->dev_type = dev->dev_type; - dev_info->feature_flags = dev->feature_flags; - dev_info->capabilities = aesni_gcm_pmd_capabilities; + dev_info->driver_id = dev->driver_id; + dev_info->feature_flags = dev->feature_flags; + dev_info->capabilities = aesni_gcm_pmd_capabilities; dev_info->max_nb_queue_pairs = internals->max_nb_queue_pairs; dev_info->sym.max_nb_sessions = internals->max_nb_sessions; @@ -184,7 +195,7 @@ aesni_gcm_pmd_qp_set_unique_name(struct rte_cryptodev *dev, "aesni_gcm_pmd_%u_qp_%u", dev->data->dev_id, qp->id); - if (n > sizeof(qp->name)) + if (n >= sizeof(qp->name)) return -1; return 0; @@ -199,7 +210,7 @@ aesni_gcm_pmd_qp_create_processed_pkts_ring(struct aesni_gcm_qp *qp, r = rte_ring_lookup(qp->name); if (r) { - if (r->prod.size >= ring_size) { + if (rte_ring_get_size(r) >= ring_size) { GCM_LOG_INFO("Reusing existing ring %s for processed" " packets", qp->name); return r; @@ -218,7 +229,7 @@ aesni_gcm_pmd_qp_create_processed_pkts_ring(struct aesni_gcm_qp *qp, static int aesni_gcm_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, const struct rte_cryptodev_qp_conf *qp_conf, - int socket_id) + int socket_id, struct rte_mempool *session_pool) { struct aesni_gcm_qp *qp = NULL; struct aesni_gcm_private *internals = dev->data->dev_private; @@ -239,14 +250,14 @@ aesni_gcm_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, if (aesni_gcm_pmd_qp_set_unique_name(dev, qp)) goto qp_setup_cleanup; - qp->ops = &gcm_ops[internals->vector_mode]; + qp->ops = (const struct aesni_gcm_ops *)gcm_ops[internals->vector_mode]; qp->processed_pkts = aesni_gcm_pmd_qp_create_processed_pkts_ring(qp, qp_conf->nb_descriptors, socket_id); if (qp->processed_pkts == NULL) goto qp_setup_cleanup; - qp->sess_mp = dev->data->session_pool; + qp->sess_mp = session_pool; memset(&qp->qp_stats, 0, sizeof(qp->qp_stats)); @@ -290,32 +301,57 @@ aesni_gcm_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused) } /** Configure a aesni gcm session from a crypto xform chain */ -static void * -aesni_gcm_pmd_session_configure(struct rte_cryptodev *dev, - struct rte_crypto_sym_xform *xform, void *sess) +static int +aesni_gcm_pmd_session_configure(struct rte_cryptodev *dev __rte_unused, + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool) { + void *sess_private_data; + int ret; struct aesni_gcm_private *internals = dev->data->dev_private; if (unlikely(sess == NULL)) { GCM_LOG_ERR("invalid session struct"); - return NULL; + return -EINVAL; } - if (aesni_gcm_set_session_parameters(&gcm_ops[internals->vector_mode], - sess, xform) != 0) { + if (rte_mempool_get(mempool, &sess_private_data)) { + CDEV_LOG_ERR( + "Couldn't get object from session mempool"); + return -ENOMEM; + } + ret = aesni_gcm_set_session_parameters(gcm_ops[internals->vector_mode], + sess_private_data, xform); + if (ret != 0) { GCM_LOG_ERR("failed configure session parameters"); - return NULL; + + /* Return session to mempool */ + rte_mempool_put(mempool, sess_private_data); + return ret; } - return sess; + set_session_private_data(sess, dev->driver_id, + sess_private_data); + + return 0; } /** Clear the memory of session so it doesn't leave key material behind */ static void -aesni_gcm_pmd_session_clear(struct rte_cryptodev *dev __rte_unused, void *sess) +aesni_gcm_pmd_session_clear(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) { - if (sess) - memset(sess, 0, sizeof(struct aesni_gcm_session)); + uint8_t index = dev->driver_id; + void *sess_priv = get_session_private_data(sess, index); + + /* Zero out the whole structure */ + if (sess_priv) { + memset(sess_priv, 0, sizeof(struct aesni_gcm_session)); + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + set_session_private_data(sess, index, NULL); + rte_mempool_put(sess_mp, sess_priv); + } } struct rte_cryptodev_ops aesni_gcm_pmd_ops = { diff --git a/dpdk/drivers/crypto/aesni_gcm/aesni_gcm_pmd_private.h b/dpdk/drivers/crypto/aesni_gcm/aesni_gcm_pmd_private.h index 9878d6e4..1c8835b5 100644 --- a/dpdk/drivers/crypto/aesni_gcm/aesni_gcm_pmd_private.h +++ b/dpdk/drivers/crypto/aesni_gcm/aesni_gcm_pmd_private.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -35,6 +35,9 @@ #include "aesni_gcm_ops.h" +#define CRYPTODEV_NAME_AESNI_GCM_PMD crypto_aesni_gcm +/**< AES-NI GCM PMD device name */ + #define GCM_LOG_ERR(fmt, args...) \ RTE_LOG(ERR, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \ RTE_STR(CRYPTODEV_NAME_AESNI_GCM_PMD), \ @@ -55,6 +58,8 @@ #define GCM_LOG_DBG(fmt, args...) #endif +/* Maximum length for digest */ +#define DIGEST_LENGTH_MAX 16 /** private data structure for each virtual AESNI GCM device */ struct aesni_gcm_private { @@ -67,38 +72,57 @@ struct aesni_gcm_private { }; struct aesni_gcm_qp { - uint16_t id; - /**< Queue Pair Identifier */ - char name[RTE_CRYPTODEV_NAME_LEN]; - /**< Unique Queue Pair Name */ const struct aesni_gcm_ops *ops; /**< Architecture dependent function pointer table of the gcm APIs */ struct rte_ring *processed_pkts; /**< Ring for placing process packets */ + struct gcm_context_data gdata_ctx; /* (16 * 5) + 8 = 88 B */ + /**< GCM parameters */ + struct rte_cryptodev_stats qp_stats; /* 8 * 4 = 32 B */ + /**< Queue pair statistics */ struct rte_mempool *sess_mp; /**< Session Mempool */ - struct rte_cryptodev_stats qp_stats; - /**< Queue pair statistics */ + uint16_t id; + /**< Queue Pair Identifier */ + char name[RTE_CRYPTODEV_NAME_LEN]; + /**< Unique Queue Pair Name */ + uint8_t temp_digest[DIGEST_LENGTH_MAX]; + /**< Buffer used to store the digest generated + * by the driver when verifying a digest provided + * by the user (using authentication verify operation) + */ } __rte_cache_aligned; enum aesni_gcm_operation { AESNI_GCM_OP_AUTHENTICATED_ENCRYPTION, - AESNI_GCM_OP_AUTHENTICATED_DECRYPTION + AESNI_GCM_OP_AUTHENTICATED_DECRYPTION, + AESNI_GMAC_OP_GENERATE, + AESNI_GMAC_OP_VERIFY }; /** AESNI GCM private session structure */ struct aesni_gcm_session { + struct { + uint16_t length; + uint16_t offset; + } iv; + /**< IV parameters */ + uint16_t aad_length; + /**< AAD length */ + uint16_t digest_length; + /**< Digest length */ enum aesni_gcm_operation op; /**< GCM operation type */ - struct gcm_data gdata __rte_cache_aligned; + enum aesni_gcm_key key; + /**< GCM key type */ + struct gcm_key_data gdata_key; /**< GCM parameters */ }; /** * Setup GCM session parameters - * @param ops gcm ops function pointer table * @param sess aesni gcm session structure * @param xform crypto transform chain * diff --git a/dpdk/drivers/crypto/aesni_mb/Makefile b/dpdk/drivers/crypto/aesni_mb/Makefile index d3994cc6..a49f06f2 100644 --- a/dpdk/drivers/crypto/aesni_mb/Makefile +++ b/dpdk/drivers/crypto/aesni_mb/Makefile @@ -47,22 +47,18 @@ CFLAGS += $(WERROR_FLAGS) LIBABIVER := 1 # versioning export map -EXPORT_MAP := rte_pmd_aesni_version.map +EXPORT_MAP := rte_pmd_aesni_mb_version.map # external library dependencies CFLAGS += -I$(AESNI_MULTI_BUFFER_LIB_PATH) CFLAGS += -I$(AESNI_MULTI_BUFFER_LIB_PATH)/include LDLIBS += -L$(AESNI_MULTI_BUFFER_LIB_PATH) -lIPSec_MB +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_cryptodev +LDLIBS += -lrte_bus_vdev # library source files SRCS-$(CONFIG_RTE_LIBRTE_PMD_AESNI_MB) += rte_aesni_mb_pmd.c SRCS-$(CONFIG_RTE_LIBRTE_PMD_AESNI_MB) += rte_aesni_mb_pmd_ops.c -# library dependencies -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_AESNI_MB) += lib/librte_eal -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_AESNI_MB) += lib/librte_mbuf -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_AESNI_MB) += lib/librte_mempool -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_AESNI_MB) += lib/librte_ring -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_AESNI_MB) += lib/librte_cryptodev - include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/crypto/aesni_mb/aesni_mb_ops.h b/dpdk/drivers/crypto/aesni_mb/aesni_mb_ops.h index 0c119bf1..59c3ee1e 100644 --- a/dpdk/drivers/crypto/aesni_mb/aesni_mb_ops.h +++ b/dpdk/drivers/crypto/aesni_mb/aesni_mb_ops.h @@ -44,29 +44,30 @@ enum aesni_mb_vector_mode { RTE_AESNI_MB_NOT_SUPPORTED = 0, RTE_AESNI_MB_SSE, RTE_AESNI_MB_AVX, - RTE_AESNI_MB_AVX2 + RTE_AESNI_MB_AVX2, + RTE_AESNI_MB_AVX512 }; -typedef void (*md5_one_block_t)(void *data, void *digest); +typedef void (*md5_one_block_t)(const void *data, void *digest); -typedef void (*sha1_one_block_t)(void *data, void *digest); -typedef void (*sha224_one_block_t)(void *data, void *digest); -typedef void (*sha256_one_block_t)(void *data, void *digest); -typedef void (*sha384_one_block_t)(void *data, void *digest); -typedef void (*sha512_one_block_t)(void *data, void *digest); +typedef void (*sha1_one_block_t)(const void *data, void *digest); +typedef void (*sha224_one_block_t)(const void *data, void *digest); +typedef void (*sha256_one_block_t)(const void *data, void *digest); +typedef void (*sha384_one_block_t)(const void *data, void *digest); +typedef void (*sha512_one_block_t)(const void *data, void *digest); typedef void (*aes_keyexp_128_t) - (void *key, void *enc_exp_keys, void *dec_exp_keys); + (const void *key, void *enc_exp_keys, void *dec_exp_keys); typedef void (*aes_keyexp_192_t) - (void *key, void *enc_exp_keys, void *dec_exp_keys); + (const void *key, void *enc_exp_keys, void *dec_exp_keys); typedef void (*aes_keyexp_256_t) - (void *key, void *enc_exp_keys, void *dec_exp_keys); + (const void *key, void *enc_exp_keys, void *dec_exp_keys); typedef void (*aes_xcbc_expand_key_t) - (void *key, void *exp_k1, void *k2, void *k3); + (const void *key, void *exp_k1, void *k2, void *k3); /** Multi-buffer library function pointer table */ -struct aesni_mb_ops { +struct aesni_mb_op_fns { struct { init_mb_mgr_t init_mgr; /**< Initialise scheduler */ @@ -115,7 +116,7 @@ struct aesni_mb_ops { }; -static const struct aesni_mb_ops job_ops[] = { +static const struct aesni_mb_op_fns job_ops[] = { [RTE_AESNI_MB_NOT_SUPPORTED] = { .job = { NULL @@ -203,6 +204,31 @@ static const struct aesni_mb_ops job_ops[] = { aes_xcbc_expand_key_avx2 } } + }, + [RTE_AESNI_MB_AVX512] = { + .job = { + init_mb_mgr_avx512, + get_next_job_avx512, + submit_job_avx512, + get_completed_job_avx512, + flush_job_avx512 + }, + .aux = { + .one_block = { + md5_one_block_avx512, + sha1_one_block_avx512, + sha224_one_block_avx512, + sha256_one_block_avx512, + sha384_one_block_avx512, + sha512_one_block_avx512 + }, + .keyexp = { + aes_keyexp_128_avx512, + aes_keyexp_192_avx512, + aes_keyexp_256_avx512, + aes_xcbc_expand_key_avx512 + } + } } }; diff --git a/dpdk/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c b/dpdk/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c index b2d0c8ca..70043897 100644 --- a/dpdk/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c +++ b/dpdk/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2015-2016 Intel Corporation. All rights reserved. + * Copyright(c) 2015-2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -30,39 +30,22 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +#include + #include #include #include #include -#include +#include #include #include #include "rte_aesni_mb_pmd_private.h" -/** - * Global static parameter used to create a unique name for each AES-NI multi - * buffer crypto device. - */ -static unsigned unique_name_id; +static uint8_t cryptodev_driver_id; -static inline int -create_unique_device_name(char *name, size_t size) -{ - int ret; - - if (name == NULL) - return -EINVAL; - - ret = snprintf(name, size, "%s_%u", RTE_STR(CRYPTODEV_NAME_AESNI_MB_PMD), - unique_name_id++); - if (ret < 0) - return ret; - return 0; -} - -typedef void (*hash_one_block_t)(void *data, void *digest); -typedef void (*aes_keyexp_t)(void *key, void *enc_exp_keys, void *dec_exp_keys); +typedef void (*hash_one_block_t)(const void *data, void *digest); +typedef void (*aes_keyexp_t)(const void *key, void *enc_exp_keys, void *dec_exp_keys); /** * Calculate the authentication pre-computes @@ -107,41 +90,50 @@ calculate_auth_precomputes(hash_one_block_t one_block_hash, } /** Get xform chain order */ -static int +static enum aesni_mb_operation aesni_mb_get_chain_order(const struct rte_crypto_sym_xform *xform) { - /* - * Multi-buffer only supports HASH_CIPHER or CIPHER_HASH chained - * operations, all other options are invalid, so we must have exactly - * 2 xform structs chained together - */ - if (xform->next == NULL || xform->next->next != NULL) - return -1; + if (xform == NULL) + return AESNI_MB_OP_NOT_SUPPORTED; - if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH && - xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) - return HASH_CIPHER; + if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { + if (xform->next == NULL) + return AESNI_MB_OP_CIPHER_ONLY; + if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) + return AESNI_MB_OP_CIPHER_HASH; + } - if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && - xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) - return CIPHER_HASH; + if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) { + if (xform->next == NULL) + return AESNI_MB_OP_HASH_ONLY; + if (xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) + return AESNI_MB_OP_HASH_CIPHER; + } - return -1; + return AESNI_MB_OP_NOT_SUPPORTED; } /** Set session authentication parameters */ static int -aesni_mb_set_session_auth_parameters(const struct aesni_mb_ops *mb_ops, +aesni_mb_set_session_auth_parameters(const struct aesni_mb_op_fns *mb_ops, struct aesni_mb_session *sess, const struct rte_crypto_sym_xform *xform) { hash_one_block_t hash_oneblock_fn; + if (xform == NULL) { + sess->auth.algo = NULL_HASH; + return 0; + } + if (xform->type != RTE_CRYPTO_SYM_XFORM_AUTH) { MB_LOG_ERR("Crypto xform struct not of type auth"); return -1; } + /* Select auth generate/verify */ + sess->auth.operation = xform->auth.op; + /* Set Authentication Parameters */ if (xform->auth.algo == RTE_CRYPTO_AUTH_AES_XCBC_MAC) { sess->auth.algo = AES_XCBC; @@ -178,7 +170,7 @@ aesni_mb_set_session_auth_parameters(const struct aesni_mb_ops *mb_ops, break; default: MB_LOG_ERR("Unsupported authentication algorithm selection"); - return -1; + return -ENOTSUP; } /* Calculate Authentication precomputes */ @@ -193,15 +185,21 @@ aesni_mb_set_session_auth_parameters(const struct aesni_mb_ops *mb_ops, /** Set session cipher parameters */ static int -aesni_mb_set_session_cipher_parameters(const struct aesni_mb_ops *mb_ops, +aesni_mb_set_session_cipher_parameters(const struct aesni_mb_op_fns *mb_ops, struct aesni_mb_session *sess, const struct rte_crypto_sym_xform *xform) { + uint8_t is_aes = 0; aes_keyexp_t aes_keyexp_fn; + if (xform == NULL) { + sess->cipher.mode = NULL_CIPHER; + return 0; + } + if (xform->type != RTE_CRYPTO_SYM_XFORM_CIPHER) { MB_LOG_ERR("Crypto xform struct not of type cipher"); - return -1; + return -EINVAL; } /* Select cipher direction */ @@ -213,117 +211,210 @@ aesni_mb_set_session_cipher_parameters(const struct aesni_mb_ops *mb_ops, sess->cipher.direction = DECRYPT; break; default: - MB_LOG_ERR("Unsupported cipher operation parameter"); - return -1; + MB_LOG_ERR("Invalid cipher operation parameter"); + return -EINVAL; } /* Select cipher mode */ switch (xform->cipher.algo) { case RTE_CRYPTO_CIPHER_AES_CBC: sess->cipher.mode = CBC; + is_aes = 1; break; case RTE_CRYPTO_CIPHER_AES_CTR: sess->cipher.mode = CNTR; + is_aes = 1; + break; + case RTE_CRYPTO_CIPHER_AES_DOCSISBPI: + sess->cipher.mode = DOCSIS_SEC_BPI; + is_aes = 1; + break; + case RTE_CRYPTO_CIPHER_DES_CBC: + sess->cipher.mode = DES; + break; + case RTE_CRYPTO_CIPHER_DES_DOCSISBPI: + sess->cipher.mode = DOCSIS_DES; break; default: MB_LOG_ERR("Unsupported cipher mode parameter"); - return -1; + return -ENOTSUP; } - /* Check key length and choose key expansion function */ - switch (xform->cipher.key.length) { - case AES_128_BYTES: - sess->cipher.key_length_in_bytes = AES_128_BYTES; - aes_keyexp_fn = mb_ops->aux.keyexp.aes128; - break; - case AES_192_BYTES: - sess->cipher.key_length_in_bytes = AES_192_BYTES; - aes_keyexp_fn = mb_ops->aux.keyexp.aes192; - break; - case AES_256_BYTES: - sess->cipher.key_length_in_bytes = AES_256_BYTES; - aes_keyexp_fn = mb_ops->aux.keyexp.aes256; - break; - default: - MB_LOG_ERR("Unsupported cipher key length"); - return -1; - } + /* Set IV parameters */ + sess->iv.offset = xform->cipher.iv.offset; + sess->iv.length = xform->cipher.iv.length; - /* Expanded cipher keys */ - (*aes_keyexp_fn)(xform->cipher.key.data, - sess->cipher.expanded_aes_keys.encode, - sess->cipher.expanded_aes_keys.decode); + /* Check key length and choose key expansion function for AES */ + if (is_aes) { + switch (xform->cipher.key.length) { + case AES_128_BYTES: + sess->cipher.key_length_in_bytes = AES_128_BYTES; + aes_keyexp_fn = mb_ops->aux.keyexp.aes128; + break; + case AES_192_BYTES: + sess->cipher.key_length_in_bytes = AES_192_BYTES; + aes_keyexp_fn = mb_ops->aux.keyexp.aes192; + break; + case AES_256_BYTES: + sess->cipher.key_length_in_bytes = AES_256_BYTES; + aes_keyexp_fn = mb_ops->aux.keyexp.aes256; + break; + default: + MB_LOG_ERR("Invalid cipher key length"); + return -EINVAL; + } + + /* Expanded cipher keys */ + (*aes_keyexp_fn)(xform->cipher.key.data, + sess->cipher.expanded_aes_keys.encode, + sess->cipher.expanded_aes_keys.decode); + + } else { + if (xform->cipher.key.length != 8) { + MB_LOG_ERR("Invalid cipher key length"); + return -EINVAL; + } + sess->cipher.key_length_in_bytes = 8; + + des_key_schedule((uint64_t *)sess->cipher.expanded_aes_keys.encode, + xform->cipher.key.data); + des_key_schedule((uint64_t *)sess->cipher.expanded_aes_keys.decode, + xform->cipher.key.data); + } return 0; } /** Parse crypto xform chain and set private session parameters */ int -aesni_mb_set_session_parameters(const struct aesni_mb_ops *mb_ops, +aesni_mb_set_session_parameters(const struct aesni_mb_op_fns *mb_ops, struct aesni_mb_session *sess, const struct rte_crypto_sym_xform *xform) { const struct rte_crypto_sym_xform *auth_xform = NULL; const struct rte_crypto_sym_xform *cipher_xform = NULL; + int ret; /* Select Crypto operation - hash then cipher / cipher then hash */ switch (aesni_mb_get_chain_order(xform)) { - case HASH_CIPHER: + case AESNI_MB_OP_HASH_CIPHER: sess->chain_order = HASH_CIPHER; auth_xform = xform; cipher_xform = xform->next; break; - case CIPHER_HASH: + case AESNI_MB_OP_CIPHER_HASH: sess->chain_order = CIPHER_HASH; auth_xform = xform->next; cipher_xform = xform; break; + case AESNI_MB_OP_HASH_ONLY: + sess->chain_order = HASH_CIPHER; + auth_xform = xform; + cipher_xform = NULL; + break; + case AESNI_MB_OP_CIPHER_ONLY: + /* + * Multi buffer library operates only at two modes, + * CIPHER_HASH and HASH_CIPHER. When doing ciphering only, + * chain order depends on cipher operation: encryption is always + * the first operation and decryption the last one. + */ + if (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) + sess->chain_order = CIPHER_HASH; + else + sess->chain_order = HASH_CIPHER; + auth_xform = NULL; + cipher_xform = xform; + break; + case AESNI_MB_OP_NOT_SUPPORTED: default: MB_LOG_ERR("Unsupported operation chain order parameter"); - return -1; + return -ENOTSUP; } - if (aesni_mb_set_session_auth_parameters(mb_ops, sess, auth_xform)) { + /* Default IV length = 0 */ + sess->iv.length = 0; + + ret = aesni_mb_set_session_auth_parameters(mb_ops, sess, auth_xform); + if (ret != 0) { MB_LOG_ERR("Invalid/unsupported authentication parameters"); - return -1; + return ret; } - if (aesni_mb_set_session_cipher_parameters(mb_ops, sess, - cipher_xform)) { + ret = aesni_mb_set_session_cipher_parameters(mb_ops, sess, + cipher_xform); + if (ret != 0) { MB_LOG_ERR("Invalid/unsupported cipher parameters"); - return -1; + return ret; } + return 0; } +/** + * burst enqueue, place crypto operations on ingress queue for processing. + * + * @param __qp Queue Pair to process + * @param ops Crypto operations for processing + * @param nb_ops Number of crypto operations for processing + * + * @return + * - Number of crypto operations enqueued + */ +static uint16_t +aesni_mb_pmd_enqueue_burst(void *__qp, struct rte_crypto_op **ops, + uint16_t nb_ops) +{ + struct aesni_mb_qp *qp = __qp; + + unsigned int nb_enqueued; + + nb_enqueued = rte_ring_enqueue_burst(qp->ingress_queue, + (void **)ops, nb_ops, NULL); + + qp->stats.enqueued_count += nb_enqueued; + + return nb_enqueued; +} + /** Get multi buffer session */ -static struct aesni_mb_session * +static inline struct aesni_mb_session * get_session(struct aesni_mb_qp *qp, struct rte_crypto_op *op) { struct aesni_mb_session *sess = NULL; - if (op->sym->sess_type == RTE_CRYPTO_SYM_OP_WITH_SESSION) { - if (unlikely(op->sym->session->dev_type != - RTE_CRYPTODEV_AESNI_MB_PMD)) - return NULL; - - sess = (struct aesni_mb_session *)op->sym->session->_private; - } else { + if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { + if (likely(op->sym->session != NULL)) + sess = (struct aesni_mb_session *) + get_session_private_data( + op->sym->session, + cryptodev_driver_id); + } else { void *_sess = NULL; + void *_sess_private_data = NULL; if (rte_mempool_get(qp->sess_mp, (void **)&_sess)) return NULL; - sess = (struct aesni_mb_session *) - ((struct rte_cryptodev_sym_session *)_sess)->_private; + if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data)) + return NULL; - if (unlikely(aesni_mb_set_session_parameters(qp->ops, + sess = (struct aesni_mb_session *)_sess_private_data; + + if (unlikely(aesni_mb_set_session_parameters(qp->op_fns, sess, op->sym->xform) != 0)) { rte_mempool_put(qp->sess_mp, _sess); + rte_mempool_put(qp->sess_mp, _sess_private_data); sess = NULL; } + op->sym->session = (struct rte_cryptodev_sym_session *)_sess; + set_session_private_data(op->sym->session, cryptodev_driver_id, + _sess_private_data); } + if (unlikely(sess == NULL)) + op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; + return sess; } @@ -339,18 +430,19 @@ get_session(struct aesni_mb_qp *qp, struct rte_crypto_op *op) * - Completed JOB_AES_HMAC structure pointer on success * - NULL pointer if completion of JOB_AES_HMAC structure isn't possible */ -static JOB_AES_HMAC * -process_crypto_op(struct aesni_mb_qp *qp, struct rte_crypto_op *op, - struct aesni_mb_session *session) +static inline int +set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp, + struct rte_crypto_op *op, uint8_t *digest_idx) { - JOB_AES_HMAC *job; - struct rte_mbuf *m_src = op->sym->m_src, *m_dst; + struct aesni_mb_session *session; uint16_t m_offset = 0; - job = (*qp->ops->job.get_next)(&qp->mb_mgr); - if (unlikely(job == NULL)) - return job; + session = get_session(qp, op); + if (session == NULL) { + op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; + return -1; + } /* Set crypto operation */ job->chain_order = session->chain_order; @@ -385,7 +477,8 @@ process_crypto_op(struct aesni_mb_qp *qp, struct rte_crypto_op *op, if (odata == NULL) { MB_LOG_ERR("failed to allocate space in destination " "mbuf for source data"); - return NULL; + op->status = RTE_CRYPTO_OP_STATUS_ERROR; + return -1; } memcpy(odata, rte_pktmbuf_mtod(op->sym->m_src, void*), @@ -396,19 +489,10 @@ process_crypto_op(struct aesni_mb_qp *qp, struct rte_crypto_op *op, } /* Set digest output location */ - if (job->cipher_direction == DECRYPT) { - job->auth_tag_output = (uint8_t *)rte_pktmbuf_append(m_dst, - get_digest_byte_length(job->hash_alg)); - - if (job->auth_tag_output == NULL) { - MB_LOG_ERR("failed to allocate space in output mbuf " - "for temp digest"); - return NULL; - } - - memset(job->auth_tag_output, 0, - sizeof(get_digest_byte_length(job->hash_alg))); - + if (job->hash_alg != NULL_HASH && + session->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) { + job->auth_tag_output = qp->temp_digests[*digest_idx]; + *digest_idx = (*digest_idx + 1) % MAX_JOBS; } else { job->auth_tag_output = op->sym->auth.digest.data; } @@ -421,8 +505,9 @@ process_crypto_op(struct aesni_mb_qp *qp, struct rte_crypto_op *op, get_truncated_digest_byte_length(job->hash_alg); /* Set IV parameters */ - job->iv = op->sym->cipher.iv.data; - job->iv_len_in_bytes = op->sym->cipher.iv.length; + job->iv = rte_crypto_op_ctod_offset(op, uint8_t *, + session->iv.offset); + job->iv_len_in_bytes = session->iv.length; /* Data Parameter */ job->src = rte_pktmbuf_mtod(m_src, uint8_t *); @@ -436,51 +521,59 @@ process_crypto_op(struct aesni_mb_qp *qp, struct rte_crypto_op *op, /* Set user data to be crypto operation data struct */ job->user_data = op; - job->user_data2 = m_dst; - return job; + return 0; +} + +static inline void +verify_digest(struct aesni_mb_qp *qp __rte_unused, JOB_AES_HMAC *job, + struct rte_crypto_op *op) { + /* Verify digest if required */ + if (memcmp(job->auth_tag_output, op->sym->auth.digest.data, + job->auth_tag_output_len_in_bytes) != 0) + op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; } /** * Process a completed job and return rte_mbuf which job processed * + * @param qp Queue Pair to process * @param job JOB_AES_HMAC job to process * * @return - * - Returns processed mbuf which is trimmed of output digest used in - * verification of supplied digest in the case of a HASH_CIPHER operation + * - Returns processed crypto operation. * - Returns NULL on invalid job */ -static struct rte_crypto_op * +static inline struct rte_crypto_op * post_process_mb_job(struct aesni_mb_qp *qp, JOB_AES_HMAC *job) { - struct rte_crypto_op *op = - (struct rte_crypto_op *)job->user_data; - struct rte_mbuf *m_dst = - (struct rte_mbuf *)job->user_data2; + struct rte_crypto_op *op = (struct rte_crypto_op *)job->user_data; + struct aesni_mb_session *sess = get_session_private_data( + op->sym->session, + cryptodev_driver_id); - if (op == NULL || m_dst == NULL) - return NULL; + if (likely(op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)) { + switch (job->status) { + case STS_COMPLETED: + op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; - /* set status as successful by default */ - op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; - - /* check if job has been processed */ - if (unlikely(job->status != STS_COMPLETED)) { - op->status = RTE_CRYPTO_OP_STATUS_ERROR; - return op; - } else if (job->chain_order == HASH_CIPHER) { - /* Verify digest if required */ - if (memcmp(job->auth_tag_output, op->sym->auth.digest.data, - job->auth_tag_output_len_in_bytes) != 0) - op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; - - /* trim area used for digest from mbuf */ - rte_pktmbuf_trim(m_dst, get_digest_byte_length(job->hash_alg)); + if (job->hash_alg != NULL_HASH) { + if (sess->auth.operation == + RTE_CRYPTO_AUTH_OP_VERIFY) + verify_digest(qp, job, op); + } + break; + default: + op->status = RTE_CRYPTO_OP_STATUS_ERROR; + } } /* Free session if a session-less crypto op */ - if (op->sym->sess_type == RTE_CRYPTO_SYM_OP_SESSIONLESS) { + if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { + memset(sess, 0, sizeof(struct aesni_mb_session)); + memset(op->sym->session, 0, + rte_cryptodev_get_header_session_size()); + rte_mempool_put(qp->sess_mp, sess); rte_mempool_put(qp->sess_mp, op->sym->session); op->sym->session = NULL; } @@ -499,84 +592,59 @@ post_process_mb_job(struct aesni_mb_qp *qp, JOB_AES_HMAC *job) * - Number of processed jobs */ static unsigned -handle_completed_jobs(struct aesni_mb_qp *qp, JOB_AES_HMAC *job) +handle_completed_jobs(struct aesni_mb_qp *qp, JOB_AES_HMAC *job, + struct rte_crypto_op **ops, uint16_t nb_ops) { struct rte_crypto_op *op = NULL; unsigned processed_jobs = 0; - while (job) { - processed_jobs++; + while (job != NULL) { op = post_process_mb_job(qp, job); - if (op) - rte_ring_enqueue(qp->processed_ops, (void *)op); - else + + if (op) { + ops[processed_jobs++] = op; + qp->stats.dequeued_count++; + } else { qp->stats.dequeue_err_count++; - job = (*qp->ops->job.get_completed_job)(&qp->mb_mgr); + break; + } + if (processed_jobs == nb_ops) + break; + + job = (*qp->op_fns->job.get_completed_job)(&qp->mb_mgr); } return processed_jobs; } -static uint16_t -aesni_mb_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops, +static inline uint16_t +flush_mb_mgr(struct aesni_mb_qp *qp, struct rte_crypto_op **ops, uint16_t nb_ops) { - struct aesni_mb_session *sess; - struct aesni_mb_qp *qp = queue_pair; + int processed_ops = 0; - JOB_AES_HMAC *job = NULL; + /* Flush the remaining jobs */ + JOB_AES_HMAC *job = (*qp->op_fns->job.flush_job)(&qp->mb_mgr); - int i, processed_jobs = 0; - - for (i = 0; i < nb_ops; i++) { -#ifdef RTE_LIBRTE_AESNI_MB_DEBUG - if (unlikely(op->type != RTE_CRYPTO_OP_TYPE_SYMMETRIC)) { - MB_LOG_ERR("PMD only supports symmetric crypto " - "operation requests, op (%p) is not a " - "symmetric operation.", op); - qp->stats.enqueue_err_count++; - goto flush_jobs; - } -#endif - sess = get_session(qp, ops[i]); - if (unlikely(sess == NULL)) { - qp->stats.enqueue_err_count++; - goto flush_jobs; - } - - job = process_crypto_op(qp, ops[i], sess); - if (unlikely(job == NULL)) { - qp->stats.enqueue_err_count++; - goto flush_jobs; - } - - /* Submit Job */ - job = (*qp->ops->job.submit)(&qp->mb_mgr); - - /* - * If submit returns a processed job then handle it, - * before submitting subsequent jobs - */ - if (job) - processed_jobs += handle_completed_jobs(qp, job); - } - - if (processed_jobs == 0) - goto flush_jobs; - else - qp->stats.enqueued_count += processed_jobs; - return i; - -flush_jobs: - /* - * If we haven't processed any jobs in submit loop, then flush jobs - * queue to stop the output stalling - */ - job = (*qp->ops->job.flush_job)(&qp->mb_mgr); if (job) - qp->stats.enqueued_count += handle_completed_jobs(qp, job); + processed_ops += handle_completed_jobs(qp, job, + &ops[processed_ops], nb_ops - processed_ops); - return i; + return processed_ops; +} + +static inline JOB_AES_HMAC * +set_job_null_op(JOB_AES_HMAC *job, struct rte_crypto_op *op) +{ + job->chain_order = HASH_CIPHER; + job->cipher_mode = NULL_CIPHER; + job->hash_alg = NULL_HASH; + job->cipher_direction = DECRYPT; + + /* Set user data to be crypto operation data struct */ + job->user_data = op; + + return job; } static uint16_t @@ -585,24 +653,70 @@ aesni_mb_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops, { struct aesni_mb_qp *qp = queue_pair; - unsigned nb_dequeued; + struct rte_crypto_op *op; + JOB_AES_HMAC *job; - nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops, - (void **)ops, nb_ops); - qp->stats.dequeued_count += nb_dequeued; + int retval, processed_jobs = 0; - return nb_dequeued; + if (unlikely(nb_ops == 0)) + return 0; + + uint8_t digest_idx = qp->digest_idx; + do { + /* Get next operation to process from ingress queue */ + retval = rte_ring_dequeue(qp->ingress_queue, (void **)&op); + if (retval < 0) + break; + + /* Get next free mb job struct from mb manager */ + job = (*qp->op_fns->job.get_next)(&qp->mb_mgr); + if (unlikely(job == NULL)) { + /* if no free mb job structs we need to flush mb_mgr */ + processed_jobs += flush_mb_mgr(qp, + &ops[processed_jobs], + (nb_ops - processed_jobs) - 1); + + job = (*qp->op_fns->job.get_next)(&qp->mb_mgr); + } + + retval = set_mb_job_params(job, qp, op, &digest_idx); + if (unlikely(retval != 0)) { + qp->stats.dequeue_err_count++; + set_job_null_op(job, op); + } + + /* Submit job to multi-buffer for processing */ + job = (*qp->op_fns->job.submit)(&qp->mb_mgr); + + /* + * If submit returns a processed job then handle it, + * before submitting subsequent jobs + */ + if (job) + processed_jobs += handle_completed_jobs(qp, job, + &ops[processed_jobs], + nb_ops - processed_jobs); + + } while (processed_jobs < nb_ops); + + qp->digest_idx = digest_idx; + + if (processed_jobs < 1) + processed_jobs += flush_mb_mgr(qp, + &ops[processed_jobs], + nb_ops - processed_jobs); + + return processed_jobs; } - -static int cryptodev_aesni_mb_uninit(const char *name); +static int cryptodev_aesni_mb_remove(struct rte_vdev_device *vdev); static int cryptodev_aesni_mb_create(const char *name, - struct rte_crypto_vdev_init_params *init_params) + struct rte_vdev_device *vdev, + struct rte_cryptodev_pmd_init_params *init_params) { struct rte_cryptodev *dev; - char crypto_dev_name[RTE_CRYPTODEV_NAME_MAX_LEN]; struct aesni_mb_private *internals; enum aesni_mb_vector_mode vector_mode; @@ -612,34 +726,23 @@ cryptodev_aesni_mb_create(const char *name, return -EFAULT; } + dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params); + if (dev == NULL) { + MB_LOG_ERR("failed to create cryptodev vdev"); + return -ENODEV; + } + /* Check CPU for supported vector instruction set */ - if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2)) + if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F)) + vector_mode = RTE_AESNI_MB_AVX512; + else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2)) vector_mode = RTE_AESNI_MB_AVX2; else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX)) vector_mode = RTE_AESNI_MB_AVX; - else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_SSE4_1)) + else vector_mode = RTE_AESNI_MB_SSE; - else { - MB_LOG_ERR("Vector instructions are not supported by CPU"); - return -EFAULT; - } - /* create a unique device name */ - if (create_unique_device_name(crypto_dev_name, - RTE_CRYPTODEV_NAME_MAX_LEN) != 0) { - MB_LOG_ERR("failed to create unique cryptodev name"); - return -EINVAL; - } - - - dev = rte_cryptodev_pmd_virtual_dev_init(crypto_dev_name, - sizeof(struct aesni_mb_private), init_params->socket_id); - if (dev == NULL) { - MB_LOG_ERR("failed to create cryptodev vdev"); - goto init_error; - } - - dev->dev_type = RTE_CRYPTODEV_AESNI_MB_PMD; + dev->driver_id = cryptodev_driver_id; dev->dev_ops = rte_aesni_mb_pmd_ops; /* register rx/tx burst functions for data path */ @@ -660,6 +763,9 @@ cryptodev_aesni_mb_create(const char *name, case RTE_AESNI_MB_AVX2: dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX2; break; + case RTE_AESNI_MB_AVX512: + dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX512; + break; default: break; } @@ -672,56 +778,67 @@ cryptodev_aesni_mb_create(const char *name, internals->max_nb_sessions = init_params->max_nb_sessions; return 0; -init_error: - MB_LOG_ERR("driver %s: cryptodev_aesni_create failed", name); - - cryptodev_aesni_mb_uninit(crypto_dev_name); - return -EFAULT; } - static int -cryptodev_aesni_mb_init(const char *name, - const char *input_args) +cryptodev_aesni_mb_probe(struct rte_vdev_device *vdev) { - struct rte_crypto_vdev_init_params init_params = { - RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_QUEUE_PAIRS, - RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_SESSIONS, - rte_socket_id() + struct rte_cryptodev_pmd_init_params init_params = { + "", + sizeof(struct aesni_mb_private), + rte_socket_id(), + RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS, + RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_SESSIONS }; + const char *name, *args; + int retval; - rte_cryptodev_parse_vdev_init_params(&init_params, input_args); - - RTE_LOG(INFO, PMD, "Initialising %s on NUMA node %d\n", name, - init_params.socket_id); - RTE_LOG(INFO, PMD, " Max number of queue pairs = %d\n", - init_params.max_nb_queue_pairs); - RTE_LOG(INFO, PMD, " Max number of sessions = %d\n", - init_params.max_nb_sessions); - - return cryptodev_aesni_mb_create(name, &init_params); -} - -static int -cryptodev_aesni_mb_uninit(const char *name) -{ + name = rte_vdev_device_name(vdev); if (name == NULL) return -EINVAL; - RTE_LOG(INFO, PMD, "Closing AESNI crypto device %s on numa socket %u\n", - name, rte_socket_id()); + args = rte_vdev_device_args(vdev); - return 0; + retval = rte_cryptodev_pmd_parse_input_args(&init_params, args); + if (retval) { + MB_LOG_ERR("Failed to parse initialisation arguments[%s]\n", + args); + return -EINVAL; + } + + return cryptodev_aesni_mb_create(name, vdev, &init_params); } -static struct rte_driver cryptodev_aesni_mb_pmd_drv = { - .type = PMD_VDEV, - .init = cryptodev_aesni_mb_init, - .uninit = cryptodev_aesni_mb_uninit +static int +cryptodev_aesni_mb_remove(struct rte_vdev_device *vdev) +{ + struct rte_cryptodev *cryptodev; + const char *name; + + name = rte_vdev_device_name(vdev); + if (name == NULL) + return -EINVAL; + + cryptodev = rte_cryptodev_pmd_get_named_dev(name); + if (cryptodev == NULL) + return -ENODEV; + + return rte_cryptodev_pmd_destroy(cryptodev); +} + +static struct rte_vdev_driver cryptodev_aesni_mb_pmd_drv = { + .probe = cryptodev_aesni_mb_probe, + .remove = cryptodev_aesni_mb_remove }; -PMD_REGISTER_DRIVER(cryptodev_aesni_mb_pmd_drv, CRYPTODEV_NAME_AESNI_MB_PMD); -DRIVER_REGISTER_PARAM_STRING(CRYPTODEV_NAME_AESNI_MB_PMD, +static struct cryptodev_driver aesni_mb_crypto_drv; + +RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_AESNI_MB_PMD, cryptodev_aesni_mb_pmd_drv); +RTE_PMD_REGISTER_ALIAS(CRYPTODEV_NAME_AESNI_MB_PMD, cryptodev_aesni_mb_pmd); +RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_AESNI_MB_PMD, "max_nb_queue_pairs= " "max_nb_sessions= " "socket_id="); +RTE_PMD_REGISTER_CRYPTO_DRIVER(aesni_mb_crypto_drv, + cryptodev_aesni_mb_pmd_drv, + cryptodev_driver_id); diff --git a/dpdk/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c b/dpdk/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c index d3c46ace..3b3ef0c0 100644 --- a/dpdk/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c +++ b/dpdk/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2015-2016 Intel Corporation. All rights reserved. + * Copyright(c) 2015-2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -48,16 +48,16 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = { .algo = RTE_CRYPTO_AUTH_MD5_HMAC, .block_size = 64, .key_size = { - .min = 64, + .min = 1, .max = 64, - .increment = 0 + .increment = 1 }, .digest_size = { .min = 12, .max = 12, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, } }, } }, @@ -69,16 +69,16 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = { .algo = RTE_CRYPTO_AUTH_SHA1_HMAC, .block_size = 64, .key_size = { - .min = 64, + .min = 1, .max = 64, - .increment = 0 + .increment = 1 }, .digest_size = { .min = 12, .max = 12, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, } }, } }, @@ -90,16 +90,16 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = { .algo = RTE_CRYPTO_AUTH_SHA224_HMAC, .block_size = 64, .key_size = { - .min = 64, + .min = 1, .max = 64, - .increment = 0 + .increment = 1 }, .digest_size = { .min = 14, .max = 14, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, } }, } }, @@ -111,16 +111,16 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = { .algo = RTE_CRYPTO_AUTH_SHA256_HMAC, .block_size = 64, .key_size = { - .min = 64, + .min = 1, .max = 64, - .increment = 0 + .increment = 1 }, .digest_size = { .min = 16, .max = 16, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, } }, } }, @@ -132,16 +132,16 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = { .algo = RTE_CRYPTO_AUTH_SHA384_HMAC, .block_size = 128, .key_size = { - .min = 128, + .min = 1, .max = 128, - .increment = 0 + .increment = 1 }, .digest_size = { .min = 24, .max = 24, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, } }, } }, @@ -153,16 +153,16 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = { .algo = RTE_CRYPTO_AUTH_SHA512_HMAC, .block_size = 128, .key_size = { - .min = 128, + .min = 1, .max = 128, - .increment = 0 + .increment = 1 }, .digest_size = { .min = 32, .max = 32, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, } }, } }, @@ -183,7 +183,7 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = { .max = 12, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, } }, } }, @@ -219,6 +219,26 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = { .max = 32, .increment = 8 }, + .iv_size = { + .min = 12, + .max = 16, + .increment = 4 + } + }, } + }, } + }, + { /* AES DOCSIS BPI */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, + {.cipher = { + .algo = RTE_CRYPTO_CIPHER_AES_DOCSISBPI, + .block_size = 16, + .key_size = { + .min = 16, + .max = 16, + .increment = 0 + }, .iv_size = { .min = 16, .max = 16, @@ -227,13 +247,57 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = { }, } }, } }, + { /* DES CBC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, + {.cipher = { + .algo = RTE_CRYPTO_CIPHER_DES_CBC, + .block_size = 8, + .key_size = { + .min = 8, + .max = 8, + .increment = 0 + }, + .iv_size = { + .min = 8, + .max = 8, + .increment = 0 + } + }, } + }, } + }, + { /* DES DOCSIS BPI */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, + {.cipher = { + .algo = RTE_CRYPTO_CIPHER_DES_DOCSISBPI, + .block_size = 8, + .key_size = { + .min = 8, + .max = 8, + .increment = 0 + }, + .iv_size = { + .min = 8, + .max = 8, + .increment = 0 + } + }, } + }, } + }, + + + RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST() }; /** Configure device */ static int -aesni_mb_pmd_config(__rte_unused struct rte_cryptodev *dev) +aesni_mb_pmd_config(__rte_unused struct rte_cryptodev *dev, + __rte_unused struct rte_cryptodev_config *config) { return 0; } @@ -299,7 +363,7 @@ aesni_mb_pmd_info_get(struct rte_cryptodev *dev, struct aesni_mb_private *internals = dev->data->dev_private; if (dev_info != NULL) { - dev_info->dev_type = dev->dev_type; + dev_info->driver_id = dev->driver_id; dev_info->feature_flags = dev->feature_flags; dev_info->capabilities = aesni_mb_pmd_capabilities; dev_info->max_nb_queue_pairs = internals->max_nb_queue_pairs; @@ -311,8 +375,14 @@ aesni_mb_pmd_info_get(struct rte_cryptodev *dev, static int aesni_mb_pmd_qp_release(struct rte_cryptodev *dev, uint16_t qp_id) { - if (dev->data->queue_pairs[qp_id] != NULL) { - rte_free(dev->data->queue_pairs[qp_id]); + struct aesni_mb_qp *qp = dev->data->queue_pairs[qp_id]; + struct rte_ring *r = NULL; + + if (qp != NULL) { + r = rte_ring_lookup(qp->name); + if (r) + rte_ring_free(r); + rte_free(qp); dev->data->queue_pairs[qp_id] = NULL; } return 0; @@ -327,7 +397,7 @@ aesni_mb_pmd_qp_set_unique_name(struct rte_cryptodev *dev, "aesni_mb_pmd_%u_qp_%u", dev->data->dev_id, qp->id); - if (n > sizeof(qp->name)) + if (n >= sizeof(qp->name)) return -1; return 0; @@ -336,24 +406,32 @@ aesni_mb_pmd_qp_set_unique_name(struct rte_cryptodev *dev, /** Create a ring to place processed operations on */ static struct rte_ring * aesni_mb_pmd_qp_create_processed_ops_ring(struct aesni_mb_qp *qp, - unsigned ring_size, int socket_id) + const char *str, unsigned int ring_size, int socket_id) { struct rte_ring *r; + char ring_name[RTE_CRYPTODEV_NAME_LEN]; - r = rte_ring_lookup(qp->name); + unsigned int n = snprintf(ring_name, sizeof(ring_name), + "%s_%s", + qp->name, str); + + if (n >= sizeof(ring_name)) + return NULL; + + r = rte_ring_lookup(ring_name); if (r) { - if (r->prod.size >= ring_size) { + if (rte_ring_get_size(r) >= ring_size) { MB_LOG_INFO("Reusing existing ring %s for processed ops", - qp->name); + ring_name); return r; } MB_LOG_ERR("Unable to reuse existing ring %s for processed ops", - qp->name); + ring_name); return NULL; } - return rte_ring_create(qp->name, ring_size, socket_id, + return rte_ring_create(ring_name, ring_size, socket_id, RING_F_SP_ENQ | RING_F_SC_DEQ); } @@ -361,7 +439,7 @@ aesni_mb_pmd_qp_create_processed_ops_ring(struct aesni_mb_qp *qp, static int aesni_mb_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, const struct rte_cryptodev_qp_conf *qp_conf, - int socket_id) + int socket_id, struct rte_mempool *session_pool) { struct aesni_mb_qp *qp = NULL; struct aesni_mb_private *internals = dev->data->dev_private; @@ -382,20 +460,25 @@ aesni_mb_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, if (aesni_mb_pmd_qp_set_unique_name(dev, qp)) goto qp_setup_cleanup; - qp->ops = &job_ops[internals->vector_mode]; - qp->processed_ops = aesni_mb_pmd_qp_create_processed_ops_ring(qp, - qp_conf->nb_descriptors, socket_id); - if (qp->processed_ops == NULL) + qp->op_fns = &job_ops[internals->vector_mode]; + + qp->ingress_queue = aesni_mb_pmd_qp_create_processed_ops_ring(qp, + "ingress", qp_conf->nb_descriptors, socket_id); + if (qp->ingress_queue == NULL) goto qp_setup_cleanup; - qp->sess_mp = dev->data->session_pool; + qp->sess_mp = session_pool; memset(&qp->stats, 0, sizeof(qp->stats)); - /* Initialise multi-buffer manager */ - (*qp->ops->job.init_mgr)(&qp->mb_mgr); + char mp_name[RTE_MEMPOOL_NAMESIZE]; + snprintf(mp_name, RTE_MEMPOOL_NAMESIZE, + "digest_mp_%u_%u", dev->data->dev_id, qp_id); + + /* Initialise multi-buffer manager */ + (*qp->op_fns->job.init_mgr)(&qp->mb_mgr); return 0; qp_setup_cleanup: @@ -436,36 +519,58 @@ aesni_mb_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused) } /** Configure a aesni multi-buffer session from a crypto xform chain */ -static void * +static int aesni_mb_pmd_session_configure(struct rte_cryptodev *dev, - struct rte_crypto_sym_xform *xform, void *sess) + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool) { + void *sess_private_data; struct aesni_mb_private *internals = dev->data->dev_private; + int ret; if (unlikely(sess == NULL)) { MB_LOG_ERR("invalid session struct"); - return NULL; + return -EINVAL; } - if (aesni_mb_set_session_parameters(&job_ops[internals->vector_mode], - sess, xform) != 0) { + if (rte_mempool_get(mempool, &sess_private_data)) { + CDEV_LOG_ERR( + "Couldn't get object from session mempool"); + return -ENOMEM; + } + + ret = aesni_mb_set_session_parameters(&job_ops[internals->vector_mode], + sess_private_data, xform); + if (ret != 0) { MB_LOG_ERR("failed configure session parameters"); - return NULL; + + /* Return session to mempool */ + rte_mempool_put(mempool, sess_private_data); + return ret; } - return sess; + set_session_private_data(sess, dev->driver_id, + sess_private_data); + + return 0; } /** Clear the memory of session so it doesn't leave key material behind */ static void -aesni_mb_pmd_session_clear(struct rte_cryptodev *dev __rte_unused, void *sess) +aesni_mb_pmd_session_clear(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) { - /* - * Current just resetting the whole data structure, need to investigate - * whether a more selective reset of key would be more performant - */ - if (sess) - memset(sess, 0, sizeof(struct aesni_mb_session)); + uint8_t index = dev->driver_id; + void *sess_priv = get_session_private_data(sess, index); + + /* Zero out the whole structure */ + if (sess_priv) { + memset(sess_priv, 0, sizeof(struct aesni_mb_session)); + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + set_session_private_data(sess, index, NULL); + rte_mempool_put(sess_mp, sess_priv); + } } struct rte_cryptodev_ops aesni_mb_pmd_ops = { diff --git a/dpdk/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h b/dpdk/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h index 17f367f4..fe3bd730 100644 --- a/dpdk/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h +++ b/dpdk/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h @@ -35,6 +35,9 @@ #include "aesni_mb_ops.h" +#define CRYPTODEV_NAME_AESNI_MB_PMD crypto_aesni_mb +/**< AES-NI Multi buffer PMD device name */ + #define MB_LOG_ERR(fmt, args...) \ RTE_LOG(ERR, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \ RTE_STR(CRYPTODEV_NAME_AESNI_MB_PMD), \ @@ -58,6 +61,8 @@ #define HMAC_IPAD_VALUE (0x36) #define HMAC_OPAD_VALUE (0x5C) +/* Maximum length for digest (SHA-512 truncated needs 32 bytes) */ +#define DIGEST_LENGTH_MAX 32 static const unsigned auth_blocksize[] = { [MD5] = 64, [SHA1] = 64, @@ -88,6 +93,7 @@ static const unsigned auth_truncated_digest_byte_lengths[] = { [SHA_384] = 24, [SHA_512] = 32, [AES_XCBC] = 12, + [NULL_HASH] = 0 }; /** @@ -111,6 +117,7 @@ static const unsigned auth_digest_byte_lengths[] = { [SHA_384] = 48, [SHA_512] = 64, [AES_XCBC] = 16, + [NULL_HASH] = 0 }; /** @@ -125,6 +132,13 @@ get_digest_byte_length(JOB_HASH_ALG algo) return auth_digest_byte_lengths[algo]; } +enum aesni_mb_operation { + AESNI_MB_OP_HASH_CIPHER, + AESNI_MB_OP_CIPHER_HASH, + AESNI_MB_OP_HASH_ONLY, + AESNI_MB_OP_CIPHER_ONLY, + AESNI_MB_OP_NOT_SUPPORTED +}; /** private data structure for each virtual AESNI device */ struct aesni_mb_private { @@ -142,22 +156,35 @@ struct aesni_mb_qp { /**< Queue Pair Identifier */ char name[RTE_CRYPTODEV_NAME_LEN]; /**< Unique Queue Pair Name */ - const struct aesni_mb_ops *ops; + const struct aesni_mb_op_fns *op_fns; /**< Vector mode dependent pointer table of the multi-buffer APIs */ MB_MGR mb_mgr; /**< Multi-buffer instance */ - struct rte_ring *processed_ops; - /**< Ring for placing process operations */ + struct rte_ring *ingress_queue; + /**< Ring for placing operations ready for processing */ struct rte_mempool *sess_mp; /**< Session Mempool */ struct rte_cryptodev_stats stats; /**< Queue pair statistics */ + uint8_t digest_idx; + /**< Index of the next slot to be used in temp_digests, + * to store the digest for a given operation + */ + uint8_t temp_digests[MAX_JOBS][DIGEST_LENGTH_MAX]; + /**< Buffers used to store the digest generated + * by the driver when verifying a digest provided + * by the user (using authentication verify operation) + */ } __rte_cache_aligned; - /** AES-NI multi-buffer private session structure */ struct aesni_mb_session { JOB_CHAIN_ORDER chain_order; + struct { + uint16_t length; + uint16_t offset; + } iv; + /**< IV parameters */ /** Cipher Parameters */ struct { @@ -185,6 +212,8 @@ struct aesni_mb_session { /** Authentication Parameters */ struct { JOB_HASH_ALG algo; /**< Authentication Algorithm */ + enum rte_crypto_auth_operation operation; + /**< auth operation generate or verify */ union { struct { uint8_t inner[128] __rte_aligned(16); @@ -216,7 +245,7 @@ struct aesni_mb_session { * */ extern int -aesni_mb_set_session_parameters(const struct aesni_mb_ops *mb_ops, +aesni_mb_set_session_parameters(const struct aesni_mb_op_fns *mb_ops, struct aesni_mb_session *sess, const struct rte_crypto_sym_xform *xform); diff --git a/dpdk/drivers/crypto/aesni_mb/rte_pmd_aesni_version.map b/dpdk/drivers/crypto/aesni_mb/rte_pmd_aesni_mb_version.map similarity index 100% rename from dpdk/drivers/crypto/aesni_mb/rte_pmd_aesni_version.map rename to dpdk/drivers/crypto/aesni_mb/rte_pmd_aesni_mb_version.map diff --git a/dpdk/drivers/crypto/armv8/Makefile b/dpdk/drivers/crypto/armv8/Makefile new file mode 100644 index 00000000..79c260ff --- /dev/null +++ b/dpdk/drivers/crypto/armv8/Makefile @@ -0,0 +1,68 @@ +# +# BSD LICENSE +# +# Copyright (C) Cavium, Inc. 2017. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Cavium, Inc nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +include $(RTE_SDK)/mk/rte.vars.mk + +ifneq ($(MAKECMDGOALS),clean) +ifneq ($(MAKECMDGOALS),config) +ifeq ($(ARMV8_CRYPTO_LIB_PATH),) +$(error "Please define ARMV8_CRYPTO_LIB_PATH environment variable") +endif +endif +endif + +# library name +LIB = librte_pmd_armv8.a + +# build flags +CFLAGS += -O3 +CFLAGS += $(WERROR_FLAGS) + +# library version +LIBABIVER := 1 + +# versioning export map +EXPORT_MAP := rte_pmd_armv8_version.map + +# external library dependencies +CFLAGS += -I$(ARMV8_CRYPTO_LIB_PATH) +CFLAGS += -I$(ARMV8_CRYPTO_LIB_PATH)/asm/include +LDLIBS += -L$(ARMV8_CRYPTO_LIB_PATH) -larmv8_crypto +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_cryptodev +LDLIBS += -lrte_bus_vdev + +# library source files +SRCS-$(CONFIG_RTE_LIBRTE_PMD_ARMV8_CRYPTO) += rte_armv8_pmd.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_ARMV8_CRYPTO) += rte_armv8_pmd_ops.c + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/crypto/armv8/rte_armv8_pmd.c b/dpdk/drivers/crypto/armv8/rte_armv8_pmd.c new file mode 100644 index 00000000..97719f27 --- /dev/null +++ b/dpdk/drivers/crypto/armv8/rte_armv8_pmd.c @@ -0,0 +1,882 @@ +/* + * BSD LICENSE + * + * Copyright (C) Cavium, Inc. 2017. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "armv8_crypto_defs.h" + +#include "rte_armv8_pmd_private.h" + +static uint8_t cryptodev_driver_id; + +static int cryptodev_armv8_crypto_uninit(struct rte_vdev_device *vdev); + +/** + * Pointers to the supported combined mode crypto functions are stored + * in the static tables. Each combined (chained) cryptographic operation + * can be described by a set of numbers: + * - order: order of operations (cipher, auth) or (auth, cipher) + * - direction: encryption or decryption + * - calg: cipher algorithm such as AES_CBC, AES_CTR, etc. + * - aalg: authentication algorithm such as SHA1, SHA256, etc. + * - keyl: cipher key length, for example 128, 192, 256 bits + * + * In order to quickly acquire each function pointer based on those numbers, + * a hierarchy of arrays is maintained. The final level, 3D array is indexed + * by the combined mode function parameters only (cipher algorithm, + * authentication algorithm and key length). + * + * This gives 3 memory accesses to obtain a function pointer instead of + * traversing the array manually and comparing function parameters on each loop. + * + * +--+CRYPTO_FUNC + * +--+ENC| + * +--+CA| + * | +--+DEC + * ORDER| + * | +--+ENC + * +--+AC| + * +--+DEC + * + */ + +/** + * 3D array type for ARM Combined Mode crypto functions pointers. + * CRYPTO_CIPHER_MAX: max cipher ID number + * CRYPTO_AUTH_MAX: max auth ID number + * CRYPTO_CIPHER_KEYLEN_MAX: max key length ID number + */ +typedef const crypto_func_t +crypto_func_tbl_t[CRYPTO_CIPHER_MAX][CRYPTO_AUTH_MAX][CRYPTO_CIPHER_KEYLEN_MAX]; + +/* Evaluate to key length definition */ +#define KEYL(keyl) (ARMV8_CRYPTO_CIPHER_KEYLEN_ ## keyl) + +/* Local aliases for supported ciphers */ +#define CIPH_AES_CBC RTE_CRYPTO_CIPHER_AES_CBC +/* Local aliases for supported hashes */ +#define AUTH_SHA1_HMAC RTE_CRYPTO_AUTH_SHA1_HMAC +#define AUTH_SHA256_HMAC RTE_CRYPTO_AUTH_SHA256_HMAC + +/** + * Arrays containing pointers to particular cryptographic, + * combined mode functions. + * crypto_op_ca_encrypt: cipher (encrypt), authenticate + * crypto_op_ca_decrypt: cipher (decrypt), authenticate + * crypto_op_ac_encrypt: authenticate, cipher (encrypt) + * crypto_op_ac_decrypt: authenticate, cipher (decrypt) + */ +static const crypto_func_tbl_t +crypto_op_ca_encrypt = { + /* [cipher alg][auth alg][key length] = crypto_function, */ + [CIPH_AES_CBC][AUTH_SHA1_HMAC][KEYL(128)] = aes128cbc_sha1_hmac, + [CIPH_AES_CBC][AUTH_SHA256_HMAC][KEYL(128)] = aes128cbc_sha256_hmac, +}; + +static const crypto_func_tbl_t +crypto_op_ca_decrypt = { + NULL +}; + +static const crypto_func_tbl_t +crypto_op_ac_encrypt = { + NULL +}; + +static const crypto_func_tbl_t +crypto_op_ac_decrypt = { + /* [cipher alg][auth alg][key length] = crypto_function, */ + [CIPH_AES_CBC][AUTH_SHA1_HMAC][KEYL(128)] = sha1_hmac_aes128cbc_dec, + [CIPH_AES_CBC][AUTH_SHA256_HMAC][KEYL(128)] = sha256_hmac_aes128cbc_dec, +}; + +/** + * Arrays containing pointers to particular cryptographic function sets, + * covering given cipher operation directions (encrypt, decrypt) + * for each order of cipher and authentication pairs. + */ +static const crypto_func_tbl_t * +crypto_cipher_auth[] = { + &crypto_op_ca_encrypt, + &crypto_op_ca_decrypt, + NULL +}; + +static const crypto_func_tbl_t * +crypto_auth_cipher[] = { + &crypto_op_ac_encrypt, + &crypto_op_ac_decrypt, + NULL +}; + +/** + * Top level array containing pointers to particular cryptographic + * function sets, covering given order of chained operations. + * crypto_cipher_auth: cipher first, authenticate after + * crypto_auth_cipher: authenticate first, cipher after + */ +static const crypto_func_tbl_t ** +crypto_chain_order[] = { + crypto_cipher_auth, + crypto_auth_cipher, + NULL +}; + +/** + * Extract particular combined mode crypto function from the 3D array. + */ +#define CRYPTO_GET_ALGO(order, cop, calg, aalg, keyl) \ +({ \ + crypto_func_tbl_t *func_tbl = \ + (crypto_chain_order[(order)])[(cop)]; \ + \ + ((*func_tbl)[(calg)][(aalg)][KEYL(keyl)]); \ +}) + +/*----------------------------------------------------------------------------*/ + +/** + * 2D array type for ARM key schedule functions pointers. + * CRYPTO_CIPHER_MAX: max cipher ID number + * CRYPTO_CIPHER_KEYLEN_MAX: max key length ID number + */ +typedef const crypto_key_sched_t +crypto_key_sched_tbl_t[CRYPTO_CIPHER_MAX][CRYPTO_CIPHER_KEYLEN_MAX]; + +static const crypto_key_sched_tbl_t +crypto_key_sched_encrypt = { + /* [cipher alg][key length] = key_expand_func, */ + [CIPH_AES_CBC][KEYL(128)] = aes128_key_sched_enc, +}; + +static const crypto_key_sched_tbl_t +crypto_key_sched_decrypt = { + /* [cipher alg][key length] = key_expand_func, */ + [CIPH_AES_CBC][KEYL(128)] = aes128_key_sched_dec, +}; + +/** + * Top level array containing pointers to particular key generation + * function sets, covering given operation direction. + * crypto_key_sched_encrypt: keys for encryption + * crypto_key_sched_decrypt: keys for decryption + */ +static const crypto_key_sched_tbl_t * +crypto_key_sched_dir[] = { + &crypto_key_sched_encrypt, + &crypto_key_sched_decrypt, + NULL +}; + +/** + * Extract particular combined mode crypto function from the 3D array. + */ +#define CRYPTO_GET_KEY_SCHED(cop, calg, keyl) \ +({ \ + crypto_key_sched_tbl_t *ks_tbl = crypto_key_sched_dir[(cop)]; \ + \ + ((*ks_tbl)[(calg)][KEYL(keyl)]); \ +}) + +/*----------------------------------------------------------------------------*/ + +/* + *------------------------------------------------------------------------------ + * Session Prepare + *------------------------------------------------------------------------------ + */ + +/** Get xform chain order */ +static enum armv8_crypto_chain_order +armv8_crypto_get_chain_order(const struct rte_crypto_sym_xform *xform) +{ + + /* + * This driver currently covers only chained operations. + * Ignore only cipher or only authentication operations + * or chains longer than 2 xform structures. + */ + if (xform->next == NULL || xform->next->next != NULL) + return ARMV8_CRYPTO_CHAIN_NOT_SUPPORTED; + + if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) { + if (xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) + return ARMV8_CRYPTO_CHAIN_AUTH_CIPHER; + } + + if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { + if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) + return ARMV8_CRYPTO_CHAIN_CIPHER_AUTH; + } + + return ARMV8_CRYPTO_CHAIN_NOT_SUPPORTED; +} + +static inline void +auth_hmac_pad_prepare(struct armv8_crypto_session *sess, + const struct rte_crypto_sym_xform *xform) +{ + size_t i; + + /* Generate i_key_pad and o_key_pad */ + memset(sess->auth.hmac.i_key_pad, 0, sizeof(sess->auth.hmac.i_key_pad)); + rte_memcpy(sess->auth.hmac.i_key_pad, sess->auth.hmac.key, + xform->auth.key.length); + memset(sess->auth.hmac.o_key_pad, 0, sizeof(sess->auth.hmac.o_key_pad)); + rte_memcpy(sess->auth.hmac.o_key_pad, sess->auth.hmac.key, + xform->auth.key.length); + /* + * XOR key with IPAD/OPAD values to obtain i_key_pad + * and o_key_pad. + * Byte-by-byte operation may seem to be the less efficient + * here but in fact it's the opposite. + * The result ASM code is likely operate on NEON registers + * (load auth key to Qx, load IPAD/OPAD to multiple + * elements of Qy, eor 128 bits at once). + */ + for (i = 0; i < SHA_BLOCK_MAX; i++) { + sess->auth.hmac.i_key_pad[i] ^= HMAC_IPAD_VALUE; + sess->auth.hmac.o_key_pad[i] ^= HMAC_OPAD_VALUE; + } +} + +static inline int +auth_set_prerequisites(struct armv8_crypto_session *sess, + const struct rte_crypto_sym_xform *xform) +{ + uint8_t partial[64] = { 0 }; + int error; + + switch (xform->auth.algo) { + case RTE_CRYPTO_AUTH_SHA1_HMAC: + /* + * Generate authentication key, i_key_pad and o_key_pad. + */ + /* Zero memory under key */ + memset(sess->auth.hmac.key, 0, SHA1_BLOCK_SIZE); + + /* + * Now copy the given authentication key to the session + * key. + */ + rte_memcpy(sess->auth.hmac.key, xform->auth.key.data, + xform->auth.key.length); + + /* Prepare HMAC padding: key|pattern */ + auth_hmac_pad_prepare(sess, xform); + /* + * Calculate partial hash values for i_key_pad and o_key_pad. + * Will be used as initialization state for final HMAC. + */ + error = sha1_block_partial(NULL, sess->auth.hmac.i_key_pad, + partial, SHA1_BLOCK_SIZE); + if (error != 0) + return -1; + memcpy(sess->auth.hmac.i_key_pad, partial, SHA1_BLOCK_SIZE); + + error = sha1_block_partial(NULL, sess->auth.hmac.o_key_pad, + partial, SHA1_BLOCK_SIZE); + if (error != 0) + return -1; + memcpy(sess->auth.hmac.o_key_pad, partial, SHA1_BLOCK_SIZE); + + break; + case RTE_CRYPTO_AUTH_SHA256_HMAC: + /* + * Generate authentication key, i_key_pad and o_key_pad. + */ + /* Zero memory under key */ + memset(sess->auth.hmac.key, 0, SHA256_BLOCK_SIZE); + + /* + * Now copy the given authentication key to the session + * key. + */ + rte_memcpy(sess->auth.hmac.key, xform->auth.key.data, + xform->auth.key.length); + + /* Prepare HMAC padding: key|pattern */ + auth_hmac_pad_prepare(sess, xform); + /* + * Calculate partial hash values for i_key_pad and o_key_pad. + * Will be used as initialization state for final HMAC. + */ + error = sha256_block_partial(NULL, sess->auth.hmac.i_key_pad, + partial, SHA256_BLOCK_SIZE); + if (error != 0) + return -1; + memcpy(sess->auth.hmac.i_key_pad, partial, SHA256_BLOCK_SIZE); + + error = sha256_block_partial(NULL, sess->auth.hmac.o_key_pad, + partial, SHA256_BLOCK_SIZE); + if (error != 0) + return -1; + memcpy(sess->auth.hmac.o_key_pad, partial, SHA256_BLOCK_SIZE); + + break; + default: + break; + } + + return 0; +} + +static inline int +cipher_set_prerequisites(struct armv8_crypto_session *sess, + const struct rte_crypto_sym_xform *xform) +{ + crypto_key_sched_t cipher_key_sched; + + cipher_key_sched = sess->cipher.key_sched; + if (likely(cipher_key_sched != NULL)) { + /* Set up cipher session key */ + cipher_key_sched(sess->cipher.key.data, xform->cipher.key.data); + } + + return 0; +} + +static int +armv8_crypto_set_session_chained_parameters(struct armv8_crypto_session *sess, + const struct rte_crypto_sym_xform *cipher_xform, + const struct rte_crypto_sym_xform *auth_xform) +{ + enum armv8_crypto_chain_order order; + enum armv8_crypto_cipher_operation cop; + enum rte_crypto_cipher_algorithm calg; + enum rte_crypto_auth_algorithm aalg; + + /* Validate and prepare scratch order of combined operations */ + switch (sess->chain_order) { + case ARMV8_CRYPTO_CHAIN_CIPHER_AUTH: + case ARMV8_CRYPTO_CHAIN_AUTH_CIPHER: + order = sess->chain_order; + break; + default: + return -ENOTSUP; + } + /* Select cipher direction */ + sess->cipher.direction = cipher_xform->cipher.op; + /* Select cipher key */ + sess->cipher.key.length = cipher_xform->cipher.key.length; + /* Set cipher direction */ + cop = sess->cipher.direction; + /* Set cipher algorithm */ + calg = cipher_xform->cipher.algo; + + /* Select cipher algo */ + switch (calg) { + /* Cover supported cipher algorithms */ + case RTE_CRYPTO_CIPHER_AES_CBC: + sess->cipher.algo = calg; + /* IV len is always 16 bytes (block size) for AES CBC */ + sess->cipher.iv.length = 16; + break; + default: + return -ENOTSUP; + } + /* Select auth generate/verify */ + sess->auth.operation = auth_xform->auth.op; + + /* Select auth algo */ + switch (auth_xform->auth.algo) { + /* Cover supported hash algorithms */ + case RTE_CRYPTO_AUTH_SHA1_HMAC: + case RTE_CRYPTO_AUTH_SHA256_HMAC: /* Fall through */ + aalg = auth_xform->auth.algo; + sess->auth.mode = ARMV8_CRYPTO_AUTH_AS_HMAC; + break; + default: + return -ENOTSUP; + } + + /* Set the digest length */ + sess->auth.digest_length = auth_xform->auth.digest_length; + + /* Verify supported key lengths and extract proper algorithm */ + switch (cipher_xform->cipher.key.length << 3) { + case 128: + sess->crypto_func = + CRYPTO_GET_ALGO(order, cop, calg, aalg, 128); + sess->cipher.key_sched = + CRYPTO_GET_KEY_SCHED(cop, calg, 128); + break; + case 192: + case 256: + /* These key lengths are not supported yet */ + default: /* Fall through */ + sess->crypto_func = NULL; + sess->cipher.key_sched = NULL; + return -ENOTSUP; + } + + if (unlikely(sess->crypto_func == NULL)) { + /* + * If we got here that means that there must be a bug + * in the algorithms selection above. Nevertheless keep + * it here to catch bug immediately and avoid NULL pointer + * dereference in OPs processing. + */ + ARMV8_CRYPTO_LOG_ERR( + "No appropriate crypto function for given parameters"); + return -EINVAL; + } + + /* Set up cipher session prerequisites */ + if (cipher_set_prerequisites(sess, cipher_xform) != 0) + return -EINVAL; + + /* Set up authentication session prerequisites */ + if (auth_set_prerequisites(sess, auth_xform) != 0) + return -EINVAL; + + return 0; +} + +/** Parse crypto xform chain and set private session parameters */ +int +armv8_crypto_set_session_parameters(struct armv8_crypto_session *sess, + const struct rte_crypto_sym_xform *xform) +{ + const struct rte_crypto_sym_xform *cipher_xform = NULL; + const struct rte_crypto_sym_xform *auth_xform = NULL; + bool is_chained_op; + int ret; + + /* Filter out spurious/broken requests */ + if (xform == NULL) + return -EINVAL; + + sess->chain_order = armv8_crypto_get_chain_order(xform); + switch (sess->chain_order) { + case ARMV8_CRYPTO_CHAIN_CIPHER_AUTH: + cipher_xform = xform; + auth_xform = xform->next; + is_chained_op = true; + break; + case ARMV8_CRYPTO_CHAIN_AUTH_CIPHER: + auth_xform = xform; + cipher_xform = xform->next; + is_chained_op = true; + break; + default: + is_chained_op = false; + return -ENOTSUP; + } + + /* Set IV offset */ + sess->cipher.iv.offset = cipher_xform->cipher.iv.offset; + + if (is_chained_op) { + ret = armv8_crypto_set_session_chained_parameters(sess, + cipher_xform, auth_xform); + if (unlikely(ret != 0)) { + ARMV8_CRYPTO_LOG_ERR( + "Invalid/unsupported chained (cipher/auth) parameters"); + return ret; + } + } else { + ARMV8_CRYPTO_LOG_ERR("Invalid/unsupported operation"); + return -ENOTSUP; + } + + return 0; +} + +/** Provide session for operation */ +static inline struct armv8_crypto_session * +get_session(struct armv8_crypto_qp *qp, struct rte_crypto_op *op) +{ + struct armv8_crypto_session *sess = NULL; + + if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { + /* get existing session */ + if (likely(op->sym->session != NULL)) { + sess = (struct armv8_crypto_session *) + get_session_private_data( + op->sym->session, + cryptodev_driver_id); + } + } else { + /* provide internal session */ + void *_sess = NULL; + void *_sess_private_data = NULL; + + if (rte_mempool_get(qp->sess_mp, (void **)&_sess)) + return NULL; + + if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data)) + return NULL; + + sess = (struct armv8_crypto_session *)_sess_private_data; + + if (unlikely(armv8_crypto_set_session_parameters(sess, + op->sym->xform) != 0)) { + rte_mempool_put(qp->sess_mp, _sess); + rte_mempool_put(qp->sess_mp, _sess_private_data); + sess = NULL; + } + op->sym->session = (struct rte_cryptodev_sym_session *)_sess; + set_session_private_data(op->sym->session, cryptodev_driver_id, + _sess_private_data); + } + + if (unlikely(sess == NULL)) + op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; + + return sess; +} + +/* + *------------------------------------------------------------------------------ + * Process Operations + *------------------------------------------------------------------------------ + */ + +/*----------------------------------------------------------------------------*/ + +/** Process cipher operation */ +static inline void +process_armv8_chained_op(struct armv8_crypto_qp *qp, struct rte_crypto_op *op, + struct armv8_crypto_session *sess, + struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) +{ + crypto_func_t crypto_func; + crypto_arg_t arg; + struct rte_mbuf *m_asrc, *m_adst; + uint8_t *csrc, *cdst; + uint8_t *adst, *asrc; + uint64_t clen, alen; + int error; + + clen = op->sym->cipher.data.length; + alen = op->sym->auth.data.length; + + csrc = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, + op->sym->cipher.data.offset); + cdst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, + op->sym->cipher.data.offset); + + switch (sess->chain_order) { + case ARMV8_CRYPTO_CHAIN_CIPHER_AUTH: + m_asrc = m_adst = mbuf_dst; + break; + case ARMV8_CRYPTO_CHAIN_AUTH_CIPHER: + m_asrc = mbuf_src; + m_adst = mbuf_dst; + break; + default: + op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; + return; + } + asrc = rte_pktmbuf_mtod_offset(m_asrc, uint8_t *, + op->sym->auth.data.offset); + + switch (sess->auth.mode) { + case ARMV8_CRYPTO_AUTH_AS_AUTH: + /* Nothing to do here, just verify correct option */ + break; + case ARMV8_CRYPTO_AUTH_AS_HMAC: + arg.digest.hmac.key = sess->auth.hmac.key; + arg.digest.hmac.i_key_pad = sess->auth.hmac.i_key_pad; + arg.digest.hmac.o_key_pad = sess->auth.hmac.o_key_pad; + break; + default: + op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; + return; + } + + if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_GENERATE) { + adst = op->sym->auth.digest.data; + if (adst == NULL) { + adst = rte_pktmbuf_mtod_offset(m_adst, + uint8_t *, + op->sym->auth.data.offset + + op->sym->auth.data.length); + } + } else { + adst = qp->temp_digest; + } + + arg.cipher.iv = rte_crypto_op_ctod_offset(op, uint8_t *, + sess->cipher.iv.offset); + arg.cipher.key = sess->cipher.key.data; + /* Acquire combined mode function */ + crypto_func = sess->crypto_func; + ARMV8_CRYPTO_ASSERT(crypto_func != NULL); + error = crypto_func(csrc, cdst, clen, asrc, adst, alen, &arg); + if (error != 0) { + op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; + return; + } + + op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; + if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) { + if (memcmp(adst, op->sym->auth.digest.data, + sess->auth.digest_length) != 0) { + op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; + } + } +} + +/** Process crypto operation for mbuf */ +static inline int +process_op(struct armv8_crypto_qp *qp, struct rte_crypto_op *op, + struct armv8_crypto_session *sess) +{ + struct rte_mbuf *msrc, *mdst; + + msrc = op->sym->m_src; + mdst = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src; + + op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; + + switch (sess->chain_order) { + case ARMV8_CRYPTO_CHAIN_CIPHER_AUTH: + case ARMV8_CRYPTO_CHAIN_AUTH_CIPHER: /* Fall through */ + process_armv8_chained_op(qp, op, sess, msrc, mdst); + break; + default: + op->status = RTE_CRYPTO_OP_STATUS_ERROR; + break; + } + + /* Free session if a session-less crypto op */ + if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { + memset(sess, 0, sizeof(struct armv8_crypto_session)); + memset(op->sym->session, 0, + rte_cryptodev_get_header_session_size()); + rte_mempool_put(qp->sess_mp, sess); + rte_mempool_put(qp->sess_mp, op->sym->session); + op->sym->session = NULL; + } + + if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) + op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; + + if (unlikely(op->status == RTE_CRYPTO_OP_STATUS_ERROR)) + return -1; + + return 0; +} + +/* + *------------------------------------------------------------------------------ + * PMD Framework + *------------------------------------------------------------------------------ + */ + +/** Enqueue burst */ +static uint16_t +armv8_crypto_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops, + uint16_t nb_ops) +{ + struct armv8_crypto_session *sess; + struct armv8_crypto_qp *qp = queue_pair; + int i, retval; + + for (i = 0; i < nb_ops; i++) { + sess = get_session(qp, ops[i]); + if (unlikely(sess == NULL)) + goto enqueue_err; + + retval = process_op(qp, ops[i], sess); + if (unlikely(retval < 0)) + goto enqueue_err; + } + + retval = rte_ring_enqueue_burst(qp->processed_ops, (void *)ops, i, + NULL); + qp->stats.enqueued_count += retval; + + return retval; + +enqueue_err: + retval = rte_ring_enqueue_burst(qp->processed_ops, (void *)ops, i, + NULL); + if (ops[i] != NULL) + ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; + + qp->stats.enqueue_err_count++; + return retval; +} + +/** Dequeue burst */ +static uint16_t +armv8_crypto_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops, + uint16_t nb_ops) +{ + struct armv8_crypto_qp *qp = queue_pair; + + unsigned int nb_dequeued = 0; + + nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops, + (void **)ops, nb_ops, NULL); + qp->stats.dequeued_count += nb_dequeued; + + return nb_dequeued; +} + +/** Create ARMv8 crypto device */ +static int +cryptodev_armv8_crypto_create(const char *name, + struct rte_vdev_device *vdev, + struct rte_cryptodev_pmd_init_params *init_params) +{ + struct rte_cryptodev *dev; + struct armv8_crypto_private *internals; + + /* Check CPU for support for AES instruction set */ + if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_AES)) { + ARMV8_CRYPTO_LOG_ERR( + "AES instructions not supported by CPU"); + return -EFAULT; + } + + /* Check CPU for support for SHA instruction set */ + if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_SHA1) || + !rte_cpu_get_flag_enabled(RTE_CPUFLAG_SHA2)) { + ARMV8_CRYPTO_LOG_ERR( + "SHA1/SHA2 instructions not supported by CPU"); + return -EFAULT; + } + + /* Check CPU for support for Advance SIMD instruction set */ + if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_NEON)) { + ARMV8_CRYPTO_LOG_ERR( + "Advanced SIMD instructions not supported by CPU"); + return -EFAULT; + } + + dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params); + if (dev == NULL) { + ARMV8_CRYPTO_LOG_ERR("failed to create cryptodev vdev"); + goto init_error; + } + + dev->driver_id = cryptodev_driver_id; + dev->dev_ops = rte_armv8_crypto_pmd_ops; + + /* register rx/tx burst functions for data path */ + dev->dequeue_burst = armv8_crypto_pmd_dequeue_burst; + dev->enqueue_burst = armv8_crypto_pmd_enqueue_burst; + + dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | + RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | + RTE_CRYPTODEV_FF_CPU_NEON | + RTE_CRYPTODEV_FF_CPU_ARM_CE; + + /* Set vector instructions mode supported */ + internals = dev->data->dev_private; + + internals->max_nb_qpairs = init_params->max_nb_queue_pairs; + internals->max_nb_sessions = init_params->max_nb_sessions; + + return 0; + +init_error: + ARMV8_CRYPTO_LOG_ERR( + "driver %s: cryptodev_armv8_crypto_create failed", + init_params->name); + + cryptodev_armv8_crypto_uninit(vdev); + return -EFAULT; +} + +/** Initialise ARMv8 crypto device */ +static int +cryptodev_armv8_crypto_init(struct rte_vdev_device *vdev) +{ + struct rte_cryptodev_pmd_init_params init_params = { + "", + sizeof(struct armv8_crypto_private), + rte_socket_id(), + RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS, + RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_SESSIONS + }; + const char *name; + const char *input_args; + + name = rte_vdev_device_name(vdev); + if (name == NULL) + return -EINVAL; + input_args = rte_vdev_device_args(vdev); + rte_cryptodev_pmd_parse_input_args(&init_params, input_args); + + return cryptodev_armv8_crypto_create(name, vdev, &init_params); +} + +/** Uninitialise ARMv8 crypto device */ +static int +cryptodev_armv8_crypto_uninit(struct rte_vdev_device *vdev) +{ + struct rte_cryptodev *cryptodev; + const char *name; + + name = rte_vdev_device_name(vdev); + if (name == NULL) + return -EINVAL; + + RTE_LOG(INFO, PMD, + "Closing ARMv8 crypto device %s on numa socket %u\n", + name, rte_socket_id()); + + cryptodev = rte_cryptodev_pmd_get_named_dev(name); + if (cryptodev == NULL) + return -ENODEV; + + return rte_cryptodev_pmd_destroy(cryptodev); +} + +static struct rte_vdev_driver armv8_crypto_pmd_drv = { + .probe = cryptodev_armv8_crypto_init, + .remove = cryptodev_armv8_crypto_uninit +}; + +static struct cryptodev_driver armv8_crypto_drv; + +RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_ARMV8_PMD, armv8_crypto_pmd_drv); +RTE_PMD_REGISTER_ALIAS(CRYPTODEV_NAME_ARMV8_PMD, cryptodev_armv8_pmd); +RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_ARMV8_PMD, + "max_nb_queue_pairs= " + "max_nb_sessions= " + "socket_id="); +RTE_PMD_REGISTER_CRYPTO_DRIVER(armv8_crypto_drv, armv8_crypto_pmd_drv, + cryptodev_driver_id); diff --git a/dpdk/drivers/crypto/armv8/rte_armv8_pmd_ops.c b/dpdk/drivers/crypto/armv8/rte_armv8_pmd_ops.c new file mode 100644 index 00000000..63776b26 --- /dev/null +++ b/dpdk/drivers/crypto/armv8/rte_armv8_pmd_ops.c @@ -0,0 +1,393 @@ +/* + * BSD LICENSE + * + * Copyright (C) Cavium, Inc. 2017. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +#include +#include +#include + +#include "armv8_crypto_defs.h" + +#include "rte_armv8_pmd_private.h" + +static const struct rte_cryptodev_capabilities + armv8_crypto_pmd_capabilities[] = { + { /* SHA1 HMAC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA1_HMAC, + .block_size = 64, + .key_size = { + .min = 1, + .max = 64, + .increment = 1 + }, + .digest_size = { + .min = 20, + .max = 20, + .increment = 0 + }, + .iv_size = { 0 } + }, } + }, } + }, + { /* SHA256 HMAC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA256_HMAC, + .block_size = 64, + .key_size = { + .min = 1, + .max = 64, + .increment = 1 + }, + .digest_size = { + .min = 32, + .max = 32, + .increment = 0 + }, + .iv_size = { 0 } + }, } + }, } + }, + { /* AES CBC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, + {.cipher = { + .algo = RTE_CRYPTO_CIPHER_AES_CBC, + .block_size = 16, + .key_size = { + .min = 16, + .max = 16, + .increment = 0 + }, + .iv_size = { + .min = 16, + .max = 16, + .increment = 0 + } + }, } + }, } + }, + + RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST() +}; + + +/** Configure device */ +static int +armv8_crypto_pmd_config(__rte_unused struct rte_cryptodev *dev, + __rte_unused struct rte_cryptodev_config *config) +{ + return 0; +} + +/** Start device */ +static int +armv8_crypto_pmd_start(__rte_unused struct rte_cryptodev *dev) +{ + return 0; +} + +/** Stop device */ +static void +armv8_crypto_pmd_stop(__rte_unused struct rte_cryptodev *dev) +{ +} + +/** Close device */ +static int +armv8_crypto_pmd_close(__rte_unused struct rte_cryptodev *dev) +{ + return 0; +} + + +/** Get device statistics */ +static void +armv8_crypto_pmd_stats_get(struct rte_cryptodev *dev, + struct rte_cryptodev_stats *stats) +{ + int qp_id; + + for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) { + struct armv8_crypto_qp *qp = dev->data->queue_pairs[qp_id]; + + stats->enqueued_count += qp->stats.enqueued_count; + stats->dequeued_count += qp->stats.dequeued_count; + + stats->enqueue_err_count += qp->stats.enqueue_err_count; + stats->dequeue_err_count += qp->stats.dequeue_err_count; + } +} + +/** Reset device statistics */ +static void +armv8_crypto_pmd_stats_reset(struct rte_cryptodev *dev) +{ + int qp_id; + + for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) { + struct armv8_crypto_qp *qp = dev->data->queue_pairs[qp_id]; + + memset(&qp->stats, 0, sizeof(qp->stats)); + } +} + + +/** Get device info */ +static void +armv8_crypto_pmd_info_get(struct rte_cryptodev *dev, + struct rte_cryptodev_info *dev_info) +{ + struct armv8_crypto_private *internals = dev->data->dev_private; + + if (dev_info != NULL) { + dev_info->driver_id = dev->driver_id; + dev_info->feature_flags = dev->feature_flags; + dev_info->capabilities = armv8_crypto_pmd_capabilities; + dev_info->max_nb_queue_pairs = internals->max_nb_qpairs; + dev_info->sym.max_nb_sessions = internals->max_nb_sessions; + } +} + +/** Release queue pair */ +static int +armv8_crypto_pmd_qp_release(struct rte_cryptodev *dev, uint16_t qp_id) +{ + + if (dev->data->queue_pairs[qp_id] != NULL) { + rte_free(dev->data->queue_pairs[qp_id]); + dev->data->queue_pairs[qp_id] = NULL; + } + + return 0; +} + +/** set a unique name for the queue pair based on it's name, dev_id and qp_id */ +static int +armv8_crypto_pmd_qp_set_unique_name(struct rte_cryptodev *dev, + struct armv8_crypto_qp *qp) +{ + unsigned int n; + + n = snprintf(qp->name, sizeof(qp->name), "armv8_crypto_pmd_%u_qp_%u", + dev->data->dev_id, qp->id); + + if (n >= sizeof(qp->name)) + return -1; + + return 0; +} + + +/** Create a ring to place processed operations on */ +static struct rte_ring * +armv8_crypto_pmd_qp_create_processed_ops_ring(struct armv8_crypto_qp *qp, + unsigned int ring_size, int socket_id) +{ + struct rte_ring *r; + + r = rte_ring_lookup(qp->name); + if (r) { + if (rte_ring_get_size(r) >= ring_size) { + ARMV8_CRYPTO_LOG_INFO( + "Reusing existing ring %s for processed ops", + qp->name); + return r; + } + + ARMV8_CRYPTO_LOG_ERR( + "Unable to reuse existing ring %s for processed ops", + qp->name); + return NULL; + } + + return rte_ring_create(qp->name, ring_size, socket_id, + RING_F_SP_ENQ | RING_F_SC_DEQ); +} + + +/** Setup a queue pair */ +static int +armv8_crypto_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, + const struct rte_cryptodev_qp_conf *qp_conf, + int socket_id, struct rte_mempool *session_pool) +{ + struct armv8_crypto_qp *qp = NULL; + + /* Free memory prior to re-allocation if needed. */ + if (dev->data->queue_pairs[qp_id] != NULL) + armv8_crypto_pmd_qp_release(dev, qp_id); + + /* Allocate the queue pair data structure. */ + qp = rte_zmalloc_socket("ARMv8 PMD Queue Pair", sizeof(*qp), + RTE_CACHE_LINE_SIZE, socket_id); + if (qp == NULL) + return -ENOMEM; + + qp->id = qp_id; + dev->data->queue_pairs[qp_id] = qp; + + if (armv8_crypto_pmd_qp_set_unique_name(dev, qp) != 0) + goto qp_setup_cleanup; + + qp->processed_ops = armv8_crypto_pmd_qp_create_processed_ops_ring(qp, + qp_conf->nb_descriptors, socket_id); + if (qp->processed_ops == NULL) + goto qp_setup_cleanup; + + qp->sess_mp = session_pool; + + memset(&qp->stats, 0, sizeof(qp->stats)); + + return 0; + +qp_setup_cleanup: + if (qp) + rte_free(qp); + + return -1; +} + +/** Start queue pair */ +static int +armv8_crypto_pmd_qp_start(__rte_unused struct rte_cryptodev *dev, + __rte_unused uint16_t queue_pair_id) +{ + return -ENOTSUP; +} + +/** Stop queue pair */ +static int +armv8_crypto_pmd_qp_stop(__rte_unused struct rte_cryptodev *dev, + __rte_unused uint16_t queue_pair_id) +{ + return -ENOTSUP; +} + +/** Return the number of allocated queue pairs */ +static uint32_t +armv8_crypto_pmd_qp_count(struct rte_cryptodev *dev) +{ + return dev->data->nb_queue_pairs; +} + +/** Returns the size of the session structure */ +static unsigned +armv8_crypto_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused) +{ + return sizeof(struct armv8_crypto_session); +} + +/** Configure the session from a crypto xform chain */ +static int +armv8_crypto_pmd_session_configure(struct rte_cryptodev *dev, + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool) +{ + void *sess_private_data; + int ret; + + if (unlikely(sess == NULL)) { + ARMV8_CRYPTO_LOG_ERR("invalid session struct"); + return -EINVAL; + } + + if (rte_mempool_get(mempool, &sess_private_data)) { + CDEV_LOG_ERR( + "Couldn't get object from session mempool"); + return -ENOMEM; + } + + ret = armv8_crypto_set_session_parameters(sess_private_data, xform); + if (ret != 0) { + ARMV8_CRYPTO_LOG_ERR("failed configure session parameters"); + + /* Return session to mempool */ + rte_mempool_put(mempool, sess_private_data); + return ret; + } + + set_session_private_data(sess, dev->driver_id, + sess_private_data); + + return 0; +} + +/** Clear the memory of session so it doesn't leave key material behind */ +static void +armv8_crypto_pmd_session_clear(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) +{ + uint8_t index = dev->driver_id; + void *sess_priv = get_session_private_data(sess, index); + + /* Zero out the whole structure */ + if (sess_priv) { + memset(sess_priv, 0, sizeof(struct armv8_crypto_session)); + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + set_session_private_data(sess, index, NULL); + rte_mempool_put(sess_mp, sess_priv); + } +} + +struct rte_cryptodev_ops armv8_crypto_pmd_ops = { + .dev_configure = armv8_crypto_pmd_config, + .dev_start = armv8_crypto_pmd_start, + .dev_stop = armv8_crypto_pmd_stop, + .dev_close = armv8_crypto_pmd_close, + + .stats_get = armv8_crypto_pmd_stats_get, + .stats_reset = armv8_crypto_pmd_stats_reset, + + .dev_infos_get = armv8_crypto_pmd_info_get, + + .queue_pair_setup = armv8_crypto_pmd_qp_setup, + .queue_pair_release = armv8_crypto_pmd_qp_release, + .queue_pair_start = armv8_crypto_pmd_qp_start, + .queue_pair_stop = armv8_crypto_pmd_qp_stop, + .queue_pair_count = armv8_crypto_pmd_qp_count, + + .session_get_size = armv8_crypto_pmd_session_get_size, + .session_configure = armv8_crypto_pmd_session_configure, + .session_clear = armv8_crypto_pmd_session_clear +}; + +struct rte_cryptodev_ops *rte_armv8_crypto_pmd_ops = &armv8_crypto_pmd_ops; diff --git a/dpdk/drivers/crypto/armv8/rte_armv8_pmd_private.h b/dpdk/drivers/crypto/armv8/rte_armv8_pmd_private.h new file mode 100644 index 00000000..fa31f0a0 --- /dev/null +++ b/dpdk/drivers/crypto/armv8/rte_armv8_pmd_private.h @@ -0,0 +1,227 @@ +/* + * BSD LICENSE + * + * Copyright (C) Cavium, Inc. 2017. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_ARMV8_PMD_PRIVATE_H_ +#define _RTE_ARMV8_PMD_PRIVATE_H_ + +#define CRYPTODEV_NAME_ARMV8_PMD crypto_armv8 +/**< ARMv8 Crypto PMD device name */ + +#define ARMV8_CRYPTO_LOG_ERR(fmt, args...) \ + RTE_LOG(ERR, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \ + RTE_STR(CRYPTODEV_NAME_ARMV8_CRYPTO_PMD), \ + __func__, __LINE__, ## args) + +#ifdef RTE_LIBRTE_ARMV8_CRYPTO_DEBUG +#define ARMV8_CRYPTO_LOG_INFO(fmt, args...) \ + RTE_LOG(INFO, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \ + RTE_STR(CRYPTODEV_NAME_ARMV8_CRYPTO_PMD), \ + __func__, __LINE__, ## args) + +#define ARMV8_CRYPTO_LOG_DBG(fmt, args...) \ + RTE_LOG(DEBUG, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \ + RTE_STR(CRYPTODEV_NAME_ARMV8_CRYPTO_PMD), \ + __func__, __LINE__, ## args) + +#define ARMV8_CRYPTO_ASSERT(con) \ +do { \ + if (!(con)) { \ + rte_panic("%s(): " \ + con "condition failed, line %u", __func__); \ + } \ +} while (0) + +#else +#define ARMV8_CRYPTO_LOG_INFO(fmt, args...) +#define ARMV8_CRYPTO_LOG_DBG(fmt, args...) +#define ARMV8_CRYPTO_ASSERT(con) +#endif + +#define NBBY 8 /* Number of bits in a byte */ +#define BYTE_LENGTH(x) ((x) / NBBY) /* Number of bytes in x (round down) */ + +/* Maximum length for digest (SHA-256 needs 32 bytes) */ +#define DIGEST_LENGTH_MAX 32 + +/** ARMv8 operation order mode enumerator */ +enum armv8_crypto_chain_order { + ARMV8_CRYPTO_CHAIN_CIPHER_AUTH, + ARMV8_CRYPTO_CHAIN_AUTH_CIPHER, + ARMV8_CRYPTO_CHAIN_NOT_SUPPORTED, + ARMV8_CRYPTO_CHAIN_LIST_END = ARMV8_CRYPTO_CHAIN_NOT_SUPPORTED +}; + +/** ARMv8 cipher operation enumerator */ +enum armv8_crypto_cipher_operation { + ARMV8_CRYPTO_CIPHER_OP_ENCRYPT = RTE_CRYPTO_CIPHER_OP_ENCRYPT, + ARMV8_CRYPTO_CIPHER_OP_DECRYPT = RTE_CRYPTO_CIPHER_OP_DECRYPT, + ARMV8_CRYPTO_CIPHER_OP_NOT_SUPPORTED, + ARMV8_CRYPTO_CIPHER_OP_LIST_END = ARMV8_CRYPTO_CIPHER_OP_NOT_SUPPORTED +}; + +enum armv8_crypto_cipher_keylen { + ARMV8_CRYPTO_CIPHER_KEYLEN_128, + ARMV8_CRYPTO_CIPHER_KEYLEN_192, + ARMV8_CRYPTO_CIPHER_KEYLEN_256, + ARMV8_CRYPTO_CIPHER_KEYLEN_NOT_SUPPORTED, + ARMV8_CRYPTO_CIPHER_KEYLEN_LIST_END = + ARMV8_CRYPTO_CIPHER_KEYLEN_NOT_SUPPORTED +}; + +/** ARMv8 auth mode enumerator */ +enum armv8_crypto_auth_mode { + ARMV8_CRYPTO_AUTH_AS_AUTH, + ARMV8_CRYPTO_AUTH_AS_HMAC, + ARMV8_CRYPTO_AUTH_AS_CIPHER, + ARMV8_CRYPTO_AUTH_NOT_SUPPORTED, + ARMV8_CRYPTO_AUTH_LIST_END = ARMV8_CRYPTO_AUTH_NOT_SUPPORTED +}; + +#define CRYPTO_ORDER_MAX ARMV8_CRYPTO_CHAIN_LIST_END +#define CRYPTO_CIPHER_OP_MAX ARMV8_CRYPTO_CIPHER_OP_LIST_END +#define CRYPTO_CIPHER_KEYLEN_MAX ARMV8_CRYPTO_CIPHER_KEYLEN_LIST_END +#define CRYPTO_CIPHER_MAX RTE_CRYPTO_CIPHER_LIST_END +#define CRYPTO_AUTH_MAX RTE_CRYPTO_AUTH_LIST_END + +#define HMAC_IPAD_VALUE (0x36) +#define HMAC_OPAD_VALUE (0x5C) + +#define SHA256_AUTH_KEY_LENGTH (BYTE_LENGTH(256)) +#define SHA256_BLOCK_SIZE (BYTE_LENGTH(512)) + +#define SHA1_AUTH_KEY_LENGTH (BYTE_LENGTH(160)) +#define SHA1_BLOCK_SIZE (BYTE_LENGTH(512)) + +#define SHA_AUTH_KEY_MAX SHA256_AUTH_KEY_LENGTH +#define SHA_BLOCK_MAX SHA256_BLOCK_SIZE + +typedef int (*crypto_func_t)(uint8_t *, uint8_t *, uint64_t, + uint8_t *, uint8_t *, uint64_t, + crypto_arg_t *); + +typedef void (*crypto_key_sched_t)(uint8_t *, const uint8_t *); + +/** private data structure for each ARMv8 crypto device */ +struct armv8_crypto_private { + unsigned int max_nb_qpairs; + /**< Max number of queue pairs */ + unsigned int max_nb_sessions; + /**< Max number of sessions */ +}; + +/** ARMv8 crypto queue pair */ +struct armv8_crypto_qp { + uint16_t id; + /**< Queue Pair Identifier */ + struct rte_ring *processed_ops; + /**< Ring for placing process packets */ + struct rte_mempool *sess_mp; + /**< Session Mempool */ + struct rte_cryptodev_stats stats; + /**< Queue pair statistics */ + char name[RTE_CRYPTODEV_NAME_LEN]; + /**< Unique Queue Pair Name */ + uint8_t temp_digest[DIGEST_LENGTH_MAX]; + /**< Buffer used to store the digest generated + * by the driver when verifying a digest provided + * by the user (using authentication verify operation) + */ +} __rte_cache_aligned; + +/** ARMv8 crypto private session structure */ +struct armv8_crypto_session { + enum armv8_crypto_chain_order chain_order; + /**< chain order mode */ + crypto_func_t crypto_func; + /**< cryptographic function to use for this session */ + + /** Cipher Parameters */ + struct { + enum rte_crypto_cipher_operation direction; + /**< cipher operation direction */ + enum rte_crypto_cipher_algorithm algo; + /**< cipher algorithm */ + struct { + uint16_t length; + uint16_t offset; + } iv; + /**< IV parameters */ + + struct { + uint8_t data[256]; + /**< key data */ + size_t length; + /**< key length in bytes */ + } key; + + crypto_key_sched_t key_sched; + /**< Key schedule function */ + } cipher; + + /** Authentication Parameters */ + struct { + enum rte_crypto_auth_operation operation; + /**< auth operation generate or verify */ + enum armv8_crypto_auth_mode mode; + /**< auth operation mode */ + + union { + struct { + /* Add data if needed */ + } auth; + + struct { + uint8_t i_key_pad[SHA_BLOCK_MAX] + __rte_cache_aligned; + /**< inner pad (max supported block length) */ + uint8_t o_key_pad[SHA_BLOCK_MAX] + __rte_cache_aligned; + /**< outer pad (max supported block length) */ + uint8_t key[SHA_BLOCK_MAX]; + /**< HMAC key (max supported block length)*/ + } hmac; + }; + uint16_t digest_length; + /* Digest length */ + } auth; + +} __rte_cache_aligned; + +/** Set and validate ARMv8 crypto session parameters */ +extern int armv8_crypto_set_session_parameters( + struct armv8_crypto_session *sess, + const struct rte_crypto_sym_xform *xform); + +/** device specific operations function pointer structure */ +extern struct rte_cryptodev_ops *rte_armv8_crypto_pmd_ops; + +#endif /* _RTE_ARMV8_PMD_PRIVATE_H_ */ diff --git a/dpdk/drivers/crypto/armv8/rte_pmd_armv8_version.map b/dpdk/drivers/crypto/armv8/rte_pmd_armv8_version.map new file mode 100644 index 00000000..1f84b68a --- /dev/null +++ b/dpdk/drivers/crypto/armv8/rte_pmd_armv8_version.map @@ -0,0 +1,3 @@ +DPDK_17.02 { + local: *; +}; diff --git a/dpdk/drivers/crypto/dpaa2_sec/Makefile b/dpdk/drivers/crypto/dpaa2_sec/Makefile new file mode 100644 index 00000000..a08f2717 --- /dev/null +++ b/dpdk/drivers/crypto/dpaa2_sec/Makefile @@ -0,0 +1,85 @@ +# BSD LICENSE +# +# Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. +# Copyright 2016 NXP. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Freescale Semiconductor, Inc nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +include $(RTE_SDK)/mk/rte.vars.mk + +ifneq ($(MAKECMDGOALS),clean) +ifneq ($(CONFIG_RTE_LIBRTE_SECURITY),y) +$(error "RTE_LIBRTE_SECURITY is required to build RTE_LIBRTE_PMD_DPAA2_SEC") +endif +endif + +# +# library name +# +LIB = librte_pmd_dpaa2_sec.a + +# build flags +ifeq ($(CONFIG_RTE_LIBRTE_DPAA2_SEC_DEBUG_INIT),y) +CFLAGS += -O0 -g +CFLAGS += "-Wno-error" +else +CFLAGS += -O3 +CFLAGS += $(WERROR_FLAGS) +endif +CFLAGS += -D _GNU_SOURCE + +ifeq ($(CONFIG_RTE_TOOLCHAIN_GCC),y) +ifeq ($(shell test $(GCC_VERSION) -gt 70 && echo 1), 1) +CFLAGS += -Wno-implicit-fallthrough +endif +endif + +CFLAGS += -I$(RTE_SDK)/drivers/crypto/dpaa2_sec/ +CFLAGS += -I$(RTE_SDK)/drivers/crypto/dpaa2_sec/mc +CFLAGS += -I$(RTE_SDK)/drivers/bus/fslmc/ +CFLAGS += -I$(RTE_SDK)/drivers/bus/fslmc/qbman/include +CFLAGS += -I$(RTE_SDK)/drivers/bus/fslmc/mc +CFLAGS += -I$(RTE_SDK)/drivers/bus/fslmc/portal +CFLAGS += -I$(RTE_SDK)/drivers/mempool/dpaa2/ +CFLAGS += -I$(RTE_SDK)/lib/librte_eal/linuxapp/eal + +# versioning export map +EXPORT_MAP := rte_pmd_dpaa2_sec_version.map + +# library version +LIBABIVER := 1 + +# library source files +SRCS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_SEC) += dpaa2_sec_dpseci.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_SEC) += mc/dpseci.c + +LDLIBS += -lrte_bus_fslmc +LDLIBS += -lrte_mempool_dpaa2 +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_cryptodev + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c b/dpdk/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c new file mode 100644 index 00000000..67fb6e24 --- /dev/null +++ b/dpdk/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c @@ -0,0 +1,2435 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. + * Copyright 2016 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Freescale Semiconductor, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "dpaa2_sec_priv.h" +#include "dpaa2_sec_logs.h" + +/* RTA header files */ +#include +#include + +/* Minimum job descriptor consists of a oneword job descriptor HEADER and + * a pointer to the shared descriptor + */ +#define MIN_JOB_DESC_SIZE (CAAM_CMD_SZ + CAAM_PTR_SZ) +#define FSL_VENDOR_ID 0x1957 +#define FSL_DEVICE_ID 0x410 +#define FSL_SUBSYSTEM_SEC 1 +#define FSL_MC_DPSECI_DEVID 3 + +#define NO_PREFETCH 0 +/* FLE_POOL_NUM_BUFS is set as per the ipsec-secgw application */ +#define FLE_POOL_NUM_BUFS 32000 +#define FLE_POOL_BUF_SIZE 256 +#define FLE_POOL_CACHE_SIZE 512 +#define SEC_FLC_DHR_OUTBOUND -114 +#define SEC_FLC_DHR_INBOUND 0 + +enum rta_sec_era rta_sec_era = RTA_SEC_ERA_8; + +static uint8_t cryptodev_driver_id; + +static inline int +build_proto_fd(dpaa2_sec_session *sess, + struct rte_crypto_op *op, + struct qbman_fd *fd, uint16_t bpid) +{ + struct rte_crypto_sym_op *sym_op = op->sym; + struct ctxt_priv *priv = sess->ctxt; + struct sec_flow_context *flc; + struct rte_mbuf *mbuf = sym_op->m_src; + + if (likely(bpid < MAX_BPID)) + DPAA2_SET_FD_BPID(fd, bpid); + else + DPAA2_SET_FD_IVP(fd); + + /* Save the shared descriptor */ + flc = &priv->flc_desc[0].flc; + + DPAA2_SET_FD_ADDR(fd, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src)); + DPAA2_SET_FD_OFFSET(fd, sym_op->m_src->data_off); + DPAA2_SET_FD_LEN(fd, sym_op->m_src->pkt_len); + DPAA2_SET_FD_FLC(fd, ((uint64_t)flc)); + + /* save physical address of mbuf */ + op->sym->aead.digest.phys_addr = mbuf->buf_iova; + mbuf->buf_iova = (uint64_t)op; + + return 0; +} + +static inline int +build_authenc_gcm_fd(dpaa2_sec_session *sess, + struct rte_crypto_op *op, + struct qbman_fd *fd, uint16_t bpid) +{ + struct rte_crypto_sym_op *sym_op = op->sym; + struct ctxt_priv *priv = sess->ctxt; + struct qbman_fle *fle, *sge; + struct sec_flow_context *flc; + uint32_t auth_only_len = sess->ext_params.aead_ctxt.auth_only_len; + int icv_len = sess->digest_length, retval; + uint8_t *old_icv; + struct rte_mbuf *dst; + uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, + sess->iv.offset); + + PMD_INIT_FUNC_TRACE(); + + if (sym_op->m_dst) + dst = sym_op->m_dst; + else + dst = sym_op->m_src; + + /* TODO we are using the first FLE entry to store Mbuf and session ctxt. + * Currently we donot know which FLE has the mbuf stored. + * So while retreiving we can go back 1 FLE from the FD -ADDR + * to get the MBUF Addr from the previous FLE. + * We can have a better approach to use the inline Mbuf + */ + retval = rte_mempool_get(priv->fle_pool, (void **)(&fle)); + if (retval) { + RTE_LOG(ERR, PMD, "Memory alloc failed for SGE\n"); + return -1; + } + memset(fle, 0, FLE_POOL_BUF_SIZE); + DPAA2_SET_FLE_ADDR(fle, DPAA2_OP_VADDR_TO_IOVA(op)); + DPAA2_FLE_SAVE_CTXT(fle, priv); + fle = fle + 1; + sge = fle + 2; + if (likely(bpid < MAX_BPID)) { + DPAA2_SET_FD_BPID(fd, bpid); + DPAA2_SET_FLE_BPID(fle, bpid); + DPAA2_SET_FLE_BPID(fle + 1, bpid); + DPAA2_SET_FLE_BPID(sge, bpid); + DPAA2_SET_FLE_BPID(sge + 1, bpid); + DPAA2_SET_FLE_BPID(sge + 2, bpid); + DPAA2_SET_FLE_BPID(sge + 3, bpid); + } else { + DPAA2_SET_FD_IVP(fd); + DPAA2_SET_FLE_IVP(fle); + DPAA2_SET_FLE_IVP((fle + 1)); + DPAA2_SET_FLE_IVP(sge); + DPAA2_SET_FLE_IVP((sge + 1)); + DPAA2_SET_FLE_IVP((sge + 2)); + DPAA2_SET_FLE_IVP((sge + 3)); + } + + /* Save the shared descriptor */ + flc = &priv->flc_desc[0].flc; + /* Configure FD as a FRAME LIST */ + DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(fle)); + DPAA2_SET_FD_COMPOUND_FMT(fd); + DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc)); + + PMD_TX_LOG(DEBUG, "auth_off: 0x%x/length %d, digest-len=%d\n" + "iv-len=%d data_off: 0x%x\n", + sym_op->aead.data.offset, + sym_op->aead.data.length, + sym_op->aead.digest.length, + sess->iv.length, + sym_op->m_src->data_off); + + /* Configure Output FLE with Scatter/Gather Entry */ + DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge)); + if (auth_only_len) + DPAA2_SET_FLE_INTERNAL_JD(fle, auth_only_len); + fle->length = (sess->dir == DIR_ENC) ? + (sym_op->aead.data.length + icv_len + auth_only_len) : + sym_op->aead.data.length + auth_only_len; + + DPAA2_SET_FLE_SG_EXT(fle); + + /* Configure Output SGE for Encap/Decap */ + DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(dst)); + DPAA2_SET_FLE_OFFSET(sge, sym_op->aead.data.offset + + dst->data_off - auth_only_len); + sge->length = sym_op->aead.data.length + auth_only_len; + + if (sess->dir == DIR_ENC) { + sge++; + DPAA2_SET_FLE_ADDR(sge, + DPAA2_VADDR_TO_IOVA(sym_op->aead.digest.data)); + sge->length = sess->digest_length; + DPAA2_SET_FD_LEN(fd, (sym_op->aead.data.length + + sess->iv.length + auth_only_len)); + } + DPAA2_SET_FLE_FIN(sge); + + sge++; + fle++; + + /* Configure Input FLE with Scatter/Gather Entry */ + DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge)); + DPAA2_SET_FLE_SG_EXT(fle); + DPAA2_SET_FLE_FIN(fle); + fle->length = (sess->dir == DIR_ENC) ? + (sym_op->aead.data.length + sess->iv.length + auth_only_len) : + (sym_op->aead.data.length + sess->iv.length + auth_only_len + + sess->digest_length); + + /* Configure Input SGE for Encap/Decap */ + DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(IV_ptr)); + sge->length = sess->iv.length; + sge++; + if (auth_only_len) { + DPAA2_SET_FLE_ADDR(sge, + DPAA2_VADDR_TO_IOVA(sym_op->aead.aad.data)); + sge->length = auth_only_len; + DPAA2_SET_FLE_BPID(sge, bpid); + sge++; + } + + DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src)); + DPAA2_SET_FLE_OFFSET(sge, sym_op->aead.data.offset + + sym_op->m_src->data_off); + sge->length = sym_op->aead.data.length; + if (sess->dir == DIR_DEC) { + sge++; + old_icv = (uint8_t *)(sge + 1); + memcpy(old_icv, sym_op->aead.digest.data, + sess->digest_length); + DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_icv)); + sge->length = sess->digest_length; + DPAA2_SET_FD_LEN(fd, (sym_op->aead.data.length + + sess->digest_length + + sess->iv.length + + auth_only_len)); + } + DPAA2_SET_FLE_FIN(sge); + + if (auth_only_len) { + DPAA2_SET_FLE_INTERNAL_JD(fle, auth_only_len); + DPAA2_SET_FD_INTERNAL_JD(fd, auth_only_len); + } + + return 0; +} + +static inline int +build_authenc_fd(dpaa2_sec_session *sess, + struct rte_crypto_op *op, + struct qbman_fd *fd, uint16_t bpid) +{ + struct rte_crypto_sym_op *sym_op = op->sym; + struct ctxt_priv *priv = sess->ctxt; + struct qbman_fle *fle, *sge; + struct sec_flow_context *flc; + uint32_t auth_only_len = sym_op->auth.data.length - + sym_op->cipher.data.length; + int icv_len = sess->digest_length, retval; + uint8_t *old_icv; + uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, + sess->iv.offset); + struct rte_mbuf *dst; + + PMD_INIT_FUNC_TRACE(); + + if (sym_op->m_dst) + dst = sym_op->m_dst; + else + dst = sym_op->m_src; + + /* we are using the first FLE entry to store Mbuf. + * Currently we donot know which FLE has the mbuf stored. + * So while retreiving we can go back 1 FLE from the FD -ADDR + * to get the MBUF Addr from the previous FLE. + * We can have a better approach to use the inline Mbuf + */ + retval = rte_mempool_get(priv->fle_pool, (void **)(&fle)); + if (retval) { + RTE_LOG(ERR, PMD, "Memory alloc failed for SGE\n"); + return -1; + } + memset(fle, 0, FLE_POOL_BUF_SIZE); + DPAA2_SET_FLE_ADDR(fle, DPAA2_OP_VADDR_TO_IOVA(op)); + DPAA2_FLE_SAVE_CTXT(fle, priv); + fle = fle + 1; + sge = fle + 2; + if (likely(bpid < MAX_BPID)) { + DPAA2_SET_FD_BPID(fd, bpid); + DPAA2_SET_FLE_BPID(fle, bpid); + DPAA2_SET_FLE_BPID(fle + 1, bpid); + DPAA2_SET_FLE_BPID(sge, bpid); + DPAA2_SET_FLE_BPID(sge + 1, bpid); + DPAA2_SET_FLE_BPID(sge + 2, bpid); + DPAA2_SET_FLE_BPID(sge + 3, bpid); + } else { + DPAA2_SET_FD_IVP(fd); + DPAA2_SET_FLE_IVP(fle); + DPAA2_SET_FLE_IVP((fle + 1)); + DPAA2_SET_FLE_IVP(sge); + DPAA2_SET_FLE_IVP((sge + 1)); + DPAA2_SET_FLE_IVP((sge + 2)); + DPAA2_SET_FLE_IVP((sge + 3)); + } + + /* Save the shared descriptor */ + flc = &priv->flc_desc[0].flc; + /* Configure FD as a FRAME LIST */ + DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(fle)); + DPAA2_SET_FD_COMPOUND_FMT(fd); + DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc)); + + PMD_TX_LOG(DEBUG, "auth_off: 0x%x/length %d, digest-len=%d\n" + "cipher_off: 0x%x/length %d, iv-len=%d data_off: 0x%x\n", + sym_op->auth.data.offset, + sym_op->auth.data.length, + sess->digest_length, + sym_op->cipher.data.offset, + sym_op->cipher.data.length, + sess->iv.length, + sym_op->m_src->data_off); + + /* Configure Output FLE with Scatter/Gather Entry */ + DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge)); + if (auth_only_len) + DPAA2_SET_FLE_INTERNAL_JD(fle, auth_only_len); + fle->length = (sess->dir == DIR_ENC) ? + (sym_op->cipher.data.length + icv_len) : + sym_op->cipher.data.length; + + DPAA2_SET_FLE_SG_EXT(fle); + + /* Configure Output SGE for Encap/Decap */ + DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(dst)); + DPAA2_SET_FLE_OFFSET(sge, sym_op->cipher.data.offset + + dst->data_off); + sge->length = sym_op->cipher.data.length; + + if (sess->dir == DIR_ENC) { + sge++; + DPAA2_SET_FLE_ADDR(sge, + DPAA2_VADDR_TO_IOVA(sym_op->auth.digest.data)); + sge->length = sess->digest_length; + DPAA2_SET_FD_LEN(fd, (sym_op->auth.data.length + + sess->iv.length)); + } + DPAA2_SET_FLE_FIN(sge); + + sge++; + fle++; + + /* Configure Input FLE with Scatter/Gather Entry */ + DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge)); + DPAA2_SET_FLE_SG_EXT(fle); + DPAA2_SET_FLE_FIN(fle); + fle->length = (sess->dir == DIR_ENC) ? + (sym_op->auth.data.length + sess->iv.length) : + (sym_op->auth.data.length + sess->iv.length + + sess->digest_length); + + /* Configure Input SGE for Encap/Decap */ + DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(iv_ptr)); + sge->length = sess->iv.length; + sge++; + + DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src)); + DPAA2_SET_FLE_OFFSET(sge, sym_op->auth.data.offset + + sym_op->m_src->data_off); + sge->length = sym_op->auth.data.length; + if (sess->dir == DIR_DEC) { + sge++; + old_icv = (uint8_t *)(sge + 1); + memcpy(old_icv, sym_op->auth.digest.data, + sess->digest_length); + DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_icv)); + sge->length = sess->digest_length; + DPAA2_SET_FD_LEN(fd, (sym_op->auth.data.length + + sess->digest_length + + sess->iv.length)); + } + DPAA2_SET_FLE_FIN(sge); + if (auth_only_len) { + DPAA2_SET_FLE_INTERNAL_JD(fle, auth_only_len); + DPAA2_SET_FD_INTERNAL_JD(fd, auth_only_len); + } + return 0; +} + +static inline int +build_auth_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op, + struct qbman_fd *fd, uint16_t bpid) +{ + struct rte_crypto_sym_op *sym_op = op->sym; + struct qbman_fle *fle, *sge; + struct sec_flow_context *flc; + struct ctxt_priv *priv = sess->ctxt; + uint8_t *old_digest; + int retval; + + PMD_INIT_FUNC_TRACE(); + + retval = rte_mempool_get(priv->fle_pool, (void **)(&fle)); + if (retval) { + RTE_LOG(ERR, PMD, "Memory alloc failed for SGE\n"); + return -1; + } + memset(fle, 0, FLE_POOL_BUF_SIZE); + /* TODO we are using the first FLE entry to store Mbuf. + * Currently we donot know which FLE has the mbuf stored. + * So while retreiving we can go back 1 FLE from the FD -ADDR + * to get the MBUF Addr from the previous FLE. + * We can have a better approach to use the inline Mbuf + */ + DPAA2_SET_FLE_ADDR(fle, DPAA2_OP_VADDR_TO_IOVA(op)); + DPAA2_FLE_SAVE_CTXT(fle, priv); + fle = fle + 1; + + if (likely(bpid < MAX_BPID)) { + DPAA2_SET_FD_BPID(fd, bpid); + DPAA2_SET_FLE_BPID(fle, bpid); + DPAA2_SET_FLE_BPID(fle + 1, bpid); + } else { + DPAA2_SET_FD_IVP(fd); + DPAA2_SET_FLE_IVP(fle); + DPAA2_SET_FLE_IVP((fle + 1)); + } + flc = &priv->flc_desc[DESC_INITFINAL].flc; + DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc)); + + DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sym_op->auth.digest.data)); + fle->length = sess->digest_length; + + DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(fle)); + DPAA2_SET_FD_COMPOUND_FMT(fd); + fle++; + + if (sess->dir == DIR_ENC) { + DPAA2_SET_FLE_ADDR(fle, + DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src)); + DPAA2_SET_FLE_OFFSET(fle, sym_op->auth.data.offset + + sym_op->m_src->data_off); + DPAA2_SET_FD_LEN(fd, sym_op->auth.data.length); + fle->length = sym_op->auth.data.length; + } else { + sge = fle + 2; + DPAA2_SET_FLE_SG_EXT(fle); + DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge)); + + if (likely(bpid < MAX_BPID)) { + DPAA2_SET_FLE_BPID(sge, bpid); + DPAA2_SET_FLE_BPID(sge + 1, bpid); + } else { + DPAA2_SET_FLE_IVP(sge); + DPAA2_SET_FLE_IVP((sge + 1)); + } + DPAA2_SET_FLE_ADDR(sge, + DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src)); + DPAA2_SET_FLE_OFFSET(sge, sym_op->auth.data.offset + + sym_op->m_src->data_off); + + DPAA2_SET_FD_LEN(fd, sym_op->auth.data.length + + sess->digest_length); + sge->length = sym_op->auth.data.length; + sge++; + old_digest = (uint8_t *)(sge + 1); + rte_memcpy(old_digest, sym_op->auth.digest.data, + sess->digest_length); + DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_digest)); + sge->length = sess->digest_length; + fle->length = sym_op->auth.data.length + + sess->digest_length; + DPAA2_SET_FLE_FIN(sge); + } + DPAA2_SET_FLE_FIN(fle); + + return 0; +} + +static int +build_cipher_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op, + struct qbman_fd *fd, uint16_t bpid) +{ + struct rte_crypto_sym_op *sym_op = op->sym; + struct qbman_fle *fle, *sge; + int retval; + struct sec_flow_context *flc; + struct ctxt_priv *priv = sess->ctxt; + uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, + sess->iv.offset); + struct rte_mbuf *dst; + + PMD_INIT_FUNC_TRACE(); + + if (sym_op->m_dst) + dst = sym_op->m_dst; + else + dst = sym_op->m_src; + + retval = rte_mempool_get(priv->fle_pool, (void **)(&fle)); + if (retval) { + RTE_LOG(ERR, PMD, "Memory alloc failed for SGE\n"); + return -1; + } + memset(fle, 0, FLE_POOL_BUF_SIZE); + /* TODO we are using the first FLE entry to store Mbuf. + * Currently we donot know which FLE has the mbuf stored. + * So while retreiving we can go back 1 FLE from the FD -ADDR + * to get the MBUF Addr from the previous FLE. + * We can have a better approach to use the inline Mbuf + */ + DPAA2_SET_FLE_ADDR(fle, DPAA2_OP_VADDR_TO_IOVA(op)); + DPAA2_FLE_SAVE_CTXT(fle, priv); + fle = fle + 1; + sge = fle + 2; + + if (likely(bpid < MAX_BPID)) { + DPAA2_SET_FD_BPID(fd, bpid); + DPAA2_SET_FLE_BPID(fle, bpid); + DPAA2_SET_FLE_BPID(fle + 1, bpid); + DPAA2_SET_FLE_BPID(sge, bpid); + DPAA2_SET_FLE_BPID(sge + 1, bpid); + } else { + DPAA2_SET_FD_IVP(fd); + DPAA2_SET_FLE_IVP(fle); + DPAA2_SET_FLE_IVP((fle + 1)); + DPAA2_SET_FLE_IVP(sge); + DPAA2_SET_FLE_IVP((sge + 1)); + } + + flc = &priv->flc_desc[0].flc; + DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(fle)); + DPAA2_SET_FD_LEN(fd, sym_op->cipher.data.length + + sess->iv.length); + DPAA2_SET_FD_COMPOUND_FMT(fd); + DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc)); + + PMD_TX_LOG(DEBUG, "cipher_off: 0x%x/length %d,ivlen=%d data_off: 0x%x", + sym_op->cipher.data.offset, + sym_op->cipher.data.length, + sess->iv.length, + sym_op->m_src->data_off); + + DPAA2_SET_FLE_ADDR(fle, DPAA2_MBUF_VADDR_TO_IOVA(dst)); + DPAA2_SET_FLE_OFFSET(fle, sym_op->cipher.data.offset + + dst->data_off); + + fle->length = sym_op->cipher.data.length + sess->iv.length; + + PMD_TX_LOG(DEBUG, "1 - flc = %p, fle = %p FLEaddr = %x-%x, length %d", + flc, fle, fle->addr_hi, fle->addr_lo, fle->length); + + fle++; + + DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge)); + fle->length = sym_op->cipher.data.length + sess->iv.length; + + DPAA2_SET_FLE_SG_EXT(fle); + + DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(iv_ptr)); + sge->length = sess->iv.length; + + sge++; + DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src)); + DPAA2_SET_FLE_OFFSET(sge, sym_op->cipher.data.offset + + sym_op->m_src->data_off); + + sge->length = sym_op->cipher.data.length; + DPAA2_SET_FLE_FIN(sge); + DPAA2_SET_FLE_FIN(fle); + + PMD_TX_LOG(DEBUG, "fdaddr =%p bpid =%d meta =%d off =%d, len =%d", + (void *)DPAA2_GET_FD_ADDR(fd), + DPAA2_GET_FD_BPID(fd), + rte_dpaa2_bpid_info[bpid].meta_data_size, + DPAA2_GET_FD_OFFSET(fd), + DPAA2_GET_FD_LEN(fd)); + + return 0; +} + +static inline int +build_sec_fd(struct rte_crypto_op *op, + struct qbman_fd *fd, uint16_t bpid) +{ + int ret = -1; + dpaa2_sec_session *sess; + + PMD_INIT_FUNC_TRACE(); + /* + * Segmented buffer is not supported. + */ + if (!rte_pktmbuf_is_contiguous(op->sym->m_src)) { + op->status = RTE_CRYPTO_OP_STATUS_ERROR; + return -ENOTSUP; + } + + if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) + sess = (dpaa2_sec_session *)get_session_private_data( + op->sym->session, cryptodev_driver_id); + else if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) + sess = (dpaa2_sec_session *)get_sec_session_private_data( + op->sym->sec_session); + else + return -1; + + switch (sess->ctxt_type) { + case DPAA2_SEC_CIPHER: + ret = build_cipher_fd(sess, op, fd, bpid); + break; + case DPAA2_SEC_AUTH: + ret = build_auth_fd(sess, op, fd, bpid); + break; + case DPAA2_SEC_AEAD: + ret = build_authenc_gcm_fd(sess, op, fd, bpid); + break; + case DPAA2_SEC_CIPHER_HASH: + ret = build_authenc_fd(sess, op, fd, bpid); + break; + case DPAA2_SEC_IPSEC: + ret = build_proto_fd(sess, op, fd, bpid); + break; + case DPAA2_SEC_HASH_CIPHER: + default: + RTE_LOG(ERR, PMD, "error: Unsupported session\n"); + } + return ret; +} + +static uint16_t +dpaa2_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops, + uint16_t nb_ops) +{ + /* Function to transmit the frames to given device and VQ*/ + uint32_t loop; + int32_t ret; + struct qbman_fd fd_arr[MAX_TX_RING_SLOTS]; + uint32_t frames_to_send; + struct qbman_eq_desc eqdesc; + struct dpaa2_sec_qp *dpaa2_qp = (struct dpaa2_sec_qp *)qp; + struct qbman_swp *swp; + uint16_t num_tx = 0; + /*todo - need to support multiple buffer pools */ + uint16_t bpid; + struct rte_mempool *mb_pool; + + if (unlikely(nb_ops == 0)) + return 0; + + if (ops[0]->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { + RTE_LOG(ERR, PMD, "sessionless crypto op not supported\n"); + return 0; + } + /*Prepare enqueue descriptor*/ + qbman_eq_desc_clear(&eqdesc); + qbman_eq_desc_set_no_orp(&eqdesc, DPAA2_EQ_RESP_ERR_FQ); + qbman_eq_desc_set_response(&eqdesc, 0, 0); + qbman_eq_desc_set_fq(&eqdesc, dpaa2_qp->tx_vq.fqid); + + if (!DPAA2_PER_LCORE_SEC_DPIO) { + ret = dpaa2_affine_qbman_swp_sec(); + if (ret) { + RTE_LOG(ERR, PMD, "Failure in affining portal\n"); + return 0; + } + } + swp = DPAA2_PER_LCORE_SEC_PORTAL; + + while (nb_ops) { + frames_to_send = (nb_ops >> 3) ? MAX_TX_RING_SLOTS : nb_ops; + + for (loop = 0; loop < frames_to_send; loop++) { + /*Clear the unused FD fields before sending*/ + memset(&fd_arr[loop], 0, sizeof(struct qbman_fd)); + mb_pool = (*ops)->sym->m_src->pool; + bpid = mempool_to_bpid(mb_pool); + ret = build_sec_fd(*ops, &fd_arr[loop], bpid); + if (ret) { + PMD_DRV_LOG(ERR, "error: Improper packet" + " contents for crypto operation\n"); + goto skip_tx; + } + ops++; + } + loop = 0; + while (loop < frames_to_send) { + loop += qbman_swp_enqueue_multiple(swp, &eqdesc, + &fd_arr[loop], + frames_to_send - loop); + } + + num_tx += frames_to_send; + nb_ops -= frames_to_send; + } +skip_tx: + dpaa2_qp->tx_vq.tx_pkts += num_tx; + dpaa2_qp->tx_vq.err_pkts += nb_ops; + return num_tx; +} + +static inline struct rte_crypto_op * +sec_simple_fd_to_mbuf(const struct qbman_fd *fd, __rte_unused uint8_t id) +{ + struct rte_crypto_op *op; + uint16_t len = DPAA2_GET_FD_LEN(fd); + uint16_t diff = 0; + dpaa2_sec_session *sess_priv; + + struct rte_mbuf *mbuf = DPAA2_INLINE_MBUF_FROM_BUF( + DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)), + rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size); + + op = (struct rte_crypto_op *)mbuf->buf_iova; + mbuf->buf_iova = op->sym->aead.digest.phys_addr; + op->sym->aead.digest.phys_addr = 0L; + + sess_priv = (dpaa2_sec_session *)get_sec_session_private_data( + op->sym->sec_session); + if (sess_priv->dir == DIR_ENC) + mbuf->data_off += SEC_FLC_DHR_OUTBOUND; + else + mbuf->data_off += SEC_FLC_DHR_INBOUND; + diff = len - mbuf->pkt_len; + mbuf->pkt_len += diff; + mbuf->data_len += diff; + + return op; +} + +static inline struct rte_crypto_op * +sec_fd_to_mbuf(const struct qbman_fd *fd, uint8_t driver_id) +{ + struct qbman_fle *fle; + struct rte_crypto_op *op; + struct ctxt_priv *priv; + struct rte_mbuf *dst, *src; + + if (DPAA2_FD_GET_FORMAT(fd) == qbman_fd_single) + return sec_simple_fd_to_mbuf(fd, driver_id); + + fle = (struct qbman_fle *)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)); + + PMD_RX_LOG(DEBUG, "FLE addr = %x - %x, offset = %x", + fle->addr_hi, fle->addr_lo, fle->fin_bpid_offset); + + /* we are using the first FLE entry to store Mbuf. + * Currently we donot know which FLE has the mbuf stored. + * So while retreiving we can go back 1 FLE from the FD -ADDR + * to get the MBUF Addr from the previous FLE. + * We can have a better approach to use the inline Mbuf + */ + + if (unlikely(DPAA2_GET_FD_IVP(fd))) { + /* TODO complete it. */ + RTE_LOG(ERR, PMD, "error: Non inline buffer - WHAT to DO?\n"); + return NULL; + } + op = (struct rte_crypto_op *)DPAA2_IOVA_TO_VADDR( + DPAA2_GET_FLE_ADDR((fle - 1))); + + /* Prefeth op */ + src = op->sym->m_src; + rte_prefetch0(src); + + if (op->sym->m_dst) { + dst = op->sym->m_dst; + rte_prefetch0(dst); + } else + dst = src; + + PMD_RX_LOG(DEBUG, "mbuf %p BMAN buf addr %p", + (void *)dst, dst->buf_addr); + + PMD_RX_LOG(DEBUG, "fdaddr =%p bpid =%d meta =%d off =%d, len =%d", + (void *)DPAA2_GET_FD_ADDR(fd), + DPAA2_GET_FD_BPID(fd), + rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size, + DPAA2_GET_FD_OFFSET(fd), + DPAA2_GET_FD_LEN(fd)); + + /* free the fle memory */ + priv = (struct ctxt_priv *)DPAA2_GET_FLE_CTXT(fle - 1); + rte_mempool_put(priv->fle_pool, (void *)(fle - 1)); + + return op; +} + +static uint16_t +dpaa2_sec_dequeue_burst(void *qp, struct rte_crypto_op **ops, + uint16_t nb_ops) +{ + /* Function is responsible to receive frames for a given device and VQ*/ + struct dpaa2_sec_qp *dpaa2_qp = (struct dpaa2_sec_qp *)qp; + struct rte_cryptodev *dev = + (struct rte_cryptodev *)(dpaa2_qp->rx_vq.dev); + struct qbman_result *dq_storage; + uint32_t fqid = dpaa2_qp->rx_vq.fqid; + int ret, num_rx = 0; + uint8_t is_last = 0, status; + struct qbman_swp *swp; + const struct qbman_fd *fd; + struct qbman_pull_desc pulldesc; + + if (!DPAA2_PER_LCORE_SEC_DPIO) { + ret = dpaa2_affine_qbman_swp_sec(); + if (ret) { + RTE_LOG(ERR, PMD, "Failure in affining portal\n"); + return 0; + } + } + swp = DPAA2_PER_LCORE_SEC_PORTAL; + dq_storage = dpaa2_qp->rx_vq.q_storage->dq_storage[0]; + + qbman_pull_desc_clear(&pulldesc); + qbman_pull_desc_set_numframes(&pulldesc, + (nb_ops > DPAA2_DQRR_RING_SIZE) ? + DPAA2_DQRR_RING_SIZE : nb_ops); + qbman_pull_desc_set_fq(&pulldesc, fqid); + qbman_pull_desc_set_storage(&pulldesc, dq_storage, + (dma_addr_t)DPAA2_VADDR_TO_IOVA(dq_storage), + 1); + + /*Issue a volatile dequeue command. */ + while (1) { + if (qbman_swp_pull(swp, &pulldesc)) { + RTE_LOG(WARNING, PMD, + "SEC VDQ command is not issued : QBMAN busy\n"); + /* Portal was busy, try again */ + continue; + } + break; + }; + + /* Receive the packets till Last Dequeue entry is found with + * respect to the above issues PULL command. + */ + while (!is_last) { + /* Check if the previous issued command is completed. + * Also seems like the SWP is shared between the Ethernet Driver + * and the SEC driver. + */ + while (!qbman_check_command_complete(dq_storage)) + ; + + /* Loop until the dq_storage is updated with + * new token by QBMAN + */ + while (!qbman_check_new_result(dq_storage)) + ; + /* Check whether Last Pull command is Expired and + * setting Condition for Loop termination + */ + if (qbman_result_DQ_is_pull_complete(dq_storage)) { + is_last = 1; + /* Check for valid frame. */ + status = (uint8_t)qbman_result_DQ_flags(dq_storage); + if (unlikely( + (status & QBMAN_DQ_STAT_VALIDFRAME) == 0)) { + PMD_RX_LOG(DEBUG, "No frame is delivered"); + continue; + } + } + + fd = qbman_result_DQ_fd(dq_storage); + ops[num_rx] = sec_fd_to_mbuf(fd, dev->driver_id); + + if (unlikely(fd->simple.frc)) { + /* TODO Parse SEC errors */ + RTE_LOG(ERR, PMD, "SEC returned Error - %x\n", + fd->simple.frc); + ops[num_rx]->status = RTE_CRYPTO_OP_STATUS_ERROR; + } else { + ops[num_rx]->status = RTE_CRYPTO_OP_STATUS_SUCCESS; + } + + num_rx++; + dq_storage++; + } /* End of Packet Rx loop */ + + dpaa2_qp->rx_vq.rx_pkts += num_rx; + + PMD_RX_LOG(DEBUG, "SEC Received %d Packets", num_rx); + /*Return the total number of packets received to DPAA2 app*/ + return num_rx; +} + +/** Release queue pair */ +static int +dpaa2_sec_queue_pair_release(struct rte_cryptodev *dev, uint16_t queue_pair_id) +{ + struct dpaa2_sec_qp *qp = + (struct dpaa2_sec_qp *)dev->data->queue_pairs[queue_pair_id]; + + PMD_INIT_FUNC_TRACE(); + + if (qp->rx_vq.q_storage) { + dpaa2_free_dq_storage(qp->rx_vq.q_storage); + rte_free(qp->rx_vq.q_storage); + } + rte_free(qp); + + dev->data->queue_pairs[queue_pair_id] = NULL; + + return 0; +} + +/** Setup a queue pair */ +static int +dpaa2_sec_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id, + __rte_unused const struct rte_cryptodev_qp_conf *qp_conf, + __rte_unused int socket_id, + __rte_unused struct rte_mempool *session_pool) +{ + struct dpaa2_sec_dev_private *priv = dev->data->dev_private; + struct dpaa2_sec_qp *qp; + struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw; + struct dpseci_rx_queue_cfg cfg; + int32_t retcode; + + PMD_INIT_FUNC_TRACE(); + + /* If qp is already in use free ring memory and qp metadata. */ + if (dev->data->queue_pairs[qp_id] != NULL) { + PMD_DRV_LOG(INFO, "QP already setup"); + return 0; + } + + PMD_DRV_LOG(DEBUG, "dev =%p, queue =%d, conf =%p", + dev, qp_id, qp_conf); + + memset(&cfg, 0, sizeof(struct dpseci_rx_queue_cfg)); + + qp = rte_malloc(NULL, sizeof(struct dpaa2_sec_qp), + RTE_CACHE_LINE_SIZE); + if (!qp) { + RTE_LOG(ERR, PMD, "malloc failed for rx/tx queues\n"); + return -1; + } + + qp->rx_vq.dev = dev; + qp->tx_vq.dev = dev; + qp->rx_vq.q_storage = rte_malloc("sec dq storage", + sizeof(struct queue_storage_info_t), + RTE_CACHE_LINE_SIZE); + if (!qp->rx_vq.q_storage) { + RTE_LOG(ERR, PMD, "malloc failed for q_storage\n"); + return -1; + } + memset(qp->rx_vq.q_storage, 0, sizeof(struct queue_storage_info_t)); + + if (dpaa2_alloc_dq_storage(qp->rx_vq.q_storage)) { + RTE_LOG(ERR, PMD, "dpaa2_alloc_dq_storage failed\n"); + return -1; + } + + dev->data->queue_pairs[qp_id] = qp; + + cfg.options = cfg.options | DPSECI_QUEUE_OPT_USER_CTX; + cfg.user_ctx = (uint64_t)(&qp->rx_vq); + retcode = dpseci_set_rx_queue(dpseci, CMD_PRI_LOW, priv->token, + qp_id, &cfg); + return retcode; +} + +/** Start queue pair */ +static int +dpaa2_sec_queue_pair_start(__rte_unused struct rte_cryptodev *dev, + __rte_unused uint16_t queue_pair_id) +{ + PMD_INIT_FUNC_TRACE(); + + return 0; +} + +/** Stop queue pair */ +static int +dpaa2_sec_queue_pair_stop(__rte_unused struct rte_cryptodev *dev, + __rte_unused uint16_t queue_pair_id) +{ + PMD_INIT_FUNC_TRACE(); + + return 0; +} + +/** Return the number of allocated queue pairs */ +static uint32_t +dpaa2_sec_queue_pair_count(struct rte_cryptodev *dev) +{ + PMD_INIT_FUNC_TRACE(); + + return dev->data->nb_queue_pairs; +} + +/** Returns the size of the aesni gcm session structure */ +static unsigned int +dpaa2_sec_session_get_size(struct rte_cryptodev *dev __rte_unused) +{ + PMD_INIT_FUNC_TRACE(); + + return sizeof(dpaa2_sec_session); +} + +static int +dpaa2_sec_cipher_init(struct rte_cryptodev *dev, + struct rte_crypto_sym_xform *xform, + dpaa2_sec_session *session) +{ + struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private; + struct alginfo cipherdata; + int bufsize, i; + struct ctxt_priv *priv; + struct sec_flow_context *flc; + + PMD_INIT_FUNC_TRACE(); + + /* For SEC CIPHER only one descriptor is required. */ + priv = (struct ctxt_priv *)rte_zmalloc(NULL, + sizeof(struct ctxt_priv) + sizeof(struct sec_flc_desc), + RTE_CACHE_LINE_SIZE); + if (priv == NULL) { + RTE_LOG(ERR, PMD, "No Memory for priv CTXT\n"); + return -1; + } + + priv->fle_pool = dev_priv->fle_pool; + + flc = &priv->flc_desc[0].flc; + + session->cipher_key.data = rte_zmalloc(NULL, xform->cipher.key.length, + RTE_CACHE_LINE_SIZE); + if (session->cipher_key.data == NULL) { + RTE_LOG(ERR, PMD, "No Memory for cipher key\n"); + rte_free(priv); + return -1; + } + session->cipher_key.length = xform->cipher.key.length; + + memcpy(session->cipher_key.data, xform->cipher.key.data, + xform->cipher.key.length); + cipherdata.key = (uint64_t)session->cipher_key.data; + cipherdata.keylen = session->cipher_key.length; + cipherdata.key_enc_flags = 0; + cipherdata.key_type = RTA_DATA_IMM; + + /* Set IV parameters */ + session->iv.offset = xform->cipher.iv.offset; + session->iv.length = xform->cipher.iv.length; + + switch (xform->cipher.algo) { + case RTE_CRYPTO_CIPHER_AES_CBC: + cipherdata.algtype = OP_ALG_ALGSEL_AES; + cipherdata.algmode = OP_ALG_AAI_CBC; + session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CBC; + break; + case RTE_CRYPTO_CIPHER_3DES_CBC: + cipherdata.algtype = OP_ALG_ALGSEL_3DES; + cipherdata.algmode = OP_ALG_AAI_CBC; + session->cipher_alg = RTE_CRYPTO_CIPHER_3DES_CBC; + break; + case RTE_CRYPTO_CIPHER_AES_CTR: + cipherdata.algtype = OP_ALG_ALGSEL_AES; + cipherdata.algmode = OP_ALG_AAI_CTR; + session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CTR; + break; + case RTE_CRYPTO_CIPHER_3DES_CTR: + case RTE_CRYPTO_CIPHER_AES_ECB: + case RTE_CRYPTO_CIPHER_3DES_ECB: + case RTE_CRYPTO_CIPHER_AES_XTS: + case RTE_CRYPTO_CIPHER_AES_F8: + case RTE_CRYPTO_CIPHER_ARC4: + case RTE_CRYPTO_CIPHER_KASUMI_F8: + case RTE_CRYPTO_CIPHER_SNOW3G_UEA2: + case RTE_CRYPTO_CIPHER_ZUC_EEA3: + case RTE_CRYPTO_CIPHER_NULL: + RTE_LOG(ERR, PMD, "Crypto: Unsupported Cipher alg %u\n", + xform->cipher.algo); + goto error_out; + default: + RTE_LOG(ERR, PMD, "Crypto: Undefined Cipher specified %u\n", + xform->cipher.algo); + goto error_out; + } + session->dir = (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ? + DIR_ENC : DIR_DEC; + + bufsize = cnstr_shdsc_blkcipher(priv->flc_desc[0].desc, 1, 0, + &cipherdata, NULL, session->iv.length, + session->dir); + if (bufsize < 0) { + RTE_LOG(ERR, PMD, "Crypto: Descriptor build failed\n"); + goto error_out; + } + flc->dhr = 0; + flc->bpv0 = 0x1; + flc->mode_bits = 0x8000; + + flc->word1_sdl = (uint8_t)bufsize; + flc->word2_rflc_31_0 = lower_32_bits( + (uint64_t)&(((struct dpaa2_sec_qp *) + dev->data->queue_pairs[0])->rx_vq)); + flc->word3_rflc_63_32 = upper_32_bits( + (uint64_t)&(((struct dpaa2_sec_qp *) + dev->data->queue_pairs[0])->rx_vq)); + session->ctxt = priv; + + for (i = 0; i < bufsize; i++) + PMD_DRV_LOG(DEBUG, "DESC[%d]:0x%x\n", + i, priv->flc_desc[0].desc[i]); + + return 0; + +error_out: + rte_free(session->cipher_key.data); + rte_free(priv); + return -1; +} + +static int +dpaa2_sec_auth_init(struct rte_cryptodev *dev, + struct rte_crypto_sym_xform *xform, + dpaa2_sec_session *session) +{ + struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private; + struct alginfo authdata; + unsigned int bufsize, i; + struct ctxt_priv *priv; + struct sec_flow_context *flc; + + PMD_INIT_FUNC_TRACE(); + + /* For SEC AUTH three descriptors are required for various stages */ + priv = (struct ctxt_priv *)rte_zmalloc(NULL, + sizeof(struct ctxt_priv) + 3 * + sizeof(struct sec_flc_desc), + RTE_CACHE_LINE_SIZE); + if (priv == NULL) { + RTE_LOG(ERR, PMD, "No Memory for priv CTXT\n"); + return -1; + } + + priv->fle_pool = dev_priv->fle_pool; + flc = &priv->flc_desc[DESC_INITFINAL].flc; + + session->auth_key.data = rte_zmalloc(NULL, xform->auth.key.length, + RTE_CACHE_LINE_SIZE); + if (session->auth_key.data == NULL) { + RTE_LOG(ERR, PMD, "No Memory for auth key\n"); + rte_free(priv); + return -1; + } + session->auth_key.length = xform->auth.key.length; + + memcpy(session->auth_key.data, xform->auth.key.data, + xform->auth.key.length); + authdata.key = (uint64_t)session->auth_key.data; + authdata.keylen = session->auth_key.length; + authdata.key_enc_flags = 0; + authdata.key_type = RTA_DATA_IMM; + + session->digest_length = xform->auth.digest_length; + + switch (xform->auth.algo) { + case RTE_CRYPTO_AUTH_SHA1_HMAC: + authdata.algtype = OP_ALG_ALGSEL_SHA1; + authdata.algmode = OP_ALG_AAI_HMAC; + session->auth_alg = RTE_CRYPTO_AUTH_SHA1_HMAC; + break; + case RTE_CRYPTO_AUTH_MD5_HMAC: + authdata.algtype = OP_ALG_ALGSEL_MD5; + authdata.algmode = OP_ALG_AAI_HMAC; + session->auth_alg = RTE_CRYPTO_AUTH_MD5_HMAC; + break; + case RTE_CRYPTO_AUTH_SHA256_HMAC: + authdata.algtype = OP_ALG_ALGSEL_SHA256; + authdata.algmode = OP_ALG_AAI_HMAC; + session->auth_alg = RTE_CRYPTO_AUTH_SHA256_HMAC; + break; + case RTE_CRYPTO_AUTH_SHA384_HMAC: + authdata.algtype = OP_ALG_ALGSEL_SHA384; + authdata.algmode = OP_ALG_AAI_HMAC; + session->auth_alg = RTE_CRYPTO_AUTH_SHA384_HMAC; + break; + case RTE_CRYPTO_AUTH_SHA512_HMAC: + authdata.algtype = OP_ALG_ALGSEL_SHA512; + authdata.algmode = OP_ALG_AAI_HMAC; + session->auth_alg = RTE_CRYPTO_AUTH_SHA512_HMAC; + break; + case RTE_CRYPTO_AUTH_SHA224_HMAC: + authdata.algtype = OP_ALG_ALGSEL_SHA224; + authdata.algmode = OP_ALG_AAI_HMAC; + session->auth_alg = RTE_CRYPTO_AUTH_SHA224_HMAC; + break; + case RTE_CRYPTO_AUTH_AES_XCBC_MAC: + case RTE_CRYPTO_AUTH_SNOW3G_UIA2: + case RTE_CRYPTO_AUTH_NULL: + case RTE_CRYPTO_AUTH_SHA1: + case RTE_CRYPTO_AUTH_SHA256: + case RTE_CRYPTO_AUTH_SHA512: + case RTE_CRYPTO_AUTH_SHA224: + case RTE_CRYPTO_AUTH_SHA384: + case RTE_CRYPTO_AUTH_MD5: + case RTE_CRYPTO_AUTH_AES_GMAC: + case RTE_CRYPTO_AUTH_KASUMI_F9: + case RTE_CRYPTO_AUTH_AES_CMAC: + case RTE_CRYPTO_AUTH_AES_CBC_MAC: + case RTE_CRYPTO_AUTH_ZUC_EIA3: + RTE_LOG(ERR, PMD, "Crypto: Unsupported auth alg %u\n", + xform->auth.algo); + goto error_out; + default: + RTE_LOG(ERR, PMD, "Crypto: Undefined Auth specified %u\n", + xform->auth.algo); + goto error_out; + } + session->dir = (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) ? + DIR_ENC : DIR_DEC; + + bufsize = cnstr_shdsc_hmac(priv->flc_desc[DESC_INITFINAL].desc, + 1, 0, &authdata, !session->dir, + session->digest_length); + + flc->word1_sdl = (uint8_t)bufsize; + flc->word2_rflc_31_0 = lower_32_bits( + (uint64_t)&(((struct dpaa2_sec_qp *) + dev->data->queue_pairs[0])->rx_vq)); + flc->word3_rflc_63_32 = upper_32_bits( + (uint64_t)&(((struct dpaa2_sec_qp *) + dev->data->queue_pairs[0])->rx_vq)); + session->ctxt = priv; + for (i = 0; i < bufsize; i++) + PMD_DRV_LOG(DEBUG, "DESC[%d]:0x%x\n", + i, priv->flc_desc[DESC_INITFINAL].desc[i]); + + + return 0; + +error_out: + rte_free(session->auth_key.data); + rte_free(priv); + return -1; +} + +static int +dpaa2_sec_aead_init(struct rte_cryptodev *dev, + struct rte_crypto_sym_xform *xform, + dpaa2_sec_session *session) +{ + struct dpaa2_sec_aead_ctxt *ctxt = &session->ext_params.aead_ctxt; + struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private; + struct alginfo aeaddata; + unsigned int bufsize, i; + struct ctxt_priv *priv; + struct sec_flow_context *flc; + struct rte_crypto_aead_xform *aead_xform = &xform->aead; + int err; + + PMD_INIT_FUNC_TRACE(); + + /* Set IV parameters */ + session->iv.offset = aead_xform->iv.offset; + session->iv.length = aead_xform->iv.length; + session->ctxt_type = DPAA2_SEC_AEAD; + + /* For SEC AEAD only one descriptor is required */ + priv = (struct ctxt_priv *)rte_zmalloc(NULL, + sizeof(struct ctxt_priv) + sizeof(struct sec_flc_desc), + RTE_CACHE_LINE_SIZE); + if (priv == NULL) { + RTE_LOG(ERR, PMD, "No Memory for priv CTXT\n"); + return -1; + } + + priv->fle_pool = dev_priv->fle_pool; + flc = &priv->flc_desc[0].flc; + + session->aead_key.data = rte_zmalloc(NULL, aead_xform->key.length, + RTE_CACHE_LINE_SIZE); + if (session->aead_key.data == NULL && aead_xform->key.length > 0) { + RTE_LOG(ERR, PMD, "No Memory for aead key\n"); + rte_free(priv); + return -1; + } + memcpy(session->aead_key.data, aead_xform->key.data, + aead_xform->key.length); + + session->digest_length = aead_xform->digest_length; + session->aead_key.length = aead_xform->key.length; + ctxt->auth_only_len = aead_xform->aad_length; + + aeaddata.key = (uint64_t)session->aead_key.data; + aeaddata.keylen = session->aead_key.length; + aeaddata.key_enc_flags = 0; + aeaddata.key_type = RTA_DATA_IMM; + + switch (aead_xform->algo) { + case RTE_CRYPTO_AEAD_AES_GCM: + aeaddata.algtype = OP_ALG_ALGSEL_AES; + aeaddata.algmode = OP_ALG_AAI_GCM; + session->aead_alg = RTE_CRYPTO_AEAD_AES_GCM; + break; + case RTE_CRYPTO_AEAD_AES_CCM: + RTE_LOG(ERR, PMD, "Crypto: Unsupported AEAD alg %u\n", + aead_xform->algo); + goto error_out; + default: + RTE_LOG(ERR, PMD, "Crypto: Undefined AEAD specified %u\n", + aead_xform->algo); + goto error_out; + } + session->dir = (aead_xform->op == RTE_CRYPTO_AEAD_OP_ENCRYPT) ? + DIR_ENC : DIR_DEC; + + priv->flc_desc[0].desc[0] = aeaddata.keylen; + err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN, + MIN_JOB_DESC_SIZE, + (unsigned int *)priv->flc_desc[0].desc, + &priv->flc_desc[0].desc[1], 1); + + if (err < 0) { + PMD_DRV_LOG(ERR, "Crypto: Incorrect key lengths\n"); + goto error_out; + } + if (priv->flc_desc[0].desc[1] & 1) { + aeaddata.key_type = RTA_DATA_IMM; + } else { + aeaddata.key = DPAA2_VADDR_TO_IOVA(aeaddata.key); + aeaddata.key_type = RTA_DATA_PTR; + } + priv->flc_desc[0].desc[0] = 0; + priv->flc_desc[0].desc[1] = 0; + + if (session->dir == DIR_ENC) + bufsize = cnstr_shdsc_gcm_encap( + priv->flc_desc[0].desc, 1, 0, + &aeaddata, session->iv.length, + session->digest_length); + else + bufsize = cnstr_shdsc_gcm_decap( + priv->flc_desc[0].desc, 1, 0, + &aeaddata, session->iv.length, + session->digest_length); + flc->word1_sdl = (uint8_t)bufsize; + flc->word2_rflc_31_0 = lower_32_bits( + (uint64_t)&(((struct dpaa2_sec_qp *) + dev->data->queue_pairs[0])->rx_vq)); + flc->word3_rflc_63_32 = upper_32_bits( + (uint64_t)&(((struct dpaa2_sec_qp *) + dev->data->queue_pairs[0])->rx_vq)); + session->ctxt = priv; + for (i = 0; i < bufsize; i++) + PMD_DRV_LOG(DEBUG, "DESC[%d]:0x%x\n", + i, priv->flc_desc[0].desc[i]); + + return 0; + +error_out: + rte_free(session->aead_key.data); + rte_free(priv); + return -1; +} + + +static int +dpaa2_sec_aead_chain_init(struct rte_cryptodev *dev, + struct rte_crypto_sym_xform *xform, + dpaa2_sec_session *session) +{ + struct dpaa2_sec_aead_ctxt *ctxt = &session->ext_params.aead_ctxt; + struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private; + struct alginfo authdata, cipherdata; + unsigned int bufsize, i; + struct ctxt_priv *priv; + struct sec_flow_context *flc; + struct rte_crypto_cipher_xform *cipher_xform; + struct rte_crypto_auth_xform *auth_xform; + int err; + + PMD_INIT_FUNC_TRACE(); + + if (session->ext_params.aead_ctxt.auth_cipher_text) { + cipher_xform = &xform->cipher; + auth_xform = &xform->next->auth; + session->ctxt_type = + (cipher_xform->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ? + DPAA2_SEC_CIPHER_HASH : DPAA2_SEC_HASH_CIPHER; + } else { + cipher_xform = &xform->next->cipher; + auth_xform = &xform->auth; + session->ctxt_type = + (cipher_xform->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ? + DPAA2_SEC_HASH_CIPHER : DPAA2_SEC_CIPHER_HASH; + } + + /* Set IV parameters */ + session->iv.offset = cipher_xform->iv.offset; + session->iv.length = cipher_xform->iv.length; + + /* For SEC AEAD only one descriptor is required */ + priv = (struct ctxt_priv *)rte_zmalloc(NULL, + sizeof(struct ctxt_priv) + sizeof(struct sec_flc_desc), + RTE_CACHE_LINE_SIZE); + if (priv == NULL) { + RTE_LOG(ERR, PMD, "No Memory for priv CTXT\n"); + return -1; + } + + priv->fle_pool = dev_priv->fle_pool; + flc = &priv->flc_desc[0].flc; + + session->cipher_key.data = rte_zmalloc(NULL, cipher_xform->key.length, + RTE_CACHE_LINE_SIZE); + if (session->cipher_key.data == NULL && cipher_xform->key.length > 0) { + RTE_LOG(ERR, PMD, "No Memory for cipher key\n"); + rte_free(priv); + return -1; + } + session->cipher_key.length = cipher_xform->key.length; + session->auth_key.data = rte_zmalloc(NULL, auth_xform->key.length, + RTE_CACHE_LINE_SIZE); + if (session->auth_key.data == NULL && auth_xform->key.length > 0) { + RTE_LOG(ERR, PMD, "No Memory for auth key\n"); + rte_free(session->cipher_key.data); + rte_free(priv); + return -1; + } + session->auth_key.length = auth_xform->key.length; + memcpy(session->cipher_key.data, cipher_xform->key.data, + cipher_xform->key.length); + memcpy(session->auth_key.data, auth_xform->key.data, + auth_xform->key.length); + + authdata.key = (uint64_t)session->auth_key.data; + authdata.keylen = session->auth_key.length; + authdata.key_enc_flags = 0; + authdata.key_type = RTA_DATA_IMM; + + session->digest_length = auth_xform->digest_length; + + switch (auth_xform->algo) { + case RTE_CRYPTO_AUTH_SHA1_HMAC: + authdata.algtype = OP_ALG_ALGSEL_SHA1; + authdata.algmode = OP_ALG_AAI_HMAC; + session->auth_alg = RTE_CRYPTO_AUTH_SHA1_HMAC; + break; + case RTE_CRYPTO_AUTH_MD5_HMAC: + authdata.algtype = OP_ALG_ALGSEL_MD5; + authdata.algmode = OP_ALG_AAI_HMAC; + session->auth_alg = RTE_CRYPTO_AUTH_MD5_HMAC; + break; + case RTE_CRYPTO_AUTH_SHA224_HMAC: + authdata.algtype = OP_ALG_ALGSEL_SHA224; + authdata.algmode = OP_ALG_AAI_HMAC; + session->auth_alg = RTE_CRYPTO_AUTH_SHA224_HMAC; + break; + case RTE_CRYPTO_AUTH_SHA256_HMAC: + authdata.algtype = OP_ALG_ALGSEL_SHA256; + authdata.algmode = OP_ALG_AAI_HMAC; + session->auth_alg = RTE_CRYPTO_AUTH_SHA256_HMAC; + break; + case RTE_CRYPTO_AUTH_SHA384_HMAC: + authdata.algtype = OP_ALG_ALGSEL_SHA384; + authdata.algmode = OP_ALG_AAI_HMAC; + session->auth_alg = RTE_CRYPTO_AUTH_SHA384_HMAC; + break; + case RTE_CRYPTO_AUTH_SHA512_HMAC: + authdata.algtype = OP_ALG_ALGSEL_SHA512; + authdata.algmode = OP_ALG_AAI_HMAC; + session->auth_alg = RTE_CRYPTO_AUTH_SHA512_HMAC; + break; + case RTE_CRYPTO_AUTH_AES_XCBC_MAC: + case RTE_CRYPTO_AUTH_SNOW3G_UIA2: + case RTE_CRYPTO_AUTH_NULL: + case RTE_CRYPTO_AUTH_SHA1: + case RTE_CRYPTO_AUTH_SHA256: + case RTE_CRYPTO_AUTH_SHA512: + case RTE_CRYPTO_AUTH_SHA224: + case RTE_CRYPTO_AUTH_SHA384: + case RTE_CRYPTO_AUTH_MD5: + case RTE_CRYPTO_AUTH_AES_GMAC: + case RTE_CRYPTO_AUTH_KASUMI_F9: + case RTE_CRYPTO_AUTH_AES_CMAC: + case RTE_CRYPTO_AUTH_AES_CBC_MAC: + case RTE_CRYPTO_AUTH_ZUC_EIA3: + RTE_LOG(ERR, PMD, "Crypto: Unsupported auth alg %u\n", + auth_xform->algo); + goto error_out; + default: + RTE_LOG(ERR, PMD, "Crypto: Undefined Auth specified %u\n", + auth_xform->algo); + goto error_out; + } + cipherdata.key = (uint64_t)session->cipher_key.data; + cipherdata.keylen = session->cipher_key.length; + cipherdata.key_enc_flags = 0; + cipherdata.key_type = RTA_DATA_IMM; + + switch (cipher_xform->algo) { + case RTE_CRYPTO_CIPHER_AES_CBC: + cipherdata.algtype = OP_ALG_ALGSEL_AES; + cipherdata.algmode = OP_ALG_AAI_CBC; + session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CBC; + break; + case RTE_CRYPTO_CIPHER_3DES_CBC: + cipherdata.algtype = OP_ALG_ALGSEL_3DES; + cipherdata.algmode = OP_ALG_AAI_CBC; + session->cipher_alg = RTE_CRYPTO_CIPHER_3DES_CBC; + break; + case RTE_CRYPTO_CIPHER_AES_CTR: + cipherdata.algtype = OP_ALG_ALGSEL_AES; + cipherdata.algmode = OP_ALG_AAI_CTR; + session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CTR; + break; + case RTE_CRYPTO_CIPHER_SNOW3G_UEA2: + case RTE_CRYPTO_CIPHER_NULL: + case RTE_CRYPTO_CIPHER_3DES_ECB: + case RTE_CRYPTO_CIPHER_AES_ECB: + case RTE_CRYPTO_CIPHER_KASUMI_F8: + RTE_LOG(ERR, PMD, "Crypto: Unsupported Cipher alg %u\n", + cipher_xform->algo); + goto error_out; + default: + RTE_LOG(ERR, PMD, "Crypto: Undefined Cipher specified %u\n", + cipher_xform->algo); + goto error_out; + } + session->dir = (cipher_xform->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ? + DIR_ENC : DIR_DEC; + + priv->flc_desc[0].desc[0] = cipherdata.keylen; + priv->flc_desc[0].desc[1] = authdata.keylen; + err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN, + MIN_JOB_DESC_SIZE, + (unsigned int *)priv->flc_desc[0].desc, + &priv->flc_desc[0].desc[2], 2); + + if (err < 0) { + PMD_DRV_LOG(ERR, "Crypto: Incorrect key lengths\n"); + goto error_out; + } + if (priv->flc_desc[0].desc[2] & 1) { + cipherdata.key_type = RTA_DATA_IMM; + } else { + cipherdata.key = DPAA2_VADDR_TO_IOVA(cipherdata.key); + cipherdata.key_type = RTA_DATA_PTR; + } + if (priv->flc_desc[0].desc[2] & (1 << 1)) { + authdata.key_type = RTA_DATA_IMM; + } else { + authdata.key = DPAA2_VADDR_TO_IOVA(authdata.key); + authdata.key_type = RTA_DATA_PTR; + } + priv->flc_desc[0].desc[0] = 0; + priv->flc_desc[0].desc[1] = 0; + priv->flc_desc[0].desc[2] = 0; + + if (session->ctxt_type == DPAA2_SEC_CIPHER_HASH) { + bufsize = cnstr_shdsc_authenc(priv->flc_desc[0].desc, 1, + 0, &cipherdata, &authdata, + session->iv.length, + ctxt->auth_only_len, + session->digest_length, + session->dir); + } else { + RTE_LOG(ERR, PMD, "Hash before cipher not supported\n"); + goto error_out; + } + + flc->word1_sdl = (uint8_t)bufsize; + flc->word2_rflc_31_0 = lower_32_bits( + (uint64_t)&(((struct dpaa2_sec_qp *) + dev->data->queue_pairs[0])->rx_vq)); + flc->word3_rflc_63_32 = upper_32_bits( + (uint64_t)&(((struct dpaa2_sec_qp *) + dev->data->queue_pairs[0])->rx_vq)); + session->ctxt = priv; + for (i = 0; i < bufsize; i++) + PMD_DRV_LOG(DEBUG, "DESC[%d]:0x%x\n", + i, priv->flc_desc[0].desc[i]); + + return 0; + +error_out: + rte_free(session->cipher_key.data); + rte_free(session->auth_key.data); + rte_free(priv); + return -1; +} + +static int +dpaa2_sec_set_session_parameters(struct rte_cryptodev *dev, + struct rte_crypto_sym_xform *xform, void *sess) +{ + dpaa2_sec_session *session = sess; + + PMD_INIT_FUNC_TRACE(); + + if (unlikely(sess == NULL)) { + RTE_LOG(ERR, PMD, "invalid session struct\n"); + return -1; + } + + /* Default IV length = 0 */ + session->iv.length = 0; + + /* Cipher Only */ + if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && xform->next == NULL) { + session->ctxt_type = DPAA2_SEC_CIPHER; + dpaa2_sec_cipher_init(dev, xform, session); + + /* Authentication Only */ + } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH && + xform->next == NULL) { + session->ctxt_type = DPAA2_SEC_AUTH; + dpaa2_sec_auth_init(dev, xform, session); + + /* Cipher then Authenticate */ + } else if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && + xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) { + session->ext_params.aead_ctxt.auth_cipher_text = true; + dpaa2_sec_aead_chain_init(dev, xform, session); + + /* Authenticate then Cipher */ + } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH && + xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { + session->ext_params.aead_ctxt.auth_cipher_text = false; + dpaa2_sec_aead_chain_init(dev, xform, session); + + /* AEAD operation for AES-GCM kind of Algorithms */ + } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD && + xform->next == NULL) { + dpaa2_sec_aead_init(dev, xform, session); + + } else { + RTE_LOG(ERR, PMD, "Invalid crypto type\n"); + return -EINVAL; + } + + return 0; +} + +static int +dpaa2_sec_set_ipsec_session(struct rte_cryptodev *dev, + struct rte_security_session_conf *conf, + void *sess) +{ + struct rte_security_ipsec_xform *ipsec_xform = &conf->ipsec; + struct rte_crypto_auth_xform *auth_xform; + struct rte_crypto_cipher_xform *cipher_xform; + dpaa2_sec_session *session = (dpaa2_sec_session *)sess; + struct ctxt_priv *priv; + struct ipsec_encap_pdb encap_pdb; + struct ipsec_decap_pdb decap_pdb; + struct alginfo authdata, cipherdata; + unsigned int bufsize; + struct sec_flow_context *flc; + + PMD_INIT_FUNC_TRACE(); + + if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) { + cipher_xform = &conf->crypto_xform->cipher; + auth_xform = &conf->crypto_xform->next->auth; + } else { + auth_xform = &conf->crypto_xform->auth; + cipher_xform = &conf->crypto_xform->next->cipher; + } + priv = (struct ctxt_priv *)rte_zmalloc(NULL, + sizeof(struct ctxt_priv) + + sizeof(struct sec_flc_desc), + RTE_CACHE_LINE_SIZE); + + if (priv == NULL) { + RTE_LOG(ERR, PMD, "\nNo memory for priv CTXT"); + return -ENOMEM; + } + + flc = &priv->flc_desc[0].flc; + + session->ctxt_type = DPAA2_SEC_IPSEC; + session->cipher_key.data = rte_zmalloc(NULL, + cipher_xform->key.length, + RTE_CACHE_LINE_SIZE); + if (session->cipher_key.data == NULL && + cipher_xform->key.length > 0) { + RTE_LOG(ERR, PMD, "No Memory for cipher key\n"); + rte_free(priv); + return -ENOMEM; + } + + session->cipher_key.length = cipher_xform->key.length; + session->auth_key.data = rte_zmalloc(NULL, + auth_xform->key.length, + RTE_CACHE_LINE_SIZE); + if (session->auth_key.data == NULL && + auth_xform->key.length > 0) { + RTE_LOG(ERR, PMD, "No Memory for auth key\n"); + rte_free(session->cipher_key.data); + rte_free(priv); + return -ENOMEM; + } + session->auth_key.length = auth_xform->key.length; + memcpy(session->cipher_key.data, cipher_xform->key.data, + cipher_xform->key.length); + memcpy(session->auth_key.data, auth_xform->key.data, + auth_xform->key.length); + + authdata.key = (uint64_t)session->auth_key.data; + authdata.keylen = session->auth_key.length; + authdata.key_enc_flags = 0; + authdata.key_type = RTA_DATA_IMM; + switch (auth_xform->algo) { + case RTE_CRYPTO_AUTH_SHA1_HMAC: + authdata.algtype = OP_PCL_IPSEC_HMAC_SHA1_96; + authdata.algmode = OP_ALG_AAI_HMAC; + session->auth_alg = RTE_CRYPTO_AUTH_SHA1_HMAC; + break; + case RTE_CRYPTO_AUTH_MD5_HMAC: + authdata.algtype = OP_PCL_IPSEC_HMAC_MD5_96; + authdata.algmode = OP_ALG_AAI_HMAC; + session->auth_alg = RTE_CRYPTO_AUTH_MD5_HMAC; + break; + case RTE_CRYPTO_AUTH_SHA256_HMAC: + authdata.algtype = OP_PCL_IPSEC_HMAC_SHA2_256_128; + authdata.algmode = OP_ALG_AAI_HMAC; + session->auth_alg = RTE_CRYPTO_AUTH_SHA256_HMAC; + break; + case RTE_CRYPTO_AUTH_SHA384_HMAC: + authdata.algtype = OP_PCL_IPSEC_HMAC_SHA2_384_192; + authdata.algmode = OP_ALG_AAI_HMAC; + session->auth_alg = RTE_CRYPTO_AUTH_SHA384_HMAC; + break; + case RTE_CRYPTO_AUTH_SHA512_HMAC: + authdata.algtype = OP_PCL_IPSEC_HMAC_SHA2_512_256; + authdata.algmode = OP_ALG_AAI_HMAC; + session->auth_alg = RTE_CRYPTO_AUTH_SHA512_HMAC; + break; + case RTE_CRYPTO_AUTH_AES_CMAC: + authdata.algtype = OP_PCL_IPSEC_AES_CMAC_96; + session->auth_alg = RTE_CRYPTO_AUTH_AES_CMAC; + break; + case RTE_CRYPTO_AUTH_NULL: + authdata.algtype = OP_PCL_IPSEC_HMAC_NULL; + session->auth_alg = RTE_CRYPTO_AUTH_NULL; + break; + case RTE_CRYPTO_AUTH_SHA224_HMAC: + case RTE_CRYPTO_AUTH_AES_XCBC_MAC: + case RTE_CRYPTO_AUTH_SNOW3G_UIA2: + case RTE_CRYPTO_AUTH_SHA1: + case RTE_CRYPTO_AUTH_SHA256: + case RTE_CRYPTO_AUTH_SHA512: + case RTE_CRYPTO_AUTH_SHA224: + case RTE_CRYPTO_AUTH_SHA384: + case RTE_CRYPTO_AUTH_MD5: + case RTE_CRYPTO_AUTH_AES_GMAC: + case RTE_CRYPTO_AUTH_KASUMI_F9: + case RTE_CRYPTO_AUTH_AES_CBC_MAC: + case RTE_CRYPTO_AUTH_ZUC_EIA3: + RTE_LOG(ERR, PMD, "Crypto: Unsupported auth alg %u\n", + auth_xform->algo); + goto out; + default: + RTE_LOG(ERR, PMD, "Crypto: Undefined Auth specified %u\n", + auth_xform->algo); + goto out; + } + cipherdata.key = (uint64_t)session->cipher_key.data; + cipherdata.keylen = session->cipher_key.length; + cipherdata.key_enc_flags = 0; + cipherdata.key_type = RTA_DATA_IMM; + + switch (cipher_xform->algo) { + case RTE_CRYPTO_CIPHER_AES_CBC: + cipherdata.algtype = OP_PCL_IPSEC_AES_CBC; + cipherdata.algmode = OP_ALG_AAI_CBC; + session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CBC; + break; + case RTE_CRYPTO_CIPHER_3DES_CBC: + cipherdata.algtype = OP_PCL_IPSEC_3DES; + cipherdata.algmode = OP_ALG_AAI_CBC; + session->cipher_alg = RTE_CRYPTO_CIPHER_3DES_CBC; + break; + case RTE_CRYPTO_CIPHER_AES_CTR: + cipherdata.algtype = OP_PCL_IPSEC_AES_CTR; + cipherdata.algmode = OP_ALG_AAI_CTR; + session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CTR; + break; + case RTE_CRYPTO_CIPHER_NULL: + cipherdata.algtype = OP_PCL_IPSEC_NULL; + break; + case RTE_CRYPTO_CIPHER_SNOW3G_UEA2: + case RTE_CRYPTO_CIPHER_3DES_ECB: + case RTE_CRYPTO_CIPHER_AES_ECB: + case RTE_CRYPTO_CIPHER_KASUMI_F8: + RTE_LOG(ERR, PMD, "Crypto: Unsupported Cipher alg %u\n", + cipher_xform->algo); + goto out; + default: + RTE_LOG(ERR, PMD, "Crypto: Undefined Cipher specified %u\n", + cipher_xform->algo); + goto out; + } + + if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) { + struct ip ip4_hdr; + + flc->dhr = SEC_FLC_DHR_OUTBOUND; + ip4_hdr.ip_v = IPVERSION; + ip4_hdr.ip_hl = 5; + ip4_hdr.ip_len = rte_cpu_to_be_16(sizeof(ip4_hdr)); + ip4_hdr.ip_tos = ipsec_xform->tunnel.ipv4.dscp; + ip4_hdr.ip_id = 0; + ip4_hdr.ip_off = 0; + ip4_hdr.ip_ttl = ipsec_xform->tunnel.ipv4.ttl; + ip4_hdr.ip_p = 0x32; + ip4_hdr.ip_sum = 0; + ip4_hdr.ip_src = ipsec_xform->tunnel.ipv4.src_ip; + ip4_hdr.ip_dst = ipsec_xform->tunnel.ipv4.dst_ip; + ip4_hdr.ip_sum = calc_chksum((uint16_t *)(void *)&ip4_hdr, + sizeof(struct ip)); + + /* For Sec Proto only one descriptor is required. */ + memset(&encap_pdb, 0, sizeof(struct ipsec_encap_pdb)); + encap_pdb.options = (IPVERSION << PDBNH_ESP_ENCAP_SHIFT) | + PDBOPTS_ESP_OIHI_PDB_INL | + PDBOPTS_ESP_IVSRC | + PDBHMO_ESP_ENCAP_DTTL; + encap_pdb.spi = ipsec_xform->spi; + encap_pdb.ip_hdr_len = sizeof(struct ip); + + session->dir = DIR_ENC; + bufsize = cnstr_shdsc_ipsec_new_encap(priv->flc_desc[0].desc, + 1, 0, &encap_pdb, + (uint8_t *)&ip4_hdr, + &cipherdata, &authdata); + } else if (ipsec_xform->direction == + RTE_SECURITY_IPSEC_SA_DIR_INGRESS) { + flc->dhr = SEC_FLC_DHR_INBOUND; + memset(&decap_pdb, 0, sizeof(struct ipsec_decap_pdb)); + decap_pdb.options = sizeof(struct ip) << 16; + session->dir = DIR_DEC; + bufsize = cnstr_shdsc_ipsec_new_decap(priv->flc_desc[0].desc, + 1, 0, &decap_pdb, &cipherdata, &authdata); + } else + goto out; + flc->word1_sdl = (uint8_t)bufsize; + + /* Enable the stashing control bit */ + DPAA2_SET_FLC_RSC(flc); + flc->word2_rflc_31_0 = lower_32_bits( + (uint64_t)&(((struct dpaa2_sec_qp *) + dev->data->queue_pairs[0])->rx_vq) | 0x14); + flc->word3_rflc_63_32 = upper_32_bits( + (uint64_t)&(((struct dpaa2_sec_qp *) + dev->data->queue_pairs[0])->rx_vq)); + + /* Set EWS bit i.e. enable write-safe */ + DPAA2_SET_FLC_EWS(flc); + /* Set BS = 1 i.e reuse input buffers as output buffers */ + DPAA2_SET_FLC_REUSE_BS(flc); + /* Set FF = 10; reuse input buffers if they provide sufficient space */ + DPAA2_SET_FLC_REUSE_FF(flc); + + session->ctxt = priv; + + return 0; +out: + rte_free(session->auth_key.data); + rte_free(session->cipher_key.data); + rte_free(priv); + return -1; +} + +static int +dpaa2_sec_security_session_create(void *dev, + struct rte_security_session_conf *conf, + struct rte_security_session *sess, + struct rte_mempool *mempool) +{ + void *sess_private_data; + struct rte_cryptodev *cdev = (struct rte_cryptodev *)dev; + int ret; + + if (rte_mempool_get(mempool, &sess_private_data)) { + CDEV_LOG_ERR( + "Couldn't get object from session mempool"); + return -ENOMEM; + } + + switch (conf->protocol) { + case RTE_SECURITY_PROTOCOL_IPSEC: + ret = dpaa2_sec_set_ipsec_session(cdev, conf, + sess_private_data); + break; + case RTE_SECURITY_PROTOCOL_MACSEC: + return -ENOTSUP; + default: + return -EINVAL; + } + if (ret != 0) { + PMD_DRV_LOG(ERR, + "DPAA2 PMD: failed to configure session parameters"); + + /* Return session to mempool */ + rte_mempool_put(mempool, sess_private_data); + return ret; + } + + set_sec_session_private_data(sess, sess_private_data); + + return ret; +} + +/** Clear the memory of session so it doesn't leave key material behind */ +static int +dpaa2_sec_security_session_destroy(void *dev __rte_unused, + struct rte_security_session *sess) +{ + PMD_INIT_FUNC_TRACE(); + void *sess_priv = get_sec_session_private_data(sess); + + dpaa2_sec_session *s = (dpaa2_sec_session *)sess_priv; + + if (sess_priv) { + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + + rte_free(s->ctxt); + rte_free(s->cipher_key.data); + rte_free(s->auth_key.data); + memset(sess, 0, sizeof(dpaa2_sec_session)); + set_sec_session_private_data(sess, NULL); + rte_mempool_put(sess_mp, sess_priv); + } + return 0; +} + +static int +dpaa2_sec_session_configure(struct rte_cryptodev *dev, + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool) +{ + void *sess_private_data; + int ret; + + if (rte_mempool_get(mempool, &sess_private_data)) { + CDEV_LOG_ERR( + "Couldn't get object from session mempool"); + return -ENOMEM; + } + + ret = dpaa2_sec_set_session_parameters(dev, xform, sess_private_data); + if (ret != 0) { + PMD_DRV_LOG(ERR, "DPAA2 PMD: failed to configure " + "session parameters"); + + /* Return session to mempool */ + rte_mempool_put(mempool, sess_private_data); + return ret; + } + + set_session_private_data(sess, dev->driver_id, + sess_private_data); + + return 0; +} + +/** Clear the memory of session so it doesn't leave key material behind */ +static void +dpaa2_sec_session_clear(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) +{ + PMD_INIT_FUNC_TRACE(); + uint8_t index = dev->driver_id; + void *sess_priv = get_session_private_data(sess, index); + dpaa2_sec_session *s = (dpaa2_sec_session *)sess_priv; + + if (sess_priv) { + rte_free(s->ctxt); + rte_free(s->cipher_key.data); + rte_free(s->auth_key.data); + memset(sess, 0, sizeof(dpaa2_sec_session)); + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + set_session_private_data(sess, index, NULL); + rte_mempool_put(sess_mp, sess_priv); + } +} + +static int +dpaa2_sec_dev_configure(struct rte_cryptodev *dev __rte_unused, + struct rte_cryptodev_config *config __rte_unused) +{ + PMD_INIT_FUNC_TRACE(); + + return 0; +} + +static int +dpaa2_sec_dev_start(struct rte_cryptodev *dev) +{ + struct dpaa2_sec_dev_private *priv = dev->data->dev_private; + struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw; + struct dpseci_attr attr; + struct dpaa2_queue *dpaa2_q; + struct dpaa2_sec_qp **qp = (struct dpaa2_sec_qp **) + dev->data->queue_pairs; + struct dpseci_rx_queue_attr rx_attr; + struct dpseci_tx_queue_attr tx_attr; + int ret, i; + + PMD_INIT_FUNC_TRACE(); + + memset(&attr, 0, sizeof(struct dpseci_attr)); + + ret = dpseci_enable(dpseci, CMD_PRI_LOW, priv->token); + if (ret) { + PMD_INIT_LOG(ERR, "DPSECI with HW_ID = %d ENABLE FAILED\n", + priv->hw_id); + goto get_attr_failure; + } + ret = dpseci_get_attributes(dpseci, CMD_PRI_LOW, priv->token, &attr); + if (ret) { + PMD_INIT_LOG(ERR, + "DPSEC ATTRIBUTE READ FAILED, disabling DPSEC\n"); + goto get_attr_failure; + } + for (i = 0; i < attr.num_rx_queues && qp[i]; i++) { + dpaa2_q = &qp[i]->rx_vq; + dpseci_get_rx_queue(dpseci, CMD_PRI_LOW, priv->token, i, + &rx_attr); + dpaa2_q->fqid = rx_attr.fqid; + PMD_INIT_LOG(DEBUG, "rx_fqid: %d", dpaa2_q->fqid); + } + for (i = 0; i < attr.num_tx_queues && qp[i]; i++) { + dpaa2_q = &qp[i]->tx_vq; + dpseci_get_tx_queue(dpseci, CMD_PRI_LOW, priv->token, i, + &tx_attr); + dpaa2_q->fqid = tx_attr.fqid; + PMD_INIT_LOG(DEBUG, "tx_fqid: %d", dpaa2_q->fqid); + } + + return 0; +get_attr_failure: + dpseci_disable(dpseci, CMD_PRI_LOW, priv->token); + return -1; +} + +static void +dpaa2_sec_dev_stop(struct rte_cryptodev *dev) +{ + struct dpaa2_sec_dev_private *priv = dev->data->dev_private; + struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw; + int ret; + + PMD_INIT_FUNC_TRACE(); + + ret = dpseci_disable(dpseci, CMD_PRI_LOW, priv->token); + if (ret) { + PMD_INIT_LOG(ERR, "Failure in disabling dpseci %d device", + priv->hw_id); + return; + } + + ret = dpseci_reset(dpseci, CMD_PRI_LOW, priv->token); + if (ret < 0) { + PMD_INIT_LOG(ERR, "SEC Device cannot be reset:Error = %0x\n", + ret); + return; + } +} + +static int +dpaa2_sec_dev_close(struct rte_cryptodev *dev) +{ + struct dpaa2_sec_dev_private *priv = dev->data->dev_private; + struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw; + int ret; + + PMD_INIT_FUNC_TRACE(); + + /* Function is reverse of dpaa2_sec_dev_init. + * It does the following: + * 1. Detach a DPSECI from attached resources i.e. buffer pools, dpbp_id + * 2. Close the DPSECI device + * 3. Free the allocated resources. + */ + + /*Close the device at underlying layer*/ + ret = dpseci_close(dpseci, CMD_PRI_LOW, priv->token); + if (ret) { + PMD_INIT_LOG(ERR, "Failure closing dpseci device with" + " error code %d\n", ret); + return -1; + } + + /*Free the allocated memory for ethernet private data and dpseci*/ + priv->hw = NULL; + rte_free(dpseci); + + return 0; +} + +static void +dpaa2_sec_dev_infos_get(struct rte_cryptodev *dev, + struct rte_cryptodev_info *info) +{ + struct dpaa2_sec_dev_private *internals = dev->data->dev_private; + + PMD_INIT_FUNC_TRACE(); + if (info != NULL) { + info->max_nb_queue_pairs = internals->max_nb_queue_pairs; + info->feature_flags = dev->feature_flags; + info->capabilities = dpaa2_sec_capabilities; + info->sym.max_nb_sessions = internals->max_nb_sessions; + info->driver_id = cryptodev_driver_id; + } +} + +static +void dpaa2_sec_stats_get(struct rte_cryptodev *dev, + struct rte_cryptodev_stats *stats) +{ + struct dpaa2_sec_dev_private *priv = dev->data->dev_private; + struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw; + struct dpseci_sec_counters counters = {0}; + struct dpaa2_sec_qp **qp = (struct dpaa2_sec_qp **) + dev->data->queue_pairs; + int ret, i; + + PMD_INIT_FUNC_TRACE(); + if (stats == NULL) { + PMD_DRV_LOG(ERR, "invalid stats ptr NULL"); + return; + } + for (i = 0; i < dev->data->nb_queue_pairs; i++) { + if (qp[i] == NULL) { + PMD_DRV_LOG(DEBUG, "Uninitialised queue pair"); + continue; + } + + stats->enqueued_count += qp[i]->tx_vq.tx_pkts; + stats->dequeued_count += qp[i]->rx_vq.rx_pkts; + stats->enqueue_err_count += qp[i]->tx_vq.err_pkts; + stats->dequeue_err_count += qp[i]->rx_vq.err_pkts; + } + + ret = dpseci_get_sec_counters(dpseci, CMD_PRI_LOW, priv->token, + &counters); + if (ret) { + PMD_DRV_LOG(ERR, "dpseci_get_sec_counters failed\n"); + } else { + PMD_DRV_LOG(INFO, "dpseci hw stats:" + "\n\tNumber of Requests Dequeued = %lu" + "\n\tNumber of Outbound Encrypt Requests = %lu" + "\n\tNumber of Inbound Decrypt Requests = %lu" + "\n\tNumber of Outbound Bytes Encrypted = %lu" + "\n\tNumber of Outbound Bytes Protected = %lu" + "\n\tNumber of Inbound Bytes Decrypted = %lu" + "\n\tNumber of Inbound Bytes Validated = %lu", + counters.dequeued_requests, + counters.ob_enc_requests, + counters.ib_dec_requests, + counters.ob_enc_bytes, + counters.ob_prot_bytes, + counters.ib_dec_bytes, + counters.ib_valid_bytes); + } +} + +static +void dpaa2_sec_stats_reset(struct rte_cryptodev *dev) +{ + int i; + struct dpaa2_sec_qp **qp = (struct dpaa2_sec_qp **) + (dev->data->queue_pairs); + + PMD_INIT_FUNC_TRACE(); + + for (i = 0; i < dev->data->nb_queue_pairs; i++) { + if (qp[i] == NULL) { + PMD_DRV_LOG(DEBUG, "Uninitialised queue pair"); + continue; + } + qp[i]->tx_vq.rx_pkts = 0; + qp[i]->tx_vq.tx_pkts = 0; + qp[i]->tx_vq.err_pkts = 0; + qp[i]->rx_vq.rx_pkts = 0; + qp[i]->rx_vq.tx_pkts = 0; + qp[i]->rx_vq.err_pkts = 0; + } +} + +static struct rte_cryptodev_ops crypto_ops = { + .dev_configure = dpaa2_sec_dev_configure, + .dev_start = dpaa2_sec_dev_start, + .dev_stop = dpaa2_sec_dev_stop, + .dev_close = dpaa2_sec_dev_close, + .dev_infos_get = dpaa2_sec_dev_infos_get, + .stats_get = dpaa2_sec_stats_get, + .stats_reset = dpaa2_sec_stats_reset, + .queue_pair_setup = dpaa2_sec_queue_pair_setup, + .queue_pair_release = dpaa2_sec_queue_pair_release, + .queue_pair_start = dpaa2_sec_queue_pair_start, + .queue_pair_stop = dpaa2_sec_queue_pair_stop, + .queue_pair_count = dpaa2_sec_queue_pair_count, + .session_get_size = dpaa2_sec_session_get_size, + .session_configure = dpaa2_sec_session_configure, + .session_clear = dpaa2_sec_session_clear, +}; + +static const struct rte_security_capability * +dpaa2_sec_capabilities_get(void *device __rte_unused) +{ + return dpaa2_sec_security_cap; +} + +struct rte_security_ops dpaa2_sec_security_ops = { + .session_create = dpaa2_sec_security_session_create, + .session_update = NULL, + .session_stats_get = NULL, + .session_destroy = dpaa2_sec_security_session_destroy, + .set_pkt_metadata = NULL, + .capabilities_get = dpaa2_sec_capabilities_get +}; + +static int +dpaa2_sec_uninit(const struct rte_cryptodev *dev) +{ + struct dpaa2_sec_dev_private *internals = dev->data->dev_private; + + rte_free(dev->security_ctx); + + rte_mempool_free(internals->fle_pool); + + PMD_INIT_LOG(INFO, "Closing DPAA2_SEC device %s on numa socket %u\n", + dev->data->name, rte_socket_id()); + + return 0; +} + +static int +dpaa2_sec_dev_init(struct rte_cryptodev *cryptodev) +{ + struct dpaa2_sec_dev_private *internals; + struct rte_device *dev = cryptodev->device; + struct rte_dpaa2_device *dpaa2_dev; + struct rte_security_ctx *security_instance; + struct fsl_mc_io *dpseci; + uint16_t token; + struct dpseci_attr attr; + int retcode, hw_id; + char str[20]; + + PMD_INIT_FUNC_TRACE(); + dpaa2_dev = container_of(dev, struct rte_dpaa2_device, device); + if (dpaa2_dev == NULL) { + PMD_INIT_LOG(ERR, "dpaa2_device not found\n"); + return -1; + } + hw_id = dpaa2_dev->object_id; + + cryptodev->driver_id = cryptodev_driver_id; + cryptodev->dev_ops = &crypto_ops; + + cryptodev->enqueue_burst = dpaa2_sec_enqueue_burst; + cryptodev->dequeue_burst = dpaa2_sec_dequeue_burst; + cryptodev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | + RTE_CRYPTODEV_FF_HW_ACCELERATED | + RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | + RTE_CRYPTODEV_FF_SECURITY; + + internals = cryptodev->data->dev_private; + internals->max_nb_sessions = RTE_DPAA2_SEC_PMD_MAX_NB_SESSIONS; + + /* + * For secondary processes, we don't initialise any further as primary + * has already done this work. Only check we don't need a different + * RX function + */ + if (rte_eal_process_type() != RTE_PROC_PRIMARY) { + PMD_INIT_LOG(DEBUG, "Device already init by primary process"); + return 0; + } + + /* Initialize security_ctx only for primary process*/ + security_instance = rte_malloc("rte_security_instances_ops", + sizeof(struct rte_security_ctx), 0); + if (security_instance == NULL) + return -ENOMEM; + security_instance->device = (void *)cryptodev; + security_instance->ops = &dpaa2_sec_security_ops; + security_instance->sess_cnt = 0; + cryptodev->security_ctx = security_instance; + + /*Open the rte device via MC and save the handle for further use*/ + dpseci = (struct fsl_mc_io *)rte_calloc(NULL, 1, + sizeof(struct fsl_mc_io), 0); + if (!dpseci) { + PMD_INIT_LOG(ERR, + "Error in allocating the memory for dpsec object"); + return -1; + } + dpseci->regs = rte_mcp_ptr_list[0]; + + retcode = dpseci_open(dpseci, CMD_PRI_LOW, hw_id, &token); + if (retcode != 0) { + PMD_INIT_LOG(ERR, "Cannot open the dpsec device: Error = %x", + retcode); + goto init_error; + } + retcode = dpseci_get_attributes(dpseci, CMD_PRI_LOW, token, &attr); + if (retcode != 0) { + PMD_INIT_LOG(ERR, + "Cannot get dpsec device attributed: Error = %x", + retcode); + goto init_error; + } + sprintf(cryptodev->data->name, "dpsec-%u", hw_id); + + internals->max_nb_queue_pairs = attr.num_tx_queues; + cryptodev->data->nb_queue_pairs = internals->max_nb_queue_pairs; + internals->hw = dpseci; + internals->token = token; + + sprintf(str, "fle_pool_%d", cryptodev->data->dev_id); + internals->fle_pool = rte_mempool_create((const char *)str, + FLE_POOL_NUM_BUFS, + FLE_POOL_BUF_SIZE, + FLE_POOL_CACHE_SIZE, 0, + NULL, NULL, NULL, NULL, + SOCKET_ID_ANY, 0); + if (!internals->fle_pool) { + RTE_LOG(ERR, PMD, "%s create failed\n", str); + goto init_error; + } + + PMD_INIT_LOG(DEBUG, "driver %s: created\n", cryptodev->data->name); + return 0; + +init_error: + PMD_INIT_LOG(ERR, "driver %s: create failed\n", cryptodev->data->name); + + /* dpaa2_sec_uninit(crypto_dev_name); */ + return -EFAULT; +} + +static int +cryptodev_dpaa2_sec_probe(struct rte_dpaa2_driver *dpaa2_drv, + struct rte_dpaa2_device *dpaa2_dev) +{ + struct rte_cryptodev *cryptodev; + char cryptodev_name[RTE_CRYPTODEV_NAME_MAX_LEN]; + + int retval; + + sprintf(cryptodev_name, "dpsec-%d", dpaa2_dev->object_id); + + cryptodev = rte_cryptodev_pmd_allocate(cryptodev_name, rte_socket_id()); + if (cryptodev == NULL) + return -ENOMEM; + + if (rte_eal_process_type() == RTE_PROC_PRIMARY) { + cryptodev->data->dev_private = rte_zmalloc_socket( + "cryptodev private structure", + sizeof(struct dpaa2_sec_dev_private), + RTE_CACHE_LINE_SIZE, + rte_socket_id()); + + if (cryptodev->data->dev_private == NULL) + rte_panic("Cannot allocate memzone for private " + "device data"); + } + + dpaa2_dev->cryptodev = cryptodev; + cryptodev->device = &dpaa2_dev->device; + cryptodev->device->driver = &dpaa2_drv->driver; + + /* init user callbacks */ + TAILQ_INIT(&(cryptodev->link_intr_cbs)); + + /* Invoke PMD device initialization function */ + retval = dpaa2_sec_dev_init(cryptodev); + if (retval == 0) + return 0; + + if (rte_eal_process_type() == RTE_PROC_PRIMARY) + rte_free(cryptodev->data->dev_private); + + cryptodev->attached = RTE_CRYPTODEV_DETACHED; + + return -ENXIO; +} + +static int +cryptodev_dpaa2_sec_remove(struct rte_dpaa2_device *dpaa2_dev) +{ + struct rte_cryptodev *cryptodev; + int ret; + + cryptodev = dpaa2_dev->cryptodev; + if (cryptodev == NULL) + return -ENODEV; + + ret = dpaa2_sec_uninit(cryptodev); + if (ret) + return ret; + + return rte_cryptodev_pmd_destroy(cryptodev); +} + +static struct rte_dpaa2_driver rte_dpaa2_sec_driver = { + .drv_type = DPAA2_CRYPTO, + .driver = { + .name = "DPAA2 SEC PMD" + }, + .probe = cryptodev_dpaa2_sec_probe, + .remove = cryptodev_dpaa2_sec_remove, +}; + +static struct cryptodev_driver dpaa2_sec_crypto_drv; + +RTE_PMD_REGISTER_DPAA2(CRYPTODEV_NAME_DPAA2_SEC_PMD, rte_dpaa2_sec_driver); +RTE_PMD_REGISTER_CRYPTO_DRIVER(dpaa2_sec_crypto_drv, rte_dpaa2_sec_driver, + cryptodev_driver_id); diff --git a/dpdk/drivers/crypto/dpaa2_sec/dpaa2_sec_logs.h b/dpdk/drivers/crypto/dpaa2_sec/dpaa2_sec_logs.h new file mode 100644 index 00000000..b5e99f80 --- /dev/null +++ b/dpdk/drivers/crypto/dpaa2_sec/dpaa2_sec_logs.h @@ -0,0 +1,70 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. + * Copyright 2016 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Freescale Semiconductor, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _DPAA2_SEC_LOGS_H_ +#define _DPAA2_SEC_LOGS_H_ + +#define PMD_INIT_LOG(level, fmt, args...) \ + RTE_LOG(level, PMD, "%s(): " fmt "\n", __func__, ##args) + +#ifdef RTE_LIBRTE_DPAA2_SEC_DEBUG_INIT +#define PMD_INIT_FUNC_TRACE() PMD_INIT_LOG(DEBUG, " >>") +#else +#define PMD_INIT_FUNC_TRACE() do { } while (0) +#endif + +#ifdef RTE_LIBRTE_DPAA2_SEC_DEBUG_RX +#define PMD_RX_LOG(level, fmt, args...) \ + RTE_LOG(level, PMD, "%s(): " fmt "\n", __func__, ## args) +#else +#define PMD_RX_LOG(level, fmt, args...) do { } while (0) +#endif + +#ifdef RTE_LIBRTE_DPAA2_SEC_DEBUG_TX +#define PMD_TX_LOG(level, fmt, args...) \ + RTE_LOG(level, PMD, "%s(): " fmt "\n", __func__, ## args) +#else +#define PMD_TX_LOG(level, fmt, args...) do { } while (0) +#endif + +#ifdef RTE_LIBRTE_DPAA2_SEC_DEBUG_DRIVER +#define PMD_DRV_LOG_RAW(level, fmt, args...) \ + RTE_LOG(level, PMD, "%s(): " fmt, __func__, ## args) +#else +#define PMD_DRV_LOG_RAW(level, fmt, args...) do { } while (0) +#endif + +#define PMD_DRV_LOG(level, fmt, args...) \ + PMD_DRV_LOG_RAW(level, fmt "\n", ## args) + +#endif /* _DPAA2_SEC_LOGS_H_ */ diff --git a/dpdk/drivers/crypto/dpaa2_sec/dpaa2_sec_priv.h b/dpdk/drivers/crypto/dpaa2_sec/dpaa2_sec_priv.h new file mode 100644 index 00000000..8e583803 --- /dev/null +++ b/dpdk/drivers/crypto/dpaa2_sec/dpaa2_sec_priv.h @@ -0,0 +1,477 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. + * Copyright 2016 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Freescale Semiconductor, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_DPAA2_SEC_PMD_PRIVATE_H_ +#define _RTE_DPAA2_SEC_PMD_PRIVATE_H_ + +#define CRYPTODEV_NAME_DPAA2_SEC_PMD crypto_dpaa2_sec +/**< NXP DPAA2 - SEC PMD device name */ + +#define MAX_QUEUES 64 +#define MAX_DESC_SIZE 64 +/** private data structure for each DPAA2_SEC device */ +struct dpaa2_sec_dev_private { + void *mc_portal; /**< MC Portal for configuring this device */ + void *hw; /**< Hardware handle for this device.Used by NADK framework */ + struct rte_mempool *fle_pool; /* per device memory pool for FLE */ + int32_t hw_id; /**< An unique ID of this device instance */ + int32_t vfio_fd; /**< File descriptor received via VFIO */ + uint16_t token; /**< Token required by DPxxx objects */ + unsigned int max_nb_queue_pairs; + /**< Max number of queue pairs supported by device */ + unsigned int max_nb_sessions; + /**< Max number of sessions supported by device */ +}; + +struct dpaa2_sec_qp { + struct dpaa2_queue rx_vq; + struct dpaa2_queue tx_vq; +}; + +enum shr_desc_type { + DESC_UPDATE, + DESC_FINAL, + DESC_INITFINAL, +}; + +#define DIR_ENC 1 +#define DIR_DEC 0 + +#define DPAA2_SET_FLC_EWS(flc) (flc->word1_bits23_16 |= 0x1) +#define DPAA2_SET_FLC_RSC(flc) (flc->word1_bits31_24 |= 0x1) +#define DPAA2_SET_FLC_REUSE_BS(flc) (flc->mode_bits |= 0x8000) +#define DPAA2_SET_FLC_REUSE_FF(flc) (flc->mode_bits |= 0x2000) + +/* SEC Flow Context Descriptor */ +struct sec_flow_context { + /* word 0 */ + uint16_t word0_sdid; /* 11-0 SDID */ + uint16_t word0_res; /* 31-12 reserved */ + + /* word 1 */ + uint8_t word1_sdl; /* 5-0 SDL */ + /* 7-6 reserved */ + + uint8_t word1_bits_15_8; /* 11-8 CRID */ + /* 14-12 reserved */ + /* 15 CRJD */ + + uint8_t word1_bits23_16; /* 16 EWS */ + /* 17 DAC */ + /* 18,19,20 ? */ + /* 23-21 reserved */ + + uint8_t word1_bits31_24; /* 24 RSC */ + /* 25 RBMT */ + /* 31-26 reserved */ + + /* word 2 RFLC[31-0] */ + uint32_t word2_rflc_31_0; + + /* word 3 RFLC[63-32] */ + uint32_t word3_rflc_63_32; + + /* word 4 */ + uint16_t word4_iicid; /* 15-0 IICID */ + uint16_t word4_oicid; /* 31-16 OICID */ + + /* word 5 */ + uint32_t word5_ofqid:24; /* 23-0 OFQID */ + uint32_t word5_31_24:8; + /* 24 OSC */ + /* 25 OBMT */ + /* 29-26 reserved */ + /* 31-30 ICR */ + + /* word 6 */ + uint32_t word6_oflc_31_0; + + /* word 7 */ + uint32_t word7_oflc_63_32; + + /* Word 8-15 storage profiles */ + uint16_t dl; /**< DataLength(correction) */ + uint16_t reserved; /**< reserved */ + uint16_t dhr; /**< DataHeadRoom(correction) */ + uint16_t mode_bits; /**< mode bits */ + uint16_t bpv0; /**< buffer pool0 valid */ + uint16_t bpid0; /**< Bypass Memory Translation */ + uint16_t bpv1; /**< buffer pool1 valid */ + uint16_t bpid1; /**< Bypass Memory Translation */ + uint64_t word_12_15[2]; /**< word 12-15 are reserved */ +}; + +struct sec_flc_desc { + struct sec_flow_context flc; + uint32_t desc[MAX_DESC_SIZE]; +}; + +struct ctxt_priv { + struct rte_mempool *fle_pool; /* per device memory pool for FLE */ + struct sec_flc_desc flc_desc[0]; +}; + +enum dpaa2_sec_op_type { + DPAA2_SEC_NONE, /*!< No Cipher operations*/ + DPAA2_SEC_CIPHER,/*!< CIPHER operations */ + DPAA2_SEC_AUTH, /*!< Authentication Operations */ + DPAA2_SEC_AEAD, /*!< AEAD (AES-GCM/CCM) type operations */ + DPAA2_SEC_CIPHER_HASH, /*!< Authenticated Encryption with + * associated data + */ + DPAA2_SEC_HASH_CIPHER, /*!< Encryption with Authenticated + * associated data + */ + DPAA2_SEC_IPSEC, /*!< IPSEC protocol operations*/ + DPAA2_SEC_PDCP, /*!< PDCP protocol operations*/ + DPAA2_SEC_PKC, /*!< Public Key Cryptographic Operations */ + DPAA2_SEC_MAX +}; + +struct dpaa2_sec_aead_ctxt { + uint16_t auth_only_len; /*!< Length of data for Auth only */ + uint8_t auth_cipher_text; /**< Authenticate/cipher ordering */ +}; + +typedef struct dpaa2_sec_session_entry { + void *ctxt; + uint8_t ctxt_type; + uint8_t dir; /*!< Operation Direction */ + enum rte_crypto_cipher_algorithm cipher_alg; /*!< Cipher Algorithm*/ + enum rte_crypto_auth_algorithm auth_alg; /*!< Authentication Algorithm*/ + enum rte_crypto_aead_algorithm aead_alg; /*!< AEAD Algorithm*/ + union { + struct { + uint8_t *data; /**< pointer to key data */ + size_t length; /**< key length in bytes */ + } aead_key; + struct { + struct { + uint8_t *data; /**< pointer to key data */ + size_t length; /**< key length in bytes */ + } cipher_key; + struct { + uint8_t *data; /**< pointer to key data */ + size_t length; /**< key length in bytes */ + } auth_key; + }; + }; + struct { + uint16_t length; /**< IV length in bytes */ + uint16_t offset; /**< IV offset in bytes */ + } iv; + uint16_t digest_length; + uint8_t status; + union { + struct dpaa2_sec_aead_ctxt aead_ctxt; + } ext_params; +} dpaa2_sec_session; + +static const struct rte_cryptodev_capabilities dpaa2_sec_capabilities[] = { + { /* MD5 HMAC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_MD5_HMAC, + .block_size = 64, + .key_size = { + .min = 1, + .max = 64, + .increment = 1 + }, + .digest_size = { + .min = 16, + .max = 16, + .increment = 0 + }, + .iv_size = { 0 } + }, } + }, } + }, + { /* SHA1 HMAC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA1_HMAC, + .block_size = 64, + .key_size = { + .min = 1, + .max = 64, + .increment = 1 + }, + .digest_size = { + .min = 20, + .max = 20, + .increment = 0 + }, + .iv_size = { 0 } + }, } + }, } + }, + { /* SHA224 HMAC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA224_HMAC, + .block_size = 64, + .key_size = { + .min = 1, + .max = 64, + .increment = 1 + }, + .digest_size = { + .min = 28, + .max = 28, + .increment = 0 + }, + .iv_size = { 0 } + }, } + }, } + }, + { /* SHA256 HMAC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA256_HMAC, + .block_size = 64, + .key_size = { + .min = 1, + .max = 64, + .increment = 1 + }, + .digest_size = { + .min = 32, + .max = 32, + .increment = 0 + }, + .iv_size = { 0 } + }, } + }, } + }, + { /* SHA384 HMAC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA384_HMAC, + .block_size = 128, + .key_size = { + .min = 1, + .max = 128, + .increment = 1 + }, + .digest_size = { + .min = 48, + .max = 48, + .increment = 0 + }, + .iv_size = { 0 } + }, } + }, } + }, + { /* SHA512 HMAC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA512_HMAC, + .block_size = 128, + .key_size = { + .min = 1, + .max = 128, + .increment = 1 + }, + .digest_size = { + .min = 64, + .max = 64, + .increment = 0 + }, + .iv_size = { 0 } + }, } + }, } + }, + { /* AES GCM */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD, + {.aead = { + .algo = RTE_CRYPTO_AEAD_AES_GCM, + .block_size = 16, + .key_size = { + .min = 16, + .max = 32, + .increment = 8 + }, + .digest_size = { + .min = 8, + .max = 16, + .increment = 4 + }, + .aad_size = { + .min = 0, + .max = 240, + .increment = 1 + }, + .iv_size = { + .min = 12, + .max = 12, + .increment = 0 + }, + }, } + }, } + }, + { /* AES CBC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, + {.cipher = { + .algo = RTE_CRYPTO_CIPHER_AES_CBC, + .block_size = 16, + .key_size = { + .min = 16, + .max = 32, + .increment = 8 + }, + .iv_size = { + .min = 16, + .max = 16, + .increment = 0 + } + }, } + }, } + }, + { /* AES CTR */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, + {.cipher = { + .algo = RTE_CRYPTO_CIPHER_AES_CTR, + .block_size = 16, + .key_size = { + .min = 16, + .max = 32, + .increment = 8 + }, + .iv_size = { + .min = 16, + .max = 16, + .increment = 0 + }, + }, } + }, } + }, + { /* 3DES CBC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, + {.cipher = { + .algo = RTE_CRYPTO_CIPHER_3DES_CBC, + .block_size = 8, + .key_size = { + .min = 16, + .max = 24, + .increment = 8 + }, + .iv_size = { + .min = 8, + .max = 8, + .increment = 0 + } + }, } + }, } + }, + + RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST() +}; + +static const struct rte_security_capability dpaa2_sec_security_cap[] = { + { /* IPsec Lookaside Protocol offload ESP Transport Egress */ + .action = RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL, + .protocol = RTE_SECURITY_PROTOCOL_IPSEC, + .ipsec = { + .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP, + .mode = RTE_SECURITY_IPSEC_SA_MODE_TUNNEL, + .direction = RTE_SECURITY_IPSEC_SA_DIR_EGRESS, + .options = { 0 } + }, + .crypto_capabilities = dpaa2_sec_capabilities + }, + { /* IPsec Lookaside Protocol offload ESP Tunnel Ingress */ + .action = RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL, + .protocol = RTE_SECURITY_PROTOCOL_IPSEC, + .ipsec = { + .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP, + .mode = RTE_SECURITY_IPSEC_SA_MODE_TUNNEL, + .direction = RTE_SECURITY_IPSEC_SA_DIR_INGRESS, + .options = { 0 } + }, + .crypto_capabilities = dpaa2_sec_capabilities + }, + { + .action = RTE_SECURITY_ACTION_TYPE_NONE + } +}; + +/** + * Checksum + * + * @param buffer calculate chksum for buffer + * @param len buffer length + * + * @return checksum value in host cpu order + */ +static inline uint16_t +calc_chksum(void *buffer, int len) +{ + uint16_t *buf = (uint16_t *)buffer; + uint32_t sum = 0; + uint16_t result; + + for (sum = 0; len > 1; len -= 2) + sum += *buf++; + + if (len == 1) + sum += *(unsigned char *)buf; + + sum = (sum >> 16) + (sum & 0xFFFF); + sum += (sum >> 16); + result = ~sum; + + return result; +} + +#endif /* _RTE_DPAA2_SEC_PMD_PRIVATE_H_ */ diff --git a/dpdk/drivers/crypto/dpaa2_sec/hw/compat.h b/dpdk/drivers/crypto/dpaa2_sec/hw/compat.h new file mode 100644 index 00000000..b17b27ef --- /dev/null +++ b/dpdk/drivers/crypto/dpaa2_sec/hw/compat.h @@ -0,0 +1,159 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2013-2016 Freescale Semiconductor Inc. + * Copyright 2016 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __RTA_COMPAT_H__ +#define __RTA_COMPAT_H__ + +#include +#include + +#ifdef __GLIBC__ +#include +#include +#include +#include + +#include +#include + +#ifndef __BYTE_ORDER__ +#error "Undefined endianness" +#endif + +#else +#error Environment not supported! +#endif + +#ifndef __always_inline +#define __always_inline __rte_always_inline +#endif + +#ifndef __always_unused +#define __always_unused __attribute__((unused)) +#endif + +#ifndef __maybe_unused +#define __maybe_unused __attribute__((unused)) +#endif + +#if defined(__GLIBC__) && !defined(pr_debug) +#if !defined(SUPPRESS_PRINTS) && defined(RTA_DEBUG) +#define pr_debug(fmt, ...) \ + RTE_LOG(DEBUG, PMD, "%s(): " fmt "\n", __func__, ##__VA_ARGS__) +#else +#define pr_debug(fmt, ...) do { } while (0) +#endif +#endif /* pr_debug */ + +#if defined(__GLIBC__) && !defined(pr_err) +#if !defined(SUPPRESS_PRINTS) +#define pr_err(fmt, ...) \ + RTE_LOG(ERR, PMD, "%s(): " fmt "\n", __func__, ##__VA_ARGS__) +#else +#define pr_err(fmt, ...) do { } while (0) +#endif +#endif /* pr_err */ + +#if defined(__GLIBC__) && !defined(pr_warn) +#if !defined(SUPPRESS_PRINTS) +#define pr_warn(fmt, ...) \ + RTE_LOG(WARNING, PMD, "%s(): " fmt "\n", __func__, ##__VA_ARGS__) +#else +#define pr_warn(fmt, ...) do { } while (0) +#endif +#endif /* pr_warn */ + +/** + * ARRAY_SIZE - returns the number of elements in an array + * @x: array + */ +#ifndef ARRAY_SIZE +#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) +#endif + +#ifndef ALIGN +#define ALIGN(x, a) (((x) + ((__typeof__(x))(a) - 1)) & \ + ~((__typeof__(x))(a) - 1)) +#endif + +#ifndef BIT +#define BIT(nr) (1UL << (nr)) +#endif + +#ifndef upper_32_bits +/** + * upper_32_bits - return bits 32-63 of a number + * @n: the number we're accessing + */ +#define upper_32_bits(n) ((uint32_t)(((n) >> 16) >> 16)) +#endif + +#ifndef lower_32_bits +/** + * lower_32_bits - return bits 0-31 of a number + * @n: the number we're accessing + */ +#define lower_32_bits(n) ((uint32_t)(n)) +#endif + +/* Use Linux naming convention */ +#ifdef __GLIBC__ + #define swab16(x) rte_bswap16(x) + #define swab32(x) rte_bswap32(x) + #define swab64(x) rte_bswap64(x) + /* Define cpu_to_be32 macro if not defined in the build environment */ + #if !defined(cpu_to_be32) + #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + #define cpu_to_be32(x) (x) + #else + #define cpu_to_be32(x) swab32(x) + #endif + #endif + /* Define cpu_to_le32 macro if not defined in the build environment */ + #if !defined(cpu_to_le32) + #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + #define cpu_to_le32(x) swab32(x) + #else + #define cpu_to_le32(x) (x) + #endif + #endif +#endif + +#endif /* __RTA_COMPAT_H__ */ diff --git a/dpdk/drivers/crypto/dpaa2_sec/hw/desc.h b/dpdk/drivers/crypto/dpaa2_sec/hw/desc.h new file mode 100644 index 00000000..0be4439c --- /dev/null +++ b/dpdk/drivers/crypto/dpaa2_sec/hw/desc.h @@ -0,0 +1,2601 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2008-2016 Freescale Semiconductor Inc. + * Copyright 2016 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * SEC descriptor composition header. + * Definitions to support SEC descriptor instruction generation + */ + +#ifndef __RTA_DESC_H__ +#define __RTA_DESC_H__ + +/* hw/compat.h is not delivered in kernel */ +#ifndef __KERNEL__ +#include "hw/compat.h" +#endif + +/* Max size of any SEC descriptor in 32-bit words, inclusive of header */ +#define MAX_CAAM_DESCSIZE 64 + +#define CAAM_CMD_SZ sizeof(uint32_t) +#define CAAM_PTR_SZ sizeof(dma_addr_t) +#define CAAM_DESC_BYTES_MAX (CAAM_CMD_SZ * MAX_CAAM_DESCSIZE) +#define DESC_JOB_IO_LEN (CAAM_CMD_SZ * 5 + CAAM_PTR_SZ * 3) + +/* Block size of any entity covered/uncovered with a KEK/TKEK */ +#define KEK_BLOCKSIZE 16 + +/* + * Supported descriptor command types as they show up + * inside a descriptor command word. + */ +#define CMD_SHIFT 27 +#define CMD_MASK (0x1f << CMD_SHIFT) + +#define CMD_KEY (0x00 << CMD_SHIFT) +#define CMD_SEQ_KEY (0x01 << CMD_SHIFT) +#define CMD_LOAD (0x02 << CMD_SHIFT) +#define CMD_SEQ_LOAD (0x03 << CMD_SHIFT) +#define CMD_FIFO_LOAD (0x04 << CMD_SHIFT) +#define CMD_SEQ_FIFO_LOAD (0x05 << CMD_SHIFT) +#define CMD_MOVEDW (0x06 << CMD_SHIFT) +#define CMD_MOVEB (0x07 << CMD_SHIFT) +#define CMD_STORE (0x0a << CMD_SHIFT) +#define CMD_SEQ_STORE (0x0b << CMD_SHIFT) +#define CMD_FIFO_STORE (0x0c << CMD_SHIFT) +#define CMD_SEQ_FIFO_STORE (0x0d << CMD_SHIFT) +#define CMD_MOVE_LEN (0x0e << CMD_SHIFT) +#define CMD_MOVE (0x0f << CMD_SHIFT) +#define CMD_OPERATION ((uint32_t)(0x10 << CMD_SHIFT)) +#define CMD_SIGNATURE ((uint32_t)(0x12 << CMD_SHIFT)) +#define CMD_JUMP ((uint32_t)(0x14 << CMD_SHIFT)) +#define CMD_MATH ((uint32_t)(0x15 << CMD_SHIFT)) +#define CMD_DESC_HDR ((uint32_t)(0x16 << CMD_SHIFT)) +#define CMD_SHARED_DESC_HDR ((uint32_t)(0x17 << CMD_SHIFT)) +#define CMD_MATHI ((uint32_t)(0x1d << CMD_SHIFT)) +#define CMD_SEQ_IN_PTR ((uint32_t)(0x1e << CMD_SHIFT)) +#define CMD_SEQ_OUT_PTR ((uint32_t)(0x1f << CMD_SHIFT)) + +/* General-purpose class selector for all commands */ +#define CLASS_SHIFT 25 +#define CLASS_MASK (0x03 << CLASS_SHIFT) + +#define CLASS_NONE (0x00 << CLASS_SHIFT) +#define CLASS_1 (0x01 << CLASS_SHIFT) +#define CLASS_2 (0x02 << CLASS_SHIFT) +#define CLASS_BOTH (0x03 << CLASS_SHIFT) + +/* ICV Check bits for Algo Operation command */ +#define ICV_CHECK_DISABLE 0 +#define ICV_CHECK_ENABLE 1 + +/* Encap Mode check bits for Algo Operation command */ +#define DIR_ENC 1 +#define DIR_DEC 0 + +/* + * Descriptor header command constructs + * Covers shared, job, and trusted descriptor headers + */ + +/* + * Extended Job Descriptor Header + */ +#define HDR_EXT BIT(24) + +/* + * Read input frame as soon as possible (SHR HDR) + */ +#define HDR_RIF BIT(25) + +/* + * Require SEQ LIODN to be the Same (JOB HDR) + */ +#define HDR_RSLS BIT(25) + +/* + * Do Not Run - marks a descriptor not executable if there was + * a preceding error somewhere + */ +#define HDR_DNR BIT(24) + +/* + * ONE - should always be set. Combination of ONE (always + * set) and ZRO (always clear) forms an endianness sanity check + */ +#define HDR_ONE BIT(23) +#define HDR_ZRO BIT(15) + +/* Start Index or SharedDesc Length */ +#define HDR_START_IDX_SHIFT 16 +#define HDR_START_IDX_MASK (0x3f << HDR_START_IDX_SHIFT) + +/* If shared descriptor header, 6-bit length */ +#define HDR_DESCLEN_SHR_MASK 0x3f + +/* If non-shared header, 7-bit length */ +#define HDR_DESCLEN_MASK 0x7f + +/* This is a TrustedDesc (if not SharedDesc) */ +#define HDR_TRUSTED BIT(14) + +/* Make into TrustedDesc (if not SharedDesc) */ +#define HDR_MAKE_TRUSTED BIT(13) + +/* Clear Input FiFO (if SharedDesc) */ +#define HDR_CLEAR_IFIFO BIT(13) + +/* Save context if self-shared (if SharedDesc) */ +#define HDR_SAVECTX BIT(12) + +/* Next item points to SharedDesc */ +#define HDR_SHARED BIT(12) + +/* + * Reverse Execution Order - execute JobDesc first, then + * execute SharedDesc (normally SharedDesc goes first). + */ +#define HDR_REVERSE BIT(11) + +/* Propagate DNR property to SharedDesc */ +#define HDR_PROP_DNR BIT(11) + +/* DECO Select Valid */ +#define HDR_EXT_DSEL_VALID BIT(7) + +/* Fake trusted descriptor */ +#define HDR_EXT_FTD BIT(8) + +/* JobDesc/SharedDesc share property */ +#define HDR_SD_SHARE_SHIFT 8 +#define HDR_SD_SHARE_MASK (0x03 << HDR_SD_SHARE_SHIFT) +#define HDR_JD_SHARE_SHIFT 8 +#define HDR_JD_SHARE_MASK (0x07 << HDR_JD_SHARE_SHIFT) + +#define HDR_SHARE_NEVER (0x00 << HDR_SD_SHARE_SHIFT) +#define HDR_SHARE_WAIT (0x01 << HDR_SD_SHARE_SHIFT) +#define HDR_SHARE_SERIAL (0x02 << HDR_SD_SHARE_SHIFT) +#define HDR_SHARE_ALWAYS (0x03 << HDR_SD_SHARE_SHIFT) +#define HDR_SHARE_DEFER (0x04 << HDR_SD_SHARE_SHIFT) + +/* JobDesc/SharedDesc descriptor length */ +#define HDR_JD_LENGTH_MASK 0x7f +#define HDR_SD_LENGTH_MASK 0x3f + +/* + * KEY/SEQ_KEY Command Constructs + */ + +/* Key Destination Class: 01 = Class 1, 02 - Class 2 */ +#define KEY_DEST_CLASS_SHIFT 25 +#define KEY_DEST_CLASS_MASK (0x03 << KEY_DEST_CLASS_SHIFT) +#define KEY_DEST_CLASS1 (1 << KEY_DEST_CLASS_SHIFT) +#define KEY_DEST_CLASS2 (2 << KEY_DEST_CLASS_SHIFT) + +/* Scatter-Gather Table/Variable Length Field */ +#define KEY_SGF BIT(24) +#define KEY_VLF BIT(24) + +/* Immediate - Key follows command in the descriptor */ +#define KEY_IMM BIT(23) + +/* + * Already in Input Data FIFO - the Input Data Sequence is not read, since it is + * already in the Input Data FIFO. + */ +#define KEY_AIDF BIT(23) + +/* + * Encrypted - Key is encrypted either with the KEK, or + * with the TDKEK if this descriptor is trusted + */ +#define KEY_ENC BIT(22) + +/* + * No Write Back - Do not allow key to be FIFO STOREd + */ +#define KEY_NWB BIT(21) + +/* + * Enhanced Encryption of Key + */ +#define KEY_EKT BIT(20) + +/* + * Encrypted with Trusted Key + */ +#define KEY_TK BIT(15) + +/* + * Plaintext Store + */ +#define KEY_PTS BIT(14) + +/* + * KDEST - Key Destination: 0 - class key register, + * 1 - PKHA 'e', 2 - AFHA Sbox, 3 - MDHA split key + */ +#define KEY_DEST_SHIFT 16 +#define KEY_DEST_MASK (0x03 << KEY_DEST_SHIFT) + +#define KEY_DEST_CLASS_REG (0x00 << KEY_DEST_SHIFT) +#define KEY_DEST_PKHA_E (0x01 << KEY_DEST_SHIFT) +#define KEY_DEST_AFHA_SBOX (0x02 << KEY_DEST_SHIFT) +#define KEY_DEST_MDHA_SPLIT (0x03 << KEY_DEST_SHIFT) + +/* Length in bytes */ +#define KEY_LENGTH_MASK 0x000003ff + +/* + * LOAD/SEQ_LOAD/STORE/SEQ_STORE Command Constructs + */ + +/* + * Load/Store Destination: 0 = class independent CCB, + * 1 = class 1 CCB, 2 = class 2 CCB, 3 = DECO + */ +#define LDST_CLASS_SHIFT 25 +#define LDST_CLASS_MASK (0x03 << LDST_CLASS_SHIFT) +#define LDST_CLASS_IND_CCB (0x00 << LDST_CLASS_SHIFT) +#define LDST_CLASS_1_CCB (0x01 << LDST_CLASS_SHIFT) +#define LDST_CLASS_2_CCB (0x02 << LDST_CLASS_SHIFT) +#define LDST_CLASS_DECO (0x03 << LDST_CLASS_SHIFT) + +/* Scatter-Gather Table/Variable Length Field */ +#define LDST_SGF BIT(24) +#define LDST_VLF BIT(24) + +/* Immediate - Key follows this command in descriptor */ +#define LDST_IMM_MASK 1 +#define LDST_IMM_SHIFT 23 +#define LDST_IMM BIT(23) + +/* SRC/DST - Destination for LOAD, Source for STORE */ +#define LDST_SRCDST_SHIFT 16 +#define LDST_SRCDST_MASK (0x7f << LDST_SRCDST_SHIFT) + +#define LDST_SRCDST_BYTE_CONTEXT (0x20 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_BYTE_KEY (0x40 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_BYTE_INFIFO (0x7c << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_BYTE_OUTFIFO (0x7e << LDST_SRCDST_SHIFT) + +#define LDST_SRCDST_WORD_MODE_REG (0x00 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_DECO_JQCTRL (0x00 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_KEYSZ_REG (0x01 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_DECO_JQDAR (0x01 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_DATASZ_REG (0x02 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_DECO_STAT (0x02 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_ICVSZ_REG (0x03 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_BYTE_DCHKSM (0x03 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_PID (0x04 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_CHACTRL (0x06 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_DECOCTRL (0x06 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_IRQCTRL (0x07 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_DECO_PCLOVRD (0x07 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_CLRW (0x08 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_DECO_MATH0 (0x08 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_STAT (0x09 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_DECO_MATH1 (0x09 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_DECO_MATH2 (0x0a << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_DECO_AAD_SZ (0x0b << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_DECO_MATH3 (0x0b << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_CLASS1_IV_SZ (0x0c << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_ALTDS_CLASS1 (0x0f << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_PKHA_A_SZ (0x10 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_GTR (0x10 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_PKHA_B_SZ (0x11 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_PKHA_N_SZ (0x12 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_PKHA_E_SZ (0x13 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_CLASS_CTX (0x20 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_STR (0x20 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_DESCBUF (0x40 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_DESCBUF_JOB (0x41 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_DESCBUF_SHARED (0x42 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_DESCBUF_JOB_WE (0x45 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_DESCBUF_SHARED_WE (0x46 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_INFO_FIFO_SZL (0x70 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_INFO_FIFO_SZM (0x71 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_INFO_FIFO_L (0x72 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_INFO_FIFO_M (0x73 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_SZL (0x74 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_SZM (0x75 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_IFNSR (0x76 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_OFNSR (0x77 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_BYTE_ALTSOURCE (0x78 << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_INFO_FIFO (0x7a << LDST_SRCDST_SHIFT) + +/* Offset in source/destination */ +#define LDST_OFFSET_SHIFT 8 +#define LDST_OFFSET_MASK (0xff << LDST_OFFSET_SHIFT) + +/* LDOFF definitions used when DST = LDST_SRCDST_WORD_DECOCTRL */ +/* These could also be shifted by LDST_OFFSET_SHIFT - this reads better */ +#define LDOFF_CHG_SHARE_SHIFT 0 +#define LDOFF_CHG_SHARE_MASK (0x3 << LDOFF_CHG_SHARE_SHIFT) +#define LDOFF_CHG_SHARE_NEVER (0x1 << LDOFF_CHG_SHARE_SHIFT) +#define LDOFF_CHG_SHARE_OK_PROP (0x2 << LDOFF_CHG_SHARE_SHIFT) +#define LDOFF_CHG_SHARE_OK_NO_PROP (0x3 << LDOFF_CHG_SHARE_SHIFT) + +#define LDOFF_ENABLE_AUTO_NFIFO BIT(2) +#define LDOFF_DISABLE_AUTO_NFIFO BIT(3) + +#define LDOFF_CHG_NONSEQLIODN_SHIFT 4 +#define LDOFF_CHG_NONSEQLIODN_MASK (0x3 << LDOFF_CHG_NONSEQLIODN_SHIFT) +#define LDOFF_CHG_NONSEQLIODN_SEQ (0x1 << LDOFF_CHG_NONSEQLIODN_SHIFT) +#define LDOFF_CHG_NONSEQLIODN_NON_SEQ (0x2 << LDOFF_CHG_NONSEQLIODN_SHIFT) +#define LDOFF_CHG_NONSEQLIODN_TRUSTED (0x3 << LDOFF_CHG_NONSEQLIODN_SHIFT) + +#define LDOFF_CHG_SEQLIODN_SHIFT 6 +#define LDOFF_CHG_SEQLIODN_MASK (0x3 << LDOFF_CHG_SEQLIODN_SHIFT) +#define LDOFF_CHG_SEQLIODN_SEQ (0x1 << LDOFF_CHG_SEQLIODN_SHIFT) +#define LDOFF_CHG_SEQLIODN_NON_SEQ (0x2 << LDOFF_CHG_SEQLIODN_SHIFT) +#define LDOFF_CHG_SEQLIODN_TRUSTED (0x3 << LDOFF_CHG_SEQLIODN_SHIFT) + +/* Data length in bytes */ +#define LDST_LEN_SHIFT 0 +#define LDST_LEN_MASK (0xff << LDST_LEN_SHIFT) + +/* Special Length definitions when dst=deco-ctrl */ +#define LDLEN_ENABLE_OSL_COUNT BIT(7) +#define LDLEN_RST_CHA_OFIFO_PTR BIT(6) +#define LDLEN_RST_OFIFO BIT(5) +#define LDLEN_SET_OFIFO_OFF_VALID BIT(4) +#define LDLEN_SET_OFIFO_OFF_RSVD BIT(3) +#define LDLEN_SET_OFIFO_OFFSET_SHIFT 0 +#define LDLEN_SET_OFIFO_OFFSET_MASK (3 << LDLEN_SET_OFIFO_OFFSET_SHIFT) + +/* CCB Clear Written Register bits */ +#define CLRW_CLR_C1MODE BIT(0) +#define CLRW_CLR_C1DATAS BIT(2) +#define CLRW_CLR_C1ICV BIT(3) +#define CLRW_CLR_C1CTX BIT(5) +#define CLRW_CLR_C1KEY BIT(6) +#define CLRW_CLR_PK_A BIT(12) +#define CLRW_CLR_PK_B BIT(13) +#define CLRW_CLR_PK_N BIT(14) +#define CLRW_CLR_PK_E BIT(15) +#define CLRW_CLR_C2MODE BIT(16) +#define CLRW_CLR_C2KEYS BIT(17) +#define CLRW_CLR_C2DATAS BIT(18) +#define CLRW_CLR_C2CTX BIT(21) +#define CLRW_CLR_C2KEY BIT(22) +#define CLRW_RESET_CLS2_DONE BIT(26) /* era 4 */ +#define CLRW_RESET_CLS1_DONE BIT(27) /* era 4 */ +#define CLRW_RESET_CLS2_CHA BIT(28) /* era 4 */ +#define CLRW_RESET_CLS1_CHA BIT(29) /* era 4 */ +#define CLRW_RESET_OFIFO BIT(30) /* era 3 */ +#define CLRW_RESET_IFIFO_DFIFO BIT(31) /* era 3 */ + +/* CHA Control Register bits */ +#define CCTRL_RESET_CHA_ALL BIT(0) +#define CCTRL_RESET_CHA_AESA BIT(1) +#define CCTRL_RESET_CHA_DESA BIT(2) +#define CCTRL_RESET_CHA_AFHA BIT(3) +#define CCTRL_RESET_CHA_KFHA BIT(4) +#define CCTRL_RESET_CHA_SF8A BIT(5) +#define CCTRL_RESET_CHA_PKHA BIT(6) +#define CCTRL_RESET_CHA_MDHA BIT(7) +#define CCTRL_RESET_CHA_CRCA BIT(8) +#define CCTRL_RESET_CHA_RNG BIT(9) +#define CCTRL_RESET_CHA_SF9A BIT(10) +#define CCTRL_RESET_CHA_ZUCE BIT(11) +#define CCTRL_RESET_CHA_ZUCA BIT(12) +#define CCTRL_UNLOAD_PK_A0 BIT(16) +#define CCTRL_UNLOAD_PK_A1 BIT(17) +#define CCTRL_UNLOAD_PK_A2 BIT(18) +#define CCTRL_UNLOAD_PK_A3 BIT(19) +#define CCTRL_UNLOAD_PK_B0 BIT(20) +#define CCTRL_UNLOAD_PK_B1 BIT(21) +#define CCTRL_UNLOAD_PK_B2 BIT(22) +#define CCTRL_UNLOAD_PK_B3 BIT(23) +#define CCTRL_UNLOAD_PK_N BIT(24) +#define CCTRL_UNLOAD_PK_A BIT(26) +#define CCTRL_UNLOAD_PK_B BIT(27) +#define CCTRL_UNLOAD_SBOX BIT(28) + +/* IRQ Control Register (CxCIRQ) bits */ +#define CIRQ_ADI BIT(1) +#define CIRQ_DDI BIT(2) +#define CIRQ_RCDI BIT(3) +#define CIRQ_KDI BIT(4) +#define CIRQ_S8DI BIT(5) +#define CIRQ_PDI BIT(6) +#define CIRQ_MDI BIT(7) +#define CIRQ_CDI BIT(8) +#define CIRQ_RNDI BIT(9) +#define CIRQ_S9DI BIT(10) +#define CIRQ_ZEDI BIT(11) /* valid for Era 5 or higher */ +#define CIRQ_ZADI BIT(12) /* valid for Era 5 or higher */ +#define CIRQ_AEI BIT(17) +#define CIRQ_DEI BIT(18) +#define CIRQ_RCEI BIT(19) +#define CIRQ_KEI BIT(20) +#define CIRQ_S8EI BIT(21) +#define CIRQ_PEI BIT(22) +#define CIRQ_MEI BIT(23) +#define CIRQ_CEI BIT(24) +#define CIRQ_RNEI BIT(25) +#define CIRQ_S9EI BIT(26) +#define CIRQ_ZEEI BIT(27) /* valid for Era 5 or higher */ +#define CIRQ_ZAEI BIT(28) /* valid for Era 5 or higher */ + +/* + * FIFO_LOAD/FIFO_STORE/SEQ_FIFO_LOAD/SEQ_FIFO_STORE + * Command Constructs + */ + +/* + * Load Destination: 0 = skip (SEQ_FIFO_LOAD only), + * 1 = Load for Class1, 2 = Load for Class2, 3 = Load both + * Store Source: 0 = normal, 1 = Class1key, 2 = Class2key + */ +#define FIFOLD_CLASS_SHIFT 25 +#define FIFOLD_CLASS_MASK (0x03 << FIFOLD_CLASS_SHIFT) +#define FIFOLD_CLASS_SKIP (0x00 << FIFOLD_CLASS_SHIFT) +#define FIFOLD_CLASS_CLASS1 (0x01 << FIFOLD_CLASS_SHIFT) +#define FIFOLD_CLASS_CLASS2 (0x02 << FIFOLD_CLASS_SHIFT) +#define FIFOLD_CLASS_BOTH (0x03 << FIFOLD_CLASS_SHIFT) + +#define FIFOST_CLASS_SHIFT 25 +#define FIFOST_CLASS_MASK (0x03 << FIFOST_CLASS_SHIFT) +#define FIFOST_CLASS_NORMAL (0x00 << FIFOST_CLASS_SHIFT) +#define FIFOST_CLASS_CLASS1KEY (0x01 << FIFOST_CLASS_SHIFT) +#define FIFOST_CLASS_CLASS2KEY (0x02 << FIFOST_CLASS_SHIFT) +#define FIFOST_CLASS_BOTH (0x03 << FIFOST_CLASS_SHIFT) + +/* + * Scatter-Gather Table/Variable Length Field + * If set for FIFO_LOAD, refers to a SG table. Within + * SEQ_FIFO_LOAD, is variable input sequence + */ +#define FIFOLDST_SGF_SHIFT 24 +#define FIFOLDST_SGF_MASK (1 << FIFOLDST_SGF_SHIFT) +#define FIFOLDST_VLF_MASK (1 << FIFOLDST_SGF_SHIFT) +#define FIFOLDST_SGF BIT(24) +#define FIFOLDST_VLF BIT(24) + +/* + * Immediate - Data follows command in descriptor + * AIDF - Already in Input Data FIFO + */ +#define FIFOLD_IMM_SHIFT 23 +#define FIFOLD_IMM_MASK (1 << FIFOLD_IMM_SHIFT) +#define FIFOLD_AIDF_MASK (1 << FIFOLD_IMM_SHIFT) +#define FIFOLD_IMM BIT(23) +#define FIFOLD_AIDF BIT(23) + +#define FIFOST_IMM_SHIFT 23 +#define FIFOST_IMM_MASK (1 << FIFOST_IMM_SHIFT) +#define FIFOST_IMM BIT(23) + +/* Continue - Not the last FIFO store to come */ +#define FIFOST_CONT_SHIFT 23 +#define FIFOST_CONT_MASK (1 << FIFOST_CONT_SHIFT) +#define FIFOST_CONT BIT(23) + +/* + * Extended Length - use 32-bit extended length that + * follows the pointer field. Illegal with IMM set + */ +#define FIFOLDST_EXT_SHIFT 22 +#define FIFOLDST_EXT_MASK (1 << FIFOLDST_EXT_SHIFT) +#define FIFOLDST_EXT BIT(22) + +/* Input data type.*/ +#define FIFOLD_TYPE_SHIFT 16 +#define FIFOLD_CONT_TYPE_SHIFT 19 /* shift past last-flush bits */ +#define FIFOLD_TYPE_MASK (0x3f << FIFOLD_TYPE_SHIFT) + +/* PK types */ +#define FIFOLD_TYPE_PK (0x00 << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_PK_MASK (0x30 << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_PK_TYPEMASK (0x0f << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_PK_A0 (0x00 << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_PK_A1 (0x01 << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_PK_A2 (0x02 << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_PK_A3 (0x03 << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_PK_B0 (0x04 << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_PK_B1 (0x05 << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_PK_B2 (0x06 << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_PK_B3 (0x07 << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_PK_N (0x08 << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_PK_A (0x0c << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_PK_B (0x0d << FIFOLD_TYPE_SHIFT) + +/* Other types. Need to OR in last/flush bits as desired */ +#define FIFOLD_TYPE_MSG_MASK (0x38 << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_MSG (0x10 << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_MSG1OUT2 (0x18 << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_IV (0x20 << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_BITDATA (0x28 << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_AAD (0x30 << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_ICV (0x38 << FIFOLD_TYPE_SHIFT) + +/* Last/Flush bits for use with "other" types above */ +#define FIFOLD_TYPE_ACT_MASK (0x07 << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_NOACTION (0x00 << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_FLUSH1 (0x01 << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_LAST1 (0x02 << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_LAST2FLUSH (0x03 << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_LAST2 (0x04 << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_LAST2FLUSH1 (0x05 << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_LASTBOTH (0x06 << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_LASTBOTHFL (0x07 << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_NOINFOFIFO (0x0f << FIFOLD_TYPE_SHIFT) + +#define FIFOLDST_LEN_MASK 0xffff +#define FIFOLDST_EXT_LEN_MASK 0xffffffff + +/* Output data types */ +#define FIFOST_TYPE_SHIFT 16 +#define FIFOST_TYPE_MASK (0x3f << FIFOST_TYPE_SHIFT) + +#define FIFOST_TYPE_PKHA_A0 (0x00 << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_PKHA_A1 (0x01 << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_PKHA_A2 (0x02 << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_PKHA_A3 (0x03 << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_PKHA_B0 (0x04 << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_PKHA_B1 (0x05 << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_PKHA_B2 (0x06 << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_PKHA_B3 (0x07 << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_PKHA_N (0x08 << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_PKHA_A (0x0c << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_PKHA_B (0x0d << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_AF_SBOX_JKEK (0x20 << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_AF_SBOX_TKEK (0x21 << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_PKHA_E_JKEK (0x22 << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_PKHA_E_TKEK (0x23 << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_KEY_KEK (0x24 << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_KEY_TKEK (0x25 << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_SPLIT_KEK (0x26 << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_SPLIT_TKEK (0x27 << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_OUTFIFO_KEK (0x28 << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_OUTFIFO_TKEK (0x29 << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_MESSAGE_DATA (0x30 << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_MESSAGE_DATA2 (0x31 << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_RNGSTORE (0x34 << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_RNGFIFO (0x35 << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_METADATA (0x3e << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_SKIP (0x3f << FIFOST_TYPE_SHIFT) + +/* + * OPERATION Command Constructs + */ + +/* Operation type selectors - OP TYPE */ +#define OP_TYPE_SHIFT 24 +#define OP_TYPE_MASK (0x07 << OP_TYPE_SHIFT) + +#define OP_TYPE_UNI_PROTOCOL (0x00 << OP_TYPE_SHIFT) +#define OP_TYPE_PK (0x01 << OP_TYPE_SHIFT) +#define OP_TYPE_CLASS1_ALG (0x02 << OP_TYPE_SHIFT) +#define OP_TYPE_CLASS2_ALG (0x04 << OP_TYPE_SHIFT) +#define OP_TYPE_DECAP_PROTOCOL (0x06 << OP_TYPE_SHIFT) +#define OP_TYPE_ENCAP_PROTOCOL (0x07 << OP_TYPE_SHIFT) + +/* ProtocolID selectors - PROTID */ +#define OP_PCLID_SHIFT 16 +#define OP_PCLID_MASK (0xff << OP_PCLID_SHIFT) + +/* Assuming OP_TYPE = OP_TYPE_UNI_PROTOCOL */ +#define OP_PCLID_IKEV1_PRF (0x01 << OP_PCLID_SHIFT) +#define OP_PCLID_IKEV2_PRF (0x02 << OP_PCLID_SHIFT) +#define OP_PCLID_SSL30_PRF (0x08 << OP_PCLID_SHIFT) +#define OP_PCLID_TLS10_PRF (0x09 << OP_PCLID_SHIFT) +#define OP_PCLID_TLS11_PRF (0x0a << OP_PCLID_SHIFT) +#define OP_PCLID_TLS12_PRF (0x0b << OP_PCLID_SHIFT) +#define OP_PCLID_DTLS10_PRF (0x0c << OP_PCLID_SHIFT) +#define OP_PCLID_PUBLICKEYPAIR (0x14 << OP_PCLID_SHIFT) +#define OP_PCLID_DSASIGN (0x15 << OP_PCLID_SHIFT) +#define OP_PCLID_DSAVERIFY (0x16 << OP_PCLID_SHIFT) +#define OP_PCLID_DIFFIEHELLMAN (0x17 << OP_PCLID_SHIFT) +#define OP_PCLID_RSAENCRYPT (0x18 << OP_PCLID_SHIFT) +#define OP_PCLID_RSADECRYPT (0x19 << OP_PCLID_SHIFT) +#define OP_PCLID_DKP_MD5 (0x20 << OP_PCLID_SHIFT) +#define OP_PCLID_DKP_SHA1 (0x21 << OP_PCLID_SHIFT) +#define OP_PCLID_DKP_SHA224 (0x22 << OP_PCLID_SHIFT) +#define OP_PCLID_DKP_SHA256 (0x23 << OP_PCLID_SHIFT) +#define OP_PCLID_DKP_SHA384 (0x24 << OP_PCLID_SHIFT) +#define OP_PCLID_DKP_SHA512 (0x25 << OP_PCLID_SHIFT) + +/* Assuming OP_TYPE = OP_TYPE_DECAP_PROTOCOL/ENCAP_PROTOCOL */ +#define OP_PCLID_IPSEC (0x01 << OP_PCLID_SHIFT) +#define OP_PCLID_SRTP (0x02 << OP_PCLID_SHIFT) +#define OP_PCLID_MACSEC (0x03 << OP_PCLID_SHIFT) +#define OP_PCLID_WIFI (0x04 << OP_PCLID_SHIFT) +#define OP_PCLID_WIMAX (0x05 << OP_PCLID_SHIFT) +#define OP_PCLID_SSL30 (0x08 << OP_PCLID_SHIFT) +#define OP_PCLID_TLS10 (0x09 << OP_PCLID_SHIFT) +#define OP_PCLID_TLS11 (0x0a << OP_PCLID_SHIFT) +#define OP_PCLID_TLS12 (0x0b << OP_PCLID_SHIFT) +#define OP_PCLID_DTLS10 (0x0c << OP_PCLID_SHIFT) +#define OP_PCLID_BLOB (0x0d << OP_PCLID_SHIFT) +#define OP_PCLID_IPSEC_NEW (0x11 << OP_PCLID_SHIFT) +#define OP_PCLID_3G_DCRC (0x31 << OP_PCLID_SHIFT) +#define OP_PCLID_3G_RLC_PDU (0x32 << OP_PCLID_SHIFT) +#define OP_PCLID_3G_RLC_SDU (0x33 << OP_PCLID_SHIFT) +#define OP_PCLID_LTE_PDCP_USER (0x42 << OP_PCLID_SHIFT) +#define OP_PCLID_LTE_PDCP_CTRL (0x43 << OP_PCLID_SHIFT) +#define OP_PCLID_LTE_PDCP_CTRL_MIXED (0x44 << OP_PCLID_SHIFT) + +/* + * ProtocolInfo selectors + */ +#define OP_PCLINFO_MASK 0xffff + +/* for OP_PCLID_IPSEC */ +#define OP_PCL_IPSEC_CIPHER_MASK 0xff00 +#define OP_PCL_IPSEC_AUTH_MASK 0x00ff + +#define OP_PCL_IPSEC_DES_IV64 0x0100 +#define OP_PCL_IPSEC_DES 0x0200 +#define OP_PCL_IPSEC_3DES 0x0300 +#define OP_PCL_IPSEC_NULL 0x0B00 +#define OP_PCL_IPSEC_AES_CBC 0x0c00 +#define OP_PCL_IPSEC_AES_CTR 0x0d00 +#define OP_PCL_IPSEC_AES_XTS 0x1600 +#define OP_PCL_IPSEC_AES_CCM8 0x0e00 +#define OP_PCL_IPSEC_AES_CCM12 0x0f00 +#define OP_PCL_IPSEC_AES_CCM16 0x1000 +#define OP_PCL_IPSEC_AES_GCM8 0x1200 +#define OP_PCL_IPSEC_AES_GCM12 0x1300 +#define OP_PCL_IPSEC_AES_GCM16 0x1400 +#define OP_PCL_IPSEC_AES_NULL_WITH_GMAC 0x1500 + +#define OP_PCL_IPSEC_HMAC_NULL 0x0000 +#define OP_PCL_IPSEC_HMAC_MD5_96 0x0001 +#define OP_PCL_IPSEC_HMAC_SHA1_96 0x0002 +#define OP_PCL_IPSEC_AES_XCBC_MAC_96 0x0005 +#define OP_PCL_IPSEC_HMAC_MD5_128 0x0006 +#define OP_PCL_IPSEC_HMAC_SHA1_160 0x0007 +#define OP_PCL_IPSEC_AES_CMAC_96 0x0008 +#define OP_PCL_IPSEC_HMAC_SHA2_256_128 0x000c +#define OP_PCL_IPSEC_HMAC_SHA2_384_192 0x000d +#define OP_PCL_IPSEC_HMAC_SHA2_512_256 0x000e + +/* For SRTP - OP_PCLID_SRTP */ +#define OP_PCL_SRTP_CIPHER_MASK 0xff00 +#define OP_PCL_SRTP_AUTH_MASK 0x00ff + +#define OP_PCL_SRTP_AES_CTR 0x0d00 + +#define OP_PCL_SRTP_HMAC_SHA1_160 0x0007 + +/* For SSL 3.0 - OP_PCLID_SSL30 */ +#define OP_PCL_SSL30_AES_128_CBC_SHA 0x002f +#define OP_PCL_SSL30_AES_128_CBC_SHA_2 0x0030 +#define OP_PCL_SSL30_AES_128_CBC_SHA_3 0x0031 +#define OP_PCL_SSL30_AES_128_CBC_SHA_4 0x0032 +#define OP_PCL_SSL30_AES_128_CBC_SHA_5 0x0033 +#define OP_PCL_SSL30_AES_128_CBC_SHA_6 0x0034 +#define OP_PCL_SSL30_AES_128_CBC_SHA_7 0x008c +#define OP_PCL_SSL30_AES_128_CBC_SHA_8 0x0090 +#define OP_PCL_SSL30_AES_128_CBC_SHA_9 0x0094 +#define OP_PCL_SSL30_AES_128_CBC_SHA_10 0xc004 +#define OP_PCL_SSL30_AES_128_CBC_SHA_11 0xc009 +#define OP_PCL_SSL30_AES_128_CBC_SHA_12 0xc00e +#define OP_PCL_SSL30_AES_128_CBC_SHA_13 0xc013 +#define OP_PCL_SSL30_AES_128_CBC_SHA_14 0xc018 +#define OP_PCL_SSL30_AES_128_CBC_SHA_15 0xc01d +#define OP_PCL_SSL30_AES_128_CBC_SHA_16 0xc01e +#define OP_PCL_SSL30_AES_128_CBC_SHA_17 0xc01f + +#define OP_PCL_SSL30_AES_256_CBC_SHA 0x0035 +#define OP_PCL_SSL30_AES_256_CBC_SHA_2 0x0036 +#define OP_PCL_SSL30_AES_256_CBC_SHA_3 0x0037 +#define OP_PCL_SSL30_AES_256_CBC_SHA_4 0x0038 +#define OP_PCL_SSL30_AES_256_CBC_SHA_5 0x0039 +#define OP_PCL_SSL30_AES_256_CBC_SHA_6 0x003a +#define OP_PCL_SSL30_AES_256_CBC_SHA_7 0x008d +#define OP_PCL_SSL30_AES_256_CBC_SHA_8 0x0091 +#define OP_PCL_SSL30_AES_256_CBC_SHA_9 0x0095 +#define OP_PCL_SSL30_AES_256_CBC_SHA_10 0xc005 +#define OP_PCL_SSL30_AES_256_CBC_SHA_11 0xc00a +#define OP_PCL_SSL30_AES_256_CBC_SHA_12 0xc00f +#define OP_PCL_SSL30_AES_256_CBC_SHA_13 0xc014 +#define OP_PCL_SSL30_AES_256_CBC_SHA_14 0xc019 +#define OP_PCL_SSL30_AES_256_CBC_SHA_15 0xc020 +#define OP_PCL_SSL30_AES_256_CBC_SHA_16 0xc021 +#define OP_PCL_SSL30_AES_256_CBC_SHA_17 0xc022 + +#define OP_PCL_SSL30_AES_128_GCM_SHA256_1 0x009C +#define OP_PCL_SSL30_AES_256_GCM_SHA384_1 0x009D +#define OP_PCL_SSL30_AES_128_GCM_SHA256_2 0x009E +#define OP_PCL_SSL30_AES_256_GCM_SHA384_2 0x009F +#define OP_PCL_SSL30_AES_128_GCM_SHA256_3 0x00A0 +#define OP_PCL_SSL30_AES_256_GCM_SHA384_3 0x00A1 +#define OP_PCL_SSL30_AES_128_GCM_SHA256_4 0x00A2 +#define OP_PCL_SSL30_AES_256_GCM_SHA384_4 0x00A3 +#define OP_PCL_SSL30_AES_128_GCM_SHA256_5 0x00A4 +#define OP_PCL_SSL30_AES_256_GCM_SHA384_5 0x00A5 +#define OP_PCL_SSL30_AES_128_GCM_SHA256_6 0x00A6 + +#define OP_PCL_TLS_DH_ANON_AES_256_GCM_SHA384 0x00A7 +#define OP_PCL_TLS_PSK_AES_128_GCM_SHA256 0x00A8 +#define OP_PCL_TLS_PSK_AES_256_GCM_SHA384 0x00A9 +#define OP_PCL_TLS_DHE_PSK_AES_128_GCM_SHA256 0x00AA +#define OP_PCL_TLS_DHE_PSK_AES_256_GCM_SHA384 0x00AB +#define OP_PCL_TLS_RSA_PSK_AES_128_GCM_SHA256 0x00AC +#define OP_PCL_TLS_RSA_PSK_AES_256_GCM_SHA384 0x00AD +#define OP_PCL_TLS_PSK_AES_128_CBC_SHA256 0x00AE +#define OP_PCL_TLS_PSK_AES_256_CBC_SHA384 0x00AF +#define OP_PCL_TLS_DHE_PSK_AES_128_CBC_SHA256 0x00B2 +#define OP_PCL_TLS_DHE_PSK_AES_256_CBC_SHA384 0x00B3 +#define OP_PCL_TLS_RSA_PSK_AES_128_CBC_SHA256 0x00B6 +#define OP_PCL_TLS_RSA_PSK_AES_256_CBC_SHA384 0x00B7 + +#define OP_PCL_SSL30_3DES_EDE_CBC_MD5 0x0023 + +#define OP_PCL_SSL30_3DES_EDE_CBC_SHA 0x001f +#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_2 0x008b +#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_3 0x008f +#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_4 0x0093 +#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_5 0x000a +#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_6 0x000d +#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_7 0x0010 +#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_8 0x0013 +#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_9 0x0016 +#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_10 0x001b +#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_11 0xc003 +#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_12 0xc008 +#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_13 0xc00d +#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_14 0xc012 +#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_15 0xc017 +#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_16 0xc01a +#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_17 0xc01b +#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_18 0xc01c + +#define OP_PCL_SSL30_DES40_CBC_MD5 0x0029 + +#define OP_PCL_SSL30_DES_CBC_MD5 0x0022 + +#define OP_PCL_SSL30_DES40_CBC_SHA 0x0008 +#define OP_PCL_SSL30_DES40_CBC_SHA_2 0x000b +#define OP_PCL_SSL30_DES40_CBC_SHA_3 0x000e +#define OP_PCL_SSL30_DES40_CBC_SHA_4 0x0011 +#define OP_PCL_SSL30_DES40_CBC_SHA_5 0x0014 +#define OP_PCL_SSL30_DES40_CBC_SHA_6 0x0019 +#define OP_PCL_SSL30_DES40_CBC_SHA_7 0x0026 + +#define OP_PCL_SSL30_DES_CBC_SHA 0x001e +#define OP_PCL_SSL30_DES_CBC_SHA_2 0x0009 +#define OP_PCL_SSL30_DES_CBC_SHA_3 0x000c +#define OP_PCL_SSL30_DES_CBC_SHA_4 0x000f +#define OP_PCL_SSL30_DES_CBC_SHA_5 0x0012 +#define OP_PCL_SSL30_DES_CBC_SHA_6 0x0015 +#define OP_PCL_SSL30_DES_CBC_SHA_7 0x001a + +#define OP_PCL_SSL30_RC4_128_MD5 0x0024 +#define OP_PCL_SSL30_RC4_128_MD5_2 0x0004 +#define OP_PCL_SSL30_RC4_128_MD5_3 0x0018 + +#define OP_PCL_SSL30_RC4_40_MD5 0x002b +#define OP_PCL_SSL30_RC4_40_MD5_2 0x0003 +#define OP_PCL_SSL30_RC4_40_MD5_3 0x0017 + +#define OP_PCL_SSL30_RC4_128_SHA 0x0020 +#define OP_PCL_SSL30_RC4_128_SHA_2 0x008a +#define OP_PCL_SSL30_RC4_128_SHA_3 0x008e +#define OP_PCL_SSL30_RC4_128_SHA_4 0x0092 +#define OP_PCL_SSL30_RC4_128_SHA_5 0x0005 +#define OP_PCL_SSL30_RC4_128_SHA_6 0xc002 +#define OP_PCL_SSL30_RC4_128_SHA_7 0xc007 +#define OP_PCL_SSL30_RC4_128_SHA_8 0xc00c +#define OP_PCL_SSL30_RC4_128_SHA_9 0xc011 +#define OP_PCL_SSL30_RC4_128_SHA_10 0xc016 + +#define OP_PCL_SSL30_RC4_40_SHA 0x0028 + +/* For TLS 1.0 - OP_PCLID_TLS10 */ +#define OP_PCL_TLS10_AES_128_CBC_SHA 0x002f +#define OP_PCL_TLS10_AES_128_CBC_SHA_2 0x0030 +#define OP_PCL_TLS10_AES_128_CBC_SHA_3 0x0031 +#define OP_PCL_TLS10_AES_128_CBC_SHA_4 0x0032 +#define OP_PCL_TLS10_AES_128_CBC_SHA_5 0x0033 +#define OP_PCL_TLS10_AES_128_CBC_SHA_6 0x0034 +#define OP_PCL_TLS10_AES_128_CBC_SHA_7 0x008c +#define OP_PCL_TLS10_AES_128_CBC_SHA_8 0x0090 +#define OP_PCL_TLS10_AES_128_CBC_SHA_9 0x0094 +#define OP_PCL_TLS10_AES_128_CBC_SHA_10 0xc004 +#define OP_PCL_TLS10_AES_128_CBC_SHA_11 0xc009 +#define OP_PCL_TLS10_AES_128_CBC_SHA_12 0xc00e +#define OP_PCL_TLS10_AES_128_CBC_SHA_13 0xc013 +#define OP_PCL_TLS10_AES_128_CBC_SHA_14 0xc018 +#define OP_PCL_TLS10_AES_128_CBC_SHA_15 0xc01d +#define OP_PCL_TLS10_AES_128_CBC_SHA_16 0xc01e +#define OP_PCL_TLS10_AES_128_CBC_SHA_17 0xc01f + +#define OP_PCL_TLS10_AES_256_CBC_SHA 0x0035 +#define OP_PCL_TLS10_AES_256_CBC_SHA_2 0x0036 +#define OP_PCL_TLS10_AES_256_CBC_SHA_3 0x0037 +#define OP_PCL_TLS10_AES_256_CBC_SHA_4 0x0038 +#define OP_PCL_TLS10_AES_256_CBC_SHA_5 0x0039 +#define OP_PCL_TLS10_AES_256_CBC_SHA_6 0x003a +#define OP_PCL_TLS10_AES_256_CBC_SHA_7 0x008d +#define OP_PCL_TLS10_AES_256_CBC_SHA_8 0x0091 +#define OP_PCL_TLS10_AES_256_CBC_SHA_9 0x0095 +#define OP_PCL_TLS10_AES_256_CBC_SHA_10 0xc005 +#define OP_PCL_TLS10_AES_256_CBC_SHA_11 0xc00a +#define OP_PCL_TLS10_AES_256_CBC_SHA_12 0xc00f +#define OP_PCL_TLS10_AES_256_CBC_SHA_13 0xc014 +#define OP_PCL_TLS10_AES_256_CBC_SHA_14 0xc019 +#define OP_PCL_TLS10_AES_256_CBC_SHA_15 0xc020 +#define OP_PCL_TLS10_AES_256_CBC_SHA_16 0xc021 +#define OP_PCL_TLS10_AES_256_CBC_SHA_17 0xc022 + +#define OP_PCL_TLS_ECDHE_ECDSA_AES_128_CBC_SHA256 0xC023 +#define OP_PCL_TLS_ECDHE_ECDSA_AES_256_CBC_SHA384 0xC024 +#define OP_PCL_TLS_ECDH_ECDSA_AES_128_CBC_SHA256 0xC025 +#define OP_PCL_TLS_ECDH_ECDSA_AES_256_CBC_SHA384 0xC026 +#define OP_PCL_TLS_ECDHE_RSA_AES_128_CBC_SHA256 0xC027 +#define OP_PCL_TLS_ECDHE_RSA_AES_256_CBC_SHA384 0xC028 +#define OP_PCL_TLS_ECDH_RSA_AES_128_CBC_SHA256 0xC029 +#define OP_PCL_TLS_ECDH_RSA_AES_256_CBC_SHA384 0xC02A +#define OP_PCL_TLS_ECDHE_ECDSA_AES_128_GCM_SHA256 0xC02B +#define OP_PCL_TLS_ECDHE_ECDSA_AES_256_GCM_SHA384 0xC02C +#define OP_PCL_TLS_ECDH_ECDSA_AES_128_GCM_SHA256 0xC02D +#define OP_PCL_TLS_ECDH_ECDSA_AES_256_GCM_SHA384 0xC02E +#define OP_PCL_TLS_ECDHE_RSA_AES_128_GCM_SHA256 0xC02F +#define OP_PCL_TLS_ECDHE_RSA_AES_256_GCM_SHA384 0xC030 +#define OP_PCL_TLS_ECDH_RSA_AES_128_GCM_SHA256 0xC031 +#define OP_PCL_TLS_ECDH_RSA_AES_256_GCM_SHA384 0xC032 +#define OP_PCL_TLS_ECDHE_PSK_RC4_128_SHA 0xC033 +#define OP_PCL_TLS_ECDHE_PSK_3DES_EDE_CBC_SHA 0xC034 +#define OP_PCL_TLS_ECDHE_PSK_AES_128_CBC_SHA 0xC035 +#define OP_PCL_TLS_ECDHE_PSK_AES_256_CBC_SHA 0xC036 +#define OP_PCL_TLS_ECDHE_PSK_AES_128_CBC_SHA256 0xC037 +#define OP_PCL_TLS_ECDHE_PSK_AES_256_CBC_SHA384 0xC038 + +/* #define OP_PCL_TLS10_3DES_EDE_CBC_MD5 0x0023 */ + +#define OP_PCL_TLS10_3DES_EDE_CBC_SHA 0x001f +#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_2 0x008b +#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_3 0x008f +#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_4 0x0093 +#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_5 0x000a +#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_6 0x000d +#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_7 0x0010 +#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_8 0x0013 +#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_9 0x0016 +#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_10 0x001b +#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_11 0xc003 +#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_12 0xc008 +#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_13 0xc00d +#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_14 0xc012 +#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_15 0xc017 +#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_16 0xc01a +#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_17 0xc01b +#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_18 0xc01c + +#define OP_PCL_TLS10_DES40_CBC_MD5 0x0029 + +#define OP_PCL_TLS10_DES_CBC_MD5 0x0022 + +#define OP_PCL_TLS10_DES40_CBC_SHA 0x0008 +#define OP_PCL_TLS10_DES40_CBC_SHA_2 0x000b +#define OP_PCL_TLS10_DES40_CBC_SHA_3 0x000e +#define OP_PCL_TLS10_DES40_CBC_SHA_4 0x0011 +#define OP_PCL_TLS10_DES40_CBC_SHA_5 0x0014 +#define OP_PCL_TLS10_DES40_CBC_SHA_6 0x0019 +#define OP_PCL_TLS10_DES40_CBC_SHA_7 0x0026 + +#define OP_PCL_TLS10_DES_CBC_SHA 0x001e +#define OP_PCL_TLS10_DES_CBC_SHA_2 0x0009 +#define OP_PCL_TLS10_DES_CBC_SHA_3 0x000c +#define OP_PCL_TLS10_DES_CBC_SHA_4 0x000f +#define OP_PCL_TLS10_DES_CBC_SHA_5 0x0012 +#define OP_PCL_TLS10_DES_CBC_SHA_6 0x0015 +#define OP_PCL_TLS10_DES_CBC_SHA_7 0x001a + +#define OP_PCL_TLS10_RC4_128_MD5 0x0024 +#define OP_PCL_TLS10_RC4_128_MD5_2 0x0004 +#define OP_PCL_TLS10_RC4_128_MD5_3 0x0018 + +#define OP_PCL_TLS10_RC4_40_MD5 0x002b +#define OP_PCL_TLS10_RC4_40_MD5_2 0x0003 +#define OP_PCL_TLS10_RC4_40_MD5_3 0x0017 + +#define OP_PCL_TLS10_RC4_128_SHA 0x0020 +#define OP_PCL_TLS10_RC4_128_SHA_2 0x008a +#define OP_PCL_TLS10_RC4_128_SHA_3 0x008e +#define OP_PCL_TLS10_RC4_128_SHA_4 0x0092 +#define OP_PCL_TLS10_RC4_128_SHA_5 0x0005 +#define OP_PCL_TLS10_RC4_128_SHA_6 0xc002 +#define OP_PCL_TLS10_RC4_128_SHA_7 0xc007 +#define OP_PCL_TLS10_RC4_128_SHA_8 0xc00c +#define OP_PCL_TLS10_RC4_128_SHA_9 0xc011 +#define OP_PCL_TLS10_RC4_128_SHA_10 0xc016 + +#define OP_PCL_TLS10_RC4_40_SHA 0x0028 + +#define OP_PCL_TLS10_3DES_EDE_CBC_MD5 0xff23 +#define OP_PCL_TLS10_3DES_EDE_CBC_SHA160 0xff30 +#define OP_PCL_TLS10_3DES_EDE_CBC_SHA224 0xff34 +#define OP_PCL_TLS10_3DES_EDE_CBC_SHA256 0xff36 +#define OP_PCL_TLS10_3DES_EDE_CBC_SHA384 0xff33 +#define OP_PCL_TLS10_3DES_EDE_CBC_SHA512 0xff35 +#define OP_PCL_TLS10_AES_128_CBC_SHA160 0xff80 +#define OP_PCL_TLS10_AES_128_CBC_SHA224 0xff84 +#define OP_PCL_TLS10_AES_128_CBC_SHA256 0xff86 +#define OP_PCL_TLS10_AES_128_CBC_SHA384 0xff83 +#define OP_PCL_TLS10_AES_128_CBC_SHA512 0xff85 +#define OP_PCL_TLS10_AES_192_CBC_SHA160 0xff20 +#define OP_PCL_TLS10_AES_192_CBC_SHA224 0xff24 +#define OP_PCL_TLS10_AES_192_CBC_SHA256 0xff26 +#define OP_PCL_TLS10_AES_192_CBC_SHA384 0xff23 +#define OP_PCL_TLS10_AES_192_CBC_SHA512 0xff25 +#define OP_PCL_TLS10_AES_256_CBC_SHA160 0xff60 +#define OP_PCL_TLS10_AES_256_CBC_SHA224 0xff64 +#define OP_PCL_TLS10_AES_256_CBC_SHA256 0xff66 +#define OP_PCL_TLS10_AES_256_CBC_SHA384 0xff63 +#define OP_PCL_TLS10_AES_256_CBC_SHA512 0xff65 + +#define OP_PCL_TLS_PVT_AES_192_CBC_SHA160 0xff90 +#define OP_PCL_TLS_PVT_AES_192_CBC_SHA384 0xff93 +#define OP_PCL_TLS_PVT_AES_192_CBC_SHA224 0xff94 +#define OP_PCL_TLS_PVT_AES_192_CBC_SHA512 0xff95 +#define OP_PCL_TLS_PVT_AES_192_CBC_SHA256 0xff96 +#define OP_PCL_TLS_PVT_MASTER_SECRET_PRF_FE 0xfffe +#define OP_PCL_TLS_PVT_MASTER_SECRET_PRF_FF 0xffff + +/* For TLS 1.1 - OP_PCLID_TLS11 */ +#define OP_PCL_TLS11_AES_128_CBC_SHA 0x002f +#define OP_PCL_TLS11_AES_128_CBC_SHA_2 0x0030 +#define OP_PCL_TLS11_AES_128_CBC_SHA_3 0x0031 +#define OP_PCL_TLS11_AES_128_CBC_SHA_4 0x0032 +#define OP_PCL_TLS11_AES_128_CBC_SHA_5 0x0033 +#define OP_PCL_TLS11_AES_128_CBC_SHA_6 0x0034 +#define OP_PCL_TLS11_AES_128_CBC_SHA_7 0x008c +#define OP_PCL_TLS11_AES_128_CBC_SHA_8 0x0090 +#define OP_PCL_TLS11_AES_128_CBC_SHA_9 0x0094 +#define OP_PCL_TLS11_AES_128_CBC_SHA_10 0xc004 +#define OP_PCL_TLS11_AES_128_CBC_SHA_11 0xc009 +#define OP_PCL_TLS11_AES_128_CBC_SHA_12 0xc00e +#define OP_PCL_TLS11_AES_128_CBC_SHA_13 0xc013 +#define OP_PCL_TLS11_AES_128_CBC_SHA_14 0xc018 +#define OP_PCL_TLS11_AES_128_CBC_SHA_15 0xc01d +#define OP_PCL_TLS11_AES_128_CBC_SHA_16 0xc01e +#define OP_PCL_TLS11_AES_128_CBC_SHA_17 0xc01f + +#define OP_PCL_TLS11_AES_256_CBC_SHA 0x0035 +#define OP_PCL_TLS11_AES_256_CBC_SHA_2 0x0036 +#define OP_PCL_TLS11_AES_256_CBC_SHA_3 0x0037 +#define OP_PCL_TLS11_AES_256_CBC_SHA_4 0x0038 +#define OP_PCL_TLS11_AES_256_CBC_SHA_5 0x0039 +#define OP_PCL_TLS11_AES_256_CBC_SHA_6 0x003a +#define OP_PCL_TLS11_AES_256_CBC_SHA_7 0x008d +#define OP_PCL_TLS11_AES_256_CBC_SHA_8 0x0091 +#define OP_PCL_TLS11_AES_256_CBC_SHA_9 0x0095 +#define OP_PCL_TLS11_AES_256_CBC_SHA_10 0xc005 +#define OP_PCL_TLS11_AES_256_CBC_SHA_11 0xc00a +#define OP_PCL_TLS11_AES_256_CBC_SHA_12 0xc00f +#define OP_PCL_TLS11_AES_256_CBC_SHA_13 0xc014 +#define OP_PCL_TLS11_AES_256_CBC_SHA_14 0xc019 +#define OP_PCL_TLS11_AES_256_CBC_SHA_15 0xc020 +#define OP_PCL_TLS11_AES_256_CBC_SHA_16 0xc021 +#define OP_PCL_TLS11_AES_256_CBC_SHA_17 0xc022 + +/* #define OP_PCL_TLS11_3DES_EDE_CBC_MD5 0x0023 */ + +#define OP_PCL_TLS11_3DES_EDE_CBC_SHA 0x001f +#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_2 0x008b +#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_3 0x008f +#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_4 0x0093 +#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_5 0x000a +#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_6 0x000d +#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_7 0x0010 +#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_8 0x0013 +#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_9 0x0016 +#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_10 0x001b +#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_11 0xc003 +#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_12 0xc008 +#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_13 0xc00d +#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_14 0xc012 +#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_15 0xc017 +#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_16 0xc01a +#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_17 0xc01b +#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_18 0xc01c + +#define OP_PCL_TLS11_DES40_CBC_MD5 0x0029 + +#define OP_PCL_TLS11_DES_CBC_MD5 0x0022 + +#define OP_PCL_TLS11_DES40_CBC_SHA 0x0008 +#define OP_PCL_TLS11_DES40_CBC_SHA_2 0x000b +#define OP_PCL_TLS11_DES40_CBC_SHA_3 0x000e +#define OP_PCL_TLS11_DES40_CBC_SHA_4 0x0011 +#define OP_PCL_TLS11_DES40_CBC_SHA_5 0x0014 +#define OP_PCL_TLS11_DES40_CBC_SHA_6 0x0019 +#define OP_PCL_TLS11_DES40_CBC_SHA_7 0x0026 + +#define OP_PCL_TLS11_DES_CBC_SHA 0x001e +#define OP_PCL_TLS11_DES_CBC_SHA_2 0x0009 +#define OP_PCL_TLS11_DES_CBC_SHA_3 0x000c +#define OP_PCL_TLS11_DES_CBC_SHA_4 0x000f +#define OP_PCL_TLS11_DES_CBC_SHA_5 0x0012 +#define OP_PCL_TLS11_DES_CBC_SHA_6 0x0015 +#define OP_PCL_TLS11_DES_CBC_SHA_7 0x001a + +#define OP_PCL_TLS11_RC4_128_MD5 0x0024 +#define OP_PCL_TLS11_RC4_128_MD5_2 0x0004 +#define OP_PCL_TLS11_RC4_128_MD5_3 0x0018 + +#define OP_PCL_TLS11_RC4_40_MD5 0x002b +#define OP_PCL_TLS11_RC4_40_MD5_2 0x0003 +#define OP_PCL_TLS11_RC4_40_MD5_3 0x0017 + +#define OP_PCL_TLS11_RC4_128_SHA 0x0020 +#define OP_PCL_TLS11_RC4_128_SHA_2 0x008a +#define OP_PCL_TLS11_RC4_128_SHA_3 0x008e +#define OP_PCL_TLS11_RC4_128_SHA_4 0x0092 +#define OP_PCL_TLS11_RC4_128_SHA_5 0x0005 +#define OP_PCL_TLS11_RC4_128_SHA_6 0xc002 +#define OP_PCL_TLS11_RC4_128_SHA_7 0xc007 +#define OP_PCL_TLS11_RC4_128_SHA_8 0xc00c +#define OP_PCL_TLS11_RC4_128_SHA_9 0xc011 +#define OP_PCL_TLS11_RC4_128_SHA_10 0xc016 + +#define OP_PCL_TLS11_RC4_40_SHA 0x0028 + +#define OP_PCL_TLS11_3DES_EDE_CBC_MD5 0xff23 +#define OP_PCL_TLS11_3DES_EDE_CBC_SHA160 0xff30 +#define OP_PCL_TLS11_3DES_EDE_CBC_SHA224 0xff34 +#define OP_PCL_TLS11_3DES_EDE_CBC_SHA256 0xff36 +#define OP_PCL_TLS11_3DES_EDE_CBC_SHA384 0xff33 +#define OP_PCL_TLS11_3DES_EDE_CBC_SHA512 0xff35 +#define OP_PCL_TLS11_AES_128_CBC_SHA160 0xff80 +#define OP_PCL_TLS11_AES_128_CBC_SHA224 0xff84 +#define OP_PCL_TLS11_AES_128_CBC_SHA256 0xff86 +#define OP_PCL_TLS11_AES_128_CBC_SHA384 0xff83 +#define OP_PCL_TLS11_AES_128_CBC_SHA512 0xff85 +#define OP_PCL_TLS11_AES_192_CBC_SHA160 0xff20 +#define OP_PCL_TLS11_AES_192_CBC_SHA224 0xff24 +#define OP_PCL_TLS11_AES_192_CBC_SHA256 0xff26 +#define OP_PCL_TLS11_AES_192_CBC_SHA384 0xff23 +#define OP_PCL_TLS11_AES_192_CBC_SHA512 0xff25 +#define OP_PCL_TLS11_AES_256_CBC_SHA160 0xff60 +#define OP_PCL_TLS11_AES_256_CBC_SHA224 0xff64 +#define OP_PCL_TLS11_AES_256_CBC_SHA256 0xff66 +#define OP_PCL_TLS11_AES_256_CBC_SHA384 0xff63 +#define OP_PCL_TLS11_AES_256_CBC_SHA512 0xff65 + + +/* For TLS 1.2 - OP_PCLID_TLS12 */ +#define OP_PCL_TLS12_AES_128_CBC_SHA 0x002f +#define OP_PCL_TLS12_AES_128_CBC_SHA_2 0x0030 +#define OP_PCL_TLS12_AES_128_CBC_SHA_3 0x0031 +#define OP_PCL_TLS12_AES_128_CBC_SHA_4 0x0032 +#define OP_PCL_TLS12_AES_128_CBC_SHA_5 0x0033 +#define OP_PCL_TLS12_AES_128_CBC_SHA_6 0x0034 +#define OP_PCL_TLS12_AES_128_CBC_SHA_7 0x008c +#define OP_PCL_TLS12_AES_128_CBC_SHA_8 0x0090 +#define OP_PCL_TLS12_AES_128_CBC_SHA_9 0x0094 +#define OP_PCL_TLS12_AES_128_CBC_SHA_10 0xc004 +#define OP_PCL_TLS12_AES_128_CBC_SHA_11 0xc009 +#define OP_PCL_TLS12_AES_128_CBC_SHA_12 0xc00e +#define OP_PCL_TLS12_AES_128_CBC_SHA_13 0xc013 +#define OP_PCL_TLS12_AES_128_CBC_SHA_14 0xc018 +#define OP_PCL_TLS12_AES_128_CBC_SHA_15 0xc01d +#define OP_PCL_TLS12_AES_128_CBC_SHA_16 0xc01e +#define OP_PCL_TLS12_AES_128_CBC_SHA_17 0xc01f + +#define OP_PCL_TLS12_AES_256_CBC_SHA 0x0035 +#define OP_PCL_TLS12_AES_256_CBC_SHA_2 0x0036 +#define OP_PCL_TLS12_AES_256_CBC_SHA_3 0x0037 +#define OP_PCL_TLS12_AES_256_CBC_SHA_4 0x0038 +#define OP_PCL_TLS12_AES_256_CBC_SHA_5 0x0039 +#define OP_PCL_TLS12_AES_256_CBC_SHA_6 0x003a +#define OP_PCL_TLS12_AES_256_CBC_SHA_7 0x008d +#define OP_PCL_TLS12_AES_256_CBC_SHA_8 0x0091 +#define OP_PCL_TLS12_AES_256_CBC_SHA_9 0x0095 +#define OP_PCL_TLS12_AES_256_CBC_SHA_10 0xc005 +#define OP_PCL_TLS12_AES_256_CBC_SHA_11 0xc00a +#define OP_PCL_TLS12_AES_256_CBC_SHA_12 0xc00f +#define OP_PCL_TLS12_AES_256_CBC_SHA_13 0xc014 +#define OP_PCL_TLS12_AES_256_CBC_SHA_14 0xc019 +#define OP_PCL_TLS12_AES_256_CBC_SHA_15 0xc020 +#define OP_PCL_TLS12_AES_256_CBC_SHA_16 0xc021 +#define OP_PCL_TLS12_AES_256_CBC_SHA_17 0xc022 + +/* #define OP_PCL_TLS12_3DES_EDE_CBC_MD5 0x0023 */ + +#define OP_PCL_TLS12_3DES_EDE_CBC_SHA 0x001f +#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_2 0x008b +#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_3 0x008f +#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_4 0x0093 +#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_5 0x000a +#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_6 0x000d +#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_7 0x0010 +#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_8 0x0013 +#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_9 0x0016 +#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_10 0x001b +#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_11 0xc003 +#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_12 0xc008 +#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_13 0xc00d +#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_14 0xc012 +#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_15 0xc017 +#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_16 0xc01a +#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_17 0xc01b +#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_18 0xc01c + +#define OP_PCL_TLS12_DES40_CBC_MD5 0x0029 + +#define OP_PCL_TLS12_DES_CBC_MD5 0x0022 + +#define OP_PCL_TLS12_DES40_CBC_SHA 0x0008 +#define OP_PCL_TLS12_DES40_CBC_SHA_2 0x000b +#define OP_PCL_TLS12_DES40_CBC_SHA_3 0x000e +#define OP_PCL_TLS12_DES40_CBC_SHA_4 0x0011 +#define OP_PCL_TLS12_DES40_CBC_SHA_5 0x0014 +#define OP_PCL_TLS12_DES40_CBC_SHA_6 0x0019 +#define OP_PCL_TLS12_DES40_CBC_SHA_7 0x0026 + +#define OP_PCL_TLS12_DES_CBC_SHA 0x001e +#define OP_PCL_TLS12_DES_CBC_SHA_2 0x0009 +#define OP_PCL_TLS12_DES_CBC_SHA_3 0x000c +#define OP_PCL_TLS12_DES_CBC_SHA_4 0x000f +#define OP_PCL_TLS12_DES_CBC_SHA_5 0x0012 +#define OP_PCL_TLS12_DES_CBC_SHA_6 0x0015 +#define OP_PCL_TLS12_DES_CBC_SHA_7 0x001a + +#define OP_PCL_TLS12_RC4_128_MD5 0x0024 +#define OP_PCL_TLS12_RC4_128_MD5_2 0x0004 +#define OP_PCL_TLS12_RC4_128_MD5_3 0x0018 + +#define OP_PCL_TLS12_RC4_40_MD5 0x002b +#define OP_PCL_TLS12_RC4_40_MD5_2 0x0003 +#define OP_PCL_TLS12_RC4_40_MD5_3 0x0017 + +#define OP_PCL_TLS12_RC4_128_SHA 0x0020 +#define OP_PCL_TLS12_RC4_128_SHA_2 0x008a +#define OP_PCL_TLS12_RC4_128_SHA_3 0x008e +#define OP_PCL_TLS12_RC4_128_SHA_4 0x0092 +#define OP_PCL_TLS12_RC4_128_SHA_5 0x0005 +#define OP_PCL_TLS12_RC4_128_SHA_6 0xc002 +#define OP_PCL_TLS12_RC4_128_SHA_7 0xc007 +#define OP_PCL_TLS12_RC4_128_SHA_8 0xc00c +#define OP_PCL_TLS12_RC4_128_SHA_9 0xc011 +#define OP_PCL_TLS12_RC4_128_SHA_10 0xc016 + +#define OP_PCL_TLS12_RC4_40_SHA 0x0028 + +/* #define OP_PCL_TLS12_AES_128_CBC_SHA256 0x003c */ +#define OP_PCL_TLS12_AES_128_CBC_SHA256_2 0x003e +#define OP_PCL_TLS12_AES_128_CBC_SHA256_3 0x003f +#define OP_PCL_TLS12_AES_128_CBC_SHA256_4 0x0040 +#define OP_PCL_TLS12_AES_128_CBC_SHA256_5 0x0067 +#define OP_PCL_TLS12_AES_128_CBC_SHA256_6 0x006c + +/* #define OP_PCL_TLS12_AES_256_CBC_SHA256 0x003d */ +#define OP_PCL_TLS12_AES_256_CBC_SHA256_2 0x0068 +#define OP_PCL_TLS12_AES_256_CBC_SHA256_3 0x0069 +#define OP_PCL_TLS12_AES_256_CBC_SHA256_4 0x006a +#define OP_PCL_TLS12_AES_256_CBC_SHA256_5 0x006b +#define OP_PCL_TLS12_AES_256_CBC_SHA256_6 0x006d + +/* AEAD_AES_xxx_CCM/GCM remain to be defined... */ + +#define OP_PCL_TLS12_3DES_EDE_CBC_MD5 0xff23 +#define OP_PCL_TLS12_3DES_EDE_CBC_SHA160 0xff30 +#define OP_PCL_TLS12_3DES_EDE_CBC_SHA224 0xff34 +#define OP_PCL_TLS12_3DES_EDE_CBC_SHA256 0xff36 +#define OP_PCL_TLS12_3DES_EDE_CBC_SHA384 0xff33 +#define OP_PCL_TLS12_3DES_EDE_CBC_SHA512 0xff35 +#define OP_PCL_TLS12_AES_128_CBC_SHA160 0xff80 +#define OP_PCL_TLS12_AES_128_CBC_SHA224 0xff84 +#define OP_PCL_TLS12_AES_128_CBC_SHA256 0xff86 +#define OP_PCL_TLS12_AES_128_CBC_SHA384 0xff83 +#define OP_PCL_TLS12_AES_128_CBC_SHA512 0xff85 +#define OP_PCL_TLS12_AES_192_CBC_SHA160 0xff20 +#define OP_PCL_TLS12_AES_192_CBC_SHA224 0xff24 +#define OP_PCL_TLS12_AES_192_CBC_SHA256 0xff26 +#define OP_PCL_TLS12_AES_192_CBC_SHA384 0xff23 +#define OP_PCL_TLS12_AES_192_CBC_SHA512 0xff25 +#define OP_PCL_TLS12_AES_256_CBC_SHA160 0xff60 +#define OP_PCL_TLS12_AES_256_CBC_SHA224 0xff64 +#define OP_PCL_TLS12_AES_256_CBC_SHA256 0xff66 +#define OP_PCL_TLS12_AES_256_CBC_SHA384 0xff63 +#define OP_PCL_TLS12_AES_256_CBC_SHA512 0xff65 + +/* For DTLS - OP_PCLID_DTLS */ + +#define OP_PCL_DTLS_AES_128_CBC_SHA 0x002f +#define OP_PCL_DTLS_AES_128_CBC_SHA_2 0x0030 +#define OP_PCL_DTLS_AES_128_CBC_SHA_3 0x0031 +#define OP_PCL_DTLS_AES_128_CBC_SHA_4 0x0032 +#define OP_PCL_DTLS_AES_128_CBC_SHA_5 0x0033 +#define OP_PCL_DTLS_AES_128_CBC_SHA_6 0x0034 +#define OP_PCL_DTLS_AES_128_CBC_SHA_7 0x008c +#define OP_PCL_DTLS_AES_128_CBC_SHA_8 0x0090 +#define OP_PCL_DTLS_AES_128_CBC_SHA_9 0x0094 +#define OP_PCL_DTLS_AES_128_CBC_SHA_10 0xc004 +#define OP_PCL_DTLS_AES_128_CBC_SHA_11 0xc009 +#define OP_PCL_DTLS_AES_128_CBC_SHA_12 0xc00e +#define OP_PCL_DTLS_AES_128_CBC_SHA_13 0xc013 +#define OP_PCL_DTLS_AES_128_CBC_SHA_14 0xc018 +#define OP_PCL_DTLS_AES_128_CBC_SHA_15 0xc01d +#define OP_PCL_DTLS_AES_128_CBC_SHA_16 0xc01e +#define OP_PCL_DTLS_AES_128_CBC_SHA_17 0xc01f + +#define OP_PCL_DTLS_AES_256_CBC_SHA 0x0035 +#define OP_PCL_DTLS_AES_256_CBC_SHA_2 0x0036 +#define OP_PCL_DTLS_AES_256_CBC_SHA_3 0x0037 +#define OP_PCL_DTLS_AES_256_CBC_SHA_4 0x0038 +#define OP_PCL_DTLS_AES_256_CBC_SHA_5 0x0039 +#define OP_PCL_DTLS_AES_256_CBC_SHA_6 0x003a +#define OP_PCL_DTLS_AES_256_CBC_SHA_7 0x008d +#define OP_PCL_DTLS_AES_256_CBC_SHA_8 0x0091 +#define OP_PCL_DTLS_AES_256_CBC_SHA_9 0x0095 +#define OP_PCL_DTLS_AES_256_CBC_SHA_10 0xc005 +#define OP_PCL_DTLS_AES_256_CBC_SHA_11 0xc00a +#define OP_PCL_DTLS_AES_256_CBC_SHA_12 0xc00f +#define OP_PCL_DTLS_AES_256_CBC_SHA_13 0xc014 +#define OP_PCL_DTLS_AES_256_CBC_SHA_14 0xc019 +#define OP_PCL_DTLS_AES_256_CBC_SHA_15 0xc020 +#define OP_PCL_DTLS_AES_256_CBC_SHA_16 0xc021 +#define OP_PCL_DTLS_AES_256_CBC_SHA_17 0xc022 + +/* #define OP_PCL_DTLS_3DES_EDE_CBC_MD5 0x0023 */ + +#define OP_PCL_DTLS_3DES_EDE_CBC_SHA 0x001f +#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_2 0x008b +#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_3 0x008f +#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_4 0x0093 +#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_5 0x000a +#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_6 0x000d +#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_7 0x0010 +#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_8 0x0013 +#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_9 0x0016 +#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_10 0x001b +#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_11 0xc003 +#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_12 0xc008 +#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_13 0xc00d +#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_14 0xc012 +#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_15 0xc017 +#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_16 0xc01a +#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_17 0xc01b +#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_18 0xc01c + +#define OP_PCL_DTLS_DES40_CBC_MD5 0x0029 + +#define OP_PCL_DTLS_DES_CBC_MD5 0x0022 + +#define OP_PCL_DTLS_DES40_CBC_SHA 0x0008 +#define OP_PCL_DTLS_DES40_CBC_SHA_2 0x000b +#define OP_PCL_DTLS_DES40_CBC_SHA_3 0x000e +#define OP_PCL_DTLS_DES40_CBC_SHA_4 0x0011 +#define OP_PCL_DTLS_DES40_CBC_SHA_5 0x0014 +#define OP_PCL_DTLS_DES40_CBC_SHA_6 0x0019 +#define OP_PCL_DTLS_DES40_CBC_SHA_7 0x0026 + + +#define OP_PCL_DTLS_DES_CBC_SHA 0x001e +#define OP_PCL_DTLS_DES_CBC_SHA_2 0x0009 +#define OP_PCL_DTLS_DES_CBC_SHA_3 0x000c +#define OP_PCL_DTLS_DES_CBC_SHA_4 0x000f +#define OP_PCL_DTLS_DES_CBC_SHA_5 0x0012 +#define OP_PCL_DTLS_DES_CBC_SHA_6 0x0015 +#define OP_PCL_DTLS_DES_CBC_SHA_7 0x001a + +#define OP_PCL_DTLS_3DES_EDE_CBC_MD5 0xff23 +#define OP_PCL_DTLS_3DES_EDE_CBC_SHA160 0xff30 +#define OP_PCL_DTLS_3DES_EDE_CBC_SHA224 0xff34 +#define OP_PCL_DTLS_3DES_EDE_CBC_SHA256 0xff36 +#define OP_PCL_DTLS_3DES_EDE_CBC_SHA384 0xff33 +#define OP_PCL_DTLS_3DES_EDE_CBC_SHA512 0xff35 +#define OP_PCL_DTLS_AES_128_CBC_SHA160 0xff80 +#define OP_PCL_DTLS_AES_128_CBC_SHA224 0xff84 +#define OP_PCL_DTLS_AES_128_CBC_SHA256 0xff86 +#define OP_PCL_DTLS_AES_128_CBC_SHA384 0xff83 +#define OP_PCL_DTLS_AES_128_CBC_SHA512 0xff85 +#define OP_PCL_DTLS_AES_192_CBC_SHA160 0xff20 +#define OP_PCL_DTLS_AES_192_CBC_SHA224 0xff24 +#define OP_PCL_DTLS_AES_192_CBC_SHA256 0xff26 +#define OP_PCL_DTLS_AES_192_CBC_SHA384 0xff23 +#define OP_PCL_DTLS_AES_192_CBC_SHA512 0xff25 +#define OP_PCL_DTLS_AES_256_CBC_SHA160 0xff60 +#define OP_PCL_DTLS_AES_256_CBC_SHA224 0xff64 +#define OP_PCL_DTLS_AES_256_CBC_SHA256 0xff66 +#define OP_PCL_DTLS_AES_256_CBC_SHA384 0xff63 +#define OP_PCL_DTLS_AES_256_CBC_SHA512 0xff65 + +/* 802.16 WiMAX protinfos */ +#define OP_PCL_WIMAX_OFDM 0x0201 +#define OP_PCL_WIMAX_OFDMA 0x0231 + +/* 802.11 WiFi protinfos */ +#define OP_PCL_WIFI 0xac04 + +/* MacSec protinfos */ +#define OP_PCL_MACSEC 0x0001 + +/* 3G DCRC protinfos */ +#define OP_PCL_3G_DCRC_CRC7 0x0710 +#define OP_PCL_3G_DCRC_CRC11 0x0B10 + +/* 3G RLC protinfos */ +#define OP_PCL_3G_RLC_NULL 0x0000 +#define OP_PCL_3G_RLC_KASUMI 0x0001 +#define OP_PCL_3G_RLC_SNOW 0x0002 + +/* LTE protinfos */ +#define OP_PCL_LTE_NULL 0x0000 +#define OP_PCL_LTE_SNOW 0x0001 +#define OP_PCL_LTE_AES 0x0002 +#define OP_PCL_LTE_ZUC 0x0003 + +/* LTE mixed protinfos */ +#define OP_PCL_LTE_MIXED_AUTH_SHIFT 0 +#define OP_PCL_LTE_MIXED_AUTH_MASK (3 << OP_PCL_LTE_MIXED_AUTH_SHIFT) +#define OP_PCL_LTE_MIXED_ENC_SHIFT 8 +#define OP_PCL_LTE_MIXED_ENC_MASK (3 < OP_PCL_LTE_MIXED_ENC_SHIFT) +#define OP_PCL_LTE_MIXED_AUTH_NULL (OP_PCL_LTE_NULL << \ + OP_PCL_LTE_MIXED_AUTH_SHIFT) +#define OP_PCL_LTE_MIXED_AUTH_SNOW (OP_PCL_LTE_SNOW << \ + OP_PCL_LTE_MIXED_AUTH_SHIFT) +#define OP_PCL_LTE_MIXED_AUTH_AES (OP_PCL_LTE_AES << \ + OP_PCL_LTE_MIXED_AUTH_SHIFT) +#define OP_PCL_LTE_MIXED_AUTH_ZUC (OP_PCL_LTE_ZUC << \ + OP_PCL_LTE_MIXED_AUTH_SHIFT) +#define OP_PCL_LTE_MIXED_ENC_NULL (OP_PCL_LTE_NULL << \ + OP_PCL_LTE_MIXED_ENC_SHIFT) +#define OP_PCL_LTE_MIXED_ENC_SNOW (OP_PCL_LTE_SNOW << \ + OP_PCL_LTE_MIXED_ENC_SHIFT) +#define OP_PCL_LTE_MIXED_ENC_AES (OP_PCL_LTE_AES << \ + OP_PCL_LTE_MIXED_ENC_SHIFT) +#define OP_PCL_LTE_MIXED_ENC_ZUC (OP_PCL_LTE_ZUC << \ + OP_PCL_LTE_MIXED_ENC_SHIFT) + +/* PKI unidirectional protocol protinfo bits */ +#define OP_PCL_PKPROT_DSA_MSG BIT(10) +#define OP_PCL_PKPROT_HASH_SHIFT 7 +#define OP_PCL_PKPROT_HASH_MASK (7 << OP_PCL_PKPROT_HASH_SHIFT) +#define OP_PCL_PKPROT_HASH_MD5 (0 << OP_PCL_PKPROT_HASH_SHIFT) +#define OP_PCL_PKPROT_HASH_SHA1 (1 << OP_PCL_PKPROT_HASH_SHIFT) +#define OP_PCL_PKPROT_HASH_SHA224 (2 << OP_PCL_PKPROT_HASH_SHIFT) +#define OP_PCL_PKPROT_HASH_SHA256 (3 << OP_PCL_PKPROT_HASH_SHIFT) +#define OP_PCL_PKPROT_HASH_SHA384 (4 << OP_PCL_PKPROT_HASH_SHIFT) +#define OP_PCL_PKPROT_HASH_SHA512 (5 << OP_PCL_PKPROT_HASH_SHIFT) +#define OP_PCL_PKPROT_EKT_Z BIT(6) +#define OP_PCL_PKPROT_DECRYPT_Z BIT(5) +#define OP_PCL_PKPROT_EKT_PRI BIT(4) +#define OP_PCL_PKPROT_TEST BIT(3) +#define OP_PCL_PKPROT_DECRYPT_PRI BIT(2) +#define OP_PCL_PKPROT_ECC BIT(1) +#define OP_PCL_PKPROT_F2M BIT(0) + +/* Blob protinfos */ +#define OP_PCL_BLOB_TKEK_SHIFT 9 +#define OP_PCL_BLOB_TKEK BIT(9) +#define OP_PCL_BLOB_EKT_SHIFT 8 +#define OP_PCL_BLOB_EKT BIT(8) +#define OP_PCL_BLOB_REG_SHIFT 4 +#define OP_PCL_BLOB_REG_MASK (0xF << OP_PCL_BLOB_REG_SHIFT) +#define OP_PCL_BLOB_REG_MEMORY (0x0 << OP_PCL_BLOB_REG_SHIFT) +#define OP_PCL_BLOB_REG_KEY1 (0x1 << OP_PCL_BLOB_REG_SHIFT) +#define OP_PCL_BLOB_REG_KEY2 (0x3 << OP_PCL_BLOB_REG_SHIFT) +#define OP_PCL_BLOB_AFHA_SBOX (0x5 << OP_PCL_BLOB_REG_SHIFT) +#define OP_PCL_BLOB_REG_SPLIT (0x7 << OP_PCL_BLOB_REG_SHIFT) +#define OP_PCL_BLOB_REG_PKE (0x9 << OP_PCL_BLOB_REG_SHIFT) +#define OP_PCL_BLOB_SEC_MEM_SHIFT 3 +#define OP_PCL_BLOB_SEC_MEM BIT(3) +#define OP_PCL_BLOB_BLACK BIT(2) +#define OP_PCL_BLOB_FORMAT_SHIFT 0 +#define OP_PCL_BLOB_FORMAT_MASK 0x3 +#define OP_PCL_BLOB_FORMAT_NORMAL 0 +#define OP_PCL_BLOB_FORMAT_MASTER_VER 2 +#define OP_PCL_BLOB_FORMAT_TEST 3 + +/* IKE / IKEv2 protinfos */ +#define OP_PCL_IKE_HMAC_MD5 0x0100 +#define OP_PCL_IKE_HMAC_SHA1 0x0200 +#define OP_PCL_IKE_HMAC_AES128_CBC 0x0400 +#define OP_PCL_IKE_HMAC_SHA256 0x0500 +#define OP_PCL_IKE_HMAC_SHA384 0x0600 +#define OP_PCL_IKE_HMAC_SHA512 0x0700 +#define OP_PCL_IKE_HMAC_AES128_CMAC 0x0800 + +/* PKI unidirectional protocol protinfo bits */ +#define OP_PCL_PKPROT_TEST BIT(3) +#define OP_PCL_PKPROT_DECRYPT BIT(2) +#define OP_PCL_PKPROT_ECC BIT(1) +#define OP_PCL_PKPROT_F2M BIT(0) + +/* RSA Protinfo */ +#define OP_PCL_RSAPROT_OP_MASK 3 +#define OP_PCL_RSAPROT_OP_ENC_F_IN 0 +#define OP_PCL_RSAPROT_OP_ENC_F_OUT 1 +#define OP_PCL_RSAPROT_OP_DEC_ND 0 +#define OP_PCL_RSAPROT_OP_DEC_PQD 1 +#define OP_PCL_RSAPROT_OP_DEC_PQDPDQC 2 +#define OP_PCL_RSAPROT_FFF_SHIFT 4 +#define OP_PCL_RSAPROT_FFF_MASK (7 << OP_PCL_RSAPROT_FFF_SHIFT) +#define OP_PCL_RSAPROT_FFF_RED (0 << OP_PCL_RSAPROT_FFF_SHIFT) +#define OP_PCL_RSAPROT_FFF_ENC (1 << OP_PCL_RSAPROT_FFF_SHIFT) +#define OP_PCL_RSAPROT_FFF_TK_ENC (5 << OP_PCL_RSAPROT_FFF_SHIFT) +#define OP_PCL_RSAPROT_FFF_EKT (3 << OP_PCL_RSAPROT_FFF_SHIFT) +#define OP_PCL_RSAPROT_FFF_TK_EKT (7 << OP_PCL_RSAPROT_FFF_SHIFT) +#define OP_PCL_RSAPROT_PPP_SHIFT 8 +#define OP_PCL_RSAPROT_PPP_MASK (7 << OP_PCL_RSAPROT_PPP_SHIFT) +#define OP_PCL_RSAPROT_PPP_RED (0 << OP_PCL_RSAPROT_PPP_SHIFT) +#define OP_PCL_RSAPROT_PPP_ENC (1 << OP_PCL_RSAPROT_PPP_SHIFT) +#define OP_PCL_RSAPROT_PPP_TK_ENC (5 << OP_PCL_RSAPROT_PPP_SHIFT) +#define OP_PCL_RSAPROT_PPP_EKT (3 << OP_PCL_RSAPROT_PPP_SHIFT) +#define OP_PCL_RSAPROT_PPP_TK_EKT (7 << OP_PCL_RSAPROT_PPP_SHIFT) +#define OP_PCL_RSAPROT_FMT_PKCSV15 BIT(12) + +/* Derived Key Protocol (DKP) Protinfo */ +#define OP_PCL_DKP_SRC_SHIFT 14 +#define OP_PCL_DKP_SRC_MASK (3 << OP_PCL_DKP_SRC_SHIFT) +#define OP_PCL_DKP_SRC_IMM (0 << OP_PCL_DKP_SRC_SHIFT) +#define OP_PCL_DKP_SRC_SEQ (1 << OP_PCL_DKP_SRC_SHIFT) +#define OP_PCL_DKP_SRC_PTR (2 << OP_PCL_DKP_SRC_SHIFT) +#define OP_PCL_DKP_SRC_SGF (3 << OP_PCL_DKP_SRC_SHIFT) +#define OP_PCL_DKP_DST_SHIFT 12 +#define OP_PCL_DKP_DST_MASK (3 << OP_PCL_DKP_DST_SHIFT) +#define OP_PCL_DKP_DST_IMM (0 << OP_PCL_DKP_DST_SHIFT) +#define OP_PCL_DKP_DST_SEQ (1 << OP_PCL_DKP_DST_SHIFT) +#define OP_PCL_DKP_DST_PTR (2 << OP_PCL_DKP_DST_SHIFT) +#define OP_PCL_DKP_DST_SGF (3 << OP_PCL_DKP_DST_SHIFT) +#define OP_PCL_DKP_KEY_SHIFT 0 +#define OP_PCL_DKP_KEY_MASK (0xfff << OP_PCL_DKP_KEY_SHIFT) + +/* For non-protocol/alg-only op commands */ +#define OP_ALG_TYPE_SHIFT 24 +#define OP_ALG_TYPE_MASK (0x7 << OP_ALG_TYPE_SHIFT) +#define OP_ALG_TYPE_CLASS1 (0x2 << OP_ALG_TYPE_SHIFT) +#define OP_ALG_TYPE_CLASS2 (0x4 << OP_ALG_TYPE_SHIFT) + +#define OP_ALG_ALGSEL_SHIFT 16 +#define OP_ALG_ALGSEL_MASK (0xff << OP_ALG_ALGSEL_SHIFT) +#define OP_ALG_ALGSEL_SUBMASK (0x0f << OP_ALG_ALGSEL_SHIFT) +#define OP_ALG_ALGSEL_AES (0x10 << OP_ALG_ALGSEL_SHIFT) +#define OP_ALG_ALGSEL_DES (0x20 << OP_ALG_ALGSEL_SHIFT) +#define OP_ALG_ALGSEL_3DES (0x21 << OP_ALG_ALGSEL_SHIFT) +#define OP_ALG_ALGSEL_ARC4 (0x30 << OP_ALG_ALGSEL_SHIFT) +#define OP_ALG_ALGSEL_MD5 (0x40 << OP_ALG_ALGSEL_SHIFT) +#define OP_ALG_ALGSEL_SHA1 (0x41 << OP_ALG_ALGSEL_SHIFT) +#define OP_ALG_ALGSEL_SHA224 (0x42 << OP_ALG_ALGSEL_SHIFT) +#define OP_ALG_ALGSEL_SHA256 (0x43 << OP_ALG_ALGSEL_SHIFT) +#define OP_ALG_ALGSEL_SHA384 (0x44 << OP_ALG_ALGSEL_SHIFT) +#define OP_ALG_ALGSEL_SHA512 (0x45 << OP_ALG_ALGSEL_SHIFT) +#define OP_ALG_ALGSEL_RNG (0x50 << OP_ALG_ALGSEL_SHIFT) +#define OP_ALG_ALGSEL_SNOW_F8 (0x60 << OP_ALG_ALGSEL_SHIFT) +#define OP_ALG_ALGSEL_KASUMI (0x70 << OP_ALG_ALGSEL_SHIFT) +#define OP_ALG_ALGSEL_CRC (0x90 << OP_ALG_ALGSEL_SHIFT) +#define OP_ALG_ALGSEL_SNOW_F9 (0xA0 << OP_ALG_ALGSEL_SHIFT) +#define OP_ALG_ALGSEL_ZUCE (0xB0 << OP_ALG_ALGSEL_SHIFT) +#define OP_ALG_ALGSEL_ZUCA (0xC0 << OP_ALG_ALGSEL_SHIFT) + +#define OP_ALG_AAI_SHIFT 4 +#define OP_ALG_AAI_MASK (0x3ff << OP_ALG_AAI_SHIFT) + +/* block cipher AAI set */ +#define OP_ALG_AESA_MODE_MASK (0xF0 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_CTR (0x00 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_CTR_MOD128 (0x00 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_CTR_MOD8 (0x01 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_CTR_MOD16 (0x02 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_CTR_MOD24 (0x03 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_CTR_MOD32 (0x04 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_CTR_MOD40 (0x05 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_CTR_MOD48 (0x06 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_CTR_MOD56 (0x07 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_CTR_MOD64 (0x08 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_CTR_MOD72 (0x09 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_CTR_MOD80 (0x0a << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_CTR_MOD88 (0x0b << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_CTR_MOD96 (0x0c << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_CTR_MOD104 (0x0d << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_CTR_MOD112 (0x0e << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_CTR_MOD120 (0x0f << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_CBC (0x10 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_ECB (0x20 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_CFB (0x30 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_OFB (0x40 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_XTS (0x50 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_CMAC (0x60 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_XCBC_MAC (0x70 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_CCM (0x80 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_GCM (0x90 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_CBC_XCBCMAC (0xa0 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_CTR_XCBCMAC (0xb0 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_CBC_CMAC (0xc0 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_CTR_CMAC_LTE (0xd0 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_CTR_CMAC (0xe0 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_CHECKODD (0x80 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_DK (0x100 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_C2K (0x200 << OP_ALG_AAI_SHIFT) + +/* randomizer AAI set */ +#define OP_ALG_RNG_MODE_MASK (0x30 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_RNG (0x00 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_RNG_NZB (0x10 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_RNG_OBP (0x20 << OP_ALG_AAI_SHIFT) + +/* RNG4 AAI set */ +#define OP_ALG_AAI_RNG4_SH_SHIFT OP_ALG_AAI_SHIFT +#define OP_ALG_AAI_RNG4_SH_MASK (0x03 << OP_ALG_AAI_RNG4_SH_SHIFT) +#define OP_ALG_AAI_RNG4_SH_0 (0x00 << OP_ALG_AAI_RNG4_SH_SHIFT) +#define OP_ALG_AAI_RNG4_SH_1 (0x01 << OP_ALG_AAI_RNG4_SH_SHIFT) +#define OP_ALG_AAI_RNG4_PS (0x40 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_RNG4_AI (0x80 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_RNG4_SK (0x100 << OP_ALG_AAI_SHIFT) + +/* hmac/smac AAI set */ +#define OP_ALG_AAI_HASH (0x00 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_HMAC (0x01 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_SMAC (0x02 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_HMAC_PRECOMP (0x04 << OP_ALG_AAI_SHIFT) + +/* CRC AAI set*/ +#define OP_ALG_CRC_POLY_MASK (0x07 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_802 (0x01 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_3385 (0x02 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_CUST_POLY (0x04 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_DIS (0x10 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_DOS (0x20 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_DOC (0x40 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_IVZ (0x80 << OP_ALG_AAI_SHIFT) + +/* Kasumi/SNOW/ZUC AAI set */ +#define OP_ALG_AAI_F8 (0xc0 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_F9 (0xc8 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_GSM (0x10 << OP_ALG_AAI_SHIFT) +#define OP_ALG_AAI_EDGE (0x20 << OP_ALG_AAI_SHIFT) + +#define OP_ALG_AS_SHIFT 2 +#define OP_ALG_AS_MASK (0x3 << OP_ALG_AS_SHIFT) +#define OP_ALG_AS_UPDATE (0 << OP_ALG_AS_SHIFT) +#define OP_ALG_AS_INIT (1 << OP_ALG_AS_SHIFT) +#define OP_ALG_AS_FINALIZE (2 << OP_ALG_AS_SHIFT) +#define OP_ALG_AS_INITFINAL (3 << OP_ALG_AS_SHIFT) + +#define OP_ALG_ICV_SHIFT 1 +#define OP_ALG_ICV_MASK (1 << OP_ALG_ICV_SHIFT) +#define OP_ALG_ICV_OFF 0 +#define OP_ALG_ICV_ON BIT(1) + +#define OP_ALG_DIR_SHIFT 0 +#define OP_ALG_DIR_MASK 1 +#define OP_ALG_DECRYPT 0 +#define OP_ALG_ENCRYPT BIT(0) + +/* PKHA algorithm type set */ +#define OP_ALG_PK 0x00800000 +#define OP_ALG_PK_FUN_MASK 0x3f /* clrmem, modmath, or cpymem */ + +/* PKHA mode clear memory functions */ +#define OP_ALG_PKMODE_A_RAM BIT(19) +#define OP_ALG_PKMODE_B_RAM BIT(18) +#define OP_ALG_PKMODE_E_RAM BIT(17) +#define OP_ALG_PKMODE_N_RAM BIT(16) +#define OP_ALG_PKMODE_CLEARMEM BIT(0) + +/* PKHA mode clear memory functions */ +#define OP_ALG_PKMODE_CLEARMEM_ALL (OP_ALG_PKMODE_CLEARMEM | \ + OP_ALG_PKMODE_A_RAM | \ + OP_ALG_PKMODE_B_RAM | \ + OP_ALG_PKMODE_N_RAM | \ + OP_ALG_PKMODE_E_RAM) +#define OP_ALG_PKMODE_CLEARMEM_ABE (OP_ALG_PKMODE_CLEARMEM | \ + OP_ALG_PKMODE_A_RAM | \ + OP_ALG_PKMODE_B_RAM | \ + OP_ALG_PKMODE_E_RAM) +#define OP_ALG_PKMODE_CLEARMEM_ABN (OP_ALG_PKMODE_CLEARMEM | \ + OP_ALG_PKMODE_A_RAM | \ + OP_ALG_PKMODE_B_RAM | \ + OP_ALG_PKMODE_N_RAM) +#define OP_ALG_PKMODE_CLEARMEM_AB (OP_ALG_PKMODE_CLEARMEM | \ + OP_ALG_PKMODE_A_RAM | \ + OP_ALG_PKMODE_B_RAM) +#define OP_ALG_PKMODE_CLEARMEM_AEN (OP_ALG_PKMODE_CLEARMEM | \ + OP_ALG_PKMODE_A_RAM | \ + OP_ALG_PKMODE_E_RAM | \ + OP_ALG_PKMODE_N_RAM) +#define OP_ALG_PKMODE_CLEARMEM_AE (OP_ALG_PKMODE_CLEARMEM | \ + OP_ALG_PKMODE_A_RAM | \ + OP_ALG_PKMODE_E_RAM) +#define OP_ALG_PKMODE_CLEARMEM_AN (OP_ALG_PKMODE_CLEARMEM | \ + OP_ALG_PKMODE_A_RAM | \ + OP_ALG_PKMODE_N_RAM) +#define OP_ALG_PKMODE_CLEARMEM_A (OP_ALG_PKMODE_CLEARMEM | \ + OP_ALG_PKMODE_A_RAM) +#define OP_ALG_PKMODE_CLEARMEM_BEN (OP_ALG_PKMODE_CLEARMEM | \ + OP_ALG_PKMODE_B_RAM | \ + OP_ALG_PKMODE_E_RAM | \ + OP_ALG_PKMODE_N_RAM) +#define OP_ALG_PKMODE_CLEARMEM_BE (OP_ALG_PKMODE_CLEARMEM | \ + OP_ALG_PKMODE_B_RAM | \ + OP_ALG_PKMODE_E_RAM) +#define OP_ALG_PKMODE_CLEARMEM_BN (OP_ALG_PKMODE_CLEARMEM | \ + OP_ALG_PKMODE_B_RAM | \ + OP_ALG_PKMODE_N_RAM) +#define OP_ALG_PKMODE_CLEARMEM_B (OP_ALG_PKMODE_CLEARMEM | \ + OP_ALG_PKMODE_B_RAM) +#define OP_ALG_PKMODE_CLEARMEM_EN (OP_ALG_PKMODE_CLEARMEM | \ + OP_ALG_PKMODE_E_RAM | \ + OP_ALG_PKMODE_N_RAM) +#define OP_ALG_PKMODE_CLEARMEM_E (OP_ALG_PKMODE_CLEARMEM | \ + OP_ALG_PKMODE_E_RAM) +#define OP_ALG_PKMODE_CLEARMEM_N (OP_ALG_PKMODE_CLEARMEM | \ + OP_ALG_PKMODE_N_RAM) + +/* PKHA mode modular-arithmetic functions */ +#define OP_ALG_PKMODE_MOD_IN_MONTY BIT(19) +#define OP_ALG_PKMODE_MOD_OUT_MONTY BIT(18) +#define OP_ALG_PKMODE_MOD_F2M BIT(17) +#define OP_ALG_PKMODE_MOD_R2_IN BIT(16) +#define OP_ALG_PKMODE_PRJECTV BIT(11) +#define OP_ALG_PKMODE_TIME_EQ BIT(10) + +#define OP_ALG_PKMODE_OUT_B 0x000 +#define OP_ALG_PKMODE_OUT_A 0x100 + +/* + * PKHA mode modular-arithmetic integer functions + * Can be ORed with OP_ALG_PKMODE_OUT_A to change destination from B + */ +#define OP_ALG_PKMODE_MOD_ADD 0x002 +#define OP_ALG_PKMODE_MOD_SUB_AB 0x003 +#define OP_ALG_PKMODE_MOD_SUB_BA 0x004 +#define OP_ALG_PKMODE_MOD_MULT 0x005 +#define OP_ALG_PKMODE_MOD_MULT_IM (0x005 | OP_ALG_PKMODE_MOD_IN_MONTY) +#define OP_ALG_PKMODE_MOD_MULT_IM_OM (0x005 | OP_ALG_PKMODE_MOD_IN_MONTY \ + | OP_ALG_PKMODE_MOD_OUT_MONTY) +#define OP_ALG_PKMODE_MOD_EXPO 0x006 +#define OP_ALG_PKMODE_MOD_EXPO_TEQ (0x006 | OP_ALG_PKMODE_TIME_EQ) +#define OP_ALG_PKMODE_MOD_EXPO_IM (0x006 | OP_ALG_PKMODE_MOD_IN_MONTY) +#define OP_ALG_PKMODE_MOD_EXPO_IM_TEQ (0x006 | OP_ALG_PKMODE_MOD_IN_MONTY \ + | OP_ALG_PKMODE_TIME_EQ) +#define OP_ALG_PKMODE_MOD_REDUCT 0x007 +#define OP_ALG_PKMODE_MOD_INV 0x008 +#define OP_ALG_PKMODE_MOD_ECC_ADD 0x009 +#define OP_ALG_PKMODE_MOD_ECC_DBL 0x00a +#define OP_ALG_PKMODE_MOD_ECC_MULT 0x00b +#define OP_ALG_PKMODE_MOD_MONT_CNST 0x00c +#define OP_ALG_PKMODE_MOD_CRT_CNST 0x00d +#define OP_ALG_PKMODE_MOD_GCD 0x00e +#define OP_ALG_PKMODE_MOD_PRIMALITY 0x00f +#define OP_ALG_PKMODE_MOD_SML_EXP 0x016 + +/* + * PKHA mode modular-arithmetic F2m functions + * Can be ORed with OP_ALG_PKMODE_OUT_A to change destination from B + */ +#define OP_ALG_PKMODE_F2M_ADD (0x002 | OP_ALG_PKMODE_MOD_F2M) +#define OP_ALG_PKMODE_F2M_MUL (0x005 | OP_ALG_PKMODE_MOD_F2M) +#define OP_ALG_PKMODE_F2M_MUL_IM (0x005 | OP_ALG_PKMODE_MOD_F2M \ + | OP_ALG_PKMODE_MOD_IN_MONTY) +#define OP_ALG_PKMODE_F2M_MUL_IM_OM (0x005 | OP_ALG_PKMODE_MOD_F2M \ + | OP_ALG_PKMODE_MOD_IN_MONTY \ + | OP_ALG_PKMODE_MOD_OUT_MONTY) +#define OP_ALG_PKMODE_F2M_EXP (0x006 | OP_ALG_PKMODE_MOD_F2M) +#define OP_ALG_PKMODE_F2M_EXP_TEQ (0x006 | OP_ALG_PKMODE_MOD_F2M \ + | OP_ALG_PKMODE_TIME_EQ) +#define OP_ALG_PKMODE_F2M_AMODN (0x007 | OP_ALG_PKMODE_MOD_F2M) +#define OP_ALG_PKMODE_F2M_INV (0x008 | OP_ALG_PKMODE_MOD_F2M) +#define OP_ALG_PKMODE_F2M_R2 (0x00c | OP_ALG_PKMODE_MOD_F2M) +#define OP_ALG_PKMODE_F2M_GCD (0x00e | OP_ALG_PKMODE_MOD_F2M) +#define OP_ALG_PKMODE_F2M_SML_EXP (0x016 | OP_ALG_PKMODE_MOD_F2M) + +/* + * PKHA mode ECC Integer arithmetic functions + * Can be ORed with OP_ALG_PKMODE_OUT_A to change destination from B + */ +#define OP_ALG_PKMODE_ECC_MOD_ADD 0x009 +#define OP_ALG_PKMODE_ECC_MOD_ADD_IM_OM_PROJ \ + (0x009 | OP_ALG_PKMODE_MOD_IN_MONTY \ + | OP_ALG_PKMODE_MOD_OUT_MONTY \ + | OP_ALG_PKMODE_PRJECTV) +#define OP_ALG_PKMODE_ECC_MOD_DBL 0x00a +#define OP_ALG_PKMODE_ECC_MOD_DBL_IM_OM_PROJ \ + (0x00a | OP_ALG_PKMODE_MOD_IN_MONTY \ + | OP_ALG_PKMODE_MOD_OUT_MONTY \ + | OP_ALG_PKMODE_PRJECTV) +#define OP_ALG_PKMODE_ECC_MOD_MUL 0x00b +#define OP_ALG_PKMODE_ECC_MOD_MUL_TEQ (0x00b | OP_ALG_PKMODE_TIME_EQ) +#define OP_ALG_PKMODE_ECC_MOD_MUL_R2 (0x00b | OP_ALG_PKMODE_MOD_R2_IN) +#define OP_ALG_PKMODE_ECC_MOD_MUL_R2_TEQ \ + (0x00b | OP_ALG_PKMODE_MOD_R2_IN \ + | OP_ALG_PKMODE_TIME_EQ) +#define OP_ALG_PKMODE_ECC_MOD_MUL_R2_PROJ \ + (0x00b | OP_ALG_PKMODE_MOD_R2_IN \ + | OP_ALG_PKMODE_PRJECTV) +#define OP_ALG_PKMODE_ECC_MOD_MUL_R2_PROJ_TEQ \ + (0x00b | OP_ALG_PKMODE_MOD_R2_IN \ + | OP_ALG_PKMODE_PRJECTV \ + | OP_ALG_PKMODE_TIME_EQ) + +/* + * PKHA mode ECC F2m arithmetic functions + * Can be ORed with OP_ALG_PKMODE_OUT_A to change destination from B + */ +#define OP_ALG_PKMODE_ECC_F2M_ADD (0x009 | OP_ALG_PKMODE_MOD_F2M) +#define OP_ALG_PKMODE_ECC_F2M_ADD_IM_OM_PROJ \ + (0x009 | OP_ALG_PKMODE_MOD_F2M \ + | OP_ALG_PKMODE_MOD_IN_MONTY \ + | OP_ALG_PKMODE_MOD_OUT_MONTY \ + | OP_ALG_PKMODE_PRJECTV) +#define OP_ALG_PKMODE_ECC_F2M_DBL (0x00a | OP_ALG_PKMODE_MOD_F2M) +#define OP_ALG_PKMODE_ECC_F2M_DBL_IM_OM_PROJ \ + (0x00a | OP_ALG_PKMODE_MOD_F2M \ + | OP_ALG_PKMODE_MOD_IN_MONTY \ + | OP_ALG_PKMODE_MOD_OUT_MONTY \ + | OP_ALG_PKMODE_PRJECTV) +#define OP_ALG_PKMODE_ECC_F2M_MUL (0x00b | OP_ALG_PKMODE_MOD_F2M) +#define OP_ALG_PKMODE_ECC_F2M_MUL_TEQ \ + (0x00b | OP_ALG_PKMODE_MOD_F2M \ + | OP_ALG_PKMODE_TIME_EQ) +#define OP_ALG_PKMODE_ECC_F2M_MUL_R2 \ + (0x00b | OP_ALG_PKMODE_MOD_F2M \ + | OP_ALG_PKMODE_MOD_R2_IN) +#define OP_ALG_PKMODE_ECC_F2M_MUL_R2_TEQ \ + (0x00b | OP_ALG_PKMODE_MOD_F2M \ + | OP_ALG_PKMODE_MOD_R2_IN \ + | OP_ALG_PKMODE_TIME_EQ) +#define OP_ALG_PKMODE_ECC_F2M_MUL_R2_PROJ \ + (0x00b | OP_ALG_PKMODE_MOD_F2M \ + | OP_ALG_PKMODE_MOD_R2_IN \ + | OP_ALG_PKMODE_PRJECTV) +#define OP_ALG_PKMODE_ECC_F2M_MUL_R2_PROJ_TEQ \ + (0x00b | OP_ALG_PKMODE_MOD_F2M \ + | OP_ALG_PKMODE_MOD_R2_IN \ + | OP_ALG_PKMODE_PRJECTV \ + | OP_ALG_PKMODE_TIME_EQ) + +/* PKHA mode copy-memory functions */ +#define OP_ALG_PKMODE_SRC_REG_SHIFT 17 +#define OP_ALG_PKMODE_SRC_REG_MASK (7 << OP_ALG_PKMODE_SRC_REG_SHIFT) +#define OP_ALG_PKMODE_DST_REG_SHIFT 10 +#define OP_ALG_PKMODE_DST_REG_MASK (7 << OP_ALG_PKMODE_DST_REG_SHIFT) +#define OP_ALG_PKMODE_SRC_SEG_SHIFT 8 +#define OP_ALG_PKMODE_SRC_SEG_MASK (3 << OP_ALG_PKMODE_SRC_SEG_SHIFT) +#define OP_ALG_PKMODE_DST_SEG_SHIFT 6 +#define OP_ALG_PKMODE_DST_SEG_MASK (3 << OP_ALG_PKMODE_DST_SEG_SHIFT) + +#define OP_ALG_PKMODE_SRC_REG_A (0 << OP_ALG_PKMODE_SRC_REG_SHIFT) +#define OP_ALG_PKMODE_SRC_REG_B (1 << OP_ALG_PKMODE_SRC_REG_SHIFT) +#define OP_ALG_PKMODE_SRC_REG_N (3 << OP_ALG_PKMODE_SRC_REG_SHIFT) +#define OP_ALG_PKMODE_DST_REG_A (0 << OP_ALG_PKMODE_DST_REG_SHIFT) +#define OP_ALG_PKMODE_DST_REG_B (1 << OP_ALG_PKMODE_DST_REG_SHIFT) +#define OP_ALG_PKMODE_DST_REG_E (2 << OP_ALG_PKMODE_DST_REG_SHIFT) +#define OP_ALG_PKMODE_DST_REG_N (3 << OP_ALG_PKMODE_DST_REG_SHIFT) +#define OP_ALG_PKMODE_SRC_SEG_0 (0 << OP_ALG_PKMODE_SRC_SEG_SHIFT) +#define OP_ALG_PKMODE_SRC_SEG_1 (1 << OP_ALG_PKMODE_SRC_SEG_SHIFT) +#define OP_ALG_PKMODE_SRC_SEG_2 (2 << OP_ALG_PKMODE_SRC_SEG_SHIFT) +#define OP_ALG_PKMODE_SRC_SEG_3 (3 << OP_ALG_PKMODE_SRC_SEG_SHIFT) +#define OP_ALG_PKMODE_DST_SEG_0 (0 << OP_ALG_PKMODE_DST_SEG_SHIFT) +#define OP_ALG_PKMODE_DST_SEG_1 (1 << OP_ALG_PKMODE_DST_SEG_SHIFT) +#define OP_ALG_PKMODE_DST_SEG_2 (2 << OP_ALG_PKMODE_DST_SEG_SHIFT) +#define OP_ALG_PKMODE_DST_SEG_3 (3 << OP_ALG_PKMODE_DST_SEG_SHIFT) + +/* PKHA mode copy-memory functions - amount based on N SIZE */ +#define OP_ALG_PKMODE_COPY_NSZ 0x10 +#define OP_ALG_PKMODE_COPY_NSZ_A0_B0 (OP_ALG_PKMODE_COPY_NSZ | \ + OP_ALG_PKMODE_SRC_REG_A | \ + OP_ALG_PKMODE_DST_REG_B) +#define OP_ALG_PKMODE_COPY_NSZ_A0_B1 (OP_ALG_PKMODE_COPY_NSZ | \ + OP_ALG_PKMODE_SRC_REG_A | \ + OP_ALG_PKMODE_DST_REG_B | \ + OP_ALG_PKMODE_DST_SEG_1) +#define OP_ALG_PKMODE_COPY_NSZ_A0_B2 (OP_ALG_PKMODE_COPY_NSZ | \ + OP_ALG_PKMODE_SRC_REG_A | \ + OP_ALG_PKMODE_DST_REG_B | \ + OP_ALG_PKMODE_DST_SEG_2) +#define OP_ALG_PKMODE_COPY_NSZ_A0_B3 (OP_ALG_PKMODE_COPY_NSZ | \ + OP_ALG_PKMODE_SRC_REG_A | \ + OP_ALG_PKMODE_DST_REG_B | \ + OP_ALG_PKMODE_DST_SEG_3) + +#define OP_ALG_PKMODE_COPY_NSZ_A1_B0 (OP_ALG_PKMODE_COPY_NSZ | \ + OP_ALG_PKMODE_SRC_REG_A | \ + OP_ALG_PKMODE_SRC_SEG_1 | \ + OP_ALG_PKMODE_DST_REG_B) +#define OP_ALG_PKMODE_COPY_NSZ_A1_B1 (OP_ALG_PKMODE_COPY_NSZ | \ + OP_ALG_PKMODE_SRC_REG_A | \ + OP_ALG_PKMODE_SRC_SEG_1 | \ + OP_ALG_PKMODE_DST_REG_B | \ + OP_ALG_PKMODE_DST_SEG_1) +#define OP_ALG_PKMODE_COPY_NSZ_A1_B2 (OP_ALG_PKMODE_COPY_NSZ | \ + OP_ALG_PKMODE_SRC_REG_A | \ + OP_ALG_PKMODE_SRC_SEG_1 | \ + OP_ALG_PKMODE_DST_REG_B | \ + OP_ALG_PKMODE_DST_SEG_2) +#define OP_ALG_PKMODE_COPY_NSZ_A1_B3 (OP_ALG_PKMODE_COPY_NSZ | \ + OP_ALG_PKMODE_SRC_REG_A | \ + OP_ALG_PKMODE_SRC_SEG_1 | \ + OP_ALG_PKMODE_DST_REG_B | \ + OP_ALG_PKMODE_DST_SEG_3) + +#define OP_ALG_PKMODE_COPY_NSZ_A2_B0 (OP_ALG_PKMODE_COPY_NSZ | \ + OP_ALG_PKMODE_SRC_REG_A | \ + OP_ALG_PKMODE_SRC_SEG_2 | \ + OP_ALG_PKMODE_DST_REG_B) +#define OP_ALG_PKMODE_COPY_NSZ_A2_B1 (OP_ALG_PKMODE_COPY_NSZ | \ + OP_ALG_PKMODE_SRC_REG_A | \ + OP_ALG_PKMODE_SRC_SEG_2 | \ + OP_ALG_PKMODE_DST_REG_B | \ + OP_ALG_PKMODE_DST_SEG_1) +#define OP_ALG_PKMODE_COPY_NSZ_A2_B2 (OP_ALG_PKMODE_COPY_NSZ | \ + OP_ALG_PKMODE_SRC_REG_A | \ + OP_ALG_PKMODE_SRC_SEG_2 | \ + OP_ALG_PKMODE_DST_REG_B | \ + OP_ALG_PKMODE_DST_SEG_2) +#define OP_ALG_PKMODE_COPY_NSZ_A2_B3 (OP_ALG_PKMODE_COPY_NSZ | \ + OP_ALG_PKMODE_SRC_REG_A | \ + OP_ALG_PKMODE_SRC_SEG_2 | \ + OP_ALG_PKMODE_DST_REG_B | \ + OP_ALG_PKMODE_DST_SEG_3) + +#define OP_ALG_PKMODE_COPY_NSZ_A3_B0 (OP_ALG_PKMODE_COPY_NSZ | \ + OP_ALG_PKMODE_SRC_REG_A | \ + OP_ALG_PKMODE_SRC_SEG_3 | \ + OP_ALG_PKMODE_DST_REG_B) +#define OP_ALG_PKMODE_COPY_NSZ_A3_B1 (OP_ALG_PKMODE_COPY_NSZ | \ + OP_ALG_PKMODE_SRC_REG_A | \ + OP_ALG_PKMODE_SRC_SEG_3 | \ + OP_ALG_PKMODE_DST_REG_B | \ + OP_ALG_PKMODE_DST_SEG_1) +#define OP_ALG_PKMODE_COPY_NSZ_A3_B2 (OP_ALG_PKMODE_COPY_NSZ | \ + OP_ALG_PKMODE_SRC_REG_A | \ + OP_ALG_PKMODE_SRC_SEG_3 | \ + OP_ALG_PKMODE_DST_REG_B | \ + OP_ALG_PKMODE_DST_SEG_2) +#define OP_ALG_PKMODE_COPY_NSZ_A3_B3 (OP_ALG_PKMODE_COPY_NSZ | \ + OP_ALG_PKMODE_SRC_REG_A | \ + OP_ALG_PKMODE_SRC_SEG_3 | \ + OP_ALG_PKMODE_DST_REG_B | \ + OP_ALG_PKMODE_DST_SEG_3) + +#define OP_ALG_PKMODE_COPY_NSZ_B0_A0 (OP_ALG_PKMODE_COPY_NSZ | \ + OP_ALG_PKMODE_SRC_REG_B | \ + OP_ALG_PKMODE_DST_REG_A) +#define OP_ALG_PKMODE_COPY_NSZ_B0_A1 (OP_ALG_PKMODE_COPY_NSZ | \ + OP_ALG_PKMODE_SRC_REG_B | \ + OP_ALG_PKMODE_DST_REG_A | \ + OP_ALG_PKMODE_DST_SEG_1) +#define OP_ALG_PKMODE_COPY_NSZ_B0_A2 (OP_ALG_PKMODE_COPY_NSZ | \ + OP_ALG_PKMODE_SRC_REG_B | \ + OP_ALG_PKMODE_DST_REG_A | \ + OP_ALG_PKMODE_DST_SEG_2) +#define OP_ALG_PKMODE_COPY_NSZ_B0_A3 (OP_ALG_PKMODE_COPY_NSZ | \ + OP_ALG_PKMODE_SRC_REG_B | \ + OP_ALG_PKMODE_DST_REG_A | \ + OP_ALG_PKMODE_DST_SEG_3) + +#define OP_ALG_PKMODE_COPY_NSZ_B1_A0 (OP_ALG_PKMODE_COPY_NSZ | \ + OP_ALG_PKMODE_SRC_REG_B | \ + OP_ALG_PKMODE_SRC_SEG_1 | \ + OP_ALG_PKMODE_DST_REG_A) +#define OP_ALG_PKMODE_COPY_NSZ_B1_A1 (OP_ALG_PKMODE_COPY_NSZ | \ + OP_ALG_PKMODE_SRC_REG_B | \ + OP_ALG_PKMODE_SRC_SEG_1 | \ + OP_ALG_PKMODE_DST_REG_A | \ + OP_ALG_PKMODE_DST_SEG_1) +#define OP_ALG_PKMODE_COPY_NSZ_B1_A2 (OP_ALG_PKMODE_COPY_NSZ | \ + OP_ALG_PKMODE_SRC_REG_B | \ + OP_ALG_PKMODE_SRC_SEG_1 | \ + OP_ALG_PKMODE_DST_REG_A | \ + OP_ALG_PKMODE_DST_SEG_2) +#define OP_ALG_PKMODE_COPY_NSZ_B1_A3 (OP_ALG_PKMODE_COPY_NSZ | \ + OP_ALG_PKMODE_SRC_REG_B | \ + OP_ALG_PKMODE_SRC_SEG_1 | \ + OP_ALG_PKMODE_DST_REG_A | \ + OP_ALG_PKMODE_DST_SEG_3) + +#define OP_ALG_PKMODE_COPY_NSZ_B2_A0 (OP_ALG_PKMODE_COPY_NSZ | \ + OP_ALG_PKMODE_SRC_REG_B | \ + OP_ALG_PKMODE_SRC_SEG_2 | \ + OP_ALG_PKMODE_DST_REG_A) +#define OP_ALG_PKMODE_COPY_NSZ_B2_A1 (OP_ALG_PKMODE_COPY_NSZ | \ + OP_ALG_PKMODE_SRC_REG_B | \ + OP_ALG_PKMODE_SRC_SEG_2 | \ + OP_ALG_PKMODE_DST_REG_A | \ + OP_ALG_PKMODE_DST_SEG_1) +#define OP_ALG_PKMODE_COPY_NSZ_B2_A2 (OP_ALG_PKMODE_COPY_NSZ | \ + OP_ALG_PKMODE_SRC_REG_B | \ + OP_ALG_PKMODE_SRC_SEG_2 | \ + OP_ALG_PKMODE_DST_REG_A | \ + OP_ALG_PKMODE_DST_SEG_2) +#define OP_ALG_PKMODE_COPY_NSZ_B2_A3 (OP_ALG_PKMODE_COPY_NSZ | \ + OP_ALG_PKMODE_SRC_REG_B | \ + OP_ALG_PKMODE_SRC_SEG_2 | \ + OP_ALG_PKMODE_DST_REG_A | \ + OP_ALG_PKMODE_DST_SEG_3) + +#define OP_ALG_PKMODE_COPY_NSZ_B3_A0 (OP_ALG_PKMODE_COPY_NSZ | \ + OP_ALG_PKMODE_SRC_REG_B | \ + OP_ALG_PKMODE_SRC_SEG_3 | \ + OP_ALG_PKMODE_DST_REG_A) +#define OP_ALG_PKMODE_COPY_NSZ_B3_A1 (OP_ALG_PKMODE_COPY_NSZ | \ + OP_ALG_PKMODE_SRC_REG_B | \ + OP_ALG_PKMODE_SRC_SEG_3 | \ + OP_ALG_PKMODE_DST_REG_A | \ + OP_ALG_PKMODE_DST_SEG_1) +#define OP_ALG_PKMODE_COPY_NSZ_B3_A2 (OP_ALG_PKMODE_COPY_NSZ | \ + OP_ALG_PKMODE_SRC_REG_B | \ + OP_ALG_PKMODE_SRC_SEG_3 | \ + OP_ALG_PKMODE_DST_REG_A | \ + OP_ALG_PKMODE_DST_SEG_2) +#define OP_ALG_PKMODE_COPY_NSZ_B3_A3 (OP_ALG_PKMODE_COPY_NSZ | \ + OP_ALG_PKMODE_SRC_REG_B | \ + OP_ALG_PKMODE_SRC_SEG_3 | \ + OP_ALG_PKMODE_DST_REG_A | \ + OP_ALG_PKMODE_DST_SEG_3) + +#define OP_ALG_PKMODE_COPY_NSZ_A_B (OP_ALG_PKMODE_COPY_NSZ | \ + OP_ALG_PKMODE_SRC_REG_A | \ + OP_ALG_PKMODE_DST_REG_B) +#define OP_ALG_PKMODE_COPY_NSZ_A_E (OP_ALG_PKMODE_COPY_NSZ | \ + OP_ALG_PKMODE_SRC_REG_A | \ + OP_ALG_PKMODE_DST_REG_E) +#define OP_ALG_PKMODE_COPY_NSZ_A_N (OP_ALG_PKMODE_COPY_NSZ | \ + OP_ALG_PKMODE_SRC_REG_A | \ + OP_ALG_PKMODE_DST_REG_N) +#define OP_ALG_PKMODE_COPY_NSZ_B_A (OP_ALG_PKMODE_COPY_NSZ | \ + OP_ALG_PKMODE_SRC_REG_B | \ + OP_ALG_PKMODE_DST_REG_A) +#define OP_ALG_PKMODE_COPY_NSZ_B_E (OP_ALG_PKMODE_COPY_NSZ | \ + OP_ALG_PKMODE_SRC_REG_B | \ + OP_ALG_PKMODE_DST_REG_E) +#define OP_ALG_PKMODE_COPY_NSZ_B_N (OP_ALG_PKMODE_COPY_NSZ | \ + OP_ALG_PKMODE_SRC_REG_B | \ + OP_ALG_PKMODE_DST_REG_N) +#define OP_ALG_PKMODE_COPY_NSZ_N_A (OP_ALG_PKMODE_COPY_NSZ | \ + OP_ALG_PKMODE_SRC_REG_N | \ + OP_ALG_PKMODE_DST_REG_A) +#define OP_ALG_PKMODE_COPY_NSZ_N_B (OP_ALG_PKMODE_COPY_NSZ | \ + OP_ALG_PKMODE_SRC_REG_N | \ + OP_ALG_PKMODE_DST_REG_B) +#define OP_ALG_PKMODE_COPY_NSZ_N_E (OP_ALG_PKMODE_COPY_NSZ | \ + OP_ALG_PKMODE_SRC_REG_N | \ + OP_ALG_PKMODE_DST_REG_E) + +/* PKHA mode copy-memory functions - amount based on SRC SIZE */ +#define OP_ALG_PKMODE_COPY_SSZ 0x11 +#define OP_ALG_PKMODE_COPY_SSZ_A0_B0 (OP_ALG_PKMODE_COPY_SSZ | \ + OP_ALG_PKMODE_SRC_REG_A | \ + OP_ALG_PKMODE_DST_REG_B) +#define OP_ALG_PKMODE_COPY_SSZ_A0_B1 (OP_ALG_PKMODE_COPY_SSZ | \ + OP_ALG_PKMODE_SRC_REG_A | \ + OP_ALG_PKMODE_DST_REG_B | \ + OP_ALG_PKMODE_DST_SEG_1) +#define OP_ALG_PKMODE_COPY_SSZ_A0_B2 (OP_ALG_PKMODE_COPY_SSZ | \ + OP_ALG_PKMODE_SRC_REG_A | \ + OP_ALG_PKMODE_DST_REG_B | \ + OP_ALG_PKMODE_DST_SEG_2) +#define OP_ALG_PKMODE_COPY_SSZ_A0_B3 (OP_ALG_PKMODE_COPY_SSZ | \ + OP_ALG_PKMODE_SRC_REG_A | \ + OP_ALG_PKMODE_DST_REG_B | \ + OP_ALG_PKMODE_DST_SEG_3) + +#define OP_ALG_PKMODE_COPY_SSZ_A1_B0 (OP_ALG_PKMODE_COPY_SSZ | \ + OP_ALG_PKMODE_SRC_REG_A | \ + OP_ALG_PKMODE_SRC_SEG_1 | \ + OP_ALG_PKMODE_DST_REG_B) +#define OP_ALG_PKMODE_COPY_SSZ_A1_B1 (OP_ALG_PKMODE_COPY_SSZ | \ + OP_ALG_PKMODE_SRC_REG_A | \ + OP_ALG_PKMODE_SRC_SEG_1 | \ + OP_ALG_PKMODE_DST_REG_B | \ + OP_ALG_PKMODE_DST_SEG_1) +#define OP_ALG_PKMODE_COPY_SSZ_A1_B2 (OP_ALG_PKMODE_COPY_SSZ | \ + OP_ALG_PKMODE_SRC_REG_A | \ + OP_ALG_PKMODE_SRC_SEG_1 | \ + OP_ALG_PKMODE_DST_REG_B | \ + OP_ALG_PKMODE_DST_SEG_2) +#define OP_ALG_PKMODE_COPY_SSZ_A1_B3 (OP_ALG_PKMODE_COPY_SSZ | \ + OP_ALG_PKMODE_SRC_REG_A | \ + OP_ALG_PKMODE_SRC_SEG_1 | \ + OP_ALG_PKMODE_DST_REG_B | \ + OP_ALG_PKMODE_DST_SEG_3) + +#define OP_ALG_PKMODE_COPY_SSZ_A2_B0 (OP_ALG_PKMODE_COPY_SSZ | \ + OP_ALG_PKMODE_SRC_REG_A | \ + OP_ALG_PKMODE_SRC_SEG_2 | \ + OP_ALG_PKMODE_DST_REG_B) +#define OP_ALG_PKMODE_COPY_SSZ_A2_B1 (OP_ALG_PKMODE_COPY_SSZ | \ + OP_ALG_PKMODE_SRC_REG_A | \ + OP_ALG_PKMODE_SRC_SEG_2 | \ + OP_ALG_PKMODE_DST_REG_B | \ + OP_ALG_PKMODE_DST_SEG_1) +#define OP_ALG_PKMODE_COPY_SSZ_A2_B2 (OP_ALG_PKMODE_COPY_SSZ | \ + OP_ALG_PKMODE_SRC_REG_A | \ + OP_ALG_PKMODE_SRC_SEG_2 | \ + OP_ALG_PKMODE_DST_REG_B | \ + OP_ALG_PKMODE_DST_SEG_2) +#define OP_ALG_PKMODE_COPY_SSZ_A2_B3 (OP_ALG_PKMODE_COPY_SSZ | \ + OP_ALG_PKMODE_SRC_REG_A | \ + OP_ALG_PKMODE_SRC_SEG_2 | \ + OP_ALG_PKMODE_DST_REG_B | \ + OP_ALG_PKMODE_DST_SEG_3) + +#define OP_ALG_PKMODE_COPY_SSZ_A3_B0 (OP_ALG_PKMODE_COPY_SSZ | \ + OP_ALG_PKMODE_SRC_REG_A | \ + OP_ALG_PKMODE_SRC_SEG_3 | \ + OP_ALG_PKMODE_DST_REG_B) +#define OP_ALG_PKMODE_COPY_SSZ_A3_B1 (OP_ALG_PKMODE_COPY_SSZ | \ + OP_ALG_PKMODE_SRC_REG_A | \ + OP_ALG_PKMODE_SRC_SEG_3 | \ + OP_ALG_PKMODE_DST_REG_B | \ + OP_ALG_PKMODE_DST_SEG_1) +#define OP_ALG_PKMODE_COPY_SSZ_A3_B2 (OP_ALG_PKMODE_COPY_SSZ | \ + OP_ALG_PKMODE_SRC_REG_A | \ + OP_ALG_PKMODE_SRC_SEG_3 | \ + OP_ALG_PKMODE_DST_REG_B | \ + OP_ALG_PKMODE_DST_SEG_2) +#define OP_ALG_PKMODE_COPY_SSZ_A3_B3 (OP_ALG_PKMODE_COPY_SSZ | \ + OP_ALG_PKMODE_SRC_REG_A | \ + OP_ALG_PKMODE_SRC_SEG_3 | \ + OP_ALG_PKMODE_DST_REG_B | \ + OP_ALG_PKMODE_DST_SEG_3) + +#define OP_ALG_PKMODE_COPY_SSZ_B0_A0 (OP_ALG_PKMODE_COPY_SSZ | \ + OP_ALG_PKMODE_SRC_REG_B | \ + OP_ALG_PKMODE_DST_REG_A) +#define OP_ALG_PKMODE_COPY_SSZ_B0_A1 (OP_ALG_PKMODE_COPY_SSZ | \ + OP_ALG_PKMODE_SRC_REG_B | \ + OP_ALG_PKMODE_DST_REG_A | \ + OP_ALG_PKMODE_DST_SEG_1) +#define OP_ALG_PKMODE_COPY_SSZ_B0_A2 (OP_ALG_PKMODE_COPY_SSZ | \ + OP_ALG_PKMODE_SRC_REG_B | \ + OP_ALG_PKMODE_DST_REG_A | \ + OP_ALG_PKMODE_DST_SEG_2) +#define OP_ALG_PKMODE_COPY_SSZ_B0_A3 (OP_ALG_PKMODE_COPY_SSZ | \ + OP_ALG_PKMODE_SRC_REG_B | \ + OP_ALG_PKMODE_DST_REG_A | \ + OP_ALG_PKMODE_DST_SEG_3) + +#define OP_ALG_PKMODE_COPY_SSZ_B1_A0 (OP_ALG_PKMODE_COPY_SSZ | \ + OP_ALG_PKMODE_SRC_REG_B | \ + OP_ALG_PKMODE_SRC_SEG_1 | \ + OP_ALG_PKMODE_DST_REG_A) +#define OP_ALG_PKMODE_COPY_SSZ_B1_A1 (OP_ALG_PKMODE_COPY_SSZ | \ + OP_ALG_PKMODE_SRC_REG_B | \ + OP_ALG_PKMODE_SRC_SEG_1 | \ + OP_ALG_PKMODE_DST_REG_A | \ + OP_ALG_PKMODE_DST_SEG_1) +#define OP_ALG_PKMODE_COPY_SSZ_B1_A2 (OP_ALG_PKMODE_COPY_SSZ | \ + OP_ALG_PKMODE_SRC_REG_B | \ + OP_ALG_PKMODE_SRC_SEG_1 | \ + OP_ALG_PKMODE_DST_REG_A | \ + OP_ALG_PKMODE_DST_SEG_2) +#define OP_ALG_PKMODE_COPY_SSZ_B1_A3 (OP_ALG_PKMODE_COPY_SSZ | \ + OP_ALG_PKMODE_SRC_REG_B | \ + OP_ALG_PKMODE_SRC_SEG_1 | \ + OP_ALG_PKMODE_DST_REG_A | \ + OP_ALG_PKMODE_DST_SEG_3) + +#define OP_ALG_PKMODE_COPY_SSZ_B2_A0 (OP_ALG_PKMODE_COPY_SSZ | \ + OP_ALG_PKMODE_SRC_REG_B | \ + OP_ALG_PKMODE_SRC_SEG_2 | \ + OP_ALG_PKMODE_DST_REG_A) +#define OP_ALG_PKMODE_COPY_SSZ_B2_A1 (OP_ALG_PKMODE_COPY_SSZ | \ + OP_ALG_PKMODE_SRC_REG_B | \ + OP_ALG_PKMODE_SRC_SEG_2 | \ + OP_ALG_PKMODE_DST_REG_A | \ + OP_ALG_PKMODE_DST_SEG_1) +#define OP_ALG_PKMODE_COPY_SSZ_B2_A2 (OP_ALG_PKMODE_COPY_SSZ | \ + OP_ALG_PKMODE_SRC_REG_B | \ + OP_ALG_PKMODE_SRC_SEG_2 | \ + OP_ALG_PKMODE_DST_REG_A | \ + OP_ALG_PKMODE_DST_SEG_2) +#define OP_ALG_PKMODE_COPY_SSZ_B2_A3 (OP_ALG_PKMODE_COPY_SSZ | \ + OP_ALG_PKMODE_SRC_REG_B | \ + OP_ALG_PKMODE_SRC_SEG_2 | \ + OP_ALG_PKMODE_DST_REG_A | \ + OP_ALG_PKMODE_DST_SEG_3) + +#define OP_ALG_PKMODE_COPY_SSZ_B3_A0 (OP_ALG_PKMODE_COPY_SSZ | \ + OP_ALG_PKMODE_SRC_REG_B | \ + OP_ALG_PKMODE_SRC_SEG_3 | \ + OP_ALG_PKMODE_DST_REG_A) +#define OP_ALG_PKMODE_COPY_SSZ_B3_A1 (OP_ALG_PKMODE_COPY_SSZ | \ + OP_ALG_PKMODE_SRC_REG_B | \ + OP_ALG_PKMODE_SRC_SEG_3 | \ + OP_ALG_PKMODE_DST_REG_A | \ + OP_ALG_PKMODE_DST_SEG_1) +#define OP_ALG_PKMODE_COPY_SSZ_B3_A2 (OP_ALG_PKMODE_COPY_SSZ | \ + OP_ALG_PKMODE_SRC_REG_B | \ + OP_ALG_PKMODE_SRC_SEG_3 | \ + OP_ALG_PKMODE_DST_REG_A | \ + OP_ALG_PKMODE_DST_SEG_2) +#define OP_ALG_PKMODE_COPY_SSZ_B3_A3 (OP_ALG_PKMODE_COPY_SSZ | \ + OP_ALG_PKMODE_SRC_REG_B | \ + OP_ALG_PKMODE_SRC_SEG_3 | \ + OP_ALG_PKMODE_DST_REG_A | \ + OP_ALG_PKMODE_DST_SEG_3) + +#define OP_ALG_PKMODE_COPY_SSZ_A_B (OP_ALG_PKMODE_COPY_SSZ | \ + OP_ALG_PKMODE_SRC_REG_A | \ + OP_ALG_PKMODE_DST_REG_B) +#define OP_ALG_PKMODE_COPY_SSZ_A_E (OP_ALG_PKMODE_COPY_SSZ | \ + OP_ALG_PKMODE_SRC_REG_A | \ + OP_ALG_PKMODE_DST_REG_E) +#define OP_ALG_PKMODE_COPY_SSZ_A_N (OP_ALG_PKMODE_COPY_SSZ | \ + OP_ALG_PKMODE_SRC_REG_A | \ + OP_ALG_PKMODE_DST_REG_N) +#define OP_ALG_PKMODE_COPY_SSZ_B_A (OP_ALG_PKMODE_COPY_SSZ | \ + OP_ALG_PKMODE_SRC_REG_B | \ + OP_ALG_PKMODE_DST_REG_A) +#define OP_ALG_PKMODE_COPY_SSZ_B_E (OP_ALG_PKMODE_COPY_SSZ | \ + OP_ALG_PKMODE_SRC_REG_B | \ + OP_ALG_PKMODE_DST_REG_E) +#define OP_ALG_PKMODE_COPY_SSZ_B_N (OP_ALG_PKMODE_COPY_SSZ | \ + OP_ALG_PKMODE_SRC_REG_B | \ + OP_ALG_PKMODE_DST_REG_N) +#define OP_ALG_PKMODE_COPY_SSZ_N_A (OP_ALG_PKMODE_COPY_SSZ | \ + OP_ALG_PKMODE_SRC_REG_N | \ + OP_ALG_PKMODE_DST_REG_A) +#define OP_ALG_PKMODE_COPY_SSZ_N_B (OP_ALG_PKMODE_COPY_SSZ | \ + OP_ALG_PKMODE_SRC_REG_N | \ + OP_ALG_PKMODE_DST_REG_B) +#define OP_ALG_PKMODE_COPY_SSZ_N_E (OP_ALG_PKMODE_COPY_SSZ | \ + OP_ALG_PKMODE_SRC_REG_N | \ + OP_ALG_PKMODE_DST_REG_E) + +/* + * SEQ_IN_PTR Command Constructs + */ + +/* Release Buffers */ +#define SQIN_RBS BIT(26) + +/* Sequence pointer is really a descriptor */ +#define SQIN_INL BIT(25) + +/* Sequence pointer is a scatter-gather table */ +#define SQIN_SGF BIT(24) + +/* Appends to a previous pointer */ +#define SQIN_PRE BIT(23) + +/* Use extended length following pointer */ +#define SQIN_EXT BIT(22) + +/* Restore sequence with pointer/length */ +#define SQIN_RTO BIT(21) + +/* Replace job descriptor */ +#define SQIN_RJD BIT(20) + +/* Sequence Out Pointer - start a new input sequence using output sequence */ +#define SQIN_SOP BIT(19) + +#define SQIN_LEN_SHIFT 0 +#define SQIN_LEN_MASK (0xffff << SQIN_LEN_SHIFT) + +/* + * SEQ_OUT_PTR Command Constructs + */ + +/* Sequence pointer is a scatter-gather table */ +#define SQOUT_SGF BIT(24) + +/* Appends to a previous pointer */ +#define SQOUT_PRE BIT(23) + +/* Restore sequence with pointer/length */ +#define SQOUT_RTO BIT(21) + +/* + * Ignore length field, add current output frame length back to SOL register. + * Reset tracking length of bytes written to output frame. + * Must be used together with SQOUT_RTO. + */ +#define SQOUT_RST BIT(20) + +/* Allow "write safe" transactions for this Output Sequence */ +#define SQOUT_EWS BIT(19) + +/* Use extended length following pointer */ +#define SQOUT_EXT BIT(22) + +#define SQOUT_LEN_SHIFT 0 +#define SQOUT_LEN_MASK (0xffff << SQOUT_LEN_SHIFT) + +/* + * SIGNATURE Command Constructs + */ + +/* TYPE field is all that's relevant */ +#define SIGN_TYPE_SHIFT 16 +#define SIGN_TYPE_MASK (0x0f << SIGN_TYPE_SHIFT) + +#define SIGN_TYPE_FINAL (0x00 << SIGN_TYPE_SHIFT) +#define SIGN_TYPE_FINAL_RESTORE (0x01 << SIGN_TYPE_SHIFT) +#define SIGN_TYPE_FINAL_NONZERO (0x02 << SIGN_TYPE_SHIFT) +#define SIGN_TYPE_IMM_2 (0x0a << SIGN_TYPE_SHIFT) +#define SIGN_TYPE_IMM_3 (0x0b << SIGN_TYPE_SHIFT) +#define SIGN_TYPE_IMM_4 (0x0c << SIGN_TYPE_SHIFT) + +/* + * MOVE Command Constructs + */ + +#define MOVE_AUX_SHIFT 25 +#define MOVE_AUX_MASK (3 << MOVE_AUX_SHIFT) +#define MOVE_AUX_MS (2 << MOVE_AUX_SHIFT) +#define MOVE_AUX_LS (1 << MOVE_AUX_SHIFT) + +#define MOVE_WAITCOMP_SHIFT 24 +#define MOVE_WAITCOMP_MASK (1 << MOVE_WAITCOMP_SHIFT) +#define MOVE_WAITCOMP BIT(24) + +#define MOVE_SRC_SHIFT 20 +#define MOVE_SRC_MASK (0x0f << MOVE_SRC_SHIFT) +#define MOVE_SRC_CLASS1CTX (0x00 << MOVE_SRC_SHIFT) +#define MOVE_SRC_CLASS2CTX (0x01 << MOVE_SRC_SHIFT) +#define MOVE_SRC_OUTFIFO (0x02 << MOVE_SRC_SHIFT) +#define MOVE_SRC_DESCBUF (0x03 << MOVE_SRC_SHIFT) +#define MOVE_SRC_MATH0 (0x04 << MOVE_SRC_SHIFT) +#define MOVE_SRC_MATH1 (0x05 << MOVE_SRC_SHIFT) +#define MOVE_SRC_MATH2 (0x06 << MOVE_SRC_SHIFT) +#define MOVE_SRC_MATH3 (0x07 << MOVE_SRC_SHIFT) +#define MOVE_SRC_INFIFO (0x08 << MOVE_SRC_SHIFT) +#define MOVE_SRC_INFIFO_CL (0x09 << MOVE_SRC_SHIFT) +#define MOVE_SRC_INFIFO_NO_NFIFO (0x0a << MOVE_SRC_SHIFT) + +#define MOVE_DEST_SHIFT 16 +#define MOVE_DEST_MASK (0x0f << MOVE_DEST_SHIFT) +#define MOVE_DEST_CLASS1CTX (0x00 << MOVE_DEST_SHIFT) +#define MOVE_DEST_CLASS2CTX (0x01 << MOVE_DEST_SHIFT) +#define MOVE_DEST_OUTFIFO (0x02 << MOVE_DEST_SHIFT) +#define MOVE_DEST_DESCBUF (0x03 << MOVE_DEST_SHIFT) +#define MOVE_DEST_MATH0 (0x04 << MOVE_DEST_SHIFT) +#define MOVE_DEST_MATH1 (0x05 << MOVE_DEST_SHIFT) +#define MOVE_DEST_MATH2 (0x06 << MOVE_DEST_SHIFT) +#define MOVE_DEST_MATH3 (0x07 << MOVE_DEST_SHIFT) +#define MOVE_DEST_CLASS1INFIFO (0x08 << MOVE_DEST_SHIFT) +#define MOVE_DEST_CLASS2INFIFO (0x09 << MOVE_DEST_SHIFT) +#define MOVE_DEST_INFIFO (0x0a << MOVE_DEST_SHIFT) +#define MOVE_DEST_PK_A (0x0c << MOVE_DEST_SHIFT) +#define MOVE_DEST_CLASS1KEY (0x0d << MOVE_DEST_SHIFT) +#define MOVE_DEST_CLASS2KEY (0x0e << MOVE_DEST_SHIFT) +#define MOVE_DEST_ALTSOURCE (0x0f << MOVE_DEST_SHIFT) + +#define MOVE_OFFSET_SHIFT 8 +#define MOVE_OFFSET_MASK (0xff << MOVE_OFFSET_SHIFT) + +#define MOVE_LEN_SHIFT 0 +#define MOVE_LEN_MASK (0xff << MOVE_LEN_SHIFT) + +#define MOVELEN_MRSEL_SHIFT 0 +#define MOVELEN_MRSEL_MASK (0x3 << MOVE_LEN_SHIFT) +#define MOVELEN_MRSEL_MATH0 (0 << MOVELEN_MRSEL_SHIFT) +#define MOVELEN_MRSEL_MATH1 (1 << MOVELEN_MRSEL_SHIFT) +#define MOVELEN_MRSEL_MATH2 (2 << MOVELEN_MRSEL_SHIFT) +#define MOVELEN_MRSEL_MATH3 (3 << MOVELEN_MRSEL_SHIFT) + +#define MOVELEN_SIZE_SHIFT 6 +#define MOVELEN_SIZE_MASK (0x3 << MOVELEN_SIZE_SHIFT) +#define MOVELEN_SIZE_WORD (0x01 << MOVELEN_SIZE_SHIFT) +#define MOVELEN_SIZE_BYTE (0x02 << MOVELEN_SIZE_SHIFT) +#define MOVELEN_SIZE_DWORD (0x03 << MOVELEN_SIZE_SHIFT) + +/* + * MATH Command Constructs + */ + +#define MATH_IFB_SHIFT 26 +#define MATH_IFB_MASK (1 << MATH_IFB_SHIFT) +#define MATH_IFB BIT(26) + +#define MATH_NFU_SHIFT 25 +#define MATH_NFU_MASK (1 << MATH_NFU_SHIFT) +#define MATH_NFU BIT(25) + +/* STL for MATH, SSEL for MATHI */ +#define MATH_STL_SHIFT 24 +#define MATH_STL_MASK (1 << MATH_STL_SHIFT) +#define MATH_STL BIT(24) + +#define MATH_SSEL_SHIFT 24 +#define MATH_SSEL_MASK (1 << MATH_SSEL_SHIFT) +#define MATH_SSEL BIT(24) + +#define MATH_SWP_SHIFT 0 +#define MATH_SWP_MASK (1 << MATH_SWP_SHIFT) +#define MATH_SWP BIT(0) + +/* Function selectors */ +#define MATH_FUN_SHIFT 20 +#define MATH_FUN_MASK (0x0f << MATH_FUN_SHIFT) +#define MATH_FUN_ADD (0x00 << MATH_FUN_SHIFT) +#define MATH_FUN_ADDC (0x01 << MATH_FUN_SHIFT) +#define MATH_FUN_SUB (0x02 << MATH_FUN_SHIFT) +#define MATH_FUN_SUBB (0x03 << MATH_FUN_SHIFT) +#define MATH_FUN_OR (0x04 << MATH_FUN_SHIFT) +#define MATH_FUN_AND (0x05 << MATH_FUN_SHIFT) +#define MATH_FUN_XOR (0x06 << MATH_FUN_SHIFT) +#define MATH_FUN_LSHIFT (0x07 << MATH_FUN_SHIFT) +#define MATH_FUN_RSHIFT (0x08 << MATH_FUN_SHIFT) +#define MATH_FUN_SHLD (0x09 << MATH_FUN_SHIFT) +#define MATH_FUN_ZBYT (0x0a << MATH_FUN_SHIFT) /* ZBYT is for MATH */ +#define MATH_FUN_FBYT (0x0a << MATH_FUN_SHIFT) /* FBYT is for MATHI */ +#define MATH_FUN_BSWAP (0x0b << MATH_FUN_SHIFT) + +/* Source 0 selectors */ +#define MATH_SRC0_SHIFT 16 +#define MATH_SRC0_MASK (0x0f << MATH_SRC0_SHIFT) +#define MATH_SRC0_REG0 (0x00 << MATH_SRC0_SHIFT) +#define MATH_SRC0_REG1 (0x01 << MATH_SRC0_SHIFT) +#define MATH_SRC0_REG2 (0x02 << MATH_SRC0_SHIFT) +#define MATH_SRC0_REG3 (0x03 << MATH_SRC0_SHIFT) +#define MATH_SRC0_IMM (0x04 << MATH_SRC0_SHIFT) +#define MATH_SRC0_DPOVRD (0x07 << MATH_SRC0_SHIFT) +#define MATH_SRC0_SEQINLEN (0x08 << MATH_SRC0_SHIFT) +#define MATH_SRC0_SEQOUTLEN (0x09 << MATH_SRC0_SHIFT) +#define MATH_SRC0_VARSEQINLEN (0x0a << MATH_SRC0_SHIFT) +#define MATH_SRC0_VARSEQOUTLEN (0x0b << MATH_SRC0_SHIFT) +#define MATH_SRC0_ZERO (0x0c << MATH_SRC0_SHIFT) +#define MATH_SRC0_ONE (0x0f << MATH_SRC0_SHIFT) + +/* Source 1 selectors */ +#define MATH_SRC1_SHIFT 12 +#define MATHI_SRC1_SHIFT 16 +#define MATH_SRC1_MASK (0x0f << MATH_SRC1_SHIFT) +#define MATH_SRC1_REG0 (0x00 << MATH_SRC1_SHIFT) +#define MATH_SRC1_REG1 (0x01 << MATH_SRC1_SHIFT) +#define MATH_SRC1_REG2 (0x02 << MATH_SRC1_SHIFT) +#define MATH_SRC1_REG3 (0x03 << MATH_SRC1_SHIFT) +#define MATH_SRC1_IMM (0x04 << MATH_SRC1_SHIFT) +#define MATH_SRC1_DPOVRD (0x07 << MATH_SRC1_SHIFT) +#define MATH_SRC1_VARSEQINLEN (0x08 << MATH_SRC1_SHIFT) +#define MATH_SRC1_VARSEQOUTLEN (0x09 << MATH_SRC1_SHIFT) +#define MATH_SRC1_INFIFO (0x0a << MATH_SRC1_SHIFT) +#define MATH_SRC1_OUTFIFO (0x0b << MATH_SRC1_SHIFT) +#define MATH_SRC1_ONE (0x0c << MATH_SRC1_SHIFT) +#define MATH_SRC1_JOBSOURCE (0x0d << MATH_SRC1_SHIFT) +#define MATH_SRC1_ZERO (0x0f << MATH_SRC1_SHIFT) + +/* Destination selectors */ +#define MATH_DEST_SHIFT 8 +#define MATHI_DEST_SHIFT 12 +#define MATH_DEST_MASK (0x0f << MATH_DEST_SHIFT) +#define MATH_DEST_REG0 (0x00 << MATH_DEST_SHIFT) +#define MATH_DEST_REG1 (0x01 << MATH_DEST_SHIFT) +#define MATH_DEST_REG2 (0x02 << MATH_DEST_SHIFT) +#define MATH_DEST_REG3 (0x03 << MATH_DEST_SHIFT) +#define MATH_DEST_DPOVRD (0x07 << MATH_DEST_SHIFT) +#define MATH_DEST_SEQINLEN (0x08 << MATH_DEST_SHIFT) +#define MATH_DEST_SEQOUTLEN (0x09 << MATH_DEST_SHIFT) +#define MATH_DEST_VARSEQINLEN (0x0a << MATH_DEST_SHIFT) +#define MATH_DEST_VARSEQOUTLEN (0x0b << MATH_DEST_SHIFT) +#define MATH_DEST_NONE (0x0f << MATH_DEST_SHIFT) + +/* MATHI Immediate value */ +#define MATHI_IMM_SHIFT 4 +#define MATHI_IMM_MASK (0xff << MATHI_IMM_SHIFT) + +/* Length selectors */ +#define MATH_LEN_SHIFT 0 +#define MATH_LEN_MASK (0x0f << MATH_LEN_SHIFT) +#define MATH_LEN_1BYTE 0x01 +#define MATH_LEN_2BYTE 0x02 +#define MATH_LEN_4BYTE 0x04 +#define MATH_LEN_8BYTE 0x08 + +/* + * JUMP Command Constructs + */ + +#define JUMP_CLASS_SHIFT 25 +#define JUMP_CLASS_MASK (3 << JUMP_CLASS_SHIFT) +#define JUMP_CLASS_NONE 0 +#define JUMP_CLASS_CLASS1 (1 << JUMP_CLASS_SHIFT) +#define JUMP_CLASS_CLASS2 (2 << JUMP_CLASS_SHIFT) +#define JUMP_CLASS_BOTH (3 << JUMP_CLASS_SHIFT) + +#define JUMP_JSL_SHIFT 24 +#define JUMP_JSL_MASK (1 << JUMP_JSL_SHIFT) +#define JUMP_JSL BIT(24) + +#define JUMP_TYPE_SHIFT 20 +#define JUMP_TYPE_MASK (0x0f << JUMP_TYPE_SHIFT) +#define JUMP_TYPE_LOCAL (0x00 << JUMP_TYPE_SHIFT) +#define JUMP_TYPE_LOCAL_INC (0x01 << JUMP_TYPE_SHIFT) +#define JUMP_TYPE_GOSUB (0x02 << JUMP_TYPE_SHIFT) +#define JUMP_TYPE_LOCAL_DEC (0x03 << JUMP_TYPE_SHIFT) +#define JUMP_TYPE_NONLOCAL (0x04 << JUMP_TYPE_SHIFT) +#define JUMP_TYPE_RETURN (0x06 << JUMP_TYPE_SHIFT) +#define JUMP_TYPE_HALT (0x08 << JUMP_TYPE_SHIFT) +#define JUMP_TYPE_HALT_USER (0x0c << JUMP_TYPE_SHIFT) + +#define JUMP_TEST_SHIFT 16 +#define JUMP_TEST_MASK (0x03 << JUMP_TEST_SHIFT) +#define JUMP_TEST_ALL (0x00 << JUMP_TEST_SHIFT) +#define JUMP_TEST_INVALL (0x01 << JUMP_TEST_SHIFT) +#define JUMP_TEST_ANY (0x02 << JUMP_TEST_SHIFT) +#define JUMP_TEST_INVANY (0x03 << JUMP_TEST_SHIFT) + +/* Condition codes. JSL bit is factored in */ +#define JUMP_COND_SHIFT 8 +#define JUMP_COND_MASK ((0xff << JUMP_COND_SHIFT) | JUMP_JSL) +#define JUMP_COND_PK_0 BIT(15) +#define JUMP_COND_PK_GCD_1 BIT(14) +#define JUMP_COND_PK_PRIME BIT(13) +#define JUMP_COND_MATH_N BIT(11) +#define JUMP_COND_MATH_Z BIT(10) +#define JUMP_COND_MATH_C BIT(9) +#define JUMP_COND_MATH_NV BIT(8) + +#define JUMP_COND_JQP (BIT(15) | JUMP_JSL) +#define JUMP_COND_SHRD (BIT(14) | JUMP_JSL) +#define JUMP_COND_SELF (BIT(13) | JUMP_JSL) +#define JUMP_COND_CALM (BIT(12) | JUMP_JSL) +#define JUMP_COND_NIP (BIT(11) | JUMP_JSL) +#define JUMP_COND_NIFP (BIT(10) | JUMP_JSL) +#define JUMP_COND_NOP (BIT(9) | JUMP_JSL) +#define JUMP_COND_NCP (BIT(8) | JUMP_JSL) + +/* Source / destination selectors */ +#define JUMP_SRC_DST_SHIFT 12 +#define JUMP_SRC_DST_MASK (0x0f << JUMP_SRC_DST_SHIFT) +#define JUMP_SRC_DST_MATH0 (0x00 << JUMP_SRC_DST_SHIFT) +#define JUMP_SRC_DST_MATH1 (0x01 << JUMP_SRC_DST_SHIFT) +#define JUMP_SRC_DST_MATH2 (0x02 << JUMP_SRC_DST_SHIFT) +#define JUMP_SRC_DST_MATH3 (0x03 << JUMP_SRC_DST_SHIFT) +#define JUMP_SRC_DST_DPOVRD (0x07 << JUMP_SRC_DST_SHIFT) +#define JUMP_SRC_DST_SEQINLEN (0x08 << JUMP_SRC_DST_SHIFT) +#define JUMP_SRC_DST_SEQOUTLEN (0x09 << JUMP_SRC_DST_SHIFT) +#define JUMP_SRC_DST_VARSEQINLEN (0x0a << JUMP_SRC_DST_SHIFT) +#define JUMP_SRC_DST_VARSEQOUTLEN (0x0b << JUMP_SRC_DST_SHIFT) + +#define JUMP_OFFSET_SHIFT 0 +#define JUMP_OFFSET_MASK (0xff << JUMP_OFFSET_SHIFT) + +/* + * NFIFO ENTRY + * Data Constructs + * + */ +#define NFIFOENTRY_DEST_SHIFT 30 +#define NFIFOENTRY_DEST_MASK ((uint32_t)(3 << NFIFOENTRY_DEST_SHIFT)) +#define NFIFOENTRY_DEST_DECO (0 << NFIFOENTRY_DEST_SHIFT) +#define NFIFOENTRY_DEST_CLASS1 (1 << NFIFOENTRY_DEST_SHIFT) +#define NFIFOENTRY_DEST_CLASS2 ((uint32_t)(2 << NFIFOENTRY_DEST_SHIFT)) +#define NFIFOENTRY_DEST_BOTH ((uint32_t)(3 << NFIFOENTRY_DEST_SHIFT)) + +#define NFIFOENTRY_LC2_SHIFT 29 +#define NFIFOENTRY_LC2_MASK (1 << NFIFOENTRY_LC2_SHIFT) +#define NFIFOENTRY_LC2 BIT(29) + +#define NFIFOENTRY_LC1_SHIFT 28 +#define NFIFOENTRY_LC1_MASK (1 << NFIFOENTRY_LC1_SHIFT) +#define NFIFOENTRY_LC1 BIT(28) + +#define NFIFOENTRY_FC2_SHIFT 27 +#define NFIFOENTRY_FC2_MASK (1 << NFIFOENTRY_FC2_SHIFT) +#define NFIFOENTRY_FC2 BIT(27) + +#define NFIFOENTRY_FC1_SHIFT 26 +#define NFIFOENTRY_FC1_MASK (1 << NFIFOENTRY_FC1_SHIFT) +#define NFIFOENTRY_FC1 BIT(26) + +#define NFIFOENTRY_STYPE_SHIFT 24 +#define NFIFOENTRY_STYPE_MASK (3 << NFIFOENTRY_STYPE_SHIFT) +#define NFIFOENTRY_STYPE_DFIFO (0 << NFIFOENTRY_STYPE_SHIFT) +#define NFIFOENTRY_STYPE_OFIFO (1 << NFIFOENTRY_STYPE_SHIFT) +#define NFIFOENTRY_STYPE_PAD (2 << NFIFOENTRY_STYPE_SHIFT) +#define NFIFOENTRY_STYPE_SNOOP (3 << NFIFOENTRY_STYPE_SHIFT) +#define NFIFOENTRY_STYPE_ALTSOURCE ((0 << NFIFOENTRY_STYPE_SHIFT) \ + | NFIFOENTRY_AST) +#define NFIFOENTRY_STYPE_OFIFO_SYNC ((1 << NFIFOENTRY_STYPE_SHIFT) \ + | NFIFOENTRY_AST) +#define NFIFOENTRY_STYPE_SNOOP_ALT ((3 << NFIFOENTRY_STYPE_SHIFT) \ + | NFIFOENTRY_AST) + +#define NFIFOENTRY_DTYPE_SHIFT 20 +#define NFIFOENTRY_DTYPE_MASK (0xF << NFIFOENTRY_DTYPE_SHIFT) + +#define NFIFOENTRY_DTYPE_SBOX (0x0 << NFIFOENTRY_DTYPE_SHIFT) +#define NFIFOENTRY_DTYPE_AAD (0x1 << NFIFOENTRY_DTYPE_SHIFT) +#define NFIFOENTRY_DTYPE_IV (0x2 << NFIFOENTRY_DTYPE_SHIFT) +#define NFIFOENTRY_DTYPE_SAD (0x3 << NFIFOENTRY_DTYPE_SHIFT) +#define NFIFOENTRY_DTYPE_ICV (0xA << NFIFOENTRY_DTYPE_SHIFT) +#define NFIFOENTRY_DTYPE_SKIP (0xE << NFIFOENTRY_DTYPE_SHIFT) +#define NFIFOENTRY_DTYPE_MSG (0xF << NFIFOENTRY_DTYPE_SHIFT) + +#define NFIFOENTRY_DTYPE_PK_A0 (0x0 << NFIFOENTRY_DTYPE_SHIFT) +#define NFIFOENTRY_DTYPE_PK_A1 (0x1 << NFIFOENTRY_DTYPE_SHIFT) +#define NFIFOENTRY_DTYPE_PK_A2 (0x2 << NFIFOENTRY_DTYPE_SHIFT) +#define NFIFOENTRY_DTYPE_PK_A3 (0x3 << NFIFOENTRY_DTYPE_SHIFT) +#define NFIFOENTRY_DTYPE_PK_B0 (0x4 << NFIFOENTRY_DTYPE_SHIFT) +#define NFIFOENTRY_DTYPE_PK_B1 (0x5 << NFIFOENTRY_DTYPE_SHIFT) +#define NFIFOENTRY_DTYPE_PK_B2 (0x6 << NFIFOENTRY_DTYPE_SHIFT) +#define NFIFOENTRY_DTYPE_PK_B3 (0x7 << NFIFOENTRY_DTYPE_SHIFT) +#define NFIFOENTRY_DTYPE_PK_N (0x8 << NFIFOENTRY_DTYPE_SHIFT) +#define NFIFOENTRY_DTYPE_PK_E (0x9 << NFIFOENTRY_DTYPE_SHIFT) +#define NFIFOENTRY_DTYPE_PK_A (0xC << NFIFOENTRY_DTYPE_SHIFT) +#define NFIFOENTRY_DTYPE_PK_B (0xD << NFIFOENTRY_DTYPE_SHIFT) + +#define NFIFOENTRY_BND_SHIFT 19 +#define NFIFOENTRY_BND_MASK (1 << NFIFOENTRY_BND_SHIFT) +#define NFIFOENTRY_BND BIT(19) + +#define NFIFOENTRY_PTYPE_SHIFT 16 +#define NFIFOENTRY_PTYPE_MASK (0x7 << NFIFOENTRY_PTYPE_SHIFT) + +#define NFIFOENTRY_PTYPE_ZEROS (0x0 << NFIFOENTRY_PTYPE_SHIFT) +#define NFIFOENTRY_PTYPE_RND_NOZEROS (0x1 << NFIFOENTRY_PTYPE_SHIFT) +#define NFIFOENTRY_PTYPE_INCREMENT (0x2 << NFIFOENTRY_PTYPE_SHIFT) +#define NFIFOENTRY_PTYPE_RND (0x3 << NFIFOENTRY_PTYPE_SHIFT) +#define NFIFOENTRY_PTYPE_ZEROS_NZ (0x4 << NFIFOENTRY_PTYPE_SHIFT) +#define NFIFOENTRY_PTYPE_RND_NZ_LZ (0x5 << NFIFOENTRY_PTYPE_SHIFT) +#define NFIFOENTRY_PTYPE_N (0x6 << NFIFOENTRY_PTYPE_SHIFT) +#define NFIFOENTRY_PTYPE_RND_NZ_N (0x7 << NFIFOENTRY_PTYPE_SHIFT) + +#define NFIFOENTRY_OC_SHIFT 15 +#define NFIFOENTRY_OC_MASK (1 << NFIFOENTRY_OC_SHIFT) +#define NFIFOENTRY_OC BIT(15) + +#define NFIFOENTRY_PR_SHIFT 15 +#define NFIFOENTRY_PR_MASK (1 << NFIFOENTRY_PR_SHIFT) +#define NFIFOENTRY_PR BIT(15) + +#define NFIFOENTRY_AST_SHIFT 14 +#define NFIFOENTRY_AST_MASK (1 << NFIFOENTRY_AST_SHIFT) +#define NFIFOENTRY_AST BIT(14) + +#define NFIFOENTRY_BM_SHIFT 11 +#define NFIFOENTRY_BM_MASK (1 << NFIFOENTRY_BM_SHIFT) +#define NFIFOENTRY_BM BIT(11) + +#define NFIFOENTRY_PS_SHIFT 10 +#define NFIFOENTRY_PS_MASK (1 << NFIFOENTRY_PS_SHIFT) +#define NFIFOENTRY_PS BIT(10) + +#define NFIFOENTRY_DLEN_SHIFT 0 +#define NFIFOENTRY_DLEN_MASK (0xFFF << NFIFOENTRY_DLEN_SHIFT) + +#define NFIFOENTRY_PLEN_SHIFT 0 +#define NFIFOENTRY_PLEN_MASK (0xFF << NFIFOENTRY_PLEN_SHIFT) + +/* Append Load Immediate Command */ +#define FD_CMD_APPEND_LOAD_IMMEDIATE BIT(31) + +/* Set SEQ LIODN equal to the Non-SEQ LIODN for the job */ +#define FD_CMD_SET_SEQ_LIODN_EQUAL_NONSEQ_LIODN BIT(30) + +/* Frame Descriptor Command for Replacement Job Descriptor */ +#define FD_CMD_REPLACE_JOB_DESC BIT(29) + +#endif /* __RTA_DESC_H__ */ diff --git a/dpdk/drivers/crypto/dpaa2_sec/hw/desc/algo.h b/dpdk/drivers/crypto/dpaa2_sec/hw/desc/algo.h new file mode 100644 index 00000000..5ae3a1ac --- /dev/null +++ b/dpdk/drivers/crypto/dpaa2_sec/hw/desc/algo.h @@ -0,0 +1,679 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2008-2016 Freescale Semiconductor Inc. + * Copyright 2016 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __DESC_ALGO_H__ +#define __DESC_ALGO_H__ + +#include "hw/rta.h" +#include "common.h" + +/** + * DOC: Algorithms - Shared Descriptor Constructors + * + * Shared descriptors for algorithms (i.e. not for protocols). + */ + +/** + * cnstr_shdsc_snow_f8 - SNOW/f8 (UEA2) as a shared descriptor + * @descbuf: pointer to descriptor-under-construction buffer + * @ps: if 36/40bit addressing is desired, this parameter must be true + * @swap: must be true when core endianness doesn't match SEC endianness + * @cipherdata: pointer to block cipher transform definitions + * @dir: Cipher direction (DIR_ENC/DIR_DEC) + * @count: UEA2 count value (32 bits) + * @bearer: UEA2 bearer ID (5 bits) + * @direction: UEA2 direction (1 bit) + * + * Return: size of descriptor written in words or negative number on error + */ +static inline int +cnstr_shdsc_snow_f8(uint32_t *descbuf, bool ps, bool swap, + struct alginfo *cipherdata, uint8_t dir, + uint32_t count, uint8_t bearer, uint8_t direction) +{ + struct program prg; + struct program *p = &prg; + uint32_t ct = count; + uint8_t br = bearer; + uint8_t dr = direction; + uint32_t context[2] = {ct, (br << 27) | (dr << 26)}; + + PROGRAM_CNTXT_INIT(p, descbuf, 0); + if (swap) { + PROGRAM_SET_BSWAP(p); + + context[0] = swab32(context[0]); + context[1] = swab32(context[1]); + } + + if (ps) + PROGRAM_SET_36BIT_ADDR(p); + SHR_HDR(p, SHR_ALWAYS, 1, 0); + + KEY(p, KEY1, cipherdata->key_enc_flags, cipherdata->key, + cipherdata->keylen, INLINE_KEY(cipherdata)); + MATHB(p, SEQINSZ, SUB, MATH2, VSEQINSZ, 4, 0); + MATHB(p, SEQINSZ, SUB, MATH2, VSEQOUTSZ, 4, 0); + ALG_OPERATION(p, OP_ALG_ALGSEL_SNOW_F8, OP_ALG_AAI_F8, + OP_ALG_AS_INITFINAL, 0, dir); + LOAD(p, (uintptr_t)context, CONTEXT1, 0, 8, IMMED | COPY); + SEQFIFOLOAD(p, MSG1, 0, VLF | LAST1); + SEQFIFOSTORE(p, MSG, 0, 0, VLF); + + return PROGRAM_FINALIZE(p); +} + +/** + * cnstr_shdsc_snow_f9 - SNOW/f9 (UIA2) as a shared descriptor + * @descbuf: pointer to descriptor-under-construction buffer + * @ps: if 36/40bit addressing is desired, this parameter must be true + * @swap: must be true when core endianness doesn't match SEC endianness + * @authdata: pointer to authentication transform definitions + * @dir: cipher direction (DIR_ENC/DIR_DEC) + * @count: UEA2 count value (32 bits) + * @fresh: UEA2 fresh value ID (32 bits) + * @direction: UEA2 direction (1 bit) + * @datalen: size of data + * + * Return: size of descriptor written in words or negative number on error + */ +static inline int +cnstr_shdsc_snow_f9(uint32_t *descbuf, bool ps, bool swap, + struct alginfo *authdata, uint8_t dir, uint32_t count, + uint32_t fresh, uint8_t direction, uint32_t datalen) +{ + struct program prg; + struct program *p = &prg; + uint64_t ct = count; + uint64_t fr = fresh; + uint64_t dr = direction; + uint64_t context[2]; + + context[0] = (ct << 32) | (dr << 26); + context[1] = fr << 32; + + PROGRAM_CNTXT_INIT(p, descbuf, 0); + if (swap) { + PROGRAM_SET_BSWAP(p); + + context[0] = swab64(context[0]); + context[1] = swab64(context[1]); + } + if (ps) + PROGRAM_SET_36BIT_ADDR(p); + SHR_HDR(p, SHR_ALWAYS, 1, 0); + + KEY(p, KEY2, authdata->key_enc_flags, authdata->key, authdata->keylen, + INLINE_KEY(authdata)); + MATHB(p, SEQINSZ, SUB, MATH2, VSEQINSZ, 4, 0); + ALG_OPERATION(p, OP_ALG_ALGSEL_SNOW_F9, OP_ALG_AAI_F9, + OP_ALG_AS_INITFINAL, 0, dir); + LOAD(p, (uintptr_t)context, CONTEXT2, 0, 16, IMMED | COPY); + SEQFIFOLOAD(p, BIT_DATA, datalen, CLASS2 | LAST2); + /* Save lower half of MAC out into a 32-bit sequence */ + SEQSTORE(p, CONTEXT2, 0, 4, 0); + + return PROGRAM_FINALIZE(p); +} + +/** + * cnstr_shdsc_blkcipher - block cipher transformation + * @descbuf: pointer to descriptor-under-construction buffer + * @ps: if 36/40bit addressing is desired, this parameter must be true + * @swap: must be true when core endianness doesn't match SEC endianness + * @cipherdata: pointer to block cipher transform definitions + * Valid algorithm values one of OP_ALG_ALGSEL_* {DES, 3DES, AES} + * Valid modes for: + * AES: OP_ALG_AAI_* {CBC, CTR} + * DES, 3DES: OP_ALG_AAI_CBC + * @iv: IV data; if NULL, "ivlen" bytes from the input frame will be read as IV + * @ivlen: IV length + * @dir: DIR_ENC/DIR_DEC + * + * Return: size of descriptor written in words or negative number on error + */ +static inline int +cnstr_shdsc_blkcipher(uint32_t *descbuf, bool ps, bool swap, + struct alginfo *cipherdata, uint8_t *iv, + uint32_t ivlen, uint8_t dir) +{ + struct program prg; + struct program *p = &prg; + uint32_t iv_off = 0; + const bool need_dk = (dir == DIR_DEC) && + (cipherdata->algtype == OP_ALG_ALGSEL_AES) && + (cipherdata->algmode == OP_ALG_AAI_CBC); + LABEL(keyjmp); + LABEL(skipdk); + REFERENCE(pkeyjmp); + REFERENCE(pskipdk); + + PROGRAM_CNTXT_INIT(p, descbuf, 0); + if (swap) + PROGRAM_SET_BSWAP(p); + if (ps) + PROGRAM_SET_36BIT_ADDR(p); + SHR_HDR(p, SHR_SERIAL, 1, SC); + + pkeyjmp = JUMP(p, keyjmp, LOCAL_JUMP, ALL_TRUE, SHRD); + /* Insert Key */ + KEY(p, KEY1, cipherdata->key_enc_flags, cipherdata->key, + cipherdata->keylen, INLINE_KEY(cipherdata)); + + if (need_dk) { + ALG_OPERATION(p, cipherdata->algtype, cipherdata->algmode, + OP_ALG_AS_INITFINAL, ICV_CHECK_DISABLE, dir); + + pskipdk = JUMP(p, skipdk, LOCAL_JUMP, ALL_TRUE, 0); + } + SET_LABEL(p, keyjmp); + + if (need_dk) { + ALG_OPERATION(p, OP_ALG_ALGSEL_AES, cipherdata->algmode | + OP_ALG_AAI_DK, OP_ALG_AS_INITFINAL, + ICV_CHECK_DISABLE, dir); + SET_LABEL(p, skipdk); + } else { + ALG_OPERATION(p, cipherdata->algtype, cipherdata->algmode, + OP_ALG_AS_INITFINAL, ICV_CHECK_DISABLE, dir); + } + + if (cipherdata->algmode == OP_ALG_AAI_CTR) + iv_off = 16; + + if (iv) + /* IV load, convert size */ + LOAD(p, (uintptr_t)iv, CONTEXT1, iv_off, ivlen, IMMED | COPY); + else + /* IV is present first before the actual message */ + SEQLOAD(p, CONTEXT1, iv_off, ivlen, 0); + + MATHB(p, SEQINSZ, SUB, MATH2, VSEQINSZ, 4, 0); + MATHB(p, SEQINSZ, SUB, MATH2, VSEQOUTSZ, 4, 0); + + /* Insert sequence load/store with VLF */ + SEQFIFOLOAD(p, MSG1, 0, VLF | LAST1); + SEQFIFOSTORE(p, MSG, 0, 0, VLF); + + PATCH_JUMP(p, pkeyjmp, keyjmp); + if (need_dk) + PATCH_JUMP(p, pskipdk, skipdk); + + return PROGRAM_FINALIZE(p); +} + +/** + * cnstr_shdsc_hmac - HMAC shared + * @descbuf: pointer to descriptor-under-construction buffer + * @ps: if 36/40bit addressing is desired, this parameter must be true + * @swap: must be true when core endianness doesn't match SEC endianness + * @authdata: pointer to authentication transform definitions; + * message digest algorithm: OP_ALG_ALGSEL_MD5/ SHA1-512. + * @do_icv: 0 if ICV checking is not desired, any other value if ICV checking + * is needed for all the packets processed by this shared descriptor + * @trunc_len: Length of the truncated ICV to be written in the output buffer, 0 + * if no truncation is needed + * + * Note: There's no support for keys longer than the block size of the + * underlying hash function, according to the selected algorithm. + * + * Return: size of descriptor written in words or negative number on error + */ +static inline int +cnstr_shdsc_hmac(uint32_t *descbuf, bool ps, bool swap, + struct alginfo *authdata, uint8_t do_icv, + uint8_t trunc_len) +{ + struct program prg; + struct program *p = &prg; + uint8_t storelen, opicv, dir; + LABEL(keyjmp); + LABEL(jmpprecomp); + REFERENCE(pkeyjmp); + REFERENCE(pjmpprecomp); + + /* Compute fixed-size store based on alg selection */ + switch (authdata->algtype) { + case OP_ALG_ALGSEL_MD5: + storelen = 16; + break; + case OP_ALG_ALGSEL_SHA1: + storelen = 20; + break; + case OP_ALG_ALGSEL_SHA224: + storelen = 28; + break; + case OP_ALG_ALGSEL_SHA256: + storelen = 32; + break; + case OP_ALG_ALGSEL_SHA384: + storelen = 48; + break; + case OP_ALG_ALGSEL_SHA512: + storelen = 64; + break; + default: + return -EINVAL; + } + + trunc_len = trunc_len && (trunc_len < storelen) ? trunc_len : storelen; + + opicv = do_icv ? ICV_CHECK_ENABLE : ICV_CHECK_DISABLE; + dir = do_icv ? DIR_DEC : DIR_ENC; + + PROGRAM_CNTXT_INIT(p, descbuf, 0); + if (swap) + PROGRAM_SET_BSWAP(p); + if (ps) + PROGRAM_SET_36BIT_ADDR(p); + SHR_HDR(p, SHR_SERIAL, 1, SC); + + pkeyjmp = JUMP(p, keyjmp, LOCAL_JUMP, ALL_TRUE, SHRD); + KEY(p, KEY2, authdata->key_enc_flags, authdata->key, authdata->keylen, + INLINE_KEY(authdata)); + + /* Do operation */ + ALG_OPERATION(p, authdata->algtype, OP_ALG_AAI_HMAC, + OP_ALG_AS_INITFINAL, opicv, dir); + + pjmpprecomp = JUMP(p, jmpprecomp, LOCAL_JUMP, ALL_TRUE, 0); + SET_LABEL(p, keyjmp); + + ALG_OPERATION(p, authdata->algtype, OP_ALG_AAI_HMAC_PRECOMP, + OP_ALG_AS_INITFINAL, opicv, dir); + + SET_LABEL(p, jmpprecomp); + + /* compute sequences */ + if (opicv == ICV_CHECK_ENABLE) + MATHB(p, SEQINSZ, SUB, trunc_len, VSEQINSZ, 4, IMMED2); + else + MATHB(p, SEQINSZ, SUB, MATH2, VSEQINSZ, 4, 0); + + /* Do load (variable length) */ + SEQFIFOLOAD(p, MSG2, 0, VLF | LAST2); + + if (opicv == ICV_CHECK_ENABLE) + SEQFIFOLOAD(p, ICV2, trunc_len, LAST2); + else + SEQSTORE(p, CONTEXT2, 0, trunc_len, 0); + + PATCH_JUMP(p, pkeyjmp, keyjmp); + PATCH_JUMP(p, pjmpprecomp, jmpprecomp); + + return PROGRAM_FINALIZE(p); +} + +/** + * cnstr_shdsc_kasumi_f8 - KASUMI F8 (Confidentiality) as a shared descriptor + * (ETSI "Document 1: f8 and f9 specification") + * @descbuf: pointer to descriptor-under-construction buffer + * @ps: if 36/40bit addressing is desired, this parameter must be true + * @swap: must be true when core endianness doesn't match SEC endianness + * @cipherdata: pointer to block cipher transform definitions + * @dir: cipher direction (DIR_ENC/DIR_DEC) + * @count: count value (32 bits) + * @bearer: bearer ID (5 bits) + * @direction: direction (1 bit) + * + * Return: size of descriptor written in words or negative number on error + */ +static inline int +cnstr_shdsc_kasumi_f8(uint32_t *descbuf, bool ps, bool swap, + struct alginfo *cipherdata, uint8_t dir, + uint32_t count, uint8_t bearer, uint8_t direction) +{ + struct program prg; + struct program *p = &prg; + uint64_t ct = count; + uint64_t br = bearer; + uint64_t dr = direction; + uint32_t context[2] = { ct, (br << 27) | (dr << 26) }; + + PROGRAM_CNTXT_INIT(p, descbuf, 0); + if (swap) { + PROGRAM_SET_BSWAP(p); + + context[0] = swab32(context[0]); + context[1] = swab32(context[1]); + } + if (ps) + PROGRAM_SET_36BIT_ADDR(p); + SHR_HDR(p, SHR_ALWAYS, 1, 0); + + KEY(p, KEY1, cipherdata->key_enc_flags, cipherdata->key, + cipherdata->keylen, INLINE_KEY(cipherdata)); + MATHB(p, SEQINSZ, SUB, MATH2, VSEQINSZ, 4, 0); + MATHB(p, SEQINSZ, SUB, MATH2, VSEQOUTSZ, 4, 0); + ALG_OPERATION(p, OP_ALG_ALGSEL_KASUMI, OP_ALG_AAI_F8, + OP_ALG_AS_INITFINAL, 0, dir); + LOAD(p, (uintptr_t)context, CONTEXT1, 0, 8, IMMED | COPY); + SEQFIFOLOAD(p, MSG1, 0, VLF | LAST1); + SEQFIFOSTORE(p, MSG, 0, 0, VLF); + + return PROGRAM_FINALIZE(p); +} + +/** + * cnstr_shdsc_kasumi_f9 - KASUMI F9 (Integrity) as a shared descriptor + * (ETSI "Document 1: f8 and f9 specification") + * @descbuf: pointer to descriptor-under-construction buffer + * @ps: if 36/40bit addressing is desired, this parameter must be true + * @swap: must be true when core endianness doesn't match SEC endianness + * @authdata: pointer to authentication transform definitions + * @dir: cipher direction (DIR_ENC/DIR_DEC) + * @count: count value (32 bits) + * @fresh: fresh value ID (32 bits) + * @direction: direction (1 bit) + * @datalen: size of data + * + * Return: size of descriptor written in words or negative number on error + */ +static inline int +cnstr_shdsc_kasumi_f9(uint32_t *descbuf, bool ps, bool swap, + struct alginfo *authdata, uint8_t dir, + uint32_t count, uint32_t fresh, uint8_t direction, + uint32_t datalen) +{ + struct program prg; + struct program *p = &prg; + uint16_t ctx_offset = 16; + uint32_t context[6] = {count, direction << 26, fresh, 0, 0, 0}; + + PROGRAM_CNTXT_INIT(p, descbuf, 0); + if (swap) { + PROGRAM_SET_BSWAP(p); + + context[0] = swab32(context[0]); + context[1] = swab32(context[1]); + context[2] = swab32(context[2]); + } + if (ps) + PROGRAM_SET_36BIT_ADDR(p); + SHR_HDR(p, SHR_ALWAYS, 1, 0); + + KEY(p, KEY1, authdata->key_enc_flags, authdata->key, authdata->keylen, + INLINE_KEY(authdata)); + MATHB(p, SEQINSZ, SUB, MATH2, VSEQINSZ, 4, 0); + ALG_OPERATION(p, OP_ALG_ALGSEL_KASUMI, OP_ALG_AAI_F9, + OP_ALG_AS_INITFINAL, 0, dir); + LOAD(p, (uintptr_t)context, CONTEXT1, 0, 24, IMMED | COPY); + SEQFIFOLOAD(p, BIT_DATA, datalen, CLASS1 | LAST1); + /* Save output MAC of DWORD 2 into a 32-bit sequence */ + SEQSTORE(p, CONTEXT1, ctx_offset, 4, 0); + + return PROGRAM_FINALIZE(p); +} + +/** + * cnstr_shdsc_gcm_encap - AES-GCM encap as a shared descriptor + * @descbuf: pointer to descriptor-under-construction buffer + * @ps: if 36/40bit addressing is desired, this parameter must be true + * @swap: must be true when core endianness doesn't match SEC endianness + * @cipherdata: pointer to block cipher transform definitions + * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with + * OP_ALG_AAI_GCM. + * @ivlen: Initialization vector length + * @icvsize: integrity check value (ICV) size (truncated or full) + * + * Return: size of descriptor written in words or negative number on error + */ +static inline int +cnstr_shdsc_gcm_encap(uint32_t *descbuf, bool ps, bool swap, + struct alginfo *cipherdata, + uint32_t ivlen, uint32_t icvsize) +{ + struct program prg; + struct program *p = &prg; + + LABEL(keyjmp); + LABEL(zeroassocjump2); + LABEL(zeroassocjump1); + LABEL(zeropayloadjump); + REFERENCE(pkeyjmp); + REFERENCE(pzeroassocjump2); + REFERENCE(pzeroassocjump1); + REFERENCE(pzeropayloadjump); + + PROGRAM_CNTXT_INIT(p, descbuf, 0); + + if (swap) + PROGRAM_SET_BSWAP(p); + if (ps) + PROGRAM_SET_36BIT_ADDR(p); + + SHR_HDR(p, SHR_SERIAL, 1, SC); + + pkeyjmp = JUMP(p, keyjmp, LOCAL_JUMP, ALL_TRUE, SELF | SHRD); + /* Insert Key */ + KEY(p, KEY1, cipherdata->key_enc_flags, cipherdata->key, + cipherdata->keylen, INLINE_KEY(cipherdata)); + + SET_LABEL(p, keyjmp); + + /* class 1 operation */ + ALG_OPERATION(p, cipherdata->algtype, cipherdata->algmode, + OP_ALG_AS_INITFINAL, ICV_CHECK_DISABLE, DIR_ENC); + + MATHB(p, DPOVRD, AND, 0x7fffffff, MATH3, 4, IMMED2); + + /* if assoclen + cryptlen is ZERO, skip to ICV write */ + MATHB(p, SEQINSZ, SUB, ivlen, VSEQOUTSZ, 4, IMMED2); + pzeroassocjump2 = JUMP(p, zeroassocjump2, LOCAL_JUMP, ALL_TRUE, MATH_Z); + + SEQFIFOLOAD(p, IV1, ivlen, FLUSH1); + + /* if assoclen is ZERO, skip reading the assoc data */ + MATHB(p, ZERO, ADD, MATH3, VSEQINSZ, 4, 0); + pzeroassocjump1 = JUMP(p, zeroassocjump1, LOCAL_JUMP, ALL_TRUE, MATH_Z); + + MATHB(p, ZERO, ADD, MATH3, VSEQOUTSZ, 4, 0); + + /* skip assoc data */ + SEQFIFOSTORE(p, SKIP, 0, 0, VLF); + + /* cryptlen = seqinlen - assoclen */ + MATHB(p, SEQINSZ, SUB, MATH3, VSEQOUTSZ, 4, 0); + + /* if cryptlen is ZERO jump to zero-payload commands */ + pzeropayloadjump = JUMP(p, zeropayloadjump, LOCAL_JUMP, ALL_TRUE, + MATH_Z); + + /* read assoc data */ + SEQFIFOLOAD(p, AAD1, 0, CLASS1 | VLF | FLUSH1); + SET_LABEL(p, zeroassocjump1); + + MATHB(p, SEQINSZ, SUB, MATH0, VSEQINSZ, 4, 0); + + /* write encrypted data */ + SEQFIFOSTORE(p, MSG, 0, 0, VLF); + + /* read payload data */ + SEQFIFOLOAD(p, MSG1, 0, CLASS1 | VLF | LAST1); + + /* jump the zero-payload commands */ + JUMP(p, 4, LOCAL_JUMP, ALL_TRUE, 0); + + /* zero-payload commands */ + SET_LABEL(p, zeropayloadjump); + + /* read assoc data */ + SEQFIFOLOAD(p, AAD1, 0, CLASS1 | VLF | LAST1); + + JUMP(p, 2, LOCAL_JUMP, ALL_TRUE, 0); + + /* There is no input data */ + SET_LABEL(p, zeroassocjump2); + + SEQFIFOLOAD(p, IV1, ivlen, FLUSH1 | LAST1); + + /* write ICV */ + SEQSTORE(p, CONTEXT1, 0, icvsize, 0); + + PATCH_JUMP(p, pkeyjmp, keyjmp); + PATCH_JUMP(p, pzeroassocjump2, zeroassocjump2); + PATCH_JUMP(p, pzeroassocjump1, zeroassocjump1); + PATCH_JUMP(p, pzeropayloadjump, zeropayloadjump); + + return PROGRAM_FINALIZE(p); +} + +/** + * cnstr_shdsc_gcm_decap - AES-GCM decap as a shared descriptor + * @descbuf: pointer to descriptor-under-construction buffer + * @ps: if 36/40bit addressing is desired, this parameter must be true + * @swap: must be true when core endianness doesn't match SEC endianness + * @cipherdata: pointer to block cipher transform definitions + * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with + * OP_ALG_AAI_GCM. + * @icvsize: integrity check value (ICV) size (truncated or full) + * + * Return: size of descriptor written in words or negative number on error + */ +static inline int +cnstr_shdsc_gcm_decap(uint32_t *descbuf, bool ps, bool swap, + struct alginfo *cipherdata, + uint32_t ivlen, uint32_t icvsize) +{ + struct program prg; + struct program *p = &prg; + + LABEL(keyjmp); + LABEL(zeroassocjump1); + LABEL(zeropayloadjump); + REFERENCE(pkeyjmp); + REFERENCE(pzeroassocjump1); + REFERENCE(pzeropayloadjump); + + PROGRAM_CNTXT_INIT(p, descbuf, 0); + + if (swap) + PROGRAM_SET_BSWAP(p); + if (ps) + PROGRAM_SET_36BIT_ADDR(p); + + SHR_HDR(p, SHR_SERIAL, 1, SC); + + pkeyjmp = JUMP(p, keyjmp, LOCAL_JUMP, ALL_TRUE, SELF | SHRD); + /* Insert Key */ + KEY(p, KEY1, cipherdata->key_enc_flags, cipherdata->key, + cipherdata->keylen, INLINE_KEY(cipherdata)); + + SET_LABEL(p, keyjmp); + + /* class 1 operation */ + ALG_OPERATION(p, cipherdata->algtype, cipherdata->algmode, + OP_ALG_AS_INITFINAL, ICV_CHECK_ENABLE, DIR_DEC); + + MATHB(p, DPOVRD, AND, 0x7fffffff, MATH3, 4, IMMED2); + SEQFIFOLOAD(p, IV1, ivlen, FLUSH1); + + /* if assoclen is ZERO, skip reading the assoc data */ + MATHB(p, ZERO, ADD, MATH3, VSEQINSZ, 4, 0); + pzeroassocjump1 = JUMP(p, zeroassocjump1, LOCAL_JUMP, ALL_TRUE, MATH_Z); + + MATHB(p, ZERO, ADD, MATH3, VSEQOUTSZ, 4, 0); + + /* skip assoc data */ + SEQFIFOSTORE(p, SKIP, 0, 0, VLF); + + /* read assoc data */ + SEQFIFOLOAD(p, AAD1, 0, CLASS1 | VLF | FLUSH1); + + SET_LABEL(p, zeroassocjump1); + + /* cryptlen = seqoutlen - assoclen */ + MATHB(p, SEQOUTSZ, SUB, MATH0, VSEQINSZ, 4, 0); + + /* jump to zero-payload command if cryptlen is zero */ + pzeropayloadjump = JUMP(p, zeropayloadjump, LOCAL_JUMP, ALL_TRUE, + MATH_Z); + + MATHB(p, SEQOUTSZ, SUB, MATH0, VSEQOUTSZ, 4, 0); + + /* store encrypted data */ + SEQFIFOSTORE(p, MSG, 0, 0, VLF); + + /* read payload data */ + SEQFIFOLOAD(p, MSG1, 0, CLASS1 | VLF | FLUSH1); + + /* zero-payload command */ + SET_LABEL(p, zeropayloadjump); + + /* read ICV */ + SEQFIFOLOAD(p, ICV1, icvsize, CLASS1 | LAST1); + + PATCH_JUMP(p, pkeyjmp, keyjmp); + PATCH_JUMP(p, pzeroassocjump1, zeroassocjump1); + PATCH_JUMP(p, pzeropayloadjump, zeropayloadjump); + + return PROGRAM_FINALIZE(p); +} + +/** + * cnstr_shdsc_crc - CRC32 Accelerator (IEEE 802 CRC32 protocol mode) + * @descbuf: pointer to descriptor-under-construction buffer + * @swap: must be true when core endianness doesn't match SEC endianness + * + * Return: size of descriptor written in words or negative number on error + */ +static inline int +cnstr_shdsc_crc(uint32_t *descbuf, bool swap) +{ + struct program prg; + struct program *p = &prg; + + PROGRAM_CNTXT_INIT(p, descbuf, 0); + if (swap) + PROGRAM_SET_BSWAP(p); + + SHR_HDR(p, SHR_ALWAYS, 1, 0); + + MATHB(p, SEQINSZ, SUB, MATH2, VSEQINSZ, 4, 0); + ALG_OPERATION(p, OP_ALG_ALGSEL_CRC, + OP_ALG_AAI_802 | OP_ALG_AAI_DOC, + OP_ALG_AS_FINALIZE, 0, DIR_ENC); + SEQFIFOLOAD(p, MSG2, 0, VLF | LAST2); + SEQSTORE(p, CONTEXT2, 0, 4, 0); + + return PROGRAM_FINALIZE(p); +} + +#endif /* __DESC_ALGO_H__ */ diff --git a/dpdk/drivers/crypto/dpaa2_sec/hw/desc/common.h b/dpdk/drivers/crypto/dpaa2_sec/hw/desc/common.h new file mode 100644 index 00000000..c2ac99be --- /dev/null +++ b/dpdk/drivers/crypto/dpaa2_sec/hw/desc/common.h @@ -0,0 +1,131 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2008-2016 Freescale Semiconductor Inc. + * Copyright 2016 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __DESC_COMMON_H__ +#define __DESC_COMMON_H__ + +#include "hw/rta.h" + +/** + * DOC: Shared Descriptor Constructors - shared structures + * + * Data structures shared between algorithm, protocol implementations. + */ + +/** + * struct alginfo - Container for algorithm details + * @algtype: algorithm selector; for valid values, see documentation of the + * functions where it is used. + * @keylen: length of the provided algorithm key, in bytes + * @key: address where algorithm key resides; virtual address if key_type is + * RTA_DATA_IMM, physical (bus) address if key_type is RTA_DATA_PTR or + * RTA_DATA_IMM_DMA. + * @key_enc_flags: key encryption flags; see encrypt_flags parameter of KEY + * command for valid values. + * @key_type: enum rta_data_type + * @algmode: algorithm mode selector; for valid values, see documentation of the + * functions where it is used. + */ +struct alginfo { + uint32_t algtype; + uint32_t keylen; + uint64_t key; + uint32_t key_enc_flags; + enum rta_data_type key_type; + uint16_t algmode; +}; + +#define INLINE_KEY(alginfo) inline_flags(alginfo->key_type) + +/** + * rta_inline_query() - Provide indications on which data items can be inlined + * and which shall be referenced in a shared descriptor. + * @sd_base_len: Shared descriptor base length - bytes consumed by the commands, + * excluding the data items to be inlined (or corresponding + * pointer if an item is not inlined). Each cnstr_* function that + * generates descriptors should have a define mentioning + * corresponding length. + * @jd_len: Maximum length of the job descriptor(s) that will be used + * together with the shared descriptor. + * @data_len: Array of lengths of the data items trying to be inlined + * @inl_mask: 32bit mask with bit x = 1 if data item x can be inlined, 0 + * otherwise. + * @count: Number of data items (size of @data_len array); must be <= 32 + * + * Return: 0 if data can be inlined / referenced, negative value if not. If 0, + * check @inl_mask for details. + */ +static inline int +rta_inline_query(unsigned int sd_base_len, + unsigned int jd_len, + unsigned int *data_len, + uint32_t *inl_mask, + unsigned int count) +{ + int rem_bytes = (int)(CAAM_DESC_BYTES_MAX - sd_base_len - jd_len); + unsigned int i; + + *inl_mask = 0; + for (i = 0; (i < count) && (rem_bytes > 0); i++) { + if (rem_bytes - (int)(data_len[i] + + (count - i - 1) * CAAM_PTR_SZ) >= 0) { + rem_bytes -= data_len[i]; + *inl_mask |= (1 << i); + } else { + rem_bytes -= CAAM_PTR_SZ; + } + } + + return (rem_bytes >= 0) ? 0 : -1; +} + +/** + * struct protcmd - Container for Protocol Operation Command fields + * @optype: command type + * @protid: protocol Identifier + * @protinfo: protocol Information + */ +struct protcmd { + uint32_t optype; + uint32_t protid; + uint16_t protinfo; +}; + +#endif /* __DESC_COMMON_H__ */ diff --git a/dpdk/drivers/crypto/dpaa2_sec/hw/desc/ipsec.h b/dpdk/drivers/crypto/dpaa2_sec/hw/desc/ipsec.h new file mode 100644 index 00000000..cc637361 --- /dev/null +++ b/dpdk/drivers/crypto/dpaa2_sec/hw/desc/ipsec.h @@ -0,0 +1,1554 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2008-2016 Freescale Semiconductor Inc. + * Copyright 2016 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __DESC_IPSEC_H__ +#define __DESC_IPSEC_H__ + +#include "hw/rta.h" +#include "common.h" + +/** + * DOC: IPsec Shared Descriptor Constructors + * + * Shared descriptors for IPsec protocol. + */ + +/* General IPSec ESP encap / decap PDB options */ + +/** + * PDBOPTS_ESP_ESN - Extended sequence included + */ +#define PDBOPTS_ESP_ESN 0x10 + +/** + * PDBOPTS_ESP_IPVSN - Process IPv6 header + * + * Valid only for IPsec legacy mode. + */ +#define PDBOPTS_ESP_IPVSN 0x02 + +/** + * PDBOPTS_ESP_TUNNEL - Tunnel mode next-header byte + * + * Valid only for IPsec legacy mode. + */ +#define PDBOPTS_ESP_TUNNEL 0x01 + +/* IPSec ESP Encap PDB options */ + +/** + * PDBOPTS_ESP_UPDATE_CSUM - Update ip header checksum + * + * Valid only for IPsec legacy mode. + */ +#define PDBOPTS_ESP_UPDATE_CSUM 0x80 + +/** + * PDBOPTS_ESP_DIFFSERV - Copy TOS/TC from inner iphdr + * + * Valid only for IPsec legacy mode. + */ +#define PDBOPTS_ESP_DIFFSERV 0x40 + +/** + * PDBOPTS_ESP_IVSRC - IV comes from internal random gen + */ +#define PDBOPTS_ESP_IVSRC 0x20 + +/** + * PDBOPTS_ESP_IPHDRSRC - IP header comes from PDB + * + * Valid only for IPsec legacy mode. + */ +#define PDBOPTS_ESP_IPHDRSRC 0x08 + +/** + * PDBOPTS_ESP_INCIPHDR - Prepend IP header to output frame + * + * Valid only for IPsec legacy mode. + */ +#define PDBOPTS_ESP_INCIPHDR 0x04 + +/** + * PDBOPTS_ESP_OIHI_MASK - Mask for Outer IP Header Included + * + * Valid only for IPsec new mode. + */ +#define PDBOPTS_ESP_OIHI_MASK 0x0c + +/** + * PDBOPTS_ESP_OIHI_PDB_INL - Prepend IP header to output frame from PDB (where + * it is inlined). + * + * Valid only for IPsec new mode. + */ +#define PDBOPTS_ESP_OIHI_PDB_INL 0x0c + +/** + * PDBOPTS_ESP_OIHI_PDB_REF - Prepend IP header to output frame from PDB + * (referenced by pointer). + * + * Vlid only for IPsec new mode. + */ +#define PDBOPTS_ESP_OIHI_PDB_REF 0x08 + +/** + * PDBOPTS_ESP_OIHI_IF - Prepend IP header to output frame from input frame + * + * Valid only for IPsec new mode. + */ +#define PDBOPTS_ESP_OIHI_IF 0x04 + +/** + * PDBOPTS_ESP_NAT - Enable RFC 3948 UDP-encapsulated-ESP + * + * Valid only for IPsec new mode. + */ +#define PDBOPTS_ESP_NAT 0x02 + +/** + * PDBOPTS_ESP_NUC - Enable NAT UDP Checksum + * + * Valid only for IPsec new mode. + */ +#define PDBOPTS_ESP_NUC 0x01 + +/* IPSec ESP Decap PDB options */ + +/** + * PDBOPTS_ESP_ARS_MASK - antireplay window mask + */ +#define PDBOPTS_ESP_ARS_MASK 0xc0 + +/** + * PDBOPTS_ESP_ARSNONE - No antireplay window + */ +#define PDBOPTS_ESP_ARSNONE 0x00 + +/** + * PDBOPTS_ESP_ARS64 - 64-entry antireplay window + */ +#define PDBOPTS_ESP_ARS64 0xc0 + +/** + * PDBOPTS_ESP_ARS128 - 128-entry antireplay window + * + * Valid only for IPsec new mode. + */ +#define PDBOPTS_ESP_ARS128 0x80 + +/** + * PDBOPTS_ESP_ARS32 - 32-entry antireplay window + */ +#define PDBOPTS_ESP_ARS32 0x40 + +/** + * PDBOPTS_ESP_VERIFY_CSUM - Validate ip header checksum + * + * Valid only for IPsec legacy mode. + */ +#define PDBOPTS_ESP_VERIFY_CSUM 0x20 + +/** + * PDBOPTS_ESP_TECN - Implement RRFC6040 ECN tunneling from outer header to + * inner header. + * + * Valid only for IPsec new mode. + */ +#define PDBOPTS_ESP_TECN 0x20 + +/** + * PDBOPTS_ESP_OUTFMT - Output only decapsulation + * + * Valid only for IPsec legacy mode. + */ +#define PDBOPTS_ESP_OUTFMT 0x08 + +/** + * PDBOPTS_ESP_AOFL - Adjust out frame len + * + * Valid only for IPsec legacy mode and for SEC >= 5.3. + */ +#define PDBOPTS_ESP_AOFL 0x04 + +/** + * PDBOPTS_ESP_ETU - EtherType Update + * + * Add corresponding ethertype (0x0800 for IPv4, 0x86dd for IPv6) in the output + * frame. + * Valid only for IPsec new mode. + */ +#define PDBOPTS_ESP_ETU 0x01 + +#define PDBHMO_ESP_DECAP_SHIFT 28 +#define PDBHMO_ESP_ENCAP_SHIFT 28 +#define PDBNH_ESP_ENCAP_SHIFT 16 +#define PDBNH_ESP_ENCAP_MASK (0xff << PDBNH_ESP_ENCAP_SHIFT) +#define PDBHDRLEN_ESP_DECAP_SHIFT 16 +#define PDBHDRLEN_MASK (0x0fff << PDBHDRLEN_ESP_DECAP_SHIFT) +#define PDB_NH_OFFSET_SHIFT 8 +#define PDB_NH_OFFSET_MASK (0xff << PDB_NH_OFFSET_SHIFT) + +/** + * PDBHMO_ESP_DECAP_DTTL - IPsec ESP decrement TTL (IPv4) / Hop limit (IPv6) + * HMO option. + */ +#define PDBHMO_ESP_DECAP_DTTL (0x02 << PDBHMO_ESP_DECAP_SHIFT) + +/** + * PDBHMO_ESP_ENCAP_DTTL - IPsec ESP increment TTL (IPv4) / Hop limit (IPv6) + * HMO option. + */ +#define PDBHMO_ESP_ENCAP_DTTL (0x02 << PDBHMO_ESP_ENCAP_SHIFT) + +/** + * PDBHMO_ESP_DIFFSERV - (Decap) DiffServ Copy - Copy the IPv4 TOS or IPv6 + * Traffic Class byte from the outer IP header to the + * inner IP header. + */ +#define PDBHMO_ESP_DIFFSERV (0x01 << PDBHMO_ESP_DECAP_SHIFT) + +/** + * PDBHMO_ESP_SNR - (Encap) - Sequence Number Rollover control + * + * Configures behaviour in case of SN / ESN rollover: + * error if SNR = 1, rollover allowed if SNR = 0. + * Valid only for IPsec new mode. + */ +#define PDBHMO_ESP_SNR (0x01 << PDBHMO_ESP_ENCAP_SHIFT) + +/** + * PDBHMO_ESP_DFBIT - (Encap) Copy DF bit - if an IPv4 tunnel mode outer IP + * header is coming from the PDB, copy the DF bit from the + * inner IP header to the outer IP header. + */ +#define PDBHMO_ESP_DFBIT (0x04 << PDBHMO_ESP_ENCAP_SHIFT) + +/** + * PDBHMO_ESP_DFV - (Decap) - DF bit value + * + * If ODF = 1, DF bit in output frame is replaced by DFV. + * Valid only from SEC Era 5 onwards. + */ +#define PDBHMO_ESP_DFV (0x04 << PDBHMO_ESP_DECAP_SHIFT) + +/** + * PDBHMO_ESP_ODF - (Decap) Override DF bit in IPv4 header of decapsulated + * output frame. + * + * If ODF = 1, DF is replaced with the value of DFV bit. + * Valid only from SEC Era 5 onwards. + */ +#define PDBHMO_ESP_ODF (0x08 << PDBHMO_ESP_DECAP_SHIFT) + +/** + * struct ipsec_encap_cbc - PDB part for IPsec CBC encapsulation + * @iv: 16-byte array initialization vector + */ +struct ipsec_encap_cbc { + uint8_t iv[16]; +}; + + +/** + * struct ipsec_encap_ctr - PDB part for IPsec CTR encapsulation + * @ctr_nonce: 4-byte array nonce + * @ctr_initial: initial count constant + * @iv: initialization vector + */ +struct ipsec_encap_ctr { + uint8_t ctr_nonce[4]; + uint32_t ctr_initial; + uint64_t iv; +}; + +/** + * struct ipsec_encap_ccm - PDB part for IPsec CCM encapsulation + * @salt: 3-byte array salt (lower 24 bits) + * @ccm_opt: CCM algorithm options - MSB-LSB description: + * b0_flags (8b) - CCM B0; use 0x5B for 8-byte ICV, 0x6B for 12-byte ICV, + * 0x7B for 16-byte ICV (cf. RFC4309, RFC3610) + * ctr_flags (8b) - counter flags; constant equal to 0x3 + * ctr_initial (16b) - initial count constant + * @iv: initialization vector + */ +struct ipsec_encap_ccm { + uint8_t salt[4]; + uint32_t ccm_opt; + uint64_t iv; +}; + +/** + * struct ipsec_encap_gcm - PDB part for IPsec GCM encapsulation + * @salt: 3-byte array salt (lower 24 bits) + * @rsvd: reserved, do not use + * @iv: initialization vector + */ +struct ipsec_encap_gcm { + uint8_t salt[4]; + uint32_t rsvd; + uint64_t iv; +}; + +/** + * struct ipsec_encap_pdb - PDB for IPsec encapsulation + * @options: MSB-LSB description (both for legacy and new modes) + * hmo (header manipulation options) - 4b + * reserved - 4b + * next header (legacy) / reserved (new) - 8b + * next header offset (legacy) / AOIPHO (actual outer IP header offset) - 8b + * option flags (depend on selected algorithm) - 8b + * @seq_num_ext_hi: (optional) IPsec Extended Sequence Number (ESN) + * @seq_num: IPsec sequence number + * @spi: IPsec SPI (Security Parameters Index) + * @ip_hdr_len: optional IP Header length (in bytes) + * reserved - 16b + * Opt. IP Hdr Len - 16b + * @ip_hdr: optional IP Header content (only for IPsec legacy mode) + */ +struct ipsec_encap_pdb { + uint32_t options; + uint32_t seq_num_ext_hi; + uint32_t seq_num; + union { + struct ipsec_encap_cbc cbc; + struct ipsec_encap_ctr ctr; + struct ipsec_encap_ccm ccm; + struct ipsec_encap_gcm gcm; + }; + uint32_t spi; + uint32_t ip_hdr_len; + uint8_t ip_hdr[0]; +}; + +static inline unsigned int +__rta_copy_ipsec_encap_pdb(struct program *program, + struct ipsec_encap_pdb *pdb, + uint32_t algtype) +{ + unsigned int start_pc = program->current_pc; + + __rta_out32(program, pdb->options); + __rta_out32(program, pdb->seq_num_ext_hi); + __rta_out32(program, pdb->seq_num); + + switch (algtype & OP_PCL_IPSEC_CIPHER_MASK) { + case OP_PCL_IPSEC_DES_IV64: + case OP_PCL_IPSEC_DES: + case OP_PCL_IPSEC_3DES: + case OP_PCL_IPSEC_AES_CBC: + case OP_PCL_IPSEC_NULL: + rta_copy_data(program, pdb->cbc.iv, sizeof(pdb->cbc.iv)); + break; + + case OP_PCL_IPSEC_AES_CTR: + rta_copy_data(program, pdb->ctr.ctr_nonce, + sizeof(pdb->ctr.ctr_nonce)); + __rta_out32(program, pdb->ctr.ctr_initial); + __rta_out64(program, true, pdb->ctr.iv); + break; + + case OP_PCL_IPSEC_AES_CCM8: + case OP_PCL_IPSEC_AES_CCM12: + case OP_PCL_IPSEC_AES_CCM16: + rta_copy_data(program, pdb->ccm.salt, sizeof(pdb->ccm.salt)); + __rta_out32(program, pdb->ccm.ccm_opt); + __rta_out64(program, true, pdb->ccm.iv); + break; + + case OP_PCL_IPSEC_AES_GCM8: + case OP_PCL_IPSEC_AES_GCM12: + case OP_PCL_IPSEC_AES_GCM16: + case OP_PCL_IPSEC_AES_NULL_WITH_GMAC: + rta_copy_data(program, pdb->gcm.salt, sizeof(pdb->gcm.salt)); + __rta_out32(program, pdb->gcm.rsvd); + __rta_out64(program, true, pdb->gcm.iv); + break; + } + + __rta_out32(program, pdb->spi); + __rta_out32(program, pdb->ip_hdr_len); + + return start_pc; +} + +/** + * struct ipsec_decap_cbc - PDB part for IPsec CBC decapsulation + * @rsvd: reserved, do not use + */ +struct ipsec_decap_cbc { + uint32_t rsvd[2]; +}; + +/** + * struct ipsec_decap_ctr - PDB part for IPsec CTR decapsulation + * @ctr_nonce: 4-byte array nonce + * @ctr_initial: initial count constant + */ +struct ipsec_decap_ctr { + uint8_t ctr_nonce[4]; + uint32_t ctr_initial; +}; + +/** + * struct ipsec_decap_ccm - PDB part for IPsec CCM decapsulation + * @salt: 3-byte salt (lower 24 bits) + * @ccm_opt: CCM algorithm options - MSB-LSB description: + * b0_flags (8b) - CCM B0; use 0x5B for 8-byte ICV, 0x6B for 12-byte ICV, + * 0x7B for 16-byte ICV (cf. RFC4309, RFC3610) + * ctr_flags (8b) - counter flags; constant equal to 0x3 + * ctr_initial (16b) - initial count constant + */ +struct ipsec_decap_ccm { + uint8_t salt[4]; + uint32_t ccm_opt; +}; + +/** + * struct ipsec_decap_gcm - PDB part for IPsec GCN decapsulation + * @salt: 4-byte salt + * @rsvd: reserved, do not use + */ +struct ipsec_decap_gcm { + uint8_t salt[4]; + uint32_t rsvd; +}; + +/** + * struct ipsec_decap_pdb - PDB for IPsec decapsulation + * @options: MSB-LSB description (both for legacy and new modes) + * hmo (header manipulation options) - 4b + * IP header length - 12b + * next header offset (legacy) / AOIPHO (actual outer IP header offset) - 8b + * option flags (depend on selected algorithm) - 8b + * @seq_num_ext_hi: (optional) IPsec Extended Sequence Number (ESN) + * @seq_num: IPsec sequence number + * @anti_replay: Anti-replay window; size depends on ARS (option flags); + * format must be Big Endian, irrespective of platform + */ +struct ipsec_decap_pdb { + uint32_t options; + union { + struct ipsec_decap_cbc cbc; + struct ipsec_decap_ctr ctr; + struct ipsec_decap_ccm ccm; + struct ipsec_decap_gcm gcm; + }; + uint32_t seq_num_ext_hi; + uint32_t seq_num; + uint32_t anti_replay[4]; +}; + +static inline unsigned int +__rta_copy_ipsec_decap_pdb(struct program *program, + struct ipsec_decap_pdb *pdb, + uint32_t algtype) +{ + unsigned int start_pc = program->current_pc; + unsigned int i, ars; + + __rta_out32(program, pdb->options); + + switch (algtype & OP_PCL_IPSEC_CIPHER_MASK) { + case OP_PCL_IPSEC_DES_IV64: + case OP_PCL_IPSEC_DES: + case OP_PCL_IPSEC_3DES: + case OP_PCL_IPSEC_AES_CBC: + case OP_PCL_IPSEC_NULL: + __rta_out32(program, pdb->cbc.rsvd[0]); + __rta_out32(program, pdb->cbc.rsvd[1]); + break; + + case OP_PCL_IPSEC_AES_CTR: + rta_copy_data(program, pdb->ctr.ctr_nonce, + sizeof(pdb->ctr.ctr_nonce)); + __rta_out32(program, pdb->ctr.ctr_initial); + break; + + case OP_PCL_IPSEC_AES_CCM8: + case OP_PCL_IPSEC_AES_CCM12: + case OP_PCL_IPSEC_AES_CCM16: + rta_copy_data(program, pdb->ccm.salt, sizeof(pdb->ccm.salt)); + __rta_out32(program, pdb->ccm.ccm_opt); + break; + + case OP_PCL_IPSEC_AES_GCM8: + case OP_PCL_IPSEC_AES_GCM12: + case OP_PCL_IPSEC_AES_GCM16: + case OP_PCL_IPSEC_AES_NULL_WITH_GMAC: + rta_copy_data(program, pdb->gcm.salt, sizeof(pdb->gcm.salt)); + __rta_out32(program, pdb->gcm.rsvd); + break; + } + + __rta_out32(program, pdb->seq_num_ext_hi); + __rta_out32(program, pdb->seq_num); + + switch (pdb->options & PDBOPTS_ESP_ARS_MASK) { + case PDBOPTS_ESP_ARS128: + ars = 4; + break; + case PDBOPTS_ESP_ARS64: + ars = 2; + break; + case PDBOPTS_ESP_ARS32: + ars = 1; + break; + case PDBOPTS_ESP_ARSNONE: + default: + ars = 0; + break; + } + + for (i = 0; i < ars; i++) + __rta_out_be32(program, pdb->anti_replay[i]); + + return start_pc; +} + +/** + * enum ipsec_icv_size - Type selectors for icv size in IPsec protocol + * @IPSEC_ICV_MD5_SIZE: full-length MD5 ICV + * @IPSEC_ICV_MD5_TRUNC_SIZE: truncated MD5 ICV + */ +enum ipsec_icv_size { + IPSEC_ICV_MD5_SIZE = 16, + IPSEC_ICV_MD5_TRUNC_SIZE = 12 +}; + +/* + * IPSec ESP Datapath Protocol Override Register (DPOVRD) + */ + +#define IPSEC_DECO_DPOVRD_USE 0x80 + +struct ipsec_deco_dpovrd { + uint8_t ovrd_ecn; + uint8_t ip_hdr_len; + uint8_t nh_offset; + union { + uint8_t next_header; /* next header if encap */ + uint8_t rsvd; /* reserved if decap */ + }; +}; + +struct ipsec_new_encap_deco_dpovrd { +#define IPSEC_NEW_ENCAP_DECO_DPOVRD_USE 0x8000 + uint16_t ovrd_ip_hdr_len; /* OVRD + outer IP header material + * length + */ +#define IPSEC_NEW_ENCAP_OIMIF 0x80 + uint8_t oimif_aoipho; /* OIMIF + actual outer IP header + * offset + */ + uint8_t rsvd; +}; + +struct ipsec_new_decap_deco_dpovrd { + uint8_t ovrd; + uint8_t aoipho_hi; /* upper nibble of actual outer IP + * header + */ + uint16_t aoipho_lo_ip_hdr_len; /* lower nibble of actual outer IP + * header + outer IP header material + */ +}; + +static inline void +__gen_auth_key(struct program *program, struct alginfo *authdata) +{ + uint32_t dkp_protid; + + switch (authdata->algtype & OP_PCL_IPSEC_AUTH_MASK) { + case OP_PCL_IPSEC_HMAC_MD5_96: + case OP_PCL_IPSEC_HMAC_MD5_128: + dkp_protid = OP_PCLID_DKP_MD5; + break; + case OP_PCL_IPSEC_HMAC_SHA1_96: + case OP_PCL_IPSEC_HMAC_SHA1_160: + dkp_protid = OP_PCLID_DKP_SHA1; + break; + case OP_PCL_IPSEC_HMAC_SHA2_256_128: + dkp_protid = OP_PCLID_DKP_SHA256; + break; + case OP_PCL_IPSEC_HMAC_SHA2_384_192: + dkp_protid = OP_PCLID_DKP_SHA384; + break; + case OP_PCL_IPSEC_HMAC_SHA2_512_256: + dkp_protid = OP_PCLID_DKP_SHA512; + break; + default: + KEY(program, KEY2, authdata->key_enc_flags, authdata->key, + authdata->keylen, INLINE_KEY(authdata)); + return; + } + + if (authdata->key_type == RTA_DATA_PTR) + DKP_PROTOCOL(program, dkp_protid, OP_PCL_DKP_SRC_PTR, + OP_PCL_DKP_DST_PTR, (uint16_t)authdata->keylen, + authdata->key, authdata->key_type); + else + DKP_PROTOCOL(program, dkp_protid, OP_PCL_DKP_SRC_IMM, + OP_PCL_DKP_DST_IMM, (uint16_t)authdata->keylen, + authdata->key, authdata->key_type); +} + +/** + * cnstr_shdsc_ipsec_encap - IPSec ESP encapsulation protocol-level shared + * descriptor. + * @descbuf: pointer to buffer used for descriptor construction + * @ps: if 36/40bit addressing is desired, this parameter must be true + * @swap: if true, perform descriptor byte swapping on a 4-byte boundary + * @pdb: pointer to the PDB to be used with this descriptor + * This structure will be copied inline to the descriptor under + * construction. No error checking will be made. Refer to the + * block guide for a details of the encapsulation PDB. + * @cipherdata: pointer to block cipher transform definitions + * Valid algorithm values - one of OP_PCL_IPSEC_* + * @authdata: pointer to authentication transform definitions + * If an authentication key is required by the protocol: + * -For SEC Eras 1-5, an MDHA split key must be provided; + * Note that the size of the split key itself must be specified. + * -For SEC Eras 6+, a "normal" key must be provided; DKP (Derived + * Key Protocol) will be used to compute MDHA on the fly in HW. + * Valid algorithm values - one of OP_PCL_IPSEC_* + * + * Return: size of descriptor written in words or negative number on error + */ +static inline int +cnstr_shdsc_ipsec_encap(uint32_t *descbuf, bool ps, bool swap, + struct ipsec_encap_pdb *pdb, + struct alginfo *cipherdata, + struct alginfo *authdata) +{ + struct program prg; + struct program *p = &prg; + + LABEL(keyjmp); + REFERENCE(pkeyjmp); + LABEL(hdr); + REFERENCE(phdr); + + PROGRAM_CNTXT_INIT(p, descbuf, 0); + if (swap) + PROGRAM_SET_BSWAP(p); + if (ps) + PROGRAM_SET_36BIT_ADDR(p); + phdr = SHR_HDR(p, SHR_SERIAL, hdr, 0); + __rta_copy_ipsec_encap_pdb(p, pdb, cipherdata->algtype); + COPY_DATA(p, pdb->ip_hdr, pdb->ip_hdr_len); + SET_LABEL(p, hdr); + pkeyjmp = JUMP(p, keyjmp, LOCAL_JUMP, ALL_TRUE, BOTH|SHRD); + if (authdata->keylen) { + if (rta_sec_era < RTA_SEC_ERA_6) + KEY(p, MDHA_SPLIT_KEY, authdata->key_enc_flags, + authdata->key, authdata->keylen, + INLINE_KEY(authdata)); + else + __gen_auth_key(p, authdata); + } + if (cipherdata->keylen) + KEY(p, KEY1, cipherdata->key_enc_flags, cipherdata->key, + cipherdata->keylen, INLINE_KEY(cipherdata)); + SET_LABEL(p, keyjmp); + PROTOCOL(p, OP_TYPE_ENCAP_PROTOCOL, + OP_PCLID_IPSEC, + (uint16_t)(cipherdata->algtype | authdata->algtype)); + PATCH_JUMP(p, pkeyjmp, keyjmp); + PATCH_HDR(p, phdr, hdr); + return PROGRAM_FINALIZE(p); +} + +/** + * cnstr_shdsc_ipsec_decap - IPSec ESP decapsulation protocol-level shared + * descriptor. + * @descbuf: pointer to buffer used for descriptor construction + * @ps: if 36/40bit addressing is desired, this parameter must be true + * @swap: if true, perform descriptor byte swapping on a 4-byte boundary + * @pdb: pointer to the PDB to be used with this descriptor + * This structure will be copied inline to the descriptor under + * construction. No error checking will be made. Refer to the + * block guide for details about the decapsulation PDB. + * @cipherdata: pointer to block cipher transform definitions. + * Valid algorithm values - one of OP_PCL_IPSEC_* + * @authdata: pointer to authentication transform definitions + * If an authentication key is required by the protocol: + * -For SEC Eras 1-5, an MDHA split key must be provided; + * Note that the size of the split key itself must be specified. + * -For SEC Eras 6+, a "normal" key must be provided; DKP (Derived + * Key Protocol) will be used to compute MDHA on the fly in HW. + * Valid algorithm values - one of OP_PCL_IPSEC_* + * + * Return: size of descriptor written in words or negative number on error + */ +static inline int +cnstr_shdsc_ipsec_decap(uint32_t *descbuf, bool ps, bool swap, + struct ipsec_decap_pdb *pdb, + struct alginfo *cipherdata, + struct alginfo *authdata) +{ + struct program prg; + struct program *p = &prg; + + LABEL(keyjmp); + REFERENCE(pkeyjmp); + LABEL(hdr); + REFERENCE(phdr); + + PROGRAM_CNTXT_INIT(p, descbuf, 0); + if (swap) + PROGRAM_SET_BSWAP(p); + if (ps) + PROGRAM_SET_36BIT_ADDR(p); + phdr = SHR_HDR(p, SHR_SERIAL, hdr, 0); + __rta_copy_ipsec_decap_pdb(p, pdb, cipherdata->algtype); + SET_LABEL(p, hdr); + pkeyjmp = JUMP(p, keyjmp, LOCAL_JUMP, ALL_TRUE, BOTH|SHRD); + if (authdata->keylen) { + if (rta_sec_era < RTA_SEC_ERA_6) + KEY(p, MDHA_SPLIT_KEY, authdata->key_enc_flags, + authdata->key, authdata->keylen, + INLINE_KEY(authdata)); + else + __gen_auth_key(p, authdata); + } + if (cipherdata->keylen) + KEY(p, KEY1, cipherdata->key_enc_flags, cipherdata->key, + cipherdata->keylen, INLINE_KEY(cipherdata)); + SET_LABEL(p, keyjmp); + PROTOCOL(p, OP_TYPE_DECAP_PROTOCOL, + OP_PCLID_IPSEC, + (uint16_t)(cipherdata->algtype | authdata->algtype)); + PATCH_JUMP(p, pkeyjmp, keyjmp); + PATCH_HDR(p, phdr, hdr); + return PROGRAM_FINALIZE(p); +} + +/** + * cnstr_shdsc_ipsec_encap_des_aes_xcbc - IPSec DES-CBC/3DES-CBC and + * AES-XCBC-MAC-96 ESP encapsulation shared descriptor. + * @descbuf: pointer to buffer used for descriptor construction + * @pdb: pointer to the PDB to be used with this descriptor + * This structure will be copied inline to the descriptor under + * construction. No error checking will be made. Refer to the + * block guide for a details of the encapsulation PDB. + * @cipherdata: pointer to block cipher transform definitions + * Valid algorithm values - OP_PCL_IPSEC_DES, OP_PCL_IPSEC_3DES. + * @authdata: pointer to authentication transform definitions + * Valid algorithm value: OP_PCL_IPSEC_AES_XCBC_MAC_96. + * + * Supported only for platforms with 32-bit address pointers and SEC ERA 4 or + * higher. The tunnel/transport mode of the IPsec ESP is supported only if the + * Outer/Transport IP Header is present in the encapsulation output packet. + * The descriptor performs DES-CBC/3DES-CBC & HMAC-MD5-96 and then rereads + * the input packet to do the AES-XCBC-MAC-96 calculation and to overwrite + * the MD5 ICV. + * The descriptor uses all the benefits of the built-in protocol by computing + * the IPsec ESP with a hardware supported algorithms combination + * (DES-CBC/3DES-CBC & HMAC-MD5-96). The HMAC-MD5 authentication algorithm + * was chosen in order to speed up the computational time for this intermediate + * step. + * Warning: The user must allocate at least 32 bytes for the authentication key + * (in order to use it also with HMAC-MD5-96),even when using a shorter key + * for the AES-XCBC-MAC-96. + * + * Return: size of descriptor written in words or negative number on error + */ +static inline int +cnstr_shdsc_ipsec_encap_des_aes_xcbc(uint32_t *descbuf, + struct ipsec_encap_pdb *pdb, + struct alginfo *cipherdata, + struct alginfo *authdata) +{ + struct program prg; + struct program *p = &prg; + + LABEL(hdr); + LABEL(shd_ptr); + LABEL(keyjump); + LABEL(outptr); + LABEL(swapped_seqin_fields); + LABEL(swapped_seqin_ptr); + REFERENCE(phdr); + REFERENCE(pkeyjump); + REFERENCE(move_outlen); + REFERENCE(move_seqout_ptr); + REFERENCE(swapped_seqin_ptr_jump); + REFERENCE(write_swapped_seqin_ptr); + + PROGRAM_CNTXT_INIT(p, descbuf, 0); + phdr = SHR_HDR(p, SHR_SERIAL, hdr, 0); + __rta_copy_ipsec_encap_pdb(p, pdb, cipherdata->algtype); + COPY_DATA(p, pdb->ip_hdr, pdb->ip_hdr_len); + SET_LABEL(p, hdr); + pkeyjump = JUMP(p, keyjump, LOCAL_JUMP, ALL_TRUE, SHRD | SELF); + /* + * Hard-coded KEY arguments. The descriptor uses all the benefits of + * the built-in protocol by computing the IPsec ESP with a hardware + * supported algorithms combination (DES-CBC/3DES-CBC & HMAC-MD5-96). + * The HMAC-MD5 authentication algorithm was chosen with + * the keys options from below in order to speed up the computational + * time for this intermediate step. + * Warning: The user must allocate at least 32 bytes for + * the authentication key (in order to use it also with HMAC-MD5-96), + * even when using a shorter key for the AES-XCBC-MAC-96. + */ + KEY(p, MDHA_SPLIT_KEY, 0, authdata->key, 32, INLINE_KEY(authdata)); + SET_LABEL(p, keyjump); + LOAD(p, LDST_SRCDST_WORD_CLRW | CLRW_CLR_C1MODE | CLRW_CLR_C1DATAS | + CLRW_CLR_C1CTX | CLRW_CLR_C1KEY | CLRW_RESET_CLS1_CHA, CLRW, 0, 4, + IMMED); + KEY(p, KEY1, cipherdata->key_enc_flags, cipherdata->key, + cipherdata->keylen, INLINE_KEY(cipherdata)); + PROTOCOL(p, OP_TYPE_ENCAP_PROTOCOL, OP_PCLID_IPSEC, + (uint16_t)(cipherdata->algtype | OP_PCL_IPSEC_HMAC_MD5_96)); + /* Swap SEQINPTR to SEQOUTPTR. */ + move_seqout_ptr = MOVE(p, DESCBUF, 0, MATH1, 0, 16, WAITCOMP | IMMED); + MATHB(p, MATH1, AND, ~(CMD_SEQ_IN_PTR ^ CMD_SEQ_OUT_PTR), MATH1, + 8, IFB | IMMED2); +/* + * TODO: RTA currently doesn't support creating a LOAD command + * with another command as IMM. + * To be changed when proper support is added in RTA. + */ + LOAD(p, 0xa00000e5, MATH3, 4, 4, IMMED); + MATHB(p, MATH3, SHLD, MATH3, MATH3, 8, 0); + write_swapped_seqin_ptr = MOVE(p, MATH1, 0, DESCBUF, 0, 20, WAITCOMP | + IMMED); + swapped_seqin_ptr_jump = JUMP(p, swapped_seqin_ptr, LOCAL_JUMP, + ALL_TRUE, 0); + LOAD(p, LDST_SRCDST_WORD_CLRW | CLRW_CLR_C1MODE | CLRW_CLR_C1DATAS | + CLRW_CLR_C1CTX | CLRW_CLR_C1KEY | CLRW_RESET_CLS1_CHA, CLRW, 0, 4, + 0); + SEQOUTPTR(p, 0, 65535, RTO); + move_outlen = MOVE(p, DESCBUF, 0, MATH0, 4, 8, WAITCOMP | IMMED); + MATHB(p, MATH0, SUB, + (uint64_t)(pdb->ip_hdr_len + IPSEC_ICV_MD5_TRUNC_SIZE), + VSEQINSZ, 4, IMMED2); + MATHB(p, MATH0, SUB, IPSEC_ICV_MD5_TRUNC_SIZE, VSEQOUTSZ, 4, IMMED2); + KEY(p, KEY1, authdata->key_enc_flags, authdata->key, authdata->keylen, + 0); + ALG_OPERATION(p, OP_ALG_ALGSEL_AES, OP_ALG_AAI_XCBC_MAC, + OP_ALG_AS_INITFINAL, ICV_CHECK_DISABLE, DIR_ENC); + SEQFIFOLOAD(p, SKIP, pdb->ip_hdr_len, 0); + SEQFIFOLOAD(p, MSG1, 0, VLF | FLUSH1 | LAST1); + SEQFIFOSTORE(p, SKIP, 0, 0, VLF); + SEQSTORE(p, CONTEXT1, 0, IPSEC_ICV_MD5_TRUNC_SIZE, 0); +/* + * TODO: RTA currently doesn't support adding labels in or after Job Descriptor. + * To be changed when proper support is added in RTA. + */ + /* Label the Shared Descriptor Pointer */ + SET_LABEL(p, shd_ptr); + shd_ptr += 1; + /* Label the Output Pointer */ + SET_LABEL(p, outptr); + outptr += 3; + /* Label the first word after JD */ + SET_LABEL(p, swapped_seqin_fields); + swapped_seqin_fields += 8; + /* Label the second word after JD */ + SET_LABEL(p, swapped_seqin_ptr); + swapped_seqin_ptr += 9; + + PATCH_HDR(p, phdr, hdr); + PATCH_JUMP(p, pkeyjump, keyjump); + PATCH_JUMP(p, swapped_seqin_ptr_jump, swapped_seqin_ptr); + PATCH_MOVE(p, move_outlen, outptr); + PATCH_MOVE(p, move_seqout_ptr, shd_ptr); + PATCH_MOVE(p, write_swapped_seqin_ptr, swapped_seqin_fields); + return PROGRAM_FINALIZE(p); +} + +/** + * cnstr_shdsc_ipsec_decap_des_aes_xcbc - IPSec DES-CBC/3DES-CBC and + * AES-XCBC-MAC-96 ESP decapsulation shared descriptor. + * @descbuf: pointer to buffer used for descriptor construction + * @pdb: pointer to the PDB to be used with this descriptor + * This structure will be copied inline to the descriptor under + * construction. No error checking will be made. Refer to the + * block guide for a details of the encapsulation PDB. + * @cipherdata: pointer to block cipher transform definitions + * Valid algorithm values - OP_PCL_IPSEC_DES, OP_PCL_IPSEC_3DES. + * @authdata: pointer to authentication transform definitions + * Valid algorithm value: OP_PCL_IPSEC_AES_XCBC_MAC_96. + * + * Supported only for platforms with 32-bit address pointers and SEC ERA 4 or + * higher. The tunnel/transport mode of the IPsec ESP is supported only if the + * Outer/Transport IP Header is present in the decapsulation input packet. + * The descriptor computes the AES-XCBC-MAC-96 to check if the received ICV + * is correct, rereads the input packet to compute the MD5 ICV, overwrites + * the XCBC ICV, and then sends the modified input packet to the + * DES-CBC/3DES-CBC & HMAC-MD5-96 IPsec. + * The descriptor uses all the benefits of the built-in protocol by computing + * the IPsec ESP with a hardware supported algorithms combination + * (DES-CBC/3DES-CBC & HMAC-MD5-96). The HMAC-MD5 authentication algorithm + * was chosen in order to speed up the computational time for this intermediate + * step. + * Warning: The user must allocate at least 32 bytes for the authentication key + * (in order to use it also with HMAC-MD5-96),even when using a shorter key + * for the AES-XCBC-MAC-96. + * + * Return: size of descriptor written in words or negative number on error + */ +static inline int +cnstr_shdsc_ipsec_decap_des_aes_xcbc(uint32_t *descbuf, + struct ipsec_decap_pdb *pdb, + struct alginfo *cipherdata, + struct alginfo *authdata) +{ + struct program prg; + struct program *p = &prg; + uint32_t ip_hdr_len = (pdb->options & PDBHDRLEN_MASK) >> + PDBHDRLEN_ESP_DECAP_SHIFT; + + LABEL(hdr); + LABEL(jump_cmd); + LABEL(keyjump); + LABEL(outlen); + LABEL(seqin_ptr); + LABEL(seqout_ptr); + LABEL(swapped_seqout_fields); + LABEL(swapped_seqout_ptr); + REFERENCE(seqout_ptr_jump); + REFERENCE(phdr); + REFERENCE(pkeyjump); + REFERENCE(move_jump); + REFERENCE(move_jump_back); + REFERENCE(move_seqin_ptr); + REFERENCE(swapped_seqout_ptr_jump); + REFERENCE(write_swapped_seqout_ptr); + + PROGRAM_CNTXT_INIT(p, descbuf, 0); + phdr = SHR_HDR(p, SHR_SERIAL, hdr, 0); + __rta_copy_ipsec_decap_pdb(p, pdb, cipherdata->algtype); + SET_LABEL(p, hdr); + pkeyjump = JUMP(p, keyjump, LOCAL_JUMP, ALL_TRUE, SHRD | SELF); + /* + * Hard-coded KEY arguments. The descriptor uses all the benefits of + * the built-in protocol by computing the IPsec ESP with a hardware + * supported algorithms combination (DES-CBC/3DES-CBC & HMAC-MD5-96). + * The HMAC-MD5 authentication algorithm was chosen with + * the keys options from bellow in order to speed up the computational + * time for this intermediate step. + * Warning: The user must allocate at least 32 bytes for + * the authentication key (in order to use it also with HMAC-MD5-96), + * even when using a shorter key for the AES-XCBC-MAC-96. + */ + KEY(p, MDHA_SPLIT_KEY, 0, authdata->key, 32, INLINE_KEY(authdata)); + SET_LABEL(p, keyjump); + LOAD(p, LDST_SRCDST_WORD_CLRW | CLRW_CLR_C1MODE | CLRW_CLR_C1DATAS | + CLRW_CLR_C1CTX | CLRW_CLR_C1KEY | CLRW_RESET_CLS1_CHA, CLRW, 0, 4, + 0); + KEY(p, KEY1, authdata->key_enc_flags, authdata->key, authdata->keylen, + INLINE_KEY(authdata)); + MATHB(p, SEQINSZ, SUB, + (uint64_t)(ip_hdr_len + IPSEC_ICV_MD5_TRUNC_SIZE), MATH0, 4, + IMMED2); + MATHB(p, MATH0, SUB, ZERO, VSEQINSZ, 4, 0); + ALG_OPERATION(p, OP_ALG_ALGSEL_MD5, OP_ALG_AAI_HMAC_PRECOMP, + OP_ALG_AS_INITFINAL, ICV_CHECK_DISABLE, DIR_ENC); + ALG_OPERATION(p, OP_ALG_ALGSEL_AES, OP_ALG_AAI_XCBC_MAC, + OP_ALG_AS_INITFINAL, ICV_CHECK_ENABLE, DIR_DEC); + SEQFIFOLOAD(p, SKIP, ip_hdr_len, 0); + SEQFIFOLOAD(p, MSG1, 0, VLF | FLUSH1); + SEQFIFOLOAD(p, ICV1, IPSEC_ICV_MD5_TRUNC_SIZE, FLUSH1 | LAST1); + /* Swap SEQOUTPTR to SEQINPTR. */ + move_seqin_ptr = MOVE(p, DESCBUF, 0, MATH1, 0, 16, WAITCOMP | IMMED); + MATHB(p, MATH1, OR, CMD_SEQ_IN_PTR ^ CMD_SEQ_OUT_PTR, MATH1, 8, + IFB | IMMED2); +/* + * TODO: RTA currently doesn't support creating a LOAD command + * with another command as IMM. + * To be changed when proper support is added in RTA. + */ + LOAD(p, 0xA00000e1, MATH3, 4, 4, IMMED); + MATHB(p, MATH3, SHLD, MATH3, MATH3, 8, 0); + write_swapped_seqout_ptr = MOVE(p, MATH1, 0, DESCBUF, 0, 20, WAITCOMP | + IMMED); + swapped_seqout_ptr_jump = JUMP(p, swapped_seqout_ptr, LOCAL_JUMP, + ALL_TRUE, 0); +/* + * TODO: To be changed when proper support is added in RTA (can't load + * a command that is also written by RTA). + * Change when proper RTA support is added. + */ + SET_LABEL(p, jump_cmd); + WORD(p, 0xA00000f3); + SEQINPTR(p, 0, 65535, RTO); + MATHB(p, MATH0, SUB, ZERO, VSEQINSZ, 4, 0); + MATHB(p, MATH0, ADD, ip_hdr_len, VSEQOUTSZ, 4, IMMED2); + move_jump = MOVE(p, DESCBUF, 0, OFIFO, 0, 8, WAITCOMP | IMMED); + move_jump_back = MOVE(p, OFIFO, 0, DESCBUF, 0, 8, IMMED); + SEQFIFOLOAD(p, SKIP, ip_hdr_len, 0); + SEQFIFOLOAD(p, MSG2, 0, VLF | LAST2); + SEQFIFOSTORE(p, SKIP, 0, 0, VLF); + SEQSTORE(p, CONTEXT2, 0, IPSEC_ICV_MD5_TRUNC_SIZE, 0); + seqout_ptr_jump = JUMP(p, seqout_ptr, LOCAL_JUMP, ALL_TRUE, CALM); + + LOAD(p, LDST_SRCDST_WORD_CLRW | CLRW_CLR_C1MODE | CLRW_CLR_C1DATAS | + CLRW_CLR_C1CTX | CLRW_CLR_C1KEY | CLRW_CLR_C2MODE | + CLRW_CLR_C2DATAS | CLRW_CLR_C2CTX | CLRW_RESET_CLS1_CHA, CLRW, 0, + 4, 0); + SEQINPTR(p, 0, 65535, RTO); + MATHB(p, MATH0, ADD, + (uint64_t)(ip_hdr_len + IPSEC_ICV_MD5_TRUNC_SIZE), SEQINSZ, 4, + IMMED2); + KEY(p, KEY1, cipherdata->key_enc_flags, cipherdata->key, + cipherdata->keylen, INLINE_KEY(cipherdata)); + PROTOCOL(p, OP_TYPE_DECAP_PROTOCOL, OP_PCLID_IPSEC, + (uint16_t)(cipherdata->algtype | OP_PCL_IPSEC_HMAC_MD5_96)); +/* + * TODO: RTA currently doesn't support adding labels in or after Job Descriptor. + * To be changed when proper support is added in RTA. + */ + /* Label the SEQ OUT PTR */ + SET_LABEL(p, seqout_ptr); + seqout_ptr += 2; + /* Label the Output Length */ + SET_LABEL(p, outlen); + outlen += 4; + /* Label the SEQ IN PTR */ + SET_LABEL(p, seqin_ptr); + seqin_ptr += 5; + /* Label the first word after JD */ + SET_LABEL(p, swapped_seqout_fields); + swapped_seqout_fields += 8; + /* Label the second word after JD */ + SET_LABEL(p, swapped_seqout_ptr); + swapped_seqout_ptr += 9; + + PATCH_HDR(p, phdr, hdr); + PATCH_JUMP(p, pkeyjump, keyjump); + PATCH_JUMP(p, seqout_ptr_jump, seqout_ptr); + PATCH_JUMP(p, swapped_seqout_ptr_jump, swapped_seqout_ptr); + PATCH_MOVE(p, move_jump, jump_cmd); + PATCH_MOVE(p, move_jump_back, seqin_ptr); + PATCH_MOVE(p, move_seqin_ptr, outlen); + PATCH_MOVE(p, write_swapped_seqout_ptr, swapped_seqout_fields); + return PROGRAM_FINALIZE(p); +} + +/** + * IPSEC_NEW_ENC_BASE_DESC_LEN - IPsec new mode encap shared descriptor length + * + * Accounts only for the "base" commands and is intended to be used by upper + * layers to determine whether Outer IP Header and/or keys can be inlined or + * not. To be used as first parameter of rta_inline_query(). + */ +#define IPSEC_NEW_ENC_BASE_DESC_LEN (5 * CAAM_CMD_SZ + \ + sizeof(struct ipsec_encap_pdb)) + +/** + * IPSEC_NEW_NULL_ENC_BASE_DESC_LEN - IPsec new mode encap shared descriptor + * length for the case of + * NULL encryption / authentication + * + * Accounts only for the "base" commands and is intended to be used by upper + * layers to determine whether Outer IP Header and/or key can be inlined or + * not. To be used as first parameter of rta_inline_query(). + */ +#define IPSEC_NEW_NULL_ENC_BASE_DESC_LEN (4 * CAAM_CMD_SZ + \ + sizeof(struct ipsec_encap_pdb)) + +/** + * cnstr_shdsc_ipsec_new_encap - IPSec new mode ESP encapsulation + * protocol-level shared descriptor. + * @descbuf: pointer to buffer used for descriptor construction + * @ps: if 36/40bit addressing is desired, this parameter must be true + * @swap: must be true when core endianness doesn't match SEC endianness + * @pdb: pointer to the PDB to be used with this descriptor + * This structure will be copied inline to the descriptor under + * construction. No error checking will be made. Refer to the + * block guide for details about the encapsulation PDB. + * @opt_ip_hdr: pointer to Optional IP Header + * -if OIHI = PDBOPTS_ESP_OIHI_PDB_INL, opt_ip_hdr points to the buffer to + * be inlined in the PDB. Number of bytes (buffer size) copied is provided + * in pdb->ip_hdr_len. + * -if OIHI = PDBOPTS_ESP_OIHI_PDB_REF, opt_ip_hdr points to the address of + * the Optional IP Header. The address will be inlined in the PDB verbatim. + * -for other values of OIHI options field, opt_ip_hdr is not used. + * @cipherdata: pointer to block cipher transform definitions + * Valid algorithm values - one of OP_PCL_IPSEC_* + * @authdata: pointer to authentication transform definitions. + * If an authentication key is required by the protocol, a "normal" + * key must be provided; DKP (Derived Key Protocol) will be used to + * compute MDHA on the fly in HW. + * Valid algorithm values - one of OP_PCL_IPSEC_* + * + * Return: size of descriptor written in words or negative number on error + */ +static inline int +cnstr_shdsc_ipsec_new_encap(uint32_t *descbuf, bool ps, + bool swap, + struct ipsec_encap_pdb *pdb, + uint8_t *opt_ip_hdr, + struct alginfo *cipherdata, + struct alginfo *authdata) +{ + struct program prg; + struct program *p = &prg; + + LABEL(keyjmp); + REFERENCE(pkeyjmp); + LABEL(hdr); + REFERENCE(phdr); + + if (rta_sec_era < RTA_SEC_ERA_8) { + pr_err("IPsec new mode encap: available only for Era %d or above\n", + USER_SEC_ERA(RTA_SEC_ERA_8)); + return -ENOTSUP; + } + + PROGRAM_CNTXT_INIT(p, descbuf, 0); + if (swap) + PROGRAM_SET_BSWAP(p); + if (ps) + PROGRAM_SET_36BIT_ADDR(p); + phdr = SHR_HDR(p, SHR_SERIAL, hdr, 0); + + __rta_copy_ipsec_encap_pdb(p, pdb, cipherdata->algtype); + + switch (pdb->options & PDBOPTS_ESP_OIHI_MASK) { + case PDBOPTS_ESP_OIHI_PDB_INL: + COPY_DATA(p, opt_ip_hdr, pdb->ip_hdr_len); + break; + case PDBOPTS_ESP_OIHI_PDB_REF: + if (ps) + COPY_DATA(p, opt_ip_hdr, 8); + else + COPY_DATA(p, opt_ip_hdr, 4); + break; + default: + break; + } + SET_LABEL(p, hdr); + + pkeyjmp = JUMP(p, keyjmp, LOCAL_JUMP, ALL_TRUE, SHRD); + if (authdata->keylen) + __gen_auth_key(p, authdata); + if (cipherdata->keylen) + KEY(p, KEY1, cipherdata->key_enc_flags, cipherdata->key, + cipherdata->keylen, INLINE_KEY(cipherdata)); + SET_LABEL(p, keyjmp); + PROTOCOL(p, OP_TYPE_ENCAP_PROTOCOL, + OP_PCLID_IPSEC_NEW, + (uint16_t)(cipherdata->algtype | authdata->algtype)); + PATCH_JUMP(p, pkeyjmp, keyjmp); + PATCH_HDR(p, phdr, hdr); + return PROGRAM_FINALIZE(p); +} + +/** + * IPSEC_NEW_DEC_BASE_DESC_LEN - IPsec new mode decap shared descriptor length + * + * Accounts only for the "base" commands and is intended to be used by upper + * layers to determine whether keys can be inlined or not. To be used as first + * parameter of rta_inline_query(). + */ +#define IPSEC_NEW_DEC_BASE_DESC_LEN (5 * CAAM_CMD_SZ + \ + sizeof(struct ipsec_decap_pdb)) + +/** + * IPSEC_NEW_NULL_DEC_BASE_DESC_LEN - IPsec new mode decap shared descriptor + * length for the case of + * NULL decryption / authentication + * + * Accounts only for the "base" commands and is intended to be used by upper + * layers to determine whether key can be inlined or not. To be used as first + * parameter of rta_inline_query(). + */ +#define IPSEC_NEW_NULL_DEC_BASE_DESC_LEN (4 * CAAM_CMD_SZ + \ + sizeof(struct ipsec_decap_pdb)) + +/** + * cnstr_shdsc_ipsec_new_decap - IPSec new mode ESP decapsulation protocol-level + * shared descriptor. + * @descbuf: pointer to buffer used for descriptor construction + * @ps: if 36/40bit addressing is desired, this parameter must be true + * @swap: must be true when core endianness doesn't match SEC endianness + * @pdb: pointer to the PDB to be used with this descriptor + * This structure will be copied inline to the descriptor under + * construction. No error checking will be made. Refer to the + * block guide for details about the decapsulation PDB. + * @cipherdata: pointer to block cipher transform definitions + * Valid algorithm values 0 one of OP_PCL_IPSEC_* + * @authdata: pointer to authentication transform definitions. + * If an authentication key is required by the protocol, a "normal" + * key must be provided; DKP (Derived Key Protocol) will be used to + * compute MDHA on the fly in HW. + * Valid algorithm values - one of OP_PCL_IPSEC_* + * + * Return: size of descriptor written in words or negative number on error + */ +static inline int +cnstr_shdsc_ipsec_new_decap(uint32_t *descbuf, bool ps, + bool swap, + struct ipsec_decap_pdb *pdb, + struct alginfo *cipherdata, + struct alginfo *authdata) +{ + struct program prg; + struct program *p = &prg; + + LABEL(keyjmp); + REFERENCE(pkeyjmp); + LABEL(hdr); + REFERENCE(phdr); + + if (rta_sec_era < RTA_SEC_ERA_8) { + pr_err("IPsec new mode decap: available only for Era %d or above\n", + USER_SEC_ERA(RTA_SEC_ERA_8)); + return -ENOTSUP; + } + + PROGRAM_CNTXT_INIT(p, descbuf, 0); + if (swap) + PROGRAM_SET_BSWAP(p); + if (ps) + PROGRAM_SET_36BIT_ADDR(p); + phdr = SHR_HDR(p, SHR_SERIAL, hdr, 0); + __rta_copy_ipsec_decap_pdb(p, pdb, cipherdata->algtype); + SET_LABEL(p, hdr); + pkeyjmp = JUMP(p, keyjmp, LOCAL_JUMP, ALL_TRUE, SHRD); + if (authdata->keylen) + __gen_auth_key(p, authdata); + if (cipherdata->keylen) + KEY(p, KEY1, cipherdata->key_enc_flags, cipherdata->key, + cipherdata->keylen, INLINE_KEY(cipherdata)); + SET_LABEL(p, keyjmp); + PROTOCOL(p, OP_TYPE_DECAP_PROTOCOL, + OP_PCLID_IPSEC_NEW, + (uint16_t)(cipherdata->algtype | authdata->algtype)); + PATCH_JUMP(p, pkeyjmp, keyjmp); + PATCH_HDR(p, phdr, hdr); + return PROGRAM_FINALIZE(p); +} + +/** + * IPSEC_AUTH_VAR_BASE_DESC_LEN - IPsec encap/decap shared descriptor length + * for the case of variable-length authentication + * only data. + * Note: Only for SoCs with SEC_ERA >= 3. + * + * Accounts only for the "base" commands and is intended to be used by upper + * layers to determine whether keys can be inlined or not. To be used as first + * parameter of rta_inline_query(). + */ +#define IPSEC_AUTH_VAR_BASE_DESC_LEN (27 * CAAM_CMD_SZ) + +/** + * IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN - IPsec AES decap shared descriptor + * length for variable-length authentication only + * data. + * Note: Only for SoCs with SEC_ERA >= 3. + * + * Accounts only for the "base" commands and is intended to be used by upper + * layers to determine whether key can be inlined or not. To be used as first + * parameter of rta_inline_query(). + */ +#define IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN \ + (IPSEC_AUTH_VAR_BASE_DESC_LEN + CAAM_CMD_SZ) + +/** + * IPSEC_AUTH_BASE_DESC_LEN - IPsec encap/decap shared descriptor length + * + * Accounts only for the "base" commands and is intended to be used by upper + * layers to determine whether key can be inlined or not. To be used as first + * parameter of rta_inline_query(). + */ +#define IPSEC_AUTH_BASE_DESC_LEN (19 * CAAM_CMD_SZ) + +/** + * IPSEC_AUTH_AES_DEC_BASE_DESC_LEN - IPsec AES decap shared descriptor length + * + * Accounts only for the "base" commands and is intended to be used by upper + * layers to determine whether key can be inlined or not. To be used as first + * parameter of rta_inline_query(). + */ +#define IPSEC_AUTH_AES_DEC_BASE_DESC_LEN (IPSEC_AUTH_BASE_DESC_LEN + \ + CAAM_CMD_SZ) + +/** + * cnstr_shdsc_authenc - authenc-like descriptor + * @descbuf: pointer to buffer used for descriptor construction + * @ps: if 36/40bit addressing is desired, this parameter must be true + * @swap: if true, perform descriptor byte swapping on a 4-byte boundary + * @cipherdata: pointer to block cipher transform definitions. + * Valid algorithm values one of OP_ALG_ALGSEL_* {DES, 3DES, AES} + * Valid modes for: + * AES: OP_ALG_AAI_* {CBC, CTR} + * DES, 3DES: OP_ALG_AAI_CBC + * @authdata: pointer to authentication transform definitions. + * Valid algorithm values - one of OP_ALG_ALGSEL_* {MD5, SHA1, + * SHA224, SHA256, SHA384, SHA512} + * Note: The key for authentication is supposed to be given as plain text. + * Note: There's no support for keys longer than the block size of the + * underlying hash function, according to the selected algorithm. + * + * @ivlen: length of the IV to be read from the input frame, before any data + * to be processed + * @auth_only_len: length of the data to be authenticated-only (commonly IP + * header, IV, Sequence number and SPI) + * Note: Extended Sequence Number processing is NOT supported + * + * @trunc_len: the length of the ICV to be written to the output frame. If 0, + * then the corresponding length of the digest, according to the + * selected algorithm shall be used. + * @dir: Protocol direction, encapsulation or decapsulation (DIR_ENC/DIR_DEC) + * + * Note: Here's how the input frame needs to be formatted so that the processing + * will be done correctly: + * For encapsulation: + * Input: + * +----+----------------+---------------------------------------------+ + * | IV | Auth-only data | Padded data to be authenticated & Encrypted | + * +----+----------------+---------------------------------------------+ + * Output: + * +--------------------------------------+ + * | Authenticated & Encrypted data | ICV | + * +--------------------------------+-----+ + + * For decapsulation: + * Input: + * +----+----------------+--------------------------------+-----+ + * | IV | Auth-only data | Authenticated & Encrypted data | ICV | + * +----+----------------+--------------------------------+-----+ + * Output: + * +----+--------------------------+ + * | Decrypted & authenticated data | + * +----+--------------------------+ + * + * Note: This descriptor can use per-packet commands, encoded as below in the + * DPOVRD register: + * 32 24 16 0 + * +------+---------------------+ + * | 0x80 | 0x00| auth_only_len | + * +------+---------------------+ + * + * This mechanism is available only for SoCs having SEC ERA >= 3. In other + * words, this will not work for P4080TO2 + * + * Note: The descriptor does not add any kind of padding to the input data, + * so the upper layer needs to ensure that the data is padded properly, + * according to the selected cipher. Failure to do so will result in + * the descriptor failing with a data-size error. + * + * Return: size of descriptor written in words or negative number on error + */ +static inline int +cnstr_shdsc_authenc(uint32_t *descbuf, bool ps, bool swap, + struct alginfo *cipherdata, + struct alginfo *authdata, + uint16_t ivlen, uint16_t auth_only_len, + uint8_t trunc_len, uint8_t dir) +{ + struct program prg; + struct program *p = &prg; + const bool need_dk = (dir == DIR_DEC) && + (cipherdata->algtype == OP_ALG_ALGSEL_AES) && + (cipherdata->algmode == OP_ALG_AAI_CBC); + + LABEL(skip_patch_len); + LABEL(keyjmp); + LABEL(skipkeys); + LABEL(aonly_len_offset); + REFERENCE(pskip_patch_len); + REFERENCE(pkeyjmp); + REFERENCE(pskipkeys); + REFERENCE(read_len); + REFERENCE(write_len); + + PROGRAM_CNTXT_INIT(p, descbuf, 0); + + if (swap) + PROGRAM_SET_BSWAP(p); + if (ps) + PROGRAM_SET_36BIT_ADDR(p); + + /* + * Since we currently assume that key length is equal to hash digest + * size, it's ok to truncate keylen value. + */ + trunc_len = trunc_len && (trunc_len < authdata->keylen) ? + trunc_len : (uint8_t)authdata->keylen; + + SHR_HDR(p, SHR_SERIAL, 1, SC); + + /* + * M0 will contain the value provided by the user when creating + * the shared descriptor. If the user provided an override in + * DPOVRD, then M0 will contain that value + */ + MATHB(p, MATH0, ADD, auth_only_len, MATH0, 4, IMMED2); + + if (rta_sec_era >= RTA_SEC_ERA_3) { + /* + * Check if the user wants to override the auth-only len + */ + MATHB(p, DPOVRD, ADD, 0x80000000, MATH2, 4, IMMED2); + + /* + * No need to patch the length of the auth-only data read if + * the user did not override it + */ + pskip_patch_len = JUMP(p, skip_patch_len, LOCAL_JUMP, ALL_TRUE, + MATH_N); + + /* Get auth-only len in M0 */ + MATHB(p, MATH2, AND, 0xFFFF, MATH0, 4, IMMED2); + + /* + * Since M0 is used in calculations, don't mangle it, copy + * its content to M1 and use this for patching. + */ + MATHB(p, MATH0, ADD, MATH1, MATH1, 4, 0); + + read_len = MOVE(p, DESCBUF, 0, MATH1, 0, 6, WAITCOMP | IMMED); + write_len = MOVE(p, MATH1, 0, DESCBUF, 0, 8, WAITCOMP | IMMED); + + SET_LABEL(p, skip_patch_len); + } + /* + * MATH0 contains the value in DPOVRD w/o the MSB, or the initial + * value, as provided by the user at descriptor creation time + */ + if (dir == DIR_ENC) + MATHB(p, MATH0, ADD, ivlen, MATH0, 4, IMMED2); + else + MATHB(p, MATH0, ADD, ivlen + trunc_len, MATH0, 4, IMMED2); + + pkeyjmp = JUMP(p, keyjmp, LOCAL_JUMP, ALL_TRUE, SHRD); + + KEY(p, KEY2, authdata->key_enc_flags, authdata->key, authdata->keylen, + INLINE_KEY(authdata)); + + /* Insert Key */ + KEY(p, KEY1, cipherdata->key_enc_flags, cipherdata->key, + cipherdata->keylen, INLINE_KEY(cipherdata)); + + /* Do operation */ + ALG_OPERATION(p, authdata->algtype, OP_ALG_AAI_HMAC, + OP_ALG_AS_INITFINAL, + dir == DIR_ENC ? ICV_CHECK_DISABLE : ICV_CHECK_ENABLE, + dir); + + if (need_dk) + ALG_OPERATION(p, OP_ALG_ALGSEL_AES, cipherdata->algmode, + OP_ALG_AS_INITFINAL, ICV_CHECK_DISABLE, dir); + pskipkeys = JUMP(p, skipkeys, LOCAL_JUMP, ALL_TRUE, 0); + + SET_LABEL(p, keyjmp); + + ALG_OPERATION(p, authdata->algtype, OP_ALG_AAI_HMAC_PRECOMP, + OP_ALG_AS_INITFINAL, + dir == DIR_ENC ? ICV_CHECK_DISABLE : ICV_CHECK_ENABLE, + dir); + + if (need_dk) { + ALG_OPERATION(p, OP_ALG_ALGSEL_AES, cipherdata->algmode | + OP_ALG_AAI_DK, OP_ALG_AS_INITFINAL, + ICV_CHECK_DISABLE, dir); + SET_LABEL(p, skipkeys); + } else { + SET_LABEL(p, skipkeys); + ALG_OPERATION(p, cipherdata->algtype, cipherdata->algmode, + OP_ALG_AS_INITFINAL, ICV_CHECK_DISABLE, dir); + } + + /* + * Prepare the length of the data to be both encrypted/decrypted + * and authenticated/checked + */ + MATHB(p, SEQINSZ, SUB, MATH0, VSEQINSZ, 4, 0); + + MATHB(p, VSEQINSZ, SUB, MATH3, VSEQOUTSZ, 4, 0); + + /* Prepare for writing the output frame */ + SEQFIFOSTORE(p, MSG, 0, 0, VLF); + + SET_LABEL(p, aonly_len_offset); + + /* Read IV */ + if (cipherdata->algmode == OP_ALG_AAI_CTR) + SEQLOAD(p, CONTEXT1, 16, ivlen, 0); + else + SEQLOAD(p, CONTEXT1, 0, ivlen, 0); + + /* + * Read data needed only for authentication. This is overwritten above + * if the user requested it. + */ + SEQFIFOLOAD(p, MSG2, auth_only_len, 0); + + if (dir == DIR_ENC) { + /* + * Read input plaintext, encrypt and authenticate & write to + * output + */ + SEQFIFOLOAD(p, MSGOUTSNOOP, 0, VLF | LAST1 | LAST2 | FLUSH1); + + /* Finally, write the ICV */ + SEQSTORE(p, CONTEXT2, 0, trunc_len, 0); + } else { + /* + * Read input ciphertext, decrypt and authenticate & write to + * output + */ + SEQFIFOLOAD(p, MSGINSNOOP, 0, VLF | LAST1 | LAST2 | FLUSH1); + + /* Read the ICV to check */ + SEQFIFOLOAD(p, ICV2, trunc_len, LAST2); + } + + PATCH_JUMP(p, pkeyjmp, keyjmp); + PATCH_JUMP(p, pskipkeys, skipkeys); + PATCH_JUMP(p, pskipkeys, skipkeys); + + if (rta_sec_era >= RTA_SEC_ERA_3) { + PATCH_JUMP(p, pskip_patch_len, skip_patch_len); + PATCH_MOVE(p, read_len, aonly_len_offset); + PATCH_MOVE(p, write_len, aonly_len_offset); + } + + return PROGRAM_FINALIZE(p); +} + +#endif /* __DESC_IPSEC_H__ */ diff --git a/dpdk/drivers/crypto/dpaa2_sec/hw/rta.h b/dpdk/drivers/crypto/dpaa2_sec/hw/rta.h new file mode 100644 index 00000000..283ad600 --- /dev/null +++ b/dpdk/drivers/crypto/dpaa2_sec/hw/rta.h @@ -0,0 +1,954 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2008-2016 Freescale Semiconductor Inc. + * Copyright 2016 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __RTA_RTA_H__ +#define __RTA_RTA_H__ + +#include "rta/sec_run_time_asm.h" +#include "rta/fifo_load_store_cmd.h" +#include "rta/header_cmd.h" +#include "rta/jump_cmd.h" +#include "rta/key_cmd.h" +#include "rta/load_cmd.h" +#include "rta/math_cmd.h" +#include "rta/move_cmd.h" +#include "rta/nfifo_cmd.h" +#include "rta/operation_cmd.h" +#include "rta/protocol_cmd.h" +#include "rta/seq_in_out_ptr_cmd.h" +#include "rta/signature_cmd.h" +#include "rta/store_cmd.h" + +/** + * DOC: About + * + * RTA (Runtime Assembler) Library is an easy and flexible runtime method for + * writing SEC descriptors. It implements a thin abstraction layer above + * SEC commands set; the resulting code is compact and similar to a + * descriptor sequence. + * + * RTA library improves comprehension of the SEC code, adds flexibility for + * writing complex descriptors and keeps the code lightweight. Should be used + * by whom needs to encode descriptors at runtime, with comprehensible flow + * control in descriptor. + */ + +/** + * DOC: Usage + * + * RTA is used in kernel space by the SEC / CAAM (Cryptographic Acceleration and + * Assurance Module) kernel module (drivers/crypto/caam) and SEC / CAAM QI + * kernel module (Freescale QorIQ SDK). + * + * RTA is used in user space by USDPAA - User Space DataPath Acceleration + * Architecture (Freescale QorIQ SDK). + */ + +/** + * DOC: Descriptor Buffer Management Routines + * + * Contains details of RTA descriptor buffer management and SEC Era + * management routines. + */ + +/** + * PROGRAM_CNTXT_INIT - must be called before any descriptor run-time assembly + * call type field carry info i.e. whether descriptor is + * shared or job descriptor. + * @program: pointer to struct program + * @buffer: input buffer where the descriptor will be placed (uint32_t *) + * @offset: offset in input buffer from where the data will be written + * (unsigned int) + */ +#define PROGRAM_CNTXT_INIT(program, buffer, offset) \ + rta_program_cntxt_init(program, buffer, offset) + +/** + * PROGRAM_FINALIZE - must be called to mark completion of RTA call. + * @program: pointer to struct program + * + * Return: total size of the descriptor in words or negative number on error. + */ +#define PROGRAM_FINALIZE(program) rta_program_finalize(program) + +/** + * PROGRAM_SET_36BIT_ADDR - must be called to set pointer size to 36 bits + * @program: pointer to struct program + * + * Return: current size of the descriptor in words (unsigned int). + */ +#define PROGRAM_SET_36BIT_ADDR(program) rta_program_set_36bit_addr(program) + +/** + * PROGRAM_SET_BSWAP - must be called to enable byte swapping + * @program: pointer to struct program + * + * Byte swapping on a 4-byte boundary will be performed at the end - when + * calling PROGRAM_FINALIZE(). + * + * Return: current size of the descriptor in words (unsigned int). + */ +#define PROGRAM_SET_BSWAP(program) rta_program_set_bswap(program) + +/** + * WORD - must be called to insert in descriptor buffer a 32bit value + * @program: pointer to struct program + * @val: input value to be written in descriptor buffer (uint32_t) + * + * Return: the descriptor buffer offset where this command is inserted + * (unsigned int). + */ +#define WORD(program, val) rta_word(program, val) + +/** + * DWORD - must be called to insert in descriptor buffer a 64bit value + * @program: pointer to struct program + * @val: input value to be written in descriptor buffer (uint64_t) + * + * Return: the descriptor buffer offset where this command is inserted + * (unsigned int). + */ +#define DWORD(program, val) rta_dword(program, val) + +/** + * COPY_DATA - must be called to insert in descriptor buffer data larger than + * 64bits. + * @program: pointer to struct program + * @data: input data to be written in descriptor buffer (uint8_t *) + * @len: length of input data (unsigned int) + * + * Return: the descriptor buffer offset where this command is inserted + * (unsigned int). + */ +#define COPY_DATA(program, data, len) rta_copy_data(program, (data), (len)) + +/** + * DESC_LEN - determines job / shared descriptor buffer length (in words) + * @buffer: descriptor buffer (uint32_t *) + * + * Return: descriptor buffer length in words (unsigned int). + */ +#define DESC_LEN(buffer) rta_desc_len(buffer) + +/** + * DESC_BYTES - determines job / shared descriptor buffer length (in bytes) + * @buffer: descriptor buffer (uint32_t *) + * + * Return: descriptor buffer length in bytes (unsigned int). + */ +#define DESC_BYTES(buffer) rta_desc_bytes(buffer) + +/* + * SEC HW block revision. + * + * This *must not be confused with SEC version*: + * - SEC HW block revision format is "v" + * - SEC revision format is "x.y" + */ +extern enum rta_sec_era rta_sec_era; + +/** + * rta_set_sec_era - Set SEC Era HW block revision for which the RTA library + * will generate the descriptors. + * @era: SEC Era (enum rta_sec_era) + * + * Return: 0 if the ERA was set successfully, -1 otherwise (int) + * + * Warning 1: Must be called *only once*, *before* using any other RTA API + * routine. + * + * Warning 2: *Not thread safe*. + */ +static inline int +rta_set_sec_era(enum rta_sec_era era) +{ + if (era > MAX_SEC_ERA) { + rta_sec_era = DEFAULT_SEC_ERA; + pr_err("Unsupported SEC ERA. Defaulting to ERA %d\n", + DEFAULT_SEC_ERA + 1); + return -1; + } + + rta_sec_era = era; + return 0; +} + +/** + * rta_get_sec_era - Get SEC Era HW block revision for which the RTA library + * will generate the descriptors. + * + * Return: SEC Era (unsigned int). + */ +static inline unsigned int +rta_get_sec_era(void) +{ + return rta_sec_era; +} + +/** + * DOC: SEC Commands Routines + * + * Contains details of RTA wrapper routines over SEC engine commands. + */ + +/** + * SHR_HDR - Configures Shared Descriptor HEADER command + * @program: pointer to struct program + * @share: descriptor share state (enum rta_share_type) + * @start_idx: index in descriptor buffer where the execution of the shared + * descriptor should start (@c unsigned int). + * @flags: operational flags: RIF, DNR, CIF, SC, PD + * + * Return: On success, descriptor buffer offset where this command is inserted. + * On error, a negative error code; first error program counter will + * point to offset in descriptor buffer where the instruction should + * have been written. + */ +#define SHR_HDR(program, share, start_idx, flags) \ + rta_shr_header(program, share, start_idx, flags) + +/** + * JOB_HDR - Configures JOB Descriptor HEADER command + * @program: pointer to struct program + * @share: descriptor share state (enum rta_share_type) + * @start_idx: index in descriptor buffer where the execution of the job + * descriptor should start (unsigned int). In case SHR bit is present + * in flags, this will be the shared descriptor length. + * @share_desc: pointer to shared descriptor, in case SHR bit is set (uint64_t) + * @flags: operational flags: RSMS, DNR, TD, MTD, REO, SHR + * + * Return: On success, descriptor buffer offset where this command is inserted. + * On error, a negative error code; first error program counter will + * point to offset in descriptor buffer where the instruction should + * have been written. + */ +#define JOB_HDR(program, share, start_idx, share_desc, flags) \ + rta_job_header(program, share, start_idx, share_desc, flags, 0) + +/** + * JOB_HDR_EXT - Configures JOB Descriptor HEADER command + * @program: pointer to struct program + * @share: descriptor share state (enum rta_share_type) + * @start_idx: index in descriptor buffer where the execution of the job + * descriptor should start (unsigned int). In case SHR bit is present + * in flags, this will be the shared descriptor length. + * @share_desc: pointer to shared descriptor, in case SHR bit is set (uint64_t) + * @flags: operational flags: RSMS, DNR, TD, MTD, REO, SHR + * @ext_flags: extended header flags: DSV (DECO Select Valid), DECO Id (limited + * by DSEL_MASK). + * + * Return: On success, descriptor buffer offset where this command is inserted. + * On error, a negative error code; first error program counter will + * point to offset in descriptor buffer where the instruction should + * have been written. + */ +#define JOB_HDR_EXT(program, share, start_idx, share_desc, flags, ext_flags) \ + rta_job_header(program, share, start_idx, share_desc, flags | EXT, \ + ext_flags) + +/** + * MOVE - Configures MOVE and MOVE_LEN commands + * @program: pointer to struct program + * @src: internal source of data that will be moved: CONTEXT1, CONTEXT2, OFIFO, + * DESCBUF, MATH0-MATH3, IFIFOABD, IFIFOAB1, IFIFOAB2, AB1, AB2, ABD. + * @src_offset: offset in source data (uint16_t) + * @dst: internal destination of data that will be moved: CONTEXT1, CONTEXT2, + * OFIFO, DESCBUF, MATH0-MATH3, IFIFOAB1, IFIFOAB2, IFIFO, PKA, KEY1, + * KEY2, ALTSOURCE. + * @dst_offset: offset in destination data (uint16_t) + * @length: size of data to be moved: for MOVE must be specified as immediate + * value and IMMED flag must be set; for MOVE_LEN must be specified + * using MATH0-MATH3. + * @opt: operational flags: WAITCOMP, FLUSH1, FLUSH2, LAST1, LAST2, SIZE_WORD, + * SIZE_BYTE, SIZE_DWORD, IMMED (not valid for MOVE_LEN). + * + * Return: On success, descriptor buffer offset where this command is inserted. + * On error, a negative error code; first error program counter will + * point to offset in descriptor buffer where the instruction should + * have been written. + */ +#define MOVE(program, src, src_offset, dst, dst_offset, length, opt) \ + rta_move(program, __MOVE, src, src_offset, dst, dst_offset, length, opt) + +/** + * MOVEB - Configures MOVEB command + * @program: pointer to struct program + * @src: internal source of data that will be moved: CONTEXT1, CONTEXT2, OFIFO, + * DESCBUF, MATH0-MATH3, IFIFOABD, IFIFOAB1, IFIFOAB2, AB1, AB2, ABD. + * @src_offset: offset in source data (uint16_t) + * @dst: internal destination of data that will be moved: CONTEXT1, CONTEXT2, + * OFIFO, DESCBUF, MATH0-MATH3, IFIFOAB1, IFIFOAB2, IFIFO, PKA, KEY1, + * KEY2, ALTSOURCE. + * @dst_offset: offset in destination data (uint16_t) + * @length: size of data to be moved: for MOVE must be specified as immediate + * value and IMMED flag must be set; for MOVE_LEN must be specified + * using MATH0-MATH3. + * @opt: operational flags: WAITCOMP, FLUSH1, FLUSH2, LAST1, LAST2, SIZE_WORD, + * SIZE_BYTE, SIZE_DWORD, IMMED (not valid for MOVE_LEN). + * + * Identical with MOVE command if byte swapping not enabled; else - when src/dst + * is descriptor buffer or MATH registers, data type is byte array when MOVE + * data type is 4-byte array and vice versa. + * + * Return: On success, descriptor buffer offset where this command is inserted. + * On error, a negative error code; first error program counter will + * point to offset in descriptor buffer where the instruction should + * have been written. + */ +#define MOVEB(program, src, src_offset, dst, dst_offset, length, opt) \ + rta_move(program, __MOVEB, src, src_offset, dst, dst_offset, length, \ + opt) + +/** + * MOVEDW - Configures MOVEDW command + * @program: pointer to struct program + * @src: internal source of data that will be moved: CONTEXT1, CONTEXT2, OFIFO, + * DESCBUF, MATH0-MATH3, IFIFOABD, IFIFOAB1, IFIFOAB2, AB1, AB2, ABD. + * @src_offset: offset in source data (uint16_t) + * @dst: internal destination of data that will be moved: CONTEXT1, CONTEXT2, + * OFIFO, DESCBUF, MATH0-MATH3, IFIFOAB1, IFIFOAB2, IFIFO, PKA, KEY1, + * KEY2, ALTSOURCE. + * @dst_offset: offset in destination data (uint16_t) + * @length: size of data to be moved: for MOVE must be specified as immediate + * value and IMMED flag must be set; for MOVE_LEN must be specified + * using MATH0-MATH3. + * @opt: operational flags: WAITCOMP, FLUSH1, FLUSH2, LAST1, LAST2, SIZE_WORD, + * SIZE_BYTE, SIZE_DWORD, IMMED (not valid for MOVE_LEN). + * + * Identical with MOVE command, with the following differences: data type is + * 8-byte array; word swapping is performed when SEC is programmed in little + * endian mode. + * + * Return: On success, descriptor buffer offset where this command is inserted. + * On error, a negative error code; first error program counter will + * point to offset in descriptor buffer where the instruction should + * have been written. + */ +#define MOVEDW(program, src, src_offset, dst, dst_offset, length, opt) \ + rta_move(program, __MOVEDW, src, src_offset, dst, dst_offset, length, \ + opt) + +/** + * FIFOLOAD - Configures FIFOLOAD command to load message data, PKHA data, IV, + * ICV, AAD and bit length message data into Input Data FIFO. + * @program: pointer to struct program + * @data: input data type to store: PKHA registers, IFIFO, MSG1, MSG2, + * MSGOUTSNOOP, MSGINSNOOP, IV1, IV2, AAD1, ICV1, ICV2, BIT_DATA, SKIP. + * @src: pointer or actual data in case of immediate load; IMMED, COPY and DCOPY + * flags indicate action taken (inline imm data, inline ptr, inline from + * ptr). + * @length: number of bytes to load (uint32_t) + * @flags: operational flags: SGF, IMMED, EXT, CLASS1, CLASS2, BOTH, FLUSH1, + * LAST1, LAST2, COPY, DCOPY. + * + * Return: On success, descriptor buffer offset where this command is inserted. + * On error, a negative error code; first error program counter will + * point to offset in descriptor buffer where the instruction should + * have been written. + */ +#define FIFOLOAD(program, data, src, length, flags) \ + rta_fifo_load(program, data, src, length, flags) + +/** + * SEQFIFOLOAD - Configures SEQ FIFOLOAD command to load message data, PKHA + * data, IV, ICV, AAD and bit length message data into Input Data + * FIFO. + * @program: pointer to struct program + * @data: input data type to store: PKHA registers, IFIFO, MSG1, MSG2, + * MSGOUTSNOOP, MSGINSNOOP, IV1, IV2, AAD1, ICV1, ICV2, BIT_DATA, SKIP. + * @length: number of bytes to load; can be set to 0 for SEQ command w/ VLF set + * (uint32_t). + * @flags: operational flags: VLF, CLASS1, CLASS2, BOTH, FLUSH1, LAST1, LAST2, + * AIDF. + * + * Return: On success, descriptor buffer offset where this command is inserted. + * On error, a negative error code; first error program counter will + * point to offset in descriptor buffer where the instruction should + * have been written. + */ +#define SEQFIFOLOAD(program, data, length, flags) \ + rta_fifo_load(program, data, NONE, length, flags|SEQ) + +/** + * FIFOSTORE - Configures FIFOSTORE command, to move data from Output Data FIFO + * to external memory via DMA. + * @program: pointer to struct program + * @data: output data type to store: PKHA registers, IFIFO, OFIFO, RNG, + * RNGOFIFO, AFHA_SBOX, MDHA_SPLIT_KEY, MSG, KEY1, KEY2, SKIP. + * @encrypt_flags: store data encryption mode: EKT, TK + * @dst: pointer to store location (uint64_t) + * @length: number of bytes to load (uint32_t) + * @flags: operational flags: SGF, CONT, EXT, CLASS1, CLASS2, BOTH + * + * Return: On success, descriptor buffer offset where this command is inserted. + * On error, a negative error code; first error program counter will + * point to offset in descriptor buffer where the instruction should + * have been written. + */ +#define FIFOSTORE(program, data, encrypt_flags, dst, length, flags) \ + rta_fifo_store(program, data, encrypt_flags, dst, length, flags) + +/** + * SEQFIFOSTORE - Configures SEQ FIFOSTORE command, to move data from Output + * Data FIFO to external memory via DMA. + * @program: pointer to struct program + * @data: output data type to store: PKHA registers, IFIFO, OFIFO, RNG, + * RNGOFIFO, AFHA_SBOX, MDHA_SPLIT_KEY, MSG, KEY1, KEY2, METADATA, SKIP. + * @encrypt_flags: store data encryption mode: EKT, TK + * @length: number of bytes to load; can be set to 0 for SEQ command w/ VLF set + * (uint32_t). + * @flags: operational flags: VLF, CONT, EXT, CLASS1, CLASS2, BOTH + * + * Return: On success, descriptor buffer offset where this command is inserted. + * On error, a negative error code; first error program counter will + * point to offset in descriptor buffer where the instruction should + * have been written. + */ +#define SEQFIFOSTORE(program, data, encrypt_flags, length, flags) \ + rta_fifo_store(program, data, encrypt_flags, 0, length, flags|SEQ) + +/** + * KEY - Configures KEY and SEQ KEY commands + * @program: pointer to struct program + * @key_dst: key store location: KEY1, KEY2, PKE, AFHA_SBOX, MDHA_SPLIT_KEY + * @encrypt_flags: key encryption mode: ENC, EKT, TK, NWB, PTS + * @src: pointer or actual data in case of immediate load (uint64_t); IMMED, + * COPY and DCOPY flags indicate action taken (inline imm data, + * inline ptr, inline from ptr). + * @length: number of bytes to load; can be set to 0 for SEQ command w/ VLF set + * (uint32_t). + * @flags: operational flags: for KEY: SGF, IMMED, COPY, DCOPY; for SEQKEY: SEQ, + * VLF, AIDF. + * + * Return: On success, descriptor buffer offset where this command is inserted. + * On error, a negative error code; first error program counter will + * point to offset in descriptor buffer where the instruction should + * have been written. + */ +#define KEY(program, key_dst, encrypt_flags, src, length, flags) \ + rta_key(program, key_dst, encrypt_flags, src, length, flags) + +/** + * SEQINPTR - Configures SEQ IN PTR command + * @program: pointer to struct program + * @src: starting address for Input Sequence (uint64_t) + * @length: number of bytes in (or to be added to) Input Sequence (uint32_t) + * @flags: operational flags: RBS, INL, SGF, PRE, EXT, RTO, RJD, SOP (when PRE, + * RTO or SOP are set, @src parameter must be 0). + * + * Return: On success, descriptor buffer offset where this command is inserted. + * On error, a negative error code; first error program counter will + * point to offset in descriptor buffer where the instruction should + * have been written. + */ +#define SEQINPTR(program, src, length, flags) \ + rta_seq_in_ptr(program, src, length, flags) + +/** + * SEQOUTPTR - Configures SEQ OUT PTR command + * @program: pointer to struct program + * @dst: starting address for Output Sequence (uint64_t) + * @length: number of bytes in (or to be added to) Output Sequence (uint32_t) + * @flags: operational flags: SGF, PRE, EXT, RTO, RST, EWS (when PRE or RTO are + * set, @dst parameter must be 0). + * + * Return: On success, descriptor buffer offset where this command is inserted. + * On error, a negative error code; first error program counter will + * point to offset in descriptor buffer where the instruction should + * have been written. + */ +#define SEQOUTPTR(program, dst, length, flags) \ + rta_seq_out_ptr(program, dst, length, flags) + +/** + * ALG_OPERATION - Configures ALGORITHM OPERATION command + * @program: pointer to struct program + * @cipher_alg: algorithm to be used + * @aai: Additional Algorithm Information; contains mode information that is + * associated with the algorithm (check desc.h for specific values). + * @algo_state: algorithm state; defines the state of the algorithm that is + * being executed (check desc.h file for specific values). + * @icv_check: ICV checking; selects whether the algorithm should check + * calculated ICV with known ICV: ICV_CHECK_ENABLE, + * ICV_CHECK_DISABLE. + * @enc: selects between encryption and decryption: DIR_ENC, DIR_DEC + * + * Return: On success, descriptor buffer offset where this command is inserted. + * On error, a negative error code; first error program counter will + * point to offset in descriptor buffer where the instruction should + * have been written. + */ +#define ALG_OPERATION(program, cipher_alg, aai, algo_state, icv_check, enc) \ + rta_operation(program, cipher_alg, aai, algo_state, icv_check, enc) + +/** + * PROTOCOL - Configures PROTOCOL OPERATION command + * @program: pointer to struct program + * @optype: operation type: OP_TYPE_UNI_PROTOCOL / OP_TYPE_DECAP_PROTOCOL / + * OP_TYPE_ENCAP_PROTOCOL. + * @protid: protocol identifier value (check desc.h file for specific values) + * @protoinfo: protocol dependent value (check desc.h file for specific values) + * + * Return: On success, descriptor buffer offset where this command is inserted. + * On error, a negative error code; first error program counter will + * point to offset in descriptor buffer where the instruction should + * have been written. + */ +#define PROTOCOL(program, optype, protid, protoinfo) \ + rta_proto_operation(program, optype, protid, protoinfo) + +/** + * DKP_PROTOCOL - Configures DKP (Derived Key Protocol) PROTOCOL command + * @program: pointer to struct program + * @protid: protocol identifier value - one of the following: + * OP_PCLID_DKP_{MD5 | SHA1 | SHA224 | SHA256 | SHA384 | SHA512} + * @key_src: How the initial ("negotiated") key is provided to the DKP protocol. + * Valid values - one of OP_PCL_DKP_SRC_{IMM, SEQ, PTR, SGF}. Not all + * (key_src,key_dst) combinations are allowed. + * @key_dst: How the derived ("split") key is returned by the DKP protocol. + * Valid values - one of OP_PCL_DKP_DST_{IMM, SEQ, PTR, SGF}. Not all + * (key_src,key_dst) combinations are allowed. + * @keylen: length of the initial key, in bytes (uint16_t) + * @key: address where algorithm key resides; virtual address if key_type is + * RTA_DATA_IMM, physical (bus) address if key_type is RTA_DATA_PTR or + * RTA_DATA_IMM_DMA. + * @key_type: enum rta_data_type + * Return: On success, descriptor buffer offset where this command is inserted. + * On error, a negative error code; first error program counter will + * point to offset in descriptor buffer where the instruction should + * have been written. + */ +#define DKP_PROTOCOL(program, protid, key_src, key_dst, keylen, key, key_type) \ + rta_dkp_proto(program, protid, key_src, key_dst, keylen, key, key_type) + +/** + * PKHA_OPERATION - Configures PKHA OPERATION command + * @program: pointer to struct program + * @op_pkha: PKHA operation; indicates the modular arithmetic function to + * execute (check desc.h file for specific values). + * + * Return: On success, descriptor buffer offset where this command is inserted. + * On error, a negative error code; first error program counter will + * point to offset in descriptor buffer where the instruction should + * have been written. + */ +#define PKHA_OPERATION(program, op_pkha) rta_pkha_operation(program, op_pkha) + +/** + * JUMP - Configures JUMP command + * @program: pointer to struct program + * @addr: local offset for local jumps or address pointer for non-local jumps; + * IMM or PTR macros must be used to indicate type. + * @jump_type: type of action taken by jump (enum rta_jump_type) + * @test_type: defines how jump conditions are evaluated (enum rta_jump_cond) + * @cond: jump conditions: operational flags - DONE1, DONE2, BOTH; various + * sharing and wait conditions (JSL = 1) - NIFP, NIP, NOP, NCP, CALM, + * SELF, SHARED, JQP; Math and PKHA status conditions (JSL = 0) - Z, N, + * NV, C, PK0, PK1, PKP. + * + * Return: On success, descriptor buffer offset where this command is inserted. + * On error, a negative error code; first error program counter will + * point to offset in descriptor buffer where the instruction should + * have been written. + */ +#define JUMP(program, addr, jump_type, test_type, cond) \ + rta_jump(program, addr, jump_type, test_type, cond, NONE) + +/** + * JUMP_INC - Configures JUMP_INC command + * @program: pointer to struct program + * @addr: local offset; IMM or PTR macros must be used to indicate type + * @test_type: defines how jump conditions are evaluated (enum rta_jump_cond) + * @cond: jump conditions: Math status conditions (JSL = 0): Z, N, NV, C + * @src_dst: register to increment / decrement: MATH0-MATH3, DPOVRD, SEQINSZ, + * SEQOUTSZ, VSEQINSZ, VSEQOUTSZ. + * + * Return: On success, descriptor buffer offset where this command is inserted. + * On error, a negative error code; first error program counter will + * point to offset in descriptor buffer where the instruction should + * have been written. + */ +#define JUMP_INC(program, addr, test_type, cond, src_dst) \ + rta_jump(program, addr, LOCAL_JUMP_INC, test_type, cond, src_dst) + +/** + * JUMP_DEC - Configures JUMP_DEC command + * @program: pointer to struct program + * @addr: local offset; IMM or PTR macros must be used to indicate type + * @test_type: defines how jump conditions are evaluated (enum rta_jump_cond) + * @cond: jump conditions: Math status conditions (JSL = 0): Z, N, NV, C + * @src_dst: register to increment / decrement: MATH0-MATH3, DPOVRD, SEQINSZ, + * SEQOUTSZ, VSEQINSZ, VSEQOUTSZ. + * + * Return: On success, descriptor buffer offset where this command is inserted. + * On error, a negative error code; first error program counter will + * point to offset in descriptor buffer where the instruction should + * have been written. + */ +#define JUMP_DEC(program, addr, test_type, cond, src_dst) \ + rta_jump(program, addr, LOCAL_JUMP_DEC, test_type, cond, src_dst) + +/** + * LOAD - Configures LOAD command to load data registers from descriptor or from + * a memory location. + * @program: pointer to struct program + * @addr: immediate value or pointer to the data to be loaded; IMMED, COPY and + * DCOPY flags indicate action taken (inline imm data, inline ptr, inline + * from ptr). + * @dst: destination register (uint64_t) + * @offset: start point to write data in destination register (uint32_t) + * @length: number of bytes to load (uint32_t) + * @flags: operational flags: VLF, IMMED, COPY, DCOPY + * + * Return: On success, descriptor buffer offset where this command is inserted. + * On error, a negative error code; first error program counter will + * point to offset in descriptor buffer where the instruction should + * have been written. + */ +#define LOAD(program, addr, dst, offset, length, flags) \ + rta_load(program, addr, dst, offset, length, flags) + +/** + * SEQLOAD - Configures SEQ LOAD command to load data registers from descriptor + * or from a memory location. + * @program: pointer to struct program + * @dst: destination register (uint64_t) + * @offset: start point to write data in destination register (uint32_t) + * @length: number of bytes to load (uint32_t) + * @flags: operational flags: SGF + * + * Return: On success, descriptor buffer offset where this command is inserted. + * On error, a negative error code; first error program counter will + * point to offset in descriptor buffer where the instruction should + * have been written. + */ +#define SEQLOAD(program, dst, offset, length, flags) \ + rta_load(program, NONE, dst, offset, length, flags|SEQ) + +/** + * STORE - Configures STORE command to read data from registers and write them + * to a memory location. + * @program: pointer to struct program + * @src: immediate value or source register for data to be stored: KEY1SZ, + * KEY2SZ, DJQDA, MODE1, MODE2, DJQCTRL, DATA1SZ, DATA2SZ, DSTAT, ICV1SZ, + * ICV2SZ, DPID, CCTRL, ICTRL, CLRW, CSTAT, MATH0-MATH3, PKHA registers, + * CONTEXT1, CONTEXT2, DESCBUF, JOBDESCBUF, SHAREDESCBUF. In case of + * immediate value, IMMED, COPY and DCOPY flags indicate action taken + * (inline imm data, inline ptr, inline from ptr). + * @offset: start point for reading from source register (uint16_t) + * @dst: pointer to store location (uint64_t) + * @length: number of bytes to store (uint32_t) + * @flags: operational flags: VLF, IMMED, COPY, DCOPY + * + * Return: On success, descriptor buffer offset where this command is inserted. + * On error, a negative error code; first error program counter will + * point to offset in descriptor buffer where the instruction should + * have been written. + */ +#define STORE(program, src, offset, dst, length, flags) \ + rta_store(program, src, offset, dst, length, flags) + +/** + * SEQSTORE - Configures SEQ STORE command to read data from registers and write + * them to a memory location. + * @program: pointer to struct program + * @src: immediate value or source register for data to be stored: KEY1SZ, + * KEY2SZ, DJQDA, MODE1, MODE2, DJQCTRL, DATA1SZ, DATA2SZ, DSTAT, ICV1SZ, + * ICV2SZ, DPID, CCTRL, ICTRL, CLRW, CSTAT, MATH0-MATH3, PKHA registers, + * CONTEXT1, CONTEXT2, DESCBUF, JOBDESCBUF, SHAREDESCBUF. In case of + * immediate value, IMMED, COPY and DCOPY flags indicate action taken + * (inline imm data, inline ptr, inline from ptr). + * @offset: start point for reading from source register (uint16_t) + * @length: number of bytes to store (uint32_t) + * @flags: operational flags: SGF, IMMED, COPY, DCOPY + * + * Return: On success, descriptor buffer offset where this command is inserted. + * On error, a negative error code; first error program counter will + * point to offset in descriptor buffer where the instruction should + * have been written. + */ +#define SEQSTORE(program, src, offset, length, flags) \ + rta_store(program, src, offset, NONE, length, flags|SEQ) + +/** + * MATHB - Configures MATHB command to perform binary operations + * @program: pointer to struct program + * @operand1: first operand: MATH0-MATH3, DPOVRD, SEQINSZ, SEQOUTSZ, VSEQINSZ, + * VSEQOUTSZ, ZERO, ONE, NONE, Immediate value. IMMED must be used to + * indicate immediate value. + * @operator: function to be performed: ADD, ADDC, SUB, SUBB, OR, AND, XOR, + * LSHIFT, RSHIFT, SHLD. + * @operand2: second operand: MATH0-MATH3, DPOVRD, VSEQINSZ, VSEQOUTSZ, ABD, + * OFIFO, JOBSRC, ZERO, ONE, Immediate value. IMMED2 must be used to + * indicate immediate value. + * @result: destination for the result: MATH0-MATH3, DPOVRD, SEQINSZ, SEQOUTSZ, + * NONE, VSEQINSZ, VSEQOUTSZ. + * @length: length in bytes of the operation and the immediate value, if there + * is one (int). + * @opt: operational flags: IFB, NFU, STL, SWP, IMMED, IMMED2 + * + * Return: On success, descriptor buffer offset where this command is inserted. + * On error, a negative error code; first error program counter will + * point to offset in descriptor buffer where the instruction should + * have been written. + */ +#define MATHB(program, operand1, operator, operand2, result, length, opt) \ + rta_math(program, operand1, MATH_FUN_##operator, operand2, result, \ + length, opt) + +/** + * MATHI - Configures MATHI command to perform binary operations + * @program: pointer to struct program + * @operand: if !SSEL: MATH0-MATH3, DPOVRD, SEQINSZ, SEQOUTSZ, VSEQINSZ, + * VSEQOUTSZ, ZERO, ONE. + * if SSEL: MATH0-MATH3, DPOVRD, VSEQINSZ, VSEQOUTSZ, ABD, OFIFO, + * JOBSRC, ZERO, ONE. + * @operator: function to be performed: ADD, ADDC, SUB, SUBB, OR, AND, XOR, + * LSHIFT, RSHIFT, FBYT (for !SSEL only). + * @imm: Immediate value (uint8_t). IMMED must be used to indicate immediate + * value. + * @result: destination for the result: MATH0-MATH3, DPOVRD, SEQINSZ, SEQOUTSZ, + * NONE, VSEQINSZ, VSEQOUTSZ. + * @length: length in bytes of the operation and the immediate value, if there + * is one (int). @imm is left-extended with zeros if needed. + * @opt: operational flags: NFU, SSEL, SWP, IMMED + * + * If !SSEL, @operand <@operator> @imm -> @result + * If SSEL, @imm <@operator> @operand -> @result + * + * Return: On success, descriptor buffer offset where this command is inserted. + * On error, a negative error code; first error program counter will + * point to offset in descriptor buffer where the instruction should + * have been written. + */ +#define MATHI(program, operand, operator, imm, result, length, opt) \ + rta_mathi(program, operand, MATH_FUN_##operator, imm, result, length, \ + opt) + +/** + * MATHU - Configures MATHU command to perform unary operations + * @program: pointer to struct program + * @operand1: operand: MATH0-MATH3, DPOVRD, SEQINSZ, SEQOUTSZ, VSEQINSZ, + * VSEQOUTSZ, ZERO, ONE, NONE, Immediate value. IMMED must be used to + * indicate immediate value. + * @operator: function to be performed: ZBYT, BSWAP + * @result: destination for the result: MATH0-MATH3, DPOVRD, SEQINSZ, SEQOUTSZ, + * NONE, VSEQINSZ, VSEQOUTSZ. + * @length: length in bytes of the operation and the immediate value, if there + * is one (int). + * @opt: operational flags: NFU, STL, SWP, IMMED + * + * Return: On success, descriptor buffer offset where this command is inserted. + * On error, a negative error code; first error program counter will + * point to offset in descriptor buffer where the instruction should + * have been written. + */ +#define MATHU(program, operand1, operator, result, length, opt) \ + rta_math(program, operand1, MATH_FUN_##operator, NONE, result, length, \ + opt) + +/** + * SIGNATURE - Configures SIGNATURE command + * @program: pointer to struct program + * @sign_type: signature type: SIGN_TYPE_FINAL, SIGN_TYPE_FINAL_RESTORE, + * SIGN_TYPE_FINAL_NONZERO, SIGN_TYPE_IMM_2, SIGN_TYPE_IMM_3, + * SIGN_TYPE_IMM_4. + * + * After SIGNATURE command, DWORD or WORD must be used to insert signature in + * descriptor buffer. + * + * Return: On success, descriptor buffer offset where this command is inserted. + * On error, a negative error code; first error program counter will + * point to offset in descriptor buffer where the instruction should + * have been written. + */ +#define SIGNATURE(program, sign_type) rta_signature(program, sign_type) + +/** + * NFIFOADD - Configures NFIFO command, a shortcut of RTA Load command to write + * to iNfo FIFO. + * @program: pointer to struct program + * @src: source for the input data in Alignment Block:IFIFO, OFIFO, PAD, + * MSGOUTSNOOP, ALTSOURCE, OFIFO_SYNC, MSGOUTSNOOP_ALT. + * @data: type of data that is going through the Input Data FIFO: MSG, MSG1, + * MSG2, IV1, IV2, ICV1, ICV2, SAD1, AAD1, AAD2, AFHA_SBOX, SKIP, + * PKHA registers, AB1, AB2, ABD. + * @length: length of the data copied in FIFO registers (uint32_t) + * @flags: select options between: + * -operational flags: LAST1, LAST2, FLUSH1, FLUSH2, OC, BP + * -when PAD is selected as source: BM, PR, PS + * -padding type: PAD_ZERO, PAD_NONZERO, PAD_INCREMENT, PAD_RANDOM, + * PAD_ZERO_N1, PAD_NONZERO_0, PAD_N1, PAD_NONZERO_N + * + * Return: On success, descriptor buffer offset where this command is inserted. + * On error, a negative error code; first error program counter will + * point to offset in descriptor buffer where the instruction should + * have been written. + */ +#define NFIFOADD(program, src, data, length, flags) \ + rta_nfifo_load(program, src, data, length, flags) + +/** + * DOC: Self Referential Code Management Routines + * + * Contains details of RTA self referential code routines. + */ + +/** + * REFERENCE - initialize a variable used for storing an index inside a + * descriptor buffer. + * @ref: reference to a descriptor buffer's index where an update is required + * with a value that will be known latter in the program flow. + */ +#define REFERENCE(ref) int ref = -1 + +/** + * LABEL - initialize a variable used for storing an index inside a descriptor + * buffer. + * @label: label stores the value with what should be updated the REFERENCE line + * in the descriptor buffer. + */ +#define LABEL(label) unsigned int label = 0 + +/** + * SET_LABEL - set a LABEL value + * @program: pointer to struct program + * @label: value that will be inserted in a line previously written in the + * descriptor buffer. + */ +#define SET_LABEL(program, label) (label = rta_set_label(program)) + +/** + * PATCH_JUMP - Auxiliary command to resolve self referential code + * @program: buffer to be updated (struct program *) + * @line: position in descriptor buffer where the update will be done; this + * value is previously retained in program flow using a reference near + * the sequence to be modified. + * @new_ref: updated value that will be inserted in descriptor buffer at the + * specified line; this value is previously obtained using SET_LABEL + * macro near the line that will be used as reference (unsigned int). + * For JUMP command, the value represents the offset field (in words). + * + * Return: 0 in case of success, a negative error code if it fails + */ +#define PATCH_JUMP(program, line, new_ref) rta_patch_jmp(program, line, new_ref) + +/** + * PATCH_MOVE - Auxiliary command to resolve self referential code + * @program: buffer to be updated (struct program *) + * @line: position in descriptor buffer where the update will be done; this + * value is previously retained in program flow using a reference near + * the sequence to be modified. + * @new_ref: updated value that will be inserted in descriptor buffer at the + * specified line; this value is previously obtained using SET_LABEL + * macro near the line that will be used as reference (unsigned int). + * For MOVE command, the value represents the offset field (in words). + * + * Return: 0 in case of success, a negative error code if it fails + */ +#define PATCH_MOVE(program, line, new_ref) \ + rta_patch_move(program, line, new_ref) + +/** + * PATCH_LOAD - Auxiliary command to resolve self referential code + * @program: buffer to be updated (struct program *) + * @line: position in descriptor buffer where the update will be done; this + * value is previously retained in program flow using a reference near + * the sequence to be modified. + * @new_ref: updated value that will be inserted in descriptor buffer at the + * specified line; this value is previously obtained using SET_LABEL + * macro near the line that will be used as reference (unsigned int). + * For LOAD command, the value represents the offset field (in words). + * + * Return: 0 in case of success, a negative error code if it fails + */ +#define PATCH_LOAD(program, line, new_ref) \ + rta_patch_load(program, line, new_ref) + +/** + * PATCH_STORE - Auxiliary command to resolve self referential code + * @program: buffer to be updated (struct program *) + * @line: position in descriptor buffer where the update will be done; this + * value is previously retained in program flow using a reference near + * the sequence to be modified. + * @new_ref: updated value that will be inserted in descriptor buffer at the + * specified line; this value is previously obtained using SET_LABEL + * macro near the line that will be used as reference (unsigned int). + * For STORE command, the value represents the offset field (in words). + * + * Return: 0 in case of success, a negative error code if it fails + */ +#define PATCH_STORE(program, line, new_ref) \ + rta_patch_store(program, line, new_ref) + +/** + * PATCH_HDR - Auxiliary command to resolve self referential code + * @program: buffer to be updated (struct program *) + * @line: position in descriptor buffer where the update will be done; this + * value is previously retained in program flow using a reference near + * the sequence to be modified. + * @new_ref: updated value that will be inserted in descriptor buffer at the + * specified line; this value is previously obtained using SET_LABEL + * macro near the line that will be used as reference (unsigned int). + * For HEADER command, the value represents the start index field. + * + * Return: 0 in case of success, a negative error code if it fails + */ +#define PATCH_HDR(program, line, new_ref) \ + rta_patch_header(program, line, new_ref) + +/** + * PATCH_RAW - Auxiliary command to resolve self referential code + * @program: buffer to be updated (struct program *) + * @line: position in descriptor buffer where the update will be done; this + * value is previously retained in program flow using a reference near + * the sequence to be modified. + * @mask: mask to be used for applying the new value (unsigned int). The mask + * selects which bits from the provided @new_val are taken into + * consideration when overwriting the existing value. + * @new_val: updated value that will be masked using the provided mask value + * and inserted in descriptor buffer at the specified line. + * + * Return: 0 in case of success, a negative error code if it fails + */ +#define PATCH_RAW(program, line, mask, new_val) \ + rta_patch_raw(program, line, mask, new_val) + +#endif /* __RTA_RTA_H__ */ diff --git a/dpdk/drivers/crypto/dpaa2_sec/hw/rta/fifo_load_store_cmd.h b/dpdk/drivers/crypto/dpaa2_sec/hw/rta/fifo_load_store_cmd.h new file mode 100644 index 00000000..dc0a3342 --- /dev/null +++ b/dpdk/drivers/crypto/dpaa2_sec/hw/rta/fifo_load_store_cmd.h @@ -0,0 +1,346 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2008-2016 Freescale Semiconductor Inc. + * Copyright 2016 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __RTA_FIFO_LOAD_STORE_CMD_H__ +#define __RTA_FIFO_LOAD_STORE_CMD_H__ + +extern enum rta_sec_era rta_sec_era; + +static const uint32_t fifo_load_table[][2] = { +/*1*/ { PKA0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_PK_A0 }, + { PKA1, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_PK_A1 }, + { PKA2, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_PK_A2 }, + { PKA3, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_PK_A3 }, + { PKB0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_PK_B0 }, + { PKB1, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_PK_B1 }, + { PKB2, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_PK_B2 }, + { PKB3, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_PK_B3 }, + { PKA, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_PK_A }, + { PKB, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_PK_B }, + { PKN, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_PK_N }, + { SKIP, FIFOLD_CLASS_SKIP }, + { MSG1, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG }, + { MSG2, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG }, + { MSGOUTSNOOP, FIFOLD_CLASS_BOTH | FIFOLD_TYPE_MSG1OUT2 }, + { MSGINSNOOP, FIFOLD_CLASS_BOTH | FIFOLD_TYPE_MSG }, + { IV1, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_IV }, + { IV2, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_IV }, + { AAD1, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_AAD }, + { ICV1, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_ICV }, + { ICV2, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_ICV }, + { BIT_DATA, FIFOLD_TYPE_BITDATA }, +/*23*/ { IFIFO, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_NOINFOFIFO } +}; + +/* + * Allowed FIFO_LOAD input data types for each SEC Era. + * Values represent the number of entries from fifo_load_table[] that are + * supported. + */ +static const unsigned int fifo_load_table_sz[] = {22, 22, 23, 23, + 23, 23, 23, 23}; + +static inline int +rta_fifo_load(struct program *program, uint32_t src, + uint64_t loc, uint32_t length, uint32_t flags) +{ + uint32_t opcode = 0; + uint32_t ext_length = 0, val = 0; + int ret = -EINVAL; + bool is_seq_cmd = false; + unsigned int start_pc = program->current_pc; + + /* write command type field */ + if (flags & SEQ) { + opcode = CMD_SEQ_FIFO_LOAD; + is_seq_cmd = true; + } else { + opcode = CMD_FIFO_LOAD; + } + + /* Parameters checking */ + if (is_seq_cmd) { + if ((flags & IMMED) || (flags & SGF)) { + pr_err("SEQ FIFO LOAD: Invalid command\n"); + goto err; + } + if ((rta_sec_era <= RTA_SEC_ERA_5) && (flags & AIDF)) { + pr_err("SEQ FIFO LOAD: Flag(s) not supported by SEC Era %d\n", + USER_SEC_ERA(rta_sec_era)); + goto err; + } + if ((flags & VLF) && ((flags & EXT) || (length >> 16))) { + pr_err("SEQ FIFO LOAD: Invalid usage of VLF\n"); + goto err; + } + } else { + if (src == SKIP) { + pr_err("FIFO LOAD: Invalid src\n"); + goto err; + } + if ((flags & AIDF) || (flags & VLF)) { + pr_err("FIFO LOAD: Invalid command\n"); + goto err; + } + if ((flags & IMMED) && (flags & SGF)) { + pr_err("FIFO LOAD: Invalid usage of SGF and IMM\n"); + goto err; + } + if ((flags & IMMED) && ((flags & EXT) || (length >> 16))) { + pr_err("FIFO LOAD: Invalid usage of EXT and IMM\n"); + goto err; + } + } + + /* write input data type field */ + ret = __rta_map_opcode(src, fifo_load_table, + fifo_load_table_sz[rta_sec_era], &val); + if (ret < 0) { + pr_err("FIFO LOAD: Source value is not supported. SEC Program Line: %d\n", + program->current_pc); + goto err; + } + opcode |= val; + + if (flags & CLASS1) + opcode |= FIFOLD_CLASS_CLASS1; + if (flags & CLASS2) + opcode |= FIFOLD_CLASS_CLASS2; + if (flags & BOTH) + opcode |= FIFOLD_CLASS_BOTH; + + /* write fields: SGF|VLF, IMM, [LC1, LC2, F1] */ + if (flags & FLUSH1) + opcode |= FIFOLD_TYPE_FLUSH1; + if (flags & LAST1) + opcode |= FIFOLD_TYPE_LAST1; + if (flags & LAST2) + opcode |= FIFOLD_TYPE_LAST2; + if (!is_seq_cmd) { + if (flags & SGF) + opcode |= FIFOLDST_SGF; + if (flags & IMMED) + opcode |= FIFOLD_IMM; + } else { + if (flags & VLF) + opcode |= FIFOLDST_VLF; + if (flags & AIDF) + opcode |= FIFOLD_AIDF; + } + + /* + * Verify if extended length is required. In case of BITDATA, calculate + * number of full bytes and additional valid bits. + */ + if ((flags & EXT) || (length >> 16)) { + opcode |= FIFOLDST_EXT; + if (src == BIT_DATA) { + ext_length = (length / 8); + length = (length % 8); + } else { + ext_length = length; + length = 0; + } + } + opcode |= (uint16_t) length; + + __rta_out32(program, opcode); + program->current_instruction++; + + /* write pointer or immediate data field */ + if (flags & IMMED) + __rta_inline_data(program, loc, flags & __COPY_MASK, length); + else if (!is_seq_cmd) + __rta_out64(program, program->ps, loc); + + /* write extended length field */ + if (opcode & FIFOLDST_EXT) + __rta_out32(program, ext_length); + + return (int)start_pc; + + err: + program->first_error_pc = start_pc; + program->current_instruction++; + return ret; +} + +static const uint32_t fifo_store_table[][2] = { +/*1*/ { PKA0, FIFOST_TYPE_PKHA_A0 }, + { PKA1, FIFOST_TYPE_PKHA_A1 }, + { PKA2, FIFOST_TYPE_PKHA_A2 }, + { PKA3, FIFOST_TYPE_PKHA_A3 }, + { PKB0, FIFOST_TYPE_PKHA_B0 }, + { PKB1, FIFOST_TYPE_PKHA_B1 }, + { PKB2, FIFOST_TYPE_PKHA_B2 }, + { PKB3, FIFOST_TYPE_PKHA_B3 }, + { PKA, FIFOST_TYPE_PKHA_A }, + { PKB, FIFOST_TYPE_PKHA_B }, + { PKN, FIFOST_TYPE_PKHA_N }, + { PKE, FIFOST_TYPE_PKHA_E_JKEK }, + { RNG, FIFOST_TYPE_RNGSTORE }, + { RNGOFIFO, FIFOST_TYPE_RNGFIFO }, + { AFHA_SBOX, FIFOST_TYPE_AF_SBOX_JKEK }, + { MDHA_SPLIT_KEY, FIFOST_CLASS_CLASS2KEY | FIFOST_TYPE_SPLIT_KEK }, + { MSG, FIFOST_TYPE_MESSAGE_DATA }, + { KEY1, FIFOST_CLASS_CLASS1KEY | FIFOST_TYPE_KEY_KEK }, + { KEY2, FIFOST_CLASS_CLASS2KEY | FIFOST_TYPE_KEY_KEK }, + { OFIFO, FIFOST_TYPE_OUTFIFO_KEK}, + { SKIP, FIFOST_TYPE_SKIP }, +/*22*/ { METADATA, FIFOST_TYPE_METADATA}, + { MSG_CKSUM, FIFOST_TYPE_MESSAGE_DATA2 } +}; + +/* + * Allowed FIFO_STORE output data types for each SEC Era. + * Values represent the number of entries from fifo_store_table[] that are + * supported. + */ +static const unsigned int fifo_store_table_sz[] = {21, 21, 21, 21, + 22, 22, 22, 23}; + +static inline int +rta_fifo_store(struct program *program, uint32_t src, + uint32_t encrypt_flags, uint64_t dst, + uint32_t length, uint32_t flags) +{ + uint32_t opcode = 0; + uint32_t val = 0; + int ret = -EINVAL; + bool is_seq_cmd = false; + unsigned int start_pc = program->current_pc; + + /* write command type field */ + if (flags & SEQ) { + opcode = CMD_SEQ_FIFO_STORE; + is_seq_cmd = true; + } else { + opcode = CMD_FIFO_STORE; + } + + /* Parameter checking */ + if (is_seq_cmd) { + if ((flags & VLF) && ((length >> 16) || (flags & EXT))) { + pr_err("SEQ FIFO STORE: Invalid usage of VLF\n"); + goto err; + } + if (dst) { + pr_err("SEQ FIFO STORE: Invalid command\n"); + goto err; + } + if ((src == METADATA) && (flags & (CONT | EXT))) { + pr_err("SEQ FIFO STORE: Invalid flags\n"); + goto err; + } + } else { + if (((src == RNGOFIFO) && ((dst) || (flags & EXT))) || + (src == METADATA)) { + pr_err("FIFO STORE: Invalid destination\n"); + goto err; + } + } + if ((rta_sec_era == RTA_SEC_ERA_7) && (src == AFHA_SBOX)) { + pr_err("FIFO STORE: AFHA S-box not supported by SEC Era %d\n", + USER_SEC_ERA(rta_sec_era)); + goto err; + } + + /* write output data type field */ + ret = __rta_map_opcode(src, fifo_store_table, + fifo_store_table_sz[rta_sec_era], &val); + if (ret < 0) { + pr_err("FIFO STORE: Source type not supported. SEC Program Line: %d\n", + program->current_pc); + goto err; + } + opcode |= val; + + if (encrypt_flags & TK) + opcode |= (0x1 << FIFOST_TYPE_SHIFT); + if (encrypt_flags & EKT) { + if (rta_sec_era == RTA_SEC_ERA_1) { + pr_err("FIFO STORE: AES-CCM source types not supported\n"); + ret = -EINVAL; + goto err; + } + opcode |= (0x10 << FIFOST_TYPE_SHIFT); + opcode &= (uint32_t)~(0x20 << FIFOST_TYPE_SHIFT); + } + + /* write flags fields */ + if (flags & CONT) + opcode |= FIFOST_CONT; + if ((flags & VLF) && (is_seq_cmd)) + opcode |= FIFOLDST_VLF; + if ((flags & SGF) && (!is_seq_cmd)) + opcode |= FIFOLDST_SGF; + if (flags & CLASS1) + opcode |= FIFOST_CLASS_CLASS1KEY; + if (flags & CLASS2) + opcode |= FIFOST_CLASS_CLASS2KEY; + if (flags & BOTH) + opcode |= FIFOST_CLASS_BOTH; + + /* Verify if extended length is required */ + if ((length >> 16) || (flags & EXT)) + opcode |= FIFOLDST_EXT; + else + opcode |= (uint16_t) length; + + __rta_out32(program, opcode); + program->current_instruction++; + + /* write pointer field */ + if ((!is_seq_cmd) && (dst)) + __rta_out64(program, program->ps, dst); + + /* write extended length field */ + if (opcode & FIFOLDST_EXT) + __rta_out32(program, length); + + return (int)start_pc; + + err: + program->first_error_pc = start_pc; + program->current_instruction++; + return ret; +} + +#endif /* __RTA_FIFO_LOAD_STORE_CMD_H__ */ diff --git a/dpdk/drivers/crypto/dpaa2_sec/hw/rta/header_cmd.h b/dpdk/drivers/crypto/dpaa2_sec/hw/rta/header_cmd.h new file mode 100644 index 00000000..96c9f1e0 --- /dev/null +++ b/dpdk/drivers/crypto/dpaa2_sec/hw/rta/header_cmd.h @@ -0,0 +1,251 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2008-2016 Freescale Semiconductor Inc. + * Copyright 2016 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __RTA_HEADER_CMD_H__ +#define __RTA_HEADER_CMD_H__ + +extern enum rta_sec_era rta_sec_era; + +/* Allowed job header flags for each SEC Era. */ +static const uint32_t job_header_flags[] = { + DNR | TD | MTD | SHR | REO, + DNR | TD | MTD | SHR | REO | RSMS, + DNR | TD | MTD | SHR | REO | RSMS, + DNR | TD | MTD | SHR | REO | RSMS, + DNR | TD | MTD | SHR | REO | RSMS | EXT, + DNR | TD | MTD | SHR | REO | RSMS | EXT, + DNR | TD | MTD | SHR | REO | RSMS | EXT, + DNR | TD | MTD | SHR | REO | EXT +}; + +/* Allowed shared header flags for each SEC Era. */ +static const uint32_t shr_header_flags[] = { + DNR | SC | PD, + DNR | SC | PD | CIF, + DNR | SC | PD | CIF, + DNR | SC | PD | CIF | RIF, + DNR | SC | PD | CIF | RIF, + DNR | SC | PD | CIF | RIF, + DNR | SC | PD | CIF | RIF, + DNR | SC | PD | CIF | RIF +}; + +static inline int +rta_shr_header(struct program *program, + enum rta_share_type share, + unsigned int start_idx, + uint32_t flags) +{ + uint32_t opcode = CMD_SHARED_DESC_HDR; + unsigned int start_pc = program->current_pc; + + if (flags & ~shr_header_flags[rta_sec_era]) { + pr_err("SHR_DESC: Flag(s) not supported by SEC Era %d\n", + USER_SEC_ERA(rta_sec_era)); + goto err; + } + + switch (share) { + case SHR_ALWAYS: + opcode |= HDR_SHARE_ALWAYS; + break; + case SHR_SERIAL: + opcode |= HDR_SHARE_SERIAL; + break; + case SHR_NEVER: + /* + * opcode |= HDR_SHARE_NEVER; + * HDR_SHARE_NEVER is 0 + */ + break; + case SHR_WAIT: + opcode |= HDR_SHARE_WAIT; + break; + default: + pr_err("SHR_DESC: SHARE VALUE is not supported. SEC Program Line: %d\n", + program->current_pc); + goto err; + } + + opcode |= HDR_ONE; + opcode |= (start_idx << HDR_START_IDX_SHIFT) & HDR_START_IDX_MASK; + + if (flags & DNR) + opcode |= HDR_DNR; + if (flags & CIF) + opcode |= HDR_CLEAR_IFIFO; + if (flags & SC) + opcode |= HDR_SAVECTX; + if (flags & PD) + opcode |= HDR_PROP_DNR; + if (flags & RIF) + opcode |= HDR_RIF; + + __rta_out32(program, opcode); + program->current_instruction++; + + if (program->current_instruction == 1) + program->shrhdr = program->buffer; + + return (int)start_pc; + + err: + program->first_error_pc = start_pc; + program->current_instruction++; + return -EINVAL; +} + +static inline int +rta_job_header(struct program *program, + enum rta_share_type share, + unsigned int start_idx, + uint64_t shr_desc, uint32_t flags, + uint32_t ext_flags) +{ + uint32_t opcode = CMD_DESC_HDR; + uint32_t hdr_ext = 0; + unsigned int start_pc = program->current_pc; + + if (flags & ~job_header_flags[rta_sec_era]) { + pr_err("JOB_DESC: Flag(s) not supported by SEC Era %d\n", + USER_SEC_ERA(rta_sec_era)); + goto err; + } + + switch (share) { + case SHR_ALWAYS: + opcode |= HDR_SHARE_ALWAYS; + break; + case SHR_SERIAL: + opcode |= HDR_SHARE_SERIAL; + break; + case SHR_NEVER: + /* + * opcode |= HDR_SHARE_NEVER; + * HDR_SHARE_NEVER is 0 + */ + break; + case SHR_WAIT: + opcode |= HDR_SHARE_WAIT; + break; + case SHR_DEFER: + opcode |= HDR_SHARE_DEFER; + break; + default: + pr_err("JOB_DESC: SHARE VALUE is not supported. SEC Program Line: %d\n", + program->current_pc); + goto err; + } + + if ((flags & TD) && (flags & REO)) { + pr_err("JOB_DESC: REO flag not supported for trusted descriptors. SEC Program Line: %d\n", + program->current_pc); + goto err; + } + + if ((rta_sec_era < RTA_SEC_ERA_7) && (flags & MTD) && !(flags & TD)) { + pr_err("JOB_DESC: Trying to MTD a descriptor that is not a TD. SEC Program Line: %d\n", + program->current_pc); + goto err; + } + + if ((flags & EXT) && !(flags & SHR) && (start_idx < 2)) { + pr_err("JOB_DESC: Start index must be >= 2 in case of no SHR and EXT. SEC Program Line: %d\n", + program->current_pc); + goto err; + } + + opcode |= HDR_ONE; + opcode |= ((start_idx << HDR_START_IDX_SHIFT) & HDR_START_IDX_MASK); + + if (flags & EXT) { + opcode |= HDR_EXT; + + if (ext_flags & DSV) { + hdr_ext |= HDR_EXT_DSEL_VALID; + hdr_ext |= ext_flags & DSEL_MASK; + } + + if (ext_flags & FTD) { + if (rta_sec_era <= RTA_SEC_ERA_5) { + pr_err("JOB_DESC: Fake trusted descriptor not supported by SEC Era %d\n", + USER_SEC_ERA(rta_sec_era)); + goto err; + } + + hdr_ext |= HDR_EXT_FTD; + } + } + if (flags & RSMS) + opcode |= HDR_RSLS; + if (flags & DNR) + opcode |= HDR_DNR; + if (flags & TD) + opcode |= HDR_TRUSTED; + if (flags & MTD) + opcode |= HDR_MAKE_TRUSTED; + if (flags & REO) + opcode |= HDR_REVERSE; + if (flags & SHR) + opcode |= HDR_SHARED; + + __rta_out32(program, opcode); + program->current_instruction++; + + if (program->current_instruction == 1) { + program->jobhdr = program->buffer; + + if (opcode & HDR_SHARED) + __rta_out64(program, program->ps, shr_desc); + } + + if (flags & EXT) + __rta_out32(program, hdr_ext); + + /* Note: descriptor length is set in program_finalize routine */ + return (int)start_pc; + + err: + program->first_error_pc = start_pc; + program->current_instruction++; + return -EINVAL; +} + +#endif /* __RTA_HEADER_CMD_H__ */ diff --git a/dpdk/drivers/crypto/dpaa2_sec/hw/rta/jump_cmd.h b/dpdk/drivers/crypto/dpaa2_sec/hw/rta/jump_cmd.h new file mode 100644 index 00000000..cba65073 --- /dev/null +++ b/dpdk/drivers/crypto/dpaa2_sec/hw/rta/jump_cmd.h @@ -0,0 +1,207 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2008-2016 Freescale Semiconductor Inc. + * Copyright 2016 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __RTA_JUMP_CMD_H__ +#define __RTA_JUMP_CMD_H__ + +extern enum rta_sec_era rta_sec_era; + +static const uint32_t jump_test_cond[][2] = { + { NIFP, JUMP_COND_NIFP }, + { NIP, JUMP_COND_NIP }, + { NOP, JUMP_COND_NOP }, + { NCP, JUMP_COND_NCP }, + { CALM, JUMP_COND_CALM }, + { SELF, JUMP_COND_SELF }, + { SHRD, JUMP_COND_SHRD }, + { JQP, JUMP_COND_JQP }, + { MATH_Z, JUMP_COND_MATH_Z }, + { MATH_N, JUMP_COND_MATH_N }, + { MATH_NV, JUMP_COND_MATH_NV }, + { MATH_C, JUMP_COND_MATH_C }, + { PK_0, JUMP_COND_PK_0 }, + { PK_GCD_1, JUMP_COND_PK_GCD_1 }, + { PK_PRIME, JUMP_COND_PK_PRIME }, + { CLASS1, JUMP_CLASS_CLASS1 }, + { CLASS2, JUMP_CLASS_CLASS2 }, + { BOTH, JUMP_CLASS_BOTH } +}; + +static const uint32_t jump_test_math_cond[][2] = { + { MATH_Z, JUMP_COND_MATH_Z }, + { MATH_N, JUMP_COND_MATH_N }, + { MATH_NV, JUMP_COND_MATH_NV }, + { MATH_C, JUMP_COND_MATH_C } +}; + +static const uint32_t jump_src_dst[][2] = { + { MATH0, JUMP_SRC_DST_MATH0 }, + { MATH1, JUMP_SRC_DST_MATH1 }, + { MATH2, JUMP_SRC_DST_MATH2 }, + { MATH3, JUMP_SRC_DST_MATH3 }, + { DPOVRD, JUMP_SRC_DST_DPOVRD }, + { SEQINSZ, JUMP_SRC_DST_SEQINLEN }, + { SEQOUTSZ, JUMP_SRC_DST_SEQOUTLEN }, + { VSEQINSZ, JUMP_SRC_DST_VARSEQINLEN }, + { VSEQOUTSZ, JUMP_SRC_DST_VARSEQOUTLEN } +}; + +static inline int +rta_jump(struct program *program, uint64_t address, + enum rta_jump_type jump_type, + enum rta_jump_cond test_type, + uint32_t test_condition, uint32_t src_dst) +{ + uint32_t opcode = CMD_JUMP; + unsigned int start_pc = program->current_pc; + int ret = -EINVAL; + + if (((jump_type == GOSUB) || (jump_type == RETURN)) && + (rta_sec_era < RTA_SEC_ERA_4)) { + pr_err("JUMP: Jump type not supported by SEC Era %d\n", + USER_SEC_ERA(rta_sec_era)); + goto err; + } + + if (((jump_type == LOCAL_JUMP_INC) || (jump_type == LOCAL_JUMP_DEC)) && + (rta_sec_era <= RTA_SEC_ERA_5)) { + pr_err("JUMP_INCDEC: Jump type not supported by SEC Era %d\n", + USER_SEC_ERA(rta_sec_era)); + goto err; + } + + switch (jump_type) { + case (LOCAL_JUMP): + /* + * opcode |= JUMP_TYPE_LOCAL; + * JUMP_TYPE_LOCAL is 0 + */ + break; + case (HALT): + opcode |= JUMP_TYPE_HALT; + break; + case (HALT_STATUS): + opcode |= JUMP_TYPE_HALT_USER; + break; + case (FAR_JUMP): + opcode |= JUMP_TYPE_NONLOCAL; + break; + case (GOSUB): + opcode |= JUMP_TYPE_GOSUB; + break; + case (RETURN): + opcode |= JUMP_TYPE_RETURN; + break; + case (LOCAL_JUMP_INC): + opcode |= JUMP_TYPE_LOCAL_INC; + break; + case (LOCAL_JUMP_DEC): + opcode |= JUMP_TYPE_LOCAL_DEC; + break; + default: + pr_err("JUMP: Invalid jump type. SEC Program Line: %d\n", + program->current_pc); + goto err; + } + + switch (test_type) { + case (ALL_TRUE): + /* + * opcode |= JUMP_TEST_ALL; + * JUMP_TEST_ALL is 0 + */ + break; + case (ALL_FALSE): + opcode |= JUMP_TEST_INVALL; + break; + case (ANY_TRUE): + opcode |= JUMP_TEST_ANY; + break; + case (ANY_FALSE): + opcode |= JUMP_TEST_INVANY; + break; + default: + pr_err("JUMP: test type not supported. SEC Program Line: %d\n", + program->current_pc); + goto err; + } + + /* write test condition field */ + if ((jump_type != LOCAL_JUMP_INC) && (jump_type != LOCAL_JUMP_DEC)) { + __rta_map_flags(test_condition, jump_test_cond, + ARRAY_SIZE(jump_test_cond), &opcode); + } else { + uint32_t val = 0; + + ret = __rta_map_opcode(src_dst, jump_src_dst, + ARRAY_SIZE(jump_src_dst), &val); + if (ret < 0) { + pr_err("JUMP_INCDEC: SRC_DST not supported. SEC PC: %d; Instr: %d\n", + program->current_pc, + program->current_instruction); + goto err; + } + opcode |= val; + + __rta_map_flags(test_condition, jump_test_math_cond, + ARRAY_SIZE(jump_test_math_cond), &opcode); + } + + /* write local offset field for local jumps and user-defined halt */ + if ((jump_type == LOCAL_JUMP) || (jump_type == LOCAL_JUMP_INC) || + (jump_type == LOCAL_JUMP_DEC) || (jump_type == GOSUB) || + (jump_type == HALT_STATUS)) + opcode |= (uint32_t)(address & JUMP_OFFSET_MASK); + + __rta_out32(program, opcode); + program->current_instruction++; + + if (jump_type == FAR_JUMP) + __rta_out64(program, program->ps, address); + + return (int)start_pc; + + err: + program->first_error_pc = start_pc; + program->current_instruction++; + return ret; +} + +#endif /* __RTA_JUMP_CMD_H__ */ diff --git a/dpdk/drivers/crypto/dpaa2_sec/hw/rta/key_cmd.h b/dpdk/drivers/crypto/dpaa2_sec/hw/rta/key_cmd.h new file mode 100644 index 00000000..8e097856 --- /dev/null +++ b/dpdk/drivers/crypto/dpaa2_sec/hw/rta/key_cmd.h @@ -0,0 +1,222 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2008-2016 Freescale Semiconductor Inc. + * Copyright 2016 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __RTA_KEY_CMD_H__ +#define __RTA_KEY_CMD_H__ + +extern enum rta_sec_era rta_sec_era; + +/* Allowed encryption flags for each SEC Era */ +static const uint32_t key_enc_flags[] = { + ENC, + ENC | NWB | EKT | TK, + ENC | NWB | EKT | TK, + ENC | NWB | EKT | TK, + ENC | NWB | EKT | TK, + ENC | NWB | EKT | TK, + ENC | NWB | EKT | TK | PTS, + ENC | NWB | EKT | TK | PTS +}; + +static inline int +rta_key(struct program *program, uint32_t key_dst, + uint32_t encrypt_flags, uint64_t src, uint32_t length, + uint32_t flags) +{ + uint32_t opcode = 0; + bool is_seq_cmd = false; + unsigned int start_pc = program->current_pc; + + if (encrypt_flags & ~key_enc_flags[rta_sec_era]) { + pr_err("KEY: Flag(s) not supported by SEC Era %d\n", + USER_SEC_ERA(rta_sec_era)); + goto err; + } + + /* write cmd type */ + if (flags & SEQ) { + opcode = CMD_SEQ_KEY; + is_seq_cmd = true; + } else { + opcode = CMD_KEY; + } + + /* check parameters */ + if (is_seq_cmd) { + if ((flags & IMMED) || (flags & SGF)) { + pr_err("SEQKEY: Invalid flag. SEC PC: %d; Instr: %d\n", + program->current_pc, + program->current_instruction); + goto err; + } + if ((rta_sec_era <= RTA_SEC_ERA_5) && + ((flags & VLF) || (flags & AIDF))) { + pr_err("SEQKEY: Flag(s) not supported by SEC Era %d\n", + USER_SEC_ERA(rta_sec_era)); + goto err; + } + } else { + if ((flags & AIDF) || (flags & VLF)) { + pr_err("KEY: Invalid flag. SEC PC: %d; Instr: %d\n", + program->current_pc, + program->current_instruction); + goto err; + } + if ((flags & SGF) && (flags & IMMED)) { + pr_err("KEY: Invalid flag. SEC PC: %d; Instr: %d\n", + program->current_pc, + program->current_instruction); + goto err; + } + } + + if ((encrypt_flags & PTS) && + ((encrypt_flags & ENC) || (encrypt_flags & NWB) || + (key_dst == PKE))) { + pr_err("KEY: Invalid flag / destination. SEC PC: %d; Instr: %d\n", + program->current_pc, program->current_instruction); + goto err; + } + + if (key_dst == AFHA_SBOX) { + if (rta_sec_era == RTA_SEC_ERA_7) { + pr_err("KEY: AFHA S-box not supported by SEC Era %d\n", + USER_SEC_ERA(rta_sec_era)); + goto err; + } + + if (flags & IMMED) { + pr_err("KEY: Invalid flag. SEC PC: %d; Instr: %d\n", + program->current_pc, + program->current_instruction); + goto err; + } + + /* + * Sbox data loaded into the ARC-4 processor must be exactly + * 258 bytes long, or else a data sequence error is generated. + */ + if (length != 258) { + pr_err("KEY: Invalid length. SEC PC: %d; Instr: %d\n", + program->current_pc, + program->current_instruction); + goto err; + } + } + + /* write key destination and class fields */ + switch (key_dst) { + case (KEY1): + opcode |= KEY_DEST_CLASS1; + break; + case (KEY2): + opcode |= KEY_DEST_CLASS2; + break; + case (PKE): + opcode |= KEY_DEST_CLASS1 | KEY_DEST_PKHA_E; + break; + case (AFHA_SBOX): + opcode |= KEY_DEST_CLASS1 | KEY_DEST_AFHA_SBOX; + break; + case (MDHA_SPLIT_KEY): + opcode |= KEY_DEST_CLASS2 | KEY_DEST_MDHA_SPLIT; + break; + default: + pr_err("KEY: Invalid destination. SEC PC: %d; Instr: %d\n", + program->current_pc, program->current_instruction); + goto err; + } + + /* write key length */ + length &= KEY_LENGTH_MASK; + opcode |= length; + + /* write key command specific flags */ + if (encrypt_flags & ENC) { + /* Encrypted (black) keys must be padded to 8 bytes (CCM) or + * 16 bytes (ECB) depending on EKT bit. AES-CCM encrypted keys + * (EKT = 1) have 6-byte nonce and 6-byte MAC after padding. + */ + opcode |= KEY_ENC; + if (encrypt_flags & EKT) { + opcode |= KEY_EKT; + length = ALIGN(length, 8); + length += 12; + } else { + length = ALIGN(length, 16); + } + if (encrypt_flags & TK) + opcode |= KEY_TK; + } + if (encrypt_flags & NWB) + opcode |= KEY_NWB; + if (encrypt_flags & PTS) + opcode |= KEY_PTS; + + /* write general command flags */ + if (!is_seq_cmd) { + if (flags & IMMED) + opcode |= KEY_IMM; + if (flags & SGF) + opcode |= KEY_SGF; + } else { + if (flags & AIDF) + opcode |= KEY_AIDF; + if (flags & VLF) + opcode |= KEY_VLF; + } + + __rta_out32(program, opcode); + program->current_instruction++; + + if (flags & IMMED) + __rta_inline_data(program, src, flags & __COPY_MASK, length); + else + __rta_out64(program, program->ps, src); + + return (int)start_pc; + + err: + program->first_error_pc = start_pc; + program->current_instruction++; + return -EINVAL; +} + +#endif /* __RTA_KEY_CMD_H__ */ diff --git a/dpdk/drivers/crypto/dpaa2_sec/hw/rta/load_cmd.h b/dpdk/drivers/crypto/dpaa2_sec/hw/rta/load_cmd.h new file mode 100644 index 00000000..2e786a20 --- /dev/null +++ b/dpdk/drivers/crypto/dpaa2_sec/hw/rta/load_cmd.h @@ -0,0 +1,335 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2008-2016 Freescale Semiconductor Inc. + * Copyright 2016 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __RTA_LOAD_CMD_H__ +#define __RTA_LOAD_CMD_H__ + +extern enum rta_sec_era rta_sec_era; + +/* Allowed length and offset masks for each SEC Era in case DST = DCTRL */ +static const uint32_t load_len_mask_allowed[] = { + 0x000000ee, + 0x000000fe, + 0x000000fe, + 0x000000fe, + 0x000000fe, + 0x000000fe, + 0x000000fe, + 0x000000fe +}; + +static const uint32_t load_off_mask_allowed[] = { + 0x0000000f, + 0x000000ff, + 0x000000ff, + 0x000000ff, + 0x000000ff, + 0x000000ff, + 0x000000ff, + 0x000000ff +}; + +#define IMM_MUST 0 +#define IMM_CAN 1 +#define IMM_NO 2 +#define IMM_DSNM 3 /* it doesn't matter the src type */ + +enum e_lenoff { + LENOF_03, + LENOF_4, + LENOF_48, + LENOF_448, + LENOF_18, + LENOF_32, + LENOF_24, + LENOF_16, + LENOF_8, + LENOF_128, + LENOF_256, + DSNM /* it doesn't matter the length/offset values */ +}; + +struct load_map { + uint32_t dst; + uint32_t dst_opcode; + enum e_lenoff len_off; + uint8_t imm_src; + +}; + +static const struct load_map load_dst[] = { +/*1*/ { KEY1SZ, LDST_CLASS_1_CCB | LDST_SRCDST_WORD_KEYSZ_REG, + LENOF_4, IMM_MUST }, + { KEY2SZ, LDST_CLASS_2_CCB | LDST_SRCDST_WORD_KEYSZ_REG, + LENOF_4, IMM_MUST }, + { DATA1SZ, LDST_CLASS_1_CCB | LDST_SRCDST_WORD_DATASZ_REG, + LENOF_448, IMM_MUST }, + { DATA2SZ, LDST_CLASS_2_CCB | LDST_SRCDST_WORD_DATASZ_REG, + LENOF_448, IMM_MUST }, + { ICV1SZ, LDST_CLASS_1_CCB | LDST_SRCDST_WORD_ICVSZ_REG, + LENOF_4, IMM_MUST }, + { ICV2SZ, LDST_CLASS_2_CCB | LDST_SRCDST_WORD_ICVSZ_REG, + LENOF_4, IMM_MUST }, + { CCTRL, LDST_CLASS_IND_CCB | LDST_SRCDST_WORD_CHACTRL, + LENOF_4, IMM_MUST }, + { DCTRL, LDST_CLASS_DECO | LDST_IMM | LDST_SRCDST_WORD_DECOCTRL, + DSNM, IMM_DSNM }, + { ICTRL, LDST_CLASS_IND_CCB | LDST_SRCDST_WORD_IRQCTRL, + LENOF_4, IMM_MUST }, + { DPOVRD, LDST_CLASS_DECO | LDST_SRCDST_WORD_DECO_PCLOVRD, + LENOF_4, IMM_MUST }, + { CLRW, LDST_CLASS_IND_CCB | LDST_SRCDST_WORD_CLRW, + LENOF_4, IMM_MUST }, + { AAD1SZ, LDST_CLASS_1_CCB | LDST_SRCDST_WORD_DECO_AAD_SZ, + LENOF_4, IMM_MUST }, + { IV1SZ, LDST_CLASS_1_CCB | LDST_SRCDST_WORD_CLASS1_IV_SZ, + LENOF_4, IMM_MUST }, + { ALTDS1, LDST_CLASS_1_CCB | LDST_SRCDST_WORD_ALTDS_CLASS1, + LENOF_448, IMM_MUST }, + { PKASZ, LDST_CLASS_1_CCB | LDST_SRCDST_WORD_PKHA_A_SZ, + LENOF_4, IMM_MUST, }, + { PKBSZ, LDST_CLASS_1_CCB | LDST_SRCDST_WORD_PKHA_B_SZ, + LENOF_4, IMM_MUST }, + { PKNSZ, LDST_CLASS_1_CCB | LDST_SRCDST_WORD_PKHA_N_SZ, + LENOF_4, IMM_MUST }, + { PKESZ, LDST_CLASS_1_CCB | LDST_SRCDST_WORD_PKHA_E_SZ, + LENOF_4, IMM_MUST }, + { NFIFO, LDST_CLASS_IND_CCB | LDST_SRCDST_WORD_INFO_FIFO, + LENOF_48, IMM_MUST }, + { IFIFO, LDST_SRCDST_BYTE_INFIFO, LENOF_18, IMM_MUST }, + { OFIFO, LDST_SRCDST_BYTE_OUTFIFO, LENOF_18, IMM_MUST }, + { MATH0, LDST_CLASS_DECO | LDST_SRCDST_WORD_DECO_MATH0, + LENOF_32, IMM_CAN }, + { MATH1, LDST_CLASS_DECO | LDST_SRCDST_WORD_DECO_MATH1, + LENOF_24, IMM_CAN }, + { MATH2, LDST_CLASS_DECO | LDST_SRCDST_WORD_DECO_MATH2, + LENOF_16, IMM_CAN }, + { MATH3, LDST_CLASS_DECO | LDST_SRCDST_WORD_DECO_MATH3, + LENOF_8, IMM_CAN }, + { CONTEXT1, LDST_CLASS_1_CCB | LDST_SRCDST_BYTE_CONTEXT, + LENOF_128, IMM_CAN }, + { CONTEXT2, LDST_CLASS_2_CCB | LDST_SRCDST_BYTE_CONTEXT, + LENOF_128, IMM_CAN }, + { KEY1, LDST_CLASS_1_CCB | LDST_SRCDST_BYTE_KEY, + LENOF_32, IMM_CAN }, + { KEY2, LDST_CLASS_2_CCB | LDST_SRCDST_BYTE_KEY, + LENOF_32, IMM_CAN }, + { DESCBUF, LDST_CLASS_DECO | LDST_SRCDST_WORD_DESCBUF, + LENOF_256, IMM_NO }, + { DPID, LDST_CLASS_DECO | LDST_SRCDST_WORD_PID, + LENOF_448, IMM_MUST }, +/*32*/ { IDFNS, LDST_SRCDST_WORD_IFNSR, LENOF_18, IMM_MUST }, + { ODFNS, LDST_SRCDST_WORD_OFNSR, LENOF_18, IMM_MUST }, + { ALTSOURCE, LDST_SRCDST_BYTE_ALTSOURCE, LENOF_18, IMM_MUST }, +/*35*/ { NFIFO_SZL, LDST_SRCDST_WORD_INFO_FIFO_SZL, LENOF_48, IMM_MUST }, + { NFIFO_SZM, LDST_SRCDST_WORD_INFO_FIFO_SZM, LENOF_03, IMM_MUST }, + { NFIFO_L, LDST_SRCDST_WORD_INFO_FIFO_L, LENOF_48, IMM_MUST }, + { NFIFO_M, LDST_SRCDST_WORD_INFO_FIFO_M, LENOF_03, IMM_MUST }, + { SZL, LDST_SRCDST_WORD_SZL, LENOF_48, IMM_MUST }, +/*40*/ { SZM, LDST_SRCDST_WORD_SZM, LENOF_03, IMM_MUST } +}; + +/* + * Allowed LOAD destinations for each SEC Era. + * Values represent the number of entries from load_dst[] that are supported. + */ +static const unsigned int load_dst_sz[] = { 31, 34, 34, 40, 40, 40, 40, 40 }; + +static inline int +load_check_len_offset(int pos, uint32_t length, uint32_t offset) +{ + if ((load_dst[pos].dst == DCTRL) && + ((length & ~load_len_mask_allowed[rta_sec_era]) || + (offset & ~load_off_mask_allowed[rta_sec_era]))) + goto err; + + switch (load_dst[pos].len_off) { + case (LENOF_03): + if ((length > 3) || (offset)) + goto err; + break; + case (LENOF_4): + if ((length != 4) || (offset != 0)) + goto err; + break; + case (LENOF_48): + if (!(((length == 4) && (offset == 0)) || + ((length == 8) && (offset == 0)))) + goto err; + break; + case (LENOF_448): + if (!(((length == 4) && (offset == 0)) || + ((length == 4) && (offset == 4)) || + ((length == 8) && (offset == 0)))) + goto err; + break; + case (LENOF_18): + if ((length < 1) || (length > 8) || (offset != 0)) + goto err; + break; + case (LENOF_32): + if ((length > 32) || (offset > 32) || ((offset + length) > 32)) + goto err; + break; + case (LENOF_24): + if ((length > 24) || (offset > 24) || ((offset + length) > 24)) + goto err; + break; + case (LENOF_16): + if ((length > 16) || (offset > 16) || ((offset + length) > 16)) + goto err; + break; + case (LENOF_8): + if ((length > 8) || (offset > 8) || ((offset + length) > 8)) + goto err; + break; + case (LENOF_128): + if ((length > 128) || (offset > 128) || + ((offset + length) > 128)) + goto err; + break; + case (LENOF_256): + if ((length < 1) || (length > 256) || ((length + offset) > 256)) + goto err; + break; + case (DSNM): + break; + default: + goto err; + } + + return 0; +err: + return -EINVAL; +} + +static inline int +rta_load(struct program *program, uint64_t src, uint64_t dst, + uint32_t offset, uint32_t length, uint32_t flags) +{ + uint32_t opcode = 0; + int pos = -1, ret = -EINVAL; + unsigned int start_pc = program->current_pc, i; + + if (flags & SEQ) + opcode = CMD_SEQ_LOAD; + else + opcode = CMD_LOAD; + + if ((length & 0xffffff00) || (offset & 0xffffff00)) { + pr_err("LOAD: Bad length/offset passed. Should be 8 bits\n"); + goto err; + } + + if (flags & SGF) + opcode |= LDST_SGF; + if (flags & VLF) + opcode |= LDST_VLF; + + /* check load destination, length and offset and source type */ + for (i = 0; i < load_dst_sz[rta_sec_era]; i++) + if (dst == load_dst[i].dst) { + pos = (int)i; + break; + } + if (-1 == pos) { + pr_err("LOAD: Invalid dst. SEC Program Line: %d\n", + program->current_pc); + goto err; + } + + if (flags & IMMED) { + if (load_dst[pos].imm_src == IMM_NO) { + pr_err("LOAD: Invalid source type. SEC Program Line: %d\n", + program->current_pc); + goto err; + } + opcode |= LDST_IMM; + } else if (load_dst[pos].imm_src == IMM_MUST) { + pr_err("LOAD IMM: Invalid source type. SEC Program Line: %d\n", + program->current_pc); + goto err; + } + + ret = load_check_len_offset(pos, length, offset); + if (ret < 0) { + pr_err("LOAD: Invalid length/offset. SEC Program Line: %d\n", + program->current_pc); + goto err; + } + + opcode |= load_dst[pos].dst_opcode; + + /* DESC BUFFER: length / offset values are specified in 4-byte words */ + if (dst == DESCBUF) { + opcode |= (length >> 2); + opcode |= ((offset >> 2) << LDST_OFFSET_SHIFT); + } else { + opcode |= length; + opcode |= (offset << LDST_OFFSET_SHIFT); + } + + __rta_out32(program, opcode); + program->current_instruction++; + + /* DECO CONTROL: skip writing pointer of imm data */ + if (dst == DCTRL) + return (int)start_pc; + + /* + * For data copy, 3 possible ways to specify how to copy data: + * - IMMED & !COPY: copy data directly from src( max 8 bytes) + * - IMMED & COPY: copy data imm from the location specified by user + * - !IMMED and is not SEQ cmd: copy the address + */ + if (flags & IMMED) + __rta_inline_data(program, src, flags & __COPY_MASK, length); + else if (!(flags & SEQ)) + __rta_out64(program, program->ps, src); + + return (int)start_pc; + + err: + program->first_error_pc = start_pc; + program->current_instruction++; + return ret; +} + +#endif /* __RTA_LOAD_CMD_H__*/ diff --git a/dpdk/drivers/crypto/dpaa2_sec/hw/rta/math_cmd.h b/dpdk/drivers/crypto/dpaa2_sec/hw/rta/math_cmd.h new file mode 100644 index 00000000..64a84b47 --- /dev/null +++ b/dpdk/drivers/crypto/dpaa2_sec/hw/rta/math_cmd.h @@ -0,0 +1,402 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2008-2016 Freescale Semiconductor Inc. + * Copyright 2016 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __RTA_MATH_CMD_H__ +#define __RTA_MATH_CMD_H__ + +extern enum rta_sec_era rta_sec_era; + +static const uint32_t math_op1[][2] = { +/*1*/ { MATH0, MATH_SRC0_REG0 }, + { MATH1, MATH_SRC0_REG1 }, + { MATH2, MATH_SRC0_REG2 }, + { MATH3, MATH_SRC0_REG3 }, + { SEQINSZ, MATH_SRC0_SEQINLEN }, + { SEQOUTSZ, MATH_SRC0_SEQOUTLEN }, + { VSEQINSZ, MATH_SRC0_VARSEQINLEN }, + { VSEQOUTSZ, MATH_SRC0_VARSEQOUTLEN }, + { ZERO, MATH_SRC0_ZERO }, +/*10*/ { NONE, 0 }, /* dummy value */ + { DPOVRD, MATH_SRC0_DPOVRD }, + { ONE, MATH_SRC0_ONE } +}; + +/* + * Allowed MATH op1 sources for each SEC Era. + * Values represent the number of entries from math_op1[] that are supported. + */ +static const unsigned int math_op1_sz[] = {10, 10, 12, 12, 12, 12, 12, 12}; + +static const uint32_t math_op2[][2] = { +/*1*/ { MATH0, MATH_SRC1_REG0 }, + { MATH1, MATH_SRC1_REG1 }, + { MATH2, MATH_SRC1_REG2 }, + { MATH3, MATH_SRC1_REG3 }, + { ABD, MATH_SRC1_INFIFO }, + { OFIFO, MATH_SRC1_OUTFIFO }, + { ONE, MATH_SRC1_ONE }, +/*8*/ { NONE, 0 }, /* dummy value */ + { JOBSRC, MATH_SRC1_JOBSOURCE }, + { DPOVRD, MATH_SRC1_DPOVRD }, + { VSEQINSZ, MATH_SRC1_VARSEQINLEN }, + { VSEQOUTSZ, MATH_SRC1_VARSEQOUTLEN }, +/*13*/ { ZERO, MATH_SRC1_ZERO } +}; + +/* + * Allowed MATH op2 sources for each SEC Era. + * Values represent the number of entries from math_op2[] that are supported. + */ +static const unsigned int math_op2_sz[] = {8, 9, 13, 13, 13, 13, 13, 13}; + +static const uint32_t math_result[][2] = { +/*1*/ { MATH0, MATH_DEST_REG0 }, + { MATH1, MATH_DEST_REG1 }, + { MATH2, MATH_DEST_REG2 }, + { MATH3, MATH_DEST_REG3 }, + { SEQINSZ, MATH_DEST_SEQINLEN }, + { SEQOUTSZ, MATH_DEST_SEQOUTLEN }, + { VSEQINSZ, MATH_DEST_VARSEQINLEN }, + { VSEQOUTSZ, MATH_DEST_VARSEQOUTLEN }, +/*9*/ { NONE, MATH_DEST_NONE }, + { DPOVRD, MATH_DEST_DPOVRD } +}; + +/* + * Allowed MATH result destinations for each SEC Era. + * Values represent the number of entries from math_result[] that are + * supported. + */ +static const unsigned int math_result_sz[] = {9, 9, 10, 10, 10, 10, 10, 10}; + +static inline int +rta_math(struct program *program, uint64_t operand1, + uint32_t op, uint64_t operand2, uint32_t result, + int length, uint32_t options) +{ + uint32_t opcode = CMD_MATH; + uint32_t val = 0; + int ret = -EINVAL; + unsigned int start_pc = program->current_pc; + + if (((op == MATH_FUN_BSWAP) && (rta_sec_era < RTA_SEC_ERA_4)) || + ((op == MATH_FUN_ZBYT) && (rta_sec_era < RTA_SEC_ERA_2))) { + pr_err("MATH: operation not supported by SEC Era %d. SEC PC: %d; Instr: %d\n", + USER_SEC_ERA(rta_sec_era), program->current_pc, + program->current_instruction); + goto err; + } + + if (options & SWP) { + if (rta_sec_era < RTA_SEC_ERA_7) { + pr_err("MATH: operation not supported by SEC Era %d. SEC PC: %d; Instr: %d\n", + USER_SEC_ERA(rta_sec_era), program->current_pc, + program->current_instruction); + goto err; + } + + if ((options & IFB) || + (!(options & IMMED) && !(options & IMMED2)) || + ((options & IMMED) && (options & IMMED2))) { + pr_err("MATH: SWP - invalid configuration. SEC PC: %d; Instr: %d\n", + program->current_pc, + program->current_instruction); + goto err; + } + } + + /* + * SHLD operation is different from others and we + * assume that we can have _NONE as first operand + * or _SEQINSZ as second operand + */ + if ((op != MATH_FUN_SHLD) && ((operand1 == NONE) || + (operand2 == SEQINSZ))) { + pr_err("MATH: Invalid operand. SEC PC: %d; Instr: %d\n", + program->current_pc, program->current_instruction); + goto err; + } + + /* + * We first check if it is unary operation. In that + * case second operand must be _NONE + */ + if (((op == MATH_FUN_ZBYT) || (op == MATH_FUN_BSWAP)) && + (operand2 != NONE)) { + pr_err("MATH: Invalid operand2. SEC PC: %d; Instr: %d\n", + program->current_pc, program->current_instruction); + goto err; + } + + /* Write first operand field */ + if (options & IMMED) { + opcode |= MATH_SRC0_IMM; + } else { + ret = __rta_map_opcode((uint32_t)operand1, math_op1, + math_op1_sz[rta_sec_era], &val); + if (ret < 0) { + pr_err("MATH: operand1 not supported. SEC PC: %d; Instr: %d\n", + program->current_pc, + program->current_instruction); + goto err; + } + opcode |= val; + } + + /* Write second operand field */ + if (options & IMMED2) { + opcode |= MATH_SRC1_IMM; + } else { + ret = __rta_map_opcode((uint32_t)operand2, math_op2, + math_op2_sz[rta_sec_era], &val); + if (ret < 0) { + pr_err("MATH: operand2 not supported. SEC PC: %d; Instr: %d\n", + program->current_pc, + program->current_instruction); + goto err; + } + opcode |= val; + } + + /* Write result field */ + ret = __rta_map_opcode(result, math_result, math_result_sz[rta_sec_era], + &val); + if (ret < 0) { + pr_err("MATH: result not supported. SEC PC: %d; Instr: %d\n", + program->current_pc, program->current_instruction); + goto err; + } + opcode |= val; + + /* + * as we encode operations with their "real" values, we do not + * to translate but we do need to validate the value + */ + switch (op) { + /*Binary operators */ + case (MATH_FUN_ADD): + case (MATH_FUN_ADDC): + case (MATH_FUN_SUB): + case (MATH_FUN_SUBB): + case (MATH_FUN_OR): + case (MATH_FUN_AND): + case (MATH_FUN_XOR): + case (MATH_FUN_LSHIFT): + case (MATH_FUN_RSHIFT): + case (MATH_FUN_SHLD): + /* Unary operators */ + case (MATH_FUN_ZBYT): + case (MATH_FUN_BSWAP): + opcode |= op; + break; + default: + pr_err("MATH: operator is not supported. SEC PC: %d; Instr: %d\n", + program->current_pc, program->current_instruction); + ret = -EINVAL; + goto err; + } + + opcode |= (options & ~(IMMED | IMMED2)); + + /* Verify length */ + switch (length) { + case (1): + opcode |= MATH_LEN_1BYTE; + break; + case (2): + opcode |= MATH_LEN_2BYTE; + break; + case (4): + opcode |= MATH_LEN_4BYTE; + break; + case (8): + opcode |= MATH_LEN_8BYTE; + break; + default: + pr_err("MATH: length is not supported. SEC PC: %d; Instr: %d\n", + program->current_pc, program->current_instruction); + ret = -EINVAL; + goto err; + } + + __rta_out32(program, opcode); + program->current_instruction++; + + /* Write immediate value */ + if ((options & IMMED) && !(options & IMMED2)) { + __rta_out64(program, (length > 4) && !(options & IFB), + operand1); + } else if ((options & IMMED2) && !(options & IMMED)) { + __rta_out64(program, (length > 4) && !(options & IFB), + operand2); + } else if ((options & IMMED) && (options & IMMED2)) { + __rta_out32(program, lower_32_bits(operand1)); + __rta_out32(program, lower_32_bits(operand2)); + } + + return (int)start_pc; + + err: + program->first_error_pc = start_pc; + program->current_instruction++; + return ret; +} + +static inline int +rta_mathi(struct program *program, uint64_t operand, + uint32_t op, uint8_t imm, uint32_t result, + int length, uint32_t options) +{ + uint32_t opcode = CMD_MATHI; + uint32_t val = 0; + int ret = -EINVAL; + unsigned int start_pc = program->current_pc; + + if (rta_sec_era < RTA_SEC_ERA_6) { + pr_err("MATHI: Command not supported by SEC Era %d. SEC PC: %d; Instr: %d\n", + USER_SEC_ERA(rta_sec_era), program->current_pc, + program->current_instruction); + goto err; + } + + if (((op == MATH_FUN_FBYT) && (options & SSEL))) { + pr_err("MATHI: Illegal combination - FBYT and SSEL. SEC PC: %d; Instr: %d\n", + program->current_pc, program->current_instruction); + goto err; + } + + if ((options & SWP) && (rta_sec_era < RTA_SEC_ERA_7)) { + pr_err("MATHI: SWP not supported by SEC Era %d. SEC PC: %d; Instr: %d\n", + USER_SEC_ERA(rta_sec_era), program->current_pc, + program->current_instruction); + goto err; + } + + /* Write first operand field */ + if (!(options & SSEL)) + ret = __rta_map_opcode((uint32_t)operand, math_op1, + math_op1_sz[rta_sec_era], &val); + else + ret = __rta_map_opcode((uint32_t)operand, math_op2, + math_op2_sz[rta_sec_era], &val); + if (ret < 0) { + pr_err("MATHI: operand not supported. SEC PC: %d; Instr: %d\n", + program->current_pc, program->current_instruction); + goto err; + } + + if (!(options & SSEL)) + opcode |= val; + else + opcode |= (val << (MATHI_SRC1_SHIFT - MATH_SRC1_SHIFT)); + + /* Write second operand field */ + opcode |= (imm << MATHI_IMM_SHIFT); + + /* Write result field */ + ret = __rta_map_opcode(result, math_result, math_result_sz[rta_sec_era], + &val); + if (ret < 0) { + pr_err("MATHI: result not supported. SEC PC: %d; Instr: %d\n", + program->current_pc, program->current_instruction); + goto err; + } + opcode |= (val << (MATHI_DEST_SHIFT - MATH_DEST_SHIFT)); + + /* + * as we encode operations with their "real" values, we do not have to + * translate but we do need to validate the value + */ + switch (op) { + case (MATH_FUN_ADD): + case (MATH_FUN_ADDC): + case (MATH_FUN_SUB): + case (MATH_FUN_SUBB): + case (MATH_FUN_OR): + case (MATH_FUN_AND): + case (MATH_FUN_XOR): + case (MATH_FUN_LSHIFT): + case (MATH_FUN_RSHIFT): + case (MATH_FUN_FBYT): + opcode |= op; + break; + default: + pr_err("MATHI: operator not supported. SEC PC: %d; Instr: %d\n", + program->current_pc, program->current_instruction); + ret = -EINVAL; + goto err; + } + + opcode |= options; + + /* Verify length */ + switch (length) { + case (1): + opcode |= MATH_LEN_1BYTE; + break; + case (2): + opcode |= MATH_LEN_2BYTE; + break; + case (4): + opcode |= MATH_LEN_4BYTE; + break; + case (8): + opcode |= MATH_LEN_8BYTE; + break; + default: + pr_err("MATHI: length %d not supported. SEC PC: %d; Instr: %d\n", + length, program->current_pc, + program->current_instruction); + ret = -EINVAL; + goto err; + } + + __rta_out32(program, opcode); + program->current_instruction++; + + return (int)start_pc; + + err: + program->first_error_pc = start_pc; + program->current_instruction++; + return ret; +} + +#endif /* __RTA_MATH_CMD_H__ */ diff --git a/dpdk/drivers/crypto/dpaa2_sec/hw/rta/move_cmd.h b/dpdk/drivers/crypto/dpaa2_sec/hw/rta/move_cmd.h new file mode 100644 index 00000000..3b9ba792 --- /dev/null +++ b/dpdk/drivers/crypto/dpaa2_sec/hw/rta/move_cmd.h @@ -0,0 +1,445 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2008-2016 Freescale Semiconductor Inc. + * Copyright 2016 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __RTA_MOVE_CMD_H__ +#define __RTA_MOVE_CMD_H__ + +#define MOVE_SET_AUX_SRC 0x01 +#define MOVE_SET_AUX_DST 0x02 +#define MOVE_SET_AUX_LS 0x03 +#define MOVE_SET_LEN_16b 0x04 + +#define MOVE_SET_AUX_MATH 0x10 +#define MOVE_SET_AUX_MATH_SRC (MOVE_SET_AUX_SRC | MOVE_SET_AUX_MATH) +#define MOVE_SET_AUX_MATH_DST (MOVE_SET_AUX_DST | MOVE_SET_AUX_MATH) + +#define MASK_16b 0xFF + +/* MOVE command type */ +#define __MOVE 1 +#define __MOVEB 2 +#define __MOVEDW 3 + +extern enum rta_sec_era rta_sec_era; + +static const uint32_t move_src_table[][2] = { +/*1*/ { CONTEXT1, MOVE_SRC_CLASS1CTX }, + { CONTEXT2, MOVE_SRC_CLASS2CTX }, + { OFIFO, MOVE_SRC_OUTFIFO }, + { DESCBUF, MOVE_SRC_DESCBUF }, + { MATH0, MOVE_SRC_MATH0 }, + { MATH1, MOVE_SRC_MATH1 }, + { MATH2, MOVE_SRC_MATH2 }, + { MATH3, MOVE_SRC_MATH3 }, +/*9*/ { IFIFOABD, MOVE_SRC_INFIFO }, + { IFIFOAB1, MOVE_SRC_INFIFO_CL | MOVE_AUX_LS }, + { IFIFOAB2, MOVE_SRC_INFIFO_CL }, +/*12*/ { ABD, MOVE_SRC_INFIFO_NO_NFIFO }, + { AB1, MOVE_SRC_INFIFO_NO_NFIFO | MOVE_AUX_LS }, + { AB2, MOVE_SRC_INFIFO_NO_NFIFO | MOVE_AUX_MS } +}; + +/* Allowed MOVE / MOVE_LEN sources for each SEC Era. + * Values represent the number of entries from move_src_table[] that are + * supported. + */ +static const unsigned int move_src_table_sz[] = {9, 11, 14, 14, 14, 14, 14, 14}; + +static const uint32_t move_dst_table[][2] = { +/*1*/ { CONTEXT1, MOVE_DEST_CLASS1CTX }, + { CONTEXT2, MOVE_DEST_CLASS2CTX }, + { OFIFO, MOVE_DEST_OUTFIFO }, + { DESCBUF, MOVE_DEST_DESCBUF }, + { MATH0, MOVE_DEST_MATH0 }, + { MATH1, MOVE_DEST_MATH1 }, + { MATH2, MOVE_DEST_MATH2 }, + { MATH3, MOVE_DEST_MATH3 }, + { IFIFOAB1, MOVE_DEST_CLASS1INFIFO }, + { IFIFOAB2, MOVE_DEST_CLASS2INFIFO }, + { PKA, MOVE_DEST_PK_A }, + { KEY1, MOVE_DEST_CLASS1KEY }, + { KEY2, MOVE_DEST_CLASS2KEY }, +/*14*/ { IFIFO, MOVE_DEST_INFIFO }, +/*15*/ { ALTSOURCE, MOVE_DEST_ALTSOURCE} +}; + +/* Allowed MOVE / MOVE_LEN destinations for each SEC Era. + * Values represent the number of entries from move_dst_table[] that are + * supported. + */ +static const +unsigned int move_dst_table_sz[] = {13, 14, 14, 15, 15, 15, 15, 15}; + +static inline int +set_move_offset(struct program *program __maybe_unused, + uint64_t src, uint16_t src_offset, + uint64_t dst, uint16_t dst_offset, + uint16_t *offset, uint16_t *opt); + +static inline int +math_offset(uint16_t offset); + +static inline int +rta_move(struct program *program, int cmd_type, uint64_t src, + uint16_t src_offset, uint64_t dst, + uint16_t dst_offset, uint32_t length, uint32_t flags) +{ + uint32_t opcode = 0; + uint16_t offset = 0, opt = 0; + uint32_t val = 0; + int ret = -EINVAL; + bool is_move_len_cmd = false; + unsigned int start_pc = program->current_pc; + + if ((rta_sec_era < RTA_SEC_ERA_7) && (cmd_type != __MOVE)) { + pr_err("MOVE: MOVEB / MOVEDW not supported by SEC Era %d. SEC PC: %d; Instr: %d\n", + USER_SEC_ERA(rta_sec_era), program->current_pc, + program->current_instruction); + goto err; + } + + /* write command type */ + if (cmd_type == __MOVEB) { + opcode = CMD_MOVEB; + } else if (cmd_type == __MOVEDW) { + opcode = CMD_MOVEDW; + } else if (!(flags & IMMED)) { + if (rta_sec_era < RTA_SEC_ERA_3) { + pr_err("MOVE: MOVE_LEN not supported by SEC Era %d. SEC PC: %d; Instr: %d\n", + USER_SEC_ERA(rta_sec_era), program->current_pc, + program->current_instruction); + goto err; + } + + if ((length != MATH0) && (length != MATH1) && + (length != MATH2) && (length != MATH3)) { + pr_err("MOVE: MOVE_LEN length must be MATH[0-3]. SEC PC: %d; Instr: %d\n", + program->current_pc, + program->current_instruction); + goto err; + } + + opcode = CMD_MOVE_LEN; + is_move_len_cmd = true; + } else { + opcode = CMD_MOVE; + } + + /* write offset first, to check for invalid combinations or incorrect + * offset values sooner; decide which offset should be here + * (src or dst) + */ + ret = set_move_offset(program, src, src_offset, dst, dst_offset, + &offset, &opt); + if (ret < 0) + goto err; + + opcode |= (offset << MOVE_OFFSET_SHIFT) & MOVE_OFFSET_MASK; + + /* set AUX field if required */ + if (opt == MOVE_SET_AUX_SRC) { + opcode |= ((src_offset / 16) << MOVE_AUX_SHIFT) & MOVE_AUX_MASK; + } else if (opt == MOVE_SET_AUX_DST) { + opcode |= ((dst_offset / 16) << MOVE_AUX_SHIFT) & MOVE_AUX_MASK; + } else if (opt == MOVE_SET_AUX_LS) { + opcode |= MOVE_AUX_LS; + } else if (opt & MOVE_SET_AUX_MATH) { + if (opt & MOVE_SET_AUX_SRC) + offset = src_offset; + else + offset = dst_offset; + + if (rta_sec_era < RTA_SEC_ERA_6) { + if (offset) + pr_debug("MOVE: Offset not supported by SEC Era %d. SEC PC: %d; Instr: %d\n", + USER_SEC_ERA(rta_sec_era), + program->current_pc, + program->current_instruction); + /* nothing to do for offset = 0 */ + } else { + ret = math_offset(offset); + if (ret < 0) { + pr_err("MOVE: Invalid offset in MATH register. SEC PC: %d; Instr: %d\n", + program->current_pc, + program->current_instruction); + goto err; + } + + opcode |= (uint32_t)ret; + } + } + + /* write source field */ + ret = __rta_map_opcode((uint32_t)src, move_src_table, + move_src_table_sz[rta_sec_era], &val); + if (ret < 0) { + pr_err("MOVE: Invalid SRC. SEC PC: %d; Instr: %d\n", + program->current_pc, program->current_instruction); + goto err; + } + opcode |= val; + + /* write destination field */ + ret = __rta_map_opcode((uint32_t)dst, move_dst_table, + move_dst_table_sz[rta_sec_era], &val); + if (ret < 0) { + pr_err("MOVE: Invalid DST. SEC PC: %d; Instr: %d\n", + program->current_pc, program->current_instruction); + goto err; + } + opcode |= val; + + /* write flags */ + if (flags & (FLUSH1 | FLUSH2)) + opcode |= MOVE_AUX_MS; + if (flags & (LAST2 | LAST1)) + opcode |= MOVE_AUX_LS; + if (flags & WAITCOMP) + opcode |= MOVE_WAITCOMP; + + if (!is_move_len_cmd) { + /* write length */ + if (opt == MOVE_SET_LEN_16b) + opcode |= (length & (MOVE_OFFSET_MASK | MOVE_LEN_MASK)); + else + opcode |= (length & MOVE_LEN_MASK); + } else { + /* write mrsel */ + switch (length) { + case (MATH0): + /* + * opcode |= MOVELEN_MRSEL_MATH0; + * MOVELEN_MRSEL_MATH0 is 0 + */ + break; + case (MATH1): + opcode |= MOVELEN_MRSEL_MATH1; + break; + case (MATH2): + opcode |= MOVELEN_MRSEL_MATH2; + break; + case (MATH3): + opcode |= MOVELEN_MRSEL_MATH3; + break; + } + + /* write size */ + if (rta_sec_era >= RTA_SEC_ERA_7) { + if (flags & SIZE_WORD) + opcode |= MOVELEN_SIZE_WORD; + else if (flags & SIZE_BYTE) + opcode |= MOVELEN_SIZE_BYTE; + else if (flags & SIZE_DWORD) + opcode |= MOVELEN_SIZE_DWORD; + } + } + + __rta_out32(program, opcode); + program->current_instruction++; + + return (int)start_pc; + + err: + program->first_error_pc = start_pc; + program->current_instruction++; + return ret; +} + +static inline int +set_move_offset(struct program *program __maybe_unused, + uint64_t src, uint16_t src_offset, + uint64_t dst, uint16_t dst_offset, + uint16_t *offset, uint16_t *opt) +{ + switch (src) { + case (CONTEXT1): + case (CONTEXT2): + if (dst == DESCBUF) { + *opt = MOVE_SET_AUX_SRC; + *offset = dst_offset; + } else if ((dst == KEY1) || (dst == KEY2)) { + if ((src_offset) && (dst_offset)) { + pr_err("MOVE: Bad offset. SEC PC: %d; Instr: %d\n", + program->current_pc, + program->current_instruction); + goto err; + } + if (dst_offset) { + *opt = MOVE_SET_AUX_LS; + *offset = dst_offset; + } else { + *offset = src_offset; + } + } else { + if ((dst == MATH0) || (dst == MATH1) || + (dst == MATH2) || (dst == MATH3)) { + *opt = MOVE_SET_AUX_MATH_DST; + } else if (((dst == OFIFO) || (dst == ALTSOURCE)) && + (src_offset % 4)) { + pr_err("MOVE: Bad offset alignment. SEC PC: %d; Instr: %d\n", + program->current_pc, + program->current_instruction); + goto err; + } + + *offset = src_offset; + } + break; + + case (OFIFO): + if (dst == OFIFO) { + pr_err("MOVE: Invalid DST. SEC PC: %d; Instr: %d\n", + program->current_pc, + program->current_instruction); + goto err; + } + if (((dst == IFIFOAB1) || (dst == IFIFOAB2) || + (dst == IFIFO) || (dst == PKA)) && + (src_offset || dst_offset)) { + pr_err("MOVE: Offset should be zero. SEC PC: %d; Instr: %d\n", + program->current_pc, + program->current_instruction); + goto err; + } + *offset = dst_offset; + break; + + case (DESCBUF): + if ((dst == CONTEXT1) || (dst == CONTEXT2)) { + *opt = MOVE_SET_AUX_DST; + } else if ((dst == MATH0) || (dst == MATH1) || + (dst == MATH2) || (dst == MATH3)) { + *opt = MOVE_SET_AUX_MATH_DST; + } else if (dst == DESCBUF) { + pr_err("MOVE: Invalid DST. SEC PC: %d; Instr: %d\n", + program->current_pc, + program->current_instruction); + goto err; + } else if (((dst == OFIFO) || (dst == ALTSOURCE)) && + (src_offset % 4)) { + pr_err("MOVE: Invalid offset alignment. SEC PC: %d; Instr %d\n", + program->current_pc, + program->current_instruction); + goto err; + } + + *offset = src_offset; + break; + + case (MATH0): + case (MATH1): + case (MATH2): + case (MATH3): + if ((dst == OFIFO) || (dst == ALTSOURCE)) { + if (src_offset % 4) { + pr_err("MOVE: Bad offset alignment. SEC PC: %d; Instr: %d\n", + program->current_pc, + program->current_instruction); + goto err; + } + *offset = src_offset; + } else if ((dst == IFIFOAB1) || (dst == IFIFOAB2) || + (dst == IFIFO) || (dst == PKA)) { + *offset = src_offset; + } else { + *offset = dst_offset; + + /* + * This condition is basically the negation of: + * dst in { CONTEXT[1-2], MATH[0-3] } + */ + if ((dst != KEY1) && (dst != KEY2)) + *opt = MOVE_SET_AUX_MATH_SRC; + } + break; + + case (IFIFOABD): + case (IFIFOAB1): + case (IFIFOAB2): + case (ABD): + case (AB1): + case (AB2): + if ((dst == IFIFOAB1) || (dst == IFIFOAB2) || + (dst == IFIFO) || (dst == PKA) || (dst == ALTSOURCE)) { + pr_err("MOVE: Bad DST. SEC PC: %d; Instr: %d\n", + program->current_pc, + program->current_instruction); + goto err; + } else { + if (dst == OFIFO) { + *opt = MOVE_SET_LEN_16b; + } else { + if (dst_offset % 4) { + pr_err("MOVE: Bad offset alignment. SEC PC: %d; Instr: %d\n", + program->current_pc, + program->current_instruction); + goto err; + } + *offset = dst_offset; + } + } + break; + default: + break; + } + + return 0; + err: + return -EINVAL; +} + +static inline int +math_offset(uint16_t offset) +{ + switch (offset) { + case 0: + return 0; + case 4: + return MOVE_AUX_LS; + case 6: + return MOVE_AUX_MS; + case 7: + return MOVE_AUX_LS | MOVE_AUX_MS; + } + + return -EINVAL; +} + +#endif /* __RTA_MOVE_CMD_H__ */ diff --git a/dpdk/drivers/crypto/dpaa2_sec/hw/rta/nfifo_cmd.h b/dpdk/drivers/crypto/dpaa2_sec/hw/rta/nfifo_cmd.h new file mode 100644 index 00000000..f1ead2da --- /dev/null +++ b/dpdk/drivers/crypto/dpaa2_sec/hw/rta/nfifo_cmd.h @@ -0,0 +1,196 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2008-2016 Freescale Semiconductor Inc. + * Copyright 2016 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __RTA_NFIFO_CMD_H__ +#define __RTA_NFIFO_CMD_H__ + +extern enum rta_sec_era rta_sec_era; + +static const uint32_t nfifo_src[][2] = { +/*1*/ { IFIFO, NFIFOENTRY_STYPE_DFIFO }, + { OFIFO, NFIFOENTRY_STYPE_OFIFO }, + { PAD, NFIFOENTRY_STYPE_PAD }, +/*4*/ { MSGOUTSNOOP, NFIFOENTRY_STYPE_SNOOP | NFIFOENTRY_DEST_BOTH }, +/*5*/ { ALTSOURCE, NFIFOENTRY_STYPE_ALTSOURCE }, + { OFIFO_SYNC, NFIFOENTRY_STYPE_OFIFO_SYNC }, +/*7*/ { MSGOUTSNOOP_ALT, NFIFOENTRY_STYPE_SNOOP_ALT | NFIFOENTRY_DEST_BOTH } +}; + +/* + * Allowed NFIFO LOAD sources for each SEC Era. + * Values represent the number of entries from nfifo_src[] that are supported. + */ +static const unsigned int nfifo_src_sz[] = {4, 5, 5, 5, 5, 5, 5, 7}; + +static const uint32_t nfifo_data[][2] = { + { MSG, NFIFOENTRY_DTYPE_MSG }, + { MSG1, NFIFOENTRY_DEST_CLASS1 | NFIFOENTRY_DTYPE_MSG }, + { MSG2, NFIFOENTRY_DEST_CLASS2 | NFIFOENTRY_DTYPE_MSG }, + { IV1, NFIFOENTRY_DEST_CLASS1 | NFIFOENTRY_DTYPE_IV }, + { IV2, NFIFOENTRY_DEST_CLASS2 | NFIFOENTRY_DTYPE_IV }, + { ICV1, NFIFOENTRY_DEST_CLASS1 | NFIFOENTRY_DTYPE_ICV }, + { ICV2, NFIFOENTRY_DEST_CLASS2 | NFIFOENTRY_DTYPE_ICV }, + { SAD1, NFIFOENTRY_DEST_CLASS1 | NFIFOENTRY_DTYPE_SAD }, + { AAD1, NFIFOENTRY_DEST_CLASS1 | NFIFOENTRY_DTYPE_AAD }, + { AAD2, NFIFOENTRY_DEST_CLASS2 | NFIFOENTRY_DTYPE_AAD }, + { AFHA_SBOX, NFIFOENTRY_DEST_CLASS1 | NFIFOENTRY_DTYPE_SBOX }, + { SKIP, NFIFOENTRY_DTYPE_SKIP }, + { PKE, NFIFOENTRY_DEST_CLASS1 | NFIFOENTRY_DTYPE_PK_E }, + { PKN, NFIFOENTRY_DEST_CLASS1 | NFIFOENTRY_DTYPE_PK_N }, + { PKA, NFIFOENTRY_DEST_CLASS1 | NFIFOENTRY_DTYPE_PK_A }, + { PKA0, NFIFOENTRY_DEST_CLASS1 | NFIFOENTRY_DTYPE_PK_A0 }, + { PKA1, NFIFOENTRY_DEST_CLASS1 | NFIFOENTRY_DTYPE_PK_A1 }, + { PKA2, NFIFOENTRY_DEST_CLASS1 | NFIFOENTRY_DTYPE_PK_A2 }, + { PKA3, NFIFOENTRY_DEST_CLASS1 | NFIFOENTRY_DTYPE_PK_A3 }, + { PKB, NFIFOENTRY_DEST_CLASS1 | NFIFOENTRY_DTYPE_PK_B }, + { PKB0, NFIFOENTRY_DEST_CLASS1 | NFIFOENTRY_DTYPE_PK_B0 }, + { PKB1, NFIFOENTRY_DEST_CLASS1 | NFIFOENTRY_DTYPE_PK_B1 }, + { PKB2, NFIFOENTRY_DEST_CLASS1 | NFIFOENTRY_DTYPE_PK_B2 }, + { PKB3, NFIFOENTRY_DEST_CLASS1 | NFIFOENTRY_DTYPE_PK_B3 }, + { AB1, NFIFOENTRY_DEST_CLASS1 }, + { AB2, NFIFOENTRY_DEST_CLASS2 }, + { ABD, NFIFOENTRY_DEST_DECO } +}; + +static const uint32_t nfifo_flags[][2] = { +/*1*/ { LAST1, NFIFOENTRY_LC1 }, + { LAST2, NFIFOENTRY_LC2 }, + { FLUSH1, NFIFOENTRY_FC1 }, + { BP, NFIFOENTRY_BND }, + { PAD_ZERO, NFIFOENTRY_PTYPE_ZEROS }, + { PAD_NONZERO, NFIFOENTRY_PTYPE_RND_NOZEROS }, + { PAD_INCREMENT, NFIFOENTRY_PTYPE_INCREMENT }, + { PAD_RANDOM, NFIFOENTRY_PTYPE_RND }, + { PAD_ZERO_N1, NFIFOENTRY_PTYPE_ZEROS_NZ }, + { PAD_NONZERO_0, NFIFOENTRY_PTYPE_RND_NZ_LZ }, + { PAD_N1, NFIFOENTRY_PTYPE_N }, +/*12*/ { PAD_NONZERO_N, NFIFOENTRY_PTYPE_RND_NZ_N }, + { FLUSH2, NFIFOENTRY_FC2 }, + { OC, NFIFOENTRY_OC } +}; + +/* + * Allowed NFIFO LOAD flags for each SEC Era. + * Values represent the number of entries from nfifo_flags[] that are supported. + */ +static const unsigned int nfifo_flags_sz[] = {12, 14, 14, 14, 14, 14, 14, 14}; + +static const uint32_t nfifo_pad_flags[][2] = { + { BM, NFIFOENTRY_BM }, + { PS, NFIFOENTRY_PS }, + { PR, NFIFOENTRY_PR } +}; + +/* + * Allowed NFIFO LOAD pad flags for each SEC Era. + * Values represent the number of entries from nfifo_pad_flags[] that are + * supported. + */ +static const unsigned int nfifo_pad_flags_sz[] = {2, 2, 2, 2, 3, 3, 3, 3}; + +static inline int +rta_nfifo_load(struct program *program, uint32_t src, + uint32_t data, uint32_t length, uint32_t flags) +{ + uint32_t opcode = 0, val; + int ret = -EINVAL; + uint32_t load_cmd = CMD_LOAD | LDST_IMM | LDST_CLASS_IND_CCB | + LDST_SRCDST_WORD_INFO_FIFO; + unsigned int start_pc = program->current_pc; + + if ((data == AFHA_SBOX) && (rta_sec_era == RTA_SEC_ERA_7)) { + pr_err("NFIFO: AFHA S-box not supported by SEC Era %d\n", + USER_SEC_ERA(rta_sec_era)); + goto err; + } + + /* write source field */ + ret = __rta_map_opcode(src, nfifo_src, nfifo_src_sz[rta_sec_era], &val); + if (ret < 0) { + pr_err("NFIFO: Invalid SRC. SEC PC: %d; Instr: %d\n", + program->current_pc, program->current_instruction); + goto err; + } + opcode |= val; + + /* write type field */ + ret = __rta_map_opcode(data, nfifo_data, ARRAY_SIZE(nfifo_data), &val); + if (ret < 0) { + pr_err("NFIFO: Invalid data. SEC PC: %d; Instr: %d\n", + program->current_pc, program->current_instruction); + goto err; + } + opcode |= val; + + /* write DL field */ + if (!(flags & EXT)) { + opcode |= length & NFIFOENTRY_DLEN_MASK; + load_cmd |= 4; + } else { + load_cmd |= 8; + } + + /* write flags */ + __rta_map_flags(flags, nfifo_flags, nfifo_flags_sz[rta_sec_era], + &opcode); + + /* in case of padding, check the destination */ + if (src == PAD) + __rta_map_flags(flags, nfifo_pad_flags, + nfifo_pad_flags_sz[rta_sec_era], &opcode); + + /* write LOAD command first */ + __rta_out32(program, load_cmd); + __rta_out32(program, opcode); + + if (flags & EXT) + __rta_out32(program, length & NFIFOENTRY_DLEN_MASK); + + program->current_instruction++; + + return (int)start_pc; + + err: + program->first_error_pc = start_pc; + program->current_instruction++; + return ret; +} + +#endif /* __RTA_NFIFO_CMD_H__ */ diff --git a/dpdk/drivers/crypto/dpaa2_sec/hw/rta/operation_cmd.h b/dpdk/drivers/crypto/dpaa2_sec/hw/rta/operation_cmd.h new file mode 100644 index 00000000..698ff530 --- /dev/null +++ b/dpdk/drivers/crypto/dpaa2_sec/hw/rta/operation_cmd.h @@ -0,0 +1,599 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2008-2016 Freescale Semiconductor Inc. + * Copyright 2016 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __RTA_OPERATION_CMD_H__ +#define __RTA_OPERATION_CMD_H__ + +extern enum rta_sec_era rta_sec_era; + +static inline int +__rta_alg_aai_aes(uint16_t aai) +{ + uint16_t aes_mode = aai & OP_ALG_AESA_MODE_MASK; + + if (aai & OP_ALG_AAI_C2K) { + if (rta_sec_era < RTA_SEC_ERA_5) + return -1; + if ((aes_mode != OP_ALG_AAI_CCM) && + (aes_mode != OP_ALG_AAI_GCM)) + return -EINVAL; + } + + switch (aes_mode) { + case OP_ALG_AAI_CBC_CMAC: + case OP_ALG_AAI_CTR_CMAC_LTE: + case OP_ALG_AAI_CTR_CMAC: + if (rta_sec_era < RTA_SEC_ERA_2) + return -EINVAL; + /* no break */ + case OP_ALG_AAI_CTR: + case OP_ALG_AAI_CBC: + case OP_ALG_AAI_ECB: + case OP_ALG_AAI_OFB: + case OP_ALG_AAI_CFB: + case OP_ALG_AAI_XTS: + case OP_ALG_AAI_CMAC: + case OP_ALG_AAI_XCBC_MAC: + case OP_ALG_AAI_CCM: + case OP_ALG_AAI_GCM: + case OP_ALG_AAI_CBC_XCBCMAC: + case OP_ALG_AAI_CTR_XCBCMAC: + return 0; + } + + return -EINVAL; +} + +static inline int +__rta_alg_aai_des(uint16_t aai) +{ + uint16_t aai_code = (uint16_t)(aai & ~OP_ALG_AAI_CHECKODD); + + switch (aai_code) { + case OP_ALG_AAI_CBC: + case OP_ALG_AAI_ECB: + case OP_ALG_AAI_CFB: + case OP_ALG_AAI_OFB: + return 0; + } + + return -EINVAL; +} + +static inline int +__rta_alg_aai_md5(uint16_t aai) +{ + switch (aai) { + case OP_ALG_AAI_HMAC: + if (rta_sec_era < RTA_SEC_ERA_2) + return -EINVAL; + /* no break */ + case OP_ALG_AAI_SMAC: + case OP_ALG_AAI_HASH: + case OP_ALG_AAI_HMAC_PRECOMP: + return 0; + } + + return -EINVAL; +} + +static inline int +__rta_alg_aai_sha(uint16_t aai) +{ + switch (aai) { + case OP_ALG_AAI_HMAC: + if (rta_sec_era < RTA_SEC_ERA_2) + return -EINVAL; + /* no break */ + case OP_ALG_AAI_HASH: + case OP_ALG_AAI_HMAC_PRECOMP: + return 0; + } + + return -EINVAL; +} + +static inline int +__rta_alg_aai_rng(uint16_t aai) +{ + uint16_t rng_mode = aai & OP_ALG_RNG_MODE_MASK; + uint16_t rng_sh = aai & OP_ALG_AAI_RNG4_SH_MASK; + + switch (rng_mode) { + case OP_ALG_AAI_RNG: + case OP_ALG_AAI_RNG_NZB: + case OP_ALG_AAI_RNG_OBP: + break; + default: + return -EINVAL; + } + + /* State Handle bits are valid only for SEC Era >= 5 */ + if ((rta_sec_era < RTA_SEC_ERA_5) && rng_sh) + return -EINVAL; + + /* PS, AI, SK bits are also valid only for SEC Era >= 5 */ + if ((rta_sec_era < RTA_SEC_ERA_5) && (aai & + (OP_ALG_AAI_RNG4_PS | OP_ALG_AAI_RNG4_AI | OP_ALG_AAI_RNG4_SK))) + return -EINVAL; + + switch (rng_sh) { + case OP_ALG_AAI_RNG4_SH_0: + case OP_ALG_AAI_RNG4_SH_1: + return 0; + } + + return -EINVAL; +} + +static inline int +__rta_alg_aai_crc(uint16_t aai) +{ + uint16_t aai_code = aai & OP_ALG_CRC_POLY_MASK; + + switch (aai_code) { + case OP_ALG_AAI_802: + case OP_ALG_AAI_3385: + case OP_ALG_AAI_CUST_POLY: + return 0; + } + + return -EINVAL; +} + +static inline int +__rta_alg_aai_kasumi(uint16_t aai) +{ + switch (aai) { + case OP_ALG_AAI_GSM: + case OP_ALG_AAI_EDGE: + case OP_ALG_AAI_F8: + case OP_ALG_AAI_F9: + return 0; + } + + return -EINVAL; +} + +static inline int +__rta_alg_aai_snow_f9(uint16_t aai) +{ + if (aai == OP_ALG_AAI_F9) + return 0; + + return -EINVAL; +} + +static inline int +__rta_alg_aai_snow_f8(uint16_t aai) +{ + if (aai == OP_ALG_AAI_F8) + return 0; + + return -EINVAL; +} + +static inline int +__rta_alg_aai_zuce(uint16_t aai) +{ + if (aai == OP_ALG_AAI_F8) + return 0; + + return -EINVAL; +} + +static inline int +__rta_alg_aai_zuca(uint16_t aai) +{ + if (aai == OP_ALG_AAI_F9) + return 0; + + return -EINVAL; +} + +struct alg_aai_map { + uint32_t chipher_algo; + int (*aai_func)(uint16_t); + uint32_t class; +}; + +static const struct alg_aai_map alg_table[] = { +/*1*/ { OP_ALG_ALGSEL_AES, __rta_alg_aai_aes, OP_TYPE_CLASS1_ALG }, + { OP_ALG_ALGSEL_DES, __rta_alg_aai_des, OP_TYPE_CLASS1_ALG }, + { OP_ALG_ALGSEL_3DES, __rta_alg_aai_des, OP_TYPE_CLASS1_ALG }, + { OP_ALG_ALGSEL_MD5, __rta_alg_aai_md5, OP_TYPE_CLASS2_ALG }, + { OP_ALG_ALGSEL_SHA1, __rta_alg_aai_md5, OP_TYPE_CLASS2_ALG }, + { OP_ALG_ALGSEL_SHA224, __rta_alg_aai_sha, OP_TYPE_CLASS2_ALG }, + { OP_ALG_ALGSEL_SHA256, __rta_alg_aai_sha, OP_TYPE_CLASS2_ALG }, + { OP_ALG_ALGSEL_SHA384, __rta_alg_aai_sha, OP_TYPE_CLASS2_ALG }, + { OP_ALG_ALGSEL_SHA512, __rta_alg_aai_sha, OP_TYPE_CLASS2_ALG }, + { OP_ALG_ALGSEL_RNG, __rta_alg_aai_rng, OP_TYPE_CLASS1_ALG }, +/*11*/ { OP_ALG_ALGSEL_CRC, __rta_alg_aai_crc, OP_TYPE_CLASS2_ALG }, + { OP_ALG_ALGSEL_ARC4, NULL, OP_TYPE_CLASS1_ALG }, + { OP_ALG_ALGSEL_SNOW_F8, __rta_alg_aai_snow_f8, OP_TYPE_CLASS1_ALG }, +/*14*/ { OP_ALG_ALGSEL_KASUMI, __rta_alg_aai_kasumi, OP_TYPE_CLASS1_ALG }, + { OP_ALG_ALGSEL_SNOW_F9, __rta_alg_aai_snow_f9, OP_TYPE_CLASS2_ALG }, + { OP_ALG_ALGSEL_ZUCE, __rta_alg_aai_zuce, OP_TYPE_CLASS1_ALG }, +/*17*/ { OP_ALG_ALGSEL_ZUCA, __rta_alg_aai_zuca, OP_TYPE_CLASS2_ALG } +}; + +/* + * Allowed OPERATION algorithms for each SEC Era. + * Values represent the number of entries from alg_table[] that are supported. + */ +static const unsigned int alg_table_sz[] = {14, 15, 15, 15, 17, 17, 11, 17}; + +static inline int +rta_operation(struct program *program, uint32_t cipher_algo, + uint16_t aai, uint8_t algo_state, + int icv_checking, int enc) +{ + uint32_t opcode = CMD_OPERATION; + unsigned int i, found = 0; + unsigned int start_pc = program->current_pc; + int ret; + + for (i = 0; i < alg_table_sz[rta_sec_era]; i++) { + if (alg_table[i].chipher_algo == cipher_algo) { + opcode |= cipher_algo | alg_table[i].class; + /* nothing else to verify */ + if (alg_table[i].aai_func == NULL) { + found = 1; + break; + } + + aai &= OP_ALG_AAI_MASK; + + ret = (*alg_table[i].aai_func)(aai); + if (ret < 0) { + pr_err("OPERATION: Bad AAI Type. SEC Program Line: %d\n", + program->current_pc); + goto err; + } + opcode |= aai; + found = 1; + break; + } + } + if (!found) { + pr_err("OPERATION: Invalid Command. SEC Program Line: %d\n", + program->current_pc); + ret = -EINVAL; + goto err; + } + + switch (algo_state) { + case OP_ALG_AS_UPDATE: + case OP_ALG_AS_INIT: + case OP_ALG_AS_FINALIZE: + case OP_ALG_AS_INITFINAL: + opcode |= algo_state; + break; + default: + pr_err("Invalid Operation Command\n"); + ret = -EINVAL; + goto err; + } + + switch (icv_checking) { + case ICV_CHECK_DISABLE: + /* + * opcode |= OP_ALG_ICV_OFF; + * OP_ALG_ICV_OFF is 0 + */ + break; + case ICV_CHECK_ENABLE: + opcode |= OP_ALG_ICV_ON; + break; + default: + pr_err("Invalid Operation Command\n"); + ret = -EINVAL; + goto err; + } + + switch (enc) { + case DIR_DEC: + /* + * opcode |= OP_ALG_DECRYPT; + * OP_ALG_DECRYPT is 0 + */ + break; + case DIR_ENC: + opcode |= OP_ALG_ENCRYPT; + break; + default: + pr_err("Invalid Operation Command\n"); + ret = -EINVAL; + goto err; + } + + __rta_out32(program, opcode); + program->current_instruction++; + return (int)start_pc; + + err: + program->first_error_pc = start_pc; + return ret; +} + +/* + * OPERATION PKHA routines + */ +static inline int +__rta_pkha_clearmem(uint32_t pkha_op) +{ + switch (pkha_op) { + case (OP_ALG_PKMODE_CLEARMEM_ALL): + case (OP_ALG_PKMODE_CLEARMEM_ABE): + case (OP_ALG_PKMODE_CLEARMEM_ABN): + case (OP_ALG_PKMODE_CLEARMEM_AB): + case (OP_ALG_PKMODE_CLEARMEM_AEN): + case (OP_ALG_PKMODE_CLEARMEM_AE): + case (OP_ALG_PKMODE_CLEARMEM_AN): + case (OP_ALG_PKMODE_CLEARMEM_A): + case (OP_ALG_PKMODE_CLEARMEM_BEN): + case (OP_ALG_PKMODE_CLEARMEM_BE): + case (OP_ALG_PKMODE_CLEARMEM_BN): + case (OP_ALG_PKMODE_CLEARMEM_B): + case (OP_ALG_PKMODE_CLEARMEM_EN): + case (OP_ALG_PKMODE_CLEARMEM_N): + case (OP_ALG_PKMODE_CLEARMEM_E): + return 0; + } + + return -EINVAL; +} + +static inline int +__rta_pkha_mod_arithmetic(uint32_t pkha_op) +{ + pkha_op &= (uint32_t)~OP_ALG_PKMODE_OUT_A; + + switch (pkha_op) { + case (OP_ALG_PKMODE_MOD_ADD): + case (OP_ALG_PKMODE_MOD_SUB_AB): + case (OP_ALG_PKMODE_MOD_SUB_BA): + case (OP_ALG_PKMODE_MOD_MULT): + case (OP_ALG_PKMODE_MOD_MULT_IM): + case (OP_ALG_PKMODE_MOD_MULT_IM_OM): + case (OP_ALG_PKMODE_MOD_EXPO): + case (OP_ALG_PKMODE_MOD_EXPO_TEQ): + case (OP_ALG_PKMODE_MOD_EXPO_IM): + case (OP_ALG_PKMODE_MOD_EXPO_IM_TEQ): + case (OP_ALG_PKMODE_MOD_REDUCT): + case (OP_ALG_PKMODE_MOD_INV): + case (OP_ALG_PKMODE_MOD_MONT_CNST): + case (OP_ALG_PKMODE_MOD_CRT_CNST): + case (OP_ALG_PKMODE_MOD_GCD): + case (OP_ALG_PKMODE_MOD_PRIMALITY): + case (OP_ALG_PKMODE_MOD_SML_EXP): + case (OP_ALG_PKMODE_F2M_ADD): + case (OP_ALG_PKMODE_F2M_MUL): + case (OP_ALG_PKMODE_F2M_MUL_IM): + case (OP_ALG_PKMODE_F2M_MUL_IM_OM): + case (OP_ALG_PKMODE_F2M_EXP): + case (OP_ALG_PKMODE_F2M_EXP_TEQ): + case (OP_ALG_PKMODE_F2M_AMODN): + case (OP_ALG_PKMODE_F2M_INV): + case (OP_ALG_PKMODE_F2M_R2): + case (OP_ALG_PKMODE_F2M_GCD): + case (OP_ALG_PKMODE_F2M_SML_EXP): + case (OP_ALG_PKMODE_ECC_F2M_ADD): + case (OP_ALG_PKMODE_ECC_F2M_ADD_IM_OM_PROJ): + case (OP_ALG_PKMODE_ECC_F2M_DBL): + case (OP_ALG_PKMODE_ECC_F2M_DBL_IM_OM_PROJ): + case (OP_ALG_PKMODE_ECC_F2M_MUL): + case (OP_ALG_PKMODE_ECC_F2M_MUL_TEQ): + case (OP_ALG_PKMODE_ECC_F2M_MUL_R2): + case (OP_ALG_PKMODE_ECC_F2M_MUL_R2_TEQ): + case (OP_ALG_PKMODE_ECC_F2M_MUL_R2_PROJ): + case (OP_ALG_PKMODE_ECC_F2M_MUL_R2_PROJ_TEQ): + case (OP_ALG_PKMODE_ECC_MOD_ADD): + case (OP_ALG_PKMODE_ECC_MOD_ADD_IM_OM_PROJ): + case (OP_ALG_PKMODE_ECC_MOD_DBL): + case (OP_ALG_PKMODE_ECC_MOD_DBL_IM_OM_PROJ): + case (OP_ALG_PKMODE_ECC_MOD_MUL): + case (OP_ALG_PKMODE_ECC_MOD_MUL_TEQ): + case (OP_ALG_PKMODE_ECC_MOD_MUL_R2): + case (OP_ALG_PKMODE_ECC_MOD_MUL_R2_TEQ): + case (OP_ALG_PKMODE_ECC_MOD_MUL_R2_PROJ): + case (OP_ALG_PKMODE_ECC_MOD_MUL_R2_PROJ_TEQ): + return 0; + } + + return -EINVAL; +} + +static inline int +__rta_pkha_copymem(uint32_t pkha_op) +{ + switch (pkha_op) { + case (OP_ALG_PKMODE_COPY_NSZ_A0_B0): + case (OP_ALG_PKMODE_COPY_NSZ_A0_B1): + case (OP_ALG_PKMODE_COPY_NSZ_A0_B2): + case (OP_ALG_PKMODE_COPY_NSZ_A0_B3): + case (OP_ALG_PKMODE_COPY_NSZ_A1_B0): + case (OP_ALG_PKMODE_COPY_NSZ_A1_B1): + case (OP_ALG_PKMODE_COPY_NSZ_A1_B2): + case (OP_ALG_PKMODE_COPY_NSZ_A1_B3): + case (OP_ALG_PKMODE_COPY_NSZ_A2_B0): + case (OP_ALG_PKMODE_COPY_NSZ_A2_B1): + case (OP_ALG_PKMODE_COPY_NSZ_A2_B2): + case (OP_ALG_PKMODE_COPY_NSZ_A2_B3): + case (OP_ALG_PKMODE_COPY_NSZ_A3_B0): + case (OP_ALG_PKMODE_COPY_NSZ_A3_B1): + case (OP_ALG_PKMODE_COPY_NSZ_A3_B2): + case (OP_ALG_PKMODE_COPY_NSZ_A3_B3): + case (OP_ALG_PKMODE_COPY_NSZ_B0_A0): + case (OP_ALG_PKMODE_COPY_NSZ_B0_A1): + case (OP_ALG_PKMODE_COPY_NSZ_B0_A2): + case (OP_ALG_PKMODE_COPY_NSZ_B0_A3): + case (OP_ALG_PKMODE_COPY_NSZ_B1_A0): + case (OP_ALG_PKMODE_COPY_NSZ_B1_A1): + case (OP_ALG_PKMODE_COPY_NSZ_B1_A2): + case (OP_ALG_PKMODE_COPY_NSZ_B1_A3): + case (OP_ALG_PKMODE_COPY_NSZ_B2_A0): + case (OP_ALG_PKMODE_COPY_NSZ_B2_A1): + case (OP_ALG_PKMODE_COPY_NSZ_B2_A2): + case (OP_ALG_PKMODE_COPY_NSZ_B2_A3): + case (OP_ALG_PKMODE_COPY_NSZ_B3_A0): + case (OP_ALG_PKMODE_COPY_NSZ_B3_A1): + case (OP_ALG_PKMODE_COPY_NSZ_B3_A2): + case (OP_ALG_PKMODE_COPY_NSZ_B3_A3): + case (OP_ALG_PKMODE_COPY_NSZ_A_E): + case (OP_ALG_PKMODE_COPY_NSZ_A_N): + case (OP_ALG_PKMODE_COPY_NSZ_B_E): + case (OP_ALG_PKMODE_COPY_NSZ_B_N): + case (OP_ALG_PKMODE_COPY_NSZ_N_A): + case (OP_ALG_PKMODE_COPY_NSZ_N_B): + case (OP_ALG_PKMODE_COPY_NSZ_N_E): + case (OP_ALG_PKMODE_COPY_SSZ_A0_B0): + case (OP_ALG_PKMODE_COPY_SSZ_A0_B1): + case (OP_ALG_PKMODE_COPY_SSZ_A0_B2): + case (OP_ALG_PKMODE_COPY_SSZ_A0_B3): + case (OP_ALG_PKMODE_COPY_SSZ_A1_B0): + case (OP_ALG_PKMODE_COPY_SSZ_A1_B1): + case (OP_ALG_PKMODE_COPY_SSZ_A1_B2): + case (OP_ALG_PKMODE_COPY_SSZ_A1_B3): + case (OP_ALG_PKMODE_COPY_SSZ_A2_B0): + case (OP_ALG_PKMODE_COPY_SSZ_A2_B1): + case (OP_ALG_PKMODE_COPY_SSZ_A2_B2): + case (OP_ALG_PKMODE_COPY_SSZ_A2_B3): + case (OP_ALG_PKMODE_COPY_SSZ_A3_B0): + case (OP_ALG_PKMODE_COPY_SSZ_A3_B1): + case (OP_ALG_PKMODE_COPY_SSZ_A3_B2): + case (OP_ALG_PKMODE_COPY_SSZ_A3_B3): + case (OP_ALG_PKMODE_COPY_SSZ_B0_A0): + case (OP_ALG_PKMODE_COPY_SSZ_B0_A1): + case (OP_ALG_PKMODE_COPY_SSZ_B0_A2): + case (OP_ALG_PKMODE_COPY_SSZ_B0_A3): + case (OP_ALG_PKMODE_COPY_SSZ_B1_A0): + case (OP_ALG_PKMODE_COPY_SSZ_B1_A1): + case (OP_ALG_PKMODE_COPY_SSZ_B1_A2): + case (OP_ALG_PKMODE_COPY_SSZ_B1_A3): + case (OP_ALG_PKMODE_COPY_SSZ_B2_A0): + case (OP_ALG_PKMODE_COPY_SSZ_B2_A1): + case (OP_ALG_PKMODE_COPY_SSZ_B2_A2): + case (OP_ALG_PKMODE_COPY_SSZ_B2_A3): + case (OP_ALG_PKMODE_COPY_SSZ_B3_A0): + case (OP_ALG_PKMODE_COPY_SSZ_B3_A1): + case (OP_ALG_PKMODE_COPY_SSZ_B3_A2): + case (OP_ALG_PKMODE_COPY_SSZ_B3_A3): + case (OP_ALG_PKMODE_COPY_SSZ_A_E): + case (OP_ALG_PKMODE_COPY_SSZ_A_N): + case (OP_ALG_PKMODE_COPY_SSZ_B_E): + case (OP_ALG_PKMODE_COPY_SSZ_B_N): + case (OP_ALG_PKMODE_COPY_SSZ_N_A): + case (OP_ALG_PKMODE_COPY_SSZ_N_B): + case (OP_ALG_PKMODE_COPY_SSZ_N_E): + return 0; + } + + return -EINVAL; +} + +static inline int +rta_pkha_operation(struct program *program, uint32_t op_pkha) +{ + uint32_t opcode = CMD_OPERATION | OP_TYPE_PK | OP_ALG_PK; + uint32_t pkha_func; + unsigned int start_pc = program->current_pc; + int ret = -EINVAL; + + pkha_func = op_pkha & OP_ALG_PK_FUN_MASK; + + switch (pkha_func) { + case (OP_ALG_PKMODE_CLEARMEM): + ret = __rta_pkha_clearmem(op_pkha); + if (ret < 0) { + pr_err("OPERATION PKHA: Type not supported. SEC Program Line: %d\n", + program->current_pc); + goto err; + } + break; + case (OP_ALG_PKMODE_MOD_ADD): + case (OP_ALG_PKMODE_MOD_SUB_AB): + case (OP_ALG_PKMODE_MOD_SUB_BA): + case (OP_ALG_PKMODE_MOD_MULT): + case (OP_ALG_PKMODE_MOD_EXPO): + case (OP_ALG_PKMODE_MOD_REDUCT): + case (OP_ALG_PKMODE_MOD_INV): + case (OP_ALG_PKMODE_MOD_MONT_CNST): + case (OP_ALG_PKMODE_MOD_CRT_CNST): + case (OP_ALG_PKMODE_MOD_GCD): + case (OP_ALG_PKMODE_MOD_PRIMALITY): + case (OP_ALG_PKMODE_MOD_SML_EXP): + case (OP_ALG_PKMODE_ECC_MOD_ADD): + case (OP_ALG_PKMODE_ECC_MOD_DBL): + case (OP_ALG_PKMODE_ECC_MOD_MUL): + ret = __rta_pkha_mod_arithmetic(op_pkha); + if (ret < 0) { + pr_err("OPERATION PKHA: Type not supported. SEC Program Line: %d\n", + program->current_pc); + goto err; + } + break; + case (OP_ALG_PKMODE_COPY_NSZ): + case (OP_ALG_PKMODE_COPY_SSZ): + ret = __rta_pkha_copymem(op_pkha); + if (ret < 0) { + pr_err("OPERATION PKHA: Type not supported. SEC Program Line: %d\n", + program->current_pc); + goto err; + } + break; + default: + pr_err("Invalid Operation Command\n"); + goto err; + } + + opcode |= op_pkha; + + __rta_out32(program, opcode); + program->current_instruction++; + return (int)start_pc; + + err: + program->first_error_pc = start_pc; + program->current_instruction++; + return ret; +} + +#endif /* __RTA_OPERATION_CMD_H__ */ diff --git a/dpdk/drivers/crypto/dpaa2_sec/hw/rta/protocol_cmd.h b/dpdk/drivers/crypto/dpaa2_sec/hw/rta/protocol_cmd.h new file mode 100644 index 00000000..f134235a --- /dev/null +++ b/dpdk/drivers/crypto/dpaa2_sec/hw/rta/protocol_cmd.h @@ -0,0 +1,732 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2008-2016 Freescale Semiconductor Inc. + * Copyright 2016 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __RTA_PROTOCOL_CMD_H__ +#define __RTA_PROTOCOL_CMD_H__ + +extern enum rta_sec_era rta_sec_era; + +static inline int +__rta_ssl_proto(uint16_t protoinfo) +{ + switch (protoinfo) { + case OP_PCL_SSL30_RC4_40_MD5_2: + case OP_PCL_SSL30_RC4_128_MD5_2: + case OP_PCL_SSL30_RC4_128_SHA_5: + case OP_PCL_SSL30_RC4_40_MD5_3: + case OP_PCL_SSL30_RC4_128_MD5_3: + case OP_PCL_SSL30_RC4_128_SHA: + case OP_PCL_SSL30_RC4_128_MD5: + case OP_PCL_SSL30_RC4_40_SHA: + case OP_PCL_SSL30_RC4_40_MD5: + case OP_PCL_SSL30_RC4_128_SHA_2: + case OP_PCL_SSL30_RC4_128_SHA_3: + case OP_PCL_SSL30_RC4_128_SHA_4: + case OP_PCL_SSL30_RC4_128_SHA_6: + case OP_PCL_SSL30_RC4_128_SHA_7: + case OP_PCL_SSL30_RC4_128_SHA_8: + case OP_PCL_SSL30_RC4_128_SHA_9: + case OP_PCL_SSL30_RC4_128_SHA_10: + case OP_PCL_TLS_ECDHE_PSK_RC4_128_SHA: + if (rta_sec_era == RTA_SEC_ERA_7) + return -EINVAL; + /* fall through if not Era 7 */ + case OP_PCL_SSL30_DES40_CBC_SHA: + case OP_PCL_SSL30_DES_CBC_SHA_2: + case OP_PCL_SSL30_3DES_EDE_CBC_SHA_5: + case OP_PCL_SSL30_DES40_CBC_SHA_2: + case OP_PCL_SSL30_DES_CBC_SHA_3: + case OP_PCL_SSL30_3DES_EDE_CBC_SHA_6: + case OP_PCL_SSL30_DES40_CBC_SHA_3: + case OP_PCL_SSL30_DES_CBC_SHA_4: + case OP_PCL_SSL30_3DES_EDE_CBC_SHA_7: + case OP_PCL_SSL30_DES40_CBC_SHA_4: + case OP_PCL_SSL30_DES_CBC_SHA_5: + case OP_PCL_SSL30_3DES_EDE_CBC_SHA_8: + case OP_PCL_SSL30_DES40_CBC_SHA_5: + case OP_PCL_SSL30_DES_CBC_SHA_6: + case OP_PCL_SSL30_3DES_EDE_CBC_SHA_9: + case OP_PCL_SSL30_DES40_CBC_SHA_6: + case OP_PCL_SSL30_DES_CBC_SHA_7: + case OP_PCL_SSL30_3DES_EDE_CBC_SHA_10: + case OP_PCL_SSL30_DES_CBC_SHA: + case OP_PCL_SSL30_3DES_EDE_CBC_SHA: + case OP_PCL_SSL30_DES_CBC_MD5: + case OP_PCL_SSL30_3DES_EDE_CBC_MD5: + case OP_PCL_SSL30_DES40_CBC_SHA_7: + case OP_PCL_SSL30_DES40_CBC_MD5: + case OP_PCL_SSL30_AES_128_CBC_SHA: + case OP_PCL_SSL30_AES_128_CBC_SHA_2: + case OP_PCL_SSL30_AES_128_CBC_SHA_3: + case OP_PCL_SSL30_AES_128_CBC_SHA_4: + case OP_PCL_SSL30_AES_128_CBC_SHA_5: + case OP_PCL_SSL30_AES_128_CBC_SHA_6: + case OP_PCL_SSL30_AES_256_CBC_SHA: + case OP_PCL_SSL30_AES_256_CBC_SHA_2: + case OP_PCL_SSL30_AES_256_CBC_SHA_3: + case OP_PCL_SSL30_AES_256_CBC_SHA_4: + case OP_PCL_SSL30_AES_256_CBC_SHA_5: + case OP_PCL_SSL30_AES_256_CBC_SHA_6: + case OP_PCL_TLS12_AES_128_CBC_SHA256_2: + case OP_PCL_TLS12_AES_128_CBC_SHA256_3: + case OP_PCL_TLS12_AES_128_CBC_SHA256_4: + case OP_PCL_TLS12_AES_128_CBC_SHA256_5: + case OP_PCL_TLS12_AES_256_CBC_SHA256_2: + case OP_PCL_TLS12_AES_256_CBC_SHA256_3: + case OP_PCL_TLS12_AES_256_CBC_SHA256_4: + case OP_PCL_TLS12_AES_256_CBC_SHA256_5: + case OP_PCL_TLS12_AES_128_CBC_SHA256_6: + case OP_PCL_TLS12_AES_256_CBC_SHA256_6: + case OP_PCL_SSL30_3DES_EDE_CBC_SHA_2: + case OP_PCL_SSL30_AES_128_CBC_SHA_7: + case OP_PCL_SSL30_AES_256_CBC_SHA_7: + case OP_PCL_SSL30_3DES_EDE_CBC_SHA_3: + case OP_PCL_SSL30_AES_128_CBC_SHA_8: + case OP_PCL_SSL30_AES_256_CBC_SHA_8: + case OP_PCL_SSL30_3DES_EDE_CBC_SHA_4: + case OP_PCL_SSL30_AES_128_CBC_SHA_9: + case OP_PCL_SSL30_AES_256_CBC_SHA_9: + case OP_PCL_SSL30_AES_128_GCM_SHA256_1: + case OP_PCL_SSL30_AES_256_GCM_SHA384_1: + case OP_PCL_SSL30_AES_128_GCM_SHA256_2: + case OP_PCL_SSL30_AES_256_GCM_SHA384_2: + case OP_PCL_SSL30_AES_128_GCM_SHA256_3: + case OP_PCL_SSL30_AES_256_GCM_SHA384_3: + case OP_PCL_SSL30_AES_128_GCM_SHA256_4: + case OP_PCL_SSL30_AES_256_GCM_SHA384_4: + case OP_PCL_SSL30_AES_128_GCM_SHA256_5: + case OP_PCL_SSL30_AES_256_GCM_SHA384_5: + case OP_PCL_SSL30_AES_128_GCM_SHA256_6: + case OP_PCL_TLS_DH_ANON_AES_256_GCM_SHA384: + case OP_PCL_TLS_PSK_AES_128_GCM_SHA256: + case OP_PCL_TLS_PSK_AES_256_GCM_SHA384: + case OP_PCL_TLS_DHE_PSK_AES_128_GCM_SHA256: + case OP_PCL_TLS_DHE_PSK_AES_256_GCM_SHA384: + case OP_PCL_TLS_RSA_PSK_AES_128_GCM_SHA256: + case OP_PCL_TLS_RSA_PSK_AES_256_GCM_SHA384: + case OP_PCL_TLS_PSK_AES_128_CBC_SHA256: + case OP_PCL_TLS_PSK_AES_256_CBC_SHA384: + case OP_PCL_TLS_DHE_PSK_AES_128_CBC_SHA256: + case OP_PCL_TLS_DHE_PSK_AES_256_CBC_SHA384: + case OP_PCL_TLS_RSA_PSK_AES_128_CBC_SHA256: + case OP_PCL_TLS_RSA_PSK_AES_256_CBC_SHA384: + case OP_PCL_SSL30_3DES_EDE_CBC_SHA_11: + case OP_PCL_SSL30_AES_128_CBC_SHA_10: + case OP_PCL_SSL30_AES_256_CBC_SHA_10: + case OP_PCL_SSL30_3DES_EDE_CBC_SHA_12: + case OP_PCL_SSL30_AES_128_CBC_SHA_11: + case OP_PCL_SSL30_AES_256_CBC_SHA_11: + case OP_PCL_SSL30_AES_128_CBC_SHA_12: + case OP_PCL_SSL30_3DES_EDE_CBC_SHA_13: + case OP_PCL_SSL30_AES_256_CBC_SHA_12: + case OP_PCL_SSL30_3DES_EDE_CBC_SHA_14: + case OP_PCL_SSL30_AES_128_CBC_SHA_13: + case OP_PCL_SSL30_AES_256_CBC_SHA_13: + case OP_PCL_SSL30_3DES_EDE_CBC_SHA_15: + case OP_PCL_SSL30_AES_128_CBC_SHA_14: + case OP_PCL_SSL30_AES_256_CBC_SHA_14: + case OP_PCL_SSL30_3DES_EDE_CBC_SHA_16: + case OP_PCL_SSL30_3DES_EDE_CBC_SHA_17: + case OP_PCL_SSL30_3DES_EDE_CBC_SHA_18: + case OP_PCL_SSL30_AES_128_CBC_SHA_15: + case OP_PCL_SSL30_AES_128_CBC_SHA_16: + case OP_PCL_SSL30_AES_128_CBC_SHA_17: + case OP_PCL_SSL30_AES_256_CBC_SHA_15: + case OP_PCL_SSL30_AES_256_CBC_SHA_16: + case OP_PCL_SSL30_AES_256_CBC_SHA_17: + case OP_PCL_TLS_ECDHE_ECDSA_AES_128_CBC_SHA256: + case OP_PCL_TLS_ECDHE_ECDSA_AES_256_CBC_SHA384: + case OP_PCL_TLS_ECDH_ECDSA_AES_128_CBC_SHA256: + case OP_PCL_TLS_ECDH_ECDSA_AES_256_CBC_SHA384: + case OP_PCL_TLS_ECDHE_RSA_AES_128_CBC_SHA256: + case OP_PCL_TLS_ECDHE_RSA_AES_256_CBC_SHA384: + case OP_PCL_TLS_ECDH_RSA_AES_128_CBC_SHA256: + case OP_PCL_TLS_ECDH_RSA_AES_256_CBC_SHA384: + case OP_PCL_TLS_ECDHE_ECDSA_AES_128_GCM_SHA256: + case OP_PCL_TLS_ECDHE_ECDSA_AES_256_GCM_SHA384: + case OP_PCL_TLS_ECDH_ECDSA_AES_128_GCM_SHA256: + case OP_PCL_TLS_ECDH_ECDSA_AES_256_GCM_SHA384: + case OP_PCL_TLS_ECDHE_RSA_AES_128_GCM_SHA256: + case OP_PCL_TLS_ECDHE_RSA_AES_256_GCM_SHA384: + case OP_PCL_TLS_ECDH_RSA_AES_128_GCM_SHA256: + case OP_PCL_TLS_ECDH_RSA_AES_256_GCM_SHA384: + case OP_PCL_TLS_ECDHE_PSK_3DES_EDE_CBC_SHA: + case OP_PCL_TLS_ECDHE_PSK_AES_128_CBC_SHA: + case OP_PCL_TLS_ECDHE_PSK_AES_256_CBC_SHA: + case OP_PCL_TLS_ECDHE_PSK_AES_128_CBC_SHA256: + case OP_PCL_TLS_ECDHE_PSK_AES_256_CBC_SHA384: + case OP_PCL_TLS12_3DES_EDE_CBC_MD5: + case OP_PCL_TLS12_3DES_EDE_CBC_SHA160: + case OP_PCL_TLS12_3DES_EDE_CBC_SHA224: + case OP_PCL_TLS12_3DES_EDE_CBC_SHA256: + case OP_PCL_TLS12_3DES_EDE_CBC_SHA384: + case OP_PCL_TLS12_3DES_EDE_CBC_SHA512: + case OP_PCL_TLS12_AES_128_CBC_SHA160: + case OP_PCL_TLS12_AES_128_CBC_SHA224: + case OP_PCL_TLS12_AES_128_CBC_SHA256: + case OP_PCL_TLS12_AES_128_CBC_SHA384: + case OP_PCL_TLS12_AES_128_CBC_SHA512: + case OP_PCL_TLS12_AES_192_CBC_SHA160: + case OP_PCL_TLS12_AES_192_CBC_SHA224: + case OP_PCL_TLS12_AES_192_CBC_SHA256: + case OP_PCL_TLS12_AES_192_CBC_SHA512: + case OP_PCL_TLS12_AES_256_CBC_SHA160: + case OP_PCL_TLS12_AES_256_CBC_SHA224: + case OP_PCL_TLS12_AES_256_CBC_SHA256: + case OP_PCL_TLS12_AES_256_CBC_SHA384: + case OP_PCL_TLS12_AES_256_CBC_SHA512: + case OP_PCL_TLS_PVT_AES_192_CBC_SHA160: + case OP_PCL_TLS_PVT_AES_192_CBC_SHA384: + case OP_PCL_TLS_PVT_AES_192_CBC_SHA224: + case OP_PCL_TLS_PVT_AES_192_CBC_SHA512: + case OP_PCL_TLS_PVT_AES_192_CBC_SHA256: + case OP_PCL_TLS_PVT_MASTER_SECRET_PRF_FE: + case OP_PCL_TLS_PVT_MASTER_SECRET_PRF_FF: + return 0; + } + + return -EINVAL; +} + +static inline int +__rta_ike_proto(uint16_t protoinfo) +{ + switch (protoinfo) { + case OP_PCL_IKE_HMAC_MD5: + case OP_PCL_IKE_HMAC_SHA1: + case OP_PCL_IKE_HMAC_AES128_CBC: + case OP_PCL_IKE_HMAC_SHA256: + case OP_PCL_IKE_HMAC_SHA384: + case OP_PCL_IKE_HMAC_SHA512: + case OP_PCL_IKE_HMAC_AES128_CMAC: + return 0; + } + + return -EINVAL; +} + +static inline int +__rta_ipsec_proto(uint16_t protoinfo) +{ + uint16_t proto_cls1 = protoinfo & OP_PCL_IPSEC_CIPHER_MASK; + uint16_t proto_cls2 = protoinfo & OP_PCL_IPSEC_AUTH_MASK; + + switch (proto_cls1) { + case OP_PCL_IPSEC_AES_NULL_WITH_GMAC: + if (rta_sec_era < RTA_SEC_ERA_2) + return -EINVAL; + /* no break */ + case OP_PCL_IPSEC_AES_CCM8: + case OP_PCL_IPSEC_AES_CCM12: + case OP_PCL_IPSEC_AES_CCM16: + case OP_PCL_IPSEC_AES_GCM8: + case OP_PCL_IPSEC_AES_GCM12: + case OP_PCL_IPSEC_AES_GCM16: + /* CCM, GCM, GMAC require PROTINFO[7:0] = 0 */ + if (proto_cls2 == OP_PCL_IPSEC_HMAC_NULL) + return 0; + return -EINVAL; + case OP_PCL_IPSEC_NULL: + if (rta_sec_era < RTA_SEC_ERA_2) + return -EINVAL; + /* no break */ + case OP_PCL_IPSEC_DES_IV64: + case OP_PCL_IPSEC_DES: + case OP_PCL_IPSEC_3DES: + case OP_PCL_IPSEC_AES_CBC: + case OP_PCL_IPSEC_AES_CTR: + break; + default: + return -EINVAL; + } + + switch (proto_cls2) { + case OP_PCL_IPSEC_HMAC_NULL: + case OP_PCL_IPSEC_HMAC_MD5_96: + case OP_PCL_IPSEC_HMAC_SHA1_96: + case OP_PCL_IPSEC_AES_XCBC_MAC_96: + case OP_PCL_IPSEC_HMAC_MD5_128: + case OP_PCL_IPSEC_HMAC_SHA1_160: + case OP_PCL_IPSEC_AES_CMAC_96: + case OP_PCL_IPSEC_HMAC_SHA2_256_128: + case OP_PCL_IPSEC_HMAC_SHA2_384_192: + case OP_PCL_IPSEC_HMAC_SHA2_512_256: + return 0; + } + + return -EINVAL; +} + +static inline int +__rta_srtp_proto(uint16_t protoinfo) +{ + uint16_t proto_cls1 = protoinfo & OP_PCL_SRTP_CIPHER_MASK; + uint16_t proto_cls2 = protoinfo & OP_PCL_SRTP_AUTH_MASK; + + switch (proto_cls1) { + case OP_PCL_SRTP_AES_CTR: + switch (proto_cls2) { + case OP_PCL_SRTP_HMAC_SHA1_160: + return 0; + } + /* no break */ + } + + return -EINVAL; +} + +static inline int +__rta_macsec_proto(uint16_t protoinfo) +{ + switch (protoinfo) { + case OP_PCL_MACSEC: + return 0; + } + + return -EINVAL; +} + +static inline int +__rta_wifi_proto(uint16_t protoinfo) +{ + switch (protoinfo) { + case OP_PCL_WIFI: + return 0; + } + + return -EINVAL; +} + +static inline int +__rta_wimax_proto(uint16_t protoinfo) +{ + switch (protoinfo) { + case OP_PCL_WIMAX_OFDM: + case OP_PCL_WIMAX_OFDMA: + return 0; + } + + return -EINVAL; +} + +/* Allowed blob proto flags for each SEC Era */ +static const uint32_t proto_blob_flags[] = { + OP_PCL_BLOB_FORMAT_MASK | OP_PCL_BLOB_BLACK, + OP_PCL_BLOB_FORMAT_MASK | OP_PCL_BLOB_BLACK | OP_PCL_BLOB_TKEK | + OP_PCL_BLOB_EKT | OP_PCL_BLOB_REG_MASK, + OP_PCL_BLOB_FORMAT_MASK | OP_PCL_BLOB_BLACK | OP_PCL_BLOB_TKEK | + OP_PCL_BLOB_EKT | OP_PCL_BLOB_REG_MASK, + OP_PCL_BLOB_FORMAT_MASK | OP_PCL_BLOB_BLACK | OP_PCL_BLOB_TKEK | + OP_PCL_BLOB_EKT | OP_PCL_BLOB_REG_MASK | OP_PCL_BLOB_SEC_MEM, + OP_PCL_BLOB_FORMAT_MASK | OP_PCL_BLOB_BLACK | OP_PCL_BLOB_TKEK | + OP_PCL_BLOB_EKT | OP_PCL_BLOB_REG_MASK | OP_PCL_BLOB_SEC_MEM +}; + +static inline int +__rta_blob_proto(uint16_t protoinfo) +{ + if (protoinfo & ~proto_blob_flags[rta_sec_era]) + return -EINVAL; + + switch (protoinfo & OP_PCL_BLOB_FORMAT_MASK) { + case OP_PCL_BLOB_FORMAT_NORMAL: + case OP_PCL_BLOB_FORMAT_MASTER_VER: + case OP_PCL_BLOB_FORMAT_TEST: + break; + default: + return -EINVAL; + } + + switch (protoinfo & OP_PCL_BLOB_REG_MASK) { + case OP_PCL_BLOB_AFHA_SBOX: + if (rta_sec_era < RTA_SEC_ERA_3) + return -EINVAL; + /* no break */ + case OP_PCL_BLOB_REG_MEMORY: + case OP_PCL_BLOB_REG_KEY1: + case OP_PCL_BLOB_REG_KEY2: + case OP_PCL_BLOB_REG_SPLIT: + case OP_PCL_BLOB_REG_PKE: + return 0; + } + + return -EINVAL; +} + +static inline int +__rta_dlc_proto(uint16_t protoinfo) +{ + if ((rta_sec_era < RTA_SEC_ERA_2) && + (protoinfo & (OP_PCL_PKPROT_DSA_MSG | OP_PCL_PKPROT_HASH_MASK | + OP_PCL_PKPROT_EKT_Z | OP_PCL_PKPROT_DECRYPT_Z | + OP_PCL_PKPROT_DECRYPT_PRI))) + return -EINVAL; + + switch (protoinfo & OP_PCL_PKPROT_HASH_MASK) { + case OP_PCL_PKPROT_HASH_MD5: + case OP_PCL_PKPROT_HASH_SHA1: + case OP_PCL_PKPROT_HASH_SHA224: + case OP_PCL_PKPROT_HASH_SHA256: + case OP_PCL_PKPROT_HASH_SHA384: + case OP_PCL_PKPROT_HASH_SHA512: + break; + default: + return -EINVAL; + } + + return 0; +} + +static inline int +__rta_rsa_enc_proto(uint16_t protoinfo) +{ + switch (protoinfo & OP_PCL_RSAPROT_OP_MASK) { + case OP_PCL_RSAPROT_OP_ENC_F_IN: + if ((protoinfo & OP_PCL_RSAPROT_FFF_MASK) != + OP_PCL_RSAPROT_FFF_RED) + return -EINVAL; + break; + case OP_PCL_RSAPROT_OP_ENC_F_OUT: + switch (protoinfo & OP_PCL_RSAPROT_FFF_MASK) { + case OP_PCL_RSAPROT_FFF_RED: + case OP_PCL_RSAPROT_FFF_ENC: + case OP_PCL_RSAPROT_FFF_EKT: + case OP_PCL_RSAPROT_FFF_TK_ENC: + case OP_PCL_RSAPROT_FFF_TK_EKT: + break; + default: + return -EINVAL; + } + break; + default: + return -EINVAL; + } + + return 0; +} + +static inline int +__rta_rsa_dec_proto(uint16_t protoinfo) +{ + switch (protoinfo & OP_PCL_RSAPROT_OP_MASK) { + case OP_PCL_RSAPROT_OP_DEC_ND: + case OP_PCL_RSAPROT_OP_DEC_PQD: + case OP_PCL_RSAPROT_OP_DEC_PQDPDQC: + break; + default: + return -EINVAL; + } + + switch (protoinfo & OP_PCL_RSAPROT_PPP_MASK) { + case OP_PCL_RSAPROT_PPP_RED: + case OP_PCL_RSAPROT_PPP_ENC: + case OP_PCL_RSAPROT_PPP_EKT: + case OP_PCL_RSAPROT_PPP_TK_ENC: + case OP_PCL_RSAPROT_PPP_TK_EKT: + break; + default: + return -EINVAL; + } + + if (protoinfo & OP_PCL_RSAPROT_FMT_PKCSV15) + switch (protoinfo & OP_PCL_RSAPROT_FFF_MASK) { + case OP_PCL_RSAPROT_FFF_RED: + case OP_PCL_RSAPROT_FFF_ENC: + case OP_PCL_RSAPROT_FFF_EKT: + case OP_PCL_RSAPROT_FFF_TK_ENC: + case OP_PCL_RSAPROT_FFF_TK_EKT: + break; + default: + return -EINVAL; + } + + return 0; +} + +/* + * DKP Protocol - Restrictions on key (SRC,DST) combinations + * For e.g. key_in_out[0][0] = 1 means (SRC=IMM,DST=IMM) combination is allowed + */ +static const uint8_t key_in_out[4][4] = { {1, 0, 0, 0}, + {1, 1, 1, 1}, + {1, 0, 1, 0}, + {1, 0, 0, 1} }; + +static inline int +__rta_dkp_proto(uint16_t protoinfo) +{ + int key_src = (protoinfo & OP_PCL_DKP_SRC_MASK) >> OP_PCL_DKP_SRC_SHIFT; + int key_dst = (protoinfo & OP_PCL_DKP_DST_MASK) >> OP_PCL_DKP_DST_SHIFT; + + if (!key_in_out[key_src][key_dst]) { + pr_err("PROTO_DESC: Invalid DKP key (SRC,DST)\n"); + return -EINVAL; + } + + return 0; +} + + +static inline int +__rta_3g_dcrc_proto(uint16_t protoinfo) +{ + if (rta_sec_era == RTA_SEC_ERA_7) + return -EINVAL; + + switch (protoinfo) { + case OP_PCL_3G_DCRC_CRC7: + case OP_PCL_3G_DCRC_CRC11: + return 0; + } + + return -EINVAL; +} + +static inline int +__rta_3g_rlc_proto(uint16_t protoinfo) +{ + if (rta_sec_era == RTA_SEC_ERA_7) + return -EINVAL; + + switch (protoinfo) { + case OP_PCL_3G_RLC_NULL: + case OP_PCL_3G_RLC_KASUMI: + case OP_PCL_3G_RLC_SNOW: + return 0; + } + + return -EINVAL; +} + +static inline int +__rta_lte_pdcp_proto(uint16_t protoinfo) +{ + if (rta_sec_era == RTA_SEC_ERA_7) + return -EINVAL; + + switch (protoinfo) { + case OP_PCL_LTE_ZUC: + if (rta_sec_era < RTA_SEC_ERA_5) + break; + case OP_PCL_LTE_NULL: + case OP_PCL_LTE_SNOW: + case OP_PCL_LTE_AES: + return 0; + } + + return -EINVAL; +} + +static inline int +__rta_lte_pdcp_mixed_proto(uint16_t protoinfo) +{ + switch (protoinfo & OP_PCL_LTE_MIXED_AUTH_MASK) { + case OP_PCL_LTE_MIXED_AUTH_NULL: + case OP_PCL_LTE_MIXED_AUTH_SNOW: + case OP_PCL_LTE_MIXED_AUTH_AES: + case OP_PCL_LTE_MIXED_AUTH_ZUC: + break; + default: + return -EINVAL; + } + + switch (protoinfo & OP_PCL_LTE_MIXED_ENC_MASK) { + case OP_PCL_LTE_MIXED_ENC_NULL: + case OP_PCL_LTE_MIXED_ENC_SNOW: + case OP_PCL_LTE_MIXED_ENC_AES: + case OP_PCL_LTE_MIXED_ENC_ZUC: + return 0; + } + + return -EINVAL; +} + +struct proto_map { + uint32_t optype; + uint32_t protid; + int (*protoinfo_func)(uint16_t); +}; + +static const struct proto_map proto_table[] = { +/*1*/ {OP_TYPE_UNI_PROTOCOL, OP_PCLID_SSL30_PRF, __rta_ssl_proto}, + {OP_TYPE_UNI_PROTOCOL, OP_PCLID_TLS10_PRF, __rta_ssl_proto}, + {OP_TYPE_UNI_PROTOCOL, OP_PCLID_TLS11_PRF, __rta_ssl_proto}, + {OP_TYPE_UNI_PROTOCOL, OP_PCLID_TLS12_PRF, __rta_ssl_proto}, + {OP_TYPE_UNI_PROTOCOL, OP_PCLID_DTLS10_PRF, __rta_ssl_proto}, + {OP_TYPE_UNI_PROTOCOL, OP_PCLID_IKEV1_PRF, __rta_ike_proto}, + {OP_TYPE_UNI_PROTOCOL, OP_PCLID_IKEV2_PRF, __rta_ike_proto}, + {OP_TYPE_UNI_PROTOCOL, OP_PCLID_PUBLICKEYPAIR, __rta_dlc_proto}, + {OP_TYPE_UNI_PROTOCOL, OP_PCLID_DSASIGN, __rta_dlc_proto}, + {OP_TYPE_UNI_PROTOCOL, OP_PCLID_DSAVERIFY, __rta_dlc_proto}, + {OP_TYPE_DECAP_PROTOCOL, OP_PCLID_IPSEC, __rta_ipsec_proto}, + {OP_TYPE_DECAP_PROTOCOL, OP_PCLID_SRTP, __rta_srtp_proto}, + {OP_TYPE_DECAP_PROTOCOL, OP_PCLID_SSL30, __rta_ssl_proto}, + {OP_TYPE_DECAP_PROTOCOL, OP_PCLID_TLS10, __rta_ssl_proto}, + {OP_TYPE_DECAP_PROTOCOL, OP_PCLID_TLS11, __rta_ssl_proto}, + {OP_TYPE_DECAP_PROTOCOL, OP_PCLID_TLS12, __rta_ssl_proto}, + {OP_TYPE_DECAP_PROTOCOL, OP_PCLID_DTLS10, __rta_ssl_proto}, + {OP_TYPE_DECAP_PROTOCOL, OP_PCLID_MACSEC, __rta_macsec_proto}, + {OP_TYPE_DECAP_PROTOCOL, OP_PCLID_WIFI, __rta_wifi_proto}, + {OP_TYPE_DECAP_PROTOCOL, OP_PCLID_WIMAX, __rta_wimax_proto}, +/*21*/ {OP_TYPE_DECAP_PROTOCOL, OP_PCLID_BLOB, __rta_blob_proto}, + {OP_TYPE_UNI_PROTOCOL, OP_PCLID_DIFFIEHELLMAN, __rta_dlc_proto}, + {OP_TYPE_UNI_PROTOCOL, OP_PCLID_RSAENCRYPT, __rta_rsa_enc_proto}, + {OP_TYPE_UNI_PROTOCOL, OP_PCLID_RSADECRYPT, __rta_rsa_dec_proto}, + {OP_TYPE_DECAP_PROTOCOL, OP_PCLID_3G_DCRC, __rta_3g_dcrc_proto}, + {OP_TYPE_DECAP_PROTOCOL, OP_PCLID_3G_RLC_PDU, __rta_3g_rlc_proto}, + {OP_TYPE_DECAP_PROTOCOL, OP_PCLID_3G_RLC_SDU, __rta_3g_rlc_proto}, + {OP_TYPE_DECAP_PROTOCOL, OP_PCLID_LTE_PDCP_USER, __rta_lte_pdcp_proto}, +/*29*/ {OP_TYPE_DECAP_PROTOCOL, OP_PCLID_LTE_PDCP_CTRL, __rta_lte_pdcp_proto}, + {OP_TYPE_UNI_PROTOCOL, OP_PCLID_DKP_MD5, __rta_dkp_proto}, + {OP_TYPE_UNI_PROTOCOL, OP_PCLID_DKP_SHA1, __rta_dkp_proto}, + {OP_TYPE_UNI_PROTOCOL, OP_PCLID_DKP_SHA224, __rta_dkp_proto}, + {OP_TYPE_UNI_PROTOCOL, OP_PCLID_DKP_SHA256, __rta_dkp_proto}, + {OP_TYPE_UNI_PROTOCOL, OP_PCLID_DKP_SHA384, __rta_dkp_proto}, +/*35*/ {OP_TYPE_UNI_PROTOCOL, OP_PCLID_DKP_SHA512, __rta_dkp_proto}, + {OP_TYPE_DECAP_PROTOCOL, OP_PCLID_PUBLICKEYPAIR, __rta_dlc_proto}, +/*37*/ {OP_TYPE_DECAP_PROTOCOL, OP_PCLID_DSASIGN, __rta_dlc_proto}, +/*38*/ {OP_TYPE_DECAP_PROTOCOL, OP_PCLID_LTE_PDCP_CTRL_MIXED, + __rta_lte_pdcp_mixed_proto}, + {OP_TYPE_DECAP_PROTOCOL, OP_PCLID_IPSEC_NEW, __rta_ipsec_proto}, +}; + +/* + * Allowed OPERATION protocols for each SEC Era. + * Values represent the number of entries from proto_table[] that are supported. + */ +static const unsigned int proto_table_sz[] = {21, 29, 29, 29, 29, 35, 37, 39}; + +static inline int +rta_proto_operation(struct program *program, uint32_t optype, + uint32_t protid, uint16_t protoinfo) +{ + uint32_t opcode = CMD_OPERATION; + unsigned int i, found = 0; + uint32_t optype_tmp = optype; + unsigned int start_pc = program->current_pc; + int ret = -EINVAL; + + for (i = 0; i < proto_table_sz[rta_sec_era]; i++) { + /* clear last bit in optype to match also decap proto */ + optype_tmp &= (uint32_t)~(1 << OP_TYPE_SHIFT); + if (optype_tmp == proto_table[i].optype) { + if (proto_table[i].protid == protid) { + /* nothing else to verify */ + if (proto_table[i].protoinfo_func == NULL) { + found = 1; + break; + } + /* check protoinfo */ + ret = (*proto_table[i].protoinfo_func) + (protoinfo); + if (ret < 0) { + pr_err("PROTO_DESC: Bad PROTO Type. SEC Program Line: %d\n", + program->current_pc); + goto err; + } + found = 1; + break; + } + } + } + if (!found) { + pr_err("PROTO_DESC: Operation Type Mismatch. SEC Program Line: %d\n", + program->current_pc); + goto err; + } + + __rta_out32(program, opcode | optype | protid | protoinfo); + program->current_instruction++; + return (int)start_pc; + + err: + program->first_error_pc = start_pc; + program->current_instruction++; + return ret; +} + +static inline int +rta_dkp_proto(struct program *program, uint32_t protid, + uint16_t key_src, uint16_t key_dst, + uint16_t keylen, uint64_t key, + enum rta_data_type key_type) +{ + unsigned int start_pc = program->current_pc; + unsigned int in_words = 0, out_words = 0; + int ret; + + key_src &= OP_PCL_DKP_SRC_MASK; + key_dst &= OP_PCL_DKP_DST_MASK; + keylen &= OP_PCL_DKP_KEY_MASK; + + ret = rta_proto_operation(program, OP_TYPE_UNI_PROTOCOL, protid, + key_src | key_dst | keylen); + if (ret < 0) + return ret; + + if ((key_src == OP_PCL_DKP_SRC_PTR) || + (key_src == OP_PCL_DKP_SRC_SGF)) { + __rta_out64(program, program->ps, key); + in_words = program->ps ? 2 : 1; + } else if (key_src == OP_PCL_DKP_SRC_IMM) { + __rta_inline_data(program, key, inline_flags(key_type), keylen); + in_words = (unsigned int)((keylen + 3) / 4); + } + + if ((key_dst == OP_PCL_DKP_DST_PTR) || + (key_dst == OP_PCL_DKP_DST_SGF)) { + out_words = in_words; + } else if (key_dst == OP_PCL_DKP_DST_IMM) { + out_words = split_key_len(protid) / 4; + } + + if (out_words < in_words) { + pr_err("PROTO_DESC: DKP doesn't currently support a smaller descriptor\n"); + program->first_error_pc = start_pc; + return -EINVAL; + } + + /* If needed, reserve space in resulting descriptor for derived key */ + program->current_pc += (out_words - in_words); + + return (int)start_pc; +} + +#endif /* __RTA_PROTOCOL_CMD_H__ */ diff --git a/dpdk/drivers/crypto/dpaa2_sec/hw/rta/sec_run_time_asm.h b/dpdk/drivers/crypto/dpaa2_sec/hw/rta/sec_run_time_asm.h new file mode 100644 index 00000000..11995aa2 --- /dev/null +++ b/dpdk/drivers/crypto/dpaa2_sec/hw/rta/sec_run_time_asm.h @@ -0,0 +1,823 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2008-2016 Freescale Semiconductor Inc. + * Copyright 2016 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __RTA_SEC_RUN_TIME_ASM_H__ +#define __RTA_SEC_RUN_TIME_ASM_H__ + +#include "hw/desc.h" + +/* hw/compat.h is not delivered in kernel */ +#ifndef __KERNEL__ +#include "hw/compat.h" +#endif + +/** + * enum rta_sec_era - SEC HW block revisions supported by the RTA library + * @RTA_SEC_ERA_1: SEC Era 1 + * @RTA_SEC_ERA_2: SEC Era 2 + * @RTA_SEC_ERA_3: SEC Era 3 + * @RTA_SEC_ERA_4: SEC Era 4 + * @RTA_SEC_ERA_5: SEC Era 5 + * @RTA_SEC_ERA_6: SEC Era 6 + * @RTA_SEC_ERA_7: SEC Era 7 + * @RTA_SEC_ERA_8: SEC Era 8 + * @MAX_SEC_ERA: maximum SEC HW block revision supported by RTA library + */ +enum rta_sec_era { + RTA_SEC_ERA_1, + RTA_SEC_ERA_2, + RTA_SEC_ERA_3, + RTA_SEC_ERA_4, + RTA_SEC_ERA_5, + RTA_SEC_ERA_6, + RTA_SEC_ERA_7, + RTA_SEC_ERA_8, + MAX_SEC_ERA = RTA_SEC_ERA_8 +}; + +/** + * DEFAULT_SEC_ERA - the default value for the SEC era in case the user provides + * an unsupported value. + */ +#define DEFAULT_SEC_ERA MAX_SEC_ERA + +/** + * USER_SEC_ERA - translates the SEC Era from internal to user representation. + * @sec_era: SEC Era in internal (library) representation + */ +#define USER_SEC_ERA(sec_era) (sec_era + 1) + +/** + * INTL_SEC_ERA - translates the SEC Era from user representation to internal. + * @sec_era: SEC Era in user representation + */ +#define INTL_SEC_ERA(sec_era) (sec_era - 1) + +/** + * enum rta_jump_type - Types of action taken by JUMP command + * @LOCAL_JUMP: conditional jump to an offset within the descriptor buffer + * @FAR_JUMP: conditional jump to a location outside the descriptor buffer, + * indicated by the POINTER field after the JUMP command. + * @HALT: conditional halt - stop the execution of the current descriptor and + * writes PKHA / Math condition bits as status / error code. + * @HALT_STATUS: conditional halt with user-specified status - stop the + * execution of the current descriptor and writes the value of + * "LOCAL OFFSET" JUMP field as status / error code. + * @GOSUB: conditional subroutine call - similar to @LOCAL_JUMP, but also saves + * return address in the Return Address register; subroutine calls + * cannot be nested. + * @RETURN: conditional subroutine return - similar to @LOCAL_JUMP, but the + * offset is taken from the Return Address register. + * @LOCAL_JUMP_INC: similar to @LOCAL_JUMP, but increment the register specified + * in "SRC_DST" JUMP field before evaluating the jump + * condition. + * @LOCAL_JUMP_DEC: similar to @LOCAL_JUMP, but decrement the register specified + * in "SRC_DST" JUMP field before evaluating the jump + * condition. + */ +enum rta_jump_type { + LOCAL_JUMP, + FAR_JUMP, + HALT, + HALT_STATUS, + GOSUB, + RETURN, + LOCAL_JUMP_INC, + LOCAL_JUMP_DEC +}; + +/** + * enum rta_jump_cond - How test conditions are evaluated by JUMP command + * @ALL_TRUE: perform action if ALL selected conditions are true + * @ALL_FALSE: perform action if ALL selected conditions are false + * @ANY_TRUE: perform action if ANY of the selected conditions is true + * @ANY_FALSE: perform action if ANY of the selected conditions is false + */ +enum rta_jump_cond { + ALL_TRUE, + ALL_FALSE, + ANY_TRUE, + ANY_FALSE +}; + +/** + * enum rta_share_type - Types of sharing for JOB_HDR and SHR_HDR commands + * @SHR_NEVER: nothing is shared; descriptors can execute in parallel (i.e. no + * dependencies are allowed between them). + * @SHR_WAIT: shared descriptor and keys are shared once the descriptor sets + * "OK to share" in DECO Control Register (DCTRL). + * @SHR_SERIAL: shared descriptor and keys are shared once the descriptor has + * completed. + * @SHR_ALWAYS: shared descriptor is shared anytime after the descriptor is + * loaded. + * @SHR_DEFER: valid only for JOB_HDR; sharing type is the one specified + * in the shared descriptor associated with the job descriptor. + */ +enum rta_share_type { + SHR_NEVER, + SHR_WAIT, + SHR_SERIAL, + SHR_ALWAYS, + SHR_DEFER +}; + +/** + * enum rta_data_type - Indicates how is the data provided and how to include it + * in the descriptor. + * @RTA_DATA_PTR: Data is in memory and accessed by reference; data address is a + * physical (bus) address. + * @RTA_DATA_IMM: Data is inlined in descriptor and accessed as immediate data; + * data address is a virtual address. + * @RTA_DATA_IMM_DMA: (AIOP only) Data is inlined in descriptor and accessed as + * immediate data; data address is a physical (bus) address + * in external memory and CDMA is programmed to transfer the + * data into descriptor buffer being built in Workspace Area. + */ +enum rta_data_type { + RTA_DATA_PTR = 1, + RTA_DATA_IMM, + RTA_DATA_IMM_DMA +}; + +/* Registers definitions */ +enum rta_regs { + /* CCB Registers */ + CONTEXT1 = 1, + CONTEXT2, + KEY1, + KEY2, + KEY1SZ, + KEY2SZ, + ICV1SZ, + ICV2SZ, + DATA1SZ, + DATA2SZ, + ALTDS1, + IV1SZ, + AAD1SZ, + MODE1, + MODE2, + CCTRL, + DCTRL, + ICTRL, + CLRW, + CSTAT, + IFIFO, + NFIFO, + OFIFO, + PKASZ, + PKBSZ, + PKNSZ, + PKESZ, + /* DECO Registers */ + MATH0, + MATH1, + MATH2, + MATH3, + DESCBUF, + JOBDESCBUF, + SHAREDESCBUF, + DPOVRD, + DJQDA, + DSTAT, + DPID, + DJQCTRL, + ALTSOURCE, + SEQINSZ, + SEQOUTSZ, + VSEQINSZ, + VSEQOUTSZ, + /* PKHA Registers */ + PKA, + PKN, + PKA0, + PKA1, + PKA2, + PKA3, + PKB, + PKB0, + PKB1, + PKB2, + PKB3, + PKE, + /* Pseudo registers */ + AB1, + AB2, + ABD, + IFIFOABD, + IFIFOAB1, + IFIFOAB2, + AFHA_SBOX, + MDHA_SPLIT_KEY, + JOBSRC, + ZERO, + ONE, + AAD1, + IV1, + IV2, + MSG1, + MSG2, + MSG, + MSG_CKSUM, + MSGOUTSNOOP, + MSGINSNOOP, + ICV1, + ICV2, + SKIP, + NONE, + RNGOFIFO, + RNG, + IDFNS, + ODFNS, + NFIFOSZ, + SZ, + PAD, + SAD1, + AAD2, + BIT_DATA, + NFIFO_SZL, + NFIFO_SZM, + NFIFO_L, + NFIFO_M, + SZL, + SZM, + JOBDESCBUF_EFF, + SHAREDESCBUF_EFF, + METADATA, + GTR, + STR, + OFIFO_SYNC, + MSGOUTSNOOP_ALT +}; + +/* Command flags */ +#define FLUSH1 BIT(0) +#define LAST1 BIT(1) +#define LAST2 BIT(2) +#define IMMED BIT(3) +#define SGF BIT(4) +#define VLF BIT(5) +#define EXT BIT(6) +#define CONT BIT(7) +#define SEQ BIT(8) +#define AIDF BIT(9) +#define FLUSH2 BIT(10) +#define CLASS1 BIT(11) +#define CLASS2 BIT(12) +#define BOTH BIT(13) + +/** + * DCOPY - (AIOP only) command param is pointer to external memory + * + * CDMA must be used to transfer the key via DMA into Workspace Area. + * Valid only in combination with IMMED flag. + */ +#define DCOPY BIT(30) + +#define COPY BIT(31) /* command param is pointer (not immediate) + * valid only in combination when IMMED + */ + +#define __COPY_MASK (COPY | DCOPY) + +/* SEQ IN/OUT PTR Command specific flags */ +#define RBS BIT(16) +#define INL BIT(17) +#define PRE BIT(18) +#define RTO BIT(19) +#define RJD BIT(20) +#define SOP BIT(21) +#define RST BIT(22) +#define EWS BIT(23) + +#define ENC BIT(14) /* Encrypted Key */ +#define EKT BIT(15) /* AES CCM Encryption (default is + * AES ECB Encryption) + */ +#define TK BIT(16) /* Trusted Descriptor Key (default is + * Job Descriptor Key) + */ +#define NWB BIT(17) /* No Write Back Key */ +#define PTS BIT(18) /* Plaintext Store */ + +/* HEADER Command specific flags */ +#define RIF BIT(16) +#define DNR BIT(17) +#define CIF BIT(18) +#define PD BIT(19) +#define RSMS BIT(20) +#define TD BIT(21) +#define MTD BIT(22) +#define REO BIT(23) +#define SHR BIT(24) +#define SC BIT(25) +/* Extended HEADER specific flags */ +#define DSV BIT(7) +#define DSEL_MASK 0x00000007 /* DECO Select */ +#define FTD BIT(8) + +/* JUMP Command specific flags */ +#define NIFP BIT(20) +#define NIP BIT(21) +#define NOP BIT(22) +#define NCP BIT(23) +#define CALM BIT(24) + +#define MATH_Z BIT(25) +#define MATH_N BIT(26) +#define MATH_NV BIT(27) +#define MATH_C BIT(28) +#define PK_0 BIT(29) +#define PK_GCD_1 BIT(30) +#define PK_PRIME BIT(31) +#define SELF BIT(0) +#define SHRD BIT(1) +#define JQP BIT(2) + +/* NFIFOADD specific flags */ +#define PAD_ZERO BIT(16) +#define PAD_NONZERO BIT(17) +#define PAD_INCREMENT BIT(18) +#define PAD_RANDOM BIT(19) +#define PAD_ZERO_N1 BIT(20) +#define PAD_NONZERO_0 BIT(21) +#define PAD_N1 BIT(23) +#define PAD_NONZERO_N BIT(24) +#define OC BIT(25) +#define BM BIT(26) +#define PR BIT(27) +#define PS BIT(28) +#define BP BIT(29) + +/* MOVE Command specific flags */ +#define WAITCOMP BIT(16) +#define SIZE_WORD BIT(17) +#define SIZE_BYTE BIT(18) +#define SIZE_DWORD BIT(19) + +/* MATH command specific flags */ +#define IFB MATH_IFB +#define NFU MATH_NFU +#define STL MATH_STL +#define SSEL MATH_SSEL +#define SWP MATH_SWP +#define IMMED2 BIT(31) + +/** + * struct program - descriptor buffer management structure + * @current_pc: current offset in descriptor + * @current_instruction: current instruction in descriptor + * @first_error_pc: offset of the first error in descriptor + * @start_pc: start offset in descriptor buffer + * @buffer: buffer carrying descriptor + * @shrhdr: shared descriptor header + * @jobhdr: job descriptor header + * @ps: pointer fields size; if ps is true, pointers will be 36bits in + * length; if ps is false, pointers will be 32bits in length + * @bswap: if true, perform byte swap on a 4-byte boundary + */ +struct program { + unsigned int current_pc; + unsigned int current_instruction; + unsigned int first_error_pc; + unsigned int start_pc; + uint32_t *buffer; + uint32_t *shrhdr; + uint32_t *jobhdr; + bool ps; + bool bswap; +}; + +static inline void +rta_program_cntxt_init(struct program *program, + uint32_t *buffer, unsigned int offset) +{ + program->current_pc = 0; + program->current_instruction = 0; + program->first_error_pc = 0; + program->start_pc = offset; + program->buffer = buffer; + program->shrhdr = NULL; + program->jobhdr = NULL; + program->ps = false; + program->bswap = false; +} + +static inline int +rta_program_finalize(struct program *program) +{ + /* Descriptor is usually not allowed to go beyond 64 words size */ + if (program->current_pc > MAX_CAAM_DESCSIZE) + pr_warn("Descriptor Size exceeded max limit of 64 words\n"); + + /* Descriptor is erroneous */ + if (program->first_error_pc) { + pr_err("Descriptor creation error\n"); + return -EINVAL; + } + + /* Update descriptor length in shared and job descriptor headers */ + if (program->shrhdr != NULL) + *program->shrhdr |= program->bswap ? + swab32(program->current_pc) : + program->current_pc; + else if (program->jobhdr != NULL) + *program->jobhdr |= program->bswap ? + swab32(program->current_pc) : + program->current_pc; + + return (int)program->current_pc; +} + +static inline unsigned int +rta_program_set_36bit_addr(struct program *program) +{ + program->ps = true; + return program->current_pc; +} + +static inline unsigned int +rta_program_set_bswap(struct program *program) +{ + program->bswap = true; + return program->current_pc; +} + +static inline void +__rta_out32(struct program *program, uint32_t val) +{ + program->buffer[program->current_pc] = program->bswap ? + swab32(val) : val; + program->current_pc++; +} + +static inline void +__rta_out_be32(struct program *program, uint32_t val) +{ + program->buffer[program->current_pc] = cpu_to_be32(val); + program->current_pc++; +} + +static inline void +__rta_out_le32(struct program *program, uint32_t val) +{ + program->buffer[program->current_pc] = cpu_to_le32(val); + program->current_pc++; +} + +static inline void +__rta_out64(struct program *program, bool is_ext, uint64_t val) +{ + if (is_ext) { + /* + * Since we are guaranteed only a 4-byte alignment in the + * descriptor buffer, we have to do 2 x 32-bit (word) writes. + * For the order of the 2 words to be correct, we need to + * take into account the endianness of the CPU. + */ +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + __rta_out32(program, program->bswap ? lower_32_bits(val) : + upper_32_bits(val)); + + __rta_out32(program, program->bswap ? upper_32_bits(val) : + lower_32_bits(val)); +#else + __rta_out32(program, program->bswap ? upper_32_bits(val) : + lower_32_bits(val)); + + __rta_out32(program, program->bswap ? lower_32_bits(val) : + upper_32_bits(val)); +#endif + } else { + __rta_out32(program, lower_32_bits(val)); + } +} + +static inline unsigned int +rta_word(struct program *program, uint32_t val) +{ + unsigned int start_pc = program->current_pc; + + __rta_out32(program, val); + + return start_pc; +} + +static inline unsigned int +rta_dword(struct program *program, uint64_t val) +{ + unsigned int start_pc = program->current_pc; + + __rta_out64(program, true, val); + + return start_pc; +} + +static inline uint32_t +inline_flags(enum rta_data_type data_type) +{ + switch (data_type) { + case RTA_DATA_PTR: + return 0; + case RTA_DATA_IMM: + return IMMED | COPY; + case RTA_DATA_IMM_DMA: + return IMMED | DCOPY; + default: + /* warn and default to RTA_DATA_PTR */ + pr_warn("RTA: defaulting to RTA_DATA_PTR parameter type\n"); + return 0; + } +} + +static inline unsigned int +rta_copy_data(struct program *program, uint8_t *data, unsigned int length) +{ + unsigned int i; + unsigned int start_pc = program->current_pc; + uint8_t *tmp = (uint8_t *)&program->buffer[program->current_pc]; + + for (i = 0; i < length; i++) + *tmp++ = data[i]; + program->current_pc += (length + 3) / 4; + + return start_pc; +} + +#if defined(__EWL__) && defined(AIOP) +static inline void +__rta_dma_data(void *ws_dst, uint64_t ext_address, uint16_t size) +{ cdma_read(ws_dst, ext_address, size); } +#else +static inline void +__rta_dma_data(void *ws_dst __maybe_unused, + uint64_t ext_address __maybe_unused, + uint16_t size __maybe_unused) +{ pr_warn("RTA: DCOPY not supported, DMA will be skipped\n"); } +#endif /* defined(__EWL__) && defined(AIOP) */ + +static inline void +__rta_inline_data(struct program *program, uint64_t data, + uint32_t copy_data, uint32_t length) +{ + if (!copy_data) { + __rta_out64(program, length > 4, data); + } else if (copy_data & COPY) { + uint8_t *tmp = (uint8_t *)&program->buffer[program->current_pc]; + uint32_t i; + + for (i = 0; i < length; i++) + *tmp++ = ((uint8_t *)(uintptr_t)data)[i]; + program->current_pc += ((length + 3) / 4); + } else if (copy_data & DCOPY) { + __rta_dma_data(&program->buffer[program->current_pc], data, + (uint16_t)length); + program->current_pc += ((length + 3) / 4); + } +} + +static inline unsigned int +rta_desc_len(uint32_t *buffer) +{ + if ((*buffer & CMD_MASK) == CMD_DESC_HDR) + return *buffer & HDR_DESCLEN_MASK; + else + return *buffer & HDR_DESCLEN_SHR_MASK; +} + +static inline unsigned int +rta_desc_bytes(uint32_t *buffer) +{ + return (unsigned int)(rta_desc_len(buffer) * CAAM_CMD_SZ); +} + +/** + * split_key_len - Compute MDHA split key length for a given algorithm + * @hash: Hashing algorithm selection, one of OP_ALG_ALGSEL_* or + * OP_PCLID_DKP_* - MD5, SHA1, SHA224, SHA256, SHA384, SHA512. + * + * Return: MDHA split key length + */ +static inline uint32_t +split_key_len(uint32_t hash) +{ + /* Sizes for MDHA pads (*not* keys): MD5, SHA1, 224, 256, 384, 512 */ + static const uint8_t mdpadlen[] = { 16, 20, 32, 32, 64, 64 }; + uint32_t idx; + + idx = (hash & OP_ALG_ALGSEL_SUBMASK) >> OP_ALG_ALGSEL_SHIFT; + + return (uint32_t)(mdpadlen[idx] * 2); +} + +/** + * split_key_pad_len - Compute MDHA split key pad length for a given algorithm + * @hash: Hashing algorithm selection, one of OP_ALG_ALGSEL_* - MD5, SHA1, + * SHA224, SHA384, SHA512. + * + * Return: MDHA split key pad length + */ +static inline uint32_t +split_key_pad_len(uint32_t hash) +{ + return ALIGN(split_key_len(hash), 16); +} + +static inline unsigned int +rta_set_label(struct program *program) +{ + return program->current_pc + program->start_pc; +} + +static inline int +rta_patch_move(struct program *program, int line, unsigned int new_ref) +{ + uint32_t opcode; + bool bswap = program->bswap; + + if (line < 0) + return -EINVAL; + + opcode = bswap ? swab32(program->buffer[line]) : program->buffer[line]; + + opcode &= (uint32_t)~MOVE_OFFSET_MASK; + opcode |= (new_ref << (MOVE_OFFSET_SHIFT + 2)) & MOVE_OFFSET_MASK; + program->buffer[line] = bswap ? swab32(opcode) : opcode; + + return 0; +} + +static inline int +rta_patch_jmp(struct program *program, int line, unsigned int new_ref) +{ + uint32_t opcode; + bool bswap = program->bswap; + + if (line < 0) + return -EINVAL; + + opcode = bswap ? swab32(program->buffer[line]) : program->buffer[line]; + + opcode &= (uint32_t)~JUMP_OFFSET_MASK; + opcode |= (new_ref - (line + program->start_pc)) & JUMP_OFFSET_MASK; + program->buffer[line] = bswap ? swab32(opcode) : opcode; + + return 0; +} + +static inline int +rta_patch_header(struct program *program, int line, unsigned int new_ref) +{ + uint32_t opcode; + bool bswap = program->bswap; + + if (line < 0) + return -EINVAL; + + opcode = bswap ? swab32(program->buffer[line]) : program->buffer[line]; + + opcode &= (uint32_t)~HDR_START_IDX_MASK; + opcode |= (new_ref << HDR_START_IDX_SHIFT) & HDR_START_IDX_MASK; + program->buffer[line] = bswap ? swab32(opcode) : opcode; + + return 0; +} + +static inline int +rta_patch_load(struct program *program, int line, unsigned int new_ref) +{ + uint32_t opcode; + bool bswap = program->bswap; + + if (line < 0) + return -EINVAL; + + opcode = (bswap ? swab32(program->buffer[line]) : + program->buffer[line]) & (uint32_t)~LDST_OFFSET_MASK; + + if (opcode & (LDST_SRCDST_WORD_DESCBUF | LDST_CLASS_DECO)) + opcode |= (new_ref << LDST_OFFSET_SHIFT) & LDST_OFFSET_MASK; + else + opcode |= (new_ref << (LDST_OFFSET_SHIFT + 2)) & + LDST_OFFSET_MASK; + + program->buffer[line] = bswap ? swab32(opcode) : opcode; + + return 0; +} + +static inline int +rta_patch_store(struct program *program, int line, unsigned int new_ref) +{ + uint32_t opcode; + bool bswap = program->bswap; + + if (line < 0) + return -EINVAL; + + opcode = bswap ? swab32(program->buffer[line]) : program->buffer[line]; + + opcode &= (uint32_t)~LDST_OFFSET_MASK; + + switch (opcode & LDST_SRCDST_MASK) { + case LDST_SRCDST_WORD_DESCBUF: + case LDST_SRCDST_WORD_DESCBUF_JOB: + case LDST_SRCDST_WORD_DESCBUF_SHARED: + case LDST_SRCDST_WORD_DESCBUF_JOB_WE: + case LDST_SRCDST_WORD_DESCBUF_SHARED_WE: + opcode |= ((new_ref) << LDST_OFFSET_SHIFT) & LDST_OFFSET_MASK; + break; + default: + opcode |= (new_ref << (LDST_OFFSET_SHIFT + 2)) & + LDST_OFFSET_MASK; + } + + program->buffer[line] = bswap ? swab32(opcode) : opcode; + + return 0; +} + +static inline int +rta_patch_raw(struct program *program, int line, unsigned int mask, + unsigned int new_val) +{ + uint32_t opcode; + bool bswap = program->bswap; + + if (line < 0) + return -EINVAL; + + opcode = bswap ? swab32(program->buffer[line]) : program->buffer[line]; + + opcode &= (uint32_t)~mask; + opcode |= new_val & mask; + program->buffer[line] = bswap ? swab32(opcode) : opcode; + + return 0; +} + +static inline int +__rta_map_opcode(uint32_t name, const uint32_t (*map_table)[2], + unsigned int num_of_entries, uint32_t *val) +{ + unsigned int i; + + for (i = 0; i < num_of_entries; i++) + if (map_table[i][0] == name) { + *val = map_table[i][1]; + return 0; + } + + return -EINVAL; +} + +static inline void +__rta_map_flags(uint32_t flags, const uint32_t (*flags_table)[2], + unsigned int num_of_entries, uint32_t *opcode) +{ + unsigned int i; + + for (i = 0; i < num_of_entries; i++) { + if (flags_table[i][0] & flags) + *opcode |= flags_table[i][1]; + } +} + +#endif /* __RTA_SEC_RUN_TIME_ASM_H__ */ diff --git a/dpdk/drivers/crypto/dpaa2_sec/hw/rta/seq_in_out_ptr_cmd.h b/dpdk/drivers/crypto/dpaa2_sec/hw/rta/seq_in_out_ptr_cmd.h new file mode 100644 index 00000000..4aea5770 --- /dev/null +++ b/dpdk/drivers/crypto/dpaa2_sec/hw/rta/seq_in_out_ptr_cmd.h @@ -0,0 +1,208 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2008-2016 Freescale Semiconductor Inc. + * Copyright 2016 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __RTA_SEQ_IN_OUT_PTR_CMD_H__ +#define __RTA_SEQ_IN_OUT_PTR_CMD_H__ + +extern enum rta_sec_era rta_sec_era; + +/* Allowed SEQ IN PTR flags for each SEC Era. */ +static const uint32_t seq_in_ptr_flags[] = { + RBS | INL | SGF | PRE | EXT | RTO, + RBS | INL | SGF | PRE | EXT | RTO | RJD, + RBS | INL | SGF | PRE | EXT | RTO | RJD, + RBS | INL | SGF | PRE | EXT | RTO | RJD, + RBS | INL | SGF | PRE | EXT | RTO | RJD | SOP, + RBS | INL | SGF | PRE | EXT | RTO | RJD | SOP, + RBS | INL | SGF | PRE | EXT | RTO | RJD | SOP, + RBS | INL | SGF | PRE | EXT | RTO | RJD | SOP +}; + +/* Allowed SEQ OUT PTR flags for each SEC Era. */ +static const uint32_t seq_out_ptr_flags[] = { + SGF | PRE | EXT, + SGF | PRE | EXT | RTO, + SGF | PRE | EXT | RTO, + SGF | PRE | EXT | RTO, + SGF | PRE | EXT | RTO | RST | EWS, + SGF | PRE | EXT | RTO | RST | EWS, + SGF | PRE | EXT | RTO | RST | EWS, + SGF | PRE | EXT | RTO | RST | EWS +}; + +static inline int +rta_seq_in_ptr(struct program *program, uint64_t src, + uint32_t length, uint32_t flags) +{ + uint32_t opcode = CMD_SEQ_IN_PTR; + unsigned int start_pc = program->current_pc; + int ret = -EINVAL; + + /* Parameters checking */ + if ((flags & RTO) && (flags & PRE)) { + pr_err("SEQ IN PTR: Invalid usage of RTO and PRE flags\n"); + goto err; + } + if (flags & ~seq_in_ptr_flags[rta_sec_era]) { + pr_err("SEQ IN PTR: Flag(s) not supported by SEC Era %d\n", + USER_SEC_ERA(rta_sec_era)); + goto err; + } + if ((flags & INL) && (flags & RJD)) { + pr_err("SEQ IN PTR: Invalid usage of INL and RJD flags\n"); + goto err; + } + if ((src) && (flags & (SOP | RTO | PRE))) { + pr_err("SEQ IN PTR: Invalid usage of RTO or PRE flag\n"); + goto err; + } + if ((flags & SOP) && (flags & (RBS | PRE | RTO | EXT))) { + pr_err("SEQ IN PTR: Invalid usage of SOP and (RBS or PRE or RTO or EXT) flags\n"); + goto err; + } + + /* write flag fields */ + if (flags & RBS) + opcode |= SQIN_RBS; + if (flags & INL) + opcode |= SQIN_INL; + if (flags & SGF) + opcode |= SQIN_SGF; + if (flags & PRE) + opcode |= SQIN_PRE; + if (flags & RTO) + opcode |= SQIN_RTO; + if (flags & RJD) + opcode |= SQIN_RJD; + if (flags & SOP) + opcode |= SQIN_SOP; + if ((length >> 16) || (flags & EXT)) { + if (flags & SOP) { + pr_err("SEQ IN PTR: Invalid usage of SOP and EXT flags\n"); + goto err; + } + + opcode |= SQIN_EXT; + } else { + opcode |= length & SQIN_LEN_MASK; + } + + __rta_out32(program, opcode); + program->current_instruction++; + + /* write pointer or immediate data field */ + if (!(opcode & (SQIN_PRE | SQIN_RTO | SQIN_SOP))) + __rta_out64(program, program->ps, src); + + /* write extended length field */ + if (opcode & SQIN_EXT) + __rta_out32(program, length); + + return (int)start_pc; + + err: + program->first_error_pc = start_pc; + program->current_instruction++; + return ret; +} + +static inline int +rta_seq_out_ptr(struct program *program, uint64_t dst, + uint32_t length, uint32_t flags) +{ + uint32_t opcode = CMD_SEQ_OUT_PTR; + unsigned int start_pc = program->current_pc; + int ret = -EINVAL; + + /* Parameters checking */ + if (flags & ~seq_out_ptr_flags[rta_sec_era]) { + pr_err("SEQ OUT PTR: Flag(s) not supported by SEC Era %d\n", + USER_SEC_ERA(rta_sec_era)); + goto err; + } + if ((flags & RTO) && (flags & PRE)) { + pr_err("SEQ OUT PTR: Invalid usage of RTO and PRE flags\n"); + goto err; + } + if ((dst) && (flags & (RTO | PRE))) { + pr_err("SEQ OUT PTR: Invalid usage of RTO or PRE flag\n"); + goto err; + } + if ((flags & RST) && !(flags & RTO)) { + pr_err("SEQ OUT PTR: RST flag must be used with RTO flag\n"); + goto err; + } + + /* write flag fields */ + if (flags & SGF) + opcode |= SQOUT_SGF; + if (flags & PRE) + opcode |= SQOUT_PRE; + if (flags & RTO) + opcode |= SQOUT_RTO; + if (flags & RST) + opcode |= SQOUT_RST; + if (flags & EWS) + opcode |= SQOUT_EWS; + if ((length >> 16) || (flags & EXT)) + opcode |= SQOUT_EXT; + else + opcode |= length & SQOUT_LEN_MASK; + + __rta_out32(program, opcode); + program->current_instruction++; + + /* write pointer or immediate data field */ + if (!(opcode & (SQOUT_PRE | SQOUT_RTO))) + __rta_out64(program, program->ps, dst); + + /* write extended length field */ + if (opcode & SQOUT_EXT) + __rta_out32(program, length); + + return (int)start_pc; + + err: + program->first_error_pc = start_pc; + program->current_instruction++; + return ret; +} + +#endif /* __RTA_SEQ_IN_OUT_PTR_CMD_H__ */ diff --git a/dpdk/drivers/crypto/dpaa2_sec/hw/rta/signature_cmd.h b/dpdk/drivers/crypto/dpaa2_sec/hw/rta/signature_cmd.h new file mode 100644 index 00000000..00597f30 --- /dev/null +++ b/dpdk/drivers/crypto/dpaa2_sec/hw/rta/signature_cmd.h @@ -0,0 +1,75 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2008-2016 Freescale Semiconductor Inc. + * Copyright 2016 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __RTA_SIGNATURE_CMD_H__ +#define __RTA_SIGNATURE_CMD_H__ + +static inline int +rta_signature(struct program *program, uint32_t sign_type) +{ + uint32_t opcode = CMD_SIGNATURE; + unsigned int start_pc = program->current_pc; + + switch (sign_type) { + case (SIGN_TYPE_FINAL): + case (SIGN_TYPE_FINAL_RESTORE): + case (SIGN_TYPE_FINAL_NONZERO): + case (SIGN_TYPE_IMM_2): + case (SIGN_TYPE_IMM_3): + case (SIGN_TYPE_IMM_4): + opcode |= sign_type; + break; + default: + pr_err("SIGNATURE Command: Invalid type selection\n"); + goto err; + } + + __rta_out32(program, opcode); + program->current_instruction++; + + return (int)start_pc; + + err: + program->first_error_pc = start_pc; + program->current_instruction++; + return -EINVAL; +} + +#endif /* __RTA_SIGNATURE_CMD_H__ */ diff --git a/dpdk/drivers/crypto/dpaa2_sec/hw/rta/store_cmd.h b/dpdk/drivers/crypto/dpaa2_sec/hw/rta/store_cmd.h new file mode 100644 index 00000000..1b75692e --- /dev/null +++ b/dpdk/drivers/crypto/dpaa2_sec/hw/rta/store_cmd.h @@ -0,0 +1,185 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2008-2016 Freescale Semiconductor Inc. + * Copyright 2016 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __RTA_STORE_CMD_H__ +#define __RTA_STORE_CMD_H__ + +extern enum rta_sec_era rta_sec_era; + +static const uint32_t store_src_table[][2] = { +/*1*/ { KEY1SZ, LDST_CLASS_1_CCB | LDST_SRCDST_WORD_KEYSZ_REG }, + { KEY2SZ, LDST_CLASS_2_CCB | LDST_SRCDST_WORD_KEYSZ_REG }, + { DJQDA, LDST_CLASS_DECO | LDST_SRCDST_WORD_DECO_JQDAR }, + { MODE1, LDST_CLASS_1_CCB | LDST_SRCDST_WORD_MODE_REG }, + { MODE2, LDST_CLASS_2_CCB | LDST_SRCDST_WORD_MODE_REG }, + { DJQCTRL, LDST_CLASS_DECO | LDST_SRCDST_WORD_DECO_JQCTRL }, + { DATA1SZ, LDST_CLASS_1_CCB | LDST_SRCDST_WORD_DATASZ_REG }, + { DATA2SZ, LDST_CLASS_2_CCB | LDST_SRCDST_WORD_DATASZ_REG }, + { DSTAT, LDST_CLASS_DECO | LDST_SRCDST_WORD_DECO_STAT }, + { ICV1SZ, LDST_CLASS_1_CCB | LDST_SRCDST_WORD_ICVSZ_REG }, + { ICV2SZ, LDST_CLASS_2_CCB | LDST_SRCDST_WORD_ICVSZ_REG }, + { DPID, LDST_CLASS_DECO | LDST_SRCDST_WORD_PID }, + { CCTRL, LDST_SRCDST_WORD_CHACTRL }, + { ICTRL, LDST_SRCDST_WORD_IRQCTRL }, + { CLRW, LDST_SRCDST_WORD_CLRW }, + { MATH0, LDST_CLASS_DECO | LDST_SRCDST_WORD_DECO_MATH0 }, + { CSTAT, LDST_SRCDST_WORD_STAT }, + { MATH1, LDST_CLASS_DECO | LDST_SRCDST_WORD_DECO_MATH1 }, + { MATH2, LDST_CLASS_DECO | LDST_SRCDST_WORD_DECO_MATH2 }, + { AAD1SZ, LDST_CLASS_1_CCB | LDST_SRCDST_WORD_DECO_AAD_SZ }, + { MATH3, LDST_CLASS_DECO | LDST_SRCDST_WORD_DECO_MATH3 }, + { IV1SZ, LDST_CLASS_1_CCB | LDST_SRCDST_WORD_CLASS1_IV_SZ }, + { PKASZ, LDST_CLASS_1_CCB | LDST_SRCDST_WORD_PKHA_A_SZ }, + { PKBSZ, LDST_CLASS_1_CCB | LDST_SRCDST_WORD_PKHA_B_SZ }, + { PKESZ, LDST_CLASS_1_CCB | LDST_SRCDST_WORD_PKHA_E_SZ }, + { PKNSZ, LDST_CLASS_1_CCB | LDST_SRCDST_WORD_PKHA_N_SZ }, + { CONTEXT1, LDST_CLASS_1_CCB | LDST_SRCDST_BYTE_CONTEXT }, + { CONTEXT2, LDST_CLASS_2_CCB | LDST_SRCDST_BYTE_CONTEXT }, + { DESCBUF, LDST_CLASS_DECO | LDST_SRCDST_WORD_DESCBUF }, +/*30*/ { JOBDESCBUF, LDST_CLASS_DECO | LDST_SRCDST_WORD_DESCBUF_JOB }, + { SHAREDESCBUF, LDST_CLASS_DECO | LDST_SRCDST_WORD_DESCBUF_SHARED }, +/*32*/ { JOBDESCBUF_EFF, LDST_CLASS_DECO | + LDST_SRCDST_WORD_DESCBUF_JOB_WE }, + { SHAREDESCBUF_EFF, LDST_CLASS_DECO | + LDST_SRCDST_WORD_DESCBUF_SHARED_WE }, +/*34*/ { GTR, LDST_CLASS_DECO | LDST_SRCDST_WORD_GTR }, + { STR, LDST_CLASS_DECO | LDST_SRCDST_WORD_STR } +}; + +/* + * Allowed STORE sources for each SEC ERA. + * Values represent the number of entries from source_src_table[] that are + * supported. + */ +static const unsigned int store_src_table_sz[] = {29, 31, 33, 33, + 33, 33, 35, 35}; + +static inline int +rta_store(struct program *program, uint64_t src, + uint16_t offset, uint64_t dst, uint32_t length, + uint32_t flags) +{ + uint32_t opcode = 0, val; + int ret = -EINVAL; + unsigned int start_pc = program->current_pc; + + if (flags & SEQ) + opcode = CMD_SEQ_STORE; + else + opcode = CMD_STORE; + + /* parameters check */ + if ((flags & IMMED) && (flags & SGF)) { + pr_err("STORE: Invalid flag. SEC PC: %d; Instr: %d\n", + program->current_pc, program->current_instruction); + goto err; + } + if ((flags & IMMED) && (offset != 0)) { + pr_err("STORE: Invalid flag. SEC PC: %d; Instr: %d\n", + program->current_pc, program->current_instruction); + goto err; + } + + if ((flags & SEQ) && ((src == JOBDESCBUF) || (src == SHAREDESCBUF) || + (src == JOBDESCBUF_EFF) || + (src == SHAREDESCBUF_EFF))) { + pr_err("STORE: Invalid SRC type. SEC PC: %d; Instr: %d\n", + program->current_pc, program->current_instruction); + goto err; + } + + if (flags & IMMED) + opcode |= LDST_IMM; + + if ((flags & SGF) || (flags & VLF)) + opcode |= LDST_VLF; + + /* + * source for data to be stored can be specified as: + * - register location; set in src field[9-15]; + * - if IMMED flag is set, data is set in value field [0-31]; + * user can give this value as actual value or pointer to data + */ + if (!(flags & IMMED)) { + ret = __rta_map_opcode((uint32_t)src, store_src_table, + store_src_table_sz[rta_sec_era], &val); + if (ret < 0) { + pr_err("STORE: Invalid source. SEC PC: %d; Instr: %d\n", + program->current_pc, + program->current_instruction); + goto err; + } + opcode |= val; + } + + /* DESC BUFFER: length / offset values are specified in 4-byte words */ + if ((src == DESCBUF) || (src == JOBDESCBUF) || (src == SHAREDESCBUF) || + (src == JOBDESCBUF_EFF) || (src == SHAREDESCBUF_EFF)) { + opcode |= (length >> 2); + opcode |= (uint32_t)((offset >> 2) << LDST_OFFSET_SHIFT); + } else { + opcode |= length; + opcode |= (uint32_t)(offset << LDST_OFFSET_SHIFT); + } + + __rta_out32(program, opcode); + program->current_instruction++; + + if ((src == JOBDESCBUF) || (src == SHAREDESCBUF) || + (src == JOBDESCBUF_EFF) || (src == SHAREDESCBUF_EFF)) + return (int)start_pc; + + /* for STORE, a pointer to where the data will be stored if needed */ + if (!(flags & SEQ)) + __rta_out64(program, program->ps, dst); + + /* for IMMED data, place the data here */ + if (flags & IMMED) + __rta_inline_data(program, src, flags & __COPY_MASK, length); + + return (int)start_pc; + + err: + program->first_error_pc = start_pc; + program->current_instruction++; + return ret; +} + +#endif /* __RTA_STORE_CMD_H__ */ diff --git a/dpdk/drivers/crypto/dpaa2_sec/mc/dpseci.c b/dpdk/drivers/crypto/dpaa2_sec/mc/dpseci.c new file mode 100644 index 00000000..2a216afc --- /dev/null +++ b/dpdk/drivers/crypto/dpaa2_sec/mc/dpseci.c @@ -0,0 +1,676 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2013-2016 Freescale Semiconductor Inc. + * Copyright 2016 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#include +#include +#include +#include + +/** + * dpseci_open() - Open a control session for the specified object + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @dpseci_id: DPSECI unique ID + * @token: Returned token; use in subsequent API calls + * + * This function can be used to open a control session for an + * already created object; an object may have been declared in + * the DPL or by calling the dpseci_create() function. + * This function returns a unique authentication token, + * associated with the specific object ID and the specific MC + * portal; this token must be used in all subsequent commands for + * this specific object. + * + * Return: '0' on Success; Error code otherwise. + */ +int dpseci_open(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + int dpseci_id, + uint16_t *token) +{ + struct dpseci_cmd_open *cmd_params; + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_OPEN, + cmd_flags, + 0); + cmd_params = (struct dpseci_cmd_open *)cmd.params; + cmd_params->dpseci_id = cpu_to_le32(dpseci_id); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + *token = mc_cmd_hdr_read_token(&cmd); + + return 0; +} + +/** + * dpseci_close() - Close the control session of the object + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPSECI object + * + * After this function is called, no further operations are + * allowed on the object without opening a new control session. + * + * Return: '0' on Success; Error code otherwise. + */ +int dpseci_close(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token) +{ + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_CLOSE, + cmd_flags, + token); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpseci_create() - Create the DPSECI object + * @mc_io: Pointer to MC portal's I/O object + * @dprc_token: Parent container token; '0' for default container + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @cfg: Configuration structure + * @obj_id: Returned object id + * + * Create the DPSECI object, allocate required resources and + * perform required initialization. + * + * The object can be created either by declaring it in the + * DPL file, or by calling this function. + * + * The function accepts an authentication token of a parent + * container that this object should be assigned to. The token + * can be '0' so the object will be assigned to the default container. + * The newly created object can be opened with the returned + * object id and using the container's associated tokens and MC portals. + * + * Return: '0' on Success; Error code otherwise. + */ +int dpseci_create(struct fsl_mc_io *mc_io, + uint16_t dprc_token, + uint32_t cmd_flags, + const struct dpseci_cfg *cfg, + uint32_t *obj_id) +{ + struct dpseci_cmd_create *cmd_params; + struct mc_command cmd = { 0 }; + int err, i; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_CREATE, + cmd_flags, + dprc_token); + cmd_params = (struct dpseci_cmd_create *)cmd.params; + for (i = 0; i < DPSECI_PRIO_NUM; i++) + cmd_params->priorities[i] = cfg->priorities[i]; + cmd_params->num_tx_queues = cfg->num_tx_queues; + cmd_params->num_rx_queues = cfg->num_rx_queues; + cmd_params->options = cfg->options; + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + *obj_id = mc_cmd_read_object_id(&cmd); + + return 0; +} + +/** + * dpseci_destroy() - Destroy the DPSECI object and release all its resources. + * @mc_io: Pointer to MC portal's I/O object + * @dprc_token: Parent container token; '0' for default container + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @object_id: The object id; it must be a valid id within the container that + * created this object; + * + * The function accepts the authentication token of the parent container that + * created the object (not the one that currently owns the object). The object + * is searched within parent using the provided 'object_id'. + * All tokens to the object must be closed before calling destroy. + * + * Return: '0' on Success; error code otherwise. + */ +int dpseci_destroy(struct fsl_mc_io *mc_io, + uint16_t dprc_token, + uint32_t cmd_flags, + uint32_t object_id) +{ + struct dpseci_cmd_destroy *cmd_params; + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_DESTROY, + cmd_flags, + dprc_token); + cmd_params = (struct dpseci_cmd_destroy *)cmd.params; + cmd_params->dpseci_id = cpu_to_le32(object_id); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpseci_enable() - Enable the DPSECI, allow sending and receiving frames. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPSECI object + * + * Return: '0' on Success; Error code otherwise. + */ +int dpseci_enable(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token) +{ + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_ENABLE, + cmd_flags, + token); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpseci_disable() - Disable the DPSECI, stop sending and receiving frames. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPSECI object + * + * Return: '0' on Success; Error code otherwise. + */ +int dpseci_disable(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token) +{ + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_DISABLE, + cmd_flags, + token); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpseci_is_enabled() - Check if the DPSECI is enabled. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPSECI object + * @en: Returns '1' if object is enabled; '0' otherwise + * + * Return: '0' on Success; Error code otherwise. + */ +int dpseci_is_enabled(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + int *en) +{ + struct dpseci_rsp_is_enabled *rsp_params; + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_IS_ENABLED, + cmd_flags, + token); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + rsp_params = (struct dpseci_rsp_is_enabled *)cmd.params; + *en = dpseci_get_field(rsp_params->en, ENABLE); + + return 0; +} + +/** + * dpseci_reset() - Reset the DPSECI, returns the object to initial state. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPSECI object + * + * Return: '0' on Success; Error code otherwise. + */ +int dpseci_reset(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token) +{ + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_RESET, + cmd_flags, + token); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpseci_get_attributes() - Retrieve DPSECI attributes. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPSECI object + * @attr: Returned object's attributes + * + * Return: '0' on Success; Error code otherwise. + */ +int dpseci_get_attributes(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + struct dpseci_attr *attr) +{ + struct dpseci_rsp_get_attr *rsp_params; + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_ATTR, + cmd_flags, + token); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + rsp_params = (struct dpseci_rsp_get_attr *)cmd.params; + attr->id = le32_to_cpu(rsp_params->id); + attr->options = rsp_params->options; + attr->num_tx_queues = rsp_params->num_tx_queues; + attr->num_rx_queues = rsp_params->num_rx_queues; + + return 0; +} + +/** + * dpseci_set_rx_queue() - Set Rx queue configuration + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPSECI object + * @queue: Select the queue relative to number of + * priorities configured at DPSECI creation; use + * DPSECI_ALL_QUEUES to configure all Rx queues identically. + * @cfg: Rx queue configuration + * + * Return: '0' on Success; Error code otherwise. + */ +int dpseci_set_rx_queue(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t queue, + const struct dpseci_rx_queue_cfg *cfg) +{ + struct dpseci_cmd_set_rx_queue *cmd_params; + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_SET_RX_QUEUE, + cmd_flags, + token); + cmd_params = (struct dpseci_cmd_set_rx_queue *)cmd.params; + cmd_params->dest_id = cpu_to_le32(cfg->dest_cfg.dest_id); + cmd_params->dest_priority = cfg->dest_cfg.priority; + cmd_params->queue = queue; + cmd_params->user_ctx = cpu_to_le64(cfg->user_ctx); + cmd_params->options = cpu_to_le32(cfg->options); + dpseci_set_field(cmd_params->dest_type, + DEST_TYPE, + cfg->dest_cfg.dest_type); + dpseci_set_field(cmd_params->order_preservation_en, + ORDER_PRESERVATION, + cfg->order_preservation_en); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpseci_get_rx_queue() - Retrieve Rx queue attributes. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPSECI object + * @queue: Select the queue relative to number of + * priorities configured at DPSECI creation + * @attr: Returned Rx queue attributes + * + * Return: '0' on Success; Error code otherwise. + */ +int dpseci_get_rx_queue(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t queue, + struct dpseci_rx_queue_attr *attr) +{ + struct dpseci_rsp_get_rx_queue *rsp_params; + struct dpseci_cmd_get_queue *cmd_params; + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_RX_QUEUE, + cmd_flags, + token); + cmd_params = (struct dpseci_cmd_get_queue *)cmd.params; + cmd_params->queue = queue; + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + rsp_params = (struct dpseci_rsp_get_rx_queue *)cmd.params; + attr->user_ctx = le64_to_cpu(rsp_params->user_ctx); + attr->fqid = le32_to_cpu(rsp_params->fqid); + attr->dest_cfg.dest_id = le32_to_cpu(rsp_params->dest_id); + attr->dest_cfg.priority = rsp_params->dest_priority; + attr->dest_cfg.dest_type = + dpseci_get_field(rsp_params->dest_type, + DEST_TYPE); + attr->order_preservation_en = + dpseci_get_field(rsp_params->order_preservation_en, + ORDER_PRESERVATION); + + return 0; +} + +/** + * dpseci_get_tx_queue() - Retrieve Tx queue attributes. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPSECI object + * @queue: Select the queue relative to number of + * priorities configured at DPSECI creation + * @attr: Returned Tx queue attributes + * + * Return: '0' on Success; Error code otherwise. + */ +int dpseci_get_tx_queue(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t queue, + struct dpseci_tx_queue_attr *attr) +{ + struct dpseci_rsp_get_tx_queue *rsp_params; + struct dpseci_cmd_get_queue *cmd_params; + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_TX_QUEUE, + cmd_flags, + token); + cmd_params = (struct dpseci_cmd_get_queue *)cmd.params; + cmd_params->queue = queue; + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + rsp_params = (struct dpseci_rsp_get_tx_queue *)cmd.params; + attr->fqid = le32_to_cpu(rsp_params->fqid); + attr->priority = rsp_params->priority; + + return 0; +} + +/** + * dpseci_get_sec_attr() - Retrieve SEC accelerator attributes. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPSECI object + * @attr: Returned SEC attributes + * + * Return: '0' on Success; Error code otherwise. + */ +int dpseci_get_sec_attr(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + struct dpseci_sec_attr *attr) +{ + struct dpseci_rsp_get_sec_attr *rsp_params; + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_SEC_ATTR, + cmd_flags, + token); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + rsp_params = (struct dpseci_rsp_get_sec_attr *)cmd.params; + attr->ip_id = le16_to_cpu(rsp_params->ip_id); + attr->major_rev = rsp_params->major_rev; + attr->minor_rev = rsp_params->minor_rev; + attr->era = rsp_params->era; + attr->deco_num = rsp_params->deco_num; + attr->zuc_auth_acc_num = rsp_params->zuc_auth_acc_num; + attr->zuc_enc_acc_num = rsp_params->zuc_enc_acc_num; + attr->snow_f8_acc_num = rsp_params->snow_f8_acc_num; + attr->snow_f9_acc_num = rsp_params->snow_f9_acc_num; + attr->crc_acc_num = rsp_params->crc_acc_num; + attr->pk_acc_num = rsp_params->pk_acc_num; + attr->kasumi_acc_num = rsp_params->kasumi_acc_num; + attr->rng_acc_num = rsp_params->rng_acc_num; + attr->md_acc_num = rsp_params->md_acc_num; + attr->arc4_acc_num = rsp_params->arc4_acc_num; + attr->des_acc_num = rsp_params->des_acc_num; + attr->aes_acc_num = rsp_params->aes_acc_num; + + return 0; +} + +/** + * dpseci_get_sec_counters() - Retrieve SEC accelerator counters. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPSECI object + * @counters: Returned SEC counters + * + * Return: '0' on Success; Error code otherwise. + */ +int dpseci_get_sec_counters(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + struct dpseci_sec_counters *counters) +{ + struct dpseci_rsp_get_sec_counters *rsp_params; + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_SEC_COUNTERS, + cmd_flags, + token); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + rsp_params = (struct dpseci_rsp_get_sec_counters *)cmd.params; + counters->dequeued_requests = + le64_to_cpu(rsp_params->dequeued_requests); + counters->ob_enc_requests = le64_to_cpu(rsp_params->ob_enc_requests); + counters->ib_dec_requests = le64_to_cpu(rsp_params->ib_dec_requests); + counters->ob_enc_bytes = le64_to_cpu(rsp_params->ob_enc_bytes); + counters->ob_prot_bytes = le64_to_cpu(rsp_params->ob_prot_bytes); + counters->ib_dec_bytes = le64_to_cpu(rsp_params->ib_dec_bytes); + counters->ib_valid_bytes = le64_to_cpu(rsp_params->ib_valid_bytes); + + return 0; +} + +/** + * dpseci_get_api_version() - Get Data Path SEC Interface API version + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @major_ver: Major version of data path sec API + * @minor_ver: Minor version of data path sec API + * + * Return: '0' on Success; Error code otherwise. + */ +int dpseci_get_api_version(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t *major_ver, + uint16_t *minor_ver) +{ + struct dpseci_rsp_get_api_version *rsp_params; + struct mc_command cmd = { 0 }; + int err; + + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_API_VERSION, + cmd_flags, + 0); + + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + rsp_params = (struct dpseci_rsp_get_api_version *)cmd.params; + *major_ver = le16_to_cpu(rsp_params->major); + *minor_ver = le16_to_cpu(rsp_params->minor); + + return 0; +} + +int dpseci_set_congestion_notification( + struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + const struct dpseci_congestion_notification_cfg *cfg) +{ + struct dpseci_cmd_set_congestion_notification *cmd_params; + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header( + DPSECI_CMDID_SET_CONGESTION_NOTIFICATION, + cmd_flags, + token); + + cmd_params = + (struct dpseci_cmd_set_congestion_notification *)cmd.params; + cmd_params->dest_id = cfg->dest_cfg.dest_id; + cmd_params->dest_priority = cfg->dest_cfg.priority; + cmd_params->message_ctx = cfg->message_ctx; + cmd_params->message_iova = cfg->message_iova; + cmd_params->notification_mode = cfg->notification_mode; + cmd_params->threshold_entry = cfg->threshold_entry; + cmd_params->threshold_exit = cfg->threshold_exit; + dpseci_set_field(cmd_params->type_units, + DEST_TYPE, + cfg->dest_cfg.dest_type); + dpseci_set_field(cmd_params->type_units, + CG_UNITS, + cfg->units); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +int dpseci_get_congestion_notification( + struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + struct dpseci_congestion_notification_cfg *cfg) +{ + struct dpseci_cmd_set_congestion_notification *rsp_params; + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header( + DPSECI_CMDID_GET_CONGESTION_NOTIFICATION, + cmd_flags, + token); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + rsp_params = + (struct dpseci_cmd_set_congestion_notification *)cmd.params; + + cfg->dest_cfg.dest_id = le32_to_cpu(rsp_params->dest_id); + cfg->dest_cfg.priority = rsp_params->dest_priority; + cfg->notification_mode = le16_to_cpu(rsp_params->notification_mode); + cfg->message_ctx = le64_to_cpu(rsp_params->message_ctx); + cfg->message_iova = le64_to_cpu(rsp_params->message_iova); + cfg->threshold_entry = le32_to_cpu(rsp_params->threshold_entry); + cfg->threshold_exit = le32_to_cpu(rsp_params->threshold_exit); + cfg->units = dpseci_get_field(rsp_params->type_units, CG_UNITS); + cfg->dest_cfg.dest_type = dpseci_get_field(rsp_params->type_units, + DEST_TYPE); + + return 0; +} diff --git a/dpdk/drivers/crypto/dpaa2_sec/mc/fsl_dpseci.h b/dpdk/drivers/crypto/dpaa2_sec/mc/fsl_dpseci.h new file mode 100644 index 00000000..4acb595c --- /dev/null +++ b/dpdk/drivers/crypto/dpaa2_sec/mc/fsl_dpseci.h @@ -0,0 +1,432 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2013-2016 Freescale Semiconductor Inc. + * Copyright 2016-2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef __FSL_DPSECI_H +#define __FSL_DPSECI_H + +/* Data Path SEC Interface API + * Contains initialization APIs and runtime control APIs for DPSECI + */ + +struct fsl_mc_io; + +/** + * General DPSECI macros + */ + +/** + * Maximum number of Tx/Rx priorities per DPSECI object + */ +#define DPSECI_PRIO_NUM 8 + +/** + * All queues considered; see dpseci_set_rx_queue() + */ +#define DPSECI_ALL_QUEUES (uint8_t)(-1) + +int dpseci_open(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + int dpseci_id, + uint16_t *token); + +int dpseci_close(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token); + +/** + * Enable the Congestion Group support + */ +#define DPSECI_OPT_HAS_CG 0x000020 + +/** + * struct dpseci_cfg - Structure representing DPSECI configuration + * @options: Any combination of the following options: + * DPSECI_OPT_HAS_CG + * DPSECI_OPT_HAS_OPR + * DPSECI_OPT_OPR_SHARED + * @num_tx_queues: num of queues towards the SEC + * @num_rx_queues: num of queues back from the SEC + * @priorities: Priorities for the SEC hardware processing; + * each place in the array is the priority of the tx queue + * towards the SEC, + * valid priorities are configured with values 1-8; + */ +struct dpseci_cfg { + uint32_t options; + uint8_t num_tx_queues; + uint8_t num_rx_queues; + uint8_t priorities[DPSECI_PRIO_NUM]; +}; + +int dpseci_create(struct fsl_mc_io *mc_io, + uint16_t dprc_token, + uint32_t cmd_flags, + const struct dpseci_cfg *cfg, + uint32_t *obj_id); + +int dpseci_destroy(struct fsl_mc_io *mc_io, + uint16_t dprc_token, + uint32_t cmd_flags, + uint32_t object_id); + +int dpseci_enable(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token); + +int dpseci_disable(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token); + +int dpseci_is_enabled(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + int *en); + +int dpseci_reset(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token); + +/** + * struct dpseci_attr - Structure representing DPSECI attributes + * @id: DPSECI object ID + * @num_tx_queues: number of queues towards the SEC + * @num_rx_queues: number of queues back from the SEC + * @options: Any combination of the following options: + * DPSECI_OPT_HAS_CG + * DPSECI_OPT_HAS_OPR + * DPSECI_OPT_OPR_SHARED + */ +struct dpseci_attr { + int id; + uint8_t num_tx_queues; + uint8_t num_rx_queues; + uint32_t options; +}; + +int dpseci_get_attributes(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + struct dpseci_attr *attr); + +/** + * enum dpseci_dest - DPSECI destination types + * @DPSECI_DEST_NONE: Unassigned destination; The queue is set in parked mode + * and does not generate FQDAN notifications; user is expected to + * dequeue from the queue based on polling or other user-defined + * method + * @DPSECI_DEST_DPIO: The queue is set in schedule mode and generates FQDAN + * notifications to the specified DPIO; user is expected to dequeue + * from the queue only after notification is received + * @DPSECI_DEST_DPCON: The queue is set in schedule mode and does not generate + * FQDAN notifications, but is connected to the specified DPCON + * object; user is expected to dequeue from the DPCON channel + */ +enum dpseci_dest { + DPSECI_DEST_NONE = 0, + DPSECI_DEST_DPIO = 1, + DPSECI_DEST_DPCON = 2 +}; + +/** + * struct dpseci_dest_cfg - Structure representing DPSECI destination parameters + * @dest_type: Destination type + * @dest_id: Either DPIO ID or DPCON ID, depending on the destination type + * @priority: Priority selection within the DPIO or DPCON channel; valid values + * are 0-1 or 0-7, depending on the number of priorities in that + * channel; not relevant for 'DPSECI_DEST_NONE' option + */ +struct dpseci_dest_cfg { + enum dpseci_dest dest_type; + int dest_id; + uint8_t priority; +}; + +/** + * DPSECI queue modification options + */ + +/** + * Select to modify the user's context associated with the queue + */ +#define DPSECI_QUEUE_OPT_USER_CTX 0x00000001 + +/** + * Select to modify the queue's destination + */ +#define DPSECI_QUEUE_OPT_DEST 0x00000002 + +/** + * Select to modify the queue's order preservation + */ +#define DPSECI_QUEUE_OPT_ORDER_PRESERVATION 0x00000004 + +/** + * struct dpseci_rx_queue_cfg - DPSECI RX queue configuration + * @options: Flags representing the suggested modifications to the queue; + * Use any combination of 'DPSECI_QUEUE_OPT_' flags + * @order_preservation_en: order preservation configuration for the rx queue + * valid only if 'DPSECI_QUEUE_OPT_ORDER_PRESERVATION' is contained in 'options' + * @user_ctx: User context value provided in the frame descriptor of each + * dequeued frame; + * valid only if 'DPSECI_QUEUE_OPT_USER_CTX' is contained in 'options' + * @dest_cfg: Queue destination parameters; + * valid only if 'DPSECI_QUEUE_OPT_DEST' is contained in 'options' + */ +struct dpseci_rx_queue_cfg { + uint32_t options; + int order_preservation_en; + uint64_t user_ctx; + struct dpseci_dest_cfg dest_cfg; +}; + +int dpseci_set_rx_queue(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t queue, + const struct dpseci_rx_queue_cfg *cfg); + +/** + * struct dpseci_rx_queue_attr - Structure representing attributes of Rx queues + * @user_ctx: User context value provided in the frame descriptor of each + * dequeued frame + * @order_preservation_en: Status of the order preservation configuration + * on the queue + * @dest_cfg: Queue destination configuration + * @fqid: Virtual FQID value to be used for dequeue operations + */ +struct dpseci_rx_queue_attr { + uint64_t user_ctx; + int order_preservation_en; + struct dpseci_dest_cfg dest_cfg; + uint32_t fqid; +}; + +int dpseci_get_rx_queue(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t queue, + struct dpseci_rx_queue_attr *attr); + +/** + * struct dpseci_tx_queue_attr - Structure representing attributes of Tx queues + * @fqid: Virtual FQID to be used for sending frames to SEC hardware + * @priority: SEC hardware processing priority for the queue + */ +struct dpseci_tx_queue_attr { + uint32_t fqid; + uint8_t priority; +}; + +int dpseci_get_tx_queue(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t queue, + struct dpseci_tx_queue_attr *attr); + +/** + * struct dpseci_sec_attr - Structure representing attributes of the SEC + * hardware accelerator + * @ip_id: ID for SEC. + * @major_rev: Major revision number for SEC. + * @minor_rev: Minor revision number for SEC. + * @era: SEC Era. + * @deco_num: The number of copies of the DECO that are implemented + * in this version of SEC. + * @zuc_auth_acc_num: The number of copies of ZUCA that are implemented + * in this version of SEC. + * @zuc_enc_acc_num: The number of copies of ZUCE that are implemented + * in this version of SEC. + * @snow_f8_acc_num: The number of copies of the SNOW-f8 module that are + * implemented in this version of SEC. + * @snow_f9_acc_num: The number of copies of the SNOW-f9 module that are + * implemented in this version of SEC. + * @crc_acc_num: The number of copies of the CRC module that are + * implemented in this version of SEC. + * @pk_acc_num: The number of copies of the Public Key module that are + * implemented in this version of SEC. + * @kasumi_acc_num: The number of copies of the Kasumi module that are + * implemented in this version of SEC. + * @rng_acc_num: The number of copies of the Random Number Generator that + * are implemented in this version of SEC. + * @md_acc_num: The number of copies of the MDHA (Hashing module) that + * are implemented in this version of SEC. + * @arc4_acc_num: The number of copies of the ARC4 module that are + * implemented in this version of SEC. + * @des_acc_num: The number of copies of the DES module that are + * implemented in this version of SEC. + * @aes_acc_num: The number of copies of the AES module that are + * implemented in this version of SEC. + **/ + +struct dpseci_sec_attr { + uint16_t ip_id; + uint8_t major_rev; + uint8_t minor_rev; + uint8_t era; + uint8_t deco_num; + uint8_t zuc_auth_acc_num; + uint8_t zuc_enc_acc_num; + uint8_t snow_f8_acc_num; + uint8_t snow_f9_acc_num; + uint8_t crc_acc_num; + uint8_t pk_acc_num; + uint8_t kasumi_acc_num; + uint8_t rng_acc_num; + uint8_t md_acc_num; + uint8_t arc4_acc_num; + uint8_t des_acc_num; + uint8_t aes_acc_num; +}; + +int dpseci_get_sec_attr(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + struct dpseci_sec_attr *attr); + +/** + * struct dpseci_sec_counters - Structure representing global SEC counters and + * not per dpseci counters + * @dequeued_requests: Number of Requests Dequeued + * @ob_enc_requests: Number of Outbound Encrypt Requests + * @ib_dec_requests: Number of Inbound Decrypt Requests + * @ob_enc_bytes: Number of Outbound Bytes Encrypted + * @ob_prot_bytes: Number of Outbound Bytes Protected + * @ib_dec_bytes: Number of Inbound Bytes Decrypted + * @ib_valid_bytes: Number of Inbound Bytes Validated + */ +struct dpseci_sec_counters { + uint64_t dequeued_requests; + uint64_t ob_enc_requests; + uint64_t ib_dec_requests; + uint64_t ob_enc_bytes; + uint64_t ob_prot_bytes; + uint64_t ib_dec_bytes; + uint64_t ib_valid_bytes; +}; + +int dpseci_get_sec_counters(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + struct dpseci_sec_counters *counters); + +int dpseci_get_api_version(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t *major_ver, + uint16_t *minor_ver); +/** + * enum dpseci_congestion_unit - DPSECI congestion units + * @DPSECI_CONGESTION_UNIT_BYTES: bytes units + * @DPSECI_CONGESTION_UNIT_FRAMES: frames units + */ +enum dpseci_congestion_unit { + DPSECI_CONGESTION_UNIT_BYTES = 0, + DPSECI_CONGESTION_UNIT_FRAMES +}; + +/** + * CSCN message is written to message_iova once entering a + * congestion state (see 'threshold_entry') + */ +#define DPSECI_CGN_MODE_WRITE_MEM_ON_ENTER 0x00000001 +/** + * CSCN message is written to message_iova once exiting a + * congestion state (see 'threshold_exit') + */ +#define DPSECI_CGN_MODE_WRITE_MEM_ON_EXIT 0x00000002 +/** + * CSCN write will attempt to allocate into a cache (coherent write); + * valid only if 'DPSECI_CGN_MODE_WRITE_MEM_' is selected + */ +#define DPSECI_CGN_MODE_COHERENT_WRITE 0x00000004 +/** + * if 'dpseci_dest_cfg.dest_type != DPSECI_DEST_NONE' CSCN message is sent to + * DPIO/DPCON's WQ channel once entering a congestion state + * (see 'threshold_entry') + */ +#define DPSECI_CGN_MODE_NOTIFY_DEST_ON_ENTER 0x00000008 +/** + * if 'dpseci_dest_cfg.dest_type != DPSECI_DEST_NONE' CSCN message is sent to + * DPIO/DPCON's WQ channel once exiting a congestion state + * (see 'threshold_exit') + */ +#define DPSECI_CGN_MODE_NOTIFY_DEST_ON_EXIT 0x00000010 +/** + * if 'dpseci_dest_cfg.dest_type != DPSECI_DEST_NONE' when the CSCN is written + * to the sw-portal's DQRR, the DQRI interrupt is asserted immediately + * (if enabled) + */ +#define DPSECI_CGN_MODE_INTR_COALESCING_DISABLED 0x00000020 + +/** + * struct dpseci_congestion_notification_cfg - congestion notification + * configuration + * @units: units type + * @threshold_entry: above this threshold we enter a congestion state. + * set it to '0' to disable it + * @threshold_exit: below this threshold we exit the congestion state. + * @message_ctx: The context that will be part of the CSCN message + * @message_iova: I/O virtual address (must be in DMA-able memory), + * must be 16B aligned; + * @dest_cfg: CSCN can be send to either DPIO or DPCON WQ channel + * @notification_mode: Mask of available options; use 'DPSECI_CGN_MODE_' + * values + */ +struct dpseci_congestion_notification_cfg { + enum dpseci_congestion_unit units; + uint32_t threshold_entry; + uint32_t threshold_exit; + uint64_t message_ctx; + uint64_t message_iova; + struct dpseci_dest_cfg dest_cfg; + uint16_t notification_mode; +}; + +int dpseci_set_congestion_notification( + struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + const struct dpseci_congestion_notification_cfg *cfg); + +int dpseci_get_congestion_notification( + struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + struct dpseci_congestion_notification_cfg *cfg); + +#endif /* __FSL_DPSECI_H */ diff --git a/dpdk/drivers/crypto/dpaa2_sec/mc/fsl_dpseci_cmd.h b/dpdk/drivers/crypto/dpaa2_sec/mc/fsl_dpseci_cmd.h new file mode 100644 index 00000000..a100a0ec --- /dev/null +++ b/dpdk/drivers/crypto/dpaa2_sec/mc/fsl_dpseci_cmd.h @@ -0,0 +1,223 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2013-2016 Freescale Semiconductor Inc. + * Copyright 2016-2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef _FSL_DPSECI_CMD_H +#define _FSL_DPSECI_CMD_H + +/* DPSECI Version */ +#define DPSECI_VER_MAJOR 5 +#define DPSECI_VER_MINOR 1 + +/* Command versioning */ +#define DPSECI_CMD_BASE_VERSION 1 +#define DPSECI_CMD_BASE_VERSION_V2 2 +#define DPSECI_CMD_ID_OFFSET 4 + +#define DPSECI_CMD_V1(id) \ + ((id << DPSECI_CMD_ID_OFFSET) | DPSECI_CMD_BASE_VERSION) +#define DPSECI_CMD_V2(id) \ + ((id << DPSECI_CMD_ID_OFFSET) | DPSECI_CMD_BASE_VERSION_V2) + +/* Command IDs */ +#define DPSECI_CMDID_CLOSE DPSECI_CMD_V1(0x800) +#define DPSECI_CMDID_OPEN DPSECI_CMD_V1(0x809) +#define DPSECI_CMDID_CREATE DPSECI_CMD_V2(0x909) +#define DPSECI_CMDID_DESTROY DPSECI_CMD_V1(0x989) +#define DPSECI_CMDID_GET_API_VERSION DPSECI_CMD_V1(0xa09) + +#define DPSECI_CMDID_ENABLE DPSECI_CMD_V1(0x002) +#define DPSECI_CMDID_DISABLE DPSECI_CMD_V1(0x003) +#define DPSECI_CMDID_GET_ATTR DPSECI_CMD_V1(0x004) +#define DPSECI_CMDID_RESET DPSECI_CMD_V1(0x005) +#define DPSECI_CMDID_IS_ENABLED DPSECI_CMD_V1(0x006) + +#define DPSECI_CMDID_SET_RX_QUEUE DPSECI_CMD_V1(0x194) +#define DPSECI_CMDID_GET_RX_QUEUE DPSECI_CMD_V1(0x196) +#define DPSECI_CMDID_GET_TX_QUEUE DPSECI_CMD_V1(0x197) +#define DPSECI_CMDID_GET_SEC_ATTR DPSECI_CMD_V1(0x198) +#define DPSECI_CMDID_GET_SEC_COUNTERS DPSECI_CMD_V1(0x199) + +#define DPSECI_CMDID_SET_CONGESTION_NOTIFICATION DPSECI_CMD_V1(0x170) +#define DPSECI_CMDID_GET_CONGESTION_NOTIFICATION DPSECI_CMD_V1(0x171) + +/* Macros for accessing command fields smaller than 1byte */ +#define DPSECI_MASK(field) \ + GENMASK(DPSECI_##field##_SHIFT + DPSECI_##field##_SIZE - 1, \ + DPSECI_##field##_SHIFT) +#define dpseci_set_field(var, field, val) \ + ((var) |= (((val) << DPSECI_##field##_SHIFT) & DPSECI_MASK(field))) +#define dpseci_get_field(var, field) \ + (((var) & DPSECI_MASK(field)) >> DPSECI_##field##_SHIFT) + +#pragma pack(push, 1) +struct dpseci_cmd_open { + uint32_t dpseci_id; +}; + +struct dpseci_cmd_create { + uint8_t priorities[8]; + uint8_t num_tx_queues; + uint8_t num_rx_queues; + uint8_t pad[6]; + uint32_t options; +}; + +struct dpseci_cmd_destroy { + uint32_t dpseci_id; +}; + +#define DPSECI_ENABLE_SHIFT 0 +#define DPSECI_ENABLE_SIZE 1 + +struct dpseci_rsp_is_enabled { + /* only the first LSB */ + uint8_t en; +}; + +struct dpseci_rsp_get_attr { + uint32_t id; + uint32_t pad; + uint8_t num_tx_queues; + uint8_t num_rx_queues; + uint8_t pad1[6]; + uint32_t options; +}; + +#define DPSECI_DEST_TYPE_SHIFT 0 +#define DPSECI_DEST_TYPE_SIZE 4 + +#define DPSECI_ORDER_PRESERVATION_SHIFT 0 +#define DPSECI_ORDER_PRESERVATION_SIZE 1 + +struct dpseci_cmd_set_rx_queue { + uint32_t dest_id; + uint8_t dest_priority; + uint8_t queue; + /* from LSB: dest_type:4 */ + uint8_t dest_type; + uint8_t pad; + uint64_t user_ctx; + uint32_t options; + /* only the LSB */ + uint8_t order_preservation_en; +}; + +struct dpseci_cmd_get_queue { + uint8_t pad[5]; + uint8_t queue; +}; + +struct dpseci_rsp_get_rx_queue { + uint32_t dest_id; + uint8_t dest_priority; + uint8_t pad1; + /* from LSB: dest_type:4 */ + uint8_t dest_type; + uint8_t pad2; + uint64_t user_ctx; + uint32_t fqid; + /* only the LSB */ + uint8_t order_preservation_en; + +}; + +struct dpseci_rsp_get_tx_queue { + uint32_t pad; + uint32_t fqid; + uint8_t priority; +}; + +struct dpseci_rsp_get_sec_attr { + uint16_t ip_id; + uint8_t major_rev; + uint8_t minor_rev; + uint8_t era; + uint8_t pad1[3]; + uint8_t deco_num; + uint8_t zuc_auth_acc_num; + uint8_t zuc_enc_acc_num; + uint8_t pad2; + uint8_t snow_f8_acc_num; + uint8_t snow_f9_acc_num; + uint8_t crc_acc_num; + uint8_t pad3; + uint8_t pk_acc_num; + uint8_t kasumi_acc_num; + uint8_t rng_acc_num; + uint8_t pad4; + uint8_t md_acc_num; + uint8_t arc4_acc_num; + uint8_t des_acc_num; + uint8_t aes_acc_num; +}; + +struct dpseci_rsp_get_sec_counters { + uint64_t dequeued_requests; + uint64_t ob_enc_requests; + uint64_t ib_dec_requests; + uint64_t ob_enc_bytes; + uint64_t ob_prot_bytes; + uint64_t ib_dec_bytes; + uint64_t ib_valid_bytes; +}; + +struct dpseci_rsp_get_api_version { + uint16_t major; + uint16_t minor; +}; + +#define DPSECI_DEST_TYPE_SHIFT 0 +#define DPSECI_DEST_TYPE_SIZE 4 +#define DPSECI_CG_UNITS_SHIFT 4 +#define DPSECI_CG_UNITS_SIZE 2 + +struct dpseci_cmd_set_congestion_notification { + uint32_t dest_id; + uint16_t notification_mode; + uint8_t dest_priority; + /* from LSB: dest_type: 4 units:2 */ + uint8_t type_units; + uint64_t message_iova; + uint64_t message_ctx; + uint32_t threshold_entry; + uint32_t threshold_exit; +}; + +#pragma pack(pop) +#endif /* _FSL_DPSECI_CMD_H */ diff --git a/dpdk/drivers/crypto/dpaa2_sec/rte_pmd_dpaa2_sec_version.map b/dpdk/drivers/crypto/dpaa2_sec/rte_pmd_dpaa2_sec_version.map new file mode 100644 index 00000000..8591cc0b --- /dev/null +++ b/dpdk/drivers/crypto/dpaa2_sec/rte_pmd_dpaa2_sec_version.map @@ -0,0 +1,4 @@ +DPDK_17.05 { + + local: *; +}; diff --git a/dpdk/drivers/crypto/dpaa_sec/Makefile b/dpdk/drivers/crypto/dpaa_sec/Makefile new file mode 100644 index 00000000..17bc79c6 --- /dev/null +++ b/dpdk/drivers/crypto/dpaa_sec/Makefile @@ -0,0 +1,73 @@ +# BSD LICENSE +# +# Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. +# Copyright 2017 NXP. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Freescale Semiconductor, Inc nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +include $(RTE_SDK)/mk/rte.vars.mk + +# +# library name +# +LIB = librte_pmd_dpaa_sec.a + +# build flags +CFLAGS += -D _GNU_SOURCE +ifeq ($(CONFIG_RTE_LIBRTE_DPAA_SEC_DEBUG_INIT),y) +CFLAGS += -O0 -g +CFLAGS += "-Wno-error" +else +CFLAGS += -O3 +CFLAGS += $(WERROR_FLAGS) +endif + +CFLAGS += -I$(RTE_SDK)/drivers/bus/dpaa +CFLAGS += -I$(RTE_SDK)/drivers/bus/dpaa/include +CFLAGS += -I$(RTE_SDK)/drivers/crypto/dpaa_sec/ +#sharing the hw flib headers from dpaa2_sec pmd +CFLAGS += -I$(RTE_SDK)/drivers/crypto/dpaa2_sec/ +CFLAGS += -I$(RTE_SDK)/lib/librte_eal/common/include +CFLAGS += -I$(RTE_SDK)/lib/librte_eal/linuxapp/eal +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_cryptodev + +# versioning export map +EXPORT_MAP := rte_pmd_dpaa_sec_version.map + +# library version +LIBABIVER := 1 + +# library source files +SRCS-$(CONFIG_RTE_LIBRTE_PMD_DPAA_SEC) += dpaa_sec.c + +# library dependencies + +LDLIBS += -lrte_bus_dpaa +LDLIBS += -lrte_mempool_dpaa + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/crypto/dpaa_sec/dpaa_sec.c b/dpdk/drivers/crypto/dpaa_sec/dpaa_sec.c new file mode 100644 index 00000000..438dd3bc --- /dev/null +++ b/dpdk/drivers/crypto/dpaa_sec/dpaa_sec.c @@ -0,0 +1,1548 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. + * Copyright 2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of NXP nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +/* RTA header files */ +#include +#include +#include + +#include +#include +#include + +enum rta_sec_era rta_sec_era; + +static uint8_t cryptodev_driver_id; + +static __thread struct rte_crypto_op **dpaa_sec_ops; +static __thread int dpaa_sec_op_nb; + +static inline void +dpaa_sec_op_ending(struct dpaa_sec_op_ctx *ctx) +{ + if (!ctx->fd_status) { + ctx->op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; + } else { + PMD_RX_LOG(ERR, "SEC return err: 0x%x", ctx->fd_status); + ctx->op->status = RTE_CRYPTO_OP_STATUS_ERROR; + } + + /* report op status to sym->op and then free the ctx memeory */ + rte_mempool_put(ctx->ctx_pool, (void *)ctx); +} + +static inline struct dpaa_sec_op_ctx * +dpaa_sec_alloc_ctx(dpaa_sec_session *ses) +{ + struct dpaa_sec_op_ctx *ctx; + int retval; + + retval = rte_mempool_get(ses->ctx_pool, (void **)(&ctx)); + if (!ctx || retval) { + PMD_TX_LOG(ERR, "Alloc sec descriptor failed!"); + return NULL; + } + /* + * Clear SG memory. There are 16 SG entries of 16 Bytes each. + * one call to dcbz_64() clear 64 bytes, hence calling it 4 times + * to clear all the SG entries. dpaa_sec_alloc_ctx() is called for + * each packet, memset is costlier than dcbz_64(). + */ + dcbz_64(&ctx->job.sg[SG_CACHELINE_0]); + dcbz_64(&ctx->job.sg[SG_CACHELINE_1]); + dcbz_64(&ctx->job.sg[SG_CACHELINE_2]); + dcbz_64(&ctx->job.sg[SG_CACHELINE_3]); + + ctx->ctx_pool = ses->ctx_pool; + + return ctx; +} + +static inline rte_iova_t +dpaa_mem_vtop(void *vaddr) +{ + const struct rte_memseg *memseg = rte_eal_get_physmem_layout(); + uint64_t vaddr_64, paddr; + int i; + + vaddr_64 = (uint64_t)vaddr; + for (i = 0; i < RTE_MAX_MEMSEG && memseg[i].addr_64 != 0; i++) { + if (vaddr_64 >= memseg[i].addr_64 && + vaddr_64 < memseg[i].addr_64 + memseg[i].len) { + paddr = memseg[i].iova + + (vaddr_64 - memseg[i].addr_64); + + return (rte_iova_t)paddr; + } + } + return (rte_iova_t)(NULL); +} + +static inline void * +dpaa_mem_ptov(rte_iova_t paddr) +{ + const struct rte_memseg *memseg = rte_eal_get_physmem_layout(); + int i; + + for (i = 0; i < RTE_MAX_MEMSEG && memseg[i].addr_64 != 0; i++) { + if (paddr >= memseg[i].iova && + (char *)paddr < (char *)memseg[i].iova + memseg[i].len) + return (void *)(memseg[i].addr_64 + + (paddr - memseg[i].iova)); + } + return NULL; +} + +static void +ern_sec_fq_handler(struct qman_portal *qm __rte_unused, + struct qman_fq *fq, + const struct qm_mr_entry *msg) +{ + RTE_LOG_DP(ERR, PMD, "sec fq %d error, RC = %x, seqnum = %x\n", + fq->fqid, msg->ern.rc, msg->ern.seqnum); +} + +/* initialize the queue with dest chan as caam chan so that + * all the packets in this queue could be dispatched into caam + */ +static int +dpaa_sec_init_rx(struct qman_fq *fq_in, rte_iova_t hwdesc, + uint32_t fqid_out) +{ + struct qm_mcc_initfq fq_opts; + uint32_t flags; + int ret = -1; + + /* Clear FQ options */ + memset(&fq_opts, 0x00, sizeof(struct qm_mcc_initfq)); + + flags = QMAN_FQ_FLAG_LOCKED | QMAN_FQ_FLAG_DYNAMIC_FQID | + QMAN_FQ_FLAG_TO_DCPORTAL; + + ret = qman_create_fq(0, flags, fq_in); + if (unlikely(ret != 0)) { + PMD_INIT_LOG(ERR, "qman_create_fq failed"); + return ret; + } + + flags = QMAN_INITFQ_FLAG_SCHED; + fq_opts.we_mask = QM_INITFQ_WE_DESTWQ | QM_INITFQ_WE_CONTEXTA | + QM_INITFQ_WE_CONTEXTB; + + qm_fqd_context_a_set64(&fq_opts.fqd, hwdesc); + fq_opts.fqd.context_b = fqid_out; + fq_opts.fqd.dest.channel = qm_channel_caam; + fq_opts.fqd.dest.wq = 0; + + fq_in->cb.ern = ern_sec_fq_handler; + + ret = qman_init_fq(fq_in, flags, &fq_opts); + if (unlikely(ret != 0)) + PMD_INIT_LOG(ERR, "qman_init_fq failed"); + + return ret; +} + +/* something is put into in_fq and caam put the crypto result into out_fq */ +static enum qman_cb_dqrr_result +dqrr_out_fq_cb_rx(struct qman_portal *qm __always_unused, + struct qman_fq *fq __always_unused, + const struct qm_dqrr_entry *dqrr) +{ + const struct qm_fd *fd; + struct dpaa_sec_job *job; + struct dpaa_sec_op_ctx *ctx; + + if (dpaa_sec_op_nb >= DPAA_SEC_BURST) + return qman_cb_dqrr_defer; + + if (!(dqrr->stat & QM_DQRR_STAT_FD_VALID)) + return qman_cb_dqrr_consume; + + fd = &dqrr->fd; + /* sg is embedded in an op ctx, + * sg[0] is for output + * sg[1] for input + */ + job = dpaa_mem_ptov(qm_fd_addr_get64(fd)); + ctx = container_of(job, struct dpaa_sec_op_ctx, job); + ctx->fd_status = fd->status; + dpaa_sec_ops[dpaa_sec_op_nb++] = ctx->op; + dpaa_sec_op_ending(ctx); + + return qman_cb_dqrr_consume; +} + +/* caam result is put into this queue */ +static int +dpaa_sec_init_tx(struct qman_fq *fq) +{ + int ret; + struct qm_mcc_initfq opts; + uint32_t flags; + + flags = QMAN_FQ_FLAG_NO_ENQUEUE | QMAN_FQ_FLAG_LOCKED | + QMAN_FQ_FLAG_DYNAMIC_FQID; + + ret = qman_create_fq(0, flags, fq); + if (unlikely(ret)) { + PMD_INIT_LOG(ERR, "qman_create_fq failed"); + return ret; + } + + memset(&opts, 0, sizeof(opts)); + opts.we_mask = QM_INITFQ_WE_DESTWQ | QM_INITFQ_WE_FQCTRL | + QM_INITFQ_WE_CONTEXTA | QM_INITFQ_WE_CONTEXTB; + + /* opts.fqd.dest.channel = dpaa_sec_pool_chan; */ + + fq->cb.dqrr = dqrr_out_fq_cb_rx; + fq->cb.ern = ern_sec_fq_handler; + + ret = qman_init_fq(fq, 0, &opts); + if (unlikely(ret)) { + PMD_INIT_LOG(ERR, "unable to init caam source fq!"); + return ret; + } + + return ret; +} + +static inline int is_cipher_only(dpaa_sec_session *ses) +{ + return ((ses->cipher_alg != RTE_CRYPTO_CIPHER_NULL) && + (ses->auth_alg == RTE_CRYPTO_AUTH_NULL)); +} + +static inline int is_auth_only(dpaa_sec_session *ses) +{ + return ((ses->cipher_alg == RTE_CRYPTO_CIPHER_NULL) && + (ses->auth_alg != RTE_CRYPTO_AUTH_NULL)); +} + +static inline int is_aead(dpaa_sec_session *ses) +{ + return ((ses->cipher_alg == 0) && + (ses->auth_alg == 0) && + (ses->aead_alg != 0)); +} + +static inline int is_auth_cipher(dpaa_sec_session *ses) +{ + return ((ses->cipher_alg != RTE_CRYPTO_CIPHER_NULL) && + (ses->auth_alg != RTE_CRYPTO_AUTH_NULL)); +} + +static inline int is_encode(dpaa_sec_session *ses) +{ + return ses->dir == DIR_ENC; +} + +static inline int is_decode(dpaa_sec_session *ses) +{ + return ses->dir == DIR_DEC; +} + +static inline void +caam_auth_alg(dpaa_sec_session *ses, struct alginfo *alginfo_a) +{ + switch (ses->auth_alg) { + case RTE_CRYPTO_AUTH_NULL: + ses->digest_length = 0; + break; + case RTE_CRYPTO_AUTH_MD5_HMAC: + alginfo_a->algtype = OP_ALG_ALGSEL_MD5; + alginfo_a->algmode = OP_ALG_AAI_HMAC; + break; + case RTE_CRYPTO_AUTH_SHA1_HMAC: + alginfo_a->algtype = OP_ALG_ALGSEL_SHA1; + alginfo_a->algmode = OP_ALG_AAI_HMAC; + break; + case RTE_CRYPTO_AUTH_SHA224_HMAC: + alginfo_a->algtype = OP_ALG_ALGSEL_SHA224; + alginfo_a->algmode = OP_ALG_AAI_HMAC; + break; + case RTE_CRYPTO_AUTH_SHA256_HMAC: + alginfo_a->algtype = OP_ALG_ALGSEL_SHA256; + alginfo_a->algmode = OP_ALG_AAI_HMAC; + break; + case RTE_CRYPTO_AUTH_SHA384_HMAC: + alginfo_a->algtype = OP_ALG_ALGSEL_SHA384; + alginfo_a->algmode = OP_ALG_AAI_HMAC; + break; + case RTE_CRYPTO_AUTH_SHA512_HMAC: + alginfo_a->algtype = OP_ALG_ALGSEL_SHA512; + alginfo_a->algmode = OP_ALG_AAI_HMAC; + break; + default: + PMD_INIT_LOG(ERR, "unsupported auth alg %u", ses->auth_alg); + } +} + +static inline void +caam_cipher_alg(dpaa_sec_session *ses, struct alginfo *alginfo_c) +{ + switch (ses->cipher_alg) { + case RTE_CRYPTO_CIPHER_NULL: + break; + case RTE_CRYPTO_CIPHER_AES_CBC: + alginfo_c->algtype = OP_ALG_ALGSEL_AES; + alginfo_c->algmode = OP_ALG_AAI_CBC; + break; + case RTE_CRYPTO_CIPHER_3DES_CBC: + alginfo_c->algtype = OP_ALG_ALGSEL_3DES; + alginfo_c->algmode = OP_ALG_AAI_CBC; + break; + case RTE_CRYPTO_CIPHER_AES_CTR: + alginfo_c->algtype = OP_ALG_ALGSEL_AES; + alginfo_c->algmode = OP_ALG_AAI_CTR; + break; + default: + PMD_INIT_LOG(ERR, "unsupported cipher alg %d", ses->cipher_alg); + } +} + +static inline void +caam_aead_alg(dpaa_sec_session *ses, struct alginfo *alginfo) +{ + switch (ses->aead_alg) { + case RTE_CRYPTO_AEAD_AES_GCM: + alginfo->algtype = OP_ALG_ALGSEL_AES; + alginfo->algmode = OP_ALG_AAI_GCM; + break; + default: + PMD_INIT_LOG(ERR, "unsupported AEAD alg %d", ses->aead_alg); + } +} + + +/* prepare command block of the session */ +static int +dpaa_sec_prep_cdb(dpaa_sec_session *ses) +{ + struct alginfo alginfo_c = {0}, alginfo_a = {0}, alginfo = {0}; + uint32_t shared_desc_len = 0; + struct sec_cdb *cdb = &ses->qp->cdb; + int err; +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + int swap = false; +#else + int swap = true; +#endif + + memset(cdb, 0, sizeof(struct sec_cdb)); + + if (is_cipher_only(ses)) { + caam_cipher_alg(ses, &alginfo_c); + if (alginfo_c.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) { + PMD_TX_LOG(ERR, "not supported cipher alg\n"); + return -ENOTSUP; + } + + alginfo_c.key = (uint64_t)ses->cipher_key.data; + alginfo_c.keylen = ses->cipher_key.length; + alginfo_c.key_enc_flags = 0; + alginfo_c.key_type = RTA_DATA_IMM; + + shared_desc_len = cnstr_shdsc_blkcipher( + cdb->sh_desc, true, + swap, &alginfo_c, + NULL, + ses->iv.length, + ses->dir); + } else if (is_auth_only(ses)) { + caam_auth_alg(ses, &alginfo_a); + if (alginfo_a.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) { + PMD_TX_LOG(ERR, "not supported auth alg\n"); + return -ENOTSUP; + } + + alginfo_a.key = (uint64_t)ses->auth_key.data; + alginfo_a.keylen = ses->auth_key.length; + alginfo_a.key_enc_flags = 0; + alginfo_a.key_type = RTA_DATA_IMM; + + shared_desc_len = cnstr_shdsc_hmac(cdb->sh_desc, true, + swap, &alginfo_a, + !ses->dir, + ses->digest_length); + } else if (is_aead(ses)) { + caam_aead_alg(ses, &alginfo); + if (alginfo.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) { + PMD_TX_LOG(ERR, "not supported aead alg\n"); + return -ENOTSUP; + } + alginfo.key = (uint64_t)ses->aead_key.data; + alginfo.keylen = ses->aead_key.length; + alginfo.key_enc_flags = 0; + alginfo.key_type = RTA_DATA_IMM; + + if (ses->dir == DIR_ENC) + shared_desc_len = cnstr_shdsc_gcm_encap( + cdb->sh_desc, true, swap, + &alginfo, + ses->iv.length, + ses->digest_length); + else + shared_desc_len = cnstr_shdsc_gcm_decap( + cdb->sh_desc, true, swap, + &alginfo, + ses->iv.length, + ses->digest_length); + } else { + caam_cipher_alg(ses, &alginfo_c); + if (alginfo_c.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) { + PMD_TX_LOG(ERR, "not supported cipher alg\n"); + return -ENOTSUP; + } + + alginfo_c.key = (uint64_t)ses->cipher_key.data; + alginfo_c.keylen = ses->cipher_key.length; + alginfo_c.key_enc_flags = 0; + alginfo_c.key_type = RTA_DATA_IMM; + + caam_auth_alg(ses, &alginfo_a); + if (alginfo_a.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) { + PMD_TX_LOG(ERR, "not supported auth alg\n"); + return -ENOTSUP; + } + + alginfo_a.key = (uint64_t)ses->auth_key.data; + alginfo_a.keylen = ses->auth_key.length; + alginfo_a.key_enc_flags = 0; + alginfo_a.key_type = RTA_DATA_IMM; + + cdb->sh_desc[0] = alginfo_c.keylen; + cdb->sh_desc[1] = alginfo_a.keylen; + err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN, + MIN_JOB_DESC_SIZE, + (unsigned int *)cdb->sh_desc, + &cdb->sh_desc[2], 2); + + if (err < 0) { + PMD_TX_LOG(ERR, "Crypto: Incorrect key lengths"); + return err; + } + if (cdb->sh_desc[2] & 1) + alginfo_c.key_type = RTA_DATA_IMM; + else { + alginfo_c.key = (uint64_t)dpaa_mem_vtop( + (void *)alginfo_c.key); + alginfo_c.key_type = RTA_DATA_PTR; + } + if (cdb->sh_desc[2] & (1<<1)) + alginfo_a.key_type = RTA_DATA_IMM; + else { + alginfo_a.key = (uint64_t)dpaa_mem_vtop( + (void *)alginfo_a.key); + alginfo_a.key_type = RTA_DATA_PTR; + } + cdb->sh_desc[0] = 0; + cdb->sh_desc[1] = 0; + cdb->sh_desc[2] = 0; + + /* Auth_only_len is set as 0 here and it will be overwritten + * in fd for each packet. + */ + shared_desc_len = cnstr_shdsc_authenc(cdb->sh_desc, + true, swap, &alginfo_c, &alginfo_a, + ses->iv.length, 0, + ses->digest_length, ses->dir); + } + cdb->sh_hdr.hi.field.idlen = shared_desc_len; + cdb->sh_hdr.hi.word = rte_cpu_to_be_32(cdb->sh_hdr.hi.word); + cdb->sh_hdr.lo.word = rte_cpu_to_be_32(cdb->sh_hdr.lo.word); + + return 0; +} + +static inline unsigned int +dpaa_volatile_deq(struct qman_fq *fq, unsigned int len, bool exact) +{ + unsigned int pkts = 0; + int ret; + struct qm_mcr_queryfq_np np; + enum qman_fq_state state; + uint32_t flags; + uint32_t vdqcr; + + qman_query_fq_np(fq, &np); + if (np.frm_cnt) { + vdqcr = QM_VDQCR_NUMFRAMES_SET(len); + if (exact) + vdqcr |= QM_VDQCR_EXACT; + ret = qman_volatile_dequeue(fq, 0, vdqcr); + if (ret) + return 0; + do { + pkts += qman_poll_dqrr(len); + qman_fq_state(fq, &state, &flags); + } while (flags & QMAN_FQ_STATE_VDQCR); + } + return pkts; +} + +/* qp is lockless, should be accessed by only one thread */ +static int +dpaa_sec_deq(struct dpaa_sec_qp *qp, struct rte_crypto_op **ops, int nb_ops) +{ + struct qman_fq *fq; + + fq = &qp->outq; + dpaa_sec_op_nb = 0; + dpaa_sec_ops = ops; + + if (unlikely(nb_ops > DPAA_SEC_BURST)) + nb_ops = DPAA_SEC_BURST; + + return dpaa_volatile_deq(fq, nb_ops, 1); +} + +/** + * packet looks like: + * |<----data_len------->| + * |ip_header|ah_header|icv|payload| + * ^ + * | + * mbuf->pkt.data + */ +static inline struct dpaa_sec_job * +build_auth_only(struct rte_crypto_op *op, dpaa_sec_session *ses) +{ + struct rte_crypto_sym_op *sym = op->sym; + struct rte_mbuf *mbuf = sym->m_src; + struct dpaa_sec_job *cf; + struct dpaa_sec_op_ctx *ctx; + struct qm_sg_entry *sg; + rte_iova_t start_addr; + uint8_t *old_digest; + + ctx = dpaa_sec_alloc_ctx(ses); + if (!ctx) + return NULL; + + cf = &ctx->job; + ctx->op = op; + old_digest = ctx->digest; + + start_addr = rte_pktmbuf_iova(mbuf); + /* output */ + sg = &cf->sg[0]; + qm_sg_entry_set64(sg, sym->auth.digest.phys_addr); + sg->length = ses->digest_length; + cpu_to_hw_sg(sg); + + /* input */ + sg = &cf->sg[1]; + if (is_decode(ses)) { + /* need to extend the input to a compound frame */ + sg->extension = 1; + qm_sg_entry_set64(sg, dpaa_mem_vtop(&cf->sg[2])); + sg->length = sym->auth.data.length + ses->digest_length; + sg->final = 1; + cpu_to_hw_sg(sg); + + sg = &cf->sg[2]; + /* hash result or digest, save digest first */ + rte_memcpy(old_digest, sym->auth.digest.data, + ses->digest_length); + qm_sg_entry_set64(sg, start_addr + sym->auth.data.offset); + sg->length = sym->auth.data.length; + cpu_to_hw_sg(sg); + + /* let's check digest by hw */ + start_addr = dpaa_mem_vtop(old_digest); + sg++; + qm_sg_entry_set64(sg, start_addr); + sg->length = ses->digest_length; + sg->final = 1; + cpu_to_hw_sg(sg); + } else { + qm_sg_entry_set64(sg, start_addr + sym->auth.data.offset); + sg->length = sym->auth.data.length; + sg->final = 1; + cpu_to_hw_sg(sg); + } + + return cf; +} + +static inline struct dpaa_sec_job * +build_cipher_only(struct rte_crypto_op *op, dpaa_sec_session *ses) +{ + struct rte_crypto_sym_op *sym = op->sym; + struct dpaa_sec_job *cf; + struct dpaa_sec_op_ctx *ctx; + struct qm_sg_entry *sg; + rte_iova_t src_start_addr, dst_start_addr; + uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, + ses->iv.offset); + + ctx = dpaa_sec_alloc_ctx(ses); + if (!ctx) + return NULL; + + cf = &ctx->job; + ctx->op = op; + + src_start_addr = rte_pktmbuf_iova(sym->m_src); + + if (sym->m_dst) + dst_start_addr = rte_pktmbuf_iova(sym->m_dst); + else + dst_start_addr = src_start_addr; + + /* output */ + sg = &cf->sg[0]; + qm_sg_entry_set64(sg, dst_start_addr + sym->cipher.data.offset); + sg->length = sym->cipher.data.length + ses->iv.length; + cpu_to_hw_sg(sg); + + /* input */ + sg = &cf->sg[1]; + + /* need to extend the input to a compound frame */ + sg->extension = 1; + sg->final = 1; + sg->length = sym->cipher.data.length + ses->iv.length; + qm_sg_entry_set64(sg, dpaa_mem_vtop(&cf->sg[2])); + cpu_to_hw_sg(sg); + + sg = &cf->sg[2]; + qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr)); + sg->length = ses->iv.length; + cpu_to_hw_sg(sg); + + sg++; + qm_sg_entry_set64(sg, src_start_addr + sym->cipher.data.offset); + sg->length = sym->cipher.data.length; + sg->final = 1; + cpu_to_hw_sg(sg); + + return cf; +} + +static inline struct dpaa_sec_job * +build_cipher_auth_gcm(struct rte_crypto_op *op, dpaa_sec_session *ses) +{ + struct rte_crypto_sym_op *sym = op->sym; + struct dpaa_sec_job *cf; + struct dpaa_sec_op_ctx *ctx; + struct qm_sg_entry *sg; + uint32_t length = 0; + rte_iova_t src_start_addr, dst_start_addr; + uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, + ses->iv.offset); + + src_start_addr = sym->m_src->buf_iova + sym->m_src->data_off; + + if (sym->m_dst) + dst_start_addr = sym->m_dst->buf_iova + sym->m_dst->data_off; + else + dst_start_addr = src_start_addr; + + ctx = dpaa_sec_alloc_ctx(ses); + if (!ctx) + return NULL; + + cf = &ctx->job; + ctx->op = op; + + /* input */ + rte_prefetch0(cf->sg); + sg = &cf->sg[2]; + qm_sg_entry_set64(&cf->sg[1], dpaa_mem_vtop(sg)); + if (is_encode(ses)) { + qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr)); + sg->length = ses->iv.length; + length += sg->length; + cpu_to_hw_sg(sg); + + sg++; + if (ses->auth_only_len) { + qm_sg_entry_set64(sg, + dpaa_mem_vtop(sym->aead.aad.data)); + sg->length = ses->auth_only_len; + length += sg->length; + cpu_to_hw_sg(sg); + sg++; + } + qm_sg_entry_set64(sg, src_start_addr + sym->aead.data.offset); + sg->length = sym->aead.data.length; + length += sg->length; + sg->final = 1; + cpu_to_hw_sg(sg); + } else { + qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr)); + sg->length = ses->iv.length; + length += sg->length; + cpu_to_hw_sg(sg); + + sg++; + if (ses->auth_only_len) { + qm_sg_entry_set64(sg, + dpaa_mem_vtop(sym->aead.aad.data)); + sg->length = ses->auth_only_len; + length += sg->length; + cpu_to_hw_sg(sg); + sg++; + } + qm_sg_entry_set64(sg, src_start_addr + sym->aead.data.offset); + sg->length = sym->aead.data.length; + length += sg->length; + cpu_to_hw_sg(sg); + + memcpy(ctx->digest, sym->aead.digest.data, + ses->digest_length); + sg++; + + qm_sg_entry_set64(sg, dpaa_mem_vtop(ctx->digest)); + sg->length = ses->digest_length; + length += sg->length; + sg->final = 1; + cpu_to_hw_sg(sg); + } + /* input compound frame */ + cf->sg[1].length = length; + cf->sg[1].extension = 1; + cf->sg[1].final = 1; + cpu_to_hw_sg(&cf->sg[1]); + + /* output */ + sg++; + qm_sg_entry_set64(&cf->sg[0], dpaa_mem_vtop(sg)); + qm_sg_entry_set64(sg, + dst_start_addr + sym->aead.data.offset - ses->auth_only_len); + sg->length = sym->aead.data.length + ses->auth_only_len; + length = sg->length; + if (is_encode(ses)) { + cpu_to_hw_sg(sg); + /* set auth output */ + sg++; + qm_sg_entry_set64(sg, sym->aead.digest.phys_addr); + sg->length = ses->digest_length; + length += sg->length; + } + sg->final = 1; + cpu_to_hw_sg(sg); + + /* output compound frame */ + cf->sg[0].length = length; + cf->sg[0].extension = 1; + cpu_to_hw_sg(&cf->sg[0]); + + return cf; +} + +static inline struct dpaa_sec_job * +build_cipher_auth(struct rte_crypto_op *op, dpaa_sec_session *ses) +{ + struct rte_crypto_sym_op *sym = op->sym; + struct dpaa_sec_job *cf; + struct dpaa_sec_op_ctx *ctx; + struct qm_sg_entry *sg; + rte_iova_t src_start_addr, dst_start_addr; + uint32_t length = 0; + uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, + ses->iv.offset); + + src_start_addr = sym->m_src->buf_iova + sym->m_src->data_off; + if (sym->m_dst) + dst_start_addr = sym->m_dst->buf_iova + sym->m_dst->data_off; + else + dst_start_addr = src_start_addr; + + ctx = dpaa_sec_alloc_ctx(ses); + if (!ctx) + return NULL; + + cf = &ctx->job; + ctx->op = op; + + /* input */ + rte_prefetch0(cf->sg); + sg = &cf->sg[2]; + qm_sg_entry_set64(&cf->sg[1], dpaa_mem_vtop(sg)); + if (is_encode(ses)) { + qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr)); + sg->length = ses->iv.length; + length += sg->length; + cpu_to_hw_sg(sg); + + sg++; + qm_sg_entry_set64(sg, src_start_addr + sym->auth.data.offset); + sg->length = sym->auth.data.length; + length += sg->length; + sg->final = 1; + cpu_to_hw_sg(sg); + } else { + qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr)); + sg->length = ses->iv.length; + length += sg->length; + cpu_to_hw_sg(sg); + + sg++; + + qm_sg_entry_set64(sg, src_start_addr + sym->auth.data.offset); + sg->length = sym->auth.data.length; + length += sg->length; + cpu_to_hw_sg(sg); + + memcpy(ctx->digest, sym->auth.digest.data, + ses->digest_length); + sg++; + + qm_sg_entry_set64(sg, dpaa_mem_vtop(ctx->digest)); + sg->length = ses->digest_length; + length += sg->length; + sg->final = 1; + cpu_to_hw_sg(sg); + } + /* input compound frame */ + cf->sg[1].length = length; + cf->sg[1].extension = 1; + cf->sg[1].final = 1; + cpu_to_hw_sg(&cf->sg[1]); + + /* output */ + sg++; + qm_sg_entry_set64(&cf->sg[0], dpaa_mem_vtop(sg)); + qm_sg_entry_set64(sg, dst_start_addr + sym->cipher.data.offset); + sg->length = sym->cipher.data.length; + length = sg->length; + if (is_encode(ses)) { + cpu_to_hw_sg(sg); + /* set auth output */ + sg++; + qm_sg_entry_set64(sg, sym->auth.digest.phys_addr); + sg->length = ses->digest_length; + length += sg->length; + } + sg->final = 1; + cpu_to_hw_sg(sg); + + /* output compound frame */ + cf->sg[0].length = length; + cf->sg[0].extension = 1; + cpu_to_hw_sg(&cf->sg[0]); + + return cf; +} + +static int +dpaa_sec_enqueue_op(struct rte_crypto_op *op, struct dpaa_sec_qp *qp) +{ + struct dpaa_sec_job *cf; + dpaa_sec_session *ses; + struct qm_fd fd; + int ret; + uint32_t auth_only_len = op->sym->auth.data.length - + op->sym->cipher.data.length; + + ses = (dpaa_sec_session *)get_session_private_data(op->sym->session, + cryptodev_driver_id); + + if (unlikely(!qp->ses || qp->ses != ses)) { + qp->ses = ses; + ses->qp = qp; + ret = dpaa_sec_prep_cdb(ses); + if (ret) + return ret; + } + + /* + * Segmented buffer is not supported. + */ + if (!rte_pktmbuf_is_contiguous(op->sym->m_src)) { + op->status = RTE_CRYPTO_OP_STATUS_ERROR; + return -ENOTSUP; + } + if (is_auth_only(ses)) { + cf = build_auth_only(op, ses); + } else if (is_cipher_only(ses)) { + cf = build_cipher_only(op, ses); + } else if (is_aead(ses)) { + cf = build_cipher_auth_gcm(op, ses); + auth_only_len = ses->auth_only_len; + } else if (is_auth_cipher(ses)) { + cf = build_cipher_auth(op, ses); + } else { + PMD_TX_LOG(ERR, "not supported sec op"); + return -ENOTSUP; + } + if (unlikely(!cf)) + return -ENOMEM; + + memset(&fd, 0, sizeof(struct qm_fd)); + qm_fd_addr_set64(&fd, dpaa_mem_vtop(cf->sg)); + fd._format1 = qm_fd_compound; + fd.length29 = 2 * sizeof(struct qm_sg_entry); + /* Auth_only_len is set as 0 in descriptor and it is overwritten + * here in the fd.cmd which will update the DPOVRD reg. + */ + if (auth_only_len) + fd.cmd = 0x80000000 | auth_only_len; + do { + ret = qman_enqueue(&qp->inq, &fd, 0); + } while (ret != 0); + + return 0; +} + +static uint16_t +dpaa_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops, + uint16_t nb_ops) +{ + /* Function to transmit the frames to given device and queuepair */ + uint32_t loop; + int32_t ret; + struct dpaa_sec_qp *dpaa_qp = (struct dpaa_sec_qp *)qp; + uint16_t num_tx = 0; + + if (unlikely(nb_ops == 0)) + return 0; + + /*Prepare each packet which is to be sent*/ + for (loop = 0; loop < nb_ops; loop++) { + if (ops[loop]->sess_type != RTE_CRYPTO_OP_WITH_SESSION) { + PMD_TX_LOG(ERR, "sessionless crypto op not supported"); + return 0; + } + ret = dpaa_sec_enqueue_op(ops[loop], dpaa_qp); + if (!ret) + num_tx++; + } + dpaa_qp->tx_pkts += num_tx; + dpaa_qp->tx_errs += nb_ops - num_tx; + + return num_tx; +} + +static uint16_t +dpaa_sec_dequeue_burst(void *qp, struct rte_crypto_op **ops, + uint16_t nb_ops) +{ + uint16_t num_rx; + struct dpaa_sec_qp *dpaa_qp = (struct dpaa_sec_qp *)qp; + + num_rx = dpaa_sec_deq(dpaa_qp, ops, nb_ops); + + dpaa_qp->rx_pkts += num_rx; + dpaa_qp->rx_errs += nb_ops - num_rx; + + PMD_RX_LOG(DEBUG, "SEC Received %d Packets\n", num_rx); + + return num_rx; +} + +/** Release queue pair */ +static int +dpaa_sec_queue_pair_release(struct rte_cryptodev *dev, + uint16_t qp_id) +{ + struct dpaa_sec_dev_private *internals; + struct dpaa_sec_qp *qp = NULL; + + PMD_INIT_FUNC_TRACE(); + + PMD_INIT_LOG(DEBUG, "dev =%p, queue =%d", dev, qp_id); + + internals = dev->data->dev_private; + if (qp_id >= internals->max_nb_queue_pairs) { + PMD_INIT_LOG(ERR, "Max supported qpid %d", + internals->max_nb_queue_pairs); + return -EINVAL; + } + + qp = &internals->qps[qp_id]; + qp->internals = NULL; + dev->data->queue_pairs[qp_id] = NULL; + + return 0; +} + +/** Setup a queue pair */ +static int +dpaa_sec_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id, + __rte_unused const struct rte_cryptodev_qp_conf *qp_conf, + __rte_unused int socket_id, + __rte_unused struct rte_mempool *session_pool) +{ + struct dpaa_sec_dev_private *internals; + struct dpaa_sec_qp *qp = NULL; + + PMD_INIT_LOG(DEBUG, "dev =%p, queue =%d, conf =%p", + dev, qp_id, qp_conf); + + internals = dev->data->dev_private; + if (qp_id >= internals->max_nb_queue_pairs) { + PMD_INIT_LOG(ERR, "Max supported qpid %d", + internals->max_nb_queue_pairs); + return -EINVAL; + } + + qp = &internals->qps[qp_id]; + qp->internals = internals; + dev->data->queue_pairs[qp_id] = qp; + + return 0; +} + +/** Start queue pair */ +static int +dpaa_sec_queue_pair_start(__rte_unused struct rte_cryptodev *dev, + __rte_unused uint16_t queue_pair_id) +{ + PMD_INIT_FUNC_TRACE(); + + return 0; +} + +/** Stop queue pair */ +static int +dpaa_sec_queue_pair_stop(__rte_unused struct rte_cryptodev *dev, + __rte_unused uint16_t queue_pair_id) +{ + PMD_INIT_FUNC_TRACE(); + + return 0; +} + +/** Return the number of allocated queue pairs */ +static uint32_t +dpaa_sec_queue_pair_count(struct rte_cryptodev *dev) +{ + PMD_INIT_FUNC_TRACE(); + + return dev->data->nb_queue_pairs; +} + +/** Returns the size of session structure */ +static unsigned int +dpaa_sec_session_get_size(struct rte_cryptodev *dev __rte_unused) +{ + PMD_INIT_FUNC_TRACE(); + + return sizeof(dpaa_sec_session); +} + +static int +dpaa_sec_cipher_init(struct rte_cryptodev *dev __rte_unused, + struct rte_crypto_sym_xform *xform, + dpaa_sec_session *session) +{ + session->cipher_alg = xform->cipher.algo; + session->iv.length = xform->cipher.iv.length; + session->iv.offset = xform->cipher.iv.offset; + session->cipher_key.data = rte_zmalloc(NULL, xform->cipher.key.length, + RTE_CACHE_LINE_SIZE); + if (session->cipher_key.data == NULL && xform->cipher.key.length > 0) { + PMD_INIT_LOG(ERR, "No Memory for cipher key\n"); + return -ENOMEM; + } + session->cipher_key.length = xform->cipher.key.length; + + memcpy(session->cipher_key.data, xform->cipher.key.data, + xform->cipher.key.length); + session->dir = (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ? + DIR_ENC : DIR_DEC; + + return 0; +} + +static int +dpaa_sec_auth_init(struct rte_cryptodev *dev __rte_unused, + struct rte_crypto_sym_xform *xform, + dpaa_sec_session *session) +{ + session->auth_alg = xform->auth.algo; + session->auth_key.data = rte_zmalloc(NULL, xform->auth.key.length, + RTE_CACHE_LINE_SIZE); + if (session->auth_key.data == NULL && xform->auth.key.length > 0) { + PMD_INIT_LOG(ERR, "No Memory for auth key\n"); + return -ENOMEM; + } + session->auth_key.length = xform->auth.key.length; + session->digest_length = xform->auth.digest_length; + + memcpy(session->auth_key.data, xform->auth.key.data, + xform->auth.key.length); + session->dir = (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) ? + DIR_ENC : DIR_DEC; + + return 0; +} + +static int +dpaa_sec_aead_init(struct rte_cryptodev *dev __rte_unused, + struct rte_crypto_sym_xform *xform, + dpaa_sec_session *session) +{ + session->aead_alg = xform->aead.algo; + session->iv.length = xform->aead.iv.length; + session->iv.offset = xform->aead.iv.offset; + session->auth_only_len = xform->aead.aad_length; + session->aead_key.data = rte_zmalloc(NULL, xform->aead.key.length, + RTE_CACHE_LINE_SIZE); + if (session->aead_key.data == NULL && xform->aead.key.length > 0) { + PMD_INIT_LOG(ERR, "No Memory for aead key\n"); + return -ENOMEM; + } + session->aead_key.length = xform->aead.key.length; + session->digest_length = xform->aead.digest_length; + + memcpy(session->aead_key.data, xform->aead.key.data, + xform->aead.key.length); + session->dir = (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) ? + DIR_ENC : DIR_DEC; + + return 0; +} + +static int +dpaa_sec_qp_attach_sess(struct rte_cryptodev *dev, uint16_t qp_id, void *ses) +{ + dpaa_sec_session *sess = ses; + struct dpaa_sec_qp *qp; + + PMD_INIT_FUNC_TRACE(); + + qp = dev->data->queue_pairs[qp_id]; + if (qp->ses != NULL) { + PMD_INIT_LOG(ERR, "qp in-use by another session\n"); + return -EBUSY; + } + + qp->ses = sess; + sess->qp = qp; + + return dpaa_sec_prep_cdb(sess); +} + +static int +dpaa_sec_qp_detach_sess(struct rte_cryptodev *dev, uint16_t qp_id, void *ses) +{ + dpaa_sec_session *sess = ses; + struct dpaa_sec_qp *qp; + + PMD_INIT_FUNC_TRACE(); + + qp = dev->data->queue_pairs[qp_id]; + if (qp->ses != NULL) { + qp->ses = NULL; + sess->qp = NULL; + return 0; + } + + PMD_DRV_LOG(ERR, "No session attached to qp"); + return -EINVAL; +} + +static int +dpaa_sec_set_session_parameters(struct rte_cryptodev *dev, + struct rte_crypto_sym_xform *xform, void *sess) +{ + struct dpaa_sec_dev_private *internals = dev->data->dev_private; + dpaa_sec_session *session = sess; + + PMD_INIT_FUNC_TRACE(); + + if (unlikely(sess == NULL)) { + RTE_LOG(ERR, PMD, "invalid session struct\n"); + return -EINVAL; + } + + /* Default IV length = 0 */ + session->iv.length = 0; + + /* Cipher Only */ + if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && xform->next == NULL) { + session->auth_alg = RTE_CRYPTO_AUTH_NULL; + dpaa_sec_cipher_init(dev, xform, session); + + /* Authentication Only */ + } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH && + xform->next == NULL) { + session->cipher_alg = RTE_CRYPTO_CIPHER_NULL; + dpaa_sec_auth_init(dev, xform, session); + + /* Cipher then Authenticate */ + } else if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && + xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) { + if (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { + dpaa_sec_cipher_init(dev, xform, session); + dpaa_sec_auth_init(dev, xform->next, session); + } else { + PMD_DRV_LOG(ERR, "Not supported: Auth then Cipher"); + return -EINVAL; + } + + /* Authenticate then Cipher */ + } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH && + xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { + if (xform->next->cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT) { + dpaa_sec_auth_init(dev, xform, session); + dpaa_sec_cipher_init(dev, xform->next, session); + } else { + PMD_DRV_LOG(ERR, "Not supported: Auth then Cipher"); + return -EINVAL; + } + + /* AEAD operation for AES-GCM kind of Algorithms */ + } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD && + xform->next == NULL) { + dpaa_sec_aead_init(dev, xform, session); + + } else { + PMD_DRV_LOG(ERR, "Invalid crypto type"); + return -EINVAL; + } + session->ctx_pool = internals->ctx_pool; + + return 0; +} + +static int +dpaa_sec_session_configure(struct rte_cryptodev *dev, + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool) +{ + void *sess_private_data; + int ret; + + PMD_INIT_FUNC_TRACE(); + + if (rte_mempool_get(mempool, &sess_private_data)) { + CDEV_LOG_ERR( + "Couldn't get object from session mempool"); + return -ENOMEM; + } + + ret = dpaa_sec_set_session_parameters(dev, xform, sess_private_data); + if (ret != 0) { + PMD_DRV_LOG(ERR, "DPAA PMD: failed to configure " + "session parameters"); + + /* Return session to mempool */ + rte_mempool_put(mempool, sess_private_data); + return ret; + } + + set_session_private_data(sess, dev->driver_id, + sess_private_data); + + return 0; +} + +/** Clear the memory of session so it doesn't leave key material behind */ +static void +dpaa_sec_session_clear(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) +{ + PMD_INIT_FUNC_TRACE(); + uint8_t index = dev->driver_id; + void *sess_priv = get_session_private_data(sess, index); + dpaa_sec_session *s = (dpaa_sec_session *)sess_priv; + + if (sess_priv) { + rte_free(s->cipher_key.data); + rte_free(s->auth_key.data); + memset(s, 0, sizeof(dpaa_sec_session)); + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + set_session_private_data(sess, index, NULL); + rte_mempool_put(sess_mp, sess_priv); + } +} + +static int +dpaa_sec_dev_configure(struct rte_cryptodev *dev __rte_unused, + struct rte_cryptodev_config *config __rte_unused) +{ + PMD_INIT_FUNC_TRACE(); + + return 0; +} + +static int +dpaa_sec_dev_start(struct rte_cryptodev *dev __rte_unused) +{ + PMD_INIT_FUNC_TRACE(); + return 0; +} + +static void +dpaa_sec_dev_stop(struct rte_cryptodev *dev __rte_unused) +{ + PMD_INIT_FUNC_TRACE(); +} + +static int +dpaa_sec_dev_close(struct rte_cryptodev *dev __rte_unused) +{ + PMD_INIT_FUNC_TRACE(); + return 0; +} + +static void +dpaa_sec_dev_infos_get(struct rte_cryptodev *dev, + struct rte_cryptodev_info *info) +{ + struct dpaa_sec_dev_private *internals = dev->data->dev_private; + + PMD_INIT_FUNC_TRACE(); + if (info != NULL) { + info->max_nb_queue_pairs = internals->max_nb_queue_pairs; + info->feature_flags = dev->feature_flags; + info->capabilities = dpaa_sec_capabilities; + info->sym.max_nb_sessions = internals->max_nb_sessions; + info->sym.max_nb_sessions_per_qp = + RTE_DPAA_SEC_PMD_MAX_NB_SESSIONS / RTE_MAX_NB_SEC_QPS; + info->driver_id = cryptodev_driver_id; + } +} + +static struct rte_cryptodev_ops crypto_ops = { + .dev_configure = dpaa_sec_dev_configure, + .dev_start = dpaa_sec_dev_start, + .dev_stop = dpaa_sec_dev_stop, + .dev_close = dpaa_sec_dev_close, + .dev_infos_get = dpaa_sec_dev_infos_get, + .queue_pair_setup = dpaa_sec_queue_pair_setup, + .queue_pair_release = dpaa_sec_queue_pair_release, + .queue_pair_start = dpaa_sec_queue_pair_start, + .queue_pair_stop = dpaa_sec_queue_pair_stop, + .queue_pair_count = dpaa_sec_queue_pair_count, + .session_get_size = dpaa_sec_session_get_size, + .session_configure = dpaa_sec_session_configure, + .session_clear = dpaa_sec_session_clear, + .qp_attach_session = dpaa_sec_qp_attach_sess, + .qp_detach_session = dpaa_sec_qp_detach_sess, +}; + +static int +dpaa_sec_uninit(struct rte_cryptodev *dev) +{ + struct dpaa_sec_dev_private *internals = dev->data->dev_private; + + if (dev == NULL) + return -ENODEV; + + rte_mempool_free(internals->ctx_pool); + rte_free(internals); + + PMD_INIT_LOG(INFO, "Closing DPAA_SEC device %s on numa socket %u\n", + dev->data->name, rte_socket_id()); + + return 0; +} + +static int +dpaa_sec_dev_init(struct rte_cryptodev *cryptodev) +{ + struct dpaa_sec_dev_private *internals; + struct dpaa_sec_qp *qp; + uint32_t i; + int ret; + char str[20]; + + PMD_INIT_FUNC_TRACE(); + + cryptodev->driver_id = cryptodev_driver_id; + cryptodev->dev_ops = &crypto_ops; + + cryptodev->enqueue_burst = dpaa_sec_enqueue_burst; + cryptodev->dequeue_burst = dpaa_sec_dequeue_burst; + cryptodev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | + RTE_CRYPTODEV_FF_HW_ACCELERATED | + RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING; + + internals = cryptodev->data->dev_private; + internals->max_nb_queue_pairs = RTE_MAX_NB_SEC_QPS; + internals->max_nb_sessions = RTE_DPAA_SEC_PMD_MAX_NB_SESSIONS; + + for (i = 0; i < internals->max_nb_queue_pairs; i++) { + /* init qman fq for queue pair */ + qp = &internals->qps[i]; + ret = dpaa_sec_init_tx(&qp->outq); + if (ret) { + PMD_INIT_LOG(ERR, "config tx of queue pair %d", i); + goto init_error; + } + ret = dpaa_sec_init_rx(&qp->inq, dpaa_mem_vtop(&qp->cdb), + qman_fq_fqid(&qp->outq)); + if (ret) { + PMD_INIT_LOG(ERR, "config rx of queue pair %d", i); + goto init_error; + } + } + + sprintf(str, "ctx_pool_%d", cryptodev->data->dev_id); + internals->ctx_pool = rte_mempool_create((const char *)str, + CTX_POOL_NUM_BUFS, + CTX_POOL_BUF_SIZE, + CTX_POOL_CACHE_SIZE, 0, + NULL, NULL, NULL, NULL, + SOCKET_ID_ANY, 0); + if (!internals->ctx_pool) { + RTE_LOG(ERR, PMD, "%s create failed\n", str); + goto init_error; + } + + PMD_INIT_LOG(DEBUG, "driver %s: created\n", cryptodev->data->name); + return 0; + +init_error: + PMD_INIT_LOG(ERR, "driver %s: create failed\n", cryptodev->data->name); + + dpaa_sec_uninit(cryptodev); + return -EFAULT; +} + +static int +cryptodev_dpaa_sec_probe(struct rte_dpaa_driver *dpaa_drv, + struct rte_dpaa_device *dpaa_dev) +{ + struct rte_cryptodev *cryptodev; + char cryptodev_name[RTE_CRYPTODEV_NAME_MAX_LEN]; + + int retval; + + sprintf(cryptodev_name, "dpaa_sec-%d", dpaa_dev->id.dev_id); + + cryptodev = rte_cryptodev_pmd_allocate(cryptodev_name, rte_socket_id()); + if (cryptodev == NULL) + return -ENOMEM; + + if (rte_eal_process_type() == RTE_PROC_PRIMARY) { + cryptodev->data->dev_private = rte_zmalloc_socket( + "cryptodev private structure", + sizeof(struct dpaa_sec_dev_private), + RTE_CACHE_LINE_SIZE, + rte_socket_id()); + + if (cryptodev->data->dev_private == NULL) + rte_panic("Cannot allocate memzone for private " + "device data"); + } + + dpaa_dev->crypto_dev = cryptodev; + cryptodev->device = &dpaa_dev->device; + cryptodev->device->driver = &dpaa_drv->driver; + + /* init user callbacks */ + TAILQ_INIT(&(cryptodev->link_intr_cbs)); + + /* if sec device version is not configured */ + if (!rta_get_sec_era()) { + const struct device_node *caam_node; + + for_each_compatible_node(caam_node, NULL, "fsl,sec-v4.0") { + const uint32_t *prop = of_get_property(caam_node, + "fsl,sec-era", + NULL); + if (prop) { + rta_set_sec_era( + INTL_SEC_ERA(rte_cpu_to_be_32(*prop))); + break; + } + } + } + + /* Invoke PMD device initialization function */ + retval = dpaa_sec_dev_init(cryptodev); + if (retval == 0) + return 0; + + /* In case of error, cleanup is done */ + if (rte_eal_process_type() == RTE_PROC_PRIMARY) + rte_free(cryptodev->data->dev_private); + + rte_cryptodev_pmd_release_device(cryptodev); + + return -ENXIO; +} + +static int +cryptodev_dpaa_sec_remove(struct rte_dpaa_device *dpaa_dev) +{ + struct rte_cryptodev *cryptodev; + int ret; + + cryptodev = dpaa_dev->crypto_dev; + if (cryptodev == NULL) + return -ENODEV; + + ret = dpaa_sec_uninit(cryptodev); + if (ret) + return ret; + + return rte_cryptodev_pmd_destroy(cryptodev); +} + +static struct rte_dpaa_driver rte_dpaa_sec_driver = { + .drv_type = FSL_DPAA_CRYPTO, + .driver = { + .name = "DPAA SEC PMD" + }, + .probe = cryptodev_dpaa_sec_probe, + .remove = cryptodev_dpaa_sec_remove, +}; + +static struct cryptodev_driver dpaa_sec_crypto_drv; + +RTE_PMD_REGISTER_DPAA(CRYPTODEV_NAME_DPAA_SEC_PMD, rte_dpaa_sec_driver); +RTE_PMD_REGISTER_CRYPTO_DRIVER(dpaa_sec_crypto_drv, rte_dpaa_sec_driver, + cryptodev_driver_id); diff --git a/dpdk/drivers/crypto/dpaa_sec/dpaa_sec.h b/dpdk/drivers/crypto/dpaa_sec/dpaa_sec.h new file mode 100644 index 00000000..af3f2550 --- /dev/null +++ b/dpdk/drivers/crypto/dpaa_sec/dpaa_sec.h @@ -0,0 +1,402 @@ +/*- + * BSD LICENSE + * + * Copyright 2016 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of NXP nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _DPAA_SEC_H_ +#define _DPAA_SEC_H_ + +#define NUM_POOL_CHANNELS 4 +#define DPAA_SEC_BURST 32 +#define DPAA_SEC_ALG_UNSUPPORT (-1) +#define TDES_CBC_IV_LEN 8 +#define AES_CBC_IV_LEN 16 +#define AES_CTR_IV_LEN 16 +#define AES_GCM_IV_LEN 12 + +/* Minimum job descriptor consists of a oneword job descriptor HEADER and + * a pointer to the shared descriptor. + */ +#define MIN_JOB_DESC_SIZE (CAAM_CMD_SZ + CAAM_PTR_SZ) +/* CTX_POOL_NUM_BUFS is set as per the ipsec-secgw application */ +#define CTX_POOL_NUM_BUFS 32000 +#define CTX_POOL_BUF_SIZE sizeof(struct dpaa_sec_op_ctx) +#define CTX_POOL_CACHE_SIZE 512 + +#define DIR_ENC 1 +#define DIR_DEC 0 + +enum dpaa_sec_op_type { + DPAA_SEC_NONE, /*!< No Cipher operations*/ + DPAA_SEC_CIPHER,/*!< CIPHER operations */ + DPAA_SEC_AUTH, /*!< Authentication Operations */ + DPAA_SEC_AEAD, /*!< Authenticated Encryption with associated data */ + DPAA_SEC_IPSEC, /*!< IPSEC protocol operations*/ + DPAA_SEC_PDCP, /*!< PDCP protocol operations*/ + DPAA_SEC_PKC, /*!< Public Key Cryptographic Operations */ + DPAA_SEC_MAX +}; + +typedef struct dpaa_sec_session_entry { + uint8_t dir; /*!< Operation Direction */ + enum rte_crypto_cipher_algorithm cipher_alg; /*!< Cipher Algorithm*/ + enum rte_crypto_auth_algorithm auth_alg; /*!< Authentication Algorithm*/ + enum rte_crypto_aead_algorithm aead_alg; /*!< Authentication Algorithm*/ + union { + struct { + uint8_t *data; /**< pointer to key data */ + size_t length; /**< key length in bytes */ + } aead_key; + struct { + struct { + uint8_t *data; /**< pointer to key data */ + size_t length; /**< key length in bytes */ + } cipher_key; + struct { + uint8_t *data; /**< pointer to key data */ + size_t length; /**< key length in bytes */ + } auth_key; + }; + }; + struct { + uint16_t length; + uint16_t offset; + } iv; /**< Initialisation vector parameters */ + uint16_t auth_only_len; /*!< Length of data for Auth only */ + uint32_t digest_length; + struct dpaa_sec_qp *qp; + struct rte_mempool *ctx_pool; /* session mempool for dpaa_sec_op_ctx */ +} dpaa_sec_session; + +#define DPAA_SEC_MAX_DESC_SIZE 64 +/* code or cmd block to caam */ +struct sec_cdb { + struct { + union { + uint32_t word; + struct { +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + uint16_t rsvd63_48; + unsigned int rsvd47_39:9; + unsigned int idlen:7; +#else + unsigned int idlen:7; + unsigned int rsvd47_39:9; + uint16_t rsvd63_48; +#endif + } field; + } __packed hi; + + union { + uint32_t word; + struct { +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + unsigned int rsvd31_30:2; + unsigned int fsgt:1; + unsigned int lng:1; + unsigned int offset:2; + unsigned int abs:1; + unsigned int add_buf:1; + uint8_t pool_id; + uint16_t pool_buffer_size; +#else + uint16_t pool_buffer_size; + uint8_t pool_id; + unsigned int add_buf:1; + unsigned int abs:1; + unsigned int offset:2; + unsigned int lng:1; + unsigned int fsgt:1; + unsigned int rsvd31_30:2; +#endif + } field; + } __packed lo; + } __packed sh_hdr; + + uint32_t sh_desc[DPAA_SEC_MAX_DESC_SIZE]; +}; + +struct dpaa_sec_qp { + struct dpaa_sec_dev_private *internals; + struct sec_cdb cdb; /* cmd block associated with qp */ + dpaa_sec_session *ses; /* session associated with qp */ + struct qman_fq inq; + struct qman_fq outq; + int rx_pkts; + int rx_errs; + int tx_pkts; + int tx_errs; +}; + +#define RTE_MAX_NB_SEC_QPS RTE_DPAA_SEC_PMD_MAX_NB_SESSIONS +/* internal sec queue interface */ +struct dpaa_sec_dev_private { + void *sec_hw; + struct rte_mempool *ctx_pool; /* per dev mempool for dpaa_sec_op_ctx */ + struct dpaa_sec_qp qps[RTE_MAX_NB_SEC_QPS]; /* i/o queue for sec */ + unsigned int max_nb_queue_pairs; + unsigned int max_nb_sessions; +}; + +#define MAX_SG_ENTRIES 16 +#define SG_CACHELINE_0 0 +#define SG_CACHELINE_1 4 +#define SG_CACHELINE_2 8 +#define SG_CACHELINE_3 12 +struct dpaa_sec_job { + /* sg[0] output, sg[1] input, others are possible sub frames */ + struct qm_sg_entry sg[MAX_SG_ENTRIES]; +}; + +#define DPAA_MAX_NB_MAX_DIGEST 32 +struct dpaa_sec_op_ctx { + struct dpaa_sec_job job; + struct rte_crypto_op *op; + struct rte_mempool *ctx_pool; /* mempool pointer for dpaa_sec_op_ctx */ + uint32_t fd_status; + uint8_t digest[DPAA_MAX_NB_MAX_DIGEST]; +}; + +static const struct rte_cryptodev_capabilities dpaa_sec_capabilities[] = { + { /* MD5 HMAC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_MD5_HMAC, + .block_size = 64, + .key_size = { + .min = 1, + .max = 64, + .increment = 1 + }, + .digest_size = { + .min = 16, + .max = 16, + .increment = 0 + }, + }, } + }, } + }, + { /* SHA1 HMAC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA1_HMAC, + .block_size = 64, + .key_size = { + .min = 1, + .max = 64, + .increment = 1 + }, + .digest_size = { + .min = 20, + .max = 20, + .increment = 0 + }, + }, } + }, } + }, + { /* SHA224 HMAC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA224_HMAC, + .block_size = 64, + .key_size = { + .min = 1, + .max = 64, + .increment = 1 + }, + .digest_size = { + .min = 28, + .max = 28, + .increment = 0 + }, + }, } + }, } + }, + { /* SHA256 HMAC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA256_HMAC, + .block_size = 64, + .key_size = { + .min = 1, + .max = 64, + .increment = 1 + }, + .digest_size = { + .min = 32, + .max = 32, + .increment = 0 + }, + }, } + }, } + }, + { /* SHA384 HMAC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA384_HMAC, + .block_size = 128, + .key_size = { + .min = 1, + .max = 128, + .increment = 1 + }, + .digest_size = { + .min = 48, + .max = 48, + .increment = 0 + }, + }, } + }, } + }, + { /* SHA512 HMAC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA512_HMAC, + .block_size = 128, + .key_size = { + .min = 1, + .max = 128, + .increment = 1 + }, + .digest_size = { + .min = 64, + .max = 64, + .increment = 0 + }, + }, } + }, } + }, + { /* AES GCM */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD, + {.auth = { + .algo = RTE_CRYPTO_AEAD_AES_GCM, + .block_size = 16, + .key_size = { + .min = 16, + .max = 32, + .increment = 8 + }, + .digest_size = { + .min = 8, + .max = 16, + .increment = 4 + }, + .aad_size = { + .min = 0, + .max = 240, + .increment = 1 + }, + .iv_size = { + .min = 12, + .max = 12, + .increment = 0 + }, + }, } + }, } + }, + { /* AES CBC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, + {.cipher = { + .algo = RTE_CRYPTO_CIPHER_AES_CBC, + .block_size = 16, + .key_size = { + .min = 16, + .max = 32, + .increment = 8 + }, + .iv_size = { + .min = 16, + .max = 16, + .increment = 0 + } + }, } + }, } + }, + { /* AES CTR */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, + {.cipher = { + .algo = RTE_CRYPTO_CIPHER_AES_CTR, + .block_size = 16, + .key_size = { + .min = 16, + .max = 32, + .increment = 8 + }, + .iv_size = { + .min = 16, + .max = 16, + .increment = 0 + } + }, } + }, } + }, + { /* 3DES CBC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, + {.cipher = { + .algo = RTE_CRYPTO_CIPHER_3DES_CBC, + .block_size = 8, + .key_size = { + .min = 16, + .max = 24, + .increment = 8 + }, + .iv_size = { + .min = 8, + .max = 8, + .increment = 0 + } + }, } + }, } + }, + + RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST() +}; + +#endif /* _DPAA_SEC_H_ */ diff --git a/dpdk/drivers/crypto/dpaa_sec/dpaa_sec_log.h b/dpdk/drivers/crypto/dpaa_sec/dpaa_sec_log.h new file mode 100644 index 00000000..dfe69757 --- /dev/null +++ b/dpdk/drivers/crypto/dpaa_sec/dpaa_sec_log.h @@ -0,0 +1,70 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. + * Copyright NXP 2017. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of NXP nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _DPAA_SEC_LOG_H_ +#define _DPAA_SEC_LOG_H_ + +#define PMD_INIT_LOG(level, fmt, args...) \ + RTE_LOG(level, PMD, "%s(): " fmt "\n", __func__, ##args) + +#ifdef RTE_LIBRTE_DPAA_SEC_DEBUG_INIT +#define PMD_INIT_FUNC_TRACE() PMD_INIT_LOG(DEBUG, " >>") +#else +#define PMD_INIT_FUNC_TRACE() do { } while (0) +#endif + +#ifdef RTE_LIBRTE_DPAA_SEC_DEBUG_RX +#define PMD_RX_LOG(level, fmt, args...) \ + RTE_LOG_DP(level, PMD, "%s(): " fmt "\n", __func__, ## args) +#else +#define PMD_RX_LOG(level, fmt, args...) do { } while (0) +#endif + +#ifdef RTE_LIBRTE_DPAA_SEC_DEBUG_TX +#define PMD_TX_LOG(level, fmt, args...) \ + RTE_LOG_DP(level, PMD, "%s(): " fmt "\n", __func__, ## args) +#else +#define PMD_TX_LOG(level, fmt, args...) do { } while (0) +#endif + +#ifdef RTE_LIBRTE_DPAA_SEC_DEBUG_DRIVER +#define PMD_DRV_LOG_RAW(level, fmt, args...) \ + RTE_LOG(level, PMD, "%s(): " fmt, __func__, ## args) +#else +#define PMD_DRV_LOG_RAW(level, fmt, args...) do { } while (0) +#endif + +#define PMD_DRV_LOG(level, fmt, args...) \ + PMD_DRV_LOG_RAW(level, fmt "\n", ## args) + +#endif /* _DPAA_SEC_LOG_H_ */ diff --git a/dpdk/drivers/crypto/dpaa_sec/rte_pmd_dpaa_sec_version.map b/dpdk/drivers/crypto/dpaa_sec/rte_pmd_dpaa_sec_version.map new file mode 100644 index 00000000..a70bd197 --- /dev/null +++ b/dpdk/drivers/crypto/dpaa_sec/rte_pmd_dpaa_sec_version.map @@ -0,0 +1,4 @@ +DPDK_17.11 { + + local: *; +}; diff --git a/dpdk/drivers/crypto/kasumi/Makefile b/dpdk/drivers/crypto/kasumi/Makefile index 9fb0be85..cf56b7a0 100644 --- a/dpdk/drivers/crypto/kasumi/Makefile +++ b/dpdk/drivers/crypto/kasumi/Makefile @@ -54,16 +54,12 @@ CFLAGS += -I$(LIBSSO_KASUMI_PATH) CFLAGS += -I$(LIBSSO_KASUMI_PATH)/include CFLAGS += -I$(LIBSSO_KASUMI_PATH)/build LDLIBS += -L$(LIBSSO_KASUMI_PATH)/build -lsso_kasumi +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_cryptodev +LDLIBS += -lrte_bus_vdev # library source files SRCS-$(CONFIG_RTE_LIBRTE_PMD_KASUMI) += rte_kasumi_pmd.c SRCS-$(CONFIG_RTE_LIBRTE_PMD_KASUMI) += rte_kasumi_pmd_ops.c -# library dependencies -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_KASUMI) += lib/librte_eal -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_KASUMI) += lib/librte_mbuf -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_KASUMI) += lib/librte_mempool -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_KASUMI) += lib/librte_ring -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_KASUMI) += lib/librte_cryptodev - include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/crypto/kasumi/rte_kasumi_pmd.c b/dpdk/drivers/crypto/kasumi/rte_kasumi_pmd.c index df1eb529..f5db5e32 100644 --- a/dpdk/drivers/crypto/kasumi/rte_kasumi_pmd.c +++ b/dpdk/drivers/crypto/kasumi/rte_kasumi_pmd.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -31,11 +31,10 @@ */ #include -#include #include #include #include -#include +#include #include #include @@ -43,30 +42,10 @@ #define KASUMI_KEY_LENGTH 16 #define KASUMI_IV_LENGTH 8 -#define KASUMI_DIGEST_LENGTH 4 #define KASUMI_MAX_BURST 4 #define BYTE_LEN 8 -/** - * Global static parameter used to create a unique name for each KASUMI - * crypto device. - */ -static unsigned unique_name_id; - -static inline int -create_unique_device_name(char *name, size_t size) -{ - int ret; - - if (name == NULL) - return -EINVAL; - - ret = snprintf(name, size, "%s_%u", RTE_STR(CRYPTODEV_NAME_KASUMI_PMD), - unique_name_id++); - if (ret < 0) - return ret; - return 0; -} +static uint8_t cryptodev_driver_id; /** Get xform chain order. */ static enum kasumi_operation @@ -129,25 +108,39 @@ kasumi_set_session_parameters(struct kasumi_session *sess, case KASUMI_OP_NOT_SUPPORTED: default: KASUMI_LOG_ERR("Unsupported operation chain order parameter"); - return -EINVAL; + return -ENOTSUP; } if (cipher_xform) { /* Only KASUMI F8 supported */ if (cipher_xform->cipher.algo != RTE_CRYPTO_CIPHER_KASUMI_F8) + return -ENOTSUP; + + sess->cipher_iv_offset = cipher_xform->cipher.iv.offset; + if (cipher_xform->cipher.iv.length != KASUMI_IV_LENGTH) { + KASUMI_LOG_ERR("Wrong IV length"); return -EINVAL; + } + /* Initialize key */ - sso_kasumi_init_f8_key_sched(xform->cipher.key.data, + sso_kasumi_init_f8_key_sched(cipher_xform->cipher.key.data, &sess->pKeySched_cipher); } if (auth_xform) { /* Only KASUMI F9 supported */ if (auth_xform->auth.algo != RTE_CRYPTO_AUTH_KASUMI_F9) + return -ENOTSUP; + + if (auth_xform->auth.digest_length != KASUMI_DIGEST_LENGTH) { + KASUMI_LOG_ERR("Wrong digest length"); return -EINVAL; + } + sess->auth_op = auth_xform->auth.op; + /* Initialize key */ - sso_kasumi_init_f9_key_sched(xform->auth.key.data, + sso_kasumi_init_f9_key_sched(auth_xform->auth.key.data, &sess->pKeySched_hash); } @@ -161,27 +154,40 @@ kasumi_set_session_parameters(struct kasumi_session *sess, static struct kasumi_session * kasumi_get_session(struct kasumi_qp *qp, struct rte_crypto_op *op) { - struct kasumi_session *sess; + struct kasumi_session *sess = NULL; - if (op->sym->sess_type == RTE_CRYPTO_SYM_OP_WITH_SESSION) { - if (unlikely(op->sym->session->dev_type != - RTE_CRYPTODEV_KASUMI_PMD)) + if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { + if (likely(op->sym->session != NULL)) + sess = (struct kasumi_session *) + get_session_private_data( + op->sym->session, + cryptodev_driver_id); + } else { + void *_sess = NULL; + void *_sess_private_data = NULL; + + if (rte_mempool_get(qp->sess_mp, (void **)&_sess)) return NULL; - sess = (struct kasumi_session *)op->sym->session->_private; - } else { - struct rte_cryptodev_session *c_sess = NULL; - - if (rte_mempool_get(qp->sess_mp, (void **)&c_sess)) + if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data)) return NULL; - sess = (struct kasumi_session *)c_sess->_private; + sess = (struct kasumi_session *)_sess_private_data; if (unlikely(kasumi_set_session_parameters(sess, - op->sym->xform) != 0)) - return NULL; + op->sym->xform) != 0)) { + rte_mempool_put(qp->sess_mp, _sess); + rte_mempool_put(qp->sess_mp, _sess_private_data); + sess = NULL; + } + op->sym->session = (struct rte_cryptodev_sym_session *)_sess; + set_session_private_data(op->sym->session, cryptodev_driver_id, + _sess_private_data); } + if (unlikely(sess == NULL)) + op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; + return sess; } @@ -194,17 +200,11 @@ process_kasumi_cipher_op(struct rte_crypto_op **ops, unsigned i; uint8_t processed_ops = 0; uint8_t *src[num_ops], *dst[num_ops]; - uint64_t IV[num_ops]; + uint8_t *iv_ptr; + uint64_t iv[num_ops]; uint32_t num_bytes[num_ops]; for (i = 0; i < num_ops; i++) { - /* Sanity checks. */ - if (ops[i]->sym->cipher.iv.length != KASUMI_IV_LENGTH) { - ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; - KASUMI_LOG_ERR("iv"); - break; - } - src[i] = rte_pktmbuf_mtod(ops[i]->sym->m_src, uint8_t *) + (ops[i]->sym->cipher.data.offset >> 3); dst[i] = ops[i]->sym->m_dst ? @@ -212,14 +212,16 @@ process_kasumi_cipher_op(struct rte_crypto_op **ops, (ops[i]->sym->cipher.data.offset >> 3) : rte_pktmbuf_mtod(ops[i]->sym->m_src, uint8_t *) + (ops[i]->sym->cipher.data.offset >> 3); - IV[i] = *((uint64_t *)(ops[i]->sym->cipher.iv.data)); + iv_ptr = rte_crypto_op_ctod_offset(ops[i], uint8_t *, + session->cipher_iv_offset); + iv[i] = *((uint64_t *)(iv_ptr)); num_bytes[i] = ops[i]->sym->cipher.data.length >> 3; processed_ops++; } if (processed_ops != 0) - sso_kasumi_f8_n_buffer(&session->pKeySched_cipher, IV, + sso_kasumi_f8_n_buffer(&session->pKeySched_cipher, iv, src, dst, num_bytes, processed_ops); return processed_ops; @@ -231,16 +233,10 @@ process_kasumi_cipher_op_bit(struct rte_crypto_op *op, struct kasumi_session *session) { uint8_t *src, *dst; - uint64_t IV; + uint8_t *iv_ptr; + uint64_t iv; uint32_t length_in_bits, offset_in_bits; - /* Sanity checks. */ - if (unlikely(op->sym->cipher.iv.length != KASUMI_IV_LENGTH)) { - op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; - KASUMI_LOG_ERR("iv"); - return 0; - } - offset_in_bits = op->sym->cipher.data.offset; src = rte_pktmbuf_mtod(op->sym->m_src, uint8_t *); if (op->sym->m_dst == NULL) { @@ -249,10 +245,12 @@ process_kasumi_cipher_op_bit(struct rte_crypto_op *op, return 0; } dst = rte_pktmbuf_mtod(op->sym->m_dst, uint8_t *); - IV = *((uint64_t *)(op->sym->cipher.iv.data)); + iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, + session->cipher_iv_offset); + iv = *((uint64_t *)(iv_ptr)); length_in_bits = op->sym->cipher.data.length; - sso_kasumi_f8_1_buffer_bit(&session->pKeySched_cipher, IV, + sso_kasumi_f8_1_buffer_bit(&session->pKeySched_cipher, iv, src, dst, length_in_bits, offset_in_bits); return 1; @@ -260,7 +258,7 @@ process_kasumi_cipher_op_bit(struct rte_crypto_op *op, /** Generate/verify hash from mbufs with same hash key. */ static int -process_kasumi_hash_op(struct rte_crypto_op **ops, +process_kasumi_hash_op(struct kasumi_qp *qp, struct rte_crypto_op **ops, struct kasumi_session *session, uint8_t num_ops) { @@ -269,23 +267,8 @@ process_kasumi_hash_op(struct rte_crypto_op **ops, uint8_t *src, *dst; uint32_t length_in_bits; uint32_t num_bytes; - uint32_t shift_bits; - uint64_t IV; - uint8_t direction; for (i = 0; i < num_ops; i++) { - if (unlikely(ops[i]->sym->auth.aad.length != KASUMI_IV_LENGTH)) { - ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; - KASUMI_LOG_ERR("aad"); - break; - } - - if (unlikely(ops[i]->sym->auth.digest.length != KASUMI_DIGEST_LENGTH)) { - ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; - KASUMI_LOG_ERR("digest"); - break; - } - /* Data must be byte aligned */ if ((ops[i]->sym->auth.data.offset % BYTE_LEN) != 0) { ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; @@ -297,34 +280,23 @@ process_kasumi_hash_op(struct rte_crypto_op **ops, src = rte_pktmbuf_mtod(ops[i]->sym->m_src, uint8_t *) + (ops[i]->sym->auth.data.offset >> 3); - /* IV from AAD */ - IV = *((uint64_t *)(ops[i]->sym->auth.aad.data)); /* Direction from next bit after end of message */ - num_bytes = (length_in_bits >> 3) + 1; - shift_bits = (BYTE_LEN - 1 - length_in_bits) % BYTE_LEN; - direction = (src[num_bytes - 1] >> shift_bits) & 0x01; + num_bytes = length_in_bits >> 3; if (session->auth_op == RTE_CRYPTO_AUTH_OP_VERIFY) { - dst = (uint8_t *)rte_pktmbuf_append(ops[i]->sym->m_src, - ops[i]->sym->auth.digest.length); + dst = qp->temp_digest; + sso_kasumi_f9_1_buffer(&session->pKeySched_hash, src, + num_bytes, dst); - sso_kasumi_f9_1_buffer_user(&session->pKeySched_hash, - IV, src, - length_in_bits, dst, direction); /* Verify digest. */ if (memcmp(dst, ops[i]->sym->auth.digest.data, - ops[i]->sym->auth.digest.length) != 0) + KASUMI_DIGEST_LENGTH) != 0) ops[i]->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; - - /* Trim area used for digest from mbuf. */ - rte_pktmbuf_trim(ops[i]->sym->m_src, - ops[i]->sym->auth.digest.length); } else { dst = ops[i]->sym->auth.digest.data; - sso_kasumi_f9_1_buffer_user(&session->pKeySched_hash, - IV, src, - length_in_bits, dst, direction); + sso_kasumi_f9_1_buffer(&session->pKeySched_hash, src, + num_bytes, dst); } processed_ops++; } @@ -347,16 +319,16 @@ process_ops(struct rte_crypto_op **ops, struct kasumi_session *session, session, num_ops); break; case KASUMI_OP_ONLY_AUTH: - processed_ops = process_kasumi_hash_op(ops, session, + processed_ops = process_kasumi_hash_op(qp, ops, session, num_ops); break; case KASUMI_OP_CIPHER_AUTH: processed_ops = process_kasumi_cipher_op(ops, session, num_ops); - process_kasumi_hash_op(ops, session, processed_ops); + process_kasumi_hash_op(qp, ops, session, processed_ops); break; case KASUMI_OP_AUTH_CIPHER: - processed_ops = process_kasumi_hash_op(ops, session, + processed_ops = process_kasumi_hash_op(qp, ops, session, num_ops); process_kasumi_cipher_op(ops, session, processed_ops); break; @@ -373,14 +345,18 @@ process_ops(struct rte_crypto_op **ops, struct kasumi_session *session, if (ops[i]->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) ops[i]->status = RTE_CRYPTO_OP_STATUS_SUCCESS; /* Free session if a session-less crypto op. */ - if (ops[i]->sym->sess_type == RTE_CRYPTO_SYM_OP_SESSIONLESS) { + if (ops[i]->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { + memset(session, 0, sizeof(struct kasumi_session)); + memset(ops[i]->sym->session, 0, + rte_cryptodev_get_header_session_size()); + rte_mempool_put(qp->sess_mp, session); rte_mempool_put(qp->sess_mp, ops[i]->sym->session); ops[i]->sym->session = NULL; } } enqueued_ops = rte_ring_enqueue_burst(qp->processed_ops, - (void **)ops, processed_ops); + (void **)ops, processed_ops, NULL); qp->qp_stats.enqueued_count += enqueued_ops; *accumulated_enqueued_ops += enqueued_ops; @@ -400,15 +376,15 @@ process_op_bit(struct rte_crypto_op *op, struct kasumi_session *session, session); break; case KASUMI_OP_ONLY_AUTH: - processed_op = process_kasumi_hash_op(&op, session, 1); + processed_op = process_kasumi_hash_op(qp, &op, session, 1); break; case KASUMI_OP_CIPHER_AUTH: processed_op = process_kasumi_cipher_op_bit(op, session); if (processed_op == 1) - process_kasumi_hash_op(&op, session, 1); + process_kasumi_hash_op(qp, &op, session, 1); break; case KASUMI_OP_AUTH_CIPHER: - processed_op = process_kasumi_hash_op(&op, session, 1); + processed_op = process_kasumi_hash_op(qp, &op, session, 1); if (processed_op == 1) process_kasumi_cipher_op_bit(op, session); break; @@ -425,13 +401,14 @@ process_op_bit(struct rte_crypto_op *op, struct kasumi_session *session, op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; /* Free session if a session-less crypto op. */ - if (op->sym->sess_type == RTE_CRYPTO_SYM_OP_SESSIONLESS) { - rte_mempool_put(qp->sess_mp, op->sym->session); + if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { + memset(op->sym->session, 0, sizeof(struct kasumi_session)); + rte_cryptodev_sym_session_free(op->sym->session); op->sym->session = NULL; } enqueued_op = rte_ring_enqueue_burst(qp->processed_ops, (void **)&op, - processed_op); + processed_op, NULL); qp->qp_stats.enqueued_count += enqueued_op; *accumulated_enqueued_ops += enqueued_op; @@ -455,6 +432,19 @@ kasumi_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops, for (i = 0; i < nb_ops; i++) { curr_c_op = ops[i]; +#ifdef RTE_LIBRTE_PMD_KASUMI_DEBUG + if (!rte_pktmbuf_is_contiguous(curr_c_op->sym->m_src) || + (curr_c_op->sym->m_dst != NULL && + !rte_pktmbuf_is_contiguous( + curr_c_op->sym->m_dst))) { + KASUMI_LOG_ERR("PMD supports only contiguous mbufs, " + "op (%p) provides noncontiguous mbuf as " + "source/destination buffer.\n", curr_c_op); + curr_c_op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; + break; + } +#endif + /* Set status as enqueued (not processed yet) by default. */ curr_c_op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; @@ -550,48 +540,36 @@ kasumi_pmd_dequeue_burst(void *queue_pair, unsigned nb_dequeued; nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops, - (void **)c_ops, nb_ops); + (void **)c_ops, nb_ops, NULL); qp->qp_stats.dequeued_count += nb_dequeued; return nb_dequeued; } -static int cryptodev_kasumi_uninit(const char *name); +static int cryptodev_kasumi_remove(struct rte_vdev_device *vdev); static int cryptodev_kasumi_create(const char *name, - struct rte_crypto_vdev_init_params *init_params) + struct rte_vdev_device *vdev, + struct rte_cryptodev_pmd_init_params *init_params) { struct rte_cryptodev *dev; - char crypto_dev_name[RTE_CRYPTODEV_NAME_MAX_LEN]; struct kasumi_private *internals; uint64_t cpu_flags = 0; - /* Check CPU for supported vector instruction set */ - if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX)) - cpu_flags |= RTE_CRYPTODEV_FF_CPU_AVX; - else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_SSE4_1)) - cpu_flags |= RTE_CRYPTODEV_FF_CPU_SSE; - else { - KASUMI_LOG_ERR("Vector instructions are not supported by CPU"); - return -EFAULT; - } - - /* Create a unique device name. */ - if (create_unique_device_name(crypto_dev_name, - RTE_CRYPTODEV_NAME_MAX_LEN) != 0) { - KASUMI_LOG_ERR("failed to create unique cryptodev name"); - return -EINVAL; - } - - dev = rte_cryptodev_pmd_virtual_dev_init(crypto_dev_name, - sizeof(struct kasumi_private), init_params->socket_id); + dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params); if (dev == NULL) { KASUMI_LOG_ERR("failed to create cryptodev vdev"); goto init_error; } - dev->dev_type = RTE_CRYPTODEV_KASUMI_PMD; + /* Check CPU for supported vector instruction set */ + if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX)) + cpu_flags |= RTE_CRYPTODEV_FF_CPU_AVX; + else + cpu_flags |= RTE_CRYPTODEV_FF_CPU_SSE; + + dev->driver_id = cryptodev_driver_id; dev->dev_ops = rte_kasumi_pmd_ops; /* Register RX/TX burst functions for data path. */ @@ -609,55 +587,65 @@ cryptodev_kasumi_create(const char *name, return 0; init_error: - KASUMI_LOG_ERR("driver %s: cryptodev_kasumi_create failed", name); + KASUMI_LOG_ERR("driver %s: cryptodev_kasumi_create failed", + init_params->name); - cryptodev_kasumi_uninit(crypto_dev_name); + cryptodev_kasumi_remove(vdev); return -EFAULT; } static int -cryptodev_kasumi_init(const char *name, - const char *input_args) +cryptodev_kasumi_probe(struct rte_vdev_device *vdev) { - struct rte_crypto_vdev_init_params init_params = { - RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_QUEUE_PAIRS, - RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_SESSIONS, - rte_socket_id() + struct rte_cryptodev_pmd_init_params init_params = { + "", + sizeof(struct kasumi_private), + rte_socket_id(), + RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS, + RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_SESSIONS }; + const char *name; + const char *input_args; - rte_cryptodev_parse_vdev_init_params(&init_params, input_args); + name = rte_vdev_device_name(vdev); + if (name == NULL) + return -EINVAL; + input_args = rte_vdev_device_args(vdev); - RTE_LOG(INFO, PMD, "Initialising %s on NUMA node %d\n", name, - init_params.socket_id); - RTE_LOG(INFO, PMD, " Max number of queue pairs = %d\n", - init_params.max_nb_queue_pairs); - RTE_LOG(INFO, PMD, " Max number of sessions = %d\n", - init_params.max_nb_sessions); + rte_cryptodev_pmd_parse_input_args(&init_params, input_args); - return cryptodev_kasumi_create(name, &init_params); + return cryptodev_kasumi_create(name, vdev, &init_params); } static int -cryptodev_kasumi_uninit(const char *name) +cryptodev_kasumi_remove(struct rte_vdev_device *vdev) { + struct rte_cryptodev *cryptodev; + const char *name; + + name = rte_vdev_device_name(vdev); if (name == NULL) return -EINVAL; - RTE_LOG(INFO, PMD, "Closing KASUMI crypto device %s" - " on numa socket %u\n", - name, rte_socket_id()); + cryptodev = rte_cryptodev_pmd_get_named_dev(name); + if (cryptodev == NULL) + return -ENODEV; - return 0; + return rte_cryptodev_pmd_destroy(cryptodev); } -static struct rte_driver cryptodev_kasumi_pmd_drv = { - .type = PMD_VDEV, - .init = cryptodev_kasumi_init, - .uninit = cryptodev_kasumi_uninit +static struct rte_vdev_driver cryptodev_kasumi_pmd_drv = { + .probe = cryptodev_kasumi_probe, + .remove = cryptodev_kasumi_remove }; -PMD_REGISTER_DRIVER(cryptodev_kasumi_pmd_drv, CRYPTODEV_NAME_KASUMI_PMD); -DRIVER_REGISTER_PARAM_STRING(CRYPTODEV_NAME_KASUMI_PMD, +static struct cryptodev_driver kasumi_crypto_drv; + +RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_KASUMI_PMD, cryptodev_kasumi_pmd_drv); +RTE_PMD_REGISTER_ALIAS(CRYPTODEV_NAME_KASUMI_PMD, cryptodev_kasumi_pmd); +RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_KASUMI_PMD, "max_nb_queue_pairs= " "max_nb_sessions= " "socket_id="); +RTE_PMD_REGISTER_CRYPTO_DRIVER(kasumi_crypto_drv, cryptodev_kasumi_pmd_drv, + cryptodev_driver_id); diff --git a/dpdk/drivers/crypto/kasumi/rte_kasumi_pmd_ops.c b/dpdk/drivers/crypto/kasumi/rte_kasumi_pmd_ops.c index b9285a43..4da12f37 100644 --- a/dpdk/drivers/crypto/kasumi/rte_kasumi_pmd_ops.c +++ b/dpdk/drivers/crypto/kasumi/rte_kasumi_pmd_ops.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -56,11 +56,7 @@ static const struct rte_cryptodev_capabilities kasumi_pmd_capabilities[] = { .max = 4, .increment = 0 }, - .aad_size = { - .min = 8, - .max = 8, - .increment = 0 - } + .iv_size = { 0 } }, } }, } }, @@ -89,7 +85,8 @@ static const struct rte_cryptodev_capabilities kasumi_pmd_capabilities[] = { /** Configure device */ static int -kasumi_pmd_config(__rte_unused struct rte_cryptodev *dev) +kasumi_pmd_config(__rte_unused struct rte_cryptodev *dev, + __rte_unused struct rte_cryptodev_config *config) { return 0; } @@ -155,7 +152,7 @@ kasumi_pmd_info_get(struct rte_cryptodev *dev, struct kasumi_private *internals = dev->data->dev_private; if (dev_info != NULL) { - dev_info->dev_type = dev->dev_type; + dev_info->driver_id = dev->driver_id; dev_info->max_nb_queue_pairs = internals->max_nb_queue_pairs; dev_info->sym.max_nb_sessions = internals->max_nb_sessions; dev_info->feature_flags = dev->feature_flags; @@ -186,7 +183,7 @@ kasumi_pmd_qp_set_unique_name(struct rte_cryptodev *dev, "kasumi_pmd_%u_qp_%u", dev->data->dev_id, qp->id); - if (n > sizeof(qp->name)) + if (n >= sizeof(qp->name)) return -1; return 0; @@ -201,7 +198,7 @@ kasumi_pmd_qp_create_processed_ops_ring(struct kasumi_qp *qp, r = rte_ring_lookup(qp->name); if (r) { - if (r->prod.size == ring_size) { + if (rte_ring_get_size(r) == ring_size) { KASUMI_LOG_INFO("Reusing existing ring %s" " for processed packets", qp->name); @@ -222,7 +219,7 @@ kasumi_pmd_qp_create_processed_ops_ring(struct kasumi_qp *qp, static int kasumi_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, const struct rte_cryptodev_qp_conf *qp_conf, - int socket_id) + int socket_id, struct rte_mempool *session_pool) { struct kasumi_qp *qp = NULL; @@ -247,7 +244,7 @@ kasumi_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, if (qp->processed_ops == NULL) goto qp_setup_cleanup; - qp->sess_mp = dev->data->session_pool; + qp->sess_mp = session_pool; memset(&qp->qp_stats, 0, sizeof(qp->qp_stats)); @@ -290,33 +287,56 @@ kasumi_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused) } /** Configure a KASUMI session from a crypto xform chain */ -static void * +static int kasumi_pmd_session_configure(struct rte_cryptodev *dev __rte_unused, - struct rte_crypto_sym_xform *xform, void *sess) + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool) { + void *sess_private_data; + int ret; + if (unlikely(sess == NULL)) { KASUMI_LOG_ERR("invalid session struct"); - return NULL; + return -EINVAL; } - if (kasumi_set_session_parameters(sess, xform) != 0) { + if (rte_mempool_get(mempool, &sess_private_data)) { + CDEV_LOG_ERR( + "Couldn't get object from session mempool"); + return -ENOMEM; + } + + ret = kasumi_set_session_parameters(sess_private_data, xform); + if (ret != 0) { KASUMI_LOG_ERR("failed configure session parameters"); - return NULL; + + /* Return session to mempool */ + rte_mempool_put(mempool, sess_private_data); + return ret; } - return sess; + set_session_private_data(sess, dev->driver_id, + sess_private_data); + + return 0; } /** Clear the memory of session so it doesn't leave key material behind */ static void -kasumi_pmd_session_clear(struct rte_cryptodev *dev __rte_unused, void *sess) +kasumi_pmd_session_clear(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) { - /* - * Current just resetting the whole data structure, need to investigate - * whether a more selective reset of key would be more performant - */ - if (sess) - memset(sess, 0, sizeof(struct kasumi_session)); + uint8_t index = dev->driver_id; + void *sess_priv = get_session_private_data(sess, index); + + /* Zero out the whole structure */ + if (sess_priv) { + memset(sess_priv, 0, sizeof(struct kasumi_session)); + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + set_session_private_data(sess, index, NULL); + rte_mempool_put(sess_mp, sess_priv); + } } struct rte_cryptodev_ops kasumi_pmd_ops = { diff --git a/dpdk/drivers/crypto/kasumi/rte_kasumi_pmd_private.h b/dpdk/drivers/crypto/kasumi/rte_kasumi_pmd_private.h index fb586caa..5f7044b8 100644 --- a/dpdk/drivers/crypto/kasumi/rte_kasumi_pmd_private.h +++ b/dpdk/drivers/crypto/kasumi/rte_kasumi_pmd_private.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -35,6 +35,9 @@ #include +#define CRYPTODEV_NAME_KASUMI_PMD crypto_kasumi +/**< KASUMI PMD device name */ + #define KASUMI_LOG_ERR(fmt, args...) \ RTE_LOG(ERR, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \ RTE_STR(CRYPTODEV_NAME_KASUMI_PMD), \ @@ -55,6 +58,8 @@ #define KASUMI_LOG_DBG(fmt, args...) #endif +#define KASUMI_DIGEST_LENGTH 4 + /** private data structure for each virtual KASUMI device */ struct kasumi_private { unsigned max_nb_queue_pairs; @@ -75,6 +80,11 @@ struct kasumi_qp { /**< Session Mempool */ struct rte_cryptodev_stats qp_stats; /**< Queue pair statistics */ + uint8_t temp_digest[KASUMI_DIGEST_LENGTH]; + /**< Buffer used to store the digest generated + * by the driver when verifying a digest provided + * by the user (using authentication verify operation) + */ } __rte_cache_aligned; enum kasumi_operation { @@ -92,6 +102,7 @@ struct kasumi_session { sso_kasumi_key_sched_t pKeySched_hash; enum kasumi_operation op; enum rte_crypto_auth_operation auth_op; + uint16_t cipher_iv_offset; } __rte_cache_aligned; diff --git a/dpdk/drivers/crypto/mrvl/Makefile b/dpdk/drivers/crypto/mrvl/Makefile new file mode 100644 index 00000000..3532f7cf --- /dev/null +++ b/dpdk/drivers/crypto/mrvl/Makefile @@ -0,0 +1,66 @@ +# BSD LICENSE +# +# Copyright(c) 2017 Marvell International Ltd. +# Copyright(c) 2017 Semihalf. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +include $(RTE_SDK)/mk/rte.vars.mk + +ifneq ($(MAKECMDGOALS),clean) +ifneq ($(MAKECMDGOALS),config) +ifeq ($(LIBMUSDK_PATH),) +$(error "Please define LIBMUSDK_PATH environment variable") +endif +endif +endif + +# library name +LIB = librte_pmd_mrvl_crypto.a + +# build flags +CFLAGS += -O3 +CFLAGS += $(WERROR_FLAGS) +CFLAGS += -I$(LIBMUSDK_PATH)/include +CFLAGS += -DMVCONF_ARCH_DMA_ADDR_T_64BIT + +# library version +LIBABIVER := 1 + +# versioning export map +EXPORT_MAP := rte_mrvl_pmd_version.map + +# external library dependencies +LDLIBS += -L$(LIBMUSDK_PATH)/lib -lmusdk +LDLIBS += -lrte_bus_vdev + +# library source files +SRCS-$(CONFIG_RTE_LIBRTE_PMD_MRVL_CRYPTO) += rte_mrvl_pmd.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_MRVL_CRYPTO) += rte_mrvl_pmd_ops.c + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/crypto/mrvl/rte_mrvl_compat.h b/dpdk/drivers/crypto/mrvl/rte_mrvl_compat.h new file mode 100644 index 00000000..c29fa105 --- /dev/null +++ b/dpdk/drivers/crypto/mrvl/rte_mrvl_compat.h @@ -0,0 +1,50 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Marvell International Ltd. + * Copyright(c) 2017 Semihalf. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_MRVL_COMPAT_H_ +#define _RTE_MRVL_COMPAT_H_ + +/* Unluckily, container_of is defined by both DPDK and MUSDK, + * we'll declare only one version. + * + * Note that it is not used in this PMD anyway. + */ +#ifdef container_of +#undef container_of +#endif +#include "drivers/mv_sam.h" +#include "drivers/mv_sam_cio.h" +#include "drivers/mv_sam_session.h" + +#endif /* _RTE_MRVL_COMPAT_H_ */ diff --git a/dpdk/drivers/crypto/mrvl/rte_mrvl_pmd.c b/dpdk/drivers/crypto/mrvl/rte_mrvl_pmd.c new file mode 100644 index 00000000..31f3fe58 --- /dev/null +++ b/dpdk/drivers/crypto/mrvl/rte_mrvl_pmd.c @@ -0,0 +1,857 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Marvell International Ltd. + * Copyright(c) 2017 Semihalf. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "rte_mrvl_pmd_private.h" + +#define MRVL_MUSDK_DMA_MEMSIZE 41943040 + +static uint8_t cryptodev_driver_id; + +/** + * Flag if particular crypto algorithm is supported by PMD/MUSDK. + * + * The idea is to have Not Supported value as default (0). + * This way we need only to define proper map sizes, + * non-initialized entries will be by default not supported. + */ +enum algo_supported { + ALGO_NOT_SUPPORTED = 0, + ALGO_SUPPORTED = 1, +}; + +/** Map elements for cipher mapping.*/ +struct cipher_params_mapping { + enum algo_supported supported; /**< On/Off switch */ + enum sam_cipher_alg cipher_alg; /**< Cipher algorithm */ + enum sam_cipher_mode cipher_mode; /**< Cipher mode */ + unsigned int max_key_len; /**< Maximum key length (in bytes)*/ +} +/* We want to squeeze in multiple maps into the cache line. */ +__rte_aligned(32); + +/** Map elements for auth mapping.*/ +struct auth_params_mapping { + enum algo_supported supported; /**< On/off switch */ + enum sam_auth_alg auth_alg; /**< Auth algorithm */ +} +/* We want to squeeze in multiple maps into the cache line. */ +__rte_aligned(32); + +/** + * Map of supported cipher algorithms. + */ +static const +struct cipher_params_mapping cipher_map[RTE_CRYPTO_CIPHER_LIST_END] = { + [RTE_CRYPTO_CIPHER_3DES_CBC] = { + .supported = ALGO_SUPPORTED, + .cipher_alg = SAM_CIPHER_3DES, + .cipher_mode = SAM_CIPHER_CBC, + .max_key_len = BITS2BYTES(192) }, + [RTE_CRYPTO_CIPHER_3DES_CTR] = { + .supported = ALGO_SUPPORTED, + .cipher_alg = SAM_CIPHER_3DES, + .cipher_mode = SAM_CIPHER_CTR, + .max_key_len = BITS2BYTES(192) }, + [RTE_CRYPTO_CIPHER_3DES_ECB] = { + .supported = ALGO_SUPPORTED, + .cipher_alg = SAM_CIPHER_3DES, + .cipher_mode = SAM_CIPHER_ECB, + .max_key_len = BITS2BYTES(192) }, + [RTE_CRYPTO_CIPHER_AES_CBC] = { + .supported = ALGO_SUPPORTED, + .cipher_alg = SAM_CIPHER_AES, + .cipher_mode = SAM_CIPHER_CBC, + .max_key_len = BITS2BYTES(256) }, + [RTE_CRYPTO_CIPHER_AES_CTR] = { + .supported = ALGO_SUPPORTED, + .cipher_alg = SAM_CIPHER_AES, + .cipher_mode = SAM_CIPHER_CTR, + .max_key_len = BITS2BYTES(256) }, +}; + +/** + * Map of supported auth algorithms. + */ +static const +struct auth_params_mapping auth_map[RTE_CRYPTO_AUTH_LIST_END] = { + [RTE_CRYPTO_AUTH_MD5_HMAC] = { + .supported = ALGO_SUPPORTED, + .auth_alg = SAM_AUTH_HMAC_MD5 }, + [RTE_CRYPTO_AUTH_MD5] = { + .supported = ALGO_SUPPORTED, + .auth_alg = SAM_AUTH_HASH_MD5 }, + [RTE_CRYPTO_AUTH_SHA1_HMAC] = { + .supported = ALGO_SUPPORTED, + .auth_alg = SAM_AUTH_HMAC_SHA1 }, + [RTE_CRYPTO_AUTH_SHA1] = { + .supported = ALGO_SUPPORTED, + .auth_alg = SAM_AUTH_HASH_SHA1 }, + [RTE_CRYPTO_AUTH_SHA224] = { + .supported = ALGO_SUPPORTED, + .auth_alg = SAM_AUTH_HASH_SHA2_224 }, + [RTE_CRYPTO_AUTH_SHA256_HMAC] = { + .supported = ALGO_SUPPORTED, + .auth_alg = SAM_AUTH_HMAC_SHA2_256 }, + [RTE_CRYPTO_AUTH_SHA256] = { + .supported = ALGO_SUPPORTED, + .auth_alg = SAM_AUTH_HASH_SHA2_256 }, + [RTE_CRYPTO_AUTH_SHA384_HMAC] = { + .supported = ALGO_SUPPORTED, + .auth_alg = SAM_AUTH_HMAC_SHA2_384 }, + [RTE_CRYPTO_AUTH_SHA384] = { + .supported = ALGO_SUPPORTED, + .auth_alg = SAM_AUTH_HASH_SHA2_384 }, + [RTE_CRYPTO_AUTH_SHA512_HMAC] = { + .supported = ALGO_SUPPORTED, + .auth_alg = SAM_AUTH_HMAC_SHA2_512 }, + [RTE_CRYPTO_AUTH_SHA512] = { + .supported = ALGO_SUPPORTED, + .auth_alg = SAM_AUTH_HASH_SHA2_512 }, + [RTE_CRYPTO_AUTH_AES_GMAC] = { + .supported = ALGO_SUPPORTED, + .auth_alg = SAM_AUTH_AES_GMAC }, +}; + +/** + * Map of supported aead algorithms. + */ +static const +struct cipher_params_mapping aead_map[RTE_CRYPTO_AEAD_LIST_END] = { + [RTE_CRYPTO_AEAD_AES_GCM] = { + .supported = ALGO_SUPPORTED, + .cipher_alg = SAM_CIPHER_AES, + .cipher_mode = SAM_CIPHER_GCM, + .max_key_len = BITS2BYTES(256) }, +}; + +/* + *----------------------------------------------------------------------------- + * Forward declarations. + *----------------------------------------------------------------------------- + */ +static int cryptodev_mrvl_crypto_uninit(struct rte_vdev_device *vdev); + +/* + *----------------------------------------------------------------------------- + * Session Preparation. + *----------------------------------------------------------------------------- + */ + +/** + * Get xform chain order. + * + * @param xform Pointer to configuration structure chain for crypto operations. + * @returns Order of crypto operations. + */ +static enum mrvl_crypto_chain_order +mrvl_crypto_get_chain_order(const struct rte_crypto_sym_xform *xform) +{ + /* Currently, Marvell supports max 2 operations in chain */ + if (xform->next != NULL && xform->next->next != NULL) + return MRVL_CRYPTO_CHAIN_NOT_SUPPORTED; + + if (xform->next != NULL) { + if ((xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) && + (xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER)) + return MRVL_CRYPTO_CHAIN_AUTH_CIPHER; + + if ((xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) && + (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)) + return MRVL_CRYPTO_CHAIN_CIPHER_AUTH; + } else { + if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) + return MRVL_CRYPTO_CHAIN_AUTH_ONLY; + + if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) + return MRVL_CRYPTO_CHAIN_CIPHER_ONLY; + + if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) + return MRVL_CRYPTO_CHAIN_COMBINED; + } + return MRVL_CRYPTO_CHAIN_NOT_SUPPORTED; +} + +/** + * Set session parameters for cipher part. + * + * @param sess Crypto session pointer. + * @param cipher_xform Pointer to configuration structure for cipher operations. + * @returns 0 in case of success, negative value otherwise. + */ +static int +mrvl_crypto_set_cipher_session_parameters(struct mrvl_crypto_session *sess, + const struct rte_crypto_sym_xform *cipher_xform) +{ + /* Make sure we've got proper struct */ + if (cipher_xform->type != RTE_CRYPTO_SYM_XFORM_CIPHER) { + MRVL_CRYPTO_LOG_ERR("Wrong xform struct provided!"); + return -EINVAL; + } + + /* See if map data is present and valid */ + if ((cipher_xform->cipher.algo > RTE_DIM(cipher_map)) || + (cipher_map[cipher_xform->cipher.algo].supported + != ALGO_SUPPORTED)) { + MRVL_CRYPTO_LOG_ERR("Cipher algorithm not supported!"); + return -EINVAL; + } + + sess->cipher_iv_offset = cipher_xform->cipher.iv.offset; + + sess->sam_sess_params.dir = + (cipher_xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ? + SAM_DIR_ENCRYPT : SAM_DIR_DECRYPT; + sess->sam_sess_params.cipher_alg = + cipher_map[cipher_xform->cipher.algo].cipher_alg; + sess->sam_sess_params.cipher_mode = + cipher_map[cipher_xform->cipher.algo].cipher_mode; + + /* Assume IV will be passed together with data. */ + sess->sam_sess_params.cipher_iv = NULL; + + /* Get max key length. */ + if (cipher_xform->cipher.key.length > + cipher_map[cipher_xform->cipher.algo].max_key_len) { + MRVL_CRYPTO_LOG_ERR("Wrong key length!"); + return -EINVAL; + } + + sess->sam_sess_params.cipher_key_len = cipher_xform->cipher.key.length; + sess->sam_sess_params.cipher_key = cipher_xform->cipher.key.data; + + return 0; +} + +/** + * Set session parameters for authentication part. + * + * @param sess Crypto session pointer. + * @param auth_xform Pointer to configuration structure for auth operations. + * @returns 0 in case of success, negative value otherwise. + */ +static int +mrvl_crypto_set_auth_session_parameters(struct mrvl_crypto_session *sess, + const struct rte_crypto_sym_xform *auth_xform) +{ + /* Make sure we've got proper struct */ + if (auth_xform->type != RTE_CRYPTO_SYM_XFORM_AUTH) { + MRVL_CRYPTO_LOG_ERR("Wrong xform struct provided!"); + return -EINVAL; + } + + /* See if map data is present and valid */ + if ((auth_xform->auth.algo > RTE_DIM(auth_map)) || + (auth_map[auth_xform->auth.algo].supported != ALGO_SUPPORTED)) { + MRVL_CRYPTO_LOG_ERR("Auth algorithm not supported!"); + return -EINVAL; + } + + sess->sam_sess_params.dir = + (auth_xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) ? + SAM_DIR_ENCRYPT : SAM_DIR_DECRYPT; + sess->sam_sess_params.auth_alg = + auth_map[auth_xform->auth.algo].auth_alg; + sess->sam_sess_params.u.basic.auth_icv_len = + auth_xform->auth.digest_length; + /* auth_key must be NULL if auth algorithm does not use HMAC */ + sess->sam_sess_params.auth_key = auth_xform->auth.key.length ? + auth_xform->auth.key.data : NULL; + sess->sam_sess_params.auth_key_len = auth_xform->auth.key.length; + + return 0; +} + +/** + * Set session parameters for aead part. + * + * @param sess Crypto session pointer. + * @param aead_xform Pointer to configuration structure for aead operations. + * @returns 0 in case of success, negative value otherwise. + */ +static int +mrvl_crypto_set_aead_session_parameters(struct mrvl_crypto_session *sess, + const struct rte_crypto_sym_xform *aead_xform) +{ + /* Make sure we've got proper struct */ + if (aead_xform->type != RTE_CRYPTO_SYM_XFORM_AEAD) { + MRVL_CRYPTO_LOG_ERR("Wrong xform struct provided!"); + return -EINVAL; + } + + /* See if map data is present and valid */ + if ((aead_xform->aead.algo > RTE_DIM(aead_map)) || + (aead_map[aead_xform->aead.algo].supported + != ALGO_SUPPORTED)) { + MRVL_CRYPTO_LOG_ERR("AEAD algorithm not supported!"); + return -EINVAL; + } + + sess->sam_sess_params.dir = + (aead_xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) ? + SAM_DIR_ENCRYPT : SAM_DIR_DECRYPT; + sess->sam_sess_params.cipher_alg = + aead_map[aead_xform->aead.algo].cipher_alg; + sess->sam_sess_params.cipher_mode = + aead_map[aead_xform->aead.algo].cipher_mode; + + /* Assume IV will be passed together with data. */ + sess->sam_sess_params.cipher_iv = NULL; + + /* Get max key length. */ + if (aead_xform->aead.key.length > + aead_map[aead_xform->aead.algo].max_key_len) { + MRVL_CRYPTO_LOG_ERR("Wrong key length!"); + return -EINVAL; + } + + sess->sam_sess_params.cipher_key = aead_xform->aead.key.data; + sess->sam_sess_params.cipher_key_len = aead_xform->aead.key.length; + + if (sess->sam_sess_params.cipher_mode == SAM_CIPHER_GCM) + sess->sam_sess_params.auth_alg = SAM_AUTH_AES_GCM; + + sess->sam_sess_params.u.basic.auth_icv_len = + aead_xform->aead.digest_length; + + sess->sam_sess_params.u.basic.auth_aad_len = + aead_xform->aead.aad_length; + + return 0; +} + +/** + * Parse crypto transform chain and setup session parameters. + * + * @param dev Pointer to crypto device + * @param sess Poiner to crypto session + * @param xform Pointer to configuration structure chain for crypto operations. + * @returns 0 in case of success, negative value otherwise. + */ +int +mrvl_crypto_set_session_parameters(struct mrvl_crypto_session *sess, + const struct rte_crypto_sym_xform *xform) +{ + const struct rte_crypto_sym_xform *cipher_xform = NULL; + const struct rte_crypto_sym_xform *auth_xform = NULL; + const struct rte_crypto_sym_xform *aead_xform = NULL; + + /* Filter out spurious/broken requests */ + if (xform == NULL) + return -EINVAL; + + sess->chain_order = mrvl_crypto_get_chain_order(xform); + switch (sess->chain_order) { + case MRVL_CRYPTO_CHAIN_CIPHER_AUTH: + cipher_xform = xform; + auth_xform = xform->next; + break; + case MRVL_CRYPTO_CHAIN_AUTH_CIPHER: + auth_xform = xform; + cipher_xform = xform->next; + break; + case MRVL_CRYPTO_CHAIN_CIPHER_ONLY: + cipher_xform = xform; + break; + case MRVL_CRYPTO_CHAIN_AUTH_ONLY: + auth_xform = xform; + break; + case MRVL_CRYPTO_CHAIN_COMBINED: + aead_xform = xform; + break; + default: + return -EINVAL; + } + + if ((cipher_xform != NULL) && + (mrvl_crypto_set_cipher_session_parameters( + sess, cipher_xform) < 0)) { + MRVL_CRYPTO_LOG_ERR("Invalid/unsupported cipher parameters"); + return -EINVAL; + } + + if ((auth_xform != NULL) && + (mrvl_crypto_set_auth_session_parameters( + sess, auth_xform) < 0)) { + MRVL_CRYPTO_LOG_ERR("Invalid/unsupported auth parameters"); + return -EINVAL; + } + + if ((aead_xform != NULL) && + (mrvl_crypto_set_aead_session_parameters( + sess, aead_xform) < 0)) { + MRVL_CRYPTO_LOG_ERR("Invalid/unsupported aead parameters"); + return -EINVAL; + } + + return 0; +} + +/* + *----------------------------------------------------------------------------- + * Process Operations + *----------------------------------------------------------------------------- + */ + +/** + * Prepare a single request. + * + * This function basically translates DPDK crypto request into one + * understandable by MUDSK's SAM. If this is a first request in a session, + * it starts the session. + * + * @param request Pointer to pre-allocated && reset request buffer [Out]. + * @param src_bd Pointer to pre-allocated source descriptor [Out]. + * @param dst_bd Pointer to pre-allocated destination descriptor [Out]. + * @param op Pointer to DPDK crypto operation struct [In]. + */ +static inline int +mrvl_request_prepare(struct sam_cio_op_params *request, + struct sam_buf_info *src_bd, + struct sam_buf_info *dst_bd, + struct rte_crypto_op *op) +{ + struct mrvl_crypto_session *sess; + struct rte_mbuf *dst_mbuf; + uint8_t *digest; + + if (unlikely(op->sess_type == RTE_CRYPTO_OP_SESSIONLESS)) { + MRVL_CRYPTO_LOG_ERR("MRVL CRYPTO PMD only supports session " + "oriented requests, op (%p) is sessionless.", + op); + return -EINVAL; + } + + sess = (struct mrvl_crypto_session *)get_session_private_data( + op->sym->session, cryptodev_driver_id); + if (unlikely(sess == NULL)) { + MRVL_CRYPTO_LOG_ERR("Session was not created for this device"); + return -EINVAL; + } + + /* + * If application delivered us null dst buffer, it means it expects + * us to deliver the result in src buffer. + */ + dst_mbuf = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src; + + request->sa = sess->sam_sess; + request->cookie = op; + + /* Single buffers only, sorry. */ + request->num_bufs = 1; + request->src = src_bd; + src_bd->vaddr = rte_pktmbuf_mtod(op->sym->m_src, void *); + src_bd->paddr = rte_pktmbuf_iova(op->sym->m_src); + src_bd->len = rte_pktmbuf_data_len(op->sym->m_src); + + /* Empty source. */ + if (rte_pktmbuf_data_len(op->sym->m_src) == 0) { + /* EIP does not support 0 length buffers. */ + MRVL_CRYPTO_LOG_ERR("Buffer length == 0 not supported!"); + return -1; + } + + /* Empty destination. */ + if (rte_pktmbuf_data_len(dst_mbuf) == 0) { + /* Make dst buffer fit at least source data. */ + if (rte_pktmbuf_append(dst_mbuf, + rte_pktmbuf_data_len(op->sym->m_src)) == NULL) { + MRVL_CRYPTO_LOG_ERR("Unable to set big enough dst buffer!"); + return -1; + } + } + + request->dst = dst_bd; + dst_bd->vaddr = rte_pktmbuf_mtod(dst_mbuf, void *); + dst_bd->paddr = rte_pktmbuf_iova(dst_mbuf); + + /* + * We can use all available space in dst_mbuf, + * not only what's used currently. + */ + dst_bd->len = dst_mbuf->buf_len - rte_pktmbuf_headroom(dst_mbuf); + + if (sess->chain_order == MRVL_CRYPTO_CHAIN_COMBINED) { + request->cipher_len = op->sym->aead.data.length; + request->cipher_offset = op->sym->aead.data.offset; + request->cipher_iv = rte_crypto_op_ctod_offset(op, uint8_t *, + sess->cipher_iv_offset); + + request->auth_aad = op->sym->aead.aad.data; + request->auth_offset = request->cipher_offset; + request->auth_len = request->cipher_len; + } else { + request->cipher_len = op->sym->cipher.data.length; + request->cipher_offset = op->sym->cipher.data.offset; + request->cipher_iv = rte_crypto_op_ctod_offset(op, uint8_t *, + sess->cipher_iv_offset); + + request->auth_offset = op->sym->auth.data.offset; + request->auth_len = op->sym->auth.data.length; + } + + digest = sess->chain_order == MRVL_CRYPTO_CHAIN_COMBINED ? + op->sym->aead.digest.data : op->sym->auth.digest.data; + if (digest == NULL) { + /* No auth - no worry. */ + return 0; + } + + request->auth_icv_offset = request->auth_offset + request->auth_len; + + /* + * EIP supports only scenarios where ICV(digest buffer) is placed at + * auth_icv_offset. Any other placement means risking errors. + */ + if (sess->sam_sess_params.dir == SAM_DIR_ENCRYPT) { + /* + * This should be the most common case anyway, + * EIP will overwrite DST buffer at auth_icv_offset. + */ + if (rte_pktmbuf_mtod_offset( + dst_mbuf, uint8_t *, + request->auth_icv_offset) == digest) { + return 0; + } + } else {/* sess->sam_sess_params.dir == SAM_DIR_DECRYPT */ + /* + * EIP will look for digest at auth_icv_offset + * offset in SRC buffer. + */ + if (rte_pktmbuf_mtod_offset( + op->sym->m_src, uint8_t *, + request->auth_icv_offset) == digest) { + return 0; + } + } + + /* + * If we landed here it means that digest pointer is + * at different than expected place. + */ + return -1; +} + +/* + *----------------------------------------------------------------------------- + * PMD Framework handlers + *----------------------------------------------------------------------------- + */ + +/** + * Enqueue burst. + * + * @param queue_pair Pointer to queue pair. + * @param ops Pointer to ops requests array. + * @param nb_ops Number of elements in ops requests array. + * @returns Number of elements consumed from ops. + */ +static uint16_t +mrvl_crypto_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops, + uint16_t nb_ops) +{ + uint16_t iter_ops = 0; + uint16_t to_enq = 0; + uint16_t consumed = 0; + int ret; + struct sam_cio_op_params requests[nb_ops]; + /* + * DPDK uses single fragment buffers, so we can KISS descriptors. + * SAM does not store bd pointers, so on-stack scope will be enough. + */ + struct sam_buf_info src_bd[nb_ops]; + struct sam_buf_info dst_bd[nb_ops]; + struct mrvl_crypto_qp *qp = (struct mrvl_crypto_qp *)queue_pair; + + if (nb_ops == 0) + return 0; + + /* Prepare the burst. */ + memset(&requests, 0, sizeof(requests)); + + /* Iterate through */ + for (; iter_ops < nb_ops; ++iter_ops) { + if (mrvl_request_prepare(&requests[iter_ops], + &src_bd[iter_ops], + &dst_bd[iter_ops], + ops[iter_ops]) < 0) { + MRVL_CRYPTO_LOG_ERR( + "Error while parameters preparation!"); + qp->stats.enqueue_err_count++; + ops[iter_ops]->status = RTE_CRYPTO_OP_STATUS_ERROR; + + /* + * Number of handled ops is increased + * (even if the result of handling is error). + */ + ++consumed; + break; + } + + ops[iter_ops]->status = + RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; + + /* Increase the number of ops to enqueue. */ + ++to_enq; + } /* for (; iter_ops < nb_ops;... */ + + if (to_enq > 0) { + /* Send the burst */ + ret = sam_cio_enq(qp->cio, requests, &to_enq); + consumed += to_enq; + if (ret < 0) { + /* + * Trust SAM that in this case returned value will be at + * some point correct (now it is returned unmodified). + */ + qp->stats.enqueue_err_count += to_enq; + for (iter_ops = 0; iter_ops < to_enq; ++iter_ops) + ops[iter_ops]->status = + RTE_CRYPTO_OP_STATUS_ERROR; + } + } + + qp->stats.enqueued_count += to_enq; + return consumed; +} + +/** + * Dequeue burst. + * + * @param queue_pair Pointer to queue pair. + * @param ops Pointer to ops requests array. + * @param nb_ops Number of elements in ops requests array. + * @returns Number of elements dequeued. + */ +static uint16_t +mrvl_crypto_pmd_dequeue_burst(void *queue_pair, + struct rte_crypto_op **ops, + uint16_t nb_ops) +{ + int ret; + struct mrvl_crypto_qp *qp = queue_pair; + struct sam_cio *cio = qp->cio; + struct sam_cio_op_result results[nb_ops]; + uint16_t i; + + ret = sam_cio_deq(cio, results, &nb_ops); + if (ret < 0) { + /* Count all dequeued as error. */ + qp->stats.dequeue_err_count += nb_ops; + + /* But act as they were dequeued anyway*/ + qp->stats.dequeued_count += nb_ops; + + return 0; + } + + /* Unpack and check results. */ + for (i = 0; i < nb_ops; ++i) { + ops[i] = results[i].cookie; + + switch (results[i].status) { + case SAM_CIO_OK: + ops[i]->status = RTE_CRYPTO_OP_STATUS_SUCCESS; + break; + case SAM_CIO_ERR_ICV: + MRVL_CRYPTO_LOG_DBG("CIO returned SAM_CIO_ERR_ICV."); + ops[i]->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; + break; + default: + MRVL_CRYPTO_LOG_DBG( + "CIO returned Error: %d", results[i].status); + ops[i]->status = RTE_CRYPTO_OP_STATUS_ERROR; + break; + } + } + + qp->stats.dequeued_count += nb_ops; + return nb_ops; +} + +/** + * Create a new crypto device. + * + * @param name Driver name. + * @param vdev Pointer to device structure. + * @param init_params Pointer to initialization parameters. + * @returns 0 in case of success, negative value otherwise. + */ +static int +cryptodev_mrvl_crypto_create(const char *name, + struct rte_vdev_device *vdev, + struct rte_cryptodev_pmd_init_params *init_params) +{ + struct rte_cryptodev *dev; + struct mrvl_crypto_private *internals; + struct sam_init_params sam_params; + int ret; + + dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params); + if (dev == NULL) { + MRVL_CRYPTO_LOG_ERR("failed to create cryptodev vdev"); + goto init_error; + } + + dev->driver_id = cryptodev_driver_id; + dev->dev_ops = rte_mrvl_crypto_pmd_ops; + + /* Register rx/tx burst functions for data path. */ + dev->enqueue_burst = mrvl_crypto_pmd_enqueue_burst; + dev->dequeue_burst = mrvl_crypto_pmd_dequeue_burst; + + dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | + RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | + RTE_CRYPTODEV_FF_HW_ACCELERATED; + + /* Set vector instructions mode supported */ + internals = dev->data->dev_private; + + internals->max_nb_qpairs = init_params->max_nb_queue_pairs; + internals->max_nb_sessions = init_params->max_nb_sessions; + + /* + * ret == -EEXIST is correct, it means DMA + * has been already initialized. + */ + ret = mv_sys_dma_mem_init(MRVL_MUSDK_DMA_MEMSIZE); + if (ret < 0) { + if (ret != -EEXIST) + return ret; + + MRVL_CRYPTO_LOG_INFO( + "DMA memory has been already initialized by a different driver."); + } + + sam_params.max_num_sessions = internals->max_nb_sessions; + + return sam_init(&sam_params); + +init_error: + MRVL_CRYPTO_LOG_ERR( + "driver %s: %s failed", init_params->name, __func__); + + cryptodev_mrvl_crypto_uninit(vdev); + return -EFAULT; +} + +/** + * Initialize the crypto device. + * + * @param vdev Pointer to device structure. + * @returns 0 in case of success, negative value otherwise. + */ +static int +cryptodev_mrvl_crypto_init(struct rte_vdev_device *vdev) +{ + struct rte_cryptodev_pmd_init_params init_params = { }; + const char *name, *args; + int ret; + + name = rte_vdev_device_name(vdev); + if (name == NULL) + return -EINVAL; + args = rte_vdev_device_args(vdev); + + init_params.private_data_size = sizeof(struct mrvl_crypto_private); + init_params.max_nb_queue_pairs = sam_get_num_inst() * SAM_HW_RING_NUM; + init_params.max_nb_sessions = + RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_SESSIONS; + init_params.socket_id = rte_socket_id(); + + ret = rte_cryptodev_pmd_parse_input_args(&init_params, args); + if (ret) { + RTE_LOG(ERR, PMD, + "Failed to parse initialisation arguments[%s]\n", + args); + return -EINVAL; + } + + return cryptodev_mrvl_crypto_create(name, vdev, &init_params); +} + +/** + * Uninitialize the crypto device + * + * @param vdev Pointer to device structure. + * @returns 0 in case of success, negative value otherwise. + */ +static int +cryptodev_mrvl_crypto_uninit(struct rte_vdev_device *vdev) +{ + struct rte_cryptodev *cryptodev; + const char *name = rte_vdev_device_name(vdev); + + if (name == NULL) + return -EINVAL; + + RTE_LOG(INFO, PMD, + "Closing Marvell crypto device %s on numa socket %u\n", + name, rte_socket_id()); + + sam_deinit(); + + cryptodev = rte_cryptodev_pmd_get_named_dev(name); + if (cryptodev == NULL) + return -ENODEV; + + return rte_cryptodev_pmd_destroy(cryptodev); +} + +/** + * Basic driver handlers for use in the constructor. + */ +static struct rte_vdev_driver cryptodev_mrvl_pmd_drv = { + .probe = cryptodev_mrvl_crypto_init, + .remove = cryptodev_mrvl_crypto_uninit +}; + +static struct cryptodev_driver mrvl_crypto_drv; + +/* Register the driver in constructor. */ +RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_MRVL_PMD, cryptodev_mrvl_pmd_drv); +RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_MRVL_PMD, + "max_nb_queue_pairs= " + "max_nb_sessions= " + "socket_id="); +RTE_PMD_REGISTER_CRYPTO_DRIVER(mrvl_crypto_drv, cryptodev_mrvl_pmd_drv, + cryptodev_driver_id); diff --git a/dpdk/drivers/crypto/mrvl/rte_mrvl_pmd_ops.c b/dpdk/drivers/crypto/mrvl/rte_mrvl_pmd_ops.c new file mode 100644 index 00000000..434cf850 --- /dev/null +++ b/dpdk/drivers/crypto/mrvl/rte_mrvl_pmd_ops.c @@ -0,0 +1,778 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Marvell International Ltd. + * Copyright(c) 2017 Semihalf. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +#include +#include +#include + +#include "rte_mrvl_pmd_private.h" + +/** + * Capabilities list to be used in reporting to DPDK. + */ +static const struct rte_cryptodev_capabilities + mrvl_crypto_pmd_capabilities[] = { + { /* MD5 HMAC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_MD5_HMAC, + .block_size = 64, + .key_size = { + .min = 1, + .max = 64, + .increment = 1 + }, + .digest_size = { + .min = 16, + .max = 16, + .increment = 0 + }, + }, } + }, } + }, + { /* MD5 */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_MD5, + .block_size = 64, + .key_size = { + .min = 0, + .max = 0, + .increment = 0 + }, + .digest_size = { + .min = 16, + .max = 16, + .increment = 0 + }, + }, } + }, } + }, + { /* SHA1 HMAC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA1_HMAC, + .block_size = 64, + .key_size = { + .min = 1, + .max = 64, + .increment = 1 + }, + .digest_size = { + .min = 20, + .max = 20, + .increment = 0 + }, + }, } + }, } + }, + { /* SHA1 */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA1, + .block_size = 64, + .key_size = { + .min = 0, + .max = 0, + .increment = 0 + }, + .digest_size = { + .min = 20, + .max = 20, + .increment = 0 + }, + }, } + }, } + }, + { /* SHA224 */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA224, + .block_size = 64, + .key_size = { + .min = 0, + .max = 0, + .increment = 0 + }, + .digest_size = { + .min = 28, + .max = 28, + .increment = 0 + }, + }, } + }, } + }, + { /* SHA256 HMAC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA256_HMAC, + .block_size = 64, + .key_size = { + .min = 1, + .max = 64, + .increment = 1 + }, + .digest_size = { + .min = 32, + .max = 32, + .increment = 0 + }, + }, } + }, } + }, + { /* SHA256 */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA256, + .block_size = 64, + .key_size = { + .min = 0, + .max = 0, + .increment = 0 + }, + .digest_size = { + .min = 32, + .max = 32, + .increment = 0 + }, + }, } + }, } + }, + { /* SHA384 HMAC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA384_HMAC, + .block_size = 128, + .key_size = { + .min = 1, + .max = 128, + .increment = 1 + }, + .digest_size = { + .min = 48, + .max = 48, + .increment = 0 + }, + }, } + }, } + }, + { /* SHA384 */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA384, + .block_size = 128, + .key_size = { + .min = 0, + .max = 0, + .increment = 0 + }, + .digest_size = { + .min = 48, + .max = 48, + .increment = 0 + }, + }, } + }, } + }, + { /* SHA512 HMAC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA512_HMAC, + .block_size = 128, + .key_size = { + .min = 1, + .max = 128, + .increment = 1 + }, + .digest_size = { + .min = 64, + .max = 64, + .increment = 0 + }, + }, } + }, } + }, + { /* SHA512 */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA512, + .block_size = 128, + .key_size = { + .min = 0, + .max = 0, + .increment = 0 + }, + .digest_size = { + .min = 64, + .max = 64, + .increment = 0 + }, + }, } + }, } + }, + { /* AES CBC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, + {.cipher = { + .algo = RTE_CRYPTO_CIPHER_AES_CBC, + .block_size = 16, + .key_size = { + .min = 16, + .max = 32, + .increment = 8 + }, + .iv_size = { + .min = 16, + .max = 16, + .increment = 0 + } + }, } + }, } + }, + { /* AES CTR */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, + {.cipher = { + .algo = RTE_CRYPTO_CIPHER_AES_CTR, + .block_size = 16, + .key_size = { + .min = 16, + .max = 32, + .increment = 8 + }, + .iv_size = { + .min = 16, + .max = 16, + .increment = 0 + } + }, } + }, } + }, + { /* AES GCM */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD, + {.aead = { + .algo = RTE_CRYPTO_AEAD_AES_GCM, + .block_size = 16, + .key_size = { + .min = 16, + .max = 32, + .increment = 8 + }, + .digest_size = { + .min = 16, + .max = 16, + .increment = 0 + }, + .aad_size = { + .min = 8, + .max = 12, + .increment = 4 + }, + .iv_size = { + .min = 12, + .max = 16, + .increment = 4 + } + }, } + }, } + }, + { /* AES GMAC (AUTH) */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_AES_GMAC, + .block_size = 16, + .key_size = { + .min = 16, + .max = 32, + .increment = 8 + }, + .digest_size = { + .min = 16, + .max = 16, + .increment = 0 + }, + .iv_size = { + .min = 8, + .max = 65532, + .increment = 4 + } + }, } + }, } + }, + { /* 3DES CBC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, + {.cipher = { + .algo = RTE_CRYPTO_CIPHER_3DES_CBC, + .block_size = 8, + .key_size = { + .min = 24, + .max = 24, + .increment = 0 + }, + .iv_size = { + .min = 8, + .max = 8, + .increment = 0 + } + }, } + }, } + }, + { /* 3DES CTR */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, + {.cipher = { + .algo = RTE_CRYPTO_CIPHER_3DES_CTR, + .block_size = 8, + .key_size = { + .min = 24, + .max = 24, + .increment = 0 + }, + .iv_size = { + .min = 8, + .max = 8, + .increment = 0 + } + }, } + }, } + }, + + RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST() +}; + + +/** + * Configure device (PMD ops callback). + * + * @param dev Pointer to the device structure. + * @param config Pointer to configuration structure. + * @returns 0. Always. + */ +static int +mrvl_crypto_pmd_config(__rte_unused struct rte_cryptodev *dev, + __rte_unused struct rte_cryptodev_config *config) +{ + return 0; +} + +/** + * Start device (PMD ops callback). + * + * @param dev Pointer to the device structure. + * @returns 0. Always. + */ +static int +mrvl_crypto_pmd_start(__rte_unused struct rte_cryptodev *dev) +{ + return 0; +} + +/** + * Stop device (PMD ops callback). + * + * @param dev Pointer to the device structure. + * @returns 0. Always. + */ +static void +mrvl_crypto_pmd_stop(__rte_unused struct rte_cryptodev *dev) +{ +} + +/** + * Get device statistics (PMD ops callback). + * + * @param dev Pointer to the device structure. + * @param stats Pointer to statistics structure [out]. + */ +static void +mrvl_crypto_pmd_stats_get(struct rte_cryptodev *dev, + struct rte_cryptodev_stats *stats) +{ + int qp_id; + + for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) { + struct mrvl_crypto_qp *qp = dev->data->queue_pairs[qp_id]; + + stats->enqueued_count += qp->stats.enqueued_count; + stats->dequeued_count += qp->stats.dequeued_count; + + stats->enqueue_err_count += qp->stats.enqueue_err_count; + stats->dequeue_err_count += qp->stats.dequeue_err_count; + } +} + +/** + * Reset device statistics (PMD ops callback). + * + * @param dev Pointer to the device structure. + */ +static void +mrvl_crypto_pmd_stats_reset(struct rte_cryptodev *dev) +{ + int qp_id; + + for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) { + struct mrvl_crypto_qp *qp = dev->data->queue_pairs[qp_id]; + + memset(&qp->stats, 0, sizeof(qp->stats)); + } +} + +/** + * Get device info (PMD ops callback). + * + * @param dev Pointer to the device structure. + * @param dev_info Pointer to the device info structure [out]. + */ +static void +mrvl_crypto_pmd_info_get(struct rte_cryptodev *dev, + struct rte_cryptodev_info *dev_info) +{ + struct mrvl_crypto_private *internals = dev->data->dev_private; + + if (dev_info != NULL) { + dev_info->driver_id = dev->driver_id; + dev_info->feature_flags = dev->feature_flags; + dev_info->capabilities = mrvl_crypto_pmd_capabilities; + dev_info->max_nb_queue_pairs = internals->max_nb_qpairs; + dev_info->sym.max_nb_sessions = internals->max_nb_sessions; + } +} + +/** + * Release queue pair (PMD ops callback). + * + * @param dev Pointer to the device structure. + * @param qp_id ID of Queue Pair to release. + * @returns 0. Always. + */ +static int +mrvl_crypto_pmd_qp_release(struct rte_cryptodev *dev, uint16_t qp_id) +{ + struct mrvl_crypto_qp *qp = + (struct mrvl_crypto_qp *)dev->data->queue_pairs[qp_id]; + + if (dev->data->queue_pairs[qp_id] != NULL) { + sam_cio_flush(qp->cio); + sam_cio_deinit(qp->cio); + rte_free(dev->data->queue_pairs[qp_id]); + dev->data->queue_pairs[qp_id] = NULL; + } + + return 0; +} + +/** + * Close device (PMD ops callback). + * + * @param dev Pointer to the device structure. + * @returns 0. Always. + */ +static int +mrvl_crypto_pmd_close(struct rte_cryptodev *dev) +{ + int qp_id; + + for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) + mrvl_crypto_pmd_qp_release(dev, qp_id); + + return 0; +} + +/** + * Setup a queue pair (PMD ops callback). + * + * @param dev Pointer to the device structure. + * @param qp_id ID of the Queue Pair. + * @param qp_conf Queue pair configuration (nb of descriptors). + * @param socket_id NUMA socket to allocate memory on. + * @returns 0 upon success, negative value otherwise. + */ +static int +mrvl_crypto_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, + const struct rte_cryptodev_qp_conf *qp_conf, + int socket_id, struct rte_mempool *session_pool) +{ + struct mrvl_crypto_qp *qp = NULL; + char match[RTE_CRYPTODEV_NAME_LEN]; + unsigned int n; + + /* Allocate the queue pair data structure. */ + qp = rte_zmalloc_socket("MRVL Crypto PMD Queue Pair", sizeof(*qp), + RTE_CACHE_LINE_SIZE, socket_id); + if (qp == NULL) + return -ENOMEM; + + /* Free old qp prior setup if needed. */ + if (dev->data->queue_pairs[qp_id] != NULL) + mrvl_crypto_pmd_qp_release(dev, qp_id); + + do { /* Error handling block */ + + /* + * This extra check is necessary due to a bug in + * crypto library. + */ + int num = sam_get_num_inst(); + if (num == 0) { + MRVL_CRYPTO_LOG_ERR("No crypto engines detected.\n"); + return -1; + } + + /* + * In case two crypto engines are enabled qps will + * be evenly spread among them. Even and odd qps will + * be handled by cio-0 and cio-1 respectively. qp-cio mapping + * will look as follows: + * + * qp: 0 1 2 3 + * cio-x:y: cio-0:0, cio-1:0, cio-0:1, cio-1:1 + * + * qp: 4 5 6 7 + * cio-x:y: cio-0:2, cio-1:2, cio-0:3, cio-1:3 + * + * In case just one engine is enabled mapping will look as + * follows: + * qp: 0 1 2 3 + * cio-x:y: cio-0:0, cio-0:1, cio-0:2, cio-0:3 + */ + n = snprintf(match, sizeof(match), "cio-%u:%u", + qp_id % num, qp_id / num); + + if (n >= sizeof(match)) + break; + + qp->cio_params.match = match; + qp->cio_params.size = qp_conf->nb_descriptors; + + if (sam_cio_init(&qp->cio_params, &qp->cio) < 0) + break; + + qp->sess_mp = session_pool; + + memset(&qp->stats, 0, sizeof(qp->stats)); + dev->data->queue_pairs[qp_id] = qp; + return 0; + } while (0); + + rte_free(qp); + return -1; +} + +/** Start queue pair (PMD ops callback) - not supported. + * + * @param dev Pointer to the device structure. + * @param qp_id ID of the Queue Pair. + * @returns -ENOTSUP. Always. + */ +static int +mrvl_crypto_pmd_qp_start(__rte_unused struct rte_cryptodev *dev, + __rte_unused uint16_t queue_pair_id) +{ + return -ENOTSUP; +} + +/** Stop queue pair (PMD ops callback) - not supported. + * + * @param dev Pointer to the device structure. + * @param qp_id ID of the Queue Pair. + * @returns -ENOTSUP. Always. + */ +static int +mrvl_crypto_pmd_qp_stop(__rte_unused struct rte_cryptodev *dev, + __rte_unused uint16_t queue_pair_id) +{ + return -ENOTSUP; +} + +/** Return the number of allocated queue pairs (PMD ops callback). + * + * @param dev Pointer to the device structure. + * @returns Number of allocated queue pairs. + */ +static uint32_t +mrvl_crypto_pmd_qp_count(struct rte_cryptodev *dev) +{ + return dev->data->nb_queue_pairs; +} + +/** Returns the size of the session structure (PMD ops callback). + * + * @param dev Pointer to the device structure [Unused]. + * @returns Size of Marvell crypto session. + */ +static unsigned +mrvl_crypto_pmd_session_get_size(__rte_unused struct rte_cryptodev *dev) +{ + return sizeof(struct mrvl_crypto_session); +} + +/** Configure the session from a crypto xform chain (PMD ops callback). + * + * @param dev Pointer to the device structure. + * @param xform Pointer to the crytpo configuration structure. + * @param sess Pointer to the empty session structure. + * @returns 0 upon success, negative value otherwise. + */ +static int +mrvl_crypto_pmd_session_configure(__rte_unused struct rte_cryptodev *dev, + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mp) +{ + struct mrvl_crypto_session *mrvl_sess; + void *sess_private_data; + int ret; + + if (sess == NULL) { + MRVL_CRYPTO_LOG_ERR("Invalid session struct."); + return -EINVAL; + } + + if (rte_mempool_get(mp, &sess_private_data)) { + CDEV_LOG_ERR("Couldn't get object from session mempool."); + return -ENOMEM; + } + + ret = mrvl_crypto_set_session_parameters(sess_private_data, xform); + if (ret != 0) { + MRVL_CRYPTO_LOG_ERR("Failed to configure session parameters."); + + /* Return session to mempool */ + rte_mempool_put(mp, sess_private_data); + return ret; + } + + set_session_private_data(sess, dev->driver_id, sess_private_data); + + mrvl_sess = (struct mrvl_crypto_session *)sess_private_data; + if (sam_session_create(&mrvl_sess->sam_sess_params, + &mrvl_sess->sam_sess) < 0) { + MRVL_CRYPTO_LOG_DBG("Failed to create session!"); + return -EIO; + } + + return 0; +} + +/** + * Clear the memory of session so it doesn't leave key material behind. + * + * @param dev Pointer to the device structure. + * @returns 0. Always. + */ +static void +mrvl_crypto_pmd_session_clear(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) +{ + + uint8_t index = dev->driver_id; + void *sess_priv = get_session_private_data(sess, index); + + /* Zero out the whole structure */ + if (sess_priv) { + struct mrvl_crypto_session *mrvl_sess = + (struct mrvl_crypto_session *)sess_priv; + + if (mrvl_sess->sam_sess && + sam_session_destroy(mrvl_sess->sam_sess) < 0) { + MRVL_CRYPTO_LOG_INFO("Error while destroying session!"); + } + + memset(sess, 0, sizeof(struct mrvl_crypto_session)); + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + set_session_private_data(sess, index, NULL); + rte_mempool_put(sess_mp, sess_priv); + } +} + +/** + * PMD handlers for crypto ops. + */ +static struct rte_cryptodev_ops mrvl_crypto_pmd_ops = { + .dev_configure = mrvl_crypto_pmd_config, + .dev_start = mrvl_crypto_pmd_start, + .dev_stop = mrvl_crypto_pmd_stop, + .dev_close = mrvl_crypto_pmd_close, + + .dev_infos_get = mrvl_crypto_pmd_info_get, + + .stats_get = mrvl_crypto_pmd_stats_get, + .stats_reset = mrvl_crypto_pmd_stats_reset, + + .queue_pair_setup = mrvl_crypto_pmd_qp_setup, + .queue_pair_release = mrvl_crypto_pmd_qp_release, + .queue_pair_start = mrvl_crypto_pmd_qp_start, + .queue_pair_stop = mrvl_crypto_pmd_qp_stop, + .queue_pair_count = mrvl_crypto_pmd_qp_count, + + .session_get_size = mrvl_crypto_pmd_session_get_size, + .session_configure = mrvl_crypto_pmd_session_configure, + .session_clear = mrvl_crypto_pmd_session_clear +}; + +struct rte_cryptodev_ops *rte_mrvl_crypto_pmd_ops = &mrvl_crypto_pmd_ops; diff --git a/dpdk/drivers/crypto/mrvl/rte_mrvl_pmd_private.h b/dpdk/drivers/crypto/mrvl/rte_mrvl_pmd_private.h new file mode 100644 index 00000000..923faaf9 --- /dev/null +++ b/dpdk/drivers/crypto/mrvl/rte_mrvl_pmd_private.h @@ -0,0 +1,123 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Marvell International Ltd. + * Copyright(c) 2017 Semihalf. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_MRVL_PMD_PRIVATE_H_ +#define _RTE_MRVL_PMD_PRIVATE_H_ + +#include "rte_mrvl_compat.h" + +#define CRYPTODEV_NAME_MRVL_PMD crypto_mrvl +/**< Marvell PMD device name */ + +#define MRVL_CRYPTO_LOG_ERR(fmt, args...) \ + RTE_LOG(ERR, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \ + RTE_STR(CRYPTODEV_NAME_MRVL_PMD), \ + __func__, __LINE__, ## args) + +#ifdef RTE_LIBRTE_PMD_MRVL_CRYPTO_DEBUG +#define MRVL_CRYPTO_LOG_INFO(fmt, args...) \ + RTE_LOG(INFO, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \ + RTE_STR(CRYPTODEV_NAME_MRVL_PMD), \ + __func__, __LINE__, ## args) + +#define MRVL_CRYPTO_LOG_DBG(fmt, args...) \ + RTE_LOG(DEBUG, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \ + RTE_STR(CRYPTODEV_NAME_MRVL_PMD), \ + __func__, __LINE__, ## args) + +#else +#define MRVL_CRYPTO_LOG_INFO(fmt, args...) +#define MRVL_CRYPTO_LOG_DBG(fmt, args...) +#endif + +/** + * Handy bits->bytes conversion macro. + */ +#define BITS2BYTES(x) ((x) >> 3) + +/** The operation order mode enumerator. */ +enum mrvl_crypto_chain_order { + MRVL_CRYPTO_CHAIN_CIPHER_ONLY, + MRVL_CRYPTO_CHAIN_AUTH_ONLY, + MRVL_CRYPTO_CHAIN_CIPHER_AUTH, + MRVL_CRYPTO_CHAIN_AUTH_CIPHER, + MRVL_CRYPTO_CHAIN_COMBINED, + MRVL_CRYPTO_CHAIN_NOT_SUPPORTED, +}; + +/** Private data structure for each crypto device. */ +struct mrvl_crypto_private { + unsigned int max_nb_qpairs; /**< Max number of queue pairs */ + unsigned int max_nb_sessions; /**< Max number of sessions */ +}; + +/** MRVL crypto queue pair structure. */ +struct mrvl_crypto_qp { + /** SAM CIO (MUSDK Queue Pair equivalent).*/ + struct sam_cio *cio; + + /** Session Mempool. */ + struct rte_mempool *sess_mp; + + /** Queue pair statistics. */ + struct rte_cryptodev_stats stats; + + /** CIO initialization parameters.*/ + struct sam_cio_params cio_params; +} __rte_cache_aligned; + +/** MRVL crypto private session structure. */ +struct mrvl_crypto_session { + /** Crypto operations chain order. */ + enum mrvl_crypto_chain_order chain_order; + + /** Session initialization parameters. */ + struct sam_session_params sam_sess_params; + + /** SAM session pointer. */ + struct sam_sa *sam_sess; + + /** Cipher IV offset. */ + uint16_t cipher_iv_offset; +} __rte_cache_aligned; + +/** Set and validate MRVL crypto session parameters */ +extern int +mrvl_crypto_set_session_parameters(struct mrvl_crypto_session *sess, + const struct rte_crypto_sym_xform *xform); + +/** device specific operations function pointer structure */ +extern struct rte_cryptodev_ops *rte_mrvl_crypto_pmd_ops; + +#endif /* _RTE_MRVL_PMD_PRIVATE_H_ */ diff --git a/dpdk/drivers/crypto/mrvl/rte_pmd_mrvl_version.map b/dpdk/drivers/crypto/mrvl/rte_pmd_mrvl_version.map new file mode 100644 index 00000000..a7530317 --- /dev/null +++ b/dpdk/drivers/crypto/mrvl/rte_pmd_mrvl_version.map @@ -0,0 +1,3 @@ +DPDK_17.11 { + local: *; +}; diff --git a/dpdk/drivers/crypto/null/Makefile b/dpdk/drivers/crypto/null/Makefile index c143929f..49ada097 100644 --- a/dpdk/drivers/crypto/null/Makefile +++ b/dpdk/drivers/crypto/null/Makefile @@ -37,6 +37,9 @@ LIB = librte_pmd_null_crypto.a # build flags CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_cryptodev +LDLIBS += -lrte_bus_vdev # library version LIBABIVER := 1 @@ -51,11 +54,4 @@ SRCS-$(CONFIG_RTE_LIBRTE_PMD_NULL_CRYPTO) += null_crypto_pmd_ops.c # export include files SYMLINK-y-include += -# library dependencies -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_NULL_CRYPTO) += lib/librte_eal -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_NULL_CRYPTO) += lib/librte_mbuf -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_NULL_CRYPTO) += lib/librte_mempool -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_NULL_CRYPTO) += lib/librte_ring -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_NULL_CRYPTO) += lib/librte_cryptodev - include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/crypto/null/null_crypto_pmd.c b/dpdk/drivers/crypto/null/null_crypto_pmd.c index 909b04f9..f031d3b8 100644 --- a/dpdk/drivers/crypto/null/null_crypto_pmd.c +++ b/dpdk/drivers/crypto/null/null_crypto_pmd.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -31,33 +31,13 @@ */ #include -#include #include -#include +#include #include #include "null_crypto_pmd_private.h" -/** - * Global static parameter used to create a unique name for each crypto device. - */ -static unsigned unique_name_id; - -static inline int -create_unique_device_name(char *name, size_t size) -{ - int ret; - - if (name == NULL) - return -EINVAL; - - ret = snprintf(name, size, "%s_%u", RTE_STR(CRYPTODEV_NAME_NULL_PMD), - unique_name_id++); - if (ret < 0) - return ret; - return 0; -} - +static uint8_t cryptodev_driver_id; /** verify and set session parameters */ int @@ -66,7 +46,7 @@ null_crypto_set_session_parameters( const struct rte_crypto_sym_xform *xform) { if (xform == NULL) { - return -1; + return -EINVAL; } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH && xform->next == NULL) { /* Authentication Only */ @@ -91,7 +71,7 @@ null_crypto_set_session_parameters( return 0; } - return -1; + return -ENOTSUP; } /** Process crypto operation for mbuf */ @@ -102,6 +82,14 @@ process_op(const struct null_crypto_qp *qp, struct rte_crypto_op *op, /* set status as successful by default */ op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; + /* Free session if a session-less crypto op. */ + if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { + memset(op->sym->session, 0, + sizeof(struct null_crypto_session)); + rte_cryptodev_sym_session_free(op->sym->session); + op->sym->session = NULL; + } + /* * if crypto session and operation are valid just enqueue the packet * in the processed ring @@ -110,26 +98,37 @@ process_op(const struct null_crypto_qp *qp, struct rte_crypto_op *op, } static struct null_crypto_session * -get_session(struct null_crypto_qp *qp, struct rte_crypto_sym_op *op) +get_session(struct null_crypto_qp *qp, struct rte_crypto_op *op) { - struct null_crypto_session *sess; + struct null_crypto_session *sess = NULL; + struct rte_crypto_sym_op *sym_op = op->sym; - if (op->sess_type == RTE_CRYPTO_SYM_OP_WITH_SESSION) { - if (unlikely(op->session == NULL || - op->session->dev_type != RTE_CRYPTODEV_NULL_PMD)) + if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { + if (likely(sym_op->session != NULL)) + sess = (struct null_crypto_session *) + get_session_private_data( + sym_op->session, cryptodev_driver_id); + } else { + void *_sess = NULL; + void *_sess_private_data = NULL; + + if (rte_mempool_get(qp->sess_mp, (void **)&_sess)) return NULL; - sess = (struct null_crypto_session *)op->session->_private; - } else { - struct rte_cryptodev_session *c_sess = NULL; - - if (rte_mempool_get(qp->sess_mp, (void **)&c_sess)) + if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data)) return NULL; - sess = (struct null_crypto_session *)c_sess->_private; + sess = (struct null_crypto_session *)_sess_private_data; - if (null_crypto_set_session_parameters(sess, op->xform) != 0) - return NULL; + if (unlikely(null_crypto_set_session_parameters(sess, + sym_op->xform) != 0)) { + rte_mempool_put(qp->sess_mp, _sess); + rte_mempool_put(qp->sess_mp, _sess_private_data); + sess = NULL; + } + sym_op->session = (struct rte_cryptodev_sym_session *)_sess; + set_session_private_data(sym_op->session, cryptodev_driver_id, + _sess_private_data); } return sess; @@ -146,7 +145,7 @@ null_crypto_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops, int i, retval; for (i = 0; i < nb_ops; i++) { - sess = get_session(qp, ops[i]->sym); + sess = get_session(qp, ops[i]); if (unlikely(sess == NULL)) goto enqueue_err; @@ -176,39 +175,28 @@ null_crypto_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops, unsigned nb_dequeued; nb_dequeued = rte_ring_dequeue_burst(qp->processed_pkts, - (void **)ops, nb_ops); + (void **)ops, nb_ops, NULL); qp->qp_stats.dequeued_count += nb_dequeued; return nb_dequeued; } -static int cryptodev_null_uninit(const char *name); - /** Create crypto device */ static int cryptodev_null_create(const char *name, - struct rte_crypto_vdev_init_params *init_params) + struct rte_vdev_device *vdev, + struct rte_cryptodev_pmd_init_params *init_params) { struct rte_cryptodev *dev; - char crypto_dev_name[RTE_CRYPTODEV_NAME_MAX_LEN]; struct null_crypto_private *internals; - /* create a unique device name */ - if (create_unique_device_name(crypto_dev_name, - RTE_CRYPTODEV_NAME_MAX_LEN) != 0) { - NULL_CRYPTO_LOG_ERR("failed to create unique cryptodev name"); - return -EINVAL; - } - - dev = rte_cryptodev_pmd_virtual_dev_init(crypto_dev_name, - sizeof(struct null_crypto_private), - init_params->socket_id); + dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params); if (dev == NULL) { NULL_CRYPTO_LOG_ERR("failed to create cryptodev vdev"); - goto init_error; + return -EFAULT; } - dev->dev_type = RTE_CRYPTODEV_NULL_PMD; + dev->driver_id = cryptodev_driver_id; dev->dev_ops = null_crypto_pmd_ops; /* register rx/tx burst functions for data path */ @@ -216,7 +204,8 @@ cryptodev_null_create(const char *name, dev->enqueue_burst = null_crypto_pmd_enqueue_burst; dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | - RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING; + RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | + RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER; internals = dev->data->dev_private; @@ -224,58 +213,67 @@ cryptodev_null_create(const char *name, internals->max_nb_sessions = init_params->max_nb_sessions; return 0; - -init_error: - NULL_CRYPTO_LOG_ERR("driver %s: cryptodev_null_create failed", name); - cryptodev_null_uninit(crypto_dev_name); - - return -EFAULT; } /** Initialise null crypto device */ static int -cryptodev_null_init(const char *name, - const char *input_args) +cryptodev_null_probe(struct rte_vdev_device *dev) { - struct rte_crypto_vdev_init_params init_params = { - RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_QUEUE_PAIRS, - RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_SESSIONS, - rte_socket_id() + struct rte_cryptodev_pmd_init_params init_params = { + "", + sizeof(struct null_crypto_private), + rte_socket_id(), + RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS, + RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_SESSIONS }; + const char *name, *args; + int retval; - rte_cryptodev_parse_vdev_init_params(&init_params, input_args); - - RTE_LOG(INFO, PMD, "Initialising %s on NUMA node %d\n", name, - init_params.socket_id); - RTE_LOG(INFO, PMD, " Max number of queue pairs = %d\n", - init_params.max_nb_queue_pairs); - RTE_LOG(INFO, PMD, " Max number of sessions = %d\n", - init_params.max_nb_sessions); - - return cryptodev_null_create(name, &init_params); -} - -/** Uninitialise null crypto device */ -static int -cryptodev_null_uninit(const char *name) -{ + name = rte_vdev_device_name(dev); if (name == NULL) return -EINVAL; - RTE_LOG(INFO, PMD, "Closing null crypto device %s on numa socket %u\n", - name, rte_socket_id()); + args = rte_vdev_device_args(dev); - return 0; + retval = rte_cryptodev_pmd_parse_input_args(&init_params, args); + if (retval) { + RTE_LOG(ERR, PMD, + "Failed to parse initialisation arguments[%s]\n", args); + return -EINVAL; + } + + return cryptodev_null_create(name, dev, &init_params); } -static struct rte_driver cryptodev_null_pmd_drv = { - .type = PMD_VDEV, - .init = cryptodev_null_init, - .uninit = cryptodev_null_uninit +static int +cryptodev_null_remove_dev(struct rte_vdev_device *vdev) +{ + struct rte_cryptodev *cryptodev; + const char *name; + + name = rte_vdev_device_name(vdev); + if (name == NULL) + return -EINVAL; + + cryptodev = rte_cryptodev_pmd_get_named_dev(name); + if (cryptodev == NULL) + return -ENODEV; + + return rte_cryptodev_pmd_destroy(cryptodev); +} + +static struct rte_vdev_driver cryptodev_null_pmd_drv = { + .probe = cryptodev_null_probe, + .remove = cryptodev_null_remove_dev, }; -PMD_REGISTER_DRIVER(cryptodev_null_pmd_drv, CRYPTODEV_NAME_NULL_PMD); -DRIVER_REGISTER_PARAM_STRING(CRYPTODEV_NAME_NULL_PMD, +static struct cryptodev_driver null_crypto_drv; + +RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_NULL_PMD, cryptodev_null_pmd_drv); +RTE_PMD_REGISTER_ALIAS(CRYPTODEV_NAME_NULL_PMD, cryptodev_null_pmd); +RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_NULL_PMD, "max_nb_queue_pairs= " "max_nb_sessions= " "socket_id="); +RTE_PMD_REGISTER_CRYPTO_DRIVER(null_crypto_drv, cryptodev_null_pmd_drv, + cryptodev_driver_id); diff --git a/dpdk/drivers/crypto/null/null_crypto_pmd_ops.c b/dpdk/drivers/crypto/null/null_crypto_pmd_ops.c index 26ff6319..c427050a 100644 --- a/dpdk/drivers/crypto/null/null_crypto_pmd_ops.c +++ b/dpdk/drivers/crypto/null/null_crypto_pmd_ops.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -56,7 +56,7 @@ static const struct rte_cryptodev_capabilities null_crypto_pmd_capabilities[] = .max = 0, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, }, }, }, }, @@ -72,11 +72,7 @@ static const struct rte_cryptodev_capabilities null_crypto_pmd_capabilities[] = .max = 0, .increment = 0 }, - .iv_size = { - .min = 0, - .max = 0, - .increment = 0 - } + .iv_size = { 0 } }, }, }, } }, @@ -85,7 +81,8 @@ static const struct rte_cryptodev_capabilities null_crypto_pmd_capabilities[] = /** Configure device */ static int -null_crypto_pmd_config(__rte_unused struct rte_cryptodev *dev) +null_crypto_pmd_config(__rte_unused struct rte_cryptodev *dev, + __rte_unused struct rte_cryptodev_config *config) { return 0; } @@ -150,7 +147,7 @@ null_crypto_pmd_info_get(struct rte_cryptodev *dev, struct null_crypto_private *internals = dev->data->dev_private; if (dev_info != NULL) { - dev_info->dev_type = dev->dev_type; + dev_info->driver_id = dev->driver_id; dev_info->max_nb_queue_pairs = internals->max_nb_qpairs; dev_info->sym.max_nb_sessions = internals->max_nb_sessions; dev_info->feature_flags = dev->feature_flags; @@ -178,7 +175,7 @@ null_crypto_pmd_qp_set_unique_name(struct rte_cryptodev *dev, "null_crypto_pmd_%u_qp_%u", dev->data->dev_id, qp->id); - if (n > sizeof(qp->name)) + if (n >= sizeof(qp->name)) return -1; return 0; @@ -193,7 +190,7 @@ null_crypto_pmd_qp_create_processed_pkts_ring(struct null_crypto_qp *qp, r = rte_ring_lookup(qp->name); if (r) { - if (r->prod.size >= ring_size) { + if (rte_ring_get_size(r) >= ring_size) { NULL_CRYPTO_LOG_INFO( "Reusing existing ring %s for processed packets", qp->name); @@ -214,7 +211,7 @@ null_crypto_pmd_qp_create_processed_pkts_ring(struct null_crypto_qp *qp, static int null_crypto_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, const struct rte_cryptodev_qp_conf *qp_conf, - int socket_id) + int socket_id, struct rte_mempool *session_pool) { struct null_crypto_private *internals = dev->data->dev_private; struct null_crypto_qp *qp; @@ -257,7 +254,7 @@ null_crypto_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, goto qp_setup_cleanup; } - qp->sess_mp = dev->data->session_pool; + qp->sess_mp = session_pool; memset(&qp->qp_stats, 0, sizeof(qp->qp_stats)); @@ -301,33 +298,56 @@ null_crypto_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused) } /** Configure a null crypto session from a crypto xform chain */ -static void * +static int null_crypto_pmd_session_configure(struct rte_cryptodev *dev __rte_unused, - struct rte_crypto_sym_xform *xform, void *sess) + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mp) { - int retval; + void *sess_private_data; + int ret; if (unlikely(sess == NULL)) { NULL_CRYPTO_LOG_ERR("invalid session struct"); - return NULL; - } - retval = null_crypto_set_session_parameters( - (struct null_crypto_session *)sess, xform); - if (retval != 0) { - NULL_CRYPTO_LOG_ERR("failed configure session parameters"); - return NULL; + return -EINVAL; } - return sess; + if (rte_mempool_get(mp, &sess_private_data)) { + CDEV_LOG_ERR( + "Couldn't get object from session mempool"); + return -ENOMEM; + } + + ret = null_crypto_set_session_parameters(sess_private_data, xform); + if (ret != 0) { + NULL_CRYPTO_LOG_ERR("failed configure session parameters"); + + /* Return session to mempool */ + rte_mempool_put(mp, sess_private_data); + return ret; + } + + set_session_private_data(sess, dev->driver_id, + sess_private_data); + + return 0; } /** Clear the memory of session so it doesn't leave key material behind */ static void -null_crypto_pmd_session_clear(struct rte_cryptodev *dev __rte_unused, - void *sess) +null_crypto_pmd_session_clear(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) { - if (sess) - memset(sess, 0, sizeof(struct null_crypto_session)); + uint8_t index = dev->driver_id; + void *sess_priv = get_session_private_data(sess, index); + + /* Zero out the whole structure */ + if (sess_priv) { + memset(sess_priv, 0, sizeof(struct null_crypto_session)); + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + set_session_private_data(sess, index, NULL); + rte_mempool_put(sess_mp, sess_priv); + } } struct rte_cryptodev_ops pmd_ops = { diff --git a/dpdk/drivers/crypto/null/null_crypto_pmd_private.h b/dpdk/drivers/crypto/null/null_crypto_pmd_private.h index acebc973..7c7852fb 100644 --- a/dpdk/drivers/crypto/null/null_crypto_pmd_private.h +++ b/dpdk/drivers/crypto/null/null_crypto_pmd_private.h @@ -33,7 +33,8 @@ #ifndef _NULL_CRYPTO_PMD_PRIVATE_H_ #define _NULL_CRYPTO_PMD_PRIVATE_H_ -#include "rte_config.h" +#define CRYPTODEV_NAME_NULL_PMD crypto_null +/**< Null crypto PMD device name */ #define NULL_CRYPTO_LOG_ERR(fmt, args...) \ RTE_LOG(ERR, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \ diff --git a/dpdk/drivers/crypto/openssl/Makefile b/dpdk/drivers/crypto/openssl/Makefile new file mode 100644 index 00000000..1a006432 --- /dev/null +++ b/dpdk/drivers/crypto/openssl/Makefile @@ -0,0 +1,56 @@ +# BSD LICENSE +# +# Copyright(c) 2016 Intel Corporation. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +include $(RTE_SDK)/mk/rte.vars.mk + +# library name +LIB = librte_pmd_openssl.a + +# build flags +CFLAGS += -O3 +CFLAGS += $(WERROR_FLAGS) + +# library version +LIBABIVER := 1 + +# versioning export map +EXPORT_MAP := rte_pmd_openssl_version.map + +# external library dependencies +LDLIBS += -lcrypto +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_cryptodev +LDLIBS += -lrte_bus_vdev + +# library source files +SRCS-$(CONFIG_RTE_LIBRTE_PMD_OPENSSL) += rte_openssl_pmd.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_OPENSSL) += rte_openssl_pmd_ops.c + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/crypto/openssl/rte_openssl_pmd.c b/dpdk/drivers/crypto/openssl/rte_openssl_pmd.c new file mode 100644 index 00000000..06e1a6de --- /dev/null +++ b/dpdk/drivers/crypto/openssl/rte_openssl_pmd.c @@ -0,0 +1,1765 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "rte_openssl_pmd_private.h" + +#define DES_BLOCK_SIZE 8 + +static uint8_t cryptodev_driver_id; + +#if (OPENSSL_VERSION_NUMBER < 0x10100000L) +static HMAC_CTX *HMAC_CTX_new(void) +{ + HMAC_CTX *ctx = OPENSSL_malloc(sizeof(*ctx)); + + if (ctx != NULL) + HMAC_CTX_init(ctx); + return ctx; +} + +static void HMAC_CTX_free(HMAC_CTX *ctx) +{ + if (ctx != NULL) { + HMAC_CTX_cleanup(ctx); + OPENSSL_free(ctx); + } +} +#endif + +static int cryptodev_openssl_remove(struct rte_vdev_device *vdev); + +/*----------------------------------------------------------------------------*/ + +/** + * Increment counter by 1 + * Counter is 64 bit array, big-endian + */ +static void +ctr_inc(uint8_t *ctr) +{ + uint64_t *ctr64 = (uint64_t *)ctr; + + *ctr64 = __builtin_bswap64(*ctr64); + (*ctr64)++; + *ctr64 = __builtin_bswap64(*ctr64); +} + +/* + *------------------------------------------------------------------------------ + * Session Prepare + *------------------------------------------------------------------------------ + */ + +/** Get xform chain order */ +static enum openssl_chain_order +openssl_get_chain_order(const struct rte_crypto_sym_xform *xform) +{ + enum openssl_chain_order res = OPENSSL_CHAIN_NOT_SUPPORTED; + + if (xform != NULL) { + if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) { + if (xform->next == NULL) + res = OPENSSL_CHAIN_ONLY_AUTH; + else if (xform->next->type == + RTE_CRYPTO_SYM_XFORM_CIPHER) + res = OPENSSL_CHAIN_AUTH_CIPHER; + } + if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { + if (xform->next == NULL) + res = OPENSSL_CHAIN_ONLY_CIPHER; + else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) + res = OPENSSL_CHAIN_CIPHER_AUTH; + } + if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) + res = OPENSSL_CHAIN_COMBINED; + } + + return res; +} + +/** Get session cipher key from input cipher key */ +static void +get_cipher_key(uint8_t *input_key, int keylen, uint8_t *session_key) +{ + memcpy(session_key, input_key, keylen); +} + +/** Get key ede 24 bytes standard from input key */ +static int +get_cipher_key_ede(uint8_t *key, int keylen, uint8_t *key_ede) +{ + int res = 0; + + /* Initialize keys - 24 bytes: [key1-key2-key3] */ + switch (keylen) { + case 24: + memcpy(key_ede, key, 24); + break; + case 16: + /* K3 = K1 */ + memcpy(key_ede, key, 16); + memcpy(key_ede + 16, key, 8); + break; + case 8: + /* K1 = K2 = K3 (DES compatibility) */ + memcpy(key_ede, key, 8); + memcpy(key_ede + 8, key, 8); + memcpy(key_ede + 16, key, 8); + break; + default: + OPENSSL_LOG_ERR("Unsupported key size"); + res = -EINVAL; + } + + return res; +} + +/** Get adequate openssl function for input cipher algorithm */ +static uint8_t +get_cipher_algo(enum rte_crypto_cipher_algorithm sess_algo, size_t keylen, + const EVP_CIPHER **algo) +{ + int res = 0; + + if (algo != NULL) { + switch (sess_algo) { + case RTE_CRYPTO_CIPHER_3DES_CBC: + switch (keylen) { + case 16: + *algo = EVP_des_ede_cbc(); + break; + case 24: + *algo = EVP_des_ede3_cbc(); + break; + default: + res = -EINVAL; + } + break; + case RTE_CRYPTO_CIPHER_3DES_CTR: + break; + case RTE_CRYPTO_CIPHER_AES_CBC: + switch (keylen) { + case 16: + *algo = EVP_aes_128_cbc(); + break; + case 24: + *algo = EVP_aes_192_cbc(); + break; + case 32: + *algo = EVP_aes_256_cbc(); + break; + default: + res = -EINVAL; + } + break; + case RTE_CRYPTO_CIPHER_AES_CTR: + switch (keylen) { + case 16: + *algo = EVP_aes_128_ctr(); + break; + case 24: + *algo = EVP_aes_192_ctr(); + break; + case 32: + *algo = EVP_aes_256_ctr(); + break; + default: + res = -EINVAL; + } + break; + default: + res = -EINVAL; + break; + } + } else { + res = -EINVAL; + } + + return res; +} + +/** Get adequate openssl function for input auth algorithm */ +static uint8_t +get_auth_algo(enum rte_crypto_auth_algorithm sessalgo, + const EVP_MD **algo) +{ + int res = 0; + + if (algo != NULL) { + switch (sessalgo) { + case RTE_CRYPTO_AUTH_MD5: + case RTE_CRYPTO_AUTH_MD5_HMAC: + *algo = EVP_md5(); + break; + case RTE_CRYPTO_AUTH_SHA1: + case RTE_CRYPTO_AUTH_SHA1_HMAC: + *algo = EVP_sha1(); + break; + case RTE_CRYPTO_AUTH_SHA224: + case RTE_CRYPTO_AUTH_SHA224_HMAC: + *algo = EVP_sha224(); + break; + case RTE_CRYPTO_AUTH_SHA256: + case RTE_CRYPTO_AUTH_SHA256_HMAC: + *algo = EVP_sha256(); + break; + case RTE_CRYPTO_AUTH_SHA384: + case RTE_CRYPTO_AUTH_SHA384_HMAC: + *algo = EVP_sha384(); + break; + case RTE_CRYPTO_AUTH_SHA512: + case RTE_CRYPTO_AUTH_SHA512_HMAC: + *algo = EVP_sha512(); + break; + default: + res = -EINVAL; + break; + } + } else { + res = -EINVAL; + } + + return res; +} + +/** Get adequate openssl function for input cipher algorithm */ +static uint8_t +get_aead_algo(enum rte_crypto_aead_algorithm sess_algo, size_t keylen, + const EVP_CIPHER **algo) +{ + int res = 0; + + if (algo != NULL) { + switch (sess_algo) { + case RTE_CRYPTO_AEAD_AES_GCM: + switch (keylen) { + case 16: + *algo = EVP_aes_128_gcm(); + break; + case 24: + *algo = EVP_aes_192_gcm(); + break; + case 32: + *algo = EVP_aes_256_gcm(); + break; + default: + res = -EINVAL; + } + break; + case RTE_CRYPTO_AEAD_AES_CCM: + switch (keylen) { + case 16: + *algo = EVP_aes_128_ccm(); + break; + case 24: + *algo = EVP_aes_192_ccm(); + break; + case 32: + *algo = EVP_aes_256_ccm(); + break; + default: + res = -EINVAL; + } + break; + default: + res = -EINVAL; + break; + } + } else { + res = -EINVAL; + } + + return res; +} + +/* Set session AEAD encryption parameters */ +static int +openssl_set_sess_aead_enc_param(struct openssl_session *sess, + enum rte_crypto_aead_algorithm algo, + uint8_t tag_len, uint8_t *key) +{ + int iv_type = 0; + unsigned int do_ccm; + + sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_ENCRYPT; + sess->auth.operation = RTE_CRYPTO_AUTH_OP_GENERATE; + + /* Select AEAD algo */ + switch (algo) { + case RTE_CRYPTO_AEAD_AES_GCM: + iv_type = EVP_CTRL_GCM_SET_IVLEN; + if (tag_len != 16) + return -EINVAL; + do_ccm = 0; + break; + case RTE_CRYPTO_AEAD_AES_CCM: + iv_type = EVP_CTRL_CCM_SET_IVLEN; + /* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */ + if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1) + return -EINVAL; + do_ccm = 1; + break; + default: + return -ENOTSUP; + } + + sess->cipher.mode = OPENSSL_CIPHER_LIB; + sess->cipher.ctx = EVP_CIPHER_CTX_new(); + + if (get_aead_algo(algo, sess->cipher.key.length, + &sess->cipher.evp_algo) != 0) + return -EINVAL; + + get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data); + + sess->chain_order = OPENSSL_CHAIN_COMBINED; + + if (EVP_EncryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo, + NULL, NULL, NULL) <= 0) + return -EINVAL; + + if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type, sess->iv.length, + NULL) <= 0) + return -EINVAL; + + if (do_ccm) + EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, EVP_CTRL_CCM_SET_TAG, + tag_len, NULL); + + if (EVP_EncryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0) + return -EINVAL; + + return 0; +} + +/* Set session AEAD decryption parameters */ +static int +openssl_set_sess_aead_dec_param(struct openssl_session *sess, + enum rte_crypto_aead_algorithm algo, + uint8_t tag_len, uint8_t *key) +{ + int iv_type = 0; + unsigned int do_ccm = 0; + + sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_DECRYPT; + sess->auth.operation = RTE_CRYPTO_AUTH_OP_VERIFY; + + /* Select AEAD algo */ + switch (algo) { + case RTE_CRYPTO_AEAD_AES_GCM: + iv_type = EVP_CTRL_GCM_SET_IVLEN; + if (tag_len != 16) + return -EINVAL; + break; + case RTE_CRYPTO_AEAD_AES_CCM: + iv_type = EVP_CTRL_CCM_SET_IVLEN; + /* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */ + if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1) + return -EINVAL; + do_ccm = 1; + break; + default: + return -ENOTSUP; + } + + sess->cipher.mode = OPENSSL_CIPHER_LIB; + sess->cipher.ctx = EVP_CIPHER_CTX_new(); + + if (get_aead_algo(algo, sess->cipher.key.length, + &sess->cipher.evp_algo) != 0) + return -EINVAL; + + get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data); + + sess->chain_order = OPENSSL_CHAIN_COMBINED; + + if (EVP_DecryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo, + NULL, NULL, NULL) <= 0) + return -EINVAL; + + if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type, + sess->iv.length, NULL) <= 0) + return -EINVAL; + + if (do_ccm) + EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, EVP_CTRL_CCM_SET_TAG, + tag_len, NULL); + + if (EVP_DecryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0) + return -EINVAL; + + return 0; +} + +/** Set session cipher parameters */ +static int +openssl_set_session_cipher_parameters(struct openssl_session *sess, + const struct rte_crypto_sym_xform *xform) +{ + /* Select cipher direction */ + sess->cipher.direction = xform->cipher.op; + /* Select cipher key */ + sess->cipher.key.length = xform->cipher.key.length; + + /* Set IV parameters */ + sess->iv.offset = xform->cipher.iv.offset; + sess->iv.length = xform->cipher.iv.length; + + /* Select cipher algo */ + switch (xform->cipher.algo) { + case RTE_CRYPTO_CIPHER_3DES_CBC: + case RTE_CRYPTO_CIPHER_AES_CBC: + case RTE_CRYPTO_CIPHER_AES_CTR: + sess->cipher.mode = OPENSSL_CIPHER_LIB; + sess->cipher.algo = xform->cipher.algo; + sess->cipher.ctx = EVP_CIPHER_CTX_new(); + + if (get_cipher_algo(sess->cipher.algo, sess->cipher.key.length, + &sess->cipher.evp_algo) != 0) + return -EINVAL; + + get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, + sess->cipher.key.data); + if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { + if (EVP_EncryptInit_ex(sess->cipher.ctx, + sess->cipher.evp_algo, + NULL, xform->cipher.key.data, + NULL) != 1) { + return -EINVAL; + } + } else if (sess->cipher.direction == + RTE_CRYPTO_CIPHER_OP_DECRYPT) { + if (EVP_DecryptInit_ex(sess->cipher.ctx, + sess->cipher.evp_algo, + NULL, xform->cipher.key.data, + NULL) != 1) { + return -EINVAL; + } + } + + break; + + case RTE_CRYPTO_CIPHER_3DES_CTR: + sess->cipher.mode = OPENSSL_CIPHER_DES3CTR; + sess->cipher.ctx = EVP_CIPHER_CTX_new(); + + if (get_cipher_key_ede(xform->cipher.key.data, + sess->cipher.key.length, + sess->cipher.key.data) != 0) + return -EINVAL; + break; + + case RTE_CRYPTO_CIPHER_DES_CBC: + sess->cipher.algo = xform->cipher.algo; + sess->cipher.ctx = EVP_CIPHER_CTX_new(); + sess->cipher.evp_algo = EVP_des_cbc(); + + get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, + sess->cipher.key.data); + if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { + if (EVP_EncryptInit_ex(sess->cipher.ctx, + sess->cipher.evp_algo, + NULL, xform->cipher.key.data, + NULL) != 1) { + return -EINVAL; + } + } else if (sess->cipher.direction == + RTE_CRYPTO_CIPHER_OP_DECRYPT) { + if (EVP_DecryptInit_ex(sess->cipher.ctx, + sess->cipher.evp_algo, + NULL, xform->cipher.key.data, + NULL) != 1) { + return -EINVAL; + } + } + + break; + + case RTE_CRYPTO_CIPHER_DES_DOCSISBPI: + sess->cipher.algo = xform->cipher.algo; + sess->chain_order = OPENSSL_CHAIN_CIPHER_BPI; + sess->cipher.ctx = EVP_CIPHER_CTX_new(); + sess->cipher.evp_algo = EVP_des_cbc(); + + sess->cipher.bpi_ctx = EVP_CIPHER_CTX_new(); + /* IV will be ECB encrypted whether direction is encrypt or decrypt */ + if (EVP_EncryptInit_ex(sess->cipher.bpi_ctx, EVP_des_ecb(), + NULL, xform->cipher.key.data, 0) != 1) + return -EINVAL; + + get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, + sess->cipher.key.data); + if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { + if (EVP_EncryptInit_ex(sess->cipher.ctx, + sess->cipher.evp_algo, + NULL, xform->cipher.key.data, + NULL) != 1) { + return -EINVAL; + } + } else if (sess->cipher.direction == + RTE_CRYPTO_CIPHER_OP_DECRYPT) { + if (EVP_DecryptInit_ex(sess->cipher.ctx, + sess->cipher.evp_algo, + NULL, xform->cipher.key.data, + NULL) != 1) { + return -EINVAL; + } + } + + break; + default: + sess->cipher.algo = RTE_CRYPTO_CIPHER_NULL; + return -ENOTSUP; + } + + return 0; +} + +/* Set session auth parameters */ +static int +openssl_set_session_auth_parameters(struct openssl_session *sess, + const struct rte_crypto_sym_xform *xform) +{ + /* Select auth generate/verify */ + sess->auth.operation = xform->auth.op; + sess->auth.algo = xform->auth.algo; + + sess->auth.digest_length = xform->auth.digest_length; + + /* Select auth algo */ + switch (xform->auth.algo) { + case RTE_CRYPTO_AUTH_AES_GMAC: + /* + * OpenSSL requires GMAC to be a GCM operation + * with no cipher data length + */ + sess->cipher.key.length = xform->auth.key.length; + + /* Set IV parameters */ + sess->iv.offset = xform->auth.iv.offset; + sess->iv.length = xform->auth.iv.length; + + if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_GENERATE) + return openssl_set_sess_aead_enc_param(sess, + RTE_CRYPTO_AEAD_AES_GCM, + xform->auth.digest_length, + xform->auth.key.data); + else + return openssl_set_sess_aead_dec_param(sess, + RTE_CRYPTO_AEAD_AES_GCM, + xform->auth.digest_length, + xform->auth.key.data); + break; + + case RTE_CRYPTO_AUTH_MD5: + case RTE_CRYPTO_AUTH_SHA1: + case RTE_CRYPTO_AUTH_SHA224: + case RTE_CRYPTO_AUTH_SHA256: + case RTE_CRYPTO_AUTH_SHA384: + case RTE_CRYPTO_AUTH_SHA512: + sess->auth.mode = OPENSSL_AUTH_AS_AUTH; + if (get_auth_algo(xform->auth.algo, + &sess->auth.auth.evp_algo) != 0) + return -EINVAL; + sess->auth.auth.ctx = EVP_MD_CTX_create(); + break; + + case RTE_CRYPTO_AUTH_MD5_HMAC: + case RTE_CRYPTO_AUTH_SHA1_HMAC: + case RTE_CRYPTO_AUTH_SHA224_HMAC: + case RTE_CRYPTO_AUTH_SHA256_HMAC: + case RTE_CRYPTO_AUTH_SHA384_HMAC: + case RTE_CRYPTO_AUTH_SHA512_HMAC: + sess->auth.mode = OPENSSL_AUTH_AS_HMAC; + sess->auth.hmac.ctx = HMAC_CTX_new(); + if (get_auth_algo(xform->auth.algo, + &sess->auth.hmac.evp_algo) != 0) + return -EINVAL; + + if (HMAC_Init_ex(sess->auth.hmac.ctx, + xform->auth.key.data, + xform->auth.key.length, + sess->auth.hmac.evp_algo, NULL) != 1) + return -EINVAL; + break; + + default: + return -ENOTSUP; + } + + return 0; +} + +/* Set session AEAD parameters */ +static int +openssl_set_session_aead_parameters(struct openssl_session *sess, + const struct rte_crypto_sym_xform *xform) +{ + /* Select cipher key */ + sess->cipher.key.length = xform->aead.key.length; + + /* Set IV parameters */ + if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM) + /* + * For AES-CCM, the actual IV is placed + * one byte after the start of the IV field, + * according to the API. + */ + sess->iv.offset = xform->aead.iv.offset + 1; + else + sess->iv.offset = xform->aead.iv.offset; + + sess->iv.length = xform->aead.iv.length; + + sess->auth.aad_length = xform->aead.aad_length; + sess->auth.digest_length = xform->aead.digest_length; + + sess->aead_algo = xform->aead.algo; + /* Select cipher direction */ + if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) + return openssl_set_sess_aead_enc_param(sess, xform->aead.algo, + xform->aead.digest_length, xform->aead.key.data); + else + return openssl_set_sess_aead_dec_param(sess, xform->aead.algo, + xform->aead.digest_length, xform->aead.key.data); +} + +/** Parse crypto xform chain and set private session parameters */ +int +openssl_set_session_parameters(struct openssl_session *sess, + const struct rte_crypto_sym_xform *xform) +{ + const struct rte_crypto_sym_xform *cipher_xform = NULL; + const struct rte_crypto_sym_xform *auth_xform = NULL; + const struct rte_crypto_sym_xform *aead_xform = NULL; + int ret; + + sess->chain_order = openssl_get_chain_order(xform); + switch (sess->chain_order) { + case OPENSSL_CHAIN_ONLY_CIPHER: + cipher_xform = xform; + break; + case OPENSSL_CHAIN_ONLY_AUTH: + auth_xform = xform; + break; + case OPENSSL_CHAIN_CIPHER_AUTH: + cipher_xform = xform; + auth_xform = xform->next; + break; + case OPENSSL_CHAIN_AUTH_CIPHER: + auth_xform = xform; + cipher_xform = xform->next; + break; + case OPENSSL_CHAIN_COMBINED: + aead_xform = xform; + break; + default: + return -EINVAL; + } + + /* Default IV length = 0 */ + sess->iv.length = 0; + + /* cipher_xform must be check before auth_xform */ + if (cipher_xform) { + ret = openssl_set_session_cipher_parameters( + sess, cipher_xform); + if (ret != 0) { + OPENSSL_LOG_ERR( + "Invalid/unsupported cipher parameters"); + return ret; + } + } + + if (auth_xform) { + ret = openssl_set_session_auth_parameters(sess, auth_xform); + if (ret != 0) { + OPENSSL_LOG_ERR( + "Invalid/unsupported auth parameters"); + return ret; + } + } + + if (aead_xform) { + ret = openssl_set_session_aead_parameters(sess, aead_xform); + if (ret != 0) { + OPENSSL_LOG_ERR( + "Invalid/unsupported AEAD parameters"); + return ret; + } + } + + return 0; +} + +/** Reset private session parameters */ +void +openssl_reset_session(struct openssl_session *sess) +{ + EVP_CIPHER_CTX_free(sess->cipher.ctx); + + if (sess->chain_order == OPENSSL_CHAIN_CIPHER_BPI) + EVP_CIPHER_CTX_free(sess->cipher.bpi_ctx); + + switch (sess->auth.mode) { + case OPENSSL_AUTH_AS_AUTH: + EVP_MD_CTX_destroy(sess->auth.auth.ctx); + break; + case OPENSSL_AUTH_AS_HMAC: + EVP_PKEY_free(sess->auth.hmac.pkey); + HMAC_CTX_free(sess->auth.hmac.ctx); + break; + default: + break; + } +} + +/** Provide session for operation */ +static struct openssl_session * +get_session(struct openssl_qp *qp, struct rte_crypto_op *op) +{ + struct openssl_session *sess = NULL; + + if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { + /* get existing session */ + if (likely(op->sym->session != NULL)) + sess = (struct openssl_session *) + get_session_private_data( + op->sym->session, + cryptodev_driver_id); + } else { + /* provide internal session */ + void *_sess = NULL; + void *_sess_private_data = NULL; + + if (rte_mempool_get(qp->sess_mp, (void **)&_sess)) + return NULL; + + if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data)) + return NULL; + + sess = (struct openssl_session *)_sess_private_data; + + if (unlikely(openssl_set_session_parameters(sess, + op->sym->xform) != 0)) { + rte_mempool_put(qp->sess_mp, _sess); + rte_mempool_put(qp->sess_mp, _sess_private_data); + sess = NULL; + } + op->sym->session = (struct rte_cryptodev_sym_session *)_sess; + set_session_private_data(op->sym->session, cryptodev_driver_id, + _sess_private_data); + } + + if (sess == NULL) + op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; + + return sess; +} + +/* + *------------------------------------------------------------------------------ + * Process Operations + *------------------------------------------------------------------------------ + */ +static inline int +process_openssl_encryption_update(struct rte_mbuf *mbuf_src, int offset, + uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx) +{ + struct rte_mbuf *m; + int dstlen; + int l, n = srclen; + uint8_t *src; + + for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); + m = m->next) + offset -= rte_pktmbuf_data_len(m); + + if (m == 0) + return -1; + + src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); + + l = rte_pktmbuf_data_len(m) - offset; + if (srclen <= l) { + if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0) + return -1; + *dst += l; + return 0; + } + + if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) + return -1; + + *dst += dstlen; + n -= l; + + for (m = m->next; (m != NULL) && (n > 0); m = m->next) { + src = rte_pktmbuf_mtod(m, uint8_t *); + l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; + if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) + return -1; + *dst += dstlen; + n -= l; + } + + return 0; +} + +static inline int +process_openssl_decryption_update(struct rte_mbuf *mbuf_src, int offset, + uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx) +{ + struct rte_mbuf *m; + int dstlen; + int l, n = srclen; + uint8_t *src; + + for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); + m = m->next) + offset -= rte_pktmbuf_data_len(m); + + if (m == 0) + return -1; + + src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); + + l = rte_pktmbuf_data_len(m) - offset; + if (srclen <= l) { + if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0) + return -1; + *dst += l; + return 0; + } + + if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) + return -1; + + *dst += dstlen; + n -= l; + + for (m = m->next; (m != NULL) && (n > 0); m = m->next) { + src = rte_pktmbuf_mtod(m, uint8_t *); + l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; + if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) + return -1; + *dst += dstlen; + n -= l; + } + + return 0; +} + +/** Process standard openssl cipher encryption */ +static int +process_openssl_cipher_encrypt(struct rte_mbuf *mbuf_src, uint8_t *dst, + int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx) +{ + int totlen; + + if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) + goto process_cipher_encrypt_err; + + EVP_CIPHER_CTX_set_padding(ctx, 0); + + if (process_openssl_encryption_update(mbuf_src, offset, &dst, + srclen, ctx)) + goto process_cipher_encrypt_err; + + if (EVP_EncryptFinal_ex(ctx, dst, &totlen) <= 0) + goto process_cipher_encrypt_err; + + return 0; + +process_cipher_encrypt_err: + OPENSSL_LOG_ERR("Process openssl cipher encrypt failed"); + return -EINVAL; +} + +/** Process standard openssl cipher encryption */ +static int +process_openssl_cipher_bpi_encrypt(uint8_t *src, uint8_t *dst, + uint8_t *iv, int srclen, + EVP_CIPHER_CTX *ctx) +{ + uint8_t i; + uint8_t encrypted_iv[DES_BLOCK_SIZE]; + int encrypted_ivlen; + + if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, + iv, DES_BLOCK_SIZE) <= 0) + goto process_cipher_encrypt_err; + + for (i = 0; i < srclen; i++) + *(dst + i) = *(src + i) ^ (encrypted_iv[i]); + + return 0; + +process_cipher_encrypt_err: + OPENSSL_LOG_ERR("Process openssl cipher bpi encrypt failed"); + return -EINVAL; +} +/** Process standard openssl cipher decryption */ +static int +process_openssl_cipher_decrypt(struct rte_mbuf *mbuf_src, uint8_t *dst, + int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx) +{ + int totlen; + + if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) + goto process_cipher_decrypt_err; + + EVP_CIPHER_CTX_set_padding(ctx, 0); + + if (process_openssl_decryption_update(mbuf_src, offset, &dst, + srclen, ctx)) + goto process_cipher_decrypt_err; + + if (EVP_DecryptFinal_ex(ctx, dst, &totlen) <= 0) + goto process_cipher_decrypt_err; + return 0; + +process_cipher_decrypt_err: + OPENSSL_LOG_ERR("Process openssl cipher decrypt failed"); + return -EINVAL; +} + +/** Process cipher des 3 ctr encryption, decryption algorithm */ +static int +process_openssl_cipher_des3ctr(struct rte_mbuf *mbuf_src, uint8_t *dst, + int offset, uint8_t *iv, uint8_t *key, int srclen, + EVP_CIPHER_CTX *ctx) +{ + uint8_t ebuf[8], ctr[8]; + int unused, n; + struct rte_mbuf *m; + uint8_t *src; + int l; + + for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); + m = m->next) + offset -= rte_pktmbuf_data_len(m); + + if (m == 0) + goto process_cipher_des3ctr_err; + + src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); + l = rte_pktmbuf_data_len(m) - offset; + + /* We use 3DES encryption also for decryption. + * IV is not important for 3DES ecb + */ + if (EVP_EncryptInit_ex(ctx, EVP_des_ede3_ecb(), NULL, key, NULL) <= 0) + goto process_cipher_des3ctr_err; + + memcpy(ctr, iv, 8); + + for (n = 0; n < srclen; n++) { + if (n % 8 == 0) { + if (EVP_EncryptUpdate(ctx, + (unsigned char *)&ebuf, &unused, + (const unsigned char *)&ctr, 8) <= 0) + goto process_cipher_des3ctr_err; + ctr_inc(ctr); + } + dst[n] = *(src++) ^ ebuf[n % 8]; + + l--; + if (!l) { + m = m->next; + if (m) { + src = rte_pktmbuf_mtod(m, uint8_t *); + l = rte_pktmbuf_data_len(m); + } + } + } + + return 0; + +process_cipher_des3ctr_err: + OPENSSL_LOG_ERR("Process openssl cipher des 3 ede ctr failed"); + return -EINVAL; +} + +/** Process AES-GCM encrypt algorithm */ +static int +process_openssl_auth_encryption_gcm(struct rte_mbuf *mbuf_src, int offset, + int srclen, uint8_t *aad, int aadlen, uint8_t *iv, + uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx) +{ + int len = 0, unused = 0; + uint8_t empty[] = {}; + + if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) + goto process_auth_encryption_gcm_err; + + if (aadlen > 0) + if (EVP_EncryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0) + goto process_auth_encryption_gcm_err; + + if (srclen > 0) + if (process_openssl_encryption_update(mbuf_src, offset, &dst, + srclen, ctx)) + goto process_auth_encryption_gcm_err; + + /* Workaround open ssl bug in version less then 1.0.1f */ + if (EVP_EncryptUpdate(ctx, empty, &unused, empty, 0) <= 0) + goto process_auth_encryption_gcm_err; + + if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0) + goto process_auth_encryption_gcm_err; + + if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, tag) <= 0) + goto process_auth_encryption_gcm_err; + + return 0; + +process_auth_encryption_gcm_err: + OPENSSL_LOG_ERR("Process openssl auth encryption gcm failed"); + return -EINVAL; +} + +/** Process AES-CCM encrypt algorithm */ +static int +process_openssl_auth_encryption_ccm(struct rte_mbuf *mbuf_src, int offset, + int srclen, uint8_t *aad, int aadlen, uint8_t *iv, + uint8_t *dst, uint8_t *tag, uint8_t taglen, EVP_CIPHER_CTX *ctx) +{ + int len = 0; + + if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) + goto process_auth_encryption_ccm_err; + + if (EVP_EncryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0) + goto process_auth_encryption_ccm_err; + + if (aadlen > 0) + /* + * For AES-CCM, the actual AAD is placed + * 18 bytes after the start of the AAD field, + * according to the API. + */ + if (EVP_EncryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0) + goto process_auth_encryption_ccm_err; + + if (srclen > 0) + if (process_openssl_encryption_update(mbuf_src, offset, &dst, + srclen, ctx)) + goto process_auth_encryption_ccm_err; + + if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0) + goto process_auth_encryption_ccm_err; + + if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_GET_TAG, taglen, tag) <= 0) + goto process_auth_encryption_ccm_err; + + return 0; + +process_auth_encryption_ccm_err: + OPENSSL_LOG_ERR("Process openssl auth encryption ccm failed"); + return -EINVAL; +} + +/** Process AES-GCM decrypt algorithm */ +static int +process_openssl_auth_decryption_gcm(struct rte_mbuf *mbuf_src, int offset, + int srclen, uint8_t *aad, int aadlen, uint8_t *iv, + uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx) +{ + int len = 0, unused = 0; + uint8_t empty[] = {}; + + if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, 16, tag) <= 0) + goto process_auth_decryption_gcm_err; + + if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) + goto process_auth_decryption_gcm_err; + + if (aadlen > 0) + if (EVP_DecryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0) + goto process_auth_decryption_gcm_err; + + if (srclen > 0) + if (process_openssl_decryption_update(mbuf_src, offset, &dst, + srclen, ctx)) + goto process_auth_decryption_gcm_err; + + /* Workaround open ssl bug in version less then 1.0.1f */ + if (EVP_DecryptUpdate(ctx, empty, &unused, empty, 0) <= 0) + goto process_auth_decryption_gcm_err; + + if (EVP_DecryptFinal_ex(ctx, dst, &len) <= 0) + return -EFAULT; + + return 0; + +process_auth_decryption_gcm_err: + OPENSSL_LOG_ERR("Process openssl auth decryption gcm failed"); + return -EINVAL; +} + +/** Process AES-CCM decrypt algorithm */ +static int +process_openssl_auth_decryption_ccm(struct rte_mbuf *mbuf_src, int offset, + int srclen, uint8_t *aad, int aadlen, uint8_t *iv, + uint8_t *dst, uint8_t *tag, uint8_t tag_len, + EVP_CIPHER_CTX *ctx) +{ + int len = 0; + + if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_SET_TAG, tag_len, tag) <= 0) + goto process_auth_decryption_ccm_err; + + if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) + goto process_auth_decryption_ccm_err; + + if (EVP_DecryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0) + goto process_auth_decryption_ccm_err; + + if (aadlen > 0) + /* + * For AES-CCM, the actual AAD is placed + * 18 bytes after the start of the AAD field, + * according to the API. + */ + if (EVP_DecryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0) + goto process_auth_decryption_ccm_err; + + if (srclen > 0) + if (process_openssl_decryption_update(mbuf_src, offset, &dst, + srclen, ctx)) + return -EFAULT; + + return 0; + +process_auth_decryption_ccm_err: + OPENSSL_LOG_ERR("Process openssl auth decryption ccm failed"); + return -EINVAL; +} + +/** Process standard openssl auth algorithms */ +static int +process_openssl_auth(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, + __rte_unused uint8_t *iv, __rte_unused EVP_PKEY * pkey, + int srclen, EVP_MD_CTX *ctx, const EVP_MD *algo) +{ + size_t dstlen; + struct rte_mbuf *m; + int l, n = srclen; + uint8_t *src; + + for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); + m = m->next) + offset -= rte_pktmbuf_data_len(m); + + if (m == 0) + goto process_auth_err; + + if (EVP_DigestInit_ex(ctx, algo, NULL) <= 0) + goto process_auth_err; + + src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); + + l = rte_pktmbuf_data_len(m) - offset; + if (srclen <= l) { + if (EVP_DigestUpdate(ctx, (char *)src, srclen) <= 0) + goto process_auth_err; + goto process_auth_final; + } + + if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0) + goto process_auth_err; + + n -= l; + + for (m = m->next; (m != NULL) && (n > 0); m = m->next) { + src = rte_pktmbuf_mtod(m, uint8_t *); + l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; + if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0) + goto process_auth_err; + n -= l; + } + +process_auth_final: + if (EVP_DigestFinal_ex(ctx, dst, (unsigned int *)&dstlen) <= 0) + goto process_auth_err; + return 0; + +process_auth_err: + OPENSSL_LOG_ERR("Process openssl auth failed"); + return -EINVAL; +} + +/** Process standard openssl auth algorithms with hmac */ +static int +process_openssl_auth_hmac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, + int srclen, HMAC_CTX *ctx) +{ + unsigned int dstlen; + struct rte_mbuf *m; + int l, n = srclen; + uint8_t *src; + + for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); + m = m->next) + offset -= rte_pktmbuf_data_len(m); + + if (m == 0) + goto process_auth_err; + + src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); + + l = rte_pktmbuf_data_len(m) - offset; + if (srclen <= l) { + if (HMAC_Update(ctx, (unsigned char *)src, srclen) != 1) + goto process_auth_err; + goto process_auth_final; + } + + if (HMAC_Update(ctx, (unsigned char *)src, l) != 1) + goto process_auth_err; + + n -= l; + + for (m = m->next; (m != NULL) && (n > 0); m = m->next) { + src = rte_pktmbuf_mtod(m, uint8_t *); + l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; + if (HMAC_Update(ctx, (unsigned char *)src, l) != 1) + goto process_auth_err; + n -= l; + } + +process_auth_final: + if (HMAC_Final(ctx, dst, &dstlen) != 1) + goto process_auth_err; + + if (unlikely(HMAC_Init_ex(ctx, NULL, 0, NULL, NULL) != 1)) + goto process_auth_err; + + return 0; + +process_auth_err: + OPENSSL_LOG_ERR("Process openssl auth failed"); + return -EINVAL; +} + +/*----------------------------------------------------------------------------*/ + +/** Process auth/cipher combined operation */ +static void +process_openssl_combined_op + (struct rte_crypto_op *op, struct openssl_session *sess, + struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) +{ + /* cipher */ + uint8_t *dst = NULL, *iv, *tag, *aad; + int srclen, aadlen, status = -1; + uint32_t offset; + uint8_t taglen; + + /* + * Segmented destination buffer is not supported for + * encryption/decryption + */ + if (!rte_pktmbuf_is_contiguous(mbuf_dst)) { + op->status = RTE_CRYPTO_OP_STATUS_ERROR; + return; + } + + iv = rte_crypto_op_ctod_offset(op, uint8_t *, + sess->iv.offset); + if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) { + srclen = 0; + offset = op->sym->auth.data.offset; + aadlen = op->sym->auth.data.length; + aad = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, + op->sym->auth.data.offset); + tag = op->sym->auth.digest.data; + if (tag == NULL) + tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, + offset + aadlen); + } else { + srclen = op->sym->aead.data.length; + dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, + op->sym->aead.data.offset); + offset = op->sym->aead.data.offset; + aad = op->sym->aead.aad.data; + aadlen = sess->auth.aad_length; + tag = op->sym->aead.digest.data; + if (tag == NULL) + tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, + offset + srclen); + } + + taglen = sess->auth.digest_length; + + if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { + if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC || + sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM) + status = process_openssl_auth_encryption_gcm( + mbuf_src, offset, srclen, + aad, aadlen, iv, + dst, tag, sess->cipher.ctx); + else + status = process_openssl_auth_encryption_ccm( + mbuf_src, offset, srclen, + aad, aadlen, iv, + dst, tag, taglen, sess->cipher.ctx); + + } else { + if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC || + sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM) + status = process_openssl_auth_decryption_gcm( + mbuf_src, offset, srclen, + aad, aadlen, iv, + dst, tag, sess->cipher.ctx); + else + status = process_openssl_auth_decryption_ccm( + mbuf_src, offset, srclen, + aad, aadlen, iv, + dst, tag, taglen, sess->cipher.ctx); + } + + if (status != 0) { + if (status == (-EFAULT) && + sess->auth.operation == + RTE_CRYPTO_AUTH_OP_VERIFY) + op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; + else + op->status = RTE_CRYPTO_OP_STATUS_ERROR; + } +} + +/** Process cipher operation */ +static void +process_openssl_cipher_op + (struct rte_crypto_op *op, struct openssl_session *sess, + struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) +{ + uint8_t *dst, *iv; + int srclen, status; + + /* + * Segmented destination buffer is not supported for + * encryption/decryption + */ + if (!rte_pktmbuf_is_contiguous(mbuf_dst)) { + op->status = RTE_CRYPTO_OP_STATUS_ERROR; + return; + } + + srclen = op->sym->cipher.data.length; + dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, + op->sym->cipher.data.offset); + + iv = rte_crypto_op_ctod_offset(op, uint8_t *, + sess->iv.offset); + + if (sess->cipher.mode == OPENSSL_CIPHER_LIB) + if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) + status = process_openssl_cipher_encrypt(mbuf_src, dst, + op->sym->cipher.data.offset, iv, + srclen, sess->cipher.ctx); + else + status = process_openssl_cipher_decrypt(mbuf_src, dst, + op->sym->cipher.data.offset, iv, + srclen, sess->cipher.ctx); + else + status = process_openssl_cipher_des3ctr(mbuf_src, dst, + op->sym->cipher.data.offset, iv, + sess->cipher.key.data, srclen, + sess->cipher.ctx); + + if (status != 0) + op->status = RTE_CRYPTO_OP_STATUS_ERROR; +} + +/** Process cipher operation */ +static void +process_openssl_docsis_bpi_op(struct rte_crypto_op *op, + struct openssl_session *sess, struct rte_mbuf *mbuf_src, + struct rte_mbuf *mbuf_dst) +{ + uint8_t *src, *dst, *iv; + uint8_t block_size, last_block_len; + int srclen, status = 0; + + srclen = op->sym->cipher.data.length; + src = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, + op->sym->cipher.data.offset); + dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, + op->sym->cipher.data.offset); + + iv = rte_crypto_op_ctod_offset(op, uint8_t *, + sess->iv.offset); + + block_size = DES_BLOCK_SIZE; + + last_block_len = srclen % block_size; + if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { + /* Encrypt only with ECB mode XOR IV */ + if (srclen < block_size) { + status = process_openssl_cipher_bpi_encrypt(src, dst, + iv, srclen, + sess->cipher.bpi_ctx); + } else { + srclen -= last_block_len; + /* Encrypt with the block aligned stream with CBC mode */ + status = process_openssl_cipher_encrypt(mbuf_src, dst, + op->sym->cipher.data.offset, iv, + srclen, sess->cipher.ctx); + if (last_block_len) { + /* Point at last block */ + dst += srclen; + /* + * IV is the last encrypted block from + * the previous operation + */ + iv = dst - block_size; + src += srclen; + srclen = last_block_len; + /* Encrypt the last frame with ECB mode */ + status |= process_openssl_cipher_bpi_encrypt(src, + dst, iv, + srclen, sess->cipher.bpi_ctx); + } + } + } else { + /* Decrypt only with ECB mode (encrypt, as it is same operation) */ + if (srclen < block_size) { + status = process_openssl_cipher_bpi_encrypt(src, dst, + iv, + srclen, + sess->cipher.bpi_ctx); + } else { + if (last_block_len) { + /* Point at last block */ + dst += srclen - last_block_len; + src += srclen - last_block_len; + /* + * IV is the last full block + */ + iv = src - block_size; + /* + * Decrypt the last frame with ECB mode + * (encrypt, as it is the same operation) + */ + status = process_openssl_cipher_bpi_encrypt(src, + dst, iv, + last_block_len, sess->cipher.bpi_ctx); + /* Prepare parameters for CBC mode op */ + iv = rte_crypto_op_ctod_offset(op, uint8_t *, + sess->iv.offset); + dst += last_block_len - srclen; + srclen -= last_block_len; + } + + /* Decrypt with CBC mode */ + status |= process_openssl_cipher_decrypt(mbuf_src, dst, + op->sym->cipher.data.offset, iv, + srclen, sess->cipher.ctx); + } + } + + if (status != 0) + op->status = RTE_CRYPTO_OP_STATUS_ERROR; +} + +/** Process auth operation */ +static void +process_openssl_auth_op(struct openssl_qp *qp, struct rte_crypto_op *op, + struct openssl_session *sess, struct rte_mbuf *mbuf_src, + struct rte_mbuf *mbuf_dst) +{ + uint8_t *dst; + int srclen, status; + + srclen = op->sym->auth.data.length; + + if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) + dst = qp->temp_digest; + else { + dst = op->sym->auth.digest.data; + if (dst == NULL) + dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, + op->sym->auth.data.offset + + op->sym->auth.data.length); + } + + switch (sess->auth.mode) { + case OPENSSL_AUTH_AS_AUTH: + status = process_openssl_auth(mbuf_src, dst, + op->sym->auth.data.offset, NULL, NULL, srclen, + sess->auth.auth.ctx, sess->auth.auth.evp_algo); + break; + case OPENSSL_AUTH_AS_HMAC: + status = process_openssl_auth_hmac(mbuf_src, dst, + op->sym->auth.data.offset, srclen, + sess->auth.hmac.ctx); + break; + default: + status = -1; + break; + } + + if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) { + if (memcmp(dst, op->sym->auth.digest.data, + sess->auth.digest_length) != 0) { + op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; + } + } + + if (status != 0) + op->status = RTE_CRYPTO_OP_STATUS_ERROR; +} + +/** Process crypto operation for mbuf */ +static int +process_op(struct openssl_qp *qp, struct rte_crypto_op *op, + struct openssl_session *sess) +{ + struct rte_mbuf *msrc, *mdst; + int retval; + + msrc = op->sym->m_src; + mdst = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src; + + op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; + + switch (sess->chain_order) { + case OPENSSL_CHAIN_ONLY_CIPHER: + process_openssl_cipher_op(op, sess, msrc, mdst); + break; + case OPENSSL_CHAIN_ONLY_AUTH: + process_openssl_auth_op(qp, op, sess, msrc, mdst); + break; + case OPENSSL_CHAIN_CIPHER_AUTH: + process_openssl_cipher_op(op, sess, msrc, mdst); + process_openssl_auth_op(qp, op, sess, mdst, mdst); + break; + case OPENSSL_CHAIN_AUTH_CIPHER: + process_openssl_auth_op(qp, op, sess, msrc, mdst); + process_openssl_cipher_op(op, sess, msrc, mdst); + break; + case OPENSSL_CHAIN_COMBINED: + process_openssl_combined_op(op, sess, msrc, mdst); + break; + case OPENSSL_CHAIN_CIPHER_BPI: + process_openssl_docsis_bpi_op(op, sess, msrc, mdst); + break; + default: + op->status = RTE_CRYPTO_OP_STATUS_ERROR; + break; + } + + /* Free session if a session-less crypto op */ + if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { + openssl_reset_session(sess); + memset(sess, 0, sizeof(struct openssl_session)); + memset(op->sym->session, 0, + rte_cryptodev_get_header_session_size()); + rte_mempool_put(qp->sess_mp, sess); + rte_mempool_put(qp->sess_mp, op->sym->session); + op->sym->session = NULL; + } + + if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) + op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; + + if (op->status != RTE_CRYPTO_OP_STATUS_ERROR) + retval = rte_ring_enqueue(qp->processed_ops, (void *)op); + else + retval = -1; + + return retval; +} + +/* + *------------------------------------------------------------------------------ + * PMD Framework + *------------------------------------------------------------------------------ + */ + +/** Enqueue burst */ +static uint16_t +openssl_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops, + uint16_t nb_ops) +{ + struct openssl_session *sess; + struct openssl_qp *qp = queue_pair; + int i, retval; + + for (i = 0; i < nb_ops; i++) { + sess = get_session(qp, ops[i]); + if (unlikely(sess == NULL)) + goto enqueue_err; + + retval = process_op(qp, ops[i], sess); + if (unlikely(retval < 0)) + goto enqueue_err; + } + + qp->stats.enqueued_count += i; + return i; + +enqueue_err: + qp->stats.enqueue_err_count++; + return i; +} + +/** Dequeue burst */ +static uint16_t +openssl_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops, + uint16_t nb_ops) +{ + struct openssl_qp *qp = queue_pair; + + unsigned int nb_dequeued = 0; + + nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops, + (void **)ops, nb_ops, NULL); + qp->stats.dequeued_count += nb_dequeued; + + return nb_dequeued; +} + +/** Create OPENSSL crypto device */ +static int +cryptodev_openssl_create(const char *name, + struct rte_vdev_device *vdev, + struct rte_cryptodev_pmd_init_params *init_params) +{ + struct rte_cryptodev *dev; + struct openssl_private *internals; + + dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params); + if (dev == NULL) { + OPENSSL_LOG_ERR("failed to create cryptodev vdev"); + goto init_error; + } + + dev->driver_id = cryptodev_driver_id; + dev->dev_ops = rte_openssl_pmd_ops; + + /* register rx/tx burst functions for data path */ + dev->dequeue_burst = openssl_pmd_dequeue_burst; + dev->enqueue_burst = openssl_pmd_enqueue_burst; + + dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | + RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | + RTE_CRYPTODEV_FF_CPU_AESNI | + RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER; + + /* Set vector instructions mode supported */ + internals = dev->data->dev_private; + + internals->max_nb_qpairs = init_params->max_nb_queue_pairs; + internals->max_nb_sessions = init_params->max_nb_sessions; + + return 0; + +init_error: + OPENSSL_LOG_ERR("driver %s: cryptodev_openssl_create failed", + init_params->name); + + cryptodev_openssl_remove(vdev); + return -EFAULT; +} + +/** Initialise OPENSSL crypto device */ +static int +cryptodev_openssl_probe(struct rte_vdev_device *vdev) +{ + struct rte_cryptodev_pmd_init_params init_params = { + "", + sizeof(struct openssl_private), + rte_socket_id(), + RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS, + RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_SESSIONS + }; + const char *name; + const char *input_args; + + name = rte_vdev_device_name(vdev); + if (name == NULL) + return -EINVAL; + input_args = rte_vdev_device_args(vdev); + + rte_cryptodev_pmd_parse_input_args(&init_params, input_args); + + return cryptodev_openssl_create(name, vdev, &init_params); +} + +/** Uninitialise OPENSSL crypto device */ +static int +cryptodev_openssl_remove(struct rte_vdev_device *vdev) +{ + struct rte_cryptodev *cryptodev; + const char *name; + + name = rte_vdev_device_name(vdev); + if (name == NULL) + return -EINVAL; + + cryptodev = rte_cryptodev_pmd_get_named_dev(name); + if (cryptodev == NULL) + return -ENODEV; + + return rte_cryptodev_pmd_destroy(cryptodev); +} + +static struct rte_vdev_driver cryptodev_openssl_pmd_drv = { + .probe = cryptodev_openssl_probe, + .remove = cryptodev_openssl_remove +}; + +static struct cryptodev_driver openssl_crypto_drv; + +RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_OPENSSL_PMD, + cryptodev_openssl_pmd_drv); +RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_OPENSSL_PMD, + "max_nb_queue_pairs= " + "max_nb_sessions= " + "socket_id="); +RTE_PMD_REGISTER_CRYPTO_DRIVER(openssl_crypto_drv, cryptodev_openssl_pmd_drv, + cryptodev_driver_id); diff --git a/dpdk/drivers/crypto/openssl/rte_openssl_pmd_ops.c b/dpdk/drivers/crypto/openssl/rte_openssl_pmd_ops.c new file mode 100644 index 00000000..c5722399 --- /dev/null +++ b/dpdk/drivers/crypto/openssl/rte_openssl_pmd_ops.c @@ -0,0 +1,785 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +#include +#include +#include + +#include "rte_openssl_pmd_private.h" + + +static const struct rte_cryptodev_capabilities openssl_pmd_capabilities[] = { + { /* MD5 HMAC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_MD5_HMAC, + .block_size = 64, + .key_size = { + .min = 1, + .max = 64, + .increment = 1 + }, + .digest_size = { + .min = 16, + .max = 16, + .increment = 0 + }, + .iv_size = { 0 } + }, } + }, } + }, + { /* MD5 */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_MD5, + .block_size = 64, + .key_size = { + .min = 0, + .max = 0, + .increment = 0 + }, + .digest_size = { + .min = 16, + .max = 16, + .increment = 0 + }, + .iv_size = { 0 } + }, } + }, } + }, + { /* SHA1 HMAC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA1_HMAC, + .block_size = 64, + .key_size = { + .min = 1, + .max = 64, + .increment = 1 + }, + .digest_size = { + .min = 20, + .max = 20, + .increment = 0 + }, + .iv_size = { 0 } + }, } + }, } + }, + { /* SHA1 */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA1, + .block_size = 64, + .key_size = { + .min = 0, + .max = 0, + .increment = 0 + }, + .digest_size = { + .min = 20, + .max = 20, + .increment = 0 + }, + .iv_size = { 0 } + }, } + }, } + }, + { /* SHA224 HMAC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA224_HMAC, + .block_size = 64, + .key_size = { + .min = 1, + .max = 64, + .increment = 1 + }, + .digest_size = { + .min = 28, + .max = 28, + .increment = 0 + }, + .iv_size = { 0 } + }, } + }, } + }, + { /* SHA224 */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA224, + .block_size = 64, + .key_size = { + .min = 0, + .max = 0, + .increment = 0 + }, + .digest_size = { + .min = 28, + .max = 28, + .increment = 0 + }, + .iv_size = { 0 } + }, } + }, } + }, + { /* SHA256 HMAC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA256_HMAC, + .block_size = 64, + .key_size = { + .min = 1, + .max = 64, + .increment = 1 + }, + .digest_size = { + .min = 32, + .max = 32, + .increment = 0 + }, + .iv_size = { 0 } + }, } + }, } + }, + { /* SHA256 */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA256, + .block_size = 64, + .key_size = { + .min = 0, + .max = 0, + .increment = 0 + }, + .digest_size = { + .min = 32, + .max = 32, + .increment = 0 + }, + .iv_size = { 0 } + }, } + }, } + }, + { /* SHA384 HMAC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA384_HMAC, + .block_size = 128, + .key_size = { + .min = 1, + .max = 128, + .increment = 1 + }, + .digest_size = { + .min = 48, + .max = 48, + .increment = 0 + }, + .iv_size = { 0 } + }, } + }, } + }, + { /* SHA384 */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA384, + .block_size = 128, + .key_size = { + .min = 0, + .max = 0, + .increment = 0 + }, + .digest_size = { + .min = 48, + .max = 48, + .increment = 0 + }, + .iv_size = { 0 } + }, } + }, } + }, + { /* SHA512 HMAC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA512_HMAC, + .block_size = 128, + .key_size = { + .min = 1, + .max = 128, + .increment = 1 + }, + .digest_size = { + .min = 64, + .max = 64, + .increment = 0 + }, + .iv_size = { 0 } + }, } + }, } + }, + { /* SHA512 */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA512, + .block_size = 128, + .key_size = { + .min = 0, + .max = 0, + .increment = 0 + }, + .digest_size = { + .min = 64, + .max = 64, + .increment = 0 + }, + .iv_size = { 0 } + }, } + }, } + }, + { /* AES CBC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, + {.cipher = { + .algo = RTE_CRYPTO_CIPHER_AES_CBC, + .block_size = 16, + .key_size = { + .min = 16, + .max = 32, + .increment = 8 + }, + .iv_size = { + .min = 16, + .max = 16, + .increment = 0 + } + }, } + }, } + }, + { /* AES CTR */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, + {.cipher = { + .algo = RTE_CRYPTO_CIPHER_AES_CTR, + .block_size = 16, + .key_size = { + .min = 16, + .max = 32, + .increment = 8 + }, + .iv_size = { + .min = 16, + .max = 16, + .increment = 0 + } + }, } + }, } + }, + { /* AES GCM */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD, + {.aead = { + .algo = RTE_CRYPTO_AEAD_AES_GCM, + .block_size = 16, + .key_size = { + .min = 16, + .max = 32, + .increment = 8 + }, + .digest_size = { + .min = 16, + .max = 16, + .increment = 0 + }, + .aad_size = { + .min = 0, + .max = 65535, + .increment = 1 + }, + .iv_size = { + .min = 12, + .max = 16, + .increment = 4 + }, + }, } + }, } + }, + { /* AES CCM */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD, + {.aead = { + .algo = RTE_CRYPTO_AEAD_AES_CCM, + .block_size = 16, + .key_size = { + .min = 16, + .max = 32, + .increment = 8 + }, + .digest_size = { + .min = 4, + .max = 16, + .increment = 2 + }, + .aad_size = { + .min = 0, + .max = 65535, + .increment = 1 + }, + .iv_size = { + .min = 7, + .max = 13, + .increment = 1 + }, + }, } + }, } + }, + { /* AES GMAC (AUTH) */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_AES_GMAC, + .block_size = 16, + .key_size = { + .min = 16, + .max = 32, + .increment = 8 + }, + .digest_size = { + .min = 16, + .max = 16, + .increment = 0 + }, + .iv_size = { + .min = 12, + .max = 16, + .increment = 4 + } + }, } + }, } + }, + { /* 3DES CBC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, + {.cipher = { + .algo = RTE_CRYPTO_CIPHER_3DES_CBC, + .block_size = 8, + .key_size = { + .min = 16, + .max = 24, + .increment = 8 + }, + .iv_size = { + .min = 8, + .max = 8, + .increment = 0 + } + }, } + }, } + }, + { /* 3DES CTR */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, + {.cipher = { + .algo = RTE_CRYPTO_CIPHER_3DES_CTR, + .block_size = 8, + .key_size = { + .min = 16, + .max = 24, + .increment = 8 + }, + .iv_size = { + .min = 8, + .max = 8, + .increment = 0 + } + }, } + }, } + }, + { /* DES CBC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, + {.cipher = { + .algo = RTE_CRYPTO_CIPHER_DES_CBC, + .block_size = 8, + .key_size = { + .min = 8, + .max = 8, + .increment = 0 + }, + .iv_size = { + .min = 8, + .max = 8, + .increment = 0 + } + }, } + }, } + }, + { /* DES DOCSIS BPI */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, + {.cipher = { + .algo = RTE_CRYPTO_CIPHER_DES_DOCSISBPI, + .block_size = 8, + .key_size = { + .min = 8, + .max = 8, + .increment = 0 + }, + .iv_size = { + .min = 8, + .max = 8, + .increment = 0 + } + }, } + }, } + }, + + RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST() +}; + + +/** Configure device */ +static int +openssl_pmd_config(__rte_unused struct rte_cryptodev *dev, + __rte_unused struct rte_cryptodev_config *config) +{ + return 0; +} + +/** Start device */ +static int +openssl_pmd_start(__rte_unused struct rte_cryptodev *dev) +{ + return 0; +} + +/** Stop device */ +static void +openssl_pmd_stop(__rte_unused struct rte_cryptodev *dev) +{ +} + +/** Close device */ +static int +openssl_pmd_close(__rte_unused struct rte_cryptodev *dev) +{ + return 0; +} + + +/** Get device statistics */ +static void +openssl_pmd_stats_get(struct rte_cryptodev *dev, + struct rte_cryptodev_stats *stats) +{ + int qp_id; + + for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) { + struct openssl_qp *qp = dev->data->queue_pairs[qp_id]; + + stats->enqueued_count += qp->stats.enqueued_count; + stats->dequeued_count += qp->stats.dequeued_count; + + stats->enqueue_err_count += qp->stats.enqueue_err_count; + stats->dequeue_err_count += qp->stats.dequeue_err_count; + } +} + +/** Reset device statistics */ +static void +openssl_pmd_stats_reset(struct rte_cryptodev *dev) +{ + int qp_id; + + for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) { + struct openssl_qp *qp = dev->data->queue_pairs[qp_id]; + + memset(&qp->stats, 0, sizeof(qp->stats)); + } +} + + +/** Get device info */ +static void +openssl_pmd_info_get(struct rte_cryptodev *dev, + struct rte_cryptodev_info *dev_info) +{ + struct openssl_private *internals = dev->data->dev_private; + + if (dev_info != NULL) { + dev_info->driver_id = dev->driver_id; + dev_info->feature_flags = dev->feature_flags; + dev_info->capabilities = openssl_pmd_capabilities; + dev_info->max_nb_queue_pairs = internals->max_nb_qpairs; + dev_info->sym.max_nb_sessions = internals->max_nb_sessions; + } +} + +/** Release queue pair */ +static int +openssl_pmd_qp_release(struct rte_cryptodev *dev, uint16_t qp_id) +{ + if (dev->data->queue_pairs[qp_id] != NULL) { + rte_free(dev->data->queue_pairs[qp_id]); + dev->data->queue_pairs[qp_id] = NULL; + } + return 0; +} + +/** set a unique name for the queue pair based on it's name, dev_id and qp_id */ +static int +openssl_pmd_qp_set_unique_name(struct rte_cryptodev *dev, + struct openssl_qp *qp) +{ + unsigned int n = snprintf(qp->name, sizeof(qp->name), + "openssl_pmd_%u_qp_%u", + dev->data->dev_id, qp->id); + + if (n >= sizeof(qp->name)) + return -1; + + return 0; +} + + +/** Create a ring to place processed operations on */ +static struct rte_ring * +openssl_pmd_qp_create_processed_ops_ring(struct openssl_qp *qp, + unsigned int ring_size, int socket_id) +{ + struct rte_ring *r; + + r = rte_ring_lookup(qp->name); + if (r) { + if (rte_ring_get_size(r) >= ring_size) { + OPENSSL_LOG_INFO( + "Reusing existing ring %s for processed ops", + qp->name); + return r; + } + + OPENSSL_LOG_ERR( + "Unable to reuse existing ring %s for processed ops", + qp->name); + return NULL; + } + + return rte_ring_create(qp->name, ring_size, socket_id, + RING_F_SP_ENQ | RING_F_SC_DEQ); +} + + +/** Setup a queue pair */ +static int +openssl_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, + const struct rte_cryptodev_qp_conf *qp_conf, + int socket_id, struct rte_mempool *session_pool) +{ + struct openssl_qp *qp = NULL; + + /* Free memory prior to re-allocation if needed. */ + if (dev->data->queue_pairs[qp_id] != NULL) + openssl_pmd_qp_release(dev, qp_id); + + /* Allocate the queue pair data structure. */ + qp = rte_zmalloc_socket("OPENSSL PMD Queue Pair", sizeof(*qp), + RTE_CACHE_LINE_SIZE, socket_id); + if (qp == NULL) + return -ENOMEM; + + qp->id = qp_id; + dev->data->queue_pairs[qp_id] = qp; + + if (openssl_pmd_qp_set_unique_name(dev, qp)) + goto qp_setup_cleanup; + + qp->processed_ops = openssl_pmd_qp_create_processed_ops_ring(qp, + qp_conf->nb_descriptors, socket_id); + if (qp->processed_ops == NULL) + goto qp_setup_cleanup; + + qp->sess_mp = session_pool; + + memset(&qp->stats, 0, sizeof(qp->stats)); + + return 0; + +qp_setup_cleanup: + if (qp) + rte_free(qp); + + return -1; +} + +/** Start queue pair */ +static int +openssl_pmd_qp_start(__rte_unused struct rte_cryptodev *dev, + __rte_unused uint16_t queue_pair_id) +{ + return -ENOTSUP; +} + +/** Stop queue pair */ +static int +openssl_pmd_qp_stop(__rte_unused struct rte_cryptodev *dev, + __rte_unused uint16_t queue_pair_id) +{ + return -ENOTSUP; +} + +/** Return the number of allocated queue pairs */ +static uint32_t +openssl_pmd_qp_count(struct rte_cryptodev *dev) +{ + return dev->data->nb_queue_pairs; +} + +/** Returns the size of the session structure */ +static unsigned +openssl_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused) +{ + return sizeof(struct openssl_session); +} + +/** Configure the session from a crypto xform chain */ +static int +openssl_pmd_session_configure(struct rte_cryptodev *dev __rte_unused, + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool) +{ + void *sess_private_data; + int ret; + + if (unlikely(sess == NULL)) { + OPENSSL_LOG_ERR("invalid session struct"); + return -EINVAL; + } + + if (rte_mempool_get(mempool, &sess_private_data)) { + CDEV_LOG_ERR( + "Couldn't get object from session mempool"); + return -ENOMEM; + } + + ret = openssl_set_session_parameters(sess_private_data, xform); + if (ret != 0) { + OPENSSL_LOG_ERR("failed configure session parameters"); + + /* Return session to mempool */ + rte_mempool_put(mempool, sess_private_data); + return ret; + } + + set_session_private_data(sess, dev->driver_id, + sess_private_data); + + return 0; +} + + +/** Clear the memory of session so it doesn't leave key material behind */ +static void +openssl_pmd_session_clear(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) +{ + uint8_t index = dev->driver_id; + void *sess_priv = get_session_private_data(sess, index); + + /* Zero out the whole structure */ + if (sess_priv) { + openssl_reset_session(sess_priv); + memset(sess_priv, 0, sizeof(struct openssl_session)); + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + set_session_private_data(sess, index, NULL); + rte_mempool_put(sess_mp, sess_priv); + } +} + +struct rte_cryptodev_ops openssl_pmd_ops = { + .dev_configure = openssl_pmd_config, + .dev_start = openssl_pmd_start, + .dev_stop = openssl_pmd_stop, + .dev_close = openssl_pmd_close, + + .stats_get = openssl_pmd_stats_get, + .stats_reset = openssl_pmd_stats_reset, + + .dev_infos_get = openssl_pmd_info_get, + + .queue_pair_setup = openssl_pmd_qp_setup, + .queue_pair_release = openssl_pmd_qp_release, + .queue_pair_start = openssl_pmd_qp_start, + .queue_pair_stop = openssl_pmd_qp_stop, + .queue_pair_count = openssl_pmd_qp_count, + + .session_get_size = openssl_pmd_session_get_size, + .session_configure = openssl_pmd_session_configure, + .session_clear = openssl_pmd_session_clear +}; + +struct rte_cryptodev_ops *rte_openssl_pmd_ops = &openssl_pmd_ops; diff --git a/dpdk/drivers/crypto/openssl/rte_openssl_pmd_private.h b/dpdk/drivers/crypto/openssl/rte_openssl_pmd_private.h new file mode 100644 index 00000000..26bf862c --- /dev/null +++ b/dpdk/drivers/crypto/openssl/rte_openssl_pmd_private.h @@ -0,0 +1,200 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _OPENSSL_PMD_PRIVATE_H_ +#define _OPENSSL_PMD_PRIVATE_H_ + +#include +#include +#include + +#define CRYPTODEV_NAME_OPENSSL_PMD crypto_openssl +/**< Open SSL Crypto PMD device name */ + +#define OPENSSL_LOG_ERR(fmt, args...) \ + RTE_LOG(ERR, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \ + RTE_STR(CRYPTODEV_NAME_OPENSSL_PMD), \ + __func__, __LINE__, ## args) + +#ifdef RTE_LIBRTE_OPENSSL_DEBUG +#define OPENSSL_LOG_INFO(fmt, args...) \ + RTE_LOG(INFO, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \ + RTE_STR(CRYPTODEV_NAME_OPENSSL_PMD), \ + __func__, __LINE__, ## args) + +#define OPENSSL_LOG_DBG(fmt, args...) \ + RTE_LOG(DEBUG, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \ + RTE_STR(CRYPTODEV_NAME_OPENSSL_PMD), \ + __func__, __LINE__, ## args) +#else +#define OPENSSL_LOG_INFO(fmt, args...) +#define OPENSSL_LOG_DBG(fmt, args...) +#endif + +/* Maximum length for digest (SHA-512 needs 64 bytes) */ +#define DIGEST_LENGTH_MAX 64 + +/** OPENSSL operation order mode enumerator */ +enum openssl_chain_order { + OPENSSL_CHAIN_ONLY_CIPHER, + OPENSSL_CHAIN_ONLY_AUTH, + OPENSSL_CHAIN_CIPHER_BPI, + OPENSSL_CHAIN_CIPHER_AUTH, + OPENSSL_CHAIN_AUTH_CIPHER, + OPENSSL_CHAIN_COMBINED, + OPENSSL_CHAIN_NOT_SUPPORTED +}; + +/** OPENSSL cipher mode enumerator */ +enum openssl_cipher_mode { + OPENSSL_CIPHER_LIB, + OPENSSL_CIPHER_DES3CTR, +}; + +/** OPENSSL auth mode enumerator */ +enum openssl_auth_mode { + OPENSSL_AUTH_AS_AUTH, + OPENSSL_AUTH_AS_HMAC, +}; + +/** private data structure for each OPENSSL crypto device */ +struct openssl_private { + unsigned int max_nb_qpairs; + /**< Max number of queue pairs */ + unsigned int max_nb_sessions; + /**< Max number of sessions */ +}; + +/** OPENSSL crypto queue pair */ +struct openssl_qp { + uint16_t id; + /**< Queue Pair Identifier */ + char name[RTE_CRYPTODEV_NAME_LEN]; + /**< Unique Queue Pair Name */ + struct rte_ring *processed_ops; + /**< Ring for placing process packets */ + struct rte_mempool *sess_mp; + /**< Session Mempool */ + struct rte_cryptodev_stats stats; + /**< Queue pair statistics */ + uint8_t temp_digest[DIGEST_LENGTH_MAX]; + /**< Buffer used to store the digest generated + * by the driver when verifying a digest provided + * by the user (using authentication verify operation) + */ +} __rte_cache_aligned; + +/** OPENSSL crypto private session structure */ +struct openssl_session { + enum openssl_chain_order chain_order; + /**< chain order mode */ + + struct { + uint16_t length; + uint16_t offset; + } iv; + /**< IV parameters */ + + enum rte_crypto_aead_algorithm aead_algo; + /**< AEAD algorithm */ + + /** Cipher Parameters */ + struct { + enum rte_crypto_cipher_operation direction; + /**< cipher operation direction */ + enum openssl_cipher_mode mode; + /**< cipher operation mode */ + enum rte_crypto_cipher_algorithm algo; + /**< cipher algorithm */ + + struct { + uint8_t data[32]; + /**< key data */ + size_t length; + /**< key length in bytes */ + } key; + + const EVP_CIPHER *evp_algo; + /**< pointer to EVP algorithm function */ + EVP_CIPHER_CTX *ctx; + /**< pointer to EVP context structure */ + EVP_CIPHER_CTX *bpi_ctx; + } cipher; + + /** Authentication Parameters */ + struct { + enum rte_crypto_auth_operation operation; + /**< auth operation generate or verify */ + enum openssl_auth_mode mode; + /**< auth operation mode */ + enum rte_crypto_auth_algorithm algo; + /**< cipher algorithm */ + + union { + struct { + const EVP_MD *evp_algo; + /**< pointer to EVP algorithm function */ + EVP_MD_CTX *ctx; + /**< pointer to EVP context structure */ + } auth; + + struct { + EVP_PKEY *pkey; + /**< pointer to EVP key */ + const EVP_MD *evp_algo; + /**< pointer to EVP algorithm function */ + HMAC_CTX *ctx; + /**< pointer to EVP context structure */ + } hmac; + }; + + uint16_t aad_length; + /**< AAD length */ + uint16_t digest_length; + /**< digest length */ + } auth; + +} __rte_cache_aligned; + +/** Set and validate OPENSSL crypto session parameters */ +extern int +openssl_set_session_parameters(struct openssl_session *sess, + const struct rte_crypto_sym_xform *xform); + +/** Reset OPENSSL crypto session parameters */ +extern void +openssl_reset_session(struct openssl_session *sess); + +/** device specific operations function pointer structure */ +extern struct rte_cryptodev_ops *rte_openssl_pmd_ops; + +#endif /* _OPENSSL_PMD_PRIVATE_H_ */ diff --git a/dpdk/drivers/crypto/openssl/rte_pmd_openssl_version.map b/dpdk/drivers/crypto/openssl/rte_pmd_openssl_version.map new file mode 100644 index 00000000..cc5829e3 --- /dev/null +++ b/dpdk/drivers/crypto/openssl/rte_pmd_openssl_version.map @@ -0,0 +1,3 @@ +DPDK_16.11 { + local: *; +}; diff --git a/dpdk/drivers/crypto/qat/Makefile b/dpdk/drivers/crypto/qat/Makefile index 20a70d4a..745d0197 100644 --- a/dpdk/drivers/crypto/qat/Makefile +++ b/dpdk/drivers/crypto/qat/Makefile @@ -43,6 +43,9 @@ CFLAGS += -O3 # external library include paths CFLAGS += -I$(SRCDIR)/qat_adf LDLIBS += -lcrypto +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_cryptodev +LDLIBS += -lrte_pci -lrte_bus_pci # library source files SRCS-$(CONFIG_RTE_LIBRTE_PMD_QAT) += qat_crypto.c @@ -56,11 +59,4 @@ SYMLINK-y-include += # versioning export map EXPORT_MAP := rte_pmd_qat_version.map -# library dependencies -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_QAT) += lib/librte_eal -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_QAT) += lib/librte_mbuf -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_QAT) += lib/librte_mempool -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_QAT) += lib/librte_cryptodev - - include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/crypto/qat/qat_adf/adf_transport_access_macros.h b/dpdk/drivers/crypto/qat/qat_adf/adf_transport_access_macros.h index 47f1c91a..d218f85d 100644 --- a/dpdk/drivers/crypto/qat/qat_adf/adf_transport_access_macros.h +++ b/dpdk/drivers/crypto/qat/qat_adf/adf_transport_access_macros.h @@ -47,14 +47,15 @@ #ifndef ADF_TRANSPORT_ACCESS_MACROS_H #define ADF_TRANSPORT_ACCESS_MACROS_H +#include + /* CSR write macro */ -#define ADF_CSR_WR(csrAddr, csrOffset, val) \ - (void)((*((volatile uint32_t *)(((uint8_t *)csrAddr) + csrOffset)) \ - = (val))) +#define ADF_CSR_WR(csrAddr, csrOffset, val) \ + rte_write32(val, (((uint8_t *)csrAddr) + csrOffset)) /* CSR read macro */ -#define ADF_CSR_RD(csrAddr, csrOffset) \ - (*((volatile uint32_t *)(((uint8_t *)csrAddr) + csrOffset))) +#define ADF_CSR_RD(csrAddr, csrOffset) \ + rte_read32((((uint8_t *)csrAddr) + csrOffset)) #define ADF_BANK_INT_SRC_SEL_MASK_0 0x4444444CUL #define ADF_BANK_INT_SRC_SEL_MASK_X 0x44444444UL diff --git a/dpdk/drivers/crypto/qat/qat_adf/icp_qat_hw.h b/dpdk/drivers/crypto/qat/qat_adf/icp_qat_hw.h index 4d4d8e4d..d03688c7 100644 --- a/dpdk/drivers/crypto/qat/qat_adf/icp_qat_hw.h +++ b/dpdk/drivers/crypto/qat/qat_adf/icp_qat_hw.h @@ -237,6 +237,11 @@ enum icp_qat_hw_cipher_dir { ICP_QAT_HW_CIPHER_DECRYPT = 1, }; +enum icp_qat_hw_auth_op { + ICP_QAT_HW_AUTH_VERIFY = 0, + ICP_QAT_HW_AUTH_GENERATE = 1, +}; + enum icp_qat_hw_cipher_convert { ICP_QAT_HW_CIPHER_NO_CONVERT = 0, ICP_QAT_HW_CIPHER_KEY_CONVERT = 1, @@ -293,14 +298,32 @@ enum icp_qat_hw_cipher_convert { #define ICP_QAT_HW_ZUC_3G_EEA3_KEY_SZ 16 #define ICP_QAT_HW_ZUC_3G_EEA3_IV_SZ 16 #define ICP_QAT_HW_MODE_F8_NUM_REG_TO_CLEAR 2 -#define INIT_SHRAM_CONSTANTS_TABLE_SZ 1024 -struct icp_qat_hw_cipher_aes256_f8 { - struct icp_qat_hw_cipher_config cipher_config; - uint8_t key[ICP_QAT_HW_AES_256_F8_KEY_SZ]; -}; +#define ICP_QAT_HW_CIPHER_MAX_KEY_SZ ICP_QAT_HW_AES_256_F8_KEY_SZ + +/* These defines describe position of the bit-fields + * in the flags byte in B0 + */ +#define ICP_QAT_HW_CCM_B0_FLAGS_ADATA_SHIFT 6 +#define ICP_QAT_HW_CCM_B0_FLAGS_T_SHIFT 3 + +#define ICP_QAT_HW_CCM_BUILD_B0_FLAGS(Adata, t, q) \ + ((((Adata) > 0 ? 1 : 0) << ICP_QAT_HW_CCM_B0_FLAGS_ADATA_SHIFT) \ + | ((((t) - 2) >> 1) << ICP_QAT_HW_CCM_B0_FLAGS_T_SHIFT) \ + | ((q) - 1)) + +#define ICP_QAT_HW_CCM_NQ_CONST 15 +#define ICP_QAT_HW_CCM_AAD_B0_LEN 16 +#define ICP_QAT_HW_CCM_AAD_LEN_INFO 2 +#define ICP_QAT_HW_CCM_AAD_DATA_OFFSET (ICP_QAT_HW_CCM_AAD_B0_LEN + \ + ICP_QAT_HW_CCM_AAD_LEN_INFO) +#define ICP_QAT_HW_CCM_AAD_ALIGNMENT 16 +#define ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE 4 +#define ICP_QAT_HW_CCM_NONCE_OFFSET 1 struct icp_qat_hw_cipher_algo_blk { - struct icp_qat_hw_cipher_aes256_f8 aes; + struct icp_qat_hw_cipher_config cipher_config; + uint8_t key[ICP_QAT_HW_CIPHER_MAX_KEY_SZ]; } __rte_cache_aligned; + #endif diff --git a/dpdk/drivers/crypto/qat/qat_adf/qat_algs.h b/dpdk/drivers/crypto/qat/qat_adf/qat_algs.h index 243c1b40..802ba95d 100644 --- a/dpdk/drivers/crypto/qat/qat_adf/qat_algs.h +++ b/dpdk/drivers/crypto/qat/qat_adf/qat_algs.h @@ -17,7 +17,7 @@ * qat-linux@intel.com * * BSD LICENSE - * Copyright(c) 2015-2016 Intel Corporation. + * Copyright(c) 2015-2017 Intel Corporation. * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: @@ -47,9 +47,23 @@ #ifndef _ICP_QAT_ALGS_H_ #define _ICP_QAT_ALGS_H_ #include +#include #include "icp_qat_hw.h" #include "icp_qat_fw.h" #include "icp_qat_fw_la.h" +#include "../qat_crypto.h" + +/* + * Key Modifier (KM) value used in KASUMI algorithm in F9 mode to XOR + * Integrity Key (IK) + */ +#define KASUMI_F9_KEY_MODIFIER_4_BYTES 0xAAAAAAAA + +#define KASUMI_F8_KEY_MODIFIER_4_BYTES 0x55555555 + +/* 3DES key sizes */ +#define QAT_3DES_KEY_SZ_OPT1 24 /* Keys are independent */ +#define QAT_3DES_KEY_SZ_OPT2 16 /* K3=K1 */ #define QAT_AES_HW_CONFIG_CBC_ENC(alg) \ ICP_QAT_HW_CIPHER_CONFIG_BUILD(ICP_QAT_HW_CIPHER_CBC_MODE, alg, \ @@ -67,13 +81,33 @@ struct qat_alg_buf { uint64_t addr; } __rte_packed; +enum qat_crypto_proto_flag { + QAT_CRYPTO_PROTO_FLAG_NONE = 0, + QAT_CRYPTO_PROTO_FLAG_CCM = 1, + QAT_CRYPTO_PROTO_FLAG_GCM = 2, + QAT_CRYPTO_PROTO_FLAG_SNOW3G = 3, + QAT_CRYPTO_PROTO_FLAG_ZUC = 4 +}; + +/* + * Maximum number of SGL entries + */ +#define QAT_SGL_MAX_NUMBER 16 + struct qat_alg_buf_list { uint64_t resrvd; uint32_t num_bufs; uint32_t num_mapped_bufs; - struct qat_alg_buf bufers[]; + struct qat_alg_buf bufers[QAT_SGL_MAX_NUMBER]; } __rte_packed __rte_cache_aligned; +struct qat_crypto_op_cookie { + struct qat_alg_buf_list qat_sgl_list_src; + struct qat_alg_buf_list qat_sgl_list_dst; + rte_iova_t qat_sgl_src_phys_addr; + rte_iova_t qat_sgl_dst_phys_addr; +}; + /* Common content descriptor */ struct qat_alg_cd { struct icp_qat_hw_cipher_algo_blk cipher; @@ -86,20 +120,25 @@ struct qat_session { enum icp_qat_hw_cipher_dir qat_dir; enum icp_qat_hw_cipher_mode qat_mode; enum icp_qat_hw_auth_algo qat_hash_alg; + enum icp_qat_hw_auth_op auth_op; + void *bpi_ctx; struct qat_alg_cd cd; - phys_addr_t cd_paddr; - struct icp_qat_fw_la_bulk_req fw_req; - struct qat_crypto_instance *inst; - uint8_t salt[ICP_QAT_HW_AES_BLK_SZ]; - rte_spinlock_t lock; /* protects this struct */ -}; - -struct qat_alg_ablkcipher_cd { - struct icp_qat_hw_cipher_algo_blk *cd; - phys_addr_t cd_paddr; + uint8_t *cd_cur_ptr; + rte_iova_t cd_paddr; struct icp_qat_fw_la_bulk_req fw_req; + uint8_t aad_len; struct qat_crypto_instance *inst; + struct { + uint16_t offset; + uint16_t length; + } cipher_iv; + struct { + uint16_t offset; + uint16_t length; + } auth_iv; + uint16_t digest_length; rte_spinlock_t lock; /* protects this struct */ + enum qat_device_gen min_qat_dev_gen; }; int qat_get_inter_state_size(enum icp_qat_hw_auth_algo qat_hash_alg); @@ -111,21 +150,20 @@ int qat_alg_aead_session_create_content_desc_cipher(struct qat_session *cd, int qat_alg_aead_session_create_content_desc_auth(struct qat_session *cdesc, uint8_t *authkey, uint32_t authkeylen, - uint32_t add_auth_data_length, + uint32_t aad_length, uint32_t digestsize, unsigned int operation); -void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header); - -void qat_alg_ablkcipher_init_enc(struct qat_alg_ablkcipher_cd *cd, - int alg, const uint8_t *key, - unsigned int keylen); - -void qat_alg_ablkcipher_init_dec(struct qat_alg_ablkcipher_cd *cd, - int alg, const uint8_t *key, - unsigned int keylen); +void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header, + enum qat_crypto_proto_flag proto_flags); int qat_alg_validate_aes_key(int key_len, enum icp_qat_hw_cipher_algo *alg); +int qat_alg_validate_aes_docsisbpi_key(int key_len, + enum icp_qat_hw_cipher_algo *alg); int qat_alg_validate_snow3g_key(int key_len, enum icp_qat_hw_cipher_algo *alg); - +int qat_alg_validate_kasumi_key(int key_len, enum icp_qat_hw_cipher_algo *alg); +int qat_alg_validate_3des_key(int key_len, enum icp_qat_hw_cipher_algo *alg); +int qat_alg_validate_des_key(int key_len, enum icp_qat_hw_cipher_algo *alg); +int qat_cipher_get_block_size(enum icp_qat_hw_cipher_algo qat_cipher_alg); +int qat_alg_validate_zuc_key(int key_len, enum icp_qat_hw_cipher_algo *alg); #endif diff --git a/dpdk/drivers/crypto/qat/qat_adf/qat_algs_build_desc.c b/dpdk/drivers/crypto/qat/qat_adf/qat_algs_build_desc.c index 185bb334..26f854c2 100644 --- a/dpdk/drivers/crypto/qat/qat_adf/qat_algs_build_desc.c +++ b/dpdk/drivers/crypto/qat/qat_adf/qat_algs_build_desc.c @@ -17,7 +17,7 @@ * qat-linux@intel.com * * BSD LICENSE - * Copyright(c) 2015-2016 Intel Corporation. + * Copyright(c) 2015-2017 Intel Corporation. * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: @@ -54,11 +54,31 @@ #include #include "../qat_logs.h" -#include "qat_algs.h" #include /* Needed to calculate pre-compute values */ #include /* Needed to calculate pre-compute values */ +#include /* Needed to calculate pre-compute values */ +#include "qat_algs.h" + +/* returns block size in bytes per cipher algo */ +int qat_cipher_get_block_size(enum icp_qat_hw_cipher_algo qat_cipher_alg) +{ + switch (qat_cipher_alg) { + case ICP_QAT_HW_CIPHER_ALGO_DES: + return ICP_QAT_HW_DES_BLK_SZ; + case ICP_QAT_HW_CIPHER_ALGO_3DES: + return ICP_QAT_HW_3DES_BLK_SZ; + case ICP_QAT_HW_CIPHER_ALGO_AES128: + case ICP_QAT_HW_CIPHER_ALGO_AES192: + case ICP_QAT_HW_CIPHER_ALGO_AES256: + return ICP_QAT_HW_AES_BLK_SZ; + default: + PMD_DRV_LOG(ERR, "invalid block cipher alg %u", qat_cipher_alg); + return -EFAULT; + }; + return -EFAULT; +} /* * Returns size in bytes per hash algo for state1 size field in cd_ctrl @@ -70,9 +90,15 @@ static int qat_hash_get_state1_size(enum icp_qat_hw_auth_algo qat_hash_alg) case ICP_QAT_HW_AUTH_ALGO_SHA1: return QAT_HW_ROUND_UP(ICP_QAT_HW_SHA1_STATE1_SZ, QAT_HW_DEFAULT_ALIGNMENT); + case ICP_QAT_HW_AUTH_ALGO_SHA224: + return QAT_HW_ROUND_UP(ICP_QAT_HW_SHA224_STATE1_SZ, + QAT_HW_DEFAULT_ALIGNMENT); case ICP_QAT_HW_AUTH_ALGO_SHA256: return QAT_HW_ROUND_UP(ICP_QAT_HW_SHA256_STATE1_SZ, QAT_HW_DEFAULT_ALIGNMENT); + case ICP_QAT_HW_AUTH_ALGO_SHA384: + return QAT_HW_ROUND_UP(ICP_QAT_HW_SHA384_STATE1_SZ, + QAT_HW_DEFAULT_ALIGNMENT); case ICP_QAT_HW_AUTH_ALGO_SHA512: return QAT_HW_ROUND_UP(ICP_QAT_HW_SHA512_STATE1_SZ, QAT_HW_DEFAULT_ALIGNMENT); @@ -83,9 +109,24 @@ static int qat_hash_get_state1_size(enum icp_qat_hw_auth_algo qat_hash_alg) case ICP_QAT_HW_AUTH_ALGO_GALOIS_64: return QAT_HW_ROUND_UP(ICP_QAT_HW_GALOIS_128_STATE1_SZ, QAT_HW_DEFAULT_ALIGNMENT); + case ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3: + return QAT_HW_ROUND_UP(ICP_QAT_HW_ZUC_3G_EIA3_STATE1_SZ, + QAT_HW_DEFAULT_ALIGNMENT); case ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2: return QAT_HW_ROUND_UP(ICP_QAT_HW_SNOW_3G_UIA2_STATE1_SZ, QAT_HW_DEFAULT_ALIGNMENT); + case ICP_QAT_HW_AUTH_ALGO_MD5: + return QAT_HW_ROUND_UP(ICP_QAT_HW_MD5_STATE1_SZ, + QAT_HW_DEFAULT_ALIGNMENT); + case ICP_QAT_HW_AUTH_ALGO_KASUMI_F9: + return QAT_HW_ROUND_UP(ICP_QAT_HW_KASUMI_F9_STATE1_SZ, + QAT_HW_DEFAULT_ALIGNMENT); + case ICP_QAT_HW_AUTH_ALGO_NULL: + return QAT_HW_ROUND_UP(ICP_QAT_HW_NULL_STATE1_SZ, + QAT_HW_DEFAULT_ALIGNMENT); + case ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC: + return QAT_HW_ROUND_UP(ICP_QAT_HW_AES_CBC_MAC_STATE1_SZ, + QAT_HW_DEFAULT_ALIGNMENT); case ICP_QAT_HW_AUTH_ALGO_DELIMITER: /* return maximum state1 size in this case */ return QAT_HW_ROUND_UP(ICP_QAT_HW_SHA512_STATE1_SZ, @@ -103,10 +144,16 @@ static int qat_hash_get_digest_size(enum icp_qat_hw_auth_algo qat_hash_alg) switch (qat_hash_alg) { case ICP_QAT_HW_AUTH_ALGO_SHA1: return ICP_QAT_HW_SHA1_STATE1_SZ; + case ICP_QAT_HW_AUTH_ALGO_SHA224: + return ICP_QAT_HW_SHA224_STATE1_SZ; case ICP_QAT_HW_AUTH_ALGO_SHA256: return ICP_QAT_HW_SHA256_STATE1_SZ; + case ICP_QAT_HW_AUTH_ALGO_SHA384: + return ICP_QAT_HW_SHA384_STATE1_SZ; case ICP_QAT_HW_AUTH_ALGO_SHA512: return ICP_QAT_HW_SHA512_STATE1_SZ; + case ICP_QAT_HW_AUTH_ALGO_MD5: + return ICP_QAT_HW_MD5_STATE1_SZ; case ICP_QAT_HW_AUTH_ALGO_DELIMITER: /* return maximum digest size in this case */ return ICP_QAT_HW_SHA512_STATE1_SZ; @@ -123,12 +170,18 @@ static int qat_hash_get_block_size(enum icp_qat_hw_auth_algo qat_hash_alg) switch (qat_hash_alg) { case ICP_QAT_HW_AUTH_ALGO_SHA1: return SHA_CBLOCK; + case ICP_QAT_HW_AUTH_ALGO_SHA224: + return SHA256_CBLOCK; case ICP_QAT_HW_AUTH_ALGO_SHA256: return SHA256_CBLOCK; + case ICP_QAT_HW_AUTH_ALGO_SHA384: + return SHA512_CBLOCK; case ICP_QAT_HW_AUTH_ALGO_SHA512: return SHA512_CBLOCK; case ICP_QAT_HW_AUTH_ALGO_GALOIS_128: return 16; + case ICP_QAT_HW_AUTH_ALGO_MD5: + return MD5_CBLOCK; case ICP_QAT_HW_AUTH_ALGO_DELIMITER: /* return maximum block size in this case */ return SHA512_CBLOCK; @@ -150,6 +203,17 @@ static int partial_hash_sha1(uint8_t *data_in, uint8_t *data_out) return 0; } +static int partial_hash_sha224(uint8_t *data_in, uint8_t *data_out) +{ + SHA256_CTX ctx; + + if (!SHA224_Init(&ctx)) + return -EFAULT; + SHA256_Transform(&ctx, data_in); + rte_memcpy(data_out, &ctx, SHA256_DIGEST_LENGTH); + return 0; +} + static int partial_hash_sha256(uint8_t *data_in, uint8_t *data_out) { SHA256_CTX ctx; @@ -161,6 +225,17 @@ static int partial_hash_sha256(uint8_t *data_in, uint8_t *data_out) return 0; } +static int partial_hash_sha384(uint8_t *data_in, uint8_t *data_out) +{ + SHA512_CTX ctx; + + if (!SHA384_Init(&ctx)) + return -EFAULT; + SHA512_Transform(&ctx, data_in); + rte_memcpy(data_out, &ctx, SHA512_DIGEST_LENGTH); + return 0; +} + static int partial_hash_sha512(uint8_t *data_in, uint8_t *data_out) { SHA512_CTX ctx; @@ -172,6 +247,18 @@ static int partial_hash_sha512(uint8_t *data_in, uint8_t *data_out) return 0; } +static int partial_hash_md5(uint8_t *data_in, uint8_t *data_out) +{ + MD5_CTX ctx; + + if (!MD5_Init(&ctx)) + return -EFAULT; + MD5_Transform(&ctx, data_in); + rte_memcpy(data_out, &ctx, MD5_DIGEST_LENGTH); + + return 0; +} + static int partial_hash_compute(enum icp_qat_hw_auth_algo hash_alg, uint8_t *data_in, uint8_t *data_out) @@ -199,6 +286,13 @@ static int partial_hash_compute(enum icp_qat_hw_auth_algo hash_alg, *hash_state_out_be32 = rte_bswap32(*(((uint32_t *)digest)+i)); break; + case ICP_QAT_HW_AUTH_ALGO_SHA224: + if (partial_hash_sha224(data_in, digest)) + return -EFAULT; + for (i = 0; i < digest_size >> 2; i++, hash_state_out_be32++) + *hash_state_out_be32 = + rte_bswap32(*(((uint32_t *)digest)+i)); + break; case ICP_QAT_HW_AUTH_ALGO_SHA256: if (partial_hash_sha256(data_in, digest)) return -EFAULT; @@ -206,6 +300,13 @@ static int partial_hash_compute(enum icp_qat_hw_auth_algo hash_alg, *hash_state_out_be32 = rte_bswap32(*(((uint32_t *)digest)+i)); break; + case ICP_QAT_HW_AUTH_ALGO_SHA384: + if (partial_hash_sha384(data_in, digest)) + return -EFAULT; + for (i = 0; i < digest_size >> 3; i++, hash_state_out_be64++) + *hash_state_out_be64 = + rte_bswap64(*(((uint64_t *)digest)+i)); + break; case ICP_QAT_HW_AUTH_ALGO_SHA512: if (partial_hash_sha512(data_in, digest)) return -EFAULT; @@ -213,6 +314,10 @@ static int partial_hash_compute(enum icp_qat_hw_auth_algo hash_alg, *hash_state_out_be64 = rte_bswap64(*(((uint64_t *)digest)+i)); break; + case ICP_QAT_HW_AUTH_ALGO_MD5: + if (partial_hash_md5(data_in, data_out)) + return -EFAULT; + break; default: PMD_DRV_LOG(ERR, "invalid hash alg %u", hash_alg); return -EFAULT; @@ -254,6 +359,11 @@ static int qat_alg_do_precomputes(enum icp_qat_hw_auth_algo hash_alg, in = rte_zmalloc("working mem for key", ICP_QAT_HW_AES_XCBC_MAC_STATE2_SZ, 16); + if (in == NULL) { + PMD_DRV_LOG(ERR, "Failed to alloc memory"); + return -ENOMEM; + } + rte_memcpy(in, qat_aes_xcbc_key_seed, ICP_QAT_HW_AES_XCBC_MAC_STATE2_SZ); for (x = 0; x < HASH_XCBC_PRECOMP_KEY_NUM; x++) { @@ -284,6 +394,11 @@ static int qat_alg_do_precomputes(enum icp_qat_hw_auth_algo hash_alg, ICP_QAT_HW_GALOIS_E_CTR0_SZ); in = rte_zmalloc("working mem for key", ICP_QAT_HW_GALOIS_H_SZ, 16); + if (in == NULL) { + PMD_DRV_LOG(ERR, "Failed to alloc memory"); + return -ENOMEM; + } + memset(in, 0, ICP_QAT_HW_GALOIS_H_SZ); if (AES_set_encrypt_key(auth_key, auth_keylen << 3, &enc_key) != 0) { @@ -344,7 +459,8 @@ static int qat_alg_do_precomputes(enum icp_qat_hw_auth_algo hash_alg, return 0; } -void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header) +void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header, + enum qat_crypto_proto_flag proto_flags) { PMD_INIT_FUNC_TRACE(); header->hdr_flags = @@ -357,10 +473,58 @@ void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header) ICP_QAT_FW_LA_PARTIAL_NONE); ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(header->serv_specif_flags, ICP_QAT_FW_CIPH_IV_16BYTE_DATA); - ICP_QAT_FW_LA_PROTO_SET(header->serv_specif_flags, - ICP_QAT_FW_LA_NO_PROTO); + + switch (proto_flags) { + case QAT_CRYPTO_PROTO_FLAG_NONE: + ICP_QAT_FW_LA_PROTO_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_NO_PROTO); + break; + case QAT_CRYPTO_PROTO_FLAG_CCM: + ICP_QAT_FW_LA_PROTO_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_CCM_PROTO); + break; + case QAT_CRYPTO_PROTO_FLAG_GCM: + ICP_QAT_FW_LA_PROTO_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_GCM_PROTO); + break; + case QAT_CRYPTO_PROTO_FLAG_SNOW3G: + ICP_QAT_FW_LA_PROTO_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_SNOW_3G_PROTO); + break; + case QAT_CRYPTO_PROTO_FLAG_ZUC: + ICP_QAT_FW_LA_ZUC_3G_PROTO_FLAG_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_ZUC_3G_PROTO); + break; + } + ICP_QAT_FW_LA_UPDATE_STATE_SET(header->serv_specif_flags, ICP_QAT_FW_LA_NO_UPDATE_STATE); + ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_NO_DIGEST_IN_BUFFER); +} + +/* + * Snow3G and ZUC should never use this function + * and set its protocol flag in both cipher and auth part of content + * descriptor building function + */ +static enum qat_crypto_proto_flag +qat_get_crypto_proto_flag(uint16_t flags) +{ + int proto = ICP_QAT_FW_LA_PROTO_GET(flags); + enum qat_crypto_proto_flag qat_proto_flag = + QAT_CRYPTO_PROTO_FLAG_NONE; + + switch (proto) { + case ICP_QAT_FW_LA_GCM_PROTO: + qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_GCM; + break; + case ICP_QAT_FW_LA_CCM_PROTO: + qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_CCM; + break; + } + + return qat_proto_flag; } int qat_alg_aead_session_create_content_desc_cipher(struct qat_session *cdesc, @@ -375,139 +539,150 @@ int qat_alg_aead_session_create_content_desc_cipher(struct qat_session *cdesc, struct icp_qat_fw_cipher_cd_ctrl_hdr *cipher_cd_ctrl = ptr; struct icp_qat_fw_auth_cd_ctrl_hdr *hash_cd_ctrl = ptr; enum icp_qat_hw_cipher_convert key_convert; - uint16_t proto = ICP_QAT_FW_LA_NO_PROTO; /* no CCM/GCM/Snow3G */ - uint16_t cipher_offset = 0; - + enum qat_crypto_proto_flag qat_proto_flag = + QAT_CRYPTO_PROTO_FLAG_NONE; + uint32_t total_key_size; + uint16_t cipher_offset, cd_size; + uint32_t wordIndex = 0; + uint32_t *temp_key = NULL; PMD_INIT_FUNC_TRACE(); - if (cdesc->qat_cmd == ICP_QAT_FW_LA_CMD_HASH_CIPHER && - cdesc->qat_hash_alg != ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2) { - cipher = - (struct icp_qat_hw_cipher_algo_blk *)((char *)&cdesc->cd + - sizeof(struct icp_qat_hw_auth_algo_blk)); - cipher_offset = sizeof(struct icp_qat_hw_auth_algo_blk); - } else { - cipher = (struct icp_qat_hw_cipher_algo_blk *)&cdesc->cd; - cipher_offset = 0; - } - /* CD setup */ - if (cdesc->qat_dir == ICP_QAT_HW_CIPHER_ENCRYPT) { - ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags, - ICP_QAT_FW_LA_RET_AUTH_RES); - ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags, - ICP_QAT_FW_LA_NO_CMP_AUTH_RES); - } else { + if (cdesc->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER) { + cd_pars->u.s.content_desc_addr = cdesc->cd_paddr; + ICP_QAT_FW_COMN_CURR_ID_SET(cipher_cd_ctrl, + ICP_QAT_FW_SLICE_CIPHER); + ICP_QAT_FW_COMN_NEXT_ID_SET(cipher_cd_ctrl, + ICP_QAT_FW_SLICE_DRAM_WR); ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags, ICP_QAT_FW_LA_NO_RET_AUTH_RES); ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags, - ICP_QAT_FW_LA_CMP_AUTH_RES); + ICP_QAT_FW_LA_NO_CMP_AUTH_RES); + cdesc->cd_cur_ptr = (uint8_t *)&cdesc->cd; + } else if (cdesc->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER_HASH) { + cd_pars->u.s.content_desc_addr = cdesc->cd_paddr; + ICP_QAT_FW_COMN_CURR_ID_SET(cipher_cd_ctrl, + ICP_QAT_FW_SLICE_CIPHER); + ICP_QAT_FW_COMN_NEXT_ID_SET(cipher_cd_ctrl, + ICP_QAT_FW_SLICE_AUTH); + ICP_QAT_FW_COMN_CURR_ID_SET(hash_cd_ctrl, + ICP_QAT_FW_SLICE_AUTH); + ICP_QAT_FW_COMN_NEXT_ID_SET(hash_cd_ctrl, + ICP_QAT_FW_SLICE_DRAM_WR); + cdesc->cd_cur_ptr = (uint8_t *)&cdesc->cd; + } else if (cdesc->qat_cmd != ICP_QAT_FW_LA_CMD_HASH_CIPHER) { + PMD_DRV_LOG(ERR, "Invalid param, must be a cipher command."); + return -EFAULT; } if (cdesc->qat_mode == ICP_QAT_HW_CIPHER_CTR_MODE) { - /* CTR Streaming ciphers are a special case. Decrypt = encrypt + /* + * CTR Streaming ciphers are a special case. Decrypt = encrypt * Overriding default values previously set */ cdesc->qat_dir = ICP_QAT_HW_CIPHER_ENCRYPT; key_convert = ICP_QAT_HW_CIPHER_NO_CONVERT; - } else if (cdesc->qat_dir == ICP_QAT_HW_CIPHER_ENCRYPT) + } else if (cdesc->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2 + || cdesc->qat_cipher_alg == + ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3) + key_convert = ICP_QAT_HW_CIPHER_KEY_CONVERT; + else if (cdesc->qat_dir == ICP_QAT_HW_CIPHER_ENCRYPT) key_convert = ICP_QAT_HW_CIPHER_NO_CONVERT; else key_convert = ICP_QAT_HW_CIPHER_KEY_CONVERT; - if (cdesc->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2) - key_convert = ICP_QAT_HW_CIPHER_KEY_CONVERT; - - /* For Snow3G, set key convert and other bits */ if (cdesc->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2) { - key_convert = ICP_QAT_HW_CIPHER_KEY_CONVERT; - ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags, - ICP_QAT_FW_LA_NO_RET_AUTH_RES); - if (cdesc->qat_cmd == ICP_QAT_FW_LA_CMD_HASH_CIPHER) { - ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags, - ICP_QAT_FW_LA_RET_AUTH_RES); - ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags, - ICP_QAT_FW_LA_NO_CMP_AUTH_RES); - } - } + total_key_size = ICP_QAT_HW_SNOW_3G_UEA2_KEY_SZ + + ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ; + cipher_cd_ctrl->cipher_state_sz = + ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ >> 3; + qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_SNOW3G; - cipher->aes.cipher_config.val = + } else if (cdesc->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_KASUMI) { + total_key_size = ICP_QAT_HW_KASUMI_F8_KEY_SZ; + cipher_cd_ctrl->cipher_state_sz = ICP_QAT_HW_KASUMI_BLK_SZ >> 3; + cipher_cd_ctrl->cipher_padding_sz = + (2 * ICP_QAT_HW_KASUMI_BLK_SZ) >> 3; + } else if (cdesc->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_3DES) { + total_key_size = ICP_QAT_HW_3DES_KEY_SZ; + cipher_cd_ctrl->cipher_state_sz = ICP_QAT_HW_3DES_BLK_SZ >> 3; + qat_proto_flag = + qat_get_crypto_proto_flag(header->serv_specif_flags); + } else if (cdesc->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_DES) { + total_key_size = ICP_QAT_HW_DES_KEY_SZ; + cipher_cd_ctrl->cipher_state_sz = ICP_QAT_HW_DES_BLK_SZ >> 3; + qat_proto_flag = + qat_get_crypto_proto_flag(header->serv_specif_flags); + } else if (cdesc->qat_cipher_alg == + ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3) { + total_key_size = ICP_QAT_HW_ZUC_3G_EEA3_KEY_SZ + + ICP_QAT_HW_ZUC_3G_EEA3_IV_SZ; + cipher_cd_ctrl->cipher_state_sz = + ICP_QAT_HW_ZUC_3G_EEA3_IV_SZ >> 3; + qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_ZUC; + cdesc->min_qat_dev_gen = QAT_GEN2; + } else { + total_key_size = cipherkeylen; + cipher_cd_ctrl->cipher_state_sz = ICP_QAT_HW_AES_BLK_SZ >> 3; + qat_proto_flag = + qat_get_crypto_proto_flag(header->serv_specif_flags); + } + cipher_cd_ctrl->cipher_key_sz = total_key_size >> 3; + cipher_offset = cdesc->cd_cur_ptr-((uint8_t *)&cdesc->cd); + cipher_cd_ctrl->cipher_cfg_offset = cipher_offset >> 3; + + header->service_cmd_id = cdesc->qat_cmd; + qat_alg_init_common_hdr(header, qat_proto_flag); + + cipher = (struct icp_qat_hw_cipher_algo_blk *)cdesc->cd_cur_ptr; + cipher->cipher_config.val = ICP_QAT_HW_CIPHER_CONFIG_BUILD(cdesc->qat_mode, cdesc->qat_cipher_alg, key_convert, cdesc->qat_dir); - memcpy(cipher->aes.key, cipherkey, cipherkeylen); - proto = ICP_QAT_FW_LA_PROTO_GET(header->serv_specif_flags); - if (cdesc->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2) - proto = ICP_QAT_FW_LA_SNOW_3G_PROTO; + if (cdesc->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_KASUMI) { + temp_key = (uint32_t *)(cdesc->cd_cur_ptr + + sizeof(struct icp_qat_hw_cipher_config) + + cipherkeylen); + memcpy(cipher->key, cipherkey, cipherkeylen); + memcpy(temp_key, cipherkey, cipherkeylen); - /* Request template setup */ - qat_alg_init_common_hdr(header); - header->service_cmd_id = cdesc->qat_cmd; - ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags, - ICP_QAT_FW_LA_NO_DIGEST_IN_BUFFER); - /* Configure the common header protocol flags */ - ICP_QAT_FW_LA_PROTO_SET(header->serv_specif_flags, proto); - cd_pars->u.s.content_desc_addr = cdesc->cd_paddr; - cd_pars->u.s.content_desc_params_sz = sizeof(cdesc->cd) >> 3; + /* XOR Key with KASUMI F8 key modifier at 4 bytes level */ + for (wordIndex = 0; wordIndex < (cipherkeylen >> 2); + wordIndex++) + temp_key[wordIndex] ^= KASUMI_F8_KEY_MODIFIER_4_BYTES; - /* Cipher CD config setup */ - if (cdesc->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2) { - cipher_cd_ctrl->cipher_key_sz = - (ICP_QAT_HW_SNOW_3G_UEA2_KEY_SZ + - ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ) >> 3; - cipher_cd_ctrl->cipher_state_sz = - ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ >> 3; - cipher_cd_ctrl->cipher_cfg_offset = cipher_offset >> 3; - if (cdesc->qat_cmd == ICP_QAT_FW_LA_CMD_HASH_CIPHER) { - ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags, - ICP_QAT_FW_LA_DIGEST_IN_BUFFER); - } + cdesc->cd_cur_ptr += sizeof(struct icp_qat_hw_cipher_config) + + cipherkeylen + cipherkeylen; } else { - cipher_cd_ctrl->cipher_key_sz = cipherkeylen >> 3; - cipher_cd_ctrl->cipher_state_sz = ICP_QAT_HW_AES_BLK_SZ >> 3; - cipher_cd_ctrl->cipher_cfg_offset = cipher_offset >> 3; + memcpy(cipher->key, cipherkey, cipherkeylen); + cdesc->cd_cur_ptr += sizeof(struct icp_qat_hw_cipher_config) + + cipherkeylen; } - if (cdesc->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER) { - ICP_QAT_FW_COMN_CURR_ID_SET(cipher_cd_ctrl, - ICP_QAT_FW_SLICE_CIPHER); - ICP_QAT_FW_COMN_NEXT_ID_SET(cipher_cd_ctrl, - ICP_QAT_FW_SLICE_DRAM_WR); - } else if (cdesc->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER_HASH) { - ICP_QAT_FW_COMN_CURR_ID_SET(cipher_cd_ctrl, - ICP_QAT_FW_SLICE_CIPHER); - ICP_QAT_FW_COMN_NEXT_ID_SET(cipher_cd_ctrl, - ICP_QAT_FW_SLICE_AUTH); - ICP_QAT_FW_COMN_CURR_ID_SET(hash_cd_ctrl, - ICP_QAT_FW_SLICE_AUTH); - ICP_QAT_FW_COMN_NEXT_ID_SET(hash_cd_ctrl, - ICP_QAT_FW_SLICE_DRAM_WR); - } else if (cdesc->qat_cmd == ICP_QAT_FW_LA_CMD_HASH_CIPHER) { - ICP_QAT_FW_COMN_CURR_ID_SET(hash_cd_ctrl, - ICP_QAT_FW_SLICE_AUTH); - ICP_QAT_FW_COMN_NEXT_ID_SET(hash_cd_ctrl, - ICP_QAT_FW_SLICE_CIPHER); - ICP_QAT_FW_COMN_CURR_ID_SET(cipher_cd_ctrl, - ICP_QAT_FW_SLICE_CIPHER); - ICP_QAT_FW_COMN_NEXT_ID_SET(cipher_cd_ctrl, - ICP_QAT_FW_SLICE_DRAM_WR); - } else { - PMD_DRV_LOG(ERR, "invalid param, only authenticated " - "encryption supported"); - return -EFAULT; + if (total_key_size > cipherkeylen) { + uint32_t padding_size = total_key_size-cipherkeylen; + if ((cdesc->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_3DES) + && (cipherkeylen == QAT_3DES_KEY_SZ_OPT2)) + /* K3 not provided so use K1 = K3*/ + memcpy(cdesc->cd_cur_ptr, cipherkey, padding_size); + else + memset(cdesc->cd_cur_ptr, 0, padding_size); + cdesc->cd_cur_ptr += padding_size; } + cd_size = cdesc->cd_cur_ptr-(uint8_t *)&cdesc->cd; + cd_pars->u.s.content_desc_params_sz = RTE_ALIGN_CEIL(cd_size, 8) >> 3; + return 0; } int qat_alg_aead_session_create_content_desc_auth(struct qat_session *cdesc, uint8_t *authkey, uint32_t authkeylen, - uint32_t add_auth_data_length, + uint32_t aad_length, uint32_t digestsize, unsigned int operation) { - struct icp_qat_hw_cipher_algo_blk *cipher; - struct icp_qat_hw_auth_algo_blk *hash; + struct icp_qat_hw_auth_setup *hash; struct icp_qat_hw_cipher_algo_blk *cipherconfig; struct icp_qat_fw_la_bulk_req *req_tmpl = &cdesc->fw_req; struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars; @@ -519,227 +694,22 @@ int qat_alg_aead_session_create_content_desc_auth(struct qat_session *cdesc, (struct icp_qat_fw_la_auth_req_params *) ((char *)&req_tmpl->serv_specif_rqpars + sizeof(struct icp_qat_fw_la_cipher_req_params)); - enum icp_qat_hw_cipher_convert key_convert; - uint16_t proto = ICP_QAT_FW_LA_NO_PROTO; /* no CCM/GCM/Snow3G */ - uint16_t state1_size = 0; - uint16_t state2_size = 0; - uint16_t cipher_offset = 0, hash_offset = 0; + uint16_t state1_size = 0, state2_size = 0; + uint16_t hash_offset, cd_size; + uint32_t *aad_len = NULL; + uint32_t wordIndex = 0; + uint32_t *pTempKey; + enum qat_crypto_proto_flag qat_proto_flag = + QAT_CRYPTO_PROTO_FLAG_NONE; PMD_INIT_FUNC_TRACE(); - if (cdesc->qat_cmd == ICP_QAT_FW_LA_CMD_HASH_CIPHER && - cdesc->qat_hash_alg != ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2) { - hash = (struct icp_qat_hw_auth_algo_blk *)&cdesc->cd; - cipher = - (struct icp_qat_hw_cipher_algo_blk *)((char *)&cdesc->cd + - sizeof(struct icp_qat_hw_auth_algo_blk)); - hash_offset = 0; - cipher_offset = ((char *)hash - (char *)cipher); - } else { - cipher = (struct icp_qat_hw_cipher_algo_blk *)&cdesc->cd; - hash = (struct icp_qat_hw_auth_algo_blk *)((char *)&cdesc->cd + - sizeof(struct icp_qat_hw_cipher_algo_blk)); - cipher_offset = 0; - hash_offset = ((char *)hash - (char *)cipher); - } - - /* CD setup */ - if (cdesc->qat_dir == ICP_QAT_HW_CIPHER_ENCRYPT) { - ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags, - ICP_QAT_FW_LA_RET_AUTH_RES); - ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags, - ICP_QAT_FW_LA_NO_CMP_AUTH_RES); - } else { - ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags, - ICP_QAT_FW_LA_NO_RET_AUTH_RES); - ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags, - ICP_QAT_FW_LA_CMP_AUTH_RES); - } - - if (cdesc->qat_mode == ICP_QAT_HW_CIPHER_CTR_MODE) { - /* CTR Streaming ciphers are a special case. Decrypt = encrypt - * Overriding default values previously set - */ - cdesc->qat_dir = ICP_QAT_HW_CIPHER_ENCRYPT; - key_convert = ICP_QAT_HW_CIPHER_NO_CONVERT; - } else if (cdesc->qat_dir == ICP_QAT_HW_CIPHER_ENCRYPT) - key_convert = ICP_QAT_HW_CIPHER_NO_CONVERT; - else - key_convert = ICP_QAT_HW_CIPHER_KEY_CONVERT; - - if (cdesc->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2) - key_convert = ICP_QAT_HW_CIPHER_KEY_CONVERT; - - cipher->aes.cipher_config.val = - ICP_QAT_HW_CIPHER_CONFIG_BUILD(cdesc->qat_mode, - cdesc->qat_cipher_alg, key_convert, - cdesc->qat_dir); - - hash->sha.inner_setup.auth_config.reserved = 0; - hash->sha.inner_setup.auth_config.config = - ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE1, - cdesc->qat_hash_alg, digestsize); - hash->sha.inner_setup.auth_counter.counter = - rte_bswap32(qat_hash_get_block_size(cdesc->qat_hash_alg)); - if (cdesc->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2) { - hash->sha.inner_setup.auth_counter.counter = 0; - hash->sha.outer_setup.auth_config.reserved = 0; - cipherconfig = (struct icp_qat_hw_cipher_algo_blk *) - ((char *)&cdesc->cd + - sizeof(struct icp_qat_hw_auth_algo_blk) - + 16); - cipherconfig->aes.cipher_config.val = - ICP_QAT_HW_CIPHER_CONFIG_BUILD(ICP_QAT_HW_CIPHER_ECB_MODE, - ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2, - ICP_QAT_HW_CIPHER_KEY_CONVERT, - ICP_QAT_HW_CIPHER_ENCRYPT); - memcpy(cipherconfig->aes.key, authkey, authkeylen); - memset(cipherconfig->aes.key + authkeylen, 0, - ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ); - } - - /* Do precomputes */ - if (cdesc->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_AES_XCBC_MAC) { - if (qat_alg_do_precomputes(cdesc->qat_hash_alg, - authkey, authkeylen, (uint8_t *)(hash->sha.state1 + - ICP_QAT_HW_AES_XCBC_MAC_STATE1_SZ), &state2_size)) { - PMD_DRV_LOG(ERR, "(XCBC)precompute failed"); - return -EFAULT; - } - } else if ((cdesc->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128) || - (cdesc->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64)) { - if (qat_alg_do_precomputes(cdesc->qat_hash_alg, - authkey, authkeylen, (uint8_t *)(hash->sha.state1 + - ICP_QAT_HW_GALOIS_128_STATE1_SZ), &state2_size)) { - PMD_DRV_LOG(ERR, "(GCM)precompute failed"); - return -EFAULT; - } - /* - * Write (the length of AAD) into bytes 16-19 of state2 - * in big-endian format. This field is 8 bytes - */ - uint32_t *aad_len = (uint32_t *)&hash->sha.state1[ - ICP_QAT_HW_GALOIS_128_STATE1_SZ + - ICP_QAT_HW_GALOIS_H_SZ]; - *aad_len = rte_bswap32(add_auth_data_length); - - proto = ICP_QAT_FW_LA_GCM_PROTO; - } else if (cdesc->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2) { - proto = ICP_QAT_FW_LA_SNOW_3G_PROTO; - state1_size = qat_hash_get_state1_size(cdesc->qat_hash_alg); - } else { - if (qat_alg_do_precomputes(cdesc->qat_hash_alg, - authkey, authkeylen, (uint8_t *)(hash->sha.state1), - &state1_size)) { - PMD_DRV_LOG(ERR, "(SHA)precompute failed"); - return -EFAULT; - } - } - - /* Request template setup */ - qat_alg_init_common_hdr(header); - header->service_cmd_id = cdesc->qat_cmd; - ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags, - ICP_QAT_FW_LA_DIGEST_IN_BUFFER); - /* Configure the common header protocol flags */ - ICP_QAT_FW_LA_PROTO_SET(header->serv_specif_flags, proto); - cd_pars->u.s.content_desc_addr = cdesc->cd_paddr; - cd_pars->u.s.content_desc_params_sz = sizeof(cdesc->cd) >> 3; - - if (cdesc->qat_cmd == ICP_QAT_FW_LA_CMD_AUTH) { - ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags, - ICP_QAT_FW_LA_NO_DIGEST_IN_BUFFER); - ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(header->serv_specif_flags, - ICP_QAT_FW_CIPH_IV_64BIT_PTR); - ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags, - ICP_QAT_FW_LA_RET_AUTH_RES); - ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags, - ICP_QAT_FW_LA_NO_CMP_AUTH_RES); - } - if (operation == RTE_CRYPTO_AUTH_OP_VERIFY) { - ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags, - ICP_QAT_FW_LA_NO_RET_AUTH_RES); - ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags, - ICP_QAT_FW_LA_CMP_AUTH_RES); - } - - /* Cipher CD config setup */ - cipher_cd_ctrl->cipher_state_sz = ICP_QAT_HW_AES_BLK_SZ >> 3; - cipher_cd_ctrl->cipher_cfg_offset = cipher_offset >> 3; - - if (cdesc->qat_cmd != ICP_QAT_FW_LA_CMD_AUTH) { - cipher_cd_ctrl->cipher_state_sz = ICP_QAT_HW_AES_BLK_SZ >> 3; - cipher_cd_ctrl->cipher_cfg_offset = cipher_offset>>3; - } else { - cipher_cd_ctrl->cipher_state_sz = 0; - cipher_cd_ctrl->cipher_cfg_offset = 0; - } - - /* Auth CD config setup */ - hash_cd_ctrl->hash_cfg_offset = hash_offset >> 3; - hash_cd_ctrl->hash_flags = ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED; - hash_cd_ctrl->inner_res_sz = digestsize; - hash_cd_ctrl->final_sz = digestsize; - hash_cd_ctrl->inner_state1_sz = state1_size; - - switch (cdesc->qat_hash_alg) { - case ICP_QAT_HW_AUTH_ALGO_SHA1: - hash_cd_ctrl->inner_state2_sz = - RTE_ALIGN_CEIL(ICP_QAT_HW_SHA1_STATE2_SZ, 8); - break; - case ICP_QAT_HW_AUTH_ALGO_SHA256: - hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA256_STATE2_SZ; - break; - case ICP_QAT_HW_AUTH_ALGO_SHA512: - hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA512_STATE2_SZ; - break; - case ICP_QAT_HW_AUTH_ALGO_AES_XCBC_MAC: - hash_cd_ctrl->inner_state2_sz = - ICP_QAT_HW_AES_XCBC_MAC_STATE2_SZ; - hash_cd_ctrl->inner_state1_sz = - ICP_QAT_HW_AES_XCBC_MAC_STATE1_SZ; - memset(hash->sha.state1, 0, ICP_QAT_HW_AES_XCBC_MAC_STATE1_SZ); - break; - case ICP_QAT_HW_AUTH_ALGO_GALOIS_128: - case ICP_QAT_HW_AUTH_ALGO_GALOIS_64: - hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_GALOIS_H_SZ + - ICP_QAT_HW_GALOIS_LEN_A_SZ + - ICP_QAT_HW_GALOIS_E_CTR0_SZ; - hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_GALOIS_128_STATE1_SZ; - memset(hash->sha.state1, 0, ICP_QAT_HW_GALOIS_128_STATE1_SZ); - break; - case ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2: - hash_cd_ctrl->inner_state2_sz = - ICP_QAT_HW_SNOW_3G_UIA2_STATE2_SZ; - hash_cd_ctrl->inner_state1_sz = - ICP_QAT_HW_SNOW_3G_UIA2_STATE1_SZ; - memset(hash->sha.state1, 0, ICP_QAT_HW_SNOW_3G_UIA2_STATE1_SZ); - break; - default: - PMD_DRV_LOG(ERR, "invalid HASH alg %u", cdesc->qat_hash_alg); - return -EFAULT; - } - - hash_cd_ctrl->inner_state2_offset = hash_cd_ctrl->hash_cfg_offset + - ((sizeof(struct icp_qat_hw_auth_setup) + - RTE_ALIGN_CEIL(hash_cd_ctrl->inner_state1_sz, 8)) - >> 3); - auth_param->auth_res_sz = digestsize; - if (cdesc->qat_cmd == ICP_QAT_FW_LA_CMD_AUTH) { ICP_QAT_FW_COMN_CURR_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_AUTH); ICP_QAT_FW_COMN_NEXT_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR); - } else if (cdesc->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER_HASH) { - ICP_QAT_FW_COMN_CURR_ID_SET(cipher_cd_ctrl, - ICP_QAT_FW_SLICE_CIPHER); - ICP_QAT_FW_COMN_NEXT_ID_SET(cipher_cd_ctrl, - ICP_QAT_FW_SLICE_AUTH); - ICP_QAT_FW_COMN_CURR_ID_SET(hash_cd_ctrl, - ICP_QAT_FW_SLICE_AUTH); - ICP_QAT_FW_COMN_NEXT_ID_SET(hash_cd_ctrl, - ICP_QAT_FW_SLICE_DRAM_WR); + cdesc->cd_cur_ptr = (uint8_t *)&cdesc->cd; } else if (cdesc->qat_cmd == ICP_QAT_FW_LA_CMD_HASH_CIPHER) { ICP_QAT_FW_COMN_CURR_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_AUTH); @@ -749,64 +719,253 @@ int qat_alg_aead_session_create_content_desc_auth(struct qat_session *cdesc, ICP_QAT_FW_SLICE_CIPHER); ICP_QAT_FW_COMN_NEXT_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR); - } else { - PMD_DRV_LOG(ERR, "invalid param, only authenticated " - "encryption supported"); + cdesc->cd_cur_ptr = (uint8_t *)&cdesc->cd; + } else if (cdesc->qat_cmd != ICP_QAT_FW_LA_CMD_CIPHER_HASH) { + PMD_DRV_LOG(ERR, "Invalid param, must be a hash command."); return -EFAULT; } + + if (operation == RTE_CRYPTO_AUTH_OP_VERIFY) { + ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_NO_RET_AUTH_RES); + ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_CMP_AUTH_RES); + cdesc->auth_op = ICP_QAT_HW_AUTH_VERIFY; + } else { + ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_RET_AUTH_RES); + ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_NO_CMP_AUTH_RES); + cdesc->auth_op = ICP_QAT_HW_AUTH_GENERATE; + } + + /* + * Setup the inner hash config + */ + hash_offset = cdesc->cd_cur_ptr-((uint8_t *)&cdesc->cd); + hash = (struct icp_qat_hw_auth_setup *)cdesc->cd_cur_ptr; + hash->auth_config.reserved = 0; + hash->auth_config.config = + ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE1, + cdesc->qat_hash_alg, digestsize); + + if (cdesc->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2 + || cdesc->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_KASUMI_F9 + || cdesc->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3) + hash->auth_counter.counter = 0; + else + hash->auth_counter.counter = rte_bswap32( + qat_hash_get_block_size(cdesc->qat_hash_alg)); + + cdesc->cd_cur_ptr += sizeof(struct icp_qat_hw_auth_setup); + + /* + * cd_cur_ptr now points at the state1 information. + */ + switch (cdesc->qat_hash_alg) { + case ICP_QAT_HW_AUTH_ALGO_SHA1: + if (qat_alg_do_precomputes(ICP_QAT_HW_AUTH_ALGO_SHA1, + authkey, authkeylen, cdesc->cd_cur_ptr, &state1_size)) { + PMD_DRV_LOG(ERR, "(SHA)precompute failed"); + return -EFAULT; + } + state2_size = RTE_ALIGN_CEIL(ICP_QAT_HW_SHA1_STATE2_SZ, 8); + break; + case ICP_QAT_HW_AUTH_ALGO_SHA224: + if (qat_alg_do_precomputes(ICP_QAT_HW_AUTH_ALGO_SHA224, + authkey, authkeylen, cdesc->cd_cur_ptr, &state1_size)) { + PMD_DRV_LOG(ERR, "(SHA)precompute failed"); + return -EFAULT; + } + state2_size = ICP_QAT_HW_SHA224_STATE2_SZ; + break; + case ICP_QAT_HW_AUTH_ALGO_SHA256: + if (qat_alg_do_precomputes(ICP_QAT_HW_AUTH_ALGO_SHA256, + authkey, authkeylen, cdesc->cd_cur_ptr, &state1_size)) { + PMD_DRV_LOG(ERR, "(SHA)precompute failed"); + return -EFAULT; + } + state2_size = ICP_QAT_HW_SHA256_STATE2_SZ; + break; + case ICP_QAT_HW_AUTH_ALGO_SHA384: + if (qat_alg_do_precomputes(ICP_QAT_HW_AUTH_ALGO_SHA384, + authkey, authkeylen, cdesc->cd_cur_ptr, &state1_size)) { + PMD_DRV_LOG(ERR, "(SHA)precompute failed"); + return -EFAULT; + } + state2_size = ICP_QAT_HW_SHA384_STATE2_SZ; + break; + case ICP_QAT_HW_AUTH_ALGO_SHA512: + if (qat_alg_do_precomputes(ICP_QAT_HW_AUTH_ALGO_SHA512, + authkey, authkeylen, cdesc->cd_cur_ptr, &state1_size)) { + PMD_DRV_LOG(ERR, "(SHA)precompute failed"); + return -EFAULT; + } + state2_size = ICP_QAT_HW_SHA512_STATE2_SZ; + break; + case ICP_QAT_HW_AUTH_ALGO_AES_XCBC_MAC: + state1_size = ICP_QAT_HW_AES_XCBC_MAC_STATE1_SZ; + if (qat_alg_do_precomputes(ICP_QAT_HW_AUTH_ALGO_AES_XCBC_MAC, + authkey, authkeylen, cdesc->cd_cur_ptr + state1_size, + &state2_size)) { + PMD_DRV_LOG(ERR, "(XCBC)precompute failed"); + return -EFAULT; + } + break; + case ICP_QAT_HW_AUTH_ALGO_GALOIS_128: + case ICP_QAT_HW_AUTH_ALGO_GALOIS_64: + qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_GCM; + state1_size = ICP_QAT_HW_GALOIS_128_STATE1_SZ; + if (qat_alg_do_precomputes(cdesc->qat_hash_alg, + authkey, authkeylen, cdesc->cd_cur_ptr + state1_size, + &state2_size)) { + PMD_DRV_LOG(ERR, "(GCM)precompute failed"); + return -EFAULT; + } + /* + * Write (the length of AAD) into bytes 16-19 of state2 + * in big-endian format. This field is 8 bytes + */ + auth_param->u2.aad_sz = + RTE_ALIGN_CEIL(aad_length, 16); + auth_param->hash_state_sz = (auth_param->u2.aad_sz) >> 3; + + aad_len = (uint32_t *)(cdesc->cd_cur_ptr + + ICP_QAT_HW_GALOIS_128_STATE1_SZ + + ICP_QAT_HW_GALOIS_H_SZ); + *aad_len = rte_bswap32(aad_length); + cdesc->aad_len = aad_length; + break; + case ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2: + qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_SNOW3G; + state1_size = qat_hash_get_state1_size( + ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2); + state2_size = ICP_QAT_HW_SNOW_3G_UIA2_STATE2_SZ; + memset(cdesc->cd_cur_ptr, 0, state1_size + state2_size); + + cipherconfig = (struct icp_qat_hw_cipher_algo_blk *) + (cdesc->cd_cur_ptr + state1_size + state2_size); + cipherconfig->cipher_config.val = + ICP_QAT_HW_CIPHER_CONFIG_BUILD(ICP_QAT_HW_CIPHER_ECB_MODE, + ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2, + ICP_QAT_HW_CIPHER_KEY_CONVERT, + ICP_QAT_HW_CIPHER_ENCRYPT); + memcpy(cipherconfig->key, authkey, authkeylen); + memset(cipherconfig->key + authkeylen, + 0, ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ); + cdesc->cd_cur_ptr += sizeof(struct icp_qat_hw_cipher_config) + + authkeylen + ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ; + auth_param->hash_state_sz = ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ >> 3; + break; + case ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3: + hash->auth_config.config = + ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE0, + cdesc->qat_hash_alg, digestsize); + qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_ZUC; + state1_size = qat_hash_get_state1_size( + ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3); + state2_size = ICP_QAT_HW_ZUC_3G_EIA3_STATE2_SZ; + memset(cdesc->cd_cur_ptr, 0, state1_size + state2_size + + ICP_QAT_HW_ZUC_3G_EEA3_IV_SZ); + + memcpy(cdesc->cd_cur_ptr + state1_size, authkey, authkeylen); + cdesc->cd_cur_ptr += state1_size + state2_size + + ICP_QAT_HW_ZUC_3G_EEA3_IV_SZ; + auth_param->hash_state_sz = ICP_QAT_HW_ZUC_3G_EEA3_IV_SZ >> 3; + cdesc->min_qat_dev_gen = QAT_GEN2; + + break; + case ICP_QAT_HW_AUTH_ALGO_MD5: + if (qat_alg_do_precomputes(ICP_QAT_HW_AUTH_ALGO_MD5, + authkey, authkeylen, cdesc->cd_cur_ptr, + &state1_size)) { + PMD_DRV_LOG(ERR, "(MD5)precompute failed"); + return -EFAULT; + } + state2_size = ICP_QAT_HW_MD5_STATE2_SZ; + break; + case ICP_QAT_HW_AUTH_ALGO_NULL: + state1_size = qat_hash_get_state1_size( + ICP_QAT_HW_AUTH_ALGO_NULL); + state2_size = ICP_QAT_HW_NULL_STATE2_SZ; + break; + case ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC: + qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_CCM; + state1_size = qat_hash_get_state1_size( + ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC); + state2_size = ICP_QAT_HW_AES_CBC_MAC_KEY_SZ + + ICP_QAT_HW_AES_CCM_CBC_E_CTR0_SZ; + + if (aad_length > 0) { + aad_length += ICP_QAT_HW_CCM_AAD_B0_LEN + + ICP_QAT_HW_CCM_AAD_LEN_INFO; + auth_param->u2.aad_sz = + RTE_ALIGN_CEIL(aad_length, + ICP_QAT_HW_CCM_AAD_ALIGNMENT); + } else { + auth_param->u2.aad_sz = ICP_QAT_HW_CCM_AAD_B0_LEN; + } + + cdesc->aad_len = aad_length; + hash->auth_counter.counter = 0; + + hash_cd_ctrl->outer_prefix_sz = digestsize; + auth_param->hash_state_sz = digestsize; + + memcpy(cdesc->cd_cur_ptr + state1_size, authkey, authkeylen); + break; + case ICP_QAT_HW_AUTH_ALGO_KASUMI_F9: + state1_size = qat_hash_get_state1_size( + ICP_QAT_HW_AUTH_ALGO_KASUMI_F9); + state2_size = ICP_QAT_HW_KASUMI_F9_STATE2_SZ; + memset(cdesc->cd_cur_ptr, 0, state1_size + state2_size); + pTempKey = (uint32_t *)(cdesc->cd_cur_ptr + state1_size + + authkeylen); + /* + * The Inner Hash Initial State2 block must contain IK + * (Initialisation Key), followed by IK XOR-ed with KM + * (Key Modifier): IK||(IK^KM). + */ + /* write the auth key */ + memcpy(cdesc->cd_cur_ptr + state1_size, authkey, authkeylen); + /* initialise temp key with auth key */ + memcpy(pTempKey, authkey, authkeylen); + /* XOR Key with KASUMI F9 key modifier at 4 bytes level */ + for (wordIndex = 0; wordIndex < (authkeylen >> 2); wordIndex++) + pTempKey[wordIndex] ^= KASUMI_F9_KEY_MODIFIER_4_BYTES; + break; + default: + PMD_DRV_LOG(ERR, "Invalid HASH alg %u", cdesc->qat_hash_alg); + return -EFAULT; + } + + /* Request template setup */ + qat_alg_init_common_hdr(header, qat_proto_flag); + header->service_cmd_id = cdesc->qat_cmd; + + /* Auth CD config setup */ + hash_cd_ctrl->hash_cfg_offset = hash_offset >> 3; + hash_cd_ctrl->hash_flags = ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED; + hash_cd_ctrl->inner_res_sz = digestsize; + hash_cd_ctrl->final_sz = digestsize; + hash_cd_ctrl->inner_state1_sz = state1_size; + auth_param->auth_res_sz = digestsize; + + hash_cd_ctrl->inner_state2_sz = state2_size; + hash_cd_ctrl->inner_state2_offset = hash_cd_ctrl->hash_cfg_offset + + ((sizeof(struct icp_qat_hw_auth_setup) + + RTE_ALIGN_CEIL(hash_cd_ctrl->inner_state1_sz, 8)) + >> 3); + + cdesc->cd_cur_ptr += state1_size + state2_size; + cd_size = cdesc->cd_cur_ptr-(uint8_t *)&cdesc->cd; + + cd_pars->u.s.content_desc_addr = cdesc->cd_paddr; + cd_pars->u.s.content_desc_params_sz = RTE_ALIGN_CEIL(cd_size, 8) >> 3; + return 0; } -static void qat_alg_ablkcipher_init_com(struct icp_qat_fw_la_bulk_req *req, - struct icp_qat_hw_cipher_algo_blk *cd, - const uint8_t *key, unsigned int keylen) -{ - struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars; - struct icp_qat_fw_comn_req_hdr *header = &req->comn_hdr; - struct icp_qat_fw_cipher_cd_ctrl_hdr *cd_ctrl = (void *)&req->cd_ctrl; - - PMD_INIT_FUNC_TRACE(); - rte_memcpy(cd->aes.key, key, keylen); - qat_alg_init_common_hdr(header); - header->service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER; - cd_pars->u.s.content_desc_params_sz = - sizeof(struct icp_qat_hw_cipher_algo_blk) >> 3; - /* Cipher CD config setup */ - cd_ctrl->cipher_key_sz = keylen >> 3; - cd_ctrl->cipher_state_sz = ICP_QAT_HW_AES_BLK_SZ >> 3; - cd_ctrl->cipher_cfg_offset = 0; - ICP_QAT_FW_COMN_CURR_ID_SET(cd_ctrl, ICP_QAT_FW_SLICE_CIPHER); - ICP_QAT_FW_COMN_NEXT_ID_SET(cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR); -} - -void qat_alg_ablkcipher_init_enc(struct qat_alg_ablkcipher_cd *cdesc, - int alg, const uint8_t *key, - unsigned int keylen) -{ - struct icp_qat_hw_cipher_algo_blk *enc_cd = cdesc->cd; - struct icp_qat_fw_la_bulk_req *req = &cdesc->fw_req; - struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars; - - PMD_INIT_FUNC_TRACE(); - qat_alg_ablkcipher_init_com(req, enc_cd, key, keylen); - cd_pars->u.s.content_desc_addr = cdesc->cd_paddr; - enc_cd->aes.cipher_config.val = QAT_AES_HW_CONFIG_CBC_ENC(alg); -} - -void qat_alg_ablkcipher_init_dec(struct qat_alg_ablkcipher_cd *cdesc, - int alg, const uint8_t *key, - unsigned int keylen) -{ - struct icp_qat_hw_cipher_algo_blk *dec_cd = cdesc->cd; - struct icp_qat_fw_la_bulk_req *req = &cdesc->fw_req; - struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars; - - PMD_INIT_FUNC_TRACE(); - qat_alg_ablkcipher_init_com(req, dec_cd, key, keylen); - cd_pars->u.s.content_desc_addr = cdesc->cd_paddr; - dec_cd->aes.cipher_config.val = QAT_AES_HW_CONFIG_CBC_DEC(alg); -} - int qat_alg_validate_aes_key(int key_len, enum icp_qat_hw_cipher_algo *alg) { switch (key_len) { @@ -825,6 +984,19 @@ int qat_alg_validate_aes_key(int key_len, enum icp_qat_hw_cipher_algo *alg) return 0; } +int qat_alg_validate_aes_docsisbpi_key(int key_len, + enum icp_qat_hw_cipher_algo *alg) +{ + switch (key_len) { + case ICP_QAT_HW_AES_128_KEY_SZ: + *alg = ICP_QAT_HW_CIPHER_ALGO_AES128; + break; + default: + return -EINVAL; + } + return 0; +} + int qat_alg_validate_snow3g_key(int key_len, enum icp_qat_hw_cipher_algo *alg) { switch (key_len) { @@ -836,3 +1008,52 @@ int qat_alg_validate_snow3g_key(int key_len, enum icp_qat_hw_cipher_algo *alg) } return 0; } + +int qat_alg_validate_kasumi_key(int key_len, enum icp_qat_hw_cipher_algo *alg) +{ + switch (key_len) { + case ICP_QAT_HW_KASUMI_KEY_SZ: + *alg = ICP_QAT_HW_CIPHER_ALGO_KASUMI; + break; + default: + return -EINVAL; + } + return 0; +} + +int qat_alg_validate_des_key(int key_len, enum icp_qat_hw_cipher_algo *alg) +{ + switch (key_len) { + case ICP_QAT_HW_DES_KEY_SZ: + *alg = ICP_QAT_HW_CIPHER_ALGO_DES; + break; + default: + return -EINVAL; + } + return 0; +} + +int qat_alg_validate_3des_key(int key_len, enum icp_qat_hw_cipher_algo *alg) +{ + switch (key_len) { + case QAT_3DES_KEY_SZ_OPT1: + case QAT_3DES_KEY_SZ_OPT2: + *alg = ICP_QAT_HW_CIPHER_ALGO_3DES; + break; + default: + return -EINVAL; + } + return 0; +} + +int qat_alg_validate_zuc_key(int key_len, enum icp_qat_hw_cipher_algo *alg) +{ + switch (key_len) { + case ICP_QAT_HW_ZUC_3G_EEA3_KEY_SZ: + *alg = ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3; + break; + default: + return -EINVAL; + } + return 0; +} diff --git a/dpdk/drivers/crypto/qat/qat_crypto.c b/dpdk/drivers/crypto/qat/qat_crypto.c index d51ca968..acd979d2 100644 --- a/dpdk/drivers/crypto/qat/qat_crypto.c +++ b/dpdk/drivers/crypto/qat/qat_crypto.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2015-2016 Intel Corporation. All rights reserved. + * Copyright(c) 2015-2017 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -44,22 +44,24 @@ #include #include #include -#include #include -#include #include #include #include #include #include -#include #include -#include #include #include #include #include #include +#include +#include +#include +#include + +#include #include "qat_logs.h" #include "qat_algs.h" @@ -67,244 +69,179 @@ #include "adf_transport_access_macros.h" #define BYTE_LENGTH 8 +/* bpi is only used for partial blocks of DES and AES + * so AES block len can be assumed as max len for iv, src and dst + */ +#define BPI_MAX_ENCR_IV_LEN ICP_QAT_HW_AES_BLK_SZ -static const struct rte_cryptodev_capabilities qat_pmd_capabilities[] = { - { /* SHA1 HMAC */ - .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, - {.sym = { - .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, - {.auth = { - .algo = RTE_CRYPTO_AUTH_SHA1_HMAC, - .block_size = 64, - .key_size = { - .min = 64, - .max = 64, - .increment = 0 - }, - .digest_size = { - .min = 20, - .max = 20, - .increment = 0 - }, - .aad_size = { 0 } - }, } - }, } - }, - { /* SHA256 HMAC */ - .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, - {.sym = { - .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, - {.auth = { - .algo = RTE_CRYPTO_AUTH_SHA256_HMAC, - .block_size = 64, - .key_size = { - .min = 64, - .max = 64, - .increment = 0 - }, - .digest_size = { - .min = 32, - .max = 32, - .increment = 0 - }, - .aad_size = { 0 } - }, } - }, } - }, - { /* SHA512 HMAC */ - .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, - {.sym = { - .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, - {.auth = { - .algo = RTE_CRYPTO_AUTH_SHA512_HMAC, - .block_size = 128, - .key_size = { - .min = 128, - .max = 128, - .increment = 0 - }, - .digest_size = { - .min = 64, - .max = 64, - .increment = 0 - }, - .aad_size = { 0 } - }, } - }, } - }, - { /* AES XCBC MAC */ - .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, - {.sym = { - .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, - {.auth = { - .algo = RTE_CRYPTO_AUTH_AES_XCBC_MAC, - .block_size = 16, - .key_size = { - .min = 16, - .max = 16, - .increment = 0 - }, - .digest_size = { - .min = 16, - .max = 16, - .increment = 0 - }, - .aad_size = { 0 } - }, } - }, } - }, - { /* AES GCM (AUTH) */ - .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, - {.sym = { - .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, - {.auth = { - .algo = RTE_CRYPTO_AUTH_AES_GCM, - .block_size = 16, - .key_size = { - .min = 16, - .max = 32, - .increment = 8 - }, - .digest_size = { - .min = 8, - .max = 16, - .increment = 4 - }, - .aad_size = { - .min = 8, - .max = 12, - .increment = 4 - } - }, } - }, } - }, - { /* SNOW3G (UIA2) */ - .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, - {.sym = { - .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, - {.auth = { - .algo = RTE_CRYPTO_AUTH_SNOW3G_UIA2, - .block_size = 16, - .key_size = { - .min = 16, - .max = 16, - .increment = 0 - }, - .digest_size = { - .min = 4, - .max = 4, - .increment = 0 - }, - .aad_size = { - .min = 16, - .max = 16, - .increment = 0 - } - }, } - }, } - }, - { /* AES GCM (CIPHER) */ - .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, - {.sym = { - .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, - {.cipher = { - .algo = RTE_CRYPTO_CIPHER_AES_GCM, - .block_size = 16, - .key_size = { - .min = 16, - .max = 32, - .increment = 8 - }, - .iv_size = { - .min = 16, - .max = 16, - .increment = 0 - } - }, } - }, } - }, - { /* AES CBC */ - .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, - {.sym = { - .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, - {.cipher = { - .algo = RTE_CRYPTO_CIPHER_AES_CBC, - .block_size = 16, - .key_size = { - .min = 16, - .max = 32, - .increment = 8 - }, - .iv_size = { - .min = 16, - .max = 16, - .increment = 0 - } - }, } - }, } - }, - { /* SNOW3G (UEA2) */ - .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, - {.sym = { - .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, - {.cipher = { - .algo = RTE_CRYPTO_CIPHER_SNOW3G_UEA2, - .block_size = 16, - .key_size = { - .min = 16, - .max = 16, - .increment = 0 - }, - .iv_size = { - .min = 16, - .max = 16, - .increment = 0 - } - }, } - }, } - }, - { /* AES CTR */ - .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, - {.sym = { - .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, - {.cipher = { - .algo = RTE_CRYPTO_CIPHER_AES_CTR, - .block_size = 16, - .key_size = { - .min = 16, - .max = 32, - .increment = 8 - }, - .iv_size = { - .min = 16, - .max = 16, - .increment = 0 - } - }, } - }, } - }, - RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST() -}; +static int +qat_is_cipher_alg_supported(enum rte_crypto_cipher_algorithm algo, + struct qat_pmd_private *internals) { + int i = 0; + const struct rte_cryptodev_capabilities *capability; + + while ((capability = &(internals->qat_dev_capabilities[i++]))->op != + RTE_CRYPTO_OP_TYPE_UNDEFINED) { + if (capability->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC) + continue; + + if (capability->sym.xform_type != RTE_CRYPTO_SYM_XFORM_CIPHER) + continue; + + if (capability->sym.cipher.algo == algo) + return 1; + } + return 0; +} + +static int +qat_is_auth_alg_supported(enum rte_crypto_auth_algorithm algo, + struct qat_pmd_private *internals) { + int i = 0; + const struct rte_cryptodev_capabilities *capability; + + while ((capability = &(internals->qat_dev_capabilities[i++]))->op != + RTE_CRYPTO_OP_TYPE_UNDEFINED) { + if (capability->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC) + continue; + + if (capability->sym.xform_type != RTE_CRYPTO_SYM_XFORM_AUTH) + continue; + + if (capability->sym.auth.algo == algo) + return 1; + } + return 0; +} + +/** Encrypt a single partial block + * Depends on openssl libcrypto + * Uses ECB+XOR to do CFB encryption, same result, more performant + */ +static inline int +bpi_cipher_encrypt(uint8_t *src, uint8_t *dst, + uint8_t *iv, int ivlen, int srclen, + void *bpi_ctx) +{ + EVP_CIPHER_CTX *ctx = (EVP_CIPHER_CTX *)bpi_ctx; + int encrypted_ivlen; + uint8_t encrypted_iv[BPI_MAX_ENCR_IV_LEN]; + uint8_t *encr = encrypted_iv; + + /* ECB method: encrypt the IV, then XOR this with plaintext */ + if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, iv, ivlen) + <= 0) + goto cipher_encrypt_err; + + for (; srclen != 0; --srclen, ++dst, ++src, ++encr) + *dst = *src ^ *encr; + + return 0; + +cipher_encrypt_err: + PMD_DRV_LOG(ERR, "libcrypto ECB cipher encrypt failed"); + return -EINVAL; +} + +/** Decrypt a single partial block + * Depends on openssl libcrypto + * Uses ECB+XOR to do CFB encryption, same result, more performant + */ +static inline int +bpi_cipher_decrypt(uint8_t *src, uint8_t *dst, + uint8_t *iv, int ivlen, int srclen, + void *bpi_ctx) +{ + EVP_CIPHER_CTX *ctx = (EVP_CIPHER_CTX *)bpi_ctx; + int encrypted_ivlen; + uint8_t encrypted_iv[BPI_MAX_ENCR_IV_LEN]; + uint8_t *encr = encrypted_iv; + + /* ECB method: encrypt (not decrypt!) the IV, then XOR with plaintext */ + if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, iv, ivlen) + <= 0) + goto cipher_decrypt_err; + + for (; srclen != 0; --srclen, ++dst, ++src, ++encr) + *dst = *src ^ *encr; + + return 0; + +cipher_decrypt_err: + PMD_DRV_LOG(ERR, "libcrypto ECB cipher encrypt for BPI IV failed"); + return -EINVAL; +} + +/** Creates a context in either AES or DES in ECB mode + * Depends on openssl libcrypto + */ +static int +bpi_cipher_ctx_init(enum rte_crypto_cipher_algorithm cryptodev_algo, + enum rte_crypto_cipher_operation direction __rte_unused, + uint8_t *key, void **ctx) +{ + const EVP_CIPHER *algo = NULL; + int ret; + *ctx = EVP_CIPHER_CTX_new(); + + if (*ctx == NULL) { + ret = -ENOMEM; + goto ctx_init_err; + } + + if (cryptodev_algo == RTE_CRYPTO_CIPHER_DES_DOCSISBPI) + algo = EVP_des_ecb(); + else + algo = EVP_aes_128_ecb(); + + /* IV will be ECB encrypted whether direction is encrypt or decrypt*/ + if (EVP_EncryptInit_ex(*ctx, algo, NULL, key, 0) != 1) { + ret = -EINVAL; + goto ctx_init_err; + } + + return 0; + +ctx_init_err: + if (*ctx != NULL) + EVP_CIPHER_CTX_free(*ctx); + return ret; +} + +/** Frees a context previously created + * Depends on openssl libcrypto + */ +static void +bpi_cipher_ctx_free(void *bpi_ctx) +{ + if (bpi_ctx != NULL) + EVP_CIPHER_CTX_free((EVP_CIPHER_CTX *)bpi_ctx); +} static inline uint32_t adf_modulo(uint32_t data, uint32_t shift); static inline int -qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg); +qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, + struct qat_crypto_op_cookie *qat_op_cookie, struct qat_qp *qp); -void qat_crypto_sym_clear_session(struct rte_cryptodev *dev, - void *session) +void +qat_crypto_sym_clear_session(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) { - struct qat_session *sess = session; - phys_addr_t cd_paddr; - PMD_INIT_FUNC_TRACE(); - if (session) { - cd_paddr = sess->cd_paddr; - memset(sess, 0, qat_crypto_sym_get_session_private_size(dev)); - sess->cd_paddr = cd_paddr; - } else - PMD_DRV_LOG(ERR, "NULL session"); + uint8_t index = dev->driver_id; + void *sess_priv = get_session_private_data(sess, index); + struct qat_session *s = (struct qat_session *)sess_priv; + + if (sess_priv) { + if (s->bpi_ctx) + bpi_cipher_ctx_free(s->bpi_ctx); + memset(s, 0, qat_crypto_sym_get_session_private_size(dev)); + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + set_session_private_data(sess, index, NULL); + rte_mempool_put(sess_mp, sess_priv); + } } static int @@ -318,6 +255,25 @@ qat_get_cmd_id(const struct rte_crypto_sym_xform *xform) if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH && xform->next == NULL) return ICP_QAT_FW_LA_CMD_AUTH; + /* AEAD */ + if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) { + /* AES-GCM and AES-CCM works with different direction + * GCM first encrypts and generate hash where AES-CCM + * first generate hash and encrypts. Similar relation + * applies to decryption. + */ + if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) + if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_GCM) + return ICP_QAT_FW_LA_CMD_CIPHER_HASH; + else + return ICP_QAT_FW_LA_CMD_HASH_CIPHER; + else + if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_GCM) + return ICP_QAT_FW_LA_CMD_HASH_CIPHER; + else + return ICP_QAT_FW_LA_CMD_CIPHER_HASH; + } + if (xform->next == NULL) return -1; @@ -359,40 +315,37 @@ qat_get_cipher_xform(struct rte_crypto_sym_xform *xform) return NULL; } -void * + +int qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev, - struct rte_crypto_sym_xform *xform, void *session_private) + struct rte_crypto_sym_xform *xform, + struct qat_session *session) { struct qat_pmd_private *internals = dev->data->dev_private; - - struct qat_session *session = session_private; - struct rte_crypto_cipher_xform *cipher_xform = NULL; + int ret; /* Get cipher xform from crypto xform chain */ cipher_xform = qat_get_cipher_xform(xform); + session->cipher_iv.offset = cipher_xform->iv.offset; + session->cipher_iv.length = cipher_xform->iv.length; + switch (cipher_xform->algo) { case RTE_CRYPTO_CIPHER_AES_CBC: if (qat_alg_validate_aes_key(cipher_xform->key.length, &session->qat_cipher_alg) != 0) { PMD_DRV_LOG(ERR, "Invalid AES cipher key size"); + ret = -EINVAL; goto error_out; } session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE; break; - case RTE_CRYPTO_CIPHER_AES_GCM: - if (qat_alg_validate_aes_key(cipher_xform->key.length, - &session->qat_cipher_alg) != 0) { - PMD_DRV_LOG(ERR, "Invalid AES cipher key size"); - goto error_out; - } - session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE; - break; case RTE_CRYPTO_CIPHER_AES_CTR: if (qat_alg_validate_aes_key(cipher_xform->key.length, &session->qat_cipher_alg) != 0) { PMD_DRV_LOG(ERR, "Invalid AES cipher key size"); + ret = -EINVAL; goto error_out; } session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE; @@ -400,23 +353,117 @@ qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev, case RTE_CRYPTO_CIPHER_SNOW3G_UEA2: if (qat_alg_validate_snow3g_key(cipher_xform->key.length, &session->qat_cipher_alg) != 0) { - PMD_DRV_LOG(ERR, "Invalid SNOW3G cipher key size"); + PMD_DRV_LOG(ERR, "Invalid SNOW 3G cipher key size"); + ret = -EINVAL; goto error_out; } session->qat_mode = ICP_QAT_HW_CIPHER_ECB_MODE; break; case RTE_CRYPTO_CIPHER_NULL: - case RTE_CRYPTO_CIPHER_3DES_ECB: - case RTE_CRYPTO_CIPHER_3DES_CBC: - case RTE_CRYPTO_CIPHER_AES_ECB: - case RTE_CRYPTO_CIPHER_AES_CCM: + session->qat_mode = ICP_QAT_HW_CIPHER_ECB_MODE; + break; case RTE_CRYPTO_CIPHER_KASUMI_F8: - PMD_DRV_LOG(ERR, "Crypto: Unsupported Cipher alg %u", + if (qat_alg_validate_kasumi_key(cipher_xform->key.length, + &session->qat_cipher_alg) != 0) { + PMD_DRV_LOG(ERR, "Invalid KASUMI cipher key size"); + ret = -EINVAL; + goto error_out; + } + session->qat_mode = ICP_QAT_HW_CIPHER_F8_MODE; + break; + case RTE_CRYPTO_CIPHER_3DES_CBC: + if (qat_alg_validate_3des_key(cipher_xform->key.length, + &session->qat_cipher_alg) != 0) { + PMD_DRV_LOG(ERR, "Invalid 3DES cipher key size"); + ret = -EINVAL; + goto error_out; + } + session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE; + break; + case RTE_CRYPTO_CIPHER_DES_CBC: + if (qat_alg_validate_des_key(cipher_xform->key.length, + &session->qat_cipher_alg) != 0) { + PMD_DRV_LOG(ERR, "Invalid DES cipher key size"); + ret = -EINVAL; + goto error_out; + } + session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE; + break; + case RTE_CRYPTO_CIPHER_3DES_CTR: + if (qat_alg_validate_3des_key(cipher_xform->key.length, + &session->qat_cipher_alg) != 0) { + PMD_DRV_LOG(ERR, "Invalid 3DES cipher key size"); + ret = -EINVAL; + goto error_out; + } + session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE; + break; + case RTE_CRYPTO_CIPHER_DES_DOCSISBPI: + ret = bpi_cipher_ctx_init( + cipher_xform->algo, + cipher_xform->op, + cipher_xform->key.data, + &session->bpi_ctx); + if (ret != 0) { + PMD_DRV_LOG(ERR, "failed to create DES BPI ctx"); + goto error_out; + } + if (qat_alg_validate_des_key(cipher_xform->key.length, + &session->qat_cipher_alg) != 0) { + PMD_DRV_LOG(ERR, "Invalid DES cipher key size"); + ret = -EINVAL; + goto error_out; + } + session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE; + break; + case RTE_CRYPTO_CIPHER_AES_DOCSISBPI: + ret = bpi_cipher_ctx_init( + cipher_xform->algo, + cipher_xform->op, + cipher_xform->key.data, + &session->bpi_ctx); + if (ret != 0) { + PMD_DRV_LOG(ERR, "failed to create AES BPI ctx"); + goto error_out; + } + if (qat_alg_validate_aes_docsisbpi_key(cipher_xform->key.length, + &session->qat_cipher_alg) != 0) { + PMD_DRV_LOG(ERR, "Invalid AES DOCSISBPI key size"); + ret = -EINVAL; + goto error_out; + } + session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE; + break; + case RTE_CRYPTO_CIPHER_ZUC_EEA3: + if (!qat_is_cipher_alg_supported( + cipher_xform->algo, internals)) { + PMD_DRV_LOG(ERR, "%s not supported on this device", + rte_crypto_cipher_algorithm_strings + [cipher_xform->algo]); + ret = -ENOTSUP; + goto error_out; + } + if (qat_alg_validate_zuc_key(cipher_xform->key.length, + &session->qat_cipher_alg) != 0) { + PMD_DRV_LOG(ERR, "Invalid ZUC cipher key size"); + ret = -EINVAL; + goto error_out; + } + session->qat_mode = ICP_QAT_HW_CIPHER_ECB_MODE; + break; + case RTE_CRYPTO_CIPHER_3DES_ECB: + case RTE_CRYPTO_CIPHER_AES_ECB: + case RTE_CRYPTO_CIPHER_AES_F8: + case RTE_CRYPTO_CIPHER_AES_XTS: + case RTE_CRYPTO_CIPHER_ARC4: + PMD_DRV_LOG(ERR, "Crypto QAT PMD: Unsupported Cipher alg %u", cipher_xform->algo); + ret = -ENOTSUP; goto error_out; default: PMD_DRV_LOG(ERR, "Crypto: Undefined Cipher specified %u\n", cipher_xform->algo); + ret = -EINVAL; goto error_out; } @@ -427,50 +474,119 @@ qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev, if (qat_alg_aead_session_create_content_desc_cipher(session, cipher_xform->key.data, - cipher_xform->key.length)) + cipher_xform->key.length)) { + ret = -EINVAL; goto error_out; + } - return session; + return 0; error_out: - rte_mempool_put(internals->sess_mp, session); - return NULL; + if (session->bpi_ctx) { + bpi_cipher_ctx_free(session->bpi_ctx); + session->bpi_ctx = NULL; + } + return ret; } - -void * +int qat_crypto_sym_configure_session(struct rte_cryptodev *dev, + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool) +{ + void *sess_private_data; + int ret; + + if (rte_mempool_get(mempool, &sess_private_data)) { + CDEV_LOG_ERR( + "Couldn't get object from session mempool"); + return -ENOMEM; + } + + ret = qat_crypto_set_session_parameters(dev, xform, sess_private_data); + if (ret != 0) { + PMD_DRV_LOG(ERR, "Crypto QAT PMD: failed to configure " + "session parameters"); + + /* Return session to mempool */ + rte_mempool_put(mempool, sess_private_data); + return ret; + } + + set_session_private_data(sess, dev->driver_id, + sess_private_data); + + return 0; +} + +int +qat_crypto_set_session_parameters(struct rte_cryptodev *dev, struct rte_crypto_sym_xform *xform, void *session_private) { - struct qat_pmd_private *internals = dev->data->dev_private; - struct qat_session *session = session_private; + int ret; int qat_cmd_id; - PMD_INIT_FUNC_TRACE(); + /* Set context descriptor physical address */ + session->cd_paddr = rte_mempool_virt2iova(session) + + offsetof(struct qat_session, cd); + + session->min_qat_dev_gen = QAT_GEN1; + /* Get requested QAT command id */ qat_cmd_id = qat_get_cmd_id(xform); if (qat_cmd_id < 0 || qat_cmd_id >= ICP_QAT_FW_LA_CMD_DELIMITER) { PMD_DRV_LOG(ERR, "Unsupported xform chain requested"); - goto error_out; + return -ENOTSUP; } session->qat_cmd = (enum icp_qat_fw_la_cmd_id)qat_cmd_id; switch (session->qat_cmd) { case ICP_QAT_FW_LA_CMD_CIPHER: - session = qat_crypto_sym_configure_session_cipher(dev, xform, session); + ret = qat_crypto_sym_configure_session_cipher(dev, xform, session); + if (ret < 0) + return ret; break; case ICP_QAT_FW_LA_CMD_AUTH: - session = qat_crypto_sym_configure_session_auth(dev, xform, session); + ret = qat_crypto_sym_configure_session_auth(dev, xform, session); + if (ret < 0) + return ret; break; case ICP_QAT_FW_LA_CMD_CIPHER_HASH: - session = qat_crypto_sym_configure_session_cipher(dev, xform, session); - session = qat_crypto_sym_configure_session_auth(dev, xform, session); + if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) { + ret = qat_crypto_sym_configure_session_aead(xform, + session); + if (ret < 0) + return ret; + } else { + ret = qat_crypto_sym_configure_session_cipher(dev, + xform, session); + if (ret < 0) + return ret; + ret = qat_crypto_sym_configure_session_auth(dev, + xform, session); + if (ret < 0) + return ret; + } break; case ICP_QAT_FW_LA_CMD_HASH_CIPHER: - session = qat_crypto_sym_configure_session_auth(dev, xform, session); - session = qat_crypto_sym_configure_session_cipher(dev, xform, session); + if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) { + ret = qat_crypto_sym_configure_session_aead(xform, + session); + if (ret < 0) + return ret; + } else { + ret = qat_crypto_sym_configure_session_auth(dev, + xform, session); + if (ret < 0) + return ret; + ret = qat_crypto_sym_configure_session_cipher(dev, + xform, session); + if (ret < 0) + return ret; + } break; case ICP_QAT_FW_LA_CMD_TRNG_GET_RANDOM: case ICP_QAT_FW_LA_CMD_TRNG_TEST: @@ -483,100 +599,243 @@ qat_crypto_sym_configure_session(struct rte_cryptodev *dev, case ICP_QAT_FW_LA_CMD_DELIMITER: PMD_DRV_LOG(ERR, "Unsupported Service %u", session->qat_cmd); - goto error_out; + return -ENOTSUP; default: PMD_DRV_LOG(ERR, "Unsupported Service %u", session->qat_cmd); - goto error_out; + return -ENOTSUP; } - return session; -error_out: - rte_mempool_put(internals->sess_mp, session); - return NULL; + return 0; } -struct qat_session * +int qat_crypto_sym_configure_session_auth(struct rte_cryptodev *dev, struct rte_crypto_sym_xform *xform, - struct qat_session *session_private) + struct qat_session *session) { - - struct qat_pmd_private *internals = dev->data->dev_private; - struct qat_session *session = session_private; struct rte_crypto_auth_xform *auth_xform = NULL; - struct rte_crypto_cipher_xform *cipher_xform = NULL; + struct qat_pmd_private *internals = dev->data->dev_private; auth_xform = qat_get_auth_xform(xform); + uint8_t *key_data = auth_xform->key.data; + uint8_t key_length = auth_xform->key.length; switch (auth_xform->algo) { case RTE_CRYPTO_AUTH_SHA1_HMAC: session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA1; break; + case RTE_CRYPTO_AUTH_SHA224_HMAC: + session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA224; + break; case RTE_CRYPTO_AUTH_SHA256_HMAC: session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA256; break; + case RTE_CRYPTO_AUTH_SHA384_HMAC: + session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA384; + break; case RTE_CRYPTO_AUTH_SHA512_HMAC: session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA512; break; case RTE_CRYPTO_AUTH_AES_XCBC_MAC: session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_AES_XCBC_MAC; break; - case RTE_CRYPTO_AUTH_AES_GCM: + case RTE_CRYPTO_AUTH_AES_GMAC: + if (qat_alg_validate_aes_key(auth_xform->key.length, + &session->qat_cipher_alg) != 0) { + PMD_DRV_LOG(ERR, "Invalid AES key size"); + return -EINVAL; + } + session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE; session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_GALOIS_128; + break; case RTE_CRYPTO_AUTH_SNOW3G_UIA2: session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2; break; + case RTE_CRYPTO_AUTH_MD5_HMAC: + session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_MD5; + break; case RTE_CRYPTO_AUTH_NULL: + session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_NULL; + break; + case RTE_CRYPTO_AUTH_KASUMI_F9: + session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_KASUMI_F9; + break; + case RTE_CRYPTO_AUTH_ZUC_EIA3: + if (!qat_is_auth_alg_supported(auth_xform->algo, internals)) { + PMD_DRV_LOG(ERR, "%s not supported on this device", + rte_crypto_auth_algorithm_strings + [auth_xform->algo]); + return -ENOTSUP; + } + session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3; + break; case RTE_CRYPTO_AUTH_SHA1: case RTE_CRYPTO_AUTH_SHA256: case RTE_CRYPTO_AUTH_SHA512: case RTE_CRYPTO_AUTH_SHA224: - case RTE_CRYPTO_AUTH_SHA224_HMAC: case RTE_CRYPTO_AUTH_SHA384: - case RTE_CRYPTO_AUTH_SHA384_HMAC: case RTE_CRYPTO_AUTH_MD5: - case RTE_CRYPTO_AUTH_MD5_HMAC: - case RTE_CRYPTO_AUTH_AES_CCM: - case RTE_CRYPTO_AUTH_AES_GMAC: - case RTE_CRYPTO_AUTH_KASUMI_F9: case RTE_CRYPTO_AUTH_AES_CMAC: case RTE_CRYPTO_AUTH_AES_CBC_MAC: - case RTE_CRYPTO_AUTH_ZUC_EIA3: PMD_DRV_LOG(ERR, "Crypto: Unsupported hash alg %u", auth_xform->algo); - goto error_out; + return -ENOTSUP; default: PMD_DRV_LOG(ERR, "Crypto: Undefined Hash algo %u specified", auth_xform->algo); - goto error_out; + return -EINVAL; } - cipher_xform = qat_get_cipher_xform(xform); - if ((session->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128) || - (session->qat_hash_alg == - ICP_QAT_HW_AUTH_ALGO_GALOIS_64)) { - if (qat_alg_aead_session_create_content_desc_auth(session, - cipher_xform->key.data, - cipher_xform->key.length, - auth_xform->add_auth_data_length, - auth_xform->digest_length, - auth_xform->op)) - goto error_out; + session->auth_iv.offset = auth_xform->iv.offset; + session->auth_iv.length = auth_xform->iv.length; + + if (auth_xform->algo == RTE_CRYPTO_AUTH_AES_GMAC) { + if (auth_xform->op == RTE_CRYPTO_AUTH_OP_GENERATE) { + session->qat_cmd = ICP_QAT_FW_LA_CMD_CIPHER_HASH; + session->qat_dir = ICP_QAT_HW_CIPHER_ENCRYPT; + /* + * It needs to create cipher desc content first, + * then authentication + */ + if (qat_alg_aead_session_create_content_desc_cipher(session, + auth_xform->key.data, + auth_xform->key.length)) + return -EINVAL; + + if (qat_alg_aead_session_create_content_desc_auth(session, + key_data, + key_length, + 0, + auth_xform->digest_length, + auth_xform->op)) + return -EINVAL; + } else { + session->qat_cmd = ICP_QAT_FW_LA_CMD_HASH_CIPHER; + session->qat_dir = ICP_QAT_HW_CIPHER_DECRYPT; + /* + * It needs to create authentication desc content first, + * then cipher + */ + if (qat_alg_aead_session_create_content_desc_auth(session, + key_data, + key_length, + 0, + auth_xform->digest_length, + auth_xform->op)) + return -EINVAL; + + if (qat_alg_aead_session_create_content_desc_cipher(session, + auth_xform->key.data, + auth_xform->key.length)) + return -EINVAL; + } + /* Restore to authentication only only */ + session->qat_cmd = ICP_QAT_FW_LA_CMD_AUTH; } else { if (qat_alg_aead_session_create_content_desc_auth(session, - auth_xform->key.data, - auth_xform->key.length, - auth_xform->add_auth_data_length, + key_data, + key_length, + 0, auth_xform->digest_length, auth_xform->op)) - goto error_out; + return -EINVAL; } - return session; -error_out: - rte_mempool_put(internals->sess_mp, session); - return NULL; + session->digest_length = auth_xform->digest_length; + return 0; +} + +int +qat_crypto_sym_configure_session_aead(struct rte_crypto_sym_xform *xform, + struct qat_session *session) +{ + struct rte_crypto_aead_xform *aead_xform = &xform->aead; + enum rte_crypto_auth_operation crypto_operation; + + /* + * Store AEAD IV parameters as cipher IV, + * to avoid unnecessary memory usage + */ + session->cipher_iv.offset = xform->aead.iv.offset; + session->cipher_iv.length = xform->aead.iv.length; + + switch (aead_xform->algo) { + case RTE_CRYPTO_AEAD_AES_GCM: + if (qat_alg_validate_aes_key(aead_xform->key.length, + &session->qat_cipher_alg) != 0) { + PMD_DRV_LOG(ERR, "Invalid AES key size"); + return -EINVAL; + } + session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE; + session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_GALOIS_128; + break; + case RTE_CRYPTO_AEAD_AES_CCM: + if (qat_alg_validate_aes_key(aead_xform->key.length, + &session->qat_cipher_alg) != 0) { + PMD_DRV_LOG(ERR, "Invalid AES key size"); + return -EINVAL; + } + session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE; + session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC; + break; + default: + PMD_DRV_LOG(ERR, "Crypto: Undefined AEAD specified %u\n", + aead_xform->algo); + return -EINVAL; + } + + if ((aead_xform->op == RTE_CRYPTO_AEAD_OP_ENCRYPT && + aead_xform->algo == RTE_CRYPTO_AEAD_AES_GCM) || + (aead_xform->op == RTE_CRYPTO_AEAD_OP_DECRYPT && + aead_xform->algo == RTE_CRYPTO_AEAD_AES_CCM)) { + session->qat_dir = ICP_QAT_HW_CIPHER_ENCRYPT; + /* + * It needs to create cipher desc content first, + * then authentication + */ + + crypto_operation = aead_xform->algo == RTE_CRYPTO_AEAD_AES_GCM ? + RTE_CRYPTO_AUTH_OP_GENERATE : RTE_CRYPTO_AUTH_OP_VERIFY; + + if (qat_alg_aead_session_create_content_desc_cipher(session, + aead_xform->key.data, + aead_xform->key.length)) + return -EINVAL; + + if (qat_alg_aead_session_create_content_desc_auth(session, + aead_xform->key.data, + aead_xform->key.length, + aead_xform->aad_length, + aead_xform->digest_length, + crypto_operation)) + return -EINVAL; + } else { + session->qat_dir = ICP_QAT_HW_CIPHER_DECRYPT; + /* + * It needs to create authentication desc content first, + * then cipher + */ + + crypto_operation = aead_xform->algo == RTE_CRYPTO_AEAD_AES_GCM ? + RTE_CRYPTO_AUTH_OP_VERIFY : RTE_CRYPTO_AUTH_OP_GENERATE; + + if (qat_alg_aead_session_create_content_desc_auth(session, + aead_xform->key.data, + aead_xform->key.length, + aead_xform->aad_length, + aead_xform->digest_length, + crypto_operation)) + return -EINVAL; + + if (qat_alg_aead_session_create_content_desc_cipher(session, + aead_xform->key.data, + aead_xform->key.length)) + return -EINVAL; + } + + session->digest_length = aead_xform->digest_length; + return 0; } unsigned qat_crypto_sym_get_session_private_size( @@ -585,6 +844,124 @@ unsigned qat_crypto_sym_get_session_private_size( return RTE_ALIGN_CEIL(sizeof(struct qat_session), 8); } +static inline uint32_t +qat_bpicipher_preprocess(struct qat_session *ctx, + struct rte_crypto_op *op) +{ + int block_len = qat_cipher_get_block_size(ctx->qat_cipher_alg); + struct rte_crypto_sym_op *sym_op = op->sym; + uint8_t last_block_len = block_len > 0 ? + sym_op->cipher.data.length % block_len : 0; + + if (last_block_len && + ctx->qat_dir == ICP_QAT_HW_CIPHER_DECRYPT) { + + /* Decrypt last block */ + uint8_t *last_block, *dst, *iv; + uint32_t last_block_offset = sym_op->cipher.data.offset + + sym_op->cipher.data.length - last_block_len; + last_block = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_src, + uint8_t *, last_block_offset); + + if (unlikely(sym_op->m_dst != NULL)) + /* out-of-place operation (OOP) */ + dst = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_dst, + uint8_t *, last_block_offset); + else + dst = last_block; + + if (last_block_len < sym_op->cipher.data.length) + /* use previous block ciphertext as IV */ + iv = last_block - block_len; + else + /* runt block, i.e. less than one full block */ + iv = rte_crypto_op_ctod_offset(op, uint8_t *, + ctx->cipher_iv.offset); + +#ifdef RTE_LIBRTE_PMD_QAT_DEBUG_TX + rte_hexdump(stdout, "BPI: src before pre-process:", last_block, + last_block_len); + if (sym_op->m_dst != NULL) + rte_hexdump(stdout, "BPI: dst before pre-process:", dst, + last_block_len); +#endif + bpi_cipher_decrypt(last_block, dst, iv, block_len, + last_block_len, ctx->bpi_ctx); +#ifdef RTE_LIBRTE_PMD_QAT_DEBUG_TX + rte_hexdump(stdout, "BPI: src after pre-process:", last_block, + last_block_len); + if (sym_op->m_dst != NULL) + rte_hexdump(stdout, "BPI: dst after pre-process:", dst, + last_block_len); +#endif + } + + return sym_op->cipher.data.length - last_block_len; +} + +static inline uint32_t +qat_bpicipher_postprocess(struct qat_session *ctx, + struct rte_crypto_op *op) +{ + int block_len = qat_cipher_get_block_size(ctx->qat_cipher_alg); + struct rte_crypto_sym_op *sym_op = op->sym; + uint8_t last_block_len = block_len > 0 ? + sym_op->cipher.data.length % block_len : 0; + + if (last_block_len > 0 && + ctx->qat_dir == ICP_QAT_HW_CIPHER_ENCRYPT) { + + /* Encrypt last block */ + uint8_t *last_block, *dst, *iv; + uint32_t last_block_offset; + + last_block_offset = sym_op->cipher.data.offset + + sym_op->cipher.data.length - last_block_len; + last_block = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_src, + uint8_t *, last_block_offset); + + if (unlikely(sym_op->m_dst != NULL)) + /* out-of-place operation (OOP) */ + dst = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_dst, + uint8_t *, last_block_offset); + else + dst = last_block; + + if (last_block_len < sym_op->cipher.data.length) + /* use previous block ciphertext as IV */ + iv = dst - block_len; + else + /* runt block, i.e. less than one full block */ + iv = rte_crypto_op_ctod_offset(op, uint8_t *, + ctx->cipher_iv.offset); + +#ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX + rte_hexdump(stdout, "BPI: src before post-process:", last_block, + last_block_len); + if (sym_op->m_dst != NULL) + rte_hexdump(stdout, "BPI: dst before post-process:", + dst, last_block_len); +#endif + bpi_cipher_encrypt(last_block, dst, iv, block_len, + last_block_len, ctx->bpi_ctx); +#ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX + rte_hexdump(stdout, "BPI: src after post-process:", last_block, + last_block_len); + if (sym_op->m_dst != NULL) + rte_hexdump(stdout, "BPI: dst after post-process:", dst, + last_block_len); +#endif + } + return sym_op->cipher.data.length - last_block_len; +} + +static inline void +txq_write_tail(struct qat_qp *qp, struct qat_queue *q) { + WRITE_CSR_RING_TAIL(qp->mmap_bar_addr, q->hw_bundle_number, + q->hw_queue_number, q->tail); + q->nb_pending_requests = 0; + q->csr_tail = q->tail; +} uint16_t qat_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops, @@ -609,19 +986,25 @@ qat_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops, tail = queue->tail; /* Find how many can actually fit on the ring */ - overflow = rte_atomic16_add_return(&tmp_qp->inflights16, nb_ops) - - queue->max_inflights; + tmp_qp->inflights16 += nb_ops; + overflow = tmp_qp->inflights16 - queue->max_inflights; if (overflow > 0) { - rte_atomic16_sub(&tmp_qp->inflights16, overflow); + tmp_qp->inflights16 -= overflow; nb_ops_possible = nb_ops - overflow; if (nb_ops_possible == 0) return 0; } while (nb_ops_sent != nb_ops_possible) { - ret = qat_write_hw_desc_entry(*cur_op, base_addr + tail); + ret = qat_write_hw_desc_entry(*cur_op, base_addr + tail, + tmp_qp->op_cookies[tail / queue->msg_size], tmp_qp); if (ret != 0) { tmp_qp->stats.enqueue_err_count++; + /* + * This message cannot be enqueued, + * decrease number of ops that wasn't sent + */ + tmp_qp->inflights16 -= nb_ops_possible - nb_ops_sent; if (nb_ops_sent == 0) return 0; goto kick_tail; @@ -632,26 +1015,59 @@ qat_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops, cur_op++; } kick_tail: - WRITE_CSR_RING_TAIL(tmp_qp->mmap_bar_addr, queue->hw_bundle_number, - queue->hw_queue_number, tail); queue->tail = tail; tmp_qp->stats.enqueued_count += nb_ops_sent; + queue->nb_pending_requests += nb_ops_sent; + if (tmp_qp->inflights16 < QAT_CSR_TAIL_FORCE_WRITE_THRESH || + queue->nb_pending_requests > QAT_CSR_TAIL_WRITE_THRESH) { + txq_write_tail(tmp_qp, queue); + } return nb_ops_sent; } +static inline +void rxq_free_desc(struct qat_qp *qp, struct qat_queue *q) +{ + uint32_t old_head, new_head; + uint32_t max_head; + + old_head = q->csr_head; + new_head = q->head; + max_head = qp->nb_descriptors * q->msg_size; + + /* write out free descriptors */ + void *cur_desc = (uint8_t *)q->base_addr + old_head; + + if (new_head < old_head) { + memset(cur_desc, ADF_RING_EMPTY_SIG, max_head - old_head); + memset(q->base_addr, ADF_RING_EMPTY_SIG, new_head); + } else { + memset(cur_desc, ADF_RING_EMPTY_SIG, new_head - old_head); + } + q->nb_processed_responses = 0; + q->csr_head = new_head; + + /* write current head to CSR */ + WRITE_CSR_RING_HEAD(qp->mmap_bar_addr, q->hw_bundle_number, + q->hw_queue_number, new_head); +} + uint16_t qat_pmd_dequeue_op_burst(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) { - struct qat_queue *queue; + struct qat_queue *rx_queue, *tx_queue; struct qat_qp *tmp_qp = (struct qat_qp *)qp; uint32_t msg_counter = 0; struct rte_crypto_op *rx_op; struct icp_qat_fw_comn_resp *resp_msg; + uint32_t head; - queue = &(tmp_qp->rx_q); + rx_queue = &(tmp_qp->rx_q); + tx_queue = &(tmp_qp->tx_q); + head = rx_queue->head; resp_msg = (struct icp_qat_fw_comn_resp *) - ((uint8_t *)queue->base_addr + queue->head); + ((uint8_t *)rx_queue->base_addr + head); while (*(uint32_t *)resp_msg != ADF_RING_EMPTY_SIG && msg_counter != nb_ops) { @@ -660,43 +1076,154 @@ qat_pmd_dequeue_op_burst(void *qp, struct rte_crypto_op **ops, #ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX rte_hexdump(stdout, "qat_response:", (uint8_t *)resp_msg, - sizeof(struct icp_qat_fw_comn_resp)); + sizeof(struct icp_qat_fw_comn_resp)); #endif if (ICP_QAT_FW_COMN_STATUS_FLAG_OK != ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET( resp_msg->comn_hdr.comn_status)) { rx_op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; } else { + struct qat_session *sess = (struct qat_session *) + get_session_private_data( + rx_op->sym->session, + cryptodev_qat_driver_id); + + if (sess->bpi_ctx) + qat_bpicipher_postprocess(sess, rx_op); rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; } - *(uint32_t *)resp_msg = ADF_RING_EMPTY_SIG; - queue->head = adf_modulo(queue->head + - queue->msg_size, - ADF_RING_SIZE_MODULO(queue->queue_size)); + + head = adf_modulo(head + rx_queue->msg_size, rx_queue->modulo); resp_msg = (struct icp_qat_fw_comn_resp *) - ((uint8_t *)queue->base_addr + - queue->head); + ((uint8_t *)rx_queue->base_addr + head); *ops = rx_op; ops++; msg_counter++; } if (msg_counter > 0) { - WRITE_CSR_RING_HEAD(tmp_qp->mmap_bar_addr, - queue->hw_bundle_number, - queue->hw_queue_number, queue->head); - rte_atomic16_sub(&tmp_qp->inflights16, msg_counter); + rx_queue->head = head; tmp_qp->stats.dequeued_count += msg_counter; + rx_queue->nb_processed_responses += msg_counter; + tmp_qp->inflights16 -= msg_counter; + + if (rx_queue->nb_processed_responses > QAT_CSR_HEAD_WRITE_THRESH) + rxq_free_desc(tmp_qp, rx_queue); + } + /* also check if tail needs to be advanced */ + if (tmp_qp->inflights16 <= QAT_CSR_TAIL_FORCE_WRITE_THRESH && + tx_queue->tail != tx_queue->csr_tail) { + txq_write_tail(tmp_qp, tx_queue); } return msg_counter; } static inline int -qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg) +qat_sgl_fill_array(struct rte_mbuf *buf, uint64_t buff_start, + struct qat_alg_buf_list *list, uint32_t data_len) { + int nr = 1; + + uint32_t buf_len = rte_pktmbuf_iova(buf) - + buff_start + rte_pktmbuf_data_len(buf); + + list->bufers[0].addr = buff_start; + list->bufers[0].resrvd = 0; + list->bufers[0].len = buf_len; + + if (data_len <= buf_len) { + list->num_bufs = nr; + list->bufers[0].len = data_len; + return 0; + } + + buf = buf->next; + while (buf) { + if (unlikely(nr == QAT_SGL_MAX_NUMBER)) { + PMD_DRV_LOG(ERR, "QAT PMD exceeded size of QAT SGL" + " entry(%u)", + QAT_SGL_MAX_NUMBER); + return -EINVAL; + } + + list->bufers[nr].len = rte_pktmbuf_data_len(buf); + list->bufers[nr].resrvd = 0; + list->bufers[nr].addr = rte_pktmbuf_iova(buf); + + buf_len += list->bufers[nr].len; + buf = buf->next; + + if (buf_len > data_len) { + list->bufers[nr].len -= + buf_len - data_len; + buf = NULL; + } + ++nr; + } + list->num_bufs = nr; + + return 0; +} + +static inline void +set_cipher_iv(uint16_t iv_length, uint16_t iv_offset, + struct icp_qat_fw_la_cipher_req_params *cipher_param, + struct rte_crypto_op *op, + struct icp_qat_fw_la_bulk_req *qat_req) +{ + /* copy IV into request if it fits */ + if (iv_length <= sizeof(cipher_param->u.cipher_IV_array)) { + rte_memcpy(cipher_param->u.cipher_IV_array, + rte_crypto_op_ctod_offset(op, uint8_t *, + iv_offset), + iv_length); + } else { + ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET( + qat_req->comn_hdr.serv_specif_flags, + ICP_QAT_FW_CIPH_IV_64BIT_PTR); + cipher_param->u.s.cipher_IV_ptr = + rte_crypto_op_ctophys_offset(op, + iv_offset); + } +} + +/** Set IV for CCM is special case, 0th byte is set to q-1 + * where q is padding of nonce in 16 byte block + */ +static inline void +set_cipher_iv_ccm(uint16_t iv_length, uint16_t iv_offset, + struct icp_qat_fw_la_cipher_req_params *cipher_param, + struct rte_crypto_op *op, uint8_t q, uint8_t aad_len_field_sz) +{ + rte_memcpy(((uint8_t *)cipher_param->u.cipher_IV_array) + + ICP_QAT_HW_CCM_NONCE_OFFSET, + rte_crypto_op_ctod_offset(op, uint8_t *, + iv_offset) + ICP_QAT_HW_CCM_NONCE_OFFSET, + iv_length); + *(uint8_t *)&cipher_param->u.cipher_IV_array[0] = + q - ICP_QAT_HW_CCM_NONCE_OFFSET; + + if (aad_len_field_sz) + rte_memcpy(&op->sym->aead.aad.data[ICP_QAT_HW_CCM_NONCE_OFFSET], + rte_crypto_op_ctod_offset(op, uint8_t *, + iv_offset) + ICP_QAT_HW_CCM_NONCE_OFFSET, + iv_length); +} + +static inline int +qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, + struct qat_crypto_op_cookie *qat_op_cookie, struct qat_qp *qp) +{ + int ret = 0; struct qat_session *ctx; struct icp_qat_fw_la_cipher_req_params *cipher_param; struct icp_qat_fw_la_auth_req_params *auth_param; register struct icp_qat_fw_la_bulk_req *qat_req; + uint8_t do_auth = 0, do_cipher = 0, do_aead = 0; + uint32_t cipher_len = 0, cipher_ofs = 0; + uint32_t auth_len = 0, auth_ofs = 0; + uint32_t min_ofs = 0; + uint64_t src_buf_start = 0, dst_buf_start = 0; + uint8_t do_sgl = 0; #ifdef RTE_LIBRTE_PMD_QAT_DEBUG_TX if (unlikely(op->type != RTE_CRYPTO_OP_TYPE_SYMMETRIC)) { @@ -706,112 +1233,366 @@ qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg) return -EINVAL; } #endif - if (unlikely(op->sym->sess_type == RTE_CRYPTO_SYM_OP_SESSIONLESS)) { + if (unlikely(op->sess_type == RTE_CRYPTO_OP_SESSIONLESS)) { PMD_DRV_LOG(ERR, "QAT PMD only supports session oriented" " requests, op (%p) is sessionless.", op); return -EINVAL; } - if (unlikely(op->sym->session->dev_type != RTE_CRYPTODEV_QAT_SYM_PMD)) { + ctx = (struct qat_session *)get_session_private_data( + op->sym->session, cryptodev_qat_driver_id); + + if (unlikely(ctx == NULL)) { PMD_DRV_LOG(ERR, "Session was not created for this device"); return -EINVAL; } - ctx = (struct qat_session *)op->sym->session->_private; - qat_req = (struct icp_qat_fw_la_bulk_req *)out_msg; - *qat_req = ctx->fw_req; - qat_req->comn_mid.opaque_data = (uint64_t)(uintptr_t)op; - - qat_req->comn_mid.dst_length = - qat_req->comn_mid.src_length = - rte_pktmbuf_data_len(op->sym->m_src); - - qat_req->comn_mid.dest_data_addr = - qat_req->comn_mid.src_data_addr = - rte_pktmbuf_mtophys(op->sym->m_src); - - if (unlikely(op->sym->m_dst != NULL)) { - qat_req->comn_mid.dest_data_addr = - rte_pktmbuf_mtophys(op->sym->m_dst); - qat_req->comn_mid.dst_length = - rte_pktmbuf_data_len(op->sym->m_dst); + if (unlikely(ctx->min_qat_dev_gen > qp->qat_dev_gen)) { + PMD_DRV_LOG(ERR, "Session alg not supported on this device gen"); + op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; + return -EINVAL; } + + + qat_req = (struct icp_qat_fw_la_bulk_req *)out_msg; + rte_mov128((uint8_t *)qat_req, (const uint8_t *)&(ctx->fw_req)); + qat_req->comn_mid.opaque_data = (uint64_t)(uintptr_t)op; cipher_param = (void *)&qat_req->serv_specif_rqpars; auth_param = (void *)((uint8_t *)cipher_param + sizeof(*cipher_param)); - cipher_param->cipher_length = op->sym->cipher.data.length; - cipher_param->cipher_offset = op->sym->cipher.data.offset; - if (ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2) { - if (unlikely((cipher_param->cipher_length % BYTE_LENGTH != 0) || - (cipher_param->cipher_offset - % BYTE_LENGTH != 0))) { - PMD_DRV_LOG(ERR, " For Snow3g, QAT PMD only " - "supports byte aligned values"); - op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; - return -EINVAL; + if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_HASH_CIPHER || + ctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER_HASH) { + /* AES-GCM or AES-CCM */ + if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128 || + ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64 || + (ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_AES128 + && ctx->qat_mode == ICP_QAT_HW_CIPHER_CTR_MODE + && ctx->qat_hash_alg == + ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC)) { + do_aead = 1; + } else { + do_auth = 1; + do_cipher = 1; } - cipher_param->cipher_length >>= 3; - cipher_param->cipher_offset >>= 3; + } else if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_AUTH) { + do_auth = 1; + do_cipher = 0; + } else if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER) { + do_auth = 0; + do_cipher = 1; } - if (op->sym->cipher.iv.length && (op->sym->cipher.iv.length <= - sizeof(cipher_param->u.cipher_IV_array))) { - rte_memcpy(cipher_param->u.cipher_IV_array, - op->sym->cipher.iv.data, - op->sym->cipher.iv.length); - } else { - ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET( - qat_req->comn_hdr.serv_specif_flags, - ICP_QAT_FW_CIPH_IV_64BIT_PTR); - cipher_param->u.s.cipher_IV_ptr = op->sym->cipher.iv.phys_addr; - } - if (op->sym->auth.digest.phys_addr) { - ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET( - qat_req->comn_hdr.serv_specif_flags, - ICP_QAT_FW_LA_NO_DIGEST_IN_BUFFER); - auth_param->auth_res_addr = op->sym->auth.digest.phys_addr; - } - auth_param->auth_off = op->sym->auth.data.offset; - auth_param->auth_len = op->sym->auth.data.length; - if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2) { - if (unlikely((auth_param->auth_off % BYTE_LENGTH != 0) || - (auth_param->auth_len % BYTE_LENGTH != 0))) { - PMD_DRV_LOG(ERR, " For Snow3g, QAT PMD only " - "supports byte aligned values"); - op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; - return -EINVAL; - } - auth_param->auth_off >>= 3; - auth_param->auth_len >>= 3; - } - auth_param->u1.aad_adr = op->sym->auth.aad.phys_addr; - /* (GCM) aad length(240 max) will be at this location after precompute */ - if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128 || - ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64) { - struct icp_qat_hw_auth_algo_blk *hash; + if (do_cipher) { - if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_HASH_CIPHER) - hash = (struct icp_qat_hw_auth_algo_blk *)((char *)&ctx->cd); - else - hash = (struct icp_qat_hw_auth_algo_blk *)((char *)&ctx->cd + - sizeof(struct icp_qat_hw_cipher_algo_blk)); + if (ctx->qat_cipher_alg == + ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2 || + ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_KASUMI || + ctx->qat_cipher_alg == + ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3) { - auth_param->u2.aad_sz = ALIGN_POW2_ROUNDUP(hash->sha.state1[ - ICP_QAT_HW_GALOIS_128_STATE1_SZ + - ICP_QAT_HW_GALOIS_H_SZ + 3], 16); - if (op->sym->cipher.iv.length == 12) { - /* - * For GCM a 12 bit IV is allowed, - * but we need to inform the f/w + if (unlikely( + (cipher_param->cipher_length % BYTE_LENGTH != 0) + || (cipher_param->cipher_offset + % BYTE_LENGTH != 0))) { + PMD_DRV_LOG(ERR, + "SNOW3G/KASUMI/ZUC in QAT PMD only supports byte aligned values"); + op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; + return -EINVAL; + } + cipher_len = op->sym->cipher.data.length >> 3; + cipher_ofs = op->sym->cipher.data.offset >> 3; + + } else if (ctx->bpi_ctx) { + /* DOCSIS - only send complete blocks to device + * Process any partial block using CFB mode. + * Even if 0 complete blocks, still send this to device + * to get into rx queue for post-process and dequeuing */ - ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET( - qat_req->comn_hdr.serv_specif_flags, - ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS); + cipher_len = qat_bpicipher_preprocess(ctx, op); + cipher_ofs = op->sym->cipher.data.offset; + } else { + cipher_len = op->sym->cipher.data.length; + cipher_ofs = op->sym->cipher.data.offset; } - } - auth_param->hash_state_sz = (auth_param->u2.aad_sz) >> 3; + set_cipher_iv(ctx->cipher_iv.length, ctx->cipher_iv.offset, + cipher_param, op, qat_req); + min_ofs = cipher_ofs; + } + + if (do_auth) { + + if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2 || + ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_KASUMI_F9 || + ctx->qat_hash_alg == + ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3) { + if (unlikely((auth_param->auth_off % BYTE_LENGTH != 0) + || (auth_param->auth_len % BYTE_LENGTH != 0))) { + PMD_DRV_LOG(ERR, + "For SNOW3G/KASUMI/ZUC, QAT PMD only supports byte aligned values"); + op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; + return -EINVAL; + } + auth_ofs = op->sym->auth.data.offset >> 3; + auth_len = op->sym->auth.data.length >> 3; + + auth_param->u1.aad_adr = + rte_crypto_op_ctophys_offset(op, + ctx->auth_iv.offset); + + } else if (ctx->qat_hash_alg == + ICP_QAT_HW_AUTH_ALGO_GALOIS_128 || + ctx->qat_hash_alg == + ICP_QAT_HW_AUTH_ALGO_GALOIS_64) { + /* AES-GMAC */ + set_cipher_iv(ctx->auth_iv.length, + ctx->auth_iv.offset, + cipher_param, op, qat_req); + auth_ofs = op->sym->auth.data.offset; + auth_len = op->sym->auth.data.length; + + auth_param->u1.aad_adr = 0; + auth_param->u2.aad_sz = 0; + + /* + * If len(iv)==12B fw computes J0 + */ + if (ctx->auth_iv.length == 12) { + ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET( + qat_req->comn_hdr.serv_specif_flags, + ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS); + + } + } else { + auth_ofs = op->sym->auth.data.offset; + auth_len = op->sym->auth.data.length; + + } + min_ofs = auth_ofs; + + if (likely(ctx->qat_hash_alg != ICP_QAT_HW_AUTH_ALGO_NULL)) + auth_param->auth_res_addr = + op->sym->auth.digest.phys_addr; + + } + + if (do_aead) { + /* + * This address may used for setting AAD physical pointer + * into IV offset from op + */ + rte_iova_t aad_phys_addr_aead = op->sym->aead.aad.phys_addr; + if (ctx->qat_hash_alg == + ICP_QAT_HW_AUTH_ALGO_GALOIS_128 || + ctx->qat_hash_alg == + ICP_QAT_HW_AUTH_ALGO_GALOIS_64) { + /* + * If len(iv)==12B fw computes J0 + */ + if (ctx->cipher_iv.length == 12) { + ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET( + qat_req->comn_hdr.serv_specif_flags, + ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS); + } + + set_cipher_iv(ctx->cipher_iv.length, + ctx->cipher_iv.offset, + cipher_param, op, qat_req); + + } else if (ctx->qat_hash_alg == + ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC) { + + /* In case of AES-CCM this may point to user selected memory + * or iv offset in cypto_op + */ + uint8_t *aad_data = op->sym->aead.aad.data; + /* This is true AAD length, it not includes 18 bytes of + * preceding data + */ + uint8_t aad_ccm_real_len = 0; + + uint8_t aad_len_field_sz = 0; + uint32_t msg_len_be = + rte_bswap32(op->sym->aead.data.length); + + if (ctx->aad_len > ICP_QAT_HW_CCM_AAD_DATA_OFFSET) { + aad_len_field_sz = ICP_QAT_HW_CCM_AAD_LEN_INFO; + aad_ccm_real_len = ctx->aad_len - + ICP_QAT_HW_CCM_AAD_B0_LEN - + ICP_QAT_HW_CCM_AAD_LEN_INFO; + } else { + /* + * aad_len not greater than 18, so no actual aad data, + * then use IV after op for B0 block + */ + aad_data = rte_crypto_op_ctod_offset(op, uint8_t *, + ctx->cipher_iv.offset); + aad_phys_addr_aead = + rte_crypto_op_ctophys_offset(op, + ctx->cipher_iv.offset); + } + + uint8_t q = ICP_QAT_HW_CCM_NQ_CONST - ctx->cipher_iv.length; + + aad_data[0] = ICP_QAT_HW_CCM_BUILD_B0_FLAGS(aad_len_field_sz, + ctx->digest_length, q); + + if (q > ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE) { + memcpy(aad_data + ctx->cipher_iv.length + + ICP_QAT_HW_CCM_NONCE_OFFSET + + (q - ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE), + (uint8_t *)&msg_len_be, + ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE); + } else { + memcpy(aad_data + ctx->cipher_iv.length + + ICP_QAT_HW_CCM_NONCE_OFFSET, + (uint8_t *)&msg_len_be + + (ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE + - q), q); + } + + if (aad_len_field_sz > 0) { + *(uint16_t *)&aad_data[ICP_QAT_HW_CCM_AAD_B0_LEN] + = rte_bswap16(aad_ccm_real_len); + + if ((aad_ccm_real_len + aad_len_field_sz) + % ICP_QAT_HW_CCM_AAD_B0_LEN) { + uint8_t pad_len = 0; + uint8_t pad_idx = 0; + + pad_len = ICP_QAT_HW_CCM_AAD_B0_LEN - + ((aad_ccm_real_len + aad_len_field_sz) % + ICP_QAT_HW_CCM_AAD_B0_LEN); + pad_idx = ICP_QAT_HW_CCM_AAD_B0_LEN + + aad_ccm_real_len + aad_len_field_sz; + memset(&aad_data[pad_idx], + 0, pad_len); + } + + } + + set_cipher_iv_ccm(ctx->cipher_iv.length, + ctx->cipher_iv.offset, + cipher_param, op, q, + aad_len_field_sz); + + } + + cipher_len = op->sym->aead.data.length; + cipher_ofs = op->sym->aead.data.offset; + auth_len = op->sym->aead.data.length; + auth_ofs = op->sym->aead.data.offset; + + auth_param->u1.aad_adr = aad_phys_addr_aead; + auth_param->auth_res_addr = op->sym->aead.digest.phys_addr; + min_ofs = op->sym->aead.data.offset; + } + + if (op->sym->m_src->next || (op->sym->m_dst && op->sym->m_dst->next)) + do_sgl = 1; + + /* adjust for chain case */ + if (do_cipher && do_auth) + min_ofs = cipher_ofs < auth_ofs ? cipher_ofs : auth_ofs; + + if (unlikely(min_ofs >= rte_pktmbuf_data_len(op->sym->m_src) && do_sgl)) + min_ofs = 0; + + if (unlikely(op->sym->m_dst != NULL)) { + /* Out-of-place operation (OOP) + * Don't align DMA start. DMA the minimum data-set + * so as not to overwrite data in dest buffer + */ + src_buf_start = + rte_pktmbuf_iova_offset(op->sym->m_src, min_ofs); + dst_buf_start = + rte_pktmbuf_iova_offset(op->sym->m_dst, min_ofs); + + } else { + /* In-place operation + * Start DMA at nearest aligned address below min_ofs + */ + src_buf_start = + rte_pktmbuf_iova_offset(op->sym->m_src, min_ofs) + & QAT_64_BTYE_ALIGN_MASK; + + if (unlikely((rte_pktmbuf_iova(op->sym->m_src) - + rte_pktmbuf_headroom(op->sym->m_src)) + > src_buf_start)) { + /* alignment has pushed addr ahead of start of mbuf + * so revert and take the performance hit + */ + src_buf_start = + rte_pktmbuf_iova_offset(op->sym->m_src, + min_ofs); + } + dst_buf_start = src_buf_start; + } + + if (do_cipher || do_aead) { + cipher_param->cipher_offset = + (uint32_t)rte_pktmbuf_iova_offset( + op->sym->m_src, cipher_ofs) - src_buf_start; + cipher_param->cipher_length = cipher_len; + } else { + cipher_param->cipher_offset = 0; + cipher_param->cipher_length = 0; + } + + if (do_auth || do_aead) { + auth_param->auth_off = (uint32_t)rte_pktmbuf_iova_offset( + op->sym->m_src, auth_ofs) - src_buf_start; + auth_param->auth_len = auth_len; + } else { + auth_param->auth_off = 0; + auth_param->auth_len = 0; + } + + qat_req->comn_mid.dst_length = + qat_req->comn_mid.src_length = + (cipher_param->cipher_offset + cipher_param->cipher_length) + > (auth_param->auth_off + auth_param->auth_len) ? + (cipher_param->cipher_offset + cipher_param->cipher_length) + : (auth_param->auth_off + auth_param->auth_len); + + if (do_sgl) { + + ICP_QAT_FW_COMN_PTR_TYPE_SET(qat_req->comn_hdr.comn_req_flags, + QAT_COMN_PTR_TYPE_SGL); + ret = qat_sgl_fill_array(op->sym->m_src, src_buf_start, + &qat_op_cookie->qat_sgl_list_src, + qat_req->comn_mid.src_length); + if (ret) { + PMD_DRV_LOG(ERR, "QAT PMD Cannot fill sgl array"); + return ret; + } + + if (likely(op->sym->m_dst == NULL)) + qat_req->comn_mid.dest_data_addr = + qat_req->comn_mid.src_data_addr = + qat_op_cookie->qat_sgl_src_phys_addr; + else { + ret = qat_sgl_fill_array(op->sym->m_dst, + dst_buf_start, + &qat_op_cookie->qat_sgl_list_dst, + qat_req->comn_mid.dst_length); + + if (ret) { + PMD_DRV_LOG(ERR, "QAT PMD Cannot " + "fill sgl array"); + return ret; + } + + qat_req->comn_mid.src_data_addr = + qat_op_cookie->qat_sgl_src_phys_addr; + qat_req->comn_mid.dest_data_addr = + qat_op_cookie->qat_sgl_dst_phys_addr; + } + } else { + qat_req->comn_mid.src_data_addr = src_buf_start; + qat_req->comn_mid.dest_data_addr = dst_buf_start; + } #ifdef RTE_LIBRTE_PMD_QAT_DEBUG_TX rte_hexdump(stdout, "qat_req:", qat_req, @@ -819,12 +1600,32 @@ qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg) rte_hexdump(stdout, "src_data:", rte_pktmbuf_mtod(op->sym->m_src, uint8_t*), rte_pktmbuf_data_len(op->sym->m_src)); - rte_hexdump(stdout, "iv:", op->sym->cipher.iv.data, - op->sym->cipher.iv.length); - rte_hexdump(stdout, "digest:", op->sym->auth.digest.data, - op->sym->auth.digest.length); - rte_hexdump(stdout, "aad:", op->sym->auth.aad.data, - op->sym->auth.aad.length); + if (do_cipher) { + uint8_t *cipher_iv_ptr = rte_crypto_op_ctod_offset(op, + uint8_t *, + ctx->cipher_iv.offset); + rte_hexdump(stdout, "cipher iv:", cipher_iv_ptr, + ctx->cipher_iv.length); + } + + if (do_auth) { + if (ctx->auth_iv.length) { + uint8_t *auth_iv_ptr = rte_crypto_op_ctod_offset(op, + uint8_t *, + ctx->auth_iv.offset); + rte_hexdump(stdout, "auth iv:", auth_iv_ptr, + ctx->auth_iv.length); + } + rte_hexdump(stdout, "digest:", op->sym->auth.digest.data, + ctx->digest_length); + } + + if (do_aead) { + rte_hexdump(stdout, "digest:", op->sym->aead.digest.data, + ctx->digest_length); + rte_hexdump(stdout, "aad:", op->sym->aead.aad.data, + ctx->aad_len); + } #endif return 0; } @@ -837,21 +1638,11 @@ static inline uint32_t adf_modulo(uint32_t data, uint32_t shift) return data - mult; } -void qat_crypto_sym_session_init(struct rte_mempool *mp, void *sym_sess) -{ - struct rte_cryptodev_sym_session *sess = sym_sess; - struct qat_session *s = (void *)sess->_private; - - PMD_INIT_FUNC_TRACE(); - s->cd_paddr = rte_mempool_virt2phy(mp, sess) + - offsetof(struct qat_session, cd) + - offsetof(struct rte_cryptodev_sym_session, _private); -} - -int qat_dev_config(__rte_unused struct rte_cryptodev *dev) +int qat_dev_config(__rte_unused struct rte_cryptodev *dev, + __rte_unused struct rte_cryptodev_config *config) { PMD_INIT_FUNC_TRACE(); - return -ENOTSUP; + return 0; } int qat_dev_start(__rte_unused struct rte_cryptodev *dev) @@ -880,8 +1671,8 @@ int qat_dev_close(struct rte_cryptodev *dev) return 0; } -void qat_dev_info_get(__rte_unused struct rte_cryptodev *dev, - struct rte_cryptodev_info *info) +void qat_dev_info_get(struct rte_cryptodev *dev, + struct rte_cryptodev_info *info) { struct qat_pmd_private *internals = dev->data->dev_private; @@ -891,9 +1682,10 @@ void qat_dev_info_get(__rte_unused struct rte_cryptodev *dev, ADF_NUM_SYM_QPS_PER_BUNDLE * ADF_NUM_BUNDLES_PER_DEV; info->feature_flags = dev->feature_flags; - info->capabilities = qat_pmd_capabilities; + info->capabilities = internals->qat_dev_capabilities; info->sym.max_nb_sessions = internals->max_nb_sessions; - info->dev_type = RTE_CRYPTODEV_QAT_SYM_PMD; + info->driver_id = cryptodev_qat_driver_id; + info->pci_dev = RTE_DEV_TO_PCI(dev->device); } } @@ -915,9 +1707,9 @@ void qat_crypto_sym_stats_get(struct rte_cryptodev *dev, } stats->enqueued_count += qp[i]->stats.enqueued_count; - stats->dequeued_count += qp[i]->stats.enqueued_count; + stats->dequeued_count += qp[i]->stats.dequeued_count; stats->enqueue_err_count += qp[i]->stats.enqueue_err_count; - stats->dequeue_err_count += qp[i]->stats.enqueue_err_count; + stats->dequeue_err_count += qp[i]->stats.dequeue_err_count; } } diff --git a/dpdk/drivers/crypto/qat/qat_crypto.h b/dpdk/drivers/crypto/qat/qat_crypto.h index 0afe74ee..c64d7756 100644 --- a/dpdk/drivers/crypto/qat/qat_crypto.h +++ b/dpdk/drivers/crypto/qat/qat_crypto.h @@ -37,12 +37,32 @@ #include #include +#include "qat_crypto_capabilities.h" + +#define CRYPTODEV_NAME_QAT_SYM_PMD crypto_qat +/**< Intel QAT Symmetric Crypto PMD device name */ + /* * This macro rounds up a number to a be a multiple of * the alignment when the alignment is a power of 2 */ #define ALIGN_POW2_ROUNDUP(num, align) \ (((num) + (align) - 1) & ~((align) - 1)) +#define QAT_64_BTYE_ALIGN_MASK (~0x3f) + +#define QAT_CSR_HEAD_WRITE_THRESH 32U +/* number of requests to accumulate before writing head CSR */ +#define QAT_CSR_TAIL_WRITE_THRESH 32U +/* number of requests to accumulate before writing tail CSR */ +#define QAT_CSR_TAIL_FORCE_WRITE_THRESH 256U +/* number of inflights below which no tail write coalescing should occur */ + +struct qat_session; + +enum qat_device_gen { + QAT_GEN1 = 1, + QAT_GEN2, +}; /** * Structure associated with each queue. @@ -50,7 +70,7 @@ struct qat_queue { char memz_name[RTE_MEMZONE_NAMESIZE]; void *base_addr; /* Base address */ - phys_addr_t base_phys_addr; /* Queue physical address */ + rte_iova_t base_phys_addr; /* Queue physical address */ uint32_t head; /* Shadow copy of the head */ uint32_t tail; /* Shadow copy of the tail */ uint32_t modulo; @@ -60,28 +80,41 @@ struct qat_queue { uint8_t hw_bundle_number; uint8_t hw_queue_number; /* HW queue aka ring offset on bundle */ + uint32_t csr_head; /* last written head value */ + uint32_t csr_tail; /* last written tail value */ + uint16_t nb_processed_responses; + /* number of responses processed since last CSR head write */ + uint16_t nb_pending_requests; + /* number of requests pending since last CSR tail write */ }; struct qat_qp { void *mmap_bar_addr; - rte_atomic16_t inflights16; + uint16_t inflights16; struct qat_queue tx_q; struct qat_queue rx_q; struct rte_cryptodev_stats stats; + struct rte_mempool *op_cookie_pool; + void **op_cookies; + uint32_t nb_descriptors; + enum qat_device_gen qat_dev_gen; } __rte_cache_aligned; /** private data structure for each QAT device */ struct qat_pmd_private { - char sess_mp_name[RTE_MEMPOOL_NAMESIZE]; - struct rte_mempool *sess_mp; - unsigned max_nb_queue_pairs; /**< Max number of queue pairs supported by device */ unsigned max_nb_sessions; /**< Max number of sessions supported by device */ + enum qat_device_gen qat_dev_gen; + /**< QAT device generation */ + const struct rte_cryptodev_capabilities *qat_dev_capabilities; }; -int qat_dev_config(struct rte_cryptodev *dev); +extern uint8_t cryptodev_qat_driver_id; + +int qat_dev_config(struct rte_cryptodev *dev, + struct rte_cryptodev_config *config); int qat_dev_start(struct rte_cryptodev *dev); void qat_dev_stop(struct rte_cryptodev *dev); int qat_dev_close(struct rte_cryptodev *dev); @@ -93,7 +126,8 @@ void qat_crypto_sym_stats_get(struct rte_cryptodev *dev, void qat_crypto_sym_stats_reset(struct rte_cryptodev *dev); int qat_crypto_sym_qp_setup(struct rte_cryptodev *dev, uint16_t queue_pair_id, - const struct rte_cryptodev_qp_conf *rx_conf, int socket_id); + const struct rte_cryptodev_qp_conf *rx_conf, int socket_id, + struct rte_mempool *session_pool); int qat_crypto_sym_qp_release(struct rte_cryptodev *dev, uint16_t queue_pair_id); @@ -104,26 +138,35 @@ qat_pmd_session_mempool_create(struct rte_cryptodev *dev, extern unsigned qat_crypto_sym_get_session_private_size(struct rte_cryptodev *dev); -extern void -qat_crypto_sym_session_init(struct rte_mempool *mempool, void *priv_sess); - -extern void * +extern int qat_crypto_sym_configure_session(struct rte_cryptodev *dev, + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool); + + +int +qat_crypto_set_session_parameters(struct rte_cryptodev *dev, struct rte_crypto_sym_xform *xform, void *session_private); -struct qat_session * +int +qat_crypto_sym_configure_session_aead(struct rte_crypto_sym_xform *xform, + struct qat_session *session); + +int qat_crypto_sym_configure_session_auth(struct rte_cryptodev *dev, struct rte_crypto_sym_xform *xform, - struct qat_session *session_private); + struct qat_session *session); -void * +int qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev, - struct rte_crypto_sym_xform *xform, void *session_private); + struct rte_crypto_sym_xform *xform, + struct qat_session *session); extern void -qat_crypto_sym_clear_session(struct rte_cryptodev *dev, void *session); - +qat_crypto_sym_clear_session(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *session); extern uint16_t qat_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops, diff --git a/dpdk/drivers/crypto/qat/qat_crypto_capabilities.h b/dpdk/drivers/crypto/qat/qat_crypto_capabilities.h new file mode 100644 index 00000000..89ba27d4 --- /dev/null +++ b/dpdk/drivers/crypto/qat/qat_crypto_capabilities.h @@ -0,0 +1,586 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _QAT_CRYPTO_CAPABILITIES_H_ +#define _QAT_CRYPTO_CAPABILITIES_H_ + +#define QAT_BASE_GEN1_SYM_CAPABILITIES \ + { /* SHA1 HMAC */ \ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, \ + {.sym = { \ + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, \ + {.auth = { \ + .algo = RTE_CRYPTO_AUTH_SHA1_HMAC, \ + .block_size = 64, \ + .key_size = { \ + .min = 1, \ + .max = 64, \ + .increment = 1 \ + }, \ + .digest_size = { \ + .min = 1, \ + .max = 20, \ + .increment = 1 \ + }, \ + .iv_size = { 0 } \ + }, } \ + }, } \ + }, \ + { /* SHA224 HMAC */ \ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, \ + {.sym = { \ + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, \ + {.auth = { \ + .algo = RTE_CRYPTO_AUTH_SHA224_HMAC, \ + .block_size = 64, \ + .key_size = { \ + .min = 1, \ + .max = 64, \ + .increment = 1 \ + }, \ + .digest_size = { \ + .min = 1, \ + .max = 28, \ + .increment = 1 \ + }, \ + .iv_size = { 0 } \ + }, } \ + }, } \ + }, \ + { /* SHA256 HMAC */ \ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, \ + {.sym = { \ + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, \ + {.auth = { \ + .algo = RTE_CRYPTO_AUTH_SHA256_HMAC, \ + .block_size = 64, \ + .key_size = { \ + .min = 1, \ + .max = 64, \ + .increment = 1 \ + }, \ + .digest_size = { \ + .min = 1, \ + .max = 32, \ + .increment = 1 \ + }, \ + .iv_size = { 0 } \ + }, } \ + }, } \ + }, \ + { /* SHA384 HMAC */ \ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, \ + {.sym = { \ + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, \ + {.auth = { \ + .algo = RTE_CRYPTO_AUTH_SHA384_HMAC, \ + .block_size = 128, \ + .key_size = { \ + .min = 1, \ + .max = 128, \ + .increment = 1 \ + }, \ + .digest_size = { \ + .min = 1, \ + .max = 48, \ + .increment = 1 \ + }, \ + .iv_size = { 0 } \ + }, } \ + }, } \ + }, \ + { /* SHA512 HMAC */ \ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, \ + {.sym = { \ + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, \ + {.auth = { \ + .algo = RTE_CRYPTO_AUTH_SHA512_HMAC, \ + .block_size = 128, \ + .key_size = { \ + .min = 1, \ + .max = 128, \ + .increment = 1 \ + }, \ + .digest_size = { \ + .min = 1, \ + .max = 64, \ + .increment = 1 \ + }, \ + .iv_size = { 0 } \ + }, } \ + }, } \ + }, \ + { /* MD5 HMAC */ \ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, \ + {.sym = { \ + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, \ + {.auth = { \ + .algo = RTE_CRYPTO_AUTH_MD5_HMAC, \ + .block_size = 64, \ + .key_size = { \ + .min = 1, \ + .max = 64, \ + .increment = 1 \ + }, \ + .digest_size = { \ + .min = 1, \ + .max = 16, \ + .increment = 1 \ + }, \ + .iv_size = { 0 } \ + }, } \ + }, } \ + }, \ + { /* AES XCBC MAC */ \ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, \ + {.sym = { \ + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, \ + {.auth = { \ + .algo = RTE_CRYPTO_AUTH_AES_XCBC_MAC, \ + .block_size = 16, \ + .key_size = { \ + .min = 16, \ + .max = 16, \ + .increment = 0 \ + }, \ + .digest_size = { \ + .min = 16, \ + .max = 16, \ + .increment = 0 \ + }, \ + .aad_size = { 0 }, \ + .iv_size = { 0 } \ + }, } \ + }, } \ + }, \ + { /* AES CCM */ \ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, \ + {.sym = { \ + .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD, \ + {.aead = { \ + .algo = RTE_CRYPTO_AEAD_AES_CCM, \ + .block_size = 16, \ + .key_size = { \ + .min = 16, \ + .max = 16, \ + .increment = 0 \ + }, \ + .digest_size = { \ + .min = 4, \ + .max = 16, \ + .increment = 2 \ + }, \ + .aad_size = { \ + .min = 0, \ + .max = 224, \ + .increment = 1 \ + }, \ + .iv_size = { \ + .min = 7, \ + .max = 13, \ + .increment = 1 \ + }, \ + }, } \ + }, } \ + }, \ + { /* AES GCM */ \ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, \ + {.sym = { \ + .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD, \ + {.aead = { \ + .algo = RTE_CRYPTO_AEAD_AES_GCM, \ + .block_size = 16, \ + .key_size = { \ + .min = 16, \ + .max = 32, \ + .increment = 8 \ + }, \ + .digest_size = { \ + .min = 8, \ + .max = 16, \ + .increment = 4 \ + }, \ + .aad_size = { \ + .min = 0, \ + .max = 240, \ + .increment = 1 \ + }, \ + .iv_size = { \ + .min = 12, \ + .max = 12, \ + .increment = 0 \ + }, \ + }, } \ + }, } \ + }, \ + { /* AES GMAC (AUTH) */ \ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, \ + {.sym = { \ + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, \ + {.auth = { \ + .algo = RTE_CRYPTO_AUTH_AES_GMAC, \ + .block_size = 16, \ + .key_size = { \ + .min = 16, \ + .max = 32, \ + .increment = 8 \ + }, \ + .digest_size = { \ + .min = 8, \ + .max = 16, \ + .increment = 4 \ + }, \ + .iv_size = { \ + .min = 12, \ + .max = 12, \ + .increment = 0 \ + } \ + }, } \ + }, } \ + }, \ + { /* SNOW 3G (UIA2) */ \ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, \ + {.sym = { \ + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, \ + {.auth = { \ + .algo = RTE_CRYPTO_AUTH_SNOW3G_UIA2, \ + .block_size = 16, \ + .key_size = { \ + .min = 16, \ + .max = 16, \ + .increment = 0 \ + }, \ + .digest_size = { \ + .min = 4, \ + .max = 4, \ + .increment = 0 \ + }, \ + .iv_size = { \ + .min = 16, \ + .max = 16, \ + .increment = 0 \ + } \ + }, } \ + }, } \ + }, \ + { /* AES CBC */ \ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, \ + {.sym = { \ + .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, \ + {.cipher = { \ + .algo = RTE_CRYPTO_CIPHER_AES_CBC, \ + .block_size = 16, \ + .key_size = { \ + .min = 16, \ + .max = 32, \ + .increment = 8 \ + }, \ + .iv_size = { \ + .min = 16, \ + .max = 16, \ + .increment = 0 \ + } \ + }, } \ + }, } \ + }, \ + { /* AES DOCSIS BPI */ \ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, \ + {.sym = { \ + .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, \ + {.cipher = { \ + .algo = RTE_CRYPTO_CIPHER_AES_DOCSISBPI,\ + .block_size = 16, \ + .key_size = { \ + .min = 16, \ + .max = 16, \ + .increment = 0 \ + }, \ + .iv_size = { \ + .min = 16, \ + .max = 16, \ + .increment = 0 \ + } \ + }, } \ + }, } \ + }, \ + { /* SNOW 3G (UEA2) */ \ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, \ + {.sym = { \ + .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, \ + {.cipher = { \ + .algo = RTE_CRYPTO_CIPHER_SNOW3G_UEA2, \ + .block_size = 16, \ + .key_size = { \ + .min = 16, \ + .max = 16, \ + .increment = 0 \ + }, \ + .iv_size = { \ + .min = 16, \ + .max = 16, \ + .increment = 0 \ + } \ + }, } \ + }, } \ + }, \ + { /* AES CTR */ \ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, \ + {.sym = { \ + .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, \ + {.cipher = { \ + .algo = RTE_CRYPTO_CIPHER_AES_CTR, \ + .block_size = 16, \ + .key_size = { \ + .min = 16, \ + .max = 32, \ + .increment = 8 \ + }, \ + .iv_size = { \ + .min = 16, \ + .max = 16, \ + .increment = 0 \ + } \ + }, } \ + }, } \ + }, \ + { /* NULL (AUTH) */ \ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, \ + {.sym = { \ + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, \ + {.auth = { \ + .algo = RTE_CRYPTO_AUTH_NULL, \ + .block_size = 1, \ + .key_size = { \ + .min = 0, \ + .max = 0, \ + .increment = 0 \ + }, \ + .digest_size = { \ + .min = 0, \ + .max = 0, \ + .increment = 0 \ + }, \ + .iv_size = { 0 } \ + }, }, \ + }, }, \ + }, \ + { /* NULL (CIPHER) */ \ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, \ + {.sym = { \ + .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, \ + {.cipher = { \ + .algo = RTE_CRYPTO_CIPHER_NULL, \ + .block_size = 1, \ + .key_size = { \ + .min = 0, \ + .max = 0, \ + .increment = 0 \ + }, \ + .iv_size = { \ + .min = 0, \ + .max = 0, \ + .increment = 0 \ + } \ + }, }, \ + }, } \ + }, \ + { /* KASUMI (F8) */ \ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, \ + {.sym = { \ + .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, \ + {.cipher = { \ + .algo = RTE_CRYPTO_CIPHER_KASUMI_F8, \ + .block_size = 8, \ + .key_size = { \ + .min = 16, \ + .max = 16, \ + .increment = 0 \ + }, \ + .iv_size = { \ + .min = 8, \ + .max = 8, \ + .increment = 0 \ + } \ + }, } \ + }, } \ + }, \ + { /* KASUMI (F9) */ \ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, \ + {.sym = { \ + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, \ + {.auth = { \ + .algo = RTE_CRYPTO_AUTH_KASUMI_F9, \ + .block_size = 8, \ + .key_size = { \ + .min = 16, \ + .max = 16, \ + .increment = 0 \ + }, \ + .digest_size = { \ + .min = 4, \ + .max = 4, \ + .increment = 0 \ + }, \ + .iv_size = { 0 } \ + }, } \ + }, } \ + }, \ + { /* 3DES CBC */ \ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, \ + {.sym = { \ + .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, \ + {.cipher = { \ + .algo = RTE_CRYPTO_CIPHER_3DES_CBC, \ + .block_size = 8, \ + .key_size = { \ + .min = 16, \ + .max = 24, \ + .increment = 8 \ + }, \ + .iv_size = { \ + .min = 8, \ + .max = 8, \ + .increment = 0 \ + } \ + }, } \ + }, } \ + }, \ + { /* 3DES CTR */ \ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, \ + {.sym = { \ + .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, \ + {.cipher = { \ + .algo = RTE_CRYPTO_CIPHER_3DES_CTR, \ + .block_size = 8, \ + .key_size = { \ + .min = 16, \ + .max = 24, \ + .increment = 8 \ + }, \ + .iv_size = { \ + .min = 8, \ + .max = 8, \ + .increment = 0 \ + } \ + }, } \ + }, } \ + }, \ + { /* DES CBC */ \ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, \ + {.sym = { \ + .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, \ + {.cipher = { \ + .algo = RTE_CRYPTO_CIPHER_DES_CBC, \ + .block_size = 8, \ + .key_size = { \ + .min = 8, \ + .max = 8, \ + .increment = 0 \ + }, \ + .iv_size = { \ + .min = 8, \ + .max = 8, \ + .increment = 0 \ + } \ + }, } \ + }, } \ + }, \ + { /* DES DOCSISBPI */ \ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, \ + {.sym = { \ + .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, \ + {.cipher = { \ + .algo = RTE_CRYPTO_CIPHER_DES_DOCSISBPI,\ + .block_size = 8, \ + .key_size = { \ + .min = 8, \ + .max = 8, \ + .increment = 0 \ + }, \ + .iv_size = { \ + .min = 8, \ + .max = 8, \ + .increment = 0 \ + } \ + }, } \ + }, } \ + } + +#define QAT_EXTRA_GEN2_SYM_CAPABILITIES \ + { /* ZUC (EEA3) */ \ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, \ + {.sym = { \ + .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, \ + {.cipher = { \ + .algo = RTE_CRYPTO_CIPHER_ZUC_EEA3, \ + .block_size = 16, \ + .key_size = { \ + .min = 16, \ + .max = 16, \ + .increment = 0 \ + }, \ + .iv_size = { \ + .min = 16, \ + .max = 16, \ + .increment = 0 \ + } \ + }, } \ + }, } \ + }, \ + { /* ZUC (EIA3) */ \ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, \ + {.sym = { \ + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, \ + {.auth = { \ + .algo = RTE_CRYPTO_AUTH_ZUC_EIA3, \ + .block_size = 16, \ + .key_size = { \ + .min = 16, \ + .max = 16, \ + .increment = 0 \ + }, \ + .digest_size = { \ + .min = 4, \ + .max = 4, \ + .increment = 0 \ + }, \ + .iv_size = { \ + .min = 16, \ + .max = 16, \ + .increment = 0 \ + } \ + }, } \ + }, } \ + } + +#endif /* _QAT_CRYPTO_CAPABILITIES_H_ */ diff --git a/dpdk/drivers/crypto/qat/qat_qp.c b/dpdk/drivers/crypto/qat/qat_qp.c index 5de47e36..c02971ee 100644 --- a/dpdk/drivers/crypto/qat/qat_qp.c +++ b/dpdk/drivers/crypto/qat/qat_qp.c @@ -36,11 +36,14 @@ #include #include #include +#include +#include #include #include #include "qat_logs.h" #include "qat_crypto.h" +#include "qat_algs.h" #include "adf_transport_access_macros.h" #define ADF_MAX_SYM_DESC 4096 @@ -120,22 +123,20 @@ queue_dma_zone_reserve(const char *queue_name, uint32_t queue_size, break; default: memzone_flags = RTE_MEMZONE_SIZE_HINT_ONLY; -} -#ifdef RTE_LIBRTE_XEN_DOM0 - return rte_memzone_reserve_bounded(queue_name, queue_size, - socket_id, 0, RTE_CACHE_LINE_SIZE, RTE_PGSIZE_2M); -#else + } return rte_memzone_reserve_aligned(queue_name, queue_size, socket_id, memzone_flags, queue_size); -#endif } int qat_crypto_sym_qp_setup(struct rte_cryptodev *dev, uint16_t queue_pair_id, const struct rte_cryptodev_qp_conf *qp_conf, - int socket_id) + int socket_id, struct rte_mempool *session_pool __rte_unused) { struct qat_qp *qp; + struct rte_pci_device *pci_dev; int ret; + char op_cookie_pool_name[RTE_RING_NAMESIZE]; + uint32_t i; PMD_INIT_FUNC_TRACE(); @@ -153,7 +154,9 @@ int qat_crypto_sym_qp_setup(struct rte_cryptodev *dev, uint16_t queue_pair_id, return -EINVAL; } - if (dev->pci_dev->mem_resource[0].addr == NULL) { + pci_dev = RTE_DEV_TO_PCI(dev->device); + + if (pci_dev->mem_resource[0].addr == NULL) { PMD_DRV_LOG(ERR, "Could not find VF config space " "(UIO driver attached?)."); return -EINVAL; @@ -166,7 +169,6 @@ int qat_crypto_sym_qp_setup(struct rte_cryptodev *dev, uint16_t queue_pair_id, queue_pair_id); return -EINVAL; } - /* Allocate the queue pair data structure. */ qp = rte_zmalloc("qat PMD qp metadata", sizeof(*qp), RTE_CACHE_LINE_SIZE); @@ -174,8 +176,18 @@ int qat_crypto_sym_qp_setup(struct rte_cryptodev *dev, uint16_t queue_pair_id, PMD_DRV_LOG(ERR, "Failed to alloc mem for qp struct"); return -ENOMEM; } - qp->mmap_bar_addr = dev->pci_dev->mem_resource[0].addr; - rte_atomic16_init(&qp->inflights16); + qp->nb_descriptors = qp_conf->nb_descriptors; + qp->op_cookies = rte_zmalloc("qat PMD op cookie pointer", + qp_conf->nb_descriptors * sizeof(*qp->op_cookies), + RTE_CACHE_LINE_SIZE); + if (qp->op_cookies == NULL) { + PMD_DRV_LOG(ERR, "Failed to alloc mem for cookie"); + rte_free(qp); + return -ENOMEM; + } + + qp->mmap_bar_addr = pci_dev->mem_resource[0].addr; + qp->inflights16 = 0; if (qat_tx_queue_create(dev, &(qp->tx_q), queue_pair_id, qp_conf->nb_descriptors, socket_id) != 0) { @@ -191,12 +203,57 @@ int qat_crypto_sym_qp_setup(struct rte_cryptodev *dev, uint16_t queue_pair_id, qat_queue_delete(&(qp->tx_q)); goto create_err; } + adf_configure_queues(qp); adf_queue_arb_enable(&qp->tx_q, qp->mmap_bar_addr); + snprintf(op_cookie_pool_name, RTE_RING_NAMESIZE, "%s_qp_op_%d_%hu", + pci_dev->driver->driver.name, dev->data->dev_id, + queue_pair_id); + + qp->op_cookie_pool = rte_mempool_lookup(op_cookie_pool_name); + if (qp->op_cookie_pool == NULL) + qp->op_cookie_pool = rte_mempool_create(op_cookie_pool_name, + qp->nb_descriptors, + sizeof(struct qat_crypto_op_cookie), 64, 0, + NULL, NULL, NULL, NULL, socket_id, + 0); + if (!qp->op_cookie_pool) { + PMD_DRV_LOG(ERR, "QAT PMD Cannot create" + " op mempool"); + goto create_err; + } + + for (i = 0; i < qp->nb_descriptors; i++) { + if (rte_mempool_get(qp->op_cookie_pool, &qp->op_cookies[i])) { + PMD_DRV_LOG(ERR, "QAT PMD Cannot get op_cookie"); + goto create_err; + } + + struct qat_crypto_op_cookie *sql_cookie = + qp->op_cookies[i]; + + sql_cookie->qat_sgl_src_phys_addr = + rte_mempool_virt2iova(sql_cookie) + + offsetof(struct qat_crypto_op_cookie, + qat_sgl_list_src); + + sql_cookie->qat_sgl_dst_phys_addr = + rte_mempool_virt2iova(sql_cookie) + + offsetof(struct qat_crypto_op_cookie, + qat_sgl_list_dst); + } + + struct qat_pmd_private *internals + = dev->data->dev_private; + qp->qat_dev_gen = internals->qat_dev_gen; + dev->data->queue_pairs[queue_pair_id] = qp; return 0; create_err: + if (qp->op_cookie_pool) + rte_mempool_free(qp->op_cookie_pool); + rte_free(qp->op_cookies); rte_free(qp); return -EFAULT; } @@ -205,6 +262,7 @@ int qat_crypto_sym_qp_release(struct rte_cryptodev *dev, uint16_t queue_pair_id) { struct qat_qp *qp = (struct qat_qp *)dev->data->queue_pairs[queue_pair_id]; + uint32_t i; PMD_INIT_FUNC_TRACE(); if (qp == NULL) { @@ -213,7 +271,7 @@ int qat_crypto_sym_qp_release(struct rte_cryptodev *dev, uint16_t queue_pair_id) } /* Don't free memory if there are still responses to be processed */ - if (rte_atomic16_read(&(qp->inflights16)) == 0) { + if (qp->inflights16 == 0) { qat_queue_delete(&(qp->tx_q)); qat_queue_delete(&(qp->rx_q)); } else { @@ -221,6 +279,14 @@ int qat_crypto_sym_qp_release(struct rte_cryptodev *dev, uint16_t queue_pair_id) } adf_queue_arb_disable(&(qp->tx_q), qp->mmap_bar_addr); + + for (i = 0; i < qp->nb_descriptors; i++) + rte_mempool_put(qp->op_cookie_pool, qp->op_cookies[i]); + + if (qp->op_cookie_pool) + rte_mempool_free(qp->op_cookie_pool); + + rte_free(qp->op_cookies); rte_free(qp); dev->data->queue_pairs[queue_pair_id] = NULL; return 0; @@ -289,6 +355,7 @@ qat_queue_create(struct rte_cryptodev *dev, struct qat_queue *queue, void *io_addr; const struct rte_memzone *qp_mz; uint32_t queue_size_bytes = nb_desc*desc_size; + struct rte_pci_device *pci_dev; PMD_INIT_FUNC_TRACE(); if (desc_size > ADF_MSG_SIZE_TO_BYTES(ADF_MAX_MSG_SIZE)) { @@ -296,11 +363,13 @@ qat_queue_create(struct rte_cryptodev *dev, struct qat_queue *queue, return -EINVAL; } + pci_dev = RTE_DEV_TO_PCI(dev->device); + /* * Allocate a memzone for the queue - create a unique name. */ snprintf(queue->memz_name, sizeof(queue->memz_name), "%s_%s_%d_%d_%d", - dev->driver->pci_drv.name, "qp_mem", dev->data->dev_id, + pci_dev->driver->driver.name, "qp_mem", dev->data->dev_id, queue->hw_bundle_number, queue->hw_queue_number); qp_mz = queue_dma_zone_reserve(queue->memz_name, queue_size_bytes, socket_id); @@ -310,7 +379,7 @@ qat_queue_create(struct rte_cryptodev *dev, struct qat_queue *queue, } queue->base_addr = (char *)qp_mz->addr; - queue->base_phys_addr = qp_mz->phys_addr; + queue->base_phys_addr = qp_mz->iova; if (qat_qp_check_queue_alignment(queue->base_phys_addr, queue_size_bytes)) { PMD_DRV_LOG(ERR, "Invalid alignment on queue create " @@ -349,7 +418,8 @@ qat_queue_create(struct rte_cryptodev *dev, struct qat_queue *queue, queue_base = BUILD_RING_BASE_ADDR(queue->base_phys_addr, queue->queue_size); - io_addr = dev->pci_dev->mem_resource[0].addr; + + io_addr = pci_dev->mem_resource[0].addr; WRITE_CSR_RING_BASE(io_addr, queue->hw_bundle_number, queue->hw_queue_number, queue_base); diff --git a/dpdk/drivers/crypto/qat/rte_qat_cryptodev.c b/dpdk/drivers/crypto/qat/rte_qat_cryptodev.c index 82ab047f..4f8e4bfe 100644 --- a/dpdk/drivers/crypto/qat/rte_qat_cryptodev.c +++ b/dpdk/drivers/crypto/qat/rte_qat_cryptodev.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2015-2016 Intel Corporation. All rights reserved. + * Copyright(c) 2015-2017 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -31,14 +31,29 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +#include #include #include #include +#include #include #include "qat_crypto.h" #include "qat_logs.h" +uint8_t cryptodev_qat_driver_id; + +static const struct rte_cryptodev_capabilities qat_gen1_capabilities[] = { + QAT_BASE_GEN1_SYM_CAPABILITIES, + RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST() +}; + +static const struct rte_cryptodev_capabilities qat_gen2_capabilities[] = { + QAT_BASE_GEN1_SYM_CAPABILITIES, + QAT_EXTRA_GEN2_SYM_CAPABILITIES, + RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST() +}; + static struct rte_cryptodev_ops crypto_qat_ops = { /* Device related operations */ @@ -59,7 +74,6 @@ static struct rte_cryptodev_ops crypto_qat_ops = { /* Crypto related operations */ .session_get_size = qat_crypto_sym_get_session_private_size, .session_configure = qat_crypto_sym_configure_session, - .session_initialize = qat_crypto_sym_session_init, .session_clear = qat_crypto_sym_clear_session }; @@ -67,26 +81,37 @@ static struct rte_cryptodev_ops crypto_qat_ops = { * The set of PCI devices this driver supports */ -static struct rte_pci_id pci_id_qat_map[] = { +static const struct rte_pci_id pci_id_qat_map[] = { { RTE_PCI_DEVICE(0x8086, 0x0443), }, + { + RTE_PCI_DEVICE(0x8086, 0x37c9), + }, + { + RTE_PCI_DEVICE(0x8086, 0x19e3), + }, + { + RTE_PCI_DEVICE(0x8086, 0x6f55), + }, {.device_id = 0}, }; static int -crypto_qat_dev_init(__attribute__((unused)) struct rte_cryptodev_driver *crypto_drv, - struct rte_cryptodev *cryptodev) +crypto_qat_create(const char *name, struct rte_pci_device *pci_dev, + struct rte_cryptodev_pmd_init_params *init_params) { + struct rte_cryptodev *cryptodev; struct qat_pmd_private *internals; PMD_INIT_FUNC_TRACE(); - PMD_DRV_LOG(DEBUG, "Found crypto device at %02x:%02x.%x", - cryptodev->pci_dev->addr.bus, - cryptodev->pci_dev->addr.devid, - cryptodev->pci_dev->addr.function); - cryptodev->dev_type = RTE_CRYPTODEV_QAT_SYM_PMD; + cryptodev = rte_cryptodev_pmd_create(name, &pci_dev->device, + init_params); + if (cryptodev == NULL) + return -ENODEV; + + cryptodev->driver_id = cryptodev_qat_driver_id; cryptodev->dev_ops = &crypto_qat_ops; cryptodev->enqueue_burst = qat_pmd_enqueue_op_burst; @@ -94,10 +119,27 @@ crypto_qat_dev_init(__attribute__((unused)) struct rte_cryptodev_driver *crypto_ cryptodev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | RTE_CRYPTODEV_FF_HW_ACCELERATED | - RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING; + RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | + RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER; internals = cryptodev->data->dev_private; - internals->max_nb_sessions = RTE_QAT_PMD_MAX_NB_SESSIONS; + internals->max_nb_sessions = init_params->max_nb_sessions; + switch (pci_dev->id.device_id) { + case 0x0443: + internals->qat_dev_gen = QAT_GEN1; + internals->qat_dev_capabilities = qat_gen1_capabilities; + break; + case 0x37c9: + case 0x19e3: + case 0x6f55: + internals->qat_dev_gen = QAT_GEN2; + internals->qat_dev_capabilities = qat_gen2_capabilities; + break; + default: + PMD_DRV_LOG(ERR, + "Invalid dev_id, can't determine capabilities"); + break; + } /* * For secondary processes, we don't initialise any further as primary @@ -112,27 +154,56 @@ crypto_qat_dev_init(__attribute__((unused)) struct rte_cryptodev_driver *crypto_ return 0; } -static struct rte_cryptodev_driver rte_qat_pmd = { - { - .id_table = pci_id_qat_map, - .drv_flags = RTE_PCI_DRV_NEED_MAPPING, - }, - .cryptodev_init = crypto_qat_dev_init, - .dev_private_size = sizeof(struct qat_pmd_private), -}; - -static int -rte_qat_pmd_init(const char *name __rte_unused, const char *params __rte_unused) +static int crypto_qat_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, + struct rte_pci_device *pci_dev) { - PMD_INIT_FUNC_TRACE(); - return rte_cryptodev_pmd_driver_register(&rte_qat_pmd, PMD_PDEV); + struct rte_cryptodev_pmd_init_params init_params = { + .name = "", + .socket_id = rte_socket_id(), + .private_data_size = sizeof(struct qat_pmd_private), + .max_nb_sessions = RTE_QAT_PMD_MAX_NB_SESSIONS + }; + char name[RTE_CRYPTODEV_NAME_MAX_LEN]; + + PMD_DRV_LOG(DEBUG, "Found QAT device at %02x:%02x.%x", + pci_dev->addr.bus, + pci_dev->addr.devid, + pci_dev->addr.function); + + rte_pci_device_name(&pci_dev->addr, name, sizeof(name)); + + return crypto_qat_create(name, pci_dev, &init_params); } -static struct rte_driver pmd_qat_drv = { - .type = PMD_PDEV, - .init = rte_qat_pmd_init, +static int crypto_qat_pci_remove(struct rte_pci_device *pci_dev) +{ + struct rte_cryptodev *cryptodev; + char cryptodev_name[RTE_CRYPTODEV_NAME_MAX_LEN]; + + if (pci_dev == NULL) + return -EINVAL; + + rte_pci_device_name(&pci_dev->addr, cryptodev_name, + sizeof(cryptodev_name)); + + cryptodev = rte_cryptodev_pmd_get_named_dev(cryptodev_name); + if (cryptodev == NULL) + return -ENODEV; + + /* free crypto device */ + return rte_cryptodev_pmd_destroy(cryptodev); +} + +static struct rte_pci_driver rte_qat_pmd = { + .id_table = pci_id_qat_map, + .drv_flags = RTE_PCI_DRV_NEED_MAPPING, + .probe = crypto_qat_pci_probe, + .remove = crypto_qat_pci_remove }; -PMD_REGISTER_DRIVER(pmd_qat_drv, CRYPTODEV_NAME_QAT_SYM_PMD); -DRIVER_REGISTER_PCI_TABLE(CRYPTODEV_NAME_QAT_SYM_PMD, pci_id_qat_map); +static struct cryptodev_driver qat_crypto_drv; +RTE_PMD_REGISTER_PCI(CRYPTODEV_NAME_QAT_SYM_PMD, rte_qat_pmd); +RTE_PMD_REGISTER_PCI_TABLE(CRYPTODEV_NAME_QAT_SYM_PMD, pci_id_qat_map); +RTE_PMD_REGISTER_CRYPTO_DRIVER(qat_crypto_drv, rte_qat_pmd, + cryptodev_qat_driver_id); diff --git a/dpdk/drivers/crypto/scheduler/Makefile b/dpdk/drivers/crypto/scheduler/Makefile new file mode 100644 index 00000000..123b0f6d --- /dev/null +++ b/dpdk/drivers/crypto/scheduler/Makefile @@ -0,0 +1,64 @@ +# BSD LICENSE +# +# Copyright(c) 2017 Intel Corporation. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +include $(RTE_SDK)/mk/rte.vars.mk + +# library name +LIB = librte_pmd_crypto_scheduler.a + +# build flags +CFLAGS += -O3 +CFLAGS += $(WERROR_FLAGS) +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_cryptodev -lrte_kvargs -lrte_reorder +LDLIBS += -lrte_bus_vdev + +# library version +LIBABIVER := 1 + +# versioning export map +EXPORT_MAP := rte_pmd_crypto_scheduler_version.map + +# +# Export include files +# +SYMLINK-y-include += rte_cryptodev_scheduler_operations.h +SYMLINK-y-include += rte_cryptodev_scheduler.h + +# library source files +SRCS-$(CONFIG_RTE_LIBRTE_PMD_CRYPTO_SCHEDULER) += scheduler_pmd.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_CRYPTO_SCHEDULER) += scheduler_pmd_ops.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_CRYPTO_SCHEDULER) += rte_cryptodev_scheduler.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_CRYPTO_SCHEDULER) += scheduler_roundrobin.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_CRYPTO_SCHEDULER) += scheduler_pkt_size_distr.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_CRYPTO_SCHEDULER) += scheduler_failover.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_CRYPTO_SCHEDULER) += scheduler_multicore.c + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/crypto/scheduler/rte_cryptodev_scheduler.c b/dpdk/drivers/crypto/scheduler/rte_cryptodev_scheduler.c new file mode 100644 index 00000000..822ce27c --- /dev/null +++ b/dpdk/drivers/crypto/scheduler/rte_cryptodev_scheduler.c @@ -0,0 +1,601 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include +#include +#include +#include + +#include "rte_cryptodev_scheduler.h" +#include "scheduler_pmd_private.h" + +/** update the scheduler pmd's capability with attaching device's + * capability. + * For each device to be attached, the scheduler's capability should be + * the common capability set of all slaves + **/ +static uint32_t +sync_caps(struct rte_cryptodev_capabilities *caps, + uint32_t nb_caps, + const struct rte_cryptodev_capabilities *slave_caps) +{ + uint32_t sync_nb_caps = nb_caps, nb_slave_caps = 0; + uint32_t i; + + while (slave_caps[nb_slave_caps].op != RTE_CRYPTO_OP_TYPE_UNDEFINED) + nb_slave_caps++; + + if (nb_caps == 0) { + rte_memcpy(caps, slave_caps, sizeof(*caps) * nb_slave_caps); + return nb_slave_caps; + } + + for (i = 0; i < sync_nb_caps; i++) { + struct rte_cryptodev_capabilities *cap = &caps[i]; + uint32_t j; + + for (j = 0; j < nb_slave_caps; j++) { + const struct rte_cryptodev_capabilities *s_cap = + &slave_caps[j]; + + if (s_cap->op != cap->op || s_cap->sym.xform_type != + cap->sym.xform_type) + continue; + + if (s_cap->sym.xform_type == + RTE_CRYPTO_SYM_XFORM_AUTH) { + if (s_cap->sym.auth.algo != + cap->sym.auth.algo) + continue; + + cap->sym.auth.digest_size.min = + s_cap->sym.auth.digest_size.min < + cap->sym.auth.digest_size.min ? + s_cap->sym.auth.digest_size.min : + cap->sym.auth.digest_size.min; + cap->sym.auth.digest_size.max = + s_cap->sym.auth.digest_size.max < + cap->sym.auth.digest_size.max ? + s_cap->sym.auth.digest_size.max : + cap->sym.auth.digest_size.max; + + } + + if (s_cap->sym.xform_type == + RTE_CRYPTO_SYM_XFORM_CIPHER) + if (s_cap->sym.cipher.algo != + cap->sym.cipher.algo) + continue; + + /* no common cap found */ + break; + } + + if (j < nb_slave_caps) + continue; + + /* remove a uncommon cap from the array */ + for (j = i; j < sync_nb_caps - 1; j++) + rte_memcpy(&caps[j], &caps[j+1], sizeof(*cap)); + + memset(&caps[sync_nb_caps - 1], 0, sizeof(*cap)); + sync_nb_caps--; + } + + return sync_nb_caps; +} + +static int +update_scheduler_capability(struct scheduler_ctx *sched_ctx) +{ + struct rte_cryptodev_capabilities tmp_caps[256] = { {0} }; + uint32_t nb_caps = 0, i; + + if (sched_ctx->capabilities) + rte_free(sched_ctx->capabilities); + + for (i = 0; i < sched_ctx->nb_slaves; i++) { + struct rte_cryptodev_info dev_info; + + rte_cryptodev_info_get(sched_ctx->slaves[i].dev_id, &dev_info); + + nb_caps = sync_caps(tmp_caps, nb_caps, dev_info.capabilities); + if (nb_caps == 0) + return -1; + } + + sched_ctx->capabilities = rte_zmalloc_socket(NULL, + sizeof(struct rte_cryptodev_capabilities) * + (nb_caps + 1), 0, SOCKET_ID_ANY); + if (!sched_ctx->capabilities) + return -ENOMEM; + + rte_memcpy(sched_ctx->capabilities, tmp_caps, + sizeof(struct rte_cryptodev_capabilities) * nb_caps); + + return 0; +} + +static void +update_scheduler_feature_flag(struct rte_cryptodev *dev) +{ + struct scheduler_ctx *sched_ctx = dev->data->dev_private; + uint32_t i; + + dev->feature_flags = 0; + + for (i = 0; i < sched_ctx->nb_slaves; i++) { + struct rte_cryptodev_info dev_info; + + rte_cryptodev_info_get(sched_ctx->slaves[i].dev_id, &dev_info); + + dev->feature_flags |= dev_info.feature_flags; + } +} + +static void +update_max_nb_qp(struct scheduler_ctx *sched_ctx) +{ + uint32_t i; + uint32_t max_nb_qp; + + if (!sched_ctx->nb_slaves) + return; + + max_nb_qp = sched_ctx->nb_slaves ? UINT32_MAX : 0; + + for (i = 0; i < sched_ctx->nb_slaves; i++) { + struct rte_cryptodev_info dev_info; + + rte_cryptodev_info_get(sched_ctx->slaves[i].dev_id, &dev_info); + max_nb_qp = dev_info.max_nb_queue_pairs < max_nb_qp ? + dev_info.max_nb_queue_pairs : max_nb_qp; + } + + sched_ctx->max_nb_queue_pairs = max_nb_qp; +} + +/** Attach a device to the scheduler. */ +int +rte_cryptodev_scheduler_slave_attach(uint8_t scheduler_id, uint8_t slave_id) +{ + struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id); + struct scheduler_ctx *sched_ctx; + struct scheduler_slave *slave; + struct rte_cryptodev_info dev_info; + uint32_t i; + + if (!dev) { + CS_LOG_ERR("Operation not supported"); + return -ENOTSUP; + } + + if (dev->driver_id != cryptodev_driver_id) { + CS_LOG_ERR("Operation not supported"); + return -ENOTSUP; + } + + if (dev->data->dev_started) { + CS_LOG_ERR("Illegal operation"); + return -EBUSY; + } + + sched_ctx = dev->data->dev_private; + if (sched_ctx->nb_slaves >= + RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES) { + CS_LOG_ERR("Too many slaves attached"); + return -ENOMEM; + } + + for (i = 0; i < sched_ctx->nb_slaves; i++) + if (sched_ctx->slaves[i].dev_id == slave_id) { + CS_LOG_ERR("Slave already added"); + return -ENOTSUP; + } + + slave = &sched_ctx->slaves[sched_ctx->nb_slaves]; + + rte_cryptodev_info_get(slave_id, &dev_info); + + slave->dev_id = slave_id; + slave->driver_id = dev_info.driver_id; + sched_ctx->nb_slaves++; + + if (update_scheduler_capability(sched_ctx) < 0) { + slave->dev_id = 0; + slave->driver_id = 0; + sched_ctx->nb_slaves--; + + CS_LOG_ERR("capabilities update failed"); + return -ENOTSUP; + } + + update_scheduler_feature_flag(dev); + + update_max_nb_qp(sched_ctx); + + return 0; +} + +int +rte_cryptodev_scheduler_slave_detach(uint8_t scheduler_id, uint8_t slave_id) +{ + struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id); + struct scheduler_ctx *sched_ctx; + uint32_t i, slave_pos; + + if (!dev) { + CS_LOG_ERR("Operation not supported"); + return -ENOTSUP; + } + + if (dev->driver_id != cryptodev_driver_id) { + CS_LOG_ERR("Operation not supported"); + return -ENOTSUP; + } + + if (dev->data->dev_started) { + CS_LOG_ERR("Illegal operation"); + return -EBUSY; + } + + sched_ctx = dev->data->dev_private; + + for (slave_pos = 0; slave_pos < sched_ctx->nb_slaves; slave_pos++) + if (sched_ctx->slaves[slave_pos].dev_id == slave_id) + break; + if (slave_pos == sched_ctx->nb_slaves) { + CS_LOG_ERR("Cannot find slave"); + return -ENOTSUP; + } + + if (sched_ctx->ops.slave_detach(dev, slave_id) < 0) { + CS_LOG_ERR("Failed to detach slave"); + return -ENOTSUP; + } + + for (i = slave_pos; i < sched_ctx->nb_slaves - 1; i++) { + memcpy(&sched_ctx->slaves[i], &sched_ctx->slaves[i+1], + sizeof(struct scheduler_slave)); + } + memset(&sched_ctx->slaves[sched_ctx->nb_slaves - 1], 0, + sizeof(struct scheduler_slave)); + sched_ctx->nb_slaves--; + + if (update_scheduler_capability(sched_ctx) < 0) { + CS_LOG_ERR("capabilities update failed"); + return -ENOTSUP; + } + + update_scheduler_feature_flag(dev); + + update_max_nb_qp(sched_ctx); + + return 0; +} + +int +rte_cryptodev_scheduler_mode_set(uint8_t scheduler_id, + enum rte_cryptodev_scheduler_mode mode) +{ + struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id); + struct scheduler_ctx *sched_ctx; + + if (!dev) { + CS_LOG_ERR("Operation not supported"); + return -ENOTSUP; + } + + if (dev->driver_id != cryptodev_driver_id) { + CS_LOG_ERR("Operation not supported"); + return -ENOTSUP; + } + + if (dev->data->dev_started) { + CS_LOG_ERR("Illegal operation"); + return -EBUSY; + } + + sched_ctx = dev->data->dev_private; + + if (mode == sched_ctx->mode) + return 0; + + switch (mode) { + case CDEV_SCHED_MODE_ROUNDROBIN: + if (rte_cryptodev_scheduler_load_user_scheduler(scheduler_id, + roundrobin_scheduler) < 0) { + CS_LOG_ERR("Failed to load scheduler"); + return -1; + } + break; + case CDEV_SCHED_MODE_PKT_SIZE_DISTR: + if (rte_cryptodev_scheduler_load_user_scheduler(scheduler_id, + pkt_size_based_distr_scheduler) < 0) { + CS_LOG_ERR("Failed to load scheduler"); + return -1; + } + break; + case CDEV_SCHED_MODE_FAILOVER: + if (rte_cryptodev_scheduler_load_user_scheduler(scheduler_id, + failover_scheduler) < 0) { + CS_LOG_ERR("Failed to load scheduler"); + return -1; + } + break; + case CDEV_SCHED_MODE_MULTICORE: + if (rte_cryptodev_scheduler_load_user_scheduler(scheduler_id, + multicore_scheduler) < 0) { + CS_LOG_ERR("Failed to load scheduler"); + return -1; + } + break; + default: + CS_LOG_ERR("Not yet supported"); + return -ENOTSUP; + } + + return 0; +} + +enum rte_cryptodev_scheduler_mode +rte_cryptodev_scheduler_mode_get(uint8_t scheduler_id) +{ + struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id); + struct scheduler_ctx *sched_ctx; + + if (!dev) { + CS_LOG_ERR("Operation not supported"); + return -ENOTSUP; + } + + if (dev->driver_id != cryptodev_driver_id) { + CS_LOG_ERR("Operation not supported"); + return -ENOTSUP; + } + + sched_ctx = dev->data->dev_private; + + return sched_ctx->mode; +} + +int +rte_cryptodev_scheduler_ordering_set(uint8_t scheduler_id, + uint32_t enable_reorder) +{ + struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id); + struct scheduler_ctx *sched_ctx; + + if (!dev) { + CS_LOG_ERR("Operation not supported"); + return -ENOTSUP; + } + + if (dev->driver_id != cryptodev_driver_id) { + CS_LOG_ERR("Operation not supported"); + return -ENOTSUP; + } + + if (dev->data->dev_started) { + CS_LOG_ERR("Illegal operation"); + return -EBUSY; + } + + sched_ctx = dev->data->dev_private; + + sched_ctx->reordering_enabled = enable_reorder; + + return 0; +} + +int +rte_cryptodev_scheduler_ordering_get(uint8_t scheduler_id) +{ + struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id); + struct scheduler_ctx *sched_ctx; + + if (!dev) { + CS_LOG_ERR("Operation not supported"); + return -ENOTSUP; + } + + if (dev->driver_id != cryptodev_driver_id) { + CS_LOG_ERR("Operation not supported"); + return -ENOTSUP; + } + + sched_ctx = dev->data->dev_private; + + return (int)sched_ctx->reordering_enabled; +} + +int +rte_cryptodev_scheduler_load_user_scheduler(uint8_t scheduler_id, + struct rte_cryptodev_scheduler *scheduler) { + + struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id); + struct scheduler_ctx *sched_ctx; + + if (!dev) { + CS_LOG_ERR("Operation not supported"); + return -ENOTSUP; + } + + if (dev->driver_id != cryptodev_driver_id) { + CS_LOG_ERR("Operation not supported"); + return -ENOTSUP; + } + + if (dev->data->dev_started) { + CS_LOG_ERR("Illegal operation"); + return -EBUSY; + } + + sched_ctx = dev->data->dev_private; + + if (strlen(scheduler->name) > RTE_CRYPTODEV_NAME_MAX_LEN - 1) { + CS_LOG_ERR("Invalid name %s, should be less than " + "%u bytes.\n", scheduler->name, + RTE_CRYPTODEV_NAME_MAX_LEN); + return -EINVAL; + } + snprintf(sched_ctx->name, sizeof(sched_ctx->name), "%s", + scheduler->name); + + if (strlen(scheduler->description) > + RTE_CRYPTODEV_SCHEDULER_DESC_MAX_LEN - 1) { + CS_LOG_ERR("Invalid description %s, should be less than " + "%u bytes.\n", scheduler->description, + RTE_CRYPTODEV_SCHEDULER_DESC_MAX_LEN - 1); + return -EINVAL; + } + snprintf(sched_ctx->description, sizeof(sched_ctx->description), "%s", + scheduler->description); + + /* load scheduler instance operations functions */ + sched_ctx->ops.config_queue_pair = scheduler->ops->config_queue_pair; + sched_ctx->ops.create_private_ctx = scheduler->ops->create_private_ctx; + sched_ctx->ops.scheduler_start = scheduler->ops->scheduler_start; + sched_ctx->ops.scheduler_stop = scheduler->ops->scheduler_stop; + sched_ctx->ops.slave_attach = scheduler->ops->slave_attach; + sched_ctx->ops.slave_detach = scheduler->ops->slave_detach; + sched_ctx->ops.option_set = scheduler->ops->option_set; + sched_ctx->ops.option_get = scheduler->ops->option_get; + + if (sched_ctx->private_ctx) + rte_free(sched_ctx->private_ctx); + + if (sched_ctx->ops.create_private_ctx) { + int ret = (*sched_ctx->ops.create_private_ctx)(dev); + + if (ret < 0) { + CS_LOG_ERR("Unable to create scheduler private " + "context"); + return ret; + } + } + + sched_ctx->mode = scheduler->mode; + + return 0; +} + +int +rte_cryptodev_scheduler_slaves_get(uint8_t scheduler_id, uint8_t *slaves) +{ + struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id); + struct scheduler_ctx *sched_ctx; + uint32_t nb_slaves = 0; + + if (!dev) { + CS_LOG_ERR("Operation not supported"); + return -ENOTSUP; + } + + if (dev->driver_id != cryptodev_driver_id) { + CS_LOG_ERR("Operation not supported"); + return -ENOTSUP; + } + + sched_ctx = dev->data->dev_private; + + nb_slaves = sched_ctx->nb_slaves; + + if (slaves && nb_slaves) { + uint32_t i; + + for (i = 0; i < nb_slaves; i++) + slaves[i] = sched_ctx->slaves[i].dev_id; + } + + return (int)nb_slaves; +} + +int +rte_cryptodev_scheduler_option_set(uint8_t scheduler_id, + enum rte_cryptodev_schedule_option_type option_type, + void *option) +{ + struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id); + struct scheduler_ctx *sched_ctx; + + if (option_type == CDEV_SCHED_OPTION_NOT_SET || + option_type >= CDEV_SCHED_OPTION_COUNT) { + CS_LOG_ERR("Invalid option parameter"); + return -EINVAL; + } + + if (!option) { + CS_LOG_ERR("Invalid option parameter"); + return -EINVAL; + } + + if (dev->data->dev_started) { + CS_LOG_ERR("Illegal operation"); + return -EBUSY; + } + + sched_ctx = dev->data->dev_private; + + RTE_FUNC_PTR_OR_ERR_RET(*sched_ctx->ops.option_set, -ENOTSUP); + + return (*sched_ctx->ops.option_set)(dev, option_type, option); +} + +int +rte_cryptodev_scheduler_option_get(uint8_t scheduler_id, + enum rte_cryptodev_schedule_option_type option_type, + void *option) +{ + struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id); + struct scheduler_ctx *sched_ctx; + + if (!dev) { + CS_LOG_ERR("Operation not supported"); + return -ENOTSUP; + } + + if (!option) { + CS_LOG_ERR("Invalid option parameter"); + return -EINVAL; + } + + if (dev->driver_id != cryptodev_driver_id) { + CS_LOG_ERR("Operation not supported"); + return -ENOTSUP; + } + + sched_ctx = dev->data->dev_private; + + RTE_FUNC_PTR_OR_ERR_RET(*sched_ctx->ops.option_get, -ENOTSUP); + + return (*sched_ctx->ops.option_get)(dev, option_type, option); +} diff --git a/dpdk/drivers/crypto/scheduler/rte_cryptodev_scheduler.h b/dpdk/drivers/crypto/scheduler/rte_cryptodev_scheduler.h new file mode 100644 index 00000000..df22f2a9 --- /dev/null +++ b/dpdk/drivers/crypto/scheduler/rte_cryptodev_scheduler.h @@ -0,0 +1,312 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_CRYPTO_SCHEDULER_H +#define _RTE_CRYPTO_SCHEDULER_H + +/** + * @file rte_cryptodev_scheduler.h + * + * RTE Cryptodev Scheduler Device + * + * The RTE Cryptodev Scheduler Device allows the aggregation of multiple (slave) + * Cryptodevs into a single logical crypto device, and the scheduling the + * crypto operations to the slaves based on the mode of the specified mode of + * operation specified and supported. This implementation supports 3 modes of + * operation: round robin, packet-size based, and fail-over. + */ + +#include +#include "rte_cryptodev_scheduler_operations.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** Maximum number of bonded devices per device */ +#ifndef RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES +#define RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES (8) +#endif + +/** Maximum number of multi-core worker cores */ +#define RTE_CRYPTODEV_SCHEDULER_MAX_NB_WORKER_CORES (64) + +/** Round-robin scheduling mode string */ +#define SCHEDULER_MODE_NAME_ROUND_ROBIN round-robin +/** Packet-size based distribution scheduling mode string */ +#define SCHEDULER_MODE_NAME_PKT_SIZE_DISTR packet-size-distr +/** Fail-over scheduling mode string */ +#define SCHEDULER_MODE_NAME_FAIL_OVER fail-over +/** multi-core scheduling mode string */ +#define SCHEDULER_MODE_NAME_MULTI_CORE multi-core + +/** + * Crypto scheduler PMD operation modes + */ +enum rte_cryptodev_scheduler_mode { + CDEV_SCHED_MODE_NOT_SET = 0, + /** User defined mode */ + CDEV_SCHED_MODE_USERDEFINED, + /** Round-robin mode */ + CDEV_SCHED_MODE_ROUNDROBIN, + /** Packet-size based distribution mode */ + CDEV_SCHED_MODE_PKT_SIZE_DISTR, + /** Fail-over mode */ + CDEV_SCHED_MODE_FAILOVER, + /** multi-core mode */ + CDEV_SCHED_MODE_MULTICORE, + + CDEV_SCHED_MODE_COUNT /**< number of modes */ +}; + +#define RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN (64) +#define RTE_CRYPTODEV_SCHEDULER_DESC_MAX_LEN (256) + +/** + * Crypto scheduler option types + */ +enum rte_cryptodev_schedule_option_type { + CDEV_SCHED_OPTION_NOT_SET = 0, + CDEV_SCHED_OPTION_THRESHOLD, + + CDEV_SCHED_OPTION_COUNT +}; + +/** + * Threshold option structure + */ +struct rte_cryptodev_scheduler_threshold_option { + uint32_t threshold; /**< Threshold for packet-size mode */ +}; + +struct rte_cryptodev_scheduler; + +/** + * Load a user defined scheduler + * + * @param scheduler_id + * The target scheduler device ID + * @param scheduler + * Pointer to the user defined scheduler + * + * @return + * - 0 if the scheduler is successfully loaded + * - -ENOTSUP if the operation is not supported. + * - -EBUSY if device is started. + * - -EINVAL if input values are invalid. + */ +int +rte_cryptodev_scheduler_load_user_scheduler(uint8_t scheduler_id, + struct rte_cryptodev_scheduler *scheduler); + +/** + * Attach a crypto device to the scheduler + * + * @param scheduler_id + * The target scheduler device ID + * @param slave_id + * Crypto device ID to be attached + * + * @return + * - 0 if the slave is attached. + * - -ENOTSUP if the operation is not supported. + * - -EBUSY if device is started. + * - -ENOMEM if the scheduler's slave list is full. + */ +int +rte_cryptodev_scheduler_slave_attach(uint8_t scheduler_id, uint8_t slave_id); + +/** + * Detach a crypto device from the scheduler + * + * @param scheduler_id + * The target scheduler device ID + * @param slave_id + * Crypto device ID to be detached + * + * @return + * - 0 if the slave is detached. + * - -ENOTSUP if the operation is not supported. + * - -EBUSY if device is started. + */ +int +rte_cryptodev_scheduler_slave_detach(uint8_t scheduler_id, uint8_t slave_id); + + +/** + * Set the scheduling mode + * + * @param scheduler_id + * The target scheduler device ID + * @param mode + * The scheduling mode + * + * @return + * - 0 if the mode is set. + * - -ENOTSUP if the operation is not supported. + * - -EBUSY if device is started. + */ +int +rte_cryptodev_scheduler_mode_set(uint8_t scheduler_id, + enum rte_cryptodev_scheduler_mode mode); + +/** + * Get the current scheduling mode + * + * @param scheduler_id + * The target scheduler device ID + * + * @return mode + * - non-negative enumerate value: the scheduling mode + * - -ENOTSUP if the operation is not supported. + */ +enum rte_cryptodev_scheduler_mode +rte_cryptodev_scheduler_mode_get(uint8_t scheduler_id); + +/** + * Set the crypto ops reordering feature on/off + * + * @param scheduler_id + * The target scheduler device ID + * @param enable_reorder + * Set the crypto op reordering feature + * - 0: disable reordering + * - 1: enable reordering + * + * @return + * - 0 if the ordering is set. + * - -ENOTSUP if the operation is not supported. + * - -EBUSY if device is started. + */ +int +rte_cryptodev_scheduler_ordering_set(uint8_t scheduler_id, + uint32_t enable_reorder); + +/** + * Get the current crypto ops reordering feature + * + * @param scheduler_id + * The target scheduler device ID + * + * @return + * - 0 if reordering is disabled + * - 1 if reordering is enabled + * - -ENOTSUP if the operation is not supported. + */ +int +rte_cryptodev_scheduler_ordering_get(uint8_t scheduler_id); + +/** + * Get the the attached slaves' count and/or ID + * + * @param scheduler_id + * The target scheduler device ID + * @param slaves + * If successful, the function will write back all slaves' device IDs to it. + * This parameter will either be an uint8_t array of + * RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES elements or NULL. + * + * @return + * - non-negative number: the number of slaves attached + * - -ENOTSUP if the operation is not supported. + */ +int +rte_cryptodev_scheduler_slaves_get(uint8_t scheduler_id, uint8_t *slaves); + +/** + * Set the mode specific option + * + * @param scheduler_id + * The target scheduler device ID + * @param option_type + * The option type enumerate + * @param option + * The specific mode's option structure + * + * @return + * - 0 if successful + * - negative integer if otherwise. + */ +int +rte_cryptodev_scheduler_option_set(uint8_t scheduler_id, + enum rte_cryptodev_schedule_option_type option_type, + void *option); + +/** + * Set the mode specific option + * + * @param scheduler_id + * The target scheduler device ID + * @param option_type + * The option type enumerate + * @param option + * If successful, the function will write back the current + * + * @return + * - 0 if successful + * - negative integer if otherwise. + */ +int +rte_cryptodev_scheduler_option_get(uint8_t scheduler_id, + enum rte_cryptodev_schedule_option_type option_type, + void *option); + +typedef uint16_t (*rte_cryptodev_scheduler_burst_enqueue_t)(void *qp_ctx, + struct rte_crypto_op **ops, uint16_t nb_ops); + +typedef uint16_t (*rte_cryptodev_scheduler_burst_dequeue_t)(void *qp_ctx, + struct rte_crypto_op **ops, uint16_t nb_ops); + +/** The data structure associated with each mode of scheduler. */ +struct rte_cryptodev_scheduler { + const char *name; /**< Scheduler name */ + const char *description; /**< Scheduler description */ + enum rte_cryptodev_scheduler_mode mode; /**< Scheduling mode */ + + /** Pointer to scheduler operation structure */ + struct rte_cryptodev_scheduler_ops *ops; +}; + +/** Round-robin mode scheduler */ +extern struct rte_cryptodev_scheduler *roundrobin_scheduler; +/** Packet-size based distribution mode scheduler */ +extern struct rte_cryptodev_scheduler *pkt_size_based_distr_scheduler; +/** Fail-over mode scheduler */ +extern struct rte_cryptodev_scheduler *failover_scheduler; +/** multi-core mode scheduler */ +extern struct rte_cryptodev_scheduler *multicore_scheduler; + +#ifdef __cplusplus +} +#endif +#endif /* _RTE_CRYPTO_SCHEDULER_H */ diff --git a/dpdk/drivers/crypto/scheduler/rte_cryptodev_scheduler_operations.h b/dpdk/drivers/crypto/scheduler/rte_cryptodev_scheduler_operations.h new file mode 100644 index 00000000..719165c3 --- /dev/null +++ b/dpdk/drivers/crypto/scheduler/rte_cryptodev_scheduler_operations.h @@ -0,0 +1,85 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_CRYPTO_SCHEDULER_OPERATIONS_H +#define _RTE_CRYPTO_SCHEDULER_OPERATIONS_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef int (*rte_cryptodev_scheduler_slave_attach_t)( + struct rte_cryptodev *dev, uint8_t slave_id); +typedef int (*rte_cryptodev_scheduler_slave_detach_t)( + struct rte_cryptodev *dev, uint8_t slave_id); + +typedef int (*rte_cryptodev_scheduler_start_t)(struct rte_cryptodev *dev); +typedef int (*rte_cryptodev_scheduler_stop_t)(struct rte_cryptodev *dev); + +typedef int (*rte_cryptodev_scheduler_config_queue_pair)( + struct rte_cryptodev *dev, uint16_t qp_id); + +typedef int (*rte_cryptodev_scheduler_create_private_ctx)( + struct rte_cryptodev *dev); + +typedef int (*rte_cryptodev_scheduler_config_option_set)( + struct rte_cryptodev *dev, + uint32_t option_type, + void *option); + +typedef int (*rte_cryptodev_scheduler_config_option_get)( + struct rte_cryptodev *dev, + uint32_t option_type, + void *option); + +struct rte_cryptodev_scheduler_ops { + rte_cryptodev_scheduler_slave_attach_t slave_attach; + rte_cryptodev_scheduler_slave_attach_t slave_detach; + + rte_cryptodev_scheduler_start_t scheduler_start; + rte_cryptodev_scheduler_stop_t scheduler_stop; + + rte_cryptodev_scheduler_config_queue_pair config_queue_pair; + + rte_cryptodev_scheduler_create_private_ctx create_private_ctx; + + rte_cryptodev_scheduler_config_option_set option_set; + rte_cryptodev_scheduler_config_option_get option_get; +}; + +#ifdef __cplusplus +} +#endif +#endif /* _RTE_CRYPTO_SCHEDULER_OPERATIONS_H */ diff --git a/dpdk/drivers/crypto/scheduler/rte_pmd_crypto_scheduler_version.map b/dpdk/drivers/crypto/scheduler/rte_pmd_crypto_scheduler_version.map new file mode 100644 index 00000000..5c43127c --- /dev/null +++ b/dpdk/drivers/crypto/scheduler/rte_pmd_crypto_scheduler_version.map @@ -0,0 +1,21 @@ +DPDK_17.02 { + global: + + rte_cryptodev_scheduler_load_user_scheduler; + rte_cryptodev_scheduler_slave_attach; + rte_cryptodev_scheduler_slave_detach; + rte_cryptodev_scheduler_ordering_set; + rte_cryptodev_scheduler_ordering_get; + +}; + +DPDK_17.05 { + global: + + rte_cryptodev_scheduler_mode_get; + rte_cryptodev_scheduler_mode_set; + rte_cryptodev_scheduler_option_get; + rte_cryptodev_scheduler_option_set; + rte_cryptodev_scheduler_slaves_get; + +} DPDK_17.02; diff --git a/dpdk/drivers/crypto/scheduler/scheduler_failover.c b/dpdk/drivers/crypto/scheduler/scheduler_failover.c new file mode 100644 index 00000000..2aa13f8e --- /dev/null +++ b/dpdk/drivers/crypto/scheduler/scheduler_failover.c @@ -0,0 +1,248 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include + +#include "rte_cryptodev_scheduler_operations.h" +#include "scheduler_pmd_private.h" + +#define PRIMARY_SLAVE_IDX 0 +#define SECONDARY_SLAVE_IDX 1 +#define NB_FAILOVER_SLAVES 2 +#define SLAVE_SWITCH_MASK (0x01) + +struct fo_scheduler_qp_ctx { + struct scheduler_slave primary_slave; + struct scheduler_slave secondary_slave; + + uint8_t deq_idx; +}; + +static __rte_always_inline uint16_t +failover_slave_enqueue(struct scheduler_slave *slave, + struct rte_crypto_op **ops, uint16_t nb_ops) +{ + uint16_t i, processed_ops; + + for (i = 0; i < nb_ops && i < 4; i++) + rte_prefetch0(ops[i]->sym->session); + + processed_ops = rte_cryptodev_enqueue_burst(slave->dev_id, + slave->qp_id, ops, nb_ops); + slave->nb_inflight_cops += processed_ops; + + return processed_ops; +} + +static uint16_t +schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) +{ + struct fo_scheduler_qp_ctx *qp_ctx = + ((struct scheduler_qp_ctx *)qp)->private_qp_ctx; + uint16_t enqueued_ops; + + if (unlikely(nb_ops == 0)) + return 0; + + enqueued_ops = failover_slave_enqueue(&qp_ctx->primary_slave, + ops, nb_ops); + + if (enqueued_ops < nb_ops) + enqueued_ops += failover_slave_enqueue(&qp_ctx->secondary_slave, + &ops[enqueued_ops], + nb_ops - enqueued_ops); + + return enqueued_ops; +} + + +static uint16_t +schedule_enqueue_ordering(void *qp, struct rte_crypto_op **ops, + uint16_t nb_ops) +{ + struct rte_ring *order_ring = + ((struct scheduler_qp_ctx *)qp)->order_ring; + uint16_t nb_ops_to_enq = get_max_enqueue_order_count(order_ring, + nb_ops); + uint16_t nb_ops_enqd = schedule_enqueue(qp, ops, + nb_ops_to_enq); + + scheduler_order_insert(order_ring, ops, nb_ops_enqd); + + return nb_ops_enqd; +} + +static uint16_t +schedule_dequeue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) +{ + struct fo_scheduler_qp_ctx *qp_ctx = + ((struct scheduler_qp_ctx *)qp)->private_qp_ctx; + struct scheduler_slave *slaves[NB_FAILOVER_SLAVES] = { + &qp_ctx->primary_slave, &qp_ctx->secondary_slave}; + struct scheduler_slave *slave = slaves[qp_ctx->deq_idx]; + uint16_t nb_deq_ops = 0, nb_deq_ops2 = 0; + + if (slave->nb_inflight_cops) { + nb_deq_ops = rte_cryptodev_dequeue_burst(slave->dev_id, + slave->qp_id, ops, nb_ops); + slave->nb_inflight_cops -= nb_deq_ops; + } + + qp_ctx->deq_idx = (~qp_ctx->deq_idx) & SLAVE_SWITCH_MASK; + + if (nb_deq_ops == nb_ops) + return nb_deq_ops; + + slave = slaves[qp_ctx->deq_idx]; + + if (slave->nb_inflight_cops) { + nb_deq_ops2 = rte_cryptodev_dequeue_burst(slave->dev_id, + slave->qp_id, &ops[nb_deq_ops], nb_ops - nb_deq_ops); + slave->nb_inflight_cops -= nb_deq_ops2; + } + + return nb_deq_ops + nb_deq_ops2; +} + +static uint16_t +schedule_dequeue_ordering(void *qp, struct rte_crypto_op **ops, + uint16_t nb_ops) +{ + struct rte_ring *order_ring = + ((struct scheduler_qp_ctx *)qp)->order_ring; + + schedule_dequeue(qp, ops, nb_ops); + + return scheduler_order_drain(order_ring, ops, nb_ops); +} + +static int +slave_attach(__rte_unused struct rte_cryptodev *dev, + __rte_unused uint8_t slave_id) +{ + return 0; +} + +static int +slave_detach(__rte_unused struct rte_cryptodev *dev, + __rte_unused uint8_t slave_id) +{ + return 0; +} + +static int +scheduler_start(struct rte_cryptodev *dev) +{ + struct scheduler_ctx *sched_ctx = dev->data->dev_private; + uint16_t i; + + if (sched_ctx->nb_slaves < 2) { + CS_LOG_ERR("Number of slaves shall no less than 2"); + return -ENOMEM; + } + + if (sched_ctx->reordering_enabled) { + dev->enqueue_burst = schedule_enqueue_ordering; + dev->dequeue_burst = schedule_dequeue_ordering; + } else { + dev->enqueue_burst = schedule_enqueue; + dev->dequeue_burst = schedule_dequeue; + } + + for (i = 0; i < dev->data->nb_queue_pairs; i++) { + struct fo_scheduler_qp_ctx *qp_ctx = + ((struct scheduler_qp_ctx *) + dev->data->queue_pairs[i])->private_qp_ctx; + + rte_memcpy(&qp_ctx->primary_slave, + &sched_ctx->slaves[PRIMARY_SLAVE_IDX], + sizeof(struct scheduler_slave)); + rte_memcpy(&qp_ctx->secondary_slave, + &sched_ctx->slaves[SECONDARY_SLAVE_IDX], + sizeof(struct scheduler_slave)); + } + + return 0; +} + +static int +scheduler_stop(__rte_unused struct rte_cryptodev *dev) +{ + return 0; +} + +static int +scheduler_config_qp(struct rte_cryptodev *dev, uint16_t qp_id) +{ + struct scheduler_qp_ctx *qp_ctx = dev->data->queue_pairs[qp_id]; + struct fo_scheduler_qp_ctx *fo_qp_ctx; + + fo_qp_ctx = rte_zmalloc_socket(NULL, sizeof(*fo_qp_ctx), 0, + rte_socket_id()); + if (!fo_qp_ctx) { + CS_LOG_ERR("failed allocate memory for private queue pair"); + return -ENOMEM; + } + + qp_ctx->private_qp_ctx = (void *)fo_qp_ctx; + + return 0; +} + +static int +scheduler_create_private_ctx(__rte_unused struct rte_cryptodev *dev) +{ + return 0; +} + +struct rte_cryptodev_scheduler_ops scheduler_fo_ops = { + slave_attach, + slave_detach, + scheduler_start, + scheduler_stop, + scheduler_config_qp, + scheduler_create_private_ctx, + NULL, /* option_set */ + NULL /*option_get */ +}; + +struct rte_cryptodev_scheduler fo_scheduler = { + .name = "failover-scheduler", + .description = "scheduler which enqueues to the primary slave, " + "and only then enqueues to the secondary slave " + "upon failing on enqueuing to primary", + .mode = CDEV_SCHED_MODE_FAILOVER, + .ops = &scheduler_fo_ops +}; + +struct rte_cryptodev_scheduler *failover_scheduler = &fo_scheduler; diff --git a/dpdk/drivers/crypto/scheduler/scheduler_multicore.c b/dpdk/drivers/crypto/scheduler/scheduler_multicore.c new file mode 100644 index 00000000..0cd5bce5 --- /dev/null +++ b/dpdk/drivers/crypto/scheduler/scheduler_multicore.c @@ -0,0 +1,415 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include + +#include +#include + +#include "rte_cryptodev_scheduler_operations.h" +#include "scheduler_pmd_private.h" + +#define MC_SCHED_ENQ_RING_NAME_PREFIX "MCS_ENQR_" +#define MC_SCHED_DEQ_RING_NAME_PREFIX "MCS_DEQR_" + +#define MC_SCHED_BUFFER_SIZE 32 + +#define CRYPTO_OP_STATUS_BIT_COMPLETE 0x80 + +/** multi-core scheduler context */ +struct mc_scheduler_ctx { + uint32_t num_workers; /**< Number of workers polling */ + uint32_t stop_signal; + + struct rte_ring *sched_enq_ring[RTE_CRYPTODEV_SCHEDULER_MAX_NB_WORKER_CORES]; + struct rte_ring *sched_deq_ring[RTE_CRYPTODEV_SCHEDULER_MAX_NB_WORKER_CORES]; +}; + +struct mc_scheduler_qp_ctx { + struct scheduler_slave slaves[RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES]; + uint32_t nb_slaves; + + uint32_t last_enq_worker_idx; + uint32_t last_deq_worker_idx; + + struct mc_scheduler_ctx *mc_private_ctx; +}; + +static uint16_t +schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) +{ + struct mc_scheduler_qp_ctx *mc_qp_ctx = + ((struct scheduler_qp_ctx *)qp)->private_qp_ctx; + struct mc_scheduler_ctx *mc_ctx = mc_qp_ctx->mc_private_ctx; + uint32_t worker_idx = mc_qp_ctx->last_enq_worker_idx; + uint16_t i, processed_ops = 0; + + if (unlikely(nb_ops == 0)) + return 0; + + for (i = 0; i < mc_ctx->num_workers && nb_ops != 0; i++) { + struct rte_ring *enq_ring = mc_ctx->sched_enq_ring[worker_idx]; + uint16_t nb_queue_ops = rte_ring_enqueue_burst(enq_ring, + (void *)(&ops[processed_ops]), nb_ops, NULL); + + nb_ops -= nb_queue_ops; + processed_ops += nb_queue_ops; + + if (++worker_idx == mc_ctx->num_workers) + worker_idx = 0; + } + mc_qp_ctx->last_enq_worker_idx = worker_idx; + + return processed_ops; +} + +static uint16_t +schedule_enqueue_ordering(void *qp, struct rte_crypto_op **ops, + uint16_t nb_ops) +{ + struct rte_ring *order_ring = + ((struct scheduler_qp_ctx *)qp)->order_ring; + uint16_t nb_ops_to_enq = get_max_enqueue_order_count(order_ring, + nb_ops); + uint16_t nb_ops_enqd = schedule_enqueue(qp, ops, + nb_ops_to_enq); + + scheduler_order_insert(order_ring, ops, nb_ops_enqd); + + return nb_ops_enqd; +} + + +static uint16_t +schedule_dequeue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) +{ + struct mc_scheduler_qp_ctx *mc_qp_ctx = + ((struct scheduler_qp_ctx *)qp)->private_qp_ctx; + struct mc_scheduler_ctx *mc_ctx = mc_qp_ctx->mc_private_ctx; + uint32_t worker_idx = mc_qp_ctx->last_deq_worker_idx; + uint16_t i, processed_ops = 0; + + for (i = 0; i < mc_ctx->num_workers && nb_ops != 0; i++) { + struct rte_ring *deq_ring = mc_ctx->sched_deq_ring[worker_idx]; + uint16_t nb_deq_ops = rte_ring_dequeue_burst(deq_ring, + (void *)(&ops[processed_ops]), nb_ops, NULL); + + nb_ops -= nb_deq_ops; + processed_ops += nb_deq_ops; + if (++worker_idx == mc_ctx->num_workers) + worker_idx = 0; + } + + mc_qp_ctx->last_deq_worker_idx = worker_idx; + + return processed_ops; + +} + +static uint16_t +schedule_dequeue_ordering(void *qp, struct rte_crypto_op **ops, + uint16_t nb_ops) +{ + struct rte_ring *order_ring = ((struct scheduler_qp_ctx *)qp)->order_ring; + struct rte_crypto_op *op; + uint32_t nb_objs = rte_ring_count(order_ring); + uint32_t nb_ops_to_deq = 0; + uint32_t nb_ops_deqd = 0; + + if (nb_objs > nb_ops) + nb_objs = nb_ops; + + while (nb_ops_to_deq < nb_objs) { + SCHEDULER_GET_RING_OBJ(order_ring, nb_ops_to_deq, op); + + if (!(op->status & CRYPTO_OP_STATUS_BIT_COMPLETE)) + break; + + op->status &= ~CRYPTO_OP_STATUS_BIT_COMPLETE; + nb_ops_to_deq++; + } + + if (nb_ops_to_deq) { + nb_ops_deqd = rte_ring_sc_dequeue_bulk(order_ring, + (void **)ops, nb_ops_to_deq, NULL); + } + + return nb_ops_deqd; +} + +static int +slave_attach(__rte_unused struct rte_cryptodev *dev, + __rte_unused uint8_t slave_id) +{ + return 0; +} + +static int +slave_detach(__rte_unused struct rte_cryptodev *dev, + __rte_unused uint8_t slave_id) +{ + return 0; +} + +static int +mc_scheduler_worker(struct rte_cryptodev *dev) +{ + struct scheduler_ctx *sched_ctx = dev->data->dev_private; + struct mc_scheduler_ctx *mc_ctx = sched_ctx->private_ctx; + struct rte_ring *enq_ring; + struct rte_ring *deq_ring; + uint32_t core_id = rte_lcore_id(); + int i, worker_idx = -1; + struct scheduler_slave *slave; + struct rte_crypto_op *enq_ops[MC_SCHED_BUFFER_SIZE]; + struct rte_crypto_op *deq_ops[MC_SCHED_BUFFER_SIZE]; + uint16_t processed_ops; + uint16_t pending_enq_ops = 0; + uint16_t pending_enq_ops_idx = 0; + uint16_t pending_deq_ops = 0; + uint16_t pending_deq_ops_idx = 0; + uint16_t inflight_ops = 0; + const uint8_t reordering_enabled = sched_ctx->reordering_enabled; + + for (i = 0; i < (int)sched_ctx->nb_wc; i++) { + if (sched_ctx->wc_pool[i] == core_id) { + worker_idx = i; + break; + } + } + if (worker_idx == -1) { + CS_LOG_ERR("worker on core %u:cannot find worker index!\n", core_id); + return -1; + } + + slave = &sched_ctx->slaves[worker_idx]; + enq_ring = mc_ctx->sched_enq_ring[worker_idx]; + deq_ring = mc_ctx->sched_deq_ring[worker_idx]; + + while (!mc_ctx->stop_signal) { + if (pending_enq_ops) { + processed_ops = + rte_cryptodev_enqueue_burst(slave->dev_id, + slave->qp_id, &enq_ops[pending_enq_ops_idx], + pending_enq_ops); + pending_enq_ops -= processed_ops; + pending_enq_ops_idx += processed_ops; + inflight_ops += processed_ops; + } else { + processed_ops = rte_ring_dequeue_burst(enq_ring, (void *)enq_ops, + MC_SCHED_BUFFER_SIZE, NULL); + if (processed_ops) { + pending_enq_ops_idx = rte_cryptodev_enqueue_burst( + slave->dev_id, slave->qp_id, + enq_ops, processed_ops); + pending_enq_ops = processed_ops - pending_enq_ops_idx; + inflight_ops += pending_enq_ops_idx; + } + } + + if (pending_deq_ops) { + processed_ops = rte_ring_enqueue_burst( + deq_ring, (void *)&deq_ops[pending_deq_ops_idx], + pending_deq_ops, NULL); + pending_deq_ops -= processed_ops; + pending_deq_ops_idx += processed_ops; + } else if (inflight_ops) { + processed_ops = rte_cryptodev_dequeue_burst(slave->dev_id, + slave->qp_id, deq_ops, MC_SCHED_BUFFER_SIZE); + if (processed_ops) { + inflight_ops -= processed_ops; + if (reordering_enabled) { + uint16_t j; + + for (j = 0; j < processed_ops; j++) { + deq_ops[j]->status |= + CRYPTO_OP_STATUS_BIT_COMPLETE; + } + } else { + pending_deq_ops_idx = rte_ring_enqueue_burst( + deq_ring, (void *)deq_ops, processed_ops, + NULL); + pending_deq_ops = processed_ops - + pending_deq_ops_idx; + } + } + } + + rte_pause(); + } + + return 0; +} + +static int +scheduler_start(struct rte_cryptodev *dev) +{ + struct scheduler_ctx *sched_ctx = dev->data->dev_private; + struct mc_scheduler_ctx *mc_ctx = sched_ctx->private_ctx; + uint16_t i; + + mc_ctx->stop_signal = 0; + + for (i = 0; i < sched_ctx->nb_wc; i++) + rte_eal_remote_launch( + (lcore_function_t *)mc_scheduler_worker, dev, + sched_ctx->wc_pool[i]); + + if (sched_ctx->reordering_enabled) { + dev->enqueue_burst = &schedule_enqueue_ordering; + dev->dequeue_burst = &schedule_dequeue_ordering; + } else { + dev->enqueue_burst = &schedule_enqueue; + dev->dequeue_burst = &schedule_dequeue; + } + + for (i = 0; i < dev->data->nb_queue_pairs; i++) { + struct scheduler_qp_ctx *qp_ctx = dev->data->queue_pairs[i]; + struct mc_scheduler_qp_ctx *mc_qp_ctx = + qp_ctx->private_qp_ctx; + uint32_t j; + + memset(mc_qp_ctx->slaves, 0, + RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES * + sizeof(struct scheduler_slave)); + for (j = 0; j < sched_ctx->nb_slaves; j++) { + mc_qp_ctx->slaves[j].dev_id = + sched_ctx->slaves[j].dev_id; + mc_qp_ctx->slaves[j].qp_id = i; + } + + mc_qp_ctx->nb_slaves = sched_ctx->nb_slaves; + + mc_qp_ctx->last_enq_worker_idx = 0; + mc_qp_ctx->last_deq_worker_idx = 0; + } + + return 0; +} + +static int +scheduler_stop(struct rte_cryptodev *dev) +{ + struct scheduler_ctx *sched_ctx = dev->data->dev_private; + struct mc_scheduler_ctx *mc_ctx = sched_ctx->private_ctx; + uint16_t i; + + mc_ctx->stop_signal = 1; + + for (i = 0; i < sched_ctx->nb_wc; i++) + rte_eal_wait_lcore(sched_ctx->wc_pool[i]); + + return 0; +} + +static int +scheduler_config_qp(struct rte_cryptodev *dev, uint16_t qp_id) +{ + struct scheduler_qp_ctx *qp_ctx = dev->data->queue_pairs[qp_id]; + struct mc_scheduler_qp_ctx *mc_qp_ctx; + struct scheduler_ctx *sched_ctx = dev->data->dev_private; + struct mc_scheduler_ctx *mc_ctx = sched_ctx->private_ctx; + + mc_qp_ctx = rte_zmalloc_socket(NULL, sizeof(*mc_qp_ctx), 0, + rte_socket_id()); + if (!mc_qp_ctx) { + CS_LOG_ERR("failed allocate memory for private queue pair"); + return -ENOMEM; + } + + mc_qp_ctx->mc_private_ctx = mc_ctx; + qp_ctx->private_qp_ctx = (void *)mc_qp_ctx; + + + return 0; +} + +static int +scheduler_create_private_ctx(struct rte_cryptodev *dev) +{ + struct scheduler_ctx *sched_ctx = dev->data->dev_private; + struct mc_scheduler_ctx *mc_ctx; + uint16_t i; + + if (sched_ctx->private_ctx) + rte_free(sched_ctx->private_ctx); + + mc_ctx = rte_zmalloc_socket(NULL, sizeof(struct mc_scheduler_ctx), 0, + rte_socket_id()); + if (!mc_ctx) { + CS_LOG_ERR("failed allocate memory"); + return -ENOMEM; + } + + mc_ctx->num_workers = sched_ctx->nb_wc; + for (i = 0; i < sched_ctx->nb_wc; i++) { + char r_name[16]; + + snprintf(r_name, sizeof(r_name), MC_SCHED_ENQ_RING_NAME_PREFIX "%u", i); + mc_ctx->sched_enq_ring[i] = rte_ring_create(r_name, PER_SLAVE_BUFF_SIZE, + rte_socket_id(), RING_F_SC_DEQ | RING_F_SP_ENQ); + if (!mc_ctx->sched_enq_ring[i]) { + CS_LOG_ERR("Cannot create ring for worker %u", i); + return -1; + } + snprintf(r_name, sizeof(r_name), MC_SCHED_DEQ_RING_NAME_PREFIX "%u", i); + mc_ctx->sched_deq_ring[i] = rte_ring_create(r_name, PER_SLAVE_BUFF_SIZE, + rte_socket_id(), RING_F_SC_DEQ | RING_F_SP_ENQ); + if (!mc_ctx->sched_deq_ring[i]) { + CS_LOG_ERR("Cannot create ring for worker %u", i); + return -1; + } + } + + sched_ctx->private_ctx = (void *)mc_ctx; + + return 0; +} + +struct rte_cryptodev_scheduler_ops scheduler_mc_ops = { + slave_attach, + slave_detach, + scheduler_start, + scheduler_stop, + scheduler_config_qp, + scheduler_create_private_ctx, + NULL, /* option_set */ + NULL /* option_get */ +}; + +struct rte_cryptodev_scheduler mc_scheduler = { + .name = "multicore-scheduler", + .description = "scheduler which will run burst across multiple cpu cores", + .mode = CDEV_SCHED_MODE_MULTICORE, + .ops = &scheduler_mc_ops +}; + +struct rte_cryptodev_scheduler *multicore_scheduler = &mc_scheduler; diff --git a/dpdk/drivers/crypto/scheduler/scheduler_pkt_size_distr.c b/dpdk/drivers/crypto/scheduler/scheduler_pkt_size_distr.c new file mode 100644 index 00000000..1dd1bc32 --- /dev/null +++ b/dpdk/drivers/crypto/scheduler/scheduler_pkt_size_distr.c @@ -0,0 +1,446 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include + +#include "rte_cryptodev_scheduler_operations.h" +#include "scheduler_pmd_private.h" + +#define DEF_PKT_SIZE_THRESHOLD (0xffffff80) +#define SLAVE_IDX_SWITCH_MASK (0x01) +#define PRIMARY_SLAVE_IDX 0 +#define SECONDARY_SLAVE_IDX 1 +#define NB_PKT_SIZE_SLAVES 2 + +/** pkt size based scheduler context */ +struct psd_scheduler_ctx { + uint32_t threshold; +}; + +/** pkt size based scheduler queue pair context */ +struct psd_scheduler_qp_ctx { + struct scheduler_slave primary_slave; + struct scheduler_slave secondary_slave; + uint32_t threshold; + uint8_t deq_idx; +} __rte_cache_aligned; + +/** scheduling operation variables' wrapping */ +struct psd_schedule_op { + uint8_t slave_idx; + uint16_t pos; +}; + +static uint16_t +schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) +{ + struct scheduler_qp_ctx *qp_ctx = qp; + struct psd_scheduler_qp_ctx *psd_qp_ctx = qp_ctx->private_qp_ctx; + struct rte_crypto_op *sched_ops[NB_PKT_SIZE_SLAVES][nb_ops]; + uint32_t in_flight_ops[NB_PKT_SIZE_SLAVES] = { + psd_qp_ctx->primary_slave.nb_inflight_cops, + psd_qp_ctx->secondary_slave.nb_inflight_cops + }; + struct psd_schedule_op enq_ops[NB_PKT_SIZE_SLAVES] = { + {PRIMARY_SLAVE_IDX, 0}, {SECONDARY_SLAVE_IDX, 0} + }; + struct psd_schedule_op *p_enq_op; + uint16_t i, processed_ops_pri = 0, processed_ops_sec = 0; + uint32_t job_len; + + if (unlikely(nb_ops == 0)) + return 0; + + for (i = 0; i < nb_ops && i < 4; i++) { + rte_prefetch0(ops[i]->sym); + rte_prefetch0(ops[i]->sym->session); + } + + for (i = 0; (i < (nb_ops - 8)) && (nb_ops > 8); i += 4) { + rte_prefetch0(ops[i + 4]->sym); + rte_prefetch0(ops[i + 4]->sym->session); + rte_prefetch0(ops[i + 5]->sym); + rte_prefetch0(ops[i + 5]->sym->session); + rte_prefetch0(ops[i + 6]->sym); + rte_prefetch0(ops[i + 6]->sym->session); + rte_prefetch0(ops[i + 7]->sym); + rte_prefetch0(ops[i + 7]->sym->session); + + /* job_len is initialized as cipher data length, once + * it is 0, equals to auth data length + */ + job_len = ops[i]->sym->cipher.data.length; + job_len += (ops[i]->sym->cipher.data.length == 0) * + ops[i]->sym->auth.data.length; + /* decide the target op based on the job length */ + p_enq_op = &enq_ops[!(job_len & psd_qp_ctx->threshold)]; + + /* stop schedule cops before the queue is full, this shall + * prevent the failed enqueue + */ + if (p_enq_op->pos + in_flight_ops[p_enq_op->slave_idx] == + qp_ctx->max_nb_objs) { + i = nb_ops; + break; + } + + sched_ops[p_enq_op->slave_idx][p_enq_op->pos] = ops[i]; + p_enq_op->pos++; + + job_len = ops[i+1]->sym->cipher.data.length; + job_len += (ops[i+1]->sym->cipher.data.length == 0) * + ops[i+1]->sym->auth.data.length; + p_enq_op = &enq_ops[!(job_len & psd_qp_ctx->threshold)]; + + if (p_enq_op->pos + in_flight_ops[p_enq_op->slave_idx] == + qp_ctx->max_nb_objs) { + i = nb_ops; + break; + } + + sched_ops[p_enq_op->slave_idx][p_enq_op->pos] = ops[i+1]; + p_enq_op->pos++; + + job_len = ops[i+2]->sym->cipher.data.length; + job_len += (ops[i+2]->sym->cipher.data.length == 0) * + ops[i+2]->sym->auth.data.length; + p_enq_op = &enq_ops[!(job_len & psd_qp_ctx->threshold)]; + + if (p_enq_op->pos + in_flight_ops[p_enq_op->slave_idx] == + qp_ctx->max_nb_objs) { + i = nb_ops; + break; + } + + sched_ops[p_enq_op->slave_idx][p_enq_op->pos] = ops[i+2]; + p_enq_op->pos++; + + job_len = ops[i+3]->sym->cipher.data.length; + job_len += (ops[i+3]->sym->cipher.data.length == 0) * + ops[i+3]->sym->auth.data.length; + p_enq_op = &enq_ops[!(job_len & psd_qp_ctx->threshold)]; + + if (p_enq_op->pos + in_flight_ops[p_enq_op->slave_idx] == + qp_ctx->max_nb_objs) { + i = nb_ops; + break; + } + + sched_ops[p_enq_op->slave_idx][p_enq_op->pos] = ops[i+3]; + p_enq_op->pos++; + } + + for (; i < nb_ops; i++) { + job_len = ops[i]->sym->cipher.data.length; + job_len += (ops[i]->sym->cipher.data.length == 0) * + ops[i]->sym->auth.data.length; + p_enq_op = &enq_ops[!(job_len & psd_qp_ctx->threshold)]; + + if (p_enq_op->pos + in_flight_ops[p_enq_op->slave_idx] == + qp_ctx->max_nb_objs) { + i = nb_ops; + break; + } + + sched_ops[p_enq_op->slave_idx][p_enq_op->pos] = ops[i]; + p_enq_op->pos++; + } + + processed_ops_pri = rte_cryptodev_enqueue_burst( + psd_qp_ctx->primary_slave.dev_id, + psd_qp_ctx->primary_slave.qp_id, + sched_ops[PRIMARY_SLAVE_IDX], + enq_ops[PRIMARY_SLAVE_IDX].pos); + /* enqueue shall not fail as the slave queue is monitored */ + RTE_ASSERT(processed_ops_pri == enq_ops[PRIMARY_SLAVE_IDX].pos); + + psd_qp_ctx->primary_slave.nb_inflight_cops += processed_ops_pri; + + processed_ops_sec = rte_cryptodev_enqueue_burst( + psd_qp_ctx->secondary_slave.dev_id, + psd_qp_ctx->secondary_slave.qp_id, + sched_ops[SECONDARY_SLAVE_IDX], + enq_ops[SECONDARY_SLAVE_IDX].pos); + RTE_ASSERT(processed_ops_sec == enq_ops[SECONDARY_SLAVE_IDX].pos); + + psd_qp_ctx->secondary_slave.nb_inflight_cops += processed_ops_sec; + + return processed_ops_pri + processed_ops_sec; +} + +static uint16_t +schedule_enqueue_ordering(void *qp, struct rte_crypto_op **ops, + uint16_t nb_ops) +{ + struct rte_ring *order_ring = + ((struct scheduler_qp_ctx *)qp)->order_ring; + uint16_t nb_ops_to_enq = get_max_enqueue_order_count(order_ring, + nb_ops); + uint16_t nb_ops_enqd = schedule_enqueue(qp, ops, + nb_ops_to_enq); + + scheduler_order_insert(order_ring, ops, nb_ops_enqd); + + return nb_ops_enqd; +} + +static uint16_t +schedule_dequeue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) +{ + struct psd_scheduler_qp_ctx *qp_ctx = + ((struct scheduler_qp_ctx *)qp)->private_qp_ctx; + struct scheduler_slave *slaves[NB_PKT_SIZE_SLAVES] = { + &qp_ctx->primary_slave, &qp_ctx->secondary_slave}; + struct scheduler_slave *slave = slaves[qp_ctx->deq_idx]; + uint16_t nb_deq_ops_pri = 0, nb_deq_ops_sec = 0; + + if (slave->nb_inflight_cops) { + nb_deq_ops_pri = rte_cryptodev_dequeue_burst(slave->dev_id, + slave->qp_id, ops, nb_ops); + slave->nb_inflight_cops -= nb_deq_ops_pri; + } + + qp_ctx->deq_idx = (~qp_ctx->deq_idx) & SLAVE_IDX_SWITCH_MASK; + + if (nb_deq_ops_pri == nb_ops) + return nb_deq_ops_pri; + + slave = slaves[qp_ctx->deq_idx]; + + if (slave->nb_inflight_cops) { + nb_deq_ops_sec = rte_cryptodev_dequeue_burst(slave->dev_id, + slave->qp_id, &ops[nb_deq_ops_pri], + nb_ops - nb_deq_ops_pri); + slave->nb_inflight_cops -= nb_deq_ops_sec; + + if (!slave->nb_inflight_cops) + qp_ctx->deq_idx = (~qp_ctx->deq_idx) & + SLAVE_IDX_SWITCH_MASK; + } + + return nb_deq_ops_pri + nb_deq_ops_sec; +} + +static uint16_t +schedule_dequeue_ordering(void *qp, struct rte_crypto_op **ops, + uint16_t nb_ops) +{ + struct rte_ring *order_ring = + ((struct scheduler_qp_ctx *)qp)->order_ring; + + schedule_dequeue(qp, ops, nb_ops); + + return scheduler_order_drain(order_ring, ops, nb_ops); +} + +static int +slave_attach(__rte_unused struct rte_cryptodev *dev, + __rte_unused uint8_t slave_id) +{ + return 0; +} + +static int +slave_detach(__rte_unused struct rte_cryptodev *dev, + __rte_unused uint8_t slave_id) +{ + return 0; +} + +static int +scheduler_start(struct rte_cryptodev *dev) +{ + struct scheduler_ctx *sched_ctx = dev->data->dev_private; + struct psd_scheduler_ctx *psd_ctx = sched_ctx->private_ctx; + uint16_t i; + + /* for packet size based scheduler, nb_slaves have to >= 2 */ + if (sched_ctx->nb_slaves < NB_PKT_SIZE_SLAVES) { + CS_LOG_ERR("not enough slaves to start"); + return -1; + } + + for (i = 0; i < dev->data->nb_queue_pairs; i++) { + struct scheduler_qp_ctx *qp_ctx = dev->data->queue_pairs[i]; + struct psd_scheduler_qp_ctx *ps_qp_ctx = + qp_ctx->private_qp_ctx; + + ps_qp_ctx->primary_slave.dev_id = + sched_ctx->slaves[PRIMARY_SLAVE_IDX].dev_id; + ps_qp_ctx->primary_slave.qp_id = i; + ps_qp_ctx->primary_slave.nb_inflight_cops = 0; + + ps_qp_ctx->secondary_slave.dev_id = + sched_ctx->slaves[SECONDARY_SLAVE_IDX].dev_id; + ps_qp_ctx->secondary_slave.qp_id = i; + ps_qp_ctx->secondary_slave.nb_inflight_cops = 0; + + ps_qp_ctx->threshold = psd_ctx->threshold; + } + + if (sched_ctx->reordering_enabled) { + dev->enqueue_burst = &schedule_enqueue_ordering; + dev->dequeue_burst = &schedule_dequeue_ordering; + } else { + dev->enqueue_burst = &schedule_enqueue; + dev->dequeue_burst = &schedule_dequeue; + } + + return 0; +} + +static int +scheduler_stop(struct rte_cryptodev *dev) +{ + uint16_t i; + + for (i = 0; i < dev->data->nb_queue_pairs; i++) { + struct scheduler_qp_ctx *qp_ctx = dev->data->queue_pairs[i]; + struct psd_scheduler_qp_ctx *ps_qp_ctx = qp_ctx->private_qp_ctx; + + if (ps_qp_ctx->primary_slave.nb_inflight_cops + + ps_qp_ctx->secondary_slave.nb_inflight_cops) { + CS_LOG_ERR("Some crypto ops left in slave queue"); + return -1; + } + } + + return 0; +} + +static int +scheduler_config_qp(struct rte_cryptodev *dev, uint16_t qp_id) +{ + struct scheduler_qp_ctx *qp_ctx = dev->data->queue_pairs[qp_id]; + struct psd_scheduler_qp_ctx *ps_qp_ctx; + + ps_qp_ctx = rte_zmalloc_socket(NULL, sizeof(*ps_qp_ctx), 0, + rte_socket_id()); + if (!ps_qp_ctx) { + CS_LOG_ERR("failed allocate memory for private queue pair"); + return -ENOMEM; + } + + qp_ctx->private_qp_ctx = (void *)ps_qp_ctx; + + return 0; +} + +static int +scheduler_create_private_ctx(struct rte_cryptodev *dev) +{ + struct scheduler_ctx *sched_ctx = dev->data->dev_private; + struct psd_scheduler_ctx *psd_ctx; + + if (sched_ctx->private_ctx) + rte_free(sched_ctx->private_ctx); + + psd_ctx = rte_zmalloc_socket(NULL, sizeof(struct psd_scheduler_ctx), 0, + rte_socket_id()); + if (!psd_ctx) { + CS_LOG_ERR("failed allocate memory"); + return -ENOMEM; + } + + psd_ctx->threshold = DEF_PKT_SIZE_THRESHOLD; + + sched_ctx->private_ctx = (void *)psd_ctx; + + return 0; +} +static int +scheduler_option_set(struct rte_cryptodev *dev, uint32_t option_type, + void *option) +{ + struct psd_scheduler_ctx *psd_ctx = ((struct scheduler_ctx *) + dev->data->dev_private)->private_ctx; + uint32_t threshold; + + if ((enum rte_cryptodev_schedule_option_type)option_type != + CDEV_SCHED_OPTION_THRESHOLD) { + CS_LOG_ERR("Option not supported"); + return -EINVAL; + } + + threshold = ((struct rte_cryptodev_scheduler_threshold_option *) + option)->threshold; + if (!rte_is_power_of_2(threshold)) { + CS_LOG_ERR("Threshold is not power of 2"); + return -EINVAL; + } + + psd_ctx->threshold = ~(threshold - 1); + + return 0; +} + +static int +scheduler_option_get(struct rte_cryptodev *dev, uint32_t option_type, + void *option) +{ + struct psd_scheduler_ctx *psd_ctx = ((struct scheduler_ctx *) + dev->data->dev_private)->private_ctx; + struct rte_cryptodev_scheduler_threshold_option *threshold_option; + + if ((enum rte_cryptodev_schedule_option_type)option_type != + CDEV_SCHED_OPTION_THRESHOLD) { + CS_LOG_ERR("Option not supported"); + return -EINVAL; + } + + threshold_option = option; + threshold_option->threshold = (~psd_ctx->threshold) + 1; + + return 0; +} + +struct rte_cryptodev_scheduler_ops scheduler_ps_ops = { + slave_attach, + slave_detach, + scheduler_start, + scheduler_stop, + scheduler_config_qp, + scheduler_create_private_ctx, + scheduler_option_set, + scheduler_option_get +}; + +struct rte_cryptodev_scheduler psd_scheduler = { + .name = "packet-size-based-scheduler", + .description = "scheduler which will distribute crypto op " + "burst based on the packet size", + .mode = CDEV_SCHED_MODE_PKT_SIZE_DISTR, + .ops = &scheduler_ps_ops +}; + +struct rte_cryptodev_scheduler *pkt_size_based_distr_scheduler = &psd_scheduler; diff --git a/dpdk/drivers/crypto/scheduler/scheduler_pmd.c b/dpdk/drivers/crypto/scheduler/scheduler_pmd.c new file mode 100644 index 00000000..acdf6361 --- /dev/null +++ b/dpdk/drivers/crypto/scheduler/scheduler_pmd.c @@ -0,0 +1,500 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "rte_cryptodev_scheduler.h" +#include "scheduler_pmd_private.h" + +uint8_t cryptodev_driver_id; + +struct scheduler_init_params { + struct rte_cryptodev_pmd_init_params def_p; + uint32_t nb_slaves; + enum rte_cryptodev_scheduler_mode mode; + uint32_t enable_ordering; + uint64_t wcmask; + char slave_names[RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES] + [RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN]; +}; + +#define RTE_CRYPTODEV_VDEV_NAME ("name") +#define RTE_CRYPTODEV_VDEV_SLAVE ("slave") +#define RTE_CRYPTODEV_VDEV_MODE ("mode") +#define RTE_CRYPTODEV_VDEV_ORDERING ("ordering") +#define RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG ("max_nb_queue_pairs") +#define RTE_CRYPTODEV_VDEV_MAX_NB_SESS_ARG ("max_nb_sessions") +#define RTE_CRYPTODEV_VDEV_SOCKET_ID ("socket_id") +#define RTE_CRYPTODEV_VDEV_COREMASK ("coremask") +#define RTE_CRYPTODEV_VDEV_CORELIST ("corelist") + +const char *scheduler_valid_params[] = { + RTE_CRYPTODEV_VDEV_NAME, + RTE_CRYPTODEV_VDEV_SLAVE, + RTE_CRYPTODEV_VDEV_MODE, + RTE_CRYPTODEV_VDEV_ORDERING, + RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG, + RTE_CRYPTODEV_VDEV_MAX_NB_SESS_ARG, + RTE_CRYPTODEV_VDEV_SOCKET_ID, + RTE_CRYPTODEV_VDEV_COREMASK, + RTE_CRYPTODEV_VDEV_CORELIST +}; + +struct scheduler_parse_map { + const char *name; + uint32_t val; +}; + +const struct scheduler_parse_map scheduler_mode_map[] = { + {RTE_STR(SCHEDULER_MODE_NAME_ROUND_ROBIN), + CDEV_SCHED_MODE_ROUNDROBIN}, + {RTE_STR(SCHEDULER_MODE_NAME_PKT_SIZE_DISTR), + CDEV_SCHED_MODE_PKT_SIZE_DISTR}, + {RTE_STR(SCHEDULER_MODE_NAME_FAIL_OVER), + CDEV_SCHED_MODE_FAILOVER}, + {RTE_STR(SCHEDULER_MODE_NAME_MULTI_CORE), + CDEV_SCHED_MODE_MULTICORE} +}; + +const struct scheduler_parse_map scheduler_ordering_map[] = { + {"enable", 1}, + {"disable", 0} +}; + +static int +cryptodev_scheduler_create(const char *name, + struct rte_vdev_device *vdev, + struct scheduler_init_params *init_params) +{ + struct rte_cryptodev *dev; + struct scheduler_ctx *sched_ctx; + uint32_t i; + int ret; + + dev = rte_cryptodev_pmd_create(name, &vdev->device, + &init_params->def_p); + if (dev == NULL) { + CS_LOG_ERR("driver %s: failed to create cryptodev vdev", + name); + return -EFAULT; + } + + if (init_params->wcmask != 0) + RTE_LOG(INFO, PMD, " workers core mask = %"PRIx64"\n", + init_params->wcmask); + + dev->driver_id = cryptodev_driver_id; + dev->dev_ops = rte_crypto_scheduler_pmd_ops; + + sched_ctx = dev->data->dev_private; + sched_ctx->max_nb_queue_pairs = + init_params->def_p.max_nb_queue_pairs; + + if (init_params->mode == CDEV_SCHED_MODE_MULTICORE) { + uint16_t i; + + sched_ctx->nb_wc = 0; + + for (i = 0; i < RTE_CRYPTODEV_SCHEDULER_MAX_NB_WORKER_CORES; i++) { + if (init_params->wcmask & (1ULL << i)) { + sched_ctx->wc_pool[sched_ctx->nb_wc++] = i; + RTE_LOG(INFO, PMD, + " Worker core[%u]=%u added\n", + sched_ctx->nb_wc-1, i); + } + } + } + + if (init_params->mode > CDEV_SCHED_MODE_USERDEFINED && + init_params->mode < CDEV_SCHED_MODE_COUNT) { + ret = rte_cryptodev_scheduler_mode_set(dev->data->dev_id, + init_params->mode); + if (ret < 0) { + rte_cryptodev_pmd_release_device(dev); + return ret; + } + + for (i = 0; i < RTE_DIM(scheduler_mode_map); i++) { + if (scheduler_mode_map[i].val != sched_ctx->mode) + continue; + + RTE_LOG(INFO, PMD, " Scheduling mode = %s\n", + scheduler_mode_map[i].name); + break; + } + } + + sched_ctx->reordering_enabled = init_params->enable_ordering; + + for (i = 0; i < RTE_DIM(scheduler_ordering_map); i++) { + if (scheduler_ordering_map[i].val != + sched_ctx->reordering_enabled) + continue; + + RTE_LOG(INFO, PMD, " Packet ordering = %s\n", + scheduler_ordering_map[i].name); + + break; + } + + for (i = 0; i < init_params->nb_slaves; i++) { + sched_ctx->init_slave_names[sched_ctx->nb_init_slaves] = + rte_zmalloc_socket( + NULL, + RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN, 0, + SOCKET_ID_ANY); + + if (!sched_ctx->init_slave_names[ + sched_ctx->nb_init_slaves]) { + CS_LOG_ERR("driver %s: Insufficient memory", + name); + return -ENOMEM; + } + + strncpy(sched_ctx->init_slave_names[ + sched_ctx->nb_init_slaves], + init_params->slave_names[i], + RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN - 1); + + sched_ctx->nb_init_slaves++; + } + + /* + * Initialize capabilities structure as an empty structure, + * in case device information is requested when no slaves are attached + */ + sched_ctx->capabilities = rte_zmalloc_socket(NULL, + sizeof(struct rte_cryptodev_capabilities), + 0, SOCKET_ID_ANY); + + if (!sched_ctx->capabilities) { + RTE_LOG(ERR, PMD, "Not enough memory for capability " + "information\n"); + return -ENOMEM; + } + + return 0; +} + +static int +cryptodev_scheduler_remove(struct rte_vdev_device *vdev) +{ + const char *name; + struct rte_cryptodev *dev; + struct scheduler_ctx *sched_ctx; + + if (vdev == NULL) + return -EINVAL; + + name = rte_vdev_device_name(vdev); + dev = rte_cryptodev_pmd_get_named_dev(name); + if (dev == NULL) + return -EINVAL; + + sched_ctx = dev->data->dev_private; + + if (sched_ctx->nb_slaves) { + uint32_t i; + + for (i = 0; i < sched_ctx->nb_slaves; i++) + rte_cryptodev_scheduler_slave_detach(dev->data->dev_id, + sched_ctx->slaves[i].dev_id); + } + + return rte_cryptodev_pmd_destroy(dev); +} + +/** Parse integer from integer argument */ +static int +parse_integer_arg(const char *key __rte_unused, + const char *value, void *extra_args) +{ + int *i = (int *) extra_args; + + *i = atoi(value); + if (*i < 0) { + CS_LOG_ERR("Argument has to be positive.\n"); + return -EINVAL; + } + + return 0; +} + +/** Parse integer from hexadecimal integer argument */ +static int +parse_coremask_arg(const char *key __rte_unused, + const char *value, void *extra_args) +{ + struct scheduler_init_params *params = extra_args; + + params->wcmask = strtoull(value, NULL, 16); + + return 0; +} + +/** Parse integer from list of integers argument */ +static int +parse_corelist_arg(const char *key __rte_unused, + const char *value, void *extra_args) +{ + struct scheduler_init_params *params = extra_args; + + params->wcmask = 0ULL; + + const char *token = value; + + while (isdigit(token[0])) { + char *rval; + unsigned int core = strtoul(token, &rval, 10); + + params->wcmask |= 1ULL << core; + token = (const char *)rval; + if (token[0] == '\0') + break; + token++; + } + + return 0; +} + +/** Parse name */ +static int +parse_name_arg(const char *key __rte_unused, + const char *value, void *extra_args) +{ + struct rte_cryptodev_pmd_init_params *params = extra_args; + + if (strlen(value) >= RTE_CRYPTODEV_NAME_MAX_LEN - 1) { + CS_LOG_ERR("Invalid name %s, should be less than " + "%u bytes.\n", value, + RTE_CRYPTODEV_NAME_MAX_LEN - 1); + return -EINVAL; + } + + strncpy(params->name, value, RTE_CRYPTODEV_NAME_MAX_LEN); + + return 0; +} + +/** Parse slave */ +static int +parse_slave_arg(const char *key __rte_unused, + const char *value, void *extra_args) +{ + struct scheduler_init_params *param = extra_args; + + if (param->nb_slaves >= RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES) { + CS_LOG_ERR("Too many slaves.\n"); + return -ENOMEM; + } + + strncpy(param->slave_names[param->nb_slaves++], value, + RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN - 1); + + return 0; +} + +static int +parse_mode_arg(const char *key __rte_unused, + const char *value, void *extra_args) +{ + struct scheduler_init_params *param = extra_args; + uint32_t i; + + for (i = 0; i < RTE_DIM(scheduler_mode_map); i++) { + if (strcmp(value, scheduler_mode_map[i].name) == 0) { + param->mode = (enum rte_cryptodev_scheduler_mode) + scheduler_mode_map[i].val; + break; + } + } + + if (i == RTE_DIM(scheduler_mode_map)) { + CS_LOG_ERR("Unrecognized input.\n"); + return -EINVAL; + } + + return 0; +} + +static int +parse_ordering_arg(const char *key __rte_unused, + const char *value, void *extra_args) +{ + struct scheduler_init_params *param = extra_args; + uint32_t i; + + for (i = 0; i < RTE_DIM(scheduler_ordering_map); i++) { + if (strcmp(value, scheduler_ordering_map[i].name) == 0) { + param->enable_ordering = + scheduler_ordering_map[i].val; + break; + } + } + + if (i == RTE_DIM(scheduler_ordering_map)) { + CS_LOG_ERR("Unrecognized input.\n"); + return -EINVAL; + } + + return 0; +} + +static int +scheduler_parse_init_params(struct scheduler_init_params *params, + const char *input_args) +{ + struct rte_kvargs *kvlist = NULL; + int ret = 0; + + if (params == NULL) + return -EINVAL; + + if (input_args) { + kvlist = rte_kvargs_parse(input_args, + scheduler_valid_params); + if (kvlist == NULL) + return -1; + + ret = rte_kvargs_process(kvlist, + RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG, + &parse_integer_arg, + ¶ms->def_p.max_nb_queue_pairs); + if (ret < 0) + goto free_kvlist; + + ret = rte_kvargs_process(kvlist, + RTE_CRYPTODEV_VDEV_MAX_NB_SESS_ARG, + &parse_integer_arg, + ¶ms->def_p.max_nb_sessions); + if (ret < 0) + goto free_kvlist; + + ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_SOCKET_ID, + &parse_integer_arg, + ¶ms->def_p.socket_id); + if (ret < 0) + goto free_kvlist; + + ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_COREMASK, + &parse_coremask_arg, + params); + if (ret < 0) + goto free_kvlist; + + ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_CORELIST, + &parse_corelist_arg, + params); + if (ret < 0) + goto free_kvlist; + + ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_NAME, + &parse_name_arg, + ¶ms->def_p); + if (ret < 0) + goto free_kvlist; + + ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_SLAVE, + &parse_slave_arg, params); + if (ret < 0) + goto free_kvlist; + + ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_MODE, + &parse_mode_arg, params); + if (ret < 0) + goto free_kvlist; + + ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_ORDERING, + &parse_ordering_arg, params); + if (ret < 0) + goto free_kvlist; + } + +free_kvlist: + rte_kvargs_free(kvlist); + return ret; +} + +static int +cryptodev_scheduler_probe(struct rte_vdev_device *vdev) +{ + struct scheduler_init_params init_params = { + .def_p = { + "", + sizeof(struct scheduler_ctx), + rte_socket_id(), + RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS, + RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_SESSIONS + }, + .nb_slaves = 0, + .mode = CDEV_SCHED_MODE_NOT_SET, + .enable_ordering = 0, + .slave_names = { {0} } + }; + const char *name; + + name = rte_vdev_device_name(vdev); + if (name == NULL) + return -EINVAL; + + scheduler_parse_init_params(&init_params, + rte_vdev_device_args(vdev)); + + + return cryptodev_scheduler_create(name, + vdev, + &init_params); +} + +static struct rte_vdev_driver cryptodev_scheduler_pmd_drv = { + .probe = cryptodev_scheduler_probe, + .remove = cryptodev_scheduler_remove +}; + +static struct cryptodev_driver scheduler_crypto_drv; + +RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_SCHEDULER_PMD, + cryptodev_scheduler_pmd_drv); +RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_SCHEDULER_PMD, + "max_nb_queue_pairs= " + "max_nb_sessions= " + "socket_id= " + "slave="); +RTE_PMD_REGISTER_CRYPTO_DRIVER(scheduler_crypto_drv, + cryptodev_scheduler_pmd_drv, + cryptodev_driver_id); diff --git a/dpdk/drivers/crypto/scheduler/scheduler_pmd_ops.c b/dpdk/drivers/crypto/scheduler/scheduler_pmd_ops.c new file mode 100644 index 00000000..d9b52352 --- /dev/null +++ b/dpdk/drivers/crypto/scheduler/scheduler_pmd_ops.c @@ -0,0 +1,570 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include + +#include +#include +#include +#include +#include +#include + +#include "scheduler_pmd_private.h" + +/** attaching the slaves predefined by scheduler's EAL options */ +static int +scheduler_attach_init_slave(struct rte_cryptodev *dev) +{ + struct scheduler_ctx *sched_ctx = dev->data->dev_private; + uint8_t scheduler_id = dev->data->dev_id; + int i; + + for (i = sched_ctx->nb_init_slaves - 1; i >= 0; i--) { + const char *dev_name = sched_ctx->init_slave_names[i]; + struct rte_cryptodev *slave_dev = + rte_cryptodev_pmd_get_named_dev(dev_name); + int status; + + if (!slave_dev) { + CS_LOG_ERR("Failed to locate slave dev %s", + dev_name); + return -EINVAL; + } + + status = rte_cryptodev_scheduler_slave_attach( + scheduler_id, slave_dev->data->dev_id); + + if (status < 0) { + CS_LOG_ERR("Failed to attach slave cryptodev %u", + slave_dev->data->dev_id); + return status; + } + + CS_LOG_INFO("Scheduler %s attached slave %s\n", + dev->data->name, + sched_ctx->init_slave_names[i]); + + rte_free(sched_ctx->init_slave_names[i]); + + sched_ctx->nb_init_slaves -= 1; + } + + return 0; +} +/** Configure device */ +static int +scheduler_pmd_config(struct rte_cryptodev *dev, + struct rte_cryptodev_config *config) +{ + struct scheduler_ctx *sched_ctx = dev->data->dev_private; + uint32_t i; + int ret; + + /* although scheduler_attach_init_slave presents multiple times, + * there will be only 1 meaningful execution. + */ + ret = scheduler_attach_init_slave(dev); + if (ret < 0) + return ret; + + for (i = 0; i < sched_ctx->nb_slaves; i++) { + uint8_t slave_dev_id = sched_ctx->slaves[i].dev_id; + + ret = rte_cryptodev_configure(slave_dev_id, config); + if (ret < 0) + break; + } + + return ret; +} + +static int +update_order_ring(struct rte_cryptodev *dev, uint16_t qp_id) +{ + struct scheduler_ctx *sched_ctx = dev->data->dev_private; + struct scheduler_qp_ctx *qp_ctx = dev->data->queue_pairs[qp_id]; + + if (sched_ctx->reordering_enabled) { + char order_ring_name[RTE_CRYPTODEV_NAME_MAX_LEN]; + uint32_t buff_size = rte_align32pow2( + sched_ctx->nb_slaves * PER_SLAVE_BUFF_SIZE); + + if (qp_ctx->order_ring) { + rte_ring_free(qp_ctx->order_ring); + qp_ctx->order_ring = NULL; + } + + if (!buff_size) + return 0; + + if (snprintf(order_ring_name, RTE_CRYPTODEV_NAME_MAX_LEN, + "%s_rb_%u_%u", RTE_STR(CRYPTODEV_NAME_SCHEDULER_PMD), + dev->data->dev_id, qp_id) < 0) { + CS_LOG_ERR("failed to create unique reorder buffer " + "name"); + return -ENOMEM; + } + + qp_ctx->order_ring = rte_ring_create(order_ring_name, + buff_size, rte_socket_id(), + RING_F_SP_ENQ | RING_F_SC_DEQ); + if (!qp_ctx->order_ring) { + CS_LOG_ERR("failed to create order ring"); + return -ENOMEM; + } + } else { + if (qp_ctx->order_ring) { + rte_ring_free(qp_ctx->order_ring); + qp_ctx->order_ring = NULL; + } + } + + return 0; +} + +/** Start device */ +static int +scheduler_pmd_start(struct rte_cryptodev *dev) +{ + struct scheduler_ctx *sched_ctx = dev->data->dev_private; + uint32_t i; + int ret; + + if (dev->data->dev_started) + return 0; + + /* although scheduler_attach_init_slave presents multiple times, + * there will be only 1 meaningful execution. + */ + ret = scheduler_attach_init_slave(dev); + if (ret < 0) + return ret; + + for (i = 0; i < dev->data->nb_queue_pairs; i++) { + ret = update_order_ring(dev, i); + if (ret < 0) { + CS_LOG_ERR("Failed to update reorder buffer"); + return ret; + } + } + + if (sched_ctx->mode == CDEV_SCHED_MODE_NOT_SET) { + CS_LOG_ERR("Scheduler mode is not set"); + return -1; + } + + if (!sched_ctx->nb_slaves) { + CS_LOG_ERR("No slave in the scheduler"); + return -1; + } + + RTE_FUNC_PTR_OR_ERR_RET(*sched_ctx->ops.slave_attach, -ENOTSUP); + + for (i = 0; i < sched_ctx->nb_slaves; i++) { + uint8_t slave_dev_id = sched_ctx->slaves[i].dev_id; + + if ((*sched_ctx->ops.slave_attach)(dev, slave_dev_id) < 0) { + CS_LOG_ERR("Failed to attach slave"); + return -ENOTSUP; + } + } + + RTE_FUNC_PTR_OR_ERR_RET(*sched_ctx->ops.scheduler_start, -ENOTSUP); + + if ((*sched_ctx->ops.scheduler_start)(dev) < 0) { + CS_LOG_ERR("Scheduler start failed"); + return -1; + } + + /* start all slaves */ + for (i = 0; i < sched_ctx->nb_slaves; i++) { + uint8_t slave_dev_id = sched_ctx->slaves[i].dev_id; + struct rte_cryptodev *slave_dev = + rte_cryptodev_pmd_get_dev(slave_dev_id); + + ret = (*slave_dev->dev_ops->dev_start)(slave_dev); + if (ret < 0) { + CS_LOG_ERR("Failed to start slave dev %u", + slave_dev_id); + return ret; + } + } + + return 0; +} + +/** Stop device */ +static void +scheduler_pmd_stop(struct rte_cryptodev *dev) +{ + struct scheduler_ctx *sched_ctx = dev->data->dev_private; + uint32_t i; + + if (!dev->data->dev_started) + return; + + /* stop all slaves first */ + for (i = 0; i < sched_ctx->nb_slaves; i++) { + uint8_t slave_dev_id = sched_ctx->slaves[i].dev_id; + struct rte_cryptodev *slave_dev = + rte_cryptodev_pmd_get_dev(slave_dev_id); + + (*slave_dev->dev_ops->dev_stop)(slave_dev); + } + + if (*sched_ctx->ops.scheduler_stop) + (*sched_ctx->ops.scheduler_stop)(dev); + + for (i = 0; i < sched_ctx->nb_slaves; i++) { + uint8_t slave_dev_id = sched_ctx->slaves[i].dev_id; + + if (*sched_ctx->ops.slave_detach) + (*sched_ctx->ops.slave_detach)(dev, slave_dev_id); + } +} + +/** Close device */ +static int +scheduler_pmd_close(struct rte_cryptodev *dev) +{ + struct scheduler_ctx *sched_ctx = dev->data->dev_private; + uint32_t i; + int ret; + + /* the dev should be stopped before being closed */ + if (dev->data->dev_started) + return -EBUSY; + + /* close all slaves first */ + for (i = 0; i < sched_ctx->nb_slaves; i++) { + uint8_t slave_dev_id = sched_ctx->slaves[i].dev_id; + struct rte_cryptodev *slave_dev = + rte_cryptodev_pmd_get_dev(slave_dev_id); + + ret = (*slave_dev->dev_ops->dev_close)(slave_dev); + if (ret < 0) + return ret; + } + + for (i = 0; i < dev->data->nb_queue_pairs; i++) { + struct scheduler_qp_ctx *qp_ctx = dev->data->queue_pairs[i]; + + if (qp_ctx->order_ring) { + rte_ring_free(qp_ctx->order_ring); + qp_ctx->order_ring = NULL; + } + + if (qp_ctx->private_qp_ctx) { + rte_free(qp_ctx->private_qp_ctx); + qp_ctx->private_qp_ctx = NULL; + } + } + + if (sched_ctx->private_ctx) + rte_free(sched_ctx->private_ctx); + + if (sched_ctx->capabilities) + rte_free(sched_ctx->capabilities); + + return 0; +} + +/** Get device statistics */ +static void +scheduler_pmd_stats_get(struct rte_cryptodev *dev, + struct rte_cryptodev_stats *stats) +{ + struct scheduler_ctx *sched_ctx = dev->data->dev_private; + uint32_t i; + + for (i = 0; i < sched_ctx->nb_slaves; i++) { + uint8_t slave_dev_id = sched_ctx->slaves[i].dev_id; + struct rte_cryptodev *slave_dev = + rte_cryptodev_pmd_get_dev(slave_dev_id); + struct rte_cryptodev_stats slave_stats = {0}; + + (*slave_dev->dev_ops->stats_get)(slave_dev, &slave_stats); + + stats->enqueued_count += slave_stats.enqueued_count; + stats->dequeued_count += slave_stats.dequeued_count; + + stats->enqueue_err_count += slave_stats.enqueue_err_count; + stats->dequeue_err_count += slave_stats.dequeue_err_count; + } +} + +/** Reset device statistics */ +static void +scheduler_pmd_stats_reset(struct rte_cryptodev *dev) +{ + struct scheduler_ctx *sched_ctx = dev->data->dev_private; + uint32_t i; + + for (i = 0; i < sched_ctx->nb_slaves; i++) { + uint8_t slave_dev_id = sched_ctx->slaves[i].dev_id; + struct rte_cryptodev *slave_dev = + rte_cryptodev_pmd_get_dev(slave_dev_id); + + (*slave_dev->dev_ops->stats_reset)(slave_dev); + } +} + +/** Get device info */ +static void +scheduler_pmd_info_get(struct rte_cryptodev *dev, + struct rte_cryptodev_info *dev_info) +{ + struct scheduler_ctx *sched_ctx = dev->data->dev_private; + uint32_t max_nb_sessions = sched_ctx->nb_slaves ? + UINT32_MAX : RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_SESSIONS; + uint32_t i; + + if (!dev_info) + return; + + /* although scheduler_attach_init_slave presents multiple times, + * there will be only 1 meaningful execution. + */ + scheduler_attach_init_slave(dev); + + for (i = 0; i < sched_ctx->nb_slaves; i++) { + uint8_t slave_dev_id = sched_ctx->slaves[i].dev_id; + struct rte_cryptodev_info slave_info; + + rte_cryptodev_info_get(slave_dev_id, &slave_info); + max_nb_sessions = slave_info.sym.max_nb_sessions < + max_nb_sessions ? + slave_info.sym.max_nb_sessions : + max_nb_sessions; + } + + dev_info->driver_id = dev->driver_id; + dev_info->feature_flags = dev->feature_flags; + dev_info->capabilities = sched_ctx->capabilities; + dev_info->max_nb_queue_pairs = sched_ctx->max_nb_queue_pairs; + dev_info->sym.max_nb_sessions = max_nb_sessions; +} + +/** Release queue pair */ +static int +scheduler_pmd_qp_release(struct rte_cryptodev *dev, uint16_t qp_id) +{ + struct scheduler_qp_ctx *qp_ctx = dev->data->queue_pairs[qp_id]; + + if (!qp_ctx) + return 0; + + if (qp_ctx->order_ring) + rte_ring_free(qp_ctx->order_ring); + if (qp_ctx->private_qp_ctx) + rte_free(qp_ctx->private_qp_ctx); + + rte_free(qp_ctx); + dev->data->queue_pairs[qp_id] = NULL; + + return 0; +} + +/** Setup a queue pair */ +static int +scheduler_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, + const struct rte_cryptodev_qp_conf *qp_conf, int socket_id, + struct rte_mempool *session_pool) +{ + struct scheduler_ctx *sched_ctx = dev->data->dev_private; + struct scheduler_qp_ctx *qp_ctx; + char name[RTE_CRYPTODEV_NAME_MAX_LEN]; + uint32_t i; + int ret; + + if (snprintf(name, RTE_CRYPTODEV_NAME_MAX_LEN, + "CRYTO_SCHE PMD %u QP %u", + dev->data->dev_id, qp_id) < 0) { + CS_LOG_ERR("Failed to create unique queue pair name"); + return -EFAULT; + } + + /* Free memory prior to re-allocation if needed. */ + if (dev->data->queue_pairs[qp_id] != NULL) + scheduler_pmd_qp_release(dev, qp_id); + + for (i = 0; i < sched_ctx->nb_slaves; i++) { + uint8_t slave_id = sched_ctx->slaves[i].dev_id; + + /* + * All slaves will share the same session mempool + * for session-less operations, so the objects + * must be big enough for all the drivers used. + */ + ret = rte_cryptodev_queue_pair_setup(slave_id, qp_id, + qp_conf, socket_id, session_pool); + if (ret < 0) + return ret; + } + + /* Allocate the queue pair data structure. */ + qp_ctx = rte_zmalloc_socket(name, sizeof(*qp_ctx), RTE_CACHE_LINE_SIZE, + socket_id); + if (qp_ctx == NULL) + return -ENOMEM; + + /* The actual available object number = nb_descriptors - 1 */ + qp_ctx->max_nb_objs = qp_conf->nb_descriptors - 1; + + dev->data->queue_pairs[qp_id] = qp_ctx; + + /* although scheduler_attach_init_slave presents multiple times, + * there will be only 1 meaningful execution. + */ + ret = scheduler_attach_init_slave(dev); + if (ret < 0) { + CS_LOG_ERR("Failed to attach slave"); + scheduler_pmd_qp_release(dev, qp_id); + return ret; + } + + if (*sched_ctx->ops.config_queue_pair) { + if ((*sched_ctx->ops.config_queue_pair)(dev, qp_id) < 0) { + CS_LOG_ERR("Unable to configure queue pair"); + return -1; + } + } + + return 0; +} + +/** Start queue pair */ +static int +scheduler_pmd_qp_start(__rte_unused struct rte_cryptodev *dev, + __rte_unused uint16_t queue_pair_id) +{ + return -ENOTSUP; +} + +/** Stop queue pair */ +static int +scheduler_pmd_qp_stop(__rte_unused struct rte_cryptodev *dev, + __rte_unused uint16_t queue_pair_id) +{ + return -ENOTSUP; +} + +/** Return the number of allocated queue pairs */ +static uint32_t +scheduler_pmd_qp_count(struct rte_cryptodev *dev) +{ + return dev->data->nb_queue_pairs; +} + +static uint32_t +scheduler_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused) +{ + struct scheduler_ctx *sched_ctx = dev->data->dev_private; + uint8_t i = 0; + uint32_t max_priv_sess_size = 0; + + /* Check what is the maximum private session size for all slaves */ + for (i = 0; i < sched_ctx->nb_slaves; i++) { + uint8_t slave_dev_id = sched_ctx->slaves[i].dev_id; + struct rte_cryptodev *dev = &rte_cryptodevs[slave_dev_id]; + uint32_t priv_sess_size = (*dev->dev_ops->session_get_size)(dev); + + if (max_priv_sess_size < priv_sess_size) + max_priv_sess_size = priv_sess_size; + } + + return max_priv_sess_size; +} + +static int +scheduler_pmd_session_configure(struct rte_cryptodev *dev, + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool) +{ + struct scheduler_ctx *sched_ctx = dev->data->dev_private; + uint32_t i; + int ret; + + for (i = 0; i < sched_ctx->nb_slaves; i++) { + struct scheduler_slave *slave = &sched_ctx->slaves[i]; + + ret = rte_cryptodev_sym_session_init(slave->dev_id, sess, + xform, mempool); + if (ret < 0) { + CS_LOG_ERR("unabled to config sym session"); + return ret; + } + } + + return 0; +} + +/** Clear the memory of session so it doesn't leave key material behind */ +static void +scheduler_pmd_session_clear(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) +{ + struct scheduler_ctx *sched_ctx = dev->data->dev_private; + uint32_t i; + + /* Clear private data of slaves */ + for (i = 0; i < sched_ctx->nb_slaves; i++) { + struct scheduler_slave *slave = &sched_ctx->slaves[i]; + + rte_cryptodev_sym_session_clear(slave->dev_id, sess); + } +} + +struct rte_cryptodev_ops scheduler_pmd_ops = { + .dev_configure = scheduler_pmd_config, + .dev_start = scheduler_pmd_start, + .dev_stop = scheduler_pmd_stop, + .dev_close = scheduler_pmd_close, + + .stats_get = scheduler_pmd_stats_get, + .stats_reset = scheduler_pmd_stats_reset, + + .dev_infos_get = scheduler_pmd_info_get, + + .queue_pair_setup = scheduler_pmd_qp_setup, + .queue_pair_release = scheduler_pmd_qp_release, + .queue_pair_start = scheduler_pmd_qp_start, + .queue_pair_stop = scheduler_pmd_qp_stop, + .queue_pair_count = scheduler_pmd_qp_count, + + .session_get_size = scheduler_pmd_session_get_size, + .session_configure = scheduler_pmd_session_configure, + .session_clear = scheduler_pmd_session_clear, +}; + +struct rte_cryptodev_ops *rte_crypto_scheduler_pmd_ops = &scheduler_pmd_ops; diff --git a/dpdk/drivers/crypto/scheduler/scheduler_pmd_private.h b/dpdk/drivers/crypto/scheduler/scheduler_pmd_private.h new file mode 100644 index 00000000..e606716a --- /dev/null +++ b/dpdk/drivers/crypto/scheduler/scheduler_pmd_private.h @@ -0,0 +1,159 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _SCHEDULER_PMD_PRIVATE_H +#define _SCHEDULER_PMD_PRIVATE_H + +#include "rte_cryptodev_scheduler.h" + +#define CRYPTODEV_NAME_SCHEDULER_PMD crypto_scheduler +/**< Scheduler Crypto PMD device name */ + +#define PER_SLAVE_BUFF_SIZE (256) + +#define CS_LOG_ERR(fmt, args...) \ + RTE_LOG(ERR, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \ + RTE_STR(CRYPTODEV_NAME_SCHEDULER_PMD), \ + __func__, __LINE__, ## args) + +#ifdef RTE_LIBRTE_CRYPTO_SCHEDULER_DEBUG +#define CS_LOG_INFO(fmt, args...) \ + RTE_LOG(INFO, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \ + RTE_STR(CRYPTODEV_NAME_SCHEDULER_PMD), \ + __func__, __LINE__, ## args) + +#define CS_LOG_DBG(fmt, args...) \ + RTE_LOG(DEBUG, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \ + RTE_STR(CRYPTODEV_NAME_SCHEDULER_PMD), \ + __func__, __LINE__, ## args) +#else +#define CS_LOG_INFO(fmt, args...) +#define CS_LOG_DBG(fmt, args...) +#endif + +struct scheduler_slave { + uint8_t dev_id; + uint16_t qp_id; + uint32_t nb_inflight_cops; + + uint8_t driver_id; +}; + +struct scheduler_ctx { + void *private_ctx; + /**< private scheduler context pointer */ + + struct rte_cryptodev_capabilities *capabilities; + uint32_t nb_capabilities; + + uint32_t max_nb_queue_pairs; + + struct scheduler_slave slaves[RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES]; + uint32_t nb_slaves; + + enum rte_cryptodev_scheduler_mode mode; + + struct rte_cryptodev_scheduler_ops ops; + + uint8_t reordering_enabled; + + char name[RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN]; + char description[RTE_CRYPTODEV_SCHEDULER_DESC_MAX_LEN]; + uint16_t wc_pool[RTE_CRYPTODEV_SCHEDULER_MAX_NB_WORKER_CORES]; + uint16_t nb_wc; + + char *init_slave_names[RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES]; + int nb_init_slaves; +} __rte_cache_aligned; + +struct scheduler_qp_ctx { + void *private_qp_ctx; + + uint32_t max_nb_objs; + + struct rte_ring *order_ring; + uint32_t seqn; +} __rte_cache_aligned; + + +extern uint8_t cryptodev_driver_id; + +static __rte_always_inline uint16_t +get_max_enqueue_order_count(struct rte_ring *order_ring, uint16_t nb_ops) +{ + uint32_t count = rte_ring_free_count(order_ring); + + return count > nb_ops ? nb_ops : count; +} + +static __rte_always_inline void +scheduler_order_insert(struct rte_ring *order_ring, + struct rte_crypto_op **ops, uint16_t nb_ops) +{ + rte_ring_sp_enqueue_burst(order_ring, (void **)ops, nb_ops, NULL); +} + +#define SCHEDULER_GET_RING_OBJ(order_ring, pos, op) do { \ + struct rte_crypto_op **ring = (void *)&order_ring[1]; \ + op = ring[(order_ring->cons.head + pos) & order_ring->mask]; \ +} while (0) + +static __rte_always_inline uint16_t +scheduler_order_drain(struct rte_ring *order_ring, + struct rte_crypto_op **ops, uint16_t nb_ops) +{ + struct rte_crypto_op *op; + uint32_t nb_objs = rte_ring_count(order_ring); + uint32_t nb_ops_to_deq = 0; + uint32_t nb_ops_deqd = 0; + + if (nb_objs > nb_ops) + nb_objs = nb_ops; + + while (nb_ops_to_deq < nb_objs) { + SCHEDULER_GET_RING_OBJ(order_ring, nb_ops_to_deq, op); + if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) + break; + nb_ops_to_deq++; + } + + if (nb_ops_to_deq) + nb_ops_deqd = rte_ring_sc_dequeue_bulk(order_ring, + (void **)ops, nb_ops_to_deq, NULL); + + return nb_ops_deqd; +} +/** device specific operations function pointer structure */ +extern struct rte_cryptodev_ops *rte_crypto_scheduler_pmd_ops; + +#endif /* _SCHEDULER_PMD_PRIVATE_H */ diff --git a/dpdk/drivers/crypto/scheduler/scheduler_roundrobin.c b/dpdk/drivers/crypto/scheduler/scheduler_roundrobin.c new file mode 100644 index 00000000..4a847281 --- /dev/null +++ b/dpdk/drivers/crypto/scheduler/scheduler_roundrobin.c @@ -0,0 +1,240 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include + +#include "rte_cryptodev_scheduler_operations.h" +#include "scheduler_pmd_private.h" + +struct rr_scheduler_qp_ctx { + struct scheduler_slave slaves[RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES]; + uint32_t nb_slaves; + + uint32_t last_enq_slave_idx; + uint32_t last_deq_slave_idx; +}; + +static uint16_t +schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) +{ + struct rr_scheduler_qp_ctx *rr_qp_ctx = + ((struct scheduler_qp_ctx *)qp)->private_qp_ctx; + uint32_t slave_idx = rr_qp_ctx->last_enq_slave_idx; + struct scheduler_slave *slave = &rr_qp_ctx->slaves[slave_idx]; + uint16_t i, processed_ops; + + if (unlikely(nb_ops == 0)) + return 0; + + for (i = 0; i < nb_ops && i < 4; i++) + rte_prefetch0(ops[i]->sym->session); + + processed_ops = rte_cryptodev_enqueue_burst(slave->dev_id, + slave->qp_id, ops, nb_ops); + + slave->nb_inflight_cops += processed_ops; + + rr_qp_ctx->last_enq_slave_idx += 1; + rr_qp_ctx->last_enq_slave_idx %= rr_qp_ctx->nb_slaves; + + return processed_ops; +} + +static uint16_t +schedule_enqueue_ordering(void *qp, struct rte_crypto_op **ops, + uint16_t nb_ops) +{ + struct rte_ring *order_ring = + ((struct scheduler_qp_ctx *)qp)->order_ring; + uint16_t nb_ops_to_enq = get_max_enqueue_order_count(order_ring, + nb_ops); + uint16_t nb_ops_enqd = schedule_enqueue(qp, ops, + nb_ops_to_enq); + + scheduler_order_insert(order_ring, ops, nb_ops_enqd); + + return nb_ops_enqd; +} + + +static uint16_t +schedule_dequeue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) +{ + struct rr_scheduler_qp_ctx *rr_qp_ctx = + ((struct scheduler_qp_ctx *)qp)->private_qp_ctx; + struct scheduler_slave *slave; + uint32_t last_slave_idx = rr_qp_ctx->last_deq_slave_idx; + uint16_t nb_deq_ops; + + if (unlikely(rr_qp_ctx->slaves[last_slave_idx].nb_inflight_cops == 0)) { + do { + last_slave_idx += 1; + + if (unlikely(last_slave_idx >= rr_qp_ctx->nb_slaves)) + last_slave_idx = 0; + /* looped back, means no inflight cops in the queue */ + if (last_slave_idx == rr_qp_ctx->last_deq_slave_idx) + return 0; + } while (rr_qp_ctx->slaves[last_slave_idx].nb_inflight_cops + == 0); + } + + slave = &rr_qp_ctx->slaves[last_slave_idx]; + + nb_deq_ops = rte_cryptodev_dequeue_burst(slave->dev_id, + slave->qp_id, ops, nb_ops); + + last_slave_idx += 1; + last_slave_idx %= rr_qp_ctx->nb_slaves; + + rr_qp_ctx->last_deq_slave_idx = last_slave_idx; + + slave->nb_inflight_cops -= nb_deq_ops; + + return nb_deq_ops; +} + +static uint16_t +schedule_dequeue_ordering(void *qp, struct rte_crypto_op **ops, + uint16_t nb_ops) +{ + struct rte_ring *order_ring = + ((struct scheduler_qp_ctx *)qp)->order_ring; + + schedule_dequeue(qp, ops, nb_ops); + + return scheduler_order_drain(order_ring, ops, nb_ops); +} + +static int +slave_attach(__rte_unused struct rte_cryptodev *dev, + __rte_unused uint8_t slave_id) +{ + return 0; +} + +static int +slave_detach(__rte_unused struct rte_cryptodev *dev, + __rte_unused uint8_t slave_id) +{ + return 0; +} + +static int +scheduler_start(struct rte_cryptodev *dev) +{ + struct scheduler_ctx *sched_ctx = dev->data->dev_private; + uint16_t i; + + if (sched_ctx->reordering_enabled) { + dev->enqueue_burst = &schedule_enqueue_ordering; + dev->dequeue_burst = &schedule_dequeue_ordering; + } else { + dev->enqueue_burst = &schedule_enqueue; + dev->dequeue_burst = &schedule_dequeue; + } + + for (i = 0; i < dev->data->nb_queue_pairs; i++) { + struct scheduler_qp_ctx *qp_ctx = dev->data->queue_pairs[i]; + struct rr_scheduler_qp_ctx *rr_qp_ctx = + qp_ctx->private_qp_ctx; + uint32_t j; + + memset(rr_qp_ctx->slaves, 0, + RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES * + sizeof(struct scheduler_slave)); + for (j = 0; j < sched_ctx->nb_slaves; j++) { + rr_qp_ctx->slaves[j].dev_id = + sched_ctx->slaves[j].dev_id; + rr_qp_ctx->slaves[j].qp_id = i; + } + + rr_qp_ctx->nb_slaves = sched_ctx->nb_slaves; + + rr_qp_ctx->last_enq_slave_idx = 0; + rr_qp_ctx->last_deq_slave_idx = 0; + } + + return 0; +} + +static int +scheduler_stop(__rte_unused struct rte_cryptodev *dev) +{ + return 0; +} + +static int +scheduler_config_qp(struct rte_cryptodev *dev, uint16_t qp_id) +{ + struct scheduler_qp_ctx *qp_ctx = dev->data->queue_pairs[qp_id]; + struct rr_scheduler_qp_ctx *rr_qp_ctx; + + rr_qp_ctx = rte_zmalloc_socket(NULL, sizeof(*rr_qp_ctx), 0, + rte_socket_id()); + if (!rr_qp_ctx) { + CS_LOG_ERR("failed allocate memory for private queue pair"); + return -ENOMEM; + } + + qp_ctx->private_qp_ctx = (void *)rr_qp_ctx; + + return 0; +} + +static int +scheduler_create_private_ctx(__rte_unused struct rte_cryptodev *dev) +{ + return 0; +} + +struct rte_cryptodev_scheduler_ops scheduler_rr_ops = { + slave_attach, + slave_detach, + scheduler_start, + scheduler_stop, + scheduler_config_qp, + scheduler_create_private_ctx, + NULL, /* option_set */ + NULL /* option_get */ +}; + +struct rte_cryptodev_scheduler scheduler = { + .name = "roundrobin-scheduler", + .description = "scheduler which will round robin burst across " + "slave crypto devices", + .mode = CDEV_SCHED_MODE_ROUNDROBIN, + .ops = &scheduler_rr_ops +}; + +struct rte_cryptodev_scheduler *roundrobin_scheduler = &scheduler; diff --git a/dpdk/drivers/crypto/snow3g/Makefile b/dpdk/drivers/crypto/snow3g/Makefile index bea6760b..183c6ce9 100644 --- a/dpdk/drivers/crypto/snow3g/Makefile +++ b/dpdk/drivers/crypto/snow3g/Makefile @@ -54,16 +54,12 @@ CFLAGS += -I$(LIBSSO_SNOW3G_PATH) CFLAGS += -I$(LIBSSO_SNOW3G_PATH)/include CFLAGS += -I$(LIBSSO_SNOW3G_PATH)/build LDLIBS += -L$(LIBSSO_SNOW3G_PATH)/build -lsso_snow3g +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_cryptodev +LDLIBS += -lrte_bus_vdev # library source files SRCS-$(CONFIG_RTE_LIBRTE_PMD_SNOW3G) += rte_snow3g_pmd.c SRCS-$(CONFIG_RTE_LIBRTE_PMD_SNOW3G) += rte_snow3g_pmd_ops.c -# library dependencies -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_SNOW3G) += lib/librte_eal -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_SNOW3G) += lib/librte_mbuf -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_SNOW3G) += lib/librte_mempool -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_SNOW3G) += lib/librte_ring -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_SNOW3G) += lib/librte_cryptodev - include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/crypto/snow3g/rte_snow3g_pmd.c b/dpdk/drivers/crypto/snow3g/rte_snow3g_pmd.c index ec31de28..4cc9a94f 100644 --- a/dpdk/drivers/crypto/snow3g/rte_snow3g_pmd.c +++ b/dpdk/drivers/crypto/snow3g/rte_snow3g_pmd.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -31,41 +31,20 @@ */ #include -#include #include #include #include -#include +#include #include #include #include "rte_snow3g_pmd_private.h" #define SNOW3G_IV_LENGTH 16 -#define SNOW3G_DIGEST_LENGTH 4 #define SNOW3G_MAX_BURST 8 #define BYTE_LEN 8 -/** - * Global static parameter used to create a unique name for each SNOW 3G - * crypto device. - */ -static unsigned unique_name_id; - -static inline int -create_unique_device_name(char *name, size_t size) -{ - int ret; - - if (name == NULL) - return -EINVAL; - - ret = snprintf(name, size, "%s_%u", RTE_STR(CRYPTODEV_NAME_SNOW3G_PMD), - unique_name_id++); - if (ret < 0) - return ret; - return 0; -} +static uint8_t cryptodev_driver_id; /** Get xform chain order. */ static enum snow3g_operation @@ -129,25 +108,45 @@ snow3g_set_session_parameters(struct snow3g_session *sess, case SNOW3G_OP_NOT_SUPPORTED: default: SNOW3G_LOG_ERR("Unsupported operation chain order parameter"); - return -EINVAL; + return -ENOTSUP; } if (cipher_xform) { /* Only SNOW 3G UEA2 supported */ if (cipher_xform->cipher.algo != RTE_CRYPTO_CIPHER_SNOW3G_UEA2) + return -ENOTSUP; + + if (cipher_xform->cipher.iv.length != SNOW3G_IV_LENGTH) { + SNOW3G_LOG_ERR("Wrong IV length"); return -EINVAL; + } + sess->cipher_iv_offset = cipher_xform->cipher.iv.offset; + /* Initialize key */ - sso_snow3g_init_key_sched(xform->cipher.key.data, + sso_snow3g_init_key_sched(cipher_xform->cipher.key.data, &sess->pKeySched_cipher); } if (auth_xform) { /* Only SNOW 3G UIA2 supported */ if (auth_xform->auth.algo != RTE_CRYPTO_AUTH_SNOW3G_UIA2) + return -ENOTSUP; + + if (auth_xform->auth.digest_length != SNOW3G_DIGEST_LENGTH) { + SNOW3G_LOG_ERR("Wrong digest length"); return -EINVAL; + } + sess->auth_op = auth_xform->auth.op; + + if (auth_xform->auth.iv.length != SNOW3G_IV_LENGTH) { + SNOW3G_LOG_ERR("Wrong IV length"); + return -EINVAL; + } + sess->auth_iv_offset = auth_xform->auth.iv.offset; + /* Initialize key */ - sso_snow3g_init_key_sched(xform->auth.key.data, + sso_snow3g_init_key_sched(auth_xform->auth.key.data, &sess->pKeySched_hash); } @@ -161,27 +160,41 @@ snow3g_set_session_parameters(struct snow3g_session *sess, static struct snow3g_session * snow3g_get_session(struct snow3g_qp *qp, struct rte_crypto_op *op) { - struct snow3g_session *sess; + struct snow3g_session *sess = NULL; - if (op->sym->sess_type == RTE_CRYPTO_SYM_OP_WITH_SESSION) { - if (unlikely(op->sym->session->dev_type != - RTE_CRYPTODEV_SNOW3G_PMD)) + if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { + if (likely(op->sym->session != NULL)) + sess = (struct snow3g_session *) + get_session_private_data( + op->sym->session, + cryptodev_driver_id); + } else { + void *_sess = NULL; + void *_sess_private_data = NULL; + + if (rte_mempool_get(qp->sess_mp, (void **)&_sess)) return NULL; - sess = (struct snow3g_session *)op->sym->session->_private; - } else { - struct rte_cryptodev_session *c_sess = NULL; - - if (rte_mempool_get(qp->sess_mp, (void **)&c_sess)) + if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data)) return NULL; - sess = (struct snow3g_session *)c_sess->_private; + sess = (struct snow3g_session *)_sess_private_data; if (unlikely(snow3g_set_session_parameters(sess, - op->sym->xform) != 0)) - return NULL; + op->sym->xform) != 0)) { + rte_mempool_put(qp->sess_mp, _sess); + rte_mempool_put(qp->sess_mp, _sess_private_data); + sess = NULL; + } + op->sym->session = (struct rte_cryptodev_sym_session *)_sess; + set_session_private_data(op->sym->session, cryptodev_driver_id, + _sess_private_data); } + if (unlikely(sess == NULL)) + op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; + + return sess; } @@ -194,17 +207,10 @@ process_snow3g_cipher_op(struct rte_crypto_op **ops, unsigned i; uint8_t processed_ops = 0; uint8_t *src[SNOW3G_MAX_BURST], *dst[SNOW3G_MAX_BURST]; - uint8_t *IV[SNOW3G_MAX_BURST]; + uint8_t *iv[SNOW3G_MAX_BURST]; uint32_t num_bytes[SNOW3G_MAX_BURST]; for (i = 0; i < num_ops; i++) { - /* Sanity checks. */ - if (unlikely(ops[i]->sym->cipher.iv.length != SNOW3G_IV_LENGTH)) { - ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; - SNOW3G_LOG_ERR("iv"); - break; - } - src[i] = rte_pktmbuf_mtod(ops[i]->sym->m_src, uint8_t *) + (ops[i]->sym->cipher.data.offset >> 3); dst[i] = ops[i]->sym->m_dst ? @@ -212,13 +218,14 @@ process_snow3g_cipher_op(struct rte_crypto_op **ops, (ops[i]->sym->cipher.data.offset >> 3) : rte_pktmbuf_mtod(ops[i]->sym->m_src, uint8_t *) + (ops[i]->sym->cipher.data.offset >> 3); - IV[i] = ops[i]->sym->cipher.iv.data; + iv[i] = rte_crypto_op_ctod_offset(ops[i], uint8_t *, + session->cipher_iv_offset); num_bytes[i] = ops[i]->sym->cipher.data.length >> 3; processed_ops++; } - sso_snow3g_f8_n_buffer(&session->pKeySched_cipher, IV, src, dst, + sso_snow3g_f8_n_buffer(&session->pKeySched_cipher, iv, src, dst, num_bytes, processed_ops); return processed_ops; @@ -230,16 +237,9 @@ process_snow3g_cipher_op_bit(struct rte_crypto_op *op, struct snow3g_session *session) { uint8_t *src, *dst; - uint8_t *IV; + uint8_t *iv; uint32_t length_in_bits, offset_in_bits; - /* Sanity checks. */ - if (unlikely(op->sym->cipher.iv.length != SNOW3G_IV_LENGTH)) { - op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; - SNOW3G_LOG_ERR("iv"); - return 0; - } - offset_in_bits = op->sym->cipher.data.offset; src = rte_pktmbuf_mtod(op->sym->m_src, uint8_t *); if (op->sym->m_dst == NULL) { @@ -248,10 +248,11 @@ process_snow3g_cipher_op_bit(struct rte_crypto_op *op, return 0; } dst = rte_pktmbuf_mtod(op->sym->m_dst, uint8_t *); - IV = op->sym->cipher.iv.data; + iv = rte_crypto_op_ctod_offset(op, uint8_t *, + session->cipher_iv_offset); length_in_bits = op->sym->cipher.data.length; - sso_snow3g_f8_1_buffer_bit(&session->pKeySched_cipher, IV, + sso_snow3g_f8_1_buffer_bit(&session->pKeySched_cipher, iv, src, dst, length_in_bits, offset_in_bits); return 1; @@ -259,7 +260,7 @@ process_snow3g_cipher_op_bit(struct rte_crypto_op *op, /** Generate/verify hash from mbufs with same hash key. */ static int -process_snow3g_hash_op(struct rte_crypto_op **ops, +process_snow3g_hash_op(struct snow3g_qp *qp, struct rte_crypto_op **ops, struct snow3g_session *session, uint8_t num_ops) { @@ -267,20 +268,9 @@ process_snow3g_hash_op(struct rte_crypto_op **ops, uint8_t processed_ops = 0; uint8_t *src, *dst; uint32_t length_in_bits; + uint8_t *iv; for (i = 0; i < num_ops; i++) { - if (unlikely(ops[i]->sym->auth.aad.length != SNOW3G_IV_LENGTH)) { - ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; - SNOW3G_LOG_ERR("aad"); - break; - } - - if (unlikely(ops[i]->sym->auth.digest.length != SNOW3G_DIGEST_LENGTH)) { - ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; - SNOW3G_LOG_ERR("digest"); - break; - } - /* Data must be byte aligned */ if ((ops[i]->sym->auth.data.offset % BYTE_LEN) != 0) { ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; @@ -292,27 +282,24 @@ process_snow3g_hash_op(struct rte_crypto_op **ops, src = rte_pktmbuf_mtod(ops[i]->sym->m_src, uint8_t *) + (ops[i]->sym->auth.data.offset >> 3); + iv = rte_crypto_op_ctod_offset(ops[i], uint8_t *, + session->auth_iv_offset); if (session->auth_op == RTE_CRYPTO_AUTH_OP_VERIFY) { - dst = (uint8_t *)rte_pktmbuf_append(ops[i]->sym->m_src, - ops[i]->sym->auth.digest.length); + dst = qp->temp_digest; sso_snow3g_f9_1_buffer(&session->pKeySched_hash, - ops[i]->sym->auth.aad.data, src, + iv, src, length_in_bits, dst); /* Verify digest. */ if (memcmp(dst, ops[i]->sym->auth.digest.data, - ops[i]->sym->auth.digest.length) != 0) + SNOW3G_DIGEST_LENGTH) != 0) ops[i]->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; - - /* Trim area used for digest from mbuf. */ - rte_pktmbuf_trim(ops[i]->sym->m_src, - ops[i]->sym->auth.digest.length); } else { dst = ops[i]->sym->auth.digest.data; sso_snow3g_f9_1_buffer(&session->pKeySched_hash, - ops[i]->sym->auth.aad.data, src, + iv, src, length_in_bits, dst); } processed_ops++; @@ -330,22 +317,37 @@ process_ops(struct rte_crypto_op **ops, struct snow3g_session *session, unsigned i; unsigned enqueued_ops, processed_ops; +#ifdef RTE_LIBRTE_PMD_SNOW3G_DEBUG + for (i = 0; i < num_ops; i++) { + if (!rte_pktmbuf_is_contiguous(ops[i]->sym->m_src) || + (ops[i]->sym->m_dst != NULL && + !rte_pktmbuf_is_contiguous( + ops[i]->sym->m_dst))) { + SNOW3G_LOG_ERR("PMD supports only contiguous mbufs, " + "op (%p) provides noncontiguous mbuf as " + "source/destination buffer.\n", ops[i]); + ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; + return 0; + } + } +#endif + switch (session->op) { case SNOW3G_OP_ONLY_CIPHER: processed_ops = process_snow3g_cipher_op(ops, session, num_ops); break; case SNOW3G_OP_ONLY_AUTH: - processed_ops = process_snow3g_hash_op(ops, session, + processed_ops = process_snow3g_hash_op(qp, ops, session, num_ops); break; case SNOW3G_OP_CIPHER_AUTH: processed_ops = process_snow3g_cipher_op(ops, session, num_ops); - process_snow3g_hash_op(ops, session, processed_ops); + process_snow3g_hash_op(qp, ops, session, processed_ops); break; case SNOW3G_OP_AUTH_CIPHER: - processed_ops = process_snow3g_hash_op(ops, session, + processed_ops = process_snow3g_hash_op(qp, ops, session, num_ops); process_snow3g_cipher_op(ops, session, processed_ops); break; @@ -362,14 +364,18 @@ process_ops(struct rte_crypto_op **ops, struct snow3g_session *session, if (ops[i]->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) ops[i]->status = RTE_CRYPTO_OP_STATUS_SUCCESS; /* Free session if a session-less crypto op. */ - if (ops[i]->sym->sess_type == RTE_CRYPTO_SYM_OP_SESSIONLESS) { + if (ops[i]->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { + memset(session, 0, sizeof(struct snow3g_session)); + memset(ops[i]->sym->session, 0, + rte_cryptodev_get_header_session_size()); + rte_mempool_put(qp->sess_mp, session); rte_mempool_put(qp->sess_mp, ops[i]->sym->session); ops[i]->sym->session = NULL; } } enqueued_ops = rte_ring_enqueue_burst(qp->processed_ops, - (void **)ops, processed_ops); + (void **)ops, processed_ops, NULL); qp->qp_stats.enqueued_count += enqueued_ops; *accumulated_enqueued_ops += enqueued_ops; @@ -389,15 +395,15 @@ process_op_bit(struct rte_crypto_op *op, struct snow3g_session *session, session); break; case SNOW3G_OP_ONLY_AUTH: - processed_op = process_snow3g_hash_op(&op, session, 1); + processed_op = process_snow3g_hash_op(qp, &op, session, 1); break; case SNOW3G_OP_CIPHER_AUTH: processed_op = process_snow3g_cipher_op_bit(op, session); if (processed_op == 1) - process_snow3g_hash_op(&op, session, 1); + process_snow3g_hash_op(qp, &op, session, 1); break; case SNOW3G_OP_AUTH_CIPHER: - processed_op = process_snow3g_hash_op(&op, session, 1); + processed_op = process_snow3g_hash_op(qp, &op, session, 1); if (processed_op == 1) process_snow3g_cipher_op_bit(op, session); break; @@ -414,13 +420,14 @@ process_op_bit(struct rte_crypto_op *op, struct snow3g_session *session, op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; /* Free session if a session-less crypto op. */ - if (op->sym->sess_type == RTE_CRYPTO_SYM_OP_SESSIONLESS) { - rte_mempool_put(qp->sess_mp, op->sym->session); + if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { + memset(op->sym->session, 0, sizeof(struct snow3g_session)); + rte_cryptodev_sym_session_free(op->sym->session); op->sym->session = NULL; } enqueued_op = rte_ring_enqueue_burst(qp->processed_ops, - (void **)&op, processed_op); + (void **)&op, processed_op, NULL); qp->qp_stats.enqueued_count += enqueued_op; *accumulated_enqueued_ops += enqueued_op; @@ -539,47 +546,30 @@ snow3g_pmd_dequeue_burst(void *queue_pair, unsigned nb_dequeued; nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops, - (void **)c_ops, nb_ops); + (void **)c_ops, nb_ops, NULL); qp->qp_stats.dequeued_count += nb_dequeued; return nb_dequeued; } -static int cryptodev_snow3g_uninit(const char *name); +static int cryptodev_snow3g_remove(struct rte_vdev_device *vdev); static int cryptodev_snow3g_create(const char *name, - struct rte_crypto_vdev_init_params *init_params) + struct rte_vdev_device *vdev, + struct rte_cryptodev_pmd_init_params *init_params) { struct rte_cryptodev *dev; - char crypto_dev_name[RTE_CRYPTODEV_NAME_MAX_LEN]; struct snow3g_private *internals; - uint64_t cpu_flags = 0; + uint64_t cpu_flags = RTE_CRYPTODEV_FF_CPU_SSE; - /* Check CPU for supported vector instruction set */ - if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_SSE4_1)) - cpu_flags |= RTE_CRYPTODEV_FF_CPU_SSE; - else { - SNOW3G_LOG_ERR("Vector instructions are not supported by CPU"); - return -EFAULT; - } - - - /* Create a unique device name. */ - if (create_unique_device_name(crypto_dev_name, - RTE_CRYPTODEV_NAME_MAX_LEN) != 0) { - SNOW3G_LOG_ERR("failed to create unique cryptodev name"); - return -EINVAL; - } - - dev = rte_cryptodev_pmd_virtual_dev_init(crypto_dev_name, - sizeof(struct snow3g_private), init_params->socket_id); + dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params); if (dev == NULL) { SNOW3G_LOG_ERR("failed to create cryptodev vdev"); goto init_error; } - dev->dev_type = RTE_CRYPTODEV_SNOW3G_PMD; + dev->driver_id = cryptodev_driver_id; dev->dev_ops = rte_snow3g_pmd_ops; /* Register RX/TX burst functions for data path. */ @@ -597,55 +587,65 @@ cryptodev_snow3g_create(const char *name, return 0; init_error: - SNOW3G_LOG_ERR("driver %s: cryptodev_snow3g_create failed", name); + SNOW3G_LOG_ERR("driver %s: cryptodev_snow3g_create failed", + init_params->name); - cryptodev_snow3g_uninit(crypto_dev_name); + cryptodev_snow3g_remove(vdev); return -EFAULT; } static int -cryptodev_snow3g_init(const char *name, - const char *input_args) +cryptodev_snow3g_probe(struct rte_vdev_device *vdev) { - struct rte_crypto_vdev_init_params init_params = { - RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_QUEUE_PAIRS, - RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_SESSIONS, - rte_socket_id() + struct rte_cryptodev_pmd_init_params init_params = { + "", + sizeof(struct snow3g_private), + rte_socket_id(), + RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS, + RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_SESSIONS }; + const char *name; + const char *input_args; - rte_cryptodev_parse_vdev_init_params(&init_params, input_args); + name = rte_vdev_device_name(vdev); + if (name == NULL) + return -EINVAL; + input_args = rte_vdev_device_args(vdev); - RTE_LOG(INFO, PMD, "Initialising %s on NUMA node %d\n", name, - init_params.socket_id); - RTE_LOG(INFO, PMD, " Max number of queue pairs = %d\n", - init_params.max_nb_queue_pairs); - RTE_LOG(INFO, PMD, " Max number of sessions = %d\n", - init_params.max_nb_sessions); + rte_cryptodev_pmd_parse_input_args(&init_params, input_args); - return cryptodev_snow3g_create(name, &init_params); + return cryptodev_snow3g_create(name, vdev, &init_params); } static int -cryptodev_snow3g_uninit(const char *name) +cryptodev_snow3g_remove(struct rte_vdev_device *vdev) { + struct rte_cryptodev *cryptodev; + const char *name; + + name = rte_vdev_device_name(vdev); if (name == NULL) return -EINVAL; - RTE_LOG(INFO, PMD, "Closing SNOW3G crypto device %s" - " on numa socket %u\n", - name, rte_socket_id()); + cryptodev = rte_cryptodev_pmd_get_named_dev(name); + if (cryptodev == NULL) + return -ENODEV; - return 0; + return rte_cryptodev_pmd_destroy(cryptodev); } -static struct rte_driver cryptodev_snow3g_pmd_drv = { - .type = PMD_VDEV, - .init = cryptodev_snow3g_init, - .uninit = cryptodev_snow3g_uninit +static struct rte_vdev_driver cryptodev_snow3g_pmd_drv = { + .probe = cryptodev_snow3g_probe, + .remove = cryptodev_snow3g_remove }; -PMD_REGISTER_DRIVER(cryptodev_snow3g_pmd_drv, CRYPTODEV_NAME_SNOW3G_PMD); -DRIVER_REGISTER_PARAM_STRING(CRYPTODEV_NAME_SNOW3G_PMD, +static struct cryptodev_driver snow3g_crypto_drv; + +RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_SNOW3G_PMD, cryptodev_snow3g_pmd_drv); +RTE_PMD_REGISTER_ALIAS(CRYPTODEV_NAME_SNOW3G_PMD, cryptodev_snow3g_pmd); +RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_SNOW3G_PMD, "max_nb_queue_pairs= " "max_nb_sessions= " "socket_id="); +RTE_PMD_REGISTER_CRYPTO_DRIVER(snow3g_crypto_drv, cryptodev_snow3g_pmd_drv, + cryptodev_driver_id); diff --git a/dpdk/drivers/crypto/snow3g/rte_snow3g_pmd_ops.c b/dpdk/drivers/crypto/snow3g/rte_snow3g_pmd_ops.c index 6f00b066..157a2de3 100644 --- a/dpdk/drivers/crypto/snow3g/rte_snow3g_pmd_ops.c +++ b/dpdk/drivers/crypto/snow3g/rte_snow3g_pmd_ops.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -39,7 +39,7 @@ #include "rte_snow3g_pmd_private.h" static const struct rte_cryptodev_capabilities snow3g_pmd_capabilities[] = { - { /* SNOW3G (UIA2) */ + { /* SNOW 3G (UIA2) */ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, {.sym = { .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, @@ -56,7 +56,7 @@ static const struct rte_cryptodev_capabilities snow3g_pmd_capabilities[] = { .max = 4, .increment = 0 }, - .aad_size = { + .iv_size = { .min = 16, .max = 16, .increment = 0 @@ -64,7 +64,7 @@ static const struct rte_cryptodev_capabilities snow3g_pmd_capabilities[] = { }, } }, } }, - { /* SNOW3G (UEA2) */ + { /* SNOW 3G (UEA2) */ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, {.sym = { .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, @@ -89,7 +89,8 @@ static const struct rte_cryptodev_capabilities snow3g_pmd_capabilities[] = { /** Configure device */ static int -snow3g_pmd_config(__rte_unused struct rte_cryptodev *dev) +snow3g_pmd_config(__rte_unused struct rte_cryptodev *dev, + __rte_unused struct rte_cryptodev_config *config) { return 0; } @@ -155,7 +156,7 @@ snow3g_pmd_info_get(struct rte_cryptodev *dev, struct snow3g_private *internals = dev->data->dev_private; if (dev_info != NULL) { - dev_info->dev_type = dev->dev_type; + dev_info->driver_id = dev->driver_id; dev_info->max_nb_queue_pairs = internals->max_nb_queue_pairs; dev_info->sym.max_nb_sessions = internals->max_nb_sessions; dev_info->feature_flags = dev->feature_flags; @@ -183,7 +184,7 @@ snow3g_pmd_qp_set_unique_name(struct rte_cryptodev *dev, "snow3g_pmd_%u_qp_%u", dev->data->dev_id, qp->id); - if (n > sizeof(qp->name)) + if (n >= sizeof(qp->name)) return -1; return 0; @@ -198,7 +199,7 @@ snow3g_pmd_qp_create_processed_ops_ring(struct snow3g_qp *qp, r = rte_ring_lookup(qp->name); if (r) { - if (r->prod.size >= ring_size) { + if (rte_ring_get_size(r) >= ring_size) { SNOW3G_LOG_INFO("Reusing existing ring %s" " for processed packets", qp->name); @@ -219,7 +220,7 @@ snow3g_pmd_qp_create_processed_ops_ring(struct snow3g_qp *qp, static int snow3g_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, const struct rte_cryptodev_qp_conf *qp_conf, - int socket_id) + int socket_id, struct rte_mempool *session_pool) { struct snow3g_qp *qp = NULL; @@ -228,7 +229,7 @@ snow3g_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, snow3g_pmd_qp_release(dev, qp_id); /* Allocate the queue pair data structure. */ - qp = rte_zmalloc_socket("SNOW3G PMD Queue Pair", sizeof(*qp), + qp = rte_zmalloc_socket("SNOW 3G PMD Queue Pair", sizeof(*qp), RTE_CACHE_LINE_SIZE, socket_id); if (qp == NULL) return (-ENOMEM); @@ -244,7 +245,7 @@ snow3g_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, if (qp->processed_ops == NULL) goto qp_setup_cleanup; - qp->sess_mp = dev->data->session_pool; + qp->sess_mp = session_pool; memset(&qp->qp_stats, 0, sizeof(qp->qp_stats)); @@ -288,33 +289,56 @@ snow3g_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused) } /** Configure a SNOW 3G session from a crypto xform chain */ -static void * +static int snow3g_pmd_session_configure(struct rte_cryptodev *dev __rte_unused, - struct rte_crypto_sym_xform *xform, void *sess) + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool) { + void *sess_private_data; + int ret; + if (unlikely(sess == NULL)) { SNOW3G_LOG_ERR("invalid session struct"); - return NULL; + return -EINVAL; } - if (snow3g_set_session_parameters(sess, xform) != 0) { + if (rte_mempool_get(mempool, &sess_private_data)) { + CDEV_LOG_ERR( + "Couldn't get object from session mempool"); + return -ENOMEM; + } + + ret = snow3g_set_session_parameters(sess_private_data, xform); + if (ret != 0) { SNOW3G_LOG_ERR("failed configure session parameters"); - return NULL; + + /* Return session to mempool */ + rte_mempool_put(mempool, sess_private_data); + return ret; } - return sess; + set_session_private_data(sess, dev->driver_id, + sess_private_data); + + return 0; } /** Clear the memory of session so it doesn't leave key material behind */ static void -snow3g_pmd_session_clear(struct rte_cryptodev *dev __rte_unused, void *sess) +snow3g_pmd_session_clear(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) { - /* - * Current just resetting the whole data structure, need to investigate - * whether a more selective reset of key would be more performant - */ - if (sess) - memset(sess, 0, sizeof(struct snow3g_session)); + uint8_t index = dev->driver_id; + void *sess_priv = get_session_private_data(sess, index); + + /* Zero out the whole structure */ + if (sess_priv) { + memset(sess_priv, 0, sizeof(struct snow3g_session)); + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + set_session_private_data(sess, index, NULL); + rte_mempool_put(sess_mp, sess_priv); + } } struct rte_cryptodev_ops snow3g_pmd_ops = { diff --git a/dpdk/drivers/crypto/snow3g/rte_snow3g_pmd_private.h b/dpdk/drivers/crypto/snow3g/rte_snow3g_pmd_private.h index 03973b97..7b9729f0 100644 --- a/dpdk/drivers/crypto/snow3g/rte_snow3g_pmd_private.h +++ b/dpdk/drivers/crypto/snow3g/rte_snow3g_pmd_private.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -35,6 +35,9 @@ #include +#define CRYPTODEV_NAME_SNOW3G_PMD crypto_snow3g +/**< SNOW 3G PMD device name */ + #define SNOW3G_LOG_ERR(fmt, args...) \ RTE_LOG(ERR, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \ RTE_STR(CRYPTODEV_NAME_SNOW3G_PMD), \ @@ -55,6 +58,8 @@ #define SNOW3G_LOG_DBG(fmt, args...) #endif +#define SNOW3G_DIGEST_LENGTH 4 + /** private data structure for each virtual SNOW 3G device */ struct snow3g_private { unsigned max_nb_queue_pairs; @@ -75,6 +80,11 @@ struct snow3g_qp { /**< Session Mempool */ struct rte_cryptodev_stats qp_stats; /**< Queue pair statistics */ + uint8_t temp_digest[SNOW3G_DIGEST_LENGTH]; + /**< Buffer used to store the digest generated + * by the driver when verifying a digest provided + * by the user (using authentication verify operation) + */ } __rte_cache_aligned; enum snow3g_operation { @@ -91,6 +101,8 @@ struct snow3g_session { enum rte_crypto_auth_operation auth_op; sso_snow3g_key_schedule_t pKeySched_cipher; sso_snow3g_key_schedule_t pKeySched_hash; + uint16_t cipher_iv_offset; + uint16_t auth_iv_offset; } __rte_cache_aligned; diff --git a/dpdk/drivers/crypto/zuc/Makefile b/dpdk/drivers/crypto/zuc/Makefile new file mode 100644 index 00000000..af77bc8a --- /dev/null +++ b/dpdk/drivers/crypto/zuc/Makefile @@ -0,0 +1,65 @@ +# BSD LICENSE +# +# Copyright(c) 2016 Intel Corporation. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +include $(RTE_SDK)/mk/rte.vars.mk + +ifneq ($(MAKECMDGOALS),clean) +ifeq ($(LIBSSO_ZUC_PATH),) +$(error "Please define LIBSSO_ZUC_PATH environment variable") +endif +endif + +# library name +LIB = librte_pmd_zuc.a + +# build flags +CFLAGS += -O3 +CFLAGS += $(WERROR_FLAGS) + +# library version +LIBABIVER := 1 + +# versioning export map +EXPORT_MAP := rte_pmd_zuc_version.map + +# external library dependencies +CFLAGS += -I$(LIBSSO_ZUC_PATH) +CFLAGS += -I$(LIBSSO_ZUC_PATH)/include +CFLAGS += -I$(LIBSSO_ZUC_PATH)/build +LDLIBS += -L$(LIBSSO_ZUC_PATH)/build -lsso_zuc +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_cryptodev +LDLIBS += -lrte_bus_vdev + +# library source files +SRCS-$(CONFIG_RTE_LIBRTE_PMD_ZUC) += rte_zuc_pmd.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_ZUC) += rte_zuc_pmd_ops.c + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/crypto/zuc/rte_pmd_zuc_version.map b/dpdk/drivers/crypto/zuc/rte_pmd_zuc_version.map new file mode 100644 index 00000000..cc5829e3 --- /dev/null +++ b/dpdk/drivers/crypto/zuc/rte_pmd_zuc_version.map @@ -0,0 +1,3 @@ +DPDK_16.11 { + local: *; +}; diff --git a/dpdk/drivers/crypto/zuc/rte_zuc_pmd.c b/dpdk/drivers/crypto/zuc/rte_zuc_pmd.c new file mode 100644 index 00000000..590224bf --- /dev/null +++ b/dpdk/drivers/crypto/zuc/rte_zuc_pmd.c @@ -0,0 +1,556 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "rte_zuc_pmd_private.h" + +#define ZUC_MAX_BURST 8 +#define BYTE_LEN 8 + +static uint8_t cryptodev_driver_id; + +/** Get xform chain order. */ +static enum zuc_operation +zuc_get_mode(const struct rte_crypto_sym_xform *xform) +{ + if (xform == NULL) + return ZUC_OP_NOT_SUPPORTED; + + if (xform->next) + if (xform->next->next != NULL) + return ZUC_OP_NOT_SUPPORTED; + + if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) { + if (xform->next == NULL) + return ZUC_OP_ONLY_AUTH; + else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) + return ZUC_OP_AUTH_CIPHER; + else + return ZUC_OP_NOT_SUPPORTED; + } + + if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { + if (xform->next == NULL) + return ZUC_OP_ONLY_CIPHER; + else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) + return ZUC_OP_CIPHER_AUTH; + else + return ZUC_OP_NOT_SUPPORTED; + } + + return ZUC_OP_NOT_SUPPORTED; +} + + +/** Parse crypto xform chain and set private session parameters. */ +int +zuc_set_session_parameters(struct zuc_session *sess, + const struct rte_crypto_sym_xform *xform) +{ + const struct rte_crypto_sym_xform *auth_xform = NULL; + const struct rte_crypto_sym_xform *cipher_xform = NULL; + enum zuc_operation mode; + + /* Select Crypto operation - hash then cipher / cipher then hash */ + mode = zuc_get_mode(xform); + + switch (mode) { + case ZUC_OP_CIPHER_AUTH: + auth_xform = xform->next; + + /* Fall-through */ + case ZUC_OP_ONLY_CIPHER: + cipher_xform = xform; + break; + case ZUC_OP_AUTH_CIPHER: + cipher_xform = xform->next; + /* Fall-through */ + case ZUC_OP_ONLY_AUTH: + auth_xform = xform; + break; + case ZUC_OP_NOT_SUPPORTED: + default: + ZUC_LOG_ERR("Unsupported operation chain order parameter"); + return -ENOTSUP; + } + + if (cipher_xform) { + /* Only ZUC EEA3 supported */ + if (cipher_xform->cipher.algo != RTE_CRYPTO_CIPHER_ZUC_EEA3) + return -ENOTSUP; + + if (cipher_xform->cipher.iv.length != ZUC_IV_KEY_LENGTH) { + ZUC_LOG_ERR("Wrong IV length"); + return -EINVAL; + } + sess->cipher_iv_offset = cipher_xform->cipher.iv.offset; + + /* Copy the key */ + memcpy(sess->pKey_cipher, cipher_xform->cipher.key.data, + ZUC_IV_KEY_LENGTH); + } + + if (auth_xform) { + /* Only ZUC EIA3 supported */ + if (auth_xform->auth.algo != RTE_CRYPTO_AUTH_ZUC_EIA3) + return -ENOTSUP; + + if (auth_xform->auth.digest_length != ZUC_DIGEST_LENGTH) { + ZUC_LOG_ERR("Wrong digest length"); + return -EINVAL; + } + + sess->auth_op = auth_xform->auth.op; + + if (auth_xform->auth.iv.length != ZUC_IV_KEY_LENGTH) { + ZUC_LOG_ERR("Wrong IV length"); + return -EINVAL; + } + sess->auth_iv_offset = auth_xform->auth.iv.offset; + + /* Copy the key */ + memcpy(sess->pKey_hash, auth_xform->auth.key.data, + ZUC_IV_KEY_LENGTH); + } + + + sess->op = mode; + + return 0; +} + +/** Get ZUC session. */ +static struct zuc_session * +zuc_get_session(struct zuc_qp *qp, struct rte_crypto_op *op) +{ + struct zuc_session *sess = NULL; + + if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { + if (likely(op->sym->session != NULL)) + sess = (struct zuc_session *)get_session_private_data( + op->sym->session, + cryptodev_driver_id); + } else { + void *_sess = NULL; + void *_sess_private_data = NULL; + + if (rte_mempool_get(qp->sess_mp, (void **)&_sess)) + return NULL; + + if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data)) + return NULL; + + sess = (struct zuc_session *)_sess_private_data; + + if (unlikely(zuc_set_session_parameters(sess, + op->sym->xform) != 0)) { + rte_mempool_put(qp->sess_mp, _sess); + rte_mempool_put(qp->sess_mp, _sess_private_data); + sess = NULL; + } + op->sym->session = (struct rte_cryptodev_sym_session *)_sess; + set_session_private_data(op->sym->session, cryptodev_driver_id, + _sess_private_data); + } + + if (unlikely(sess == NULL)) + op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; + + + return sess; +} + +/** Encrypt/decrypt mbufs with same cipher key. */ +static uint8_t +process_zuc_cipher_op(struct rte_crypto_op **ops, + struct zuc_session *session, + uint8_t num_ops) +{ + unsigned i; + uint8_t processed_ops = 0; + uint8_t *src[ZUC_MAX_BURST], *dst[ZUC_MAX_BURST]; + uint8_t *iv[ZUC_MAX_BURST]; + uint32_t num_bytes[ZUC_MAX_BURST]; + uint8_t *cipher_keys[ZUC_MAX_BURST]; + + for (i = 0; i < num_ops; i++) { + if (((ops[i]->sym->cipher.data.length % BYTE_LEN) != 0) + || ((ops[i]->sym->cipher.data.offset + % BYTE_LEN) != 0)) { + ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; + ZUC_LOG_ERR("Data Length or offset"); + break; + } + +#ifdef RTE_LIBRTE_PMD_ZUC_DEBUG + if (!rte_pktmbuf_is_contiguous(ops[i]->sym->m_src) || + (ops[i]->sym->m_dst != NULL && + !rte_pktmbuf_is_contiguous( + ops[i]->sym->m_dst))) { + ZUC_LOG_ERR("PMD supports only contiguous mbufs, " + "op (%p) provides noncontiguous mbuf as " + "source/destination buffer.\n", ops[i]); + ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; + break; + } +#endif + + src[i] = rte_pktmbuf_mtod(ops[i]->sym->m_src, uint8_t *) + + (ops[i]->sym->cipher.data.offset >> 3); + dst[i] = ops[i]->sym->m_dst ? + rte_pktmbuf_mtod(ops[i]->sym->m_dst, uint8_t *) + + (ops[i]->sym->cipher.data.offset >> 3) : + rte_pktmbuf_mtod(ops[i]->sym->m_src, uint8_t *) + + (ops[i]->sym->cipher.data.offset >> 3); + iv[i] = rte_crypto_op_ctod_offset(ops[i], uint8_t *, + session->cipher_iv_offset); + num_bytes[i] = ops[i]->sym->cipher.data.length >> 3; + + cipher_keys[i] = session->pKey_cipher; + + processed_ops++; + } + + sso_zuc_eea3_n_buffer(cipher_keys, iv, src, dst, + num_bytes, processed_ops); + + return processed_ops; +} + +/** Generate/verify hash from mbufs with same hash key. */ +static int +process_zuc_hash_op(struct zuc_qp *qp, struct rte_crypto_op **ops, + struct zuc_session *session, + uint8_t num_ops) +{ + unsigned i; + uint8_t processed_ops = 0; + uint8_t *src; + uint32_t *dst; + uint32_t length_in_bits; + uint8_t *iv; + + for (i = 0; i < num_ops; i++) { + /* Data must be byte aligned */ + if ((ops[i]->sym->auth.data.offset % BYTE_LEN) != 0) { + ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; + ZUC_LOG_ERR("Offset"); + break; + } + + length_in_bits = ops[i]->sym->auth.data.length; + + src = rte_pktmbuf_mtod(ops[i]->sym->m_src, uint8_t *) + + (ops[i]->sym->auth.data.offset >> 3); + iv = rte_crypto_op_ctod_offset(ops[i], uint8_t *, + session->auth_iv_offset); + + if (session->auth_op == RTE_CRYPTO_AUTH_OP_VERIFY) { + dst = (uint32_t *)qp->temp_digest; + + sso_zuc_eia3_1_buffer(session->pKey_hash, + iv, src, + length_in_bits, dst); + /* Verify digest. */ + if (memcmp(dst, ops[i]->sym->auth.digest.data, + ZUC_DIGEST_LENGTH) != 0) + ops[i]->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; + } else { + dst = (uint32_t *)ops[i]->sym->auth.digest.data; + + sso_zuc_eia3_1_buffer(session->pKey_hash, + iv, src, + length_in_bits, dst); + } + processed_ops++; + } + + return processed_ops; +} + +/** Process a batch of crypto ops which shares the same session. */ +static int +process_ops(struct rte_crypto_op **ops, struct zuc_session *session, + struct zuc_qp *qp, uint8_t num_ops, + uint16_t *accumulated_enqueued_ops) +{ + unsigned i; + unsigned enqueued_ops, processed_ops; + + switch (session->op) { + case ZUC_OP_ONLY_CIPHER: + processed_ops = process_zuc_cipher_op(ops, + session, num_ops); + break; + case ZUC_OP_ONLY_AUTH: + processed_ops = process_zuc_hash_op(qp, ops, session, + num_ops); + break; + case ZUC_OP_CIPHER_AUTH: + processed_ops = process_zuc_cipher_op(ops, session, + num_ops); + process_zuc_hash_op(qp, ops, session, processed_ops); + break; + case ZUC_OP_AUTH_CIPHER: + processed_ops = process_zuc_hash_op(qp, ops, session, + num_ops); + process_zuc_cipher_op(ops, session, processed_ops); + break; + default: + /* Operation not supported. */ + processed_ops = 0; + } + + for (i = 0; i < num_ops; i++) { + /* + * If there was no error/authentication failure, + * change status to successful. + */ + if (ops[i]->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) + ops[i]->status = RTE_CRYPTO_OP_STATUS_SUCCESS; + /* Free session if a session-less crypto op. */ + if (ops[i]->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { + memset(session, 0, sizeof(struct zuc_session)); + memset(ops[i]->sym->session, 0, + rte_cryptodev_get_header_session_size()); + rte_mempool_put(qp->sess_mp, session); + rte_mempool_put(qp->sess_mp, ops[i]->sym->session); + ops[i]->sym->session = NULL; + } + } + + enqueued_ops = rte_ring_enqueue_burst(qp->processed_ops, + (void **)ops, processed_ops, NULL); + qp->qp_stats.enqueued_count += enqueued_ops; + *accumulated_enqueued_ops += enqueued_ops; + + return enqueued_ops; +} + +static uint16_t +zuc_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops, + uint16_t nb_ops) +{ + struct rte_crypto_op *c_ops[ZUC_MAX_BURST]; + struct rte_crypto_op *curr_c_op; + + struct zuc_session *prev_sess = NULL, *curr_sess = NULL; + struct zuc_qp *qp = queue_pair; + unsigned i; + uint8_t burst_size = 0; + uint16_t enqueued_ops = 0; + uint8_t processed_ops; + + for (i = 0; i < nb_ops; i++) { + curr_c_op = ops[i]; + + /* Set status as enqueued (not processed yet) by default. */ + curr_c_op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; + + curr_sess = zuc_get_session(qp, curr_c_op); + if (unlikely(curr_sess == NULL || + curr_sess->op == ZUC_OP_NOT_SUPPORTED)) { + curr_c_op->status = + RTE_CRYPTO_OP_STATUS_INVALID_SESSION; + break; + } + + /* Batch ops that share the same session. */ + if (prev_sess == NULL) { + prev_sess = curr_sess; + c_ops[burst_size++] = curr_c_op; + } else if (curr_sess == prev_sess) { + c_ops[burst_size++] = curr_c_op; + /* + * When there are enough ops to process in a batch, + * process them, and start a new batch. + */ + if (burst_size == ZUC_MAX_BURST) { + processed_ops = process_ops(c_ops, prev_sess, + qp, burst_size, &enqueued_ops); + if (processed_ops < burst_size) { + burst_size = 0; + break; + } + + burst_size = 0; + prev_sess = NULL; + } + } else { + /* + * Different session, process the ops + * of the previous session. + */ + processed_ops = process_ops(c_ops, prev_sess, + qp, burst_size, &enqueued_ops); + if (processed_ops < burst_size) { + burst_size = 0; + break; + } + + burst_size = 0; + prev_sess = curr_sess; + + c_ops[burst_size++] = curr_c_op; + } + } + + if (burst_size != 0) { + /* Process the crypto ops of the last session. */ + processed_ops = process_ops(c_ops, prev_sess, + qp, burst_size, &enqueued_ops); + } + + qp->qp_stats.enqueue_err_count += nb_ops - enqueued_ops; + return enqueued_ops; +} + +static uint16_t +zuc_pmd_dequeue_burst(void *queue_pair, + struct rte_crypto_op **c_ops, uint16_t nb_ops) +{ + struct zuc_qp *qp = queue_pair; + + unsigned nb_dequeued; + + nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops, + (void **)c_ops, nb_ops, NULL); + qp->qp_stats.dequeued_count += nb_dequeued; + + return nb_dequeued; +} + +static int cryptodev_zuc_remove(struct rte_vdev_device *vdev); + +static int +cryptodev_zuc_create(const char *name, + struct rte_vdev_device *vdev, + struct rte_cryptodev_pmd_init_params *init_params) +{ + struct rte_cryptodev *dev; + struct zuc_private *internals; + uint64_t cpu_flags = RTE_CRYPTODEV_FF_CPU_SSE; + + + dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params); + if (dev == NULL) { + ZUC_LOG_ERR("failed to create cryptodev vdev"); + goto init_error; + } + + dev->driver_id = cryptodev_driver_id; + dev->dev_ops = rte_zuc_pmd_ops; + + /* Register RX/TX burst functions for data path. */ + dev->dequeue_burst = zuc_pmd_dequeue_burst; + dev->enqueue_burst = zuc_pmd_enqueue_burst; + + dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | + RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | + cpu_flags; + + internals = dev->data->dev_private; + + internals->max_nb_queue_pairs = init_params->max_nb_queue_pairs; + internals->max_nb_sessions = init_params->max_nb_sessions; + + return 0; +init_error: + ZUC_LOG_ERR("driver %s: cryptodev_zuc_create failed", + init_params->name); + + cryptodev_zuc_remove(vdev); + return -EFAULT; +} + +static int +cryptodev_zuc_probe(struct rte_vdev_device *vdev) +{ + struct rte_cryptodev_pmd_init_params init_params = { + "", + sizeof(struct zuc_private), + rte_socket_id(), + RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS, + RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_SESSIONS + }; + const char *name; + const char *input_args; + + name = rte_vdev_device_name(vdev); + if (name == NULL) + return -EINVAL; + input_args = rte_vdev_device_args(vdev); + + rte_cryptodev_pmd_parse_input_args(&init_params, input_args); + + return cryptodev_zuc_create(name, vdev, &init_params); +} + +static int +cryptodev_zuc_remove(struct rte_vdev_device *vdev) +{ + + struct rte_cryptodev *cryptodev; + const char *name; + + name = rte_vdev_device_name(vdev); + if (name == NULL) + return -EINVAL; + + cryptodev = rte_cryptodev_pmd_get_named_dev(name); + if (cryptodev == NULL) + return -ENODEV; + + return rte_cryptodev_pmd_destroy(cryptodev); +} + +static struct rte_vdev_driver cryptodev_zuc_pmd_drv = { + .probe = cryptodev_zuc_probe, + .remove = cryptodev_zuc_remove +}; + +static struct cryptodev_driver zuc_crypto_drv; + +RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_ZUC_PMD, cryptodev_zuc_pmd_drv); +RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_ZUC_PMD, + "max_nb_queue_pairs= " + "max_nb_sessions= " + "socket_id="); +RTE_PMD_REGISTER_CRYPTO_DRIVER(zuc_crypto_drv, cryptodev_zuc_pmd_drv, + cryptodev_driver_id); diff --git a/dpdk/drivers/crypto/zuc/rte_zuc_pmd_ops.c b/dpdk/drivers/crypto/zuc/rte_zuc_pmd_ops.c new file mode 100644 index 00000000..243c0991 --- /dev/null +++ b/dpdk/drivers/crypto/zuc/rte_zuc_pmd_ops.c @@ -0,0 +1,366 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +#include +#include +#include + +#include "rte_zuc_pmd_private.h" + +static const struct rte_cryptodev_capabilities zuc_pmd_capabilities[] = { + { /* ZUC (EIA3) */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_ZUC_EIA3, + .block_size = 16, + .key_size = { + .min = 16, + .max = 16, + .increment = 0 + }, + .digest_size = { + .min = 4, + .max = 4, + .increment = 0 + }, + .iv_size = { + .min = 16, + .max = 16, + .increment = 0 + } + }, } + }, } + }, + { /* ZUC (EEA3) */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, + {.cipher = { + .algo = RTE_CRYPTO_CIPHER_ZUC_EEA3, + .block_size = 16, + .key_size = { + .min = 16, + .max = 16, + .increment = 0 + }, + .iv_size = { + .min = 16, + .max = 16, + .increment = 0 + }, + }, } + }, } + }, + RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST() +}; + +/** Configure device */ +static int +zuc_pmd_config(__rte_unused struct rte_cryptodev *dev, + __rte_unused struct rte_cryptodev_config *config) +{ + return 0; +} + +/** Start device */ +static int +zuc_pmd_start(__rte_unused struct rte_cryptodev *dev) +{ + return 0; +} + +/** Stop device */ +static void +zuc_pmd_stop(__rte_unused struct rte_cryptodev *dev) +{ +} + +/** Close device */ +static int +zuc_pmd_close(__rte_unused struct rte_cryptodev *dev) +{ + return 0; +} + + +/** Get device statistics */ +static void +zuc_pmd_stats_get(struct rte_cryptodev *dev, + struct rte_cryptodev_stats *stats) +{ + int qp_id; + + for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) { + struct zuc_qp *qp = dev->data->queue_pairs[qp_id]; + + stats->enqueued_count += qp->qp_stats.enqueued_count; + stats->dequeued_count += qp->qp_stats.dequeued_count; + + stats->enqueue_err_count += qp->qp_stats.enqueue_err_count; + stats->dequeue_err_count += qp->qp_stats.dequeue_err_count; + } +} + +/** Reset device statistics */ +static void +zuc_pmd_stats_reset(struct rte_cryptodev *dev) +{ + int qp_id; + + for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) { + struct zuc_qp *qp = dev->data->queue_pairs[qp_id]; + + memset(&qp->qp_stats, 0, sizeof(qp->qp_stats)); + } +} + + +/** Get device info */ +static void +zuc_pmd_info_get(struct rte_cryptodev *dev, + struct rte_cryptodev_info *dev_info) +{ + struct zuc_private *internals = dev->data->dev_private; + + if (dev_info != NULL) { + dev_info->driver_id = dev->driver_id; + dev_info->max_nb_queue_pairs = internals->max_nb_queue_pairs; + dev_info->sym.max_nb_sessions = internals->max_nb_sessions; + dev_info->feature_flags = dev->feature_flags; + dev_info->capabilities = zuc_pmd_capabilities; + } +} + +/** Release queue pair */ +static int +zuc_pmd_qp_release(struct rte_cryptodev *dev, uint16_t qp_id) +{ + if (dev->data->queue_pairs[qp_id] != NULL) { + rte_free(dev->data->queue_pairs[qp_id]); + dev->data->queue_pairs[qp_id] = NULL; + } + return 0; +} + +/** set a unique name for the queue pair based on its name, dev_id and qp_id */ +static int +zuc_pmd_qp_set_unique_name(struct rte_cryptodev *dev, + struct zuc_qp *qp) +{ + unsigned n = snprintf(qp->name, sizeof(qp->name), + "zuc_pmd_%u_qp_%u", + dev->data->dev_id, qp->id); + + if (n >= sizeof(qp->name)) + return -1; + + return 0; +} + +/** Create a ring to place processed ops on */ +static struct rte_ring * +zuc_pmd_qp_create_processed_ops_ring(struct zuc_qp *qp, + unsigned ring_size, int socket_id) +{ + struct rte_ring *r; + + r = rte_ring_lookup(qp->name); + if (r) { + if (rte_ring_get_size(r) >= ring_size) { + ZUC_LOG_INFO("Reusing existing ring %s" + " for processed packets", + qp->name); + return r; + } + + ZUC_LOG_ERR("Unable to reuse existing ring %s" + " for processed packets", + qp->name); + return NULL; + } + + return rte_ring_create(qp->name, ring_size, socket_id, + RING_F_SP_ENQ | RING_F_SC_DEQ); +} + +/** Setup a queue pair */ +static int +zuc_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, + const struct rte_cryptodev_qp_conf *qp_conf, + int socket_id, struct rte_mempool *session_pool) +{ + struct zuc_qp *qp = NULL; + + /* Free memory prior to re-allocation if needed. */ + if (dev->data->queue_pairs[qp_id] != NULL) + zuc_pmd_qp_release(dev, qp_id); + + /* Allocate the queue pair data structure. */ + qp = rte_zmalloc_socket("ZUC PMD Queue Pair", sizeof(*qp), + RTE_CACHE_LINE_SIZE, socket_id); + if (qp == NULL) + return (-ENOMEM); + + qp->id = qp_id; + dev->data->queue_pairs[qp_id] = qp; + + if (zuc_pmd_qp_set_unique_name(dev, qp)) + goto qp_setup_cleanup; + + qp->processed_ops = zuc_pmd_qp_create_processed_ops_ring(qp, + qp_conf->nb_descriptors, socket_id); + if (qp->processed_ops == NULL) + goto qp_setup_cleanup; + + qp->sess_mp = session_pool; + + memset(&qp->qp_stats, 0, sizeof(qp->qp_stats)); + + return 0; + +qp_setup_cleanup: + if (qp) + rte_free(qp); + + return -1; +} + +/** Start queue pair */ +static int +zuc_pmd_qp_start(__rte_unused struct rte_cryptodev *dev, + __rte_unused uint16_t queue_pair_id) +{ + return -ENOTSUP; +} + +/** Stop queue pair */ +static int +zuc_pmd_qp_stop(__rte_unused struct rte_cryptodev *dev, + __rte_unused uint16_t queue_pair_id) +{ + return -ENOTSUP; +} + +/** Return the number of allocated queue pairs */ +static uint32_t +zuc_pmd_qp_count(struct rte_cryptodev *dev) +{ + return dev->data->nb_queue_pairs; +} + +/** Returns the size of the ZUC session structure */ +static unsigned +zuc_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused) +{ + return sizeof(struct zuc_session); +} + +/** Configure a ZUC session from a crypto xform chain */ +static int +zuc_pmd_session_configure(struct rte_cryptodev *dev __rte_unused, + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool) +{ + void *sess_private_data; + int ret; + + if (unlikely(sess == NULL)) { + ZUC_LOG_ERR("invalid session struct"); + return -EINVAL; + } + + if (rte_mempool_get(mempool, &sess_private_data)) { + CDEV_LOG_ERR( + "Couldn't get object from session mempool"); + return -ENOMEM; + } + + ret = zuc_set_session_parameters(sess_private_data, xform); + if (ret != 0) { + ZUC_LOG_ERR("failed configure session parameters"); + + /* Return session to mempool */ + rte_mempool_put(mempool, sess_private_data); + return ret; + } + + set_session_private_data(sess, dev->driver_id, + sess_private_data); + + return 0; +} + +/** Clear the memory of session so it doesn't leave key material behind */ +static void +zuc_pmd_session_clear(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) +{ + uint8_t index = dev->driver_id; + void *sess_priv = get_session_private_data(sess, index); + + /* Zero out the whole structure */ + if (sess_priv) { + memset(sess_priv, 0, sizeof(struct zuc_session)); + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + set_session_private_data(sess, index, NULL); + rte_mempool_put(sess_mp, sess_priv); + } +} + +struct rte_cryptodev_ops zuc_pmd_ops = { + .dev_configure = zuc_pmd_config, + .dev_start = zuc_pmd_start, + .dev_stop = zuc_pmd_stop, + .dev_close = zuc_pmd_close, + + .stats_get = zuc_pmd_stats_get, + .stats_reset = zuc_pmd_stats_reset, + + .dev_infos_get = zuc_pmd_info_get, + + .queue_pair_setup = zuc_pmd_qp_setup, + .queue_pair_release = zuc_pmd_qp_release, + .queue_pair_start = zuc_pmd_qp_start, + .queue_pair_stop = zuc_pmd_qp_stop, + .queue_pair_count = zuc_pmd_qp_count, + + .session_get_size = zuc_pmd_session_get_size, + .session_configure = zuc_pmd_session_configure, + .session_clear = zuc_pmd_session_clear +}; + +struct rte_cryptodev_ops *rte_zuc_pmd_ops = &zuc_pmd_ops; diff --git a/dpdk/drivers/crypto/zuc/rte_zuc_pmd_private.h b/dpdk/drivers/crypto/zuc/rte_zuc_pmd_private.h new file mode 100644 index 00000000..a57b8cd0 --- /dev/null +++ b/dpdk/drivers/crypto/zuc/rte_zuc_pmd_private.h @@ -0,0 +1,120 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_ZUC_PMD_PRIVATE_H_ +#define _RTE_ZUC_PMD_PRIVATE_H_ + +#include + +#define CRYPTODEV_NAME_ZUC_PMD crypto_zuc +/**< KASUMI PMD device name */ + +#define ZUC_LOG_ERR(fmt, args...) \ + RTE_LOG(ERR, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \ + RTE_STR(CRYPTODEV_NAME_ZUC_PMD), \ + __func__, __LINE__, ## args) + +#ifdef RTE_LIBRTE_ZUC_DEBUG +#define ZUC_LOG_INFO(fmt, args...) \ + RTE_LOG(INFO, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \ + RTE_STR(CRYPTODEV_NAME_ZUC_PMD), \ + __func__, __LINE__, ## args) + +#define ZUC_LOG_DBG(fmt, args...) \ + RTE_LOG(DEBUG, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \ + RTE_STR(CRYPTODEV_NAME_ZUC_PMD), \ + __func__, __LINE__, ## args) +#else +#define ZUC_LOG_INFO(fmt, args...) +#define ZUC_LOG_DBG(fmt, args...) +#endif + +#define ZUC_IV_KEY_LENGTH 16 +#define ZUC_DIGEST_LENGTH 4 + +/** private data structure for each virtual ZUC device */ +struct zuc_private { + unsigned max_nb_queue_pairs; + /**< Max number of queue pairs supported by device */ + unsigned max_nb_sessions; + /**< Max number of sessions supported by device */ +}; + +/** ZUC buffer queue pair */ +struct zuc_qp { + uint16_t id; + /**< Queue Pair Identifier */ + char name[RTE_CRYPTODEV_NAME_LEN]; + /**< Unique Queue Pair Name */ + struct rte_ring *processed_ops; + /**< Ring for placing processed ops */ + struct rte_mempool *sess_mp; + /**< Session Mempool */ + struct rte_cryptodev_stats qp_stats; + /**< Queue pair statistics */ + uint8_t temp_digest[ZUC_DIGEST_LENGTH]; + /**< Buffer used to store the digest generated + * by the driver when verifying a digest provided + * by the user (using authentication verify operation) + */ +} __rte_cache_aligned; + +enum zuc_operation { + ZUC_OP_ONLY_CIPHER, + ZUC_OP_ONLY_AUTH, + ZUC_OP_CIPHER_AUTH, + ZUC_OP_AUTH_CIPHER, + ZUC_OP_NOT_SUPPORTED +}; + +/** ZUC private session structure */ +struct zuc_session { + enum zuc_operation op; + enum rte_crypto_auth_operation auth_op; + uint8_t pKey_cipher[ZUC_IV_KEY_LENGTH]; + uint8_t pKey_hash[ZUC_IV_KEY_LENGTH]; + uint16_t cipher_iv_offset; + uint16_t auth_iv_offset; +} __rte_cache_aligned; + + +extern int +zuc_set_session_parameters(struct zuc_session *sess, + const struct rte_crypto_sym_xform *xform); + + +/** device specific operations function pointer structure */ +extern struct rte_cryptodev_ops *rte_zuc_pmd_ops; + + + +#endif /* _RTE_ZUC_PMD_PRIVATE_H_ */ diff --git a/dpdk/drivers/event/Makefile b/dpdk/drivers/event/Makefile new file mode 100644 index 00000000..1f9c0ba2 --- /dev/null +++ b/dpdk/drivers/event/Makefile @@ -0,0 +1,39 @@ +# BSD LICENSE +# +# Copyright(c) 2016 Cavium, Inc. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Cavium, Inc nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +include $(RTE_SDK)/mk/rte.vars.mk + +DIRS-$(CONFIG_RTE_LIBRTE_PMD_SKELETON_EVENTDEV) += skeleton +DIRS-$(CONFIG_RTE_LIBRTE_PMD_SW_EVENTDEV) += sw +DIRS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX_SSOVF) += octeontx +DIRS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_EVENTDEV) += dpaa2 + +include $(RTE_SDK)/mk/rte.subdir.mk diff --git a/dpdk/drivers/event/dpaa2/Makefile b/dpdk/drivers/event/dpaa2/Makefile new file mode 100644 index 00000000..f34eebfa --- /dev/null +++ b/dpdk/drivers/event/dpaa2/Makefile @@ -0,0 +1,64 @@ +# BSD LICENSE +# +# Copyright 2017 NXP. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of NXP nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +include $(RTE_SDK)/mk/rte.vars.mk + +# +# library name +# +LIB = librte_pmd_dpaa2_event.a + +CFLAGS += $(WERROR_FLAGS) + +CFLAGS += -I$(RTE_SDK)/drivers/bus/fslmc +CFLAGS += -I$(RTE_SDK)/drivers/bus/fslmc/qbman/include +CFLAGS += -I$(RTE_SDK)/drivers/bus/fslmc/mc +CFLAGS += -I$(RTE_SDK)/drivers/bus/fslmc/portal +CFLAGS += -I$(RTE_SDK)/drivers/mempool/dpaa2 +CFLAGS += -I$(RTE_SDK)/drivers/event/dpaa2 +CFLAGS += -I$(RTE_SDK)/lib/librte_eal/linuxapp/eal +LDLIBS += -lrte_eal -lrte_eventdev -lrte_bus_fslmc -lrte_pmd_dpaa2 +LDLIBS += -lrte_bus_vdev +CFLAGS += -I$(RTE_SDK)/drivers/net/dpaa2 +CFLAGS += -I$(RTE_SDK)/drivers/net/dpaa2/mc + +# versioning export map +EXPORT_MAP := rte_pmd_dpaa2_event_version.map + +LIBABIVER := 1 + +# +# all source are stored in SRCS-y +# +SRCS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_EVENTDEV) += dpaa2_hw_dpcon.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_EVENTDEV) += dpaa2_eventdev.c + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/event/dpaa2/dpaa2_eventdev.c b/dpdk/drivers/event/dpaa2/dpaa2_eventdev.c new file mode 100644 index 00000000..eeeb2312 --- /dev/null +++ b/dpdk/drivers/event/dpaa2/dpaa2_eventdev.c @@ -0,0 +1,854 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of NXP nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include "dpaa2_eventdev.h" +#include +#include + +/* Clarifications + * Evendev = SoC Instance + * Eventport = DPIO Instance + * Eventqueue = DPCON Instance + * 1 Eventdev can have N Eventqueue + * Soft Event Flow is DPCI Instance + */ + +static uint16_t +dpaa2_eventdev_enqueue_burst(void *port, const struct rte_event ev[], + uint16_t nb_events) +{ + struct rte_eventdev *ev_dev = + ((struct dpaa2_io_portal_t *)port)->eventdev; + struct dpaa2_eventdev *priv = ev_dev->data->dev_private; + uint32_t queue_id = ev[0].queue_id; + struct evq_info_t *evq_info = &priv->evq_info[queue_id]; + uint32_t fqid; + struct qbman_swp *swp; + struct qbman_fd fd_arr[MAX_TX_RING_SLOTS]; + uint32_t loop, frames_to_send; + struct qbman_eq_desc eqdesc[MAX_TX_RING_SLOTS]; + uint16_t num_tx = 0; + int ret; + + RTE_SET_USED(port); + + if (unlikely(!DPAA2_PER_LCORE_DPIO)) { + ret = dpaa2_affine_qbman_swp(); + if (ret) { + PMD_DRV_LOG(ERR, "Failure in affining portal\n"); + return 0; + } + } + + swp = DPAA2_PER_LCORE_PORTAL; + + while (nb_events) { + frames_to_send = (nb_events >> 3) ? + MAX_TX_RING_SLOTS : nb_events; + + for (loop = 0; loop < frames_to_send; loop++) { + const struct rte_event *event = &ev[num_tx + loop]; + + if (event->sched_type != RTE_SCHED_TYPE_ATOMIC) + fqid = evq_info->dpci->queue[ + DPAA2_EVENT_DPCI_PARALLEL_QUEUE].fqid; + else + fqid = evq_info->dpci->queue[ + DPAA2_EVENT_DPCI_ATOMIC_QUEUE].fqid; + + /* Prepare enqueue descriptor */ + qbman_eq_desc_clear(&eqdesc[loop]); + qbman_eq_desc_set_fq(&eqdesc[loop], fqid); + qbman_eq_desc_set_no_orp(&eqdesc[loop], 0); + qbman_eq_desc_set_response(&eqdesc[loop], 0, 0); + + if (event->impl_opaque) { + uint8_t dqrr_index = event->impl_opaque - 1; + + qbman_eq_desc_set_dca(&eqdesc[loop], 1, + dqrr_index, 0); + DPAA2_PER_LCORE_DPIO->dqrr_size--; + DPAA2_PER_LCORE_DPIO->dqrr_held &= + ~(1 << dqrr_index); + } + + memset(&fd_arr[loop], 0, sizeof(struct qbman_fd)); + + /* + * todo - need to align with hw context data + * to avoid copy + */ + struct rte_event *ev_temp = rte_malloc(NULL, + sizeof(struct rte_event), 0); + + if (!ev_temp) { + if (!loop) + return num_tx; + frames_to_send = loop; + PMD_DRV_LOG(ERR, "Unable to allocate memory"); + goto send_partial; + } + rte_memcpy(ev_temp, event, sizeof(struct rte_event)); + DPAA2_SET_FD_ADDR((&fd_arr[loop]), ev_temp); + DPAA2_SET_FD_LEN((&fd_arr[loop]), + sizeof(struct rte_event)); + } +send_partial: + loop = 0; + while (loop < frames_to_send) { + loop += qbman_swp_enqueue_multiple_desc(swp, + &eqdesc[loop], &fd_arr[loop], + frames_to_send - loop); + } + num_tx += frames_to_send; + nb_events -= frames_to_send; + } + + return num_tx; +} + +static uint16_t +dpaa2_eventdev_enqueue(void *port, const struct rte_event *ev) +{ + return dpaa2_eventdev_enqueue_burst(port, ev, 1); +} + +static void dpaa2_eventdev_dequeue_wait(uint64_t timeout_ticks) +{ + struct epoll_event epoll_ev; + int ret, i = 0; + + qbman_swp_interrupt_clear_status(DPAA2_PER_LCORE_PORTAL, + QBMAN_SWP_INTERRUPT_DQRI); + +RETRY: + ret = epoll_wait(DPAA2_PER_LCORE_DPIO->epoll_fd, + &epoll_ev, 1, timeout_ticks); + if (ret < 1) { + /* sometimes due to some spurious interrupts epoll_wait fails + * with errno EINTR. so here we are retrying epoll_wait in such + * case to avoid the problem. + */ + if (errno == EINTR) { + PMD_DRV_LOG(DEBUG, "epoll_wait fails\n"); + if (i++ > 10) + PMD_DRV_LOG(DEBUG, "Dequeue burst Failed\n"); + goto RETRY; + } + } +} + +static void dpaa2_eventdev_process_parallel(struct qbman_swp *swp, + const struct qbman_fd *fd, + const struct qbman_result *dq, + struct dpaa2_queue *rxq, + struct rte_event *ev) +{ + struct rte_event *ev_temp = + (struct rte_event *)DPAA2_GET_FD_ADDR(fd); + + RTE_SET_USED(rxq); + + rte_memcpy(ev, ev_temp, sizeof(struct rte_event)); + rte_free(ev_temp); + + qbman_swp_dqrr_consume(swp, dq); +} + +static void dpaa2_eventdev_process_atomic(struct qbman_swp *swp, + const struct qbman_fd *fd, + const struct qbman_result *dq, + struct dpaa2_queue *rxq, + struct rte_event *ev) +{ + struct rte_event *ev_temp = + (struct rte_event *)DPAA2_GET_FD_ADDR(fd); + uint8_t dqrr_index = qbman_get_dqrr_idx(dq); + + RTE_SET_USED(swp); + RTE_SET_USED(rxq); + + rte_memcpy(ev, ev_temp, sizeof(struct rte_event)); + rte_free(ev_temp); + ev->impl_opaque = dqrr_index + 1; + DPAA2_PER_LCORE_DPIO->dqrr_size++; + DPAA2_PER_LCORE_DPIO->dqrr_held |= 1 << dqrr_index; +} + +static uint16_t +dpaa2_eventdev_dequeue_burst(void *port, struct rte_event ev[], + uint16_t nb_events, uint64_t timeout_ticks) +{ + const struct qbman_result *dq; + struct qbman_swp *swp; + const struct qbman_fd *fd; + struct dpaa2_queue *rxq; + int num_pkts = 0, ret, i = 0; + + RTE_SET_USED(port); + + if (unlikely(!DPAA2_PER_LCORE_DPIO)) { + ret = dpaa2_affine_qbman_swp(); + if (ret) { + PMD_DRV_LOG(ERR, "Failure in affining portal\n"); + return 0; + } + } + + swp = DPAA2_PER_LCORE_PORTAL; + + /* Check if there are atomic contexts to be released */ + while (DPAA2_PER_LCORE_DPIO->dqrr_size) { + if (DPAA2_PER_LCORE_DPIO->dqrr_held & (1 << i)) { + dq = qbman_get_dqrr_from_idx(swp, i); + qbman_swp_dqrr_consume(swp, dq); + DPAA2_PER_LCORE_DPIO->dqrr_size--; + } + i++; + } + DPAA2_PER_LCORE_DPIO->dqrr_held = 0; + + do { + dq = qbman_swp_dqrr_next(swp); + if (!dq) { + if (!num_pkts && timeout_ticks) { + dpaa2_eventdev_dequeue_wait(timeout_ticks); + timeout_ticks = 0; + continue; + } + return num_pkts; + } + + fd = qbman_result_DQ_fd(dq); + + rxq = (struct dpaa2_queue *)qbman_result_DQ_fqd_ctx(dq); + if (rxq) { + rxq->cb(swp, fd, dq, rxq, &ev[num_pkts]); + } else { + qbman_swp_dqrr_consume(swp, dq); + PMD_DRV_LOG(ERR, "Null Return VQ received\n"); + return 0; + } + + num_pkts++; + } while (num_pkts < nb_events); + + return num_pkts; +} + +static uint16_t +dpaa2_eventdev_dequeue(void *port, struct rte_event *ev, + uint64_t timeout_ticks) +{ + return dpaa2_eventdev_dequeue_burst(port, ev, 1, timeout_ticks); +} + +static void +dpaa2_eventdev_info_get(struct rte_eventdev *dev, + struct rte_event_dev_info *dev_info) +{ + struct dpaa2_eventdev *priv = dev->data->dev_private; + + PMD_DRV_FUNC_TRACE(); + + RTE_SET_USED(dev); + + memset(dev_info, 0, sizeof(struct rte_event_dev_info)); + dev_info->min_dequeue_timeout_ns = + DPAA2_EVENT_MIN_DEQUEUE_TIMEOUT; + dev_info->max_dequeue_timeout_ns = + DPAA2_EVENT_MAX_DEQUEUE_TIMEOUT; + dev_info->dequeue_timeout_ns = + DPAA2_EVENT_MIN_DEQUEUE_TIMEOUT; + dev_info->max_event_queues = priv->max_event_queues; + dev_info->max_event_queue_flows = + DPAA2_EVENT_MAX_QUEUE_FLOWS; + dev_info->max_event_queue_priority_levels = + DPAA2_EVENT_MAX_QUEUE_PRIORITY_LEVELS; + dev_info->max_event_priority_levels = + DPAA2_EVENT_MAX_EVENT_PRIORITY_LEVELS; + dev_info->max_event_ports = RTE_MAX_LCORE; + dev_info->max_event_port_dequeue_depth = + DPAA2_EVENT_MAX_PORT_DEQUEUE_DEPTH; + dev_info->max_event_port_enqueue_depth = + DPAA2_EVENT_MAX_PORT_ENQUEUE_DEPTH; + dev_info->max_num_events = DPAA2_EVENT_MAX_NUM_EVENTS; + dev_info->event_dev_cap = RTE_EVENT_DEV_CAP_DISTRIBUTED_SCHED | + RTE_EVENT_DEV_CAP_BURST_MODE; +} + +static int +dpaa2_eventdev_configure(const struct rte_eventdev *dev) +{ + struct dpaa2_eventdev *priv = dev->data->dev_private; + struct rte_event_dev_config *conf = &dev->data->dev_conf; + + PMD_DRV_FUNC_TRACE(); + + priv->dequeue_timeout_ns = conf->dequeue_timeout_ns; + priv->nb_event_queues = conf->nb_event_queues; + priv->nb_event_ports = conf->nb_event_ports; + priv->nb_event_queue_flows = conf->nb_event_queue_flows; + priv->nb_event_port_dequeue_depth = conf->nb_event_port_dequeue_depth; + priv->nb_event_port_enqueue_depth = conf->nb_event_port_enqueue_depth; + priv->event_dev_cfg = conf->event_dev_cfg; + + PMD_DRV_LOG(DEBUG, "Configured eventdev devid=%d", dev->data->dev_id); + return 0; +} + +static int +dpaa2_eventdev_start(struct rte_eventdev *dev) +{ + PMD_DRV_FUNC_TRACE(); + + RTE_SET_USED(dev); + + return 0; +} + +static void +dpaa2_eventdev_stop(struct rte_eventdev *dev) +{ + PMD_DRV_FUNC_TRACE(); + + RTE_SET_USED(dev); +} + +static int +dpaa2_eventdev_close(struct rte_eventdev *dev) +{ + PMD_DRV_FUNC_TRACE(); + + RTE_SET_USED(dev); + + return 0; +} + +static void +dpaa2_eventdev_queue_def_conf(struct rte_eventdev *dev, uint8_t queue_id, + struct rte_event_queue_conf *queue_conf) +{ + PMD_DRV_FUNC_TRACE(); + + RTE_SET_USED(dev); + RTE_SET_USED(queue_id); + RTE_SET_USED(queue_conf); + + queue_conf->nb_atomic_flows = DPAA2_EVENT_QUEUE_ATOMIC_FLOWS; + queue_conf->schedule_type = RTE_SCHED_TYPE_ATOMIC | + RTE_SCHED_TYPE_PARALLEL; + queue_conf->priority = RTE_EVENT_DEV_PRIORITY_NORMAL; +} + +static void +dpaa2_eventdev_queue_release(struct rte_eventdev *dev, uint8_t queue_id) +{ + PMD_DRV_FUNC_TRACE(); + + RTE_SET_USED(dev); + RTE_SET_USED(queue_id); +} + +static int +dpaa2_eventdev_queue_setup(struct rte_eventdev *dev, uint8_t queue_id, + const struct rte_event_queue_conf *queue_conf) +{ + struct dpaa2_eventdev *priv = dev->data->dev_private; + struct evq_info_t *evq_info = + &priv->evq_info[queue_id]; + + PMD_DRV_FUNC_TRACE(); + + evq_info->event_queue_cfg = queue_conf->event_queue_cfg; + + return 0; +} + +static void +dpaa2_eventdev_port_def_conf(struct rte_eventdev *dev, uint8_t port_id, + struct rte_event_port_conf *port_conf) +{ + PMD_DRV_FUNC_TRACE(); + + RTE_SET_USED(dev); + RTE_SET_USED(port_id); + RTE_SET_USED(port_conf); + + port_conf->new_event_threshold = + DPAA2_EVENT_MAX_NUM_EVENTS; + port_conf->dequeue_depth = + DPAA2_EVENT_MAX_PORT_DEQUEUE_DEPTH; + port_conf->enqueue_depth = + DPAA2_EVENT_MAX_PORT_ENQUEUE_DEPTH; +} + +static void +dpaa2_eventdev_port_release(void *port) +{ + PMD_DRV_FUNC_TRACE(); + + RTE_SET_USED(port); +} + +static int +dpaa2_eventdev_port_setup(struct rte_eventdev *dev, uint8_t port_id, + const struct rte_event_port_conf *port_conf) +{ + PMD_DRV_FUNC_TRACE(); + + RTE_SET_USED(port_conf); + + if (!dpaa2_io_portal[port_id].dpio_dev) { + dpaa2_io_portal[port_id].dpio_dev = + dpaa2_get_qbman_swp(port_id); + rte_atomic16_inc(&dpaa2_io_portal[port_id].dpio_dev->ref_count); + if (!dpaa2_io_portal[port_id].dpio_dev) + return -1; + } + + dpaa2_io_portal[port_id].eventdev = dev; + dev->data->ports[port_id] = &dpaa2_io_portal[port_id]; + return 0; +} + +static int +dpaa2_eventdev_port_unlink(struct rte_eventdev *dev, void *port, + uint8_t queues[], uint16_t nb_unlinks) +{ + struct dpaa2_eventdev *priv = dev->data->dev_private; + struct dpaa2_io_portal_t *dpaa2_portal = port; + struct evq_info_t *evq_info; + int i; + + PMD_DRV_FUNC_TRACE(); + + for (i = 0; i < nb_unlinks; i++) { + evq_info = &priv->evq_info[queues[i]]; + qbman_swp_push_set(dpaa2_portal->dpio_dev->sw_portal, + evq_info->dpcon->channel_index, 0); + dpio_remove_static_dequeue_channel(dpaa2_portal->dpio_dev->dpio, + 0, dpaa2_portal->dpio_dev->token, + evq_info->dpcon->dpcon_id); + evq_info->link = 0; + } + + return (int)nb_unlinks; +} + +static int +dpaa2_eventdev_port_link(struct rte_eventdev *dev, void *port, + const uint8_t queues[], const uint8_t priorities[], + uint16_t nb_links) +{ + struct dpaa2_eventdev *priv = dev->data->dev_private; + struct dpaa2_io_portal_t *dpaa2_portal = port; + struct evq_info_t *evq_info; + uint8_t channel_index; + int ret, i, n; + + PMD_DRV_FUNC_TRACE(); + + for (i = 0; i < nb_links; i++) { + evq_info = &priv->evq_info[queues[i]]; + if (evq_info->link) + continue; + + ret = dpio_add_static_dequeue_channel( + dpaa2_portal->dpio_dev->dpio, + CMD_PRI_LOW, dpaa2_portal->dpio_dev->token, + evq_info->dpcon->dpcon_id, &channel_index); + if (ret < 0) { + PMD_DRV_ERR("Static dequeue cfg failed with ret: %d\n", + ret); + goto err; + } + + qbman_swp_push_set(dpaa2_portal->dpio_dev->sw_portal, + channel_index, 1); + evq_info->dpcon->channel_index = channel_index; + evq_info->link = 1; + } + + RTE_SET_USED(priorities); + + return (int)nb_links; +err: + for (n = 0; n < i; n++) { + evq_info = &priv->evq_info[queues[n]]; + qbman_swp_push_set(dpaa2_portal->dpio_dev->sw_portal, + evq_info->dpcon->channel_index, 0); + dpio_remove_static_dequeue_channel(dpaa2_portal->dpio_dev->dpio, + 0, dpaa2_portal->dpio_dev->token, + evq_info->dpcon->dpcon_id); + evq_info->link = 0; + } + return ret; +} + +static int +dpaa2_eventdev_timeout_ticks(struct rte_eventdev *dev, uint64_t ns, + uint64_t *timeout_ticks) +{ + uint32_t scale = 1; + + PMD_DRV_FUNC_TRACE(); + + RTE_SET_USED(dev); + *timeout_ticks = ns * scale; + + return 0; +} + +static void +dpaa2_eventdev_dump(struct rte_eventdev *dev, FILE *f) +{ + PMD_DRV_FUNC_TRACE(); + + RTE_SET_USED(dev); + RTE_SET_USED(f); +} + +static int +dpaa2_eventdev_eth_caps_get(const struct rte_eventdev *dev, + const struct rte_eth_dev *eth_dev, + uint32_t *caps) +{ + const char *ethdev_driver = eth_dev->device->driver->name; + + PMD_DRV_FUNC_TRACE(); + + RTE_SET_USED(dev); + + if (!strcmp(ethdev_driver, "net_dpaa2")) + *caps = RTE_EVENT_ETH_RX_ADAPTER_DPAA2_CAP; + else + *caps = RTE_EVENT_ETH_RX_ADAPTER_SW_CAP; + + return 0; +} + +static int +dpaa2_eventdev_eth_queue_add_all(const struct rte_eventdev *dev, + const struct rte_eth_dev *eth_dev, + const struct rte_event_eth_rx_adapter_queue_conf *queue_conf) +{ + struct dpaa2_eventdev *priv = dev->data->dev_private; + uint8_t ev_qid = queue_conf->ev.queue_id; + uint16_t dpcon_id = priv->evq_info[ev_qid].dpcon->dpcon_id; + int i, ret; + + PMD_DRV_FUNC_TRACE(); + + for (i = 0; i < eth_dev->data->nb_rx_queues; i++) { + ret = dpaa2_eth_eventq_attach(eth_dev, i, + dpcon_id, queue_conf); + if (ret) { + PMD_DRV_ERR("dpaa2_eth_eventq_attach failed: ret %d\n", + ret); + goto fail; + } + } + return 0; +fail: + for (i = (i - 1); i >= 0 ; i--) + dpaa2_eth_eventq_detach(eth_dev, i); + + return ret; +} + +static int +dpaa2_eventdev_eth_queue_add(const struct rte_eventdev *dev, + const struct rte_eth_dev *eth_dev, + int32_t rx_queue_id, + const struct rte_event_eth_rx_adapter_queue_conf *queue_conf) +{ + struct dpaa2_eventdev *priv = dev->data->dev_private; + uint8_t ev_qid = queue_conf->ev.queue_id; + uint16_t dpcon_id = priv->evq_info[ev_qid].dpcon->dpcon_id; + int ret; + + PMD_DRV_FUNC_TRACE(); + + if (rx_queue_id == -1) + return dpaa2_eventdev_eth_queue_add_all(dev, + eth_dev, queue_conf); + + ret = dpaa2_eth_eventq_attach(eth_dev, rx_queue_id, + dpcon_id, queue_conf); + if (ret) { + PMD_DRV_ERR("dpaa2_eth_eventq_attach failed: ret: %d\n", ret); + return ret; + } + return 0; +} + +static int +dpaa2_eventdev_eth_queue_del_all(const struct rte_eventdev *dev, + const struct rte_eth_dev *eth_dev) +{ + int i, ret; + + PMD_DRV_FUNC_TRACE(); + + RTE_SET_USED(dev); + + for (i = 0; i < eth_dev->data->nb_rx_queues; i++) { + ret = dpaa2_eth_eventq_detach(eth_dev, i); + if (ret) { + PMD_DRV_ERR("dpaa2_eth_eventq_detach failed: ret %d\n", + ret); + return ret; + } + } + + return 0; +} + +static int +dpaa2_eventdev_eth_queue_del(const struct rte_eventdev *dev, + const struct rte_eth_dev *eth_dev, + int32_t rx_queue_id) +{ + int ret; + + PMD_DRV_FUNC_TRACE(); + + if (rx_queue_id == -1) + return dpaa2_eventdev_eth_queue_del_all(dev, eth_dev); + + ret = dpaa2_eth_eventq_detach(eth_dev, rx_queue_id); + if (ret) { + PMD_DRV_ERR("dpaa2_eth_eventq_detach failed: ret: %d\n", ret); + return ret; + } + + return 0; +} + +static int +dpaa2_eventdev_eth_start(const struct rte_eventdev *dev, + const struct rte_eth_dev *eth_dev) +{ + PMD_DRV_FUNC_TRACE(); + + RTE_SET_USED(dev); + RTE_SET_USED(eth_dev); + + return 0; +} + +static int +dpaa2_eventdev_eth_stop(const struct rte_eventdev *dev, + const struct rte_eth_dev *eth_dev) +{ + PMD_DRV_FUNC_TRACE(); + + RTE_SET_USED(dev); + RTE_SET_USED(eth_dev); + + return 0; +} + +static const struct rte_eventdev_ops dpaa2_eventdev_ops = { + .dev_infos_get = dpaa2_eventdev_info_get, + .dev_configure = dpaa2_eventdev_configure, + .dev_start = dpaa2_eventdev_start, + .dev_stop = dpaa2_eventdev_stop, + .dev_close = dpaa2_eventdev_close, + .queue_def_conf = dpaa2_eventdev_queue_def_conf, + .queue_setup = dpaa2_eventdev_queue_setup, + .queue_release = dpaa2_eventdev_queue_release, + .port_def_conf = dpaa2_eventdev_port_def_conf, + .port_setup = dpaa2_eventdev_port_setup, + .port_release = dpaa2_eventdev_port_release, + .port_link = dpaa2_eventdev_port_link, + .port_unlink = dpaa2_eventdev_port_unlink, + .timeout_ticks = dpaa2_eventdev_timeout_ticks, + .dump = dpaa2_eventdev_dump, + .eth_rx_adapter_caps_get = dpaa2_eventdev_eth_caps_get, + .eth_rx_adapter_queue_add = dpaa2_eventdev_eth_queue_add, + .eth_rx_adapter_queue_del = dpaa2_eventdev_eth_queue_del, + .eth_rx_adapter_start = dpaa2_eventdev_eth_start, + .eth_rx_adapter_stop = dpaa2_eventdev_eth_stop, +}; + +static int +dpaa2_eventdev_setup_dpci(struct dpaa2_dpci_dev *dpci_dev, + struct dpaa2_dpcon_dev *dpcon_dev) +{ + struct dpci_rx_queue_cfg rx_queue_cfg; + int ret, i; + + /*Do settings to get the frame on a DPCON object*/ + rx_queue_cfg.options = DPCI_QUEUE_OPT_DEST | + DPCI_QUEUE_OPT_USER_CTX; + rx_queue_cfg.dest_cfg.dest_type = DPCI_DEST_DPCON; + rx_queue_cfg.dest_cfg.dest_id = dpcon_dev->dpcon_id; + rx_queue_cfg.dest_cfg.priority = DPAA2_EVENT_DEFAULT_DPCI_PRIO; + + dpci_dev->queue[DPAA2_EVENT_DPCI_PARALLEL_QUEUE].cb = + dpaa2_eventdev_process_parallel; + dpci_dev->queue[DPAA2_EVENT_DPCI_ATOMIC_QUEUE].cb = + dpaa2_eventdev_process_atomic; + + for (i = 0 ; i < DPAA2_EVENT_DPCI_MAX_QUEUES; i++) { + rx_queue_cfg.user_ctx = (uint64_t)(&dpci_dev->queue[i]); + ret = dpci_set_rx_queue(&dpci_dev->dpci, + CMD_PRI_LOW, + dpci_dev->token, i, + &rx_queue_cfg); + if (ret) { + PMD_DRV_LOG(ERR, + "set_rx_q failed with err code: %d", ret); + return ret; + } + } + return 0; +} + +static int +dpaa2_eventdev_create(const char *name) +{ + struct rte_eventdev *eventdev; + struct dpaa2_eventdev *priv; + struct dpaa2_dpcon_dev *dpcon_dev = NULL; + struct dpaa2_dpci_dev *dpci_dev = NULL; + int ret; + + eventdev = rte_event_pmd_vdev_init(name, + sizeof(struct dpaa2_eventdev), + rte_socket_id()); + if (eventdev == NULL) { + PMD_DRV_ERR("Failed to create eventdev vdev %s", name); + goto fail; + } + + eventdev->dev_ops = &dpaa2_eventdev_ops; + eventdev->enqueue = dpaa2_eventdev_enqueue; + eventdev->enqueue_burst = dpaa2_eventdev_enqueue_burst; + eventdev->enqueue_new_burst = dpaa2_eventdev_enqueue_burst; + eventdev->enqueue_forward_burst = dpaa2_eventdev_enqueue_burst; + eventdev->dequeue = dpaa2_eventdev_dequeue; + eventdev->dequeue_burst = dpaa2_eventdev_dequeue_burst; + + /* For secondary processes, the primary has done all the work */ + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return 0; + + priv = eventdev->data->dev_private; + priv->max_event_queues = 0; + + do { + dpcon_dev = rte_dpaa2_alloc_dpcon_dev(); + if (!dpcon_dev) + break; + priv->evq_info[priv->max_event_queues].dpcon = dpcon_dev; + + dpci_dev = rte_dpaa2_alloc_dpci_dev(); + if (!dpci_dev) { + rte_dpaa2_free_dpcon_dev(dpcon_dev); + break; + } + priv->evq_info[priv->max_event_queues].dpci = dpci_dev; + + ret = dpaa2_eventdev_setup_dpci(dpci_dev, dpcon_dev); + if (ret) { + PMD_DRV_LOG(ERR, + "dpci setup failed with err code: %d", ret); + return ret; + } + priv->max_event_queues++; + } while (dpcon_dev && dpci_dev); + + return 0; +fail: + return -EFAULT; +} + +static int +dpaa2_eventdev_probe(struct rte_vdev_device *vdev) +{ + const char *name; + + name = rte_vdev_device_name(vdev); + PMD_DRV_LOG(INFO, "Initializing %s", name); + return dpaa2_eventdev_create(name); +} + +static int +dpaa2_eventdev_remove(struct rte_vdev_device *vdev) +{ + const char *name; + + name = rte_vdev_device_name(vdev); + PMD_DRV_LOG(INFO, "Closing %s", name); + + return rte_event_pmd_vdev_uninit(name); +} + +static struct rte_vdev_driver vdev_eventdev_dpaa2_pmd = { + .probe = dpaa2_eventdev_probe, + .remove = dpaa2_eventdev_remove +}; + +RTE_PMD_REGISTER_VDEV(EVENTDEV_NAME_DPAA2_PMD, vdev_eventdev_dpaa2_pmd); diff --git a/dpdk/drivers/event/dpaa2/dpaa2_eventdev.h b/dpdk/drivers/event/dpaa2/dpaa2_eventdev.h new file mode 100644 index 00000000..ae8e07e9 --- /dev/null +++ b/dpdk/drivers/event/dpaa2/dpaa2_eventdev.h @@ -0,0 +1,122 @@ +/* + * BSD LICENSE + * + * Copyright 2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of NXP nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __DPAA2_EVENTDEV_H__ +#define __DPAA2_EVENTDEV_H__ + +#include +#include +#include +#include +#include + +#define EVENTDEV_NAME_DPAA2_PMD event_dpaa2 + +#ifdef RTE_LIBRTE_PMD_DPAA2_EVENTDEV_DEBUG +#define PMD_DRV_LOG(level, fmt, args...) \ + RTE_LOG(level, PMD, "%s(): " fmt "\n", __func__, ## args) +#define PMD_DRV_FUNC_TRACE() PMD_DRV_LOG(DEBUG, ">>") +#else +#define PMD_DRV_LOG(level, fmt, args...) do { } while (0) +#define PMD_DRV_FUNC_TRACE() do { } while (0) +#endif + +#define PMD_DRV_ERR(fmt, args...) \ + RTE_LOG(ERR, PMD, "%s(): " fmt "\n", __func__, ## args) + +#define DPAA2_EVENT_DEFAULT_DPCI_PRIO 0 + +#define DPAA2_EVENT_MAX_QUEUES 16 +#define DPAA2_EVENT_MIN_DEQUEUE_TIMEOUT 1 +#define DPAA2_EVENT_MAX_DEQUEUE_TIMEOUT (UINT32_MAX - 1) +#define DPAA2_EVENT_MAX_QUEUE_FLOWS 2048 +#define DPAA2_EVENT_MAX_QUEUE_PRIORITY_LEVELS 8 +#define DPAA2_EVENT_MAX_EVENT_PRIORITY_LEVELS 0 +#define DPAA2_EVENT_MAX_PORT_DEQUEUE_DEPTH 8 +#define DPAA2_EVENT_MAX_PORT_ENQUEUE_DEPTH 8 +#define DPAA2_EVENT_MAX_NUM_EVENTS (INT32_MAX - 1) + +#define DPAA2_EVENT_QUEUE_ATOMIC_FLOWS 2048 +#define DPAA2_EVENT_QUEUE_ORDER_SEQUENCES 2048 + +enum { + DPAA2_EVENT_DPCI_PARALLEL_QUEUE, + DPAA2_EVENT_DPCI_ATOMIC_QUEUE, + DPAA2_EVENT_DPCI_MAX_QUEUES +}; + +#define RTE_EVENT_ETH_RX_ADAPTER_DPAA2_CAP \ + (RTE_EVENT_ETH_RX_ADAPTER_CAP_INTERNAL_PORT | \ + RTE_EVENT_ETH_RX_ADAPTER_CAP_MULTI_EVENTQ | \ + RTE_EVENT_ETH_RX_ADAPTER_CAP_OVERRIDE_FLOW_ID) +/**< Ethernet Rx adapter cap to return If the packet transfers from + * the ethdev to eventdev with DPAA2 devices. + */ + +struct dpaa2_dpcon_dev { + TAILQ_ENTRY(dpaa2_dpcon_dev) next; + struct fsl_mc_io dpcon; + uint16_t token; + rte_atomic16_t in_use; + uint32_t dpcon_id; + uint16_t qbman_ch_id; + uint8_t num_priorities; + uint8_t channel_index; +}; + +struct evq_info_t { + /* DPcon device */ + struct dpaa2_dpcon_dev *dpcon; + /* Attached DPCI device */ + struct dpaa2_dpci_dev *dpci; + /* Configuration provided by the user */ + uint32_t event_queue_cfg; + uint8_t link; +}; + +struct dpaa2_eventdev { + struct evq_info_t evq_info[DPAA2_EVENT_MAX_QUEUES]; + uint32_t dequeue_timeout_ns; + uint8_t max_event_queues; + uint8_t nb_event_queues; + uint8_t nb_event_ports; + uint8_t resvd_1; + uint32_t nb_event_queue_flows; + uint32_t nb_event_port_dequeue_depth; + uint32_t nb_event_port_enqueue_depth; + uint32_t event_dev_cfg; +}; + +struct dpaa2_dpcon_dev *rte_dpaa2_alloc_dpcon_dev(void); +void rte_dpaa2_free_dpcon_dev(struct dpaa2_dpcon_dev *dpcon); + +#endif /* __DPAA2_EVENTDEV_H__ */ diff --git a/dpdk/drivers/event/dpaa2/dpaa2_hw_dpcon.c b/dpdk/drivers/event/dpaa2/dpaa2_hw_dpcon.c new file mode 100644 index 00000000..005e6234 --- /dev/null +++ b/dpdk/drivers/event/dpaa2/dpaa2_hw_dpcon.c @@ -0,0 +1,139 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of NXP nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include "dpaa2_eventdev.h" + +TAILQ_HEAD(dpcon_dev_list, dpaa2_dpcon_dev); +static struct dpcon_dev_list dpcon_dev_list + = TAILQ_HEAD_INITIALIZER(dpcon_dev_list); /*!< DPCON device list */ + +static int +rte_dpaa2_create_dpcon_device(int dev_fd __rte_unused, + struct vfio_device_info *obj_info __rte_unused, + int dpcon_id) +{ + struct dpaa2_dpcon_dev *dpcon_node; + struct dpcon_attr attr; + int ret; + + /* Allocate DPAA2 dpcon handle */ + dpcon_node = rte_malloc(NULL, sizeof(struct dpaa2_dpcon_dev), 0); + if (!dpcon_node) { + PMD_DRV_LOG(ERR, "Memory allocation failed for DPCON Device"); + return -1; + } + + /* Open the dpcon object */ + dpcon_node->dpcon.regs = rte_mcp_ptr_list[MC_PORTAL_INDEX]; + ret = dpcon_open(&dpcon_node->dpcon, + CMD_PRI_LOW, dpcon_id, &dpcon_node->token); + if (ret) { + PMD_DRV_LOG(ERR, "Resource alloc failure with err code: %d", + ret); + rte_free(dpcon_node); + return -1; + } + + /* Get the device attributes */ + ret = dpcon_get_attributes(&dpcon_node->dpcon, + CMD_PRI_LOW, dpcon_node->token, &attr); + if (ret != 0) { + PMD_DRV_LOG(ERR, "Reading device failed with err code: %d", + ret); + rte_free(dpcon_node); + return -1; + } + + /* Updating device specific private information*/ + dpcon_node->qbman_ch_id = attr.qbman_ch_id; + dpcon_node->num_priorities = attr.num_priorities; + dpcon_node->dpcon_id = dpcon_id; + rte_atomic16_init(&dpcon_node->in_use); + + TAILQ_INSERT_TAIL(&dpcon_dev_list, dpcon_node, next); + + RTE_LOG(DEBUG, PMD, "DPAA2: Added [dpcon.%d]\n", dpcon_id); + + return 0; +} + +struct dpaa2_dpcon_dev *rte_dpaa2_alloc_dpcon_dev(void) +{ + struct dpaa2_dpcon_dev *dpcon_dev = NULL; + + /* Get DPCON dev handle from list using index */ + TAILQ_FOREACH(dpcon_dev, &dpcon_dev_list, next) { + if (dpcon_dev && rte_atomic16_test_and_set(&dpcon_dev->in_use)) + break; + } + + return dpcon_dev; +} + +void rte_dpaa2_free_dpcon_dev(struct dpaa2_dpcon_dev *dpcon) +{ + struct dpaa2_dpcon_dev *dpcon_dev = NULL; + + /* Match DPCON handle and mark it free */ + TAILQ_FOREACH(dpcon_dev, &dpcon_dev_list, next) { + if (dpcon_dev == dpcon) { + rte_atomic16_dec(&dpcon_dev->in_use); + return; + } + } +} + +static struct rte_dpaa2_object rte_dpaa2_dpcon_obj = { + .dev_type = DPAA2_CON, + .create = rte_dpaa2_create_dpcon_device, +}; + +RTE_PMD_REGISTER_DPAA2_OBJECT(dpcon, rte_dpaa2_dpcon_obj); diff --git a/dpdk/drivers/event/dpaa2/rte_pmd_dpaa2_event_version.map b/dpdk/drivers/event/dpaa2/rte_pmd_dpaa2_event_version.map new file mode 100644 index 00000000..1c0b7559 --- /dev/null +++ b/dpdk/drivers/event/dpaa2/rte_pmd_dpaa2_event_version.map @@ -0,0 +1,3 @@ +DPDK_17.08 { + local: *; +}; diff --git a/dpdk/drivers/event/octeontx/Makefile b/dpdk/drivers/event/octeontx/Makefile new file mode 100644 index 00000000..26044128 --- /dev/null +++ b/dpdk/drivers/event/octeontx/Makefile @@ -0,0 +1,71 @@ +# BSD LICENSE +# +# Copyright(c) 2017 Cavium, Inc. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Cavium, Inc nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +include $(RTE_SDK)/mk/rte.vars.mk + +# +# library name +# +LIB = librte_pmd_octeontx_ssovf.a + +CFLAGS += $(WERROR_FLAGS) +CFLAGS += -I$(RTE_SDK)/drivers/mempool/octeontx/ +CFLAGS += -I$(RTE_SDK)/drivers/net/octeontx/ + +LDLIBS += -lrte_eal -lrte_eventdev -lrte_mempool_octeontx -lrte_pmd_octeontx +LDLIBS += -lrte_bus_pci +LDLIBS += -lrte_bus_vdev + +EXPORT_MAP := rte_pmd_octeontx_ssovf_version.map + +LIBABIVER := 1 + +# +# all source are stored in SRCS-y +# +SRCS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX_SSOVF) += ssovf_worker.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX_SSOVF) += ssovf_evdev.c + +ifeq ($(CONFIG_RTE_TOOLCHAIN_GCC),y) +CFLAGS_ssovf_worker.o += -fno-prefetch-loop-arrays + +ifeq ($(shell test $(GCC_VERSION) -ge 46 && echo 1), 1) +CFLAGS_ssovf_worker.o += -Ofast +else +CFLAGS_ssovf_worker.o += -O3 -ffast-math +endif + +else +CFLAGS_ssovf_worker.o += -Ofast +endif + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/event/octeontx/rte_pmd_octeontx_ssovf_version.map b/dpdk/drivers/event/octeontx/rte_pmd_octeontx_ssovf_version.map new file mode 100644 index 00000000..5352e7e3 --- /dev/null +++ b/dpdk/drivers/event/octeontx/rte_pmd_octeontx_ssovf_version.map @@ -0,0 +1,3 @@ +DPDK_17.05 { + local: *; +}; diff --git a/dpdk/drivers/event/octeontx/ssovf_evdev.c b/dpdk/drivers/event/octeontx/ssovf_evdev.c new file mode 100644 index 00000000..117b1453 --- /dev/null +++ b/dpdk/drivers/event/octeontx/ssovf_evdev.c @@ -0,0 +1,711 @@ +/* + * BSD LICENSE + * + * Copyright (C) Cavium, Inc. 2017. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ssovf_evdev.h" + +/* SSOPF Mailbox messages */ + +struct ssovf_mbox_dev_info { + uint64_t min_deq_timeout_ns; + uint64_t max_deq_timeout_ns; + uint32_t max_num_events; +}; + +static int +ssovf_mbox_dev_info(struct ssovf_mbox_dev_info *info) +{ + struct octeontx_mbox_hdr hdr = {0}; + uint16_t len = sizeof(struct ssovf_mbox_dev_info); + + hdr.coproc = SSO_COPROC; + hdr.msg = SSO_GET_DEV_INFO; + hdr.vfid = 0; + + memset(info, 0, len); + return octeontx_ssovf_mbox_send(&hdr, NULL, 0, info, len); +} + +struct ssovf_mbox_getwork_wait { + uint64_t wait_ns; +}; + +static int +ssovf_mbox_getwork_tmo_set(uint32_t timeout_ns) +{ + struct octeontx_mbox_hdr hdr = {0}; + struct ssovf_mbox_getwork_wait tmo_set; + uint16_t len = sizeof(struct ssovf_mbox_getwork_wait); + int ret; + + hdr.coproc = SSO_COPROC; + hdr.msg = SSO_SET_GETWORK_WAIT; + hdr.vfid = 0; + + tmo_set.wait_ns = timeout_ns; + ret = octeontx_ssovf_mbox_send(&hdr, &tmo_set, len, NULL, 0); + if (ret) + ssovf_log_err("Failed to set getwork timeout(%d)", ret); + + return ret; +} + +struct ssovf_mbox_grp_pri { + uint8_t wgt_left; /* Read only */ + uint8_t weight; + uint8_t affinity; + uint8_t priority; +}; + +static int +ssovf_mbox_priority_set(uint8_t queue, uint8_t prio) +{ + struct octeontx_mbox_hdr hdr = {0}; + struct ssovf_mbox_grp_pri grp; + uint16_t len = sizeof(struct ssovf_mbox_grp_pri); + int ret; + + hdr.coproc = SSO_COPROC; + hdr.msg = SSO_GRP_SET_PRIORITY; + hdr.vfid = queue; + + grp.weight = 0xff; + grp.affinity = 0xff; + grp.priority = prio / 32; /* Normalize to 0 to 7 */ + + ret = octeontx_ssovf_mbox_send(&hdr, &grp, len, NULL, 0); + if (ret) + ssovf_log_err("Failed to set grp=%d prio=%d", queue, prio); + + return ret; +} + +struct ssovf_mbox_convert_ns_getworks_iter { + uint64_t wait_ns; + uint32_t getwork_iter;/* Get_work iterations for the given wait_ns */ +}; + +static int +ssovf_mbox_timeout_ticks(uint64_t ns, uint64_t *tmo_ticks) +{ + struct octeontx_mbox_hdr hdr = {0}; + struct ssovf_mbox_convert_ns_getworks_iter ns2iter; + uint16_t len = sizeof(ns2iter); + int ret; + + hdr.coproc = SSO_COPROC; + hdr.msg = SSO_CONVERT_NS_GETWORK_ITER; + hdr.vfid = 0; + + memset(&ns2iter, 0, len); + ns2iter.wait_ns = ns; + ret = octeontx_ssovf_mbox_send(&hdr, &ns2iter, len, &ns2iter, len); + if (ret < 0 || (ret != len)) { + ssovf_log_err("Failed to get tmo ticks ns=%"PRId64"", ns); + return -EIO; + } + + *tmo_ticks = ns2iter.getwork_iter; + return 0; +} + +static void +ssovf_fastpath_fns_set(struct rte_eventdev *dev) +{ + struct ssovf_evdev *edev = ssovf_pmd_priv(dev); + + dev->enqueue = ssows_enq; + dev->enqueue_burst = ssows_enq_burst; + dev->enqueue_new_burst = ssows_enq_new_burst; + dev->enqueue_forward_burst = ssows_enq_fwd_burst; + dev->dequeue = ssows_deq; + dev->dequeue_burst = ssows_deq_burst; + + if (edev->is_timeout_deq) { + dev->dequeue = ssows_deq_timeout; + dev->dequeue_burst = ssows_deq_timeout_burst; + } +} + +static void +ssovf_info_get(struct rte_eventdev *dev, struct rte_event_dev_info *dev_info) +{ + struct ssovf_evdev *edev = ssovf_pmd_priv(dev); + + dev_info->driver_name = RTE_STR(EVENTDEV_NAME_OCTEONTX_PMD); + dev_info->min_dequeue_timeout_ns = edev->min_deq_timeout_ns; + dev_info->max_dequeue_timeout_ns = edev->max_deq_timeout_ns; + dev_info->max_event_queues = edev->max_event_queues; + dev_info->max_event_queue_flows = (1ULL << 20); + dev_info->max_event_queue_priority_levels = 8; + dev_info->max_event_priority_levels = 1; + dev_info->max_event_ports = edev->max_event_ports; + dev_info->max_event_port_dequeue_depth = 1; + dev_info->max_event_port_enqueue_depth = 1; + dev_info->max_num_events = edev->max_num_events; + dev_info->event_dev_cap = RTE_EVENT_DEV_CAP_QUEUE_QOS | + RTE_EVENT_DEV_CAP_DISTRIBUTED_SCHED | + RTE_EVENT_DEV_CAP_QUEUE_ALL_TYPES; +} + +static int +ssovf_configure(const struct rte_eventdev *dev) +{ + struct rte_event_dev_config *conf = &dev->data->dev_conf; + struct ssovf_evdev *edev = ssovf_pmd_priv(dev); + uint64_t deq_tmo_ns; + + ssovf_func_trace(); + deq_tmo_ns = conf->dequeue_timeout_ns; + if (deq_tmo_ns == 0) + deq_tmo_ns = edev->min_deq_timeout_ns; + + if (conf->event_dev_cfg & RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT) { + edev->is_timeout_deq = 1; + deq_tmo_ns = edev->min_deq_timeout_ns; + } + edev->nb_event_queues = conf->nb_event_queues; + edev->nb_event_ports = conf->nb_event_ports; + + return ssovf_mbox_getwork_tmo_set(deq_tmo_ns); +} + +static void +ssovf_queue_def_conf(struct rte_eventdev *dev, uint8_t queue_id, + struct rte_event_queue_conf *queue_conf) +{ + RTE_SET_USED(dev); + RTE_SET_USED(queue_id); + + queue_conf->nb_atomic_flows = (1ULL << 20); + queue_conf->nb_atomic_order_sequences = (1ULL << 20); + queue_conf->event_queue_cfg = RTE_EVENT_QUEUE_CFG_ALL_TYPES; + queue_conf->priority = RTE_EVENT_DEV_PRIORITY_NORMAL; +} + +static void +ssovf_queue_release(struct rte_eventdev *dev, uint8_t queue_id) +{ + RTE_SET_USED(dev); + RTE_SET_USED(queue_id); +} + +static int +ssovf_queue_setup(struct rte_eventdev *dev, uint8_t queue_id, + const struct rte_event_queue_conf *queue_conf) +{ + RTE_SET_USED(dev); + ssovf_func_trace("queue=%d prio=%d", queue_id, queue_conf->priority); + + return ssovf_mbox_priority_set(queue_id, queue_conf->priority); +} + +static void +ssovf_port_def_conf(struct rte_eventdev *dev, uint8_t port_id, + struct rte_event_port_conf *port_conf) +{ + struct ssovf_evdev *edev = ssovf_pmd_priv(dev); + + RTE_SET_USED(port_id); + port_conf->new_event_threshold = edev->max_num_events; + port_conf->dequeue_depth = 1; + port_conf->enqueue_depth = 1; +} + +static void +ssovf_port_release(void *port) +{ + rte_free(port); +} + +static int +ssovf_port_setup(struct rte_eventdev *dev, uint8_t port_id, + const struct rte_event_port_conf *port_conf) +{ + struct ssows *ws; + uint32_t reg_off; + uint8_t q; + struct ssovf_evdev *edev = ssovf_pmd_priv(dev); + + ssovf_func_trace("port=%d", port_id); + RTE_SET_USED(port_conf); + + /* Free memory prior to re-allocation if needed */ + if (dev->data->ports[port_id] != NULL) { + ssovf_port_release(dev->data->ports[port_id]); + dev->data->ports[port_id] = NULL; + } + + /* Allocate event port memory */ + ws = rte_zmalloc_socket("eventdev ssows", + sizeof(struct ssows), RTE_CACHE_LINE_SIZE, + dev->data->socket_id); + if (ws == NULL) { + ssovf_log_err("Failed to alloc memory for port=%d", port_id); + return -ENOMEM; + } + + ws->base = octeontx_ssovf_bar(OCTEONTX_SSO_HWS, port_id, 0); + if (ws->base == NULL) { + rte_free(ws); + ssovf_log_err("Failed to get hws base addr port=%d", port_id); + return -EINVAL; + } + + reg_off = SSOW_VHWS_OP_GET_WORK0; + reg_off |= 1 << 4; /* Index_ggrp_mask (Use maskset zero) */ + reg_off |= 1 << 16; /* Wait */ + ws->getwork = ws->base + reg_off; + ws->port = port_id; + + for (q = 0; q < edev->nb_event_queues; q++) { + ws->grps[q] = octeontx_ssovf_bar(OCTEONTX_SSO_GROUP, q, 2); + if (ws->grps[q] == NULL) { + rte_free(ws); + ssovf_log_err("Failed to get grp%d base addr", q); + return -EINVAL; + } + } + + dev->data->ports[port_id] = ws; + ssovf_log_dbg("port=%d ws=%p", port_id, ws); + return 0; +} + +static int +ssovf_port_link(struct rte_eventdev *dev, void *port, const uint8_t queues[], + const uint8_t priorities[], uint16_t nb_links) +{ + uint16_t link; + uint64_t val; + struct ssows *ws = port; + + ssovf_func_trace("port=%d nb_links=%d", ws->port, nb_links); + RTE_SET_USED(dev); + RTE_SET_USED(priorities); + + for (link = 0; link < nb_links; link++) { + val = queues[link]; + val |= (1ULL << 24); /* Set membership */ + ssovf_write64(val, ws->base + SSOW_VHWS_GRPMSK_CHGX(0)); + } + return (int)nb_links; +} + +static int +ssovf_port_unlink(struct rte_eventdev *dev, void *port, uint8_t queues[], + uint16_t nb_unlinks) +{ + uint16_t unlink; + uint64_t val; + struct ssows *ws = port; + + ssovf_func_trace("port=%d nb_links=%d", ws->port, nb_unlinks); + RTE_SET_USED(dev); + + for (unlink = 0; unlink < nb_unlinks; unlink++) { + val = queues[unlink]; + val &= ~(1ULL << 24); /* Clear membership */ + ssovf_write64(val, ws->base + SSOW_VHWS_GRPMSK_CHGX(0)); + } + return (int)nb_unlinks; +} + +static int +ssovf_timeout_ticks(struct rte_eventdev *dev, uint64_t ns, uint64_t *tmo_ticks) +{ + RTE_SET_USED(dev); + + return ssovf_mbox_timeout_ticks(ns, tmo_ticks); +} + +static void +ssows_dump(struct ssows *ws, FILE *f) +{ + uint8_t *base = ws->base; + uint64_t val; + + fprintf(f, "\t---------------port%d---------------\n", ws->port); + val = ssovf_read64(base + SSOW_VHWS_TAG); + fprintf(f, "\ttag=0x%x tt=%d head=%d tail=%d grp=%d index=%d tail=%d\n", + (uint32_t)(val & 0xffffffff), (int)(val >> 32) & 0x3, + (int)(val >> 34) & 0x1, (int)(val >> 35) & 0x1, + (int)(val >> 36) & 0x3ff, (int)(val >> 48) & 0x3ff, + (int)(val >> 63) & 0x1); + + val = ssovf_read64(base + SSOW_VHWS_WQP); + fprintf(f, "\twqp=0x%"PRIx64"\n", val); + + val = ssovf_read64(base + SSOW_VHWS_LINKS); + fprintf(f, "\tindex=%d valid=%d revlink=%d tail=%d head=%d grp=%d\n", + (int)(val & 0x3ff), (int)(val >> 10) & 0x1, + (int)(val >> 11) & 0x3ff, (int)(val >> 26) & 0x1, + (int)(val >> 27) & 0x1, (int)(val >> 28) & 0x3ff); + + val = ssovf_read64(base + SSOW_VHWS_PENDTAG); + fprintf(f, "\tptag=0x%x ptt=%d pgwi=%d pdesc=%d pgw=%d pgww=%d ps=%d\n", + (uint32_t)(val & 0xffffffff), (int)(val >> 32) & 0x3, + (int)(val >> 56) & 0x1, (int)(val >> 58) & 0x1, + (int)(val >> 61) & 0x1, (int)(val >> 62) & 0x1, + (int)(val >> 63) & 0x1); + + val = ssovf_read64(base + SSOW_VHWS_PENDWQP); + fprintf(f, "\tpwqp=0x%"PRIx64"\n", val); +} + +static int +ssovf_eth_rx_adapter_caps_get(const struct rte_eventdev *dev, + const struct rte_eth_dev *eth_dev, uint32_t *caps) +{ + int ret; + RTE_SET_USED(dev); + + ret = strncmp(eth_dev->data->name, "eth_octeontx", 12); + if (ret) + *caps = RTE_EVENT_ETH_RX_ADAPTER_SW_CAP; + else + *caps = RTE_EVENT_ETH_RX_ADAPTER_CAP_INTERNAL_PORT; + + return 0; +} + +static int +ssovf_eth_rx_adapter_queue_add(const struct rte_eventdev *dev, + const struct rte_eth_dev *eth_dev, int32_t rx_queue_id, + const struct rte_event_eth_rx_adapter_queue_conf *queue_conf) +{ + int ret = 0; + const struct octeontx_nic *nic = eth_dev->data->dev_private; + pki_mod_qos_t pki_qos; + RTE_SET_USED(dev); + + ret = strncmp(eth_dev->data->name, "eth_octeontx", 12); + if (ret) + return -EINVAL; + + if (rx_queue_id >= 0) + return -EINVAL; + + if (queue_conf->ev.sched_type == RTE_SCHED_TYPE_PARALLEL) + return -ENOTSUP; + + memset(&pki_qos, 0, sizeof(pki_mod_qos_t)); + + pki_qos.port_type = 0; + pki_qos.index = 0; + pki_qos.mmask.f_tag_type = 1; + pki_qos.mmask.f_port_add = 1; + pki_qos.mmask.f_grp_ok = 1; + pki_qos.mmask.f_grp_bad = 1; + pki_qos.mmask.f_grptag_ok = 1; + pki_qos.mmask.f_grptag_bad = 1; + + pki_qos.tag_type = queue_conf->ev.sched_type; + pki_qos.qos_entry.port_add = 0; + pki_qos.qos_entry.ggrp_ok = queue_conf->ev.queue_id; + pki_qos.qos_entry.ggrp_bad = queue_conf->ev.queue_id; + pki_qos.qos_entry.grptag_bad = 0; + pki_qos.qos_entry.grptag_ok = 0; + + ret = octeontx_pki_port_modify_qos(nic->port_id, &pki_qos); + if (ret < 0) + ssovf_log_err("failed to modify QOS, port=%d, q=%d", + nic->port_id, queue_conf->ev.queue_id); + + return ret; +} + +static int +ssovf_eth_rx_adapter_queue_del(const struct rte_eventdev *dev, + const struct rte_eth_dev *eth_dev, int32_t rx_queue_id) +{ + int ret = 0; + const struct octeontx_nic *nic = eth_dev->data->dev_private; + pki_del_qos_t pki_qos; + RTE_SET_USED(dev); + RTE_SET_USED(rx_queue_id); + + ret = strncmp(eth_dev->data->name, "eth_octeontx", 12); + if (ret) + return -EINVAL; + + pki_qos.port_type = 0; + pki_qos.index = 0; + memset(&pki_qos, 0, sizeof(pki_del_qos_t)); + ret = octeontx_pki_port_delete_qos(nic->port_id, &pki_qos); + if (ret < 0) + ssovf_log_err("Failed to delete QOS port=%d, q=%d", + nic->port_id, queue_conf->ev.queue_id); + return ret; +} + +static int +ssovf_eth_rx_adapter_start(const struct rte_eventdev *dev, + const struct rte_eth_dev *eth_dev) +{ + int ret; + const struct octeontx_nic *nic = eth_dev->data->dev_private; + RTE_SET_USED(dev); + + ret = strncmp(eth_dev->data->name, "eth_octeontx", 12); + if (ret) + return 0; + octeontx_pki_port_start(nic->port_id); + return 0; +} + + +static int +ssovf_eth_rx_adapter_stop(const struct rte_eventdev *dev, + const struct rte_eth_dev *eth_dev) +{ + int ret; + const struct octeontx_nic *nic = eth_dev->data->dev_private; + RTE_SET_USED(dev); + + ret = strncmp(eth_dev->data->name, "eth_octeontx", 12); + if (ret) + return 0; + octeontx_pki_port_stop(nic->port_id); + return 0; +} + +static void +ssovf_dump(struct rte_eventdev *dev, FILE *f) +{ + struct ssovf_evdev *edev = ssovf_pmd_priv(dev); + uint8_t port; + + /* Dump SSOWVF debug registers */ + for (port = 0; port < edev->nb_event_ports; port++) + ssows_dump(dev->data->ports[port], f); +} + +static int +ssovf_start(struct rte_eventdev *dev) +{ + struct ssovf_evdev *edev = ssovf_pmd_priv(dev); + struct ssows *ws; + uint8_t *base; + uint8_t i; + + ssovf_func_trace(); + for (i = 0; i < edev->nb_event_ports; i++) { + ws = dev->data->ports[i]; + ssows_reset(ws); + ws->swtag_req = 0; + } + + for (i = 0; i < edev->nb_event_queues; i++) { + /* Consume all the events through HWS0 */ + ssows_flush_events(dev->data->ports[0], i); + + base = octeontx_ssovf_bar(OCTEONTX_SSO_GROUP, i, 0); + base += SSO_VHGRP_QCTL; + ssovf_write64(1, base); /* Enable SSO group */ + } + + ssovf_fastpath_fns_set(dev); + return 0; +} + +static void +ssovf_stop(struct rte_eventdev *dev) +{ + struct ssovf_evdev *edev = ssovf_pmd_priv(dev); + struct ssows *ws; + uint8_t *base; + uint8_t i; + + ssovf_func_trace(); + for (i = 0; i < edev->nb_event_ports; i++) { + ws = dev->data->ports[i]; + ssows_reset(ws); + ws->swtag_req = 0; + } + + for (i = 0; i < edev->nb_event_queues; i++) { + /* Consume all the events through HWS0 */ + ssows_flush_events(dev->data->ports[0], i); + + base = octeontx_ssovf_bar(OCTEONTX_SSO_GROUP, i, 0); + base += SSO_VHGRP_QCTL; + ssovf_write64(0, base); /* Disable SSO group */ + } +} + +static int +ssovf_close(struct rte_eventdev *dev) +{ + struct ssovf_evdev *edev = ssovf_pmd_priv(dev); + uint8_t all_queues[RTE_EVENT_MAX_QUEUES_PER_DEV]; + uint8_t i; + + for (i = 0; i < edev->nb_event_queues; i++) + all_queues[i] = i; + + for (i = 0; i < edev->nb_event_ports; i++) + ssovf_port_unlink(dev, dev->data->ports[i], all_queues, + edev->nb_event_queues); + return 0; +} + +/* Initialize and register event driver with DPDK Application */ +static const struct rte_eventdev_ops ssovf_ops = { + .dev_infos_get = ssovf_info_get, + .dev_configure = ssovf_configure, + .queue_def_conf = ssovf_queue_def_conf, + .queue_setup = ssovf_queue_setup, + .queue_release = ssovf_queue_release, + .port_def_conf = ssovf_port_def_conf, + .port_setup = ssovf_port_setup, + .port_release = ssovf_port_release, + .port_link = ssovf_port_link, + .port_unlink = ssovf_port_unlink, + .timeout_ticks = ssovf_timeout_ticks, + + .eth_rx_adapter_caps_get = ssovf_eth_rx_adapter_caps_get, + .eth_rx_adapter_queue_add = ssovf_eth_rx_adapter_queue_add, + .eth_rx_adapter_queue_del = ssovf_eth_rx_adapter_queue_del, + .eth_rx_adapter_start = ssovf_eth_rx_adapter_start, + .eth_rx_adapter_stop = ssovf_eth_rx_adapter_stop, + + .dump = ssovf_dump, + .dev_start = ssovf_start, + .dev_stop = ssovf_stop, + .dev_close = ssovf_close +}; + +static int +ssovf_vdev_probe(struct rte_vdev_device *vdev) +{ + struct octeontx_ssovf_info oinfo; + struct ssovf_mbox_dev_info info; + struct ssovf_evdev *edev; + struct rte_eventdev *eventdev; + static int ssovf_init_once; + const char *name; + int ret; + + name = rte_vdev_device_name(vdev); + /* More than one instance is not supported */ + if (ssovf_init_once) { + ssovf_log_err("Request to create >1 %s instance", name); + return -EINVAL; + } + + eventdev = rte_event_pmd_vdev_init(name, sizeof(struct ssovf_evdev), + rte_socket_id()); + if (eventdev == NULL) { + ssovf_log_err("Failed to create eventdev vdev %s", name); + return -ENOMEM; + } + eventdev->dev_ops = &ssovf_ops; + + /* For secondary processes, the primary has done all the work */ + if (rte_eal_process_type() != RTE_PROC_PRIMARY) { + ssovf_fastpath_fns_set(eventdev); + return 0; + } + + ret = octeontx_ssovf_info(&oinfo); + if (ret) { + ssovf_log_err("Failed to probe and validate ssovfs %d", ret); + goto error; + } + + edev = ssovf_pmd_priv(eventdev); + edev->max_event_ports = oinfo.total_ssowvfs; + edev->max_event_queues = oinfo.total_ssovfs; + edev->is_timeout_deq = 0; + + ret = ssovf_mbox_dev_info(&info); + if (ret < 0 || ret != sizeof(struct ssovf_mbox_dev_info)) { + ssovf_log_err("Failed to get mbox devinfo %d", ret); + goto error; + } + + edev->min_deq_timeout_ns = info.min_deq_timeout_ns; + edev->max_deq_timeout_ns = info.max_deq_timeout_ns; + edev->max_num_events = info.max_num_events; + ssovf_log_dbg("min_deq_tmo=%"PRId64" max_deq_tmo=%"PRId64" max_evts=%d", + info.min_deq_timeout_ns, info.max_deq_timeout_ns, + info.max_num_events); + + if (!edev->max_event_ports || !edev->max_event_queues) { + ssovf_log_err("Not enough eventdev resource queues=%d ports=%d", + edev->max_event_queues, edev->max_event_ports); + ret = -ENODEV; + goto error; + } + + ssovf_log_info("Initializing %s domain=%d max_queues=%d max_ports=%d", + name, oinfo.domain, edev->max_event_queues, + edev->max_event_ports); + + ssovf_init_once = 1; + return 0; + +error: + rte_event_pmd_vdev_uninit(name); + return ret; +} + +static int +ssovf_vdev_remove(struct rte_vdev_device *vdev) +{ + const char *name; + + name = rte_vdev_device_name(vdev); + ssovf_log_info("Closing %s", name); + return rte_event_pmd_vdev_uninit(name); +} + +static struct rte_vdev_driver vdev_ssovf_pmd = { + .probe = ssovf_vdev_probe, + .remove = ssovf_vdev_remove +}; + +RTE_PMD_REGISTER_VDEV(EVENTDEV_NAME_OCTEONTX_PMD, vdev_ssovf_pmd); diff --git a/dpdk/drivers/event/octeontx/ssovf_evdev.h b/dpdk/drivers/event/octeontx/ssovf_evdev.h new file mode 100644 index 00000000..b093a3e7 --- /dev/null +++ b/dpdk/drivers/event/octeontx/ssovf_evdev.h @@ -0,0 +1,200 @@ +/* + * BSD LICENSE + * + * Copyright (C) Cavium, Inc. 2017. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __SSOVF_EVDEV_H__ +#define __SSOVF_EVDEV_H__ + +#include +#include + +#include +#include + +#define EVENTDEV_NAME_OCTEONTX_PMD event_octeontx + +#ifdef RTE_LIBRTE_PMD_OCTEONTX_SSOVF_DEBUG +#define ssovf_log_info(fmt, args...) \ + RTE_LOG(INFO, EVENTDEV, "[%s] %s() " fmt "\n", \ + RTE_STR(EVENTDEV_NAME_OCTEONTX_PMD), __func__, ## args) +#define ssovf_log_dbg(fmt, args...) \ + RTE_LOG(DEBUG, EVENTDEV, "[%s] %s() " fmt "\n", \ + RTE_STR(EVENTDEV_NAME_OCTEONTX_PMD), __func__, ## args) +#else +#define ssovf_log_info(fmt, args...) +#define ssovf_log_dbg(fmt, args...) +#endif + +#define ssovf_func_trace ssovf_log_dbg +#define ssovf_log_err(fmt, args...) \ + RTE_LOG(ERR, EVENTDEV, "[%s] %s() " fmt "\n", \ + RTE_STR(EVENTDEV_NAME_OCTEONTX_PMD), __func__, ## args) + +#define SSO_MAX_VHGRP (64) +#define SSO_MAX_VHWS (32) + +/* SSO VF register offsets */ +#define SSO_VHGRP_QCTL (0x010ULL) +#define SSO_VHGRP_INT (0x100ULL) +#define SSO_VHGRP_INT_W1S (0x108ULL) +#define SSO_VHGRP_INT_ENA_W1S (0x110ULL) +#define SSO_VHGRP_INT_ENA_W1C (0x118ULL) +#define SSO_VHGRP_INT_THR (0x140ULL) +#define SSO_VHGRP_INT_CNT (0x180ULL) +#define SSO_VHGRP_XAQ_CNT (0x1B0ULL) +#define SSO_VHGRP_AQ_CNT (0x1C0ULL) +#define SSO_VHGRP_AQ_THR (0x1E0ULL) + +/* BAR2 */ +#define SSO_VHGRP_OP_ADD_WORK0 (0x00ULL) +#define SSO_VHGRP_OP_ADD_WORK1 (0x08ULL) + +/* SSOW VF register offsets (BAR0) */ +#define SSOW_VHWS_GRPMSK_CHGX(x) (0x080ULL | ((x) << 3)) +#define SSOW_VHWS_TAG (0x300ULL) +#define SSOW_VHWS_WQP (0x308ULL) +#define SSOW_VHWS_LINKS (0x310ULL) +#define SSOW_VHWS_PENDTAG (0x340ULL) +#define SSOW_VHWS_PENDWQP (0x348ULL) +#define SSOW_VHWS_SWTP (0x400ULL) +#define SSOW_VHWS_OP_ALLOC_WE (0x410ULL) +#define SSOW_VHWS_OP_UPD_WQP_GRP0 (0x440ULL) +#define SSOW_VHWS_OP_UPD_WQP_GRP1 (0x448ULL) +#define SSOW_VHWS_OP_SWTAG_UNTAG (0x490ULL) +#define SSOW_VHWS_OP_SWTAG_CLR (0x820ULL) +#define SSOW_VHWS_OP_DESCHED (0x860ULL) +#define SSOW_VHWS_OP_DESCHED_NOSCH (0x870ULL) +#define SSOW_VHWS_OP_SWTAG_DESCHED (0x8C0ULL) +#define SSOW_VHWS_OP_SWTAG_NOSCHED (0x8D0ULL) +#define SSOW_VHWS_OP_SWTP_SET (0xC20ULL) +#define SSOW_VHWS_OP_SWTAG_NORM (0xC80ULL) +#define SSOW_VHWS_OP_SWTAG_FULL0 (0xCA0UL) +#define SSOW_VHWS_OP_SWTAG_FULL1 (0xCA8ULL) +#define SSOW_VHWS_OP_CLR_NSCHED (0x10000ULL) +#define SSOW_VHWS_OP_GET_WORK0 (0x80000ULL) +#define SSOW_VHWS_OP_GET_WORK1 (0x80008ULL) + +/* Mailbox message constants */ +#define SSO_COPROC 0x2 + +#define SSO_GETDOMAINCFG 0x1 +#define SSO_IDENTIFY 0x2 +#define SSO_GET_DEV_INFO 0x3 +#define SSO_GET_GETWORK_WAIT 0x4 +#define SSO_SET_GETWORK_WAIT 0x5 +#define SSO_CONVERT_NS_GETWORK_ITER 0x6 +#define SSO_GRP_GET_PRIORITY 0x7 +#define SSO_GRP_SET_PRIORITY 0x8 + +/* + * In Cavium OcteonTX SoC, all accesses to the device registers are + * implictly strongly ordered. So, The relaxed version of IO operation is + * safe to use with out any IO memory barriers. + */ +#define ssovf_read64 rte_read64_relaxed +#define ssovf_write64 rte_write64_relaxed + +/* ARM64 specific functions */ +#if defined(RTE_ARCH_ARM64) +#define ssovf_load_pair(val0, val1, addr) ({ \ + asm volatile( \ + "ldp %x[x0], %x[x1], [%x[p1]]" \ + :[x0]"=r"(val0), [x1]"=r"(val1) \ + :[p1]"r"(addr) \ + ); }) + +#define ssovf_store_pair(val0, val1, addr) ({ \ + asm volatile( \ + "stp %x[x0], %x[x1], [%x[p1]]" \ + ::[x0]"r"(val0), [x1]"r"(val1), [p1]"r"(addr) \ + ); }) +#else /* Un optimized functions for building on non arm64 arch */ + +#define ssovf_load_pair(val0, val1, addr) \ +do { \ + val0 = rte_read64(addr); \ + val1 = rte_read64(((uint8_t *)addr) + 8); \ +} while (0) + +#define ssovf_store_pair(val0, val1, addr) \ +do { \ + rte_write64(val0, addr); \ + rte_write64(val1, (((uint8_t *)addr) + 8)); \ +} while (0) +#endif + + +struct ssovf_evdev { + uint8_t max_event_queues; + uint8_t max_event_ports; + uint8_t is_timeout_deq; + uint8_t nb_event_queues; + uint8_t nb_event_ports; + uint32_t min_deq_timeout_ns; + uint32_t max_deq_timeout_ns; + int32_t max_num_events; +} __rte_cache_aligned; + +/* Event port aka HWS */ +struct ssows { + uint8_t cur_tt; + uint8_t cur_grp; + uint8_t swtag_req; + uint8_t *base; + uint8_t *getwork; + uint8_t *grps[SSO_MAX_VHGRP]; + uint8_t port; +} __rte_cache_aligned; + +static inline struct ssovf_evdev * +ssovf_pmd_priv(const struct rte_eventdev *eventdev) +{ + return eventdev->data->dev_private; +} + +uint16_t ssows_enq(void *port, const struct rte_event *ev); +uint16_t ssows_enq_burst(void *port, + const struct rte_event ev[], uint16_t nb_events); +uint16_t ssows_enq_new_burst(void *port, + const struct rte_event ev[], uint16_t nb_events); +uint16_t ssows_enq_fwd_burst(void *port, + const struct rte_event ev[], uint16_t nb_events); +uint16_t ssows_deq(void *port, struct rte_event *ev, uint64_t timeout_ticks); +uint16_t ssows_deq_burst(void *port, struct rte_event ev[], + uint16_t nb_events, uint64_t timeout_ticks); +uint16_t ssows_deq_timeout(void *port, struct rte_event *ev, + uint64_t timeout_ticks); +uint16_t ssows_deq_timeout_burst(void *port, struct rte_event ev[], + uint16_t nb_events, uint64_t timeout_ticks); +void ssows_flush_events(struct ssows *ws, uint8_t queue_id); +void ssows_reset(struct ssows *ws); + +#endif /* __SSOVF_EVDEV_H__ */ diff --git a/dpdk/drivers/event/octeontx/ssovf_worker.c b/dpdk/drivers/event/octeontx/ssovf_worker.c new file mode 100644 index 00000000..5e17c7b8 --- /dev/null +++ b/dpdk/drivers/event/octeontx/ssovf_worker.c @@ -0,0 +1,277 @@ +/* + * BSD LICENSE + * + * Copyright (C) Cavium, Inc. 2017. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "ssovf_worker.h" + +static __rte_always_inline void +ssows_new_event(struct ssows *ws, const struct rte_event *ev) +{ + const uint64_t event_ptr = ev->u64; + const uint32_t tag = (uint32_t)ev->event; + const uint8_t new_tt = ev->sched_type; + const uint8_t grp = ev->queue_id; + + ssows_add_work(ws, event_ptr, tag, new_tt, grp); +} + +static __rte_always_inline void +ssows_fwd_swtag(struct ssows *ws, const struct rte_event *ev, const uint8_t grp) +{ + const uint8_t cur_tt = ws->cur_tt; + const uint8_t new_tt = ev->sched_type; + const uint32_t tag = (uint32_t)ev->event; + /* + * cur_tt/new_tt SSO_SYNC_ORDERED SSO_SYNC_ATOMIC SSO_SYNC_UNTAGGED + * + * SSO_SYNC_ORDERED norm norm untag + * SSO_SYNC_ATOMIC norm norm untag + * SSO_SYNC_UNTAGGED full full NOOP + */ + if (unlikely(cur_tt == SSO_SYNC_UNTAGGED)) { + if (new_tt != SSO_SYNC_UNTAGGED) { + ssows_swtag_full(ws, ev->u64, tag, + new_tt, grp); + } + } else { + if (likely(new_tt != SSO_SYNC_UNTAGGED)) + ssows_swtag_norm(ws, tag, new_tt); + else + ssows_swtag_untag(ws); + } + ws->swtag_req = 1; +} + +#define OCT_EVENT_TYPE_GRP_FWD (RTE_EVENT_TYPE_MAX - 1) + +static __rte_always_inline void +ssows_fwd_group(struct ssows *ws, const struct rte_event *ev, const uint8_t grp) +{ + const uint64_t event_ptr = ev->u64; + const uint32_t tag = (uint32_t)ev->event; + const uint8_t cur_tt = ws->cur_tt; + const uint8_t new_tt = ev->sched_type; + + if (cur_tt == SSO_SYNC_ORDERED) { + /* Create unique tag based on custom event type and new grp */ + uint32_t newtag = OCT_EVENT_TYPE_GRP_FWD << 28; + + newtag |= grp << 20; + newtag |= tag; + ssows_swtag_norm(ws, newtag, SSO_SYNC_ATOMIC); + rte_smp_wmb(); + ssows_swtag_wait(ws); + } else { + rte_smp_wmb(); + } + ssows_add_work(ws, event_ptr, tag, new_tt, grp); +} + +static __rte_always_inline void +ssows_forward_event(struct ssows *ws, const struct rte_event *ev) +{ + const uint8_t grp = ev->queue_id; + + /* Group hasn't changed, Use SWTAG to forward the event */ + if (ws->cur_grp == grp) + ssows_fwd_swtag(ws, ev, grp); + else + /* + * Group has been changed for group based work pipelining, + * Use deschedule/add_work operation to transfer the event to + * new group/core + */ + ssows_fwd_group(ws, ev, grp); +} + +static __rte_always_inline void +ssows_release_event(struct ssows *ws) +{ + if (likely(ws->cur_tt != SSO_SYNC_UNTAGGED)) + ssows_swtag_untag(ws); +} + +__rte_always_inline uint16_t __hot +ssows_deq(void *port, struct rte_event *ev, uint64_t timeout_ticks) +{ + struct ssows *ws = port; + + RTE_SET_USED(timeout_ticks); + + if (ws->swtag_req) { + ws->swtag_req = 0; + ssows_swtag_wait(ws); + return 1; + } else { + return ssows_get_work(ws, ev); + } +} + +__rte_always_inline uint16_t __hot +ssows_deq_timeout(void *port, struct rte_event *ev, uint64_t timeout_ticks) +{ + struct ssows *ws = port; + uint64_t iter; + uint16_t ret = 1; + + if (ws->swtag_req) { + ws->swtag_req = 0; + ssows_swtag_wait(ws); + } else { + ret = ssows_get_work(ws, ev); + for (iter = 1; iter < timeout_ticks && (ret == 0); iter++) + ret = ssows_get_work(ws, ev); + } + return ret; +} + +uint16_t __hot +ssows_deq_burst(void *port, struct rte_event ev[], uint16_t nb_events, + uint64_t timeout_ticks) +{ + RTE_SET_USED(nb_events); + + return ssows_deq(port, ev, timeout_ticks); +} + +uint16_t __hot +ssows_deq_timeout_burst(void *port, struct rte_event ev[], uint16_t nb_events, + uint64_t timeout_ticks) +{ + RTE_SET_USED(nb_events); + + return ssows_deq_timeout(port, ev, timeout_ticks); +} + +__rte_always_inline uint16_t __hot +ssows_enq(void *port, const struct rte_event *ev) +{ + struct ssows *ws = port; + uint16_t ret = 1; + + switch (ev->op) { + case RTE_EVENT_OP_NEW: + rte_smp_wmb(); + ssows_new_event(ws, ev); + break; + case RTE_EVENT_OP_FORWARD: + ssows_forward_event(ws, ev); + break; + case RTE_EVENT_OP_RELEASE: + ssows_release_event(ws); + break; + default: + ret = 0; + } + return ret; +} + +uint16_t __hot +ssows_enq_burst(void *port, const struct rte_event ev[], uint16_t nb_events) +{ + RTE_SET_USED(nb_events); + return ssows_enq(port, ev); +} + +uint16_t __hot +ssows_enq_new_burst(void *port, const struct rte_event ev[], uint16_t nb_events) +{ + uint16_t i; + struct ssows *ws = port; + + rte_smp_wmb(); + for (i = 0; i < nb_events; i++) + ssows_new_event(ws, &ev[i]); + + return nb_events; +} + +uint16_t __hot +ssows_enq_fwd_burst(void *port, const struct rte_event ev[], uint16_t nb_events) +{ + struct ssows *ws = port; + RTE_SET_USED(nb_events); + + ssows_forward_event(ws, ev); + + return 1; +} + +void +ssows_flush_events(struct ssows *ws, uint8_t queue_id) +{ + uint32_t reg_off; + uint64_t aq_cnt = 1; + uint64_t cq_ds_cnt = 1; + uint64_t enable, get_work0, get_work1; + uint8_t *base = octeontx_ssovf_bar(OCTEONTX_SSO_GROUP, queue_id, 0); + + enable = ssovf_read64(base + SSO_VHGRP_QCTL); + if (!enable) + return; + + reg_off = SSOW_VHWS_OP_GET_WORK0; + reg_off |= 1 << 17; /* Grouped */ + reg_off |= 1 << 16; /* WAIT */ + reg_off |= queue_id << 4; /* INDEX_GGRP_MASK(group number) */ + while (aq_cnt || cq_ds_cnt) { + aq_cnt = ssovf_read64(base + SSO_VHGRP_AQ_CNT); + cq_ds_cnt = ssovf_read64(base + SSO_VHGRP_INT_CNT); + /* Extract cq and ds count */ + cq_ds_cnt &= 0x1FFF1FFF0000; + ssovf_load_pair(get_work0, get_work1, ws->base + reg_off); + } + + RTE_SET_USED(get_work0); + RTE_SET_USED(get_work1); +} + +void +ssows_reset(struct ssows *ws) +{ + uint64_t tag; + uint64_t pend_tag; + uint8_t pend_tt; + uint8_t tt; + + tag = ssovf_read64(ws->base + SSOW_VHWS_TAG); + pend_tag = ssovf_read64(ws->base + SSOW_VHWS_PENDTAG); + + if (pend_tag & (1ULL << 63)) { /* Tagswitch pending */ + pend_tt = (pend_tag >> 32) & 0x3; + if (pend_tt == SSO_SYNC_ORDERED || pend_tt == SSO_SYNC_ATOMIC) + ssows_desched(ws); + } else { + tt = (tag >> 32) & 0x3; + if (tt == SSO_SYNC_ORDERED || tt == SSO_SYNC_ATOMIC) + ssows_swtag_untag(ws); + } +} diff --git a/dpdk/drivers/event/octeontx/ssovf_worker.h b/dpdk/drivers/event/octeontx/ssovf_worker.h new file mode 100644 index 00000000..4c9a4c47 --- /dev/null +++ b/dpdk/drivers/event/octeontx/ssovf_worker.h @@ -0,0 +1,166 @@ +/* + * BSD LICENSE + * + * Copyright (C) Cavium, Inc. 2017. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + + +#include +#include + +#include + +#include "ssovf_evdev.h" +#include "octeontx_rxtx.h" + +enum { + SSO_SYNC_ORDERED, + SSO_SYNC_ATOMIC, + SSO_SYNC_UNTAGGED, + SSO_SYNC_EMPTY +}; + +#ifndef __hot +#define __hot __attribute__((hot)) +#endif + +/* SSO Operations */ + +static __rte_always_inline struct rte_mbuf * +ssovf_octeontx_wqe_to_pkt(uint64_t work, uint16_t port_info) +{ + struct rte_mbuf *mbuf; + octtx_wqe_t *wqe = (octtx_wqe_t *)(uintptr_t)work; + rte_prefetch_non_temporal(wqe); + + /* Get mbuf from wqe */ + mbuf = (struct rte_mbuf *)((uintptr_t)wqe - + OCTTX_PACKET_WQE_SKIP); + mbuf->packet_type = + ptype_table[wqe->s.w2.lcty][wqe->s.w2.lety][wqe->s.w2.lfty]; + mbuf->data_off = RTE_PTR_DIFF(wqe->s.w3.addr, mbuf->buf_addr); + mbuf->pkt_len = wqe->s.w1.len; + mbuf->data_len = mbuf->pkt_len; + mbuf->nb_segs = 1; + mbuf->ol_flags = 0; + mbuf->port = rte_octeontx_pchan_map[port_info >> 4][port_info & 0xF]; + rte_mbuf_refcnt_set(mbuf, 1); + return mbuf; +} + +static __rte_always_inline uint16_t +ssows_get_work(struct ssows *ws, struct rte_event *ev) +{ + uint64_t get_work0, get_work1; + uint64_t sched_type_queue; + + ssovf_load_pair(get_work0, get_work1, ws->getwork); + + sched_type_queue = (get_work0 >> 32) & 0xfff; + ws->cur_tt = sched_type_queue & 0x3; + ws->cur_grp = sched_type_queue >> 2; + sched_type_queue = sched_type_queue << 38; + ev->event = sched_type_queue | (get_work0 & 0xffffffff); + if (get_work1 && ev->event_type == RTE_EVENT_TYPE_ETHDEV) { + ev->mbuf = ssovf_octeontx_wqe_to_pkt(get_work1, + (ev->event >> 20) & 0x7F); + } else { + ev->u64 = get_work1; + } + + return !!get_work1; +} + +static __rte_always_inline void +ssows_add_work(struct ssows *ws, const uint64_t event_ptr, const uint32_t tag, + const uint8_t new_tt, const uint8_t grp) +{ + uint64_t add_work0; + + add_work0 = tag | ((uint64_t)(new_tt) << 32); + ssovf_store_pair(add_work0, event_ptr, ws->grps[grp]); +} + +static __rte_always_inline void +ssows_swtag_full(struct ssows *ws, const uint64_t event_ptr, const uint32_t tag, + const uint8_t new_tt, const uint8_t grp) +{ + uint64_t swtag_full0; + + swtag_full0 = tag | ((uint64_t)(new_tt & 0x3) << 32) | + ((uint64_t)grp << 34); + ssovf_store_pair(swtag_full0, event_ptr, (ws->base + + SSOW_VHWS_OP_SWTAG_FULL0)); +} + +static __rte_always_inline void +ssows_swtag_desched(struct ssows *ws, uint32_t tag, uint8_t new_tt, uint8_t grp) +{ + uint64_t val; + + val = tag | ((uint64_t)(new_tt & 0x3) << 32) | ((uint64_t)grp << 34); + ssovf_write64(val, ws->base + SSOW_VHWS_OP_SWTAG_DESCHED); +} + +static __rte_always_inline void +ssows_swtag_norm(struct ssows *ws, uint32_t tag, uint8_t new_tt) +{ + uint64_t val; + + val = tag | ((uint64_t)(new_tt & 0x3) << 32); + ssovf_write64(val, ws->base + SSOW_VHWS_OP_SWTAG_NORM); +} + +static __rte_always_inline void +ssows_swtag_untag(struct ssows *ws) +{ + ssovf_write64(0, ws->base + SSOW_VHWS_OP_SWTAG_UNTAG); + ws->cur_tt = SSO_SYNC_UNTAGGED; +} + +static __rte_always_inline void +ssows_upd_wqp(struct ssows *ws, uint8_t grp, uint64_t event_ptr) +{ + ssovf_store_pair((uint64_t)grp << 34, event_ptr, (ws->base + + SSOW_VHWS_OP_UPD_WQP_GRP0)); +} + +static __rte_always_inline void +ssows_desched(struct ssows *ws) +{ + ssovf_write64(0, ws->base + SSOW_VHWS_OP_DESCHED); +} + +static __rte_always_inline void +ssows_swtag_wait(struct ssows *ws) +{ + /* Wait for the SWTAG/SWTAG_FULL operation */ + while (ssovf_read64(ws->base + SSOW_VHWS_SWTP)) + ; +} diff --git a/dpdk/drivers/event/skeleton/Makefile b/dpdk/drivers/event/skeleton/Makefile new file mode 100644 index 00000000..a24738b1 --- /dev/null +++ b/dpdk/drivers/event/skeleton/Makefile @@ -0,0 +1,54 @@ +# BSD LICENSE +# +# Copyright(c) 2016 Cavium, Inc. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Cavium, Inc nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +include $(RTE_SDK)/mk/rte.vars.mk + +# +# library name +# +LIB = librte_pmd_skeleton_event.a + +CFLAGS += $(WERROR_FLAGS) +LDLIBS += -lrte_eal -lrte_eventdev +LDLIBS += -lrte_pci -lrte_bus_pci +LDLIBS += -lrte_bus_vdev + +EXPORT_MAP := rte_pmd_skeleton_event_version.map + +LIBABIVER := 1 + +# +# all source are stored in SRCS-y +# +SRCS-$(CONFIG_RTE_LIBRTE_PMD_SKELETON_EVENTDEV) += skeleton_eventdev.c + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/event/skeleton/rte_pmd_skeleton_event_version.map b/dpdk/drivers/event/skeleton/rte_pmd_skeleton_event_version.map new file mode 100644 index 00000000..8591cc0b --- /dev/null +++ b/dpdk/drivers/event/skeleton/rte_pmd_skeleton_event_version.map @@ -0,0 +1,4 @@ +DPDK_17.05 { + + local: *; +}; diff --git a/dpdk/drivers/event/skeleton/skeleton_eventdev.c b/dpdk/drivers/event/skeleton/skeleton_eventdev.c new file mode 100644 index 00000000..bb554c36 --- /dev/null +++ b/dpdk/drivers/event/skeleton/skeleton_eventdev.c @@ -0,0 +1,504 @@ +/* + * BSD LICENSE + * + * Copyright (C) Cavium, Inc. 2016. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "skeleton_eventdev.h" + +#define EVENTDEV_NAME_SKELETON_PMD event_skeleton +/**< Skeleton event device PMD name */ + +static uint16_t +skeleton_eventdev_enqueue(void *port, const struct rte_event *ev) +{ + struct skeleton_port *sp = port; + + RTE_SET_USED(sp); + RTE_SET_USED(ev); + RTE_SET_USED(port); + + return 0; +} + +static uint16_t +skeleton_eventdev_enqueue_burst(void *port, const struct rte_event ev[], + uint16_t nb_events) +{ + struct skeleton_port *sp = port; + + RTE_SET_USED(sp); + RTE_SET_USED(ev); + RTE_SET_USED(port); + RTE_SET_USED(nb_events); + + return 0; +} + +static uint16_t +skeleton_eventdev_dequeue(void *port, struct rte_event *ev, + uint64_t timeout_ticks) +{ + struct skeleton_port *sp = port; + + RTE_SET_USED(sp); + RTE_SET_USED(ev); + RTE_SET_USED(timeout_ticks); + + return 0; +} + +static uint16_t +skeleton_eventdev_dequeue_burst(void *port, struct rte_event ev[], + uint16_t nb_events, uint64_t timeout_ticks) +{ + struct skeleton_port *sp = port; + + RTE_SET_USED(sp); + RTE_SET_USED(ev); + RTE_SET_USED(nb_events); + RTE_SET_USED(timeout_ticks); + + return 0; +} + +static void +skeleton_eventdev_info_get(struct rte_eventdev *dev, + struct rte_event_dev_info *dev_info) +{ + struct skeleton_eventdev *skel = skeleton_pmd_priv(dev); + + PMD_DRV_FUNC_TRACE(); + + RTE_SET_USED(skel); + + dev_info->min_dequeue_timeout_ns = 1; + dev_info->max_dequeue_timeout_ns = 10000; + dev_info->dequeue_timeout_ns = 25; + dev_info->max_event_queues = 64; + dev_info->max_event_queue_flows = (1ULL << 20); + dev_info->max_event_queue_priority_levels = 8; + dev_info->max_event_priority_levels = 8; + dev_info->max_event_ports = 32; + dev_info->max_event_port_dequeue_depth = 16; + dev_info->max_event_port_enqueue_depth = 16; + dev_info->max_num_events = (1ULL << 20); + dev_info->event_dev_cap = RTE_EVENT_DEV_CAP_QUEUE_QOS | + RTE_EVENT_DEV_CAP_BURST_MODE | + RTE_EVENT_DEV_CAP_EVENT_QOS; +} + +static int +skeleton_eventdev_configure(const struct rte_eventdev *dev) +{ + struct rte_eventdev_data *data = dev->data; + struct rte_event_dev_config *conf = &data->dev_conf; + struct skeleton_eventdev *skel = skeleton_pmd_priv(dev); + + PMD_DRV_FUNC_TRACE(); + + RTE_SET_USED(conf); + RTE_SET_USED(skel); + + PMD_DRV_LOG(DEBUG, "Configured eventdev devid=%d", dev->data->dev_id); + return 0; +} + +static int +skeleton_eventdev_start(struct rte_eventdev *dev) +{ + struct skeleton_eventdev *skel = skeleton_pmd_priv(dev); + + PMD_DRV_FUNC_TRACE(); + + RTE_SET_USED(skel); + + return 0; +} + +static void +skeleton_eventdev_stop(struct rte_eventdev *dev) +{ + struct skeleton_eventdev *skel = skeleton_pmd_priv(dev); + + PMD_DRV_FUNC_TRACE(); + + RTE_SET_USED(skel); +} + +static int +skeleton_eventdev_close(struct rte_eventdev *dev) +{ + struct skeleton_eventdev *skel = skeleton_pmd_priv(dev); + + PMD_DRV_FUNC_TRACE(); + + RTE_SET_USED(skel); + + return 0; +} + +static void +skeleton_eventdev_queue_def_conf(struct rte_eventdev *dev, uint8_t queue_id, + struct rte_event_queue_conf *queue_conf) +{ + struct skeleton_eventdev *skel = skeleton_pmd_priv(dev); + + PMD_DRV_FUNC_TRACE(); + + RTE_SET_USED(skel); + RTE_SET_USED(queue_id); + + queue_conf->nb_atomic_flows = (1ULL << 20); + queue_conf->nb_atomic_order_sequences = (1ULL << 20); + queue_conf->event_queue_cfg = RTE_EVENT_QUEUE_CFG_ALL_TYPES; + queue_conf->priority = RTE_EVENT_DEV_PRIORITY_NORMAL; +} + +static void +skeleton_eventdev_queue_release(struct rte_eventdev *dev, uint8_t queue_id) +{ + PMD_DRV_FUNC_TRACE(); + + RTE_SET_USED(dev); + RTE_SET_USED(queue_id); +} + +static int +skeleton_eventdev_queue_setup(struct rte_eventdev *dev, uint8_t queue_id, + const struct rte_event_queue_conf *queue_conf) +{ + struct skeleton_eventdev *skel = skeleton_pmd_priv(dev); + + PMD_DRV_FUNC_TRACE(); + + RTE_SET_USED(skel); + RTE_SET_USED(queue_conf); + RTE_SET_USED(queue_id); + + return 0; +} + +static void +skeleton_eventdev_port_def_conf(struct rte_eventdev *dev, uint8_t port_id, + struct rte_event_port_conf *port_conf) +{ + struct skeleton_eventdev *skel = skeleton_pmd_priv(dev); + + PMD_DRV_FUNC_TRACE(); + + RTE_SET_USED(skel); + RTE_SET_USED(port_id); + + port_conf->new_event_threshold = 32 * 1024; + port_conf->dequeue_depth = 16; + port_conf->enqueue_depth = 16; +} + +static void +skeleton_eventdev_port_release(void *port) +{ + struct skeleton_port *sp = port; + PMD_DRV_FUNC_TRACE(); + + rte_free(sp); +} + +static int +skeleton_eventdev_port_setup(struct rte_eventdev *dev, uint8_t port_id, + const struct rte_event_port_conf *port_conf) +{ + struct skeleton_port *sp; + struct skeleton_eventdev *skel = skeleton_pmd_priv(dev); + + PMD_DRV_FUNC_TRACE(); + + RTE_SET_USED(skel); + RTE_SET_USED(port_conf); + + /* Free memory prior to re-allocation if needed */ + if (dev->data->ports[port_id] != NULL) { + PMD_DRV_LOG(DEBUG, "Freeing memory prior to re-allocation %d", + port_id); + skeleton_eventdev_port_release(dev->data->ports[port_id]); + dev->data->ports[port_id] = NULL; + } + + /* Allocate event port memory */ + sp = rte_zmalloc_socket("eventdev port", + sizeof(struct skeleton_port), RTE_CACHE_LINE_SIZE, + dev->data->socket_id); + if (sp == NULL) { + PMD_DRV_ERR("Failed to allocate sp port_id=%d", port_id); + return -ENOMEM; + } + + sp->port_id = port_id; + + PMD_DRV_LOG(DEBUG, "[%d] sp=%p", port_id, sp); + + dev->data->ports[port_id] = sp; + return 0; +} + +static int +skeleton_eventdev_port_link(struct rte_eventdev *dev, void *port, + const uint8_t queues[], const uint8_t priorities[], + uint16_t nb_links) +{ + struct skeleton_port *sp = port; + PMD_DRV_FUNC_TRACE(); + + RTE_SET_USED(dev); + RTE_SET_USED(sp); + RTE_SET_USED(queues); + RTE_SET_USED(priorities); + + /* Linked all the queues */ + return (int)nb_links; +} + +static int +skeleton_eventdev_port_unlink(struct rte_eventdev *dev, void *port, + uint8_t queues[], uint16_t nb_unlinks) +{ + struct skeleton_port *sp = port; + PMD_DRV_FUNC_TRACE(); + + RTE_SET_USED(dev); + RTE_SET_USED(sp); + RTE_SET_USED(queues); + + /* Unlinked all the queues */ + return (int)nb_unlinks; + +} + +static int +skeleton_eventdev_timeout_ticks(struct rte_eventdev *dev, uint64_t ns, + uint64_t *timeout_ticks) +{ + struct skeleton_eventdev *skel = skeleton_pmd_priv(dev); + uint32_t scale = 1; + + PMD_DRV_FUNC_TRACE(); + + RTE_SET_USED(skel); + *timeout_ticks = ns * scale; + + return 0; +} + +static void +skeleton_eventdev_dump(struct rte_eventdev *dev, FILE *f) +{ + struct skeleton_eventdev *skel = skeleton_pmd_priv(dev); + + PMD_DRV_FUNC_TRACE(); + + RTE_SET_USED(skel); + RTE_SET_USED(f); +} + + +/* Initialize and register event driver with DPDK Application */ +static const struct rte_eventdev_ops skeleton_eventdev_ops = { + .dev_infos_get = skeleton_eventdev_info_get, + .dev_configure = skeleton_eventdev_configure, + .dev_start = skeleton_eventdev_start, + .dev_stop = skeleton_eventdev_stop, + .dev_close = skeleton_eventdev_close, + .queue_def_conf = skeleton_eventdev_queue_def_conf, + .queue_setup = skeleton_eventdev_queue_setup, + .queue_release = skeleton_eventdev_queue_release, + .port_def_conf = skeleton_eventdev_port_def_conf, + .port_setup = skeleton_eventdev_port_setup, + .port_release = skeleton_eventdev_port_release, + .port_link = skeleton_eventdev_port_link, + .port_unlink = skeleton_eventdev_port_unlink, + .timeout_ticks = skeleton_eventdev_timeout_ticks, + .dump = skeleton_eventdev_dump +}; + +static int +skeleton_eventdev_init(struct rte_eventdev *eventdev) +{ + struct rte_pci_device *pci_dev; + struct skeleton_eventdev *skel = skeleton_pmd_priv(eventdev); + int ret = 0; + + PMD_DRV_FUNC_TRACE(); + + eventdev->dev_ops = &skeleton_eventdev_ops; + eventdev->enqueue = skeleton_eventdev_enqueue; + eventdev->enqueue_burst = skeleton_eventdev_enqueue_burst; + eventdev->dequeue = skeleton_eventdev_dequeue; + eventdev->dequeue_burst = skeleton_eventdev_dequeue_burst; + + /* For secondary processes, the primary has done all the work */ + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return 0; + + pci_dev = RTE_DEV_TO_PCI(eventdev->dev); + + skel->reg_base = (uintptr_t)pci_dev->mem_resource[0].addr; + if (!skel->reg_base) { + PMD_DRV_ERR("Failed to map BAR0"); + ret = -ENODEV; + goto fail; + } + + skel->device_id = pci_dev->id.device_id; + skel->vendor_id = pci_dev->id.vendor_id; + skel->subsystem_device_id = pci_dev->id.subsystem_device_id; + skel->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id; + + PMD_DRV_LOG(DEBUG, "pci device (%x:%x) %u:%u:%u:%u", + pci_dev->id.vendor_id, pci_dev->id.device_id, + pci_dev->addr.domain, pci_dev->addr.bus, + pci_dev->addr.devid, pci_dev->addr.function); + + PMD_DRV_LOG(INFO, "dev_id=%d socket_id=%d (%x:%x)", + eventdev->data->dev_id, eventdev->data->socket_id, + skel->vendor_id, skel->device_id); + +fail: + return ret; +} + +/* PCI based event device */ + +#define EVENTDEV_SKEL_VENDOR_ID 0x177d +#define EVENTDEV_SKEL_PRODUCT_ID 0x0001 + +static const struct rte_pci_id pci_id_skeleton_map[] = { + { + RTE_PCI_DEVICE(EVENTDEV_SKEL_VENDOR_ID, + EVENTDEV_SKEL_PRODUCT_ID) + }, + { + .vendor_id = 0, + }, +}; + +static int +event_skeleton_pci_probe(struct rte_pci_driver *pci_drv, + struct rte_pci_device *pci_dev) +{ + return rte_event_pmd_pci_probe(pci_drv, pci_dev, + sizeof(struct skeleton_eventdev), skeleton_eventdev_init); +} + +static int +event_skeleton_pci_remove(struct rte_pci_device *pci_dev) +{ + return rte_event_pmd_pci_remove(pci_dev, NULL); +} + +static struct rte_pci_driver pci_eventdev_skeleton_pmd = { + .id_table = pci_id_skeleton_map, + .drv_flags = RTE_PCI_DRV_NEED_MAPPING, + .probe = event_skeleton_pci_probe, + .remove = event_skeleton_pci_remove, +}; + +RTE_PMD_REGISTER_PCI(event_skeleton_pci, pci_eventdev_skeleton_pmd); +RTE_PMD_REGISTER_PCI_TABLE(event_skeleton_pci, pci_id_skeleton_map); + +/* VDEV based event device */ + +static int +skeleton_eventdev_create(const char *name, int socket_id) +{ + struct rte_eventdev *eventdev; + + eventdev = rte_event_pmd_vdev_init(name, + sizeof(struct skeleton_eventdev), socket_id); + if (eventdev == NULL) { + PMD_DRV_ERR("Failed to create eventdev vdev %s", name); + goto fail; + } + + eventdev->dev_ops = &skeleton_eventdev_ops; + eventdev->enqueue = skeleton_eventdev_enqueue; + eventdev->enqueue_burst = skeleton_eventdev_enqueue_burst; + eventdev->dequeue = skeleton_eventdev_dequeue; + eventdev->dequeue_burst = skeleton_eventdev_dequeue_burst; + + return 0; +fail: + return -EFAULT; +} + +static int +skeleton_eventdev_probe(struct rte_vdev_device *vdev) +{ + const char *name; + + name = rte_vdev_device_name(vdev); + RTE_LOG(INFO, PMD, "Initializing %s on NUMA node %d\n", name, + rte_socket_id()); + return skeleton_eventdev_create(name, rte_socket_id()); +} + +static int +skeleton_eventdev_remove(struct rte_vdev_device *vdev) +{ + const char *name; + + name = rte_vdev_device_name(vdev); + PMD_DRV_LOG(INFO, "Closing %s on NUMA node %d", name, rte_socket_id()); + + return rte_event_pmd_vdev_uninit(name); +} + +static struct rte_vdev_driver vdev_eventdev_skeleton_pmd = { + .probe = skeleton_eventdev_probe, + .remove = skeleton_eventdev_remove +}; + +RTE_PMD_REGISTER_VDEV(EVENTDEV_NAME_SKELETON_PMD, vdev_eventdev_skeleton_pmd); diff --git a/dpdk/drivers/event/skeleton/skeleton_eventdev.h b/dpdk/drivers/event/skeleton/skeleton_eventdev.h new file mode 100644 index 00000000..32064721 --- /dev/null +++ b/dpdk/drivers/event/skeleton/skeleton_eventdev.h @@ -0,0 +1,69 @@ +/* + * BSD LICENSE + * + * Copyright (C) Cavium, Inc. 2016. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __SKELETON_EVENTDEV_H__ +#define __SKELETON_EVENTDEV_H__ + +#include +#include + +#ifdef RTE_LIBRTE_PMD_SKELETON_EVENTDEV_DEBUG +#define PMD_DRV_LOG(level, fmt, args...) \ + RTE_LOG(level, PMD, "%s(): " fmt "\n", __func__, ## args) +#define PMD_DRV_FUNC_TRACE() PMD_DRV_LOG(DEBUG, ">>") +#else +#define PMD_DRV_LOG(level, fmt, args...) do { } while (0) +#define PMD_DRV_FUNC_TRACE() do { } while (0) +#endif + +#define PMD_DRV_ERR(fmt, args...) \ + RTE_LOG(ERR, PMD, "%s(): " fmt "\n", __func__, ## args) + +struct skeleton_eventdev { + uintptr_t reg_base; + uint16_t device_id; + uint16_t vendor_id; + uint16_t subsystem_device_id; + uint16_t subsystem_vendor_id; +} __rte_cache_aligned; + +struct skeleton_port { + uint8_t port_id; +} __rte_cache_aligned; + +static inline struct skeleton_eventdev * +skeleton_pmd_priv(const struct rte_eventdev *eventdev) +{ + return eventdev->data->dev_private; +} + +#endif /* __SKELETON_EVENTDEV_H__ */ diff --git a/dpdk/drivers/event/sw/Makefile b/dpdk/drivers/event/sw/Makefile new file mode 100644 index 00000000..2f2b67ba --- /dev/null +++ b/dpdk/drivers/event/sw/Makefile @@ -0,0 +1,64 @@ +# BSD LICENSE +# +# Copyright(c) 2016-2017 Intel Corporation. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +include $(RTE_SDK)/mk/rte.vars.mk + +# library name +LIB = librte_pmd_sw_event.a + +# build flags +CFLAGS += -O3 +CFLAGS += $(WERROR_FLAGS) +# for older GCC versions, allow us to initialize an event using +# designated initializers. +ifeq ($(CONFIG_RTE_TOOLCHAIN_GCC),y) +ifeq ($(shell test $(GCC_VERSION) -le 50 && echo 1), 1) +CFLAGS += -Wno-missing-field-initializers +endif +endif +LDLIBS += -lrte_eal -lrte_eventdev -lrte_kvargs -lrte_ring +LDLIBS += -lrte_bus_vdev + +# library version +LIBABIVER := 1 + +# versioning export map +EXPORT_MAP := rte_pmd_sw_event_version.map + +# library source files +SRCS-$(CONFIG_RTE_LIBRTE_PMD_SW_EVENTDEV) += sw_evdev.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_SW_EVENTDEV) += sw_evdev_worker.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_SW_EVENTDEV) += sw_evdev_scheduler.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_SW_EVENTDEV) += sw_evdev_xstats.c + +# export include files +SYMLINK-y-include += + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/event/sw/event_ring.h b/dpdk/drivers/event/sw/event_ring.h new file mode 100644 index 00000000..734a3b4b --- /dev/null +++ b/dpdk/drivers/event/sw/event_ring.h @@ -0,0 +1,181 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * Generic ring structure for passing events from one core to another. + * + * Used by the software scheduler for the producer and consumer rings for + * each port, i.e. for passing events from worker cores to scheduler and + * vice-versa. Designed for single-producer, single-consumer use with two + * cores working on each ring. + */ + +#ifndef _EVENT_RING_ +#define _EVENT_RING_ + +#include + +#include +#include +#include + +#define QE_RING_NAMESIZE 32 + +struct qe_ring { + char name[QE_RING_NAMESIZE] __rte_cache_aligned; + uint32_t ring_size; /* size of memory block allocated to the ring */ + uint32_t mask; /* mask for read/write values == ring_size -1 */ + uint32_t size; /* actual usable space in the ring */ + volatile uint32_t write_idx __rte_cache_aligned; + volatile uint32_t read_idx __rte_cache_aligned; + + struct rte_event ring[0] __rte_cache_aligned; +}; + +static inline struct qe_ring * +qe_ring_create(const char *name, unsigned int size, unsigned int socket_id) +{ + struct qe_ring *retval; + const uint32_t ring_size = rte_align32pow2(size + 1); + size_t memsize = sizeof(*retval) + + (ring_size * sizeof(retval->ring[0])); + + retval = rte_zmalloc_socket(NULL, memsize, 0, socket_id); + if (retval == NULL) + goto end; + + snprintf(retval->name, sizeof(retval->name), "EVDEV_RG_%s", name); + retval->ring_size = ring_size; + retval->mask = ring_size - 1; + retval->size = size; +end: + return retval; +} + +static inline void +qe_ring_destroy(struct qe_ring *r) +{ + rte_free(r); +} + +static __rte_always_inline unsigned int +qe_ring_count(const struct qe_ring *r) +{ + return r->write_idx - r->read_idx; +} + +static __rte_always_inline unsigned int +qe_ring_free_count(const struct qe_ring *r) +{ + return r->size - qe_ring_count(r); +} + +static __rte_always_inline unsigned int +qe_ring_enqueue_burst(struct qe_ring *r, const struct rte_event *qes, + unsigned int nb_qes, uint16_t *free_count) +{ + const uint32_t size = r->size; + const uint32_t mask = r->mask; + const uint32_t read = r->read_idx; + uint32_t write = r->write_idx; + const uint32_t space = read + size - write; + uint32_t i; + + if (space < nb_qes) + nb_qes = space; + + for (i = 0; i < nb_qes; i++, write++) + r->ring[write & mask] = qes[i]; + + rte_smp_wmb(); + + if (nb_qes != 0) + r->write_idx = write; + + *free_count = space - nb_qes; + + return nb_qes; +} + +static __rte_always_inline unsigned int +qe_ring_enqueue_burst_with_ops(struct qe_ring *r, const struct rte_event *qes, + unsigned int nb_qes, uint8_t *ops) +{ + const uint32_t size = r->size; + const uint32_t mask = r->mask; + const uint32_t read = r->read_idx; + uint32_t write = r->write_idx; + const uint32_t space = read + size - write; + uint32_t i; + + if (space < nb_qes) + nb_qes = space; + + for (i = 0; i < nb_qes; i++, write++) { + r->ring[write & mask] = qes[i]; + r->ring[write & mask].op = ops[i]; + } + + rte_smp_wmb(); + + if (nb_qes != 0) + r->write_idx = write; + + return nb_qes; +} + +static __rte_always_inline unsigned int +qe_ring_dequeue_burst(struct qe_ring *r, struct rte_event *qes, + unsigned int nb_qes) +{ + const uint32_t mask = r->mask; + uint32_t read = r->read_idx; + const uint32_t write = r->write_idx; + const uint32_t items = write - read; + uint32_t i; + + if (items < nb_qes) + nb_qes = items; + + + for (i = 0; i < nb_qes; i++, read++) + qes[i] = r->ring[read & mask]; + + rte_smp_rmb(); + + if (nb_qes != 0) + r->read_idx += nb_qes; + + return nb_qes; +} + +#endif diff --git a/dpdk/drivers/event/sw/iq_ring.h b/dpdk/drivers/event/sw/iq_ring.h new file mode 100644 index 00000000..64cf6784 --- /dev/null +++ b/dpdk/drivers/event/sw/iq_ring.h @@ -0,0 +1,172 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * Ring structure definitions used for the internal ring buffers of the + * SW eventdev implementation. These are designed for single-core use only. + */ +#ifndef _IQ_RING_ +#define _IQ_RING_ + +#include + +#include +#include +#include +#include + +#define IQ_RING_NAMESIZE 12 +#define QID_IQ_DEPTH 512 +#define QID_IQ_MASK (uint16_t)(QID_IQ_DEPTH - 1) + +struct iq_ring { + char name[IQ_RING_NAMESIZE] __rte_cache_aligned; + uint16_t write_idx; + uint16_t read_idx; + + struct rte_event ring[QID_IQ_DEPTH]; +}; + +static inline struct iq_ring * +iq_ring_create(const char *name, unsigned int socket_id) +{ + struct iq_ring *retval; + + retval = rte_malloc_socket(NULL, sizeof(*retval), 0, socket_id); + if (retval == NULL) + goto end; + + snprintf(retval->name, sizeof(retval->name), "%s", name); + retval->write_idx = retval->read_idx = 0; +end: + return retval; +} + +static inline void +iq_ring_destroy(struct iq_ring *r) +{ + rte_free(r); +} + +static __rte_always_inline uint16_t +iq_ring_count(const struct iq_ring *r) +{ + return r->write_idx - r->read_idx; +} + +static __rte_always_inline uint16_t +iq_ring_free_count(const struct iq_ring *r) +{ + return QID_IQ_MASK - iq_ring_count(r); +} + +static __rte_always_inline uint16_t +iq_ring_enqueue_burst(struct iq_ring *r, struct rte_event *qes, uint16_t nb_qes) +{ + const uint16_t read = r->read_idx; + uint16_t write = r->write_idx; + const uint16_t space = read + QID_IQ_MASK - write; + uint16_t i; + + if (space < nb_qes) + nb_qes = space; + + for (i = 0; i < nb_qes; i++, write++) + r->ring[write & QID_IQ_MASK] = qes[i]; + + r->write_idx = write; + + return nb_qes; +} + +static __rte_always_inline uint16_t +iq_ring_dequeue_burst(struct iq_ring *r, struct rte_event *qes, uint16_t nb_qes) +{ + uint16_t read = r->read_idx; + const uint16_t write = r->write_idx; + const uint16_t items = write - read; + uint16_t i; + + for (i = 0; i < nb_qes; i++, read++) + qes[i] = r->ring[read & QID_IQ_MASK]; + + if (items < nb_qes) + nb_qes = items; + + r->read_idx += nb_qes; + + return nb_qes; +} + +/* assumes there is space, from a previous dequeue_burst */ +static __rte_always_inline uint16_t +iq_ring_put_back(struct iq_ring *r, struct rte_event *qes, uint16_t nb_qes) +{ + uint16_t i, read = r->read_idx; + + for (i = nb_qes; i-- > 0; ) + r->ring[--read & QID_IQ_MASK] = qes[i]; + + r->read_idx = read; + return nb_qes; +} + +static __rte_always_inline const struct rte_event * +iq_ring_peek(const struct iq_ring *r) +{ + return &r->ring[r->read_idx & QID_IQ_MASK]; +} + +static __rte_always_inline void +iq_ring_pop(struct iq_ring *r) +{ + r->read_idx++; +} + +static __rte_always_inline int +iq_ring_enqueue(struct iq_ring *r, const struct rte_event *qe) +{ + const uint16_t read = r->read_idx; + const uint16_t write = r->write_idx; + const uint16_t space = read + QID_IQ_MASK - write; + + if (space == 0) + return -1; + + r->ring[write & QID_IQ_MASK] = *qe; + + r->write_idx = write + 1; + + return 0; +} + +#endif diff --git a/dpdk/drivers/event/sw/rte_pmd_sw_event_version.map b/dpdk/drivers/event/sw/rte_pmd_sw_event_version.map new file mode 100644 index 00000000..5352e7e3 --- /dev/null +++ b/dpdk/drivers/event/sw/rte_pmd_sw_event_version.map @@ -0,0 +1,3 @@ +DPDK_17.05 { + local: *; +}; diff --git a/dpdk/drivers/event/sw/sw_evdev.c b/dpdk/drivers/event/sw/sw_evdev.c new file mode 100644 index 00000000..d989346b --- /dev/null +++ b/dpdk/drivers/event/sw/sw_evdev.c @@ -0,0 +1,907 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "sw_evdev.h" +#include "iq_ring.h" + +#define EVENTDEV_NAME_SW_PMD event_sw +#define NUMA_NODE_ARG "numa_node" +#define SCHED_QUANTA_ARG "sched_quanta" +#define CREDIT_QUANTA_ARG "credit_quanta" + +static void +sw_info_get(struct rte_eventdev *dev, struct rte_event_dev_info *info); + +static int +sw_port_link(struct rte_eventdev *dev, void *port, const uint8_t queues[], + const uint8_t priorities[], uint16_t num) +{ + struct sw_port *p = port; + struct sw_evdev *sw = sw_pmd_priv(dev); + int i; + + RTE_SET_USED(priorities); + for (i = 0; i < num; i++) { + struct sw_qid *q = &sw->qids[queues[i]]; + unsigned int j; + + /* check for qid map overflow */ + if (q->cq_num_mapped_cqs >= RTE_DIM(q->cq_map)) { + rte_errno = -EDQUOT; + break; + } + + if (p->is_directed && p->num_qids_mapped > 0) { + rte_errno = -EDQUOT; + break; + } + + for (j = 0; j < q->cq_num_mapped_cqs; j++) { + if (q->cq_map[j] == p->id) + break; + } + + /* check if port is already linked */ + if (j < q->cq_num_mapped_cqs) + continue; + + if (q->type == SW_SCHED_TYPE_DIRECT) { + /* check directed qids only map to one port */ + if (p->num_qids_mapped > 0) { + rte_errno = -EDQUOT; + break; + } + /* check port only takes a directed flow */ + if (num > 1) { + rte_errno = -EDQUOT; + break; + } + + p->is_directed = 1; + p->num_qids_mapped = 1; + } else if (q->type == RTE_SCHED_TYPE_ORDERED) { + p->num_ordered_qids++; + p->num_qids_mapped++; + } else if (q->type == RTE_SCHED_TYPE_ATOMIC || + q->type == RTE_SCHED_TYPE_PARALLEL) { + p->num_qids_mapped++; + } + + q->cq_map[q->cq_num_mapped_cqs] = p->id; + rte_smp_wmb(); + q->cq_num_mapped_cqs++; + } + return i; +} + +static int +sw_port_unlink(struct rte_eventdev *dev, void *port, uint8_t queues[], + uint16_t nb_unlinks) +{ + struct sw_port *p = port; + struct sw_evdev *sw = sw_pmd_priv(dev); + unsigned int i, j; + + int unlinked = 0; + for (i = 0; i < nb_unlinks; i++) { + struct sw_qid *q = &sw->qids[queues[i]]; + for (j = 0; j < q->cq_num_mapped_cqs; j++) { + if (q->cq_map[j] == p->id) { + q->cq_map[j] = + q->cq_map[q->cq_num_mapped_cqs - 1]; + rte_smp_wmb(); + q->cq_num_mapped_cqs--; + unlinked++; + + p->num_qids_mapped--; + + if (q->type == RTE_SCHED_TYPE_ORDERED) + p->num_ordered_qids--; + + continue; + } + } + } + return unlinked; +} + +static int +sw_port_setup(struct rte_eventdev *dev, uint8_t port_id, + const struct rte_event_port_conf *conf) +{ + struct sw_evdev *sw = sw_pmd_priv(dev); + struct sw_port *p = &sw->ports[port_id]; + char buf[RTE_RING_NAMESIZE]; + unsigned int i; + + struct rte_event_dev_info info; + sw_info_get(dev, &info); + + /* detect re-configuring and return credits to instance if needed */ + if (p->initialized) { + /* taking credits from pool is done one quanta at a time, and + * credits may be spend (counted in p->inflights) or still + * available in the port (p->inflight_credits). We must return + * the sum to no leak credits + */ + int possible_inflights = p->inflight_credits + p->inflights; + rte_atomic32_sub(&sw->inflights, possible_inflights); + } + + *p = (struct sw_port){0}; /* zero entire structure */ + p->id = port_id; + p->sw = sw; + + /* check to see if rings exists - port_setup() can be called multiple + * times legally (assuming device is stopped). If ring exists, free it + * to so it gets re-created with the correct size + */ + snprintf(buf, sizeof(buf), "sw%d_p%u_%s", dev->data->dev_id, + port_id, "rx_worker_ring"); + struct rte_event_ring *existing_ring = rte_event_ring_lookup(buf); + if (existing_ring) + rte_event_ring_free(existing_ring); + + p->rx_worker_ring = rte_event_ring_create(buf, MAX_SW_PROD_Q_DEPTH, + dev->data->socket_id, + RING_F_SP_ENQ | RING_F_SC_DEQ | RING_F_EXACT_SZ); + if (p->rx_worker_ring == NULL) { + SW_LOG_ERR("Error creating RX worker ring for port %d\n", + port_id); + return -1; + } + + p->inflight_max = conf->new_event_threshold; + + /* check if ring exists, same as rx_worker above */ + snprintf(buf, sizeof(buf), "sw%d_p%u, %s", dev->data->dev_id, + port_id, "cq_worker_ring"); + existing_ring = rte_event_ring_lookup(buf); + if (existing_ring) + rte_event_ring_free(existing_ring); + + p->cq_worker_ring = rte_event_ring_create(buf, conf->dequeue_depth, + dev->data->socket_id, + RING_F_SP_ENQ | RING_F_SC_DEQ | RING_F_EXACT_SZ); + if (p->cq_worker_ring == NULL) { + rte_event_ring_free(p->rx_worker_ring); + SW_LOG_ERR("Error creating CQ worker ring for port %d\n", + port_id); + return -1; + } + sw->cq_ring_space[port_id] = conf->dequeue_depth; + + /* set hist list contents to empty */ + for (i = 0; i < SW_PORT_HIST_LIST; i++) { + p->hist_list[i].fid = -1; + p->hist_list[i].qid = -1; + } + dev->data->ports[port_id] = p; + + rte_smp_wmb(); + p->initialized = 1; + return 0; +} + +static void +sw_port_release(void *port) +{ + struct sw_port *p = (void *)port; + if (p == NULL) + return; + + rte_event_ring_free(p->rx_worker_ring); + rte_event_ring_free(p->cq_worker_ring); + memset(p, 0, sizeof(*p)); +} + +static int32_t +qid_init(struct sw_evdev *sw, unsigned int idx, int type, + const struct rte_event_queue_conf *queue_conf) +{ + unsigned int i; + int dev_id = sw->data->dev_id; + int socket_id = sw->data->socket_id; + char buf[IQ_RING_NAMESIZE]; + struct sw_qid *qid = &sw->qids[idx]; + + for (i = 0; i < SW_IQS_MAX; i++) { + snprintf(buf, sizeof(buf), "q_%u_iq_%d", idx, i); + qid->iq[i] = iq_ring_create(buf, socket_id); + if (!qid->iq[i]) { + SW_LOG_DBG("ring create failed"); + goto cleanup; + } + } + + /* Initialize the FID structures to no pinning (-1), and zero packets */ + const struct sw_fid_t fid = {.cq = -1, .pcount = 0}; + for (i = 0; i < RTE_DIM(qid->fids); i++) + qid->fids[i] = fid; + + qid->id = idx; + qid->type = type; + qid->priority = queue_conf->priority; + + if (qid->type == RTE_SCHED_TYPE_ORDERED) { + char ring_name[RTE_RING_NAMESIZE]; + uint32_t window_size; + + /* rte_ring and window_size_mask require require window_size to + * be a power-of-2. + */ + window_size = rte_align32pow2( + queue_conf->nb_atomic_order_sequences); + + qid->window_size = window_size - 1; + + if (!window_size) { + SW_LOG_DBG( + "invalid reorder_window_size for ordered queue\n" + ); + goto cleanup; + } + + snprintf(buf, sizeof(buf), "sw%d_iq_%d_rob", dev_id, i); + qid->reorder_buffer = rte_zmalloc_socket(buf, + window_size * sizeof(qid->reorder_buffer[0]), + 0, socket_id); + if (!qid->reorder_buffer) { + SW_LOG_DBG("reorder_buffer malloc failed\n"); + goto cleanup; + } + + memset(&qid->reorder_buffer[0], + 0, + window_size * sizeof(qid->reorder_buffer[0])); + + snprintf(ring_name, sizeof(ring_name), "sw%d_q%d_freelist", + dev_id, idx); + + /* lookup the ring, and if it already exists, free it */ + struct rte_ring *cleanup = rte_ring_lookup(ring_name); + if (cleanup) + rte_ring_free(cleanup); + + qid->reorder_buffer_freelist = rte_ring_create(ring_name, + window_size, + socket_id, + RING_F_SP_ENQ | RING_F_SC_DEQ); + if (!qid->reorder_buffer_freelist) { + SW_LOG_DBG("freelist ring create failed"); + goto cleanup; + } + + /* Populate the freelist with reorder buffer entries. Enqueue + * 'window_size - 1' entries because the rte_ring holds only + * that many. + */ + for (i = 0; i < window_size - 1; i++) { + if (rte_ring_sp_enqueue(qid->reorder_buffer_freelist, + &qid->reorder_buffer[i]) < 0) + goto cleanup; + } + + qid->reorder_buffer_index = 0; + qid->cq_next_tx = 0; + } + + qid->initialized = 1; + + return 0; + +cleanup: + for (i = 0; i < SW_IQS_MAX; i++) { + if (qid->iq[i]) + iq_ring_destroy(qid->iq[i]); + } + + if (qid->reorder_buffer) { + rte_free(qid->reorder_buffer); + qid->reorder_buffer = NULL; + } + + if (qid->reorder_buffer_freelist) { + rte_ring_free(qid->reorder_buffer_freelist); + qid->reorder_buffer_freelist = NULL; + } + + return -EINVAL; +} + +static void +sw_queue_release(struct rte_eventdev *dev, uint8_t id) +{ + struct sw_evdev *sw = sw_pmd_priv(dev); + struct sw_qid *qid = &sw->qids[id]; + uint32_t i; + + for (i = 0; i < SW_IQS_MAX; i++) + iq_ring_destroy(qid->iq[i]); + + if (qid->type == RTE_SCHED_TYPE_ORDERED) { + rte_free(qid->reorder_buffer); + rte_ring_free(qid->reorder_buffer_freelist); + } + memset(qid, 0, sizeof(*qid)); +} + +static int +sw_queue_setup(struct rte_eventdev *dev, uint8_t queue_id, + const struct rte_event_queue_conf *conf) +{ + int type; + + type = conf->schedule_type; + + if (RTE_EVENT_QUEUE_CFG_SINGLE_LINK & conf->event_queue_cfg) { + type = SW_SCHED_TYPE_DIRECT; + } else if (RTE_EVENT_QUEUE_CFG_ALL_TYPES + & conf->event_queue_cfg) { + SW_LOG_ERR("QUEUE_CFG_ALL_TYPES not supported\n"); + return -ENOTSUP; + } + + struct sw_evdev *sw = sw_pmd_priv(dev); + + if (sw->qids[queue_id].initialized) + sw_queue_release(dev, queue_id); + + return qid_init(sw, queue_id, type, conf); +} + +static void +sw_queue_def_conf(struct rte_eventdev *dev, uint8_t queue_id, + struct rte_event_queue_conf *conf) +{ + RTE_SET_USED(dev); + RTE_SET_USED(queue_id); + + static const struct rte_event_queue_conf default_conf = { + .nb_atomic_flows = 4096, + .nb_atomic_order_sequences = 1, + .schedule_type = RTE_SCHED_TYPE_ATOMIC, + .priority = RTE_EVENT_DEV_PRIORITY_NORMAL, + }; + + *conf = default_conf; +} + +static void +sw_port_def_conf(struct rte_eventdev *dev, uint8_t port_id, + struct rte_event_port_conf *port_conf) +{ + RTE_SET_USED(dev); + RTE_SET_USED(port_id); + + port_conf->new_event_threshold = 1024; + port_conf->dequeue_depth = 16; + port_conf->enqueue_depth = 16; +} + +static int +sw_dev_configure(const struct rte_eventdev *dev) +{ + struct sw_evdev *sw = sw_pmd_priv(dev); + const struct rte_eventdev_data *data = dev->data; + const struct rte_event_dev_config *conf = &data->dev_conf; + + sw->qid_count = conf->nb_event_queues; + sw->port_count = conf->nb_event_ports; + sw->nb_events_limit = conf->nb_events_limit; + rte_atomic32_set(&sw->inflights, 0); + + if (conf->event_dev_cfg & RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT) + return -ENOTSUP; + + return 0; +} + +struct rte_eth_dev; + +static int +sw_eth_rx_adapter_caps_get(const struct rte_eventdev *dev, + const struct rte_eth_dev *eth_dev, + uint32_t *caps) +{ + RTE_SET_USED(dev); + RTE_SET_USED(eth_dev); + *caps = RTE_EVENT_ETH_RX_ADAPTER_SW_CAP; + return 0; +} + +static void +sw_info_get(struct rte_eventdev *dev, struct rte_event_dev_info *info) +{ + RTE_SET_USED(dev); + + static const struct rte_event_dev_info evdev_sw_info = { + .driver_name = SW_PMD_NAME, + .max_event_queues = RTE_EVENT_MAX_QUEUES_PER_DEV, + .max_event_queue_flows = SW_QID_NUM_FIDS, + .max_event_queue_priority_levels = SW_Q_PRIORITY_MAX, + .max_event_priority_levels = SW_IQS_MAX, + .max_event_ports = SW_PORTS_MAX, + .max_event_port_dequeue_depth = MAX_SW_CONS_Q_DEPTH, + .max_event_port_enqueue_depth = MAX_SW_PROD_Q_DEPTH, + .max_num_events = SW_INFLIGHT_EVENTS_TOTAL, + .event_dev_cap = (RTE_EVENT_DEV_CAP_QUEUE_QOS | + RTE_EVENT_DEV_CAP_BURST_MODE | + RTE_EVENT_DEV_CAP_EVENT_QOS), + }; + + *info = evdev_sw_info; +} + +static void +sw_dump(struct rte_eventdev *dev, FILE *f) +{ + const struct sw_evdev *sw = sw_pmd_priv(dev); + + static const char * const q_type_strings[] = { + "Ordered", "Atomic", "Parallel", "Directed" + }; + uint32_t i; + fprintf(f, "EventDev %s: ports %d, qids %d\n", "todo-fix-name", + sw->port_count, sw->qid_count); + + fprintf(f, "\trx %"PRIu64"\n\tdrop %"PRIu64"\n\ttx %"PRIu64"\n", + sw->stats.rx_pkts, sw->stats.rx_dropped, sw->stats.tx_pkts); + fprintf(f, "\tsched calls: %"PRIu64"\n", sw->sched_called); + fprintf(f, "\tsched cq/qid call: %"PRIu64"\n", sw->sched_cq_qid_called); + fprintf(f, "\tsched no IQ enq: %"PRIu64"\n", sw->sched_no_iq_enqueues); + fprintf(f, "\tsched no CQ enq: %"PRIu64"\n", sw->sched_no_cq_enqueues); + uint32_t inflights = rte_atomic32_read(&sw->inflights); + uint32_t credits = sw->nb_events_limit - inflights; + fprintf(f, "\tinflight %d, credits: %d\n", inflights, credits); + +#define COL_RED "\x1b[31m" +#define COL_RESET "\x1b[0m" + + for (i = 0; i < sw->port_count; i++) { + int max, j; + const struct sw_port *p = &sw->ports[i]; + if (!p->initialized) { + fprintf(f, " %sPort %d not initialized.%s\n", + COL_RED, i, COL_RESET); + continue; + } + fprintf(f, " Port %d %s\n", i, + p->is_directed ? " (SingleCons)" : ""); + fprintf(f, "\trx %"PRIu64"\tdrop %"PRIu64"\ttx %"PRIu64 + "\t%sinflight %d%s\n", sw->ports[i].stats.rx_pkts, + sw->ports[i].stats.rx_dropped, + sw->ports[i].stats.tx_pkts, + (p->inflights == p->inflight_max) ? + COL_RED : COL_RESET, + sw->ports[i].inflights, COL_RESET); + + fprintf(f, "\tMax New: %u" + "\tAvg cycles PP: %"PRIu64"\tCredits: %u\n", + sw->ports[i].inflight_max, + sw->ports[i].avg_pkt_ticks, + sw->ports[i].inflight_credits); + fprintf(f, "\tReceive burst distribution:\n"); + float zp_percent = p->zero_polls * 100.0 / p->total_polls; + fprintf(f, zp_percent < 10 ? "\t\t0:%.02f%% " : "\t\t0:%.0f%% ", + zp_percent); + for (max = (int)RTE_DIM(p->poll_buckets); max-- > 0;) + if (p->poll_buckets[max] != 0) + break; + for (j = 0; j <= max; j++) { + if (p->poll_buckets[j] != 0) { + float poll_pc = p->poll_buckets[j] * 100.0 / + p->total_polls; + fprintf(f, "%u-%u:%.02f%% ", + ((j << SW_DEQ_STAT_BUCKET_SHIFT) + 1), + ((j+1) << SW_DEQ_STAT_BUCKET_SHIFT), + poll_pc); + } + } + fprintf(f, "\n"); + + if (p->rx_worker_ring) { + uint64_t used = rte_event_ring_count(p->rx_worker_ring); + uint64_t space = rte_event_ring_free_count( + p->rx_worker_ring); + const char *col = (space == 0) ? COL_RED : COL_RESET; + fprintf(f, "\t%srx ring used: %4"PRIu64"\tfree: %4" + PRIu64 COL_RESET"\n", col, used, space); + } else + fprintf(f, "\trx ring not initialized.\n"); + + if (p->cq_worker_ring) { + uint64_t used = rte_event_ring_count(p->cq_worker_ring); + uint64_t space = rte_event_ring_free_count( + p->cq_worker_ring); + const char *col = (space == 0) ? COL_RED : COL_RESET; + fprintf(f, "\t%scq ring used: %4"PRIu64"\tfree: %4" + PRIu64 COL_RESET"\n", col, used, space); + } else + fprintf(f, "\tcq ring not initialized.\n"); + } + + for (i = 0; i < sw->qid_count; i++) { + const struct sw_qid *qid = &sw->qids[i]; + if (!qid->initialized) { + fprintf(f, " %sQueue %d not initialized.%s\n", + COL_RED, i, COL_RESET); + continue; + } + int affinities_per_port[SW_PORTS_MAX] = {0}; + uint32_t inflights = 0; + + fprintf(f, " Queue %d (%s)\n", i, q_type_strings[qid->type]); + fprintf(f, "\trx %"PRIu64"\tdrop %"PRIu64"\ttx %"PRIu64"\n", + qid->stats.rx_pkts, qid->stats.rx_dropped, + qid->stats.tx_pkts); + if (qid->type == RTE_SCHED_TYPE_ORDERED) { + struct rte_ring *rob_buf_free = + qid->reorder_buffer_freelist; + if (rob_buf_free) + fprintf(f, "\tReorder entries in use: %u\n", + rte_ring_free_count(rob_buf_free)); + else + fprintf(f, + "\tReorder buffer not initialized\n"); + } + + uint32_t flow; + for (flow = 0; flow < RTE_DIM(qid->fids); flow++) + if (qid->fids[flow].cq != -1) { + affinities_per_port[qid->fids[flow].cq]++; + inflights += qid->fids[flow].pcount; + } + + uint32_t port; + fprintf(f, "\tPer Port Stats:\n"); + for (port = 0; port < sw->port_count; port++) { + fprintf(f, "\t Port %d: Pkts: %"PRIu64, port, + qid->to_port[port]); + fprintf(f, "\tFlows: %d\n", affinities_per_port[port]); + } + + uint32_t iq; + uint32_t iq_printed = 0; + for (iq = 0; iq < SW_IQS_MAX; iq++) { + if (!qid->iq[iq]) { + fprintf(f, "\tiq %d is not initialized.\n", iq); + iq_printed = 1; + continue; + } + uint32_t used = iq_ring_count(qid->iq[iq]); + uint32_t free = iq_ring_free_count(qid->iq[iq]); + const char *col = (free == 0) ? COL_RED : COL_RESET; + if (used > 0) { + fprintf(f, "\t%siq %d: Used %d\tFree %d" + COL_RESET"\n", col, iq, used, free); + iq_printed = 1; + } + } + if (iq_printed == 0) + fprintf(f, "\t-- iqs empty --\n"); + } +} + +static int +sw_start(struct rte_eventdev *dev) +{ + unsigned int i, j; + struct sw_evdev *sw = sw_pmd_priv(dev); + + rte_service_component_runstate_set(sw->service_id, 1); + + /* check a service core is mapped to this service */ + if (!rte_service_runstate_get(sw->service_id)) { + SW_LOG_ERR("Warning: No Service core enabled on service %s\n", + sw->service_name); + return -ENOENT; + } + + /* check all ports are set up */ + for (i = 0; i < sw->port_count; i++) + if (sw->ports[i].rx_worker_ring == NULL) { + SW_LOG_ERR("Port %d not configured\n", i); + return -ESTALE; + } + + /* check all queues are configured and mapped to ports*/ + for (i = 0; i < sw->qid_count; i++) + if (sw->qids[i].iq[0] == NULL || + sw->qids[i].cq_num_mapped_cqs == 0) { + SW_LOG_ERR("Queue %d not configured\n", i); + return -ENOLINK; + } + + /* build up our prioritized array of qids */ + /* We don't use qsort here, as if all/multiple entries have the same + * priority, the result is non-deterministic. From "man 3 qsort": + * "If two members compare as equal, their order in the sorted + * array is undefined." + */ + uint32_t qidx = 0; + for (j = 0; j <= RTE_EVENT_DEV_PRIORITY_LOWEST; j++) { + for (i = 0; i < sw->qid_count; i++) { + if (sw->qids[i].priority == j) { + sw->qids_prioritized[qidx] = &sw->qids[i]; + qidx++; + } + } + } + + if (sw_xstats_init(sw) < 0) + return -EINVAL; + + rte_smp_wmb(); + sw->started = 1; + + return 0; +} + +static void +sw_stop(struct rte_eventdev *dev) +{ + struct sw_evdev *sw = sw_pmd_priv(dev); + sw_xstats_uninit(sw); + sw->started = 0; + rte_smp_wmb(); +} + +static int +sw_close(struct rte_eventdev *dev) +{ + struct sw_evdev *sw = sw_pmd_priv(dev); + uint32_t i; + + for (i = 0; i < sw->qid_count; i++) + sw_queue_release(dev, i); + sw->qid_count = 0; + + for (i = 0; i < sw->port_count; i++) + sw_port_release(&sw->ports[i]); + sw->port_count = 0; + + memset(&sw->stats, 0, sizeof(sw->stats)); + sw->sched_called = 0; + sw->sched_no_iq_enqueues = 0; + sw->sched_no_cq_enqueues = 0; + sw->sched_cq_qid_called = 0; + + return 0; +} + +static int +assign_numa_node(const char *key __rte_unused, const char *value, void *opaque) +{ + int *socket_id = opaque; + *socket_id = atoi(value); + if (*socket_id >= RTE_MAX_NUMA_NODES) + return -1; + return 0; +} + +static int +set_sched_quanta(const char *key __rte_unused, const char *value, void *opaque) +{ + int *quanta = opaque; + *quanta = atoi(value); + if (*quanta < 0 || *quanta >= 4096) + return -1; + return 0; +} + +static int +set_credit_quanta(const char *key __rte_unused, const char *value, void *opaque) +{ + int *credit = opaque; + *credit = atoi(value); + if (*credit < 0 || *credit >= 128) + return -1; + return 0; +} + + +static int32_t sw_sched_service_func(void *args) +{ + struct rte_eventdev *dev = args; + sw_event_schedule(dev); + return 0; +} + +static int +sw_probe(struct rte_vdev_device *vdev) +{ + static const struct rte_eventdev_ops evdev_sw_ops = { + .dev_configure = sw_dev_configure, + .dev_infos_get = sw_info_get, + .dev_close = sw_close, + .dev_start = sw_start, + .dev_stop = sw_stop, + .dump = sw_dump, + + .queue_def_conf = sw_queue_def_conf, + .queue_setup = sw_queue_setup, + .queue_release = sw_queue_release, + .port_def_conf = sw_port_def_conf, + .port_setup = sw_port_setup, + .port_release = sw_port_release, + .port_link = sw_port_link, + .port_unlink = sw_port_unlink, + + .eth_rx_adapter_caps_get = sw_eth_rx_adapter_caps_get, + + .xstats_get = sw_xstats_get, + .xstats_get_names = sw_xstats_get_names, + .xstats_get_by_name = sw_xstats_get_by_name, + .xstats_reset = sw_xstats_reset, + }; + + static const char *const args[] = { + NUMA_NODE_ARG, + SCHED_QUANTA_ARG, + CREDIT_QUANTA_ARG, + NULL + }; + const char *name; + const char *params; + struct rte_eventdev *dev; + struct sw_evdev *sw; + int socket_id = rte_socket_id(); + int sched_quanta = SW_DEFAULT_SCHED_QUANTA; + int credit_quanta = SW_DEFAULT_CREDIT_QUANTA; + + name = rte_vdev_device_name(vdev); + params = rte_vdev_device_args(vdev); + if (params != NULL && params[0] != '\0') { + struct rte_kvargs *kvlist = rte_kvargs_parse(params, args); + + if (!kvlist) { + SW_LOG_INFO( + "Ignoring unsupported parameters when creating device '%s'\n", + name); + } else { + int ret = rte_kvargs_process(kvlist, NUMA_NODE_ARG, + assign_numa_node, &socket_id); + if (ret != 0) { + SW_LOG_ERR( + "%s: Error parsing numa node parameter", + name); + rte_kvargs_free(kvlist); + return ret; + } + + ret = rte_kvargs_process(kvlist, SCHED_QUANTA_ARG, + set_sched_quanta, &sched_quanta); + if (ret != 0) { + SW_LOG_ERR( + "%s: Error parsing sched quanta parameter", + name); + rte_kvargs_free(kvlist); + return ret; + } + + ret = rte_kvargs_process(kvlist, CREDIT_QUANTA_ARG, + set_credit_quanta, &credit_quanta); + if (ret != 0) { + SW_LOG_ERR( + "%s: Error parsing credit quanta parameter", + name); + rte_kvargs_free(kvlist); + return ret; + } + + rte_kvargs_free(kvlist); + } + } + + SW_LOG_INFO( + "Creating eventdev sw device %s, numa_node=%d, sched_quanta=%d, credit_quanta=%d\n", + name, socket_id, sched_quanta, credit_quanta); + + dev = rte_event_pmd_vdev_init(name, + sizeof(struct sw_evdev), socket_id); + if (dev == NULL) { + SW_LOG_ERR("eventdev vdev init() failed"); + return -EFAULT; + } + dev->dev_ops = &evdev_sw_ops; + dev->enqueue = sw_event_enqueue; + dev->enqueue_burst = sw_event_enqueue_burst; + dev->enqueue_new_burst = sw_event_enqueue_burst; + dev->enqueue_forward_burst = sw_event_enqueue_burst; + dev->dequeue = sw_event_dequeue; + dev->dequeue_burst = sw_event_dequeue_burst; + + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return 0; + + sw = dev->data->dev_private; + sw->data = dev->data; + + /* copy values passed from vdev command line to instance */ + sw->credit_update_quanta = credit_quanta; + sw->sched_quanta = sched_quanta; + + /* register service with EAL */ + struct rte_service_spec service; + memset(&service, 0, sizeof(struct rte_service_spec)); + snprintf(service.name, sizeof(service.name), "%s_service", name); + snprintf(sw->service_name, sizeof(sw->service_name), "%s_service", + name); + service.socket_id = socket_id; + service.callback = sw_sched_service_func; + service.callback_userdata = (void *)dev; + + int32_t ret = rte_service_component_register(&service, &sw->service_id); + if (ret) { + SW_LOG_ERR("service register() failed"); + return -ENOEXEC; + } + + dev->data->service_inited = 1; + dev->data->service_id = sw->service_id; + + return 0; +} + +static int +sw_remove(struct rte_vdev_device *vdev) +{ + const char *name; + + name = rte_vdev_device_name(vdev); + if (name == NULL) + return -EINVAL; + + SW_LOG_INFO("Closing eventdev sw device %s\n", name); + + return rte_event_pmd_vdev_uninit(name); +} + +static struct rte_vdev_driver evdev_sw_pmd_drv = { + .probe = sw_probe, + .remove = sw_remove +}; + +RTE_PMD_REGISTER_VDEV(EVENTDEV_NAME_SW_PMD, evdev_sw_pmd_drv); +RTE_PMD_REGISTER_PARAM_STRING(event_sw, NUMA_NODE_ARG "= " + SCHED_QUANTA_ARG "=" CREDIT_QUANTA_ARG "="); diff --git a/dpdk/drivers/event/sw/sw_evdev.h b/dpdk/drivers/event/sw/sw_evdev.h new file mode 100644 index 00000000..ee0edb32 --- /dev/null +++ b/dpdk/drivers/event/sw/sw_evdev.h @@ -0,0 +1,323 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _SW_EVDEV_H_ +#define _SW_EVDEV_H_ + +#include +#include +#include + +#define SW_DEFAULT_CREDIT_QUANTA 32 +#define SW_DEFAULT_SCHED_QUANTA 128 +#define SW_QID_NUM_FIDS 16384 +#define SW_IQS_MAX 4 +#define SW_Q_PRIORITY_MAX 255 +#define SW_PORTS_MAX 64 +#define MAX_SW_CONS_Q_DEPTH 128 +#define SW_INFLIGHT_EVENTS_TOTAL 4096 +/* allow for lots of over-provisioning */ +#define MAX_SW_PROD_Q_DEPTH 4096 +#define SW_FRAGMENTS_MAX 16 + +/* report dequeue burst sizes in buckets */ +#define SW_DEQ_STAT_BUCKET_SHIFT 2 +/* how many packets pulled from port by sched */ +#define SCHED_DEQUEUE_BURST_SIZE 32 + +#define SW_PORT_HIST_LIST (MAX_SW_PROD_Q_DEPTH) /* size of our history list */ +#define NUM_SAMPLES 64 /* how many data points use for average stats */ + +#define EVENTDEV_NAME_SW_PMD event_sw +#define SW_PMD_NAME RTE_STR(event_sw) +#define SW_PMD_NAME_MAX 64 + +#define SW_SCHED_TYPE_DIRECT (RTE_SCHED_TYPE_PARALLEL + 1) + +#define SW_NUM_POLL_BUCKETS (MAX_SW_CONS_Q_DEPTH >> SW_DEQ_STAT_BUCKET_SHIFT) + +enum { + QE_FLAG_VALID_SHIFT = 0, + QE_FLAG_COMPLETE_SHIFT, + QE_FLAG_NOT_EOP_SHIFT, + _QE_FLAG_COUNT +}; + +#define QE_FLAG_VALID (1 << QE_FLAG_VALID_SHIFT) /* for NEW FWD, FRAG */ +#define QE_FLAG_COMPLETE (1 << QE_FLAG_COMPLETE_SHIFT) /* set for FWD, DROP */ +#define QE_FLAG_NOT_EOP (1 << QE_FLAG_NOT_EOP_SHIFT) /* set for FRAG only */ + +static const uint8_t sw_qe_flag_map[] = { + QE_FLAG_VALID /* NEW Event */, + QE_FLAG_VALID | QE_FLAG_COMPLETE /* FWD Event */, + QE_FLAG_COMPLETE /* RELEASE Event */, + + /* Values which can be used for future support for partial + * events, i.e. where one event comes back to the scheduler + * as multiple which need to be tracked together + */ + QE_FLAG_VALID | QE_FLAG_COMPLETE | QE_FLAG_NOT_EOP, +}; + +#ifdef RTE_LIBRTE_PMD_SW_EVENTDEV_DEBUG +#define SW_LOG_INFO(fmt, args...) \ + RTE_LOG(INFO, EVENTDEV, "[%s] %s() line %u: " fmt "\n", \ + SW_PMD_NAME, \ + __func__, __LINE__, ## args) + +#define SW_LOG_DBG(fmt, args...) \ + RTE_LOG(DEBUG, EVENTDEV, "[%s] %s() line %u: " fmt "\n", \ + SW_PMD_NAME, \ + __func__, __LINE__, ## args) +#else +#define SW_LOG_INFO(fmt, args...) +#define SW_LOG_DBG(fmt, args...) +#endif + +#define SW_LOG_ERR(fmt, args...) \ + RTE_LOG(ERR, EVENTDEV, "[%s] %s() line %u: " fmt "\n", \ + SW_PMD_NAME, \ + __func__, __LINE__, ## args) + +/* Records basic event stats at a given point. Used in port and qid structs */ +struct sw_point_stats { + uint64_t rx_pkts; + uint64_t rx_dropped; + uint64_t tx_pkts; +}; + +/* structure used to track what port a flow (FID) is pinned to */ +struct sw_fid_t { + /* which CQ this FID is currently pinned to */ + int32_t cq; + /* number of packets gone to the CQ with this FID */ + uint32_t pcount; +}; + +struct reorder_buffer_entry { + uint16_t num_fragments; /**< Number of packet fragments */ + uint16_t fragment_index; /**< Points to the oldest valid frag */ + uint8_t ready; /**< Entry is ready to be reordered */ + struct rte_event fragments[SW_FRAGMENTS_MAX]; +}; + +struct sw_qid { + /* set when the QID has been initialized */ + uint8_t initialized; + /* The type of this QID */ + int8_t type; + /* Integer ID representing the queue. This is used in history lists, + * to identify the stage of processing. + */ + uint32_t id; + struct sw_point_stats stats; + + /* Internal priority rings for packets */ + struct iq_ring *iq[SW_IQS_MAX]; + uint32_t iq_pkt_mask; /* A mask to indicate packets in an IQ */ + uint64_t iq_pkt_count[SW_IQS_MAX]; + + /* Information on what CQs are polling this IQ */ + uint32_t cq_num_mapped_cqs; + uint32_t cq_next_tx; /* cq to write next (non-atomic) packet */ + uint32_t cq_map[SW_PORTS_MAX]; + uint64_t to_port[SW_PORTS_MAX]; + + /* Track flow ids for atomic load balancing */ + struct sw_fid_t fids[SW_QID_NUM_FIDS]; + + /* Track packet order for reordering when needed */ + struct reorder_buffer_entry *reorder_buffer; /*< pkts await reorder */ + struct rte_ring *reorder_buffer_freelist; /* available reorder slots */ + uint32_t reorder_buffer_index; /* oldest valid reorder buffer entry */ + uint32_t window_size; /* Used to wrap reorder_buffer_index */ + + uint8_t priority; +}; + +struct sw_hist_list_entry { + int32_t qid; + int32_t fid; + struct reorder_buffer_entry *rob_entry; +}; + +struct sw_evdev; + +struct sw_port { + /* new enqueue / dequeue API doesn't have an instance pointer, only the + * pointer to the port being enqueue/dequeued from + */ + struct sw_evdev *sw; + + /* set when the port is initialized */ + uint8_t initialized; + /* A numeric ID for the port */ + uint8_t id; + + int16_t is_directed; /** Takes from a single directed QID */ + /** + * For loadbalanced we can optimise pulling packets from + * producers if there is no reordering involved + */ + int16_t num_ordered_qids; + + /** Ring and buffer for pulling events from workers for scheduling */ + struct rte_event_ring *rx_worker_ring __rte_cache_aligned; + /** Ring and buffer for pushing packets to workers after scheduling */ + struct rte_event_ring *cq_worker_ring; + + /* hole */ + + /* num releases yet to be completed on this port */ + uint16_t outstanding_releases __rte_cache_aligned; + uint16_t inflight_max; /* app requested max inflights for this port */ + uint16_t inflight_credits; /* num credits this port has right now */ + + uint16_t last_dequeue_burst_sz; /* how big the burst was */ + uint64_t last_dequeue_ticks; /* used to track burst processing time */ + uint64_t avg_pkt_ticks; /* tracks average over NUM_SAMPLES burst */ + uint64_t total_polls; /* how many polls were counted in stats */ + uint64_t zero_polls; /* tracks polls returning nothing */ + uint32_t poll_buckets[SW_NUM_POLL_BUCKETS]; + /* bucket values in 4s for shorter reporting */ + + /* History list structs, containing info on pkts egressed to worker */ + uint16_t hist_head __rte_cache_aligned; + uint16_t hist_tail; + uint16_t inflights; + struct sw_hist_list_entry hist_list[SW_PORT_HIST_LIST]; + + /* track packets in and out of this port */ + struct sw_point_stats stats; + + + uint32_t pp_buf_start; + uint32_t pp_buf_count; + uint16_t cq_buf_count; + struct rte_event pp_buf[SCHED_DEQUEUE_BURST_SIZE]; + struct rte_event cq_buf[MAX_SW_CONS_Q_DEPTH]; + + uint8_t num_qids_mapped; +}; + +struct sw_evdev { + struct rte_eventdev_data *data; + + uint32_t port_count; + uint32_t qid_count; + uint32_t xstats_count; + struct sw_xstats_entry *xstats; + uint32_t xstats_count_mode_dev; + uint32_t xstats_count_mode_port; + uint32_t xstats_count_mode_queue; + + /* Contains all ports - load balanced and directed */ + struct sw_port ports[SW_PORTS_MAX] __rte_cache_aligned; + + rte_atomic32_t inflights __rte_cache_aligned; + + /* + * max events in this instance. Cached here for performance. + * (also available in data->conf.nb_events_limit) + */ + uint32_t nb_events_limit; + + /* Internal queues - one per logical queue */ + struct sw_qid qids[RTE_EVENT_MAX_QUEUES_PER_DEV] __rte_cache_aligned; + + /* Cache how many packets are in each cq */ + uint16_t cq_ring_space[SW_PORTS_MAX] __rte_cache_aligned; + + /* Array of pointers to load-balanced QIDs sorted by priority level */ + struct sw_qid *qids_prioritized[RTE_EVENT_MAX_QUEUES_PER_DEV]; + + /* Stats */ + struct sw_point_stats stats __rte_cache_aligned; + uint64_t sched_called; + int32_t sched_quanta; + uint64_t sched_no_iq_enqueues; + uint64_t sched_no_cq_enqueues; + uint64_t sched_cq_qid_called; + + uint8_t started; + uint32_t credit_update_quanta; + + /* store num stats and offset of the stats for each port */ + uint16_t xstats_count_per_port[SW_PORTS_MAX]; + uint16_t xstats_offset_for_port[SW_PORTS_MAX]; + /* store num stats and offset of the stats for each queue */ + uint16_t xstats_count_per_qid[RTE_EVENT_MAX_QUEUES_PER_DEV]; + uint16_t xstats_offset_for_qid[RTE_EVENT_MAX_QUEUES_PER_DEV]; + + uint32_t service_id; + char service_name[SW_PMD_NAME_MAX]; +}; + +static inline struct sw_evdev * +sw_pmd_priv(const struct rte_eventdev *eventdev) +{ + return eventdev->data->dev_private; +} + +static inline const struct sw_evdev * +sw_pmd_priv_const(const struct rte_eventdev *eventdev) +{ + return eventdev->data->dev_private; +} + +uint16_t sw_event_enqueue(void *port, const struct rte_event *ev); +uint16_t sw_event_enqueue_burst(void *port, const struct rte_event ev[], + uint16_t num); + +uint16_t sw_event_dequeue(void *port, struct rte_event *ev, uint64_t wait); +uint16_t sw_event_dequeue_burst(void *port, struct rte_event *ev, uint16_t num, + uint64_t wait); +void sw_event_schedule(struct rte_eventdev *dev); +int sw_xstats_init(struct sw_evdev *dev); +int sw_xstats_uninit(struct sw_evdev *dev); +int sw_xstats_get_names(const struct rte_eventdev *dev, + enum rte_event_dev_xstats_mode mode, uint8_t queue_port_id, + struct rte_event_dev_xstats_name *xstats_names, + unsigned int *ids, unsigned int size); +int sw_xstats_get(const struct rte_eventdev *dev, + enum rte_event_dev_xstats_mode mode, uint8_t queue_port_id, + const unsigned int ids[], uint64_t values[], unsigned int n); +uint64_t sw_xstats_get_by_name(const struct rte_eventdev *dev, + const char *name, unsigned int *id); +int sw_xstats_reset(struct rte_eventdev *dev, + enum rte_event_dev_xstats_mode mode, + int16_t queue_port_id, + const uint32_t ids[], + uint32_t nb_ids); + + +#endif /* _SW_EVDEV_H_ */ diff --git a/dpdk/drivers/event/sw/sw_evdev_scheduler.c b/dpdk/drivers/event/sw/sw_evdev_scheduler.c new file mode 100644 index 00000000..8a2c9d4f --- /dev/null +++ b/dpdk/drivers/event/sw/sw_evdev_scheduler.c @@ -0,0 +1,603 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include "sw_evdev.h" +#include "iq_ring.h" + +#define SW_IQS_MASK (SW_IQS_MAX-1) + +/* Retrieve the highest priority IQ or -1 if no pkts available. Doing the + * CLZ twice is faster than caching the value due to data dependencies + */ +#define PKT_MASK_TO_IQ(pkts) \ + (__builtin_ctz(pkts | (1 << SW_IQS_MAX))) + +#if SW_IQS_MAX != 4 +#error Misconfigured PRIO_TO_IQ caused by SW_IQS_MAX value change +#endif +#define PRIO_TO_IQ(prio) (prio >> 6) + +#define MAX_PER_IQ_DEQUEUE 48 +#define FLOWID_MASK (SW_QID_NUM_FIDS-1) +/* use cheap bit mixing, we only need to lose a few bits */ +#define SW_HASH_FLOWID(f) (((f) ^ (f >> 10)) & FLOWID_MASK) + +static inline uint32_t +sw_schedule_atomic_to_cq(struct sw_evdev *sw, struct sw_qid * const qid, + uint32_t iq_num, unsigned int count) +{ + struct rte_event qes[MAX_PER_IQ_DEQUEUE]; /* count <= MAX */ + struct rte_event blocked_qes[MAX_PER_IQ_DEQUEUE]; + uint32_t nb_blocked = 0; + uint32_t i; + + if (count > MAX_PER_IQ_DEQUEUE) + count = MAX_PER_IQ_DEQUEUE; + + /* This is the QID ID. The QID ID is static, hence it can be + * used to identify the stage of processing in history lists etc + */ + uint32_t qid_id = qid->id; + + iq_ring_dequeue_burst(qid->iq[iq_num], qes, count); + for (i = 0; i < count; i++) { + const struct rte_event *qe = &qes[i]; + const uint16_t flow_id = SW_HASH_FLOWID(qes[i].flow_id); + struct sw_fid_t *fid = &qid->fids[flow_id]; + int cq = fid->cq; + + if (cq < 0) { + uint32_t cq_idx = qid->cq_next_tx++; + if (qid->cq_next_tx == qid->cq_num_mapped_cqs) + qid->cq_next_tx = 0; + cq = qid->cq_map[cq_idx]; + + /* find least used */ + int cq_free_cnt = sw->cq_ring_space[cq]; + for (cq_idx = 0; cq_idx < qid->cq_num_mapped_cqs; + cq_idx++) { + int test_cq = qid->cq_map[cq_idx]; + int test_cq_free = sw->cq_ring_space[test_cq]; + if (test_cq_free > cq_free_cnt) { + cq = test_cq; + cq_free_cnt = test_cq_free; + } + } + + fid->cq = cq; /* this pins early */ + } + + if (sw->cq_ring_space[cq] == 0 || + sw->ports[cq].inflights == SW_PORT_HIST_LIST) { + blocked_qes[nb_blocked++] = *qe; + continue; + } + + struct sw_port *p = &sw->ports[cq]; + + /* at this point we can queue up the packet on the cq_buf */ + fid->pcount++; + p->cq_buf[p->cq_buf_count++] = *qe; + p->inflights++; + sw->cq_ring_space[cq]--; + + int head = (p->hist_head++ & (SW_PORT_HIST_LIST-1)); + p->hist_list[head].fid = flow_id; + p->hist_list[head].qid = qid_id; + + p->stats.tx_pkts++; + qid->stats.tx_pkts++; + qid->to_port[cq]++; + + /* if we just filled in the last slot, flush the buffer */ + if (sw->cq_ring_space[cq] == 0) { + struct rte_event_ring *worker = p->cq_worker_ring; + rte_event_ring_enqueue_burst(worker, p->cq_buf, + p->cq_buf_count, + &sw->cq_ring_space[cq]); + p->cq_buf_count = 0; + } + } + iq_ring_put_back(qid->iq[iq_num], blocked_qes, nb_blocked); + + return count - nb_blocked; +} + +static inline uint32_t +sw_schedule_parallel_to_cq(struct sw_evdev *sw, struct sw_qid * const qid, + uint32_t iq_num, unsigned int count, int keep_order) +{ + uint32_t i; + uint32_t cq_idx = qid->cq_next_tx; + + /* This is the QID ID. The QID ID is static, hence it can be + * used to identify the stage of processing in history lists etc + */ + uint32_t qid_id = qid->id; + + if (count > MAX_PER_IQ_DEQUEUE) + count = MAX_PER_IQ_DEQUEUE; + + if (keep_order) + /* only schedule as many as we have reorder buffer entries */ + count = RTE_MIN(count, + rte_ring_count(qid->reorder_buffer_freelist)); + + for (i = 0; i < count; i++) { + const struct rte_event *qe = iq_ring_peek(qid->iq[iq_num]); + uint32_t cq_check_count = 0; + uint32_t cq; + + /* + * for parallel, just send to next available CQ in round-robin + * fashion. So scan for an available CQ. If all CQs are full + * just return and move on to next QID + */ + do { + if (++cq_check_count > qid->cq_num_mapped_cqs) + goto exit; + cq = qid->cq_map[cq_idx]; + if (++cq_idx == qid->cq_num_mapped_cqs) + cq_idx = 0; + } while (rte_event_ring_free_count( + sw->ports[cq].cq_worker_ring) == 0 || + sw->ports[cq].inflights == SW_PORT_HIST_LIST); + + struct sw_port *p = &sw->ports[cq]; + if (sw->cq_ring_space[cq] == 0 || + p->inflights == SW_PORT_HIST_LIST) + break; + + sw->cq_ring_space[cq]--; + + qid->stats.tx_pkts++; + + const int head = (p->hist_head & (SW_PORT_HIST_LIST-1)); + p->hist_list[head].fid = SW_HASH_FLOWID(qe->flow_id); + p->hist_list[head].qid = qid_id; + + if (keep_order) + rte_ring_sc_dequeue(qid->reorder_buffer_freelist, + (void *)&p->hist_list[head].rob_entry); + + sw->ports[cq].cq_buf[sw->ports[cq].cq_buf_count++] = *qe; + iq_ring_pop(qid->iq[iq_num]); + + rte_compiler_barrier(); + p->inflights++; + p->stats.tx_pkts++; + p->hist_head++; + } +exit: + qid->cq_next_tx = cq_idx; + return i; +} + +static uint32_t +sw_schedule_dir_to_cq(struct sw_evdev *sw, struct sw_qid * const qid, + uint32_t iq_num, unsigned int count __rte_unused) +{ + uint32_t cq_id = qid->cq_map[0]; + struct sw_port *port = &sw->ports[cq_id]; + + /* get max burst enq size for cq_ring */ + uint32_t count_free = sw->cq_ring_space[cq_id]; + if (count_free == 0) + return 0; + + /* burst dequeue from the QID IQ ring */ + struct iq_ring *ring = qid->iq[iq_num]; + uint32_t ret = iq_ring_dequeue_burst(ring, + &port->cq_buf[port->cq_buf_count], count_free); + port->cq_buf_count += ret; + + /* Update QID, Port and Total TX stats */ + qid->stats.tx_pkts += ret; + port->stats.tx_pkts += ret; + + /* Subtract credits from cached value */ + sw->cq_ring_space[cq_id] -= ret; + + return ret; +} + +static uint32_t +sw_schedule_qid_to_cq(struct sw_evdev *sw) +{ + uint32_t pkts = 0; + uint32_t qid_idx; + + sw->sched_cq_qid_called++; + + for (qid_idx = 0; qid_idx < sw->qid_count; qid_idx++) { + struct sw_qid *qid = sw->qids_prioritized[qid_idx]; + + int type = qid->type; + int iq_num = PKT_MASK_TO_IQ(qid->iq_pkt_mask); + + /* zero mapped CQs indicates directed */ + if (iq_num >= SW_IQS_MAX) + continue; + + uint32_t pkts_done = 0; + uint32_t count = iq_ring_count(qid->iq[iq_num]); + + if (count > 0) { + if (type == SW_SCHED_TYPE_DIRECT) + pkts_done += sw_schedule_dir_to_cq(sw, qid, + iq_num, count); + else if (type == RTE_SCHED_TYPE_ATOMIC) + pkts_done += sw_schedule_atomic_to_cq(sw, qid, + iq_num, count); + else + pkts_done += sw_schedule_parallel_to_cq(sw, qid, + iq_num, count, + type == RTE_SCHED_TYPE_ORDERED); + } + + /* Check if the IQ that was polled is now empty, and unset it + * in the IQ mask if its empty. + */ + int all_done = (pkts_done == count); + + qid->iq_pkt_mask &= ~(all_done << (iq_num)); + pkts += pkts_done; + } + + return pkts; +} + +/* This function will perform re-ordering of packets, and injecting into + * the appropriate QID IQ. As LB and DIR QIDs are in the same array, but *NOT* + * contiguous in that array, this function accepts a "range" of QIDs to scan. + */ +static uint16_t +sw_schedule_reorder(struct sw_evdev *sw, int qid_start, int qid_end) +{ + /* Perform egress reordering */ + struct rte_event *qe; + uint32_t pkts_iter = 0; + + for (; qid_start < qid_end; qid_start++) { + struct sw_qid *qid = &sw->qids[qid_start]; + int i, num_entries_in_use; + + if (qid->type != RTE_SCHED_TYPE_ORDERED) + continue; + + num_entries_in_use = rte_ring_free_count( + qid->reorder_buffer_freelist); + + for (i = 0; i < num_entries_in_use; i++) { + struct reorder_buffer_entry *entry; + int j; + + entry = &qid->reorder_buffer[qid->reorder_buffer_index]; + + if (!entry->ready) + break; + + for (j = 0; j < entry->num_fragments; j++) { + uint16_t dest_qid; + uint16_t dest_iq; + + int idx = entry->fragment_index + j; + qe = &entry->fragments[idx]; + + dest_qid = qe->queue_id; + dest_iq = PRIO_TO_IQ(qe->priority); + + if (dest_qid >= sw->qid_count) { + sw->stats.rx_dropped++; + continue; + } + + struct sw_qid *dest_qid_ptr = + &sw->qids[dest_qid]; + const struct iq_ring *dest_iq_ptr = + dest_qid_ptr->iq[dest_iq]; + if (iq_ring_free_count(dest_iq_ptr) == 0) + break; + + pkts_iter++; + + struct sw_qid *q = &sw->qids[dest_qid]; + struct iq_ring *r = q->iq[dest_iq]; + + /* we checked for space above, so enqueue must + * succeed + */ + iq_ring_enqueue(r, qe); + q->iq_pkt_mask |= (1 << (dest_iq)); + q->iq_pkt_count[dest_iq]++; + q->stats.rx_pkts++; + } + + entry->ready = (j != entry->num_fragments); + entry->num_fragments -= j; + entry->fragment_index += j; + + if (!entry->ready) { + entry->fragment_index = 0; + + rte_ring_sp_enqueue( + qid->reorder_buffer_freelist, + entry); + + qid->reorder_buffer_index++; + qid->reorder_buffer_index %= qid->window_size; + } + } + } + return pkts_iter; +} + +static __rte_always_inline void +sw_refill_pp_buf(struct sw_evdev *sw, struct sw_port *port) +{ + RTE_SET_USED(sw); + struct rte_event_ring *worker = port->rx_worker_ring; + port->pp_buf_start = 0; + port->pp_buf_count = rte_event_ring_dequeue_burst(worker, port->pp_buf, + RTE_DIM(port->pp_buf), NULL); +} + +static __rte_always_inline uint32_t +__pull_port_lb(struct sw_evdev *sw, uint32_t port_id, int allow_reorder) +{ + static struct reorder_buffer_entry dummy_rob; + uint32_t pkts_iter = 0; + struct sw_port *port = &sw->ports[port_id]; + + /* If shadow ring has 0 pkts, pull from worker ring */ + if (port->pp_buf_count == 0) + sw_refill_pp_buf(sw, port); + + while (port->pp_buf_count) { + const struct rte_event *qe = &port->pp_buf[port->pp_buf_start]; + struct sw_hist_list_entry *hist_entry = NULL; + uint8_t flags = qe->op; + const uint16_t eop = !(flags & QE_FLAG_NOT_EOP); + int needs_reorder = 0; + /* if no-reordering, having PARTIAL == NEW */ + if (!allow_reorder && !eop) + flags = QE_FLAG_VALID; + + /* + * if we don't have space for this packet in an IQ, + * then move on to next queue. Technically, for a + * packet that needs reordering, we don't need to check + * here, but it simplifies things not to special-case + */ + uint32_t iq_num = PRIO_TO_IQ(qe->priority); + struct sw_qid *qid = &sw->qids[qe->queue_id]; + + if ((flags & QE_FLAG_VALID) && + iq_ring_free_count(qid->iq[iq_num]) == 0) + break; + + /* now process based on flags. Note that for directed + * queues, the enqueue_flush masks off all but the + * valid flag. This makes FWD and PARTIAL enqueues just + * NEW type, and makes DROPS no-op calls. + */ + if ((flags & QE_FLAG_COMPLETE) && port->inflights > 0) { + const uint32_t hist_tail = port->hist_tail & + (SW_PORT_HIST_LIST - 1); + + hist_entry = &port->hist_list[hist_tail]; + const uint32_t hist_qid = hist_entry->qid; + const uint32_t hist_fid = hist_entry->fid; + + struct sw_fid_t *fid = + &sw->qids[hist_qid].fids[hist_fid]; + fid->pcount -= eop; + if (fid->pcount == 0) + fid->cq = -1; + + if (allow_reorder) { + /* set reorder ready if an ordered QID */ + uintptr_t rob_ptr = + (uintptr_t)hist_entry->rob_entry; + const uintptr_t valid = (rob_ptr != 0); + needs_reorder = valid; + rob_ptr |= + ((valid - 1) & (uintptr_t)&dummy_rob); + struct reorder_buffer_entry *tmp_rob_ptr = + (struct reorder_buffer_entry *)rob_ptr; + tmp_rob_ptr->ready = eop * needs_reorder; + } + + port->inflights -= eop; + port->hist_tail += eop; + } + if (flags & QE_FLAG_VALID) { + port->stats.rx_pkts++; + + if (allow_reorder && needs_reorder) { + struct reorder_buffer_entry *rob_entry = + hist_entry->rob_entry; + + hist_entry->rob_entry = NULL; + /* Although fragmentation not currently + * supported by eventdev API, we support it + * here. Open: How do we alert the user that + * they've exceeded max frags? + */ + int num_frag = rob_entry->num_fragments; + if (num_frag == SW_FRAGMENTS_MAX) + sw->stats.rx_dropped++; + else { + int idx = rob_entry->num_fragments++; + rob_entry->fragments[idx] = *qe; + } + goto end_qe; + } + + /* Use the iq_num from above to push the QE + * into the qid at the right priority + */ + + qid->iq_pkt_mask |= (1 << (iq_num)); + iq_ring_enqueue(qid->iq[iq_num], qe); + qid->iq_pkt_count[iq_num]++; + qid->stats.rx_pkts++; + pkts_iter++; + } + +end_qe: + port->pp_buf_start++; + port->pp_buf_count--; + } /* while (avail_qes) */ + + return pkts_iter; +} + +static uint32_t +sw_schedule_pull_port_lb(struct sw_evdev *sw, uint32_t port_id) +{ + return __pull_port_lb(sw, port_id, 1); +} + +static uint32_t +sw_schedule_pull_port_no_reorder(struct sw_evdev *sw, uint32_t port_id) +{ + return __pull_port_lb(sw, port_id, 0); +} + +static uint32_t +sw_schedule_pull_port_dir(struct sw_evdev *sw, uint32_t port_id) +{ + uint32_t pkts_iter = 0; + struct sw_port *port = &sw->ports[port_id]; + + /* If shadow ring has 0 pkts, pull from worker ring */ + if (port->pp_buf_count == 0) + sw_refill_pp_buf(sw, port); + + while (port->pp_buf_count) { + const struct rte_event *qe = &port->pp_buf[port->pp_buf_start]; + uint8_t flags = qe->op; + + if ((flags & QE_FLAG_VALID) == 0) + goto end_qe; + + uint32_t iq_num = PRIO_TO_IQ(qe->priority); + struct sw_qid *qid = &sw->qids[qe->queue_id]; + struct iq_ring *iq_ring = qid->iq[iq_num]; + + if (iq_ring_free_count(iq_ring) == 0) + break; /* move to next port */ + + port->stats.rx_pkts++; + + /* Use the iq_num from above to push the QE + * into the qid at the right priority + */ + qid->iq_pkt_mask |= (1 << (iq_num)); + iq_ring_enqueue(iq_ring, qe); + qid->iq_pkt_count[iq_num]++; + qid->stats.rx_pkts++; + pkts_iter++; + +end_qe: + port->pp_buf_start++; + port->pp_buf_count--; + } /* while port->pp_buf_count */ + + return pkts_iter; +} + +void +sw_event_schedule(struct rte_eventdev *dev) +{ + struct sw_evdev *sw = sw_pmd_priv(dev); + uint32_t in_pkts, out_pkts; + uint32_t out_pkts_total = 0, in_pkts_total = 0; + int32_t sched_quanta = sw->sched_quanta; + uint32_t i; + + sw->sched_called++; + if (!sw->started) + return; + + do { + uint32_t in_pkts_this_iteration = 0; + + /* Pull from rx_ring for ports */ + do { + in_pkts = 0; + for (i = 0; i < sw->port_count; i++) + if (sw->ports[i].is_directed) + in_pkts += sw_schedule_pull_port_dir(sw, i); + else if (sw->ports[i].num_ordered_qids > 0) + in_pkts += sw_schedule_pull_port_lb(sw, i); + else + in_pkts += sw_schedule_pull_port_no_reorder(sw, i); + + /* QID scan for re-ordered */ + in_pkts += sw_schedule_reorder(sw, 0, + sw->qid_count); + in_pkts_this_iteration += in_pkts; + } while (in_pkts > 4 && + (int)in_pkts_this_iteration < sched_quanta); + + out_pkts = 0; + out_pkts += sw_schedule_qid_to_cq(sw); + out_pkts_total += out_pkts; + in_pkts_total += in_pkts_this_iteration; + + if (in_pkts == 0 && out_pkts == 0) + break; + } while ((int)out_pkts_total < sched_quanta); + + /* push all the internal buffered QEs in port->cq_ring to the + * worker cores: aka, do the ring transfers batched. + */ + for (i = 0; i < sw->port_count; i++) { + struct rte_event_ring *worker = sw->ports[i].cq_worker_ring; + rte_event_ring_enqueue_burst(worker, sw->ports[i].cq_buf, + sw->ports[i].cq_buf_count, + &sw->cq_ring_space[i]); + sw->ports[i].cq_buf_count = 0; + } + + sw->stats.tx_pkts += out_pkts_total; + sw->stats.rx_pkts += in_pkts_total; + + sw->sched_no_iq_enqueues += (in_pkts_total == 0); + sw->sched_no_cq_enqueues += (out_pkts_total == 0); + +} diff --git a/dpdk/drivers/event/sw/sw_evdev_worker.c b/dpdk/drivers/event/sw/sw_evdev_worker.c new file mode 100644 index 00000000..b3b3b17e --- /dev/null +++ b/dpdk/drivers/event/sw/sw_evdev_worker.c @@ -0,0 +1,212 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include + +#include "sw_evdev.h" + +#define PORT_ENQUEUE_MAX_BURST_SIZE 64 + +static inline void +sw_event_release(struct sw_port *p, uint8_t index) +{ + /* + * Drops the next outstanding event in our history. Used on dequeue + * to clear any history before dequeuing more events. + */ + RTE_SET_USED(index); + + /* create drop message */ + struct rte_event ev; + ev.op = sw_qe_flag_map[RTE_EVENT_OP_RELEASE]; + + uint16_t free_count; + rte_event_ring_enqueue_burst(p->rx_worker_ring, &ev, 1, &free_count); + + /* each release returns one credit */ + p->outstanding_releases--; + p->inflight_credits++; +} + +/* + * special-case of rte_event_ring enqueue, with overriding the ops member on + * the events that get written to the ring. + */ +static inline unsigned int +enqueue_burst_with_ops(struct rte_event_ring *r, const struct rte_event *events, + unsigned int n, uint8_t *ops) +{ + struct rte_event tmp_evs[PORT_ENQUEUE_MAX_BURST_SIZE]; + unsigned int i; + + memcpy(tmp_evs, events, n * sizeof(events[0])); + for (i = 0; i < n; i++) + tmp_evs[i].op = ops[i]; + + return rte_event_ring_enqueue_burst(r, tmp_evs, n, NULL); +} + +uint16_t +sw_event_enqueue_burst(void *port, const struct rte_event ev[], uint16_t num) +{ + int32_t i; + uint8_t new_ops[PORT_ENQUEUE_MAX_BURST_SIZE]; + struct sw_port *p = port; + struct sw_evdev *sw = (void *)p->sw; + uint32_t sw_inflights = rte_atomic32_read(&sw->inflights); + int new = 0; + + if (num > PORT_ENQUEUE_MAX_BURST_SIZE) + num = PORT_ENQUEUE_MAX_BURST_SIZE; + + for (i = 0; i < num; i++) + new += (ev[i].op == RTE_EVENT_OP_NEW); + + if (unlikely(new > 0 && p->inflight_max < sw_inflights)) + return 0; + + if (p->inflight_credits < new) { + /* check if event enqueue brings port over max threshold */ + uint32_t credit_update_quanta = sw->credit_update_quanta; + if (sw_inflights + credit_update_quanta > sw->nb_events_limit) + return 0; + + rte_atomic32_add(&sw->inflights, credit_update_quanta); + p->inflight_credits += (credit_update_quanta); + + if (p->inflight_credits < new) + return 0; + } + + uint32_t forwards = 0; + for (i = 0; i < num; i++) { + int op = ev[i].op; + int outstanding = p->outstanding_releases > 0; + const uint8_t invalid_qid = (ev[i].queue_id >= sw->qid_count); + + p->inflight_credits -= (op == RTE_EVENT_OP_NEW); + p->inflight_credits += (op == RTE_EVENT_OP_RELEASE) * + outstanding; + forwards += (op == RTE_EVENT_OP_FORWARD); + + new_ops[i] = sw_qe_flag_map[op]; + new_ops[i] &= ~(invalid_qid << QE_FLAG_VALID_SHIFT); + + /* FWD and RELEASE packets will both resolve to taken (assuming + * correct usage of the API), providing very high correct + * prediction rate. + */ + if ((new_ops[i] & QE_FLAG_COMPLETE) && outstanding) + p->outstanding_releases--; + + /* error case: branch to avoid touching p->stats */ + if (unlikely(invalid_qid)) { + p->stats.rx_dropped++; + p->inflight_credits++; + } + } + + /* handle directed port forward credits */ + p->inflight_credits -= forwards * p->is_directed; + + /* returns number of events actually enqueued */ + uint32_t enq = enqueue_burst_with_ops(p->rx_worker_ring, ev, i, + new_ops); + if (p->outstanding_releases == 0 && p->last_dequeue_burst_sz != 0) { + uint64_t burst_ticks = rte_get_timer_cycles() - + p->last_dequeue_ticks; + uint64_t burst_pkt_ticks = + burst_ticks / p->last_dequeue_burst_sz; + p->avg_pkt_ticks -= p->avg_pkt_ticks / NUM_SAMPLES; + p->avg_pkt_ticks += burst_pkt_ticks / NUM_SAMPLES; + p->last_dequeue_ticks = 0; + } + return enq; +} + +uint16_t +sw_event_enqueue(void *port, const struct rte_event *ev) +{ + return sw_event_enqueue_burst(port, ev, 1); +} + +uint16_t +sw_event_dequeue_burst(void *port, struct rte_event *ev, uint16_t num, + uint64_t wait) +{ + RTE_SET_USED(wait); + struct sw_port *p = (void *)port; + struct sw_evdev *sw = (void *)p->sw; + struct rte_event_ring *ring = p->cq_worker_ring; + uint32_t credit_update_quanta = sw->credit_update_quanta; + + /* check that all previous dequeues have been released */ + if (!p->is_directed) { + uint16_t out_rels = p->outstanding_releases; + uint16_t i; + for (i = 0; i < out_rels; i++) + sw_event_release(p, i); + } + + /* returns number of events actually dequeued */ + uint16_t ndeq = rte_event_ring_dequeue_burst(ring, ev, num, NULL); + if (unlikely(ndeq == 0)) { + p->outstanding_releases = 0; + p->zero_polls++; + p->total_polls++; + goto end; + } + + /* only add credits for directed ports - LB ports send RELEASEs */ + p->inflight_credits += ndeq * p->is_directed; + p->outstanding_releases = ndeq; + p->last_dequeue_burst_sz = ndeq; + p->last_dequeue_ticks = rte_get_timer_cycles(); + p->poll_buckets[(ndeq - 1) >> SW_DEQ_STAT_BUCKET_SHIFT]++; + p->total_polls++; + +end: + if (p->inflight_credits >= credit_update_quanta * 2 && + p->inflight_credits > credit_update_quanta + ndeq) { + rte_atomic32_sub(&sw->inflights, credit_update_quanta); + p->inflight_credits -= credit_update_quanta; + } + return ndeq; +} + +uint16_t +sw_event_dequeue(void *port, struct rte_event *ev, uint64_t wait) +{ + return sw_event_dequeue_burst(port, ev, 1, wait); +} diff --git a/dpdk/drivers/event/sw/sw_evdev_xstats.c b/dpdk/drivers/event/sw/sw_evdev_xstats.c new file mode 100644 index 00000000..61a5c33b --- /dev/null +++ b/dpdk/drivers/event/sw/sw_evdev_xstats.c @@ -0,0 +1,682 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include "sw_evdev.h" +#include "iq_ring.h" + +enum xstats_type { + /* common stats */ + rx, + tx, + dropped, + inflight, + calls, + credits, + /* device instance specific */ + no_iq_enq, + no_cq_enq, + /* port_specific */ + rx_used, + rx_free, + tx_used, + tx_free, + pkt_cycles, + poll_return, /* for zero-count and used also for port bucket loop */ + /* qid_specific */ + iq_size, + iq_used, + /* qid port mapping specific */ + pinned, + pkts, /* note: qid-to-port pkts */ +}; + +typedef uint64_t (*xstats_fn)(const struct sw_evdev *dev, + uint16_t obj_idx, /* port or queue id */ + enum xstats_type stat, int extra_arg); + +struct sw_xstats_entry { + struct rte_event_dev_xstats_name name; + xstats_fn fn; + uint16_t obj_idx; + enum xstats_type stat; + enum rte_event_dev_xstats_mode mode; + int extra_arg; + uint8_t reset_allowed; /* when set, this value can be reset */ + uint64_t reset_value; /* an offset to be taken away to emulate resets */ +}; + +static uint64_t +get_dev_stat(const struct sw_evdev *sw, uint16_t obj_idx __rte_unused, + enum xstats_type type, int extra_arg __rte_unused) +{ + switch (type) { + case rx: return sw->stats.rx_pkts; + case tx: return sw->stats.tx_pkts; + case dropped: return sw->stats.rx_dropped; + case calls: return sw->sched_called; + case no_iq_enq: return sw->sched_no_iq_enqueues; + case no_cq_enq: return sw->sched_no_cq_enqueues; + default: return -1; + } +} + +static uint64_t +get_port_stat(const struct sw_evdev *sw, uint16_t obj_idx, + enum xstats_type type, int extra_arg __rte_unused) +{ + const struct sw_port *p = &sw->ports[obj_idx]; + + switch (type) { + case rx: return p->stats.rx_pkts; + case tx: return p->stats.tx_pkts; + case dropped: return p->stats.rx_dropped; + case inflight: return p->inflights; + case pkt_cycles: return p->avg_pkt_ticks; + case calls: return p->total_polls; + case credits: return p->inflight_credits; + case poll_return: return p->zero_polls; + case rx_used: return rte_event_ring_count(p->rx_worker_ring); + case rx_free: return rte_event_ring_free_count(p->rx_worker_ring); + case tx_used: return rte_event_ring_count(p->cq_worker_ring); + case tx_free: return rte_event_ring_free_count(p->cq_worker_ring); + default: return -1; + } +} + +static uint64_t +get_port_bucket_stat(const struct sw_evdev *sw, uint16_t obj_idx, + enum xstats_type type, int extra_arg) +{ + const struct sw_port *p = &sw->ports[obj_idx]; + + switch (type) { + case poll_return: return p->poll_buckets[extra_arg]; + default: return -1; + } +} + +static uint64_t +get_qid_stat(const struct sw_evdev *sw, uint16_t obj_idx, + enum xstats_type type, int extra_arg __rte_unused) +{ + const struct sw_qid *qid = &sw->qids[obj_idx]; + + switch (type) { + case rx: return qid->stats.rx_pkts; + case tx: return qid->stats.tx_pkts; + case dropped: return qid->stats.rx_dropped; + case inflight: + do { + uint64_t infl = 0; + unsigned int i; + for (i = 0; i < RTE_DIM(qid->fids); i++) + infl += qid->fids[i].pcount; + return infl; + } while (0); + break; + case iq_size: return RTE_DIM(qid->iq[0]->ring); + default: return -1; + } +} + +static uint64_t +get_qid_iq_stat(const struct sw_evdev *sw, uint16_t obj_idx, + enum xstats_type type, int extra_arg) +{ + const struct sw_qid *qid = &sw->qids[obj_idx]; + const int iq_idx = extra_arg; + + switch (type) { + case iq_used: return iq_ring_count(qid->iq[iq_idx]); + default: return -1; + } +} + +static uint64_t +get_qid_port_stat(const struct sw_evdev *sw, uint16_t obj_idx, + enum xstats_type type, int extra_arg) +{ + const struct sw_qid *qid = &sw->qids[obj_idx]; + uint16_t port = extra_arg; + + switch (type) { + case pinned: + do { + uint64_t pin = 0; + unsigned int i; + for (i = 0; i < RTE_DIM(qid->fids); i++) + if (qid->fids[i].cq == port) + pin++; + return pin; + } while (0); + break; + case pkts: + return qid->to_port[port]; + default: return -1; + } +} + +int +sw_xstats_init(struct sw_evdev *sw) +{ + /* + * define the stats names and types. Used to build up the device + * xstats array + * There are multiple set of stats: + * - device-level, + * - per-port, + * - per-port-dequeue-burst-sizes + * - per-qid, + * - per-iq + * - per-port-per-qid + * + * For each of these sets, we have three parallel arrays, one for the + * names, the other for the stat type parameter to be passed in the fn + * call to get that stat. The third array allows resetting or not. + * All these arrays must be kept in sync + */ + static const char * const dev_stats[] = { "rx", "tx", "drop", + "sched_calls", "sched_no_iq_enq", "sched_no_cq_enq", + }; + static const enum xstats_type dev_types[] = { rx, tx, dropped, + calls, no_iq_enq, no_cq_enq, + }; + /* all device stats are allowed to be reset */ + + static const char * const port_stats[] = {"rx", "tx", "drop", + "inflight", "avg_pkt_cycles", "credits", + "rx_ring_used", "rx_ring_free", + "cq_ring_used", "cq_ring_free", + "dequeue_calls", "dequeues_returning_0", + }; + static const enum xstats_type port_types[] = { rx, tx, dropped, + inflight, pkt_cycles, credits, + rx_used, rx_free, tx_used, tx_free, + calls, poll_return, + }; + static const uint8_t port_reset_allowed[] = {1, 1, 1, + 0, 1, 0, + 0, 0, 0, 0, + 1, 1, + }; + + static const char * const port_bucket_stats[] = { + "dequeues_returning" }; + static const enum xstats_type port_bucket_types[] = { poll_return }; + /* all bucket dequeues are allowed to be reset, handled in loop below */ + + static const char * const qid_stats[] = {"rx", "tx", "drop", + "inflight", "iq_size" + }; + static const enum xstats_type qid_types[] = { rx, tx, dropped, + inflight, iq_size + }; + static const uint8_t qid_reset_allowed[] = {1, 1, 1, + 0, 0 + }; + + static const char * const qid_iq_stats[] = { "used" }; + static const enum xstats_type qid_iq_types[] = { iq_used }; + /* reset allowed */ + + static const char * const qid_port_stats[] = { "pinned_flows", + "packets" + }; + static const enum xstats_type qid_port_types[] = { pinned, pkts }; + static const uint8_t qid_port_reset_allowed[] = {0, 1}; + /* reset allowed */ + /* ---- end of stat definitions ---- */ + + /* check sizes, since a missed comma can lead to strings being + * joined by the compiler. + */ + RTE_BUILD_BUG_ON(RTE_DIM(dev_stats) != RTE_DIM(dev_types)); + RTE_BUILD_BUG_ON(RTE_DIM(port_stats) != RTE_DIM(port_types)); + RTE_BUILD_BUG_ON(RTE_DIM(qid_stats) != RTE_DIM(qid_types)); + RTE_BUILD_BUG_ON(RTE_DIM(qid_iq_stats) != RTE_DIM(qid_iq_types)); + RTE_BUILD_BUG_ON(RTE_DIM(qid_port_stats) != RTE_DIM(qid_port_types)); + RTE_BUILD_BUG_ON(RTE_DIM(port_bucket_stats) != + RTE_DIM(port_bucket_types)); + + RTE_BUILD_BUG_ON(RTE_DIM(port_stats) != RTE_DIM(port_reset_allowed)); + RTE_BUILD_BUG_ON(RTE_DIM(qid_stats) != RTE_DIM(qid_reset_allowed)); + + /* other vars */ + const uint32_t cons_bkt_shift = + (MAX_SW_CONS_Q_DEPTH >> SW_DEQ_STAT_BUCKET_SHIFT); + const unsigned int count = RTE_DIM(dev_stats) + + sw->port_count * RTE_DIM(port_stats) + + sw->port_count * RTE_DIM(port_bucket_stats) * + (cons_bkt_shift + 1) + + sw->qid_count * RTE_DIM(qid_stats) + + sw->qid_count * SW_IQS_MAX * RTE_DIM(qid_iq_stats) + + sw->qid_count * sw->port_count * + RTE_DIM(qid_port_stats); + unsigned int i, port, qid, iq, bkt, stat = 0; + + sw->xstats = rte_zmalloc_socket(NULL, sizeof(sw->xstats[0]) * count, 0, + sw->data->socket_id); + if (sw->xstats == NULL) + return -ENOMEM; + +#define sname sw->xstats[stat].name.name + for (i = 0; i < RTE_DIM(dev_stats); i++, stat++) { + sw->xstats[stat] = (struct sw_xstats_entry){ + .fn = get_dev_stat, + .stat = dev_types[i], + .mode = RTE_EVENT_DEV_XSTATS_DEVICE, + .reset_allowed = 1, + }; + snprintf(sname, sizeof(sname), "dev_%s", dev_stats[i]); + } + sw->xstats_count_mode_dev = stat; + + for (port = 0; port < sw->port_count; port++) { + sw->xstats_offset_for_port[port] = stat; + + uint32_t count_offset = stat; + + for (i = 0; i < RTE_DIM(port_stats); i++, stat++) { + sw->xstats[stat] = (struct sw_xstats_entry){ + .fn = get_port_stat, + .obj_idx = port, + .stat = port_types[i], + .mode = RTE_EVENT_DEV_XSTATS_PORT, + .reset_allowed = port_reset_allowed[i], + }; + snprintf(sname, sizeof(sname), "port_%u_%s", + port, port_stats[i]); + } + + for (bkt = 0; bkt < (rte_event_ring_get_capacity( + sw->ports[port].cq_worker_ring) >> + SW_DEQ_STAT_BUCKET_SHIFT) + 1; bkt++) { + for (i = 0; i < RTE_DIM(port_bucket_stats); i++) { + sw->xstats[stat] = (struct sw_xstats_entry){ + .fn = get_port_bucket_stat, + .obj_idx = port, + .stat = port_bucket_types[i], + .mode = RTE_EVENT_DEV_XSTATS_PORT, + .extra_arg = bkt, + .reset_allowed = 1, + }; + snprintf(sname, sizeof(sname), + "port_%u_%s_%u-%u", + port, port_bucket_stats[i], + (bkt << SW_DEQ_STAT_BUCKET_SHIFT) + 1, + (bkt + 1) << SW_DEQ_STAT_BUCKET_SHIFT); + stat++; + } + } + + sw->xstats_count_per_port[port] = stat - count_offset; + } + + sw->xstats_count_mode_port = stat - sw->xstats_count_mode_dev; + + for (qid = 0; qid < sw->qid_count; qid++) { + uint32_t count_offset = stat; + sw->xstats_offset_for_qid[qid] = stat; + + for (i = 0; i < RTE_DIM(qid_stats); i++, stat++) { + sw->xstats[stat] = (struct sw_xstats_entry){ + .fn = get_qid_stat, + .obj_idx = qid, + .stat = qid_types[i], + .mode = RTE_EVENT_DEV_XSTATS_QUEUE, + .reset_allowed = qid_reset_allowed[i], + }; + snprintf(sname, sizeof(sname), "qid_%u_%s", + qid, qid_stats[i]); + } + for (iq = 0; iq < SW_IQS_MAX; iq++) + for (i = 0; i < RTE_DIM(qid_iq_stats); i++, stat++) { + sw->xstats[stat] = (struct sw_xstats_entry){ + .fn = get_qid_iq_stat, + .obj_idx = qid, + .stat = qid_iq_types[i], + .mode = RTE_EVENT_DEV_XSTATS_QUEUE, + .extra_arg = iq, + .reset_allowed = 0, + }; + snprintf(sname, sizeof(sname), + "qid_%u_iq_%u_%s", + qid, iq, + qid_iq_stats[i]); + } + + for (port = 0; port < sw->port_count; port++) + for (i = 0; i < RTE_DIM(qid_port_stats); i++, stat++) { + sw->xstats[stat] = (struct sw_xstats_entry){ + .fn = get_qid_port_stat, + .obj_idx = qid, + .stat = qid_port_types[i], + .mode = RTE_EVENT_DEV_XSTATS_QUEUE, + .extra_arg = port, + .reset_allowed = + qid_port_reset_allowed[i], + }; + snprintf(sname, sizeof(sname), + "qid_%u_port_%u_%s", + qid, port, + qid_port_stats[i]); + } + + sw->xstats_count_per_qid[qid] = stat - count_offset; + } + + sw->xstats_count_mode_queue = stat - + (sw->xstats_count_mode_dev + sw->xstats_count_mode_port); +#undef sname + + sw->xstats_count = stat; + + return stat; +} + +int +sw_xstats_uninit(struct sw_evdev *sw) +{ + rte_free(sw->xstats); + sw->xstats_count = 0; + return 0; +} + +int +sw_xstats_get_names(const struct rte_eventdev *dev, + enum rte_event_dev_xstats_mode mode, uint8_t queue_port_id, + struct rte_event_dev_xstats_name *xstats_names, + unsigned int *ids, unsigned int size) +{ + const struct sw_evdev *sw = sw_pmd_priv_const(dev); + unsigned int i; + unsigned int xidx = 0; + RTE_SET_USED(mode); + RTE_SET_USED(queue_port_id); + + uint32_t xstats_mode_count = 0; + uint32_t start_offset = 0; + + switch (mode) { + case RTE_EVENT_DEV_XSTATS_DEVICE: + xstats_mode_count = sw->xstats_count_mode_dev; + break; + case RTE_EVENT_DEV_XSTATS_PORT: + if (queue_port_id >= (signed int)sw->port_count) + break; + xstats_mode_count = sw->xstats_count_per_port[queue_port_id]; + start_offset = sw->xstats_offset_for_port[queue_port_id]; + break; + case RTE_EVENT_DEV_XSTATS_QUEUE: + if (queue_port_id >= (signed int)sw->qid_count) + break; + xstats_mode_count = sw->xstats_count_per_qid[queue_port_id]; + start_offset = sw->xstats_offset_for_qid[queue_port_id]; + break; + default: + SW_LOG_ERR("Invalid mode received in sw_xstats_get_names()\n"); + return -EINVAL; + }; + + if (xstats_mode_count > size || !ids || !xstats_names) + return xstats_mode_count; + + for (i = 0; i < sw->xstats_count && xidx < size; i++) { + if (sw->xstats[i].mode != mode) + continue; + + if (mode != RTE_EVENT_DEV_XSTATS_DEVICE && + queue_port_id != sw->xstats[i].obj_idx) + continue; + + xstats_names[xidx] = sw->xstats[i].name; + if (ids) + ids[xidx] = start_offset + xidx; + xidx++; + } + return xidx; +} + +static int +sw_xstats_update(struct sw_evdev *sw, enum rte_event_dev_xstats_mode mode, + uint8_t queue_port_id, const unsigned int ids[], + uint64_t values[], unsigned int n, const uint32_t reset, + const uint32_t ret_if_n_lt_nstats) +{ + unsigned int i; + unsigned int xidx = 0; + RTE_SET_USED(mode); + RTE_SET_USED(queue_port_id); + + uint32_t xstats_mode_count = 0; + + switch (mode) { + case RTE_EVENT_DEV_XSTATS_DEVICE: + xstats_mode_count = sw->xstats_count_mode_dev; + break; + case RTE_EVENT_DEV_XSTATS_PORT: + if (queue_port_id >= (signed int)sw->port_count) + goto invalid_value; + xstats_mode_count = sw->xstats_count_per_port[queue_port_id]; + break; + case RTE_EVENT_DEV_XSTATS_QUEUE: + if (queue_port_id >= (signed int)sw->qid_count) + goto invalid_value; + xstats_mode_count = sw->xstats_count_per_qid[queue_port_id]; + break; + default: + SW_LOG_ERR("Invalid mode received in sw_xstats_get()\n"); + goto invalid_value; + }; + + /* this function can check num stats and return them (xstats_get() style + * behaviour) or ignore n for reset() of a single stat style behaviour. + */ + if (ret_if_n_lt_nstats && xstats_mode_count > n) + return xstats_mode_count; + + for (i = 0; i < n && xidx < xstats_mode_count; i++) { + struct sw_xstats_entry *xs = &sw->xstats[ids[i]]; + if (ids[i] > sw->xstats_count || xs->mode != mode) + continue; + + if (mode != RTE_EVENT_DEV_XSTATS_DEVICE && + queue_port_id != xs->obj_idx) + continue; + + uint64_t val = xs->fn(sw, xs->obj_idx, xs->stat, xs->extra_arg) + - xs->reset_value; + + if (values) + values[xidx] = val; + + if (xs->reset_allowed && reset) + xs->reset_value = val; + + xidx++; + } + + return xidx; +invalid_value: + return -EINVAL; +} + +int +sw_xstats_get(const struct rte_eventdev *dev, + enum rte_event_dev_xstats_mode mode, uint8_t queue_port_id, + const unsigned int ids[], uint64_t values[], unsigned int n) +{ + struct sw_evdev *sw = sw_pmd_priv(dev); + const uint32_t reset = 0; + const uint32_t ret_n_lt_stats = 0; + return sw_xstats_update(sw, mode, queue_port_id, ids, values, n, + reset, ret_n_lt_stats); +} + +uint64_t +sw_xstats_get_by_name(const struct rte_eventdev *dev, + const char *name, unsigned int *id) +{ + const struct sw_evdev *sw = sw_pmd_priv_const(dev); + unsigned int i; + + for (i = 0; i < sw->xstats_count; i++) { + struct sw_xstats_entry *xs = &sw->xstats[i]; + if (strncmp(xs->name.name, name, + RTE_EVENT_DEV_XSTATS_NAME_SIZE) == 0){ + if (id != NULL) + *id = i; + return xs->fn(sw, xs->obj_idx, xs->stat, xs->extra_arg) + - xs->reset_value; + } + } + if (id != NULL) + *id = (uint32_t)-1; + return (uint64_t)-1; +} + +static void +sw_xstats_reset_range(struct sw_evdev *sw, uint32_t start, uint32_t num) +{ + uint32_t i; + for (i = start; i < start + num; i++) { + struct sw_xstats_entry *xs = &sw->xstats[i]; + if (!xs->reset_allowed) + continue; + + uint64_t val = xs->fn(sw, xs->obj_idx, xs->stat, xs->extra_arg) + - xs->reset_value; + xs->reset_value = val; + } +} + +static int +sw_xstats_reset_queue(struct sw_evdev *sw, uint8_t queue_id, + const uint32_t ids[], uint32_t nb_ids) +{ + const uint32_t reset = 1; + const uint32_t ret_n_lt_stats = 0; + if (ids) { + uint32_t nb_reset = sw_xstats_update(sw, + RTE_EVENT_DEV_XSTATS_QUEUE, + queue_id, ids, NULL, nb_ids, + reset, ret_n_lt_stats); + return nb_reset == nb_ids ? 0 : -EINVAL; + } + + if (ids == NULL) + sw_xstats_reset_range(sw, sw->xstats_offset_for_qid[queue_id], + sw->xstats_count_per_qid[queue_id]); + + return 0; +} + +static int +sw_xstats_reset_port(struct sw_evdev *sw, uint8_t port_id, + const uint32_t ids[], uint32_t nb_ids) +{ + const uint32_t reset = 1; + const uint32_t ret_n_lt_stats = 0; + int offset = sw->xstats_offset_for_port[port_id]; + int nb_stat = sw->xstats_count_per_port[port_id]; + + if (ids) { + uint32_t nb_reset = sw_xstats_update(sw, + RTE_EVENT_DEV_XSTATS_PORT, port_id, + ids, NULL, nb_ids, + reset, ret_n_lt_stats); + return nb_reset == nb_ids ? 0 : -EINVAL; + } + + sw_xstats_reset_range(sw, offset, nb_stat); + return 0; +} + +static int +sw_xstats_reset_dev(struct sw_evdev *sw, const uint32_t ids[], uint32_t nb_ids) +{ + uint32_t i; + if (ids) { + for (i = 0; i < nb_ids; i++) { + uint32_t id = ids[i]; + if (id >= sw->xstats_count_mode_dev) + return -EINVAL; + sw_xstats_reset_range(sw, id, 1); + } + } else { + for (i = 0; i < sw->xstats_count_mode_dev; i++) + sw_xstats_reset_range(sw, i, 1); + } + + return 0; +} + +int +sw_xstats_reset(struct rte_eventdev *dev, + enum rte_event_dev_xstats_mode mode, + int16_t queue_port_id, + const uint32_t ids[], + uint32_t nb_ids) +{ + struct sw_evdev *sw = sw_pmd_priv(dev); + uint32_t i, err; + + /* handle -1 for queue_port_id here, looping over all ports/queues */ + switch (mode) { + case RTE_EVENT_DEV_XSTATS_DEVICE: + sw_xstats_reset_dev(sw, ids, nb_ids); + break; + case RTE_EVENT_DEV_XSTATS_PORT: + if (queue_port_id == -1) { + for (i = 0; i < sw->port_count; i++) { + err = sw_xstats_reset_port(sw, i, ids, nb_ids); + if (err) + return -EINVAL; + } + } else if (queue_port_id < (int16_t)sw->port_count) + sw_xstats_reset_port(sw, queue_port_id, ids, nb_ids); + break; + case RTE_EVENT_DEV_XSTATS_QUEUE: + if (queue_port_id == -1) { + for (i = 0; i < sw->qid_count; i++) { + err = sw_xstats_reset_queue(sw, i, ids, nb_ids); + if (err) + return -EINVAL; + } + } else if (queue_port_id < (int16_t)sw->qid_count) + sw_xstats_reset_queue(sw, queue_port_id, ids, nb_ids); + break; + }; + + return 0; +} diff --git a/dpdk/drivers/mempool/Makefile b/dpdk/drivers/mempool/Makefile new file mode 100644 index 00000000..f656c566 --- /dev/null +++ b/dpdk/drivers/mempool/Makefile @@ -0,0 +1,39 @@ +# BSD LICENSE +# +# Copyright 2017 NXP. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of NXP nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +include $(RTE_SDK)/mk/rte.vars.mk + +DIRS-$(CONFIG_RTE_LIBRTE_DPAA_MEMPOOL) += dpaa +DIRS-$(CONFIG_RTE_LIBRTE_DPAA2_MEMPOOL) += dpaa2 +DIRS-$(CONFIG_RTE_DRIVER_MEMPOOL_RING) += ring +DIRS-$(CONFIG_RTE_DRIVER_MEMPOOL_STACK) += stack +DIRS-$(CONFIG_RTE_LIBRTE_OCTEONTX_MEMPOOL) += octeontx + +include $(RTE_SDK)/mk/rte.subdir.mk diff --git a/dpdk/drivers/mempool/dpaa/Makefile b/dpdk/drivers/mempool/dpaa/Makefile new file mode 100644 index 00000000..c49b0ee3 --- /dev/null +++ b/dpdk/drivers/mempool/dpaa/Makefile @@ -0,0 +1,59 @@ +# BSD LICENSE +# +# Copyright 2016 NXP. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of NXP nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +include $(RTE_SDK)/mk/rte.vars.mk + +# +# library name +# +LIB = librte_mempool_dpaa.a + +CFLAGS := -I$(SRCDIR) $(CFLAGS) +CFLAGS += -O3 $(WERROR_FLAGS) +CFLAGS += -D _GNU_SOURCE +CFLAGS += -I$(RTE_SDK)/drivers/bus/dpaa +CFLAGS += -I$(RTE_SDK)/drivers/bus/dpaa/include/ +CFLAGS += -I$(RTE_SDK)/drivers/mempool/dpaa +CFLAGS += -I$(RTE_SDK)/lib/librte_mempool + +# versioning export map +EXPORT_MAP := rte_mempool_dpaa_version.map + +# Lbrary version +LIBABIVER := 1 + +# all source are stored in SRCS-y +# +SRCS-$(CONFIG_RTE_LIBRTE_DPAA_MEMPOOL) += dpaa_mempool.c + +LDLIBS += -lrte_bus_dpaa +LDLIBS += -lrte_eal -lrte_mempool -lrte_ring + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/mempool/dpaa/dpaa_mempool.c b/dpdk/drivers/mempool/dpaa/dpaa_mempool.c new file mode 100644 index 00000000..f5ee80f2 --- /dev/null +++ b/dpdk/drivers/mempool/dpaa/dpaa_mempool.c @@ -0,0 +1,284 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of NXP nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* System headers */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +struct dpaa_bp_info rte_dpaa_bpid_info[DPAA_MAX_BPOOLS]; + +static int +dpaa_mbuf_create_pool(struct rte_mempool *mp) +{ + struct bman_pool *bp; + struct bm_buffer bufs[8]; + struct dpaa_bp_info *bp_info; + uint8_t bpid; + int num_bufs = 0, ret = 0; + struct bman_pool_params params = { + .flags = BMAN_POOL_FLAG_DYNAMIC_BPID + }; + + MEMPOOL_INIT_FUNC_TRACE(); + + bp = bman_new_pool(¶ms); + if (!bp) { + DPAA_MEMPOOL_ERR("bman_new_pool() failed"); + return -ENODEV; + } + bpid = bman_get_params(bp)->bpid; + + /* Drain the pool of anything already in it. */ + do { + /* Acquire is all-or-nothing, so we drain in 8s, + * then in 1s for the remainder. + */ + if (ret != 1) + ret = bman_acquire(bp, bufs, 8, 0); + if (ret < 8) + ret = bman_acquire(bp, bufs, 1, 0); + if (ret > 0) + num_bufs += ret; + } while (ret > 0); + if (num_bufs) + DPAA_MEMPOOL_WARN("drained %u bufs from BPID %d", + num_bufs, bpid); + + rte_dpaa_bpid_info[bpid].mp = mp; + rte_dpaa_bpid_info[bpid].bpid = bpid; + rte_dpaa_bpid_info[bpid].size = mp->elt_size; + rte_dpaa_bpid_info[bpid].bp = bp; + rte_dpaa_bpid_info[bpid].meta_data_size = + sizeof(struct rte_mbuf) + rte_pktmbuf_priv_size(mp); + rte_dpaa_bpid_info[bpid].dpaa_ops_index = mp->ops_index; + + bp_info = rte_malloc(NULL, + sizeof(struct dpaa_bp_info), + RTE_CACHE_LINE_SIZE); + if (!bp_info) { + DPAA_MEMPOOL_WARN("Memory allocation failed for bp_info"); + bman_free_pool(bp); + return -ENOMEM; + } + + rte_memcpy(bp_info, (void *)&rte_dpaa_bpid_info[bpid], + sizeof(struct dpaa_bp_info)); + mp->pool_data = (void *)bp_info; + + DPAA_MEMPOOL_INFO("BMAN pool created for bpid =%d", bpid); + return 0; +} + +static void +dpaa_mbuf_free_pool(struct rte_mempool *mp) +{ + struct dpaa_bp_info *bp_info = DPAA_MEMPOOL_TO_POOL_INFO(mp); + + MEMPOOL_INIT_FUNC_TRACE(); + + if (bp_info) { + bman_free_pool(bp_info->bp); + DPAA_MEMPOOL_INFO("BMAN pool freed for bpid =%d", + bp_info->bpid); + rte_free(mp->pool_data); + mp->pool_data = NULL; + } +} + +static void +dpaa_buf_free(struct dpaa_bp_info *bp_info, uint64_t addr) +{ + struct bm_buffer buf; + int ret; + + DPAA_MEMPOOL_DEBUG("Free 0x%lx to bpid: %d", addr, bp_info->bpid); + + bm_buffer_set64(&buf, addr); +retry: + ret = bman_release(bp_info->bp, &buf, 1, 0); + if (ret) { + DPAA_MEMPOOL_DEBUG("BMAN busy. Retrying..."); + cpu_spin(CPU_SPIN_BACKOFF_CYCLES); + goto retry; + } +} + +static int +dpaa_mbuf_free_bulk(struct rte_mempool *pool, + void *const *obj_table, + unsigned int n) +{ + struct dpaa_bp_info *bp_info = DPAA_MEMPOOL_TO_POOL_INFO(pool); + int ret; + unsigned int i = 0; + + DPAA_MEMPOOL_DPDEBUG("Request to free %d buffers in bpid = %d", + n, bp_info->bpid); + + ret = rte_dpaa_portal_init((void *)0); + if (ret) { + DPAA_MEMPOOL_ERR("rte_dpaa_portal_init failed with ret: %d", + ret); + return 0; + } + + while (i < n) { + dpaa_buf_free(bp_info, + (uint64_t)rte_mempool_virt2iova(obj_table[i]) + + bp_info->meta_data_size); + i = i + 1; + } + + DPAA_MEMPOOL_DPDEBUG("freed %d buffers in bpid =%d", + n, bp_info->bpid); + + return 0; +} + +static int +dpaa_mbuf_alloc_bulk(struct rte_mempool *pool, + void **obj_table, + unsigned int count) +{ + struct rte_mbuf **m = (struct rte_mbuf **)obj_table; + struct bm_buffer bufs[DPAA_MBUF_MAX_ACQ_REL]; + struct dpaa_bp_info *bp_info; + void *bufaddr; + int i, ret; + unsigned int n = 0; + + bp_info = DPAA_MEMPOOL_TO_POOL_INFO(pool); + + DPAA_MEMPOOL_DPDEBUG("Request to alloc %d buffers in bpid = %d", + count, bp_info->bpid); + + if (unlikely(count >= (RTE_MEMPOOL_CACHE_MAX_SIZE * 2))) { + DPAA_MEMPOOL_ERR("Unable to allocate requested (%u) buffers", + count); + return -1; + } + + ret = rte_dpaa_portal_init((void *)0); + if (ret) { + DPAA_MEMPOOL_ERR("rte_dpaa_portal_init failed with ret: %d", + ret); + return -1; + } + + while (n < count) { + /* Acquire is all-or-nothing, so we drain in 7s, + * then the remainder. + */ + if ((count - n) > DPAA_MBUF_MAX_ACQ_REL) { + ret = bman_acquire(bp_info->bp, bufs, + DPAA_MBUF_MAX_ACQ_REL, 0); + } else { + ret = bman_acquire(bp_info->bp, bufs, count - n, 0); + } + /* In case of less than requested number of buffers available + * in pool, qbman_swp_acquire returns 0 + */ + if (ret <= 0) { + DPAA_MEMPOOL_DPDEBUG("Buffer acquire failed (%d)", + ret); + /* The API expect the exact number of requested + * buffers. Releasing all buffers allocated + */ + dpaa_mbuf_free_bulk(pool, obj_table, n); + return -ENOBUFS; + } + /* assigning mbuf from the acquired objects */ + for (i = 0; (i < ret) && bufs[i].addr; i++) { + /* TODO-errata - objerved that bufs may be null + * i.e. first buffer is valid, remaining 6 buffers + * may be null. + */ + bufaddr = (void *)rte_dpaa_mem_ptov(bufs[i].addr); + m[n] = (struct rte_mbuf *)((char *)bufaddr + - bp_info->meta_data_size); + DPAA_MEMPOOL_DPDEBUG("Paddr (%p), FD (%p) from BMAN", + (void *)bufaddr, (void *)m[n]); + n++; + } + } + + DPAA_MEMPOOL_DPDEBUG("Allocated %d buffers from bpid=%d", + n, bp_info->bpid); + return 0; +} + +static unsigned int +dpaa_mbuf_get_count(const struct rte_mempool *mp) +{ + struct dpaa_bp_info *bp_info; + + MEMPOOL_INIT_FUNC_TRACE(); + + if (!mp || !mp->pool_data) { + DPAA_MEMPOOL_ERR("Invalid mempool provided\n"); + return 0; + } + + bp_info = DPAA_MEMPOOL_TO_POOL_INFO(mp); + + return bman_query_free_buffers(bp_info->bp); +} + +struct rte_mempool_ops dpaa_mpool_ops = { + .name = "dpaa", + .alloc = dpaa_mbuf_create_pool, + .free = dpaa_mbuf_free_pool, + .enqueue = dpaa_mbuf_free_bulk, + .dequeue = dpaa_mbuf_alloc_bulk, + .get_count = dpaa_mbuf_get_count, +}; + +MEMPOOL_REGISTER_OPS(dpaa_mpool_ops); diff --git a/dpdk/drivers/mempool/dpaa/dpaa_mempool.h b/dpdk/drivers/mempool/dpaa/dpaa_mempool.h new file mode 100644 index 00000000..67958594 --- /dev/null +++ b/dpdk/drivers/mempool/dpaa/dpaa_mempool.h @@ -0,0 +1,76 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of NXP nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef __DPAA_MEMPOOL_H__ +#define __DPAA_MEMPOOL_H__ + +/* System headers */ +#include +#include +#include +#include + +#include + +#include +#include + +#include +#include + +#define CPU_SPIN_BACKOFF_CYCLES 512 + +/* total number of bpools on SoC */ +#define DPAA_MAX_BPOOLS 256 + +/* Maximum release/acquire from BMAN */ +#define DPAA_MBUF_MAX_ACQ_REL 8 + +struct dpaa_bp_info { + struct rte_mempool *mp; + struct bman_pool *bp; + uint32_t bpid; + uint32_t size; + uint32_t meta_data_size; + int32_t dpaa_ops_index; +}; + +#define DPAA_MEMPOOL_TO_POOL_INFO(__mp) \ + ((struct dpaa_bp_info *)__mp->pool_data) + +#define DPAA_MEMPOOL_TO_BPID(__mp) \ + (((struct dpaa_bp_info *)__mp->pool_data)->bpid) + +extern struct dpaa_bp_info rte_dpaa_bpid_info[DPAA_MAX_BPOOLS]; + +#define DPAA_BPID_TO_POOL_INFO(__bpid) (&rte_dpaa_bpid_info[__bpid]) + +#endif diff --git a/dpdk/drivers/mempool/dpaa/rte_mempool_dpaa_version.map b/dpdk/drivers/mempool/dpaa/rte_mempool_dpaa_version.map new file mode 100644 index 00000000..cc635c73 --- /dev/null +++ b/dpdk/drivers/mempool/dpaa/rte_mempool_dpaa_version.map @@ -0,0 +1,8 @@ +DPDK_17.11 { + global: + + rte_dpaa_bpid_info; + rte_dpaa_pool_table; + + local: *; +}; diff --git a/dpdk/drivers/mempool/dpaa2/Makefile b/dpdk/drivers/mempool/dpaa2/Makefile new file mode 100644 index 00000000..dd19e100 --- /dev/null +++ b/dpdk/drivers/mempool/dpaa2/Makefile @@ -0,0 +1,63 @@ +# BSD LICENSE +# +# Copyright 2016 NXP. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of NXP nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +include $(RTE_SDK)/mk/rte.vars.mk + +# +# library name +# +LIB = librte_mempool_dpaa2.a + +ifeq ($(CONFIG_RTE_LIBRTE_DPAA2_DEBUG_INIT),y) +CFLAGS += -O0 -g +CFLAGS += "-Wno-error" +else +CFLAGS += -O3 +CFLAGS += $(WERROR_FLAGS) +endif + +CFLAGS += -I$(RTE_SDK)/drivers/bus/fslmc +CFLAGS += -I$(RTE_SDK)/drivers/bus/fslmc/qbman/include +CFLAGS += -I$(RTE_SDK)/lib/librte_eal/linuxapp/eal + +# versioning export map +EXPORT_MAP := rte_mempool_dpaa2_version.map + +# Lbrary version +LIBABIVER := 1 + +# all source are stored in SRCS-y +# +SRCS-$(CONFIG_RTE_LIBRTE_DPAA2_MEMPOOL) += dpaa2_hw_mempool.c + +LDLIBS += -lrte_bus_fslmc +LDLIBS += -lrte_eal -lrte_mempool -lrte_ring + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/mempool/dpaa2/dpaa2_hw_mempool.c b/dpdk/drivers/mempool/dpaa2/dpaa2_hw_mempool.c new file mode 100644 index 00000000..8bcbaa89 --- /dev/null +++ b/dpdk/drivers/mempool/dpaa2/dpaa2_hw_mempool.c @@ -0,0 +1,391 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. + * Copyright 2016 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Freescale Semiconductor, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include "dpaa2_hw_mempool.h" + +struct dpaa2_bp_info rte_dpaa2_bpid_info[MAX_BPID]; +static struct dpaa2_bp_list *h_bp_list; + +static int +rte_hw_mbuf_create_pool(struct rte_mempool *mp) +{ + struct dpaa2_bp_list *bp_list; + struct dpaa2_dpbp_dev *avail_dpbp; + struct dpaa2_bp_info *bp_info; + struct dpbp_attr dpbp_attr; + uint32_t bpid; + int ret; + + avail_dpbp = dpaa2_alloc_dpbp_dev(); + + if (!avail_dpbp) { + PMD_DRV_LOG(ERR, "DPAA2 resources not available"); + return -ENOENT; + } + + if (unlikely(!DPAA2_PER_LCORE_DPIO)) { + ret = dpaa2_affine_qbman_swp(); + if (ret) { + RTE_LOG(ERR, PMD, "Failure in affining portal\n"); + goto err1; + } + } + + ret = dpbp_enable(&avail_dpbp->dpbp, CMD_PRI_LOW, avail_dpbp->token); + if (ret != 0) { + PMD_INIT_LOG(ERR, "Resource enable failure with" + " err code: %d\n", ret); + goto err1; + } + + ret = dpbp_get_attributes(&avail_dpbp->dpbp, CMD_PRI_LOW, + avail_dpbp->token, &dpbp_attr); + if (ret != 0) { + PMD_INIT_LOG(ERR, "Resource read failure with" + " err code: %d\n", ret); + goto err2; + } + + bp_info = rte_malloc(NULL, + sizeof(struct dpaa2_bp_info), + RTE_CACHE_LINE_SIZE); + if (!bp_info) { + PMD_INIT_LOG(ERR, "No heap memory available for bp_info"); + ret = -ENOMEM; + goto err2; + } + + /* Allocate the bp_list which will be added into global_bp_list */ + bp_list = rte_malloc(NULL, sizeof(struct dpaa2_bp_list), + RTE_CACHE_LINE_SIZE); + if (!bp_list) { + PMD_INIT_LOG(ERR, "No heap memory available"); + ret = -ENOMEM; + goto err3; + } + + /* Set parameters of buffer pool list */ + bp_list->buf_pool.num_bufs = mp->size; + bp_list->buf_pool.size = mp->elt_size + - sizeof(struct rte_mbuf) - rte_pktmbuf_priv_size(mp); + bp_list->buf_pool.bpid = dpbp_attr.bpid; + bp_list->buf_pool.h_bpool_mem = NULL; + bp_list->buf_pool.dpbp_node = avail_dpbp; + /* Identification for our offloaded pool_data structure */ + bp_list->dpaa2_ops_index = mp->ops_index; + bp_list->next = h_bp_list; + bp_list->mp = mp; + + bpid = dpbp_attr.bpid; + + rte_dpaa2_bpid_info[bpid].meta_data_size = sizeof(struct rte_mbuf) + + rte_pktmbuf_priv_size(mp); + rte_dpaa2_bpid_info[bpid].bp_list = bp_list; + rte_dpaa2_bpid_info[bpid].bpid = bpid; + + rte_memcpy(bp_info, (void *)&rte_dpaa2_bpid_info[bpid], + sizeof(struct dpaa2_bp_info)); + mp->pool_data = (void *)bp_info; + + PMD_INIT_LOG(DEBUG, "BP List created for bpid =%d", dpbp_attr.bpid); + + h_bp_list = bp_list; + return 0; +err3: + rte_free(bp_info); +err2: + dpbp_disable(&avail_dpbp->dpbp, CMD_PRI_LOW, avail_dpbp->token); +err1: + dpaa2_free_dpbp_dev(avail_dpbp); + + return ret; +} + +static void +rte_hw_mbuf_free_pool(struct rte_mempool *mp) +{ + struct dpaa2_bp_info *bpinfo; + struct dpaa2_bp_list *bp; + struct dpaa2_dpbp_dev *dpbp_node; + + if (!mp->pool_data) { + PMD_DRV_LOG(ERR, "Not a valid dpaa22 pool"); + return; + } + + bpinfo = (struct dpaa2_bp_info *)mp->pool_data; + bp = bpinfo->bp_list; + dpbp_node = bp->buf_pool.dpbp_node; + + dpbp_disable(&(dpbp_node->dpbp), CMD_PRI_LOW, dpbp_node->token); + + if (h_bp_list == bp) { + h_bp_list = h_bp_list->next; + } else { /* if it is not the first node */ + struct dpaa2_bp_list *prev = h_bp_list, *temp; + temp = h_bp_list->next; + while (temp) { + if (temp == bp) { + prev->next = temp->next; + rte_free(bp); + break; + } + prev = temp; + temp = temp->next; + } + } + + rte_free(mp->pool_data); + dpaa2_free_dpbp_dev(dpbp_node); +} + +static void +rte_dpaa2_mbuf_release(struct rte_mempool *pool __rte_unused, + void * const *obj_table, + uint32_t bpid, + uint32_t meta_data_size, + int count) +{ + struct qbman_release_desc releasedesc; + struct qbman_swp *swp; + int ret; + int i, n; + uint64_t bufs[DPAA2_MBUF_MAX_ACQ_REL]; + + if (unlikely(!DPAA2_PER_LCORE_DPIO)) { + ret = dpaa2_affine_qbman_swp(); + if (ret != 0) { + RTE_LOG(ERR, PMD, "Failed to allocate IO portal\n"); + return; + } + } + swp = DPAA2_PER_LCORE_PORTAL; + + /* Create a release descriptor required for releasing + * buffers into QBMAN + */ + qbman_release_desc_clear(&releasedesc); + qbman_release_desc_set_bpid(&releasedesc, bpid); + + n = count % DPAA2_MBUF_MAX_ACQ_REL; + if (unlikely(!n)) + goto aligned; + + /* convert mbuf to buffers for the remainder */ + for (i = 0; i < n ; i++) { +#ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA + bufs[i] = (uint64_t)rte_mempool_virt2iova(obj_table[i]) + + meta_data_size; +#else + bufs[i] = (uint64_t)obj_table[i] + meta_data_size; +#endif + } + + /* feed them to bman */ + do { + ret = qbman_swp_release(swp, &releasedesc, bufs, n); + } while (ret == -EBUSY); + +aligned: + /* if there are more buffers to free */ + while (n < count) { + /* convert mbuf to buffers */ + for (i = 0; i < DPAA2_MBUF_MAX_ACQ_REL; i++) { +#ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA + bufs[i] = (uint64_t) + rte_mempool_virt2iova(obj_table[n + i]) + + meta_data_size; +#else + bufs[i] = (uint64_t)obj_table[n + i] + meta_data_size; +#endif + } + + do { + ret = qbman_swp_release(swp, &releasedesc, bufs, + DPAA2_MBUF_MAX_ACQ_REL); + } while (ret == -EBUSY); + n += DPAA2_MBUF_MAX_ACQ_REL; + } +} + +int +rte_dpaa2_mbuf_alloc_bulk(struct rte_mempool *pool, + void **obj_table, unsigned int count) +{ +#ifdef RTE_LIBRTE_DPAA2_DEBUG_DRIVER + static int alloc; +#endif + struct qbman_swp *swp; + uint16_t bpid; + uint64_t bufs[DPAA2_MBUF_MAX_ACQ_REL]; + int i, ret; + unsigned int n = 0; + struct dpaa2_bp_info *bp_info; + + bp_info = mempool_to_bpinfo(pool); + + if (!(bp_info->bp_list)) { + RTE_LOG(ERR, PMD, "DPAA2 buffer pool not configured\n"); + return -ENOENT; + } + + bpid = bp_info->bpid; + + if (unlikely(!DPAA2_PER_LCORE_DPIO)) { + ret = dpaa2_affine_qbman_swp(); + if (ret != 0) { + RTE_LOG(ERR, PMD, "Failed to allocate IO portal\n"); + return ret; + } + } + swp = DPAA2_PER_LCORE_PORTAL; + + while (n < count) { + /* Acquire is all-or-nothing, so we drain in 7s, + * then the remainder. + */ + if ((count - n) > DPAA2_MBUF_MAX_ACQ_REL) { + ret = qbman_swp_acquire(swp, bpid, bufs, + DPAA2_MBUF_MAX_ACQ_REL); + } else { + ret = qbman_swp_acquire(swp, bpid, bufs, + count - n); + } + /* In case of less than requested number of buffers available + * in pool, qbman_swp_acquire returns 0 + */ + if (ret <= 0) { + PMD_TX_LOG(ERR, "Buffer acquire failed with" + " err code: %d", ret); + /* The API expect the exact number of requested bufs */ + /* Releasing all buffers allocated */ + rte_dpaa2_mbuf_release(pool, obj_table, bpid, + bp_info->meta_data_size, n); + return -ENOBUFS; + } + /* assigning mbuf from the acquired objects */ + for (i = 0; (i < ret) && bufs[i]; i++) { + DPAA2_MODIFY_IOVA_TO_VADDR(bufs[i], uint64_t); + obj_table[n] = (struct rte_mbuf *) + (bufs[i] - bp_info->meta_data_size); + PMD_TX_LOG(DEBUG, "Acquired %p address %p from BMAN", + (void *)bufs[i], (void *)obj_table[n]); + n++; + } + } + +#ifdef RTE_LIBRTE_DPAA2_DEBUG_DRIVER + alloc += n; + PMD_TX_LOG(DEBUG, "Total = %d , req = %d done = %d", + alloc, count, n); +#endif + return 0; +} + +static int +rte_hw_mbuf_free_bulk(struct rte_mempool *pool, + void * const *obj_table, unsigned int n) +{ + struct dpaa2_bp_info *bp_info; + + bp_info = mempool_to_bpinfo(pool); + if (!(bp_info->bp_list)) { + RTE_LOG(ERR, PMD, "DPAA2 buffer pool not configured\n"); + return -ENOENT; + } + rte_dpaa2_mbuf_release(pool, obj_table, bp_info->bpid, + bp_info->meta_data_size, n); + + return 0; +} + +static unsigned int +rte_hw_mbuf_get_count(const struct rte_mempool *mp) +{ + int ret; + unsigned int num_of_bufs = 0; + struct dpaa2_bp_info *bp_info; + struct dpaa2_dpbp_dev *dpbp_node; + + if (!mp || !mp->pool_data) { + RTE_LOG(ERR, PMD, "Invalid mempool provided\n"); + return 0; + } + + bp_info = (struct dpaa2_bp_info *)mp->pool_data; + dpbp_node = bp_info->bp_list->buf_pool.dpbp_node; + + ret = dpbp_get_num_free_bufs(&dpbp_node->dpbp, CMD_PRI_LOW, + dpbp_node->token, &num_of_bufs); + if (ret) { + RTE_LOG(ERR, PMD, "Unable to obtain free buf count (err=%d)\n", + ret); + return 0; + } + + RTE_LOG(DEBUG, PMD, "Free bufs = %u\n", num_of_bufs); + + return num_of_bufs; +} + +struct rte_mempool_ops dpaa2_mpool_ops = { + .name = "dpaa2", + .alloc = rte_hw_mbuf_create_pool, + .free = rte_hw_mbuf_free_pool, + .enqueue = rte_hw_mbuf_free_bulk, + .dequeue = rte_dpaa2_mbuf_alloc_bulk, + .get_count = rte_hw_mbuf_get_count, +}; + +MEMPOOL_REGISTER_OPS(dpaa2_mpool_ops); diff --git a/dpdk/drivers/mempool/dpaa2/dpaa2_hw_mempool.h b/dpdk/drivers/mempool/dpaa2/dpaa2_hw_mempool.h new file mode 100644 index 00000000..0971929e --- /dev/null +++ b/dpdk/drivers/mempool/dpaa2/dpaa2_hw_mempool.h @@ -0,0 +1,91 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. + * Copyright 2016 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Freescale Semiconductor, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _DPAA2_HW_DPBP_H_ +#define _DPAA2_HW_DPBP_H_ + +#define DPAA2_MAX_BUF_POOLS 8 + +struct buf_pool_cfg { + void *addr; + /**< The address from where DPAA2 will carve out the buffers */ + rte_iova_t phys_addr; + /**< Physical address of the memory provided in addr */ + uint32_t num; + /**< Number of buffers */ + uint32_t size; + /**< Size including headroom for each buffer */ + uint16_t align; + /**< Buffer alignment (in bytes) */ + uint16_t bpid; + /**< Autogenerated buffer pool ID for internal use */ +}; + +struct buf_pool { + uint32_t size; /**< Size of the Pool */ + uint32_t num_bufs; /**< Number of buffers in Pool */ + uint16_t bpid; /**< Pool ID, from pool configuration */ + uint8_t *h_bpool_mem; /**< Internal context data */ + struct dpaa2_dpbp_dev *dpbp_node; /**< Hardware context */ +}; + +/*! + * Buffer pool list configuration structure. User need to give DPAA2 the + * valid number of 'num_buf_pools'. + */ +struct dpaa2_bp_list_cfg { + struct buf_pool_cfg buf_pool; /* Configuration of each buffer pool*/ +}; + +struct dpaa2_bp_list { + struct dpaa2_bp_list *next; + struct rte_mempool *mp; /**< DPDK RTE EAL pool reference */ + int32_t dpaa2_ops_index; /**< Index into DPDK Mempool ops table */ + struct buf_pool buf_pool; +}; + +struct dpaa2_bp_info { + uint32_t meta_data_size; + uint32_t bpid; + struct dpaa2_bp_list *bp_list; +}; + +#define mempool_to_bpinfo(mp) ((struct dpaa2_bp_info *)(mp)->pool_data) +#define mempool_to_bpid(mp) ((mempool_to_bpinfo(mp))->bpid) + +extern struct dpaa2_bp_info rte_dpaa2_bpid_info[MAX_BPID]; + +int rte_dpaa2_mbuf_alloc_bulk(struct rte_mempool *pool, + void **obj_table, unsigned int count); + +#endif /* _DPAA2_HW_DPBP_H_ */ diff --git a/dpdk/drivers/mempool/dpaa2/rte_mempool_dpaa2_version.map b/dpdk/drivers/mempool/dpaa2/rte_mempool_dpaa2_version.map new file mode 100644 index 00000000..a8aa685c --- /dev/null +++ b/dpdk/drivers/mempool/dpaa2/rte_mempool_dpaa2_version.map @@ -0,0 +1,8 @@ +DPDK_17.05 { + global: + + rte_dpaa2_bpid_info; + rte_dpaa2_mbuf_alloc_bulk; + + local: *; +}; diff --git a/dpdk/drivers/mempool/octeontx/Makefile b/dpdk/drivers/mempool/octeontx/Makefile new file mode 100644 index 00000000..a2e2863c --- /dev/null +++ b/dpdk/drivers/mempool/octeontx/Makefile @@ -0,0 +1,68 @@ +# BSD LICENSE +# +# Copyright(c) 2017 Cavium Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Cavium Networks nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +include $(RTE_SDK)/mk/rte.vars.mk + +# +# library name +# +LIB = librte_mempool_octeontx.a + +CFLAGS += $(WERROR_FLAGS) +EXPORT_MAP := rte_mempool_octeontx_version.map + +LIBABIVER := 1 + +# +# all source are stored in SRCS-y +# +SRCS-$(CONFIG_RTE_LIBRTE_OCTEONTX_MEMPOOL) += octeontx_ssovf.c +SRCS-$(CONFIG_RTE_LIBRTE_OCTEONTX_MEMPOOL) += octeontx_mbox.c +SRCS-$(CONFIG_RTE_LIBRTE_OCTEONTX_MEMPOOL) += octeontx_fpavf.c +SRCS-$(CONFIG_RTE_LIBRTE_OCTEONTX_MEMPOOL) += rte_mempool_octeontx.c + +ifeq ($(CONFIG_RTE_TOOLCHAIN_GCC),y) +CFLAGS_rte_mempool_octeontx.o += -fno-prefetch-loop-arrays + +ifeq ($(shell test $(GCC_VERSION) -ge 46 && echo 1), 1) +CFLAGS_rte_mempool_octeontx.o += -Ofast +else +CFLAGS_rte_mempool_octeontx.o += -O3 -ffast-math +endif + +else +CFLAGS_rte_mempool_octeontx.o += -Ofast +endif + +LDLIBS += -lrte_eal -lrte_mempool -lrte_ring -lrte_mbuf +LDLIBS += -lrte_bus_pci + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/mempool/octeontx/octeontx_fpavf.c b/dpdk/drivers/mempool/octeontx/octeontx_fpavf.c new file mode 100644 index 00000000..28f431ed --- /dev/null +++ b/dpdk/drivers/mempool/octeontx/octeontx_fpavf.c @@ -0,0 +1,813 @@ +/* + * BSD LICENSE + * + * Copyright (C) Cavium Inc. 2017. All Right reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium networks nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "octeontx_mbox.h" +#include "octeontx_fpavf.h" + +/* FPA Mbox Message */ +#define IDENTIFY 0x0 + +#define FPA_CONFIGSET 0x1 +#define FPA_CONFIGGET 0x2 +#define FPA_START_COUNT 0x3 +#define FPA_STOP_COUNT 0x4 +#define FPA_ATTACHAURA 0x5 +#define FPA_DETACHAURA 0x6 +#define FPA_SETAURALVL 0x7 +#define FPA_GETAURALVL 0x8 + +#define FPA_COPROC 0x1 + +/* fpa mbox struct */ +struct octeontx_mbox_fpa_cfg { + int aid; + uint64_t pool_cfg; + uint64_t pool_stack_base; + uint64_t pool_stack_end; + uint64_t aura_cfg; +}; + +struct __attribute__((__packed__)) gen_req { + uint32_t value; +}; + +struct __attribute__((__packed__)) idn_req { + uint8_t domain_id; +}; + +struct __attribute__((__packed__)) gen_resp { + uint16_t domain_id; + uint16_t vfid; +}; + +struct __attribute__((__packed__)) dcfg_resp { + uint8_t sso_count; + uint8_t ssow_count; + uint8_t fpa_count; + uint8_t pko_count; + uint8_t tim_count; + uint8_t net_port_count; + uint8_t virt_port_count; +}; + +#define FPA_MAX_POOL 32 +#define FPA_PF_PAGE_SZ 4096 + +#define FPA_LN_SIZE 128 +#define FPA_ROUND_UP(x, size) \ + ((((unsigned long)(x)) + size-1) & (~(size-1))) +#define FPA_OBJSZ_2_CACHE_LINE(sz) (((sz) + RTE_CACHE_LINE_MASK) >> 7) +#define FPA_CACHE_LINE_2_OBJSZ(sz) ((sz) << 7) + +#define POOL_ENA (0x1 << 0) +#define POOL_DIS (0x0 << 0) +#define POOL_SET_NAT_ALIGN (0x1 << 1) +#define POOL_DIS_NAT_ALIGN (0x0 << 1) +#define POOL_STYPE(x) (((x) & 0x1) << 2) +#define POOL_LTYPE(x) (((x) & 0x3) << 3) +#define POOL_BUF_OFFSET(x) (((x) & 0x7fffULL) << 16) +#define POOL_BUF_SIZE(x) (((x) & 0x7ffULL) << 32) + +struct fpavf_res { + void *pool_stack_base; + void *bar0; + uint64_t stack_ln_ptr; + uint16_t domain_id; + uint16_t vf_id; /* gpool_id */ + uint16_t sz128; /* Block size in cache lines */ + bool is_inuse; +}; + +struct octeontx_fpadev { + rte_spinlock_t lock; + uint8_t total_gpool_cnt; + struct fpavf_res pool[FPA_VF_MAX]; +}; + +static struct octeontx_fpadev fpadev; + +/* lock is taken by caller */ +static int +octeontx_fpa_gpool_alloc(unsigned int object_size) +{ + struct fpavf_res *res = NULL; + uint16_t gpool; + unsigned int sz128; + + sz128 = FPA_OBJSZ_2_CACHE_LINE(object_size); + + for (gpool = 0; gpool < FPA_VF_MAX; gpool++) { + + /* Skip VF that is not mapped Or _inuse */ + if ((fpadev.pool[gpool].bar0 == NULL) || + (fpadev.pool[gpool].is_inuse == true)) + continue; + + res = &fpadev.pool[gpool]; + + RTE_ASSERT(res->domain_id != (uint16_t)~0); + RTE_ASSERT(res->vf_id != (uint16_t)~0); + RTE_ASSERT(res->stack_ln_ptr != 0); + + if (res->sz128 == 0) { + res->sz128 = sz128; + + fpavf_log_dbg("gpool %d blk_sz %d\n", gpool, sz128); + return gpool; + } + } + + return -ENOSPC; +} + +/* lock is taken by caller */ +static __rte_always_inline uintptr_t +octeontx_fpa_gpool2handle(uint16_t gpool) +{ + struct fpavf_res *res = NULL; + + RTE_ASSERT(gpool < FPA_VF_MAX); + + res = &fpadev.pool[gpool]; + return (uintptr_t)res->bar0 | gpool; +} + +static __rte_always_inline bool +octeontx_fpa_handle_valid(uintptr_t handle) +{ + struct fpavf_res *res = NULL; + uint8_t gpool; + int i; + bool ret = false; + + if (unlikely(!handle)) + return ret; + + /* get the gpool */ + gpool = octeontx_fpa_bufpool_gpool(handle); + + /* get the bar address */ + handle &= ~(uint64_t)FPA_GPOOL_MASK; + for (i = 0; i < FPA_VF_MAX; i++) { + if ((uintptr_t)fpadev.pool[i].bar0 != handle) + continue; + + /* validate gpool */ + if (gpool != i) + return false; + + res = &fpadev.pool[i]; + + if (res->sz128 == 0 || res->domain_id == (uint16_t)~0 || + res->stack_ln_ptr == 0) + ret = false; + else + ret = true; + break; + } + + return ret; +} + +static int +octeontx_fpapf_pool_setup(unsigned int gpool, unsigned int buf_size, + signed short buf_offset, unsigned int max_buf_count) +{ + void *memptr = NULL; + rte_iova_t phys_addr; + unsigned int memsz; + struct fpavf_res *fpa = NULL; + uint64_t reg; + struct octeontx_mbox_hdr hdr; + struct dcfg_resp resp; + struct octeontx_mbox_fpa_cfg cfg; + int ret = -1; + + fpa = &fpadev.pool[gpool]; + memsz = FPA_ROUND_UP(max_buf_count / fpa->stack_ln_ptr, FPA_LN_SIZE) * + FPA_LN_SIZE; + + /* Round-up to page size */ + memsz = (memsz + FPA_PF_PAGE_SZ - 1) & ~(uintptr_t)(FPA_PF_PAGE_SZ-1); + memptr = rte_malloc(NULL, memsz, RTE_CACHE_LINE_SIZE); + if (memptr == NULL) { + ret = -ENOMEM; + goto err; + } + + /* Configure stack */ + fpa->pool_stack_base = memptr; + phys_addr = rte_malloc_virt2iova(memptr); + + buf_size /= FPA_LN_SIZE; + + /* POOL setup */ + hdr.coproc = FPA_COPROC; + hdr.msg = FPA_CONFIGSET; + hdr.vfid = fpa->vf_id; + hdr.res_code = 0; + + buf_offset /= FPA_LN_SIZE; + reg = POOL_BUF_SIZE(buf_size) | POOL_BUF_OFFSET(buf_offset) | + POOL_LTYPE(0x2) | POOL_STYPE(0) | POOL_SET_NAT_ALIGN | + POOL_ENA; + + cfg.aid = 0; + cfg.pool_cfg = reg; + cfg.pool_stack_base = phys_addr; + cfg.pool_stack_end = phys_addr + memsz; + cfg.aura_cfg = (1 << 9); + + ret = octeontx_ssovf_mbox_send(&hdr, &cfg, + sizeof(struct octeontx_mbox_fpa_cfg), + &resp, sizeof(resp)); + if (ret < 0) { + ret = -EACCES; + goto err; + } + + fpavf_log_dbg(" vfid %d gpool %d aid %d pool_cfg 0x%x pool_stack_base %" PRIx64 " pool_stack_end %" PRIx64" aura_cfg %" PRIx64 "\n", + fpa->vf_id, gpool, cfg.aid, (unsigned int)cfg.pool_cfg, + cfg.pool_stack_base, cfg.pool_stack_end, cfg.aura_cfg); + + /* Now pool is in_use */ + fpa->is_inuse = true; + +err: + if (ret < 0) + rte_free(memptr); + + return ret; +} + +static int +octeontx_fpapf_pool_destroy(unsigned int gpool_index) +{ + struct octeontx_mbox_hdr hdr; + struct dcfg_resp resp; + struct octeontx_mbox_fpa_cfg cfg; + struct fpavf_res *fpa = NULL; + int ret = -1; + + fpa = &fpadev.pool[gpool_index]; + + hdr.coproc = FPA_COPROC; + hdr.msg = FPA_CONFIGSET; + hdr.vfid = fpa->vf_id; + hdr.res_code = 0; + + /* reset and free the pool */ + cfg.aid = 0; + cfg.pool_cfg = 0; + cfg.pool_stack_base = 0; + cfg.pool_stack_end = 0; + cfg.aura_cfg = 0; + + ret = octeontx_ssovf_mbox_send(&hdr, &cfg, + sizeof(struct octeontx_mbox_fpa_cfg), + &resp, sizeof(resp)); + if (ret < 0) { + ret = -EACCES; + goto err; + } + + ret = 0; +err: + /* anycase free pool stack memory */ + rte_free(fpa->pool_stack_base); + fpa->pool_stack_base = NULL; + return ret; +} + +static int +octeontx_fpapf_aura_attach(unsigned int gpool_index) +{ + struct octeontx_mbox_hdr hdr; + struct dcfg_resp resp; + struct octeontx_mbox_fpa_cfg cfg; + int ret = 0; + + if (gpool_index >= FPA_MAX_POOL) { + ret = -EINVAL; + goto err; + } + hdr.coproc = FPA_COPROC; + hdr.msg = FPA_ATTACHAURA; + hdr.vfid = gpool_index; + hdr.res_code = 0; + memset(&cfg, 0x0, sizeof(struct octeontx_mbox_fpa_cfg)); + cfg.aid = gpool_index; /* gpool is guara */ + + ret = octeontx_ssovf_mbox_send(&hdr, &cfg, + sizeof(struct octeontx_mbox_fpa_cfg), + &resp, sizeof(resp)); + if (ret < 0) { + fpavf_log_err("Could not attach fpa "); + fpavf_log_err("aura %d to pool %d. Err=%d. FuncErr=%d\n", + gpool_index, gpool_index, ret, hdr.res_code); + ret = -EACCES; + goto err; + } +err: + return ret; +} + +static int +octeontx_fpapf_aura_detach(unsigned int gpool_index) +{ + struct octeontx_mbox_fpa_cfg cfg = {0}; + struct octeontx_mbox_hdr hdr = {0}; + int ret = 0; + + if (gpool_index >= FPA_MAX_POOL) { + ret = -EINVAL; + goto err; + } + + cfg.aid = gpool_index; /* gpool is gaura */ + hdr.coproc = FPA_COPROC; + hdr.msg = FPA_DETACHAURA; + hdr.vfid = gpool_index; + ret = octeontx_ssovf_mbox_send(&hdr, &cfg, sizeof(cfg), NULL, 0); + if (ret < 0) { + fpavf_log_err("Couldn't detach FPA aura %d Err=%d FuncErr=%d\n", + gpool_index, ret, hdr.res_code); + ret = -EINVAL; + } + +err: + return ret; +} + +int +octeontx_fpavf_pool_set_range(uintptr_t handle, unsigned long memsz, + void *memva, uint16_t gpool) +{ + uint64_t va_end; + + if (unlikely(!handle)) + return -ENODEV; + + va_end = (uintptr_t)memva + memsz; + va_end &= ~RTE_CACHE_LINE_MASK; + + /* VHPOOL setup */ + fpavf_write64((uintptr_t)memva, + (void *)((uintptr_t)handle + + FPA_VF_VHPOOL_START_ADDR(gpool))); + fpavf_write64(va_end, + (void *)((uintptr_t)handle + + FPA_VF_VHPOOL_END_ADDR(gpool))); + return 0; +} + +static int +octeontx_fpapf_start_count(uint16_t gpool_index) +{ + int ret = 0; + struct octeontx_mbox_hdr hdr = {0}; + + if (gpool_index >= FPA_MAX_POOL) { + ret = -EINVAL; + goto err; + } + + hdr.coproc = FPA_COPROC; + hdr.msg = FPA_START_COUNT; + hdr.vfid = gpool_index; + ret = octeontx_ssovf_mbox_send(&hdr, NULL, 0, NULL, 0); + if (ret < 0) { + fpavf_log_err("Could not start buffer counting for "); + fpavf_log_err("FPA pool %d. Err=%d. FuncErr=%d\n", + gpool_index, ret, hdr.res_code); + ret = -EINVAL; + goto err; + } + +err: + return ret; +} + +static __rte_always_inline int +octeontx_fpavf_free(unsigned int gpool) +{ + int ret = 0; + + if (gpool >= FPA_MAX_POOL) { + ret = -EINVAL; + goto err; + } + + /* Pool is free */ + fpadev.pool[gpool].is_inuse = false; + +err: + return ret; +} + +static __rte_always_inline int +octeontx_gpool_free(uint16_t gpool) +{ + if (fpadev.pool[gpool].sz128 != 0) { + fpadev.pool[gpool].sz128 = 0; + return 0; + } + return -EINVAL; +} + +/* + * Return buffer size for a given pool + */ +int +octeontx_fpa_bufpool_block_size(uintptr_t handle) +{ + struct fpavf_res *res = NULL; + uint8_t gpool; + + if (unlikely(!octeontx_fpa_handle_valid(handle))) + return -EINVAL; + + /* get the gpool */ + gpool = octeontx_fpa_bufpool_gpool(handle); + res = &fpadev.pool[gpool]; + return FPA_CACHE_LINE_2_OBJSZ(res->sz128); +} + +int +octeontx_fpa_bufpool_free_count(uintptr_t handle) +{ + uint64_t cnt, limit, avail; + uint8_t gpool; + uintptr_t pool_bar; + + if (unlikely(!octeontx_fpa_handle_valid(handle))) + return -EINVAL; + + /* get the gpool */ + gpool = octeontx_fpa_bufpool_gpool(handle); + + /* Get pool bar address from handle */ + pool_bar = handle & ~(uint64_t)FPA_GPOOL_MASK; + + cnt = fpavf_read64((void *)((uintptr_t)pool_bar + + FPA_VF_VHAURA_CNT(gpool))); + limit = fpavf_read64((void *)((uintptr_t)pool_bar + + FPA_VF_VHAURA_CNT_LIMIT(gpool))); + + avail = fpavf_read64((void *)((uintptr_t)pool_bar + + FPA_VF_VHPOOL_AVAILABLE(gpool))); + + return RTE_MIN(avail, (limit - cnt)); +} + +uintptr_t +octeontx_fpa_bufpool_create(unsigned int object_size, unsigned int object_count, + unsigned int buf_offset, int node_id) +{ + unsigned int gpool; + uintptr_t gpool_handle; + uintptr_t pool_bar; + int res; + + RTE_SET_USED(node_id); + RTE_BUILD_BUG_ON(sizeof(struct rte_mbuf) > OCTEONTX_FPAVF_BUF_OFFSET); + + object_size = RTE_CACHE_LINE_ROUNDUP(object_size); + if (object_size > FPA_MAX_OBJ_SIZE) { + errno = EINVAL; + goto error_end; + } + + rte_spinlock_lock(&fpadev.lock); + res = octeontx_fpa_gpool_alloc(object_size); + + /* Bail if failed */ + if (unlikely(res < 0)) { + errno = res; + goto error_unlock; + } + + /* get fpavf */ + gpool = res; + + /* get pool handle */ + gpool_handle = octeontx_fpa_gpool2handle(gpool); + if (!octeontx_fpa_handle_valid(gpool_handle)) { + errno = ENOSPC; + goto error_gpool_free; + } + + /* Get pool bar address from handle */ + pool_bar = gpool_handle & ~(uint64_t)FPA_GPOOL_MASK; + + res = octeontx_fpapf_pool_setup(gpool, object_size, buf_offset, + object_count); + if (res < 0) { + errno = res; + goto error_gpool_free; + } + + /* populate AURA fields */ + res = octeontx_fpapf_aura_attach(gpool); + if (res < 0) { + errno = res; + goto error_pool_destroy; + } + + /* Release lock */ + rte_spinlock_unlock(&fpadev.lock); + + /* populate AURA registers */ + fpavf_write64(object_count, (void *)((uintptr_t)pool_bar + + FPA_VF_VHAURA_CNT(gpool))); + fpavf_write64(object_count, (void *)((uintptr_t)pool_bar + + FPA_VF_VHAURA_CNT_LIMIT(gpool))); + fpavf_write64(object_count + 1, (void *)((uintptr_t)pool_bar + + FPA_VF_VHAURA_CNT_THRESHOLD(gpool))); + + octeontx_fpapf_start_count(gpool); + + return gpool_handle; + +error_pool_destroy: + octeontx_fpavf_free(gpool); + octeontx_fpapf_pool_destroy(gpool); +error_gpool_free: + octeontx_gpool_free(gpool); +error_unlock: + rte_spinlock_unlock(&fpadev.lock); +error_end: + return (uintptr_t)NULL; +} + +/* + * Destroy a buffer pool. + */ +int +octeontx_fpa_bufpool_destroy(uintptr_t handle, int node_id) +{ + void **node, **curr, *head = NULL; + uint64_t sz; + uint64_t cnt, avail; + uint8_t gpool; + uintptr_t pool_bar; + int ret; + + RTE_SET_USED(node_id); + + /* Wait for all outstanding writes to be committed */ + rte_smp_wmb(); + + if (unlikely(!octeontx_fpa_handle_valid(handle))) + return -EINVAL; + + /* get the pool */ + gpool = octeontx_fpa_bufpool_gpool(handle); + + /* Get pool bar address from handle */ + pool_bar = handle & ~(uint64_t)FPA_GPOOL_MASK; + + /* Check for no outstanding buffers */ + cnt = fpavf_read64((void *)((uintptr_t)pool_bar + + FPA_VF_VHAURA_CNT(gpool))); + if (cnt) { + fpavf_log_dbg("buffer exist in pool cnt %" PRId64 "\n", cnt); + return -EBUSY; + } + + rte_spinlock_lock(&fpadev.lock); + + avail = fpavf_read64((void *)((uintptr_t)pool_bar + + FPA_VF_VHPOOL_AVAILABLE(gpool))); + + /* Prepare to empty the entire POOL */ + fpavf_write64(avail, (void *)((uintptr_t)pool_bar + + FPA_VF_VHAURA_CNT_LIMIT(gpool))); + fpavf_write64(avail + 1, (void *)((uintptr_t)pool_bar + + FPA_VF_VHAURA_CNT_THRESHOLD(gpool))); + + /* Empty the pool */ + /* Invalidate the POOL */ + octeontx_gpool_free(gpool); + + /* Process all buffers in the pool */ + while (avail--) { + + /* Yank a buffer from the pool */ + node = (void *)(uintptr_t) + fpavf_read64((void *) + (pool_bar + FPA_VF_VHAURA_OP_ALLOC(gpool))); + + if (node == NULL) { + fpavf_log_err("GAURA[%u] missing %" PRIx64 " buf\n", + gpool, avail); + break; + } + + /* Imsert it into an ordered linked list */ + for (curr = &head; curr[0] != NULL; curr = curr[0]) { + if ((uintptr_t)node <= (uintptr_t)curr[0]) + break; + } + node[0] = curr[0]; + curr[0] = node; + } + + /* Verify the linked list to be a perfect series */ + sz = octeontx_fpa_bufpool_block_size(handle) << 7; + for (curr = head; curr != NULL && curr[0] != NULL; + curr = curr[0]) { + if (curr == curr[0] || + ((uintptr_t)curr != ((uintptr_t)curr[0] - sz))) { + fpavf_log_err("POOL# %u buf sequence err (%p vs. %p)\n", + gpool, curr, curr[0]); + } + } + + /* Disable pool operation */ + fpavf_write64(~0ul, (void *)((uintptr_t)pool_bar + + FPA_VF_VHPOOL_START_ADDR(gpool))); + fpavf_write64(~0ul, (void *)((uintptr_t)pool_bar + + FPA_VF_VHPOOL_END_ADDR(gpool))); + + (void)octeontx_fpapf_pool_destroy(gpool); + + /* Deactivate the AURA */ + fpavf_write64(0, (void *)((uintptr_t)pool_bar + + FPA_VF_VHAURA_CNT_LIMIT(gpool))); + fpavf_write64(0, (void *)((uintptr_t)pool_bar + + FPA_VF_VHAURA_CNT_THRESHOLD(gpool))); + + ret = octeontx_fpapf_aura_detach(gpool); + if (ret) { + fpavf_log_err("Failed to dettach gaura %u. error code=%d\n", + gpool, ret); + } + + /* Free VF */ + (void)octeontx_fpavf_free(gpool); + + rte_spinlock_unlock(&fpadev.lock); + return 0; +} + +static void +octeontx_fpavf_setup(void) +{ + uint8_t i; + static bool init_once; + + if (!init_once) { + rte_spinlock_init(&fpadev.lock); + fpadev.total_gpool_cnt = 0; + + for (i = 0; i < FPA_VF_MAX; i++) { + + fpadev.pool[i].domain_id = ~0; + fpadev.pool[i].stack_ln_ptr = 0; + fpadev.pool[i].sz128 = 0; + fpadev.pool[i].bar0 = NULL; + fpadev.pool[i].pool_stack_base = NULL; + fpadev.pool[i].is_inuse = false; + } + init_once = 1; + } +} + +static int +octeontx_fpavf_identify(void *bar0) +{ + uint64_t val; + uint16_t domain_id; + uint16_t vf_id; + uint64_t stack_ln_ptr; + + val = fpavf_read64((void *)((uintptr_t)bar0 + + FPA_VF_VHAURA_CNT_THRESHOLD(0))); + + domain_id = (val >> 8) & 0xffff; + vf_id = (val >> 24) & 0xffff; + + stack_ln_ptr = fpavf_read64((void *)((uintptr_t)bar0 + + FPA_VF_VHPOOL_THRESHOLD(0))); + if (vf_id >= FPA_VF_MAX) { + fpavf_log_err("vf_id(%d) greater than max vf (32)\n", vf_id); + return -1; + } + + if (fpadev.pool[vf_id].is_inuse) { + fpavf_log_err("vf_id %d is_inuse\n", vf_id); + return -1; + } + + fpadev.pool[vf_id].domain_id = domain_id; + fpadev.pool[vf_id].vf_id = vf_id; + fpadev.pool[vf_id].bar0 = bar0; + fpadev.pool[vf_id].stack_ln_ptr = stack_ln_ptr; + + /* SUCCESS */ + return vf_id; +} + +/* FPAVF pcie device aka mempool probe */ +static int +fpavf_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) +{ + uint8_t *idreg; + int res; + struct fpavf_res *fpa = NULL; + + RTE_SET_USED(pci_drv); + RTE_SET_USED(fpa); + + /* For secondary processes, the primary has done all the work */ + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return 0; + + if (pci_dev->mem_resource[0].addr == NULL) { + fpavf_log_err("Empty bars %p ", pci_dev->mem_resource[0].addr); + return -ENODEV; + } + idreg = pci_dev->mem_resource[0].addr; + + octeontx_fpavf_setup(); + + res = octeontx_fpavf_identify(idreg); + if (res < 0) + return -1; + + fpa = &fpadev.pool[res]; + fpadev.total_gpool_cnt++; + rte_wmb(); + + fpavf_log_dbg("total_fpavfs %d bar0 %p domain %d vf %d stk_ln_ptr 0x%x", + fpadev.total_gpool_cnt, fpa->bar0, fpa->domain_id, + fpa->vf_id, (unsigned int)fpa->stack_ln_ptr); + + return 0; +} + +static const struct rte_pci_id pci_fpavf_map[] = { + { + RTE_PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, + PCI_DEVICE_ID_OCTEONTX_FPA_VF) + }, + { + .vendor_id = 0, + }, +}; + +static struct rte_pci_driver pci_fpavf = { + .id_table = pci_fpavf_map, + .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_IOVA_AS_VA, + .probe = fpavf_probe, +}; + +RTE_PMD_REGISTER_PCI(octeontx_fpavf, pci_fpavf); diff --git a/dpdk/drivers/mempool/octeontx/octeontx_fpavf.h b/dpdk/drivers/mempool/octeontx/octeontx_fpavf.h new file mode 100644 index 00000000..bc5dc3ba --- /dev/null +++ b/dpdk/drivers/mempool/octeontx/octeontx_fpavf.h @@ -0,0 +1,133 @@ +/* + * BSD LICENSE + * + * Copyright (C) 2017 Cavium Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium networks nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __OCTEONTX_FPAVF_H__ +#define __OCTEONTX_FPAVF_H__ + +#include +#include "octeontx_pool_logs.h" + +/* fpa pool Vendor ID and Device ID */ +#define PCI_VENDOR_ID_CAVIUM 0x177D +#define PCI_DEVICE_ID_OCTEONTX_FPA_VF 0xA053 + +#define FPA_VF_MAX 32 +#define FPA_GPOOL_MASK (FPA_VF_MAX-1) + +/* FPA VF register offsets */ +#define FPA_VF_INT(x) (0x200ULL | ((x) << 22)) +#define FPA_VF_INT_W1S(x) (0x210ULL | ((x) << 22)) +#define FPA_VF_INT_ENA_W1S(x) (0x220ULL | ((x) << 22)) +#define FPA_VF_INT_ENA_W1C(x) (0x230ULL | ((x) << 22)) + +#define FPA_VF_VHPOOL_AVAILABLE(vhpool) (0x04150 | ((vhpool)&0x0)) +#define FPA_VF_VHPOOL_THRESHOLD(vhpool) (0x04160 | ((vhpool)&0x0)) +#define FPA_VF_VHPOOL_START_ADDR(vhpool) (0x04200 | ((vhpool)&0x0)) +#define FPA_VF_VHPOOL_END_ADDR(vhpool) (0x04210 | ((vhpool)&0x0)) + +#define FPA_VF_VHAURA_CNT(vaura) (0x20120 | ((vaura)&0xf)<<18) +#define FPA_VF_VHAURA_CNT_ADD(vaura) (0x20128 | ((vaura)&0xf)<<18) +#define FPA_VF_VHAURA_CNT_LIMIT(vaura) (0x20130 | ((vaura)&0xf)<<18) +#define FPA_VF_VHAURA_CNT_THRESHOLD(vaura) (0x20140 | ((vaura)&0xf)<<18) +#define FPA_VF_VHAURA_OP_ALLOC(vaura) (0x30000 | ((vaura)&0xf)<<18) +#define FPA_VF_VHAURA_OP_FREE(vaura) (0x38000 | ((vaura)&0xf)<<18) + +#define FPA_VF_FREE_ADDRS_S(x, y, z) \ + ((x) | (((y) & 0x1ff) << 3) | ((((z) & 1)) << 14)) + +/* FPA VF register offsets from VF_BAR4, size 2 MByte */ +#define FPA_VF_MSIX_VEC_ADDR 0x00000 +#define FPA_VF_MSIX_VEC_CTL 0x00008 +#define FPA_VF_MSIX_PBA 0xF0000 + +#define FPA_VF0_APERTURE_SHIFT 22 +#define FPA_AURA_SET_SIZE 16 + +#define FPA_MAX_OBJ_SIZE (128 * 1024) +#define OCTEONTX_FPAVF_BUF_OFFSET 128 + +/* + * In Cavium OcteonTX SoC, all accesses to the device registers are + * implicitly strongly ordered. So, the relaxed version of IO operation is + * safe to use with out any IO memory barriers. + */ +#define fpavf_read64 rte_read64_relaxed +#define fpavf_write64 rte_write64_relaxed + +/* ARM64 specific functions */ +#if defined(RTE_ARCH_ARM64) +#define fpavf_load_pair(val0, val1, addr) ({ \ + asm volatile( \ + "ldp %x[x0], %x[x1], [%x[p1]]" \ + :[x0]"=r"(val0), [x1]"=r"(val1) \ + :[p1]"r"(addr) \ + ); }) + +#define fpavf_store_pair(val0, val1, addr) ({ \ + asm volatile( \ + "stp %x[x0], %x[x1], [%x[p1]]" \ + ::[x0]"r"(val0), [x1]"r"(val1), [p1]"r"(addr) \ + ); }) +#else /* Un optimized functions for building on non arm64 arch */ + +#define fpavf_load_pair(val0, val1, addr) \ +do { \ + val0 = rte_read64(addr); \ + val1 = rte_read64(((uint8_t *)addr) + 8); \ +} while (0) + +#define fpavf_store_pair(val0, val1, addr) \ +do { \ + rte_write64(val0, addr); \ + rte_write64(val1, (((uint8_t *)addr) + 8)); \ +} while (0) +#endif + +uintptr_t +octeontx_fpa_bufpool_create(unsigned int object_size, unsigned int object_count, + unsigned int buf_offset, int node); +int +octeontx_fpavf_pool_set_range(uintptr_t handle, unsigned long memsz, + void *memva, uint16_t gpool); +int +octeontx_fpa_bufpool_destroy(uintptr_t handle, int node); +int +octeontx_fpa_bufpool_block_size(uintptr_t handle); +int +octeontx_fpa_bufpool_free_count(uintptr_t handle); + +static __rte_always_inline uint8_t +octeontx_fpa_bufpool_gpool(uintptr_t handle) +{ + return (uint8_t)handle & FPA_GPOOL_MASK; +} +#endif /* __OCTEONTX_FPAVF_H__ */ diff --git a/dpdk/drivers/mempool/octeontx/octeontx_mbox.c b/dpdk/drivers/mempool/octeontx/octeontx_mbox.c new file mode 100644 index 00000000..a38bfe6e --- /dev/null +++ b/dpdk/drivers/mempool/octeontx/octeontx_mbox.c @@ -0,0 +1,242 @@ +/* + * BSD LICENSE + * + * Copyright (C) Cavium, Inc. 2017. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +#include +#include +#include +#include +#include + +#include "octeontx_mbox.h" +#include "octeontx_pool_logs.h" + +/* Mbox operation timeout in seconds */ +#define MBOX_WAIT_TIME_SEC 3 +#define MAX_RAM_MBOX_LEN ((SSOW_BAR4_LEN >> 1) - 8 /* Mbox header */) + +/* Mbox channel state */ +enum { + MBOX_CHAN_STATE_REQ = 1, + MBOX_CHAN_STATE_RES = 0, +}; + +/* Response messages */ +enum { + MBOX_RET_SUCCESS, + MBOX_RET_INVALID, + MBOX_RET_INTERNAL_ERR, +}; + +struct mbox { + int init_once; + uint8_t *ram_mbox_base; /* Base address of mbox message stored in ram */ + uint8_t *reg; /* Store to this register triggers PF mbox interrupt */ + uint16_t tag_own; /* Last tag which was written to own channel */ + rte_spinlock_t lock; +}; + +static struct mbox octeontx_mbox; + +/* + * Structure used for mbox synchronization + * This structure sits at the begin of Mbox RAM and used as main + * synchronization point for channel communication + */ +struct mbox_ram_hdr { + union { + uint64_t u64; + struct { + uint8_t chan_state : 1; + uint8_t coproc : 7; + uint8_t msg; + uint8_t vfid; + uint8_t res_code; + uint16_t tag; + uint16_t len; + }; + }; +}; + +static inline void +mbox_msgcpy(volatile uint8_t *d, volatile const uint8_t *s, uint16_t size) +{ + uint16_t i; + + for (i = 0; i < size; i++) + d[i] = s[i]; +} + +static inline void +mbox_send_request(struct mbox *m, struct octeontx_mbox_hdr *hdr, + const void *txmsg, uint16_t txsize) +{ + struct mbox_ram_hdr old_hdr; + struct mbox_ram_hdr new_hdr = { {0} }; + uint64_t *ram_mbox_hdr = (uint64_t *)m->ram_mbox_base; + uint8_t *ram_mbox_msg = m->ram_mbox_base + sizeof(struct mbox_ram_hdr); + + /* + * Initialize the channel with the tag left by last send. + * On success full mbox send complete, PF increments the tag by one. + * The sender can validate integrity of PF message with this scheme + */ + old_hdr.u64 = rte_read64(ram_mbox_hdr); + m->tag_own = (old_hdr.tag + 2) & (~0x1ul); /* next even number */ + + /* Copy msg body */ + if (txmsg) + mbox_msgcpy(ram_mbox_msg, txmsg, txsize); + + /* Prepare new hdr */ + new_hdr.chan_state = MBOX_CHAN_STATE_REQ; + new_hdr.coproc = hdr->coproc; + new_hdr.msg = hdr->msg; + new_hdr.vfid = hdr->vfid; + new_hdr.tag = m->tag_own; + new_hdr.len = txsize; + + /* Write the msg header */ + rte_write64(new_hdr.u64, ram_mbox_hdr); + rte_smp_wmb(); + /* Notify PF about the new msg - write to MBOX reg generates PF IRQ */ + rte_write64(0, m->reg); +} + +static inline int +mbox_wait_response(struct mbox *m, struct octeontx_mbox_hdr *hdr, + void *rxmsg, uint16_t rxsize) +{ + int res = 0, wait; + uint16_t len; + struct mbox_ram_hdr rx_hdr; + uint64_t *ram_mbox_hdr = (uint64_t *)m->ram_mbox_base; + uint8_t *ram_mbox_msg = m->ram_mbox_base + sizeof(struct mbox_ram_hdr); + + /* Wait for response */ + wait = MBOX_WAIT_TIME_SEC * 1000 * 10; + while (wait > 0) { + rte_delay_us(100); + rx_hdr.u64 = rte_read64(ram_mbox_hdr); + if (rx_hdr.chan_state == MBOX_CHAN_STATE_RES) + break; + --wait; + } + + hdr->res_code = rx_hdr.res_code; + m->tag_own++; + + /* Timeout */ + if (wait <= 0) { + res = -ETIMEDOUT; + goto error; + } + + /* Tag mismatch */ + if (m->tag_own != rx_hdr.tag) { + res = -EINVAL; + goto error; + } + + /* PF nacked the msg */ + if (rx_hdr.res_code != MBOX_RET_SUCCESS) { + res = -EBADMSG; + goto error; + } + + len = RTE_MIN(rx_hdr.len, rxsize); + if (rxmsg) + mbox_msgcpy(rxmsg, ram_mbox_msg, len); + + return len; + +error: + mbox_log_err("Failed to send mbox(%d/%d) coproc=%d msg=%d ret=(%d,%d)", + m->tag_own, rx_hdr.tag, hdr->coproc, hdr->msg, res, + hdr->res_code); + return res; +} + +static inline int +mbox_send(struct mbox *m, struct octeontx_mbox_hdr *hdr, const void *txmsg, + uint16_t txsize, void *rxmsg, uint16_t rxsize) +{ + int res = -EINVAL; + + if (m->init_once == 0 || hdr == NULL || + txsize > MAX_RAM_MBOX_LEN || rxsize > MAX_RAM_MBOX_LEN) { + mbox_log_err("Invalid init_once=%d hdr=%p txsz=%d rxsz=%d", + m->init_once, hdr, txsize, rxsize); + return res; + } + + rte_spinlock_lock(&m->lock); + + mbox_send_request(m, hdr, txmsg, txsize); + res = mbox_wait_response(m, hdr, rxmsg, rxsize); + + rte_spinlock_unlock(&m->lock); + return res; +} + +static inline int +mbox_setup(struct mbox *m) +{ + if (unlikely(m->init_once == 0)) { + rte_spinlock_init(&m->lock); + m->ram_mbox_base = octeontx_ssovf_bar(OCTEONTX_SSO_HWS, 0, 4); + m->reg = octeontx_ssovf_bar(OCTEONTX_SSO_GROUP, 0, 0); + m->reg += SSO_VHGRP_PF_MBOX(1); + + if (m->ram_mbox_base == NULL || m->reg == NULL) { + mbox_log_err("Invalid ram_mbox_base=%p or reg=%p", + m->ram_mbox_base, m->reg); + return -EINVAL; + } + m->init_once = 1; + } + return 0; +} + +int +octeontx_ssovf_mbox_send(struct octeontx_mbox_hdr *hdr, void *txdata, + uint16_t txlen, void *rxdata, uint16_t rxlen) +{ + struct mbox *m = &octeontx_mbox; + + RTE_BUILD_BUG_ON(sizeof(struct mbox_ram_hdr) != 8); + if (rte_eal_process_type() != RTE_PROC_PRIMARY || mbox_setup(m)) + return -EINVAL; + + return mbox_send(m, hdr, txdata, txlen, rxdata, rxlen); +} diff --git a/dpdk/drivers/mempool/octeontx/octeontx_mbox.h b/dpdk/drivers/mempool/octeontx/octeontx_mbox.h new file mode 100644 index 00000000..49f38257 --- /dev/null +++ b/dpdk/drivers/mempool/octeontx/octeontx_mbox.h @@ -0,0 +1,64 @@ +/* + * BSD LICENSE + * + * Copyright (C) Cavium, Inc. 2017. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __OCTEONTX_MBOX_H__ +#define __OCTEONTX_MBOX_H__ + +#include + +#define SSOW_BAR4_LEN (64 * 1024) +#define SSO_VHGRP_PF_MBOX(x) (0x200ULL | ((x) << 3)) + +struct octeontx_ssovf_info { + uint16_t domain; /* Domain id */ + uint8_t total_ssovfs; /* Total sso groups available in domain */ + uint8_t total_ssowvfs;/* Total sso hws available in domain */ +}; + +enum octeontx_ssovf_type { + OCTEONTX_SSO_GROUP, /* SSO group vf */ + OCTEONTX_SSO_HWS, /* SSO hardware workslot vf */ +}; + +struct octeontx_mbox_hdr { + uint16_t vfid; /* VF index or pf resource index local to the domain */ + uint8_t coproc; /* Coprocessor id */ + uint8_t msg; /* Message id */ + uint8_t res_code; /* Functional layer response code */ +}; + +int octeontx_ssovf_info(struct octeontx_ssovf_info *info); +void *octeontx_ssovf_bar(enum octeontx_ssovf_type, uint8_t id, uint8_t bar); +int octeontx_ssovf_mbox_send(struct octeontx_mbox_hdr *hdr, + void *txdata, uint16_t txlen, void *rxdata, uint16_t rxlen); + +#endif /* __OCTEONTX_MBOX_H__ */ diff --git a/dpdk/drivers/mempool/octeontx/octeontx_pool_logs.h b/dpdk/drivers/mempool/octeontx/octeontx_pool_logs.h new file mode 100644 index 00000000..58ccb0f0 --- /dev/null +++ b/dpdk/drivers/mempool/octeontx/octeontx_pool_logs.h @@ -0,0 +1,68 @@ +/* + * BSD LICENSE + * + * Copyright (C) 2017 Cavium Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium networks nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __OCTEONTX_POOL_LOGS_H__ +#define __OCTEONTX_POOL_LOGS_H__ + +#include + +#ifdef RTE_LIBRTE_OCTEONTX_MEMPOOL_DEBUG +#define fpavf_log_info(fmt, args...) \ + RTE_LOG(INFO, PMD, "%s() line %u: " fmt "\n", \ + __func__, __LINE__, ## args) +#define fpavf_log_dbg(fmt, args...) \ + RTE_LOG(DEBUG, PMD, "%s() line %u: " fmt "\n", \ + __func__, __LINE__, ## args) + +#define mbox_log_info(fmt, args...) \ + RTE_LOG(INFO, PMD, "%s() line %u: " fmt "\n", \ + __func__, __LINE__, ## args) +#define mbox_log_dbg(fmt, args...) \ + RTE_LOG(DEBUG, PMD, "%s() line %u: " fmt "\n", \ + __func__, __LINE__, ## args) +#else +#define fpavf_log_info(fmt, args...) +#define fpavf_log_dbg(fmt, args...) +#define mbox_log_info(fmt, args...) +#define mbox_log_dbg(fmt, args...) +#endif + +#define fpavf_func_trace fpavf_log_dbg +#define fpavf_log_err(fmt, args...) \ + RTE_LOG(ERR, PMD, "%s() line %u: " fmt "\n", \ + __func__, __LINE__, ## args) +#define mbox_func_trace mbox_log_dbg +#define mbox_log_err(fmt, args...) \ + RTE_LOG(ERR, PMD, "%s() line %u: " fmt "\n", \ + __func__, __LINE__, ## args) + +#endif /* __OCTEONTX_POOL_LOGS_H__*/ diff --git a/dpdk/drivers/mempool/octeontx/octeontx_ssovf.c b/dpdk/drivers/mempool/octeontx/octeontx_ssovf.c new file mode 100644 index 00000000..012c887d --- /dev/null +++ b/dpdk/drivers/mempool/octeontx/octeontx_ssovf.c @@ -0,0 +1,299 @@ +/* + * BSD LICENSE + * + * Copyright (C) Cavium, Inc. 2017. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include + +#include "octeontx_mbox.h" +#include "octeontx_pool_logs.h" + +#define PCI_VENDOR_ID_CAVIUM 0x177D +#define PCI_DEVICE_ID_OCTEONTX_SSOGRP_VF 0xA04B +#define PCI_DEVICE_ID_OCTEONTX_SSOWS_VF 0xA04D + +#define SSO_MAX_VHGRP (64) +#define SSO_MAX_VHWS (32) + +#define SSO_VHGRP_AQ_THR (0x1E0ULL) + +struct ssovf_res { + uint16_t domain; + uint16_t vfid; + void *bar0; + void *bar2; +}; + +struct ssowvf_res { + uint16_t domain; + uint16_t vfid; + void *bar0; + void *bar2; + void *bar4; +}; + +struct ssowvf_identify { + uint16_t domain; + uint16_t vfid; +}; + +struct ssodev { + uint8_t total_ssovfs; + uint8_t total_ssowvfs; + struct ssovf_res grp[SSO_MAX_VHGRP]; + struct ssowvf_res hws[SSO_MAX_VHWS]; +}; + +static struct ssodev sdev; + +/* Interface functions */ +int +octeontx_ssovf_info(struct octeontx_ssovf_info *info) +{ + uint8_t i; + uint16_t domain; + + if (rte_eal_process_type() != RTE_PROC_PRIMARY || info == NULL) + return -EINVAL; + + if (sdev.total_ssovfs == 0 || sdev.total_ssowvfs == 0) + return -ENODEV; + + domain = sdev.grp[0].domain; + for (i = 0; i < sdev.total_ssovfs; i++) { + /* Check vfid's are contiguous and belong to same domain */ + if (sdev.grp[i].vfid != i || + sdev.grp[i].bar0 == NULL || + sdev.grp[i].domain != domain) { + mbox_log_err("GRP error, vfid=%d/%d domain=%d/%d %p", + i, sdev.grp[i].vfid, + domain, sdev.grp[i].domain, + sdev.grp[i].bar0); + return -EINVAL; + } + } + + for (i = 0; i < sdev.total_ssowvfs; i++) { + /* Check vfid's are contiguous and belong to same domain */ + if (sdev.hws[i].vfid != i || + sdev.hws[i].bar0 == NULL || + sdev.hws[i].domain != domain) { + mbox_log_err("HWS error, vfid=%d/%d domain=%d/%d %p", + i, sdev.hws[i].vfid, + domain, sdev.hws[i].domain, + sdev.hws[i].bar0); + return -EINVAL; + } + } + + info->domain = domain; + info->total_ssovfs = sdev.total_ssovfs; + info->total_ssowvfs = sdev.total_ssowvfs; + return 0; +} + +void* +octeontx_ssovf_bar(enum octeontx_ssovf_type type, uint8_t id, uint8_t bar) +{ + if (rte_eal_process_type() != RTE_PROC_PRIMARY || + type > OCTEONTX_SSO_HWS) + return NULL; + + if (type == OCTEONTX_SSO_GROUP) { + if (id >= sdev.total_ssovfs) + return NULL; + } else { + if (id >= sdev.total_ssowvfs) + return NULL; + } + + if (type == OCTEONTX_SSO_GROUP) { + switch (bar) { + case 0: + return sdev.grp[id].bar0; + case 2: + return sdev.grp[id].bar2; + default: + return NULL; + } + } else { + switch (bar) { + case 0: + return sdev.hws[id].bar0; + case 2: + return sdev.hws[id].bar2; + case 4: + return sdev.hws[id].bar4; + default: + return NULL; + } + } +} + +/* SSOWVF pcie device aka event port probe */ + +static int +ssowvf_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) +{ + uint16_t vfid; + struct ssowvf_res *res; + struct ssowvf_identify *id; + + RTE_SET_USED(pci_drv); + + /* For secondary processes, the primary has done all the work */ + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return 0; + + if (pci_dev->mem_resource[0].addr == NULL || + pci_dev->mem_resource[2].addr == NULL || + pci_dev->mem_resource[4].addr == NULL) { + mbox_log_err("Empty bars %p %p %p", + pci_dev->mem_resource[0].addr, + pci_dev->mem_resource[2].addr, + pci_dev->mem_resource[4].addr); + return -ENODEV; + } + + if (pci_dev->mem_resource[4].len != SSOW_BAR4_LEN) { + mbox_log_err("Bar4 len mismatch %d != %d", + SSOW_BAR4_LEN, (int)pci_dev->mem_resource[4].len); + return -EINVAL; + } + + id = pci_dev->mem_resource[4].addr; + vfid = id->vfid; + if (vfid >= SSO_MAX_VHWS) { + mbox_log_err("Invalid vfid(%d/%d)", vfid, SSO_MAX_VHWS); + return -EINVAL; + } + + res = &sdev.hws[vfid]; + res->vfid = vfid; + res->bar0 = pci_dev->mem_resource[0].addr; + res->bar2 = pci_dev->mem_resource[2].addr; + res->bar4 = pci_dev->mem_resource[4].addr; + res->domain = id->domain; + + sdev.total_ssowvfs++; + rte_wmb(); + mbox_log_dbg("Domain=%d hws=%d total_ssowvfs=%d", res->domain, + res->vfid, sdev.total_ssowvfs); + return 0; +} + +static const struct rte_pci_id pci_ssowvf_map[] = { + { + RTE_PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, + PCI_DEVICE_ID_OCTEONTX_SSOWS_VF) + }, + { + .vendor_id = 0, + }, +}; + +static struct rte_pci_driver pci_ssowvf = { + .id_table = pci_ssowvf_map, + .drv_flags = RTE_PCI_DRV_NEED_MAPPING, + .probe = ssowvf_probe, +}; + +RTE_PMD_REGISTER_PCI(octeontx_ssowvf, pci_ssowvf); + +/* SSOVF pcie device aka event queue probe */ + +static int +ssovf_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) +{ + uint64_t val; + uint16_t vfid; + uint8_t *idreg; + struct ssovf_res *res; + + RTE_SET_USED(pci_drv); + + /* For secondary processes, the primary has done all the work */ + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return 0; + + if (pci_dev->mem_resource[0].addr == NULL || + pci_dev->mem_resource[2].addr == NULL) { + mbox_log_err("Empty bars %p %p", + pci_dev->mem_resource[0].addr, + pci_dev->mem_resource[2].addr); + return -ENODEV; + } + idreg = pci_dev->mem_resource[0].addr; + idreg += SSO_VHGRP_AQ_THR; + val = rte_read64(idreg); + + /* Write back the default value of aq_thr */ + rte_write64((1ULL << 33) - 1, idreg); + vfid = (val >> 16) & 0xffff; + if (vfid >= SSO_MAX_VHGRP) { + mbox_log_err("Invalid vfid (%d/%d)", vfid, SSO_MAX_VHGRP); + return -EINVAL; + } + + res = &sdev.grp[vfid]; + res->vfid = vfid; + res->bar0 = pci_dev->mem_resource[0].addr; + res->bar2 = pci_dev->mem_resource[2].addr; + res->domain = val & 0xffff; + + sdev.total_ssovfs++; + rte_wmb(); + mbox_log_dbg("Domain=%d group=%d total_ssovfs=%d", res->domain, + res->vfid, sdev.total_ssovfs); + return 0; +} + +static const struct rte_pci_id pci_ssovf_map[] = { + { + RTE_PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, + PCI_DEVICE_ID_OCTEONTX_SSOGRP_VF) + }, + { + .vendor_id = 0, + }, +}; + +static struct rte_pci_driver pci_ssovf = { + .id_table = pci_ssovf_map, + .drv_flags = RTE_PCI_DRV_NEED_MAPPING, + .probe = ssovf_probe, +}; + +RTE_PMD_REGISTER_PCI(octeontx_ssovf, pci_ssovf); diff --git a/dpdk/drivers/mempool/octeontx/rte_mempool_octeontx.c b/dpdk/drivers/mempool/octeontx/rte_mempool_octeontx.c new file mode 100644 index 00000000..fc0cab94 --- /dev/null +++ b/dpdk/drivers/mempool/octeontx/rte_mempool_octeontx.c @@ -0,0 +1,191 @@ +/* + * BSD LICENSE + * + * Copyright (C) 2017 Cavium Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include +#include +#include +#include + +#include "octeontx_fpavf.h" + +static int +octeontx_fpavf_alloc(struct rte_mempool *mp) +{ + uintptr_t pool; + uint32_t memseg_count = mp->size; + uint32_t object_size; + int rc = 0; + + object_size = mp->elt_size + mp->header_size + mp->trailer_size; + + pool = octeontx_fpa_bufpool_create(object_size, memseg_count, + OCTEONTX_FPAVF_BUF_OFFSET, + mp->socket_id); + rc = octeontx_fpa_bufpool_block_size(pool); + if (rc < 0) + goto _end; + + if ((uint32_t)rc != object_size) + fpavf_log_err("buffer size mismatch: %d instead of %u\n", + rc, object_size); + + fpavf_log_info("Pool created %p with .. ", (void *)pool); + fpavf_log_info("obj_sz %d, cnt %d\n", object_size, memseg_count); + + /* assign pool handle to mempool */ + mp->pool_id = (uint64_t)pool; + + return 0; + +_end: + return rc; +} + +static void +octeontx_fpavf_free(struct rte_mempool *mp) +{ + uintptr_t pool; + pool = (uintptr_t)mp->pool_id; + + octeontx_fpa_bufpool_destroy(pool, mp->socket_id); +} + +static __rte_always_inline void * +octeontx_fpa_bufpool_alloc(uintptr_t handle) +{ + return (void *)(uintptr_t)fpavf_read64((void *)(handle + + FPA_VF_VHAURA_OP_ALLOC(0))); +} + +static __rte_always_inline void +octeontx_fpa_bufpool_free(uintptr_t handle, void *buf) +{ + uint64_t free_addr = FPA_VF_FREE_ADDRS_S(FPA_VF_VHAURA_OP_FREE(0), + 0 /* DWB */, 1 /* FABS */); + + fpavf_write64((uintptr_t)buf, (void *)(uintptr_t)(handle + free_addr)); +} + +static int +octeontx_fpavf_enqueue(struct rte_mempool *mp, void * const *obj_table, + unsigned int n) +{ + uintptr_t pool; + unsigned int index; + + pool = (uintptr_t)mp->pool_id; + /* Get pool bar address from handle */ + pool &= ~(uint64_t)FPA_GPOOL_MASK; + for (index = 0; index < n; index++, obj_table++) + octeontx_fpa_bufpool_free(pool, *obj_table); + + return 0; +} + +static int +octeontx_fpavf_dequeue(struct rte_mempool *mp, void **obj_table, + unsigned int n) +{ + unsigned int index; + uintptr_t pool; + void *obj; + + pool = (uintptr_t)mp->pool_id; + /* Get pool bar address from handle */ + pool &= ~(uint64_t)FPA_GPOOL_MASK; + for (index = 0; index < n; index++, obj_table++) { + obj = octeontx_fpa_bufpool_alloc(pool); + if (obj == NULL) { + /* + * Failed to allocate the requested number of objects + * from the pool. Current pool implementation requires + * completing the entire request or returning error + * otherwise. + * Free already allocated buffers to the pool. + */ + for (; index > 0; index--) { + obj_table--; + octeontx_fpa_bufpool_free(pool, *obj_table); + } + return -ENOMEM; + } + *obj_table = obj; + } + + return 0; +} + +static unsigned int +octeontx_fpavf_get_count(const struct rte_mempool *mp) +{ + uintptr_t pool; + + pool = (uintptr_t)mp->pool_id; + + return octeontx_fpa_bufpool_free_count(pool); +} + +static int +octeontx_fpavf_get_capabilities(const struct rte_mempool *mp, + unsigned int *flags) +{ + RTE_SET_USED(mp); + *flags |= (MEMPOOL_F_CAPA_PHYS_CONTIG | + MEMPOOL_F_CAPA_BLK_ALIGNED_OBJECTS); + return 0; +} + +static int +octeontx_fpavf_register_memory_area(const struct rte_mempool *mp, + char *vaddr, rte_iova_t paddr, size_t len) +{ + RTE_SET_USED(paddr); + uint8_t gpool; + uintptr_t pool_bar; + + gpool = octeontx_fpa_bufpool_gpool(mp->pool_id); + pool_bar = mp->pool_id & ~(uint64_t)FPA_GPOOL_MASK; + + return octeontx_fpavf_pool_set_range(pool_bar, len, vaddr, gpool); +} + +static struct rte_mempool_ops octeontx_fpavf_ops = { + .name = "octeontx_fpavf", + .alloc = octeontx_fpavf_alloc, + .free = octeontx_fpavf_free, + .enqueue = octeontx_fpavf_enqueue, + .dequeue = octeontx_fpavf_dequeue, + .get_count = octeontx_fpavf_get_count, + .get_capabilities = octeontx_fpavf_get_capabilities, + .register_memory_area = octeontx_fpavf_register_memory_area, +}; + +MEMPOOL_REGISTER_OPS(octeontx_fpavf_ops); diff --git a/dpdk/drivers/mempool/octeontx/rte_mempool_octeontx_version.map b/dpdk/drivers/mempool/octeontx/rte_mempool_octeontx_version.map new file mode 100644 index 00000000..fe8cdeca --- /dev/null +++ b/dpdk/drivers/mempool/octeontx/rte_mempool_octeontx_version.map @@ -0,0 +1,9 @@ +DPDK_17.11 { + global: + + octeontx_ssovf_info; + octeontx_ssovf_bar; + octeontx_ssovf_mbox_send; + + local: *; +}; diff --git a/dpdk/drivers/mempool/ring/Makefile b/dpdk/drivers/mempool/ring/Makefile new file mode 100644 index 00000000..a7889b96 --- /dev/null +++ b/dpdk/drivers/mempool/ring/Makefile @@ -0,0 +1,48 @@ +# BSD LICENSE +# +# Copyright 2017 NXP. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of NXP nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +include $(RTE_SDK)/mk/rte.vars.mk + +# +# library name +# +LIB = librte_mempool_ring.a + +CFLAGS += -O3 +CFLAGS += $(WERROR_FLAGS) +LDLIBS += -lrte_eal -lrte_mempool -lrte_ring + +EXPORT_MAP := rte_mempool_ring_version.map + +LIBABIVER := 1 + +SRCS-$(CONFIG_RTE_DRIVER_MEMPOOL_RING) += rte_mempool_ring.c + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/mempool/ring/rte_mempool_ring.c b/dpdk/drivers/mempool/ring/rte_mempool_ring.c new file mode 100644 index 00000000..5c132bf6 --- /dev/null +++ b/dpdk/drivers/mempool/ring/rte_mempool_ring.c @@ -0,0 +1,165 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include + +#include +#include +#include + +static int +common_ring_mp_enqueue(struct rte_mempool *mp, void * const *obj_table, + unsigned n) +{ + return rte_ring_mp_enqueue_bulk(mp->pool_data, + obj_table, n, NULL) == 0 ? -ENOBUFS : 0; +} + +static int +common_ring_sp_enqueue(struct rte_mempool *mp, void * const *obj_table, + unsigned n) +{ + return rte_ring_sp_enqueue_bulk(mp->pool_data, + obj_table, n, NULL) == 0 ? -ENOBUFS : 0; +} + +static int +common_ring_mc_dequeue(struct rte_mempool *mp, void **obj_table, unsigned n) +{ + return rte_ring_mc_dequeue_bulk(mp->pool_data, + obj_table, n, NULL) == 0 ? -ENOBUFS : 0; +} + +static int +common_ring_sc_dequeue(struct rte_mempool *mp, void **obj_table, unsigned n) +{ + return rte_ring_sc_dequeue_bulk(mp->pool_data, + obj_table, n, NULL) == 0 ? -ENOBUFS : 0; +} + +static unsigned +common_ring_get_count(const struct rte_mempool *mp) +{ + return rte_ring_count(mp->pool_data); +} + + +static int +common_ring_alloc(struct rte_mempool *mp) +{ + int rg_flags = 0, ret; + char rg_name[RTE_RING_NAMESIZE]; + struct rte_ring *r; + + ret = snprintf(rg_name, sizeof(rg_name), + RTE_MEMPOOL_MZ_FORMAT, mp->name); + if (ret < 0 || ret >= (int)sizeof(rg_name)) { + rte_errno = ENAMETOOLONG; + return -rte_errno; + } + + /* ring flags */ + if (mp->flags & MEMPOOL_F_SP_PUT) + rg_flags |= RING_F_SP_ENQ; + if (mp->flags & MEMPOOL_F_SC_GET) + rg_flags |= RING_F_SC_DEQ; + + /* + * Allocate the ring that will be used to store objects. + * Ring functions will return appropriate errors if we are + * running as a secondary process etc., so no checks made + * in this function for that condition. + */ + r = rte_ring_create(rg_name, rte_align32pow2(mp->size + 1), + mp->socket_id, rg_flags); + if (r == NULL) + return -rte_errno; + + mp->pool_data = r; + + return 0; +} + +static void +common_ring_free(struct rte_mempool *mp) +{ + rte_ring_free(mp->pool_data); +} + +/* + * The following 4 declarations of mempool ops structs address + * the need for the backward compatible mempool handlers for + * single/multi producers and single/multi consumers as dictated by the + * flags provided to the rte_mempool_create function + */ +static const struct rte_mempool_ops ops_mp_mc = { + .name = "ring_mp_mc", + .alloc = common_ring_alloc, + .free = common_ring_free, + .enqueue = common_ring_mp_enqueue, + .dequeue = common_ring_mc_dequeue, + .get_count = common_ring_get_count, +}; + +static const struct rte_mempool_ops ops_sp_sc = { + .name = "ring_sp_sc", + .alloc = common_ring_alloc, + .free = common_ring_free, + .enqueue = common_ring_sp_enqueue, + .dequeue = common_ring_sc_dequeue, + .get_count = common_ring_get_count, +}; + +static const struct rte_mempool_ops ops_mp_sc = { + .name = "ring_mp_sc", + .alloc = common_ring_alloc, + .free = common_ring_free, + .enqueue = common_ring_mp_enqueue, + .dequeue = common_ring_sc_dequeue, + .get_count = common_ring_get_count, +}; + +static const struct rte_mempool_ops ops_sp_mc = { + .name = "ring_sp_mc", + .alloc = common_ring_alloc, + .free = common_ring_free, + .enqueue = common_ring_sp_enqueue, + .dequeue = common_ring_mc_dequeue, + .get_count = common_ring_get_count, +}; + +MEMPOOL_REGISTER_OPS(ops_mp_mc); +MEMPOOL_REGISTER_OPS(ops_sp_sc); +MEMPOOL_REGISTER_OPS(ops_mp_sc); +MEMPOOL_REGISTER_OPS(ops_sp_mc); diff --git a/dpdk/drivers/mempool/ring/rte_mempool_ring_version.map b/dpdk/drivers/mempool/ring/rte_mempool_ring_version.map new file mode 100644 index 00000000..8591cc0b --- /dev/null +++ b/dpdk/drivers/mempool/ring/rte_mempool_ring_version.map @@ -0,0 +1,4 @@ +DPDK_17.05 { + + local: *; +}; diff --git a/dpdk/drivers/mempool/stack/Makefile b/dpdk/drivers/mempool/stack/Makefile new file mode 100644 index 00000000..f8d6c574 --- /dev/null +++ b/dpdk/drivers/mempool/stack/Makefile @@ -0,0 +1,51 @@ +# BSD LICENSE +# +# Copyright 2017 NXP. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of NXP nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +include $(RTE_SDK)/mk/rte.vars.mk + +# +# library name +# +LIB = librte_mempool_stack.a + +CFLAGS += -O3 +CFLAGS += $(WERROR_FLAGS) + +# Headers +CFLAGS += -I$(RTE_SDK)/lib/librte_mempool +LDLIBS += -lrte_eal -lrte_mempool -lrte_ring + +EXPORT_MAP := rte_mempool_stack_version.map + +LIBABIVER := 1 + +SRCS-$(CONFIG_RTE_DRIVER_MEMPOOL_STACK) += rte_mempool_stack.c + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/mempool/stack/rte_mempool_stack.c b/dpdk/drivers/mempool/stack/rte_mempool_stack.c new file mode 100644 index 00000000..817f77e6 --- /dev/null +++ b/dpdk/drivers/mempool/stack/rte_mempool_stack.c @@ -0,0 +1,147 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include + +struct rte_mempool_stack { + rte_spinlock_t sl; + + uint32_t size; + uint32_t len; + void *objs[]; +}; + +static int +stack_alloc(struct rte_mempool *mp) +{ + struct rte_mempool_stack *s; + unsigned n = mp->size; + int size = sizeof(*s) + (n+16)*sizeof(void *); + + /* Allocate our local memory structure */ + s = rte_zmalloc_socket("mempool-stack", + size, + RTE_CACHE_LINE_SIZE, + mp->socket_id); + if (s == NULL) { + RTE_LOG(ERR, MEMPOOL, "Cannot allocate stack!\n"); + return -ENOMEM; + } + + rte_spinlock_init(&s->sl); + + s->size = n; + mp->pool_data = s; + + return 0; +} + +static int +stack_enqueue(struct rte_mempool *mp, void * const *obj_table, + unsigned n) +{ + struct rte_mempool_stack *s = mp->pool_data; + void **cache_objs; + unsigned index; + + rte_spinlock_lock(&s->sl); + cache_objs = &s->objs[s->len]; + + /* Is there sufficient space in the stack ? */ + if ((s->len + n) > s->size) { + rte_spinlock_unlock(&s->sl); + return -ENOBUFS; + } + + /* Add elements back into the cache */ + for (index = 0; index < n; ++index, obj_table++) + cache_objs[index] = *obj_table; + + s->len += n; + + rte_spinlock_unlock(&s->sl); + return 0; +} + +static int +stack_dequeue(struct rte_mempool *mp, void **obj_table, + unsigned n) +{ + struct rte_mempool_stack *s = mp->pool_data; + void **cache_objs; + unsigned index, len; + + rte_spinlock_lock(&s->sl); + + if (unlikely(n > s->len)) { + rte_spinlock_unlock(&s->sl); + return -ENOENT; + } + + cache_objs = s->objs; + + for (index = 0, len = s->len - 1; index < n; + ++index, len--, obj_table++) + *obj_table = cache_objs[len]; + + s->len -= n; + rte_spinlock_unlock(&s->sl); + return 0; +} + +static unsigned +stack_get_count(const struct rte_mempool *mp) +{ + struct rte_mempool_stack *s = mp->pool_data; + + return s->len; +} + +static void +stack_free(struct rte_mempool *mp) +{ + rte_free((void *)(mp->pool_data)); +} + +static struct rte_mempool_ops ops_stack = { + .name = "stack", + .alloc = stack_alloc, + .free = stack_free, + .enqueue = stack_enqueue, + .dequeue = stack_dequeue, + .get_count = stack_get_count +}; + +MEMPOOL_REGISTER_OPS(ops_stack); diff --git a/dpdk/drivers/mempool/stack/rte_mempool_stack_version.map b/dpdk/drivers/mempool/stack/rte_mempool_stack_version.map new file mode 100644 index 00000000..8591cc0b --- /dev/null +++ b/dpdk/drivers/mempool/stack/rte_mempool_stack_version.map @@ -0,0 +1,4 @@ +DPDK_17.05 { + + local: *; +}; diff --git a/dpdk/drivers/net/Makefile b/dpdk/drivers/net/Makefile index bc932309..ef09b4e1 100644 --- a/dpdk/drivers/net/Makefile +++ b/dpdk/drivers/net/Makefile @@ -31,33 +31,60 @@ include $(RTE_SDK)/mk/rte.vars.mk +# set in mk/toolchain/xxx/rte.toolchain-compat.mk +ifeq ($(CONFIG_RTE_LIBRTE_THUNDERX_NICVF_PMD),d) + $(warning thunderx pmd is not supported by old compilers) +endif + DIRS-$(CONFIG_RTE_LIBRTE_PMD_AF_PACKET) += af_packet +DIRS-$(CONFIG_RTE_LIBRTE_ARK_PMD) += ark +DIRS-$(CONFIG_RTE_LIBRTE_AVP_PMD) += avp DIRS-$(CONFIG_RTE_LIBRTE_BNX2X_PMD) += bnx2x DIRS-$(CONFIG_RTE_LIBRTE_PMD_BOND) += bonding DIRS-$(CONFIG_RTE_LIBRTE_CXGBE_PMD) += cxgbe +DIRS-$(CONFIG_RTE_LIBRTE_DPAA_PMD) += dpaa +DIRS-$(CONFIG_RTE_LIBRTE_DPAA2_PMD) += dpaa2 DIRS-$(CONFIG_RTE_LIBRTE_E1000_PMD) += e1000 DIRS-$(CONFIG_RTE_LIBRTE_ENA_PMD) += ena DIRS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic +DIRS-$(CONFIG_RTE_LIBRTE_PMD_FAILSAFE) += failsafe DIRS-$(CONFIG_RTE_LIBRTE_FM10K_PMD) += fm10k DIRS-$(CONFIG_RTE_LIBRTE_I40E_PMD) += i40e DIRS-$(CONFIG_RTE_LIBRTE_IXGBE_PMD) += ixgbe +DIRS-$(CONFIG_RTE_LIBRTE_LIO_PMD) += liquidio DIRS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4 DIRS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5 -DIRS-$(CONFIG_RTE_LIBRTE_MPIPE_PMD) += mpipe +DIRS-$(CONFIG_RTE_LIBRTE_MRVL_PMD) += mrvl DIRS-$(CONFIG_RTE_LIBRTE_NFP_PMD) += nfp DIRS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += bnxt DIRS-$(CONFIG_RTE_LIBRTE_PMD_NULL) += null +DIRS-$(CONFIG_RTE_LIBRTE_OCTEONTX_PMD) += octeontx DIRS-$(CONFIG_RTE_LIBRTE_PMD_PCAP) += pcap DIRS-$(CONFIG_RTE_LIBRTE_QEDE_PMD) += qede DIRS-$(CONFIG_RTE_LIBRTE_PMD_RING) += ring +DIRS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += sfc DIRS-$(CONFIG_RTE_LIBRTE_PMD_SZEDATA2) += szedata2 +DIRS-$(CONFIG_RTE_LIBRTE_PMD_TAP) += tap DIRS-$(CONFIG_RTE_LIBRTE_THUNDERX_NICVF_PMD) += thunderx DIRS-$(CONFIG_RTE_LIBRTE_VIRTIO_PMD) += virtio DIRS-$(CONFIG_RTE_LIBRTE_VMXNET3_PMD) += vmxnet3 -DIRS-$(CONFIG_RTE_LIBRTE_PMD_XENVIRT) += xenvirt + +ifeq ($(CONFIG_RTE_LIBRTE_KNI),y) +DIRS-$(CONFIG_RTE_LIBRTE_PMD_KNI) += kni +endif + +ifeq ($(CONFIG_RTE_LIBRTE_SCHED),y) +DIRS-$(CONFIG_RTE_LIBRTE_PMD_SOFTNIC) += softnic +endif # $(CONFIG_RTE_LIBRTE_SCHED) ifeq ($(CONFIG_RTE_LIBRTE_VHOST),y) DIRS-$(CONFIG_RTE_LIBRTE_PMD_VHOST) += vhost endif # $(CONFIG_RTE_LIBRTE_VHOST) +ifeq ($(CONFIG_RTE_LIBRTE_MRVL_PMD),y) +ifeq ($(CONFIG_RTE_LIBRTE_CFGFILE),n) +$(error "RTE_LIBRTE_CFGFILE must be enabled in configuration!") +endif +endif + include $(RTE_SDK)/mk/rte.subdir.mk diff --git a/dpdk/drivers/net/af_packet/Makefile b/dpdk/drivers/net/af_packet/Makefile index e14d6d0c..bb37d67a 100644 --- a/dpdk/drivers/net/af_packet/Makefile +++ b/dpdk/drivers/net/af_packet/Makefile @@ -44,17 +44,13 @@ LIBABIVER := 1 CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs +LDLIBS += -lrte_bus_vdev # # all source are stored in SRCS-y # SRCS-$(CONFIG_RTE_LIBRTE_PMD_AF_PACKET) += rte_eth_af_packet.c -# this lib depends upon: -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_AF_PACKET) += lib/librte_eal -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_AF_PACKET) += lib/librte_mbuf -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_AF_PACKET) += lib/librte_mempool -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_AF_PACKET) += lib/librte_ether -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_AF_PACKET) += lib/librte_kvargs - include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/net/af_packet/rte_eth_af_packet.c b/dpdk/drivers/net/af_packet/rte_eth_af_packet.c index f7955662..d5154089 100644 --- a/dpdk/drivers/net/af_packet/rte_eth_af_packet.c +++ b/dpdk/drivers/net/af_packet/rte_eth_af_packet.c @@ -38,9 +38,10 @@ #include #include +#include #include #include -#include +#include #include #include @@ -58,6 +59,7 @@ #define ETH_AF_PACKET_BLOCKSIZE_ARG "blocksz" #define ETH_AF_PACKET_FRAMESIZE_ARG "framesz" #define ETH_AF_PACKET_FRAMECOUNT_ARG "framecnt" +#define ETH_AF_PACKET_QDISC_BYPASS_ARG "qdisc_bypass" #define DFLT_BLOCK_SIZE (1 << 12) #define DFLT_FRAME_SIZE (1 << 11) @@ -74,7 +76,7 @@ struct pkt_rx_queue { unsigned int framenum; struct rte_mempool *mb_pool; - uint8_t in_port; + uint16_t in_port; volatile unsigned long rx_pkts; volatile unsigned long err_pkts; @@ -83,6 +85,7 @@ struct pkt_rx_queue { struct pkt_tx_queue { int sockfd; + unsigned int frame_data_size; struct iovec *rd; uint8_t *map; @@ -98,6 +101,7 @@ struct pmd_internals { unsigned nb_queues; int if_index; + char *if_name; struct ether_addr eth_addr; struct tpacket_req req; @@ -112,16 +116,15 @@ static const char *valid_arguments[] = { ETH_AF_PACKET_BLOCKSIZE_ARG, ETH_AF_PACKET_FRAMESIZE_ARG, ETH_AF_PACKET_FRAMECOUNT_ARG, + ETH_AF_PACKET_QDISC_BYPASS_ARG, NULL }; -static const char *drivername = "AF_PACKET PMD"; - static struct rte_eth_link pmd_link = { .link_speed = ETH_SPEED_NUM_10G, .link_duplex = ETH_LINK_FULL_DUPLEX, .link_status = ETH_LINK_DOWN, - .link_autoneg = ETH_LINK_SPEED_AUTONEG + .link_autoneg = ETH_LINK_AUTONEG }; static uint16_t @@ -161,6 +164,12 @@ eth_af_packet_rx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) pbuf = (uint8_t *) ppd + ppd->tp_mac; memcpy(rte_pktmbuf_mtod(mbuf, void *), pbuf, rte_pktmbuf_data_len(mbuf)); + /* check for vlan info */ + if (ppd->tp_status & TP_STATUS_VLAN_VALID) { + mbuf->vlan_tci = ppd->tp_vlan_tci; + mbuf->ol_flags |= (PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED); + } + /* release incoming frame and advance ring buffer */ ppd->tp_status = TP_STATUS_KERNEL; if (++framenum >= framecount) @@ -206,17 +215,41 @@ eth_af_packet_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) framenum = pkt_q->framenum; ppd = (struct tpacket2_hdr *) pkt_q->rd[framenum].iov_base; for (i = 0; i < nb_pkts; i++) { + mbuf = *bufs++; + + /* drop oversized packets */ + if (mbuf->pkt_len > pkt_q->frame_data_size) { + rte_pktmbuf_free(mbuf); + continue; + } + + /* insert vlan info if necessary */ + if (mbuf->ol_flags & PKT_TX_VLAN_PKT) { + if (rte_vlan_insert(&mbuf)) { + rte_pktmbuf_free(mbuf); + continue; + } + } + /* point at the next incoming frame */ if ((ppd->tp_status != TP_STATUS_AVAILABLE) && (poll(&pfd, 1, -1) < 0)) - continue; + break; /* copy the tx frame data */ - mbuf = bufs[num_tx]; pbuf = (uint8_t *) ppd + TPACKET2_HDRLEN - sizeof(struct sockaddr_ll); - memcpy(pbuf, rte_pktmbuf_mtod(mbuf, void*), rte_pktmbuf_data_len(mbuf)); - ppd->tp_len = ppd->tp_snaplen = rte_pktmbuf_data_len(mbuf); + + struct rte_mbuf *tmp_mbuf = mbuf; + while (tmp_mbuf) { + uint16_t data_len = rte_pktmbuf_data_len(tmp_mbuf); + memcpy(pbuf, rte_pktmbuf_mtod(tmp_mbuf, void*), data_len); + pbuf += data_len; + tmp_mbuf = tmp_mbuf->next; + } + + ppd->tp_len = mbuf->pkt_len; + ppd->tp_snaplen = mbuf->pkt_len; /* release incoming frame and advance ring buffer */ ppd->tp_status = TP_STATUS_SEND_REQUEST; @@ -230,14 +263,17 @@ eth_af_packet_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) } /* kick-off transmits */ - if (sendto(pkt_q->sockfd, NULL, 0, MSG_DONTWAIT, NULL, 0) == -1) - return 0; /* error sending -- no packets transmitted */ + if (sendto(pkt_q->sockfd, NULL, 0, MSG_DONTWAIT, NULL, 0) == -1) { + /* error sending -- no packets transmitted */ + num_tx = 0; + num_tx_bytes = 0; + } pkt_q->framenum = framenum; pkt_q->tx_pkts += num_tx; - pkt_q->err_pkts += nb_pkts - num_tx; + pkt_q->err_pkts += i - num_tx; pkt_q->tx_bytes += num_tx_bytes; - return num_tx; + return i; } static int @@ -261,9 +297,16 @@ eth_dev_stop(struct rte_eth_dev *dev) sockfd = internals->rx_queue[i].sockfd; if (sockfd != -1) close(sockfd); - sockfd = internals->tx_queue[i].sockfd; - if (sockfd != -1) - close(sockfd); + + /* Prevent use after free in case tx fd == rx fd */ + if (sockfd != internals->tx_queue[i].sockfd) { + sockfd = internals->tx_queue[i].sockfd; + if (sockfd != -1) + close(sockfd); + } + + internals->rx_queue[i].sockfd = -1; + internals->tx_queue[i].sockfd = -1; } dev->data->dev_link.link_status = ETH_LINK_DOWN; @@ -280,17 +323,15 @@ eth_dev_info(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) { struct pmd_internals *internals = dev->data->dev_private; - dev_info->driver_name = drivername; dev_info->if_index = internals->if_index; dev_info->max_mac_addrs = 1; dev_info->max_rx_pktlen = (uint32_t)ETH_FRAME_LEN; dev_info->max_rx_queues = (uint16_t)internals->nb_queues; dev_info->max_tx_queues = (uint16_t)internals->nb_queues; dev_info->min_rx_bufsize = 0; - dev_info->pci_dev = NULL; } -static void +static int eth_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *igb_stats) { unsigned i, imax; @@ -323,6 +364,7 @@ eth_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *igb_stats) igb_stats->opackets = tx_total; igb_stats->oerrors = tx_err_total; igb_stats->obytes = tx_bytes_total; + return 0; } static void @@ -370,18 +412,20 @@ eth_rx_queue_setup(struct rte_eth_dev *dev, { struct pmd_internals *internals = dev->data->dev_private; struct pkt_rx_queue *pkt_q = &internals->rx_queue[rx_queue_id]; - uint16_t buf_size; + unsigned int buf_size, data_size; pkt_q->mb_pool = mb_pool; /* Now get the space available for data in the mbuf */ - buf_size = (uint16_t)(rte_pktmbuf_data_room_size(pkt_q->mb_pool) - - RTE_PKTMBUF_HEADROOM); + buf_size = rte_pktmbuf_data_room_size(pkt_q->mb_pool) - + RTE_PKTMBUF_HEADROOM; + data_size = internals->req.tp_frame_size; + data_size -= TPACKET2_HDRLEN - sizeof(struct sockaddr_ll); - if (ETH_FRAME_LEN > buf_size) { + if (data_size > buf_size) { RTE_LOG(ERR, PMD, "%s: %d bytes will not fit in mbuf (%d bytes)\n", - dev->data->name, ETH_FRAME_LEN, buf_size); + dev->device->name, data_size, buf_size); return -ENOMEM; } @@ -405,12 +449,80 @@ eth_tx_queue_setup(struct rte_eth_dev *dev, return 0; } +static int +eth_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) +{ + struct pmd_internals *internals = dev->data->dev_private; + struct ifreq ifr = { .ifr_mtu = mtu }; + int ret; + int s; + unsigned int data_size = internals->req.tp_frame_size - + TPACKET2_HDRLEN - + sizeof(struct sockaddr_ll); + + if (mtu > data_size) + return -EINVAL; + + s = socket(PF_INET, SOCK_DGRAM, 0); + if (s < 0) + return -EINVAL; + + snprintf(ifr.ifr_name, IFNAMSIZ, "%s", internals->if_name); + ret = ioctl(s, SIOCSIFMTU, &ifr); + close(s); + + if (ret < 0) + return -EINVAL; + + return 0; +} + +static void +eth_dev_change_flags(char *if_name, uint32_t flags, uint32_t mask) +{ + struct ifreq ifr; + int s; + + s = socket(PF_INET, SOCK_DGRAM, 0); + if (s < 0) + return; + + snprintf(ifr.ifr_name, IFNAMSIZ, "%s", if_name); + if (ioctl(s, SIOCGIFFLAGS, &ifr) < 0) + goto out; + ifr.ifr_flags &= mask; + ifr.ifr_flags |= flags; + if (ioctl(s, SIOCSIFFLAGS, &ifr) < 0) + goto out; +out: + close(s); +} + +static void +eth_dev_promiscuous_enable(struct rte_eth_dev *dev) +{ + struct pmd_internals *internals = dev->data->dev_private; + + eth_dev_change_flags(internals->if_name, IFF_PROMISC, ~0); +} + +static void +eth_dev_promiscuous_disable(struct rte_eth_dev *dev) +{ + struct pmd_internals *internals = dev->data->dev_private; + + eth_dev_change_flags(internals->if_name, 0, ~IFF_PROMISC); +} + static const struct eth_dev_ops ops = { .dev_start = eth_dev_start, .dev_stop = eth_dev_stop, .dev_close = eth_dev_close, .dev_configure = eth_dev_configure, .dev_infos_get = eth_dev_info, + .mtu_set = eth_dev_mtu_set, + .promiscuous_enable = eth_dev_promiscuous_enable, + .promiscuous_disable = eth_dev_promiscuous_disable, .rx_queue_setup = eth_rx_queue_setup, .tx_queue_setup = eth_tx_queue_setup, .rx_queue_release = eth_queue_release, @@ -440,19 +552,23 @@ open_packet_iface(const char *key __rte_unused, return 0; } +static struct rte_vdev_driver pmd_af_packet_drv; + static int -rte_pmd_init_internals(const char *name, +rte_pmd_init_internals(struct rte_vdev_device *dev, const int sockfd, const unsigned nb_queues, unsigned int blocksize, unsigned int blockcnt, unsigned int framesize, unsigned int framecnt, - const unsigned numa_node, + unsigned int qdisc_bypass, struct pmd_internals **internals, struct rte_eth_dev **eth_dev, struct rte_kvargs *kvlist) { + const char *name = rte_vdev_device_name(dev); + const unsigned int numa_node = dev->device.numa_node; struct rte_eth_dev_data *data = NULL; struct rte_kvargs_pair *pair = NULL; struct ifreq ifr; @@ -465,7 +581,9 @@ rte_pmd_init_internals(const char *name, int rc, tpver, discard; int qsockfd = -1; unsigned int i, q, rdsize; - int fanout_arg __rte_unused, bypass __rte_unused; +#if defined(PACKET_FANOUT) + int fanout_arg; +#endif for (k_idx = 0; k_idx < kvlist->count; k_idx++) { pair = &kvlist->pairs[k_idx]; @@ -524,6 +642,9 @@ rte_pmd_init_internals(const char *name, name); goto error_early; } + (*internals)->if_name = strdup(pair->value); + if ((*internals)->if_name == NULL) + goto error_early; (*internals)->if_index = ifr.ifr_ifindex; if (ioctl(sockfd, SIOCGIFHWADDR, &ifr) == -1) { @@ -578,9 +699,8 @@ rte_pmd_init_internals(const char *name, } #if defined(PACKET_QDISC_BYPASS) - bypass = 1; rc = setsockopt(qsockfd, SOL_PACKET, PACKET_QDISC_BYPASS, - &bypass, sizeof(bypass)); + &qdisc_bypass, sizeof(qdisc_bypass)); if (rc == -1) { RTE_LOG(ERR, PMD, "%s: could not set PACKET_QDISC_BYPASS " @@ -588,6 +708,8 @@ rte_pmd_init_internals(const char *name, pair->value); goto error; } +#else + RTE_SET_USED(qdisc_bypass); #endif rc = setsockopt(qsockfd, SOL_PACKET, PACKET_RX_RING, req, sizeof(*req)); @@ -633,6 +755,9 @@ rte_pmd_init_internals(const char *name, tx_queue = &((*internals)->tx_queue[q]); tx_queue->framecount = req->tp_frame_nr; + tx_queue->frame_data_size = req->tp_frame_size; + tx_queue->frame_data_size -= TPACKET2_HDRLEN - + sizeof(struct sockaddr_ll); tx_queue->map = rx_queue->map + req->tp_block_size * req->tp_block_nr; @@ -666,7 +791,7 @@ rte_pmd_init_internals(const char *name, } /* reserve an ethdev entry */ - *eth_dev = rte_eth_dev_allocate(name, RTE_ETH_DEV_VIRTUAL); + *eth_dev = rte_eth_vdev_allocate(dev, 0); if (*eth_dev == NULL) goto error; @@ -680,22 +805,15 @@ rte_pmd_init_internals(const char *name, (*internals)->nb_queues = nb_queues; + rte_memcpy(data, (*eth_dev)->data, sizeof(*data)); data->dev_private = *internals; - data->port_id = (*eth_dev)->data->port_id; data->nb_rx_queues = (uint16_t)nb_queues; data->nb_tx_queues = (uint16_t)nb_queues; data->dev_link = pmd_link; data->mac_addrs = &(*internals)->eth_addr; - strncpy(data->name, - (*eth_dev)->data->name, strlen((*eth_dev)->data->name)); (*eth_dev)->data = data; (*eth_dev)->dev_ops = &ops; - (*eth_dev)->driver = NULL; - (*eth_dev)->data->dev_flags = RTE_ETH_DEV_DETACHABLE; - (*eth_dev)->data->drv_name = drivername; - (*eth_dev)->data->kdrv = RTE_KDRV_NONE; - (*eth_dev)->data->numa_node = numa_node; return 0; @@ -712,6 +830,7 @@ error: ((*internals)->rx_queue[q].sockfd != qsockfd)) close((*internals)->rx_queue[q].sockfd); } + free((*internals)->if_name); rte_free(*internals); error_early: rte_free(data); @@ -719,11 +838,11 @@ error_early: } static int -rte_eth_from_packet(const char *name, +rte_eth_from_packet(struct rte_vdev_device *dev, int const *sockfd, - const unsigned numa_node, struct rte_kvargs *kvlist) { + const char *name = rte_vdev_device_name(dev); struct pmd_internals *internals = NULL; struct rte_eth_dev *eth_dev = NULL; struct rte_kvargs_pair *pair = NULL; @@ -733,6 +852,7 @@ rte_eth_from_packet(const char *name, unsigned int framesize = DFLT_FRAME_SIZE; unsigned int framecount = DFLT_FRAME_COUNT; unsigned int qpairs = 1; + unsigned int qdisc_bypass = 1; /* do some parameter checking */ if (*sockfd < 0) @@ -784,6 +904,16 @@ rte_eth_from_packet(const char *name, } continue; } + if (strstr(pair->key, ETH_AF_PACKET_QDISC_BYPASS_ARG) != NULL) { + qdisc_bypass = atoi(pair->value); + if (qdisc_bypass > 1) { + RTE_LOG(ERR, PMD, + "%s: invalid bypass value\n", + name); + return -1; + } + continue; + } } if (framesize > blocksize) { @@ -806,11 +936,12 @@ rte_eth_from_packet(const char *name, RTE_LOG(INFO, PMD, "%s:\tframe size %d\n", name, framesize); RTE_LOG(INFO, PMD, "%s:\tframe count %d\n", name, framecount); - if (rte_pmd_init_internals(name, *sockfd, qpairs, - blocksize, blockcount, - framesize, framecount, - numa_node, &internals, ð_dev, - kvlist) < 0) + if (rte_pmd_init_internals(dev, *sockfd, qpairs, + blocksize, blockcount, + framesize, framecount, + qdisc_bypass, + &internals, ð_dev, + kvlist) < 0) return -1; eth_dev->rx_pkt_burst = eth_af_packet_rx; @@ -820,18 +951,16 @@ rte_eth_from_packet(const char *name, } static int -rte_pmd_af_packet_devinit(const char *name, const char *params) +rte_pmd_af_packet_probe(struct rte_vdev_device *dev) { - unsigned numa_node; int ret = 0; struct rte_kvargs *kvlist; int sockfd = -1; - RTE_LOG(INFO, PMD, "Initializing pmd_af_packet for %s\n", name); + RTE_LOG(INFO, PMD, "Initializing pmd_af_packet for %s\n", + rte_vdev_device_name(dev)); - numa_node = rte_socket_id(); - - kvlist = rte_kvargs_parse(params, valid_arguments); + kvlist = rte_kvargs_parse(rte_vdev_device_args(dev), valid_arguments); if (kvlist == NULL) { ret = -1; goto exit; @@ -849,7 +978,10 @@ rte_pmd_af_packet_devinit(const char *name, const char *params) goto exit; } - ret = rte_eth_from_packet(name, &sockfd, numa_node, kvlist); + if (dev->device.numa_node == SOCKET_ID_ANY) + dev->device.numa_node = rte_socket_id(); + + ret = rte_eth_from_packet(dev, &sockfd, kvlist); close(sockfd); /* no longer needed */ exit: @@ -858,7 +990,7 @@ exit: } static int -rte_pmd_af_packet_devuninit(const char *name) +rte_pmd_af_packet_remove(struct rte_vdev_device *dev) { struct rte_eth_dev *eth_dev = NULL; struct pmd_internals *internals; @@ -867,11 +999,11 @@ rte_pmd_af_packet_devuninit(const char *name) RTE_LOG(INFO, PMD, "Closing AF_PACKET ethdev on numa socket %u\n", rte_socket_id()); - if (name == NULL) + if (dev == NULL) return -1; /* find the ethdev entry */ - eth_dev = rte_eth_dev_allocated(name); + eth_dev = rte_eth_dev_allocated(rte_vdev_device_name(dev)); if (eth_dev == NULL) return -1; @@ -880,6 +1012,7 @@ rte_pmd_af_packet_devuninit(const char *name) rte_free(internals->rx_queue[q].rd); rte_free(internals->tx_queue[q].rd); } + free(internals->if_name); rte_free(eth_dev->data->dev_private); rte_free(eth_dev->data); @@ -889,16 +1022,17 @@ rte_pmd_af_packet_devuninit(const char *name) return 0; } -static struct rte_driver pmd_af_packet_drv = { - .type = PMD_VDEV, - .init = rte_pmd_af_packet_devinit, - .uninit = rte_pmd_af_packet_devuninit, +static struct rte_vdev_driver pmd_af_packet_drv = { + .probe = rte_pmd_af_packet_probe, + .remove = rte_pmd_af_packet_remove, }; -PMD_REGISTER_DRIVER(pmd_af_packet_drv, eth_af_packet); -DRIVER_REGISTER_PARAM_STRING(eth_af_packet, +RTE_PMD_REGISTER_VDEV(net_af_packet, pmd_af_packet_drv); +RTE_PMD_REGISTER_ALIAS(net_af_packet, eth_af_packet); +RTE_PMD_REGISTER_PARAM_STRING(net_af_packet, "iface= " "qpairs= " "blocksz= " "framesz= " - "framecnt="); + "framecnt= " + "qdisc_bypass=<0|1>"); diff --git a/dpdk/drivers/net/ark/Makefile b/dpdk/drivers/net/ark/Makefile new file mode 100644 index 00000000..f1433bd2 --- /dev/null +++ b/dpdk/drivers/net/ark/Makefile @@ -0,0 +1,69 @@ +# BSD LICENSE +# +# Copyright (c) 2015-2017 Atomic Rules LLC +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of copyright holder nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +include $(RTE_SDK)/mk/rte.vars.mk + +# +# library name +# +LIB = librte_pmd_ark.a + +CFLAGS += -O3 -I./ +CFLAGS += $(WERROR_FLAGS) -Werror + +EXPORT_MAP := rte_pmd_ark_version.map + +LIBABIVER := 1 + +# +# all source are stored in SRCS-y +# +SRCS-$(CONFIG_RTE_LIBRTE_ARK_PMD) += ark_ddm.c +SRCS-$(CONFIG_RTE_LIBRTE_ARK_PMD) += ark_ethdev.c +SRCS-$(CONFIG_RTE_LIBRTE_ARK_PMD) += ark_ethdev_rx.c +SRCS-$(CONFIG_RTE_LIBRTE_ARK_PMD) += ark_ethdev_tx.c +SRCS-$(CONFIG_RTE_LIBRTE_ARK_PMD) += ark_mpu.c +SRCS-$(CONFIG_RTE_LIBRTE_ARK_PMD) += ark_pktchkr.c +SRCS-$(CONFIG_RTE_LIBRTE_ARK_PMD) += ark_pktdir.c +SRCS-$(CONFIG_RTE_LIBRTE_ARK_PMD) += ark_pktgen.c +SRCS-$(CONFIG_RTE_LIBRTE_ARK_PMD) += ark_rqp.c +SRCS-$(CONFIG_RTE_LIBRTE_ARK_PMD) += ark_udm.c + +# this lib depends upon: +LDLIBS += -lpthread +ifdef CONFIG_RTE_EXEC_ENV_LINUXAPP +LDLIBS += -ldl +endif +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs +LDLIBS += -lrte_bus_pci + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/net/ark/ark_ddm.c b/dpdk/drivers/net/ark/ark_ddm.c new file mode 100644 index 00000000..929dc7d1 --- /dev/null +++ b/dpdk/drivers/net/ark/ark_ddm.c @@ -0,0 +1,151 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2015-2017 Atomic Rules LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +#include "ark_logs.h" +#include "ark_ddm.h" + +/* ************************************************************************* */ +int +ark_ddm_verify(struct ark_ddm_t *ddm) +{ + if (sizeof(struct ark_ddm_t) != ARK_DDM_EXPECTED_SIZE) { + PMD_DRV_LOG(ERR, "ARK: DDM structure looks incorrect %d vs %zd\n", + ARK_DDM_EXPECTED_SIZE, sizeof(struct ark_ddm_t)); + return -1; + } + + if (ddm->cfg.const0 != ARK_DDM_CONST) { + PMD_DRV_LOG(ERR, "ARK: DDM module not found as expected 0x%08x\n", + ddm->cfg.const0); + return -1; + } + return 0; +} + +void +ark_ddm_start(struct ark_ddm_t *ddm) +{ + ddm->cfg.command = 1; +} + +int +ark_ddm_stop(struct ark_ddm_t *ddm, const int wait) +{ + int cnt = 0; + + ddm->cfg.command = 2; + while (wait && (ddm->cfg.stop_flushed & 0x01) == 0) { + if (cnt++ > 1000) + return 1; + + usleep(10); + } + return 0; +} + +void +ark_ddm_reset(struct ark_ddm_t *ddm) +{ + int status; + + /* reset only works if ddm has stopped properly. */ + status = ark_ddm_stop(ddm, 1); + + if (status != 0) { + PMD_DEBUG_LOG(INFO, "%s stop failed doing forced reset\n", + __func__); + ddm->cfg.command = 4; + usleep(10); + } + ddm->cfg.command = 3; +} + +void +ark_ddm_setup(struct ark_ddm_t *ddm, rte_iova_t cons_addr, uint32_t interval) +{ + ddm->setup.cons_write_index_addr = cons_addr; + ddm->setup.write_index_interval = interval / 4; /* 4 ns period */ +} + +void +ark_ddm_stats_reset(struct ark_ddm_t *ddm) +{ + ddm->cfg.tlp_stats_clear = 1; +} + +void +ark_ddm_dump(struct ark_ddm_t *ddm, const char *msg) +{ + PMD_FUNC_LOG(DEBUG, "%s Stopped: %d\n", msg, + ark_ddm_is_stopped(ddm) + ); +} + +void +ark_ddm_dump_stats(struct ark_ddm_t *ddm, const char *msg) +{ + struct ark_ddm_stats_t *stats = &ddm->stats; + + PMD_STATS_LOG(INFO, "DDM Stats: %s" + ARK_SU64 ARK_SU64 ARK_SU64 + "\n", msg, + "Bytes:", stats->tx_byte_count, + "Packets:", stats->tx_pkt_count, + "MBufs", stats->tx_mbuf_count); +} + +int +ark_ddm_is_stopped(struct ark_ddm_t *ddm) +{ + return (ddm->cfg.stop_flushed & 0x01) != 0; +} + +uint64_t +ark_ddm_queue_byte_count(struct ark_ddm_t *ddm) +{ + return ddm->queue_stats.byte_count; +} + +uint64_t +ark_ddm_queue_pkt_count(struct ark_ddm_t *ddm) +{ + return ddm->queue_stats.pkt_count; +} + +void +ark_ddm_queue_reset_stats(struct ark_ddm_t *ddm) +{ + ddm->queue_stats.byte_count = 1; +} diff --git a/dpdk/drivers/net/ark/ark_ddm.h b/dpdk/drivers/net/ark/ark_ddm.h new file mode 100644 index 00000000..f67ad012 --- /dev/null +++ b/dpdk/drivers/net/ark/ark_ddm.h @@ -0,0 +1,177 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2015-2017 Atomic Rules LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _ARK_DDM_H_ +#define _ARK_DDM_H_ + +#include + +#include + + +/* The DDM or Downstream Data Mover is an internal Arkville hardware + * module for moving packet from host memory to the TX packet streams. + * This module is *not* intended for end-user manipulation, hence + * there is minimal documentation. + */ + +/* struct defining Tx meta data -- fixed in FPGA -- 16 bytes */ +struct ark_tx_meta { + uint64_t physaddr; + uint32_t delta_ns; + uint16_t data_len; /* of this MBUF */ +#define ARK_DDM_EOP 0x01 +#define ARK_DDM_SOP 0x02 + uint8_t flags; /* bit 0 indicates last mbuf in chain. */ + uint8_t reserved[1]; +}; + + +/* + * DDM core hardware structures + * These are overlay structures to a memory mapped FPGA device. These + * structs will never be instantiated in ram memory + */ +#define ARK_DDM_CFG 0x0000 +#define ARK_DDM_CONST 0xfacecafe +struct ark_ddm_cfg_t { + uint32_t r0; + volatile uint32_t tlp_stats_clear; + uint32_t const0; + volatile uint32_t tag_max; + volatile uint32_t command; + volatile uint32_t stop_flushed; +}; + +#define ARK_DDM_STATS 0x0020 +struct ark_ddm_stats_t { + volatile uint64_t tx_byte_count; + volatile uint64_t tx_pkt_count; + volatile uint64_t tx_mbuf_count; +}; + +#define ARK_DDM_MRDQ 0x0040 +struct ark_ddm_mrdq_t { + volatile uint32_t mrd_q1; + volatile uint32_t mrd_q2; + volatile uint32_t mrd_q3; + volatile uint32_t mrd_q4; + volatile uint32_t mrd_full; +}; + +#define ARK_DDM_CPLDQ 0x0068 +struct ark_ddm_cpldq_t { + volatile uint32_t cpld_q1; + volatile uint32_t cpld_q2; + volatile uint32_t cpld_q3; + volatile uint32_t cpld_q4; + volatile uint32_t cpld_full; +}; + +#define ARK_DDM_MRD_PS 0x0090 +struct ark_ddm_mrd_ps_t { + volatile uint32_t mrd_ps_min; + volatile uint32_t mrd_ps_max; + volatile uint32_t mrd_full_ps_min; + volatile uint32_t mrd_full_ps_max; + volatile uint32_t mrd_dw_ps_min; + volatile uint32_t mrd_dw_ps_max; +}; + +#define ARK_DDM_QUEUE_STATS 0x00a8 +struct ark_ddm_qstats_t { + volatile uint64_t byte_count; + volatile uint64_t pkt_count; + volatile uint64_t mbuf_count; +}; + +#define ARK_DDM_CPLD_PS 0x00c0 +struct ark_ddm_cpld_ps_t { + volatile uint32_t cpld_ps_min; + volatile uint32_t cpld_ps_max; + volatile uint32_t cpld_full_ps_min; + volatile uint32_t cpld_full_ps_max; + volatile uint32_t cpld_dw_ps_min; + volatile uint32_t cpld_dw_ps_max; +}; + +#define ARK_DDM_SETUP 0x00e0 +struct ark_ddm_setup_t { + rte_iova_t cons_write_index_addr; + uint32_t write_index_interval; /* 4ns each */ + volatile uint32_t cons_index; +}; + +#define ARK_DDM_EXPECTED_SIZE 256 +#define ARK_DDM_QOFFSET ARK_DDM_EXPECTED_SIZE +/* Consolidated structure */ +struct ark_ddm_t { + struct ark_ddm_cfg_t cfg; + uint8_t reserved0[(ARK_DDM_STATS - ARK_DDM_CFG) - + sizeof(struct ark_ddm_cfg_t)]; + struct ark_ddm_stats_t stats; + uint8_t reserved1[(ARK_DDM_MRDQ - ARK_DDM_STATS) - + sizeof(struct ark_ddm_stats_t)]; + struct ark_ddm_mrdq_t mrdq; + uint8_t reserved2[(ARK_DDM_CPLDQ - ARK_DDM_MRDQ) - + sizeof(struct ark_ddm_mrdq_t)]; + struct ark_ddm_cpldq_t cpldq; + uint8_t reserved3[(ARK_DDM_MRD_PS - ARK_DDM_CPLDQ) - + sizeof(struct ark_ddm_cpldq_t)]; + struct ark_ddm_mrd_ps_t mrd_ps; + struct ark_ddm_qstats_t queue_stats; + struct ark_ddm_cpld_ps_t cpld_ps; + uint8_t reserved5[(ARK_DDM_SETUP - ARK_DDM_CPLD_PS) - + sizeof(struct ark_ddm_cpld_ps_t)]; + struct ark_ddm_setup_t setup; + uint8_t reserved_p[(ARK_DDM_EXPECTED_SIZE - ARK_DDM_SETUP) - + sizeof(struct ark_ddm_setup_t)]; +}; + + +/* DDM function prototype */ +int ark_ddm_verify(struct ark_ddm_t *ddm); +void ark_ddm_start(struct ark_ddm_t *ddm); +int ark_ddm_stop(struct ark_ddm_t *ddm, const int wait); +void ark_ddm_reset(struct ark_ddm_t *ddm); +void ark_ddm_stats_reset(struct ark_ddm_t *ddm); +void ark_ddm_setup(struct ark_ddm_t *ddm, rte_iova_t cons_addr, + uint32_t interval); +void ark_ddm_dump_stats(struct ark_ddm_t *ddm, const char *msg); +void ark_ddm_dump(struct ark_ddm_t *ddm, const char *msg); +int ark_ddm_is_stopped(struct ark_ddm_t *ddm); +uint64_t ark_ddm_queue_byte_count(struct ark_ddm_t *ddm); +uint64_t ark_ddm_queue_pkt_count(struct ark_ddm_t *ddm); +void ark_ddm_queue_reset_stats(struct ark_ddm_t *ddm); + +#endif diff --git a/dpdk/drivers/net/ark/ark_ethdev.c b/dpdk/drivers/net/ark/ark_ethdev.c new file mode 100644 index 00000000..ff87c20e --- /dev/null +++ b/dpdk/drivers/net/ark/ark_ethdev.c @@ -0,0 +1,1047 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2015-2017 Atomic Rules LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include + +#include +#include +#include + +#include "ark_global.h" +#include "ark_logs.h" +#include "ark_ethdev_tx.h" +#include "ark_ethdev_rx.h" +#include "ark_mpu.h" +#include "ark_ddm.h" +#include "ark_udm.h" +#include "ark_rqp.h" +#include "ark_pktdir.h" +#include "ark_pktgen.h" +#include "ark_pktchkr.h" + +/* Internal prototypes */ +static int eth_ark_check_args(struct ark_adapter *ark, const char *params); +static int eth_ark_dev_init(struct rte_eth_dev *dev); +static int ark_config_device(struct rte_eth_dev *dev); +static int eth_ark_dev_uninit(struct rte_eth_dev *eth_dev); +static int eth_ark_dev_configure(struct rte_eth_dev *dev); +static int eth_ark_dev_start(struct rte_eth_dev *dev); +static void eth_ark_dev_stop(struct rte_eth_dev *dev); +static void eth_ark_dev_close(struct rte_eth_dev *dev); +static void eth_ark_dev_info_get(struct rte_eth_dev *dev, + struct rte_eth_dev_info *dev_info); +static int eth_ark_dev_link_update(struct rte_eth_dev *dev, + int wait_to_complete); +static int eth_ark_dev_set_link_up(struct rte_eth_dev *dev); +static int eth_ark_dev_set_link_down(struct rte_eth_dev *dev); +static int eth_ark_dev_stats_get(struct rte_eth_dev *dev, + struct rte_eth_stats *stats); +static void eth_ark_dev_stats_reset(struct rte_eth_dev *dev); +static void eth_ark_set_default_mac_addr(struct rte_eth_dev *dev, + struct ether_addr *mac_addr); +static int eth_ark_macaddr_add(struct rte_eth_dev *dev, + struct ether_addr *mac_addr, + uint32_t index, + uint32_t pool); +static void eth_ark_macaddr_remove(struct rte_eth_dev *dev, + uint32_t index); +static int eth_ark_set_mtu(struct rte_eth_dev *dev, uint16_t size); + +/* + * The packet generator is a functional block used to generate packet + * patterns for testing. It is not intended for nominal use. + */ +#define ARK_PKTGEN_ARG "Pkt_gen" + +/* + * The packet checker is a functional block used to verify packet + * patterns for testing. It is not intended for nominal use. + */ +#define ARK_PKTCHKR_ARG "Pkt_chkr" + +/* + * The packet director is used to select the internal ingress and + * egress packets paths during testing. It is not intended for + * nominal use. + */ +#define ARK_PKTDIR_ARG "Pkt_dir" + +/* Devinfo configurations */ +#define ARK_RX_MAX_QUEUE (4096 * 4) +#define ARK_RX_MIN_QUEUE (512) +#define ARK_RX_MAX_PKT_LEN ((16 * 1024) - 128) +#define ARK_RX_MIN_BUFSIZE (1024) + +#define ARK_TX_MAX_QUEUE (4096 * 4) +#define ARK_TX_MIN_QUEUE (256) + +static const char * const valid_arguments[] = { + ARK_PKTGEN_ARG, + ARK_PKTCHKR_ARG, + ARK_PKTDIR_ARG, + NULL +}; + +static const struct rte_pci_id pci_id_ark_map[] = { + {RTE_PCI_DEVICE(0x1d6c, 0x100d)}, + {RTE_PCI_DEVICE(0x1d6c, 0x100e)}, + {.vendor_id = 0, /* sentinel */ }, +}; + +static int +eth_ark_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, + struct rte_pci_device *pci_dev) +{ + struct rte_eth_dev *eth_dev; + int ret; + + eth_dev = rte_eth_dev_pci_allocate(pci_dev, sizeof(struct ark_adapter)); + + if (eth_dev == NULL) + return -ENOMEM; + + ret = eth_ark_dev_init(eth_dev); + if (ret) + rte_eth_dev_pci_release(eth_dev); + + return ret; +} + +static int +eth_ark_pci_remove(struct rte_pci_device *pci_dev) +{ + return rte_eth_dev_pci_generic_remove(pci_dev, eth_ark_dev_uninit); +} + +static struct rte_pci_driver rte_ark_pmd = { + .id_table = pci_id_ark_map, + .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC, + .probe = eth_ark_pci_probe, + .remove = eth_ark_pci_remove, +}; + +static const struct eth_dev_ops ark_eth_dev_ops = { + .dev_configure = eth_ark_dev_configure, + .dev_start = eth_ark_dev_start, + .dev_stop = eth_ark_dev_stop, + .dev_close = eth_ark_dev_close, + + .dev_infos_get = eth_ark_dev_info_get, + + .rx_queue_setup = eth_ark_dev_rx_queue_setup, + .rx_queue_count = eth_ark_dev_rx_queue_count, + .tx_queue_setup = eth_ark_tx_queue_setup, + + .link_update = eth_ark_dev_link_update, + .dev_set_link_up = eth_ark_dev_set_link_up, + .dev_set_link_down = eth_ark_dev_set_link_down, + + .rx_queue_start = eth_ark_rx_start_queue, + .rx_queue_stop = eth_ark_rx_stop_queue, + + .tx_queue_start = eth_ark_tx_queue_start, + .tx_queue_stop = eth_ark_tx_queue_stop, + + .stats_get = eth_ark_dev_stats_get, + .stats_reset = eth_ark_dev_stats_reset, + + .mac_addr_add = eth_ark_macaddr_add, + .mac_addr_remove = eth_ark_macaddr_remove, + .mac_addr_set = eth_ark_set_default_mac_addr, + + .mtu_set = eth_ark_set_mtu, +}; + +static int +check_for_ext(struct ark_adapter *ark) +{ + int found = 0; + + /* Get the env */ + const char *dllpath = getenv("ARK_EXT_PATH"); + + if (dllpath == NULL) { + PMD_DEBUG_LOG(DEBUG, "ARK EXT NO dll path specified\n"); + return 0; + } + PMD_DRV_LOG(INFO, "ARK EXT found dll path at %s\n", dllpath); + + /* Open and load the .so */ + ark->d_handle = dlopen(dllpath, RTLD_LOCAL | RTLD_LAZY); + if (ark->d_handle == NULL) { + PMD_DRV_LOG(ERR, "Could not load user extension %s\n", + dllpath); + return -1; + } + PMD_DRV_LOG(INFO, "SUCCESS: loaded user extension %s\n", + dllpath); + + /* Get the entry points */ + ark->user_ext.dev_init = + (void *(*)(struct rte_eth_dev *, void *, int)) + dlsym(ark->d_handle, "dev_init"); + PMD_DEBUG_LOG(DEBUG, "device ext init pointer = %p\n", + ark->user_ext.dev_init); + ark->user_ext.dev_get_port_count = + (int (*)(struct rte_eth_dev *, void *)) + dlsym(ark->d_handle, "dev_get_port_count"); + ark->user_ext.dev_uninit = + (void (*)(struct rte_eth_dev *, void *)) + dlsym(ark->d_handle, "dev_uninit"); + ark->user_ext.dev_configure = + (int (*)(struct rte_eth_dev *, void *)) + dlsym(ark->d_handle, "dev_configure"); + ark->user_ext.dev_start = + (int (*)(struct rte_eth_dev *, void *)) + dlsym(ark->d_handle, "dev_start"); + ark->user_ext.dev_stop = + (void (*)(struct rte_eth_dev *, void *)) + dlsym(ark->d_handle, "dev_stop"); + ark->user_ext.dev_close = + (void (*)(struct rte_eth_dev *, void *)) + dlsym(ark->d_handle, "dev_close"); + ark->user_ext.link_update = + (int (*)(struct rte_eth_dev *, int, void *)) + dlsym(ark->d_handle, "link_update"); + ark->user_ext.dev_set_link_up = + (int (*)(struct rte_eth_dev *, void *)) + dlsym(ark->d_handle, "dev_set_link_up"); + ark->user_ext.dev_set_link_down = + (int (*)(struct rte_eth_dev *, void *)) + dlsym(ark->d_handle, "dev_set_link_down"); + ark->user_ext.stats_get = + (int (*)(struct rte_eth_dev *, struct rte_eth_stats *, + void *)) + dlsym(ark->d_handle, "stats_get"); + ark->user_ext.stats_reset = + (void (*)(struct rte_eth_dev *, void *)) + dlsym(ark->d_handle, "stats_reset"); + ark->user_ext.mac_addr_add = + (void (*)(struct rte_eth_dev *, struct ether_addr *, uint32_t, + uint32_t, void *)) + dlsym(ark->d_handle, "mac_addr_add"); + ark->user_ext.mac_addr_remove = + (void (*)(struct rte_eth_dev *, uint32_t, void *)) + dlsym(ark->d_handle, "mac_addr_remove"); + ark->user_ext.mac_addr_set = + (void (*)(struct rte_eth_dev *, struct ether_addr *, + void *)) + dlsym(ark->d_handle, "mac_addr_set"); + ark->user_ext.set_mtu = + (int (*)(struct rte_eth_dev *, uint16_t, + void *)) + dlsym(ark->d_handle, "set_mtu"); + + return found; +} + +static int +eth_ark_dev_init(struct rte_eth_dev *dev) +{ + struct ark_adapter *ark = + (struct ark_adapter *)dev->data->dev_private; + struct rte_pci_device *pci_dev; + int ret; + int port_count = 1; + int p; + + ark->eth_dev = dev; + + PMD_FUNC_LOG(DEBUG, "\n"); + + /* Check to see if there is an extension that we need to load */ + ret = check_for_ext(ark); + if (ret) + return ret; + pci_dev = RTE_ETH_DEV_TO_PCI(dev); + rte_eth_copy_pci_info(dev, pci_dev); + + /* Use dummy function until setup */ + dev->rx_pkt_burst = ð_ark_recv_pkts_noop; + dev->tx_pkt_burst = ð_ark_xmit_pkts_noop; + + ark->bar0 = (uint8_t *)pci_dev->mem_resource[0].addr; + ark->a_bar = (uint8_t *)pci_dev->mem_resource[2].addr; + + ark->sysctrl.v = (void *)&ark->bar0[ARK_SYSCTRL_BASE]; + ark->mpurx.v = (void *)&ark->bar0[ARK_MPU_RX_BASE]; + ark->udm.v = (void *)&ark->bar0[ARK_UDM_BASE]; + ark->mputx.v = (void *)&ark->bar0[ARK_MPU_TX_BASE]; + ark->ddm.v = (void *)&ark->bar0[ARK_DDM_BASE]; + ark->cmac.v = (void *)&ark->bar0[ARK_CMAC_BASE]; + ark->external.v = (void *)&ark->bar0[ARK_EXTERNAL_BASE]; + ark->pktdir.v = (void *)&ark->bar0[ARK_PKTDIR_BASE]; + ark->pktgen.v = (void *)&ark->bar0[ARK_PKTGEN_BASE]; + ark->pktchkr.v = (void *)&ark->bar0[ARK_PKTCHKR_BASE]; + + ark->rqpacing = + (struct ark_rqpace_t *)(ark->bar0 + ARK_RCPACING_BASE); + ark->started = 0; + + PMD_DEBUG_LOG(INFO, "Sys Ctrl Const = 0x%x HW Commit_ID: %08x\n", + ark->sysctrl.t32[4], + rte_be_to_cpu_32(ark->sysctrl.t32[0x20 / 4])); + PMD_DRV_LOG(INFO, "Arkville HW Commit_ID: %08x\n", + rte_be_to_cpu_32(ark->sysctrl.t32[0x20 / 4])); + + /* If HW sanity test fails, return an error */ + if (ark->sysctrl.t32[4] != 0xcafef00d) { + PMD_DRV_LOG(ERR, + "HW Sanity test has failed, expected constant" + " 0x%x, read 0x%x (%s)\n", + 0xcafef00d, + ark->sysctrl.t32[4], __func__); + return -1; + } + if (ark->sysctrl.t32[3] != 0) { + if (ark_rqp_lasped(ark->rqpacing)) { + PMD_DRV_LOG(ERR, "Arkville Evaluation System - " + "Timer has Expired\n"); + return -1; + } + PMD_DRV_LOG(WARNING, "Arkville Evaluation System - " + "Timer is Running\n"); + } + + PMD_DRV_LOG(INFO, + "HW Sanity test has PASSED, expected constant" + " 0x%x, read 0x%x (%s)\n", + 0xcafef00d, ark->sysctrl.t32[4], __func__); + + /* We are a single function multi-port device. */ + ret = ark_config_device(dev); + dev->dev_ops = &ark_eth_dev_ops; + + dev->data->mac_addrs = rte_zmalloc("ark", ETHER_ADDR_LEN, 0); + if (!dev->data->mac_addrs) { + PMD_DRV_LOG(ERR, + "Failed to allocated memory for storing mac address" + ); + } + + if (ark->user_ext.dev_init) { + ark->user_data[dev->data->port_id] = + ark->user_ext.dev_init(dev, ark->a_bar, 0); + if (!ark->user_data[dev->data->port_id]) { + PMD_DRV_LOG(INFO, + "Failed to initialize PMD extension!" + " continuing without it\n"); + memset(&ark->user_ext, 0, sizeof(struct ark_user_ext)); + dlclose(ark->d_handle); + } + } + + if (pci_dev->device.devargs) + ret = eth_ark_check_args(ark, pci_dev->device.devargs->args); + else + PMD_DRV_LOG(INFO, "No Device args found\n"); + + if (ret) + goto error; + /* + * We will create additional devices based on the number of requested + * ports + */ + if (ark->user_ext.dev_get_port_count) + port_count = + ark->user_ext.dev_get_port_count(dev, + ark->user_data[dev->data->port_id]); + ark->num_ports = port_count; + + for (p = 0; p < port_count; p++) { + struct rte_eth_dev *eth_dev; + char name[RTE_ETH_NAME_MAX_LEN]; + + snprintf(name, sizeof(name), "arketh%d", + dev->data->port_id + p); + + if (p == 0) { + /* First port is already allocated by DPDK */ + eth_dev = ark->eth_dev; + continue; + } + + /* reserve an ethdev entry */ + eth_dev = rte_eth_dev_allocate(name); + if (!eth_dev) { + PMD_DRV_LOG(ERR, + "Could not allocate eth_dev for port %d\n", + p); + goto error; + } + + eth_dev->device = &pci_dev->device; + eth_dev->data->dev_private = ark; + eth_dev->dev_ops = ark->eth_dev->dev_ops; + eth_dev->tx_pkt_burst = ark->eth_dev->tx_pkt_burst; + eth_dev->rx_pkt_burst = ark->eth_dev->rx_pkt_burst; + + rte_eth_copy_pci_info(eth_dev, pci_dev); + + eth_dev->data->mac_addrs = rte_zmalloc(name, ETHER_ADDR_LEN, 0); + if (!eth_dev->data->mac_addrs) { + PMD_DRV_LOG(ERR, + "Memory allocation for MAC failed!" + " Exiting.\n"); + goto error; + } + + if (ark->user_ext.dev_init) { + ark->user_data[eth_dev->data->port_id] = + ark->user_ext.dev_init(dev, ark->a_bar, p); + } + } + + return ret; + + error: + if (dev->data->mac_addrs) + rte_free(dev->data->mac_addrs); + return -1; +} + +/* + *Initial device configuration when device is opened + * setup the DDM, and UDM + * Called once per PCIE device + */ +static int +ark_config_device(struct rte_eth_dev *dev) +{ + struct ark_adapter *ark = + (struct ark_adapter *)dev->data->dev_private; + uint16_t num_q, i; + struct ark_mpu_t *mpu; + + /* + * Make sure that the packet director, generator and checker are in a + * known state + */ + ark->start_pg = 0; + ark->pg = ark_pktgen_init(ark->pktgen.v, 0, 1); + if (ark->pg == NULL) + return -1; + ark_pktgen_reset(ark->pg); + ark->pc = ark_pktchkr_init(ark->pktchkr.v, 0, 1); + if (ark->pc == NULL) + return -1; + ark_pktchkr_stop(ark->pc); + ark->pd = ark_pktdir_init(ark->pktdir.v); + if (ark->pd == NULL) + return -1; + + /* Verify HW */ + if (ark_udm_verify(ark->udm.v)) + return -1; + if (ark_ddm_verify(ark->ddm.v)) + return -1; + + /* UDM */ + if (ark_udm_reset(ark->udm.v)) { + PMD_DRV_LOG(ERR, "Unable to stop and reset UDM\n"); + return -1; + } + /* Keep in reset until the MPU are cleared */ + + /* MPU reset */ + mpu = ark->mpurx.v; + num_q = ark_api_num_queues(mpu); + ark->rx_queues = num_q; + for (i = 0; i < num_q; i++) { + ark_mpu_reset(mpu); + mpu = RTE_PTR_ADD(mpu, ARK_MPU_QOFFSET); + } + + ark_udm_stop(ark->udm.v, 0); + ark_udm_configure(ark->udm.v, + RTE_PKTMBUF_HEADROOM, + RTE_MBUF_DEFAULT_DATAROOM, + ARK_RX_WRITE_TIME_NS); + ark_udm_stats_reset(ark->udm.v); + ark_udm_stop(ark->udm.v, 0); + + /* TX -- DDM */ + if (ark_ddm_stop(ark->ddm.v, 1)) + PMD_DRV_LOG(ERR, "Unable to stop DDM\n"); + + mpu = ark->mputx.v; + num_q = ark_api_num_queues(mpu); + ark->tx_queues = num_q; + for (i = 0; i < num_q; i++) { + ark_mpu_reset(mpu); + mpu = RTE_PTR_ADD(mpu, ARK_MPU_QOFFSET); + } + + ark_ddm_reset(ark->ddm.v); + ark_ddm_stats_reset(ark->ddm.v); + + ark_ddm_stop(ark->ddm.v, 0); + ark_rqp_stats_reset(ark->rqpacing); + + return 0; +} + +static int +eth_ark_dev_uninit(struct rte_eth_dev *dev) +{ + struct ark_adapter *ark = + (struct ark_adapter *)dev->data->dev_private; + + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return 0; + + if (ark->user_ext.dev_uninit) + ark->user_ext.dev_uninit(dev, + ark->user_data[dev->data->port_id]); + + ark_pktgen_uninit(ark->pg); + ark_pktchkr_uninit(ark->pc); + + dev->dev_ops = NULL; + dev->rx_pkt_burst = NULL; + dev->tx_pkt_burst = NULL; + rte_free(dev->data->mac_addrs); + return 0; +} + +static int +eth_ark_dev_configure(struct rte_eth_dev *dev) +{ + PMD_FUNC_LOG(DEBUG, "\n"); + struct ark_adapter *ark = + (struct ark_adapter *)dev->data->dev_private; + + eth_ark_dev_set_link_up(dev); + if (ark->user_ext.dev_configure) + return ark->user_ext.dev_configure(dev, + ark->user_data[dev->data->port_id]); + return 0; +} + +static void * +delay_pg_start(void *arg) +{ + struct ark_adapter *ark = (struct ark_adapter *)arg; + + /* This function is used exclusively for regression testing, We + * perform a blind sleep here to ensure that the external test + * application has time to setup the test before we generate packets + */ + usleep(100000); + ark_pktgen_run(ark->pg); + return NULL; +} + +static int +eth_ark_dev_start(struct rte_eth_dev *dev) +{ + struct ark_adapter *ark = + (struct ark_adapter *)dev->data->dev_private; + int i; + + PMD_FUNC_LOG(DEBUG, "\n"); + + /* RX Side */ + /* start UDM */ + ark_udm_start(ark->udm.v); + + for (i = 0; i < dev->data->nb_rx_queues; i++) + eth_ark_rx_start_queue(dev, i); + + /* TX Side */ + for (i = 0; i < dev->data->nb_tx_queues; i++) + eth_ark_tx_queue_start(dev, i); + + /* start DDM */ + ark_ddm_start(ark->ddm.v); + + ark->started = 1; + /* set xmit and receive function */ + dev->rx_pkt_burst = ð_ark_recv_pkts; + dev->tx_pkt_burst = ð_ark_xmit_pkts; + + if (ark->start_pg) + ark_pktchkr_run(ark->pc); + + if (ark->start_pg && (dev->data->port_id == 0)) { + pthread_t thread; + + /* Delay packet generatpr start allow the hardware to be ready + * This is only used for sanity checking with internal generator + */ + if (pthread_create(&thread, NULL, delay_pg_start, ark)) { + PMD_DRV_LOG(ERR, "Could not create pktgen " + "starter thread\n"); + return -1; + } + } + + if (ark->user_ext.dev_start) + ark->user_ext.dev_start(dev, + ark->user_data[dev->data->port_id]); + + return 0; +} + +static void +eth_ark_dev_stop(struct rte_eth_dev *dev) +{ + uint16_t i; + int status; + struct ark_adapter *ark = + (struct ark_adapter *)dev->data->dev_private; + struct ark_mpu_t *mpu; + + PMD_FUNC_LOG(DEBUG, "\n"); + + if (ark->started == 0) + return; + ark->started = 0; + + /* Stop the extension first */ + if (ark->user_ext.dev_stop) + ark->user_ext.dev_stop(dev, + ark->user_data[dev->data->port_id]); + + /* Stop the packet generator */ + if (ark->start_pg) + ark_pktgen_pause(ark->pg); + + dev->rx_pkt_burst = ð_ark_recv_pkts_noop; + dev->tx_pkt_burst = ð_ark_xmit_pkts_noop; + + /* STOP TX Side */ + for (i = 0; i < dev->data->nb_tx_queues; i++) { + status = eth_ark_tx_queue_stop(dev, i); + if (status != 0) { + uint16_t port = dev->data->port_id; + PMD_DRV_LOG(ERR, + "tx_queue stop anomaly" + " port %u, queue %u\n", + port, i); + } + } + + /* Stop DDM */ + /* Wait up to 0.1 second. each stop is up to 1000 * 10 useconds */ + for (i = 0; i < 10; i++) { + status = ark_ddm_stop(ark->ddm.v, 1); + if (status == 0) + break; + } + if (status || i != 0) { + PMD_DRV_LOG(ERR, "DDM stop anomaly. status:" + " %d iter: %u. (%s)\n", + status, + i, + __func__); + ark_ddm_dump(ark->ddm.v, "Stop anomaly"); + + mpu = ark->mputx.v; + for (i = 0; i < ark->tx_queues; i++) { + ark_mpu_dump(mpu, "DDM failure dump", i); + mpu = RTE_PTR_ADD(mpu, ARK_MPU_QOFFSET); + } + } + + /* STOP RX Side */ + /* Stop UDM multiple tries attempted */ + for (i = 0; i < 10; i++) { + status = ark_udm_stop(ark->udm.v, 1); + if (status == 0) + break; + } + if (status || i != 0) { + PMD_DRV_LOG(ERR, "UDM stop anomaly. status %d iter: %u. (%s)\n", + status, i, __func__); + ark_udm_dump(ark->udm.v, "Stop anomaly"); + + mpu = ark->mpurx.v; + for (i = 0; i < ark->rx_queues; i++) { + ark_mpu_dump(mpu, "UDM Stop anomaly", i); + mpu = RTE_PTR_ADD(mpu, ARK_MPU_QOFFSET); + } + } + + ark_udm_dump_stats(ark->udm.v, "Post stop"); + ark_udm_dump_perf(ark->udm.v, "Post stop"); + + for (i = 0; i < dev->data->nb_rx_queues; i++) + eth_ark_rx_dump_queue(dev, i, __func__); + + /* Stop the packet checker if it is running */ + if (ark->start_pg) { + ark_pktchkr_dump_stats(ark->pc); + ark_pktchkr_stop(ark->pc); + } +} + +static void +eth_ark_dev_close(struct rte_eth_dev *dev) +{ + struct ark_adapter *ark = + (struct ark_adapter *)dev->data->dev_private; + uint16_t i; + + if (ark->user_ext.dev_close) + ark->user_ext.dev_close(dev, + ark->user_data[dev->data->port_id]); + + eth_ark_dev_stop(dev); + eth_ark_udm_force_close(dev); + + /* + * TODO This should only be called once for the device during shutdown + */ + ark_rqp_dump(ark->rqpacing); + + for (i = 0; i < dev->data->nb_tx_queues; i++) { + eth_ark_tx_queue_release(dev->data->tx_queues[i]); + dev->data->tx_queues[i] = 0; + } + + for (i = 0; i < dev->data->nb_rx_queues; i++) { + eth_ark_dev_rx_queue_release(dev->data->rx_queues[i]); + dev->data->rx_queues[i] = 0; + } +} + +static void +eth_ark_dev_info_get(struct rte_eth_dev *dev, + struct rte_eth_dev_info *dev_info) +{ + struct ark_adapter *ark = + (struct ark_adapter *)dev->data->dev_private; + struct ark_mpu_t *tx_mpu = RTE_PTR_ADD(ark->bar0, ARK_MPU_TX_BASE); + struct ark_mpu_t *rx_mpu = RTE_PTR_ADD(ark->bar0, ARK_MPU_RX_BASE); + uint16_t ports = ark->num_ports; + + dev_info->max_rx_pktlen = ARK_RX_MAX_PKT_LEN; + dev_info->min_rx_bufsize = ARK_RX_MIN_BUFSIZE; + + dev_info->max_rx_queues = ark_api_num_queues_per_port(rx_mpu, ports); + dev_info->max_tx_queues = ark_api_num_queues_per_port(tx_mpu, ports); + + dev_info->rx_desc_lim = (struct rte_eth_desc_lim) { + .nb_max = ARK_RX_MAX_QUEUE, + .nb_min = ARK_RX_MIN_QUEUE, + .nb_align = ARK_RX_MIN_QUEUE}; /* power of 2 */ + + dev_info->tx_desc_lim = (struct rte_eth_desc_lim) { + .nb_max = ARK_TX_MAX_QUEUE, + .nb_min = ARK_TX_MIN_QUEUE, + .nb_align = ARK_TX_MIN_QUEUE}; /* power of 2 */ + + /* ARK PMD supports all line rates, how do we indicate that here ?? */ + dev_info->speed_capa = (ETH_LINK_SPEED_1G | + ETH_LINK_SPEED_10G | + ETH_LINK_SPEED_25G | + ETH_LINK_SPEED_40G | + ETH_LINK_SPEED_50G | + ETH_LINK_SPEED_100G); + dev_info->pci_dev = RTE_ETH_DEV_TO_PCI(dev); +} + +static int +eth_ark_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete) +{ + PMD_DEBUG_LOG(DEBUG, "link status = %d\n", + dev->data->dev_link.link_status); + struct ark_adapter *ark = + (struct ark_adapter *)dev->data->dev_private; + + if (ark->user_ext.link_update) { + return ark->user_ext.link_update + (dev, wait_to_complete, + ark->user_data[dev->data->port_id]); + } + return 0; +} + +static int +eth_ark_dev_set_link_up(struct rte_eth_dev *dev) +{ + dev->data->dev_link.link_status = 1; + struct ark_adapter *ark = + (struct ark_adapter *)dev->data->dev_private; + + if (ark->user_ext.dev_set_link_up) + return ark->user_ext.dev_set_link_up(dev, + ark->user_data[dev->data->port_id]); + return 0; +} + +static int +eth_ark_dev_set_link_down(struct rte_eth_dev *dev) +{ + dev->data->dev_link.link_status = 0; + struct ark_adapter *ark = + (struct ark_adapter *)dev->data->dev_private; + + if (ark->user_ext.dev_set_link_down) + return ark->user_ext.dev_set_link_down(dev, + ark->user_data[dev->data->port_id]); + return 0; +} + +static int +eth_ark_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) +{ + uint16_t i; + struct ark_adapter *ark = + (struct ark_adapter *)dev->data->dev_private; + + stats->ipackets = 0; + stats->ibytes = 0; + stats->opackets = 0; + stats->obytes = 0; + stats->imissed = 0; + stats->oerrors = 0; + + for (i = 0; i < dev->data->nb_tx_queues; i++) + eth_tx_queue_stats_get(dev->data->tx_queues[i], stats); + for (i = 0; i < dev->data->nb_rx_queues; i++) + eth_rx_queue_stats_get(dev->data->rx_queues[i], stats); + if (ark->user_ext.stats_get) + return ark->user_ext.stats_get(dev, stats, + ark->user_data[dev->data->port_id]); + return 0; +} + +static void +eth_ark_dev_stats_reset(struct rte_eth_dev *dev) +{ + uint16_t i; + struct ark_adapter *ark = + (struct ark_adapter *)dev->data->dev_private; + + for (i = 0; i < dev->data->nb_tx_queues; i++) + eth_tx_queue_stats_reset(dev->data->tx_queues[i]); + for (i = 0; i < dev->data->nb_rx_queues; i++) + eth_rx_queue_stats_reset(dev->data->rx_queues[i]); + if (ark->user_ext.stats_reset) + ark->user_ext.stats_reset(dev, + ark->user_data[dev->data->port_id]); +} + +static int +eth_ark_macaddr_add(struct rte_eth_dev *dev, + struct ether_addr *mac_addr, + uint32_t index, + uint32_t pool) +{ + struct ark_adapter *ark = + (struct ark_adapter *)dev->data->dev_private; + + if (ark->user_ext.mac_addr_add) { + ark->user_ext.mac_addr_add(dev, + mac_addr, + index, + pool, + ark->user_data[dev->data->port_id]); + return 0; + } + return -ENOTSUP; +} + +static void +eth_ark_macaddr_remove(struct rte_eth_dev *dev, uint32_t index) +{ + struct ark_adapter *ark = + (struct ark_adapter *)dev->data->dev_private; + + if (ark->user_ext.mac_addr_remove) + ark->user_ext.mac_addr_remove(dev, index, + ark->user_data[dev->data->port_id]); +} + +static void +eth_ark_set_default_mac_addr(struct rte_eth_dev *dev, + struct ether_addr *mac_addr) +{ + struct ark_adapter *ark = + (struct ark_adapter *)dev->data->dev_private; + + if (ark->user_ext.mac_addr_set) + ark->user_ext.mac_addr_set(dev, mac_addr, + ark->user_data[dev->data->port_id]); +} + +static int +eth_ark_set_mtu(struct rte_eth_dev *dev, uint16_t size) +{ + struct ark_adapter *ark = + (struct ark_adapter *)dev->data->dev_private; + + if (ark->user_ext.set_mtu) + return ark->user_ext.set_mtu(dev, size, + ark->user_data[dev->data->port_id]); + + return -ENOTSUP; +} + +static inline int +process_pktdir_arg(const char *key, const char *value, + void *extra_args) +{ + PMD_FUNC_LOG(DEBUG, "key = %s, value = %s\n", + key, value); + struct ark_adapter *ark = + (struct ark_adapter *)extra_args; + + ark->pkt_dir_v = strtol(value, NULL, 16); + PMD_FUNC_LOG(DEBUG, "pkt_dir_v = 0x%x\n", ark->pkt_dir_v); + return 0; +} + +static inline int +process_file_args(const char *key, const char *value, void *extra_args) +{ + PMD_FUNC_LOG(DEBUG, "key = %s, value = %s\n", + key, value); + char *args = (char *)extra_args; + + /* Open the configuration file */ + FILE *file = fopen(value, "r"); + char line[ARK_MAX_ARG_LEN]; + int size = 0; + int first = 1; + + if (file == NULL) { + PMD_DRV_LOG(ERR, "Unable to open " + "config file %s\n", value); + return -1; + } + + while (fgets(line, sizeof(line), file)) { + size += strlen(line); + if (size >= ARK_MAX_ARG_LEN) { + PMD_DRV_LOG(ERR, "Unable to parse file %s args, " + "parameter list is too long\n", value); + fclose(file); + return -1; + } + if (first) { + strncpy(args, line, ARK_MAX_ARG_LEN); + first = 0; + } else { + strncat(args, line, ARK_MAX_ARG_LEN); + } + } + PMD_FUNC_LOG(DEBUG, "file = %s\n", args); + fclose(file); + return 0; +} + +static int +eth_ark_check_args(struct ark_adapter *ark, const char *params) +{ + struct rte_kvargs *kvlist; + unsigned int k_idx; + struct rte_kvargs_pair *pair = NULL; + int ret = -1; + + kvlist = rte_kvargs_parse(params, valid_arguments); + if (kvlist == NULL) + return 0; + + ark->pkt_gen_args[0] = 0; + ark->pkt_chkr_args[0] = 0; + + for (k_idx = 0; k_idx < kvlist->count; k_idx++) { + pair = &kvlist->pairs[k_idx]; + PMD_FUNC_LOG(DEBUG, "**** Arg passed to PMD = %s:%s\n", + pair->key, + pair->value); + } + + if (rte_kvargs_process(kvlist, + ARK_PKTDIR_ARG, + &process_pktdir_arg, + ark) != 0) { + PMD_DRV_LOG(ERR, "Unable to parse arg %s\n", ARK_PKTDIR_ARG); + goto free_kvlist; + } + + if (rte_kvargs_process(kvlist, + ARK_PKTGEN_ARG, + &process_file_args, + ark->pkt_gen_args) != 0) { + PMD_DRV_LOG(ERR, "Unable to parse arg %s\n", ARK_PKTGEN_ARG); + goto free_kvlist; + } + + if (rte_kvargs_process(kvlist, + ARK_PKTCHKR_ARG, + &process_file_args, + ark->pkt_chkr_args) != 0) { + PMD_DRV_LOG(ERR, "Unable to parse arg %s\n", ARK_PKTCHKR_ARG); + goto free_kvlist; + } + + PMD_DRV_LOG(INFO, "packet director set to 0x%x\n", ark->pkt_dir_v); + /* Setup the packet director */ + ark_pktdir_setup(ark->pd, ark->pkt_dir_v); + + /* Setup the packet generator */ + if (ark->pkt_gen_args[0]) { + PMD_DRV_LOG(INFO, "Setting up the packet generator\n"); + ark_pktgen_parse(ark->pkt_gen_args); + ark_pktgen_reset(ark->pg); + ark_pktgen_setup(ark->pg); + ark->start_pg = 1; + } + + /* Setup the packet checker */ + if (ark->pkt_chkr_args[0]) { + ark_pktchkr_parse(ark->pkt_chkr_args); + ark_pktchkr_setup(ark->pc); + } + + ret = 0; + +free_kvlist: + rte_kvargs_free(kvlist); + + return ret; +} + +RTE_PMD_REGISTER_PCI(net_ark, rte_ark_pmd); +RTE_PMD_REGISTER_KMOD_DEP(net_ark, "* igb_uio | uio_pci_generic "); +RTE_PMD_REGISTER_PCI_TABLE(net_ark, pci_id_ark_map); +RTE_PMD_REGISTER_PARAM_STRING(net_ark, + ARK_PKTGEN_ARG "= " + ARK_PKTCHKR_ARG "= " + ARK_PKTDIR_ARG "="); diff --git a/dpdk/drivers/net/ark/ark_ethdev_rx.c b/dpdk/drivers/net/ark/ark_ethdev_rx.c new file mode 100644 index 00000000..987d085e --- /dev/null +++ b/dpdk/drivers/net/ark/ark_ethdev_rx.c @@ -0,0 +1,672 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2015-2017 Atomic Rules LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +#include "ark_ethdev_rx.h" +#include "ark_global.h" +#include "ark_logs.h" +#include "ark_mpu.h" +#include "ark_udm.h" + +#define ARK_RX_META_SIZE 32 +#define ARK_RX_META_OFFSET (RTE_PKTMBUF_HEADROOM - ARK_RX_META_SIZE) +#define ARK_RX_MAX_NOCHAIN (RTE_MBUF_DEFAULT_DATAROOM) + +/* Forward declarations */ +struct ark_rx_queue; +struct ark_rx_meta; + +static void dump_mbuf_data(struct rte_mbuf *mbuf, uint16_t lo, uint16_t hi); +static void ark_ethdev_rx_dump(const char *name, struct ark_rx_queue *queue); +static uint32_t eth_ark_rx_jumbo(struct ark_rx_queue *queue, + struct ark_rx_meta *meta, + struct rte_mbuf *mbuf0, + uint32_t cons_index); +static inline int eth_ark_rx_seed_mbufs(struct ark_rx_queue *queue); + +/* ************************************************************************* */ +struct ark_rx_queue { + /* array of mbufs to populate */ + struct rte_mbuf **reserve_q; + /* array of physical addresses of the mbuf data pointer */ + /* This point is a virtual address */ + rte_iova_t *paddress_q; + struct rte_mempool *mb_pool; + + struct ark_udm_t *udm; + struct ark_mpu_t *mpu; + + uint32_t queue_size; + uint32_t queue_mask; + + uint32_t seed_index; /* step 1 set with empty mbuf */ + uint32_t cons_index; /* step 3 consumed by driver */ + + /* The queue Id is used to identify the HW Q */ + uint16_t phys_qid; + + /* The queue Index is used within the dpdk device structures */ + uint16_t queue_index; + + uint32_t pad1; + + /* separate cache line */ + /* second cache line - fields only used in slow path */ + MARKER cacheline1 __rte_cache_min_aligned; + + volatile uint32_t prod_index; /* step 2 filled by FPGA */ +} __rte_cache_aligned; + + +/* ************************************************************************* */ +static int +eth_ark_rx_hw_setup(struct rte_eth_dev *dev, + struct ark_rx_queue *queue, + uint16_t rx_queue_id __rte_unused, uint16_t rx_queue_idx) +{ + rte_iova_t queue_base; + rte_iova_t phys_addr_q_base; + rte_iova_t phys_addr_prod_index; + + queue_base = rte_malloc_virt2iova(queue); + phys_addr_prod_index = queue_base + + offsetof(struct ark_rx_queue, prod_index); + + phys_addr_q_base = rte_malloc_virt2iova(queue->paddress_q); + + /* Verify HW */ + if (ark_mpu_verify(queue->mpu, sizeof(rte_iova_t))) { + PMD_DRV_LOG(ERR, "Illegal configuration rx queue\n"); + return -1; + } + + /* Stop and Reset and configure MPU */ + ark_mpu_configure(queue->mpu, phys_addr_q_base, queue->queue_size, 0); + + ark_udm_write_addr(queue->udm, phys_addr_prod_index); + + /* advance the valid pointer, but don't start until the queue starts */ + ark_mpu_reset_stats(queue->mpu); + + /* The seed is the producer index for the HW */ + ark_mpu_set_producer(queue->mpu, queue->seed_index); + dev->data->rx_queue_state[rx_queue_idx] = RTE_ETH_QUEUE_STATE_STOPPED; + + return 0; +} + +static inline void +eth_ark_rx_update_cons_index(struct ark_rx_queue *queue, uint32_t cons_index) +{ + queue->cons_index = cons_index; + eth_ark_rx_seed_mbufs(queue); + ark_mpu_set_producer(queue->mpu, queue->seed_index); +} + +/* ************************************************************************* */ +int +eth_ark_dev_rx_queue_setup(struct rte_eth_dev *dev, + uint16_t queue_idx, + uint16_t nb_desc, + unsigned int socket_id, + const struct rte_eth_rxconf *rx_conf, + struct rte_mempool *mb_pool) +{ + static int warning1; /* = 0 */ + struct ark_adapter *ark = (struct ark_adapter *)dev->data->dev_private; + + struct ark_rx_queue *queue; + uint32_t i; + int status; + + /* Future works: divide the Q's evenly with multi-ports */ + int port = dev->data->port_id; + int qidx = port + queue_idx; + + /* We may already be setup, free memory prior to re-allocation */ + if (dev->data->rx_queues[queue_idx] != NULL) { + eth_ark_dev_rx_queue_release(dev->data->rx_queues[queue_idx]); + dev->data->rx_queues[queue_idx] = NULL; + } + + if (rx_conf != NULL && warning1 == 0) { + warning1 = 1; + PMD_DRV_LOG(INFO, + "Arkville ignores rte_eth_rxconf argument.\n"); + } + + if (RTE_PKTMBUF_HEADROOM < ARK_RX_META_SIZE) { + PMD_DRV_LOG(ERR, + "Error: DPDK Arkville requires head room > %d bytes (%s)\n", + ARK_RX_META_SIZE, __func__); + return -1; /* ERROR CODE */ + } + + if (!rte_is_power_of_2(nb_desc)) { + PMD_DRV_LOG(ERR, + "DPDK Arkville configuration queue size must be power of two %u (%s)\n", + nb_desc, __func__); + return -1; /* ERROR CODE */ + } + + /* Allocate queue struct */ + queue = rte_zmalloc_socket("Ark_rxqueue", + sizeof(struct ark_rx_queue), + 64, + socket_id); + if (queue == 0) { + PMD_DRV_LOG(ERR, "Failed to allocate memory in %s\n", __func__); + return -ENOMEM; + } + + /* NOTE zmalloc is used, no need to 0 indexes, etc. */ + queue->mb_pool = mb_pool; + queue->phys_qid = qidx; + queue->queue_index = queue_idx; + queue->queue_size = nb_desc; + queue->queue_mask = nb_desc - 1; + + queue->reserve_q = + rte_zmalloc_socket("Ark_rx_queue mbuf", + nb_desc * sizeof(struct rte_mbuf *), + 64, + socket_id); + queue->paddress_q = + rte_zmalloc_socket("Ark_rx_queue paddr", + nb_desc * sizeof(rte_iova_t), + 64, + socket_id); + + if (queue->reserve_q == 0 || queue->paddress_q == 0) { + PMD_DRV_LOG(ERR, + "Failed to allocate queue memory in %s\n", + __func__); + rte_free(queue->reserve_q); + rte_free(queue->paddress_q); + rte_free(queue); + return -ENOMEM; + } + + dev->data->rx_queues[queue_idx] = queue; + queue->udm = RTE_PTR_ADD(ark->udm.v, qidx * ARK_UDM_QOFFSET); + queue->mpu = RTE_PTR_ADD(ark->mpurx.v, qidx * ARK_MPU_QOFFSET); + + /* populate mbuf reserve */ + status = eth_ark_rx_seed_mbufs(queue); + + /* MPU Setup */ + if (status == 0) + status = eth_ark_rx_hw_setup(dev, queue, qidx, queue_idx); + + if (unlikely(status != 0)) { + struct rte_mbuf *mbuf; + + PMD_DRV_LOG(ERR, "Failed to initialize RX queue %d %s\n", + qidx, + __func__); + /* Free the mbufs allocated */ + for (i = 0, mbuf = queue->reserve_q[0]; + i < nb_desc; ++i, mbuf++) { + rte_pktmbuf_free(mbuf); + } + rte_free(queue->reserve_q); + rte_free(queue->paddress_q); + rte_free(queue); + return -1; /* ERROR CODE */ + } + + return 0; +} + +/* ************************************************************************* */ +uint16_t +eth_ark_recv_pkts_noop(void *rx_queue __rte_unused, + struct rte_mbuf **rx_pkts __rte_unused, + uint16_t nb_pkts __rte_unused) +{ + return 0; +} + +/* ************************************************************************* */ +uint16_t +eth_ark_recv_pkts(void *rx_queue, + struct rte_mbuf **rx_pkts, + uint16_t nb_pkts) +{ + struct ark_rx_queue *queue; + register uint32_t cons_index, prod_index; + uint16_t nb; + struct rte_mbuf *mbuf; + struct ark_rx_meta *meta; + + queue = (struct ark_rx_queue *)rx_queue; + if (unlikely(queue == 0)) + return 0; + if (unlikely(nb_pkts == 0)) + return 0; + prod_index = queue->prod_index; + cons_index = queue->cons_index; + nb = 0; + + while (prod_index != cons_index) { + mbuf = queue->reserve_q[cons_index & queue->queue_mask]; + /* prefetch mbuf */ + rte_mbuf_prefetch_part1(mbuf); + rte_mbuf_prefetch_part2(mbuf); + + /* META DATA embedded in headroom */ + meta = RTE_PTR_ADD(mbuf->buf_addr, ARK_RX_META_OFFSET); + + mbuf->port = meta->port; + mbuf->pkt_len = meta->pkt_len; + mbuf->data_len = meta->pkt_len; + mbuf->timestamp = meta->timestamp; + mbuf->udata64 = meta->user_data; + + if (ARK_RX_DEBUG) { /* debug sanity checks */ + if ((meta->pkt_len > (1024 * 16)) || + (meta->pkt_len == 0)) { + PMD_RX_LOG(DEBUG, "RX: Bad Meta Q: %u" + " cons: %" PRIU32 + " prod: %" PRIU32 + " seed_index %" PRIU32 + "\n", + queue->phys_qid, + cons_index, + queue->prod_index, + queue->seed_index); + + + PMD_RX_LOG(DEBUG, " : UDM" + " prod: %" PRIU32 + " len: %u\n", + queue->udm->rt_cfg.prod_idx, + meta->pkt_len); + ark_mpu_dump(queue->mpu, + " ", + queue->phys_qid); + dump_mbuf_data(mbuf, 0, 256); + /* its FUBAR so fix it */ + mbuf->pkt_len = 63; + meta->pkt_len = 63; + } + /* seqn is only set under debug */ + mbuf->seqn = cons_index; + } + + if (unlikely(meta->pkt_len > ARK_RX_MAX_NOCHAIN)) + cons_index = eth_ark_rx_jumbo + (queue, meta, mbuf, cons_index + 1); + else + cons_index += 1; + + rx_pkts[nb] = mbuf; + nb++; + if (nb >= nb_pkts) + break; + } + + if (unlikely(nb != 0)) + /* report next free to FPGA */ + eth_ark_rx_update_cons_index(queue, cons_index); + + return nb; +} + +/* ************************************************************************* */ +static uint32_t +eth_ark_rx_jumbo(struct ark_rx_queue *queue, + struct ark_rx_meta *meta, + struct rte_mbuf *mbuf0, + uint32_t cons_index) +{ + struct rte_mbuf *mbuf_prev; + struct rte_mbuf *mbuf; + + uint16_t remaining; + uint16_t data_len; + uint16_t segments; + + /* first buf populated by called */ + mbuf_prev = mbuf0; + segments = 1; + data_len = RTE_MIN(meta->pkt_len, RTE_MBUF_DEFAULT_DATAROOM); + remaining = meta->pkt_len - data_len; + mbuf0->data_len = data_len; + + /* HW guarantees that the data does not exceed prod_index! */ + while (remaining != 0) { + data_len = RTE_MIN(remaining, + RTE_MBUF_DEFAULT_DATAROOM + + RTE_PKTMBUF_HEADROOM); + + remaining -= data_len; + segments += 1; + + mbuf = queue->reserve_q[cons_index & queue->queue_mask]; + mbuf_prev->next = mbuf; + mbuf_prev = mbuf; + mbuf->data_len = data_len; + mbuf->data_off = 0; + if (ARK_RX_DEBUG) + mbuf->seqn = cons_index; /* for debug only */ + + cons_index += 1; + } + + mbuf0->nb_segs = segments; + return cons_index; +} + +/* Drain the internal queue allowing hw to clear out. */ +static void +eth_ark_rx_queue_drain(struct ark_rx_queue *queue) +{ + register uint32_t cons_index; + struct rte_mbuf *mbuf; + + cons_index = queue->cons_index; + + /* NOT performance optimized, since this is a one-shot call */ + while ((cons_index ^ queue->prod_index) & queue->queue_mask) { + mbuf = queue->reserve_q[cons_index & queue->queue_mask]; + rte_pktmbuf_free(mbuf); + cons_index++; + eth_ark_rx_update_cons_index(queue, cons_index); + } +} + +uint32_t +eth_ark_dev_rx_queue_count(struct rte_eth_dev *dev, uint16_t queue_id) +{ + struct ark_rx_queue *queue; + + queue = dev->data->rx_queues[queue_id]; + return (queue->prod_index - queue->cons_index); /* mod arith */ +} + +/* ************************************************************************* */ +int +eth_ark_rx_start_queue(struct rte_eth_dev *dev, uint16_t queue_id) +{ + struct ark_rx_queue *queue; + + queue = dev->data->rx_queues[queue_id]; + if (queue == 0) + return -1; + + dev->data->rx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STARTED; + + ark_mpu_set_producer(queue->mpu, queue->seed_index); + ark_mpu_start(queue->mpu); + + ark_udm_queue_enable(queue->udm, 1); + + return 0; +} + +/* ************************************************************************* */ + +/* Queue can be restarted. data remains + */ +int +eth_ark_rx_stop_queue(struct rte_eth_dev *dev, uint16_t queue_id) +{ + struct ark_rx_queue *queue; + + queue = dev->data->rx_queues[queue_id]; + if (queue == 0) + return -1; + + ark_udm_queue_enable(queue->udm, 0); + + dev->data->rx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; + + return 0; +} + +/* ************************************************************************* */ +static inline int +eth_ark_rx_seed_mbufs(struct ark_rx_queue *queue) +{ + uint32_t limit = queue->cons_index + queue->queue_size; + uint32_t seed_index = queue->seed_index; + + uint32_t count = 0; + uint32_t seed_m = queue->seed_index & queue->queue_mask; + + uint32_t nb = limit - seed_index; + + /* Handle wrap around -- remainder is filled on the next call */ + if (unlikely(seed_m + nb > queue->queue_size)) + nb = queue->queue_size - seed_m; + + struct rte_mbuf **mbufs = &queue->reserve_q[seed_m]; + int status = rte_pktmbuf_alloc_bulk(queue->mb_pool, mbufs, nb); + + if (unlikely(status != 0)) + return -1; + + if (ARK_RX_DEBUG) { /* DEBUG */ + while (count != nb) { + struct rte_mbuf *mbuf_init = + queue->reserve_q[seed_m + count]; + + memset(mbuf_init->buf_addr, -1, 512); + *((uint32_t *)mbuf_init->buf_addr) = + seed_index + count; + *(uint16_t *)RTE_PTR_ADD(mbuf_init->buf_addr, 4) = + queue->phys_qid; + count++; + } + count = 0; + } /* DEBUG */ + queue->seed_index += nb; + + /* Duff's device https://en.wikipedia.org/wiki/Duff's_device */ + switch (nb % 4) { + case 0: + while (count != nb) { + queue->paddress_q[seed_m++] = + (*mbufs++)->buf_iova; + count++; + /* FALLTHROUGH */ + case 3: + queue->paddress_q[seed_m++] = + (*mbufs++)->buf_iova; + count++; + /* FALLTHROUGH */ + case 2: + queue->paddress_q[seed_m++] = + (*mbufs++)->buf_iova; + count++; + /* FALLTHROUGH */ + case 1: + queue->paddress_q[seed_m++] = + (*mbufs++)->buf_iova; + count++; + /* FALLTHROUGH */ + + } /* while (count != nb) */ + } /* switch */ + + return 0; +} + +void +eth_ark_rx_dump_queue(struct rte_eth_dev *dev, uint16_t queue_id, + const char *msg) +{ + struct ark_rx_queue *queue; + + queue = dev->data->rx_queues[queue_id]; + + ark_ethdev_rx_dump(msg, queue); +} + +/* ************************************************************************* */ +/* Call on device closed no user API, queue is stopped */ +void +eth_ark_dev_rx_queue_release(void *vqueue) +{ + struct ark_rx_queue *queue; + uint32_t i; + + queue = (struct ark_rx_queue *)vqueue; + if (queue == 0) + return; + + ark_udm_queue_enable(queue->udm, 0); + /* Stop the MPU since pointer are going away */ + ark_mpu_stop(queue->mpu); + + /* Need to clear out mbufs here, dropping packets along the way */ + eth_ark_rx_queue_drain(queue); + + for (i = 0; i < queue->queue_size; ++i) + rte_pktmbuf_free(queue->reserve_q[i]); + + rte_free(queue->reserve_q); + rte_free(queue->paddress_q); + rte_free(queue); +} + +void +eth_rx_queue_stats_get(void *vqueue, struct rte_eth_stats *stats) +{ + struct ark_rx_queue *queue; + struct ark_udm_t *udm; + + queue = vqueue; + if (queue == 0) + return; + udm = queue->udm; + + uint64_t ibytes = ark_udm_bytes(udm); + uint64_t ipackets = ark_udm_packets(udm); + uint64_t idropped = ark_udm_dropped(queue->udm); + + stats->q_ipackets[queue->queue_index] = ipackets; + stats->q_ibytes[queue->queue_index] = ibytes; + stats->q_errors[queue->queue_index] = idropped; + stats->ipackets += ipackets; + stats->ibytes += ibytes; + stats->imissed += idropped; +} + +void +eth_rx_queue_stats_reset(void *vqueue) +{ + struct ark_rx_queue *queue; + + queue = vqueue; + if (queue == 0) + return; + + ark_mpu_reset_stats(queue->mpu); + ark_udm_queue_stats_reset(queue->udm); +} + +void +eth_ark_udm_force_close(struct rte_eth_dev *dev) +{ + struct ark_adapter *ark = (struct ark_adapter *)dev->data->dev_private; + struct ark_rx_queue *queue; + uint32_t index; + uint16_t i; + + if (!ark_udm_is_flushed(ark->udm.v)) { + /* restart the MPUs */ + PMD_DRV_LOG(ERR, "ARK: %s UDM not flushed\n", __func__); + for (i = 0; i < dev->data->nb_rx_queues; i++) { + queue = (struct ark_rx_queue *)dev->data->rx_queues[i]; + if (queue == 0) + continue; + + ark_mpu_start(queue->mpu); + /* Add some buffers */ + index = 100000 + queue->seed_index; + ark_mpu_set_producer(queue->mpu, index); + } + /* Wait to allow data to pass */ + usleep(100); + + PMD_DEBUG_LOG(DEBUG, "UDM forced flush attempt, stopped = %d\n", + ark_udm_is_flushed(ark->udm.v)); + } + ark_udm_reset(ark->udm.v); +} + +static void +ark_ethdev_rx_dump(const char *name, struct ark_rx_queue *queue) +{ + if (queue == NULL) + return; + PMD_DEBUG_LOG(DEBUG, "RX QUEUE %d -- %s", queue->phys_qid, name); + PMD_DEBUG_LOG(DEBUG, ARK_SU32 ARK_SU32 ARK_SU32 ARK_SU32 "\n", + "queue_size", queue->queue_size, + "seed_index", queue->seed_index, + "prod_index", queue->prod_index, + "cons_index", queue->cons_index); + + ark_mpu_dump(queue->mpu, name, queue->phys_qid); + ark_mpu_dump_setup(queue->mpu, queue->phys_qid); + ark_udm_dump(queue->udm, name); + ark_udm_dump_setup(queue->udm, queue->phys_qid); +} + +/* Only used in debug. + * This function is a raw memory dump of a portion of an mbuf's memory + * region. The usual function, rte_pktmbuf_dump() only shows data + * with respect to the data_off field. This function show data + * anywhere in the mbuf's buffer. This is useful for examining + * data in the headroom or tailroom portion of an mbuf. + */ +static void +dump_mbuf_data(struct rte_mbuf *mbuf, uint16_t lo, uint16_t hi) +{ + uint16_t i, j; + + PMD_DRV_LOG(INFO, " MBUF: %p len %d, off: %d, seq: %" PRIU32 "\n", mbuf, + mbuf->pkt_len, mbuf->data_off, mbuf->seqn); + for (i = lo; i < hi; i += 16) { + uint8_t *dp = RTE_PTR_ADD(mbuf->buf_addr, i); + + PMD_DRV_LOG(INFO, " %6d: ", i); + for (j = 0; j < 16; j++) + PMD_DRV_LOG(INFO, " %02x", dp[j]); + + PMD_DRV_LOG(INFO, "\n"); + } +} diff --git a/dpdk/drivers/net/ark/ark_ethdev_rx.h b/dpdk/drivers/net/ark/ark_ethdev_rx.h new file mode 100644 index 00000000..3a54a4c9 --- /dev/null +++ b/dpdk/drivers/net/ark/ark_ethdev_rx.h @@ -0,0 +1,65 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2015-2017 Atomic Rules LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _ARK_ETHDEV_RX_H_ +#define _ARK_ETHDEV_RX_H_ + +#include + +#include +#include +#include + + +int eth_ark_dev_rx_queue_setup(struct rte_eth_dev *dev, + uint16_t queue_idx, + uint16_t nb_desc, + unsigned int socket_id, + const struct rte_eth_rxconf *rx_conf, + struct rte_mempool *mp); +uint32_t eth_ark_dev_rx_queue_count(struct rte_eth_dev *dev, + uint16_t rx_queue_id); +int eth_ark_rx_stop_queue(struct rte_eth_dev *dev, uint16_t queue_id); +int eth_ark_rx_start_queue(struct rte_eth_dev *dev, uint16_t queue_id); +uint16_t eth_ark_recv_pkts_noop(void *rx_queue, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts); +uint16_t eth_ark_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts); +void eth_ark_dev_rx_queue_release(void *rx_queue); +void eth_rx_queue_stats_get(void *vqueue, struct rte_eth_stats *stats); +void eth_rx_queue_stats_reset(void *vqueue); +void eth_ark_rx_dump_queue(struct rte_eth_dev *dev, uint16_t queue_id, + const char *msg); +void eth_ark_udm_force_close(struct rte_eth_dev *dev); + +#endif diff --git a/dpdk/drivers/net/ark/ark_ethdev_tx.c b/dpdk/drivers/net/ark/ark_ethdev_tx.c new file mode 100644 index 00000000..4ef55d10 --- /dev/null +++ b/dpdk/drivers/net/ark/ark_ethdev_tx.c @@ -0,0 +1,467 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2015-2017 Atomic Rules LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +#include "ark_ethdev_tx.h" +#include "ark_global.h" +#include "ark_mpu.h" +#include "ark_ddm.h" +#include "ark_logs.h" + +#define ARK_TX_META_SIZE 32 +#define ARK_TX_META_OFFSET (RTE_PKTMBUF_HEADROOM - ARK_TX_META_SIZE) +#define ARK_TX_MAX_NOCHAIN (RTE_MBUF_DEFAULT_DATAROOM) + + +/* ************************************************************************* */ +struct ark_tx_queue { + struct ark_tx_meta *meta_q; + struct rte_mbuf **bufs; + + /* handles for hw objects */ + struct ark_mpu_t *mpu; + struct ark_ddm_t *ddm; + + /* Stats HW tracks bytes and packets, need to count send errors */ + uint64_t tx_errors; + + uint32_t queue_size; + uint32_t queue_mask; + + /* 3 indexes to the paired data rings. */ + uint32_t prod_index; /* where to put the next one */ + uint32_t free_index; /* mbuf has been freed */ + + /* The queue Id is used to identify the HW Q */ + uint16_t phys_qid; + /* The queue Index within the dpdk device structures */ + uint16_t queue_index; + + uint32_t pad[1]; + + /* second cache line - fields only used in slow path */ + MARKER cacheline1 __rte_cache_min_aligned; + uint32_t cons_index; /* hw is done, can be freed */ +} __rte_cache_aligned; + +/* Forward declarations */ +static uint32_t eth_ark_tx_jumbo(struct ark_tx_queue *queue, + struct rte_mbuf *mbuf); +static int eth_ark_tx_hw_queue_config(struct ark_tx_queue *queue); +static void free_completed_tx(struct ark_tx_queue *queue); + +static inline void +ark_tx_hw_queue_stop(struct ark_tx_queue *queue) +{ + ark_mpu_stop(queue->mpu); +} + +/* ************************************************************************* */ +static inline void +eth_ark_tx_meta_from_mbuf(struct ark_tx_meta *meta, + const struct rte_mbuf *mbuf, + uint8_t flags) +{ + meta->physaddr = rte_mbuf_data_iova(mbuf); + meta->delta_ns = 0; + meta->data_len = rte_pktmbuf_data_len(mbuf); + meta->flags = flags; +} + +/* ************************************************************************* */ +uint16_t +eth_ark_xmit_pkts_noop(void *vtxq __rte_unused, + struct rte_mbuf **tx_pkts __rte_unused, + uint16_t nb_pkts __rte_unused) +{ + return 0; +} + +/* ************************************************************************* */ +uint16_t +eth_ark_xmit_pkts(void *vtxq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) +{ + struct ark_tx_queue *queue; + struct rte_mbuf *mbuf; + struct ark_tx_meta *meta; + + uint32_t idx; + uint32_t prod_index_limit; + int stat; + uint16_t nb; + + queue = (struct ark_tx_queue *)vtxq; + + /* free any packets after the HW is done with them */ + free_completed_tx(queue); + + prod_index_limit = queue->queue_size + queue->free_index; + + for (nb = 0; + (nb < nb_pkts) && (queue->prod_index != prod_index_limit); + ++nb) { + mbuf = tx_pkts[nb]; + + if (ARK_TX_PAD_TO_60) { + if (unlikely(rte_pktmbuf_pkt_len(mbuf) < 60)) { + /* this packet even if it is small can be split, + * be sure to add to the end mbuf + */ + uint16_t to_add = + 60 - rte_pktmbuf_pkt_len(mbuf); + char *appended = + rte_pktmbuf_append(mbuf, to_add); + + if (appended == 0) { + /* This packet is in error, + * we cannot send it so just + * count it and delete it. + */ + queue->tx_errors += 1; + rte_pktmbuf_free(mbuf); + continue; + } + memset(appended, 0, to_add); + } + } + + if (unlikely(mbuf->nb_segs != 1)) { + stat = eth_ark_tx_jumbo(queue, mbuf); + if (unlikely(stat != 0)) + break; /* Queue is full */ + } else { + idx = queue->prod_index & queue->queue_mask; + queue->bufs[idx] = mbuf; + meta = &queue->meta_q[idx]; + eth_ark_tx_meta_from_mbuf(meta, + mbuf, + ARK_DDM_SOP | + ARK_DDM_EOP); + queue->prod_index++; + } + } + + if (ARK_TX_DEBUG && (nb != nb_pkts)) { + PMD_TX_LOG(DEBUG, "TX: Failure to send:" + " req: %" PRIU32 + " sent: %" PRIU32 + " prod: %" PRIU32 + " cons: %" PRIU32 + " free: %" PRIU32 "\n", + nb_pkts, nb, + queue->prod_index, + queue->cons_index, + queue->free_index); + ark_mpu_dump(queue->mpu, + "TX Failure MPU: ", + queue->phys_qid); + } + + /* let FPGA know producer index. */ + if (likely(nb != 0)) + ark_mpu_set_producer(queue->mpu, queue->prod_index); + + return nb; +} + +/* ************************************************************************* */ +static uint32_t +eth_ark_tx_jumbo(struct ark_tx_queue *queue, struct rte_mbuf *mbuf) +{ + struct rte_mbuf *next; + struct ark_tx_meta *meta; + uint32_t free_queue_space; + uint32_t idx; + uint8_t flags = ARK_DDM_SOP; + + free_queue_space = queue->queue_mask - + (queue->prod_index - queue->free_index); + if (unlikely(free_queue_space < mbuf->nb_segs)) + return -1; + + while (mbuf != NULL) { + next = mbuf->next; + + idx = queue->prod_index & queue->queue_mask; + queue->bufs[idx] = mbuf; + meta = &queue->meta_q[idx]; + + flags |= (next == NULL) ? ARK_DDM_EOP : 0; + eth_ark_tx_meta_from_mbuf(meta, mbuf, flags); + queue->prod_index++; + + flags &= ~ARK_DDM_SOP; /* drop SOP flags */ + mbuf = next; + } + + return 0; +} + +/* ************************************************************************* */ +int +eth_ark_tx_queue_setup(struct rte_eth_dev *dev, + uint16_t queue_idx, + uint16_t nb_desc, + unsigned int socket_id, + const struct rte_eth_txconf *tx_conf __rte_unused) +{ + struct ark_adapter *ark = (struct ark_adapter *)dev->data->dev_private; + struct ark_tx_queue *queue; + int status; + + /* Future: divide the Q's evenly with multi-ports */ + int port = dev->data->port_id; + int qidx = port + queue_idx; + + if (!rte_is_power_of_2(nb_desc)) { + PMD_DRV_LOG(ERR, + "DPDK Arkville configuration queue size" + " must be power of two %u (%s)\n", + nb_desc, __func__); + return -1; + } + + /* Allocate queue struct */ + queue = rte_zmalloc_socket("Ark_txqueue", + sizeof(struct ark_tx_queue), + 64, + socket_id); + if (queue == 0) { + PMD_DRV_LOG(ERR, "Failed to allocate tx " + "queue memory in %s\n", + __func__); + return -ENOMEM; + } + + /* we use zmalloc no need to initialize fields */ + queue->queue_size = nb_desc; + queue->queue_mask = nb_desc - 1; + queue->phys_qid = qidx; + queue->queue_index = queue_idx; + dev->data->tx_queues[queue_idx] = queue; + + queue->meta_q = + rte_zmalloc_socket("Ark_txqueue meta", + nb_desc * sizeof(struct ark_tx_meta), + 64, + socket_id); + queue->bufs = + rte_zmalloc_socket("Ark_txqueue bufs", + nb_desc * sizeof(struct rte_mbuf *), + 64, + socket_id); + + if (queue->meta_q == 0 || queue->bufs == 0) { + PMD_DRV_LOG(ERR, "Failed to allocate " + "queue memory in %s\n", __func__); + rte_free(queue->meta_q); + rte_free(queue->bufs); + rte_free(queue); + return -ENOMEM; + } + + queue->ddm = RTE_PTR_ADD(ark->ddm.v, qidx * ARK_DDM_QOFFSET); + queue->mpu = RTE_PTR_ADD(ark->mputx.v, qidx * ARK_MPU_QOFFSET); + + status = eth_ark_tx_hw_queue_config(queue); + + if (unlikely(status != 0)) { + rte_free(queue->meta_q); + rte_free(queue->bufs); + rte_free(queue); + return -1; /* ERROR CODE */ + } + + return 0; +} + +/* ************************************************************************* */ +static int +eth_ark_tx_hw_queue_config(struct ark_tx_queue *queue) +{ + rte_iova_t queue_base, ring_base, cons_index_addr; + uint32_t write_interval_ns; + + /* Verify HW -- MPU */ + if (ark_mpu_verify(queue->mpu, sizeof(struct ark_tx_meta))) + return -1; + + queue_base = rte_malloc_virt2iova(queue); + ring_base = rte_malloc_virt2iova(queue->meta_q); + cons_index_addr = + queue_base + offsetof(struct ark_tx_queue, cons_index); + + ark_mpu_stop(queue->mpu); + ark_mpu_reset(queue->mpu); + + /* Stop and Reset and configure MPU */ + ark_mpu_configure(queue->mpu, ring_base, queue->queue_size, 1); + + /* + * Adjust the write interval based on queue size -- + * increase pcie traffic when low mbuf count + * Queue sizes less than 128 are not allowed + */ + switch (queue->queue_size) { + case 128: + write_interval_ns = 500; + break; + case 256: + write_interval_ns = 500; + break; + case 512: + write_interval_ns = 1000; + break; + default: + write_interval_ns = 2000; + break; + } + + /* Completion address in UDM */ + ark_ddm_setup(queue->ddm, cons_index_addr, write_interval_ns); + + return 0; +} + +/* ************************************************************************* */ +void +eth_ark_tx_queue_release(void *vtx_queue) +{ + struct ark_tx_queue *queue; + + queue = (struct ark_tx_queue *)vtx_queue; + + ark_tx_hw_queue_stop(queue); + + queue->cons_index = queue->prod_index; + free_completed_tx(queue); + + rte_free(queue->meta_q); + rte_free(queue->bufs); + rte_free(queue); +} + +/* ************************************************************************* */ +int +eth_ark_tx_queue_stop(struct rte_eth_dev *dev, uint16_t queue_id) +{ + struct ark_tx_queue *queue; + int cnt = 0; + + queue = dev->data->tx_queues[queue_id]; + + /* Wait for DDM to send out all packets. */ + while (queue->cons_index != queue->prod_index) { + usleep(100); + if (cnt++ > 10000) + return -1; + } + + ark_mpu_stop(queue->mpu); + free_completed_tx(queue); + + dev->data->tx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; + + return 0; +} + +int +eth_ark_tx_queue_start(struct rte_eth_dev *dev, uint16_t queue_id) +{ + struct ark_tx_queue *queue; + + queue = dev->data->tx_queues[queue_id]; + if (dev->data->tx_queue_state[queue_id] == RTE_ETH_QUEUE_STATE_STARTED) + return 0; + + ark_mpu_start(queue->mpu); + dev->data->tx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STARTED; + + return 0; +} + +/* ************************************************************************* */ +static void +free_completed_tx(struct ark_tx_queue *queue) +{ + struct rte_mbuf *mbuf; + struct ark_tx_meta *meta; + uint32_t top_index; + + top_index = queue->cons_index; /* read once */ + while (queue->free_index != top_index) { + meta = &queue->meta_q[queue->free_index & queue->queue_mask]; + mbuf = queue->bufs[queue->free_index & queue->queue_mask]; + + if (likely((meta->flags & ARK_DDM_SOP) != 0)) { + /* ref count of the mbuf is checked in this call. */ + rte_pktmbuf_free(mbuf); + } + queue->free_index++; + } +} + +/* ************************************************************************* */ +void +eth_tx_queue_stats_get(void *vqueue, struct rte_eth_stats *stats) +{ + struct ark_tx_queue *queue; + struct ark_ddm_t *ddm; + uint64_t bytes, pkts; + + queue = vqueue; + ddm = queue->ddm; + + bytes = ark_ddm_queue_byte_count(ddm); + pkts = ark_ddm_queue_pkt_count(ddm); + + stats->q_opackets[queue->queue_index] = pkts; + stats->q_obytes[queue->queue_index] = bytes; + stats->opackets += pkts; + stats->obytes += bytes; + stats->oerrors += queue->tx_errors; +} + +void +eth_tx_queue_stats_reset(void *vqueue) +{ + struct ark_tx_queue *queue; + struct ark_ddm_t *ddm; + + queue = vqueue; + ddm = queue->ddm; + + ark_ddm_queue_reset_stats(ddm); + queue->tx_errors = 0; +} diff --git a/dpdk/drivers/net/ark/ark_ethdev_tx.h b/dpdk/drivers/net/ark/ark_ethdev_tx.h new file mode 100644 index 00000000..8aaafc22 --- /dev/null +++ b/dpdk/drivers/net/ark/ark_ethdev_tx.h @@ -0,0 +1,59 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2015-2017 Atomic Rules LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _ARK_ETHDEV_TX_H_ +#define _ARK_ETHDEV_TX_H_ + +#include + +#include + + +uint16_t eth_ark_xmit_pkts_noop(void *vtxq, + struct rte_mbuf **tx_pkts, + uint16_t nb_pkts); +uint16_t eth_ark_xmit_pkts(void *vtxq, + struct rte_mbuf **tx_pkts, + uint16_t nb_pkts); +int eth_ark_tx_queue_setup(struct rte_eth_dev *dev, + uint16_t queue_idx, + uint16_t nb_desc, + unsigned int socket_id, + const struct rte_eth_txconf *tx_conf); +void eth_ark_tx_queue_release(void *vtx_queue); +int eth_ark_tx_queue_stop(struct rte_eth_dev *dev, uint16_t queue_id); +int eth_ark_tx_queue_start(struct rte_eth_dev *dev, uint16_t queue_id); +void eth_tx_queue_stats_get(void *vqueue, struct rte_eth_stats *stats); +void eth_tx_queue_stats_reset(void *vqueue); + +#endif diff --git a/dpdk/drivers/net/ark/ark_ext.h b/dpdk/drivers/net/ark/ark_ext.h new file mode 100644 index 00000000..d26c8198 --- /dev/null +++ b/dpdk/drivers/net/ark/ark_ext.h @@ -0,0 +1,119 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2015-2017 Atomic Rules LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _ARK_EXT_H_ +#define _ARK_EXT_H_ + +#include + +/* + * This is the template file for users who which to define a dynamic + * extension to the Arkville PMD. User's who create an extension + * should include this file and define the necessary and desired + * functions. + * Only 1 function is required for an extension, dev_init(); all other + * functions prototyped in this file are optional. + */ + +/* + * Called post PMD init. + * The implementation returns its private data that gets passed into + * all other functions as user_data + * The ARK extension implementation MUST implement this function + */ +void *dev_init(struct rte_eth_dev *dev, void *a_bar, int port_id); + +/* Called during device shutdown */ +void dev_uninit(struct rte_eth_dev *dev, void *user_data); + +/* This call is optional and allows the + * extension to specify the number of supported ports. + */ +uint8_t dev_get_port_count(struct rte_eth_dev *dev, + void *user_data); + +/* + * The following functions are optional and are directly mapped + * from the DPDK PMD ops structure. + * Each function if implemented is called after the ARK PMD + * implementation executes. + */ + +int dev_configure(struct rte_eth_dev *dev, + void *user_data); + +int dev_start(struct rte_eth_dev *dev, + void *user_data); + +void dev_stop(struct rte_eth_dev *dev, + void *user_data); + +void dev_close(struct rte_eth_dev *dev, + void *user_data); + +int link_update(struct rte_eth_dev *dev, + int wait_to_complete, + void *user_data); + +int dev_set_link_up(struct rte_eth_dev *dev, + void *user_data); + +int dev_set_link_down(struct rte_eth_dev *dev, + void *user_data); + +int stats_get(struct rte_eth_dev *dev, + struct rte_eth_stats *stats, + void *user_data); + +void stats_reset(struct rte_eth_dev *dev, + void *user_data); + +void mac_addr_add(struct rte_eth_dev *dev, + struct ether_addr *macadr, + uint32_t index, + uint32_t pool, + void *user_data); + +void mac_addr_remove(struct rte_eth_dev *dev, + uint32_t index, + void *user_data); + +void mac_addr_set(struct rte_eth_dev *dev, + struct ether_addr *mac_addr, + void *user_data); + +int set_mtu(struct rte_eth_dev *dev, + uint16_t size, + void *user_data); + +#endif diff --git a/dpdk/drivers/net/ark/ark_global.h b/dpdk/drivers/net/ark/ark_global.h new file mode 100644 index 00000000..aef2cf73 --- /dev/null +++ b/dpdk/drivers/net/ark/ark_global.h @@ -0,0 +1,162 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2015-2017 Atomic Rules LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _ARK_GLOBAL_H_ +#define _ARK_GLOBAL_H_ + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ark_pktdir.h" +#include "ark_pktgen.h" +#include "ark_pktchkr.h" + +#define ETH_ARK_ARG_MAXLEN 64 +#define ARK_SYSCTRL_BASE 0x0 +#define ARK_PKTGEN_BASE 0x10000 +#define ARK_MPU_RX_BASE 0x20000 +#define ARK_UDM_BASE 0x30000 +#define ARK_MPU_TX_BASE 0x40000 +#define ARK_DDM_BASE 0x60000 +#define ARK_CMAC_BASE 0x80000 +#define ARK_PKTDIR_BASE 0xa0000 +#define ARK_PKTCHKR_BASE 0x90000 +#define ARK_RCPACING_BASE 0xb0000 +#define ARK_EXTERNAL_BASE 0x100000 +#define ARK_MPU_QOFFSET 0x00100 +#define ARK_MAX_PORTS RTE_MAX_ETHPORTS + +#define offset8(n) n +#define offset16(n) ((n) / 2) +#define offset32(n) ((n) / 4) +#define offset64(n) ((n) / 8) + +/* Maximum length of arg list in bytes */ +#define ARK_MAX_ARG_LEN 256 + +/* + * Structure to store private data for each PF/VF instance. + */ +#define def_ptr(type, name) \ + union type { \ + uint64_t *t64; \ + uint32_t *t32; \ + uint16_t *t16; \ + uint8_t *t8; \ + void *v; \ + } name + +struct ark_user_ext { + void *(*dev_init)(struct rte_eth_dev *, void *abar, int port_id); + void (*dev_uninit)(struct rte_eth_dev *, void *); + int (*dev_get_port_count)(struct rte_eth_dev *, void *); + int (*dev_configure)(struct rte_eth_dev *, void *); + int (*dev_start)(struct rte_eth_dev *, void *); + void (*dev_stop)(struct rte_eth_dev *, void *); + void (*dev_close)(struct rte_eth_dev *, void *); + int (*link_update)(struct rte_eth_dev *, int wait_to_complete, void *); + int (*dev_set_link_up)(struct rte_eth_dev *, void *); + int (*dev_set_link_down)(struct rte_eth_dev *, void *); + int (*stats_get)(struct rte_eth_dev *, struct rte_eth_stats *, void *); + void (*stats_reset)(struct rte_eth_dev *, void *); + void (*mac_addr_add)(struct rte_eth_dev *, + struct ether_addr *, + uint32_t, + uint32_t, + void *); + void (*mac_addr_remove)(struct rte_eth_dev *, uint32_t, void *); + void (*mac_addr_set)(struct rte_eth_dev *, struct ether_addr *, void *); + int (*set_mtu)(struct rte_eth_dev *, uint16_t, void *); +}; + +struct ark_adapter { + /* User extension private data */ + void *user_data[ARK_MAX_PORTS]; + + /* Pointers to packet generator and checker */ + int start_pg; + ark_pkt_gen_t pg; + ark_pkt_chkr_t pc; + ark_pkt_dir_t pd; + + int num_ports; + + /* Packet generator/checker args */ + char pkt_gen_args[ARK_MAX_ARG_LEN]; + char pkt_chkr_args[ARK_MAX_ARG_LEN]; + uint32_t pkt_dir_v; + + /* eth device */ + struct rte_eth_dev *eth_dev; + + void *d_handle; + struct ark_user_ext user_ext; + + /* Our Bar 0 */ + uint8_t *bar0; + + /* Application Bar */ + uint8_t *a_bar; + + /* Arkville demo block offsets */ + def_ptr(sys_ctrl, sysctrl); + def_ptr(pkt_gen, pktgen); + def_ptr(mpu_rx, mpurx); + def_ptr(UDM, udm); + def_ptr(mpu_tx, mputx); + def_ptr(DDM, ddm); + def_ptr(CMAC, cmac); + def_ptr(external, external); + def_ptr(pkt_dir, pktdir); + def_ptr(pkt_chkr, pktchkr); + + int started; + uint16_t rx_queues; + uint16_t tx_queues; + + struct ark_rqpace_t *rqpacing; +}; + +typedef uint32_t *ark_t; + +#endif diff --git a/dpdk/drivers/net/ark/ark_logs.h b/dpdk/drivers/net/ark/ark_logs.h new file mode 100644 index 00000000..8aff2963 --- /dev/null +++ b/dpdk/drivers/net/ark/ark_logs.h @@ -0,0 +1,119 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2015-2017 Atomic Rules LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _ARK_DEBUG_H_ +#define _ARK_DEBUG_H_ + +#include +#include + + +/* Configuration option to pad TX packets to 60 bytes */ +#ifdef RTE_LIBRTE_ARK_PAD_TX +#define ARK_TX_PAD_TO_60 1 +#else +#define ARK_TX_PAD_TO_60 0 +#endif + +/* system camel case definition changed to upper case */ +#define PRIU32 PRIu32 +#define PRIU64 PRIu64 + +/* Format specifiers for string data pairs */ +#define ARK_SU32 "\n\t%-20s %'20" PRIU32 +#define ARK_SU64 "\n\t%-20s %'20" PRIU64 +#define ARK_SU64X "\n\t%-20s %#20" PRIx64 +#define ARK_SPTR "\n\t%-20s %20p" + + + +#define PMD_DRV_LOG(level, fmt, args...) \ + RTE_LOG(level, PMD, fmt, ## args) + +/* Conditional trace definitions */ +#define ARK_TRACE_ON(level, fmt, ...) \ + RTE_LOG(level, PMD, fmt, ##__VA_ARGS__) + +/* This pattern allows compiler check arguments even if disabled */ +#define ARK_TRACE_OFF(level, fmt, ...) \ + do {if (0) RTE_LOG(level, PMD, fmt, ##__VA_ARGS__); } \ + while (0) + + +/* tracing including the function name */ +#define ARK_FUNC_ON(level, fmt, args...) \ + RTE_LOG(level, PMD, "%s(): " fmt, __func__, ## args) + +/* tracing including the function name */ +#define ARK_FUNC_OFF(level, fmt, args...) \ + do { if (0) RTE_LOG(level, PMD, "%s(): " fmt, __func__, ## args); } \ + while (0) + + +/* Debug macro for tracing full behavior, function tracing and messages*/ +#ifdef RTE_LIBRTE_ARK_DEBUG_TRACE +#define PMD_FUNC_LOG(level, fmt, ...) ARK_FUNC_ON(level, fmt, ##__VA_ARGS__) +#define PMD_DEBUG_LOG(level, fmt, ...) ARK_TRACE_ON(level, fmt, ##__VA_ARGS__) +#else +#define PMD_FUNC_LOG(level, fmt, ...) ARK_FUNC_OFF(level, fmt, ##__VA_ARGS__) +#define PMD_DEBUG_LOG(level, fmt, ...) ARK_TRACE_OFF(level, fmt, ##__VA_ARGS__) +#endif + + +/* Debug macro for reporting FPGA statistics */ +#ifdef RTE_LIBRTE_ARK_DEBUG_STATS +#define PMD_STATS_LOG(level, fmt, ...) ARK_TRACE_ON(level, fmt, ##__VA_ARGS__) +#else +#define PMD_STATS_LOG(level, fmt, ...) ARK_TRACE_OFF(level, fmt, ##__VA_ARGS__) +#endif + + +/* Debug macro for RX path */ +#ifdef RTE_LIBRTE_ARK_DEBUG_RX +#define ARK_RX_DEBUG 1 +#define PMD_RX_LOG(level, fmt, ...) ARK_TRACE_ON(level, fmt, ##__VA_ARGS__) +#else +#define ARK_RX_DEBUG 0 +#define PMD_RX_LOG(level, fmt, ...) ARK_TRACE_OFF(level, fmt, ##__VA_ARGS__) +#endif + +/* Debug macro for TX path */ +#ifdef RTE_LIBRTE_ARK_DEBUG_TX +#define ARK_TX_DEBUG 1 +#define PMD_TX_LOG(level, fmt, ...) ARK_TRACE_ON(level, fmt, ##__VA_ARGS__) +#else +#define ARK_TX_DEBUG 0 +#define PMD_TX_LOG(level, fmt, ...) ARK_TRACE_OFF(level, fmt, ##__VA_ARGS__) +#endif + +#endif diff --git a/dpdk/drivers/net/ark/ark_mpu.c b/dpdk/drivers/net/ark/ark_mpu.c new file mode 100644 index 00000000..d4ba6dc7 --- /dev/null +++ b/dpdk/drivers/net/ark/ark_mpu.c @@ -0,0 +1,181 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2015-2017 Atomic Rules LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +#include "ark_logs.h" +#include "ark_mpu.h" + +uint16_t +ark_api_num_queues(struct ark_mpu_t *mpu) +{ + return mpu->hw.num_queues; +} + +uint16_t +ark_api_num_queues_per_port(struct ark_mpu_t *mpu, uint16_t ark_ports) +{ + return mpu->hw.num_queues / ark_ports; +} + +int +ark_mpu_verify(struct ark_mpu_t *mpu, uint32_t obj_size) +{ + uint32_t version; + + version = mpu->id.vernum & 0x0000fF00; + if ((mpu->id.idnum != 0x2055504d) || + (mpu->hw.obj_size != obj_size) || + (version != 0x00003100)) { + PMD_DRV_LOG(ERR, + " MPU module not found as expected %08x" + " \"%c%c%c%c %c%c%c%c\"\n", + mpu->id.idnum, + mpu->id.id[0], mpu->id.id[1], + mpu->id.id[2], mpu->id.id[3], + mpu->id.ver[0], mpu->id.ver[1], + mpu->id.ver[2], mpu->id.ver[3]); + PMD_DRV_LOG(ERR, + " MPU HW num_queues: %u hw_depth %u," + " obj_size: %u, obj_per_mrr: %u" + " Expected size %u\n", + mpu->hw.num_queues, + mpu->hw.hw_depth, + mpu->hw.obj_size, + mpu->hw.obj_per_mrr, + obj_size); + return -1; + } + return 0; +} + +void +ark_mpu_stop(struct ark_mpu_t *mpu) +{ + mpu->cfg.command = MPU_CMD_STOP; +} + +void +ark_mpu_start(struct ark_mpu_t *mpu) +{ + mpu->cfg.command = MPU_CMD_RUN; +} + +int +ark_mpu_reset(struct ark_mpu_t *mpu) +{ + int cnt = 0; + + mpu->cfg.command = MPU_CMD_RESET; + + while (mpu->cfg.command != MPU_CMD_IDLE) { + if (cnt++ > 1000) + break; + usleep(10); + } + if (mpu->cfg.command != MPU_CMD_IDLE) { + mpu->cfg.command = MPU_CMD_FORCE_RESET; + usleep(10); + } + ark_mpu_reset_stats(mpu); + return mpu->cfg.command != MPU_CMD_IDLE; +} + +void +ark_mpu_reset_stats(struct ark_mpu_t *mpu) +{ + mpu->stats.pci_request = 1; /* reset stats */ +} + +int +ark_mpu_configure(struct ark_mpu_t *mpu, rte_iova_t ring, uint32_t ring_size, + int is_tx) +{ + ark_mpu_reset(mpu); + + if (!rte_is_power_of_2(ring_size)) { + PMD_DRV_LOG(ERR, "ARK: Invalid ring size for MPU %d\n", + ring_size); + return -1; + } + + mpu->cfg.ring_base = ring; + mpu->cfg.ring_size = ring_size; + mpu->cfg.ring_mask = ring_size - 1; + mpu->cfg.min_host_move = is_tx ? 1 : mpu->hw.obj_per_mrr; + mpu->cfg.min_hw_move = mpu->hw.obj_per_mrr; + mpu->cfg.sw_prod_index = 0; + mpu->cfg.hw_cons_index = 0; + return 0; +} + +void +ark_mpu_dump(struct ark_mpu_t *mpu, const char *code, uint16_t qid) +{ + /* DUMP to see that we have started */ + PMD_DEBUG_LOG(DEBUG, "MPU: %s Q: %3u sw_prod %u, hw_cons: %u\n", + code, qid, + mpu->cfg.sw_prod_index, mpu->cfg.hw_cons_index); + PMD_DEBUG_LOG(DEBUG, "MPU: %s state: %d count %d, reserved %d" + " data 0x%08x_%08x 0x%08x_%08x\n", + code, + mpu->debug.state, mpu->debug.count, + mpu->debug.reserved, + mpu->debug.peek[1], + mpu->debug.peek[0], + mpu->debug.peek[3], + mpu->debug.peek[2] + ); + PMD_STATS_LOG(INFO, "MPU: %s Q: %3u" + ARK_SU64 ARK_SU64 ARK_SU64 ARK_SU64 + ARK_SU64 ARK_SU64 ARK_SU64 "\n", + code, qid, + "PCI Request:", mpu->stats.pci_request, + "Queue_empty", mpu->stats.q_empty, + "Queue_q1", mpu->stats.q_q1, + "Queue_q2", mpu->stats.q_q2, + "Queue_q3", mpu->stats.q_q3, + "Queue_q4", mpu->stats.q_q4, + "Queue_full", mpu->stats.q_full + ); +} + +void +ark_mpu_dump_setup(struct ark_mpu_t *mpu, uint16_t q_id) +{ + PMD_DEBUG_LOG(DEBUG, "MPU Setup Q: %u" + ARK_SU64X "\n", + q_id, + "ring_base", mpu->cfg.ring_base + ); +} diff --git a/dpdk/drivers/net/ark/ark_mpu.h b/dpdk/drivers/net/ark/ark_mpu.h new file mode 100644 index 00000000..f6f6c808 --- /dev/null +++ b/dpdk/drivers/net/ark/ark_mpu.h @@ -0,0 +1,154 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2015-2017 Atomic Rules LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _ARK_MPU_H_ +#define _ARK_MPU_H_ + +#include + +#include + +/* The MPU or Memory Prefetch Unit is an internal Arkville hardware + * module for moving data between host memory and the hardware FPGA. + * This module is *not* intended for end-user manipulation, hence + * there is minimal documentation. + */ + +/* + * MPU hardware structures + * These are overlay structures to a memory mapped FPGA device. These + * structs will never be instantiated in ram memory + */ + +#define ARK_MPU_ID 0x00 +struct ark_mpu_id_t { + union { + char id[4]; + uint32_t idnum; + }; + union { + char ver[4]; + uint32_t vernum; + }; + uint32_t phys_id; + uint32_t mrr_code; +}; + +#define ARK_MPU_HW 0x010 +struct ark_mpu_hw_t { + uint16_t num_queues; + uint16_t reserved; + uint32_t hw_depth; + uint32_t obj_size; + uint32_t obj_per_mrr; +}; + +#define ARK_MPU_CFG 0x040 +struct ark_mpu_cfg_t { + rte_iova_t ring_base; /* rte_iova_t is a uint64_t */ + uint32_t ring_size; + uint32_t ring_mask; + uint32_t min_host_move; + uint32_t min_hw_move; + volatile uint32_t sw_prod_index; + volatile uint32_t hw_cons_index; + volatile uint32_t command; +}; +enum ARK_MPU_COMMAND { + MPU_CMD_IDLE = 1, + MPU_CMD_RUN = 2, + MPU_CMD_STOP = 4, + MPU_CMD_RESET = 8, + MPU_CMD_FORCE_RESET = 16, + MPU_COMMAND_LIMIT = 0xfFFFFFFF +}; + +#define ARK_MPU_STATS 0x080 +struct ark_mpu_stats_t { + volatile uint64_t pci_request; + volatile uint64_t q_empty; + volatile uint64_t q_q1; + volatile uint64_t q_q2; + volatile uint64_t q_q3; + volatile uint64_t q_q4; + volatile uint64_t q_full; +}; + +#define ARK_MPU_DEBUG 0x0C0 +struct ark_mpu_debug_t { + volatile uint32_t state; + uint32_t reserved; + volatile uint32_t count; + volatile uint32_t take; + volatile uint32_t peek[4]; +}; + +/* Consolidated structure */ +struct ark_mpu_t { + struct ark_mpu_id_t id; + uint8_t reserved0[(ARK_MPU_HW - ARK_MPU_ID) + - sizeof(struct ark_mpu_id_t)]; + struct ark_mpu_hw_t hw; + uint8_t reserved1[(ARK_MPU_CFG - ARK_MPU_HW) - + sizeof(struct ark_mpu_hw_t)]; + struct ark_mpu_cfg_t cfg; + uint8_t reserved2[(ARK_MPU_STATS - ARK_MPU_CFG) - + sizeof(struct ark_mpu_cfg_t)]; + struct ark_mpu_stats_t stats; + uint8_t reserved3[(ARK_MPU_DEBUG - ARK_MPU_STATS) - + sizeof(struct ark_mpu_stats_t)]; + struct ark_mpu_debug_t debug; +}; + +uint16_t ark_api_num_queues(struct ark_mpu_t *mpu); +uint16_t ark_api_num_queues_per_port(struct ark_mpu_t *mpu, + uint16_t ark_ports); +int ark_mpu_verify(struct ark_mpu_t *mpu, uint32_t obj_size); +void ark_mpu_stop(struct ark_mpu_t *mpu); +void ark_mpu_start(struct ark_mpu_t *mpu); +int ark_mpu_reset(struct ark_mpu_t *mpu); +int ark_mpu_configure(struct ark_mpu_t *mpu, rte_iova_t ring, + uint32_t ring_size, int is_tx); + +void ark_mpu_dump(struct ark_mpu_t *mpu, const char *msg, uint16_t idx); +void ark_mpu_dump_setup(struct ark_mpu_t *mpu, uint16_t qid); +void ark_mpu_reset_stats(struct ark_mpu_t *mpu); + +/* this action is in a performance critical path */ +static inline void +ark_mpu_set_producer(struct ark_mpu_t *mpu, uint32_t idx) +{ + mpu->cfg.sw_prod_index = idx; +} + +#endif diff --git a/dpdk/drivers/net/ark/ark_pktchkr.c b/dpdk/drivers/net/ark/ark_pktchkr.c new file mode 100644 index 00000000..202a1d9b --- /dev/null +++ b/dpdk/drivers/net/ark/ark_pktchkr.c @@ -0,0 +1,478 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2015-2017 Atomic Rules LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include + +#include +#include + +#include "ark_pktchkr.h" +#include "ark_logs.h" + +static int set_arg(char *arg, char *val); +static int ark_pktchkr_is_gen_forever(ark_pkt_chkr_t handle); + +#define ARK_MAX_STR_LEN 64 +union OPTV { + int INT; + int BOOL; + uint64_t LONG; + char STR[ARK_MAX_STR_LEN]; +}; + +enum OPTYPE { + OTINT, + OTLONG, + OTBOOL, + OTSTRING +}; + +struct OPTIONS { + char opt[ARK_MAX_STR_LEN]; + enum OPTYPE t; + union OPTV v; +}; + +static struct OPTIONS toptions[] = { + {{"configure"}, OTBOOL, {1} }, + {{"port"}, OTINT, {0} }, + {{"mac-dump"}, OTBOOL, {0} }, + {{"dg-mode"}, OTBOOL, {1} }, + {{"run"}, OTBOOL, {0} }, + {{"stop"}, OTBOOL, {0} }, + {{"dump"}, OTBOOL, {0} }, + {{"en_resync"}, OTBOOL, {0} }, + {{"tuser_err_val"}, OTINT, {1} }, + {{"gen_forever"}, OTBOOL, {0} }, + {{"en_slaved_start"}, OTBOOL, {0} }, + {{"vary_length"}, OTBOOL, {0} }, + {{"incr_payload"}, OTINT, {0} }, + {{"incr_first_byte"}, OTBOOL, {0} }, + {{"ins_seq_num"}, OTBOOL, {0} }, + {{"ins_time_stamp"}, OTBOOL, {1} }, + {{"ins_udp_hdr"}, OTBOOL, {0} }, + {{"num_pkts"}, OTLONG, .v.LONG = 10000000000000L}, + {{"payload_byte"}, OTINT, {0x55} }, + {{"pkt_spacing"}, OTINT, {60} }, + {{"pkt_size_min"}, OTINT, {2005} }, + {{"pkt_size_max"}, OTINT, {1514} }, + {{"pkt_size_incr"}, OTINT, {1} }, + {{"eth_type"}, OTINT, {0x0800} }, + {{"src_mac_addr"}, OTLONG, .v.LONG = 0xdC3cF6425060L}, + {{"dst_mac_addr"}, OTLONG, .v.LONG = 0x112233445566L}, + {{"hdr_dW0"}, OTINT, {0x0016e319} }, + {{"hdr_dW1"}, OTINT, {0x27150004} }, + {{"hdr_dW2"}, OTINT, {0x76967bda} }, + {{"hdr_dW3"}, OTINT, {0x08004500} }, + {{"hdr_dW4"}, OTINT, {0x005276ed} }, + {{"hdr_dW5"}, OTINT, {0x40004006} }, + {{"hdr_dW6"}, OTINT, {0x56cfc0a8} }, + {{"start_offset"}, OTINT, {0} }, + {{"dst_ip"}, OTSTRING, .v.STR = "169.254.10.240"}, + {{"dst_port"}, OTINT, {65536} }, + {{"src_port"}, OTINT, {65536} }, +}; + +ark_pkt_chkr_t +ark_pktchkr_init(void *addr, int ord, int l2_mode) +{ + struct ark_pkt_chkr_inst *inst = + rte_malloc("ark_pkt_chkr_inst", + sizeof(struct ark_pkt_chkr_inst), 0); + if (inst == NULL) { + PMD_DRV_LOG(ERR, "Failed to malloc ark_pkt_chkr_inst.\n"); + return inst; + } + inst->sregs = (struct ark_pkt_chkr_stat_regs *)addr; + inst->cregs = + (struct ark_pkt_chkr_ctl_regs *)(((uint8_t *)addr) + 0x100); + inst->ordinal = ord; + inst->l2_mode = l2_mode; + return inst; +} + +void +ark_pktchkr_uninit(ark_pkt_chkr_t handle) +{ + rte_free(handle); +} + +void +ark_pktchkr_run(ark_pkt_chkr_t handle) +{ + struct ark_pkt_chkr_inst *inst = (struct ark_pkt_chkr_inst *)handle; + + inst->sregs->pkt_start_stop = 0; + inst->sregs->pkt_start_stop = 0x1; +} + +int +ark_pktchkr_stopped(ark_pkt_chkr_t handle) +{ + struct ark_pkt_chkr_inst *inst = (struct ark_pkt_chkr_inst *)handle; + uint32_t r = inst->sregs->pkt_start_stop; + + return (((r >> 16) & 1) == 1); +} + +void +ark_pktchkr_stop(ark_pkt_chkr_t handle) +{ + struct ark_pkt_chkr_inst *inst = (struct ark_pkt_chkr_inst *)handle; + int wait_cycle = 10; + + inst->sregs->pkt_start_stop = 0; + while (!ark_pktchkr_stopped(handle) && (wait_cycle > 0)) { + usleep(1000); + wait_cycle--; + PMD_DEBUG_LOG(DEBUG, "Waiting for pktchk %d to stop...\n", + inst->ordinal); + } + PMD_DEBUG_LOG(DEBUG, "Pktchk %d stopped.\n", inst->ordinal); +} + +int +ark_pktchkr_is_running(ark_pkt_chkr_t handle) +{ + struct ark_pkt_chkr_inst *inst = (struct ark_pkt_chkr_inst *)handle; + uint32_t r = inst->sregs->pkt_start_stop; + + return ((r & 1) == 1); +} + +static void +ark_pktchkr_set_pkt_ctrl(ark_pkt_chkr_t handle, + uint32_t gen_forever, + uint32_t vary_length, + uint32_t incr_payload, + uint32_t incr_first_byte, + uint32_t ins_seq_num, + uint32_t ins_udp_hdr, + uint32_t en_resync, + uint32_t tuser_err_val, + uint32_t ins_time_stamp) +{ + struct ark_pkt_chkr_inst *inst = (struct ark_pkt_chkr_inst *)handle; + uint32_t r = (tuser_err_val << 16) | (en_resync << 0); + + inst->sregs->pkt_ctrl = r; + if (!inst->l2_mode) + ins_udp_hdr = 0; + r = ((gen_forever << 24) | + (vary_length << 16) | + (incr_payload << 12) | + (incr_first_byte << 8) | + (ins_time_stamp << 5) | + (ins_seq_num << 4) | + ins_udp_hdr); + inst->cregs->pkt_ctrl = r; +} + +static +int +ark_pktchkr_is_gen_forever(ark_pkt_chkr_t handle) +{ + struct ark_pkt_chkr_inst *inst = (struct ark_pkt_chkr_inst *)handle; + uint32_t r = inst->cregs->pkt_ctrl; + + return (((r >> 24) & 1) == 1); +} + +int +ark_pktchkr_wait_done(ark_pkt_chkr_t handle) +{ + struct ark_pkt_chkr_inst *inst = (struct ark_pkt_chkr_inst *)handle; + + if (ark_pktchkr_is_gen_forever(handle)) { + PMD_DEBUG_LOG(ERR, "Pktchk wait_done will not terminate" + " because gen_forever=1\n"); + return -1; + } + int wait_cycle = 10; + + while (!ark_pktchkr_stopped(handle) && (wait_cycle > 0)) { + usleep(1000); + wait_cycle--; + PMD_DEBUG_LOG(DEBUG, "Waiting for packet checker %d's" + " internal pktgen to finish sending...\n", + inst->ordinal); + PMD_DEBUG_LOG(DEBUG, "Pktchk %d's pktgen done.\n", + inst->ordinal); + } + return 0; +} + +int +ark_pktchkr_get_pkts_sent(ark_pkt_chkr_t handle) +{ + struct ark_pkt_chkr_inst *inst = (struct ark_pkt_chkr_inst *)handle; + + return inst->cregs->pkts_sent; +} + +void +ark_pktchkr_set_payload_byte(ark_pkt_chkr_t handle, uint32_t b) +{ + struct ark_pkt_chkr_inst *inst = (struct ark_pkt_chkr_inst *)handle; + + inst->cregs->pkt_payload = b; +} + +void +ark_pktchkr_set_pkt_size_min(ark_pkt_chkr_t handle, uint32_t x) +{ + struct ark_pkt_chkr_inst *inst = (struct ark_pkt_chkr_inst *)handle; + + inst->cregs->pkt_size_min = x; +} + +void +ark_pktchkr_set_pkt_size_max(ark_pkt_chkr_t handle, uint32_t x) +{ + struct ark_pkt_chkr_inst *inst = (struct ark_pkt_chkr_inst *)handle; + + inst->cregs->pkt_size_max = x; +} + +void +ark_pktchkr_set_pkt_size_incr(ark_pkt_chkr_t handle, uint32_t x) +{ + struct ark_pkt_chkr_inst *inst = (struct ark_pkt_chkr_inst *)handle; + + inst->cregs->pkt_size_incr = x; +} + +void +ark_pktchkr_set_num_pkts(ark_pkt_chkr_t handle, uint32_t x) +{ + struct ark_pkt_chkr_inst *inst = (struct ark_pkt_chkr_inst *)handle; + + inst->cregs->num_pkts = x; +} + +void +ark_pktchkr_set_src_mac_addr(ark_pkt_chkr_t handle, uint64_t mac_addr) +{ + struct ark_pkt_chkr_inst *inst = (struct ark_pkt_chkr_inst *)handle; + + inst->cregs->src_mac_addr_h = (mac_addr >> 32) & 0xffff; + inst->cregs->src_mac_addr_l = mac_addr & 0xffffffff; +} + +void +ark_pktchkr_set_dst_mac_addr(ark_pkt_chkr_t handle, uint64_t mac_addr) +{ + struct ark_pkt_chkr_inst *inst = (struct ark_pkt_chkr_inst *)handle; + + inst->cregs->dst_mac_addr_h = (mac_addr >> 32) & 0xffff; + inst->cregs->dst_mac_addr_l = mac_addr & 0xffffffff; +} + +void +ark_pktchkr_set_eth_type(ark_pkt_chkr_t handle, uint32_t x) +{ + struct ark_pkt_chkr_inst *inst = (struct ark_pkt_chkr_inst *)handle; + + inst->cregs->eth_type = x; +} + +void +ark_pktchkr_set_hdr_dW(ark_pkt_chkr_t handle, uint32_t *hdr) +{ + uint32_t i; + struct ark_pkt_chkr_inst *inst = (struct ark_pkt_chkr_inst *)handle; + + for (i = 0; i < 7; i++) + inst->cregs->hdr_dw[i] = hdr[i]; +} + +void +ark_pktchkr_dump_stats(ark_pkt_chkr_t handle) +{ + struct ark_pkt_chkr_inst *inst = (struct ark_pkt_chkr_inst *)handle; + + PMD_STATS_LOG(INFO, "pkts_rcvd = (%'u)\n", + inst->sregs->pkts_rcvd); + PMD_STATS_LOG(INFO, "bytes_rcvd = (%'" PRIU64 ")\n", + inst->sregs->bytes_rcvd); + PMD_STATS_LOG(INFO, "pkts_ok = (%'u)\n", + inst->sregs->pkts_ok); + PMD_STATS_LOG(INFO, "pkts_mismatch = (%'u)\n", + inst->sregs->pkts_mismatch); + PMD_STATS_LOG(INFO, "pkts_err = (%'u)\n", + inst->sregs->pkts_err); + PMD_STATS_LOG(INFO, "first_mismatch = (%'u)\n", + inst->sregs->first_mismatch); + PMD_STATS_LOG(INFO, "resync_events = (%'u)\n", + inst->sregs->resync_events); + PMD_STATS_LOG(INFO, "pkts_missing = (%'u)\n", + inst->sregs->pkts_missing); + PMD_STATS_LOG(INFO, "min_latency = (%'u)\n", + inst->sregs->min_latency); + PMD_STATS_LOG(INFO, "max_latency = (%'u)\n", + inst->sregs->max_latency); +} + +static struct OPTIONS * +options(const char *id) +{ + unsigned int i; + + for (i = 0; i < sizeof(toptions) / sizeof(struct OPTIONS); i++) { + if (strcmp(id, toptions[i].opt) == 0) + return &toptions[i]; + } + PMD_DRV_LOG(ERR, + "pktchkr: Could not find requested option!, option = %s\n", + id); + return NULL; +} + +static int +set_arg(char *arg, char *val) +{ + struct OPTIONS *o = options(arg); + + if (o) { + switch (o->t) { + case OTINT: + case OTBOOL: + o->v.INT = atoi(val); + break; + case OTLONG: + o->v.INT = atoll(val); + break; + case OTSTRING: + snprintf(o->v.STR, ARK_MAX_STR_LEN, "%s", val); + break; + } + return 1; + } + return 0; +} + +/****** + * Arg format = "opt0=v,opt_n=v ..." + ******/ +void +ark_pktchkr_parse(char *args) +{ + char *argv, *v; + const char toks[] = "=\n\t\v\f \r"; + argv = strtok(args, toks); + v = strtok(NULL, toks); + while (argv && v) { + set_arg(argv, v); + argv = strtok(NULL, toks); + v = strtok(NULL, toks); + } +} + +static int32_t parse_ipv4_string(char const *ip_address); +static int32_t +parse_ipv4_string(char const *ip_address) +{ + unsigned int ip[4]; + + if (sscanf(ip_address, "%u.%u.%u.%u", + &ip[0], &ip[1], &ip[2], &ip[3]) != 4) + return 0; + return ip[3] + ip[2] * 0x100 + ip[1] * 0x10000ul + ip[0] * 0x1000000ul; +} + +void +ark_pktchkr_setup(ark_pkt_chkr_t handle) +{ + uint32_t hdr[7]; + int32_t dst_ip = parse_ipv4_string(options("dst_ip")->v.STR); + + if (!options("stop")->v.BOOL && options("configure")->v.BOOL) { + ark_pktchkr_set_payload_byte(handle, + options("payload_byte")->v.INT); + ark_pktchkr_set_src_mac_addr(handle, + options("src_mac_addr")->v.INT); + ark_pktchkr_set_dst_mac_addr(handle, + options("dst_mac_addr")->v.LONG); + + ark_pktchkr_set_eth_type(handle, + options("eth_type")->v.INT); + if (options("dg-mode")->v.BOOL) { + hdr[0] = options("hdr_dW0")->v.INT; + hdr[1] = options("hdr_dW1")->v.INT; + hdr[2] = options("hdr_dW2")->v.INT; + hdr[3] = options("hdr_dW3")->v.INT; + hdr[4] = options("hdr_dW4")->v.INT; + hdr[5] = options("hdr_dW5")->v.INT; + hdr[6] = options("hdr_dW6")->v.INT; + } else { + hdr[0] = dst_ip; + hdr[1] = options("dst_port")->v.INT; + hdr[2] = options("src_port")->v.INT; + hdr[3] = 0; + hdr[4] = 0; + hdr[5] = 0; + hdr[6] = 0; + } + ark_pktchkr_set_hdr_dW(handle, hdr); + ark_pktchkr_set_num_pkts(handle, + options("num_pkts")->v.INT); + ark_pktchkr_set_pkt_size_min(handle, + options("pkt_size_min")->v.INT); + ark_pktchkr_set_pkt_size_max(handle, + options("pkt_size_max")->v.INT); + ark_pktchkr_set_pkt_size_incr(handle, + options("pkt_size_incr")->v.INT); + ark_pktchkr_set_pkt_ctrl(handle, + options("gen_forever")->v.BOOL, + options("vary_length")->v.BOOL, + options("incr_payload")->v.BOOL, + options("incr_first_byte")->v.BOOL, + options("ins_seq_num")->v.INT, + options("ins_udp_hdr")->v.BOOL, + options("en_resync")->v.BOOL, + options("tuser_err_val")->v.INT, + options("ins_time_stamp")->v.INT); + } + + if (options("stop")->v.BOOL) + ark_pktchkr_stop(handle); + + if (options("run")->v.BOOL) { + PMD_DEBUG_LOG(DEBUG, "Starting packet checker on port %d\n", + options("port")->v.INT); + ark_pktchkr_run(handle); + } +} diff --git a/dpdk/drivers/net/ark/ark_pktchkr.h b/dpdk/drivers/net/ark/ark_pktchkr.h new file mode 100644 index 00000000..f4025dd6 --- /dev/null +++ b/dpdk/drivers/net/ark/ark_pktchkr.h @@ -0,0 +1,117 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2015-2017 Atomic Rules LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _ARK_PKTCHKR_H_ +#define _ARK_PKTCHKR_H_ + +#include +#include + +#define ARK_PKTCHKR_BASE_ADR 0x90000 + +typedef void *ark_pkt_chkr_t; + +/* The packet checker is an internal Arkville hardware module, which + * verifies packet streams generated from the corresponding packet + * generator. This module is used for Arkville testing. + * This module is *not* intended for end-user manipulation, hence + * there is minimal documentation. + */ + +/* + * This are overlay structures to a memory mapped FPGA device. These + * structs will never be instantiated in ram memory + */ +struct ark_pkt_chkr_stat_regs { + uint32_t r0; + uint32_t pkt_start_stop; + uint32_t pkt_ctrl; + uint32_t pkts_rcvd; + uint64_t bytes_rcvd; + uint32_t pkts_ok; + uint32_t pkts_mismatch; + uint32_t pkts_err; + uint32_t first_mismatch; + uint32_t resync_events; + uint32_t pkts_missing; + uint32_t min_latency; + uint32_t max_latency; +} __attribute__ ((packed)); + +struct ark_pkt_chkr_ctl_regs { + uint32_t pkt_ctrl; + uint32_t pkt_payload; + uint32_t pkt_size_min; + uint32_t pkt_size_max; + uint32_t pkt_size_incr; + uint32_t num_pkts; + uint32_t pkts_sent; + uint32_t src_mac_addr_l; + uint32_t src_mac_addr_h; + uint32_t dst_mac_addr_l; + uint32_t dst_mac_addr_h; + uint32_t eth_type; + uint32_t hdr_dw[7]; +} __attribute__ ((packed)); + +struct ark_pkt_chkr_inst { + struct rte_eth_dev_info *dev_info; + volatile struct ark_pkt_chkr_stat_regs *sregs; + volatile struct ark_pkt_chkr_ctl_regs *cregs; + int l2_mode; + int ordinal; +}; + +/* packet checker functions */ +ark_pkt_chkr_t ark_pktchkr_init(void *addr, int ord, int l2_mode); +void ark_pktchkr_uninit(ark_pkt_chkr_t handle); +void ark_pktchkr_run(ark_pkt_chkr_t handle); +int ark_pktchkr_stopped(ark_pkt_chkr_t handle); +void ark_pktchkr_stop(ark_pkt_chkr_t handle); +int ark_pktchkr_is_running(ark_pkt_chkr_t handle); +int ark_pktchkr_get_pkts_sent(ark_pkt_chkr_t handle); +void ark_pktchkr_set_payload_byte(ark_pkt_chkr_t handle, uint32_t b); +void ark_pktchkr_set_pkt_size_min(ark_pkt_chkr_t handle, uint32_t x); +void ark_pktchkr_set_pkt_size_max(ark_pkt_chkr_t handle, uint32_t x); +void ark_pktchkr_set_pkt_size_incr(ark_pkt_chkr_t handle, uint32_t x); +void ark_pktchkr_set_num_pkts(ark_pkt_chkr_t handle, uint32_t x); +void ark_pktchkr_set_src_mac_addr(ark_pkt_chkr_t handle, uint64_t mac_addr); +void ark_pktchkr_set_dst_mac_addr(ark_pkt_chkr_t handle, uint64_t mac_addr); +void ark_pktchkr_set_eth_type(ark_pkt_chkr_t handle, uint32_t x); +void ark_pktchkr_set_hdr_dW(ark_pkt_chkr_t handle, uint32_t *hdr); +void ark_pktchkr_parse(char *args); +void ark_pktchkr_setup(ark_pkt_chkr_t handle); +void ark_pktchkr_dump_stats(ark_pkt_chkr_t handle); +int ark_pktchkr_wait_done(ark_pkt_chkr_t handle); + +#endif diff --git a/dpdk/drivers/net/ark/ark_pktdir.c b/dpdk/drivers/net/ark/ark_pktdir.c new file mode 100644 index 00000000..eb47dedb --- /dev/null +++ b/dpdk/drivers/net/ark/ark_pktdir.c @@ -0,0 +1,85 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2015-2017 Atomic Rules LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include + +#include "ark_pktdir.h" +#include "ark_global.h" +#include "ark_logs.h" + + +ark_pkt_dir_t +ark_pktdir_init(void *base) +{ + struct ark_pkt_dir_inst *inst = + rte_malloc("ark_pkt_dir_inst", + sizeof(struct ark_pkt_dir_inst), + 0); + if (inst == NULL) { + PMD_DRV_LOG(ERR, "Failed to malloc ark_pkt_dir_inst.\n"); + return inst; + } + inst->regs = (struct ark_pkt_dir_regs *)base; + inst->regs->ctrl = 0x00110110; /* POR state */ + return inst; +} + +void +ark_pktdir_uninit(ark_pkt_dir_t handle) +{ + struct ark_pkt_dir_inst *inst = (struct ark_pkt_dir_inst *)handle; + + rte_free(inst); +} + +void +ark_pktdir_setup(ark_pkt_dir_t handle, uint32_t v) +{ + struct ark_pkt_dir_inst *inst = (struct ark_pkt_dir_inst *)handle; + inst->regs->ctrl = v; +} + +uint32_t +ark_pktdir_status(ark_pkt_dir_t handle) +{ + struct ark_pkt_dir_inst *inst = (struct ark_pkt_dir_inst *)handle; + return inst->regs->ctrl; +} + +uint32_t +ark_pktdir_stall_cnt(ark_pkt_dir_t handle) +{ + struct ark_pkt_dir_inst *inst = (struct ark_pkt_dir_inst *)handle; + return inst->regs->stall_cnt; +} diff --git a/dpdk/drivers/net/ark/ark_pktdir.h b/dpdk/drivers/net/ark/ark_pktdir.h new file mode 100644 index 00000000..e13fe821 --- /dev/null +++ b/dpdk/drivers/net/ark/ark_pktdir.h @@ -0,0 +1,70 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2015-2017 Atomic Rules LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _ARK_PKTDIR_H_ +#define _ARK_PKTDIR_H_ + +#include + +#define ARK_PKTDIR_BASE_ADR 0xa0000 + +typedef void *ark_pkt_dir_t; + + +/* The packet director is an internal Arkville hardware module for + * directing packet data in non-typical flows, such as testing. + * This module is *not* intended for end-user manipulation, hence + * there is minimal documentation. + */ + +/* + * This is an overlay structures to a memory mapped FPGA device. These + * structs will never be instantiated in ram memory + */ +struct ark_pkt_dir_regs { + uint32_t ctrl; + uint32_t status; + uint32_t stall_cnt; +} __attribute__ ((packed)); + +struct ark_pkt_dir_inst { + volatile struct ark_pkt_dir_regs *regs; +}; + +ark_pkt_dir_t ark_pktdir_init(void *base); +void ark_pktdir_uninit(ark_pkt_dir_t handle); +void ark_pktdir_setup(ark_pkt_dir_t handle, uint32_t v); +uint32_t ark_pktdir_stall_cnt(ark_pkt_dir_t handle); +uint32_t ark_pktdir_status(ark_pkt_dir_t handle); + +#endif diff --git a/dpdk/drivers/net/ark/ark_pktgen.c b/dpdk/drivers/net/ark/ark_pktgen.c new file mode 100644 index 00000000..018f37b6 --- /dev/null +++ b/dpdk/drivers/net/ark/ark_pktgen.c @@ -0,0 +1,500 @@ +/* + * BSD LICENSE + * + * Copyright (c) 2015-2017 Atomic Rules LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include + +#include + +#include +#include + +#include "ark_pktgen.h" +#include "ark_logs.h" + +#define ARK_MAX_STR_LEN 64 +union OPTV { + int INT; + int BOOL; + uint64_t LONG; + char STR[ARK_MAX_STR_LEN]; +}; + +enum OPTYPE { + OTINT, + OTLONG, + OTBOOL, + OTSTRING +}; + +struct OPTIONS { + char opt[ARK_MAX_STR_LEN]; + enum OPTYPE t; + union OPTV v; +}; + +static struct OPTIONS toptions[] = { + {{"configure"}, OTBOOL, {1} }, + {{"dg-mode"}, OTBOOL, {1} }, + {{"run"}, OTBOOL, {0} }, + {{"pause"}, OTBOOL, {0} }, + {{"reset"}, OTBOOL, {0} }, + {{"dump"}, OTBOOL, {0} }, + {{"gen_forever"}, OTBOOL, {0} }, + {{"en_slaved_start"}, OTBOOL, {0} }, + {{"vary_length"}, OTBOOL, {0} }, + {{"incr_payload"}, OTBOOL, {0} }, + {{"incr_first_byte"}, OTBOOL, {0} }, + {{"ins_seq_num"}, OTBOOL, {0} }, + {{"ins_time_stamp"}, OTBOOL, {1} }, + {{"ins_udp_hdr"}, OTBOOL, {0} }, + {{"num_pkts"}, OTLONG, .v.LONG = 100000000}, + {{"payload_byte"}, OTINT, {0x55} }, + {{"pkt_spacing"}, OTINT, {130} }, + {{"pkt_size_min"}, OTINT, {2006} }, + {{"pkt_size_max"}, OTINT, {1514} }, + {{"pkt_size_incr"}, OTINT, {1} }, + {{"eth_type"}, OTINT, {0x0800} }, + {{"src_mac_addr"}, OTLONG, .v.LONG = 0xdC3cF6425060L}, + {{"dst_mac_addr"}, OTLONG, .v.LONG = 0x112233445566L}, + {{"hdr_dW0"}, OTINT, {0x0016e319} }, + {{"hdr_dW1"}, OTINT, {0x27150004} }, + {{"hdr_dW2"}, OTINT, {0x76967bda} }, + {{"hdr_dW3"}, OTINT, {0x08004500} }, + {{"hdr_dW4"}, OTINT, {0x005276ed} }, + {{"hdr_dW5"}, OTINT, {0x40004006} }, + {{"hdr_dW6"}, OTINT, {0x56cfc0a8} }, + {{"start_offset"}, OTINT, {0} }, + {{"bytes_per_cycle"}, OTINT, {10} }, + {{"shaping"}, OTBOOL, {0} }, + {{"dst_ip"}, OTSTRING, .v.STR = "169.254.10.240"}, + {{"dst_port"}, OTINT, {65536} }, + {{"src_port"}, OTINT, {65536} }, +}; + +ark_pkt_gen_t +ark_pktgen_init(void *adr, int ord, int l2_mode) +{ + struct ark_pkt_gen_inst *inst = + rte_malloc("ark_pkt_gen_inst_pmd", + sizeof(struct ark_pkt_gen_inst), 0); + if (inst == NULL) { + PMD_DRV_LOG(ERR, "Failed to malloc ark_pkt_gen_inst.\n"); + return inst; + } + inst->regs = (struct ark_pkt_gen_regs *)adr; + inst->ordinal = ord; + inst->l2_mode = l2_mode; + return inst; +} + +void +ark_pktgen_uninit(ark_pkt_gen_t handle) +{ + rte_free(handle); +} + +void +ark_pktgen_run(ark_pkt_gen_t handle) +{ + struct ark_pkt_gen_inst *inst = (struct ark_pkt_gen_inst *)handle; + + inst->regs->pkt_start_stop = 1; +} + +uint32_t +ark_pktgen_paused(ark_pkt_gen_t handle) +{ + struct ark_pkt_gen_inst *inst = (struct ark_pkt_gen_inst *)handle; + uint32_t r = inst->regs->pkt_start_stop; + + return (((r >> 16) & 1) == 1); +} + +void +ark_pktgen_pause(ark_pkt_gen_t handle) +{ + struct ark_pkt_gen_inst *inst = (struct ark_pkt_gen_inst *)handle; + int cnt = 0; + + inst->regs->pkt_start_stop = 0; + + while (!ark_pktgen_paused(handle)) { + usleep(1000); + if (cnt++ > 100) { + PMD_DRV_LOG(ERR, "Pktgen %d failed to pause.\n", + inst->ordinal); + break; + } + } + PMD_DEBUG_LOG(DEBUG, "Pktgen %d paused.\n", inst->ordinal); +} + +void +ark_pktgen_reset(ark_pkt_gen_t handle) +{ + struct ark_pkt_gen_inst *inst = (struct ark_pkt_gen_inst *)handle; + + if (!ark_pktgen_is_running(handle) && + !ark_pktgen_paused(handle)) { + PMD_DEBUG_LOG(DEBUG, "Pktgen %d is not running" + " and is not paused. No need to reset.\n", + inst->ordinal); + return; + } + + if (ark_pktgen_is_running(handle) && + !ark_pktgen_paused(handle)) { + PMD_DEBUG_LOG(DEBUG, + "Pktgen %d is not paused. Pausing first.\n", + inst->ordinal); + ark_pktgen_pause(handle); + } + + PMD_DEBUG_LOG(DEBUG, "Resetting pktgen %d.\n", inst->ordinal); + inst->regs->pkt_start_stop = (1 << 8); +} + +uint32_t +ark_pktgen_tx_done(ark_pkt_gen_t handle) +{ + struct ark_pkt_gen_inst *inst = (struct ark_pkt_gen_inst *)handle; + uint32_t r = inst->regs->pkt_start_stop; + + return (((r >> 24) & 1) == 1); +} + +uint32_t +ark_pktgen_is_running(ark_pkt_gen_t handle) +{ + struct ark_pkt_gen_inst *inst = (struct ark_pkt_gen_inst *)handle; + uint32_t r = inst->regs->pkt_start_stop; + + return ((r & 1) == 1); +} + +uint32_t +ark_pktgen_is_gen_forever(ark_pkt_gen_t handle) +{ + struct ark_pkt_gen_inst *inst = (struct ark_pkt_gen_inst *)handle; + uint32_t r = inst->regs->pkt_ctrl; + + return (((r >> 24) & 1) == 1); +} + +void +ark_pktgen_wait_done(ark_pkt_gen_t handle) +{ + struct ark_pkt_gen_inst *inst = (struct ark_pkt_gen_inst *)handle; + int wait_cycle = 10; + + if (ark_pktgen_is_gen_forever(handle)) + PMD_DRV_LOG(ERR, "Pktgen wait_done will not terminate" + " because gen_forever=1\n"); + + while (!ark_pktgen_tx_done(handle) && (wait_cycle > 0)) { + usleep(1000); + wait_cycle--; + PMD_DEBUG_LOG(DEBUG, + "Waiting for pktgen %d to finish sending...\n", + inst->ordinal); + } + PMD_DEBUG_LOG(DEBUG, "Pktgen %d done.\n", inst->ordinal); +} + +uint32_t +ark_pktgen_get_pkts_sent(ark_pkt_gen_t handle) +{ + struct ark_pkt_gen_inst *inst = (struct ark_pkt_gen_inst *)handle; + return inst->regs->pkts_sent; +} + +void +ark_pktgen_set_payload_byte(ark_pkt_gen_t handle, uint32_t b) +{ + struct ark_pkt_gen_inst *inst = (struct ark_pkt_gen_inst *)handle; + inst->regs->pkt_payload = b; +} + +void +ark_pktgen_set_pkt_spacing(ark_pkt_gen_t handle, uint32_t x) +{ + struct ark_pkt_gen_inst *inst = (struct ark_pkt_gen_inst *)handle; + inst->regs->pkt_spacing = x; +} + +void +ark_pktgen_set_pkt_size_min(ark_pkt_gen_t handle, uint32_t x) +{ + struct ark_pkt_gen_inst *inst = (struct ark_pkt_gen_inst *)handle; + inst->regs->pkt_size_min = x; +} + +void +ark_pktgen_set_pkt_size_max(ark_pkt_gen_t handle, uint32_t x) +{ + struct ark_pkt_gen_inst *inst = (struct ark_pkt_gen_inst *)handle; + inst->regs->pkt_size_max = x; +} + +void +ark_pktgen_set_pkt_size_incr(ark_pkt_gen_t handle, uint32_t x) +{ + struct ark_pkt_gen_inst *inst = (struct ark_pkt_gen_inst *)handle; + inst->regs->pkt_size_incr = x; +} + +void +ark_pktgen_set_num_pkts(ark_pkt_gen_t handle, uint32_t x) +{ + struct ark_pkt_gen_inst *inst = (struct ark_pkt_gen_inst *)handle; + inst->regs->num_pkts = x; +} + +void +ark_pktgen_set_src_mac_addr(ark_pkt_gen_t handle, uint64_t mac_addr) +{ + struct ark_pkt_gen_inst *inst = (struct ark_pkt_gen_inst *)handle; + inst->regs->src_mac_addr_h = (mac_addr >> 32) & 0xffff; + inst->regs->src_mac_addr_l = mac_addr & 0xffffffff; +} + +void +ark_pktgen_set_dst_mac_addr(ark_pkt_gen_t handle, uint64_t mac_addr) +{ + struct ark_pkt_gen_inst *inst = (struct ark_pkt_gen_inst *)handle; + inst->regs->dst_mac_addr_h = (mac_addr >> 32) & 0xffff; + inst->regs->dst_mac_addr_l = mac_addr & 0xffffffff; +} + +void +ark_pktgen_set_eth_type(ark_pkt_gen_t handle, uint32_t x) +{ + struct ark_pkt_gen_inst *inst = (struct ark_pkt_gen_inst *)handle; + inst->regs->eth_type = x; +} + +void +ark_pktgen_set_hdr_dW(ark_pkt_gen_t handle, uint32_t *hdr) +{ + uint32_t i; + struct ark_pkt_gen_inst *inst = (struct ark_pkt_gen_inst *)handle; + + for (i = 0; i < 7; i++) + inst->regs->hdr_dw[i] = hdr[i]; +} + +void +ark_pktgen_set_start_offset(ark_pkt_gen_t handle, uint32_t x) +{ + struct ark_pkt_gen_inst *inst = (struct ark_pkt_gen_inst *)handle; + + inst->regs->start_offset = x; +} + +static struct OPTIONS * +options(const char *id) +{ + unsigned int i; + + for (i = 0; i < sizeof(toptions) / sizeof(struct OPTIONS); i++) { + if (strcmp(id, toptions[i].opt) == 0) + return &toptions[i]; + } + + PMD_DRV_LOG(ERR, + "Pktgen: Could not find requested option!, " + "option = %s\n", + id + ); + return NULL; +} + +static int pmd_set_arg(char *arg, char *val); +static int +pmd_set_arg(char *arg, char *val) +{ + struct OPTIONS *o = options(arg); + + if (o) { + switch (o->t) { + case OTINT: + case OTBOOL: + o->v.INT = atoi(val); + break; + case OTLONG: + o->v.INT = atoll(val); + break; + case OTSTRING: + snprintf(o->v.STR, ARK_MAX_STR_LEN, "%s", val); + break; + } + return 1; + } + return 0; +} + +/****** + * Arg format = "opt0=v,opt_n=v ..." + ******/ +void +ark_pktgen_parse(char *args) +{ + char *argv, *v; + const char toks[] = " =\n\t\v\f \r"; + argv = strtok(args, toks); + v = strtok(NULL, toks); + while (argv && v) { + pmd_set_arg(argv, v); + argv = strtok(NULL, toks); + v = strtok(NULL, toks); + } +} + +static int32_t parse_ipv4_string(char const *ip_address); +static int32_t +parse_ipv4_string(char const *ip_address) +{ + unsigned int ip[4]; + + if (sscanf(ip_address, "%u.%u.%u.%u", + &ip[0], &ip[1], &ip[2], &ip[3]) != 4) + return 0; + return ip[3] + ip[2] * 0x100 + ip[1] * 0x10000ul + ip[0] * 0x1000000ul; +} + +static void +ark_pktgen_set_pkt_ctrl(ark_pkt_gen_t handle, + uint32_t gen_forever, + uint32_t en_slaved_start, + uint32_t vary_length, + uint32_t incr_payload, + uint32_t incr_first_byte, + uint32_t ins_seq_num, + uint32_t ins_udp_hdr, + uint32_t ins_time_stamp) +{ + uint32_t r; + struct ark_pkt_gen_inst *inst = (struct ark_pkt_gen_inst *)handle; + + if (!inst->l2_mode) + ins_udp_hdr = 0; + + r = ((gen_forever << 24) | + (en_slaved_start << 20) | + (vary_length << 16) | + (incr_payload << 12) | + (incr_first_byte << 8) | + (ins_time_stamp << 5) | + (ins_seq_num << 4) | + ins_udp_hdr); + + inst->regs->bytes_per_cycle = options("bytes_per_cycle")->v.INT; + if (options("shaping")->v.BOOL) + r = r | (1 << 28); /* enable shaping */ + + inst->regs->pkt_ctrl = r; +} + +void +ark_pktgen_setup(ark_pkt_gen_t handle) +{ + uint32_t hdr[7]; + int32_t dst_ip = parse_ipv4_string(options("dst_ip")->v.STR); + + if (!options("pause")->v.BOOL && + (!options("reset")->v.BOOL && + (options("configure")->v.BOOL))) { + ark_pktgen_set_payload_byte(handle, + options("payload_byte")->v.INT); + ark_pktgen_set_src_mac_addr(handle, + options("src_mac_addr")->v.INT); + ark_pktgen_set_dst_mac_addr(handle, + options("dst_mac_addr")->v.LONG); + ark_pktgen_set_eth_type(handle, + options("eth_type")->v.INT); + + if (options("dg-mode")->v.BOOL) { + hdr[0] = options("hdr_dW0")->v.INT; + hdr[1] = options("hdr_dW1")->v.INT; + hdr[2] = options("hdr_dW2")->v.INT; + hdr[3] = options("hdr_dW3")->v.INT; + hdr[4] = options("hdr_dW4")->v.INT; + hdr[5] = options("hdr_dW5")->v.INT; + hdr[6] = options("hdr_dW6")->v.INT; + } else { + hdr[0] = dst_ip; + hdr[1] = options("dst_port")->v.INT; + hdr[2] = options("src_port")->v.INT; + hdr[3] = 0; + hdr[4] = 0; + hdr[5] = 0; + hdr[6] = 0; + } + ark_pktgen_set_hdr_dW(handle, hdr); + ark_pktgen_set_num_pkts(handle, + options("num_pkts")->v.INT); + ark_pktgen_set_pkt_size_min(handle, + options("pkt_size_min")->v.INT); + ark_pktgen_set_pkt_size_max(handle, + options("pkt_size_max")->v.INT); + ark_pktgen_set_pkt_size_incr(handle, + options("pkt_size_incr")->v.INT); + ark_pktgen_set_pkt_spacing(handle, + options("pkt_spacing")->v.INT); + ark_pktgen_set_start_offset(handle, + options("start_offset")->v.INT); + ark_pktgen_set_pkt_ctrl(handle, + options("gen_forever")->v.BOOL, + options("en_slaved_start")->v.BOOL, + options("vary_length")->v.BOOL, + options("incr_payload")->v.BOOL, + options("incr_first_byte")->v.BOOL, + options("ins_seq_num")->v.INT, + options("ins_udp_hdr")->v.BOOL, + options("ins_time_stamp")->v.INT); + } + + if (options("pause")->v.BOOL) + ark_pktgen_pause(handle); + + if (options("reset")->v.BOOL) + ark_pktgen_reset(handle); + if (options("run")->v.BOOL) { + PMD_DEBUG_LOG(DEBUG, "Starting packet generator on port %d\n", + options("port")->v.INT); + ark_pktgen_run(handle); + } +} diff --git a/dpdk/drivers/net/ark/ark_pktgen.h b/dpdk/drivers/net/ark/ark_pktgen.h new file mode 100644 index 00000000..bf5a241b --- /dev/null +++ b/dpdk/drivers/net/ark/ark_pktgen.h @@ -0,0 +1,108 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2015-2017 Atomic Rules LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _ARK_PKTGEN_H_ +#define _ARK_PKTGEN_H_ + +#include +#include + +#define ARK_PKTGEN_BASE_ADR 0x10000 + +typedef void *ark_pkt_gen_t; + +/* The packet generator is an internal Arkville hardware module, which + * generates known packets for use in integrity and line-rate testing. + * This module is *not* intended for end-user manipulation, hence + * there is minimal documentation. + */ + +/* + * This is an overlay structure to a memory mapped FPGA device. These + * structs will never be instantiated in ram memory + */ +struct ark_pkt_gen_regs { + uint32_t r0; + volatile uint32_t pkt_start_stop; + volatile uint32_t pkt_ctrl; + uint32_t pkt_payload; + uint32_t pkt_spacing; + uint32_t pkt_size_min; + uint32_t pkt_size_max; + uint32_t pkt_size_incr; + volatile uint32_t num_pkts; + volatile uint32_t pkts_sent; + uint32_t src_mac_addr_l; + uint32_t src_mac_addr_h; + uint32_t dst_mac_addr_l; + uint32_t dst_mac_addr_h; + uint32_t eth_type; + uint32_t hdr_dw[7]; + uint32_t start_offset; + uint32_t bytes_per_cycle; +} __attribute__ ((packed)); + +struct ark_pkt_gen_inst { + struct rte_eth_dev_info *dev_info; + struct ark_pkt_gen_regs *regs; + int l2_mode; + int ordinal; +}; + +/* packet generator functions */ +ark_pkt_gen_t ark_pktgen_init(void *arg, int ord, int l2_mode); +void ark_pktgen_uninit(ark_pkt_gen_t handle); +void ark_pktgen_run(ark_pkt_gen_t handle); +void ark_pktgen_pause(ark_pkt_gen_t handle); +uint32_t ark_pktgen_paused(ark_pkt_gen_t handle); +uint32_t ark_pktgen_is_gen_forever(ark_pkt_gen_t handle); +uint32_t ark_pktgen_is_running(ark_pkt_gen_t handle); +uint32_t ark_pktgen_tx_done(ark_pkt_gen_t handle); +void ark_pktgen_reset(ark_pkt_gen_t handle); +void ark_pktgen_wait_done(ark_pkt_gen_t handle); +uint32_t ark_pktgen_get_pkts_sent(ark_pkt_gen_t handle); +void ark_pktgen_set_payload_byte(ark_pkt_gen_t handle, uint32_t b); +void ark_pktgen_set_pkt_spacing(ark_pkt_gen_t handle, uint32_t x); +void ark_pktgen_set_pkt_size_min(ark_pkt_gen_t handle, uint32_t x); +void ark_pktgen_set_pkt_size_max(ark_pkt_gen_t handle, uint32_t x); +void ark_pktgen_set_pkt_size_incr(ark_pkt_gen_t handle, uint32_t x); +void ark_pktgen_set_num_pkts(ark_pkt_gen_t handle, uint32_t x); +void ark_pktgen_set_src_mac_addr(ark_pkt_gen_t handle, uint64_t mac_addr); +void ark_pktgen_set_dst_mac_addr(ark_pkt_gen_t handle, uint64_t mac_addr); +void ark_pktgen_set_eth_type(ark_pkt_gen_t handle, uint32_t x); +void ark_pktgen_set_hdr_dW(ark_pkt_gen_t handle, uint32_t *hdr); +void ark_pktgen_set_start_offset(ark_pkt_gen_t handle, uint32_t x); +void ark_pktgen_parse(char *argv); +void ark_pktgen_setup(ark_pkt_gen_t handle); + +#endif diff --git a/dpdk/drivers/net/ark/ark_rqp.c b/dpdk/drivers/net/ark/ark_rqp.c new file mode 100644 index 00000000..41c497b0 --- /dev/null +++ b/dpdk/drivers/net/ark/ark_rqp.c @@ -0,0 +1,97 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2015-2017 Atomic Rules LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +#include "ark_rqp.h" +#include "ark_logs.h" + +/* ************************************************************************* */ +void +ark_rqp_stats_reset(struct ark_rqpace_t *rqp) +{ + rqp->stats_clear = 1; + /* POR 992 */ + /* rqp->cpld_max = 992; */ + /* POR 64 */ + /* rqp->cplh_max = 64; */ +} + +/* ************************************************************************* */ +void +ark_rqp_dump(struct ark_rqpace_t *rqp) +{ + if (rqp->err_count_other != 0) + PMD_DRV_LOG(ERR, + "RQP Errors noted: ctrl: %d cplh_hmax %d cpld_max %d" + ARK_SU32 + ARK_SU32 "\n", + rqp->ctrl, rqp->cplh_max, rqp->cpld_max, + "Error Count", rqp->err_cnt, + "Error General", rqp->err_count_other); + + PMD_STATS_LOG(INFO, "RQP Dump: ctrl: %d cplh_hmax %d cpld_max %d" + ARK_SU32 + ARK_SU32 ARK_SU32 ARK_SU32 ARK_SU32 ARK_SU32 ARK_SU32 + ARK_SU32 ARK_SU32 ARK_SU32 ARK_SU32 ARK_SU32 ARK_SU32 + ARK_SU32 ARK_SU32 ARK_SU32 + ARK_SU32 ARK_SU32 ARK_SU32 ARK_SU32 ARK_SU32 "\n", + rqp->ctrl, rqp->cplh_max, rqp->cpld_max, + "Error Count", rqp->err_cnt, + "Error General", rqp->err_count_other, + "stall_pS", rqp->stall_ps, + "stall_pS Min", rqp->stall_ps_min, + "stall_pS Max", rqp->stall_ps_max, + "req_pS", rqp->req_ps, + "req_pS Min", rqp->req_ps_min, + "req_pS Max", rqp->req_ps_max, + "req_dWPS", rqp->req_dw_ps, + "req_dWPS Min", rqp->req_dw_ps_min, + "req_dWPS Max", rqp->req_dw_ps_max, + "cpl_pS", rqp->cpl_ps, + "cpl_pS Min", rqp->cpl_ps_min, + "cpl_pS Max", rqp->cpl_ps_max, + "cpl_dWPS", rqp->cpl_dw_ps, + "cpl_dWPS Min", rqp->cpl_dw_ps_min, + "cpl_dWPS Max", rqp->cpl_dw_ps_max, + "cplh pending", rqp->cplh_pending, + "cpld pending", rqp->cpld_pending, + "cplh pending max", rqp->cplh_pending_max, + "cpld pending max", rqp->cpld_pending_max); +} + +int +ark_rqp_lasped(struct ark_rqpace_t *rqp) +{ + return rqp->lasped; +} diff --git a/dpdk/drivers/net/ark/ark_rqp.h b/dpdk/drivers/net/ark/ark_rqp.h new file mode 100644 index 00000000..0c380071 --- /dev/null +++ b/dpdk/drivers/net/ark/ark_rqp.h @@ -0,0 +1,86 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2015-2017 Atomic Rules LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _ARK_RQP_H_ +#define _ARK_RQP_H_ + +#include + +#include + +/* The RQP or ReQuest Pacer is an internal Arkville hardware module + * which limits the PCIE data flow to insure correct operation for the + * particular hardware PCIE endpoint. + * This module is *not* intended for end-user manipulation, hence + * there is minimal documentation. + */ + +/* + * RQ Pacing core hardware structure + * This is an overlay structures to a memory mapped FPGA device. These + * structs will never be instantiated in ram memory + */ +struct ark_rqpace_t { + volatile uint32_t ctrl; + volatile uint32_t stats_clear; + volatile uint32_t cplh_max; + volatile uint32_t cpld_max; + volatile uint32_t err_cnt; + volatile uint32_t stall_ps; + volatile uint32_t stall_ps_min; + volatile uint32_t stall_ps_max; + volatile uint32_t req_ps; + volatile uint32_t req_ps_min; + volatile uint32_t req_ps_max; + volatile uint32_t req_dw_ps; + volatile uint32_t req_dw_ps_min; + volatile uint32_t req_dw_ps_max; + volatile uint32_t cpl_ps; + volatile uint32_t cpl_ps_min; + volatile uint32_t cpl_ps_max; + volatile uint32_t cpl_dw_ps; + volatile uint32_t cpl_dw_ps_min; + volatile uint32_t cpl_dw_ps_max; + volatile uint32_t cplh_pending; + volatile uint32_t cpld_pending; + volatile uint32_t cplh_pending_max; + volatile uint32_t cpld_pending_max; + volatile uint32_t err_count_other; + char eval[4]; + volatile int lasped; +}; + +void ark_rqp_dump(struct ark_rqpace_t *rqp); +void ark_rqp_stats_reset(struct ark_rqpace_t *rqp); +int ark_rqp_lasped(struct ark_rqpace_t *rqp); +#endif diff --git a/dpdk/drivers/net/ark/ark_udm.c b/dpdk/drivers/net/ark/ark_udm.c new file mode 100644 index 00000000..7a429ac7 --- /dev/null +++ b/dpdk/drivers/net/ark/ark_udm.c @@ -0,0 +1,226 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2015-2017 Atomic Rules LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +#include "ark_logs.h" +#include "ark_udm.h" + +int +ark_udm_verify(struct ark_udm_t *udm) +{ + if (sizeof(struct ark_udm_t) != ARK_UDM_EXPECT_SIZE) { + PMD_DRV_LOG(ERR, + "ARK: UDM structure looks incorrect %d vs %zd\n", + ARK_UDM_EXPECT_SIZE, sizeof(struct ark_udm_t)); + return -1; + } + + if (udm->setup.const0 != ARK_UDM_CONST) { + PMD_DRV_LOG(ERR, + "ARK: UDM module not found as expected 0x%08x\n", + udm->setup.const0); + return -1; + } + return 0; +} + +int +ark_udm_stop(struct ark_udm_t *udm, const int wait) +{ + int cnt = 0; + + udm->cfg.command = 2; + + while (wait && (udm->cfg.stop_flushed & 0x01) == 0) { + if (cnt++ > 1000) + return 1; + + usleep(10); + } + return 0; +} + +int +ark_udm_reset(struct ark_udm_t *udm) +{ + int status; + + status = ark_udm_stop(udm, 1); + if (status != 0) { + PMD_DEBUG_LOG(INFO, "%s stop failed doing forced reset\n", + __func__); + udm->cfg.command = 4; + usleep(10); + udm->cfg.command = 3; + status = ark_udm_stop(udm, 0); + PMD_DEBUG_LOG(INFO, "%s stop status %d post failure" + " and forced reset\n", + __func__, status); + } else { + udm->cfg.command = 3; + } + + return status; +} + +void +ark_udm_start(struct ark_udm_t *udm) +{ + udm->cfg.command = 1; +} + +void +ark_udm_stats_reset(struct ark_udm_t *udm) +{ + udm->pcibp.pci_clear = 1; + udm->tlp_ps.tlp_clear = 1; +} + +void +ark_udm_configure(struct ark_udm_t *udm, + uint32_t headroom, + uint32_t dataroom, + uint32_t write_interval_ns) +{ + /* headroom and data room are in DWords in the UDM */ + udm->cfg.dataroom = dataroom / 4; + udm->cfg.headroom = headroom / 4; + + /* 4 NS period ns */ + udm->rt_cfg.write_interval = write_interval_ns / 4; +} + +void +ark_udm_write_addr(struct ark_udm_t *udm, rte_iova_t addr) +{ + udm->rt_cfg.hw_prod_addr = addr; +} + +int +ark_udm_is_flushed(struct ark_udm_t *udm) +{ + return (udm->cfg.stop_flushed & 0x01) != 0; +} + +uint64_t +ark_udm_dropped(struct ark_udm_t *udm) +{ + return udm->qstats.q_pkt_drop; +} + +uint64_t +ark_udm_bytes(struct ark_udm_t *udm) +{ + return udm->qstats.q_byte_count; +} + +uint64_t +ark_udm_packets(struct ark_udm_t *udm) +{ + return udm->qstats.q_ff_packet_count; +} + +void +ark_udm_dump_stats(struct ark_udm_t *udm, const char *msg) +{ + PMD_STATS_LOG(INFO, "UDM Stats: %s" + ARK_SU64 ARK_SU64 ARK_SU64 ARK_SU64 ARK_SU64 "\n", + msg, + "Pkts Received", udm->stats.rx_packet_count, + "Pkts Finalized", udm->stats.rx_sent_packets, + "Pkts Dropped", udm->tlp.pkt_drop, + "Bytes Count", udm->stats.rx_byte_count, + "MBuf Count", udm->stats.rx_mbuf_count); +} + +void +ark_udm_dump_queue_stats(struct ark_udm_t *udm, const char *msg, uint16_t qid) +{ + PMD_STATS_LOG(INFO, "UDM Queue %3u Stats: %s" + ARK_SU64 ARK_SU64 + ARK_SU64 ARK_SU64 + ARK_SU64 "\n", + qid, msg, + "Pkts Received", udm->qstats.q_packet_count, + "Pkts Finalized", udm->qstats.q_ff_packet_count, + "Pkts Dropped", udm->qstats.q_pkt_drop, + "Bytes Count", udm->qstats.q_byte_count, + "MBuf Count", udm->qstats.q_mbuf_count); +} + +void +ark_udm_dump(struct ark_udm_t *udm, const char *msg) +{ + PMD_DEBUG_LOG(DEBUG, "UDM Dump: %s Stopped: %d\n", msg, + udm->cfg.stop_flushed); +} + +void +ark_udm_dump_setup(struct ark_udm_t *udm, uint16_t q_id) +{ + PMD_DEBUG_LOG(DEBUG, "UDM Setup Q: %u" + ARK_SU64X ARK_SU32 "\n", + q_id, + "hw_prod_addr", udm->rt_cfg.hw_prod_addr, + "prod_idx", udm->rt_cfg.prod_idx); +} + +void +ark_udm_dump_perf(struct ark_udm_t *udm, const char *msg) +{ + struct ark_udm_pcibp_t *bp = &udm->pcibp; + + PMD_STATS_LOG(INFO, "UDM Performance %s" + ARK_SU32 ARK_SU32 ARK_SU32 ARK_SU32 ARK_SU32 ARK_SU32 + "\n", + msg, + "PCI Empty", bp->pci_empty, + "PCI Q1", bp->pci_q1, + "PCI Q2", bp->pci_q2, + "PCI Q3", bp->pci_q3, + "PCI Q4", bp->pci_q4, + "PCI Full", bp->pci_full); +} + +void +ark_udm_queue_stats_reset(struct ark_udm_t *udm) +{ + udm->qstats.q_byte_count = 1; +} + +void +ark_udm_queue_enable(struct ark_udm_t *udm, int enable) +{ + udm->qstats.q_enable = enable ? 1 : 0; +} diff --git a/dpdk/drivers/net/ark/ark_udm.h b/dpdk/drivers/net/ark/ark_udm.h new file mode 100644 index 00000000..915343fe --- /dev/null +++ b/dpdk/drivers/net/ark/ark_udm.h @@ -0,0 +1,192 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2015-2017 Atomic Rules LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _ARK_UDM_H_ +#define _ARK_UDM_H_ + +#include + +#include + +/* The UDM or Upstream Data Mover is an internal Arkville hardware + * module for moving packet from the RX packet streams to host memory. + * This module is *not* intended for end-user manipulation, hence + * there is minimal documentation. + */ + +/* Meta data structure apssed from FPGA, must match layout in FPGA */ +struct ark_rx_meta { + uint64_t timestamp; + uint64_t user_data; + uint8_t port; + uint8_t dst_queue; + uint16_t pkt_len; +}; + +/* + * UDM hardware structures + * These are overlay structures to a memory mapped FPGA device. These + * structs will never be instantiated in ram memory + */ + +#define ARK_RX_WRITE_TIME_NS 2500 +#define ARK_UDM_SETUP 0 +#define ARK_UDM_CONST 0xbACECACE +struct ark_udm_setup_t { + uint32_t r0; + uint32_t r4; + volatile uint32_t cycle_count; + uint32_t const0; +}; + +#define ARK_UDM_CFG 0x010 +struct ark_udm_cfg_t { + volatile uint32_t stop_flushed; /* RO */ + volatile uint32_t command; + uint32_t dataroom; + uint32_t headroom; +}; + +typedef enum { + ARK_UDM_START = 0x1, + ARK_UDM_STOP = 0x2, + ARK_UDM_RESET = 0x3 +} ark_udm_commands; + +#define ARK_UDM_STATS 0x020 +struct ark_udm_stats_t { + volatile uint64_t rx_byte_count; + volatile uint64_t rx_packet_count; + volatile uint64_t rx_mbuf_count; + volatile uint64_t rx_sent_packets; +}; + +#define ARK_UDM_PQ 0x040 +struct ark_udm_queue_stats_t { + volatile uint64_t q_byte_count; + volatile uint64_t q_packet_count; /* includes drops */ + volatile uint64_t q_mbuf_count; + volatile uint64_t q_ff_packet_count; + volatile uint64_t q_pkt_drop; + uint32_t q_enable; +}; + +#define ARK_UDM_TLP 0x0070 +struct ark_udm_tlp_t { + volatile uint64_t pkt_drop; /* global */ + volatile uint32_t tlp_q1; + volatile uint32_t tlp_q2; + volatile uint32_t tlp_q3; + volatile uint32_t tlp_q4; + volatile uint32_t tlp_full; +}; + +#define ARK_UDM_PCIBP 0x00a0 +struct ark_udm_pcibp_t { + volatile uint32_t pci_clear; + volatile uint32_t pci_empty; + volatile uint32_t pci_q1; + volatile uint32_t pci_q2; + volatile uint32_t pci_q3; + volatile uint32_t pci_q4; + volatile uint32_t pci_full; +}; + +#define ARK_UDM_TLP_PS 0x00bc +struct ark_udm_tlp_ps_t { + volatile uint32_t tlp_clear; + volatile uint32_t tlp_ps_min; + volatile uint32_t tlp_ps_max; + volatile uint32_t tlp_full_ps_min; + volatile uint32_t tlp_full_ps_max; + volatile uint32_t tlp_dw_ps_min; + volatile uint32_t tlp_dw_ps_max; + volatile uint32_t tlp_pldw_ps_min; + volatile uint32_t tlp_pldw_ps_max; +}; + +#define ARK_UDM_RT_CFG 0x00e0 +struct ark_udm_rt_cfg_t { + rte_iova_t hw_prod_addr; + uint32_t write_interval; /* 4ns cycles */ + volatile uint32_t prod_idx; /* RO */ +}; + +/* Consolidated structure */ +#define ARK_UDM_EXPECT_SIZE (0x00fc + 4) +#define ARK_UDM_QOFFSET ARK_UDM_EXPECT_SIZE +struct ark_udm_t { + struct ark_udm_setup_t setup; + struct ark_udm_cfg_t cfg; + struct ark_udm_stats_t stats; + struct ark_udm_queue_stats_t qstats; + uint8_t reserved1[(ARK_UDM_TLP - ARK_UDM_PQ) - + sizeof(struct ark_udm_queue_stats_t)]; + struct ark_udm_tlp_t tlp; + uint8_t reserved2[(ARK_UDM_PCIBP - ARK_UDM_TLP) - + sizeof(struct ark_udm_tlp_t)]; + struct ark_udm_pcibp_t pcibp; + struct ark_udm_tlp_ps_t tlp_ps; + struct ark_udm_rt_cfg_t rt_cfg; + int8_t reserved3[(ARK_UDM_EXPECT_SIZE - ARK_UDM_RT_CFG) - + sizeof(struct ark_udm_rt_cfg_t)]; +}; + + +int ark_udm_verify(struct ark_udm_t *udm); +int ark_udm_stop(struct ark_udm_t *udm, int wait); +void ark_udm_start(struct ark_udm_t *udm); +int ark_udm_reset(struct ark_udm_t *udm); +void ark_udm_configure(struct ark_udm_t *udm, + uint32_t headroom, + uint32_t dataroom, + uint32_t write_interval_ns); +void ark_udm_write_addr(struct ark_udm_t *udm, rte_iova_t addr); +void ark_udm_stats_reset(struct ark_udm_t *udm); +void ark_udm_dump_stats(struct ark_udm_t *udm, const char *msg); +void ark_udm_dump_queue_stats(struct ark_udm_t *udm, const char *msg, + uint16_t qid); +void ark_udm_dump(struct ark_udm_t *udm, const char *msg); +void ark_udm_dump_perf(struct ark_udm_t *udm, const char *msg); +void ark_udm_dump_setup(struct ark_udm_t *udm, uint16_t q_id); +int ark_udm_is_flushed(struct ark_udm_t *udm); + +/* Per queue data */ +uint64_t ark_udm_dropped(struct ark_udm_t *udm); +uint64_t ark_udm_bytes(struct ark_udm_t *udm); +uint64_t ark_udm_packets(struct ark_udm_t *udm); + +void ark_udm_queue_stats_reset(struct ark_udm_t *udm); +void ark_udm_queue_enable(struct ark_udm_t *udm, int enable); + +#endif diff --git a/dpdk/drivers/net/ark/rte_pmd_ark_version.map b/dpdk/drivers/net/ark/rte_pmd_ark_version.map new file mode 100644 index 00000000..1062e042 --- /dev/null +++ b/dpdk/drivers/net/ark/rte_pmd_ark_version.map @@ -0,0 +1,4 @@ +DPDK_17.05 { + local: *; + +}; diff --git a/dpdk/drivers/net/avp/Makefile b/dpdk/drivers/net/avp/Makefile new file mode 100644 index 00000000..c29ecf45 --- /dev/null +++ b/dpdk/drivers/net/avp/Makefile @@ -0,0 +1,60 @@ +# BSD LICENSE +# +# Copyright(c) 2013-2017, Wind River Systems, Inc. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Wind River Systems nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +include $(RTE_SDK)/mk/rte.vars.mk + +# +# library name +# +LIB = librte_pmd_avp.a + +CFLAGS += -O3 +CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs +LDLIBS += -lrte_bus_pci + +EXPORT_MAP := rte_pmd_avp_version.map + +LIBABIVER := 1 + +# install public header files to enable compilation of the hypervisor level +# dpdk application +SYMLINK-$(CONFIG_RTE_LIBRTE_AVP_PMD)-include += rte_avp_common.h +SYMLINK-$(CONFIG_RTE_LIBRTE_AVP_PMD)-include += rte_avp_fifo.h + +# +# all source files are stored in SRCS-y +# +SRCS-$(CONFIG_RTE_LIBRTE_AVP_PMD) += avp_ethdev.c + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/net/avp/avp_ethdev.c b/dpdk/drivers/net/avp/avp_ethdev.c new file mode 100644 index 00000000..9b342bfa --- /dev/null +++ b/dpdk/drivers/net/avp/avp_ethdev.c @@ -0,0 +1,2314 @@ +/* + * BSD LICENSE + * + * Copyright (c) 2013-2017, Wind River Systems, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1) Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2) Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3) Neither the name of Wind River Systems nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "rte_avp_common.h" +#include "rte_avp_fifo.h" + +#include "avp_logs.h" + + +static int avp_dev_create(struct rte_pci_device *pci_dev, + struct rte_eth_dev *eth_dev); + +static int avp_dev_configure(struct rte_eth_dev *dev); +static int avp_dev_start(struct rte_eth_dev *dev); +static void avp_dev_stop(struct rte_eth_dev *dev); +static void avp_dev_close(struct rte_eth_dev *dev); +static void avp_dev_info_get(struct rte_eth_dev *dev, + struct rte_eth_dev_info *dev_info); +static int avp_vlan_offload_set(struct rte_eth_dev *dev, int mask); +static int avp_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete); +static void avp_dev_promiscuous_enable(struct rte_eth_dev *dev); +static void avp_dev_promiscuous_disable(struct rte_eth_dev *dev); + +static int avp_dev_rx_queue_setup(struct rte_eth_dev *dev, + uint16_t rx_queue_id, + uint16_t nb_rx_desc, + unsigned int socket_id, + const struct rte_eth_rxconf *rx_conf, + struct rte_mempool *pool); + +static int avp_dev_tx_queue_setup(struct rte_eth_dev *dev, + uint16_t tx_queue_id, + uint16_t nb_tx_desc, + unsigned int socket_id, + const struct rte_eth_txconf *tx_conf); + +static uint16_t avp_recv_scattered_pkts(void *rx_queue, + struct rte_mbuf **rx_pkts, + uint16_t nb_pkts); + +static uint16_t avp_recv_pkts(void *rx_queue, + struct rte_mbuf **rx_pkts, + uint16_t nb_pkts); + +static uint16_t avp_xmit_scattered_pkts(void *tx_queue, + struct rte_mbuf **tx_pkts, + uint16_t nb_pkts); + +static uint16_t avp_xmit_pkts(void *tx_queue, + struct rte_mbuf **tx_pkts, + uint16_t nb_pkts); + +static void avp_dev_rx_queue_release(void *rxq); +static void avp_dev_tx_queue_release(void *txq); + +static int avp_dev_stats_get(struct rte_eth_dev *dev, + struct rte_eth_stats *stats); +static void avp_dev_stats_reset(struct rte_eth_dev *dev); + + +#define AVP_MAX_RX_BURST 64 +#define AVP_MAX_TX_BURST 64 +#define AVP_MAX_MAC_ADDRS 1 +#define AVP_MIN_RX_BUFSIZE ETHER_MIN_LEN + + +/* + * Defines the number of microseconds to wait before checking the response + * queue for completion. + */ +#define AVP_REQUEST_DELAY_USECS (5000) + +/* + * Defines the number times to check the response queue for completion before + * declaring a timeout. + */ +#define AVP_MAX_REQUEST_RETRY (100) + +/* Defines the current PCI driver version number */ +#define AVP_DPDK_DRIVER_VERSION RTE_AVP_CURRENT_GUEST_VERSION + +/* + * The set of PCI devices this driver supports + */ +static const struct rte_pci_id pci_id_avp_map[] = { + { .vendor_id = RTE_AVP_PCI_VENDOR_ID, + .device_id = RTE_AVP_PCI_DEVICE_ID, + .subsystem_vendor_id = RTE_AVP_PCI_SUB_VENDOR_ID, + .subsystem_device_id = RTE_AVP_PCI_SUB_DEVICE_ID, + .class_id = RTE_CLASS_ANY_ID, + }, + + { .vendor_id = 0, /* sentinel */ + }, +}; + +/* + * dev_ops for avp, bare necessities for basic operation + */ +static const struct eth_dev_ops avp_eth_dev_ops = { + .dev_configure = avp_dev_configure, + .dev_start = avp_dev_start, + .dev_stop = avp_dev_stop, + .dev_close = avp_dev_close, + .dev_infos_get = avp_dev_info_get, + .vlan_offload_set = avp_vlan_offload_set, + .stats_get = avp_dev_stats_get, + .stats_reset = avp_dev_stats_reset, + .link_update = avp_dev_link_update, + .promiscuous_enable = avp_dev_promiscuous_enable, + .promiscuous_disable = avp_dev_promiscuous_disable, + .rx_queue_setup = avp_dev_rx_queue_setup, + .rx_queue_release = avp_dev_rx_queue_release, + .tx_queue_setup = avp_dev_tx_queue_setup, + .tx_queue_release = avp_dev_tx_queue_release, +}; + +/**@{ AVP device flags */ +#define AVP_F_PROMISC (1 << 1) +#define AVP_F_CONFIGURED (1 << 2) +#define AVP_F_LINKUP (1 << 3) +#define AVP_F_DETACHED (1 << 4) +/**@} */ + +/* Ethernet device validation marker */ +#define AVP_ETHDEV_MAGIC 0x92972862 + +/* + * Defines the AVP device attributes which are attached to an RTE ethernet + * device + */ +struct avp_dev { + uint32_t magic; /**< Memory validation marker */ + uint64_t device_id; /**< Unique system identifier */ + struct ether_addr ethaddr; /**< Host specified MAC address */ + struct rte_eth_dev_data *dev_data; + /**< Back pointer to ethernet device data */ + volatile uint32_t flags; /**< Device operational flags */ + uint16_t port_id; /**< Ethernet port identifier */ + struct rte_mempool *pool; /**< pkt mbuf mempool */ + unsigned int guest_mbuf_size; /**< local pool mbuf size */ + unsigned int host_mbuf_size; /**< host mbuf size */ + unsigned int max_rx_pkt_len; /**< maximum receive unit */ + uint32_t host_features; /**< Supported feature bitmap */ + uint32_t features; /**< Enabled feature bitmap */ + unsigned int num_tx_queues; /**< Negotiated number of transmit queues */ + unsigned int max_tx_queues; /**< Maximum number of transmit queues */ + unsigned int num_rx_queues; /**< Negotiated number of receive queues */ + unsigned int max_rx_queues; /**< Maximum number of receive queues */ + + struct rte_avp_fifo *tx_q[RTE_AVP_MAX_QUEUES]; /**< TX queue */ + struct rte_avp_fifo *rx_q[RTE_AVP_MAX_QUEUES]; /**< RX queue */ + struct rte_avp_fifo *alloc_q[RTE_AVP_MAX_QUEUES]; + /**< Allocated mbufs queue */ + struct rte_avp_fifo *free_q[RTE_AVP_MAX_QUEUES]; + /**< To be freed mbufs queue */ + + /* mutual exclusion over the 'flag' and 'resp_q/req_q' fields */ + rte_spinlock_t lock; + + /* For request & response */ + struct rte_avp_fifo *req_q; /**< Request queue */ + struct rte_avp_fifo *resp_q; /**< Response queue */ + void *host_sync_addr; /**< (host) Req/Resp Mem address */ + void *sync_addr; /**< Req/Resp Mem address */ + void *host_mbuf_addr; /**< (host) MBUF pool start address */ + void *mbuf_addr; /**< MBUF pool start address */ +} __rte_cache_aligned; + +/* RTE ethernet private data */ +struct avp_adapter { + struct avp_dev avp; +} __rte_cache_aligned; + + +/* 32-bit MMIO register write */ +#define AVP_WRITE32(_value, _addr) rte_write32_relaxed((_value), (_addr)) + +/* 32-bit MMIO register read */ +#define AVP_READ32(_addr) rte_read32_relaxed((_addr)) + +/* Macro to cast the ethernet device private data to a AVP object */ +#define AVP_DEV_PRIVATE_TO_HW(adapter) \ + (&((struct avp_adapter *)adapter)->avp) + +/* + * Defines the structure of a AVP device queue for the purpose of handling the + * receive and transmit burst callback functions + */ +struct avp_queue { + struct rte_eth_dev_data *dev_data; + /**< Backpointer to ethernet device data */ + struct avp_dev *avp; /**< Backpointer to AVP device */ + uint16_t queue_id; + /**< Queue identifier used for indexing current queue */ + uint16_t queue_base; + /**< Base queue identifier for queue servicing */ + uint16_t queue_limit; + /**< Maximum queue identifier for queue servicing */ + + uint64_t packets; + uint64_t bytes; + uint64_t errors; +}; + +/* send a request and wait for a response + * + * @warning must be called while holding the avp->lock spinlock. + */ +static int +avp_dev_process_request(struct avp_dev *avp, struct rte_avp_request *request) +{ + unsigned int retry = AVP_MAX_REQUEST_RETRY; + void *resp_addr = NULL; + unsigned int count; + int ret; + + PMD_DRV_LOG(DEBUG, "Sending request %u to host\n", request->req_id); + + request->result = -ENOTSUP; + + /* Discard any stale responses before starting a new request */ + while (avp_fifo_get(avp->resp_q, (void **)&resp_addr, 1)) + PMD_DRV_LOG(DEBUG, "Discarding stale response\n"); + + rte_memcpy(avp->sync_addr, request, sizeof(*request)); + count = avp_fifo_put(avp->req_q, &avp->host_sync_addr, 1); + if (count < 1) { + PMD_DRV_LOG(ERR, "Cannot send request %u to host\n", + request->req_id); + ret = -EBUSY; + goto done; + } + + while (retry--) { + /* wait for a response */ + usleep(AVP_REQUEST_DELAY_USECS); + + count = avp_fifo_count(avp->resp_q); + if (count >= 1) { + /* response received */ + break; + } + + if ((count < 1) && (retry == 0)) { + PMD_DRV_LOG(ERR, "Timeout while waiting for a response for %u\n", + request->req_id); + ret = -ETIME; + goto done; + } + } + + /* retrieve the response */ + count = avp_fifo_get(avp->resp_q, (void **)&resp_addr, 1); + if ((count != 1) || (resp_addr != avp->host_sync_addr)) { + PMD_DRV_LOG(ERR, "Invalid response from host, count=%u resp=%p host_sync_addr=%p\n", + count, resp_addr, avp->host_sync_addr); + ret = -ENODATA; + goto done; + } + + /* copy to user buffer */ + rte_memcpy(request, avp->sync_addr, sizeof(*request)); + ret = 0; + + PMD_DRV_LOG(DEBUG, "Result %d received for request %u\n", + request->result, request->req_id); + +done: + return ret; +} + +static int +avp_dev_ctrl_set_link_state(struct rte_eth_dev *eth_dev, unsigned int state) +{ + struct avp_dev *avp = AVP_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); + struct rte_avp_request request; + int ret; + + /* setup a link state change request */ + memset(&request, 0, sizeof(request)); + request.req_id = RTE_AVP_REQ_CFG_NETWORK_IF; + request.if_up = state; + + ret = avp_dev_process_request(avp, &request); + + return ret == 0 ? request.result : ret; +} + +static int +avp_dev_ctrl_set_config(struct rte_eth_dev *eth_dev, + struct rte_avp_device_config *config) +{ + struct avp_dev *avp = AVP_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); + struct rte_avp_request request; + int ret; + + /* setup a configure request */ + memset(&request, 0, sizeof(request)); + request.req_id = RTE_AVP_REQ_CFG_DEVICE; + memcpy(&request.config, config, sizeof(request.config)); + + ret = avp_dev_process_request(avp, &request); + + return ret == 0 ? request.result : ret; +} + +static int +avp_dev_ctrl_shutdown(struct rte_eth_dev *eth_dev) +{ + struct avp_dev *avp = AVP_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); + struct rte_avp_request request; + int ret; + + /* setup a shutdown request */ + memset(&request, 0, sizeof(request)); + request.req_id = RTE_AVP_REQ_SHUTDOWN_DEVICE; + + ret = avp_dev_process_request(avp, &request); + + return ret == 0 ? request.result : ret; +} + +/* translate from host mbuf virtual address to guest virtual address */ +static inline void * +avp_dev_translate_buffer(struct avp_dev *avp, void *host_mbuf_address) +{ + return RTE_PTR_ADD(RTE_PTR_SUB(host_mbuf_address, + (uintptr_t)avp->host_mbuf_addr), + (uintptr_t)avp->mbuf_addr); +} + +/* translate from host physical address to guest virtual address */ +static void * +avp_dev_translate_address(struct rte_eth_dev *eth_dev, + rte_iova_t host_phys_addr) +{ + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); + struct rte_mem_resource *resource; + struct rte_avp_memmap_info *info; + struct rte_avp_memmap *map; + off_t offset; + void *addr; + unsigned int i; + + addr = pci_dev->mem_resource[RTE_AVP_PCI_MEMORY_BAR].addr; + resource = &pci_dev->mem_resource[RTE_AVP_PCI_MEMMAP_BAR]; + info = (struct rte_avp_memmap_info *)resource->addr; + + offset = 0; + for (i = 0; i < info->nb_maps; i++) { + /* search all segments looking for a matching address */ + map = &info->maps[i]; + + if ((host_phys_addr >= map->phys_addr) && + (host_phys_addr < (map->phys_addr + map->length))) { + /* address is within this segment */ + offset += (host_phys_addr - map->phys_addr); + addr = RTE_PTR_ADD(addr, offset); + + PMD_DRV_LOG(DEBUG, "Translating host physical 0x%" PRIx64 " to guest virtual 0x%p\n", + host_phys_addr, addr); + + return addr; + } + offset += map->length; + } + + return NULL; +} + +/* verify that the incoming device version is compatible with our version */ +static int +avp_dev_version_check(uint32_t version) +{ + uint32_t driver = RTE_AVP_STRIP_MINOR_VERSION(AVP_DPDK_DRIVER_VERSION); + uint32_t device = RTE_AVP_STRIP_MINOR_VERSION(version); + + if (device <= driver) { + /* the host driver version is less than or equal to ours */ + return 0; + } + + return 1; +} + +/* verify that memory regions have expected version and validation markers */ +static int +avp_dev_check_regions(struct rte_eth_dev *eth_dev) +{ + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); + struct rte_avp_memmap_info *memmap; + struct rte_avp_device_info *info; + struct rte_mem_resource *resource; + unsigned int i; + + /* Dump resource info for debug */ + for (i = 0; i < PCI_MAX_RESOURCE; i++) { + resource = &pci_dev->mem_resource[i]; + if ((resource->phys_addr == 0) || (resource->len == 0)) + continue; + + PMD_DRV_LOG(DEBUG, "resource[%u]: phys=0x%" PRIx64 " len=%" PRIu64 " addr=%p\n", + i, resource->phys_addr, + resource->len, resource->addr); + + switch (i) { + case RTE_AVP_PCI_MEMMAP_BAR: + memmap = (struct rte_avp_memmap_info *)resource->addr; + if ((memmap->magic != RTE_AVP_MEMMAP_MAGIC) || + (memmap->version != RTE_AVP_MEMMAP_VERSION)) { + PMD_DRV_LOG(ERR, "Invalid memmap magic 0x%08x and version %u\n", + memmap->magic, memmap->version); + return -EINVAL; + } + break; + + case RTE_AVP_PCI_DEVICE_BAR: + info = (struct rte_avp_device_info *)resource->addr; + if ((info->magic != RTE_AVP_DEVICE_MAGIC) || + avp_dev_version_check(info->version)) { + PMD_DRV_LOG(ERR, "Invalid device info magic 0x%08x or version 0x%08x > 0x%08x\n", + info->magic, info->version, + AVP_DPDK_DRIVER_VERSION); + return -EINVAL; + } + break; + + case RTE_AVP_PCI_MEMORY_BAR: + case RTE_AVP_PCI_MMIO_BAR: + if (resource->addr == NULL) { + PMD_DRV_LOG(ERR, "Missing address space for BAR%u\n", + i); + return -EINVAL; + } + break; + + case RTE_AVP_PCI_MSIX_BAR: + default: + /* no validation required */ + break; + } + } + + return 0; +} + +static int +avp_dev_detach(struct rte_eth_dev *eth_dev) +{ + struct avp_dev *avp = AVP_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); + int ret; + + PMD_DRV_LOG(NOTICE, "Detaching port %u from AVP device 0x%" PRIx64 "\n", + eth_dev->data->port_id, avp->device_id); + + rte_spinlock_lock(&avp->lock); + + if (avp->flags & AVP_F_DETACHED) { + PMD_DRV_LOG(NOTICE, "port %u already detached\n", + eth_dev->data->port_id); + ret = 0; + goto unlock; + } + + /* shutdown the device first so the host stops sending us packets. */ + ret = avp_dev_ctrl_shutdown(eth_dev); + if (ret < 0) { + PMD_DRV_LOG(ERR, "Failed to send/recv shutdown to host, ret=%d\n", + ret); + avp->flags &= ~AVP_F_DETACHED; + goto unlock; + } + + avp->flags |= AVP_F_DETACHED; + rte_wmb(); + + /* wait for queues to acknowledge the presence of the detach flag */ + rte_delay_ms(1); + + ret = 0; + +unlock: + rte_spinlock_unlock(&avp->lock); + return ret; +} + +static void +_avp_set_rx_queue_mappings(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id) +{ + struct avp_dev *avp = + AVP_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); + struct avp_queue *rxq; + uint16_t queue_count; + uint16_t remainder; + + rxq = (struct avp_queue *)eth_dev->data->rx_queues[rx_queue_id]; + + /* + * Must map all AVP fifos as evenly as possible between the configured + * device queues. Each device queue will service a subset of the AVP + * fifos. If there is an odd number of device queues the first set of + * device queues will get the extra AVP fifos. + */ + queue_count = avp->num_rx_queues / eth_dev->data->nb_rx_queues; + remainder = avp->num_rx_queues % eth_dev->data->nb_rx_queues; + if (rx_queue_id < remainder) { + /* these queues must service one extra FIFO */ + rxq->queue_base = rx_queue_id * (queue_count + 1); + rxq->queue_limit = rxq->queue_base + (queue_count + 1) - 1; + } else { + /* these queues service the regular number of FIFO */ + rxq->queue_base = ((remainder * (queue_count + 1)) + + ((rx_queue_id - remainder) * queue_count)); + rxq->queue_limit = rxq->queue_base + queue_count - 1; + } + + PMD_DRV_LOG(DEBUG, "rxq %u at %p base %u limit %u\n", + rx_queue_id, rxq, rxq->queue_base, rxq->queue_limit); + + rxq->queue_id = rxq->queue_base; +} + +static void +_avp_set_queue_counts(struct rte_eth_dev *eth_dev) +{ + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); + struct avp_dev *avp = AVP_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); + struct rte_avp_device_info *host_info; + void *addr; + + addr = pci_dev->mem_resource[RTE_AVP_PCI_DEVICE_BAR].addr; + host_info = (struct rte_avp_device_info *)addr; + + /* + * the transmit direction is not negotiated beyond respecting the max + * number of queues because the host can handle arbitrary guest tx + * queues (host rx queues). + */ + avp->num_tx_queues = eth_dev->data->nb_tx_queues; + + /* + * the receive direction is more restrictive. The host requires a + * minimum number of guest rx queues (host tx queues) therefore + * negotiate a value that is at least as large as the host minimum + * requirement. If the host and guest values are not identical then a + * mapping will be established in the receive_queue_setup function. + */ + avp->num_rx_queues = RTE_MAX(host_info->min_rx_queues, + eth_dev->data->nb_rx_queues); + + PMD_DRV_LOG(DEBUG, "Requesting %u Tx and %u Rx queues from host\n", + avp->num_tx_queues, avp->num_rx_queues); +} + +static int +avp_dev_attach(struct rte_eth_dev *eth_dev) +{ + struct avp_dev *avp = AVP_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); + struct rte_avp_device_config config; + unsigned int i; + int ret; + + PMD_DRV_LOG(NOTICE, "Attaching port %u to AVP device 0x%" PRIx64 "\n", + eth_dev->data->port_id, avp->device_id); + + rte_spinlock_lock(&avp->lock); + + if (!(avp->flags & AVP_F_DETACHED)) { + PMD_DRV_LOG(NOTICE, "port %u already attached\n", + eth_dev->data->port_id); + ret = 0; + goto unlock; + } + + /* + * make sure that the detached flag is set prior to reconfiguring the + * queues. + */ + avp->flags |= AVP_F_DETACHED; + rte_wmb(); + + /* + * re-run the device create utility which will parse the new host info + * and setup the AVP device queue pointers. + */ + ret = avp_dev_create(RTE_ETH_DEV_TO_PCI(eth_dev), eth_dev); + if (ret < 0) { + PMD_DRV_LOG(ERR, "Failed to re-create AVP device, ret=%d\n", + ret); + goto unlock; + } + + if (avp->flags & AVP_F_CONFIGURED) { + /* + * Update the receive queue mapping to handle cases where the + * source and destination hosts have different queue + * requirements. As long as the DETACHED flag is asserted the + * queue table should not be referenced so it should be safe to + * update it. + */ + _avp_set_queue_counts(eth_dev); + for (i = 0; i < eth_dev->data->nb_rx_queues; i++) + _avp_set_rx_queue_mappings(eth_dev, i); + + /* + * Update the host with our config details so that it knows the + * device is active. + */ + memset(&config, 0, sizeof(config)); + config.device_id = avp->device_id; + config.driver_type = RTE_AVP_DRIVER_TYPE_DPDK; + config.driver_version = AVP_DPDK_DRIVER_VERSION; + config.features = avp->features; + config.num_tx_queues = avp->num_tx_queues; + config.num_rx_queues = avp->num_rx_queues; + config.if_up = !!(avp->flags & AVP_F_LINKUP); + + ret = avp_dev_ctrl_set_config(eth_dev, &config); + if (ret < 0) { + PMD_DRV_LOG(ERR, "Config request failed by host, ret=%d\n", + ret); + goto unlock; + } + } + + rte_wmb(); + avp->flags &= ~AVP_F_DETACHED; + + ret = 0; + +unlock: + rte_spinlock_unlock(&avp->lock); + return ret; +} + +static void +avp_dev_interrupt_handler(void *data) +{ + struct rte_eth_dev *eth_dev = data; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); + void *registers = pci_dev->mem_resource[RTE_AVP_PCI_MMIO_BAR].addr; + uint32_t status, value; + int ret; + + if (registers == NULL) + rte_panic("no mapped MMIO register space\n"); + + /* read the interrupt status register + * note: this register clears on read so all raised interrupts must be + * handled or remembered for later processing + */ + status = AVP_READ32( + RTE_PTR_ADD(registers, + RTE_AVP_INTERRUPT_STATUS_OFFSET)); + + if (status & RTE_AVP_MIGRATION_INTERRUPT_MASK) { + /* handle interrupt based on current status */ + value = AVP_READ32( + RTE_PTR_ADD(registers, + RTE_AVP_MIGRATION_STATUS_OFFSET)); + switch (value) { + case RTE_AVP_MIGRATION_DETACHED: + ret = avp_dev_detach(eth_dev); + break; + case RTE_AVP_MIGRATION_ATTACHED: + ret = avp_dev_attach(eth_dev); + break; + default: + PMD_DRV_LOG(ERR, "unexpected migration status, status=%u\n", + value); + ret = -EINVAL; + } + + /* acknowledge the request by writing out our current status */ + value = (ret == 0 ? value : RTE_AVP_MIGRATION_ERROR); + AVP_WRITE32(value, + RTE_PTR_ADD(registers, + RTE_AVP_MIGRATION_ACK_OFFSET)); + + PMD_DRV_LOG(NOTICE, "AVP migration interrupt handled\n"); + } + + if (status & ~RTE_AVP_MIGRATION_INTERRUPT_MASK) + PMD_DRV_LOG(WARNING, "AVP unexpected interrupt, status=0x%08x\n", + status); + + /* re-enable UIO interrupt handling */ + ret = rte_intr_enable(&pci_dev->intr_handle); + if (ret < 0) { + PMD_DRV_LOG(ERR, "Failed to re-enable UIO interrupts, ret=%d\n", + ret); + /* continue */ + } +} + +static int +avp_dev_enable_interrupts(struct rte_eth_dev *eth_dev) +{ + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); + void *registers = pci_dev->mem_resource[RTE_AVP_PCI_MMIO_BAR].addr; + int ret; + + if (registers == NULL) + return -EINVAL; + + /* enable UIO interrupt handling */ + ret = rte_intr_enable(&pci_dev->intr_handle); + if (ret < 0) { + PMD_DRV_LOG(ERR, "Failed to enable UIO interrupts, ret=%d\n", + ret); + return ret; + } + + /* inform the device that all interrupts are enabled */ + AVP_WRITE32(RTE_AVP_APP_INTERRUPTS_MASK, + RTE_PTR_ADD(registers, RTE_AVP_INTERRUPT_MASK_OFFSET)); + + return 0; +} + +static int +avp_dev_disable_interrupts(struct rte_eth_dev *eth_dev) +{ + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); + void *registers = pci_dev->mem_resource[RTE_AVP_PCI_MMIO_BAR].addr; + int ret; + + if (registers == NULL) + return 0; + + /* inform the device that all interrupts are disabled */ + AVP_WRITE32(RTE_AVP_NO_INTERRUPTS_MASK, + RTE_PTR_ADD(registers, RTE_AVP_INTERRUPT_MASK_OFFSET)); + + /* enable UIO interrupt handling */ + ret = rte_intr_disable(&pci_dev->intr_handle); + if (ret < 0) { + PMD_DRV_LOG(ERR, "Failed to disable UIO interrupts, ret=%d\n", + ret); + return ret; + } + + return 0; +} + +static int +avp_dev_setup_interrupts(struct rte_eth_dev *eth_dev) +{ + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); + int ret; + + /* register a callback handler with UIO for interrupt notifications */ + ret = rte_intr_callback_register(&pci_dev->intr_handle, + avp_dev_interrupt_handler, + (void *)eth_dev); + if (ret < 0) { + PMD_DRV_LOG(ERR, "Failed to register UIO interrupt callback, ret=%d\n", + ret); + return ret; + } + + /* enable interrupt processing */ + return avp_dev_enable_interrupts(eth_dev); +} + +static int +avp_dev_migration_pending(struct rte_eth_dev *eth_dev) +{ + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); + void *registers = pci_dev->mem_resource[RTE_AVP_PCI_MMIO_BAR].addr; + uint32_t value; + + if (registers == NULL) + return 0; + + value = AVP_READ32(RTE_PTR_ADD(registers, + RTE_AVP_MIGRATION_STATUS_OFFSET)); + if (value == RTE_AVP_MIGRATION_DETACHED) { + /* migration is in progress; ack it if we have not already */ + AVP_WRITE32(value, + RTE_PTR_ADD(registers, + RTE_AVP_MIGRATION_ACK_OFFSET)); + return 1; + } + return 0; +} + +/* + * create a AVP device using the supplied device info by first translating it + * to guest address space(s). + */ +static int +avp_dev_create(struct rte_pci_device *pci_dev, + struct rte_eth_dev *eth_dev) +{ + struct avp_dev *avp = AVP_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); + struct rte_avp_device_info *host_info; + struct rte_mem_resource *resource; + unsigned int i; + + resource = &pci_dev->mem_resource[RTE_AVP_PCI_DEVICE_BAR]; + if (resource->addr == NULL) { + PMD_DRV_LOG(ERR, "BAR%u is not mapped\n", + RTE_AVP_PCI_DEVICE_BAR); + return -EFAULT; + } + host_info = (struct rte_avp_device_info *)resource->addr; + + if ((host_info->magic != RTE_AVP_DEVICE_MAGIC) || + avp_dev_version_check(host_info->version)) { + PMD_DRV_LOG(ERR, "Invalid AVP PCI device, magic 0x%08x version 0x%08x > 0x%08x\n", + host_info->magic, host_info->version, + AVP_DPDK_DRIVER_VERSION); + return -EINVAL; + } + + PMD_DRV_LOG(DEBUG, "AVP host device is v%u.%u.%u\n", + RTE_AVP_GET_RELEASE_VERSION(host_info->version), + RTE_AVP_GET_MAJOR_VERSION(host_info->version), + RTE_AVP_GET_MINOR_VERSION(host_info->version)); + + PMD_DRV_LOG(DEBUG, "AVP host supports %u to %u TX queue(s)\n", + host_info->min_tx_queues, host_info->max_tx_queues); + PMD_DRV_LOG(DEBUG, "AVP host supports %u to %u RX queue(s)\n", + host_info->min_rx_queues, host_info->max_rx_queues); + PMD_DRV_LOG(DEBUG, "AVP host supports features 0x%08x\n", + host_info->features); + + if (avp->magic != AVP_ETHDEV_MAGIC) { + /* + * First time initialization (i.e., not during a VM + * migration) + */ + memset(avp, 0, sizeof(*avp)); + avp->magic = AVP_ETHDEV_MAGIC; + avp->dev_data = eth_dev->data; + avp->port_id = eth_dev->data->port_id; + avp->host_mbuf_size = host_info->mbuf_size; + avp->host_features = host_info->features; + rte_spinlock_init(&avp->lock); + memcpy(&avp->ethaddr.addr_bytes[0], + host_info->ethaddr, ETHER_ADDR_LEN); + /* adjust max values to not exceed our max */ + avp->max_tx_queues = + RTE_MIN(host_info->max_tx_queues, RTE_AVP_MAX_QUEUES); + avp->max_rx_queues = + RTE_MIN(host_info->max_rx_queues, RTE_AVP_MAX_QUEUES); + } else { + /* Re-attaching during migration */ + + /* TODO... requires validation of host values */ + if ((host_info->features & avp->features) != avp->features) { + PMD_DRV_LOG(ERR, "AVP host features mismatched; 0x%08x, host=0x%08x\n", + avp->features, host_info->features); + /* this should not be possible; continue for now */ + } + } + + /* the device id is allowed to change over migrations */ + avp->device_id = host_info->device_id; + + /* translate incoming host addresses to guest address space */ + PMD_DRV_LOG(DEBUG, "AVP first host tx queue at 0x%" PRIx64 "\n", + host_info->tx_phys); + PMD_DRV_LOG(DEBUG, "AVP first host alloc queue at 0x%" PRIx64 "\n", + host_info->alloc_phys); + for (i = 0; i < avp->max_tx_queues; i++) { + avp->tx_q[i] = avp_dev_translate_address(eth_dev, + host_info->tx_phys + (i * host_info->tx_size)); + + avp->alloc_q[i] = avp_dev_translate_address(eth_dev, + host_info->alloc_phys + (i * host_info->alloc_size)); + } + + PMD_DRV_LOG(DEBUG, "AVP first host rx queue at 0x%" PRIx64 "\n", + host_info->rx_phys); + PMD_DRV_LOG(DEBUG, "AVP first host free queue at 0x%" PRIx64 "\n", + host_info->free_phys); + for (i = 0; i < avp->max_rx_queues; i++) { + avp->rx_q[i] = avp_dev_translate_address(eth_dev, + host_info->rx_phys + (i * host_info->rx_size)); + avp->free_q[i] = avp_dev_translate_address(eth_dev, + host_info->free_phys + (i * host_info->free_size)); + } + + PMD_DRV_LOG(DEBUG, "AVP host request queue at 0x%" PRIx64 "\n", + host_info->req_phys); + PMD_DRV_LOG(DEBUG, "AVP host response queue at 0x%" PRIx64 "\n", + host_info->resp_phys); + PMD_DRV_LOG(DEBUG, "AVP host sync address at 0x%" PRIx64 "\n", + host_info->sync_phys); + PMD_DRV_LOG(DEBUG, "AVP host mbuf address at 0x%" PRIx64 "\n", + host_info->mbuf_phys); + avp->req_q = avp_dev_translate_address(eth_dev, host_info->req_phys); + avp->resp_q = avp_dev_translate_address(eth_dev, host_info->resp_phys); + avp->sync_addr = + avp_dev_translate_address(eth_dev, host_info->sync_phys); + avp->mbuf_addr = + avp_dev_translate_address(eth_dev, host_info->mbuf_phys); + + /* + * store the host mbuf virtual address so that we can calculate + * relative offsets for each mbuf as they are processed + */ + avp->host_mbuf_addr = host_info->mbuf_va; + avp->host_sync_addr = host_info->sync_va; + + /* + * store the maximum packet length that is supported by the host. + */ + avp->max_rx_pkt_len = host_info->max_rx_pkt_len; + PMD_DRV_LOG(DEBUG, "AVP host max receive packet length is %u\n", + host_info->max_rx_pkt_len); + + return 0; +} + +/* + * This function is based on probe() function in avp_pci.c + * It returns 0 on success. + */ +static int +eth_avp_dev_init(struct rte_eth_dev *eth_dev) +{ + struct avp_dev *avp = + AVP_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); + struct rte_pci_device *pci_dev; + int ret; + + pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); + eth_dev->dev_ops = &avp_eth_dev_ops; + eth_dev->rx_pkt_burst = &avp_recv_pkts; + eth_dev->tx_pkt_burst = &avp_xmit_pkts; + + if (rte_eal_process_type() != RTE_PROC_PRIMARY) { + /* + * no setup required on secondary processes. All data is saved + * in dev_private by the primary process. All resource should + * be mapped to the same virtual address so all pointers should + * be valid. + */ + if (eth_dev->data->scattered_rx) { + PMD_DRV_LOG(NOTICE, "AVP device configured for chained mbufs\n"); + eth_dev->rx_pkt_burst = avp_recv_scattered_pkts; + eth_dev->tx_pkt_burst = avp_xmit_scattered_pkts; + } + return 0; + } + + rte_eth_copy_pci_info(eth_dev, pci_dev); + + /* Check current migration status */ + if (avp_dev_migration_pending(eth_dev)) { + PMD_DRV_LOG(ERR, "VM live migration operation in progress\n"); + return -EBUSY; + } + + /* Check BAR resources */ + ret = avp_dev_check_regions(eth_dev); + if (ret < 0) { + PMD_DRV_LOG(ERR, "Failed to validate BAR resources, ret=%d\n", + ret); + return ret; + } + + /* Enable interrupts */ + ret = avp_dev_setup_interrupts(eth_dev); + if (ret < 0) { + PMD_DRV_LOG(ERR, "Failed to enable interrupts, ret=%d\n", ret); + return ret; + } + + /* Handle each subtype */ + ret = avp_dev_create(pci_dev, eth_dev); + if (ret < 0) { + PMD_DRV_LOG(ERR, "Failed to create device, ret=%d\n", ret); + return ret; + } + + /* Allocate memory for storing MAC addresses */ + eth_dev->data->mac_addrs = rte_zmalloc("avp_ethdev", ETHER_ADDR_LEN, 0); + if (eth_dev->data->mac_addrs == NULL) { + PMD_DRV_LOG(ERR, "Failed to allocate %d bytes needed to store MAC addresses\n", + ETHER_ADDR_LEN); + return -ENOMEM; + } + + /* Get a mac from device config */ + ether_addr_copy(&avp->ethaddr, ð_dev->data->mac_addrs[0]); + + return 0; +} + +static int +eth_avp_dev_uninit(struct rte_eth_dev *eth_dev) +{ + int ret; + + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return -EPERM; + + if (eth_dev->data == NULL) + return 0; + + ret = avp_dev_disable_interrupts(eth_dev); + if (ret != 0) { + PMD_DRV_LOG(ERR, "Failed to disable interrupts, ret=%d\n", ret); + return ret; + } + + if (eth_dev->data->mac_addrs != NULL) { + rte_free(eth_dev->data->mac_addrs); + eth_dev->data->mac_addrs = NULL; + } + + return 0; +} + +static int +eth_avp_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, + struct rte_pci_device *pci_dev) +{ + struct rte_eth_dev *eth_dev; + int ret; + + eth_dev = rte_eth_dev_pci_allocate(pci_dev, + sizeof(struct avp_adapter)); + if (eth_dev == NULL) + return -ENOMEM; + + ret = eth_avp_dev_init(eth_dev); + if (ret) + rte_eth_dev_pci_release(eth_dev); + + return ret; +} + +static int +eth_avp_pci_remove(struct rte_pci_device *pci_dev) +{ + return rte_eth_dev_pci_generic_remove(pci_dev, + eth_avp_dev_uninit); +} + +static struct rte_pci_driver rte_avp_pmd = { + .id_table = pci_id_avp_map, + .drv_flags = RTE_PCI_DRV_NEED_MAPPING, + .probe = eth_avp_pci_probe, + .remove = eth_avp_pci_remove, +}; + +static int +avp_dev_enable_scattered(struct rte_eth_dev *eth_dev, + struct avp_dev *avp) +{ + unsigned int max_rx_pkt_len; + + max_rx_pkt_len = eth_dev->data->dev_conf.rxmode.max_rx_pkt_len; + + if ((max_rx_pkt_len > avp->guest_mbuf_size) || + (max_rx_pkt_len > avp->host_mbuf_size)) { + /* + * If the guest MTU is greater than either the host or guest + * buffers then chained mbufs have to be enabled in the TX + * direction. It is assumed that the application will not need + * to send packets larger than their max_rx_pkt_len (MRU). + */ + return 1; + } + + if ((avp->max_rx_pkt_len > avp->guest_mbuf_size) || + (avp->max_rx_pkt_len > avp->host_mbuf_size)) { + /* + * If the host MRU is greater than its own mbuf size or the + * guest mbuf size then chained mbufs have to be enabled in the + * RX direction. + */ + return 1; + } + + return 0; +} + +static int +avp_dev_rx_queue_setup(struct rte_eth_dev *eth_dev, + uint16_t rx_queue_id, + uint16_t nb_rx_desc, + unsigned int socket_id, + const struct rte_eth_rxconf *rx_conf, + struct rte_mempool *pool) +{ + struct avp_dev *avp = AVP_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); + struct rte_pktmbuf_pool_private *mbp_priv; + struct avp_queue *rxq; + + if (rx_queue_id >= eth_dev->data->nb_rx_queues) { + PMD_DRV_LOG(ERR, "RX queue id is out of range: rx_queue_id=%u, nb_rx_queues=%u\n", + rx_queue_id, eth_dev->data->nb_rx_queues); + return -EINVAL; + } + + /* Save mbuf pool pointer */ + avp->pool = pool; + + /* Save the local mbuf size */ + mbp_priv = rte_mempool_get_priv(pool); + avp->guest_mbuf_size = (uint16_t)(mbp_priv->mbuf_data_room_size); + avp->guest_mbuf_size -= RTE_PKTMBUF_HEADROOM; + + if (avp_dev_enable_scattered(eth_dev, avp)) { + if (!eth_dev->data->scattered_rx) { + PMD_DRV_LOG(NOTICE, "AVP device configured for chained mbufs\n"); + eth_dev->data->scattered_rx = 1; + eth_dev->rx_pkt_burst = avp_recv_scattered_pkts; + eth_dev->tx_pkt_burst = avp_xmit_scattered_pkts; + } + } + + PMD_DRV_LOG(DEBUG, "AVP max_rx_pkt_len=(%u,%u) mbuf_size=(%u,%u)\n", + avp->max_rx_pkt_len, + eth_dev->data->dev_conf.rxmode.max_rx_pkt_len, + avp->host_mbuf_size, + avp->guest_mbuf_size); + + /* allocate a queue object */ + rxq = rte_zmalloc_socket("ethdev RX queue", sizeof(struct avp_queue), + RTE_CACHE_LINE_SIZE, socket_id); + if (rxq == NULL) { + PMD_DRV_LOG(ERR, "Failed to allocate new Rx queue object\n"); + return -ENOMEM; + } + + /* save back pointers to AVP and Ethernet devices */ + rxq->avp = avp; + rxq->dev_data = eth_dev->data; + eth_dev->data->rx_queues[rx_queue_id] = (void *)rxq; + + /* setup the queue receive mapping for the current queue. */ + _avp_set_rx_queue_mappings(eth_dev, rx_queue_id); + + PMD_DRV_LOG(DEBUG, "Rx queue %u setup at %p\n", rx_queue_id, rxq); + + (void)nb_rx_desc; + (void)rx_conf; + return 0; +} + +static int +avp_dev_tx_queue_setup(struct rte_eth_dev *eth_dev, + uint16_t tx_queue_id, + uint16_t nb_tx_desc, + unsigned int socket_id, + const struct rte_eth_txconf *tx_conf) +{ + struct avp_dev *avp = AVP_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); + struct avp_queue *txq; + + if (tx_queue_id >= eth_dev->data->nb_tx_queues) { + PMD_DRV_LOG(ERR, "TX queue id is out of range: tx_queue_id=%u, nb_tx_queues=%u\n", + tx_queue_id, eth_dev->data->nb_tx_queues); + return -EINVAL; + } + + /* allocate a queue object */ + txq = rte_zmalloc_socket("ethdev TX queue", sizeof(struct avp_queue), + RTE_CACHE_LINE_SIZE, socket_id); + if (txq == NULL) { + PMD_DRV_LOG(ERR, "Failed to allocate new Tx queue object\n"); + return -ENOMEM; + } + + /* only the configured set of transmit queues are used */ + txq->queue_id = tx_queue_id; + txq->queue_base = tx_queue_id; + txq->queue_limit = tx_queue_id; + + /* save back pointers to AVP and Ethernet devices */ + txq->avp = avp; + txq->dev_data = eth_dev->data; + eth_dev->data->tx_queues[tx_queue_id] = (void *)txq; + + PMD_DRV_LOG(DEBUG, "Tx queue %u setup at %p\n", tx_queue_id, txq); + + (void)nb_tx_desc; + (void)tx_conf; + return 0; +} + +static inline int +_avp_cmp_ether_addr(struct ether_addr *a, struct ether_addr *b) +{ + uint16_t *_a = (uint16_t *)&a->addr_bytes[0]; + uint16_t *_b = (uint16_t *)&b->addr_bytes[0]; + return (_a[0] ^ _b[0]) | (_a[1] ^ _b[1]) | (_a[2] ^ _b[2]); +} + +static inline int +_avp_mac_filter(struct avp_dev *avp, struct rte_mbuf *m) +{ + struct ether_hdr *eth = rte_pktmbuf_mtod(m, struct ether_hdr *); + + if (likely(_avp_cmp_ether_addr(&avp->ethaddr, ð->d_addr) == 0)) { + /* allow all packets destined to our address */ + return 0; + } + + if (likely(is_broadcast_ether_addr(ð->d_addr))) { + /* allow all broadcast packets */ + return 0; + } + + if (likely(is_multicast_ether_addr(ð->d_addr))) { + /* allow all multicast packets */ + return 0; + } + + if (avp->flags & AVP_F_PROMISC) { + /* allow all packets when in promiscuous mode */ + return 0; + } + + return -1; +} + +#ifdef RTE_LIBRTE_AVP_DEBUG_BUFFERS +static inline void +__avp_dev_buffer_sanity_check(struct avp_dev *avp, struct rte_avp_desc *buf) +{ + struct rte_avp_desc *first_buf; + struct rte_avp_desc *pkt_buf; + unsigned int pkt_len; + unsigned int nb_segs; + void *pkt_data; + unsigned int i; + + first_buf = avp_dev_translate_buffer(avp, buf); + + i = 0; + pkt_len = 0; + nb_segs = first_buf->nb_segs; + do { + /* Adjust pointers for guest addressing */ + pkt_buf = avp_dev_translate_buffer(avp, buf); + if (pkt_buf == NULL) + rte_panic("bad buffer: segment %u has an invalid address %p\n", + i, buf); + pkt_data = avp_dev_translate_buffer(avp, pkt_buf->data); + if (pkt_data == NULL) + rte_panic("bad buffer: segment %u has a NULL data pointer\n", + i); + if (pkt_buf->data_len == 0) + rte_panic("bad buffer: segment %u has 0 data length\n", + i); + pkt_len += pkt_buf->data_len; + nb_segs--; + i++; + + } while (nb_segs && (buf = pkt_buf->next) != NULL); + + if (nb_segs != 0) + rte_panic("bad buffer: expected %u segments found %u\n", + first_buf->nb_segs, (first_buf->nb_segs - nb_segs)); + if (pkt_len != first_buf->pkt_len) + rte_panic("bad buffer: expected length %u found %u\n", + first_buf->pkt_len, pkt_len); +} + +#define avp_dev_buffer_sanity_check(a, b) \ + __avp_dev_buffer_sanity_check((a), (b)) + +#else /* RTE_LIBRTE_AVP_DEBUG_BUFFERS */ + +#define avp_dev_buffer_sanity_check(a, b) do {} while (0) + +#endif + +/* + * Copy a host buffer chain to a set of mbufs. This function assumes that + * there exactly the required number of mbufs to copy all source bytes. + */ +static inline struct rte_mbuf * +avp_dev_copy_from_buffers(struct avp_dev *avp, + struct rte_avp_desc *buf, + struct rte_mbuf **mbufs, + unsigned int count) +{ + struct rte_mbuf *m_previous = NULL; + struct rte_avp_desc *pkt_buf; + unsigned int total_length = 0; + unsigned int copy_length; + unsigned int src_offset; + struct rte_mbuf *m; + uint16_t ol_flags; + uint16_t vlan_tci; + void *pkt_data; + unsigned int i; + + avp_dev_buffer_sanity_check(avp, buf); + + /* setup the first source buffer */ + pkt_buf = avp_dev_translate_buffer(avp, buf); + pkt_data = avp_dev_translate_buffer(avp, pkt_buf->data); + total_length = pkt_buf->pkt_len; + src_offset = 0; + + if (pkt_buf->ol_flags & RTE_AVP_RX_VLAN_PKT) { + ol_flags = PKT_RX_VLAN; + vlan_tci = pkt_buf->vlan_tci; + } else { + ol_flags = 0; + vlan_tci = 0; + } + + for (i = 0; (i < count) && (buf != NULL); i++) { + /* fill each destination buffer */ + m = mbufs[i]; + + if (m_previous != NULL) + m_previous->next = m; + + m_previous = m; + + do { + /* + * Copy as many source buffers as will fit in the + * destination buffer. + */ + copy_length = RTE_MIN((avp->guest_mbuf_size - + rte_pktmbuf_data_len(m)), + (pkt_buf->data_len - + src_offset)); + rte_memcpy(RTE_PTR_ADD(rte_pktmbuf_mtod(m, void *), + rte_pktmbuf_data_len(m)), + RTE_PTR_ADD(pkt_data, src_offset), + copy_length); + rte_pktmbuf_data_len(m) += copy_length; + src_offset += copy_length; + + if (likely(src_offset == pkt_buf->data_len)) { + /* need a new source buffer */ + buf = pkt_buf->next; + if (buf != NULL) { + pkt_buf = avp_dev_translate_buffer( + avp, buf); + pkt_data = avp_dev_translate_buffer( + avp, pkt_buf->data); + src_offset = 0; + } + } + + if (unlikely(rte_pktmbuf_data_len(m) == + avp->guest_mbuf_size)) { + /* need a new destination mbuf */ + break; + } + + } while (buf != NULL); + } + + m = mbufs[0]; + m->ol_flags = ol_flags; + m->nb_segs = count; + rte_pktmbuf_pkt_len(m) = total_length; + m->vlan_tci = vlan_tci; + + __rte_mbuf_sanity_check(m, 1); + + return m; +} + +static uint16_t +avp_recv_scattered_pkts(void *rx_queue, + struct rte_mbuf **rx_pkts, + uint16_t nb_pkts) +{ + struct avp_queue *rxq = (struct avp_queue *)rx_queue; + struct rte_avp_desc *avp_bufs[AVP_MAX_RX_BURST]; + struct rte_mbuf *mbufs[RTE_AVP_MAX_MBUF_SEGMENTS]; + struct avp_dev *avp = rxq->avp; + struct rte_avp_desc *pkt_buf; + struct rte_avp_fifo *free_q; + struct rte_avp_fifo *rx_q; + struct rte_avp_desc *buf; + unsigned int count, avail, n; + unsigned int guest_mbuf_size; + struct rte_mbuf *m; + unsigned int required; + unsigned int buf_len; + unsigned int port_id; + unsigned int i; + + if (unlikely(avp->flags & AVP_F_DETACHED)) { + /* VM live migration in progress */ + return 0; + } + + guest_mbuf_size = avp->guest_mbuf_size; + port_id = avp->port_id; + rx_q = avp->rx_q[rxq->queue_id]; + free_q = avp->free_q[rxq->queue_id]; + + /* setup next queue to service */ + rxq->queue_id = (rxq->queue_id < rxq->queue_limit) ? + (rxq->queue_id + 1) : rxq->queue_base; + + /* determine how many slots are available in the free queue */ + count = avp_fifo_free_count(free_q); + + /* determine how many packets are available in the rx queue */ + avail = avp_fifo_count(rx_q); + + /* determine how many packets can be received */ + count = RTE_MIN(count, avail); + count = RTE_MIN(count, nb_pkts); + count = RTE_MIN(count, (unsigned int)AVP_MAX_RX_BURST); + + if (unlikely(count == 0)) { + /* no free buffers, or no buffers on the rx queue */ + return 0; + } + + /* retrieve pending packets */ + n = avp_fifo_get(rx_q, (void **)&avp_bufs, count); + PMD_RX_LOG(DEBUG, "Receiving %u packets from Rx queue at %p\n", + count, rx_q); + + count = 0; + for (i = 0; i < n; i++) { + /* prefetch next entry while processing current one */ + if (i + 1 < n) { + pkt_buf = avp_dev_translate_buffer(avp, + avp_bufs[i + 1]); + rte_prefetch0(pkt_buf); + } + buf = avp_bufs[i]; + + /* Peek into the first buffer to determine the total length */ + pkt_buf = avp_dev_translate_buffer(avp, buf); + buf_len = pkt_buf->pkt_len; + + /* Allocate enough mbufs to receive the entire packet */ + required = (buf_len + guest_mbuf_size - 1) / guest_mbuf_size; + if (rte_pktmbuf_alloc_bulk(avp->pool, mbufs, required)) { + rxq->dev_data->rx_mbuf_alloc_failed++; + continue; + } + + /* Copy the data from the buffers to our mbufs */ + m = avp_dev_copy_from_buffers(avp, buf, mbufs, required); + + /* finalize mbuf */ + m->port = port_id; + + if (_avp_mac_filter(avp, m) != 0) { + /* silently discard packets not destined to our MAC */ + rte_pktmbuf_free(m); + continue; + } + + /* return new mbuf to caller */ + rx_pkts[count++] = m; + rxq->bytes += buf_len; + } + + rxq->packets += count; + + /* return the buffers to the free queue */ + avp_fifo_put(free_q, (void **)&avp_bufs[0], n); + + return count; +} + + +static uint16_t +avp_recv_pkts(void *rx_queue, + struct rte_mbuf **rx_pkts, + uint16_t nb_pkts) +{ + struct avp_queue *rxq = (struct avp_queue *)rx_queue; + struct rte_avp_desc *avp_bufs[AVP_MAX_RX_BURST]; + struct avp_dev *avp = rxq->avp; + struct rte_avp_desc *pkt_buf; + struct rte_avp_fifo *free_q; + struct rte_avp_fifo *rx_q; + unsigned int count, avail, n; + unsigned int pkt_len; + struct rte_mbuf *m; + char *pkt_data; + unsigned int i; + + if (unlikely(avp->flags & AVP_F_DETACHED)) { + /* VM live migration in progress */ + return 0; + } + + rx_q = avp->rx_q[rxq->queue_id]; + free_q = avp->free_q[rxq->queue_id]; + + /* setup next queue to service */ + rxq->queue_id = (rxq->queue_id < rxq->queue_limit) ? + (rxq->queue_id + 1) : rxq->queue_base; + + /* determine how many slots are available in the free queue */ + count = avp_fifo_free_count(free_q); + + /* determine how many packets are available in the rx queue */ + avail = avp_fifo_count(rx_q); + + /* determine how many packets can be received */ + count = RTE_MIN(count, avail); + count = RTE_MIN(count, nb_pkts); + count = RTE_MIN(count, (unsigned int)AVP_MAX_RX_BURST); + + if (unlikely(count == 0)) { + /* no free buffers, or no buffers on the rx queue */ + return 0; + } + + /* retrieve pending packets */ + n = avp_fifo_get(rx_q, (void **)&avp_bufs, count); + PMD_RX_LOG(DEBUG, "Receiving %u packets from Rx queue at %p\n", + count, rx_q); + + count = 0; + for (i = 0; i < n; i++) { + /* prefetch next entry while processing current one */ + if (i < n - 1) { + pkt_buf = avp_dev_translate_buffer(avp, + avp_bufs[i + 1]); + rte_prefetch0(pkt_buf); + } + + /* Adjust host pointers for guest addressing */ + pkt_buf = avp_dev_translate_buffer(avp, avp_bufs[i]); + pkt_data = avp_dev_translate_buffer(avp, pkt_buf->data); + pkt_len = pkt_buf->pkt_len; + + if (unlikely((pkt_len > avp->guest_mbuf_size) || + (pkt_buf->nb_segs > 1))) { + /* + * application should be using the scattered receive + * function + */ + rxq->errors++; + continue; + } + + /* process each packet to be transmitted */ + m = rte_pktmbuf_alloc(avp->pool); + if (unlikely(m == NULL)) { + rxq->dev_data->rx_mbuf_alloc_failed++; + continue; + } + + /* copy data out of the host buffer to our buffer */ + m->data_off = RTE_PKTMBUF_HEADROOM; + rte_memcpy(rte_pktmbuf_mtod(m, void *), pkt_data, pkt_len); + + /* initialize the local mbuf */ + rte_pktmbuf_data_len(m) = pkt_len; + rte_pktmbuf_pkt_len(m) = pkt_len; + m->port = avp->port_id; + + if (pkt_buf->ol_flags & RTE_AVP_RX_VLAN_PKT) { + m->ol_flags = PKT_RX_VLAN; + m->vlan_tci = pkt_buf->vlan_tci; + } + + if (_avp_mac_filter(avp, m) != 0) { + /* silently discard packets not destined to our MAC */ + rte_pktmbuf_free(m); + continue; + } + + /* return new mbuf to caller */ + rx_pkts[count++] = m; + rxq->bytes += pkt_len; + } + + rxq->packets += count; + + /* return the buffers to the free queue */ + avp_fifo_put(free_q, (void **)&avp_bufs[0], n); + + return count; +} + +/* + * Copy a chained mbuf to a set of host buffers. This function assumes that + * there are sufficient destination buffers to contain the entire source + * packet. + */ +static inline uint16_t +avp_dev_copy_to_buffers(struct avp_dev *avp, + struct rte_mbuf *mbuf, + struct rte_avp_desc **buffers, + unsigned int count) +{ + struct rte_avp_desc *previous_buf = NULL; + struct rte_avp_desc *first_buf = NULL; + struct rte_avp_desc *pkt_buf; + struct rte_avp_desc *buf; + size_t total_length; + struct rte_mbuf *m; + size_t copy_length; + size_t src_offset; + char *pkt_data; + unsigned int i; + + __rte_mbuf_sanity_check(mbuf, 1); + + m = mbuf; + src_offset = 0; + total_length = rte_pktmbuf_pkt_len(m); + for (i = 0; (i < count) && (m != NULL); i++) { + /* fill each destination buffer */ + buf = buffers[i]; + + if (i < count - 1) { + /* prefetch next entry while processing this one */ + pkt_buf = avp_dev_translate_buffer(avp, buffers[i + 1]); + rte_prefetch0(pkt_buf); + } + + /* Adjust pointers for guest addressing */ + pkt_buf = avp_dev_translate_buffer(avp, buf); + pkt_data = avp_dev_translate_buffer(avp, pkt_buf->data); + + /* setup the buffer chain */ + if (previous_buf != NULL) + previous_buf->next = buf; + else + first_buf = pkt_buf; + + previous_buf = pkt_buf; + + do { + /* + * copy as many source mbuf segments as will fit in the + * destination buffer. + */ + copy_length = RTE_MIN((avp->host_mbuf_size - + pkt_buf->data_len), + (rte_pktmbuf_data_len(m) - + src_offset)); + rte_memcpy(RTE_PTR_ADD(pkt_data, pkt_buf->data_len), + RTE_PTR_ADD(rte_pktmbuf_mtod(m, void *), + src_offset), + copy_length); + pkt_buf->data_len += copy_length; + src_offset += copy_length; + + if (likely(src_offset == rte_pktmbuf_data_len(m))) { + /* need a new source buffer */ + m = m->next; + src_offset = 0; + } + + if (unlikely(pkt_buf->data_len == + avp->host_mbuf_size)) { + /* need a new destination buffer */ + break; + } + + } while (m != NULL); + } + + first_buf->nb_segs = count; + first_buf->pkt_len = total_length; + + if (mbuf->ol_flags & PKT_TX_VLAN_PKT) { + first_buf->ol_flags |= RTE_AVP_TX_VLAN_PKT; + first_buf->vlan_tci = mbuf->vlan_tci; + } + + avp_dev_buffer_sanity_check(avp, buffers[0]); + + return total_length; +} + + +static uint16_t +avp_xmit_scattered_pkts(void *tx_queue, + struct rte_mbuf **tx_pkts, + uint16_t nb_pkts) +{ + struct rte_avp_desc *avp_bufs[(AVP_MAX_TX_BURST * + RTE_AVP_MAX_MBUF_SEGMENTS)]; + struct avp_queue *txq = (struct avp_queue *)tx_queue; + struct rte_avp_desc *tx_bufs[AVP_MAX_TX_BURST]; + struct avp_dev *avp = txq->avp; + struct rte_avp_fifo *alloc_q; + struct rte_avp_fifo *tx_q; + unsigned int count, avail, n; + unsigned int orig_nb_pkts; + struct rte_mbuf *m; + unsigned int required; + unsigned int segments; + unsigned int tx_bytes; + unsigned int i; + + orig_nb_pkts = nb_pkts; + if (unlikely(avp->flags & AVP_F_DETACHED)) { + /* VM live migration in progress */ + /* TODO ... buffer for X packets then drop? */ + txq->errors += nb_pkts; + return 0; + } + + tx_q = avp->tx_q[txq->queue_id]; + alloc_q = avp->alloc_q[txq->queue_id]; + + /* limit the number of transmitted packets to the max burst size */ + if (unlikely(nb_pkts > AVP_MAX_TX_BURST)) + nb_pkts = AVP_MAX_TX_BURST; + + /* determine how many buffers are available to copy into */ + avail = avp_fifo_count(alloc_q); + if (unlikely(avail > (AVP_MAX_TX_BURST * + RTE_AVP_MAX_MBUF_SEGMENTS))) + avail = AVP_MAX_TX_BURST * RTE_AVP_MAX_MBUF_SEGMENTS; + + /* determine how many slots are available in the transmit queue */ + count = avp_fifo_free_count(tx_q); + + /* determine how many packets can be sent */ + nb_pkts = RTE_MIN(count, nb_pkts); + + /* determine how many packets will fit in the available buffers */ + count = 0; + segments = 0; + for (i = 0; i < nb_pkts; i++) { + m = tx_pkts[i]; + if (likely(i < (unsigned int)nb_pkts - 1)) { + /* prefetch next entry while processing this one */ + rte_prefetch0(tx_pkts[i + 1]); + } + required = (rte_pktmbuf_pkt_len(m) + avp->host_mbuf_size - 1) / + avp->host_mbuf_size; + + if (unlikely((required == 0) || + (required > RTE_AVP_MAX_MBUF_SEGMENTS))) + break; + else if (unlikely(required + segments > avail)) + break; + segments += required; + count++; + } + nb_pkts = count; + + if (unlikely(nb_pkts == 0)) { + /* no available buffers, or no space on the tx queue */ + txq->errors += orig_nb_pkts; + return 0; + } + + PMD_TX_LOG(DEBUG, "Sending %u packets on Tx queue at %p\n", + nb_pkts, tx_q); + + /* retrieve sufficient send buffers */ + n = avp_fifo_get(alloc_q, (void **)&avp_bufs, segments); + if (unlikely(n != segments)) { + PMD_TX_LOG(DEBUG, "Failed to allocate buffers " + "n=%u, segments=%u, orig=%u\n", + n, segments, orig_nb_pkts); + txq->errors += orig_nb_pkts; + return 0; + } + + tx_bytes = 0; + count = 0; + for (i = 0; i < nb_pkts; i++) { + /* process each packet to be transmitted */ + m = tx_pkts[i]; + + /* determine how many buffers are required for this packet */ + required = (rte_pktmbuf_pkt_len(m) + avp->host_mbuf_size - 1) / + avp->host_mbuf_size; + + tx_bytes += avp_dev_copy_to_buffers(avp, m, + &avp_bufs[count], required); + tx_bufs[i] = avp_bufs[count]; + count += required; + + /* free the original mbuf */ + rte_pktmbuf_free(m); + } + + txq->packets += nb_pkts; + txq->bytes += tx_bytes; + +#ifdef RTE_LIBRTE_AVP_DEBUG_BUFFERS + for (i = 0; i < nb_pkts; i++) + avp_dev_buffer_sanity_check(avp, tx_bufs[i]); +#endif + + /* send the packets */ + n = avp_fifo_put(tx_q, (void **)&tx_bufs[0], nb_pkts); + if (unlikely(n != orig_nb_pkts)) + txq->errors += (orig_nb_pkts - n); + + return n; +} + + +static uint16_t +avp_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) +{ + struct avp_queue *txq = (struct avp_queue *)tx_queue; + struct rte_avp_desc *avp_bufs[AVP_MAX_TX_BURST]; + struct avp_dev *avp = txq->avp; + struct rte_avp_desc *pkt_buf; + struct rte_avp_fifo *alloc_q; + struct rte_avp_fifo *tx_q; + unsigned int count, avail, n; + struct rte_mbuf *m; + unsigned int pkt_len; + unsigned int tx_bytes; + char *pkt_data; + unsigned int i; + + if (unlikely(avp->flags & AVP_F_DETACHED)) { + /* VM live migration in progress */ + /* TODO ... buffer for X packets then drop?! */ + txq->errors++; + return 0; + } + + tx_q = avp->tx_q[txq->queue_id]; + alloc_q = avp->alloc_q[txq->queue_id]; + + /* limit the number of transmitted packets to the max burst size */ + if (unlikely(nb_pkts > AVP_MAX_TX_BURST)) + nb_pkts = AVP_MAX_TX_BURST; + + /* determine how many buffers are available to copy into */ + avail = avp_fifo_count(alloc_q); + + /* determine how many slots are available in the transmit queue */ + count = avp_fifo_free_count(tx_q); + + /* determine how many packets can be sent */ + count = RTE_MIN(count, avail); + count = RTE_MIN(count, nb_pkts); + + if (unlikely(count == 0)) { + /* no available buffers, or no space on the tx queue */ + txq->errors += nb_pkts; + return 0; + } + + PMD_TX_LOG(DEBUG, "Sending %u packets on Tx queue at %p\n", + count, tx_q); + + /* retrieve sufficient send buffers */ + n = avp_fifo_get(alloc_q, (void **)&avp_bufs, count); + if (unlikely(n != count)) { + txq->errors++; + return 0; + } + + tx_bytes = 0; + for (i = 0; i < count; i++) { + /* prefetch next entry while processing the current one */ + if (i < count - 1) { + pkt_buf = avp_dev_translate_buffer(avp, + avp_bufs[i + 1]); + rte_prefetch0(pkt_buf); + } + + /* process each packet to be transmitted */ + m = tx_pkts[i]; + + /* Adjust pointers for guest addressing */ + pkt_buf = avp_dev_translate_buffer(avp, avp_bufs[i]); + pkt_data = avp_dev_translate_buffer(avp, pkt_buf->data); + pkt_len = rte_pktmbuf_pkt_len(m); + + if (unlikely((pkt_len > avp->guest_mbuf_size) || + (pkt_len > avp->host_mbuf_size))) { + /* + * application should be using the scattered transmit + * function; send it truncated to avoid the performance + * hit of having to manage returning the already + * allocated buffer to the free list. This should not + * happen since the application should have set the + * max_rx_pkt_len based on its MTU and it should be + * policing its own packet sizes. + */ + txq->errors++; + pkt_len = RTE_MIN(avp->guest_mbuf_size, + avp->host_mbuf_size); + } + + /* copy data out of our mbuf and into the AVP buffer */ + rte_memcpy(pkt_data, rte_pktmbuf_mtod(m, void *), pkt_len); + pkt_buf->pkt_len = pkt_len; + pkt_buf->data_len = pkt_len; + pkt_buf->nb_segs = 1; + pkt_buf->next = NULL; + + if (m->ol_flags & PKT_TX_VLAN_PKT) { + pkt_buf->ol_flags |= RTE_AVP_TX_VLAN_PKT; + pkt_buf->vlan_tci = m->vlan_tci; + } + + tx_bytes += pkt_len; + + /* free the original mbuf */ + rte_pktmbuf_free(m); + } + + txq->packets += count; + txq->bytes += tx_bytes; + + /* send the packets */ + n = avp_fifo_put(tx_q, (void **)&avp_bufs[0], count); + + return n; +} + +static void +avp_dev_rx_queue_release(void *rx_queue) +{ + struct avp_queue *rxq = (struct avp_queue *)rx_queue; + struct avp_dev *avp = rxq->avp; + struct rte_eth_dev_data *data = avp->dev_data; + unsigned int i; + + for (i = 0; i < avp->num_rx_queues; i++) { + if (data->rx_queues[i] == rxq) + data->rx_queues[i] = NULL; + } +} + +static void +avp_dev_tx_queue_release(void *tx_queue) +{ + struct avp_queue *txq = (struct avp_queue *)tx_queue; + struct avp_dev *avp = txq->avp; + struct rte_eth_dev_data *data = avp->dev_data; + unsigned int i; + + for (i = 0; i < avp->num_tx_queues; i++) { + if (data->tx_queues[i] == txq) + data->tx_queues[i] = NULL; + } +} + +static int +avp_dev_configure(struct rte_eth_dev *eth_dev) +{ + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); + struct avp_dev *avp = AVP_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); + struct rte_avp_device_info *host_info; + struct rte_avp_device_config config; + int mask = 0; + void *addr; + int ret; + + rte_spinlock_lock(&avp->lock); + if (avp->flags & AVP_F_DETACHED) { + PMD_DRV_LOG(ERR, "Operation not supported during VM live migration\n"); + ret = -ENOTSUP; + goto unlock; + } + + addr = pci_dev->mem_resource[RTE_AVP_PCI_DEVICE_BAR].addr; + host_info = (struct rte_avp_device_info *)addr; + + /* Setup required number of queues */ + _avp_set_queue_counts(eth_dev); + + mask = (ETH_VLAN_STRIP_MASK | + ETH_VLAN_FILTER_MASK | + ETH_VLAN_EXTEND_MASK); + ret = avp_vlan_offload_set(eth_dev, mask); + if (ret < 0) { + PMD_DRV_LOG(ERR, "VLAN offload set failed by host, ret=%d\n", + ret); + goto unlock; + } + + /* update device config */ + memset(&config, 0, sizeof(config)); + config.device_id = host_info->device_id; + config.driver_type = RTE_AVP_DRIVER_TYPE_DPDK; + config.driver_version = AVP_DPDK_DRIVER_VERSION; + config.features = avp->features; + config.num_tx_queues = avp->num_tx_queues; + config.num_rx_queues = avp->num_rx_queues; + + ret = avp_dev_ctrl_set_config(eth_dev, &config); + if (ret < 0) { + PMD_DRV_LOG(ERR, "Config request failed by host, ret=%d\n", + ret); + goto unlock; + } + + avp->flags |= AVP_F_CONFIGURED; + ret = 0; + +unlock: + rte_spinlock_unlock(&avp->lock); + return ret; +} + +static int +avp_dev_start(struct rte_eth_dev *eth_dev) +{ + struct avp_dev *avp = AVP_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); + int ret; + + rte_spinlock_lock(&avp->lock); + if (avp->flags & AVP_F_DETACHED) { + PMD_DRV_LOG(ERR, "Operation not supported during VM live migration\n"); + ret = -ENOTSUP; + goto unlock; + } + + /* disable features that we do not support */ + eth_dev->data->dev_conf.rxmode.hw_ip_checksum = 0; + eth_dev->data->dev_conf.rxmode.hw_vlan_filter = 0; + eth_dev->data->dev_conf.rxmode.hw_vlan_extend = 0; + eth_dev->data->dev_conf.rxmode.hw_strip_crc = 0; + + /* update link state */ + ret = avp_dev_ctrl_set_link_state(eth_dev, 1); + if (ret < 0) { + PMD_DRV_LOG(ERR, "Link state change failed by host, ret=%d\n", + ret); + goto unlock; + } + + /* remember current link state */ + avp->flags |= AVP_F_LINKUP; + + ret = 0; + +unlock: + rte_spinlock_unlock(&avp->lock); + return ret; +} + +static void +avp_dev_stop(struct rte_eth_dev *eth_dev) +{ + struct avp_dev *avp = AVP_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); + int ret; + + rte_spinlock_lock(&avp->lock); + if (avp->flags & AVP_F_DETACHED) { + PMD_DRV_LOG(ERR, "Operation not supported during VM live migration\n"); + goto unlock; + } + + /* remember current link state */ + avp->flags &= ~AVP_F_LINKUP; + + /* update link state */ + ret = avp_dev_ctrl_set_link_state(eth_dev, 0); + if (ret < 0) { + PMD_DRV_LOG(ERR, "Link state change failed by host, ret=%d\n", + ret); + } + +unlock: + rte_spinlock_unlock(&avp->lock); +} + +static void +avp_dev_close(struct rte_eth_dev *eth_dev) +{ + struct avp_dev *avp = AVP_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); + int ret; + + rte_spinlock_lock(&avp->lock); + if (avp->flags & AVP_F_DETACHED) { + PMD_DRV_LOG(ERR, "Operation not supported during VM live migration\n"); + goto unlock; + } + + /* remember current link state */ + avp->flags &= ~AVP_F_LINKUP; + avp->flags &= ~AVP_F_CONFIGURED; + + ret = avp_dev_disable_interrupts(eth_dev); + if (ret < 0) { + PMD_DRV_LOG(ERR, "Failed to disable interrupts\n"); + /* continue */ + } + + /* update device state */ + ret = avp_dev_ctrl_shutdown(eth_dev); + if (ret < 0) { + PMD_DRV_LOG(ERR, "Device shutdown failed by host, ret=%d\n", + ret); + /* continue */ + } + +unlock: + rte_spinlock_unlock(&avp->lock); +} + +static int +avp_dev_link_update(struct rte_eth_dev *eth_dev, + __rte_unused int wait_to_complete) +{ + struct avp_dev *avp = AVP_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); + struct rte_eth_link *link = ð_dev->data->dev_link; + + link->link_speed = ETH_SPEED_NUM_10G; + link->link_duplex = ETH_LINK_FULL_DUPLEX; + link->link_status = !!(avp->flags & AVP_F_LINKUP); + + return -1; +} + +static void +avp_dev_promiscuous_enable(struct rte_eth_dev *eth_dev) +{ + struct avp_dev *avp = AVP_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); + + rte_spinlock_lock(&avp->lock); + if ((avp->flags & AVP_F_PROMISC) == 0) { + avp->flags |= AVP_F_PROMISC; + PMD_DRV_LOG(DEBUG, "Promiscuous mode enabled on %u\n", + eth_dev->data->port_id); + } + rte_spinlock_unlock(&avp->lock); +} + +static void +avp_dev_promiscuous_disable(struct rte_eth_dev *eth_dev) +{ + struct avp_dev *avp = AVP_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); + + rte_spinlock_lock(&avp->lock); + if ((avp->flags & AVP_F_PROMISC) != 0) { + avp->flags &= ~AVP_F_PROMISC; + PMD_DRV_LOG(DEBUG, "Promiscuous mode disabled on %u\n", + eth_dev->data->port_id); + } + rte_spinlock_unlock(&avp->lock); +} + +static void +avp_dev_info_get(struct rte_eth_dev *eth_dev, + struct rte_eth_dev_info *dev_info) +{ + struct avp_dev *avp = AVP_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); + + dev_info->pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); + dev_info->max_rx_queues = avp->max_rx_queues; + dev_info->max_tx_queues = avp->max_tx_queues; + dev_info->min_rx_bufsize = AVP_MIN_RX_BUFSIZE; + dev_info->max_rx_pktlen = avp->max_rx_pkt_len; + dev_info->max_mac_addrs = AVP_MAX_MAC_ADDRS; + if (avp->host_features & RTE_AVP_FEATURE_VLAN_OFFLOAD) { + dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP; + dev_info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT; + } +} + +static int +avp_vlan_offload_set(struct rte_eth_dev *eth_dev, int mask) +{ + struct avp_dev *avp = AVP_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); + + if (mask & ETH_VLAN_STRIP_MASK) { + if (avp->host_features & RTE_AVP_FEATURE_VLAN_OFFLOAD) { + if (eth_dev->data->dev_conf.rxmode.hw_vlan_strip) + avp->features |= RTE_AVP_FEATURE_VLAN_OFFLOAD; + else + avp->features &= ~RTE_AVP_FEATURE_VLAN_OFFLOAD; + } else { + PMD_DRV_LOG(ERR, "VLAN strip offload not supported\n"); + } + } + + if (mask & ETH_VLAN_FILTER_MASK) { + if (eth_dev->data->dev_conf.rxmode.hw_vlan_filter) + PMD_DRV_LOG(ERR, "VLAN filter offload not supported\n"); + } + + if (mask & ETH_VLAN_EXTEND_MASK) { + if (eth_dev->data->dev_conf.rxmode.hw_vlan_extend) + PMD_DRV_LOG(ERR, "VLAN extend offload not supported\n"); + } + + return 0; +} + +static int +avp_dev_stats_get(struct rte_eth_dev *eth_dev, struct rte_eth_stats *stats) +{ + struct avp_dev *avp = AVP_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); + unsigned int i; + + for (i = 0; i < avp->num_rx_queues; i++) { + struct avp_queue *rxq = avp->dev_data->rx_queues[i]; + + if (rxq) { + stats->ipackets += rxq->packets; + stats->ibytes += rxq->bytes; + stats->ierrors += rxq->errors; + + stats->q_ipackets[i] += rxq->packets; + stats->q_ibytes[i] += rxq->bytes; + stats->q_errors[i] += rxq->errors; + } + } + + for (i = 0; i < avp->num_tx_queues; i++) { + struct avp_queue *txq = avp->dev_data->tx_queues[i]; + + if (txq) { + stats->opackets += txq->packets; + stats->obytes += txq->bytes; + stats->oerrors += txq->errors; + + stats->q_opackets[i] += txq->packets; + stats->q_obytes[i] += txq->bytes; + stats->q_errors[i] += txq->errors; + } + } + + return 0; +} + +static void +avp_dev_stats_reset(struct rte_eth_dev *eth_dev) +{ + struct avp_dev *avp = AVP_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); + unsigned int i; + + for (i = 0; i < avp->num_rx_queues; i++) { + struct avp_queue *rxq = avp->dev_data->rx_queues[i]; + + if (rxq) { + rxq->bytes = 0; + rxq->packets = 0; + rxq->errors = 0; + } + } + + for (i = 0; i < avp->num_tx_queues; i++) { + struct avp_queue *txq = avp->dev_data->tx_queues[i]; + + if (txq) { + txq->bytes = 0; + txq->packets = 0; + txq->errors = 0; + } + } +} + +RTE_PMD_REGISTER_PCI(net_avp, rte_avp_pmd); +RTE_PMD_REGISTER_PCI_TABLE(net_avp, pci_id_avp_map); diff --git a/dpdk/drivers/net/avp/avp_logs.h b/dpdk/drivers/net/avp/avp_logs.h new file mode 100644 index 00000000..252cab7d --- /dev/null +++ b/dpdk/drivers/net/avp/avp_logs.h @@ -0,0 +1,59 @@ +/* + * BSD LICENSE + * + * Copyright (c) 2013-2015, Wind River Systems, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1) Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2) Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3) Neither the name of Wind River Systems nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _AVP_LOGS_H_ +#define _AVP_LOGS_H_ + +#include + +#ifdef RTE_LIBRTE_AVP_DEBUG_RX +#define PMD_RX_LOG(level, fmt, args...) \ + RTE_LOG(level, PMD, "%s() rx: " fmt, __func__, ## args) +#else +#define PMD_RX_LOG(level, fmt, args...) do { } while (0) +#endif + +#ifdef RTE_LIBRTE_AVP_DEBUG_TX +#define PMD_TX_LOG(level, fmt, args...) \ + RTE_LOG(level, PMD, "%s() tx: " fmt, __func__, ## args) +#else +#define PMD_TX_LOG(level, fmt, args...) do { } while (0) +#endif + +#ifdef RTE_LIBRTE_AVP_DEBUG_DRIVER +#define PMD_DRV_LOG(level, fmt, args...) \ + RTE_LOG(level, PMD, "%s(): " fmt, __func__, ## args) +#else +#define PMD_DRV_LOG(level, fmt, args...) do { } while (0) +#endif + +#endif /* _AVP_LOGS_H_ */ diff --git a/dpdk/drivers/net/avp/rte_avp_common.h b/dpdk/drivers/net/avp/rte_avp_common.h new file mode 100644 index 00000000..81dfe5ea --- /dev/null +++ b/dpdk/drivers/net/avp/rte_avp_common.h @@ -0,0 +1,433 @@ +/*- + * This file is provided under a dual BSD/LGPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * GNU LESSER GENERAL PUBLIC LICENSE + * + * Copyright(c) 2010-2013 Intel Corporation. All rights reserved. + * Copyright(c) 2014-2017 Wind River Systems, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2.1 of the GNU Lesser General Public License + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * Contact Information: + * Wind River Systems, Inc. + * + * + * BSD LICENSE + * + * Copyright(c) 2010-2013 Intel Corporation. All rights reserved. + * Copyright(c) 2014-2017 Wind River Systems, Inc. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef _RTE_AVP_COMMON_H_ +#define _RTE_AVP_COMMON_H_ + +#ifdef __KERNEL__ +#include +#define RTE_STD_C11 +#else +#include +#include +#include +#include +#include +#include +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * AVP name is part of network device name. + */ +#define RTE_AVP_NAMESIZE 32 + +/** + * AVP alias is a user-defined value used for lookups from secondary + * processes. Typically, this is a UUID. + */ +#define RTE_AVP_ALIASSIZE 128 + +/* + * Request id. + */ +enum rte_avp_req_id { + RTE_AVP_REQ_UNKNOWN = 0, + RTE_AVP_REQ_CHANGE_MTU, + RTE_AVP_REQ_CFG_NETWORK_IF, + RTE_AVP_REQ_CFG_DEVICE, + RTE_AVP_REQ_SHUTDOWN_DEVICE, + RTE_AVP_REQ_MAX, +}; + +/**@{ AVP device driver types */ +#define RTE_AVP_DRIVER_TYPE_UNKNOWN 0 +#define RTE_AVP_DRIVER_TYPE_DPDK 1 +#define RTE_AVP_DRIVER_TYPE_KERNEL 2 +#define RTE_AVP_DRIVER_TYPE_QEMU 3 +/**@} */ + +/**@{ AVP device operational modes */ +#define RTE_AVP_MODE_HOST 0 /**< AVP interface created in host */ +#define RTE_AVP_MODE_GUEST 1 /**< AVP interface created for export to guest */ +#define RTE_AVP_MODE_TRACE 2 /**< AVP interface created for packet tracing */ +/**@} */ + +/* + * Structure for AVP queue configuration query request/result + */ +struct rte_avp_device_config { + uint64_t device_id; /**< Unique system identifier */ + uint32_t driver_type; /**< Device Driver type */ + uint32_t driver_version; /**< Device Driver version */ + uint32_t features; /**< Negotiated features */ + uint16_t num_tx_queues; /**< Number of active transmit queues */ + uint16_t num_rx_queues; /**< Number of active receive queues */ + uint8_t if_up; /**< 1: interface up, 0: interface down */ +} __attribute__ ((__packed__)); + +/* + * Structure for AVP request. + */ +struct rte_avp_request { + uint32_t req_id; /**< Request id */ + RTE_STD_C11 + union { + uint32_t new_mtu; /**< New MTU */ + uint8_t if_up; /**< 1: interface up, 0: interface down */ + struct rte_avp_device_config config; /**< Queue configuration */ + }; + int32_t result; /**< Result for processing request */ +} __attribute__ ((__packed__)); + +/* + * FIFO struct mapped in a shared memory. It describes a circular buffer FIFO + * Write and read should wrap around. FIFO is empty when write == read + * Writing should never overwrite the read position + */ +struct rte_avp_fifo { + volatile unsigned int write; /**< Next position to be written*/ + volatile unsigned int read; /**< Next position to be read */ + unsigned int len; /**< Circular buffer length */ + unsigned int elem_size; /**< Pointer size - for 32/64 bit OS */ + void *volatile buffer[]; /**< The buffer contains mbuf pointers */ +}; + + +/* + * AVP packet buffer header used to define the exchange of packet data. + */ +struct rte_avp_desc { + uint64_t pad0; + void *pkt_mbuf; /**< Reference to packet mbuf */ + uint8_t pad1[14]; + uint16_t ol_flags; /**< Offload features. */ + void *next; /**< Reference to next buffer in chain */ + void *data; /**< Start address of data in segment buffer. */ + uint16_t data_len; /**< Amount of data in segment buffer. */ + uint8_t nb_segs; /**< Number of segments */ + uint8_t pad2; + uint16_t pkt_len; /**< Total pkt len: sum of all segment data_len. */ + uint32_t pad3; + uint16_t vlan_tci; /**< VLAN Tag Control Identifier (CPU order). */ + uint32_t pad4; +} __attribute__ ((__aligned__(RTE_CACHE_LINE_SIZE), __packed__)); + + +/**{ AVP device features */ +#define RTE_AVP_FEATURE_VLAN_OFFLOAD (1 << 0) /**< Emulated HW VLAN offload */ +/**@} */ + + +/**@{ Offload feature flags */ +#define RTE_AVP_TX_VLAN_PKT 0x0001 /**< TX packet is a 802.1q VLAN packet. */ +#define RTE_AVP_RX_VLAN_PKT 0x0800 /**< RX packet is a 802.1q VLAN packet. */ +/**@} */ + + +/**@{ AVP PCI identifiers */ +#define RTE_AVP_PCI_VENDOR_ID 0x1af4 +#define RTE_AVP_PCI_DEVICE_ID 0x1110 +/**@} */ + +/**@{ AVP PCI subsystem identifiers */ +#define RTE_AVP_PCI_SUB_VENDOR_ID RTE_AVP_PCI_VENDOR_ID +#define RTE_AVP_PCI_SUB_DEVICE_ID 0x1104 +/**@} */ + +/**@{ AVP PCI BAR definitions */ +#define RTE_AVP_PCI_MMIO_BAR 0 +#define RTE_AVP_PCI_MSIX_BAR 1 +#define RTE_AVP_PCI_MEMORY_BAR 2 +#define RTE_AVP_PCI_MEMMAP_BAR 4 +#define RTE_AVP_PCI_DEVICE_BAR 5 +#define RTE_AVP_PCI_MAX_BAR 6 +/**@} */ + +/**@{ AVP PCI BAR name definitions */ +#define RTE_AVP_MMIO_BAR_NAME "avp-mmio" +#define RTE_AVP_MSIX_BAR_NAME "avp-msix" +#define RTE_AVP_MEMORY_BAR_NAME "avp-memory" +#define RTE_AVP_MEMMAP_BAR_NAME "avp-memmap" +#define RTE_AVP_DEVICE_BAR_NAME "avp-device" +/**@} */ + +/**@{ AVP PCI MSI-X vectors */ +#define RTE_AVP_MIGRATION_MSIX_VECTOR 0 /**< Migration interrupts */ +#define RTE_AVP_MAX_MSIX_VECTORS 1 +/**@} */ + +/**@} AVP Migration status/ack register values */ +#define RTE_AVP_MIGRATION_NONE 0 /**< Migration never executed */ +#define RTE_AVP_MIGRATION_DETACHED 1 /**< Device attached during migration */ +#define RTE_AVP_MIGRATION_ATTACHED 2 /**< Device reattached during migration */ +#define RTE_AVP_MIGRATION_ERROR 3 /**< Device failed to attach/detach */ +/**@} */ + +/**@} AVP MMIO Register Offsets */ +#define RTE_AVP_REGISTER_BASE 0 +#define RTE_AVP_INTERRUPT_MASK_OFFSET (RTE_AVP_REGISTER_BASE + 0) +#define RTE_AVP_INTERRUPT_STATUS_OFFSET (RTE_AVP_REGISTER_BASE + 4) +#define RTE_AVP_MIGRATION_STATUS_OFFSET (RTE_AVP_REGISTER_BASE + 8) +#define RTE_AVP_MIGRATION_ACK_OFFSET (RTE_AVP_REGISTER_BASE + 12) +/**@} */ + +/**@} AVP Interrupt Status Mask */ +#define RTE_AVP_MIGRATION_INTERRUPT_MASK (1 << 1) +#define RTE_AVP_APP_INTERRUPTS_MASK 0xFFFFFFFF +#define RTE_AVP_NO_INTERRUPTS_MASK 0 +/**@} */ + +/* + * Maximum number of memory regions to export + */ +#define RTE_AVP_MAX_MAPS 2048 + +/* + * Description of a single memory region + */ +struct rte_avp_memmap { + void *addr; + rte_iova_t phys_addr; + uint64_t length; +}; + +/* + * AVP memory mapping validation marker + */ +#define RTE_AVP_MEMMAP_MAGIC 0x20131969 + +/**@{ AVP memory map versions */ +#define RTE_AVP_MEMMAP_VERSION_1 1 +#define RTE_AVP_MEMMAP_VERSION RTE_AVP_MEMMAP_VERSION_1 +/**@} */ + +/* + * Defines a list of memory regions exported from the host to the guest + */ +struct rte_avp_memmap_info { + uint32_t magic; /**< Memory validation marker */ + uint32_t version; /**< Data format version */ + uint32_t nb_maps; + struct rte_avp_memmap maps[RTE_AVP_MAX_MAPS]; +}; + +/* + * AVP device memory validation marker + */ +#define RTE_AVP_DEVICE_MAGIC 0x20131975 + +/**@{ AVP device map versions + * WARNING: do not change the format or names of these variables. They are + * automatically parsed from the build system to generate the SDK package + * name. + **/ +#define RTE_AVP_RELEASE_VERSION_1 1 +#define RTE_AVP_RELEASE_VERSION RTE_AVP_RELEASE_VERSION_1 +#define RTE_AVP_MAJOR_VERSION_0 0 +#define RTE_AVP_MAJOR_VERSION_1 1 +#define RTE_AVP_MAJOR_VERSION_2 2 +#define RTE_AVP_MAJOR_VERSION RTE_AVP_MAJOR_VERSION_2 +#define RTE_AVP_MINOR_VERSION_0 0 +#define RTE_AVP_MINOR_VERSION_1 1 +#define RTE_AVP_MINOR_VERSION_13 13 +#define RTE_AVP_MINOR_VERSION RTE_AVP_MINOR_VERSION_13 +/**@} */ + + +/** + * Generates a 32-bit version number from the specified version number + * components + */ +#define RTE_AVP_MAKE_VERSION(_release, _major, _minor) \ +((((_release) & 0xffff) << 16) | (((_major) & 0xff) << 8) | ((_minor) & 0xff)) + + +/** + * Represents the current version of the AVP host driver + * WARNING: in the current development branch the host and guest driver + * version should always be the same. When patching guest features back to + * GA releases the host version number should not be updated unless there was + * an actual change made to the host driver. + */ +#define RTE_AVP_CURRENT_HOST_VERSION \ +RTE_AVP_MAKE_VERSION(RTE_AVP_RELEASE_VERSION_1, \ + RTE_AVP_MAJOR_VERSION_0, \ + RTE_AVP_MINOR_VERSION_1) + + +/** + * Represents the current version of the AVP guest drivers + */ +#define RTE_AVP_CURRENT_GUEST_VERSION \ +RTE_AVP_MAKE_VERSION(RTE_AVP_RELEASE_VERSION_1, \ + RTE_AVP_MAJOR_VERSION_2, \ + RTE_AVP_MINOR_VERSION_13) + +/** + * Access AVP device version values + */ +#define RTE_AVP_GET_RELEASE_VERSION(_version) (((_version) >> 16) & 0xffff) +#define RTE_AVP_GET_MAJOR_VERSION(_version) (((_version) >> 8) & 0xff) +#define RTE_AVP_GET_MINOR_VERSION(_version) ((_version) & 0xff) +/**@}*/ + + +/** + * Remove the minor version number so that only the release and major versions + * are used for comparisons. + */ +#define RTE_AVP_STRIP_MINOR_VERSION(_version) ((_version) >> 8) + + +/** + * Defines the number of mbuf pools supported per device (1 per socket) + */ +#define RTE_AVP_MAX_MEMPOOLS 8 + +/* + * Defines address translation parameters for each support mbuf pool + */ +struct rte_avp_mempool_info { + void *addr; + rte_iova_t phys_addr; + uint64_t length; +}; + +/* + * Struct used to create a AVP device. Passed to the kernel in IOCTL call or + * via inter-VM shared memory when used in a guest. + */ +struct rte_avp_device_info { + uint32_t magic; /**< Memory validation marker */ + uint32_t version; /**< Data format version */ + + char ifname[RTE_AVP_NAMESIZE]; /**< Network device name for AVP */ + + rte_iova_t tx_phys; + rte_iova_t rx_phys; + rte_iova_t alloc_phys; + rte_iova_t free_phys; + + uint32_t features; /**< Supported feature bitmap */ + uint8_t min_rx_queues; /**< Minimum supported receive/free queues */ + uint8_t num_rx_queues; /**< Recommended number of receive/free queues */ + uint8_t max_rx_queues; /**< Maximum supported receive/free queues */ + uint8_t min_tx_queues; /**< Minimum supported transmit/alloc queues */ + uint8_t num_tx_queues; + /**< Recommended number of transmit/alloc queues */ + uint8_t max_tx_queues; /**< Maximum supported transmit/alloc queues */ + + uint32_t tx_size; /**< Size of each transmit queue */ + uint32_t rx_size; /**< Size of each receive queue */ + uint32_t alloc_size; /**< Size of each alloc queue */ + uint32_t free_size; /**< Size of each free queue */ + + /* Used by Ethtool */ + rte_iova_t req_phys; + rte_iova_t resp_phys; + rte_iova_t sync_phys; + void *sync_va; + + /* mbuf mempool (used when a single memory area is supported) */ + void *mbuf_va; + rte_iova_t mbuf_phys; + + /* mbuf mempools */ + struct rte_avp_mempool_info pool[RTE_AVP_MAX_MEMPOOLS]; + +#ifdef __KERNEL__ + /* Ethernet info */ + char ethaddr[ETH_ALEN]; +#else + char ethaddr[ETHER_ADDR_LEN]; +#endif + + uint8_t mode; /**< device mode, i.e guest, host, trace */ + + /* mbuf size */ + unsigned int mbuf_size; + + /* + * unique id to differentiate between two instantiations of the same + * AVP device (i.e., the guest needs to know if the device has been + * deleted and recreated). + */ + uint64_t device_id; + + uint32_t max_rx_pkt_len; /**< Maximum receive unit size */ +}; + +#define RTE_AVP_MAX_QUEUES 8 /**< Maximum number of queues per device */ + +/** Maximum number of chained mbufs in a packet */ +#define RTE_AVP_MAX_MBUF_SEGMENTS 5 + +#define RTE_AVP_DEVICE "avp" + +#define RTE_AVP_IOCTL_TEST _IOWR(0, 1, int) +#define RTE_AVP_IOCTL_CREATE _IOWR(0, 2, struct rte_avp_device_info) +#define RTE_AVP_IOCTL_RELEASE _IOWR(0, 3, struct rte_avp_device_info) +#define RTE_AVP_IOCTL_QUERY _IOWR(0, 4, struct rte_avp_device_config) + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_AVP_COMMON_H_ */ diff --git a/dpdk/drivers/net/avp/rte_avp_fifo.h b/dpdk/drivers/net/avp/rte_avp_fifo.h new file mode 100644 index 00000000..803eb80a --- /dev/null +++ b/dpdk/drivers/net/avp/rte_avp_fifo.h @@ -0,0 +1,169 @@ +/*- + * This file is provided under a dual BSD/LGPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * GNU LESSER GENERAL PUBLIC LICENSE + * + * Copyright(c) 2010-2013 Intel Corporation. All rights reserved. + * Copyright(c) 2014 Wind River Systems, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2.1 of the GNU Lesser General Public License + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * Contact Information: + * Wind River Systems, Inc. + * + * + * BSD LICENSE + * + * Copyright(c) 2010-2013 Intel Corporation. All rights reserved. + * Copyright(c) 2013-2017 Wind River Systems, Inc. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef _RTE_AVP_FIFO_H_ +#define _RTE_AVP_FIFO_H_ + +#include "rte_avp_common.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef __KERNEL__ +/* Write memory barrier for kernel compiles */ +#define AVP_WMB() smp_wmb() +/* Read memory barrier for kernel compiles */ +#define AVP_RMB() smp_rmb() +#else +/* Write memory barrier for userspace compiles */ +#define AVP_WMB() rte_wmb() +/* Read memory barrier for userspace compiles */ +#define AVP_RMB() rte_rmb() +#endif + +#ifndef __KERNEL__ +#include + +/** + * Initializes the avp fifo structure + */ +static inline void +avp_fifo_init(struct rte_avp_fifo *fifo, unsigned int size) +{ + /* Ensure size is power of 2 */ + if (size & (size - 1)) + rte_panic("AVP fifo size must be power of 2\n"); + + fifo->write = 0; + fifo->read = 0; + fifo->len = size; + fifo->elem_size = sizeof(void *); +} +#endif + +/** + * Adds num elements into the fifo. Return the number actually written + */ +static inline unsigned +avp_fifo_put(struct rte_avp_fifo *fifo, void **data, unsigned int num) +{ + unsigned int i = 0; + unsigned int fifo_write = fifo->write; + unsigned int fifo_read = fifo->read; + unsigned int new_write = fifo_write; + + for (i = 0; i < num; i++) { + new_write = (new_write + 1) & (fifo->len - 1); + + if (new_write == fifo_read) + break; + fifo->buffer[fifo_write] = data[i]; + fifo_write = new_write; + } + AVP_WMB(); + fifo->write = fifo_write; + return i; +} + +/** + * Get up to num elements from the fifo. Return the number actually read + */ +static inline unsigned int +avp_fifo_get(struct rte_avp_fifo *fifo, void **data, unsigned int num) +{ + unsigned int i = 0; + unsigned int new_read = fifo->read; + unsigned int fifo_write = fifo->write; + + if (new_read == fifo_write) + return 0; /* empty */ + + for (i = 0; i < num; i++) { + if (new_read == fifo_write) + break; + + data[i] = fifo->buffer[new_read]; + new_read = (new_read + 1) & (fifo->len - 1); + } + AVP_RMB(); + fifo->read = new_read; + return i; +} + +/** + * Get the num of elements in the fifo + */ +static inline unsigned int +avp_fifo_count(struct rte_avp_fifo *fifo) +{ + return (fifo->len + fifo->write - fifo->read) & (fifo->len - 1); +} + +/** + * Get the num of available elements in the fifo + */ +static inline unsigned int +avp_fifo_free_count(struct rte_avp_fifo *fifo) +{ + return (fifo->read - fifo->write - 1) & (fifo->len - 1); +} + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_AVP_FIFO_H_ */ diff --git a/dpdk/drivers/net/avp/rte_pmd_avp_version.map b/dpdk/drivers/net/avp/rte_pmd_avp_version.map new file mode 100644 index 00000000..af8f3f47 --- /dev/null +++ b/dpdk/drivers/net/avp/rte_pmd_avp_version.map @@ -0,0 +1,4 @@ +DPDK_17.05 { + + local: *; +}; diff --git a/dpdk/drivers/net/bnx2x/Makefile b/dpdk/drivers/net/bnx2x/Makefile index ab696801..90ff8b1e 100644 --- a/dpdk/drivers/net/bnx2x/Makefile +++ b/dpdk/drivers/net/bnx2x/Makefile @@ -9,6 +9,9 @@ CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) CFLAGS += -DZLIB_CONST LDLIBS += -lz +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs +LDLIBS += -lrte_bus_pci EXPORT_MAP := rte_pmd_bnx2x_version.map @@ -28,10 +31,5 @@ SRCS-$(CONFIG_RTE_LIBRTE_BNX2X_PMD) += bnx2x_ethdev.c SRCS-$(CONFIG_RTE_LIBRTE_BNX2X_PMD) += ecore_sp.c SRCS-$(CONFIG_RTE_LIBRTE_BNX2X_PMD) += elink.c SRCS-$(CONFIG_RTE_LIBRTE_BNX2X_PMD) += bnx2x_vfpf.c -SRCS-$(CONFIG_RTE_LIBRTE_BNX2X_DEBUG_PERIODIC) += debug.c - -# this lib depends upon: -DEPDIRS-$(CONFIG_RTE_LIBRTE_BNX2X_PMD) += lib/librte_eal lib/librte_ether -DEPDIRS-$(CONFIG_RTE_LIBRTE_BNX2X_PMD) += lib/librte_mempool lib/librte_mbuf include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/net/bnx2x/bnx2x.c b/dpdk/drivers/net/bnx2x/bnx2x.c index 8970334e..9394f6c5 100644 --- a/dpdk/drivers/net/bnx2x/bnx2x.c +++ b/dpdk/drivers/net/bnx2x/bnx2x.c @@ -31,7 +31,7 @@ #define BNX2X_PMD_VER_PREFIX "BNX2X PMD" #define BNX2X_PMD_VERSION_MAJOR 1 #define BNX2X_PMD_VERSION_MINOR 0 -#define BNX2X_PMD_VERSION_REVISION 1 +#define BNX2X_PMD_VERSION_REVISION 5 #define BNX2X_PMD_VERSION_PATCH 1 static inline const char * @@ -120,7 +120,7 @@ static int bnx2x_alloc_mem(struct bnx2x_softc *sc); static void bnx2x_free_mem(struct bnx2x_softc *sc); static int bnx2x_alloc_fw_stats_mem(struct bnx2x_softc *sc); static void bnx2x_free_fw_stats_mem(struct bnx2x_softc *sc); -static __attribute__ ((noinline)) +static __rte_noinline int bnx2x_nic_load(struct bnx2x_softc *sc); static int bnx2x_handle_sp_tq(struct bnx2x_softc *sc); @@ -184,7 +184,7 @@ bnx2x_dma_alloc(struct bnx2x_softc *sc, size_t size, struct bnx2x_dma *dma, PMD_DRV_LOG(ERR, "DMA alloc failed for %s", msg); return -ENOMEM; } - dma->paddr = (uint64_t) z->phys_addr; + dma->paddr = (uint64_t) z->iova; dma->vaddr = z->addr; PMD_DRV_LOG(DEBUG, "%s: virt=%p phys=%" PRIx64, msg, dma->vaddr, dma->paddr); @@ -419,7 +419,7 @@ void bnx2x_read_dmae(struct bnx2x_softc *sc, uint32_t src_addr, uint32_t len32) } void -bnx2x_write_dmae(struct bnx2x_softc *sc, phys_addr_t dma_addr, uint32_t dst_addr, +bnx2x_write_dmae(struct bnx2x_softc *sc, rte_iova_t dma_addr, uint32_t dst_addr, uint32_t len32) { struct dmae_command dmae; @@ -447,7 +447,7 @@ bnx2x_write_dmae(struct bnx2x_softc *sc, phys_addr_t dma_addr, uint32_t dst_addr } static void -bnx2x_write_dmae_phys_len(struct bnx2x_softc *sc, phys_addr_t phys_addr, +bnx2x_write_dmae_phys_len(struct bnx2x_softc *sc, rte_iova_t phys_addr, uint32_t addr, uint32_t len) { uint32_t dmae_wr_max = DMAE_LEN32_WR_MAX(sc); @@ -823,14 +823,14 @@ bnx2x_fw_command(struct bnx2x_softc *sc, uint32_t command, uint32_t param) static void __storm_memset_dma_mapping(struct bnx2x_softc *sc, uint32_t addr, - phys_addr_t mapping) + rte_iova_t mapping) { REG_WR(sc, addr, U64_LO(mapping)); REG_WR(sc, (addr + 4), U64_HI(mapping)); } static void -storm_memset_spq_addr(struct bnx2x_softc *sc, phys_addr_t mapping, +storm_memset_spq_addr(struct bnx2x_softc *sc, rte_iova_t mapping, uint16_t abs_fid) { uint32_t addr = (XSEM_REG_FAST_MEMORY + @@ -887,7 +887,7 @@ storm_memset_eq_prod(struct bnx2x_softc *sc, uint16_t eq_prod, uint16_t pfid) /* * Post a slowpath command. * - * A slowpath command is used to propogate a configuration change through + * A slowpath command is used to propagate a configuration change through * the controller in a controlled manner, allowing each STORM processor and * other H/W blocks to phase in the change. The commands sent on the * slowpath are referred to as ramrods. Depending on the ramrod used the @@ -1397,10 +1397,10 @@ bnx2x_del_all_macs(struct bnx2x_softc *sc, struct ecore_vlan_mac_obj *mac_obj, return rc; } -int +static int bnx2x_fill_accept_flags(struct bnx2x_softc *sc, uint32_t rx_mode, - unsigned long *rx_accept_flags, - unsigned long *tx_accept_flags) + unsigned long *rx_accept_flags, + unsigned long *tx_accept_flags) { /* Clear the flags first */ *rx_accept_flags = 0; @@ -1438,6 +1438,7 @@ bnx2x_fill_accept_flags(struct bnx2x_softc *sc, uint32_t rx_mode, break; + case BNX2X_RX_MODE_ALLMULTI_PROMISC: case BNX2X_RX_MODE_PROMISC: /* * According to deffinition of SI mode, iface in promisc mode @@ -1497,7 +1498,7 @@ bnx2x_set_q_rx_mode(struct bnx2x_softc *sc, uint8_t cl_id, ramrod_param.rdata = BNX2X_SP(sc, rx_mode_rdata); ramrod_param.rdata_mapping = - (phys_addr_t)BNX2X_SP_MAPPING(sc, rx_mode_rdata), + (rte_iova_t)BNX2X_SP_MAPPING(sc, rx_mode_rdata), bnx2x_set_bit(ECORE_FILTER_RX_MODE_PENDING, &sc->sp_state); ramrod_param.ramrod_flags = ramrod_flags; @@ -1961,7 +1962,7 @@ static void bnx2x_squeeze_objects(struct bnx2x_softc *sc) } /* stop the controller */ -__attribute__ ((noinline)) +__rte_noinline int bnx2x_nic_unload(struct bnx2x_softc *sc, uint32_t unload_mode, uint8_t keep_link) { @@ -2001,7 +2002,7 @@ bnx2x_nic_unload(struct bnx2x_softc *sc, uint32_t unload_mode, uint8_t keep_link /* * Nothing to do during unload if previous bnx2x_nic_load() - * did not completed succesfully - all resourses are released. + * did not completed successfully - all resourses are released. */ if ((sc->state == BNX2X_STATE_CLOSED) || (sc->state == BNX2X_STATE_ERROR)) { return 0; @@ -2134,7 +2135,7 @@ int bnx2x_tx_encap(struct bnx2x_tx_queue *txq, struct rte_mbuf *m0) tx_start_bd = &txq->tx_ring[TX_BD(bd_prod, txq)].start_bd; tx_start_bd->addr = - rte_cpu_to_le_64(rte_mbuf_data_dma_addr(m0)); + rte_cpu_to_le_64(rte_mbuf_data_iova(m0)); tx_start_bd->nbytes = rte_cpu_to_le_16(m0->data_len); tx_start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD; tx_start_bd->general_data = @@ -2219,7 +2220,7 @@ int bnx2x_tx_encap(struct bnx2x_tx_queue *txq, struct rte_mbuf *m0) } PMD_TX_LOG(DEBUG, - "start bd: nbytes %d flags %x vlan %x\n", + "start bd: nbytes %d flags %x vlan %x", tx_start_bd->nbytes, tx_start_bd->bd_flags.as_bitfield, tx_start_bd->vlan_or_ethertype); @@ -3930,7 +3931,7 @@ static void bnx2x_attn_int_deasserted2(struct bnx2x_softc *sc, uint32_t attn) mask1 = REG_RD(sc, PXP2_REG_PXP2_INT_MASK_1); val0 = REG_RD(sc, PXP2_REG_PXP2_INT_STS_0); /* - * If the olny PXP2_EOP_ERROR_BIT is set in + * If the only PXP2_EOP_ERROR_BIT is set in * STS0 and STS1 - clear it * * probably we lose additional attentions between @@ -4598,9 +4599,9 @@ static void bnx2x_init_func_obj(struct bnx2x_softc *sc) ecore_init_func_obj(sc, &sc->func_obj, BNX2X_SP(sc, func_rdata), - (phys_addr_t)BNX2X_SP_MAPPING(sc, func_rdata), + (rte_iova_t)BNX2X_SP_MAPPING(sc, func_rdata), BNX2X_SP(sc, func_afex_rdata), - (phys_addr_t)BNX2X_SP_MAPPING(sc, func_afex_rdata), + (rte_iova_t)BNX2X_SP_MAPPING(sc, func_afex_rdata), &bnx2x_func_sp_drv); } @@ -4771,7 +4772,7 @@ static void bnx2x_map_sb_state_machines(struct hc_index_data *index_data) } static void -bnx2x_init_sb(struct bnx2x_softc *sc, phys_addr_t busaddr, int vfid, +bnx2x_init_sb(struct bnx2x_softc *sc, rte_iova_t busaddr, int vfid, uint8_t vf_valid, int fw_sb_id, int igu_sb_id) { struct hc_status_block_data_e2 sb_data_e2; @@ -4917,7 +4918,7 @@ static void bnx2x_init_eth_fp(struct bnx2x_softc *sc, int idx) sc->max_cos, SC_FUNC(sc), BNX2X_SP(sc, q_rdata), - (phys_addr_t)BNX2X_SP_MAPPING(sc, q_rdata), + (rte_iova_t)BNX2X_SP_MAPPING(sc, q_rdata), q_type); /* configure classification DBs */ @@ -4927,7 +4928,7 @@ static void bnx2x_init_eth_fp(struct bnx2x_softc *sc, int idx) idx, SC_FUNC(sc), BNX2X_SP(sc, mac_rdata), - (phys_addr_t)BNX2X_SP_MAPPING(sc, mac_rdata), + (rte_iova_t)BNX2X_SP_MAPPING(sc, mac_rdata), ECORE_FILTER_MAC_PENDING, &sc->sp_state, ECORE_OBJ_TYPE_RX_TX, &sc->macs_pool); } @@ -5027,7 +5028,7 @@ static void bnx2x_init_tx_rings(struct bnx2x_softc *sc) static void bnx2x_init_def_sb(struct bnx2x_softc *sc) { struct host_sp_status_block *def_sb = sc->def_sb; - phys_addr_t mapping = sc->def_sb_dma.paddr; + rte_iova_t mapping = sc->def_sb_dma.paddr; int igu_sp_sb_index; int igu_seg_id; int port = SC_PORT(sc); @@ -5699,7 +5700,7 @@ static void bnx2x_init_objs(struct bnx2x_softc *sc) SC_FUNC(sc), SC_FUNC(sc), BNX2X_SP(sc, mcast_rdata), - (phys_addr_t)BNX2X_SP_MAPPING(sc, mcast_rdata), + (rte_iova_t)BNX2X_SP_MAPPING(sc, mcast_rdata), ECORE_FILTER_MCAST_PENDING, &sc->sp_state, o_type); @@ -5723,7 +5724,7 @@ static void bnx2x_init_objs(struct bnx2x_softc *sc) SC_FUNC(sc), SC_FUNC(sc), BNX2X_SP(sc, rss_rdata), - (phys_addr_t)BNX2X_SP_MAPPING(sc, rss_rdata), + (rte_iova_t)BNX2X_SP_MAPPING(sc, rss_rdata), ECORE_FILTER_RSS_CONF_PENDING, &sc->sp_state, ECORE_OBJ_TYPE_RX); } @@ -5909,7 +5910,7 @@ static void bnx2x_set_234_gates(struct bnx2x_softc *sc, uint8_t close) (val | HC_CONFIG_0_REG_BLOCK_DISABLE_0)); } else { -/* Prevent incomming interrupts in IGU */ +/* Prevent incoming interrupts in IGU */ val = REG_RD(sc, IGU_REG_BLOCK_CONFIGURATION); if (close) @@ -6444,9 +6445,9 @@ bnx2x_pf_rx_q_prep(struct bnx2x_softc *sc, struct bnx2x_fastpath *fp, pause->pri_map = 1; /* rxq setup */ - rxq_init->dscr_map = (phys_addr_t)rxq->rx_ring_phys_addr; - rxq_init->rcq_map = (phys_addr_t)rxq->cq_ring_phys_addr; - rxq_init->rcq_np_map = (phys_addr_t)(rxq->cq_ring_phys_addr + + rxq_init->dscr_map = (rte_iova_t)rxq->rx_ring_phys_addr; + rxq_init->rcq_map = (rte_iova_t)rxq->cq_ring_phys_addr; + rxq_init->rcq_np_map = (rte_iova_t)(rxq->cq_ring_phys_addr + BNX2X_PAGE_SIZE); /* @@ -6485,7 +6486,7 @@ bnx2x_pf_tx_q_prep(struct bnx2x_softc *sc, struct bnx2x_fastpath *fp, PMD_TX_LOG(ERR, "ERROR: TX queue is NULL"); return; } - txq_init->dscr_map = (phys_addr_t)txq->tx_ring_phys_addr; + txq_init->dscr_map = (rte_iova_t)txq->tx_ring_phys_addr; txq_init->sb_cq_index = HC_INDEX_ETH_FIRST_TX_CQ_CONS + cos; txq_init->traffic_type = LLFC_TRAFFIC_TYPE_NW; txq_init->fw_sb_id = fp->fw_sb_id; @@ -6603,7 +6604,7 @@ bnx2x_config_rss_pf(struct bnx2x_softc *sc, struct ecore_rss_config_obj *rss_obj /* Hash bits */ params.rss_result_mask = MULTI_MASK; - (void)rte_memcpy(params.ind_table, rss_obj->ind_table, + rte_memcpy(params.ind_table, rss_obj->ind_table, sizeof(params.ind_table)); if (config_hash) { @@ -6670,7 +6671,7 @@ bnx2x_set_mac_one(struct bnx2x_softc *sc, uint8_t * mac, /* fill a user request section if needed */ if (!bnx2x_test_bit(RAMROD_CONT, ramrod_flags)) { - (void)rte_memcpy(ramrod_param.user_req.u.mac.mac, mac, + rte_memcpy(ramrod_param.user_req.u.mac.mac, mac, ETH_ALEN); bnx2x_set_bit(mac_type, &ramrod_param.user_req.vlan_mac_flags); @@ -6878,7 +6879,7 @@ static void bnx2x_link_report(struct bnx2x_softc *sc) sc->link_cnt++; /* report new link params and remember the state for the next time */ - (void)rte_memcpy(&sc->last_reported_link, &cur_data, sizeof(cur_data)); + rte_memcpy(&sc->last_reported_link, &cur_data, sizeof(cur_data)); if (bnx2x_test_bit(BNX2X_LINK_REPORT_LINK_DOWN, &cur_data.link_report_flags)) { @@ -7016,34 +7017,6 @@ static int bnx2x_initial_phy_init(struct bnx2x_softc *sc, int load_mode) bnx2x_set_requested_fc(sc); - if (CHIP_REV_IS_SLOW(sc)) { - uint32_t bond = CHIP_BOND_ID(sc); - uint32_t feat = 0; - - if (CHIP_IS_E2(sc) && CHIP_IS_MODE_4_PORT(sc)) { - feat |= ELINK_FEATURE_CONFIG_EMUL_DISABLE_BMAC; - } else if (bond & 0x4) { - if (CHIP_IS_E3(sc)) { - feat |= ELINK_FEATURE_CONFIG_EMUL_DISABLE_XMAC; - } else { - feat |= ELINK_FEATURE_CONFIG_EMUL_DISABLE_BMAC; - } - } else if (bond & 0x8) { - if (CHIP_IS_E3(sc)) { - feat |= ELINK_FEATURE_CONFIG_EMUL_DISABLE_UMAC; - } else { - feat |= ELINK_FEATURE_CONFIG_EMUL_DISABLE_EMAC; - } - } - -/* disable EMAC for E3 and above */ - if (bond & 0x2) { - feat |= ELINK_FEATURE_CONFIG_EMUL_DISABLE_EMAC; - } - - sc->link_params.feature_config_flags |= feat; - } - if (load_mode == LOAD_DIAG) { lp->loopback_mode = ELINK_LOOPBACK_XGXS; /* Prefer doing PHY loopback at 10G speed, if possible */ @@ -7151,7 +7124,7 @@ void bnx2x_periodic_callout(struct bnx2x_softc *sc) } /* start the controller */ -static __attribute__ ((noinline)) +static __rte_noinline int bnx2x_nic_load(struct bnx2x_softc *sc) { uint32_t val; @@ -11086,7 +11059,7 @@ static int bnx2x_init_hw_func(struct bnx2x_softc *sc) for (i = 0; i < L2_ILT_LINES(sc); i++) { ilt->lines[cdu_ilt_start + i].page = sc->context[i].vcxt; ilt->lines[cdu_ilt_start + i].page_mapping = - (phys_addr_t)sc->context[i].vcxt_dma.paddr; + (rte_iova_t)sc->context[i].vcxt_dma.paddr; ilt->lines[cdu_ilt_start + i].size = sc->context[i].size; } ecore_ilt_init_op(sc, INITOP_SET); @@ -11384,7 +11357,7 @@ static void bnx2x_reset_port(struct bnx2x_softc *sc) } } -static void bnx2x_ilt_wr(struct bnx2x_softc *sc, uint32_t index, phys_addr_t addr) +static void bnx2x_ilt_wr(struct bnx2x_softc *sc, uint32_t index, rte_iova_t addr) { int reg; uint32_t wb_write[2]; @@ -11614,7 +11587,7 @@ static int ecore_gunzip(struct bnx2x_softc *sc, const uint8_t * zbuf, int len) } static void -ecore_write_dmae_phys_len(struct bnx2x_softc *sc, phys_addr_t phys_addr, +ecore_write_dmae_phys_len(struct bnx2x_softc *sc, rte_iova_t phys_addr, uint32_t addr, uint32_t len) { bnx2x_write_dmae_phys_len(sc, phys_addr, addr, len); diff --git a/dpdk/drivers/net/bnx2x/bnx2x.h b/dpdk/drivers/net/bnx2x/bnx2x.h index 78757a8d..17075d38 100644 --- a/dpdk/drivers/net/bnx2x/bnx2x.h +++ b/dpdk/drivers/net/bnx2x/bnx2x.h @@ -17,6 +17,9 @@ #define __BNX2X_H__ #include +#include +#include +#include #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN #ifndef __LITTLE_ENDIAN @@ -304,10 +307,7 @@ struct bnx2x_device_type { /* TCP with Timestamp Option (32) + IPv6 (40) */ /* max supported alignment is 256 (8 shift) */ -#define BNX2X_RX_ALIGN_SHIFT 8 -/* FW uses 2 cache lines alignment for start packet and size */ -#define BNX2X_FW_RX_ALIGN_START (1 << BNX2X_RX_ALIGN_SHIFT) -#define BNX2X_FW_RX_ALIGN_END (1 << BNX2X_RX_ALIGN_SHIFT) +#define BNX2X_RX_ALIGN_SHIFT RTE_MAX(6, min(8, RTE_CACHE_LINE_SIZE_LOG2)) #define BNX2X_PXP_DRAM_ALIGN (BNX2X_RX_ALIGN_SHIFT - 5) @@ -318,7 +318,7 @@ struct bnx2x_bar { /* Used to manage DMA allocations. */ struct bnx2x_dma { struct bnx2x_softc *sc; - phys_addr_t paddr; + rte_iova_t paddr; void *vaddr; int nseg; char msg[RTE_MEMZONE_NAMESIZE - 6]; @@ -371,10 +371,10 @@ struct bnx2x_fastpath { struct bnx2x_dma sb_dma; union bnx2x_host_hc_status_block status_block; - phys_addr_t tx_desc_mapping; + rte_iova_t tx_desc_mapping; - phys_addr_t rx_desc_mapping; - phys_addr_t rx_comp_mapping; + rte_iova_t rx_desc_mapping; + rte_iova_t rx_comp_mapping; uint16_t *sb_index_values; uint16_t *sb_running_index; @@ -469,7 +469,7 @@ union cdu_context { struct hw_context { struct bnx2x_dma vcxt_dma; union cdu_context *vcxt; - //phys_addr_t cxt_mapping; + //rte_iova_t cxt_mapping; size_t size; }; @@ -1031,12 +1031,13 @@ struct bnx2x_softc { struct bnx2x_mac_ops mac_ops; /* structures for VF mbox/response/bulletin */ - struct bnx2x_vf_mbx_msg *vf2pf_mbox; - struct bnx2x_dma vf2pf_mbox_mapping; - struct vf_acquire_resp_tlv acquire_resp; + struct bnx2x_vf_mbx_msg *vf2pf_mbox; + struct bnx2x_dma vf2pf_mbox_mapping; + struct vf_acquire_resp_tlv acquire_resp; struct bnx2x_vf_bulletin *pf2vf_bulletin; - struct bnx2x_dma pf2vf_bulletin_mapping; - struct bnx2x_vf_bulletin old_bulletin; + struct bnx2x_dma pf2vf_bulletin_mapping; + struct bnx2x_vf_bulletin old_bulletin; + rte_spinlock_t vf2pf_lock; int media; @@ -1147,11 +1148,12 @@ struct bnx2x_softc { #define BNX2X_RECOVERY_NIC_LOADING 5 uint32_t rx_mode; -#define BNX2X_RX_MODE_NONE 0 -#define BNX2X_RX_MODE_NORMAL 1 -#define BNX2X_RX_MODE_ALLMULTI 2 -#define BNX2X_RX_MODE_PROMISC 3 -#define BNX2X_MAX_MULTICAST 64 +#define BNX2X_RX_MODE_NONE 0 +#define BNX2X_RX_MODE_NORMAL 1 +#define BNX2X_RX_MODE_ALLMULTI 2 +#define BNX2X_RX_MODE_ALLMULTI_PROMISC 3 +#define BNX2X_RX_MODE_PROMISC 4 +#define BNX2X_MAX_MULTICAST 64 struct bnx2x_port port; @@ -1241,7 +1243,7 @@ struct bnx2x_softc { uint32_t gz_outlen; #define GUNZIP_BUF(sc) (sc->gz_buf) #define GUNZIP_OUTLEN(sc) (sc->gz_outlen) -#define GUNZIP_PHYS(sc) (phys_addr_t)(sc->gz_buf_dma.paddr) +#define GUNZIP_PHYS(sc) (rte_iova_t)(sc->gz_buf_dma.paddr) #define FW_BUF_SIZE 0x40000 struct raw_op *init_ops; @@ -1309,14 +1311,14 @@ struct bnx2x_softc { */ int fw_stats_req_size; struct bnx2x_fw_stats_req *fw_stats_req; - phys_addr_t fw_stats_req_mapping; + rte_iova_t fw_stats_req_mapping; /* * FW statistics data shortcut (points at the beginning of fw_stats * buffer + fw_stats_req_size). */ int fw_stats_data_size; struct bnx2x_fw_stats_data *fw_stats_data; - phys_addr_t fw_stats_data_mapping; + rte_iova_t fw_stats_data_mapping; /* tracking a pending STAT_QUERY ramrod */ uint16_t stats_pending; @@ -1401,8 +1403,8 @@ union bnx2x_stats_show_data { #define FUNC_FLG_LEADING 0x0020 /* PF only */ struct bnx2x_func_init_params { - phys_addr_t fw_stat_map; /* (dma) valid if FUNC_FLG_STATS */ - phys_addr_t spq_map; /* (dma) valid if FUNC_FLG_SPQ */ + rte_iova_t fw_stat_map; /* (dma) valid if FUNC_FLG_STATS */ + rte_iova_t spq_map; /* (dma) valid if FUNC_FLG_SPQ */ uint16_t func_flgs; uint16_t func_id; /* abs function id */ uint16_t pf_id; @@ -1415,33 +1417,89 @@ struct bnx2x_func_init_params { #define BAR1 2 #define BAR2 4 +static inline void +bnx2x_reg_write8(struct bnx2x_softc *sc, size_t offset, uint8_t val) +{ + PMD_DEBUG_PERIODIC_LOG(DEBUG, "offset=0x%08lx val=0x%02x", + (unsigned long)offset, val); + rte_write8(val, ((uint8_t *)sc->bar[BAR0].base_addr + offset)); +} + +static inline void +bnx2x_reg_write16(struct bnx2x_softc *sc, size_t offset, uint16_t val) +{ #ifdef RTE_LIBRTE_BNX2X_DEBUG_PERIODIC -uint8_t bnx2x_reg_read8(struct bnx2x_softc *sc, size_t offset); -uint16_t bnx2x_reg_read16(struct bnx2x_softc *sc, size_t offset); -uint32_t bnx2x_reg_read32(struct bnx2x_softc *sc, size_t offset); - -void bnx2x_reg_write8(struct bnx2x_softc *sc, size_t offset, uint8_t val); -void bnx2x_reg_write16(struct bnx2x_softc *sc, size_t offset, uint16_t val); -void bnx2x_reg_write32(struct bnx2x_softc *sc, size_t offset, uint32_t val); -#else -#define bnx2x_reg_write8(sc, offset, val)\ - *((volatile uint8_t*)((uintptr_t)sc->bar[BAR0].base_addr + offset)) = val - -#define bnx2x_reg_write16(sc, offset, val)\ - *((volatile uint16_t*)((uintptr_t)sc->bar[BAR0].base_addr + offset)) = val - -#define bnx2x_reg_write32(sc, offset, val)\ - *((volatile uint32_t*)((uintptr_t)sc->bar[BAR0].base_addr + offset)) = val - -#define bnx2x_reg_read8(sc, offset)\ - (*((volatile uint8_t*)((uintptr_t)sc->bar[BAR0].base_addr + offset))) - -#define bnx2x_reg_read16(sc, offset)\ - (*((volatile uint16_t*)((uintptr_t)sc->bar[BAR0].base_addr + offset))) - -#define bnx2x_reg_read32(sc, offset)\ - (*((volatile uint32_t*)((uintptr_t)sc->bar[BAR0].base_addr + offset))) + if ((offset % 2) != 0) + PMD_DRV_LOG(NOTICE, "Unaligned 16-bit write to 0x%08lx", + (unsigned long)offset); #endif + PMD_DEBUG_PERIODIC_LOG(DEBUG, "offset=0x%08lx val=0x%04x", + (unsigned long)offset, val); + rte_write16(val, ((uint8_t *)sc->bar[BAR0].base_addr + offset)); + +} + +static inline void +bnx2x_reg_write32(struct bnx2x_softc *sc, size_t offset, uint32_t val) +{ +#ifdef RTE_LIBRTE_BNX2X_DEBUG_PERIODIC + if ((offset % 4) != 0) + PMD_DRV_LOG(NOTICE, "Unaligned 32-bit write to 0x%08lx", + (unsigned long)offset); +#endif + + PMD_DEBUG_PERIODIC_LOG(DEBUG, "offset=0x%08lx val=0x%08x", + (unsigned long)offset, val); + rte_write32(val, ((uint8_t *)sc->bar[BAR0].base_addr + offset)); +} + +static inline uint8_t +bnx2x_reg_read8(struct bnx2x_softc *sc, size_t offset) +{ + uint8_t val; + + val = rte_read8((uint8_t *)sc->bar[BAR0].base_addr + offset); + PMD_DEBUG_PERIODIC_LOG(DEBUG, "offset=0x%08lx val=0x%02x", + (unsigned long)offset, val); + + return val; +} + +static inline uint16_t +bnx2x_reg_read16(struct bnx2x_softc *sc, size_t offset) +{ + uint16_t val; + +#ifdef RTE_LIBRTE_BNX2X_DEBUG_PERIODIC + if ((offset % 2) != 0) + PMD_DRV_LOG(NOTICE, "Unaligned 16-bit read from 0x%08lx", + (unsigned long)offset); +#endif + + val = rte_read16(((uint8_t *)sc->bar[BAR0].base_addr + offset)); + PMD_DEBUG_PERIODIC_LOG(DEBUG, "offset=0x%08lx val=0x%08x", + (unsigned long)offset, val); + + return val; +} + +static inline uint32_t +bnx2x_reg_read32(struct bnx2x_softc *sc, size_t offset) +{ + uint32_t val; + +#ifdef RTE_LIBRTE_BNX2X_DEBUG_PERIODIC + if ((offset % 4) != 0) + PMD_DRV_LOG(NOTICE, "Unaligned 32-bit read from 0x%08lx", + (unsigned long)offset); +#endif + + val = rte_read32(((uint8_t *)sc->bar[BAR0].base_addr + offset)); + PMD_DEBUG_PERIODIC_LOG(DEBUG, "offset=0x%08lx val=0x%08x", + (unsigned long)offset, val); + + return val; +} #define REG_ADDR(sc, offset) (((uint64_t)sc->bar[BAR0].base_addr) + (offset)) @@ -1468,12 +1526,12 @@ void bnx2x_reg_write32(struct bnx2x_softc *sc, size_t offset, uint32_t val); #define REG_RD_DMAE(sc, offset, valp, len32) \ do { \ (void)bnx2x_read_dmae(sc, offset, len32); \ - (void)rte_memcpy(valp, BNX2X_SP(sc, wb_data[0]), (len32) * 4); \ + rte_memcpy(valp, BNX2X_SP(sc, wb_data[0]), (len32) * 4); \ } while (0) #define REG_WR_DMAE(sc, offset, valp, len32) \ do { \ - (void)rte_memcpy(BNX2X_SP(sc, wb_data[0]), valp, (len32) * 4); \ + rte_memcpy(BNX2X_SP(sc, wb_data[0]), valp, (len32) * 4); \ (void)bnx2x_write_dmae(sc, BNX2X_SP_MAPPING(sc, wb_data), offset, len32); \ } while (0) @@ -1500,11 +1558,9 @@ void bnx2x_reg_write32(struct bnx2x_softc *sc, size_t offset, uint32_t val); #define DPM_TRIGGER_TYPE 0x40 /* Doorbell macro */ -#define BNX2X_DB_WRITE(db_bar, val) \ - *((volatile uint32_t *)(db_bar)) = (val) +#define BNX2X_DB_WRITE(db_bar, val) rte_write32_relaxed((val), (db_bar)) -#define BNX2X_DB_READ(db_bar) \ - *((volatile uint32_t *)(db_bar)) +#define BNX2X_DB_READ(db_bar) rte_read32_relaxed(db_bar) #define DOORBELL_ADDR(sc, offset) \ (volatile uint32_t *)(((char *)(sc)->bar[BAR1].base_addr + (offset))) @@ -1693,7 +1749,7 @@ uint32_t bnx2x_dmae_opcode(struct bnx2x_softc *sc, uint8_t src_type, uint8_t comp_type); void bnx2x_post_dmae(struct bnx2x_softc *sc, struct dmae_command *dmae, int idx); void bnx2x_read_dmae(struct bnx2x_softc *sc, uint32_t src_addr, uint32_t len32); -void bnx2x_write_dmae(struct bnx2x_softc *sc, phys_addr_t dma_addr, +void bnx2x_write_dmae(struct bnx2x_softc *sc, rte_iova_t dma_addr, uint32_t dst_addr, uint32_t len32); void bnx2x_set_ctx_validation(struct bnx2x_softc *sc, struct eth_context *cxt, uint32_t cid); @@ -1883,8 +1939,6 @@ int bnx2x_vf_setup_queue(struct bnx2x_softc *sc, struct bnx2x_fastpath *fp, int leading); void bnx2x_free_hsi_mem(struct bnx2x_softc *sc); int bnx2x_vf_set_rx_mode(struct bnx2x_softc *sc); -int bnx2x_fill_accept_flags(struct bnx2x_softc *sc, uint32_t rx_mode, - unsigned long *rx_accept_flags, unsigned long *tx_accept_flags); int bnx2x_check_bull(struct bnx2x_softc *sc); //#define BNX2X_PULSE @@ -1924,7 +1978,7 @@ bnx2x_set_rx_mode(struct bnx2x_softc *sc) static inline int pci_read(struct bnx2x_softc *sc, size_t addr, void *val, uint8_t size) { - if (rte_eal_pci_read_config(sc->pci_dev, val, size, addr) <= 0) { + if (rte_pci_read_config(sc->pci_dev, val, size, addr) <= 0) { PMD_DRV_LOG(ERR, "Can't read from PCI config space"); return ENXIO; } @@ -1936,7 +1990,7 @@ static inline int pci_write_word(struct bnx2x_softc *sc, size_t addr, off_t val) { uint16_t val16 = val; - if (rte_eal_pci_write_config(sc->pci_dev, &val16, + if (rte_pci_write_config(sc->pci_dev, &val16, sizeof(val16), addr) <= 0) { PMD_DRV_LOG(ERR, "Can't write to PCI config space"); return ENXIO; @@ -1948,7 +2002,7 @@ static inline int pci_write_word(struct bnx2x_softc *sc, size_t addr, off_t val) static inline int pci_write_long(struct bnx2x_softc *sc, size_t addr, off_t val) { uint32_t val32 = val; - if (rte_eal_pci_write_config(sc->pci_dev, &val32, + if (rte_pci_write_config(sc->pci_dev, &val32, sizeof(val32), addr) <= 0) { PMD_DRV_LOG(ERR, "Can't write to PCI config space"); return ENXIO; diff --git a/dpdk/drivers/net/bnx2x/bnx2x_ethdev.c b/dpdk/drivers/net/bnx2x/bnx2x_ethdev.c index f3ab3550..95861a06 100644 --- a/dpdk/drivers/net/bnx2x/bnx2x_ethdev.c +++ b/dpdk/drivers/net/bnx2x/bnx2x_ethdev.c @@ -12,12 +12,13 @@ #include "bnx2x_rxtx.h" #include +#include /* * The set of PCI devices this driver supports */ #define BROADCOM_PCI_VENDOR_ID 0x14E4 -static struct rte_pci_id pci_id_bnx2x_map[] = { +static const struct rte_pci_id pci_id_bnx2x_map[] = { { RTE_PCI_DEVICE(BROADCOM_PCI_VENDOR_ID, CHIP_NUM_57800) }, { RTE_PCI_DEVICE(BROADCOM_PCI_VENDOR_ID, CHIP_NUM_57711) }, { RTE_PCI_DEVICE(BROADCOM_PCI_VENDOR_ID, CHIP_NUM_57810) }, @@ -33,7 +34,7 @@ static struct rte_pci_id pci_id_bnx2x_map[] = { { .vendor_id = 0, } }; -static struct rte_pci_id pci_id_bnx2xvf_map[] = { +static const struct rte_pci_id pci_id_bnx2xvf_map[] = { { RTE_PCI_DEVICE(BROADCOM_PCI_VENDOR_ID, CHIP_NUM_57800_VF) }, { RTE_PCI_DEVICE(BROADCOM_PCI_VENDOR_ID, CHIP_NUM_57810_VF) }, { RTE_PCI_DEVICE(BROADCOM_PCI_VENDOR_ID, CHIP_NUM_57811_VF) }, @@ -118,13 +119,14 @@ bnx2x_interrupt_action(struct rte_eth_dev *dev) bnx2x_link_update(dev); } -static __rte_unused void -bnx2x_interrupt_handler(__rte_unused struct rte_intr_handle *handle, void *param) +static void +bnx2x_interrupt_handler(void *param) { struct rte_eth_dev *dev = (struct rte_eth_dev *)param; + struct bnx2x_softc *sc = dev->data->dev_private; bnx2x_interrupt_action(dev); - rte_intr_enable(&(dev->pci_dev->intr_handle)); + rte_intr_enable(&sc->pci_dev->intr_handle); } /* @@ -187,10 +189,10 @@ bnx2x_dev_start(struct rte_eth_dev *dev) } if (IS_PF(sc)) { - rte_intr_callback_register(&(dev->pci_dev->intr_handle), + rte_intr_callback_register(&sc->pci_dev->intr_handle, bnx2x_interrupt_handler, (void *)dev); - if(rte_intr_enable(&(dev->pci_dev->intr_handle))) + if (rte_intr_enable(&sc->pci_dev->intr_handle)) PMD_DRV_LOG(ERR, "rte_intr_enable failed"); } @@ -203,8 +205,6 @@ bnx2x_dev_start(struct rte_eth_dev *dev) /* Print important adapter info for the user. */ bnx2x_print_adapter_info(sc); - DELAY_MS(2500); - return ret; } @@ -217,8 +217,8 @@ bnx2x_dev_stop(struct rte_eth_dev *dev) PMD_INIT_FUNC_TRACE(); if (IS_PF(sc)) { - rte_intr_disable(&(dev->pci_dev->intr_handle)); - rte_intr_callback_unregister(&(dev->pci_dev->intr_handle), + rte_intr_disable(&sc->pci_dev->intr_handle); + rte_intr_callback_unregister(&sc->pci_dev->intr_handle, bnx2x_interrupt_handler, (void *)dev); } @@ -258,6 +258,8 @@ bnx2x_promisc_enable(struct rte_eth_dev *dev) PMD_INIT_FUNC_TRACE(); sc->rx_mode = BNX2X_RX_MODE_PROMISC; + if (rte_eth_allmulticast_get(dev->data->port_id) == 1) + sc->rx_mode = BNX2X_RX_MODE_ALLMULTI_PROMISC; bnx2x_set_rx_mode(sc); } @@ -268,6 +270,8 @@ bnx2x_promisc_disable(struct rte_eth_dev *dev) PMD_INIT_FUNC_TRACE(); sc->rx_mode = BNX2X_RX_MODE_NORMAL; + if (rte_eth_allmulticast_get(dev->data->port_id) == 1) + sc->rx_mode = BNX2X_RX_MODE_ALLMULTI; bnx2x_set_rx_mode(sc); } @@ -278,6 +282,8 @@ bnx2x_dev_allmulticast_enable(struct rte_eth_dev *dev) PMD_INIT_FUNC_TRACE(); sc->rx_mode = BNX2X_RX_MODE_ALLMULTI; + if (rte_eth_promiscuous_get(dev->data->port_id) == 1) + sc->rx_mode = BNX2X_RX_MODE_ALLMULTI_PROMISC; bnx2x_set_rx_mode(sc); } @@ -288,6 +294,8 @@ bnx2x_dev_allmulticast_disable(struct rte_eth_dev *dev) PMD_INIT_FUNC_TRACE(); sc->rx_mode = BNX2X_RX_MODE_NORMAL; + if (rte_eth_promiscuous_get(dev->data->port_id) == 1) + sc->rx_mode = BNX2X_RX_MODE_PROMISC; bnx2x_set_rx_mode(sc); } @@ -321,7 +329,7 @@ bnx2xvf_dev_link_update(struct rte_eth_dev *dev, __rte_unused int wait_to_comple return old_link_status == dev->data->dev_link.link_status ? -1 : 0; } -static void +static int bnx2x_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) { struct bnx2x_softc *sc = dev->data->dev_private; @@ -381,6 +389,8 @@ bnx2x_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) stats->imissed = brb_drops + brb_truncates + brb_truncate_discard + stats->rx_nombuf; + + return 0; } static int @@ -424,15 +434,17 @@ bnx2x_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, xstats[num].value = *(uint64_t *)((char *)&sc->eth_stats + bnx2x_xstats_strings[num].offset_lo); + xstats[num].id = num; } return num; } static void -bnx2x_dev_infos_get(struct rte_eth_dev *dev, __rte_unused struct rte_eth_dev_info *dev_info) +bnx2x_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) { struct bnx2x_softc *sc = dev->data->dev_private; + dev_info->pci_dev = RTE_ETH_DEV_TO_PCI(dev); dev_info->max_rx_queues = sc->max_rx_queues; dev_info->max_tx_queues = sc->max_tx_queues; dev_info->min_rx_bufsize = BNX2X_MIN_RX_BUF_SIZE; @@ -441,14 +453,17 @@ bnx2x_dev_infos_get(struct rte_eth_dev *dev, __rte_unused struct rte_eth_dev_inf dev_info->speed_capa = ETH_LINK_SPEED_10G | ETH_LINK_SPEED_20G; } -static void +static int bnx2x_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac_addr, uint32_t index, uint32_t pool) { struct bnx2x_softc *sc = dev->data->dev_private; - if (sc->mac_ops.mac_addr_add) + if (sc->mac_ops.mac_addr_add) { sc->mac_ops.mac_addr_add(dev, mac_addr, index, pool); + return 0; + } + return -ENOTSUP; } static void @@ -518,7 +533,7 @@ bnx2x_common_dev_init(struct rte_eth_dev *eth_dev, int is_vf) PMD_INIT_FUNC_TRACE(); eth_dev->dev_ops = is_vf ? &bnx2xvf_eth_dev_ops : &bnx2x_eth_dev_ops; - pci_dev = eth_dev->pci_dev; + pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); rte_eth_copy_pci_info(eth_dev, pci_dev); @@ -577,6 +592,8 @@ bnx2x_common_dev_init(struct rte_eth_dev *eth_dev, int is_vf) eth_dev->data->port_id, pci_dev->id.vendor_id, pci_dev->id.device_id); if (IS_VF(sc)) { + rte_spinlock_init(&sc->vf2pf_lock); + if (bnx2x_dma_alloc(sc, sizeof(struct bnx2x_vf_mbx_msg), &sc->vf2pf_mbox_mapping, "vf2pf_mbox", RTE_CACHE_LINE_SIZE) != 0) @@ -616,56 +633,57 @@ eth_bnx2xvf_dev_init(struct rte_eth_dev *eth_dev) return bnx2x_common_dev_init(eth_dev, 1); } -static struct eth_driver rte_bnx2x_pmd = { - .pci_drv = { - .name = "rte_bnx2x_pmd", - .id_table = pci_id_bnx2x_map, - .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC, - }, - .eth_dev_init = eth_bnx2x_dev_init, - .dev_private_size = sizeof(struct bnx2x_softc), +static struct rte_pci_driver rte_bnx2x_pmd; +static struct rte_pci_driver rte_bnx2xvf_pmd; + +static int eth_bnx2x_pci_probe(struct rte_pci_driver *pci_drv, + struct rte_pci_device *pci_dev) +{ + struct rte_eth_dev *eth_dev; + int ret; + + eth_dev = rte_eth_dev_pci_allocate(pci_dev, sizeof(struct bnx2x_softc)); + if (!eth_dev) + return -ENOMEM; + + if (pci_drv == &rte_bnx2x_pmd) + ret = eth_bnx2x_dev_init(eth_dev); + else if (pci_drv == &rte_bnx2xvf_pmd) + ret = eth_bnx2xvf_dev_init(eth_dev); + else + ret = -EINVAL; + + if (ret) + rte_eth_dev_pci_release(eth_dev); + + return ret; +} + +static int eth_bnx2x_pci_remove(struct rte_pci_device *pci_dev) +{ + return rte_eth_dev_pci_generic_remove(pci_dev, NULL); +} + +static struct rte_pci_driver rte_bnx2x_pmd = { + .id_table = pci_id_bnx2x_map, + .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC, + .probe = eth_bnx2x_pci_probe, + .remove = eth_bnx2x_pci_remove, }; /* * virtual function driver struct */ -static struct eth_driver rte_bnx2xvf_pmd = { - .pci_drv = { - .name = "rte_bnx2xvf_pmd", - .id_table = pci_id_bnx2xvf_map, - .drv_flags = RTE_PCI_DRV_NEED_MAPPING, - }, - .eth_dev_init = eth_bnx2xvf_dev_init, - .dev_private_size = sizeof(struct bnx2x_softc), +static struct rte_pci_driver rte_bnx2xvf_pmd = { + .id_table = pci_id_bnx2xvf_map, + .drv_flags = RTE_PCI_DRV_NEED_MAPPING, + .probe = eth_bnx2x_pci_probe, + .remove = eth_bnx2x_pci_remove, }; -static int rte_bnx2x_pmd_init(const char *name __rte_unused, const char *params __rte_unused) -{ - PMD_INIT_FUNC_TRACE(); - rte_eth_driver_register(&rte_bnx2x_pmd); - - return 0; -} - -static int rte_bnx2xvf_pmd_init(const char *name __rte_unused, const char *params __rte_unused) -{ - PMD_INIT_FUNC_TRACE(); - rte_eth_driver_register(&rte_bnx2xvf_pmd); - - return 0; -} - -static struct rte_driver rte_bnx2x_driver = { - .type = PMD_PDEV, - .init = rte_bnx2x_pmd_init, -}; - -static struct rte_driver rte_bnx2xvf_driver = { - .type = PMD_PDEV, - .init = rte_bnx2xvf_pmd_init, -}; - -PMD_REGISTER_DRIVER(rte_bnx2x_driver, bnx2x); -DRIVER_REGISTER_PCI_TABLE(bnx2x, pci_id_bnx2x_map); -PMD_REGISTER_DRIVER(rte_bnx2xvf_driver, bnx2xvf); -DRIVER_REGISTER_PCI_TABLE(bnx2xvf, pci_id_bnx2xvf_map); +RTE_PMD_REGISTER_PCI(net_bnx2x, rte_bnx2x_pmd); +RTE_PMD_REGISTER_PCI_TABLE(net_bnx2x, pci_id_bnx2x_map); +RTE_PMD_REGISTER_KMOD_DEP(net_bnx2x, "* igb_uio | uio_pci_generic | vfio-pci"); +RTE_PMD_REGISTER_PCI(net_bnx2xvf, rte_bnx2xvf_pmd); +RTE_PMD_REGISTER_PCI_TABLE(net_bnx2xvf, pci_id_bnx2xvf_map); +RTE_PMD_REGISTER_KMOD_DEP(net_bnx2xvf, "* igb_uio | vfio-pci"); diff --git a/dpdk/drivers/net/bnx2x/bnx2x_ethdev.h b/dpdk/drivers/net/bnx2x/bnx2x_ethdev.h index a9da9de8..967d6dc5 100644 --- a/dpdk/drivers/net/bnx2x/bnx2x_ethdev.h +++ b/dpdk/drivers/net/bnx2x/bnx2x_ethdev.h @@ -35,7 +35,6 @@ #include #include #include -#include #include #include #include diff --git a/dpdk/drivers/net/bnx2x/bnx2x_rxtx.c b/dpdk/drivers/net/bnx2x/bnx2x_rxtx.c index 0ec4f899..a0d4ac92 100644 --- a/dpdk/drivers/net/bnx2x/bnx2x_rxtx.c +++ b/dpdk/drivers/net/bnx2x/bnx2x_rxtx.c @@ -19,7 +19,8 @@ ring_dma_zone_reserve(struct rte_eth_dev *dev, const char *ring_name, const struct rte_memzone *mz; snprintf(z_name, sizeof(z_name), "%s_%s_%d_%d", - dev->driver->pci_drv.name, ring_name, dev->data->port_id, queue_id); + dev->device->driver->name, ring_name, + dev->data->port_id, queue_id); mz = rte_memzone_lookup(z_name); if (mz) @@ -59,7 +60,7 @@ bnx2x_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, uint16_t nb_desc, unsigned int socket_id, - const struct rte_eth_rxconf *rx_conf, + __rte_unused const struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp) { uint16_t j, idx; @@ -70,8 +71,8 @@ bnx2x_dev_rx_queue_setup(struct rte_eth_dev *dev, struct bnx2x_softc *sc = dev->data->dev_private; struct bnx2x_fastpath *fp = &sc->fp[queue_idx]; struct eth_rx_cqe_next_page *nextpg; - phys_addr_t *rx_bd; - phys_addr_t busaddr; + rte_iova_t *rx_bd; + rte_iova_t busaddr; /* First allocate the rx queue data structure */ rxq = rte_zmalloc_socket("ethdev RX queue", sizeof(struct bnx2x_rx_queue), @@ -84,7 +85,6 @@ bnx2x_dev_rx_queue_setup(struct rte_eth_dev *dev, rxq->mb_pool = mp; rxq->queue_id = queue_idx; rxq->port_id = dev->data->port_id; - rxq->crc_len = (uint8_t)((dev->data->dev_conf.rxmode.hw_strip_crc) ? 0 : ETHER_CRC_LEN); rxq->nb_rx_pages = 1; while (USABLE_RX_BD(rxq) < nb_desc) @@ -94,13 +94,9 @@ bnx2x_dev_rx_queue_setup(struct rte_eth_dev *dev, sc->rx_ring_size = USABLE_RX_BD(rxq); rxq->nb_cq_pages = RCQ_BD_PAGES(rxq); - rxq->rx_free_thresh = rx_conf->rx_free_thresh ? - rx_conf->rx_free_thresh : DEFAULT_RX_FREE_THRESH; - - PMD_INIT_LOG(DEBUG, "fp[%02d] req_bd=%u, thresh=%u, usable_bd=%lu, " + PMD_INIT_LOG(DEBUG, "fp[%02d] req_bd=%u, usable_bd=%lu, " "total_bd=%lu, rx_pages=%u, cq_pages=%u", - queue_idx, nb_desc, rxq->rx_free_thresh, - (unsigned long)USABLE_RX_BD(rxq), + queue_idx, nb_desc, (unsigned long)USABLE_RX_BD(rxq), (unsigned long)TOTAL_RX_BD(rxq), rxq->nb_rx_pages, rxq->nb_cq_pages); @@ -112,7 +108,7 @@ bnx2x_dev_rx_queue_setup(struct rte_eth_dev *dev, bnx2x_rx_queue_release(rxq); return -ENOMEM; } - fp->rx_desc_mapping = rxq->rx_ring_phys_addr = (uint64_t)dma->phys_addr; + fp->rx_desc_mapping = rxq->rx_ring_phys_addr = (uint64_t)dma->iova; rxq->rx_ring = (uint64_t*)dma->addr; memset((void *)rxq->rx_ring, 0, dma_size); @@ -135,7 +131,6 @@ bnx2x_dev_rx_queue_setup(struct rte_eth_dev *dev, } /* Initialize software ring entries */ - rxq->rx_mbuf_alloc = 0; for (idx = 0; idx < rxq->nb_rx_desc; idx = NEXT_RX_BD(idx)) { mbuf = rte_mbuf_raw_alloc(mp); if (NULL == mbuf) { @@ -145,8 +140,7 @@ bnx2x_dev_rx_queue_setup(struct rte_eth_dev *dev, return -ENOMEM; } rxq->sw_ring[idx] = mbuf; - rxq->rx_ring[idx] = mbuf->buf_physaddr; - rxq->rx_mbuf_alloc++; + rxq->rx_ring[idx] = mbuf->buf_iova; } rxq->pkt_first_seg = NULL; rxq->pkt_last_seg = NULL; @@ -160,7 +154,7 @@ bnx2x_dev_rx_queue_setup(struct rte_eth_dev *dev, PMD_RX_LOG(ERR, "RCQ alloc failed"); return -ENOMEM; } - fp->rx_comp_mapping = rxq->cq_ring_phys_addr = (uint64_t)dma->phys_addr; + fp->rx_comp_mapping = rxq->cq_ring_phys_addr = (uint64_t)dma->iova; rxq->cq_ring = (union eth_rx_cqe*)dma->addr; /* Link the CQ chain pages. */ @@ -279,6 +273,8 @@ bnx2x_dev_tx_queue_setup(struct rte_eth_dev *dev, txq->tx_free_thresh = tx_conf->tx_free_thresh ? tx_conf->tx_free_thresh : DEFAULT_TX_FREE_THRESH; + txq->tx_free_thresh = min(txq->tx_free_thresh, + txq->nb_tx_desc - BDS_PER_TX_PKT); PMD_INIT_LOG(DEBUG, "fp[%02d] req_bd=%u, thresh=%u, usable_bd=%lu, " "total_bd=%lu, tx_pages=%u", @@ -293,7 +289,7 @@ bnx2x_dev_tx_queue_setup(struct rte_eth_dev *dev, bnx2x_tx_queue_release(txq); return -ENOMEM; } - fp->tx_desc_mapping = txq->tx_ring_phys_addr = (uint64_t)tz->phys_addr; + fp->tx_desc_mapping = txq->tx_ring_phys_addr = (uint64_t)tz->iova; txq->tx_ring = (union eth_tx_bd_types *) tz->addr; memset(txq->tx_ring, 0, tsize); @@ -404,7 +400,7 @@ bnx2x_recv_pkts(void *p_rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) rx_mb = rxq->sw_ring[bd_cons]; rxq->sw_ring[bd_cons] = new_mb; - rxq->rx_ring[bd_prod] = new_mb->buf_physaddr; + rxq->rx_ring[bd_prod] = new_mb->buf_iova; rx_pref = NEXT_RX_BD(bd_cons) & MAX_RX_BD(rxq); rte_prefetch0(rxq->sw_ring[rx_pref]); @@ -426,7 +422,7 @@ bnx2x_recv_pkts(void *p_rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) */ if (cqe_fp->pars_flags.flags & PARSING_FLAGS_VLAN) { rx_mb->vlan_tci = cqe_fp->vlan_tag; - rx_mb->ol_flags |= PKT_RX_VLAN_PKT; + rx_mb->ol_flags |= PKT_RX_VLAN; } rx_pkts[nb_rx] = rx_mb; diff --git a/dpdk/drivers/net/bnx2x/bnx2x_rxtx.h b/dpdk/drivers/net/bnx2x/bnx2x_rxtx.h index ccb22fc1..9600e0f1 100644 --- a/dpdk/drivers/net/bnx2x/bnx2x_rxtx.h +++ b/dpdk/drivers/net/bnx2x/bnx2x_rxtx.h @@ -11,9 +11,7 @@ #ifndef _BNX2X_RXTX_H_ #define _BNX2X_RXTX_H_ - -#define DEFAULT_RX_FREE_THRESH 0 -#define DEFAULT_TX_FREE_THRESH 512 +#define DEFAULT_TX_FREE_THRESH 64 #define RTE_PMD_BNX2X_TX_MAX_BURST 1 /** @@ -42,13 +40,9 @@ struct bnx2x_rx_queue { uint16_t rx_bd_tail; /**< Index of last rx bd. */ uint16_t rx_cq_head; /**< Index of current rcq bd. */ uint16_t rx_cq_tail; /**< Index of last rcq bd. */ - uint16_t nb_rx_hold; /**< number of held free RX desc. */ - uint16_t rx_free_thresh; /**< max free RX desc to hold. */ uint16_t queue_id; /**< RX queue index. */ - uint8_t port_id; /**< Device port identifier. */ - uint8_t crc_len; /**< 0 if CRC stripped, 4 otherwise. */ + uint16_t port_id; /**< Device port identifier. */ struct bnx2x_softc *sc; /**< Ptr to dev_private data. */ - uint64_t rx_mbuf_alloc; /**< Number of allocated mbufs. */ }; /** @@ -68,7 +62,7 @@ struct bnx2x_tx_queue { uint16_t nb_tx_avail; /**< Number of TX descriptors available. */ uint16_t nb_tx_pages; /**< number of TX pages */ uint16_t queue_id; /**< TX queue index. */ - uint8_t port_id; /**< Device port identifier. */ + uint16_t port_id; /**< Device port identifier. */ struct bnx2x_softc *sc; /**< Ptr to dev_private data */ }; diff --git a/dpdk/drivers/net/bnx2x/bnx2x_stats.c b/dpdk/drivers/net/bnx2x/bnx2x_stats.c index c489cbee..b9b85963 100644 --- a/dpdk/drivers/net/bnx2x/bnx2x_stats.c +++ b/dpdk/drivers/net/bnx2x/bnx2x_stats.c @@ -1338,7 +1338,7 @@ bnx2x_prep_fw_stats_req(struct bnx2x_softc *sc) int i; int first_queue_query_index; struct stats_query_header *stats_hdr = &sc->fw_stats_req->hdr; - phys_addr_t cur_data_offset; + rte_iova_t cur_data_offset; struct stats_query_entry *cur_query_entry; stats_hdr->cmd_num = sc->fw_stats_num; @@ -1371,9 +1371,9 @@ bnx2x_prep_fw_stats_req(struct bnx2x_softc *sc) cur_query_entry = &sc->fw_stats_req->query[BNX2X_PORT_QUERY_IDX]; cur_query_entry->kind = STATS_TYPE_PORT; - /* For port query index is a DONT CARE */ + /* For port query index is a DON'T CARE */ cur_query_entry->index = SC_PORT(sc); - /* For port query funcID is a DONT CARE */ + /* For port query funcID is a DON'T CARE */ cur_query_entry->funcID = htole16(SC_FUNC(sc)); cur_query_entry->address.hi = htole32(U64_HI(cur_data_offset)); cur_query_entry->address.lo = htole32(U64_LO(cur_data_offset)); @@ -1385,7 +1385,7 @@ bnx2x_prep_fw_stats_req(struct bnx2x_softc *sc) cur_query_entry = &sc->fw_stats_req->query[BNX2X_PF_QUERY_IDX]; cur_query_entry->kind = STATS_TYPE_PF; - /* For PF query index is a DONT CARE */ + /* For PF query index is a DON'T CARE */ cur_query_entry->index = SC_PORT(sc); cur_query_entry->funcID = htole16(SC_FUNC(sc)); cur_query_entry->address.hi = htole32(U64_HI(cur_data_offset)); @@ -1450,7 +1450,7 @@ void bnx2x_memset_stats(struct bnx2x_softc *sc) if (sc->port.pmf && sc->port.port_stx) bnx2x_port_stats_base_init(sc); - /* mark the end of statistics initializiation */ + /* mark the end of statistics initialization */ sc->stats_init = false; } @@ -1536,7 +1536,7 @@ bnx2x_stats_init(struct bnx2x_softc *sc) bnx2x_port_stats_base_init(sc); } - /* mark the end of statistics initializiation */ + /* mark the end of statistics initialization */ sc->stats_init = FALSE; } diff --git a/dpdk/drivers/net/bnx2x/bnx2x_vfpf.c b/dpdk/drivers/net/bnx2x/bnx2x_vfpf.c index 1c895f88..3c08f2a2 100644 --- a/dpdk/drivers/net/bnx2x/bnx2x_vfpf.c +++ b/dpdk/drivers/net/bnx2x/bnx2x_vfpf.c @@ -64,25 +64,46 @@ bnx2x_check_bull(struct bnx2x_softc *sc) return TRUE; } -/* add tlv to a buffer */ -#define BNX2X_TLV_APPEND(_tlvs, _offset, _type, _length) \ - ((struct vf_first_tlv *)((unsigned long)_tlvs + _offset))->type = _type; \ - ((struct vf_first_tlv *)((unsigned long)_tlvs + _offset))->length = _length +/* place a given tlv on the tlv buffer at a given offset */ +static void +bnx2x_add_tlv(__rte_unused struct bnx2x_softc *sc, void *tlvs_list, + uint16_t offset, uint16_t type, uint16_t length) +{ + struct channel_tlv *tl = (struct channel_tlv *) + ((unsigned long)tlvs_list + offset); + + tl->type = type; + tl->length = length; +} /* Initiliaze header of the first tlv and clear mailbox*/ static void -bnx2x_init_first_tlv(struct bnx2x_softc *sc, struct vf_first_tlv *tlv, - uint16_t type, uint16_t len) +bnx2x_vf_prep(struct bnx2x_softc *sc, struct vf_first_tlv *first_tlv, + uint16_t type, uint16_t length) { struct bnx2x_vf_mbx_msg *mbox = sc->vf2pf_mbox; + + rte_spinlock_lock(&sc->vf2pf_lock); + PMD_DRV_LOG(DEBUG, "Preparing %d tlv for sending", type); memset(mbox, 0, sizeof(struct bnx2x_vf_mbx_msg)); - BNX2X_TLV_APPEND(tlv, 0, type, len); + bnx2x_add_tlv(sc, &first_tlv->tl, 0, type, length); /* Initialize header of the first tlv */ - tlv->reply_offset = sizeof(mbox->query); + first_tlv->reply_offset = sizeof(mbox->query); +} + +/* releases the mailbox */ +static void +bnx2x_vf_finalize(struct bnx2x_softc *sc, + __rte_unused struct vf_first_tlv *first_tlv) +{ + PMD_DRV_LOG(DEBUG, "done sending [%d] tlv over vf pf channel", + first_tlv->tl.type); + + rte_spinlock_unlock(&sc->vf2pf_lock); } #define BNX2X_VF_CMD_ADDR_LO PXP_VF_ADDR_CSDM_GLOBAL_START @@ -92,46 +113,43 @@ bnx2x_init_first_tlv(struct bnx2x_softc *sc, struct vf_first_tlv *tlv, #define BNX2X_VF_CHANNEL_TRIES 100 static int -bnx2x_do_req4pf(struct bnx2x_softc *sc, phys_addr_t phys_addr) +bnx2x_do_req4pf(struct bnx2x_softc *sc, rte_iova_t phys_addr) { uint8_t *status = &sc->vf2pf_mbox->resp.common_reply.status; uint8_t i; - if (!*status) { - bnx2x_check_bull(sc); - if (sc->old_bulletin.valid_bitmap & (1 << CHANNEL_DOWN)) { - PMD_DRV_LOG(ERR, "channel is down. Aborting message sending"); - *status = BNX2X_VF_STATUS_SUCCESS; - return 0; - } - - REG_WR(sc, BNX2X_VF_CMD_ADDR_LO, U64_LO(phys_addr)); - REG_WR(sc, BNX2X_VF_CMD_ADDR_HI, U64_HI(phys_addr)); - - /* memory barrier to ensure that FW can read phys_addr */ - wmb(); - - REG_WR8(sc, BNX2X_VF_CMD_TRIGGER, 1); - - /* Do several attempts until PF completes - * "." is used to show progress - */ - for (i = 0; i < BNX2X_VF_CHANNEL_TRIES; i++) { - DELAY_MS(BNX2X_VF_CHANNEL_DELAY); - if (*status) - break; - } - - if (!*status) { - PMD_DRV_LOG(ERR, "Response from PF timed out"); - return -EAGAIN; - } - } else { + if (*status) { PMD_DRV_LOG(ERR, "status should be zero before message" - "to pf was sent"); + " to pf was sent"); return -EINVAL; } + bnx2x_check_bull(sc); + if (sc->old_bulletin.valid_bitmap & (1 << CHANNEL_DOWN)) { + PMD_DRV_LOG(ERR, "channel is down. Aborting message sending"); + return -EINVAL; + } + + REG_WR(sc, BNX2X_VF_CMD_ADDR_LO, U64_LO(phys_addr)); + REG_WR(sc, BNX2X_VF_CMD_ADDR_HI, U64_HI(phys_addr)); + + /* memory barrier to ensure that FW can read phys_addr */ + wmb(); + + REG_WR8(sc, BNX2X_VF_CMD_TRIGGER, 1); + + /* Do several attempts until PF completes */ + for (i = 0; i < BNX2X_VF_CHANNEL_TRIES; i++) { + DELAY_MS(BNX2X_VF_CHANNEL_DELAY); + if (*status) + break; + } + + if (!*status) { + PMD_DRV_LOG(ERR, "Response from PF timed out"); + return -EAGAIN; + } + PMD_DRV_LOG(DEBUG, "Response from PF was received"); return 0; } @@ -168,31 +186,23 @@ static inline int bnx2x_read_vf_id(struct bnx2x_softc *sc) #define BNX2X_VF_OBTAIN_MAC_FILTERS 1 #define BNX2X_VF_OBTAIN_MC_FILTERS 10 -struct bnx2x_obtain_status { - int success; - int err_code; -}; - static -struct bnx2x_obtain_status bnx2x_loop_obtain_resources(struct bnx2x_softc *sc) +int bnx2x_loop_obtain_resources(struct bnx2x_softc *sc) { - int tries = 0; struct vf_acquire_resp_tlv *resp = &sc->vf2pf_mbox->resp.acquire_resp, - *sc_resp = &sc->acquire_resp; - struct vf_resource_query *res_query; - struct vf_resc *resc; - struct bnx2x_obtain_status status; + *sc_resp = &sc->acquire_resp; + struct vf_resource_query *res_query; + struct vf_resc *resc; int res_obtained = false; + int tries = 0; + int rc; do { PMD_DRV_LOG(DEBUG, "trying to get resources"); - if (bnx2x_do_req4pf(sc, sc->vf2pf_mbox_mapping.paddr)) { - /* timeout */ - status.success = 0; - status.err_code = -EAGAIN; - return status; - } + rc = bnx2x_do_req4pf(sc, sc->vf2pf_mbox_mapping.paddr); + if (rc) + return rc; memcpy(sc_resp, resp, sizeof(sc->acquire_resp)); @@ -203,12 +213,12 @@ struct bnx2x_obtain_status bnx2x_loop_obtain_resources(struct bnx2x_softc *sc) PMD_DRV_LOG(DEBUG, "resources obtained successfully"); res_obtained = true; } else if (sc_resp->status == BNX2X_VF_STATUS_NO_RESOURCES && - tries < BNX2X_VF_OBTAIN_MAX_TRIES) { + tries < BNX2X_VF_OBTAIN_MAX_TRIES) { PMD_DRV_LOG(DEBUG, "PF cannot allocate requested amount of resources"); res_query = &sc->vf2pf_mbox->query[0].acquire.res_query; - resc = &sc_resp->resc; + resc = &sc_resp->resc; /* PF refused our request. Try to decrease request params */ res_query->num_txqs = min(res_query->num_txqs, resc->num_txqs); @@ -220,30 +230,30 @@ struct bnx2x_obtain_status bnx2x_loop_obtain_resources(struct bnx2x_softc *sc) memset(&sc->vf2pf_mbox->resp, 0, sizeof(union resp_tlvs)); } else { - PMD_DRV_LOG(ERR, "Resources cannot be obtained. Status of handling: %d. Aborting", - sc_resp->status); - status.success = 0; - status.err_code = -EAGAIN; - return status; + PMD_DRV_LOG(ERR, "Failed to get the requested " + "amount of resources: %d.", + sc_resp->status); + return -EINVAL; } } while (!res_obtained); - status.success = 1; - return status; + return 0; } int bnx2x_vf_get_resources(struct bnx2x_softc *sc, uint8_t tx_count, uint8_t rx_count) { struct vf_acquire_tlv *acq = &sc->vf2pf_mbox->query[0].acquire; int vf_id; - struct bnx2x_obtain_status obtain_status; + int rc; bnx2x_vf_close(sc); - bnx2x_init_first_tlv(sc, &acq->first_tlv, BNX2X_VF_TLV_ACQUIRE, sizeof(*acq)); + bnx2x_vf_prep(sc, &acq->first_tlv, BNX2X_VF_TLV_ACQUIRE, sizeof(*acq)); vf_id = bnx2x_read_vf_id(sc); - if (vf_id < 0) - return -EAGAIN; + if (vf_id < 0) { + rc = -EAGAIN; + goto out; + } acq->vf_id = vf_id; @@ -256,19 +266,19 @@ int bnx2x_vf_get_resources(struct bnx2x_softc *sc, uint8_t tx_count, uint8_t rx_ acq->bulletin_addr = sc->pf2vf_bulletin_mapping.paddr; /* Request physical port identifier */ - BNX2X_TLV_APPEND(acq, acq->first_tlv.length, - BNX2X_VF_TLV_PHYS_PORT_ID, - sizeof(struct channel_tlv)); + bnx2x_add_tlv(sc, acq, acq->first_tlv.tl.length, + BNX2X_VF_TLV_PHYS_PORT_ID, + sizeof(struct channel_tlv)); - BNX2X_TLV_APPEND(acq, - (acq->first_tlv.length + sizeof(struct channel_tlv)), - BNX2X_VF_TLV_LIST_END, - sizeof(struct channel_list_end_tlv)); + bnx2x_add_tlv(sc, acq, + (acq->first_tlv.tl.length + sizeof(struct channel_tlv)), + BNX2X_VF_TLV_LIST_END, + sizeof(struct channel_list_end_tlv)); /* requesting the resources in loop */ - obtain_status = bnx2x_loop_obtain_resources(sc); - if (!obtain_status.success) - return obtain_status.err_code; + rc = bnx2x_loop_obtain_resources(sc); + if (rc) + goto out; struct vf_acquire_resp_tlv sc_resp = sc->acquire_resp; @@ -299,7 +309,10 @@ int bnx2x_vf_get_resources(struct bnx2x_softc *sc, uint8_t tx_count, uint8_t rx_ else eth_random_addr(sc->link_params.mac_addr); - return 0; +out: + bnx2x_vf_finalize(sc, &acq->first_tlv); + + return rc; } /* Ask PF to release VF's resources */ @@ -309,19 +322,23 @@ bnx2x_vf_close(struct bnx2x_softc *sc) struct vf_release_tlv *query; struct vf_common_reply_tlv *reply = &sc->vf2pf_mbox->resp.common_reply; int vf_id = bnx2x_read_vf_id(sc); + int rc; if (vf_id >= 0) { query = &sc->vf2pf_mbox->query[0].release; - bnx2x_init_first_tlv(sc, &query->first_tlv, BNX2X_VF_TLV_RELEASE, - sizeof(*query)); + bnx2x_vf_prep(sc, &query->first_tlv, BNX2X_VF_TLV_RELEASE, + sizeof(*query)); query->vf_id = vf_id; - BNX2X_TLV_APPEND(query, query->first_tlv.length, BNX2X_VF_TLV_LIST_END, - sizeof(struct channel_list_end_tlv)); + bnx2x_add_tlv(sc, query, query->first_tlv.tl.length, + BNX2X_VF_TLV_LIST_END, + sizeof(struct channel_list_end_tlv)); - bnx2x_do_req4pf(sc, sc->vf2pf_mbox_mapping.paddr); - if (reply->status != BNX2X_VF_STATUS_SUCCESS) + rc = bnx2x_do_req4pf(sc, sc->vf2pf_mbox_mapping.paddr); + if (rc || reply->status != BNX2X_VF_STATUS_SUCCESS) PMD_DRV_LOG(ERR, "Failed to release VF"); + + bnx2x_vf_finalize(sc, &query->first_tlv); } } @@ -331,11 +348,11 @@ bnx2x_vf_init(struct bnx2x_softc *sc) { struct vf_init_tlv *query; struct vf_common_reply_tlv *reply = &sc->vf2pf_mbox->resp.common_reply; - int i; + int i, rc; query = &sc->vf2pf_mbox->query[0].init; - bnx2x_init_first_tlv(sc, &query->first_tlv, BNX2X_VF_TLV_INIT, - sizeof(*query)); + bnx2x_vf_prep(sc, &query->first_tlv, BNX2X_VF_TLV_INIT, + sizeof(*query)); FOR_EACH_QUEUE(sc, i) { query->sb_addr[i] = (unsigned long)(sc->fp[i].sb_dma.paddr); @@ -345,17 +362,23 @@ bnx2x_vf_init(struct bnx2x_softc *sc) query->stats_addr = sc->fw_stats_data_mapping + offsetof(struct bnx2x_fw_stats_data, queue_stats); - BNX2X_TLV_APPEND(query, query->first_tlv.length, BNX2X_VF_TLV_LIST_END, - sizeof(struct channel_list_end_tlv)); + bnx2x_add_tlv(sc, query, query->first_tlv.tl.length, + BNX2X_VF_TLV_LIST_END, + sizeof(struct channel_list_end_tlv)); - bnx2x_do_req4pf(sc, sc->vf2pf_mbox_mapping.paddr); + rc = bnx2x_do_req4pf(sc, sc->vf2pf_mbox_mapping.paddr); + if (rc) + goto out; if (reply->status != BNX2X_VF_STATUS_SUCCESS) { PMD_DRV_LOG(ERR, "Failed to init VF"); - return -EINVAL; + rc = -EINVAL; + goto out; } PMD_DRV_LOG(DEBUG, "VF was initialized"); - return 0; +out: + bnx2x_vf_finalize(sc, &query->first_tlv); + return rc; } void @@ -364,44 +387,49 @@ bnx2x_vf_unload(struct bnx2x_softc *sc) struct vf_close_tlv *query; struct vf_common_reply_tlv *reply = &sc->vf2pf_mbox->resp.common_reply; struct vf_q_op_tlv *query_op; - int i, vf_id; + int i, vf_id, rc; vf_id = bnx2x_read_vf_id(sc); if (vf_id > 0) { FOR_EACH_QUEUE(sc, i) { query_op = &sc->vf2pf_mbox->query[0].q_op; - bnx2x_init_first_tlv(sc, &query_op->first_tlv, - BNX2X_VF_TLV_TEARDOWN_Q, - sizeof(*query_op)); + bnx2x_vf_prep(sc, &query_op->first_tlv, + BNX2X_VF_TLV_TEARDOWN_Q, + sizeof(*query_op)); query_op->vf_qid = i; - BNX2X_TLV_APPEND(query_op, query_op->first_tlv.length, - BNX2X_VF_TLV_LIST_END, - sizeof(struct channel_list_end_tlv)); + bnx2x_add_tlv(sc, query_op, + query_op->first_tlv.tl.length, + BNX2X_VF_TLV_LIST_END, + sizeof(struct channel_list_end_tlv)); - bnx2x_do_req4pf(sc, sc->vf2pf_mbox_mapping.paddr); - if (reply->status != BNX2X_VF_STATUS_SUCCESS) + rc = bnx2x_do_req4pf(sc, sc->vf2pf_mbox_mapping.paddr); + if (rc || reply->status != BNX2X_VF_STATUS_SUCCESS) PMD_DRV_LOG(ERR, "Bad reply for vf_q %d teardown", i); + + bnx2x_vf_finalize(sc, &query_op->first_tlv); } bnx2x_vf_set_mac(sc, false); query = &sc->vf2pf_mbox->query[0].close; - bnx2x_init_first_tlv(sc, &query->first_tlv, BNX2X_VF_TLV_CLOSE, - sizeof(*query)); + bnx2x_vf_prep(sc, &query->first_tlv, BNX2X_VF_TLV_CLOSE, + sizeof(*query)); query->vf_id = vf_id; - BNX2X_TLV_APPEND(query, query->first_tlv.length, - BNX2X_VF_TLV_LIST_END, - sizeof(struct channel_list_end_tlv)); + bnx2x_add_tlv(sc, query, query->first_tlv.tl.length, + BNX2X_VF_TLV_LIST_END, + sizeof(struct channel_list_end_tlv)); - bnx2x_do_req4pf(sc, sc->vf2pf_mbox_mapping.paddr); - if (reply->status != BNX2X_VF_STATUS_SUCCESS) + rc = bnx2x_do_req4pf(sc, sc->vf2pf_mbox_mapping.paddr); + if (rc || reply->status != BNX2X_VF_STATUS_SUCCESS) PMD_DRV_LOG(ERR, "Bad reply from PF for close message"); + + bnx2x_vf_finalize(sc, &query->first_tlv); } } @@ -466,10 +494,11 @@ bnx2x_vf_setup_queue(struct bnx2x_softc *sc, struct bnx2x_fastpath *fp, int lead struct vf_setup_q_tlv *query; struct vf_common_reply_tlv *reply = &sc->vf2pf_mbox->resp.common_reply; uint16_t flags = bnx2x_vf_q_flags(leading); + int rc; query = &sc->vf2pf_mbox->query[0].setup_q; - bnx2x_init_first_tlv(sc, &query->first_tlv, BNX2X_VF_TLV_SETUP_Q, - sizeof(*query)); + bnx2x_vf_prep(sc, &query->first_tlv, BNX2X_VF_TLV_SETUP_Q, + sizeof(*query)); query->vf_qid = fp->index; query->param_valid = VF_RXQ_VALID | VF_TXQ_VALID; @@ -477,17 +506,22 @@ bnx2x_vf_setup_queue(struct bnx2x_softc *sc, struct bnx2x_fastpath *fp, int lead bnx2x_vf_rx_q_prep(sc, fp, &query->rxq, flags); bnx2x_vf_tx_q_prep(sc, fp, &query->txq, flags); - BNX2X_TLV_APPEND(query, query->first_tlv.length, BNX2X_VF_TLV_LIST_END, - sizeof(struct channel_list_end_tlv)); + bnx2x_add_tlv(sc, query, query->first_tlv.tl.length, + BNX2X_VF_TLV_LIST_END, + sizeof(struct channel_list_end_tlv)); - bnx2x_do_req4pf(sc, sc->vf2pf_mbox_mapping.paddr); + rc = bnx2x_do_req4pf(sc, sc->vf2pf_mbox_mapping.paddr); + if (rc) + goto out; if (reply->status != BNX2X_VF_STATUS_SUCCESS) { PMD_DRV_LOG(ERR, "Failed to setup VF queue[%d]", fp->index); - return -EINVAL; + rc = -EINVAL; } +out: + bnx2x_vf_finalize(sc, &query->first_tlv); - return 0; + return rc; } int @@ -495,9 +529,10 @@ bnx2x_vf_set_mac(struct bnx2x_softc *sc, int set) { struct vf_set_q_filters_tlv *query; struct vf_common_reply_tlv *reply; + int rc; query = &sc->vf2pf_mbox->query[0].set_q_filters; - bnx2x_init_first_tlv(sc, &query->first_tlv, BNX2X_VF_TLV_SET_Q_FILTERS, + bnx2x_vf_prep(sc, &query->first_tlv, BNX2X_VF_TLV_SET_Q_FILTERS, sizeof(*query)); query->vf_qid = sc->fp->index; @@ -511,10 +546,13 @@ bnx2x_vf_set_mac(struct bnx2x_softc *sc, int set) rte_memcpy(query->filters[0].mac, sc->link_params.mac_addr, ETH_ALEN); - BNX2X_TLV_APPEND(query, query->first_tlv.length, BNX2X_VF_TLV_LIST_END, - sizeof(struct channel_list_end_tlv)); + bnx2x_add_tlv(sc, query, query->first_tlv.tl.length, + BNX2X_VF_TLV_LIST_END, + sizeof(struct channel_list_end_tlv)); - bnx2x_do_req4pf(sc, sc->vf2pf_mbox_mapping.paddr); + rc = bnx2x_do_req4pf(sc, sc->vf2pf_mbox_mapping.paddr); + if (rc) + goto out; reply = &sc->vf2pf_mbox->resp.common_reply; while (BNX2X_VF_STATUS_FAILURE == reply->status && @@ -525,16 +563,20 @@ bnx2x_vf_set_mac(struct bnx2x_softc *sc, int set) rte_memcpy(query->filters[0].mac, sc->pf2vf_bulletin->mac, ETH_ALEN); - bnx2x_do_req4pf(sc, sc->vf2pf_mbox_mapping.paddr); + rc = bnx2x_do_req4pf(sc, sc->vf2pf_mbox_mapping.paddr); + if (rc) + goto out; } if (BNX2X_VF_STATUS_SUCCESS != reply->status) { PMD_DRV_LOG(ERR, "Bad reply from PF for SET MAC message: %d", reply->status); - return -EINVAL; + rc = -EINVAL; } +out: + bnx2x_vf_finalize(sc, &query->first_tlv); - return 0; + return rc; } int @@ -543,15 +585,17 @@ bnx2x_vf_config_rss(struct bnx2x_softc *sc, { struct vf_rss_tlv *query; struct vf_common_reply_tlv *reply = &sc->vf2pf_mbox->resp.common_reply; + int rc; query = &sc->vf2pf_mbox->query[0].update_rss; - bnx2x_init_first_tlv(sc, &query->first_tlv, BNX2X_VF_TLV_UPDATE_RSS, + bnx2x_vf_prep(sc, &query->first_tlv, BNX2X_VF_TLV_UPDATE_RSS, sizeof(*query)); /* add list termination tlv */ - BNX2X_TLV_APPEND(query, query->first_tlv.length, BNX2X_VF_TLV_LIST_END, - sizeof(struct channel_list_end_tlv)); + bnx2x_add_tlv(sc, query, query->first_tlv.tl.length, + BNX2X_VF_TLV_LIST_END, + sizeof(struct channel_list_end_tlv)); rte_memcpy(query->rss_key, params->rss_key, sizeof(params->rss_key)); query->rss_key_size = T_ETH_RSS_KEY; @@ -562,13 +606,18 @@ bnx2x_vf_config_rss(struct bnx2x_softc *sc, query->rss_result_mask = params->rss_result_mask; query->rss_flags = params->rss_flags; - bnx2x_do_req4pf(sc, sc->vf2pf_mbox_mapping.paddr); + rc = bnx2x_do_req4pf(sc, sc->vf2pf_mbox_mapping.paddr); + if (rc) + goto out; + if (reply->status != BNX2X_VF_STATUS_SUCCESS) { PMD_DRV_LOG(ERR, "Failed to configure RSS"); - return -EINVAL; + rc = -EINVAL; } +out: + bnx2x_vf_finalize(sc, &query->first_tlv); - return 0; + return rc; } int @@ -576,27 +625,56 @@ bnx2x_vf_set_rx_mode(struct bnx2x_softc *sc) { struct vf_set_q_filters_tlv *query; struct vf_common_reply_tlv *reply = &sc->vf2pf_mbox->resp.common_reply; - unsigned long tx_mask; + int rc; query = &sc->vf2pf_mbox->query[0].set_q_filters; - bnx2x_init_first_tlv(sc, &query->first_tlv, BNX2X_VF_TLV_SET_Q_FILTERS, + bnx2x_vf_prep(sc, &query->first_tlv, BNX2X_VF_TLV_SET_Q_FILTERS, sizeof(*query)); query->vf_qid = 0; query->flags = BNX2X_VF_RX_MASK_CHANGED; - if (bnx2x_fill_accept_flags(sc, sc->rx_mode, &query->rx_mask, &tx_mask)) { - return -EINVAL; + switch (sc->rx_mode) { + case BNX2X_RX_MODE_NONE: /* no Rx */ + query->rx_mask = VFPF_RX_MASK_ACCEPT_NONE; + break; + case BNX2X_RX_MODE_NORMAL: + query->rx_mask = VFPF_RX_MASK_ACCEPT_MATCHED_MULTICAST; + query->rx_mask |= VFPF_RX_MASK_ACCEPT_MATCHED_UNICAST; + query->rx_mask |= VFPF_RX_MASK_ACCEPT_BROADCAST; + break; + case BNX2X_RX_MODE_ALLMULTI: + query->rx_mask = VFPF_RX_MASK_ACCEPT_ALL_MULTICAST; + query->rx_mask |= VFPF_RX_MASK_ACCEPT_MATCHED_UNICAST; + query->rx_mask |= VFPF_RX_MASK_ACCEPT_BROADCAST; + break; + case BNX2X_RX_MODE_ALLMULTI_PROMISC: + case BNX2X_RX_MODE_PROMISC: + query->rx_mask = VFPF_RX_MASK_ACCEPT_ALL_UNICAST; + query->rx_mask |= VFPF_RX_MASK_ACCEPT_ALL_MULTICAST; + query->rx_mask |= VFPF_RX_MASK_ACCEPT_BROADCAST; + break; + default: + PMD_DRV_LOG(ERR, "BAD rx mode (%d)", sc->rx_mode); + rc = -EINVAL; + goto out; } - BNX2X_TLV_APPEND(query, query->first_tlv.length, BNX2X_VF_TLV_LIST_END, - sizeof(struct channel_list_end_tlv)); + bnx2x_add_tlv(sc, query, query->first_tlv.tl.length, + BNX2X_VF_TLV_LIST_END, + sizeof(struct channel_list_end_tlv)); + + rc = bnx2x_do_req4pf(sc, sc->vf2pf_mbox_mapping.paddr); + if (rc) + goto out; - bnx2x_do_req4pf(sc, sc->vf2pf_mbox_mapping.paddr); if (reply->status != BNX2X_VF_STATUS_SUCCESS) { PMD_DRV_LOG(ERR, "Failed to set RX mode"); - return -EINVAL; + rc = -EINVAL; } - return 0; +out: + bnx2x_vf_finalize(sc, &query->first_tlv); + + return rc; } diff --git a/dpdk/drivers/net/bnx2x/bnx2x_vfpf.h b/dpdk/drivers/net/bnx2x/bnx2x_vfpf.h index f854d81b..d7cc11be 100644 --- a/dpdk/drivers/net/bnx2x/bnx2x_vfpf.h +++ b/dpdk/drivers/net/bnx2x/bnx2x_vfpf.h @@ -40,6 +40,13 @@ struct vf_resource_query { #define TLV_BUFFER_SIZE 1024 +#define VFPF_RX_MASK_ACCEPT_NONE 0x00000000 +#define VFPF_RX_MASK_ACCEPT_MATCHED_UNICAST 0x00000001 +#define VFPF_RX_MASK_ACCEPT_MATCHED_MULTICAST 0x00000002 +#define VFPF_RX_MASK_ACCEPT_ALL_UNICAST 0x00000004 +#define VFPF_RX_MASK_ACCEPT_ALL_MULTICAST 0x00000008 +#define VFPF_RX_MASK_ACCEPT_BROADCAST 0x00000010 + /* general tlv header (used for both vf->pf request and pf->vf response) */ struct channel_tlv { uint16_t type; @@ -47,8 +54,7 @@ struct channel_tlv { }; struct vf_first_tlv { - uint16_t type; - uint16_t length; + struct channel_tlv tl; uint32_t reply_offset; }; @@ -58,16 +64,14 @@ struct tlv_buffer_size { /* tlv struct for all PF replies except acquire */ struct vf_common_reply_tlv { - uint16_t type; - uint16_t length; + struct channel_tlv tl; uint8_t status; uint8_t pad[3]; }; /* used to terminate and pad a tlv list */ struct channel_list_end_tlv { - uint16_t type; - uint16_t length; + struct channel_tlv tl; uint32_t pad; }; @@ -278,7 +282,7 @@ struct bnx2x_vf_bulletin { uint16_t version; uint16_t length; - uint64_t valid_bitmap; /* bitmap indicating wich fields + uint64_t valid_bitmap; /* bitmap indicating which fields * hold valid values */ @@ -327,7 +331,6 @@ struct bnx2x_vf_mbx_msg { union resp_tlvs resp; }; -void bnx2x_add_tlv(void *tlvs_list, uint16_t offset, uint16_t type, uint16_t length); int bnx2x_vf_set_mac(struct bnx2x_softc *sc, int set); int bnx2x_vf_config_rss(struct bnx2x_softc *sc, struct ecore_config_rss_params *params); diff --git a/dpdk/drivers/net/bnx2x/debug.c b/dpdk/drivers/net/bnx2x/debug.c deleted file mode 100644 index cc50845c..00000000 --- a/dpdk/drivers/net/bnx2x/debug.c +++ /dev/null @@ -1,96 +0,0 @@ -/*- - * Copyright (c) 2007-2013 QLogic Corporation. All rights reserved. - * - * Eric Davis - * David Christensen - * Gary Zambrano - * - * Copyright (c) 2013-2015 Brocade Communications Systems, Inc. - * Copyright (c) 2015 QLogic Corporation. - * All rights reserved. - * www.qlogic.com - * - * See LICENSE.bnx2x_pmd for copyright and licensing details. - */ - -#include "bnx2x.h" - - -/* - * Debug versions of the 8/16/32 bit OS register read/write functions to - * capture/display values read/written from/to the controller. - */ -void -bnx2x_reg_write8(struct bnx2x_softc *sc, size_t offset, uint8_t val) -{ - PMD_DEBUG_PERIODIC_LOG(DEBUG, "offset=0x%08lx val=0x%02x", (unsigned long)offset, val); - *((volatile uint8_t*)((uintptr_t)sc->bar[BAR0].base_addr + offset)) = val; -} - -void -bnx2x_reg_write16(struct bnx2x_softc *sc, size_t offset, uint16_t val) -{ - if ((offset % 2) != 0) { - PMD_DRV_LOG(NOTICE, "Unaligned 16-bit write to 0x%08lx", - (unsigned long)offset); - } - - PMD_DEBUG_PERIODIC_LOG(DEBUG, "offset=0x%08lx val=0x%04x", (unsigned long)offset, val); - *((volatile uint16_t*)((uintptr_t)sc->bar[BAR0].base_addr + offset)) = val; -} - -void -bnx2x_reg_write32(struct bnx2x_softc *sc, size_t offset, uint32_t val) -{ - if ((offset % 4) != 0) { - PMD_DRV_LOG(NOTICE, "Unaligned 32-bit write to 0x%08lx", - (unsigned long)offset); - } - - PMD_DEBUG_PERIODIC_LOG(DEBUG, "offset=0x%08lx val=0x%08x", (unsigned long)offset, val); - *((volatile uint32_t*)((uintptr_t)sc->bar[BAR0].base_addr + offset)) = val; -} - -uint8_t -bnx2x_reg_read8(struct bnx2x_softc *sc, size_t offset) -{ - uint8_t val; - - val = (uint8_t)(*((volatile uint8_t*)((uintptr_t)sc->bar[BAR0].base_addr + offset))); - PMD_DEBUG_PERIODIC_LOG(DEBUG, "offset=0x%08lx val=0x%02x", (unsigned long)offset, val); - - return val; -} - -uint16_t -bnx2x_reg_read16(struct bnx2x_softc *sc, size_t offset) -{ - uint16_t val; - - if ((offset % 2) != 0) { - PMD_DRV_LOG(NOTICE, "Unaligned 16-bit read from 0x%08lx", - (unsigned long)offset); - } - - val = (uint16_t)(*((volatile uint16_t*)((uintptr_t)sc->bar[BAR0].base_addr + offset))); - PMD_DEBUG_PERIODIC_LOG(DEBUG, "offset=0x%08lx val=0x%08x", (unsigned long)offset, val); - - return val; -} - -uint32_t -bnx2x_reg_read32(struct bnx2x_softc *sc, size_t offset) -{ - uint32_t val; - - if ((offset % 4) != 0) { - PMD_DRV_LOG(NOTICE, "Unaligned 32-bit read from 0x%08lx", - (unsigned long)offset); - return 0; - } - - val = (uint32_t)(*((volatile uint32_t*)((uintptr_t)sc->bar[BAR0].base_addr + offset))); - PMD_DEBUG_PERIODIC_LOG(DEBUG, "offset=0x%08lx val=0x%08x", (unsigned long)offset, val); - - return val; -} diff --git a/dpdk/drivers/net/bnx2x/ecore_hsi.h b/dpdk/drivers/net/bnx2x/ecore_hsi.h index 5808e1ae..5cce6647 100644 --- a/dpdk/drivers/net/bnx2x/ecore_hsi.h +++ b/dpdk/drivers/net/bnx2x/ecore_hsi.h @@ -2499,7 +2499,7 @@ struct shmem2_region { * SHMEM_EEE_XXX_ADV definitions (where XXX is replaced by speed). * bit 28 when 1'b1 EEE was requested. * bit 29 when 1'b1 tx lpi was requested. - * bit 30 when 1'b1 EEE was negotiated. Tx lpi will be asserted iff + * bit 30 when 1'b1 EEE was negotiated. Tx lpi will be asserted if * 30:29 are 2'b11. * bit 31 when 1'b0 bits 15:0 contain a PORT_FEAT_CFG_EEE_ define as * value. When 1'b1 those bits contains a value times 16 microseconds. @@ -3911,7 +3911,7 @@ struct client_init_rx_data uint16_t bd_pause_thr_high /* number of remaining bds above which, we send un-pause message */; uint16_t sge_pause_thr_low /* number of remaining sges under which, we send pause message */; uint16_t sge_pause_thr_high /* number of remaining sges above which, we send un-pause message */; - uint16_t rx_cos_mask /* the bits that will be set on pfc/ safc paket whith will be genratet when this ring is full. for regular flow control set this to 1 */; + uint16_t rx_cos_mask /* the bits that will be set on pfc/ safc paket with will be genratet when this ring is full. for regular flow control set this to 1 */; uint16_t silent_vlan_value /* The vlan to compare, in case, silent vlan is set */; uint16_t silent_vlan_mask /* The vlan mask, in case, silent vlan is set */; uint32_t reserved6[2]; @@ -4903,7 +4903,7 @@ enum eth_tx_vlan_type */ enum eth_vlan_filter_mode { - ETH_VLAN_FILTER_ANY_VLAN /* Dont filter by vlan */, + ETH_VLAN_FILTER_ANY_VLAN /* Don't filter by vlan */, ETH_VLAN_FILTER_SPECIFIC_VLAN /* Only the vlan_id is allowed */, ETH_VLAN_FILTER_CLASSIFY /* Vlan will be added to CAM for classification */, MAX_ETH_VLAN_FILTER_MODE}; @@ -4937,7 +4937,7 @@ struct mac_configuration_entry #define MAC_CONFIGURATION_ENTRY_RDMA_MAC_SHIFT 1 #define MAC_CONFIGURATION_ENTRY_VLAN_FILTERING_MODE (0x3<<2) /* BitField flags (use enum eth_vlan_filter_mode) */ #define MAC_CONFIGURATION_ENTRY_VLAN_FILTERING_MODE_SHIFT 2 -#define MAC_CONFIGURATION_ENTRY_OVERRIDE_VLAN_REMOVAL (0x1<<4) /* BitField flags BitField flags 0 - cant remove vlan 1 - can remove vlan. relevant only to everest1 */ +#define MAC_CONFIGURATION_ENTRY_OVERRIDE_VLAN_REMOVAL (0x1<<4) /* BitField flags BitField flags 0 - can't remove vlan 1 - can remove vlan. relevant only to everest1 */ #define MAC_CONFIGURATION_ENTRY_OVERRIDE_VLAN_REMOVAL_SHIFT 4 #define MAC_CONFIGURATION_ENTRY_BROADCAST (0x1<<5) /* BitField flags BitField flags 0 - not broadcast 1 - broadcast. relevant only to everest1 */ #define MAC_CONFIGURATION_ENTRY_BROADCAST_SHIFT 5 @@ -5024,7 +5024,7 @@ struct tstorm_eth_function_common_config #define TSTORM_ETH_FUNCTION_COMMON_CONFIG_RSS_IPV6_TCP_CAPABILITY_SHIFT 3 #define TSTORM_ETH_FUNCTION_COMMON_CONFIG_RSS_MODE (0x7<<4) /* BitField config_flagsGeneral configuration flags RSS mode of operation (use enum eth_rss_mode) */ #define TSTORM_ETH_FUNCTION_COMMON_CONFIG_RSS_MODE_SHIFT 4 -#define TSTORM_ETH_FUNCTION_COMMON_CONFIG_VLAN_FILTERING_ENABLE (0x1<<7) /* BitField config_flagsGeneral configuration flags 0 - Dont filter by vlan, 1 - Filter according to the vlans specificied in mac_filter_config */ +#define TSTORM_ETH_FUNCTION_COMMON_CONFIG_VLAN_FILTERING_ENABLE (0x1<<7) /* BitField config_flagsGeneral configuration flags 0 - Don't filter by vlan, 1 - Filter according to the vlans specificied in mac_filter_config */ #define TSTORM_ETH_FUNCTION_COMMON_CONFIG_VLAN_FILTERING_ENABLE_SHIFT 7 #define __TSTORM_ETH_FUNCTION_COMMON_CONFIG_RESERVED0 (0xFF<<8) /* BitField config_flagsGeneral configuration flags */ #define __TSTORM_ETH_FUNCTION_COMMON_CONFIG_RESERVED0_SHIFT 8 @@ -5634,7 +5634,7 @@ struct flow_control_configuration struct function_start_data { uint8_t function_mode /* the function mode */; - uint8_t allow_npar_tx_switching /* If set, inter-pf tx switching is allowed in Switch Independant function mode. (E2/E3 Only) */; + uint8_t allow_npar_tx_switching /* If set, inter-pf tx switching is allowed in Switch Independent function mode. (E2/E3 Only) */; uint16_t sd_vlan_tag /* value of Vlan in case of switch depended multi-function mode */; uint16_t vif_id /* value of VIF id in case of NIV multi-function mode */; uint8_t path_id; diff --git a/dpdk/drivers/net/bnx2x/ecore_init.h b/dpdk/drivers/net/bnx2x/ecore_init.h index d25e2803..4576c565 100644 --- a/dpdk/drivers/net/bnx2x/ecore_init.h +++ b/dpdk/drivers/net/bnx2x/ecore_init.h @@ -304,7 +304,7 @@ static inline void ecore_dcb_config_qm(struct bnx2x_softc *sc, enum cos_mode mod /* - * congestion managment port init api description + * congestion management port init api description * the api works as follows: * the driver should pass the cmng_init_input struct, the port_init function * will prepare the required internal ram structure which will be passed back diff --git a/dpdk/drivers/net/bnx2x/ecore_sp.c b/dpdk/drivers/net/bnx2x/ecore_sp.c index e6fecd88..ef7f9fea 100644 --- a/dpdk/drivers/net/bnx2x/ecore_sp.c +++ b/dpdk/drivers/net/bnx2x/ecore_sp.c @@ -2676,7 +2676,7 @@ static void ecore_mcast_hdl_del(struct bnx2x_softc *sc, * @cmd: * @start_cnt: first line in the ramrod data that may be used * - * This function is called iff there is enough place for the current command in + * This function is called if there is enough place for the current command in * the ramrod data. * Returns number of lines filled in the ramrod data in total. */ @@ -2725,7 +2725,7 @@ static int ecore_mcast_validate_e2(__rte_unused struct bnx2x_softc *sc, /* DEL command deletes all currently configured MACs */ case ECORE_MCAST_CMD_DEL: o->set_registry_size(o, 0); - /* Don't break */ + /* fall-through */ /* RESTORE command will restore the entire multicast configuration */ case ECORE_MCAST_CMD_RESTORE: @@ -2834,7 +2834,7 @@ static int ecore_mcast_setup_e2(struct bnx2x_softc *sc, if (ECORE_LIST_IS_EMPTY(&o->pending_cmds_head)) o->clear_sched(o); - /* The below may be TRUE iff there was enough room in ramrod + /* The below may be TRUE if there was enough room in ramrod * data for all pending commands and for the current * command. Otherwise the current command would have been added * to the pending commands and p->mcast_list_len would have been diff --git a/dpdk/drivers/net/bnx2x/ecore_sp.h b/dpdk/drivers/net/bnx2x/ecore_sp.h index 9c1f55df..ff40413c 100644 --- a/dpdk/drivers/net/bnx2x/ecore_sp.h +++ b/dpdk/drivers/net/bnx2x/ecore_sp.h @@ -36,7 +36,7 @@ #include "ecore_reg.h" struct bnx2x_softc; -typedef phys_addr_t ecore_dma_addr_t; /* expected to be 64 bit wide */ +typedef rte_iova_t ecore_dma_addr_t; /* expected to be 64 bit wide */ typedef volatile int ecore_atomic_t; @@ -113,7 +113,7 @@ typedef rte_spinlock_t ECORE_MUTEX_SPIN; #define ECORE_FCOE_CID(sc) ((sc)->fp[FCOE_IDX(sc)].cl_id) #define ECORE_MEMCMP(_a, _b, _s) memcmp(_a, _b, _s) -#define ECORE_MEMCPY(_a, _b, _s) (void)rte_memcpy(_a, _b, _s) +#define ECORE_MEMCPY(_a, _b, _s) rte_memcpy(_a, _b, _s) #define ECORE_MEMSET(_a, _c, _s) memset(_a, _c, _s) #define ECORE_CPU_TO_LE16(x) htole16(x) @@ -1116,10 +1116,10 @@ struct ecore_config_rss_params { /* RSS hash values */ uint32_t rss_key[10]; - /* valid only iff ECORE_RSS_UPDATE_TOE is set */ + /* valid only if ECORE_RSS_UPDATE_TOE is set */ uint16_t toe_rss_bitmap; - /* valid iff ECORE_RSS_TUNNELING is set */ + /* valid if ECORE_RSS_TUNNELING is set */ uint16_t tunnel_value; uint16_t tunnel_mask; }; @@ -1286,14 +1286,14 @@ struct rxq_pause_params { uint16_t bd_th_hi; uint16_t rcq_th_lo; uint16_t rcq_th_hi; - uint16_t sge_th_lo; /* valid iff ECORE_Q_FLG_TPA */ - uint16_t sge_th_hi; /* valid iff ECORE_Q_FLG_TPA */ + uint16_t sge_th_lo; /* valid if ECORE_Q_FLG_TPA */ + uint16_t sge_th_hi; /* valid if ECORE_Q_FLG_TPA */ uint16_t pri_map; }; /* general */ struct ecore_general_setup_params { - /* valid iff ECORE_Q_FLG_STATS */ + /* valid if ECORE_Q_FLG_STATS */ uint8_t stat_id; uint8_t spcl_id; @@ -1312,19 +1312,19 @@ struct ecore_rxq_setup_params { uint8_t fw_sb_id; uint8_t cl_qzone_id; - /* valid iff ECORE_Q_FLG_TPA */ + /* valid if ECORE_Q_FLG_TPA */ uint16_t tpa_agg_sz; uint8_t max_tpa_queues; uint8_t rss_engine_id; - /* valid iff ECORE_Q_FLG_MCAST */ + /* valid if ECORE_Q_FLG_MCAST */ uint8_t mcast_engine_id; uint8_t cache_line_log; uint8_t sb_cq_index; - /* valid iff BXN2X_Q_FLG_SILENT_VLAN_REM */ + /* valid if BXN2X_Q_FLG_SILENT_VLAN_REM */ uint16_t silent_removal_value; uint16_t silent_removal_mask; }; @@ -1335,12 +1335,12 @@ struct ecore_txq_setup_params { uint8_t fw_sb_id; uint8_t sb_cq_index; - uint8_t cos; /* valid iff ECORE_Q_FLG_COS */ + uint8_t cos; /* valid if ECORE_Q_FLG_COS */ uint16_t traffic_type; /* equals to the leading rss client id, used for TX classification*/ uint8_t tss_leading_cl_id; - /* valid iff ECORE_Q_FLG_DEF_VLAN */ + /* valid if ECORE_Q_FLG_DEF_VLAN */ uint16_t default_vlan; }; @@ -1733,7 +1733,7 @@ void ecore_init_mcast_obj(struct bnx2x_softc *sc, * the current command will be enqueued to the tail of the * pending commands list. * - * Return: 0 is operation was successfull and there are no pending completions, + * Return: 0 is operation was successful and there are no pending completions, * negative if there were errors, positive if there are pending * completions. */ diff --git a/dpdk/drivers/net/bnx2x/elink.c b/dpdk/drivers/net/bnx2x/elink.c index d9a72f0a..9d0f3136 100644 --- a/dpdk/drivers/net/bnx2x/elink.c +++ b/dpdk/drivers/net/bnx2x/elink.c @@ -1586,26 +1586,6 @@ static elink_status_t elink_emac_enable(struct elink_params *params, /* enable emac and not bmac */ REG_WR(sc, NIG_REG_EGRESS_EMAC0_PORT + port * 4, 1); -#ifdef ELINK_INCLUDE_EMUL - /* for paladium */ - if (CHIP_REV_IS_EMUL(sc)) { - /* Use lane 1 (of lanes 0-3) */ - REG_WR(sc, NIG_REG_XGXS_LANE_SEL_P0 + port * 4, 1); - REG_WR(sc, NIG_REG_XGXS_SERDES0_MODE_SEL + port * 4, 1); - } - /* for fpga */ - else -#endif -#ifdef ELINK_INCLUDE_FPGA - if (CHIP_REV_IS_FPGA(sc)) { - /* Use lane 1 (of lanes 0-3) */ - PMD_DRV_LOG(DEBUG, "elink_emac_enable: Setting FPGA"); - - REG_WR(sc, NIG_REG_XGXS_LANE_SEL_P0 + port * 4, 1); - REG_WR(sc, NIG_REG_XGXS_SERDES0_MODE_SEL + port * 4, 0); - } else -#endif - /* ASIC */ if (vars->phy_flags & PHY_XGXS_FLAG) { uint32_t ser_lane = ((params->lane_config & PORT_HW_CFG_LANE_SWAP_CFG_MASTER_MASK) >> @@ -1628,39 +1608,28 @@ static elink_status_t elink_emac_enable(struct elink_params *params, elink_bits_en(sc, emac_base + EMAC_REG_EMAC_TX_MODE, EMAC_TX_MODE_RESET); -#if defined(ELINK_INCLUDE_EMUL) || defined(ELINK_INCLUDE_FPGA) - if (CHIP_REV_IS_SLOW(sc)) { - /* config GMII mode */ - val = REG_RD(sc, emac_base + EMAC_REG_EMAC_MODE); - elink_cb_reg_write(sc, emac_base + EMAC_REG_EMAC_MODE, - (val | EMAC_MODE_PORT_GMII)); - } else { /* ASIC */ -#endif - /* pause enable/disable */ - elink_bits_dis(sc, emac_base + EMAC_REG_EMAC_RX_MODE, - EMAC_RX_MODE_FLOW_EN); + /* pause enable/disable */ + elink_bits_dis(sc, emac_base + EMAC_REG_EMAC_RX_MODE, + EMAC_RX_MODE_FLOW_EN); - elink_bits_dis(sc, emac_base + EMAC_REG_EMAC_TX_MODE, - (EMAC_TX_MODE_EXT_PAUSE_EN | - EMAC_TX_MODE_FLOW_EN)); - if (!(params->feature_config_flags & - ELINK_FEATURE_CONFIG_PFC_ENABLED)) { - if (vars->flow_ctrl & ELINK_FLOW_CTRL_RX) - elink_bits_en(sc, emac_base + - EMAC_REG_EMAC_RX_MODE, - EMAC_RX_MODE_FLOW_EN); + elink_bits_dis(sc, emac_base + EMAC_REG_EMAC_TX_MODE, + (EMAC_TX_MODE_EXT_PAUSE_EN | + EMAC_TX_MODE_FLOW_EN)); + if (!(params->feature_config_flags & + ELINK_FEATURE_CONFIG_PFC_ENABLED)) { + if (vars->flow_ctrl & ELINK_FLOW_CTRL_RX) + elink_bits_en(sc, emac_base + + EMAC_REG_EMAC_RX_MODE, + EMAC_RX_MODE_FLOW_EN); - if (vars->flow_ctrl & ELINK_FLOW_CTRL_TX) - elink_bits_en(sc, emac_base + - EMAC_REG_EMAC_TX_MODE, - (EMAC_TX_MODE_EXT_PAUSE_EN | - EMAC_TX_MODE_FLOW_EN)); - } else - elink_bits_en(sc, emac_base + EMAC_REG_EMAC_TX_MODE, - EMAC_TX_MODE_FLOW_EN); -#if defined(ELINK_INCLUDE_EMUL) || defined(ELINK_INCLUDE_FPGA) - } -#endif + if (vars->flow_ctrl & ELINK_FLOW_CTRL_TX) + elink_bits_en(sc, emac_base + + EMAC_REG_EMAC_TX_MODE, + (EMAC_TX_MODE_EXT_PAUSE_EN | + EMAC_TX_MODE_FLOW_EN)); + } else + elink_bits_en(sc, emac_base + EMAC_REG_EMAC_TX_MODE, + EMAC_TX_MODE_FLOW_EN); /* KEEP_VLAN_TAG, promiscuous */ val = REG_RD(sc, emac_base + EMAC_REG_EMAC_RX_MODE); @@ -1727,17 +1696,7 @@ static elink_status_t elink_emac_enable(struct elink_params *params, REG_WR(sc, NIG_REG_EMAC0_PAUSE_OUT_EN + port * 4, val); REG_WR(sc, NIG_REG_EGRESS_EMAC0_OUT_EN + port * 4, 0x1); -#ifdef ELINK_INCLUDE_EMUL - if (CHIP_REV_IS_EMUL(sc)) { - /* Take the BigMac out of reset */ - REG_WR(sc, GRCBASE_MISC + MISC_REGISTERS_RESET_REG_2_SET, - (MISC_REGISTERS_RESET_REG_2_RST_BMAC0 << port)); - - /* Enable access for bmac registers */ - REG_WR(sc, NIG_REG_BMAC0_REGS_OUT_EN + port * 4, 0x1); - } else -#endif - REG_WR(sc, NIG_REG_BMAC0_REGS_OUT_EN + port * 4, 0x0); + REG_WR(sc, NIG_REG_BMAC0_REGS_OUT_EN + port * 4, 0x0); vars->mac_type = ELINK_MAC_TYPE_EMAC; return ELINK_STATUS_OK; @@ -2137,15 +2096,6 @@ static elink_status_t elink_bmac1_enable(struct elink_params *params, wb_data[1] = 0; REG_WR_DMAE(sc, bmac_addr + BIGMAC_REGISTER_RX_LLFC_MSG_FLDS, wb_data, 2); -#ifdef ELINK_INCLUDE_EMUL - /* Fix for emulation */ - if (CHIP_REV_IS_EMUL(sc)) { - wb_data[0] = 0xf000; - wb_data[1] = 0; - REG_WR_DMAE(sc, bmac_addr + BIGMAC_REGISTER_TX_PAUSE_THRESHOLD, - wb_data, 2); - } -#endif return ELINK_STATUS_OK; } @@ -2752,7 +2702,7 @@ static elink_status_t elink_eee_initial_config(struct elink_params *params, { vars->eee_status |= ((uint32_t) mode) << SHMEM_EEE_SUPPORTED_SHIFT; - /* Propogate params' bits --> vars (for migration exposure) */ + /* Propagate params' bits --> vars (for migration exposure) */ if (params->eee_mode & ELINK_EEE_MODE_ENABLE_LPI) vars->eee_status |= SHMEM_EEE_LPI_REQUESTED_BIT; else @@ -3500,7 +3450,7 @@ static void elink_warpcore_enable_AN_KR(struct elink_phy *phy, {MDIO_WC_DEVAD, MDIO_WC_REG_CL72_USERB0_CL72_TX_FIR_TAP, 0}, }; PMD_DRV_LOG(DEBUG, "Enable Auto Negotiation for KR"); - /* Set to default registers that may be overriden by 10G force */ + /* Set to default registers that may be overridden by 10G force */ for (i = 0; i < ARRAY_SIZE(reg_set); i++) elink_cl45_write(sc, phy, reg_set[i].devad, reg_set[i].reg, reg_set[i].val); @@ -4413,14 +4363,14 @@ static void elink_sync_link(struct elink_params *params, switch (vars->link_status & LINK_STATUS_SPEED_AND_DUPLEX_MASK) { case ELINK_LINK_10THD: vars->duplex = DUPLEX_HALF; - /* Fall thru */ + /* Fall through */ case ELINK_LINK_10TFD: vars->line_speed = ELINK_SPEED_10; break; case ELINK_LINK_100TXHD: vars->duplex = DUPLEX_HALF; - /* Fall thru */ + /* Fall through */ case ELINK_LINK_100T4: case ELINK_LINK_100TXFD: vars->line_speed = ELINK_SPEED_100; @@ -4428,14 +4378,14 @@ static void elink_sync_link(struct elink_params *params, case ELINK_LINK_1000THD: vars->duplex = DUPLEX_HALF; - /* Fall thru */ + /* Fall through */ case ELINK_LINK_1000TFD: vars->line_speed = ELINK_SPEED_1000; break; case ELINK_LINK_2500THD: vars->duplex = DUPLEX_HALF; - /* Fall thru */ + /* Fall through */ case ELINK_LINK_2500TFD: vars->line_speed = ELINK_SPEED_2500; break; @@ -5922,11 +5872,6 @@ elink_status_t elink_set_led(struct elink_params *params, params, mode); } } -#ifdef ELINK_INCLUDE_EMUL - if (params->feature_config_flags & - ELINK_FEATURE_CONFIG_EMUL_DISABLE_EMAC) - return rc; -#endif switch (mode) { case ELINK_LED_MODE_FRONT_PANEL_OFF: @@ -5953,6 +5898,7 @@ elink_status_t elink_set_led(struct elink_params *params, */ if (!vars->link_up) break; + /* fall-through */ case ELINK_LED_MODE_ON: if (((params->phy[ELINK_EXT_PHY1].type == PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BNX2X8727) || @@ -6395,7 +6341,7 @@ elink_status_t elink_link_update(struct elink_params * params, * hence its link is expected to be down * - SECOND_PHY means that first phy should not be able * to link up by itself (using configuration) - * - DEFAULT should be overriden during initialiazation + * - DEFAULT should be overridden during initialization */ PMD_DRV_LOG(DEBUG, "Invalid link indication" "mpc=0x%x. DISABLING LINK !!!", @@ -11589,11 +11535,13 @@ static void elink_phy_def_cfg(struct elink_params *params, switch (link_config & PORT_FEATURE_LINK_SPEED_MASK) { case PORT_FEATURE_LINK_SPEED_10M_HALF: phy->req_duplex = DUPLEX_HALF; + /* fall-through */ case PORT_FEATURE_LINK_SPEED_10M_FULL: phy->req_line_speed = ELINK_SPEED_10; break; case PORT_FEATURE_LINK_SPEED_100M_HALF: phy->req_duplex = DUPLEX_HALF; + /* fall-through */ case PORT_FEATURE_LINK_SPEED_100M_FULL: phy->req_line_speed = ELINK_SPEED_100; break; @@ -11671,10 +11619,7 @@ elink_status_t elink_phy_probe(struct elink_params * params) struct elink_phy *phy; params->num_phys = 0; PMD_DRV_LOG(DEBUG, "Begin phy probe"); -#ifdef ELINK_INCLUDE_EMUL - if (CHIP_REV_IS_EMUL(sc)) - return ELINK_STATUS_OK; -#endif + phy_config_swapped = params->multi_phy_config & PORT_HW_CFG_PHY_SWAPPED_ENABLED; @@ -11739,182 +11684,6 @@ elink_status_t elink_phy_probe(struct elink_params * params) return ELINK_STATUS_OK; } -#ifdef ELINK_INCLUDE_EMUL -static elink_status_t elink_init_e3_emul_mac(struct elink_params *params, - struct elink_vars *vars) -{ - struct bnx2x_softc *sc = params->sc; - vars->line_speed = params->req_line_speed[0]; - /* In case link speed is auto, set speed the highest as possible */ - if (params->req_line_speed[0] == ELINK_SPEED_AUTO_NEG) { - if (params->feature_config_flags & - ELINK_FEATURE_CONFIG_EMUL_DISABLE_XMAC) - vars->line_speed = ELINK_SPEED_2500; - else if (elink_is_4_port_mode(sc)) - vars->line_speed = ELINK_SPEED_10000; - else - vars->line_speed = ELINK_SPEED_20000; - } - if (vars->line_speed < ELINK_SPEED_10000) { - if ((params->feature_config_flags & - ELINK_FEATURE_CONFIG_EMUL_DISABLE_UMAC)) { - PMD_DRV_LOG(DEBUG, "Invalid line speed %d while UMAC is" - " disabled!", params->req_line_speed[0]); - return ELINK_STATUS_ERROR; - } - switch (vars->line_speed) { - case ELINK_SPEED_10: - vars->link_status = ELINK_LINK_10TFD; - break; - case ELINK_SPEED_100: - vars->link_status = ELINK_LINK_100TXFD; - break; - case ELINK_SPEED_1000: - vars->link_status = ELINK_LINK_1000TFD; - break; - case ELINK_SPEED_2500: - vars->link_status = ELINK_LINK_2500TFD; - break; - default: - PMD_DRV_LOG(DEBUG, "Invalid line speed %d for UMAC", - vars->line_speed); - return ELINK_STATUS_ERROR; - } - vars->link_status |= LINK_STATUS_LINK_UP; - - if (params->loopback_mode == ELINK_LOOPBACK_UMAC) - elink_umac_enable(params, vars, 1); - else - elink_umac_enable(params, vars, 0); - } else { - /* Link speed >= 10000 requires XMAC enabled */ - if (params->feature_config_flags & - ELINK_FEATURE_CONFIG_EMUL_DISABLE_XMAC) { - PMD_DRV_LOG(DEBUG, "Invalid line speed %d while XMAC is" - " disabled!", params->req_line_speed[0]); - return ELINK_STATUS_ERROR; - } - /* Check link speed */ - switch (vars->line_speed) { - case ELINK_SPEED_10000: - vars->link_status = ELINK_LINK_10GTFD; - break; - case ELINK_SPEED_20000: - vars->link_status = ELINK_LINK_20GTFD; - break; - default: - PMD_DRV_LOG(DEBUG, "Invalid line speed %d for XMAC", - vars->line_speed); - return ELINK_STATUS_ERROR; - } - vars->link_status |= LINK_STATUS_LINK_UP; - if (params->loopback_mode == ELINK_LOOPBACK_XMAC) - elink_xmac_enable(params, vars, 1); - else - elink_xmac_enable(params, vars, 0); - } - return ELINK_STATUS_OK; -} - -static elink_status_t elink_init_emul(struct elink_params *params, - struct elink_vars *vars) -{ - struct bnx2x_softc *sc = params->sc; - if (CHIP_IS_E3(sc)) { - if (elink_init_e3_emul_mac(params, vars) != ELINK_STATUS_OK) - return ELINK_STATUS_ERROR; - } else { - if (params->feature_config_flags & - ELINK_FEATURE_CONFIG_EMUL_DISABLE_BMAC) { - vars->line_speed = ELINK_SPEED_1000; - vars->link_status = (LINK_STATUS_LINK_UP | - ELINK_LINK_1000XFD); - if (params->loopback_mode == ELINK_LOOPBACK_EMAC) - elink_emac_enable(params, vars, 1); - else - elink_emac_enable(params, vars, 0); - } else { - vars->line_speed = ELINK_SPEED_10000; - vars->link_status = (LINK_STATUS_LINK_UP | - ELINK_LINK_10GTFD); - if (params->loopback_mode == ELINK_LOOPBACK_BMAC) - elink_bmac_enable(params, vars, 1, 1); - else - elink_bmac_enable(params, vars, 0, 1); - } - } - vars->link_up = 1; - vars->duplex = DUPLEX_FULL; - vars->flow_ctrl = ELINK_FLOW_CTRL_NONE; - - if (CHIP_IS_E1x(sc)) - elink_pbf_update(params, vars->flow_ctrl, vars->line_speed); - /* Disable drain */ - REG_WR(sc, NIG_REG_EGRESS_DRAIN0_MODE + params->port * 4, 0); - - /* update shared memory */ - elink_update_mng(params, vars->link_status); - return ELINK_STATUS_OK; -} -#endif -#ifdef ELINK_INCLUDE_FPGA -static elink_status_t elink_init_fpga(struct elink_params *params, - struct elink_vars *vars) -{ - /* Enable on E1.5 FPGA */ - struct bnx2x_softc *sc = params->sc; - vars->duplex = DUPLEX_FULL; - vars->flow_ctrl = ELINK_FLOW_CTRL_NONE; - vars->flow_ctrl = (ELINK_FLOW_CTRL_TX | ELINK_FLOW_CTRL_RX); - vars->link_status |= (LINK_STATUS_TX_FLOW_CONTROL_ENABLED | - LINK_STATUS_RX_FLOW_CONTROL_ENABLED); - if (CHIP_IS_E3(sc)) { - vars->line_speed = params->req_line_speed[0]; - switch (vars->line_speed) { - case ELINK_SPEED_AUTO_NEG: - vars->line_speed = ELINK_SPEED_2500; - case ELINK_SPEED_2500: - vars->link_status = ELINK_LINK_2500TFD; - break; - case ELINK_SPEED_1000: - vars->link_status = ELINK_LINK_1000XFD; - break; - case ELINK_SPEED_100: - vars->link_status = ELINK_LINK_100TXFD; - break; - case ELINK_SPEED_10: - vars->link_status = ELINK_LINK_10TFD; - break; - default: - PMD_DRV_LOG(DEBUG, "Invalid link speed %d", - params->req_line_speed[0]); - return ELINK_STATUS_ERROR; - } - vars->link_status |= LINK_STATUS_LINK_UP; - if (params->loopback_mode == ELINK_LOOPBACK_UMAC) - elink_umac_enable(params, vars, 1); - else - elink_umac_enable(params, vars, 0); - } else { - vars->line_speed = ELINK_SPEED_10000; - vars->link_status = (LINK_STATUS_LINK_UP | ELINK_LINK_10GTFD); - if (params->loopback_mode == ELINK_LOOPBACK_EMAC) - elink_emac_enable(params, vars, 1); - else - elink_emac_enable(params, vars, 0); - } - vars->link_up = 1; - - if (CHIP_IS_E1x(sc)) - elink_pbf_update(params, vars->flow_ctrl, vars->line_speed); - /* Disable drain */ - REG_WR(sc, NIG_REG_EGRESS_DRAIN0_MODE + params->port * 4, 0); - - /* Update shared memory */ - elink_update_mng(params, vars->link_status); - return ELINK_STATUS_OK; -} -#endif static void elink_init_bmac_loopback(struct elink_params *params, struct elink_vars *vars) { @@ -12236,12 +12005,8 @@ elink_status_t elink_phy_init(struct elink_params *params, ELINK_NIG_MASK_XGXS0_LINK10G | ELINK_NIG_MASK_SERDES0_LINK_STATUS | ELINK_NIG_MASK_MI_INT)); -#ifdef ELINK_INCLUDE_EMUL - if (!(params->feature_config_flags & - ELINK_FEATURE_CONFIG_EMUL_DISABLE_EMAC)) -#endif - elink_emac_init(params); + elink_emac_init(params); if (params->feature_config_flags & ELINK_FEATURE_CONFIG_PFC_ENABLED) vars->link_status |= LINK_STATUS_PFC_ENABLED; @@ -12253,45 +12018,36 @@ elink_status_t elink_phy_init(struct elink_params *params, set_phy_vars(params, vars); PMD_DRV_LOG(DEBUG, "Num of phys on board: %d", params->num_phys); -#ifdef ELINK_INCLUDE_FPGA - if (CHIP_REV_IS_FPGA(sc)) { - return elink_init_fpga(params, vars); - } else -#endif -#ifdef ELINK_INCLUDE_EMUL - if (CHIP_REV_IS_EMUL(sc)) { - return elink_init_emul(params, vars); - } else -#endif - switch (params->loopback_mode) { - case ELINK_LOOPBACK_BMAC: - elink_init_bmac_loopback(params, vars); - break; - case ELINK_LOOPBACK_EMAC: - elink_init_emac_loopback(params, vars); - break; - case ELINK_LOOPBACK_XMAC: - elink_init_xmac_loopback(params, vars); - break; - case ELINK_LOOPBACK_UMAC: - elink_init_umac_loopback(params, vars); - break; - case ELINK_LOOPBACK_XGXS: - case ELINK_LOOPBACK_EXT_PHY: - elink_init_xgxs_loopback(params, vars); - break; - default: - if (!CHIP_IS_E3(sc)) { - if (params->switch_cfg == ELINK_SWITCH_CFG_10G) - elink_xgxs_deassert(params); - else - elink_serdes_deassert(sc, params->port); - } - elink_link_initialize(params, vars); - DELAY(1000 * 30); - elink_link_int_enable(params); - break; + + switch (params->loopback_mode) { + case ELINK_LOOPBACK_BMAC: + elink_init_bmac_loopback(params, vars); + break; + case ELINK_LOOPBACK_EMAC: + elink_init_emac_loopback(params, vars); + break; + case ELINK_LOOPBACK_XMAC: + elink_init_xmac_loopback(params, vars); + break; + case ELINK_LOOPBACK_UMAC: + elink_init_umac_loopback(params, vars); + break; + case ELINK_LOOPBACK_XGXS: + case ELINK_LOOPBACK_EXT_PHY: + elink_init_xgxs_loopback(params, vars); + break; + default: + if (!CHIP_IS_E3(sc)) { + if (params->switch_cfg == ELINK_SWITCH_CFG_10G) + elink_xgxs_deassert(params); + else + elink_serdes_deassert(sc, params->port); } + elink_link_initialize(params, vars); + DELAY(1000 * 30); + elink_link_int_enable(params); + break; + } elink_update_mng(params, vars->link_status); elink_update_mng_eee(params, vars->eee_status); @@ -12325,22 +12081,12 @@ static elink_status_t elink_link_reset(struct elink_params *params, REG_WR(sc, NIG_REG_BMAC0_OUT_EN + port * 4, 0); REG_WR(sc, NIG_REG_EGRESS_EMAC0_OUT_EN + port * 4, 0); } -#ifdef ELINK_INCLUDE_EMUL - /* Stop BigMac rx */ - if (!(params->feature_config_flags & - ELINK_FEATURE_CONFIG_EMUL_DISABLE_BMAC)) -#endif - if (!CHIP_IS_E3(sc)) - elink_set_bmac_rx(sc, port, 0); -#ifdef ELINK_INCLUDE_EMUL - /* Stop XMAC/UMAC rx */ - if (!(params->feature_config_flags & - ELINK_FEATURE_CONFIG_EMUL_DISABLE_XMAC)) -#endif - if (CHIP_IS_E3(sc) && !CHIP_REV_IS_FPGA(sc)) { - elink_set_xmac_rxtx(params, 0); - elink_set_umac_rxtx(params, 0); - } + if (!CHIP_IS_E3(sc)) + elink_set_bmac_rx(sc, port, 0); + if (CHIP_IS_E3(sc) && !CHIP_REV_IS_FPGA(sc)) { + elink_set_xmac_rxtx(params, 0); + elink_set_umac_rxtx(params, 0); + } /* Disable emac */ if (!CHIP_IS_E3(sc)) REG_WR(sc, NIG_REG_NIG_EMAC0_EN + port * 4, 0); @@ -12376,14 +12122,11 @@ static elink_status_t elink_link_reset(struct elink_params *params, elink_bits_dis(sc, NIG_REG_LATCH_BC_0 + port * 4, 1 << ELINK_NIG_LATCH_BC_ENABLE_MI_INT); } -#if defined(ELINK_INCLUDE_EMUL) || defined(ELINK_INCLUDE_FPGA) - if (!CHIP_REV_IS_SLOW(sc)) -#endif - if (params->phy[ELINK_INT_PHY].link_reset) - params->phy[ELINK_INT_PHY].link_reset(¶ms-> - phy - [ELINK_INT_PHY], - params); + if (params->phy[ELINK_INT_PHY].link_reset) + params->phy[ELINK_INT_PHY].link_reset(¶ms-> + phy + [ELINK_INT_PHY], + params); /* Disable nig ingress interface */ if (!CHIP_IS_E3(sc)) { @@ -12868,10 +12611,6 @@ elink_status_t elink_common_init_phy(struct bnx2x_softc * sc, uint32_t phy_ver, val; uint8_t phy_index = 0; uint32_t ext_phy_type, ext_phy_config; -#if defined(ELINK_INCLUDE_EMUL) || defined(ELINK_INCLUDE_FPGA) - if (CHIP_REV_IS_EMUL(sc) || CHIP_REV_IS_FPGA(sc)) - return ELINK_STATUS_OK; -#endif elink_set_mdio_clk(sc, GRCBASE_EMAC0); elink_set_mdio_clk(sc, GRCBASE_EMAC1); @@ -12941,7 +12680,7 @@ static void elink_check_over_curr(struct elink_params *params, vars->phy_flags &= ~PHY_OVER_CURRENT_FLAG; } -/* Returns 0 if no change occured since last check; 1 otherwise. */ +/* Returns 0 if no change occurred since last check; 1 otherwise. */ static uint8_t elink_analyze_link_error(struct elink_params *params, struct elink_vars *vars, uint32_t status, uint32_t phy_flag, diff --git a/dpdk/drivers/net/bnx2x/elink.h b/dpdk/drivers/net/bnx2x/elink.h index c4f886a7..9401b7cd 100644 --- a/dpdk/drivers/net/bnx2x/elink.h +++ b/dpdk/drivers/net/bnx2x/elink.h @@ -359,10 +359,6 @@ struct elink_params { #define ELINK_FEATURE_CONFIG_PFC_ENABLED (1<<1) #define ELINK_FEATURE_CONFIG_BC_SUPPORTS_OPT_MDL_VRFY (1<<2) #define ELINK_FEATURE_CONFIG_BC_SUPPORTS_DUAL_PHY_OPT_MDL_VRFY (1<<3) -#define ELINK_FEATURE_CONFIG_EMUL_DISABLE_EMAC (1<<4) -#define ELINK_FEATURE_CONFIG_EMUL_DISABLE_BMAC (1<<5) -#define ELINK_FEATURE_CONFIG_EMUL_DISABLE_UMAC (1<<6) -#define ELINK_FEATURE_CONFIG_EMUL_DISABLE_XMAC (1<<7) #define ELINK_FEATURE_CONFIG_BC_SUPPORTS_AFEX (1<<8) #define ELINK_FEATURE_CONFIG_AUTOGREEEN_ENABLED (1<<9) #define ELINK_FEATURE_CONFIG_BC_SUPPORTS_SFP_TX_DISABLED (1<<10) diff --git a/dpdk/drivers/net/bnxt/Makefile b/dpdk/drivers/net/bnxt/Makefile index d9c5a4cb..2aa04411 100644 --- a/dpdk/drivers/net/bnxt/Makefile +++ b/dpdk/drivers/net/bnxt/Makefile @@ -38,10 +38,15 @@ include $(RTE_SDK)/mk/rte.vars.mk # LIB = librte_pmd_bnxt.a -LIBABIVER := 1 +EXPORT_MAP := rte_pmd_bnxt_version.map + +LIBABIVER := 2 CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs +LDLIBS += -lrte_bus_pci EXPORT_MAP := rte_pmd_bnxt_version.map @@ -59,16 +64,13 @@ SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += bnxt_stats.c SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += bnxt_txq.c SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += bnxt_txr.c SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += bnxt_vnic.c +SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += bnxt_irq.c +SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += rte_pmd_bnxt.c # # Export include files # SYMLINK-y-include += - -# this lib depends upon: -DEPDIRS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += lib/librte_mbuf -DEPDIRS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += lib/librte_mempool -DEPDIRS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += lib/librte_ether -DEPDIRS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += lib/librte_eal +SYMLINK-$(CONFIG_RTE_LIBRTE_BNXT_PMD)-include := rte_pmd_bnxt.h include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/net/bnxt/bnxt.h b/dpdk/drivers/net/bnxt/bnxt.h index 0e21aceb..3bc2b937 100644 --- a/dpdk/drivers/net/bnxt/bnxt.h +++ b/dpdk/drivers/net/bnxt/bnxt.h @@ -35,8 +35,11 @@ #define _BNXT_H_ #include +#include #include +#include +#include #include #include #include @@ -44,8 +47,44 @@ #include "bnxt_cpr.h" -#define BNXT_MAX_MTU 9000 +#define BNXT_MAX_MTU 9500 #define VLAN_TAG_SIZE 4 +#define BNXT_MAX_LED 4 + +struct bnxt_led_info { + uint8_t led_id; + uint8_t led_type; + uint8_t led_group_id; + uint8_t unused; + uint16_t led_state_caps; +#define BNXT_LED_ALT_BLINK_CAP(x) ((x) & \ + rte_cpu_to_le_16(HWRM_PORT_LED_QCFG_OUTPUT_LED0_STATE_BLINKALT)) + + uint16_t led_color_caps; +}; + +struct bnxt_led_cfg { + uint8_t led_id; + uint8_t led_state; + uint8_t led_color; + uint8_t unused; + uint16_t led_blink_on; + uint16_t led_blink_off; + uint8_t led_group_id; + uint8_t rsvd; +}; + +#define BNXT_LED_DFLT_ENA \ + (HWRM_PORT_LED_CFG_INPUT_ENABLES_LED0_ID | \ + HWRM_PORT_LED_CFG_INPUT_ENABLES_LED0_STATE | \ + HWRM_PORT_LED_CFG_INPUT_ENABLES_LED0_BLINK_ON | \ + HWRM_PORT_LED_CFG_INPUT_ENABLES_LED0_BLINK_OFF | \ + HWRM_PORT_LED_CFG_INPUT_ENABLES_LED0_GROUP_ID) + +#define BNXT_LED_DFLT_ENA_SHIFT 6 + +#define BNXT_LED_DFLT_ENABLES(x) \ + rte_cpu_to_le_32(BNXT_LED_DFLT_ENA << (BNXT_LED_DFLT_ENA_SHIFT * (x))) enum bnxt_hw_context { HW_CONTEXT_NONE = 0, @@ -54,16 +93,32 @@ enum bnxt_hw_context { HW_CONTEXT_IS_LB = 3, }; -struct bnxt_vf_info { - uint16_t fw_fid; - uint8_t mac_addr[ETHER_ADDR_LEN]; - uint16_t max_rsscos_ctx; - uint16_t max_cp_rings; - uint16_t max_tx_rings; - uint16_t max_rx_rings; - uint16_t max_l2_ctx; - uint16_t max_vnics; - struct bnxt_pf_info *pf; +struct bnxt_vlan_table_entry { + uint16_t tpid; + uint16_t vid; +} __attribute__((packed)); + +struct bnxt_vlan_antispoof_table_entry { + uint16_t tpid; + uint16_t vid; + uint16_t mask; +} __attribute__((packed)); + +struct bnxt_child_vf_info { + void *req_buf; + struct bnxt_vlan_table_entry *vlan_table; + struct bnxt_vlan_antispoof_table_entry *vlan_as_table; + STAILQ_HEAD(, bnxt_filter_info) filter; + uint32_t func_cfg_flags; + uint32_t l2_rx_mask; + uint16_t fid; + uint16_t max_tx_rate; + uint16_t dflt_vlan; + uint16_t vlan_count; + uint8_t mac_spoof_en; + uint8_t vlan_spoof_en; + bool random_mac; + bool persist_stats; }; struct bnxt_pf_info { @@ -72,29 +127,27 @@ struct bnxt_pf_info { #define BNXT_FIRST_VF_FID 128 #define BNXT_PF_RINGS_USED(bp) bnxt_get_num_queues(bp) #define BNXT_PF_RINGS_AVAIL(bp) (bp->pf.max_cp_rings - BNXT_PF_RINGS_USED(bp)) - uint32_t fw_fid; - uint8_t port_id; - uint8_t mac_addr[ETHER_ADDR_LEN]; - uint16_t max_rsscos_ctx; - uint16_t max_cp_rings; - uint16_t max_tx_rings; - uint16_t max_rx_rings; - uint16_t max_l2_ctx; - uint16_t max_vnics; + uint16_t port_id; uint16_t first_vf_id; uint16_t active_vfs; uint16_t max_vfs; + uint32_t func_cfg_flags; void *vf_req_buf; - phys_addr_t vf_req_buf_dma_addr; + rte_iova_t vf_req_buf_dma_addr; uint32_t vf_req_fwd[8]; - struct bnxt_vf_info *vf; + uint16_t total_vnics; + struct bnxt_child_vf_info *vf_info; +#define BNXT_EVB_MODE_NONE 0 +#define BNXT_EVB_MODE_VEB 1 +#define BNXT_EVB_MODE_VEPA 2 + uint8_t evb_mode; }; /* Max wait time is 10 * 100ms = 1s */ #define BNXT_LINK_WAIT_CNT 10 #define BNXT_LINK_WAIT_INTERVAL 100 struct bnxt_link_info { - uint8_t phy_flags; + uint32_t phy_flags; uint8_t mac_type; uint8_t phy_link_status; uint8_t loop_back; @@ -109,8 +162,11 @@ struct bnxt_link_info { uint16_t link_speed; uint16_t support_speeds; uint16_t auto_link_speed; + uint16_t force_link_speed; uint16_t auto_link_speed_mask; uint32_t preemphasis; + uint8_t phy_type; + uint8_t media_type; }; #define BNXT_COS_QUEUE_COUNT 8 @@ -119,25 +175,45 @@ struct bnxt_cos_queue_info { uint8_t profile; }; +struct rte_flow { + STAILQ_ENTRY(rte_flow) next; + struct bnxt_filter_info *filter; + struct bnxt_vnic_info *vnic; +}; + +#define BNXT_HWRM_SHORT_REQ_LEN sizeof(struct hwrm_short_input) struct bnxt { void *bar0; struct rte_eth_dev *eth_dev; + struct rte_eth_rss_conf rss_conf; struct rte_pci_device *pdev; uint32_t flags; #define BNXT_FLAG_REGISTERED (1 << 0) #define BNXT_FLAG_VF (1 << 1) +#define BNXT_FLAG_PORT_STATS (1 << 2) +#define BNXT_FLAG_JUMBO (1 << 3) +#define BNXT_FLAG_SHORT_CMD (1 << 4) +#define BNXT_FLAG_UPDATE_HASH (1 << 5) #define BNXT_PF(bp) (!((bp)->flags & BNXT_FLAG_VF)) #define BNXT_VF(bp) ((bp)->flags & BNXT_FLAG_VF) +#define BNXT_NPAR_ENABLED(bp) ((bp)->port_partition_type) +#define BNXT_NPAR_PF(bp) (BNXT_PF(bp) && BNXT_NPAR_ENABLED(bp)) unsigned int rx_nr_rings; unsigned int rx_cp_nr_rings; struct bnxt_rx_queue **rx_queues; + const void *rx_mem_zone; + struct rx_port_stats *hw_rx_port_stats; + rte_iova_t hw_rx_port_stats_map; unsigned int tx_nr_rings; unsigned int tx_cp_nr_rings; struct bnxt_tx_queue **tx_queues; + const void *tx_mem_zone; + struct tx_port_stats *hw_tx_port_stats; + rte_iova_t hw_tx_port_stats_map; /* Default completion ring */ struct bnxt_cp_ring_info *def_cp_ring; @@ -153,15 +229,19 @@ struct bnxt { STAILQ_HEAD(, bnxt_filter_info) free_filter_list; /* VNIC pointer for flow filter (VMDq) pools */ -#define MAX_FF_POOLS ETH_64_POOLS +#define MAX_FF_POOLS 256 STAILQ_HEAD(, bnxt_vnic_info) ff_pool[MAX_FF_POOLS]; + struct bnxt_irq *irq_tbl; + #define MAX_NUM_MAC_ADDR 32 uint8_t mac_addr[ETHER_ADDR_LEN]; uint16_t hwrm_cmd_seq; void *hwrm_cmd_resp_addr; - phys_addr_t hwrm_cmd_resp_dma_addr; + rte_iova_t hwrm_cmd_resp_dma_addr; + void *hwrm_short_cmd_req_addr; + rte_iova_t hwrm_short_cmd_req_dma_addr; rte_spinlock_t hwrm_lock; uint16_t max_req_len; uint16_t max_resp_len; @@ -169,9 +249,37 @@ struct bnxt { struct bnxt_link_info link_info; struct bnxt_cos_queue_info cos_queue[BNXT_COS_QUEUE_COUNT]; + uint16_t fw_fid; + uint8_t dflt_mac_addr[ETHER_ADDR_LEN]; + uint16_t max_rsscos_ctx; + uint16_t max_cp_rings; + uint16_t max_tx_rings; + uint16_t max_rx_rings; + uint16_t max_l2_ctx; + uint16_t max_vnics; + uint16_t max_stat_ctx; + uint16_t vlan; struct bnxt_pf_info pf; - struct bnxt_vf_info vf; + uint8_t port_partition_type; uint8_t dev_stopped; + uint8_t vxlan_port_cnt; + uint8_t geneve_port_cnt; + uint16_t vxlan_port; + uint16_t geneve_port; + uint16_t vxlan_fw_dst_port_id; + uint16_t geneve_fw_dst_port_id; + uint32_t fw_ver; + rte_atomic64_t rx_mbuf_alloc_fail; + + struct bnxt_led_info leds[BNXT_MAX_LED]; + uint8_t num_leds; }; +int bnxt_link_update_op(struct rte_eth_dev *eth_dev, int wait_to_complete); +int bnxt_rcv_msg_from_vf(struct bnxt *bp, uint16_t vf_id, void *msg); + +#define RX_PROD_AGG_BD_TYPE_RX_PROD_AGG 0x6 + +bool is_bnxt_supported(struct rte_eth_dev *dev); +extern const struct rte_flow_ops bnxt_flow_ops; #endif diff --git a/dpdk/drivers/net/bnxt/bnxt_cpr.c b/dpdk/drivers/net/bnxt/bnxt_cpr.c index 60c277a1..cde8adc3 100644 --- a/dpdk/drivers/net/bnxt/bnxt_cpr.c +++ b/dpdk/drivers/net/bnxt/bnxt_cpr.c @@ -42,91 +42,150 @@ /* * Async event handling */ -void bnxt_handle_async_event(struct bnxt *bp __rte_unused, +void bnxt_handle_async_event(struct bnxt *bp, struct cmpl_base *cmp) { struct hwrm_async_event_cmpl *async_cmp = (struct hwrm_async_event_cmpl *)cmp; + uint16_t event_id = rte_le_to_cpu_16(async_cmp->event_id); /* TODO: HWRM async events are not defined yet */ /* Needs to handle: link events, error events, etc. */ - switch (async_cmp->event_id) { - case 0: - /* Assume LINK_CHANGE == 0 */ - RTE_LOG(INFO, PMD, "Link change event\n"); - - /* Can just prompt the update_op routine to do a qcfg - * instead of doing the actual qcfg - */ - break; - case 1: + switch (event_id) { + case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE: + case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE: + case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE: + bnxt_link_update_op(bp->eth_dev, 1); break; default: - RTE_LOG(ERR, PMD, "handle_async_event id = 0x%x\n", - async_cmp->event_id); + RTE_LOG(DEBUG, PMD, "handle_async_event id = 0x%x\n", event_id); break; } } void bnxt_handle_fwd_req(struct bnxt *bp, struct cmpl_base *cmpl) { + struct hwrm_exec_fwd_resp_input *fwreq; struct hwrm_fwd_req_cmpl *fwd_cmpl = (struct hwrm_fwd_req_cmpl *)cmpl; struct input *fwd_cmd; - uint16_t logical_vf_id, error_code; + uint16_t fw_vf_id; + uint16_t vf_id; + uint16_t req_len; + int rc; + + if (bp->pf.active_vfs <= 0) { + RTE_LOG(ERR, PMD, "Forwarded VF with no active VFs\n"); + return; + } /* Qualify the fwd request */ - if (fwd_cmpl->source_id < bp->pf.first_vf_id) { - RTE_LOG(ERR, PMD, - "FWD req's source_id 0x%x > first_vf_id 0x%x\n", - fwd_cmpl->source_id, bp->pf.first_vf_id); - error_code = HWRM_ERR_CODE_RESOURCE_ACCESS_DENIED; - goto reject; - } else if (fwd_cmpl->req_len_type >> HWRM_FWD_REQ_CMPL_REQ_LEN_SFT > - 128 - sizeof(struct input)) { - RTE_LOG(ERR, PMD, - "FWD req's cmd len 0x%x > 108 bytes allowed\n", - fwd_cmpl->req_len_type >> HWRM_FWD_REQ_CMPL_REQ_LEN_SFT); - error_code = HWRM_ERR_CODE_INVALID_PARAMS; - goto reject; - } + fw_vf_id = rte_le_to_cpu_16(fwd_cmpl->source_id); + vf_id = fw_vf_id - bp->pf.first_vf_id; + + req_len = (rte_le_to_cpu_16(fwd_cmpl->req_len_type) & + HWRM_FWD_REQ_CMPL_REQ_LEN_MASK) >> + HWRM_FWD_REQ_CMPL_REQ_LEN_SFT; + if (req_len > sizeof(fwreq->encap_request)) + req_len = sizeof(fwreq->encap_request); /* Locate VF's forwarded command */ - logical_vf_id = fwd_cmpl->source_id - bp->pf.first_vf_id; - fwd_cmd = (struct input *)((uint8_t *)bp->pf.vf_req_buf + - (logical_vf_id * 128)); + fwd_cmd = (struct input *)bp->pf.vf_info[vf_id].req_buf; - /* Provision the request */ - switch (fwd_cmd->req_type) { - case HWRM_CFA_L2_FILTER_ALLOC: - case HWRM_CFA_L2_FILTER_FREE: - case HWRM_CFA_L2_FILTER_CFG: - case HWRM_CFA_L2_SET_RX_MASK: - break; - default: - error_code = HWRM_ERR_CODE_INVALID_PARAMS; + if (fw_vf_id < bp->pf.first_vf_id || + fw_vf_id >= (bp->pf.first_vf_id) + bp->pf.active_vfs) { + RTE_LOG(ERR, PMD, + "FWD req's source_id 0x%x out of range 0x%x - 0x%x (%d %d)\n", + fw_vf_id, bp->pf.first_vf_id, + (bp->pf.first_vf_id) + bp->pf.active_vfs - 1, + bp->pf.first_vf_id, bp->pf.active_vfs); goto reject; } - /* Forward */ - fwd_cmd->target_id = fwd_cmpl->source_id; - bnxt_hwrm_exec_fwd_resp(bp, fwd_cmd); - return; + if (bnxt_rcv_msg_from_vf(bp, vf_id, fwd_cmd) == true) { + /* + * In older firmware versions, the MAC had to be all zeros for + * the VF to set it's MAC via hwrm_func_vf_cfg. Set to all + * zeros if it's being configured and has been ok'd by caller. + */ + if (fwd_cmd->req_type == HWRM_FUNC_VF_CFG) { + struct hwrm_func_vf_cfg_input *vfc = (void *)fwd_cmd; + + if (vfc->enables & + HWRM_FUNC_VF_CFG_INPUT_ENABLES_DFLT_MAC_ADDR) { + bnxt_hwrm_func_vf_mac(bp, vf_id, + (const uint8_t *)"\x00\x00\x00\x00\x00"); + } + } + if (fwd_cmd->req_type == HWRM_CFA_L2_SET_RX_MASK) { + struct hwrm_cfa_l2_set_rx_mask_input *srm = + (void *)fwd_cmd; + + srm->vlan_tag_tbl_addr = rte_cpu_to_le_64(0); + srm->num_vlan_tags = rte_cpu_to_le_32(0); + srm->mask &= ~rte_cpu_to_le_32( + HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_VLANONLY | + HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_VLAN_NONVLAN | + HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_ANYVLAN_NONVLAN); + } + /* Forward */ + rc = bnxt_hwrm_exec_fwd_resp(bp, fw_vf_id, fwd_cmd, req_len); + if (rc) { + RTE_LOG(ERR, PMD, + "Failed to send FWD req VF 0x%x, type 0x%x.\n", + fw_vf_id - bp->pf.first_vf_id, + rte_le_to_cpu_16(fwd_cmd->req_type)); + } + return; + } reject: - /* TODO: Encap the reject error resp into the hwrm_err_iput? */ - /* Use the error_code for the reject cmd */ - RTE_LOG(ERR, PMD, - "Error 0x%x found in the forward request\n", error_code); + rc = bnxt_hwrm_reject_fwd_resp(bp, fw_vf_id, fwd_cmd, req_len); + if (rc) { + RTE_LOG(ERR, PMD, + "Failed to send REJECT req VF 0x%x, type 0x%x.\n", + fw_vf_id - bp->pf.first_vf_id, + rte_le_to_cpu_16(fwd_cmd->req_type)); + } + + return; } /* For the default completion ring only */ +int bnxt_alloc_def_cp_ring(struct bnxt *bp) +{ + struct bnxt_cp_ring_info *cpr = bp->def_cp_ring; + struct bnxt_ring *cp_ring = cpr->cp_ring_struct; + int rc; + + rc = bnxt_hwrm_ring_alloc(bp, cp_ring, + HWRM_RING_ALLOC_INPUT_RING_TYPE_L2_CMPL, + 0, HWRM_NA_SIGNATURE, + HWRM_NA_SIGNATURE); + if (rc) + goto err_out; + cpr->cp_doorbell = bp->pdev->mem_resource[2].addr; + B_CP_DIS_DB(cpr, cpr->cp_raw_cons); + if (BNXT_PF(bp)) + rc = bnxt_hwrm_func_cfg_def_cp(bp); + else + rc = bnxt_hwrm_vf_func_cfg_def_cp(bp); + +err_out: + return rc; +} + void bnxt_free_def_cp_ring(struct bnxt *bp) { struct bnxt_cp_ring_info *cpr = bp->def_cp_ring; + if (cpr == NULL) + return; + bnxt_free_ring(cpr->cp_ring_struct); + cpr->cp_ring_struct = NULL; rte_free(cpr->cp_ring_struct); rte_free(cpr); + bp->def_cp_ring = NULL; } /* For the default completion ring only */ diff --git a/dpdk/drivers/net/bnxt/bnxt_cpr.h b/dpdk/drivers/net/bnxt/bnxt_cpr.h index c176f8c5..ce2b0cb8 100644 --- a/dpdk/drivers/net/bnxt/bnxt_cpr.h +++ b/dpdk/drivers/net/bnxt/bnxt_cpr.h @@ -33,11 +33,17 @@ #ifndef _BNXT_CPR_H_ #define _BNXT_CPR_H_ +#include + +#include #define CMP_VALID(cmp, raw_cons, ring) \ (!!(((struct cmpl_base *)(cmp))->info3_v & CMPL_BASE_V) == \ !((raw_cons) & ((ring)->ring_size))) +#define CMPL_VALID(cmp, v) \ + (!!(((struct cmpl_base *)(cmp))->info3_v & CMPL_BASE_V) == !(v)) + #define CMP_TYPE(cmp) \ (((struct cmpl_base *)cmp)->type & CMPL_BASE_TYPE_MASK) @@ -45,17 +51,33 @@ #define NEXT_RAW_CMP(idx) ADV_RAW_CMP(idx, 1) #define RING_CMP(ring, idx) ((idx) & (ring)->ring_mask) #define NEXT_CMP(idx) RING_CMP(ADV_RAW_CMP(idx, 1)) +#define FLIP_VALID(cons, mask, val) ((cons) >= (mask) ? !(val) : (val)) #define DB_CP_REARM_FLAGS (DB_KEY_CP | DB_IDX_VALID) #define DB_CP_FLAGS (DB_KEY_CP | DB_IDX_VALID | DB_IRQ_DIS) #define B_CP_DB_REARM(cpr, raw_cons) \ - (*(uint32_t *)((cpr)->cp_doorbell) = (DB_CP_REARM_FLAGS | \ - RING_CMP(cpr->cp_ring_struct, raw_cons))) + rte_write32((DB_CP_REARM_FLAGS | \ + RING_CMP(((cpr)->cp_ring_struct), raw_cons)), \ + ((cpr)->cp_doorbell)) -#define B_CP_DIS_DB(cpr, raw_cons) \ +#define B_CP_DB_ARM(cpr) rte_write32((DB_KEY_CP), ((cpr)->cp_doorbell)) +#define B_CP_DB_DISARM(cpr) (*(uint32_t *)((cpr)->cp_doorbell) = \ + DB_KEY_CP | DB_IRQ_DIS) + +#define B_CP_DB_IDX_ARM(cpr, cons) \ + (*(uint32_t *)((cpr)->cp_doorbell) = (DB_CP_REARM_FLAGS | \ + (cons))) + +#define B_CP_DB_IDX_DISARM(cpr, cons) do { \ + rte_smp_wmb(); \ (*(uint32_t *)((cpr)->cp_doorbell) = (DB_CP_FLAGS | \ - RING_CMP(cpr->cp_ring_struct, raw_cons))) + (cons)); \ +} while (0) +#define B_CP_DIS_DB(cpr, raw_cons) \ + rte_write32((DB_CP_FLAGS | \ + RING_CMP(((cpr)->cp_ring_struct), raw_cons)), \ + ((cpr)->cp_doorbell)) struct bnxt_ring; struct bnxt_cp_ring_info { @@ -64,13 +86,15 @@ struct bnxt_cp_ring_info { struct cmpl_base *cp_desc_ring; - phys_addr_t cp_desc_mapping; + rte_iova_t cp_desc_mapping; struct ctx_hw_stats *hw_stats; - phys_addr_t hw_stats_map; + rte_iova_t hw_stats_map; uint32_t hw_stats_ctx_id; struct bnxt_ring *cp_ring_struct; + uint16_t cp_cons; + bool valid; }; #define RX_CMP_L2_ERRORS \ @@ -78,6 +102,7 @@ struct bnxt_cp_ring_info { struct bnxt; +int bnxt_alloc_def_cp_ring(struct bnxt *bp); void bnxt_free_def_cp_ring(struct bnxt *bp); int bnxt_init_def_ring_struct(struct bnxt *bp, unsigned int socket_id); void bnxt_handle_async_event(struct bnxt *bp, struct cmpl_base *cmp); diff --git a/dpdk/drivers/net/bnxt/bnxt_ethdev.c b/dpdk/drivers/net/bnxt/bnxt_ethdev.c index 7052ecf3..3eeca6f4 100644 --- a/dpdk/drivers/net/bnxt/bnxt_ethdev.c +++ b/dpdk/drivers/net/bnxt/bnxt_ethdev.c @@ -36,6 +36,7 @@ #include #include +#include #include #include @@ -43,6 +44,7 @@ #include "bnxt_cpr.h" #include "bnxt_filter.h" #include "bnxt_hwrm.h" +#include "bnxt_irq.h" #include "bnxt_ring.h" #include "bnxt_rxq.h" #include "bnxt_rxr.h" @@ -51,6 +53,7 @@ #include "bnxt_txr.h" #include "bnxt_vnic.h" #include "hsi_struct_def_dpdk.h" +#include "bnxt_nvm_defs.h" #define DRV_MODULE_NAME "bnxt" static const char bnxt_version[] = @@ -58,28 +61,79 @@ static const char bnxt_version[] = #define PCI_VENDOR_ID_BROADCOM 0x14E4 +#define BROADCOM_DEV_ID_STRATUS_NIC_VF 0x1609 +#define BROADCOM_DEV_ID_STRATUS_NIC 0x1614 +#define BROADCOM_DEV_ID_57414_VF 0x16c1 #define BROADCOM_DEV_ID_57301 0x16c8 #define BROADCOM_DEV_ID_57302 0x16c9 #define BROADCOM_DEV_ID_57304_PF 0x16ca #define BROADCOM_DEV_ID_57304_VF 0x16cb +#define BROADCOM_DEV_ID_57417_MF 0x16cc +#define BROADCOM_DEV_ID_NS2 0x16cd +#define BROADCOM_DEV_ID_57311 0x16ce +#define BROADCOM_DEV_ID_57312 0x16cf #define BROADCOM_DEV_ID_57402 0x16d0 #define BROADCOM_DEV_ID_57404 0x16d1 #define BROADCOM_DEV_ID_57406_PF 0x16d2 #define BROADCOM_DEV_ID_57406_VF 0x16d3 -#define BROADCOM_DEV_ID_57406_MF 0x16d4 +#define BROADCOM_DEV_ID_57402_MF 0x16d4 +#define BROADCOM_DEV_ID_57407_RJ45 0x16d5 +#define BROADCOM_DEV_ID_57412 0x16d6 +#define BROADCOM_DEV_ID_57414 0x16d7 +#define BROADCOM_DEV_ID_57416_RJ45 0x16d8 +#define BROADCOM_DEV_ID_57417_RJ45 0x16d9 +#define BROADCOM_DEV_ID_5741X_VF 0x16dc +#define BROADCOM_DEV_ID_57412_MF 0x16de #define BROADCOM_DEV_ID_57314 0x16df +#define BROADCOM_DEV_ID_57317_RJ45 0x16e0 +#define BROADCOM_DEV_ID_5731X_VF 0x16e1 +#define BROADCOM_DEV_ID_57417_SFP 0x16e2 +#define BROADCOM_DEV_ID_57416_SFP 0x16e3 +#define BROADCOM_DEV_ID_57317_SFP 0x16e4 +#define BROADCOM_DEV_ID_57404_MF 0x16e7 +#define BROADCOM_DEV_ID_57406_MF 0x16e8 +#define BROADCOM_DEV_ID_57407_SFP 0x16e9 +#define BROADCOM_DEV_ID_57407_MF 0x16ea +#define BROADCOM_DEV_ID_57414_MF 0x16ec +#define BROADCOM_DEV_ID_57416_MF 0x16ee -static struct rte_pci_id bnxt_pci_id_map[] = { +static const struct rte_pci_id bnxt_pci_id_map[] = { + { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, + BROADCOM_DEV_ID_STRATUS_NIC_VF) }, + { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_STRATUS_NIC) }, + { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57414_VF) }, { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57301) }, { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57302) }, { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57304_PF) }, { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57304_VF) }, + { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_NS2) }, { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57402) }, { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57404) }, { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57406_PF) }, { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57406_VF) }, + { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57402_MF) }, + { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57407_RJ45) }, + { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57404_MF) }, { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57406_MF) }, + { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57407_SFP) }, + { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57407_MF) }, + { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_5741X_VF) }, + { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_5731X_VF) }, { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57314) }, + { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57417_MF) }, + { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57311) }, + { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57312) }, + { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57412) }, + { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57414) }, + { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57416_RJ45) }, + { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57417_RJ45) }, + { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57412_MF) }, + { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57317_RJ45) }, + { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57417_SFP) }, + { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57416_SFP) }, + { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57317_SFP) }, + { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57414_MF) }, + { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57416_MF) }, { .vendor_id = 0, /* sentinel */ }, }; @@ -91,6 +145,9 @@ static struct rte_pci_id bnxt_pci_id_map[] = { ETH_RSS_NONFRAG_IPV6_TCP | \ ETH_RSS_NONFRAG_IPV6_UDP) +static int bnxt_vlan_offload_set_op(struct rte_eth_dev *dev, int mask); +static void bnxt_print_link_info(struct rte_eth_dev *eth_dev); + /***********************/ /* @@ -145,8 +202,25 @@ alloc_mem_err: static int bnxt_init_chip(struct bnxt *bp) { unsigned int i, rss_idx, fw_idx; + struct rte_eth_link new; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(bp->eth_dev); + struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; + uint32_t intr_vector = 0; + uint32_t queue_id, base = BNXT_MISC_VEC_ID; + uint32_t vec = BNXT_MISC_VEC_ID; int rc; + /* disable uio/vfio intr/eventfd mapping */ + rte_intr_disable(intr_handle); + + if (bp->eth_dev->data->mtu > ETHER_MTU) { + bp->eth_dev->data->dev_conf.rxmode.jumbo_frame = 1; + bp->flags |= BNXT_FLAG_JUMBO; + } else { + bp->eth_dev->data->dev_conf.rxmode.jumbo_frame = 0; + bp->flags &= ~BNXT_FLAG_JUMBO; + } + rc = bnxt_alloc_all_hwrm_stat_ctxs(bp); if (rc) { RTE_LOG(ERR, PMD, "HWRM stat ctx alloc failure rc: %x\n", rc); @@ -177,28 +251,31 @@ static int bnxt_init_chip(struct bnxt *bp) rc = bnxt_hwrm_vnic_alloc(bp, vnic); if (rc) { - RTE_LOG(ERR, PMD, "HWRM vnic alloc failure rc: %x\n", - rc); + RTE_LOG(ERR, PMD, "HWRM vnic %d alloc failure rc: %x\n", + i, rc); goto err_out; } rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic); if (rc) { RTE_LOG(ERR, PMD, - "HWRM vnic ctx alloc failure rc: %x\n", rc); + "HWRM vnic %d ctx alloc failure rc: %x\n", + i, rc); goto err_out; } rc = bnxt_hwrm_vnic_cfg(bp, vnic); if (rc) { - RTE_LOG(ERR, PMD, "HWRM vnic cfg failure rc: %x\n", rc); + RTE_LOG(ERR, PMD, "HWRM vnic %d cfg failure rc: %x\n", + i, rc); goto err_out; } rc = bnxt_set_hwrm_vnic_filters(bp, vnic); if (rc) { - RTE_LOG(ERR, PMD, "HWRM vnic filter failure rc: %x\n", - rc); + RTE_LOG(ERR, PMD, + "HWRM vnic %d filter failure rc: %x\n", + i, rc); goto err_out; } if (vnic->rss_table && vnic->hash_type) { @@ -218,19 +295,84 @@ static int bnxt_init_chip(struct bnxt *bp) rc = bnxt_hwrm_vnic_rss_cfg(bp, vnic); if (rc) { RTE_LOG(ERR, PMD, - "HWRM vnic set RSS failure rc: %x\n", - rc); + "HWRM vnic %d set RSS failure rc: %x\n", + i, rc); goto err_out; } } + + bnxt_hwrm_vnic_plcmode_cfg(bp, vnic); + + if (bp->eth_dev->data->dev_conf.rxmode.enable_lro) + bnxt_hwrm_vnic_tpa_cfg(bp, vnic, 1); + else + bnxt_hwrm_vnic_tpa_cfg(bp, vnic, 0); } - rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, &bp->vnic_info[0]); + rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, &bp->vnic_info[0], 0, NULL); if (rc) { RTE_LOG(ERR, PMD, "HWRM cfa l2 rx mask failure rc: %x\n", rc); goto err_out; } + /* check and configure queue intr-vector mapping */ + if ((rte_intr_cap_multiple(intr_handle) || + !RTE_ETH_DEV_SRIOV(bp->eth_dev).active) && + bp->eth_dev->data->dev_conf.intr_conf.rxq != 0) { + intr_vector = bp->eth_dev->data->nb_rx_queues; + RTE_LOG(INFO, PMD, "%s(): intr_vector = %d\n", __func__, + intr_vector); + if (intr_vector > bp->rx_cp_nr_rings) { + RTE_LOG(ERR, PMD, "At most %d intr queues supported", + bp->rx_cp_nr_rings); + return -ENOTSUP; + } + if (rte_intr_efd_enable(intr_handle, intr_vector)) + return -1; + } + + if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) { + intr_handle->intr_vec = + rte_zmalloc("intr_vec", + bp->eth_dev->data->nb_rx_queues * + sizeof(int), 0); + if (intr_handle->intr_vec == NULL) { + RTE_LOG(ERR, PMD, "Failed to allocate %d rx_queues" + " intr_vec", bp->eth_dev->data->nb_rx_queues); + return -ENOMEM; + } + RTE_LOG(DEBUG, PMD, "%s(): intr_handle->intr_vec = %p " + "intr_handle->nb_efd = %d intr_handle->max_intr = %d\n", + __func__, intr_handle->intr_vec, intr_handle->nb_efd, + intr_handle->max_intr); + } + + for (queue_id = 0; queue_id < bp->eth_dev->data->nb_rx_queues; + queue_id++) { + intr_handle->intr_vec[queue_id] = vec; + if (vec < base + intr_handle->nb_efd - 1) + vec++; + } + + /* enable uio/vfio intr/eventfd mapping */ + rte_intr_enable(intr_handle); + + rc = bnxt_get_hwrm_link_config(bp, &new); + if (rc) { + RTE_LOG(ERR, PMD, "HWRM Get link config failure rc: %x\n", rc); + goto err_out; + } + + if (!bp->link_info.link_up) { + rc = bnxt_set_hwrm_link_config(bp, true); + if (rc) { + RTE_LOG(ERR, PMD, + "HWRM link config failure rc: %x\n", rc); + goto err_out; + } + } + bnxt_print_link_info(bp->eth_dev); + return 0; err_out: @@ -251,7 +393,10 @@ static int bnxt_init_nic(struct bnxt *bp) { int rc; - bnxt_init_ring_grps(bp); + rc = bnxt_init_ring_grps(bp); + if (rc) + return rc; + bnxt_init_vnics(bp); bnxt_init_filters(bp); @@ -271,34 +416,46 @@ static void bnxt_dev_info_get_op(struct rte_eth_dev *eth_dev, { struct bnxt *bp = (struct bnxt *)eth_dev->data->dev_private; uint16_t max_vnics, i, j, vpool, vrxq; + unsigned int max_rx_rings; + + dev_info->pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); /* MAC Specifics */ - dev_info->max_mac_addrs = MAX_NUM_MAC_ADDR; + dev_info->max_mac_addrs = bp->max_l2_ctx; dev_info->max_hash_mac_addrs = 0; /* PF/VF specifics */ - if (BNXT_PF(bp)) { - dev_info->max_rx_queues = bp->pf.max_rx_rings; - dev_info->max_tx_queues = bp->pf.max_tx_rings; - dev_info->max_vfs = bp->pf.active_vfs; - dev_info->reta_size = bp->pf.max_rsscos_ctx; - max_vnics = bp->pf.max_vnics; - } else { - dev_info->max_rx_queues = bp->vf.max_rx_rings; - dev_info->max_tx_queues = bp->vf.max_tx_rings; - dev_info->reta_size = bp->vf.max_rsscos_ctx; - max_vnics = bp->vf.max_vnics; - } + if (BNXT_PF(bp)) + dev_info->max_vfs = bp->pdev->max_vfs; + max_rx_rings = RTE_MIN(bp->max_vnics, RTE_MIN(bp->max_l2_ctx, + RTE_MIN(bp->max_rsscos_ctx, + bp->max_stat_ctx))); + /* For the sake of symmetry, max_rx_queues = max_tx_queues */ + dev_info->max_rx_queues = max_rx_rings; + dev_info->max_tx_queues = max_rx_rings; + dev_info->reta_size = bp->max_rsscos_ctx; + dev_info->hash_key_size = 40; + max_vnics = bp->max_vnics; /* Fast path specifics */ dev_info->min_rx_bufsize = 1; dev_info->max_rx_pktlen = BNXT_MAX_MTU + ETHER_HDR_LEN + ETHER_CRC_LEN + VLAN_TAG_SIZE; - dev_info->rx_offload_capa = 0; - dev_info->tx_offload_capa = DEV_TX_OFFLOAD_IPV4_CKSUM | + dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP | + DEV_RX_OFFLOAD_IPV4_CKSUM | + DEV_RX_OFFLOAD_UDP_CKSUM | + DEV_RX_OFFLOAD_TCP_CKSUM | + DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM; + dev_info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT | + DEV_TX_OFFLOAD_IPV4_CKSUM | DEV_TX_OFFLOAD_TCP_CKSUM | DEV_TX_OFFLOAD_UDP_CKSUM | - DEV_TX_OFFLOAD_TCP_TSO; + DEV_TX_OFFLOAD_TCP_TSO | + DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM | + DEV_TX_OFFLOAD_VXLAN_TNL_TSO | + DEV_TX_OFFLOAD_GRE_TNL_TSO | + DEV_TX_OFFLOAD_IPIP_TNL_TSO | + DEV_TX_OFFLOAD_GENEVE_TNL_TSO; /* *INDENT-OFF* */ dev_info->default_rxconf = (struct rte_eth_rxconf) { @@ -322,6 +479,10 @@ static void bnxt_dev_info_get_op(struct rte_eth_dev *eth_dev, .txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS | ETH_TXQ_FLAGS_NOOFFLOADS, }; + eth_dev->data->dev_conf.intr_conf.lsc = 1; + + eth_dev->data->dev_conf.intr_conf.rxq = 1; + /* *INDENT-ON* */ /* @@ -360,7 +521,6 @@ found: static int bnxt_dev_configure_op(struct rte_eth_dev *eth_dev) { struct bnxt *bp = (struct bnxt *)eth_dev->data->dev_private; - int rc; bp->rx_queues = (void *)eth_dev->data->rx_queues; bp->tx_queues = (void *)eth_dev->data->tx_queues; @@ -375,28 +535,54 @@ static int bnxt_dev_configure_op(struct rte_eth_dev *eth_dev) eth_dev->data->mtu = eth_dev->data->dev_conf.rxmode.max_rx_pkt_len - ETHER_HDR_LEN - ETHER_CRC_LEN - VLAN_TAG_SIZE; - rc = bnxt_set_hwrm_link_config(bp, true); - return rc; + return 0; +} + +static void bnxt_print_link_info(struct rte_eth_dev *eth_dev) +{ + struct rte_eth_link *link = ð_dev->data->dev_link; + + if (link->link_status) + RTE_LOG(INFO, PMD, "Port %d Link Up - speed %u Mbps - %s\n", + eth_dev->data->port_id, + (uint32_t)link->link_speed, + (link->link_duplex == ETH_LINK_FULL_DUPLEX) ? + ("full-duplex") : ("half-duplex\n")); + else + RTE_LOG(INFO, PMD, "Port %d Link Down\n", + eth_dev->data->port_id); +} + +static int bnxt_dev_lsc_intr_setup(struct rte_eth_dev *eth_dev) +{ + bnxt_print_link_info(eth_dev); + return 0; } static int bnxt_dev_start_op(struct rte_eth_dev *eth_dev) { struct bnxt *bp = (struct bnxt *)eth_dev->data->dev_private; + int vlan_mask = 0; int rc; - bp->dev_stopped = 0; - rc = bnxt_hwrm_func_reset(bp); - if (rc) { - RTE_LOG(ERR, PMD, "hwrm chip reset failure rc: %x\n", rc); - rc = -1; - goto error; + if (bp->rx_cp_nr_rings > RTE_ETHDEV_QUEUE_STAT_CNTRS) { + RTE_LOG(ERR, PMD, + "RxQ cnt %d > CONFIG_RTE_ETHDEV_QUEUE_STAT_CNTRS %d\n", + bp->rx_cp_nr_rings, RTE_ETHDEV_QUEUE_STAT_CNTRS); } + bp->dev_stopped = 0; - rc = bnxt_alloc_mem(bp); + rc = bnxt_init_nic(bp); if (rc) goto error; - rc = bnxt_init_nic(bp); + bnxt_link_update_op(eth_dev, 1); + + if (eth_dev->data->dev_conf.rxmode.hw_vlan_filter) + vlan_mask |= ETH_VLAN_FILTER_MASK; + if (eth_dev->data->dev_conf.rxmode.hw_vlan_strip) + vlan_mask |= ETH_VLAN_STRIP_MASK; + rc = bnxt_vlan_offload_set_op(eth_dev, vlan_mask); if (rc) goto error; @@ -406,16 +592,20 @@ error: bnxt_shutdown_nic(bp); bnxt_free_tx_mbufs(bp); bnxt_free_rx_mbufs(bp); - bnxt_free_mem(bp); return rc; } static int bnxt_dev_set_link_up_op(struct rte_eth_dev *eth_dev) { struct bnxt *bp = (struct bnxt *)eth_dev->data->dev_private; + int rc = 0; - eth_dev->data->dev_link.link_status = 1; - bnxt_set_hwrm_link_config(bp, true); + if (!bp->link_info.link_up) + rc = bnxt_set_hwrm_link_config(bp, true); + if (!rc) + eth_dev->data->dev_link.link_status = 1; + + bnxt_print_link_info(eth_dev); return 0; } @@ -425,6 +615,8 @@ static int bnxt_dev_set_link_down_op(struct rte_eth_dev *eth_dev) eth_dev->data->dev_link.link_status = 0; bnxt_set_hwrm_link_config(bp, false); + bp->link_info.link_up = 0; + return 0; } @@ -437,7 +629,10 @@ static void bnxt_dev_stop_op(struct rte_eth_dev *eth_dev) /* TBD: STOP HW queues DMA */ eth_dev->data->dev_link.link_status = 0; } + bnxt_set_hwrm_link_config(bp, false); + bnxt_hwrm_port_clr_stats(bp); bnxt_shutdown_nic(bp); + bp->dev_stopped = 1; } static void bnxt_dev_close_op(struct rte_eth_dev *eth_dev) @@ -450,7 +645,14 @@ static void bnxt_dev_close_op(struct rte_eth_dev *eth_dev) bnxt_free_tx_mbufs(bp); bnxt_free_rx_mbufs(bp); bnxt_free_mem(bp); - rte_free(eth_dev->data->mac_addrs); + if (eth_dev->data->mac_addrs != NULL) { + rte_free(eth_dev->data->mac_addrs); + eth_dev->data->mac_addrs = NULL; + } + if (bp->grp_info != NULL) { + rte_free(bp->grp_info); + bp->grp_info = NULL; + } } static void bnxt_mac_addr_remove_op(struct rte_eth_dev *eth_dev, @@ -460,13 +662,14 @@ static void bnxt_mac_addr_remove_op(struct rte_eth_dev *eth_dev, uint64_t pool_mask = eth_dev->data->mac_pool_sel[index]; struct bnxt_vnic_info *vnic; struct bnxt_filter_info *filter, *temp_filter; - int i; + uint32_t pool = RTE_MIN(MAX_FF_POOLS, ETH_64_POOLS); + uint32_t i; /* * Loop through all VNICs from the specified filter flow pools to * remove the corresponding MAC addr filter */ - for (i = 0; i < MAX_FF_POOLS; i++) { + for (i = 0; i < pool; i++) { if (!(pool_mask & (1ULL << i))) continue; @@ -477,7 +680,7 @@ static void bnxt_mac_addr_remove_op(struct rte_eth_dev *eth_dev, if (filter->mac_index == index) { STAILQ_REMOVE(&vnic->filter, filter, bnxt_filter_info, next); - bnxt_hwrm_clear_filter(bp, filter); + bnxt_hwrm_clear_l2_filter(bp, filter); filter->mac_index = INVALID_MAC_INDEX; memset(&filter->l2_addr, 0, ETHER_ADDR_LEN); @@ -491,39 +694,43 @@ static void bnxt_mac_addr_remove_op(struct rte_eth_dev *eth_dev, } } -static void bnxt_mac_addr_add_op(struct rte_eth_dev *eth_dev, - struct ether_addr *mac_addr, - uint32_t index, uint32_t pool) +static int bnxt_mac_addr_add_op(struct rte_eth_dev *eth_dev, + struct ether_addr *mac_addr, + uint32_t index, uint32_t pool) { struct bnxt *bp = (struct bnxt *)eth_dev->data->dev_private; struct bnxt_vnic_info *vnic = STAILQ_FIRST(&bp->ff_pool[pool]); struct bnxt_filter_info *filter; + if (BNXT_VF(bp)) { + RTE_LOG(ERR, PMD, "Cannot add MAC address to a VF interface\n"); + return -ENOTSUP; + } + if (!vnic) { RTE_LOG(ERR, PMD, "VNIC not found for pool %d!\n", pool); - return; + return -EINVAL; } /* Attach requested MAC address to the new l2_filter */ STAILQ_FOREACH(filter, &vnic->filter, next) { if (filter->mac_index == index) { RTE_LOG(ERR, PMD, "MAC addr already existed for pool %d\n", pool); - return; + return 0; } } filter = bnxt_alloc_filter(bp); if (!filter) { RTE_LOG(ERR, PMD, "L2 filter alloc failed\n"); - return; + return -ENODEV; } STAILQ_INSERT_TAIL(&vnic->filter, filter, next); filter->mac_index = index; memcpy(filter->l2_addr, mac_addr, ETHER_ADDR_LEN); - bnxt_hwrm_set_filter(bp, vnic, filter); + return bnxt_hwrm_set_l2_filter(bp, vnic->fw_vnic_id, filter); } -static int bnxt_link_update_op(struct rte_eth_dev *eth_dev, - int wait_to_complete) +int bnxt_link_update_op(struct rte_eth_dev *eth_dev, int wait_to_complete) { int rc = 0; struct bnxt *bp = (struct bnxt *)eth_dev->data->dev_private; @@ -538,24 +745,24 @@ static int bnxt_link_update_op(struct rte_eth_dev *eth_dev, new.link_speed = ETH_LINK_SPEED_100M; new.link_duplex = ETH_LINK_FULL_DUPLEX; RTE_LOG(ERR, PMD, - "Failed to retrieve link rc = 0x%x!", rc); + "Failed to retrieve link rc = 0x%x!\n", rc); goto out; } - if (!wait_to_complete) - break; - rte_delay_ms(BNXT_LINK_WAIT_INTERVAL); + if (!wait_to_complete) + break; } while (!new.link_status && cnt--); - /* Timed out or success */ - if (new.link_status) { - /* Update only if success */ - eth_dev->data->dev_link.link_duplex = new.link_duplex; - eth_dev->data->dev_link.link_speed = new.link_speed; - } - eth_dev->data->dev_link.link_status = new.link_status; out: + /* Timed out or success */ + if (new.link_status != eth_dev->data->dev_link.link_status || + new.link_speed != eth_dev->data->dev_link.link_speed) { + memcpy(ð_dev->data->dev_link, &new, + sizeof(struct rte_eth_link)); + bnxt_print_link_info(eth_dev); + } + return rc; } @@ -570,7 +777,7 @@ static void bnxt_promiscuous_enable_op(struct rte_eth_dev *eth_dev) vnic = &bp->vnic_info[0]; vnic->flags |= BNXT_VNIC_INFO_PROMISC; - bnxt_hwrm_cfa_l2_set_rx_mask(bp, vnic); + bnxt_hwrm_cfa_l2_set_rx_mask(bp, vnic, 0, NULL); } static void bnxt_promiscuous_disable_op(struct rte_eth_dev *eth_dev) @@ -584,7 +791,7 @@ static void bnxt_promiscuous_disable_op(struct rte_eth_dev *eth_dev) vnic = &bp->vnic_info[0]; vnic->flags &= ~BNXT_VNIC_INFO_PROMISC; - bnxt_hwrm_cfa_l2_set_rx_mask(bp, vnic); + bnxt_hwrm_cfa_l2_set_rx_mask(bp, vnic, 0, NULL); } static void bnxt_allmulticast_enable_op(struct rte_eth_dev *eth_dev) @@ -598,7 +805,7 @@ static void bnxt_allmulticast_enable_op(struct rte_eth_dev *eth_dev) vnic = &bp->vnic_info[0]; vnic->flags |= BNXT_VNIC_INFO_ALLMULTI; - bnxt_hwrm_cfa_l2_set_rx_mask(bp, vnic); + bnxt_hwrm_cfa_l2_set_rx_mask(bp, vnic, 0, NULL); } static void bnxt_allmulticast_disable_op(struct rte_eth_dev *eth_dev) @@ -612,7 +819,7 @@ static void bnxt_allmulticast_disable_op(struct rte_eth_dev *eth_dev) vnic = &bp->vnic_info[0]; vnic->flags &= ~BNXT_VNIC_INFO_ALLMULTI; - bnxt_hwrm_cfa_l2_set_rx_mask(bp, vnic); + bnxt_hwrm_cfa_l2_set_rx_mask(bp, vnic, 0, NULL); } static int bnxt_reta_update_op(struct rte_eth_dev *eth_dev, @@ -650,6 +857,8 @@ static int bnxt_reta_query_op(struct rte_eth_dev *eth_dev, { struct bnxt *bp = (struct bnxt *)eth_dev->data->dev_private; struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; + struct rte_intr_handle *intr_handle + = &bp->pdev->intr_handle; /* Retrieve from the default VNIC */ if (!vnic) @@ -666,6 +875,11 @@ static int bnxt_reta_query_op(struct rte_eth_dev *eth_dev, /* EW - need to revisit here copying from u64 to u16 */ memcpy(reta_conf, vnic->rss_table, reta_size); + if (rte_intr_allow_others(intr_handle)) { + if (eth_dev->data->dev_conf.intr_conf.lsc != 0) + bnxt_dev_lsc_intr_setup(eth_dev); + } + return 0; } @@ -684,11 +898,15 @@ static int bnxt_rss_hash_update_op(struct rte_eth_dev *eth_dev, */ if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) { if (!rss_conf->rss_hf) - return -EINVAL; + RTE_LOG(ERR, PMD, "Hash type NONE\n"); } else { if (rss_conf->rss_hf & BNXT_ETH_RSS_SUPPORT) return -EINVAL; } + + bp->flags |= BNXT_FLAG_UPDATE_HASH; + memcpy(&bp->rss_conf, rss_conf, sizeof(*rss_conf)); + if (rss_conf->rss_hf & ETH_RSS_IPV4) hash_type |= HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_IPV4; if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) @@ -781,7 +999,7 @@ static int bnxt_rss_hash_conf_get_op(struct rte_eth_dev *eth_dev, } static int bnxt_flow_ctrl_get_op(struct rte_eth_dev *dev, - struct rte_eth_fc_conf *fc_conf __rte_unused) + struct rte_eth_fc_conf *fc_conf) { struct bnxt *bp = (struct bnxt *)dev->data->dev_private; struct rte_eth_link link_info; @@ -817,6 +1035,11 @@ static int bnxt_flow_ctrl_set_op(struct rte_eth_dev *dev, { struct bnxt *bp = (struct bnxt *)dev->data->dev_private; + if (BNXT_NPAR_PF(bp) || BNXT_VF(bp)) { + RTE_LOG(ERR, PMD, "Flow Control Settings cannot be modified\n"); + return -ENOTSUP; + } + switch (fc_conf->mode) { case RTE_FC_NONE: bp->link_info.auto_pause = 0; @@ -861,11 +1084,1617 @@ static int bnxt_flow_ctrl_set_op(struct rte_eth_dev *dev, return bnxt_set_hwrm_link_config(bp, true); } +/* Add UDP tunneling port */ +static int +bnxt_udp_tunnel_port_add_op(struct rte_eth_dev *eth_dev, + struct rte_eth_udp_tunnel *udp_tunnel) +{ + struct bnxt *bp = (struct bnxt *)eth_dev->data->dev_private; + uint16_t tunnel_type = 0; + int rc = 0; + + switch (udp_tunnel->prot_type) { + case RTE_TUNNEL_TYPE_VXLAN: + if (bp->vxlan_port_cnt) { + RTE_LOG(ERR, PMD, "Tunnel Port %d already programmed\n", + udp_tunnel->udp_port); + if (bp->vxlan_port != udp_tunnel->udp_port) { + RTE_LOG(ERR, PMD, "Only one port allowed\n"); + return -ENOSPC; + } + bp->vxlan_port_cnt++; + return 0; + } + tunnel_type = + HWRM_TUNNEL_DST_PORT_ALLOC_INPUT_TUNNEL_TYPE_VXLAN; + bp->vxlan_port_cnt++; + break; + case RTE_TUNNEL_TYPE_GENEVE: + if (bp->geneve_port_cnt) { + RTE_LOG(ERR, PMD, "Tunnel Port %d already programmed\n", + udp_tunnel->udp_port); + if (bp->geneve_port != udp_tunnel->udp_port) { + RTE_LOG(ERR, PMD, "Only one port allowed\n"); + return -ENOSPC; + } + bp->geneve_port_cnt++; + return 0; + } + tunnel_type = + HWRM_TUNNEL_DST_PORT_ALLOC_INPUT_TUNNEL_TYPE_GENEVE; + bp->geneve_port_cnt++; + break; + default: + RTE_LOG(ERR, PMD, "Tunnel type is not supported\n"); + return -ENOTSUP; + } + rc = bnxt_hwrm_tunnel_dst_port_alloc(bp, udp_tunnel->udp_port, + tunnel_type); + return rc; +} + +static int +bnxt_udp_tunnel_port_del_op(struct rte_eth_dev *eth_dev, + struct rte_eth_udp_tunnel *udp_tunnel) +{ + struct bnxt *bp = (struct bnxt *)eth_dev->data->dev_private; + uint16_t tunnel_type = 0; + uint16_t port = 0; + int rc = 0; + + switch (udp_tunnel->prot_type) { + case RTE_TUNNEL_TYPE_VXLAN: + if (!bp->vxlan_port_cnt) { + RTE_LOG(ERR, PMD, "No Tunnel port configured yet\n"); + return -EINVAL; + } + if (bp->vxlan_port != udp_tunnel->udp_port) { + RTE_LOG(ERR, PMD, "Req Port: %d. Configured port: %d\n", + udp_tunnel->udp_port, bp->vxlan_port); + return -EINVAL; + } + if (--bp->vxlan_port_cnt) + return 0; + + tunnel_type = + HWRM_TUNNEL_DST_PORT_FREE_INPUT_TUNNEL_TYPE_VXLAN; + port = bp->vxlan_fw_dst_port_id; + break; + case RTE_TUNNEL_TYPE_GENEVE: + if (!bp->geneve_port_cnt) { + RTE_LOG(ERR, PMD, "No Tunnel port configured yet\n"); + return -EINVAL; + } + if (bp->geneve_port != udp_tunnel->udp_port) { + RTE_LOG(ERR, PMD, "Req Port: %d. Configured port: %d\n", + udp_tunnel->udp_port, bp->geneve_port); + return -EINVAL; + } + if (--bp->geneve_port_cnt) + return 0; + + tunnel_type = + HWRM_TUNNEL_DST_PORT_FREE_INPUT_TUNNEL_TYPE_GENEVE; + port = bp->geneve_fw_dst_port_id; + break; + default: + RTE_LOG(ERR, PMD, "Tunnel type is not supported\n"); + return -ENOTSUP; + } + + rc = bnxt_hwrm_tunnel_dst_port_free(bp, port, tunnel_type); + if (!rc) { + if (tunnel_type == + HWRM_TUNNEL_DST_PORT_FREE_INPUT_TUNNEL_TYPE_VXLAN) + bp->vxlan_port = 0; + if (tunnel_type == + HWRM_TUNNEL_DST_PORT_FREE_INPUT_TUNNEL_TYPE_GENEVE) + bp->geneve_port = 0; + } + return rc; +} + +static int bnxt_del_vlan_filter(struct bnxt *bp, uint16_t vlan_id) +{ + struct bnxt_filter_info *filter, *temp_filter, *new_filter; + struct bnxt_vnic_info *vnic; + unsigned int i; + int rc = 0; + uint32_t chk = HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_OVLAN; + + /* Cycle through all VNICs */ + for (i = 0; i < bp->nr_vnics; i++) { + /* + * For each VNIC and each associated filter(s) + * if VLAN exists && VLAN matches vlan_id + * remove the MAC+VLAN filter + * add a new MAC only filter + * else + * VLAN filter doesn't exist, just skip and continue + */ + STAILQ_FOREACH(vnic, &bp->ff_pool[i], next) { + filter = STAILQ_FIRST(&vnic->filter); + while (filter) { + temp_filter = STAILQ_NEXT(filter, next); + + if (filter->enables & chk && + filter->l2_ovlan == vlan_id) { + /* Must delete the filter */ + STAILQ_REMOVE(&vnic->filter, filter, + bnxt_filter_info, next); + bnxt_hwrm_clear_l2_filter(bp, filter); + STAILQ_INSERT_TAIL( + &bp->free_filter_list, + filter, next); + + /* + * Need to examine to see if the MAC + * filter already existed or not before + * allocating a new one + */ + + new_filter = bnxt_alloc_filter(bp); + if (!new_filter) { + RTE_LOG(ERR, PMD, + "MAC/VLAN filter alloc failed\n"); + rc = -ENOMEM; + goto exit; + } + STAILQ_INSERT_TAIL(&vnic->filter, + new_filter, next); + /* Inherit MAC from previous filter */ + new_filter->mac_index = + filter->mac_index; + memcpy(new_filter->l2_addr, + filter->l2_addr, ETHER_ADDR_LEN); + /* MAC only filter */ + rc = bnxt_hwrm_set_l2_filter(bp, + vnic->fw_vnic_id, + new_filter); + if (rc) + goto exit; + RTE_LOG(INFO, PMD, + "Del Vlan filter for %d\n", + vlan_id); + } + filter = temp_filter; + } + } + } +exit: + return rc; +} + +static int bnxt_add_vlan_filter(struct bnxt *bp, uint16_t vlan_id) +{ + struct bnxt_filter_info *filter, *temp_filter, *new_filter; + struct bnxt_vnic_info *vnic; + unsigned int i; + int rc = 0; + uint32_t en = HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_OVLAN | + HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_OVLAN_MASK; + uint32_t chk = HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_OVLAN; + + /* Cycle through all VNICs */ + for (i = 0; i < bp->nr_vnics; i++) { + /* + * For each VNIC and each associated filter(s) + * if VLAN exists: + * if VLAN matches vlan_id + * VLAN filter already exists, just skip and continue + * else + * add a new MAC+VLAN filter + * else + * Remove the old MAC only filter + * Add a new MAC+VLAN filter + */ + STAILQ_FOREACH(vnic, &bp->ff_pool[i], next) { + filter = STAILQ_FIRST(&vnic->filter); + while (filter) { + temp_filter = STAILQ_NEXT(filter, next); + + if (filter->enables & chk) { + if (filter->l2_ovlan == vlan_id) + goto cont; + } else { + /* Must delete the MAC filter */ + STAILQ_REMOVE(&vnic->filter, filter, + bnxt_filter_info, next); + bnxt_hwrm_clear_l2_filter(bp, filter); + filter->l2_ovlan = 0; + STAILQ_INSERT_TAIL( + &bp->free_filter_list, + filter, next); + } + new_filter = bnxt_alloc_filter(bp); + if (!new_filter) { + RTE_LOG(ERR, PMD, + "MAC/VLAN filter alloc failed\n"); + rc = -ENOMEM; + goto exit; + } + STAILQ_INSERT_TAIL(&vnic->filter, new_filter, + next); + /* Inherit MAC from the previous filter */ + new_filter->mac_index = filter->mac_index; + memcpy(new_filter->l2_addr, filter->l2_addr, + ETHER_ADDR_LEN); + /* MAC + VLAN ID filter */ + new_filter->l2_ovlan = vlan_id; + new_filter->l2_ovlan_mask = 0xF000; + new_filter->enables |= en; + rc = bnxt_hwrm_set_l2_filter(bp, + vnic->fw_vnic_id, + new_filter); + if (rc) + goto exit; + RTE_LOG(INFO, PMD, + "Added Vlan filter for %d\n", vlan_id); +cont: + filter = temp_filter; + } + } + } +exit: + return rc; +} + +static int bnxt_vlan_filter_set_op(struct rte_eth_dev *eth_dev, + uint16_t vlan_id, int on) +{ + struct bnxt *bp = (struct bnxt *)eth_dev->data->dev_private; + + /* These operations apply to ALL existing MAC/VLAN filters */ + if (on) + return bnxt_add_vlan_filter(bp, vlan_id); + else + return bnxt_del_vlan_filter(bp, vlan_id); +} + +static int +bnxt_vlan_offload_set_op(struct rte_eth_dev *dev, int mask) +{ + struct bnxt *bp = (struct bnxt *)dev->data->dev_private; + unsigned int i; + + if (mask & ETH_VLAN_FILTER_MASK) { + if (!dev->data->dev_conf.rxmode.hw_vlan_filter) { + /* Remove any VLAN filters programmed */ + for (i = 0; i < 4095; i++) + bnxt_del_vlan_filter(bp, i); + } + RTE_LOG(INFO, PMD, "VLAN Filtering: %d\n", + dev->data->dev_conf.rxmode.hw_vlan_filter); + } + + if (mask & ETH_VLAN_STRIP_MASK) { + /* Enable or disable VLAN stripping */ + for (i = 0; i < bp->nr_vnics; i++) { + struct bnxt_vnic_info *vnic = &bp->vnic_info[i]; + if (dev->data->dev_conf.rxmode.hw_vlan_strip) + vnic->vlan_strip = true; + else + vnic->vlan_strip = false; + bnxt_hwrm_vnic_cfg(bp, vnic); + } + RTE_LOG(INFO, PMD, "VLAN Strip Offload: %d\n", + dev->data->dev_conf.rxmode.hw_vlan_strip); + } + + if (mask & ETH_VLAN_EXTEND_MASK) + RTE_LOG(ERR, PMD, "Extend VLAN Not supported\n"); + + return 0; +} + +static void +bnxt_set_default_mac_addr_op(struct rte_eth_dev *dev, struct ether_addr *addr) +{ + struct bnxt *bp = (struct bnxt *)dev->data->dev_private; + /* Default Filter is tied to VNIC 0 */ + struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; + struct bnxt_filter_info *filter; + int rc; + + if (BNXT_VF(bp)) + return; + + memcpy(bp->mac_addr, addr, sizeof(bp->mac_addr)); + memcpy(&dev->data->mac_addrs[0], bp->mac_addr, ETHER_ADDR_LEN); + + STAILQ_FOREACH(filter, &vnic->filter, next) { + /* Default Filter is at Index 0 */ + if (filter->mac_index != 0) + continue; + rc = bnxt_hwrm_clear_l2_filter(bp, filter); + if (rc) + break; + memcpy(filter->l2_addr, bp->mac_addr, ETHER_ADDR_LEN); + memset(filter->l2_addr_mask, 0xff, ETHER_ADDR_LEN); + filter->flags |= HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_PATH_RX; + filter->enables |= + HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_ADDR | + HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_ADDR_MASK; + rc = bnxt_hwrm_set_l2_filter(bp, vnic->fw_vnic_id, filter); + if (rc) + break; + filter->mac_index = 0; + RTE_LOG(DEBUG, PMD, "Set MAC addr\n"); + } +} + +static int +bnxt_dev_set_mc_addr_list_op(struct rte_eth_dev *eth_dev, + struct ether_addr *mc_addr_set, + uint32_t nb_mc_addr) +{ + struct bnxt *bp = (struct bnxt *)eth_dev->data->dev_private; + char *mc_addr_list = (char *)mc_addr_set; + struct bnxt_vnic_info *vnic; + uint32_t off = 0, i = 0; + + vnic = &bp->vnic_info[0]; + + if (nb_mc_addr > BNXT_MAX_MC_ADDRS) { + vnic->flags |= BNXT_VNIC_INFO_ALLMULTI; + goto allmulti; + } + + /* TODO Check for Duplicate mcast addresses */ + vnic->flags &= ~BNXT_VNIC_INFO_ALLMULTI; + for (i = 0; i < nb_mc_addr; i++) { + memcpy(vnic->mc_list + off, &mc_addr_list[i], ETHER_ADDR_LEN); + off += ETHER_ADDR_LEN; + } + + vnic->mc_addr_cnt = i; + +allmulti: + return bnxt_hwrm_cfa_l2_set_rx_mask(bp, vnic, 0, NULL); +} + +static int +bnxt_fw_version_get(struct rte_eth_dev *dev, char *fw_version, size_t fw_size) +{ + struct bnxt *bp = (struct bnxt *)dev->data->dev_private; + uint8_t fw_major = (bp->fw_ver >> 24) & 0xff; + uint8_t fw_minor = (bp->fw_ver >> 16) & 0xff; + uint8_t fw_updt = (bp->fw_ver >> 8) & 0xff; + int ret; + + ret = snprintf(fw_version, fw_size, "%d.%d.%d", + fw_major, fw_minor, fw_updt); + + ret += 1; /* add the size of '\0' */ + if (fw_size < (uint32_t)ret) + return ret; + else + return 0; +} + +static void +bnxt_rxq_info_get_op(struct rte_eth_dev *dev, uint16_t queue_id, + struct rte_eth_rxq_info *qinfo) +{ + struct bnxt_rx_queue *rxq; + + rxq = dev->data->rx_queues[queue_id]; + + qinfo->mp = rxq->mb_pool; + qinfo->scattered_rx = dev->data->scattered_rx; + qinfo->nb_desc = rxq->nb_rx_desc; + + qinfo->conf.rx_free_thresh = rxq->rx_free_thresh; + qinfo->conf.rx_drop_en = 0; + qinfo->conf.rx_deferred_start = 0; +} + +static void +bnxt_txq_info_get_op(struct rte_eth_dev *dev, uint16_t queue_id, + struct rte_eth_txq_info *qinfo) +{ + struct bnxt_tx_queue *txq; + + txq = dev->data->tx_queues[queue_id]; + + qinfo->nb_desc = txq->nb_tx_desc; + + qinfo->conf.tx_thresh.pthresh = txq->pthresh; + qinfo->conf.tx_thresh.hthresh = txq->hthresh; + qinfo->conf.tx_thresh.wthresh = txq->wthresh; + + qinfo->conf.tx_free_thresh = txq->tx_free_thresh; + qinfo->conf.tx_rs_thresh = 0; + qinfo->conf.txq_flags = txq->txq_flags; + qinfo->conf.tx_deferred_start = txq->tx_deferred_start; +} + +static int bnxt_mtu_set_op(struct rte_eth_dev *eth_dev, uint16_t new_mtu) +{ + struct bnxt *bp = eth_dev->data->dev_private; + struct rte_eth_dev_info dev_info; + uint32_t max_dev_mtu; + uint32_t rc = 0; + uint32_t i; + + bnxt_dev_info_get_op(eth_dev, &dev_info); + max_dev_mtu = dev_info.max_rx_pktlen - + ETHER_HDR_LEN - ETHER_CRC_LEN - VLAN_TAG_SIZE * 2; + + if (new_mtu < ETHER_MIN_MTU || new_mtu > max_dev_mtu) { + RTE_LOG(ERR, PMD, "MTU requested must be within (%d, %d)\n", + ETHER_MIN_MTU, max_dev_mtu); + return -EINVAL; + } + + + if (new_mtu > ETHER_MTU) { + bp->flags |= BNXT_FLAG_JUMBO; + eth_dev->data->dev_conf.rxmode.jumbo_frame = 1; + } else { + eth_dev->data->dev_conf.rxmode.jumbo_frame = 0; + bp->flags &= ~BNXT_FLAG_JUMBO; + } + + eth_dev->data->dev_conf.rxmode.max_rx_pkt_len = + new_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN + VLAN_TAG_SIZE * 2; + + eth_dev->data->mtu = new_mtu; + RTE_LOG(INFO, PMD, "New MTU is %d\n", eth_dev->data->mtu); + + for (i = 0; i < bp->nr_vnics; i++) { + struct bnxt_vnic_info *vnic = &bp->vnic_info[i]; + + vnic->mru = bp->eth_dev->data->mtu + ETHER_HDR_LEN + + ETHER_CRC_LEN + VLAN_TAG_SIZE * 2; + rc = bnxt_hwrm_vnic_cfg(bp, vnic); + if (rc) + break; + + rc = bnxt_hwrm_vnic_plcmode_cfg(bp, vnic); + if (rc) + return rc; + } + + return rc; +} + +static int +bnxt_vlan_pvid_set_op(struct rte_eth_dev *dev, uint16_t pvid, int on) +{ + struct bnxt *bp = (struct bnxt *)dev->data->dev_private; + uint16_t vlan = bp->vlan; + int rc; + + if (BNXT_NPAR_PF(bp) || BNXT_VF(bp)) { + RTE_LOG(ERR, PMD, + "PVID cannot be modified for this function\n"); + return -ENOTSUP; + } + bp->vlan = on ? pvid : 0; + + rc = bnxt_hwrm_set_default_vlan(bp, 0, 0); + if (rc) + bp->vlan = vlan; + return rc; +} + +static int +bnxt_dev_led_on_op(struct rte_eth_dev *dev) +{ + struct bnxt *bp = (struct bnxt *)dev->data->dev_private; + + return bnxt_hwrm_port_led_cfg(bp, true); +} + +static int +bnxt_dev_led_off_op(struct rte_eth_dev *dev) +{ + struct bnxt *bp = (struct bnxt *)dev->data->dev_private; + + return bnxt_hwrm_port_led_cfg(bp, false); +} + +static uint32_t +bnxt_rx_queue_count_op(struct rte_eth_dev *dev, uint16_t rx_queue_id) +{ + uint32_t desc = 0, raw_cons = 0, cons; + struct bnxt_cp_ring_info *cpr; + struct bnxt_rx_queue *rxq; + struct rx_pkt_cmpl *rxcmp; + uint16_t cmp_type; + uint8_t cmp = 1; + bool valid; + + rxq = dev->data->rx_queues[rx_queue_id]; + cpr = rxq->cp_ring; + valid = cpr->valid; + + while (raw_cons < rxq->nb_rx_desc) { + cons = RING_CMP(cpr->cp_ring_struct, raw_cons); + rxcmp = (struct rx_pkt_cmpl *)&cpr->cp_desc_ring[cons]; + + if (!CMPL_VALID(rxcmp, valid)) + goto nothing_to_do; + valid = FLIP_VALID(cons, cpr->cp_ring_struct->ring_mask, valid); + cmp_type = CMP_TYPE(rxcmp); + if (cmp_type == RX_TPA_END_CMPL_TYPE_RX_TPA_END) { + cmp = (rte_le_to_cpu_32( + ((struct rx_tpa_end_cmpl *) + (rxcmp))->agg_bufs_v1) & + RX_TPA_END_CMPL_AGG_BUFS_MASK) >> + RX_TPA_END_CMPL_AGG_BUFS_SFT; + desc++; + } else if (cmp_type == 0x11) { + desc++; + cmp = (rxcmp->agg_bufs_v1 & + RX_PKT_CMPL_AGG_BUFS_MASK) >> + RX_PKT_CMPL_AGG_BUFS_SFT; + } else { + cmp = 1; + } +nothing_to_do: + raw_cons += cmp ? cmp : 2; + } + + return desc; +} + +static int +bnxt_rx_descriptor_status_op(void *rx_queue, uint16_t offset) +{ + struct bnxt_rx_queue *rxq = (struct bnxt_rx_queue *)rx_queue; + struct bnxt_rx_ring_info *rxr; + struct bnxt_cp_ring_info *cpr; + struct bnxt_sw_rx_bd *rx_buf; + struct rx_pkt_cmpl *rxcmp; + uint32_t cons, cp_cons; + + if (!rxq) + return -EINVAL; + + cpr = rxq->cp_ring; + rxr = rxq->rx_ring; + + if (offset >= rxq->nb_rx_desc) + return -EINVAL; + + cons = RING_CMP(cpr->cp_ring_struct, offset); + cp_cons = cpr->cp_raw_cons; + rxcmp = (struct rx_pkt_cmpl *)&cpr->cp_desc_ring[cons]; + + if (cons > cp_cons) { + if (CMPL_VALID(rxcmp, cpr->valid)) + return RTE_ETH_RX_DESC_DONE; + } else { + if (CMPL_VALID(rxcmp, !cpr->valid)) + return RTE_ETH_RX_DESC_DONE; + } + rx_buf = &rxr->rx_buf_ring[cons]; + if (rx_buf->mbuf == NULL) + return RTE_ETH_RX_DESC_UNAVAIL; + + + return RTE_ETH_RX_DESC_AVAIL; +} + +static int +bnxt_tx_descriptor_status_op(void *tx_queue, uint16_t offset) +{ + struct bnxt_tx_queue *txq = (struct bnxt_tx_queue *)tx_queue; + struct bnxt_tx_ring_info *txr; + struct bnxt_cp_ring_info *cpr; + struct bnxt_sw_tx_bd *tx_buf; + struct tx_pkt_cmpl *txcmp; + uint32_t cons, cp_cons; + + if (!txq) + return -EINVAL; + + cpr = txq->cp_ring; + txr = txq->tx_ring; + + if (offset >= txq->nb_tx_desc) + return -EINVAL; + + cons = RING_CMP(cpr->cp_ring_struct, offset); + txcmp = (struct tx_pkt_cmpl *)&cpr->cp_desc_ring[cons]; + cp_cons = cpr->cp_raw_cons; + + if (cons > cp_cons) { + if (CMPL_VALID(txcmp, cpr->valid)) + return RTE_ETH_TX_DESC_UNAVAIL; + } else { + if (CMPL_VALID(txcmp, !cpr->valid)) + return RTE_ETH_TX_DESC_UNAVAIL; + } + tx_buf = &txr->tx_buf_ring[cons]; + if (tx_buf->mbuf == NULL) + return RTE_ETH_TX_DESC_DONE; + + return RTE_ETH_TX_DESC_FULL; +} + +static struct bnxt_filter_info * +bnxt_match_and_validate_ether_filter(struct bnxt *bp, + struct rte_eth_ethertype_filter *efilter, + struct bnxt_vnic_info *vnic0, + struct bnxt_vnic_info *vnic, + int *ret) +{ + struct bnxt_filter_info *mfilter = NULL; + int match = 0; + *ret = 0; + + if (efilter->ether_type == ETHER_TYPE_IPv4 || + efilter->ether_type == ETHER_TYPE_IPv6) { + RTE_LOG(ERR, PMD, "invalid ether_type(0x%04x) in" + " ethertype filter.", efilter->ether_type); + *ret = -EINVAL; + goto exit; + } + if (efilter->queue >= bp->rx_nr_rings) { + RTE_LOG(ERR, PMD, "Invalid queue %d\n", efilter->queue); + *ret = -EINVAL; + goto exit; + } + + vnic0 = STAILQ_FIRST(&bp->ff_pool[0]); + vnic = STAILQ_FIRST(&bp->ff_pool[efilter->queue]); + if (vnic == NULL) { + RTE_LOG(ERR, PMD, "Invalid queue %d\n", efilter->queue); + *ret = -EINVAL; + goto exit; + } + + if (efilter->flags & RTE_ETHTYPE_FLAGS_DROP) { + STAILQ_FOREACH(mfilter, &vnic0->filter, next) { + if ((!memcmp(efilter->mac_addr.addr_bytes, + mfilter->l2_addr, ETHER_ADDR_LEN) && + mfilter->flags == + HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_FLAGS_DROP && + mfilter->ethertype == efilter->ether_type)) { + match = 1; + break; + } + } + } else { + STAILQ_FOREACH(mfilter, &vnic->filter, next) + if ((!memcmp(efilter->mac_addr.addr_bytes, + mfilter->l2_addr, ETHER_ADDR_LEN) && + mfilter->ethertype == efilter->ether_type && + mfilter->flags == + HWRM_CFA_L2_FILTER_CFG_INPUT_FLAGS_PATH_RX)) { + match = 1; + break; + } + } + + if (match) + *ret = -EEXIST; + +exit: + return mfilter; +} + +static int +bnxt_ethertype_filter(struct rte_eth_dev *dev, + enum rte_filter_op filter_op, + void *arg) +{ + struct bnxt *bp = (struct bnxt *)dev->data->dev_private; + struct rte_eth_ethertype_filter *efilter = + (struct rte_eth_ethertype_filter *)arg; + struct bnxt_filter_info *bfilter, *filter1; + struct bnxt_vnic_info *vnic, *vnic0; + int ret; + + if (filter_op == RTE_ETH_FILTER_NOP) + return 0; + + if (arg == NULL) { + RTE_LOG(ERR, PMD, "arg shouldn't be NULL for operation %u.", + filter_op); + return -EINVAL; + } + + vnic0 = STAILQ_FIRST(&bp->ff_pool[0]); + vnic = STAILQ_FIRST(&bp->ff_pool[efilter->queue]); + + switch (filter_op) { + case RTE_ETH_FILTER_ADD: + bnxt_match_and_validate_ether_filter(bp, efilter, + vnic0, vnic, &ret); + if (ret < 0) + return ret; + + bfilter = bnxt_get_unused_filter(bp); + if (bfilter == NULL) { + RTE_LOG(ERR, PMD, + "Not enough resources for a new filter.\n"); + return -ENOMEM; + } + bfilter->filter_type = HWRM_CFA_NTUPLE_FILTER; + memcpy(bfilter->l2_addr, efilter->mac_addr.addr_bytes, + ETHER_ADDR_LEN); + memcpy(bfilter->dst_macaddr, efilter->mac_addr.addr_bytes, + ETHER_ADDR_LEN); + bfilter->enables |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_MACADDR; + bfilter->ethertype = efilter->ether_type; + bfilter->enables |= NTUPLE_FLTR_ALLOC_INPUT_EN_ETHERTYPE; + + filter1 = bnxt_get_l2_filter(bp, bfilter, vnic0); + if (filter1 == NULL) { + ret = -1; + goto cleanup; + } + bfilter->enables |= + HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_L2_FILTER_ID; + bfilter->fw_l2_filter_id = filter1->fw_l2_filter_id; + + bfilter->dst_id = vnic->fw_vnic_id; + + if (efilter->flags & RTE_ETHTYPE_FLAGS_DROP) { + bfilter->flags = + HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_FLAGS_DROP; + } + + ret = bnxt_hwrm_set_ntuple_filter(bp, bfilter->dst_id, bfilter); + if (ret) + goto cleanup; + STAILQ_INSERT_TAIL(&vnic->filter, bfilter, next); + break; + case RTE_ETH_FILTER_DELETE: + filter1 = bnxt_match_and_validate_ether_filter(bp, efilter, + vnic0, vnic, &ret); + if (ret == -EEXIST) { + ret = bnxt_hwrm_clear_ntuple_filter(bp, filter1); + + STAILQ_REMOVE(&vnic->filter, filter1, bnxt_filter_info, + next); + bnxt_free_filter(bp, filter1); + } else if (ret == 0) { + RTE_LOG(ERR, PMD, "No matching filter found\n"); + } + break; + default: + RTE_LOG(ERR, PMD, "unsupported operation %u.", filter_op); + ret = -EINVAL; + goto error; + } + return ret; +cleanup: + bnxt_free_filter(bp, bfilter); +error: + return ret; +} + +static inline int +parse_ntuple_filter(struct bnxt *bp, + struct rte_eth_ntuple_filter *nfilter, + struct bnxt_filter_info *bfilter) +{ + uint32_t en = 0; + + if (nfilter->queue >= bp->rx_nr_rings) { + RTE_LOG(ERR, PMD, "Invalid queue %d\n", nfilter->queue); + return -EINVAL; + } + + switch (nfilter->dst_port_mask) { + case UINT16_MAX: + bfilter->dst_port_mask = -1; + bfilter->dst_port = nfilter->dst_port; + en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_PORT | + NTUPLE_FLTR_ALLOC_INPUT_EN_DST_PORT_MASK; + break; + default: + RTE_LOG(ERR, PMD, "invalid dst_port mask."); + return -EINVAL; + } + + bfilter->ip_addr_type = NTUPLE_FLTR_ALLOC_INPUT_IP_ADDR_TYPE_IPV4; + en |= NTUPLE_FLTR_ALLOC_IN_EN_IP_PROTO; + + switch (nfilter->proto_mask) { + case UINT8_MAX: + if (nfilter->proto == 17) /* IPPROTO_UDP */ + bfilter->ip_protocol = 17; + else if (nfilter->proto == 6) /* IPPROTO_TCP */ + bfilter->ip_protocol = 6; + else + return -EINVAL; + en |= NTUPLE_FLTR_ALLOC_IN_EN_IP_PROTO; + break; + default: + RTE_LOG(ERR, PMD, "invalid protocol mask."); + return -EINVAL; + } + + switch (nfilter->dst_ip_mask) { + case UINT32_MAX: + bfilter->dst_ipaddr_mask[0] = -1; + bfilter->dst_ipaddr[0] = nfilter->dst_ip; + en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR | + NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR_MASK; + break; + default: + RTE_LOG(ERR, PMD, "invalid dst_ip mask."); + return -EINVAL; + } + + switch (nfilter->src_ip_mask) { + case UINT32_MAX: + bfilter->src_ipaddr_mask[0] = -1; + bfilter->src_ipaddr[0] = nfilter->src_ip; + en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR | + NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR_MASK; + break; + default: + RTE_LOG(ERR, PMD, "invalid src_ip mask."); + return -EINVAL; + } + + switch (nfilter->src_port_mask) { + case UINT16_MAX: + bfilter->src_port_mask = -1; + bfilter->src_port = nfilter->src_port; + en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_PORT | + NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_PORT_MASK; + break; + default: + RTE_LOG(ERR, PMD, "invalid src_port mask."); + return -EINVAL; + } + + //TODO Priority + //nfilter->priority = (uint8_t)filter->priority; + + bfilter->enables = en; + return 0; +} + +static struct bnxt_filter_info* +bnxt_match_ntuple_filter(struct bnxt *bp, + struct bnxt_filter_info *bfilter, + struct bnxt_vnic_info **mvnic) +{ + struct bnxt_filter_info *mfilter = NULL; + int i; + + for (i = bp->nr_vnics - 1; i >= 0; i--) { + struct bnxt_vnic_info *vnic = &bp->vnic_info[i]; + STAILQ_FOREACH(mfilter, &vnic->filter, next) { + if (bfilter->src_ipaddr[0] == mfilter->src_ipaddr[0] && + bfilter->src_ipaddr_mask[0] == + mfilter->src_ipaddr_mask[0] && + bfilter->src_port == mfilter->src_port && + bfilter->src_port_mask == mfilter->src_port_mask && + bfilter->dst_ipaddr[0] == mfilter->dst_ipaddr[0] && + bfilter->dst_ipaddr_mask[0] == + mfilter->dst_ipaddr_mask[0] && + bfilter->dst_port == mfilter->dst_port && + bfilter->dst_port_mask == mfilter->dst_port_mask && + bfilter->flags == mfilter->flags && + bfilter->enables == mfilter->enables) { + if (mvnic) + *mvnic = vnic; + return mfilter; + } + } + } + return NULL; +} + +static int +bnxt_cfg_ntuple_filter(struct bnxt *bp, + struct rte_eth_ntuple_filter *nfilter, + enum rte_filter_op filter_op) +{ + struct bnxt_filter_info *bfilter, *mfilter, *filter1; + struct bnxt_vnic_info *vnic, *vnic0, *mvnic; + int ret; + + if (nfilter->flags != RTE_5TUPLE_FLAGS) { + RTE_LOG(ERR, PMD, "only 5tuple is supported."); + return -EINVAL; + } + + if (nfilter->flags & RTE_NTUPLE_FLAGS_TCP_FLAG) { + RTE_LOG(ERR, PMD, "Ntuple filter: TCP flags not supported\n"); + return -EINVAL; + } + + bfilter = bnxt_get_unused_filter(bp); + if (bfilter == NULL) { + RTE_LOG(ERR, PMD, + "Not enough resources for a new filter.\n"); + return -ENOMEM; + } + ret = parse_ntuple_filter(bp, nfilter, bfilter); + if (ret < 0) + goto free_filter; + + vnic = STAILQ_FIRST(&bp->ff_pool[nfilter->queue]); + vnic0 = STAILQ_FIRST(&bp->ff_pool[0]); + filter1 = STAILQ_FIRST(&vnic0->filter); + if (filter1 == NULL) { + ret = -1; + goto free_filter; + } + + bfilter->dst_id = vnic->fw_vnic_id; + bfilter->fw_l2_filter_id = filter1->fw_l2_filter_id; + bfilter->enables |= + HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_L2_FILTER_ID; + bfilter->ethertype = 0x800; + bfilter->enables |= NTUPLE_FLTR_ALLOC_INPUT_EN_ETHERTYPE; + + mfilter = bnxt_match_ntuple_filter(bp, bfilter, &mvnic); + + if (mfilter != NULL && filter_op == RTE_ETH_FILTER_ADD && + bfilter->dst_id == mfilter->dst_id) { + RTE_LOG(ERR, PMD, "filter exists.\n"); + ret = -EEXIST; + goto free_filter; + } else if (mfilter != NULL && filter_op == RTE_ETH_FILTER_ADD && + bfilter->dst_id != mfilter->dst_id) { + mfilter->dst_id = vnic->fw_vnic_id; + ret = bnxt_hwrm_set_ntuple_filter(bp, mfilter->dst_id, mfilter); + STAILQ_REMOVE(&mvnic->filter, mfilter, bnxt_filter_info, next); + STAILQ_INSERT_TAIL(&vnic->filter, mfilter, next); + RTE_LOG(ERR, PMD, "filter with matching pattern exists.\n"); + RTE_LOG(ERR, PMD, " Updated it to the new destination queue\n"); + goto free_filter; + } + if (mfilter == NULL && filter_op == RTE_ETH_FILTER_DELETE) { + RTE_LOG(ERR, PMD, "filter doesn't exist."); + ret = -ENOENT; + goto free_filter; + } + + if (filter_op == RTE_ETH_FILTER_ADD) { + bfilter->filter_type = HWRM_CFA_NTUPLE_FILTER; + ret = bnxt_hwrm_set_ntuple_filter(bp, bfilter->dst_id, bfilter); + if (ret) + goto free_filter; + STAILQ_INSERT_TAIL(&vnic->filter, bfilter, next); + } else { + if (mfilter == NULL) { + /* This should not happen. But for Coverity! */ + ret = -ENOENT; + goto free_filter; + } + ret = bnxt_hwrm_clear_ntuple_filter(bp, mfilter); + + STAILQ_REMOVE(&vnic->filter, mfilter, bnxt_filter_info, next); + bnxt_free_filter(bp, mfilter); + mfilter->fw_l2_filter_id = -1; + bnxt_free_filter(bp, bfilter); + bfilter->fw_l2_filter_id = -1; + } + + return 0; +free_filter: + bfilter->fw_l2_filter_id = -1; + bnxt_free_filter(bp, bfilter); + return ret; +} + +static int +bnxt_ntuple_filter(struct rte_eth_dev *dev, + enum rte_filter_op filter_op, + void *arg) +{ + struct bnxt *bp = (struct bnxt *)dev->data->dev_private; + int ret; + + if (filter_op == RTE_ETH_FILTER_NOP) + return 0; + + if (arg == NULL) { + RTE_LOG(ERR, PMD, "arg shouldn't be NULL for operation %u.", + filter_op); + return -EINVAL; + } + + switch (filter_op) { + case RTE_ETH_FILTER_ADD: + ret = bnxt_cfg_ntuple_filter(bp, + (struct rte_eth_ntuple_filter *)arg, + filter_op); + break; + case RTE_ETH_FILTER_DELETE: + ret = bnxt_cfg_ntuple_filter(bp, + (struct rte_eth_ntuple_filter *)arg, + filter_op); + break; + default: + RTE_LOG(ERR, PMD, "unsupported operation %u.", filter_op); + ret = -EINVAL; + break; + } + return ret; +} + +static int +bnxt_parse_fdir_filter(struct bnxt *bp, + struct rte_eth_fdir_filter *fdir, + struct bnxt_filter_info *filter) +{ + enum rte_fdir_mode fdir_mode = + bp->eth_dev->data->dev_conf.fdir_conf.mode; + struct bnxt_vnic_info *vnic0, *vnic; + struct bnxt_filter_info *filter1; + uint32_t en = 0; + int i; + + if (fdir_mode == RTE_FDIR_MODE_PERFECT_TUNNEL) + return -EINVAL; + + filter->l2_ovlan = fdir->input.flow_ext.vlan_tci; + en |= EM_FLOW_ALLOC_INPUT_EN_OVLAN_VID; + + switch (fdir->input.flow_type) { + case RTE_ETH_FLOW_IPV4: + case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: + /* FALLTHROUGH */ + filter->src_ipaddr[0] = fdir->input.flow.ip4_flow.src_ip; + en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR; + filter->dst_ipaddr[0] = fdir->input.flow.ip4_flow.dst_ip; + en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR; + filter->ip_protocol = fdir->input.flow.ip4_flow.proto; + en |= NTUPLE_FLTR_ALLOC_IN_EN_IP_PROTO; + filter->ip_addr_type = + NTUPLE_FLTR_ALLOC_INPUT_IP_ADDR_TYPE_IPV4; + filter->src_ipaddr_mask[0] = 0xffffffff; + en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR_MASK; + filter->dst_ipaddr_mask[0] = 0xffffffff; + en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR_MASK; + filter->ethertype = 0x800; + filter->enables |= NTUPLE_FLTR_ALLOC_INPUT_EN_ETHERTYPE; + break; + case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: + filter->src_port = fdir->input.flow.tcp4_flow.src_port; + en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_PORT; + filter->dst_port = fdir->input.flow.tcp4_flow.dst_port; + en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_PORT; + filter->dst_port_mask = 0xffff; + en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_PORT_MASK; + filter->src_port_mask = 0xffff; + en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_PORT_MASK; + filter->src_ipaddr[0] = fdir->input.flow.tcp4_flow.ip.src_ip; + en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR; + filter->dst_ipaddr[0] = fdir->input.flow.tcp4_flow.ip.dst_ip; + en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR; + filter->ip_protocol = 6; + en |= NTUPLE_FLTR_ALLOC_IN_EN_IP_PROTO; + filter->ip_addr_type = + NTUPLE_FLTR_ALLOC_INPUT_IP_ADDR_TYPE_IPV4; + filter->src_ipaddr_mask[0] = 0xffffffff; + en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR_MASK; + filter->dst_ipaddr_mask[0] = 0xffffffff; + en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR_MASK; + filter->ethertype = 0x800; + filter->enables |= NTUPLE_FLTR_ALLOC_INPUT_EN_ETHERTYPE; + break; + case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: + filter->src_port = fdir->input.flow.udp4_flow.src_port; + en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_PORT; + filter->dst_port = fdir->input.flow.udp4_flow.dst_port; + en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_PORT; + filter->dst_port_mask = 0xffff; + en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_PORT_MASK; + filter->src_port_mask = 0xffff; + en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_PORT_MASK; + filter->src_ipaddr[0] = fdir->input.flow.udp4_flow.ip.src_ip; + en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR; + filter->dst_ipaddr[0] = fdir->input.flow.udp4_flow.ip.dst_ip; + en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR; + filter->ip_protocol = 17; + en |= NTUPLE_FLTR_ALLOC_IN_EN_IP_PROTO; + filter->ip_addr_type = + NTUPLE_FLTR_ALLOC_INPUT_IP_ADDR_TYPE_IPV4; + filter->src_ipaddr_mask[0] = 0xffffffff; + en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR_MASK; + filter->dst_ipaddr_mask[0] = 0xffffffff; + en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR_MASK; + filter->ethertype = 0x800; + filter->enables |= NTUPLE_FLTR_ALLOC_INPUT_EN_ETHERTYPE; + break; + case RTE_ETH_FLOW_IPV6: + case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: + /* FALLTHROUGH */ + filter->ip_addr_type = + NTUPLE_FLTR_ALLOC_INPUT_IP_ADDR_TYPE_IPV6; + filter->ip_protocol = fdir->input.flow.ipv6_flow.proto; + en |= NTUPLE_FLTR_ALLOC_IN_EN_IP_PROTO; + rte_memcpy(filter->src_ipaddr, + fdir->input.flow.ipv6_flow.src_ip, 16); + en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR; + rte_memcpy(filter->dst_ipaddr, + fdir->input.flow.ipv6_flow.dst_ip, 16); + en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR; + memset(filter->dst_ipaddr_mask, 0xff, 16); + en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR_MASK; + memset(filter->src_ipaddr_mask, 0xff, 16); + en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR_MASK; + filter->ethertype = 0x86dd; + filter->enables |= NTUPLE_FLTR_ALLOC_INPUT_EN_ETHERTYPE; + break; + case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: + filter->src_port = fdir->input.flow.tcp6_flow.src_port; + en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_PORT; + filter->dst_port = fdir->input.flow.tcp6_flow.dst_port; + en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_PORT; + filter->dst_port_mask = 0xffff; + en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_PORT_MASK; + filter->src_port_mask = 0xffff; + en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_PORT_MASK; + filter->ip_addr_type = + NTUPLE_FLTR_ALLOC_INPUT_IP_ADDR_TYPE_IPV6; + filter->ip_protocol = fdir->input.flow.tcp6_flow.ip.proto; + en |= NTUPLE_FLTR_ALLOC_IN_EN_IP_PROTO; + rte_memcpy(filter->src_ipaddr, + fdir->input.flow.tcp6_flow.ip.src_ip, 16); + en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR; + rte_memcpy(filter->dst_ipaddr, + fdir->input.flow.tcp6_flow.ip.dst_ip, 16); + en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR; + memset(filter->dst_ipaddr_mask, 0xff, 16); + en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR_MASK; + memset(filter->src_ipaddr_mask, 0xff, 16); + en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR_MASK; + filter->ethertype = 0x86dd; + filter->enables |= NTUPLE_FLTR_ALLOC_INPUT_EN_ETHERTYPE; + break; + case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: + filter->src_port = fdir->input.flow.udp6_flow.src_port; + en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_PORT; + filter->dst_port = fdir->input.flow.udp6_flow.dst_port; + en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_PORT; + filter->dst_port_mask = 0xffff; + en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_PORT_MASK; + filter->src_port_mask = 0xffff; + en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_PORT_MASK; + filter->ip_addr_type = + NTUPLE_FLTR_ALLOC_INPUT_IP_ADDR_TYPE_IPV6; + filter->ip_protocol = fdir->input.flow.udp6_flow.ip.proto; + en |= NTUPLE_FLTR_ALLOC_IN_EN_IP_PROTO; + rte_memcpy(filter->src_ipaddr, + fdir->input.flow.udp6_flow.ip.src_ip, 16); + en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR; + rte_memcpy(filter->dst_ipaddr, + fdir->input.flow.udp6_flow.ip.dst_ip, 16); + en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR; + memset(filter->dst_ipaddr_mask, 0xff, 16); + en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR_MASK; + memset(filter->src_ipaddr_mask, 0xff, 16); + en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR_MASK; + filter->ethertype = 0x86dd; + filter->enables |= NTUPLE_FLTR_ALLOC_INPUT_EN_ETHERTYPE; + break; + case RTE_ETH_FLOW_L2_PAYLOAD: + filter->ethertype = fdir->input.flow.l2_flow.ether_type; + en |= NTUPLE_FLTR_ALLOC_INPUT_EN_ETHERTYPE; + break; + case RTE_ETH_FLOW_VXLAN: + if (fdir->action.behavior == RTE_ETH_FDIR_REJECT) + return -EINVAL; + filter->vni = fdir->input.flow.tunnel_flow.tunnel_id; + filter->tunnel_type = + CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN; + en |= HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_TUNNEL_TYPE; + break; + case RTE_ETH_FLOW_NVGRE: + if (fdir->action.behavior == RTE_ETH_FDIR_REJECT) + return -EINVAL; + filter->vni = fdir->input.flow.tunnel_flow.tunnel_id; + filter->tunnel_type = + CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_NVGRE; + en |= HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_TUNNEL_TYPE; + break; + case RTE_ETH_FLOW_UNKNOWN: + case RTE_ETH_FLOW_RAW: + case RTE_ETH_FLOW_FRAG_IPV4: + case RTE_ETH_FLOW_NONFRAG_IPV4_SCTP: + case RTE_ETH_FLOW_FRAG_IPV6: + case RTE_ETH_FLOW_NONFRAG_IPV6_SCTP: + case RTE_ETH_FLOW_IPV6_EX: + case RTE_ETH_FLOW_IPV6_TCP_EX: + case RTE_ETH_FLOW_IPV6_UDP_EX: + case RTE_ETH_FLOW_GENEVE: + /* FALLTHROUGH */ + default: + return -EINVAL; + } + + vnic0 = STAILQ_FIRST(&bp->ff_pool[0]); + vnic = STAILQ_FIRST(&bp->ff_pool[fdir->action.rx_queue]); + if (vnic == NULL) { + RTE_LOG(ERR, PMD, "Invalid queue %d\n", fdir->action.rx_queue); + return -EINVAL; + } + + + if (fdir_mode == RTE_FDIR_MODE_PERFECT_MAC_VLAN) { + rte_memcpy(filter->dst_macaddr, + fdir->input.flow.mac_vlan_flow.mac_addr.addr_bytes, 6); + en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_MACADDR; + } + + if (fdir->action.behavior == RTE_ETH_FDIR_REJECT) { + filter->flags = HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_FLAGS_DROP; + filter1 = STAILQ_FIRST(&vnic0->filter); + //filter1 = bnxt_get_l2_filter(bp, filter, vnic0); + } else { + filter->dst_id = vnic->fw_vnic_id; + for (i = 0; i < ETHER_ADDR_LEN; i++) + if (filter->dst_macaddr[i] == 0x00) + filter1 = STAILQ_FIRST(&vnic0->filter); + else + filter1 = bnxt_get_l2_filter(bp, filter, vnic); + } + + if (filter1 == NULL) + return -EINVAL; + + en |= HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_L2_FILTER_ID; + filter->fw_l2_filter_id = filter1->fw_l2_filter_id; + + filter->enables = en; + + return 0; +} + +static struct bnxt_filter_info * +bnxt_match_fdir(struct bnxt *bp, struct bnxt_filter_info *nf) +{ + struct bnxt_filter_info *mf = NULL; + int i; + + for (i = bp->nr_vnics - 1; i >= 0; i--) { + struct bnxt_vnic_info *vnic = &bp->vnic_info[i]; + + STAILQ_FOREACH(mf, &vnic->filter, next) { + if (mf->filter_type == nf->filter_type && + mf->flags == nf->flags && + mf->src_port == nf->src_port && + mf->src_port_mask == nf->src_port_mask && + mf->dst_port == nf->dst_port && + mf->dst_port_mask == nf->dst_port_mask && + mf->ip_protocol == nf->ip_protocol && + mf->ip_addr_type == nf->ip_addr_type && + mf->ethertype == nf->ethertype && + mf->vni == nf->vni && + mf->tunnel_type == nf->tunnel_type && + mf->l2_ovlan == nf->l2_ovlan && + mf->l2_ovlan_mask == nf->l2_ovlan_mask && + mf->l2_ivlan == nf->l2_ivlan && + mf->l2_ivlan_mask == nf->l2_ivlan_mask && + !memcmp(mf->l2_addr, nf->l2_addr, ETHER_ADDR_LEN) && + !memcmp(mf->l2_addr_mask, nf->l2_addr_mask, + ETHER_ADDR_LEN) && + !memcmp(mf->src_macaddr, nf->src_macaddr, + ETHER_ADDR_LEN) && + !memcmp(mf->dst_macaddr, nf->dst_macaddr, + ETHER_ADDR_LEN) && + !memcmp(mf->src_ipaddr, nf->src_ipaddr, + sizeof(nf->src_ipaddr)) && + !memcmp(mf->src_ipaddr_mask, nf->src_ipaddr_mask, + sizeof(nf->src_ipaddr_mask)) && + !memcmp(mf->dst_ipaddr, nf->dst_ipaddr, + sizeof(nf->dst_ipaddr)) && + !memcmp(mf->dst_ipaddr_mask, nf->dst_ipaddr_mask, + sizeof(nf->dst_ipaddr_mask))) + return mf; + } + } + return NULL; +} + +static int +bnxt_fdir_filter(struct rte_eth_dev *dev, + enum rte_filter_op filter_op, + void *arg) +{ + struct bnxt *bp = (struct bnxt *)dev->data->dev_private; + struct rte_eth_fdir_filter *fdir = (struct rte_eth_fdir_filter *)arg; + struct bnxt_filter_info *filter, *match; + struct bnxt_vnic_info *vnic; + int ret = 0, i; + + if (filter_op == RTE_ETH_FILTER_NOP) + return 0; + + if (arg == NULL && filter_op != RTE_ETH_FILTER_FLUSH) + return -EINVAL; + + switch (filter_op) { + case RTE_ETH_FILTER_ADD: + case RTE_ETH_FILTER_DELETE: + /* FALLTHROUGH */ + filter = bnxt_get_unused_filter(bp); + if (filter == NULL) { + RTE_LOG(ERR, PMD, + "Not enough resources for a new flow.\n"); + return -ENOMEM; + } + + ret = bnxt_parse_fdir_filter(bp, fdir, filter); + if (ret != 0) + goto free_filter; + filter->filter_type = HWRM_CFA_NTUPLE_FILTER; + + match = bnxt_match_fdir(bp, filter); + if (match != NULL && filter_op == RTE_ETH_FILTER_ADD) { + RTE_LOG(ERR, PMD, "Flow already exists.\n"); + ret = -EEXIST; + goto free_filter; + } + if (match == NULL && filter_op == RTE_ETH_FILTER_DELETE) { + RTE_LOG(ERR, PMD, "Flow does not exist.\n"); + ret = -ENOENT; + goto free_filter; + } + + if (fdir->action.behavior == RTE_ETH_FDIR_REJECT) + vnic = STAILQ_FIRST(&bp->ff_pool[0]); + else + vnic = + STAILQ_FIRST(&bp->ff_pool[fdir->action.rx_queue]); + + if (filter_op == RTE_ETH_FILTER_ADD) { + ret = bnxt_hwrm_set_ntuple_filter(bp, + filter->dst_id, + filter); + if (ret) + goto free_filter; + STAILQ_INSERT_TAIL(&vnic->filter, filter, next); + } else { + ret = bnxt_hwrm_clear_ntuple_filter(bp, match); + STAILQ_REMOVE(&vnic->filter, match, + bnxt_filter_info, next); + bnxt_free_filter(bp, match); + filter->fw_l2_filter_id = -1; + bnxt_free_filter(bp, filter); + } + break; + case RTE_ETH_FILTER_FLUSH: + for (i = bp->nr_vnics - 1; i >= 0; i--) { + struct bnxt_vnic_info *vnic = &bp->vnic_info[i]; + + STAILQ_FOREACH(filter, &vnic->filter, next) { + if (filter->filter_type == + HWRM_CFA_NTUPLE_FILTER) { + ret = + bnxt_hwrm_clear_ntuple_filter(bp, + filter); + STAILQ_REMOVE(&vnic->filter, filter, + bnxt_filter_info, next); + } + } + } + return ret; + case RTE_ETH_FILTER_UPDATE: + case RTE_ETH_FILTER_STATS: + case RTE_ETH_FILTER_INFO: + /* FALLTHROUGH */ + RTE_LOG(ERR, PMD, "operation %u not implemented", filter_op); + break; + default: + RTE_LOG(ERR, PMD, "unknown operation %u", filter_op); + ret = -EINVAL; + break; + } + return ret; + +free_filter: + filter->fw_l2_filter_id = -1; + bnxt_free_filter(bp, filter); + return ret; +} + +static int +bnxt_filter_ctrl_op(struct rte_eth_dev *dev __rte_unused, + enum rte_filter_type filter_type, + enum rte_filter_op filter_op, void *arg) +{ + int ret = 0; + + switch (filter_type) { + case RTE_ETH_FILTER_TUNNEL: + RTE_LOG(ERR, PMD, + "filter type: %d: To be implemented\n", filter_type); + break; + case RTE_ETH_FILTER_FDIR: + ret = bnxt_fdir_filter(dev, filter_op, arg); + break; + case RTE_ETH_FILTER_NTUPLE: + ret = bnxt_ntuple_filter(dev, filter_op, arg); + break; + case RTE_ETH_FILTER_ETHERTYPE: + ret = bnxt_ethertype_filter(dev, filter_op, arg); + break; + case RTE_ETH_FILTER_GENERIC: + if (filter_op != RTE_ETH_FILTER_GET) + return -EINVAL; + *(const void **)arg = &bnxt_flow_ops; + break; + default: + RTE_LOG(ERR, PMD, + "Filter type (%d) not supported", filter_type); + ret = -EINVAL; + break; + } + return ret; +} + +static const uint32_t * +bnxt_dev_supported_ptypes_get_op(struct rte_eth_dev *dev) +{ + static const uint32_t ptypes[] = { + RTE_PTYPE_L2_ETHER_VLAN, + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN, + RTE_PTYPE_L3_IPV6_EXT_UNKNOWN, + RTE_PTYPE_L4_ICMP, + RTE_PTYPE_L4_TCP, + RTE_PTYPE_L4_UDP, + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN, + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN, + RTE_PTYPE_INNER_L4_ICMP, + RTE_PTYPE_INNER_L4_TCP, + RTE_PTYPE_INNER_L4_UDP, + RTE_PTYPE_UNKNOWN + }; + + if (dev->rx_pkt_burst == bnxt_recv_pkts) + return ptypes; + return NULL; +} + + + +static int +bnxt_get_eeprom_length_op(struct rte_eth_dev *dev) +{ + struct bnxt *bp = (struct bnxt *)dev->data->dev_private; + int rc; + uint32_t dir_entries; + uint32_t entry_length; + + RTE_LOG(INFO, PMD, "%s(): %04x:%02x:%02x:%02x\n", + __func__, bp->pdev->addr.domain, bp->pdev->addr.bus, + bp->pdev->addr.devid, bp->pdev->addr.function); + + rc = bnxt_hwrm_nvm_get_dir_info(bp, &dir_entries, &entry_length); + if (rc != 0) + return rc; + + return dir_entries * entry_length; +} + +static int +bnxt_get_eeprom_op(struct rte_eth_dev *dev, + struct rte_dev_eeprom_info *in_eeprom) +{ + struct bnxt *bp = (struct bnxt *)dev->data->dev_private; + uint32_t index; + uint32_t offset; + + RTE_LOG(INFO, PMD, "%s(): %04x:%02x:%02x:%02x in_eeprom->offset = %d " + "len = %d\n", __func__, bp->pdev->addr.domain, + bp->pdev->addr.bus, bp->pdev->addr.devid, + bp->pdev->addr.function, in_eeprom->offset, in_eeprom->length); + + if (in_eeprom->offset == 0) /* special offset value to get directory */ + return bnxt_get_nvram_directory(bp, in_eeprom->length, + in_eeprom->data); + + index = in_eeprom->offset >> 24; + offset = in_eeprom->offset & 0xffffff; + + if (index != 0) + return bnxt_hwrm_get_nvram_item(bp, index - 1, offset, + in_eeprom->length, in_eeprom->data); + + return 0; +} + +static bool bnxt_dir_type_is_ape_bin_format(uint16_t dir_type) +{ + switch (dir_type) { + case BNX_DIR_TYPE_CHIMP_PATCH: + case BNX_DIR_TYPE_BOOTCODE: + case BNX_DIR_TYPE_BOOTCODE_2: + case BNX_DIR_TYPE_APE_FW: + case BNX_DIR_TYPE_APE_PATCH: + case BNX_DIR_TYPE_KONG_FW: + case BNX_DIR_TYPE_KONG_PATCH: + case BNX_DIR_TYPE_BONO_FW: + case BNX_DIR_TYPE_BONO_PATCH: + return true; + } + + return false; +} + +static bool bnxt_dir_type_is_other_exec_format(uint16_t dir_type) +{ + switch (dir_type) { + case BNX_DIR_TYPE_AVS: + case BNX_DIR_TYPE_EXP_ROM_MBA: + case BNX_DIR_TYPE_PCIE: + case BNX_DIR_TYPE_TSCF_UCODE: + case BNX_DIR_TYPE_EXT_PHY: + case BNX_DIR_TYPE_CCM: + case BNX_DIR_TYPE_ISCSI_BOOT: + case BNX_DIR_TYPE_ISCSI_BOOT_IPV6: + case BNX_DIR_TYPE_ISCSI_BOOT_IPV4N6: + return true; + } + + return false; +} + +static bool bnxt_dir_type_is_executable(uint16_t dir_type) +{ + return bnxt_dir_type_is_ape_bin_format(dir_type) || + bnxt_dir_type_is_other_exec_format(dir_type); +} + +static int +bnxt_set_eeprom_op(struct rte_eth_dev *dev, + struct rte_dev_eeprom_info *in_eeprom) +{ + struct bnxt *bp = (struct bnxt *)dev->data->dev_private; + uint8_t index, dir_op; + uint16_t type, ext, ordinal, attr; + + RTE_LOG(INFO, PMD, "%s(): %04x:%02x:%02x:%02x in_eeprom->offset = %d " + "len = %d\n", __func__, bp->pdev->addr.domain, + bp->pdev->addr.bus, bp->pdev->addr.devid, + bp->pdev->addr.function, in_eeprom->offset, in_eeprom->length); + + if (!BNXT_PF(bp)) { + RTE_LOG(ERR, PMD, "NVM write not supported from a VF\n"); + return -EINVAL; + } + + type = in_eeprom->magic >> 16; + + if (type == 0xffff) { /* special value for directory operations */ + index = in_eeprom->magic & 0xff; + dir_op = in_eeprom->magic >> 8; + if (index == 0) + return -EINVAL; + switch (dir_op) { + case 0x0e: /* erase */ + if (in_eeprom->offset != ~in_eeprom->magic) + return -EINVAL; + return bnxt_hwrm_erase_nvram_directory(bp, index - 1); + default: + return -EINVAL; + } + } + + /* Create or re-write an NVM item: */ + if (bnxt_dir_type_is_executable(type) == true) + return -EOPNOTSUPP; + ext = in_eeprom->magic & 0xffff; + ordinal = in_eeprom->offset >> 16; + attr = in_eeprom->offset & 0xffff; + + return bnxt_hwrm_flash_nvram(bp, type, ordinal, ext, attr, + in_eeprom->data, in_eeprom->length); + return 0; +} + /* * Initialization */ -static struct eth_dev_ops bnxt_dev_ops = { +static const struct eth_dev_ops bnxt_dev_ops = { .dev_infos_get = bnxt_dev_info_get_op, .dev_close = bnxt_dev_close_op, .dev_configure = bnxt_dev_configure_op, @@ -879,6 +2708,8 @@ static struct eth_dev_ops bnxt_dev_ops = { .rx_queue_release = bnxt_rx_queue_release_op, .tx_queue_setup = bnxt_tx_queue_setup_op, .tx_queue_release = bnxt_tx_queue_release_op, + .rx_queue_intr_enable = bnxt_rx_queue_intr_enable_op, + .rx_queue_intr_disable = bnxt_rx_queue_intr_disable_op, .reta_update = bnxt_reta_update_op, .reta_query = bnxt_reta_query_op, .rss_hash_update = bnxt_rss_hash_update_op, @@ -892,23 +2723,54 @@ static struct eth_dev_ops bnxt_dev_ops = { .mac_addr_remove = bnxt_mac_addr_remove_op, .flow_ctrl_get = bnxt_flow_ctrl_get_op, .flow_ctrl_set = bnxt_flow_ctrl_set_op, + .udp_tunnel_port_add = bnxt_udp_tunnel_port_add_op, + .udp_tunnel_port_del = bnxt_udp_tunnel_port_del_op, + .vlan_filter_set = bnxt_vlan_filter_set_op, + .vlan_offload_set = bnxt_vlan_offload_set_op, + .vlan_pvid_set = bnxt_vlan_pvid_set_op, + .mtu_set = bnxt_mtu_set_op, + .mac_addr_set = bnxt_set_default_mac_addr_op, + .xstats_get = bnxt_dev_xstats_get_op, + .xstats_get_names = bnxt_dev_xstats_get_names_op, + .xstats_reset = bnxt_dev_xstats_reset_op, + .fw_version_get = bnxt_fw_version_get, + .set_mc_addr_list = bnxt_dev_set_mc_addr_list_op, + .rxq_info_get = bnxt_rxq_info_get_op, + .txq_info_get = bnxt_txq_info_get_op, + .dev_led_on = bnxt_dev_led_on_op, + .dev_led_off = bnxt_dev_led_off_op, + .xstats_get_by_id = bnxt_dev_xstats_get_by_id_op, + .xstats_get_names_by_id = bnxt_dev_xstats_get_names_by_id_op, + .rx_queue_count = bnxt_rx_queue_count_op, + .rx_descriptor_status = bnxt_rx_descriptor_status_op, + .tx_descriptor_status = bnxt_tx_descriptor_status_op, + .filter_ctrl = bnxt_filter_ctrl_op, + .dev_supported_ptypes_get = bnxt_dev_supported_ptypes_get_op, + .get_eeprom_length = bnxt_get_eeprom_length_op, + .get_eeprom = bnxt_get_eeprom_op, + .set_eeprom = bnxt_set_eeprom_op, }; static bool bnxt_vf_pciid(uint16_t id) { if (id == BROADCOM_DEV_ID_57304_VF || - id == BROADCOM_DEV_ID_57406_VF) + id == BROADCOM_DEV_ID_57406_VF || + id == BROADCOM_DEV_ID_5731X_VF || + id == BROADCOM_DEV_ID_5741X_VF || + id == BROADCOM_DEV_ID_57414_VF || + id == BROADCOM_DEV_ID_STRATUS_NIC_VF) return true; return false; } static int bnxt_init_board(struct rte_eth_dev *eth_dev) { - int rc; struct bnxt *bp = eth_dev->data->dev_private; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); + int rc; /* enable device (incl. PCI PM wakeup), and bus-mastering */ - if (!eth_dev->pci_dev->mem_resource[0].addr) { + if (!pci_dev->mem_resource[0].addr) { RTE_LOG(ERR, PMD, "Cannot find PCI device base address, aborting\n"); rc = -ENODEV; @@ -916,9 +2778,9 @@ static int bnxt_init_board(struct rte_eth_dev *eth_dev) } bp->eth_dev = eth_dev; - bp->pdev = eth_dev->pci_dev; + bp->pdev = pci_dev; - bp->bar0 = (void *)eth_dev->pci_dev->mem_resource[0].addr; + bp->bar0 = (void *)pci_dev->mem_resource[0].addr; if (!bp->bar0) { RTE_LOG(ERR, PMD, "Cannot map device registers, aborting\n"); rc = -ENOMEM; @@ -935,28 +2797,40 @@ init_err_disable: return rc; } +static int bnxt_dev_uninit(struct rte_eth_dev *eth_dev); + +#define ALLOW_FUNC(x) \ + { \ + typeof(x) arg = (x); \ + bp->pf.vf_req_fwd[((arg) >> 5)] &= \ + ~rte_cpu_to_le_32(1 << ((arg) & 0x1f)); \ + } static int bnxt_dev_init(struct rte_eth_dev *eth_dev) { + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); + char mz_name[RTE_MEMZONE_NAMESIZE]; + const struct rte_memzone *mz = NULL; static int version_printed; + uint32_t total_alloc_len; + rte_iova_t mz_phys_addr; struct bnxt *bp; int rc; if (version_printed++ == 0) - RTE_LOG(INFO, PMD, "%s", bnxt_version); + RTE_LOG(INFO, PMD, "%s\n", bnxt_version); - if (eth_dev->pci_dev->addr.function >= 2 && - eth_dev->pci_dev->addr.function < 4) { - RTE_LOG(ERR, PMD, "Function not enabled %x:\n", - eth_dev->pci_dev->addr.function); - rc = -ENOMEM; - goto error; - } + rte_eth_copy_pci_info(eth_dev, pci_dev); - rte_eth_copy_pci_info(eth_dev, eth_dev->pci_dev); bp = eth_dev->data->dev_private; - if (bnxt_vf_pciid(eth_dev->pci_dev->id.device_id)) + rte_atomic64_init(&bp->rx_mbuf_alloc_fail); + bp->dev_stopped = 1; + + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + goto skip_init; + + if (bnxt_vf_pciid(pci_dev->id.device_id)) bp->flags |= BNXT_FLAG_VF; rc = bnxt_init_board(eth_dev); @@ -965,10 +2839,87 @@ bnxt_dev_init(struct rte_eth_dev *eth_dev) "Board initialization failed rc: %x\n", rc); goto error; } +skip_init: eth_dev->dev_ops = &bnxt_dev_ops; + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return 0; eth_dev->rx_pkt_burst = &bnxt_recv_pkts; eth_dev->tx_pkt_burst = &bnxt_xmit_pkts; + if (BNXT_PF(bp) && pci_dev->id.device_id != BROADCOM_DEV_ID_NS2) { + snprintf(mz_name, RTE_MEMZONE_NAMESIZE, + "bnxt_%04x:%02x:%02x:%02x-%s", pci_dev->addr.domain, + pci_dev->addr.bus, pci_dev->addr.devid, + pci_dev->addr.function, "rx_port_stats"); + mz_name[RTE_MEMZONE_NAMESIZE - 1] = 0; + mz = rte_memzone_lookup(mz_name); + total_alloc_len = RTE_CACHE_LINE_ROUNDUP( + sizeof(struct rx_port_stats) + 512); + if (!mz) { + mz = rte_memzone_reserve(mz_name, total_alloc_len, + SOCKET_ID_ANY, + RTE_MEMZONE_2MB | + RTE_MEMZONE_SIZE_HINT_ONLY); + if (mz == NULL) + return -ENOMEM; + } + memset(mz->addr, 0, mz->len); + mz_phys_addr = mz->iova; + if ((unsigned long)mz->addr == mz_phys_addr) { + RTE_LOG(WARNING, PMD, + "Memzone physical address same as virtual.\n"); + RTE_LOG(WARNING, PMD, + "Using rte_mem_virt2iova()\n"); + mz_phys_addr = rte_mem_virt2iova(mz->addr); + if (mz_phys_addr == 0) { + RTE_LOG(ERR, PMD, + "unable to map address to physical memory\n"); + return -ENOMEM; + } + } + + bp->rx_mem_zone = (const void *)mz; + bp->hw_rx_port_stats = mz->addr; + bp->hw_rx_port_stats_map = mz_phys_addr; + + snprintf(mz_name, RTE_MEMZONE_NAMESIZE, + "bnxt_%04x:%02x:%02x:%02x-%s", pci_dev->addr.domain, + pci_dev->addr.bus, pci_dev->addr.devid, + pci_dev->addr.function, "tx_port_stats"); + mz_name[RTE_MEMZONE_NAMESIZE - 1] = 0; + mz = rte_memzone_lookup(mz_name); + total_alloc_len = RTE_CACHE_LINE_ROUNDUP( + sizeof(struct tx_port_stats) + 512); + if (!mz) { + mz = rte_memzone_reserve(mz_name, total_alloc_len, + SOCKET_ID_ANY, + RTE_MEMZONE_2MB | + RTE_MEMZONE_SIZE_HINT_ONLY); + if (mz == NULL) + return -ENOMEM; + } + memset(mz->addr, 0, mz->len); + mz_phys_addr = mz->iova; + if ((unsigned long)mz->addr == mz_phys_addr) { + RTE_LOG(WARNING, PMD, + "Memzone physical address same as virtual.\n"); + RTE_LOG(WARNING, PMD, + "Using rte_mem_virt2iova()\n"); + mz_phys_addr = rte_mem_virt2iova(mz->addr); + if (mz_phys_addr == 0) { + RTE_LOG(ERR, PMD, + "unable to map address to physical memory\n"); + return -ENOMEM; + } + } + + bp->tx_mem_zone = (const void *)mz; + bp->hw_tx_port_stats = mz->addr; + bp->hw_tx_port_stats_map = mz_phys_addr; + + bp->flags |= BNXT_FLAG_PORT_STATS; + } + rc = bnxt_alloc_hwrm_resources(bp); if (rc) { RTE_LOG(ERR, PMD, @@ -980,39 +2931,74 @@ bnxt_dev_init(struct rte_eth_dev *eth_dev) goto error_free; bnxt_hwrm_queue_qportcfg(bp); + bnxt_hwrm_func_qcfg(bp); + /* Get the MAX capabilities for this function */ rc = bnxt_hwrm_func_qcaps(bp); if (rc) { RTE_LOG(ERR, PMD, "hwrm query capability failure rc: %x\n", rc); goto error_free; } + if (bp->max_tx_rings == 0) { + RTE_LOG(ERR, PMD, "No TX rings available!\n"); + rc = -EBUSY; + goto error_free; + } eth_dev->data->mac_addrs = rte_zmalloc("bnxt_mac_addr_tbl", - ETHER_ADDR_LEN * MAX_NUM_MAC_ADDR, 0); + ETHER_ADDR_LEN * bp->max_l2_ctx, 0); if (eth_dev->data->mac_addrs == NULL) { RTE_LOG(ERR, PMD, "Failed to alloc %u bytes needed to store MAC addr tbl", - ETHER_ADDR_LEN * MAX_NUM_MAC_ADDR); + ETHER_ADDR_LEN * bp->max_l2_ctx); rc = -ENOMEM; goto error_free; } /* Copy the permanent MAC from the qcap response address now. */ - if (BNXT_PF(bp)) - memcpy(bp->mac_addr, bp->pf.mac_addr, sizeof(bp->mac_addr)); - else - memcpy(bp->mac_addr, bp->vf.mac_addr, sizeof(bp->mac_addr)); + memcpy(bp->mac_addr, bp->dflt_mac_addr, sizeof(bp->mac_addr)); memcpy(ð_dev->data->mac_addrs[0], bp->mac_addr, ETHER_ADDR_LEN); + + if (bp->max_ring_grps < bp->rx_cp_nr_rings) { + /* 1 ring is for default completion ring */ + RTE_LOG(ERR, PMD, "Insufficient resource: Ring Group\n"); + rc = -ENOSPC; + goto error_free; + } + bp->grp_info = rte_zmalloc("bnxt_grp_info", sizeof(*bp->grp_info) * bp->max_ring_grps, 0); if (!bp->grp_info) { RTE_LOG(ERR, PMD, - "Failed to alloc %zu bytes needed to store group info table\n", + "Failed to alloc %zu bytes to store group info table\n", sizeof(*bp->grp_info) * bp->max_ring_grps); rc = -ENOMEM; goto error_free; } - rc = bnxt_hwrm_func_driver_register(bp, 0, - bp->pf.vf_req_fwd); + /* Forward all requests if firmware is new enough */ + if (((bp->fw_ver >= ((20 << 24) | (6 << 16) | (100 << 8))) && + (bp->fw_ver < ((20 << 24) | (7 << 16)))) || + ((bp->fw_ver >= ((20 << 24) | (8 << 16))))) { + memset(bp->pf.vf_req_fwd, 0xff, sizeof(bp->pf.vf_req_fwd)); + } else { + RTE_LOG(WARNING, PMD, + "Firmware too old for VF mailbox functionality\n"); + memset(bp->pf.vf_req_fwd, 0, sizeof(bp->pf.vf_req_fwd)); + } + + /* + * The following are used for driver cleanup. If we disallow these, + * VF drivers can't clean up cleanly. + */ + ALLOW_FUNC(HWRM_FUNC_DRV_UNRGTR); + ALLOW_FUNC(HWRM_VNIC_FREE); + ALLOW_FUNC(HWRM_RING_FREE); + ALLOW_FUNC(HWRM_RING_GRP_FREE); + ALLOW_FUNC(HWRM_VNIC_RSS_COS_LB_CTX_FREE); + ALLOW_FUNC(HWRM_CFA_L2_FILTER_FREE); + ALLOW_FUNC(HWRM_STAT_CTX_FREE); + ALLOW_FUNC(HWRM_PORT_PHY_QCFG); + ALLOW_FUNC(HWRM_VNIC_TPA_CFG); + rc = bnxt_hwrm_func_driver_register(bp); if (rc) { RTE_LOG(ERR, PMD, "Failed to register driver"); @@ -1022,15 +3008,66 @@ bnxt_dev_init(struct rte_eth_dev *eth_dev) RTE_LOG(INFO, PMD, DRV_MODULE_NAME " found at mem %" PRIx64 ", node addr %pM\n", - eth_dev->pci_dev->mem_resource[0].phys_addr, - eth_dev->pci_dev->mem_resource[0].addr); + pci_dev->mem_resource[0].phys_addr, + pci_dev->mem_resource[0].addr); - bp->dev_stopped = 0; + rc = bnxt_hwrm_func_reset(bp); + if (rc) { + RTE_LOG(ERR, PMD, "hwrm chip reset failure rc: %x\n", rc); + rc = -1; + goto error_free; + } + + if (BNXT_PF(bp)) { + //if (bp->pf.active_vfs) { + // TODO: Deallocate VF resources? + //} + if (bp->pdev->max_vfs) { + rc = bnxt_hwrm_allocate_vfs(bp, bp->pdev->max_vfs); + if (rc) { + RTE_LOG(ERR, PMD, "Failed to allocate VFs\n"); + goto error_free; + } + } else { + rc = bnxt_hwrm_allocate_pf_only(bp); + if (rc) { + RTE_LOG(ERR, PMD, + "Failed to allocate PF resources\n"); + goto error_free; + } + } + } + + bnxt_hwrm_port_led_qcaps(bp); + + rc = bnxt_setup_int(bp); + if (rc) + goto error_free; + + rc = bnxt_alloc_mem(bp); + if (rc) + goto error_free_int; + + rc = bnxt_request_int(bp); + if (rc) + goto error_free_int; + + rc = bnxt_alloc_def_cp_ring(bp); + if (rc) + goto error_free_int; + + bnxt_enable_int(bp); return 0; +error_free_int: + bnxt_disable_int(bp); + bnxt_free_def_cp_ring(bp); + bnxt_hwrm_func_buf_unrgtr(bp); + bnxt_free_int(bp); + bnxt_free_mem(bp); error_free: - eth_dev->driver->eth_dev_uninit(eth_dev); + bnxt_dev_uninit(eth_dev); error: return rc; } @@ -1040,39 +3077,69 @@ bnxt_dev_uninit(struct rte_eth_dev *eth_dev) { struct bnxt *bp = eth_dev->data->dev_private; int rc; - if (eth_dev->data->mac_addrs) + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return -EPERM; + + bnxt_disable_int(bp); + bnxt_free_int(bp); + bnxt_free_mem(bp); + if (eth_dev->data->mac_addrs != NULL) { rte_free(eth_dev->data->mac_addrs); - if (bp->grp_info) + eth_dev->data->mac_addrs = NULL; + } + if (bp->grp_info != NULL) { rte_free(bp->grp_info); + bp->grp_info = NULL; + } rc = bnxt_hwrm_func_driver_unregister(bp, 0); bnxt_free_hwrm_resources(bp); + rte_memzone_free((const struct rte_memzone *)bp->tx_mem_zone); + rte_memzone_free((const struct rte_memzone *)bp->rx_mem_zone); if (bp->dev_stopped == 0) bnxt_dev_close_op(eth_dev); + if (bp->pf.vf_info) + rte_free(bp->pf.vf_info); + eth_dev->dev_ops = NULL; + eth_dev->rx_pkt_burst = NULL; + eth_dev->tx_pkt_burst = NULL; + return rc; } -static struct eth_driver bnxt_rte_pmd = { - .pci_drv = { - .name = "rte_" DRV_MODULE_NAME "_pmd", - .id_table = bnxt_pci_id_map, - .drv_flags = RTE_PCI_DRV_NEED_MAPPING, - }, - .eth_dev_init = bnxt_dev_init, - .eth_dev_uninit = bnxt_dev_uninit, - .dev_private_size = sizeof(struct bnxt), -}; - -static int bnxt_rte_pmd_init(const char *name, const char *params __rte_unused) +static int bnxt_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, + struct rte_pci_device *pci_dev) { - RTE_LOG(INFO, PMD, "bnxt_rte_pmd_init() called for %s\n", name); - rte_eth_driver_register(&bnxt_rte_pmd); - return 0; + return rte_eth_dev_pci_generic_probe(pci_dev, sizeof(struct bnxt), + bnxt_dev_init); } -static struct rte_driver bnxt_pmd_drv = { - .type = PMD_PDEV, - .init = bnxt_rte_pmd_init, +static int bnxt_pci_remove(struct rte_pci_device *pci_dev) +{ + return rte_eth_dev_pci_generic_remove(pci_dev, bnxt_dev_uninit); +} + +static struct rte_pci_driver bnxt_rte_pmd = { + .id_table = bnxt_pci_id_map, + .drv_flags = RTE_PCI_DRV_NEED_MAPPING | + RTE_PCI_DRV_INTR_LSC, + .probe = bnxt_pci_probe, + .remove = bnxt_pci_remove, }; -PMD_REGISTER_DRIVER(bnxt_pmd_drv, bnxt); -DRIVER_REGISTER_PCI_TABLE(bnxt, bnxt_pci_id_map); +static bool +is_device_supported(struct rte_eth_dev *dev, struct rte_pci_driver *drv) +{ + if (strcmp(dev->device->driver->name, drv->driver.name)) + return false; + + return true; +} + +bool is_bnxt_supported(struct rte_eth_dev *dev) +{ + return is_device_supported(dev, &bnxt_rte_pmd); +} + +RTE_PMD_REGISTER_PCI(net_bnxt, bnxt_rte_pmd); +RTE_PMD_REGISTER_PCI_TABLE(net_bnxt, bnxt_pci_id_map); +RTE_PMD_REGISTER_KMOD_DEP(net_bnxt, "* igb_uio | uio_pci_generic | vfio-pci"); diff --git a/dpdk/drivers/net/bnxt/bnxt_filter.c b/dpdk/drivers/net/bnxt/bnxt_filter.c index f03a1dc8..32af6061 100644 --- a/dpdk/drivers/net/bnxt/bnxt_filter.c +++ b/dpdk/drivers/net/bnxt/bnxt_filter.c @@ -35,6 +35,9 @@ #include #include +#include +#include +#include #include "bnxt.h" #include "bnxt_filter.h" @@ -68,24 +71,34 @@ struct bnxt_filter_info *bnxt_alloc_filter(struct bnxt *bp) return filter; } +struct bnxt_filter_info *bnxt_alloc_vf_filter(struct bnxt *bp, uint16_t vf) +{ + struct bnxt_filter_info *filter; + + filter = rte_zmalloc("bnxt_vf_filter_info", sizeof(*filter), 0); + if (!filter) { + RTE_LOG(ERR, PMD, "Failed to alloc memory for VF %hu filters\n", + vf); + return NULL; + } + + filter->fw_l2_filter_id = UINT64_MAX; + STAILQ_INSERT_TAIL(&bp->pf.vf_info[vf].filter, filter, next); + return filter; +} + void bnxt_init_filters(struct bnxt *bp) { struct bnxt_filter_info *filter; int i, max_filters; - if (BNXT_PF(bp)) { - struct bnxt_pf_info *pf = &bp->pf; - - max_filters = pf->max_l2_ctx; - } else { - struct bnxt_vf_info *vf = &bp->vf; - - max_filters = vf->max_l2_ctx; - } + max_filters = bp->max_l2_ctx; STAILQ_INIT(&bp->free_filter_list); for (i = 0; i < max_filters; i++) { filter = &bp->filter_info[i]; filter->fw_l2_filter_id = -1; + filter->fw_em_filter_id = -1; + filter->fw_ntuple_filter_id = -1; STAILQ_INSERT_TAIL(&bp->free_filter_list, filter, next); } } @@ -110,6 +123,12 @@ void bnxt_free_all_filters(struct bnxt *bp) STAILQ_INIT(&vnic->filter); } } + + for (i = 0; i < bp->pf.max_vfs; i++) { + STAILQ_FOREACH(filter, &bp->pf.vf_info[i].filter, next) { + bnxt_hwrm_clear_l2_filter(bp, filter); + } + } } void bnxt_free_filter_mem(struct bnxt *bp) @@ -118,28 +137,23 @@ void bnxt_free_filter_mem(struct bnxt *bp) uint16_t max_filters, i; int rc = 0; + if (bp->filter_info == NULL) + return; + /* Ensure that all filters are freed */ - if (BNXT_PF(bp)) { - struct bnxt_pf_info *pf = &bp->pf; - - max_filters = pf->max_l2_ctx; - } else { - struct bnxt_vf_info *vf = &bp->vf; - - max_filters = vf->max_l2_ctx; - } + max_filters = bp->max_l2_ctx; for (i = 0; i < max_filters; i++) { filter = &bp->filter_info[i]; if (filter->fw_l2_filter_id != ((uint64_t)-1)) { RTE_LOG(ERR, PMD, "HWRM filter is not freed??\n"); /* Call HWRM to try to free filter again */ - rc = bnxt_hwrm_clear_filter(bp, filter); + rc = bnxt_hwrm_clear_l2_filter(bp, filter); if (rc) RTE_LOG(ERR, PMD, "HWRM filter cannot be freed rc = %d\n", rc); } - filter->fw_l2_filter_id = -1; + filter->fw_l2_filter_id = UINT64_MAX; } STAILQ_INIT(&bp->free_filter_list); @@ -152,15 +166,7 @@ int bnxt_alloc_filter_mem(struct bnxt *bp) struct bnxt_filter_info *filter_mem; uint16_t max_filters; - if (BNXT_PF(bp)) { - struct bnxt_pf_info *pf = &bp->pf; - - max_filters = pf->max_l2_ctx; - } else { - struct bnxt_vf_info *vf = &bp->vf; - - max_filters = vf->max_l2_ctx; - } + max_filters = bp->max_l2_ctx; /* Allocate memory for VNIC pool and filter pool */ filter_mem = rte_zmalloc("bnxt_filter_info", max_filters * sizeof(struct bnxt_filter_info), @@ -173,3 +179,1063 @@ int bnxt_alloc_filter_mem(struct bnxt *bp) bp->filter_info = filter_mem; return 0; } + +struct bnxt_filter_info *bnxt_get_unused_filter(struct bnxt *bp) +{ + struct bnxt_filter_info *filter; + + /* Find the 1st unused filter from the free_filter_list pool*/ + filter = STAILQ_FIRST(&bp->free_filter_list); + if (!filter) { + RTE_LOG(ERR, PMD, "No more free filter resources\n"); + return NULL; + } + STAILQ_REMOVE_HEAD(&bp->free_filter_list, next); + + return filter; +} + +void bnxt_free_filter(struct bnxt *bp, struct bnxt_filter_info *filter) +{ + STAILQ_INSERT_TAIL(&bp->free_filter_list, filter, next); +} + +static int +bnxt_flow_agrs_validate(const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error) +{ + if (!pattern) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM_NUM, + NULL, "NULL pattern."); + return -rte_errno; + } + + if (!actions) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION_NUM, + NULL, "NULL action."); + return -rte_errno; + } + + if (!attr) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR, + NULL, "NULL attribute."); + return -rte_errno; + } + + return 0; +} + +static const struct rte_flow_item * +nxt_non_void_pattern(const struct rte_flow_item *cur) +{ + while (1) { + if (cur->type != RTE_FLOW_ITEM_TYPE_VOID) + return cur; + cur++; + } +} + +static const struct rte_flow_action * +nxt_non_void_action(const struct rte_flow_action *cur) +{ + while (1) { + if (cur->type != RTE_FLOW_ACTION_TYPE_VOID) + return cur; + cur++; + } +} + +static inline int check_zero_bytes(const uint8_t *bytes, int len) +{ + int i; + for (i = 0; i < len; i++) + if (bytes[i] != 0x00) + return 0; + return 1; +} + +static int +bnxt_filter_type_check(const struct rte_flow_item pattern[], + struct rte_flow_error *error __rte_unused) +{ + const struct rte_flow_item *item = nxt_non_void_pattern(pattern); + int use_ntuple = 1; + + while (item->type != RTE_FLOW_ITEM_TYPE_END) { + switch (item->type) { + case RTE_FLOW_ITEM_TYPE_ETH: + use_ntuple = 1; + break; + case RTE_FLOW_ITEM_TYPE_VLAN: + use_ntuple = 0; + break; + case RTE_FLOW_ITEM_TYPE_IPV4: + case RTE_FLOW_ITEM_TYPE_IPV6: + case RTE_FLOW_ITEM_TYPE_TCP: + case RTE_FLOW_ITEM_TYPE_UDP: + /* FALLTHROUGH */ + /* need ntuple match, reset exact match */ + if (!use_ntuple) { + RTE_LOG(ERR, PMD, + "VLAN flow cannot use NTUPLE filter\n"); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Cannot use VLAN with NTUPLE"); + return -rte_errno; + } + use_ntuple |= 1; + break; + default: + RTE_LOG(ERR, PMD, "Unknown Flow type"); + use_ntuple |= 1; + } + item++; + } + return use_ntuple; +} + +static int +bnxt_validate_and_parse_flow_type(struct bnxt *bp, + const struct rte_flow_item pattern[], + struct rte_flow_error *error, + struct bnxt_filter_info *filter) +{ + const struct rte_flow_item *item = nxt_non_void_pattern(pattern); + const struct rte_flow_item_vlan *vlan_spec, *vlan_mask; + const struct rte_flow_item_ipv4 *ipv4_spec, *ipv4_mask; + const struct rte_flow_item_ipv6 *ipv6_spec, *ipv6_mask; + const struct rte_flow_item_tcp *tcp_spec, *tcp_mask; + const struct rte_flow_item_udp *udp_spec, *udp_mask; + const struct rte_flow_item_eth *eth_spec, *eth_mask; + const struct rte_flow_item_nvgre *nvgre_spec; + const struct rte_flow_item_nvgre *nvgre_mask; + const struct rte_flow_item_vxlan *vxlan_spec; + const struct rte_flow_item_vxlan *vxlan_mask; + uint8_t vni_mask[] = {0xFF, 0xFF, 0xFF}; + uint8_t tni_mask[] = {0xFF, 0xFF, 0xFF}; + const struct rte_flow_item_vf *vf_spec; + uint32_t tenant_id_be = 0; + bool vni_masked = 0; + bool tni_masked = 0; + uint32_t vf = 0; + int use_ntuple; + uint32_t en = 0; + int dflt_vnic; + + use_ntuple = bnxt_filter_type_check(pattern, error); + RTE_LOG(DEBUG, PMD, "Use NTUPLE %d\n", use_ntuple); + if (use_ntuple < 0) + return use_ntuple; + + filter->filter_type = use_ntuple ? + HWRM_CFA_NTUPLE_FILTER : HWRM_CFA_EM_FILTER; + + while (item->type != RTE_FLOW_ITEM_TYPE_END) { + if (item->last) { + /* last or range is NOT supported as match criteria */ + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "No support for range"); + return -rte_errno; + } + if (!item->spec || !item->mask) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "spec/mask is NULL"); + return -rte_errno; + } + switch (item->type) { + case RTE_FLOW_ITEM_TYPE_ETH: + eth_spec = (const struct rte_flow_item_eth *)item->spec; + eth_mask = (const struct rte_flow_item_eth *)item->mask; + + /* Source MAC address mask cannot be partially set. + * Should be All 0's or all 1's. + * Destination MAC address mask must not be partially + * set. Should be all 1's or all 0's. + */ + if ((!is_zero_ether_addr(ð_mask->src) && + !is_broadcast_ether_addr(ð_mask->src)) || + (!is_zero_ether_addr(ð_mask->dst) && + !is_broadcast_ether_addr(ð_mask->dst))) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "MAC_addr mask not valid"); + return -rte_errno; + } + + /* Mask is not allowed. Only exact matches are */ + if ((eth_mask->type & UINT16_MAX) != UINT16_MAX) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "ethertype mask not valid"); + return -rte_errno; + } + + if (is_broadcast_ether_addr(ð_mask->dst)) { + rte_memcpy(filter->dst_macaddr, + ð_spec->dst, 6); + en |= use_ntuple ? + NTUPLE_FLTR_ALLOC_INPUT_EN_DST_MACADDR : + EM_FLOW_ALLOC_INPUT_EN_DST_MACADDR; + } + if (is_broadcast_ether_addr(ð_mask->src)) { + rte_memcpy(filter->src_macaddr, + ð_spec->src, 6); + en |= use_ntuple ? + NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_MACADDR : + EM_FLOW_ALLOC_INPUT_EN_SRC_MACADDR; + } /* + * else { + * RTE_LOG(ERR, PMD, "Handle this condition\n"); + * } + */ + if (eth_spec->type) { + filter->ethertype = + rte_be_to_cpu_16(eth_spec->type); + en |= use_ntuple ? + NTUPLE_FLTR_ALLOC_INPUT_EN_ETHERTYPE : + EM_FLOW_ALLOC_INPUT_EN_ETHERTYPE; + } + + break; + case RTE_FLOW_ITEM_TYPE_VLAN: + vlan_spec = + (const struct rte_flow_item_vlan *)item->spec; + vlan_mask = + (const struct rte_flow_item_vlan *)item->mask; + if (vlan_mask->tci & 0xFFFF && !vlan_mask->tpid) { + /* Only the VLAN ID can be matched. */ + filter->l2_ovlan = + rte_be_to_cpu_16(vlan_spec->tci & + 0xFFF); + en |= EM_FLOW_ALLOC_INPUT_EN_OVLAN_VID; + } else { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "VLAN mask is invalid"); + return -rte_errno; + } + + break; + case RTE_FLOW_ITEM_TYPE_IPV4: + /* If mask is not involved, we could use EM filters. */ + ipv4_spec = + (const struct rte_flow_item_ipv4 *)item->spec; + ipv4_mask = + (const struct rte_flow_item_ipv4 *)item->mask; + /* Only IP DST and SRC fields are maskable. */ + if (ipv4_mask->hdr.version_ihl || + ipv4_mask->hdr.type_of_service || + ipv4_mask->hdr.total_length || + ipv4_mask->hdr.packet_id || + ipv4_mask->hdr.fragment_offset || + ipv4_mask->hdr.time_to_live || + ipv4_mask->hdr.next_proto_id || + ipv4_mask->hdr.hdr_checksum) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid IPv4 mask."); + return -rte_errno; + } + filter->dst_ipaddr[0] = ipv4_spec->hdr.dst_addr; + filter->src_ipaddr[0] = ipv4_spec->hdr.src_addr; + if (use_ntuple) + en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR | + NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR; + else + en |= EM_FLOW_ALLOC_INPUT_EN_SRC_IPADDR | + EM_FLOW_ALLOC_INPUT_EN_DST_IPADDR; + if (ipv4_mask->hdr.src_addr) { + filter->src_ipaddr_mask[0] = + ipv4_mask->hdr.src_addr; + en |= !use_ntuple ? 0 : + NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR_MASK; + } + if (ipv4_mask->hdr.dst_addr) { + filter->dst_ipaddr_mask[0] = + ipv4_mask->hdr.dst_addr; + en |= !use_ntuple ? 0 : + NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR_MASK; + } + filter->ip_addr_type = use_ntuple ? + HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_IP_ADDR_TYPE_IPV4 : + HWRM_CFA_EM_FLOW_ALLOC_INPUT_IP_ADDR_TYPE_IPV4; + if (ipv4_spec->hdr.next_proto_id) { + filter->ip_protocol = + ipv4_spec->hdr.next_proto_id; + if (use_ntuple) + en |= NTUPLE_FLTR_ALLOC_IN_EN_IP_PROTO; + else + en |= EM_FLOW_ALLOC_INPUT_EN_IP_PROTO; + } + break; + case RTE_FLOW_ITEM_TYPE_IPV6: + ipv6_spec = + (const struct rte_flow_item_ipv6 *)item->spec; + ipv6_mask = + (const struct rte_flow_item_ipv6 *)item->mask; + + /* Only IP DST and SRC fields are maskable. */ + if (ipv6_mask->hdr.vtc_flow || + ipv6_mask->hdr.payload_len || + ipv6_mask->hdr.proto || + ipv6_mask->hdr.hop_limits) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid IPv6 mask."); + return -rte_errno; + } + + if (use_ntuple) + en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR | + NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR; + else + en |= EM_FLOW_ALLOC_INPUT_EN_SRC_IPADDR | + EM_FLOW_ALLOC_INPUT_EN_DST_IPADDR; + rte_memcpy(filter->src_ipaddr, + ipv6_spec->hdr.src_addr, 16); + rte_memcpy(filter->dst_ipaddr, + ipv6_spec->hdr.dst_addr, 16); + if (!check_zero_bytes(ipv6_mask->hdr.src_addr, 16)) { + rte_memcpy(filter->src_ipaddr_mask, + ipv6_mask->hdr.src_addr, 16); + en |= !use_ntuple ? 0 : + NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR_MASK; + } + if (!check_zero_bytes(ipv6_mask->hdr.dst_addr, 16)) { + rte_memcpy(filter->dst_ipaddr_mask, + ipv6_mask->hdr.dst_addr, 16); + en |= !use_ntuple ? 0 : + NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR_MASK; + } + filter->ip_addr_type = use_ntuple ? + NTUPLE_FLTR_ALLOC_INPUT_IP_ADDR_TYPE_IPV6 : + EM_FLOW_ALLOC_INPUT_IP_ADDR_TYPE_IPV6; + break; + case RTE_FLOW_ITEM_TYPE_TCP: + tcp_spec = (const struct rte_flow_item_tcp *)item->spec; + tcp_mask = (const struct rte_flow_item_tcp *)item->mask; + + /* Check TCP mask. Only DST & SRC ports are maskable */ + if (tcp_mask->hdr.sent_seq || + tcp_mask->hdr.recv_ack || + tcp_mask->hdr.data_off || + tcp_mask->hdr.tcp_flags || + tcp_mask->hdr.rx_win || + tcp_mask->hdr.cksum || + tcp_mask->hdr.tcp_urp) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid TCP mask"); + return -rte_errno; + } + filter->src_port = tcp_spec->hdr.src_port; + filter->dst_port = tcp_spec->hdr.dst_port; + if (use_ntuple) + en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_PORT | + NTUPLE_FLTR_ALLOC_INPUT_EN_DST_PORT; + else + en |= EM_FLOW_ALLOC_INPUT_EN_SRC_PORT | + EM_FLOW_ALLOC_INPUT_EN_DST_PORT; + if (tcp_mask->hdr.dst_port) { + filter->dst_port_mask = tcp_mask->hdr.dst_port; + en |= !use_ntuple ? 0 : + NTUPLE_FLTR_ALLOC_INPUT_EN_DST_PORT_MASK; + } + if (tcp_mask->hdr.src_port) { + filter->src_port_mask = tcp_mask->hdr.src_port; + en |= !use_ntuple ? 0 : + NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_PORT_MASK; + } + break; + case RTE_FLOW_ITEM_TYPE_UDP: + udp_spec = (const struct rte_flow_item_udp *)item->spec; + udp_mask = (const struct rte_flow_item_udp *)item->mask; + + if (udp_mask->hdr.dgram_len || + udp_mask->hdr.dgram_cksum) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid UDP mask"); + return -rte_errno; + } + + filter->src_port = udp_spec->hdr.src_port; + filter->dst_port = udp_spec->hdr.dst_port; + if (use_ntuple) + en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_PORT | + NTUPLE_FLTR_ALLOC_INPUT_EN_DST_PORT; + else + en |= EM_FLOW_ALLOC_INPUT_EN_SRC_PORT | + EM_FLOW_ALLOC_INPUT_EN_DST_PORT; + + if (udp_mask->hdr.dst_port) { + filter->dst_port_mask = udp_mask->hdr.dst_port; + en |= !use_ntuple ? 0 : + NTUPLE_FLTR_ALLOC_INPUT_EN_DST_PORT_MASK; + } + if (udp_mask->hdr.src_port) { + filter->src_port_mask = udp_mask->hdr.src_port; + en |= !use_ntuple ? 0 : + NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_PORT_MASK; + } + break; + case RTE_FLOW_ITEM_TYPE_VXLAN: + vxlan_spec = + (const struct rte_flow_item_vxlan *)item->spec; + vxlan_mask = + (const struct rte_flow_item_vxlan *)item->mask; + /* Check if VXLAN item is used to describe protocol. + * If yes, both spec and mask should be NULL. + * If no, both spec and mask shouldn't be NULL. + */ + if ((!vxlan_spec && vxlan_mask) || + (vxlan_spec && !vxlan_mask)) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid VXLAN item"); + return -rte_errno; + } + + if (vxlan_spec->rsvd1 || vxlan_spec->rsvd0[0] || + vxlan_spec->rsvd0[1] || vxlan_spec->rsvd0[2] || + vxlan_spec->flags != 0x8) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid VXLAN item"); + return -rte_errno; + } + + /* Check if VNI is masked. */ + if (vxlan_spec && vxlan_mask) { + vni_masked = + !!memcmp(vxlan_mask->vni, vni_mask, + RTE_DIM(vni_mask)); + if (vni_masked) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid VNI mask"); + return -rte_errno; + } + + rte_memcpy(((uint8_t *)&tenant_id_be + 1), + vxlan_spec->vni, 3); + filter->vni = + rte_be_to_cpu_32(tenant_id_be); + filter->tunnel_type = + CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN; + } + break; + case RTE_FLOW_ITEM_TYPE_NVGRE: + nvgre_spec = + (const struct rte_flow_item_nvgre *)item->spec; + nvgre_mask = + (const struct rte_flow_item_nvgre *)item->mask; + /* Check if NVGRE item is used to describe protocol. + * If yes, both spec and mask should be NULL. + * If no, both spec and mask shouldn't be NULL. + */ + if ((!nvgre_spec && nvgre_mask) || + (nvgre_spec && !nvgre_mask)) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid NVGRE item"); + return -rte_errno; + } + + if (nvgre_spec->c_k_s_rsvd0_ver != 0x2000 || + nvgre_spec->protocol != 0x6558) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid NVGRE item"); + return -rte_errno; + } + + if (nvgre_spec && nvgre_mask) { + tni_masked = + !!memcmp(nvgre_mask->tni, tni_mask, + RTE_DIM(tni_mask)); + if (tni_masked) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid TNI mask"); + return -rte_errno; + } + rte_memcpy(((uint8_t *)&tenant_id_be + 1), + nvgre_spec->tni, 3); + filter->vni = + rte_be_to_cpu_32(tenant_id_be); + filter->tunnel_type = + CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_NVGRE; + } + break; + case RTE_FLOW_ITEM_TYPE_VF: + vf_spec = (const struct rte_flow_item_vf *)item->spec; + vf = vf_spec->id; + if (!BNXT_PF(bp)) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Configuring on a VF!"); + return -rte_errno; + } + + if (vf >= bp->pdev->max_vfs) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Incorrect VF id!"); + return -rte_errno; + } + + filter->mirror_vnic_id = + dflt_vnic = bnxt_hwrm_func_qcfg_vf_dflt_vnic_id(bp, vf); + if (dflt_vnic < 0) { + /* This simply indicates there's no driver + * loaded. This is not an error. + */ + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Unable to get default VNIC for VF"); + return -rte_errno; + } + filter->mirror_vnic_id = dflt_vnic; + en |= NTUPLE_FLTR_ALLOC_INPUT_EN_MIRROR_VNIC_ID; + break; + default: + break; + } + item++; + } + filter->enables = en; + + return 0; +} + +/* Parse attributes */ +static int +bnxt_flow_parse_attr(const struct rte_flow_attr *attr, + struct rte_flow_error *error) +{ + /* Must be input direction */ + if (!attr->ingress) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, + attr, "Only support ingress."); + return -rte_errno; + } + + /* Not supported */ + if (attr->egress) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, + attr, "No support for egress."); + return -rte_errno; + } + + /* Not supported */ + if (attr->priority) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, + attr, "No support for priority."); + return -rte_errno; + } + + /* Not supported */ + if (attr->group) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_GROUP, + attr, "No support for group."); + return -rte_errno; + } + + return 0; +} + +struct bnxt_filter_info * +bnxt_get_l2_filter(struct bnxt *bp, struct bnxt_filter_info *nf, + struct bnxt_vnic_info *vnic) +{ + struct bnxt_filter_info *filter1, *f0; + struct bnxt_vnic_info *vnic0; + int rc; + + vnic0 = STAILQ_FIRST(&bp->ff_pool[0]); + f0 = STAILQ_FIRST(&vnic0->filter); + + //This flow has same DST MAC as the port/l2 filter. + if (memcmp(f0->l2_addr, nf->dst_macaddr, ETHER_ADDR_LEN) == 0) + return f0; + + //This flow needs DST MAC which is not same as port/l2 + RTE_LOG(DEBUG, PMD, "Create L2 filter for DST MAC\n"); + filter1 = bnxt_get_unused_filter(bp); + if (filter1 == NULL) + return NULL; + filter1->flags = HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_PATH_RX; + filter1->enables = HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_ADDR | + L2_FILTER_ALLOC_INPUT_EN_L2_ADDR_MASK; + memcpy(filter1->l2_addr, nf->dst_macaddr, ETHER_ADDR_LEN); + memset(filter1->l2_addr_mask, 0xff, ETHER_ADDR_LEN); + rc = bnxt_hwrm_set_l2_filter(bp, vnic->fw_vnic_id, + filter1); + if (rc) { + bnxt_free_filter(bp, filter1); + return NULL; + } + STAILQ_INSERT_TAIL(&vnic->filter, filter1, next); + return filter1; +} + +static int +bnxt_validate_and_parse_flow(struct rte_eth_dev *dev, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + const struct rte_flow_attr *attr, + struct rte_flow_error *error, + struct bnxt_filter_info *filter) +{ + const struct rte_flow_action *act = nxt_non_void_action(actions); + struct bnxt *bp = (struct bnxt *)dev->data->dev_private; + const struct rte_flow_action_queue *act_q; + const struct rte_flow_action_vf *act_vf; + struct bnxt_vnic_info *vnic, *vnic0; + struct bnxt_filter_info *filter1; + uint32_t vf = 0; + int dflt_vnic; + int rc; + + if (bp->eth_dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS) { + RTE_LOG(ERR, PMD, "Cannot create flow on RSS queues\n"); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "Cannot create flow on RSS queues"); + rc = -rte_errno; + goto ret; + } + + rc = bnxt_validate_and_parse_flow_type(bp, pattern, error, filter); + if (rc != 0) + goto ret; + + rc = bnxt_flow_parse_attr(attr, error); + if (rc != 0) + goto ret; + //Since we support ingress attribute only - right now. + filter->flags = HWRM_CFA_EM_FLOW_ALLOC_INPUT_FLAGS_PATH_RX; + + switch (act->type) { + case RTE_FLOW_ACTION_TYPE_QUEUE: + /* Allow this flow. Redirect to a VNIC. */ + act_q = (const struct rte_flow_action_queue *)act->conf; + if (act_q->index >= bp->rx_nr_rings) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, act, + "Invalid queue ID."); + rc = -rte_errno; + goto ret; + } + RTE_LOG(DEBUG, PMD, "Queue index %d\n", act_q->index); + + vnic0 = STAILQ_FIRST(&bp->ff_pool[0]); + vnic = STAILQ_FIRST(&bp->ff_pool[act_q->index]); + if (vnic == NULL) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, act, + "No matching VNIC for queue ID."); + rc = -rte_errno; + goto ret; + } + filter->dst_id = vnic->fw_vnic_id; + filter1 = bnxt_get_l2_filter(bp, filter, vnic); + if (filter1 == NULL) { + rc = -ENOSPC; + goto ret; + } + filter->fw_l2_filter_id = filter1->fw_l2_filter_id; + RTE_LOG(DEBUG, PMD, "VNIC found\n"); + break; + case RTE_FLOW_ACTION_TYPE_DROP: + vnic0 = STAILQ_FIRST(&bp->ff_pool[0]); + filter1 = bnxt_get_l2_filter(bp, filter, vnic0); + if (filter1 == NULL) { + rc = -ENOSPC; + goto ret; + } + filter->fw_l2_filter_id = filter1->fw_l2_filter_id; + if (filter->filter_type == HWRM_CFA_EM_FILTER) + filter->flags = + HWRM_CFA_EM_FLOW_ALLOC_INPUT_FLAGS_DROP; + else + filter->flags = + HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_FLAGS_DROP; + break; + case RTE_FLOW_ACTION_TYPE_COUNT: + vnic0 = STAILQ_FIRST(&bp->ff_pool[0]); + filter1 = bnxt_get_l2_filter(bp, filter, vnic0); + if (filter1 == NULL) { + rc = -ENOSPC; + goto ret; + } + filter->fw_l2_filter_id = filter1->fw_l2_filter_id; + filter->flags = HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_FLAGS_METER; + break; + case RTE_FLOW_ACTION_TYPE_VF: + act_vf = (const struct rte_flow_action_vf *)act->conf; + vf = act_vf->id; + if (!BNXT_PF(bp)) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + act, + "Configuring on a VF!"); + rc = -rte_errno; + goto ret; + } + + if (vf >= bp->pdev->max_vfs) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + act, + "Incorrect VF id!"); + rc = -rte_errno; + goto ret; + } + + filter->mirror_vnic_id = + dflt_vnic = bnxt_hwrm_func_qcfg_vf_dflt_vnic_id(bp, vf); + if (dflt_vnic < 0) { + /* This simply indicates there's no driver loaded. + * This is not an error. + */ + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + act, + "Unable to get default VNIC for VF"); + rc = -rte_errno; + goto ret; + } + filter->mirror_vnic_id = dflt_vnic; + filter->enables |= NTUPLE_FLTR_ALLOC_INPUT_EN_MIRROR_VNIC_ID; + + vnic0 = STAILQ_FIRST(&bp->ff_pool[0]); + filter1 = bnxt_get_l2_filter(bp, filter, vnic0); + if (filter1 == NULL) { + rc = -ENOSPC; + goto ret; + } + filter->fw_l2_filter_id = filter1->fw_l2_filter_id; + break; + + default: + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, act, + "Invalid action."); + rc = -rte_errno; + goto ret; + } + +//done: + act = nxt_non_void_action(++act); + if (act->type != RTE_FLOW_ACTION_TYPE_END) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + act, "Invalid action."); + rc = -rte_errno; + goto ret; + } +ret: + return rc; +} + +static int +bnxt_flow_validate(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error) +{ + struct bnxt *bp = (struct bnxt *)dev->data->dev_private; + struct bnxt_filter_info *filter; + int ret = 0; + + ret = bnxt_flow_agrs_validate(attr, pattern, actions, error); + if (ret != 0) + return ret; + + filter = bnxt_get_unused_filter(bp); + if (filter == NULL) { + RTE_LOG(ERR, PMD, "Not enough resources for a new flow.\n"); + return -ENOMEM; + } + + ret = bnxt_validate_and_parse_flow(dev, pattern, actions, attr, + error, filter); + /* No need to hold on to this filter if we are just validating flow */ + filter->fw_l2_filter_id = -1; + bnxt_free_filter(bp, filter); + + return ret; +} + +static int +bnxt_match_filter(struct bnxt *bp, struct bnxt_filter_info *nf) +{ + struct bnxt_filter_info *mf; + struct rte_flow *flow; + int i; + + for (i = bp->nr_vnics - 1; i >= 0; i--) { + struct bnxt_vnic_info *vnic = &bp->vnic_info[i]; + + STAILQ_FOREACH(flow, &vnic->flow_list, next) { + mf = flow->filter; + + if (mf->filter_type == nf->filter_type && + mf->flags == nf->flags && + mf->src_port == nf->src_port && + mf->src_port_mask == nf->src_port_mask && + mf->dst_port == nf->dst_port && + mf->dst_port_mask == nf->dst_port_mask && + mf->ip_protocol == nf->ip_protocol && + mf->ip_addr_type == nf->ip_addr_type && + mf->ethertype == nf->ethertype && + mf->vni == nf->vni && + mf->tunnel_type == nf->tunnel_type && + mf->l2_ovlan == nf->l2_ovlan && + mf->l2_ovlan_mask == nf->l2_ovlan_mask && + mf->l2_ivlan == nf->l2_ivlan && + mf->l2_ivlan_mask == nf->l2_ivlan_mask && + !memcmp(mf->l2_addr, nf->l2_addr, ETHER_ADDR_LEN) && + !memcmp(mf->l2_addr_mask, nf->l2_addr_mask, + ETHER_ADDR_LEN) && + !memcmp(mf->src_macaddr, nf->src_macaddr, + ETHER_ADDR_LEN) && + !memcmp(mf->dst_macaddr, nf->dst_macaddr, + ETHER_ADDR_LEN) && + !memcmp(mf->src_ipaddr, nf->src_ipaddr, + sizeof(nf->src_ipaddr)) && + !memcmp(mf->src_ipaddr_mask, nf->src_ipaddr_mask, + sizeof(nf->src_ipaddr_mask)) && + !memcmp(mf->dst_ipaddr, nf->dst_ipaddr, + sizeof(nf->dst_ipaddr)) && + !memcmp(mf->dst_ipaddr_mask, nf->dst_ipaddr_mask, + sizeof(nf->dst_ipaddr_mask))) { + if (mf->dst_id == nf->dst_id) + return -EEXIST; + /* Same Flow, Different queue + * Clear the old ntuple filter + */ + if (nf->filter_type == HWRM_CFA_EM_FILTER) + bnxt_hwrm_clear_em_filter(bp, mf); + if (nf->filter_type == HWRM_CFA_NTUPLE_FILTER) + bnxt_hwrm_clear_ntuple_filter(bp, mf); + /* Free the old filter, update flow + * with new filter + */ + bnxt_free_filter(bp, mf); + flow->filter = nf; + return -EXDEV; + } + } + } + return 0; +} + +static struct rte_flow * +bnxt_flow_create(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error) +{ + struct bnxt *bp = (struct bnxt *)dev->data->dev_private; + struct bnxt_filter_info *filter; + struct bnxt_vnic_info *vnic = NULL; + bool update_flow = false; + struct rte_flow *flow; + unsigned int i; + int ret = 0; + + flow = rte_zmalloc("bnxt_flow", sizeof(struct rte_flow), 0); + if (!flow) { + rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_HANDLE, NULL, + "Failed to allocate memory"); + return flow; + } + + ret = bnxt_flow_agrs_validate(attr, pattern, actions, error); + if (ret != 0) { + RTE_LOG(ERR, PMD, "Not a validate flow.\n"); + goto free_flow; + } + + filter = bnxt_get_unused_filter(bp); + if (filter == NULL) { + RTE_LOG(ERR, PMD, "Not enough resources for a new flow.\n"); + goto free_flow; + } + + ret = bnxt_validate_and_parse_flow(dev, pattern, actions, attr, + error, filter); + if (ret != 0) + goto free_filter; + + ret = bnxt_match_filter(bp, filter); + if (ret == -EEXIST) { + RTE_LOG(DEBUG, PMD, "Flow already exists.\n"); + /* Clear the filter that was created as part of + * validate_and_parse_flow() above + */ + bnxt_hwrm_clear_l2_filter(bp, filter); + goto free_filter; + } else if (ret == -EXDEV) { + RTE_LOG(DEBUG, PMD, "Flow with same pattern exists"); + RTE_LOG(DEBUG, PMD, "Updating with different destination\n"); + update_flow = true; + } + + if (filter->filter_type == HWRM_CFA_EM_FILTER) { + filter->enables |= + HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_L2_FILTER_ID; + ret = bnxt_hwrm_set_em_filter(bp, filter->dst_id, filter); + } + if (filter->filter_type == HWRM_CFA_NTUPLE_FILTER) { + filter->enables |= + HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_L2_FILTER_ID; + ret = bnxt_hwrm_set_ntuple_filter(bp, filter->dst_id, filter); + } + + for (i = 0; i < bp->nr_vnics; i++) { + vnic = &bp->vnic_info[i]; + if (filter->dst_id == vnic->fw_vnic_id) + break; + } + + if (!ret) { + flow->filter = filter; + flow->vnic = vnic; + if (update_flow) { + ret = -EXDEV; + goto free_flow; + } + RTE_LOG(ERR, PMD, "Successfully created flow.\n"); + STAILQ_INSERT_TAIL(&vnic->flow_list, flow, next); + return flow; + } +free_filter: + bnxt_free_filter(bp, filter); +free_flow: + if (ret == -EEXIST) + rte_flow_error_set(error, ret, + RTE_FLOW_ERROR_TYPE_HANDLE, NULL, + "Matching Flow exists."); + else if (ret == -EXDEV) + rte_flow_error_set(error, ret, + RTE_FLOW_ERROR_TYPE_HANDLE, NULL, + "Flow with pattern exists, updating destination queue"); + else + rte_flow_error_set(error, -ret, + RTE_FLOW_ERROR_TYPE_HANDLE, NULL, + "Failed to create flow."); + rte_free(flow); + flow = NULL; + return flow; +} + +static int +bnxt_flow_destroy(struct rte_eth_dev *dev, + struct rte_flow *flow, + struct rte_flow_error *error) +{ + struct bnxt *bp = (struct bnxt *)dev->data->dev_private; + struct bnxt_filter_info *filter = flow->filter; + struct bnxt_vnic_info *vnic = flow->vnic; + int ret = 0; + + ret = bnxt_match_filter(bp, filter); + if (ret == 0) + RTE_LOG(ERR, PMD, "Could not find matching flow\n"); + if (filter->filter_type == HWRM_CFA_EM_FILTER) + ret = bnxt_hwrm_clear_em_filter(bp, filter); + if (filter->filter_type == HWRM_CFA_NTUPLE_FILTER) + ret = bnxt_hwrm_clear_ntuple_filter(bp, filter); + + bnxt_hwrm_clear_l2_filter(bp, filter); + if (!ret) { + STAILQ_REMOVE(&vnic->flow_list, flow, rte_flow, next); + rte_free(flow); + } else { + rte_flow_error_set(error, -ret, + RTE_FLOW_ERROR_TYPE_HANDLE, NULL, + "Failed to destroy flow."); + } + + return ret; +} + +static int +bnxt_flow_flush(struct rte_eth_dev *dev, struct rte_flow_error *error) +{ + struct bnxt *bp = (struct bnxt *)dev->data->dev_private; + struct bnxt_vnic_info *vnic; + struct rte_flow *flow; + unsigned int i; + int ret = 0; + + for (i = 0; i < bp->nr_vnics; i++) { + vnic = &bp->vnic_info[i]; + STAILQ_FOREACH(flow, &vnic->flow_list, next) { + struct bnxt_filter_info *filter = flow->filter; + + if (filter->filter_type == HWRM_CFA_EM_FILTER) + ret = bnxt_hwrm_clear_em_filter(bp, filter); + if (filter->filter_type == HWRM_CFA_NTUPLE_FILTER) + ret = bnxt_hwrm_clear_ntuple_filter(bp, filter); + + if (ret) { + rte_flow_error_set(error, -ret, + RTE_FLOW_ERROR_TYPE_HANDLE, + NULL, + "Failed to flush flow in HW."); + return -rte_errno; + } + + STAILQ_REMOVE(&vnic->flow_list, flow, + rte_flow, next); + rte_free(flow); + } + } + + return ret; +} + +const struct rte_flow_ops bnxt_flow_ops = { + .validate = bnxt_flow_validate, + .create = bnxt_flow_create, + .destroy = bnxt_flow_destroy, + .flush = bnxt_flow_flush, +}; diff --git a/dpdk/drivers/net/bnxt/bnxt_filter.h b/dpdk/drivers/net/bnxt/bnxt_filter.h index 06fe134a..2591a87e 100644 --- a/dpdk/drivers/net/bnxt/bnxt_filter.h +++ b/dpdk/drivers/net/bnxt/bnxt_filter.h @@ -40,8 +40,15 @@ struct bnxt; struct bnxt_filter_info { STAILQ_ENTRY(bnxt_filter_info) next; uint64_t fw_l2_filter_id; + uint64_t fw_em_filter_id; + uint64_t fw_ntuple_filter_id; #define INVALID_MAC_INDEX ((uint16_t)-1) uint16_t mac_index; +#define HWRM_CFA_L2_FILTER 0 +#define HWRM_CFA_EM_FILTER 1 +#define HWRM_CFA_NTUPLE_FILTER 2 + uint8_t filter_type; //L2 or EM or NTUPLE filter + uint32_t dst_id; /* Filter Characteristics */ uint32_t flags; @@ -63,12 +70,96 @@ struct bnxt_filter_info { uint32_t vni; uint8_t pri_hint; uint64_t l2_filter_id_hint; + uint32_t src_id; + uint8_t src_type; + uint8_t src_macaddr[6]; + uint8_t dst_macaddr[6]; + uint32_t dst_ipaddr[4]; + uint32_t dst_ipaddr_mask[4]; + uint32_t src_ipaddr[4]; + uint32_t src_ipaddr_mask[4]; + uint16_t dst_port; + uint16_t dst_port_mask; + uint16_t src_port; + uint16_t src_port_mask; + uint16_t ip_protocol; + uint16_t ip_addr_type; + uint16_t ethertype; }; struct bnxt_filter_info *bnxt_alloc_filter(struct bnxt *bp); +struct bnxt_filter_info *bnxt_alloc_vf_filter(struct bnxt *bp, uint16_t vf); void bnxt_init_filters(struct bnxt *bp); void bnxt_free_all_filters(struct bnxt *bp); void bnxt_free_filter_mem(struct bnxt *bp); int bnxt_alloc_filter_mem(struct bnxt *bp); +struct bnxt_filter_info *bnxt_get_unused_filter(struct bnxt *bp); +void bnxt_free_filter(struct bnxt *bp, struct bnxt_filter_info *filter); +struct bnxt_filter_info *bnxt_get_l2_filter(struct bnxt *bp, + struct bnxt_filter_info *nf, struct bnxt_vnic_info *vnic); +#define NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_MACADDR \ + HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_SRC_MACADDR +#define EM_FLOW_ALLOC_INPUT_EN_SRC_MACADDR \ + HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_SRC_MACADDR +#define NTUPLE_FLTR_ALLOC_INPUT_EN_DST_MACADDR \ + HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_DST_MACADDR +#define EM_FLOW_ALLOC_INPUT_EN_DST_MACADDR \ + HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_DST_MACADDR +#define NTUPLE_FLTR_ALLOC_INPUT_EN_ETHERTYPE \ + HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_ETHERTYPE +#define EM_FLOW_ALLOC_INPUT_EN_ETHERTYPE \ + HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_ETHERTYPE +#define EM_FLOW_ALLOC_INPUT_EN_OVLAN_VID \ + HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_OVLAN_VID +#define NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR \ + HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_SRC_IPADDR +#define NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR_MASK \ + HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_SRC_IPADDR_MASK +#define NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR \ + HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_DST_IPADDR +#define NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR_MASK \ + HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_DST_IPADDR_MASK +#define NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_PORT \ + HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_SRC_PORT +#define NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_PORT_MASK \ + HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_SRC_PORT_MASK +#define NTUPLE_FLTR_ALLOC_INPUT_EN_DST_PORT \ + HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_DST_PORT +#define NTUPLE_FLTR_ALLOC_INPUT_EN_DST_PORT_MASK \ + HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_DST_PORT_MASK +#define NTUPLE_FLTR_ALLOC_IN_EN_IP_PROTO \ + HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_IP_PROTOCOL +#define EM_FLOW_ALLOC_INPUT_EN_SRC_IPADDR \ + HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_SRC_IPADDR +#define EM_FLOW_ALLOC_INPUT_EN_DST_IPADDR \ + HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_DST_IPADDR +#define EM_FLOW_ALLOC_INPUT_EN_SRC_PORT \ + HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_SRC_PORT +#define EM_FLOW_ALLOC_INPUT_EN_DST_PORT \ + HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_DST_PORT +#define EM_FLOW_ALLOC_INPUT_EN_IP_PROTO \ + HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_IP_PROTOCOL +#define EM_FLOW_ALLOC_INPUT_IP_ADDR_TYPE_IPV6 \ + HWRM_CFA_EM_FLOW_ALLOC_INPUT_IP_ADDR_TYPE_IPV6 +#define NTUPLE_FLTR_ALLOC_INPUT_IP_ADDR_TYPE_IPV6 \ + HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_IP_ADDR_TYPE_IPV6 +#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN \ + HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_TUNNEL_TYPE_VXLAN +#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_NVGRE \ + HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_TUNNEL_TYPE_NVGRE +#define L2_FILTER_ALLOC_INPUT_EN_L2_ADDR_MASK \ + HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_ADDR_MASK +#define NTUPLE_FLTR_ALLOC_INPUT_IP_PROTOCOL_UDP \ + HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_IP_PROTOCOL_UDP +#define NTUPLE_FLTR_ALLOC_INPUT_IP_PROTOCOL_TCP \ + HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_IP_PROTOCOL_TCP +#define NTUPLE_FLTR_ALLOC_INPUT_IP_PROTOCOL_UNKNOWN \ + HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_IP_PROTOCOL_UNKNOWN +#define NTUPLE_FLTR_ALLOC_INPUT_IP_ADDR_TYPE_IPV4 \ + HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_IP_ADDR_TYPE_IPV4 +#define NTUPLE_FLTR_ALLOC_INPUT_EN_MIRROR_VNIC_ID \ + HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_MIRROR_VNIC_ID +#define NTUPLE_FLTR_ALLOC_INPUT_EN_MIRROR_VNIC_ID \ + HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_MIRROR_VNIC_ID #endif diff --git a/dpdk/drivers/net/bnxt/bnxt_hwrm.c b/dpdk/drivers/net/bnxt/bnxt_hwrm.c index 2ed4c2f1..ce214d7c 100644 --- a/dpdk/drivers/net/bnxt/bnxt_hwrm.c +++ b/dpdk/drivers/net/bnxt/bnxt_hwrm.c @@ -31,6 +31,8 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +#include + #include #include #include @@ -50,7 +52,41 @@ #include "bnxt_vnic.h" #include "hsi_struct_def_dpdk.h" -#define HWRM_CMD_TIMEOUT 2000 +#include + +#define HWRM_CMD_TIMEOUT 10000 + +struct bnxt_plcmodes_cfg { + uint32_t flags; + uint16_t jumbo_thresh; + uint16_t hds_offset; + uint16_t hds_threshold; +}; + +static int page_getenum(size_t size) +{ + if (size <= 1 << 4) + return 4; + if (size <= 1 << 12) + return 12; + if (size <= 1 << 13) + return 13; + if (size <= 1 << 16) + return 16; + if (size <= 1 << 21) + return 21; + if (size <= 1 << 22) + return 22; + if (size <= 1 << 30) + return 30; + RTE_LOG(ERR, PMD, "Page size %zu out of range\n", size); + return sizeof(void *) * 8 - 1; +} + +static int page_roundup(size_t size) +{ + return 1 << page_getenum(size); +} /* * HWRM Functions (sent to HWRM) @@ -59,7 +95,7 @@ * command was failed by the ChiMP. */ -static int bnxt_hwrm_send_message_locked(struct bnxt *bp, void *msg, +static int bnxt_hwrm_send_message(struct bnxt *bp, void *msg, uint32_t msg_len) { unsigned int i; @@ -68,23 +104,47 @@ static int bnxt_hwrm_send_message_locked(struct bnxt *bp, void *msg, uint32_t *data = msg; uint8_t *bar; uint8_t *valid; + uint16_t max_req_len = bp->max_req_len; + struct hwrm_short_input short_input = { 0 }; + + if (bp->flags & BNXT_FLAG_SHORT_CMD) { + void *short_cmd_req = bp->hwrm_short_cmd_req_addr; + + memset(short_cmd_req, 0, bp->max_req_len); + memcpy(short_cmd_req, req, msg_len); + + short_input.req_type = rte_cpu_to_le_16(req->req_type); + short_input.signature = rte_cpu_to_le_16( + HWRM_SHORT_REQ_SIGNATURE_SHORT_CMD); + short_input.size = rte_cpu_to_le_16(msg_len); + short_input.req_addr = + rte_cpu_to_le_64(bp->hwrm_short_cmd_req_dma_addr); + + data = (uint32_t *)&short_input; + msg_len = sizeof(short_input); + + /* Sync memory write before updating doorbell */ + rte_wmb(); + + max_req_len = BNXT_HWRM_SHORT_REQ_LEN; + } /* Write request msg to hwrm channel */ for (i = 0; i < msg_len; i += 4) { bar = (uint8_t *)bp->bar0 + i; - *(volatile uint32_t *)bar = *data; + rte_write32(*data, bar); data++; } /* Zero the rest of the request space */ - for (; i < bp->max_req_len; i += 4) { + for (; i < max_req_len; i += 4) { bar = (uint8_t *)bp->bar0 + i; - *(volatile uint32_t *)bar = 0; + rte_write32(0, bar); } /* Ring channel doorbell */ bar = (uint8_t *)bp->bar0 + 0x100; - *(volatile uint32_t *)bar = 1; + rte_write32(1, bar); /* Poll for the valid bit */ for (i = 0; i < HWRM_CMD_TIMEOUT; i++) { @@ -101,7 +161,7 @@ static int bnxt_hwrm_send_message_locked(struct bnxt *bp, void *msg, } if (i >= HWRM_CMD_TIMEOUT) { - RTE_LOG(ERR, PMD, "Error sending msg %x\n", + RTE_LOG(ERR, PMD, "Error sending msg 0x%04x\n", req->req_type); goto err_ret; } @@ -111,37 +171,58 @@ err_ret: return -1; } -static int bnxt_hwrm_send_message(struct bnxt *bp, void *msg, uint32_t msg_len) -{ - int rc; - - rte_spinlock_lock(&bp->hwrm_lock); - rc = bnxt_hwrm_send_message_locked(bp, msg, msg_len); - rte_spinlock_unlock(&bp->hwrm_lock); - return rc; -} - -#define HWRM_PREP(req, type, cr, resp) \ +/* + * HWRM_PREP() should be used to prepare *ALL* HWRM commands. It grabs the + * spinlock, and does initial processing. + * + * HWRM_CHECK_RESULT() returns errors on failure and may not be used. It + * releases the spinlock only if it returns. If the regular int return codes + * are not used by the function, HWRM_CHECK_RESULT() should not be used + * directly, rather it should be copied and modified to suit the function. + * + * HWRM_UNLOCK() must be called after all response processing is completed. + */ +#define HWRM_PREP(req, type) do { \ + rte_spinlock_lock(&bp->hwrm_lock); \ memset(bp->hwrm_cmd_resp_addr, 0, bp->max_resp_len); \ req.req_type = rte_cpu_to_le_16(HWRM_##type); \ - req.cmpl_ring = rte_cpu_to_le_16(cr); \ + req.cmpl_ring = rte_cpu_to_le_16(-1); \ req.seq_id = rte_cpu_to_le_16(bp->hwrm_cmd_seq++); \ req.target_id = rte_cpu_to_le_16(0xffff); \ - req.resp_addr = rte_cpu_to_le_64(bp->hwrm_cmd_resp_dma_addr) + req.resp_addr = rte_cpu_to_le_64(bp->hwrm_cmd_resp_dma_addr); \ +} while (0) -#define HWRM_CHECK_RESULT \ - { \ - if (rc) { \ - RTE_LOG(ERR, PMD, "%s failed rc:%d\n", \ - __func__, rc); \ - return rc; \ +#define HWRM_CHECK_RESULT() do {\ + if (rc) { \ + RTE_LOG(ERR, PMD, "%s failed rc:%d\n", \ + __func__, rc); \ + rte_spinlock_unlock(&bp->hwrm_lock); \ + return rc; \ + } \ + if (resp->error_code) { \ + rc = rte_le_to_cpu_16(resp->error_code); \ + if (resp->resp_len >= 16) { \ + struct hwrm_err_output *tmp_hwrm_err_op = \ + (void *)resp; \ + RTE_LOG(ERR, PMD, \ + "%s error %d:%d:%08x:%04x\n", \ + __func__, \ + rc, tmp_hwrm_err_op->cmd_err, \ + rte_le_to_cpu_32(\ + tmp_hwrm_err_op->opaque_0), \ + rte_le_to_cpu_16(\ + tmp_hwrm_err_op->opaque_1)); \ } \ - if (resp->error_code) { \ - rc = rte_le_to_cpu_16(resp->error_code); \ - RTE_LOG(ERR, PMD, "%s error %d\n", __func__, rc); \ - return rc; \ + else { \ + RTE_LOG(ERR, PMD, \ + "%s error %d\n", __func__, rc); \ } \ - } + rte_spinlock_unlock(&bp->hwrm_lock); \ + return rc; \ + } \ +} while (0) + +#define HWRM_UNLOCK() rte_spinlock_unlock(&bp->hwrm_lock) int bnxt_hwrm_cfa_l2_clear_rx_mask(struct bnxt *bp, struct bnxt_vnic_info *vnic) { @@ -149,80 +230,165 @@ int bnxt_hwrm_cfa_l2_clear_rx_mask(struct bnxt *bp, struct bnxt_vnic_info *vnic) struct hwrm_cfa_l2_set_rx_mask_input req = {.req_type = 0 }; struct hwrm_cfa_l2_set_rx_mask_output *resp = bp->hwrm_cmd_resp_addr; - HWRM_PREP(req, CFA_L2_SET_RX_MASK, -1, resp); + HWRM_PREP(req, CFA_L2_SET_RX_MASK); req.vnic_id = rte_cpu_to_le_16(vnic->fw_vnic_id); req.mask = 0; rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); - HWRM_CHECK_RESULT; + HWRM_CHECK_RESULT(); + HWRM_UNLOCK(); return rc; } -int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, struct bnxt_vnic_info *vnic) +int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, + struct bnxt_vnic_info *vnic, + uint16_t vlan_count, + struct bnxt_vlan_table_entry *vlan_table) { int rc = 0; struct hwrm_cfa_l2_set_rx_mask_input req = {.req_type = 0 }; struct hwrm_cfa_l2_set_rx_mask_output *resp = bp->hwrm_cmd_resp_addr; uint32_t mask = 0; - HWRM_PREP(req, CFA_L2_SET_RX_MASK, -1, resp); + HWRM_PREP(req, CFA_L2_SET_RX_MASK); req.vnic_id = rte_cpu_to_le_16(vnic->fw_vnic_id); /* FIXME add multicast flag, when multicast adding options is supported * by ethtool. */ + if (vnic->flags & BNXT_VNIC_INFO_BCAST) + mask |= HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_BCAST; + if (vnic->flags & BNXT_VNIC_INFO_UNTAGGED) + mask |= HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_VLAN_NONVLAN; if (vnic->flags & BNXT_VNIC_INFO_PROMISC) - mask = HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_PROMISCUOUS; + mask |= HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_PROMISCUOUS; if (vnic->flags & BNXT_VNIC_INFO_ALLMULTI) - mask = HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_ALL_MCAST; - req.mask = rte_cpu_to_le_32(HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_BCAST | - mask); + mask |= HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_ALL_MCAST; + if (vnic->flags & BNXT_VNIC_INFO_MCAST) + mask |= HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_MCAST; + if (vnic->mc_addr_cnt) { + mask |= HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_MCAST; + req.num_mc_entries = rte_cpu_to_le_32(vnic->mc_addr_cnt); + req.mc_tbl_addr = rte_cpu_to_le_64(vnic->mc_list_dma_addr); + } + if (vlan_table) { + if (!(mask & HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_VLAN_NONVLAN)) + mask |= HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_VLANONLY; + req.vlan_tag_tbl_addr = rte_cpu_to_le_64( + rte_mem_virt2iova(vlan_table)); + req.num_vlan_tags = rte_cpu_to_le_32((uint32_t)vlan_count); + } + req.mask = rte_cpu_to_le_32(mask); rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); - HWRM_CHECK_RESULT; + HWRM_CHECK_RESULT(); + HWRM_UNLOCK(); return rc; } -int bnxt_hwrm_clear_filter(struct bnxt *bp, +int bnxt_hwrm_cfa_vlan_antispoof_cfg(struct bnxt *bp, uint16_t fid, + uint16_t vlan_count, + struct bnxt_vlan_antispoof_table_entry *vlan_table) +{ + int rc = 0; + struct hwrm_cfa_vlan_antispoof_cfg_input req = {.req_type = 0 }; + struct hwrm_cfa_vlan_antispoof_cfg_output *resp = + bp->hwrm_cmd_resp_addr; + + /* + * Older HWRM versions did not support this command, and the set_rx_mask + * list was used for anti-spoof. In 1.8.0, the TX path configuration was + * removed from set_rx_mask call, and this command was added. + * + * This command is also present from 1.7.8.11 and higher, + * as well as 1.7.8.0 + */ + if (bp->fw_ver < ((1 << 24) | (8 << 16))) { + if (bp->fw_ver != ((1 << 24) | (7 << 16) | (8 << 8))) { + if (bp->fw_ver < ((1 << 24) | (7 << 16) | (8 << 8) | + (11))) + return 0; + } + } + HWRM_PREP(req, CFA_VLAN_ANTISPOOF_CFG); + req.fid = rte_cpu_to_le_16(fid); + + req.vlan_tag_mask_tbl_addr = + rte_cpu_to_le_64(rte_mem_virt2iova(vlan_table)); + req.num_vlan_entries = rte_cpu_to_le_32((uint32_t)vlan_count); + + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + + HWRM_CHECK_RESULT(); + HWRM_UNLOCK(); + + return rc; +} + +int bnxt_hwrm_clear_l2_filter(struct bnxt *bp, struct bnxt_filter_info *filter) { int rc = 0; struct hwrm_cfa_l2_filter_free_input req = {.req_type = 0 }; struct hwrm_cfa_l2_filter_free_output *resp = bp->hwrm_cmd_resp_addr; - HWRM_PREP(req, CFA_L2_FILTER_FREE, -1, resp); + if (filter->fw_l2_filter_id == UINT64_MAX) + return 0; + + HWRM_PREP(req, CFA_L2_FILTER_FREE); req.l2_filter_id = rte_cpu_to_le_64(filter->fw_l2_filter_id); rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); - HWRM_CHECK_RESULT; + HWRM_CHECK_RESULT(); + HWRM_UNLOCK(); filter->fw_l2_filter_id = -1; return 0; } -int bnxt_hwrm_set_filter(struct bnxt *bp, - struct bnxt_vnic_info *vnic, +int bnxt_hwrm_set_l2_filter(struct bnxt *bp, + uint16_t dst_id, struct bnxt_filter_info *filter) { int rc = 0; struct hwrm_cfa_l2_filter_alloc_input req = {.req_type = 0 }; struct hwrm_cfa_l2_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr; + struct rte_eth_conf *dev_conf = &bp->eth_dev->data->dev_conf; + const struct rte_eth_vmdq_rx_conf *conf = + &dev_conf->rx_adv_conf.vmdq_rx_conf; uint32_t enables = 0; + uint16_t j = dst_id - 1; - HWRM_PREP(req, CFA_L2_FILTER_ALLOC, -1, resp); + //TODO: Is there a better way to add VLANs to each VNIC in case of VMDQ + if ((dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG) && + conf->pool_map[j].pools & (1UL << j)) { + RTE_LOG(DEBUG, PMD, + "Add vlan %u to vmdq pool %u\n", + conf->pool_map[j].vlan_id, j); + + filter->l2_ivlan = conf->pool_map[j].vlan_id; + filter->enables |= + HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_IVLAN | + HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_IVLAN_MASK; + } + + if (filter->fw_l2_filter_id != UINT64_MAX) + bnxt_hwrm_clear_l2_filter(bp, filter); + + HWRM_PREP(req, CFA_L2_FILTER_ALLOC); req.flags = rte_cpu_to_le_32(filter->flags); enables = filter->enables | HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_DST_ID; - req.dst_id = rte_cpu_to_le_16(vnic->fw_vnic_id); + req.dst_id = rte_cpu_to_le_16(dst_id); if (enables & HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_ADDR) @@ -235,35 +401,28 @@ int bnxt_hwrm_set_filter(struct bnxt *bp, if (enables & HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_OVLAN) req.l2_ovlan = filter->l2_ovlan; + if (enables & + HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_IVLAN) + req.l2_ovlan = filter->l2_ivlan; if (enables & HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_OVLAN_MASK) req.l2_ovlan_mask = filter->l2_ovlan_mask; + if (enables & + HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_IVLAN_MASK) + req.l2_ovlan_mask = filter->l2_ivlan_mask; + if (enables & HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_SRC_ID) + req.src_id = rte_cpu_to_le_32(filter->src_id); + if (enables & HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_SRC_TYPE) + req.src_type = filter->src_type; req.enables = rte_cpu_to_le_32(enables); rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); - HWRM_CHECK_RESULT; + HWRM_CHECK_RESULT(); filter->fw_l2_filter_id = rte_le_to_cpu_64(resp->l2_filter_id); - - return rc; -} - -int bnxt_hwrm_exec_fwd_resp(struct bnxt *bp, void *fwd_cmd) -{ - int rc; - struct hwrm_exec_fwd_resp_input req = {.req_type = 0 }; - struct hwrm_exec_fwd_resp_output *resp = bp->hwrm_cmd_resp_addr; - - HWRM_PREP(req, EXEC_FWD_RESP, -1, resp); - - memcpy(req.encap_request, fwd_cmd, - sizeof(req.encap_request)); - - rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); - - HWRM_CHECK_RESULT; + HWRM_UNLOCK(); return rc; } @@ -273,43 +432,78 @@ int bnxt_hwrm_func_qcaps(struct bnxt *bp) int rc = 0; struct hwrm_func_qcaps_input req = {.req_type = 0 }; struct hwrm_func_qcaps_output *resp = bp->hwrm_cmd_resp_addr; + uint16_t new_max_vfs; + int i; - HWRM_PREP(req, FUNC_QCAPS, -1, resp); + HWRM_PREP(req, FUNC_QCAPS); req.fid = rte_cpu_to_le_16(0xffff); rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); - HWRM_CHECK_RESULT; + HWRM_CHECK_RESULT(); bp->max_ring_grps = rte_le_to_cpu_32(resp->max_hw_ring_grps); if (BNXT_PF(bp)) { - struct bnxt_pf_info *pf = &bp->pf; - - pf->fw_fid = rte_le_to_cpu_32(resp->fid); - pf->port_id = resp->port_id; - memcpy(pf->mac_addr, resp->perm_mac_address, ETHER_ADDR_LEN); - pf->max_rsscos_ctx = rte_le_to_cpu_16(resp->max_rsscos_ctx); - pf->max_cp_rings = rte_le_to_cpu_16(resp->max_cmpl_rings); - pf->max_tx_rings = rte_le_to_cpu_16(resp->max_tx_rings); - pf->max_rx_rings = rte_le_to_cpu_16(resp->max_rx_rings); - pf->max_l2_ctx = rte_le_to_cpu_16(resp->max_l2_ctxs); - pf->max_vnics = rte_le_to_cpu_16(resp->max_vnics); - pf->first_vf_id = rte_le_to_cpu_16(resp->first_vf_id); - pf->max_vfs = rte_le_to_cpu_16(resp->max_vfs); - } else { - struct bnxt_vf_info *vf = &bp->vf; - - vf->fw_fid = rte_le_to_cpu_32(resp->fid); - memcpy(vf->mac_addr, &resp->perm_mac_address, ETHER_ADDR_LEN); - vf->max_rsscos_ctx = rte_le_to_cpu_16(resp->max_rsscos_ctx); - vf->max_cp_rings = rte_le_to_cpu_16(resp->max_cmpl_rings); - vf->max_tx_rings = rte_le_to_cpu_16(resp->max_tx_rings); - vf->max_rx_rings = rte_le_to_cpu_16(resp->max_rx_rings); - vf->max_l2_ctx = rte_le_to_cpu_16(resp->max_l2_ctxs); - vf->max_vnics = rte_le_to_cpu_16(resp->max_vnics); + bp->pf.port_id = resp->port_id; + bp->pf.first_vf_id = rte_le_to_cpu_16(resp->first_vf_id); + new_max_vfs = bp->pdev->max_vfs; + if (new_max_vfs != bp->pf.max_vfs) { + if (bp->pf.vf_info) + rte_free(bp->pf.vf_info); + bp->pf.vf_info = rte_malloc("bnxt_vf_info", + sizeof(bp->pf.vf_info[0]) * new_max_vfs, 0); + bp->pf.max_vfs = new_max_vfs; + for (i = 0; i < new_max_vfs; i++) { + bp->pf.vf_info[i].fid = bp->pf.first_vf_id + i; + bp->pf.vf_info[i].vlan_table = + rte_zmalloc("VF VLAN table", + getpagesize(), + getpagesize()); + if (bp->pf.vf_info[i].vlan_table == NULL) + RTE_LOG(ERR, PMD, + "Fail to alloc VLAN table for VF %d\n", + i); + else + rte_mem_lock_page( + bp->pf.vf_info[i].vlan_table); + bp->pf.vf_info[i].vlan_as_table = + rte_zmalloc("VF VLAN AS table", + getpagesize(), + getpagesize()); + if (bp->pf.vf_info[i].vlan_as_table == NULL) + RTE_LOG(ERR, PMD, + "Alloc VLAN AS table for VF %d fail\n", + i); + else + rte_mem_lock_page( + bp->pf.vf_info[i].vlan_as_table); + STAILQ_INIT(&bp->pf.vf_info[i].filter); + } + } } + bp->fw_fid = rte_le_to_cpu_32(resp->fid); + memcpy(bp->dflt_mac_addr, &resp->mac_address, ETHER_ADDR_LEN); + bp->max_rsscos_ctx = rte_le_to_cpu_16(resp->max_rsscos_ctx); + bp->max_cp_rings = rte_le_to_cpu_16(resp->max_cmpl_rings); + bp->max_tx_rings = rte_le_to_cpu_16(resp->max_tx_rings); + bp->max_rx_rings = rte_le_to_cpu_16(resp->max_rx_rings); + bp->max_l2_ctx = rte_le_to_cpu_16(resp->max_l2_ctxs); + /* TODO: For now, do not support VMDq/RFS on VFs. */ + if (BNXT_PF(bp)) { + if (bp->pf.max_vfs) + bp->max_vnics = 1; + else + bp->max_vnics = rte_le_to_cpu_16(resp->max_vnics); + } else { + bp->max_vnics = 1; + } + bp->max_stat_ctx = rte_le_to_cpu_16(resp->max_stat_ctx); + if (BNXT_PF(bp)) + bp->pf.total_vnics = rte_le_to_cpu_16(resp->max_vnics); + HWRM_UNLOCK(); + return rc; } @@ -319,19 +513,19 @@ int bnxt_hwrm_func_reset(struct bnxt *bp) struct hwrm_func_reset_input req = {.req_type = 0 }; struct hwrm_func_reset_output *resp = bp->hwrm_cmd_resp_addr; - HWRM_PREP(req, FUNC_RESET, -1, resp); + HWRM_PREP(req, FUNC_RESET); req.enables = rte_cpu_to_le_32(0); rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); - HWRM_CHECK_RESULT; + HWRM_CHECK_RESULT(); + HWRM_UNLOCK(); return rc; } -int bnxt_hwrm_func_driver_register(struct bnxt *bp, uint32_t flags, - uint32_t *vf_req_fwd) +int bnxt_hwrm_func_driver_register(struct bnxt *bp) { int rc; struct hwrm_func_drv_rgtr_input req = {.req_type = 0 }; @@ -340,18 +534,28 @@ int bnxt_hwrm_func_driver_register(struct bnxt *bp, uint32_t flags, if (bp->flags & BNXT_FLAG_REGISTERED) return 0; - HWRM_PREP(req, FUNC_DRV_RGTR, -1, resp); - req.flags = flags; - req.enables = HWRM_FUNC_DRV_RGTR_INPUT_ENABLES_VER; + HWRM_PREP(req, FUNC_DRV_RGTR); + req.enables = rte_cpu_to_le_32(HWRM_FUNC_DRV_RGTR_INPUT_ENABLES_VER | + HWRM_FUNC_DRV_RGTR_INPUT_ENABLES_ASYNC_EVENT_FWD); req.ver_maj = RTE_VER_YEAR; req.ver_min = RTE_VER_MONTH; req.ver_upd = RTE_VER_MINOR; - memcpy(req.vf_req_fwd, vf_req_fwd, sizeof(req.vf_req_fwd)); + if (BNXT_PF(bp)) { + req.enables |= rte_cpu_to_le_32( + HWRM_FUNC_DRV_RGTR_INPUT_ENABLES_VF_INPUT_FWD); + memcpy(req.vf_req_fwd, bp->pf.vf_req_fwd, + RTE_MIN(sizeof(req.vf_req_fwd), + sizeof(bp->pf.vf_req_fwd))); + } + + req.async_event_fwd[0] |= rte_cpu_to_le_32(0x1); /* TODO: Use MACRO */ + //memset(req.async_event_fwd, 0xff, sizeof(req.async_event_fwd)); rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); - HWRM_CHECK_RESULT; + HWRM_CHECK_RESULT(); + HWRM_UNLOCK(); bp->flags |= BNXT_FLAG_REGISTERED; @@ -367,25 +571,27 @@ int bnxt_hwrm_ver_get(struct bnxt *bp) uint32_t fw_version; uint16_t max_resp_len; char type[RTE_MEMZONE_NAMESIZE]; + uint32_t dev_caps_cfg; - HWRM_PREP(req, VER_GET, -1, resp); + bp->max_req_len = HWRM_MAX_REQ_LEN; + HWRM_PREP(req, VER_GET); req.hwrm_intf_maj = HWRM_VERSION_MAJOR; req.hwrm_intf_min = HWRM_VERSION_MINOR; req.hwrm_intf_upd = HWRM_VERSION_UPDATE; - /* - * Hold the lock since we may be adjusting the response pointers. - */ - rte_spinlock_lock(&bp->hwrm_lock); - rc = bnxt_hwrm_send_message_locked(bp, &req, sizeof(req)); + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); - HWRM_CHECK_RESULT; + HWRM_CHECK_RESULT(); RTE_LOG(INFO, PMD, "%d.%d.%d:%d.%d.%d\n", resp->hwrm_intf_maj, resp->hwrm_intf_min, resp->hwrm_intf_upd, resp->hwrm_fw_maj, resp->hwrm_fw_min, resp->hwrm_fw_bld); + bp->fw_ver = (resp->hwrm_fw_maj << 24) | (resp->hwrm_fw_min << 16) | + (resp->hwrm_fw_bld << 8) | resp->hwrm_fw_rsvd; + RTE_LOG(INFO, PMD, "Driver HWRM version: %d.%d.%d\n", + HWRM_VERSION_MAJOR, HWRM_VERSION_MINOR, HWRM_VERSION_UPDATE); my_version = HWRM_VERSION_MAJOR << 16; my_version |= HWRM_VERSION_MINOR << 8; @@ -420,8 +626,10 @@ int bnxt_hwrm_ver_get(struct bnxt *bp) RTE_LOG(ERR, PMD, "Unsupported request length\n"); rc = -EINVAL; } - bp->max_req_len = resp->max_req_win_len; + bp->max_req_len = rte_le_to_cpu_16(resp->max_req_win_len); max_resp_len = resp->max_resp_len; + dev_caps_cfg = rte_le_to_cpu_32(resp->dev_caps_cfg); + if (bp->max_resp_len != max_resp_len) { sprintf(type, "bnxt_hwrm_%04x:%02x:%02x:%02x", bp->pdev->addr.domain, bp->pdev->addr.bus, @@ -434,13 +642,48 @@ int bnxt_hwrm_ver_get(struct bnxt *bp) rc = -ENOMEM; goto error; } + rte_mem_lock_page(bp->hwrm_cmd_resp_addr); bp->hwrm_cmd_resp_dma_addr = - rte_malloc_virt2phy(bp->hwrm_cmd_resp_addr); + rte_mem_virt2iova(bp->hwrm_cmd_resp_addr); + if (bp->hwrm_cmd_resp_dma_addr == 0) { + RTE_LOG(ERR, PMD, + "Unable to map response buffer to physical memory.\n"); + rc = -ENOMEM; + goto error; + } bp->max_resp_len = max_resp_len; } + if ((dev_caps_cfg & + HWRM_VER_GET_OUTPUT_DEV_CAPS_CFG_SHORT_CMD_SUPPORTED) && + (dev_caps_cfg & + HWRM_VER_GET_OUTPUT_DEV_CAPS_CFG_SHORT_CMD_INPUTUIRED)) { + RTE_LOG(DEBUG, PMD, "Short command supported\n"); + + rte_free(bp->hwrm_short_cmd_req_addr); + + bp->hwrm_short_cmd_req_addr = rte_malloc(type, + bp->max_req_len, 0); + if (bp->hwrm_short_cmd_req_addr == NULL) { + rc = -ENOMEM; + goto error; + } + rte_mem_lock_page(bp->hwrm_short_cmd_req_addr); + bp->hwrm_short_cmd_req_dma_addr = + rte_mem_virt2iova(bp->hwrm_short_cmd_req_addr); + if (bp->hwrm_short_cmd_req_dma_addr == 0) { + rte_free(bp->hwrm_short_cmd_req_addr); + RTE_LOG(ERR, PMD, + "Unable to map buffer to physical memory.\n"); + rc = -ENOMEM; + goto error; + } + + bp->flags |= BNXT_FLAG_SHORT_CMD; + } + error: - rte_spinlock_unlock(&bp->hwrm_lock); + HWRM_UNLOCK(); return rc; } @@ -453,12 +696,13 @@ int bnxt_hwrm_func_driver_unregister(struct bnxt *bp, uint32_t flags) if (!(bp->flags & BNXT_FLAG_REGISTERED)) return 0; - HWRM_PREP(req, FUNC_DRV_UNRGTR, -1, resp); + HWRM_PREP(req, FUNC_DRV_UNRGTR); req.flags = flags; rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); - HWRM_CHECK_RESULT; + HWRM_CHECK_RESULT(); + HWRM_UNLOCK(); bp->flags &= ~BNXT_FLAG_REGISTERED; @@ -468,54 +712,63 @@ int bnxt_hwrm_func_driver_unregister(struct bnxt *bp, uint32_t flags) static int bnxt_hwrm_port_phy_cfg(struct bnxt *bp, struct bnxt_link_info *conf) { int rc = 0; - struct hwrm_port_phy_cfg_input req = {.req_type = 0}; + struct hwrm_port_phy_cfg_input req = {0}; struct hwrm_port_phy_cfg_output *resp = bp->hwrm_cmd_resp_addr; + uint32_t enables = 0; - HWRM_PREP(req, PORT_PHY_CFG, -1, resp); + HWRM_PREP(req, PORT_PHY_CFG); - req.flags = conf->phy_flags; if (conf->link_up) { - req.force_link_speed = conf->link_speed; + /* Setting Fixed Speed. But AutoNeg is ON, So disable it */ + if (bp->link_info.auto_mode && conf->link_speed) { + req.auto_mode = HWRM_PORT_PHY_CFG_INPUT_AUTO_MODE_NONE; + RTE_LOG(DEBUG, PMD, "Disabling AutoNeg\n"); + } + + req.flags = rte_cpu_to_le_32(conf->phy_flags); + req.force_link_speed = rte_cpu_to_le_16(conf->link_speed); + enables |= HWRM_PORT_PHY_CFG_INPUT_ENABLES_AUTO_MODE; /* * Note, ChiMP FW 20.2.1 and 20.2.2 return an error when we set * any auto mode, even "none". */ - if (req.auto_mode == HWRM_PORT_PHY_CFG_INPUT_AUTO_MODE_NONE) { - req.flags |= HWRM_PORT_PHY_CFG_INPUT_FLAGS_FORCE; - } else { - req.auto_mode = conf->auto_mode; - req.enables |= - HWRM_PORT_PHY_CFG_INPUT_ENABLES_AUTO_MODE; - req.auto_link_speed_mask = conf->auto_link_speed_mask; - req.enables |= - HWRM_PORT_PHY_CFG_INPUT_ENABLES_AUTO_LINK_SPEED_MASK; - req.auto_link_speed = conf->auto_link_speed; - req.enables |= - HWRM_PORT_PHY_CFG_INPUT_ENABLES_AUTO_LINK_SPEED; + if (!conf->link_speed) { + /* No speeds specified. Enable AutoNeg - all speeds */ + req.auto_mode = + HWRM_PORT_PHY_CFG_INPUT_AUTO_MODE_ALL_SPEEDS; } + /* AutoNeg - Advertise speeds specified. */ + if (conf->auto_link_speed_mask && + !(conf->phy_flags & HWRM_PORT_PHY_CFG_INPUT_FLAGS_FORCE)) { + req.auto_mode = + HWRM_PORT_PHY_CFG_INPUT_AUTO_MODE_SPEED_MASK; + req.auto_link_speed_mask = + conf->auto_link_speed_mask; + enables |= + HWRM_PORT_PHY_CFG_INPUT_ENABLES_AUTO_LINK_SPEED_MASK; + } + req.auto_duplex = conf->duplex; - req.enables |= HWRM_PORT_PHY_CFG_INPUT_ENABLES_AUTO_DUPLEX; + enables |= HWRM_PORT_PHY_CFG_INPUT_ENABLES_AUTO_DUPLEX; req.auto_pause = conf->auto_pause; - /* Set force_pause if there is no auto or if there is a force */ - if (req.auto_pause) - req.enables |= - HWRM_PORT_PHY_CFG_INPUT_ENABLES_AUTO_PAUSE; - else - req.enables |= - HWRM_PORT_PHY_CFG_INPUT_ENABLES_FORCE_PAUSE; req.force_pause = conf->force_pause; - if (req.force_pause) - req.enables |= - HWRM_PORT_PHY_CFG_INPUT_ENABLES_FORCE_PAUSE; + /* Set force_pause if there is no auto or if there is a force */ + if (req.auto_pause && !req.force_pause) + enables |= HWRM_PORT_PHY_CFG_INPUT_ENABLES_AUTO_PAUSE; + else + enables |= HWRM_PORT_PHY_CFG_INPUT_ENABLES_FORCE_PAUSE; + + req.enables = rte_cpu_to_le_32(enables); } else { - req.flags &= ~HWRM_PORT_PHY_CFG_INPUT_FLAGS_RESTART_AUTONEG; - req.flags |= HWRM_PORT_PHY_CFG_INPUT_FLAGS_FORCE_LINK_DOWN; - req.force_link_speed = 0; + req.flags = + rte_cpu_to_le_32(HWRM_PORT_PHY_CFG_INPUT_FLAGS_FORCE_LINK_DWN); + RTE_LOG(INFO, PMD, "Force Link Down\n"); } rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); - HWRM_CHECK_RESULT; + HWRM_CHECK_RESULT(); + HWRM_UNLOCK(); return rc; } @@ -524,36 +777,47 @@ static int bnxt_hwrm_port_phy_qcfg(struct bnxt *bp, struct bnxt_link_info *link_info) { int rc = 0; - struct hwrm_port_phy_qcfg_input req = {.req_type = 0}; + struct hwrm_port_phy_qcfg_input req = {0}; struct hwrm_port_phy_qcfg_output *resp = bp->hwrm_cmd_resp_addr; - HWRM_PREP(req, PORT_PHY_QCFG, -1, resp); + HWRM_PREP(req, PORT_PHY_QCFG); rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); - HWRM_CHECK_RESULT; + HWRM_CHECK_RESULT(); link_info->phy_link_status = resp->link; - if (link_info->phy_link_status == HWRM_PORT_PHY_QCFG_OUTPUT_LINK_LINK) { - link_info->link_up = 1; - link_info->link_speed = rte_le_to_cpu_16(resp->link_speed); - } else { - link_info->link_up = 0; - link_info->link_speed = 0; - } - link_info->duplex = resp->duplex; + link_info->link_up = + (link_info->phy_link_status == + HWRM_PORT_PHY_QCFG_OUTPUT_LINK_LINK) ? 1 : 0; + link_info->link_speed = rte_le_to_cpu_16(resp->link_speed); + link_info->duplex = resp->duplex_cfg; link_info->pause = resp->pause; link_info->auto_pause = resp->auto_pause; link_info->force_pause = resp->force_pause; link_info->auto_mode = resp->auto_mode; + link_info->phy_type = resp->phy_type; + link_info->media_type = resp->media_type; link_info->support_speeds = rte_le_to_cpu_16(resp->support_speeds); link_info->auto_link_speed = rte_le_to_cpu_16(resp->auto_link_speed); link_info->preemphasis = rte_le_to_cpu_32(resp->preemphasis); + link_info->force_link_speed = rte_le_to_cpu_16(resp->force_link_speed); link_info->phy_ver[0] = resp->phy_maj; link_info->phy_ver[1] = resp->phy_min; link_info->phy_ver[2] = resp->phy_bld; + HWRM_UNLOCK(); + + RTE_LOG(DEBUG, PMD, "Link Speed %d\n", link_info->link_speed); + RTE_LOG(DEBUG, PMD, "Auto Mode %d\n", link_info->auto_mode); + RTE_LOG(DEBUG, PMD, "Support Speeds %x\n", link_info->support_speeds); + RTE_LOG(DEBUG, PMD, "Auto Link Speed %x\n", link_info->auto_link_speed); + RTE_LOG(DEBUG, PMD, "Auto Link Speed Mask %x\n", + link_info->auto_link_speed_mask); + RTE_LOG(DEBUG, PMD, "Forced Link Speed %x\n", + link_info->force_link_speed); + return rc; } @@ -563,11 +827,11 @@ int bnxt_hwrm_queue_qportcfg(struct bnxt *bp) struct hwrm_queue_qportcfg_input req = {.req_type = 0 }; struct hwrm_queue_qportcfg_output *resp = bp->hwrm_cmd_resp_addr; - HWRM_PREP(req, QUEUE_QPORTCFG, -1, resp); + HWRM_PREP(req, QUEUE_QPORTCFG); rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); - HWRM_CHECK_RESULT; + HWRM_CHECK_RESULT(); #define GET_QUEUE_INFO(x) \ bp->cos_queue[x].id = resp->queue_id##x; \ @@ -582,53 +846,56 @@ int bnxt_hwrm_queue_qportcfg(struct bnxt *bp) GET_QUEUE_INFO(6); GET_QUEUE_INFO(7); + HWRM_UNLOCK(); + return rc; } int bnxt_hwrm_ring_alloc(struct bnxt *bp, struct bnxt_ring *ring, uint32_t ring_type, uint32_t map_index, - uint32_t stats_ctx_id) + uint32_t stats_ctx_id, uint32_t cmpl_ring_id) { int rc = 0; + uint32_t enables = 0; struct hwrm_ring_alloc_input req = {.req_type = 0 }; struct hwrm_ring_alloc_output *resp = bp->hwrm_cmd_resp_addr; - HWRM_PREP(req, RING_ALLOC, -1, resp); - - req.enables = rte_cpu_to_le_32(0); + HWRM_PREP(req, RING_ALLOC); req.page_tbl_addr = rte_cpu_to_le_64(ring->bd_dma); req.fbo = rte_cpu_to_le_32(0); /* Association of ring index with doorbell index */ req.logical_id = rte_cpu_to_le_16(map_index); + req.length = rte_cpu_to_le_32(ring->ring_size); switch (ring_type) { case HWRM_RING_ALLOC_INPUT_RING_TYPE_TX: req.queue_id = bp->cos_queue[0].id; + /* FALLTHROUGH */ case HWRM_RING_ALLOC_INPUT_RING_TYPE_RX: req.ring_type = ring_type; - req.cmpl_ring_id = - rte_cpu_to_le_16(bp->grp_info[map_index].cp_fw_ring_id); - req.length = rte_cpu_to_le_32(ring->ring_size); + req.cmpl_ring_id = rte_cpu_to_le_16(cmpl_ring_id); req.stat_ctx_id = rte_cpu_to_le_16(stats_ctx_id); - req.enables = rte_cpu_to_le_32(rte_le_to_cpu_32(req.enables) | - HWRM_RING_ALLOC_INPUT_ENABLES_STAT_CTX_ID_VALID); + if (stats_ctx_id != INVALID_STATS_CTX_ID) + enables |= + HWRM_RING_ALLOC_INPUT_ENABLES_STAT_CTX_ID_VALID; break; - case HWRM_RING_ALLOC_INPUT_RING_TYPE_CMPL: + case HWRM_RING_ALLOC_INPUT_RING_TYPE_L2_CMPL: req.ring_type = ring_type; /* * TODO: Some HWRM versions crash with * HWRM_RING_ALLOC_INPUT_INT_MODE_POLL */ req.int_mode = HWRM_RING_ALLOC_INPUT_INT_MODE_MSIX; - req.length = rte_cpu_to_le_32(ring->ring_size); break; default: RTE_LOG(ERR, PMD, "hwrm alloc invalid ring type %d\n", ring_type); + HWRM_UNLOCK(); return -1; } + req.enables = rte_cpu_to_le_32(enables); rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); @@ -636,25 +903,30 @@ int bnxt_hwrm_ring_alloc(struct bnxt *bp, if (rc == 0 && resp->error_code) rc = rte_le_to_cpu_16(resp->error_code); switch (ring_type) { - case HWRM_RING_FREE_INPUT_RING_TYPE_CMPL: + case HWRM_RING_FREE_INPUT_RING_TYPE_L2_CMPL: RTE_LOG(ERR, PMD, "hwrm_ring_alloc cp failed. rc:%d\n", rc); + HWRM_UNLOCK(); return rc; case HWRM_RING_FREE_INPUT_RING_TYPE_RX: RTE_LOG(ERR, PMD, "hwrm_ring_alloc rx failed. rc:%d\n", rc); + HWRM_UNLOCK(); return rc; case HWRM_RING_FREE_INPUT_RING_TYPE_TX: RTE_LOG(ERR, PMD, "hwrm_ring_alloc tx failed. rc:%d\n", rc); + HWRM_UNLOCK(); return rc; default: RTE_LOG(ERR, PMD, "Invalid ring. rc:%d\n", rc); + HWRM_UNLOCK(); return rc; } } ring->fw_ring_id = rte_le_to_cpu_16(resp->ring_id); + HWRM_UNLOCK(); return rc; } @@ -665,7 +937,7 @@ int bnxt_hwrm_ring_free(struct bnxt *bp, struct hwrm_ring_free_input req = {.req_type = 0 }; struct hwrm_ring_free_output *resp = bp->hwrm_cmd_resp_addr; - HWRM_PREP(req, RING_FREE, -1, resp); + HWRM_PREP(req, RING_FREE); req.ring_type = ring_type; req.ring_id = rte_cpu_to_le_16(ring->fw_ring_id); @@ -675,9 +947,10 @@ int bnxt_hwrm_ring_free(struct bnxt *bp, if (rc || resp->error_code) { if (rc == 0 && resp->error_code) rc = rte_le_to_cpu_16(resp->error_code); + HWRM_UNLOCK(); switch (ring_type) { - case HWRM_RING_FREE_INPUT_RING_TYPE_CMPL: + case HWRM_RING_FREE_INPUT_RING_TYPE_L2_CMPL: RTE_LOG(ERR, PMD, "hwrm_ring_free cp failed. rc:%d\n", rc); return rc; @@ -694,6 +967,7 @@ int bnxt_hwrm_ring_free(struct bnxt *bp, return rc; } } + HWRM_UNLOCK(); return 0; } @@ -703,7 +977,7 @@ int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp, unsigned int idx) struct hwrm_ring_grp_alloc_input req = {.req_type = 0 }; struct hwrm_ring_grp_alloc_output *resp = bp->hwrm_cmd_resp_addr; - HWRM_PREP(req, RING_GRP_ALLOC, -1, resp); + HWRM_PREP(req, RING_GRP_ALLOC); req.cr = rte_cpu_to_le_16(bp->grp_info[idx].cp_fw_ring_id); req.rr = rte_cpu_to_le_16(bp->grp_info[idx].rx_fw_ring_id); @@ -712,11 +986,13 @@ int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp, unsigned int idx) rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); - HWRM_CHECK_RESULT; + HWRM_CHECK_RESULT(); bp->grp_info[idx].fw_grp_id = rte_le_to_cpu_16(resp->ring_group_id); + HWRM_UNLOCK(); + return rc; } @@ -726,13 +1002,14 @@ int bnxt_hwrm_ring_grp_free(struct bnxt *bp, unsigned int idx) struct hwrm_ring_grp_free_input req = {.req_type = 0 }; struct hwrm_ring_grp_free_output *resp = bp->hwrm_cmd_resp_addr; - HWRM_PREP(req, RING_GRP_FREE, -1, resp); + HWRM_PREP(req, RING_GRP_FREE); req.ring_group_id = rte_cpu_to_le_16(bp->grp_info[idx].fw_grp_id); rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); - HWRM_CHECK_RESULT; + HWRM_CHECK_RESULT(); + HWRM_UNLOCK(); bp->grp_info[idx].fw_grp_id = INVALID_HW_RING_ID; return rc; @@ -744,64 +1021,61 @@ int bnxt_hwrm_stat_clear(struct bnxt *bp, struct bnxt_cp_ring_info *cpr) struct hwrm_stat_ctx_clr_stats_input req = {.req_type = 0 }; struct hwrm_stat_ctx_clr_stats_output *resp = bp->hwrm_cmd_resp_addr; - HWRM_PREP(req, STAT_CTX_CLR_STATS, -1, resp); - if (cpr->hw_stats_ctx_id == (uint32_t)HWRM_NA_SIGNATURE) return rc; + HWRM_PREP(req, STAT_CTX_CLR_STATS); + req.stat_ctx_id = rte_cpu_to_le_16(cpr->hw_stats_ctx_id); - req.seq_id = rte_cpu_to_le_16(bp->hwrm_cmd_seq++); rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); - HWRM_CHECK_RESULT; + HWRM_CHECK_RESULT(); + HWRM_UNLOCK(); return rc; } -int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp, - struct bnxt_cp_ring_info *cpr, unsigned int idx) +int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, + unsigned int idx __rte_unused) { int rc; struct hwrm_stat_ctx_alloc_input req = {.req_type = 0 }; struct hwrm_stat_ctx_alloc_output *resp = bp->hwrm_cmd_resp_addr; - HWRM_PREP(req, STAT_CTX_ALLOC, -1, resp); + HWRM_PREP(req, STAT_CTX_ALLOC); - req.update_period_ms = rte_cpu_to_le_32(1000); + req.update_period_ms = rte_cpu_to_le_32(0); - req.seq_id = rte_cpu_to_le_16(bp->hwrm_cmd_seq++); req.stats_dma_addr = rte_cpu_to_le_64(cpr->hw_stats_map); rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); - HWRM_CHECK_RESULT; + HWRM_CHECK_RESULT(); cpr->hw_stats_ctx_id = rte_le_to_cpu_16(resp->stat_ctx_id); - bp->grp_info[idx].fw_stats_ctx = cpr->hw_stats_ctx_id; + + HWRM_UNLOCK(); return rc; } -int bnxt_hwrm_stat_ctx_free(struct bnxt *bp, - struct bnxt_cp_ring_info *cpr, unsigned int idx) +int bnxt_hwrm_stat_ctx_free(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, + unsigned int idx __rte_unused) { int rc; struct hwrm_stat_ctx_free_input req = {.req_type = 0 }; struct hwrm_stat_ctx_free_output *resp = bp->hwrm_cmd_resp_addr; - HWRM_PREP(req, STAT_CTX_FREE, -1, resp); + HWRM_PREP(req, STAT_CTX_FREE); req.stat_ctx_id = rte_cpu_to_le_16(cpr->hw_stats_ctx_id); - req.seq_id = rte_cpu_to_le_16(bp->hwrm_cmd_seq++); rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); - HWRM_CHECK_RESULT; - - cpr->hw_stats_ctx_id = HWRM_NA_SIGNATURE; - bp->grp_info[idx].fw_stats_ctx = cpr->hw_stats_ctx_id; + HWRM_CHECK_RESULT(); + HWRM_UNLOCK(); return rc; } @@ -809,30 +1083,88 @@ int bnxt_hwrm_stat_ctx_free(struct bnxt *bp, int bnxt_hwrm_vnic_alloc(struct bnxt *bp, struct bnxt_vnic_info *vnic) { int rc = 0, i, j; - struct hwrm_vnic_alloc_input req = {.req_type = 0 }; + struct hwrm_vnic_alloc_input req = { 0 }; struct hwrm_vnic_alloc_output *resp = bp->hwrm_cmd_resp_addr; /* map ring groups to this vnic */ - for (i = vnic->start_grp_id, j = 0; i <= vnic->end_grp_id; i++, j++) { - if (bp->grp_info[i].fw_grp_id == (uint16_t)HWRM_NA_SIGNATURE) { - RTE_LOG(ERR, PMD, - "Not enough ring groups avail:%x req:%x\n", j, - (vnic->end_grp_id - vnic->start_grp_id) + 1); - break; - } + RTE_LOG(DEBUG, PMD, "Alloc VNIC. Start %x, End %x\n", + vnic->start_grp_id, vnic->end_grp_id); + for (i = vnic->start_grp_id, j = 0; i <= vnic->end_grp_id; i++, j++) vnic->fw_grp_ids[j] = bp->grp_info[i].fw_grp_id; - } + vnic->dflt_ring_grp = bp->grp_info[vnic->start_grp_id].fw_grp_id; + vnic->rss_rule = (uint16_t)HWRM_NA_SIGNATURE; + vnic->cos_rule = (uint16_t)HWRM_NA_SIGNATURE; + vnic->lb_rule = (uint16_t)HWRM_NA_SIGNATURE; + vnic->mru = bp->eth_dev->data->mtu + ETHER_HDR_LEN + + ETHER_CRC_LEN + VLAN_TAG_SIZE; + HWRM_PREP(req, VNIC_ALLOC); - vnic->fw_rss_cos_lb_ctx = (uint16_t)HWRM_NA_SIGNATURE; - vnic->ctx_is_rss_cos_lb = HW_CONTEXT_NONE; + if (vnic->func_default) + req.flags = HWRM_VNIC_ALLOC_INPUT_FLAGS_DEFAULT; + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); - HWRM_PREP(req, VNIC_ALLOC, -1, resp); + HWRM_CHECK_RESULT(); + + vnic->fw_vnic_id = rte_le_to_cpu_16(resp->vnic_id); + HWRM_UNLOCK(); + RTE_LOG(DEBUG, PMD, "VNIC ID %x\n", vnic->fw_vnic_id); + return rc; +} + +static int bnxt_hwrm_vnic_plcmodes_qcfg(struct bnxt *bp, + struct bnxt_vnic_info *vnic, + struct bnxt_plcmodes_cfg *pmode) +{ + int rc = 0; + struct hwrm_vnic_plcmodes_qcfg_input req = {.req_type = 0 }; + struct hwrm_vnic_plcmodes_qcfg_output *resp = bp->hwrm_cmd_resp_addr; + + HWRM_PREP(req, VNIC_PLCMODES_QCFG); + + req.vnic_id = rte_cpu_to_le_32(vnic->fw_vnic_id); rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); - HWRM_CHECK_RESULT; + HWRM_CHECK_RESULT(); + + pmode->flags = rte_le_to_cpu_32(resp->flags); + /* dflt_vnic bit doesn't exist in the _cfg command */ + pmode->flags &= ~(HWRM_VNIC_PLCMODES_QCFG_OUTPUT_FLAGS_DFLT_VNIC); + pmode->jumbo_thresh = rte_le_to_cpu_16(resp->jumbo_thresh); + pmode->hds_offset = rte_le_to_cpu_16(resp->hds_offset); + pmode->hds_threshold = rte_le_to_cpu_16(resp->hds_threshold); + + HWRM_UNLOCK(); + + return rc; +} + +static int bnxt_hwrm_vnic_plcmodes_cfg(struct bnxt *bp, + struct bnxt_vnic_info *vnic, + struct bnxt_plcmodes_cfg *pmode) +{ + int rc = 0; + struct hwrm_vnic_plcmodes_cfg_input req = {.req_type = 0 }; + struct hwrm_vnic_plcmodes_cfg_output *resp = bp->hwrm_cmd_resp_addr; + + HWRM_PREP(req, VNIC_PLCMODES_CFG); + + req.vnic_id = rte_cpu_to_le_32(vnic->fw_vnic_id); + req.flags = rte_cpu_to_le_32(pmode->flags); + req.jumbo_thresh = rte_cpu_to_le_16(pmode->jumbo_thresh); + req.hds_offset = rte_cpu_to_le_16(pmode->hds_offset); + req.hds_threshold = rte_cpu_to_le_16(pmode->hds_threshold); + req.enables = rte_cpu_to_le_32( + HWRM_VNIC_PLCMODES_CFG_INPUT_ENABLES_HDS_THRESHOLD_VALID | + HWRM_VNIC_PLCMODES_CFG_INPUT_ENABLES_HDS_OFFSET_VALID | + HWRM_VNIC_PLCMODES_CFG_INPUT_ENABLES_JUMBO_THRESH_VALID + ); + + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + + HWRM_CHECK_RESULT(); + HWRM_UNLOCK(); - vnic->fw_vnic_id = rte_le_to_cpu_16(resp->vnic_id); return rc; } @@ -841,31 +1173,108 @@ int bnxt_hwrm_vnic_cfg(struct bnxt *bp, struct bnxt_vnic_info *vnic) int rc = 0; struct hwrm_vnic_cfg_input req = {.req_type = 0 }; struct hwrm_vnic_cfg_output *resp = bp->hwrm_cmd_resp_addr; + uint32_t ctx_enable_flag = 0; + struct bnxt_plcmodes_cfg pmodes; - HWRM_PREP(req, VNIC_CFG, -1, resp); + if (vnic->fw_vnic_id == INVALID_HW_RING_ID) { + RTE_LOG(DEBUG, PMD, "VNIC ID %x\n", vnic->fw_vnic_id); + return rc; + } + + rc = bnxt_hwrm_vnic_plcmodes_qcfg(bp, vnic, &pmodes); + if (rc) + return rc; + + HWRM_PREP(req, VNIC_CFG); /* Only RSS support for now TBD: COS & LB */ req.enables = - rte_cpu_to_le_32(HWRM_VNIC_CFG_INPUT_ENABLES_DFLT_RING_GRP | - HWRM_VNIC_CFG_INPUT_ENABLES_RSS_RULE | - HWRM_VNIC_CFG_INPUT_ENABLES_MRU); + rte_cpu_to_le_32(HWRM_VNIC_CFG_INPUT_ENABLES_DFLT_RING_GRP); + if (vnic->lb_rule != 0xffff) + ctx_enable_flag |= HWRM_VNIC_CFG_INPUT_ENABLES_LB_RULE; + if (vnic->cos_rule != 0xffff) + ctx_enable_flag |= HWRM_VNIC_CFG_INPUT_ENABLES_COS_RULE; + if (vnic->rss_rule != 0xffff) { + ctx_enable_flag |= HWRM_VNIC_CFG_INPUT_ENABLES_MRU; + ctx_enable_flag |= HWRM_VNIC_CFG_INPUT_ENABLES_RSS_RULE; + } + req.enables |= rte_cpu_to_le_32(ctx_enable_flag); req.vnic_id = rte_cpu_to_le_16(vnic->fw_vnic_id); - req.dflt_ring_grp = - rte_cpu_to_le_16(bp->grp_info[vnic->start_grp_id].fw_grp_id); - req.rss_rule = rte_cpu_to_le_16(vnic->fw_rss_cos_lb_ctx); - req.cos_rule = rte_cpu_to_le_16(0xffff); - req.lb_rule = rte_cpu_to_le_16(0xffff); - req.mru = rte_cpu_to_le_16(bp->eth_dev->data->mtu + ETHER_HDR_LEN + - ETHER_CRC_LEN + VLAN_TAG_SIZE); + req.dflt_ring_grp = rte_cpu_to_le_16(vnic->dflt_ring_grp); + req.rss_rule = rte_cpu_to_le_16(vnic->rss_rule); + req.cos_rule = rte_cpu_to_le_16(vnic->cos_rule); + req.lb_rule = rte_cpu_to_le_16(vnic->lb_rule); + req.mru = rte_cpu_to_le_16(vnic->mru); if (vnic->func_default) - req.flags = 1; + req.flags |= + rte_cpu_to_le_32(HWRM_VNIC_CFG_INPUT_FLAGS_DEFAULT); if (vnic->vlan_strip) req.flags |= rte_cpu_to_le_32(HWRM_VNIC_CFG_INPUT_FLAGS_VLAN_STRIP_MODE); + if (vnic->bd_stall) + req.flags |= + rte_cpu_to_le_32(HWRM_VNIC_CFG_INPUT_FLAGS_BD_STALL_MODE); + if (vnic->roce_dual) + req.flags |= rte_cpu_to_le_32( + HWRM_VNIC_QCFG_OUTPUT_FLAGS_ROCE_DUAL_VNIC_MODE); + if (vnic->roce_only) + req.flags |= rte_cpu_to_le_32( + HWRM_VNIC_QCFG_OUTPUT_FLAGS_ROCE_ONLY_VNIC_MODE); + if (vnic->rss_dflt_cr) + req.flags |= rte_cpu_to_le_32( + HWRM_VNIC_QCFG_OUTPUT_FLAGS_RSS_DFLT_CR_MODE); rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); - HWRM_CHECK_RESULT; + HWRM_CHECK_RESULT(); + HWRM_UNLOCK(); + + rc = bnxt_hwrm_vnic_plcmodes_cfg(bp, vnic, &pmodes); + + return rc; +} + +int bnxt_hwrm_vnic_qcfg(struct bnxt *bp, struct bnxt_vnic_info *vnic, + int16_t fw_vf_id) +{ + int rc = 0; + struct hwrm_vnic_qcfg_input req = {.req_type = 0 }; + struct hwrm_vnic_qcfg_output *resp = bp->hwrm_cmd_resp_addr; + + if (vnic->fw_vnic_id == INVALID_HW_RING_ID) { + RTE_LOG(DEBUG, PMD, "VNIC QCFG ID %d\n", vnic->fw_vnic_id); + return rc; + } + HWRM_PREP(req, VNIC_QCFG); + + req.enables = + rte_cpu_to_le_32(HWRM_VNIC_QCFG_INPUT_ENABLES_VF_ID_VALID); + req.vnic_id = rte_cpu_to_le_16(vnic->fw_vnic_id); + req.vf_id = rte_cpu_to_le_16(fw_vf_id); + + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + + HWRM_CHECK_RESULT(); + + vnic->dflt_ring_grp = rte_le_to_cpu_16(resp->dflt_ring_grp); + vnic->rss_rule = rte_le_to_cpu_16(resp->rss_rule); + vnic->cos_rule = rte_le_to_cpu_16(resp->cos_rule); + vnic->lb_rule = rte_le_to_cpu_16(resp->lb_rule); + vnic->mru = rte_le_to_cpu_16(resp->mru); + vnic->func_default = rte_le_to_cpu_32( + resp->flags) & HWRM_VNIC_QCFG_OUTPUT_FLAGS_DEFAULT; + vnic->vlan_strip = rte_le_to_cpu_32(resp->flags) & + HWRM_VNIC_QCFG_OUTPUT_FLAGS_VLAN_STRIP_MODE; + vnic->bd_stall = rte_le_to_cpu_32(resp->flags) & + HWRM_VNIC_QCFG_OUTPUT_FLAGS_BD_STALL_MODE; + vnic->roce_dual = rte_le_to_cpu_32(resp->flags) & + HWRM_VNIC_QCFG_OUTPUT_FLAGS_ROCE_DUAL_VNIC_MODE; + vnic->roce_only = rte_le_to_cpu_32(resp->flags) & + HWRM_VNIC_QCFG_OUTPUT_FLAGS_ROCE_ONLY_VNIC_MODE; + vnic->rss_dflt_cr = rte_le_to_cpu_32(resp->flags) & + HWRM_VNIC_QCFG_OUTPUT_FLAGS_RSS_DFLT_CR_MODE; + + HWRM_UNLOCK(); return rc; } @@ -877,13 +1286,15 @@ int bnxt_hwrm_vnic_ctx_alloc(struct bnxt *bp, struct bnxt_vnic_info *vnic) struct hwrm_vnic_rss_cos_lb_ctx_alloc_output *resp = bp->hwrm_cmd_resp_addr; - HWRM_PREP(req, VNIC_RSS_COS_LB_CTX_ALLOC, -1, resp); + HWRM_PREP(req, VNIC_RSS_COS_LB_CTX_ALLOC); rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); - HWRM_CHECK_RESULT; + HWRM_CHECK_RESULT(); - vnic->fw_rss_cos_lb_ctx = rte_le_to_cpu_16(resp->rss_cos_lb_ctx_id); + vnic->rss_rule = rte_le_to_cpu_16(resp->rss_cos_lb_ctx_id); + HWRM_UNLOCK(); + RTE_LOG(DEBUG, PMD, "VNIC RSS Rule %x\n", vnic->rss_rule); return rc; } @@ -895,15 +1306,20 @@ int bnxt_hwrm_vnic_ctx_free(struct bnxt *bp, struct bnxt_vnic_info *vnic) struct hwrm_vnic_rss_cos_lb_ctx_free_output *resp = bp->hwrm_cmd_resp_addr; - HWRM_PREP(req, VNIC_RSS_COS_LB_CTX_FREE, -1, resp); + if (vnic->rss_rule == 0xffff) { + RTE_LOG(DEBUG, PMD, "VNIC RSS Rule %x\n", vnic->rss_rule); + return rc; + } + HWRM_PREP(req, VNIC_RSS_COS_LB_CTX_FREE); - req.rss_cos_lb_ctx_id = rte_cpu_to_le_16(vnic->fw_rss_cos_lb_ctx); + req.rss_cos_lb_ctx_id = rte_cpu_to_le_16(vnic->rss_rule); rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); - HWRM_CHECK_RESULT; + HWRM_CHECK_RESULT(); + HWRM_UNLOCK(); - vnic->fw_rss_cos_lb_ctx = INVALID_HW_RING_ID; + vnic->rss_rule = INVALID_HW_RING_ID; return rc; } @@ -914,16 +1330,19 @@ int bnxt_hwrm_vnic_free(struct bnxt *bp, struct bnxt_vnic_info *vnic) struct hwrm_vnic_free_input req = {.req_type = 0 }; struct hwrm_vnic_free_output *resp = bp->hwrm_cmd_resp_addr; - if (vnic->fw_vnic_id == INVALID_HW_RING_ID) + if (vnic->fw_vnic_id == INVALID_HW_RING_ID) { + RTE_LOG(DEBUG, PMD, "VNIC FREE ID %x\n", vnic->fw_vnic_id); return rc; + } - HWRM_PREP(req, VNIC_FREE, -1, resp); + HWRM_PREP(req, VNIC_FREE); req.vnic_id = rte_cpu_to_le_16(vnic->fw_vnic_id); rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); - HWRM_CHECK_RESULT; + HWRM_CHECK_RESULT(); + HWRM_UNLOCK(); vnic->fw_vnic_id = INVALID_HW_RING_ID; return rc; @@ -936,7 +1355,7 @@ int bnxt_hwrm_vnic_rss_cfg(struct bnxt *bp, struct hwrm_vnic_rss_cfg_input req = {.req_type = 0 }; struct hwrm_vnic_rss_cfg_output *resp = bp->hwrm_cmd_resp_addr; - HWRM_PREP(req, VNIC_RSS_CFG, -1, resp); + HWRM_PREP(req, VNIC_RSS_CFG); req.hash_type = rte_cpu_to_le_32(vnic->hash_type); @@ -944,11 +1363,181 @@ int bnxt_hwrm_vnic_rss_cfg(struct bnxt *bp, rte_cpu_to_le_64(vnic->rss_table_dma_addr); req.hash_key_tbl_addr = rte_cpu_to_le_64(vnic->rss_hash_key_dma_addr); - req.rss_ctx_idx = rte_cpu_to_le_16(vnic->fw_rss_cos_lb_ctx); + req.rss_ctx_idx = rte_cpu_to_le_16(vnic->rss_rule); rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); - HWRM_CHECK_RESULT; + HWRM_CHECK_RESULT(); + HWRM_UNLOCK(); + + return rc; +} + +int bnxt_hwrm_vnic_plcmode_cfg(struct bnxt *bp, + struct bnxt_vnic_info *vnic) +{ + int rc = 0; + struct hwrm_vnic_plcmodes_cfg_input req = {.req_type = 0 }; + struct hwrm_vnic_plcmodes_cfg_output *resp = bp->hwrm_cmd_resp_addr; + uint16_t size; + + HWRM_PREP(req, VNIC_PLCMODES_CFG); + + req.flags = rte_cpu_to_le_32( + HWRM_VNIC_PLCMODES_CFG_INPUT_FLAGS_JUMBO_PLACEMENT); + + req.enables = rte_cpu_to_le_32( + HWRM_VNIC_PLCMODES_CFG_INPUT_ENABLES_JUMBO_THRESH_VALID); + + size = rte_pktmbuf_data_room_size(bp->rx_queues[0]->mb_pool); + size -= RTE_PKTMBUF_HEADROOM; + + req.jumbo_thresh = rte_cpu_to_le_16(size); + req.vnic_id = rte_cpu_to_le_32(vnic->fw_vnic_id); + + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + + HWRM_CHECK_RESULT(); + HWRM_UNLOCK(); + + return rc; +} + +int bnxt_hwrm_vnic_tpa_cfg(struct bnxt *bp, + struct bnxt_vnic_info *vnic, bool enable) +{ + int rc = 0; + struct hwrm_vnic_tpa_cfg_input req = {.req_type = 0 }; + struct hwrm_vnic_tpa_cfg_output *resp = bp->hwrm_cmd_resp_addr; + + HWRM_PREP(req, VNIC_TPA_CFG); + + if (enable) { + req.enables = rte_cpu_to_le_32( + HWRM_VNIC_TPA_CFG_INPUT_ENABLES_MAX_AGG_SEGS | + HWRM_VNIC_TPA_CFG_INPUT_ENABLES_MAX_AGGS | + HWRM_VNIC_TPA_CFG_INPUT_ENABLES_MIN_AGG_LEN); + req.flags = rte_cpu_to_le_32( + HWRM_VNIC_TPA_CFG_INPUT_FLAGS_TPA | + HWRM_VNIC_TPA_CFG_INPUT_FLAGS_ENCAP_TPA | + HWRM_VNIC_TPA_CFG_INPUT_FLAGS_RSC_WND_UPDATE | + HWRM_VNIC_TPA_CFG_INPUT_FLAGS_GRO | + HWRM_VNIC_TPA_CFG_INPUT_FLAGS_AGG_WITH_ECN | + HWRM_VNIC_TPA_CFG_INPUT_FLAGS_AGG_WITH_SAME_GRE_SEQ); + req.vnic_id = rte_cpu_to_le_32(vnic->fw_vnic_id); + req.max_agg_segs = rte_cpu_to_le_16(5); + req.max_aggs = + rte_cpu_to_le_16(HWRM_VNIC_TPA_CFG_INPUT_MAX_AGGS_MAX); + req.min_agg_len = rte_cpu_to_le_32(512); + } + + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + + HWRM_CHECK_RESULT(); + HWRM_UNLOCK(); + + return rc; +} + +int bnxt_hwrm_func_vf_mac(struct bnxt *bp, uint16_t vf, const uint8_t *mac_addr) +{ + struct hwrm_func_cfg_input req = {0}; + struct hwrm_func_cfg_output *resp = bp->hwrm_cmd_resp_addr; + int rc; + + req.flags = rte_cpu_to_le_32(bp->pf.vf_info[vf].func_cfg_flags); + req.enables = rte_cpu_to_le_32( + HWRM_FUNC_CFG_INPUT_ENABLES_DFLT_MAC_ADDR); + memcpy(req.dflt_mac_addr, mac_addr, sizeof(req.dflt_mac_addr)); + req.fid = rte_cpu_to_le_16(bp->pf.vf_info[vf].fid); + + HWRM_PREP(req, FUNC_CFG); + + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + HWRM_CHECK_RESULT(); + HWRM_UNLOCK(); + + bp->pf.vf_info[vf].random_mac = false; + + return rc; +} + +int bnxt_hwrm_func_qstats_tx_drop(struct bnxt *bp, uint16_t fid, + uint64_t *dropped) +{ + int rc = 0; + struct hwrm_func_qstats_input req = {.req_type = 0}; + struct hwrm_func_qstats_output *resp = bp->hwrm_cmd_resp_addr; + + HWRM_PREP(req, FUNC_QSTATS); + + req.fid = rte_cpu_to_le_16(fid); + + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + + HWRM_CHECK_RESULT(); + + if (dropped) + *dropped = rte_le_to_cpu_64(resp->tx_drop_pkts); + + HWRM_UNLOCK(); + + return rc; +} + +int bnxt_hwrm_func_qstats(struct bnxt *bp, uint16_t fid, + struct rte_eth_stats *stats) +{ + int rc = 0; + struct hwrm_func_qstats_input req = {.req_type = 0}; + struct hwrm_func_qstats_output *resp = bp->hwrm_cmd_resp_addr; + + HWRM_PREP(req, FUNC_QSTATS); + + req.fid = rte_cpu_to_le_16(fid); + + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + + HWRM_CHECK_RESULT(); + + stats->ipackets = rte_le_to_cpu_64(resp->rx_ucast_pkts); + stats->ipackets += rte_le_to_cpu_64(resp->rx_mcast_pkts); + stats->ipackets += rte_le_to_cpu_64(resp->rx_bcast_pkts); + stats->ibytes = rte_le_to_cpu_64(resp->rx_ucast_bytes); + stats->ibytes += rte_le_to_cpu_64(resp->rx_mcast_bytes); + stats->ibytes += rte_le_to_cpu_64(resp->rx_bcast_bytes); + + stats->opackets = rte_le_to_cpu_64(resp->tx_ucast_pkts); + stats->opackets += rte_le_to_cpu_64(resp->tx_mcast_pkts); + stats->opackets += rte_le_to_cpu_64(resp->tx_bcast_pkts); + stats->obytes = rte_le_to_cpu_64(resp->tx_ucast_bytes); + stats->obytes += rte_le_to_cpu_64(resp->tx_mcast_bytes); + stats->obytes += rte_le_to_cpu_64(resp->tx_bcast_bytes); + + stats->ierrors = rte_le_to_cpu_64(resp->rx_err_pkts); + stats->oerrors = rte_le_to_cpu_64(resp->tx_err_pkts); + + stats->imissed = rte_le_to_cpu_64(resp->rx_drop_pkts); + + HWRM_UNLOCK(); + + return rc; +} + +int bnxt_hwrm_func_clr_stats(struct bnxt *bp, uint16_t fid) +{ + int rc = 0; + struct hwrm_func_clr_stats_input req = {.req_type = 0}; + struct hwrm_func_clr_stats_output *resp = bp->hwrm_cmd_resp_addr; + + HWRM_PREP(req, FUNC_CLR_STATS); + + req.fid = rte_cpu_to_le_16(fid); + + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + + HWRM_CHECK_RESULT(); + HWRM_UNLOCK(); return rc; } @@ -989,14 +1578,16 @@ int bnxt_free_all_hwrm_stat_ctxs(struct bnxt *bp) struct bnxt_cp_ring_info *cpr; for (i = 0; i < bp->rx_cp_nr_rings + bp->tx_cp_nr_rings; i++) { - unsigned int idx = i + 1; - if (i >= bp->rx_cp_nr_rings) + if (i >= bp->rx_cp_nr_rings) { cpr = bp->tx_queues[i - bp->rx_cp_nr_rings]->cp_ring; - else + } else { cpr = bp->rx_queues[i]->cp_ring; + bp->grp_info[i].fw_stats_ctx = -1; + } if (cpr->hw_stats_ctx_id != HWRM_NA_SIGNATURE) { - rc = bnxt_hwrm_stat_ctx_free(bp, cpr, idx); + rc = bnxt_hwrm_stat_ctx_free(bp, cpr, i); + cpr->hw_stats_ctx_id = HWRM_NA_SIGNATURE; if (rc) return rc; } @@ -1013,7 +1604,6 @@ int bnxt_alloc_all_hwrm_stat_ctxs(struct bnxt *bp) struct bnxt_tx_queue *txq; struct bnxt_rx_queue *rxq; struct bnxt_cp_ring_info *cpr; - unsigned int idx = i + 1; if (i >= bp->rx_cp_nr_rings) { txq = bp->tx_queues[i - bp->rx_cp_nr_rings]; @@ -1023,7 +1613,7 @@ int bnxt_alloc_all_hwrm_stat_ctxs(struct bnxt *bp) cpr = rxq->cp_ring; } - rc = bnxt_hwrm_stat_ctx_alloc(bp, cpr, idx); + rc = bnxt_hwrm_stat_ctx_alloc(bp, cpr, i); if (rc) return rc; @@ -1033,18 +1623,13 @@ int bnxt_alloc_all_hwrm_stat_ctxs(struct bnxt *bp) int bnxt_free_all_hwrm_ring_grps(struct bnxt *bp) { - uint16_t i; + uint16_t idx; uint32_t rc = 0; - for (i = 0; i < bp->rx_cp_nr_rings; i++) { - unsigned int idx = i + 1; + for (idx = 0; idx < bp->rx_cp_nr_rings; idx++) { - if (bp->grp_info[idx].fw_grp_id == INVALID_HW_RING_ID) { - RTE_LOG(ERR, PMD, - "Attempt to free invalid ring group %d\n", - idx); + if (bp->grp_info[idx].fw_grp_id == INVALID_HW_RING_ID) continue; - } rc = bnxt_hwrm_ring_grp_free(bp, idx); @@ -1054,15 +1639,14 @@ int bnxt_free_all_hwrm_ring_grps(struct bnxt *bp) return rc; } -static void bnxt_free_cp_ring(struct bnxt *bp, - struct bnxt_cp_ring_info *cpr, unsigned int idx) +static void bnxt_free_cp_ring(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, + unsigned int idx __rte_unused) { struct bnxt_ring *cp_ring = cpr->cp_ring_struct; bnxt_hwrm_ring_free(bp, cp_ring, - HWRM_RING_FREE_INPUT_RING_TYPE_CMPL); + HWRM_RING_FREE_INPUT_RING_TYPE_L2_CMPL); cp_ring->fw_ring_id = INVALID_HW_RING_ID; - bp->grp_info[idx].cp_fw_ring_id = INVALID_HW_RING_ID; memset(cpr->cp_desc_ring, 0, cpr->cp_ring_struct->ring_size * sizeof(*cpr->cp_desc_ring)); cpr->cp_raw_cons = 0; @@ -1093,8 +1677,10 @@ int bnxt_free_all_hwrm_rings(struct bnxt *bp) txr->tx_prod = 0; txr->tx_cons = 0; } - if (cpr->cp_ring_struct->fw_ring_id != INVALID_HW_RING_ID) + if (cpr->cp_ring_struct->fw_ring_id != INVALID_HW_RING_ID) { bnxt_free_cp_ring(bp, cpr, idx); + cpr->cp_ring_struct->fw_ring_id = INVALID_HW_RING_ID; + } } for (i = 0; i < bp->rx_cp_nr_rings; i++) { @@ -1117,16 +1703,32 @@ int bnxt_free_all_hwrm_rings(struct bnxt *bp) sizeof(*rxr->rx_buf_ring)); rxr->rx_prod = 0; } - if (cpr->cp_ring_struct->fw_ring_id != INVALID_HW_RING_ID) + ring = rxr->ag_ring_struct; + if (ring->fw_ring_id != INVALID_HW_RING_ID) { + bnxt_hwrm_ring_free(bp, ring, + HWRM_RING_FREE_INPUT_RING_TYPE_RX); + ring->fw_ring_id = INVALID_HW_RING_ID; + memset(rxr->ag_buf_ring, 0, + rxr->ag_ring_struct->ring_size * + sizeof(*rxr->ag_buf_ring)); + rxr->ag_prod = 0; + bp->grp_info[i].ag_fw_ring_id = INVALID_HW_RING_ID; + } + if (cpr->cp_ring_struct->fw_ring_id != INVALID_HW_RING_ID) { bnxt_free_cp_ring(bp, cpr, idx); + bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID; + cpr->cp_ring_struct->fw_ring_id = INVALID_HW_RING_ID; + } } /* Default completion ring */ { struct bnxt_cp_ring_info *cpr = bp->def_cp_ring; - if (cpr->cp_ring_struct->fw_ring_id != INVALID_HW_RING_ID) + if (cpr->cp_ring_struct->fw_ring_id != INVALID_HW_RING_ID) { bnxt_free_cp_ring(bp, cpr, 0); + cpr->cp_ring_struct->fw_ring_id = INVALID_HW_RING_ID; + } } return rc; @@ -1138,14 +1740,7 @@ int bnxt_alloc_all_hwrm_ring_grps(struct bnxt *bp) uint32_t rc = 0; for (i = 0; i < bp->rx_cp_nr_rings; i++) { - unsigned int idx = i + 1; - - if (bp->grp_info[idx].cp_fw_ring_id == INVALID_HW_RING_ID || - bp->grp_info[idx].rx_fw_ring_id == INVALID_HW_RING_ID) - continue; - - rc = bnxt_hwrm_ring_grp_alloc(bp, idx); - + rc = bnxt_hwrm_ring_grp_alloc(bp, i); if (rc) return rc; } @@ -1156,8 +1751,11 @@ void bnxt_free_hwrm_resources(struct bnxt *bp) { /* Release memzone */ rte_free(bp->hwrm_cmd_resp_addr); + rte_free(bp->hwrm_short_cmd_req_addr); bp->hwrm_cmd_resp_addr = NULL; + bp->hwrm_short_cmd_req_addr = NULL; bp->hwrm_cmd_resp_dma_addr = 0; + bp->hwrm_short_cmd_req_dma_addr = 0; } int bnxt_alloc_hwrm_resources(struct bnxt *bp) @@ -1167,13 +1765,18 @@ int bnxt_alloc_hwrm_resources(struct bnxt *bp) sprintf(type, "bnxt_hwrm_%04x:%02x:%02x:%02x", pdev->addr.domain, pdev->addr.bus, pdev->addr.devid, pdev->addr.function); - bp->max_req_len = HWRM_MAX_REQ_LEN; bp->max_resp_len = HWRM_MAX_RESP_LEN; bp->hwrm_cmd_resp_addr = rte_malloc(type, bp->max_resp_len, 0); + rte_mem_lock_page(bp->hwrm_cmd_resp_addr); if (bp->hwrm_cmd_resp_addr == NULL) return -ENOMEM; bp->hwrm_cmd_resp_dma_addr = - rte_malloc_virt2phy(bp->hwrm_cmd_resp_addr); + rte_mem_virt2iova(bp->hwrm_cmd_resp_addr); + if (bp->hwrm_cmd_resp_dma_addr == 0) { + RTE_LOG(ERR, PMD, + "unable to map response address to physical memory\n"); + return -ENOMEM; + } rte_spinlock_init(&bp->hwrm_lock); return 0; @@ -1185,9 +1788,39 @@ int bnxt_clear_hwrm_vnic_filters(struct bnxt *bp, struct bnxt_vnic_info *vnic) int rc = 0; STAILQ_FOREACH(filter, &vnic->filter, next) { - rc = bnxt_hwrm_clear_filter(bp, filter); - if (rc) - break; + if (filter->filter_type == HWRM_CFA_EM_FILTER) + rc = bnxt_hwrm_clear_em_filter(bp, filter); + else if (filter->filter_type == HWRM_CFA_NTUPLE_FILTER) + rc = bnxt_hwrm_clear_ntuple_filter(bp, filter); + else + rc = bnxt_hwrm_clear_l2_filter(bp, filter); + //if (rc) + //break; + } + return rc; +} + +static int +bnxt_clear_hwrm_vnic_flows(struct bnxt *bp, struct bnxt_vnic_info *vnic) +{ + struct bnxt_filter_info *filter; + struct rte_flow *flow; + int rc = 0; + + STAILQ_FOREACH(flow, &vnic->flow_list, next) { + filter = flow->filter; + RTE_LOG(ERR, PMD, "filter type %d\n", filter->filter_type); + if (filter->filter_type == HWRM_CFA_EM_FILTER) + rc = bnxt_hwrm_clear_em_filter(bp, filter); + else if (filter->filter_type == HWRM_CFA_NTUPLE_FILTER) + rc = bnxt_hwrm_clear_ntuple_filter(bp, filter); + else + rc = bnxt_hwrm_clear_l2_filter(bp, filter); + + STAILQ_REMOVE(&vnic->flow_list, flow, rte_flow, next); + rte_free(flow); + //if (rc) + //break; } return rc; } @@ -1198,37 +1831,62 @@ int bnxt_set_hwrm_vnic_filters(struct bnxt *bp, struct bnxt_vnic_info *vnic) int rc = 0; STAILQ_FOREACH(filter, &vnic->filter, next) { - rc = bnxt_hwrm_set_filter(bp, vnic, filter); + if (filter->filter_type == HWRM_CFA_EM_FILTER) + rc = bnxt_hwrm_set_em_filter(bp, filter->dst_id, + filter); + else if (filter->filter_type == HWRM_CFA_NTUPLE_FILTER) + rc = bnxt_hwrm_set_ntuple_filter(bp, filter->dst_id, + filter); + else + rc = bnxt_hwrm_set_l2_filter(bp, vnic->fw_vnic_id, + filter); if (rc) break; } return rc; } +void bnxt_free_tunnel_ports(struct bnxt *bp) +{ + if (bp->vxlan_port_cnt) + bnxt_hwrm_tunnel_dst_port_free(bp, bp->vxlan_fw_dst_port_id, + HWRM_TUNNEL_DST_PORT_FREE_INPUT_TUNNEL_TYPE_VXLAN); + bp->vxlan_port = 0; + if (bp->geneve_port_cnt) + bnxt_hwrm_tunnel_dst_port_free(bp, bp->geneve_fw_dst_port_id, + HWRM_TUNNEL_DST_PORT_FREE_INPUT_TUNNEL_TYPE_GENEVE); + bp->geneve_port = 0; +} + void bnxt_free_all_hwrm_resources(struct bnxt *bp) { - struct bnxt_vnic_info *vnic; - unsigned int i; + int i; if (bp->vnic_info == NULL) return; - vnic = &bp->vnic_info[0]; - bnxt_hwrm_cfa_l2_clear_rx_mask(bp, vnic); - - /* VNIC resources */ - for (i = 0; i < bp->nr_vnics; i++) { + /* + * Cleanup VNICs in reverse order, to make sure the L2 filter + * from vnic0 is last to be cleaned up. + */ + for (i = bp->nr_vnics - 1; i >= 0; i--) { struct bnxt_vnic_info *vnic = &bp->vnic_info[i]; + bnxt_clear_hwrm_vnic_flows(bp, vnic); + bnxt_clear_hwrm_vnic_filters(bp, vnic); bnxt_hwrm_vnic_ctx_free(bp, vnic); + + bnxt_hwrm_vnic_tpa_cfg(bp, vnic, false); + bnxt_hwrm_vnic_free(bp, vnic); } /* Ring resources */ bnxt_free_all_hwrm_rings(bp); bnxt_free_all_hwrm_ring_grps(bp); bnxt_free_all_hwrm_stat_ctxs(bp); + bnxt_free_tunnel_ports(bp); } static uint16_t bnxt_parse_eth_link_duplex(uint32_t conf_link_speed) @@ -1246,46 +1904,51 @@ static uint16_t bnxt_parse_eth_link_duplex(uint32_t conf_link_speed) return hw_link_duplex; } +static uint16_t bnxt_check_eth_link_autoneg(uint32_t conf_link) +{ + return (conf_link & ETH_LINK_SPEED_FIXED) ? 0 : 1; +} + static uint16_t bnxt_parse_eth_link_speed(uint32_t conf_link_speed) { uint16_t eth_link_speed = 0; - if ((conf_link_speed & ETH_LINK_SPEED_FIXED) == ETH_LINK_SPEED_AUTONEG) + if (conf_link_speed == ETH_LINK_SPEED_AUTONEG) return ETH_LINK_SPEED_AUTONEG; switch (conf_link_speed & ~ETH_LINK_SPEED_FIXED) { case ETH_LINK_SPEED_100M: case ETH_LINK_SPEED_100M_HD: eth_link_speed = - HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_10MB; + HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_100MB; break; case ETH_LINK_SPEED_1G: eth_link_speed = - HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_1GB; + HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_1GB; break; case ETH_LINK_SPEED_2_5G: eth_link_speed = - HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_2_5GB; + HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_2_5GB; break; case ETH_LINK_SPEED_10G: eth_link_speed = - HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_10GB; + HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEED_10GB; break; case ETH_LINK_SPEED_20G: eth_link_speed = - HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_20GB; + HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_20GB; break; case ETH_LINK_SPEED_25G: eth_link_speed = - HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_25GB; + HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_25GB; break; case ETH_LINK_SPEED_40G: eth_link_speed = - HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_40GB; + HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEED_40GB; break; case ETH_LINK_SPEED_50G: eth_link_speed = - HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_50GB; + HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEED_50GB; break; default: RTE_LOG(ERR, PMD, @@ -1301,7 +1964,7 @@ static uint16_t bnxt_parse_eth_link_speed(uint32_t conf_link_speed) ETH_LINK_SPEED_10G | ETH_LINK_SPEED_20G | ETH_LINK_SPEED_25G | \ ETH_LINK_SPEED_40G | ETH_LINK_SPEED_50G) -static int bnxt_valid_link_speed(uint32_t link_speed, uint8_t port_id) +static int bnxt_valid_link_speed(uint32_t link_speed, uint16_t port_id) { uint32_t one_speed; @@ -1334,12 +1997,16 @@ static int bnxt_valid_link_speed(uint32_t link_speed, uint8_t port_id) return 0; } -static uint16_t bnxt_parse_eth_link_speed_mask(uint32_t link_speed) +static uint16_t +bnxt_parse_eth_link_speed_mask(struct bnxt *bp, uint32_t link_speed) { uint16_t ret = 0; - if (link_speed == ETH_LINK_SPEED_AUTONEG) + if (link_speed == ETH_LINK_SPEED_AUTONEG) { + if (bp->link_info.support_speeds) + return bp->link_info.support_speeds; link_speed = BNXT_SUPPORTED_SPEEDS; + } if (link_speed & ETH_LINK_SPEED_100M) ret |= HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_100MB; @@ -1431,16 +2098,16 @@ int bnxt_get_hwrm_link_config(struct bnxt *bp, struct rte_eth_link *link) "Get link config failed with rc %d\n", rc); goto exit; } - if (link_info->link_up) + if (link_info->link_speed) link->link_speed = bnxt_parse_hw_link_speed(link_info->link_speed); else - link->link_speed = ETH_LINK_SPEED_10M; + link->link_speed = ETH_SPEED_NUM_NONE; link->link_duplex = bnxt_parse_hw_link_duplex(link_info->duplex); link->link_status = link_info->link_up; link->link_autoneg = link_info->auto_mode == HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_MODE_NONE ? - ETH_LINK_SPEED_FIXED : ETH_LINK_SPEED_AUTONEG; + ETH_LINK_FIXED : ETH_LINK_AUTONEG; exit: return rc; } @@ -1450,7 +2117,10 @@ int bnxt_set_hwrm_link_config(struct bnxt *bp, bool link_up) int rc = 0; struct rte_eth_conf *dev_conf = &bp->eth_dev->data->dev_conf; struct bnxt_link_info link_req; - uint16_t speed; + uint16_t speed, autoneg; + + if (BNXT_NPAR_PF(bp) || BNXT_VF(bp)) + return 0; rc = bnxt_valid_link_speed(dev_conf->link_speeds, bp->eth_dev->data->port_id); @@ -1458,27 +2128,46 @@ int bnxt_set_hwrm_link_config(struct bnxt *bp, bool link_up) goto error; memset(&link_req, 0, sizeof(link_req)); - speed = bnxt_parse_eth_link_speed(dev_conf->link_speeds); link_req.link_up = link_up; - if (speed == 0) { - link_req.phy_flags = + if (!link_up) + goto port_phy_cfg; + + autoneg = bnxt_check_eth_link_autoneg(dev_conf->link_speeds); + speed = bnxt_parse_eth_link_speed(dev_conf->link_speeds); + link_req.phy_flags = HWRM_PORT_PHY_CFG_INPUT_FLAGS_RESET_PHY; + /* Autoneg can be done only when the FW allows */ + if (autoneg == 1 && !(bp->link_info.auto_link_speed || + bp->link_info.force_link_speed)) { + link_req.phy_flags |= HWRM_PORT_PHY_CFG_INPUT_FLAGS_RESTART_AUTONEG; - link_req.auto_mode = - HWRM_PORT_PHY_CFG_INPUT_AUTO_MODE_ONE_OR_BELOW; link_req.auto_link_speed_mask = - bnxt_parse_eth_link_speed_mask(dev_conf->link_speeds); - link_req.auto_link_speed = - HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_50GB; + bnxt_parse_eth_link_speed_mask(bp, + dev_conf->link_speeds); } else { - link_req.auto_mode = HWRM_PORT_PHY_CFG_INPUT_AUTO_MODE_NONE; - link_req.phy_flags = HWRM_PORT_PHY_CFG_INPUT_FLAGS_FORCE | - HWRM_PORT_PHY_CFG_INPUT_FLAGS_RESET_PHY; - link_req.link_speed = speed; + if (bp->link_info.phy_type == + HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASET || + bp->link_info.phy_type == + HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASETE || + bp->link_info.media_type == + HWRM_PORT_PHY_QCFG_OUTPUT_MEDIA_TYPE_TP) { + RTE_LOG(ERR, PMD, "10GBase-T devices must autoneg\n"); + return -EINVAL; + } + + link_req.phy_flags |= HWRM_PORT_PHY_CFG_INPUT_FLAGS_FORCE; + /* If user wants a particular speed try that first. */ + if (speed) + link_req.link_speed = speed; + else if (bp->link_info.force_link_speed) + link_req.link_speed = bp->link_info.force_link_speed; + else + link_req.link_speed = bp->link_info.auto_link_speed; } link_req.duplex = bnxt_parse_eth_link_duplex(dev_conf->link_speeds); link_req.auto_pause = bp->link_info.auto_pause; link_req.force_pause = bp->link_info.force_pause; +port_phy_cfg: rc = bnxt_hwrm_port_phy_cfg(bp, &link_req); if (rc) { RTE_LOG(ERR, PMD, @@ -1488,3 +2177,1424 @@ int bnxt_set_hwrm_link_config(struct bnxt *bp, bool link_up) error: return rc; } + +/* JIRA 22088 */ +int bnxt_hwrm_func_qcfg(struct bnxt *bp) +{ + struct hwrm_func_qcfg_input req = {0}; + struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr; + int rc = 0; + + HWRM_PREP(req, FUNC_QCFG); + req.fid = rte_cpu_to_le_16(0xffff); + + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + + HWRM_CHECK_RESULT(); + + /* Hard Coded.. 0xfff VLAN ID mask */ + bp->vlan = rte_le_to_cpu_16(resp->vlan) & 0xfff; + + switch (resp->port_partition_type) { + case HWRM_FUNC_QCFG_OUTPUT_PORT_PARTITION_TYPE_NPAR1_0: + case HWRM_FUNC_QCFG_OUTPUT_PORT_PARTITION_TYPE_NPAR1_5: + case HWRM_FUNC_QCFG_OUTPUT_PORT_PARTITION_TYPE_NPAR2_0: + bp->port_partition_type = resp->port_partition_type; + break; + default: + bp->port_partition_type = 0; + break; + } + + HWRM_UNLOCK(); + + return rc; +} + +static void copy_func_cfg_to_qcaps(struct hwrm_func_cfg_input *fcfg, + struct hwrm_func_qcaps_output *qcaps) +{ + qcaps->max_rsscos_ctx = fcfg->num_rsscos_ctxs; + memcpy(qcaps->mac_address, fcfg->dflt_mac_addr, + sizeof(qcaps->mac_address)); + qcaps->max_l2_ctxs = fcfg->num_l2_ctxs; + qcaps->max_rx_rings = fcfg->num_rx_rings; + qcaps->max_tx_rings = fcfg->num_tx_rings; + qcaps->max_cmpl_rings = fcfg->num_cmpl_rings; + qcaps->max_stat_ctx = fcfg->num_stat_ctxs; + qcaps->max_vfs = 0; + qcaps->first_vf_id = 0; + qcaps->max_vnics = fcfg->num_vnics; + qcaps->max_decap_records = 0; + qcaps->max_encap_records = 0; + qcaps->max_tx_wm_flows = 0; + qcaps->max_tx_em_flows = 0; + qcaps->max_rx_wm_flows = 0; + qcaps->max_rx_em_flows = 0; + qcaps->max_flow_id = 0; + qcaps->max_mcast_filters = fcfg->num_mcast_filters; + qcaps->max_sp_tx_rings = 0; + qcaps->max_hw_ring_grps = fcfg->num_hw_ring_grps; +} + +static int bnxt_hwrm_pf_func_cfg(struct bnxt *bp, int tx_rings) +{ + struct hwrm_func_cfg_input req = {0}; + struct hwrm_func_cfg_output *resp = bp->hwrm_cmd_resp_addr; + int rc; + + req.enables = rte_cpu_to_le_32(HWRM_FUNC_CFG_INPUT_ENABLES_MTU | + HWRM_FUNC_CFG_INPUT_ENABLES_MRU | + HWRM_FUNC_CFG_INPUT_ENABLES_NUM_RSSCOS_CTXS | + HWRM_FUNC_CFG_INPUT_ENABLES_NUM_STAT_CTXS | + HWRM_FUNC_CFG_INPUT_ENABLES_NUM_CMPL_RINGS | + HWRM_FUNC_CFG_INPUT_ENABLES_NUM_TX_RINGS | + HWRM_FUNC_CFG_INPUT_ENABLES_NUM_RX_RINGS | + HWRM_FUNC_CFG_INPUT_ENABLES_NUM_L2_CTXS | + HWRM_FUNC_CFG_INPUT_ENABLES_NUM_VNICS | + HWRM_FUNC_CFG_INPUT_ENABLES_NUM_HW_RING_GRPS); + req.flags = rte_cpu_to_le_32(bp->pf.func_cfg_flags); + req.mtu = rte_cpu_to_le_16(BNXT_MAX_MTU); + req.mru = rte_cpu_to_le_16(bp->eth_dev->data->mtu + ETHER_HDR_LEN + + ETHER_CRC_LEN + VLAN_TAG_SIZE); + req.num_rsscos_ctxs = rte_cpu_to_le_16(bp->max_rsscos_ctx); + req.num_stat_ctxs = rte_cpu_to_le_16(bp->max_stat_ctx); + req.num_cmpl_rings = rte_cpu_to_le_16(bp->max_cp_rings); + req.num_tx_rings = rte_cpu_to_le_16(tx_rings); + req.num_rx_rings = rte_cpu_to_le_16(bp->max_rx_rings); + req.num_l2_ctxs = rte_cpu_to_le_16(bp->max_l2_ctx); + req.num_vnics = rte_cpu_to_le_16(bp->max_vnics); + req.num_hw_ring_grps = rte_cpu_to_le_16(bp->max_ring_grps); + req.fid = rte_cpu_to_le_16(0xffff); + + HWRM_PREP(req, FUNC_CFG); + + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + + HWRM_CHECK_RESULT(); + HWRM_UNLOCK(); + + return rc; +} + +static void populate_vf_func_cfg_req(struct bnxt *bp, + struct hwrm_func_cfg_input *req, + int num_vfs) +{ + req->enables = rte_cpu_to_le_32(HWRM_FUNC_CFG_INPUT_ENABLES_MTU | + HWRM_FUNC_CFG_INPUT_ENABLES_MRU | + HWRM_FUNC_CFG_INPUT_ENABLES_NUM_RSSCOS_CTXS | + HWRM_FUNC_CFG_INPUT_ENABLES_NUM_STAT_CTXS | + HWRM_FUNC_CFG_INPUT_ENABLES_NUM_CMPL_RINGS | + HWRM_FUNC_CFG_INPUT_ENABLES_NUM_TX_RINGS | + HWRM_FUNC_CFG_INPUT_ENABLES_NUM_RX_RINGS | + HWRM_FUNC_CFG_INPUT_ENABLES_NUM_L2_CTXS | + HWRM_FUNC_CFG_INPUT_ENABLES_NUM_VNICS | + HWRM_FUNC_CFG_INPUT_ENABLES_NUM_HW_RING_GRPS); + + req->mtu = rte_cpu_to_le_16(bp->eth_dev->data->mtu + ETHER_HDR_LEN + + ETHER_CRC_LEN + VLAN_TAG_SIZE); + req->mru = rte_cpu_to_le_16(bp->eth_dev->data->mtu + ETHER_HDR_LEN + + ETHER_CRC_LEN + VLAN_TAG_SIZE); + req->num_rsscos_ctxs = rte_cpu_to_le_16(bp->max_rsscos_ctx / + (num_vfs + 1)); + req->num_stat_ctxs = rte_cpu_to_le_16(bp->max_stat_ctx / (num_vfs + 1)); + req->num_cmpl_rings = rte_cpu_to_le_16(bp->max_cp_rings / + (num_vfs + 1)); + req->num_tx_rings = rte_cpu_to_le_16(bp->max_tx_rings / (num_vfs + 1)); + req->num_rx_rings = rte_cpu_to_le_16(bp->max_rx_rings / (num_vfs + 1)); + req->num_l2_ctxs = rte_cpu_to_le_16(bp->max_l2_ctx / (num_vfs + 1)); + /* TODO: For now, do not support VMDq/RFS on VFs. */ + req->num_vnics = rte_cpu_to_le_16(1); + req->num_hw_ring_grps = rte_cpu_to_le_16(bp->max_ring_grps / + (num_vfs + 1)); +} + +static void add_random_mac_if_needed(struct bnxt *bp, + struct hwrm_func_cfg_input *cfg_req, + int vf) +{ + struct ether_addr mac; + + if (bnxt_hwrm_func_qcfg_vf_default_mac(bp, vf, &mac)) + return; + + if (memcmp(mac.addr_bytes, "\x00\x00\x00\x00\x00", 6) == 0) { + cfg_req->enables |= + rte_cpu_to_le_32(HWRM_FUNC_CFG_INPUT_ENABLES_DFLT_MAC_ADDR); + eth_random_addr(cfg_req->dflt_mac_addr); + bp->pf.vf_info[vf].random_mac = true; + } else { + memcpy(cfg_req->dflt_mac_addr, mac.addr_bytes, ETHER_ADDR_LEN); + } +} + +static void reserve_resources_from_vf(struct bnxt *bp, + struct hwrm_func_cfg_input *cfg_req, + int vf) +{ + struct hwrm_func_qcaps_input req = {0}; + struct hwrm_func_qcaps_output *resp = bp->hwrm_cmd_resp_addr; + int rc; + + /* Get the actual allocated values now */ + HWRM_PREP(req, FUNC_QCAPS); + req.fid = rte_cpu_to_le_16(bp->pf.vf_info[vf].fid); + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + + if (rc) { + RTE_LOG(ERR, PMD, "hwrm_func_qcaps failed rc:%d\n", rc); + copy_func_cfg_to_qcaps(cfg_req, resp); + } else if (resp->error_code) { + rc = rte_le_to_cpu_16(resp->error_code); + RTE_LOG(ERR, PMD, "hwrm_func_qcaps error %d\n", rc); + copy_func_cfg_to_qcaps(cfg_req, resp); + } + + bp->max_rsscos_ctx -= rte_le_to_cpu_16(resp->max_rsscos_ctx); + bp->max_stat_ctx -= rte_le_to_cpu_16(resp->max_stat_ctx); + bp->max_cp_rings -= rte_le_to_cpu_16(resp->max_cmpl_rings); + bp->max_tx_rings -= rte_le_to_cpu_16(resp->max_tx_rings); + bp->max_rx_rings -= rte_le_to_cpu_16(resp->max_rx_rings); + bp->max_l2_ctx -= rte_le_to_cpu_16(resp->max_l2_ctxs); + /* + * TODO: While not supporting VMDq with VFs, max_vnics is always + * forced to 1 in this case + */ + //bp->max_vnics -= rte_le_to_cpu_16(esp->max_vnics); + bp->max_ring_grps -= rte_le_to_cpu_16(resp->max_hw_ring_grps); + + HWRM_UNLOCK(); +} + +int bnxt_hwrm_func_qcfg_current_vf_vlan(struct bnxt *bp, int vf) +{ + struct hwrm_func_qcfg_input req = {0}; + struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr; + int rc; + + /* Check for zero MAC address */ + HWRM_PREP(req, FUNC_QCFG); + req.fid = rte_cpu_to_le_16(bp->pf.vf_info[vf].fid); + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + if (rc) { + RTE_LOG(ERR, PMD, "hwrm_func_qcfg failed rc:%d\n", rc); + return -1; + } else if (resp->error_code) { + rc = rte_le_to_cpu_16(resp->error_code); + RTE_LOG(ERR, PMD, "hwrm_func_qcfg error %d\n", rc); + return -1; + } + rc = rte_le_to_cpu_16(resp->vlan); + + HWRM_UNLOCK(); + + return rc; +} + +static int update_pf_resource_max(struct bnxt *bp) +{ + struct hwrm_func_qcfg_input req = {0}; + struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr; + int rc; + + /* And copy the allocated numbers into the pf struct */ + HWRM_PREP(req, FUNC_QCFG); + req.fid = rte_cpu_to_le_16(0xffff); + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + HWRM_CHECK_RESULT(); + + /* Only TX ring value reflects actual allocation? TODO */ + bp->max_tx_rings = rte_le_to_cpu_16(resp->alloc_tx_rings); + bp->pf.evb_mode = resp->evb_mode; + + HWRM_UNLOCK(); + + return rc; +} + +int bnxt_hwrm_allocate_pf_only(struct bnxt *bp) +{ + int rc; + + if (!BNXT_PF(bp)) { + RTE_LOG(ERR, PMD, "Attempt to allcoate VFs on a VF!\n"); + return -1; + } + + rc = bnxt_hwrm_func_qcaps(bp); + if (rc) + return rc; + + bp->pf.func_cfg_flags &= + ~(HWRM_FUNC_CFG_INPUT_FLAGS_STD_TX_RING_MODE_ENABLE | + HWRM_FUNC_CFG_INPUT_FLAGS_STD_TX_RING_MODE_DISABLE); + bp->pf.func_cfg_flags |= + HWRM_FUNC_CFG_INPUT_FLAGS_STD_TX_RING_MODE_DISABLE; + rc = bnxt_hwrm_pf_func_cfg(bp, bp->max_tx_rings); + return rc; +} + +int bnxt_hwrm_allocate_vfs(struct bnxt *bp, int num_vfs) +{ + struct hwrm_func_cfg_input req = {0}; + struct hwrm_func_cfg_output *resp = bp->hwrm_cmd_resp_addr; + int i; + size_t sz; + int rc = 0; + size_t req_buf_sz; + + if (!BNXT_PF(bp)) { + RTE_LOG(ERR, PMD, "Attempt to allcoate VFs on a VF!\n"); + return -1; + } + + rc = bnxt_hwrm_func_qcaps(bp); + + if (rc) + return rc; + + bp->pf.active_vfs = num_vfs; + + /* + * First, configure the PF to only use one TX ring. This ensures that + * there are enough rings for all VFs. + * + * If we don't do this, when we call func_alloc() later, we will lock + * extra rings to the PF that won't be available during func_cfg() of + * the VFs. + * + * This has been fixed with firmware versions above 20.6.54 + */ + bp->pf.func_cfg_flags &= + ~(HWRM_FUNC_CFG_INPUT_FLAGS_STD_TX_RING_MODE_ENABLE | + HWRM_FUNC_CFG_INPUT_FLAGS_STD_TX_RING_MODE_DISABLE); + bp->pf.func_cfg_flags |= + HWRM_FUNC_CFG_INPUT_FLAGS_STD_TX_RING_MODE_ENABLE; + rc = bnxt_hwrm_pf_func_cfg(bp, 1); + if (rc) + return rc; + + /* + * Now, create and register a buffer to hold forwarded VF requests + */ + req_buf_sz = num_vfs * HWRM_MAX_REQ_LEN; + bp->pf.vf_req_buf = rte_malloc("bnxt_vf_fwd", req_buf_sz, + page_roundup(num_vfs * HWRM_MAX_REQ_LEN)); + if (bp->pf.vf_req_buf == NULL) { + rc = -ENOMEM; + goto error_free; + } + for (sz = 0; sz < req_buf_sz; sz += getpagesize()) + rte_mem_lock_page(((char *)bp->pf.vf_req_buf) + sz); + for (i = 0; i < num_vfs; i++) + bp->pf.vf_info[i].req_buf = ((char *)bp->pf.vf_req_buf) + + (i * HWRM_MAX_REQ_LEN); + + rc = bnxt_hwrm_func_buf_rgtr(bp); + if (rc) + goto error_free; + + populate_vf_func_cfg_req(bp, &req, num_vfs); + + bp->pf.active_vfs = 0; + for (i = 0; i < num_vfs; i++) { + add_random_mac_if_needed(bp, &req, i); + + HWRM_PREP(req, FUNC_CFG); + req.flags = rte_cpu_to_le_32(bp->pf.vf_info[i].func_cfg_flags); + req.fid = rte_cpu_to_le_16(bp->pf.vf_info[i].fid); + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + + /* Clear enable flag for next pass */ + req.enables &= ~rte_cpu_to_le_32( + HWRM_FUNC_CFG_INPUT_ENABLES_DFLT_MAC_ADDR); + + if (rc || resp->error_code) { + RTE_LOG(ERR, PMD, + "Failed to initizlie VF %d\n", i); + RTE_LOG(ERR, PMD, + "Not all VFs available. (%d, %d)\n", + rc, resp->error_code); + HWRM_UNLOCK(); + break; + } + + HWRM_UNLOCK(); + + reserve_resources_from_vf(bp, &req, i); + bp->pf.active_vfs++; + bnxt_hwrm_func_clr_stats(bp, bp->pf.vf_info[i].fid); + } + + /* + * Now configure the PF to use "the rest" of the resources + * We're using STD_TX_RING_MODE here though which will limit the TX + * rings. This will allow QoS to function properly. Not setting this + * will cause PF rings to break bandwidth settings. + */ + rc = bnxt_hwrm_pf_func_cfg(bp, bp->max_tx_rings); + if (rc) + goto error_free; + + rc = update_pf_resource_max(bp); + if (rc) + goto error_free; + + return rc; + +error_free: + bnxt_hwrm_func_buf_unrgtr(bp); + return rc; +} + +int bnxt_hwrm_pf_evb_mode(struct bnxt *bp) +{ + struct hwrm_func_cfg_input req = {0}; + struct hwrm_func_cfg_output *resp = bp->hwrm_cmd_resp_addr; + int rc; + + HWRM_PREP(req, FUNC_CFG); + + req.fid = rte_cpu_to_le_16(0xffff); + req.enables = rte_cpu_to_le_32(HWRM_FUNC_CFG_INPUT_ENABLES_EVB_MODE); + req.evb_mode = bp->pf.evb_mode; + + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + HWRM_CHECK_RESULT(); + HWRM_UNLOCK(); + + return rc; +} + +int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, uint16_t port, + uint8_t tunnel_type) +{ + struct hwrm_tunnel_dst_port_alloc_input req = {0}; + struct hwrm_tunnel_dst_port_alloc_output *resp = bp->hwrm_cmd_resp_addr; + int rc = 0; + + HWRM_PREP(req, TUNNEL_DST_PORT_ALLOC); + req.tunnel_type = tunnel_type; + req.tunnel_dst_port_val = port; + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + HWRM_CHECK_RESULT(); + + switch (tunnel_type) { + case HWRM_TUNNEL_DST_PORT_ALLOC_INPUT_TUNNEL_TYPE_VXLAN: + bp->vxlan_fw_dst_port_id = resp->tunnel_dst_port_id; + bp->vxlan_port = port; + break; + case HWRM_TUNNEL_DST_PORT_ALLOC_INPUT_TUNNEL_TYPE_GENEVE: + bp->geneve_fw_dst_port_id = resp->tunnel_dst_port_id; + bp->geneve_port = port; + break; + default: + break; + } + + HWRM_UNLOCK(); + + return rc; +} + +int bnxt_hwrm_tunnel_dst_port_free(struct bnxt *bp, uint16_t port, + uint8_t tunnel_type) +{ + struct hwrm_tunnel_dst_port_free_input req = {0}; + struct hwrm_tunnel_dst_port_free_output *resp = bp->hwrm_cmd_resp_addr; + int rc = 0; + + HWRM_PREP(req, TUNNEL_DST_PORT_FREE); + + req.tunnel_type = tunnel_type; + req.tunnel_dst_port_id = rte_cpu_to_be_16(port); + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + + HWRM_CHECK_RESULT(); + HWRM_UNLOCK(); + + return rc; +} + +int bnxt_hwrm_func_cfg_vf_set_flags(struct bnxt *bp, uint16_t vf, + uint32_t flags) +{ + struct hwrm_func_cfg_output *resp = bp->hwrm_cmd_resp_addr; + struct hwrm_func_cfg_input req = {0}; + int rc; + + HWRM_PREP(req, FUNC_CFG); + + req.fid = rte_cpu_to_le_16(bp->pf.vf_info[vf].fid); + req.flags = rte_cpu_to_le_32(flags); + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + + HWRM_CHECK_RESULT(); + HWRM_UNLOCK(); + + return rc; +} + +void vf_vnic_set_rxmask_cb(struct bnxt_vnic_info *vnic, void *flagp) +{ + uint32_t *flag = flagp; + + vnic->flags = *flag; +} + +int bnxt_set_rx_mask_no_vlan(struct bnxt *bp, struct bnxt_vnic_info *vnic) +{ + return bnxt_hwrm_cfa_l2_set_rx_mask(bp, vnic, 0, NULL); +} + +int bnxt_hwrm_func_buf_rgtr(struct bnxt *bp) +{ + int rc = 0; + struct hwrm_func_buf_rgtr_input req = {.req_type = 0 }; + struct hwrm_func_buf_rgtr_output *resp = bp->hwrm_cmd_resp_addr; + + HWRM_PREP(req, FUNC_BUF_RGTR); + + req.req_buf_num_pages = rte_cpu_to_le_16(1); + req.req_buf_page_size = rte_cpu_to_le_16( + page_getenum(bp->pf.active_vfs * HWRM_MAX_REQ_LEN)); + req.req_buf_len = rte_cpu_to_le_16(HWRM_MAX_REQ_LEN); + req.req_buf_page_addr[0] = + rte_cpu_to_le_64(rte_mem_virt2iova(bp->pf.vf_req_buf)); + if (req.req_buf_page_addr[0] == 0) { + RTE_LOG(ERR, PMD, + "unable to map buffer address to physical memory\n"); + return -ENOMEM; + } + + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + + HWRM_CHECK_RESULT(); + HWRM_UNLOCK(); + + return rc; +} + +int bnxt_hwrm_func_buf_unrgtr(struct bnxt *bp) +{ + int rc = 0; + struct hwrm_func_buf_unrgtr_input req = {.req_type = 0 }; + struct hwrm_func_buf_unrgtr_output *resp = bp->hwrm_cmd_resp_addr; + + HWRM_PREP(req, FUNC_BUF_UNRGTR); + + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + + HWRM_CHECK_RESULT(); + HWRM_UNLOCK(); + + return rc; +} + +int bnxt_hwrm_func_cfg_def_cp(struct bnxt *bp) +{ + struct hwrm_func_cfg_output *resp = bp->hwrm_cmd_resp_addr; + struct hwrm_func_cfg_input req = {0}; + int rc; + + HWRM_PREP(req, FUNC_CFG); + + req.fid = rte_cpu_to_le_16(0xffff); + req.flags = rte_cpu_to_le_32(bp->pf.func_cfg_flags); + req.enables = rte_cpu_to_le_32( + HWRM_FUNC_CFG_INPUT_ENABLES_ASYNC_EVENT_CR); + req.async_event_cr = rte_cpu_to_le_16( + bp->def_cp_ring->cp_ring_struct->fw_ring_id); + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + + HWRM_CHECK_RESULT(); + HWRM_UNLOCK(); + + return rc; +} + +int bnxt_hwrm_vf_func_cfg_def_cp(struct bnxt *bp) +{ + struct hwrm_func_vf_cfg_output *resp = bp->hwrm_cmd_resp_addr; + struct hwrm_func_vf_cfg_input req = {0}; + int rc; + + HWRM_PREP(req, FUNC_VF_CFG); + + req.enables = rte_cpu_to_le_32( + HWRM_FUNC_CFG_INPUT_ENABLES_ASYNC_EVENT_CR); + req.async_event_cr = rte_cpu_to_le_16( + bp->def_cp_ring->cp_ring_struct->fw_ring_id); + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + + HWRM_CHECK_RESULT(); + HWRM_UNLOCK(); + + return rc; +} + +int bnxt_hwrm_set_default_vlan(struct bnxt *bp, int vf, uint8_t is_vf) +{ + struct hwrm_func_cfg_input req = {0}; + struct hwrm_func_cfg_output *resp = bp->hwrm_cmd_resp_addr; + uint16_t dflt_vlan, fid; + uint32_t func_cfg_flags; + int rc = 0; + + HWRM_PREP(req, FUNC_CFG); + + if (is_vf) { + dflt_vlan = bp->pf.vf_info[vf].dflt_vlan; + fid = bp->pf.vf_info[vf].fid; + func_cfg_flags = bp->pf.vf_info[vf].func_cfg_flags; + } else { + fid = rte_cpu_to_le_16(0xffff); + func_cfg_flags = bp->pf.func_cfg_flags; + dflt_vlan = bp->vlan; + } + + req.flags = rte_cpu_to_le_32(func_cfg_flags); + req.fid = rte_cpu_to_le_16(fid); + req.enables |= rte_cpu_to_le_32(HWRM_FUNC_CFG_INPUT_ENABLES_DFLT_VLAN); + req.dflt_vlan = rte_cpu_to_le_16(dflt_vlan); + + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + + HWRM_CHECK_RESULT(); + HWRM_UNLOCK(); + + return rc; +} + +int bnxt_hwrm_func_bw_cfg(struct bnxt *bp, uint16_t vf, + uint16_t max_bw, uint16_t enables) +{ + struct hwrm_func_cfg_output *resp = bp->hwrm_cmd_resp_addr; + struct hwrm_func_cfg_input req = {0}; + int rc; + + HWRM_PREP(req, FUNC_CFG); + + req.fid = rte_cpu_to_le_16(bp->pf.vf_info[vf].fid); + req.enables |= rte_cpu_to_le_32(enables); + req.flags = rte_cpu_to_le_32(bp->pf.vf_info[vf].func_cfg_flags); + req.max_bw = rte_cpu_to_le_32(max_bw); + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + + HWRM_CHECK_RESULT(); + HWRM_UNLOCK(); + + return rc; +} + +int bnxt_hwrm_set_vf_vlan(struct bnxt *bp, int vf) +{ + struct hwrm_func_cfg_input req = {0}; + struct hwrm_func_cfg_output *resp = bp->hwrm_cmd_resp_addr; + int rc = 0; + + HWRM_PREP(req, FUNC_CFG); + + req.flags = rte_cpu_to_le_32(bp->pf.vf_info[vf].func_cfg_flags); + req.fid = rte_cpu_to_le_16(bp->pf.vf_info[vf].fid); + req.enables |= rte_cpu_to_le_32(HWRM_FUNC_CFG_INPUT_ENABLES_DFLT_VLAN); + req.dflt_vlan = rte_cpu_to_le_16(bp->pf.vf_info[vf].dflt_vlan); + + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + + HWRM_CHECK_RESULT(); + HWRM_UNLOCK(); + + return rc; +} + +int bnxt_hwrm_reject_fwd_resp(struct bnxt *bp, uint16_t target_id, + void *encaped, size_t ec_size) +{ + int rc = 0; + struct hwrm_reject_fwd_resp_input req = {.req_type = 0}; + struct hwrm_reject_fwd_resp_output *resp = bp->hwrm_cmd_resp_addr; + + if (ec_size > sizeof(req.encap_request)) + return -1; + + HWRM_PREP(req, REJECT_FWD_RESP); + + req.encap_resp_target_id = rte_cpu_to_le_16(target_id); + memcpy(req.encap_request, encaped, ec_size); + + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + + HWRM_CHECK_RESULT(); + HWRM_UNLOCK(); + + return rc; +} + +int bnxt_hwrm_func_qcfg_vf_default_mac(struct bnxt *bp, uint16_t vf, + struct ether_addr *mac) +{ + struct hwrm_func_qcfg_input req = {0}; + struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr; + int rc; + + HWRM_PREP(req, FUNC_QCFG); + + req.fid = rte_cpu_to_le_16(bp->pf.vf_info[vf].fid); + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + + HWRM_CHECK_RESULT(); + + memcpy(mac->addr_bytes, resp->mac_address, ETHER_ADDR_LEN); + + HWRM_UNLOCK(); + + return rc; +} + +int bnxt_hwrm_exec_fwd_resp(struct bnxt *bp, uint16_t target_id, + void *encaped, size_t ec_size) +{ + int rc = 0; + struct hwrm_exec_fwd_resp_input req = {.req_type = 0}; + struct hwrm_exec_fwd_resp_output *resp = bp->hwrm_cmd_resp_addr; + + if (ec_size > sizeof(req.encap_request)) + return -1; + + HWRM_PREP(req, EXEC_FWD_RESP); + + req.encap_resp_target_id = rte_cpu_to_le_16(target_id); + memcpy(req.encap_request, encaped, ec_size); + + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + + HWRM_CHECK_RESULT(); + HWRM_UNLOCK(); + + return rc; +} + +int bnxt_hwrm_ctx_qstats(struct bnxt *bp, uint32_t cid, int idx, + struct rte_eth_stats *stats, uint8_t rx) +{ + int rc = 0; + struct hwrm_stat_ctx_query_input req = {.req_type = 0}; + struct hwrm_stat_ctx_query_output *resp = bp->hwrm_cmd_resp_addr; + + HWRM_PREP(req, STAT_CTX_QUERY); + + req.stat_ctx_id = rte_cpu_to_le_32(cid); + + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + + HWRM_CHECK_RESULT(); + + if (rx) { + stats->q_ipackets[idx] = rte_le_to_cpu_64(resp->rx_ucast_pkts); + stats->q_ipackets[idx] += rte_le_to_cpu_64(resp->rx_mcast_pkts); + stats->q_ipackets[idx] += rte_le_to_cpu_64(resp->rx_bcast_pkts); + stats->q_ibytes[idx] = rte_le_to_cpu_64(resp->rx_ucast_bytes); + stats->q_ibytes[idx] += rte_le_to_cpu_64(resp->rx_mcast_bytes); + stats->q_ibytes[idx] += rte_le_to_cpu_64(resp->rx_bcast_bytes); + stats->q_errors[idx] = rte_le_to_cpu_64(resp->rx_err_pkts); + stats->q_errors[idx] += rte_le_to_cpu_64(resp->rx_drop_pkts); + } else { + stats->q_opackets[idx] = rte_le_to_cpu_64(resp->tx_ucast_pkts); + stats->q_opackets[idx] += rte_le_to_cpu_64(resp->tx_mcast_pkts); + stats->q_opackets[idx] += rte_le_to_cpu_64(resp->tx_bcast_pkts); + stats->q_obytes[idx] = rte_le_to_cpu_64(resp->tx_ucast_bytes); + stats->q_obytes[idx] += rte_le_to_cpu_64(resp->tx_mcast_bytes); + stats->q_obytes[idx] += rte_le_to_cpu_64(resp->tx_bcast_bytes); + stats->q_errors[idx] += rte_le_to_cpu_64(resp->tx_err_pkts); + } + + + HWRM_UNLOCK(); + + return rc; +} + +int bnxt_hwrm_port_qstats(struct bnxt *bp) +{ + struct hwrm_port_qstats_input req = {0}; + struct hwrm_port_qstats_output *resp = bp->hwrm_cmd_resp_addr; + struct bnxt_pf_info *pf = &bp->pf; + int rc; + + if (!(bp->flags & BNXT_FLAG_PORT_STATS)) + return 0; + + HWRM_PREP(req, PORT_QSTATS); + + req.port_id = rte_cpu_to_le_16(pf->port_id); + req.tx_stat_host_addr = rte_cpu_to_le_64(bp->hw_tx_port_stats_map); + req.rx_stat_host_addr = rte_cpu_to_le_64(bp->hw_rx_port_stats_map); + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + + HWRM_CHECK_RESULT(); + HWRM_UNLOCK(); + + return rc; +} + +int bnxt_hwrm_port_clr_stats(struct bnxt *bp) +{ + struct hwrm_port_clr_stats_input req = {0}; + struct hwrm_port_clr_stats_output *resp = bp->hwrm_cmd_resp_addr; + struct bnxt_pf_info *pf = &bp->pf; + int rc; + + if (!(bp->flags & BNXT_FLAG_PORT_STATS)) + return 0; + + HWRM_PREP(req, PORT_CLR_STATS); + + req.port_id = rte_cpu_to_le_16(pf->port_id); + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + + HWRM_CHECK_RESULT(); + HWRM_UNLOCK(); + + return rc; +} + +int bnxt_hwrm_port_led_qcaps(struct bnxt *bp) +{ + struct hwrm_port_led_qcaps_output *resp = bp->hwrm_cmd_resp_addr; + struct hwrm_port_led_qcaps_input req = {0}; + int rc; + + if (BNXT_VF(bp)) + return 0; + + HWRM_PREP(req, PORT_LED_QCAPS); + req.port_id = bp->pf.port_id; + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + + HWRM_CHECK_RESULT(); + + if (resp->num_leds > 0 && resp->num_leds < BNXT_MAX_LED) { + unsigned int i; + + bp->num_leds = resp->num_leds; + memcpy(bp->leds, &resp->led0_id, + sizeof(bp->leds[0]) * bp->num_leds); + for (i = 0; i < bp->num_leds; i++) { + struct bnxt_led_info *led = &bp->leds[i]; + + uint16_t caps = led->led_state_caps; + + if (!led->led_group_id || + !BNXT_LED_ALT_BLINK_CAP(caps)) { + bp->num_leds = 0; + break; + } + } + } + + HWRM_UNLOCK(); + + return rc; +} + +int bnxt_hwrm_port_led_cfg(struct bnxt *bp, bool led_on) +{ + struct hwrm_port_led_cfg_output *resp = bp->hwrm_cmd_resp_addr; + struct hwrm_port_led_cfg_input req = {0}; + struct bnxt_led_cfg *led_cfg; + uint8_t led_state = HWRM_PORT_LED_QCFG_OUTPUT_LED0_STATE_DEFAULT; + uint16_t duration = 0; + int rc, i; + + if (!bp->num_leds || BNXT_VF(bp)) + return -EOPNOTSUPP; + + HWRM_PREP(req, PORT_LED_CFG); + + if (led_on) { + led_state = HWRM_PORT_LED_CFG_INPUT_LED0_STATE_BLINKALT; + duration = rte_cpu_to_le_16(500); + } + req.port_id = bp->pf.port_id; + req.num_leds = bp->num_leds; + led_cfg = (struct bnxt_led_cfg *)&req.led0_id; + for (i = 0; i < bp->num_leds; i++, led_cfg++) { + req.enables |= BNXT_LED_DFLT_ENABLES(i); + led_cfg->led_id = bp->leds[i].led_id; + led_cfg->led_state = led_state; + led_cfg->led_blink_on = duration; + led_cfg->led_blink_off = duration; + led_cfg->led_group_id = bp->leds[i].led_group_id; + } + + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + + HWRM_CHECK_RESULT(); + HWRM_UNLOCK(); + + return rc; +} + +int bnxt_hwrm_nvm_get_dir_info(struct bnxt *bp, uint32_t *entries, + uint32_t *length) +{ + int rc; + struct hwrm_nvm_get_dir_info_input req = {0}; + struct hwrm_nvm_get_dir_info_output *resp = bp->hwrm_cmd_resp_addr; + + HWRM_PREP(req, NVM_GET_DIR_INFO); + + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + + HWRM_CHECK_RESULT(); + HWRM_UNLOCK(); + + if (!rc) { + *entries = rte_le_to_cpu_32(resp->entries); + *length = rte_le_to_cpu_32(resp->entry_length); + } + return rc; +} + +int bnxt_get_nvram_directory(struct bnxt *bp, uint32_t len, uint8_t *data) +{ + int rc; + uint32_t dir_entries; + uint32_t entry_length; + uint8_t *buf; + size_t buflen; + rte_iova_t dma_handle; + struct hwrm_nvm_get_dir_entries_input req = {0}; + struct hwrm_nvm_get_dir_entries_output *resp = bp->hwrm_cmd_resp_addr; + + rc = bnxt_hwrm_nvm_get_dir_info(bp, &dir_entries, &entry_length); + if (rc != 0) + return rc; + + *data++ = dir_entries; + *data++ = entry_length; + len -= 2; + memset(data, 0xff, len); + + buflen = dir_entries * entry_length; + buf = rte_malloc("nvm_dir", buflen, 0); + rte_mem_lock_page(buf); + if (buf == NULL) + return -ENOMEM; + dma_handle = rte_mem_virt2iova(buf); + if (dma_handle == 0) { + RTE_LOG(ERR, PMD, + "unable to map response address to physical memory\n"); + return -ENOMEM; + } + HWRM_PREP(req, NVM_GET_DIR_ENTRIES); + req.host_dest_addr = rte_cpu_to_le_64(dma_handle); + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + + HWRM_CHECK_RESULT(); + HWRM_UNLOCK(); + + if (rc == 0) + memcpy(data, buf, len > buflen ? buflen : len); + + rte_free(buf); + + return rc; +} + +int bnxt_hwrm_get_nvram_item(struct bnxt *bp, uint32_t index, + uint32_t offset, uint32_t length, + uint8_t *data) +{ + int rc; + uint8_t *buf; + rte_iova_t dma_handle; + struct hwrm_nvm_read_input req = {0}; + struct hwrm_nvm_read_output *resp = bp->hwrm_cmd_resp_addr; + + buf = rte_malloc("nvm_item", length, 0); + rte_mem_lock_page(buf); + if (!buf) + return -ENOMEM; + + dma_handle = rte_mem_virt2iova(buf); + if (dma_handle == 0) { + RTE_LOG(ERR, PMD, + "unable to map response address to physical memory\n"); + return -ENOMEM; + } + HWRM_PREP(req, NVM_READ); + req.host_dest_addr = rte_cpu_to_le_64(dma_handle); + req.dir_idx = rte_cpu_to_le_16(index); + req.offset = rte_cpu_to_le_32(offset); + req.len = rte_cpu_to_le_32(length); + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + HWRM_CHECK_RESULT(); + HWRM_UNLOCK(); + if (rc == 0) + memcpy(data, buf, length); + + rte_free(buf); + return rc; +} + +int bnxt_hwrm_erase_nvram_directory(struct bnxt *bp, uint8_t index) +{ + int rc; + struct hwrm_nvm_erase_dir_entry_input req = {0}; + struct hwrm_nvm_erase_dir_entry_output *resp = bp->hwrm_cmd_resp_addr; + + HWRM_PREP(req, NVM_ERASE_DIR_ENTRY); + req.dir_idx = rte_cpu_to_le_16(index); + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + HWRM_CHECK_RESULT(); + HWRM_UNLOCK(); + + return rc; +} + + +int bnxt_hwrm_flash_nvram(struct bnxt *bp, uint16_t dir_type, + uint16_t dir_ordinal, uint16_t dir_ext, + uint16_t dir_attr, const uint8_t *data, + size_t data_len) +{ + int rc; + struct hwrm_nvm_write_input req = {0}; + struct hwrm_nvm_write_output *resp = bp->hwrm_cmd_resp_addr; + rte_iova_t dma_handle; + uint8_t *buf; + + HWRM_PREP(req, NVM_WRITE); + + req.dir_type = rte_cpu_to_le_16(dir_type); + req.dir_ordinal = rte_cpu_to_le_16(dir_ordinal); + req.dir_ext = rte_cpu_to_le_16(dir_ext); + req.dir_attr = rte_cpu_to_le_16(dir_attr); + req.dir_data_length = rte_cpu_to_le_32(data_len); + + buf = rte_malloc("nvm_write", data_len, 0); + rte_mem_lock_page(buf); + if (!buf) + return -ENOMEM; + + dma_handle = rte_mem_virt2iova(buf); + if (dma_handle == 0) { + RTE_LOG(ERR, PMD, + "unable to map response address to physical memory\n"); + return -ENOMEM; + } + memcpy(buf, data, data_len); + req.host_src_addr = rte_cpu_to_le_64(dma_handle); + + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + + HWRM_CHECK_RESULT(); + HWRM_UNLOCK(); + + rte_free(buf); + return rc; +} + +static void +bnxt_vnic_count(struct bnxt_vnic_info *vnic __rte_unused, void *cbdata) +{ + uint32_t *count = cbdata; + + *count = *count + 1; +} + +static int bnxt_vnic_count_hwrm_stub(struct bnxt *bp __rte_unused, + struct bnxt_vnic_info *vnic __rte_unused) +{ + return 0; +} + +int bnxt_vf_vnic_count(struct bnxt *bp, uint16_t vf) +{ + uint32_t count = 0; + + bnxt_hwrm_func_vf_vnic_query_and_config(bp, vf, bnxt_vnic_count, + &count, bnxt_vnic_count_hwrm_stub); + + return count; +} + +static int bnxt_hwrm_func_vf_vnic_query(struct bnxt *bp, uint16_t vf, + uint16_t *vnic_ids) +{ + struct hwrm_func_vf_vnic_ids_query_input req = {0}; + struct hwrm_func_vf_vnic_ids_query_output *resp = + bp->hwrm_cmd_resp_addr; + int rc; + + /* First query all VNIC ids */ + HWRM_PREP(req, FUNC_VF_VNIC_IDS_QUERY); + + req.vf_id = rte_cpu_to_le_16(bp->pf.first_vf_id + vf); + req.max_vnic_id_cnt = rte_cpu_to_le_32(bp->pf.total_vnics); + req.vnic_id_tbl_addr = rte_cpu_to_le_64(rte_mem_virt2iova(vnic_ids)); + + if (req.vnic_id_tbl_addr == 0) { + HWRM_UNLOCK(); + RTE_LOG(ERR, PMD, + "unable to map VNIC ID table address to physical memory\n"); + return -ENOMEM; + } + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + if (rc) { + HWRM_UNLOCK(); + RTE_LOG(ERR, PMD, "hwrm_func_vf_vnic_query failed rc:%d\n", rc); + return -1; + } else if (resp->error_code) { + rc = rte_le_to_cpu_16(resp->error_code); + HWRM_UNLOCK(); + RTE_LOG(ERR, PMD, "hwrm_func_vf_vnic_query error %d\n", rc); + return -1; + } + rc = rte_le_to_cpu_32(resp->vnic_id_cnt); + + HWRM_UNLOCK(); + + return rc; +} + +/* + * This function queries the VNIC IDs for a specified VF. It then calls + * the vnic_cb to update the necessary field in vnic_info with cbdata. + * Then it calls the hwrm_cb function to program this new vnic configuration. + */ +int bnxt_hwrm_func_vf_vnic_query_and_config(struct bnxt *bp, uint16_t vf, + void (*vnic_cb)(struct bnxt_vnic_info *, void *), void *cbdata, + int (*hwrm_cb)(struct bnxt *bp, struct bnxt_vnic_info *vnic)) +{ + struct bnxt_vnic_info vnic; + int rc = 0; + int i, num_vnic_ids; + uint16_t *vnic_ids; + size_t vnic_id_sz; + size_t sz; + + /* First query all VNIC ids */ + vnic_id_sz = bp->pf.total_vnics * sizeof(*vnic_ids); + vnic_ids = rte_malloc("bnxt_hwrm_vf_vnic_ids_query", vnic_id_sz, + RTE_CACHE_LINE_SIZE); + if (vnic_ids == NULL) { + rc = -ENOMEM; + return rc; + } + for (sz = 0; sz < vnic_id_sz; sz += getpagesize()) + rte_mem_lock_page(((char *)vnic_ids) + sz); + + num_vnic_ids = bnxt_hwrm_func_vf_vnic_query(bp, vf, vnic_ids); + + if (num_vnic_ids < 0) + return num_vnic_ids; + + /* Retrieve VNIC, update bd_stall then update */ + + for (i = 0; i < num_vnic_ids; i++) { + memset(&vnic, 0, sizeof(struct bnxt_vnic_info)); + vnic.fw_vnic_id = rte_le_to_cpu_16(vnic_ids[i]); + rc = bnxt_hwrm_vnic_qcfg(bp, &vnic, bp->pf.first_vf_id + vf); + if (rc) + break; + if (vnic.mru <= 4) /* Indicates unallocated */ + continue; + + vnic_cb(&vnic, cbdata); + + rc = hwrm_cb(bp, &vnic); + if (rc) + break; + } + + rte_free(vnic_ids); + + return rc; +} + +int bnxt_hwrm_func_cfg_vf_set_vlan_anti_spoof(struct bnxt *bp, uint16_t vf, + bool on) +{ + struct hwrm_func_cfg_output *resp = bp->hwrm_cmd_resp_addr; + struct hwrm_func_cfg_input req = {0}; + int rc; + + HWRM_PREP(req, FUNC_CFG); + + req.fid = rte_cpu_to_le_16(bp->pf.vf_info[vf].fid); + req.enables |= rte_cpu_to_le_32( + HWRM_FUNC_CFG_INPUT_ENABLES_VLAN_ANTISPOOF_MODE); + req.vlan_antispoof_mode = on ? + HWRM_FUNC_CFG_INPUT_VLAN_ANTISPOOF_MODE_VALIDATE_VLAN : + HWRM_FUNC_CFG_INPUT_VLAN_ANTISPOOF_MODE_NOCHECK; + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + + HWRM_CHECK_RESULT(); + HWRM_UNLOCK(); + + return rc; +} + +int bnxt_hwrm_func_qcfg_vf_dflt_vnic_id(struct bnxt *bp, int vf) +{ + struct bnxt_vnic_info vnic; + uint16_t *vnic_ids; + size_t vnic_id_sz; + int num_vnic_ids, i; + size_t sz; + int rc; + + vnic_id_sz = bp->pf.total_vnics * sizeof(*vnic_ids); + vnic_ids = rte_malloc("bnxt_hwrm_vf_vnic_ids_query", vnic_id_sz, + RTE_CACHE_LINE_SIZE); + if (vnic_ids == NULL) { + rc = -ENOMEM; + return rc; + } + + for (sz = 0; sz < vnic_id_sz; sz += getpagesize()) + rte_mem_lock_page(((char *)vnic_ids) + sz); + + rc = bnxt_hwrm_func_vf_vnic_query(bp, vf, vnic_ids); + if (rc <= 0) + goto exit; + num_vnic_ids = rc; + + /* + * Loop through to find the default VNIC ID. + * TODO: The easier way would be to obtain the resp->dflt_vnic_id + * by sending the hwrm_func_qcfg command to the firmware. + */ + for (i = 0; i < num_vnic_ids; i++) { + memset(&vnic, 0, sizeof(struct bnxt_vnic_info)); + vnic.fw_vnic_id = rte_le_to_cpu_16(vnic_ids[i]); + rc = bnxt_hwrm_vnic_qcfg(bp, &vnic, + bp->pf.first_vf_id + vf); + if (rc) + goto exit; + if (vnic.func_default) { + rte_free(vnic_ids); + return vnic.fw_vnic_id; + } + } + /* Could not find a default VNIC. */ + RTE_LOG(ERR, PMD, "No default VNIC\n"); +exit: + rte_free(vnic_ids); + return -1; +} + +int bnxt_hwrm_set_em_filter(struct bnxt *bp, + uint16_t dst_id, + struct bnxt_filter_info *filter) +{ + int rc = 0; + struct hwrm_cfa_em_flow_alloc_input req = {.req_type = 0 }; + struct hwrm_cfa_em_flow_alloc_output *resp = bp->hwrm_cmd_resp_addr; + uint32_t enables = 0; + + if (filter->fw_em_filter_id != UINT64_MAX) + bnxt_hwrm_clear_em_filter(bp, filter); + + HWRM_PREP(req, CFA_EM_FLOW_ALLOC); + + req.flags = rte_cpu_to_le_32(filter->flags); + + enables = filter->enables | + HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_DST_ID; + req.dst_id = rte_cpu_to_le_16(dst_id); + + if (filter->ip_addr_type) { + req.ip_addr_type = filter->ip_addr_type; + enables |= HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_IPADDR_TYPE; + } + if (enables & + HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_L2_FILTER_ID) + req.l2_filter_id = rte_cpu_to_le_64(filter->fw_l2_filter_id); + if (enables & + HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_SRC_MACADDR) + memcpy(req.src_macaddr, filter->src_macaddr, + ETHER_ADDR_LEN); + if (enables & + HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_DST_MACADDR) + memcpy(req.dst_macaddr, filter->dst_macaddr, + ETHER_ADDR_LEN); + if (enables & + HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_OVLAN_VID) + req.ovlan_vid = filter->l2_ovlan; + if (enables & + HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_IVLAN_VID) + req.ivlan_vid = filter->l2_ivlan; + if (enables & + HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_ETHERTYPE) + req.ethertype = rte_cpu_to_be_16(filter->ethertype); + if (enables & + HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_IP_PROTOCOL) + req.ip_protocol = filter->ip_protocol; + if (enables & + HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_SRC_IPADDR) + req.src_ipaddr[0] = rte_cpu_to_be_32(filter->src_ipaddr[0]); + if (enables & + HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_DST_IPADDR) + req.dst_ipaddr[0] = rte_cpu_to_be_32(filter->dst_ipaddr[0]); + if (enables & + HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_SRC_PORT) + req.src_port = rte_cpu_to_be_16(filter->src_port); + if (enables & + HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_DST_PORT) + req.dst_port = rte_cpu_to_be_16(filter->dst_port); + if (enables & + HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_MIRROR_VNIC_ID) + req.mirror_vnic_id = filter->mirror_vnic_id; + + req.enables = rte_cpu_to_le_32(enables); + + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + + HWRM_CHECK_RESULT(); + + filter->fw_em_filter_id = rte_le_to_cpu_64(resp->em_filter_id); + HWRM_UNLOCK(); + + return rc; +} + +int bnxt_hwrm_clear_em_filter(struct bnxt *bp, struct bnxt_filter_info *filter) +{ + int rc = 0; + struct hwrm_cfa_em_flow_free_input req = {.req_type = 0 }; + struct hwrm_cfa_em_flow_free_output *resp = bp->hwrm_cmd_resp_addr; + + if (filter->fw_em_filter_id == UINT64_MAX) + return 0; + + RTE_LOG(ERR, PMD, "Clear EM filter\n"); + HWRM_PREP(req, CFA_EM_FLOW_FREE); + + req.em_filter_id = rte_cpu_to_le_64(filter->fw_em_filter_id); + + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + + HWRM_CHECK_RESULT(); + HWRM_UNLOCK(); + + filter->fw_em_filter_id = -1; + filter->fw_l2_filter_id = -1; + + return 0; +} + +int bnxt_hwrm_set_ntuple_filter(struct bnxt *bp, + uint16_t dst_id, + struct bnxt_filter_info *filter) +{ + int rc = 0; + struct hwrm_cfa_ntuple_filter_alloc_input req = {.req_type = 0 }; + struct hwrm_cfa_ntuple_filter_alloc_output *resp = + bp->hwrm_cmd_resp_addr; + uint32_t enables = 0; + + if (filter->fw_ntuple_filter_id != UINT64_MAX) + bnxt_hwrm_clear_ntuple_filter(bp, filter); + + HWRM_PREP(req, CFA_NTUPLE_FILTER_ALLOC); + + req.flags = rte_cpu_to_le_32(filter->flags); + + enables = filter->enables | + HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_DST_ID; + req.dst_id = rte_cpu_to_le_16(dst_id); + + + if (filter->ip_addr_type) { + req.ip_addr_type = filter->ip_addr_type; + enables |= + HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_IPADDR_TYPE; + } + if (enables & + HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_L2_FILTER_ID) + req.l2_filter_id = rte_cpu_to_le_64(filter->fw_l2_filter_id); + if (enables & + HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_SRC_MACADDR) + memcpy(req.src_macaddr, filter->src_macaddr, + ETHER_ADDR_LEN); + //if (enables & + //HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_DST_MACADDR) + //memcpy(req.dst_macaddr, filter->dst_macaddr, + //ETHER_ADDR_LEN); + if (enables & + HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_ETHERTYPE) + req.ethertype = rte_cpu_to_be_16(filter->ethertype); + if (enables & + HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_IP_PROTOCOL) + req.ip_protocol = filter->ip_protocol; + if (enables & + HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_SRC_IPADDR) + req.src_ipaddr[0] = rte_cpu_to_le_32(filter->src_ipaddr[0]); + if (enables & + HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_SRC_IPADDR_MASK) + req.src_ipaddr_mask[0] = + rte_cpu_to_le_32(filter->src_ipaddr_mask[0]); + if (enables & + HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_DST_IPADDR) + req.dst_ipaddr[0] = rte_cpu_to_le_32(filter->dst_ipaddr[0]); + if (enables & + HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_DST_IPADDR_MASK) + req.dst_ipaddr_mask[0] = + rte_cpu_to_be_32(filter->dst_ipaddr_mask[0]); + if (enables & + HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_SRC_PORT) + req.src_port = rte_cpu_to_le_16(filter->src_port); + if (enables & + HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_SRC_PORT_MASK) + req.src_port_mask = rte_cpu_to_le_16(filter->src_port_mask); + if (enables & + HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_DST_PORT) + req.dst_port = rte_cpu_to_le_16(filter->dst_port); + if (enables & + HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_DST_PORT_MASK) + req.dst_port_mask = rte_cpu_to_le_16(filter->dst_port_mask); + if (enables & + HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_MIRROR_VNIC_ID) + req.mirror_vnic_id = filter->mirror_vnic_id; + + req.enables = rte_cpu_to_le_32(enables); + + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + + HWRM_CHECK_RESULT(); + + filter->fw_ntuple_filter_id = rte_le_to_cpu_64(resp->ntuple_filter_id); + HWRM_UNLOCK(); + + return rc; +} + +int bnxt_hwrm_clear_ntuple_filter(struct bnxt *bp, + struct bnxt_filter_info *filter) +{ + int rc = 0; + struct hwrm_cfa_ntuple_filter_free_input req = {.req_type = 0 }; + struct hwrm_cfa_ntuple_filter_free_output *resp = + bp->hwrm_cmd_resp_addr; + + if (filter->fw_ntuple_filter_id == UINT64_MAX) + return 0; + + HWRM_PREP(req, CFA_NTUPLE_FILTER_FREE); + + req.ntuple_filter_id = rte_cpu_to_le_64(filter->fw_ntuple_filter_id); + + rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + + HWRM_CHECK_RESULT(); + HWRM_UNLOCK(); + + filter->fw_ntuple_filter_id = -1; + + return 0; +} diff --git a/dpdk/drivers/net/bnxt/bnxt_hwrm.h b/dpdk/drivers/net/bnxt/bnxt_hwrm.h index a508024f..85083e61 100644 --- a/dpdk/drivers/net/bnxt/bnxt_hwrm.h +++ b/dpdk/drivers/net/bnxt/bnxt_hwrm.h @@ -45,27 +45,42 @@ struct bnxt_cp_ring_info; int bnxt_hwrm_cfa_l2_clear_rx_mask(struct bnxt *bp, struct bnxt_vnic_info *vnic); -int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, struct bnxt_vnic_info *vnic); -int bnxt_hwrm_clear_filter(struct bnxt *bp, +int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, struct bnxt_vnic_info *vnic, + uint16_t vlan_count, + struct bnxt_vlan_table_entry *vlan_table); +int bnxt_hwrm_cfa_vlan_antispoof_cfg(struct bnxt *bp, uint16_t fid, + uint16_t vlan_count, + struct bnxt_vlan_antispoof_table_entry *vlan_table); +int bnxt_hwrm_clear_l2_filter(struct bnxt *bp, struct bnxt_filter_info *filter); -int bnxt_hwrm_set_filter(struct bnxt *bp, - struct bnxt_vnic_info *vnic, +int bnxt_hwrm_set_l2_filter(struct bnxt *bp, + uint16_t dst_id, struct bnxt_filter_info *filter); +int bnxt_hwrm_exec_fwd_resp(struct bnxt *bp, uint16_t target_id, + void *encaped, size_t ec_size); +int bnxt_hwrm_reject_fwd_resp(struct bnxt *bp, uint16_t target_id, + void *encaped, size_t ec_size); -int bnxt_hwrm_exec_fwd_resp(struct bnxt *bp, void *fwd_cmd); - -int bnxt_hwrm_func_driver_register(struct bnxt *bp, uint32_t flags, - uint32_t *vf_req_fwd); +int bnxt_hwrm_func_buf_rgtr(struct bnxt *bp); +int bnxt_hwrm_func_buf_unrgtr(struct bnxt *bp); +int bnxt_hwrm_func_driver_register(struct bnxt *bp); int bnxt_hwrm_func_qcaps(struct bnxt *bp); int bnxt_hwrm_func_reset(struct bnxt *bp); int bnxt_hwrm_func_driver_unregister(struct bnxt *bp, uint32_t flags); +int bnxt_hwrm_func_qstats(struct bnxt *bp, uint16_t fid, + struct rte_eth_stats *stats); +int bnxt_hwrm_func_qstats_tx_drop(struct bnxt *bp, uint16_t fid, + uint64_t *dropped); +int bnxt_hwrm_func_clr_stats(struct bnxt *bp, uint16_t fid); +int bnxt_hwrm_func_cfg_def_cp(struct bnxt *bp); +int bnxt_hwrm_vf_func_cfg_def_cp(struct bnxt *bp); int bnxt_hwrm_queue_qportcfg(struct bnxt *bp); int bnxt_hwrm_ring_alloc(struct bnxt *bp, struct bnxt_ring *ring, uint32_t ring_type, uint32_t map_index, - uint32_t stats_ctx_id); + uint32_t stats_ctx_id, uint32_t cmpl_ring_id); int bnxt_hwrm_ring_free(struct bnxt *bp, struct bnxt_ring *ring, uint32_t ring_type); int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp, unsigned int idx); @@ -76,16 +91,24 @@ int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, unsigned int idx); int bnxt_hwrm_stat_ctx_free(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, unsigned int idx); +int bnxt_hwrm_ctx_qstats(struct bnxt *bp, uint32_t cid, int idx, + struct rte_eth_stats *stats, uint8_t rx); int bnxt_hwrm_ver_get(struct bnxt *bp); int bnxt_hwrm_vnic_alloc(struct bnxt *bp, struct bnxt_vnic_info *vnic); int bnxt_hwrm_vnic_cfg(struct bnxt *bp, struct bnxt_vnic_info *vnic); +int bnxt_hwrm_vnic_qcfg(struct bnxt *bp, struct bnxt_vnic_info *vnic, + int16_t fw_vf_id); int bnxt_hwrm_vnic_ctx_alloc(struct bnxt *bp, struct bnxt_vnic_info *vnic); int bnxt_hwrm_vnic_ctx_free(struct bnxt *bp, struct bnxt_vnic_info *vnic); int bnxt_hwrm_vnic_free(struct bnxt *bp, struct bnxt_vnic_info *vnic); int bnxt_hwrm_vnic_rss_cfg(struct bnxt *bp, struct bnxt_vnic_info *vnic); +int bnxt_hwrm_vnic_plcmode_cfg(struct bnxt *bp, + struct bnxt_vnic_info *vnic); +int bnxt_hwrm_vnic_tpa_cfg(struct bnxt *bp, + struct bnxt_vnic_info *vnic, bool enable); int bnxt_alloc_all_hwrm_stat_ctxs(struct bnxt *bp); int bnxt_clear_all_hwrm_stat_ctxs(struct bnxt *bp); @@ -100,5 +123,56 @@ void bnxt_free_hwrm_resources(struct bnxt *bp); int bnxt_alloc_hwrm_resources(struct bnxt *bp); int bnxt_get_hwrm_link_config(struct bnxt *bp, struct rte_eth_link *link); int bnxt_set_hwrm_link_config(struct bnxt *bp, bool link_up); +int bnxt_hwrm_func_qcfg(struct bnxt *bp); +int bnxt_hwrm_allocate_pf_only(struct bnxt *bp); +int bnxt_hwrm_allocate_vfs(struct bnxt *bp, int num_vfs); +int bnxt_hwrm_func_vf_mac(struct bnxt *bp, uint16_t vf, + const uint8_t *mac_addr); +int bnxt_hwrm_pf_evb_mode(struct bnxt *bp); +int bnxt_hwrm_func_bw_cfg(struct bnxt *bp, uint16_t vf, + uint16_t max_bw, uint16_t enables); +int bnxt_hwrm_set_vf_vlan(struct bnxt *bp, int vf); +int bnxt_hwrm_func_qcfg_vf_default_mac(struct bnxt *bp, uint16_t vf, + struct ether_addr *mac); +int bnxt_hwrm_func_qcfg_current_vf_vlan(struct bnxt *bp, int vf); +int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, uint16_t port, + uint8_t tunnel_type); +int bnxt_hwrm_tunnel_dst_port_free(struct bnxt *bp, uint16_t port, + uint8_t tunnel_type); +void bnxt_free_tunnel_ports(struct bnxt *bp); +int bnxt_hwrm_set_default_vlan(struct bnxt *bp, int vf, uint8_t is_vf); +int bnxt_hwrm_port_qstats(struct bnxt *bp); +int bnxt_hwrm_port_clr_stats(struct bnxt *bp); +int bnxt_hwrm_port_led_cfg(struct bnxt *bp, bool led_on); +int bnxt_hwrm_port_led_qcaps(struct bnxt *bp); +int bnxt_hwrm_func_cfg_vf_set_flags(struct bnxt *bp, uint16_t vf, + uint32_t flags); +void vf_vnic_set_rxmask_cb(struct bnxt_vnic_info *vnic, void *flagp); +int bnxt_set_rx_mask_no_vlan(struct bnxt *bp, struct bnxt_vnic_info *vnic); +int bnxt_vf_vnic_count(struct bnxt *bp, uint16_t vf); +int bnxt_hwrm_func_vf_vnic_query_and_config(struct bnxt *bp, uint16_t vf, + void (*vnic_cb)(struct bnxt_vnic_info *, void *), void *cbdata, + int (*hwrm_cb)(struct bnxt *bp, struct bnxt_vnic_info *vnic)); +int bnxt_hwrm_func_cfg_vf_set_vlan_anti_spoof(struct bnxt *bp, uint16_t vf, + bool on); +int bnxt_hwrm_func_qcfg_vf_dflt_vnic_id(struct bnxt *bp, int vf); +int bnxt_hwrm_set_em_filter(struct bnxt *bp, uint16_t dst_id, + struct bnxt_filter_info *filter); +int bnxt_hwrm_clear_em_filter(struct bnxt *bp, struct bnxt_filter_info *filter); +int bnxt_hwrm_set_ntuple_filter(struct bnxt *bp, uint16_t dst_id, + struct bnxt_filter_info *filter); +int bnxt_hwrm_clear_ntuple_filter(struct bnxt *bp, + struct bnxt_filter_info *filter); +int bnxt_get_nvram_directory(struct bnxt *bp, uint32_t len, uint8_t *data); +int bnxt_hwrm_nvm_get_dir_info(struct bnxt *bp, uint32_t *entries, + uint32_t *length); +int bnxt_hwrm_get_nvram_item(struct bnxt *bp, uint32_t index, + uint32_t offset, uint32_t length, + uint8_t *data); +int bnxt_hwrm_erase_nvram_directory(struct bnxt *bp, uint8_t index); +int bnxt_hwrm_flash_nvram(struct bnxt *bp, uint16_t dir_type, + uint16_t dir_ordinal, uint16_t dir_ext, + uint16_t dir_attr, const uint8_t *data, + size_t data_len); #endif diff --git a/dpdk/drivers/net/bnxt/bnxt_irq.c b/dpdk/drivers/net/bnxt/bnxt_irq.c new file mode 100644 index 00000000..49436cfd --- /dev/null +++ b/dpdk/drivers/net/bnxt/bnxt_irq.c @@ -0,0 +1,173 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2014-2015 Broadcom Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Broadcom Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +#include + +#include "bnxt.h" +#include "bnxt_cpr.h" +#include "bnxt_irq.h" +#include "bnxt_ring.h" +#include "hsi_struct_def_dpdk.h" + +/* + * Interrupts + */ + +static void bnxt_int_handler(void *param) +{ + struct rte_eth_dev *eth_dev = (struct rte_eth_dev *)param; + struct bnxt *bp = (struct bnxt *)eth_dev->data->dev_private; + struct bnxt_cp_ring_info *cpr = bp->def_cp_ring; + struct cmpl_base *cmp; + uint32_t raw_cons; + uint32_t cons; + + if (cpr == NULL) + return; + + raw_cons = cpr->cp_raw_cons; + while (1) { + if (!cpr || !cpr->cp_ring_struct) + return; + + cons = RING_CMP(cpr->cp_ring_struct, raw_cons); + cmp = &cpr->cp_desc_ring[cons]; + + if (!CMP_VALID(cmp, raw_cons, cpr->cp_ring_struct)) + break; + + switch (CMP_TYPE(cmp)) { + case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT: + /* Handle any async event */ + bnxt_handle_async_event(bp, cmp); + break; + case CMPL_BASE_TYPE_HWRM_FWD_REQ: + /* Handle HWRM forwarded responses */ + bnxt_handle_fwd_req(bp, cmp); + break; + default: + /* Ignore any other events */ + if (cmp->type & rte_cpu_to_le_16(0x01)) { + if (!CMP_VALID(cmp, raw_cons, + cpr->cp_ring_struct)) + goto no_more; + } + RTE_LOG(INFO, PMD, + "Ignoring %02x completion\n", CMP_TYPE(cmp)); + break; + } + raw_cons = NEXT_RAW_CMP(raw_cons); + + }; +no_more: + cpr->cp_raw_cons = raw_cons; + B_CP_DB_REARM(cpr, cpr->cp_raw_cons); +} + +void bnxt_free_int(struct bnxt *bp) +{ + struct bnxt_irq *irq; + + irq = bp->irq_tbl; + if (irq) { + if (irq->requested) { + rte_intr_disable(&bp->pdev->intr_handle); + rte_intr_callback_unregister(&bp->pdev->intr_handle, + irq->handler, + (void *)bp->eth_dev); + irq->requested = 0; + } + rte_free((void *)bp->irq_tbl); + bp->irq_tbl = NULL; + } +} + +void bnxt_disable_int(struct bnxt *bp) +{ + struct bnxt_cp_ring_info *cpr = bp->def_cp_ring; + + /* Only the default completion ring */ + if (cpr != NULL && cpr->cp_doorbell != NULL) + B_CP_DB_DISARM(cpr); +} + +void bnxt_enable_int(struct bnxt *bp) +{ + struct bnxt_cp_ring_info *cpr = bp->def_cp_ring; + + B_CP_DB_ARM(cpr); +} + +int bnxt_setup_int(struct bnxt *bp) +{ + uint16_t total_vecs; + const int len = sizeof(bp->irq_tbl[0].name); + int i, rc = 0; + + /* DPDK host only supports 1 MSI-X vector */ + total_vecs = 1; + bp->irq_tbl = rte_calloc("bnxt_irq_tbl", total_vecs, + sizeof(struct bnxt_irq), 0); + if (bp->irq_tbl) { + for (i = 0; i < total_vecs; i++) { + bp->irq_tbl[i].vector = i; + snprintf(bp->irq_tbl[i].name, len, + "%s-%d", bp->eth_dev->device->name, i); + bp->irq_tbl[i].handler = bnxt_int_handler; + } + } else { + rc = -ENOMEM; + goto setup_exit; + } + return 0; + +setup_exit: + RTE_LOG(ERR, PMD, "bnxt_irq_tbl setup failed\n"); + return rc; +} + +int bnxt_request_int(struct bnxt *bp) +{ + int rc = 0; + + struct bnxt_irq *irq = bp->irq_tbl; + + rte_intr_callback_register(&bp->pdev->intr_handle, irq->handler, + (void *)bp->eth_dev); + rte_intr_enable(&bp->pdev->intr_handle); + + irq->requested = 1; + return rc; +} diff --git a/dpdk/drivers/net/bnxt/bnxt_irq.h b/dpdk/drivers/net/bnxt/bnxt_irq.h new file mode 100644 index 00000000..4d2f7af9 --- /dev/null +++ b/dpdk/drivers/net/bnxt/bnxt_irq.h @@ -0,0 +1,54 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2014-2015 Broadcom Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Broadcom Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _BNXT_IRQ_H_ +#define _BNXT_IRQ_H_ + +#define BNXT_MISC_VEC_ID RTE_INTR_VEC_ZERO_OFFSET +#define BNXT_RX_VEC_START RTE_INTR_VEC_RXTX_OFFSET + +struct bnxt_irq { + rte_intr_callback_fn handler; + unsigned int vector; + uint8_t requested; + char name[RTE_ETH_NAME_MAX_LEN + 2]; +}; + +struct bnxt; +void bnxt_free_int(struct bnxt *bp); +void bnxt_disable_int(struct bnxt *bp); +void bnxt_enable_int(struct bnxt *bp); +int bnxt_setup_int(struct bnxt *bp); +int bnxt_request_int(struct bnxt *bp); + +#endif diff --git a/dpdk/drivers/net/bnxt/bnxt_nvm_defs.h b/dpdk/drivers/net/bnxt/bnxt_nvm_defs.h new file mode 100644 index 00000000..c5ccc9bc --- /dev/null +++ b/dpdk/drivers/net/bnxt/bnxt_nvm_defs.h @@ -0,0 +1,75 @@ +/* Broadcom NetXtreme-C/E network driver. + * + * Copyright (c) 2014-2016 Broadcom Corporation + * Copyright (c) 2016-2017 Broadcom Limited + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation. + */ + +#ifndef _BNXT_NVM_DEFS_H_ +#define _BNXT_NVM_DEFS_H_ + +enum bnxt_nvm_directory_type { + BNX_DIR_TYPE_UNUSED = 0, + BNX_DIR_TYPE_PKG_LOG = 1, + BNX_DIR_TYPE_UPDATE = 2, + BNX_DIR_TYPE_CHIMP_PATCH = 3, + BNX_DIR_TYPE_BOOTCODE = 4, + BNX_DIR_TYPE_VPD = 5, + BNX_DIR_TYPE_EXP_ROM_MBA = 6, + BNX_DIR_TYPE_AVS = 7, + BNX_DIR_TYPE_PCIE = 8, + BNX_DIR_TYPE_PORT_MACRO = 9, + BNX_DIR_TYPE_APE_FW = 10, + BNX_DIR_TYPE_APE_PATCH = 11, + BNX_DIR_TYPE_KONG_FW = 12, + BNX_DIR_TYPE_KONG_PATCH = 13, + BNX_DIR_TYPE_BONO_FW = 14, + BNX_DIR_TYPE_BONO_PATCH = 15, + BNX_DIR_TYPE_TANG_FW = 16, + BNX_DIR_TYPE_TANG_PATCH = 17, + BNX_DIR_TYPE_BOOTCODE_2 = 18, + BNX_DIR_TYPE_CCM = 19, + BNX_DIR_TYPE_PCI_CFG = 20, + BNX_DIR_TYPE_TSCF_UCODE = 21, + BNX_DIR_TYPE_ISCSI_BOOT = 22, + BNX_DIR_TYPE_ISCSI_BOOT_IPV6 = 24, + BNX_DIR_TYPE_ISCSI_BOOT_IPV4N6 = 25, + BNX_DIR_TYPE_ISCSI_BOOT_CFG6 = 26, + BNX_DIR_TYPE_EXT_PHY = 27, + BNX_DIR_TYPE_SHARED_CFG = 40, + BNX_DIR_TYPE_PORT_CFG = 41, + BNX_DIR_TYPE_FUNC_CFG = 42, + BNX_DIR_TYPE_MGMT_CFG = 48, + BNX_DIR_TYPE_MGMT_DATA = 49, + BNX_DIR_TYPE_MGMT_WEB_DATA = 50, + BNX_DIR_TYPE_MGMT_WEB_META = 51, + BNX_DIR_TYPE_MGMT_EVENT_LOG = 52, + BNX_DIR_TYPE_MGMT_AUDIT_LOG = 53 +}; + +#define BNX_DIR_ORDINAL_FIRST 0 + +#define BNX_DIR_EXT_NONE 0 +#define BNX_DIR_EXT_INACTIVE (1 << 0) +#define BNX_DIR_EXT_UPDATE (1 << 1) + +#define BNX_DIR_ATTR_NONE 0 +#define BNX_DIR_ATTR_NO_CHKSUM (1 << 0) +#define BNX_DIR_ATTR_PROP_STREAM (1 << 1) + +#define BNX_PKG_LOG_MAX_LENGTH 4096 + +enum bnxnvm_pkglog_field_index { + BNX_PKG_LOG_FIELD_IDX_INSTALLED_TIMESTAMP = 0, + BNX_PKG_LOG_FIELD_IDX_PKG_DESCRIPTION = 1, + BNX_PKG_LOG_FIELD_IDX_PKG_VERSION = 2, + BNX_PKG_LOG_FIELD_IDX_PKG_TIMESTAMP = 3, + BNX_PKG_LOG_FIELD_IDX_PKG_CHECKSUM = 4, + BNX_PKG_LOG_FIELD_IDX_INSTALLED_ITEMS = 5, + BNX_PKG_LOG_FIELD_IDX_INSTALLED_MASK = 6 +}; + +#endif /* Don't add anything after this line */ diff --git a/dpdk/drivers/net/bnxt/bnxt_ring.c b/dpdk/drivers/net/bnxt/bnxt_ring.c index 3f81ffcc..59d1035f 100644 --- a/dpdk/drivers/net/bnxt/bnxt_ring.c +++ b/dpdk/drivers/net/bnxt/bnxt_ring.c @@ -31,7 +31,9 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +#include #include +#include #include "bnxt.h" #include "bnxt_cpr.h" @@ -61,13 +63,15 @@ void bnxt_free_ring(struct bnxt_ring *ring) * Ring groups */ -void bnxt_init_ring_grps(struct bnxt *bp) +int bnxt_init_ring_grps(struct bnxt *bp) { unsigned int i; for (i = 0; i < bp->max_ring_grps; i++) memset(&bp->grp_info[i], (uint8_t)HWRM_NA_SIGNATURE, sizeof(struct bnxt_ring_grp_info)); + + return 0; } /* @@ -96,6 +100,8 @@ int bnxt_alloc_rings(struct bnxt *bp, uint16_t qidx, struct rte_pci_device *pdev = bp->pdev; const struct rte_memzone *mz = NULL; char mz_name[RTE_MEMZONE_NAMESIZE]; + rte_iova_t mz_phys_addr; + int sz; int stats_len = (tx_ring_info || rx_ring_info) ? RTE_CACHE_LINE_ROUNDUP(sizeof(struct ctx_hw_stats64)) : 0; @@ -112,8 +118,15 @@ int bnxt_alloc_rings(struct bnxt *bp, uint16_t qidx, int rx_vmem_len = rx_ring_info ? RTE_CACHE_LINE_ROUNDUP(rx_ring_info-> rx_ring_struct->vmem_size) : 0; + int ag_vmem_start = 0; + int ag_vmem_len = 0; + int cp_ring_start = 0; + + ag_vmem_start = rx_vmem_start + rx_vmem_len; + ag_vmem_len = rx_ring_info ? RTE_CACHE_LINE_ROUNDUP( + rx_ring_info->ag_ring_struct->vmem_size) : 0; + cp_ring_start = ag_vmem_start + ag_vmem_len; - int cp_ring_start = rx_vmem_start + rx_vmem_len; int cp_ring_len = RTE_CACHE_LINE_ROUNDUP(cp_ring->ring_size * sizeof(struct cmpl_base)); @@ -127,7 +140,23 @@ int bnxt_alloc_rings(struct bnxt *bp, uint16_t qidx, RTE_CACHE_LINE_ROUNDUP(rx_ring_info->rx_ring_struct->ring_size * sizeof(struct rx_prod_pkt_bd)) : 0; - int total_alloc_len = rx_ring_start + rx_ring_len; + int ag_ring_start = rx_ring_start + rx_ring_len; + int ag_ring_len = rx_ring_len * AGG_RING_SIZE_FACTOR; + + int ag_bitmap_start = ag_ring_start + ag_ring_len; + int ag_bitmap_len = rx_ring_info ? + RTE_CACHE_LINE_ROUNDUP(rte_bitmap_get_memory_footprint( + rx_ring_info->rx_ring_struct->ring_size * + AGG_RING_SIZE_FACTOR)) : 0; + + int tpa_info_start = ag_bitmap_start + ag_bitmap_len; + int tpa_info_len = rx_ring_info ? + RTE_CACHE_LINE_ROUNDUP(BNXT_TPA_MAX * + sizeof(struct bnxt_tpa_info)) : 0; + + int total_alloc_len = tpa_info_start; + if (bp->eth_dev->data->dev_conf.rxmode.enable_lro) + total_alloc_len += tpa_info_len; snprintf(mz_name, RTE_MEMZONE_NAMESIZE, "bnxt_%04x:%02x:%02x:%02x-%04x_%s", pdev->addr.domain, @@ -136,21 +165,37 @@ int bnxt_alloc_rings(struct bnxt *bp, uint16_t qidx, mz_name[RTE_MEMZONE_NAMESIZE - 1] = 0; mz = rte_memzone_lookup(mz_name); if (!mz) { - mz = rte_memzone_reserve(mz_name, total_alloc_len, + mz = rte_memzone_reserve_aligned(mz_name, total_alloc_len, SOCKET_ID_ANY, RTE_MEMZONE_2MB | - RTE_MEMZONE_SIZE_HINT_ONLY); + RTE_MEMZONE_SIZE_HINT_ONLY, + getpagesize()); if (mz == NULL) return -ENOMEM; } memset(mz->addr, 0, mz->len); + mz_phys_addr = mz->iova; + if ((unsigned long)mz->addr == mz_phys_addr) { + RTE_LOG(WARNING, PMD, + "Memzone physical address same as virtual.\n"); + RTE_LOG(WARNING, PMD, + "Using rte_mem_virt2iova()\n"); + for (sz = 0; sz < total_alloc_len; sz += getpagesize()) + rte_mem_lock_page(((char *)mz->addr) + sz); + mz_phys_addr = rte_mem_virt2iova(mz->addr); + if (mz_phys_addr == 0) { + RTE_LOG(ERR, PMD, + "unable to map ring address to physical memory\n"); + return -ENOMEM; + } + } if (tx_ring_info) { tx_ring = tx_ring_info->tx_ring_struct; tx_ring->bd = ((char *)mz->addr + tx_ring_start); tx_ring_info->tx_desc_ring = (struct tx_bd_long *)tx_ring->bd; - tx_ring->bd_dma = mz->phys_addr + tx_ring_start; + tx_ring->bd_dma = mz_phys_addr + tx_ring_start; tx_ring_info->tx_desc_mapping = tx_ring->bd_dma; tx_ring->mem_zone = (const void *)mz; @@ -170,7 +215,7 @@ int bnxt_alloc_rings(struct bnxt *bp, uint16_t qidx, rx_ring->bd = ((char *)mz->addr + rx_ring_start); rx_ring_info->rx_desc_ring = (struct rx_prod_pkt_bd *)rx_ring->bd; - rx_ring->bd_dma = mz->phys_addr + rx_ring_start; + rx_ring->bd_dma = mz_phys_addr + rx_ring_start; rx_ring_info->rx_desc_mapping = rx_ring->bd_dma; rx_ring->mem_zone = (const void *)mz; @@ -182,10 +227,39 @@ int bnxt_alloc_rings(struct bnxt *bp, uint16_t qidx, rx_ring_info->rx_buf_ring = (struct bnxt_sw_rx_bd *)rx_ring->vmem; } + + rx_ring = rx_ring_info->ag_ring_struct; + + rx_ring->bd = ((char *)mz->addr + ag_ring_start); + rx_ring_info->ag_desc_ring = + (struct rx_prod_pkt_bd *)rx_ring->bd; + rx_ring->bd_dma = mz->iova + ag_ring_start; + rx_ring_info->ag_desc_mapping = rx_ring->bd_dma; + rx_ring->mem_zone = (const void *)mz; + + if (!rx_ring->bd) + return -ENOMEM; + if (rx_ring->vmem_size) { + rx_ring->vmem = + (void **)((char *)mz->addr + ag_vmem_start); + rx_ring_info->ag_buf_ring = + (struct bnxt_sw_rx_bd *)rx_ring->vmem; + } + + rx_ring_info->ag_bitmap = + rte_bitmap_init(rx_ring_info->rx_ring_struct->ring_size * + AGG_RING_SIZE_FACTOR, (uint8_t *)mz->addr + + ag_bitmap_start, ag_bitmap_len); + + /* TPA info */ + if (bp->eth_dev->data->dev_conf.rxmode.enable_lro) + rx_ring_info->tpa_info = + ((struct bnxt_tpa_info *)((char *)mz->addr + + tpa_info_start)); } cp_ring->bd = ((char *)mz->addr + cp_ring_start); - cp_ring->bd_dma = mz->phys_addr + cp_ring_start; + cp_ring->bd_dma = mz_phys_addr + cp_ring_start; cp_ring_info->cp_desc_ring = cp_ring->bd; cp_ring_info->cp_desc_mapping = cp_ring->bd_dma; cp_ring->mem_zone = (const void *)mz; @@ -196,7 +270,7 @@ int bnxt_alloc_rings(struct bnxt *bp, uint16_t qidx, *cp_ring->vmem = ((char *)mz->addr + stats_len); if (stats_len) { cp_ring_info->hw_stats = mz->addr; - cp_ring_info->hw_stats_map = mz->phys_addr; + cp_ring_info->hw_stats_map = mz_phys_addr; } cp_ring_info->hw_stats_ctx_id = HWRM_NA_SIGNATURE; return 0; @@ -209,25 +283,10 @@ int bnxt_alloc_rings(struct bnxt *bp, uint16_t qidx, */ int bnxt_alloc_hwrm_rings(struct bnxt *bp) { + struct rte_pci_device *pci_dev = bp->pdev; unsigned int i; int rc = 0; - /* Default completion ring */ - { - struct bnxt_cp_ring_info *cpr = bp->def_cp_ring; - struct bnxt_ring *cp_ring = cpr->cp_ring_struct; - - rc = bnxt_hwrm_ring_alloc(bp, cp_ring, - HWRM_RING_ALLOC_INPUT_RING_TYPE_CMPL, - 0, HWRM_NA_SIGNATURE); - if (rc) - goto err_out; - cpr->cp_doorbell = - (char *)bp->eth_dev->pci_dev->mem_resource[2].addr; - B_CP_DIS_DB(cpr, cpr->cp_raw_cons); - bp->grp_info[0].cp_fw_ring_id = cp_ring->fw_ring_id; - } - for (i = 0; i < bp->rx_cp_nr_rings; i++) { struct bnxt_rx_queue *rxq = bp->rx_queues[i]; struct bnxt_cp_ring_info *cpr = rxq->cp_ring; @@ -235,37 +294,66 @@ int bnxt_alloc_hwrm_rings(struct bnxt *bp) struct bnxt_rx_ring_info *rxr = rxq->rx_ring; struct bnxt_ring *ring = rxr->rx_ring_struct; unsigned int idx = i + 1; + unsigned int map_idx = idx + bp->rx_cp_nr_rings; + + bp->grp_info[i].fw_stats_ctx = cpr->hw_stats_ctx_id; /* Rx cmpl */ rc = bnxt_hwrm_ring_alloc(bp, cp_ring, - HWRM_RING_ALLOC_INPUT_RING_TYPE_CMPL, - idx, HWRM_NA_SIGNATURE); + HWRM_RING_ALLOC_INPUT_RING_TYPE_L2_CMPL, + idx, HWRM_NA_SIGNATURE, + HWRM_NA_SIGNATURE); if (rc) goto err_out; - cpr->cp_doorbell = - (char *)bp->eth_dev->pci_dev->mem_resource[2].addr + + cpr->cp_doorbell = (char *)pci_dev->mem_resource[2].addr + idx * 0x80; - bp->grp_info[idx].cp_fw_ring_id = cp_ring->fw_ring_id; + bp->grp_info[i].cp_fw_ring_id = cp_ring->fw_ring_id; B_CP_DIS_DB(cpr, cpr->cp_raw_cons); /* Rx ring */ rc = bnxt_hwrm_ring_alloc(bp, ring, HWRM_RING_ALLOC_INPUT_RING_TYPE_RX, - idx, cpr->hw_stats_ctx_id); + idx, cpr->hw_stats_ctx_id, + cp_ring->fw_ring_id); if (rc) goto err_out; rxr->rx_prod = 0; - rxr->rx_doorbell = - (char *)bp->eth_dev->pci_dev->mem_resource[2].addr + + rxr->rx_doorbell = (char *)pci_dev->mem_resource[2].addr + idx * 0x80; - bp->grp_info[idx].rx_fw_ring_id = ring->fw_ring_id; + bp->grp_info[i].rx_fw_ring_id = ring->fw_ring_id; B_RX_DB(rxr->rx_doorbell, rxr->rx_prod); + + ring = rxr->ag_ring_struct; + /* Agg ring */ + if (ring == NULL) { + RTE_LOG(ERR, PMD, "Alloc AGG Ring is NULL!\n"); + goto err_out; + } + + rc = bnxt_hwrm_ring_alloc(bp, ring, + HWRM_RING_ALLOC_INPUT_RING_TYPE_RX, + map_idx, HWRM_NA_SIGNATURE, + cp_ring->fw_ring_id); + if (rc) + goto err_out; + RTE_LOG(DEBUG, PMD, "Alloc AGG Done!\n"); + rxr->ag_prod = 0; + rxr->ag_doorbell = + (char *)pci_dev->mem_resource[2].addr + + map_idx * 0x80; + bp->grp_info[i].ag_fw_ring_id = ring->fw_ring_id; + B_RX_DB(rxr->ag_doorbell, rxr->ag_prod); + + rxq->rx_buf_use_size = BNXT_MAX_MTU + ETHER_HDR_LEN + + ETHER_CRC_LEN + (2 * VLAN_TAG_SIZE); if (bnxt_init_one_rx_ring(rxq)) { - RTE_LOG(ERR, PMD, "bnxt_init_one_rx_ring failed!"); + RTE_LOG(ERR, PMD, "bnxt_init_one_rx_ring failed!\n"); bnxt_rx_queue_release_op(rxq); return -ENOMEM; } B_RX_DB(rxr->rx_doorbell, rxr->rx_prod); + B_RX_DB(rxr->ag_doorbell, rxr->ag_prod); + rxq->index = idx; } for (i = 0; i < bp->tx_cp_nr_rings; i++) { @@ -274,31 +362,31 @@ int bnxt_alloc_hwrm_rings(struct bnxt *bp) struct bnxt_ring *cp_ring = cpr->cp_ring_struct; struct bnxt_tx_ring_info *txr = txq->tx_ring; struct bnxt_ring *ring = txr->tx_ring_struct; - unsigned int idx = 1 + bp->rx_cp_nr_rings + i; + unsigned int idx = i + 1 + bp->rx_cp_nr_rings; /* Tx cmpl */ rc = bnxt_hwrm_ring_alloc(bp, cp_ring, - HWRM_RING_ALLOC_INPUT_RING_TYPE_CMPL, - idx, HWRM_NA_SIGNATURE); + HWRM_RING_ALLOC_INPUT_RING_TYPE_L2_CMPL, + idx, HWRM_NA_SIGNATURE, + HWRM_NA_SIGNATURE); if (rc) goto err_out; - cpr->cp_doorbell = - (char *)bp->eth_dev->pci_dev->mem_resource[2].addr + + cpr->cp_doorbell = (char *)pci_dev->mem_resource[2].addr + idx * 0x80; - bp->grp_info[idx].cp_fw_ring_id = cp_ring->fw_ring_id; B_CP_DIS_DB(cpr, cpr->cp_raw_cons); /* Tx ring */ rc = bnxt_hwrm_ring_alloc(bp, ring, HWRM_RING_ALLOC_INPUT_RING_TYPE_TX, - idx, cpr->hw_stats_ctx_id); + idx, cpr->hw_stats_ctx_id, + cp_ring->fw_ring_id); if (rc) goto err_out; - txr->tx_doorbell = - (char *)bp->eth_dev->pci_dev->mem_resource[2].addr + + txr->tx_doorbell = (char *)pci_dev->mem_resource[2].addr + idx * 0x80; + txq->index = idx; } err_out: diff --git a/dpdk/drivers/net/bnxt/bnxt_ring.h b/dpdk/drivers/net/bnxt/bnxt_ring.h index 8656549a..a88ab55f 100644 --- a/dpdk/drivers/net/bnxt/bnxt_ring.h +++ b/dpdk/drivers/net/bnxt/bnxt_ring.h @@ -41,7 +41,7 @@ #define RING_NEXT(ring, idx) (((idx) + 1) & (ring)->ring_mask) #define RTE_MBUF_DATA_DMA_ADDR(mb) \ - ((uint64_t)((mb)->buf_physaddr + (mb)->data_off)) + ((uint64_t)((mb)->buf_iova + (mb)->data_off)) #define DB_IDX_MASK 0xffffff #define DB_IDX_VALID (0x1 << 26) @@ -57,7 +57,8 @@ #define DEFAULT_RX_RING_SIZE 256 #define DEFAULT_TX_RING_SIZE 256 -#define MAX_TPA 128 +#define BNXT_TPA_MAX 64 +#define AGG_RING_SIZE_FACTOR 2 /* These assume 4k pages */ #define MAX_RX_DESC_CNT (8 * 1024) @@ -65,10 +66,11 @@ #define MAX_CP_DESC_CNT (16 * 1024) #define INVALID_HW_RING_ID ((uint16_t)-1) +#define INVALID_STATS_CTX_ID ((uint16_t)-1) struct bnxt_ring { void *bd; - phys_addr_t bd_dma; + rte_iova_t bd_dma; uint32_t ring_size; uint32_t ring_mask; @@ -92,7 +94,7 @@ struct bnxt_tx_ring_info; struct bnxt_rx_ring_info; struct bnxt_cp_ring_info; void bnxt_free_ring(struct bnxt_ring *ring); -void bnxt_init_ring_grps(struct bnxt *bp); +int bnxt_init_ring_grps(struct bnxt *bp); int bnxt_alloc_rings(struct bnxt *bp, uint16_t qidx, struct bnxt_tx_ring_info *tx_ring_info, struct bnxt_rx_ring_info *rx_ring_info, diff --git a/dpdk/drivers/net/bnxt/bnxt_rxq.c b/dpdk/drivers/net/bnxt/bnxt_rxq.c index cddf17d5..b4e9f38d 100644 --- a/dpdk/drivers/net/bnxt/bnxt_rxq.c +++ b/dpdk/drivers/net/bnxt/bnxt_rxq.c @@ -60,10 +60,13 @@ void bnxt_free_rxq_stats(struct bnxt_rx_queue *rxq) int bnxt_mq_rx_configure(struct bnxt *bp) { struct rte_eth_conf *dev_conf = &bp->eth_dev->data->dev_conf; - unsigned int i, j, nb_q_per_grp, ring_idx; - int start_grp_id, end_grp_id, rc = 0; + const struct rte_eth_vmdq_rx_conf *conf = + &dev_conf->rx_adv_conf.vmdq_rx_conf; + unsigned int i, j, nb_q_per_grp = 1, ring_idx = 0; + int start_grp_id, end_grp_id = 1, rc = 0; struct bnxt_vnic_info *vnic; struct bnxt_filter_info *filter; + enum rte_eth_nb_pools pools = bp->rx_cp_nr_rings, max_pools = 0; struct bnxt_rx_queue *rxq; bp->nr_vnics = 0; @@ -76,6 +79,7 @@ int bnxt_mq_rx_configure(struct bnxt *bp) rc = -ENOMEM; goto err_out; } + vnic->flags |= BNXT_VNIC_INFO_BCAST; STAILQ_INSERT_TAIL(&bp->ff_pool[0], vnic, next); bp->nr_vnics++; @@ -84,9 +88,8 @@ int bnxt_mq_rx_configure(struct bnxt *bp) vnic->func_default = true; vnic->ff_pool_idx = 0; - vnic->start_grp_id = 1; - vnic->end_grp_id = vnic->start_grp_id + - bp->rx_cp_nr_rings - 1; + vnic->start_grp_id = 0; + vnic->end_grp_id = vnic->start_grp_id; filter = bnxt_alloc_filter(bp); if (!filter) { RTE_LOG(ERR, PMD, "L2 filter alloc failed\n"); @@ -98,113 +101,125 @@ int bnxt_mq_rx_configure(struct bnxt *bp) } /* Multi-queue mode */ - if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG) { + if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_DCB_RSS) { /* VMDq ONLY, VMDq+RSS, VMDq+DCB, VMDq+DCB+RSS */ - enum rte_eth_nb_pools pools; switch (dev_conf->rxmode.mq_mode) { case ETH_MQ_RX_VMDQ_RSS: case ETH_MQ_RX_VMDQ_ONLY: - { - const struct rte_eth_vmdq_rx_conf *conf = - &dev_conf->rx_adv_conf.vmdq_rx_conf; - - /* ETH_8/64_POOLs */ - pools = conf->nb_queue_pools; - break; - } + /* ETH_8/64_POOLs */ + pools = conf->nb_queue_pools; + /* For each pool, allocate MACVLAN CFA rule & VNIC */ + max_pools = RTE_MIN(bp->max_vnics, + RTE_MIN(bp->max_l2_ctx, + RTE_MIN(bp->max_rsscos_ctx, + ETH_64_POOLS))); + if (pools > max_pools) + pools = max_pools; + break; + case ETH_MQ_RX_RSS: + pools = 1; + break; default: RTE_LOG(ERR, PMD, "Unsupported mq_mod %d\n", dev_conf->rxmode.mq_mode); rc = -EINVAL; goto err_out; } - /* For each pool, allocate MACVLAN CFA rule & VNIC */ - if (!pools) { - RTE_LOG(ERR, PMD, - "VMDq pool not set, defaulted to 64\n"); - pools = ETH_64_POOLS; + } + + nb_q_per_grp = bp->rx_cp_nr_rings / pools; + start_grp_id = 0; + end_grp_id = nb_q_per_grp; + + for (i = 0; i < pools; i++) { + vnic = bnxt_alloc_vnic(bp); + if (!vnic) { + RTE_LOG(ERR, PMD, "VNIC alloc failed\n"); + rc = -ENOMEM; + goto err_out; } - nb_q_per_grp = bp->rx_cp_nr_rings / pools; - start_grp_id = 1; - end_grp_id = start_grp_id + nb_q_per_grp - 1; + vnic->flags |= BNXT_VNIC_INFO_BCAST; + STAILQ_INSERT_TAIL(&bp->ff_pool[i], vnic, next); + bp->nr_vnics++; - ring_idx = 0; - for (i = 0; i < pools; i++) { - vnic = bnxt_alloc_vnic(bp); - if (!vnic) { - RTE_LOG(ERR, PMD, - "VNIC alloc failed\n"); - rc = -ENOMEM; - goto err_out; - } - STAILQ_INSERT_TAIL(&bp->ff_pool[i], vnic, next); - bp->nr_vnics++; - - for (j = 0; j < nb_q_per_grp; j++, ring_idx++) { - rxq = bp->eth_dev->data->rx_queues[ring_idx]; - rxq->vnic = vnic; - } - if (i == 0) - vnic->func_default = true; - vnic->ff_pool_idx = i; - vnic->start_grp_id = start_grp_id; - vnic->end_grp_id = end_grp_id; - - filter = bnxt_alloc_filter(bp); - if (!filter) { - RTE_LOG(ERR, PMD, - "L2 filter alloc failed\n"); - rc = -ENOMEM; - goto err_out; - } - /* - * TODO: Configure & associate CFA rule for - * each VNIC for each VMDq with MACVLAN, MACVLAN+TC - */ - STAILQ_INSERT_TAIL(&vnic->filter, filter, next); - - start_grp_id = end_grp_id + 1; - end_grp_id += nb_q_per_grp; + for (j = 0; j < nb_q_per_grp; j++, ring_idx++) { + rxq = bp->eth_dev->data->rx_queues[ring_idx]; + rxq->vnic = vnic; } - goto out; - } + if (i == 0) { + if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_DCB) { + bp->eth_dev->data->promiscuous = 1; + vnic->flags |= BNXT_VNIC_INFO_PROMISC; + } + vnic->func_default = true; + } + vnic->ff_pool_idx = i; + vnic->start_grp_id = start_grp_id; + vnic->end_grp_id = end_grp_id; - /* Non-VMDq mode - RSS, DCB, RSS+DCB */ - /* Init default VNIC for RSS or DCB only */ - vnic = bnxt_alloc_vnic(bp); - if (!vnic) { - RTE_LOG(ERR, PMD, "VNIC alloc failed\n"); - rc = -ENOMEM; - goto err_out; - } - /* Partition the rx queues for the single pool */ - for (i = 0; i < bp->rx_cp_nr_rings; i++) { - rxq = bp->eth_dev->data->rx_queues[i]; - rxq->vnic = vnic; - } - STAILQ_INSERT_TAIL(&bp->ff_pool[0], vnic, next); - bp->nr_vnics++; + if (i) { + if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_DCB || + !(dev_conf->rxmode.mq_mode & ETH_MQ_RX_RSS)) + vnic->rss_dflt_cr = true; + goto skip_filter_allocation; + } + filter = bnxt_alloc_filter(bp); + if (!filter) { + RTE_LOG(ERR, PMD, "L2 filter alloc failed\n"); + rc = -ENOMEM; + goto err_out; + } + /* + * TODO: Configure & associate CFA rule for + * each VNIC for each VMDq with MACVLAN, MACVLAN+TC + */ + STAILQ_INSERT_TAIL(&vnic->filter, filter, next); - vnic->func_default = true; - vnic->ff_pool_idx = 0; - vnic->start_grp_id = 1; - vnic->end_grp_id = vnic->start_grp_id + - bp->rx_cp_nr_rings - 1; - filter = bnxt_alloc_filter(bp); - if (!filter) { - RTE_LOG(ERR, PMD, "L2 filter alloc failed\n"); - rc = -ENOMEM; - goto err_out; +skip_filter_allocation: + start_grp_id = end_grp_id; + end_grp_id += nb_q_per_grp; } - STAILQ_INSERT_TAIL(&vnic->filter, filter, next); - - if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) - vnic->hash_type = - HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_IPV4 | - HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_IPV6; out: + if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) { + struct rte_eth_rss_conf *rss = &dev_conf->rx_adv_conf.rss_conf; + uint16_t hash_type = 0; + + if (bp->flags & BNXT_FLAG_UPDATE_HASH) { + rss = &bp->rss_conf; + bp->flags &= ~BNXT_FLAG_UPDATE_HASH; + } + + if (rss->rss_hf & ETH_RSS_IPV4) + hash_type |= HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_IPV4; + if (rss->rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) + hash_type |= HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_TCP_IPV4; + if (rss->rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) + hash_type |= HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_UDP_IPV4; + if (rss->rss_hf & ETH_RSS_IPV6) + hash_type |= HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_IPV6; + if (rss->rss_hf & ETH_RSS_NONFRAG_IPV6_TCP) + hash_type |= HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_TCP_IPV6; + if (rss->rss_hf & ETH_RSS_NONFRAG_IPV6_UDP) + hash_type |= HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_UDP_IPV6; + + for (i = 0; i < bp->nr_vnics; i++) { + STAILQ_FOREACH(vnic, &bp->ff_pool[i], next) { + vnic->hash_type = hash_type; + + /* + * Use the supplied key if the key length is + * acceptable and the rss_key is not NULL + */ + if (rss->rss_key && + rss->rss_key_len <= HW_HASH_KEY_SIZE) + memcpy(vnic->rss_hash_key, + rss->rss_key, rss->rss_key_len); + } + } + } + return rc; err_out: @@ -213,9 +228,10 @@ err_out: return rc; } -static void bnxt_rx_queue_release_mbufs(struct bnxt_rx_queue *rxq __rte_unused) +static void bnxt_rx_queue_release_mbufs(struct bnxt_rx_queue *rxq) { struct bnxt_sw_rx_bd *sw_ring; + struct bnxt_tpa_info *tpa_info; uint16_t i; if (rxq) { @@ -228,6 +244,27 @@ static void bnxt_rx_queue_release_mbufs(struct bnxt_rx_queue *rxq __rte_unused) } } } + /* Free up mbufs in Agg ring */ + sw_ring = rxq->rx_ring->ag_buf_ring; + if (sw_ring) { + for (i = 0; i < rxq->nb_rx_desc; i++) { + if (sw_ring[i].mbuf) { + rte_pktmbuf_free_seg(sw_ring[i].mbuf); + sw_ring[i].mbuf = NULL; + } + } + } + + /* Free up mbufs in TPA */ + tpa_info = rxq->rx_ring->tpa_info; + if (tpa_info) { + for (i = 0; i < BNXT_TPA_MAX; i++) { + if (tpa_info[i].mbuf) { + rte_pktmbuf_free_seg(tpa_info[i].mbuf); + tpa_info[i].mbuf = NULL; + } + } + } } } @@ -251,6 +288,8 @@ void bnxt_rx_queue_release_op(void *rx_queue) /* Free RX ring hardware descriptors */ bnxt_free_ring(rxq->rx_ring->rx_ring_struct); + /* Free RX Agg ring hardware descriptors */ + bnxt_free_ring(rxq->rx_ring->ag_ring_struct); /* Free RX completion ring hardware descriptors */ bnxt_free_ring(rxq->cp_ring->cp_ring_struct); @@ -273,7 +312,7 @@ int bnxt_rx_queue_setup_op(struct rte_eth_dev *eth_dev, int rc = 0; if (!nb_desc || nb_desc > MAX_RX_DESC_CNT) { - RTE_LOG(ERR, PMD, "nb_desc %d is invalid", nb_desc); + RTE_LOG(ERR, PMD, "nb_desc %d is invalid\n", nb_desc); rc = -EINVAL; goto out; } @@ -286,7 +325,7 @@ int bnxt_rx_queue_setup_op(struct rte_eth_dev *eth_dev, rxq = rte_zmalloc_socket("bnxt_rx_queue", sizeof(struct bnxt_rx_queue), RTE_CACHE_LINE_SIZE, socket_id); if (!rxq) { - RTE_LOG(ERR, PMD, "bnxt_rx_queue allocation failed!"); + RTE_LOG(ERR, PMD, "bnxt_rx_queue allocation failed!\n"); rc = -ENOMEM; goto out; } @@ -295,6 +334,9 @@ int bnxt_rx_queue_setup_op(struct rte_eth_dev *eth_dev, rxq->nb_rx_desc = nb_desc; rxq->rx_free_thresh = rx_conf->rx_free_thresh; + RTE_LOG(DEBUG, PMD, "RX Buf size is %d\n", rxq->rx_buf_use_size); + RTE_LOG(DEBUG, PMD, "RX Buf MTU %d\n", eth_dev->data->mtu); + rc = bnxt_init_rx_ring_struct(rxq, socket_id); if (rc) goto out; @@ -308,7 +350,8 @@ int bnxt_rx_queue_setup_op(struct rte_eth_dev *eth_dev, /* Allocate RX ring hardware descriptors */ if (bnxt_alloc_rings(bp, queue_idx, NULL, rxq->rx_ring, rxq->cp_ring, "rxr")) { - RTE_LOG(ERR, PMD, "ring_dma_zone_reserve for rx_ring failed!"); + RTE_LOG(ERR, PMD, + "ring_dma_zone_reserve for rx_ring failed!\n"); bnxt_rx_queue_release_op(rxq); rc = -ENOMEM; goto out; @@ -317,3 +360,41 @@ int bnxt_rx_queue_setup_op(struct rte_eth_dev *eth_dev, out: return rc; } + +int +bnxt_rx_queue_intr_enable_op(struct rte_eth_dev *eth_dev, uint16_t queue_id) +{ + struct bnxt_rx_queue *rxq; + struct bnxt_cp_ring_info *cpr; + int rc = 0; + + if (eth_dev->data->rx_queues) { + rxq = eth_dev->data->rx_queues[queue_id]; + if (!rxq) { + rc = -EINVAL; + return rc; + } + cpr = rxq->cp_ring; + B_CP_DB_ARM(cpr); + } + return rc; +} + +int +bnxt_rx_queue_intr_disable_op(struct rte_eth_dev *eth_dev, uint16_t queue_id) +{ + struct bnxt_rx_queue *rxq; + struct bnxt_cp_ring_info *cpr; + int rc = 0; + + if (eth_dev->data->rx_queues) { + rxq = eth_dev->data->rx_queues[queue_id]; + if (!rxq) { + rc = -EINVAL; + return rc; + } + cpr = rxq->cp_ring; + B_CP_DB_DISARM(cpr); + } + return rc; +} diff --git a/dpdk/drivers/net/bnxt/bnxt_rxq.h b/dpdk/drivers/net/bnxt/bnxt_rxq.h index 95543298..508731ee 100644 --- a/dpdk/drivers/net/bnxt/bnxt_rxq.h +++ b/dpdk/drivers/net/bnxt/bnxt_rxq.h @@ -48,16 +48,19 @@ struct bnxt_rx_queue { uint16_t rx_free_thresh; /* max free RX desc to hold */ uint16_t queue_id; /* RX queue index */ uint16_t reg_idx; /* RX queue register index */ - uint8_t port_id; /* Device port identifier */ + uint16_t port_id; /* Device port identifier */ uint8_t crc_len; /* 0 if CRC stripped, 4 otherwise */ struct bnxt *bp; + int index; struct bnxt_vnic_info *vnic; uint32_t rx_buf_size; uint32_t rx_buf_use_size; /* useable size */ struct bnxt_rx_ring_info *rx_ring; struct bnxt_cp_ring_info *cp_ring; + + struct bnxt_tpa_info *rx_tpa; }; void bnxt_free_rxq_stats(struct bnxt_rx_queue *rxq); @@ -70,5 +73,9 @@ int bnxt_rx_queue_setup_op(struct rte_eth_dev *eth_dev, const struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp); void bnxt_free_rx_mbufs(struct bnxt *bp); +int bnxt_rx_queue_intr_enable_op(struct rte_eth_dev *eth_dev, + uint16_t queue_id); +int bnxt_rx_queue_intr_disable_op(struct rte_eth_dev *eth_dev, + uint16_t queue_id); #endif diff --git a/dpdk/drivers/net/bnxt/bnxt_rxr.c b/dpdk/drivers/net/bnxt/bnxt_rxr.c index 5d93de26..5128335f 100644 --- a/dpdk/drivers/net/bnxt/bnxt_rxr.c +++ b/dpdk/drivers/net/bnxt/bnxt_rxr.c @@ -34,6 +34,7 @@ #include #include +#include #include #include #include @@ -67,8 +68,10 @@ static inline int bnxt_alloc_rx_data(struct bnxt_rx_queue *rxq, struct rte_mbuf *data; data = __bnxt_alloc_rx_data(rxq->mb_pool); - if (!data) + if (!data) { + rte_atomic64_inc(&rxq->bp->rx_mbuf_alloc_fail); return -ENOMEM; + } rx_buf->mbuf = data; @@ -77,24 +80,304 @@ static inline int bnxt_alloc_rx_data(struct bnxt_rx_queue *rxq, return 0; } -static void bnxt_reuse_rx_mbuf(struct bnxt_rx_ring_info *rxr, uint16_t cons, +static inline int bnxt_alloc_ag_data(struct bnxt_rx_queue *rxq, + struct bnxt_rx_ring_info *rxr, + uint16_t prod) +{ + struct rx_prod_pkt_bd *rxbd = &rxr->ag_desc_ring[prod]; + struct bnxt_sw_rx_bd *rx_buf = &rxr->ag_buf_ring[prod]; + struct rte_mbuf *data; + + data = __bnxt_alloc_rx_data(rxq->mb_pool); + if (!data) { + rte_atomic64_inc(&rxq->bp->rx_mbuf_alloc_fail); + return -ENOMEM; + } + + if (rxbd == NULL) + RTE_LOG(ERR, PMD, "Jumbo Frame. rxbd is NULL\n"); + if (rx_buf == NULL) + RTE_LOG(ERR, PMD, "Jumbo Frame. rx_buf is NULL\n"); + + + rx_buf->mbuf = data; + + rxbd->addr = rte_cpu_to_le_64(RTE_MBUF_DATA_DMA_ADDR(rx_buf->mbuf)); + + return 0; +} + +static inline void bnxt_reuse_rx_mbuf(struct bnxt_rx_ring_info *rxr, struct rte_mbuf *mbuf) { - uint16_t prod = rxr->rx_prod; + uint16_t prod = RING_NEXT(rxr->rx_ring_struct, rxr->rx_prod); struct bnxt_sw_rx_bd *prod_rx_buf; - struct rx_prod_pkt_bd *prod_bd, *cons_bd; + struct rx_prod_pkt_bd *prod_bd; prod_rx_buf = &rxr->rx_buf_ring[prod]; + RTE_ASSERT(prod_rx_buf->mbuf == NULL); + RTE_ASSERT(mbuf != NULL); + prod_rx_buf->mbuf = mbuf; prod_bd = &rxr->rx_desc_ring[prod]; - cons_bd = &rxr->rx_desc_ring[cons]; + + prod_bd->addr = rte_cpu_to_le_64(RTE_MBUF_DATA_DMA_ADDR(mbuf)); + + rxr->rx_prod = prod; +} + +#ifdef BNXT_DEBUG +static void bnxt_reuse_ag_mbuf(struct bnxt_rx_ring_info *rxr, uint16_t cons, + struct rte_mbuf *mbuf) +{ + uint16_t prod = rxr->ag_prod; + struct bnxt_sw_rx_bd *prod_rx_buf; + struct rx_prod_pkt_bd *prod_bd, *cons_bd; + + prod_rx_buf = &rxr->ag_buf_ring[prod]; + + prod_rx_buf->mbuf = mbuf; + + prod_bd = &rxr->ag_desc_ring[prod]; + cons_bd = &rxr->ag_desc_ring[cons]; prod_bd->addr = cons_bd->addr; } +#endif -static uint16_t bnxt_rx_pkt(struct rte_mbuf **rx_pkt, +static inline +struct rte_mbuf *bnxt_consume_rx_buf(struct bnxt_rx_ring_info *rxr, + uint16_t cons) +{ + struct bnxt_sw_rx_bd *cons_rx_buf; + struct rte_mbuf *mbuf; + + cons_rx_buf = &rxr->rx_buf_ring[cons]; + RTE_ASSERT(cons_rx_buf->mbuf != NULL); + mbuf = cons_rx_buf->mbuf; + cons_rx_buf->mbuf = NULL; + return mbuf; +} + +static void bnxt_tpa_start(struct bnxt_rx_queue *rxq, + struct rx_tpa_start_cmpl *tpa_start, + struct rx_tpa_start_cmpl_hi *tpa_start1) +{ + struct bnxt_rx_ring_info *rxr = rxq->rx_ring; + uint8_t agg_id = rte_le_to_cpu_32(tpa_start->agg_id & + RX_TPA_START_CMPL_AGG_ID_MASK) >> RX_TPA_START_CMPL_AGG_ID_SFT; + uint16_t data_cons; + struct bnxt_tpa_info *tpa_info; + struct rte_mbuf *mbuf; + + data_cons = tpa_start->opaque; + tpa_info = &rxr->tpa_info[agg_id]; + + mbuf = bnxt_consume_rx_buf(rxr, data_cons); + + bnxt_reuse_rx_mbuf(rxr, tpa_info->mbuf); + + tpa_info->mbuf = mbuf; + tpa_info->len = rte_le_to_cpu_32(tpa_start->len); + + mbuf->nb_segs = 1; + mbuf->next = NULL; + mbuf->pkt_len = rte_le_to_cpu_32(tpa_start->len); + mbuf->data_len = mbuf->pkt_len; + mbuf->port = rxq->port_id; + mbuf->ol_flags = PKT_RX_LRO; + if (likely(tpa_start->flags_type & + rte_cpu_to_le_32(RX_TPA_START_CMPL_FLAGS_RSS_VALID))) { + mbuf->hash.rss = rte_le_to_cpu_32(tpa_start->rss_hash); + mbuf->ol_flags |= PKT_RX_RSS_HASH; + } else { + mbuf->hash.fdir.id = rte_le_to_cpu_16(tpa_start1->cfa_code); + mbuf->ol_flags |= PKT_RX_FDIR | PKT_RX_FDIR_ID; + } + if (tpa_start1->flags2 & + rte_cpu_to_le_32(RX_TPA_START_CMPL_FLAGS2_META_FORMAT_VLAN)) { + mbuf->vlan_tci = rte_le_to_cpu_32(tpa_start1->metadata); + mbuf->ol_flags |= PKT_RX_VLAN; + } + if (likely(tpa_start1->flags2 & + rte_cpu_to_le_32(RX_TPA_START_CMPL_FLAGS2_L4_CS_CALC))) + mbuf->ol_flags |= PKT_RX_L4_CKSUM_GOOD; + + /* recycle next mbuf */ + data_cons = RING_NEXT(rxr->rx_ring_struct, data_cons); + bnxt_reuse_rx_mbuf(rxr, bnxt_consume_rx_buf(rxr, data_cons)); +} + +static int bnxt_agg_bufs_valid(struct bnxt_cp_ring_info *cpr, + uint8_t agg_bufs, uint32_t raw_cp_cons) +{ + uint16_t last_cp_cons; + struct rx_pkt_cmpl *agg_cmpl; + + raw_cp_cons = ADV_RAW_CMP(raw_cp_cons, agg_bufs); + last_cp_cons = RING_CMP(cpr->cp_ring_struct, raw_cp_cons); + agg_cmpl = (struct rx_pkt_cmpl *)&cpr->cp_desc_ring[last_cp_cons]; + cpr->valid = FLIP_VALID(raw_cp_cons, + cpr->cp_ring_struct->ring_mask, + cpr->valid); + return CMP_VALID(agg_cmpl, raw_cp_cons, cpr->cp_ring_struct); +} + +/* TPA consume agg buffer out of order, allocate connected data only */ +static int bnxt_prod_ag_mbuf(struct bnxt_rx_queue *rxq) +{ + struct bnxt_rx_ring_info *rxr = rxq->rx_ring; + uint16_t next = RING_NEXT(rxr->ag_ring_struct, rxr->ag_prod); + + /* TODO batch allocation for better performance */ + while (rte_bitmap_get(rxr->ag_bitmap, next)) { + if (unlikely(bnxt_alloc_ag_data(rxq, rxr, next))) { + RTE_LOG(ERR, PMD, + "agg mbuf alloc failed: prod=0x%x\n", next); + break; + } + rte_bitmap_clear(rxr->ag_bitmap, next); + rxr->ag_prod = next; + next = RING_NEXT(rxr->ag_ring_struct, next); + } + + return 0; +} + +static int bnxt_rx_pages(struct bnxt_rx_queue *rxq, + struct rte_mbuf *mbuf, uint32_t *tmp_raw_cons, + uint8_t agg_buf) +{ + struct bnxt_cp_ring_info *cpr = rxq->cp_ring; + struct bnxt_rx_ring_info *rxr = rxq->rx_ring; + int i; + uint16_t cp_cons, ag_cons; + struct rx_pkt_cmpl *rxcmp; + struct rte_mbuf *last = mbuf; + + for (i = 0; i < agg_buf; i++) { + struct bnxt_sw_rx_bd *ag_buf; + struct rte_mbuf *ag_mbuf; + *tmp_raw_cons = NEXT_RAW_CMP(*tmp_raw_cons); + cp_cons = RING_CMP(cpr->cp_ring_struct, *tmp_raw_cons); + rxcmp = (struct rx_pkt_cmpl *) + &cpr->cp_desc_ring[cp_cons]; + +#ifdef BNXT_DEBUG + bnxt_dump_cmpl(cp_cons, rxcmp); +#endif + + ag_cons = rxcmp->opaque; + RTE_ASSERT(ag_cons <= rxr->ag_ring_struct->ring_mask); + ag_buf = &rxr->ag_buf_ring[ag_cons]; + ag_mbuf = ag_buf->mbuf; + RTE_ASSERT(ag_mbuf != NULL); + + ag_mbuf->data_len = rte_le_to_cpu_16(rxcmp->len); + + mbuf->nb_segs++; + mbuf->pkt_len += ag_mbuf->data_len; + + last->next = ag_mbuf; + last = ag_mbuf; + + ag_buf->mbuf = NULL; + + /* + * As aggregation buffer consumed out of order in TPA module, + * use bitmap to track freed slots to be allocated and notified + * to NIC + */ + rte_bitmap_set(rxr->ag_bitmap, ag_cons); + } + bnxt_prod_ag_mbuf(rxq); + return 0; +} + +static inline struct rte_mbuf *bnxt_tpa_end( + struct bnxt_rx_queue *rxq, + uint32_t *raw_cp_cons, + struct rx_tpa_end_cmpl *tpa_end, + struct rx_tpa_end_cmpl_hi *tpa_end1 __rte_unused) +{ + struct bnxt_cp_ring_info *cpr = rxq->cp_ring; + struct bnxt_rx_ring_info *rxr = rxq->rx_ring; + uint8_t agg_id = (tpa_end->agg_id & RX_TPA_END_CMPL_AGG_ID_MASK) + >> RX_TPA_END_CMPL_AGG_ID_SFT; + struct rte_mbuf *mbuf; + uint8_t agg_bufs; + struct bnxt_tpa_info *tpa_info; + + tpa_info = &rxr->tpa_info[agg_id]; + mbuf = tpa_info->mbuf; + RTE_ASSERT(mbuf != NULL); + + rte_prefetch0(mbuf); + agg_bufs = (rte_le_to_cpu_32(tpa_end->agg_bufs_v1) & + RX_TPA_END_CMPL_AGG_BUFS_MASK) >> RX_TPA_END_CMPL_AGG_BUFS_SFT; + if (agg_bufs) { + if (!bnxt_agg_bufs_valid(cpr, agg_bufs, *raw_cp_cons)) + return NULL; + bnxt_rx_pages(rxq, mbuf, raw_cp_cons, agg_bufs); + } + mbuf->l4_len = tpa_end->payload_offset; + + struct rte_mbuf *new_data = __bnxt_alloc_rx_data(rxq->mb_pool); + RTE_ASSERT(new_data != NULL); + if (!new_data) { + rte_atomic64_inc(&rxq->bp->rx_mbuf_alloc_fail); + return NULL; + } + tpa_info->mbuf = new_data; + + return mbuf; +} + +static uint32_t +bnxt_parse_pkt_type(struct rx_pkt_cmpl *rxcmp, struct rx_pkt_cmpl_hi *rxcmp1) +{ + uint32_t pkt_type = 0; + uint32_t t_ipcs = 0, ip = 0, ip6 = 0; + uint32_t tcp = 0, udp = 0, icmp = 0; + uint32_t vlan = 0; + + vlan = !!(rxcmp1->flags2 & + rte_cpu_to_le_32(RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN)); + t_ipcs = !!(rxcmp1->flags2 & + rte_cpu_to_le_32(RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC)); + ip6 = !!(rxcmp1->flags2 & + rte_cpu_to_le_32(RX_PKT_CMPL_FLAGS2_IP_TYPE)); + icmp = !!(rxcmp->flags_type & + rte_cpu_to_le_16(RX_PKT_CMPL_FLAGS_ITYPE_ICMP)); + tcp = !!(rxcmp->flags_type & + rte_cpu_to_le_16(RX_PKT_CMPL_FLAGS_ITYPE_TCP)); + udp = !!(rxcmp->flags_type & + rte_cpu_to_le_16(RX_PKT_CMPL_FLAGS_ITYPE_UDP)); + ip = !!(rxcmp->flags_type & + rte_cpu_to_le_16(RX_PKT_CMPL_FLAGS_ITYPE_IP)); + + pkt_type |= ((ip || tcp || udp || icmp) && !t_ipcs && !ip6) ? + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN : 0; + pkt_type |= ((ip || tcp || udp || icmp) && !t_ipcs && ip6) ? + RTE_PTYPE_L3_IPV6_EXT_UNKNOWN : 0; + pkt_type |= (!t_ipcs && icmp) ? RTE_PTYPE_L4_ICMP : 0; + pkt_type |= (!t_ipcs && udp) ? RTE_PTYPE_L4_UDP : 0; + pkt_type |= (!t_ipcs && tcp) ? RTE_PTYPE_L4_TCP : 0; + pkt_type |= ((ip || tcp || udp || icmp) && t_ipcs && !ip6) ? + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN : 0; + pkt_type |= ((ip || tcp || udp || icmp) && t_ipcs && ip6) ? + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN : 0; + pkt_type |= (t_ipcs && icmp) ? RTE_PTYPE_INNER_L4_ICMP : 0; + pkt_type |= (t_ipcs && udp) ? RTE_PTYPE_INNER_L4_UDP : 0; + pkt_type |= (t_ipcs && tcp) ? RTE_PTYPE_INNER_L4_TCP : 0; + pkt_type |= vlan ? RTE_PTYPE_L2_ETHER_VLAN : 0; + + return pkt_type; +} + +static int bnxt_rx_pkt(struct rte_mbuf **rx_pkt, struct bnxt_rx_queue *rxq, uint32_t *raw_cons) { struct bnxt_cp_ring_info *cpr = rxq->cp_ring; @@ -104,9 +387,13 @@ static uint16_t bnxt_rx_pkt(struct rte_mbuf **rx_pkt, uint32_t tmp_raw_cons = *raw_cons; uint16_t cons, prod, cp_cons = RING_CMP(cpr->cp_ring_struct, tmp_raw_cons); - struct bnxt_sw_rx_bd *rx_buf; +#ifdef BNXT_DEBUG + uint16_t ag_cons; +#endif struct rte_mbuf *mbuf; int rc = 0; + uint8_t agg_buf = 0; + uint16_t cmp_type; rxcmp = (struct rx_pkt_cmpl *) &cpr->cp_desc_ring[cp_cons]; @@ -118,12 +405,41 @@ static uint16_t bnxt_rx_pkt(struct rte_mbuf **rx_pkt, if (!CMP_VALID(rxcmp1, tmp_raw_cons, cpr->cp_ring_struct)) return -EBUSY; + cpr->valid = FLIP_VALID(cp_cons, + cpr->cp_ring_struct->ring_mask, + cpr->valid); + + cmp_type = CMP_TYPE(rxcmp); + if (cmp_type == RX_TPA_START_CMPL_TYPE_RX_TPA_START) { + bnxt_tpa_start(rxq, (struct rx_tpa_start_cmpl *)rxcmp, + (struct rx_tpa_start_cmpl_hi *)rxcmp1); + rc = -EINVAL; /* Continue w/o new mbuf */ + goto next_rx; + } else if (cmp_type == RX_TPA_END_CMPL_TYPE_RX_TPA_END) { + mbuf = bnxt_tpa_end(rxq, &tmp_raw_cons, + (struct rx_tpa_end_cmpl *)rxcmp, + (struct rx_tpa_end_cmpl_hi *)rxcmp1); + if (unlikely(!mbuf)) + return -EBUSY; + *rx_pkt = mbuf; + goto next_rx; + } else if (cmp_type != 0x11) { + rc = -EINVAL; + goto next_rx; + } + + agg_buf = (rxcmp->agg_bufs_v1 & RX_PKT_CMPL_AGG_BUFS_MASK) + >> RX_PKT_CMPL_AGG_BUFS_SFT; + if (agg_buf && !bnxt_agg_bufs_valid(cpr, agg_buf, tmp_raw_cons)) + return -EBUSY; + prod = rxr->rx_prod; - /* EW - GRO deferred to phase 3 */ cons = rxcmp->opaque; - rx_buf = &rxr->rx_buf_ring[cons]; - mbuf = rx_buf->mbuf; + mbuf = bnxt_consume_rx_buf(rxr, cons); + if (mbuf == NULL) + return -EBUSY; + rte_prefetch0(mbuf); mbuf->nb_segs = 1; @@ -139,22 +455,41 @@ static uint16_t bnxt_rx_pkt(struct rte_mbuf **rx_pkt, mbuf->hash.fdir.id = rxcmp1->cfa_code; mbuf->ol_flags |= PKT_RX_FDIR | PKT_RX_FDIR_ID; } + + if (agg_buf) + bnxt_rx_pages(rxq, mbuf, &tmp_raw_cons, agg_buf); + if (rxcmp1->flags2 & RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN) { mbuf->vlan_tci = rxcmp1->metadata & (RX_PKT_CMPL_METADATA_VID_MASK | RX_PKT_CMPL_METADATA_DE | RX_PKT_CMPL_METADATA_PRI_MASK); - mbuf->ol_flags |= PKT_RX_VLAN_PKT; + mbuf->ol_flags |= PKT_RX_VLAN; } - rx_buf->mbuf = NULL; + if (likely(RX_CMP_IP_CS_OK(rxcmp1))) + mbuf->ol_flags |= PKT_RX_IP_CKSUM_GOOD; + else + mbuf->ol_flags |= PKT_RX_IP_CKSUM_BAD; + + if (likely(RX_CMP_L4_CS_OK(rxcmp1))) + mbuf->ol_flags |= PKT_RX_L4_CKSUM_GOOD; + else + mbuf->ol_flags |= PKT_RX_L4_CKSUM_BAD; + + mbuf->packet_type = bnxt_parse_pkt_type(rxcmp, rxcmp1); + +#ifdef BNXT_DEBUG if (rxcmp1->errors_v2 & RX_CMP_L2_ERRORS) { /* Re-install the mbuf back to the rx ring */ bnxt_reuse_rx_mbuf(rxr, cons, mbuf); + if (agg_buf) + bnxt_reuse_ag_mbuf(rxr, ag_cons, mbuf); rc = -EIO; goto next_rx; } +#endif /* * TODO: Redesign this.... * If the allocation fails, the packet does not get received. @@ -170,24 +505,21 @@ static uint16_t bnxt_rx_pkt(struct rte_mbuf **rx_pkt, * calls in favour of a tight loop with the same function being called * in it. */ + prod = RING_NEXT(rxr->rx_ring_struct, prod); if (bnxt_alloc_rx_data(rxq, rxr, prod)) { RTE_LOG(ERR, PMD, "mbuf alloc failed with prod=0x%x\n", prod); rc = -ENOMEM; - goto next_rx; + goto rx; } - + rxr->rx_prod = prod; /* * All MBUFs are allocated with the same size under DPDK, * no optimization for rx_copy_thresh */ - - /* AGG buf operation is deferred */ - - /* EW - VLAN reception. Must compare against the ol_flags */ - +rx: *rx_pkt = mbuf; + next_rx: - rxr->rx_prod = RING_NEXT(rxr->rx_ring_struct, prod); *raw_cons = tmp_raw_cons; @@ -203,45 +535,73 @@ uint16_t bnxt_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint32_t raw_cons = cpr->cp_raw_cons; uint32_t cons; int nb_rx_pkts = 0; - bool rx_event = false; struct rx_pkt_cmpl *rxcmp; + uint16_t prod = rxr->rx_prod; + uint16_t ag_prod = rxr->ag_prod; + int rc = 0; /* Handle RX burst request */ while (1) { - int rc; - cons = RING_CMP(cpr->cp_ring_struct, raw_cons); rte_prefetch0(&cpr->cp_desc_ring[cons]); rxcmp = (struct rx_pkt_cmpl *)&cpr->cp_desc_ring[cons]; if (!CMP_VALID(rxcmp, raw_cons, cpr->cp_ring_struct)) break; + cpr->valid = FLIP_VALID(cons, + cpr->cp_ring_struct->ring_mask, + cpr->valid); /* TODO: Avoid magic numbers... */ if ((CMP_TYPE(rxcmp) & 0x30) == 0x10) { rc = bnxt_rx_pkt(&rx_pkts[nb_rx_pkts], rxq, &raw_cons); - if (likely(!rc)) + if (likely(!rc) || rc == -ENOMEM) nb_rx_pkts++; - else if (rc == -EBUSY) /* partial completion */ + if (rc == -EBUSY) /* partial completion */ break; - rx_event = true; } raw_cons = NEXT_RAW_CMP(raw_cons); if (nb_rx_pkts == nb_pkts) break; } - if (raw_cons == cpr->cp_raw_cons) { + + cpr->cp_raw_cons = raw_cons; + if (prod == rxr->rx_prod && ag_prod == rxr->ag_prod) { /* * For PMD, there is no need to keep on pushing to REARM * the doorbell if there are no new completions */ return nb_rx_pkts; } - cpr->cp_raw_cons = raw_cons; B_CP_DIS_DB(cpr, cpr->cp_raw_cons); - if (rx_event) - B_RX_DB(rxr->rx_doorbell, rxr->rx_prod); + B_RX_DB(rxr->rx_doorbell, rxr->rx_prod); + /* Ring the AGG ring DB */ + B_RX_DB(rxr->ag_doorbell, rxr->ag_prod); + + /* Attempt to alloc Rx buf in case of a previous allocation failure. */ + if (rc == -ENOMEM) { + int i; + + for (i = prod; i <= nb_rx_pkts; + i = RING_NEXT(rxr->rx_ring_struct, i)) { + struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[i]; + + /* Buffer already allocated for this index. */ + if (rx_buf->mbuf != NULL) + continue; + + /* This slot is empty. Alloc buffer for Rx */ + if (!bnxt_alloc_rx_data(rxq, rxr, i)) { + rxr->rx_prod = i; + B_RX_DB(rxr->rx_doorbell, rxr->rx_prod); + } else { + RTE_LOG(ERR, PMD, "Alloc mbuf failed\n"); + break; + } + } + } + return nb_rx_pkts; } @@ -257,6 +617,12 @@ void bnxt_free_rx_rings(struct bnxt *bp) bnxt_free_ring(rxq->rx_ring->rx_ring_struct); rte_free(rxq->rx_ring->rx_ring_struct); + + /* Free the Aggregator ring */ + bnxt_free_ring(rxq->rx_ring->ag_ring_struct); + rte_free(rxq->rx_ring->ag_ring_struct); + rxq->rx_ring->ag_ring_struct = NULL; + rte_free(rxq->rx_ring); bnxt_free_ring(rxq->cp_ring->cp_ring_struct); @@ -270,13 +636,11 @@ void bnxt_free_rx_rings(struct bnxt *bp) int bnxt_init_rx_ring_struct(struct bnxt_rx_queue *rxq, unsigned int socket_id) { - struct bnxt *bp = rxq->bp; struct bnxt_cp_ring_info *cpr; struct bnxt_rx_ring_info *rxr; struct bnxt_ring *ring; - rxq->rx_buf_use_size = bp->eth_dev->data->mtu + - ETHER_HDR_LEN + ETHER_CRC_LEN + + rxq->rx_buf_use_size = BNXT_MAX_MTU + ETHER_HDR_LEN + ETHER_CRC_LEN + (2 * VLAN_TAG_SIZE); rxq->rx_buf_size = rxq->rx_buf_use_size + sizeof(struct rte_mbuf); @@ -313,13 +677,29 @@ int bnxt_init_rx_ring_struct(struct bnxt_rx_queue *rxq, unsigned int socket_id) if (ring == NULL) return -ENOMEM; cpr->cp_ring_struct = ring; - ring->ring_size = rxr->rx_ring_struct->ring_size * 2; + ring->ring_size = rte_align32pow2(rxr->rx_ring_struct->ring_size * + (2 + AGG_RING_SIZE_FACTOR)); ring->ring_mask = ring->ring_size - 1; ring->bd = (void *)cpr->cp_desc_ring; ring->bd_dma = cpr->cp_desc_mapping; ring->vmem_size = 0; ring->vmem = NULL; + /* Allocate Aggregator rings */ + ring = rte_zmalloc_socket("bnxt_rx_ring_struct", + sizeof(struct bnxt_ring), + RTE_CACHE_LINE_SIZE, socket_id); + if (ring == NULL) + return -ENOMEM; + rxr->ag_ring_struct = ring; + ring->ring_size = rte_align32pow2(rxq->nb_rx_desc * + AGG_RING_SIZE_FACTOR); + ring->ring_mask = ring->ring_size - 1; + ring->bd = (void *)rxr->ag_desc_ring; + ring->bd_dma = rxr->ag_desc_mapping; + ring->vmem_size = ring->ring_size * sizeof(struct bnxt_sw_rx_bd); + ring->vmem = (void **)&rxr->ag_buf_ring; + return 0; } @@ -332,8 +712,8 @@ static void bnxt_init_rxbds(struct bnxt_ring *ring, uint32_t type, if (!rx_bd_ring) return; for (j = 0; j < ring->ring_size; j++) { - rx_bd_ring[j].flags_type = type; - rx_bd_ring[j].len = len; + rx_bd_ring[j].flags_type = rte_cpu_to_le_16(type); + rx_bd_ring[j].len = rte_cpu_to_le_16(len); rx_bd_ring[j].opaque = j; } } @@ -344,12 +724,17 @@ int bnxt_init_one_rx_ring(struct bnxt_rx_queue *rxq) struct bnxt_ring *ring; uint32_t prod, type; unsigned int i; + uint16_t size; - type = RX_PROD_PKT_BD_TYPE_RX_PROD_PKT | RX_PROD_PKT_BD_FLAGS_EOP_PAD; + size = rte_pktmbuf_data_room_size(rxq->mb_pool) - RTE_PKTMBUF_HEADROOM; + if (rxq->rx_buf_use_size <= size) + size = rxq->rx_buf_use_size; + + type = RX_PROD_PKT_BD_TYPE_RX_PROD_PKT; rxr = rxq->rx_ring; ring = rxr->rx_ring_struct; - bnxt_init_rxbds(ring, type, rxq->rx_buf_use_size); + bnxt_init_rxbds(ring, type, size); prod = rxr->rx_prod; for (i = 0; i < ring->ring_size; i++) { @@ -362,6 +747,36 @@ int bnxt_init_one_rx_ring(struct bnxt_rx_queue *rxq) rxr->rx_prod = prod; prod = RING_NEXT(rxr->rx_ring_struct, prod); } + RTE_LOG(DEBUG, PMD, "%s\n", __func__); + + ring = rxr->ag_ring_struct; + type = RX_PROD_AGG_BD_TYPE_RX_PROD_AGG; + bnxt_init_rxbds(ring, type, size); + prod = rxr->ag_prod; + + for (i = 0; i < ring->ring_size; i++) { + if (bnxt_alloc_ag_data(rxq, rxr, prod) != 0) { + RTE_LOG(WARNING, PMD, + "init'ed AG ring %d with %d/%d mbufs only\n", + rxq->queue_id, i, ring->ring_size); + break; + } + rxr->ag_prod = prod; + prod = RING_NEXT(rxr->ag_ring_struct, prod); + } + RTE_LOG(DEBUG, PMD, "%s AGG Done!\n", __func__); + + if (rxr->tpa_info) { + for (i = 0; i < BNXT_TPA_MAX; i++) { + rxr->tpa_info[i].mbuf = + __bnxt_alloc_rx_data(rxq->mb_pool); + if (!rxr->tpa_info[i].mbuf) { + rte_atomic64_inc(&rxq->bp->rx_mbuf_alloc_fail); + return -ENOMEM; + } + } + } + RTE_LOG(DEBUG, PMD, "%s TPA alloc Done!\n", __func__); return 0; } diff --git a/dpdk/drivers/net/bnxt/bnxt_rxr.h b/dpdk/drivers/net/bnxt/bnxt_rxr.h index f766b26c..a94373d1 100644 --- a/dpdk/drivers/net/bnxt/bnxt_rxr.h +++ b/dpdk/drivers/net/bnxt/bnxt_rxr.h @@ -37,20 +37,82 @@ #define B_RX_DB(db, prod) \ (*(uint32_t *)db = (DB_KEY_RX | prod)) +#define BNXT_TPA_L4_SIZE(x) \ + { \ + typeof(x) hdr_info = (x); \ + (((hdr_info) & 0xf8000000) ? ((hdr_info) >> 27) : 32) \ + } + +#define BNXT_TPA_INNER_L3_OFF(hdr_info) \ + (((hdr_info) >> 18) & 0x1ff) + +#define BNXT_TPA_INNER_L2_OFF(hdr_info) \ + (((hdr_info) >> 9) & 0x1ff) + +#define BNXT_TPA_OUTER_L3_OFF(hdr_info) \ + ((hdr_info) & 0x1ff) + +#define RX_CMP_L4_CS_BITS rte_cpu_to_le_32(RX_PKT_CMPL_FLAGS2_L4_CS_CALC) + +#define RX_CMP_L4_CS_ERR_BITS rte_cpu_to_le_32(RX_PKT_CMPL_ERRORS_L4_CS_ERROR) + +#define RX_CMP_L4_CS_OK(rxcmp1) \ + (((rxcmp1)->flags2 & RX_CMP_L4_CS_BITS) && \ + !((rxcmp1)->errors_v2 & RX_CMP_L4_CS_ERR_BITS)) + +#define RX_CMP_IP_CS_ERR_BITS rte_cpu_to_le_32(RX_PKT_CMPL_ERRORS_IP_CS_ERROR) + +#define RX_CMP_IP_CS_BITS rte_cpu_to_le_32(RX_PKT_CMPL_FLAGS2_IP_CS_CALC) + +#define RX_CMP_IP_CS_OK(rxcmp1) \ + (((rxcmp1)->flags2 & RX_CMP_IP_CS_BITS) && \ + !((rxcmp1)->errors_v2 & RX_CMP_IP_CS_ERR_BITS)) + +enum pkt_hash_types { + PKT_HASH_TYPE_NONE, /* Undefined type */ + PKT_HASH_TYPE_L2, /* Input: src_MAC, dest_MAC */ + PKT_HASH_TYPE_L3, /* Input: src_IP, dst_IP */ + PKT_HASH_TYPE_L4, /* Input: src_IP, dst_IP, src_port, dst_port */ +}; + +struct bnxt_tpa_info { + struct rte_mbuf *mbuf; + uint16_t len; + unsigned short gso_type; + uint32_t flags2; + uint32_t metadata; + enum pkt_hash_types hash_type; + uint32_t rss_hash; + uint32_t hdr_info; +}; + struct bnxt_sw_rx_bd { struct rte_mbuf *mbuf; /* data associated with RX descriptor */ }; struct bnxt_rx_ring_info { uint16_t rx_prod; + uint16_t ag_prod; void *rx_doorbell; + void *ag_doorbell; struct rx_prod_pkt_bd *rx_desc_ring; + struct rx_prod_pkt_bd *ag_desc_ring; struct bnxt_sw_rx_bd *rx_buf_ring; /* sw ring */ + struct bnxt_sw_rx_bd *ag_buf_ring; /* sw ring */ - phys_addr_t rx_desc_mapping; + rte_iova_t rx_desc_mapping; + rte_iova_t ag_desc_mapping; struct bnxt_ring *rx_ring_struct; + struct bnxt_ring *ag_ring_struct; + + /* + * To deal with out of order return from TPA, use free buffer indicator + */ + struct rte_bitmap *ag_bitmap; + + struct bnxt_tpa_info *tpa_info; }; uint16_t bnxt_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, diff --git a/dpdk/drivers/net/bnxt/bnxt_stats.c b/dpdk/drivers/net/bnxt/bnxt_stats.c index 6f1c7602..fe83d370 100644 --- a/dpdk/drivers/net/bnxt/bnxt_stats.c +++ b/dpdk/drivers/net/bnxt/bnxt_stats.c @@ -43,6 +43,171 @@ #include "bnxt_txq.h" #include "hsi_struct_def_dpdk.h" +static const struct bnxt_xstats_name_off bnxt_rx_stats_strings[] = { + {"rx_64b_frames", offsetof(struct rx_port_stats, + rx_64b_frames)}, + {"rx_65b_127b_frames", offsetof(struct rx_port_stats, + rx_65b_127b_frames)}, + {"rx_128b_255b_frames", offsetof(struct rx_port_stats, + rx_128b_255b_frames)}, + {"rx_256b_511b_frames", offsetof(struct rx_port_stats, + rx_256b_511b_frames)}, + {"rx_512b_1023b_frames", offsetof(struct rx_port_stats, + rx_512b_1023b_frames)}, + {"rx_1024b_1518_frames", offsetof(struct rx_port_stats, + rx_1024b_1518_frames)}, + {"rx_good_vlan_frames", offsetof(struct rx_port_stats, + rx_good_vlan_frames)}, + {"rx_1519b_2047b_frames", offsetof(struct rx_port_stats, + rx_1519b_2047b_frames)}, + {"rx_2048b_4095b_frames", offsetof(struct rx_port_stats, + rx_2048b_4095b_frames)}, + {"rx_4096b_9216b_frames", offsetof(struct rx_port_stats, + rx_4096b_9216b_frames)}, + {"rx_9217b_16383b_frames", offsetof(struct rx_port_stats, + rx_9217b_16383b_frames)}, + {"rx_total_frames", offsetof(struct rx_port_stats, + rx_total_frames)}, + {"rx_ucast_frames", offsetof(struct rx_port_stats, + rx_ucast_frames)}, + {"rx_mcast_frames", offsetof(struct rx_port_stats, + rx_mcast_frames)}, + {"rx_bcast_frames", offsetof(struct rx_port_stats, + rx_bcast_frames)}, + {"rx_fcs_err_frames", offsetof(struct rx_port_stats, + rx_fcs_err_frames)}, + {"rx_ctrl_frames", offsetof(struct rx_port_stats, + rx_ctrl_frames)}, + {"rx_pause_frames", offsetof(struct rx_port_stats, + rx_pause_frames)}, + {"rx_pfc_frames", offsetof(struct rx_port_stats, + rx_pfc_frames)}, + {"rx_align_err_frames", offsetof(struct rx_port_stats, + rx_align_err_frames)}, + {"rx_ovrsz_frames", offsetof(struct rx_port_stats, + rx_ovrsz_frames)}, + {"rx_jbr_frames", offsetof(struct rx_port_stats, + rx_jbr_frames)}, + {"rx_mtu_err_frames", offsetof(struct rx_port_stats, + rx_mtu_err_frames)}, + {"rx_tagged_frames", offsetof(struct rx_port_stats, + rx_tagged_frames)}, + {"rx_double_tagged_frames", offsetof(struct rx_port_stats, + rx_double_tagged_frames)}, + {"rx_good_frames", offsetof(struct rx_port_stats, + rx_good_frames)}, + {"rx_undrsz_frames", offsetof(struct rx_port_stats, + rx_undrsz_frames)}, + {"rx_eee_lpi_events", offsetof(struct rx_port_stats, + rx_eee_lpi_events)}, + {"rx_eee_lpi_duration", offsetof(struct rx_port_stats, + rx_eee_lpi_duration)}, + {"rx_bytes", offsetof(struct rx_port_stats, + rx_bytes)}, + {"rx_runt_bytes", offsetof(struct rx_port_stats, + rx_runt_bytes)}, + {"rx_runt_frames", offsetof(struct rx_port_stats, + rx_runt_frames)}, +}; + +static const struct bnxt_xstats_name_off bnxt_tx_stats_strings[] = { + {"tx_64b_frames", offsetof(struct tx_port_stats, + tx_64b_frames)}, + {"tx_65b_127b_frames", offsetof(struct tx_port_stats, + tx_65b_127b_frames)}, + {"tx_128b_255b_frames", offsetof(struct tx_port_stats, + tx_128b_255b_frames)}, + {"tx_256b_511b_frames", offsetof(struct tx_port_stats, + tx_256b_511b_frames)}, + {"tx_512b_1023b_frames", offsetof(struct tx_port_stats, + tx_512b_1023b_frames)}, + {"tx_1024b_1518_frames", offsetof(struct tx_port_stats, + tx_1024b_1518_frames)}, + {"tx_good_vlan_frames", offsetof(struct tx_port_stats, + tx_good_vlan_frames)}, + {"tx_1519b_2047_frames", offsetof(struct tx_port_stats, + tx_1519b_2047_frames)}, + {"tx_2048b_4095b_frames", offsetof(struct tx_port_stats, + tx_2048b_4095b_frames)}, + {"tx_4096b_9216b_frames", offsetof(struct tx_port_stats, + tx_4096b_9216b_frames)}, + {"tx_9217b_16383b_frames", offsetof(struct tx_port_stats, + tx_9217b_16383b_frames)}, + {"tx_good_frames", offsetof(struct tx_port_stats, + tx_good_frames)}, + {"tx_total_frames", offsetof(struct tx_port_stats, + tx_total_frames)}, + {"tx_ucast_frames", offsetof(struct tx_port_stats, + tx_ucast_frames)}, + {"tx_mcast_frames", offsetof(struct tx_port_stats, + tx_mcast_frames)}, + {"tx_bcast_frames", offsetof(struct tx_port_stats, + tx_bcast_frames)}, + {"tx_pause_frames", offsetof(struct tx_port_stats, + tx_pause_frames)}, + {"tx_pfc_frames", offsetof(struct tx_port_stats, + tx_pfc_frames)}, + {"tx_jabber_frames", offsetof(struct tx_port_stats, + tx_jabber_frames)}, + {"tx_fcs_err_frames", offsetof(struct tx_port_stats, + tx_fcs_err_frames)}, + {"tx_err", offsetof(struct tx_port_stats, + tx_err)}, + {"tx_fifo_underruns", offsetof(struct tx_port_stats, + tx_fifo_underruns)}, + {"tx_eee_lpi_events", offsetof(struct tx_port_stats, + tx_eee_lpi_events)}, + {"tx_eee_lpi_duration", offsetof(struct tx_port_stats, + tx_eee_lpi_duration)}, + {"tx_total_collisions", offsetof(struct tx_port_stats, + tx_total_collisions)}, + {"tx_bytes", offsetof(struct tx_port_stats, + tx_bytes)}, +}; + +static const struct bnxt_xstats_name_off bnxt_func_stats_strings[] = { + {"tx_ucast_pkts", offsetof(struct hwrm_func_qstats_output, + tx_ucast_pkts)}, + {"tx_mcast_pkts", offsetof(struct hwrm_func_qstats_output, + tx_mcast_pkts)}, + {"tx_bcast_pkts", offsetof(struct hwrm_func_qstats_output, + tx_bcast_pkts)}, + {"tx_err_pkts", offsetof(struct hwrm_func_qstats_output, + tx_err_pkts)}, + {"tx_drop_pkts", offsetof(struct hwrm_func_qstats_output, + tx_drop_pkts)}, + {"tx_ucast_bytes", offsetof(struct hwrm_func_qstats_output, + tx_ucast_bytes)}, + {"tx_mcast_bytes", offsetof(struct hwrm_func_qstats_output, + tx_mcast_bytes)}, + {"tx_bcast_bytes", offsetof(struct hwrm_func_qstats_output, + tx_bcast_bytes)}, + {"rx_ucast_pkts", offsetof(struct hwrm_func_qstats_output, + rx_ucast_pkts)}, + {"rx_mcast_pkts", offsetof(struct hwrm_func_qstats_output, + rx_mcast_pkts)}, + {"rx_bcast_pkts", offsetof(struct hwrm_func_qstats_output, + rx_bcast_pkts)}, + {"rx_err_pkts", offsetof(struct hwrm_func_qstats_output, + rx_err_pkts)}, + {"rx_drop_pkts", offsetof(struct hwrm_func_qstats_output, + rx_drop_pkts)}, + {"rx_ucast_bytes", offsetof(struct hwrm_func_qstats_output, + rx_ucast_bytes)}, + {"rx_mcast_bytes", offsetof(struct hwrm_func_qstats_output, + rx_mcast_bytes)}, + {"rx_bcast_bytes", offsetof(struct hwrm_func_qstats_output, + rx_bcast_bytes)}, + {"rx_agg_pkts", offsetof(struct hwrm_func_qstats_output, + rx_agg_pkts)}, + {"rx_agg_bytes", offsetof(struct hwrm_func_qstats_output, + rx_agg_bytes)}, + {"rx_agg_events", offsetof(struct hwrm_func_qstats_output, + rx_agg_events)}, + {"rx_agg_aborts", offsetof(struct hwrm_func_qstats_output, + rx_agg_aborts)}, +}; + /* * Statistics functions */ @@ -63,9 +228,10 @@ void bnxt_free_stats(struct bnxt *bp) } } -void bnxt_stats_get_op(struct rte_eth_dev *eth_dev, +int bnxt_stats_get_op(struct rte_eth_dev *eth_dev, struct rte_eth_stats *bnxt_stats) { + int rc = 0; unsigned int i; struct bnxt *bp = eth_dev->data->dev_private; @@ -74,64 +240,27 @@ void bnxt_stats_get_op(struct rte_eth_dev *eth_dev, for (i = 0; i < bp->rx_cp_nr_rings; i++) { struct bnxt_rx_queue *rxq = bp->rx_queues[i]; struct bnxt_cp_ring_info *cpr = rxq->cp_ring; - struct ctx_hw_stats64 *hw_stats = - (struct ctx_hw_stats64 *)cpr->hw_stats; - bnxt_stats->q_ipackets[i] += - rte_le_to_cpu_64(hw_stats->rx_ucast_pkts); - bnxt_stats->q_ipackets[i] += - rte_le_to_cpu_64(hw_stats->rx_mcast_pkts); - bnxt_stats->q_ipackets[i] += - rte_le_to_cpu_64(hw_stats->rx_bcast_pkts); - - bnxt_stats->q_ibytes[i] += - rte_le_to_cpu_64(hw_stats->rx_ucast_bytes); - bnxt_stats->q_ibytes[i] += - rte_le_to_cpu_64(hw_stats->rx_mcast_bytes); - bnxt_stats->q_ibytes[i] += - rte_le_to_cpu_64(hw_stats->rx_bcast_bytes); - - /* - * TBD: No clear mapping to this... we don't seem - * to have a stat specifically for dropped due to - * insufficient mbufs. - */ - bnxt_stats->q_errors[i] = 0; - - /* These get replaced once the *_QSTATS commands work */ - bnxt_stats->ipackets += bnxt_stats->q_ipackets[i]; - bnxt_stats->ibytes += bnxt_stats->q_ibytes[i]; - bnxt_stats->imissed += bnxt_stats->q_errors[i]; - bnxt_stats->ierrors += - rte_le_to_cpu_64(hw_stats->rx_err_pkts); + rc = bnxt_hwrm_ctx_qstats(bp, cpr->hw_stats_ctx_id, i, + bnxt_stats, 1); + if (unlikely(rc)) + return rc; } for (i = 0; i < bp->tx_cp_nr_rings; i++) { struct bnxt_tx_queue *txq = bp->tx_queues[i]; struct bnxt_cp_ring_info *cpr = txq->cp_ring; - struct ctx_hw_stats64 *hw_stats = - (struct ctx_hw_stats64 *)cpr->hw_stats; - bnxt_stats->q_opackets[i] += - rte_le_to_cpu_64(hw_stats->tx_ucast_pkts); - bnxt_stats->q_opackets[i] += - rte_le_to_cpu_64(hw_stats->tx_mcast_pkts); - bnxt_stats->q_opackets[i] += - rte_le_to_cpu_64(hw_stats->tx_bcast_pkts); - - bnxt_stats->q_obytes[i] += - rte_le_to_cpu_64(hw_stats->tx_ucast_bytes); - bnxt_stats->q_obytes[i] += - rte_le_to_cpu_64(hw_stats->tx_mcast_bytes); - bnxt_stats->q_obytes[i] += - rte_le_to_cpu_64(hw_stats->tx_bcast_bytes); - - /* These get replaced once the *_QSTATS commands work */ - bnxt_stats->opackets += bnxt_stats->q_opackets[i]; - bnxt_stats->obytes += bnxt_stats->q_obytes[i]; - bnxt_stats->oerrors += rte_le_to_cpu_64(hw_stats->tx_drop_pkts); - bnxt_stats->oerrors += rte_le_to_cpu_64(hw_stats->tx_err_pkts); + rc = bnxt_hwrm_ctx_qstats(bp, cpr->hw_stats_ctx_id, i, + bnxt_stats, 0); + if (unlikely(rc)) + return rc; } + rc = bnxt_hwrm_func_qstats(bp, 0xffff, bnxt_stats); + if (unlikely(rc)) + return rc; + bnxt_stats->rx_nombuf = rte_atomic64_read(&bp->rx_mbuf_alloc_fail); + return rc; } void bnxt_stats_reset_op(struct rte_eth_dev *eth_dev) @@ -139,4 +268,154 @@ void bnxt_stats_reset_op(struct rte_eth_dev *eth_dev) struct bnxt *bp = (struct bnxt *)eth_dev->data->dev_private; bnxt_clear_all_hwrm_stat_ctxs(bp); + rte_atomic64_clear(&bp->rx_mbuf_alloc_fail); +} + +int bnxt_dev_xstats_get_op(struct rte_eth_dev *eth_dev, + struct rte_eth_xstat *xstats, unsigned int n) +{ + struct bnxt *bp = (struct bnxt *)eth_dev->data->dev_private; + + unsigned int count, i; + uint64_t tx_drop_pkts; + + if (!(bp->flags & BNXT_FLAG_PORT_STATS)) { + RTE_LOG(ERR, PMD, "xstats not supported for VF\n"); + return 0; + } + + bnxt_hwrm_port_qstats(bp); + bnxt_hwrm_func_qstats_tx_drop(bp, 0xffff, &tx_drop_pkts); + + count = RTE_DIM(bnxt_rx_stats_strings) + + RTE_DIM(bnxt_tx_stats_strings) + 1; /* For tx_drop_pkts */ + + if (n < count) + return count; + + count = 0; + for (i = 0; i < RTE_DIM(bnxt_rx_stats_strings); i++) { + uint64_t *rx_stats = (uint64_t *)bp->hw_rx_port_stats; + xstats[count].value = rte_le_to_cpu_64( + *(uint64_t *)((char *)rx_stats + + bnxt_rx_stats_strings[i].offset)); + count++; + } + + for (i = 0; i < RTE_DIM(bnxt_tx_stats_strings); i++) { + uint64_t *tx_stats = (uint64_t *)bp->hw_tx_port_stats; + xstats[count].value = rte_le_to_cpu_64( + *(uint64_t *)((char *)tx_stats + + bnxt_tx_stats_strings[i].offset)); + count++; + } + + /* The Tx drop pkts aka the Anti spoof coounter */ + xstats[count].value = rte_le_to_cpu_64(tx_drop_pkts); + count++; + + return count; +} + +int bnxt_dev_xstats_get_names_op(__rte_unused struct rte_eth_dev *eth_dev, + struct rte_eth_xstat_name *xstats_names, + __rte_unused unsigned int limit) +{ + /* Account for the Tx drop pkts aka the Anti spoof counter */ + const unsigned int stat_cnt = RTE_DIM(bnxt_rx_stats_strings) + + RTE_DIM(bnxt_tx_stats_strings) + 1; + unsigned int i, count; + + if (xstats_names != NULL) { + count = 0; + + for (i = 0; i < RTE_DIM(bnxt_rx_stats_strings); i++) { + snprintf(xstats_names[count].name, + sizeof(xstats_names[count].name), + "%s", + bnxt_rx_stats_strings[i].name); + count++; + } + + for (i = 0; i < RTE_DIM(bnxt_tx_stats_strings); i++) { + snprintf(xstats_names[count].name, + sizeof(xstats_names[count].name), + "%s", + bnxt_tx_stats_strings[i].name); + count++; + } + + snprintf(xstats_names[count].name, + sizeof(xstats_names[count].name), + "%s", + bnxt_func_stats_strings[4].name); + count++; + } + return stat_cnt; +} + +void bnxt_dev_xstats_reset_op(struct rte_eth_dev *eth_dev) +{ + struct bnxt *bp = (struct bnxt *)eth_dev->data->dev_private; + + if (bp->flags & BNXT_FLAG_PORT_STATS && !BNXT_NPAR_PF(bp)) + bnxt_hwrm_port_clr_stats(bp); + + if (BNXT_VF(bp)) + RTE_LOG(ERR, PMD, "Operation not supported on a VF device\n"); + if (BNXT_NPAR_PF(bp)) + RTE_LOG(ERR, PMD, "Operation not supported on a MF device\n"); + if (!(bp->flags & BNXT_FLAG_PORT_STATS)) + RTE_LOG(ERR, PMD, "Operation not supported\n"); +} + +int bnxt_dev_xstats_get_by_id_op(struct rte_eth_dev *dev, const uint64_t *ids, + uint64_t *values, unsigned int limit) +{ + /* Account for the Tx drop pkts aka the Anti spoof counter */ + const unsigned int stat_cnt = RTE_DIM(bnxt_rx_stats_strings) + + RTE_DIM(bnxt_tx_stats_strings) + 1; + struct rte_eth_xstat xstats[stat_cnt]; + uint64_t values_copy[stat_cnt]; + uint16_t i; + + if (!ids) + return bnxt_dev_xstats_get_op(dev, xstats, stat_cnt); + + bnxt_dev_xstats_get_by_id_op(dev, NULL, values_copy, stat_cnt); + for (i = 0; i < limit; i++) { + if (ids[i] >= stat_cnt) { + RTE_LOG(ERR, PMD, "id value isn't valid"); + return -1; + } + values[i] = values_copy[ids[i]]; + } + return stat_cnt; +} + +int bnxt_dev_xstats_get_names_by_id_op(struct rte_eth_dev *dev, + struct rte_eth_xstat_name *xstats_names, + const uint64_t *ids, unsigned int limit) +{ + /* Account for the Tx drop pkts aka the Anti spoof counter */ + const unsigned int stat_cnt = RTE_DIM(bnxt_rx_stats_strings) + + RTE_DIM(bnxt_tx_stats_strings) + 1; + struct rte_eth_xstat_name xstats_names_copy[stat_cnt]; + uint16_t i; + + if (!ids) + return bnxt_dev_xstats_get_names_op(dev, xstats_names, + stat_cnt); + bnxt_dev_xstats_get_names_by_id_op(dev, xstats_names_copy, NULL, + stat_cnt); + + for (i = 0; i < limit; i++) { + if (ids[i] >= stat_cnt) { + RTE_LOG(ERR, PMD, "id value isn't valid"); + return -1; + } + strcpy(xstats_names[i].name, + xstats_names_copy[ids[i]].name); + } + return stat_cnt; } diff --git a/dpdk/drivers/net/bnxt/bnxt_stats.h b/dpdk/drivers/net/bnxt/bnxt_stats.h index 65408a44..51d16f5d 100644 --- a/dpdk/drivers/net/bnxt/bnxt_stats.h +++ b/dpdk/drivers/net/bnxt/bnxt_stats.h @@ -37,8 +37,23 @@ #include void bnxt_free_stats(struct bnxt *bp); -void bnxt_stats_get_op(struct rte_eth_dev *eth_dev, +int bnxt_stats_get_op(struct rte_eth_dev *eth_dev, struct rte_eth_stats *bnxt_stats); void bnxt_stats_reset_op(struct rte_eth_dev *eth_dev); +int bnxt_dev_xstats_get_names_op(__rte_unused struct rte_eth_dev *eth_dev, + struct rte_eth_xstat_name *xstats_names, + __rte_unused unsigned int limit); +int bnxt_dev_xstats_get_op(struct rte_eth_dev *eth_dev, + struct rte_eth_xstat *xstats, unsigned int n); +void bnxt_dev_xstats_reset_op(struct rte_eth_dev *eth_dev); +int bnxt_dev_xstats_get_by_id_op(struct rte_eth_dev *dev, const uint64_t *ids, + uint64_t *values, unsigned int limit); +int bnxt_dev_xstats_get_names_by_id_op(struct rte_eth_dev *dev, + struct rte_eth_xstat_name *xstats_names, + const uint64_t *ids, unsigned int limit); +struct bnxt_xstats_name_off { + char name[RTE_ETH_XSTATS_NAME_SIZE]; + uint64_t offset; +}; #endif diff --git a/dpdk/drivers/net/bnxt/bnxt_txq.h b/dpdk/drivers/net/bnxt/bnxt_txq.h index 16f3a0bd..f753c10f 100644 --- a/dpdk/drivers/net/bnxt/bnxt_txq.h +++ b/dpdk/drivers/net/bnxt/bnxt_txq.h @@ -46,7 +46,7 @@ struct bnxt_tx_queue { uint16_t tx_next_rs; /* next desc to set RS bit */ uint16_t queue_id; /* TX queue index */ uint16_t reg_idx; /* TX queue register index */ - uint8_t port_id; /* Device port identifier */ + uint16_t port_id; /* Device port identifier */ uint8_t pthresh; /* Prefetch threshold register */ uint8_t hthresh; /* Host threshold register */ uint8_t wthresh; /* Write-back threshold reg */ diff --git a/dpdk/drivers/net/bnxt/bnxt_txr.c b/dpdk/drivers/net/bnxt/bnxt_txr.c index 8bf8fee3..03d2652c 100644 --- a/dpdk/drivers/net/bnxt/bnxt_txr.c +++ b/dpdk/drivers/net/bnxt/bnxt_txr.c @@ -101,7 +101,7 @@ int bnxt_init_tx_ring_struct(struct bnxt_tx_queue *txq, unsigned int socket_id) if (ring == NULL) return -ENOMEM; txr->tx_ring_struct = ring; - ring->ring_size = rte_align32pow2(txq->nb_tx_desc + 1); + ring->ring_size = rte_align32pow2(txq->nb_tx_desc); ring->ring_mask = ring->ring_size - 1; ring->bd = (void *)txr->tx_desc_ring; ring->bd_dma = txr->tx_desc_mapping; @@ -161,7 +161,7 @@ static uint16_t bnxt_start_xmit(struct rte_mbuf *tx_pkt, if (tx_pkt->ol_flags & (PKT_TX_TCP_SEG | PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM | PKT_TX_IP_CKSUM | - PKT_TX_VLAN_PKT)) + PKT_TX_VLAN_PKT | PKT_TX_OUTER_IP_CKSUM)) long_bd = true; tx_buf = &txr->tx_buf_ring[txr->tx_prod]; @@ -211,20 +211,44 @@ static uint16_t bnxt_start_xmit(struct rte_mbuf *tx_pkt, if (tx_pkt->ol_flags & PKT_TX_TCP_SEG) { /* TSO */ - txbd1->lflags = TX_BD_LONG_LFLAGS_LSO; + txbd1->lflags |= TX_BD_LONG_LFLAGS_LSO; txbd1->hdr_size = tx_pkt->l2_len + tx_pkt->l3_len + - tx_pkt->l4_len; + tx_pkt->l4_len + tx_pkt->outer_l2_len + + tx_pkt->outer_l3_len; txbd1->mss = tx_pkt->tso_segsz; - } else if (tx_pkt->ol_flags & (PKT_TX_TCP_CKSUM | - PKT_TX_UDP_CKSUM)) { - /* TCP/UDP CSO */ - txbd1->lflags = TX_BD_LONG_LFLAGS_TCP_UDP_CHKSUM; + } else if ((tx_pkt->ol_flags & PKT_TX_OIP_IIP_TCP_UDP_CKSUM) == + PKT_TX_OIP_IIP_TCP_UDP_CKSUM) { + /* Outer IP, Inner IP, Inner TCP/UDP CSO */ + txbd1->lflags |= TX_BD_FLG_TIP_IP_TCP_UDP_CHKSUM; + txbd1->mss = 0; + } else if ((tx_pkt->ol_flags & PKT_TX_IIP_TCP_UDP_CKSUM) == + PKT_TX_IIP_TCP_UDP_CKSUM) { + /* (Inner) IP, (Inner) TCP/UDP CSO */ + txbd1->lflags |= TX_BD_FLG_IP_TCP_UDP_CHKSUM; + txbd1->mss = 0; + } else if ((tx_pkt->ol_flags & PKT_TX_OIP_TCP_UDP_CKSUM) == + PKT_TX_OIP_TCP_UDP_CKSUM) { + /* Outer IP, (Inner) TCP/UDP CSO */ + txbd1->lflags |= TX_BD_FLG_TIP_TCP_UDP_CHKSUM; + txbd1->mss = 0; + } else if ((tx_pkt->ol_flags & PKT_TX_OIP_IIP_CKSUM) == + PKT_TX_OIP_IIP_CKSUM) { + /* Outer IP, Inner IP CSO */ + txbd1->lflags |= TX_BD_FLG_TIP_IP_CHKSUM; + txbd1->mss = 0; + } else if ((tx_pkt->ol_flags & PKT_TX_TCP_UDP_CKSUM) == + PKT_TX_TCP_UDP_CKSUM) { + /* TCP/UDP CSO */ + txbd1->lflags |= TX_BD_LONG_LFLAGS_TCP_UDP_CHKSUM; txbd1->mss = 0; - } else if (tx_pkt->ol_flags & PKT_TX_IP_CKSUM) { /* IP CSO */ - txbd1->lflags = TX_BD_LONG_LFLAGS_IP_CHKSUM; + txbd1->lflags |= TX_BD_LONG_LFLAGS_IP_CHKSUM; + txbd1->mss = 0; + } else if (tx_pkt->ol_flags & PKT_TX_OUTER_IP_CKSUM) { + /* IP CSO */ + txbd1->lflags |= TX_BD_LONG_LFLAGS_T_IP_CHKSUM; txbd1->mss = 0; } } else { @@ -294,11 +318,14 @@ static int bnxt_handle_tx_cp(struct bnxt_tx_queue *txq) if (!CMP_VALID(txcmp, raw_cons, cpr->cp_ring_struct)) break; + cpr->valid = FLIP_VALID(cons, + cpr->cp_ring_struct->ring_mask, + cpr->valid); if (CMP_TYPE(txcmp) == TX_CMPL_TYPE_TX_L2) nb_tx_pkts++; else - RTE_LOG(DEBUG, PMD, + RTE_LOG_DP(DEBUG, PMD, "Unhandled CMP type %02x\n", CMP_TYPE(txcmp)); raw_cons = NEXT_RAW_CMP(raw_cons); diff --git a/dpdk/drivers/net/bnxt/bnxt_txr.h b/dpdk/drivers/net/bnxt/bnxt_txr.h index 2176acaa..2feac51d 100644 --- a/dpdk/drivers/net/bnxt/bnxt_txr.h +++ b/dpdk/drivers/net/bnxt/bnxt_txr.h @@ -34,11 +34,12 @@ #ifndef _BNXT_TXR_H_ #define _BNXT_TXR_H_ +#include + #define MAX_TX_RINGS 16 #define BNXT_TX_PUSH_THRESH 92 -#define B_TX_DB(db, prod) \ - (*(uint32_t *)db = (DB_KEY_TX | prod)) +#define B_TX_DB(db, prod) rte_write32((DB_KEY_TX | (prod)), db) struct bnxt_tx_ring_info { uint16_t tx_prod; @@ -48,7 +49,7 @@ struct bnxt_tx_ring_info { struct tx_bd_long *tx_desc_ring; struct bnxt_sw_tx_bd *tx_buf_ring; - phys_addr_t tx_desc_mapping; + rte_iova_t tx_desc_mapping; #define BNXT_DEV_STATE_CLOSING 0x1 uint32_t dev_state; @@ -68,4 +69,25 @@ int bnxt_init_tx_ring_struct(struct bnxt_tx_queue *txq, unsigned int socket_id); uint16_t bnxt_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts); +#define PKT_TX_OIP_IIP_TCP_UDP_CKSUM (PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM | \ + PKT_TX_IP_CKSUM | PKT_TX_OUTER_IP_CKSUM) +#define PKT_TX_IIP_TCP_UDP_CKSUM (PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM | \ + PKT_TX_IP_CKSUM) +#define PKT_TX_OIP_TCP_UDP_CKSUM (PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM | \ + PKT_TX_OUTER_IP_CKSUM) +#define PKT_TX_OIP_IIP_CKSUM (PKT_TX_IP_CKSUM | \ + PKT_TX_OUTER_IP_CKSUM) +#define PKT_TX_TCP_UDP_CKSUM (PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM) + + +#define TX_BD_FLG_TIP_IP_TCP_UDP_CHKSUM (TX_BD_LONG_LFLAGS_TCP_UDP_CHKSUM | \ + TX_BD_LONG_LFLAGS_T_IP_CHKSUM | \ + TX_BD_LONG_LFLAGS_IP_CHKSUM) +#define TX_BD_FLG_IP_TCP_UDP_CHKSUM (TX_BD_LONG_LFLAGS_TCP_UDP_CHKSUM | \ + TX_BD_LONG_LFLAGS_IP_CHKSUM) +#define TX_BD_FLG_TIP_IP_CHKSUM (TX_BD_LONG_LFLAGS_T_IP_CHKSUM | \ + TX_BD_LONG_LFLAGS_IP_CHKSUM) +#define TX_BD_FLG_TIP_TCP_UDP_CHKSUM (TX_BD_LONG_LFLAGS_TCP_UDP_CHKSUM | \ + TX_BD_LONG_LFLAGS_T_IP_CHKSUM) + #endif diff --git a/dpdk/drivers/net/bnxt/bnxt_vnic.c b/dpdk/drivers/net/bnxt/bnxt_vnic.c index 1b5f54c4..5bac2605 100644 --- a/dpdk/drivers/net/bnxt/bnxt_vnic.c +++ b/dpdk/drivers/net/bnxt/bnxt_vnic.c @@ -69,27 +69,21 @@ void bnxt_init_vnics(struct bnxt *bp) uint16_t max_vnics; int i, j; - if (BNXT_PF(bp)) { - struct bnxt_pf_info *pf = &bp->pf; - - max_vnics = pf->max_vnics; - } else { - struct bnxt_vf_info *vf = &bp->vf; - - max_vnics = vf->max_vnics; - } + max_vnics = bp->max_vnics; STAILQ_INIT(&bp->free_vnic_list); for (i = 0; i < max_vnics; i++) { vnic = &bp->vnic_info[i]; vnic->fw_vnic_id = (uint16_t)HWRM_NA_SIGNATURE; - vnic->fw_rss_cos_lb_ctx = (uint16_t)HWRM_NA_SIGNATURE; - vnic->ctx_is_rss_cos_lb = HW_CONTEXT_NONE; + vnic->rss_rule = (uint16_t)HWRM_NA_SIGNATURE; + vnic->cos_rule = (uint16_t)HWRM_NA_SIGNATURE; + vnic->lb_rule = (uint16_t)HWRM_NA_SIGNATURE; for (j = 0; j < MAX_QUEUES_PER_VNIC; j++) vnic->fw_grp_ids[j] = (uint16_t)HWRM_NA_SIGNATURE; prandom_bytes(vnic->rss_hash_key, HW_HASH_KEY_SIZE); STAILQ_INIT(&vnic->filter); + STAILQ_INIT(&vnic->flow_list); STAILQ_INSERT_TAIL(&bp->free_vnic_list, vnic, next); } for (i = 0; i < MAX_FF_POOLS; i++) @@ -175,21 +169,15 @@ int bnxt_alloc_vnic_attributes(struct bnxt *bp) struct rte_pci_device *pdev = bp->pdev; const struct rte_memzone *mz; char mz_name[RTE_MEMZONE_NAMESIZE]; - uint16_t entry_length = RTE_CACHE_LINE_ROUNDUP( + uint32_t entry_length = RTE_CACHE_LINE_ROUNDUP( HW_HASH_INDEX_SIZE * sizeof(*vnic->rss_table) + - HW_HASH_KEY_SIZE); + HW_HASH_KEY_SIZE + + BNXT_MAX_MC_ADDRS * ETHER_ADDR_LEN); uint16_t max_vnics; int i; + rte_iova_t mz_phys_addr; - if (BNXT_PF(bp)) { - struct bnxt_pf_info *pf = &bp->pf; - - max_vnics = pf->max_vnics; - } else { - struct bnxt_vf_info *vf = &bp->vf; - - max_vnics = vf->max_vnics; - } + max_vnics = bp->max_vnics; snprintf(mz_name, RTE_MEMZONE_NAMESIZE, "bnxt_%04x:%02x:%02x:%02x_vnicattr", pdev->addr.domain, pdev->addr.bus, pdev->addr.devid, pdev->addr.function); @@ -204,6 +192,19 @@ int bnxt_alloc_vnic_attributes(struct bnxt *bp) if (!mz) return -ENOMEM; } + mz_phys_addr = mz->iova; + if ((unsigned long)mz->addr == mz_phys_addr) { + RTE_LOG(WARNING, PMD, + "Memzone physical address same as virtual.\n"); + RTE_LOG(WARNING, PMD, + "Using rte_mem_virt2iova()\n"); + mz_phys_addr = rte_mem_virt2iova(mz->addr); + if (mz_phys_addr == 0) { + RTE_LOG(ERR, PMD, + "unable to map vnic address to physical memory\n"); + return -ENOMEM; + } + } for (i = 0; i < max_vnics; i++) { vnic = &bp->vnic_info[i]; @@ -213,12 +214,16 @@ int bnxt_alloc_vnic_attributes(struct bnxt *bp) (void *)((char *)mz->addr + (entry_length * i)); memset(vnic->rss_table, -1, entry_length); - vnic->rss_table_dma_addr = mz->phys_addr + (entry_length * i); + vnic->rss_table_dma_addr = mz_phys_addr + (entry_length * i); vnic->rss_hash_key = (void *)((char *)vnic->rss_table + HW_HASH_INDEX_SIZE * sizeof(*vnic->rss_table)); vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + HW_HASH_INDEX_SIZE * sizeof(*vnic->rss_table); + vnic->mc_list = (void *)((char *)vnic->rss_hash_key + + HW_HASH_KEY_SIZE); + vnic->mc_list_dma_addr = vnic->rss_hash_key_dma_addr + + HW_HASH_KEY_SIZE; } return 0; @@ -229,15 +234,10 @@ void bnxt_free_vnic_mem(struct bnxt *bp) struct bnxt_vnic_info *vnic; uint16_t max_vnics, i; - if (BNXT_PF(bp)) { - struct bnxt_pf_info *pf = &bp->pf; + if (bp->vnic_info == NULL) + return; - max_vnics = pf->max_vnics; - } else { - struct bnxt_vf_info *vf = &bp->vf; - - max_vnics = vf->max_vnics; - } + max_vnics = bp->max_vnics; for (i = 0; i < max_vnics; i++) { vnic = &bp->vnic_info[i]; if (vnic->fw_vnic_id != (uint16_t)HWRM_NA_SIGNATURE) { @@ -255,15 +255,7 @@ int bnxt_alloc_vnic_mem(struct bnxt *bp) struct bnxt_vnic_info *vnic_mem; uint16_t max_vnics; - if (BNXT_PF(bp)) { - struct bnxt_pf_info *pf = &bp->pf; - - max_vnics = pf->max_vnics; - } else { - struct bnxt_vf_info *vf = &bp->vf; - - max_vnics = vf->max_vnics; - } + max_vnics = bp->max_vnics; /* Allocate memory for VNIC pool and filter pool */ vnic_mem = rte_zmalloc("bnxt_vnic_info", max_vnics * sizeof(struct bnxt_vnic_info), 0); diff --git a/dpdk/drivers/net/bnxt/bnxt_vnic.h b/dpdk/drivers/net/bnxt/bnxt_vnic.h index 9671ba42..875dc3c1 100644 --- a/dpdk/drivers/net/bnxt/bnxt_vnic.h +++ b/dpdk/drivers/net/bnxt/bnxt_vnic.h @@ -42,8 +42,7 @@ struct bnxt_vnic_info { uint8_t ff_pool_idx; uint16_t fw_vnic_id; /* returned by Chimp during alloc */ - uint16_t fw_rss_cos_lb_ctx; - uint16_t ctx_is_rss_cos_lb; + uint16_t rss_rule; #define MAX_NUM_TRAFFIC_CLASSES 8 #define MAX_NUM_RSS_QUEUES_PER_VNIC 16 #define MAX_QUEUES_PER_VNIC (MAX_NUM_RSS_QUEUES_PER_VNIC + \ @@ -51,19 +50,37 @@ struct bnxt_vnic_info { uint16_t start_grp_id; uint16_t end_grp_id; uint16_t fw_grp_ids[MAX_QUEUES_PER_VNIC]; + uint16_t dflt_ring_grp; + uint16_t mru; uint16_t hash_type; - phys_addr_t rss_table_dma_addr; + rte_iova_t rss_table_dma_addr; uint16_t *rss_table; - phys_addr_t rss_hash_key_dma_addr; + rte_iova_t rss_hash_key_dma_addr; void *rss_hash_key; + rte_iova_t mc_list_dma_addr; + char *mc_list; + uint32_t mc_addr_cnt; +#define BNXT_MAX_MC_ADDRS 16 uint32_t flags; #define BNXT_VNIC_INFO_PROMISC (1 << 0) #define BNXT_VNIC_INFO_ALLMULTI (1 << 1) +#define BNXT_VNIC_INFO_BCAST (1 << 2) +#define BNXT_VNIC_INFO_UCAST (1 << 3) +#define BNXT_VNIC_INFO_MCAST (1 << 4) +#define BNXT_VNIC_INFO_TAGGED (1 << 5) +#define BNXT_VNIC_INFO_UNTAGGED (1 << 6) + uint16_t cos_rule; + uint16_t lb_rule; bool vlan_strip; bool func_default; + bool bd_stall; + bool roce_dual; + bool roce_only; + bool rss_dflt_cr; STAILQ_HEAD(, bnxt_filter_info) filter; + STAILQ_HEAD(, rte_flow) flow_list; }; struct bnxt; diff --git a/dpdk/drivers/net/bnxt/hsi_struct_def_dpdk.h b/dpdk/drivers/net/bnxt/hsi_struct_def_dpdk.h index f2db3eaf..c16edbad 100644 --- a/dpdk/drivers/net/bnxt/hsi_struct_def_dpdk.h +++ b/dpdk/drivers/net/bnxt/hsi_struct_def_dpdk.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) Broadcom Limited. + * Copyright(c) 2001-2017 Broadcom Limited. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -31,71 +31,91 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#ifndef _HSI_STRUCT_DEF_EXTERNAL_H_ -#define _HSI_STRUCT_DEF_EXTERNAL_H_ - -/* - * per-context HW statistics -- chip view - */ - -struct ctx_hw_stats64 { - uint64_t rx_ucast_pkts; - uint64_t rx_mcast_pkts; - uint64_t rx_bcast_pkts; - uint64_t rx_drop_pkts; - uint64_t rx_err_pkts; - uint64_t rx_ucast_bytes; - uint64_t rx_mcast_bytes; - uint64_t rx_bcast_bytes; - - uint64_t tx_ucast_pkts; - uint64_t tx_mcast_pkts; - uint64_t tx_bcast_pkts; - uint64_t tx_drop_pkts; - uint64_t tx_err_pkts; - uint64_t tx_ucast_bytes; - uint64_t tx_mcast_bytes; - uint64_t tx_bcast_bytes; - - uint64_t tpa_pkts; - uint64_t tpa_bytes; - uint64_t tpa_events; - uint64_t tpa_aborts; -} ctx_hw_stats64_t; - -/* HW Resource Manager Specification 1.2.0 */ +#ifndef _HSI_STRUCT_DEF_DPDK_ +#define _HSI_STRUCT_DEF_DPDK_ +/* HSI and HWRM Specification 1.8.2 */ #define HWRM_VERSION_MAJOR 1 -#define HWRM_VERSION_MINOR 2 -#define HWRM_VERSION_UPDATE 0 +#define HWRM_VERSION_MINOR 8 +#define HWRM_VERSION_UPDATE 2 +#define HWRM_VERSION_RSVD 0 /* non-zero means beta version */ + +#define HWRM_VERSION_STR "1.8.2.0" /* * Following is the signature for HWRM message field that indicates not - * applicable (All F's). Need to cast it the size of the field if needed. + * applicable (All F's). Need to cast it the size of the field if needed. */ -#define HWRM_NA_SIGNATURE ((uint32_t)(-1)) +#define HWRM_NA_SIGNATURE ((uint32_t)(-1)) #define HWRM_MAX_REQ_LEN (128) /* hwrm_func_buf_rgtr */ -#define HWRM_MAX_RESP_LEN (176) /* hwrm_func_qstats */ -#define HW_HASH_INDEX_SIZE 0x80 /* 7 bit indirection table index. */ -#define HW_HASH_KEY_SIZE 40 +#define HWRM_MAX_RESP_LEN (280) /* hwrm_selftest_qlist */ +#define HW_HASH_INDEX_SIZE 0x80 /* 7 bit indirection table index. */ +#define HW_HASH_KEY_SIZE 40 #define HWRM_RESP_VALID_KEY 1 /* valid key for HWRM response */ +#define HWRM_ROCE_SP_HSI_VERSION_MAJOR 1 +#define HWRM_ROCE_SP_HSI_VERSION_MINOR 8 +#define HWRM_ROCE_SP_HSI_VERSION_UPDATE 2 /* * Request types */ #define HWRM_VER_GET (UINT32_C(0x0)) +#define HWRM_FUNC_BUF_UNRGTR (UINT32_C(0xe)) +#define HWRM_FUNC_VF_CFG (UINT32_C(0xf)) + /* Reserved for future use */ +#define RESERVED1 (UINT32_C(0x10)) #define HWRM_FUNC_RESET (UINT32_C(0x11)) +#define HWRM_FUNC_GETFID (UINT32_C(0x12)) +#define HWRM_FUNC_VF_ALLOC (UINT32_C(0x13)) +#define HWRM_FUNC_VF_FREE (UINT32_C(0x14)) #define HWRM_FUNC_QCAPS (UINT32_C(0x15)) +#define HWRM_FUNC_QCFG (UINT32_C(0x16)) +#define HWRM_FUNC_CFG (UINT32_C(0x17)) +#define HWRM_FUNC_QSTATS (UINT32_C(0x18)) +#define HWRM_FUNC_CLR_STATS (UINT32_C(0x19)) #define HWRM_FUNC_DRV_UNRGTR (UINT32_C(0x1a)) +#define HWRM_FUNC_VF_RESC_FREE (UINT32_C(0x1b)) +#define HWRM_FUNC_VF_VNIC_IDS_QUERY (UINT32_C(0x1c)) #define HWRM_FUNC_DRV_RGTR (UINT32_C(0x1d)) +#define HWRM_FUNC_DRV_QVER (UINT32_C(0x1e)) +#define HWRM_FUNC_BUF_RGTR (UINT32_C(0x1f)) #define HWRM_PORT_PHY_CFG (UINT32_C(0x20)) +#define HWRM_PORT_MAC_CFG (UINT32_C(0x21)) +#define HWRM_PORT_QSTATS (UINT32_C(0x23)) +#define HWRM_PORT_LPBK_QSTATS (UINT32_C(0x24)) +#define HWRM_PORT_CLR_STATS (UINT32_C(0x25)) #define HWRM_PORT_PHY_QCFG (UINT32_C(0x27)) +#define HWRM_PORT_MAC_QCFG (UINT32_C(0x28)) +#define HWRM_PORT_PHY_QCAPS (UINT32_C(0x2a)) +#define HWRM_PORT_LED_CFG (UINT32_C(0x2d)) +#define HWRM_PORT_LED_QCFG (UINT32_C(0x2e)) +#define HWRM_PORT_LED_QCAPS (UINT32_C(0x2f)) #define HWRM_QUEUE_QPORTCFG (UINT32_C(0x30)) +#define HWRM_QUEUE_QCFG (UINT32_C(0x31)) +#define HWRM_QUEUE_CFG (UINT32_C(0x32)) +#define HWRM_FUNC_VLAN_CFG (UINT32_C(0x33)) +#define HWRM_FUNC_VLAN_QCFG (UINT32_C(0x34)) +#define HWRM_QUEUE_PFCENABLE_QCFG (UINT32_C(0x35)) +#define HWRM_QUEUE_PFCENABLE_CFG (UINT32_C(0x36)) +#define HWRM_QUEUE_PRI2COS_QCFG (UINT32_C(0x37)) +#define HWRM_QUEUE_PRI2COS_CFG (UINT32_C(0x38)) +#define HWRM_QUEUE_COS2BW_QCFG (UINT32_C(0x39)) +#define HWRM_QUEUE_COS2BW_CFG (UINT32_C(0x3a)) +#define HWRM_VNIC_ALLOC (UINT32_C(0x40)) #define HWRM_VNIC_ALLOC (UINT32_C(0x40)) #define HWRM_VNIC_FREE (UINT32_C(0x41)) #define HWRM_VNIC_CFG (UINT32_C(0x42)) +#define HWRM_VNIC_QCFG (UINT32_C(0x43)) +#define HWRM_VNIC_TPA_CFG (UINT32_C(0x44)) #define HWRM_VNIC_RSS_CFG (UINT32_C(0x46)) +#define HWRM_VNIC_RSS_QCFG (UINT32_C(0x47)) +#define HWRM_VNIC_PLCMODES_CFG (UINT32_C(0x48)) +#define HWRM_VNIC_PLCMODES_QCFG (UINT32_C(0x49)) +#define HWRM_VNIC_QCAPS (UINT32_C(0x4a)) #define HWRM_RING_ALLOC (UINT32_C(0x50)) #define HWRM_RING_FREE (UINT32_C(0x51)) +#define HWRM_RING_CMPL_RING_QAGGINT_PARAMS (UINT32_C(0x52)) +#define HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS (UINT32_C(0x53)) +#define HWRM_RING_RESET (UINT32_C(0x5e)) #define HWRM_RING_GRP_ALLOC (UINT32_C(0x60)) #define HWRM_RING_GRP_FREE (UINT32_C(0x61)) #define HWRM_VNIC_RSS_COS_LB_CTX_ALLOC (UINT32_C(0x70)) @@ -104,3483 +124,1708 @@ struct ctx_hw_stats64 { #define HWRM_CFA_L2_FILTER_FREE (UINT32_C(0x91)) #define HWRM_CFA_L2_FILTER_CFG (UINT32_C(0x92)) #define HWRM_CFA_L2_SET_RX_MASK (UINT32_C(0x93)) + /* Reserved for future use */ +#define HWRM_CFA_VLAN_ANTISPOOF_CFG (UINT32_C(0x94)) +#define HWRM_CFA_TUNNEL_FILTER_ALLOC (UINT32_C(0x95)) +#define HWRM_CFA_TUNNEL_FILTER_FREE (UINT32_C(0x96)) +#define HWRM_CFA_NTUPLE_FILTER_ALLOC (UINT32_C(0x99)) +#define HWRM_CFA_NTUPLE_FILTER_FREE (UINT32_C(0x9a)) +#define HWRM_CFA_NTUPLE_FILTER_CFG (UINT32_C(0x9b)) +#define HWRM_CFA_EM_FLOW_ALLOC (UINT32_C(0x9c)) +#define HWRM_CFA_EM_FLOW_FREE (UINT32_C(0x9d)) +#define HWRM_CFA_EM_FLOW_CFG (UINT32_C(0x9e)) +#define HWRM_TUNNEL_DST_PORT_QUERY (UINT32_C(0xa0)) +#define HWRM_TUNNEL_DST_PORT_ALLOC (UINT32_C(0xa1)) +#define HWRM_TUNNEL_DST_PORT_FREE (UINT32_C(0xa2)) #define HWRM_STAT_CTX_ALLOC (UINT32_C(0xb0)) #define HWRM_STAT_CTX_FREE (UINT32_C(0xb1)) +#define HWRM_STAT_CTX_QUERY (UINT32_C(0xb2)) #define HWRM_STAT_CTX_CLR_STATS (UINT32_C(0xb3)) +#define HWRM_FW_RESET (UINT32_C(0xc0)) +#define HWRM_FW_QSTATUS (UINT32_C(0xc1)) #define HWRM_EXEC_FWD_RESP (UINT32_C(0xd0)) +#define HWRM_REJECT_FWD_RESP (UINT32_C(0xd1)) +#define HWRM_FWD_RESP (UINT32_C(0xd2)) +#define HWRM_FWD_ASYNC_EVENT_CMPL (UINT32_C(0xd3)) +#define HWRM_TEMP_MONITOR_QUERY (UINT32_C(0xe0)) +#define HWRM_WOL_FILTER_ALLOC (UINT32_C(0xf0)) +#define HWRM_WOL_FILTER_FREE (UINT32_C(0xf1)) +#define HWRM_WOL_FILTER_QCFG (UINT32_C(0xf2)) +#define HWRM_WOL_REASON_QCFG (UINT32_C(0xf3)) +#define HWRM_DBG_DUMP (UINT32_C(0xff14)) +#define HWRM_NVM_VALIDATE_OPTION (UINT32_C(0xffef)) +#define HWRM_NVM_FLUSH (UINT32_C(0xfff0)) +#define HWRM_NVM_GET_VARIABLE (UINT32_C(0xfff1)) +#define HWRM_NVM_SET_VARIABLE (UINT32_C(0xfff2)) +#define HWRM_NVM_INSTALL_UPDATE (UINT32_C(0xfff3)) +#define HWRM_NVM_MODIFY (UINT32_C(0xfff4)) +#define HWRM_NVM_VERIFY_UPDATE (UINT32_C(0xfff5)) +#define HWRM_NVM_GET_DEV_INFO (UINT32_C(0xfff6)) +#define HWRM_NVM_ERASE_DIR_ENTRY (UINT32_C(0xfff7)) +#define HWRM_NVM_MOD_DIR_ENTRY (UINT32_C(0xfff8)) +#define HWRM_NVM_FIND_DIR_ENTRY (UINT32_C(0xfff9)) +#define HWRM_NVM_GET_DIR_ENTRIES (UINT32_C(0xfffa)) +#define HWRM_NVM_GET_DIR_INFO (UINT32_C(0xfffb)) +#define HWRM_NVM_RAW_DUMP (UINT32_C(0xfffc)) +#define HWRM_NVM_READ (UINT32_C(0xfffd)) +#define HWRM_NVM_WRITE (UINT32_C(0xfffe)) +#define HWRM_NVM_RAW_WRITE_BLK (UINT32_C(0xffff)) -/* Return Codes */ -#define HWRM_ERR_CODE_INVALID_PARAMS (UINT32_C(0x2)) -#define HWRM_ERR_CODE_RESOURCE_ACCESS_DENIED (UINT32_C(0x3)) - -/* Short TX BD (16 bytes) */ +/* + * Note: The Host Software Interface (HSI) and Hardware Resource Manager (HWRM) + * specification describes the data structures used in Ethernet packet or RDMA + * message data transfers as well as an abstract interface for managing Ethernet + * NIC hardware resources. + */ +/* Ethernet Data path Host Structures */ +/* + * Description: The following three sections document the host structures used + * between device and software drivers for communicating Ethernet packets. + */ +/* BD Ring Structures */ +/* + * Description: This structure is used to inform the NIC of a location for and + * an aggregation buffer that will be used for packet data that is received. An + * aggregation buffer creates a different kind of completion operation for a + * packet where a variable number of BDs may be used to place the packet in the + * host. RX Rings that have aggregation buffers are known as aggregation rings + * and must contain only aggregation buffers. + */ +/* Short TX BD (16 bytes) */ struct tx_bd_short { + uint16_t flags_type; /* - * All bits in this field must be valid on the first BD of a packet. - * Only the packet_end bit must be valid for the remaining BDs of a - * packet. + * All bits in this field must be valid on the first BD of a + * packet. Only the packet_end bit must be valid for the + * remaining BDs of a packet. */ /* This value identifies the type of buffer descriptor. */ - #define TX_BD_SHORT_TYPE_MASK UINT32_C(0x3f) - #define TX_BD_SHORT_TYPE_SFT 0 - /* - * Indicates that this BD is 16B long and is used for normal L2 - * packet transmission. - */ - #define TX_BD_SHORT_TYPE_TX_BD_SHORT (UINT32_C(0x0) << 0) + #define TX_BD_SHORT_TYPE_MASK UINT32_C(0x3f) + #define TX_BD_SHORT_TYPE_SFT 0 /* - * If set to 1, the packet ends with the data in the buffer pointed to - * by this descriptor. This flag must be valid on every BD. + * Indicates that this BD is 16B long and is + * used for normal L2 packet transmission. */ - #define TX_BD_SHORT_FLAGS_PACKET_END UINT32_C(0x40) + #define TX_BD_SHORT_TYPE_TX_BD_SHORT UINT32_C(0x0) /* - * If set to 1, the device will not generate a completion for this - * transmit packet unless there is an error in it's processing. If this - * bit is set to 0, then the packet will be completed normally. This bit - * must be valid only on the first BD of a packet. + * If set to 1, the packet ends with the data in the buffer + * pointed to by this descriptor. This flag must be valid on + * every BD. */ - #define TX_BD_SHORT_FLAGS_NO_CMPL UINT32_C(0x80) + #define TX_BD_SHORT_FLAGS_PACKET_END UINT32_C(0x40) /* - * This value indicates how many 16B BD locations are consumed in the - * ring by this packet. A value of 1 indicates that this BD is the only - * BD (and that the it is a short BD). A value of 3 indicates either 3 - * short BDs or 1 long BD and one short BD in the packet. A value of 0 - * indicates that there are 32 BD locations in the packet (the maximum). - * This field is valid only on the first BD of a packet. + * If set to 1, the device will not generate a completion for + * this transmit packet unless there is an error in it's + * processing. If this bit is set to 0, then the packet will be + * completed normally. This bit must be valid only on the first + * BD of a packet. */ - #define TX_BD_SHORT_FLAGS_BD_CNT_MASK UINT32_C(0x1f00) - #define TX_BD_SHORT_FLAGS_BD_CNT_SFT 8 + #define TX_BD_SHORT_FLAGS_NO_CMPL UINT32_C(0x80) /* - * This value is a hint for the length of the entire packet. It is used - * by the chip to optimize internal processing. The packet will be - * dropped if the hint is too short. This field is valid only on the - * first BD of a packet. + * This value indicates how many 16B BD locations are consumed + * in the ring by this packet. A value of 1 indicates that this + * BD is the only BD (and that the it is a short BD). A value of + * 3 indicates either 3 short BDs or 1 long BD and one short BD + * in the packet. A value of 0 indicates that there are 32 BD + * locations in the packet (the maximum). This field is valid + * only on the first BD of a packet. */ - #define TX_BD_SHORT_FLAGS_LHINT_MASK UINT32_C(0x6000) - #define TX_BD_SHORT_FLAGS_LHINT_SFT 13 - /* indicates packet length < 512B */ - #define TX_BD_SHORT_FLAGS_LHINT_LT512 (UINT32_C(0x0) << 13) - /* indicates 512 <= packet length < 1KB */ - #define TX_BD_SHORT_FLAGS_LHINT_LT1K (UINT32_C(0x1) << 13) - /* indicates 1KB <= packet length < 2KB */ - #define TX_BD_SHORT_FLAGS_LHINT_LT2K (UINT32_C(0x2) << 13) - /* indicates packet length >= 2KB */ - #define TX_BD_SHORT_FLAGS_LHINT_GTE2K (UINT32_C(0x3) << 13) - #define TX_BD_SHORT_FLAGS_LHINT_LAST TX_BD_SHORT_FLAGS_LHINT_GTE2K + #define TX_BD_SHORT_FLAGS_BD_CNT_MASK UINT32_C(0x1f00) + #define TX_BD_SHORT_FLAGS_BD_CNT_SFT 8 /* - * If set to 1, the device immediately updates the Send Consumer Index - * after the buffer associated with this descriptor has been transferred - * via DMA to NIC memory from host memory. An interrupt may or may not - * be generated according to the state of the interrupt avoidance - * mechanisms. If this bit is set to 0, then the Consumer Index is only - * updated as soon as one of the host interrupt coalescing conditions - * has been met. This bit must be valid on the first BD of a packet. + * This value is a hint for the length of the entire packet. It + * is used by the chip to optimize internal processing. The + * packet will be dropped if the hint is too short. This field + * is valid only on the first BD of a packet. */ - #define TX_BD_SHORT_FLAGS_COAL_NOW UINT32_C(0x8000) + #define TX_BD_SHORT_FLAGS_LHINT_MASK UINT32_C(0x6000) + #define TX_BD_SHORT_FLAGS_LHINT_SFT 13 + /* indicates packet length < 512B */ + #define TX_BD_SHORT_FLAGS_LHINT_LT512 (UINT32_C(0x0) << 13) + /* indicates 512 <= packet length < 1KB */ + #define TX_BD_SHORT_FLAGS_LHINT_LT1K (UINT32_C(0x1) << 13) + /* indicates 1KB <= packet length < 2KB */ + #define TX_BD_SHORT_FLAGS_LHINT_LT2K (UINT32_C(0x2) << 13) + /* indicates packet length >= 2KB */ + #define TX_BD_SHORT_FLAGS_LHINT_GTE2K (UINT32_C(0x3) << 13) + #define TX_BD_SHORT_FLAGS_LHINT_LAST \ + TX_BD_SHORT_FLAGS_LHINT_GTE2K /* - * All bits in this field must be valid on the first BD of a packet. - * Only the packet_end bit must be valid for the remaining BDs of a - * packet. + * If set to 1, the device immediately updates the Send Consumer + * Index after the buffer associated with this descriptor has + * been transferred via DMA to NIC memory from host memory. An + * interrupt may or may not be generated according to the state + * of the interrupt avoidance mechanisms. If this bit is set to + * 0, then the Consumer Index is only updated as soon as one of + * the host interrupt coalescing conditions has been met. This + * bit must be valid on the first BD of a packet. */ - #define TX_BD_SHORT_FLAGS_MASK UINT32_C(0xffc0) - #define TX_BD_SHORT_FLAGS_SFT 6 - uint16_t flags_type; - + #define TX_BD_SHORT_FLAGS_COAL_NOW UINT32_C(0x8000) /* - * This is the length of the host physical buffer this BD describes in - * bytes. This field must be valid on all BDs of a packet. + * All bits in this field must be valid on the first BD of a + * packet. Only the packet_end bit must be valid for the + * remaining BDs of a packet. */ + #define TX_BD_SHORT_FLAGS_MASK UINT32_C(0xffc0) + #define TX_BD_SHORT_FLAGS_SFT 6 uint16_t len; /* - * The opaque data field is pass through to the completion and can be - * used for any data that the driver wants to associate with the - * transmit BD. This field must be valid on the first BD of a packet. - */ - uint32_t opaque; - - /* - * This is the host physical address for the portion of the packet - * described by this TX BD. This value must be valid on all BDs of a + * This is the length of the host physical buffer this BD + * describes in bytes. This field must be valid on all BDs of a * packet. */ + uint32_t opaque; + /* + * The opaque data field is pass through to the completion and + * can be used for any data that the driver wants to associate + * with the transmit BD. This field must be valid on the first + * BD of a packet. + */ uint64_t addr; + /* + * This is the host physical address for the portion of the + * packet described by this TX BD. This value must be valid on + * all BDs of a packet. + */ } __attribute__((packed)); -/* Long TX BD (32 bytes split to 2 16-byte struct) */ +/* Long TX BD (32 bytes split to 2 16-byte struct) */ struct tx_bd_long { + uint16_t flags_type; /* - * All bits in this field must be valid on the first BD of a packet. - * Only the packet_end bit must be valid for the remaining BDs of a - * packet. + * All bits in this field must be valid on the first BD of a + * packet. Only the packet_end bit must be valid for the + * remaining BDs of a packet. */ /* This value identifies the type of buffer descriptor. */ - #define TX_BD_LONG_TYPE_MASK UINT32_C(0x3f) - #define TX_BD_LONG_TYPE_SFT 0 - /* - * Indicates that this BD is 32B long and is used for normal L2 - * packet transmission. - */ - #define TX_BD_LONG_TYPE_TX_BD_LONG (UINT32_C(0x10) << 0) + #define TX_BD_LONG_TYPE_MASK UINT32_C(0x3f) + #define TX_BD_LONG_TYPE_SFT 0 /* - * If set to 1, the packet ends with the data in the buffer pointed to - * by this descriptor. This flag must be valid on every BD. + * Indicates that this BD is 32B long and is + * used for normal L2 packet transmission. */ - #define TX_BD_LONG_FLAGS_PACKET_END UINT32_C(0x40) + #define TX_BD_LONG_TYPE_TX_BD_LONG UINT32_C(0x10) /* - * If set to 1, the device will not generate a completion for this - * transmit packet unless there is an error in it's processing. If this - * bit is set to 0, then the packet will be completed normally. This bit - * must be valid only on the first BD of a packet. + * If set to 1, the packet ends with the data in the buffer + * pointed to by this descriptor. This flag must be valid on + * every BD. */ - #define TX_BD_LONG_FLAGS_NO_CMPL UINT32_C(0x80) + #define TX_BD_LONG_FLAGS_PACKET_END UINT32_C(0x40) /* - * This value indicates how many 16B BD locations are consumed in the - * ring by this packet. A value of 1 indicates that this BD is the only - * BD (and that the it is a short BD). A value of 3 indicates either 3 - * short BDs or 1 long BD and one short BD in the packet. A value of 0 - * indicates that there are 32 BD locations in the packet (the maximum). - * This field is valid only on the first BD of a packet. + * If set to 1, the device will not generate a completion for + * this transmit packet unless there is an error in it's + * processing. If this bit is set to 0, then the packet will be + * completed normally. This bit must be valid only on the first + * BD of a packet. */ - #define TX_BD_LONG_FLAGS_BD_CNT_MASK UINT32_C(0x1f00) - #define TX_BD_LONG_FLAGS_BD_CNT_SFT 8 + #define TX_BD_LONG_FLAGS_NO_CMPL UINT32_C(0x80) /* - * This value is a hint for the length of the entire packet. It is used - * by the chip to optimize internal processing. The packet will be - * dropped if the hint is too short. This field is valid only on the - * first BD of a packet. + * This value indicates how many 16B BD locations are consumed + * in the ring by this packet. A value of 1 indicates that this + * BD is the only BD (and that the it is a short BD). A value of + * 3 indicates either 3 short BDs or 1 long BD and one short BD + * in the packet. A value of 0 indicates that there are 32 BD + * locations in the packet (the maximum). This field is valid + * only on the first BD of a packet. */ - #define TX_BD_LONG_FLAGS_LHINT_MASK UINT32_C(0x6000) - #define TX_BD_LONG_FLAGS_LHINT_SFT 13 - /* indicates packet length < 512B */ - #define TX_BD_LONG_FLAGS_LHINT_LT512 (UINT32_C(0x0) << 13) - /* indicates 512 <= packet length < 1KB */ - #define TX_BD_LONG_FLAGS_LHINT_LT1K (UINT32_C(0x1) << 13) - /* indicates 1KB <= packet length < 2KB */ - #define TX_BD_LONG_FLAGS_LHINT_LT2K (UINT32_C(0x2) << 13) - /* indicates packet length >= 2KB */ - #define TX_BD_LONG_FLAGS_LHINT_GTE2K (UINT32_C(0x3) << 13) - #define TX_BD_LONG_FLAGS_LHINT_LAST TX_BD_LONG_FLAGS_LHINT_GTE2K + #define TX_BD_LONG_FLAGS_BD_CNT_MASK UINT32_C(0x1f00) + #define TX_BD_LONG_FLAGS_BD_CNT_SFT 8 /* - * If set to 1, the device immediately updates the Send Consumer Index - * after the buffer associated with this descriptor has been transferred - * via DMA to NIC memory from host memory. An interrupt may or may not - * be generated according to the state of the interrupt avoidance - * mechanisms. If this bit is set to 0, then the Consumer Index is only - * updated as soon as one of the host interrupt coalescing conditions - * has been met. This bit must be valid on the first BD of a packet. + * This value is a hint for the length of the entire packet. It + * is used by the chip to optimize internal processing. The + * packet will be dropped if the hint is too short. This field + * is valid only on the first BD of a packet. */ - #define TX_BD_LONG_FLAGS_COAL_NOW UINT32_C(0x8000) + #define TX_BD_LONG_FLAGS_LHINT_MASK UINT32_C(0x6000) + #define TX_BD_LONG_FLAGS_LHINT_SFT 13 + /* indicates packet length < 512B */ + #define TX_BD_LONG_FLAGS_LHINT_LT512 (UINT32_C(0x0) << 13) + /* indicates 512 <= packet length < 1KB */ + #define TX_BD_LONG_FLAGS_LHINT_LT1K (UINT32_C(0x1) << 13) + /* indicates 1KB <= packet length < 2KB */ + #define TX_BD_LONG_FLAGS_LHINT_LT2K (UINT32_C(0x2) << 13) + /* indicates packet length >= 2KB */ + #define TX_BD_LONG_FLAGS_LHINT_GTE2K (UINT32_C(0x3) << 13) + #define TX_BD_LONG_FLAGS_LHINT_LAST \ + TX_BD_LONG_FLAGS_LHINT_GTE2K /* - * All bits in this field must be valid on the first BD of a packet. - * Only the packet_end bit must be valid for the remaining BDs of a - * packet. + * If set to 1, the device immediately updates the Send Consumer + * Index after the buffer associated with this descriptor has + * been transferred via DMA to NIC memory from host memory. An + * interrupt may or may not be generated according to the state + * of the interrupt avoidance mechanisms. If this bit is set to + * 0, then the Consumer Index is only updated as soon as one of + * the host interrupt coalescing conditions has been met. This + * bit must be valid on the first BD of a packet. */ - #define TX_BD_LONG_FLAGS_MASK UINT32_C(0xffc0) - #define TX_BD_LONG_FLAGS_SFT 6 - uint16_t flags_type; - + #define TX_BD_LONG_FLAGS_COAL_NOW UINT32_C(0x8000) /* - * This is the length of the host physical buffer this BD describes in - * bytes. This field must be valid on all BDs of a packet. + * All bits in this field must be valid on the first BD of a + * packet. Only the packet_end bit must be valid for the + * remaining BDs of a packet. */ + #define TX_BD_LONG_FLAGS_MASK UINT32_C(0xffc0) + #define TX_BD_LONG_FLAGS_SFT 6 uint16_t len; - /* - * The opaque data field is pass through to the completion and can be - * used for any data that the driver wants to associate with the - * transmit BD. This field must be valid on the first BD of a packet. + * This is the length of the host physical buffer this BD + * describes in bytes. This field must be valid on all BDs of a + * packet. */ uint32_t opaque; - /* - * This is the host physical address for the portion of the packet - * described by this TX BD. This value must be valid on all BDs of a - * packet. + * The opaque data field is pass through to the completion and + * can be used for any data that the driver wants to associate + * with the transmit BD. This field must be valid on the first + * BD of a packet. */ uint64_t addr; + /* + * This is the host physical address for the portion of the + * packet described by this TX BD. This value must be valid on + * all BDs of a packet. + */ } __attribute__((packed)); /* last 16 bytes of Long TX BD */ - struct tx_bd_long_hi { + uint16_t lflags; /* - * All bits in this field must be valid on the first BD of a packet. - * Their value on other BDs of the packet will be ignored. + * All bits in this field must be valid on the first BD of a + * packet. Their value on other BDs of the packet will be + * ignored. */ /* - * If set to 1, the controller replaces the TCP/UPD checksum fields of - * normal TCP/UPD checksum, or the inner TCP/UDP checksum field of the - * encapsulated TCP/UDP packets with the hardware calculated TCP/UDP - * checksum for the packet associated with this descriptor. This bit - * must be valid on the first BD of a packet. + * If set to 1, the controller replaces the TCP/UPD checksum + * fields of normal TCP/UPD checksum, or the inner TCP/UDP + * checksum field of the encapsulated TCP/UDP packets with the + * hardware calculated TCP/UDP checksum for the packet + * associated with this descriptor. The flag is ignored if the + * LSO flag is set. This bit must be valid on the first BD of a + * packet. */ #define TX_BD_LONG_LFLAGS_TCP_UDP_CHKSUM UINT32_C(0x1) /* - * If set to 1, the controller replaces the IP checksum of the normal - * packets, or the inner IP checksum of the encapsulated packets with - * the hardware calculated IP checksum for the packet associated with - * this descriptor. This bit must be valid on the first BD of a packet. + * If set to 1, the controller replaces the IP checksum of the + * normal packets, or the inner IP checksum of the encapsulated + * packets with the hardware calculated IP checksum for the + * packet associated with this descriptor. This bit must be + * valid on the first BD of a packet. */ - #define TX_BD_LONG_LFLAGS_IP_CHKSUM UINT32_C(0x2) + #define TX_BD_LONG_LFLAGS_IP_CHKSUM UINT32_C(0x2) /* - * If set to 1, the controller will not append an Ethernet CRC to the - * end of the frame. This bit must be valid on the first BD of a packet. - * Packet must be 64B or longer when this flag is set. It is not useful - * to use this bit with any form of TX offload such as CSO or LSO. The - * intent is that the packet from the host already has a valid Ethernet - * CRC on the packet. + * If set to 1, the controller will not append an Ethernet CRC + * to the end of the frame. This bit must be valid on the first + * BD of a packet. Packet must be 64B or longer when this flag + * is set. It is not useful to use this bit with any form of TX + * offload such as CSO or LSO. The intent is that the packet + * from the host already has a valid Ethernet CRC on the packet. */ - #define TX_BD_LONG_LFLAGS_NOCRC UINT32_C(0x4) + #define TX_BD_LONG_LFLAGS_NOCRC UINT32_C(0x4) /* - * If set to 1, the device will record the time at which the packet was - * actually transmitted at the TX MAC. This bit must be valid on the - * first BD of a packet. + * If set to 1, the device will record the time at which the + * packet was actually transmitted at the TX MAC. This bit must + * be valid on the first BD of a packet. */ - #define TX_BD_LONG_LFLAGS_STAMP UINT32_C(0x8) + #define TX_BD_LONG_LFLAGS_STAMP UINT32_C(0x8) /* - * If set to 1, The controller replaces the tunnel IP checksum field - * with hardware calculated IP checksum for the IP header of the packet - * associated with this descriptor. In case of VXLAN, the controller - * also replaces the outer header UDP checksum with hardware calculated - * UDP checksum for the packet associated with this descriptor. + * If set to 1, The controller replaces the tunnel IP checksum + * field with hardware calculated IP checksum for the IP header + * of the packet associated with this descriptor. For outer UDP + * checksum, global outer UDP checksum TE_NIC register needs to + * be enabled. If the global outer UDP checksum TE_NIC register + * bit is set, outer UDP checksum will be calculated for the + * following cases: 1. Packets with tcp_udp_chksum flag set to + * offload checksum for inner packet AND the inner packet is + * TCP/UDP. If the inner packet is ICMP for example (non- + * TCP/UDP), even if the tcp_udp_chksum is set, the outer UDP + * checksum will not be calculated. 2. Packets with lso flag set + * which implies inner TCP checksum calculation as part of LSO + * operation. */ - #define TX_BD_LONG_LFLAGS_T_IP_CHKSUM UINT32_C(0x10) + #define TX_BD_LONG_LFLAGS_T_IP_CHKSUM UINT32_C(0x10) /* - * If set to 1, the device will treat this packet with LSO(Large Send - * Offload) processing for both normal or encapsulated packets, which is - * a form of TCP segmentation. When this bit is 1, the hdr_size and mss - * fields must be valid. The driver doesn't need to set t_ip_chksum, - * ip_chksum, and tcp_udp_chksum flags since the controller will replace - * the appropriate checksum fields for segmented packets. When this bit - * is 1, the hdr_size and mss fields must be valid. + * If set to 1, the device will treat this packet with LSO(Large + * Send Offload) processing for both normal or encapsulated + * packets, which is a form of TCP segmentation. When this bit + * is 1, the hdr_size and mss fields must be valid. The driver + * doesn't need to set t_ip_chksum, ip_chksum, and + * tcp_udp_chksum flags since the controller will replace the + * appropriate checksum fields for segmented packets. When this + * bit is 1, the hdr_size and mss fields must be valid. */ - #define TX_BD_LONG_LFLAGS_LSO UINT32_C(0x20) + #define TX_BD_LONG_LFLAGS_LSO UINT32_C(0x20) /* - * If set to zero when LSO is '1', then the IPID will be treated as a - * 16b number and will be wrapped if it exceeds a value of 0xffff. If - * set to one when LSO is '1', then the IPID will be treated as a 15b - * number and will be wrapped if it exceeds a value 0f 0x7fff. + * If set to zero when LSO is '1', then the IPID will be treated + * as a 16b number and will be wrapped if it exceeds a value of + * 0xffff. If set to one when LSO is '1', then the IPID will be + * treated as a 15b number and will be wrapped if it exceeds a + * value 0f 0x7fff. */ - #define TX_BD_LONG_LFLAGS_IPID_FMT UINT32_C(0x40) + #define TX_BD_LONG_LFLAGS_IPID_FMT UINT32_C(0x40) /* - * If set to zero when LSO is '1', then the IPID of the tunnel IP header - * will not be modified during LSO operations. If set to one when LSO is - * '1', then the IPID of the tunnel IP header will be incremented for - * each subsequent segment of an LSO operation. + * If set to zero when LSO is '1', then the IPID of the tunnel + * IP header will not be modified during LSO operations. If set + * to one when LSO is '1', then the IPID of the tunnel IP header + * will be incremented for each subsequent segment of an LSO + * operation. The flag is ignored if the LSO packet is a normal + * (non-tunneled) TCP packet. */ - #define TX_BD_LONG_LFLAGS_T_IPID UINT32_C(0x80) + #define TX_BD_LONG_LFLAGS_T_IPID UINT32_C(0x80) /* - * If set to '1', then the RoCE ICRC will be appended to the packet. - * Packet must be a valid RoCE format packet. + * If set to '1', then the RoCE ICRC will be appended to the + * packet. Packet must be a valid RoCE format packet. */ - #define TX_BD_LONG_LFLAGS_ROCE_CRC UINT32_C(0x100) + #define TX_BD_LONG_LFLAGS_ROCE_CRC UINT32_C(0x100) /* - * If set to '1', then the FCoE CRC will be appended to the packet. - * Packet must be a valid FCoE format packet. + * If set to '1', then the FCoE CRC will be appended to the + * packet. Packet must be a valid FCoE format packet. */ - #define TX_BD_LONG_LFLAGS_FCOE_CRC UINT32_C(0x200) - uint16_t lflags; - - /* - * When LSO is '1', this field must contain the offset of the TCP - * payload from the beginning of the packet in as 16b words. In case of - * encapsulated/tunneling packet, this field contains the offset of the - * inner TCP payload from beginning of the packet as 16-bit words. This - * value must be valid on the first BD of a packet. - */ - #define TX_BD_LONG_HDR_SIZE_MASK UINT32_C(0x1ff) - #define TX_BD_LONG_HDR_SIZE_SFT 0 + #define TX_BD_LONG_LFLAGS_FCOE_CRC UINT32_C(0x200) uint16_t hdr_size; - /* - * This is the MSS value that will be used to do the LSO processing. The - * value is the length in bytes of the TCP payload for each segment - * generated by the LSO operation. This value must be valid on the first + * When LSO is '1', this field must contain the offset of the + * TCP payload from the beginning of the packet in as 16b words. + * In case of encapsulated/tunneling packet, this field contains + * the offset of the inner TCP payload from beginning of the + * packet as 16-bit words. This value must be valid on the first * BD of a packet. */ - #define TX_BD_LONG_MSS_MASK UINT32_C(0x7fff) - #define TX_BD_LONG_MSS_SFT 0 + #define TX_BD_LONG_HDR_SIZE_MASK UINT32_C(0x1ff) + #define TX_BD_LONG_HDR_SIZE_SFT 0 uint32_t mss; - - uint16_t unused_2; - /* - * This value selects a CFA action to perform on the packet. Set this - * value to zero if no CFA action is desired. This value must be valid - * on the first BD of a packet. + * This is the MSS value that will be used to do the LSO + * processing. The value is the length in bytes of the TCP + * payload for each segment generated by the LSO operation. This + * value must be valid on the first BD of a packet. */ + #define TX_BD_LONG_MSS_MASK UINT32_C(0x7fff) + #define TX_BD_LONG_MSS_SFT 0 + uint16_t unused_2; uint16_t cfa_action; - /* - * This value is action meta-data that defines CFA edit operations that - * are done in addition to any action editing. + * This value selects a CFA action to perform on the packet. Set + * this value to zero if no CFA action is desired. This value + * must be valid on the first BD of a packet. + */ + uint32_t cfa_meta; + /* + * This value is action meta-data that defines CFA edit + * operations that are done in addition to any action editing. */ /* When key=1, This is the VLAN tag VID value. */ #define TX_BD_LONG_CFA_META_VLAN_VID_MASK UINT32_C(0xfff) #define TX_BD_LONG_CFA_META_VLAN_VID_SFT 0 /* When key=1, This is the VLAN tag DE value. */ - #define TX_BD_LONG_CFA_META_VLAN_DE UINT32_C(0x1000) + #define TX_BD_LONG_CFA_META_VLAN_DE UINT32_C(0x1000) /* When key=1, This is the VLAN tag PRI value. */ #define TX_BD_LONG_CFA_META_VLAN_PRI_MASK UINT32_C(0xe000) #define TX_BD_LONG_CFA_META_VLAN_PRI_SFT 13 /* When key=1, This is the VLAN tag TPID select value. */ #define TX_BD_LONG_CFA_META_VLAN_TPID_MASK UINT32_C(0x70000) #define TX_BD_LONG_CFA_META_VLAN_TPID_SFT 16 - /* 0x88a8 */ + /* 0x88a8 */ #define TX_BD_LONG_CFA_META_VLAN_TPID_TPID88A8 (UINT32_C(0x0) << 16) - /* 0x8100 */ + /* 0x8100 */ #define TX_BD_LONG_CFA_META_VLAN_TPID_TPID8100 (UINT32_C(0x1) << 16) - /* 0x9100 */ + /* 0x9100 */ #define TX_BD_LONG_CFA_META_VLAN_TPID_TPID9100 (UINT32_C(0x2) << 16) - /* 0x9200 */ + /* 0x9200 */ #define TX_BD_LONG_CFA_META_VLAN_TPID_TPID9200 (UINT32_C(0x3) << 16) - /* 0x9300 */ + /* 0x9300 */ #define TX_BD_LONG_CFA_META_VLAN_TPID_TPID9300 (UINT32_C(0x4) << 16) - /* Value programmed in CFA VLANTPID register. */ + /* Value programmed in CFA VLANTPID register. */ #define TX_BD_LONG_CFA_META_VLAN_TPID_TPIDCFG (UINT32_C(0x5) << 16) #define TX_BD_LONG_CFA_META_VLAN_TPID_LAST \ - TX_BD_LONG_CFA_META_VLAN_TPID_TPIDCFG + TX_BD_LONG_CFA_META_VLAN_TPID_TPIDCFG /* When key=1, This is the VLAN tag TPID select value. */ #define TX_BD_LONG_CFA_META_VLAN_RESERVED_MASK UINT32_C(0xff80000) #define TX_BD_LONG_CFA_META_VLAN_RESERVED_SFT 19 /* - * This field identifies the type of edit to be performed on the packet. - * This value must be valid on the first BD of a packet. + * This field identifies the type of edit to be performed on the + * packet. This value must be valid on the first BD of a packet. + */ + #define TX_BD_LONG_CFA_META_KEY_MASK UINT32_C(0xf0000000) + #define TX_BD_LONG_CFA_META_KEY_SFT 28 + /* No editing */ + #define TX_BD_LONG_CFA_META_KEY_NONE (UINT32_C(0x0) << 28) + /* + * - meta[17:16] - TPID select value (0 = + * 0x8100). - meta[15:12] - PRI/DE value. - + * meta[11:0] - VID value. */ - #define TX_BD_LONG_CFA_META_KEY_MASK UINT32_C(0xf0000000) - #define TX_BD_LONG_CFA_META_KEY_SFT 28 - /* No editing */ - #define TX_BD_LONG_CFA_META_KEY_NONE (UINT32_C(0x0) << 28) - /* - * - meta[17:16] - TPID select value (0 = 0x8100). - meta[15:12] - * - PRI/DE value. - meta[11:0] - VID value. - */ #define TX_BD_LONG_CFA_META_KEY_VLAN_TAG (UINT32_C(0x1) << 28) - #define TX_BD_LONG_CFA_META_KEY_LAST TX_BD_LONG_CFA_META_KEY_VLAN_TAG - uint32_t cfa_meta; + #define TX_BD_LONG_CFA_META_KEY_LAST \ + TX_BD_LONG_CFA_META_KEY_VLAN_TAG } __attribute__((packed)); -/* RX Producer Packet BD (16 bytes) */ +/* RX Producer Packet BD (16 bytes) */ struct rx_prod_pkt_bd { + uint16_t flags_type; /* This value identifies the type of buffer descriptor. */ - #define RX_PROD_PKT_BD_TYPE_MASK UINT32_C(0x3f) - #define RX_PROD_PKT_BD_TYPE_SFT 0 - /* - * Indicates that this BD is 16B long and is an RX Producer (ie. - * empty) buffer descriptor. - */ - #define RX_PROD_PKT_BD_TYPE_RX_PROD_PKT (UINT32_C(0x4) << 0) + #define RX_PROD_PKT_BD_TYPE_MASK UINT32_C(0x3f) + #define RX_PROD_PKT_BD_TYPE_SFT 0 /* - * If set to 1, the packet will be placed at the address plus 2B. The 2 - * Bytes of padding will be written as zero. + * Indicates that this BD is 16B long and is an + * RX Producer (ie. empty) buffer descriptor. + */ + #define RX_PROD_PKT_BD_TYPE_RX_PROD_PKT UINT32_C(0x4) + /* + * If set to 1, the packet will be placed at the address plus + * 2B. The 2 Bytes of padding will be written as zero. */ /* - * This is intended to be used when the host buffer is cache-line - * aligned to produce packets that are easy to parse in host memory - * while still allowing writes to be cache line aligned. + * This is intended to be used when the host buffer is cache- + * line aligned to produce packets that are easy to parse in + * host memory while still allowing writes to be cache line + * aligned. */ - #define RX_PROD_PKT_BD_FLAGS_SOP_PAD UINT32_C(0x40) + #define RX_PROD_PKT_BD_FLAGS_SOP_PAD UINT32_C(0x40) /* - * If set to 1, the packet write will be padded out to the nearest - * cache-line with zero value padding. + * If set to 1, the packet write will be padded out to the + * nearest cache-line with zero value padding. */ /* - * If receive buffers start/end on cache-line boundaries, this feature - * will ensure that all data writes on the PCI bus start/end on cache - * line boundaries. + * If receive buffers start/end on cache-line boundaries, this + * feature will ensure that all data writes on the PCI bus + * start/end on cache line boundaries. */ - #define RX_PROD_PKT_BD_FLAGS_EOP_PAD UINT32_C(0x80) + #define RX_PROD_PKT_BD_FLAGS_EOP_PAD UINT32_C(0x80) /* - * This value is the number of additional buffers in the ring that - * describe the buffer space to be consumed for the this packet. If the - * value is zero, then the packet must fit within the space described by - * this BD. If this value is 1 or more, it indicates how many additional - * "buffer" BDs are in the ring immediately following this BD to be used - * for the same network packet. Even if the packet to be placed does not - * need all the additional buffers, they will be consumed anyway. + * This value is the number of additional buffers in the ring + * that describe the buffer space to be consumed for the this + * packet. If the value is zero, then the packet must fit within + * the space described by this BD. If this value is 1 or more, + * it indicates how many additional "buffer" BDs are in the ring + * immediately following this BD to be used for the same network + * packet. Even if the packet to be placed does not need all the + * additional buffers, they will be consumed anyway. */ #define RX_PROD_PKT_BD_FLAGS_BUFFERS_MASK UINT32_C(0x300) #define RX_PROD_PKT_BD_FLAGS_BUFFERS_SFT 8 - #define RX_PROD_PKT_BD_FLAGS_MASK UINT32_C(0xffc0) - #define RX_PROD_PKT_BD_FLAGS_SFT 6 - uint16_t flags_type; - - /* - * This is the length in Bytes of the host physical buffer where data - * for the packet may be placed in host memory. - */ - /* - * While this is a Byte resolution value, it is often advantageous to - * ensure that the buffers provided end on a host cache line. - */ + #define RX_PROD_PKT_BD_FLAGS_MASK UINT32_C(0xffc0) + #define RX_PROD_PKT_BD_FLAGS_SFT 6 uint16_t len; - /* - * The opaque data field is pass through to the completion and can be - * used for any data that the driver wants to associate with this - * receive buffer set. + * This is the length in Bytes of the host physical buffer where + * data for the packet may be placed in host memory. + */ + /* + * While this is a Byte resolution value, it is often + * advantageous to ensure that the buffers provided end on a + * host cache line. */ uint32_t opaque; - /* - * This is the host physical address where data for the packet may by - * placed in host memory. - */ - /* - * While this is a Byte resolution value, it is often advantageous to - * ensure that the buffers provide start on a host cache line. + * The opaque data field is pass through to the completion and + * can be used for any data that the driver wants to associate + * with this receive buffer set. */ uint64_t addr; + /* + * This is the host physical address where data for the packet + * may by placed in host memory. + */ + /* + * While this is a Byte resolution value, it is often + * advantageous to ensure that the buffers provide start on a + * host cache line. + */ } __attribute__((packed)); /* Completion Ring Structures */ /* Note: This structure is used by the HWRM to communicate HWRM Error. */ -/* Base Completion Record (16 bytes) */ +/* Base Completion Record (16 bytes) */ struct cmpl_base { + uint16_t type; /* unused is 10 b */ /* - * This field indicates the exact type of the completion. By convention, - * the LSB identifies the length of the record in 16B units. Even values - * indicate 16B records. Odd values indicate 32B records. + * This field indicates the exact type of the completion. By + * convention, the LSB identifies the length of the record in + * 16B units. Even values indicate 16B records. Odd values + * indicate 32B records. */ - #define CMPL_BASE_TYPE_MASK UINT32_C(0x3f) - #define CMPL_BASE_TYPE_SFT 0 - /* TX L2 completion: Completion of TX packet. Length = 16B */ - #define CMPL_BASE_TYPE_TX_L2 (UINT32_C(0x0) << 0) - /* - * RX L2 completion: Completion of and L2 RX packet. - * Length = 32B - */ - #define CMPL_BASE_TYPE_RX_L2 (UINT32_C(0x11) << 0) - /* - * RX Aggregation Buffer completion : Completion of an L2 - * aggregation buffer in support of TPA, HDS, or Jumbo packet - * completion. Length = 16B - */ - #define CMPL_BASE_TYPE_RX_AGG (UINT32_C(0x12) << 0) - /* - * RX L2 TPA Start Completion: Completion at the beginning of a - * TPA operation. Length = 32B - */ - #define CMPL_BASE_TYPE_RX_TPA_START (UINT32_C(0x13) << 0) - /* - * RX L2 TPA End Completion: Completion at the end of a TPA - * operation. Length = 32B - */ - #define CMPL_BASE_TYPE_RX_TPA_END (UINT32_C(0x15) << 0) - /* - * Statistics Ejection Completion: Completion of statistics data - * ejection buffer. Length = 16B - */ - #define CMPL_BASE_TYPE_STAT_EJECT (UINT32_C(0x1a) << 0) - /* HWRM Command Completion: Completion of an HWRM command. */ - #define CMPL_BASE_TYPE_HWRM_DONE (UINT32_C(0x20) << 0) - /* Forwarded HWRM Request */ - #define CMPL_BASE_TYPE_HWRM_FWD_REQ (UINT32_C(0x22) << 0) - /* Forwarded HWRM Response */ - #define CMPL_BASE_TYPE_HWRM_FWD_RESP (UINT32_C(0x24) << 0) - /* HWRM Asynchronous Event Information */ - #define CMPL_BASE_TYPE_HWRM_ASYNC_EVENT (UINT32_C(0x2e) << 0) - /* CQ Notification */ - #define CMPL_BASE_TYPE_CQ_NOTIFICATION (UINT32_C(0x30) << 0) - /* SRQ Threshold Event */ - #define CMPL_BASE_TYPE_SRQ_EVENT (UINT32_C(0x32) << 0) - /* DBQ Threshold Event */ - #define CMPL_BASE_TYPE_DBQ_EVENT (UINT32_C(0x34) << 0) - /* QP Async Notification */ - #define CMPL_BASE_TYPE_QP_EVENT (UINT32_C(0x38) << 0) - /* Function Async Notification */ - #define CMPL_BASE_TYPE_FUNC_EVENT (UINT32_C(0x3a) << 0) - uint16_t type; - - uint16_t info1; - uint32_t info2; - + #define CMPL_BASE_TYPE_MASK UINT32_C(0x3f) + #define CMPL_BASE_TYPE_SFT 0 + /* TX L2 completion: Completion of TX packet. Length = 16B */ + #define CMPL_BASE_TYPE_TX_L2 UINT32_C(0x0) /* - * This value is written by the NIC such that it will be different for - * each pass through the completion queue. The even passes will write 1. - * The odd passes will write 0. + * RX L2 completion: Completion of and L2 RX + * packet. Length = 32B */ - #define CMPL_BASE_V UINT32_C(0x1) - /* info3 is 31 b */ - #define CMPL_BASE_INFO3_MASK UINT32_C(0xfffffffe) - #define CMPL_BASE_INFO3_SFT 1 + #define CMPL_BASE_TYPE_RX_L2 UINT32_C(0x11) + /* + * RX Aggregation Buffer completion : Completion + * of an L2 aggregation buffer in support of + * TPA, HDS, or Jumbo packet completion. Length + * = 16B + */ + #define CMPL_BASE_TYPE_RX_AGG UINT32_C(0x12) + /* + * RX L2 TPA Start Completion: Completion at the + * beginning of a TPA operation. Length = 32B + */ + #define CMPL_BASE_TYPE_RX_TPA_START UINT32_C(0x13) + /* + * RX L2 TPA End Completion: Completion at the + * end of a TPA operation. Length = 32B + */ + #define CMPL_BASE_TYPE_RX_TPA_END UINT32_C(0x15) + /* + * Statistics Ejection Completion: Completion of + * statistics data ejection buffer. Length = 16B + */ + #define CMPL_BASE_TYPE_STAT_EJECT UINT32_C(0x1a) + /* HWRM Command Completion: Completion of an HWRM command. */ + #define CMPL_BASE_TYPE_HWRM_DONE UINT32_C(0x20) + /* Forwarded HWRM Request */ + #define CMPL_BASE_TYPE_HWRM_FWD_REQ UINT32_C(0x22) + /* Forwarded HWRM Response */ + #define CMPL_BASE_TYPE_HWRM_FWD_RESP UINT32_C(0x24) + /* HWRM Asynchronous Event Information */ + #define CMPL_BASE_TYPE_HWRM_ASYNC_EVENT UINT32_C(0x2e) + /* CQ Notification */ + #define CMPL_BASE_TYPE_CQ_NOTIFICATION UINT32_C(0x30) + /* SRQ Threshold Event */ + #define CMPL_BASE_TYPE_SRQ_EVENT UINT32_C(0x32) + /* DBQ Threshold Event */ + #define CMPL_BASE_TYPE_DBQ_EVENT UINT32_C(0x34) + /* QP Async Notification */ + #define CMPL_BASE_TYPE_QP_EVENT UINT32_C(0x38) + /* Function Async Notification */ + #define CMPL_BASE_TYPE_FUNC_EVENT UINT32_C(0x3a) + /* unused is 10 b */ + uint16_t info1; + /* info1 is 16 b */ + uint32_t info2; + /* info2 is 32 b */ uint32_t info3_v; - + /* info3 is 31 b */ + /* + * This value is written by the NIC such that it will be + * different for each pass through the completion queue. The + * even passes will write 1. The odd passes will write 0. + */ + #define CMPL_BASE_V UINT32_C(0x1) + /* info3 is 31 b */ + #define CMPL_BASE_INFO3_MASK UINT32_C(0xfffffffe) + #define CMPL_BASE_INFO3_SFT 1 uint32_t info4; + /* info4 is 32 b */ } __attribute__((packed)); -/* TX Completion Record (16 bytes) */ +/* TX Completion Record (16 bytes) */ struct tx_cmpl { - /* - * This field indicates the exact type of the completion. By convention, - * the LSB identifies the length of the record in 16B units. Even values - * indicate 16B records. Odd values indicate 32B records. - */ - #define TX_CMPL_TYPE_MASK UINT32_C(0x3f) - #define TX_CMPL_TYPE_SFT 0 - /* TX L2 completion: Completion of TX packet. Length = 16B */ - #define TX_CMPL_TYPE_TX_L2 (UINT32_C(0x0) << 0) - /* - * When this bit is '1', it indicates a packet that has an error of some - * type. Type of error is indicated in error_flags. - */ - #define TX_CMPL_FLAGS_ERROR UINT32_C(0x40) - /* - * When this bit is '1', it indicates that the packet completed was - * transmitted using the push acceleration data provided by the driver. - * When this bit is '0', it indicates that the packet had not push - * acceleration data written or was executed as a normal packet even - * though push data was provided. - */ - #define TX_CMPL_FLAGS_PUSH UINT32_C(0x80) - #define TX_CMPL_FLAGS_MASK UINT32_C(0xffc0) - #define TX_CMPL_FLAGS_SFT 6 uint16_t flags_type; - + /* + * This field indicates the exact type of the completion. By + * convention, the LSB identifies the length of the record in + * 16B units. Even values indicate 16B records. Odd values + * indicate 32B records. + */ + #define TX_CMPL_TYPE_MASK UINT32_C(0x3f) + #define TX_CMPL_TYPE_SFT 0 + /* TX L2 completion: Completion of TX packet. Length = 16B */ + #define TX_CMPL_TYPE_TX_L2 UINT32_C(0x0) + /* + * When this bit is '1', it indicates a packet that has an error + * of some type. Type of error is indicated in error_flags. + */ + #define TX_CMPL_FLAGS_ERROR UINT32_C(0x40) + /* + * When this bit is '1', it indicates that the packet completed + * was transmitted using the push acceleration data provided by + * the driver. When this bit is '0', it indicates that the + * packet had not push acceleration data written or was executed + * as a normal packet even though push data was provided. + */ + #define TX_CMPL_FLAGS_PUSH UINT32_C(0x80) + #define TX_CMPL_FLAGS_MASK UINT32_C(0xffc0) + #define TX_CMPL_FLAGS_SFT 6 uint16_t unused_0; - - /* - * This is a copy of the opaque field from the first TX BD of this - * transmitted packet. - */ + /* unused1 is 16 b */ uint32_t opaque; - /* - * This value is written by the NIC such that it will be different for - * each pass through the completion queue. The even passes will write 1. - * The odd passes will write 0. + * This is a copy of the opaque field from the first TX BD of + * this transmitted packet. */ - #define TX_CMPL_V UINT32_C(0x1) + uint16_t errors_v; /* - * This error indicates that there was some sort of problem with the BDs - * for the packet. + * This value is written by the NIC such that it will be + * different for each pass through the completion queue. The + * even passes will write 1. The odd passes will write 0. + */ + #define TX_CMPL_V UINT32_C(0x1) + /* + * This error indicates that there was some sort of problem with + * the BDs for the packet. */ #define TX_CMPL_ERRORS_BUFFER_ERROR_MASK UINT32_C(0xe) - #define TX_CMPL_ERRORS_BUFFER_ERROR_SFT 1 - /* No error */ + #define TX_CMPL_ERRORS_BUFFER_ERROR_SFT 1 + /* No error */ #define TX_CMPL_ERRORS_BUFFER_ERROR_NO_ERROR (UINT32_C(0x0) << 1) - /* Bad Format: BDs were not formatted correctly. */ + /* Bad Format: BDs were not formatted correctly. */ #define TX_CMPL_ERRORS_BUFFER_ERROR_BAD_FMT (UINT32_C(0x2) << 1) #define TX_CMPL_ERRORS_BUFFER_ERROR_LAST \ - TX_CMPL_ERRORS_BUFFER_ERROR_BAD_FMT + TX_CMPL_ERRORS_BUFFER_ERROR_BAD_FMT /* - * When this bit is '1', it indicates that the length of the packet was - * zero. No packet was transmitted. + * When this bit is '1', it indicates that the length of the + * packet was zero. No packet was transmitted. */ - #define TX_CMPL_ERRORS_ZERO_LENGTH_PKT UINT32_C(0x10) + #define TX_CMPL_ERRORS_ZERO_LENGTH_PKT UINT32_C(0x10) /* - * When this bit is '1', it indicates that the packet was longer than - * the programmed limit in TDI. No packet was transmitted. + * When this bit is '1', it indicates that the packet was longer + * than the programmed limit in TDI. No packet was transmitted. */ #define TX_CMPL_ERRORS_EXCESSIVE_BD_LENGTH UINT32_C(0x20) /* - * When this bit is '1', it indicates that one or more of the BDs - * associated with this packet generated a PCI error. This probably - * means the address was not valid. + * When this bit is '1', it indicates that one or more of the + * BDs associated with this packet generated a PCI error. This + * probably means the address was not valid. */ - #define TX_CMPL_ERRORS_DMA_ERROR UINT32_C(0x40) + #define TX_CMPL_ERRORS_DMA_ERROR UINT32_C(0x40) /* - * When this bit is '1', it indicates that the packet was longer than - * indicated by the hint. No packet was transmitted. + * When this bit is '1', it indicates that the packet was longer + * than indicated by the hint. No packet was transmitted. */ - #define TX_CMPL_ERRORS_HINT_TOO_SHORT UINT32_C(0x80) + #define TX_CMPL_ERRORS_HINT_TOO_SHORT UINT32_C(0x80) /* - * When this bit is '1', it indicates that the packet was dropped due to - * Poison TLP error on one or more of the TLPs in the PXP completion. + * When this bit is '1', it indicates that the packet was + * dropped due to Poison TLP error on one or more of the TLPs in + * the PXP completion. */ - #define TX_CMPL_ERRORS_POISON_TLP_ERROR UINT32_C(0x100) - #define TX_CMPL_ERRORS_MASK UINT32_C(0xfffe) - #define TX_CMPL_ERRORS_SFT 1 - uint16_t errors_v; - + #define TX_CMPL_ERRORS_POISON_TLP_ERROR UINT32_C(0x100) + #define TX_CMPL_ERRORS_MASK UINT32_C(0xfffe) + #define TX_CMPL_ERRORS_SFT 1 uint16_t unused_1; + /* unused2 is 16 b */ uint32_t unused_2; -} __attribute__((packed)) tx_cmpl_t, *ptx_cmpl_t; + /* unused3 is 32 b */ +} __attribute__((packed)); -/* RX Packet Completion Record (32 bytes split to 2 16-byte struct) */ +/* RX Packet Completion Record (32 bytes split to 2 16-byte struct) */ struct rx_pkt_cmpl { + uint16_t flags_type; /* - * This field indicates the exact type of the completion. By convention, - * the LSB identifies the length of the record in 16B units. Even values - * indicate 16B records. Odd values indicate 32B records. + * This field indicates the exact type of the completion. By + * convention, the LSB identifies the length of the record in + * 16B units. Even values indicate 16B records. Odd values + * indicate 32B records. */ - #define RX_PKT_CMPL_TYPE_MASK UINT32_C(0x3f) - #define RX_PKT_CMPL_TYPE_SFT 0 - /* - * RX L2 completion: Completion of and L2 RX packet. - * Length = 32B - */ - #define RX_PKT_CMPL_TYPE_RX_L2 (UINT32_C(0x11) << 0) + #define RX_PKT_CMPL_TYPE_MASK UINT32_C(0x3f) + #define RX_PKT_CMPL_TYPE_SFT 0 /* - * When this bit is '1', it indicates a packet that has an error of some - * type. Type of error is indicated in error_flags. + * RX L2 completion: Completion of and L2 RX + * packet. Length = 32B */ - #define RX_PKT_CMPL_FLAGS_ERROR UINT32_C(0x40) + #define RX_PKT_CMPL_TYPE_RX_L2 UINT32_C(0x11) + /* + * When this bit is '1', it indicates a packet that has an error + * of some type. Type of error is indicated in error_flags. + */ + #define RX_PKT_CMPL_FLAGS_ERROR UINT32_C(0x40) /* This field indicates how the packet was placed in the buffer. */ #define RX_PKT_CMPL_FLAGS_PLACEMENT_MASK UINT32_C(0x380) - #define RX_PKT_CMPL_FLAGS_PLACEMENT_SFT 7 - /* Normal: Packet was placed using normal algorithm. */ + #define RX_PKT_CMPL_FLAGS_PLACEMENT_SFT 7 + /* Normal: Packet was placed using normal algorithm. */ #define RX_PKT_CMPL_FLAGS_PLACEMENT_NORMAL (UINT32_C(0x0) << 7) - /* Jumbo: Packet was placed using jumbo algorithm. */ + /* Jumbo: Packet was placed using jumbo algorithm. */ #define RX_PKT_CMPL_FLAGS_PLACEMENT_JUMBO (UINT32_C(0x1) << 7) - /* - * Header/Data Separation: Packet was placed using Header/Data - * separation algorithm. The separation location is indicated by - * the itype field. - */ - #define RX_PKT_CMPL_FLAGS_PLACEMENT_HDS (UINT32_C(0x2) << 7) - #define RX_PKT_CMPL_FLAGS_PLACEMENT_LAST \ - RX_PKT_CMPL_FLAGS_PLACEMENT_HDS - /* This bit is '1' if the RSS field in this completion is valid. */ - #define RX_PKT_CMPL_FLAGS_RSS_VALID UINT32_C(0x400) /* - * This value indicates what the inner packet determined for the packet - * was. + * Header/Data Separation: Packet was placed + * using Header/Data separation algorithm. The + * separation location is indicated by the itype + * field. */ - #define RX_PKT_CMPL_FLAGS_ITYPE_MASK UINT32_C(0xf000) - #define RX_PKT_CMPL_FLAGS_ITYPE_SFT 12 - /* Not Known: Indicates that the packet type was not known. */ + #define RX_PKT_CMPL_FLAGS_PLACEMENT_HDS (UINT32_C(0x2) << 7) + #define RX_PKT_CMPL_FLAGS_PLACEMENT_LAST \ + RX_PKT_CMPL_FLAGS_PLACEMENT_HDS + /* This bit is '1' if the RSS field in this completion is valid. */ + #define RX_PKT_CMPL_FLAGS_RSS_VALID UINT32_C(0x400) + /* unused is 1 b */ + #define RX_PKT_CMPL_FLAGS_UNUSED UINT32_C(0x800) + /* + * This value indicates what the inner packet determined for the + * packet was. + */ + #define RX_PKT_CMPL_FLAGS_ITYPE_MASK UINT32_C(0xf000) + #define RX_PKT_CMPL_FLAGS_ITYPE_SFT 12 + /* Not Known: Indicates that the packet type was not known. */ #define RX_PKT_CMPL_FLAGS_ITYPE_NOT_KNOWN (UINT32_C(0x0) << 12) - /* - * IP Packet: Indicates that the packet was an IP packet, but - * further classification was not possible. - */ - #define RX_PKT_CMPL_FLAGS_ITYPE_IP (UINT32_C(0x1) << 12) - /* - * TCP Packet: Indicates that the packet was IP and TCP. This - * indicates that the payload_offset field is valid. - */ - #define RX_PKT_CMPL_FLAGS_ITYPE_TCP (UINT32_C(0x2) << 12) - /* - * UDP Packet: Indicates that the packet was IP and UDP. This - * indicates that the payload_offset field is valid. - */ - #define RX_PKT_CMPL_FLAGS_ITYPE_UDP (UINT32_C(0x3) << 12) - /* - * FCoE Packet: Indicates that the packet was recognized as a - * FCoE. This also indicates that the payload_offset field is - * valid. - */ - #define RX_PKT_CMPL_FLAGS_ITYPE_FCOE (UINT32_C(0x4) << 12) - /* - * RoCE Packet: Indicates that the packet was recognized as a - * RoCE. This also indicates that the payload_offset field is - * valid. - */ - #define RX_PKT_CMPL_FLAGS_ITYPE_ROCE (UINT32_C(0x5) << 12) - /* - * ICMP Packet: Indicates that the packet was recognized as - * ICMP. This indicates that the payload_offset field is valid. - */ - #define RX_PKT_CMPL_FLAGS_ITYPE_ICMP (UINT32_C(0x7) << 12) - /* - * PtP packet wo/timestamp: Indicates that the packet was - * recognized as a PtP packet. - */ - #define RX_PKT_CMPL_FLAGS_ITYPE_PTP_WO_TIMESTAMP \ - (UINT32_C(0x8) << 12) - /* - * PtP packet w/timestamp: Indicates that the packet was - * recognized as a PtP packet and that a timestamp was taken for - * the packet. - */ + /* + * IP Packet: Indicates that the packet was an + * IP packet, but further classification was not + * possible. + */ + #define RX_PKT_CMPL_FLAGS_ITYPE_IP (UINT32_C(0x1) << 12) + /* + * TCP Packet: Indicates that the packet was IP + * and TCP. This indicates that the + * payload_offset field is valid. + */ + #define RX_PKT_CMPL_FLAGS_ITYPE_TCP (UINT32_C(0x2) << 12) + /* + * UDP Packet: Indicates that the packet was IP + * and UDP. This indicates that the + * payload_offset field is valid. + */ + #define RX_PKT_CMPL_FLAGS_ITYPE_UDP (UINT32_C(0x3) << 12) + /* + * FCoE Packet: Indicates that the packet was + * recognized as a FCoE. This also indicates + * that the payload_offset field is valid. + */ + #define RX_PKT_CMPL_FLAGS_ITYPE_FCOE (UINT32_C(0x4) << 12) + /* + * RoCE Packet: Indicates that the packet was + * recognized as a RoCE. This also indicates + * that the payload_offset field is valid. + */ + #define RX_PKT_CMPL_FLAGS_ITYPE_ROCE (UINT32_C(0x5) << 12) + /* + * ICMP Packet: Indicates that the packet was + * recognized as ICMP. This indicates that the + * payload_offset field is valid. + */ + #define RX_PKT_CMPL_FLAGS_ITYPE_ICMP (UINT32_C(0x7) << 12) + /* + * PtP packet wo/timestamp: Indicates that the + * packet was recognized as a PtP packet. + */ + #define RX_PKT_CMPL_FLAGS_ITYPE_PTP_WO_TIMESTAMP (UINT32_C(0x8) << 12) + /* + * PtP packet w/timestamp: Indicates that the + * packet was recognized as a PtP packet and + * that a timestamp was taken for the packet. + */ #define RX_PKT_CMPL_FLAGS_ITYPE_PTP_W_TIMESTAMP (UINT32_C(0x9) << 12) #define RX_PKT_CMPL_FLAGS_ITYPE_LAST \ - RX_PKT_CMPL_FLAGS_ITYPE_PTP_W_TIMESTAMP - #define RX_PKT_CMPL_FLAGS_MASK UINT32_C(0xffc0) - #define RX_PKT_CMPL_FLAGS_SFT 6 - uint16_t flags_type; - - /* - * This is the length of the data for the packet stored in the buffer(s) - * identified by the opaque value. This includes the packet BD and any - * associated buffer BDs. This does not include the the length of any - * data places in aggregation BDs. - */ + RX_PKT_CMPL_FLAGS_ITYPE_PTP_W_TIMESTAMP + #define RX_PKT_CMPL_FLAGS_MASK UINT32_C(0xffc0) + #define RX_PKT_CMPL_FLAGS_SFT 6 uint16_t len; - /* - * This is a copy of the opaque field from the RX BD this completion - * corresponds to. + * This is the length of the data for the packet stored in the + * buffer(s) identified by the opaque value. This includes the + * packet BD and any associated buffer BDs. This does not + * include the the length of any data places in aggregation BDs. */ uint32_t opaque; - /* - * This value is written by the NIC such that it will be different for - * each pass through the completion queue. The even passes will write 1. - * The odd passes will write 0. + * This is a copy of the opaque field from the RX BD this + * completion corresponds to. */ - #define RX_PKT_CMPL_V1 UINT32_C(0x1) - /* - * This value is the number of aggregation buffers that follow this - * entry in the completion ring that are a part of this packet. If the - * value is zero, then the packet is completely contained in the buffer - * space provided for the packet in the RX ring. - */ - #define RX_PKT_CMPL_AGG_BUFS_MASK UINT32_C(0x3e) - #define RX_PKT_CMPL_AGG_BUFS_SFT 1 uint8_t agg_bufs_v1; - + /* unused1 is 2 b */ + /* + * This value is written by the NIC such that it will be + * different for each pass through the completion queue. The + * even passes will write 1. The odd passes will write 0. + */ + #define RX_PKT_CMPL_V1 UINT32_C(0x1) + /* + * This value is the number of aggregation buffers that follow + * this entry in the completion ring that are a part of this + * packet. If the value is zero, then the packet is completely + * contained in the buffer space provided for the packet in the + * RX ring. + */ + #define RX_PKT_CMPL_AGG_BUFS_MASK UINT32_C(0x3e) + #define RX_PKT_CMPL_AGG_BUFS_SFT 1 + /* unused1 is 2 b */ + uint8_t rss_hash_type; /* * This is the RSS hash type for the packet. The value is packed - * {tuple_extrac_op[1:0],rss_profile_id[4:0],tuple_extrac_op[2]}. - */ - uint8_t rss_hash_type; - - /* - * This value indicates the offset from the beginning of the packet - * where the inner payload starts. This value is valid for TCP, UDP, - * FCoE, and RoCE packets. + * {tuple_extrac_op[1:0],rss_profile_id[4:0],tuple_extrac_op[2]} + * . The value of tuple_extrac_op provides the information about + * what fields the hash was computed on. * 0: The RSS hash was + * computed over source IP address, destination IP address, + * source port, and destination port of inner IP and TCP or UDP + * headers. Note: For non-tunneled packets, the packet headers + * are considered inner packet headers for the RSS hash + * computation purpose. * 1: The RSS hash was computed over + * source IP address and destination IP address of inner IP + * header. Note: For non-tunneled packets, the packet headers + * are considered inner packet headers for the RSS hash + * computation purpose. * 2: The RSS hash was computed over + * source IP address, destination IP address, source port, and + * destination port of IP and TCP or UDP headers of outer tunnel + * headers. Note: For non-tunneled packets, this value is not + * applicable. * 3: The RSS hash was computed over source IP + * address and destination IP address of IP header of outer + * tunnel headers. Note: For non-tunneled packets, this value is + * not applicable. Note that 4-tuples values listed above are + * applicable for layer 4 protocols supported and enabled for + * RSS in the hardware, HWRM firmware, and drivers. For example, + * if RSS hash is supported and enabled for TCP traffic only, + * then the values of tuple_extract_op corresponding to 4-tuples + * are only valid for TCP traffic. */ uint8_t payload_offset; - - uint8_t unused_1; - /* - * This value is the RSS hash value calculated for the packet based on - * the mode bits and key value in the VNIC. + * This value indicates the offset in bytes from the beginning + * of the packet where the inner payload starts. This value is + * valid for TCP, UDP, FCoE, and RoCE packets. A value of zero + * indicates that header is 256B into the packet. */ + uint8_t unused_1; + /* unused2 is 8 b */ uint32_t rss_hash; + /* + * This value is the RSS hash value calculated for the packet + * based on the mode bits and key value in the VNIC. + */ } __attribute__((packed)); /* last 16 bytes of RX Packet Completion Record */ struct rx_pkt_cmpl_hi { + uint32_t flags2; /* - * This indicates that the ip checksum was calculated for the inner - * packet and that the ip_cs_error field indicates if there was an - * error. + * This indicates that the ip checksum was calculated for the + * inner packet and that the ip_cs_error field indicates if + * there was an error. */ - #define RX_PKT_CMPL_FLAGS2_IP_CS_CALC UINT32_C(0x1) + #define RX_PKT_CMPL_FLAGS2_IP_CS_CALC UINT32_C(0x1) /* - * This indicates that the TCP, UDP or ICMP checksum was calculated for - * the inner packet and that the l4_cs_error field indicates if there - * was an error. + * This indicates that the TCP, UDP or ICMP checksum was + * calculated for the inner packet and that the l4_cs_error + * field indicates if there was an error. */ - #define RX_PKT_CMPL_FLAGS2_L4_CS_CALC UINT32_C(0x2) + #define RX_PKT_CMPL_FLAGS2_L4_CS_CALC UINT32_C(0x2) /* - * This indicates that the ip checksum was calculated for the tunnel - * header and that the t_ip_cs_error field indicates if there was an - * error. + * This indicates that the ip checksum was calculated for the + * tunnel header and that the t_ip_cs_error field indicates if + * there was an error. */ - #define RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC UINT32_C(0x4) + #define RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC UINT32_C(0x4) /* - * This indicates that the UDP checksum was calculated for the tunnel - * packet and that the t_l4_cs_error field indicates if there was an - * error. + * This indicates that the UDP checksum was calculated for the + * tunnel packet and that the t_l4_cs_error field indicates if + * there was an error. */ - #define RX_PKT_CMPL_FLAGS2_T_L4_CS_CALC UINT32_C(0x8) + #define RX_PKT_CMPL_FLAGS2_T_L4_CS_CALC UINT32_C(0x8) /* This value indicates what format the metadata field is. */ #define RX_PKT_CMPL_FLAGS2_META_FORMAT_MASK UINT32_C(0xf0) #define RX_PKT_CMPL_FLAGS2_META_FORMAT_SFT 4 - /* No metadata informtaion. Value is zero. */ + /* No metadata informtaion. Value is zero. */ #define RX_PKT_CMPL_FLAGS2_META_FORMAT_NONE (UINT32_C(0x0) << 4) - /* - * The metadata field contains the VLAN tag and TPID value. - - * metadata[11:0] contains the vlan VID value. - metadata[12] - * contains the vlan DE value. - metadata[15:13] contains the - * vlan PRI value. - metadata[31:16] contains the vlan TPID - * value. - */ + /* + * The metadata field contains the VLAN tag and + * TPID value. - metadata[11:0] contains the + * vlan VID value. - metadata[12] contains the + * vlan DE value. - metadata[15:13] contains the + * vlan PRI value. - metadata[31:16] contains + * the vlan TPID value. + */ #define RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN (UINT32_C(0x1) << 4) #define RX_PKT_CMPL_FLAGS2_META_FORMAT_LAST \ - RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN + RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN /* - * This field indicates the IP type for the inner-most IP header. A - * value of '0' indicates IPv4. A value of '1' indicates IPv6. This - * value is only valid if itype indicates a packet with an IP header. + * This field indicates the IP type for the inner-most IP + * header. A value of '0' indicates IPv4. A value of '1' + * indicates IPv6. This value is only valid if itype indicates a + * packet with an IP header. */ - #define RX_PKT_CMPL_FLAGS2_IP_TYPE UINT32_C(0x100) - uint32_t flags2; - + #define RX_PKT_CMPL_FLAGS2_IP_TYPE UINT32_C(0x100) + uint32_t metadata; /* - * This is data from the CFA block as indicated by the meta_format - * field. + * This is data from the CFA block as indicated by the + * meta_format field. */ /* When meta_format=1, this value is the VLAN VID. */ - #define RX_PKT_CMPL_METADATA_VID_MASK UINT32_C(0xfff) - #define RX_PKT_CMPL_METADATA_VID_SFT 0 + #define RX_PKT_CMPL_METADATA_VID_MASK UINT32_C(0xfff) + #define RX_PKT_CMPL_METADATA_VID_SFT 0 /* When meta_format=1, this value is the VLAN DE. */ - #define RX_PKT_CMPL_METADATA_DE UINT32_C(0x1000) + #define RX_PKT_CMPL_METADATA_DE UINT32_C(0x1000) /* When meta_format=1, this value is the VLAN PRI. */ - #define RX_PKT_CMPL_METADATA_PRI_MASK UINT32_C(0xe000) - #define RX_PKT_CMPL_METADATA_PRI_SFT 13 + #define RX_PKT_CMPL_METADATA_PRI_MASK UINT32_C(0xe000) + #define RX_PKT_CMPL_METADATA_PRI_SFT 13 /* When meta_format=1, this value is the VLAN TPID. */ - #define RX_PKT_CMPL_METADATA_TPID_MASK UINT32_C(0xffff0000) - #define RX_PKT_CMPL_METADATA_TPID_SFT 16 - uint32_t metadata; - + #define RX_PKT_CMPL_METADATA_TPID_MASK UINT32_C(0xffff0000) + #define RX_PKT_CMPL_METADATA_TPID_SFT 16 + uint16_t errors_v2; /* - * This value is written by the NIC such that it will be different for - * each pass through the completion queue. The even passes will write 1. - * The odd passes will write 0. + * This value is written by the NIC such that it will be + * different for each pass through the completion queue. The + * even passes will write 1. The odd passes will write 0. */ - #define RX_PKT_CMPL_V2 UINT32_C(0x1) + #define RX_PKT_CMPL_V2 UINT32_C(0x1) /* - * This error indicates that there was some sort of problem with the BDs - * for the packet that was found after part of the packet was already - * placed. The packet should be treated as invalid. + * This error indicates that there was some sort of problem with + * the BDs for the packet that was found after part of the + * packet was already placed. The packet should be treated as + * invalid. */ #define RX_PKT_CMPL_ERRORS_BUFFER_ERROR_MASK UINT32_C(0xe) #define RX_PKT_CMPL_ERRORS_BUFFER_ERROR_SFT 1 - /* No buffer error */ - #define RX_PKT_CMPL_ERRORS_BUFFER_ERROR_NO_BUFFER \ - (UINT32_C(0x0) << 1) - /* - * Did Not Fit: Packet did not fit into packet buffer provided. - * For regular placement, this means the packet did not fit in - * the buffer provided. For HDS and jumbo placement, this means - * that the packet could not be placed into 7 physical buffers - * or less. - */ - #define RX_PKT_CMPL_ERRORS_BUFFER_ERROR_DID_NOT_FIT \ - (UINT32_C(0x1) << 1) - /* - * Not On Chip: All BDs needed for the packet were not on-chip - * when the packet arrived. - */ - #define RX_PKT_CMPL_ERRORS_BUFFER_ERROR_NOT_ON_CHIP \ - (UINT32_C(0x2) << 1) - /* Bad Format: BDs were not formatted correctly. */ - #define RX_PKT_CMPL_ERRORS_BUFFER_ERROR_BAD_FORMAT \ - (UINT32_C(0x3) << 1) - #define RX_PKT_CMPL_ERRORS_BUFFER_ERROR_LAST \ - RX_PKT_CMPL_ERRORS_BUFFER_ERROR_BAD_FORMAT - /* This indicates that there was an error in the IP header checksum. */ - #define RX_PKT_CMPL_ERRORS_IP_CS_ERROR UINT32_C(0x10) + /* No buffer error */ + #define RX_PKT_CMPL_ERRORS_BUFFER_ERROR_NO_BUFFER (UINT32_C(0x0) << 1) /* - * This indicates that there was an error in the TCP, UDP or ICMP - * checksum. + * Did Not Fit: Packet did not fit into packet + * buffer provided. For regular placement, this + * means the packet did not fit in the buffer + * provided. For HDS and jumbo placement, this + * means that the packet could not be placed + * into 7 physical buffers or less. */ - #define RX_PKT_CMPL_ERRORS_L4_CS_ERROR UINT32_C(0x20) + #define RX_PKT_CMPL_ERRORS_BUFFER_ERROR_DID_NOT_FIT \ + (UINT32_C(0x1) << 1) /* - * This indicates that there was an error in the tunnel IP header - * checksum. + * Not On Chip: All BDs needed for the packet + * were not on-chip when the packet arrived. + */ + #define RX_PKT_CMPL_ERRORS_BUFFER_ERROR_NOT_ON_CHIP \ + (UINT32_C(0x2) << 1) + /* Bad Format: BDs were not formatted correctly. */ + #define RX_PKT_CMPL_ERRORS_BUFFER_ERROR_BAD_FORMAT \ + (UINT32_C(0x3) << 1) + #define RX_PKT_CMPL_ERRORS_BUFFER_ERROR_LAST \ + RX_PKT_CMPL_ERRORS_BUFFER_ERROR_BAD_FORMAT + /* This indicates that there was an error in the IP header checksum. */ + #define RX_PKT_CMPL_ERRORS_IP_CS_ERROR UINT32_C(0x10) + /* + * This indicates that there was an error in the TCP, UDP or + * ICMP checksum. + */ + #define RX_PKT_CMPL_ERRORS_L4_CS_ERROR UINT32_C(0x20) + /* + * This indicates that there was an error in the tunnel IP + * header checksum. */ #define RX_PKT_CMPL_ERRORS_T_IP_CS_ERROR UINT32_C(0x40) - /* This indicates that there was an error in the tunnel UDP checksum. */ + /* + * This indicates that there was an error in the tunnel UDP + * checksum. + */ #define RX_PKT_CMPL_ERRORS_T_L4_CS_ERROR UINT32_C(0x80) /* - * This indicates that there was a CRC error on either an FCoE or RoCE - * packet. The itype indicates the packet type. + * This indicates that there was a CRC error on either an FCoE + * or RoCE packet. The itype indicates the packet type. */ - #define RX_PKT_CMPL_ERRORS_CRC_ERROR UINT32_C(0x100) + #define RX_PKT_CMPL_ERRORS_CRC_ERROR UINT32_C(0x100) /* - * This indicates that there was an error in the tunnel portion of the - * packet when this field is non-zero. + * This indicates that there was an error in the tunnel portion + * of the packet when this field is non-zero. */ #define RX_PKT_CMPL_ERRORS_T_PKT_ERROR_MASK UINT32_C(0xe00) #define RX_PKT_CMPL_ERRORS_T_PKT_ERROR_SFT 9 - /* - * No additional error occurred on the tunnel portion of the - * packet of the packet does not have a tunnel. - */ - #define RX_PKT_CMPL_ERRORS_T_PKT_ERROR_NO_ERROR (UINT32_C(0x0) << 9) - /* - * Indicates that IP header version does not match expectation - * from L2 Ethertype for IPv4 and IPv6 in the tunnel header. - */ - #define RX_PKT_CMPL_ERRORS_T_PKT_ERROR_T_L3_BAD_VERSION \ - (UINT32_C(0x1) << 9) - /* - * Indicates that header length is out of range in the tunnel - * header. Valid for IPv4. - */ - #define RX_PKT_CMPL_ERRORS_T_PKT_ERROR_T_L3_BAD_HDR_LEN \ - (UINT32_C(0x2) << 9) - /* - * Indicates that the physical packet is shorter than that - * claimed by the PPPoE header length for a tunnel PPPoE packet. - */ - #define RX_PKT_CMPL_ERRORS_T_PKT_ERROR_TUNNEL_TOTAL_ERROR \ - (UINT32_C(0x3) << 9) - /* - * Indicates that physical packet is shorter than that claimed - * by the tunnel l3 header length. Valid for IPv4, or IPv6 - * tunnel packet packets. - */ - #define RX_PKT_CMPL_ERRORS_T_PKT_ERROR_T_IP_TOTAL_ERROR \ - (UINT32_C(0x4) << 9) - /* - * Indicates that the physical packet is shorter than that - * claimed by the tunnel UDP header length for a tunnel UDP - * packet that is not fragmented. - */ - #define RX_PKT_CMPL_ERRORS_T_PKT_ERROR_T_UDP_TOTAL_ERROR \ - (UINT32_C(0x5) << 9) - /* - * indicates that the IPv4 TTL or IPv6 hop limit check have - * failed (e.g. TTL = 0) in the tunnel header. Valid for IPv4, - * and IPv6. - */ - #define RX_PKT_CMPL_ERRORS_T_PKT_ERROR_T_L3_BAD_TTL \ - (UINT32_C(0x6) << 9) - #define RX_PKT_CMPL_ERRORS_T_PKT_ERROR_LAST \ - RX_PKT_CMPL_ERRORS_T_PKT_ERROR_T_L3_BAD_TTL /* - * This indicates that there was an error in the inner portion of the - * packet when this field is non-zero. + * No additional error occurred on the tunnel + * portion of the packet of the packet does not + * have a tunnel. + */ + #define RX_PKT_CMPL_ERRORS_T_PKT_ERROR_NO_ERROR (UINT32_C(0x0) << 9) + /* + * Indicates that IP header version does not + * match expectation from L2 Ethertype for IPv4 + * and IPv6 in the tunnel header. + */ + #define RX_PKT_CMPL_ERRORS_T_PKT_ERROR_T_L3_BAD_VERSION \ + (UINT32_C(0x1) << 9) + /* + * Indicates that header length is out of range + * in the tunnel header. Valid for IPv4. + */ + #define RX_PKT_CMPL_ERRORS_T_PKT_ERROR_T_L3_BAD_HDR_LEN \ + (UINT32_C(0x2) << 9) + /* + * Indicates that the physical packet is shorter + * than that claimed by the PPPoE header length + * for a tunnel PPPoE packet. + */ + #define RX_PKT_CMPL_ERRORS_T_PKT_ERROR_TUNNEL_TOTAL_ERROR \ + (UINT32_C(0x3) << 9) + /* + * Indicates that physical packet is shorter + * than that claimed by the tunnel l3 header + * length. Valid for IPv4, or IPv6 tunnel packet + * packets. + */ + #define RX_PKT_CMPL_ERRORS_T_PKT_ERROR_T_IP_TOTAL_ERROR \ + (UINT32_C(0x4) << 9) + /* + * Indicates that the physical packet is shorter + * than that claimed by the tunnel UDP header + * length for a tunnel UDP packet that is not + * fragmented. + */ + #define RX_PKT_CMPL_ERRORS_T_PKT_ERROR_T_UDP_TOTAL_ERROR \ + (UINT32_C(0x5) << 9) + /* + * indicates that the IPv4 TTL or IPv6 hop limit + * check have failed (e.g. TTL = 0) in the + * tunnel header. Valid for IPv4, and IPv6. + */ + #define RX_PKT_CMPL_ERRORS_T_PKT_ERROR_T_L3_BAD_TTL \ + (UINT32_C(0x6) << 9) + #define RX_PKT_CMPL_ERRORS_T_PKT_ERROR_LAST \ + RX_PKT_CMPL_ERRORS_T_PKT_ERROR_T_L3_BAD_TTL + /* + * This indicates that there was an error in the inner portion + * of the packet when this field is non-zero. */ #define RX_PKT_CMPL_ERRORS_PKT_ERROR_MASK UINT32_C(0xf000) #define RX_PKT_CMPL_ERRORS_PKT_ERROR_SFT 12 - /* - * No additional error occurred on the tunnel portion of the - * packet of the packet does not have a tunnel. - */ + /* + * No additional error occurred on the tunnel + * portion of the packet of the packet does not + * have a tunnel. + */ #define RX_PKT_CMPL_ERRORS_PKT_ERROR_NO_ERROR (UINT32_C(0x0) << 12) - /* - * Indicates that IP header version does not match expectation - * from L2 Ethertype for IPv4 and IPv6 or that option other than - * VFT was parsed on FCoE packet. - */ + /* + * Indicates that IP header version does not + * match expectation from L2 Ethertype for IPv4 + * and IPv6 or that option other than VFT was + * parsed on FCoE packet. + */ #define RX_PKT_CMPL_ERRORS_PKT_ERROR_L3_BAD_VERSION \ - (UINT32_C(0x1) << 12) - /* - * indicates that header length is out of range. Valid for IPv4 - * and RoCE - */ + (UINT32_C(0x1) << 12) + /* + * indicates that header length is out of range. + * Valid for IPv4 and RoCE + */ #define RX_PKT_CMPL_ERRORS_PKT_ERROR_L3_BAD_HDR_LEN \ - (UINT32_C(0x2) << 12) - /* - * indicates that the IPv4 TTL or IPv6 hop limit check have - * failed (e.g. TTL = 0). Valid for IPv4, and IPv6 - */ + (UINT32_C(0x2) << 12) + /* + * indicates that the IPv4 TTL or IPv6 hop limit + * check have failed (e.g. TTL = 0). Valid for + * IPv4, and IPv6 + */ #define RX_PKT_CMPL_ERRORS_PKT_ERROR_L3_BAD_TTL (UINT32_C(0x3) << 12) - /* - * Indicates that physical packet is shorter than that claimed - * by the l3 header length. Valid for IPv4, IPv6 packet or RoCE - * packets. - */ + /* + * Indicates that physical packet is shorter + * than that claimed by the l3 header length. + * Valid for IPv4, IPv6 packet or RoCE packets. + */ #define RX_PKT_CMPL_ERRORS_PKT_ERROR_IP_TOTAL_ERROR \ - (UINT32_C(0x4) << 12) - /* - * Indicates that the physical packet is shorter than that - * claimed by the UDP header length for a UDP packet that is not - * fragmented. - */ + (UINT32_C(0x4) << 12) + /* + * Indicates that the physical packet is shorter + * than that claimed by the UDP header length + * for a UDP packet that is not fragmented. + */ #define RX_PKT_CMPL_ERRORS_PKT_ERROR_UDP_TOTAL_ERROR \ - (UINT32_C(0x5) << 12) - /* - * Indicates that TCP header length > IP payload. Valid for TCP - * packets only. - */ + (UINT32_C(0x5) << 12) + /* + * Indicates that TCP header length > IP + * payload. Valid for TCP packets only. + */ #define RX_PKT_CMPL_ERRORS_PKT_ERROR_L4_BAD_HDR_LEN \ - (UINT32_C(0x6) << 12) - /* Indicates that TCP header length < 5. Valid for TCP. */ + (UINT32_C(0x6) << 12) + /* Indicates that TCP header length < 5. Valid for TCP. */ #define RX_PKT_CMPL_ERRORS_PKT_ERROR_L4_BAD_HDR_LEN_TOO_SMALL \ - (UINT32_C(0x7) << 12) - /* - * Indicates that TCP option headers result in a TCP header size - * that does not match data offset in TCP header. Valid for TCP. - */ + (UINT32_C(0x7) << 12) + /* + * Indicates that TCP option headers result in a + * TCP header size that does not match data + * offset in TCP header. Valid for TCP. + */ #define RX_PKT_CMPL_ERRORS_PKT_ERROR_L4_BAD_OPT_LEN \ - (UINT32_C(0x8) << 12) + (UINT32_C(0x8) << 12) #define RX_PKT_CMPL_ERRORS_PKT_ERROR_LAST \ - RX_PKT_CMPL_ERRORS_PKT_ERROR_L4_BAD_OPT_LEN - #define RX_PKT_CMPL_ERRORS_MASK UINT32_C(0xfffe) - #define RX_PKT_CMPL_ERRORS_SFT 1 - uint16_t errors_v2; - - /* - * This field identifies the CFA action rule that was used for this - * packet. - */ + RX_PKT_CMPL_ERRORS_PKT_ERROR_L4_BAD_OPT_LEN + #define RX_PKT_CMPL_ERRORS_MASK UINT32_C(0xfffe) + #define RX_PKT_CMPL_ERRORS_SFT 1 uint16_t cfa_code; - /* - * This value holds the reordering sequence number for the packet. If - * the reordering sequence is not valid, then this value is zero. The - * reordering domain for the packet is in the bottom 8 to 10b of the - * rss_hash value. The bottom 20b of this value contain the ordering - * domain value for the packet. + * This field identifies the CFA action rule that was used for + * this packet. */ - #define RX_PKT_CMPL_REORDER_MASK UINT32_C(0xffffff) - #define RX_PKT_CMPL_REORDER_SFT 0 uint32_t reorder; + /* + * This value holds the reordering sequence number for the + * packet. If the reordering sequence is not valid, then this + * value is zero. The reordering domain for the packet is in the + * bottom 8 to 10b of the rss_hash value. The bottom 20b of this + * value contain the ordering domain value for the packet. + */ + #define RX_PKT_CMPL_REORDER_MASK UINT32_C(0xffffff) + #define RX_PKT_CMPL_REORDER_SFT 0 } __attribute__((packed)); -/* HWRM Forwarded Request (16 bytes) */ +/* RX L2 TPA Start Completion Record (32 bytes split to 2 16-byte struct) */ +struct rx_tpa_start_cmpl { + uint16_t flags_type; + /* + * This field indicates the exact type of the completion. By + * convention, the LSB identifies the length of the record in + * 16B units. Even values indicate 16B records. Odd values + * indicate 32B records. + */ + #define RX_TPA_START_CMPL_TYPE_MASK UINT32_C(0x3f) + #define RX_TPA_START_CMPL_TYPE_SFT 0 + /* + * RX L2 TPA Start Completion: Completion at the + * beginning of a TPA operation. Length = 32B + */ + #define RX_TPA_START_CMPL_TYPE_RX_TPA_START UINT32_C(0x13) + /* This bit will always be '0' for TPA start completions. */ + #define RX_TPA_START_CMPL_FLAGS_ERROR UINT32_C(0x40) + /* This field indicates how the packet was placed in the buffer. */ + #define RX_TPA_START_CMPL_FLAGS_PLACEMENT_MASK UINT32_C(0x380) + #define RX_TPA_START_CMPL_FLAGS_PLACEMENT_SFT 7 + /* + * Jumbo: TPA Packet was placed using jumbo + * algorithm. This means that the first buffer + * will be filled with data before moving to + * aggregation buffers. Each aggregation buffer + * will be filled before moving to the next + * aggregation buffer. + */ + #define RX_TPA_START_CMPL_FLAGS_PLACEMENT_JUMBO (UINT32_C(0x1) << 7) + /* + * Header/Data Separation: Packet was placed + * using Header/Data separation algorithm. The + * separation location is indicated by the itype + * field. + */ + #define RX_TPA_START_CMPL_FLAGS_PLACEMENT_HDS (UINT32_C(0x2) << 7) + /* + * GRO/Jumbo: Packet will be placed using + * GRO/Jumbo where the first packet is filled + * with data. Subsequent packets will be placed + * such that any one packet does not span two + * aggregation buffers unless it starts at the + * beginning of an aggregation buffer. + */ + #define RX_TPA_START_CMPL_FLAGS_PLACEMENT_GRO_JUMBO \ + (UINT32_C(0x5) << 7) + /* + * GRO/Header-Data Separation: Packet will be + * placed using GRO/HDS where the header is in + * the first packet. Payload of each packet will + * be placed such that any one packet does not + * span two aggregation buffers unless it starts + * at the beginning of an aggregation buffer. + */ + #define RX_TPA_START_CMPL_FLAGS_PLACEMENT_GRO_HDS (UINT32_C(0x6) << 7) + #define RX_TPA_START_CMPL_FLAGS_PLACEMENT_LAST \ + RX_TPA_START_CMPL_FLAGS_PLACEMENT_GRO_HDS + /* This bit is '1' if the RSS field in this completion is valid. */ + #define RX_TPA_START_CMPL_FLAGS_RSS_VALID UINT32_C(0x400) + /* unused is 1 b */ + #define RX_TPA_START_CMPL_FLAGS_UNUSED UINT32_C(0x800) + /* + * This value indicates what the inner packet determined for the + * packet was. + */ + #define RX_TPA_START_CMPL_FLAGS_ITYPE_MASK UINT32_C(0xf000) + #define RX_TPA_START_CMPL_FLAGS_ITYPE_SFT 12 + /* TCP Packet: Indicates that the packet was IP and TCP. */ + #define RX_TPA_START_CMPL_FLAGS_ITYPE_TCP (UINT32_C(0x2) << 12) + #define RX_TPA_START_CMPL_FLAGS_ITYPE_LAST \ + RX_TPA_START_CMPL_FLAGS_ITYPE_TCP + #define RX_TPA_START_CMPL_FLAGS_MASK UINT32_C(0xffc0) + #define RX_TPA_START_CMPL_FLAGS_SFT 6 + uint16_t len; + /* + * This value indicates the amount of packet data written to the + * buffer the opaque field in this completion corresponds to. + */ + uint32_t opaque; + /* + * This is a copy of the opaque field from the RX BD this + * completion corresponds to. + */ + uint8_t v1; + /* unused1 is 7 b */ + /* + * This value is written by the NIC such that it will be + * different for each pass through the completion queue. The + * even passes will write 1. The odd passes will write 0. + */ + #define RX_TPA_START_CMPL_V1 UINT32_C(0x1) + /* unused1 is 7 b */ + uint8_t rss_hash_type; + /* + * This is the RSS hash type for the packet. The value is packed + * {tuple_extrac_op[1:0],rss_profile_id[4:0],tuple_extrac_op[2]} + * . The value of tuple_extrac_op provides the information about + * what fields the hash was computed on. * 0: The RSS hash was + * computed over source IP address, destination IP address, + * source port, and destination port of inner IP and TCP or UDP + * headers. Note: For non-tunneled packets, the packet headers + * are considered inner packet headers for the RSS hash + * computation purpose. * 1: The RSS hash was computed over + * source IP address and destination IP address of inner IP + * header. Note: For non-tunneled packets, the packet headers + * are considered inner packet headers for the RSS hash + * computation purpose. * 2: The RSS hash was computed over + * source IP address, destination IP address, source port, and + * destination port of IP and TCP or UDP headers of outer tunnel + * headers. Note: For non-tunneled packets, this value is not + * applicable. * 3: The RSS hash was computed over source IP + * address and destination IP address of IP header of outer + * tunnel headers. Note: For non-tunneled packets, this value is + * not applicable. Note that 4-tuples values listed above are + * applicable for layer 4 protocols supported and enabled for + * RSS in the hardware, HWRM firmware, and drivers. For example, + * if RSS hash is supported and enabled for TCP traffic only, + * then the values of tuple_extract_op corresponding to 4-tuples + * are only valid for TCP traffic. + */ + uint16_t agg_id; + /* + * This is the aggregation ID that the completion is associated + * with. Use this number to correlate the TPA start completion + * with the TPA end completion. + */ + /* unused2 is 9 b */ + /* + * This is the aggregation ID that the completion is associated + * with. Use this number to correlate the TPA start completion + * with the TPA end completion. + */ + #define RX_TPA_START_CMPL_AGG_ID_MASK UINT32_C(0xfe00) + #define RX_TPA_START_CMPL_AGG_ID_SFT 9 + uint32_t rss_hash; + /* + * This value is the RSS hash value calculated for the packet + * based on the mode bits and key value in the VNIC. + */ +} __attribute__((packed)); + +/* last 16 bytes of RX L2 TPA Start Completion Record */ +struct rx_tpa_start_cmpl_hi { + uint32_t flags2; + /* + * This indicates that the ip checksum was calculated for the + * inner packet and that the sum passed for all segments + * included in the aggregation. + */ + #define RX_TPA_START_CMPL_FLAGS2_IP_CS_CALC UINT32_C(0x1) + /* + * This indicates that the TCP, UDP or ICMP checksum was + * calculated for the inner packet and that the sum passed for + * all segments included in the aggregation. + */ + #define RX_TPA_START_CMPL_FLAGS2_L4_CS_CALC UINT32_C(0x2) + /* + * This indicates that the ip checksum was calculated for the + * tunnel header and that the sum passed for all segments + * included in the aggregation. + */ + #define RX_TPA_START_CMPL_FLAGS2_T_IP_CS_CALC UINT32_C(0x4) + /* + * This indicates that the UDP checksum was calculated for the + * tunnel packet and that the sum passed for all segments + * included in the aggregation. + */ + #define RX_TPA_START_CMPL_FLAGS2_T_L4_CS_CALC UINT32_C(0x8) + /* This value indicates what format the metadata field is. */ + #define RX_TPA_START_CMPL_FLAGS2_META_FORMAT_MASK UINT32_C(0xf0) + #define RX_TPA_START_CMPL_FLAGS2_META_FORMAT_SFT 4 + /* No metadata informtaion. Value is zero. */ + #define RX_TPA_START_CMPL_FLAGS2_META_FORMAT_NONE (UINT32_C(0x0) << 4) + /* + * The metadata field contains the VLAN tag and + * TPID value. - metadata[11:0] contains the + * vlan VID value. - metadata[12] contains the + * vlan DE value. - metadata[15:13] contains the + * vlan PRI value. - metadata[31:16] contains + * the vlan TPID value. + */ + #define RX_TPA_START_CMPL_FLAGS2_META_FORMAT_VLAN (UINT32_C(0x1) << 4) + #define RX_TPA_START_CMPL_FLAGS2_META_FORMAT_LAST \ + RX_TPA_START_CMPL_FLAGS2_META_FORMAT_VLAN + /* + * This field indicates the IP type for the inner-most IP + * header. A value of '0' indicates IPv4. A value of '1' + * indicates IPv6. + */ + #define RX_TPA_START_CMPL_FLAGS2_IP_TYPE UINT32_C(0x100) + uint32_t metadata; + /* + * This is data from the CFA block as indicated by the + * meta_format field. + */ + /* When meta_format=1, this value is the VLAN VID. */ + #define RX_TPA_START_CMPL_METADATA_VID_MASK UINT32_C(0xfff) + #define RX_TPA_START_CMPL_METADATA_VID_SFT 0 + /* When meta_format=1, this value is the VLAN DE. */ + #define RX_TPA_START_CMPL_METADATA_DE UINT32_C(0x1000) + /* When meta_format=1, this value is the VLAN PRI. */ + #define RX_TPA_START_CMPL_METADATA_PRI_MASK UINT32_C(0xe000) + #define RX_TPA_START_CMPL_METADATA_PRI_SFT 13 + /* When meta_format=1, this value is the VLAN TPID. */ + #define RX_TPA_START_CMPL_METADATA_TPID_MASK UINT32_C(0xffff0000) + #define RX_TPA_START_CMPL_METADATA_TPID_SFT 16 + uint16_t v2; + /* unused4 is 15 b */ + /* + * This value is written by the NIC such that it will be + * different for each pass through the completion queue. The + * even passes will write 1. The odd passes will write 0. + */ + #define RX_TPA_START_CMPL_V2 UINT32_C(0x1) + /* unused4 is 15 b */ + uint16_t cfa_code; + /* + * This field identifies the CFA action rule that was used for + * this packet. + */ + uint32_t inner_l4_size_inner_l3_offset_inner_l2_offset_outer_l3_offset; + /* + * This is the size in bytes of the inner most L4 header. This + * can be subtracted from the payload_offset to determine the + * start of the inner most L4 header. + */ + /* + * This is the offset from the beginning of the packet in bytes + * for the outer L3 header. If there is no outer L3 header, then + * this value is zero. + */ + #define RX_TPA_START_CMPL_OUTER_L3_OFFSET_MASK UINT32_C(0x1ff) + #define RX_TPA_START_CMPL_OUTER_L3_OFFSET_SFT 0 + /* + * This is the offset from the beginning of the packet in bytes + * for the inner most L2 header. + */ + #define RX_TPA_START_CMPL_INNER_L2_OFFSET_MASK UINT32_C(0x3fe00) + #define RX_TPA_START_CMPL_INNER_L2_OFFSET_SFT 9 + /* + * This is the offset from the beginning of the packet in bytes + * for the inner most L3 header. + */ + #define RX_TPA_START_CMPL_INNER_L3_OFFSET_MASK UINT32_C(0x7fc0000) + #define RX_TPA_START_CMPL_INNER_L3_OFFSET_SFT 18 + /* + * This is the size in bytes of the inner most L4 header. This + * can be subtracted from the payload_offset to determine the + * start of the inner most L4 header. + */ + #define RX_TPA_START_CMPL_INNER_L4_SIZE_MASK UINT32_C(0xf8000000) + #define RX_TPA_START_CMPL_INNER_L4_SIZE_SFT 27 +} __attribute__((packed)); + +/* RX TPA End Completion Record (32 bytes split to 2 16-byte struct) */ +struct rx_tpa_end_cmpl { + uint16_t flags_type; + /* + * This field indicates the exact type of the completion. By + * convention, the LSB identifies the length of the record in + * 16B units. Even values indicate 16B records. Odd values + * indicate 32B records. + */ + #define RX_TPA_END_CMPL_TYPE_MASK UINT32_C(0x3f) + #define RX_TPA_END_CMPL_TYPE_SFT 0 + /* + * RX L2 TPA End Completion: Completion at the + * end of a TPA operation. Length = 32B + */ + #define RX_TPA_END_CMPL_TYPE_RX_TPA_END UINT32_C(0x15) + /* + * When this bit is '1', it indicates a packet that has an error + * of some type. Type of error is indicated in error_flags. + */ + #define RX_TPA_END_CMPL_FLAGS_ERROR UINT32_C(0x40) + /* This field indicates how the packet was placed in the buffer. */ + #define RX_TPA_END_CMPL_FLAGS_PLACEMENT_MASK UINT32_C(0x380) + #define RX_TPA_END_CMPL_FLAGS_PLACEMENT_SFT 7 + /* + * Jumbo: TPA Packet was placed using jumbo + * algorithm. This means that the first buffer + * will be filled with data before moving to + * aggregation buffers. Each aggregation buffer + * will be filled before moving to the next + * aggregation buffer. + */ + #define RX_TPA_END_CMPL_FLAGS_PLACEMENT_JUMBO (UINT32_C(0x1) << 7) + /* + * Header/Data Separation: Packet was placed + * using Header/Data separation algorithm. The + * separation location is indicated by the itype + * field. + */ + #define RX_TPA_END_CMPL_FLAGS_PLACEMENT_HDS (UINT32_C(0x2) << 7) + /* + * GRO/Jumbo: Packet will be placed using + * GRO/Jumbo where the first packet is filled + * with data. Subsequent packets will be placed + * such that any one packet does not span two + * aggregation buffers unless it starts at the + * beginning of an aggregation buffer. + */ + #define RX_TPA_END_CMPL_FLAGS_PLACEMENT_GRO_JUMBO (UINT32_C(0x5) << 7) + /* + * GRO/Header-Data Separation: Packet will be + * placed using GRO/HDS where the header is in + * the first packet. Payload of each packet will + * be placed such that any one packet does not + * span two aggregation buffers unless it starts + * at the beginning of an aggregation buffer. + */ + #define RX_TPA_END_CMPL_FLAGS_PLACEMENT_GRO_HDS (UINT32_C(0x6) << 7) + #define RX_TPA_END_CMPL_FLAGS_PLACEMENT_LAST \ + RX_TPA_END_CMPL_FLAGS_PLACEMENT_GRO_HDS + /* unused is 2 b */ + #define RX_TPA_END_CMPL_FLAGS_UNUSED_MASK UINT32_C(0xc00) + #define RX_TPA_END_CMPL_FLAGS_UNUSED_SFT 10 + /* + * This value indicates what the inner packet determined for the + * packet was. - 2 TCP Packet Indicates that the packet was IP + * and TCP. This indicates that the ip_cs field is valid and + * that the tcp_udp_cs field is valid and contains the TCP + * checksum. This also indicates that the payload_offset field + * is valid. + */ + #define RX_TPA_END_CMPL_FLAGS_ITYPE_MASK UINT32_C(0xf000) + #define RX_TPA_END_CMPL_FLAGS_ITYPE_SFT 12 + #define RX_TPA_END_CMPL_FLAGS_MASK UINT32_C(0xffc0) + #define RX_TPA_END_CMPL_FLAGS_SFT 6 + uint16_t len; + /* + * This value is zero for TPA End completions. There is no data + * in the buffer that corresponds to the opaque value in this + * completion. + */ + uint32_t opaque; + /* + * This is a copy of the opaque field from the RX BD this + * completion corresponds to. + */ + uint8_t agg_bufs_v1; + /* unused1 is 1 b */ + /* + * This value is written by the NIC such that it will be + * different for each pass through the completion queue. The + * even passes will write 1. The odd passes will write 0. + */ + #define RX_TPA_END_CMPL_V1 UINT32_C(0x1) + /* + * This value is the number of aggregation buffers that follow + * this entry in the completion ring that are a part of this + * aggregation packet. If the value is zero, then the packet is + * completely contained in the buffer space provided in the + * aggregation start completion. + */ + #define RX_TPA_END_CMPL_AGG_BUFS_MASK UINT32_C(0x7e) + #define RX_TPA_END_CMPL_AGG_BUFS_SFT 1 + /* unused1 is 1 b */ + uint8_t tpa_segs; + /* This value is the number of segments in the TPA operation. */ + uint8_t payload_offset; + /* + * This value indicates the offset in bytes from the beginning + * of the packet where the inner payload starts. This value is + * valid for TCP, UDP, FCoE, and RoCE packets. A value of zero + * indicates an offset of 256 bytes. + */ + uint8_t agg_id; + /* + * This is the aggregation ID that the completion is associated + * with. Use this number to correlate the TPA start completion + * with the TPA end completion. + */ + /* unused2 is 1 b */ + /* + * This is the aggregation ID that the completion is associated + * with. Use this number to correlate the TPA start completion + * with the TPA end completion. + */ + #define RX_TPA_END_CMPL_AGG_ID_MASK UINT32_C(0xfe) + #define RX_TPA_END_CMPL_AGG_ID_SFT 1 + uint32_t tsdelta; + /* + * For non-GRO packets, this value is the timestamp delta + * between earliest and latest timestamp values for TPA packet. + * If packets were not time stamped, then delta will be zero. + * For GRO packets, this field is zero except for the following + * sub-fields. - tsdelta[31] Timestamp present indication. When + * '0', no Timestamp option is in the packet. When '1', then a + * Timestamp option is present in the packet. + */ +} __attribute__((packed)); + +/* last 16 bytes of RX TPA End Completion Record */ +struct rx_tpa_end_cmpl_hi { + uint32_t tpa_dup_acks; + /* unused3 is 28 b */ + /* + * This value is the number of duplicate ACKs that have been + * received as part of the TPA operation. + */ + #define RX_TPA_END_CMPL_TPA_DUP_ACKS_MASK UINT32_C(0xf) + #define RX_TPA_END_CMPL_TPA_DUP_ACKS_SFT 0 + /* unused3 is 28 b */ + uint16_t tpa_seg_len; + /* + * This value is the valid when TPA completion is active. It + * indicates the length of the longest segment of the TPA + * operation for LRO mode and the length of the first segment in + * GRO mode. This value may be used by GRO software to re- + * construct the original packet stream from the TPA packet. + * This is the length of all but the last segment for GRO. In + * LRO mode this value may be used to indicate MSS size to the + * stack. + */ + uint16_t unused_3; + /* unused4 is 16 b */ + uint16_t errors_v2; + /* + * This value is written by the NIC such that it will be + * different for each pass through the completion queue. The + * even passes will write 1. The odd passes will write 0. + */ + #define RX_TPA_END_CMPL_V2 UINT32_C(0x1) + /* + * This error indicates that there was some sort of problem with + * the BDs for the packet that was found after part of the + * packet was already placed. The packet should be treated as + * invalid. + */ + #define RX_TPA_END_CMPL_ERRORS_BUFFER_ERROR_MASK UINT32_C(0xe) + #define RX_TPA_END_CMPL_ERRORS_BUFFER_ERROR_SFT 1 + /* + * This error occurs when there is a fatal HW + * problem in the chip only. It indicates that + * there were not BDs on chip but that there was + * adequate reservation. provided by the TPA + * block. + */ + #define RX_TPA_END_CMPL_ERRORS_BUFFER_ERROR_NOT_ON_CHIP \ + (UINT32_C(0x2) << 1) + /* + * This error occurs when TPA block was not + * configured to reserve adequate BDs for TPA + * operations on this RX ring. All data for the + * TPA operation was not placed. This error can + * also be generated when the number of segments + * is not programmed correctly in TPA and the 33 + * total aggregation buffers allowed for the TPA + * operation has been exceeded. + */ + #define RX_TPA_END_CMPL_ERRORS_BUFFER_ERROR_RSV_ERROR \ + (UINT32_C(0x4) << 1) + #define RX_TPA_END_CMPL_ERRORS_BUFFER_ERROR_LAST \ + RX_TPA_END_CMPL_ERRORS_BUFFER_ERROR_RSV_ERROR + #define RX_TPA_END_CMPL_ERRORS_MASK UINT32_C(0xfffe) + #define RX_TPA_END_CMPL_ERRORS_SFT 1 + uint16_t unused_4; + /* unused5 is 16 b */ + uint32_t start_opaque; + /* + * This is the opaque value that was completed for the TPA start + * completion that corresponds to this TPA end completion. + */ +} __attribute__((packed)); + +/* HWRM Forwarded Request (16 bytes) */ struct hwrm_fwd_req_cmpl { - /* Length of forwarded request in bytes. */ - /* - * This field indicates the exact type of the completion. By convention, - * the LSB identifies the length of the record in 16B units. Even values - * indicate 16B records. Odd values indicate 32B records. - */ - #define HWRM_FWD_REQ_CMPL_TYPE_MASK UINT32_C(0x3f) - #define HWRM_FWD_REQ_CMPL_TYPE_SFT 0 - /* Forwarded HWRM Request */ - #define HWRM_FWD_REQ_CMPL_TYPE_HWRM_FWD_REQ (UINT32_C(0x22) << 0) - /* Length of forwarded request in bytes. */ - #define HWRM_FWD_REQ_CMPL_REQ_LEN_MASK UINT32_C(0xffc0) - #define HWRM_FWD_REQ_CMPL_REQ_LEN_SFT 6 uint16_t req_len_type; - + /* Length of forwarded request in bytes. */ /* - * Source ID of this request. Typically used in forwarding requests and - * responses. 0x0 - 0xFFF8 - Used for function ids 0xFFF8 - 0xFFFE - - * Reserved for internal processors 0xFFFF - HWRM + * This field indicates the exact type of the completion. By + * convention, the LSB identifies the length of the record in + * 16B units. Even values indicate 16B records. Odd values + * indicate 32B records. */ + #define HWRM_FWD_INPUT_CMPL_TYPE_MASK UINT32_C(0x3f) + #define HWRM_FWD_INPUT_CMPL_TYPE_SFT 0 + /* Forwarded HWRM Request */ + #define HWRM_FWD_INPUT_CMPL_TYPE_HWRM_FWD_INPUT UINT32_C(0x22) + /* Length of forwarded request in bytes. */ + #define HWRM_FWD_REQ_CMPL_REQ_LEN_MASK UINT32_C(0xffc0) + #define HWRM_FWD_REQ_CMPL_REQ_LEN_SFT 6 uint16_t source_id; - + /* + * Source ID of this request. Typically used in forwarding + * requests and responses. 0x0 - 0xFFF8 - Used for function ids + * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - + * HWRM + */ uint32_t unused_0; - + /* unused1 is 32 b */ + uint32_t req_buf_addr_v[2]; /* Address of forwarded request. */ /* - * This value is written by the NIC such that it will be different for - * each pass through the completion queue. The even passes will write 1. - * The odd passes will write 0. + * This value is written by the NIC such that it will be + * different for each pass through the completion queue. The + * even passes will write 1. The odd passes will write 0. */ - #define HWRM_FWD_REQ_CMPL_V UINT32_C(0x1) + #define HWRM_FWD_INPUT_CMPL_V UINT32_C(0x1) /* Address of forwarded request. */ #define HWRM_FWD_REQ_CMPL_REQ_BUF_ADDR_MASK UINT32_C(0xfffffffe) #define HWRM_FWD_REQ_CMPL_REQ_BUF_ADDR_SFT 1 - uint64_t req_buf_addr_v; } __attribute__((packed)); -/* HWRM Asynchronous Event Completion Record (16 bytes) */ +/* HWRM Asynchronous Event Completion Record (16 bytes) */ struct hwrm_async_event_cmpl { - /* - * This field indicates the exact type of the completion. By convention, - * the LSB identifies the length of the record in 16B units. Even values - * indicate 16B records. Odd values indicate 32B records. - */ - #define HWRM_ASYNC_EVENT_CMPL_TYPE_MASK UINT32_C(0x3f) - #define HWRM_ASYNC_EVENT_CMPL_TYPE_SFT 0 - /* HWRM Asynchronous Event Information */ - #define HWRM_ASYNC_EVENT_CMPL_TYPE_HWRM_ASYNC_EVENT \ - (UINT32_C(0x2e) << 0) uint16_t type; - - /* Identifiers of events. */ - /* Link status changed */ - #define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE \ - (UINT32_C(0x0) << 0) - /* Link MTU changed */ - #define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_MTU_CHANGE \ - (UINT32_C(0x1) << 0) - /* Link speed changed */ - #define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE \ - (UINT32_C(0x2) << 0) - /* DCB Configuration changed */ - #define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DCB_CONFIG_CHANGE \ - (UINT32_C(0x3) << 0) - /* Port connection not allowed */ - #define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED \ - (UINT32_C(0x4) << 0) - /* Link speed configuration was not allowed */ - #define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_NOT_ALLOWED \ - (UINT32_C(0x5) << 0) - /* Function driver unloaded */ - #define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_FUNC_DRVR_UNLOAD \ - (UINT32_C(0x10) << 0) - /* Function driver loaded */ - #define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_FUNC_DRVR_LOAD \ - (UINT32_C(0x11) << 0) - /* PF driver unloaded */ - #define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD \ - (UINT32_C(0x20) << 0) - /* PF driver loaded */ - #define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_LOAD \ - (UINT32_C(0x21) << 0) - /* VF Function Level Reset (FLR) */ - #define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_VF_FLR (UINT32_C(0x30) << 0) - /* VF MAC Address Change */ - #define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_VF_MAC_ADDR_CHANGE \ - (UINT32_C(0x31) << 0) - /* PF-VF communication channel status change. */ - #define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PF_VF_COMM_STATUS_CHANGE \ - (UINT32_C(0x32) << 0) - /* HWRM Error */ - #define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_HWRM_ERROR \ - (UINT32_C(0xff) << 0) + /* unused1 is 10 b */ + /* + * This field indicates the exact type of the completion. By + * convention, the LSB identifies the length of the record in + * 16B units. Even values indicate 16B records. Odd values + * indicate 32B records. + */ + #define HWRM_ASYNC_EVENT_CMPL_TYPE_MASK UINT32_C(0x3f) + #define HWRM_ASYNC_EVENT_CMPL_TYPE_SFT 0 + /* HWRM Asynchronous Event Information */ + #define HWRM_ASYNC_EVENT_CMPL_TYPE_HWRM_ASYNC_EVENT UINT32_C(0x2e) + /* unused1 is 10 b */ uint16_t event_id; - - /* Event specific data */ + /* Identifiers of events. */ + /* Link status changed */ + #define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE UINT32_C(0x0) + /* Link MTU changed */ + #define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_MTU_CHANGE UINT32_C(0x1) + /* Link speed changed */ + #define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE UINT32_C(0x2) + /* DCB Configuration changed */ + #define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DCB_CONFIG_CHANGE UINT32_C(0x3) + /* Port connection not allowed */ + #define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED UINT32_C(0x4) + /* Link speed configuration was not allowed */ + #define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_NOT_ALLOWED \ + UINT32_C(0x5) + /* Link speed configuration change */ + #define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE UINT32_C(0x6) + /* Port PHY configuration change */ + #define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE UINT32_C(0x7) + /* Function driver unloaded */ + #define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_FUNC_DRVR_UNLOAD UINT32_C(0x10) + /* Function driver loaded */ + #define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_FUNC_DRVR_LOAD UINT32_C(0x11) + /* Function FLR related processing has completed */ + #define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_FUNC_FLR_PROC_CMPLT UINT32_C(0x12) + /* PF driver unloaded */ + #define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD UINT32_C(0x20) + /* PF driver loaded */ + #define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_LOAD UINT32_C(0x21) + /* VF Function Level Reset (FLR) */ + #define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_VF_FLR UINT32_C(0x30) + /* VF MAC Address Change */ + #define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_VF_MAC_ADDR_CHANGE UINT32_C(0x31) + /* PF-VF communication channel status change. */ + #define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PF_VF_COMM_STATUS_CHANGE \ + UINT32_C(0x32) + /* VF Configuration Change */ + #define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE UINT32_C(0x33) + /* LLFC/PFC Configuration Change */ + #define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LLFC_PFC_CHANGE UINT32_C(0x34) + /* HWRM Error */ + #define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_HWRM_ERROR UINT32_C(0xff) uint32_t event_data2; - - /* opaque is 7 b */ - /* - * This value is written by the NIC such that it will be different for - * each pass through the completion queue. The even passes will write 1. - * The odd passes will write 0. - */ - #define HWRM_ASYNC_EVENT_CMPL_V UINT32_C(0x1) - /* opaque is 7 b */ - #define HWRM_ASYNC_EVENT_CMPL_OPAQUE_MASK UINT32_C(0xfe) - #define HWRM_ASYNC_EVENT_CMPL_OPAQUE_SFT 1 - uint8_t opaque_v; - - /* 8-lsb timestamp from POR (100-msec resolution) */ - uint8_t timestamp_lo; - - /* 16-lsb timestamp from POR (100-msec resolution) */ - uint16_t timestamp_hi; - /* Event specific data */ + uint8_t opaque_v; + /* opaque is 7 b */ + /* + * This value is written by the NIC such that it will be + * different for each pass through the completion queue. The + * even passes will write 1. The odd passes will write 0. + */ + #define HWRM_ASYNC_EVENT_CMPL_V UINT32_C(0x1) + /* opaque is 7 b */ + #define HWRM_ASYNC_EVENT_CMPL_OPAQUE_MASK UINT32_C(0xfe) + #define HWRM_ASYNC_EVENT_CMPL_OPAQUE_SFT 1 + uint8_t timestamp_lo; + /* 8-lsb timestamp from POR (100-msec resolution) */ + uint16_t timestamp_hi; + /* 16-lsb timestamp from POR (100-msec resolution) */ uint32_t event_data1; -} __attribute__((packed)); - -/* - * Note: The Hardware Resource Manager (HWRM) manages various hardware resources - * inside the chip. The HWRM is implemented in firmware, and runs on embedded - * processors inside the chip. This firmware is vital part of the chip's - * hardware. The chip can not be used by driver without it. - */ - -/* Input (16 bytes) */ -struct input { - /* - * This value indicates what type of request this is. The format for the - * rest of the command is determined by this field. - */ - uint16_t req_type; - - /* - * This value indicates the what completion ring the request will be - * optionally completed on. If the value is -1, then no CR completion - * will be generated. Any other value must be a valid CR ring_id value - * for this function. - */ - uint16_t cmpl_ring; - - /* This value indicates the command sequence number. */ - uint16_t seq_id; - - /* - * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids - * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM - */ - uint16_t target_id; - - /* - * This is the host address where the response will be written when the - * request is complete. This area must be 16B aligned and must be - * cleared to zero before the request is made. - */ - uint64_t resp_addr; -} __attribute__((packed)); - -/* Output (8 bytes) */ -struct output { - /* - * Pass/Fail or error type Note: receiver to verify the in parameters, - * and fail the call with an error when appropriate - */ - uint16_t error_code; - - /* This field returns the type of original request. */ - uint16_t req_type; - - /* This field provides original sequence number of the command. */ - uint16_t seq_id; - - /* - * This field is the length of the response in bytes. The last byte of - * the response is a valid flag that will read as '1' when the command - * has been completely written to memory. - */ - uint16_t resp_len; -} __attribute__((packed)); - -/* hwrm_cfa_l2_filter_alloc */ -/* - * A filter is used to identify traffic that contains a matching set of - * parameters like unicast or broadcast MAC address or a VLAN tag amongst - * other things which then allows the ASIC to direct the incoming traffic - * to an appropriate VNIC or Rx ring. - */ - -/* Input (96 bytes) */ -struct hwrm_cfa_l2_filter_alloc_input { - /* - * This value indicates what type of request this is. The format for the - * rest of the command is determined by this field. - */ - uint16_t req_type; - - /* - * This value indicates the what completion ring the request will be - * optionally completed on. If the value is -1, then no CR completion - * will be generated. Any other value must be a valid CR ring_id value - * for this function. - */ - uint16_t cmpl_ring; - - /* This value indicates the command sequence number. */ - uint16_t seq_id; - - /* - * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids - * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM - */ - uint16_t target_id; - - /* - * This is the host address where the response will be written when the - * request is complete. This area must be 16B aligned and must be - * cleared to zero before the request is made. - */ - uint64_t resp_addr; - - /* - * Enumeration denoting the RX, TX type of the resource. This - * enumeration is used for resources that are similar for both TX and RX - * paths of the chip. - */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_PATH \ - UINT32_C(0x1) - /* tx path */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_PATH_TX \ - (UINT32_C(0x0) << 0) - /* rx path */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_PATH_RX \ - (UINT32_C(0x1) << 0) - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_PATH_LAST \ - HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_PATH_RX - /* - * Setting of this flag indicates the applicability to the loopback - * path. - */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_LOOPBACK \ - UINT32_C(0x2) - /* - * Setting of this flag indicates drop action. If this flag is not set, - * then it should be considered accept action. - */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_DROP \ - UINT32_C(0x4) - /* - * If this flag is set, all t_l2_* fields are invalid and they should - * not be specified. If this flag is set, then l2_* fields refer to - * fields of outermost L2 header. - */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_OUTERMOST \ - UINT32_C(0x8) - uint32_t flags; - - /* This bit must be '1' for the l2_addr field to be configured. */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_ADDR \ - UINT32_C(0x1) - /* This bit must be '1' for the l2_addr_mask field to be configured. */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_ADDR_MASK \ - UINT32_C(0x2) - /* This bit must be '1' for the l2_ovlan field to be configured. */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_OVLAN \ - UINT32_C(0x4) - /* This bit must be '1' for the l2_ovlan_mask field to be configured. */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_OVLAN_MASK \ - UINT32_C(0x8) - /* This bit must be '1' for the l2_ivlan field to be configured. */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_IVLAN \ - UINT32_C(0x10) - /* This bit must be '1' for the l2_ivlan_mask field to be configured. */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_IVLAN_MASK \ - UINT32_C(0x20) - /* This bit must be '1' for the t_l2_addr field to be configured. */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_T_L2_ADDR \ - UINT32_C(0x40) - /* - * This bit must be '1' for the t_l2_addr_mask field to be configured. - */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_T_L2_ADDR_MASK \ - UINT32_C(0x80) - /* This bit must be '1' for the t_l2_ovlan field to be configured. */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_T_L2_OVLAN \ - UINT32_C(0x100) - /* - * This bit must be '1' for the t_l2_ovlan_mask field to be configured. - */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_T_L2_OVLAN_MASK \ - UINT32_C(0x200) - /* This bit must be '1' for the t_l2_ivlan field to be configured. */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_T_L2_IVLAN \ - UINT32_C(0x400) - /* - * This bit must be '1' for the t_l2_ivlan_mask field to be configured. - */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_T_L2_IVLAN_MASK \ - UINT32_C(0x800) - /* This bit must be '1' for the src_type field to be configured. */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_SRC_TYPE \ - UINT32_C(0x1000) - /* This bit must be '1' for the src_id field to be configured. */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_SRC_ID \ - UINT32_C(0x2000) - /* This bit must be '1' for the tunnel_type field to be configured. */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_TUNNEL_TYPE \ - UINT32_C(0x4000) - /* This bit must be '1' for the dst_id field to be configured. */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_DST_ID \ - UINT32_C(0x8000) - /* - * This bit must be '1' for the mirror_vnic_id field to be configured. - */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_MIRROR_VNIC_ID \ - UINT32_C(0x10000) - uint32_t enables; - - /* - * This value sets the match value for the L2 MAC address. Destination - * MAC address for RX path. Source MAC address for TX path. - */ - uint8_t l2_addr[6]; - - uint8_t unused_0; - uint8_t unused_1; - - /* - * This value sets the mask value for the L2 address. A value of 0 will - * mask the corresponding bit from compare. - */ - uint8_t l2_addr_mask[6]; - - /* This value sets VLAN ID value for outer VLAN. */ - uint16_t l2_ovlan; - - /* - * This value sets the mask value for the ovlan id. A value of 0 will - * mask the corresponding bit from compare. - */ - uint16_t l2_ovlan_mask; - - /* This value sets VLAN ID value for inner VLAN. */ - uint16_t l2_ivlan; - - /* - * This value sets the mask value for the ivlan id. A value of 0 will - * mask the corresponding bit from compare. - */ - uint16_t l2_ivlan_mask; - - uint8_t unused_2; - uint8_t unused_3; - - /* - * This value sets the match value for the tunnel L2 MAC address. - * Destination MAC address for RX path. Source MAC address for TX path. - */ - uint8_t t_l2_addr[6]; - - uint8_t unused_4; - uint8_t unused_5; - - /* - * This value sets the mask value for the tunnel L2 address. A value of - * 0 will mask the corresponding bit from compare. - */ - uint8_t t_l2_addr_mask[6]; - - /* This value sets VLAN ID value for tunnel outer VLAN. */ - uint16_t t_l2_ovlan; - - /* - * This value sets the mask value for the tunnel ovlan id. A value of 0 - * will mask the corresponding bit from compare. - */ - uint16_t t_l2_ovlan_mask; - - /* This value sets VLAN ID value for tunnel inner VLAN. */ - uint16_t t_l2_ivlan; - - /* - * This value sets the mask value for the tunnel ivlan id. A value of 0 - * will mask the corresponding bit from compare. - */ - uint16_t t_l2_ivlan_mask; - - /* This value identifies the type of source of the packet. */ - /* Network port */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_SRC_TYPE_NPORT \ - (UINT32_C(0x0) << 0) - /* Physical function */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_SRC_TYPE_PF \ - (UINT32_C(0x1) << 0) - /* Virtual function */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_SRC_TYPE_VF \ - (UINT32_C(0x2) << 0) - /* Virtual NIC of a function */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_SRC_TYPE_VNIC \ - (UINT32_C(0x3) << 0) - /* Embedded processor for CFA management */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_SRC_TYPE_KONG \ - (UINT32_C(0x4) << 0) - /* Embedded processor for OOB management */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_SRC_TYPE_APE \ - (UINT32_C(0x5) << 0) - /* Embedded processor for RoCE */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_SRC_TYPE_BONO \ - (UINT32_C(0x6) << 0) - /* Embedded processor for network proxy functions */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_SRC_TYPE_TANG \ - (UINT32_C(0x7) << 0) - uint8_t src_type; - - uint8_t unused_6; - /* - * This value is the id of the source. For a network port, it represents - * port_id. For a physical function, it represents fid. For a virtual - * function, it represents vf_id. For a vnic, it represents vnic_id. For - * embedded processors, this id is not valid. Notes: 1. The function ID - * is implied if it src_id is not provided for a src_type that is either - */ - uint32_t src_id; - - /* Tunnel Type. */ - /* Non-tunnel */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_TUNNEL_TYPE_NONTUNNEL \ - (UINT32_C(0x0) << 0) - /* Virtual eXtensible Local Area Network (VXLAN) */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_TUNNEL_TYPE_VXLAN \ - (UINT32_C(0x1) << 0) - /* - * Network Virtualization Generic Routing Encapsulation (NVGRE) - */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_TUNNEL_TYPE_NVGRE \ - (UINT32_C(0x2) << 0) - /* - * Generic Routing Encapsulation (GRE) inside Ethernet payload - */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_TUNNEL_TYPE_L2GRE \ - (UINT32_C(0x3) << 0) - /* IP in IP */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_TUNNEL_TYPE_IPIP \ - (UINT32_C(0x4) << 0) - /* Generic Network Virtualization Encapsulation (Geneve) */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_TUNNEL_TYPE_GENEVE \ - (UINT32_C(0x5) << 0) - /* Multi-Protocol Lable Switching (MPLS) */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_TUNNEL_TYPE_MPLS \ - (UINT32_C(0x6) << 0) - /* Stateless Transport Tunnel (STT) */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_TUNNEL_TYPE_STT \ - (UINT32_C(0x7) << 0) - /* - * Generic Routing Encapsulation (GRE) inside IP datagram - * payload - */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_TUNNEL_TYPE_IPGRE \ - (UINT32_C(0x8) << 0) - /* Any tunneled traffic */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_TUNNEL_TYPE_ANYTUNNEL \ - (UINT32_C(0xff) << 0) - uint8_t tunnel_type; - - uint8_t unused_7; - - /* - * If set, this value shall represent the Logical VNIC ID of the - * destination VNIC for the RX path and network port id of the - * destination port for the TX path. - */ - uint16_t dst_id; - - /* Logical VNIC ID of the VNIC where traffic is mirrored. */ - uint16_t mirror_vnic_id; - - /* - * This hint is provided to help in placing the filter in the filter - * table. - */ - /* No preference */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_PRI_HINT_NO_PREFER \ - (UINT32_C(0x0) << 0) - /* Above the given filter */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_PRI_HINT_ABOVE_FILTER \ - (UINT32_C(0x1) << 0) - /* Below the given filter */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_PRI_HINT_BELOW_FILTER \ - (UINT32_C(0x2) << 0) - /* As high as possible */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_PRI_HINT_MAX \ - (UINT32_C(0x3) << 0) - /* As low as possible */ - #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_PRI_HINT_MIN \ - (UINT32_C(0x4) << 0) - uint8_t pri_hint; - - uint8_t unused_8; - uint32_t unused_9; - - /* - * This is the ID of the filter that goes along with the pri_hint. This - * field is valid only for the following values. 1 - Above the given - * filter 2 - Below the given filter - */ - uint64_t l2_filter_id_hint; -} __attribute__((packed)); - -/* Output (24 bytes) */ -struct hwrm_cfa_l2_filter_alloc_output { - /* - * Pass/Fail or error type Note: receiver to verify the in parameters, - * and fail the call with an error when appropriate - */ - uint16_t error_code; - - /* This field returns the type of original request. */ - uint16_t req_type; - - /* This field provides original sequence number of the command. */ - uint16_t seq_id; - - /* - * This field is the length of the response in bytes. The last byte of - * the response is a valid flag that will read as '1' when the command - * has been completely written to memory. - */ - uint16_t resp_len; - - /* - * This value identifies a set of CFA data structures used for an L2 - * context. - */ - uint64_t l2_filter_id; - - /* - * This is the ID of the flow associated with this filter. This value - * shall be used to match and associate the flow identifier returned in - * completion records. A value of 0xFFFFFFFF shall indicate no flow id. - */ - uint32_t flow_id; - - uint8_t unused_0; - uint8_t unused_1; - uint8_t unused_2; - - /* - * This field is used in Output records to indicate that the output is - * completely written to RAM. This field should be read as '1' to - * indicate that the output has been completely written. When writing a - * command completion or response to an internal processor, the order of - * writes has to be such that this field is written last. - */ - uint8_t valid; -} __attribute__((packed)); - -/* hwrm_cfa_l2_filter_free */ -/* - * Description: Free a L2 filter. The HWRM shall free all associated filter - * resources with the L2 filter. - */ - -/* Input (24 bytes) */ -struct hwrm_cfa_l2_filter_free_input { - /* - * This value indicates what type of request this is. The format for the - * rest of the command is determined by this field. - */ - uint16_t req_type; - - /* - * This value indicates the what completion ring the request will be - * optionally completed on. If the value is -1, then no CR completion - * will be generated. Any other value must be a valid CR ring_id value - * for this function. - */ - uint16_t cmpl_ring; - - /* This value indicates the command sequence number. */ - uint16_t seq_id; - - /* - * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids - * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM - */ - uint16_t target_id; - - /* - * This is the host address where the response will be written when the - * request is complete. This area must be 16B aligned and must be - * cleared to zero before the request is made. - */ - uint64_t resp_addr; - - /* - * This value identifies a set of CFA data structures used for an L2 - * context. - */ - uint64_t l2_filter_id; -} __attribute__((packed)); - -/* Output (16 bytes) */ -struct hwrm_cfa_l2_filter_free_output { - /* - * Pass/Fail or error type Note: receiver to verify the in parameters, - * and fail the call with an error when appropriate - */ - uint16_t error_code; - - /* This field returns the type of original request. */ - uint16_t req_type; - - /* This field provides original sequence number of the command. */ - uint16_t seq_id; - - /* - * This field is the length of the response in bytes. The last byte of - * the response is a valid flag that will read as '1' when the command - * has been completely written to memory. - */ - uint16_t resp_len; - - uint32_t unused_0; - uint8_t unused_1; - uint8_t unused_2; - uint8_t unused_3; - - /* - * This field is used in Output records to indicate that the output is - * completely written to RAM. This field should be read as '1' to - * indicate that the output has been completely written. When writing a - * command completion or response to an internal processor, the order of - * writes has to be such that this field is written last. - */ - uint8_t valid; -} __attribute__((packed)); - -/* hwrm_cfa_l2_set_rx_mask */ -/* Description: This command will set rx mask of the function. */ - -/* Input (40 bytes) */ -struct hwrm_cfa_l2_set_rx_mask_input { - /* - * This value indicates what type of request this is. The format for the - * rest of the command is determined by this field. - */ - uint16_t req_type; - - /* - * This value indicates the what completion ring the request will be - * optionally completed on. If the value is -1, then no CR completion - * will be generated. Any other value must be a valid CR ring_id value - * for this function. - */ - uint16_t cmpl_ring; - - /* This value indicates the command sequence number. */ - uint16_t seq_id; - - /* - * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids - * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM - */ - uint16_t target_id; - - /* - * This is the host address where the response will be written when the - * request is complete. This area must be 16B aligned and must be - * cleared to zero before the request is made. - */ - uint64_t resp_addr; - - /* VNIC ID */ - uint32_t vnic_id; - - /* Reserved for future use. */ - #define HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_RESERVED UINT32_C(0x1) - /* - * When this bit is '1', the function is requested to accept multi-cast - * packets specified by the multicast addr table. - */ - #define HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_MCAST UINT32_C(0x2) - /* - * When this bit is '1', the function is requested to accept all multi- - * cast packets. - */ - #define HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_ALL_MCAST UINT32_C(0x4) - /* - * When this bit is '1', the function is requested to accept broadcast - * packets. - */ - #define HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_BCAST UINT32_C(0x8) - /* - * When this bit is '1', the function is requested to be put in the - * promiscuous mode. The HWRM should accept any function to set up - * promiscuous mode. The HWRM shall follow the semantics below for the - * promiscuous mode support. # When partitioning is not enabled on a - * port (i.e. single PF on the port), then the PF shall be allowed to be - * in the promiscuous mode. When the PF is in the promiscuous mode, then - * it shall receive all host bound traffic on that port. # When - * partitioning is enabled on a port (i.e. multiple PFs per port) and a - * PF on that port is in the promiscuous mode, then the PF receives all - * traffic within that partition as identified by a unique identifier - * for the PF (e.g. S-Tag). If a unique outer VLAN for the PF is - * specified, then the setting of promiscuous mode on that PF shall - * result in the PF receiving all host bound traffic with matching outer - * VLAN. # A VF shall can be set in the promiscuous mode. In the - * promiscuous mode, the VF does not receive any traffic unless a unique - * outer VLAN for the VF is specified. If a unique outer VLAN for the VF - * is specified, then the setting of promiscuous mode on that VF shall - * result in the VF receiving all host bound traffic with the matching - * outer VLAN. # The HWRM shall allow the setting of promiscuous mode on - * a function independently from the promiscuous mode settings on other - * functions. - */ - #define HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_PROMISCUOUS UINT32_C(0x10) - /* - * If this flag is set, the corresponding RX filters shall be set up to - * cover multicast/broadcast filters for the outermost Layer 2 - * destination MAC address field. - */ - #define HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_OUTERMOST UINT32_C(0x20) - uint32_t mask; - - /* This is the address for mcast address tbl. */ - uint64_t mc_tbl_addr; - - /* - * This value indicates how many entries in mc_tbl are valid. Each entry - * is 6 bytes. - */ - uint32_t num_mc_entries; - - uint32_t unused_0; -} __attribute__((packed)); - -/* Output (16 bytes) */ -struct hwrm_cfa_l2_set_rx_mask_output { - /* - * Pass/Fail or error type Note: receiver to verify the in parameters, - * and fail the call with an error when appropriate - */ - uint16_t error_code; - - /* This field returns the type of original request. */ - uint16_t req_type; - - /* This field provides original sequence number of the command. */ - uint16_t seq_id; - - /* - * This field is the length of the response in bytes. The last byte of - * the response is a valid flag that will read as '1' when the command - * has been completely written to memory. - */ - uint16_t resp_len; - - uint32_t unused_0; - uint8_t unused_1; - uint8_t unused_2; - uint8_t unused_3; - - /* - * This field is used in Output records to indicate that the output is - * completely written to RAM. This field should be read as '1' to - * indicate that the output has been completely written. When writing a - * command completion or response to an internal processor, the order of - * writes has to be such that this field is written last. - */ - uint8_t valid; -} __attribute__((packed)); - -/* hwrm_exec_fwd_resp */ -/* - * Description: This command is used to send an encapsulated request to the - * HWRM. This command instructs the HWRM to execute the request and forward the - * response of the encapsulated request to the location specified in the - * original request that is encapsulated. The target id of this command shall be - * set to 0xFFFF (HWRM). The response location in this command shall be used to - * acknowledge the receipt of the encapsulated request and forwarding of the - * response. - */ - -/* Input (128 bytes) */ -struct hwrm_exec_fwd_resp_input { - /* - * This value indicates what type of request this is. The format for the - * rest of the command is determined by this field. - */ - uint16_t req_type; - - /* - * This value indicates the what completion ring the request will be - * optionally completed on. If the value is -1, then no CR completion - * will be generated. Any other value must be a valid CR ring_id value - * for this function. - */ - uint16_t cmpl_ring; - - /* This value indicates the command sequence number. */ - uint16_t seq_id; - - /* - * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids - * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM - */ - uint16_t target_id; - - /* - * This is the host address where the response will be written when the - * request is complete. This area must be 16B aligned and must be - * cleared to zero before the request is made. - */ - uint64_t resp_addr; - - /* - * This is an encapsulated request. This request should be executed by - * the HWRM and the response should be provided in the response buffer - * inside the encapsulated request. - */ - uint32_t encap_request[26]; - - /* - * This value indicates the target id of the response to the - * encapsulated request. 0x0 - 0xFFF8 - Used for function ids 0xFFF8 - - * 0xFFFE - Reserved for internal processors 0xFFFF - HWRM - */ - uint16_t encap_resp_target_id; - - uint16_t unused_0[3]; -} __attribute__((packed)); - -/* Output (16 bytes) */ -struct hwrm_exec_fwd_resp_output { - /* - * Pass/Fail or error type Note: receiver to verify the in parameters, - * and fail the call with an error when appropriate - */ - uint16_t error_code; - - /* This field returns the type of original request. */ - uint16_t req_type; - - /* This field provides original sequence number of the command. */ - uint16_t seq_id; - - /* - * This field is the length of the response in bytes. The last byte of - * the response is a valid flag that will read as '1' when the command - * has been completely written to memory. - */ - uint16_t resp_len; - - uint32_t unused_0; - uint8_t unused_1; - uint8_t unused_2; - uint8_t unused_3; - - /* - * This field is used in Output records to indicate that the output is - * completely written to RAM. This field should be read as '1' to - * indicate that the output has been completely written. When writing a - * command completion or response to an internal processor, the order of - * writes has to be such that this field is written last. - */ - uint8_t valid; -} __attribute__((packed)); - -/* hwrm_func_qcaps */ -/* - * Description: This command returns capabilities of a function. The input FID - * value is used to indicate what function is being queried. This allows a - * physical function driver to query virtual functions that are children of the - * physical function. The output FID value is needed to configure Rings and - * MSI-X vectors so their DMA operations appear correctly on the PCI bus. - */ - -/* Input (24 bytes) */ -struct hwrm_func_qcaps_input { - /* - * This value indicates what type of request this is. The format for the - * rest of the command is determined by this field. - */ - uint16_t req_type; - - /* - * This value indicates the what completion ring the request will be - * optionally completed on. If the value is -1, then no CR completion - * will be generated. Any other value must be a valid CR ring_id value - * for this function. - */ - uint16_t cmpl_ring; - - /* This value indicates the command sequence number. */ - uint16_t seq_id; - - /* - * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids - * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM - */ - uint16_t target_id; - - /* - * This is the host address where the response will be written when the - * request is complete. This area must be 16B aligned and must be - * cleared to zero before the request is made. - */ - uint64_t resp_addr; - - /* - * Function ID of the function that is being queried. 0xFF... (All Fs) - * if the query is for the requesting function. - */ - uint16_t fid; - - uint16_t unused_0[3]; -} __attribute__((packed)); - -/* Output (80 bytes) */ -struct hwrm_func_qcaps_output { - /* - * Pass/Fail or error type Note: receiver to verify the in parameters, - * and fail the call with an error when appropriate - */ - uint16_t error_code; - - /* This field returns the type of original request. */ - uint16_t req_type; - - /* This field provides original sequence number of the command. */ - uint16_t seq_id; - - /* - * This field is the length of the response in bytes. The last byte of - * the response is a valid flag that will read as '1' when the command - * has been completely written to memory. - */ - uint16_t resp_len; - - /* - * FID value. This value is used to identify operations on the PCI bus - * as belonging to a particular PCI function. - */ - uint16_t fid; - - /* - * Port ID of port that this function is associated with. Valid only for - * the PF. 0xFF... (All Fs) if this function is not associated with any - * port. 0xFF... (All Fs) if this function is called from a VF. - */ - uint16_t port_id; - - /* If 1, then Push mode is supported on this function. */ - #define HWRM_FUNC_QCAPS_OUTPUT_FLAGS_PUSH_MODE_SUPPORTED UINT32_C(0x1) - /* - * If 1, then the global MSI-X auto-masking is enabled for the device. - */ - #define HWRM_FUNC_QCAPS_OUTPUT_FLAGS_GLOBAL_MSIX_AUTOMASKING \ - UINT32_C(0x2) - /* - * If 1, then the Precision Time Protocol (PTP) processing is supported - * on this function. The HWRM should enable PTP on only a single - * Physical Function (PF) per port. - */ - #define HWRM_FUNC_QCAPS_OUTPUT_FLAGS_PTP_SUPPORTED UINT32_C(0x4) - uint32_t flags; - - /* - * This value is current MAC address configured for this function. A - * value of 00-00-00-00-00-00 indicates no MAC address is currently - * configured. - */ - uint8_t perm_mac_address[6]; - - /* - * The maximum number of RSS/COS contexts that can be allocated to the - * function. - */ - uint16_t max_rsscos_ctx; - - /* - * The maximum number of completion rings that can be allocated to the - * function. - */ - uint16_t max_cmpl_rings; - - /* - * The maximum number of transmit rings that can be allocated to the - * function. - */ - uint16_t max_tx_rings; - - /* - * The maximum number of receive rings that can be allocated to the - * function. - */ - uint16_t max_rx_rings; - - /* - * The maximum number of L2 contexts that can be allocated to the - * function. - */ - uint16_t max_l2_ctxs; - - /* The maximum number of VNICs that can be allocated to the function. */ - uint16_t max_vnics; - - /* - * The identifier for the first VF enabled on a PF. This is valid only - * on the PF with SR-IOV enabled. 0xFF... (All Fs) if this command is - * called on a PF with SR-IOV disabled or on a VF. - */ - uint16_t first_vf_id; - - /* - * The maximum number of VFs that can be allocated to the function. This - * is valid only on the PF with SR-IOV enabled. 0xFF... (All Fs) if this - * command is called on a PF with SR-IOV disabled or on a VF. - */ - uint16_t max_vfs; - - /* - * The maximum number of statistic contexts that can be allocated to the - * function. - */ - uint16_t max_stat_ctx; - - /* - * The maximum number of Encapsulation records that can be offloaded by - * this function. - */ - uint32_t max_encap_records; - - /* - * The maximum number of decapsulation records that can be offloaded by - * this function. - */ - uint32_t max_decap_records; - - /* - * The maximum number of Exact Match (EM) flows that can be offloaded by - * this function on the TX side. - */ - uint32_t max_tx_em_flows; - - /* - * The maximum number of Wildcard Match (WM) flows that can be offloaded - * by this function on the TX side. - */ - uint32_t max_tx_wm_flows; - - /* - * The maximum number of Exact Match (EM) flows that can be offloaded by - * this function on the RX side. - */ - uint32_t max_rx_em_flows; - - /* - * The maximum number of Wildcard Match (WM) flows that can be offloaded - * by this function on the RX side. - */ - uint32_t max_rx_wm_flows; - - /* - * The maximum number of multicast filters that can be supported by this - * function on the RX side. - */ - uint32_t max_mcast_filters; - - /* - * The maximum value of flow_id that can be supported in completion - * records. - */ - uint32_t max_flow_id; - - /* - * The maximum number of HW ring groups that can be supported on this - * function. - */ - uint32_t max_hw_ring_grps; - - uint8_t unused_0; - uint8_t unused_1; - uint8_t unused_2; - - /* - * This field is used in Output records to indicate that the output is - * completely written to RAM. This field should be read as '1' to - * indicate that the output has been completely written. When writing a - * command completion or response to an internal processor, the order of - * writes has to be such that this field is written last. - */ - uint8_t valid; -} __attribute__((packed)); - -/* hwrm_func_reset */ -/* - * Description: This command resets a hardware function (PCIe function) and - * frees any resources used by the function. This command shall be initiated by - * the driver after an FLR has occurred to prepare the function for re-use. This - * command may also be initiated by a driver prior to doing it's own - * configuration. This command puts the function into the reset state. In the - * reset state, global and port related features of the chip are not available. - */ -/* - * Note: This command will reset a function that has already been disabled or - * idled. The command returns all the resources owned by the function so a new - * driver may allocate and configure resources normally. - */ - -/* Input (24 bytes) */ -struct hwrm_func_reset_input { - /* - * This value indicates what type of request this is. The format for the - * rest of the command is determined by this field. - */ - uint16_t req_type; - - /* - * This value indicates the what completion ring the request will be - * optionally completed on. If the value is -1, then no CR completion - * will be generated. Any other value must be a valid CR ring_id value - * for this function. - */ - uint16_t cmpl_ring; - - /* This value indicates the command sequence number. */ - uint16_t seq_id; - - /* - * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids - * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM - */ - uint16_t target_id; - - /* - * This is the host address where the response will be written when the - * request is complete. This area must be 16B aligned and must be - * cleared to zero before the request is made. - */ - uint64_t resp_addr; - - /* This bit must be '1' for the vf_id_valid field to be configured. */ - #define HWRM_FUNC_RESET_INPUT_ENABLES_VF_ID_VALID \ - UINT32_C(0x1) - uint32_t enables; - - /* - * The ID of the VF that this PF is trying to reset. Only the parent PF - * shall be allowed to reset a child VF. A parent PF driver shall use - * this field only when a specific child VF is requested to be reset. - */ - uint16_t vf_id; - - /* This value indicates the level of a function reset. */ - /* - * Reset the caller function and its children VFs (if any). If - * no children functions exist, then reset the caller function - * only. - */ - #define HWRM_FUNC_RESET_INPUT_FUNC_RESET_LEVEL_RESETALL \ - (UINT32_C(0x0) << 0) - /* Reset the caller function only */ - #define HWRM_FUNC_RESET_INPUT_FUNC_RESET_LEVEL_RESETME \ - (UINT32_C(0x1) << 0) - /* - * Reset all children VFs of the caller function driver if the - * caller is a PF driver. It is an error to specify this level - * by a VF driver. It is an error to specify this level by a PF - * driver with no children VFs. - */ - #define HWRM_FUNC_RESET_INPUT_FUNC_RESET_LEVEL_RESETCHILDREN \ - (UINT32_C(0x2) << 0) - /* - * Reset a specific VF of the caller function driver if the - * caller is the parent PF driver. It is an error to specify - * this level by a VF driver. It is an error to specify this - * level by a PF driver that is not the parent of the VF that is - * being requested to reset. - */ - #define HWRM_FUNC_RESET_INPUT_FUNC_RESET_LEVEL_RESETVF \ - (UINT32_C(0x3) << 0) - uint8_t func_reset_level; - - uint8_t unused_0; -} __attribute__((packed)); - -/* Output (16 bytes) */ -struct hwrm_func_reset_output { - /* - * Pass/Fail or error type Note: receiver to verify the in parameters, - * and fail the call with an error when appropriate - */ - uint16_t error_code; - - /* This field returns the type of original request. */ - uint16_t req_type; - - /* This field provides original sequence number of the command. */ - uint16_t seq_id; - - /* - * This field is the length of the response in bytes. The last byte of - * the response is a valid flag that will read as '1' when the command - * has been completely written to memory. - */ - uint16_t resp_len; - - uint32_t unused_0; - uint8_t unused_1; - uint8_t unused_2; - uint8_t unused_3; - - /* - * This field is used in Output records to indicate that the output is - * completely written to RAM. This field should be read as '1' to - * indicate that the output has been completely written. When writing a - * command completion or response to an internal processor, the order of - * writes has to be such that this field is written last. - */ - uint8_t valid; -} __attribute__((packed)); - -/* hwrm_port_phy_cfg */ -/* - * Description: This command configures the PHY device for the port. It allows - * setting of the most generic settings for the PHY. The HWRM shall complete - * this command as soon as PHY settings are configured. They may not be applied - * when the command response is provided. A VF driver shall not be allowed to - * configure PHY using this command. In a network partition mode, a PF driver - * shall not be allowed to configure PHY using this command. - */ - -/* Input (56 bytes) */ -struct hwrm_port_phy_cfg_input { - /* - * This value indicates what type of request this is. The format for the - * rest of the command is determined by this field. - */ - uint16_t req_type; - - /* - * This value indicates the what completion ring the request will be - * optionally completed on. If the value is -1, then no CR completion - * will be generated. Any other value must be a valid CR ring_id value - * for this function. - */ - uint16_t cmpl_ring; - - /* This value indicates the command sequence number. */ - uint16_t seq_id; - - /* - * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids - * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM - */ - uint16_t target_id; - - /* - * This is the host address where the response will be written when the - * request is complete. This area must be 16B aligned and must be - * cleared to zero before the request is made. - */ - uint64_t resp_addr; - - /* - * When this bit is set to '1', the PHY for the port shall be reset. # - * If this bit is set to 1, then the HWRM shall reset the PHY after - * applying PHY configuration changes specified in this command. # In - * order to guarantee that PHY configuration changes specified in this - * command take effect, the HWRM client should set this flag to 1. # If - * this bit is not set to 1, then the HWRM may reset the PHY depending - * on the current PHY configuration and settings specified in this - * command. - */ - #define HWRM_PORT_PHY_CFG_INPUT_FLAGS_RESET_PHY UINT32_C(0x1) - /* - * When this bit is set to '1', the link shall be forced to be taken - * down. # When this bit is set to '1", all other command input settings - * related to the link speed shall be ignored. Once the link state is - * forced down, it can be explicitly cleared from that state by setting - * this flag to '0'. # If this flag is set to '0', then the link shall - * be cleared from forced down state if the link is in forced down - * state. There may be conditions (e.g. out-of-band or sideband - * configuration changes for the link) outside the scope of the HWRM - * implementation that may clear forced down link state. - */ - #define HWRM_PORT_PHY_CFG_INPUT_FLAGS_FORCE_LINK_DOWN UINT32_C(0x2) - /* - * When this bit is set to '1', the link shall be forced to the - * force_link_speed value. When this bit is set to '1', the HWRM client - * should not enable any of the auto negotiation related fields - * represented by auto_XXX fields in this command. When this bit is set - * to '1' and the HWRM client has enabled a auto_XXX field in this - * command, then the HWRM shall ignore the enabled auto_XXX field. When - * this bit is set to zero, the link shall be allowed to autoneg. - */ - #define HWRM_PORT_PHY_CFG_INPUT_FLAGS_FORCE UINT32_C(0x4) - /* - * When this bit is set to '1', the auto-negotiation process shall be - * restarted on the link. - */ - #define HWRM_PORT_PHY_CFG_INPUT_FLAGS_RESTART_AUTONEG UINT32_C(0x8) - /* - * When this bit is set to '1', Energy Efficient Ethernet (EEE) is - * requested to be enabled on this link. If EEE is not supported on this - * port, then this flag shall be ignored by the HWRM. - */ - #define HWRM_PORT_PHY_CFG_INPUT_FLAGS_EEE_ENABLE UINT32_C(0x10) - /* - * When this bit is set to '1', Energy Efficient Ethernet (EEE) is - * requested to be disabled on this link. If EEE is not supported on - * this port, then this flag shall be ignored by the HWRM. - */ - #define HWRM_PORT_PHY_CFG_INPUT_FLAGS_EEE_DISABLE UINT32_C(0x20) - /* - * When this bit is set to '1' and EEE is enabled on this link, then TX - * LPI is requested to be enabled on the link. If EEE is not supported - * on this port, then this flag shall be ignored by the HWRM. If EEE is - * disabled on this port, then this flag shall be ignored by the HWRM. - */ - #define HWRM_PORT_PHY_CFG_INPUT_FLAGS_EEE_TX_LPI UINT32_C(0x40) - uint32_t flags; - - /* This bit must be '1' for the auto_mode field to be configured. */ - #define HWRM_PORT_PHY_CFG_INPUT_ENABLES_AUTO_MODE UINT32_C(0x1) - /* This bit must be '1' for the auto_duplex field to be configured. */ - #define HWRM_PORT_PHY_CFG_INPUT_ENABLES_AUTO_DUPLEX UINT32_C(0x2) - /* This bit must be '1' for the auto_pause field to be configured. */ - #define HWRM_PORT_PHY_CFG_INPUT_ENABLES_AUTO_PAUSE UINT32_C(0x4) - /* - * This bit must be '1' for the auto_link_speed field to be configured. - */ - #define HWRM_PORT_PHY_CFG_INPUT_ENABLES_AUTO_LINK_SPEED UINT32_C(0x8) - /* - * This bit must be '1' for the auto_link_speed_mask field to be - * configured. - */ - #define HWRM_PORT_PHY_CFG_INPUT_ENABLES_AUTO_LINK_SPEED_MASK \ - UINT32_C(0x10) - /* This bit must be '1' for the wirespeed field to be configured. */ - #define HWRM_PORT_PHY_CFG_INPUT_ENABLES_WIRESPEED UINT32_C(0x20) - /* This bit must be '1' for the lpbk field to be configured. */ - #define HWRM_PORT_PHY_CFG_INPUT_ENABLES_LPBK UINT32_C(0x40) - /* This bit must be '1' for the preemphasis field to be configured. */ - #define HWRM_PORT_PHY_CFG_INPUT_ENABLES_PREEMPHASIS UINT32_C(0x80) - /* This bit must be '1' for the force_pause field to be configured. */ - #define HWRM_PORT_PHY_CFG_INPUT_ENABLES_FORCE_PAUSE UINT32_C(0x100) - /* - * This bit must be '1' for the eee_link_speed_mask field to be - * configured. - */ - #define HWRM_PORT_PHY_CFG_INPUT_ENABLES_EEE_LINK_SPEED_MASK \ - UINT32_C(0x200) - /* This bit must be '1' for the tx_lpi_timer field to be configured. */ - #define HWRM_PORT_PHY_CFG_INPUT_ENABLES_TX_LPI_TIMER UINT32_C(0x400) - uint32_t enables; - - /* Port ID of port that is to be configured. */ - uint16_t port_id; - - /* - * This is the speed that will be used if the force bit is '1'. If - * unsupported speed is selected, an error will be generated. - */ - /* 100Mb link speed */ - #define HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEED_100MB \ - (UINT32_C(0x1) << 0) - /* 1Gb link speed */ - #define HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEED_1GB \ - (UINT32_C(0xa) << 0) - /* 2Gb link speed */ - #define HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEED_2GB \ - (UINT32_C(0x14) << 0) - /* 2.5Gb link speed */ - #define HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEED_2_5GB \ - (UINT32_C(0x19) << 0) - /* 10Gb link speed */ - #define HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEED_10GB \ - (UINT32_C(0x64) << 0) - /* 20Mb link speed */ - #define HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEED_20GB \ - (UINT32_C(0xc8) << 0) - /* 25Gb link speed */ - #define HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEED_25GB \ - (UINT32_C(0xfa) << 0) - /* 40Gb link speed */ - #define HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEED_40GB \ - (UINT32_C(0x190) << 0) - /* 50Gb link speed */ - #define HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEED_50GB \ - (UINT32_C(0x1f4) << 0) - /* 100Gb link speed */ - #define HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEED_100GB \ - (UINT32_C(0x3e8) << 0) - /* 10Mb link speed */ - #define HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEED_10MB \ - (UINT32_C(0xffff) << 0) - uint16_t force_link_speed; - - /* - * This value is used to identify what autoneg mode is used when the - * link speed is not being forced. - */ - /* - * Disable autoneg or autoneg disabled. No speeds are selected. - */ - #define HWRM_PORT_PHY_CFG_INPUT_AUTO_MODE_NONE (UINT32_C(0x0) << 0) - /* Select all possible speeds for autoneg mode. */ - #define HWRM_PORT_PHY_CFG_INPUT_AUTO_MODE_ALL_SPEEDS \ - (UINT32_C(0x1) << 0) - /* - * Select only the auto_link_speed speed for autoneg mode. This - * mode has been DEPRECATED. An HWRM client should not use this - * mode. - */ - #define HWRM_PORT_PHY_CFG_INPUT_AUTO_MODE_ONE_SPEED \ - (UINT32_C(0x2) << 0) - /* - * Select the auto_link_speed or any speed below that speed for - * autoneg. This mode has been DEPRECATED. An HWRM client should - * not use this mode. - */ - #define HWRM_PORT_PHY_CFG_INPUT_AUTO_MODE_ONE_OR_BELOW \ - (UINT32_C(0x3) << 0) - /* - * Select the speeds based on the corresponding link speed mask - * value that is provided. - */ - #define HWRM_PORT_PHY_CFG_INPUT_AUTO_MODE_SPEED_MASK \ - (UINT32_C(0x4) << 0) - uint8_t auto_mode; - - /* - * This is the duplex setting that will be used if the autoneg_mode is - * "one_speed" or "one_or_below". - */ - /* Half Duplex will be requested. */ - #define HWRM_PORT_PHY_CFG_INPUT_AUTO_DUPLEX_HALF \ - (UINT32_C(0x0) << 0) - /* Full duplex will be requested. */ - #define HWRM_PORT_PHY_CFG_INPUT_AUTO_DUPLEX_FULL \ - (UINT32_C(0x1) << 0) - /* Both Half and Full dupex will be requested. */ - #define HWRM_PORT_PHY_CFG_INPUT_AUTO_DUPLEX_BOTH \ - (UINT32_C(0x2) << 0) - uint8_t auto_duplex; - - /* - * This value is used to configure the pause that will be used for - * autonegotiation. Add text on the usage of auto_pause and force_pause. - */ - /* - * When this bit is '1', Generation of tx pause messages has been - * requested. Disabled otherwise. - */ - #define HWRM_PORT_PHY_CFG_INPUT_AUTO_PAUSE_TX UINT32_C(0x1) - /* - * When this bit is '1', Reception of rx pause messages has been - * requested. Disabled otherwise. - */ - #define HWRM_PORT_PHY_CFG_INPUT_AUTO_PAUSE_RX UINT32_C(0x2) - /* - * When set to 1, the advertisement of pause is enabled. # When the - * auto_mode is not set to none and this flag is set to 1, then the - * auto_pause bits on this port are being advertised and autoneg pause - * results are being interpreted. # When the auto_mode is not set to - * none and this flag is set to 0, the pause is forced as indicated in - * force_pause, and also advertised as auto_pause bits, but the autoneg - * results are not interpreted since the pause configuration is being - * forced. # When the auto_mode is set to none and this flag is set to - * 1, auto_pause bits should be ignored and should be set to 0. - */ - #define HWRM_PORT_PHY_CFG_INPUT_AUTO_PAUSE_AUTONEG_PAUSE UINT32_C(0x4) - uint8_t auto_pause; - - uint8_t unused_0; - - /* - * This is the speed that will be used if the autoneg_mode is - * "one_speed" or "one_or_below". If an unsupported speed is selected, - * an error will be generated. - */ - /* 100Mb link speed */ - #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_100MB \ - (UINT32_C(0x1) << 0) - /* 1Gb link speed */ - #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_1GB \ - (UINT32_C(0xa) << 0) - /* 2Gb link speed */ - #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_2GB \ - (UINT32_C(0x14) << 0) - /* 2.5Gb link speed */ - #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_2_5GB \ - (UINT32_C(0x19) << 0) - /* 10Gb link speed */ - #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_10GB \ - (UINT32_C(0x64) << 0) - /* 20Mb link speed */ - #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_20GB \ - (UINT32_C(0xc8) << 0) - /* 25Gb link speed */ - #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_25GB \ - (UINT32_C(0xfa) << 0) - /* 40Gb link speed */ - #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_40GB \ - (UINT32_C(0x190) << 0) - /* 50Gb link speed */ - #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_50GB \ - (UINT32_C(0x1f4) << 0) - /* 100Gb link speed */ - #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_100GB \ - (UINT32_C(0x3e8) << 0) - /* 10Mb link speed */ - #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_10MB \ - (UINT32_C(0xffff) << 0) - uint16_t auto_link_speed; - - /* - * This is a mask of link speeds that will be used if autoneg_mode is - * "mask". If unsupported speed is enabled an error will be generated. - */ - /* 100Mb link speed (Half-duplex) */ - #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_100MBHD \ - UINT32_C(0x1) - /* 100Mb link speed (Full-duplex) */ - #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_100MB \ - UINT32_C(0x2) - /* 1Gb link speed (Half-duplex) */ - #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_1GBHD \ - UINT32_C(0x4) - /* 1Gb link speed (Full-duplex) */ - #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_1GB \ - UINT32_C(0x8) - /* 2Gb link speed */ - #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_2GB \ - UINT32_C(0x10) - /* 2.5Gb link speed */ - #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_2_5GB \ - UINT32_C(0x20) - /* 10Gb link speed */ - #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_10GB \ - UINT32_C(0x40) - /* 20Gb link speed */ - #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_20GB \ - UINT32_C(0x80) - /* 25Gb link speed */ - #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_25GB \ - UINT32_C(0x100) - /* 40Gb link speed */ - #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_40GB \ - UINT32_C(0x200) - /* 50Gb link speed */ - #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_50GB \ - UINT32_C(0x400) - /* 100Gb link speed */ - #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_100GB \ - UINT32_C(0x800) - /* 10Mb link speed (Half-duplex) */ - #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_10MBHD \ - UINT32_C(0x1000) - /* 10Mb link speed (Full-duplex) */ - #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_10MB \ - UINT32_C(0x2000) - uint16_t auto_link_speed_mask; - - /* This value controls the wirespeed feature. */ - /* Wirespeed feature is disabled. */ - #define HWRM_PORT_PHY_CFG_INPUT_WIRESPEED_OFF (UINT32_C(0x0) << 0) - /* Wirespeed feature is enabled. */ - #define HWRM_PORT_PHY_CFG_INPUT_WIRESPEED_ON (UINT32_C(0x1) << 0) - uint8_t wirespeed; - - /* This value controls the loopback setting for the PHY. */ - /* No loopback is selected. Normal operation. */ - #define HWRM_PORT_PHY_CFG_INPUT_LPBK_NONE (UINT32_C(0x0) << 0) - /* - * The HW will be configured with local loopback such that host - * data is sent back to the host without modification. - */ - #define HWRM_PORT_PHY_CFG_INPUT_LPBK_LOCAL (UINT32_C(0x1) << 0) - /* - * The HW will be configured with remote loopback such that port - * logic will send packets back out the transmitter that are - * received. - */ - #define HWRM_PORT_PHY_CFG_INPUT_LPBK_REMOTE (UINT32_C(0x2) << 0) - uint8_t lpbk; - - /* - * This value is used to configure the pause that will be used for force - * mode. - */ - /* - * When this bit is '1', Generation of tx pause messages is supported. - * Disabled otherwise. - */ - #define HWRM_PORT_PHY_CFG_INPUT_FORCE_PAUSE_TX UINT32_C(0x1) - /* - * When this bit is '1', Reception of rx pause messages is supported. - * Disabled otherwise. - */ - #define HWRM_PORT_PHY_CFG_INPUT_FORCE_PAUSE_RX UINT32_C(0x2) - uint8_t force_pause; - - uint8_t unused_1; - - /* - * This value controls the pre-emphasis to be used for the link. Driver - * should not set this value (use enable.preemphasis = 0) unless driver - * is sure of setting. Normally HWRM FW will determine proper pre- - * emphasis. - */ - uint32_t preemphasis; - - /* - * Setting for link speed mask that is used to advertise speeds during - * autonegotiation when EEE is enabled. This field is valid only when - * EEE is enabled. The speeds specified in this field shall be a subset - * of speeds specified in auto_link_speed_mask. If EEE is enabled,then - * at least one speed shall be provided in this mask. - */ - /* Reserved */ - #define HWRM_PORT_PHY_CFG_INPUT_EEE_LINK_SPEED_MASK_RSVD1 UINT32_C(0x1) - /* 100Mb link speed (Full-duplex) */ - #define HWRM_PORT_PHY_CFG_INPUT_EEE_LINK_SPEED_MASK_100MB UINT32_C(0x2) - /* Reserved */ - #define HWRM_PORT_PHY_CFG_INPUT_EEE_LINK_SPEED_MASK_RSVD2 UINT32_C(0x4) - /* 1Gb link speed (Full-duplex) */ - #define HWRM_PORT_PHY_CFG_INPUT_EEE_LINK_SPEED_MASK_1GB UINT32_C(0x8) - /* Reserved */ - #define HWRM_PORT_PHY_CFG_INPUT_EEE_LINK_SPEED_MASK_RSVD3 \ - UINT32_C(0x10) - /* Reserved */ - #define HWRM_PORT_PHY_CFG_INPUT_EEE_LINK_SPEED_MASK_RSVD4 \ - UINT32_C(0x20) - /* 10Gb link speed */ - #define HWRM_PORT_PHY_CFG_INPUT_EEE_LINK_SPEED_MASK_10GB \ - UINT32_C(0x40) - uint16_t eee_link_speed_mask; - - uint8_t unused_2; - uint8_t unused_3; - - /* - * Reuested setting of TX LPI timer in microseconds. This field is valid - * only when EEE is enabled and TX LPI is enabled. - */ - #define HWRM_PORT_PHY_CFG_INPUT_TX_LPI_TIMER_MASK \ - UINT32_C(0xffffff) - #define HWRM_PORT_PHY_CFG_INPUT_TX_LPI_TIMER_SFT 0 - uint32_t tx_lpi_timer; - - uint32_t unused_4; -} __attribute__((packed)); - -/* Output (16 bytes) */ -struct hwrm_port_phy_cfg_output { - /* - * Pass/Fail or error type Note: receiver to verify the in parameters, - * and fail the call with an error when appropriate - */ - uint16_t error_code; - - /* This field returns the type of original request. */ - uint16_t req_type; - - /* This field provides original sequence number of the command. */ - uint16_t seq_id; - - /* - * This field is the length of the response in bytes. The last byte of - * the response is a valid flag that will read as '1' when the command - * has been completely written to memory. - */ - uint16_t resp_len; - - uint32_t unused_0; - uint8_t unused_1; - uint8_t unused_2; - uint8_t unused_3; - - /* - * This field is used in Output records to indicate that the output is - * completely written to RAM. This field should be read as '1' to - * indicate that the output has been completely written. When writing a - * command completion or response to an internal processor, the order of - * writes has to be such that this field is written last. - */ - uint8_t valid; -} __attribute__((packed)); - -/* hwrm_port_phy_qcfg */ -/* Description: This command queries the PHY configuration for the port. */ -/* Input (24 bytes) */ - -struct hwrm_port_phy_qcfg_input { - /* - * This value indicates what type of request this is. The format for the - * rest of the command is determined by this field. - */ - uint16_t req_type; - - /* - * This value indicates the what completion ring the request will be - * optionally completed on. If the value is -1, then no CR completion - * will be generated. Any other value must be a valid CR ring_id value - * for this function. - */ - uint16_t cmpl_ring; - - /* This value indicates the command sequence number. */ - uint16_t seq_id; - - /* - * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids - * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM - */ - uint16_t target_id; - - /* - * This is the host address where the response will be written when the - * request is complete. This area must be 16B aligned and must be - * cleared to zero before the request is made. - */ - uint64_t resp_addr; - - /* Port ID of port that is to be queried. */ - uint16_t port_id; - - uint16_t unused_0[3]; -} __attribute__((packed)); - -/* Output (96 bytes) */ -struct hwrm_port_phy_qcfg_output { - /* - * Pass/Fail or error type Note: receiver to verify the in parameters, - * and fail the call with an error when appropriate - */ - uint16_t error_code; - - /* This field returns the type of original request. */ - uint16_t req_type; - - /* This field provides original sequence number of the command. */ - uint16_t seq_id; - - /* - * This field is the length of the response in bytes. The last byte of - * the response is a valid flag that will read as '1' when the command - * has been completely written to memory. - */ - uint16_t resp_len; - - /* This value indicates the current link status. */ - /* There is no link or cable detected. */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_NO_LINK (UINT32_C(0x0) << 0) - /* There is no link, but a cable has been detected. */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SIGNAL (UINT32_C(0x1) << 0) - /* There is a link. */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_LINK (UINT32_C(0x2) << 0) - uint8_t link; - - uint8_t unused_0; - - /* This value indicates the current link speed of the connection. */ - /* 100Mb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_100MB \ - (UINT32_C(0x1) << 0) - /* 1Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_1GB \ - (UINT32_C(0xa) << 0) - /* 2Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_2GB \ - (UINT32_C(0x14) << 0) - /* 2.5Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_2_5GB \ - (UINT32_C(0x19) << 0) - /* 10Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_10GB \ - (UINT32_C(0x64) << 0) - /* 20Mb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_20GB \ - (UINT32_C(0xc8) << 0) - /* 25Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_25GB \ - (UINT32_C(0xfa) << 0) - /* 40Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_40GB \ - (UINT32_C(0x190) << 0) - /* 50Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_50GB \ - (UINT32_C(0x1f4) << 0) - /* 100Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_100GB \ - (UINT32_C(0x3e8) << 0) - /* 10Mb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_10MB \ - (UINT32_C(0xffff) << 0) - uint16_t link_speed; - - /* This value is indicates the duplex of the current connection. */ - /* Half Duplex connection. */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_DUPLEX_HALF (UINT32_C(0x0) << 0) - /* Full duplex connection. */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_DUPLEX_FULL (UINT32_C(0x1) << 0) - uint8_t duplex; - - /* - * This value is used to indicate the current pause configuration. When - * autoneg is enabled, this value represents the autoneg results of - * pause configuration. - */ - /* - * When this bit is '1', Generation of tx pause messages is supported. - * Disabled otherwise. - */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_TX UINT32_C(0x1) - /* - * When this bit is '1', Reception of rx pause messages is supported. - * Disabled otherwise. - */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_RX UINT32_C(0x2) - uint8_t pause; - - /* - * The supported speeds for the port. This is a bit mask. For each speed - * that is supported, the corrresponding bit will be set to '1'. - */ - /* 100Mb link speed (Half-duplex) */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_100MBHD \ - UINT32_C(0x1) - /* 100Mb link speed (Full-duplex) */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_100MB \ - UINT32_C(0x2) - /* 1Gb link speed (Half-duplex) */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_1GBHD \ - UINT32_C(0x4) - /* 1Gb link speed (Full-duplex) */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_1GB \ - UINT32_C(0x8) - /* 2Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_2GB \ - UINT32_C(0x10) - /* 2.5Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_2_5GB \ - UINT32_C(0x20) - /* 10Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_10GB \ - UINT32_C(0x40) - /* 20Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_20GB \ - UINT32_C(0x80) - /* 25Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_25GB \ - UINT32_C(0x100) - /* 40Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_40GB \ - UINT32_C(0x200) - /* 50Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_50GB \ - UINT32_C(0x400) - /* 100Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_100GB \ - UINT32_C(0x800) - /* 10Mb link speed (Half-duplex) */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_10MBHD \ - UINT32_C(0x1000) - /* 10Mb link speed (Full-duplex) */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_10MB \ - UINT32_C(0x2000) - uint16_t support_speeds; - - /* - * Current setting of forced link speed. When the link speed is not - * being forced, this value shall be set to 0. - */ - /* 100Mb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_FORCE_LINK_SPEED_100MB \ - (UINT32_C(0x1) << 0) - /* 1Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_FORCE_LINK_SPEED_1GB \ - (UINT32_C(0xa) << 0) - /* 2Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_FORCE_LINK_SPEED_2GB \ - (UINT32_C(0x14) << 0) - /* 2.5Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_FORCE_LINK_SPEED_2_5GB \ - (UINT32_C(0x19) << 0) - /* 10Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_FORCE_LINK_SPEED_10GB \ - (UINT32_C(0x64) << 0) - /* 20Mb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_FORCE_LINK_SPEED_20GB \ - (UINT32_C(0xc8) << 0) - /* 25Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_FORCE_LINK_SPEED_25GB \ - (UINT32_C(0xfa) << 0) - /* 40Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_FORCE_LINK_SPEED_40GB \ - (UINT32_C(0x190) << 0) - /* 50Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_FORCE_LINK_SPEED_50GB \ - (UINT32_C(0x1f4) << 0) - /* 100Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_FORCE_LINK_SPEED_100GB \ - (UINT32_C(0x3e8) << 0) - /* 10Mb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_FORCE_LINK_SPEED_10MB \ - (UINT32_C(0xffff) << 0) - uint16_t force_link_speed; - - /* Current setting of auto negotiation mode. */ - /* - * Disable autoneg or autoneg disabled. No speeds are selected. - */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_MODE_NONE \ - (UINT32_C(0x0) << 0) - /* Select all possible speeds for autoneg mode. */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_MODE_ALL_SPEEDS \ - (UINT32_C(0x1) << 0) - /* - * Select only the auto_link_speed speed for autoneg mode. This - * mode has been DEPRECATED. An HWRM client should not use this - * mode. - */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_MODE_ONE_SPEED \ - (UINT32_C(0x2) << 0) - /* - * Select the auto_link_speed or any speed below that speed for - * autoneg. This mode has been DEPRECATED. An HWRM client should - * not use this mode. - */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_MODE_ONE_OR_BELOW \ - (UINT32_C(0x3) << 0) - /* - * Select the speeds based on the corresponding link speed mask - * value that is provided. - */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_MODE_SPEED_MASK \ - (UINT32_C(0x4) << 0) - uint8_t auto_mode; - - /* - * Current setting of pause autonegotiation. Move autoneg_pause flag - * here. - */ - /* - * When this bit is '1', Generation of tx pause messages has been - * requested. Disabled otherwise. - */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_PAUSE_TX UINT32_C(0x1) - /* - * When this bit is '1', Reception of rx pause messages has been - * requested. Disabled otherwise. - */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_PAUSE_RX UINT32_C(0x2) - /* - * When set to 1, the advertisement of pause is enabled. # When the - * auto_mode is not set to none and this flag is set to 1, then the - * auto_pause bits on this port are being advertised and autoneg pause - * results are being interpreted. # When the auto_mode is not set to - * none and this flag is set to 0, the pause is forced as indicated in - * force_pause, and also advertised as auto_pause bits, but the autoneg - * results are not interpreted since the pause configuration is being - * forced. # When the auto_mode is set to none and this flag is set to - * 1, auto_pause bits should be ignored and should be set to 0. - */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_PAUSE_AUTONEG_PAUSE \ - UINT32_C(0x4) - uint8_t auto_pause; - - /* - * Current setting for auto_link_speed. This field is only valid when - * auto_mode is set to "one_speed" or "one_or_below". - */ - /* 100Mb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_100MB \ - (UINT32_C(0x1) << 0) - /* 1Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_1GB \ - (UINT32_C(0xa) << 0) - /* 2Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_2GB \ - (UINT32_C(0x14) << 0) - /* 2.5Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_2_5GB \ - (UINT32_C(0x19) << 0) - /* 10Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_10GB \ - (UINT32_C(0x64) << 0) - /* 20Mb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_20GB \ - (UINT32_C(0xc8) << 0) - /* 25Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_25GB \ - (UINT32_C(0xfa) << 0) - /* 40Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_40GB \ - (UINT32_C(0x190) << 0) - /* 50Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_50GB \ - (UINT32_C(0x1f4) << 0) - /* 100Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_100GB \ - (UINT32_C(0x3e8) << 0) - /* 10Mb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_10MB \ - (UINT32_C(0xffff) << 0) - uint16_t auto_link_speed; - - /* - * Current setting for auto_link_speed_mask that is used to advertise - * speeds during autonegotiation. This field is only valid when - * auto_mode is set to "mask". The speeds specified in this field shall - * be a subset of supported speeds on this port. - */ - /* 100Mb link speed (Half-duplex) */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_MASK_100MBHD \ - UINT32_C(0x1) - /* 100Mb link speed (Full-duplex) */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_MASK_100MB \ - UINT32_C(0x2) - /* 1Gb link speed (Half-duplex) */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_MASK_1GBHD \ - UINT32_C(0x4) - /* 1Gb link speed (Full-duplex) */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_MASK_1GB \ - UINT32_C(0x8) - /* 2Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_MASK_2GB \ - UINT32_C(0x10) - /* 2.5Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_MASK_2_5GB \ - UINT32_C(0x20) - /* 10Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_MASK_10GB \ - UINT32_C(0x40) - /* 20Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_MASK_20GB \ - UINT32_C(0x80) - /* 25Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_MASK_25GB \ - UINT32_C(0x100) - /* 40Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_MASK_40GB \ - UINT32_C(0x200) - /* 50Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_MASK_50GB \ - UINT32_C(0x400) - /* 100Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_MASK_100GB \ - UINT32_C(0x800) - /* 10Mb link speed (Half-duplex) */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_MASK_10MBHD \ - UINT32_C(0x1000) - /* 10Mb link speed (Full-duplex) */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_MASK_10MB \ - UINT32_C(0x2000) - uint16_t auto_link_speed_mask; - - /* Current setting for wirespeed. */ - /* Wirespeed feature is disabled. */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_WIRESPEED_OFF (UINT32_C(0x0) << 0) - /* Wirespeed feature is enabled. */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_WIRESPEED_ON (UINT32_C(0x1) << 0) - uint8_t wirespeed; - - /* Current setting for loopback. */ - /* No loopback is selected. Normal operation. */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_LPBK_NONE (UINT32_C(0x0) << 0) - /* - * The HW will be configured with local loopback such that host - * data is sent back to the host without modification. - */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_LPBK_LOCAL (UINT32_C(0x1) << 0) - /* - * The HW will be configured with remote loopback such that port - * logic will send packets back out the transmitter that are - * received. - */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_LPBK_REMOTE (UINT32_C(0x2) << 0) - uint8_t lpbk; - - /* - * Current setting of forced pause. When the pause configuration is not - * being forced, then this value shall be set to 0. - */ - /* - * When this bit is '1', Generation of tx pause messages is supported. - * Disabled otherwise. - */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_FORCE_PAUSE_TX \ - UINT32_C(0x1) - /* - * When this bit is '1', Reception of rx pause messages is supported. - * Disabled otherwise. - */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_FORCE_PAUSE_RX \ - UINT32_C(0x2) - uint8_t force_pause; - - /* - * This value indicates the current status of the optics module on this - * port. - */ - /* Module is inserted and accepted */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_MODULE_STATUS_NONE \ - (UINT32_C(0x0) << 0) - /* Module is rejected and transmit side Laser is disabled. */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_MODULE_STATUS_DISABLETX \ - (UINT32_C(0x1) << 0) - /* Module mismatch warning. */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_MODULE_STATUS_WARNINGMSG \ - (UINT32_C(0x2) << 0) - /* Module is rejected and powered down. */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_MODULE_STATUS_PWRDOWN \ - (UINT32_C(0x3) << 0) - /* Module is not inserted. */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_MODULE_STATUS_NOTINSERTED \ - (UINT32_C(0x4) << 0) - /* Module status is not applicable. */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_MODULE_STATUS_NOTAPPLICABLE \ - (UINT32_C(0xff) << 0) - uint8_t module_status; - - /* Current setting for preemphasis. */ - uint32_t preemphasis; - - /* This field represents the major version of the PHY. */ - uint8_t phy_maj; - - /* This field represents the minor version of the PHY. */ - uint8_t phy_min; - - /* This field represents the build version of the PHY. */ - uint8_t phy_bld; - - /* This value represents a PHY type. */ - /* Unknown */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_UNKNOWN \ - (UINT32_C(0x0) << 0) - /* BASE-CR */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASECR \ - (UINT32_C(0x1) << 0) - /* BASE-KR4 (Deprecated) */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKR4 \ - (UINT32_C(0x2) << 0) - /* BASE-LR */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASELR \ - (UINT32_C(0x3) << 0) - /* BASE-SR */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASESR \ - (UINT32_C(0x4) << 0) - /* BASE-KR2 (Deprecated) */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKR2 \ - (UINT32_C(0x5) << 0) - /* BASE-KX */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKX \ - (UINT32_C(0x6) << 0) - /* BASE-KR */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKR \ - (UINT32_C(0x7) << 0) - /* BASE-T */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASET \ - (UINT32_C(0x8) << 0) - /* EEE capable BASE-T */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASETE \ - (UINT32_C(0x9) << 0) - /* SGMII connected external PHY */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_SGMIIEXTPHY \ - (UINT32_C(0xa) << 0) - uint8_t phy_type; - - /* This value represents a media type. */ - /* Unknown */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_MEDIA_TYPE_UNKNOWN \ - (UINT32_C(0x0) << 0) - /* Twisted Pair */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_MEDIA_TYPE_TP (UINT32_C(0x1) << 0) - /* Direct Attached Copper */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_MEDIA_TYPE_DAC \ - (UINT32_C(0x2) << 0) - /* Fiber */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_MEDIA_TYPE_FIBRE \ - (UINT32_C(0x3) << 0) - uint8_t media_type; - - /* This value represents a transceiver type. */ - /* PHY and MAC are in the same package */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_XCVR_PKG_TYPE_XCVR_INTERNAL \ - (UINT32_C(0x1) << 0) - /* PHY and MAC are in different packages */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_XCVR_PKG_TYPE_XCVR_EXTERNAL \ - (UINT32_C(0x2) << 0) - uint8_t xcvr_pkg_type; - - /* - * This field represents flags related to EEE configuration. These EEE - * configuration flags are valid only when the auto_mode is not set to - * none (in other words autonegotiation is enabled). - */ - /* This field represents PHY address. */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_ADDR_MASK UINT32_C(0x1f) - #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_ADDR_SFT 0 - /* - * When set to 1, Energy Efficient Ethernet (EEE) mode is enabled. - * Speeds for autoneg with EEE mode enabled are based on - * eee_link_speed_mask. - */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_EEE_CONFIG_EEE_ENABLED \ - UINT32_C(0x20) - /* - * This flag is valid only when eee_enabled is set to 1. # If - * eee_enabled is set to 0, then EEE mode is disabled and this flag - * shall be ignored. # If eee_enabled is set to 1 and this flag is set - * to 1, then Energy Efficient Ethernet (EEE) mode is enabled and in - * use. # If eee_enabled is set to 1 and this flag is set to 0, then - * Energy Efficient Ethernet (EEE) mode is enabled but is currently not - * in use. - */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_EEE_CONFIG_EEE_ACTIVE \ - UINT32_C(0x40) - /* - * This flag is valid only when eee_enabled is set to 1. # If - * eee_enabled is set to 0, then EEE mode is disabled and this flag - * shall be ignored. # If eee_enabled is set to 1 and this flag is set - * to 1, then Energy Efficient Ethernet (EEE) mode is enabled and TX LPI - * is enabled. # If eee_enabled is set to 1 and this flag is set to 0, - * then Energy Efficient Ethernet (EEE) mode is enabled but TX LPI is - * disabled. - */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_EEE_CONFIG_EEE_TX_LPI \ - UINT32_C(0x80) - /* - * This field represents flags related to EEE configuration. These EEE - * configuration flags are valid only when the auto_mode is not set to - * none (in other words autonegotiation is enabled). - */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_EEE_CONFIG_MASK \ - UINT32_C(0xe0) - #define HWRM_PORT_PHY_QCFG_OUTPUT_EEE_CONFIG_SFT 5 - uint8_t eee_config_phy_addr; - - /* Reserved field, set to 0 */ - /* - * When set to 1, the parallel detection is used to determine the speed - * of the link partner. Parallel detection is used when a - * autonegotiation capable device is connected to a link parter that is - * not capable of autonegotiation. - */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_PARALLEL_DETECT \ - UINT32_C(0x1) - /* Reserved field, set to 0 */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_RESERVED_MASK UINT32_C(0xfe) - #define HWRM_PORT_PHY_QCFG_OUTPUT_RESERVED_SFT 1 - uint8_t parallel_detect; - - /* - * The advertised speeds for the port by the link partner. Each - * advertised speed will be set to '1'. - */ - /* 100Mb link speed (Half-duplex) */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_SPEEDS_100MBHD \ - UINT32_C(0x1) - /* 100Mb link speed (Full-duplex) */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_SPEEDS_100MB \ - UINT32_C(0x2) - /* 1Gb link speed (Half-duplex) */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_SPEEDS_1GBHD \ - UINT32_C(0x4) - /* 1Gb link speed (Full-duplex) */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_SPEEDS_1GB \ - UINT32_C(0x8) - /* 2Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_SPEEDS_2GB \ - UINT32_C(0x10) - /* 2.5Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_SPEEDS_2_5GB \ - UINT32_C(0x20) - /* 10Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_SPEEDS_10GB \ - UINT32_C(0x40) - /* 20Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_SPEEDS_20GB \ - UINT32_C(0x80) - /* 25Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_SPEEDS_25GB \ - UINT32_C(0x100) - /* 40Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_SPEEDS_40GB \ - UINT32_C(0x200) - /* 50Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_SPEEDS_50GB \ - UINT32_C(0x400) - /* 100Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_SPEEDS_100GB \ - UINT32_C(0x800) - /* 10Mb link speed (Half-duplex) */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_SPEEDS_10MBHD \ - UINT32_C(0x1000) - /* 10Mb link speed (Full-duplex) */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_SPEEDS_10MB \ - UINT32_C(0x2000) - uint16_t link_partner_adv_speeds; - - /* - * The advertised autoneg for the port by the link partner. This field - * is deprecated and should be set to 0. - */ - /* - * Disable autoneg or autoneg disabled. No speeds are selected. - */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_AUTO_MODE_NONE \ - (UINT32_C(0x0) << 0) - /* Select all possible speeds for autoneg mode. */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_AUTO_MODE_ALL_SPEEDS\ - (UINT32_C(0x1) << 0) - /* - * Select only the auto_link_speed speed for autoneg mode. This - * mode has been DEPRECATED. An HWRM client should not use this - * mode. - */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_AUTO_MODE_ONE_SPEED \ - (UINT32_C(0x2) << 0) - /* - * Select the auto_link_speed or any speed below that speed for - * autoneg. This mode has been DEPRECATED. An HWRM client should - * not use this mode. - */ - #define \ - HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_AUTO_MODE_ONE_OR_BELOW \ - (UINT32_C(0x3) << 0) - /* - * Select the speeds based on the corresponding link speed mask - * value that is provided. - */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_AUTO_MODE_SPEED_MASK\ - (UINT32_C(0x4) << 0) - uint8_t link_partner_adv_auto_mode; - - /* The advertised pause settings on the port by the link partner. */ - /* - * When this bit is '1', Generation of tx pause messages is supported. - * Disabled otherwise. - */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_PAUSE_TX \ - UINT32_C(0x1) - /* - * When this bit is '1', Reception of rx pause messages is supported. - * Disabled otherwise. - */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_PAUSE_RX \ - UINT32_C(0x2) - uint8_t link_partner_adv_pause; - - /* - * Current setting for link speed mask that is used to advertise speeds - * during autonegotiation when EEE is enabled. This field is valid only - * when eee_enabled flags is set to 1. The speeds specified in this - * field shall be a subset of speeds specified in auto_link_speed_mask. - */ - /* Reserved */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_ADV_EEE_LINK_SPEED_MASK_RSVD1 \ - UINT32_C(0x1) - /* 100Mb link speed (Full-duplex) */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_ADV_EEE_LINK_SPEED_MASK_100MB \ - UINT32_C(0x2) - /* Reserved */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_ADV_EEE_LINK_SPEED_MASK_RSVD2 \ - UINT32_C(0x4) - /* 1Gb link speed (Full-duplex) */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_ADV_EEE_LINK_SPEED_MASK_1GB \ - UINT32_C(0x8) - /* Reserved */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_ADV_EEE_LINK_SPEED_MASK_RSVD3 \ - UINT32_C(0x10) - /* Reserved */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_ADV_EEE_LINK_SPEED_MASK_RSVD4 \ - UINT32_C(0x20) - /* 10Gb link speed */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_ADV_EEE_LINK_SPEED_MASK_10GB \ - UINT32_C(0x40) - uint16_t adv_eee_link_speed_mask; - - /* - * Current setting for link speed mask that is advertised by the link - * partner when EEE is enabled. This field is valid only when - * eee_enabled flags is set to 1. - */ - /* Reserved */ - #define \ - HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_EEE_LINK_SPEED_MASK_RSVD1 \ - UINT32_C(0x1) - /* 100Mb link speed (Full-duplex) */ - #define \ - HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_EEE_LINK_SPEED_MASK_100MB \ - UINT32_C(0x2) - /* Reserved */ - #define \ - HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_EEE_LINK_SPEED_MASK_RSVD2 \ - UINT32_C(0x4) - /* 1Gb link speed (Full-duplex) */ - #define \ - HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_EEE_LINK_SPEED_MASK_1GB \ - UINT32_C(0x8) - /* Reserved */ - #define \ - HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_EEE_LINK_SPEED_MASK_RSVD3 \ - UINT32_C(0x10) - /* Reserved */ - #define \ - HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_EEE_LINK_SPEED_MASK_RSVD4 \ - UINT32_C(0x20) - /* 10Gb link speed */ - #define \ - HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_EEE_LINK_SPEED_MASK_10GB \ - UINT32_C(0x40) - uint16_t link_partner_adv_eee_link_speed_mask; - - /* This value represents transceiver identifier type. */ - /* - * Current setting of TX LPI timer in microseconds. This field is valid - * only when_eee_enabled flag is set to 1 and tx_lpi_enabled is set to - * 1. - */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_TX_LPI_TIMER_MASK \ - UINT32_C(0xffffff) - #define HWRM_PORT_PHY_QCFG_OUTPUT_TX_LPI_TIMER_SFT 0 - /* This value represents transceiver identifier type. */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_XCVR_IDENTIFIER_TYPE_MASK \ - UINT32_C(0xff000000) - #define HWRM_PORT_PHY_QCFG_OUTPUT_XCVR_IDENTIFIER_TYPE_SFT \ - 24 - /* Unknown */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_XCVR_IDENTIFIER_TYPE_UNKNOWN \ - (UINT32_C(0x0) << 24) - /* SFP/SFP+/SFP28 */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_XCVR_IDENTIFIER_TYPE_SFP \ - (UINT32_C(0x3) << 24) - /* QSFP */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_XCVR_IDENTIFIER_TYPE_QSFP \ - (UINT32_C(0xc) << 24) - /* QSFP+ */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_XCVR_IDENTIFIER_TYPE_QSFPPLUS \ - (UINT32_C(0xd) << 24) - /* QSFP28 */ - #define HWRM_PORT_PHY_QCFG_OUTPUT_XCVR_IDENTIFIER_TYPE_QSFP28 \ - (UINT32_C(0x11) << 24) - uint32_t xcvr_identifier_type_tx_lpi_timer; - - uint32_t unused_1; - - /* - * Up to 16 bytes of null padded ASCII string representing PHY vendor. - * If the string is set to null, then the vendor name is not available. - */ - char phy_vendor_name[16]; - - /* - * Up to 16 bytes of null padded ASCII string that identifies vendor - * specific part number of the PHY. If the string is set to null, then - * the vendor specific part number is not available. - */ - char phy_vendor_partnumber[16]; - - uint32_t unused_2; - uint8_t unused_3; - uint8_t unused_4; - uint8_t unused_5; - - /* - * This field is used in Output records to indicate that the output is - * completely written to RAM. This field should be read as '1' to - * indicate that the output has been completely written. When writing a - * command completion or response to an internal processor, the order of - * writes has to be such that this field is written last. - */ - uint8_t valid; + /* Event specific data */ } __attribute__((packed)); /* hwrm_ver_get */ @@ -3592,313 +1837,536 @@ struct hwrm_port_phy_qcfg_output { * interface or firmware version with major = 0, minor = 0, and update = 0 shall * be considered an invalid version. */ - -/* Input (24 bytes) */ +/* Input (24 bytes) */ struct hwrm_ver_get_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint8_t hwrm_intf_maj; + /* + * This field represents the major version of HWRM interface + * specification supported by the driver HWRM implementation. + * The interface major version is intended to change only when + * non backward compatible changes are made to the HWRM + * interface specification. + */ + uint8_t hwrm_intf_min; + /* + * This field represents the minor version of HWRM interface + * specification supported by the driver HWRM implementation. A + * change in interface minor version is used to reflect + * significant backward compatible modification to HWRM + * interface specification. This can be due to addition or + * removal of functionality. HWRM interface specifications with + * the same major version but different minor versions are + * compatible. + */ + uint8_t hwrm_intf_upd; + /* + * This field represents the update version of HWRM interface + * specification supported by the driver HWRM implementation. + * The interface update version is used to reflect minor changes + * or bug fixes to a released HWRM interface specification. + */ + uint8_t unused_0[5]; +} __attribute__((packed)); + +/* Output (128 bytes) */ +struct hwrm_ver_get_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint8_t hwrm_intf_maj; + /* + * This field represents the major version of HWRM interface + * specification supported by the HWRM implementation. The + * interface major version is intended to change only when non + * backward compatible changes are made to the HWRM interface + * specification. A HWRM implementation that is compliant with + * this specification shall provide value of 1 in this field. + */ + uint8_t hwrm_intf_min; + /* + * This field represents the minor version of HWRM interface + * specification supported by the HWRM implementation. A change + * in interface minor version is used to reflect significant + * backward compatible modification to HWRM interface + * specification. This can be due to addition or removal of + * functionality. HWRM interface specifications with the same + * major version but different minor versions are compatible. A + * HWRM implementation that is compliant with this specification + * shall provide value of 2 in this field. + */ + uint8_t hwrm_intf_upd; + /* + * This field represents the update version of HWRM interface + * specification supported by the HWRM implementation. The + * interface update version is used to reflect minor changes or + * bug fixes to a released HWRM interface specification. A HWRM + * implementation that is compliant with this specification + * shall provide value of 2 in this field. + */ + uint8_t hwrm_intf_rsvd; + uint8_t hwrm_fw_maj; + /* + * This field represents the major version of HWRM firmware. A + * change in firmware major version represents a major firmware + * release. + */ + uint8_t hwrm_fw_min; + /* + * This field represents the minor version of HWRM firmware. A + * change in firmware minor version represents significant + * firmware functionality changes. + */ + uint8_t hwrm_fw_bld; + /* + * This field represents the build version of HWRM firmware. A + * change in firmware build version represents bug fixes to a + * released firmware. + */ + uint8_t hwrm_fw_rsvd; + /* + * This field is a reserved field. This field can be used to + * represent firmware branches or customer specific releases + * tied to a specific (major,minor,update) version of the HWRM + * firmware. + */ + uint8_t mgmt_fw_maj; + /* + * This field represents the major version of mgmt firmware. A + * change in major version represents a major release. + */ + uint8_t mgmt_fw_min; + /* + * This field represents the minor version of mgmt firmware. A + * change in minor version represents significant functionality + * changes. + */ + uint8_t mgmt_fw_bld; + /* + * This field represents the build version of mgmt firmware. A + * change in update version represents bug fixes. + */ + uint8_t mgmt_fw_rsvd; + /* + * This field is a reserved field. This field can be used to + * represent firmware branches or customer specific releases + * tied to a specific (major,minor,update) version + */ + uint8_t netctrl_fw_maj; + /* + * This field represents the major version of network control + * firmware. A change in major version represents a major + * release. + */ + uint8_t netctrl_fw_min; + /* + * This field represents the minor version of network control + * firmware. A change in minor version represents significant + * functionality changes. + */ + uint8_t netctrl_fw_bld; + /* + * This field represents the build version of network control + * firmware. A change in update version represents bug fixes. + */ + uint8_t netctrl_fw_rsvd; + /* + * This field is a reserved field. This field can be used to + * represent firmware branches or customer specific releases + * tied to a specific (major,minor,update) version + */ + uint32_t dev_caps_cfg; + /* + * This field is used to indicate device's capabilities and + * configurations. + */ + /* + * If set to 1, then secure firmware update behavior is + * supported. If set to 0, then secure firmware update behavior + * is not supported. + */ + #define HWRM_VER_GET_OUTPUT_DEV_CAPS_CFG_SECURE_FW_UPD_SUPPORTED \ + UINT32_C(0x1) + /* + * If set to 1, then firmware based DCBX agent is supported. If + * set to 0, then firmware based DCBX agent capability is not + * supported on this device. + */ + #define HWRM_VER_GET_OUTPUT_DEV_CAPS_CFG_FW_DCBX_AGENT_SUPPORTED \ + UINT32_C(0x2) + /* + * If set to 1, then HWRM short command format is supported. If + * set to 0, then HWRM short command format is not supported. + */ + #define HWRM_VER_GET_OUTPUT_DEV_CAPS_CFG_SHORT_CMD_SUPPORTED \ + UINT32_C(0x4) + /* + * If set to 1, then HWRM short command format is required. If + * set to 0, then HWRM short command format is not required. + */ + #define HWRM_VER_GET_OUTPUT_DEV_CAPS_CFG_SHORT_CMD_INPUTUIRED \ + UINT32_C(0x8) + uint8_t roce_fw_maj; + /* + * This field represents the major version of RoCE firmware. A + * change in major version represents a major release. + */ + uint8_t roce_fw_min; + /* + * This field represents the minor version of RoCE firmware. A + * change in minor version represents significant functionality + * changes. + */ + uint8_t roce_fw_bld; + /* + * This field represents the build version of RoCE firmware. A + * change in update version represents bug fixes. + */ + uint8_t roce_fw_rsvd; + /* + * This field is a reserved field. This field can be used to + * represent firmware branches or customer specific releases + * tied to a specific (major,minor,update) version + */ + char hwrm_fw_name[16]; + /* + * This field represents the name of HWRM FW (ASCII chars with + * NULL at the end). + */ + char mgmt_fw_name[16]; + /* + * This field represents the name of mgmt FW (ASCII chars with + * NULL at the end). + */ + char netctrl_fw_name[16]; + /* + * This field represents the name of network control firmware + * (ASCII chars with NULL at the end). + */ + uint32_t reserved2[4]; + /* + * This field is reserved for future use. The responder should + * set it to 0. The requester should ignore this field. + */ + char roce_fw_name[16]; + /* + * This field represents the name of RoCE FW (ASCII chars with + * NULL at the end). + */ + uint16_t chip_num; + /* This field returns the chip number. */ + uint8_t chip_rev; + /* This field returns the revision of chip. */ + uint8_t chip_metal; + /* This field returns the chip metal number. */ + uint8_t chip_bond_id; + /* This field returns the bond id of the chip. */ + uint8_t chip_platform_type; + /* + * This value indicates the type of platform used for chip + * implementation. + */ + /* ASIC */ + #define HWRM_VER_GET_OUTPUT_CHIP_PLATFORM_TYPE_ASIC UINT32_C(0x0) + /* FPGA platform of the chip. */ + #define HWRM_VER_GET_OUTPUT_CHIP_PLATFORM_TYPE_FPGA UINT32_C(0x1) + /* Palladium platform of the chip. */ + #define HWRM_VER_GET_OUTPUT_CHIP_PLATFORM_TYPE_PALLADIUM UINT32_C(0x2) + uint16_t max_req_win_len; + /* + * This field returns the maximum value of request window that + * is supported by the HWRM. The request window is mapped into + * device address space using MMIO. + */ + uint16_t max_resp_len; + /* This field returns the maximum value of response buffer in bytes. */ + uint16_t def_req_timeout; + /* + * This field returns the default request timeout value in + * milliseconds. + */ + uint8_t init_pending; + /* + * This field will indicate if any subsystems is not fully + * initialized. + */ + /* + * If set to 1, device is not ready. If set to 0, device is + * ready to accept all HWRM commands. + */ + #define HWRM_VER_GET_OUTPUT_INIT_PENDING_DEV_NOT_RDY UINT32_C(0x1) + uint8_t unused_0; + uint8_t unused_1; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* hwrm_func_reset */ +/* + * Description: This command resets a hardware function (PCIe function) and + * frees any resources used by the function. This command shall be initiated by + * the driver after an FLR has occurred to prepare the function for re-use. This + * command may also be initiated by a driver prior to doing it's own + * configuration. This command puts the function into the reset state. In the + * reset state, global and port related features of the chip are not available. + */ +/* + * Note: This command will reset a function that has already been disabled or + * idled. The command returns all the resources owned by the function so a new + * driver may allocate and configure resources normally. + */ +/* Input (24 bytes) */ +struct hwrm_func_reset_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint32_t enables; + /* This bit must be '1' for the vf_id_valid field to be configured. */ + #define HWRM_FUNC_RESET_INPUT_ENABLES_VF_ID_VALID UINT32_C(0x1) + uint16_t vf_id; + /* + * The ID of the VF that this PF is trying to reset. Only the + * parent PF shall be allowed to reset a child VF. A parent PF + * driver shall use this field only when a specific child VF is + * requested to be reset. + */ + uint8_t func_reset_level; + /* This value indicates the level of a function reset. */ + /* + * Reset the caller function and its children + * VFs (if any). If no children functions exist, + * then reset the caller function only. + */ + #define HWRM_FUNC_RESET_INPUT_FUNC_RESET_LEVEL_RESETALL UINT32_C(0x0) + /* Reset the caller function only */ + #define HWRM_FUNC_RESET_INPUT_FUNC_RESET_LEVEL_RESETME UINT32_C(0x1) + /* + * Reset all children VFs of the caller function + * driver if the caller is a PF driver. It is an + * error to specify this level by a VF driver. + * It is an error to specify this level by a PF + * driver with no children VFs. + */ + #define HWRM_FUNC_RESET_INPUT_FUNC_RESET_LEVEL_RESETCHILDREN \ + UINT32_C(0x2) + /* + * Reset a specific VF of the caller function + * driver if the caller is the parent PF driver. + * It is an error to specify this level by a VF + * driver. It is an error to specify this level + * by a PF driver that is not the parent of the + * VF that is being requested to reset. + */ + #define HWRM_FUNC_RESET_INPUT_FUNC_RESET_LEVEL_RESETVF UINT32_C(0x3) + uint8_t unused_0; +} __attribute__((packed)); + +/* Output (16 bytes) */ +struct hwrm_func_reset_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint32_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t unused_3; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* hwrm_func_vf_cfg */ +/* + * Description: This command allows configuration of a VF by its driver. If this + * function is called by a PF driver, then the HWRM shall fail this command. If + * guest VLAN and/or MAC address are provided in this command, then the HWRM + * shall set up appropriate MAC/VLAN filters for the VF that is being + * configured. A VF driver should set VF MTU/MRU using this command prior to + * allocating RX VNICs or TX rings for the corresponding VF. + */ +/* Input (32 bytes) */ +struct hwrm_func_vf_cfg_input { + uint16_t req_type; /* * This value indicates what type of request this is. The format for the * rest of the command is determined by this field. */ - uint16_t req_type; - + uint16_t cmpl_ring; /* * This value indicates the what completion ring the request will be * optionally completed on. If the value is -1, then no CR completion * will be generated. Any other value must be a valid CR ring_id value * for this function. */ - uint16_t cmpl_ring; - - /* This value indicates the command sequence number. */ uint16_t seq_id; - + /* This value indicates the command sequence number. */ + uint16_t target_id; /* * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM */ - uint16_t target_id; - + uint64_t resp_addr; /* * This is the host address where the response will be written when the * request is complete. This area must be 16B aligned and must be * cleared to zero before the request is made. */ - uint64_t resp_addr; - + uint32_t enables; + /* This bit must be '1' for the mtu field to be configured. */ + #define HWRM_FUNC_VF_CFG_INPUT_ENABLES_MTU UINT32_C(0x1) + /* This bit must be '1' for the guest_vlan field to be configured. */ + #define HWRM_FUNC_VF_CFG_INPUT_ENABLES_GUEST_VLAN UINT32_C(0x2) /* - * This field represents the major version of HWRM interface - * specification supported by the driver HWRM implementation. The - * interface major version is intended to change only when non backward - * compatible changes are made to the HWRM interface specification. + * This bit must be '1' for the async_event_cr field to be configured. */ - uint8_t hwrm_intf_maj; - + #define HWRM_FUNC_VF_CFG_INPUT_ENABLES_ASYNC_EVENT_CR UINT32_C(0x4) + /* This bit must be '1' for the dflt_mac_addr field to be configured. */ + #define HWRM_FUNC_VF_CFG_INPUT_ENABLES_DFLT_MAC_ADDR UINT32_C(0x8) + uint16_t mtu; /* - * This field represents the minor version of HWRM interface - * specification supported by the driver HWRM implementation. A change - * in interface minor version is used to reflect significant backward - * compatible modification to HWRM interface specification. This can be - * due to addition or removal of functionality. HWRM interface - * specifications with the same major version but different minor - * versions are compatible. + * The maximum transmission unit requested on the function. The HWRM + * should make sure that the mtu of the function does not exceed the mtu + * of the physical port that this function is associated with. In + * addition to requesting mtu per function, it is possible to configure + * mtu per transmit ring. By default, the mtu of each transmit ring + * associated with a function is equal to the mtu of the function. The + * HWRM should make sure that the mtu of each transmit ring that is + * assigned to a function has a valid mtu. */ - uint8_t hwrm_intf_min; - + uint16_t guest_vlan; /* - * This field represents the update version of HWRM interface - * specification supported by the driver HWRM implementation. The - * interface update version is used to reflect minor changes or bug - * fixes to a released HWRM interface specification. + * The guest VLAN for the function being configured. This field's format + * is same as 802.1Q Tag's Tag Control Information (TCI) format that + * includes both Priority Code Point (PCP) and VLAN Identifier (VID). + */ + uint16_t async_event_cr; + /* + * ID of the target completion ring for receiving asynchronous event + * completions. If this field is not valid, then the HWRM shall use the + * default completion ring of the function that is being configured as + * the target completion ring for providing any asynchronous event + * completions for that function. If this field is valid, then the HWRM + * shall use the completion ring identified by this ID as the target + * completion ring for providing any asynchronous event completions for + * the function that is being configured. + */ + uint8_t dflt_mac_addr[6]; + /* + * This value is the current MAC address requested by the VF driver to + * be configured on this VF. A value of 00-00-00-00-00-00 indicates no + * MAC address configuration is requested by the VF driver. The parent + * PF driver may reject or overwrite this MAC address. */ - uint8_t hwrm_intf_upd; - - uint8_t unused_0[5]; } __attribute__((packed)); -/* Output (128 bytes) */ -struct hwrm_ver_get_output { +/* Output (16 bytes) */ + +struct hwrm_func_vf_cfg_output { + uint16_t error_code; /* * Pass/Fail or error type Note: receiver to verify the in parameters, * and fail the call with an error when appropriate */ - uint16_t error_code; - - /* This field returns the type of original request. */ uint16_t req_type; - - /* This field provides original sequence number of the command. */ + /* This field returns the type of original request. */ uint16_t seq_id; - - /* - * This field is the length of the response in bytes. The last byte of - * the response is a valid flag that will read as '1' when the command - * has been completely written to memory. - */ + /* This field provides original sequence number of the command. */ uint16_t resp_len; - /* - * This field represents the major version of HWRM interface - * specification supported by the HWRM implementation. The interface - * major version is intended to change only when non backward compatible - * changes are made to the HWRM interface specification. A HWRM - * implementation that is compliant with this specification shall - * provide value of 1 in this field. + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. */ - uint8_t hwrm_intf_maj; - - /* - * This field represents the minor version of HWRM interface - * specification supported by the HWRM implementation. A change in - * interface minor version is used to reflect significant backward - * compatible modification to HWRM interface specification. This can be - * due to addition or removal of functionality. HWRM interface - * specifications with the same major version but different minor - * versions are compatible. A HWRM implementation that is compliant with - * this specification shall provide value of 0 in this field. - */ - uint8_t hwrm_intf_min; - - /* - * This field represents the update version of HWRM interface - * specification supported by the HWRM implementation. The interface - * update version is used to reflect minor changes or bug fixes to a - * released HWRM interface specification. A HWRM implementation that is - * compliant with this specification shall provide value of 1 in this - * field. - */ - uint8_t hwrm_intf_upd; - - uint8_t hwrm_intf_rsvd; - - /* - * This field represents the major version of HWRM firmware. A change in - * firmware major version represents a major firmware release. - */ - uint8_t hwrm_fw_maj; - - /* - * This field represents the minor version of HWRM firmware. A change in - * firmware minor version represents significant firmware functionality - * changes. - */ - uint8_t hwrm_fw_min; - - /* - * This field represents the build version of HWRM firmware. A change in - * firmware build version represents bug fixes to a released firmware. - */ - uint8_t hwrm_fw_bld; - - /* - * This field is a reserved field. This field can be used to represent - * firmware branches or customer specific releases tied to a specific - * (major,minor,update) version of the HWRM firmware. - */ - uint8_t hwrm_fw_rsvd; - - /* - * This field represents the major version of mgmt firmware. A change in - * major version represents a major release. - */ - uint8_t mgmt_fw_maj; - - /* - * This field represents the minor version of mgmt firmware. A change in - * minor version represents significant functionality changes. - */ - uint8_t mgmt_fw_min; - - /* - * This field represents the build version of mgmt firmware. A change in - * update version represents bug fixes. - */ - uint8_t mgmt_fw_bld; - - /* - * This field is a reserved field. This field can be used to represent - * firmware branches or customer specific releases tied to a specific - * (major,minor,update) version - */ - uint8_t mgmt_fw_rsvd; - - /* - * This field represents the major version of network control firmware. - * A change in major version represents a major release. - */ - uint8_t netctrl_fw_maj; - - /* - * This field represents the minor version of network control firmware. - * A change in minor version represents significant functionality - * changes. - */ - uint8_t netctrl_fw_min; - - /* - * This field represents the build version of network control firmware. - * A change in update version represents bug fixes. - */ - uint8_t netctrl_fw_bld; - - /* - * This field is a reserved field. This field can be used to represent - * firmware branches or customer specific releases tied to a specific - * (major,minor,update) version - */ - uint8_t netctrl_fw_rsvd; - - /* - * This field is reserved for future use. The responder should set it to - * 0. The requester should ignore this field. - */ - uint32_t reserved1; - - /* - * This field represents the major version of RoCE firmware. A change in - * major version represents a major release. - */ - uint8_t roce_fw_maj; - - /* - * This field represents the minor version of RoCE firmware. A change in - * minor version represents significant functionality changes. - */ - uint8_t roce_fw_min; - - /* - * This field represents the build version of RoCE firmware. A change in - * update version represents bug fixes. - */ - uint8_t roce_fw_bld; - - /* - * This field is a reserved field. This field can be used to represent - * firmware branches or customer specific releases tied to a specific - * (major,minor,update) version - */ - uint8_t roce_fw_rsvd; - - /* - * This field represents the name of HWRM FW (ASCII chars without NULL - * at the end). - */ - char hwrm_fw_name[16]; - - /* - * This field represents the name of mgmt FW (ASCII chars without NULL - * at the end). - */ - char mgmt_fw_name[16]; - - /* - * This field represents the name of network control firmware (ASCII - * chars without NULL at the end). - */ - char netctrl_fw_name[16]; - - /* - * This field is reserved for future use. The responder should set it to - * 0. The requester should ignore this field. - */ - uint32_t reserved2[4]; - - /* - * This field represents the name of RoCE FW (ASCII chars without NULL - * at the end). - */ - char roce_fw_name[16]; - - /* This field returns the chip number. */ - uint16_t chip_num; - - /* This field returns the revision of chip. */ - uint8_t chip_rev; - - /* This field returns the chip metal number. */ - uint8_t chip_metal; - - /* This field returns the bond id of the chip. */ - uint8_t chip_bond_id; - - /* - * This value indicates the type of platform used for chip - * implementation. - */ - /* ASIC */ - #define HWRM_VER_GET_OUTPUT_CHIP_PLATFORM_TYPE_ASIC \ - (UINT32_C(0x0) << 0) - /* FPGA platform of the chip. */ - #define HWRM_VER_GET_OUTPUT_CHIP_PLATFORM_TYPE_FPGA \ - (UINT32_C(0x1) << 0) - /* Palladium platform of the chip. */ - #define HWRM_VER_GET_OUTPUT_CHIP_PLATFORM_TYPE_PALLADIUM \ - (UINT32_C(0x2) << 0) - uint8_t chip_platform_type; - - /* - * This field returns the maximum value of request window that is - * supported by the HWRM. The request window is mapped into device - * address space using MMIO. - */ - uint16_t max_req_win_len; - - /* - * This field returns the maximum value of response buffer in bytes. If - * a request specifies the response buffer length that is greater than - * this value, then the HWRM should fail it. The value of this field - * shall be 4KB or more. - */ - uint16_t max_resp_len; - - /* - * This field returns the default request timeout value in milliseconds. - */ - uint16_t def_req_timeout; - - uint8_t unused_0; + uint32_t unused_0; uint8_t unused_1; uint8_t unused_2; - + uint8_t unused_3; + uint8_t valid; /* * This field is used in Output records to indicate that the output is * completely written to RAM. This field should be read as '1' to @@ -3906,7 +2374,4404 @@ struct hwrm_ver_get_output { * command completion or response to an internal processor, the order of * writes has to be such that this field is written last. */ +} __attribute__((packed)); + +/* hwrm_func_qcaps */ +/* + * Description: This command returns capabilities of a function. The input FID + * value is used to indicate what function is being queried. This allows a + * physical function driver to query virtual functions that are children of the + * physical function. The output FID value is needed to configure Rings and + * MSI-X vectors so their DMA operations appear correctly on the PCI bus. + */ +/* Input (24 bytes) */ +struct hwrm_func_qcaps_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint16_t fid; + /* + * Function ID of the function that is being queried. 0xFF... + * (All Fs) if the query is for the requesting function. + */ + uint16_t unused_0[3]; +} __attribute__((packed)); + +/* Output (80 bytes) */ +struct hwrm_func_qcaps_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint16_t fid; + /* + * FID value. This value is used to identify operations on the + * PCI bus as belonging to a particular PCI function. + */ + uint16_t port_id; + /* + * Port ID of port that this function is associated with. Valid + * only for the PF. 0xFF... (All Fs) if this function is not + * associated with any port. 0xFF... (All Fs) if this function + * is called from a VF. + */ + uint32_t flags; + /* If 1, then Push mode is supported on this function. */ + #define HWRM_FUNC_QCAPS_OUTPUT_FLAGS_PUSH_MODE_SUPPORTED UINT32_C(0x1) + /* + * If 1, then the global MSI-X auto-masking is enabled for the + * device. + */ + #define HWRM_FUNC_QCAPS_OUTPUT_FLAGS_GLOBAL_MSIX_AUTOMASKING \ + UINT32_C(0x2) + /* + * If 1, then the Precision Time Protocol (PTP) processing is + * supported on this function. The HWRM should enable PTP on + * only a single Physical Function (PF) per port. + */ + #define HWRM_FUNC_QCAPS_OUTPUT_FLAGS_PTP_SUPPORTED UINT32_C(0x4) + /* + * If 1, then RDMA over Converged Ethernet (RoCE) v1 is + * supported on this function. + */ + #define HWRM_FUNC_QCAPS_OUTPUT_FLAGS_ROCE_V1_SUPPORTED UINT32_C(0x8) + /* + * If 1, then RDMA over Converged Ethernet (RoCE) v2 is + * supported on this function. + */ + #define HWRM_FUNC_QCAPS_OUTPUT_FLAGS_ROCE_V2_SUPPORTED UINT32_C(0x10) + /* + * If 1, then control and configuration of WoL magic packet are + * supported on this function. + */ + #define HWRM_FUNC_QCAPS_OUTPUT_FLAGS_WOL_MAGICPKT_SUPPORTED \ + UINT32_C(0x20) + /* + * If 1, then control and configuration of bitmap pattern packet + * are supported on this function. + */ + #define HWRM_FUNC_QCAPS_OUTPUT_FLAGS_WOL_BMP_SUPPORTED UINT32_C(0x40) + /* + * If set to 1, then the control and configuration of rate limit + * of an allocated TX ring on the queried function is supported. + */ + #define HWRM_FUNC_QCAPS_OUTPUT_FLAGS_TX_RING_RL_SUPPORTED UINT32_C(0x80) + /* + * If 1, then control and configuration of minimum and maximum + * bandwidths are supported on the queried function. + */ + #define HWRM_FUNC_QCAPS_OUTPUT_FLAGS_TX_BW_CFG_SUPPORTED UINT32_C(0x100) + /* + * If the query is for a VF, then this flag shall be ignored. If + * this query is for a PF and this flag is set to 1, then the PF + * has the capability to set the rate limits on the TX rings of + * its children VFs. If this query is for a PF and this flag is + * set to 0, then the PF does not have the capability to set the + * rate limits on the TX rings of its children VFs. + */ + #define HWRM_FUNC_QCAPS_OUTPUT_FLAGS_VF_TX_RING_RL_SUPPORTED \ + UINT32_C(0x200) + /* + * If the query is for a VF, then this flag shall be ignored. If + * this query is for a PF and this flag is set to 1, then the PF + * has the capability to set the minimum and/or maximum + * bandwidths for its children VFs. If this query is for a PF + * and this flag is set to 0, then the PF does not have the + * capability to set the minimum or maximum bandwidths for its + * children VFs. + */ + #define HWRM_FUNC_QCAPS_OUTPUT_FLAGS_VF_BW_CFG_SUPPORTED UINT32_C(0x400) + /* + * Standard TX Ring mode is used for the allocation of TX ring + * and underlying scheduling resources that allow bandwidth + * reservation and limit settings on the queried function. If + * set to 1, then standard TX ring mode is supported on the + * queried function. If set to 0, then standard TX ring mode is + * not available on the queried function. + */ + #define HWRM_FUNC_QCAPS_OUTPUT_FLAGS_STD_TX_RING_MODE_SUPPORTED \ + UINT32_C(0x800) + uint8_t mac_address[6]; + /* + * This value is current MAC address configured for this + * function. A value of 00-00-00-00-00-00 indicates no MAC + * address is currently configured. + */ + uint16_t max_rsscos_ctx; + /* + * The maximum number of RSS/COS contexts that can be allocated + * to the function. + */ + uint16_t max_cmpl_rings; + /* + * The maximum number of completion rings that can be allocated + * to the function. + */ + uint16_t max_tx_rings; + /* + * The maximum number of transmit rings that can be allocated to + * the function. + */ + uint16_t max_rx_rings; + /* + * The maximum number of receive rings that can be allocated to + * the function. + */ + uint16_t max_l2_ctxs; + /* + * The maximum number of L2 contexts that can be allocated to + * the function. + */ + uint16_t max_vnics; + /* + * The maximum number of VNICs that can be allocated to the + * function. + */ + uint16_t first_vf_id; + /* + * The identifier for the first VF enabled on a PF. This is + * valid only on the PF with SR-IOV enabled. 0xFF... (All Fs) if + * this command is called on a PF with SR-IOV disabled or on a + * VF. + */ + uint16_t max_vfs; + /* + * The maximum number of VFs that can be allocated to the + * function. This is valid only on the PF with SR-IOV enabled. + * 0xFF... (All Fs) if this command is called on a PF with SR- + * IOV disabled or on a VF. + */ + uint16_t max_stat_ctx; + /* + * The maximum number of statistic contexts that can be + * allocated to the function. + */ + uint32_t max_encap_records; + /* + * The maximum number of Encapsulation records that can be + * offloaded by this function. + */ + uint32_t max_decap_records; + /* + * The maximum number of decapsulation records that can be + * offloaded by this function. + */ + uint32_t max_tx_em_flows; + /* + * The maximum number of Exact Match (EM) flows that can be + * offloaded by this function on the TX side. + */ + uint32_t max_tx_wm_flows; + /* + * The maximum number of Wildcard Match (WM) flows that can be + * offloaded by this function on the TX side. + */ + uint32_t max_rx_em_flows; + /* + * The maximum number of Exact Match (EM) flows that can be + * offloaded by this function on the RX side. + */ + uint32_t max_rx_wm_flows; + /* + * The maximum number of Wildcard Match (WM) flows that can be + * offloaded by this function on the RX side. + */ + uint32_t max_mcast_filters; + /* + * The maximum number of multicast filters that can be supported + * by this function on the RX side. + */ + uint32_t max_flow_id; + /* + * The maximum value of flow_id that can be supported in + * completion records. + */ + uint32_t max_hw_ring_grps; + /* + * The maximum number of HW ring groups that can be supported on + * this function. + */ + uint16_t max_sp_tx_rings; + /* + * The maximum number of strict priority transmit rings that can + * be allocated to the function. This number indicates the + * maximum number of TX rings that can be assigned strict + * priorities out of the maximum number of TX rings that can be + * allocated (max_tx_rings) to the function. + */ + uint8_t unused_0; uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* hwrm_func_qcfg */ +/* + * Description: This command returns the current configuration of a function. + * The input FID value is used to indicate what function is being queried. This + * allows a physical function driver to query virtual functions that are + * children of the physical function. The output FID value is needed to + * configure Rings and MSI-X vectors so their DMA operations appear correctly on + * the PCI bus. This command should be called by every driver after + * 'hwrm_func_cfg' to get the actual number of resources allocated by the HWRM. + * The values returned by hwrm_func_qcfg are the values the driver shall use. + * These values may be different than what was originally requested in the + * 'hwrm_func_cfg' command. + */ +/* Input (24 bytes) */ +struct hwrm_func_qcfg_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint16_t fid; + /* + * Function ID of the function that is being queried. 0xFF... + * (All Fs) if the query is for the requesting function. + */ + uint16_t unused_0[3]; +} __attribute__((packed)); + +/* Output (72 bytes) */ +struct hwrm_func_qcfg_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint16_t fid; + /* + * FID value. This value is used to identify operations on the + * PCI bus as belonging to a particular PCI function. + */ + uint16_t port_id; + /* + * Port ID of port that this function is associated with. + * 0xFF... (All Fs) if this function is not associated with any + * port. + */ + uint16_t vlan; + /* + * This value is the current VLAN setting for this function. The + * value of 0 for this field indicates no priority tagging or + * VLAN is used. This field's format is same as 802.1Q Tag's Tag + * Control Information (TCI) format that includes both Priority + * Code Point (PCP) and VLAN Identifier (VID). + */ + uint16_t flags; + /* + * If 1, then magic packet based Out-Of-Box WoL is enabled on + * the port associated with this function. + */ + #define HWRM_FUNC_QCFG_OUTPUT_FLAGS_OOB_WOL_MAGICPKT_ENABLED \ + UINT32_C(0x1) + /* + * If 1, then bitmap pattern based Out-Of-Box WoL packet is + * enabled on the port associated with this function. + */ + #define HWRM_FUNC_QCFG_OUTPUT_FLAGS_OOB_WOL_BMP_ENABLED UINT32_C(0x2) + /* + * If set to 1, then FW based DCBX agent is enabled and running + * on the port associated with this function. If set to 0, then + * DCBX agent is not running in the firmware. + */ + #define HWRM_FUNC_QCFG_OUTPUT_FLAGS_FW_DCBX_AGENT_ENABLED \ + UINT32_C(0x4) + /* + * Standard TX Ring mode is used for the allocation of TX ring + * and underlying scheduling resources that allow bandwidth + * reservation and limit settings on the queried function. If + * set to 1, then standard TX ring mode is enabled on the + * queried function. If set to 0, then the standard TX ring mode + * is disabled on the queried function. In this extended TX ring + * resource mode, the minimum and maximum bandwidth settings are + * not supported to allow the allocation of TX rings to span + * multiple scheduler nodes. + */ + #define HWRM_FUNC_QCFG_OUTPUT_FLAGS_STD_TX_RING_MODE_ENABLED \ + UINT32_C(0x8) + /* + * If set to 1 then FW based LLDP agent is enabled and running + * on the port associated with this function. If set to 0 then + * the LLDP agent is not running in the firmware. + */ + #define HWRM_FUNC_QCFG_OUTPUT_FLAGS_FW_LLDP_AGENT_ENABLED UINT32_C(0x10) + /* + * If set to 1, then multi-host mode is active for this + * function. If set to 0, then multi-host mode is inactive for + * this function or not applicable for this device. + */ + #define HWRM_FUNC_QCFG_OUTPUT_FLAGS_MULTI_HOST UINT32_C(0x20) + uint8_t mac_address[6]; + /* + * This value is current MAC address configured for this + * function. A value of 00-00-00-00-00-00 indicates no MAC + * address is currently configured. + */ + uint16_t pci_id; + /* + * This value is current PCI ID of this function. If ARI is + * enabled, then it is Bus Number (8b):Function Number(8b). + * Otherwise, it is Bus Number (8b):Device Number (4b):Function + * Number(4b). + */ + uint16_t alloc_rsscos_ctx; + /* + * The number of RSS/COS contexts currently allocated to the + * function. + */ + uint16_t alloc_cmpl_rings; + /* + * The number of completion rings currently allocated to the + * function. This does not include the rings allocated to any + * children functions if any. + */ + uint16_t alloc_tx_rings; + /* + * The number of transmit rings currently allocated to the + * function. This does not include the rings allocated to any + * children functions if any. + */ + uint16_t alloc_rx_rings; + /* + * The number of receive rings currently allocated to the + * function. This does not include the rings allocated to any + * children functions if any. + */ + uint16_t alloc_l2_ctx; + /* The allocated number of L2 contexts to the function. */ + uint16_t alloc_vnics; + /* The allocated number of vnics to the function. */ + uint16_t mtu; + /* + * The maximum transmission unit of the function. For rings + * allocated on this function, this default value is used if + * ring MTU is not specified. + */ + uint16_t mru; + /* + * The maximum receive unit of the function. For vnics allocated + * on this function, this default value is used if vnic MRU is + * not specified. + */ + uint16_t stat_ctx_id; + /* The statistics context assigned to a function. */ + uint8_t port_partition_type; + /* + * The HWRM shall return Unknown value for this field when this + * command is used to query VF's configuration. + */ + /* Single physical function */ + #define HWRM_FUNC_QCFG_OUTPUT_PORT_PARTITION_TYPE_SPF UINT32_C(0x0) + /* Multiple physical functions */ + #define HWRM_FUNC_QCFG_OUTPUT_PORT_PARTITION_TYPE_MPFS UINT32_C(0x1) + /* Network Partitioning 1.0 */ + #define HWRM_FUNC_QCFG_OUTPUT_PORT_PARTITION_TYPE_NPAR1_0 UINT32_C(0x2) + /* Network Partitioning 1.5 */ + #define HWRM_FUNC_QCFG_OUTPUT_PORT_PARTITION_TYPE_NPAR1_5 UINT32_C(0x3) + /* Network Partitioning 2.0 */ + #define HWRM_FUNC_QCFG_OUTPUT_PORT_PARTITION_TYPE_NPAR2_0 UINT32_C(0x4) + /* Unknown */ + #define HWRM_FUNC_QCFG_OUTPUT_PORT_PARTITION_TYPE_UNKNOWN UINT32_C(0xff) + uint8_t port_pf_cnt; + /* + * This field will indicate number of physical functions on this + * port_partition. HWRM shall return unavail (i.e. value of 0) + * for this field when this command is used to query VF's + * configuration or from older firmware that doesn't support + * this field. + */ + /* number of PFs is not available */ + #define HWRM_FUNC_QCFG_OUTPUT_PORT_PF_CNT_UNAVAIL UINT32_C(0x0) + uint16_t dflt_vnic_id; + /* The default VNIC ID assigned to a function that is being queried. */ + uint16_t max_mtu_configured; + /* + * This value specifies the MAX MTU that can be configured by + * host drivers. This 'max_mtu_configure' can be HW max MTU or + * OEM applications specified value. Host drivers can't + * configure the MTU greater than this value. Host drivers + * should read this value prior to configuring the MTU. FW will + * fail the host request with MTU greater than + * 'max_mtu_configured'. + */ + uint32_t min_bw; + /* + * Minimum BW allocated for this function. The HWRM will + * translate this value into byte counter and time interval used + * for the scheduler inside the device. A value of 0 indicates + * the minimum bandwidth is not configured. + */ + /* The bandwidth value. */ + #define HWRM_FUNC_QCFG_OUTPUT_MIN_BW_BW_VALUE_MASK UINT32_C(0xfffffff) + #define HWRM_FUNC_QCFG_OUTPUT_MIN_BW_BW_VALUE_SFT 0 + /* The granularity of the value (bits or bytes). */ + #define HWRM_FUNC_QCFG_OUTPUT_MIN_BW_SCALE UINT32_C(0x10000000) + /* Value is in bits. */ + #define HWRM_FUNC_QCFG_OUTPUT_MIN_BW_SCALE_BITS (UINT32_C(0x0) << 28) + /* Value is in bytes. */ + #define HWRM_FUNC_QCFG_OUTPUT_MIN_BW_SCALE_BYTES \ + (UINT32_C(0x1) << 28) + #define HWRM_FUNC_QCFG_OUTPUT_MIN_BW_SCALE_LAST \ + FUNC_QCFG_OUTPUT_MIN_BW_SCALE_BYTES + /* bw_value_unit is 3 b */ + #define HWRM_FUNC_QCFG_OUTPUT_MIN_BW_BW_VALUE_UNIT_MASK \ + UINT32_C(0xe0000000) + #define HWRM_FUNC_QCFG_OUTPUT_MIN_BW_BW_VALUE_UNIT_SFT 29 + /* Value is in Mb or MB (base 10). */ + #define HWRM_FUNC_QCFG_OUTPUT_MIN_BW_BW_VALUE_UNIT_MEGA \ + (UINT32_C(0x0) << 29) + /* Value is in Kb or KB (base 10). */ + #define HWRM_FUNC_QCFG_OUTPUT_MIN_BW_BW_VALUE_UNIT_KILO \ + (UINT32_C(0x2) << 29) + /* Value is in bits or bytes. */ + #define HWRM_FUNC_QCFG_OUTPUT_MIN_BW_BW_VALUE_UNIT_BASE \ + (UINT32_C(0x4) << 29) + /* Value is in Gb or GB (base 10). */ + #define HWRM_FUNC_QCFG_OUTPUT_MIN_BW_BW_VALUE_UNIT_GIGA \ + (UINT32_C(0x6) << 29) + /* Value is in 1/100th of a percentage of total bandwidth. */ + #define HWRM_FUNC_QCFG_OUTPUT_MIN_BW_BW_VALUE_UNIT_PERCENT1_100 \ + (UINT32_C(0x1) << 29) + /* Invalid unit */ + #define HWRM_FUNC_QCFG_OUTPUT_MIN_BW_BW_VALUE_UNIT_INVALID \ + (UINT32_C(0x7) << 29) + #define HWRM_FUNC_QCFG_OUTPUT_MIN_BW_BW_VALUE_UNIT_LAST \ + FUNC_QCFG_OUTPUT_MIN_BW_BW_VALUE_UNIT_INVALID + uint32_t max_bw; + /* + * Maximum BW allocated for this function. The HWRM will + * translate this value into byte counter and time interval used + * for the scheduler inside the device. A value of 0 indicates + * that the maximum bandwidth is not configured. + */ + /* The bandwidth value. */ + #define HWRM_FUNC_QCFG_OUTPUT_MAX_BW_BW_VALUE_MASK UINT32_C(0xfffffff) + #define HWRM_FUNC_QCFG_OUTPUT_MAX_BW_BW_VALUE_SFT 0 + /* The granularity of the value (bits or bytes). */ + #define HWRM_FUNC_QCFG_OUTPUT_MAX_BW_SCALE UINT32_C(0x10000000) + /* Value is in bits. */ + #define HWRM_FUNC_QCFG_OUTPUT_MAX_BW_SCALE_BITS (UINT32_C(0x0) << 28) + /* Value is in bytes. */ + #define HWRM_FUNC_QCFG_OUTPUT_MAX_BW_SCALE_BYTES \ + (UINT32_C(0x1) << 28) + #define HWRM_FUNC_QCFG_OUTPUT_MAX_BW_SCALE_LAST \ + FUNC_QCFG_OUTPUT_MAX_BW_SCALE_BYTES + /* bw_value_unit is 3 b */ + #define HWRM_FUNC_QCFG_OUTPUT_MAX_BW_BW_VALUE_UNIT_MASK \ + UINT32_C(0xe0000000) + #define HWRM_FUNC_QCFG_OUTPUT_MAX_BW_BW_VALUE_UNIT_SFT 29 + /* Value is in Mb or MB (base 10). */ + #define HWRM_FUNC_QCFG_OUTPUT_MAX_BW_BW_VALUE_UNIT_MEGA \ + (UINT32_C(0x0) << 29) + /* Value is in Kb or KB (base 10). */ + #define HWRM_FUNC_QCFG_OUTPUT_MAX_BW_BW_VALUE_UNIT_KILO \ + (UINT32_C(0x2) << 29) + /* Value is in bits or bytes. */ + #define HWRM_FUNC_QCFG_OUTPUT_MAX_BW_BW_VALUE_UNIT_BASE \ + (UINT32_C(0x4) << 29) + /* Value is in Gb or GB (base 10). */ + #define HWRM_FUNC_QCFG_OUTPUT_MAX_BW_BW_VALUE_UNIT_GIGA \ + (UINT32_C(0x6) << 29) + /* Value is in 1/100th of a percentage of total bandwidth. */ + #define HWRM_FUNC_QCFG_OUTPUT_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 \ + (UINT32_C(0x1) << 29) + /* Invalid unit */ + #define HWRM_FUNC_QCFG_OUTPUT_MAX_BW_BW_VALUE_UNIT_INVALID \ + (UINT32_C(0x7) << 29) + #define HWRM_FUNC_QCFG_OUTPUT_MAX_BW_BW_VALUE_UNIT_LAST \ + FUNC_QCFG_OUTPUT_MAX_BW_BW_VALUE_UNIT_INVALID + uint8_t evb_mode; + /* + * This value indicates the Edge virtual bridge mode for the + * domain that this function belongs to. + */ + /* No Edge Virtual Bridging (EVB) */ + #define HWRM_FUNC_QCFG_OUTPUT_EVB_MODE_NO_EVB UINT32_C(0x0) + /* Virtual Ethernet Bridge (VEB) */ + #define HWRM_FUNC_QCFG_OUTPUT_EVB_MODE_VEB UINT32_C(0x1) + /* Virtual Ethernet Port Aggregator (VEPA) */ + #define HWRM_FUNC_QCFG_OUTPUT_EVB_MODE_VEPA UINT32_C(0x2) + uint8_t unused_0; + uint16_t alloc_vfs; + /* + * The number of VFs that are allocated to the function. This is + * valid only on the PF with SR-IOV enabled. 0xFF... (All Fs) if + * this command is called on a PF with SR-IOV disabled or on a + * VF. + */ + uint32_t alloc_mcast_filters; + /* + * The number of allocated multicast filters for this function + * on the RX side. + */ + uint32_t alloc_hw_ring_grps; + /* The number of allocated HW ring groups for this function. */ + uint16_t alloc_sp_tx_rings; + /* + * The number of strict priority transmit rings out of currently + * allocated TX rings to the function (alloc_tx_rings). + */ + uint8_t unused_1; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* hwrm_func_vlan_qcfg */ +/* + * Description: This command should be called by PF driver to get the current + * C-TAG, S-TAG and correcponsing PCP and TPID values configured for the + * function. + */ +/* Input (24 bytes) */ +struct hwrm_func_vlan_qcfg_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint16_t fid; + /* + * Function ID of the function that is being configured. If set + * to 0xFF... (All Fs), then the configuration is for the + * requesting function. + */ + uint16_t unused_0[3]; +}; + +/* Output (40 bytes) */ +struct hwrm_func_vlan_qcfg_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint32_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t unused_3; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ + uint16_t stag_vid; + /* S-TAG VLAN identifier configured for the function. */ + uint8_t stag_pcp; + /* S-TAG PCP value configured for the function. */ + uint8_t unused_4; + uint16_t stag_tpid; + /* + * S-TAG TPID value configured for the function. This field is + * specified in network byte order. + */ + uint16_t ctag_vid; + /* C-TAG VLAN identifier configured for the function. */ + uint8_t ctag_pcp; + /* C-TAG PCP value configured for the function. */ + uint8_t unused_5; + uint16_t ctag_tpid; + /* + * C-TAG TPID value configured for the function. This field is + * specified in network byte order. + */ + uint32_t rsvd2; + /* Future use. */ + uint32_t rsvd3; + /* Future use. */ + uint32_t unused_6; +}; + +/* hwrm_func_vlan_cfg */ +/* + * Description: This command allows PF driver to configure C-TAG, S-TAG and + * corresponding PCP and TPID values for a function. + */ +/* Input (48 bytes) */ +struct hwrm_func_vlan_cfg_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint16_t fid; + /* + * Function ID of the function that is being configured. If set + * to 0xFF... (All Fs), then the configuration is for the + * requesting function. + */ + uint8_t unused_0; + uint8_t unused_1; + uint32_t enables; + /* This bit must be '1' for the stag_vid field to be configured. */ + #define HWRM_FUNC_VLAN_CFG_INPUT_ENABLES_STAG_VID UINT32_C(0x1) + /* This bit must be '1' for the ctag_vid field to be configured. */ + #define HWRM_FUNC_VLAN_CFG_INPUT_ENABLES_CTAG_VID UINT32_C(0x2) + /* This bit must be '1' for the stag_pcp field to be configured. */ + #define HWRM_FUNC_VLAN_CFG_INPUT_ENABLES_STAG_PCP UINT32_C(0x4) + /* This bit must be '1' for the ctag_pcp field to be configured. */ + #define HWRM_FUNC_VLAN_CFG_INPUT_ENABLES_CTAG_PCP UINT32_C(0x8) + /* This bit must be '1' for the stag_tpid field to be configured. */ + #define HWRM_FUNC_VLAN_CFG_INPUT_ENABLES_STAG_TPID UINT32_C(0x10) + /* This bit must be '1' for the ctag_tpid field to be configured. */ + #define HWRM_FUNC_VLAN_CFG_INPUT_ENABLES_CTAG_TPID UINT32_C(0x20) + uint16_t stag_vid; + /* S-TAG VLAN identifier configured for the function. */ + uint8_t stag_pcp; + /* S-TAG PCP value configured for the function. */ + uint8_t unused_2; + uint16_t stag_tpid; + /* + * S-TAG TPID value configured for the function. This field is + * specified in network byte order. + */ + uint16_t ctag_vid; + /* C-TAG VLAN identifier configured for the function. */ + uint8_t ctag_pcp; + /* C-TAG PCP value configured for the function. */ + uint8_t unused_3; + uint16_t ctag_tpid; + /* + * C-TAG TPID value configured for the function. This field is + * specified in network byte order. + */ + uint32_t rsvd1; + /* Future use. */ + uint32_t rsvd2; + /* Future use. */ + uint32_t unused_4; +}; + +/* Output (16 bytes) */ +struct hwrm_func_vlan_cfg_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint32_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t unused_3; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +}; + +/* hwrm_func_cfg */ +/* + * Description: This command allows configuration of a PF by the corresponding + * PF driver. This command also allows configuration of a child VF by its parent + * PF driver. The input FID value is used to indicate what function is being + * configured. This allows a PF driver to configure the PF owned by itself or a + * virtual function that is a child of the PF. This command allows to reserve + * resources for a VF by its parent PF. To reverse the process, the command + * should be called with all enables flags cleared for resources. This will free + * allocated resources for the VF and return them to the resource pool. If this + * command is requested by a VF driver to configure or reserve resources, then + * the HWRM shall fail this command. If default MAC address and/or VLAN are + * provided in this command, then the HWRM shall set up appropriate MAC/VLAN + * filters for the function that is being configured. If source properties + * checks are enabled and default MAC address and/or IP address are provided in + * this command, then the HWRM shall set appropriate source property checks + * based on provided MAC and/or IP addresses. The parent PF driver should not + * set MTU/MRU for a VF using this command. This is to allow MTU/MRU setting by + * the VF driver. If the MTU or MRU for a VF is set by the PF driver, then the + * HWRM should ignore it. A function's MTU/MRU should be set prior to allocating + * RX VNICs or TX rings. A PF driver calls hwrm_func_cfg to allocate resources + * for itself or its children VFs. All function drivers shall call hwrm_func_cfg + * to reserve resources. A request to hwrm_func_cfg may not be fully granted; + * that is, a request for resources may be larger than what can be supported by + * the device and the HWRM will allocate the best set of resources available, + * but that may be less than requested. If all the amounts requested could not + * be fulfilled, the HWRM shall allocate what it could and return a status code + * of success. A function driver should call hwrm_func_qcfg immediately after + * hwrm_func_cfg to determine what resources were assigned to the configured + * function. A call by a PF driver to hwrm_func_cfg to allocate resources for + * itself shall only allocate resources for the PF driver to use, not for its + * children VFs. Likewise, a call to hwrm_func_qcfg shall return the resources + * available for the PF driver to use, not what is available to its children + * VFs. + */ +/* Input (88 bytes) */ +struct hwrm_func_cfg_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint16_t fid; + /* + * Function ID of the function that is being configured. If set + * to 0xFF... (All Fs), then the the configuration is for the + * requesting function. + */ + uint8_t unused_0; + uint8_t unused_1; + uint32_t flags; + /* + * When this bit is '1', the function is disabled with source + * MAC address check. This is an anti-spoofing check. If this + * flag is set, then the function shall be configured to + * disallow transmission of frames with the source MAC address + * that is configured for this function. + */ + #define HWRM_FUNC_CFG_INPUT_FLAGS_SRC_MAC_ADDR_CHECK_DISABLE \ + UINT32_C(0x1) + /* + * When this bit is '1', the function is enabled with source MAC + * address check. This is an anti-spoofing check. If this flag + * is set, then the function shall be configured to allow + * transmission of frames with the source MAC address that is + * configured for this function. + */ + #define HWRM_FUNC_CFG_INPUT_FLAGS_SRC_MAC_ADDR_CHECK_ENABLE \ + UINT32_C(0x2) + /* reserved */ + #define HWRM_FUNC_CFG_INPUT_FLAGS_RSVD_MASK UINT32_C(0x1fc) + #define HWRM_FUNC_CFG_INPUT_FLAGS_RSVD_SFT 2 + /* + * Standard TX Ring mode is used for the allocation of TX ring + * and underlying scheduling resources that allow bandwidth + * reservation and limit settings on the queried function. If + * set to 1, then standard TX ring mode is requested to be + * enabled on the function being configured. + */ + #define HWRM_FUNC_CFG_INPUT_FLAGS_STD_TX_RING_MODE_ENABLE \ + UINT32_C(0x200) + /* + * Standard TX Ring mode is used for the allocation of TX ring + * and underlying scheduling resources that allow bandwidth + * reservation and limit settings on the queried function. If + * set to 1, then the standard TX ring mode is requested to be + * disabled on the function being configured. In this extended + * TX ring resource mode, the minimum and maximum bandwidth + * settings are not supported to allow the allocation of TX + * rings to span multiple scheduler nodes. + */ + #define HWRM_FUNC_CFG_INPUT_FLAGS_STD_TX_RING_MODE_DISABLE \ + UINT32_C(0x400) + /* + * If this bit is set, virtual mac address configured in this + * command will be persistent over warm boot. + */ + #define HWRM_FUNC_CFG_INPUT_FLAGS_VIRT_MAC_PERSIST UINT32_C(0x800) + /* + * This bit only applies to the VF. If this bit is set, the + * statistic context counters will not be cleared when the + * statistic context is freed or a function reset is called on + * VF. This bit will be cleared when the PF is unloaded or a + * function reset is called on the PF. + */ + #define HWRM_FUNC_CFG_INPUT_FLAGS_NO_AUTOCLEAR_STATISTIC \ + UINT32_C(0x1000) + /* + * This bit requests that the firmware test to see if all the + * assets requested in this command (i.e. number of TX rings) + * are available. The firmware will return an error if the + * requested assets are not available. The firwmare will NOT + * reserve the assets if they are available. + */ + #define HWRM_FUNC_CFG_INPUT_FLAGS_TX_ASSETS_TEST UINT32_C(0x2000) + uint32_t enables; + /* This bit must be '1' for the mtu field to be configured. */ + #define HWRM_FUNC_CFG_INPUT_ENABLES_MTU UINT32_C(0x1) + /* This bit must be '1' for the mru field to be configured. */ + #define HWRM_FUNC_CFG_INPUT_ENABLES_MRU UINT32_C(0x2) + /* + * This bit must be '1' for the num_rsscos_ctxs field to be + * configured. + */ + #define HWRM_FUNC_CFG_INPUT_ENABLES_NUM_RSSCOS_CTXS UINT32_C(0x4) + /* + * This bit must be '1' for the num_cmpl_rings field to be + * configured. + */ + #define HWRM_FUNC_CFG_INPUT_ENABLES_NUM_CMPL_RINGS UINT32_C(0x8) + /* This bit must be '1' for the num_tx_rings field to be configured. */ + #define HWRM_FUNC_CFG_INPUT_ENABLES_NUM_TX_RINGS UINT32_C(0x10) + /* This bit must be '1' for the num_rx_rings field to be configured. */ + #define HWRM_FUNC_CFG_INPUT_ENABLES_NUM_RX_RINGS UINT32_C(0x20) + /* This bit must be '1' for the num_l2_ctxs field to be configured. */ + #define HWRM_FUNC_CFG_INPUT_ENABLES_NUM_L2_CTXS UINT32_C(0x40) + /* This bit must be '1' for the num_vnics field to be configured. */ + #define HWRM_FUNC_CFG_INPUT_ENABLES_NUM_VNICS UINT32_C(0x80) + /* + * This bit must be '1' for the num_stat_ctxs field to be + * configured. + */ + #define HWRM_FUNC_CFG_INPUT_ENABLES_NUM_STAT_CTXS UINT32_C(0x100) + /* + * This bit must be '1' for the dflt_mac_addr field to be + * configured. + */ + #define HWRM_FUNC_CFG_INPUT_ENABLES_DFLT_MAC_ADDR UINT32_C(0x200) + /* This bit must be '1' for the dflt_vlan field to be configured. */ + #define HWRM_FUNC_CFG_INPUT_ENABLES_DFLT_VLAN UINT32_C(0x400) + /* This bit must be '1' for the dflt_ip_addr field to be configured. */ + #define HWRM_FUNC_CFG_INPUT_ENABLES_DFLT_IP_ADDR UINT32_C(0x800) + /* This bit must be '1' for the min_bw field to be configured. */ + #define HWRM_FUNC_CFG_INPUT_ENABLES_MIN_BW UINT32_C(0x1000) + /* This bit must be '1' for the max_bw field to be configured. */ + #define HWRM_FUNC_CFG_INPUT_ENABLES_MAX_BW UINT32_C(0x2000) + /* + * This bit must be '1' for the async_event_cr field to be + * configured. + */ + #define HWRM_FUNC_CFG_INPUT_ENABLES_ASYNC_EVENT_CR UINT32_C(0x4000) + /* + * This bit must be '1' for the vlan_antispoof_mode field to be + * configured. + */ + #define HWRM_FUNC_CFG_INPUT_ENABLES_VLAN_ANTISPOOF_MODE UINT32_C(0x8000) + /* + * This bit must be '1' for the allowed_vlan_pris field to be + * configured. + */ + #define HWRM_FUNC_CFG_INPUT_ENABLES_ALLOWED_VLAN_PRIS UINT32_C(0x10000) + /* This bit must be '1' for the evb_mode field to be configured. */ + #define HWRM_FUNC_CFG_INPUT_ENABLES_EVB_MODE UINT32_C(0x20000) + /* + * This bit must be '1' for the num_mcast_filters field to be + * configured. + */ + #define HWRM_FUNC_CFG_INPUT_ENABLES_NUM_MCAST_FILTERS UINT32_C(0x40000) + /* + * This bit must be '1' for the num_hw_ring_grps field to be + * configured. + */ + #define HWRM_FUNC_CFG_INPUT_ENABLES_NUM_HW_RING_GRPS UINT32_C(0x80000) + uint16_t mtu; + /* + * The maximum transmission unit of the function. The HWRM + * should make sure that the mtu of the function does not exceed + * the mtu of the physical port that this function is associated + * with. In addition to configuring mtu per function, it is + * possible to configure mtu per transmit ring. By default, the + * mtu of each transmit ring associated with a function is equal + * to the mtu of the function. The HWRM should make sure that + * the mtu of each transmit ring that is assigned to a function + * has a valid mtu. + */ + uint16_t mru; + /* + * The maximum receive unit of the function. The HWRM should + * make sure that the mru of the function does not exceed the + * mru of the physical port that this function is associated + * with. In addition to configuring mru per function, it is + * possible to configure mru per vnic. By default, the mru of + * each vnic associated with a function is equal to the mru of + * the function. The HWRM should make sure that the mru of each + * vnic that is assigned to a function has a valid mru. + */ + uint16_t num_rsscos_ctxs; + /* The number of RSS/COS contexts requested for the function. */ + uint16_t num_cmpl_rings; + /* + * The number of completion rings requested for the function. + * This does not include the rings allocated to any children + * functions if any. + */ + uint16_t num_tx_rings; + /* + * The number of transmit rings requested for the function. This + * does not include the rings allocated to any children + * functions if any. + */ + uint16_t num_rx_rings; + /* + * The number of receive rings requested for the function. This + * does not include the rings allocated to any children + * functions if any. + */ + uint16_t num_l2_ctxs; + /* The requested number of L2 contexts for the function. */ + uint16_t num_vnics; + /* The requested number of vnics for the function. */ + uint16_t num_stat_ctxs; + /* The requested number of statistic contexts for the function. */ + uint16_t num_hw_ring_grps; + /* + * The number of HW ring groups that should be reserved for this + * function. + */ + uint8_t dflt_mac_addr[6]; + /* The default MAC address for the function being configured. */ + uint16_t dflt_vlan; + /* + * The default VLAN for the function being configured. This + * field's format is same as 802.1Q Tag's Tag Control + * Information (TCI) format that includes both Priority Code + * Point (PCP) and VLAN Identifier (VID). + */ + uint32_t dflt_ip_addr[4]; + /* + * The default IP address for the function being configured. + * This address is only used in enabling source property check. + */ + uint32_t min_bw; + /* + * Minimum BW allocated for this function. The HWRM will + * translate this value into byte counter and time interval used + * for the scheduler inside the device. + */ + /* The bandwidth value. */ + #define HWRM_FUNC_CFG_INPUT_MIN_BW_BW_VALUE_MASK UINT32_C(0xfffffff) + #define HWRM_FUNC_CFG_INPUT_MIN_BW_BW_VALUE_SFT 0 + /* The granularity of the value (bits or bytes). */ + #define HWRM_FUNC_CFG_INPUT_MIN_BW_SCALE UINT32_C(0x10000000) + /* Value is in bits. */ + #define HWRM_FUNC_CFG_INPUT_MIN_BW_SCALE_BITS (UINT32_C(0x0) << 28) + /* Value is in bytes. */ + #define HWRM_FUNC_CFG_INPUT_MIN_BW_SCALE_BYTES (UINT32_C(0x1) << 28) + #define HWRM_FUNC_CFG_INPUT_MIN_BW_SCALE_LAST \ + FUNC_CFG_INPUT_MIN_BW_SCALE_BYTES + /* bw_value_unit is 3 b */ + #define HWRM_FUNC_CFG_INPUT_MIN_BW_BW_VALUE_UNIT_MASK \ + UINT32_C(0xe0000000) + #define HWRM_FUNC_CFG_INPUT_MIN_BW_BW_VALUE_UNIT_SFT 29 + /* Value is in Mb or MB (base 10). */ + #define HWRM_FUNC_CFG_INPUT_MIN_BW_BW_VALUE_UNIT_MEGA \ + (UINT32_C(0x0) << 29) + /* Value is in Kb or KB (base 10). */ + #define HWRM_FUNC_CFG_INPUT_MIN_BW_BW_VALUE_UNIT_KILO \ + (UINT32_C(0x2) << 29) + /* Value is in bits or bytes. */ + #define HWRM_FUNC_CFG_INPUT_MIN_BW_BW_VALUE_UNIT_BASE \ + (UINT32_C(0x4) << 29) + /* Value is in Gb or GB (base 10). */ + #define HWRM_FUNC_CFG_INPUT_MIN_BW_BW_VALUE_UNIT_GIGA \ + (UINT32_C(0x6) << 29) + /* Value is in 1/100th of a percentage of total bandwidth. */ + #define HWRM_FUNC_CFG_INPUT_MIN_BW_BW_VALUE_UNIT_PERCENT1_100 \ + (UINT32_C(0x1) << 29) + /* Invalid unit */ + #define HWRM_FUNC_CFG_INPUT_MIN_BW_BW_VALUE_UNIT_INVALID \ + (UINT32_C(0x7) << 29) + #define HWRM_FUNC_CFG_INPUT_MIN_BW_BW_VALUE_UNIT_LAST \ + FUNC_CFG_INPUT_MIN_BW_BW_VALUE_UNIT_INVALID + uint32_t max_bw; + /* + * Maximum BW allocated for this function. The HWRM will + * translate this value into byte counter and time interval used + * for the scheduler inside the device. + */ + /* The bandwidth value. */ + #define HWRM_FUNC_CFG_INPUT_MAX_BW_BW_VALUE_MASK \ + UINT32_C(0xfffffff) + #define HWRM_FUNC_CFG_INPUT_MAX_BW_BW_VALUE_SFT 0 + /* The granularity of the value (bits or bytes). */ + #define HWRM_FUNC_CFG_INPUT_MAX_BW_SCALE UINT32_C(0x10000000) + /* Value is in bits. */ + #define HWRM_FUNC_CFG_INPUT_MAX_BW_SCALE_BITS (UINT32_C(0x0) << 28) + /* Value is in bytes. */ + #define HWRM_FUNC_CFG_INPUT_MAX_BW_SCALE_BYTES (UINT32_C(0x1) << 28) + #define HWRM_FUNC_CFG_INPUT_MAX_BW_SCALE_LAST \ + FUNC_CFG_INPUT_MAX_BW_SCALE_BYTES + /* bw_value_unit is 3 b */ + #define HWRM_FUNC_CFG_INPUT_MAX_BW_BW_VALUE_UNIT_MASK \ + UINT32_C(0xe0000000) + #define HWRM_FUNC_CFG_INPUT_MAX_BW_BW_VALUE_UNIT_SFT 29 + /* Value is in Mb or MB (base 10). */ + #define HWRM_FUNC_CFG_INPUT_MAX_BW_BW_VALUE_UNIT_MEGA \ + (UINT32_C(0x0) << 29) + /* Value is in Kb or KB (base 10). */ + #define HWRM_FUNC_CFG_INPUT_MAX_BW_BW_VALUE_UNIT_KILO \ + (UINT32_C(0x2) << 29) + /* Value is in bits or bytes. */ + #define HWRM_FUNC_CFG_INPUT_MAX_BW_BW_VALUE_UNIT_BASE \ + (UINT32_C(0x4) << 29) + /* Value is in Gb or GB (base 10). */ + #define HWRM_FUNC_CFG_INPUT_MAX_BW_BW_VALUE_UNIT_GIGA \ + (UINT32_C(0x6) << 29) + /* Value is in 1/100th of a percentage of total bandwidth. */ + #define HWRM_FUNC_CFG_INPUT_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 \ + (UINT32_C(0x1) << 29) + /* Invalid unit */ + #define HWRM_FUNC_CFG_INPUT_MAX_BW_BW_VALUE_UNIT_INVALID \ + (UINT32_C(0x7) << 29) + #define HWRM_FUNC_CFG_INPUT_MAX_BW_BW_VALUE_UNIT_LAST \ + FUNC_CFG_INPUT_MAX_BW_BW_VALUE_UNIT_INVALID + uint16_t async_event_cr; + /* + * ID of the target completion ring for receiving asynchronous + * event completions. If this field is not valid, then the HWRM + * shall use the default completion ring of the function that is + * being configured as the target completion ring for providing + * any asynchronous event completions for that function. If this + * field is valid, then the HWRM shall use the completion ring + * identified by this ID as the target completion ring for + * providing any asynchronous event completions for the function + * that is being configured. + */ + uint8_t vlan_antispoof_mode; + /* VLAN Anti-spoofing mode. */ + /* No VLAN anti-spoofing checks are enabled */ + #define HWRM_FUNC_CFG_INPUT_VLAN_ANTISPOOF_MODE_NOCHECK UINT32_C(0x0) + /* Validate VLAN against the configured VLAN(s) */ + #define HWRM_FUNC_CFG_INPUT_VLAN_ANTISPOOF_MODE_VALIDATE_VLAN \ + UINT32_C(0x1) + /* Insert VLAN if it does not exist, otherwise discard */ + #define HWRM_FUNC_CFG_INPUT_VLAN_ANTISPOOF_MODE_INSERT_IF_VLANDNE \ + UINT32_C(0x2) + /* + * Insert VLAN if it does not exist, override + * VLAN if it exists + */ + #define \ + HWRM_FUNC_CFG_INPUT_VLAN_ANTISPOOF_MODE_INSERT_OR_OVERRIDE_VLAN \ + UINT32_C(0x3) + uint8_t allowed_vlan_pris; + /* + * This bit field defines VLAN PRIs that are allowed on this + * function. If nth bit is set, then VLAN PRI n is allowed on + * this function. + */ + uint8_t evb_mode; + /* + * The HWRM shall allow a PF driver to change EVB mode for the + * partition it belongs to. The HWRM shall not allow a VF driver + * to change the EVB mode. The HWRM shall take into account the + * switching of EVB mode from one to another and reconfigure + * hardware resources as appropriately. The switching from VEB + * to VEPA mode requires the disabling of the loopback traffic. + * Additionally, source knock outs are handled differently in + * VEB and VEPA modes. + */ + /* No Edge Virtual Bridging (EVB) */ + #define HWRM_FUNC_CFG_INPUT_EVB_MODE_NO_EVB UINT32_C(0x0) + /* Virtual Ethernet Bridge (VEB) */ + #define HWRM_FUNC_CFG_INPUT_EVB_MODE_VEB UINT32_C(0x1) + /* Virtual Ethernet Port Aggregator (VEPA) */ + #define HWRM_FUNC_CFG_INPUT_EVB_MODE_VEPA UINT32_C(0x2) + uint8_t unused_2; + uint16_t num_mcast_filters; + /* + * The number of multicast filters that should be reserved for + * this function on the RX side. + */ +} __attribute__((packed)); + +/* Output (16 bytes) */ +struct hwrm_func_cfg_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint32_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t unused_3; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* hwrm_func_qstats */ +/* + * Description: This command returns statistics of a function. The input FID + * value is used to indicate what function is being queried. This allows a + * physical function driver to query virtual functions that are children of the + * physical function. The HWRM shall return any unsupported counter with a value + * of 0xFFFFFFFF for 32-bit counters and 0xFFFFFFFFFFFFFFFF for 64-bit counters. + */ +/* Input (24 bytes) */ +struct hwrm_func_qstats_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint16_t fid; + /* + * Function ID of the function that is being queried. 0xFF... + * (All Fs) if the query is for the requesting function. + */ + uint16_t unused_0[3]; +} __attribute__((packed)); + +/* Output (176 bytes) */ +struct hwrm_func_qstats_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint64_t tx_ucast_pkts; + /* Number of transmitted unicast packets on the function. */ + uint64_t tx_mcast_pkts; + /* Number of transmitted multicast packets on the function. */ + uint64_t tx_bcast_pkts; + /* Number of transmitted broadcast packets on the function. */ + uint64_t tx_err_pkts; + /* + * Number of transmitted packets that were discarded due to + * internal NIC resource problems. For transmit, this can only + * happen if TMP is configured to allow dropping in HOL blocking + * conditions, which is not a normal configuration. + */ + uint64_t tx_drop_pkts; + /* + * Number of dropped packets on transmit path on the function. + * These are packets that have been marked for drop by the TE + * CFA block or are packets that exceeded the transmit MTU limit + * for the function. + */ + uint64_t tx_ucast_bytes; + /* Number of transmitted bytes for unicast traffic on the function. */ + uint64_t tx_mcast_bytes; + /* + * Number of transmitted bytes for multicast traffic on the + * function. + */ + uint64_t tx_bcast_bytes; + /* + * Number of transmitted bytes for broadcast traffic on the + * function. + */ + uint64_t rx_ucast_pkts; + /* Number of received unicast packets on the function. */ + uint64_t rx_mcast_pkts; + /* Number of received multicast packets on the function. */ + uint64_t rx_bcast_pkts; + /* Number of received broadcast packets on the function. */ + uint64_t rx_err_pkts; + /* + * Number of received packets that were discarded on the + * function due to resource limitations. This can happen for 3 + * reasons. # The BD used for the packet has a bad format. # + * There were no BDs available in the ring for the packet. # + * There were no BDs available on-chip for the packet. + */ + uint64_t rx_drop_pkts; + /* + * Number of dropped packets on received path on the function. + * These are packets that have been marked for drop by the RE + * CFA. + */ + uint64_t rx_ucast_bytes; + /* Number of received bytes for unicast traffic on the function. */ + uint64_t rx_mcast_bytes; + /* Number of received bytes for multicast traffic on the function. */ + uint64_t rx_bcast_bytes; + /* Number of received bytes for broadcast traffic on the function. */ + uint64_t rx_agg_pkts; + /* Number of aggregated unicast packets on the function. */ + uint64_t rx_agg_bytes; + /* Number of aggregated unicast bytes on the function. */ + uint64_t rx_agg_events; + /* Number of aggregation events on the function. */ + uint64_t rx_agg_aborts; + /* Number of aborted aggregations on the function. */ + uint32_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t unused_3; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* hwrm_func_clr_stats */ +/* + * Description: This command clears statistics of a function. The input FID + * value is used to indicate what function's statistics is being cleared. This + * allows a physical function driver to clear statistics of virtual functions + * that are children of the physical function. + */ +/* Input (24 bytes) */ +struct hwrm_func_clr_stats_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint16_t fid; + /* + * Function ID of the function. 0xFF... (All Fs) if the query is + * for the requesting function. + */ + uint16_t unused_0[3]; +} __attribute__((packed)); + +/* Output (16 bytes) */ +struct hwrm_func_clr_stats_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint32_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t unused_3; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* hwrm_func_vf_vnic_ids_query */ +/* Description: This command is used to query vf vnic ids. */ +/* Input (32 bytes) */ +struct hwrm_func_vf_vnic_ids_query_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint16_t vf_id; + /* + * This value is used to identify a Virtual Function (VF). The + * scope of VF ID is local within a PF. + */ + uint8_t unused_0; + uint8_t unused_1; + uint32_t max_vnic_id_cnt; + /* Max number of vnic ids in vnic id table */ + uint64_t vnic_id_tbl_addr; + /* This is the address for VF VNIC ID table */ +} __attribute__((packed)); + +/* Output (16 bytes) */ +struct hwrm_func_vf_vnic_ids_query_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint32_t vnic_id_cnt; + /* + * Actual number of vnic ids Each VNIC ID is written as a 32-bit + * number. + */ + uint8_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* hwrm_func_drv_rgtr */ +/* + * Description: This command is used by the function driver to register its + * information with the HWRM. A function driver shall implement this command. A + * function driver shall use this command during the driver initialization right + * after the HWRM version discovery and default ring resources allocation. + */ +/* Input (80 bytes) */ +struct hwrm_func_drv_rgtr_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint32_t flags; + /* + * When this bit is '1', the function driver is requesting all + * requests from its children VF drivers to be forwarded to + * itself. This flag can only be set by the PF driver. If a VF + * driver sets this flag, it should be ignored by the HWRM. + */ + #define HWRM_FUNC_DRV_RGTR_INPUT_FLAGS_FWD_ALL_MODE UINT32_C(0x1) + /* + * When this bit is '1', the function is requesting none of the + * requests from its children VF drivers to be forwarded to + * itself. This flag can only be set by the PF driver. If a VF + * driver sets this flag, it should be ignored by the HWRM. + */ + #define HWRM_FUNC_DRV_RGTR_INPUT_FLAGS_FWD_NONE_MODE UINT32_C(0x2) + uint32_t enables; + /* This bit must be '1' for the os_type field to be configured. */ + #define HWRM_FUNC_DRV_RGTR_INPUT_ENABLES_OS_TYPE UINT32_C(0x1) + /* This bit must be '1' for the ver field to be configured. */ + #define HWRM_FUNC_DRV_RGTR_INPUT_ENABLES_VER UINT32_C(0x2) + /* This bit must be '1' for the timestamp field to be configured. */ + #define HWRM_FUNC_DRV_RGTR_INPUT_ENABLES_TIMESTAMP UINT32_C(0x4) + /* This bit must be '1' for the vf_req_fwd field to be configured. */ + #define HWRM_FUNC_DRV_RGTR_INPUT_ENABLES_VF_INPUT_FWD UINT32_C(0x8) + /* + * This bit must be '1' for the async_event_fwd field to be + * configured. + */ + #define HWRM_FUNC_DRV_RGTR_INPUT_ENABLES_ASYNC_EVENT_FWD UINT32_C(0x10) + uint16_t os_type; + /* + * This value indicates the type of OS. The values are based on + * CIM_OperatingSystem.mof file as published by the DMTF. + */ + /* Unknown */ + #define HWRM_FUNC_DRV_RGTR_INPUT_OS_TYPE_UNKNOWN UINT32_C(0x0) + /* Other OS not listed below. */ + #define HWRM_FUNC_DRV_RGTR_INPUT_OS_TYPE_OTHER UINT32_C(0x1) + /* MSDOS OS. */ + #define HWRM_FUNC_DRV_RGTR_INPUT_OS_TYPE_MSDOS UINT32_C(0xe) + /* Windows OS. */ + #define HWRM_FUNC_DRV_RGTR_INPUT_OS_TYPE_WINDOWS UINT32_C(0x12) + /* Solaris OS. */ + #define HWRM_FUNC_DRV_RGTR_INPUT_OS_TYPE_SOLARIS UINT32_C(0x1d) + /* Linux OS. */ + #define HWRM_FUNC_DRV_RGTR_INPUT_OS_TYPE_LINUX UINT32_C(0x24) + /* FreeBSD OS. */ + #define HWRM_FUNC_DRV_RGTR_INPUT_OS_TYPE_FREEBSD UINT32_C(0x2a) + /* VMware ESXi OS. */ + #define HWRM_FUNC_DRV_RGTR_INPUT_OS_TYPE_ESXI UINT32_C(0x68) + /* Microsoft Windows 8 64-bit OS. */ + #define HWRM_FUNC_DRV_RGTR_INPUT_OS_TYPE_WIN864 UINT32_C(0x73) + /* Microsoft Windows Server 2012 R2 OS. */ + #define HWRM_FUNC_DRV_RGTR_INPUT_OS_TYPE_WIN2012R2 UINT32_C(0x74) + uint8_t ver_maj; + /* This is the major version of the driver. */ + uint8_t ver_min; + /* This is the minor version of the driver. */ + uint8_t ver_upd; + /* This is the update version of the driver. */ + uint8_t unused_0; + uint16_t unused_1; + uint32_t timestamp; + /* + * This is a 32-bit timestamp provided by the driver for keep + * alive. The timestamp is in multiples of 1ms. + */ + uint32_t unused_2; + uint32_t vf_req_fwd[8]; + /* + * This is a 256-bit bit mask provided by the PF driver for + * letting the HWRM know what commands issued by the VF driver + * to the HWRM should be forwarded to the PF driver. Nth bit + * refers to the Nth req_type. Setting Nth bit to 1 indicates + * that requests from the VF driver with req_type equal to N + * shall be forwarded to the parent PF driver. This field is not + * valid for the VF driver. + */ + uint32_t async_event_fwd[8]; + /* + * This is a 256-bit bit mask provided by the function driver + * (PF or VF driver) to indicate the list of asynchronous event + * completions to be forwarded. Nth bit refers to the Nth + * event_id. Setting Nth bit to 1 by the function driver shall + * result in the HWRM forwarding asynchronous event completion + * with event_id equal to N. If all bits are set to 0 (value of + * 0), then the HWRM shall not forward any asynchronous event + * completion to this function driver. + */ +} __attribute__((packed)); + +/* Output (16 bytes) */ +struct hwrm_func_drv_rgtr_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint32_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t unused_3; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* hwrm_func_drv_unrgtr */ +/* + * Description: This command is used by the function driver to un register with + * the HWRM. A function driver shall implement this command. A function driver + * shall use this command during the driver unloading. + */ +/* Input (24 bytes) */ +struct hwrm_func_drv_unrgtr_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint32_t flags; + /* + * When this bit is '1', the function driver is notifying the + * HWRM to prepare for the shutdown. + */ + #define HWRM_FUNC_DRV_UNRGTR_INPUT_FLAGS_PREPARE_FOR_SHUTDOWN \ + UINT32_C(0x1) + uint32_t unused_0; +} __attribute__((packed)); + +/* Output (16 bytes) */ +struct hwrm_func_drv_unrgtr_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint32_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t unused_3; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* hwrm_func_buf_rgtr */ +/* + * Description: This command is used by the PF driver to register buffers used + * in the PF-VF communication with the HWRM. The PF driver uses this command to + * register buffers for each PF-VF channel. A parent PF may issue this command + * per child VF. If VF ID is not valid, then this command is used to register + * buffers for all children VFs of the PF. + */ +/* Input (128 bytes) */ +struct hwrm_func_buf_rgtr_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint32_t enables; + /* This bit must be '1' for the vf_id field to be configured. */ + #define HWRM_FUNC_BUF_RGTR_INPUT_ENABLES_VF_ID UINT32_C(0x1) + /* This bit must be '1' for the err_buf_addr field to be configured. */ + #define HWRM_FUNC_BUF_RGTR_INPUT_ENABLES_ERR_BUF_ADDR UINT32_C(0x2) + uint16_t vf_id; + /* + * This value is used to identify a Virtual Function (VF). The + * scope of VF ID is local within a PF. + */ + uint16_t req_buf_num_pages; + /* + * This field represents the number of pages used for request + * buffer(s). + */ + uint16_t req_buf_page_size; + /* This field represents the page size used for request buffer(s). */ + /* 16 bytes */ + #define HWRM_FUNC_BUF_RGTR_INPUT_INPUT_BUF_PAGE_SIZE_16B UINT32_C(0x4) + /* 4 Kbytes */ + #define HWRM_FUNC_BUF_RGTR_INPUT_INPUT_BUF_PAGE_SIZE_4K UINT32_C(0xc) + /* 8 Kbytes */ + #define HWRM_FUNC_BUF_RGTR_INPUT_INPUT_BUF_PAGE_SIZE_8K UINT32_C(0xd) + /* 64 Kbytes */ + #define HWRM_FUNC_BUF_RGTR_INPUT_INPUT_BUF_PAGE_SIZE_64K UINT32_C(0x10) + /* 2 Mbytes */ + #define HWRM_FUNC_BUF_RGTR_INPUT_INPUT_BUF_PAGE_SIZE_2M UINT32_C(0x15) + /* 4 Mbytes */ + #define HWRM_FUNC_BUF_RGTR_INPUT_INPUT_BUF_PAGE_SIZE_4M UINT32_C(0x16) + /* 1 Gbytes */ + #define HWRM_FUNC_BUF_RGTR_INPUT_INPUT_BUF_PAGE_SIZE_1G UINT32_C(0x1e) + uint16_t req_buf_len; + /* The length of the request buffer per VF in bytes. */ + uint16_t resp_buf_len; + /* The length of the response buffer in bytes. */ + uint8_t unused_0; + uint8_t unused_1; + uint64_t req_buf_page_addr[10]; + /* This field represents the page address of req buffer. */ + uint64_t error_buf_addr; + /* + * This field is used to receive the error reporting from the + * chipset. Only applicable for PFs. + */ + uint64_t resp_buf_addr; + /* This field is used to receive the response forwarded by the HWRM. */ +} __attribute__((packed)); + +/* Output (16 bytes) */ +struct hwrm_func_buf_rgtr_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint32_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t unused_3; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* hwrm_func_buf_unrgtr */ +/* + * Description: This command is used by the PF driver to unregister buffers used + * in the PF-VF communication with the HWRM. The PF driver uses this command to + * unregister buffers for PF-VF communication. A parent PF may issue this + * command to unregister buffers for communication between the PF and a specific + * VF. If the VF ID is not valid, then this command is used to unregister + * buffers used for communications with all children VFs of the PF. + */ +/* Input (24 bytes) */ +struct hwrm_func_buf_unrgtr_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint32_t enables; + /* This bit must be '1' for the vf_id field to be configured. */ + #define HWRM_FUNC_BUF_UNRGTR_INPUT_ENABLES_VF_ID UINT32_C(0x1) + uint16_t vf_id; + /* + * This value is used to identify a Virtual Function (VF). The + * scope of VF ID is local within a PF. + */ + uint16_t unused_0; +} __attribute__((packed)); + +/* Output (16 bytes) */ +struct hwrm_func_buf_unrgtr_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint32_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t unused_3; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* hwrm_port_phy_cfg */ +/* + * Description: This command configures the PHY device for the port. It allows + * setting of the most generic settings for the PHY. The HWRM shall complete + * this command as soon as PHY settings are configured. They may not be applied + * when the command response is provided. A VF driver shall not be allowed to + * configure PHY using this command. In a network partition mode, a PF driver + * shall not be allowed to configure PHY using this command. + */ +/* Input (56 bytes) */ +struct hwrm_port_phy_cfg_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint32_t flags; + /* + * When this bit is set to '1', the PHY for the port shall be + * reset. # If this bit is set to 1, then the HWRM shall reset + * the PHY after applying PHY configuration changes specified in + * this command. # In order to guarantee that PHY configuration + * changes specified in this command take effect, the HWRM + * client should set this flag to 1. # If this bit is not set to + * 1, then the HWRM may reset the PHY depending on the current + * PHY configuration and settings specified in this command. + */ + #define HWRM_PORT_PHY_CFG_INPUT_FLAGS_RESET_PHY UINT32_C(0x1) + /* deprecated bit. Do not use!!! */ + #define HWRM_PORT_PHY_CFG_INPUT_FLAGS_DEPRECATED UINT32_C(0x2) + /* + * When this bit is set to '1', the link shall be forced to the + * force_link_speed value. When this bit is set to '1', the HWRM + * client should not enable any of the auto negotiation related + * fields represented by auto_XXX fields in this command. When + * this bit is set to '1' and the HWRM client has enabled a + * auto_XXX field in this command, then the HWRM shall ignore + * the enabled auto_XXX field. When this bit is set to zero, the + * link shall be allowed to autoneg. + */ + #define HWRM_PORT_PHY_CFG_INPUT_FLAGS_FORCE UINT32_C(0x4) + /* + * When this bit is set to '1', the auto-negotiation process + * shall be restarted on the link. + */ + #define HWRM_PORT_PHY_CFG_INPUT_FLAGS_RESTART_AUTONEG UINT32_C(0x8) + /* + * When this bit is set to '1', Energy Efficient Ethernet (EEE) + * is requested to be enabled on this link. If EEE is not + * supported on this port, then this flag shall be ignored by + * the HWRM. + */ + #define HWRM_PORT_PHY_CFG_INPUT_FLAGS_EEE_ENABLE UINT32_C(0x10) + /* + * When this bit is set to '1', Energy Efficient Ethernet (EEE) + * is requested to be disabled on this link. If EEE is not + * supported on this port, then this flag shall be ignored by + * the HWRM. + */ + #define HWRM_PORT_PHY_CFG_INPUT_FLAGS_EEE_DISABLE UINT32_C(0x20) + /* + * When this bit is set to '1' and EEE is enabled on this link, + * then TX LPI is requested to be enabled on the link. If EEE is + * not supported on this port, then this flag shall be ignored + * by the HWRM. If EEE is disabled on this port, then this flag + * shall be ignored by the HWRM. + */ + #define HWRM_PORT_PHY_CFG_INPUT_FLAGS_EEE_TX_LPI_ENABLE UINT32_C(0x40) + /* + * When this bit is set to '1' and EEE is enabled on this link, + * then TX LPI is requested to be disabled on the link. If EEE + * is not supported on this port, then this flag shall be + * ignored by the HWRM. If EEE is disabled on this port, then + * this flag shall be ignored by the HWRM. + */ + #define HWRM_PORT_PHY_CFG_INPUT_FLAGS_EEE_TX_LPI_DISABLE UINT32_C(0x80) + /* + * When set to 1, then the HWRM shall enable FEC + * autonegotitation on this port if supported. When set to 0, + * then this flag shall be ignored. If FEC autonegotiation is + * not supported, then the HWRM shall ignore this flag. + */ + #define HWRM_PORT_PHY_CFG_INPUT_FLAGS_FEC_AUTONEG_ENABLE UINT32_C(0x100) + /* + * When set to 1, then the HWRM shall disable FEC + * autonegotiation on this port if supported. When set to 0, + * then this flag shall be ignored. If FEC autonegotiation is + * not supported, then the HWRM shall ignore this flag. + */ + #define HWRM_PORT_PHY_CFG_INPUT_FLAGS_FEC_AUTONEG_DISABLE \ + UINT32_C(0x200) + /* + * When set to 1, then the HWRM shall enable FEC CLAUSE 74 (Fire + * Code) on this port if supported. When set to 0, then this + * flag shall be ignored. If FEC CLAUSE 74 is not supported, + * then the HWRM shall ignore this flag. + */ + #define HWRM_PORT_PHY_CFG_INPUT_FLAGS_FEC_CLAUSE74_ENABLE \ + UINT32_C(0x400) + /* + * When set to 1, then the HWRM shall disable FEC CLAUSE 74 + * (Fire Code) on this port if supported. When set to 0, then + * this flag shall be ignored. If FEC CLAUSE 74 is not + * supported, then the HWRM shall ignore this flag. + */ + #define HWRM_PORT_PHY_CFG_INPUT_FLAGS_FEC_CLAUSE74_DISABLE \ + UINT32_C(0x800) + /* + * When set to 1, then the HWRM shall enable FEC CLAUSE 91 (Reed + * Solomon) on this port if supported. When set to 0, then this + * flag shall be ignored. If FEC CLAUSE 91 is not supported, + * then the HWRM shall ignore this flag. + */ + #define HWRM_PORT_PHY_CFG_INPUT_FLAGS_FEC_CLAUSE91_ENABLE \ + UINT32_C(0x1000) + /* + * When set to 1, then the HWRM shall disable FEC CLAUSE 91 + * (Reed Solomon) on this port if supported. When set to 0, then + * this flag shall be ignored. If FEC CLAUSE 91 is not + * supported, then the HWRM shall ignore this flag. + */ + #define HWRM_PORT_PHY_CFG_INPUT_FLAGS_FEC_CLAUSE91_DISABLE \ + UINT32_C(0x2000) + /* + * When this bit is set to '1', the link shall be forced to be + * taken down. # When this bit is set to '1", all other command + * input settings related to the link speed shall be ignored. + * Once the link state is forced down, it can be explicitly + * cleared from that state by setting this flag to '0'. # If + * this flag is set to '0', then the link shall be cleared from + * forced down state if the link is in forced down state. There + * may be conditions (e.g. out-of-band or sideband configuration + * changes for the link) outside the scope of the HWRM + * implementation that may clear forced down link state. + */ + #define HWRM_PORT_PHY_CFG_INPUT_FLAGS_FORCE_LINK_DWN UINT32_C(0x4000) + uint32_t enables; + /* This bit must be '1' for the auto_mode field to be configured. */ + #define HWRM_PORT_PHY_CFG_INPUT_ENABLES_AUTO_MODE UINT32_C(0x1) + /* This bit must be '1' for the auto_duplex field to be configured. */ + #define HWRM_PORT_PHY_CFG_INPUT_ENABLES_AUTO_DUPLEX UINT32_C(0x2) + /* This bit must be '1' for the auto_pause field to be configured. */ + #define HWRM_PORT_PHY_CFG_INPUT_ENABLES_AUTO_PAUSE UINT32_C(0x4) + /* + * This bit must be '1' for the auto_link_speed field to be + * configured. + */ + #define HWRM_PORT_PHY_CFG_INPUT_ENABLES_AUTO_LINK_SPEED UINT32_C(0x8) + /* + * This bit must be '1' for the auto_link_speed_mask field to be + * configured. + */ + #define HWRM_PORT_PHY_CFG_INPUT_ENABLES_AUTO_LINK_SPEED_MASK \ + UINT32_C(0x10) + /* This bit must be '1' for the wirespeed field to be configured. */ + #define HWRM_PORT_PHY_CFG_INPUT_ENABLES_WIOUTPUTEED UINT32_C(0x20) + /* This bit must be '1' for the lpbk field to be configured. */ + #define HWRM_PORT_PHY_CFG_INPUT_ENABLES_LPBK UINT32_C(0x40) + /* This bit must be '1' for the preemphasis field to be configured. */ + #define HWRM_PORT_PHY_CFG_INPUT_ENABLES_PREEMPHASIS UINT32_C(0x80) + /* This bit must be '1' for the force_pause field to be configured. */ + #define HWRM_PORT_PHY_CFG_INPUT_ENABLES_FORCE_PAUSE UINT32_C(0x100) + /* + * This bit must be '1' for the eee_link_speed_mask field to be + * configured. + */ + #define HWRM_PORT_PHY_CFG_INPUT_ENABLES_EEE_LINK_SPEED_MASK \ + UINT32_C(0x200) + /* This bit must be '1' for the tx_lpi_timer field to be configured. */ + #define HWRM_PORT_PHY_CFG_INPUT_ENABLES_TX_LPI_TIMER UINT32_C(0x400) + uint16_t port_id; + /* Port ID of port that is to be configured. */ + uint16_t force_link_speed; + /* + * This is the speed that will be used if the force bit is '1'. + * If unsupported speed is selected, an error will be generated. + */ + /* 100Mb link speed */ + #define HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEED_100MB UINT32_C(0x1) + /* 1Gb link speed */ + #define HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEED_1GB UINT32_C(0xa) + /* 2Gb link speed */ + #define HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEED_2GB UINT32_C(0x14) + /* 2.5Gb link speed */ + #define HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEED_2_5GB UINT32_C(0x19) + /* 10Gb link speed */ + #define HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEED_10GB UINT32_C(0x64) + /* 20Mb link speed */ + #define HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEED_20GB UINT32_C(0xc8) + /* 25Gb link speed */ + #define HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEED_25GB UINT32_C(0xfa) + /* 40Gb link speed */ + #define HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEED_40GB UINT32_C(0x190) + /* 50Gb link speed */ + #define HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEED_50GB UINT32_C(0x1f4) + /* 100Gb link speed */ + #define HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEED_100GB UINT32_C(0x3e8) + /* 10Mb link speed */ + #define HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEED_10MB UINT32_C(0xffff) + uint8_t auto_mode; + /* + * This value is used to identify what autoneg mode is used when + * the link speed is not being forced. + */ + /* + * Disable autoneg or autoneg disabled. No + * speeds are selected. + */ + #define HWRM_PORT_PHY_CFG_INPUT_AUTO_MODE_NONE UINT32_C(0x0) + /* Select all possible speeds for autoneg mode. */ + #define HWRM_PORT_PHY_CFG_INPUT_AUTO_MODE_ALL_SPEEDS UINT32_C(0x1) + /* + * Select only the auto_link_speed speed for + * autoneg mode. This mode has been DEPRECATED. + * An HWRM client should not use this mode. + */ + #define HWRM_PORT_PHY_CFG_INPUT_AUTO_MODE_ONE_SPEED UINT32_C(0x2) + /* + * Select the auto_link_speed or any speed below + * that speed for autoneg. This mode has been + * DEPRECATED. An HWRM client should not use + * this mode. + */ + #define HWRM_PORT_PHY_CFG_INPUT_AUTO_MODE_ONE_OR_BELOW UINT32_C(0x3) + /* + * Select the speeds based on the corresponding + * link speed mask value that is provided. + */ + #define HWRM_PORT_PHY_CFG_INPUT_AUTO_MODE_SPEED_MASK UINT32_C(0x4) + uint8_t auto_duplex; + /* + * This is the duplex setting that will be used if the + * autoneg_mode is "one_speed" or "one_or_below". + */ + /* Half Duplex will be requested. */ + #define HWRM_PORT_PHY_CFG_INPUT_AUTO_DUPLEX_HALF UINT32_C(0x0) + /* Full duplex will be requested. */ + #define HWRM_PORT_PHY_CFG_INPUT_AUTO_DUPLEX_FULL UINT32_C(0x1) + /* Both Half and Full dupex will be requested. */ + #define HWRM_PORT_PHY_CFG_INPUT_AUTO_DUPLEX_BOTH UINT32_C(0x2) + uint8_t auto_pause; + /* + * This value is used to configure the pause that will be used + * for autonegotiation. Add text on the usage of auto_pause and + * force_pause. + */ + /* + * When this bit is '1', Generation of tx pause messages has + * been requested. Disabled otherwise. + */ + #define HWRM_PORT_PHY_CFG_INPUT_AUTO_PAUSE_TX UINT32_C(0x1) + /* + * When this bit is '1', Reception of rx pause messages has been + * requested. Disabled otherwise. + */ + #define HWRM_PORT_PHY_CFG_INPUT_AUTO_PAUSE_RX UINT32_C(0x2) + /* + * When set to 1, the advertisement of pause is enabled. # When + * the auto_mode is not set to none and this flag is set to 1, + * then the auto_pause bits on this port are being advertised + * and autoneg pause results are being interpreted. # When the + * auto_mode is not set to none and this flag is set to 0, the + * pause is forced as indicated in force_pause, and also + * advertised as auto_pause bits, but the autoneg results are + * not interpreted since the pause configuration is being + * forced. # When the auto_mode is set to none and this flag is + * set to 1, auto_pause bits should be ignored and should be set + * to 0. + */ + #define HWRM_PORT_PHY_CFG_INPUT_AUTO_PAUSE_AUTONEG_PAUSE UINT32_C(0x4) + uint8_t unused_0; + uint16_t auto_link_speed; + /* + * This is the speed that will be used if the autoneg_mode is + * "one_speed" or "one_or_below". If an unsupported speed is + * selected, an error will be generated. + */ + /* 100Mb link speed */ + #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_100MB UINT32_C(0x1) + /* 1Gb link speed */ + #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_1GB UINT32_C(0xa) + /* 2Gb link speed */ + #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_2GB UINT32_C(0x14) + /* 2.5Gb link speed */ + #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_2_5GB UINT32_C(0x19) + /* 10Gb link speed */ + #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_10GB UINT32_C(0x64) + /* 20Mb link speed */ + #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_20GB UINT32_C(0xc8) + /* 25Gb link speed */ + #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_25GB UINT32_C(0xfa) + /* 40Gb link speed */ + #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_40GB UINT32_C(0x190) + /* 50Gb link speed */ + #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_50GB UINT32_C(0x1f4) + /* 100Gb link speed */ + #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_100GB UINT32_C(0x3e8) + /* 10Mb link speed */ + #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_10MB UINT32_C(0xffff) + uint16_t auto_link_speed_mask; + /* + * This is a mask of link speeds that will be used if + * autoneg_mode is "mask". If unsupported speed is enabled an + * error will be generated. + */ + /* 100Mb link speed (Half-duplex) */ + #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_100MBHD \ + UINT32_C(0x1) + /* 100Mb link speed (Full-duplex) */ + #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_100MB \ + UINT32_C(0x2) + /* 1Gb link speed (Half-duplex) */ + #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_1GBHD \ + UINT32_C(0x4) + /* 1Gb link speed (Full-duplex) */ + #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_1GB \ + UINT32_C(0x8) + /* 2Gb link speed */ + #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_2GB \ + UINT32_C(0x10) + /* 2.5Gb link speed */ + #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_2_5GB \ + UINT32_C(0x20) + /* 10Gb link speed */ + #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_10GB UINT32_C(0x40) + /* 20Gb link speed */ + #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_20GB UINT32_C(0x80) + /* 25Gb link speed */ + #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_25GB \ + UINT32_C(0x100) + /* 40Gb link speed */ + #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_40GB \ + UINT32_C(0x200) + /* 50Gb link speed */ + #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_50GB \ + UINT32_C(0x400) + /* 100Gb link speed */ + #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_100GB \ + UINT32_C(0x800) + /* 10Mb link speed (Half-duplex) */ + #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_10MBHD \ + UINT32_C(0x1000) + /* 10Mb link speed (Full-duplex) */ + #define HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_10MB \ + UINT32_C(0x2000) + uint8_t wirespeed; + /* This value controls the wirespeed feature. */ + /* Wirespeed feature is disabled. */ + #define HWRM_PORT_PHY_CFG_INPUT_WIOUTPUTEED_OFF UINT32_C(0x0) + /* Wirespeed feature is enabled. */ + #define HWRM_PORT_PHY_CFG_INPUT_WIOUTPUTEED_ON UINT32_C(0x1) + uint8_t lpbk; + /* This value controls the loopback setting for the PHY. */ + /* No loopback is selected. Normal operation. */ + #define HWRM_PORT_PHY_CFG_INPUT_LPBK_NONE UINT32_C(0x0) + /* + * The HW will be configured with local loopback + * such that host data is sent back to the host + * without modification. + */ + #define HWRM_PORT_PHY_CFG_INPUT_LPBK_LOCAL UINT32_C(0x1) + /* + * The HW will be configured with remote + * loopback such that port logic will send + * packets back out the transmitter that are + * received. + */ + #define HWRM_PORT_PHY_CFG_INPUT_LPBK_REMOTE UINT32_C(0x2) + uint8_t force_pause; + /* + * This value is used to configure the pause that will be used + * for force mode. + */ + /* + * When this bit is '1', Generation of tx pause messages is + * supported. Disabled otherwise. + */ + #define HWRM_PORT_PHY_CFG_INPUT_FORCE_PAUSE_TX UINT32_C(0x1) + /* + * When this bit is '1', Reception of rx pause messages is + * supported. Disabled otherwise. + */ + #define HWRM_PORT_PHY_CFG_INPUT_FORCE_PAUSE_RX UINT32_C(0x2) + uint8_t unused_1; + uint32_t preemphasis; + /* + * This value controls the pre-emphasis to be used for the link. + * Driver should not set this value (use enable.preemphasis = 0) + * unless driver is sure of setting. Normally HWRM FW will + * determine proper pre-emphasis. + */ + uint16_t eee_link_speed_mask; + /* + * Setting for link speed mask that is used to advertise speeds + * during autonegotiation when EEE is enabled. This field is + * valid only when EEE is enabled. The speeds specified in this + * field shall be a subset of speeds specified in + * auto_link_speed_mask. If EEE is enabled,then at least one + * speed shall be provided in this mask. + */ + /* Reserved */ + #define HWRM_PORT_PHY_CFG_INPUT_EEE_LINK_SPEED_MASK_RSVD1 UINT32_C(0x1) + /* 100Mb link speed (Full-duplex) */ + #define HWRM_PORT_PHY_CFG_INPUT_EEE_LINK_SPEED_MASK_100MB UINT32_C(0x2) + /* Reserved */ + #define HWRM_PORT_PHY_CFG_INPUT_EEE_LINK_SPEED_MASK_RSVD2 UINT32_C(0x4) + /* 1Gb link speed (Full-duplex) */ + #define HWRM_PORT_PHY_CFG_INPUT_EEE_LINK_SPEED_MASK_1GB UINT32_C(0x8) + /* Reserved */ + #define HWRM_PORT_PHY_CFG_INPUT_EEE_LINK_SPEED_MASK_RSVD3 UINT32_C(0x10) + /* Reserved */ + #define HWRM_PORT_PHY_CFG_INPUT_EEE_LINK_SPEED_MASK_RSVD4 UINT32_C(0x20) + /* 10Gb link speed */ + #define HWRM_PORT_PHY_CFG_INPUT_EEE_LINK_SPEED_MASK_10GB UINT32_C(0x40) + uint8_t unused_2; + uint8_t unused_3; + uint32_t tx_lpi_timer; + uint32_t unused_4; + /* + * Reuested setting of TX LPI timer in microseconds. This field + * is valid only when EEE is enabled and TX LPI is enabled. + */ + #define HWRM_PORT_PHY_CFG_INPUT_TX_LPI_TIMER_MASK UINT32_C(0xffffff) + #define HWRM_PORT_PHY_CFG_INPUT_TX_LPI_TIMER_SFT 0 +} __attribute__((packed)); + +/* Output (16 bytes) */ +struct hwrm_port_phy_cfg_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint32_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t unused_3; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* hwrm_port_phy_qcfg */ +/* Description: This command queries the PHY configuration for the port. */ +/* Input (24 bytes) */ +struct hwrm_port_phy_qcfg_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint16_t port_id; + /* Port ID of port that is to be queried. */ + uint16_t unused_0[3]; +} __attribute__((packed)); + +/* Output (96 bytes) */ +struct hwrm_port_phy_qcfg_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint8_t link; + /* This value indicates the current link status. */ + /* There is no link or cable detected. */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_NO_LINK UINT32_C(0x0) + /* There is no link, but a cable has been detected. */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SIGNAL UINT32_C(0x1) + /* There is a link. */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_LINK UINT32_C(0x2) + uint8_t unused_0; + uint16_t link_speed; + /* This value indicates the current link speed of the connection. */ + /* 100Mb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_100MB UINT32_C(0x1) + /* 1Gb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_1GB UINT32_C(0xa) + /* 2Gb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_2GB UINT32_C(0x14) + /* 2.5Gb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_2_5GB UINT32_C(0x19) + /* 10Gb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_10GB UINT32_C(0x64) + /* 20Mb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_20GB UINT32_C(0xc8) + /* 25Gb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_25GB UINT32_C(0xfa) + /* 40Gb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_40GB UINT32_C(0x190) + /* 50Gb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_50GB UINT32_C(0x1f4) + /* 100Gb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_100GB UINT32_C(0x3e8) + /* 10Mb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_10MB UINT32_C(0xffff) + uint8_t duplex_cfg; + /* This value is indicates the duplex of the current connection. */ + /* Half Duplex connection. */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_DUPLEX_CFG_HALF UINT32_C(0x0) + /* Full duplex connection. */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_DUPLEX_CFG_FULL UINT32_C(0x1) + uint8_t pause; + /* + * This value is used to indicate the current pause + * configuration. When autoneg is enabled, this value represents + * the autoneg results of pause configuration. + */ + /* + * When this bit is '1', Generation of tx pause messages is + * supported. Disabled otherwise. + */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_TX UINT32_C(0x1) + /* + * When this bit is '1', Reception of rx pause messages is + * supported. Disabled otherwise. + */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_RX UINT32_C(0x2) + uint16_t support_speeds; + /* + * The supported speeds for the port. This is a bit mask. For + * each speed that is supported, the corrresponding bit will be + * set to '1'. + */ + /* 100Mb link speed (Half-duplex) */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_100MBHD UINT32_C(0x1) + /* 100Mb link speed (Full-duplex) */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_100MB UINT32_C(0x2) + /* 1Gb link speed (Half-duplex) */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_1GBHD UINT32_C(0x4) + /* 1Gb link speed (Full-duplex) */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_1GB UINT32_C(0x8) + /* 2Gb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_2GB UINT32_C(0x10) + /* 2.5Gb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_2_5GB UINT32_C(0x20) + /* 10Gb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_10GB UINT32_C(0x40) + /* 20Gb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_20GB UINT32_C(0x80) + /* 25Gb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_25GB UINT32_C(0x100) + /* 40Gb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_40GB UINT32_C(0x200) + /* 50Gb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_50GB UINT32_C(0x400) + /* 100Gb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_100GB UINT32_C(0x800) + /* 10Mb link speed (Half-duplex) */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_10MBHD UINT32_C(0x1000) + /* 10Mb link speed (Full-duplex) */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_10MB UINT32_C(0x2000) + uint16_t force_link_speed; + /* + * Current setting of forced link speed. When the link speed is + * not being forced, this value shall be set to 0. + */ + /* 100Mb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_FORCE_LINK_SPEED_100MB UINT32_C(0x1) + /* 1Gb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_FORCE_LINK_SPEED_1GB UINT32_C(0xa) + /* 2Gb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_FORCE_LINK_SPEED_2GB UINT32_C(0x14) + /* 2.5Gb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_FORCE_LINK_SPEED_2_5GB UINT32_C(0x19) + /* 10Gb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_FORCE_LINK_SPEED_10GB UINT32_C(0x64) + /* 20Mb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_FORCE_LINK_SPEED_20GB UINT32_C(0xc8) + /* 25Gb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_FORCE_LINK_SPEED_25GB UINT32_C(0xfa) + /* 40Gb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_FORCE_LINK_SPEED_40GB UINT32_C(0x190) + /* 50Gb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_FORCE_LINK_SPEED_50GB UINT32_C(0x1f4) + /* 100Gb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_FORCE_LINK_SPEED_100GB UINT32_C(0x3e8) + /* 10Mb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_FORCE_LINK_SPEED_10MB UINT32_C(0xffff) + uint8_t auto_mode; + /* Current setting of auto negotiation mode. */ + /* + * Disable autoneg or autoneg disabled. No + * speeds are selected. + */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_MODE_NONE UINT32_C(0x0) + /* Select all possible speeds for autoneg mode. */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_MODE_ALL_SPEEDS UINT32_C(0x1) + /* + * Select only the auto_link_speed speed for + * autoneg mode. This mode has been DEPRECATED. + * An HWRM client should not use this mode. + */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_MODE_ONE_SPEED UINT32_C(0x2) + /* + * Select the auto_link_speed or any speed below + * that speed for autoneg. This mode has been + * DEPRECATED. An HWRM client should not use + * this mode. + */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_MODE_ONE_OR_BELOW UINT32_C(0x3) + /* + * Select the speeds based on the corresponding + * link speed mask value that is provided. + */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_MODE_SPEED_MASK UINT32_C(0x4) + uint8_t auto_pause; + /* + * Current setting of pause autonegotiation. Move autoneg_pause + * flag here. + */ + /* + * When this bit is '1', Generation of tx pause messages has + * been requested. Disabled otherwise. + */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_PAUSE_TX UINT32_C(0x1) + /* + * When this bit is '1', Reception of rx pause messages has been + * requested. Disabled otherwise. + */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_PAUSE_RX UINT32_C(0x2) + /* + * When set to 1, the advertisement of pause is enabled. # When + * the auto_mode is not set to none and this flag is set to 1, + * then the auto_pause bits on this port are being advertised + * and autoneg pause results are being interpreted. # When the + * auto_mode is not set to none and this flag is set to 0, the + * pause is forced as indicated in force_pause, and also + * advertised as auto_pause bits, but the autoneg results are + * not interpreted since the pause configuration is being + * forced. # When the auto_mode is set to none and this flag is + * set to 1, auto_pause bits should be ignored and should be set + * to 0. + */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_PAUSE_AUTONEG_PAUSE UINT32_C(0x4) + uint16_t auto_link_speed; + /* + * Current setting for auto_link_speed. This field is only valid + * when auto_mode is set to "one_speed" or "one_or_below". + */ + /* 100Mb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_100MB UINT32_C(0x1) + /* 1Gb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_1GB UINT32_C(0xa) + /* 2Gb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_2GB UINT32_C(0x14) + /* 2.5Gb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_2_5GB UINT32_C(0x19) + /* 10Gb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_10GB UINT32_C(0x64) + /* 20Mb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_20GB UINT32_C(0xc8) + /* 25Gb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_25GB UINT32_C(0xfa) + /* 40Gb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_40GB UINT32_C(0x190) + /* 50Gb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_50GB UINT32_C(0x1f4) + /* 100Gb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_100GB UINT32_C(0x3e8) + /* 10Mb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_10MB UINT32_C(0xffff) + uint16_t auto_link_speed_mask; + /* + * Current setting for auto_link_speed_mask that is used to + * advertise speeds during autonegotiation. This field is only + * valid when auto_mode is set to "mask". The speeds specified + * in this field shall be a subset of supported speeds on this + * port. + */ + /* 100Mb link speed (Half-duplex) */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_MASK_100MBHD \ + UINT32_C(0x1) + /* 100Mb link speed (Full-duplex) */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_MASK_100MB \ + UINT32_C(0x2) + /* 1Gb link speed (Half-duplex) */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_MASK_1GBHD \ + UINT32_C(0x4) + /* 1Gb link speed (Full-duplex) */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_MASK_1GB UINT32_C(0x8) + /* 2Gb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_MASK_2GB \ + UINT32_C(0x10) + /* 2.5Gb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_MASK_2_5GB \ + UINT32_C(0x20) + /* 10Gb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_MASK_10GB \ + UINT32_C(0x40) + /* 20Gb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_MASK_20GB \ + UINT32_C(0x80) + /* 25Gb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_MASK_25GB \ + UINT32_C(0x100) + /* 40Gb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_MASK_40GB \ + UINT32_C(0x200) + /* 50Gb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_MASK_50GB \ + UINT32_C(0x400) + /* 100Gb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_MASK_100GB \ + UINT32_C(0x800) + /* 10Mb link speed (Half-duplex) */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_MASK_10MBHD \ + UINT32_C(0x1000) + /* 10Mb link speed (Full-duplex) */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_LINK_SPEED_MASK_10MB \ + UINT32_C(0x2000) + uint8_t wirespeed; + /* Current setting for wirespeed. */ + /* Wirespeed feature is disabled. */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_WIOUTPUTEED_OFF UINT32_C(0x0) + /* Wirespeed feature is enabled. */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_WIOUTPUTEED_ON UINT32_C(0x1) + uint8_t lpbk; + /* Current setting for loopback. */ + /* No loopback is selected. Normal operation. */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_LPBK_NONE UINT32_C(0x0) + /* + * The HW will be configured with local loopback + * such that host data is sent back to the host + * without modification. + */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_LPBK_LOCAL UINT32_C(0x1) + /* + * The HW will be configured with remote + * loopback such that port logic will send + * packets back out the transmitter that are + * received. + */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_LPBK_REMOTE UINT32_C(0x2) + uint8_t force_pause; + /* + * Current setting of forced pause. When the pause configuration + * is not being forced, then this value shall be set to 0. + */ + /* + * When this bit is '1', Generation of tx pause messages is + * supported. Disabled otherwise. + */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_FORCE_PAUSE_TX UINT32_C(0x1) + /* + * When this bit is '1', Reception of rx pause messages is + * supported. Disabled otherwise. + */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_FORCE_PAUSE_RX UINT32_C(0x2) + uint8_t module_status; + /* + * This value indicates the current status of the optics module + * on this port. + */ + /* Module is inserted and accepted */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_MODULE_STATUS_NONE UINT32_C(0x0) + /* Module is rejected and transmit side Laser is disabled. */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_MODULE_STATUS_DISABLETX UINT32_C(0x1) + /* Module mismatch warning. */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_MODULE_STATUS_WARNINGMSG UINT32_C(0x2) + /* Module is rejected and powered down. */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_MODULE_STATUS_PWRDOWN UINT32_C(0x3) + /* Module is not inserted. */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_MODULE_STATUS_NOTINSERTED \ + UINT32_C(0x4) + /* Module status is not applicable. */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_MODULE_STATUS_NOTAPPLICABLE \ + UINT32_C(0xff) + uint32_t preemphasis; + /* Current setting for preemphasis. */ + uint8_t phy_maj; + /* This field represents the major version of the PHY. */ + uint8_t phy_min; + /* This field represents the minor version of the PHY. */ + uint8_t phy_bld; + /* This field represents the build version of the PHY. */ + uint8_t phy_type; + /* This value represents a PHY type. */ + /* Unknown */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_UNKNOWN UINT32_C(0x0) + /* BASE-CR */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASECR UINT32_C(0x1) + /* BASE-KR4 (Deprecated) */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKR4 UINT32_C(0x2) + /* BASE-LR */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASELR UINT32_C(0x3) + /* BASE-SR */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASESR UINT32_C(0x4) + /* BASE-KR2 (Deprecated) */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKR2 UINT32_C(0x5) + /* BASE-KX */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKX UINT32_C(0x6) + /* BASE-KR */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKR UINT32_C(0x7) + /* BASE-T */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASET UINT32_C(0x8) + /* EEE capable BASE-T */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASETE UINT32_C(0x9) + /* SGMII connected external PHY */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_SGMIIEXTPHY UINT32_C(0xa) + /* 25G_BASECR_CA_L */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_25G_BASECR_CA_L UINT32_C(0xb) + /* 25G_BASECR_CA_S */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_25G_BASECR_CA_S UINT32_C(0xc) + /* 25G_BASECR_CA_N */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_25G_BASECR_CA_N UINT32_C(0xd) + /* 25G_BASESR */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_25G_BASESR UINT32_C(0xe) + /* 100G_BASECR4 */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_100G_BASECR4 UINT32_C(0xf) + /* 100G_BASESR4 */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_100G_BASESR4 UINT32_C(0x10) + /* 100G_BASELR4 */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_100G_BASELR4 UINT32_C(0x11) + /* 100G_BASEER4 */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_100G_BASEER4 UINT32_C(0x12) + /* 100G_BASESR10 */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_100G_BASESR10 UINT32_C(0x13) + /* 40G_BASECR4 */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_40G_BASECR4 UINT32_C(0x14) + /* 40G_BASESR4 */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_40G_BASESR4 UINT32_C(0x15) + /* 40G_BASELR4 */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_40G_BASELR4 UINT32_C(0x16) + /* 40G_BASEER4 */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_40G_BASEER4 UINT32_C(0x17) + /* 40G_ACTIVE_CABLE */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_40G_ACTIVE_CABLE \ + UINT32_C(0x18) + #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_1G_BASET UINT32_C(0x19) + /* 1G_baseSX */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_1G_BASESX UINT32_C(0x1a) + /* 1G_baseCX */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_1G_BASECX UINT32_C(0x1b) + uint8_t media_type; + /* This value represents a media type. */ + /* Unknown */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_MEDIA_TYPE_UNKNOWN UINT32_C(0x0) + /* Twisted Pair */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_MEDIA_TYPE_TP UINT32_C(0x1) + /* Direct Attached Copper */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_MEDIA_TYPE_DAC UINT32_C(0x2) + /* Fiber */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_MEDIA_TYPE_FIBRE UINT32_C(0x3) + uint8_t xcvr_pkg_type; + /* This value represents a transceiver type. */ + /* PHY and MAC are in the same package */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_XCVR_PKG_TYPE_XCVR_INTERNAL \ + UINT32_C(0x1) + /* PHY and MAC are in different packages */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_XCVR_PKG_TYPE_XCVR_EXTERNAL \ + UINT32_C(0x2) + uint8_t eee_config_phy_addr; + /* + * This field represents flags related to EEE configuration. + * These EEE configuration flags are valid only when the + * auto_mode is not set to none (in other words autonegotiation + * is enabled). + */ + /* This field represents PHY address. */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_ADDR_MASK UINT32_C(0x1f) + #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_ADDR_SFT 0 + /* + * When set to 1, Energy Efficient Ethernet (EEE) mode is + * enabled. Speeds for autoneg with EEE mode enabled are based + * on eee_link_speed_mask. + */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_EEE_CONFIG_EEE_ENABLED UINT32_C(0x20) + /* + * This flag is valid only when eee_enabled is set to 1. # If + * eee_enabled is set to 0, then EEE mode is disabled and this + * flag shall be ignored. # If eee_enabled is set to 1 and this + * flag is set to 1, then Energy Efficient Ethernet (EEE) mode + * is enabled and in use. # If eee_enabled is set to 1 and this + * flag is set to 0, then Energy Efficient Ethernet (EEE) mode + * is enabled but is currently not in use. + */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_EEE_CONFIG_EEE_ACTIVE UINT32_C(0x40) + /* + * This flag is valid only when eee_enabled is set to 1. # If + * eee_enabled is set to 0, then EEE mode is disabled and this + * flag shall be ignored. # If eee_enabled is set to 1 and this + * flag is set to 1, then Energy Efficient Ethernet (EEE) mode + * is enabled and TX LPI is enabled. # If eee_enabled is set to + * 1 and this flag is set to 0, then Energy Efficient Ethernet + * (EEE) mode is enabled but TX LPI is disabled. + */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_EEE_CONFIG_EEE_TX_LPI UINT32_C(0x80) + /* + * This field represents flags related to EEE configuration. + * These EEE configuration flags are valid only when the + * auto_mode is not set to none (in other words autonegotiation + * is enabled). + */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_EEE_CONFIG_MASK UINT32_C(0xe0) + #define HWRM_PORT_PHY_QCFG_OUTPUT_EEE_CONFIG_SFT 5 + uint8_t parallel_detect; + /* Reserved field, set to 0 */ + /* + * When set to 1, the parallel detection is used to determine + * the speed of the link partner. Parallel detection is used + * when a autonegotiation capable device is connected to a link + * parter that is not capable of autonegotiation. + */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_PARALLEL_DETECT UINT32_C(0x1) + /* Reserved field, set to 0 */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_RESERVED_MASK UINT32_C(0xfe) + #define HWRM_PORT_PHY_QCFG_OUTPUT_RESERVED_SFT 1 + uint16_t link_partner_adv_speeds; + /* + * The advertised speeds for the port by the link partner. Each + * advertised speed will be set to '1'. + */ + /* 100Mb link speed (Half-duplex) */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_SPEEDS_100MBHD \ + UINT32_C(0x1) + /* 100Mb link speed (Full-duplex) */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_SPEEDS_100MB \ + UINT32_C(0x2) + /* 1Gb link speed (Half-duplex) */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_SPEEDS_1GBHD \ + UINT32_C(0x4) + /* 1Gb link speed (Full-duplex) */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_SPEEDS_1GB \ + UINT32_C(0x8) + /* 2Gb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_SPEEDS_2GB \ + UINT32_C(0x10) + /* 2.5Gb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_SPEEDS_2_5GB \ + UINT32_C(0x20) + /* 10Gb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_SPEEDS_10GB \ + UINT32_C(0x40) + /* 20Gb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_SPEEDS_20GB \ + UINT32_C(0x80) + /* 25Gb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_SPEEDS_25GB \ + UINT32_C(0x100) + /* 40Gb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_SPEEDS_40GB \ + UINT32_C(0x200) + /* 50Gb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_SPEEDS_50GB \ + UINT32_C(0x400) + /* 100Gb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_SPEEDS_100GB \ + UINT32_C(0x800) + /* 10Mb link speed (Half-duplex) */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_SPEEDS_10MBHD \ + UINT32_C(0x1000) + /* 10Mb link speed (Full-duplex) */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_SPEEDS_10MB \ + UINT32_C(0x2000) + uint8_t link_partner_adv_auto_mode; + /* + * The advertised autoneg for the port by the link partner. This + * field is deprecated and should be set to 0. + */ + /* + * Disable autoneg or autoneg disabled. No + * speeds are selected. + */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_AUTO_MODE_NONE \ + UINT32_C(0x0) + /* Select all possible speeds for autoneg mode. */ + #define \ + HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_AUTO_MODE_ALL_SPEEDS \ + UINT32_C(0x1) + /* + * Select only the auto_link_speed speed for + * autoneg mode. This mode has been DEPRECATED. + * An HWRM client should not use this mode. + */ + #define \ + HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_AUTO_MODE_ONE_SPEED \ + UINT32_C(0x2) + /* + * Select the auto_link_speed or any speed below + * that speed for autoneg. This mode has been + * DEPRECATED. An HWRM client should not use + * this mode. + */ + #define \ + HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_AUTO_MODE_ONE_OR_BELOW \ + UINT32_C(0x3) + /* + * Select the speeds based on the corresponding + * link speed mask value that is provided. + */ + #define \ + HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_AUTO_MODE_SPEED_MASK \ + UINT32_C(0x4) + uint8_t link_partner_adv_pause; + /* The advertised pause settings on the port by the link partner. */ + /* + * When this bit is '1', Generation of tx pause messages is + * supported. Disabled otherwise. + */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_PAUSE_TX \ + UINT32_C(0x1) + /* + * When this bit is '1', Reception of rx pause messages is + * supported. Disabled otherwise. + */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_PAUSE_RX \ + UINT32_C(0x2) + uint16_t adv_eee_link_speed_mask; + /* + * Current setting for link speed mask that is used to advertise + * speeds during autonegotiation when EEE is enabled. This field + * is valid only when eee_enabled flags is set to 1. The speeds + * specified in this field shall be a subset of speeds specified + * in auto_link_speed_mask. + */ + /* Reserved */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_ADV_EEE_LINK_SPEED_MASK_RSVD1 \ + UINT32_C(0x1) + /* 100Mb link speed (Full-duplex) */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_ADV_EEE_LINK_SPEED_MASK_100MB \ + UINT32_C(0x2) + /* Reserved */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_ADV_EEE_LINK_SPEED_MASK_RSVD2 \ + UINT32_C(0x4) + /* 1Gb link speed (Full-duplex) */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_ADV_EEE_LINK_SPEED_MASK_1GB \ + UINT32_C(0x8) + /* Reserved */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_ADV_EEE_LINK_SPEED_MASK_RSVD3 \ + UINT32_C(0x10) + /* Reserved */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_ADV_EEE_LINK_SPEED_MASK_RSVD4 \ + UINT32_C(0x20) + /* 10Gb link speed */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_ADV_EEE_LINK_SPEED_MASK_10GB \ + UINT32_C(0x40) + uint16_t link_partner_adv_eee_link_speed_mask; + /* + * Current setting for link speed mask that is advertised by the + * link partner when EEE is enabled. This field is valid only + * when eee_enabled flags is set to 1. + */ + /* Reserved */ + #define \ + HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_EEE_LINK_SPEED_MASK_RSVD1 \ + UINT32_C(0x1) + /* 100Mb link speed (Full-duplex) */ + #define \ + HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_EEE_LINK_SPEED_MASK_100MB \ + UINT32_C(0x2) + /* Reserved */ + #define \ + HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_EEE_LINK_SPEED_MASK_RSVD2 \ + UINT32_C(0x4) + /* 1Gb link speed (Full-duplex) */ + #define \ + HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_EEE_LINK_SPEED_MASK_1GB \ + UINT32_C(0x8) + /* Reserved */ + #define \ + HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_EEE_LINK_SPEED_MASK_RSVD3 \ + UINT32_C(0x10) + /* Reserved */ + #define \ + HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_EEE_LINK_SPEED_MASK_RSVD4 \ + UINT32_C(0x20) + /* 10Gb link speed */ + #define \ + HWRM_PORT_PHY_QCFG_OUTPUT_LINK_PARTNER_ADV_EEE_LINK_SPEED_MASK_10GB \ + UINT32_C(0x40) + uint32_t xcvr_identifier_type_tx_lpi_timer; + /* This value represents transceiver identifier type. */ + /* + * Current setting of TX LPI timer in microseconds. This field + * is valid only when_eee_enabled flag is set to 1 and + * tx_lpi_enabled is set to 1. + */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_TX_LPI_TIMER_MASK UINT32_C(0xffffff) + #define HWRM_PORT_PHY_QCFG_OUTPUT_TX_LPI_TIMER_SFT 0 + /* This value represents transceiver identifier type. */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_XCVR_IDENTIFIER_TYPE_MASK \ + UINT32_C(0xff000000) + #define HWRM_PORT_PHY_QCFG_OUTPUT_XCVR_IDENTIFIER_TYPE_SFT 24 + /* Unknown */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_XCVR_IDENTIFIER_TYPE_UNKNOWN \ + (UINT32_C(0x0) << 24) + /* SFP/SFP+/SFP28 */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_XCVR_IDENTIFIER_TYPE_SFP \ + (UINT32_C(0x3) << 24) + /* QSFP */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_XCVR_IDENTIFIER_TYPE_QSFP \ + (UINT32_C(0xc) << 24) + /* QSFP+ */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_XCVR_IDENTIFIER_TYPE_QSFPPLUS \ + (UINT32_C(0xd) << 24) + /* QSFP28 */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_XCVR_IDENTIFIER_TYPE_QSFP28 \ + (UINT32_C(0x11) << 24) + uint16_t fec_cfg; + /* + * This value represents the current configuration of Forward + * Error Correction (FEC) on the port. + */ + /* + * When set to 1, then FEC is not supported on this port. If + * this flag is set to 1, then all other FEC configuration flags + * shall be ignored. When set to 0, then FEC is supported as + * indicated by other configuration flags. If no cable is + * attached and the HWRM does not yet know the FEC capability, + * then the HWRM shall set this flag to 1 when reporting FEC + * capability. + */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_FEC_CFG_FEC_NONE_SUPPORTED \ + UINT32_C(0x1) + /* + * When set to 1, then FEC autonegotiation is supported on this + * port. When set to 0, then FEC autonegotiation is not + * supported on this port. + */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_FEC_CFG_FEC_AUTONEG_SUPPORTED \ + UINT32_C(0x2) + /* + * When set to 1, then FEC autonegotiation is enabled on this + * port. When set to 0, then FEC autonegotiation is disabled if + * supported. This flag should be ignored if FEC autonegotiation + * is not supported on this port. + */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_FEC_CFG_FEC_AUTONEG_ENABLED \ + UINT32_C(0x4) + /* + * When set to 1, then FEC CLAUSE 74 (Fire Code) is supported on + * this port. When set to 0, then FEC CLAUSE 74 (Fire Code) is + * not supported on this port. + */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_FEC_CFG_FEC_CLAUSE74_SUPPORTED \ + UINT32_C(0x8) + /* + * When set to 1, then FEC CLAUSE 74 (Fire Code) is enabled on + * this port. When set to 0, then FEC CLAUSE 74 (Fire Code) is + * disabled if supported. This flag should be ignored if FEC + * CLAUSE 74 is not supported on this port. + */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_FEC_CFG_FEC_CLAUSE74_ENABLED \ + UINT32_C(0x10) + /* + * When set to 1, then FEC CLAUSE 91 (Reed Solomon) is supported + * on this port. When set to 0, then FEC CLAUSE 91 (Reed + * Solomon) is not supported on this port. + */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_FEC_CFG_FEC_CLAUSE91_SUPPORTED \ + UINT32_C(0x20) + /* + * When set to 1, then FEC CLAUSE 91 (Reed Solomon) is enabled + * on this port. When set to 0, then FEC CLAUSE 91 (Reed + * Solomon) is disabled if supported. This flag should be + * ignored if FEC CLAUSE 91 is not supported on this port. + */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_FEC_CFG_FEC_CLAUSE91_ENABLED \ + UINT32_C(0x40) + uint8_t duplex_state; + /* + * This value is indicates the duplex of the current connection + * state. + */ + /* Half Duplex connection. */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_DUPLEX_STATE_HALF UINT32_C(0x0) + /* Full duplex connection. */ + #define HWRM_PORT_PHY_QCFG_OUTPUT_DUPLEX_STATE_FULL UINT32_C(0x1) + uint8_t unused_1; + char phy_vendor_name[16]; + /* + * Up to 16 bytes of null padded ASCII string representing PHY + * vendor. If the string is set to null, then the vendor name is + * not available. + */ + char phy_vendor_partnumber[16]; + /* + * Up to 16 bytes of null padded ASCII string that identifies + * vendor specific part number of the PHY. If the string is set + * to null, then the vendor specific part number is not + * available. + */ + uint32_t unused_2; + uint8_t unused_3; + uint8_t unused_4; + uint8_t unused_5; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* hwrm_port_qstats */ +/* Description: This function returns per port Ethernet statistics. */ +/* Input (40 bytes) */ +struct hwrm_port_qstats_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint16_t port_id; + /* Port ID of port that is being queried. */ + uint8_t unused_0; + uint8_t unused_1; + uint8_t unused_2[3]; + uint8_t unused_3; + uint64_t tx_stat_host_addr; + /* This is the host address where Tx port statistics will be stored */ + uint64_t rx_stat_host_addr; + /* This is the host address where Rx port statistics will be stored */ +} __attribute__((packed)); + +/* Output (16 bytes) */ +struct hwrm_port_qstats_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint16_t tx_stat_size; + /* The size of TX port statistics block in bytes. */ + uint16_t rx_stat_size; + /* The size of RX port statistics block in bytes. */ + uint8_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* hwrm_port_clr_stats */ +/* + * Description: This function clears per port statistics. The HWRM shall not + * allow a VF driver to clear port statistics. The HWRM shall not allow a PF + * driver to clear port statistics in a partitioning mode. The HWRM may allow a + * PF driver to clear port statistics in the non-partitioning mode. + */ +/* Input (24 bytes) */ +struct hwrm_port_clr_stats_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint16_t port_id; + /* Port ID of port that is being queried. */ + uint16_t unused_0[3]; +} __attribute__((packed)); + +/* Output (16 bytes) */ +struct hwrm_port_clr_stats_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint32_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t unused_3; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* hwrm_port_led_cfg */ +/* + * Description: This function is used to configure LEDs on a given port. Each + * port has individual set of LEDs associated with it. These LEDs are used for + * speed/link configuration as well as activity indicator configuration. Up to + * three LEDs can be configured, one for activity and two for speeds. + */ +/* Input (64 bytes) */ +struct hwrm_port_led_cfg_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint32_t enables; + /* This bit must be '1' for the led0_id field to be configured. */ + #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED0_ID UINT32_C(0x1) + /* This bit must be '1' for the led0_state field to be configured. */ + #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED0_STATE UINT32_C(0x2) + /* This bit must be '1' for the led0_color field to be configured. */ + #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED0_COLOR UINT32_C(0x4) + /* + * This bit must be '1' for the led0_blink_on field to be + * configured. + */ + #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED0_BLINK_ON UINT32_C(0x8) + /* + * This bit must be '1' for the led0_blink_off field to be + * configured. + */ + #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED0_BLINK_OFF UINT32_C(0x10) + /* + * This bit must be '1' for the led0_group_id field to be + * configured. + */ + #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED0_GROUP_ID UINT32_C(0x20) + /* This bit must be '1' for the led1_id field to be configured. */ + #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED1_ID UINT32_C(0x40) + /* This bit must be '1' for the led1_state field to be configured. */ + #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED1_STATE UINT32_C(0x80) + /* This bit must be '1' for the led1_color field to be configured. */ + #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED1_COLOR UINT32_C(0x100) + /* + * This bit must be '1' for the led1_blink_on field to be + * configured. + */ + #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED1_BLINK_ON UINT32_C(0x200) + /* + * This bit must be '1' for the led1_blink_off field to be + * configured. + */ + #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED1_BLINK_OFF UINT32_C(0x400) + /* + * This bit must be '1' for the led1_group_id field to be + * configured. + */ + #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED1_GROUP_ID UINT32_C(0x800) + /* This bit must be '1' for the led2_id field to be configured. */ + #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED2_ID UINT32_C(0x1000) + /* This bit must be '1' for the led2_state field to be configured. */ + #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED2_STATE UINT32_C(0x2000) + /* This bit must be '1' for the led2_color field to be configured. */ + #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED2_COLOR UINT32_C(0x4000) + /* + * This bit must be '1' for the led2_blink_on field to be + * configured. + */ + #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED2_BLINK_ON UINT32_C(0x8000) + /* + * This bit must be '1' for the led2_blink_off field to be + * configured. + */ + #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED2_BLINK_OFF UINT32_C(0x10000) + /* + * This bit must be '1' for the led2_group_id field to be + * configured. + */ + #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED2_GROUP_ID UINT32_C(0x20000) + /* This bit must be '1' for the led3_id field to be configured. */ + #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED3_ID UINT32_C(0x40000) + /* This bit must be '1' for the led3_state field to be configured. */ + #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED3_STATE UINT32_C(0x80000) + /* This bit must be '1' for the led3_color field to be configured. */ + #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED3_COLOR UINT32_C(0x100000) + /* + * This bit must be '1' for the led3_blink_on field to be + * configured. + */ + #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED3_BLINK_ON UINT32_C(0x200000) + /* + * This bit must be '1' for the led3_blink_off field to be + * configured. + */ + #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED3_BLINK_OFF \ + UINT32_C(0x400000) + /* + * This bit must be '1' for the led3_group_id field to be + * configured. + */ + #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED3_GROUP_ID UINT32_C(0x800000) + uint16_t port_id; + /* Port ID of port whose LEDs are configured. */ + uint8_t num_leds; + /* + * The number of LEDs that are being configured. Up to 4 LEDs + * can be configured with this command. + */ + uint8_t rsvd; + /* Reserved field. */ + uint8_t led0_id; + /* An identifier for the LED #0. */ + uint8_t led0_state; + /* The requested state of the LED #0. */ + /* Default state of the LED */ + #define HWRM_PORT_LED_CFG_INPUT_LED0_STATE_DEFAULT UINT32_C(0x0) + /* Off */ + #define HWRM_PORT_LED_CFG_INPUT_LED0_STATE_OFF UINT32_C(0x1) + /* On */ + #define HWRM_PORT_LED_CFG_INPUT_LED0_STATE_ON UINT32_C(0x2) + /* Blink */ + #define HWRM_PORT_LED_CFG_INPUT_LED0_STATE_BLINK UINT32_C(0x3) + /* Blink Alternately */ + #define HWRM_PORT_LED_CFG_INPUT_LED0_STATE_BLINKALT UINT32_C(0x4) + uint8_t led0_color; + /* The requested color of LED #0. */ + /* Default */ + #define HWRM_PORT_LED_CFG_INPUT_LED0_COLOR_DEFAULT UINT32_C(0x0) + /* Amber */ + #define HWRM_PORT_LED_CFG_INPUT_LED0_COLOR_AMBER UINT32_C(0x1) + /* Green */ + #define HWRM_PORT_LED_CFG_INPUT_LED0_COLOR_GREEN UINT32_C(0x2) + /* Green or Amber */ + #define HWRM_PORT_LED_CFG_INPUT_LED0_COLOR_GREENAMBER UINT32_C(0x3) + uint8_t unused_0; + uint16_t led0_blink_on; + /* + * If the LED #0 state is "blink" or "blinkalt", then this field + * represents the requested time in milliseconds to keep LED on + * between cycles. + */ + uint16_t led0_blink_off; + /* + * If the LED #0 state is "blink" or "blinkalt", then this field + * represents the requested time in milliseconds to keep LED off + * between cycles. + */ + uint8_t led0_group_id; + /* + * An identifier for the group of LEDs that LED #0 belongs to. + * If set to 0, then the LED #0 shall not be grouped and shall + * be treated as an individual resource. For all other non-zero + * values of this field, LED #0 shall be grouped together with + * the LEDs with the same group ID value. + */ + uint8_t rsvd0; + /* Reserved field. */ + uint8_t led1_id; + /* An identifier for the LED #1. */ + uint8_t led1_state; + /* The requested state of the LED #1. */ + /* Default state of the LED */ + #define HWRM_PORT_LED_CFG_INPUT_LED1_STATE_DEFAULT UINT32_C(0x0) + /* Off */ + #define HWRM_PORT_LED_CFG_INPUT_LED1_STATE_OFF UINT32_C(0x1) + /* On */ + #define HWRM_PORT_LED_CFG_INPUT_LED1_STATE_ON UINT32_C(0x2) + /* Blink */ + #define HWRM_PORT_LED_CFG_INPUT_LED1_STATE_BLINK UINT32_C(0x3) + /* Blink Alternately */ + #define HWRM_PORT_LED_CFG_INPUT_LED1_STATE_BLINKALT UINT32_C(0x4) + uint8_t led1_color; + /* The requested color of LED #1. */ + /* Default */ + #define HWRM_PORT_LED_CFG_INPUT_LED1_COLOR_DEFAULT UINT32_C(0x0) + /* Amber */ + #define HWRM_PORT_LED_CFG_INPUT_LED1_COLOR_AMBER UINT32_C(0x1) + /* Green */ + #define HWRM_PORT_LED_CFG_INPUT_LED1_COLOR_GREEN UINT32_C(0x2) + /* Green or Amber */ + #define HWRM_PORT_LED_CFG_INPUT_LED1_COLOR_GREENAMBER UINT32_C(0x3) + uint8_t unused_1; + uint16_t led1_blink_on; + /* + * If the LED #1 state is "blink" or "blinkalt", then this field + * represents the requested time in milliseconds to keep LED on + * between cycles. + */ + uint16_t led1_blink_off; + /* + * If the LED #1 state is "blink" or "blinkalt", then this field + * represents the requested time in milliseconds to keep LED off + * between cycles. + */ + uint8_t led1_group_id; + /* + * An identifier for the group of LEDs that LED #1 belongs to. + * If set to 0, then the LED #1 shall not be grouped and shall + * be treated as an individual resource. For all other non-zero + * values of this field, LED #1 shall be grouped together with + * the LEDs with the same group ID value. + */ + uint8_t rsvd1; + /* Reserved field. */ + uint8_t led2_id; + /* An identifier for the LED #2. */ + uint8_t led2_state; + /* The requested state of the LED #2. */ + /* Default state of the LED */ + #define HWRM_PORT_LED_CFG_INPUT_LED2_STATE_DEFAULT UINT32_C(0x0) + /* Off */ + #define HWRM_PORT_LED_CFG_INPUT_LED2_STATE_OFF UINT32_C(0x1) + /* On */ + #define HWRM_PORT_LED_CFG_INPUT_LED2_STATE_ON UINT32_C(0x2) + /* Blink */ + #define HWRM_PORT_LED_CFG_INPUT_LED2_STATE_BLINK UINT32_C(0x3) + /* Blink Alternately */ + #define HWRM_PORT_LED_CFG_INPUT_LED2_STATE_BLINKALT UINT32_C(0x4) + uint8_t led2_color; + /* The requested color of LED #2. */ + /* Default */ + #define HWRM_PORT_LED_CFG_INPUT_LED2_COLOR_DEFAULT UINT32_C(0x0) + /* Amber */ + #define HWRM_PORT_LED_CFG_INPUT_LED2_COLOR_AMBER UINT32_C(0x1) + /* Green */ + #define HWRM_PORT_LED_CFG_INPUT_LED2_COLOR_GREEN UINT32_C(0x2) + /* Green or Amber */ + #define HWRM_PORT_LED_CFG_INPUT_LED2_COLOR_GREENAMBER UINT32_C(0x3) + uint8_t unused_2; + uint16_t led2_blink_on; + /* + * If the LED #2 state is "blink" or "blinkalt", then this field + * represents the requested time in milliseconds to keep LED on + * between cycles. + */ + uint16_t led2_blink_off; + /* + * If the LED #2 state is "blink" or "blinkalt", then this field + * represents the requested time in milliseconds to keep LED off + * between cycles. + */ + uint8_t led2_group_id; + /* + * An identifier for the group of LEDs that LED #2 belongs to. + * If set to 0, then the LED #2 shall not be grouped and shall + * be treated as an individual resource. For all other non-zero + * values of this field, LED #2 shall be grouped together with + * the LEDs with the same group ID value. + */ + uint8_t rsvd2; + /* Reserved field. */ + uint8_t led3_id; + /* An identifier for the LED #3. */ + uint8_t led3_state; + /* The requested state of the LED #3. */ + /* Default state of the LED */ + #define HWRM_PORT_LED_CFG_INPUT_LED3_STATE_DEFAULT UINT32_C(0x0) + /* Off */ + #define HWRM_PORT_LED_CFG_INPUT_LED3_STATE_OFF UINT32_C(0x1) + /* On */ + #define HWRM_PORT_LED_CFG_INPUT_LED3_STATE_ON UINT32_C(0x2) + /* Blink */ + #define HWRM_PORT_LED_CFG_INPUT_LED3_STATE_BLINK UINT32_C(0x3) + /* Blink Alternately */ + #define HWRM_PORT_LED_CFG_INPUT_LED3_STATE_BLINKALT UINT32_C(0x4) + uint8_t led3_color; + /* The requested color of LED #3. */ + /* Default */ + #define HWRM_PORT_LED_CFG_INPUT_LED3_COLOR_DEFAULT UINT32_C(0x0) + /* Amber */ + #define HWRM_PORT_LED_CFG_INPUT_LED3_COLOR_AMBER UINT32_C(0x1) + /* Green */ + #define HWRM_PORT_LED_CFG_INPUT_LED3_COLOR_GREEN UINT32_C(0x2) + /* Green or Amber */ + #define HWRM_PORT_LED_CFG_INPUT_LED3_COLOR_GREENAMBER UINT32_C(0x3) + uint8_t unused_3; + uint16_t led3_blink_on; + /* + * If the LED #3 state is "blink" or "blinkalt", then this field + * represents the requested time in milliseconds to keep LED on + * between cycles. + */ + uint16_t led3_blink_off; + /* + * If the LED #3 state is "blink" or "blinkalt", then this field + * represents the requested time in milliseconds to keep LED off + * between cycles. + */ + uint8_t led3_group_id; + /* + * An identifier for the group of LEDs that LED #3 belongs to. + * If set to 0, then the LED #3 shall not be grouped and shall + * be treated as an individual resource. For all other non-zero + * values of this field, LED #3 shall be grouped together with + * the LEDs with the same group ID value. + */ + uint8_t rsvd3; + /* Reserved field. */ +} __attribute__((packed)); + +/* Output (16 bytes) */ +struct hwrm_port_led_cfg_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint32_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t unused_3; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* hwrm_port_led_qcfg */ +/* + * Description: This function is used to query configuration of LEDs on a given + * port. Each port has individual set of LEDs associated with it. These LEDs are + * used for speed/link configuration as well as activity indicator + * configuration. Up to three LEDs can be configured, one for activity and two + * for speeds. + */ +/* Input (24 bytes) */ +struct hwrm_port_led_qcfg_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint16_t port_id; + /* Port ID of port whose LED configuration is being queried. */ + uint16_t unused_0[3]; +} __attribute__((packed)); + +/* Output (56 bytes) */ +struct hwrm_port_led_qcfg_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint8_t num_leds; + /* + * The number of LEDs that are configured on this port. Up to 4 + * LEDs can be returned in the response. + */ + uint8_t led0_id; + /* An identifier for the LED #0. */ + uint8_t led0_type; + /* The type of LED #0. */ + /* Speed LED */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED0_TYPE_SPEED UINT32_C(0x0) + /* Activity LED */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED0_TYPE_ACTIVITY UINT32_C(0x1) + /* Invalid */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED0_TYPE_INVALID UINT32_C(0xff) + uint8_t led0_state; + /* The current state of the LED #0. */ + /* Default state of the LED */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED0_STATE_DEFAULT UINT32_C(0x0) + /* Off */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED0_STATE_OFF UINT32_C(0x1) + /* On */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED0_STATE_ON UINT32_C(0x2) + /* Blink */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED0_STATE_BLINK UINT32_C(0x3) + /* Blink Alternately */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED0_STATE_BLINKALT UINT32_C(0x4) + uint8_t led0_color; + /* The color of LED #0. */ + /* Default */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED0_COLOR_DEFAULT UINT32_C(0x0) + /* Amber */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED0_COLOR_AMBER UINT32_C(0x1) + /* Green */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED0_COLOR_GREEN UINT32_C(0x2) + /* Green or Amber */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED0_COLOR_GREENAMBER UINT32_C(0x3) + uint8_t unused_0; + uint16_t led0_blink_on; + /* + * If the LED #0 state is "blink" or "blinkalt", then this field + * represents the requested time in milliseconds to keep LED on + * between cycles. + */ + uint16_t led0_blink_off; + /* + * If the LED #0 state is "blink" or "blinkalt", then this field + * represents the requested time in milliseconds to keep LED off + * between cycles. + */ + uint8_t led0_group_id; + /* + * An identifier for the group of LEDs that LED #0 belongs to. + * If set to 0, then the LED #0 is not grouped. For all other + * non-zero values of this field, LED #0 is grouped together + * with the LEDs with the same group ID value. + */ + uint8_t led1_id; + /* An identifier for the LED #1. */ + uint8_t led1_type; + /* The type of LED #1. */ + /* Speed LED */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED1_TYPE_SPEED UINT32_C(0x0) + /* Activity LED */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED1_TYPE_ACTIVITY UINT32_C(0x1) + /* Invalid */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED1_TYPE_INVALID UINT32_C(0xff) + uint8_t led1_state; + /* The current state of the LED #1. */ + /* Default state of the LED */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED1_STATE_DEFAULT UINT32_C(0x0) + /* Off */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED1_STATE_OFF UINT32_C(0x1) + /* On */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED1_STATE_ON UINT32_C(0x2) + /* Blink */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED1_STATE_BLINK UINT32_C(0x3) + /* Blink Alternately */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED1_STATE_BLINKALT UINT32_C(0x4) + uint8_t led1_color; + /* The color of LED #1. */ + /* Default */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED1_COLOR_DEFAULT UINT32_C(0x0) + /* Amber */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED1_COLOR_AMBER UINT32_C(0x1) + /* Green */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED1_COLOR_GREEN UINT32_C(0x2) + /* Green or Amber */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED1_COLOR_GREENAMBER UINT32_C(0x3) + uint8_t unused_1; + uint16_t led1_blink_on; + /* + * If the LED #1 state is "blink" or "blinkalt", then this field + * represents the requested time in milliseconds to keep LED on + * between cycles. + */ + uint16_t led1_blink_off; + /* + * If the LED #1 state is "blink" or "blinkalt", then this field + * represents the requested time in milliseconds to keep LED off + * between cycles. + */ + uint8_t led1_group_id; + /* + * An identifier for the group of LEDs that LED #1 belongs to. + * If set to 0, then the LED #1 is not grouped. For all other + * non-zero values of this field, LED #1 is grouped together + * with the LEDs with the same group ID value. + */ + uint8_t led2_id; + /* An identifier for the LED #2. */ + uint8_t led2_type; + /* The type of LED #2. */ + /* Speed LED */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED2_TYPE_SPEED UINT32_C(0x0) + /* Activity LED */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED2_TYPE_ACTIVITY UINT32_C(0x1) + /* Invalid */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED2_TYPE_INVALID UINT32_C(0xff) + uint8_t led2_state; + /* The current state of the LED #2. */ + /* Default state of the LED */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED2_STATE_DEFAULT UINT32_C(0x0) + /* Off */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED2_STATE_OFF UINT32_C(0x1) + /* On */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED2_STATE_ON UINT32_C(0x2) + /* Blink */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED2_STATE_BLINK UINT32_C(0x3) + /* Blink Alternately */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED2_STATE_BLINKALT UINT32_C(0x4) + uint8_t led2_color; + /* The color of LED #2. */ + /* Default */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED2_COLOR_DEFAULT UINT32_C(0x0) + /* Amber */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED2_COLOR_AMBER UINT32_C(0x1) + /* Green */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED2_COLOR_GREEN UINT32_C(0x2) + /* Green or Amber */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED2_COLOR_GREENAMBER UINT32_C(0x3) + uint8_t unused_2; + uint16_t led2_blink_on; + /* + * If the LED #2 state is "blink" or "blinkalt", then this field + * represents the requested time in milliseconds to keep LED on + * between cycles. + */ + uint16_t led2_blink_off; + /* + * If the LED #2 state is "blink" or "blinkalt", then this field + * represents the requested time in milliseconds to keep LED off + * between cycles. + */ + uint8_t led2_group_id; + /* + * An identifier for the group of LEDs that LED #2 belongs to. + * If set to 0, then the LED #2 is not grouped. For all other + * non-zero values of this field, LED #2 is grouped together + * with the LEDs with the same group ID value. + */ + uint8_t led3_id; + /* An identifier for the LED #3. */ + uint8_t led3_type; + /* The type of LED #3. */ + /* Speed LED */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED3_TYPE_SPEED UINT32_C(0x0) + /* Activity LED */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED3_TYPE_ACTIVITY UINT32_C(0x1) + /* Invalid */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED3_TYPE_INVALID UINT32_C(0xff) + uint8_t led3_state; + /* The current state of the LED #3. */ + /* Default state of the LED */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED3_STATE_DEFAULT UINT32_C(0x0) + /* Off */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED3_STATE_OFF UINT32_C(0x1) + /* On */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED3_STATE_ON UINT32_C(0x2) + /* Blink */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED3_STATE_BLINK UINT32_C(0x3) + /* Blink Alternately */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED3_STATE_BLINKALT UINT32_C(0x4) + uint8_t led3_color; + /* The color of LED #3. */ + /* Default */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED3_COLOR_DEFAULT UINT32_C(0x0) + /* Amber */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED3_COLOR_AMBER UINT32_C(0x1) + /* Green */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED3_COLOR_GREEN UINT32_C(0x2) + /* Green or Amber */ + #define HWRM_PORT_LED_QCFG_OUTPUT_LED3_COLOR_GREENAMBER UINT32_C(0x3) + uint8_t unused_3; + uint16_t led3_blink_on; + /* + * If the LED #3 state is "blink" or "blinkalt", then this field + * represents the requested time in milliseconds to keep LED on + * between cycles. + */ + uint16_t led3_blink_off; + /* + * If the LED #3 state is "blink" or "blinkalt", then this field + * represents the requested time in milliseconds to keep LED off + * between cycles. + */ + uint8_t led3_group_id; + /* + * An identifier for the group of LEDs that LED #3 belongs to. + * If set to 0, then the LED #3 is not grouped. For all other + * non-zero values of this field, LED #3 is grouped together + * with the LEDs with the same group ID value. + */ + uint8_t unused_4; + uint16_t unused_5; + uint8_t unused_6; + uint8_t unused_7; + uint8_t unused_8; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* hwrm_port_led_qcaps */ +/* + * Description: This function is used to query capabilities of LEDs on a given + * port. Each port has individual set of LEDs associated with it. These LEDs are + * used for speed/link configuration as well as activity indicator + * configuration. + */ +/* Input (24 bytes) */ +struct hwrm_port_led_qcaps_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint16_t port_id; + /* Port ID of port whose LED configuration is being queried. */ + uint16_t unused_0[3]; +} __attribute__((packed)); + +/* Output (48 bytes) */ +struct hwrm_port_led_qcaps_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint8_t num_leds; + /* + * The number of LEDs that are configured on this port. Up to 4 + * LEDs can be returned in the response. + */ + uint8_t unused_0[3]; + /* Reserved for future use. */ + uint8_t led0_id; + /* An identifier for the LED #0. */ + uint8_t led0_type; + /* The type of LED #0. */ + /* Speed LED */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED0_TYPE_SPEED UINT32_C(0x0) + /* Activity LED */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED0_TYPE_ACTIVITY UINT32_C(0x1) + /* Invalid */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED0_TYPE_INVALID UINT32_C(0xff) + uint8_t led0_group_id; + /* + * An identifier for the group of LEDs that LED #0 belongs to. + * If set to 0, then the LED #0 cannot be grouped. For all other + * non-zero values of this field, LED #0 is grouped together + * with the LEDs with the same group ID value. + */ + uint8_t unused_1; + uint16_t led0_state_caps; + /* The states supported by LED #0. */ + /* + * If set to 1, this LED is enabled. If set to 0, this LED is + * disabled. + */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED0_STATE_CAPS_ENABLED UINT32_C(0x1) + /* + * If set to 1, off state is supported on this LED. If set to 0, + * off state is not supported on this LED. + */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED0_STATE_CAPS_OFF_SUPPORTED \ + UINT32_C(0x2) + /* + * If set to 1, on state is supported on this LED. If set to 0, + * on state is not supported on this LED. + */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED0_STATE_CAPS_ON_SUPPORTED \ + UINT32_C(0x4) + /* + * If set to 1, blink state is supported on this LED. If set to + * 0, blink state is not supported on this LED. + */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED0_STATE_CAPS_BLINK_SUPPORTED \ + UINT32_C(0x8) + /* + * If set to 1, blink_alt state is supported on this LED. If set + * to 0, blink_alt state is not supported on this LED. + */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED0_STATE_CAPS_BLINK_ALT_SUPPORTED \ + UINT32_C(0x10) + uint16_t led0_color_caps; + /* The colors supported by LED #0. */ + /* reserved */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED0_COLOR_CAPS_RSVD UINT32_C(0x1) + /* + * If set to 1, Amber color is supported on this LED. If set to + * 0, Amber color is not supported on this LED. + */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED0_COLOR_CAPS_AMBER_SUPPORTED \ + UINT32_C(0x2) + /* + * If set to 1, Green color is supported on this LED. If set to + * 0, Green color is not supported on this LED. + */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED0_COLOR_CAPS_GREEN_SUPPORTED \ + UINT32_C(0x4) + uint8_t led1_id; + /* An identifier for the LED #1. */ + uint8_t led1_type; + /* The type of LED #1. */ + /* Speed LED */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED1_TYPE_SPEED UINT32_C(0x0) + /* Activity LED */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED1_TYPE_ACTIVITY UINT32_C(0x1) + /* Invalid */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED1_TYPE_INVALID UINT32_C(0xff) + uint8_t led1_group_id; + /* + * An identifier for the group of LEDs that LED #1 belongs to. + * If set to 0, then the LED #0 cannot be grouped. For all other + * non-zero values of this field, LED #0 is grouped together + * with the LEDs with the same group ID value. + */ + uint8_t unused_2; + uint16_t led1_state_caps; + /* The states supported by LED #1. */ + /* + * If set to 1, this LED is enabled. If set to 0, this LED is + * disabled. + */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED1_STATE_CAPS_ENABLED UINT32_C(0x1) + /* + * If set to 1, off state is supported on this LED. If set to 0, + * off state is not supported on this LED. + */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED1_STATE_CAPS_OFF_SUPPORTED \ + UINT32_C(0x2) + /* + * If set to 1, on state is supported on this LED. If set to 0, + * on state is not supported on this LED. + */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED1_STATE_CAPS_ON_SUPPORTED \ + UINT32_C(0x4) + /* + * If set to 1, blink state is supported on this LED. If set to + * 0, blink state is not supported on this LED. + */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED1_STATE_CAPS_BLINK_SUPPORTED \ + UINT32_C(0x8) + /* + * If set to 1, blink_alt state is supported on this LED. If set + * to 0, blink_alt state is not supported on this LED. + */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED1_STATE_CAPS_BLINK_ALT_SUPPORTED \ + UINT32_C(0x10) + uint16_t led1_color_caps; + /* The colors supported by LED #1. */ + /* reserved */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED1_COLOR_CAPS_RSVD UINT32_C(0x1) + /* + * If set to 1, Amber color is supported on this LED. If set to + * 0, Amber color is not supported on this LED. + */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED1_COLOR_CAPS_AMBER_SUPPORTED \ + UINT32_C(0x2) + /* + * If set to 1, Green color is supported on this LED. If set to + * 0, Green color is not supported on this LED. + */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED1_COLOR_CAPS_GREEN_SUPPORTED \ + UINT32_C(0x4) + uint8_t led2_id; + /* An identifier for the LED #2. */ + uint8_t led2_type; + /* The type of LED #2. */ + /* Speed LED */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED2_TYPE_SPEED UINT32_C(0x0) + /* Activity LED */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED2_TYPE_ACTIVITY UINT32_C(0x1) + /* Invalid */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED2_TYPE_INVALID UINT32_C(0xff) + uint8_t led2_group_id; + /* + * An identifier for the group of LEDs that LED #0 belongs to. + * If set to 0, then the LED #0 cannot be grouped. For all other + * non-zero values of this field, LED #0 is grouped together + * with the LEDs with the same group ID value. + */ + uint8_t unused_3; + uint16_t led2_state_caps; + /* The states supported by LED #2. */ + /* + * If set to 1, this LED is enabled. If set to 0, this LED is + * disabled. + */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED2_STATE_CAPS_ENABLED UINT32_C(0x1) + /* + * If set to 1, off state is supported on this LED. If set to 0, + * off state is not supported on this LED. + */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED2_STATE_CAPS_OFF_SUPPORTED \ + UINT32_C(0x2) + /* + * If set to 1, on state is supported on this LED. If set to 0, + * on state is not supported on this LED. + */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED2_STATE_CAPS_ON_SUPPORTED \ + UINT32_C(0x4) + /* + * If set to 1, blink state is supported on this LED. If set to + * 0, blink state is not supported on this LED. + */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED2_STATE_CAPS_BLINK_SUPPORTED \ + UINT32_C(0x8) + /* + * If set to 1, blink_alt state is supported on this LED. If set + * to 0, blink_alt state is not supported on this LED. + */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED2_STATE_CAPS_BLINK_ALT_SUPPORTED \ + UINT32_C(0x10) + uint16_t led2_color_caps; + /* The colors supported by LED #2. */ + /* reserved */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED2_COLOR_CAPS_RSVD UINT32_C(0x1) + /* + * If set to 1, Amber color is supported on this LED. If set to + * 0, Amber color is not supported on this LED. + */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED2_COLOR_CAPS_AMBER_SUPPORTED \ + UINT32_C(0x2) + /* + * If set to 1, Green color is supported on this LED. If set to + * 0, Green color is not supported on this LED. + */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED2_COLOR_CAPS_GREEN_SUPPORTED \ + UINT32_C(0x4) + uint8_t led3_id; + /* An identifier for the LED #3. */ + uint8_t led3_type; + /* The type of LED #3. */ + /* Speed LED */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED3_TYPE_SPEED UINT32_C(0x0) + /* Activity LED */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED3_TYPE_ACTIVITY UINT32_C(0x1) + /* Invalid */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED3_TYPE_INVALID UINT32_C(0xff) + uint8_t led3_group_id; + /* + * An identifier for the group of LEDs that LED #3 belongs to. + * If set to 0, then the LED #0 cannot be grouped. For all other + * non-zero values of this field, LED #0 is grouped together + * with the LEDs with the same group ID value. + */ + uint8_t unused_4; + uint16_t led3_state_caps; + /* The states supported by LED #3. */ + /* + * If set to 1, this LED is enabled. If set to 0, this LED is + * disabled. + */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED3_STATE_CAPS_ENABLED UINT32_C(0x1) + /* + * If set to 1, off state is supported on this LED. If set to 0, + * off state is not supported on this LED. + */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED3_STATE_CAPS_OFF_SUPPORTED \ + UINT32_C(0x2) + /* + * If set to 1, on state is supported on this LED. If set to 0, + * on state is not supported on this LED. + */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED3_STATE_CAPS_ON_SUPPORTED \ + UINT32_C(0x4) + /* + * If set to 1, blink state is supported on this LED. If set to + * 0, blink state is not supported on this LED. + */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED3_STATE_CAPS_BLINK_SUPPORTED \ + UINT32_C(0x8) + /* + * If set to 1, blink_alt state is supported on this LED. If set + * to 0, blink_alt state is not supported on this LED. + */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED3_STATE_CAPS_BLINK_ALT_SUPPORTED \ + UINT32_C(0x10) + uint16_t led3_color_caps; + /* The colors supported by LED #3. */ + /* reserved */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED3_COLOR_CAPS_RSVD UINT32_C(0x1) + /* + * If set to 1, Amber color is supported on this LED. If set to + * 0, Amber color is not supported on this LED. + */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED3_COLOR_CAPS_AMBER_SUPPORTED \ + UINT32_C(0x2) + /* + * If set to 1, Green color is supported on this LED. If set to + * 0, Green color is not supported on this LED. + */ + #define HWRM_PORT_LED_QCAPS_OUTPUT_LED3_COLOR_CAPS_GREEN_SUPPORTED \ + UINT32_C(0x4) + uint8_t unused_5; + uint8_t unused_6; + uint8_t unused_7; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ } __attribute__((packed)); /* hwrm_queue_qportcfg */ @@ -3920,794 +6785,340 @@ struct hwrm_ver_get_output { * then the driver shall only use queues for which service profiles are pre- * configured. */ - -/* Input (24 bytes) */ +/* Input (24 bytes) */ struct hwrm_queue_qportcfg_input { - /* - * This value indicates what type of request this is. The format for the - * rest of the command is determined by this field. - */ uint16_t req_type; - /* - * This value indicates the what completion ring the request will be - * optionally completed on. If the value is -1, then no CR completion - * will be generated. Any other value must be a valid CR ring_id value - * for this function. + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. */ uint16_t cmpl_ring; - - /* This value indicates the command sequence number. */ - uint16_t seq_id; - /* - * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids - * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ uint16_t target_id; - /* - * This is the host address where the response will be written when the - * request is complete. This area must be 16B aligned and must be - * cleared to zero before the request is made. + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM */ uint64_t resp_addr; - + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint32_t flags; /* * Enumeration denoting the RX, TX type of the resource. This - * enumeration is used for resources that are similar for both TX and RX - * paths of the chip. + * enumeration is used for resources that are similar for both + * TX and RX paths of the chip. */ - #define HWRM_QUEUE_QPORTCFG_INPUT_FLAGS_PATH \ - UINT32_C(0x1) - /* tx path */ - #define HWRM_QUEUE_QPORTCFG_INPUT_FLAGS_PATH_TX \ - (UINT32_C(0x0) << 0) - /* rx path */ - #define HWRM_QUEUE_QPORTCFG_INPUT_FLAGS_PATH_RX \ - (UINT32_C(0x1) << 0) + #define HWRM_QUEUE_QPORTCFG_INPUT_FLAGS_PATH UINT32_C(0x1) + /* tx path */ + #define HWRM_QUEUE_QPORTCFG_INPUT_FLAGS_PATH_TX UINT32_C(0x0) + /* rx path */ + #define HWRM_QUEUE_QPORTCFG_INPUT_FLAGS_PATH_RX UINT32_C(0x1) #define HWRM_QUEUE_QPORTCFG_INPUT_FLAGS_PATH_LAST \ - HWRM_QUEUE_QPORTCFG_INPUT_FLAGS_PATH_RX - uint32_t flags; - - /* - * Port ID of port for which the queue configuration is being queried. - * This field is only required when sent by IPC. - */ + QUEUE_QPORTCFG_INPUT_FLAGS_PATH_RX uint16_t port_id; - + /* + * Port ID of port for which the queue configuration is being + * queried. This field is only required when sent by IPC. + */ uint16_t unused_0; } __attribute__((packed)); -/* hwrm_ring_alloc */ -/* - * Description: This command allocates and does basic preparation for a ring. - */ - -/* Input (80 bytes) */ -struct hwrm_ring_alloc_input { - /* - * This value indicates what type of request this is. The format for the - * rest of the command is determined by this field. - */ - uint16_t req_type; - - /* - * This value indicates the what completion ring the request will be - * optionally completed on. If the value is -1, then no CR completion - * will be generated. Any other value must be a valid CR ring_id value - * for this function. - */ - uint16_t cmpl_ring; - - /* This value indicates the command sequence number. */ - uint16_t seq_id; - - /* - * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids - * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM - */ - uint16_t target_id; - - /* - * This is the host address where the response will be written when the - * request is complete. This area must be 16B aligned and must be - * cleared to zero before the request is made. - */ - uint64_t resp_addr; - - /* This bit must be '1' for the Reserved1 field to be configured. */ - #define HWRM_RING_ALLOC_INPUT_ENABLES_RESERVED1 UINT32_C(0x1) - /* This bit must be '1' for the Reserved2 field to be configured. */ - #define HWRM_RING_ALLOC_INPUT_ENABLES_RESERVED2 UINT32_C(0x2) - /* This bit must be '1' for the Reserved3 field to be configured. */ - #define HWRM_RING_ALLOC_INPUT_ENABLES_RESERVED3 UINT32_C(0x4) - /* - * This bit must be '1' for the stat_ctx_id_valid field to be - * configured. - */ - #define HWRM_RING_ALLOC_INPUT_ENABLES_STAT_CTX_ID_VALID UINT32_C(0x8) - /* This bit must be '1' for the Reserved4 field to be configured. */ - #define HWRM_RING_ALLOC_INPUT_ENABLES_RESERVED4 UINT32_C(0x10) - /* This bit must be '1' for the max_bw_valid field to be configured. */ - #define HWRM_RING_ALLOC_INPUT_ENABLES_MAX_BW_VALID UINT32_C(0x20) - uint32_t enables; - - /* Ring Type. */ - /* Completion Ring (CR) */ - #define HWRM_RING_ALLOC_INPUT_RING_TYPE_CMPL (UINT32_C(0x0) << 0) - /* TX Ring (TR) */ - #define HWRM_RING_ALLOC_INPUT_RING_TYPE_TX (UINT32_C(0x1) << 0) - /* RX Ring (RR) */ - #define HWRM_RING_ALLOC_INPUT_RING_TYPE_RX (UINT32_C(0x2) << 0) - uint8_t ring_type; - - uint8_t unused_0; - uint16_t unused_1; - - /* This value is a pointer to the page table for the Ring. */ - uint64_t page_tbl_addr; - - /* First Byte Offset of the first entry in the first page. */ - uint32_t fbo; - - /* - * Actual page size in 2^page_size. The supported range is increments in - * powers of 2 from 16 bytes to 1GB. - 4 = 16 B Page size is 16 B. - 12 - * = 4 KB Page size is 4 KB. - 13 = 8 KB Page size is 8 KB. - 16 = 64 KB - * Page size is 64 KB. - 22 = 2 MB Page size is 2 MB. - 23 = 4 MB Page - * size is 4 MB. - 31 = 1 GB Page size is 1 GB. - */ - uint8_t page_size; - - /* - * This value indicates the depth of page table. For this version of the - * specification, value other than 0 or 1 shall be considered as an - * invalid value. When the page_tbl_depth = 0, then it is treated as a - * special case with the following. 1. FBO and page size fields are not - * valid. 2. page_tbl_addr is the physical address of the first element - * of the ring. - */ - uint8_t page_tbl_depth; - - uint8_t unused_2; - uint8_t unused_3; - - /* - * Number of 16B units in the ring. Minimum size for a ring is 16 16B - * entries. - */ - uint32_t length; - - /* - * Logical ring number for the ring to be allocated. This value - * determines the position in the doorbell area where the update to the - * ring will be made. For completion rings, this value is also the MSI-X - * vector number for the function the completion ring is associated - * with. - */ - uint16_t logical_id; - - /* - * This field is used only when ring_type is a TX ring. This value - * indicates what completion ring the TX ring is associated with. - */ - uint16_t cmpl_ring_id; - - /* - * This field is used only when ring_type is a TX ring. This value - * indicates what CoS queue the TX ring is associated with. - */ - uint16_t queue_id; - - uint8_t unused_4; - uint8_t unused_5; - - /* This field is reserved for the future use. It shall be set to 0. */ - uint32_t reserved1; - /* This field is reserved for the future use. It shall be set to 0. */ - uint16_t reserved2; - - uint8_t unused_6; - uint8_t unused_7; - /* This field is reserved for the future use. It shall be set to 0. */ - uint32_t reserved3; - - /* - * This field is used only when ring_type is a TX ring. This input - * indicates what statistics context this ring should be associated - * with. - */ - uint32_t stat_ctx_id; - - /* This field is reserved for the future use. It shall be set to 0. */ - uint32_t reserved4; - - /* - * This field is used only when ring_type is a TX ring. Maximum BW - * allocated to this TX ring in Mbps. The HWRM will translate this value - * into byte counter and time interval used for this ring inside the - * device. - */ - uint32_t max_bw; - - /* - * This field is used only when ring_type is a Completion ring. This - * value indicates what interrupt mode should be used on this completion - * ring. Note: In the legacy interrupt mode, no more than 16 completion - * rings are allowed. - */ - /* Legacy INTA */ - #define HWRM_RING_ALLOC_INPUT_INT_MODE_LEGACY (UINT32_C(0x0) << 0) - /* Reserved */ - #define HWRM_RING_ALLOC_INPUT_INT_MODE_RSVD (UINT32_C(0x1) << 0) - /* MSI-X */ - #define HWRM_RING_ALLOC_INPUT_INT_MODE_MSIX (UINT32_C(0x2) << 0) - /* No Interrupt - Polled mode */ - #define HWRM_RING_ALLOC_INPUT_INT_MODE_POLL (UINT32_C(0x3) << 0) - uint8_t int_mode; - - uint8_t unused_8[3]; -} __attribute__((packed)); - -/* Output (16 bytes) */ - -struct hwrm_ring_alloc_output { - /* - * Pass/Fail or error type Note: receiver to verify the in parameters, - * and fail the call with an error when appropriate - */ +/* Output (32 bytes) */ +struct hwrm_queue_qportcfg_output { uint16_t error_code; - + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; /* This field returns the type of original request. */ - uint16_t req_type; - + uint16_t seq_id; /* This field provides original sequence number of the command. */ - uint16_t seq_id; - - /* - * This field is the length of the response in bytes. The last byte of - * the response is a valid flag that will read as '1' when the command - * has been completely written to memory. - */ uint16_t resp_len; - - /* Physical number of ring allocated. */ - uint16_t ring_id; - - /* Logical number of ring allocated. */ - uint16_t logical_ring_id; - - uint8_t unused_0; - uint8_t unused_1; - uint8_t unused_2; - /* - * This field is used in Output records to indicate that the output is - * completely written to RAM. This field should be read as '1' to - * indicate that the output has been completely written. When writing a - * command completion or response to an internal processor, the order of - * writes has to be such that this field is written last. + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. */ + uint8_t max_configurable_queues; + /* + * The maximum number of queues that can be configured on this + * port. Valid values range from 1 through 8. + */ + uint8_t max_configurable_lossless_queues; + /* + * The maximum number of lossless queues that can be configured + * on this port. Valid values range from 0 through 8. + */ + uint8_t queue_cfg_allowed; + /* + * Bitmask indicating which queues can be configured by the + * hwrm_queue_cfg command. Each bit represents a specific queue + * where bit 0 represents queue 0 and bit 7 represents queue 7. + * # A value of 0 indicates that the queue is not configurable + * by the hwrm_queue_cfg command. # A value of 1 indicates that + * the queue is configurable. # A hwrm_queue_cfg command shall + * return error when trying to configure a queue not + * configurable. + */ + uint8_t queue_cfg_info; + /* Information about queue configuration. */ + /* + * If this flag is set to '1', then the queues are configured + * asymmetrically on TX and RX sides. If this flag is set to + * '0', then the queues are configured symmetrically on TX and + * RX sides. For symmetric configuration, the queue + * configuration including queue ids and service profiles on the + * TX side is the same as the corresponding queue configuration + * on the RX side. + */ + #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_CFG_INFO_ASYM_CFG UINT32_C(0x1) + uint8_t queue_pfcenable_cfg_allowed; + /* + * Bitmask indicating which queues can be configured by the + * hwrm_queue_pfcenable_cfg command. Each bit represents a + * specific priority where bit 0 represents priority 0 and bit 7 + * represents priority 7. # A value of 0 indicates that the + * priority is not configurable by the hwrm_queue_pfcenable_cfg + * command. # A value of 1 indicates that the priority is + * configurable. # A hwrm_queue_pfcenable_cfg command shall + * return error when trying to configure a priority that is not + * configurable. + */ + uint8_t queue_pri2cos_cfg_allowed; + /* + * Bitmask indicating which queues can be configured by the + * hwrm_queue_pri2cos_cfg command. Each bit represents a + * specific queue where bit 0 represents queue 0 and bit 7 + * represents queue 7. # A value of 0 indicates that the queue + * is not configurable by the hwrm_queue_pri2cos_cfg command. # + * A value of 1 indicates that the queue is configurable. # A + * hwrm_queue_pri2cos_cfg command shall return error when trying + * to configure a queue that is not configurable. + */ + uint8_t queue_cos2bw_cfg_allowed; + /* + * Bitmask indicating which queues can be configured by the + * hwrm_queue_pri2cos_cfg command. Each bit represents a + * specific queue where bit 0 represents queue 0 and bit 7 + * represents queue 7. # A value of 0 indicates that the queue + * is not configurable by the hwrm_queue_pri2cos_cfg command. # + * A value of 1 indicates that the queue is configurable. # A + * hwrm_queue_pri2cos_cfg command shall return error when trying + * to configure a queue not configurable. + */ + uint8_t queue_id0; + /* + * ID of CoS Queue 0. FF - Invalid id # This ID can be used on + * any subsequent call to an hwrm command that takes a queue id. + * # IDs must always be queried by this command before any use + * by the driver or software. # Any driver or software should + * not make any assumptions about queue IDs. # A value of 0xff + * indicates that the queue is not available. # Available queues + * may not be in sequential order. + */ + uint8_t queue_id0_service_profile; + /* This value is applicable to CoS queues only. */ + /* Lossy (best-effort) */ + #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID0_SERVICE_PROFILE_LOSSY \ + UINT32_C(0x0) + /* Lossless */ + #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID0_SERVICE_PROFILE_LOSSLESS \ + UINT32_C(0x1) + /* + * Set to 0xFF... (All Fs) if there is no + * service profile specified + */ + #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID0_SERVICE_PROFILE_UNKNOWN \ + UINT32_C(0xff) + uint8_t queue_id1; + /* + * ID of CoS Queue 1. FF - Invalid id # This ID can be used on + * any subsequent call to an hwrm command that takes a queue id. + * # IDs must always be queried by this command before any use + * by the driver or software. # Any driver or software should + * not make any assumptions about queue IDs. # A value of 0xff + * indicates that the queue is not available. # Available queues + * may not be in sequential order. + */ + uint8_t queue_id1_service_profile; + /* This value is applicable to CoS queues only. */ + /* Lossy (best-effort) */ + #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID1_SERVICE_PROFILE_LOSSY \ + UINT32_C(0x0) + /* Lossless */ + #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID1_SERVICE_PROFILE_LOSSLESS \ + UINT32_C(0x1) + /* + * Set to 0xFF... (All Fs) if there is no + * service profile specified + */ + #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID1_SERVICE_PROFILE_UNKNOWN \ + UINT32_C(0xff) + uint8_t queue_id2; + /* + * ID of CoS Queue 2. FF - Invalid id # This ID can be used on + * any subsequent call to an hwrm command that takes a queue id. + * # IDs must always be queried by this command before any use + * by the driver or software. # Any driver or software should + * not make any assumptions about queue IDs. # A value of 0xff + * indicates that the queue is not available. # Available queues + * may not be in sequential order. + */ + uint8_t queue_id2_service_profile; + /* This value is applicable to CoS queues only. */ + /* Lossy (best-effort) */ + #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID2_SERVICE_PROFILE_LOSSY \ + UINT32_C(0x0) + /* Lossless */ + #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID2_SERVICE_PROFILE_LOSSLESS \ + UINT32_C(0x1) + /* + * Set to 0xFF... (All Fs) if there is no + * service profile specified + */ + #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID2_SERVICE_PROFILE_UNKNOWN \ + UINT32_C(0xff) + uint8_t queue_id3; + /* + * ID of CoS Queue 3. FF - Invalid id # This ID can be used on + * any subsequent call to an hwrm command that takes a queue id. + * # IDs must always be queried by this command before any use + * by the driver or software. # Any driver or software should + * not make any assumptions about queue IDs. # A value of 0xff + * indicates that the queue is not available. # Available queues + * may not be in sequential order. + */ + uint8_t queue_id3_service_profile; + /* This value is applicable to CoS queues only. */ + /* Lossy (best-effort) */ + #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID3_SERVICE_PROFILE_LOSSY \ + UINT32_C(0x0) + /* Lossless */ + #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID3_SERVICE_PROFILE_LOSSLESS \ + UINT32_C(0x1) + /* + * Set to 0xFF... (All Fs) if there is no + * service profile specified + */ + #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID3_SERVICE_PROFILE_UNKNOWN \ + UINT32_C(0xff) + uint8_t queue_id4; + /* + * ID of CoS Queue 4. FF - Invalid id # This ID can be used on + * any subsequent call to an hwrm command that takes a queue id. + * # IDs must always be queried by this command before any use + * by the driver or software. # Any driver or software should + * not make any assumptions about queue IDs. # A value of 0xff + * indicates that the queue is not available. # Available queues + * may not be in sequential order. + */ + uint8_t queue_id4_service_profile; + /* This value is applicable to CoS queues only. */ + /* Lossy (best-effort) */ + #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID4_SERVICE_PROFILE_LOSSY \ + UINT32_C(0x0) + /* Lossless */ + #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID4_SERVICE_PROFILE_LOSSLESS \ + UINT32_C(0x1) + /* + * Set to 0xFF... (All Fs) if there is no + * service profile specified + */ + #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID4_SERVICE_PROFILE_UNKNOWN \ + UINT32_C(0xff) + uint8_t queue_id5; + /* + * ID of CoS Queue 5. FF - Invalid id # This ID can be used on + * any subsequent call to an hwrm command that takes a queue id. + * # IDs must always be queried by this command before any use + * by the driver or software. # Any driver or software should + * not make any assumptions about queue IDs. # A value of 0xff + * indicates that the queue is not available. # Available queues + * may not be in sequential order. + */ + uint8_t queue_id5_service_profile; + /* This value is applicable to CoS queues only. */ + /* Lossy (best-effort) */ + #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID5_SERVICE_PROFILE_LOSSY \ + UINT32_C(0x0) + /* Lossless */ + #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID5_SERVICE_PROFILE_LOSSLESS \ + UINT32_C(0x1) + /* + * Set to 0xFF... (All Fs) if there is no + * service profile specified + */ + #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID5_SERVICE_PROFILE_UNKNOWN \ + UINT32_C(0xff) + uint8_t queue_id6; + /* + * ID of CoS Queue 6. FF - Invalid id # This ID can be used on + * any subsequent call to an hwrm command that takes a queue id. + * # IDs must always be queried by this command before any use + * by the driver or software. # Any driver or software should + * not make any assumptions about queue IDs. # A value of 0xff + * indicates that the queue is not available. # Available queues + * may not be in sequential order. + */ + uint8_t queue_id6_service_profile; + /* This value is applicable to CoS queues only. */ + /* Lossy (best-effort) */ + #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID6_SERVICE_PROFILE_LOSSY \ + UINT32_C(0x0) + /* Lossless */ + #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID6_SERVICE_PROFILE_LOSSLESS \ + UINT32_C(0x1) + /* + * Set to 0xFF... (All Fs) if there is no + * service profile specified + */ + #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID6_SERVICE_PROFILE_UNKNOWN \ + UINT32_C(0xff) + uint8_t queue_id7; + /* + * ID of CoS Queue 7. FF - Invalid id # This ID can be used on + * any subsequent call to an hwrm command that takes a queue id. + * # IDs must always be queried by this command before any use + * by the driver or software. # Any driver or software should + * not make any assumptions about queue IDs. # A value of 0xff + * indicates that the queue is not available. # Available queues + * may not be in sequential order. + */ + uint8_t queue_id7_service_profile; + /* This value is applicable to CoS queues only. */ + /* Lossy (best-effort) */ + #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID7_SERVICE_PROFILE_LOSSY \ + UINT32_C(0x0) + /* Lossless */ + #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID7_SERVICE_PROFILE_LOSSLESS \ + UINT32_C(0x1) + /* + * Set to 0xFF... (All Fs) if there is no + * service profile specified + */ + #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID7_SERVICE_PROFILE_UNKNOWN \ + UINT32_C(0xff) uint8_t valid; -} __attribute__((packed)); - -/* hwrm_ring_free */ -/* - * Description: This command is used to free a ring and associated resources. - */ -/* Input (24 bytes) */ - -struct hwrm_ring_free_input { /* - * This value indicates what type of request this is. The format for the - * rest of the command is determined by this field. + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. */ - uint16_t req_type; - - /* - * This value indicates the what completion ring the request will be - * optionally completed on. If the value is -1, then no CR completion - * will be generated. Any other value must be a valid CR ring_id value - * for this function. - */ - uint16_t cmpl_ring; - - /* This value indicates the command sequence number. */ - uint16_t seq_id; - - /* - * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids - * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM - */ - uint16_t target_id; - - /* - * This is the host address where the response will be written when the - * request is complete. This area must be 16B aligned and must be - * cleared to zero before the request is made. - */ - uint64_t resp_addr; - - /* Ring Type. */ - /* Completion Ring (CR) */ - #define HWRM_RING_FREE_INPUT_RING_TYPE_CMPL (UINT32_C(0x0) << 0) - /* TX Ring (TR) */ - #define HWRM_RING_FREE_INPUT_RING_TYPE_TX (UINT32_C(0x1) << 0) - /* RX Ring (RR) */ - #define HWRM_RING_FREE_INPUT_RING_TYPE_RX (UINT32_C(0x2) << 0) - uint8_t ring_type; - - uint8_t unused_0; - - /* Physical number of ring allocated. */ - uint16_t ring_id; - - uint32_t unused_1; -} __attribute__((packed)); - -/* Output (16 bytes) */ -struct hwrm_ring_free_output { - /* - * Pass/Fail or error type Note: receiver to verify the in parameters, - * and fail the call with an error when appropriate - */ - uint16_t error_code; - - /* This field returns the type of original request. */ - uint16_t req_type; - - /* This field provides original sequence number of the command. */ - uint16_t seq_id; - - /* - * This field is the length of the response in bytes. The last byte of - * the response is a valid flag that will read as '1' when the command - * has been completely written to memory. - */ - uint16_t resp_len; - - uint32_t unused_0; - uint8_t unused_1; - uint8_t unused_2; - uint8_t unused_3; - - /* - * This field is used in Output records to indicate that the output is - * completely written to RAM. This field should be read as '1' to - * indicate that the output has been completely written. When writing a - * command completion or response to an internal processor, the order of - * writes has to be such that this field is written last. - */ - uint8_t valid; -} __attribute__((packed)); - -/* hwrm_ring_grp_alloc */ -/* - * Description: This API allocates and does basic preparation for a ring group. - */ - -/* Input (24 bytes) */ -struct hwrm_ring_grp_alloc_input { - /* - * This value indicates what type of request this is. The format for the - * rest of the command is determined by this field. - */ - uint16_t req_type; - - /* - * This value indicates the what completion ring the request will be - * optionally completed on. If the value is -1, then no CR completion - * will be generated. Any other value must be a valid CR ring_id value - * for this function. - */ - uint16_t cmpl_ring; - - /* This value indicates the command sequence number. */ - uint16_t seq_id; - - /* - * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids - * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM - */ - uint16_t target_id; - - /* - * This is the host address where the response will be written when the - * request is complete. This area must be 16B aligned and must be - * cleared to zero before the request is made. - */ - uint64_t resp_addr; - - /* This value identifies the CR associated with the ring group. */ - uint16_t cr; - - /* This value identifies the main RR associated with the ring group. */ - uint16_t rr; - - /* - * This value identifies the aggregation RR associated with the ring - * group. If this value is 0xFF... (All Fs), then no Aggregation ring - * will be set. - */ - uint16_t ar; - - /* - * This value identifies the statistics context associated with the ring - * group. - */ - uint16_t sc; -} __attribute__((packed)); - -/* Output (16 bytes) */ -struct hwrm_ring_grp_alloc_output { - /* - * Pass/Fail or error type Note: receiver to verify the in parameters, - * and fail the call with an error when appropriate - */ - uint16_t error_code; - - /* This field returns the type of original request. */ - uint16_t req_type; - - /* This field provides original sequence number of the command. */ - uint16_t seq_id; - - /* - * This field is the length of the response in bytes. The last byte of - * the response is a valid flag that will read as '1' when the command - * has been completely written to memory. - */ - uint16_t resp_len; - - /* - * This is the ring group ID value. Use this value to program the - * default ring group for the VNIC or as table entries in an RSS/COS - * context. - */ - uint32_t ring_group_id; - - uint8_t unused_0; - uint8_t unused_1; - uint8_t unused_2; - - /* - * This field is used in Output records to indicate that the output is - * completely written to RAM. This field should be read as '1' to - * indicate that the output has been completely written. When writing a - * command completion or response to an internal processor, the order of - * writes has to be such that this field is written last. - */ - uint8_t valid; -} __attribute__((packed)); - -/* hwrm_ring_grp_free */ -/* - * Description: This API frees a ring group and associated resources. # If a - * ring in the ring group is reset or free, then the associated rings in the - * ring group shall also be reset/free using hwrm_ring_free. # A function driver - * shall always use hwrm_ring_grp_free after freeing all rings in a group. # As - * a part of executing this command, the HWRM shall reset all associated ring - * group resources. - */ - -/* Input (24 bytes) */ -struct hwrm_ring_grp_free_input { - /* - * This value indicates what type of request this is. The format for the - * rest of the command is determined by this field. - */ - uint16_t req_type; - - /* - * This value indicates the what completion ring the request will be - * optionally completed on. If the value is -1, then no CR completion - * will be generated. Any other value must be a valid CR ring_id value - * for this function. - */ - uint16_t cmpl_ring; - - /* This value indicates the command sequence number. */ - uint16_t seq_id; - - /* - * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids - * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM - */ - uint16_t target_id; - - /* - * This is the host address where the response will be written when the - * request is complete. This area must be 16B aligned and must be - * cleared to zero before the request is made. - */ - uint64_t resp_addr; - - /* This is the ring group ID value. */ - uint32_t ring_group_id; - - uint32_t unused_0; -} __attribute__((packed)); - -/* Output (16 bytes) */ -struct hwrm_ring_grp_free_output { - /* - * Pass/Fail or error type Note: receiver to verify the in parameters, - * and fail the call with an error when appropriate - */ - uint16_t error_code; - - /* This field returns the type of original request. */ - uint16_t req_type; - - /* This field provides original sequence number of the command. */ - uint16_t seq_id; - - /* - * This field is the length of the response in bytes. The last byte of - * the response is a valid flag that will read as '1' when the command - * has been completely written to memory. - */ - uint16_t resp_len; - - uint32_t unused_0; - uint8_t unused_1; - uint8_t unused_2; - uint8_t unused_3; - - /* - * This field is used in Output records to indicate that the output is - * completely written to RAM. This field should be read as '1' to - * indicate that the output has been completely written. When writing a - * command completion or response to an internal processor, the order of - * writes has to be such that this field is written last. - */ - uint8_t valid; -} __attribute__((packed)); - -/* hwrm_stat_ctx_alloc */ -/* - * Description: This command allocates and does basic preparation for a stat - * context. - */ - -/* Input (32 bytes) */ -struct hwrm_stat_ctx_alloc_input { - /* - * This value indicates what type of request this is. The format for the - * rest of the command is determined by this field. - */ - uint16_t req_type; - - /* - * This value indicates the what completion ring the request will be - * optionally completed on. If the value is -1, then no CR completion - * will be generated. Any other value must be a valid CR ring_id value - * for this function. - */ - uint16_t cmpl_ring; - - /* This value indicates the command sequence number. */ - uint16_t seq_id; - - /* - * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids - * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM - */ - uint16_t target_id; - - /* - * This is the host address where the response will be written when the - * request is complete. This area must be 16B aligned and must be - * cleared to zero before the request is made. - */ - uint64_t resp_addr; - - /* This is the address for statistic block. */ - uint64_t stats_dma_addr; - - /* - * The statistic block update period in ms. e.g. 250ms, 500ms, 750ms, - * 1000ms. - */ - uint32_t update_period_ms; - - uint32_t unused_0; -} __attribute__((packed)); - -/* Output (16 bytes) */ -struct hwrm_stat_ctx_alloc_output { - /* - * Pass/Fail or error type Note: receiver to verify the in parameters, - * and fail the call with an error when appropriate - */ - uint16_t error_code; - - /* This field returns the type of original request. */ - uint16_t req_type; - - /* This field provides original sequence number of the command. */ - uint16_t seq_id; - - /* - * This field is the length of the response in bytes. The last byte of - * the response is a valid flag that will read as '1' when the command - * has been completely written to memory. - */ - uint16_t resp_len; - - /* This is the statistics context ID value. */ - uint32_t stat_ctx_id; - - uint8_t unused_0; - uint8_t unused_1; - uint8_t unused_2; - - /* - * This field is used in Output records to indicate that the output is - * completely written to RAM. This field should be read as '1' to - * indicate that the output has been completely written. When writing a - * command completion or response to an internal processor, the order of - * writes has to be such that this field is written last. - */ - uint8_t valid; -} __attribute__((packed)); - -/* hwrm_stat_ctx_clr_stats */ -/* Description: This command clears statistics of a context. */ - -/* Input (24 bytes) */ -struct hwrm_stat_ctx_clr_stats_input { - /* - * This value indicates what type of request this is. The format for the - * rest of the command is determined by this field. - */ - uint16_t req_type; - - /* - * This value indicates the what completion ring the request will be - * optionally completed on. If the value is -1, then no CR completion - * will be generated. Any other value must be a valid CR ring_id value - * for this function. - */ - uint16_t cmpl_ring; - - /* This value indicates the command sequence number. */ - uint16_t seq_id; - - /* - * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids - * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM - */ - uint16_t target_id; - - /* - * This is the host address where the response will be written when the - * request is complete. This area must be 16B aligned and must be - * cleared to zero before the request is made. - */ - uint64_t resp_addr; - - /* ID of the statistics context that is being queried. */ - uint32_t stat_ctx_id; - - uint32_t unused_0; -} __attribute__((packed)); - -/* Output (16 bytes) */ -struct hwrm_stat_ctx_clr_stats_output { - /* - * Pass/Fail or error type Note: receiver to verify the in parameters, - * and fail the call with an error when appropriate - */ - uint16_t error_code; - - /* This field returns the type of original request. */ - uint16_t req_type; - - /* This field provides original sequence number of the command. */ - uint16_t seq_id; - - /* - * This field is the length of the response in bytes. The last byte of - * the response is a valid flag that will read as '1' when the command - * has been completely written to memory. - */ - uint16_t resp_len; - - uint32_t unused_0; - uint8_t unused_1; - uint8_t unused_2; - uint8_t unused_3; - - /* - * This field is used in Output records to indicate that the output is - * completely written to RAM. This field should be read as '1' to - * indicate that the output has been completely written. When writing a - * command completion or response to an internal processor, the order of - * writes has to be such that this field is written last. - */ - uint8_t valid; -} __attribute__((packed)); - -/* hwrm_stat_ctx_free */ -/* Description: This command is used to free a stat context. */ -/* Input (24 bytes) */ - -struct hwrm_stat_ctx_free_input { - /* - * This value indicates what type of request this is. The format for the - * rest of the command is determined by this field. - */ - uint16_t req_type; - - /* - * This value indicates the what completion ring the request will be - * optionally completed on. If the value is -1, then no CR completion - * will be generated. Any other value must be a valid CR ring_id value - * for this function. - */ - uint16_t cmpl_ring; - - /* This value indicates the command sequence number. */ - uint16_t seq_id; - - /* - * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids - * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM - */ - uint16_t target_id; - - /* - * This is the host address where the response will be written when the - * request is complete. This area must be 16B aligned and must be - * cleared to zero before the request is made. - */ - uint64_t resp_addr; - - /* ID of the statistics context that is being queried. */ - uint32_t stat_ctx_id; - - uint32_t unused_0; -} __attribute__((packed)); - -/* Output (16 bytes) */ - -struct hwrm_stat_ctx_free_output { - /* - * Pass/Fail or error type Note: receiver to verify the in parameters, - * and fail the call with an error when appropriate - */ - uint16_t error_code; - - /* This field returns the type of original request. */ - uint16_t req_type; - - /* This field provides original sequence number of the command. */ - uint16_t seq_id; - - /* - * This field is the length of the response in bytes. The last byte of - * the response is a valid flag that will read as '1' when the command - * has been completely written to memory. - */ - uint16_t resp_len; - - /* This is the statistics context ID value. */ - uint32_t stat_ctx_id; - - uint8_t unused_0; - uint8_t unused_1; - uint8_t unused_2; - - /* - * This field is used in Output records to indicate that the output is - * completely written to RAM. This field should be read as '1' to - * indicate that the output has been completely written. When writing a - * command completion or response to an internal processor, the order of - * writes has to be such that this field is written last. - */ - uint8_t valid; } __attribute__((packed)); /* hwrm_vnic_alloc */ @@ -4727,239 +7138,74 @@ struct hwrm_stat_ctx_free_output { * enabled, then the internal VNIC to SVIF mapping data structures shall be * programmed at the time of VNIC allocation. */ - -/* Input (24 bytes) */ +/* Input (24 bytes) */ struct hwrm_vnic_alloc_input { - /* - * This value indicates what type of request this is. The format for the - * rest of the command is determined by this field. - */ uint16_t req_type; - /* - * This value indicates the what completion ring the request will be - * optionally completed on. If the value is -1, then no CR completion - * will be generated. Any other value must be a valid CR ring_id value - * for this function. + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. */ uint16_t cmpl_ring; - - /* This value indicates the command sequence number. */ - uint16_t seq_id; - /* - * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids - * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ uint16_t target_id; - /* - * This is the host address where the response will be written when the - * request is complete. This area must be 16B aligned and must be - * cleared to zero before the request is made. + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM */ uint64_t resp_addr; - /* - * When this bit is '1', this VNIC is requested to be the default VNIC - * for this function. + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. */ - #define HWRM_VNIC_ALLOC_INPUT_FLAGS_DEFAULT UINT32_C(0x1) uint32_t flags; - + /* + * When this bit is '1', this VNIC is requested to be the + * default VNIC for this function. + */ + #define HWRM_VNIC_ALLOC_INPUT_FLAGS_DEFAULT UINT32_C(0x1) uint32_t unused_0; } __attribute__((packed)); -/* Output (16 bytes) */ +/* Output (16 bytes) */ struct hwrm_vnic_alloc_output { - /* - * Pass/Fail or error type Note: receiver to verify the in parameters, - * and fail the call with an error when appropriate - */ uint16_t error_code; - - /* This field returns the type of original request. */ - uint16_t req_type; - - /* This field provides original sequence number of the command. */ - uint16_t seq_id; - /* - * This field is the length of the response in bytes. The last byte of - * the response is a valid flag that will read as '1' when the command - * has been completely written to memory. + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ uint16_t resp_len; - - /* Logical vnic ID */ + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ uint32_t vnic_id; - + /* Logical vnic ID */ uint8_t unused_0; uint8_t unused_1; uint8_t unused_2; - - /* - * This field is used in Output records to indicate that the output is - * completely written to RAM. This field should be read as '1' to - * indicate that the output has been completely written. When writing a - * command completion or response to an internal processor, the order of - * writes has to be such that this field is written last. - */ uint8_t valid; -} __attribute__((packed)); - -/* hwrm_vnic_cfg */ -/* Description: Configure the RX VNIC structure. */ - -/* Input (40 bytes) */ -struct hwrm_vnic_cfg_input { /* - * This value indicates what type of request this is. The format for the - * rest of the command is determined by this field. + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. */ - uint16_t req_type; - - /* - * This value indicates the what completion ring the request will be - * optionally completed on. If the value is -1, then no CR completion - * will be generated. Any other value must be a valid CR ring_id value - * for this function. - */ - uint16_t cmpl_ring; - - /* This value indicates the command sequence number. */ - uint16_t seq_id; - - /* - * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids - * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM - */ - uint16_t target_id; - - /* - * This is the host address where the response will be written when the - * request is complete. This area must be 16B aligned and must be - * cleared to zero before the request is made. - */ - uint64_t resp_addr; - - /* - * When this bit is '1', the VNIC is requested to be the default VNIC - * for the function. - */ - #define HWRM_VNIC_CFG_INPUT_FLAGS_DEFAULT UINT32_C(0x1) - /* - * When this bit is '1', the VNIC is being configured to strip VLAN in - * the RX path. If set to '0', then VLAN stripping is disabled on this - * VNIC. - */ - #define HWRM_VNIC_CFG_INPUT_FLAGS_VLAN_STRIP_MODE UINT32_C(0x2) - /* - * When this bit is '1', the VNIC is being configured to buffer receive - * packets in the hardware until the host posts new receive buffers. If - * set to '0', then bd_stall is being configured to be disabled on this - * VNIC. - */ - #define HWRM_VNIC_CFG_INPUT_FLAGS_BD_STALL_MODE UINT32_C(0x4) - /* - * When this bit is '1', the VNIC is being configured to receive both - * RoCE and non-RoCE traffic. If set to '0', then this VNIC is not - * configured to be operating in dual VNIC mode. - */ - #define HWRM_VNIC_CFG_INPUT_FLAGS_ROCE_DUAL_VNIC_MODE UINT32_C(0x8) - /* - * When this flag is set to '1', the VNIC is requested to be configured - * to receive only RoCE traffic. If this flag is set to '0', then this - * flag shall be ignored by the HWRM. If roce_dual_vnic_mode flag is set - * to '1', then the HWRM client shall not set this flag to '1'. - */ - #define HWRM_VNIC_CFG_INPUT_FLAGS_ROCE_ONLY_VNIC_MODE UINT32_C(0x10) - uint32_t flags; - - /* This bit must be '1' for the dflt_ring_grp field to be configured. */ - #define HWRM_VNIC_CFG_INPUT_ENABLES_DFLT_RING_GRP UINT32_C(0x1) - /* This bit must be '1' for the rss_rule field to be configured. */ - #define HWRM_VNIC_CFG_INPUT_ENABLES_RSS_RULE UINT32_C(0x2) - /* This bit must be '1' for the cos_rule field to be configured. */ - #define HWRM_VNIC_CFG_INPUT_ENABLES_COS_RULE UINT32_C(0x4) - /* This bit must be '1' for the lb_rule field to be configured. */ - #define HWRM_VNIC_CFG_INPUT_ENABLES_LB_RULE UINT32_C(0x8) - /* This bit must be '1' for the mru field to be configured. */ - #define HWRM_VNIC_CFG_INPUT_ENABLES_MRU UINT32_C(0x10) - uint32_t enables; - - /* Logical vnic ID */ - uint16_t vnic_id; - - /* - * Default Completion ring for the VNIC. This ring will be chosen if - * packet does not match any RSS rules and if there is no COS rule. - */ - uint16_t dflt_ring_grp; - - /* - * RSS ID for RSS rule/table structure. 0xFF... (All Fs) if there is no - * RSS rule. - */ - uint16_t rss_rule; - - /* - * RSS ID for COS rule/table structure. 0xFF... (All Fs) if there is no - * COS rule. - */ - uint16_t cos_rule; - - /* - * RSS ID for load balancing rule/table structure. 0xFF... (All Fs) if - * there is no LB rule. - */ - uint16_t lb_rule; - - /* - * The maximum receive unit of the vnic. Each vnic is associated with a - * function. The vnic mru value overwrites the mru setting of the - * associated function. The HWRM shall make sure that vnic mru does not - * exceed the mru of the port the function is associated with. - */ - uint16_t mru; - - uint32_t unused_0; -} __attribute__((packed)); - -/* Output (16 bytes) */ -struct hwrm_vnic_cfg_output { - /* - * Pass/Fail or error type Note: receiver to verify the in parameters, - * and fail the call with an error when appropriate - */ - uint16_t error_code; - - /* This field returns the type of original request. */ - uint16_t req_type; - - /* This field provides original sequence number of the command. */ - uint16_t seq_id; - - /* - * This field is the length of the response in bytes. The last byte of - * the response is a valid flag that will read as '1' when the command - * has been completely written to memory. - */ - uint16_t resp_len; - - uint32_t unused_0; - uint8_t unused_1; - uint8_t unused_2; - uint8_t unused_3; - - /* - * This field is used in Output records to indicate that the output is - * completely written to RAM. This field should be read as '1' to - * indicate that the output has been completely written. When writing a - * command completion or response to an internal processor, the order of - * writes has to be such that this field is written last. - */ - uint8_t valid; } __attribute__((packed)); /* hwrm_vnic_free */ @@ -4968,192 +7214,816 @@ struct hwrm_vnic_cfg_output { * VNIC as well as the VNIC. Reset and release all resources associated with the * VNIC. */ - -/* Input (24 bytes) */ +/* Input (24 bytes) */ struct hwrm_vnic_free_input { - /* - * This value indicates what type of request this is. The format for the - * rest of the command is determined by this field. - */ uint16_t req_type; - /* - * This value indicates the what completion ring the request will be - * optionally completed on. If the value is -1, then no CR completion - * will be generated. Any other value must be a valid CR ring_id value - * for this function. + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. */ uint16_t cmpl_ring; - - /* This value indicates the command sequence number. */ - uint16_t seq_id; - /* - * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids - * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ uint16_t target_id; - /* - * This is the host address where the response will be written when the - * request is complete. This area must be 16B aligned and must be - * cleared to zero before the request is made. + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM */ uint64_t resp_addr; - - /* Logical vnic ID */ + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ uint32_t vnic_id; - + /* Logical vnic ID */ uint32_t unused_0; } __attribute__((packed)); -/* Output (16 bytes) */ +/* Output (16 bytes) */ struct hwrm_vnic_free_output { - /* - * Pass/Fail or error type Note: receiver to verify the in parameters, - * and fail the call with an error when appropriate - */ uint16_t error_code; - - /* This field returns the type of original request. */ - uint16_t req_type; - - /* This field provides original sequence number of the command. */ - uint16_t seq_id; - /* - * This field is the length of the response in bytes. The last byte of - * the response is a valid flag that will read as '1' when the command - * has been completely written to memory. + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ uint16_t resp_len; - + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ uint32_t unused_0; uint8_t unused_1; uint8_t unused_2; uint8_t unused_3; - - /* - * This field is used in Output records to indicate that the output is - * completely written to RAM. This field should be read as '1' to - * indicate that the output has been completely written. When writing a - * command completion or response to an internal processor, the order of - * writes has to be such that this field is written last. - */ uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* hwrm_vnic_cfg */ +/* Description: Configure the RX VNIC structure. */ +/* Input (40 bytes) */ +struct hwrm_vnic_cfg_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint32_t flags; + /* + * When this bit is '1', the VNIC is requested to be the default + * VNIC for the function. + */ + #define HWRM_VNIC_CFG_INPUT_FLAGS_DEFAULT UINT32_C(0x1) + /* + * When this bit is '1', the VNIC is being configured to strip + * VLAN in the RX path. If set to '0', then VLAN stripping is + * disabled on this VNIC. + */ + #define HWRM_VNIC_CFG_INPUT_FLAGS_VLAN_STRIP_MODE UINT32_C(0x2) + /* + * When this bit is '1', the VNIC is being configured to buffer + * receive packets in the hardware until the host posts new + * receive buffers. If set to '0', then bd_stall is being + * configured to be disabled on this VNIC. + */ + #define HWRM_VNIC_CFG_INPUT_FLAGS_BD_STALL_MODE UINT32_C(0x4) + /* + * When this bit is '1', the VNIC is being configured to receive + * both RoCE and non-RoCE traffic. If set to '0', then this VNIC + * is not configured to be operating in dual VNIC mode. + */ + #define HWRM_VNIC_CFG_INPUT_FLAGS_ROCE_DUAL_VNIC_MODE UINT32_C(0x8) + /* + * When this flag is set to '1', the VNIC is requested to be + * configured to receive only RoCE traffic. If this flag is set + * to '0', then this flag shall be ignored by the HWRM. If + * roce_dual_vnic_mode flag is set to '1', then the HWRM client + * shall not set this flag to '1'. + */ + #define HWRM_VNIC_CFG_INPUT_FLAGS_ROCE_ONLY_VNIC_MODE UINT32_C(0x10) + /* + * When a VNIC uses one destination ring group for certain + * application (e.g. Receive Flow Steering) where exact match is + * used to direct packets to a VNIC with one destination ring + * group only, there is no need to configure RSS indirection + * table for that VNIC as only one destination ring group is + * used. This flag is used to enable a mode where RSS is enabled + * in the VNIC using a RSS context for computing RSS hash but + * the RSS indirection table is not configured using + * hwrm_vnic_rss_cfg. If this mode is enabled, then the driver + * should not program RSS indirection table for the RSS context + * that is used for computing RSS hash only. + */ + #define HWRM_VNIC_CFG_INPUT_FLAGS_RSS_DFLT_CR_MODE UINT32_C(0x20) + /* + * When this bit is '1', the VNIC is being configured to receive + * both RoCE and non-RoCE traffic, but forward only the RoCE + * traffic further. Also, RoCE traffic can be mirrored to L2 + * driver. + */ + #define HWRM_VNIC_CFG_INPUT_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_MODE \ + UINT32_C(0x40) + uint32_t enables; + /* + * This bit must be '1' for the dflt_ring_grp field to be + * configured. + */ + #define HWRM_VNIC_CFG_INPUT_ENABLES_DFLT_RING_GRP UINT32_C(0x1) + /* This bit must be '1' for the rss_rule field to be configured. */ + #define HWRM_VNIC_CFG_INPUT_ENABLES_RSS_RULE UINT32_C(0x2) + /* This bit must be '1' for the cos_rule field to be configured. */ + #define HWRM_VNIC_CFG_INPUT_ENABLES_COS_RULE UINT32_C(0x4) + /* This bit must be '1' for the lb_rule field to be configured. */ + #define HWRM_VNIC_CFG_INPUT_ENABLES_LB_RULE UINT32_C(0x8) + /* This bit must be '1' for the mru field to be configured. */ + #define HWRM_VNIC_CFG_INPUT_ENABLES_MRU UINT32_C(0x10) + uint16_t vnic_id; + /* Logical vnic ID */ + uint16_t dflt_ring_grp; + /* + * Default Completion ring for the VNIC. This ring will be + * chosen if packet does not match any RSS rules and if there is + * no COS rule. + */ + uint16_t rss_rule; + /* + * RSS ID for RSS rule/table structure. 0xFF... (All Fs) if + * there is no RSS rule. + */ + uint16_t cos_rule; + /* + * RSS ID for COS rule/table structure. 0xFF... (All Fs) if + * there is no COS rule. + */ + uint16_t lb_rule; + /* + * RSS ID for load balancing rule/table structure. 0xFF... (All + * Fs) if there is no LB rule. + */ + uint16_t mru; + /* + * The maximum receive unit of the vnic. Each vnic is associated + * with a function. The vnic mru value overwrites the mru + * setting of the associated function. The HWRM shall make sure + * that vnic mru does not exceed the mru of the port the + * function is associated with. + */ + uint32_t unused_0; +} __attribute__((packed)); + +/* Output (16 bytes) */ +struct hwrm_vnic_cfg_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint32_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t unused_3; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* hwrm_vnic_qcfg */ +/* + * Description: Query the RX VNIC structure. This function can be used by a PF + * driver to query its own VNIC resource or VNIC resource of its child VF. This + * function can also be used by a VF driver to query its own VNIC resource. + */ +/* Input (32 bytes) */ +struct hwrm_vnic_qcfg_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint32_t enables; + /* This bit must be '1' for the vf_id_valid field to be configured. */ + #define HWRM_VNIC_QCFG_INPUT_ENABLES_VF_ID_VALID UINT32_C(0x1) + uint32_t vnic_id; + /* Logical vnic ID */ + uint16_t vf_id; + /* ID of Virtual Function whose VNIC resource is being queried. */ + uint16_t unused_0[3]; +} __attribute__((packed)); + +/* Output (32 bytes) */ +struct hwrm_vnic_qcfg_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint16_t dflt_ring_grp; + /* Default Completion ring for the VNIC. */ + uint16_t rss_rule; + /* + * RSS ID for RSS rule/table structure. 0xFF... (All Fs) if + * there is no RSS rule. + */ + uint16_t cos_rule; + /* + * RSS ID for COS rule/table structure. 0xFF... (All Fs) if + * there is no COS rule. + */ + uint16_t lb_rule; + /* + * RSS ID for load balancing rule/table structure. 0xFF... (All + * Fs) if there is no LB rule. + */ + uint16_t mru; + /* The maximum receive unit of the vnic. */ + uint8_t unused_0; + uint8_t unused_1; + uint32_t flags; + /* + * When this bit is '1', the VNIC is the default VNIC for the + * function. + */ + #define HWRM_VNIC_QCFG_OUTPUT_FLAGS_DEFAULT UINT32_C(0x1) + /* + * When this bit is '1', the VNIC is configured to strip VLAN in + * the RX path. If set to '0', then VLAN stripping is disabled + * on this VNIC. + */ + #define HWRM_VNIC_QCFG_OUTPUT_FLAGS_VLAN_STRIP_MODE UINT32_C(0x2) + /* + * When this bit is '1', the VNIC is configured to buffer + * receive packets in the hardware until the host posts new + * receive buffers. If set to '0', then bd_stall is disabled on + * this VNIC. + */ + #define HWRM_VNIC_QCFG_OUTPUT_FLAGS_BD_STALL_MODE UINT32_C(0x4) + /* + * When this bit is '1', the VNIC is configured to receive both + * RoCE and non-RoCE traffic. If set to '0', then this VNIC is + * not configured to operate in dual VNIC mode. + */ + #define HWRM_VNIC_QCFG_OUTPUT_FLAGS_ROCE_DUAL_VNIC_MODE UINT32_C(0x8) + /* + * When this flag is set to '1', the VNIC is configured to + * receive only RoCE traffic. When this flag is set to '0', the + * VNIC is not configured to receive only RoCE traffic. If + * roce_dual_vnic_mode flag and this flag both are set to '1', + * then it is an invalid configuration of the VNIC. The HWRM + * should not allow that type of mis-configuration by HWRM + * clients. + */ + #define HWRM_VNIC_QCFG_OUTPUT_FLAGS_ROCE_ONLY_VNIC_MODE UINT32_C(0x10) + /* + * When a VNIC uses one destination ring group for certain + * application (e.g. Receive Flow Steering) where exact match is + * used to direct packets to a VNIC with one destination ring + * group only, there is no need to configure RSS indirection + * table for that VNIC as only one destination ring group is + * used. When this bit is set to '1', then the VNIC is enabled + * in a mode where RSS is enabled in the VNIC using a RSS + * context for computing RSS hash but the RSS indirection table + * is not configured. + */ + #define HWRM_VNIC_QCFG_OUTPUT_FLAGS_RSS_DFLT_CR_MODE UINT32_C(0x20) + /* + * When this bit is '1', the VNIC is configured to receive both + * RoCE and non-RoCE traffic, but forward only RoCE traffic + * further. Also RoCE traffic can be mirrored to L2 driver. + */ + #define HWRM_VNIC_QCFG_OUTPUT_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_MODE \ + UINT32_C(0x40) + uint32_t unused_2; + uint8_t unused_3; + uint8_t unused_4; + uint8_t unused_5; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + + +/* hwrm_vnic_tpa_cfg */ +/* Description: This function is used to enable/configure TPA on the VNIC. */ +/* Input (40 bytes) */ +struct hwrm_vnic_tpa_cfg_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint32_t flags; + /* + * When this bit is '1', the VNIC shall be configured to perform + * transparent packet aggregation (TPA) of non-tunneled TCP + * packets. + */ + #define HWRM_VNIC_TPA_CFG_INPUT_FLAGS_TPA UINT32_C(0x1) + /* + * When this bit is '1', the VNIC shall be configured to perform + * transparent packet aggregation (TPA) of tunneled TCP packets. + */ + #define HWRM_VNIC_TPA_CFG_INPUT_FLAGS_ENCAP_TPA UINT32_C(0x2) + /* + * When this bit is '1', the VNIC shall be configured to perform + * transparent packet aggregation (TPA) according to Windows + * Receive Segment Coalescing (RSC) rules. + */ + #define HWRM_VNIC_TPA_CFG_INPUT_FLAGS_RSC_WND_UPDATE UINT32_C(0x4) + /* + * When this bit is '1', the VNIC shall be configured to perform + * transparent packet aggregation (TPA) according to Linux + * Generic Receive Offload (GRO) rules. + */ + #define HWRM_VNIC_TPA_CFG_INPUT_FLAGS_GRO UINT32_C(0x8) + /* + * When this bit is '1', the VNIC shall be configured to perform + * transparent packet aggregation (TPA) for TCP packets with IP + * ECN set to non-zero. + */ + #define HWRM_VNIC_TPA_CFG_INPUT_FLAGS_AGG_WITH_ECN UINT32_C(0x10) + /* + * When this bit is '1', the VNIC shall be configured to perform + * transparent packet aggregation (TPA) for GRE tunneled TCP + * packets only if all packets have the same GRE sequence. + */ + #define HWRM_VNIC_TPA_CFG_INPUT_FLAGS_AGG_WITH_SAME_GRE_SEQ \ + UINT32_C(0x20) + /* + * When this bit is '1' and the GRO mode is enabled, the VNIC + * shall be configured to perform transparent packet aggregation + * (TPA) for TCP/IPv4 packets with consecutively increasing + * IPIDs. In other words, the last packet that is being + * aggregated to an already existing aggregation context shall + * have IPID 1 more than the IPID of the last packet that was + * aggregated in that aggregation context. + */ + #define HWRM_VNIC_TPA_CFG_INPUT_FLAGS_GRO_IPID_CHECK UINT32_C(0x40) + /* + * When this bit is '1' and the GRO mode is enabled, the VNIC + * shall be configured to perform transparent packet aggregation + * (TPA) for TCP packets with the same TTL (IPv4) or Hop limit + * (IPv6) value. + */ + #define HWRM_VNIC_TPA_CFG_INPUT_FLAGS_GRO_TTL_CHECK UINT32_C(0x80) + uint32_t enables; + /* This bit must be '1' for the max_agg_segs field to be configured. */ + #define HWRM_VNIC_TPA_CFG_INPUT_ENABLES_MAX_AGG_SEGS UINT32_C(0x1) + /* This bit must be '1' for the max_aggs field to be configured. */ + #define HWRM_VNIC_TPA_CFG_INPUT_ENABLES_MAX_AGGS UINT32_C(0x2) + /* + * This bit must be '1' for the max_agg_timer field to be + * configured. + */ + #define HWRM_VNIC_TPA_CFG_INPUT_ENABLES_MAX_AGG_TIMER UINT32_C(0x4) + /* This bit must be '1' for the min_agg_len field to be configured. */ + #define HWRM_VNIC_TPA_CFG_INPUT_ENABLES_MIN_AGG_LEN UINT32_C(0x8) + uint16_t vnic_id; + /* Logical vnic ID */ + uint16_t max_agg_segs; + /* + * This is the maximum number of TCP segments that can be + * aggregated (unit is Log2). Max value is 31. + */ + /* 1 segment */ + #define HWRM_VNIC_TPA_CFG_INPUT_MAX_AGG_SEGS_1 UINT32_C(0x0) + /* 2 segments */ + #define HWRM_VNIC_TPA_CFG_INPUT_MAX_AGG_SEGS_2 UINT32_C(0x1) + /* 4 segments */ + #define HWRM_VNIC_TPA_CFG_INPUT_MAX_AGG_SEGS_4 UINT32_C(0x2) + /* 8 segments */ + #define HWRM_VNIC_TPA_CFG_INPUT_MAX_AGG_SEGS_8 UINT32_C(0x3) + /* Any segment size larger than this is not valid */ + #define HWRM_VNIC_TPA_CFG_INPUT_MAX_AGG_SEGS_MAX UINT32_C(0x1f) + uint16_t max_aggs; + /* + * This is the maximum number of aggregations this VNIC is + * allowed (unit is Log2). Max value is 7 + */ + /* 1 aggregation */ + #define HWRM_VNIC_TPA_CFG_INPUT_MAX_AGGS_1 UINT32_C(0x0) + /* 2 aggregations */ + #define HWRM_VNIC_TPA_CFG_INPUT_MAX_AGGS_2 UINT32_C(0x1) + /* 4 aggregations */ + #define HWRM_VNIC_TPA_CFG_INPUT_MAX_AGGS_4 UINT32_C(0x2) + /* 8 aggregations */ + #define HWRM_VNIC_TPA_CFG_INPUT_MAX_AGGS_8 UINT32_C(0x3) + /* 16 aggregations */ + #define HWRM_VNIC_TPA_CFG_INPUT_MAX_AGGS_16 UINT32_C(0x4) + /* Any aggregation size larger than this is not valid */ + #define HWRM_VNIC_TPA_CFG_INPUT_MAX_AGGS_MAX UINT32_C(0x7) + uint8_t unused_0; + uint8_t unused_1; + uint32_t max_agg_timer; + /* + * This is the maximum amount of time allowed for an aggregation + * context to complete after it was initiated. + */ + uint32_t min_agg_len; + /* + * This is the minimum amount of payload length required to + * start an aggregation context. + */ +} __attribute__((packed)); + +/* Output (16 bytes) */ +struct hwrm_vnic_tpa_cfg_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint32_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t unused_3; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ } __attribute__((packed)); /* hwrm_vnic_rss_cfg */ /* Description: This function is used to enable RSS configuration. */ - -/* Input (48 bytes) */ +/* Input (48 bytes) */ struct hwrm_vnic_rss_cfg_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint32_t hash_type; + /* + * When this bit is '1', the RSS hash shall be computed over + * source and destination IPv4 addresses of IPv4 packets. + */ + #define HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_IPV4 UINT32_C(0x1) + /* + * When this bit is '1', the RSS hash shall be computed over + * source/destination IPv4 addresses and source/destination + * ports of TCP/IPv4 packets. + */ + #define HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_TCP_IPV4 UINT32_C(0x2) + /* + * When this bit is '1', the RSS hash shall be computed over + * source/destination IPv4 addresses and source/destination + * ports of UDP/IPv4 packets. + */ + #define HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_UDP_IPV4 UINT32_C(0x4) + /* + * When this bit is '1', the RSS hash shall be computed over + * source and destination IPv4 addresses of IPv6 packets. + */ + #define HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_IPV6 UINT32_C(0x8) + /* + * When this bit is '1', the RSS hash shall be computed over + * source/destination IPv6 addresses and source/destination + * ports of TCP/IPv6 packets. + */ + #define HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_TCP_IPV6 UINT32_C(0x10) + /* + * When this bit is '1', the RSS hash shall be computed over + * source/destination IPv6 addresses and source/destination + * ports of UDP/IPv6 packets. + */ + #define HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_UDP_IPV6 UINT32_C(0x20) + uint32_t unused_0; + uint64_t ring_grp_tbl_addr; + /* This is the address for rss ring group table */ + uint64_t hash_key_tbl_addr; + /* This is the address for rss hash key table */ + uint16_t rss_ctx_idx; + /* Index to the rss indirection table. */ + uint16_t unused_1[3]; +} __attribute__((packed)); + +/* Output (16 bytes) */ +struct hwrm_vnic_rss_cfg_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint32_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t unused_3; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* hwrm_vnic_plcmodes_cfg */ +/* + * Description: This function can be used to set placement mode configuration of + * the VNIC. + */ +/* Input (40 bytes) */ +struct hwrm_vnic_plcmodes_cfg_input { + uint16_t req_type; /* * This value indicates what type of request this is. The format for the * rest of the command is determined by this field. */ - uint16_t req_type; - + uint16_t cmpl_ring; /* * This value indicates the what completion ring the request will be * optionally completed on. If the value is -1, then no CR completion * will be generated. Any other value must be a valid CR ring_id value * for this function. */ - uint16_t cmpl_ring; - - /* This value indicates the command sequence number. */ uint16_t seq_id; - + /* This value indicates the command sequence number. */ + uint16_t target_id; /* * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM */ - uint16_t target_id; - + uint64_t resp_addr; /* * This is the host address where the response will be written when the * request is complete. This area must be 16B aligned and must be * cleared to zero before the request is made. */ - uint64_t resp_addr; - + uint32_t flags; /* - * When this bit is '1', the RSS hash shall be computed over source and - * destination IPv4 addresses of IPv4 packets. + * When this bit is '1', the VNIC shall be configured to use regular + * placement algorithm. By default, the regular placement algorithm + * shall be enabled on the VNIC. */ - #define HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_IPV4 UINT32_C(0x1) + #define HWRM_VNIC_PLCMODES_CFG_INPUT_FLAGS_REGULAR_PLACEMENT \ + UINT32_C(0x1) /* - * When this bit is '1', the RSS hash shall be computed over - * source/destination IPv4 addresses and source/destination ports of - * TCP/IPv4 packets. + * When this bit is '1', the VNIC shall be configured use the jumbo + * placement algorithm. */ - #define HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_TCP_IPV4 UINT32_C(0x2) + #define HWRM_VNIC_PLCMODES_CFG_INPUT_FLAGS_JUMBO_PLACEMENT \ + UINT32_C(0x2) /* - * When this bit is '1', the RSS hash shall be computed over - * source/destination IPv4 addresses and source/destination ports of - * UDP/IPv4 packets. + * When this bit is '1', the VNIC shall be configured to enable Header- + * Data split for IPv4 packets according to the following rules: # If + * the packet is identified as TCP/IPv4, then the packet is split at the + * beginning of the TCP payload. # If the packet is identified as + * UDP/IPv4, then the packet is split at the beginning of UDP payload. # + * If the packet is identified as non-TCP and non-UDP IPv4 packet, then + * the packet is split at the beginning of the upper layer protocol + * header carried in the IPv4 packet. */ - #define HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_UDP_IPV4 UINT32_C(0x4) + #define HWRM_VNIC_PLCMODES_CFG_INPUT_FLAGS_HDS_IPV4 UINT32_C(0x4) /* - * When this bit is '1', the RSS hash shall be computed over source and - * destination IPv4 addresses of IPv6 packets. + * When this bit is '1', the VNIC shall be configured to enable Header- + * Data split for IPv6 packets according to the following rules: # If + * the packet is identified as TCP/IPv6, then the packet is split at the + * beginning of the TCP payload. # If the packet is identified as + * UDP/IPv6, then the packet is split at the beginning of UDP payload. # + * If the packet is identified as non-TCP and non-UDP IPv6 packet, then + * the packet is split at the beginning of the upper layer protocol + * header carried in the IPv6 packet. */ - #define HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_IPV6 UINT32_C(0x8) + #define HWRM_VNIC_PLCMODES_CFG_INPUT_FLAGS_HDS_IPV6 UINT32_C(0x8) /* - * When this bit is '1', the RSS hash shall be computed over - * source/destination IPv6 addresses and source/destination ports of - * TCP/IPv6 packets. + * When this bit is '1', the VNIC shall be configured to enable Header- + * Data split for FCoE packets at the beginning of FC payload. */ - #define HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_TCP_IPV6 UINT32_C(0x10) + #define HWRM_VNIC_PLCMODES_CFG_INPUT_FLAGS_HDS_FCOE UINT32_C(0x10) /* - * When this bit is '1', the RSS hash shall be computed over - * source/destination IPv6 addresses and source/destination ports of - * UDP/IPv6 packets. + * When this bit is '1', the VNIC shall be configured to enable Header- + * Data split for RoCE packets at the beginning of RoCE payload (after + * BTH/GRH headers). */ - #define HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_UDP_IPV6 UINT32_C(0x20) - uint32_t hash_type; - - uint32_t unused_0; - - /* This is the address for rss ring group table */ - uint64_t ring_grp_tbl_addr; - - /* This is the address for rss hash key table */ - uint64_t hash_key_tbl_addr; - - /* Index to the rss indirection table. */ - uint16_t rss_ctx_idx; - - uint16_t unused_1[3]; + #define HWRM_VNIC_PLCMODES_CFG_INPUT_FLAGS_HDS_ROCE UINT32_C(0x20) + uint32_t enables; + /* + * This bit must be '1' for the jumbo_thresh_valid field to be + * configured. + */ + #define HWRM_VNIC_PLCMODES_CFG_INPUT_ENABLES_JUMBO_THRESH_VALID \ + UINT32_C(0x1) + /* + * This bit must be '1' for the hds_offset_valid field to be configured. + */ + #define HWRM_VNIC_PLCMODES_CFG_INPUT_ENABLES_HDS_OFFSET_VALID \ + UINT32_C(0x2) + /* + * This bit must be '1' for the hds_threshold_valid field to be + * configured. + */ + #define HWRM_VNIC_PLCMODES_CFG_INPUT_ENABLES_HDS_THRESHOLD_VALID \ + UINT32_C(0x4) + uint32_t vnic_id; + /* Logical vnic ID */ + uint16_t jumbo_thresh; + /* + * When jumbo placement algorithm is enabled, this value is used to + * determine the threshold for jumbo placement. Packets with length + * larger than this value will be placed according to the jumbo + * placement algorithm. + */ + uint16_t hds_offset; + /* + * This value is used to determine the offset into packet buffer where + * the split data (payload) will be placed according to one of of HDS + * placement algorithm. The lengths of packet buffers provided for split + * data shall be larger than this value. + */ + uint16_t hds_threshold; + /* + * When one of the HDS placement algorithm is enabled, this value is + * used to determine the threshold for HDS placement. Packets with + * length larger than this value will be placed according to the HDS + * placement algorithm. This value shall be in multiple of 4 bytes. + */ + uint16_t unused_0[3]; } __attribute__((packed)); /* Output (16 bytes) */ -struct hwrm_vnic_rss_cfg_output { +struct hwrm_vnic_plcmodes_cfg_output { + uint16_t error_code; /* * Pass/Fail or error type Note: receiver to verify the in parameters, * and fail the call with an error when appropriate */ - uint16_t error_code; - - /* This field returns the type of original request. */ uint16_t req_type; - - /* This field provides original sequence number of the command. */ + /* This field returns the type of original request. */ uint16_t seq_id; - + /* This field provides original sequence number of the command. */ + uint16_t resp_len; /* * This field is the length of the response in bytes. The last byte of * the response is a valid flag that will read as '1' when the command * has been completely written to memory. */ - uint16_t resp_len; - uint32_t unused_0; uint8_t unused_1; uint8_t unused_2; uint8_t unused_3; - + uint8_t valid; /* * This field is used in Output records to indicate that the output is * completely written to RAM. This field should be read as '1' to @@ -5161,73 +8031,127 @@ struct hwrm_vnic_rss_cfg_output { * command completion or response to an internal processor, the order of * writes has to be such that this field is written last. */ - uint8_t valid; } __attribute__((packed)); -/* Input (16 bytes) */ -struct hwrm_vnic_rss_cos_lb_ctx_alloc_input { +/* hwrm_vnic_plcmodes_qcfg */ +/* + * Description: This function can be used to query placement mode configuration + * of the VNIC. + */ +/* Input (24 bytes) */ +struct hwrm_vnic_plcmodes_qcfg_input { + uint16_t req_type; /* * This value indicates what type of request this is. The format for the * rest of the command is determined by this field. */ - uint16_t req_type; - + uint16_t cmpl_ring; /* * This value indicates the what completion ring the request will be * optionally completed on. If the value is -1, then no CR completion * will be generated. Any other value must be a valid CR ring_id value * for this function. */ - uint16_t cmpl_ring; - - /* This value indicates the command sequence number. */ uint16_t seq_id; - + /* This value indicates the command sequence number. */ + uint16_t target_id; /* * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM */ - uint16_t target_id; - + uint64_t resp_addr; /* * This is the host address where the response will be written when the * request is complete. This area must be 16B aligned and must be * cleared to zero before the request is made. */ - uint64_t resp_addr; + uint32_t vnic_id; + /* Logical vnic ID */ + uint32_t unused_0; } __attribute__((packed)); -/* Output (16 bytes) */ - -struct hwrm_vnic_rss_cos_lb_ctx_alloc_output { +/* Output (24 bytes) */ +struct hwrm_vnic_plcmodes_qcfg_output { + uint16_t error_code; /* * Pass/Fail or error type Note: receiver to verify the in parameters, * and fail the call with an error when appropriate */ - uint16_t error_code; - - /* This field returns the type of original request. */ uint16_t req_type; - - /* This field provides original sequence number of the command. */ + /* This field returns the type of original request. */ uint16_t seq_id; - + /* This field provides original sequence number of the command. */ + uint16_t resp_len; /* * This field is the length of the response in bytes. The last byte of * the response is a valid flag that will read as '1' when the command * has been completely written to memory. */ - uint16_t resp_len; - - /* rss_cos_lb_ctx_id is 16 b */ - uint16_t rss_cos_lb_ctx_id; - + uint32_t flags; + /* + * When this bit is '1', the VNIC is configured to use regular placement + * algorithm. + */ + #define HWRM_VNIC_PLCMODES_QCFG_OUTPUT_FLAGS_REGULAR_PLACEMENT \ + UINT32_C(0x1) + /* + * When this bit is '1', the VNIC is configured to use the jumbo + * placement algorithm. + */ + #define HWRM_VNIC_PLCMODES_QCFG_OUTPUT_FLAGS_JUMBO_PLACEMENT \ + UINT32_C(0x2) + /* + * When this bit is '1', the VNIC is configured to enable Header-Data + * split for IPv4 packets. + */ + #define HWRM_VNIC_PLCMODES_QCFG_OUTPUT_FLAGS_HDS_IPV4 UINT32_C(0x4) + /* + * When this bit is '1', the VNIC is configured to enable Header-Data + * split for IPv6 packets. + */ + #define HWRM_VNIC_PLCMODES_QCFG_OUTPUT_FLAGS_HDS_IPV6 UINT32_C(0x8) + /* + * When this bit is '1', the VNIC is configured to enable Header-Data + * split for FCoE packets. + */ + #define HWRM_VNIC_PLCMODES_QCFG_OUTPUT_FLAGS_HDS_FCOE UINT32_C(0x10) + /* + * When this bit is '1', the VNIC is configured to enable Header-Data + * split for RoCE packets. + */ + #define HWRM_VNIC_PLCMODES_QCFG_OUTPUT_FLAGS_HDS_ROCE UINT32_C(0x20) + /* + * When this bit is '1', the VNIC is configured to be the default VNIC + * of the requesting function. + */ + #define HWRM_VNIC_PLCMODES_QCFG_OUTPUT_FLAGS_DFLT_VNIC UINT32_C(0x40) + uint16_t jumbo_thresh; + /* + * When jumbo placement algorithm is enabled, this value is used to + * determine the threshold for jumbo placement. Packets with length + * larger than this value will be placed according to the jumbo + * placement algorithm. + */ + uint16_t hds_offset; + /* + * This value is used to determine the offset into packet buffer where + * the split data (payload) will be placed according to one of of HDS + * placement algorithm. The lengths of packet buffers provided for split + * data shall be larger than this value. + */ + uint16_t hds_threshold; + /* + * When one of the HDS placement algorithm is enabled, this value is + * used to determine the threshold for HDS placement. Packets with + * length larger than this value will be placed according to the HDS + * placement algorithm. This value shall be in multiple of 4 bytes. + */ uint8_t unused_0; uint8_t unused_1; uint8_t unused_2; uint8_t unused_3; uint8_t unused_4; - + uint8_t valid; /* * This field is used in Output records to indicate that the output is * completely written to RAM. This field should be read as '1' to @@ -5235,557 +8159,1467 @@ struct hwrm_vnic_rss_cos_lb_ctx_alloc_output { * command completion or response to an internal processor, the order of * writes has to be such that this field is written last. */ +} __attribute__((packed)); + +/* hwrm_vnic_rss_cos_lb_ctx_alloc */ +/* Description: This function is used to allocate COS/Load Balance context. */ +/* Input (16 bytes) */ +struct hwrm_vnic_rss_cos_lb_ctx_alloc_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ +} __attribute__((packed)); + +/* Output (16 bytes) */ +struct hwrm_vnic_rss_cos_lb_ctx_alloc_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint16_t rss_cos_lb_ctx_id; + /* rss_cos_lb_ctx_id is 16 b */ + uint8_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t unused_3; + uint8_t unused_4; uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ } __attribute__((packed)); /* hwrm_vnic_rss_cos_lb_ctx_free */ /* Description: This function can be used to free COS/Load Balance context. */ -/* Input (24 bytes) */ - +/* Input (24 bytes) */ struct hwrm_vnic_rss_cos_lb_ctx_free_input { - /* - * This value indicates what type of request this is. The format for the - * rest of the command is determined by this field. - */ uint16_t req_type; - /* - * This value indicates the what completion ring the request will be - * optionally completed on. If the value is -1, then no CR completion - * will be generated. Any other value must be a valid CR ring_id value - * for this function. + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. */ uint16_t cmpl_ring; - - /* This value indicates the command sequence number. */ - uint16_t seq_id; - /* - * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids - * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ uint16_t target_id; - /* - * This is the host address where the response will be written when the - * request is complete. This area must be 16B aligned and must be - * cleared to zero before the request is made. + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM */ uint64_t resp_addr; - - /* rss_cos_lb_ctx_id is 16 b */ + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ uint16_t rss_cos_lb_ctx_id; - + /* rss_cos_lb_ctx_id is 16 b */ uint16_t unused_0[3]; } __attribute__((packed)); -/* Output (16 bytes) */ +/* Output (16 bytes) */ struct hwrm_vnic_rss_cos_lb_ctx_free_output { - /* - * Pass/Fail or error type Note: receiver to verify the in parameters, - * and fail the call with an error when appropriate - */ uint16_t error_code; - - /* This field returns the type of original request. */ - uint16_t req_type; - - /* This field provides original sequence number of the command. */ - uint16_t seq_id; - /* - * This field is the length of the response in bytes. The last byte of - * the response is a valid flag that will read as '1' when the command - * has been completely written to memory. + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ uint16_t resp_len; - + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ uint32_t unused_0; uint8_t unused_1; uint8_t unused_2; uint8_t unused_3; - - /* - * This field is used in Output records to indicate that the output is - * completely written to RAM. This field should be read as '1' to - * indicate that the output has been completely written. When writing a - * command completion or response to an internal processor, the order of - * writes has to be such that this field is written last. - */ uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ } __attribute__((packed)); -/* Output (32 bytes) */ -struct hwrm_queue_qportcfg_output { - /* - * Pass/Fail or error type Note: receiver to verify the in parameters, - * and fail the call with an error when appropriate - */ - uint16_t error_code; - - /* This field returns the type of original request. */ - uint16_t req_type; - - /* This field provides original sequence number of the command. */ - uint16_t seq_id; - - /* - * This field is the length of the response in bytes. The last byte of - * the response is a valid flag that will read as '1' when the command - * has been completely written to memory. - */ - uint16_t resp_len; - - /* The maximum number of queues that can be configured. */ - uint8_t max_configurable_queues; - - /* The maximum number of lossless queues that can be configured. */ - uint8_t max_configurable_lossless_queues; - - /* - * 0 - Not allowed. Non-zero - Allowed. If this value is non-zero, then - * the HWRM shall allow the host SW driver to configure queues using - * hwrm_queue_cfg. - */ - uint8_t queue_cfg_allowed; - - /* - * 0 - Not allowed. Non-zero - Allowed If this value is non-zero, then - * the HWRM shall allow the host SW driver to configure queue buffers - * using hwrm_queue_buffers_cfg. - */ - uint8_t queue_buffers_cfg_allowed; - - /* - * 0 - Not allowed. Non-zero - Allowed If this value is non-zero, then - * the HWRM shall allow the host SW driver to configure PFC using - * hwrm_queue_pfcenable_cfg. - */ - uint8_t queue_pfcenable_cfg_allowed; - - /* - * 0 - Not allowed. Non-zero - Allowed If this value is non-zero, then - * the HWRM shall allow the host SW driver to configure Priority to CoS - * mapping using hwrm_queue_pri2cos_cfg. - */ - uint8_t queue_pri2cos_cfg_allowed; - - /* - * 0 - Not allowed. Non-zero - Allowed If this value is non-zero, then - * the HWRM shall allow the host SW driver to configure CoS Bandwidth - * configuration using hwrm_queue_cos2bw_cfg. - */ - uint8_t queue_cos2bw_cfg_allowed; - - /* ID of CoS Queue 0. FF - Invalid id */ - uint8_t queue_id0; - - /* This value is applicable to CoS queues only. */ - /* Lossy (best-effort) */ - #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID0_SERVICE_PROFILE_LOSSY \ - (UINT32_C(0x0) << 0) - /* Lossless */ - #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID0_SERVICE_PROFILE_LOSSLESS \ - (UINT32_C(0x1) << 0) - /* - * Set to 0xFF... (All Fs) if there is no service profile - * specified - */ - #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID0_SERVICE_PROFILE_UNKNOWN \ - (UINT32_C(0xff) << 0) - uint8_t queue_id0_service_profile; - - /* ID of CoS Queue 1. FF - Invalid id */ - uint8_t queue_id1; - /* This value is applicable to CoS queues only. */ - /* Lossy (best-effort) */ - #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID1_SERVICE_PROFILE_LOSSY \ - (UINT32_C(0x0) << 0) - /* Lossless */ - #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID1_SERVICE_PROFILE_LOSSLESS \ - (UINT32_C(0x1) << 0) - /* - * Set to 0xFF... (All Fs) if there is no service profile - * specified - */ - #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID1_SERVICE_PROFILE_UNKNOWN \ - (UINT32_C(0xff) << 0) - uint8_t queue_id1_service_profile; - - /* ID of CoS Queue 2. FF - Invalid id */ - uint8_t queue_id2; - /* This value is applicable to CoS queues only. */ - /* Lossy (best-effort) */ - #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID2_SERVICE_PROFILE_LOSSY \ - (UINT32_C(0x0) << 0) - /* Lossless */ - #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID2_SERVICE_PROFILE_LOSSLESS \ - (UINT32_C(0x1) << 0) - /* - * Set to 0xFF... (All Fs) if there is no service profile - * specified - */ - #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID2_SERVICE_PROFILE_UNKNOWN \ - (UINT32_C(0xff) << 0) - uint8_t queue_id2_service_profile; - - /* ID of CoS Queue 3. FF - Invalid id */ - uint8_t queue_id3; - - /* This value is applicable to CoS queues only. */ - /* Lossy (best-effort) */ - #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID3_SERVICE_PROFILE_LOSSY \ - (UINT32_C(0x0) << 0) - /* Lossless */ - #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID3_SERVICE_PROFILE_LOSSLESS \ - (UINT32_C(0x1) << 0) - /* - * Set to 0xFF... (All Fs) if there is no service profile - * specified - */ - #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID3_SERVICE_PROFILE_UNKNOWN \ - (UINT32_C(0xff) << 0) - uint8_t queue_id3_service_profile; - - /* ID of CoS Queue 4. FF - Invalid id */ - uint8_t queue_id4; - /* This value is applicable to CoS queues only. */ - /* Lossy (best-effort) */ - #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID4_SERVICE_PROFILE_LOSSY \ - (UINT32_C(0x0) << 0) - /* Lossless */ - #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID4_SERVICE_PROFILE_LOSSLESS \ - (UINT32_C(0x1) << 0) - /* - * Set to 0xFF... (All Fs) if there is no service profile - * specified - */ - #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID4_SERVICE_PROFILE_UNKNOWN \ - (UINT32_C(0xff) << 0) - uint8_t queue_id4_service_profile; - - /* ID of CoS Queue 5. FF - Invalid id */ - uint8_t queue_id5; - - /* This value is applicable to CoS queues only. */ - /* Lossy (best-effort) */ - #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID5_SERVICE_PROFILE_LOSSY \ - (UINT32_C(0x0) << 0) - /* Lossless */ - #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID5_SERVICE_PROFILE_LOSSLESS \ - (UINT32_C(0x1) << 0) - /* - * Set to 0xFF... (All Fs) if there is no service profile - * specified - */ - #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID5_SERVICE_PROFILE_UNKNOWN \ - (UINT32_C(0xff) << 0) - uint8_t queue_id5_service_profile; - - /* ID of CoS Queue 6. FF - Invalid id */ - uint8_t queue_id6_service_profile; - /* This value is applicable to CoS queues only. */ - /* Lossy (best-effort) */ - #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID6_SERVICE_PROFILE_LOSSY \ - (UINT32_C(0x0) << 0) - /* Lossless */ - #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID6_SERVICE_PROFILE_LOSSLESS \ - (UINT32_C(0x1) << 0) - /* - * Set to 0xFF... (All Fs) if there is no service profile - * specified - */ - #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID6_SERVICE_PROFILE_UNKNOWN \ - (UINT32_C(0xff) << 0) - uint8_t queue_id6; - - /* ID of CoS Queue 7. FF - Invalid id */ - uint8_t queue_id7; - - /* This value is applicable to CoS queues only. */ - /* Lossy (best-effort) */ - #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID7_SERVICE_PROFILE_LOSSY \ - (UINT32_C(0x0) << 0) - /* Lossless */ - #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID7_SERVICE_PROFILE_LOSSLESS \ - (UINT32_C(0x1) << 0) - /* - * Set to 0xFF... (All Fs) if there is no service profile - * specified - */ - #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID7_SERVICE_PROFILE_UNKNOWN \ - (UINT32_C(0xff) << 0) - uint8_t queue_id7_service_profile; - - /* - * This field is used in Output records to indicate that the output is - * completely written to RAM. This field should be read as '1' to - * indicate that the output has been completely written. When writing a - * command completion or response to an internal processor, the order of - * writes has to be such that this field is written last. - */ - uint8_t valid; -} __attribute__((packed)); - -/* hwrm_func_drv_rgtr */ +/* hwrm_ring_alloc */ /* - * Description: This command is used by the function driver to register its - * information with the HWRM. A function driver shall implement this command. A - * function driver shall use this command during the driver initialization right - * after the HWRM version discovery and default ring resources allocation. + * Description: This command allocates and does basic preparation for a ring. */ - -/* Input (80 bytes) */ -struct hwrm_func_drv_rgtr_input { - /* - * This value indicates what type of request this is. The format for the - * rest of the command is determined by this field. - */ +/* Input (80 bytes) */ +struct hwrm_ring_alloc_input { uint16_t req_type; - /* - * This value indicates the what completion ring the request will be - * optionally completed on. If the value is -1, then no CR completion - * will be generated. Any other value must be a valid CR ring_id value - * for this function. + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. */ uint16_t cmpl_ring; - - /* This value indicates the command sequence number. */ - uint16_t seq_id; - /* - * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids - * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ uint16_t target_id; - /* - * This is the host address where the response will be written when the - * request is complete. This area must be 16B aligned and must be - * cleared to zero before the request is made. + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM */ uint64_t resp_addr; - /* - * When this bit is '1', the function driver is requesting all requests - * from its children VF drivers to be forwarded to itself. This flag can - * only be set by the PF driver. If a VF driver sets this flag, it - * should be ignored by the HWRM. + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. */ - #define HWRM_FUNC_DRV_RGTR_INPUT_FLAGS_FWD_ALL_MODE UINT32_C(0x1) - /* - * When this bit is '1', the function is requesting none of the requests - * from its children VF drivers to be forwarded to itself. This flag can - * only be set by the PF driver. If a VF driver sets this flag, it - * should be ignored by the HWRM. - */ - #define HWRM_FUNC_DRV_RGTR_INPUT_FLAGS_FWD_NONE_MODE UINT32_C(0x2) - uint32_t flags; - - /* This bit must be '1' for the os_type field to be configured. */ - #define HWRM_FUNC_DRV_RGTR_INPUT_ENABLES_OS_TYPE UINT32_C(0x1) - /* This bit must be '1' for the ver field to be configured. */ - #define HWRM_FUNC_DRV_RGTR_INPUT_ENABLES_VER UINT32_C(0x2) - /* This bit must be '1' for the timestamp field to be configured. */ - #define HWRM_FUNC_DRV_RGTR_INPUT_ENABLES_TIMESTAMP UINT32_C(0x4) - /* This bit must be '1' for the vf_req_fwd field to be configured. */ - #define HWRM_FUNC_DRV_RGTR_INPUT_ENABLES_VF_REQ_FWD UINT32_C(0x8) - /* - * This bit must be '1' for the async_event_fwd field to be configured. - */ - #define HWRM_FUNC_DRV_RGTR_INPUT_ENABLES_ASYNC_EVENT_FWD \ - UINT32_C(0x10) uint32_t enables; - - /* This value indicates the type of OS. */ - /* Unknown */ - #define HWRM_FUNC_DRV_RGTR_INPUT_OS_TYPE_UNKNOWN \ - (UINT32_C(0x0) << 0) - /* Other OS not listed below. */ - #define HWRM_FUNC_DRV_RGTR_INPUT_OS_TYPE_OTHER \ - (UINT32_C(0x1) << 0) - /* MSDOS OS. */ - #define HWRM_FUNC_DRV_RGTR_INPUT_OS_TYPE_MSDOS \ - (UINT32_C(0xe) << 0) - /* Windows OS. */ - #define HWRM_FUNC_DRV_RGTR_INPUT_OS_TYPE_WINDOWS \ - (UINT32_C(0x12) << 0) - /* Solaris OS. */ - #define HWRM_FUNC_DRV_RGTR_INPUT_OS_TYPE_SOLARIS \ - (UINT32_C(0x1d) << 0) - /* Linux OS. */ - #define HWRM_FUNC_DRV_RGTR_INPUT_OS_TYPE_LINUX \ - (UINT32_C(0x24) << 0) - /* FreeBSD OS. */ - #define HWRM_FUNC_DRV_RGTR_INPUT_OS_TYPE_FREEBSD \ - (UINT32_C(0x2a) << 0) - /* VMware ESXi OS. */ - #define HWRM_FUNC_DRV_RGTR_INPUT_OS_TYPE_ESXI \ - (UINT32_C(0x68) << 0) - /* Microsoft Windows 8 64-bit OS. */ - #define HWRM_FUNC_DRV_RGTR_INPUT_OS_TYPE_WIN864 \ - (UINT32_C(0x73) << 0) - /* Microsoft Windows Server 2012 R2 OS. */ - #define HWRM_FUNC_DRV_RGTR_INPUT_OS_TYPE_WIN2012R2 \ - (UINT32_C(0x74) << 0) - uint16_t os_type; - - /* This is the major version of the driver. */ - uint8_t ver_maj; - - /* This is the minor version of the driver. */ - uint8_t ver_min; - - /* This is the update version of the driver. */ - uint8_t ver_upd; - + /* This bit must be '1' for the Reserved1 field to be configured. */ + #define HWRM_RING_ALLOC_INPUT_ENABLES_RESERVED1 UINT32_C(0x1) + /* This bit must be '1' for the ring_arb_cfg field to be configured. */ + #define HWRM_RING_ALLOC_INPUT_ENABLES_RING_ARB_CFG UINT32_C(0x2) + /* This bit must be '1' for the Reserved3 field to be configured. */ + #define HWRM_RING_ALLOC_INPUT_ENABLES_RESERVED3 UINT32_C(0x4) + /* + * This bit must be '1' for the stat_ctx_id_valid field to be + * configured. + */ + #define HWRM_RING_ALLOC_INPUT_ENABLES_STAT_CTX_ID_VALID UINT32_C(0x8) + /* This bit must be '1' for the Reserved4 field to be configured. */ + #define HWRM_RING_ALLOC_INPUT_ENABLES_RESERVED4 UINT32_C(0x10) + /* This bit must be '1' for the max_bw_valid field to be configured. */ + #define HWRM_RING_ALLOC_INPUT_ENABLES_MAX_BW_VALID UINT32_C(0x20) + uint8_t ring_type; + /* Ring Type. */ + /* L2 Completion Ring (CR) */ + #define HWRM_RING_ALLOC_INPUT_RING_TYPE_L2_CMPL UINT32_C(0x0) + /* TX Ring (TR) */ + #define HWRM_RING_ALLOC_INPUT_RING_TYPE_TX UINT32_C(0x1) + /* RX Ring (RR) */ + #define HWRM_RING_ALLOC_INPUT_RING_TYPE_RX UINT32_C(0x2) + /* RoCE Notification Completion Ring (ROCE_CR) */ + #define HWRM_RING_ALLOC_INPUT_RING_TYPE_ROCE_CMPL UINT32_C(0x3) uint8_t unused_0; uint16_t unused_1; - + uint64_t page_tbl_addr; + /* This value is a pointer to the page table for the Ring. */ + uint32_t fbo; + /* First Byte Offset of the first entry in the first page. */ + uint8_t page_size; /* - * This is a 32-bit timestamp provided by the driver for keep alive. The - * timestamp is in multiples of 1ms. + * Actual page size in 2^page_size. The supported range is + * increments in powers of 2 from 16 bytes to 1GB. - 4 = 16 B + * Page size is 16 B. - 12 = 4 KB Page size is 4 KB. - 13 = 8 KB + * Page size is 8 KB. - 16 = 64 KB Page size is 64 KB. - 21 = 2 + * MB Page size is 2 MB. - 22 = 4 MB Page size is 4 MB. - 30 = 1 + * GB Page size is 1 GB. */ - uint32_t timestamp; - - uint32_t unused_2; - + uint8_t page_tbl_depth; /* - * This is a 256-bit bit mask provided by the PF driver for letting the - * HWRM know what commands issued by the VF driver to the HWRM should be - * forwarded to the PF driver. Nth bit refers to the Nth req_type. - * Setting Nth bit to 1 indicates that requests from the VF driver with - * req_type equal to N shall be forwarded to the parent PF driver. This - * field is not valid for the VF driver. + * This value indicates the depth of page table. For this + * version of the specification, value other than 0 or 1 shall + * be considered as an invalid value. When the page_tbl_depth = + * 0, then it is treated as a special case with the following. + * 1. FBO and page size fields are not valid. 2. page_tbl_addr + * is the physical address of the first element of the ring. */ - uint32_t vf_req_fwd[8]; - + uint8_t unused_2; + uint8_t unused_3; + uint32_t length; /* - * This is a 256-bit bit mask provided by the function driver (PF or VF - * driver) to indicate the list of asynchronous event completions to be - * forwarded. Nth bit refers to the Nth event_id. Setting Nth bit to 1 - * by the function driver shall result in the HWRM forwarding - * asynchronous event completion with event_id equal to N. If all bits - * are set to 0 (value of 0), then the HWRM shall not forward any - * asynchronous event completion to this function driver. + * Number of 16B units in the ring. Minimum size for a ring is + * 16 16B entries. */ - uint32_t async_event_fwd[8]; + uint16_t logical_id; + /* + * Logical ring number for the ring to be allocated. This value + * determines the position in the doorbell area where the update + * to the ring will be made. For completion rings, this value is + * also the MSI-X vector number for the function the completion + * ring is associated with. + */ + uint16_t cmpl_ring_id; + /* + * This field is used only when ring_type is a TX ring. This + * value indicates what completion ring the TX ring is + * associated with. + */ + uint16_t queue_id; + /* + * This field is used only when ring_type is a TX ring. This + * value indicates what CoS queue the TX ring is associated + * with. + */ + uint8_t unused_4; + uint8_t unused_5; + uint32_t reserved1; + /* This field is reserved for the future use. It shall be set to 0. */ + uint16_t ring_arb_cfg; + /* + * This field is used only when ring_type is a TX ring. This + * field is used to configure arbitration related parameters for + * a TX ring. + */ + /* Arbitration policy used for the ring. */ + #define HWRM_RING_ALLOC_INPUT_RING_ARB_CFG_ARB_POLICY_MASK UINT32_C(0xf) + #define HWRM_RING_ALLOC_INPUT_RING_ARB_CFG_ARB_POLICY_SFT 0 + /* + * Use strict priority for the TX ring. Priority + * value is specified in arb_policy_param + */ + #define HWRM_RING_ALLOC_INPUT_RING_ARB_CFG_ARB_POLICY_SP \ + (UINT32_C(0x1) << 0) + /* + * Use weighted fair queue arbitration for the + * TX ring. Weight is specified in + * arb_policy_param + */ + #define HWRM_RING_ALLOC_INPUT_RING_ARB_CFG_ARB_POLICY_WFQ \ + (UINT32_C(0x2) << 0) + #define HWRM_RING_ALLOC_INPUT_RING_ARB_CFG_ARB_POLICY_LAST \ + RING_ALLOC_INPUT_RING_ARB_CFG_ARB_POLICY_WFQ + /* Reserved field. */ + #define HWRM_RING_ALLOC_INPUT_RING_ARB_CFG_RSVD_MASK UINT32_C(0xf0) + #define HWRM_RING_ALLOC_INPUT_RING_ARB_CFG_RSVD_SFT 4 + /* + * Arbitration policy specific parameter. # For strict priority + * arbitration policy, this field represents a priority value. + * If set to 0, then the priority is not specified and the HWRM + * is allowed to select any priority for this TX ring. # For + * weighted fair queue arbitration policy, this field represents + * a weight value. If set to 0, then the weight is not specified + * and the HWRM is allowed to select any weight for this TX + * ring. + */ + #define HWRM_RING_ALLOC_INPUT_RING_ARB_CFG_ARB_POLICY_PARAM_MASK \ + UINT32_C(0xff00) + #define HWRM_RING_ALLOC_INPUT_RING_ARB_CFG_ARB_POLICY_PARAM_SFT 8 + uint8_t unused_6; + uint8_t unused_7; + uint32_t reserved3; + /* This field is reserved for the future use. It shall be set to 0. */ + uint32_t stat_ctx_id; + /* + * This field is used only when ring_type is a TX ring. This + * input indicates what statistics context this ring should be + * associated with. + */ + uint32_t reserved4; + /* This field is reserved for the future use. It shall be set to 0. */ + uint32_t max_bw; + /* + * This field is used only when ring_type is a TX ring to + * specify maximum BW allocated to the TX ring. The HWRM will + * translate this value into byte counter and time interval used + * for this ring inside the device. + */ + /* The bandwidth value. */ + #define HWRM_RING_ALLOC_INPUT_MAX_BW_BW_VALUE_MASK UINT32_C(0xfffffff) + #define HWRM_RING_ALLOC_INPUT_MAX_BW_BW_VALUE_SFT 0 + /* The granularity of the value (bits or bytes). */ + #define HWRM_RING_ALLOC_INPUT_MAX_BW_SCALE UINT32_C(0x10000000) + /* Value is in bits. */ + #define HWRM_RING_ALLOC_INPUT_MAX_BW_SCALE_BITS (UINT32_C(0x0) << 28) + /* Value is in bytes. */ + #define HWRM_RING_ALLOC_INPUT_MAX_BW_SCALE_BYTES (UINT32_C(0x1) << 28) + #define HWRM_RING_ALLOC_INPUT_MAX_BW_SCALE_LAST \ + RING_ALLOC_INPUT_MAX_BW_SCALE_BYTES + /* bw_value_unit is 3 b */ + #define HWRM_RING_ALLOC_INPUT_MAX_BW_BW_VALUE_UNIT_MASK \ + UINT32_C(0xe0000000) + #define HWRM_RING_ALLOC_INPUT_MAX_BW_BW_VALUE_UNIT_SFT 29 + /* Value is in Mb or MB (base 10). */ + #define HWRM_RING_ALLOC_INPUT_MAX_BW_BW_VALUE_UNIT_MEGA \ + (UINT32_C(0x0) << 29) + /* Value is in Kb or KB (base 10). */ + #define HWRM_RING_ALLOC_INPUT_MAX_BW_BW_VALUE_UNIT_KILO \ + (UINT32_C(0x2) << 29) + /* Value is in bits or bytes. */ + #define HWRM_RING_ALLOC_INPUT_MAX_BW_BW_VALUE_UNIT_BASE \ + (UINT32_C(0x4) << 29) + /* Value is in Gb or GB (base 10). */ + #define HWRM_RING_ALLOC_INPUT_MAX_BW_BW_VALUE_UNIT_GIGA \ + (UINT32_C(0x6) << 29) + /* Value is in 1/100th of a percentage of total bandwidth. */ + #define HWRM_RING_ALLOC_INPUT_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 \ + (UINT32_C(0x1) << 29) + /* Invalid unit */ + #define HWRM_RING_ALLOC_INPUT_MAX_BW_BW_VALUE_UNIT_INVALID \ + (UINT32_C(0x7) << 29) + #define HWRM_RING_ALLOC_INPUT_MAX_BW_BW_VALUE_UNIT_LAST \ + RING_ALLOC_INPUT_MAX_BW_BW_VALUE_UNIT_INVALID + uint8_t int_mode; + /* + * This field is used only when ring_type is a Completion ring. + * This value indicates what interrupt mode should be used on + * this completion ring. Note: In the legacy interrupt mode, no + * more than 16 completion rings are allowed. + */ + /* Legacy INTA */ + #define HWRM_RING_ALLOC_INPUT_INT_MODE_LEGACY UINT32_C(0x0) + /* Reserved */ + #define HWRM_RING_ALLOC_INPUT_INT_MODE_RSVD UINT32_C(0x1) + /* MSI-X */ + #define HWRM_RING_ALLOC_INPUT_INT_MODE_MSIX UINT32_C(0x2) + /* No Interrupt - Polled mode */ + #define HWRM_RING_ALLOC_INPUT_INT_MODE_POLL UINT32_C(0x3) + uint8_t unused_8[3]; } __attribute__((packed)); -/* Output (16 bytes) */ - -struct hwrm_func_drv_rgtr_output { - /* - * Pass/Fail or error type Note: receiver to verify the in parameters, - * and fail the call with an error when appropriate - */ +/* Output (16 bytes) */ +struct hwrm_ring_alloc_output { uint16_t error_code; - - /* This field returns the type of original request. */ - uint16_t req_type; - - /* This field provides original sequence number of the command. */ - uint16_t seq_id; - /* - * This field is the length of the response in bytes. The last byte of - * the response is a valid flag that will read as '1' when the command - * has been completely written to memory. + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint16_t ring_id; + /* + * Physical number of ring allocated. This value shall be unique + * for a ring type. + */ + uint16_t logical_ring_id; + /* Logical number of ring allocated. */ + uint8_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); +/* hwrm_ring_free */ +/* + * Description: This command is used to free a ring and associated resources. + * With QoS and DCBx agents, it is possible the traffic classes will be moved + * from one CoS queue to another. When this occurs, the driver shall call + * 'hwrm_ring_free' to free the allocated rings and then call 'hwrm_ring_alloc' + * to re-allocate each ring and assign it to a new CoS queue. hwrm_ring_free + * shall be called on a ring only after it has been idle for 500ms or more and + * no frames have been posted to the ring during this time. All frames queued + * for transmission shall be completed and at least 500ms time elapsed from the + * last completion before calling this command. + */ +/* Input (24 bytes) */ +struct hwrm_ring_free_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint8_t ring_type; + /* Ring Type. */ + /* L2 Completion Ring (CR) */ + #define HWRM_RING_FREE_INPUT_RING_TYPE_L2_CMPL UINT32_C(0x0) + /* TX Ring (TR) */ + #define HWRM_RING_FREE_INPUT_RING_TYPE_TX UINT32_C(0x1) + /* RX Ring (RR) */ + #define HWRM_RING_FREE_INPUT_RING_TYPE_RX UINT32_C(0x2) + /* RoCE Notification Completion Ring (ROCE_CR) */ + #define HWRM_RING_FREE_INPUT_RING_TYPE_ROCE_CMPL UINT32_C(0x3) + uint8_t unused_0; + uint16_t ring_id; + /* Physical number of ring allocated. */ + uint32_t unused_1; +} __attribute__((packed)); + +/* Output (16 bytes) */ +struct hwrm_ring_free_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ uint32_t unused_0; uint8_t unused_1; uint8_t unused_2; uint8_t unused_3; - - /* - * This field is used in Output records to indicate that the output is - * completely written to RAM. This field should be read as '1' to - * indicate that the output has been completely written. When writing a - * command completion or response to an internal processor, the order of - * writes has to be such that this field is written last. - */ uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ } __attribute__((packed)); -/* hwrm_func_drv_unrgtr */ +/* hwrm_ring_grp_alloc */ /* - * Description: This command is used by the function driver to un register with - * the HWRM. A function driver shall implement this command. A function driver - * shall use this command during the driver unloading. + * Description: This API allocates and does basic preparation for a ring group. */ -/* Input (24 bytes) */ +/* Input (24 bytes) */ +struct hwrm_ring_grp_alloc_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint16_t cr; + /* This value identifies the CR associated with the ring group. */ + uint16_t rr; + /* This value identifies the main RR associated with the ring group. */ + uint16_t ar; + /* + * This value identifies the aggregation RR associated with the + * ring group. If this value is 0xFF... (All Fs), then no + * Aggregation ring will be set. + */ + uint16_t sc; + /* + * This value identifies the statistics context associated with + * the ring group. + */ +} __attribute__((packed)); -struct hwrm_func_drv_unrgtr_input { +/* Output (16 bytes) */ +struct hwrm_ring_grp_alloc_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint32_t ring_group_id; + /* + * This is the ring group ID value. Use this value to program + * the default ring group for the VNIC or as table entries in an + * RSS/COS context. + */ + uint8_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* hwrm_ring_grp_free */ +/* + * Description: This API frees a ring group and associated resources. # If a + * ring in the ring group is reset or free, then the associated rings in the + * ring group shall also be reset/free using hwrm_ring_free. # A function driver + * shall always use hwrm_ring_grp_free after freeing all rings in a group. # As + * a part of executing this command, the HWRM shall reset all associated ring + * group resources. + */ +/* Input (24 bytes) */ +struct hwrm_ring_grp_free_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint32_t ring_group_id; + /* This is the ring group ID value. */ + uint32_t unused_0; +} __attribute__((packed)); + +/* Output (16 bytes) */ +struct hwrm_ring_grp_free_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint32_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t unused_3; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* hwrm_cfa_l2_filter_alloc */ +/* + * Description: An L2 filter is a filter resource that is used to identify a + * vnic or ring for a packet based on layer 2 fields. Layer 2 fields for + * encapsulated packets include both outer L2 header and/or inner l2 header of + * encapsulated packet. The L2 filter resource covers the following OS specific + * L2 filters. Linux/FreeBSD (per function): # Broadcast enable/disable # List + * of individual multicast filters # All multicast enable/disable filter # + * Unicast filters # Promiscuous mode VMware: # Broadcast enable/disable (per + * physical function) # All multicast enable/disable (per function) # Unicast + * filters per ring or vnic # Promiscuous mode per PF Windows: # Broadcast + * enable/disable (per physical function) # List of individual multicast filters + * (Driver needs to advertise the maximum number of filters supported) # All + * multicast enable/disable per physical function # Unicast filters per vnic # + * Promiscuous mode per PF Implementation notes on the use of VNIC in this + * command: # By default, these filters belong to default vnic for the function. + * # Once these filters are set up, only destination VNIC can be modified. # If + * the destination VNIC is not specified in this command, then the HWRM shall + * only create an l2 context id. HWRM Implementation notes for multicast + * filters: # The hwrm_filter_alloc command can be used to set up multicast + * filters (perfect match or partial match). Each individual function driver can + * set up multicast filters independently. # The HWRM needs to keep track of + * multicast filters set up by function drivers and maintain multicast group + * replication records to enable a subset of functions to receive traffic for a + * specific multicast address. # When a specific multicast filter cannot be set, + * the HWRM shall return an error. In this error case, the driver should fall + * back to using one general filter (rather than specific) for all multicast + * traffic. # When the SR-IOV is enabled, the HWRM needs to additionally track + * source knockout per multicast group record. Examples of setting unicast + * filters: For a unicast MAC based filter, one can use a combination of the + * fields and masks provided in this command to set up the filter. Below are + * some examples: # MAC + no VLAN filter: This filter is used to identify + * traffic that does not contain any VLAN tags and matches destination (or + * source) MAC address. This filter can be set up by setting only l2_addr field + * to be a valid field. All other fields are not valid. The following value is + * set for l2_addr. l2_addr = MAC # MAC + Any VLAN filter: This filter is used + * to identify traffic that carries single VLAN tag and matches (destination or + * source) MAC address. This filter can be set up by setting only l2_addr and + * l2_ovlan_mask fields to be valid fields. All other fields are not valid. The + * following values are set for those two valid fields. l2_addr = MAC, + * l2_ovlan_mask = 0xFFFF # MAC + no VLAN or VLAN ID=0: This filter is used to + * identify untagged traffic that does not contain any VLAN tags or a VLAN tag + * with VLAN ID = 0 and matches destination (or source) MAC address. This filter + * can be set up by setting only l2_addr and l2_ovlan fields to be valid fields. + * All other fields are not valid. The following value are set for l2_addr and + * l2_ovlan. l2_addr = MAC, l2_ovlan = 0x0 # MAC + no VLAN or any VLAN: This + * filter is used to identify traffic that contains zero or 1 VLAN tag and + * matches destination (or source) MAC address. This filter can be set up by + * setting only l2_addr, l2_ovlan, and l2_mask fields to be valid fields. All + * other fields are not valid. The following value are set for l2_addr, + * l2_ovlan, and l2_mask fields. l2_addr = MAC, l2_ovlan = 0x0, l2_ovlan_mask = + * 0xFFFF # MAC + VLAN ID filter: This filter can be set up by setting only + * l2_addr, l2_ovlan, and l2_ovlan_mask fields to be valid fields. All other + * fields are not valid. The following values are set for those three valid + * fields. l2_addr = MAC, l2_ovlan = VLAN ID, l2_ovlan_mask = 0xF000 + */ +/* Input (96 bytes) */ +struct hwrm_cfa_l2_filter_alloc_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint32_t flags; + /* + * Enumeration denoting the RX, TX type of the resource. This + * enumeration is used for resources that are similar for both + * TX and RX paths of the chip. + */ + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_PATH UINT32_C(0x1) + /* tx path */ + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_PATH_TX \ + (UINT32_C(0x0) << 0) + /* rx path */ + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_PATH_RX \ + (UINT32_C(0x1) << 0) + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_PATH_LAST \ + CFA_L2_FILTER_ALLOC_INPUT_FLAGS_PATH_RX + /* + * Setting of this flag indicates the applicability to the + * loopback path. + */ + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_LOOPBACK UINT32_C(0x2) + /* + * Setting of this flag indicates drop action. If this flag is + * not set, then it should be considered accept action. + */ + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_DROP UINT32_C(0x4) + /* + * If this flag is set, all t_l2_* fields are invalid and they + * should not be specified. If this flag is set, then l2_* + * fields refer to fields of outermost L2 header. + */ + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_OUTERMOST UINT32_C(0x8) + uint32_t enables; + /* This bit must be '1' for the l2_addr field to be configured. */ + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_ADDR UINT32_C(0x1) + /* This bit must be '1' for the l2_addr_mask field to be configured. */ + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_ADDR_MASK \ + UINT32_C(0x2) + /* This bit must be '1' for the l2_ovlan field to be configured. */ + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_OVLAN UINT32_C(0x4) + /* + * This bit must be '1' for the l2_ovlan_mask field to be + * configured. + */ + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_OVLAN_MASK \ + UINT32_C(0x8) + /* This bit must be '1' for the l2_ivlan field to be configured. */ + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_IVLAN UINT32_C(0x10) + /* + * This bit must be '1' for the l2_ivlan_mask field to be + * configured. + */ + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_IVLAN_MASK \ + UINT32_C(0x20) + /* This bit must be '1' for the t_l2_addr field to be configured. */ + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_T_L2_ADDR UINT32_C(0x40) + /* + * This bit must be '1' for the t_l2_addr_mask field to be + * configured. + */ + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_T_L2_ADDR_MASK \ + UINT32_C(0x80) + /* This bit must be '1' for the t_l2_ovlan field to be configured. */ + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_T_L2_OVLAN \ + UINT32_C(0x100) + /* + * This bit must be '1' for the t_l2_ovlan_mask field to be + * configured. + */ + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_T_L2_OVLAN_MASK \ + UINT32_C(0x200) + /* This bit must be '1' for the t_l2_ivlan field to be configured. */ + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_T_L2_IVLAN \ + UINT32_C(0x400) + /* + * This bit must be '1' for the t_l2_ivlan_mask field to be + * configured. + */ + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_T_L2_IVLAN_MASK \ + UINT32_C(0x800) + /* This bit must be '1' for the src_type field to be configured. */ + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_SRC_TYPE UINT32_C(0x1000) + /* This bit must be '1' for the src_id field to be configured. */ + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_SRC_ID UINT32_C(0x2000) + /* This bit must be '1' for the tunnel_type field to be configured. */ + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_TUNNEL_TYPE \ + UINT32_C(0x4000) + /* This bit must be '1' for the dst_id field to be configured. */ + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_DST_ID UINT32_C(0x8000) + /* + * This bit must be '1' for the mirror_vnic_id field to be + * configured. + */ + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_MIRROR_VNIC_ID \ + UINT32_C(0x10000) + uint8_t l2_addr[6]; + /* + * This value sets the match value for the L2 MAC address. + * Destination MAC address for RX path. Source MAC address for + * TX path. + */ + uint8_t unused_0; + uint8_t unused_1; + uint8_t l2_addr_mask[6]; + /* + * This value sets the mask value for the L2 address. A value of + * 0 will mask the corresponding bit from compare. + */ + uint16_t l2_ovlan; + /* This value sets VLAN ID value for outer VLAN. */ + uint16_t l2_ovlan_mask; + /* + * This value sets the mask value for the ovlan id. A value of 0 + * will mask the corresponding bit from compare. + */ + uint16_t l2_ivlan; + /* This value sets VLAN ID value for inner VLAN. */ + uint16_t l2_ivlan_mask; + /* + * This value sets the mask value for the ivlan id. A value of 0 + * will mask the corresponding bit from compare. + */ + uint8_t unused_2; + uint8_t unused_3; + uint8_t t_l2_addr[6]; + /* + * This value sets the match value for the tunnel L2 MAC + * address. Destination MAC address for RX path. Source MAC + * address for TX path. + */ + uint8_t unused_4; + uint8_t unused_5; + uint8_t t_l2_addr_mask[6]; + /* + * This value sets the mask value for the tunnel L2 address. A + * value of 0 will mask the corresponding bit from compare. + */ + uint16_t t_l2_ovlan; + /* This value sets VLAN ID value for tunnel outer VLAN. */ + uint16_t t_l2_ovlan_mask; + /* + * This value sets the mask value for the tunnel ovlan id. A + * value of 0 will mask the corresponding bit from compare. + */ + uint16_t t_l2_ivlan; + /* This value sets VLAN ID value for tunnel inner VLAN. */ + uint16_t t_l2_ivlan_mask; + /* + * This value sets the mask value for the tunnel ivlan id. A + * value of 0 will mask the corresponding bit from compare. + */ + uint8_t src_type; + /* This value identifies the type of source of the packet. */ + /* Network port */ + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_SRC_TYPE_NPORT UINT32_C(0x0) + /* Physical function */ + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_SRC_TYPE_PF UINT32_C(0x1) + /* Virtual function */ + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_SRC_TYPE_VF UINT32_C(0x2) + /* Virtual NIC of a function */ + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_SRC_TYPE_VNIC UINT32_C(0x3) + /* Embedded processor for CFA management */ + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_SRC_TYPE_KONG UINT32_C(0x4) + /* Embedded processor for OOB management */ + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_SRC_TYPE_APE UINT32_C(0x5) + /* Embedded processor for RoCE */ + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_SRC_TYPE_BONO UINT32_C(0x6) + /* Embedded processor for network proxy functions */ + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_SRC_TYPE_TANG UINT32_C(0x7) + uint8_t unused_6; + uint32_t src_id; + /* + * This value is the id of the source. For a network port, it + * represents port_id. For a physical function, it represents + * fid. For a virtual function, it represents vf_id. For a vnic, + * it represents vnic_id. For embedded processors, this id is + * not valid. Notes: 1. The function ID is implied if it src_id + * is not provided for a src_type that is either + */ + uint8_t tunnel_type; + /* Tunnel Type. */ + /* Non-tunnel */ + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_TUNNEL_TYPE_NONTUNNEL \ + UINT32_C(0x0) + /* Virtual eXtensible Local Area Network (VXLAN) */ + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_TUNNEL_TYPE_VXLAN \ + UINT32_C(0x1) + /* + * Network Virtualization Generic Routing + * Encapsulation (NVGRE) + */ + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_TUNNEL_TYPE_NVGRE \ + UINT32_C(0x2) + /* + * Generic Routing Encapsulation (GRE) inside + * Ethernet payload + */ + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_TUNNEL_TYPE_L2GRE UINT32_C(0x3) + /* IP in IP */ + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_TUNNEL_TYPE_IPIP UINT32_C(0x4) + /* Generic Network Virtualization Encapsulation (Geneve) */ + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_TUNNEL_TYPE_GENEVE UINT32_C(0x5) + /* Multi-Protocol Lable Switching (MPLS) */ + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_TUNNEL_TYPE_MPLS UINT32_C(0x6) + /* Stateless Transport Tunnel (STT) */ + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_TUNNEL_TYPE_STT UINT32_C(0x7) + /* + * Generic Routing Encapsulation (GRE) inside IP + * datagram payload + */ + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_TUNNEL_TYPE_IPGRE UINT32_C(0x8) + /* + * IPV4 over virtual eXtensible Local Area + * Network (IPV4oVXLAN) + */ + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_TUNNEL_TYPE_VXLAN_V4 \ + UINT32_C(0x9) + /* Any tunneled traffic */ + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_TUNNEL_TYPE_ANYTUNNEL \ + UINT32_C(0xff) + uint8_t unused_7; + uint16_t dst_id; + /* + * If set, this value shall represent the Logical VNIC ID of the + * destination VNIC for the RX path and network port id of the + * destination port for the TX path. + */ + uint16_t mirror_vnic_id; + /* Logical VNIC ID of the VNIC where traffic is mirrored. */ + uint8_t pri_hint; + /* + * This hint is provided to help in placing the filter in the + * filter table. + */ + /* No preference */ + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_PRI_HINT_NO_PREFER \ + UINT32_C(0x0) + /* Above the given filter */ + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_PRI_HINT_ABOVE_FILTER \ + UINT32_C(0x1) + /* Below the given filter */ + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_PRI_HINT_BELOW_FILTER \ + UINT32_C(0x2) + /* As high as possible */ + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_PRI_HINT_MAX UINT32_C(0x3) + /* As low as possible */ + #define HWRM_CFA_L2_FILTER_ALLOC_INPUT_PRI_HINT_MIN UINT32_C(0x4) + uint8_t unused_8; + uint32_t unused_9; + uint64_t l2_filter_id_hint; + /* + * This is the ID of the filter that goes along with the + * pri_hint. This field is valid only for the following values. + * 1 - Above the given filter 2 - Below the given filter + */ +} __attribute__((packed)); + +/* Output (24 bytes) */ +struct hwrm_cfa_l2_filter_alloc_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint64_t l2_filter_id; + /* + * This value identifies a set of CFA data structures used for + * an L2 context. + */ + uint32_t flow_id; + /* + * This is the ID of the flow associated with this filter. This + * value shall be used to match and associate the flow + * identifier returned in completion records. A value of + * 0xFFFFFFFF shall indicate no flow id. + */ + uint8_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* hwrm_cfa_l2_filter_free */ +/* + * Description: Free a L2 filter. The HWRM shall free all associated filter + * resources with the L2 filter. + */ +/* Input (24 bytes) */ +struct hwrm_cfa_l2_filter_free_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint64_t l2_filter_id; + /* + * This value identifies a set of CFA data structures used for + * an L2 context. + */ +} __attribute__((packed)); + +/* Output (16 bytes) */ +struct hwrm_cfa_l2_filter_free_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint32_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t unused_3; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* hwrm_cfa_l2_filter_cfg */ +/* Description: Change the configuration of an existing L2 filter */ +/* Input (40 bytes) */ +struct hwrm_cfa_l2_filter_cfg_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint32_t flags; + /* + * Enumeration denoting the RX, TX type of the resource. This + * enumeration is used for resources that are similar for both + * TX and RX paths of the chip. + */ + #define HWRM_CFA_L2_FILTER_CFG_INPUT_FLAGS_PATH UINT32_C(0x1) + /* tx path */ + #define HWRM_CFA_L2_FILTER_CFG_INPUT_FLAGS_PATH_TX \ + (UINT32_C(0x0) << 0) + /* rx path */ + #define HWRM_CFA_L2_FILTER_CFG_INPUT_FLAGS_PATH_RX \ + (UINT32_C(0x1) << 0) + #define HWRM_CFA_L2_FILTER_CFG_INPUT_FLAGS_PATH_LAST \ + CFA_L2_FILTER_CFG_INPUT_FLAGS_PATH_RX + /* + * Setting of this flag indicates drop action. If this flag is + * not set, then it should be considered accept action. + */ + #define HWRM_CFA_L2_FILTER_CFG_INPUT_FLAGS_DROP UINT32_C(0x2) + uint32_t enables; + /* This bit must be '1' for the dst_id field to be configured. */ + #define HWRM_CFA_L2_FILTER_CFG_INPUT_ENABLES_DST_ID UINT32_C(0x1) + /* + * This bit must be '1' for the new_mirror_vnic_id field to be + * configured. + */ + #define HWRM_CFA_L2_FILTER_CFG_INPUT_ENABLES_NEW_MIRROR_VNIC_ID \ + UINT32_C(0x2) + uint64_t l2_filter_id; + /* + * This value identifies a set of CFA data structures used for + * an L2 context. + */ + uint32_t dst_id; + /* + * If set, this value shall represent the Logical VNIC ID of the + * destination VNIC for the RX path and network port id of the + * destination port for the TX path. + */ + uint32_t new_mirror_vnic_id; + /* New Logical VNIC ID of the VNIC where traffic is mirrored. */ +} __attribute__((packed)); + +/* Output (16 bytes) */ +struct hwrm_cfa_l2_filter_cfg_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint32_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t unused_3; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* hwrm_cfa_l2_set_rx_mask */ +/* Description: This command will set rx mask of the function. */ +/* Input (56 bytes) */ +struct hwrm_cfa_l2_set_rx_mask_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint32_t vnic_id; + /* VNIC ID */ + uint32_t mask; + /* Reserved for future use. */ + #define HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_RESERVED UINT32_C(0x1) + /* + * When this bit is '1', the function is requested to accept + * multi-cast packets specified by the multicast addr table. + */ + #define HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_MCAST UINT32_C(0x2) + /* + * When this bit is '1', the function is requested to accept all + * multi-cast packets. + */ + #define HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_ALL_MCAST UINT32_C(0x4) + /* + * When this bit is '1', the function is requested to accept + * broadcast packets. + */ + #define HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_BCAST UINT32_C(0x8) + /* + * When this bit is '1', the function is requested to be put in + * the promiscuous mode. The HWRM should accept any function to + * set up promiscuous mode. The HWRM shall follow the semantics + * below for the promiscuous mode support. # When partitioning + * is not enabled on a port (i.e. single PF on the port), then + * the PF shall be allowed to be in the promiscuous mode. When + * the PF is in the promiscuous mode, then it shall receive all + * host bound traffic on that port. # When partitioning is + * enabled on a port (i.e. multiple PFs per port) and a PF on + * that port is in the promiscuous mode, then the PF receives + * all traffic within that partition as identified by a unique + * identifier for the PF (e.g. S-Tag). If a unique outer VLAN + * for the PF is specified, then the setting of promiscuous mode + * on that PF shall result in the PF receiving all host bound + * traffic with matching outer VLAN. # A VF shall can be set in + * the promiscuous mode. In the promiscuous mode, the VF does + * not receive any traffic unless a unique outer VLAN for the VF + * is specified. If a unique outer VLAN for the VF is specified, + * then the setting of promiscuous mode on that VF shall result + * in the VF receiving all host bound traffic with the matching + * outer VLAN. # The HWRM shall allow the setting of promiscuous + * mode on a function independently from the promiscuous mode + * settings on other functions. + */ + #define HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_PROMISCUOUS UINT32_C(0x10) + /* + * If this flag is set, the corresponding RX filters shall be + * set up to cover multicast/broadcast filters for the outermost + * Layer 2 destination MAC address field. + */ + #define HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_OUTERMOST UINT32_C(0x20) + /* + * If this flag is set, the corresponding RX filters shall be + * set up to cover multicast/broadcast filters for the VLAN- + * tagged packets that match the TPID and VID fields of VLAN + * tags in the VLAN tag table specified in this command. + */ + #define HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_VLANONLY UINT32_C(0x40) + /* + * If this flag is set, the corresponding RX filters shall be + * set up to cover multicast/broadcast filters for non-VLAN + * tagged packets and VLAN-tagged packets that match the TPID + * and VID fields of VLAN tags in the VLAN tag table specified + * in this command. + */ + #define HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_VLAN_NONVLAN UINT32_C(0x80) + /* + * If this flag is set, the corresponding RX filters shall be + * set up to cover multicast/broadcast filters for non-VLAN + * tagged packets and VLAN-tagged packets matching any VLAN tag. + * If this flag is set, then the HWRM shall ignore VLAN tags + * specified in vlan_tag_tbl. If none of vlanonly, vlan_nonvlan, + * and anyvlan_nonvlan flags is set, then the HWRM shall ignore + * VLAN tags specified in vlan_tag_tbl. The HWRM client shall + * set at most one flag out of vlanonly, vlan_nonvlan, and + * anyvlan_nonvlan. + */ + #define HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_ANYVLAN_NONVLAN \ + UINT32_C(0x100) + uint64_t mc_tbl_addr; + /* This is the address for mcast address tbl. */ + uint32_t num_mc_entries; + /* + * This value indicates how many entries in mc_tbl are valid. + * Each entry is 6 bytes. + */ + uint32_t unused_0; + uint64_t vlan_tag_tbl_addr; + /* + * This is the address for VLAN tag table. Each VLAN entry in + * the table is 4 bytes of a VLAN tag including TPID, PCP, DEI, + * and VID fields in network byte order. + */ + uint32_t num_vlan_tags; + /* + * This value indicates how many entries in vlan_tag_tbl are + * valid. Each entry is 4 bytes. + */ + uint32_t unused_1; +} __attribute__((packed)); + +/* Output (16 bytes) */ +struct hwrm_cfa_l2_set_rx_mask_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint32_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t unused_3; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* Command specific Error Codes (8 bytes) */ +struct hwrm_cfa_l2_set_rx_mask_cmd_err { + uint8_t code; + /* + * command specific error codes that goes to the cmd_err field + * in Common HWRM Error Response. + */ + /* Unknown error */ + #define HWRM_CFA_L2_SET_RX_MASK_CMD_ERR_CODE_UNKNOWN UINT32_C(0x0) + /* + * Unable to complete operation due to conflict + * with Ntuple Filter + */ + #define \ + HWRM_CFA_L2_SET_RX_MASK_CMD_ERR_CODE_NTUPLE_FILTER_CONFLICT_ERR \ + UINT32_C(0x1) + uint8_t unused_0[7]; +} __attribute__((packed)); + +/* hwrm_cfa_vlan_antispoof_cfg */ +/* Description: Configures vlan anti-spoof filters for VF. */ +/* Input (32 bytes) */ +struct hwrm_cfa_vlan_antispoof_cfg_input { + uint16_t req_type; /* * This value indicates what type of request this is. The format for the * rest of the command is determined by this field. */ - uint16_t req_type; - + uint16_t cmpl_ring; /* * This value indicates the what completion ring the request will be * optionally completed on. If the value is -1, then no CR completion * will be generated. Any other value must be a valid CR ring_id value * for this function. */ - uint16_t cmpl_ring; - - /* This value indicates the command sequence number. */ uint16_t seq_id; - + /* This value indicates the command sequence number. */ + uint16_t target_id; /* * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM */ - uint16_t target_id; - + uint64_t resp_addr; /* * This is the host address where the response will be written when the * request is complete. This area must be 16B aligned and must be * cleared to zero before the request is made. */ - uint64_t resp_addr; - + uint16_t fid; /* - * When this bit is '1', the function driver is notifying the HWRM to - * prepare for the shutdown. + * Function ID of the function that is being configured. Only valid for + * a VF FID configured by the PF. */ - #define HWRM_FUNC_DRV_UNRGTR_INPUT_FLAGS_PREPARE_FOR_SHUTDOWN \ - UINT32_C(0x1) - uint32_t flags; - - uint32_t unused_0; -} __attribute__((packed)); + uint8_t unused_0; + uint8_t unused_1; + uint32_t num_vlan_entries; + /* Number of VLAN entries in the vlan_tag_mask_tbl. */ + uint64_t vlan_tag_mask_tbl_addr; + /* + * The vlan_tag_mask_tbl_addr is the DMA address of the VLAN antispoof + * table. Each table entry contains the 16-bit TPID (0x8100 or 0x88a8 + * only), 16-bit VLAN ID, and a 16-bit mask, all in network order to + * match hwrm_cfa_l2_set_rx_mask. For an individual VLAN entry, the mask + * value should be 0xfff for the 12-bit VLAN ID. + */ +}; /* Output (16 bytes) */ -struct hwrm_func_drv_unrgtr_output { +struct hwrm_cfa_vlan_antispoof_cfg_output { + uint16_t error_code; /* * Pass/Fail or error type Note: receiver to verify the in parameters, * and fail the call with an error when appropriate */ - uint16_t error_code; - - /* This field returns the type of original request. */ uint16_t req_type; - - /* This field provides original sequence number of the command. */ + /* This field returns the type of original request. */ uint16_t seq_id; - + /* This field provides original sequence number of the command. */ + uint16_t resp_len; /* * This field is the length of the response in bytes. The last byte of * the response is a valid flag that will read as '1' when the command * has been completely written to memory. */ - uint16_t resp_len; - uint32_t unused_0; uint8_t unused_1; uint8_t unused_2; uint8_t unused_3; - + uint8_t valid; /* * This field is used in Output records to indicate that the output is * completely written to RAM. This field should be read as '1' to @@ -5793,7 +9627,2668 @@ struct hwrm_func_drv_unrgtr_output { * command completion or response to an internal processor, the order of * writes has to be such that this field is written last. */ +}; + +/* hwrm_cfa_ntuple_filter_alloc */ +/* + * Description: This is a ntuple filter that uses fields from L4/L3 header and + * optionally fields from L2. The ntuple filters apply to receive traffic only. + * All L2/L3/L4 header fields are specified in network byte order. These filters + * can be used for Receive Flow Steering (RFS). # For ethertype value, only + * 0x0800 (IPv4) and 0x86dd (IPv6) shall be supported for ntuple filters. # If a + * field specified in this command is not enabled as a valid field, then that + * field shall not be used in matching packet header fields against this filter. + */ +/* Input (128 bytes) */ +struct hwrm_cfa_ntuple_filter_alloc_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint32_t flags; + /* + * Setting of this flag indicates the applicability to the + * loopback path. + */ + #define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_FLAGS_LOOPBACK \ + UINT32_C(0x1) + /* + * Setting of this flag indicates drop action. If this flag is + * not set, then it should be considered accept action. + */ + #define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_FLAGS_DROP UINT32_C(0x2) + /* + * Setting of this flag indicates that a meter is expected to be + * attached to this flow. This hint can be used when choosing + * the action record format required for the flow. + */ + #define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_FLAGS_METER UINT32_C(0x4) + uint32_t enables; + /* This bit must be '1' for the l2_filter_id field to be configured. */ + #define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_L2_FILTER_ID \ + UINT32_C(0x1) + /* This bit must be '1' for the ethertype field to be configured. */ + #define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_ETHERTYPE \ + UINT32_C(0x2) + /* This bit must be '1' for the tunnel_type field to be configured. */ + #define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_TUNNEL_TYPE \ + UINT32_C(0x4) + /* This bit must be '1' for the src_macaddr field to be configured. */ + #define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_SRC_MACADDR \ + UINT32_C(0x8) + /* This bit must be '1' for the ipaddr_type field to be configured. */ + #define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_IPADDR_TYPE \ + UINT32_C(0x10) + /* This bit must be '1' for the src_ipaddr field to be configured. */ + #define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_SRC_IPADDR \ + UINT32_C(0x20) + /* + * This bit must be '1' for the src_ipaddr_mask field to be + * configured. + */ + #define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_SRC_IPADDR_MASK \ + UINT32_C(0x40) + /* This bit must be '1' for the dst_ipaddr field to be configured. */ + #define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_DST_IPADDR \ + UINT32_C(0x80) + /* + * This bit must be '1' for the dst_ipaddr_mask field to be + * configured. + */ + #define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_DST_IPADDR_MASK \ + UINT32_C(0x100) + /* This bit must be '1' for the ip_protocol field to be configured. */ + #define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_IP_PROTOCOL \ + UINT32_C(0x200) + /* This bit must be '1' for the src_port field to be configured. */ + #define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_SRC_PORT \ + UINT32_C(0x400) + /* + * This bit must be '1' for the src_port_mask field to be + * configured. + */ + #define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_SRC_PORT_MASK \ + UINT32_C(0x800) + /* This bit must be '1' for the dst_port field to be configured. */ + #define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_DST_PORT \ + UINT32_C(0x1000) + /* + * This bit must be '1' for the dst_port_mask field to be + * configured. + */ + #define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_DST_PORT_MASK \ + UINT32_C(0x2000) + /* This bit must be '1' for the pri_hint field to be configured. */ + #define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_PRI_HINT \ + UINT32_C(0x4000) + /* + * This bit must be '1' for the ntuple_filter_id field to be + * configured. + */ + #define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_NTUPLE_FILTER_ID \ + UINT32_C(0x8000) + /* This bit must be '1' for the dst_id field to be configured. */ + #define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_DST_ID \ + UINT32_C(0x10000) + /* + * This bit must be '1' for the mirror_vnic_id field to be + * configured. + */ + #define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_MIRROR_VNIC_ID \ + UINT32_C(0x20000) + /* This bit must be '1' for the dst_macaddr field to be configured. */ + #define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_DST_MACADDR \ + UINT32_C(0x40000) + uint64_t l2_filter_id; + /* + * This value identifies a set of CFA data structures used for + * an L2 context. + */ + uint8_t src_macaddr[6]; + /* + * This value indicates the source MAC address in the Ethernet + * header. + */ + uint16_t ethertype; + /* This value indicates the ethertype in the Ethernet header. */ + uint8_t ip_addr_type; + /* + * This value indicates the type of IP address. 4 - IPv4 6 - + * IPv6 All others are invalid. + */ + /* invalid */ + #define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_IP_ADDR_TYPE_UNKNOWN \ + UINT32_C(0x0) + /* IPv4 */ + #define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_IP_ADDR_TYPE_IPV4 \ + UINT32_C(0x4) + /* IPv6 */ + #define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_IP_ADDR_TYPE_IPV6 \ + UINT32_C(0x6) + uint8_t ip_protocol; + /* + * The value of protocol filed in IP header. Applies to UDP and + * TCP traffic. 6 - TCP 17 - UDP + */ + /* invalid */ + #define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_IP_PROTOCOL_UNKNOWN \ + UINT32_C(0x0) + /* TCP */ + #define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_IP_PROTOCOL_TCP \ + UINT32_C(0x6) + /* UDP */ + #define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_IP_PROTOCOL_UDP \ + UINT32_C(0x11) + uint16_t dst_id; + /* + * If set, this value shall represent the Logical VNIC ID of the + * destination VNIC for the RX path and network port id of the + * destination port for the TX path. + */ + uint16_t mirror_vnic_id; + /* Logical VNIC ID of the VNIC where traffic is mirrored. */ + uint8_t tunnel_type; + /* + * This value indicates the tunnel type for this filter. If this + * field is not specified, then the filter shall apply to both + * non-tunneled and tunneled packets. If this field conflicts + * with the tunnel_type specified in the l2_filter_id, then the + * HWRM shall return an error for this command. + */ + /* Non-tunnel */ + #define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_TUNNEL_TYPE_NONTUNNEL \ + UINT32_C(0x0) + /* Virtual eXtensible Local Area Network (VXLAN) */ + #define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_TUNNEL_TYPE_VXLAN \ + UINT32_C(0x1) + /* + * Network Virtualization Generic Routing + * Encapsulation (NVGRE) + */ + #define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_TUNNEL_TYPE_NVGRE \ + UINT32_C(0x2) + /* + * Generic Routing Encapsulation (GRE) inside + * Ethernet payload + */ + #define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_TUNNEL_TYPE_L2GRE \ + UINT32_C(0x3) + /* IP in IP */ + #define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_TUNNEL_TYPE_IPIP \ + UINT32_C(0x4) + /* Generic Network Virtualization Encapsulation (Geneve) */ + #define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_TUNNEL_TYPE_GENEVE \ + UINT32_C(0x5) + /* Multi-Protocol Lable Switching (MPLS) */ + #define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_TUNNEL_TYPE_MPLS \ + UINT32_C(0x6) + /* Stateless Transport Tunnel (STT) */ + #define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_TUNNEL_TYPE_STT UINT32_C(0x7) + /* + * Generic Routing Encapsulation (GRE) inside IP + * datagram payload + */ + #define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_TUNNEL_TYPE_IPGRE \ + UINT32_C(0x8) + /* Any tunneled traffic */ + #define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_TUNNEL_TYPE_ANYTUNNEL \ + UINT32_C(0xff) + uint8_t pri_hint; + /* + * This hint is provided to help in placing the filter in the + * filter table. + */ + /* No preference */ + #define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_PRI_HINT_NO_PREFER \ + UINT32_C(0x0) + /* Above the given filter */ + #define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_PRI_HINT_ABOVE UINT32_C(0x1) + /* Below the given filter */ + #define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_PRI_HINT_BELOW UINT32_C(0x2) + /* As high as possible */ + #define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_PRI_HINT_HIGHEST \ + UINT32_C(0x3) + /* As low as possible */ + #define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_PRI_HINT_LOWEST UINT32_C(0x4) + uint32_t src_ipaddr[4]; + /* + * The value of source IP address to be used in filtering. For + * IPv4, first four bytes represent the IP address. + */ + uint32_t src_ipaddr_mask[4]; + /* + * The value of source IP address mask to be used in filtering. + * For IPv4, first four bytes represent the IP address mask. + */ + uint32_t dst_ipaddr[4]; + /* + * The value of destination IP address to be used in filtering. + * For IPv4, first four bytes represent the IP address. + */ + uint32_t dst_ipaddr_mask[4]; + /* + * The value of destination IP address mask to be used in + * filtering. For IPv4, first four bytes represent the IP + * address mask. + */ + uint16_t src_port; + /* + * The value of source port to be used in filtering. Applies to + * UDP and TCP traffic. + */ + uint16_t src_port_mask; + /* + * The value of source port mask to be used in filtering. + * Applies to UDP and TCP traffic. + */ + uint16_t dst_port; + /* + * The value of destination port to be used in filtering. + * Applies to UDP and TCP traffic. + */ + uint16_t dst_port_mask; + /* + * The value of destination port mask to be used in filtering. + * Applies to UDP and TCP traffic. + */ + uint64_t ntuple_filter_id_hint; + /* This is the ID of the filter that goes along with the pri_hint. */ +} __attribute__((packed)); + +/* Output (24 bytes) */ +struct hwrm_cfa_ntuple_filter_alloc_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint64_t ntuple_filter_id; + /* This value is an opaque id into CFA data structures. */ + uint32_t flow_id; + /* + * This is the ID of the flow associated with this filter. This + * value shall be used to match and associate the flow + * identifier returned in completion records. A value of + * 0xFFFFFFFF shall indicate no flow id. + */ + uint8_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* Command specific Error Codes (8 bytes) */ +struct hwrm_cfa_ntuple_filter_alloc_cmd_err { + uint8_t code; + /* + * command specific error codes that goes to the cmd_err field + * in Common HWRM Error Response. + */ + /* Unknown error */ + #define HWRM_CFA_NTUPLE_FILTER_ALLOC_CMD_ERR_CODE_UNKNOWN UINT32_C(0x0) + /* + * Unable to complete operation due to conflict + * with Rx Mask VLAN + */ + #define \ + HWRM_CFA_NTUPLE_FILTER_ALLOC_CMD_ERR_CODE_RX_MASK_VLAN_CONFLICT_ERR \ + UINT32_C(0x1) + uint8_t unused_0[7]; +} __attribute__((packed)); + +/* hwrm_cfa_ntuple_filter_free */ +/* Description: Free an ntuple filter */ +/* Input (24 bytes) */ +struct hwrm_cfa_ntuple_filter_free_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint64_t ntuple_filter_id; + /* This value is an opaque id into CFA data structures. */ +} __attribute__((packed)); + +/* Output (16 bytes) */ +struct hwrm_cfa_ntuple_filter_free_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint32_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t unused_3; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* hwrm_cfa_ntuple_filter_cfg */ +/* + * Description: Configure an ntuple filter with a new destination VNIC and/or + * meter. + */ +/* Input (48 bytes) */ +struct hwrm_cfa_ntuple_filter_cfg_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint32_t enables; + /* This bit must be '1' for the new_dst_id field to be configured. */ + #define HWRM_CFA_NTUPLE_FILTER_CFG_INPUT_ENABLES_NEW_DST_ID \ + UINT32_C(0x1) + /* + * This bit must be '1' for the new_mirror_vnic_id field to be + * configured. + */ + #define HWRM_CFA_NTUPLE_FILTER_CFG_INPUT_ENABLES_NEW_MIRROR_VNIC_ID \ + UINT32_C(0x2) + /* + * This bit must be '1' for the new_meter_instance_id field to + * be configured. + */ + #define HWRM_CFA_NTUPLE_FILTER_CFG_INPUT_ENABLES_NEW_METER_INSTANCE_ID \ + UINT32_C(0x4) + uint32_t unused_0; + uint64_t ntuple_filter_id; + /* This value is an opaque id into CFA data structures. */ + uint32_t new_dst_id; + /* + * If set, this value shall represent the new Logical VNIC ID of + * the destination VNIC for the RX path and new network port id + * of the destination port for the TX path. + */ + uint32_t new_mirror_vnic_id; + /* New Logical VNIC ID of the VNIC where traffic is mirrored. */ + uint16_t new_meter_instance_id; + /* + * New meter to attach to the flow. Specifying the invalid + * instance ID is used to remove any existing meter from the + * flow. + */ + /* + * A value of 0xfff is considered invalid and + * implies the instance is not configured. + */ + #define HWRM_CFA_NTUPLE_FILTER_CFG_INPUT_NEW_METER_INSTANCE_ID_INVALID \ + UINT32_C(0xffff) + uint16_t unused_1[3]; +} __attribute__((packed)); + +/* Output (16 bytes) */ +struct hwrm_cfa_ntuple_filter_cfg_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint32_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t unused_3; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* hwrm_cfa_em_flow_alloc */ +/* + * Description: This is a generic Exact Match (EM) flow that uses fields from + * L4/L3/L2 headers. The EM flows apply to transmit and receive traffic. All + * L2/L3/L4 header fields are specified in network byte order. For each EM flow, + * there is an associated set of actions specified. For tunneled packets, all + * L2/L3/L4 fields specified are fields of inner headers unless otherwise + * specified. # If a field specified in this command is not enabled as a valid + * field, then that field shall not be used in matching packet header fields + * against this EM flow entry. + */ +/* Input (112 bytes) */ +struct hwrm_cfa_em_flow_alloc_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint32_t flags; + /* + * Enumeration denoting the RX, TX type of the resource. This + * enumeration is used for resources that are similar for both + * TX and RX paths of the chip. + */ + #define HWRM_CFA_EM_FLOW_ALLOC_INPUT_FLAGS_PATH UINT32_C(0x1) + /* tx path */ + #define HWRM_CFA_EM_FLOW_ALLOC_INPUT_FLAGS_PATH_TX \ + (UINT32_C(0x0) << 0) + /* rx path */ + #define HWRM_CFA_EM_FLOW_ALLOC_INPUT_FLAGS_PATH_RX \ + (UINT32_C(0x1) << 0) + #define HWRM_CFA_EM_FLOW_ALLOC_INPUT_FLAGS_PATH_LAST \ + CFA_EM_FLOW_ALLOC_INPUT_FLAGS_PATH_RX + /* + * Setting of this flag indicates enabling of a byte counter for + * a given flow. + */ + #define HWRM_CFA_EM_FLOW_ALLOC_INPUT_FLAGS_BYTE_CTR UINT32_C(0x2) + /* + * Setting of this flag indicates enabling of a packet counter + * for a given flow. + */ + #define HWRM_CFA_EM_FLOW_ALLOC_INPUT_FLAGS_PKT_CTR UINT32_C(0x4) + /* + * Setting of this flag indicates de-capsulation action for the + * given flow. + */ + #define HWRM_CFA_EM_FLOW_ALLOC_INPUT_FLAGS_DECAP UINT32_C(0x8) + /* + * Setting of this flag indicates encapsulation action for the + * given flow. + */ + #define HWRM_CFA_EM_FLOW_ALLOC_INPUT_FLAGS_ENCAP UINT32_C(0x10) + /* + * Setting of this flag indicates drop action. If this flag is + * not set, then it should be considered accept action. + */ + #define HWRM_CFA_EM_FLOW_ALLOC_INPUT_FLAGS_DROP UINT32_C(0x20) + /* + * Setting of this flag indicates that a meter is expected to be + * attached to this flow. This hint can be used when choosing + * the action record format required for the flow. + */ + #define HWRM_CFA_EM_FLOW_ALLOC_INPUT_FLAGS_METER UINT32_C(0x40) + uint32_t enables; + /* This bit must be '1' for the l2_filter_id field to be configured. */ + #define HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_L2_FILTER_ID UINT32_C(0x1) + /* This bit must be '1' for the tunnel_type field to be configured. */ + #define HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_TUNNEL_TYPE UINT32_C(0x2) + /* This bit must be '1' for the tunnel_id field to be configured. */ + #define HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_TUNNEL_ID UINT32_C(0x4) + /* This bit must be '1' for the src_macaddr field to be configured. */ + #define HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_SRC_MACADDR UINT32_C(0x8) + /* This bit must be '1' for the dst_macaddr field to be configured. */ + #define HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_DST_MACADDR UINT32_C(0x10) + /* This bit must be '1' for the ovlan_vid field to be configured. */ + #define HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_OVLAN_VID UINT32_C(0x20) + /* This bit must be '1' for the ivlan_vid field to be configured. */ + #define HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_IVLAN_VID UINT32_C(0x40) + /* This bit must be '1' for the ethertype field to be configured. */ + #define HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_ETHERTYPE UINT32_C(0x80) + /* This bit must be '1' for the src_ipaddr field to be configured. */ + #define HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_SRC_IPADDR UINT32_C(0x100) + /* This bit must be '1' for the dst_ipaddr field to be configured. */ + #define HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_DST_IPADDR UINT32_C(0x200) + /* This bit must be '1' for the ipaddr_type field to be configured. */ + #define HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_IPADDR_TYPE UINT32_C(0x400) + /* This bit must be '1' for the ip_protocol field to be configured. */ + #define HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_IP_PROTOCOL UINT32_C(0x800) + /* This bit must be '1' for the src_port field to be configured. */ + #define HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_SRC_PORT UINT32_C(0x1000) + /* This bit must be '1' for the dst_port field to be configured. */ + #define HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_DST_PORT UINT32_C(0x2000) + /* This bit must be '1' for the dst_id field to be configured. */ + #define HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_DST_ID UINT32_C(0x4000) + /* + * This bit must be '1' for the mirror_vnic_id field to be + * configured. + */ + #define HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_MIRROR_VNIC_ID \ + UINT32_C(0x8000) + /* + * This bit must be '1' for the encap_record_id field to be + * configured. + */ + #define HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_ENCAP_RECORD_ID \ + UINT32_C(0x10000) + /* + * This bit must be '1' for the meter_instance_id field to be + * configured. + */ + #define HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_METER_INSTANCE_ID \ + UINT32_C(0x20000) + uint64_t l2_filter_id; + /* + * This value identifies a set of CFA data structures used for + * an L2 context. + */ + uint8_t tunnel_type; + /* Tunnel Type. */ + /* Non-tunnel */ + #define HWRM_CFA_EM_FLOW_ALLOC_INPUT_TUNNEL_TYPE_NONTUNNEL \ + UINT32_C(0x0) + /* Virtual eXtensible Local Area Network (VXLAN) */ + #define HWRM_CFA_EM_FLOW_ALLOC_INPUT_TUNNEL_TYPE_VXLAN UINT32_C(0x1) + /* + * Network Virtualization Generic Routing + * Encapsulation (NVGRE) + */ + #define HWRM_CFA_EM_FLOW_ALLOC_INPUT_TUNNEL_TYPE_NVGRE UINT32_C(0x2) + /* + * Generic Routing Encapsulation (GRE) inside + * Ethernet payload + */ + #define HWRM_CFA_EM_FLOW_ALLOC_INPUT_TUNNEL_TYPE_L2GRE UINT32_C(0x3) + /* IP in IP */ + #define HWRM_CFA_EM_FLOW_ALLOC_INPUT_TUNNEL_TYPE_IPIP UINT32_C(0x4) + /* Generic Network Virtualization Encapsulation (Geneve) */ + #define HWRM_CFA_EM_FLOW_ALLOC_INPUT_TUNNEL_TYPE_GENEVE UINT32_C(0x5) + /* Multi-Protocol Lable Switching (MPLS) */ + #define HWRM_CFA_EM_FLOW_ALLOC_INPUT_TUNNEL_TYPE_MPLS UINT32_C(0x6) + /* Stateless Transport Tunnel (STT) */ + #define HWRM_CFA_EM_FLOW_ALLOC_INPUT_TUNNEL_TYPE_STT UINT32_C(0x7) + /* + * Generic Routing Encapsulation (GRE) inside IP + * datagram payload + */ + #define HWRM_CFA_EM_FLOW_ALLOC_INPUT_TUNNEL_TYPE_IPGRE UINT32_C(0x8) + /* + * IPV4 over virtual eXtensible Local Area + * Network (IPV4oVXLAN) + */ + #define HWRM_CFA_EM_FLOW_ALLOC_INPUT_TUNNEL_TYPE_VXLAN_V4 UINT32_C(0x9) + /* Any tunneled traffic */ + #define HWRM_CFA_EM_FLOW_ALLOC_INPUT_TUNNEL_TYPE_ANYTUNNEL \ + UINT32_C(0xff) + uint8_t unused_0; + uint16_t unused_1; + uint32_t tunnel_id; + /* + * Tunnel identifier. Virtual Network Identifier (VNI). Only + * valid with tunnel_types VXLAN, NVGRE, and Geneve. Only lower + * 24-bits of VNI field are used in setting up the filter. + */ + uint8_t src_macaddr[6]; + /* + * This value indicates the source MAC address in the Ethernet + * header. + */ + uint16_t meter_instance_id; + /* The meter instance to attach to the flow. */ + /* + * A value of 0xfff is considered invalid and + * implies the instance is not configured. + */ + #define HWRM_CFA_EM_FLOW_ALLOC_INPUT_METER_INSTANCE_ID_INVALID \ + UINT32_C(0xffff) + uint8_t dst_macaddr[6]; + /* + * This value indicates the destination MAC address in the + * Ethernet header. + */ + uint16_t ovlan_vid; + /* + * This value indicates the VLAN ID of the outer VLAN tag in the + * Ethernet header. + */ + uint16_t ivlan_vid; + /* + * This value indicates the VLAN ID of the inner VLAN tag in the + * Ethernet header. + */ + uint16_t ethertype; + /* This value indicates the ethertype in the Ethernet header. */ + uint8_t ip_addr_type; + /* + * This value indicates the type of IP address. 4 - IPv4 6 - + * IPv6 All others are invalid. + */ + /* invalid */ + #define HWRM_CFA_EM_FLOW_ALLOC_INPUT_IP_ADDR_TYPE_UNKNOWN UINT32_C(0x0) + /* IPv4 */ + #define HWRM_CFA_EM_FLOW_ALLOC_INPUT_IP_ADDR_TYPE_IPV4 UINT32_C(0x4) + /* IPv6 */ + #define HWRM_CFA_EM_FLOW_ALLOC_INPUT_IP_ADDR_TYPE_IPV6 UINT32_C(0x6) + uint8_t ip_protocol; + /* + * The value of protocol filed in IP header. Applies to UDP and + * TCP traffic. 6 - TCP 17 - UDP + */ + /* invalid */ + #define HWRM_CFA_EM_FLOW_ALLOC_INPUT_IP_PROTOCOL_UNKNOWN UINT32_C(0x0) + /* TCP */ + #define HWRM_CFA_EM_FLOW_ALLOC_INPUT_IP_PROTOCOL_TCP UINT32_C(0x6) + /* UDP */ + #define HWRM_CFA_EM_FLOW_ALLOC_INPUT_IP_PROTOCOL_UDP UINT32_C(0x11) + uint8_t unused_2; + uint8_t unused_3; + uint32_t src_ipaddr[4]; + /* + * The value of source IP address to be used in filtering. For + * IPv4, first four bytes represent the IP address. + */ + uint32_t dst_ipaddr[4]; + /* + * big_endian = True The value of destination IP address to be + * used in filtering. For IPv4, first four bytes represent the + * IP address. + */ + uint16_t src_port; + /* + * The value of source port to be used in filtering. Applies to + * UDP and TCP traffic. + */ + uint16_t dst_port; + /* + * The value of destination port to be used in filtering. + * Applies to UDP and TCP traffic. + */ + uint16_t dst_id; + /* + * If set, this value shall represent the Logical VNIC ID of the + * destination VNIC for the RX path and network port id of the + * destination port for the TX path. + */ + uint16_t mirror_vnic_id; + /* Logical VNIC ID of the VNIC where traffic is mirrored. */ + uint32_t encap_record_id; + /* Logical ID of the encapsulation record. */ + uint32_t unused_4; +} __attribute__((packed)); + +/* Output (24 bytes) */ +struct hwrm_cfa_em_flow_alloc_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint64_t em_filter_id; + /* This value is an opaque id into CFA data structures. */ + uint32_t flow_id; + /* + * This is the ID of the flow associated with this filter. This + * value shall be used to match and associate the flow + * identifier returned in completion records. A value of + * 0xFFFFFFFF shall indicate no flow id. + */ + uint8_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* hwrm_cfa_em_flow_free */ +/* Description: Free an EM flow table entry */ +/* Input (24 bytes) */ +struct hwrm_cfa_em_flow_free_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint64_t em_filter_id; + /* This value is an opaque id into CFA data structures. */ +} __attribute__((packed)); + +/* Output (16 bytes) */ +struct hwrm_cfa_em_flow_free_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint32_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t unused_3; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* hwrm_cfa_em_flow_cfg */ +/* + * Description: Configure an EM flow with a new destination VNIC and/or meter. + */ +/* Input (48 bytes) */ +struct hwrm_cfa_em_flow_cfg_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint32_t enables; + /* This bit must be '1' for the new_dst_id field to be configured. */ + #define HWRM_CFA_EM_FLOW_CFG_INPUT_ENABLES_NEW_DST_ID UINT32_C(0x1) + /* + * This bit must be '1' for the new_mirror_vnic_id field to be + * configured. + */ + #define HWRM_CFA_EM_FLOW_CFG_INPUT_ENABLES_NEW_MIRROR_VNIC_ID \ + UINT32_C(0x2) + /* + * This bit must be '1' for the new_meter_instance_id field to + * be configured. + */ + #define HWRM_CFA_EM_FLOW_CFG_INPUT_ENABLES_NEW_METER_INSTANCE_ID \ + UINT32_C(0x4) + uint32_t unused_0; + uint64_t em_filter_id; + /* This value is an opaque id into CFA data structures. */ + uint32_t new_dst_id; + /* + * If set, this value shall represent the new Logical VNIC ID of + * the destination VNIC for the RX path and network port id of + * the destination port for the TX path. + */ + uint32_t new_mirror_vnic_id; + /* New Logical VNIC ID of the VNIC where traffic is mirrored. */ + uint16_t new_meter_instance_id; + /* + * New meter to attach to the flow. Specifying the invalid + * instance ID is used to remove any existing meter from the + * flow. + */ + /* + * A value of 0xfff is considered invalid and + * implies the instance is not configured. + */ + #define HWRM_CFA_EM_FLOW_CFG_INPUT_NEW_METER_INSTANCE_ID_INVALID \ + UINT32_C(0xffff) + uint16_t unused_1[3]; +} __attribute__((packed)); + +/* Output (16 bytes) */ +struct hwrm_cfa_em_flow_cfg_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint32_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t unused_3; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* hwrm_tunnel_dst_port_query */ +/* + * Description: This function is called by a driver to query tunnel type + * specific destination port configuration. + */ +/* Input (24 bytes) */ +struct hwrm_tunnel_dst_port_query_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint8_t tunnel_type; + /* Tunnel Type. */ + /* Virtual eXtensible Local Area Network (VXLAN) */ + #define HWRM_TUNNEL_DST_PORT_QUERY_INPUT_TUNNEL_TYPE_VXLAN \ + UINT32_C(0x1) + /* Generic Network Virtualization Encapsulation (Geneve) */ + #define HWRM_TUNNEL_DST_PORT_QUERY_INPUT_TUNNEL_TYPE_GENEVE \ + UINT32_C(0x5) + /* + * IPV4 over virtual eXtensible Local Area + * Network (IPV4oVXLAN) + */ + #define HWRM_TUNNEL_DST_PORT_QUERY_INPUT_TUNNEL_TYPE_VXLAN_V4 \ + UINT32_C(0x9) + uint8_t unused_0[7]; +} __attribute__((packed)); + +/* Output (16 bytes) */ +struct hwrm_tunnel_dst_port_query_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint16_t tunnel_dst_port_id; + /* + * This field represents the identifier of L4 destination port + * used for the given tunnel type. This field is valid for + * specific tunnel types that use layer 4 (e.g. UDP) transports + * for tunneling. + */ + uint16_t tunnel_dst_port_val; + /* + * This field represents the value of L4 destination port + * identified by tunnel_dst_port_id. This field is valid for + * specific tunnel types that use layer 4 (e.g. UDP) transports + * for tunneling. This field is in network byte order. A value + * of 0 means that the destination port is not configured. + */ + uint8_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* hwrm_tunnel_dst_port_alloc */ +/* + * Description: This function is called by a driver to allocate l4 destination + * port for a specific tunnel type. The destination port value is provided in + * the input. If the HWRM supports only one global destination port for a tunnel + * type, then the HWRM shall keep track of its usage as described below. # The + * first caller that allocates a destination port shall always succeed and the + * HWRM shall save the destination port configuration for that tunnel type and + * increment the usage count to 1. # Subsequent callers allocating the same + * destination port for that tunnel type shall succeed and the HWRM shall + * increment the usage count for that port for each subsequent caller that + * succeeds. # Any subsequent caller trying to allocate a different destination + * port for that tunnel type shall fail until the usage count for the original + * destination port goes to zero. # A caller that frees a port will cause the + * usage count for that port to decrement. + */ +/* Input (24 bytes) */ +struct hwrm_tunnel_dst_port_alloc_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint8_t tunnel_type; + /* Tunnel Type. */ + /* Virtual eXtensible Local Area Network (VXLAN) */ + #define HWRM_TUNNEL_DST_PORT_ALLOC_INPUT_TUNNEL_TYPE_VXLAN UINT32_C(0x1) + /* Generic Network Virtualization Encapsulation (Geneve) */ + #define HWRM_TUNNEL_DST_PORT_ALLOC_INPUT_TUNNEL_TYPE_GENEVE \ + UINT32_C(0x5) + /* + * IPV4 over virtual eXtensible Local Area + * Network (IPV4oVXLAN) + */ + #define HWRM_TUNNEL_DST_PORT_ALLOC_INPUT_TUNNEL_TYPE_VXLAN_V4 \ + UINT32_C(0x9) + uint8_t unused_0; + uint16_t tunnel_dst_port_val; + /* + * This field represents the value of L4 destination port used + * for the given tunnel type. This field is valid for specific + * tunnel types that use layer 4 (e.g. UDP) transports for + * tunneling. This field is in network byte order. A value of 0 + * shall fail the command. + */ + uint32_t unused_1; +} __attribute__((packed)); + +/* Output (16 bytes) */ +struct hwrm_tunnel_dst_port_alloc_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint16_t tunnel_dst_port_id; + /* + * Identifier of a tunnel L4 destination port value. Only + * applies to tunnel types that has l4 destination port + * parameters. + */ + uint8_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t unused_3; + uint8_t unused_4; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* hwrm_tunnel_dst_port_free */ +/* + * Description: This function is called by a driver to free l4 destination port + * for a specific tunnel type. + */ +/* Input (24 bytes) */ +struct hwrm_tunnel_dst_port_free_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint8_t tunnel_type; + /* Tunnel Type. */ + /* Virtual eXtensible Local Area Network (VXLAN) */ + #define HWRM_TUNNEL_DST_PORT_FREE_INPUT_TUNNEL_TYPE_VXLAN UINT32_C(0x1) + /* Generic Network Virtualization Encapsulation (Geneve) */ + #define HWRM_TUNNEL_DST_PORT_FREE_INPUT_TUNNEL_TYPE_GENEVE UINT32_C(0x5) + /* + * IPV4 over virtual eXtensible Local Area + * Network (IPV4oVXLAN) + */ + #define HWRM_TUNNEL_DST_PORT_FREE_INPUT_TUNNEL_TYPE_VXLAN_V4 \ + UINT32_C(0x9) + uint8_t unused_0; + uint16_t tunnel_dst_port_id; + /* + * Identifier of a tunnel L4 destination port value. Only + * applies to tunnel types that has l4 destination port + * parameters. + */ + uint32_t unused_1; +} __attribute__((packed)); + +/* Output (16 bytes) */ +struct hwrm_tunnel_dst_port_free_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint32_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t unused_3; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* hwrm_stat_ctx_alloc */ +/* + * Description: This command allocates and does basic preparation for a stat + * context. + */ +/* Input (32 bytes) */ +struct hwrm_stat_ctx_alloc_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint64_t stats_dma_addr; + /* This is the address for statistic block. */ + uint32_t update_period_ms; + /* + * The statistic block update period in ms. e.g. 250ms, 500ms, + * 750ms, 1000ms. If update_period_ms is 0, then the stats + * update shall be never done and the DMA address shall not be + * used. In this case, the stat block can only be read by + * hwrm_stat_ctx_query command. + */ + uint8_t stat_ctx_flags; + /* + * This field is used to specify statistics context specific + * configuration flags. + */ + /* + * When this bit is set to '1', the statistics context shall be + * allocated for RoCE traffic only. In this case, traffic other + * than offloaded RoCE traffic shall not be included in this + * statistic context. When this bit is set to '0', the + * statistics context shall be used for the network traffic + * other than offloaded RoCE traffic. + */ + #define HWRM_STAT_CTX_ALLOC_INPUT_STAT_CTX_FLAGS_ROCE UINT32_C(0x1) + uint8_t unused_0[3]; +} __attribute__((packed)); + +/* Output (16 bytes) */ +struct hwrm_stat_ctx_alloc_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint32_t stat_ctx_id; + /* This is the statistics context ID value. */ + uint8_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* hwrm_stat_ctx_free */ +/* Description: This command is used to free a stat context. */ +/* Input (24 bytes) */ +struct hwrm_stat_ctx_free_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint32_t stat_ctx_id; + /* ID of the statistics context that is being queried. */ + uint32_t unused_0; +} __attribute__((packed)); + +/* Output (16 bytes) */ +struct hwrm_stat_ctx_free_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint32_t stat_ctx_id; + /* This is the statistics context ID value. */ + uint8_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* hwrm_stat_ctx_query */ +/* Description: This command returns statistics of a context. */ +/* Input (24 bytes) */ +struct hwrm_stat_ctx_query_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format for the + * rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request will be + * optionally completed on. If the value is -1, then no CR completion + * will be generated. Any other value must be a valid CR ring_id value + * for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids + * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written when the + * request is complete. This area must be 16B aligned and must be + * cleared to zero before the request is made. + */ + uint32_t stat_ctx_id; + /* ID of the statistics context that is being queried. */ + uint32_t unused_0; +} __attribute__((packed)); + +/* Output (176 bytes) */ +struct hwrm_stat_ctx_query_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in parameters, + * and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last byte of + * the response is a valid flag that will read as '1' when the command + * has been completely written to memory. + */ + uint64_t tx_ucast_pkts; + /* Number of transmitted unicast packets */ + uint64_t tx_mcast_pkts; + /* Number of transmitted multicast packets */ + uint64_t tx_bcast_pkts; + /* Number of transmitted broadcast packets */ + uint64_t tx_err_pkts; + /* Number of transmitted packets with error */ + uint64_t tx_drop_pkts; + /* Number of dropped packets on transmit path */ + uint64_t tx_ucast_bytes; + /* Number of transmitted bytes for unicast traffic */ + uint64_t tx_mcast_bytes; + /* Number of transmitted bytes for multicast traffic */ + uint64_t tx_bcast_bytes; + /* Number of transmitted bytes for broadcast traffic */ + uint64_t rx_ucast_pkts; + /* Number of received unicast packets */ + uint64_t rx_mcast_pkts; + /* Number of received multicast packets */ + uint64_t rx_bcast_pkts; + /* Number of received broadcast packets */ + uint64_t rx_err_pkts; + /* Number of received packets with error */ + uint64_t rx_drop_pkts; + /* Number of dropped packets on received path */ + uint64_t rx_ucast_bytes; + /* Number of received bytes for unicast traffic */ + uint64_t rx_mcast_bytes; + /* Number of received bytes for multicast traffic */ + uint64_t rx_bcast_bytes; + /* Number of received bytes for broadcast traffic */ + uint64_t rx_agg_pkts; + /* Number of aggregated unicast packets */ + uint64_t rx_agg_bytes; + /* Number of aggregated unicast bytes */ + uint64_t rx_agg_events; + /* Number of aggregation events */ + uint64_t rx_agg_aborts; + /* Number of aborted aggregations */ + uint32_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t unused_3; + uint8_t valid; + /* + * This field is used in Output records to indicate that the output is + * completely written to RAM. This field should be read as '1' to + * indicate that the output has been completely written. When writing a + * command completion or response to an internal processor, the order of + * writes has to be such that this field is written last. + */ +} __attribute__((packed)); + +/* hwrm_stat_ctx_clr_stats */ +/* Description: This command clears statistics of a context. */ +/* Input (24 bytes) */ +struct hwrm_stat_ctx_clr_stats_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint32_t stat_ctx_id; + /* ID of the statistics context that is being queried. */ + uint32_t unused_0; +} __attribute__((packed)); + +/* Output (16 bytes) */ +struct hwrm_stat_ctx_clr_stats_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint32_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t unused_3; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* hwrm_exec_fwd_resp */ +/* + * Description: This command is used to send an encapsulated request to the + * HWRM. This command instructs the HWRM to execute the request and forward the + * response of the encapsulated request to the location specified in the + * original request that is encapsulated. The target id of this command shall be + * set to 0xFFFF (HWRM). The response location in this command shall be used to + * acknowledge the receipt of the encapsulated request and forwarding of the + * response. + */ +/* Input (128 bytes) */ +struct hwrm_exec_fwd_resp_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint32_t encap_request[26]; + /* + * This is an encapsulated request. This request should be + * executed by the HWRM and the response should be provided in + * the response buffer inside the encapsulated request. + */ + uint16_t encap_resp_target_id; + /* + * This value indicates the target id of the response to the + * encapsulated request. 0x0 - 0xFFF8 - Used for function ids + * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - + * HWRM + */ + uint16_t unused_0[3]; +} __attribute__((packed)); + +/* Output (16 bytes) */ +struct hwrm_exec_fwd_resp_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint32_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t unused_3; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* hwrm_reject_fwd_resp */ +/* + * Description: This command is used to send an encapsulated request to the + * HWRM. This command instructs the HWRM to reject the request and forward the + * error response of the encapsulated request to the location specified in the + * original request that is encapsulated. The target id of this command shall be + * set to 0xFFFF (HWRM). The response location in this command shall be used to + * acknowledge the receipt of the encapsulated request and forwarding of the + * response. + */ +/* Input (128 bytes) */ +struct hwrm_reject_fwd_resp_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint32_t encap_request[26]; + /* + * This is an encapsulated request. This request should be + * rejected by the HWRM and the error response should be + * provided in the response buffer inside the encapsulated + * request. + */ + uint16_t encap_resp_target_id; + /* + * This value indicates the target id of the response to the + * encapsulated request. 0x0 - 0xFFF8 - Used for function ids + * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - + * HWRM + */ + uint16_t unused_0[3]; +} __attribute__((packed)); + +/* Output (16 bytes) */ +struct hwrm_reject_fwd_resp_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint32_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t unused_3; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* hwrm_nvm_get_dir_entries */ +/* Input (24 bytes) */ +struct hwrm_nvm_get_dir_entries_input { + uint16_t req_type; + uint16_t cmpl_ring; + uint16_t seq_id; + uint16_t target_id; + uint64_t resp_addr; + uint64_t host_dest_addr; +} __attribute__((packed)); + +/* Output (16 bytes) */ +struct hwrm_nvm_get_dir_entries_output { + uint16_t error_code; + uint16_t req_type; + uint16_t seq_id; + uint16_t resp_len; + uint32_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t unused_3; uint8_t valid; } __attribute__((packed)); -#endif + +/* hwrm_nvm_erase_dir_entry */ +/* Input (24 bytes) */ +struct hwrm_nvm_erase_dir_entry_input { + uint16_t req_type; + uint16_t cmpl_ring; + uint16_t seq_id; + uint16_t target_id; + uint64_t resp_addr; + uint16_t dir_idx; + uint16_t unused_0[3]; +}; + +/* Output (16 bytes) */ +struct hwrm_nvm_erase_dir_entry_output { + uint16_t error_code; + uint16_t req_type; + uint16_t seq_id; + uint16_t resp_len; + uint32_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t unused_3; + uint8_t valid; +}; + +/* hwrm_nvm_get_dir_info */ +/* Input (16 bytes) */ +struct hwrm_nvm_get_dir_info_input { + uint16_t req_type; + uint16_t cmpl_ring; + uint16_t seq_id; + uint16_t target_id; + uint64_t resp_addr; +} __attribute__((packed)); + +/* Output (24 bytes) */ +struct hwrm_nvm_get_dir_info_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint32_t entries; + /* Number of directory entries in the directory. */ + uint32_t entry_length; + /* Size of each directory entry, in bytes. */ + uint32_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t unused_3; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* hwrm_nvm_write */ +/* + * Note: Write to the allocated NVRAM of an item referenced by an existing + * directory entry. + */ +/* Input (48 bytes) */ +struct hwrm_nvm_write_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint64_t host_src_addr; + /* 64-bit Host Source Address. This is where the source data is. */ + uint16_t dir_type; + /* + * The Directory Entry Type (valid values are defined in the + * bnxnvm_directory_type enum defined in the file + * bnxnvm_defs.h). + */ + uint16_t dir_ordinal; + /* + * Directory ordinal. The 0-based instance of the combined + * Directory Entry Type and Extension. + */ + uint16_t dir_ext; + /* + * The Directory Entry Extension flags (see BNX_DIR_EXT_* in the + * file bnxnvm_defs.h). + */ + uint16_t dir_attr; + /* + * Directory Entry Attribute flags (see BNX_DIR_ATTR_* in the + * file bnxnvm_defs.h). + */ + uint32_t dir_data_length; + /* + * Length of data to write, in bytes. May be less than or equal + * to the allocated size for the directory entry. The data + * length stored in the directory entry will be updated to + * reflect this value once the write is complete. + */ + uint16_t option; + /* Option. */ + uint16_t flags; + /* + * When this bit is '1', the original active image will not be + * removed. TBD: what purpose is this? + */ + #define HWRM_NVM_WRITE_INPUT_FLAGS_KEEP_ORIG_ACTIVE_IMG UINT32_C(0x1) + uint32_t dir_item_length; + /* + * The requested length of the allocated NVM for the item, in + * bytes. This value may be greater than or equal to the + * specified data length (dir_data_length). If this value is + * less than the specified data length, it will be ignored. The + * response will contain the actual allocated item length, which + * may be greater than the requested item length. The purpose + * for allocating more than the required number of bytes for an + * item's data is to pre-allocate extra storage (padding) to + * accommodate the potential future growth of an item (e.g. + * upgraded firmware with a size increase, log growth, expanded + * configuration data). + */ + uint32_t unused_0; +} __attribute__((packed)); + +/* Output (16 bytes) */ +struct hwrm_nvm_write_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint32_t dir_item_length; + /* + * Length of the allocated NVM for the item, in bytes. The value + * may be greater than or equal to the specified data length or + * the requested item length. The actual item length used when + * creating a new directory entry will be a multiple of an NVM + * block size. + */ + uint16_t dir_idx; + /* The directory index of the created or modified item. */ + uint8_t unused_0; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* hwrm_nvm_read */ +/* + * Note: Read the contents of an NVRAM item as referenced (indexed) by an + * existing directory entry. + */ +/* Input (40 bytes) */ +struct hwrm_nvm_read_input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ + uint64_t host_dest_addr; + /* + * 64-bit Host Destination Address. This is the host address + * where the data will be written to. + */ + uint16_t dir_idx; + /* The 0-based index of the directory entry. */ + uint8_t unused_0; + uint8_t unused_1; + uint32_t offset; + /* The NVRAM byte-offset to read from. */ + uint32_t len; + /* The length of the data to be read, in bytes. */ + uint32_t unused_2; +} __attribute__((packed)); + +/* Output (16 bytes) */ +struct hwrm_nvm_read_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint32_t unused_0; + uint8_t unused_1; + uint8_t unused_2; + uint8_t unused_3; + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* Hardware Resource Manager Specification */ +/* Description: This structure is used to specify port description. */ +/* + * Note: The Hardware Resource Manager (HWRM) manages various hardware resources + * inside the chip. The HWRM is implemented in firmware, and runs on embedded + * processors inside the chip. This firmware service is vital part of the chip. + * The chip can not be used by a driver or HWRM client without the HWRM. + */ +/* Input (16 bytes) */ +struct input { + uint16_t req_type; + /* + * This value indicates what type of request this is. The format + * for the rest of the command is determined by this field. + */ + uint16_t cmpl_ring; + /* + * This value indicates the what completion ring the request + * will be optionally completed on. If the value is -1, then no + * CR completion will be generated. Any other value must be a + * valid CR ring_id value for this function. + */ + uint16_t seq_id; + /* This value indicates the command sequence number. */ + uint16_t target_id; + /* + * Target ID of this command. 0x0 - 0xFFF8 - Used for function + * ids 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF + * - HWRM + */ + uint64_t resp_addr; + /* + * This is the host address where the response will be written + * when the request is complete. This area must be 16B aligned + * and must be cleared to zero before the request is made. + */ +} __attribute__((packed)); + +/* Output (8 bytes) */ +struct output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ +} __attribute__((packed)); + +/* Short Command Structure (16 bytes) */ +struct hwrm_short_input { + uint16_t req_type; + /* + * This field indicates the type of request in the request + * buffer. The format for the rest of the command (request) is + * determined by this field. + */ + uint16_t signature; + /* + * This field indicates a signature that is used to identify + * short form of the command listed here. This field shall be + * set to 17185 (0x4321). + */ + /* Signature indicating this is a short form of HWRM command */ + #define HWRM_SHORT_REQ_SIGNATURE_SHORT_CMD UINT32_C(0x4321) + uint16_t unused_0; + /* Reserved for future use. */ + uint16_t size; + /* This value indicates the length of the request. */ + uint64_t req_addr; + /* + * This is the host address where the request was written. This + * area must be 16B aligned. + */ +} __attribute__((packed)); + +#define HWRM_GET_HWRM_ERROR_CODE(arg) \ + { \ + typeof(arg) x = (arg); \ + ((x) == 0xf ? "HWRM_ERROR" : \ + ((x) == 0xffff ? "CMD_NOT_SUPPORTED" : \ + ((x) == 0xfffe ? "UNKNOWN_ERR" : \ + ((x) == 0x4 ? "RESOURCE_ALLOC_ERROR" : \ + ((x) == 0x5 ? "INVALID_FLAGS" : \ + ((x) == 0x6 ? "INVALID_ENABLES" : \ + ((x) == 0x0 ? "SUCCESS" : \ + ((x) == 0x1 ? "FAIL" : \ + ((x) == 0x2 ? "INVALID_PARAMS" : \ + ((x) == 0x3 ? "RESOURCE_ACCESS_DENIED" : \ + "Unknown error_code")))))))))) \ + } + +/* Return Codes (8 bytes) */ +struct ret_codes { + uint16_t error_code; + /* These are numbers assigned to return/error codes. */ + /* Request was successfully executed by the HWRM. */ + #define HWRM_ERR_CODE_SUCCESS (UINT32_C(0x0)) + /* THe HWRM failed to execute the request. */ + #define HWRM_ERR_CODE_FAIL (UINT32_C(0x1)) + /* + * The request contains invalid argument(s) or + * input parameters. + */ + #define HWRM_ERR_CODE_INVALID_PARAMS (UINT32_C(0x2)) + /* + * The requester is not allowed to access the + * requested resource. This error code shall be + * provided in a response to a request to query + * or modify an existing resource that is not + * accessible by the requester. + */ + #define HWRM_ERR_CODE_RESOURCE_ACCESS_DENIED (UINT32_C(0x3)) + /* + * The HWRM is unable to allocate the requested + * resource. This code only applies to requests + * for HWRM resource allocations. + */ + #define HWRM_ERR_CODE_RESOURCE_ALLOC_ERROR (UINT32_C(0x4)) + /* Invalid combination of flags is specified in the request. */ + #define HWRM_ERR_CODE_INVALID_FLAGS (UINT32_C(0x5)) + /* + * Invalid combination of enables fields is + * specified in the request. + */ + #define HWRM_ERR_CODE_INVALID_ENABLES (UINT32_C(0x6)) + /* + * Generic HWRM execution error that represents + * an internal error. + */ + #define HWRM_ERR_CODE_HWRM_ERROR (UINT32_C(0xf)) + /* Unknown error */ + #define HWRM_ERR_CODE_UNKNOWN_ERR (UINT32_C(0xfffe)) + /* Unsupported or invalid command */ + #define HWRM_ERR_CODE_CMD_NOT_SUPPORTED (UINT32_C(0xffff)) + uint16_t unused_0[3]; +} __attribute__((packed)); + +/* Output (16 bytes) */ +struct hwrm_err_output { + uint16_t error_code; + /* + * Pass/Fail or error type Note: receiver to verify the in + * parameters, and fail the call with an error when appropriate + */ + uint16_t req_type; + /* This field returns the type of original request. */ + uint16_t seq_id; + /* This field provides original sequence number of the command. */ + uint16_t resp_len; + /* + * This field is the length of the response in bytes. The last + * byte of the response is a valid flag that will read as '1' + * when the command has been completely written to memory. + */ + uint32_t opaque_0; + /* debug info for this error response. */ + uint16_t opaque_1; + /* debug info for this error response. */ + uint8_t cmd_err; + /* + * In the case of an error response, command specific error code + * is returned in this field. + */ + uint8_t valid; + /* + * This field is used in Output records to indicate that the + * output is completely written to RAM. This field should be + * read as '1' to indicate that the output has been completely + * written. When writing a command completion or response to an + * internal processor, the order of writes has to be such that + * this field is written last. + */ +} __attribute__((packed)); + +/* Port Tx Statistics Formats (408 bytes) */ +struct tx_port_stats { + uint64_t tx_64b_frames; + /* Total Number of 64 Bytes frames transmitted */ + uint64_t tx_65b_127b_frames; + /* Total Number of 65-127 Bytes frames transmitted */ + uint64_t tx_128b_255b_frames; + /* Total Number of 128-255 Bytes frames transmitted */ + uint64_t tx_256b_511b_frames; + /* Total Number of 256-511 Bytes frames transmitted */ + uint64_t tx_512b_1023b_frames; + /* Total Number of 512-1023 Bytes frames transmitted */ + uint64_t tx_1024b_1518_frames; + /* Total Number of 1024-1518 Bytes frames transmitted */ + uint64_t tx_good_vlan_frames; + /* + * Total Number of each good VLAN (exludes FCS errors) frame + * transmitted which is 1519 to 1522 bytes in length inclusive + * (excluding framing bits but including FCS bytes). + */ + uint64_t tx_1519b_2047_frames; + /* Total Number of 1519-2047 Bytes frames transmitted */ + uint64_t tx_2048b_4095b_frames; + /* Total Number of 2048-4095 Bytes frames transmitted */ + uint64_t tx_4096b_9216b_frames; + /* Total Number of 4096-9216 Bytes frames transmitted */ + uint64_t tx_9217b_16383b_frames; + /* Total Number of 9217-16383 Bytes frames transmitted */ + uint64_t tx_good_frames; + /* Total Number of good frames transmitted */ + uint64_t tx_total_frames; + /* Total Number of frames transmitted */ + uint64_t tx_ucast_frames; + /* Total number of unicast frames transmitted */ + uint64_t tx_mcast_frames; + /* Total number of multicast frames transmitted */ + uint64_t tx_bcast_frames; + /* Total number of broadcast frames transmitted */ + uint64_t tx_pause_frames; + /* Total number of PAUSE control frames transmitted */ + uint64_t tx_pfc_frames; + /* Total number of PFC/per-priority PAUSE control frames transmitted */ + uint64_t tx_jabber_frames; + /* Total number of jabber frames transmitted */ + uint64_t tx_fcs_err_frames; + /* Total number of frames transmitted with FCS error */ + uint64_t tx_control_frames; + /* Total number of control frames transmitted */ + uint64_t tx_oversz_frames; + /* Total number of over-sized frames transmitted */ + uint64_t tx_single_dfrl_frames; + /* Total number of frames with single deferral */ + uint64_t tx_multi_dfrl_frames; + /* Total number of frames with multiple deferrals */ + uint64_t tx_single_coll_frames; + /* Total number of frames with single collision */ + uint64_t tx_multi_coll_frames; + /* Total number of frames with multiple collisions */ + uint64_t tx_late_coll_frames; + /* Total number of frames with late collisions */ + uint64_t tx_excessive_coll_frames; + /* Total number of frames with excessive collisions */ + uint64_t tx_frag_frames; + /* Total number of fragmented frames transmitted */ + uint64_t tx_err; + /* Total number of transmit errors */ + uint64_t tx_tagged_frames; + /* Total number of single VLAN tagged frames transmitted */ + uint64_t tx_dbl_tagged_frames; + /* Total number of double VLAN tagged frames transmitted */ + uint64_t tx_runt_frames; + /* Total number of runt frames transmitted */ + uint64_t tx_fifo_underruns; + /* Total number of TX FIFO under runs */ + uint64_t tx_pfc_ena_frames_pri0; + /* + * Total number of PFC frames with PFC enabled bit for Pri 0 + * transmitted + */ + uint64_t tx_pfc_ena_frames_pri1; + /* + * Total number of PFC frames with PFC enabled bit for Pri 1 + * transmitted + */ + uint64_t tx_pfc_ena_frames_pri2; + /* + * Total number of PFC frames with PFC enabled bit for Pri 2 + * transmitted + */ + uint64_t tx_pfc_ena_frames_pri3; + /* + * Total number of PFC frames with PFC enabled bit for Pri 3 + * transmitted + */ + uint64_t tx_pfc_ena_frames_pri4; + /* + * Total number of PFC frames with PFC enabled bit for Pri 4 + * transmitted + */ + uint64_t tx_pfc_ena_frames_pri5; + /* + * Total number of PFC frames with PFC enabled bit for Pri 5 + * transmitted + */ + uint64_t tx_pfc_ena_frames_pri6; + /* + * Total number of PFC frames with PFC enabled bit for Pri 6 + * transmitted + */ + uint64_t tx_pfc_ena_frames_pri7; + /* + * Total number of PFC frames with PFC enabled bit for Pri 7 + * transmitted + */ + uint64_t tx_eee_lpi_events; + /* Total number of EEE LPI Events on TX */ + uint64_t tx_eee_lpi_duration; + /* EEE LPI Duration Counter on TX */ + uint64_t tx_llfc_logical_msgs; + /* + * Total number of Link Level Flow Control (LLFC) messages + * transmitted + */ + uint64_t tx_hcfc_msgs; + /* Total number of HCFC messages transmitted */ + uint64_t tx_total_collisions; + /* Total number of TX collisions */ + uint64_t tx_bytes; + /* Total number of transmitted bytes */ + uint64_t tx_xthol_frames; + /* Total number of end-to-end HOL frames */ + uint64_t tx_stat_discard; + /* Total Tx Drops per Port reported by STATS block */ + uint64_t tx_stat_error; + /* Total Tx Error Drops per Port reported by STATS block */ +} __attribute__((packed)); + +/* Port Rx Statistics Formats (528 bytes) */ +struct rx_port_stats { + uint64_t rx_64b_frames; + /* Total Number of 64 Bytes frames received */ + uint64_t rx_65b_127b_frames; + /* Total Number of 65-127 Bytes frames received */ + uint64_t rx_128b_255b_frames; + /* Total Number of 128-255 Bytes frames received */ + uint64_t rx_256b_511b_frames; + /* Total Number of 256-511 Bytes frames received */ + uint64_t rx_512b_1023b_frames; + /* Total Number of 512-1023 Bytes frames received */ + uint64_t rx_1024b_1518_frames; + /* Total Number of 1024-1518 Bytes frames received */ + uint64_t rx_good_vlan_frames; + /* + * Total Number of each good VLAN (exludes FCS errors) frame + * received which is 1519 to 1522 bytes in length inclusive + * (excluding framing bits but including FCS bytes). + */ + uint64_t rx_1519b_2047b_frames; + /* Total Number of 1519-2047 Bytes frames received */ + uint64_t rx_2048b_4095b_frames; + /* Total Number of 2048-4095 Bytes frames received */ + uint64_t rx_4096b_9216b_frames; + /* Total Number of 4096-9216 Bytes frames received */ + uint64_t rx_9217b_16383b_frames; + /* Total Number of 9217-16383 Bytes frames received */ + uint64_t rx_total_frames; + /* Total number of frames received */ + uint64_t rx_ucast_frames; + /* Total number of unicast frames received */ + uint64_t rx_mcast_frames; + /* Total number of multicast frames received */ + uint64_t rx_bcast_frames; + /* Total number of broadcast frames received */ + uint64_t rx_fcs_err_frames; + /* Total number of received frames with FCS error */ + uint64_t rx_ctrl_frames; + /* Total number of control frames received */ + uint64_t rx_pause_frames; + /* Total number of PAUSE frames received */ + uint64_t rx_pfc_frames; + /* Total number of PFC frames received */ + uint64_t rx_unsupported_opcode_frames; + /* Total number of frames received with an unsupported opcode */ + uint64_t rx_unsupported_da_pausepfc_frames; + /* + * Total number of frames received with an unsupported DA for + * pause and PFC + */ + uint64_t rx_wrong_sa_frames; + /* Total number of frames received with an unsupported SA */ + uint64_t rx_align_err_frames; + /* Total number of received packets with alignment error */ + uint64_t rx_oor_len_frames; + /* Total number of received frames with out-of-range length */ + uint64_t rx_code_err_frames; + /* Total number of received frames with error termination */ + uint64_t rx_false_carrier_frames; + /* + * Total number of received frames with a false carrier is + * detected during idle, as defined by RX_ER samples active and + * RXD is 0xE. The event is reported along with the statistics + * generated on the next received frame. Only one false carrier + * condition can be detected and logged between frames. Carrier + * event, valid for 10M/100M speed modes only. + */ + uint64_t rx_ovrsz_frames; + /* Total number of over-sized frames received */ + uint64_t rx_jbr_frames; + /* Total number of jabber packets received */ + uint64_t rx_mtu_err_frames; + /* Total number of received frames with MTU error */ + uint64_t rx_match_crc_frames; + /* Total number of received frames with CRC match */ + uint64_t rx_promiscuous_frames; + /* Total number of frames received promiscuously */ + uint64_t rx_tagged_frames; + /* Total number of received frames with one or two VLAN tags */ + uint64_t rx_double_tagged_frames; + /* Total number of received frames with two VLAN tags */ + uint64_t rx_trunc_frames; + /* Total number of truncated frames received */ + uint64_t rx_good_frames; + /* Total number of good frames (without errors) received */ + uint64_t rx_pfc_xon2xoff_frames_pri0; + /* + * Total number of received PFC frames with transition from XON + * to XOFF on Pri 0 + */ + uint64_t rx_pfc_xon2xoff_frames_pri1; + /* + * Total number of received PFC frames with transition from XON + * to XOFF on Pri 1 + */ + uint64_t rx_pfc_xon2xoff_frames_pri2; + /* + * Total number of received PFC frames with transition from XON + * to XOFF on Pri 2 + */ + uint64_t rx_pfc_xon2xoff_frames_pri3; + /* + * Total number of received PFC frames with transition from XON + * to XOFF on Pri 3 + */ + uint64_t rx_pfc_xon2xoff_frames_pri4; + /* + * Total number of received PFC frames with transition from XON + * to XOFF on Pri 4 + */ + uint64_t rx_pfc_xon2xoff_frames_pri5; + /* + * Total number of received PFC frames with transition from XON + * to XOFF on Pri 5 + */ + uint64_t rx_pfc_xon2xoff_frames_pri6; + /* + * Total number of received PFC frames with transition from XON + * to XOFF on Pri 6 + */ + uint64_t rx_pfc_xon2xoff_frames_pri7; + /* + * Total number of received PFC frames with transition from XON + * to XOFF on Pri 7 + */ + uint64_t rx_pfc_ena_frames_pri0; + /* + * Total number of received PFC frames with PFC enabled bit for + * Pri 0 + */ + uint64_t rx_pfc_ena_frames_pri1; + /* + * Total number of received PFC frames with PFC enabled bit for + * Pri 1 + */ + uint64_t rx_pfc_ena_frames_pri2; + /* + * Total number of received PFC frames with PFC enabled bit for + * Pri 2 + */ + uint64_t rx_pfc_ena_frames_pri3; + /* + * Total number of received PFC frames with PFC enabled bit for + * Pri 3 + */ + uint64_t rx_pfc_ena_frames_pri4; + /* + * Total number of received PFC frames with PFC enabled bit for + * Pri 4 + */ + uint64_t rx_pfc_ena_frames_pri5; + /* + * Total number of received PFC frames with PFC enabled bit for + * Pri 5 + */ + uint64_t rx_pfc_ena_frames_pri6; + /* + * Total number of received PFC frames with PFC enabled bit for + * Pri 6 + */ + uint64_t rx_pfc_ena_frames_pri7; + /* + * Total number of received PFC frames with PFC enabled bit for + * Pri 7 + */ + uint64_t rx_sch_crc_err_frames; + /* Total Number of frames received with SCH CRC error */ + uint64_t rx_undrsz_frames; + /* Total Number of under-sized frames received */ + uint64_t rx_frag_frames; + /* Total Number of fragmented frames received */ + uint64_t rx_eee_lpi_events; + /* Total number of RX EEE LPI Events */ + uint64_t rx_eee_lpi_duration; + /* EEE LPI Duration Counter on RX */ + uint64_t rx_llfc_physical_msgs; + /* + * Total number of physical type Link Level Flow Control (LLFC) + * messages received + */ + uint64_t rx_llfc_logical_msgs; + /* + * Total number of logical type Link Level Flow Control (LLFC) + * messages received + */ + uint64_t rx_llfc_msgs_with_crc_err; + /* + * Total number of logical type Link Level Flow Control (LLFC) + * messages received with CRC error + */ + uint64_t rx_hcfc_msgs; + /* Total number of HCFC messages received */ + uint64_t rx_hcfc_msgs_with_crc_err; + /* Total number of HCFC messages received with CRC error */ + uint64_t rx_bytes; + /* Total number of received bytes */ + uint64_t rx_runt_bytes; + /* Total number of bytes received in runt frames */ + uint64_t rx_runt_frames; + /* Total number of runt frames received */ + uint64_t rx_stat_discard; + /* Total Rx Discards per Port reported by STATS block */ + uint64_t rx_stat_err; + /* Total Rx Error Drops per Port reported by STATS block */ +} __attribute__((packed)); + +/* Periodic Statistics Context DMA to host (160 bytes) */ +/* + * per-context HW statistics -- chip view + */ + +struct ctx_hw_stats64 { + uint64_t rx_ucast_pkts; + uint64_t rx_mcast_pkts; + uint64_t rx_bcast_pkts; + uint64_t rx_drop_pkts; + uint64_t rx_discard_pkts; + uint64_t rx_ucast_bytes; + uint64_t rx_mcast_bytes; + uint64_t rx_bcast_bytes; + + uint64_t tx_ucast_pkts; + uint64_t tx_mcast_pkts; + uint64_t tx_bcast_pkts; + uint64_t tx_drop_pkts; + uint64_t tx_discard_pkts; + uint64_t tx_ucast_bytes; + uint64_t tx_mcast_bytes; + uint64_t tx_bcast_bytes; + + uint64_t tpa_pkts; + uint64_t tpa_bytes; + uint64_t tpa_events; + uint64_t tpa_aborts; +} __attribute__((packed)); + +#endif /* _HSI_STRUCT_DEF_DPDK_ */ diff --git a/dpdk/drivers/net/bnxt/rte_pmd_bnxt.c b/dpdk/drivers/net/bnxt/rte_pmd_bnxt.c new file mode 100644 index 00000000..a3134074 --- /dev/null +++ b/dpdk/drivers/net/bnxt/rte_pmd_bnxt.c @@ -0,0 +1,839 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Broadcom Limited. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Broadcom Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "bnxt.h" +#include "bnxt_filter.h" +#include "bnxt_hwrm.h" +#include "bnxt_vnic.h" +#include "rte_pmd_bnxt.h" +#include "hsi_struct_def_dpdk.h" + +int bnxt_rcv_msg_from_vf(struct bnxt *bp, uint16_t vf_id, void *msg) +{ + struct rte_pmd_bnxt_mb_event_param ret_param; + + ret_param.retval = RTE_PMD_BNXT_MB_EVENT_PROCEED; + ret_param.vf_id = vf_id; + ret_param.msg = msg; + + _rte_eth_dev_callback_process(bp->eth_dev, RTE_ETH_EVENT_VF_MBOX, + NULL, &ret_param); + + /* Default to approve */ + if (ret_param.retval == RTE_PMD_BNXT_MB_EVENT_PROCEED) + ret_param.retval = RTE_PMD_BNXT_MB_EVENT_NOOP_ACK; + + return ret_param.retval == RTE_PMD_BNXT_MB_EVENT_NOOP_ACK ? + true : false; +} + +int rte_pmd_bnxt_set_tx_loopback(uint16_t port, uint8_t on) +{ + struct rte_eth_dev *eth_dev; + struct bnxt *bp; + int rc; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + if (on > 1) + return -EINVAL; + + eth_dev = &rte_eth_devices[port]; + if (!is_bnxt_supported(eth_dev)) + return -ENOTSUP; + + bp = (struct bnxt *)eth_dev->data->dev_private; + + if (!BNXT_PF(bp)) { + RTE_LOG(ERR, PMD, + "Attempt to set Tx loopback on non-PF port %d!\n", + port); + return -ENOTSUP; + } + + if (on) + bp->pf.evb_mode = BNXT_EVB_MODE_VEB; + else + bp->pf.evb_mode = BNXT_EVB_MODE_VEPA; + + rc = bnxt_hwrm_pf_evb_mode(bp); + + return rc; +} + +static void +rte_pmd_bnxt_set_all_queues_drop_en_cb(struct bnxt_vnic_info *vnic, void *onptr) +{ + uint8_t *on = onptr; + vnic->bd_stall = !(*on); +} + +int rte_pmd_bnxt_set_all_queues_drop_en(uint16_t port, uint8_t on) +{ + struct rte_eth_dev *eth_dev; + struct bnxt *bp; + uint32_t i; + int rc = -EINVAL; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + if (on > 1) + return -EINVAL; + + eth_dev = &rte_eth_devices[port]; + if (!is_bnxt_supported(eth_dev)) + return -ENOTSUP; + + bp = (struct bnxt *)eth_dev->data->dev_private; + + if (!BNXT_PF(bp)) { + RTE_LOG(ERR, PMD, + "Attempt to set all queues drop on non-PF port!\n"); + return -ENOTSUP; + } + + if (bp->vnic_info == NULL) + return -ENODEV; + + /* Stall PF */ + for (i = 0; i < bp->nr_vnics; i++) { + bp->vnic_info[i].bd_stall = !on; + rc = bnxt_hwrm_vnic_cfg(bp, &bp->vnic_info[i]); + if (rc) { + RTE_LOG(ERR, PMD, "Failed to update PF VNIC %d.\n", i); + return rc; + } + } + + /* Stall all active VFs */ + for (i = 0; i < bp->pf.active_vfs; i++) { + rc = bnxt_hwrm_func_vf_vnic_query_and_config(bp, i, + rte_pmd_bnxt_set_all_queues_drop_en_cb, &on, + bnxt_hwrm_vnic_cfg); + if (rc) { + RTE_LOG(ERR, PMD, "Failed to update VF VNIC %d.\n", i); + break; + } + } + + return rc; +} + +int rte_pmd_bnxt_set_vf_mac_addr(uint16_t port, uint16_t vf, + struct ether_addr *mac_addr) +{ + struct rte_eth_dev *dev; + struct rte_eth_dev_info dev_info; + struct bnxt *bp; + int rc; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + dev = &rte_eth_devices[port]; + if (!is_bnxt_supported(dev)) + return -ENOTSUP; + + rte_eth_dev_info_get(port, &dev_info); + bp = (struct bnxt *)dev->data->dev_private; + + if (vf >= dev_info.max_vfs || mac_addr == NULL) + return -EINVAL; + + if (!BNXT_PF(bp)) { + RTE_LOG(ERR, PMD, + "Attempt to set VF %d mac address on non-PF port %d!\n", + vf, port); + return -ENOTSUP; + } + + rc = bnxt_hwrm_func_vf_mac(bp, vf, (uint8_t *)mac_addr); + + return rc; +} + +int rte_pmd_bnxt_set_vf_rate_limit(uint16_t port, uint16_t vf, + uint16_t tx_rate, uint64_t q_msk) +{ + struct rte_eth_dev *eth_dev; + struct rte_eth_dev_info dev_info; + struct bnxt *bp; + uint16_t tot_rate = 0; + uint64_t idx; + int rc; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + eth_dev = &rte_eth_devices[port]; + if (!is_bnxt_supported(eth_dev)) + return -ENOTSUP; + + rte_eth_dev_info_get(port, &dev_info); + bp = (struct bnxt *)eth_dev->data->dev_private; + + if (!bp->pf.active_vfs) + return -EINVAL; + + if (vf >= bp->pf.max_vfs) + return -EINVAL; + + /* Add up the per queue BW and configure MAX BW of the VF */ + for (idx = 0; idx < 64; idx++) { + if ((1ULL << idx) & q_msk) + tot_rate += tx_rate; + } + + /* Requested BW can't be greater than link speed */ + if (tot_rate > eth_dev->data->dev_link.link_speed) { + RTE_LOG(ERR, PMD, "Rate > Link speed. Set to %d\n", tot_rate); + return -EINVAL; + } + + /* Requested BW already configured */ + if (tot_rate == bp->pf.vf_info[vf].max_tx_rate) + return 0; + + rc = bnxt_hwrm_func_bw_cfg(bp, vf, tot_rate, + HWRM_FUNC_CFG_INPUT_ENABLES_MAX_BW); + + if (!rc) + bp->pf.vf_info[vf].max_tx_rate = tot_rate; + + return rc; +} + +int rte_pmd_bnxt_set_vf_mac_anti_spoof(uint16_t port, uint16_t vf, uint8_t on) +{ + struct rte_eth_dev_info dev_info; + struct rte_eth_dev *dev; + uint32_t func_flags; + struct bnxt *bp; + int rc; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + if (on > 1) + return -EINVAL; + + dev = &rte_eth_devices[port]; + if (!is_bnxt_supported(dev)) + return -ENOTSUP; + + rte_eth_dev_info_get(port, &dev_info); + bp = (struct bnxt *)dev->data->dev_private; + + if (!BNXT_PF(bp)) { + RTE_LOG(ERR, PMD, + "Attempt to set mac spoof on non-PF port %d!\n", port); + return -EINVAL; + } + + if (vf >= dev_info.max_vfs) + return -EINVAL; + + /* Prev setting same as new setting. */ + if (on == bp->pf.vf_info[vf].mac_spoof_en) + return 0; + + func_flags = bp->pf.vf_info[vf].func_cfg_flags; + func_flags &= ~(HWRM_FUNC_CFG_INPUT_FLAGS_SRC_MAC_ADDR_CHECK_ENABLE | + HWRM_FUNC_CFG_INPUT_FLAGS_SRC_MAC_ADDR_CHECK_DISABLE); + + if (on) + func_flags |= + HWRM_FUNC_CFG_INPUT_FLAGS_SRC_MAC_ADDR_CHECK_ENABLE; + else + func_flags |= + HWRM_FUNC_CFG_INPUT_FLAGS_SRC_MAC_ADDR_CHECK_DISABLE; + + rc = bnxt_hwrm_func_cfg_vf_set_flags(bp, vf, func_flags); + if (!rc) { + bp->pf.vf_info[vf].mac_spoof_en = on; + bp->pf.vf_info[vf].func_cfg_flags = func_flags; + } + + return rc; +} + +int rte_pmd_bnxt_set_vf_vlan_anti_spoof(uint16_t port, uint16_t vf, uint8_t on) +{ + struct rte_eth_dev_info dev_info; + struct rte_eth_dev *dev; + struct bnxt *bp; + int rc; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + if (on > 1) + return -EINVAL; + + dev = &rte_eth_devices[port]; + if (!is_bnxt_supported(dev)) + return -ENOTSUP; + + rte_eth_dev_info_get(port, &dev_info); + bp = (struct bnxt *)dev->data->dev_private; + + if (!BNXT_PF(bp)) { + RTE_LOG(ERR, PMD, + "Attempt to set VLAN spoof on non-PF port %d!\n", port); + return -EINVAL; + } + + if (vf >= dev_info.max_vfs) + return -EINVAL; + + rc = bnxt_hwrm_func_cfg_vf_set_vlan_anti_spoof(bp, vf, on); + if (!rc) { + bp->pf.vf_info[vf].vlan_spoof_en = on; + if (on) { + if (bnxt_hwrm_cfa_vlan_antispoof_cfg(bp, + bp->pf.first_vf_id + vf, + bp->pf.vf_info[vf].vlan_count, + bp->pf.vf_info[vf].vlan_as_table)) + rc = -1; + } + } else { + RTE_LOG(ERR, PMD, "Failed to update VF VNIC %d.\n", vf); + } + + return rc; +} + +static void +rte_pmd_bnxt_set_vf_vlan_stripq_cb(struct bnxt_vnic_info *vnic, void *onptr) +{ + uint8_t *on = onptr; + vnic->vlan_strip = *on; +} + +int +rte_pmd_bnxt_set_vf_vlan_stripq(uint16_t port, uint16_t vf, uint8_t on) +{ + struct rte_eth_dev *dev; + struct rte_eth_dev_info dev_info; + struct bnxt *bp; + int rc; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + dev = &rte_eth_devices[port]; + if (!is_bnxt_supported(dev)) + return -ENOTSUP; + + rte_eth_dev_info_get(port, &dev_info); + bp = (struct bnxt *)dev->data->dev_private; + + if (vf >= dev_info.max_vfs) + return -EINVAL; + + if (!BNXT_PF(bp)) { + RTE_LOG(ERR, PMD, + "Attempt to set VF %d stripq on non-PF port %d!\n", + vf, port); + return -ENOTSUP; + } + + rc = bnxt_hwrm_func_vf_vnic_query_and_config(bp, vf, + rte_pmd_bnxt_set_vf_vlan_stripq_cb, &on, + bnxt_hwrm_vnic_cfg); + if (rc) + RTE_LOG(ERR, PMD, "Failed to update VF VNIC %d.\n", vf); + + return rc; +} + +int rte_pmd_bnxt_set_vf_rxmode(uint16_t port, uint16_t vf, + uint16_t rx_mask, uint8_t on) +{ + struct rte_eth_dev *dev; + struct rte_eth_dev_info dev_info; + uint16_t flag = 0; + struct bnxt *bp; + int rc; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + dev = &rte_eth_devices[port]; + if (!is_bnxt_supported(dev)) + return -ENOTSUP; + + rte_eth_dev_info_get(port, &dev_info); + bp = (struct bnxt *)dev->data->dev_private; + + if (!bp->pf.vf_info) + return -EINVAL; + + if (vf >= bp->pdev->max_vfs) + return -EINVAL; + + if (rx_mask & ETH_VMDQ_ACCEPT_UNTAG) { + RTE_LOG(ERR, PMD, "Currently cannot toggle this setting\n"); + return -ENOTSUP; + } + + /* Is this really the correct mapping? VFd seems to think it is. */ + if (rx_mask & ETH_VMDQ_ACCEPT_HASH_UC) + flag |= BNXT_VNIC_INFO_PROMISC; + + if (rx_mask & ETH_VMDQ_ACCEPT_BROADCAST) + flag |= BNXT_VNIC_INFO_BCAST; + if (rx_mask & ETH_VMDQ_ACCEPT_MULTICAST) + flag |= BNXT_VNIC_INFO_ALLMULTI | BNXT_VNIC_INFO_MCAST; + + if (on) + bp->pf.vf_info[vf].l2_rx_mask |= flag; + else + bp->pf.vf_info[vf].l2_rx_mask &= ~flag; + + rc = bnxt_hwrm_func_vf_vnic_query_and_config(bp, vf, + vf_vnic_set_rxmask_cb, + &bp->pf.vf_info[vf].l2_rx_mask, + bnxt_set_rx_mask_no_vlan); + if (rc) + RTE_LOG(ERR, PMD, "bnxt_hwrm_func_vf_vnic_set_rxmask failed\n"); + + return rc; +} + +static int bnxt_set_vf_table(struct bnxt *bp, uint16_t vf) +{ + int rc = 0; + int dflt_vnic; + struct bnxt_vnic_info vnic; + + if (!BNXT_PF(bp)) { + RTE_LOG(ERR, PMD, + "Attempt to set VLAN table on non-PF port!\n"); + return -EINVAL; + } + + if (vf >= bp->pdev->max_vfs) + return -EINVAL; + + dflt_vnic = bnxt_hwrm_func_qcfg_vf_dflt_vnic_id(bp, vf); + if (dflt_vnic < 0) { + /* This simply indicates there's no driver loaded. + * This is not an error. + */ + RTE_LOG(ERR, PMD, "Unable to get default VNIC for VF %d\n", vf); + } else { + memset(&vnic, 0, sizeof(vnic)); + vnic.fw_vnic_id = dflt_vnic; + if (bnxt_hwrm_vnic_qcfg(bp, &vnic, + bp->pf.first_vf_id + vf) == 0) { + if (bnxt_hwrm_cfa_l2_set_rx_mask(bp, &vnic, + bp->pf.vf_info[vf].vlan_count, + bp->pf.vf_info[vf].vlan_table)) + rc = -1; + } else { + rc = -1; + } + } + + return rc; +} + +int rte_pmd_bnxt_set_vf_vlan_filter(uint16_t port, uint16_t vlan, + uint64_t vf_mask, uint8_t vlan_on) +{ + struct bnxt_vlan_table_entry *ve; + struct bnxt_vlan_antispoof_table_entry *vase; + struct rte_eth_dev *dev; + struct bnxt *bp; + uint16_t cnt; + int rc = 0; + int i, j; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + dev = &rte_eth_devices[port]; + if (!is_bnxt_supported(dev)) + return -ENOTSUP; + + bp = (struct bnxt *)dev->data->dev_private; + if (!bp->pf.vf_info) + return -EINVAL; + + for (i = 0; vf_mask; i++, vf_mask >>= 1) { + cnt = bp->pf.vf_info[i].vlan_count; + if ((vf_mask & 1) == 0) + continue; + + if (bp->pf.vf_info[i].vlan_table == NULL) { + rc = -1; + continue; + } + if (bp->pf.vf_info[i].vlan_as_table == NULL) { + rc = -1; + continue; + } + if (vlan_on) { + /* First, search for a duplicate... */ + for (j = 0; j < cnt; j++) { + if (rte_be_to_cpu_16( + bp->pf.vf_info[i].vlan_table[j].vid) == vlan) + break; + } + if (j == cnt) { + /* Now check that there's space */ + if (cnt == getpagesize() / sizeof(struct + bnxt_vlan_antispoof_table_entry)) { + RTE_LOG(ERR, PMD, + "VLAN anti-spoof table is full\n"); + RTE_LOG(ERR, PMD, + "VF %d cannot add VLAN %u\n", + i, vlan); + rc = -1; + continue; + } + + /* cnt is one less than vlan_count */ + cnt = bp->pf.vf_info[i].vlan_count++; + /* + * And finally, add to the + * end of the table + */ + vase = &bp->pf.vf_info[i].vlan_as_table[cnt]; + // TODO: Hardcoded TPID + vase->tpid = rte_cpu_to_be_16(0x8100); + vase->vid = rte_cpu_to_be_16(vlan); + vase->mask = rte_cpu_to_be_16(0xfff); + ve = &bp->pf.vf_info[i].vlan_table[cnt]; + /* TODO: Hardcoded TPID */ + ve->tpid = rte_cpu_to_be_16(0x8100); + ve->vid = rte_cpu_to_be_16(vlan); + } + } else { + for (j = 0; j < cnt; j++) { + if (rte_be_to_cpu_16( + bp->pf.vf_info[i].vlan_table[j].vid) != vlan) + continue; + memmove(&bp->pf.vf_info[i].vlan_table[j], + &bp->pf.vf_info[i].vlan_table[j + 1], + getpagesize() - ((j + 1) * + sizeof(struct bnxt_vlan_table_entry))); + memmove(&bp->pf.vf_info[i].vlan_as_table[j], + &bp->pf.vf_info[i].vlan_as_table[j + 1], + getpagesize() - ((j + 1) * sizeof(struct + bnxt_vlan_antispoof_table_entry))); + j--; + cnt = --bp->pf.vf_info[i].vlan_count; + } + } + bnxt_set_vf_table(bp, i); + } + + return rc; +} + +int rte_pmd_bnxt_get_vf_stats(uint16_t port, + uint16_t vf_id, + struct rte_eth_stats *stats) +{ + struct rte_eth_dev *dev; + struct rte_eth_dev_info dev_info; + struct bnxt *bp; + + dev = &rte_eth_devices[port]; + if (!is_bnxt_supported(dev)) + return -ENOTSUP; + + rte_eth_dev_info_get(port, &dev_info); + bp = (struct bnxt *)dev->data->dev_private; + + if (vf_id >= dev_info.max_vfs) + return -EINVAL; + + if (!BNXT_PF(bp)) { + RTE_LOG(ERR, PMD, + "Attempt to get VF %d stats on non-PF port %d!\n", + vf_id, port); + return -ENOTSUP; + } + + return bnxt_hwrm_func_qstats(bp, bp->pf.first_vf_id + vf_id, stats); +} + +int rte_pmd_bnxt_reset_vf_stats(uint16_t port, + uint16_t vf_id) +{ + struct rte_eth_dev *dev; + struct rte_eth_dev_info dev_info; + struct bnxt *bp; + + dev = &rte_eth_devices[port]; + if (!is_bnxt_supported(dev)) + return -ENOTSUP; + + rte_eth_dev_info_get(port, &dev_info); + bp = (struct bnxt *)dev->data->dev_private; + + if (vf_id >= dev_info.max_vfs) + return -EINVAL; + + if (!BNXT_PF(bp)) { + RTE_LOG(ERR, PMD, + "Attempt to reset VF %d stats on non-PF port %d!\n", + vf_id, port); + return -ENOTSUP; + } + + return bnxt_hwrm_func_clr_stats(bp, bp->pf.first_vf_id + vf_id); +} + +int rte_pmd_bnxt_get_vf_rx_status(uint16_t port, uint16_t vf_id) +{ + struct rte_eth_dev *dev; + struct rte_eth_dev_info dev_info; + struct bnxt *bp; + + dev = &rte_eth_devices[port]; + if (!is_bnxt_supported(dev)) + return -ENOTSUP; + + rte_eth_dev_info_get(port, &dev_info); + bp = (struct bnxt *)dev->data->dev_private; + + if (vf_id >= dev_info.max_vfs) + return -EINVAL; + + if (!BNXT_PF(bp)) { + RTE_LOG(ERR, PMD, + "Attempt to query VF %d RX stats on non-PF port %d!\n", + vf_id, port); + return -ENOTSUP; + } + + return bnxt_vf_vnic_count(bp, vf_id); +} + +int rte_pmd_bnxt_get_vf_tx_drop_count(uint16_t port, uint16_t vf_id, + uint64_t *count) +{ + struct rte_eth_dev *dev; + struct rte_eth_dev_info dev_info; + struct bnxt *bp; + + dev = &rte_eth_devices[port]; + if (!is_bnxt_supported(dev)) + return -ENOTSUP; + + rte_eth_dev_info_get(port, &dev_info); + bp = (struct bnxt *)dev->data->dev_private; + + if (vf_id >= dev_info.max_vfs) + return -EINVAL; + + if (!BNXT_PF(bp)) { + RTE_LOG(ERR, PMD, + "Attempt to query VF %d TX drops on non-PF port %d!\n", + vf_id, port); + return -ENOTSUP; + } + + return bnxt_hwrm_func_qstats_tx_drop(bp, bp->pf.first_vf_id + vf_id, + count); +} + +int rte_pmd_bnxt_mac_addr_add(uint16_t port, struct ether_addr *addr, + uint32_t vf_id) +{ + struct rte_eth_dev *dev; + struct rte_eth_dev_info dev_info; + struct bnxt *bp; + struct bnxt_filter_info *filter; + struct bnxt_vnic_info vnic; + struct ether_addr dflt_mac; + int rc; + + dev = &rte_eth_devices[port]; + if (!is_bnxt_supported(dev)) + return -ENOTSUP; + + rte_eth_dev_info_get(port, &dev_info); + bp = (struct bnxt *)dev->data->dev_private; + + if (vf_id >= dev_info.max_vfs) + return -EINVAL; + + if (!BNXT_PF(bp)) { + RTE_LOG(ERR, PMD, + "Attempt to config VF %d MAC on non-PF port %d!\n", + vf_id, port); + return -ENOTSUP; + } + + /* If the VF currently uses a random MAC, update default to this one */ + if (bp->pf.vf_info[vf_id].random_mac) { + if (rte_pmd_bnxt_get_vf_rx_status(port, vf_id) <= 0) + bnxt_hwrm_func_vf_mac(bp, vf_id, (uint8_t *)addr); + } + + /* query the default VNIC id used by the function */ + rc = bnxt_hwrm_func_qcfg_vf_dflt_vnic_id(bp, vf_id); + if (rc < 0) + goto exit; + + memset(&vnic, 0, sizeof(struct bnxt_vnic_info)); + vnic.fw_vnic_id = rte_le_to_cpu_16(rc); + rc = bnxt_hwrm_vnic_qcfg(bp, &vnic, bp->pf.first_vf_id + vf_id); + if (rc < 0) + goto exit; + + STAILQ_FOREACH(filter, &bp->pf.vf_info[vf_id].filter, next) { + if (filter->flags == + HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_PATH_RX && + filter->enables == + (HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_ADDR | + HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_ADDR_MASK) && + memcmp(addr, filter->l2_addr, ETHER_ADDR_LEN) == 0) { + bnxt_hwrm_clear_l2_filter(bp, filter); + break; + } + } + + if (filter == NULL) + filter = bnxt_alloc_vf_filter(bp, vf_id); + + filter->fw_l2_filter_id = UINT64_MAX; + filter->flags = HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_PATH_RX; + filter->enables = HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_ADDR | + HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_ADDR_MASK; + memcpy(filter->l2_addr, addr, ETHER_ADDR_LEN); + memset(filter->l2_addr_mask, 0xff, ETHER_ADDR_LEN); + + /* Do not add a filter for the default MAC */ + if (bnxt_hwrm_func_qcfg_vf_default_mac(bp, vf_id, &dflt_mac) || + memcmp(filter->l2_addr, dflt_mac.addr_bytes, ETHER_ADDR_LEN)) + rc = bnxt_hwrm_set_l2_filter(bp, vnic.fw_vnic_id, filter); + +exit: + return rc; +} + +int +rte_pmd_bnxt_set_vf_vlan_insert(uint16_t port, uint16_t vf, + uint16_t vlan_id) +{ + struct rte_eth_dev *dev; + struct rte_eth_dev_info dev_info; + struct bnxt *bp; + int rc; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + dev = &rte_eth_devices[port]; + if (!is_bnxt_supported(dev)) + return -ENOTSUP; + + rte_eth_dev_info_get(port, &dev_info); + bp = (struct bnxt *)dev->data->dev_private; + + if (vf >= dev_info.max_vfs) + return -EINVAL; + + if (!BNXT_PF(bp)) { + RTE_LOG(ERR, PMD, + "Attempt to set VF %d vlan insert on non-PF port %d!\n", + vf, port); + return -ENOTSUP; + } + + bp->pf.vf_info[vf].dflt_vlan = vlan_id; + if (bnxt_hwrm_func_qcfg_current_vf_vlan(bp, vf) == + bp->pf.vf_info[vf].dflt_vlan) + return 0; + + rc = bnxt_hwrm_set_vf_vlan(bp, vf); + + return rc; +} + +int rte_pmd_bnxt_set_vf_persist_stats(uint16_t port, uint16_t vf, uint8_t on) +{ + struct rte_eth_dev_info dev_info; + struct rte_eth_dev *dev; + uint32_t func_flags; + struct bnxt *bp; + int rc; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + if (on > 1) + return -EINVAL; + + dev = &rte_eth_devices[port]; + rte_eth_dev_info_get(port, &dev_info); + bp = (struct bnxt *)dev->data->dev_private; + + if (!BNXT_PF(bp)) { + RTE_LOG(ERR, PMD, + "Attempt to set persist stats on non-PF port %d!\n", + port); + return -EINVAL; + } + + if (vf >= dev_info.max_vfs) + return -EINVAL; + + /* Prev setting same as new setting. */ + if (on == bp->pf.vf_info[vf].persist_stats) + return 0; + + func_flags = bp->pf.vf_info[vf].func_cfg_flags; + + if (on) + func_flags |= + HWRM_FUNC_CFG_INPUT_FLAGS_NO_AUTOCLEAR_STATISTIC; + else + func_flags &= + ~HWRM_FUNC_CFG_INPUT_FLAGS_NO_AUTOCLEAR_STATISTIC; + + rc = bnxt_hwrm_func_cfg_vf_set_flags(bp, vf, func_flags); + if (!rc) { + bp->pf.vf_info[vf].persist_stats = on; + bp->pf.vf_info[vf].func_cfg_flags = func_flags; + } + + return rc; +} diff --git a/dpdk/drivers/net/bnxt/rte_pmd_bnxt.h b/dpdk/drivers/net/bnxt/rte_pmd_bnxt.h new file mode 100644 index 00000000..f881d30d --- /dev/null +++ b/dpdk/drivers/net/bnxt/rte_pmd_bnxt.h @@ -0,0 +1,354 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Broadcom Limited. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Broadcom Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _PMD_BNXT_H_ +#define _PMD_BNXT_H_ + +#include + +/* + * Response sent back to the caller after callback + */ +enum rte_pmd_bnxt_mb_event_rsp { + RTE_PMD_BNXT_MB_EVENT_NOOP_ACK, /**< skip mbox request and ACK */ + RTE_PMD_BNXT_MB_EVENT_NOOP_NACK, /**< skip mbox request and NACK */ + RTE_PMD_BNXT_MB_EVENT_PROCEED, /**< proceed with mbox request */ + RTE_PMD_BNXT_MB_EVENT_MAX /**< max value of this enum */ +}; + +/* mailbox message types */ +#define BNXT_VF_RESET 0x01 /* VF requests reset */ +#define BNXT_VF_SET_MAC_ADDR 0x02 /* VF requests PF to set MAC addr */ +#define BNXT_VF_SET_VLAN 0x03 /* VF requests PF to set VLAN */ +#define BNXT_VF_SET_MTU 0x04 /* VF requests PF to set MTU */ +#define BNXT_VF_SET_MRU 0x05 /* VF requests PF to set MRU */ + +/* + * Data sent to the caller when the callback is executed. + */ +struct rte_pmd_bnxt_mb_event_param { + uint16_t vf_id; /* Virtual Function number */ + int retval; /* return value */ + void *msg; /* pointer to message */ +}; + +/** + * Enable/Disable VF MAC anti spoof + * + * @param port + * The port identifier of the Ethernet device. + * @param vf + * VF id. + * @param on + * 1 - Enable VF MAC anti spoof. + * 0 - Disable VF MAC anti spoof. + * + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-EINVAL) if bad parameter. + */ +int rte_pmd_bnxt_set_vf_mac_anti_spoof(uint16_t port, uint16_t vf, uint8_t on); + +/** + * Set the VF MAC address. + * + * @param port + * The port identifier of the Ethernet device. + * @param vf + * VF id. + * @param mac_addr + * VF MAC address. + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-EINVAL) if *vf* or *mac_addr* is invalid. + */ +int rte_pmd_bnxt_set_vf_mac_addr(uint16_t port, uint16_t vf, + struct ether_addr *mac_addr); + +/** + * Enable/Disable vf vlan strip for all queues in a pool + * + * @param port + * The port identifier of the Ethernet device. + * @param vf + * ID specifying VF. + * @param on + * 1 - Enable VF's vlan strip on RX queues. + * 0 - Disable VF's vlan strip on RX queues. + * + * @return + * - (0) if successful. + * - (-ENOTSUP) if hardware doesn't support this feature. + * - (-ENODEV) if *port* invalid. + * - (-EINVAL) if bad parameter. + */ +int +rte_pmd_bnxt_set_vf_vlan_stripq(uint16_t port, uint16_t vf, uint8_t on); + +/** + * Enable/Disable vf vlan insert + * + * @param port + * The port identifier of the Ethernet device. + * @param vf + * ID specifying VF. + * @param vlan_id + * 0 - Disable VF's vlan insert. + * n - Enable; n is inserted as the vlan id. + * + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-EINVAL) if bad parameter. + */ +int +rte_pmd_bnxt_set_vf_vlan_insert(uint16_t port, uint16_t vf, + uint16_t vlan_id); + +/** + * Enable/Disable hardware VF VLAN filtering by an Ethernet device of + * received VLAN packets tagged with a given VLAN Tag Identifier. + * + * @param port + * The port identifier of the Ethernet device. + * @param vlan + * The VLAN Tag Identifier whose filtering must be enabled or disabled. + * @param vf_mask + * Bitmap listing which VFs participate in the VLAN filtering. + * @param vlan_on + * 1 - Enable VFs VLAN filtering. + * 0 - Disable VFs VLAN filtering. + * @return + * - (0) if successful. + * - (-ENOTSUP) if hardware doesn't support. + * - (-ENODEV) if *port_id* invalid. + * - (-EINVAL) if bad parameter. + */ +int rte_pmd_bnxt_set_vf_vlan_filter(uint16_t port, uint16_t vlan, + uint64_t vf_mask, uint8_t vlan_on); + +/** + * Enable/Disable tx loopback + * + * @param port + * The port identifier of the Ethernet device. + * @param on + * 1 - Enable tx loopback. + * 0 - Disable tx loopback. + * + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-EINVAL) if bad parameter. + */ +int rte_pmd_bnxt_set_tx_loopback(uint16_t port, uint8_t on); + +/** + * set all queues drop enable bit + * + * @param port + * The port identifier of the Ethernet device. + * @param on + * 1 - set the queue drop enable bit for all pools. + * 0 - reset the queue drop enable bit for all pools. + * + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-EINVAL) if bad parameter. + */ +int rte_pmd_bnxt_set_all_queues_drop_en(uint16_t port, uint8_t on); + +/** + * Set the VF rate limit. + * + * @param port + * The port identifier of the Ethernet device. + * @param vf + * VF id. + * @param tx_rate + * Tx rate for the VF + * @param q_msk + * Mask of the Tx queue + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-EINVAL) if *vf* or *mac_addr* is invalid. + */ +int rte_pmd_bnxt_set_vf_rate_limit(uint16_t port, uint16_t vf, + uint16_t tx_rate, uint64_t q_msk); + +/** + * Get VF's statistics + * + * @param port + * The port identifier of the Ethernet device. + * @param vf_id + * VF on which to get. + * @param stats + * A pointer to a structure of type *rte_eth_stats* to be filled with + * the values of device counters supported statistics: + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-EINVAL) if bad parameter. + */ + +int rte_pmd_bnxt_get_vf_stats(uint16_t port, + uint16_t vf_id, + struct rte_eth_stats *stats); + +/** + * Clear VF's statistics + * + * @param port + * The port identifier of the Ethernet device. + * @param vf_id + * VF on which to get. + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-EINVAL) if bad parameter. + */ +int rte_pmd_bnxt_reset_vf_stats(uint16_t port, + uint16_t vf_id); + +/** + * Enable/Disable VF VLAN anti spoof + * + * @param port + * The port identifier of the Ethernet device. + * @param vf + * VF id. + * @param on + * 1 - Enable VF VLAN anti spoof. + * 0 - Disable VF VLAN anti spoof. + * + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-EINVAL) if bad parameter. + */ +int rte_pmd_bnxt_set_vf_vlan_anti_spoof(uint16_t port, uint16_t vf, uint8_t on); + +/** + * Set RX L2 Filtering mode of a VF of an Ethernet device. + * + * @param port + * The port identifier of the Ethernet device. + * @param vf + * VF id. + * @param rx_mask + * The RX mode mask + * @param on + * 1 - Enable a VF RX mode. + * 0 - Disable a VF RX mode. + * @return + * - (0) if successful. + * - (-ENODEV) if *port_id* invalid. + * - (-EINVAL) if bad parameter. + */ +int rte_pmd_bnxt_set_vf_rxmode(uint16_t port, uint16_t vf, + uint16_t rx_mask, uint8_t on); + +/** + * Returns the number of default RX queues on a VF + * + * @param port + * The port identifier of the Ethernet device. + * @param vf + * VF id. + * @return + * - Non-negative value - Number of default RX queues + * - (-EINVAL) if bad parameter. + * - (-ENOTSUP) if on a function without VFs + * - (-ENOMEM) on an allocation failure + * - (-1) firmware interface error + */ +int rte_pmd_bnxt_get_vf_rx_status(uint16_t port, uint16_t vf_id); + +/** + * Queries the TX drop counter for the function + * + * @param port + * The port identifier of the Ethernet device. + * @param vf_id + * VF on which to get. + * @param count + * Pointer to a uint64_t that will be populated with the counter value. + * @return + * - Positive Non-zero value - Error code from HWRM + * - (-EINVAL) invalid vf_id specified. + * - (-ENOTSUP) Ethernet device is not a PF + */ +int rte_pmd_bnxt_get_vf_tx_drop_count(uint16_t port, uint16_t vf_id, + uint64_t *count); + +/** + * Programs the MAC address for the function specified + * + * @param port + * The port identifier of the Ethernet device. + * @param mac_addr + * The MAC address to be programmed in the filter. + * @param vf_id + * VF on which to get. + * @return + * - Positive Non-zero value - Error code from HWRM + * - (-EINVAL) invalid vf_id specified. + * - (-ENOTSUP) Ethernet device is not a PF + * - (-ENOMEM) on an allocation failure + */ +int rte_pmd_bnxt_mac_addr_add(uint16_t port, struct ether_addr *mac_addr, + uint32_t vf_id); + +/** + * Enable/Disable VF statistics retention + * + * @param port + * The port identifier of the Ethernet device. + * @param vf + * VF id. + * @param on + * 1 - Prevent VF statistics from automatically resetting + * 0 - Allow VF statistics to automatically reset + * + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-EINVAL) if bad parameter. + */ +int rte_pmd_bnxt_set_vf_persist_stats(uint16_t port, uint16_t vf, uint8_t on); +#endif /* _PMD_BNXT_H_ */ diff --git a/dpdk/drivers/net/bnxt/rte_pmd_bnxt_version.map b/dpdk/drivers/net/bnxt/rte_pmd_bnxt_version.map index 349c6e1c..4750d40a 100644 --- a/dpdk/drivers/net/bnxt/rte_pmd_bnxt_version.map +++ b/dpdk/drivers/net/bnxt/rte_pmd_bnxt_version.map @@ -1,4 +1,22 @@ -DPDK_16.04 { +DPDK_17.08 { + global: + + rte_pmd_bnxt_get_vf_rx_status; + rte_pmd_bnxt_get_vf_stats; + rte_pmd_bnxt_get_vf_tx_drop_count; + rte_pmd_bnxt_mac_addr_add; + rte_pmd_bnxt_reset_vf_stats; + rte_pmd_bnxt_set_all_queues_drop_en; + rte_pmd_bnxt_set_tx_loopback; + rte_pmd_bnxt_set_vf_mac_addr; + rte_pmd_bnxt_set_vf_mac_anti_spoof; + rte_pmd_bnxt_set_vf_rate_limit; + rte_pmd_bnxt_set_vf_rxmode; + rte_pmd_bnxt_set_vf_vlan_anti_spoof; + rte_pmd_bnxt_set_vf_vlan_filter; + rte_pmd_bnxt_set_vf_vlan_insert; + rte_pmd_bnxt_set_vf_vlan_stripq; + rte_pmd_bnxt_set_vf_persist_stats; local: *; }; diff --git a/dpdk/drivers/net/bonding/Makefile b/dpdk/drivers/net/bonding/Makefile index 504f2e8b..dea1bd5c 100644 --- a/dpdk/drivers/net/bonding/Makefile +++ b/dpdk/drivers/net/bonding/Makefile @@ -38,10 +38,14 @@ LIB = librte_pmd_bond.a CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs -lrte_cmdline +LDLIBS += -lrte_pci -lrte_bus_pci +LDLIBS += -lrte_bus_vdev -EXPORT_MAP := rte_eth_bond_version.map +EXPORT_MAP := rte_pmd_bond_version.map -LIBABIVER := 1 +LIBABIVER := 2 # # all source are stored in SRCS-y @@ -58,13 +62,4 @@ SRCS-$(CONFIG_RTE_LIBRTE_PMD_BOND) += rte_eth_bond_alb.c SYMLINK-y-include += rte_eth_bond.h SYMLINK-y-include += rte_eth_bond_8023ad.h -# this lib depends upon: -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_BOND) += lib/librte_mbuf -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_BOND) += lib/librte_ether -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_BOND) += lib/librte_eal -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_BOND) += lib/librte_kvargs -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_BOND) += lib/librte_cmdline -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_BOND) += lib/librte_mempool -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_BOND) += lib/librte_ring - include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/net/bonding/rte_eth_bond.h b/dpdk/drivers/net/bonding/rte_eth_bond.h index 8efbf071..87ff2917 100644 --- a/dpdk/drivers/net/bonding/rte_eth_bond.h +++ b/dpdk/drivers/net/bonding/rte_eth_bond.h @@ -151,7 +151,7 @@ rte_eth_bond_free(const char *name); * 0 on success, negative value otherwise */ int -rte_eth_bond_slave_add(uint8_t bonded_port_id, uint8_t slave_port_id); +rte_eth_bond_slave_add(uint16_t bonded_port_id, uint16_t slave_port_id); /** * Remove a slave rte_eth_dev device from the bonded device @@ -163,7 +163,7 @@ rte_eth_bond_slave_add(uint8_t bonded_port_id, uint8_t slave_port_id); * 0 on success, negative value otherwise */ int -rte_eth_bond_slave_remove(uint8_t bonded_port_id, uint8_t slave_port_id); +rte_eth_bond_slave_remove(uint16_t bonded_port_id, uint16_t slave_port_id); /** * Set link bonding mode of bonded device @@ -175,7 +175,7 @@ rte_eth_bond_slave_remove(uint8_t bonded_port_id, uint8_t slave_port_id); * 0 on success, negative value otherwise */ int -rte_eth_bond_mode_set(uint8_t bonded_port_id, uint8_t mode); +rte_eth_bond_mode_set(uint16_t bonded_port_id, uint8_t mode); /** * Get link bonding mode of bonded device @@ -186,7 +186,7 @@ rte_eth_bond_mode_set(uint8_t bonded_port_id, uint8_t mode); * link bonding mode on success, negative value otherwise */ int -rte_eth_bond_mode_get(uint8_t bonded_port_id); +rte_eth_bond_mode_get(uint16_t bonded_port_id); /** * Set slave rte_eth_dev as primary slave of bonded device @@ -198,7 +198,7 @@ rte_eth_bond_mode_get(uint8_t bonded_port_id); * 0 on success, negative value otherwise */ int -rte_eth_bond_primary_set(uint8_t bonded_port_id, uint8_t slave_port_id); +rte_eth_bond_primary_set(uint16_t bonded_port_id, uint16_t slave_port_id); /** * Get primary slave of bonded device @@ -209,7 +209,7 @@ rte_eth_bond_primary_set(uint8_t bonded_port_id, uint8_t slave_port_id); * Port Id of primary slave on success, -1 on failure */ int -rte_eth_bond_primary_get(uint8_t bonded_port_id); +rte_eth_bond_primary_get(uint16_t bonded_port_id); /** * Populate an array with list of the slaves port id's of the bonded device @@ -223,7 +223,8 @@ rte_eth_bond_primary_get(uint8_t bonded_port_id); * negative value otherwise */ int -rte_eth_bond_slaves_get(uint8_t bonded_port_id, uint8_t slaves[], uint8_t len); +rte_eth_bond_slaves_get(uint16_t bonded_port_id, uint16_t slaves[], + uint16_t len); /** * Populate an array with list of the active slaves port id's of the bonded @@ -238,8 +239,8 @@ rte_eth_bond_slaves_get(uint8_t bonded_port_id, uint8_t slaves[], uint8_t len); * negative value otherwise */ int -rte_eth_bond_active_slaves_get(uint8_t bonded_port_id, uint8_t slaves[], - uint8_t len); +rte_eth_bond_active_slaves_get(uint16_t bonded_port_id, uint16_t slaves[], + uint16_t len); /** * Set explicit MAC address to use on bonded device and it's slaves. @@ -252,7 +253,7 @@ rte_eth_bond_active_slaves_get(uint8_t bonded_port_id, uint8_t slaves[], * 0 on success, negative value otherwise */ int -rte_eth_bond_mac_address_set(uint8_t bonded_port_id, +rte_eth_bond_mac_address_set(uint16_t bonded_port_id, struct ether_addr *mac_addr); /** @@ -265,7 +266,7 @@ rte_eth_bond_mac_address_set(uint8_t bonded_port_id, * 0 on success, negative value otherwise */ int -rte_eth_bond_mac_address_reset(uint8_t bonded_port_id); +rte_eth_bond_mac_address_reset(uint16_t bonded_port_id); /** * Set the transmit policy for bonded device to use when it is operating in @@ -279,7 +280,7 @@ rte_eth_bond_mac_address_reset(uint8_t bonded_port_id); * 0 on success, negative value otherwise. */ int -rte_eth_bond_xmit_policy_set(uint8_t bonded_port_id, uint8_t policy); +rte_eth_bond_xmit_policy_set(uint16_t bonded_port_id, uint8_t policy); /** * Get the transmit policy set on bonded device for balance mode operation @@ -290,7 +291,7 @@ rte_eth_bond_xmit_policy_set(uint8_t bonded_port_id, uint8_t policy); * Balance transmit policy on success, negative value otherwise. */ int -rte_eth_bond_xmit_policy_get(uint8_t bonded_port_id); +rte_eth_bond_xmit_policy_get(uint16_t bonded_port_id); /** * Set the link monitoring frequency (in ms) for monitoring the link status of @@ -304,7 +305,7 @@ rte_eth_bond_xmit_policy_get(uint8_t bonded_port_id); */ int -rte_eth_bond_link_monitoring_set(uint8_t bonded_port_id, uint32_t internal_ms); +rte_eth_bond_link_monitoring_set(uint16_t bonded_port_id, uint32_t internal_ms); /** * Get the current link monitoring frequency (in ms) for monitoring of the link @@ -316,7 +317,7 @@ rte_eth_bond_link_monitoring_set(uint8_t bonded_port_id, uint32_t internal_ms); * Monitoring interval on success, negative value otherwise. */ int -rte_eth_bond_link_monitoring_get(uint8_t bonded_port_id); +rte_eth_bond_link_monitoring_get(uint16_t bonded_port_id); /** @@ -330,7 +331,8 @@ rte_eth_bond_link_monitoring_get(uint8_t bonded_port_id); * 0 on success, negative value otherwise. */ int -rte_eth_bond_link_down_prop_delay_set(uint8_t bonded_port_id, uint32_t delay_ms); +rte_eth_bond_link_down_prop_delay_set(uint16_t bonded_port_id, + uint32_t delay_ms); /** * Get the period in milliseconds set for delaying the disabling of a bonded @@ -342,7 +344,7 @@ rte_eth_bond_link_down_prop_delay_set(uint8_t bonded_port_id, uint32_t delay_ms) * Delay period on success, negative value otherwise. */ int -rte_eth_bond_link_down_prop_delay_get(uint8_t bonded_port_id); +rte_eth_bond_link_down_prop_delay_get(uint16_t bonded_port_id); /** * Set the period in milliseconds for delaying the enabling of a bonded link @@ -355,7 +357,8 @@ rte_eth_bond_link_down_prop_delay_get(uint8_t bonded_port_id); * 0 on success, negative value otherwise. */ int -rte_eth_bond_link_up_prop_delay_set(uint8_t bonded_port_id, uint32_t delay_ms); +rte_eth_bond_link_up_prop_delay_set(uint16_t bonded_port_id, + uint32_t delay_ms); /** * Get the period in milliseconds set for delaying the enabling of a bonded @@ -367,7 +370,7 @@ rte_eth_bond_link_up_prop_delay_set(uint8_t bonded_port_id, uint32_t delay_ms); * Delay period on success, negative value otherwise. */ int -rte_eth_bond_link_up_prop_delay_get(uint8_t bonded_port_id); +rte_eth_bond_link_up_prop_delay_get(uint16_t bonded_port_id); #ifdef __cplusplus diff --git a/dpdk/drivers/net/bonding/rte_eth_bond_8023ad.c b/dpdk/drivers/net/bonding/rte_eth_bond_8023ad.c index 2f7ae70c..1351808c 100644 --- a/dpdk/drivers/net/bonding/rte_eth_bond_8023ad.c +++ b/dpdk/drivers/net/bonding/rte_eth_bond_8023ad.c @@ -44,7 +44,6 @@ #include "rte_eth_bond_private.h" static void bond_mode_8023ad_ext_periodic_cb(void *arg); - #ifdef RTE_LIBRTE_BOND_DEBUG_8023AD #define MODE4_DEBUG(fmt, ...) RTE_LOG(DEBUG, PMD, "%6u [Port %u: %s] " fmt, \ bond_dbg_get_time_diff_ms(), slave_id, \ @@ -210,7 +209,7 @@ set_warning_flags(struct port *port, uint16_t flags) } static void -show_warnings(uint8_t slave_id) +show_warnings(uint16_t slave_id) { struct port *port = &mode_8023ad_ports[slave_id]; uint8_t warnings; @@ -279,7 +278,7 @@ record_default(struct port *port) * @param port Port on which LACPDU was received. */ static void -rx_machine(struct bond_dev_private *internals, uint8_t slave_id, +rx_machine(struct bond_dev_private *internals, uint16_t slave_id, struct lacpdu *lacp) { struct port *agg, *port = &mode_8023ad_ports[slave_id]; @@ -400,7 +399,7 @@ rx_machine(struct bond_dev_private *internals, uint8_t slave_id, * @param port Port to handle state machine. */ static void -periodic_machine(struct bond_dev_private *internals, uint8_t slave_id) +periodic_machine(struct bond_dev_private *internals, uint16_t slave_id) { struct port *port = &mode_8023ad_ports[slave_id]; /* Calculate if either site is LACP enabled */ @@ -435,7 +434,7 @@ periodic_machine(struct bond_dev_private *internals, uint8_t slave_id) * In other case (was fast and now it is slow) just switch * timeout to slow without forcing send of LACP (because standard * say so)*/ - if (!is_partner_fast) + if (is_partner_fast) SM_FLAG_SET(port, NTT); } else return; /* Nothing changed */ @@ -462,7 +461,7 @@ periodic_machine(struct bond_dev_private *internals, uint8_t slave_id) * @param port Port to handle state machine. */ static void -mux_machine(struct bond_dev_private *internals, uint8_t slave_id) +mux_machine(struct bond_dev_private *internals, uint16_t slave_id) { struct port *port = &mode_8023ad_ports[slave_id]; @@ -565,7 +564,7 @@ mux_machine(struct bond_dev_private *internals, uint8_t slave_id) * @param port */ static void -tx_machine(struct bond_dev_private *internals, uint8_t slave_id) +tx_machine(struct bond_dev_private *internals, uint16_t slave_id) { struct port *agg, *port = &mode_8023ad_ports[slave_id]; @@ -632,21 +631,53 @@ tx_machine(struct bond_dev_private *internals, uint8_t slave_id) lacpdu->tlv_type_terminator = TLV_TYPE_TERMINATOR_INFORMATION; lacpdu->terminator_length = 0; - if (rte_ring_enqueue(port->tx_ring, lacp_pkt) == -ENOBUFS) { - /* If TX ring full, drop packet and free message. Retransmission - * will happen in next function call. */ - rte_pktmbuf_free(lacp_pkt); - set_warning_flags(port, WRN_TX_QUEUE_FULL); - return; + MODE4_DEBUG("Sending LACP frame\n"); + BOND_PRINT_LACP(lacpdu); + + if (internals->mode4.dedicated_queues.enabled == 0) { + int retval = rte_ring_enqueue(port->tx_ring, lacp_pkt); + if (retval != 0) { + /* If TX ring full, drop packet and free message. + Retransmission will happen in next function call. */ + rte_pktmbuf_free(lacp_pkt); + set_warning_flags(port, WRN_TX_QUEUE_FULL); + return; + } + } else { + uint16_t pkts_sent = rte_eth_tx_burst(slave_id, + internals->mode4.dedicated_queues.tx_qid, + &lacp_pkt, 1); + if (pkts_sent != 1) { + rte_pktmbuf_free(lacp_pkt); + set_warning_flags(port, WRN_TX_QUEUE_FULL); + return; + } } - MODE4_DEBUG("sending LACP frame\n"); - BOND_PRINT_LACP(lacpdu); timer_set(&port->tx_machine_timer, internals->mode4.tx_period_timeout); SM_FLAG_CLR(port, NTT); } +static uint8_t +max_index(uint64_t *a, int n) +{ + if (n <= 0) + return -1; + + int i, max_i = 0; + uint64_t max = a[0]; + + for (i = 1; i < n; ++i) { + if (a[i] > max) { + max = a[i]; + max_i = i; + } + } + + return max_i; +} + /** * Function assigns port to aggregator. * @@ -657,8 +688,13 @@ static void selection_logic(struct bond_dev_private *internals, uint8_t slave_id) { struct port *agg, *port; - uint8_t slaves_count, new_agg_id, i; - uint8_t *slaves; + uint16_t slaves_count, new_agg_id, i, j = 0; + uint16_t *slaves; + uint64_t agg_bandwidth[8] = {0}; + uint64_t agg_count[8] = {0}; + uint16_t default_slave = 0; + uint8_t mode_count_id, mode_band_id; + struct rte_eth_link link_info; slaves = internals->active_slaves; slaves_count = internals->active_slave_count; @@ -671,6 +707,10 @@ selection_logic(struct bond_dev_private *internals, uint8_t slave_id) if (agg->aggregator_port_id != slaves[i]) continue; + agg_count[agg->aggregator_port_id] += 1; + rte_eth_link_get_nowait(slaves[i], &link_info); + agg_bandwidth[agg->aggregator_port_id] += link_info.link_speed; + /* Actors system ID is not checked since all slave device have the same * ID (MAC address). */ if ((agg->actor.key == port->actor.key && @@ -681,15 +721,36 @@ selection_logic(struct bond_dev_private *internals, uint8_t slave_id) (agg->actor.key & rte_cpu_to_be_16(BOND_LINK_FULL_DUPLEX_KEY)) != 0) { - break; + if (j == 0) + default_slave = i; + j++; } } - /* By default, port uses it self as agregator */ - if (i == slaves_count) - new_agg_id = slave_id; - else - new_agg_id = slaves[i]; + switch (internals->mode4.agg_selection) { + case AGG_COUNT: + mode_count_id = max_index( + (uint64_t *)agg_count, slaves_count); + new_agg_id = mode_count_id; + break; + case AGG_BANDWIDTH: + mode_band_id = max_index( + (uint64_t *)agg_bandwidth, slaves_count); + new_agg_id = mode_band_id; + break; + case AGG_STABLE: + if (default_slave == slaves_count) + new_agg_id = slave_id; + else + new_agg_id = slaves[default_slave]; + break; + default: + if (default_slave == slaves_count) + new_agg_id = slave_id; + else + new_agg_id = slaves[default_slave]; + break; + } if (new_agg_id != port->aggregator_port_id) { port->aggregator_port_id = new_agg_id; @@ -740,6 +801,22 @@ link_speed_key(uint16_t speed) { return key_speed; } +static void +rx_machine_update(struct bond_dev_private *internals, uint8_t slave_id, + struct rte_mbuf *lacp_pkt) { + struct lacpdu_header *lacp; + + if (lacp_pkt != NULL) { + lacp = rte_pktmbuf_mtod(lacp_pkt, struct lacpdu_header *); + RTE_ASSERT(lacp->lacpdu.subtype == SLOW_SUBTYPE_LACP); + + /* This is LACP frame so pass it to rx_machine */ + rx_machine(internals, slave_id, &lacp->lacpdu); + rte_pktmbuf_free(lacp_pkt); + } else + rx_machine(internals, slave_id, NULL); +} + static void bond_mode_8023ad_periodic_cb(void *arg) { @@ -748,8 +825,8 @@ bond_mode_8023ad_periodic_cb(void *arg) struct port *port; struct rte_eth_link link_info; struct ether_addr slave_addr; + struct rte_mbuf *lacp_pkt = NULL; - void *pkt = NULL; uint8_t i, slave_id; @@ -758,7 +835,7 @@ bond_mode_8023ad_periodic_cb(void *arg) uint16_t key; slave_id = internals->active_slaves[i]; - rte_eth_link_get(slave_id, &link_info); + rte_eth_link_get_nowait(slave_id, &link_info); rte_eth_macaddr_get(slave_id, &slave_addr); if (link_info.link_status != 0) { @@ -809,20 +886,28 @@ bond_mode_8023ad_periodic_cb(void *arg) SM_FLAG_SET(port, LACP_ENABLED); - /* Find LACP packet to this port. Do not check subtype, it is done in - * function that queued packet */ - if (rte_ring_dequeue(port->rx_ring, &pkt) == 0) { - struct rte_mbuf *lacp_pkt = pkt; - struct lacpdu_header *lacp; + if (internals->mode4.dedicated_queues.enabled == 0) { + /* Find LACP packet to this port. Do not check subtype, + * it is done in function that queued packet + */ + int retval = rte_ring_dequeue(port->rx_ring, + (void **)&lacp_pkt); - lacp = rte_pktmbuf_mtod(lacp_pkt, struct lacpdu_header *); - RTE_ASSERT(lacp->lacpdu.subtype == SLOW_SUBTYPE_LACP); + if (retval != 0) + lacp_pkt = NULL; - /* This is LACP frame so pass it to rx_machine */ - rx_machine(internals, slave_id, &lacp->lacpdu); - rte_pktmbuf_free(lacp_pkt); - } else - rx_machine(internals, slave_id, NULL); + rx_machine_update(internals, slave_id, lacp_pkt); + } else { + uint16_t rx_count = rte_eth_rx_burst(slave_id, + internals->mode4.dedicated_queues.rx_qid, + &lacp_pkt, 1); + + if (rx_count == 1) + bond_mode_8023ad_handle_slow_pkt(internals, + slave_id, lacp_pkt); + else + rx_machine_update(internals, slave_id, NULL); + } periodic_machine(internals, slave_id); mux_machine(internals, slave_id); @@ -838,7 +923,8 @@ bond_mode_8023ad_periodic_cb(void *arg) } void -bond_mode_8023ad_activate_slave(struct rte_eth_dev *bond_dev, uint8_t slave_id) +bond_mode_8023ad_activate_slave(struct rte_eth_dev *bond_dev, + uint16_t slave_id) { struct bond_dev_private *internals = bond_dev->data->dev_private; @@ -866,13 +952,13 @@ bond_mode_8023ad_activate_slave(struct rte_eth_dev *bond_dev, uint8_t slave_id) memcpy(&port->actor, &initial, sizeof(struct port_params)); /* Standard requires that port ID must be grater than 0. * Add 1 do get corresponding port_number */ - port->actor.port_number = rte_cpu_to_be_16((uint16_t)slave_id + 1); + port->actor.port_number = rte_cpu_to_be_16(slave_id + 1); memcpy(&port->partner, &initial, sizeof(struct port_params)); /* default states */ port->actor_state = STATE_AGGREGATION | STATE_LACP_ACTIVE | STATE_DEFAULTED; - port->partner_state = STATE_LACP_ACTIVE; + port->partner_state = STATE_LACP_ACTIVE | STATE_AGGREGATION; port->sm_flags = SM_FLAGS_BEGIN; /* use this port as agregator */ @@ -886,10 +972,13 @@ bond_mode_8023ad_activate_slave(struct rte_eth_dev *bond_dev, uint8_t slave_id) RTE_ASSERT(port->rx_ring == NULL); RTE_ASSERT(port->tx_ring == NULL); - socket_id = rte_eth_devices[slave_id].data->numa_node; - element_size = sizeof(struct slow_protocol_frame) + sizeof(struct rte_mbuf) - + RTE_PKTMBUF_HEADROOM; + socket_id = rte_eth_dev_socket_id(slave_id); + if (socket_id == (int)LCORE_ID_ANY) + socket_id = rte_socket_id(); + + element_size = sizeof(struct slow_protocol_frame) + + RTE_PKTMBUF_HEADROOM; /* The size of the mempool should be at least: * the sum of the TX descriptors + BOND_MODE_8023AX_SLAVE_TX_PKTS */ @@ -900,13 +989,12 @@ bond_mode_8023ad_activate_slave(struct rte_eth_dev *bond_dev, uint8_t slave_id) } snprintf(mem_name, RTE_DIM(mem_name), "slave_port%u_pool", slave_id); - port->mbuf_pool = rte_mempool_create(mem_name, - total_tx_desc, element_size, - RTE_MEMPOOL_CACHE_MAX_SIZE >= 32 ? 32 : RTE_MEMPOOL_CACHE_MAX_SIZE, - sizeof(struct rte_pktmbuf_pool_private), rte_pktmbuf_pool_init, - NULL, rte_pktmbuf_init, NULL, socket_id, MEMPOOL_F_NO_SPREAD); + port->mbuf_pool = rte_pktmbuf_pool_create(mem_name, total_tx_desc, + RTE_MEMPOOL_CACHE_MAX_SIZE >= 32 ? + 32 : RTE_MEMPOOL_CACHE_MAX_SIZE, + 0, element_size, socket_id); - /* Any memory allocation failure in initalization is critical because + /* Any memory allocation failure in initialization is critical because * resources can't be free, so reinitialization is impossible. */ if (port->mbuf_pool == NULL) { rte_panic("Slave %u: Failed to create memory pool '%s': %s\n", @@ -934,37 +1022,30 @@ bond_mode_8023ad_activate_slave(struct rte_eth_dev *bond_dev, uint8_t slave_id) } int -bond_mode_8023ad_deactivate_slave(struct rte_eth_dev *bond_dev, - uint8_t slave_id) +bond_mode_8023ad_deactivate_slave(struct rte_eth_dev *bond_dev __rte_unused, + uint16_t slave_id) { - struct bond_dev_private *internals = bond_dev->data->dev_private; void *pkt = NULL; - struct port *port; - uint8_t i; - - /* Given slave must be in active list */ - RTE_ASSERT(find_slave_by_id(internals->active_slaves, - internals->active_slave_count, slave_id) < internals->active_slave_count); - - /* Exclude slave from transmit policy. If this slave is an aggregator - * make all aggregated slaves unselected to force selection logic - * to select suitable aggregator for this port. */ - for (i = 0; i < internals->active_slave_count; i++) { - port = &mode_8023ad_ports[internals->active_slaves[i]]; - if (port->aggregator_port_id != slave_id) - continue; - - port->selected = UNSELECTED; - - /* Use default aggregator */ - port->aggregator_port_id = internals->active_slaves[i]; - } + struct port *port = NULL; + uint8_t old_partner_state; port = &mode_8023ad_ports[slave_id]; - port->selected = UNSELECTED; - port->actor_state &= ~(STATE_SYNCHRONIZATION | STATE_DISTRIBUTING | - STATE_COLLECTING); + ACTOR_STATE_CLR(port, AGGREGATION); + port->selected = UNSELECTED; + + old_partner_state = port->partner_state; + record_default(port); + + /* If partner timeout state changes then disable timer */ + if (!((old_partner_state ^ port->partner_state) & + STATE_LACP_SHORT_TIMEOUT)) + timer_cancel(&port->current_while_timer); + + PARTNER_STATE_CLR(port, AGGREGATION); + ACTOR_STATE_CLR(port, EXPIRED); + + /* flush rx/tx rings */ while (rte_ring_dequeue(port->rx_ring, &pkt) == 0) rte_pktmbuf_free((struct rte_mbuf *)pkt); @@ -979,7 +1060,7 @@ bond_mode_8023ad_mac_address_update(struct rte_eth_dev *bond_dev) struct bond_dev_private *internals = bond_dev->data->dev_private; struct ether_addr slave_addr; struct port *slave, *agg_slave; - uint8_t slave_id, i, j; + uint16_t slave_id, i, j; bond_mode_8023ad_stop(bond_dev); @@ -1024,17 +1105,8 @@ bond_mode_8023ad_conf_get(struct rte_eth_dev *dev, conf->tx_period_ms = mode4->tx_period_timeout / ms_ticks; conf->update_timeout_ms = mode4->update_timeout_us / 1000; conf->rx_marker_period_ms = mode4->rx_marker_timeout / ms_ticks; -} - -static void -bond_mode_8023ad_conf_get_v1607(struct rte_eth_dev *dev, - struct rte_eth_bond_8023ad_conf *conf) -{ - struct bond_dev_private *internals = dev->data->dev_private; - struct mode8023ad_private *mode4 = &internals->mode4; - - bond_mode_8023ad_conf_get(dev, conf); conf->slowrx_cb = mode4->slowrx_cb; + conf->agg_selection = mode4->agg_selection; } static void @@ -1049,6 +1121,7 @@ bond_mode_8023ad_conf_get_default(struct rte_eth_bond_8023ad_conf *conf) conf->rx_marker_period_ms = BOND_8023AD_RX_MARKER_PERIOD_MS; conf->update_timeout_ms = BOND_MODE_8023AX_UPDATE_TIMEOUT_MS; conf->slowrx_cb = NULL; + conf->agg_selection = AGG_STABLE; } static void @@ -1065,29 +1138,12 @@ bond_mode_8023ad_conf_assign(struct mode8023ad_private *mode4, mode4->tx_period_timeout = conf->tx_period_ms * ms_ticks; mode4->rx_marker_timeout = conf->rx_marker_period_ms * ms_ticks; mode4->update_timeout_us = conf->update_timeout_ms * 1000; + + mode4->dedicated_queues.enabled = 0; + mode4->dedicated_queues.rx_qid = UINT16_MAX; + mode4->dedicated_queues.tx_qid = UINT16_MAX; } -static void -bond_mode_8023ad_setup_v20(struct rte_eth_dev *dev, - struct rte_eth_bond_8023ad_conf *conf) -{ - struct rte_eth_bond_8023ad_conf def_conf; - struct bond_dev_private *internals = dev->data->dev_private; - struct mode8023ad_private *mode4 = &internals->mode4; - - if (conf == NULL) { - conf = &def_conf; - bond_mode_8023ad_conf_get_default(conf); - } - - bond_mode_8023ad_stop(dev); - bond_mode_8023ad_conf_assign(mode4, conf); - - if (dev->data->dev_started) - bond_mode_8023ad_start(dev); -} - - void bond_mode_8023ad_setup(struct rte_eth_dev *dev, struct rte_eth_bond_8023ad_conf *conf) @@ -1104,6 +1160,7 @@ bond_mode_8023ad_setup(struct rte_eth_dev *dev, bond_mode_8023ad_stop(dev); bond_mode_8023ad_conf_assign(mode4, conf); mode4->slowrx_cb = conf->slowrx_cb; + mode4->agg_selection = AGG_STABLE; if (dev->data->dev_started) bond_mode_8023ad_start(dev); @@ -1116,7 +1173,8 @@ bond_mode_8023ad_enable(struct rte_eth_dev *bond_dev) uint8_t i; for (i = 0; i < internals->active_slave_count; i++) - bond_mode_8023ad_activate_slave(bond_dev, i); + bond_mode_8023ad_activate_slave(bond_dev, + internals->active_slaves[i]); return 0; } @@ -1151,7 +1209,7 @@ bond_mode_8023ad_stop(struct rte_eth_dev *bond_dev) void bond_mode_8023ad_handle_slow_pkt(struct bond_dev_private *internals, - uint8_t slave_id, struct rte_mbuf *pkt) + uint16_t slave_id, struct rte_mbuf *pkt) { struct mode8023ad_private *mode4 = &internals->mode4; struct port *port = &mode_8023ad_ports[slave_id]; @@ -1189,18 +1247,36 @@ bond_mode_8023ad_handle_slow_pkt(struct bond_dev_private *internals, m_hdr->marker.tlv_type_marker = MARKER_TLV_TYPE_RESP; rte_eth_macaddr_get(slave_id, &m_hdr->eth_hdr.s_addr); - if (unlikely(rte_ring_enqueue(port->tx_ring, pkt) == -ENOBUFS)) { - /* reset timer */ - port->rx_marker_timer = 0; - wrn = WRN_TX_QUEUE_FULL; - goto free_out; + if (internals->mode4.dedicated_queues.enabled == 0) { + int retval = rte_ring_enqueue(port->tx_ring, pkt); + if (retval != 0) { + /* reset timer */ + port->rx_marker_timer = 0; + wrn = WRN_TX_QUEUE_FULL; + goto free_out; + } + } else { + /* Send packet directly to the slow queue */ + uint16_t tx_count = rte_eth_tx_burst(slave_id, + internals->mode4.dedicated_queues.tx_qid, + &pkt, 1); + if (tx_count != 1) { + /* reset timer */ + port->rx_marker_timer = 0; + wrn = WRN_TX_QUEUE_FULL; + goto free_out; + } } } else if (likely(subtype == SLOW_SUBTYPE_LACP)) { - if (unlikely(rte_ring_enqueue(port->rx_ring, pkt) == -ENOBUFS)) { - /* If RX fing full free lacpdu message and drop packet */ - wrn = WRN_RX_QUEUE_FULL; - goto free_out; - } + if (internals->mode4.dedicated_queues.enabled == 0) { + int retval = rte_ring_enqueue(port->rx_ring, pkt); + if (retval != 0) { + /* If RX fing full free lacpdu message and drop packet */ + wrn = WRN_RX_QUEUE_FULL; + goto free_out; + } + } else + rx_machine_update(internals, slave_id, pkt); } else { wrn = WRN_UNKNOWN_SLOW_TYPE; goto free_out; @@ -1214,7 +1290,7 @@ free_out: } int -rte_eth_bond_8023ad_conf_get_v20(uint8_t port_id, +rte_eth_bond_8023ad_conf_get(uint16_t port_id, struct rte_eth_bond_8023ad_conf *conf) { struct rte_eth_dev *bond_dev; @@ -1229,31 +1305,52 @@ rte_eth_bond_8023ad_conf_get_v20(uint8_t port_id, bond_mode_8023ad_conf_get(bond_dev, conf); return 0; } -VERSION_SYMBOL(rte_eth_bond_8023ad_conf_get, _v20, 2.0); int -rte_eth_bond_8023ad_conf_get_v1607(uint8_t port_id, - struct rte_eth_bond_8023ad_conf *conf) +rte_eth_bond_8023ad_agg_selection_set(uint16_t port_id, + enum rte_bond_8023ad_agg_selection agg_selection) { struct rte_eth_dev *bond_dev; + struct bond_dev_private *internals; + struct mode8023ad_private *mode4; + + bond_dev = &rte_eth_devices[port_id]; + internals = bond_dev->data->dev_private; if (valid_bonded_port_id(port_id) != 0) return -EINVAL; - - if (conf == NULL) + if (internals->mode != 4) return -EINVAL; - bond_dev = &rte_eth_devices[port_id]; - bond_mode_8023ad_conf_get_v1607(bond_dev, conf); + mode4 = &internals->mode4; + if (agg_selection == AGG_COUNT || agg_selection == AGG_BANDWIDTH + || agg_selection == AGG_STABLE) + mode4->agg_selection = agg_selection; return 0; } -BIND_DEFAULT_SYMBOL(rte_eth_bond_8023ad_conf_get, _v1607, 16.07); -MAP_STATIC_SYMBOL(int rte_eth_bond_8023ad_conf_get(uint8_t port_id, - struct rte_eth_bond_8023ad_conf *conf), - rte_eth_bond_8023ad_conf_get_v1607); + +int rte_eth_bond_8023ad_agg_selection_get(uint16_t port_id) +{ + struct rte_eth_dev *bond_dev; + struct bond_dev_private *internals; + struct mode8023ad_private *mode4; + + bond_dev = &rte_eth_devices[port_id]; + internals = bond_dev->data->dev_private; + + if (valid_bonded_port_id(port_id) != 0) + return -EINVAL; + if (internals->mode != 4) + return -EINVAL; + mode4 = &internals->mode4; + + return mode4->agg_selection; +} + + static int -bond_8023ad_setup_validate(uint8_t port_id, +bond_8023ad_setup_validate(uint16_t port_id, struct rte_eth_bond_8023ad_conf *conf) { if (valid_bonded_port_id(port_id) != 0) @@ -1277,26 +1374,9 @@ bond_8023ad_setup_validate(uint8_t port_id, return 0; } -int -rte_eth_bond_8023ad_setup_v20(uint8_t port_id, - struct rte_eth_bond_8023ad_conf *conf) -{ - struct rte_eth_dev *bond_dev; - int err; - - err = bond_8023ad_setup_validate(port_id, conf); - if (err != 0) - return err; - - bond_dev = &rte_eth_devices[port_id]; - bond_mode_8023ad_setup_v20(bond_dev, conf); - - return 0; -} -VERSION_SYMBOL(rte_eth_bond_8023ad_setup, _v20, 2.0); int -rte_eth_bond_8023ad_setup_v1607(uint8_t port_id, +rte_eth_bond_8023ad_setup(uint16_t port_id, struct rte_eth_bond_8023ad_conf *conf) { struct rte_eth_dev *bond_dev; @@ -1311,13 +1391,13 @@ rte_eth_bond_8023ad_setup_v1607(uint8_t port_id, return 0; } -BIND_DEFAULT_SYMBOL(rte_eth_bond_8023ad_setup, _v1607, 16.07); -MAP_STATIC_SYMBOL(int rte_eth_bond_8023ad_setup(uint8_t port_id, - struct rte_eth_bond_8023ad_conf *conf), - rte_eth_bond_8023ad_setup_v1607); + + + + int -rte_eth_bond_8023ad_slave_info(uint8_t port_id, uint8_t slave_id, +rte_eth_bond_8023ad_slave_info(uint16_t port_id, uint16_t slave_id, struct rte_eth_bond_8023ad_slave_info *info) { struct rte_eth_dev *bond_dev; @@ -1350,7 +1430,7 @@ rte_eth_bond_8023ad_slave_info(uint8_t port_id, uint8_t slave_id, } static int -bond_8023ad_ext_validate(uint8_t port_id, uint8_t slave_id) +bond_8023ad_ext_validate(uint16_t port_id, uint16_t slave_id) { struct rte_eth_dev *bond_dev; struct bond_dev_private *internals; @@ -1378,7 +1458,8 @@ bond_8023ad_ext_validate(uint8_t port_id, uint8_t slave_id) } int -rte_eth_bond_8023ad_ext_collect(uint8_t port_id, uint8_t slave_id, int enabled) +rte_eth_bond_8023ad_ext_collect(uint16_t port_id, uint16_t slave_id, + int enabled) { struct port *port; int res; @@ -1398,7 +1479,8 @@ rte_eth_bond_8023ad_ext_collect(uint8_t port_id, uint8_t slave_id, int enabled) } int -rte_eth_bond_8023ad_ext_distrib(uint8_t port_id, uint8_t slave_id, int enabled) +rte_eth_bond_8023ad_ext_distrib(uint16_t port_id, uint16_t slave_id, + int enabled) { struct port *port; int res; @@ -1418,7 +1500,7 @@ rte_eth_bond_8023ad_ext_distrib(uint8_t port_id, uint8_t slave_id, int enabled) } int -rte_eth_bond_8023ad_ext_distrib_get(uint8_t port_id, uint8_t slave_id) +rte_eth_bond_8023ad_ext_distrib_get(uint16_t port_id, uint16_t slave_id) { struct port *port; int err; @@ -1432,7 +1514,7 @@ rte_eth_bond_8023ad_ext_distrib_get(uint8_t port_id, uint8_t slave_id) } int -rte_eth_bond_8023ad_ext_collect_get(uint8_t port_id, uint8_t slave_id) +rte_eth_bond_8023ad_ext_collect_get(uint16_t port_id, uint16_t slave_id) { struct port *port; int err; @@ -1446,7 +1528,7 @@ rte_eth_bond_8023ad_ext_collect_get(uint8_t port_id, uint8_t slave_id) } int -rte_eth_bond_8023ad_ext_slowtx(uint8_t port_id, uint8_t slave_id, +rte_eth_bond_8023ad_ext_slowtx(uint16_t port_id, uint16_t slave_id, struct rte_mbuf *lacp_pkt) { struct port *port; @@ -1505,3 +1587,49 @@ bond_mode_8023ad_ext_periodic_cb(void *arg) rte_eal_alarm_set(internals->mode4.update_timeout_us, bond_mode_8023ad_ext_periodic_cb, arg); } + +int +rte_eth_bond_8023ad_dedicated_queues_enable(uint16_t port) +{ + int retval = 0; + struct rte_eth_dev *dev = &rte_eth_devices[port]; + struct bond_dev_private *internals = (struct bond_dev_private *) + dev->data->dev_private; + + if (check_for_bonded_ethdev(dev) != 0) + return -1; + + if (bond_8023ad_slow_pkt_hw_filter_supported(port) != 0) + return -1; + + /* Device must be stopped to set up slow queue */ + if (dev->data->dev_started) + return -1; + + internals->mode4.dedicated_queues.enabled = 1; + + bond_ethdev_mode_set(dev, internals->mode); + return retval; +} + +int +rte_eth_bond_8023ad_dedicated_queues_disable(uint16_t port) +{ + int retval = 0; + struct rte_eth_dev *dev = &rte_eth_devices[port]; + struct bond_dev_private *internals = (struct bond_dev_private *) + dev->data->dev_private; + + if (check_for_bonded_ethdev(dev) != 0) + return -1; + + /* Device must be stopped to set up slow queue */ + if (dev->data->dev_started) + return -1; + + internals->mode4.dedicated_queues.enabled = 0; + + bond_ethdev_mode_set(dev, internals->mode); + + return retval; +} diff --git a/dpdk/drivers/net/bonding/rte_eth_bond_8023ad.h b/dpdk/drivers/net/bonding/rte_eth_bond_8023ad.h index 6b8ff575..2874336d 100644 --- a/dpdk/drivers/net/bonding/rte_eth_bond_8023ad.h +++ b/dpdk/drivers/net/bonding/rte_eth_bond_8023ad.h @@ -64,7 +64,7 @@ extern "C" { #define MARKER_TLV_TYPE_INFO 0x01 #define MARKER_TLV_TYPE_RESP 0x02 -typedef void (*rte_eth_bond_8023ad_ext_slowrx_fn)(uint8_t slave_id, +typedef void (*rte_eth_bond_8023ad_ext_slowrx_fn)(uint16_t slave_id, struct rte_mbuf *lacp_pkt); enum rte_bond_8023ad_selection { @@ -73,6 +73,12 @@ enum rte_bond_8023ad_selection { SELECTED }; +enum rte_bond_8023ad_agg_selection { + AGG_BANDWIDTH, + AGG_COUNT, + AGG_STABLE +}; + /** Generic slow protocol structure */ struct slow_protocol { uint8_t subtype; @@ -161,6 +167,7 @@ struct rte_eth_bond_8023ad_conf { uint32_t rx_marker_period_ms; uint32_t update_timeout_ms; rte_eth_bond_8023ad_ext_slowrx_fn slowrx_cb; + enum rte_bond_8023ad_agg_selection agg_selection; }; struct rte_eth_bond_8023ad_slave_info { @@ -169,7 +176,7 @@ struct rte_eth_bond_8023ad_slave_info { struct port_params actor; uint8_t partner_state; struct port_params partner; - uint8_t agg_port_id; + uint16_t agg_port_id; }; /** @@ -185,13 +192,7 @@ struct rte_eth_bond_8023ad_slave_info { * -EINVAL if conf is NULL */ int -rte_eth_bond_8023ad_conf_get(uint8_t port_id, - struct rte_eth_bond_8023ad_conf *conf); -int -rte_eth_bond_8023ad_conf_get_v20(uint8_t port_id, - struct rte_eth_bond_8023ad_conf *conf); -int -rte_eth_bond_8023ad_conf_get_v1607(uint8_t port_id, +rte_eth_bond_8023ad_conf_get(uint16_t port_id, struct rte_eth_bond_8023ad_conf *conf); /** @@ -206,13 +207,7 @@ rte_eth_bond_8023ad_conf_get_v1607(uint8_t port_id, * -EINVAL if configuration is invalid. */ int -rte_eth_bond_8023ad_setup(uint8_t port_id, - struct rte_eth_bond_8023ad_conf *conf); -int -rte_eth_bond_8023ad_setup_v20(uint8_t port_id, - struct rte_eth_bond_8023ad_conf *conf); -int -rte_eth_bond_8023ad_setup_v1607(uint8_t port_id, +rte_eth_bond_8023ad_setup(uint16_t port_id, struct rte_eth_bond_8023ad_conf *conf); /** @@ -228,7 +223,7 @@ rte_eth_bond_8023ad_setup_v1607(uint8_t port_id, * bonded device or is not inactive). */ int -rte_eth_bond_8023ad_slave_info(uint8_t port_id, uint8_t slave_id, +rte_eth_bond_8023ad_slave_info(uint16_t port_id, uint16_t slave_id, struct rte_eth_bond_8023ad_slave_info *conf); #ifdef __cplusplus @@ -246,7 +241,8 @@ rte_eth_bond_8023ad_slave_info(uint8_t port_id, uint8_t slave_id, * -EINVAL if slave is not valid. */ int -rte_eth_bond_8023ad_ext_collect(uint8_t port_id, uint8_t slave_id, int enabled); +rte_eth_bond_8023ad_ext_collect(uint16_t port_id, uint16_t slave_id, + int enabled); /** * Get COLLECTING flag from slave port actor state. @@ -259,7 +255,7 @@ rte_eth_bond_8023ad_ext_collect(uint8_t port_id, uint8_t slave_id, int enabled); * -EINVAL if slave is not valid. */ int -rte_eth_bond_8023ad_ext_collect_get(uint8_t port_id, uint8_t slave_id); +rte_eth_bond_8023ad_ext_collect_get(uint16_t port_id, uint16_t slave_id); /** * Configure a slave port to start distributing. @@ -272,7 +268,8 @@ rte_eth_bond_8023ad_ext_collect_get(uint8_t port_id, uint8_t slave_id); * -EINVAL if slave is not valid. */ int -rte_eth_bond_8023ad_ext_distrib(uint8_t port_id, uint8_t slave_id, int enabled); +rte_eth_bond_8023ad_ext_distrib(uint16_t port_id, uint16_t slave_id, + int enabled); /** * Get DISTRIBUTING flag from slave port actor state. @@ -285,7 +282,7 @@ rte_eth_bond_8023ad_ext_distrib(uint8_t port_id, uint8_t slave_id, int enabled); * -EINVAL if slave is not valid. */ int -rte_eth_bond_8023ad_ext_distrib_get(uint8_t port_id, uint8_t slave_id); +rte_eth_bond_8023ad_ext_distrib_get(uint16_t port_id, uint16_t slave_id); /** * LACPDU transmit path for external 802.3ad state machine. Caller retains @@ -299,7 +296,68 @@ rte_eth_bond_8023ad_ext_distrib_get(uint8_t port_id, uint8_t slave_id); * 0 on success, negative value otherwise. */ int -rte_eth_bond_8023ad_ext_slowtx(uint8_t port_id, uint8_t slave_id, +rte_eth_bond_8023ad_ext_slowtx(uint16_t port_id, uint16_t slave_id, struct rte_mbuf *lacp_pkt); +/** + * Enable dedicated hw queues for 802.3ad control plane traffic on on slaves + * + * This function creates an additional tx and rx queue on each slave for + * dedicated 802.3ad control plane traffic . A flow filtering rule is + * programmed on each slave to redirect all LACP slow packets to that rx queue + * for processing in the LACP state machine, this removes the need to filter + * these packets in the bonded devices data path. The additional tx queue is + * used to enable the LACP state machine to enqueue LACP packets directly to + * slave hw independently of the bonded devices data path. + * + * To use this feature all slaves must support the programming of the flow + * filter rule required for rx and have enough queues that one rx and tx queue + * can be reserved for the LACP state machines control packets. + * + * Bonding port must be stopped to change this configuration. + * + * @param port_id Bonding device id + * + * @return + * 0 on success, negative value otherwise. + */ +int +rte_eth_bond_8023ad_dedicated_queues_enable(uint16_t port_id); + +/** + * Disable slow queue on slaves + * + * This function disables hardware slow packet filter. + * + * Bonding port must be stopped to change this configuration. + * + * @see rte_eth_bond_8023ad_slow_pkt_hw_filter_enable + * + * @param port_id Bonding device id + * @return + * 0 on success, negative value otherwise. + * + */ +int +rte_eth_bond_8023ad_dedicated_queues_disable(uint16_t port_id); + +/* + * Get aggregator mode for 8023ad + * @param port_id Bonding device id + * + * @return + * agregator mode on success, negative value otherwise + */ +int +rte_eth_bond_8023ad_agg_selection_get(uint16_t port_id); + +/** + * Set aggregator mode for 8023ad + * @param port_id Bonding device id + * @return + * 0 on success, negative value otherwise + */ +int +rte_eth_bond_8023ad_agg_selection_set(uint16_t port_id, + enum rte_bond_8023ad_agg_selection agg_selection); #endif /* RTE_ETH_BOND_8023AD_H_ */ diff --git a/dpdk/drivers/net/bonding/rte_eth_bond_8023ad_private.h b/dpdk/drivers/net/bonding/rte_eth_bond_8023ad_private.h index ca8858be..433c7000 100644 --- a/dpdk/drivers/net/bonding/rte_eth_bond_8023ad_private.h +++ b/dpdk/drivers/net/bonding/rte_eth_bond_8023ad_private.h @@ -39,6 +39,7 @@ #include #include #include +#include #include "rte_eth_bond_8023ad.h" @@ -162,6 +163,9 @@ struct port { uint64_t warning_timer; volatile uint16_t warnings_to_show; + + /** Memory pool used to allocate slow queues */ + struct rte_mempool *slow_pool; }; struct mode8023ad_private { @@ -175,6 +179,23 @@ struct mode8023ad_private { uint64_t update_timeout_us; rte_eth_bond_8023ad_ext_slowrx_fn slowrx_cb; uint8_t external_sm; + + struct rte_eth_link slave_link; + /***< slave link properties */ + + /** + * Configuration of dedicated hardware queues for control plane + * traffic + */ + struct { + uint8_t enabled; + + struct rte_flow *flow[RTE_MAX_ETHPORTS]; + + uint16_t rx_qid; + uint16_t tx_qid; + } dedicated_queues; + enum rte_bond_8023ad_agg_selection agg_selection; }; /** @@ -258,7 +279,7 @@ bond_mode_8023ad_stop(struct rte_eth_dev *dev); */ void bond_mode_8023ad_handle_slow_pkt(struct bond_dev_private *internals, - uint8_t slave_id, struct rte_mbuf *pkt); + uint16_t slave_id, struct rte_mbuf *pkt); /** * @internal @@ -272,7 +293,7 @@ bond_mode_8023ad_handle_slow_pkt(struct bond_dev_private *internals, * 0 on success, negative value otherwise. */ void -bond_mode_8023ad_activate_slave(struct rte_eth_dev *dev, uint8_t port_id); +bond_mode_8023ad_activate_slave(struct rte_eth_dev *dev, uint16_t port_id); /** * @internal @@ -286,7 +307,7 @@ bond_mode_8023ad_activate_slave(struct rte_eth_dev *dev, uint8_t port_id); * 0 on success, negative value otherwise. */ int -bond_mode_8023ad_deactivate_slave(struct rte_eth_dev *dev, uint8_t slave_pos); +bond_mode_8023ad_deactivate_slave(struct rte_eth_dev *dev, uint16_t slave_pos); /** * Updates state when MAC was changed on bonded device or one of its slaves. @@ -295,4 +316,14 @@ bond_mode_8023ad_deactivate_slave(struct rte_eth_dev *dev, uint8_t slave_pos); void bond_mode_8023ad_mac_address_update(struct rte_eth_dev *bond_dev); +int +bond_ethdev_8023ad_flow_verify(struct rte_eth_dev *bond_dev, + uint16_t slave_port); + +int +bond_ethdev_8023ad_flow_set(struct rte_eth_dev *bond_dev, uint16_t slave_port); + +int +bond_8023ad_slow_pkt_hw_filter_supported(uint16_t port_id); + #endif /* RTE_ETH_BOND_8023AD_H_ */ diff --git a/dpdk/drivers/net/bonding/rte_eth_bond_alb.c b/dpdk/drivers/net/bonding/rte_eth_bond_alb.c index 38f5c4d4..f7efbb78 100644 --- a/dpdk/drivers/net/bonding/rte_eth_bond_alb.c +++ b/dpdk/drivers/net/bonding/rte_eth_bond_alb.c @@ -80,7 +80,8 @@ bond_mode_alb_enable(struct rte_eth_dev *bond_dev) * The value is chosen to be cache aligned. */ data_size = 256 + RTE_PKTMBUF_HEADROOM; - snprintf(mem_name, sizeof(mem_name), "%s_MODE6", bond_dev->data->name); + snprintf(mem_name, sizeof(mem_name), "%s_ALB", + bond_dev->device->name); internals->mode6.mempool = rte_pktmbuf_pool_create(mem_name, 512 * RTE_MAX_ETHPORTS, RTE_MEMPOOL_CACHE_MAX_SIZE >= 32 ? @@ -89,7 +90,7 @@ bond_mode_alb_enable(struct rte_eth_dev *bond_dev) if (internals->mode6.mempool == NULL) { RTE_LOG(ERR, PMD, "%s: Failed to initialize ALB mempool.\n", - bond_dev->data->name); + bond_dev->device->name); goto mempool_alloc_error; } } @@ -147,7 +148,7 @@ void bond_mode_alb_arp_recv(struct ether_hdr *eth_h, uint16_t offset, rte_spinlock_unlock(&internals->mode6.lock); } -uint8_t +uint16_t bond_mode_alb_arp_xmit(struct ether_hdr *eth_h, uint16_t offset, struct bond_dev_private *internals) { @@ -219,13 +220,13 @@ bond_mode_alb_arp_xmit(struct ether_hdr *eth_h, uint16_t offset, return internals->current_primary_port; } -uint8_t +uint16_t bond_mode_alb_arp_upd(struct client_data *client_info, struct rte_mbuf *pkt, struct bond_dev_private *internals) { struct ether_hdr *eth_h; struct arp_hdr *arp_h; - uint8_t slave_idx; + uint16_t slave_idx; rte_spinlock_lock(&internals->mode6.lock); eth_h = rte_pktmbuf_mtod(pkt, struct ether_hdr *); diff --git a/dpdk/drivers/net/bonding/rte_eth_bond_alb.h b/dpdk/drivers/net/bonding/rte_eth_bond_alb.h index fd7c3aeb..9f17f7c8 100644 --- a/dpdk/drivers/net/bonding/rte_eth_bond_alb.h +++ b/dpdk/drivers/net/bonding/rte_eth_bond_alb.h @@ -51,7 +51,7 @@ struct client_data { uint32_t cli_ip; /**< Client IP address */ - uint8_t slave_idx; + uint16_t slave_idx; /**< Index of slave on which we connect with that client */ uint8_t in_use; /**< Flag indicating if entry in client table is currently used */ @@ -113,7 +113,7 @@ bond_mode_alb_arp_recv(struct ether_hdr *eth_h, uint16_t offset, * @return * Index of slave on which packet should be sent. */ -uint8_t +uint16_t bond_mode_alb_arp_xmit(struct ether_hdr *eth_h, uint16_t offset, struct bond_dev_private *internals); @@ -127,7 +127,7 @@ bond_mode_alb_arp_xmit(struct ether_hdr *eth_h, uint16_t offset, * @return * Index of slawe on which packet should be sent. */ -uint8_t +uint16_t bond_mode_alb_arp_upd(struct client_data *client_info, struct rte_mbuf *pkt, struct bond_dev_private *internals); diff --git a/dpdk/drivers/net/bonding/rte_eth_bond_api.c b/dpdk/drivers/net/bonding/rte_eth_bond_api.c index 3c169730..b8340356 100644 --- a/dpdk/drivers/net/bonding/rte_eth_bond_api.c +++ b/dpdk/drivers/net/bonding/rte_eth_bond_api.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. + * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -37,47 +37,69 @@ #include #include #include +#include +#include #include "rte_eth_bond.h" #include "rte_eth_bond_private.h" #include "rte_eth_bond_8023ad_private.h" -#define DEFAULT_POLLING_INTERVAL_10_MS (10) - -const char pmd_bond_driver_name[] = "rte_bond_pmd"; - int check_for_bonded_ethdev(const struct rte_eth_dev *eth_dev) { /* Check valid pointer */ - if (eth_dev->data->drv_name == NULL) + if (eth_dev->device->driver->name == NULL) return -1; /* return 0 if driver name matches */ - return eth_dev->data->drv_name != pmd_bond_driver_name; + return eth_dev->device->driver->name != pmd_bond_drv.driver.name; } int -valid_bonded_port_id(uint8_t port_id) +valid_bonded_port_id(uint16_t port_id) { RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -1); return check_for_bonded_ethdev(&rte_eth_devices[port_id]); } int -valid_slave_port_id(uint8_t port_id) +check_for_master_bonded_ethdev(const struct rte_eth_dev *eth_dev) +{ + int i; + struct bond_dev_private *internals; + + if (check_for_bonded_ethdev(eth_dev) != 0) + return 0; + + internals = eth_dev->data->dev_private; + + /* Check if any of slave devices is a bonded device */ + for (i = 0; i < internals->slave_count; i++) + if (valid_bonded_port_id(internals->slaves[i].port_id) == 0) + return 1; + + return 0; +} + +int +valid_slave_port_id(uint16_t port_id, uint8_t mode) { RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -1); /* Verify that port_id refers to a non bonded port */ - if (check_for_bonded_ethdev(&rte_eth_devices[port_id]) == 0) + if (check_for_bonded_ethdev(&rte_eth_devices[port_id]) == 0 && + mode == BONDING_MODE_8023AD) { + RTE_BOND_LOG(ERR, "Cannot add slave to bonded device in 802.3ad" + " mode as slave is also a bonded device, only " + "physical devices can be support in this mode."); return -1; + } return 0; } void -activate_slave(struct rte_eth_dev *eth_dev, uint8_t port_id) +activate_slave(struct rte_eth_dev *eth_dev, uint16_t port_id) { struct bond_dev_private *internals = eth_dev->data->dev_private; uint8_t active_count = internals->active_slave_count; @@ -104,11 +126,11 @@ activate_slave(struct rte_eth_dev *eth_dev, uint8_t port_id) } void -deactivate_slave(struct rte_eth_dev *eth_dev, uint8_t port_id) +deactivate_slave(struct rte_eth_dev *eth_dev, uint16_t port_id) { - uint8_t slave_pos; + uint16_t slave_pos; struct bond_dev_private *internals = eth_dev->data->dev_private; - uint8_t active_count = internals->active_slave_count; + uint16_t active_count = internals->active_slave_count; if (internals->mode == BONDING_MODE_8023AD) { bond_mode_8023ad_stop(eth_dev); @@ -145,182 +167,104 @@ deactivate_slave(struct rte_eth_dev *eth_dev, uint8_t port_id) } } -uint8_t -number_of_sockets(void) -{ - int sockets = 0; - int i; - const struct rte_memseg *ms = rte_eal_get_physmem_layout(); - - for (i = 0; ((i < RTE_MAX_MEMSEG) && (ms[i].addr != NULL)); i++) { - if (sockets < ms[i].socket_id) - sockets = ms[i].socket_id; - } - - /* Number of sockets = maximum socket_id + 1 */ - return ++sockets; -} - int rte_eth_bond_create(const char *name, uint8_t mode, uint8_t socket_id) { - struct bond_dev_private *internals = NULL; - struct rte_eth_dev *eth_dev = NULL; - - /* now do all data allocation - for eth_dev structure, dummy pci driver - * and internal (private) data - */ + struct bond_dev_private *internals; + char devargs[52]; + uint16_t port_id; + int ret; if (name == NULL) { RTE_BOND_LOG(ERR, "Invalid name specified"); - goto err; + return -EINVAL; } - if (socket_id >= number_of_sockets()) { - RTE_BOND_LOG(ERR, - "Invalid socket id specified to create bonded device on."); - goto err; - } + ret = snprintf(devargs, sizeof(devargs), + "driver=net_bonding,mode=%d,socket_id=%d", mode, socket_id); + if (ret < 0 || ret >= (int)sizeof(devargs)) + return -ENOMEM; - internals = rte_zmalloc_socket(name, sizeof(*internals), 0, socket_id); - if (internals == NULL) { - RTE_BOND_LOG(ERR, "Unable to malloc internals on socket"); - goto err; - } + ret = rte_vdev_init(name, devargs); + if (ret) + return -ENOMEM; - /* reserve an ethdev entry */ - eth_dev = rte_eth_dev_allocate(name, RTE_ETH_DEV_VIRTUAL); - if (eth_dev == NULL) { - RTE_BOND_LOG(ERR, "Unable to allocate rte_eth_dev"); - goto err; - } + ret = rte_eth_dev_get_port_by_name(name, &port_id); + RTE_ASSERT(!ret); - eth_dev->data->dev_private = internals; - eth_dev->data->nb_rx_queues = (uint16_t)1; - eth_dev->data->nb_tx_queues = (uint16_t)1; + /* + * To make bond_ethdev_configure() happy we need to free the + * internals->kvlist here. + * + * Also see comment in bond_ethdev_configure(). + */ + internals = rte_eth_devices[port_id].data->dev_private; + rte_kvargs_free(internals->kvlist); + internals->kvlist = NULL; - TAILQ_INIT(&(eth_dev->link_intr_cbs)); - - eth_dev->data->dev_link.link_status = ETH_LINK_DOWN; - - eth_dev->data->mac_addrs = rte_zmalloc_socket(name, ETHER_ADDR_LEN, 0, - socket_id); - if (eth_dev->data->mac_addrs == NULL) { - RTE_BOND_LOG(ERR, "Unable to malloc mac_addrs"); - goto err; - } - - eth_dev->data->dev_started = 0; - eth_dev->data->promiscuous = 0; - eth_dev->data->scattered_rx = 0; - eth_dev->data->all_multicast = 0; - - eth_dev->dev_ops = &default_dev_ops; - eth_dev->data->dev_flags = RTE_ETH_DEV_INTR_LSC | - RTE_ETH_DEV_DETACHABLE; - eth_dev->driver = NULL; - eth_dev->data->kdrv = RTE_KDRV_NONE; - eth_dev->data->drv_name = pmd_bond_driver_name; - eth_dev->data->numa_node = socket_id; - - rte_spinlock_init(&internals->lock); - - internals->port_id = eth_dev->data->port_id; - internals->mode = BONDING_MODE_INVALID; - internals->current_primary_port = RTE_MAX_ETHPORTS + 1; - internals->balance_xmit_policy = BALANCE_XMIT_POLICY_LAYER2; - internals->xmit_hash = xmit_l2_hash; - internals->user_defined_mac = 0; - internals->link_props_set = 0; - - internals->link_status_polling_enabled = 0; - - internals->link_status_polling_interval_ms = DEFAULT_POLLING_INTERVAL_10_MS; - internals->link_down_delay_ms = 0; - internals->link_up_delay_ms = 0; - - internals->slave_count = 0; - internals->active_slave_count = 0; - internals->rx_offload_capa = 0; - internals->tx_offload_capa = 0; - internals->candidate_max_rx_pktlen = 0; - internals->max_rx_pktlen = 0; - - /* Initially allow to choose any offload type */ - internals->flow_type_rss_offloads = ETH_RSS_PROTO_MASK; - - memset(internals->active_slaves, 0, sizeof(internals->active_slaves)); - memset(internals->slaves, 0, sizeof(internals->slaves)); - - /* Set mode 4 default configuration */ - bond_mode_8023ad_setup(eth_dev, NULL); - if (bond_ethdev_mode_set(eth_dev, mode)) { - RTE_BOND_LOG(ERR, "Failed to set bonded device %d mode too %d", - eth_dev->data->port_id, mode); - goto err; - } - - return eth_dev->data->port_id; - -err: - rte_free(internals); - if (eth_dev != NULL) { - rte_free(eth_dev->data->mac_addrs); - rte_eth_dev_release_port(eth_dev); - } - return -1; + return port_id; } int rte_eth_bond_free(const char *name) { - struct rte_eth_dev *eth_dev = NULL; - struct bond_dev_private *internals; - - /* now free all data allocation - for eth_dev structure, - * dummy pci driver and internal (private) data - */ - - /* find an ethdev entry */ - eth_dev = rte_eth_dev_allocated(name); - if (eth_dev == NULL) - return -ENODEV; - - internals = eth_dev->data->dev_private; - if (internals->slave_count != 0) - return -EBUSY; - - if (eth_dev->data->dev_started == 1) { - bond_ethdev_stop(eth_dev); - bond_ethdev_close(eth_dev); - } - - eth_dev->dev_ops = NULL; - eth_dev->rx_pkt_burst = NULL; - eth_dev->tx_pkt_burst = NULL; - - rte_free(eth_dev->data->dev_private); - rte_free(eth_dev->data->mac_addrs); - - rte_eth_dev_release_port(eth_dev); - - return 0; + return rte_vdev_uninit(name); } static int -__eth_bond_slave_add_lock_free(uint8_t bonded_port_id, uint8_t slave_port_id) +slave_vlan_filter_set(uint16_t bonded_port_id, uint16_t slave_port_id) +{ + struct rte_eth_dev *bonded_eth_dev; + struct bond_dev_private *internals; + int found; + int res = 0; + uint64_t slab = 0; + uint32_t pos = 0; + uint16_t first; + + bonded_eth_dev = &rte_eth_devices[bonded_port_id]; + if (bonded_eth_dev->data->dev_conf.rxmode.hw_vlan_filter == 0) + return 0; + + internals = bonded_eth_dev->data->dev_private; + found = rte_bitmap_scan(internals->vlan_filter_bmp, &pos, &slab); + first = pos; + + if (!found) + return 0; + + do { + uint32_t i; + uint64_t mask; + + for (i = 0, mask = 1; + i < RTE_BITMAP_SLAB_BIT_SIZE; + i ++, mask <<= 1) { + if (unlikely(slab & mask)) + res = rte_eth_dev_vlan_filter(slave_port_id, + (uint16_t)pos, 1); + } + found = rte_bitmap_scan(internals->vlan_filter_bmp, + &pos, &slab); + } while (found && first != pos && res == 0); + + return res; +} + +static int +__eth_bond_slave_add_lock_free(uint16_t bonded_port_id, uint16_t slave_port_id) { struct rte_eth_dev *bonded_eth_dev, *slave_eth_dev; struct bond_dev_private *internals; struct rte_eth_link link_props; struct rte_eth_dev_info dev_info; - if (valid_slave_port_id(slave_port_id) != 0) - return -1; - bonded_eth_dev = &rte_eth_devices[bonded_port_id]; internals = bonded_eth_dev->data->dev_private; + if (valid_slave_port_id(slave_port_id, internals->mode) != 0) + return -1; + slave_eth_dev = &rte_eth_devices[slave_port_id]; if (slave_eth_dev->data->dev_flags & RTE_ETH_DEV_BONDED_SLAVE) { RTE_BOND_LOG(ERR, "Slave device is already a slave of a bonded device"); @@ -347,8 +291,13 @@ __eth_bond_slave_add_lock_free(uint8_t bonded_port_id, uint8_t slave_port_id) if (internals->slave_count < 1) { /* if MAC is not user defined then use MAC of first slave add to * bonded device */ - if (!internals->user_defined_mac) - mac_address_set(bonded_eth_dev, slave_eth_dev->data->mac_addrs); + if (!internals->user_defined_mac) { + if (mac_address_set(bonded_eth_dev, + slave_eth_dev->data->mac_addrs)) { + RTE_BOND_LOG(ERR, "Failed to set MAC address"); + return -1; + } + } /* Inherit eth dev link properties from first slave */ link_properties_set(bonded_eth_dev, @@ -377,6 +326,14 @@ __eth_bond_slave_add_lock_free(uint8_t bonded_port_id, uint8_t slave_port_id) internals->tx_offload_capa &= dev_info.tx_offload_capa; internals->flow_type_rss_offloads &= dev_info.flow_type_rss_offloads; + if (link_properties_valid(bonded_eth_dev, + &slave_eth_dev->data->dev_link) != 0) { + RTE_BOND_LOG(ERR, "Invalid link properties for slave %d" + " in bonding mode %d", slave_port_id, + internals->mode); + return -1; + } + /* RETA size is GCD of all slaves RETA sizes, so, if all sizes will be * the power of 2, the lower one is GCD */ @@ -427,12 +384,15 @@ __eth_bond_slave_add_lock_free(uint8_t bonded_port_id, uint8_t slave_port_id) activate_slave(bonded_eth_dev, slave_port_id); } } + + slave_vlan_filter_set(bonded_port_id, slave_port_id); + return 0; } int -rte_eth_bond_slave_add(uint8_t bonded_port_id, uint8_t slave_port_id) +rte_eth_bond_slave_add(uint16_t bonded_port_id, uint16_t slave_port_id) { struct rte_eth_dev *bonded_eth_dev; struct bond_dev_private *internals; @@ -456,19 +416,20 @@ rte_eth_bond_slave_add(uint8_t bonded_port_id, uint8_t slave_port_id) } static int -__eth_bond_slave_remove_lock_free(uint8_t bonded_port_id, uint8_t slave_port_id) +__eth_bond_slave_remove_lock_free(uint16_t bonded_port_id, + uint16_t slave_port_id) { struct rte_eth_dev *bonded_eth_dev; struct bond_dev_private *internals; struct rte_eth_dev *slave_eth_dev; int i, slave_idx; - if (valid_slave_port_id(slave_port_id) != 0) - return -1; - bonded_eth_dev = &rte_eth_devices[bonded_port_id]; internals = bonded_eth_dev->data->dev_private; + if (valid_slave_port_id(slave_port_id, internals->mode) < 0) + return -1; + /* first remove from active slave list */ slave_idx = find_slave_by_id(internals->active_slaves, internals->active_slave_count, slave_port_id); @@ -497,7 +458,7 @@ __eth_bond_slave_remove_lock_free(uint8_t bonded_port_id, uint8_t slave_port_id) &rte_eth_devices[bonded_port_id].data->port_id); /* Restore original MAC address of slave device */ - mac_address_set(&rte_eth_devices[slave_port_id], + rte_eth_dev_default_mac_addr_set(slave_port_id, &(internals->slaves[slave_idx].persisted_mac_addr)); slave_eth_dev = &rte_eth_devices[slave_port_id]; @@ -516,9 +477,6 @@ __eth_bond_slave_remove_lock_free(uint8_t bonded_port_id, uint8_t slave_port_id) } if (internals->active_slave_count < 1) { - /* reset device link properties as no slaves are active */ - link_properties_reset(&rte_eth_devices[bonded_port_id]); - /* if no slaves are any longer attached to bonded device and MAC is not * user defined then clear MAC of bonded device as it will be reset * when a new slave is added */ @@ -538,7 +496,7 @@ __eth_bond_slave_remove_lock_free(uint8_t bonded_port_id, uint8_t slave_port_id) } int -rte_eth_bond_slave_remove(uint8_t bonded_port_id, uint8_t slave_port_id) +rte_eth_bond_slave_remove(uint16_t bonded_port_id, uint16_t slave_port_id) { struct rte_eth_dev *bonded_eth_dev; struct bond_dev_private *internals; @@ -560,16 +518,24 @@ rte_eth_bond_slave_remove(uint8_t bonded_port_id, uint8_t slave_port_id) } int -rte_eth_bond_mode_set(uint8_t bonded_port_id, uint8_t mode) +rte_eth_bond_mode_set(uint16_t bonded_port_id, uint8_t mode) { + struct rte_eth_dev *bonded_eth_dev; + if (valid_bonded_port_id(bonded_port_id) != 0) return -1; - return bond_ethdev_mode_set(&rte_eth_devices[bonded_port_id], mode); + bonded_eth_dev = &rte_eth_devices[bonded_port_id]; + + if (check_for_master_bonded_ethdev(bonded_eth_dev) != 0 && + mode == BONDING_MODE_8023AD) + return -1; + + return bond_ethdev_mode_set(bonded_eth_dev, mode); } int -rte_eth_bond_mode_get(uint8_t bonded_port_id) +rte_eth_bond_mode_get(uint16_t bonded_port_id) { struct bond_dev_private *internals; @@ -582,17 +548,17 @@ rte_eth_bond_mode_get(uint8_t bonded_port_id) } int -rte_eth_bond_primary_set(uint8_t bonded_port_id, uint8_t slave_port_id) +rte_eth_bond_primary_set(uint16_t bonded_port_id, uint16_t slave_port_id) { struct bond_dev_private *internals; if (valid_bonded_port_id(bonded_port_id) != 0) return -1; - if (valid_slave_port_id(slave_port_id) != 0) - return -1; + internals = rte_eth_devices[bonded_port_id].data->dev_private; - internals = rte_eth_devices[bonded_port_id].data->dev_private; + if (valid_slave_port_id(slave_port_id, internals->mode) != 0) + return -1; internals->user_defined_primary_port = 1; internals->primary_port = slave_port_id; @@ -603,7 +569,7 @@ rte_eth_bond_primary_set(uint8_t bonded_port_id, uint8_t slave_port_id) } int -rte_eth_bond_primary_get(uint8_t bonded_port_id) +rte_eth_bond_primary_get(uint16_t bonded_port_id) { struct bond_dev_private *internals; @@ -619,7 +585,8 @@ rte_eth_bond_primary_get(uint8_t bonded_port_id) } int -rte_eth_bond_slaves_get(uint8_t bonded_port_id, uint8_t slaves[], uint8_t len) +rte_eth_bond_slaves_get(uint16_t bonded_port_id, uint16_t slaves[], + uint16_t len) { struct bond_dev_private *internals; uint8_t i; @@ -642,8 +609,8 @@ rte_eth_bond_slaves_get(uint8_t bonded_port_id, uint8_t slaves[], uint8_t len) } int -rte_eth_bond_active_slaves_get(uint8_t bonded_port_id, uint8_t slaves[], - uint8_t len) +rte_eth_bond_active_slaves_get(uint16_t bonded_port_id, uint16_t slaves[], + uint16_t len) { struct bond_dev_private *internals; @@ -658,13 +625,14 @@ rte_eth_bond_active_slaves_get(uint8_t bonded_port_id, uint8_t slaves[], if (internals->active_slave_count > len) return -1; - memcpy(slaves, internals->active_slaves, internals->active_slave_count); + memcpy(slaves, internals->active_slaves, + internals->active_slave_count * sizeof(internals->active_slaves[0])); return internals->active_slave_count; } int -rte_eth_bond_mac_address_set(uint8_t bonded_port_id, +rte_eth_bond_mac_address_set(uint16_t bonded_port_id, struct ether_addr *mac_addr) { struct rte_eth_dev *bonded_eth_dev; @@ -690,7 +658,7 @@ rte_eth_bond_mac_address_set(uint8_t bonded_port_id, } int -rte_eth_bond_mac_address_reset(uint8_t bonded_port_id) +rte_eth_bond_mac_address_reset(uint16_t bonded_port_id) { struct rte_eth_dev *bonded_eth_dev; struct bond_dev_private *internals; @@ -719,7 +687,7 @@ rte_eth_bond_mac_address_reset(uint8_t bonded_port_id) } int -rte_eth_bond_xmit_policy_set(uint8_t bonded_port_id, uint8_t policy) +rte_eth_bond_xmit_policy_set(uint16_t bonded_port_id, uint8_t policy) { struct bond_dev_private *internals; @@ -749,7 +717,7 @@ rte_eth_bond_xmit_policy_set(uint8_t bonded_port_id, uint8_t policy) } int -rte_eth_bond_xmit_policy_get(uint8_t bonded_port_id) +rte_eth_bond_xmit_policy_get(uint16_t bonded_port_id) { struct bond_dev_private *internals; @@ -762,7 +730,7 @@ rte_eth_bond_xmit_policy_get(uint8_t bonded_port_id) } int -rte_eth_bond_link_monitoring_set(uint8_t bonded_port_id, uint32_t internal_ms) +rte_eth_bond_link_monitoring_set(uint16_t bonded_port_id, uint32_t internal_ms) { struct bond_dev_private *internals; @@ -776,7 +744,7 @@ rte_eth_bond_link_monitoring_set(uint8_t bonded_port_id, uint32_t internal_ms) } int -rte_eth_bond_link_monitoring_get(uint8_t bonded_port_id) +rte_eth_bond_link_monitoring_get(uint16_t bonded_port_id) { struct bond_dev_private *internals; @@ -789,7 +757,8 @@ rte_eth_bond_link_monitoring_get(uint8_t bonded_port_id) } int -rte_eth_bond_link_down_prop_delay_set(uint8_t bonded_port_id, uint32_t delay_ms) +rte_eth_bond_link_down_prop_delay_set(uint16_t bonded_port_id, + uint32_t delay_ms) { struct bond_dev_private *internals; @@ -804,7 +773,7 @@ rte_eth_bond_link_down_prop_delay_set(uint8_t bonded_port_id, uint32_t delay_ms) } int -rte_eth_bond_link_down_prop_delay_get(uint8_t bonded_port_id) +rte_eth_bond_link_down_prop_delay_get(uint16_t bonded_port_id) { struct bond_dev_private *internals; @@ -817,7 +786,7 @@ rte_eth_bond_link_down_prop_delay_get(uint8_t bonded_port_id) } int -rte_eth_bond_link_up_prop_delay_set(uint8_t bonded_port_id, uint32_t delay_ms) +rte_eth_bond_link_up_prop_delay_set(uint16_t bonded_port_id, uint32_t delay_ms) { struct bond_dev_private *internals; @@ -832,7 +801,7 @@ rte_eth_bond_link_up_prop_delay_set(uint8_t bonded_port_id, uint32_t delay_ms) } int -rte_eth_bond_link_up_prop_delay_get(uint8_t bonded_port_id) +rte_eth_bond_link_up_prop_delay_get(uint16_t bonded_port_id) { struct bond_dev_private *internals; diff --git a/dpdk/drivers/net/bonding/rte_eth_bond_args.c b/dpdk/drivers/net/bonding/rte_eth_bond_args.c index 02ecde64..e816da31 100644 --- a/dpdk/drivers/net/bonding/rte_eth_bond_args.c +++ b/dpdk/drivers/net/bonding/rte_eth_bond_args.c @@ -32,6 +32,8 @@ */ #include +#include +#include #include #include @@ -47,22 +49,21 @@ const char *pmd_bond_init_valid_arguments[] = { PMD_BOND_XMIT_POLICY_KVARG, PMD_BOND_SOCKET_ID_KVARG, PMD_BOND_MAC_ADDR_KVARG, - + PMD_BOND_AGG_MODE_KVARG, + "driver", NULL }; static inline int find_port_id_by_pci_addr(const struct rte_pci_addr *pci_addr) { + struct rte_pci_device *pci_dev; struct rte_pci_addr *eth_pci_addr; unsigned i; for (i = 0; i < rte_eth_dev_count(); i++) { - - if (rte_eth_devices[i].pci_dev == NULL) - continue; - - eth_pci_addr = &(rte_eth_devices[i].pci_dev->addr); + pci_dev = RTE_ETH_DEV_TO_PCI(&rte_eth_devices[i]); + eth_pci_addr = &pci_dev->addr; if (pci_addr->bus == eth_pci_addr->bus && pci_addr->devid == eth_pci_addr->devid && @@ -82,12 +83,22 @@ find_port_id_by_dev_name(const char *name) if (rte_eth_devices[i].data == NULL) continue; - if (strcmp(rte_eth_devices[i].data->name, name) == 0) + if (strcmp(rte_eth_devices[i].device->name, name) == 0) return i; } return -1; } +static inline int +bond_pci_addr_cmp(const struct rte_device *dev, const void *_pci_addr) +{ + struct rte_pci_device *pdev; + const struct rte_pci_addr *paddr = _pci_addr; + + pdev = RTE_DEV_TO_PCI(*(struct rte_device **)(void *)&dev); + return rte_eal_compare_pci_addr(&pdev->addr, paddr); +} + /** * Parses a port identifier string to a port id by pci address, then by name, * and finally port id. @@ -96,10 +107,23 @@ static inline int parse_port_id(const char *port_str) { struct rte_pci_addr dev_addr; + struct rte_bus *pci_bus; + struct rte_device *dev; int port_id; + pci_bus = rte_bus_find_by_name("pci"); + if (pci_bus == NULL) { + RTE_LOG(ERR, PMD, "unable to find PCI bus\n"); + return -1; + } + /* try parsing as pci address, physical devices */ - if (eal_parse_pci_DomBDF(port_str, &dev_addr) == 0) { + if (pci_bus->parse(port_str, &dev_addr) == 0) { + dev = pci_bus->find_device(NULL, bond_pci_addr_cmp, &dev_addr); + if (dev == NULL) { + RTE_LOG(ERR, PMD, "unable to find PCI device\n"); + return -1; + } port_id = find_port_id_by_pci_addr(&dev_addr); if (port_id < 0) return -1; @@ -126,7 +150,7 @@ parse_port_id(const char *port_str) } int -bond_ethdev_parse_slave_port_kvarg(const char *key __rte_unused, +bond_ethdev_parse_slave_port_kvarg(const char *key, const char *value, void *extra_args) { struct bond_ethdev_slave_ports *slave_ports; @@ -143,7 +167,7 @@ bond_ethdev_parse_slave_port_kvarg(const char *key __rte_unused, return -1; } else slave_ports->slaves[slave_ports->slave_count++] = - (uint8_t)port_id; + port_id; } return 0; } @@ -181,6 +205,38 @@ bond_ethdev_parse_slave_mode_kvarg(const char *key __rte_unused, } } +int +bond_ethdev_parse_slave_agg_mode_kvarg(const char *key __rte_unused, + const char *value, void *extra_args) +{ + uint8_t *agg_mode; + + if (value == NULL || extra_args == NULL) + return -1; + + agg_mode = extra_args; + + errno = 0; + if (strncmp(value, "stable", 6) == 0) + *agg_mode = AGG_STABLE; + + if (strncmp(value, "bandwidth", 9) == 0) + *agg_mode = AGG_BANDWIDTH; + + if (strncmp(value, "count", 5) == 0) + *agg_mode = AGG_COUNT; + + switch (*agg_mode) { + case AGG_STABLE: + case AGG_BANDWIDTH: + case AGG_COUNT: + return 0; + default: + RTE_BOND_LOG(ERR, "Invalid agg mode value stable/bandwidth/count"); + return -1; + } +} + int bond_ethdev_parse_socket_id_kvarg(const char *key __rte_unused, const char *value, void *extra_args) @@ -196,8 +252,8 @@ bond_ethdev_parse_socket_id_kvarg(const char *key __rte_unused, if (*endptr != 0 || errno != 0) return -1; - /* validate mode value */ - if (socket_id >= 0 && socket_id < number_of_sockets()) { + /* validate socket id value */ + if (socket_id >= 0) { *(uint8_t *)extra_args = (uint8_t)socket_id; return 0; } diff --git a/dpdk/drivers/net/bonding/rte_eth_bond_pmd.c b/dpdk/drivers/net/bonding/rte_eth_bond_pmd.c index 25fe00a6..1d3fbeba 100644 --- a/dpdk/drivers/net/bonding/rte_eth_bond_pmd.c +++ b/dpdk/drivers/net/bonding/rte_eth_bond_pmd.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -36,13 +36,14 @@ #include #include #include +#include #include #include #include #include #include #include -#include +#include #include #include @@ -51,6 +52,7 @@ #include "rte_eth_bond_8023ad_private.h" #define REORDER_PERIOD_MS 10 +#define DEFAULT_POLLING_INTERVAL_10_MS (10) #define HASH_L4_PORTS(h) ((h)->src_port ^ (h)->dst_port) @@ -122,6 +124,275 @@ bond_ethdev_rx_burst_active_backup(void *queue, struct rte_mbuf **bufs, bd_rx_q->queue_id, bufs, nb_pkts); } +static inline uint8_t +is_lacp_packets(uint16_t ethertype, uint8_t subtype, struct rte_mbuf *mbuf) +{ + const uint16_t ether_type_slow_be = rte_be_to_cpu_16(ETHER_TYPE_SLOW); + + return !((mbuf->ol_flags & PKT_RX_VLAN) ? mbuf->vlan_tci : 0) && + (ethertype == ether_type_slow_be && + (subtype == SLOW_SUBTYPE_MARKER || subtype == SLOW_SUBTYPE_LACP)); +} + +/***************************************************************************** + * Flow director's setup for mode 4 optimization + */ + +static struct rte_flow_item_eth flow_item_eth_type_8023ad = { + .dst.addr_bytes = { 0 }, + .src.addr_bytes = { 0 }, + .type = RTE_BE16(ETHER_TYPE_SLOW), +}; + +static struct rte_flow_item_eth flow_item_eth_mask_type_8023ad = { + .dst.addr_bytes = { 0 }, + .src.addr_bytes = { 0 }, + .type = 0xFFFF, +}; + +static struct rte_flow_item flow_item_8023ad[] = { + { + .type = RTE_FLOW_ITEM_TYPE_ETH, + .spec = &flow_item_eth_type_8023ad, + .last = NULL, + .mask = &flow_item_eth_mask_type_8023ad, + }, + { + .type = RTE_FLOW_ITEM_TYPE_END, + .spec = NULL, + .last = NULL, + .mask = NULL, + } +}; + +const struct rte_flow_attr flow_attr_8023ad = { + .group = 0, + .priority = 0, + .ingress = 1, + .egress = 0, + .reserved = 0, +}; + +int +bond_ethdev_8023ad_flow_verify(struct rte_eth_dev *bond_dev, + uint16_t slave_port) { + struct rte_eth_dev_info slave_info; + struct rte_flow_error error; + struct bond_dev_private *internals = (struct bond_dev_private *) + (bond_dev->data->dev_private); + + const struct rte_flow_action_queue lacp_queue_conf = { + .index = 0, + }; + + const struct rte_flow_action actions[] = { + { + .type = RTE_FLOW_ACTION_TYPE_QUEUE, + .conf = &lacp_queue_conf + }, + { + .type = RTE_FLOW_ACTION_TYPE_END, + } + }; + + int ret = rte_flow_validate(slave_port, &flow_attr_8023ad, + flow_item_8023ad, actions, &error); + if (ret < 0) { + RTE_BOND_LOG(ERR, "%s: %s (slave_port=%d queue_id=%d)", + __func__, error.message, slave_port, + internals->mode4.dedicated_queues.rx_qid); + return -1; + } + + rte_eth_dev_info_get(slave_port, &slave_info); + if (slave_info.max_rx_queues < bond_dev->data->nb_rx_queues || + slave_info.max_tx_queues < bond_dev->data->nb_tx_queues) { + RTE_BOND_LOG(ERR, + "%s: Slave %d capabilities doesn't allow to allocate additional queues", + __func__, slave_port); + return -1; + } + + return 0; +} + +int +bond_8023ad_slow_pkt_hw_filter_supported(uint16_t port_id) { + struct rte_eth_dev *bond_dev = &rte_eth_devices[port_id]; + struct bond_dev_private *internals = (struct bond_dev_private *) + (bond_dev->data->dev_private); + struct rte_eth_dev_info bond_info; + uint16_t idx; + + /* Verify if all slaves in bonding supports flow director and */ + if (internals->slave_count > 0) { + rte_eth_dev_info_get(bond_dev->data->port_id, &bond_info); + + internals->mode4.dedicated_queues.rx_qid = bond_info.nb_rx_queues; + internals->mode4.dedicated_queues.tx_qid = bond_info.nb_tx_queues; + + for (idx = 0; idx < internals->slave_count; idx++) { + if (bond_ethdev_8023ad_flow_verify(bond_dev, + internals->slaves[idx].port_id) != 0) + return -1; + } + } + + return 0; +} + +int +bond_ethdev_8023ad_flow_set(struct rte_eth_dev *bond_dev, uint16_t slave_port) { + + struct rte_flow_error error; + struct bond_dev_private *internals = (struct bond_dev_private *) + (bond_dev->data->dev_private); + + struct rte_flow_action_queue lacp_queue_conf = { + .index = internals->mode4.dedicated_queues.rx_qid, + }; + + const struct rte_flow_action actions[] = { + { + .type = RTE_FLOW_ACTION_TYPE_QUEUE, + .conf = &lacp_queue_conf + }, + { + .type = RTE_FLOW_ACTION_TYPE_END, + } + }; + + internals->mode4.dedicated_queues.flow[slave_port] = rte_flow_create(slave_port, + &flow_attr_8023ad, flow_item_8023ad, actions, &error); + if (internals->mode4.dedicated_queues.flow[slave_port] == NULL) { + RTE_BOND_LOG(ERR, "bond_ethdev_8023ad_flow_set: %s " + "(slave_port=%d queue_id=%d)", + error.message, slave_port, + internals->mode4.dedicated_queues.rx_qid); + return -1; + } + + return 0; +} + +static uint16_t +bond_ethdev_rx_burst_8023ad_fast_queue(void *queue, struct rte_mbuf **bufs, + uint16_t nb_pkts) +{ + struct bond_rx_queue *bd_rx_q = (struct bond_rx_queue *)queue; + struct bond_dev_private *internals = bd_rx_q->dev_private; + uint16_t num_rx_total = 0; /* Total number of received packets */ + uint16_t slaves[RTE_MAX_ETHPORTS]; + uint16_t slave_count; + + uint16_t i, idx; + + /* Copy slave list to protect against slave up/down changes during tx + * bursting */ + slave_count = internals->active_slave_count; + memcpy(slaves, internals->active_slaves, + sizeof(internals->active_slaves[0]) * slave_count); + + for (i = 0, idx = internals->active_slave; + i < slave_count && num_rx_total < nb_pkts; i++, idx++) { + idx = idx % slave_count; + + /* Read packets from this slave */ + num_rx_total += rte_eth_rx_burst(slaves[idx], bd_rx_q->queue_id, + &bufs[num_rx_total], nb_pkts - num_rx_total); + } + + internals->active_slave = idx; + + return num_rx_total; +} + +static uint16_t +bond_ethdev_tx_burst_8023ad_fast_queue(void *queue, struct rte_mbuf **bufs, + uint16_t nb_pkts) +{ + struct bond_dev_private *internals; + struct bond_tx_queue *bd_tx_q; + + uint16_t num_of_slaves; + uint16_t slaves[RTE_MAX_ETHPORTS]; + /* positions in slaves, not ID */ + uint8_t distributing_offsets[RTE_MAX_ETHPORTS]; + uint8_t distributing_count; + + uint16_t num_tx_slave, num_tx_total = 0, num_tx_fail_total = 0; + uint16_t i, op_slave_idx; + + struct rte_mbuf *slave_bufs[RTE_MAX_ETHPORTS][nb_pkts]; + + /* Total amount of packets in slave_bufs */ + uint16_t slave_nb_pkts[RTE_MAX_ETHPORTS] = { 0 }; + /* Slow packets placed in each slave */ + + if (unlikely(nb_pkts == 0)) + return 0; + + bd_tx_q = (struct bond_tx_queue *)queue; + internals = bd_tx_q->dev_private; + + /* Copy slave list to protect against slave up/down changes during tx + * bursting */ + num_of_slaves = internals->active_slave_count; + if (num_of_slaves < 1) + return num_tx_total; + + memcpy(slaves, internals->active_slaves, sizeof(slaves[0]) * + num_of_slaves); + + distributing_count = 0; + for (i = 0; i < num_of_slaves; i++) { + struct port *port = &mode_8023ad_ports[slaves[i]]; + if (ACTOR_STATE(port, DISTRIBUTING)) + distributing_offsets[distributing_count++] = i; + } + + if (likely(distributing_count > 0)) { + /* Populate slaves mbuf with the packets which are to be sent */ + for (i = 0; i < nb_pkts; i++) { + /* Select output slave using hash based on xmit policy */ + op_slave_idx = internals->xmit_hash(bufs[i], + distributing_count); + + /* Populate slave mbuf arrays with mbufs for that slave. + * Use only slaves that are currently distributing. + */ + uint8_t slave_offset = + distributing_offsets[op_slave_idx]; + slave_bufs[slave_offset][slave_nb_pkts[slave_offset]] = + bufs[i]; + slave_nb_pkts[slave_offset]++; + } + } + + /* Send packet burst on each slave device */ + for (i = 0; i < num_of_slaves; i++) { + if (slave_nb_pkts[i] == 0) + continue; + + num_tx_slave = rte_eth_tx_burst(slaves[i], bd_tx_q->queue_id, + slave_bufs[i], slave_nb_pkts[i]); + + num_tx_total += num_tx_slave; + num_tx_fail_total += slave_nb_pkts[i] - num_tx_slave; + + /* If tx burst fails move packets to end of bufs */ + if (unlikely(num_tx_slave < slave_nb_pkts[i])) { + uint16_t j = nb_pkts - num_tx_fail_total; + for ( ; num_tx_slave < slave_nb_pkts[i]; j++, + num_tx_slave++) + bufs[j] = slave_bufs[i][num_tx_slave]; + } + } + + return num_tx_total; +} + + static uint16_t bond_ethdev_rx_burst_8023ad(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) @@ -135,12 +406,13 @@ bond_ethdev_rx_burst_8023ad(void *queue, struct rte_mbuf **bufs, const uint16_t ether_type_slow_be = rte_be_to_cpu_16(ETHER_TYPE_SLOW); uint16_t num_rx_total = 0; /* Total number of received packets */ - uint8_t slaves[RTE_MAX_ETHPORTS]; - uint8_t slave_count; + uint16_t slaves[RTE_MAX_ETHPORTS]; + uint16_t slave_count, idx; uint8_t collecting; /* current slave collecting status */ const uint8_t promisc = internals->promiscuous_en; uint8_t i, j, k; + uint8_t subtype; rte_eth_macaddr_get(internals->port_id, &bond_mac); /* Copy slave list to protect against slave up/down changes during tx @@ -149,12 +421,18 @@ bond_ethdev_rx_burst_8023ad(void *queue, struct rte_mbuf **bufs, memcpy(slaves, internals->active_slaves, sizeof(internals->active_slaves[0]) * slave_count); + idx = internals->active_slave; + if (idx >= slave_count) { + internals->active_slave = 0; + idx = 0; + } for (i = 0; i < slave_count && num_rx_total < nb_pkts; i++) { j = num_rx_total; - collecting = ACTOR_STATE(&mode_8023ad_ports[slaves[i]], COLLECTING); + collecting = ACTOR_STATE(&mode_8023ad_ports[slaves[idx]], + COLLECTING); /* Read packets from this slave */ - num_rx_total += rte_eth_rx_burst(slaves[i], bd_rx_q->queue_id, + num_rx_total += rte_eth_rx_burst(slaves[idx], bd_rx_q->queue_id, &bufs[num_rx_total], nb_pkts - num_rx_total); for (k = j; k < 2 && k < num_rx_total; k++) @@ -162,21 +440,30 @@ bond_ethdev_rx_burst_8023ad(void *queue, struct rte_mbuf **bufs, /* Handle slow protocol packets. */ while (j < num_rx_total) { + + /* If packet is not pure L2 and is known, skip it */ + if ((bufs[j]->packet_type & ~RTE_PTYPE_L2_ETHER) != 0) { + j++; + continue; + } + if (j + 3 < num_rx_total) rte_prefetch0(rte_pktmbuf_mtod(bufs[j + 3], void *)); hdr = rte_pktmbuf_mtod(bufs[j], struct ether_hdr *); + subtype = ((struct slow_protocol_frame *)hdr)->slow_protocol.subtype; + /* Remove packet from array if it is slow packet or slave is not - * in collecting state or bondign interface is not in promiscus + * in collecting state or bonding interface is not in promiscuous * mode and packet address does not match. */ - if (unlikely(hdr->ether_type == ether_type_slow_be || + if (unlikely(is_lacp_packets(hdr->ether_type, subtype, bufs[j]) || !collecting || (!promisc && !is_multicast_ether_addr(&hdr->d_addr) && !is_same_ether_addr(&bond_mac, &hdr->d_addr)))) { if (hdr->ether_type == ether_type_slow_be) { - bond_mode_8023ad_handle_slow_pkt(internals, slaves[i], - bufs[j]); + bond_mode_8023ad_handle_slow_pkt( + internals, slaves[idx], bufs[j]); } else rte_pktmbuf_free(bufs[j]); @@ -189,8 +476,11 @@ bond_ethdev_rx_burst_8023ad(void *queue, struct rte_mbuf **bufs, } else j++; } + if (unlikely(++idx == slave_count)) + idx = 0; } + internals->active_slave = idx; return num_rx_total; } @@ -248,7 +538,7 @@ ipv4_addr_to_dot(uint32_t be_ipv4_addr, char *buf, uint8_t buf_size) #define MAX_CLIENTS_NUMBER 128 uint8_t active_clients; struct client_stats_t { - uint8_t port; + uint16_t port; uint32_t ipv4_addr; uint32_t ipv4_rx_packets; uint32_t ipv4_tx_packets; @@ -256,7 +546,7 @@ struct client_stats_t { struct client_stats_t client_stats[MAX_CLIENTS_NUMBER]; static void -update_client_stats(uint32_t addr, uint8_t port, uint32_t *TXorRXindicator) +update_client_stats(uint32_t addr, uint16_t port, uint32_t *TXorRXindicator) { int i = 0; @@ -314,7 +604,7 @@ update_client_stats(uint32_t addr, uint8_t port, uint32_t *TXorRXindicator) static void mode6_debug(const char __attribute__((unused)) *info, struct ether_hdr *eth_h, - uint8_t port, uint32_t __attribute__((unused)) *burstnumber) + uint16_t port, uint32_t __attribute__((unused)) *burstnumber) { struct ipv4_hdr *ipv4_h; #ifdef RTE_LIBRTE_BOND_DEBUG_ALB @@ -395,8 +685,8 @@ bond_ethdev_tx_burst_round_robin(void *queue, struct rte_mbuf **bufs, struct rte_mbuf *slave_bufs[RTE_MAX_ETHPORTS][nb_pkts]; uint16_t slave_nb_pkts[RTE_MAX_ETHPORTS] = { 0 }; - uint8_t num_of_slaves; - uint8_t slaves[RTE_MAX_ETHPORTS]; + uint16_t num_of_slaves; + uint16_t slaves[RTE_MAX_ETHPORTS]; uint16_t num_tx_total = 0, num_tx_slave; @@ -626,12 +916,12 @@ bandwidth_cmp(const void *a, const void *b) } static void -bandwidth_left(uint8_t port_id, uint64_t load, uint8_t update_idx, +bandwidth_left(uint16_t port_id, uint64_t load, uint8_t update_idx, struct bwg_slave *bwg_slave) { struct rte_eth_link link_status; - rte_eth_link_get(port_id, &link_status); + rte_eth_link_get_nowait(port_id, &link_status); uint64_t link_bwg = link_status.link_speed * 1000000ULL / 8; if (link_bwg == 0) return; @@ -692,10 +982,10 @@ bond_ethdev_tx_burst_tlb(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) struct rte_eth_dev *primary_port = &rte_eth_devices[internals->primary_port]; uint16_t num_tx_total = 0; - uint8_t i, j; + uint16_t i, j; - uint8_t num_of_slaves = internals->active_slave_count; - uint8_t slaves[RTE_MAX_ETHPORTS]; + uint16_t num_of_slaves = internals->active_slave_count; + uint16_t slaves[RTE_MAX_ETHPORTS]; struct ether_hdr *ether_hdr; struct ether_addr primary_slave_addr; @@ -770,8 +1060,8 @@ bond_ethdev_tx_burst_alb(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) uint16_t slave_bufs_pkts[RTE_MAX_ETHPORTS + 1] = { 0 }; /* - * We create separate transmit buffers for update packets as they wont be - * counted in num_tx_total. + * We create separate transmit buffers for update packets as they won't + * be counted in num_tx_total. */ struct rte_mbuf *update_bufs[RTE_MAX_ETHPORTS][ALB_HASH_TABLE_SIZE]; uint16_t update_bufs_pkts[RTE_MAX_ETHPORTS] = { 0 }; @@ -781,7 +1071,7 @@ bond_ethdev_tx_burst_alb(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) uint16_t num_send, num_not_send = 0; uint16_t num_tx_total = 0; - uint8_t slave_idx; + uint16_t slave_idx; int i, j; @@ -888,7 +1178,6 @@ bond_ethdev_tx_burst_alb(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) } num_tx_total += num_send; - num_not_send += slave_bufs_pkts[RTE_MAX_ETHPORTS] - num_send; } return num_tx_total; @@ -901,8 +1190,8 @@ bond_ethdev_tx_burst_balance(void *queue, struct rte_mbuf **bufs, struct bond_dev_private *internals; struct bond_tx_queue *bd_tx_q; - uint8_t num_of_slaves; - uint8_t slaves[RTE_MAX_ETHPORTS]; + uint16_t num_of_slaves; + uint16_t slaves[RTE_MAX_ETHPORTS]; uint16_t num_tx_total = 0, num_tx_slave = 0, tx_fail_total = 0; @@ -962,8 +1251,8 @@ bond_ethdev_tx_burst_8023ad(void *queue, struct rte_mbuf **bufs, struct bond_dev_private *internals; struct bond_tx_queue *bd_tx_q; - uint8_t num_of_slaves; - uint8_t slaves[RTE_MAX_ETHPORTS]; + uint16_t num_of_slaves; + uint16_t slaves[RTE_MAX_ETHPORTS]; /* positions in slaves, not ID */ uint8_t distributing_offsets[RTE_MAX_ETHPORTS]; uint8_t distributing_count; @@ -997,7 +1286,8 @@ bond_ethdev_tx_burst_8023ad(void *queue, struct rte_mbuf **bufs, struct port *port = &mode_8023ad_ports[slaves[i]]; slave_slow_nb_pkts[i] = rte_ring_dequeue_burst(port->tx_ring, - slow_pkts, BOND_MODE_8023AX_SLAVE_TX_PKTS); + slow_pkts, BOND_MODE_8023AX_SLAVE_TX_PKTS, + NULL); slave_nb_pkts[i] = slave_slow_nb_pkts[i]; for (j = 0; j < slave_slow_nb_pkts[i]; j++) @@ -1055,7 +1345,7 @@ bond_ethdev_tx_burst_broadcast(void *queue, struct rte_mbuf **bufs, struct bond_tx_queue *bd_tx_q; uint8_t tx_failed_flag = 0, num_of_slaves; - uint8_t slaves[RTE_MAX_ETHPORTS]; + uint16_t slaves[RTE_MAX_ETHPORTS]; uint16_t max_nb_of_tx_pkts = 0; @@ -1108,39 +1398,44 @@ bond_ethdev_tx_burst_broadcast(void *queue, struct rte_mbuf **bufs, } void -link_properties_set(struct rte_eth_dev *bonded_eth_dev, - struct rte_eth_link *slave_dev_link) +link_properties_set(struct rte_eth_dev *ethdev, struct rte_eth_link *slave_link) { - struct rte_eth_link *bonded_dev_link = &bonded_eth_dev->data->dev_link; - struct bond_dev_private *internals = bonded_eth_dev->data->dev_private; + struct bond_dev_private *bond_ctx = ethdev->data->dev_private; - if (slave_dev_link->link_status && - bonded_eth_dev->data->dev_started) { - bonded_dev_link->link_duplex = slave_dev_link->link_duplex; - bonded_dev_link->link_speed = slave_dev_link->link_speed; + if (bond_ctx->mode == BONDING_MODE_8023AD) { + /** + * If in mode 4 then save the link properties of the first + * slave, all subsequent slaves must match these properties + */ + struct rte_eth_link *bond_link = &bond_ctx->mode4.slave_link; - internals->link_props_set = 1; + bond_link->link_autoneg = slave_link->link_autoneg; + bond_link->link_duplex = slave_link->link_duplex; + bond_link->link_speed = slave_link->link_speed; + } else { + /** + * In any other mode the link properties are set to default + * values of AUTONEG/DUPLEX + */ + ethdev->data->dev_link.link_autoneg = ETH_LINK_AUTONEG; + ethdev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX; } } -void -link_properties_reset(struct rte_eth_dev *bonded_eth_dev) -{ - struct bond_dev_private *internals = bonded_eth_dev->data->dev_private; - - memset(&(bonded_eth_dev->data->dev_link), 0, - sizeof(bonded_eth_dev->data->dev_link)); - - internals->link_props_set = 0; -} - int -link_properties_valid(struct rte_eth_link *bonded_dev_link, - struct rte_eth_link *slave_dev_link) +link_properties_valid(struct rte_eth_dev *ethdev, + struct rte_eth_link *slave_link) { - if (bonded_dev_link->link_duplex != slave_dev_link->link_duplex || - bonded_dev_link->link_speed != slave_dev_link->link_speed) - return -1; + struct bond_dev_private *bond_ctx = ethdev->data->dev_private; + + if (bond_ctx->mode == BONDING_MODE_8023AD) { + struct rte_eth_link *bond_link = &bond_ctx->mode4.slave_link; + + if (bond_link->link_duplex != slave_link->link_duplex || + bond_link->link_autoneg != slave_link->link_autoneg || + bond_link->link_speed != slave_link->link_speed) + return -1; + } return 0; } @@ -1205,7 +1500,8 @@ mac_address_slaves_update(struct rte_eth_dev *bonded_eth_dev) case BONDING_MODE_BALANCE: case BONDING_MODE_BROADCAST: for (i = 0; i < internals->slave_count; i++) { - if (mac_address_set(&rte_eth_devices[internals->slaves[i].port_id], + if (rte_eth_dev_default_mac_addr_set( + internals->slaves[i].port_id, bonded_eth_dev->data->mac_addrs)) { RTE_BOND_LOG(ERR, "Failed to update port Id %d MAC address", internals->slaves[i].port_id); @@ -1223,15 +1519,16 @@ mac_address_slaves_update(struct rte_eth_dev *bonded_eth_dev) for (i = 0; i < internals->slave_count; i++) { if (internals->slaves[i].port_id == internals->current_primary_port) { - if (mac_address_set(&rte_eth_devices[internals->primary_port], + if (rte_eth_dev_default_mac_addr_set( + internals->primary_port, bonded_eth_dev->data->mac_addrs)) { RTE_BOND_LOG(ERR, "Failed to update port Id %d MAC address", internals->current_primary_port); return -1; } } else { - if (mac_address_set( - &rte_eth_devices[internals->slaves[i].port_id], + if (rte_eth_dev_default_mac_addr_set( + internals->slaves[i].port_id, &internals->slaves[i].persisted_mac_addr)) { RTE_BOND_LOG(ERR, "Failed to update port Id %d MAC address", internals->slaves[i].port_id); @@ -1272,11 +1569,19 @@ bond_ethdev_mode_set(struct rte_eth_dev *eth_dev, int mode) if (bond_mode_8023ad_enable(eth_dev) != 0) return -1; - eth_dev->rx_pkt_burst = bond_ethdev_rx_burst_8023ad; - eth_dev->tx_pkt_burst = bond_ethdev_tx_burst_8023ad; - RTE_LOG(WARNING, PMD, - "Using mode 4, it is necessary to do TX burst and RX burst " - "at least every 100ms.\n"); + if (internals->mode4.dedicated_queues.enabled == 0) { + eth_dev->rx_pkt_burst = bond_ethdev_rx_burst_8023ad; + eth_dev->tx_pkt_burst = bond_ethdev_tx_burst_8023ad; + RTE_LOG(WARNING, PMD, + "Using mode 4, it is necessary to do TX burst " + "and RX burst at least every 100ms.\n"); + } else { + /* Use flow director's optimization */ + eth_dev->rx_pkt_burst = + bond_ethdev_rx_burst_8023ad_fast_queue; + eth_dev->tx_pkt_burst = + bond_ethdev_tx_burst_8023ad_fast_queue; + } break; case BONDING_MODE_TLB: eth_dev->tx_pkt_burst = bond_ethdev_tx_burst_tlb; @@ -1298,17 +1603,81 @@ bond_ethdev_mode_set(struct rte_eth_dev *eth_dev, int mode) return 0; } + +static int +slave_configure_slow_queue(struct rte_eth_dev *bonded_eth_dev, + struct rte_eth_dev *slave_eth_dev) +{ + int errval = 0; + struct bond_dev_private *internals = (struct bond_dev_private *) + bonded_eth_dev->data->dev_private; + struct port *port = &mode_8023ad_ports[slave_eth_dev->data->port_id]; + + if (port->slow_pool == NULL) { + char mem_name[256]; + int slave_id = slave_eth_dev->data->port_id; + + snprintf(mem_name, RTE_DIM(mem_name), "slave_port%u_slow_pool", + slave_id); + port->slow_pool = rte_pktmbuf_pool_create(mem_name, 8191, + 250, 0, RTE_MBUF_DEFAULT_BUF_SIZE, + slave_eth_dev->data->numa_node); + + /* Any memory allocation failure in initialization is critical because + * resources can't be free, so reinitialization is impossible. */ + if (port->slow_pool == NULL) { + rte_panic("Slave %u: Failed to create memory pool '%s': %s\n", + slave_id, mem_name, rte_strerror(rte_errno)); + } + } + + if (internals->mode4.dedicated_queues.enabled == 1) { + /* Configure slow Rx queue */ + + errval = rte_eth_rx_queue_setup(slave_eth_dev->data->port_id, + internals->mode4.dedicated_queues.rx_qid, 128, + rte_eth_dev_socket_id(slave_eth_dev->data->port_id), + NULL, port->slow_pool); + if (errval != 0) { + RTE_BOND_LOG(ERR, + "rte_eth_rx_queue_setup: port=%d queue_id %d, err (%d)", + slave_eth_dev->data->port_id, + internals->mode4.dedicated_queues.rx_qid, + errval); + return errval; + } + + errval = rte_eth_tx_queue_setup(slave_eth_dev->data->port_id, + internals->mode4.dedicated_queues.tx_qid, 512, + rte_eth_dev_socket_id(slave_eth_dev->data->port_id), + NULL); + if (errval != 0) { + RTE_BOND_LOG(ERR, + "rte_eth_tx_queue_setup: port=%d queue_id %d, err (%d)", + slave_eth_dev->data->port_id, + internals->mode4.dedicated_queues.tx_qid, + errval); + return errval; + } + } + return 0; +} + int slave_configure(struct rte_eth_dev *bonded_eth_dev, struct rte_eth_dev *slave_eth_dev) { struct bond_rx_queue *bd_rx_q; struct bond_tx_queue *bd_tx_q; + uint16_t nb_rx_queues; + uint16_t nb_tx_queues; - uint16_t old_nb_tx_queues = slave_eth_dev->data->nb_tx_queues; - uint16_t old_nb_rx_queues = slave_eth_dev->data->nb_rx_queues; int errval; uint16_t q_id; + struct rte_flow_error flow_error; + + struct bond_dev_private *internals = (struct bond_dev_private *) + bonded_eth_dev->data->dev_private; /* Stop slave */ rte_eth_dev_stop(slave_eth_dev->data->port_id); @@ -1335,10 +1704,22 @@ slave_configure(struct rte_eth_dev *bonded_eth_dev, bonded_eth_dev->data->dev_conf.rxmode.mq_mode; } + slave_eth_dev->data->dev_conf.rxmode.hw_vlan_filter = + bonded_eth_dev->data->dev_conf.rxmode.hw_vlan_filter; + + nb_rx_queues = bonded_eth_dev->data->nb_rx_queues; + nb_tx_queues = bonded_eth_dev->data->nb_tx_queues; + + if (internals->mode == BONDING_MODE_8023AD) { + if (internals->mode4.dedicated_queues.enabled == 1) { + nb_rx_queues++; + nb_tx_queues++; + } + } + /* Configure device */ errval = rte_eth_dev_configure(slave_eth_dev->data->port_id, - bonded_eth_dev->data->nb_rx_queues, - bonded_eth_dev->data->nb_tx_queues, + nb_rx_queues, nb_tx_queues, &(slave_eth_dev->data->dev_conf)); if (errval != 0) { RTE_BOND_LOG(ERR, "Cannot configure slave device: port %u , err (%d)", @@ -1347,9 +1728,7 @@ slave_configure(struct rte_eth_dev *bonded_eth_dev, } /* Setup Rx Queues */ - /* Use existing queues, if any */ - for (q_id = old_nb_rx_queues; - q_id < bonded_eth_dev->data->nb_rx_queues; q_id++) { + for (q_id = 0; q_id < bonded_eth_dev->data->nb_rx_queues; q_id++) { bd_rx_q = (struct bond_rx_queue *)bonded_eth_dev->data->rx_queues[q_id]; errval = rte_eth_rx_queue_setup(slave_eth_dev->data->port_id, q_id, @@ -1365,9 +1744,7 @@ slave_configure(struct rte_eth_dev *bonded_eth_dev, } /* Setup Tx Queues */ - /* Use existing queues, if any */ - for (q_id = old_nb_tx_queues; - q_id < bonded_eth_dev->data->nb_tx_queues; q_id++) { + for (q_id = 0; q_id < bonded_eth_dev->data->nb_tx_queues; q_id++) { bd_tx_q = (struct bond_tx_queue *)bonded_eth_dev->data->tx_queues[q_id]; errval = rte_eth_tx_queue_setup(slave_eth_dev->data->port_id, q_id, @@ -1376,12 +1753,35 @@ slave_configure(struct rte_eth_dev *bonded_eth_dev, &bd_tx_q->tx_conf); if (errval != 0) { RTE_BOND_LOG(ERR, - "rte_eth_tx_queue_setup: port=%d queue_id %d, err (%d)", - slave_eth_dev->data->port_id, q_id, errval); + "rte_eth_tx_queue_setup: port=%d queue_id %d, err (%d)", + slave_eth_dev->data->port_id, q_id, errval); return errval; } } + if (internals->mode == BONDING_MODE_8023AD && + internals->mode4.dedicated_queues.enabled == 1) { + if (slave_configure_slow_queue(bonded_eth_dev, slave_eth_dev) + != 0) + return errval; + + if (bond_ethdev_8023ad_flow_verify(bonded_eth_dev, + slave_eth_dev->data->port_id) != 0) { + RTE_BOND_LOG(ERR, + "rte_eth_tx_queue_setup: port=%d queue_id %d, err (%d)", + slave_eth_dev->data->port_id, q_id, errval); + return -1; + } + + if (internals->mode4.dedicated_queues.flow[slave_eth_dev->data->port_id] != NULL) + rte_flow_destroy(slave_eth_dev->data->port_id, + internals->mode4.dedicated_queues.flow[slave_eth_dev->data->port_id], + &flow_error); + + bond_ethdev_8023ad_flow_set(bonded_eth_dev, + slave_eth_dev->data->port_id); + } + /* Start device */ errval = rte_eth_dev_start(slave_eth_dev->data->port_id); if (errval != 0) { @@ -1415,9 +1815,12 @@ slave_configure(struct rte_eth_dev *bonded_eth_dev, } /* If lsc interrupt is set, check initial slave's link status */ - if (slave_eth_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) + if (slave_eth_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) { + slave_eth_dev->dev_ops->link_update(slave_eth_dev, 0); bond_ethdev_lsc_event_callback(slave_eth_dev->data->port_id, - RTE_ETH_EVENT_INTR_LSC, &bonded_eth_dev->data->port_id); + RTE_ETH_EVENT_INTR_LSC, &bonded_eth_dev->data->port_id, + NULL); + } return 0; } @@ -1439,6 +1842,9 @@ slave_remove(struct bond_dev_private *internals, (internals->slave_count - i - 1)); internals->slave_count--; + + /* force reconfiguration of slave interfaces */ + _rte_eth_dev_reset(slave_eth_dev); } static void @@ -1469,7 +1875,7 @@ slave_add(struct bond_dev_private *internals, void bond_ethdev_primary_set(struct bond_dev_private *internals, - uint8_t slave_port_id) + uint16_t slave_port_id) { int i; @@ -1534,13 +1940,25 @@ bond_ethdev_start(struct rte_eth_dev *eth_dev) if (internals->promiscuous_en) bond_ethdev_promiscuous_enable(eth_dev); + if (internals->mode == BONDING_MODE_8023AD) { + if (internals->mode4.dedicated_queues.enabled == 1) { + internals->mode4.dedicated_queues.rx_qid = + eth_dev->data->nb_rx_queues; + internals->mode4.dedicated_queues.tx_qid = + eth_dev->data->nb_tx_queues; + } + } + + /* Reconfigure each slave device if starting bonded device */ for (i = 0; i < internals->slave_count; i++) { - if (slave_configure(eth_dev, - &(rte_eth_devices[internals->slaves[i].port_id])) != 0) { + struct rte_eth_dev *slave_ethdev = + &(rte_eth_devices[internals->slaves[i].port_id]); + if (slave_configure(eth_dev, slave_ethdev) != 0) { RTE_BOND_LOG(ERR, - "bonded port (%d) failed to reconfigure slave device (%d)", - eth_dev->data->port_id, internals->slaves[i].port_id); + "bonded port (%d) failed to reconfigure slave device (%d)", + eth_dev->data->port_id, + internals->slaves[i].port_id); return -1; } /* We will need to poll for link status if any slave doesn't @@ -1637,7 +2055,25 @@ bond_ethdev_stop(struct rte_eth_dev *eth_dev) void bond_ethdev_close(struct rte_eth_dev *dev) { + struct bond_dev_private *internals = dev->data->dev_private; + uint8_t bond_port_id = internals->port_id; + int skipped = 0; + + RTE_LOG(INFO, EAL, "Closing bonded device %s\n", dev->device->name); + while (internals->slave_count != skipped) { + uint16_t port_id = internals->slaves[skipped].port_id; + + rte_eth_dev_stop(port_id); + + if (rte_eth_bond_slave_remove(bond_port_id, port_id) != 0) { + RTE_LOG(ERR, EAL, + "Failed to remove port %d from bonded device " + "%s\n", port_id, dev->device->name); + skipped++; + } + } bond_ethdev_free_queues(dev); + rte_bitmap_reset(internals->vlan_filter_bmp); } /* forward declaration */ @@ -1648,16 +2084,49 @@ bond_ethdev_info(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) { struct bond_dev_private *internals = dev->data->dev_private; + uint16_t max_nb_rx_queues = UINT16_MAX; + uint16_t max_nb_tx_queues = UINT16_MAX; + dev_info->max_mac_addrs = 1; dev_info->max_rx_pktlen = internals->candidate_max_rx_pktlen ? - internals->candidate_max_rx_pktlen : 2048; + internals->candidate_max_rx_pktlen : + ETHER_MAX_JUMBO_FRAME_LEN; - dev_info->max_rx_queues = (uint16_t)128; - dev_info->max_tx_queues = (uint16_t)512; + /* Max number of tx/rx queues that the bonded device can support is the + * minimum values of the bonded slaves, as all slaves must be capable + * of supporting the same number of tx/rx queues. + */ + if (internals->slave_count > 0) { + struct rte_eth_dev_info slave_info; + uint8_t idx; + + for (idx = 0; idx < internals->slave_count; idx++) { + rte_eth_dev_info_get(internals->slaves[idx].port_id, + &slave_info); + + if (slave_info.max_rx_queues < max_nb_rx_queues) + max_nb_rx_queues = slave_info.max_rx_queues; + + if (slave_info.max_tx_queues < max_nb_tx_queues) + max_nb_tx_queues = slave_info.max_tx_queues; + } + } + + dev_info->max_rx_queues = max_nb_rx_queues; + dev_info->max_tx_queues = max_nb_tx_queues; + + /** + * If dedicated hw queues enabled for link bonding device in LACP mode + * then we need to reduce the maximum number of data path queues by 1. + */ + if (internals->mode == BONDING_MODE_8023AD && + internals->mode4.dedicated_queues.enabled == 1) { + dev_info->max_rx_queues--; + dev_info->max_tx_queues--; + } dev_info->min_rx_bufsize = 0; - dev_info->pci_dev = NULL; dev_info->rx_offload_capa = internals->rx_offload_capa; dev_info->tx_offload_capa = internals->tx_offload_capa; @@ -1666,6 +2135,35 @@ bond_ethdev_info(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) dev_info->reta_size = internals->reta_size; } +static int +bond_ethdev_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) +{ + int res; + uint16_t i; + struct bond_dev_private *internals = dev->data->dev_private; + + /* don't do this while a slave is being added */ + rte_spinlock_lock(&internals->lock); + + if (on) + rte_bitmap_set(internals->vlan_filter_bmp, vlan_id); + else + rte_bitmap_clear(internals->vlan_filter_bmp, vlan_id); + + for (i = 0; i < internals->slave_count; i++) { + uint16_t port_id = internals->slaves[i].port_id; + + res = rte_eth_dev_vlan_filter(port_id, vlan_id, on); + if (res == ENOTSUP) + RTE_LOG(WARNING, PMD, + "Setting VLAN filter on slave port %u not supported.\n", + port_id); + } + + rte_spinlock_unlock(&internals->lock); + return 0; +} + static int bond_ethdev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t rx_queue_id, uint16_t nb_rx_desc, unsigned int socket_id __rte_unused, @@ -1777,7 +2275,8 @@ bond_ethdev_slave_link_status_change_monitor(void *cb_arg) bond_ethdev_lsc_event_callback(internals->slaves[i].port_id, RTE_ETH_EVENT_INTR_LSC, - &bonded_ethdev->data->port_id); + &bonded_ethdev->data->port_id, + NULL); } } rte_spinlock_unlock(&internals->lock); @@ -1790,37 +2289,91 @@ bond_ethdev_slave_link_status_change_monitor(void *cb_arg) } static int -bond_ethdev_link_update(struct rte_eth_dev *bonded_eth_dev, - int wait_to_complete) +bond_ethdev_link_update(struct rte_eth_dev *ethdev, int wait_to_complete) { - struct bond_dev_private *internals = bonded_eth_dev->data->dev_private; + void (*link_update)(uint16_t port_id, struct rte_eth_link *eth_link); - if (!bonded_eth_dev->data->dev_started || - internals->active_slave_count == 0) { - bonded_eth_dev->data->dev_link.link_status = ETH_LINK_DOWN; + struct bond_dev_private *bond_ctx; + struct rte_eth_link slave_link; + + uint32_t idx; + + bond_ctx = ethdev->data->dev_private; + + ethdev->data->dev_link.link_speed = ETH_SPEED_NUM_NONE; + + if (ethdev->data->dev_started == 0 || + bond_ctx->active_slave_count == 0) { + ethdev->data->dev_link.link_status = ETH_LINK_DOWN; return 0; - } else { - struct rte_eth_dev *slave_eth_dev; - int i, link_up = 0; - - for (i = 0; i < internals->active_slave_count; i++) { - slave_eth_dev = &rte_eth_devices[internals->active_slaves[i]]; - - (*slave_eth_dev->dev_ops->link_update)(slave_eth_dev, - wait_to_complete); - if (slave_eth_dev->data->dev_link.link_status == ETH_LINK_UP) { - link_up = 1; - break; - } - } - - bonded_eth_dev->data->dev_link.link_status = link_up; } + ethdev->data->dev_link.link_status = ETH_LINK_UP; + + if (wait_to_complete) + link_update = rte_eth_link_get; + else + link_update = rte_eth_link_get_nowait; + + switch (bond_ctx->mode) { + case BONDING_MODE_BROADCAST: + /** + * Setting link speed to UINT32_MAX to ensure we pick up the + * value of the first active slave + */ + ethdev->data->dev_link.link_speed = UINT32_MAX; + + /** + * link speed is minimum value of all the slaves link speed as + * packet loss will occur on this slave if transmission at rates + * greater than this are attempted + */ + for (idx = 1; idx < bond_ctx->active_slave_count; idx++) { + link_update(bond_ctx->active_slaves[0], &slave_link); + + if (slave_link.link_speed < + ethdev->data->dev_link.link_speed) + ethdev->data->dev_link.link_speed = + slave_link.link_speed; + } + break; + case BONDING_MODE_ACTIVE_BACKUP: + /* Current primary slave */ + link_update(bond_ctx->current_primary_port, &slave_link); + + ethdev->data->dev_link.link_speed = slave_link.link_speed; + break; + case BONDING_MODE_8023AD: + ethdev->data->dev_link.link_autoneg = + bond_ctx->mode4.slave_link.link_autoneg; + ethdev->data->dev_link.link_duplex = + bond_ctx->mode4.slave_link.link_duplex; + /* fall through to update link speed */ + case BONDING_MODE_ROUND_ROBIN: + case BONDING_MODE_BALANCE: + case BONDING_MODE_TLB: + case BONDING_MODE_ALB: + default: + /** + * In theses mode the maximum theoretical link speed is the sum + * of all the slaves + */ + ethdev->data->dev_link.link_speed = ETH_SPEED_NUM_NONE; + + for (idx = 0; idx < bond_ctx->active_slave_count; idx++) { + link_update(bond_ctx->active_slaves[idx], &slave_link); + + ethdev->data->dev_link.link_speed += + slave_link.link_speed; + } + } + + return 0; } -static void + +static int bond_ethdev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) { struct bond_dev_private *internals = dev->data->dev_private; @@ -1848,6 +2401,8 @@ bond_ethdev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) } } + + return 0; } static void @@ -1923,35 +2478,35 @@ bond_ethdev_delayed_lsc_propagation(void *arg) return; _rte_eth_dev_callback_process((struct rte_eth_dev *)arg, - RTE_ETH_EVENT_INTR_LSC); + RTE_ETH_EVENT_INTR_LSC, NULL, NULL); } -void -bond_ethdev_lsc_event_callback(uint8_t port_id, enum rte_eth_event_type type, - void *param) +int +bond_ethdev_lsc_event_callback(uint16_t port_id, enum rte_eth_event_type type, + void *param, void *ret_param __rte_unused) { - struct rte_eth_dev *bonded_eth_dev, *slave_eth_dev; + struct rte_eth_dev *bonded_eth_dev; struct bond_dev_private *internals; struct rte_eth_link link; + int rc = -1; int i, valid_slave = 0; uint8_t active_pos; uint8_t lsc_flag = 0; if (type != RTE_ETH_EVENT_INTR_LSC || param == NULL) - return; + return rc; bonded_eth_dev = &rte_eth_devices[*(uint8_t *)param]; - slave_eth_dev = &rte_eth_devices[port_id]; if (check_for_bonded_ethdev(bonded_eth_dev)) - return; + return rc; internals = bonded_eth_dev->data->dev_private; /* If the device isn't started don't handle interrupts */ if (!bonded_eth_dev->data->dev_started) - return; + return rc; /* verify that port_id is a valid slave of bonded port */ for (i = 0; i < internals->slave_count; i++) { @@ -1962,7 +2517,7 @@ bond_ethdev_lsc_event_callback(uint8_t port_id, enum rte_eth_event_type type, } if (!valid_slave) - return; + return rc; /* Search for port in active port list */ active_pos = find_slave_by_id(internals->active_slaves, @@ -1971,7 +2526,7 @@ bond_ethdev_lsc_event_callback(uint8_t port_id, enum rte_eth_event_type type, rte_eth_link_get_nowait(port_id, &link); if (link.link_status) { if (active_pos < internals->active_slave_count) - return; + return rc; /* if no active slave ports then set this port to be primary port */ if (internals->active_slave_count < 1) { @@ -1981,20 +2536,6 @@ bond_ethdev_lsc_event_callback(uint8_t port_id, enum rte_eth_event_type type, lsc_flag = 1; mac_address_slaves_update(bonded_eth_dev); - - /* Inherit eth dev link properties from first active slave */ - link_properties_set(bonded_eth_dev, - &(slave_eth_dev->data->dev_link)); - } else { - if (link_properties_valid( - &bonded_eth_dev->data->dev_link, &link) != 0) { - slave_eth_dev->data->dev_flags &= - (~RTE_ETH_DEV_BONDED_SLAVE); - RTE_LOG(ERR, PMD, - "port %u invalid speed/duplex\n", - port_id); - return; - } } activate_slave(bonded_eth_dev, port_id); @@ -2005,19 +2546,13 @@ bond_ethdev_lsc_event_callback(uint8_t port_id, enum rte_eth_event_type type, bond_ethdev_primary_set(internals, port_id); } else { if (active_pos == internals->active_slave_count) - return; + return rc; /* Remove from active slave list */ deactivate_slave(bonded_eth_dev, port_id); - /* No active slaves, change link status to down and reset other - * link properties */ - if (internals->active_slave_count < 1) { + if (internals->active_slave_count < 1) lsc_flag = 1; - bonded_eth_dev->data->dev_link.link_status = ETH_LINK_DOWN; - - link_properties_reset(bonded_eth_dev); - } /* Update primary id, take first active slave from list or if none * available set to -1 */ @@ -2030,6 +2565,12 @@ bond_ethdev_lsc_event_callback(uint8_t port_id, enum rte_eth_event_type type, } } + /** + * Update bonded device link properties after any change to active + * slaves + */ + bond_ethdev_link_update(bonded_eth_dev, 0); + if (lsc_flag) { /* Cancel any possible outstanding interrupts if delays are enabled */ if (internals->link_up_delay_ms > 0 || @@ -2044,7 +2585,8 @@ bond_ethdev_lsc_event_callback(uint8_t port_id, enum rte_eth_event_type type, (void *)bonded_eth_dev); else _rte_eth_dev_callback_process(bonded_eth_dev, - RTE_ETH_EVENT_INTR_LSC); + RTE_ETH_EVENT_INTR_LSC, + NULL, NULL); } else { if (internals->link_down_delay_ms > 0) @@ -2053,9 +2595,11 @@ bond_ethdev_lsc_event_callback(uint8_t port_id, enum rte_eth_event_type type, (void *)bonded_eth_dev); else _rte_eth_dev_callback_process(bonded_eth_dev, - RTE_ETH_EVENT_INTR_LSC); + RTE_ETH_EVENT_INTR_LSC, + NULL, NULL); } } + return 0; } static int @@ -2171,6 +2715,7 @@ const struct eth_dev_ops default_dev_ops = { .dev_close = bond_ethdev_close, .dev_configure = bond_ethdev_configure, .dev_infos_get = bond_ethdev_info, + .vlan_filter_set = bond_ethdev_vlan_filter_set, .rx_queue_setup = bond_ethdev_rx_queue_setup, .tx_queue_setup = bond_ethdev_tx_queue_setup, .rx_queue_release = bond_ethdev_rx_queue_release, @@ -2187,16 +2732,126 @@ const struct eth_dev_ops default_dev_ops = { }; static int -bond_init(const char *name, const char *params) +bond_alloc(struct rte_vdev_device *dev, uint8_t mode) { + const char *name = rte_vdev_device_name(dev); + uint8_t socket_id = dev->device.numa_node; + struct bond_dev_private *internals = NULL; + struct rte_eth_dev *eth_dev = NULL; + uint32_t vlan_filter_bmp_size; + + /* now do all data allocation - for eth_dev structure, dummy pci driver + * and internal (private) data + */ + + /* reserve an ethdev entry */ + eth_dev = rte_eth_vdev_allocate(dev, sizeof(*internals)); + if (eth_dev == NULL) { + RTE_BOND_LOG(ERR, "Unable to allocate rte_eth_dev"); + goto err; + } + + internals = eth_dev->data->dev_private; + eth_dev->data->nb_rx_queues = (uint16_t)1; + eth_dev->data->nb_tx_queues = (uint16_t)1; + + eth_dev->data->mac_addrs = rte_zmalloc_socket(name, ETHER_ADDR_LEN, 0, + socket_id); + if (eth_dev->data->mac_addrs == NULL) { + RTE_BOND_LOG(ERR, "Unable to malloc mac_addrs"); + goto err; + } + + eth_dev->dev_ops = &default_dev_ops; + eth_dev->data->dev_flags = RTE_ETH_DEV_INTR_LSC; + + rte_spinlock_init(&internals->lock); + + internals->port_id = eth_dev->data->port_id; + internals->mode = BONDING_MODE_INVALID; + internals->current_primary_port = RTE_MAX_ETHPORTS + 1; + internals->balance_xmit_policy = BALANCE_XMIT_POLICY_LAYER2; + internals->xmit_hash = xmit_l2_hash; + internals->user_defined_mac = 0; + + internals->link_status_polling_enabled = 0; + + internals->link_status_polling_interval_ms = + DEFAULT_POLLING_INTERVAL_10_MS; + internals->link_down_delay_ms = 0; + internals->link_up_delay_ms = 0; + + internals->slave_count = 0; + internals->active_slave_count = 0; + internals->rx_offload_capa = 0; + internals->tx_offload_capa = 0; + internals->candidate_max_rx_pktlen = 0; + internals->max_rx_pktlen = 0; + + /* Initially allow to choose any offload type */ + internals->flow_type_rss_offloads = ETH_RSS_PROTO_MASK; + + memset(internals->active_slaves, 0, sizeof(internals->active_slaves)); + memset(internals->slaves, 0, sizeof(internals->slaves)); + + /* Set mode 4 default configuration */ + bond_mode_8023ad_setup(eth_dev, NULL); + if (bond_ethdev_mode_set(eth_dev, mode)) { + RTE_BOND_LOG(ERR, "Failed to set bonded device %d mode too %d", + eth_dev->data->port_id, mode); + goto err; + } + + vlan_filter_bmp_size = + rte_bitmap_get_memory_footprint(ETHER_MAX_VLAN_ID + 1); + internals->vlan_filter_bmpmem = rte_malloc(name, vlan_filter_bmp_size, + RTE_CACHE_LINE_SIZE); + if (internals->vlan_filter_bmpmem == NULL) { + RTE_BOND_LOG(ERR, + "Failed to allocate vlan bitmap for bonded device %u\n", + eth_dev->data->port_id); + goto err; + } + + internals->vlan_filter_bmp = rte_bitmap_init(ETHER_MAX_VLAN_ID + 1, + internals->vlan_filter_bmpmem, vlan_filter_bmp_size); + if (internals->vlan_filter_bmp == NULL) { + RTE_BOND_LOG(ERR, + "Failed to init vlan bitmap for bonded device %u\n", + eth_dev->data->port_id); + rte_free(internals->vlan_filter_bmpmem); + goto err; + } + + return eth_dev->data->port_id; + +err: + rte_free(internals); + if (eth_dev != NULL) { + rte_free(eth_dev->data->mac_addrs); + rte_eth_dev_release_port(eth_dev); + } + return -1; +} + +static int +bond_probe(struct rte_vdev_device *dev) +{ + const char *name; struct bond_dev_private *internals; struct rte_kvargs *kvlist; - uint8_t bonding_mode, socket_id; + uint8_t bonding_mode, socket_id/*, agg_mode*/; int arg_count, port_id; + uint8_t agg_mode; + if (!dev) + return -EINVAL; + + name = rte_vdev_device_name(dev); RTE_LOG(INFO, EAL, "Initializing pmd_bond for %s\n", name); - kvlist = rte_kvargs_parse(params, pmd_bond_init_valid_arguments); + kvlist = rte_kvargs_parse(rte_vdev_device_args(dev), + pmd_bond_init_valid_arguments); if (kvlist == NULL) return -1; @@ -2233,8 +2888,10 @@ bond_init(const char *name, const char *params) socket_id = rte_socket_id(); } + dev->device.numa_node = socket_id; + /* Create link bonding eth device */ - port_id = rte_eth_bond_create(name, bonding_mode, socket_id); + port_id = bond_alloc(dev, bonding_mode); if (port_id < 0) { RTE_LOG(ERR, EAL, "Failed to create socket %s in mode %u on " "socket %u.\n", name, bonding_mode, socket_id); @@ -2243,6 +2900,25 @@ bond_init(const char *name, const char *params) internals = rte_eth_devices[port_id].data->dev_private; internals->kvlist = kvlist; + + if (rte_kvargs_count(kvlist, PMD_BOND_AGG_MODE_KVARG) == 1) { + if (rte_kvargs_process(kvlist, + PMD_BOND_AGG_MODE_KVARG, + &bond_ethdev_parse_slave_agg_mode_kvarg, + &agg_mode) != 0) { + RTE_LOG(ERR, EAL, + "Failed to parse agg selection mode for bonded device %s\n", + name); + goto parse_error; + } + + if (internals->mode == BONDING_MODE_8023AD) + rte_eth_bond_8023ad_agg_selection_set(port_id, + agg_mode); + } else { + rte_eth_bond_8023ad_agg_selection_set(port_id, AGG_STABLE); + } + RTE_LOG(INFO, EAL, "Create bonded device %s on port %d in mode %u on " "socket %u.\n", name, port_id, bonding_mode, socket_id); return 0; @@ -2254,21 +2930,51 @@ parse_error: } static int -bond_uninit(const char *name) +bond_remove(struct rte_vdev_device *dev) { - int ret; + struct rte_eth_dev *eth_dev; + struct bond_dev_private *internals; + const char *name; - if (name == NULL) + if (!dev) return -EINVAL; + name = rte_vdev_device_name(dev); RTE_LOG(INFO, EAL, "Uninitializing pmd_bond for %s\n", name); - /* free link bonding eth device */ - ret = rte_eth_bond_free(name); - if (ret < 0) - RTE_LOG(ERR, EAL, "Failed to free %s\n", name); + /* now free all data allocation - for eth_dev structure, + * dummy pci driver and internal (private) data + */ - return ret; + /* find an ethdev entry */ + eth_dev = rte_eth_dev_allocated(name); + if (eth_dev == NULL) + return -ENODEV; + + RTE_ASSERT(eth_dev->device == &dev->device); + + internals = eth_dev->data->dev_private; + if (internals->slave_count != 0) + return -EBUSY; + + if (eth_dev->data->dev_started == 1) { + bond_ethdev_stop(eth_dev); + bond_ethdev_close(eth_dev); + } + + eth_dev->dev_ops = NULL; + eth_dev->rx_pkt_burst = NULL; + eth_dev->tx_pkt_burst = NULL; + + internals = eth_dev->data->dev_private; + rte_bitmap_free(internals->vlan_filter_bmp); + rte_free(internals->vlan_filter_bmpmem); + rte_free(eth_dev->data->dev_private); + rte_free(eth_dev->data->mac_addrs); + + rte_eth_dev_release_port(eth_dev); + + return 0; } /* this part will resolve the slave portids after all the other pdev and vdev @@ -2276,11 +2982,12 @@ bond_uninit(const char *name) static int bond_ethdev_configure(struct rte_eth_dev *dev) { - char *name = dev->data->name; + const char *name = dev->device->name; struct bond_dev_private *internals = dev->data->dev_private; struct rte_kvargs *kvlist = internals->kvlist; int arg_count; - uint8_t port_id = dev - rte_eth_devices; + uint16_t port_id = dev - rte_eth_devices; + uint8_t agg_mode; static const uint8_t default_rss_key[40] = { 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2, 0x41, 0x67, 0x25, 0x3D, @@ -2368,6 +3075,20 @@ bond_ethdev_configure(struct rte_eth_dev *dev) return -1; } + if (rte_kvargs_count(kvlist, PMD_BOND_AGG_MODE_KVARG) == 1) { + if (rte_kvargs_process(kvlist, + PMD_BOND_AGG_MODE_KVARG, + &bond_ethdev_parse_slave_agg_mode_kvarg, + &agg_mode) != 0) { + RTE_LOG(ERR, EAL, + "Failed to parse agg selection mode for bonded device %s\n", + name); + } + if (internals->mode == BONDING_MODE_8023AD) + rte_eth_bond_8023ad_agg_selection_set(port_id, + agg_mode); + } + /* Parse/add slave ports to bonded device */ if (rte_kvargs_count(kvlist, PMD_BOND_SLAVE_PORT_KVARG) > 0) { struct bond_ethdev_slave_ports slave_ports; @@ -2399,7 +3120,7 @@ bond_ethdev_configure(struct rte_eth_dev *dev) /* Parse/set primary slave port id*/ arg_count = rte_kvargs_count(kvlist, PMD_BOND_PRIMARY_SLAVE_KVARG); if (arg_count == 1) { - uint8_t primary_slave_port_id; + uint16_t primary_slave_port_id; if (rte_kvargs_process(kvlist, PMD_BOND_PRIMARY_SLAVE_KVARG, @@ -2412,7 +3133,7 @@ bond_ethdev_configure(struct rte_eth_dev *dev) } /* Set balance mode transmit policy*/ - if (rte_eth_bond_primary_set(port_id, (uint8_t)primary_slave_port_id) + if (rte_eth_bond_primary_set(port_id, primary_slave_port_id) != 0) { RTE_LOG(ERR, EAL, "Failed to set primary slave port %d on bonded device %s\n", @@ -2518,19 +3239,20 @@ bond_ethdev_configure(struct rte_eth_dev *dev) return 0; } -static struct rte_driver bond_drv = { - .type = PMD_VDEV, - .init = bond_init, - .uninit = bond_uninit, +struct rte_vdev_driver pmd_bond_drv = { + .probe = bond_probe, + .remove = bond_remove, }; -PMD_REGISTER_DRIVER(bond_drv, eth_bond); +RTE_PMD_REGISTER_VDEV(net_bonding, pmd_bond_drv); +RTE_PMD_REGISTER_ALIAS(net_bonding, eth_bond); -DRIVER_REGISTER_PARAM_STRING(eth_bond, +RTE_PMD_REGISTER_PARAM_STRING(net_bonding, "slave= " "primary= " "mode=[0-6] " "xmit_policy=[l2 | l23 | l34] " + "agg_mode=[count | stable | bandwidth] " "socket_id= " "mac= " "lsc_poll_period_ms= " diff --git a/dpdk/drivers/net/bonding/rte_eth_bond_private.h b/dpdk/drivers/net/bonding/rte_eth_bond_private.h index 2bdc9efa..a5cfa6ac 100644 --- a/dpdk/drivers/net/bonding/rte_eth_bond_private.h +++ b/dpdk/drivers/net/bonding/rte_eth_bond_private.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -36,6 +36,7 @@ #include #include +#include #include "rte_eth_bond.h" #include "rte_eth_bond_8023ad_private.h" @@ -44,6 +45,7 @@ #define PMD_BOND_SLAVE_PORT_KVARG ("slave") #define PMD_BOND_PRIMARY_SLAVE_KVARG ("primary") #define PMD_BOND_MODE_KVARG ("mode") +#define PMD_BOND_AGG_MODE_KVARG ("agg_mode") #define PMD_BOND_XMIT_POLICY_KVARG ("xmit_policy") #define PMD_BOND_SOCKET_ID_KVARG ("socket_id") #define PMD_BOND_MAC_ADDR_KVARG ("mac") @@ -62,7 +64,7 @@ extern const char *pmd_bond_init_valid_arguments[]; -extern const char pmd_bond_driver_name[]; +extern struct rte_vdev_driver pmd_bond_drv; /** Port Queue Mapping Structure */ struct bond_rx_queue { @@ -91,12 +93,12 @@ struct bond_tx_queue { /** Bonded slave devices structure */ struct bond_ethdev_slave_ports { - uint8_t slaves[RTE_MAX_ETHPORTS]; /**< Slave port id array */ - uint8_t slave_count; /**< Number of slaves */ + uint16_t slaves[RTE_MAX_ETHPORTS]; /**< Slave port id array */ + uint16_t slave_count; /**< Number of slaves */ }; struct bond_slave_details { - uint8_t port_id; + uint16_t port_id; uint8_t link_status_poll_enabled; uint8_t link_status_wait_to_complete; @@ -112,14 +114,14 @@ typedef uint16_t (*xmit_hash_t)(const struct rte_mbuf *buf, uint8_t slave_count) /** Link Bonding PMD device private configuration Structure */ struct bond_dev_private { - uint8_t port_id; /**< Port Id of Bonded Port */ + uint16_t port_id; /**< Port Id of Bonded Port */ uint8_t mode; /**< Link Bonding Mode */ rte_spinlock_t lock; - uint8_t primary_port; /**< Primary Slave Port */ - uint8_t current_primary_port; /**< Primary Slave Port */ - uint8_t user_defined_primary_port; + uint16_t primary_port; /**< Primary Slave Port */ + uint16_t current_primary_port; /**< Primary Slave Port */ + uint16_t user_defined_primary_port; /**< Flag for whether primary port is user defined or not */ uint8_t balance_xmit_policy; @@ -131,8 +133,7 @@ struct bond_dev_private { /**< Flag for whether MAC address is user defined or not */ uint8_t promiscuous_en; /**< Enabled/disable promiscuous mode on bonding device */ - uint8_t link_props_set; - /**< flag to denote if the link properties are set */ + uint8_t link_status_polling_enabled; uint32_t link_status_polling_interval_ms; @@ -143,15 +144,17 @@ struct bond_dev_private { uint16_t nb_rx_queues; /**< Total number of rx queues */ uint16_t nb_tx_queues; /**< Total number of tx queues*/ - uint8_t active_slave_count; /**< Number of active slaves */ - uint8_t active_slaves[RTE_MAX_ETHPORTS]; /**< Active slave list */ + uint16_t active_slave; /**< Next active_slave to poll */ + uint16_t active_slave_count; /**< Number of active slaves */ + uint16_t active_slaves[RTE_MAX_ETHPORTS]; /**< Active slave list */ - uint8_t slave_count; /**< Number of bonded slaves */ + uint16_t slave_count; /**< Number of bonded slaves */ struct bond_slave_details slaves[RTE_MAX_ETHPORTS]; /**< Arary of bonded slaves details */ struct mode8023ad_private mode4; - uint8_t tlb_slaves_order[RTE_MAX_ETHPORTS]; /* TLB active slaves send order */ + uint16_t tlb_slaves_order[RTE_MAX_ETHPORTS]; + /**< TLB active slaves send order */ struct mode_alb_private mode6; uint32_t rx_offload_capa; /** Rx offload capability */ @@ -172,19 +175,25 @@ struct bond_dev_private { uint32_t candidate_max_rx_pktlen; uint32_t max_rx_pktlen; + + void *vlan_filter_bmpmem; /* enabled vlan filter bitmap */ + struct rte_bitmap *vlan_filter_bmp; }; extern const struct eth_dev_ops default_dev_ops; +int +check_for_master_bonded_ethdev(const struct rte_eth_dev *eth_dev); + int check_for_bonded_ethdev(const struct rte_eth_dev *eth_dev); -/* Search given slave array to find possition of given id. +/* Search given slave array to find position of given id. * Return slave pos or slaves_count if not found. */ -static inline uint8_t -find_slave_by_id(uint8_t *slaves, uint8_t slaves_count, uint8_t slave_id) { +static inline uint16_t +find_slave_by_id(uint16_t *slaves, uint16_t slaves_count, uint16_t slave_id) { - uint8_t pos; + uint16_t pos; for (pos = 0; pos < slaves_count; pos++) { if (slave_id == slaves[pos]) break; @@ -194,28 +203,25 @@ find_slave_by_id(uint8_t *slaves, uint8_t slaves_count, uint8_t slave_id) { } int -valid_port_id(uint8_t port_id); +valid_port_id(uint16_t port_id); int -valid_bonded_port_id(uint8_t port_id); +valid_bonded_port_id(uint16_t port_id); int -valid_slave_port_id(uint8_t port_id); +valid_slave_port_id(uint16_t port_id, uint8_t mode); void -deactivate_slave(struct rte_eth_dev *eth_dev, uint8_t port_id); +deactivate_slave(struct rte_eth_dev *eth_dev, uint16_t port_id); void -activate_slave(struct rte_eth_dev *eth_dev, uint8_t port_id); +activate_slave(struct rte_eth_dev *eth_dev, uint16_t port_id); void link_properties_set(struct rte_eth_dev *bonded_eth_dev, struct rte_eth_link *slave_dev_link); -void -link_properties_reset(struct rte_eth_dev *bonded_eth_dev); - int -link_properties_valid(struct rte_eth_link *bonded_dev_link, +link_properties_valid(struct rte_eth_dev *bonded_eth_dev, struct rte_eth_link *slave_dev_link); int @@ -227,9 +233,6 @@ mac_address_get(struct rte_eth_dev *eth_dev, struct ether_addr *dst_mac_addr); int mac_address_slaves_update(struct rte_eth_dev *bonded_eth_dev); -uint8_t -number_of_sockets(void); - int bond_ethdev_mode_set(struct rte_eth_dev *eth_dev, int mode); @@ -256,38 +259,42 @@ xmit_l34_hash(const struct rte_mbuf *buf, uint8_t slave_count); void bond_ethdev_primary_set(struct bond_dev_private *internals, - uint8_t slave_port_id); - -void -bond_ethdev_lsc_event_callback(uint8_t port_id, enum rte_eth_event_type type, - void *param); + uint16_t slave_port_id); int -bond_ethdev_parse_slave_port_kvarg(const char *key __rte_unused, +bond_ethdev_lsc_event_callback(uint16_t port_id, enum rte_eth_event_type type, + void *param, void *ret_param); + +int +bond_ethdev_parse_slave_port_kvarg(const char *key, const char *value, void *extra_args); int -bond_ethdev_parse_slave_mode_kvarg(const char *key __rte_unused, +bond_ethdev_parse_slave_mode_kvarg(const char *key, const char *value, void *extra_args); int -bond_ethdev_parse_socket_id_kvarg(const char *key __rte_unused, +bond_ethdev_parse_slave_agg_mode_kvarg(const char *key __rte_unused, const char *value, void *extra_args); int -bond_ethdev_parse_primary_slave_port_id_kvarg(const char *key __rte_unused, +bond_ethdev_parse_socket_id_kvarg(const char *key, const char *value, void *extra_args); int -bond_ethdev_parse_balance_xmit_policy_kvarg(const char *key __rte_unused, +bond_ethdev_parse_primary_slave_port_id_kvarg(const char *key, const char *value, void *extra_args); int -bond_ethdev_parse_bond_mac_addr_kvarg(const char *key __rte_unused, +bond_ethdev_parse_balance_xmit_policy_kvarg(const char *key, const char *value, void *extra_args); int -bond_ethdev_parse_time_ms_kvarg(const char *key __rte_unused, +bond_ethdev_parse_bond_mac_addr_kvarg(const char *key, + const char *value, void *extra_args); + +int +bond_ethdev_parse_time_ms_kvarg(const char *key, const char *value, void *extra_args); void diff --git a/dpdk/drivers/net/bonding/rte_eth_bond_version.map b/dpdk/drivers/net/bonding/rte_eth_bond_version.map deleted file mode 100644 index 2de0a7d3..00000000 --- a/dpdk/drivers/net/bonding/rte_eth_bond_version.map +++ /dev/null @@ -1,45 +0,0 @@ -DPDK_2.0 { - global: - - rte_eth_bond_8023ad_conf_get; - rte_eth_bond_8023ad_setup; - rte_eth_bond_active_slaves_get; - rte_eth_bond_create; - rte_eth_bond_link_monitoring_set; - rte_eth_bond_mac_address_reset; - rte_eth_bond_mac_address_set; - rte_eth_bond_mode_get; - rte_eth_bond_mode_set; - rte_eth_bond_primary_get; - rte_eth_bond_primary_set; - rte_eth_bond_slave_add; - rte_eth_bond_slave_remove; - rte_eth_bond_slaves_get; - rte_eth_bond_xmit_policy_get; - rte_eth_bond_xmit_policy_set; - - local: *; -}; - -DPDK_2.1 { - global: - - rte_eth_bond_free; - -} DPDK_2.0; - -DPDK_16.04 { -}; - -DPDK_16.07 { - global: - - rte_eth_bond_8023ad_ext_collect; - rte_eth_bond_8023ad_ext_collect_get; - rte_eth_bond_8023ad_ext_distrib; - rte_eth_bond_8023ad_ext_distrib_get; - rte_eth_bond_8023ad_ext_slowtx; - rte_eth_bond_8023ad_conf_get; - rte_eth_bond_8023ad_setup; - -} DPDK_16.04; diff --git a/dpdk/drivers/net/bonding/rte_pmd_bond_version.map b/dpdk/drivers/net/bonding/rte_pmd_bond_version.map new file mode 100644 index 00000000..ec3374b0 --- /dev/null +++ b/dpdk/drivers/net/bonding/rte_pmd_bond_version.map @@ -0,0 +1,54 @@ +DPDK_2.0 { + global: + + rte_eth_bond_active_slaves_get; + rte_eth_bond_create; + rte_eth_bond_link_monitoring_set; + rte_eth_bond_mac_address_reset; + rte_eth_bond_mac_address_set; + rte_eth_bond_mode_get; + rte_eth_bond_mode_set; + rte_eth_bond_primary_get; + rte_eth_bond_primary_set; + rte_eth_bond_slave_add; + rte_eth_bond_slave_remove; + rte_eth_bond_slaves_get; + rte_eth_bond_xmit_policy_get; + rte_eth_bond_xmit_policy_set; + + local: *; +}; + +DPDK_2.1 { + global: + + rte_eth_bond_free; + +} DPDK_2.0; + +DPDK_16.04 { +}; + +DPDK_16.07 { + global: + + rte_eth_bond_8023ad_ext_collect; + rte_eth_bond_8023ad_ext_collect_get; + rte_eth_bond_8023ad_ext_distrib; + rte_eth_bond_8023ad_ext_distrib_get; + rte_eth_bond_8023ad_ext_slowtx; + +} DPDK_16.04; + +DPDK_17.08 { + global: + + rte_eth_bond_8023ad_dedicated_queues_enable; + rte_eth_bond_8023ad_dedicated_queues_disable; + rte_eth_bond_8023ad_agg_selection_get; + rte_eth_bond_8023ad_agg_selection_set; + rte_eth_bond_8023ad_conf_get; + rte_eth_bond_8023ad_setup; + + +} DPDK_16.07; diff --git a/dpdk/drivers/net/cxgbe/Makefile b/dpdk/drivers/net/cxgbe/Makefile index bfcc3159..65df1425 100644 --- a/dpdk/drivers/net/cxgbe/Makefile +++ b/dpdk/drivers/net/cxgbe/Makefile @@ -62,12 +62,15 @@ endif CFLAGS_BASE_DRIVER = endif +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs +LDLIBS += -lrte_bus_pci # # Add extra flags for base driver files (also known as shared code) # to disable warnings in them # -BASE_DRIVER_OBJS=$(patsubst %.c,%.o,$(notdir $(wildcard $(SRCDIR)/base/*.c))) +BASE_DRIVER_OBJS=$(sort $(patsubst %.c,%.o,$(notdir $(wildcard $(SRCDIR)/base/*.c)))) $(foreach obj, $(BASE_DRIVER_OBJS), $(eval CFLAGS_$(obj)+=$(CFLAGS_BASE_DRIVER))) VPATH += $(SRCDIR)/base @@ -81,9 +84,4 @@ SRCS-$(CONFIG_RTE_LIBRTE_CXGBE_PMD) += cxgbe_main.c SRCS-$(CONFIG_RTE_LIBRTE_CXGBE_PMD) += sge.c SRCS-$(CONFIG_RTE_LIBRTE_CXGBE_PMD) += t4_hw.c -# this lib depends upon: -DEPDIRS-$(CONFIG_RTE_LIBRTE_CXGBE_PMD) += lib/librte_eal lib/librte_ether -DEPDIRS-$(CONFIG_RTE_LIBRTE_CXGBE_PMD) += lib/librte_mempool lib/librte_mbuf -DEPDIRS-$(CONFIG_RTE_LIBRTE_CXGBE_PMD) += lib/librte_net - include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/net/cxgbe/base/adapter.h b/dpdk/drivers/net/cxgbe/base/adapter.h index 5e3bd509..f2057af1 100644 --- a/dpdk/drivers/net/cxgbe/base/adapter.h +++ b/dpdk/drivers/net/cxgbe/base/adapter.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2014-2016 Chelsio Communications. + * Copyright(c) 2014-2017 Chelsio Communications. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -36,7 +36,9 @@ #ifndef __T4_ADAPTER_H__ #define __T4_ADAPTER_H__ +#include #include +#include #include "cxgbe_compat.h" #include "t4_regs_values.h" @@ -147,6 +149,7 @@ struct sge_rspq { /* state for an SGE response queue */ void __iomem *bar2_addr; /* address of BAR2 Queue registers */ unsigned int bar2_qid; /* Queue ID for BAR2 Queue registers */ + struct sge_qstat *stat; unsigned int cidx; /* consumer index */ unsigned int gts_idx; /* last gts write sent */ @@ -324,7 +327,7 @@ struct adapter { int use_unpacked_mode; /* unpacked rx mode state */ }; -#define CXGBE_PCI_REG(reg) (*((volatile uint32_t *)(reg))) +#define CXGBE_PCI_REG(reg) rte_read32(reg) static inline uint64_t cxgbe_read_addr64(volatile void *addr) { @@ -350,16 +353,21 @@ static inline uint32_t cxgbe_read_addr(volatile void *addr) #define CXGBE_READ_REG64(adap, reg) \ cxgbe_read_addr64(CXGBE_PCI_REG_ADDR((adap), (reg))) -#define CXGBE_PCI_REG_WRITE(reg, value) ({ \ - CXGBE_PCI_REG((reg)) = (value); }) +#define CXGBE_PCI_REG_WRITE(reg, value) rte_write32((value), (reg)) + +#define CXGBE_PCI_REG_WRITE_RELAXED(reg, value) \ + rte_write32_relaxed((value), (reg)) #define CXGBE_WRITE_REG(adap, reg, value) \ CXGBE_PCI_REG_WRITE(CXGBE_PCI_REG_ADDR((adap), (reg)), (value)) +#define CXGBE_WRITE_REG_RELAXED(adap, reg, value) \ + CXGBE_PCI_REG_WRITE_RELAXED(CXGBE_PCI_REG_ADDR((adap), (reg)), (value)) + static inline uint64_t cxgbe_write_addr64(volatile void *addr, uint64_t val) { - CXGBE_PCI_REG(addr) = val; - CXGBE_PCI_REG(((volatile uint8_t *)(addr) + 4)) = (val >> 32); + CXGBE_PCI_REG_WRITE(addr, val); + CXGBE_PCI_REG_WRITE(((volatile uint8_t *)(addr) + 4), (val >> 32)); return val; } @@ -383,7 +391,7 @@ static inline u32 t4_read_reg(struct adapter *adapter, u32 reg_addr) } /** - * t4_write_reg - write a HW register + * t4_write_reg - write a HW register with barrier * @adapter: the adapter * @reg_addr: the register address * @val: the value to write @@ -397,6 +405,22 @@ static inline void t4_write_reg(struct adapter *adapter, u32 reg_addr, u32 val) CXGBE_WRITE_REG(adapter, reg_addr, val); } +/** + * t4_write_reg_relaxed - write a HW register with no barrier + * @adapter: the adapter + * @reg_addr: the register address + * @val: the value to write + * + * Write a 32-bit value into the given HW register. + */ +static inline void t4_write_reg_relaxed(struct adapter *adapter, u32 reg_addr, + u32 val) +{ + CXGBE_DEBUG_REG(adapter, "setting register 0x%x to 0x%x\n", reg_addr, + val); + CXGBE_WRITE_REG_RELAXED(adapter, reg_addr, val); +} + /** * t4_read_reg64 - read a 64-bit HW register * @adapter: the adapter @@ -437,7 +461,10 @@ static inline void t4_write_reg64(struct adapter *adapter, u32 reg_addr, #define PCI_CAP_ID_EXP 0x10 /* PCI Express */ #define PCI_CAP_LIST_ID 0 /* Capability ID */ #define PCI_CAP_LIST_NEXT 1 /* Next capability in the list */ +#define PCI_EXP_DEVCTL 0x0008 /* Device control */ #define PCI_EXP_DEVCTL2 40 /* Device Control 2 */ +#define PCI_EXP_DEVCTL_EXT_TAG 0x0100 /* Extended Tag Field Enable */ +#define PCI_EXP_DEVCTL_PAYLOAD 0x00E0 /* Max payload */ #define PCI_CAP_ID_VPD 0x03 /* Vital Product Data */ #define PCI_VPD_ADDR 2 /* Address to access (15 bits!) */ #define PCI_VPD_ADDR_F 0x8000 /* Write 0, 1 indicates completion */ @@ -456,7 +483,7 @@ static inline void t4_os_pci_write_cfg4(struct adapter *adapter, size_t addr, { u32 val32 = val; - if (rte_eal_pci_write_config(adapter->pdev, &val32, sizeof(val32), + if (rte_pci_write_config(adapter->pdev, &val32, sizeof(val32), addr) < 0) dev_err(adapter, "Can't write to PCI config space\n"); } @@ -472,7 +499,7 @@ static inline void t4_os_pci_write_cfg4(struct adapter *adapter, size_t addr, static inline void t4_os_pci_read_cfg4(struct adapter *adapter, size_t addr, u32 *val) { - if (rte_eal_pci_read_config(adapter->pdev, val, sizeof(*val), + if (rte_pci_read_config(adapter->pdev, val, sizeof(*val), addr) < 0) dev_err(adapter, "Can't read from PCI config space\n"); } @@ -490,7 +517,7 @@ static inline void t4_os_pci_write_cfg2(struct adapter *adapter, size_t addr, { u16 val16 = val; - if (rte_eal_pci_write_config(adapter->pdev, &val16, sizeof(val16), + if (rte_pci_write_config(adapter->pdev, &val16, sizeof(val16), addr) < 0) dev_err(adapter, "Can't write to PCI config space\n"); } @@ -506,7 +533,7 @@ static inline void t4_os_pci_write_cfg2(struct adapter *adapter, size_t addr, static inline void t4_os_pci_read_cfg2(struct adapter *adapter, size_t addr, u16 *val) { - if (rte_eal_pci_read_config(adapter->pdev, val, sizeof(*val), + if (rte_pci_read_config(adapter->pdev, val, sizeof(*val), addr) < 0) dev_err(adapter, "Can't read from PCI config space\n"); } @@ -522,7 +549,7 @@ static inline void t4_os_pci_read_cfg2(struct adapter *adapter, size_t addr, static inline void t4_os_pci_read_cfg(struct adapter *adapter, size_t addr, u8 *val) { - if (rte_eal_pci_read_config(adapter->pdev, val, sizeof(*val), + if (rte_pci_read_config(adapter->pdev, val, sizeof(*val), addr) < 0) dev_err(adapter, "Can't read from PCI config space\n"); } @@ -684,7 +711,8 @@ void reclaim_completed_tx(struct sge_txq *q); void t4_free_sge_resources(struct adapter *adap); void t4_sge_tx_monitor_start(struct adapter *adap); void t4_sge_tx_monitor_stop(struct adapter *adap); -int t4_eth_xmit(struct sge_eth_txq *txq, struct rte_mbuf *mbuf); +int t4_eth_xmit(struct sge_eth_txq *txq, struct rte_mbuf *mbuf, + uint16_t nb_pkts); int t4_ethrx_handler(struct sge_rspq *q, const __be64 *rsp, const struct pkt_gl *gl); int t4_sge_init(struct adapter *adap); diff --git a/dpdk/drivers/net/cxgbe/base/common.h b/dpdk/drivers/net/cxgbe/base/common.h index 11f139c9..1eda57d0 100644 --- a/dpdk/drivers/net/cxgbe/base/common.h +++ b/dpdk/drivers/net/cxgbe/base/common.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2014-2016 Chelsio Communications. + * Copyright(c) 2014-2017 Chelsio Communications. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -68,6 +68,12 @@ enum { PAUSE_AUTONEG = 1 << 2 }; +enum { + FEC_RS = 1 << 0, + FEC_BASER_RS = 1 << 1, + FEC_RESERVED = 1 << 2, +}; + struct port_stats { u64 tx_octets; /* total # of octets in good frames */ u64 tx_frames; /* all good frames */ @@ -151,6 +157,11 @@ struct tp_params { u32 vlan_pri_map; /* cached TP_VLAN_PRI_MAP */ u32 ingress_config; /* cached TP_INGRESS_CONFIG */ + /* cached TP_OUT_CONFIG compressed error vector + * and passing outer header info for encapsulated packets. + */ + int rx_pkt_encap; + /* * TP_VLAN_PRI_MAP Compressed Filter Tuple field offsets. This is a * subset of the set of fields which may be present in the Compressed @@ -210,7 +221,9 @@ struct adapter_params { unsigned int sf_nsec; /* # of flash sectors */ unsigned int fw_vers; + unsigned int bs_vers; unsigned int tp_vers; + unsigned int er_vers; unsigned short mtus[NMTUS]; unsigned short a_wnd[NCCTRL_WIN]; @@ -231,10 +244,12 @@ struct adapter_params { struct link_config { unsigned short supported; /* link capabilities */ unsigned short advertising; /* advertised capabilities */ - unsigned short requested_speed; /* speed user has requested */ - unsigned short speed; /* actual link speed */ + unsigned int requested_speed; /* speed user has requested */ + unsigned int speed; /* actual link speed */ unsigned char requested_fc; /* flow control user has requested */ unsigned char fc; /* actual link flow control */ + unsigned char requested_fec; /* Forward Error Correction user */ + unsigned char fec; /* has requested and actual FEC */ unsigned char autoneg; /* autonegotiating? */ unsigned char link_ok; /* link up? */ }; @@ -272,6 +287,7 @@ int t4_fw_bye(struct adapter *adap, unsigned int mbox); int t4_fw_reset(struct adapter *adap, unsigned int mbox, int reset); int t4_fw_halt(struct adapter *adap, unsigned int mbox, int reset); int t4_fw_restart(struct adapter *adap, unsigned int mbox, int reset); +int t4_fl_pkt_align(struct adapter *adap); int t4_fixup_host_params_compat(struct adapter *adap, unsigned int page_size, unsigned int cache_line_size, enum chip_type chip_compat); @@ -374,7 +390,8 @@ int t4_get_vpd_params(struct adapter *adapter, struct vpd_params *p); int t4_read_flash(struct adapter *adapter, unsigned int addr, unsigned int nwords, u32 *data, int byte_oriented); int t4_flash_cfg_addr(struct adapter *adapter); -unsigned int t4_get_mps_bg_map(struct adapter *adapter, int idx); +unsigned int t4_get_mps_bg_map(struct adapter *adapter, unsigned int pidx); +unsigned int t4_get_tp_ch_map(struct adapter *adapter, unsigned int pidx); const char *t4_get_port_type_description(enum fw_port_type port_type); void t4_get_port_stats(struct adapter *adap, int idx, struct port_stats *p); void t4_get_port_stats_offset(struct adapter *adap, int idx, @@ -382,9 +399,10 @@ void t4_get_port_stats_offset(struct adapter *adap, int idx, struct port_stats *offset); void t4_clr_port_stats(struct adapter *adap, int idx); void t4_reset_link_config(struct adapter *adap, int idx); -int t4_get_fw_version(struct adapter *adapter, u32 *vers); -int t4_get_tp_version(struct adapter *adapter, u32 *vers); +int t4_get_version_info(struct adapter *adapter); +void t4_dump_version_info(struct adapter *adapter); int t4_get_flash_params(struct adapter *adapter); +int t4_get_chip_type(struct adapter *adap, int ver); int t4_prep_adapter(struct adapter *adapter); int t4_port_init(struct adapter *adap, int mbox, int pf, int vf); int t4_init_rss_mode(struct adapter *adap, int mbox); diff --git a/dpdk/drivers/net/cxgbe/base/t4_chip_type.h b/dpdk/drivers/net/cxgbe/base/t4_chip_type.h index 1ca68039..cd7a9282 100644 --- a/dpdk/drivers/net/cxgbe/base/t4_chip_type.h +++ b/dpdk/drivers/net/cxgbe/base/t4_chip_type.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2014-2015 Chelsio Communications. + * Copyright(c) 2014-2017 Chelsio Communications. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -49,6 +49,7 @@ #define CHELSIO_T4 0x4 #define CHELSIO_T5 0x5 +#define CHELSIO_T6 0x6 #define CHELSIO_CHIP_CODE(version, revision) (((version) << 4) | (revision)) #define CHELSIO_CHIP_VERSION(code) (((code) >> 4) & 0xf) @@ -64,6 +65,10 @@ enum chip_type { T5_A1 = CHELSIO_CHIP_CODE(CHELSIO_T5, 1), T5_FIRST_REV = T5_A0, T5_LAST_REV = T5_A1, + + T6_A0 = CHELSIO_CHIP_CODE(CHELSIO_T6, 0), + T6_FIRST_REV = T6_A0, + T6_LAST_REV = T6_A0, }; static inline int is_t4(enum chip_type chip) @@ -76,4 +81,8 @@ static inline int is_t5(enum chip_type chip) return (CHELSIO_CHIP_VERSION(chip) == CHELSIO_T5); } +static inline int is_t6(enum chip_type chip) +{ + return (CHELSIO_CHIP_VERSION(chip) == CHELSIO_T6); +} #endif /* __T4_CHIP_TYPE_H__ */ diff --git a/dpdk/drivers/net/cxgbe/base/t4_hw.c b/dpdk/drivers/net/cxgbe/base/t4_hw.c index 7e79adf6..282e2e62 100644 --- a/dpdk/drivers/net/cxgbe/base/t4_hw.c +++ b/dpdk/drivers/net/cxgbe/base/t4_hw.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2014-2016 Chelsio Communications. + * Copyright(c) 2014-2017 Chelsio Communications. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -40,13 +40,11 @@ #include #include #include -#include #include #include #include #include #include -#include #include #include #include @@ -57,7 +55,8 @@ #include "t4_regs_values.h" #include "t4fw_interface.h" -static void init_link_config(struct link_config *lc, unsigned int caps); +static void init_link_config(struct link_config *lc, unsigned int pcaps, + unsigned int acaps); /** * t4_read_mtu_tbl - returns the values in the HW path MTU table @@ -359,6 +358,9 @@ int t4_wr_mbox_meat_timeout(struct adapter *adap, int mbox, struct mbox_entry entry; u32 pcie_fw = 0; + if (!temp) + return -ENOMEM; + if ((size & 15) || size > MBOX_LEN) { free(temp); return -EINVAL; @@ -400,6 +402,7 @@ int t4_wr_mbox_meat_timeout(struct adapter *adap, int mbox, t4_os_atomic_list_del(&entry, &adap->mbox_list, &adap->mbox_lock); t4_report_fw_error(adap); + free(temp); return (pcie_fw & F_PCIE_FW_ERR) ? -ENXIO : -EBUSY; } @@ -443,6 +446,7 @@ int t4_wr_mbox_meat_timeout(struct adapter *adap, int mbox, &adap->mbox_list, &adap->mbox_lock)); t4_report_fw_error(adap); + free(temp); return (v == X_MBOWNER_FW ? -EBUSY : -ETIMEDOUT); } @@ -543,6 +547,7 @@ int t4_wr_mbox_meat_timeout(struct adapter *adap, int mbox, T4_OS_MBOX_LOCKING( t4_os_atomic_list_del(&entry, &adap->mbox_list, &adap->mbox_lock)); + free(temp); return -G_FW_CMD_RETVAL((int)res); } } @@ -581,6 +586,7 @@ unsigned int t4_get_regs_len(struct adapter *adapter) switch (chip_version) { case CHELSIO_T5: + case CHELSIO_T6: return T5_REGMAP_SIZE; } @@ -1376,6 +1382,567 @@ void t4_get_regs(struct adapter *adap, void *buf, size_t buf_size) 0x51300, 0x51308, }; + static const unsigned int t6_reg_ranges[] = { + 0x1008, 0x101c, + 0x1024, 0x10a8, + 0x10b4, 0x10f8, + 0x1100, 0x1114, + 0x111c, 0x112c, + 0x1138, 0x113c, + 0x1144, 0x114c, + 0x1180, 0x1184, + 0x1190, 0x1194, + 0x11a0, 0x11a4, + 0x11b0, 0x11b4, + 0x11fc, 0x1274, + 0x1280, 0x133c, + 0x1800, 0x18fc, + 0x3000, 0x302c, + 0x3060, 0x30b0, + 0x30b8, 0x30d8, + 0x30e0, 0x30fc, + 0x3140, 0x357c, + 0x35a8, 0x35cc, + 0x35ec, 0x35ec, + 0x3600, 0x5624, + 0x56cc, 0x56ec, + 0x56f4, 0x5720, + 0x5728, 0x575c, + 0x580c, 0x5814, + 0x5890, 0x589c, + 0x58a4, 0x58ac, + 0x58b8, 0x58bc, + 0x5940, 0x595c, + 0x5980, 0x598c, + 0x59b0, 0x59c8, + 0x59d0, 0x59dc, + 0x59fc, 0x5a18, + 0x5a60, 0x5a6c, + 0x5a80, 0x5a8c, + 0x5a94, 0x5a9c, + 0x5b94, 0x5bfc, + 0x5c10, 0x5e48, + 0x5e50, 0x5e94, + 0x5ea0, 0x5eb0, + 0x5ec0, 0x5ec0, + 0x5ec8, 0x5ed0, + 0x5ee0, 0x5ee0, + 0x5ef0, 0x5ef0, + 0x5f00, 0x5f00, + 0x6000, 0x6020, + 0x6028, 0x6040, + 0x6058, 0x609c, + 0x60a8, 0x619c, + 0x7700, 0x7798, + 0x77c0, 0x7880, + 0x78cc, 0x78fc, + 0x7b00, 0x7b58, + 0x7b60, 0x7b84, + 0x7b8c, 0x7c54, + 0x7d00, 0x7d38, + 0x7d40, 0x7d84, + 0x7d8c, 0x7ddc, + 0x7de4, 0x7e04, + 0x7e10, 0x7e1c, + 0x7e24, 0x7e38, + 0x7e40, 0x7e44, + 0x7e4c, 0x7e78, + 0x7e80, 0x7edc, + 0x7ee8, 0x7efc, + 0x8dc0, 0x8de4, + 0x8df8, 0x8e04, + 0x8e10, 0x8e84, + 0x8ea0, 0x8f88, + 0x8fb8, 0x9058, + 0x9060, 0x9060, + 0x9068, 0x90f8, + 0x9100, 0x9124, + 0x9400, 0x9470, + 0x9600, 0x9600, + 0x9608, 0x9638, + 0x9640, 0x9704, + 0x9710, 0x971c, + 0x9800, 0x9808, + 0x9820, 0x983c, + 0x9850, 0x9864, + 0x9c00, 0x9c6c, + 0x9c80, 0x9cec, + 0x9d00, 0x9d6c, + 0x9d80, 0x9dec, + 0x9e00, 0x9e6c, + 0x9e80, 0x9eec, + 0x9f00, 0x9f6c, + 0x9f80, 0xa020, + 0xd004, 0xd03c, + 0xd100, 0xd118, + 0xd200, 0xd214, + 0xd220, 0xd234, + 0xd240, 0xd254, + 0xd260, 0xd274, + 0xd280, 0xd294, + 0xd2a0, 0xd2b4, + 0xd2c0, 0xd2d4, + 0xd2e0, 0xd2f4, + 0xd300, 0xd31c, + 0xdfc0, 0xdfe0, + 0xe000, 0xf008, + 0xf010, 0xf018, + 0xf020, 0xf028, + 0x11000, 0x11014, + 0x11048, 0x1106c, + 0x11074, 0x11088, + 0x11098, 0x11120, + 0x1112c, 0x1117c, + 0x11190, 0x112e0, + 0x11300, 0x1130c, + 0x12000, 0x1206c, + 0x19040, 0x1906c, + 0x19078, 0x19080, + 0x1908c, 0x190e8, + 0x190f0, 0x190f8, + 0x19100, 0x19110, + 0x19120, 0x19124, + 0x19150, 0x19194, + 0x1919c, 0x191b0, + 0x191d0, 0x191e8, + 0x19238, 0x19290, + 0x192a4, 0x192b0, + 0x192bc, 0x192bc, + 0x19348, 0x1934c, + 0x193f8, 0x19418, + 0x19420, 0x19428, + 0x19430, 0x19444, + 0x1944c, 0x1946c, + 0x19474, 0x19474, + 0x19490, 0x194cc, + 0x194f0, 0x194f8, + 0x19c00, 0x19c48, + 0x19c50, 0x19c80, + 0x19c94, 0x19c98, + 0x19ca0, 0x19cbc, + 0x19ce4, 0x19ce4, + 0x19cf0, 0x19cf8, + 0x19d00, 0x19d28, + 0x19d50, 0x19d78, + 0x19d94, 0x19d98, + 0x19da0, 0x19dc8, + 0x19df0, 0x19e10, + 0x19e50, 0x19e6c, + 0x19ea0, 0x19ebc, + 0x19ec4, 0x19ef4, + 0x19f04, 0x19f2c, + 0x19f34, 0x19f34, + 0x19f40, 0x19f50, + 0x19f90, 0x19fac, + 0x19fc4, 0x19fc8, + 0x19fd0, 0x19fe4, + 0x1a000, 0x1a004, + 0x1a010, 0x1a06c, + 0x1a0b0, 0x1a0e4, + 0x1a0ec, 0x1a0f8, + 0x1a100, 0x1a108, + 0x1a114, 0x1a120, + 0x1a128, 0x1a130, + 0x1a138, 0x1a138, + 0x1a190, 0x1a1c4, + 0x1a1fc, 0x1a1fc, + 0x1e008, 0x1e00c, + 0x1e040, 0x1e044, + 0x1e04c, 0x1e04c, + 0x1e284, 0x1e290, + 0x1e2c0, 0x1e2c0, + 0x1e2e0, 0x1e2e0, + 0x1e300, 0x1e384, + 0x1e3c0, 0x1e3c8, + 0x1e408, 0x1e40c, + 0x1e440, 0x1e444, + 0x1e44c, 0x1e44c, + 0x1e684, 0x1e690, + 0x1e6c0, 0x1e6c0, + 0x1e6e0, 0x1e6e0, + 0x1e700, 0x1e784, + 0x1e7c0, 0x1e7c8, + 0x1e808, 0x1e80c, + 0x1e840, 0x1e844, + 0x1e84c, 0x1e84c, + 0x1ea84, 0x1ea90, + 0x1eac0, 0x1eac0, + 0x1eae0, 0x1eae0, + 0x1eb00, 0x1eb84, + 0x1ebc0, 0x1ebc8, + 0x1ec08, 0x1ec0c, + 0x1ec40, 0x1ec44, + 0x1ec4c, 0x1ec4c, + 0x1ee84, 0x1ee90, + 0x1eec0, 0x1eec0, + 0x1eee0, 0x1eee0, + 0x1ef00, 0x1ef84, + 0x1efc0, 0x1efc8, + 0x1f008, 0x1f00c, + 0x1f040, 0x1f044, + 0x1f04c, 0x1f04c, + 0x1f284, 0x1f290, + 0x1f2c0, 0x1f2c0, + 0x1f2e0, 0x1f2e0, + 0x1f300, 0x1f384, + 0x1f3c0, 0x1f3c8, + 0x1f408, 0x1f40c, + 0x1f440, 0x1f444, + 0x1f44c, 0x1f44c, + 0x1f684, 0x1f690, + 0x1f6c0, 0x1f6c0, + 0x1f6e0, 0x1f6e0, + 0x1f700, 0x1f784, + 0x1f7c0, 0x1f7c8, + 0x1f808, 0x1f80c, + 0x1f840, 0x1f844, + 0x1f84c, 0x1f84c, + 0x1fa84, 0x1fa90, + 0x1fac0, 0x1fac0, + 0x1fae0, 0x1fae0, + 0x1fb00, 0x1fb84, + 0x1fbc0, 0x1fbc8, + 0x1fc08, 0x1fc0c, + 0x1fc40, 0x1fc44, + 0x1fc4c, 0x1fc4c, + 0x1fe84, 0x1fe90, + 0x1fec0, 0x1fec0, + 0x1fee0, 0x1fee0, + 0x1ff00, 0x1ff84, + 0x1ffc0, 0x1ffc8, + 0x30000, 0x30030, + 0x30100, 0x30168, + 0x30190, 0x301a0, + 0x301a8, 0x301b8, + 0x301c4, 0x301c8, + 0x301d0, 0x301d0, + 0x30200, 0x30320, + 0x30400, 0x304b4, + 0x304c0, 0x3052c, + 0x30540, 0x3061c, + 0x30800, 0x308a0, + 0x308c0, 0x30908, + 0x30910, 0x309b8, + 0x30a00, 0x30a04, + 0x30a0c, 0x30a14, + 0x30a1c, 0x30a2c, + 0x30a44, 0x30a50, + 0x30a74, 0x30a74, + 0x30a7c, 0x30afc, + 0x30b08, 0x30c24, + 0x30d00, 0x30d14, + 0x30d1c, 0x30d3c, + 0x30d44, 0x30d4c, + 0x30d54, 0x30d74, + 0x30d7c, 0x30d7c, + 0x30de0, 0x30de0, + 0x30e00, 0x30ed4, + 0x30f00, 0x30fa4, + 0x30fc0, 0x30fc4, + 0x31000, 0x31004, + 0x31080, 0x310fc, + 0x31208, 0x31220, + 0x3123c, 0x31254, + 0x31300, 0x31300, + 0x31308, 0x3131c, + 0x31338, 0x3133c, + 0x31380, 0x31380, + 0x31388, 0x313a8, + 0x313b4, 0x313b4, + 0x31400, 0x31420, + 0x31438, 0x3143c, + 0x31480, 0x31480, + 0x314a8, 0x314a8, + 0x314b0, 0x314b4, + 0x314c8, 0x314d4, + 0x31a40, 0x31a4c, + 0x31af0, 0x31b20, + 0x31b38, 0x31b3c, + 0x31b80, 0x31b80, + 0x31ba8, 0x31ba8, + 0x31bb0, 0x31bb4, + 0x31bc8, 0x31bd4, + 0x32140, 0x3218c, + 0x321f0, 0x321f4, + 0x32200, 0x32200, + 0x32218, 0x32218, + 0x32400, 0x32400, + 0x32408, 0x3241c, + 0x32618, 0x32620, + 0x32664, 0x32664, + 0x326a8, 0x326a8, + 0x326ec, 0x326ec, + 0x32a00, 0x32abc, + 0x32b00, 0x32b38, + 0x32b20, 0x32b38, + 0x32b40, 0x32b58, + 0x32b60, 0x32b78, + 0x32c00, 0x32c00, + 0x32c08, 0x32c3c, + 0x33000, 0x3302c, + 0x33034, 0x33050, + 0x33058, 0x33058, + 0x33060, 0x3308c, + 0x3309c, 0x330ac, + 0x330c0, 0x330c0, + 0x330c8, 0x330d0, + 0x330d8, 0x330e0, + 0x330ec, 0x3312c, + 0x33134, 0x33150, + 0x33158, 0x33158, + 0x33160, 0x3318c, + 0x3319c, 0x331ac, + 0x331c0, 0x331c0, + 0x331c8, 0x331d0, + 0x331d8, 0x331e0, + 0x331ec, 0x33290, + 0x33298, 0x332c4, + 0x332e4, 0x33390, + 0x33398, 0x333c4, + 0x333e4, 0x3342c, + 0x33434, 0x33450, + 0x33458, 0x33458, + 0x33460, 0x3348c, + 0x3349c, 0x334ac, + 0x334c0, 0x334c0, + 0x334c8, 0x334d0, + 0x334d8, 0x334e0, + 0x334ec, 0x3352c, + 0x33534, 0x33550, + 0x33558, 0x33558, + 0x33560, 0x3358c, + 0x3359c, 0x335ac, + 0x335c0, 0x335c0, + 0x335c8, 0x335d0, + 0x335d8, 0x335e0, + 0x335ec, 0x33690, + 0x33698, 0x336c4, + 0x336e4, 0x33790, + 0x33798, 0x337c4, + 0x337e4, 0x337fc, + 0x33814, 0x33814, + 0x33854, 0x33868, + 0x33880, 0x3388c, + 0x338c0, 0x338d0, + 0x338e8, 0x338ec, + 0x33900, 0x3392c, + 0x33934, 0x33950, + 0x33958, 0x33958, + 0x33960, 0x3398c, + 0x3399c, 0x339ac, + 0x339c0, 0x339c0, + 0x339c8, 0x339d0, + 0x339d8, 0x339e0, + 0x339ec, 0x33a90, + 0x33a98, 0x33ac4, + 0x33ae4, 0x33b10, + 0x33b24, 0x33b28, + 0x33b38, 0x33b50, + 0x33bf0, 0x33c10, + 0x33c24, 0x33c28, + 0x33c38, 0x33c50, + 0x33cf0, 0x33cfc, + 0x34000, 0x34030, + 0x34100, 0x34168, + 0x34190, 0x341a0, + 0x341a8, 0x341b8, + 0x341c4, 0x341c8, + 0x341d0, 0x341d0, + 0x34200, 0x34320, + 0x34400, 0x344b4, + 0x344c0, 0x3452c, + 0x34540, 0x3461c, + 0x34800, 0x348a0, + 0x348c0, 0x34908, + 0x34910, 0x349b8, + 0x34a00, 0x34a04, + 0x34a0c, 0x34a14, + 0x34a1c, 0x34a2c, + 0x34a44, 0x34a50, + 0x34a74, 0x34a74, + 0x34a7c, 0x34afc, + 0x34b08, 0x34c24, + 0x34d00, 0x34d14, + 0x34d1c, 0x34d3c, + 0x34d44, 0x34d4c, + 0x34d54, 0x34d74, + 0x34d7c, 0x34d7c, + 0x34de0, 0x34de0, + 0x34e00, 0x34ed4, + 0x34f00, 0x34fa4, + 0x34fc0, 0x34fc4, + 0x35000, 0x35004, + 0x35080, 0x350fc, + 0x35208, 0x35220, + 0x3523c, 0x35254, + 0x35300, 0x35300, + 0x35308, 0x3531c, + 0x35338, 0x3533c, + 0x35380, 0x35380, + 0x35388, 0x353a8, + 0x353b4, 0x353b4, + 0x35400, 0x35420, + 0x35438, 0x3543c, + 0x35480, 0x35480, + 0x354a8, 0x354a8, + 0x354b0, 0x354b4, + 0x354c8, 0x354d4, + 0x35a40, 0x35a4c, + 0x35af0, 0x35b20, + 0x35b38, 0x35b3c, + 0x35b80, 0x35b80, + 0x35ba8, 0x35ba8, + 0x35bb0, 0x35bb4, + 0x35bc8, 0x35bd4, + 0x36140, 0x3618c, + 0x361f0, 0x361f4, + 0x36200, 0x36200, + 0x36218, 0x36218, + 0x36400, 0x36400, + 0x36408, 0x3641c, + 0x36618, 0x36620, + 0x36664, 0x36664, + 0x366a8, 0x366a8, + 0x366ec, 0x366ec, + 0x36a00, 0x36abc, + 0x36b00, 0x36b38, + 0x36b20, 0x36b38, + 0x36b40, 0x36b58, + 0x36b60, 0x36b78, + 0x36c00, 0x36c00, + 0x36c08, 0x36c3c, + 0x37000, 0x3702c, + 0x37034, 0x37050, + 0x37058, 0x37058, + 0x37060, 0x3708c, + 0x3709c, 0x370ac, + 0x370c0, 0x370c0, + 0x370c8, 0x370d0, + 0x370d8, 0x370e0, + 0x370ec, 0x3712c, + 0x37134, 0x37150, + 0x37158, 0x37158, + 0x37160, 0x3718c, + 0x3719c, 0x371ac, + 0x371c0, 0x371c0, + 0x371c8, 0x371d0, + 0x371d8, 0x371e0, + 0x371ec, 0x37290, + 0x37298, 0x372c4, + 0x372e4, 0x37390, + 0x37398, 0x373c4, + 0x373e4, 0x3742c, + 0x37434, 0x37450, + 0x37458, 0x37458, + 0x37460, 0x3748c, + 0x3749c, 0x374ac, + 0x374c0, 0x374c0, + 0x374c8, 0x374d0, + 0x374d8, 0x374e0, + 0x374ec, 0x3752c, + 0x37534, 0x37550, + 0x37558, 0x37558, + 0x37560, 0x3758c, + 0x3759c, 0x375ac, + 0x375c0, 0x375c0, + 0x375c8, 0x375d0, + 0x375d8, 0x375e0, + 0x375ec, 0x37690, + 0x37698, 0x376c4, + 0x376e4, 0x37790, + 0x37798, 0x377c4, + 0x377e4, 0x377fc, + 0x37814, 0x37814, + 0x37854, 0x37868, + 0x37880, 0x3788c, + 0x378c0, 0x378d0, + 0x378e8, 0x378ec, + 0x37900, 0x3792c, + 0x37934, 0x37950, + 0x37958, 0x37958, + 0x37960, 0x3798c, + 0x3799c, 0x379ac, + 0x379c0, 0x379c0, + 0x379c8, 0x379d0, + 0x379d8, 0x379e0, + 0x379ec, 0x37a90, + 0x37a98, 0x37ac4, + 0x37ae4, 0x37b10, + 0x37b24, 0x37b28, + 0x37b38, 0x37b50, + 0x37bf0, 0x37c10, + 0x37c24, 0x37c28, + 0x37c38, 0x37c50, + 0x37cf0, 0x37cfc, + 0x40040, 0x40040, + 0x40080, 0x40084, + 0x40100, 0x40100, + 0x40140, 0x401bc, + 0x40200, 0x40214, + 0x40228, 0x40228, + 0x40240, 0x40258, + 0x40280, 0x40280, + 0x40304, 0x40304, + 0x40330, 0x4033c, + 0x41304, 0x413c8, + 0x413d0, 0x413dc, + 0x413f0, 0x413f0, + 0x41400, 0x4140c, + 0x41414, 0x4141c, + 0x41480, 0x414d0, + 0x44000, 0x4407c, + 0x440c0, 0x441ac, + 0x441b4, 0x4427c, + 0x442c0, 0x443ac, + 0x443b4, 0x4447c, + 0x444c0, 0x445ac, + 0x445b4, 0x4467c, + 0x446c0, 0x447ac, + 0x447b4, 0x4487c, + 0x448c0, 0x449ac, + 0x449b4, 0x44a7c, + 0x44ac0, 0x44bac, + 0x44bb4, 0x44c7c, + 0x44cc0, 0x44dac, + 0x44db4, 0x44e7c, + 0x44ec0, 0x44fac, + 0x44fb4, 0x4507c, + 0x450c0, 0x451ac, + 0x451b4, 0x451fc, + 0x45800, 0x45804, + 0x45810, 0x45830, + 0x45840, 0x45860, + 0x45868, 0x45868, + 0x45880, 0x45884, + 0x458a0, 0x458b0, + 0x45a00, 0x45a04, + 0x45a10, 0x45a30, + 0x45a40, 0x45a60, + 0x45a68, 0x45a68, + 0x45a80, 0x45a84, + 0x45aa0, 0x45ab0, + 0x460c0, 0x460e4, + 0x47000, 0x4703c, + 0x47044, 0x4708c, + 0x47200, 0x47250, + 0x47400, 0x47408, + 0x47414, 0x47420, + 0x47600, 0x47618, + 0x47800, 0x47814, + 0x47820, 0x4782c, + 0x50000, 0x50084, + 0x50090, 0x500cc, + 0x50300, 0x50384, + 0x50400, 0x50400, + 0x50800, 0x50884, + 0x50890, 0x508cc, + 0x50b00, 0x50b84, + 0x50c00, 0x50c00, + 0x51000, 0x51020, + 0x51028, 0x510b0, + 0x51300, 0x51324, + }; + u32 *buf_end = (u32 *)((char *)buf + buf_size); const unsigned int *reg_ranges; int reg_ranges_size, range; @@ -1390,6 +1957,11 @@ void t4_get_regs(struct adapter *adap, void *buf, size_t buf_size) reg_ranges_size = ARRAY_SIZE(t5_reg_ranges); break; + case CHELSIO_T6: + reg_ranges = t6_reg_ranges; + reg_ranges_size = ARRAY_SIZE(t6_reg_ranges); + break; + default: dev_err(adap, "Unsupported chip version %d\n", chip_version); @@ -1532,7 +2104,7 @@ int t4_seeprom_write(struct adapter *adapter, u32 addr, u32 data) { unsigned int base = adapter->params.pci.vpd_cap_addr; int ret; - u32 stats_reg; + u32 stats_reg = 0; int max_poll; /* VPD Accesses must alway be 4-byte aligned! @@ -1925,6 +2497,43 @@ int t4_read_flash(struct adapter *adapter, unsigned int addr, return 0; } +/** + * t4_get_exprom_version - return the Expansion ROM version (if any) + * @adapter: the adapter + * @vers: where to place the version + * + * Reads the Expansion ROM header from FLASH and returns the version + * number (if present) through the @vers return value pointer. We return + * this in the Firmware Version Format since it's convenient. Return + * 0 on success, -ENOENT if no Expansion ROM is present. + */ +static int t4_get_exprom_version(struct adapter *adapter, u32 *vers) +{ + struct exprom_header { + unsigned char hdr_arr[16]; /* must start with 0x55aa */ + unsigned char hdr_ver[4]; /* Expansion ROM version */ + } *hdr; + u32 exprom_header_buf[DIV_ROUND_UP(sizeof(struct exprom_header), + sizeof(u32))]; + int ret; + + ret = t4_read_flash(adapter, FLASH_EXP_ROM_START, + ARRAY_SIZE(exprom_header_buf), + exprom_header_buf, 0); + if (ret) + return ret; + + hdr = (struct exprom_header *)exprom_header_buf; + if (hdr->hdr_arr[0] != 0x55 || hdr->hdr_arr[1] != 0xaa) + return -ENOENT; + + *vers = (V_FW_HDR_FW_VER_MAJOR(hdr->hdr_ver[0]) | + V_FW_HDR_FW_VER_MINOR(hdr->hdr_ver[1]) | + V_FW_HDR_FW_VER_MICRO(hdr->hdr_ver[2]) | + V_FW_HDR_FW_VER_BUILD(hdr->hdr_ver[3])); + return 0; +} + /** * t4_get_fw_version - read the firmware version * @adapter: the adapter @@ -1932,12 +2541,26 @@ int t4_read_flash(struct adapter *adapter, unsigned int addr, * * Reads the FW version from flash. */ -int t4_get_fw_version(struct adapter *adapter, u32 *vers) +static int t4_get_fw_version(struct adapter *adapter, u32 *vers) { return t4_read_flash(adapter, FLASH_FW_START + offsetof(struct fw_hdr, fw_ver), 1, vers, 0); } +/** + * t4_get_bs_version - read the firmware bootstrap version + * @adapter: the adapter + * @vers: where to place the version + * + * Reads the FW Bootstrap version from flash. + */ +static int t4_get_bs_version(struct adapter *adapter, u32 *vers) +{ + return t4_read_flash(adapter, FLASH_FWBOOTSTRAP_START + + offsetof(struct fw_hdr, fw_ver), 1, + vers, 0); +} + /** * t4_get_tp_version - read the TP microcode version * @adapter: the adapter @@ -1945,16 +2568,110 @@ int t4_get_fw_version(struct adapter *adapter, u32 *vers) * * Reads the TP microcode version from flash. */ -int t4_get_tp_version(struct adapter *adapter, u32 *vers) +static int t4_get_tp_version(struct adapter *adapter, u32 *vers) { return t4_read_flash(adapter, FLASH_FW_START + offsetof(struct fw_hdr, tp_microcode_ver), 1, vers, 0); } -#define ADVERT_MASK (FW_PORT_CAP_SPEED_100M | FW_PORT_CAP_SPEED_1G |\ - FW_PORT_CAP_SPEED_10G | FW_PORT_CAP_SPEED_40G | \ - FW_PORT_CAP_SPEED_100G | FW_PORT_CAP_ANEG) +/** + * t4_get_version_info - extract various chip/firmware version information + * @adapter: the adapter + * + * Reads various chip/firmware version numbers and stores them into the + * adapter Adapter Parameters structure. If any of the efforts fails + * the first failure will be returned, but all of the version numbers + * will be read. + */ +int t4_get_version_info(struct adapter *adapter) +{ + int ret = 0; + +#define FIRST_RET(__getvinfo) \ + do { \ + int __ret = __getvinfo; \ + if (__ret && !ret) \ + ret = __ret; \ + } while (0) + + FIRST_RET(t4_get_fw_version(adapter, &adapter->params.fw_vers)); + FIRST_RET(t4_get_bs_version(adapter, &adapter->params.bs_vers)); + FIRST_RET(t4_get_tp_version(adapter, &adapter->params.tp_vers)); + FIRST_RET(t4_get_exprom_version(adapter, &adapter->params.er_vers)); + +#undef FIRST_RET + + return ret; +} + +/** + * t4_dump_version_info - dump all of the adapter configuration IDs + * @adapter: the adapter + * + * Dumps all of the various bits of adapter configuration version/revision + * IDs information. This is typically called at some point after + * t4_get_version_info() has been called. + */ +void t4_dump_version_info(struct adapter *adapter) +{ + /** + * Device information. + */ + dev_info(adapter, "Chelsio rev %d\n", + CHELSIO_CHIP_RELEASE(adapter->params.chip)); + + /** + * Firmware Version. + */ + if (!adapter->params.fw_vers) + dev_warn(adapter, "No firmware loaded\n"); + else + dev_info(adapter, "Firmware version: %u.%u.%u.%u\n", + G_FW_HDR_FW_VER_MAJOR(adapter->params.fw_vers), + G_FW_HDR_FW_VER_MINOR(adapter->params.fw_vers), + G_FW_HDR_FW_VER_MICRO(adapter->params.fw_vers), + G_FW_HDR_FW_VER_BUILD(adapter->params.fw_vers)); + + /** + * Bootstrap Firmware Version. + */ + if (!adapter->params.bs_vers) + dev_warn(adapter, "No bootstrap loaded\n"); + else + dev_info(adapter, "Bootstrap version: %u.%u.%u.%u\n", + G_FW_HDR_FW_VER_MAJOR(adapter->params.bs_vers), + G_FW_HDR_FW_VER_MINOR(adapter->params.bs_vers), + G_FW_HDR_FW_VER_MICRO(adapter->params.bs_vers), + G_FW_HDR_FW_VER_BUILD(adapter->params.bs_vers)); + + /** + * TP Microcode Version. + */ + if (!adapter->params.tp_vers) + dev_warn(adapter, "No TP Microcode loaded\n"); + else + dev_info(adapter, "TP Microcode version: %u.%u.%u.%u\n", + G_FW_HDR_FW_VER_MAJOR(adapter->params.tp_vers), + G_FW_HDR_FW_VER_MINOR(adapter->params.tp_vers), + G_FW_HDR_FW_VER_MICRO(adapter->params.tp_vers), + G_FW_HDR_FW_VER_BUILD(adapter->params.tp_vers)); + + /** + * Expansion ROM version. + */ + if (!adapter->params.er_vers) + dev_info(adapter, "No Expansion ROM loaded\n"); + else + dev_info(adapter, "Expansion ROM version: %u.%u.%u.%u\n", + G_FW_HDR_FW_VER_MAJOR(adapter->params.er_vers), + G_FW_HDR_FW_VER_MINOR(adapter->params.er_vers), + G_FW_HDR_FW_VER_MICRO(adapter->params.er_vers), + G_FW_HDR_FW_VER_BUILD(adapter->params.er_vers)); +} + +#define ADVERT_MASK (V_FW_PORT_CAP_SPEED(M_FW_PORT_CAP_SPEED) | \ + FW_PORT_CAP_ANEG) /** * t4_link_l1cfg - apply link configuration to MAC/PHY @@ -1973,14 +2690,24 @@ int t4_link_l1cfg(struct adapter *adap, unsigned int mbox, unsigned int port, struct link_config *lc) { struct fw_port_cmd c; - unsigned int fc = 0, mdi = V_FW_PORT_CAP_MDI(FW_PORT_CAP_MDI_AUTO); + unsigned int mdi = V_FW_PORT_CAP_MDI(FW_PORT_CAP_MDI_AUTO); + unsigned int fc, fec; lc->link_ok = 0; + fc = 0; if (lc->requested_fc & PAUSE_RX) fc |= FW_PORT_CAP_FC_RX; if (lc->requested_fc & PAUSE_TX) fc |= FW_PORT_CAP_FC_TX; + fec = 0; + if (lc->requested_fec & FEC_RS) + fec |= FW_PORT_CAP_FEC_RS; + if (lc->requested_fec & FEC_BASER_RS) + fec |= FW_PORT_CAP_FEC_BASER_RS; + if (lc->requested_fec & FEC_RESERVED) + fec |= FW_PORT_CAP_FEC_RESERVED; + memset(&c, 0, sizeof(c)); c.op_to_portid = cpu_to_be32(V_FW_CMD_OP(FW_PORT_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_EXEC | @@ -1991,13 +2718,16 @@ int t4_link_l1cfg(struct adapter *adap, unsigned int mbox, unsigned int port, if (!(lc->supported & FW_PORT_CAP_ANEG)) { c.u.l1cfg.rcap = cpu_to_be32((lc->supported & ADVERT_MASK) | - fc); - lc->fc = lc->requested_fc & (PAUSE_RX | PAUSE_TX); + fc | fec); + lc->fc = lc->requested_fc & ~PAUSE_AUTONEG; + lc->fec = lc->requested_fec; } else if (lc->autoneg == AUTONEG_DISABLE) { - c.u.l1cfg.rcap = cpu_to_be32(lc->requested_speed | fc | mdi); - lc->fc = lc->requested_fc & (PAUSE_RX | PAUSE_TX); + c.u.l1cfg.rcap = cpu_to_be32(lc->requested_speed | fc | + fec | mdi); + lc->fc = lc->requested_fc & ~PAUSE_AUTONEG; + lc->fec = lc->requested_fec; } else { - c.u.l1cfg.rcap = cpu_to_be32(lc->advertising | fc | mdi); + c.u.l1cfg.rcap = cpu_to_be32(lc->advertising | fc | fec | mdi); } return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL); @@ -2041,7 +2771,9 @@ int t4_flash_cfg_addr(struct adapter *adapter) void t4_intr_enable(struct adapter *adapter) { u32 val = 0; - u32 pf = G_SOURCEPF(t4_read_reg(adapter, A_PL_WHOAMI)); + u32 whoami = t4_read_reg(adapter, A_PL_WHOAMI); + u32 pf = CHELSIO_CHIP_VERSION(adapter->params.chip) <= CHELSIO_T5 ? + G_SOURCEPF(whoami) : G_T6_SOURCEPF(whoami); if (CHELSIO_CHIP_VERSION(adapter->params.chip) <= CHELSIO_T5) val = F_ERR_DROPPED_DB | F_ERR_EGR_CTXT_PRIO | F_DBFIFO_HP_INT; @@ -2066,7 +2798,9 @@ void t4_intr_enable(struct adapter *adapter) */ void t4_intr_disable(struct adapter *adapter) { - u32 pf = G_SOURCEPF(t4_read_reg(adapter, A_PL_WHOAMI)); + u32 whoami = t4_read_reg(adapter, A_PL_WHOAMI); + u32 pf = CHELSIO_CHIP_VERSION(adapter->params.chip) <= CHELSIO_T5 ? + G_SOURCEPF(whoami) : G_T6_SOURCEPF(whoami); t4_write_reg(adapter, MYPF_REG(A_PL_PF_INT_ENABLE), 0); t4_set_reg_field(adapter, A_PL_INT_MAP0, 1 << pf, 0); @@ -2095,6 +2829,12 @@ const char *t4_get_port_type_description(enum fw_port_type port_type) "QSA", "QSFP", "BP40_BA", + "KR4_100G", + "CR4_QSFP", + "CR_QSFP", + "CR2_QSFP", + "SFP28", + "KR_SFP28", }; if (port_type < ARRAY_SIZE(port_type_description)) @@ -2105,21 +2845,90 @@ const char *t4_get_port_type_description(enum fw_port_type port_type) /** * t4_get_mps_bg_map - return the buffer groups associated with a port * @adap: the adapter - * @idx: the port index + * @pidx: the port index * * Returns a bitmap indicating which MPS buffer groups are associated * with the given port. Bit i is set if buffer group i is used by the * port. */ -unsigned int t4_get_mps_bg_map(struct adapter *adap, int idx) +unsigned int t4_get_mps_bg_map(struct adapter *adap, unsigned int pidx) { - u32 n = G_NUMPORTS(t4_read_reg(adap, A_MPS_CMN_CTL)); + unsigned int chip_version = CHELSIO_CHIP_VERSION(adap->params.chip); + unsigned int nports = 1 << G_NUMPORTS(t4_read_reg(adap, + A_MPS_CMN_CTL)); - if (n == 0) - return idx == 0 ? 0xf : 0; - if (n == 1) - return idx < 2 ? (3 << (2 * idx)) : 0; - return 1 << idx; + if (pidx >= nports) { + dev_warn(adap, "MPS Port Index %d >= Nports %d\n", + pidx, nports); + return 0; + } + + switch (chip_version) { + case CHELSIO_T4: + case CHELSIO_T5: + switch (nports) { + case 1: return 0xf; + case 2: return 3 << (2 * pidx); + case 4: return 1 << pidx; + } + break; + + case CHELSIO_T6: + switch (nports) { + case 2: return 1 << (2 * pidx); + } + break; + } + + dev_err(adap, "Need MPS Buffer Group Map for Chip %0x, Nports %d\n", + chip_version, nports); + return 0; +} + +/** + * t4_get_tp_ch_map - return TP ingress channels associated with a port + * @adapter: the adapter + * @pidx: the port index + * + * Returns a bitmap indicating which TP Ingress Channels are associated with + * a given Port. Bit i is set if TP Ingress Channel i is used by the Port. + */ +unsigned int t4_get_tp_ch_map(struct adapter *adapter, unsigned int pidx) +{ + unsigned int chip_version = CHELSIO_CHIP_VERSION(adapter->params.chip); + unsigned int nports = 1 << G_NUMPORTS(t4_read_reg(adapter, + A_MPS_CMN_CTL)); + + if (pidx >= nports) { + dev_warn(adap, "TP Port Index %d >= Nports %d\n", + pidx, nports); + return 0; + } + + switch (chip_version) { + case CHELSIO_T4: + case CHELSIO_T5: + /* Note that this happens to be the same values as the MPS + * Buffer Group Map for these Chips. But we replicate the code + * here because they're really separate concepts. + */ + switch (nports) { + case 1: return 0xf; + case 2: return 3 << (2 * pidx); + case 4: return 1 << pidx; + } + break; + + case CHELSIO_T6: + switch (nports) { + case 2: return 1 << pidx; + } + break; + } + + dev_err(adapter, "Need TP Channel Map for Chip %0x, Nports %d\n", + chip_version, nports); + return 0; } /** @@ -2133,6 +2942,7 @@ unsigned int t4_get_mps_bg_map(struct adapter *adap, int idx) void t4_get_port_stats(struct adapter *adap, int idx, struct port_stats *p) { u32 bgmap = t4_get_mps_bg_map(adap, idx); + u32 stat_ctl = t4_read_reg(adap, A_MPS_STAT_CTL); #define GET_STAT(name) \ t4_read_reg64(adap, \ @@ -2165,6 +2975,15 @@ void t4_get_port_stats(struct adapter *adap, int idx, struct port_stats *p) p->tx_ppp6 = GET_STAT(TX_PORT_PPP6); p->tx_ppp7 = GET_STAT(TX_PORT_PPP7); + if (CHELSIO_CHIP_VERSION(adap->params.chip) >= CHELSIO_T5) { + if (stat_ctl & F_COUNTPAUSESTATTX) { + p->tx_frames -= p->tx_pause; + p->tx_octets -= p->tx_pause * 64; + } + if (stat_ctl & F_COUNTPAUSEMCTX) + p->tx_mcast_frames -= p->tx_pause; + } + p->rx_octets = GET_STAT(RX_PORT_BYTES); p->rx_frames = GET_STAT(RX_PORT_FRAMES); p->rx_bcast_frames = GET_STAT(RX_PORT_BCAST); @@ -2192,6 +3011,16 @@ void t4_get_port_stats(struct adapter *adap, int idx, struct port_stats *p) p->rx_ppp5 = GET_STAT(RX_PORT_PPP5); p->rx_ppp6 = GET_STAT(RX_PORT_PPP6); p->rx_ppp7 = GET_STAT(RX_PORT_PPP7); + + if (CHELSIO_CHIP_VERSION(adap->params.chip) >= CHELSIO_T5) { + if (stat_ctl & F_COUNTPAUSESTATRX) { + p->rx_frames -= p->rx_pause; + p->rx_octets -= p->rx_pause * 64; + } + if (stat_ctl & F_COUNTPAUSEMCRX) + p->rx_mcast_frames -= p->rx_pause; + } + p->rx_ovflow0 = (bgmap & 1) ? GET_STAT_COM(RX_BG_0_MAC_DROP_FRAME) : 0; p->rx_ovflow1 = (bgmap & 2) ? GET_STAT_COM(RX_BG_1_MAC_DROP_FRAME) : 0; p->rx_ovflow2 = (bgmap & 4) ? GET_STAT_COM(RX_BG_2_MAC_DROP_FRAME) : 0; @@ -2549,6 +3378,49 @@ int t4_fw_restart(struct adapter *adap, unsigned int mbox, int reset) return 0; } +/** + * t4_fl_pkt_align - return the fl packet alignment + * @adap: the adapter + * + * T4 has a single field to specify the packing and padding boundary. + * T5 onwards has separate fields for this and hence the alignment for + * next packet offset is maximum of these two. + */ +int t4_fl_pkt_align(struct adapter *adap) +{ + u32 sge_control, sge_control2; + unsigned int ingpadboundary, ingpackboundary, fl_align, ingpad_shift; + + sge_control = t4_read_reg(adap, A_SGE_CONTROL); + + /* T4 uses a single control field to specify both the PCIe Padding and + * Packing Boundary. T5 introduced the ability to specify these + * separately. The actual Ingress Packet Data alignment boundary + * within Packed Buffer Mode is the maximum of these two + * specifications. + */ + if (CHELSIO_CHIP_VERSION(adap->params.chip) <= CHELSIO_T5) + ingpad_shift = X_INGPADBOUNDARY_SHIFT; + else + ingpad_shift = X_T6_INGPADBOUNDARY_SHIFT; + + ingpadboundary = 1 << (G_INGPADBOUNDARY(sge_control) + ingpad_shift); + + fl_align = ingpadboundary; + if (!is_t4(adap->params.chip)) { + sge_control2 = t4_read_reg(adap, A_SGE_CONTROL2); + ingpackboundary = G_INGPACKBOUNDARY(sge_control2); + if (ingpackboundary == X_INGPACKBOUNDARY_16B) + ingpackboundary = 16; + else + ingpackboundary = 1 << (ingpackboundary + + X_INGPACKBOUNDARY_SHIFT); + + fl_align = max(ingpadboundary, ingpackboundary); + } + return fl_align; +} + /** * t4_fixup_host_params_compat - fix up host-dependent parameters * @adap: the adapter @@ -2594,6 +3466,10 @@ int t4_fixup_host_params_compat(struct adapter *adap, X_INGPADBOUNDARY_SHIFT) | V_EGRSTATUSPAGESIZE(stat_len != 64)); else { + unsigned int pack_align; + unsigned int ingpad, ingpack; + unsigned int pcie_cap; + /* * T5 introduced the separation of the Free List Padding and * Packing Boundaries. Thus, we can select a smaller Padding @@ -2607,12 +3483,34 @@ int t4_fixup_host_params_compat(struct adapter *adap, * Size (the minimum unit of transfer to/from Memory). If we * have a Padding Boundary which is smaller than the Memory * Line Size, that'll involve a Read-Modify-Write cycle on the - * Memory Controller which is never good. For T5 the smallest - * Padding Boundary which we can select is 32 bytes which is - * larger than any known Memory Controller Line Size so we'll - * use that. + * Memory Controller which is never good. */ + /* We want the Packing Boundary to be based on the Cache Line + * Size in order to help avoid False Sharing performance + * issues between CPUs, etc. We also want the Packing + * Boundary to incorporate the PCI-E Maximum Payload Size. We + * get best performance when the Packing Boundary is a + * multiple of the Maximum Payload Size. + */ + pack_align = fl_align; + pcie_cap = t4_os_find_pci_capability(adap, PCI_CAP_ID_EXP); + if (pcie_cap) { + unsigned int mps, mps_log; + u16 devctl; + + /* The PCIe Device Control Maximum Payload Size field + * [bits 7:5] encodes sizes as powers of 2 starting at + * 128 bytes. + */ + t4_os_pci_read_cfg2(adap, pcie_cap + PCI_EXP_DEVCTL, + &devctl); + mps_log = ((devctl & PCI_EXP_DEVCTL_PAYLOAD) >> 5) + 7; + mps = 1 << mps_log; + if (mps > pack_align) + pack_align = mps; + } + /* * N.B. T5 has a different interpretation of the "0" value for * the Packing Boundary. This corresponds to 16 bytes instead @@ -2621,19 +3519,36 @@ int t4_fixup_host_params_compat(struct adapter *adap, * on the other hand, if we wanted 32 bytes, the best we can * really do is 64 bytes ... */ - if (fl_align <= 32) { + if (pack_align <= 16) { + ingpack = X_INGPACKBOUNDARY_16B; + fl_align = 16; + } else if (pack_align == 32) { + ingpack = X_INGPACKBOUNDARY_64B; fl_align = 64; - fl_align_log = 6; + } else { + unsigned int pack_align_log = cxgbe_fls(pack_align) - 1; + + ingpack = pack_align_log - X_INGPACKBOUNDARY_SHIFT; + fl_align = pack_align; } + + /* Use the smallest Ingress Padding which isn't smaller than + * the Memory Controller Read/Write Size. We'll take that as + * being 8 bytes since we don't know of any system with a + * wider Memory Controller Bus Width. + */ + if (is_t5(adap->params.chip)) + ingpad = X_INGPADBOUNDARY_32B; + else + ingpad = X_T6_INGPADBOUNDARY_8B; t4_set_reg_field(adap, A_SGE_CONTROL, V_INGPADBOUNDARY(M_INGPADBOUNDARY) | F_EGRSTATUSPAGESIZE, - V_INGPADBOUNDARY(X_INGPCIEBOUNDARY_32B) | + V_INGPADBOUNDARY(ingpad) | V_EGRSTATUSPAGESIZE(stat_len != 64)); t4_set_reg_field(adap, A_SGE_CONTROL2, V_INGPACKBOUNDARY(M_INGPACKBOUNDARY), - V_INGPACKBOUNDARY(fl_align_log - - X_INGPACKBOUNDARY_SHIFT)); + V_INGPACKBOUNDARY(ingpack)); } /* @@ -3171,7 +4086,7 @@ int t4_handle_fw_rpl(struct adapter *adap, const __be64 *rpl) if (opcode == FW_PORT_CMD && action == FW_PORT_ACTION_GET_PORT_INFO) { /* link/module state change message */ - int speed = 0, fc = 0, i; + unsigned int speed = 0, fc = 0, i; int chan = G_FW_PORT_CMD_PORTID(be32_to_cpu(p->op_to_portid)); struct port_info *pi = NULL; struct link_config *lc; @@ -3189,8 +4104,12 @@ int t4_handle_fw_rpl(struct adapter *adap, const __be64 *rpl) speed = ETH_SPEED_NUM_1G; else if (stat & V_FW_PORT_CMD_LSPEED(FW_PORT_CAP_SPEED_10G)) speed = ETH_SPEED_NUM_10G; + else if (stat & V_FW_PORT_CMD_LSPEED(FW_PORT_CAP_SPEED_25G)) + speed = ETH_SPEED_NUM_25G; else if (stat & V_FW_PORT_CMD_LSPEED(FW_PORT_CAP_SPEED_40G)) speed = ETH_SPEED_NUM_40G; + else if (stat & V_FW_PORT_CMD_LSPEED(FW_PORT_CAP_SPEED_100G)) + speed = ETH_SPEED_NUM_100G; for_each_port(adap, i) { pi = adap2pinfo(adap, i); @@ -3248,19 +4167,37 @@ void t4_reset_link_config(struct adapter *adap, int idx) /** * init_link_config - initialize a link's SW state * @lc: structure holding the link state - * @caps: link capabilities + * @pcaps: link Port Capabilities + * @acaps: link current Advertised Port Capabilities * * Initializes the SW state maintained for each link, including the link's * capabilities and default speed/flow-control/autonegotiation settings. */ -static void init_link_config(struct link_config *lc, - unsigned int caps) +static void init_link_config(struct link_config *lc, unsigned int pcaps, + unsigned int acaps) { - lc->supported = caps; + unsigned int fec; + + lc->supported = pcaps; lc->requested_speed = 0; lc->speed = 0; lc->requested_fc = 0; lc->fc = 0; + + /** + * For Forward Error Control, we default to whatever the Firmware + * tells us the Link is currently advertising. + */ + fec = 0; + if (acaps & FW_PORT_CAP_FEC_RS) + fec |= FEC_RS; + if (acaps & FW_PORT_CAP_FEC_BASER_RS) + fec |= FEC_BASER_RS; + if (acaps & FW_PORT_CAP_FEC_RESERVED) + fec |= FEC_RESERVED; + lc->requested_fec = fec; + lc->fec = fec; + if (lc->supported & FW_PORT_CAP_ANEG) { lc->advertising = lc->supported & ADVERT_MASK; lc->autoneg = AUTONEG_ENABLE; @@ -3289,8 +4226,12 @@ static int t4_wait_dev_ready(struct adapter *adapter) msleep(500); whoami = t4_read_reg(adapter, A_PL_WHOAMI); - return (whoami != 0xffffffff && whoami != X_CIM_PF_NOACCESS - ? 0 : -EIO); + if (whoami != 0xffffffff && whoami != X_CIM_PF_NOACCESS) + return 0; + + dev_err(adapter, "Device didn't become ready for access, whoami = %#x\n", + whoami); + return -EIO; } struct flash_desc { @@ -3306,47 +4247,96 @@ int t4_get_flash_params(struct adapter *adapter) * sectors. */ static struct flash_desc supported_flash[] = { - { 0x150201, 4 << 20 }, /* Spansion 4MB S25FL032P */ + { 0x00150201, 4 << 20 }, /* Spansion 4MB S25FL032P */ }; int ret; - unsigned int i; - u32 info = 0; + u32 flashid = 0; + unsigned int part, manufacturer; + unsigned int density, size; + /** + * Issue a Read ID Command to the Flash part. We decode supported + * Flash parts and their sizes from this. There's a newer Query + * Command which can retrieve detailed geometry information but + * many Flash parts don't support it. + */ ret = sf1_write(adapter, 1, 1, 0, SF_RD_ID); if (!ret) - ret = sf1_read(adapter, 3, 0, 1, &info); + ret = sf1_read(adapter, 3, 0, 1, &flashid); t4_write_reg(adapter, A_SF_OP, 0); /* unlock SF */ if (ret < 0) return ret; - for (i = 0; i < ARRAY_SIZE(supported_flash); ++i) - if (supported_flash[i].vendor_and_model_id == info) { - adapter->params.sf_size = supported_flash[i].size_mb; + for (part = 0; part < ARRAY_SIZE(supported_flash); part++) { + if (supported_flash[part].vendor_and_model_id == flashid) { + adapter->params.sf_size = + supported_flash[part].size_mb; adapter->params.sf_nsec = adapter->params.sf_size / SF_SEC_SIZE; - return 0; + goto found; + } + } + + manufacturer = flashid & 0xff; + switch (manufacturer) { + case 0x20: { /* Micron/Numonix */ + /** + * This Density -> Size decoding table is taken from Micron + * Data Sheets. + */ + density = (flashid >> 16) & 0xff; + switch (density) { + case 0x14: + size = 1 << 20; /* 1MB */ + break; + case 0x15: + size = 1 << 21; /* 2MB */ + break; + case 0x16: + size = 1 << 22; /* 4MB */ + break; + case 0x17: + size = 1 << 23; /* 8MB */ + break; + case 0x18: + size = 1 << 24; /* 16MB */ + break; + case 0x19: + size = 1 << 25; /* 32MB */ + break; + case 0x20: + size = 1 << 26; /* 64MB */ + break; + case 0x21: + size = 1 << 27; /* 128MB */ + break; + case 0x22: + size = 1 << 28; /* 256MB */ + break; + default: + dev_err(adapter, "Micron Flash Part has bad size, ID = %#x, Density code = %#x\n", + flashid, density); + return -EINVAL; } - if ((info & 0xff) != 0x20) /* not a Numonix flash */ + adapter->params.sf_size = size; + adapter->params.sf_nsec = size / SF_SEC_SIZE; + break; + } + default: + dev_err(adapter, "Unsupported Flash Part, ID = %#x\n", flashid); return -EINVAL; - info >>= 16; /* log2 of size */ - if (info >= 0x14 && info < 0x18) - adapter->params.sf_nsec = 1 << (info - 16); - else if (info == 0x18) - adapter->params.sf_nsec = 64; - else - return -EINVAL; - adapter->params.sf_size = 1 << info; + } +found: /* * We should reject adapters with FLASHes which are too small. So, emit * a warning. */ - if (adapter->params.sf_size < FLASH_MIN_SIZE) { - dev_warn(adapter, "WARNING!!! FLASH size %#x < %#x!!!\n", - adapter->params.sf_size, FLASH_MIN_SIZE); - } + if (adapter->params.sf_size < FLASH_MIN_SIZE) + dev_warn(adapter, "WARNING: Flash Part ID %#x, size %#x < %#x\n", + flashid, adapter->params.sf_size, FLASH_MIN_SIZE); return 0; } @@ -3366,6 +4356,33 @@ static void set_pcie_completion_timeout(struct adapter *adapter, } } +/** + * t4_get_chip_type - Determine chip type from device ID + * @adap: the adapter + * @ver: adapter version + */ +int t4_get_chip_type(struct adapter *adap, int ver) +{ + enum chip_type chip = 0; + u32 pl_rev = G_REV(t4_read_reg(adap, A_PL_REV)); + + /* Retrieve adapter's device ID */ + switch (ver) { + case CHELSIO_T5: + chip |= CHELSIO_CHIP_CODE(CHELSIO_T5, pl_rev); + break; + case CHELSIO_T6: + chip |= CHELSIO_CHIP_CODE(CHELSIO_T6, pl_rev); + break; + default: + dev_err(adap, "Device %d is not supported\n", + adap->params.pci.device_id); + return -EINVAL; + } + + return chip; +} + /** * t4_prep_adapter - prepare SW and HW for operation * @adapter: the adapter @@ -3403,6 +4420,15 @@ int t4_prep_adapter(struct adapter *adapter) adapter->params.arch.nchan = NCHAN; adapter->params.arch.vfcount = 128; break; + case CHELSIO_T6: + adapter->params.chip |= CHELSIO_CHIP_CODE(CHELSIO_T6, pl_rev); + adapter->params.arch.sge_fl_db = 0; + adapter->params.arch.mps_tcam_size = + NUM_MPS_T5_CLS_SRAM_L_INSTANCES; + adapter->params.arch.mps_rplc_size = 256; + adapter->params.arch.nchan = 2; + adapter->params.arch.vfcount = 256; + break; default: dev_err(adapter, "%s: Device %d is not supported\n", __func__, adapter->params.pci.device_id); @@ -3413,8 +4439,11 @@ int t4_prep_adapter(struct adapter *adapter) t4_os_find_pci_capability(adapter, PCI_CAP_ID_VPD); ret = t4_get_flash_params(adapter); - if (ret < 0) + if (ret < 0) { + dev_err(adapter, "Unable to retrieve Flash Parameters, ret = %d\n", + -ret); return ret; + } adapter->params.cim_la_size = CIMLA_SIZE; @@ -3586,6 +4615,14 @@ int t4_init_tp_params(struct adapter *adap) &adap->params.tp.ingress_config, 1, A_TP_INGRESS_CONFIG); + /* For T6, cache the adapter's compressed error vector + * and passing outer header info for encapsulated packets. + */ + if (CHELSIO_CHIP_VERSION(adap->params.chip) > CHELSIO_T5) { + v = t4_read_reg(adap, A_TP_OUT_CONFIG); + adap->params.tp.rx_pkt_encap = (v & F_CRXPKTENC) ? 1 : 0; + } + /* * Now that we have TP_VLAN_PRI_MAP cached, we can calculate the field * shift positions of several elements of the Compressed Filter Tuple @@ -3724,7 +4761,8 @@ int t4_port_init(struct adapter *adap, int mbox, int pf, int vf) p->port_type = G_FW_PORT_CMD_PTYPE(ret); p->mod_type = FW_PORT_MOD_TYPE_NA; - init_link_config(&p->link_cfg, be16_to_cpu(c.u.info.pcap)); + init_link_config(&p->link_cfg, be16_to_cpu(c.u.info.pcap), + be16_to_cpu(c.u.info.acap)); j++; } return 0; diff --git a/dpdk/drivers/net/cxgbe/base/t4_hw.h b/dpdk/drivers/net/cxgbe/base/t4_hw.h index 5e62c417..07498841 100644 --- a/dpdk/drivers/net/cxgbe/base/t4_hw.h +++ b/dpdk/drivers/net/cxgbe/base/t4_hw.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2014-2016 Chelsio Communications. + * Copyright(c) 2014-2017 Chelsio Communications. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -123,6 +123,14 @@ struct rsp_ctrl { #define FLASH_MAX_SIZE(nsecs) ((nsecs) * SF_SEC_SIZE) enum { + /* + * Various Expansion-ROM boot images, etc. + */ + FLASH_EXP_ROM_START_SEC = 0, + FLASH_EXP_ROM_NSECS = 6, + FLASH_EXP_ROM_START = FLASH_START(FLASH_EXP_ROM_START_SEC), + FLASH_EXP_ROM_MAX_SIZE = FLASH_MAX_SIZE(FLASH_EXP_ROM_NSECS), + /* * Location of firmware image in FLASH. */ @@ -131,6 +139,14 @@ enum { FLASH_FW_START = FLASH_START(FLASH_FW_START_SEC), FLASH_FW_MAX_SIZE = FLASH_MAX_SIZE(FLASH_FW_NSECS), + /* + * Location of bootstrap firmware image in FLASH. + */ + FLASH_FWBOOTSTRAP_START_SEC = 27, + FLASH_FWBOOTSTRAP_NSECS = 1, + FLASH_FWBOOTSTRAP_START = FLASH_START(FLASH_FWBOOTSTRAP_START_SEC), + FLASH_FWBOOTSTRAP_MAX_SIZE = FLASH_MAX_SIZE(FLASH_FWBOOTSTRAP_NSECS), + /* * Location of Firmware Configuration File in FLASH. */ diff --git a/dpdk/drivers/net/cxgbe/base/t4_msg.h b/dpdk/drivers/net/cxgbe/base/t4_msg.h index 4b04cd0d..6acd749a 100644 --- a/dpdk/drivers/net/cxgbe/base/t4_msg.h +++ b/dpdk/drivers/net/cxgbe/base/t4_msg.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2014-2015 Chelsio Communications. + * Copyright(c) 2014-2017 Chelsio Communications. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -262,6 +262,20 @@ struct cpl_rx_pkt { #define V_RXF_IP6(x) ((x) << S_RXF_IP6) #define F_RXF_IP6 V_RXF_IP6(1U) +/* rx_pkt.err_vec fields */ +/* In T6, rx_pkt.err_vec indicates + * RxError Error vector (16b) or + * Encapsulating header length (8b), + * Outer encapsulation type (2b) and + * compressed error vector (6b) if CRxPktEnc is + * enabled in TP_OUT_CONFIG + */ +#define S_T6_COMPR_RXERR_VEC 0 +#define M_T6_COMPR_RXERR_VEC 0x3F +#define V_T6_COMPR_RXERR_VEC(x) ((x) << S_T6_COMPR_RXERR_VEC) +#define G_T6_COMPR_RXERR_VEC(x) \ + (((x) >> S_T6_COMPR_RXERR_VEC) & M_T6_COMPR_RXERR_VEC) + /* cpl_fw*.type values */ enum { FW_TYPE_RSSCPL = 4, diff --git a/dpdk/drivers/net/cxgbe/base/t4_pci_id_tbl.h b/dpdk/drivers/net/cxgbe/base/t4_pci_id_tbl.h index 110fadb0..1230e738 100644 --- a/dpdk/drivers/net/cxgbe/base/t4_pci_id_tbl.h +++ b/dpdk/drivers/net/cxgbe/base/t4_pci_id_tbl.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2014-2015 Chelsio Communications. + * Copyright(c) 2014-2017 Chelsio Communications. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -144,6 +144,19 @@ CH_PCI_DEVICE_ID_TABLE_DEFINE_BEGIN CH_PCI_ID_TABLE_FENTRY(0x5090), /* Custom T540-CR */ CH_PCI_ID_TABLE_FENTRY(0x5091), /* Custom T522-CR */ CH_PCI_ID_TABLE_FENTRY(0x5092), /* Custom T520-CR */ + + /* T6 adapter */ + CH_PCI_ID_TABLE_FENTRY(0x6001), /* T6225-CR */ + CH_PCI_ID_TABLE_FENTRY(0x6002), /* T6225-SO-CR */ + CH_PCI_ID_TABLE_FENTRY(0x6003), /* T6425-CR */ + CH_PCI_ID_TABLE_FENTRY(0x6005), /* T6225-OCP */ + CH_PCI_ID_TABLE_FENTRY(0x6007), /* T62100-LP-CR */ + CH_PCI_ID_TABLE_FENTRY(0x6008), /* T62100-SO-CR */ + CH_PCI_ID_TABLE_FENTRY(0x600d), /* T62100-CR */ + CH_PCI_ID_TABLE_FENTRY(0x6011), /* T6225-LL-CR */ + CH_PCI_ID_TABLE_FENTRY(0x6014), /* T61100-OCP-SO */ + CH_PCI_ID_TABLE_FENTRY(0x6080), /* Custom T6225-CR SFP28 */ + CH_PCI_ID_TABLE_FENTRY(0x6081), /* Custom T62100-CR */ CH_PCI_DEVICE_ID_TABLE_DEFINE_END; #endif /* CH_PCI_DEVICE_ID_TABLE_DEFINE_BEGIN */ diff --git a/dpdk/drivers/net/cxgbe/base/t4_regs.h b/dpdk/drivers/net/cxgbe/base/t4_regs.h index 9057e409..1100e16f 100644 --- a/dpdk/drivers/net/cxgbe/base/t4_regs.h +++ b/dpdk/drivers/net/cxgbe/base/t4_regs.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2014-2015 Chelsio Communications. + * Copyright(c) 2014-2017 Chelsio Communications. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -420,6 +420,26 @@ #define A_PCIE_FW 0x30b8 #define A_PCIE_FW_PF 0x30bc +#define A_PCIE_CFG2 0x3018 + +#define S_TOTMAXTAG 0 +#define M_TOTMAXTAG 0x3U +#define V_TOTMAXTAG(x) ((x) << S_TOTMAXTAG) + +#define S_T6_TOTMAXTAG 0 +#define M_T6_TOTMAXTAG 0x7U +#define V_T6_TOTMAXTAG(x) ((x) << S_T6_TOTMAXTAG) + +#define A_PCIE_CMD_CFG 0x5980 + +#define S_MINTAG 0 +#define M_MINTAG 0xffU +#define V_MINTAG(x) ((x) << S_MINTAG) + +#define S_T6_MINTAG 0 +#define M_T6_MINTAG 0xffU +#define V_T6_MINTAG(x) ((x) << S_T6_MINTAG) + /* registers for module CIM */ #define CIM_BASE_ADDR 0x7b00 @@ -443,6 +463,12 @@ #define F_UPCRST V_UPCRST(1U) /* registers for module TP */ +#define A_TP_OUT_CONFIG 0x7d04 + +#define S_CRXPKTENC 3 +#define V_CRXPKTENC(x) ((x) << S_CRXPKTENC) +#define F_CRXPKTENC V_CRXPKTENC(1U) + #define TP_BASE_ADDR 0x7d00 #define A_TP_TIMER_RESOLUTION 0x7d90 @@ -553,6 +579,24 @@ #define V_VF(x) ((x) << S_VF) #define G_VF(x) (((x) >> S_VF) & M_VF) +#define A_MPS_STAT_CTL 0x9600 + +#define S_COUNTPAUSEMCRX 5 +#define V_COUNTPAUSEMCRX(x) ((x) << S_COUNTPAUSEMCRX) +#define F_COUNTPAUSEMCRX V_COUNTPAUSEMCRX(1U) + +#define S_COUNTPAUSESTATRX 4 +#define V_COUNTPAUSESTATRX(x) ((x) << S_COUNTPAUSESTATRX) +#define F_COUNTPAUSESTATRX V_COUNTPAUSESTATRX(1U) + +#define S_COUNTPAUSEMCTX 3 +#define V_COUNTPAUSEMCTX(x) ((x) << S_COUNTPAUSEMCTX) +#define F_COUNTPAUSEMCTX V_COUNTPAUSEMCTX(1U) + +#define S_COUNTPAUSESTATTX 2 +#define V_COUNTPAUSESTATTX(x) ((x) << S_COUNTPAUSESTATTX) +#define F_COUNTPAUSESTATTX V_COUNTPAUSESTATTX(1U) + #define A_MPS_PORT_STAT_TX_PORT_BYTES_L 0x400 #define A_MPS_PORT_STAT_TX_PORT_BYTES_H 0x404 #define A_MPS_PORT_STAT_TX_PORT_FRAMES_L 0x408 @@ -763,6 +807,11 @@ #define V_SOURCEPF(x) ((x) << S_SOURCEPF) #define G_SOURCEPF(x) (((x) >> S_SOURCEPF) & M_SOURCEPF) +#define S_T6_SOURCEPF 9 +#define M_T6_SOURCEPF 0x7U +#define V_T6_SOURCEPF(x) ((x) << S_T6_SOURCEPF) +#define G_T6_SOURCEPF(x) (((x) >> S_T6_SOURCEPF) & M_T6_SOURCEPF) + #define A_PL_PF_INT_ENABLE 0x3c4 #define S_PFSW 3 diff --git a/dpdk/drivers/net/cxgbe/base/t4_regs_values.h b/dpdk/drivers/net/cxgbe/base/t4_regs_values.h index d7d3144c..9085ff6d 100644 --- a/dpdk/drivers/net/cxgbe/base/t4_regs_values.h +++ b/dpdk/drivers/net/cxgbe/base/t4_regs_values.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2014-2015 Chelsio Communications. + * Copyright(c) 2014-2017 Chelsio Communications. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -55,10 +55,15 @@ #define X_RXPKTCPLMODE_SPLIT 1 #define X_INGPCIEBOUNDARY_32B 0 #define X_INGPADBOUNDARY_SHIFT 5 +#define X_INGPADBOUNDARY_32B 0 + +#define X_T6_INGPADBOUNDARY_SHIFT 3 +#define X_T6_INGPADBOUNDARY_8B 0 /* CONTROL2 register */ #define X_INGPACKBOUNDARY_SHIFT 5 #define X_INGPACKBOUNDARY_16B 0 +#define X_INGPACKBOUNDARY_64B 1 /* GTS register */ #define X_TIMERREG_RESTART_COUNTER 6 @@ -77,7 +82,7 @@ /* * Ingress Context field values */ -#define X_UPDATEDELIVERY_INTERRUPT 1 +#define X_UPDATEDELIVERY_STATUS_PAGE 2 #define X_RSPD_TYPE_FLBUF 0 #define X_RSPD_TYPE_CPL 1 diff --git a/dpdk/drivers/net/cxgbe/base/t4fw_interface.h b/dpdk/drivers/net/cxgbe/base/t4fw_interface.h index 74f19fe7..6ca4f318 100644 --- a/dpdk/drivers/net/cxgbe/base/t4fw_interface.h +++ b/dpdk/drivers/net/cxgbe/base/t4fw_interface.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2014-2015 Chelsio Communications. + * Copyright(c) 2014-2017 Chelsio Communications. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -84,6 +84,7 @@ enum fw_memtype { enum fw_wr_opcodes { FW_ETH_TX_PKT_WR = 0x08, FW_ETH_TX_PKTS_WR = 0x09, + FW_ETH_TX_PKTS2_WR = 0x78, }; /* @@ -591,6 +592,13 @@ struct fw_iq_cmd { #define G_FW_IQ_CMD_IQESIZE(x) \ (((x) >> S_FW_IQ_CMD_IQESIZE) & M_FW_IQ_CMD_IQESIZE) +#define S_FW_IQ_CMD_IQRO 30 +#define M_FW_IQ_CMD_IQRO 0x1 +#define V_FW_IQ_CMD_IQRO(x) ((x) << S_FW_IQ_CMD_IQRO) +#define G_FW_IQ_CMD_IQRO(x) \ + (((x) >> S_FW_IQ_CMD_IQRO) & M_FW_IQ_CMD_IQRO) +#define F_FW_IQ_CMD_IQRO V_FW_IQ_CMD_IQRO(1U) + #define S_FW_IQ_CMD_IQFLINTCONGEN 27 #define M_FW_IQ_CMD_IQFLINTCONGEN 0x1 #define V_FW_IQ_CMD_IQFLINTCONGEN(x) ((x) << S_FW_IQ_CMD_IQFLINTCONGEN) @@ -1061,7 +1069,7 @@ struct fw_vi_stats_cmd { enum fw_port_cap { FW_PORT_CAP_SPEED_100M = 0x0001, FW_PORT_CAP_SPEED_1G = 0x0002, - FW_PORT_CAP_SPEED_2_5G = 0x0004, + FW_PORT_CAP_SPEED_25G = 0x0004, FW_PORT_CAP_SPEED_10G = 0x0008, FW_PORT_CAP_SPEED_40G = 0x0010, FW_PORT_CAP_SPEED_100G = 0x0020, @@ -1070,13 +1078,19 @@ enum fw_port_cap { FW_PORT_CAP_ANEG = 0x0100, FW_PORT_CAP_MDIX = 0x0200, FW_PORT_CAP_MDIAUTO = 0x0400, - FW_PORT_CAP_FEC = 0x0800, - FW_PORT_CAP_TECHKR = 0x1000, - FW_PORT_CAP_TECHKX4 = 0x2000, + FW_PORT_CAP_FEC_RS = 0x0800, + FW_PORT_CAP_FEC_BASER_RS = 0x1000, + FW_PORT_CAP_FEC_RESERVED = 0x2000, FW_PORT_CAP_802_3_PAUSE = 0x4000, FW_PORT_CAP_802_3_ASM_DIR = 0x8000, }; +#define S_FW_PORT_CAP_SPEED 0 +#define M_FW_PORT_CAP_SPEED 0x3f +#define V_FW_PORT_CAP_SPEED(x) ((x) << S_FW_PORT_CAP_SPEED) +#define G_FW_PORT_CAP_SPEED(x) \ + (((x) >> S_FW_PORT_CAP_SPEED) & M_FW_PORT_CAP_SPEED) + enum fw_port_mdi { FW_PORT_CAP_MDI_AUTO, }; @@ -1279,7 +1293,12 @@ enum fw_port_type { FW_PORT_TYPE_QSFP = 14, /* No, 4, Yes, No, No, No, 40G */ FW_PORT_TYPE_BP40_BA = 15, /* No, 4, No, No, Yes, Yes, 40G/10G/1G, BP ANGE */ - + FW_PORT_TYPE_KR4_100G = 16, /* No, 4, 100G/40G/25G, Backplane */ + FW_PORT_TYPE_CR4_QSFP = 17, /* No, 4, 100G/40G/25G */ + FW_PORT_TYPE_CR_QSFP = 18, /* No, 1, 25G Spider cable */ + FW_PORT_TYPE_CR2_QSFP = 19, /* No, 2, 50G */ + FW_PORT_TYPE_SFP28 = 20, /* No, 1, 25G/10G/1G */ + FW_PORT_TYPE_KR_SFP28 = 21, /* No, 1, 25G/10G/1G using Backplane */ FW_PORT_TYPE_NONE = M_FW_PORT_CMD_PTYPE }; diff --git a/dpdk/drivers/net/cxgbe/cxgbe.h b/dpdk/drivers/net/cxgbe/cxgbe.h index 0201c990..f9891545 100644 --- a/dpdk/drivers/net/cxgbe/cxgbe.h +++ b/dpdk/drivers/net/cxgbe/cxgbe.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2014-2015 Chelsio Communications. + * Copyright(c) 2014-2017 Chelsio Communications. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -47,6 +47,7 @@ #define CXGBE_MAX_RX_PKTLEN (9000 + ETHER_HDR_LEN + ETHER_CRC_LEN) /* max pkt */ int cxgbe_probe(struct adapter *adapter); +void cxgbe_get_speed_caps(struct port_info *pi, u32 *speed_caps); int cxgbe_up(struct adapter *adap); int cxgbe_down(struct port_info *pi); void cxgbe_close(struct adapter *adapter); @@ -59,5 +60,6 @@ int setup_sge_fwevtq(struct adapter *adapter); void cfg_queues(struct rte_eth_dev *eth_dev); int cfg_queue_count(struct rte_eth_dev *eth_dev); int setup_rss(struct port_info *pi); +void cxgbe_enable_rx_queues(struct port_info *pi); #endif /* _CXGBE_H_ */ diff --git a/dpdk/drivers/net/cxgbe/cxgbe_compat.h b/dpdk/drivers/net/cxgbe/cxgbe_compat.h index e68f8f59..03bba9fe 100644 --- a/dpdk/drivers/net/cxgbe/cxgbe_compat.h +++ b/dpdk/drivers/net/cxgbe/cxgbe_compat.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2014-2015 Chelsio Communications. + * Copyright(c) 2014-2017 Chelsio Communications. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -45,6 +45,7 @@ #include #include #include +#include #define dev_printf(level, fmt, args...) \ RTE_LOG(level, PMD, "rte_cxgbe_pmd: " fmt, ## args) @@ -225,6 +226,15 @@ static inline int cxgbe_fls(int x) return x ? sizeof(x) * 8 - __builtin_clz(x) : 0; } +/** + * cxgbe_ffs - find first bit set + * @x: the word to search + */ +static inline int cxgbe_ffs(int x) +{ + return x ? __builtin_ffs(x) : 0; +} + static inline unsigned long ilog2(unsigned long n) { unsigned int e = 0; @@ -254,7 +264,7 @@ static inline unsigned long ilog2(unsigned long n) static inline void writel(unsigned int val, volatile void __iomem *addr) { - *(volatile unsigned int *)addr = val; + rte_write32(val, addr); } static inline void writeq(u64 val, volatile void __iomem *addr) @@ -263,4 +273,9 @@ static inline void writeq(u64 val, volatile void __iomem *addr) writel(val >> 32, (void *)((uintptr_t)addr + 4)); } +static inline void writel_relaxed(unsigned int val, volatile void __iomem *addr) +{ + rte_write32_relaxed(val, addr); +} + #endif /* _CXGBE_COMPAT_H_ */ diff --git a/dpdk/drivers/net/cxgbe/cxgbe_ethdev.c b/dpdk/drivers/net/cxgbe/cxgbe_ethdev.c index 9208a615..dc153c73 100644 --- a/dpdk/drivers/net/cxgbe/cxgbe_ethdev.c +++ b/dpdk/drivers/net/cxgbe/cxgbe_ethdev.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2014-2016 Chelsio Communications. + * Copyright(c) 2014-2017 Chelsio Communications. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -48,16 +48,16 @@ #include #include #include +#include #include #include #include -#include #include #include #include #include #include -#include +#include #include #include #include @@ -68,7 +68,7 @@ * Macros needed to support the PCI Device ID Table ... */ #define CH_PCI_DEVICE_ID_TABLE_DEFINE_BEGIN \ - static struct rte_pci_id cxgb4_pci_tbl[] = { + static const struct rte_pci_id cxgb4_pci_tbl[] = { #define CH_PCI_DEVICE_ID_FUNCTION 0x4 #define PCI_VENDOR_ID_CHELSIO 0x1425 @@ -103,7 +103,8 @@ static uint16_t cxgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, pkts_remain = nb_pkts - total_sent; for (pkts_sent = 0; pkts_sent < pkts_remain; pkts_sent++) { - ret = t4_eth_xmit(txq, tx_pkts[total_sent + pkts_sent]); + ret = t4_eth_xmit(txq, tx_pkts[total_sent + pkts_sent], + nb_pkts); if (ret < 0) break; } @@ -147,6 +148,8 @@ static void cxgbe_dev_info_get(struct rte_eth_dev *eth_dev, .nb_align = 1, }; + device_info->pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); + device_info->min_rx_bufsize = CXGBE_MIN_RX_BUFSIZE; device_info->max_rx_pktlen = CXGBE_MAX_RX_PKTLEN; device_info->max_rx_queues = max_queues; @@ -171,7 +174,7 @@ static void cxgbe_dev_info_get(struct rte_eth_dev *eth_dev, device_info->rx_desc_lim = cxgbe_desc_lim; device_info->tx_desc_lim = cxgbe_desc_lim; - device_info->speed_capa = ETH_LINK_SPEED_10G | ETH_LINK_SPEED_40G; + cxgbe_get_speed_caps(pi, &device_info->speed_capa); } static void cxgbe_dev_promiscuous_enable(struct rte_eth_dev *eth_dev) @@ -335,6 +338,8 @@ static int cxgbe_dev_start(struct rte_eth_dev *eth_dev) goto out; } + cxgbe_enable_rx_queues(pi); + err = setup_rss(pi); if (err) goto out; @@ -614,7 +619,7 @@ static int cxgbe_dev_rx_queue_setup(struct rte_eth_dev *eth_dev, err = t4_sge_alloc_rxq(adapter, &rxq->rspq, false, eth_dev, msi_idx, &rxq->fl, t4_ethrx_handler, - t4_get_mps_bg_map(adapter, pi->tx_chan), mp, + t4_get_tp_ch_map(adapter, pi->tx_chan), mp, queue_idx, socket_id); dev_debug(adapter, "%s: err = %d; port_id = %d; cntxt_id = %u\n", @@ -642,7 +647,7 @@ static void cxgbe_dev_rx_queue_release(void *q) /* * Get port statistics. */ -static void cxgbe_dev_stats_get(struct rte_eth_dev *eth_dev, +static int cxgbe_dev_stats_get(struct rte_eth_dev *eth_dev, struct rte_eth_stats *eth_stats) { struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private); @@ -654,8 +659,6 @@ static void cxgbe_dev_stats_get(struct rte_eth_dev *eth_dev, cxgbe_stats_get(pi, &ps); /* RX Stats */ - eth_stats->ipackets = ps.rx_frames; - eth_stats->ibytes = ps.rx_octets; eth_stats->imissed = ps.rx_ovflow0 + ps.rx_ovflow1 + ps.rx_ovflow2 + ps.rx_ovflow3 + ps.rx_trunc0 + ps.rx_trunc1 + @@ -675,6 +678,8 @@ static void cxgbe_dev_stats_get(struct rte_eth_dev *eth_dev, eth_stats->q_ipackets[i] = rxq->stats.pkts; eth_stats->q_ibytes[i] = rxq->stats.rx_bytes; + eth_stats->ipackets += eth_stats->q_ipackets[i]; + eth_stats->ibytes += eth_stats->q_ibytes[i]; } for (i = 0; i < pi->n_tx_qsets; i++) { @@ -685,6 +690,7 @@ static void cxgbe_dev_stats_get(struct rte_eth_dev *eth_dev, eth_stats->q_obytes[i] = txq->stats.tx_bytes; eth_stats->q_errors[i] = txq->stats.mapping_err; } + return 0; } /* @@ -1005,7 +1011,7 @@ static int eth_cxgbe_dev_init(struct rte_eth_dev *eth_dev) if (rte_eal_process_type() != RTE_PROC_PRIMARY) return 0; - pci_dev = eth_dev->pci_dev; + pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); snprintf(name, sizeof(name), "cxgbeadapter%d", eth_dev->data->port_id); adapter = rte_zmalloc(name, sizeof(*adapter), 0); @@ -1037,35 +1043,25 @@ out_free_adapter: return err; } -static struct eth_driver rte_cxgbe_pmd = { - .pci_drv = { - .name = "rte_cxgbe_pmd", - .id_table = cxgb4_pci_tbl, - .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC, - }, - .eth_dev_init = eth_cxgbe_dev_init, - .dev_private_size = sizeof(struct port_info), -}; - -/* - * Driver initialization routine. - * Invoked once at EAL init time. - * Register itself as the [Poll Mode] Driver of PCI CXGBE devices. - */ -static int rte_cxgbe_pmd_init(const char *name __rte_unused, - const char *params __rte_unused) +static int eth_cxgbe_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, + struct rte_pci_device *pci_dev) { - CXGBE_FUNC_TRACE(); - - rte_eth_driver_register(&rte_cxgbe_pmd); - return 0; + return rte_eth_dev_pci_generic_probe(pci_dev, + sizeof(struct port_info), eth_cxgbe_dev_init); } -static struct rte_driver rte_cxgbe_driver = { - .type = PMD_PDEV, - .init = rte_cxgbe_pmd_init, +static int eth_cxgbe_pci_remove(struct rte_pci_device *pci_dev) +{ + return rte_eth_dev_pci_generic_remove(pci_dev, NULL); +} + +static struct rte_pci_driver rte_cxgbe_pmd = { + .id_table = cxgb4_pci_tbl, + .drv_flags = RTE_PCI_DRV_NEED_MAPPING, + .probe = eth_cxgbe_pci_probe, + .remove = eth_cxgbe_pci_remove, }; -PMD_REGISTER_DRIVER(rte_cxgbe_driver, cxgb4); -DRIVER_REGISTER_PCI_TABLE(cxgb4, cxgb4_pci_tbl); - +RTE_PMD_REGISTER_PCI(net_cxgbe, rte_cxgbe_pmd); +RTE_PMD_REGISTER_PCI_TABLE(net_cxgbe, cxgb4_pci_tbl); +RTE_PMD_REGISTER_KMOD_DEP(net_cxgbe, "* igb_uio | uio_pci_generic | vfio-pci"); diff --git a/dpdk/drivers/net/cxgbe/cxgbe_main.c b/dpdk/drivers/net/cxgbe/cxgbe_main.c index ceaf5ab2..5b828c23 100644 --- a/dpdk/drivers/net/cxgbe/cxgbe_main.c +++ b/dpdk/drivers/net/cxgbe/cxgbe_main.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2014-2016 Chelsio Communications. + * Copyright(c) 2014-2017 Chelsio Communications. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -51,13 +51,12 @@ #include #include #include -#include #include #include #include #include #include -#include +#include #include #include #include @@ -205,9 +204,12 @@ static inline bool is_x_1g_port(const struct link_config *lc) static inline bool is_x_10g_port(const struct link_config *lc) { - return ((lc->supported & FW_PORT_CAP_SPEED_10G) != 0 || - (lc->supported & FW_PORT_CAP_SPEED_40G) != 0 || - (lc->supported & FW_PORT_CAP_SPEED_100G) != 0); + unsigned int speeds, high_speeds; + + speeds = V_FW_PORT_CAP_SPEED(G_FW_PORT_CAP_SPEED(lc->supported)); + high_speeds = speeds & ~(FW_PORT_CAP_SPEED_100M | FW_PORT_CAP_SPEED_1G); + + return high_speeds != 0; } inline void init_rspq(struct adapter *adap, struct sge_rspq *q, @@ -297,7 +299,7 @@ void cfg_queues(struct rte_eth_dev *eth_dev) for (i = 0; i < ARRAY_SIZE(s->ethrxq); i++) { struct sge_eth_rxq *r = &s->ethrxq[i]; - init_rspq(adap, &r->rspq, 0, 0, 1024, 64); + init_rspq(adap, &r->rspq, 5, 32, 1024, 64); r->usembufs = 1; r->fl.size = (r->usembufs ? 1024 : 72); } @@ -362,6 +364,17 @@ static int init_rss(struct adapter *adap) return 0; } +/** + * Dump basic information about the adapter. + */ +static void print_adapter_info(struct adapter *adap) +{ + /** + * Hardware/Firmware/etc. Version/Revision IDs. + */ + t4_dump_version_info(adap); +} + static void print_port_info(struct adapter *adap) { int i; @@ -373,13 +386,17 @@ static void print_port_info(struct adapter *adap) char *bufp = buf; if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_100M) - bufp += sprintf(bufp, "100/"); + bufp += sprintf(bufp, "100M/"); if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_1G) - bufp += sprintf(bufp, "1000/"); + bufp += sprintf(bufp, "1G/"); if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_10G) bufp += sprintf(bufp, "10G/"); + if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_25G) + bufp += sprintf(bufp, "25G/"); if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_40G) bufp += sprintf(bufp, "40G/"); + if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_100G) + bufp += sprintf(bufp, "100G/"); if (bufp != buf) --bufp; sprintf(bufp, "BASE-%s", @@ -395,6 +412,36 @@ static void print_port_info(struct adapter *adap) } } +static void configure_pcie_ext_tag(struct adapter *adapter) +{ + u16 v; + int pos = t4_os_find_pci_capability(adapter, PCI_CAP_ID_EXP); + + if (!pos) + return; + + if (pos > 0) { + t4_os_pci_read_cfg2(adapter, pos + PCI_EXP_DEVCTL, &v); + v |= PCI_EXP_DEVCTL_EXT_TAG; + t4_os_pci_write_cfg2(adapter, pos + PCI_EXP_DEVCTL, v); + if (is_t6(adapter->params.chip)) { + t4_set_reg_field(adapter, A_PCIE_CFG2, + V_T6_TOTMAXTAG(M_T6_TOTMAXTAG), + V_T6_TOTMAXTAG(7)); + t4_set_reg_field(adapter, A_PCIE_CMD_CFG, + V_T6_MINTAG(M_T6_MINTAG), + V_T6_MINTAG(8)); + } else { + t4_set_reg_field(adapter, A_PCIE_CFG2, + V_TOTMAXTAG(M_TOTMAXTAG), + V_TOTMAXTAG(3)); + t4_set_reg_field(adapter, A_PCIE_CMD_CFG, + V_MINTAG(M_MINTAG), + V_MINTAG(8)); + } + } +} + /* * Tweak configuration based on system architecture, etc. Most of these have * defaults assigned to them by Firmware Configuration Files (if we're using @@ -426,6 +473,9 @@ static int adap_init0_tweaks(struct adapter *adapter) V_CREDITCNT(M_CREDITCNT) | M_CREDITCNTPACKING, V_CREDITCNT(3) | V_CREDITCNTPACKING(1)); + t4_set_reg_field(adapter, A_SGE_INGRESS_RX_THRESHOLD, + V_THRESHOLD_3(M_THRESHOLD_3), V_THRESHOLD_3(32U)); + t4_set_reg_field(adapter, A_SGE_CONTROL2, V_IDMAARBROUNDROBIN(1U), V_IDMAARBROUNDROBIN(1U)); @@ -574,7 +624,7 @@ static int adap_init0_config(struct adapter *adapter, int reset) /* * Return successfully and note that we're operating with parameters * not supplied by the driver, rather than from hard-wired - * initialization constants burried in the driver. + * initialization constants buried in the driver. */ dev_info(adapter, "Successfully configured using Firmware Configuration File \"%s\", version %#x, computed checksum %#x\n", @@ -640,18 +690,7 @@ static int adap_init0(struct adapter *adap) state = (enum dev_state)((unsigned)state & ~DEV_STATE_INIT); } - t4_get_fw_version(adap, &adap->params.fw_vers); - t4_get_tp_version(adap, &adap->params.tp_vers); - - dev_info(adap, "fw: %u.%u.%u.%u, TP: %u.%u.%u.%u\n", - G_FW_HDR_FW_VER_MAJOR(adap->params.fw_vers), - G_FW_HDR_FW_VER_MINOR(adap->params.fw_vers), - G_FW_HDR_FW_VER_MICRO(adap->params.fw_vers), - G_FW_HDR_FW_VER_BUILD(adap->params.fw_vers), - G_FW_HDR_FW_VER_MAJOR(adap->params.tp_vers), - G_FW_HDR_FW_VER_MINOR(adap->params.tp_vers), - G_FW_HDR_FW_VER_MICRO(adap->params.tp_vers), - G_FW_HDR_FW_VER_BUILD(adap->params.tp_vers)); + t4_get_version_info(adap); ret = t4_get_core_clock(adap, &adap->params.vpd); if (ret < 0) { @@ -660,26 +699,6 @@ static int adap_init0(struct adapter *adap) goto bye; } - /* - * Find out what ports are available to us. Note that we need to do - * this before calling adap_init0_no_config() since it needs nports - * and portvec ... - */ - v = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | - V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_PORTVEC); - ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 1, &v, &port_vec); - if (ret < 0) { - dev_err(adap, "%s: failure in t4_queury_params; error = %d\n", - __func__, ret); - goto bye; - } - - adap->params.nports = hweight32(port_vec); - adap->params.portvec = port_vec; - - dev_debug(adap, "%s: adap->params.nports = %u\n", __func__, - adap->params.nports); - /* * If the firmware is initialized already (and we're not forcing a * master initialization), note that we're living with existing @@ -704,6 +723,22 @@ static int adap_init0(struct adapter *adap) goto bye; } + /* Find out what ports are available to us. */ + v = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | + V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_PORTVEC); + ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 1, &v, &port_vec); + if (ret < 0) { + dev_err(adap, "%s: failure in t4_query_params; error = %d\n", + __func__, ret); + goto bye; + } + + adap->params.nports = hweight32(port_vec); + adap->params.portvec = port_vec; + + dev_debug(adap, "%s: adap->params.nports = %u\n", __func__, + adap->params.nports); + /* * Give the SGE code a chance to pull in anything that it needs ... * Note that this must be called after we retrieve our VPD parameters @@ -792,6 +827,7 @@ static int adap_init0(struct adapter *adap) } t4_init_sge_params(adap); t4_init_tp_params(adap); + configure_pcie_ext_tag(adap); adap->params.drv_memwin = MEMWIN_NIC; adap->flags |= FW_OK; @@ -832,10 +868,10 @@ void t4_os_portmod_changed(const struct adapter *adap, int port_id) dev_info(adap, "Port%d: %s port module inserted\n", pi->port_id, mod_str[pi->mod_type]); else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) - dev_info(adap, "Port%d: unsupported optical port module inserted\n", + dev_info(adap, "Port%d: unsupported port module inserted\n", pi->port_id); else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) - dev_info(adap, "Port%d: unknown port module inserted, forcing TWINAX\n", + dev_info(adap, "Port%d: unknown port module inserted\n", pi->port_id); else if (pi->mod_type == FW_PORT_MOD_TYPE_ERROR) dev_info(adap, "Port%d: transceiver module error\n", @@ -959,7 +995,7 @@ int setup_rss(struct port_info *pi) dev_debug(adapter, "%s: pi->rss_size = %u; pi->n_rx_qsets = %u\n", __func__, pi->rss_size, pi->n_rx_qsets); - if (!pi->flags & PORT_RSS_DONE) { + if (!(pi->flags & PORT_RSS_DONE)) { if (adapter->flags & FULL_INIT_DONE) { /* Fill default values with equal distribution */ for (j = 0; j < pi->rss_size; j++) @@ -977,33 +1013,126 @@ int setup_rss(struct port_info *pi) /* * Enable NAPI scheduling and interrupt generation for all Rx queues. */ -static void enable_rx(struct adapter *adap) +static void enable_rx(struct adapter *adap, struct sge_rspq *q) { - struct sge *s = &adap->sge; - struct sge_rspq *q = &s->fw_evtq; - int i, j; - /* 0-increment GTS to start the timer and enable interrupts */ t4_write_reg(adap, MYPF_REG(A_SGE_PF_GTS), V_SEINTARM(q->intr_params) | V_INGRESSQID(q->cntxt_id)); +} - for_each_port(adap, i) { - const struct port_info *pi = &adap->port[i]; - struct rte_eth_dev *eth_dev = pi->eth_dev; +void cxgbe_enable_rx_queues(struct port_info *pi) +{ + struct adapter *adap = pi->adapter; + struct sge *s = &adap->sge; + unsigned int i; - for (j = 0; j < eth_dev->data->nb_rx_queues; j++) { - q = eth_dev->data->rx_queues[j]; + for (i = 0; i < pi->n_rx_qsets; i++) + enable_rx(adap, &s->ethrxq[pi->first_qset + i].rspq); +} - /* - * 0-increment GTS to start the timer and enable - * interrupts - */ - t4_write_reg(adap, MYPF_REG(A_SGE_PF_GTS), - V_SEINTARM(q->intr_params) | - V_INGRESSQID(q->cntxt_id)); - } +/** + * fw_caps_to_speed_caps - translate Firmware Port Caps to Speed Caps. + * @port_type: Firmware Port Type + * @fw_caps: Firmware Port Capabilities + * @speed_caps: Device Info Speed Capabilities + * + * Translate a Firmware Port Capabilities specification to Device Info + * Speed Capabilities. + */ +static void fw_caps_to_speed_caps(enum fw_port_type port_type, + unsigned int fw_caps, + u32 *speed_caps) +{ +#define SET_SPEED(__speed_name) \ + do { \ + *speed_caps |= ETH_LINK_ ## __speed_name; \ + } while (0) + +#define FW_CAPS_TO_SPEED(__fw_name) \ + do { \ + if (fw_caps & FW_PORT_CAP_ ## __fw_name) \ + SET_SPEED(__fw_name); \ + } while (0) + + switch (port_type) { + case FW_PORT_TYPE_BT_SGMII: + case FW_PORT_TYPE_BT_XFI: + case FW_PORT_TYPE_BT_XAUI: + FW_CAPS_TO_SPEED(SPEED_100M); + FW_CAPS_TO_SPEED(SPEED_1G); + FW_CAPS_TO_SPEED(SPEED_10G); + break; + + case FW_PORT_TYPE_KX4: + case FW_PORT_TYPE_KX: + case FW_PORT_TYPE_FIBER_XFI: + case FW_PORT_TYPE_FIBER_XAUI: + case FW_PORT_TYPE_SFP: + case FW_PORT_TYPE_QSFP_10G: + case FW_PORT_TYPE_QSA: + FW_CAPS_TO_SPEED(SPEED_1G); + FW_CAPS_TO_SPEED(SPEED_10G); + break; + + case FW_PORT_TYPE_KR: + SET_SPEED(SPEED_10G); + break; + + case FW_PORT_TYPE_BP_AP: + case FW_PORT_TYPE_BP4_AP: + SET_SPEED(SPEED_1G); + SET_SPEED(SPEED_10G); + break; + + case FW_PORT_TYPE_BP40_BA: + case FW_PORT_TYPE_QSFP: + SET_SPEED(SPEED_40G); + break; + + case FW_PORT_TYPE_CR_QSFP: + case FW_PORT_TYPE_SFP28: + case FW_PORT_TYPE_KR_SFP28: + FW_CAPS_TO_SPEED(SPEED_1G); + FW_CAPS_TO_SPEED(SPEED_10G); + FW_CAPS_TO_SPEED(SPEED_25G); + break; + + case FW_PORT_TYPE_CR2_QSFP: + SET_SPEED(SPEED_50G); + break; + + case FW_PORT_TYPE_KR4_100G: + case FW_PORT_TYPE_CR4_QSFP: + FW_CAPS_TO_SPEED(SPEED_25G); + FW_CAPS_TO_SPEED(SPEED_40G); + FW_CAPS_TO_SPEED(SPEED_100G); + break; + + default: + break; } + +#undef FW_CAPS_TO_SPEED +#undef SET_SPEED +} + +/** + * cxgbe_get_speed_caps - Fetch supported speed capabilities + * @pi: Underlying port's info + * @speed_caps: Device Info speed capabilities + * + * Fetch supported speed capabilities of the underlying port. + */ +void cxgbe_get_speed_caps(struct port_info *pi, u32 *speed_caps) +{ + *speed_caps = 0; + + fw_caps_to_speed_caps(pi->port_type, pi->link_cfg.supported, + speed_caps); + + if (!(pi->link_cfg.supported & FW_PORT_CAP_ANEG)) + *speed_caps |= ETH_LINK_SPEED_FIXED; } /** @@ -1016,7 +1145,7 @@ static void enable_rx(struct adapter *adap) */ int cxgbe_up(struct adapter *adap) { - enable_rx(adap); + enable_rx(adap, &adap->sge.fw_evtq); t4_sge_tx_monitor_start(adap); t4_intr_enable(adap); adap->flags |= FULL_INIT_DONE; @@ -1072,10 +1201,20 @@ void cxgbe_close(struct adapter *adapter) int cxgbe_probe(struct adapter *adapter) { struct port_info *pi; + int chip; int func, i; int err = 0; + u32 whoami; + + whoami = t4_read_reg(adapter, A_PL_WHOAMI); + chip = t4_get_chip_type(adapter, + CHELSIO_PCI_ID_VER(adapter->pdev->id.device_id)); + if (chip < 0) + return chip; + + func = CHELSIO_CHIP_VERSION(chip) <= CHELSIO_T5 ? + G_SOURCEPF(whoami) : G_T6_SOURCEPF(whoami); - func = G_SOURCEPF(t4_read_reg(adapter, A_PL_WHOAMI)); adapter->mbox = func; adapter->pf = func; @@ -1150,7 +1289,7 @@ int cxgbe_probe(struct adapter *adapter) */ /* reserve an ethdev entry */ - pi->eth_dev = rte_eth_dev_allocate(name, RTE_ETH_DEV_PCI); + pi->eth_dev = rte_eth_dev_allocate(name); if (!pi->eth_dev) goto out_free; @@ -1163,16 +1302,13 @@ int cxgbe_probe(struct adapter *adapter) pi->eth_dev->data = data; allocate_mac: - pi->eth_dev->pci_dev = adapter->pdev; + pi->eth_dev->device = &adapter->pdev->device; pi->eth_dev->data->dev_private = pi; - pi->eth_dev->driver = adapter->eth_dev->driver; pi->eth_dev->dev_ops = adapter->eth_dev->dev_ops; pi->eth_dev->tx_pkt_burst = adapter->eth_dev->tx_pkt_burst; pi->eth_dev->rx_pkt_burst = adapter->eth_dev->rx_pkt_burst; - rte_eth_copy_pci_info(pi->eth_dev, pi->eth_dev->pci_dev); - - TAILQ_INIT(&pi->eth_dev->link_intr_cbs); + rte_eth_copy_pci_info(pi->eth_dev, adapter->pdev); pi->eth_dev->data->mac_addrs = rte_zmalloc(name, ETHER_ADDR_LEN, 0); @@ -1195,6 +1331,7 @@ allocate_mac: cfg_queues(adapter->eth_dev); + print_adapter_info(adapter); print_port_info(adapter); err = init_rss(adapter); diff --git a/dpdk/drivers/net/cxgbe/sge.c b/dpdk/drivers/net/cxgbe/sge.c index ab5a842a..fc10d958 100644 --- a/dpdk/drivers/net/cxgbe/sge.c +++ b/dpdk/drivers/net/cxgbe/sge.c @@ -57,7 +57,6 @@ #include #include #include -#include #include #include #include @@ -150,7 +149,7 @@ static int map_mbuf(struct rte_mbuf *mbuf, dma_addr_t *addr) struct rte_mbuf *m = mbuf; for (; m; m = m->next, addr++) { - *addr = m->buf_physaddr + rte_pktmbuf_headroom(m); + *addr = m->buf_iova + rte_pktmbuf_headroom(m); if (*addr == 0) goto out_err; } @@ -338,12 +337,12 @@ static inline void ring_fl_db(struct adapter *adap, struct sge_fl *q) * mechanism. */ if (unlikely(!q->bar2_addr)) { - t4_write_reg(adap, MYPF_REG(A_SGE_PF_KDOORBELL), - val | V_QID(q->cntxt_id)); + t4_write_reg_relaxed(adap, MYPF_REG(A_SGE_PF_KDOORBELL), + val | V_QID(q->cntxt_id)); } else { - writel(val | V_QID(q->bar2_qid), - (void *)((uintptr_t)q->bar2_addr + - SGE_UDB_KDOORBELL)); + writel_relaxed(val | V_QID(q->bar2_qid), + (void *)((uintptr_t)q->bar2_addr + + SGE_UDB_KDOORBELL)); /* * This Write memory Barrier will force the write to @@ -388,7 +387,7 @@ static unsigned int refill_fl_usembufs(struct adapter *adap, struct sge_fl *q, struct rte_pktmbuf_pool_private *mbp_priv; u8 jumbo_en = rxq->rspq.eth_dev->data->dev_conf.rxmode.jumbo_frame; - /* Use jumbo mtu buffers iff mbuf data room size can fit jumbo data. */ + /* Use jumbo mtu buffers if mbuf data room size can fit jumbo data. */ mbp_priv = rte_mempool_get_priv(rxq->rspq.mb_pool); if (jumbo_en && ((mbp_priv->mbuf_data_room_size - RTE_PKTMBUF_HEADROOM) >= 9000)) @@ -415,12 +414,18 @@ static unsigned int refill_fl_usembufs(struct adapter *adap, struct sge_fl *q, } rte_mbuf_refcnt_set(mbuf, 1); - mbuf->data_off = RTE_PKTMBUF_HEADROOM; + mbuf->data_off = + (uint16_t)(RTE_PTR_ALIGN((char *)mbuf->buf_addr + + RTE_PKTMBUF_HEADROOM, + adap->sge.fl_align) - + (char *)mbuf->buf_addr); mbuf->next = NULL; mbuf->nb_segs = 1; mbuf->port = rxq->rspq.port_id; - mapping = (dma_addr_t)(mbuf->buf_physaddr + mbuf->data_off); + mapping = (dma_addr_t)RTE_ALIGN(mbuf->buf_iova + + mbuf->data_off, + adap->sge.fl_align); mapping |= buf_size_idx; *d++ = cpu_to_be64(mapping); set_rx_sw_desc(sd, mbuf, mapping); @@ -591,7 +596,7 @@ static inline unsigned int calc_tx_flits(const struct rte_mbuf *m) * Write Header (incorporated as part of the cpl_tx_pkt_lso and * cpl_tx_pkt structures), followed by either a TX Packet Write CPL * message or, if we're doing a Large Send Offload, an LSO CPL message - * with an embeded TX Packet Write CPL message. + * with an embedded TX Packet Write CPL message. */ flits = sgl_len(m->nb_segs); if (m->tso_segsz) @@ -681,6 +686,10 @@ static void write_sgl(struct rte_mbuf *mbuf, struct sge_txq *q, #define Q_IDXDIFF(q, idx) IDXDIFF((q)->pidx, (q)->idx, (q)->size) #define R_IDXDIFF(q, idx) IDXDIFF((q)->cidx, (q)->idx, (q)->size) +#define PIDXDIFF(head, tail, wrap) \ + ((tail) >= (head) ? (tail) - (head) : (wrap) - (head) + (tail)) +#define P_IDXDIFF(q, idx) PIDXDIFF((q)->cidx, idx, (q)->size) + /** * ring_tx_db - ring a Tx queue's doorbell * @adap: the adapter @@ -771,7 +780,7 @@ static u64 hwcsum(enum chip_type chip, const struct rte_mbuf *m) } if (likely(csum_type >= TX_CSUM_TCPIP)) { - int hdr_len = V_TXPKT_IPHDR_LEN(m->l3_len); + u64 hdr_len = V_TXPKT_IPHDR_LEN(m->l3_len); int eth_hdr_len = m->l2_len; if (CHELSIO_CHIP_VERSION(chip) <= CHELSIO_T5) @@ -846,7 +855,7 @@ static inline void ship_tx_pkt_coalesce_wr(struct adapter *adap, /* fill the pkts WR header */ wr = (void *)&q->desc[q->pidx]; - wr->op_pkd = htonl(V_FW_WR_OP(FW_ETH_TX_PKTS_WR)); + wr->op_pkd = htonl(V_FW_WR_OP(FW_ETH_TX_PKTS2_WR)); wr_mid = V_FW_WR_LEN16(DIV_ROUND_UP(q->coalesce.flits, 2)); ndesc = flits_to_desc(q->coalesce.flits); @@ -890,15 +899,11 @@ static inline int should_tx_packet_coalesce(struct sge_eth_txq *txq, struct sge_txq *q = &txq->q; unsigned int flits, ndesc; unsigned char type = 0; - int credits, hw_cidx = ntohs(q->stat->cidx); - int in_use = q->pidx - hw_cidx + flits_to_desc(q->coalesce.flits); + int credits; /* use coal WR type 1 when no frags are present */ type = (mbuf->nb_segs == 1) ? 1 : 0; - if (in_use < 0) - in_use += q->size; - if (unlikely(type != q->coalesce.type && q->coalesce.idx)) ship_tx_pkt_coalesce_wr(adap, txq); @@ -973,7 +978,7 @@ static inline int tx_do_packet_coalesce(struct sge_eth_txq *txq, struct rte_mbuf *mbuf, int flits, struct adapter *adap, const struct port_info *pi, - dma_addr_t *addr) + dma_addr_t *addr, uint16_t nb_pkts) { u64 cntrl, *end; struct sge_txq *q = &txq->q; @@ -983,6 +988,10 @@ static inline int tx_do_packet_coalesce(struct sge_eth_txq *txq, struct tx_sw_desc *sd; unsigned int idx = q->coalesce.idx, len = mbuf->pkt_len; +#ifdef RTE_LIBRTE_CXGBE_TPUT + RTE_SET_USED(nb_pkts); +#endif + if (q->coalesce.type == 0) { mc = (struct ulp_txpkt *)q->coalesce.ptr; mc->cmd_dest = htonl(V_ULPTX_CMD(4) | V_ULP_TXPKT_DEST(0) | @@ -1052,7 +1061,11 @@ static inline int tx_do_packet_coalesce(struct sge_eth_txq *txq, sd->coalesce.idx = (idx & 1) + 1; /* send the coaelsced work request if max reached */ - if (++q->coalesce.idx == ETH_COALESCE_PKT_NUM) + if (++q->coalesce.idx == ETH_COALESCE_PKT_NUM +#ifndef RTE_LIBRTE_CXGBE_TPUT + || q->coalesce.idx >= nb_pkts +#endif + ) ship_tx_pkt_coalesce_wr(adap, txq); return 0; } @@ -1064,7 +1077,8 @@ static inline int tx_do_packet_coalesce(struct sge_eth_txq *txq, * * Add a packet to an SGE Ethernet Tx queue. Runs with softirqs disabled. */ -int t4_eth_xmit(struct sge_eth_txq *txq, struct rte_mbuf *mbuf) +int t4_eth_xmit(struct sge_eth_txq *txq, struct rte_mbuf *mbuf, + uint16_t nb_pkts) { const struct port_info *pi; struct cpl_tx_pkt_lso_core *lso; @@ -1118,7 +1132,7 @@ out_free: } rte_prefetch0((volatile void *)addr); return tx_do_packet_coalesce(txq, mbuf, cflits, adap, - pi, addr); + pi, addr, nb_pkts); } else { return -EBUSY; } @@ -1189,9 +1203,15 @@ out_free: else lso->len = htonl(V_LSO_T5_XFER_SIZE(m->pkt_len)); cpl = (void *)(lso + 1); - cntrl = V_TXPKT_CSUM_TYPE(v6 ? TX_CSUM_TCPIP6 : TX_CSUM_TCPIP) | - V_TXPKT_IPHDR_LEN(l3hdr_len) | - V_TXPKT_ETHHDR_LEN(eth_xtra_len); + + if (CHELSIO_CHIP_VERSION(adap->params.chip) <= CHELSIO_T5) + cntrl = V_TXPKT_ETHHDR_LEN(eth_xtra_len); + else + cntrl = V_T6_TXPKT_ETHHDR_LEN(eth_xtra_len); + + cntrl |= V_TXPKT_CSUM_TYPE(v6 ? TX_CSUM_TCPIP6 : + TX_CSUM_TCPIP) | + V_TXPKT_IPHDR_LEN(l3hdr_len); txq->stats.tso++; txq->stats.tx_cso += m->tso_segsz; } @@ -1298,7 +1318,7 @@ alloc_sw_ring: if (metadata) *(void **)metadata = s; - *phys = (uint64_t)tz->phys_addr; + *phys = (uint64_t)tz->iova; return tz->addr; } @@ -1348,10 +1368,16 @@ int t4_ethrx_handler(struct sge_rspq *q, const __be64 *rsp, const struct rss_header *rss_hdr; bool csum_ok; struct sge_eth_rxq *rxq = container_of(q, struct sge_eth_rxq, rspq); + u16 err_vec; rss_hdr = (const void *)rsp; pkt = (const void *)&rsp[1]; - csum_ok = pkt->csum_calc && !pkt->err_vec; + /* Compressed error vector is enabled for T6 only */ + if (q->adapter->params.tp.rx_pkt_encap) + err_vec = G_T6_COMPR_RXERR_VEC(ntohs(pkt->err_vec)); + else + err_vec = ntohs(pkt->err_vec); + csum_ok = pkt->csum_calc && !err_vec; mbuf = t4_pktgl_to_mbuf(si); if (unlikely(!mbuf)) { @@ -1379,7 +1405,7 @@ int t4_ethrx_handler(struct sge_rspq *q, const __be64 *rsp, } if (pkt->vlan_ex) { - mbuf->ol_flags |= PKT_RX_VLAN_PKT; + mbuf->ol_flags |= PKT_RX_VLAN; mbuf->vlan_tci = ntohs(pkt->vlan); } rxq->stats.pkts++; @@ -1388,20 +1414,6 @@ int t4_ethrx_handler(struct sge_rspq *q, const __be64 *rsp, return 0; } -/** - * is_new_response - check if a response is newly written - * @r: the response descriptor - * @q: the response queue - * - * Returns true if a response descriptor contains a yet unprocessed - * response. - */ -static inline bool is_new_response(const struct rsp_ctrl *r, - const struct sge_rspq *q) -{ - return (r->u.type_gen >> S_RSPD_GEN) == q->gen; -} - #define CXGB4_MSG_AN ((void *)1) /** @@ -1443,12 +1455,12 @@ static int process_responses(struct sge_rspq *q, int budget, struct sge_eth_rxq *rxq = container_of(q, struct sge_eth_rxq, rspq); while (likely(budget_left)) { + if (q->cidx == ntohs(q->stat->pidx)) + break; + rc = (const struct rsp_ctrl *) ((const char *)q->cur_desc + (q->iqe_len - sizeof(*rc))); - if (!is_new_response(rc, q)) - break; - /* * Ensure response has been read */ @@ -1456,63 +1468,101 @@ static int process_responses(struct sge_rspq *q, int budget, rsp_type = G_RSPD_TYPE(rc->u.type_gen); if (likely(rsp_type == X_RSPD_TYPE_FLBUF)) { - const struct rx_sw_desc *rsd = - &rxq->fl.sdesc[rxq->fl.cidx]; - const struct rss_header *rss_hdr = - (const void *)q->cur_desc; - const struct cpl_rx_pkt *cpl = - (const void *)&q->cur_desc[1]; - bool csum_ok = cpl->csum_calc && !cpl->err_vec; - struct rte_mbuf *pkt, *npkt; - u32 len, bufsz; + unsigned int stat_pidx; + int stat_pidx_diff; - len = ntohl(rc->pldbuflen_qid); - BUG_ON(!(len & F_RSPD_NEWBUF)); - pkt = rsd->buf; - npkt = pkt; - len = G_RSPD_LEN(len); - pkt->pkt_len = len; + stat_pidx = ntohs(q->stat->pidx); + stat_pidx_diff = P_IDXDIFF(q, stat_pidx); + while (stat_pidx_diff && budget_left) { + const struct rx_sw_desc *rsd = + &rxq->fl.sdesc[rxq->fl.cidx]; + const struct rss_header *rss_hdr = + (const void *)q->cur_desc; + const struct cpl_rx_pkt *cpl = + (const void *)&q->cur_desc[1]; + struct rte_mbuf *pkt, *npkt; + u32 len, bufsz; + bool csum_ok; + u16 err_vec; - /* Chain mbufs into len if necessary */ - while (len) { - struct rte_mbuf *new_pkt = rsd->buf; + rc = (const struct rsp_ctrl *) + ((const char *)q->cur_desc + + (q->iqe_len - sizeof(*rc))); - bufsz = min(get_buf_size(q->adapter, rsd), len); - new_pkt->data_len = bufsz; - unmap_rx_buf(&rxq->fl); - len -= bufsz; - npkt->next = new_pkt; - npkt = new_pkt; - pkt->nb_segs++; - rsd = &rxq->fl.sdesc[rxq->fl.cidx]; + rsp_type = G_RSPD_TYPE(rc->u.type_gen); + if (unlikely(rsp_type != X_RSPD_TYPE_FLBUF)) + break; + + len = ntohl(rc->pldbuflen_qid); + BUG_ON(!(len & F_RSPD_NEWBUF)); + pkt = rsd->buf; + npkt = pkt; + len = G_RSPD_LEN(len); + pkt->pkt_len = len; + + /* Compressed error vector is enabled for + * T6 only + */ + if (q->adapter->params.tp.rx_pkt_encap) + err_vec = G_T6_COMPR_RXERR_VEC( + ntohs(cpl->err_vec)); + else + err_vec = ntohs(cpl->err_vec); + csum_ok = cpl->csum_calc && !err_vec; + + /* Chain mbufs into len if necessary */ + while (len) { + struct rte_mbuf *new_pkt = rsd->buf; + + bufsz = min(get_buf_size(q->adapter, + rsd), len); + new_pkt->data_len = bufsz; + unmap_rx_buf(&rxq->fl); + len -= bufsz; + npkt->next = new_pkt; + npkt = new_pkt; + pkt->nb_segs++; + rsd = &rxq->fl.sdesc[rxq->fl.cidx]; + } + npkt->next = NULL; + pkt->nb_segs--; + + if (cpl->l2info & htonl(F_RXF_IP)) { + pkt->packet_type = RTE_PTYPE_L3_IPV4; + if (unlikely(!csum_ok)) + pkt->ol_flags |= + PKT_RX_IP_CKSUM_BAD; + + if ((cpl->l2info & + htonl(F_RXF_UDP | F_RXF_TCP)) && + !csum_ok) + pkt->ol_flags |= + PKT_RX_L4_CKSUM_BAD; + } else if (cpl->l2info & htonl(F_RXF_IP6)) { + pkt->packet_type = RTE_PTYPE_L3_IPV6; + } + + if (!rss_hdr->filter_tid && + rss_hdr->hash_type) { + pkt->ol_flags |= PKT_RX_RSS_HASH; + pkt->hash.rss = + ntohl(rss_hdr->hash_val); + } + + if (cpl->vlan_ex) { + pkt->ol_flags |= PKT_RX_VLAN; + pkt->vlan_tci = ntohs(cpl->vlan); + } + + rxq->stats.pkts++; + rxq->stats.rx_bytes += pkt->pkt_len; + rx_pkts[budget - budget_left] = pkt; + + rspq_next(q); + budget_left--; + stat_pidx_diff--; } - npkt->next = NULL; - pkt->nb_segs--; - - if (cpl->l2info & htonl(F_RXF_IP)) { - pkt->packet_type = RTE_PTYPE_L3_IPV4; - if (unlikely(!csum_ok)) - pkt->ol_flags |= PKT_RX_IP_CKSUM_BAD; - - if ((cpl->l2info & - htonl(F_RXF_UDP | F_RXF_TCP)) && !csum_ok) - pkt->ol_flags |= PKT_RX_L4_CKSUM_BAD; - } else if (cpl->l2info & htonl(F_RXF_IP6)) { - pkt->packet_type = RTE_PTYPE_L3_IPV6; - } - - if (!rss_hdr->filter_tid && rss_hdr->hash_type) { - pkt->ol_flags |= PKT_RX_RSS_HASH; - pkt->hash.rss = ntohl(rss_hdr->hash_val); - } - - if (cpl->vlan_ex) { - pkt->ol_flags |= PKT_RX_VLAN_PKT; - pkt->vlan_tci = ntohs(cpl->vlan); - } - rxq->stats.pkts++; - rxq->stats.rx_bytes += pkt->pkt_len; - rx_pkts[budget - budget_left] = pkt; + continue; } else if (likely(rsp_type == X_RSPD_TYPE_CPL)) { ret = q->handler(q, q->cur_desc, NULL); } else { @@ -1527,35 +1577,6 @@ static int process_responses(struct sge_rspq *q, int budget, rspq_next(q); budget_left--; - - if (R_IDXDIFF(q, gts_idx) >= 64) { - unsigned int cidx_inc = R_IDXDIFF(q, gts_idx); - unsigned int params; - u32 val; - - if (fl_cap(&rxq->fl) - rxq->fl.avail >= 64) - __refill_fl(q->adapter, &rxq->fl); - params = V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX); - q->next_intr_params = params; - val = V_CIDXINC(cidx_inc) | V_SEINTARM(params); - - if (unlikely(!q->bar2_addr)) - t4_write_reg(q->adapter, MYPF_REG(A_SGE_PF_GTS), - val | - V_INGRESSQID((u32)q->cntxt_id)); - else { - writel(val | V_INGRESSQID(q->bar2_qid), - (void *)((uintptr_t)q->bar2_addr + - SGE_UDB_GTS)); - /* - * This Write memory Barrier will force the - * write to the User Doorbell area to be - * flushed. - */ - wmb(); - } - q->gts_idx = q->cidx; - } } /* @@ -1573,10 +1594,38 @@ static int process_responses(struct sge_rspq *q, int budget, int cxgbe_poll(struct sge_rspq *q, struct rte_mbuf **rx_pkts, unsigned int budget, unsigned int *work_done) { - int err = 0; + struct sge_eth_rxq *rxq = container_of(q, struct sge_eth_rxq, rspq); + unsigned int cidx_inc; + unsigned int params; + u32 val; *work_done = process_responses(q, budget, rx_pkts); - return err; + + if (*work_done) { + cidx_inc = R_IDXDIFF(q, gts_idx); + + if (q->offset >= 0 && fl_cap(&rxq->fl) - rxq->fl.avail >= 64) + __refill_fl(q->adapter, &rxq->fl); + + params = q->intr_params; + q->next_intr_params = params; + val = V_CIDXINC(cidx_inc) | V_SEINTARM(params); + + if (unlikely(!q->bar2_addr)) { + t4_write_reg(q->adapter, MYPF_REG(A_SGE_PF_GTS), + val | V_INGRESSQID((u32)q->cntxt_id)); + } else { + writel(val | V_INGRESSQID(q->bar2_qid), + (void *)((uintptr_t)q->bar2_addr + SGE_UDB_GTS)); + /* This Write memory Barrier will force the + * write to the User Doorbell area to be + * flushed. + */ + wmb(); + } + q->gts_idx = q->cidx; + } + return 0; } /** @@ -1645,7 +1694,8 @@ int t4_sge_alloc_rxq(struct adapter *adap, struct sge_rspq *iq, bool fwevtq, iq->size = cxgbe_roundup(iq->size, 16); snprintf(z_name, sizeof(z_name), "%s_%s_%d_%d", - eth_dev->driver->pci_drv.name, fwevtq ? "fwq_ring" : "rx_ring", + eth_dev->device->driver->name, + fwevtq ? "fwq_ring" : "rx_ring", eth_dev->data->port_id, queue_id); snprintf(z_name_sw, sizeof(z_name_sw), "%s_sw_ring", z_name); @@ -1665,24 +1715,25 @@ int t4_sge_alloc_rxq(struct adapter *adap, struct sge_rspq *iq, bool fwevtq, V_FW_IQ_CMD_IQASYNCH(fwevtq) | V_FW_IQ_CMD_VIID(pi->viid) | V_FW_IQ_CMD_IQANDST(intr_idx < 0) | - V_FW_IQ_CMD_IQANUD(X_UPDATEDELIVERY_INTERRUPT) | + V_FW_IQ_CMD_IQANUD(X_UPDATEDELIVERY_STATUS_PAGE) | V_FW_IQ_CMD_IQANDSTINDEX(intr_idx >= 0 ? intr_idx : -intr_idx - 1)); c.iqdroprss_to_iqesize = - htons(V_FW_IQ_CMD_IQPCIECH(pi->tx_chan) | + htons(V_FW_IQ_CMD_IQPCIECH(cong > 0 ? cxgbe_ffs(cong) - 1 : + pi->tx_chan) | F_FW_IQ_CMD_IQGTSMODE | V_FW_IQ_CMD_IQINTCNTTHRESH(iq->pktcnt_idx) | V_FW_IQ_CMD_IQESIZE(ilog2(iq->iqe_len) - 4)); c.iqsize = htons(iq->size); c.iqaddr = cpu_to_be64(iq->phys_addr); if (cong >= 0) - c.iqns_to_fl0congen = htonl(F_FW_IQ_CMD_IQFLINTCONGEN); + c.iqns_to_fl0congen = htonl(F_FW_IQ_CMD_IQFLINTCONGEN | + F_FW_IQ_CMD_IQRO); if (fl) { struct sge_eth_rxq *rxq = container_of(fl, struct sge_eth_rxq, fl); - enum chip_type chip = (enum chip_type)CHELSIO_CHIP_VERSION( - adap->params.chip); + unsigned int chip_ver = CHELSIO_CHIP_VERSION(adap->params.chip); /* * Allocate the ring for the hardware free list (with space @@ -1697,7 +1748,7 @@ int t4_sge_alloc_rxq(struct adapter *adap, struct sge_rspq *iq, bool fwevtq, fl->size = cxgbe_roundup(fl->size, 8); snprintf(z_name, sizeof(z_name), "%s_%s_%d_%d", - eth_dev->driver->pci_drv.name, + eth_dev->device->driver->name, fwevtq ? "fwq_ring" : "fl_ring", eth_dev->data->port_id, queue_id); snprintf(z_name_sw, sizeof(z_name_sw), "%s_sw_ring", z_name); @@ -1728,9 +1779,12 @@ int t4_sge_alloc_rxq(struct adapter *adap, struct sge_rspq *iq, bool fwevtq, * Hence maximum allowed burst size will be 448 bytes. */ c.fl0dcaen_to_fl0cidxfthresh = - htons(V_FW_IQ_CMD_FL0FBMIN(X_FETCHBURSTMIN_128B) | - V_FW_IQ_CMD_FL0FBMAX((chip <= CHELSIO_T5) ? - X_FETCHBURSTMAX_512B : X_FETCHBURSTMAX_256B)); + htons(V_FW_IQ_CMD_FL0FBMIN(chip_ver <= CHELSIO_T5 ? + X_FETCHBURSTMIN_128B : + X_FETCHBURSTMIN_64B) | + V_FW_IQ_CMD_FL0FBMAX(chip_ver <= CHELSIO_T5 ? + X_FETCHBURSTMAX_512B : + X_FETCHBURSTMAX_256B)); c.fl0size = htons(flsz); c.fl0addr = cpu_to_be64(fl->addr); } @@ -1749,6 +1803,7 @@ int t4_sge_alloc_rxq(struct adapter *adap, struct sge_rspq *iq, bool fwevtq, iq->bar2_addr = bar2_address(adap, iq->cntxt_id, T4_BAR2_QTYPE_INGRESS, &iq->bar2_qid); iq->size--; /* subtract status entry */ + iq->stat = (void *)&iq->desc[iq->size * 8]; iq->eth_dev = eth_dev; iq->handler = hnd; iq->port_id = pi->port_id; @@ -1893,7 +1948,7 @@ int t4_sge_alloc_eth_txq(struct adapter *adap, struct sge_eth_txq *txq, nentries = txq->q.size + s->stat_len / sizeof(struct tx_desc); snprintf(z_name, sizeof(z_name), "%s_%s_%d_%d", - eth_dev->driver->pci_drv.name, "tx_ring", + eth_dev->device->driver->name, "tx_ring", eth_dev->data->port_id, queue_id); snprintf(z_name_sw, sizeof(z_name_sw), "%s_sw_ring", z_name); @@ -2186,8 +2241,7 @@ static int t4_sge_init_soft(struct adapter *adap) int t4_sge_init(struct adapter *adap) { struct sge *s = &adap->sge; - u32 sge_control, sge_control2, sge_conm_ctrl; - unsigned int ingpadboundary, ingpackboundary; + u32 sge_control, sge_conm_ctrl; int ret, egress_threshold; /* @@ -2197,34 +2251,7 @@ int t4_sge_init(struct adapter *adap) sge_control = t4_read_reg(adap, A_SGE_CONTROL); s->pktshift = G_PKTSHIFT(sge_control); s->stat_len = (sge_control & F_EGRSTATUSPAGESIZE) ? 128 : 64; - - /* - * T4 uses a single control field to specify both the PCIe Padding and - * Packing Boundary. T5 introduced the ability to specify these - * separately. The actual Ingress Packet Data alignment boundary - * within Packed Buffer Mode is the maximum of these two - * specifications. - */ - ingpadboundary = 1 << (G_INGPADBOUNDARY(sge_control) + - X_INGPADBOUNDARY_SHIFT); - s->fl_align = ingpadboundary; - - if (!is_t4(adap->params.chip) && !adap->use_unpacked_mode) { - /* - * T5 has a weird interpretation of one of the PCIe Packing - * Boundary values. No idea why ... - */ - sge_control2 = t4_read_reg(adap, A_SGE_CONTROL2); - ingpackboundary = G_INGPACKBOUNDARY(sge_control2); - if (ingpackboundary == X_INGPACKBOUNDARY_16B) - ingpackboundary = 16; - else - ingpackboundary = 1 << (ingpackboundary + - X_INGPACKBOUNDARY_SHIFT); - - s->fl_align = max(ingpadboundary, ingpackboundary); - } - + s->fl_align = t4_fl_pkt_align(adap); ret = t4_sge_init_soft(adap); if (ret < 0) { dev_err(adap, "%s: t4_sge_init_soft failed, error %d\n", diff --git a/dpdk/drivers/net/dpaa/Makefile b/dpdk/drivers/net/dpaa/Makefile new file mode 100644 index 00000000..171686ec --- /dev/null +++ b/dpdk/drivers/net/dpaa/Makefile @@ -0,0 +1,63 @@ +# BSD LICENSE +# +# Copyright 2017 NXP. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of NXP nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +include $(RTE_SDK)/mk/rte.vars.mk +RTE_SDK_DPAA=$(RTE_SDK)/drivers/net/dpaa + +# +# library name +# +LIB = librte_pmd_dpaa.a + +CFLAGS := -I$(SRCDIR) $(CFLAGS) +CFLAGS += -O3 $(WERROR_FLAGS) +CFLAGS += -Wno-pointer-arith +CFLAGS += -I$(RTE_SDK_DPAA)/ +CFLAGS += -I$(RTE_SDK_DPAA)/include +CFLAGS += -I$(RTE_SDK)/drivers/bus/dpaa +CFLAGS += -I$(RTE_SDK)/drivers/bus/dpaa/include/ +CFLAGS += -I$(RTE_SDK)/drivers/mempool/dpaa +CFLAGS += -I$(RTE_SDK)/lib/librte_eal/common/include +CFLAGS += -I$(RTE_SDK)/lib/librte_eal/linuxapp/eal/include + +EXPORT_MAP := rte_pmd_dpaa_version.map + +LIBABIVER := 1 + +# Interfaces with DPDK +SRCS-$(CONFIG_RTE_LIBRTE_DPAA_PMD) += dpaa_ethdev.c +SRCS-$(CONFIG_RTE_LIBRTE_DPAA_PMD) += dpaa_rxtx.c + +LDLIBS += -lrte_bus_dpaa +LDLIBS += -lrte_mempool_dpaa +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/net/dpaa/dpaa_ethdev.c b/dpdk/drivers/net/dpaa/dpaa_ethdev.c new file mode 100644 index 00000000..e4375c37 --- /dev/null +++ b/dpdk/drivers/net/dpaa/dpaa_ethdev.c @@ -0,0 +1,1119 @@ +/*- + * BSD LICENSE + * + * Copyright 2016 Freescale Semiconductor, Inc. All rights reserved. + * Copyright 2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Freescale Semiconductor, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +/* System headers */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include + +#include +#include +#include +#include + +/* Keep track of whether QMAN and BMAN have been globally initialized */ +static int is_global_init; + +struct rte_dpaa_xstats_name_off { + char name[RTE_ETH_XSTATS_NAME_SIZE]; + uint32_t offset; +}; + +static const struct rte_dpaa_xstats_name_off dpaa_xstats_strings[] = { + {"rx_align_err", + offsetof(struct dpaa_if_stats, raln)}, + {"rx_valid_pause", + offsetof(struct dpaa_if_stats, rxpf)}, + {"rx_fcs_err", + offsetof(struct dpaa_if_stats, rfcs)}, + {"rx_vlan_frame", + offsetof(struct dpaa_if_stats, rvlan)}, + {"rx_frame_err", + offsetof(struct dpaa_if_stats, rerr)}, + {"rx_drop_err", + offsetof(struct dpaa_if_stats, rdrp)}, + {"rx_undersized", + offsetof(struct dpaa_if_stats, rund)}, + {"rx_oversize_err", + offsetof(struct dpaa_if_stats, rovr)}, + {"rx_fragment_pkt", + offsetof(struct dpaa_if_stats, rfrg)}, + {"tx_valid_pause", + offsetof(struct dpaa_if_stats, txpf)}, + {"tx_fcs_err", + offsetof(struct dpaa_if_stats, terr)}, + {"tx_vlan_frame", + offsetof(struct dpaa_if_stats, tvlan)}, + {"rx_undersized", + offsetof(struct dpaa_if_stats, tund)}, +}; + +static int +dpaa_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) +{ + struct dpaa_if *dpaa_intf = dev->data->dev_private; + + PMD_INIT_FUNC_TRACE(); + + if (mtu < ETHER_MIN_MTU) + return -EINVAL; + if (mtu > ETHER_MAX_LEN) + dev->data->dev_conf.rxmode.jumbo_frame = 1; + else + dev->data->dev_conf.rxmode.jumbo_frame = 0; + + dev->data->dev_conf.rxmode.max_rx_pkt_len = mtu; + + fman_if_set_maxfrm(dpaa_intf->fif, mtu); + + return 0; +} + +static int +dpaa_eth_dev_configure(struct rte_eth_dev *dev __rte_unused) +{ + PMD_INIT_FUNC_TRACE(); + + if (dev->data->dev_conf.rxmode.jumbo_frame == 1) { + if (dev->data->dev_conf.rxmode.max_rx_pkt_len <= + DPAA_MAX_RX_PKT_LEN) + return dpaa_mtu_set(dev, + dev->data->dev_conf.rxmode.max_rx_pkt_len); + else + return -1; + } + return 0; +} + +static const uint32_t * +dpaa_supported_ptypes_get(struct rte_eth_dev *dev) +{ + static const uint32_t ptypes[] = { + /*todo -= add more types */ + RTE_PTYPE_L2_ETHER, + RTE_PTYPE_L3_IPV4, + RTE_PTYPE_L3_IPV4_EXT, + RTE_PTYPE_L3_IPV6, + RTE_PTYPE_L3_IPV6_EXT, + RTE_PTYPE_L4_TCP, + RTE_PTYPE_L4_UDP, + RTE_PTYPE_L4_SCTP + }; + + PMD_INIT_FUNC_TRACE(); + + if (dev->rx_pkt_burst == dpaa_eth_queue_rx) + return ptypes; + return NULL; +} + +static int dpaa_eth_dev_start(struct rte_eth_dev *dev) +{ + struct dpaa_if *dpaa_intf = dev->data->dev_private; + + PMD_INIT_FUNC_TRACE(); + + /* Change tx callback to the real one */ + dev->tx_pkt_burst = dpaa_eth_queue_tx; + fman_if_enable_rx(dpaa_intf->fif); + + return 0; +} + +static void dpaa_eth_dev_stop(struct rte_eth_dev *dev) +{ + struct dpaa_if *dpaa_intf = dev->data->dev_private; + + PMD_INIT_FUNC_TRACE(); + + fman_if_disable_rx(dpaa_intf->fif); + dev->tx_pkt_burst = dpaa_eth_tx_drop_all; +} + +static void dpaa_eth_dev_close(struct rte_eth_dev *dev) +{ + PMD_INIT_FUNC_TRACE(); + + dpaa_eth_dev_stop(dev); +} + +static int +dpaa_fw_version_get(struct rte_eth_dev *dev __rte_unused, + char *fw_version, + size_t fw_size) +{ + int ret; + FILE *svr_file = NULL; + unsigned int svr_ver = 0; + + PMD_INIT_FUNC_TRACE(); + + svr_file = fopen(DPAA_SOC_ID_FILE, "r"); + if (!svr_file) { + DPAA_PMD_ERR("Unable to open SoC device"); + return -ENOTSUP; /* Not supported on this infra */ + } + if (fscanf(svr_file, "svr:%x", &svr_ver) <= 0) + DPAA_PMD_ERR("Unable to read SoC device"); + + fclose(svr_file); + + ret = snprintf(fw_version, fw_size, "SVR:%x-fman-v%x", + svr_ver, fman_ip_rev); + ret += 1; /* add the size of '\0' */ + + if (fw_size < (uint32_t)ret) + return ret; + else + return 0; +} + +static void dpaa_eth_dev_info(struct rte_eth_dev *dev, + struct rte_eth_dev_info *dev_info) +{ + struct dpaa_if *dpaa_intf = dev->data->dev_private; + + PMD_INIT_FUNC_TRACE(); + + dev_info->max_rx_queues = dpaa_intf->nb_rx_queues; + dev_info->max_tx_queues = dpaa_intf->nb_tx_queues; + dev_info->min_rx_bufsize = DPAA_MIN_RX_BUF_SIZE; + dev_info->max_rx_pktlen = DPAA_MAX_RX_PKT_LEN; + dev_info->max_mac_addrs = DPAA_MAX_MAC_FILTER; + dev_info->max_hash_mac_addrs = 0; + dev_info->max_vfs = 0; + dev_info->max_vmdq_pools = ETH_16_POOLS; + dev_info->flow_type_rss_offloads = DPAA_RSS_OFFLOAD_ALL; + dev_info->speed_capa = (ETH_LINK_SPEED_1G | + ETH_LINK_SPEED_10G); + dev_info->rx_offload_capa = + (DEV_RX_OFFLOAD_IPV4_CKSUM | + DEV_RX_OFFLOAD_UDP_CKSUM | + DEV_RX_OFFLOAD_TCP_CKSUM); + dev_info->tx_offload_capa = + (DEV_TX_OFFLOAD_IPV4_CKSUM | + DEV_TX_OFFLOAD_UDP_CKSUM | + DEV_TX_OFFLOAD_TCP_CKSUM); +} + +static int dpaa_eth_link_update(struct rte_eth_dev *dev, + int wait_to_complete __rte_unused) +{ + struct dpaa_if *dpaa_intf = dev->data->dev_private; + struct rte_eth_link *link = &dev->data->dev_link; + + PMD_INIT_FUNC_TRACE(); + + if (dpaa_intf->fif->mac_type == fman_mac_1g) + link->link_speed = 1000; + else if (dpaa_intf->fif->mac_type == fman_mac_10g) + link->link_speed = 10000; + else + DPAA_PMD_ERR("invalid link_speed: %s, %d", + dpaa_intf->name, dpaa_intf->fif->mac_type); + + link->link_status = dpaa_intf->valid; + link->link_duplex = ETH_LINK_FULL_DUPLEX; + link->link_autoneg = ETH_LINK_AUTONEG; + return 0; +} + +static int dpaa_eth_stats_get(struct rte_eth_dev *dev, + struct rte_eth_stats *stats) +{ + struct dpaa_if *dpaa_intf = dev->data->dev_private; + + PMD_INIT_FUNC_TRACE(); + + fman_if_stats_get(dpaa_intf->fif, stats); + return 0; +} + +static void dpaa_eth_stats_reset(struct rte_eth_dev *dev) +{ + struct dpaa_if *dpaa_intf = dev->data->dev_private; + + PMD_INIT_FUNC_TRACE(); + + fman_if_stats_reset(dpaa_intf->fif); +} + +static int +dpaa_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, + unsigned int n) +{ + struct dpaa_if *dpaa_intf = dev->data->dev_private; + unsigned int i = 0, num = RTE_DIM(dpaa_xstats_strings); + uint64_t values[sizeof(struct dpaa_if_stats) / 8]; + + if (xstats == NULL) + return 0; + + if (n < num) + return num; + + fman_if_stats_get_all(dpaa_intf->fif, values, + sizeof(struct dpaa_if_stats) / 8); + + for (i = 0; i < num; i++) { + xstats[i].id = i; + xstats[i].value = values[dpaa_xstats_strings[i].offset / 8]; + } + return i; +} + +static int +dpaa_xstats_get_names(__rte_unused struct rte_eth_dev *dev, + struct rte_eth_xstat_name *xstats_names, + __rte_unused unsigned int limit) +{ + unsigned int i, stat_cnt = RTE_DIM(dpaa_xstats_strings); + + if (xstats_names != NULL) + for (i = 0; i < stat_cnt; i++) + snprintf(xstats_names[i].name, + sizeof(xstats_names[i].name), + "%s", + dpaa_xstats_strings[i].name); + + return stat_cnt; +} + +static int +dpaa_xstats_get_by_id(struct rte_eth_dev *dev, const uint64_t *ids, + uint64_t *values, unsigned int n) +{ + unsigned int i, stat_cnt = RTE_DIM(dpaa_xstats_strings); + uint64_t values_copy[sizeof(struct dpaa_if_stats) / 8]; + + if (!ids) { + struct dpaa_if *dpaa_intf = dev->data->dev_private; + + if (n < stat_cnt) + return stat_cnt; + + if (!values) + return 0; + + fman_if_stats_get_all(dpaa_intf->fif, values_copy, + sizeof(struct dpaa_if_stats)); + + for (i = 0; i < stat_cnt; i++) + values[i] = + values_copy[dpaa_xstats_strings[i].offset / 8]; + + return stat_cnt; + } + + dpaa_xstats_get_by_id(dev, NULL, values_copy, stat_cnt); + + for (i = 0; i < n; i++) { + if (ids[i] >= stat_cnt) { + DPAA_PMD_ERR("id value isn't valid"); + return -1; + } + values[i] = values_copy[ids[i]]; + } + return n; +} + +static int +dpaa_xstats_get_names_by_id( + struct rte_eth_dev *dev, + struct rte_eth_xstat_name *xstats_names, + const uint64_t *ids, + unsigned int limit) +{ + unsigned int i, stat_cnt = RTE_DIM(dpaa_xstats_strings); + struct rte_eth_xstat_name xstats_names_copy[stat_cnt]; + + if (!ids) + return dpaa_xstats_get_names(dev, xstats_names, limit); + + dpaa_xstats_get_names(dev, xstats_names_copy, limit); + + for (i = 0; i < limit; i++) { + if (ids[i] >= stat_cnt) { + DPAA_PMD_ERR("id value isn't valid"); + return -1; + } + strcpy(xstats_names[i].name, xstats_names_copy[ids[i]].name); + } + return limit; +} + +static void dpaa_eth_promiscuous_enable(struct rte_eth_dev *dev) +{ + struct dpaa_if *dpaa_intf = dev->data->dev_private; + + PMD_INIT_FUNC_TRACE(); + + fman_if_promiscuous_enable(dpaa_intf->fif); +} + +static void dpaa_eth_promiscuous_disable(struct rte_eth_dev *dev) +{ + struct dpaa_if *dpaa_intf = dev->data->dev_private; + + PMD_INIT_FUNC_TRACE(); + + fman_if_promiscuous_disable(dpaa_intf->fif); +} + +static void dpaa_eth_multicast_enable(struct rte_eth_dev *dev) +{ + struct dpaa_if *dpaa_intf = dev->data->dev_private; + + PMD_INIT_FUNC_TRACE(); + + fman_if_set_mcast_filter_table(dpaa_intf->fif); +} + +static void dpaa_eth_multicast_disable(struct rte_eth_dev *dev) +{ + struct dpaa_if *dpaa_intf = dev->data->dev_private; + + PMD_INIT_FUNC_TRACE(); + + fman_if_reset_mcast_filter_table(dpaa_intf->fif); +} + +static +int dpaa_eth_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, + uint16_t nb_desc __rte_unused, + unsigned int socket_id __rte_unused, + const struct rte_eth_rxconf *rx_conf __rte_unused, + struct rte_mempool *mp) +{ + struct dpaa_if *dpaa_intf = dev->data->dev_private; + + PMD_INIT_FUNC_TRACE(); + + DPAA_PMD_INFO("Rx queue setup for queue index: %d", queue_idx); + + if (!dpaa_intf->bp_info || dpaa_intf->bp_info->mp != mp) { + struct fman_if_ic_params icp; + uint32_t fd_offset; + uint32_t bp_size; + + if (!mp->pool_data) { + DPAA_PMD_ERR("Not an offloaded buffer pool!"); + return -1; + } + dpaa_intf->bp_info = DPAA_MEMPOOL_TO_POOL_INFO(mp); + + memset(&icp, 0, sizeof(icp)); + /* set ICEOF for to the default value , which is 0*/ + icp.iciof = DEFAULT_ICIOF; + icp.iceof = DEFAULT_RX_ICEOF; + icp.icsz = DEFAULT_ICSZ; + fman_if_set_ic_params(dpaa_intf->fif, &icp); + + fd_offset = RTE_PKTMBUF_HEADROOM + DPAA_HW_BUF_RESERVE; + fman_if_set_fdoff(dpaa_intf->fif, fd_offset); + + /* Buffer pool size should be equal to Dataroom Size*/ + bp_size = rte_pktmbuf_data_room_size(mp); + fman_if_set_bp(dpaa_intf->fif, mp->size, + dpaa_intf->bp_info->bpid, bp_size); + dpaa_intf->valid = 1; + DPAA_PMD_INFO("if =%s - fd_offset = %d offset = %d", + dpaa_intf->name, fd_offset, + fman_if_get_fdoff(dpaa_intf->fif)); + } + dev->data->rx_queues[queue_idx] = &dpaa_intf->rx_queues[queue_idx]; + + return 0; +} + +static +void dpaa_eth_rx_queue_release(void *rxq __rte_unused) +{ + PMD_INIT_FUNC_TRACE(); +} + +static +int dpaa_eth_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, + uint16_t nb_desc __rte_unused, + unsigned int socket_id __rte_unused, + const struct rte_eth_txconf *tx_conf __rte_unused) +{ + struct dpaa_if *dpaa_intf = dev->data->dev_private; + + PMD_INIT_FUNC_TRACE(); + + DPAA_PMD_INFO("Tx queue setup for queue index: %d", queue_idx); + dev->data->tx_queues[queue_idx] = &dpaa_intf->tx_queues[queue_idx]; + return 0; +} + +static void dpaa_eth_tx_queue_release(void *txq __rte_unused) +{ + PMD_INIT_FUNC_TRACE(); +} + +static int dpaa_link_down(struct rte_eth_dev *dev) +{ + PMD_INIT_FUNC_TRACE(); + + dpaa_eth_dev_stop(dev); + return 0; +} + +static int dpaa_link_up(struct rte_eth_dev *dev) +{ + PMD_INIT_FUNC_TRACE(); + + dpaa_eth_dev_start(dev); + return 0; +} + +static int +dpaa_flow_ctrl_set(struct rte_eth_dev *dev, + struct rte_eth_fc_conf *fc_conf) +{ + struct dpaa_if *dpaa_intf = dev->data->dev_private; + struct rte_eth_fc_conf *net_fc; + + PMD_INIT_FUNC_TRACE(); + + if (!(dpaa_intf->fc_conf)) { + dpaa_intf->fc_conf = rte_zmalloc(NULL, + sizeof(struct rte_eth_fc_conf), MAX_CACHELINE); + if (!dpaa_intf->fc_conf) { + DPAA_PMD_ERR("unable to save flow control info"); + return -ENOMEM; + } + } + net_fc = dpaa_intf->fc_conf; + + if (fc_conf->high_water < fc_conf->low_water) { + DPAA_PMD_ERR("Incorrect Flow Control Configuration"); + return -EINVAL; + } + + if (fc_conf->mode == RTE_FC_NONE) { + return 0; + } else if (fc_conf->mode == RTE_FC_TX_PAUSE || + fc_conf->mode == RTE_FC_FULL) { + fman_if_set_fc_threshold(dpaa_intf->fif, fc_conf->high_water, + fc_conf->low_water, + dpaa_intf->bp_info->bpid); + if (fc_conf->pause_time) + fman_if_set_fc_quanta(dpaa_intf->fif, + fc_conf->pause_time); + } + + /* Save the information in dpaa device */ + net_fc->pause_time = fc_conf->pause_time; + net_fc->high_water = fc_conf->high_water; + net_fc->low_water = fc_conf->low_water; + net_fc->send_xon = fc_conf->send_xon; + net_fc->mac_ctrl_frame_fwd = fc_conf->mac_ctrl_frame_fwd; + net_fc->mode = fc_conf->mode; + net_fc->autoneg = fc_conf->autoneg; + + return 0; +} + +static int +dpaa_flow_ctrl_get(struct rte_eth_dev *dev, + struct rte_eth_fc_conf *fc_conf) +{ + struct dpaa_if *dpaa_intf = dev->data->dev_private; + struct rte_eth_fc_conf *net_fc = dpaa_intf->fc_conf; + int ret; + + PMD_INIT_FUNC_TRACE(); + + if (net_fc) { + fc_conf->pause_time = net_fc->pause_time; + fc_conf->high_water = net_fc->high_water; + fc_conf->low_water = net_fc->low_water; + fc_conf->send_xon = net_fc->send_xon; + fc_conf->mac_ctrl_frame_fwd = net_fc->mac_ctrl_frame_fwd; + fc_conf->mode = net_fc->mode; + fc_conf->autoneg = net_fc->autoneg; + return 0; + } + ret = fman_if_get_fc_threshold(dpaa_intf->fif); + if (ret) { + fc_conf->mode = RTE_FC_TX_PAUSE; + fc_conf->pause_time = fman_if_get_fc_quanta(dpaa_intf->fif); + } else { + fc_conf->mode = RTE_FC_NONE; + } + + return 0; +} + +static int +dpaa_dev_add_mac_addr(struct rte_eth_dev *dev, + struct ether_addr *addr, + uint32_t index, + __rte_unused uint32_t pool) +{ + int ret; + struct dpaa_if *dpaa_intf = dev->data->dev_private; + + PMD_INIT_FUNC_TRACE(); + + ret = fman_if_add_mac_addr(dpaa_intf->fif, addr->addr_bytes, index); + + if (ret) + RTE_LOG(ERR, PMD, "error: Adding the MAC ADDR failed:" + " err = %d", ret); + return 0; +} + +static void +dpaa_dev_remove_mac_addr(struct rte_eth_dev *dev, + uint32_t index) +{ + struct dpaa_if *dpaa_intf = dev->data->dev_private; + + PMD_INIT_FUNC_TRACE(); + + fman_if_clear_mac_addr(dpaa_intf->fif, index); +} + +static void +dpaa_dev_set_mac_addr(struct rte_eth_dev *dev, + struct ether_addr *addr) +{ + int ret; + struct dpaa_if *dpaa_intf = dev->data->dev_private; + + PMD_INIT_FUNC_TRACE(); + + ret = fman_if_add_mac_addr(dpaa_intf->fif, addr->addr_bytes, 0); + if (ret) + RTE_LOG(ERR, PMD, "error: Setting the MAC ADDR failed %d", ret); +} + +static struct eth_dev_ops dpaa_devops = { + .dev_configure = dpaa_eth_dev_configure, + .dev_start = dpaa_eth_dev_start, + .dev_stop = dpaa_eth_dev_stop, + .dev_close = dpaa_eth_dev_close, + .dev_infos_get = dpaa_eth_dev_info, + .dev_supported_ptypes_get = dpaa_supported_ptypes_get, + + .rx_queue_setup = dpaa_eth_rx_queue_setup, + .tx_queue_setup = dpaa_eth_tx_queue_setup, + .rx_queue_release = dpaa_eth_rx_queue_release, + .tx_queue_release = dpaa_eth_tx_queue_release, + + .flow_ctrl_get = dpaa_flow_ctrl_get, + .flow_ctrl_set = dpaa_flow_ctrl_set, + + .link_update = dpaa_eth_link_update, + .stats_get = dpaa_eth_stats_get, + .xstats_get = dpaa_dev_xstats_get, + .xstats_get_by_id = dpaa_xstats_get_by_id, + .xstats_get_names_by_id = dpaa_xstats_get_names_by_id, + .xstats_get_names = dpaa_xstats_get_names, + .xstats_reset = dpaa_eth_stats_reset, + .stats_reset = dpaa_eth_stats_reset, + .promiscuous_enable = dpaa_eth_promiscuous_enable, + .promiscuous_disable = dpaa_eth_promiscuous_disable, + .allmulticast_enable = dpaa_eth_multicast_enable, + .allmulticast_disable = dpaa_eth_multicast_disable, + .mtu_set = dpaa_mtu_set, + .dev_set_link_down = dpaa_link_down, + .dev_set_link_up = dpaa_link_up, + .mac_addr_add = dpaa_dev_add_mac_addr, + .mac_addr_remove = dpaa_dev_remove_mac_addr, + .mac_addr_set = dpaa_dev_set_mac_addr, + + .fw_version_get = dpaa_fw_version_get, +}; + +static int dpaa_fc_set_default(struct dpaa_if *dpaa_intf) +{ + struct rte_eth_fc_conf *fc_conf; + int ret; + + PMD_INIT_FUNC_TRACE(); + + if (!(dpaa_intf->fc_conf)) { + dpaa_intf->fc_conf = rte_zmalloc(NULL, + sizeof(struct rte_eth_fc_conf), MAX_CACHELINE); + if (!dpaa_intf->fc_conf) { + DPAA_PMD_ERR("unable to save flow control info"); + return -ENOMEM; + } + } + fc_conf = dpaa_intf->fc_conf; + ret = fman_if_get_fc_threshold(dpaa_intf->fif); + if (ret) { + fc_conf->mode = RTE_FC_TX_PAUSE; + fc_conf->pause_time = fman_if_get_fc_quanta(dpaa_intf->fif); + } else { + fc_conf->mode = RTE_FC_NONE; + } + + return 0; +} + +/* Initialise an Rx FQ */ +static int dpaa_rx_queue_init(struct qman_fq *fq, + uint32_t fqid) +{ + struct qm_mcc_initfq opts = {0}; + int ret; + + PMD_INIT_FUNC_TRACE(); + + ret = qman_reserve_fqid(fqid); + if (ret) { + DPAA_PMD_ERR("reserve rx fqid %d failed with ret: %d", + fqid, ret); + return -EINVAL; + } + + DPAA_PMD_DEBUG("creating rx fq %p, fqid %d", fq, fqid); + ret = qman_create_fq(fqid, QMAN_FQ_FLAG_NO_ENQUEUE, fq); + if (ret) { + DPAA_PMD_ERR("create rx fqid %d failed with ret: %d", + fqid, ret); + return ret; + } + + opts.we_mask = QM_INITFQ_WE_DESTWQ | QM_INITFQ_WE_FQCTRL | + QM_INITFQ_WE_CONTEXTA; + + opts.fqd.dest.wq = DPAA_IF_RX_PRIORITY; + opts.fqd.fq_ctrl = QM_FQCTRL_AVOIDBLOCK | QM_FQCTRL_CTXASTASHING | + QM_FQCTRL_PREFERINCACHE; + opts.fqd.context_a.stashing.exclusive = 0; + opts.fqd.context_a.stashing.annotation_cl = DPAA_IF_RX_ANNOTATION_STASH; + opts.fqd.context_a.stashing.data_cl = DPAA_IF_RX_DATA_STASH; + opts.fqd.context_a.stashing.context_cl = DPAA_IF_RX_CONTEXT_STASH; + + /*Enable tail drop */ + opts.we_mask = opts.we_mask | QM_INITFQ_WE_TDTHRESH; + opts.fqd.fq_ctrl = opts.fqd.fq_ctrl | QM_FQCTRL_TDE; + qm_fqd_taildrop_set(&opts.fqd.td, CONG_THRESHOLD_RX_Q, 1); + + ret = qman_init_fq(fq, 0, &opts); + if (ret) + DPAA_PMD_ERR("init rx fqid %d failed with ret: %d", fqid, ret); + return ret; +} + +/* Initialise a Tx FQ */ +static int dpaa_tx_queue_init(struct qman_fq *fq, + struct fman_if *fman_intf) +{ + struct qm_mcc_initfq opts = {0}; + int ret; + + PMD_INIT_FUNC_TRACE(); + + ret = qman_create_fq(0, QMAN_FQ_FLAG_DYNAMIC_FQID | + QMAN_FQ_FLAG_TO_DCPORTAL, fq); + if (ret) { + DPAA_PMD_ERR("create tx fq failed with ret: %d", ret); + return ret; + } + opts.we_mask = QM_INITFQ_WE_DESTWQ | QM_INITFQ_WE_FQCTRL | + QM_INITFQ_WE_CONTEXTB | QM_INITFQ_WE_CONTEXTA; + opts.fqd.dest.channel = fman_intf->tx_channel_id; + opts.fqd.dest.wq = DPAA_IF_TX_PRIORITY; + opts.fqd.fq_ctrl = QM_FQCTRL_PREFERINCACHE; + opts.fqd.context_b = 0; + /* no tx-confirmation */ + opts.fqd.context_a.hi = 0x80000000 | fman_dealloc_bufs_mask_hi; + opts.fqd.context_a.lo = 0 | fman_dealloc_bufs_mask_lo; + DPAA_PMD_DEBUG("init tx fq %p, fqid %d", fq, fq->fqid); + ret = qman_init_fq(fq, QMAN_INITFQ_FLAG_SCHED, &opts); + if (ret) + DPAA_PMD_ERR("init tx fqid %d failed %d", fq->fqid, ret); + return ret; +} + +#ifdef RTE_LIBRTE_DPAA_DEBUG_DRIVER +/* Initialise a DEBUG FQ ([rt]x_error, rx_default). */ +static int dpaa_debug_queue_init(struct qman_fq *fq, uint32_t fqid) +{ + struct qm_mcc_initfq opts = {0}; + int ret; + + PMD_INIT_FUNC_TRACE(); + + ret = qman_reserve_fqid(fqid); + if (ret) { + DPAA_PMD_ERR("Reserve debug fqid %d failed with ret: %d", + fqid, ret); + return -EINVAL; + } + /* "map" this Rx FQ to one of the interfaces Tx FQID */ + DPAA_PMD_DEBUG("Creating debug fq %p, fqid %d", fq, fqid); + ret = qman_create_fq(fqid, QMAN_FQ_FLAG_NO_ENQUEUE, fq); + if (ret) { + DPAA_PMD_ERR("create debug fqid %d failed with ret: %d", + fqid, ret); + return ret; + } + opts.we_mask = QM_INITFQ_WE_DESTWQ | QM_INITFQ_WE_FQCTRL; + opts.fqd.dest.wq = DPAA_IF_DEBUG_PRIORITY; + ret = qman_init_fq(fq, 0, &opts); + if (ret) + DPAA_PMD_ERR("init debug fqid %d failed with ret: %d", + fqid, ret); + return ret; +} +#endif + +/* Initialise a network interface */ +static int +dpaa_dev_init(struct rte_eth_dev *eth_dev) +{ + int num_cores, num_rx_fqs, fqid; + int loop, ret = 0; + int dev_id; + struct rte_dpaa_device *dpaa_device; + struct dpaa_if *dpaa_intf; + struct fm_eth_port_cfg *cfg; + struct fman_if *fman_intf; + struct fman_if_bpool *bp, *tmp_bp; + + PMD_INIT_FUNC_TRACE(); + + /* For secondary processes, the primary has done all the work */ + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return 0; + + dpaa_device = DEV_TO_DPAA_DEVICE(eth_dev->device); + dev_id = dpaa_device->id.dev_id; + dpaa_intf = eth_dev->data->dev_private; + cfg = &dpaa_netcfg->port_cfg[dev_id]; + fman_intf = cfg->fman_if; + + dpaa_intf->name = dpaa_device->name; + + /* save fman_if & cfg in the interface struture */ + dpaa_intf->fif = fman_intf; + dpaa_intf->ifid = dev_id; + dpaa_intf->cfg = cfg; + + /* Initialize Rx FQ's */ + if (getenv("DPAA_NUM_RX_QUEUES")) + num_rx_fqs = atoi(getenv("DPAA_NUM_RX_QUEUES")); + else + num_rx_fqs = DPAA_DEFAULT_NUM_PCD_QUEUES; + + /* Each device can not have more than DPAA_PCD_FQID_MULTIPLIER RX + * queues. + */ + if (num_rx_fqs <= 0 || num_rx_fqs > DPAA_PCD_FQID_MULTIPLIER) { + DPAA_PMD_ERR("Invalid number of RX queues\n"); + return -EINVAL; + } + + dpaa_intf->rx_queues = rte_zmalloc(NULL, + sizeof(struct qman_fq) * num_rx_fqs, MAX_CACHELINE); + if (!dpaa_intf->rx_queues) { + DPAA_PMD_ERR("Failed to alloc mem for RX queues\n"); + return -ENOMEM; + } + + for (loop = 0; loop < num_rx_fqs; loop++) { + fqid = DPAA_PCD_FQID_START + dpaa_intf->ifid * + DPAA_PCD_FQID_MULTIPLIER + loop; + ret = dpaa_rx_queue_init(&dpaa_intf->rx_queues[loop], fqid); + if (ret) + goto free_rx; + dpaa_intf->rx_queues[loop].dpaa_intf = dpaa_intf; + } + dpaa_intf->nb_rx_queues = num_rx_fqs; + + /* Initialise Tx FQs. Have as many Tx FQ's as number of cores */ + num_cores = rte_lcore_count(); + dpaa_intf->tx_queues = rte_zmalloc(NULL, sizeof(struct qman_fq) * + num_cores, MAX_CACHELINE); + if (!dpaa_intf->tx_queues) { + DPAA_PMD_ERR("Failed to alloc mem for TX queues\n"); + ret = -ENOMEM; + goto free_rx; + } + + for (loop = 0; loop < num_cores; loop++) { + ret = dpaa_tx_queue_init(&dpaa_intf->tx_queues[loop], + fman_intf); + if (ret) + goto free_tx; + dpaa_intf->tx_queues[loop].dpaa_intf = dpaa_intf; + } + dpaa_intf->nb_tx_queues = num_cores; + +#ifdef RTE_LIBRTE_DPAA_DEBUG_DRIVER + dpaa_debug_queue_init(&dpaa_intf->debug_queues[ + DPAA_DEBUG_FQ_RX_ERROR], fman_intf->fqid_rx_err); + dpaa_intf->debug_queues[DPAA_DEBUG_FQ_RX_ERROR].dpaa_intf = dpaa_intf; + dpaa_debug_queue_init(&dpaa_intf->debug_queues[ + DPAA_DEBUG_FQ_TX_ERROR], fman_intf->fqid_tx_err); + dpaa_intf->debug_queues[DPAA_DEBUG_FQ_TX_ERROR].dpaa_intf = dpaa_intf; +#endif + + DPAA_PMD_DEBUG("All frame queues created"); + + /* Get the initial configuration for flow control */ + dpaa_fc_set_default(dpaa_intf); + + /* reset bpool list, initialize bpool dynamically */ + list_for_each_entry_safe(bp, tmp_bp, &cfg->fman_if->bpool_list, node) { + list_del(&bp->node); + free(bp); + } + + /* Populate ethdev structure */ + eth_dev->dev_ops = &dpaa_devops; + eth_dev->rx_pkt_burst = dpaa_eth_queue_rx; + eth_dev->tx_pkt_burst = dpaa_eth_tx_drop_all; + + /* Allocate memory for storing MAC addresses */ + eth_dev->data->mac_addrs = rte_zmalloc("mac_addr", + ETHER_ADDR_LEN * DPAA_MAX_MAC_FILTER, 0); + if (eth_dev->data->mac_addrs == NULL) { + DPAA_PMD_ERR("Failed to allocate %d bytes needed to " + "store MAC addresses", + ETHER_ADDR_LEN * DPAA_MAX_MAC_FILTER); + ret = -ENOMEM; + goto free_tx; + } + + /* copy the primary mac address */ + ether_addr_copy(&fman_intf->mac_addr, ð_dev->data->mac_addrs[0]); + + RTE_LOG(INFO, PMD, "net: dpaa: %s: %02x:%02x:%02x:%02x:%02x:%02x\n", + dpaa_device->name, + fman_intf->mac_addr.addr_bytes[0], + fman_intf->mac_addr.addr_bytes[1], + fman_intf->mac_addr.addr_bytes[2], + fman_intf->mac_addr.addr_bytes[3], + fman_intf->mac_addr.addr_bytes[4], + fman_intf->mac_addr.addr_bytes[5]); + + /* Disable RX mode */ + fman_if_discard_rx_errors(fman_intf); + fman_if_disable_rx(fman_intf); + /* Disable promiscuous mode */ + fman_if_promiscuous_disable(fman_intf); + /* Disable multicast */ + fman_if_reset_mcast_filter_table(fman_intf); + /* Reset interface statistics */ + fman_if_stats_reset(fman_intf); + + return 0; + +free_tx: + rte_free(dpaa_intf->tx_queues); + dpaa_intf->tx_queues = NULL; + dpaa_intf->nb_tx_queues = 0; + +free_rx: + rte_free(dpaa_intf->rx_queues); + dpaa_intf->rx_queues = NULL; + dpaa_intf->nb_rx_queues = 0; + return ret; +} + +static int +dpaa_dev_uninit(struct rte_eth_dev *dev) +{ + struct dpaa_if *dpaa_intf = dev->data->dev_private; + + PMD_INIT_FUNC_TRACE(); + + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return -EPERM; + + if (!dpaa_intf) { + DPAA_PMD_WARN("Already closed or not started"); + return -1; + } + + dpaa_eth_dev_close(dev); + + /* release configuration memory */ + if (dpaa_intf->fc_conf) + rte_free(dpaa_intf->fc_conf); + + rte_free(dpaa_intf->rx_queues); + dpaa_intf->rx_queues = NULL; + + rte_free(dpaa_intf->tx_queues); + dpaa_intf->tx_queues = NULL; + + /* free memory for storing MAC addresses */ + rte_free(dev->data->mac_addrs); + dev->data->mac_addrs = NULL; + + dev->dev_ops = NULL; + dev->rx_pkt_burst = NULL; + dev->tx_pkt_burst = NULL; + + return 0; +} + +static int +rte_dpaa_probe(struct rte_dpaa_driver *dpaa_drv, + struct rte_dpaa_device *dpaa_dev) +{ + int diag; + int ret; + struct rte_eth_dev *eth_dev; + + PMD_INIT_FUNC_TRACE(); + + /* In case of secondary process, the device is already configured + * and no further action is required, except portal initialization + * and verifying secondary attachment to port name. + */ + if (rte_eal_process_type() != RTE_PROC_PRIMARY) { + eth_dev = rte_eth_dev_attach_secondary(dpaa_dev->name); + if (!eth_dev) + return -ENOMEM; + return 0; + } + + if (!is_global_init) { + /* One time load of Qman/Bman drivers */ + ret = qman_global_init(); + if (ret) { + DPAA_PMD_ERR("QMAN initialization failed: %d", + ret); + return ret; + } + ret = bman_global_init(); + if (ret) { + DPAA_PMD_ERR("BMAN initialization failed: %d", + ret); + return ret; + } + + is_global_init = 1; + } + + ret = rte_dpaa_portal_init((void *)1); + if (ret) { + DPAA_PMD_ERR("Unable to initialize portal"); + return ret; + } + + eth_dev = rte_eth_dev_allocate(dpaa_dev->name); + if (eth_dev == NULL) + return -ENOMEM; + + eth_dev->data->dev_private = rte_zmalloc( + "ethdev private structure", + sizeof(struct dpaa_if), + RTE_CACHE_LINE_SIZE); + if (!eth_dev->data->dev_private) { + DPAA_PMD_ERR("Cannot allocate memzone for port data"); + rte_eth_dev_release_port(eth_dev); + return -ENOMEM; + } + + eth_dev->device = &dpaa_dev->device; + eth_dev->device->driver = &dpaa_drv->driver; + dpaa_dev->eth_dev = eth_dev; + + /* Invoke PMD device initialization function */ + diag = dpaa_dev_init(eth_dev); + if (diag == 0) + return 0; + + if (rte_eal_process_type() == RTE_PROC_PRIMARY) + rte_free(eth_dev->data->dev_private); + + rte_eth_dev_release_port(eth_dev); + return diag; +} + +static int +rte_dpaa_remove(struct rte_dpaa_device *dpaa_dev) +{ + struct rte_eth_dev *eth_dev; + + PMD_INIT_FUNC_TRACE(); + + eth_dev = dpaa_dev->eth_dev; + dpaa_dev_uninit(eth_dev); + + if (rte_eal_process_type() == RTE_PROC_PRIMARY) + rte_free(eth_dev->data->dev_private); + + rte_eth_dev_release_port(eth_dev); + + return 0; +} + +static struct rte_dpaa_driver rte_dpaa_pmd = { + .drv_type = FSL_DPAA_ETH, + .probe = rte_dpaa_probe, + .remove = rte_dpaa_remove, +}; + +RTE_PMD_REGISTER_DPAA(net_dpaa, rte_dpaa_pmd); diff --git a/dpdk/drivers/net/dpaa/dpaa_ethdev.h b/dpdk/drivers/net/dpaa/dpaa_ethdev.h new file mode 100644 index 00000000..ec5ae13d --- /dev/null +++ b/dpdk/drivers/net/dpaa/dpaa_ethdev.h @@ -0,0 +1,182 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2014-2016 Freescale Semiconductor, Inc. All rights reserved. + * Copyright 2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Freescale Semiconductor, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef __DPAA_ETHDEV_H__ +#define __DPAA_ETHDEV_H__ + +/* System headers */ +#include +#include + +#include +#include +#include +#include +#include + +/* DPAA SoC identifier; If this is not available, it can be concluded + * that board is non-DPAA. Single slot is currently supported. + */ +#define DPAA_SOC_ID_FILE "/sys/devices/soc0/soc_id" + +#define DPAA_MBUF_HW_ANNOTATION 64 +#define DPAA_FD_PTA_SIZE 64 + +#if (DPAA_MBUF_HW_ANNOTATION + DPAA_FD_PTA_SIZE) > RTE_PKTMBUF_HEADROOM +#error "Annotation requirement is more than RTE_PKTMBUF_HEADROOM" +#endif + +/* we will re-use the HEADROOM for annotation in RX */ +#define DPAA_HW_BUF_RESERVE 0 +#define DPAA_PACKET_LAYOUT_ALIGN 64 + +/* Alignment to use for cpu-local structs to avoid coherency problems. */ +#define MAX_CACHELINE 64 + +#define DPAA_MIN_RX_BUF_SIZE 512 +#define DPAA_MAX_RX_PKT_LEN 10240 + +/* RX queue tail drop threshold + * currently considering 32 KB packets. + */ +#define CONG_THRESHOLD_RX_Q (32 * 1024) + +/*max mac filter for memac(8) including primary mac addr*/ +#define DPAA_MAX_MAC_FILTER (MEMAC_NUM_OF_PADDRS + 1) + +/*Maximum number of slots available in TX ring*/ +#define MAX_TX_RING_SLOTS 8 + +/* PCD frame queues */ +#define DPAA_PCD_FQID_START 0x400 +#define DPAA_PCD_FQID_MULTIPLIER 0x100 +#define DPAA_DEFAULT_NUM_PCD_QUEUES 1 + +#define DPAA_IF_TX_PRIORITY 3 +#define DPAA_IF_RX_PRIORITY 4 +#define DPAA_IF_DEBUG_PRIORITY 7 + +#define DPAA_IF_RX_ANNOTATION_STASH 1 +#define DPAA_IF_RX_DATA_STASH 1 +#define DPAA_IF_RX_CONTEXT_STASH 0 + +/* Each "debug" FQ is represented by one of these */ +#define DPAA_DEBUG_FQ_RX_ERROR 0 +#define DPAA_DEBUG_FQ_TX_ERROR 1 + +#define DPAA_RSS_OFFLOAD_ALL ( \ + ETH_RSS_FRAG_IPV4 | \ + ETH_RSS_NONFRAG_IPV4_TCP | \ + ETH_RSS_NONFRAG_IPV4_UDP | \ + ETH_RSS_NONFRAG_IPV4_SCTP | \ + ETH_RSS_FRAG_IPV6 | \ + ETH_RSS_NONFRAG_IPV6_TCP | \ + ETH_RSS_NONFRAG_IPV6_UDP | \ + ETH_RSS_NONFRAG_IPV6_SCTP) + +#define DPAA_TX_CKSUM_OFFLOAD_MASK ( \ + PKT_TX_IP_CKSUM | \ + PKT_TX_TCP_CKSUM | \ + PKT_TX_UDP_CKSUM) + +/* DPAA Frame descriptor macros */ + +#define DPAA_FD_CMD_FCO 0x80000000 +/**< Frame queue Context Override */ +#define DPAA_FD_CMD_RPD 0x40000000 +/**< Read Prepended Data */ +#define DPAA_FD_CMD_UPD 0x20000000 +/**< Update Prepended Data */ +#define DPAA_FD_CMD_DTC 0x10000000 +/**< Do IP/TCP/UDP Checksum */ +#define DPAA_FD_CMD_DCL4C 0x10000000 +/**< Didn't calculate L4 Checksum */ +#define DPAA_FD_CMD_CFQ 0x00ffffff +/**< Confirmation Frame Queue */ + +/* Each network interface is represented by one of these */ +struct dpaa_if { + int valid; + char *name; + const struct fm_eth_port_cfg *cfg; + struct qman_fq *rx_queues; + struct qman_fq *tx_queues; + struct qman_fq debug_queues[2]; + uint16_t nb_rx_queues; + uint16_t nb_tx_queues; + uint32_t ifid; + struct fman_if *fif; + struct dpaa_bp_info *bp_info; + struct rte_eth_fc_conf *fc_conf; +}; + +struct dpaa_if_stats { + /* Rx Statistics Counter */ + uint64_t reoct; /** +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "dpaa_ethdev.h" +#include "dpaa_rxtx.h" +#include +#include + +#include +#include +#include +#include +#include + +#define DPAA_MBUF_TO_CONTIG_FD(_mbuf, _fd, _bpid) \ + do { \ + (_fd)->cmd = 0; \ + (_fd)->opaque_addr = 0; \ + (_fd)->opaque = QM_FD_CONTIG << DPAA_FD_FORMAT_SHIFT; \ + (_fd)->opaque |= ((_mbuf)->data_off) << DPAA_FD_OFFSET_SHIFT; \ + (_fd)->opaque |= (_mbuf)->pkt_len; \ + (_fd)->addr = (_mbuf)->buf_iova; \ + (_fd)->bpid = _bpid; \ + } while (0) + +#if (defined RTE_LIBRTE_DPAA_DEBUG_DRIVER) +void dpaa_display_frame(const struct qm_fd *fd) +{ + int ii; + char *ptr; + + printf("%s::bpid %x addr %08x%08x, format %d off %d, len %d stat %x\n", + __func__, fd->bpid, fd->addr_hi, fd->addr_lo, fd->format, + fd->offset, fd->length20, fd->status); + + ptr = (char *)rte_dpaa_mem_ptov(fd->addr); + ptr += fd->offset; + printf("%02x ", *ptr); + for (ii = 1; ii < fd->length20; ii++) { + printf("%02x ", *ptr); + if ((ii % 16) == 0) + printf("\n"); + ptr++; + } + printf("\n"); +} +#else +#define dpaa_display_frame(a) +#endif + +static inline void dpaa_slow_parsing(struct rte_mbuf *m __rte_unused, + uint64_t prs __rte_unused) +{ + DPAA_DP_LOG(DEBUG, "Slow parsing"); + /*TBD:XXX: to be implemented*/ +} + +static inline void dpaa_eth_packet_info(struct rte_mbuf *m, + uint64_t fd_virt_addr) +{ + struct annotations_t *annot = GET_ANNOTATIONS(fd_virt_addr); + uint64_t prs = *((uint64_t *)(&annot->parse)) & DPAA_PARSE_MASK; + + DPAA_DP_LOG(DEBUG, " Parsing mbuf: %p with annotations: %p", m, annot); + + switch (prs) { + case DPAA_PKT_TYPE_NONE: + m->packet_type = 0; + break; + case DPAA_PKT_TYPE_ETHER: + m->packet_type = RTE_PTYPE_L2_ETHER; + break; + case DPAA_PKT_TYPE_IPV4: + m->packet_type = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4; + break; + case DPAA_PKT_TYPE_IPV6: + m->packet_type = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV6; + break; + case DPAA_PKT_TYPE_IPV4_FRAG: + case DPAA_PKT_TYPE_IPV4_FRAG_UDP: + case DPAA_PKT_TYPE_IPV4_FRAG_TCP: + case DPAA_PKT_TYPE_IPV4_FRAG_SCTP: + m->packet_type = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_FRAG; + break; + case DPAA_PKT_TYPE_IPV6_FRAG: + case DPAA_PKT_TYPE_IPV6_FRAG_UDP: + case DPAA_PKT_TYPE_IPV6_FRAG_TCP: + case DPAA_PKT_TYPE_IPV6_FRAG_SCTP: + m->packet_type = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_FRAG; + break; + case DPAA_PKT_TYPE_IPV4_EXT: + m->packet_type = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT; + break; + case DPAA_PKT_TYPE_IPV6_EXT: + m->packet_type = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV6_EXT; + break; + case DPAA_PKT_TYPE_IPV4_TCP: + m->packet_type = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_TCP; + break; + case DPAA_PKT_TYPE_IPV6_TCP: + m->packet_type = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_TCP; + break; + case DPAA_PKT_TYPE_IPV4_UDP: + m->packet_type = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_UDP; + break; + case DPAA_PKT_TYPE_IPV6_UDP: + m->packet_type = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_UDP; + break; + case DPAA_PKT_TYPE_IPV4_EXT_UDP: + m->packet_type = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_UDP; + break; + case DPAA_PKT_TYPE_IPV6_EXT_UDP: + m->packet_type = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_UDP; + break; + case DPAA_PKT_TYPE_IPV4_EXT_TCP: + m->packet_type = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_TCP; + break; + case DPAA_PKT_TYPE_IPV6_EXT_TCP: + m->packet_type = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_TCP; + break; + case DPAA_PKT_TYPE_IPV4_SCTP: + m->packet_type = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_SCTP; + break; + case DPAA_PKT_TYPE_IPV6_SCTP: + m->packet_type = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_SCTP; + break; + /* More switch cases can be added */ + default: + dpaa_slow_parsing(m, prs); + } + + m->tx_offload = annot->parse.ip_off[0]; + m->tx_offload |= (annot->parse.l4_off - annot->parse.ip_off[0]) + << DPAA_PKT_L3_LEN_SHIFT; + + /* Set the hash values */ + m->hash.rss = (uint32_t)(rte_be_to_cpu_64(annot->hash)); + m->ol_flags = PKT_RX_RSS_HASH; + /* All packets with Bad checksum are dropped by interface (and + * corresponding notification issued to RX error queues). + */ + m->ol_flags |= PKT_RX_IP_CKSUM_GOOD; + + /* Check if Vlan is present */ + if (prs & DPAA_PARSE_VLAN_MASK) + m->ol_flags |= PKT_RX_VLAN; + /* Packet received without stripping the vlan */ +} + +static inline void dpaa_checksum(struct rte_mbuf *mbuf) +{ + struct ether_hdr *eth_hdr = rte_pktmbuf_mtod(mbuf, struct ether_hdr *); + char *l3_hdr = (char *)eth_hdr + mbuf->l2_len; + struct ipv4_hdr *ipv4_hdr = (struct ipv4_hdr *)l3_hdr; + struct ipv6_hdr *ipv6_hdr = (struct ipv6_hdr *)l3_hdr; + + DPAA_DP_LOG(DEBUG, "Calculating checksum for mbuf: %p", mbuf); + + if (((mbuf->packet_type & RTE_PTYPE_L3_MASK) == RTE_PTYPE_L3_IPV4) || + ((mbuf->packet_type & RTE_PTYPE_L3_MASK) == + RTE_PTYPE_L3_IPV4_EXT)) { + ipv4_hdr = (struct ipv4_hdr *)l3_hdr; + ipv4_hdr->hdr_checksum = 0; + ipv4_hdr->hdr_checksum = rte_ipv4_cksum(ipv4_hdr); + } else if (((mbuf->packet_type & RTE_PTYPE_L3_MASK) == + RTE_PTYPE_L3_IPV6) || + ((mbuf->packet_type & RTE_PTYPE_L3_MASK) == + RTE_PTYPE_L3_IPV6_EXT)) + ipv6_hdr = (struct ipv6_hdr *)l3_hdr; + + if ((mbuf->packet_type & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_TCP) { + struct tcp_hdr *tcp_hdr = (struct tcp_hdr *)(l3_hdr + + mbuf->l3_len); + tcp_hdr->cksum = 0; + if (eth_hdr->ether_type == htons(ETHER_TYPE_IPv4)) + tcp_hdr->cksum = rte_ipv4_udptcp_cksum(ipv4_hdr, + tcp_hdr); + else /* assume ethertype == ETHER_TYPE_IPv6 */ + tcp_hdr->cksum = rte_ipv6_udptcp_cksum(ipv6_hdr, + tcp_hdr); + } else if ((mbuf->packet_type & RTE_PTYPE_L4_MASK) == + RTE_PTYPE_L4_UDP) { + struct udp_hdr *udp_hdr = (struct udp_hdr *)(l3_hdr + + mbuf->l3_len); + udp_hdr->dgram_cksum = 0; + if (eth_hdr->ether_type == htons(ETHER_TYPE_IPv4)) + udp_hdr->dgram_cksum = rte_ipv4_udptcp_cksum(ipv4_hdr, + udp_hdr); + else /* assume ethertype == ETHER_TYPE_IPv6 */ + udp_hdr->dgram_cksum = rte_ipv6_udptcp_cksum(ipv6_hdr, + udp_hdr); + } +} + +static inline void dpaa_checksum_offload(struct rte_mbuf *mbuf, + struct qm_fd *fd, char *prs_buf) +{ + struct dpaa_eth_parse_results_t *prs; + + DPAA_DP_LOG(DEBUG, " Offloading checksum for mbuf: %p", mbuf); + + prs = GET_TX_PRS(prs_buf); + prs->l3r = 0; + prs->l4r = 0; + if (((mbuf->packet_type & RTE_PTYPE_L3_MASK) == RTE_PTYPE_L3_IPV4) || + ((mbuf->packet_type & RTE_PTYPE_L3_MASK) == + RTE_PTYPE_L3_IPV4_EXT)) + prs->l3r = DPAA_L3_PARSE_RESULT_IPV4; + else if (((mbuf->packet_type & RTE_PTYPE_L3_MASK) == + RTE_PTYPE_L3_IPV6) || + ((mbuf->packet_type & RTE_PTYPE_L3_MASK) == + RTE_PTYPE_L3_IPV6_EXT)) + prs->l3r = DPAA_L3_PARSE_RESULT_IPV6; + + if ((mbuf->packet_type & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_TCP) + prs->l4r = DPAA_L4_PARSE_RESULT_TCP; + else if ((mbuf->packet_type & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_UDP) + prs->l4r = DPAA_L4_PARSE_RESULT_UDP; + + prs->ip_off[0] = mbuf->l2_len; + prs->l4_off = mbuf->l3_len + mbuf->l2_len; + /* Enable L3 (and L4, if TCP or UDP) HW checksum*/ + fd->cmd = DPAA_FD_CMD_RPD | DPAA_FD_CMD_DTC; +} + +struct rte_mbuf * +dpaa_eth_sg_to_mbuf(struct qm_fd *fd, uint32_t ifid) +{ + struct dpaa_bp_info *bp_info = DPAA_BPID_TO_POOL_INFO(fd->bpid); + struct rte_mbuf *first_seg, *prev_seg, *cur_seg, *temp; + struct qm_sg_entry *sgt, *sg_temp; + void *vaddr, *sg_vaddr; + int i = 0; + uint8_t fd_offset = fd->offset; + + DPAA_DP_LOG(DEBUG, "Received an SG frame"); + + vaddr = rte_dpaa_mem_ptov(qm_fd_addr(fd)); + if (!vaddr) { + DPAA_PMD_ERR("unable to convert physical address"); + return NULL; + } + sgt = vaddr + fd_offset; + sg_temp = &sgt[i++]; + hw_sg_to_cpu(sg_temp); + temp = (struct rte_mbuf *)((char *)vaddr - bp_info->meta_data_size); + sg_vaddr = rte_dpaa_mem_ptov(qm_sg_entry_get64(sg_temp)); + + first_seg = (struct rte_mbuf *)((char *)sg_vaddr - + bp_info->meta_data_size); + first_seg->data_off = sg_temp->offset; + first_seg->data_len = sg_temp->length; + first_seg->pkt_len = sg_temp->length; + rte_mbuf_refcnt_set(first_seg, 1); + + first_seg->port = ifid; + first_seg->nb_segs = 1; + first_seg->ol_flags = 0; + prev_seg = first_seg; + while (i < DPAA_SGT_MAX_ENTRIES) { + sg_temp = &sgt[i++]; + hw_sg_to_cpu(sg_temp); + sg_vaddr = rte_dpaa_mem_ptov(qm_sg_entry_get64(sg_temp)); + cur_seg = (struct rte_mbuf *)((char *)sg_vaddr - + bp_info->meta_data_size); + cur_seg->data_off = sg_temp->offset; + cur_seg->data_len = sg_temp->length; + first_seg->pkt_len += sg_temp->length; + first_seg->nb_segs += 1; + rte_mbuf_refcnt_set(cur_seg, 1); + prev_seg->next = cur_seg; + if (sg_temp->final) { + cur_seg->next = NULL; + break; + } + prev_seg = cur_seg; + } + + dpaa_eth_packet_info(first_seg, (uint64_t)vaddr); + rte_pktmbuf_free_seg(temp); + + return first_seg; +} + +static inline struct rte_mbuf *dpaa_eth_fd_to_mbuf(struct qm_fd *fd, + uint32_t ifid) +{ + struct dpaa_bp_info *bp_info = DPAA_BPID_TO_POOL_INFO(fd->bpid); + struct rte_mbuf *mbuf; + void *ptr; + uint8_t format = + (fd->opaque & DPAA_FD_FORMAT_MASK) >> DPAA_FD_FORMAT_SHIFT; + uint16_t offset = + (fd->opaque & DPAA_FD_OFFSET_MASK) >> DPAA_FD_OFFSET_SHIFT; + uint32_t length = fd->opaque & DPAA_FD_LENGTH_MASK; + + DPAA_DP_LOG(DEBUG, " FD--->MBUF"); + + if (unlikely(format == qm_fd_sg)) + return dpaa_eth_sg_to_mbuf(fd, ifid); + + /* Ignoring case when format != qm_fd_contig */ + dpaa_display_frame(fd); + ptr = rte_dpaa_mem_ptov(fd->addr); + /* Ignoring case when ptr would be NULL. That is only possible incase + * of a corrupted packet + */ + + mbuf = (struct rte_mbuf *)((char *)ptr - bp_info->meta_data_size); + /* Prefetch the Parse results and packet data to L1 */ + rte_prefetch0((void *)((uint8_t *)ptr + DEFAULT_RX_ICEOF)); + rte_prefetch0((void *)((uint8_t *)ptr + offset)); + + mbuf->data_off = offset; + mbuf->data_len = length; + mbuf->pkt_len = length; + + mbuf->port = ifid; + mbuf->nb_segs = 1; + mbuf->ol_flags = 0; + mbuf->next = NULL; + rte_mbuf_refcnt_set(mbuf, 1); + dpaa_eth_packet_info(mbuf, (uint64_t)mbuf->buf_addr); + + return mbuf; +} + +uint16_t dpaa_eth_queue_rx(void *q, + struct rte_mbuf **bufs, + uint16_t nb_bufs) +{ + struct qman_fq *fq = q; + struct qm_dqrr_entry *dq; + uint32_t num_rx = 0, ifid = ((struct dpaa_if *)fq->dpaa_intf)->ifid; + int ret; + + ret = rte_dpaa_portal_init((void *)0); + if (ret) { + DPAA_PMD_ERR("Failure in affining portal"); + return 0; + } + + ret = qman_set_vdq(fq, (nb_bufs > DPAA_MAX_DEQUEUE_NUM_FRAMES) ? + DPAA_MAX_DEQUEUE_NUM_FRAMES : nb_bufs); + if (ret) + return 0; + + do { + dq = qman_dequeue(fq); + if (!dq) + continue; + bufs[num_rx++] = dpaa_eth_fd_to_mbuf(&dq->fd, ifid); + qman_dqrr_consume(fq, dq); + } while (fq->flags & QMAN_FQ_STATE_VDQCR); + + return num_rx; +} + +static void *dpaa_get_pktbuf(struct dpaa_bp_info *bp_info) +{ + int ret; + uint64_t buf = 0; + struct bm_buffer bufs; + + ret = bman_acquire(bp_info->bp, &bufs, 1, 0); + if (ret <= 0) { + DPAA_PMD_WARN("Failed to allocate buffers %d", ret); + return (void *)buf; + } + + DPAA_DP_LOG(DEBUG, "got buffer 0x%lx from pool %d", + (uint64_t)bufs.addr, bufs.bpid); + + buf = (uint64_t)rte_dpaa_mem_ptov(bufs.addr) - bp_info->meta_data_size; + if (!buf) + goto out; + +out: + return (void *)buf; +} + +static struct rte_mbuf *dpaa_get_dmable_mbuf(struct rte_mbuf *mbuf, + struct dpaa_if *dpaa_intf) +{ + struct rte_mbuf *dpaa_mbuf; + + /* allocate pktbuffer on bpid for dpaa port */ + dpaa_mbuf = dpaa_get_pktbuf(dpaa_intf->bp_info); + if (!dpaa_mbuf) + return NULL; + + memcpy((uint8_t *)(dpaa_mbuf->buf_addr) + mbuf->data_off, (void *) + ((uint8_t *)(mbuf->buf_addr) + mbuf->data_off), mbuf->pkt_len); + + /* Copy only the required fields */ + dpaa_mbuf->data_off = mbuf->data_off; + dpaa_mbuf->pkt_len = mbuf->pkt_len; + dpaa_mbuf->ol_flags = mbuf->ol_flags; + dpaa_mbuf->packet_type = mbuf->packet_type; + dpaa_mbuf->tx_offload = mbuf->tx_offload; + rte_pktmbuf_free(mbuf); + return dpaa_mbuf; +} + +int +dpaa_eth_mbuf_to_sg_fd(struct rte_mbuf *mbuf, + struct qm_fd *fd, + uint32_t bpid) +{ + struct rte_mbuf *cur_seg = mbuf, *prev_seg = NULL; + struct dpaa_bp_info *bp_info = DPAA_BPID_TO_POOL_INFO(bpid); + struct rte_mbuf *temp, *mi; + struct qm_sg_entry *sg_temp, *sgt; + int i = 0; + + DPAA_DP_LOG(DEBUG, "Creating SG FD to transmit"); + + temp = rte_pktmbuf_alloc(bp_info->mp); + if (!temp) { + DPAA_PMD_ERR("Failure in allocation of mbuf"); + return -1; + } + if (temp->buf_len < ((mbuf->nb_segs * sizeof(struct qm_sg_entry)) + + temp->data_off)) { + DPAA_PMD_ERR("Insufficient space in mbuf for SG entries"); + return -1; + } + + fd->cmd = 0; + fd->opaque_addr = 0; + + if (mbuf->ol_flags & DPAA_TX_CKSUM_OFFLOAD_MASK) { + if (!mbuf->packet_type) { + struct rte_net_hdr_lens hdr_lens; + + mbuf->packet_type = rte_net_get_ptype(mbuf, &hdr_lens, + RTE_PTYPE_L2_MASK | RTE_PTYPE_L3_MASK + | RTE_PTYPE_L4_MASK); + mbuf->l2_len = hdr_lens.l2_len; + mbuf->l3_len = hdr_lens.l3_len; + } + if (temp->data_off < DEFAULT_TX_ICEOF + + sizeof(struct dpaa_eth_parse_results_t)) + temp->data_off = DEFAULT_TX_ICEOF + + sizeof(struct dpaa_eth_parse_results_t); + dcbz_64(temp->buf_addr); + dpaa_checksum_offload(mbuf, fd, temp->buf_addr); + } + + sgt = temp->buf_addr + temp->data_off; + fd->format = QM_FD_SG; + fd->addr = temp->buf_iova; + fd->offset = temp->data_off; + fd->bpid = bpid; + fd->length20 = mbuf->pkt_len; + + while (i < DPAA_SGT_MAX_ENTRIES) { + sg_temp = &sgt[i++]; + sg_temp->opaque = 0; + sg_temp->val = 0; + sg_temp->addr = cur_seg->buf_iova; + sg_temp->offset = cur_seg->data_off; + sg_temp->length = cur_seg->data_len; + if (RTE_MBUF_DIRECT(cur_seg)) { + if (rte_mbuf_refcnt_read(cur_seg) > 1) { + /*If refcnt > 1, invalid bpid is set to ensure + * buffer is not freed by HW. + */ + sg_temp->bpid = 0xff; + rte_mbuf_refcnt_update(cur_seg, -1); + } else { + sg_temp->bpid = + DPAA_MEMPOOL_TO_BPID(cur_seg->pool); + } + cur_seg = cur_seg->next; + } else { + /* Get owner MBUF from indirect buffer */ + mi = rte_mbuf_from_indirect(cur_seg); + if (rte_mbuf_refcnt_read(mi) > 1) { + /*If refcnt > 1, invalid bpid is set to ensure + * owner buffer is not freed by HW. + */ + sg_temp->bpid = 0xff; + } else { + sg_temp->bpid = DPAA_MEMPOOL_TO_BPID(mi->pool); + rte_mbuf_refcnt_update(mi, 1); + } + prev_seg = cur_seg; + cur_seg = cur_seg->next; + prev_seg->next = NULL; + rte_pktmbuf_free(prev_seg); + } + if (cur_seg == NULL) { + sg_temp->final = 1; + cpu_to_hw_sg(sg_temp); + break; + } + cpu_to_hw_sg(sg_temp); + } + return 0; +} + +/* Handle mbufs which are not segmented (non SG) */ +static inline void +tx_on_dpaa_pool_unsegmented(struct rte_mbuf *mbuf, + struct dpaa_bp_info *bp_info, + struct qm_fd *fd_arr) +{ + struct rte_mbuf *mi = NULL; + + if (RTE_MBUF_DIRECT(mbuf)) { + if (rte_mbuf_refcnt_read(mbuf) > 1) { + /* In case of direct mbuf and mbuf being cloned, + * BMAN should _not_ release buffer. + */ + DPAA_MBUF_TO_CONTIG_FD(mbuf, fd_arr, 0xff); + /* Buffer should be releasd by EAL */ + rte_mbuf_refcnt_update(mbuf, -1); + } else { + /* In case of direct mbuf and no cloning, mbuf can be + * released by BMAN. + */ + DPAA_MBUF_TO_CONTIG_FD(mbuf, fd_arr, bp_info->bpid); + } + } else { + /* This is data-containing core mbuf: 'mi' */ + mi = rte_mbuf_from_indirect(mbuf); + if (rte_mbuf_refcnt_read(mi) > 1) { + /* In case of indirect mbuf, and mbuf being cloned, + * BMAN should _not_ release it and let EAL release + * it through pktmbuf_free below. + */ + DPAA_MBUF_TO_CONTIG_FD(mbuf, fd_arr, 0xff); + } else { + /* In case of indirect mbuf, and no cloning, core mbuf + * should be released by BMAN. + * Increate refcnt of core mbuf so that when + * pktmbuf_free is called and mbuf is released, EAL + * doesn't try to release core mbuf which would have + * been released by BMAN. + */ + rte_mbuf_refcnt_update(mi, 1); + DPAA_MBUF_TO_CONTIG_FD(mbuf, fd_arr, bp_info->bpid); + } + rte_pktmbuf_free(mbuf); + } + + if (mbuf->ol_flags & DPAA_TX_CKSUM_OFFLOAD_MASK) { + if (!mbuf->packet_type) { + struct rte_net_hdr_lens hdr_lens; + + mbuf->packet_type = rte_net_get_ptype(mbuf, &hdr_lens, + RTE_PTYPE_L2_MASK | RTE_PTYPE_L3_MASK + | RTE_PTYPE_L4_MASK); + mbuf->l2_len = hdr_lens.l2_len; + mbuf->l3_len = hdr_lens.l3_len; + } + if (mbuf->data_off < (DEFAULT_TX_ICEOF + + sizeof(struct dpaa_eth_parse_results_t))) { + DPAA_DP_LOG(DEBUG, "Checksum offload Err: " + "Not enough Headroom " + "space for correct Checksum offload." + "So Calculating checksum in Software."); + dpaa_checksum(mbuf); + } else { + dpaa_checksum_offload(mbuf, fd_arr, mbuf->buf_addr); + } + } +} + +/* Handle all mbufs on dpaa BMAN managed pool */ +static inline uint16_t +tx_on_dpaa_pool(struct rte_mbuf *mbuf, + struct dpaa_bp_info *bp_info, + struct qm_fd *fd_arr) +{ + DPAA_DP_LOG(DEBUG, "BMAN offloaded buffer, mbuf: %p", mbuf); + + if (mbuf->nb_segs == 1) { + /* Case for non-segmented buffers */ + tx_on_dpaa_pool_unsegmented(mbuf, bp_info, fd_arr); + } else if (mbuf->nb_segs > 1 && + mbuf->nb_segs <= DPAA_SGT_MAX_ENTRIES) { + if (dpaa_eth_mbuf_to_sg_fd(mbuf, fd_arr, bp_info->bpid)) { + DPAA_PMD_DEBUG("Unable to create Scatter Gather FD"); + return 1; + } + } else { + DPAA_PMD_DEBUG("Number of Segments not supported"); + return 1; + } + + return 0; +} + +/* Handle all mbufs on an external pool (non-dpaa) */ +static inline uint16_t +tx_on_external_pool(struct qman_fq *txq, struct rte_mbuf *mbuf, + struct qm_fd *fd_arr) +{ + struct dpaa_if *dpaa_intf = txq->dpaa_intf; + struct rte_mbuf *dmable_mbuf; + + DPAA_DP_LOG(DEBUG, "Non-BMAN offloaded buffer." + "Allocating an offloaded buffer"); + dmable_mbuf = dpaa_get_dmable_mbuf(mbuf, dpaa_intf); + if (!dmable_mbuf) { + DPAA_DP_LOG(DEBUG, "no dpaa buffers."); + return 1; + } + + DPAA_MBUF_TO_CONTIG_FD(dmable_mbuf, fd_arr, dpaa_intf->bp_info->bpid); + + return 0; +} + +uint16_t +dpaa_eth_queue_tx(void *q, struct rte_mbuf **bufs, uint16_t nb_bufs) +{ + struct rte_mbuf *mbuf, *mi = NULL; + struct rte_mempool *mp; + struct dpaa_bp_info *bp_info; + struct qm_fd fd_arr[MAX_TX_RING_SLOTS]; + uint32_t frames_to_send, loop, i = 0; + uint16_t state; + int ret; + + ret = rte_dpaa_portal_init((void *)0); + if (ret) { + DPAA_PMD_ERR("Failure in affining portal"); + return 0; + } + + DPAA_DP_LOG(DEBUG, "Transmitting %d buffers on queue: %p", nb_bufs, q); + + while (nb_bufs) { + frames_to_send = (nb_bufs >> 3) ? MAX_TX_RING_SLOTS : nb_bufs; + for (loop = 0; loop < frames_to_send; loop++, i++) { + mbuf = bufs[i]; + if (RTE_MBUF_DIRECT(mbuf)) { + mp = mbuf->pool; + } else { + mi = rte_mbuf_from_indirect(mbuf); + mp = mi->pool; + } + + bp_info = DPAA_MEMPOOL_TO_POOL_INFO(mp); + if (likely(mp->ops_index == bp_info->dpaa_ops_index)) { + state = tx_on_dpaa_pool(mbuf, bp_info, + &fd_arr[loop]); + if (unlikely(state)) { + /* Set frames_to_send & nb_bufs so + * that packets are transmitted till + * previous frame. + */ + frames_to_send = loop; + nb_bufs = loop; + goto send_pkts; + } + } else { + state = tx_on_external_pool(q, mbuf, + &fd_arr[loop]); + if (unlikely(state)) { + /* Set frames_to_send & nb_bufs so + * that packets are transmitted till + * previous frame. + */ + frames_to_send = loop; + nb_bufs = loop; + goto send_pkts; + } + } + } + +send_pkts: + loop = 0; + while (loop < frames_to_send) { + loop += qman_enqueue_multi(q, &fd_arr[loop], + frames_to_send - loop); + } + nb_bufs -= frames_to_send; + } + + DPAA_DP_LOG(DEBUG, "Transmitted %d buffers on queue: %p", i, q); + + return i; +} + +uint16_t dpaa_eth_tx_drop_all(void *q __rte_unused, + struct rte_mbuf **bufs __rte_unused, + uint16_t nb_bufs __rte_unused) +{ + DPAA_DP_LOG(DEBUG, "Drop all packets"); + + /* Drop all incoming packets. No need to free packets here + * because the rte_eth f/w frees up the packets through tx_buffer + * callback in case this functions returns count less than nb_bufs + */ + return 0; +} diff --git a/dpdk/drivers/net/dpaa/dpaa_rxtx.h b/dpdk/drivers/net/dpaa/dpaa_rxtx.h new file mode 100644 index 00000000..2ffc4ffe --- /dev/null +++ b/dpdk/drivers/net/dpaa/dpaa_rxtx.h @@ -0,0 +1,297 @@ +/*- + * BSD LICENSE + * + * Copyright 2016 Freescale Semiconductor, Inc. All rights reserved. + * Copyright 2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Freescale Semiconductor, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __DPDK_RXTX_H__ +#define __DPDK_RXTX_H__ + +/* internal offset from where IC is copied to packet buffer*/ +#define DEFAULT_ICIOF 32 +/* IC transfer size */ +#define DEFAULT_ICSZ 48 + +/* IC offsets from buffer header address */ +#define DEFAULT_RX_ICEOF 16 +#define DEFAULT_TX_ICEOF 16 + +/* + * Values for the L3R field of the FM Parse Results + */ +/* L3 Type field: First IP Present IPv4 */ +#define DPAA_L3_PARSE_RESULT_IPV4 0x80 +/* L3 Type field: First IP Present IPv6 */ +#define DPAA_L3_PARSE_RESULT_IPV6 0x40 +/* Values for the L4R field of the FM Parse Results + * See $8.8.4.7.20 - L4 HXS - L4 Results from DPAA-Rev2 Reference Manual. + */ +/* L4 Type field: UDP */ +#define DPAA_L4_PARSE_RESULT_UDP 0x40 +/* L4 Type field: TCP */ +#define DPAA_L4_PARSE_RESULT_TCP 0x20 + +#define DPAA_SGT_MAX_ENTRIES 16 /* maximum number of entries in SG Table */ + +#define DPAA_MAX_DEQUEUE_NUM_FRAMES 63 + /** +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "../dpaa2_ethdev.h" + +static int +dpaa2_distset_to_dpkg_profile_cfg( + uint64_t req_dist_set, + struct dpkg_profile_cfg *kg_cfg); + +int +dpaa2_setup_flow_dist(struct rte_eth_dev *eth_dev, + uint64_t req_dist_set) +{ + struct dpaa2_dev_priv *priv = eth_dev->data->dev_private; + struct fsl_mc_io *dpni = priv->hw; + struct dpni_rx_tc_dist_cfg tc_cfg; + struct dpkg_profile_cfg kg_cfg; + void *p_params; + int ret, tc_index = 0; + + p_params = rte_malloc( + NULL, DIST_PARAM_IOVA_SIZE, RTE_CACHE_LINE_SIZE); + if (!p_params) { + PMD_INIT_LOG(ERR, "Memory unavailable"); + return -ENOMEM; + } + memset(p_params, 0, DIST_PARAM_IOVA_SIZE); + memset(&tc_cfg, 0, sizeof(struct dpni_rx_tc_dist_cfg)); + + ret = dpaa2_distset_to_dpkg_profile_cfg(req_dist_set, &kg_cfg); + if (ret) { + PMD_INIT_LOG(ERR, "given rss_hf (%lx) not supported", + req_dist_set); + rte_free(p_params); + return ret; + } + tc_cfg.key_cfg_iova = (uint64_t)(DPAA2_VADDR_TO_IOVA(p_params)); + tc_cfg.dist_size = eth_dev->data->nb_rx_queues; + tc_cfg.dist_mode = DPNI_DIST_MODE_HASH; + + ret = dpkg_prepare_key_cfg(&kg_cfg, p_params); + if (ret) { + PMD_INIT_LOG(ERR, "Unable to prepare extract parameters"); + rte_free(p_params); + return ret; + } + + ret = dpni_set_rx_tc_dist(dpni, CMD_PRI_LOW, priv->token, tc_index, + &tc_cfg); + rte_free(p_params); + if (ret) { + PMD_INIT_LOG(ERR, + "Setting distribution for Rx failed with err: %d", + ret); + return ret; + } + + return 0; +} + +int dpaa2_remove_flow_dist( + struct rte_eth_dev *eth_dev, + uint8_t tc_index) +{ + struct dpaa2_dev_priv *priv = eth_dev->data->dev_private; + struct fsl_mc_io *dpni = priv->hw; + struct dpni_rx_tc_dist_cfg tc_cfg; + struct dpkg_profile_cfg kg_cfg; + void *p_params; + int ret; + + p_params = rte_malloc( + NULL, DIST_PARAM_IOVA_SIZE, RTE_CACHE_LINE_SIZE); + if (!p_params) { + PMD_INIT_LOG(ERR, "Memory unavailable"); + return -ENOMEM; + } + memset(p_params, 0, DIST_PARAM_IOVA_SIZE); + memset(&tc_cfg, 0, sizeof(struct dpni_rx_tc_dist_cfg)); + kg_cfg.num_extracts = 0; + tc_cfg.key_cfg_iova = (uint64_t)(DPAA2_VADDR_TO_IOVA(p_params)); + tc_cfg.dist_size = 0; + tc_cfg.dist_mode = DPNI_DIST_MODE_NONE; + + ret = dpkg_prepare_key_cfg(&kg_cfg, p_params); + if (ret) { + PMD_INIT_LOG(ERR, "Unable to prepare extract parameters"); + rte_free(p_params); + return ret; + } + + ret = dpni_set_rx_tc_dist(dpni, CMD_PRI_LOW, priv->token, tc_index, + &tc_cfg); + rte_free(p_params); + if (ret) + PMD_INIT_LOG(ERR, + "Setting distribution for Rx failed with err:%d", + ret); + return ret; +} + +static int +dpaa2_distset_to_dpkg_profile_cfg( + uint64_t req_dist_set, + struct dpkg_profile_cfg *kg_cfg) +{ + uint32_t loop = 0, i = 0, dist_field = 0; + int l2_configured = 0, l3_configured = 0; + int l4_configured = 0, sctp_configured = 0; + + memset(kg_cfg, 0, sizeof(struct dpkg_profile_cfg)); + while (req_dist_set) { + if (req_dist_set % 2 != 0) { + dist_field = 1U << loop; + switch (dist_field) { + case ETH_RSS_L2_PAYLOAD: + + if (l2_configured) + break; + l2_configured = 1; + + kg_cfg->extracts[i].extract.from_hdr.prot = + NET_PROT_ETH; + kg_cfg->extracts[i].extract.from_hdr.field = + NH_FLD_ETH_TYPE; + kg_cfg->extracts[i].type = + DPKG_EXTRACT_FROM_HDR; + kg_cfg->extracts[i].extract.from_hdr.type = + DPKG_FULL_FIELD; + i++; + break; + + case ETH_RSS_IPV4: + case ETH_RSS_FRAG_IPV4: + case ETH_RSS_NONFRAG_IPV4_OTHER: + case ETH_RSS_IPV6: + case ETH_RSS_FRAG_IPV6: + case ETH_RSS_NONFRAG_IPV6_OTHER: + case ETH_RSS_IPV6_EX: + + if (l3_configured) + break; + l3_configured = 1; + + kg_cfg->extracts[i].extract.from_hdr.prot = + NET_PROT_IP; + kg_cfg->extracts[i].extract.from_hdr.field = + NH_FLD_IP_SRC; + kg_cfg->extracts[i].type = + DPKG_EXTRACT_FROM_HDR; + kg_cfg->extracts[i].extract.from_hdr.type = + DPKG_FULL_FIELD; + i++; + + kg_cfg->extracts[i].extract.from_hdr.prot = + NET_PROT_IP; + kg_cfg->extracts[i].extract.from_hdr.field = + NH_FLD_IP_DST; + kg_cfg->extracts[i].type = + DPKG_EXTRACT_FROM_HDR; + kg_cfg->extracts[i].extract.from_hdr.type = + DPKG_FULL_FIELD; + i++; + + kg_cfg->extracts[i].extract.from_hdr.prot = + NET_PROT_IP; + kg_cfg->extracts[i].extract.from_hdr.field = + NH_FLD_IP_PROTO; + kg_cfg->extracts[i].type = + DPKG_EXTRACT_FROM_HDR; + kg_cfg->extracts[i].extract.from_hdr.type = + DPKG_FULL_FIELD; + kg_cfg->num_extracts++; + i++; + break; + + case ETH_RSS_NONFRAG_IPV4_TCP: + case ETH_RSS_NONFRAG_IPV6_TCP: + case ETH_RSS_NONFRAG_IPV4_UDP: + case ETH_RSS_NONFRAG_IPV6_UDP: + case ETH_RSS_IPV6_TCP_EX: + case ETH_RSS_IPV6_UDP_EX: + + if (l4_configured) + break; + l4_configured = 1; + + kg_cfg->extracts[i].extract.from_hdr.prot = + NET_PROT_TCP; + kg_cfg->extracts[i].extract.from_hdr.field = + NH_FLD_TCP_PORT_SRC; + kg_cfg->extracts[i].type = + DPKG_EXTRACT_FROM_HDR; + kg_cfg->extracts[i].extract.from_hdr.type = + DPKG_FULL_FIELD; + i++; + + kg_cfg->extracts[i].extract.from_hdr.prot = + NET_PROT_TCP; + kg_cfg->extracts[i].extract.from_hdr.field = + NH_FLD_TCP_PORT_SRC; + kg_cfg->extracts[i].type = + DPKG_EXTRACT_FROM_HDR; + kg_cfg->extracts[i].extract.from_hdr.type = + DPKG_FULL_FIELD; + i++; + break; + + case ETH_RSS_NONFRAG_IPV4_SCTP: + case ETH_RSS_NONFRAG_IPV6_SCTP: + + if (sctp_configured) + break; + sctp_configured = 1; + + kg_cfg->extracts[i].extract.from_hdr.prot = + NET_PROT_SCTP; + kg_cfg->extracts[i].extract.from_hdr.field = + NH_FLD_SCTP_PORT_SRC; + kg_cfg->extracts[i].type = + DPKG_EXTRACT_FROM_HDR; + kg_cfg->extracts[i].extract.from_hdr.type = + DPKG_FULL_FIELD; + i++; + + kg_cfg->extracts[i].extract.from_hdr.prot = + NET_PROT_SCTP; + kg_cfg->extracts[i].extract.from_hdr.field = + NH_FLD_SCTP_PORT_DST; + kg_cfg->extracts[i].type = + DPKG_EXTRACT_FROM_HDR; + kg_cfg->extracts[i].extract.from_hdr.type = + DPKG_FULL_FIELD; + i++; + break; + + default: + PMD_INIT_LOG(WARNING, + "Unsupported flow dist option %x", + dist_field); + return -EINVAL; + } + } + req_dist_set = req_dist_set >> 1; + loop++; + } + kg_cfg->num_extracts = i; + return 0; +} + +int +dpaa2_attach_bp_list(struct dpaa2_dev_priv *priv, + void *blist) +{ + /* Function to attach a DPNI with a buffer pool list. Buffer pool list + * handle is passed in blist. + */ + int32_t retcode; + struct fsl_mc_io *dpni = priv->hw; + struct dpni_pools_cfg bpool_cfg; + struct dpaa2_bp_list *bp_list = (struct dpaa2_bp_list *)blist; + struct dpni_buffer_layout layout; + int tot_size; + + /* ... rx buffer layout . + * Check alignment for buffer layouts first + */ + + /* ... rx buffer layout ... */ + tot_size = RTE_PKTMBUF_HEADROOM; + tot_size = RTE_ALIGN_CEIL(tot_size, DPAA2_PACKET_LAYOUT_ALIGN); + + memset(&layout, 0, sizeof(struct dpni_buffer_layout)); + layout.options = DPNI_BUF_LAYOUT_OPT_DATA_HEAD_ROOM | + DPNI_BUF_LAYOUT_OPT_FRAME_STATUS | + DPNI_BUF_LAYOUT_OPT_PARSER_RESULT | + DPNI_BUF_LAYOUT_OPT_DATA_ALIGN | + DPNI_BUF_LAYOUT_OPT_PRIVATE_DATA_SIZE; + + layout.pass_frame_status = 1; + layout.private_data_size = DPAA2_FD_PTA_SIZE; + layout.pass_parser_result = 1; + layout.data_align = DPAA2_PACKET_LAYOUT_ALIGN; + layout.data_head_room = tot_size - DPAA2_FD_PTA_SIZE - + DPAA2_MBUF_HW_ANNOTATION; + retcode = dpni_set_buffer_layout(dpni, CMD_PRI_LOW, priv->token, + DPNI_QUEUE_RX, &layout); + if (retcode) { + PMD_INIT_LOG(ERR, "Err(%d) in setting rx buffer layout\n", + retcode); + return retcode; + } + + /*Attach buffer pool to the network interface as described by the user*/ + bpool_cfg.num_dpbp = 1; + bpool_cfg.pools[0].dpbp_id = bp_list->buf_pool.dpbp_node->dpbp_id; + bpool_cfg.pools[0].backup_pool = 0; + bpool_cfg.pools[0].buffer_size = RTE_ALIGN_CEIL(bp_list->buf_pool.size, + DPAA2_PACKET_LAYOUT_ALIGN); + bpool_cfg.pools[0].priority_mask = 0; + + retcode = dpni_set_pools(dpni, CMD_PRI_LOW, priv->token, &bpool_cfg); + if (retcode != 0) { + PMD_INIT_LOG(ERR, "Error in attaching the buffer pool list" + " bpid = %d Error code = %d\n", + bpool_cfg.pools[0].dpbp_id, retcode); + return retcode; + } + + priv->bp_list = bp_list; + return 0; +} diff --git a/dpdk/drivers/net/dpaa2/base/dpaa2_hw_dpni_annot.h b/dpdk/drivers/net/dpaa2/base/dpaa2_hw_dpni_annot.h new file mode 100644 index 00000000..e68febff --- /dev/null +++ b/dpdk/drivers/net/dpaa2/base/dpaa2_hw_dpni_annot.h @@ -0,0 +1,257 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. + * Copyright 2016 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Freescale Semiconductor, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** + * @file + * + * DPNI packet parse results - implementation internal + */ + +#ifndef _DPAA2_HW_DPNI_ANNOT_H_ +#define _DPAA2_HW_DPNI_ANNOT_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +/* Annotation valid bits in FD FRC */ +#define DPAA2_FD_FRC_FASV 0x8000 +#define DPAA2_FD_FRC_FAEADV 0x4000 +#define DPAA2_FD_FRC_FAPRV 0x2000 +#define DPAA2_FD_FRC_FAIADV 0x1000 +#define DPAA2_FD_FRC_FASWOV 0x0800 +#define DPAA2_FD_FRC_FAICFDV 0x0400 + +/* Annotation bits in FD CTRL */ +#define DPAA2_FD_CTRL_ASAL 0x00020000 /* ASAL = 128 */ +#define DPAA2_FD_CTRL_PTA 0x00800000 +#define DPAA2_FD_CTRL_PTV1 0x00400000 + +/* Frame annotation status */ +struct dpaa2_fas { + uint8_t reserved; + uint8_t ppid; + __le16 ifpid; + __le32 status; +} __packed; + +/** + * HW Packet Annotation Register structures + */ +struct dpaa2_annot_hdr { + /**< word1: Frame Annotation Status (8 bytes)*/ + uint64_t word1; + + /**< word2: Time Stamp (8 bytes)*/ + uint64_t word2; + + /**< word3: Next Hdr + FAF Extension + FAF (2 + 2 + 4 bytes)*/ + uint64_t word3; + + /**< word4: Frame Annotation Flags-FAF (8 bytes) */ + uint64_t word4; + + /**< word5: + * ShimOffset_1 + ShimOffset_2 + IPPIDOffset + EthOffset + + * LLC+SNAPOffset + VLANTCIOffset_1 + VLANTCIOffset_n + + * LastETypeOffset (1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 bytes) + */ + uint64_t word5; + + /**< word6: + * PPPoEOffset + MPLSOffset_1 + MPLSOffset_n + ARPorIPOffset_1 + * + IPOffset_norMInEncapO + GREOffset + L4Offset + + * GTPorESPorIPSecOffset(1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 bytes) + */ + uint64_t word6; + + /**< word7: + * RoutingHdrOfset1 + RoutingHdrOfset2 + NxtHdrOffset + * + IPv6FragOffset + GrossRunningSum + * + RunningSum(1 + 1 + 1 + 1 + 2 + 2 bytes) + */ + uint64_t word7; + + /**< word8: + * ParseErrorcode + Soft Parsing Context (1 + 7 bytes) + */ + uint64_t word8; +}; + +/** + * Internal Macros to get/set Packet annotation header + */ + +/** General Macro to define a particular bit position*/ +#define BIT_POS(x) ((uint64_t)1 << ((x))) +/** Set a bit in the variable */ +#define BIT_SET_AT_POS(var, pos) ((var) |= (pos)) +/** Reset the bit in the variable */ +#define BIT_RESET_AT_POS(var, pos) ((var) &= ~(pos)) +/** Check the bit is set in the variable */ +#define BIT_ISSET_AT_POS(var, pos) (((var) & (pos)) ? 1 : 0) +/** + * Macrso to define bit position in word3 + */ +#define NEXT_HDR(var) ((uint64_t)(var) & 0xFFFF000000000000) +#define FAF_EXTN_IPV6_ROUTE_HDR_PRESENT(var) BIT_POS(16) +#define FAF_EXTN_RESERVED(var) ((uint64_t)(var) & 0x00007FFF00000000) +#define FAF_USER_DEFINED_RESERVED(var) ((uint64_t)(var) & 0x00000000FF000000) +#define SHIM_SHELL_SOFT_PARSING_ERRROR BIT_POS(23) +#define PARSING_ERROR BIT_POS(22) +#define L2_ETH_MAC_PRESENT BIT_POS(21) +#define L2_ETH_MAC_UNICAST BIT_POS(20) +#define L2_ETH_MAC_MULTICAST BIT_POS(19) +#define L2_ETH_MAC_BROADCAST BIT_POS(18) +#define L2_ETH_FRAME_IS_BPDU BIT_POS(17) +#define L2_ETH_FCOE_PRESENT BIT_POS(16) +#define L2_ETH_FIP_PRESENT BIT_POS(15) +#define L2_ETH_PARSING_ERROR BIT_POS(14) +#define L2_LLC_SNAP_PRESENT BIT_POS(13) +#define L2_UNKNOWN_LLC_OUI BIT_POS(12) +#define L2_LLC_SNAP_ERROR BIT_POS(11) +#define L2_VLAN_1_PRESENT BIT_POS(10) +#define L2_VLAN_N_PRESENT BIT_POS(9) +#define L2_VLAN_CFI_BIT_PRESENT BIT_POS(8) +#define L2_VLAN_PARSING_ERROR BIT_POS(7) +#define L2_PPPOE_PPP_PRESENT BIT_POS(6) +#define L2_PPPOE_PPP_PARSING_ERROR BIT_POS(5) +#define L2_MPLS_1_PRESENT BIT_POS(4) +#define L2_MPLS_N_PRESENT BIT_POS(3) +#define L2_MPLS_PARSING_ERROR BIT_POS(2) +#define L2_ARP_PRESENT BIT_POS(1) +#define L2_ARP_PARSING_ERROR BIT_POS(0) +/** + * Macrso to define bit position in word4 + */ +#define L2_UNKNOWN_PROTOCOL BIT_POS(63) +#define L2_SOFT_PARSING_ERROR BIT_POS(62) +#define L3_IPV4_1_PRESENT BIT_POS(61) +#define L3_IPV4_1_UNICAST BIT_POS(60) +#define L3_IPV4_1_MULTICAST BIT_POS(59) +#define L3_IPV4_1_BROADCAST BIT_POS(58) +#define L3_IPV4_N_PRESENT BIT_POS(57) +#define L3_IPV4_N_UNICAST BIT_POS(56) +#define L3_IPV4_N_MULTICAST BIT_POS(55) +#define L3_IPV4_N_BROADCAST BIT_POS(54) +#define L3_IPV6_1_PRESENT BIT_POS(53) +#define L3_IPV6_1_UNICAST BIT_POS(52) +#define L3_IPV6_1_MULTICAST BIT_POS(51) +#define L3_IPV6_N_PRESENT BIT_POS(50) +#define L3_IPV6_N_UNICAST BIT_POS(49) +#define L3_IPV6_N_MULTICAST BIT_POS(48) +#define L3_IP_1_OPT_PRESENT BIT_POS(47) +#define L3_IP_1_UNKNOWN_PROTOCOL BIT_POS(46) +#define L3_IP_1_MORE_FRAGMENT BIT_POS(45) +#define L3_IP_1_FIRST_FRAGMENT BIT_POS(44) +#define L3_IP_1_PARSING_ERROR BIT_POS(43) +#define L3_IP_N_OPT_PRESENT BIT_POS(42) +#define L3_IP_N_UNKNOWN_PROTOCOL BIT_POS(41) +#define L3_IP_N_MORE_FRAGMENT BIT_POS(40) +#define L3_IP_N_FIRST_FRAGMENT BIT_POS(39) +#define L3_PROTO_ICMP_PRESENT BIT_POS(38) +#define L3_PROTO_IGMP_PRESENT BIT_POS(37) +#define L3_PROTO_ICMPV6_PRESENT BIT_POS(36) +#define L3_PROTO_UDP_LIGHT_PRESENT BIT_POS(35) +#define L3_IP_N_PARSING_ERROR BIT_POS(34) +#define L3_MIN_ENCAP_PRESENT BIT_POS(33) +#define L3_MIN_ENCAP_SBIT_PRESENT BIT_POS(32) +#define L3_MIN_ENCAP_PARSING_ERROR BIT_POS(31) +#define L3_PROTO_GRE_PRESENT BIT_POS(30) +#define L3_PROTO_GRE_RBIT_PRESENT BIT_POS(29) +#define L3_PROTO_GRE_PARSING_ERROR BIT_POS(28) +#define L3_IP_UNKNOWN_PROTOCOL BIT_POS(27) +#define L3_SOFT_PARSING_ERROR BIT_POS(26) +#define L3_PROTO_UDP_PRESENT BIT_POS(25) +#define L3_PROTO_UDP_PARSING_ERROR BIT_POS(24) +#define L3_PROTO_TCP_PRESENT BIT_POS(23) +#define L3_PROTO_TCP_OPT_PRESENT BIT_POS(22) +#define L3_PROTO_TCP_CTRL_BIT_6_TO_11_PRESENT BIT_POS(21) +#define L3_PROTO_TCP_CTRL_BIT_3_TO_5_PRESENT BIT_POS(20) +#define L3_PROTO_TCP_PARSING_ERROR BIT_POS(19) +#define L3_PROTO_IPSEC_PRESENT BIT_POS(18) +#define L3_PROTO_IPSEC_ESP_PRESENT BIT_POS(17) +#define L3_PROTO_IPSEC_AH_PRESENT BIT_POS(16) +#define L3_PROTO_IPSEC_PARSING_ERROR BIT_POS(15) +#define L3_PROTO_SCTP_PRESENT BIT_POS(14) +#define L3_PROTO_SCTP_PARSING_ERROR BIT_POS(13) +#define L3_PROTO_DCCP_PRESENT BIT_POS(12) +#define L3_PROTO_DCCP_PARSING_ERROR BIT_POS(11) +#define L4_UNKNOWN_PROTOCOL BIT_POS(10) +#define L4_SOFT_PARSING_ERROR BIT_POS(9) +#define L3_PROTO_GTP_PRESENT BIT_POS(8) +#define L3_PROTO_GTP_PARSING_ERROR BIT_POS(7) +#define L3_PROTO_ESP_PRESENT BIT_POS(6) +#define L3_PROTO_ESP_PARSING_ERROR BIT_POS(5) +#define L3_PROTO_ISCSI_PRESENT BIT_POS(4) +#define L3_PROTO_CAPWAN__CTRL_PRESENT BIT_POS(3) +#define L3_PROTO_CAPWAN__DATA_PRESENT BIT_POS(2) +#define L5_SOFT_PARSING_ERROR BIT_POS(1) +#define L3_IPV6_ROUTE_HDR_PRESENT BIT_POS(0) + +/* Debug frame, otherwise supposed to be discarded */ +#define DPAA2_ETH_FAS_DISC 0x80000000 +/* MACSEC frame */ +#define DPAA2_ETH_FAS_MS 0x40000000 +#define DPAA2_ETH_FAS_PTP 0x08000000 +/* Ethernet multicast frame */ +#define DPAA2_ETH_FAS_MC 0x04000000 +/* Ethernet broadcast frame */ +#define DPAA2_ETH_FAS_BC 0x02000000 +#define DPAA2_ETH_FAS_KSE 0x00040000 +#define DPAA2_ETH_FAS_EOFHE 0x00020000 +#define DPAA2_ETH_FAS_MNLE 0x00010000 +#define DPAA2_ETH_FAS_TIDE 0x00008000 +#define DPAA2_ETH_FAS_PIEE 0x00004000 +/* Frame length error */ +#define DPAA2_ETH_FAS_FLE 0x00002000 +/* Frame physical error; our favourite pastime */ +#define DPAA2_ETH_FAS_FPE 0x00001000 +#define DPAA2_ETH_FAS_PTE 0x00000080 +#define DPAA2_ETH_FAS_ISP 0x00000040 +#define DPAA2_ETH_FAS_PHE 0x00000020 +#define DPAA2_ETH_FAS_BLE 0x00000010 +/* L3 csum validation performed */ +#define DPAA2_ETH_FAS_L3CV 0x00000008 +/* L3 csum error */ +#define DPAA2_ETH_FAS_L3CE 0x00000004 +/* L4 csum validation performed */ +#define DPAA2_ETH_FAS_L4CV 0x00000002 +/* L4 csum error */ +#define DPAA2_ETH_FAS_L4CE 0x00000001 + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/dpdk/drivers/net/dpaa2/dpaa2_ethdev.c b/dpdk/drivers/net/dpaa2/dpaa2_ethdev.c new file mode 100644 index 00000000..202f84f0 --- /dev/null +++ b/dpdk/drivers/net/dpaa2/dpaa2_ethdev.c @@ -0,0 +1,2011 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. + * Copyright 2016 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Freescale Semiconductor, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include "dpaa2_ethdev.h" + +struct rte_dpaa2_xstats_name_off { + char name[RTE_ETH_XSTATS_NAME_SIZE]; + uint8_t page_id; /* dpni statistics page id */ + uint8_t stats_id; /* stats id in the given page */ +}; + +static const struct rte_dpaa2_xstats_name_off dpaa2_xstats_strings[] = { + {"ingress_multicast_frames", 0, 2}, + {"ingress_multicast_bytes", 0, 3}, + {"ingress_broadcast_frames", 0, 4}, + {"ingress_broadcast_bytes", 0, 5}, + {"egress_multicast_frames", 1, 2}, + {"egress_multicast_bytes", 1, 3}, + {"egress_broadcast_frames", 1, 4}, + {"egress_broadcast_bytes", 1, 5}, + {"ingress_filtered_frames", 2, 0}, + {"ingress_discarded_frames", 2, 1}, + {"ingress_nobuffer_discards", 2, 2}, + {"egress_discarded_frames", 2, 3}, + {"egress_confirmed_frames", 2, 4}, +}; + +static struct rte_dpaa2_driver rte_dpaa2_pmd; +static int dpaa2_dev_uninit(struct rte_eth_dev *eth_dev); +static int dpaa2_dev_link_update(struct rte_eth_dev *dev, + int wait_to_complete); +static int dpaa2_dev_set_link_up(struct rte_eth_dev *dev); +static int dpaa2_dev_set_link_down(struct rte_eth_dev *dev); +static int dpaa2_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu); + +/** + * Atomically reads the link status information from global + * structure rte_eth_dev. + * + * @param dev + * - Pointer to the structure rte_eth_dev to read from. + * - Pointer to the buffer to be saved with the link status. + * + * @return + * - On success, zero. + * - On failure, negative value. + */ +static inline int +dpaa2_dev_atomic_read_link_status(struct rte_eth_dev *dev, + struct rte_eth_link *link) +{ + struct rte_eth_link *dst = link; + struct rte_eth_link *src = &dev->data->dev_link; + + if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst, + *(uint64_t *)src) == 0) + return -1; + + return 0; +} + +/** + * Atomically writes the link status information into global + * structure rte_eth_dev. + * + * @param dev + * - Pointer to the structure rte_eth_dev to read from. + * - Pointer to the buffer to be saved with the link status. + * + * @return + * - On success, zero. + * - On failure, negative value. + */ +static inline int +dpaa2_dev_atomic_write_link_status(struct rte_eth_dev *dev, + struct rte_eth_link *link) +{ + struct rte_eth_link *dst = &dev->data->dev_link; + struct rte_eth_link *src = link; + + if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst, + *(uint64_t *)src) == 0) + return -1; + + return 0; +} + +static int +dpaa2_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) +{ + int ret; + struct dpaa2_dev_priv *priv = dev->data->dev_private; + struct fsl_mc_io *dpni = priv->hw; + + PMD_INIT_FUNC_TRACE(); + + if (dpni == NULL) { + RTE_LOG(ERR, PMD, "dpni is NULL\n"); + return -1; + } + + if (on) + ret = dpni_add_vlan_id(dpni, CMD_PRI_LOW, + priv->token, vlan_id); + else + ret = dpni_remove_vlan_id(dpni, CMD_PRI_LOW, + priv->token, vlan_id); + + if (ret < 0) + PMD_DRV_LOG(ERR, "ret = %d Unable to add/rem vlan %d hwid =%d", + ret, vlan_id, priv->hw_id); + + return ret; +} + +static int +dpaa2_vlan_offload_set(struct rte_eth_dev *dev, int mask) +{ + struct dpaa2_dev_priv *priv = dev->data->dev_private; + struct fsl_mc_io *dpni = priv->hw; + int ret; + + PMD_INIT_FUNC_TRACE(); + + if (mask & ETH_VLAN_FILTER_MASK) { + if (dev->data->dev_conf.rxmode.hw_vlan_filter) + ret = dpni_enable_vlan_filter(dpni, CMD_PRI_LOW, + priv->token, true); + else + ret = dpni_enable_vlan_filter(dpni, CMD_PRI_LOW, + priv->token, false); + if (ret < 0) + RTE_LOG(ERR, PMD, "Unable to set vlan filter = %d\n", + ret); + } + + if (mask & ETH_VLAN_EXTEND_MASK) { + if (dev->data->dev_conf.rxmode.hw_vlan_extend) + RTE_LOG(INFO, PMD, + "VLAN extend offload not supported\n"); + } + + return 0; +} + +static int +dpaa2_fw_version_get(struct rte_eth_dev *dev, + char *fw_version, + size_t fw_size) +{ + int ret; + struct dpaa2_dev_priv *priv = dev->data->dev_private; + struct fsl_mc_io *dpni = priv->hw; + struct mc_soc_version mc_plat_info = {0}; + struct mc_version mc_ver_info = {0}; + + PMD_INIT_FUNC_TRACE(); + + if (mc_get_soc_version(dpni, CMD_PRI_LOW, &mc_plat_info)) + RTE_LOG(WARNING, PMD, "\tmc_get_soc_version failed\n"); + + if (mc_get_version(dpni, CMD_PRI_LOW, &mc_ver_info)) + RTE_LOG(WARNING, PMD, "\tmc_get_version failed\n"); + + ret = snprintf(fw_version, fw_size, + "%x-%d.%d.%d", + mc_plat_info.svr, + mc_ver_info.major, + mc_ver_info.minor, + mc_ver_info.revision); + + ret += 1; /* add the size of '\0' */ + if (fw_size < (uint32_t)ret) + return ret; + else + return 0; +} + +static void +dpaa2_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) +{ + struct dpaa2_dev_priv *priv = dev->data->dev_private; + + PMD_INIT_FUNC_TRACE(); + + dev_info->if_index = priv->hw_id; + + dev_info->max_mac_addrs = priv->max_mac_filters; + dev_info->max_rx_pktlen = DPAA2_MAX_RX_PKT_LEN; + dev_info->min_rx_bufsize = DPAA2_MIN_RX_BUF_SIZE; + dev_info->max_rx_queues = (uint16_t)priv->nb_rx_queues; + dev_info->max_tx_queues = (uint16_t)priv->nb_tx_queues; + dev_info->rx_offload_capa = + DEV_RX_OFFLOAD_IPV4_CKSUM | + DEV_RX_OFFLOAD_UDP_CKSUM | + DEV_RX_OFFLOAD_TCP_CKSUM | + DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM; + dev_info->tx_offload_capa = + DEV_TX_OFFLOAD_IPV4_CKSUM | + DEV_TX_OFFLOAD_UDP_CKSUM | + DEV_TX_OFFLOAD_TCP_CKSUM | + DEV_TX_OFFLOAD_SCTP_CKSUM | + DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM; + dev_info->speed_capa = ETH_LINK_SPEED_1G | + ETH_LINK_SPEED_2_5G | + ETH_LINK_SPEED_10G; +} + +static int +dpaa2_alloc_rx_tx_queues(struct rte_eth_dev *dev) +{ + struct dpaa2_dev_priv *priv = dev->data->dev_private; + uint16_t dist_idx; + uint32_t vq_id; + struct dpaa2_queue *mc_q, *mcq; + uint32_t tot_queues; + int i; + struct dpaa2_queue *dpaa2_q; + + PMD_INIT_FUNC_TRACE(); + + tot_queues = priv->nb_rx_queues + priv->nb_tx_queues; + mc_q = rte_malloc(NULL, sizeof(struct dpaa2_queue) * tot_queues, + RTE_CACHE_LINE_SIZE); + if (!mc_q) { + PMD_INIT_LOG(ERR, "malloc failed for rx/tx queues\n"); + return -1; + } + + for (i = 0; i < priv->nb_rx_queues; i++) { + mc_q->dev = dev; + priv->rx_vq[i] = mc_q++; + dpaa2_q = (struct dpaa2_queue *)priv->rx_vq[i]; + dpaa2_q->q_storage = rte_malloc("dq_storage", + sizeof(struct queue_storage_info_t), + RTE_CACHE_LINE_SIZE); + if (!dpaa2_q->q_storage) + goto fail; + + memset(dpaa2_q->q_storage, 0, + sizeof(struct queue_storage_info_t)); + if (dpaa2_alloc_dq_storage(dpaa2_q->q_storage)) + goto fail; + } + + for (i = 0; i < priv->nb_tx_queues; i++) { + mc_q->dev = dev; + mc_q->flow_id = 0xffff; + priv->tx_vq[i] = mc_q++; + dpaa2_q = (struct dpaa2_queue *)priv->tx_vq[i]; + dpaa2_q->cscn = rte_malloc(NULL, + sizeof(struct qbman_result), 16); + if (!dpaa2_q->cscn) + goto fail_tx; + } + + vq_id = 0; + for (dist_idx = 0; dist_idx < priv->nb_rx_queues; dist_idx++) { + mcq = (struct dpaa2_queue *)priv->rx_vq[vq_id]; + mcq->tc_index = DPAA2_DEF_TC; + mcq->flow_id = dist_idx; + vq_id++; + } + + return 0; +fail_tx: + i -= 1; + while (i >= 0) { + dpaa2_q = (struct dpaa2_queue *)priv->tx_vq[i]; + rte_free(dpaa2_q->cscn); + priv->tx_vq[i--] = NULL; + } + i = priv->nb_rx_queues; +fail: + i -= 1; + mc_q = priv->rx_vq[0]; + while (i >= 0) { + dpaa2_q = (struct dpaa2_queue *)priv->rx_vq[i]; + dpaa2_free_dq_storage(dpaa2_q->q_storage); + rte_free(dpaa2_q->q_storage); + priv->rx_vq[i--] = NULL; + } + rte_free(mc_q); + return -1; +} + +static int +dpaa2_eth_dev_configure(struct rte_eth_dev *dev) +{ + struct dpaa2_dev_priv *priv = dev->data->dev_private; + struct fsl_mc_io *dpni = priv->hw; + struct rte_eth_conf *eth_conf = &dev->data->dev_conf; + int rx_ip_csum_offload = false; + int ret; + + PMD_INIT_FUNC_TRACE(); + + if (eth_conf->rxmode.jumbo_frame == 1) { + if (eth_conf->rxmode.max_rx_pkt_len <= DPAA2_MAX_RX_PKT_LEN) { + ret = dpaa2_dev_mtu_set(dev, + eth_conf->rxmode.max_rx_pkt_len); + if (ret) { + PMD_INIT_LOG(ERR, + "unable to set mtu. check config\n"); + return ret; + } + } else { + return -1; + } + } + + if (eth_conf->rxmode.mq_mode == ETH_MQ_RX_RSS) { + ret = dpaa2_setup_flow_dist(dev, + eth_conf->rx_adv_conf.rss_conf.rss_hf); + if (ret) { + PMD_INIT_LOG(ERR, "unable to set flow distribution." + "please check queue config\n"); + return ret; + } + } + + if (eth_conf->rxmode.hw_ip_checksum) + rx_ip_csum_offload = true; + + ret = dpni_set_offload(dpni, CMD_PRI_LOW, priv->token, + DPNI_OFF_RX_L3_CSUM, rx_ip_csum_offload); + if (ret) { + PMD_INIT_LOG(ERR, "Error to set RX l3 csum:Error = %d\n", ret); + return ret; + } + + ret = dpni_set_offload(dpni, CMD_PRI_LOW, priv->token, + DPNI_OFF_RX_L4_CSUM, rx_ip_csum_offload); + if (ret) { + PMD_INIT_LOG(ERR, "Error to get RX l4 csum:Error = %d\n", ret); + return ret; + } + + ret = dpni_set_offload(dpni, CMD_PRI_LOW, priv->token, + DPNI_OFF_TX_L3_CSUM, true); + if (ret) { + PMD_INIT_LOG(ERR, "Error to set TX l3 csum:Error = %d\n", ret); + return ret; + } + + ret = dpni_set_offload(dpni, CMD_PRI_LOW, priv->token, + DPNI_OFF_TX_L4_CSUM, true); + if (ret) { + PMD_INIT_LOG(ERR, "Error to get TX l4 csum:Error = %d\n", ret); + return ret; + } + + /* update the current status */ + dpaa2_dev_link_update(dev, 0); + + return 0; +} + +/* Function to setup RX flow information. It contains traffic class ID, + * flow ID, destination configuration etc. + */ +static int +dpaa2_dev_rx_queue_setup(struct rte_eth_dev *dev, + uint16_t rx_queue_id, + uint16_t nb_rx_desc __rte_unused, + unsigned int socket_id __rte_unused, + const struct rte_eth_rxconf *rx_conf __rte_unused, + struct rte_mempool *mb_pool) +{ + struct dpaa2_dev_priv *priv = dev->data->dev_private; + struct fsl_mc_io *dpni = (struct fsl_mc_io *)priv->hw; + struct mc_soc_version mc_plat_info = {0}; + struct dpaa2_queue *dpaa2_q; + struct dpni_queue cfg; + uint8_t options = 0; + uint8_t flow_id; + uint32_t bpid; + int ret; + + PMD_INIT_FUNC_TRACE(); + + PMD_DRV_LOG(DEBUG, "dev =%p, queue =%d, pool = %p, conf =%p", + dev, rx_queue_id, mb_pool, rx_conf); + + if (!priv->bp_list || priv->bp_list->mp != mb_pool) { + bpid = mempool_to_bpid(mb_pool); + ret = dpaa2_attach_bp_list(priv, + rte_dpaa2_bpid_info[bpid].bp_list); + if (ret) + return ret; + } + dpaa2_q = (struct dpaa2_queue *)priv->rx_vq[rx_queue_id]; + dpaa2_q->mb_pool = mb_pool; /**< mbuf pool to populate RX ring. */ + + /*Get the flow id from given VQ id*/ + flow_id = rx_queue_id % priv->nb_rx_queues; + memset(&cfg, 0, sizeof(struct dpni_queue)); + + options = options | DPNI_QUEUE_OPT_USER_CTX; + cfg.user_context = (uint64_t)(dpaa2_q); + + /*if ls2088 or rev2 device, enable the stashing */ + + if (mc_get_soc_version(dpni, CMD_PRI_LOW, &mc_plat_info)) + PMD_INIT_LOG(ERR, "\tmc_get_soc_version failed\n"); + + if ((mc_plat_info.svr & 0xffff0000) != SVR_LS2080A) { + options |= DPNI_QUEUE_OPT_FLC; + cfg.flc.stash_control = true; + cfg.flc.value &= 0xFFFFFFFFFFFFFFC0; + /* 00 00 00 - last 6 bit represent annotation, context stashing, + * data stashing setting 01 01 00 (0x14) to enable + * 1 line data, 1 line annotation + */ + cfg.flc.value |= 0x14; + } + ret = dpni_set_queue(dpni, CMD_PRI_LOW, priv->token, DPNI_QUEUE_RX, + dpaa2_q->tc_index, flow_id, options, &cfg); + if (ret) { + PMD_INIT_LOG(ERR, "Error in setting the rx flow: = %d\n", ret); + return -1; + } + + if (!(priv->flags & DPAA2_RX_TAILDROP_OFF)) { + struct dpni_taildrop taildrop; + + taildrop.enable = 1; + /*enabling per rx queue congestion control */ + taildrop.threshold = CONG_THRESHOLD_RX_Q; + taildrop.units = DPNI_CONGESTION_UNIT_BYTES; + taildrop.oal = CONG_RX_OAL; + PMD_DRV_LOG(DEBUG, "Enabling Early Drop on queue = %d", + rx_queue_id); + ret = dpni_set_taildrop(dpni, CMD_PRI_LOW, priv->token, + DPNI_CP_QUEUE, DPNI_QUEUE_RX, + dpaa2_q->tc_index, flow_id, &taildrop); + if (ret) { + PMD_INIT_LOG(ERR, "Error in setting the rx flow" + " err : = %d\n", ret); + return -1; + } + } + + dev->data->rx_queues[rx_queue_id] = dpaa2_q; + return 0; +} + +static int +dpaa2_dev_tx_queue_setup(struct rte_eth_dev *dev, + uint16_t tx_queue_id, + uint16_t nb_tx_desc __rte_unused, + unsigned int socket_id __rte_unused, + const struct rte_eth_txconf *tx_conf __rte_unused) +{ + struct dpaa2_dev_priv *priv = dev->data->dev_private; + struct dpaa2_queue *dpaa2_q = (struct dpaa2_queue *) + priv->tx_vq[tx_queue_id]; + struct fsl_mc_io *dpni = priv->hw; + struct dpni_queue tx_conf_cfg; + struct dpni_queue tx_flow_cfg; + uint8_t options = 0, flow_id; + uint32_t tc_id; + int ret; + + PMD_INIT_FUNC_TRACE(); + + /* Return if queue already configured */ + if (dpaa2_q->flow_id != 0xffff) { + dev->data->tx_queues[tx_queue_id] = dpaa2_q; + return 0; + } + + memset(&tx_conf_cfg, 0, sizeof(struct dpni_queue)); + memset(&tx_flow_cfg, 0, sizeof(struct dpni_queue)); + + tc_id = tx_queue_id; + flow_id = 0; + + ret = dpni_set_queue(dpni, CMD_PRI_LOW, priv->token, DPNI_QUEUE_TX, + tc_id, flow_id, options, &tx_flow_cfg); + if (ret) { + PMD_INIT_LOG(ERR, "Error in setting the tx flow: " + "tc_id=%d, flow =%d ErrorCode = %x\n", + tc_id, flow_id, -ret); + return -1; + } + + dpaa2_q->flow_id = flow_id; + + if (tx_queue_id == 0) { + /*Set tx-conf and error configuration*/ + ret = dpni_set_tx_confirmation_mode(dpni, CMD_PRI_LOW, + priv->token, + DPNI_CONF_DISABLE); + if (ret) { + PMD_INIT_LOG(ERR, "Error in set tx conf mode settings" + " ErrorCode = %x", ret); + return -1; + } + } + dpaa2_q->tc_index = tc_id; + + if (!(priv->flags & DPAA2_TX_CGR_OFF)) { + struct dpni_congestion_notification_cfg cong_notif_cfg; + + cong_notif_cfg.units = DPNI_CONGESTION_UNIT_FRAMES; + cong_notif_cfg.threshold_entry = CONG_ENTER_TX_THRESHOLD; + /* Notify that the queue is not congested when the data in + * the queue is below this thershold. + */ + cong_notif_cfg.threshold_exit = CONG_EXIT_TX_THRESHOLD; + cong_notif_cfg.message_ctx = 0; + cong_notif_cfg.message_iova = (uint64_t)dpaa2_q->cscn; + cong_notif_cfg.dest_cfg.dest_type = DPNI_DEST_NONE; + cong_notif_cfg.notification_mode = + DPNI_CONG_OPT_WRITE_MEM_ON_ENTER | + DPNI_CONG_OPT_WRITE_MEM_ON_EXIT | + DPNI_CONG_OPT_COHERENT_WRITE; + + ret = dpni_set_congestion_notification(dpni, CMD_PRI_LOW, + priv->token, + DPNI_QUEUE_TX, + tc_id, + &cong_notif_cfg); + if (ret) { + PMD_INIT_LOG(ERR, + "Error in setting tx congestion notification: = %d", + -ret); + return -ret; + } + } + dev->data->tx_queues[tx_queue_id] = dpaa2_q; + return 0; +} + +static void +dpaa2_dev_rx_queue_release(void *q __rte_unused) +{ + PMD_INIT_FUNC_TRACE(); +} + +static void +dpaa2_dev_tx_queue_release(void *q __rte_unused) +{ + PMD_INIT_FUNC_TRACE(); +} + +static const uint32_t * +dpaa2_supported_ptypes_get(struct rte_eth_dev *dev) +{ + static const uint32_t ptypes[] = { + /*todo -= add more types */ + RTE_PTYPE_L2_ETHER, + RTE_PTYPE_L3_IPV4, + RTE_PTYPE_L3_IPV4_EXT, + RTE_PTYPE_L3_IPV6, + RTE_PTYPE_L3_IPV6_EXT, + RTE_PTYPE_L4_TCP, + RTE_PTYPE_L4_UDP, + RTE_PTYPE_L4_SCTP, + RTE_PTYPE_L4_ICMP, + RTE_PTYPE_UNKNOWN + }; + + if (dev->rx_pkt_burst == dpaa2_dev_prefetch_rx) + return ptypes; + return NULL; +} + +/** + * Dpaa2 link Interrupt handler + * + * @param param + * The address of parameter (struct rte_eth_dev *) regsitered before. + * + * @return + * void + */ +static void +dpaa2_interrupt_handler(void *param) +{ + struct rte_eth_dev *dev = param; + struct dpaa2_dev_priv *priv = dev->data->dev_private; + struct fsl_mc_io *dpni = (struct fsl_mc_io *)priv->hw; + int ret; + int irq_index = DPNI_IRQ_INDEX; + unsigned int status = 0, clear = 0; + + PMD_INIT_FUNC_TRACE(); + + if (dpni == NULL) { + RTE_LOG(ERR, PMD, "dpni is NULL"); + return; + } + + ret = dpni_get_irq_status(dpni, CMD_PRI_LOW, priv->token, + irq_index, &status); + if (unlikely(ret)) { + RTE_LOG(ERR, PMD, "Can't get irq status (err %d)", ret); + clear = 0xffffffff; + goto out; + } + + if (status & DPNI_IRQ_EVENT_LINK_CHANGED) { + clear = DPNI_IRQ_EVENT_LINK_CHANGED; + dpaa2_dev_link_update(dev, 0); + /* calling all the apps registered for link status event */ + _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, + NULL, NULL); + } +out: + ret = dpni_clear_irq_status(dpni, CMD_PRI_LOW, priv->token, + irq_index, clear); + if (unlikely(ret)) + RTE_LOG(ERR, PMD, "Can't clear irq status (err %d)", ret); +} + +static int +dpaa2_eth_setup_irqs(struct rte_eth_dev *dev, int enable) +{ + int err = 0; + struct dpaa2_dev_priv *priv = dev->data->dev_private; + struct fsl_mc_io *dpni = (struct fsl_mc_io *)priv->hw; + int irq_index = DPNI_IRQ_INDEX; + unsigned int mask = DPNI_IRQ_EVENT_LINK_CHANGED; + + PMD_INIT_FUNC_TRACE(); + + err = dpni_set_irq_mask(dpni, CMD_PRI_LOW, priv->token, + irq_index, mask); + if (err < 0) { + PMD_INIT_LOG(ERR, "Error: dpni_set_irq_mask():%d (%s)", err, + strerror(-err)); + return err; + } + + err = dpni_set_irq_enable(dpni, CMD_PRI_LOW, priv->token, + irq_index, enable); + if (err < 0) + PMD_INIT_LOG(ERR, "Error: dpni_set_irq_enable():%d (%s)", err, + strerror(-err)); + + return err; +} + +static int +dpaa2_dev_start(struct rte_eth_dev *dev) +{ + struct rte_device *rdev = dev->device; + struct rte_dpaa2_device *dpaa2_dev; + struct rte_eth_dev_data *data = dev->data; + struct dpaa2_dev_priv *priv = data->dev_private; + struct fsl_mc_io *dpni = (struct fsl_mc_io *)priv->hw; + struct dpni_queue cfg; + struct dpni_error_cfg err_cfg; + uint16_t qdid; + struct dpni_queue_id qid; + struct dpaa2_queue *dpaa2_q; + int ret, i; + struct rte_intr_handle *intr_handle; + + dpaa2_dev = container_of(rdev, struct rte_dpaa2_device, device); + intr_handle = &dpaa2_dev->intr_handle; + + PMD_INIT_FUNC_TRACE(); + + ret = dpni_enable(dpni, CMD_PRI_LOW, priv->token); + if (ret) { + PMD_INIT_LOG(ERR, "Failure %d in enabling dpni %d device\n", + ret, priv->hw_id); + return ret; + } + + /* Power up the phy. Needed to make the link go UP */ + dpaa2_dev_set_link_up(dev); + + ret = dpni_get_qdid(dpni, CMD_PRI_LOW, priv->token, + DPNI_QUEUE_TX, &qdid); + if (ret) { + PMD_INIT_LOG(ERR, "Error to get qdid:ErrorCode = %d\n", ret); + return ret; + } + priv->qdid = qdid; + + for (i = 0; i < data->nb_rx_queues; i++) { + dpaa2_q = (struct dpaa2_queue *)data->rx_queues[i]; + ret = dpni_get_queue(dpni, CMD_PRI_LOW, priv->token, + DPNI_QUEUE_RX, dpaa2_q->tc_index, + dpaa2_q->flow_id, &cfg, &qid); + if (ret) { + PMD_INIT_LOG(ERR, "Error to get flow " + "information Error code = %d\n", ret); + return ret; + } + dpaa2_q->fqid = qid.fqid; + } + + /*checksum errors, send them to normal path and set it in annotation */ + err_cfg.errors = DPNI_ERROR_L3CE | DPNI_ERROR_L4CE; + + err_cfg.error_action = DPNI_ERROR_ACTION_CONTINUE; + err_cfg.set_frame_annotation = true; + + ret = dpni_set_errors_behavior(dpni, CMD_PRI_LOW, + priv->token, &err_cfg); + if (ret) { + PMD_INIT_LOG(ERR, "Error to dpni_set_errors_behavior:" + "code = %d\n", ret); + return ret; + } + /* VLAN Offload Settings */ + if (priv->max_vlan_filters) { + ret = dpaa2_vlan_offload_set(dev, ETH_VLAN_FILTER_MASK); + if (ret) { + PMD_INIT_LOG(ERR, "Error to dpaa2_vlan_offload_set:" + "code = %d\n", ret); + return ret; + } + } + + + /* if the interrupts were configured on this devices*/ + if (intr_handle && (intr_handle->fd) && + (dev->data->dev_conf.intr_conf.lsc != 0)) { + /* Registering LSC interrupt handler */ + rte_intr_callback_register(intr_handle, + dpaa2_interrupt_handler, + (void *)dev); + + /* enable vfio intr/eventfd mapping + * Interrupt index 0 is required, so we can not use + * rte_intr_enable. + */ + rte_dpaa2_intr_enable(intr_handle, DPNI_IRQ_INDEX); + + /* enable dpni_irqs */ + dpaa2_eth_setup_irqs(dev, 1); + } + + return 0; +} + +/** + * This routine disables all traffic on the adapter by issuing a + * global reset on the MAC. + */ +static void +dpaa2_dev_stop(struct rte_eth_dev *dev) +{ + struct dpaa2_dev_priv *priv = dev->data->dev_private; + struct fsl_mc_io *dpni = (struct fsl_mc_io *)priv->hw; + int ret; + struct rte_eth_link link; + struct rte_intr_handle *intr_handle = dev->intr_handle; + + PMD_INIT_FUNC_TRACE(); + + /* reset interrupt callback */ + if (intr_handle && (intr_handle->fd) && + (dev->data->dev_conf.intr_conf.lsc != 0)) { + /*disable dpni irqs */ + dpaa2_eth_setup_irqs(dev, 0); + + /* disable vfio intr before callback unregister */ + rte_dpaa2_intr_disable(intr_handle, DPNI_IRQ_INDEX); + + /* Unregistering LSC interrupt handler */ + rte_intr_callback_unregister(intr_handle, + dpaa2_interrupt_handler, + (void *)dev); + } + + dpaa2_dev_set_link_down(dev); + + ret = dpni_disable(dpni, CMD_PRI_LOW, priv->token); + if (ret) { + PMD_INIT_LOG(ERR, "Failure (ret %d) in disabling dpni %d dev\n", + ret, priv->hw_id); + return; + } + + /* clear the recorded link status */ + memset(&link, 0, sizeof(link)); + dpaa2_dev_atomic_write_link_status(dev, &link); +} + +static void +dpaa2_dev_close(struct rte_eth_dev *dev) +{ + struct rte_eth_dev_data *data = dev->data; + struct dpaa2_dev_priv *priv = dev->data->dev_private; + struct fsl_mc_io *dpni = (struct fsl_mc_io *)priv->hw; + int i, ret; + struct rte_eth_link link; + struct dpaa2_queue *dpaa2_q; + + PMD_INIT_FUNC_TRACE(); + + for (i = 0; i < data->nb_tx_queues; i++) { + dpaa2_q = (struct dpaa2_queue *)data->tx_queues[i]; + if (!dpaa2_q->cscn) { + rte_free(dpaa2_q->cscn); + dpaa2_q->cscn = NULL; + } + } + + /* Clean the device first */ + ret = dpni_reset(dpni, CMD_PRI_LOW, priv->token); + if (ret) { + PMD_INIT_LOG(ERR, "Failure cleaning dpni device with" + " error code %d\n", ret); + return; + } + + memset(&link, 0, sizeof(link)); + dpaa2_dev_atomic_write_link_status(dev, &link); +} + +static void +dpaa2_dev_promiscuous_enable( + struct rte_eth_dev *dev) +{ + int ret; + struct dpaa2_dev_priv *priv = dev->data->dev_private; + struct fsl_mc_io *dpni = (struct fsl_mc_io *)priv->hw; + + PMD_INIT_FUNC_TRACE(); + + if (dpni == NULL) { + RTE_LOG(ERR, PMD, "dpni is NULL\n"); + return; + } + + ret = dpni_set_unicast_promisc(dpni, CMD_PRI_LOW, priv->token, true); + if (ret < 0) + RTE_LOG(ERR, PMD, "Unable to enable U promisc mode %d\n", ret); + + ret = dpni_set_multicast_promisc(dpni, CMD_PRI_LOW, priv->token, true); + if (ret < 0) + RTE_LOG(ERR, PMD, "Unable to enable M promisc mode %d\n", ret); +} + +static void +dpaa2_dev_promiscuous_disable( + struct rte_eth_dev *dev) +{ + int ret; + struct dpaa2_dev_priv *priv = dev->data->dev_private; + struct fsl_mc_io *dpni = (struct fsl_mc_io *)priv->hw; + + PMD_INIT_FUNC_TRACE(); + + if (dpni == NULL) { + RTE_LOG(ERR, PMD, "dpni is NULL\n"); + return; + } + + ret = dpni_set_unicast_promisc(dpni, CMD_PRI_LOW, priv->token, false); + if (ret < 0) + RTE_LOG(ERR, PMD, "Unable to disable U promisc mode %d\n", ret); + + if (dev->data->all_multicast == 0) { + ret = dpni_set_multicast_promisc(dpni, CMD_PRI_LOW, + priv->token, false); + if (ret < 0) + RTE_LOG(ERR, PMD, + "Unable to disable M promisc mode %d\n", + ret); + } +} + +static void +dpaa2_dev_allmulticast_enable( + struct rte_eth_dev *dev) +{ + int ret; + struct dpaa2_dev_priv *priv = dev->data->dev_private; + struct fsl_mc_io *dpni = (struct fsl_mc_io *)priv->hw; + + PMD_INIT_FUNC_TRACE(); + + if (dpni == NULL) { + RTE_LOG(ERR, PMD, "dpni is NULL\n"); + return; + } + + ret = dpni_set_multicast_promisc(dpni, CMD_PRI_LOW, priv->token, true); + if (ret < 0) + RTE_LOG(ERR, PMD, "Unable to enable multicast mode %d\n", ret); +} + +static void +dpaa2_dev_allmulticast_disable(struct rte_eth_dev *dev) +{ + int ret; + struct dpaa2_dev_priv *priv = dev->data->dev_private; + struct fsl_mc_io *dpni = (struct fsl_mc_io *)priv->hw; + + PMD_INIT_FUNC_TRACE(); + + if (dpni == NULL) { + RTE_LOG(ERR, PMD, "dpni is NULL\n"); + return; + } + + /* must remain on for all promiscuous */ + if (dev->data->promiscuous == 1) + return; + + ret = dpni_set_multicast_promisc(dpni, CMD_PRI_LOW, priv->token, false); + if (ret < 0) + RTE_LOG(ERR, PMD, "Unable to disable multicast mode %d\n", ret); +} + +static int +dpaa2_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) +{ + int ret; + struct dpaa2_dev_priv *priv = dev->data->dev_private; + struct fsl_mc_io *dpni = (struct fsl_mc_io *)priv->hw; + uint32_t frame_size = mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; + + PMD_INIT_FUNC_TRACE(); + + if (dpni == NULL) { + RTE_LOG(ERR, PMD, "dpni is NULL\n"); + return -EINVAL; + } + + /* check that mtu is within the allowed range */ + if ((mtu < ETHER_MIN_MTU) || (frame_size > DPAA2_MAX_RX_PKT_LEN)) + return -EINVAL; + + if (frame_size > ETHER_MAX_LEN) + dev->data->dev_conf.rxmode.jumbo_frame = 1; + else + dev->data->dev_conf.rxmode.jumbo_frame = 0; + + /* Set the Max Rx frame length as 'mtu' + + * Maximum Ethernet header length + */ + ret = dpni_set_max_frame_length(dpni, CMD_PRI_LOW, priv->token, + mtu + ETH_VLAN_HLEN); + if (ret) { + PMD_DRV_LOG(ERR, "setting the max frame length failed"); + return -1; + } + PMD_DRV_LOG(INFO, "MTU is configured %d for the device", mtu); + return 0; +} + +static int +dpaa2_dev_add_mac_addr(struct rte_eth_dev *dev, + struct ether_addr *addr, + __rte_unused uint32_t index, + __rte_unused uint32_t pool) +{ + int ret; + struct dpaa2_dev_priv *priv = dev->data->dev_private; + struct fsl_mc_io *dpni = (struct fsl_mc_io *)priv->hw; + + PMD_INIT_FUNC_TRACE(); + + if (dpni == NULL) { + RTE_LOG(ERR, PMD, "dpni is NULL\n"); + return -1; + } + + ret = dpni_add_mac_addr(dpni, CMD_PRI_LOW, + priv->token, addr->addr_bytes); + if (ret) + RTE_LOG(ERR, PMD, + "error: Adding the MAC ADDR failed: err = %d\n", ret); + return 0; +} + +static void +dpaa2_dev_remove_mac_addr(struct rte_eth_dev *dev, + uint32_t index) +{ + int ret; + struct dpaa2_dev_priv *priv = dev->data->dev_private; + struct fsl_mc_io *dpni = (struct fsl_mc_io *)priv->hw; + struct rte_eth_dev_data *data = dev->data; + struct ether_addr *macaddr; + + PMD_INIT_FUNC_TRACE(); + + macaddr = &data->mac_addrs[index]; + + if (dpni == NULL) { + RTE_LOG(ERR, PMD, "dpni is NULL\n"); + return; + } + + ret = dpni_remove_mac_addr(dpni, CMD_PRI_LOW, + priv->token, macaddr->addr_bytes); + if (ret) + RTE_LOG(ERR, PMD, + "error: Removing the MAC ADDR failed: err = %d\n", ret); +} + +static void +dpaa2_dev_set_mac_addr(struct rte_eth_dev *dev, + struct ether_addr *addr) +{ + int ret; + struct dpaa2_dev_priv *priv = dev->data->dev_private; + struct fsl_mc_io *dpni = (struct fsl_mc_io *)priv->hw; + + PMD_INIT_FUNC_TRACE(); + + if (dpni == NULL) { + RTE_LOG(ERR, PMD, "dpni is NULL\n"); + return; + } + + ret = dpni_set_primary_mac_addr(dpni, CMD_PRI_LOW, + priv->token, addr->addr_bytes); + + if (ret) + RTE_LOG(ERR, PMD, + "error: Setting the MAC ADDR failed %d\n", ret); +} +static +int dpaa2_dev_stats_get(struct rte_eth_dev *dev, + struct rte_eth_stats *stats) +{ + struct dpaa2_dev_priv *priv = dev->data->dev_private; + struct fsl_mc_io *dpni = (struct fsl_mc_io *)priv->hw; + int32_t retcode; + uint8_t page0 = 0, page1 = 1, page2 = 2; + union dpni_statistics value; + + memset(&value, 0, sizeof(union dpni_statistics)); + + PMD_INIT_FUNC_TRACE(); + + if (!dpni) { + RTE_LOG(ERR, PMD, "dpni is NULL\n"); + return -EINVAL; + } + + if (!stats) { + RTE_LOG(ERR, PMD, "stats is NULL\n"); + return -EINVAL; + } + + /*Get Counters from page_0*/ + retcode = dpni_get_statistics(dpni, CMD_PRI_LOW, priv->token, + page0, 0, &value); + if (retcode) + goto err; + + stats->ipackets = value.page_0.ingress_all_frames; + stats->ibytes = value.page_0.ingress_all_bytes; + + /*Get Counters from page_1*/ + retcode = dpni_get_statistics(dpni, CMD_PRI_LOW, priv->token, + page1, 0, &value); + if (retcode) + goto err; + + stats->opackets = value.page_1.egress_all_frames; + stats->obytes = value.page_1.egress_all_bytes; + + /*Get Counters from page_2*/ + retcode = dpni_get_statistics(dpni, CMD_PRI_LOW, priv->token, + page2, 0, &value); + if (retcode) + goto err; + + /* Ingress drop frame count due to configured rules */ + stats->ierrors = value.page_2.ingress_filtered_frames; + /* Ingress drop frame count due to error */ + stats->ierrors += value.page_2.ingress_discarded_frames; + + stats->oerrors = value.page_2.egress_discarded_frames; + stats->imissed = value.page_2.ingress_nobuffer_discards; + + return 0; + +err: + RTE_LOG(ERR, PMD, "Operation not completed:Error Code = %d\n", retcode); + return retcode; +}; + +static int +dpaa2_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, + unsigned int n) +{ + struct dpaa2_dev_priv *priv = dev->data->dev_private; + struct fsl_mc_io *dpni = (struct fsl_mc_io *)priv->hw; + int32_t retcode; + union dpni_statistics value[3] = {}; + unsigned int i = 0, num = RTE_DIM(dpaa2_xstats_strings); + + if (xstats == NULL) + return 0; + + if (n < num) + return num; + + /* Get Counters from page_0*/ + retcode = dpni_get_statistics(dpni, CMD_PRI_LOW, priv->token, + 0, 0, &value[0]); + if (retcode) + goto err; + + /* Get Counters from page_1*/ + retcode = dpni_get_statistics(dpni, CMD_PRI_LOW, priv->token, + 1, 0, &value[1]); + if (retcode) + goto err; + + /* Get Counters from page_2*/ + retcode = dpni_get_statistics(dpni, CMD_PRI_LOW, priv->token, + 2, 0, &value[2]); + if (retcode) + goto err; + + for (i = 0; i < num; i++) { + xstats[i].id = i; + xstats[i].value = value[dpaa2_xstats_strings[i].page_id]. + raw.counter[dpaa2_xstats_strings[i].stats_id]; + } + return i; +err: + RTE_LOG(ERR, PMD, "Error in obtaining extended stats (%d)\n", retcode); + return retcode; +} + +static int +dpaa2_xstats_get_names(__rte_unused struct rte_eth_dev *dev, + struct rte_eth_xstat_name *xstats_names, + __rte_unused unsigned int limit) +{ + unsigned int i, stat_cnt = RTE_DIM(dpaa2_xstats_strings); + + if (xstats_names != NULL) + for (i = 0; i < stat_cnt; i++) + snprintf(xstats_names[i].name, + sizeof(xstats_names[i].name), + "%s", + dpaa2_xstats_strings[i].name); + + return stat_cnt; +} + +static int +dpaa2_xstats_get_by_id(struct rte_eth_dev *dev, const uint64_t *ids, + uint64_t *values, unsigned int n) +{ + unsigned int i, stat_cnt = RTE_DIM(dpaa2_xstats_strings); + uint64_t values_copy[stat_cnt]; + + if (!ids) { + struct dpaa2_dev_priv *priv = dev->data->dev_private; + struct fsl_mc_io *dpni = (struct fsl_mc_io *)priv->hw; + int32_t retcode; + union dpni_statistics value[3] = {}; + + if (n < stat_cnt) + return stat_cnt; + + if (!values) + return 0; + + /* Get Counters from page_0*/ + retcode = dpni_get_statistics(dpni, CMD_PRI_LOW, priv->token, + 0, 0, &value[0]); + if (retcode) + return 0; + + /* Get Counters from page_1*/ + retcode = dpni_get_statistics(dpni, CMD_PRI_LOW, priv->token, + 1, 0, &value[1]); + if (retcode) + return 0; + + /* Get Counters from page_2*/ + retcode = dpni_get_statistics(dpni, CMD_PRI_LOW, priv->token, + 2, 0, &value[2]); + if (retcode) + return 0; + + for (i = 0; i < stat_cnt; i++) { + values[i] = value[dpaa2_xstats_strings[i].page_id]. + raw.counter[dpaa2_xstats_strings[i].stats_id]; + } + return stat_cnt; + } + + dpaa2_xstats_get_by_id(dev, NULL, values_copy, stat_cnt); + + for (i = 0; i < n; i++) { + if (ids[i] >= stat_cnt) { + PMD_INIT_LOG(ERR, "id value isn't valid"); + return -1; + } + values[i] = values_copy[ids[i]]; + } + return n; +} + +static int +dpaa2_xstats_get_names_by_id( + struct rte_eth_dev *dev, + struct rte_eth_xstat_name *xstats_names, + const uint64_t *ids, + unsigned int limit) +{ + unsigned int i, stat_cnt = RTE_DIM(dpaa2_xstats_strings); + struct rte_eth_xstat_name xstats_names_copy[stat_cnt]; + + if (!ids) + return dpaa2_xstats_get_names(dev, xstats_names, limit); + + dpaa2_xstats_get_names(dev, xstats_names_copy, limit); + + for (i = 0; i < limit; i++) { + if (ids[i] >= stat_cnt) { + PMD_INIT_LOG(ERR, "id value isn't valid"); + return -1; + } + strcpy(xstats_names[i].name, xstats_names_copy[ids[i]].name); + } + return limit; +} + +static void +dpaa2_dev_stats_reset(struct rte_eth_dev *dev) +{ + struct dpaa2_dev_priv *priv = dev->data->dev_private; + struct fsl_mc_io *dpni = (struct fsl_mc_io *)priv->hw; + int32_t retcode; + + PMD_INIT_FUNC_TRACE(); + + if (dpni == NULL) { + RTE_LOG(ERR, PMD, "dpni is NULL\n"); + return; + } + + retcode = dpni_reset_statistics(dpni, CMD_PRI_LOW, priv->token); + if (retcode) + goto error; + + return; + +error: + RTE_LOG(ERR, PMD, "Operation not completed:Error Code = %d\n", retcode); + return; +}; + +/* return 0 means link status changed, -1 means not changed */ +static int +dpaa2_dev_link_update(struct rte_eth_dev *dev, + int wait_to_complete __rte_unused) +{ + int ret; + struct dpaa2_dev_priv *priv = dev->data->dev_private; + struct fsl_mc_io *dpni = (struct fsl_mc_io *)priv->hw; + struct rte_eth_link link, old; + struct dpni_link_state state = {0}; + + if (dpni == NULL) { + RTE_LOG(ERR, PMD, "dpni is NULL\n"); + return 0; + } + memset(&old, 0, sizeof(old)); + dpaa2_dev_atomic_read_link_status(dev, &old); + + ret = dpni_get_link_state(dpni, CMD_PRI_LOW, priv->token, &state); + if (ret < 0) { + RTE_LOG(ERR, PMD, "error: dpni_get_link_state %d\n", ret); + return -1; + } + + if ((old.link_status == state.up) && (old.link_speed == state.rate)) { + RTE_LOG(DEBUG, PMD, "No change in status\n"); + return -1; + } + + memset(&link, 0, sizeof(struct rte_eth_link)); + link.link_status = state.up; + link.link_speed = state.rate; + + if (state.options & DPNI_LINK_OPT_HALF_DUPLEX) + link.link_duplex = ETH_LINK_HALF_DUPLEX; + else + link.link_duplex = ETH_LINK_FULL_DUPLEX; + + dpaa2_dev_atomic_write_link_status(dev, &link); + + if (link.link_status) + PMD_DRV_LOG(INFO, "Port %d Link is Up\n", dev->data->port_id); + else + PMD_DRV_LOG(INFO, "Port %d Link is Down", dev->data->port_id); + return 0; +} + +/** + * Toggle the DPNI to enable, if not already enabled. + * This is not strictly PHY up/down - it is more of logical toggling. + */ +static int +dpaa2_dev_set_link_up(struct rte_eth_dev *dev) +{ + int ret = -EINVAL; + struct dpaa2_dev_priv *priv; + struct fsl_mc_io *dpni; + int en = 0; + struct dpni_link_state state = {0}; + + priv = dev->data->dev_private; + dpni = (struct fsl_mc_io *)priv->hw; + + if (dpni == NULL) { + RTE_LOG(ERR, PMD, "DPNI is NULL\n"); + return ret; + } + + /* Check if DPNI is currently enabled */ + ret = dpni_is_enabled(dpni, CMD_PRI_LOW, priv->token, &en); + if (ret) { + /* Unable to obtain dpni status; Not continuing */ + PMD_DRV_LOG(ERR, "Interface Link UP failed (%d)", ret); + return -EINVAL; + } + + /* Enable link if not already enabled */ + if (!en) { + ret = dpni_enable(dpni, CMD_PRI_LOW, priv->token); + if (ret) { + PMD_DRV_LOG(ERR, "Interface Link UP failed (%d)", ret); + return -EINVAL; + } + } + ret = dpni_get_link_state(dpni, CMD_PRI_LOW, priv->token, &state); + if (ret < 0) { + RTE_LOG(ERR, PMD, "error: dpni_get_link_state %d\n", ret); + return -1; + } + + /* changing tx burst function to start enqueues */ + dev->tx_pkt_burst = dpaa2_dev_tx; + dev->data->dev_link.link_status = state.up; + + if (state.up) + PMD_DRV_LOG(INFO, "Port %d Link is set as UP", + dev->data->port_id); + else + PMD_DRV_LOG(INFO, "Port %d Link is DOWN", dev->data->port_id); + return ret; +} + +/** + * Toggle the DPNI to disable, if not already disabled. + * This is not strictly PHY up/down - it is more of logical toggling. + */ +static int +dpaa2_dev_set_link_down(struct rte_eth_dev *dev) +{ + int ret = -EINVAL; + struct dpaa2_dev_priv *priv; + struct fsl_mc_io *dpni; + int dpni_enabled = 0; + int retries = 10; + + PMD_INIT_FUNC_TRACE(); + + priv = dev->data->dev_private; + dpni = (struct fsl_mc_io *)priv->hw; + + if (dpni == NULL) { + RTE_LOG(ERR, PMD, "Device has not yet been configured\n"); + return ret; + } + + /*changing tx burst function to avoid any more enqueues */ + dev->tx_pkt_burst = dummy_dev_tx; + + /* Loop while dpni_disable() attempts to drain the egress FQs + * and confirm them back to us. + */ + do { + ret = dpni_disable(dpni, 0, priv->token); + if (ret) { + PMD_DRV_LOG(ERR, "dpni disable failed (%d)", ret); + return ret; + } + ret = dpni_is_enabled(dpni, 0, priv->token, &dpni_enabled); + if (ret) { + PMD_DRV_LOG(ERR, "dpni_is_enabled failed (%d)", ret); + return ret; + } + if (dpni_enabled) + /* Allow the MC some slack */ + rte_delay_us(100 * 1000); + } while (dpni_enabled && --retries); + + if (!retries) { + PMD_DRV_LOG(WARNING, "Retry count exceeded disabling DPNI\n"); + /* todo- we may have to manually cleanup queues. + */ + } else { + PMD_DRV_LOG(INFO, "Port %d Link DOWN successful", + dev->data->port_id); + } + + dev->data->dev_link.link_status = 0; + + return ret; +} + +static int +dpaa2_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) +{ + int ret = -EINVAL; + struct dpaa2_dev_priv *priv; + struct fsl_mc_io *dpni; + struct dpni_link_state state = {0}; + + PMD_INIT_FUNC_TRACE(); + + priv = dev->data->dev_private; + dpni = (struct fsl_mc_io *)priv->hw; + + if (dpni == NULL || fc_conf == NULL) { + RTE_LOG(ERR, PMD, "device not configured\n"); + return ret; + } + + ret = dpni_get_link_state(dpni, CMD_PRI_LOW, priv->token, &state); + if (ret) { + RTE_LOG(ERR, PMD, "error: dpni_get_link_state %d\n", ret); + return ret; + } + + memset(fc_conf, 0, sizeof(struct rte_eth_fc_conf)); + if (state.options & DPNI_LINK_OPT_PAUSE) { + /* DPNI_LINK_OPT_PAUSE set + * if ASYM_PAUSE not set, + * RX Side flow control (handle received Pause frame) + * TX side flow control (send Pause frame) + * if ASYM_PAUSE set, + * RX Side flow control (handle received Pause frame) + * No TX side flow control (send Pause frame disabled) + */ + if (!(state.options & DPNI_LINK_OPT_ASYM_PAUSE)) + fc_conf->mode = RTE_FC_FULL; + else + fc_conf->mode = RTE_FC_RX_PAUSE; + } else { + /* DPNI_LINK_OPT_PAUSE not set + * if ASYM_PAUSE set, + * TX side flow control (send Pause frame) + * No RX side flow control (No action on pause frame rx) + * if ASYM_PAUSE not set, + * Flow control disabled + */ + if (state.options & DPNI_LINK_OPT_ASYM_PAUSE) + fc_conf->mode = RTE_FC_TX_PAUSE; + else + fc_conf->mode = RTE_FC_NONE; + } + + return ret; +} + +static int +dpaa2_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) +{ + int ret = -EINVAL; + struct dpaa2_dev_priv *priv; + struct fsl_mc_io *dpni; + struct dpni_link_state state = {0}; + struct dpni_link_cfg cfg = {0}; + + PMD_INIT_FUNC_TRACE(); + + priv = dev->data->dev_private; + dpni = (struct fsl_mc_io *)priv->hw; + + if (dpni == NULL) { + RTE_LOG(ERR, PMD, "dpni is NULL\n"); + return ret; + } + + /* It is necessary to obtain the current state before setting fc_conf + * as MC would return error in case rate, autoneg or duplex values are + * different. + */ + ret = dpni_get_link_state(dpni, CMD_PRI_LOW, priv->token, &state); + if (ret) { + RTE_LOG(ERR, PMD, "Unable to get link state (err=%d)\n", ret); + return -1; + } + + /* Disable link before setting configuration */ + dpaa2_dev_set_link_down(dev); + + /* Based on fc_conf, update cfg */ + cfg.rate = state.rate; + cfg.options = state.options; + + /* update cfg with fc_conf */ + switch (fc_conf->mode) { + case RTE_FC_FULL: + /* Full flow control; + * OPT_PAUSE set, ASYM_PAUSE not set + */ + cfg.options |= DPNI_LINK_OPT_PAUSE; + cfg.options &= ~DPNI_LINK_OPT_ASYM_PAUSE; + break; + case RTE_FC_TX_PAUSE: + /* Enable RX flow control + * OPT_PAUSE not set; + * ASYM_PAUSE set; + */ + cfg.options |= DPNI_LINK_OPT_ASYM_PAUSE; + cfg.options &= ~DPNI_LINK_OPT_PAUSE; + break; + case RTE_FC_RX_PAUSE: + /* Enable TX Flow control + * OPT_PAUSE set + * ASYM_PAUSE set + */ + cfg.options |= DPNI_LINK_OPT_PAUSE; + cfg.options |= DPNI_LINK_OPT_ASYM_PAUSE; + break; + case RTE_FC_NONE: + /* Disable Flow control + * OPT_PAUSE not set + * ASYM_PAUSE not set + */ + cfg.options &= ~DPNI_LINK_OPT_PAUSE; + cfg.options &= ~DPNI_LINK_OPT_ASYM_PAUSE; + break; + default: + RTE_LOG(ERR, PMD, "Incorrect Flow control flag (%d)\n", + fc_conf->mode); + return -1; + } + + ret = dpni_set_link_cfg(dpni, CMD_PRI_LOW, priv->token, &cfg); + if (ret) + RTE_LOG(ERR, PMD, + "Unable to set Link configuration (err=%d)\n", + ret); + + /* Enable link */ + dpaa2_dev_set_link_up(dev); + + return ret; +} + +static int +dpaa2_dev_rss_hash_update(struct rte_eth_dev *dev, + struct rte_eth_rss_conf *rss_conf) +{ + struct rte_eth_dev_data *data = dev->data; + struct rte_eth_conf *eth_conf = &data->dev_conf; + int ret; + + PMD_INIT_FUNC_TRACE(); + + if (rss_conf->rss_hf) { + ret = dpaa2_setup_flow_dist(dev, rss_conf->rss_hf); + if (ret) { + PMD_INIT_LOG(ERR, "unable to set flow dist"); + return ret; + } + } else { + ret = dpaa2_remove_flow_dist(dev, 0); + if (ret) { + PMD_INIT_LOG(ERR, "unable to remove flow dist"); + return ret; + } + } + eth_conf->rx_adv_conf.rss_conf.rss_hf = rss_conf->rss_hf; + return 0; +} + +static int +dpaa2_dev_rss_hash_conf_get(struct rte_eth_dev *dev, + struct rte_eth_rss_conf *rss_conf) +{ + struct rte_eth_dev_data *data = dev->data; + struct rte_eth_conf *eth_conf = &data->dev_conf; + + /* dpaa2 does not support rss_key, so length should be 0*/ + rss_conf->rss_key_len = 0; + rss_conf->rss_hf = eth_conf->rx_adv_conf.rss_conf.rss_hf; + return 0; +} + +int dpaa2_eth_eventq_attach(const struct rte_eth_dev *dev, + int eth_rx_queue_id, + uint16_t dpcon_id, + const struct rte_event_eth_rx_adapter_queue_conf *queue_conf) +{ + struct dpaa2_dev_priv *eth_priv = dev->data->dev_private; + struct fsl_mc_io *dpni = (struct fsl_mc_io *)eth_priv->hw; + struct dpaa2_queue *dpaa2_ethq = eth_priv->rx_vq[eth_rx_queue_id]; + uint8_t flow_id = dpaa2_ethq->flow_id; + struct dpni_queue cfg; + uint8_t options; + int ret; + + if (queue_conf->ev.sched_type == RTE_SCHED_TYPE_PARALLEL) + dpaa2_ethq->cb = dpaa2_dev_process_parallel_event; + else + return -EINVAL; + + memset(&cfg, 0, sizeof(struct dpni_queue)); + options = DPNI_QUEUE_OPT_DEST; + cfg.destination.type = DPNI_DEST_DPCON; + cfg.destination.id = dpcon_id; + cfg.destination.priority = queue_conf->ev.priority; + + options |= DPNI_QUEUE_OPT_USER_CTX; + cfg.user_context = (uint64_t)(dpaa2_ethq); + + ret = dpni_set_queue(dpni, CMD_PRI_LOW, eth_priv->token, DPNI_QUEUE_RX, + dpaa2_ethq->tc_index, flow_id, options, &cfg); + if (ret) { + RTE_LOG(ERR, PMD, "Error in dpni_set_queue: ret: %d\n", ret); + return ret; + } + + memcpy(&dpaa2_ethq->ev, &queue_conf->ev, sizeof(struct rte_event)); + + return 0; +} + +int dpaa2_eth_eventq_detach(const struct rte_eth_dev *dev, + int eth_rx_queue_id) +{ + struct dpaa2_dev_priv *eth_priv = dev->data->dev_private; + struct fsl_mc_io *dpni = (struct fsl_mc_io *)eth_priv->hw; + struct dpaa2_queue *dpaa2_ethq = eth_priv->rx_vq[eth_rx_queue_id]; + uint8_t flow_id = dpaa2_ethq->flow_id; + struct dpni_queue cfg; + uint8_t options; + int ret; + + memset(&cfg, 0, sizeof(struct dpni_queue)); + options = DPNI_QUEUE_OPT_DEST; + cfg.destination.type = DPNI_DEST_NONE; + + ret = dpni_set_queue(dpni, CMD_PRI_LOW, eth_priv->token, DPNI_QUEUE_RX, + dpaa2_ethq->tc_index, flow_id, options, &cfg); + if (ret) + RTE_LOG(ERR, PMD, "Error in dpni_set_queue: ret: %d\n", ret); + + return ret; +} + +static struct eth_dev_ops dpaa2_ethdev_ops = { + .dev_configure = dpaa2_eth_dev_configure, + .dev_start = dpaa2_dev_start, + .dev_stop = dpaa2_dev_stop, + .dev_close = dpaa2_dev_close, + .promiscuous_enable = dpaa2_dev_promiscuous_enable, + .promiscuous_disable = dpaa2_dev_promiscuous_disable, + .allmulticast_enable = dpaa2_dev_allmulticast_enable, + .allmulticast_disable = dpaa2_dev_allmulticast_disable, + .dev_set_link_up = dpaa2_dev_set_link_up, + .dev_set_link_down = dpaa2_dev_set_link_down, + .link_update = dpaa2_dev_link_update, + .stats_get = dpaa2_dev_stats_get, + .xstats_get = dpaa2_dev_xstats_get, + .xstats_get_by_id = dpaa2_xstats_get_by_id, + .xstats_get_names_by_id = dpaa2_xstats_get_names_by_id, + .xstats_get_names = dpaa2_xstats_get_names, + .stats_reset = dpaa2_dev_stats_reset, + .xstats_reset = dpaa2_dev_stats_reset, + .fw_version_get = dpaa2_fw_version_get, + .dev_infos_get = dpaa2_dev_info_get, + .dev_supported_ptypes_get = dpaa2_supported_ptypes_get, + .mtu_set = dpaa2_dev_mtu_set, + .vlan_filter_set = dpaa2_vlan_filter_set, + .vlan_offload_set = dpaa2_vlan_offload_set, + .rx_queue_setup = dpaa2_dev_rx_queue_setup, + .rx_queue_release = dpaa2_dev_rx_queue_release, + .tx_queue_setup = dpaa2_dev_tx_queue_setup, + .tx_queue_release = dpaa2_dev_tx_queue_release, + .flow_ctrl_get = dpaa2_flow_ctrl_get, + .flow_ctrl_set = dpaa2_flow_ctrl_set, + .mac_addr_add = dpaa2_dev_add_mac_addr, + .mac_addr_remove = dpaa2_dev_remove_mac_addr, + .mac_addr_set = dpaa2_dev_set_mac_addr, + .rss_hash_update = dpaa2_dev_rss_hash_update, + .rss_hash_conf_get = dpaa2_dev_rss_hash_conf_get, +}; + +static int +dpaa2_dev_init(struct rte_eth_dev *eth_dev) +{ + struct rte_device *dev = eth_dev->device; + struct rte_dpaa2_device *dpaa2_dev; + struct fsl_mc_io *dpni_dev; + struct dpni_attr attr; + struct dpaa2_dev_priv *priv = eth_dev->data->dev_private; + struct dpni_buffer_layout layout; + int ret, hw_id; + + PMD_INIT_FUNC_TRACE(); + + /* For secondary processes, the primary has done all the work */ + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return 0; + + dpaa2_dev = container_of(dev, struct rte_dpaa2_device, device); + + hw_id = dpaa2_dev->object_id; + + dpni_dev = rte_malloc(NULL, sizeof(struct fsl_mc_io), 0); + if (!dpni_dev) { + PMD_INIT_LOG(ERR, "malloc failed for dpni device\n"); + return -1; + } + + dpni_dev->regs = rte_mcp_ptr_list[0]; + ret = dpni_open(dpni_dev, CMD_PRI_LOW, hw_id, &priv->token); + if (ret) { + PMD_INIT_LOG(ERR, + "Failure in opening dpni@%d with err code %d\n", + hw_id, ret); + rte_free(dpni_dev); + return -1; + } + + /* Clean the device first */ + ret = dpni_reset(dpni_dev, CMD_PRI_LOW, priv->token); + if (ret) { + PMD_INIT_LOG(ERR, + "Failure cleaning dpni@%d with err code %d\n", + hw_id, ret); + goto init_err; + } + + ret = dpni_get_attributes(dpni_dev, CMD_PRI_LOW, priv->token, &attr); + if (ret) { + PMD_INIT_LOG(ERR, + "Failure in get dpni@%d attribute, err code %d\n", + hw_id, ret); + goto init_err; + } + + priv->num_rx_tc = attr.num_rx_tcs; + + /* Resetting the "num_rx_queues" to equal number of queues in first TC + * as only one TC is supported on Rx Side. Once Multiple TCs will be + * in use for Rx processing then this will be changed or removed. + */ + priv->nb_rx_queues = attr.num_queues; + + /* Using number of TX queues as number of TX TCs */ + priv->nb_tx_queues = attr.num_tx_tcs; + + PMD_DRV_LOG(DEBUG, "RX-TC= %d, nb_rx_queues= %d, nb_tx_queues=%d", + priv->num_rx_tc, priv->nb_rx_queues, priv->nb_tx_queues); + + priv->hw = dpni_dev; + priv->hw_id = hw_id; + priv->options = attr.options; + priv->max_mac_filters = attr.mac_filter_entries; + priv->max_vlan_filters = attr.vlan_filter_entries; + priv->flags = 0; + + /* Allocate memory for hardware structure for queues */ + ret = dpaa2_alloc_rx_tx_queues(eth_dev); + if (ret) { + PMD_INIT_LOG(ERR, "dpaa2_alloc_rx_tx_queuesFailed\n"); + goto init_err; + } + + /* Allocate memory for storing MAC addresses */ + eth_dev->data->mac_addrs = rte_zmalloc("dpni", + ETHER_ADDR_LEN * attr.mac_filter_entries, 0); + if (eth_dev->data->mac_addrs == NULL) { + PMD_INIT_LOG(ERR, + "Failed to allocate %d bytes needed to store MAC addresses", + ETHER_ADDR_LEN * attr.mac_filter_entries); + ret = -ENOMEM; + goto init_err; + } + + ret = dpni_get_primary_mac_addr(dpni_dev, CMD_PRI_LOW, + priv->token, + (uint8_t *)(eth_dev->data->mac_addrs[0].addr_bytes)); + if (ret) { + PMD_INIT_LOG(ERR, "DPNI get mac address failed:Err Code = %d\n", + ret); + goto init_err; + } + + /* ... tx buffer layout ... */ + memset(&layout, 0, sizeof(struct dpni_buffer_layout)); + layout.options = DPNI_BUF_LAYOUT_OPT_FRAME_STATUS; + layout.pass_frame_status = 1; + ret = dpni_set_buffer_layout(dpni_dev, CMD_PRI_LOW, priv->token, + DPNI_QUEUE_TX, &layout); + if (ret) { + PMD_INIT_LOG(ERR, "Error (%d) in setting tx buffer layout", + ret); + goto init_err; + } + + /* ... tx-conf and error buffer layout ... */ + memset(&layout, 0, sizeof(struct dpni_buffer_layout)); + layout.options = DPNI_BUF_LAYOUT_OPT_FRAME_STATUS; + layout.pass_frame_status = 1; + ret = dpni_set_buffer_layout(dpni_dev, CMD_PRI_LOW, priv->token, + DPNI_QUEUE_TX_CONFIRM, &layout); + if (ret) { + PMD_INIT_LOG(ERR, "Error (%d) in setting tx-conf buffer layout", + ret); + goto init_err; + } + + eth_dev->dev_ops = &dpaa2_ethdev_ops; + eth_dev->data->dev_flags |= RTE_ETH_DEV_INTR_LSC; + + eth_dev->rx_pkt_burst = dpaa2_dev_prefetch_rx; + eth_dev->tx_pkt_burst = dpaa2_dev_tx; + rte_fslmc_vfio_dmamap(); + + RTE_LOG(INFO, PMD, "%s: netdev created\n", eth_dev->data->name); + return 0; +init_err: + dpaa2_dev_uninit(eth_dev); + return ret; +} + +static int +dpaa2_dev_uninit(struct rte_eth_dev *eth_dev) +{ + struct dpaa2_dev_priv *priv = eth_dev->data->dev_private; + struct fsl_mc_io *dpni = (struct fsl_mc_io *)priv->hw; + int i, ret; + struct dpaa2_queue *dpaa2_q; + + PMD_INIT_FUNC_TRACE(); + + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return 0; + + if (!dpni) { + PMD_INIT_LOG(WARNING, "Already closed or not started"); + return -1; + } + + dpaa2_dev_close(eth_dev); + + if (priv->rx_vq[0]) { + /* cleaning up queue storage */ + for (i = 0; i < priv->nb_rx_queues; i++) { + dpaa2_q = (struct dpaa2_queue *)priv->rx_vq[i]; + if (dpaa2_q->q_storage) + rte_free(dpaa2_q->q_storage); + } + /*free the all queue memory */ + rte_free(priv->rx_vq[0]); + priv->rx_vq[0] = NULL; + } + + /* free memory for storing MAC addresses */ + if (eth_dev->data->mac_addrs) { + rte_free(eth_dev->data->mac_addrs); + eth_dev->data->mac_addrs = NULL; + } + + /* Close the device at underlying layer*/ + ret = dpni_close(dpni, CMD_PRI_LOW, priv->token); + if (ret) { + PMD_INIT_LOG(ERR, + "Failure closing dpni device with err code %d\n", + ret); + } + + /* Free the allocated memory for ethernet private data and dpni*/ + priv->hw = NULL; + rte_free(dpni); + + eth_dev->dev_ops = NULL; + eth_dev->rx_pkt_burst = NULL; + eth_dev->tx_pkt_burst = NULL; + + RTE_LOG(INFO, PMD, "%s: netdev created\n", eth_dev->data->name); + return 0; +} + +static int +rte_dpaa2_probe(struct rte_dpaa2_driver *dpaa2_drv, + struct rte_dpaa2_device *dpaa2_dev) +{ + struct rte_eth_dev *eth_dev; + int diag; + + if (rte_eal_process_type() == RTE_PROC_PRIMARY) { + eth_dev = rte_eth_dev_allocate(dpaa2_dev->device.name); + if (!eth_dev) + return -ENODEV; + eth_dev->data->dev_private = rte_zmalloc( + "ethdev private structure", + sizeof(struct dpaa2_dev_priv), + RTE_CACHE_LINE_SIZE); + if (eth_dev->data->dev_private == NULL) { + PMD_INIT_LOG(CRIT, "Cannot allocate memzone for" + " private port data\n"); + rte_eth_dev_release_port(eth_dev); + return -ENOMEM; + } + } else { + eth_dev = rte_eth_dev_attach_secondary(dpaa2_dev->device.name); + if (!eth_dev) + return -ENODEV; + } + + eth_dev->device = &dpaa2_dev->device; + eth_dev->device->driver = &dpaa2_drv->driver; + + dpaa2_dev->eth_dev = eth_dev; + eth_dev->data->rx_mbuf_alloc_failed = 0; + + /* Invoke PMD device initialization function */ + diag = dpaa2_dev_init(eth_dev); + if (diag == 0) + return 0; + + if (rte_eal_process_type() == RTE_PROC_PRIMARY) + rte_free(eth_dev->data->dev_private); + rte_eth_dev_release_port(eth_dev); + return diag; +} + +static int +rte_dpaa2_remove(struct rte_dpaa2_device *dpaa2_dev) +{ + struct rte_eth_dev *eth_dev; + + eth_dev = dpaa2_dev->eth_dev; + dpaa2_dev_uninit(eth_dev); + + if (rte_eal_process_type() == RTE_PROC_PRIMARY) + rte_free(eth_dev->data->dev_private); + rte_eth_dev_release_port(eth_dev); + + return 0; +} + +static struct rte_dpaa2_driver rte_dpaa2_pmd = { + .drv_type = DPAA2_ETH, + .probe = rte_dpaa2_probe, + .remove = rte_dpaa2_remove, +}; + +RTE_PMD_REGISTER_DPAA2(net_dpaa2, rte_dpaa2_pmd); diff --git a/dpdk/drivers/net/dpaa2/dpaa2_ethdev.h b/dpdk/drivers/net/dpaa2/dpaa2_ethdev.h new file mode 100644 index 00000000..b8e94aa3 --- /dev/null +++ b/dpdk/drivers/net/dpaa2/dpaa2_ethdev.h @@ -0,0 +1,122 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2015-2016 Freescale Semiconductor, Inc. All rights reserved. + * Copyright 2016 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Freescale Semiconductor, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _DPAA2_ETHDEV_H +#define _DPAA2_ETHDEV_H + +#include + +#include +#include + +#define DPAA2_MIN_RX_BUF_SIZE 512 +#define DPAA2_MAX_RX_PKT_LEN 10240 /*WRIOP support*/ + +#define MAX_TCS DPNI_MAX_TC +#define MAX_RX_QUEUES 16 +#define MAX_TX_QUEUES 16 + +/*default tc to be used for ,congestion, distribution etc configuration. */ +#define DPAA2_DEF_TC 0 + +/* Threshold for a Tx queue to *Enter* Congestion state. + */ +#define CONG_ENTER_TX_THRESHOLD 512 + +/* Threshold for a queue to *Exit* Congestion state. + */ +#define CONG_EXIT_TX_THRESHOLD 480 + +#define CONG_RETRY_COUNT 18000 + +/* RX queue tail drop threshold + * currently considering 32 KB packets + */ +#define CONG_THRESHOLD_RX_Q (64 * 1024) +#define CONG_RX_OAL 128 + +/* Size of the input SMMU mapped memory required by MC */ +#define DIST_PARAM_IOVA_SIZE 256 + +/* Enable TX Congestion control support + * default is disable + */ +#define DPAA2_TX_CGR_OFF 0x01 + +/* Disable RX tail drop, default is enable */ +#define DPAA2_RX_TAILDROP_OFF 0x04 + +struct dpaa2_dev_priv { + void *hw; + int32_t hw_id; + int32_t qdid; + uint16_t token; + uint8_t nb_tx_queues; + uint8_t nb_rx_queues; + void *rx_vq[MAX_RX_QUEUES]; + void *tx_vq[MAX_TX_QUEUES]; + + struct dpaa2_bp_list *bp_list; /** +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "dpaa2_ethdev.h" +#include "base/dpaa2_hw_dpni_annot.h" + +static inline uint32_t __attribute__((hot)) +dpaa2_dev_rx_parse(uint64_t hw_annot_addr) +{ + uint32_t pkt_type = RTE_PTYPE_UNKNOWN; + struct dpaa2_annot_hdr *annotation = + (struct dpaa2_annot_hdr *)hw_annot_addr; + + PMD_RX_LOG(DEBUG, "annotation = 0x%lx ", annotation->word4); + + if (BIT_ISSET_AT_POS(annotation->word3, L2_ARP_PRESENT)) { + pkt_type = RTE_PTYPE_L2_ETHER_ARP; + goto parse_done; + } else if (BIT_ISSET_AT_POS(annotation->word3, L2_ETH_MAC_PRESENT)) { + pkt_type = RTE_PTYPE_L2_ETHER; + } else { + goto parse_done; + } + + if (BIT_ISSET_AT_POS(annotation->word4, L3_IPV4_1_PRESENT | + L3_IPV4_N_PRESENT)) { + pkt_type |= RTE_PTYPE_L3_IPV4; + if (BIT_ISSET_AT_POS(annotation->word4, L3_IP_1_OPT_PRESENT | + L3_IP_N_OPT_PRESENT)) + pkt_type |= RTE_PTYPE_L3_IPV4_EXT; + + } else if (BIT_ISSET_AT_POS(annotation->word4, L3_IPV6_1_PRESENT | + L3_IPV6_N_PRESENT)) { + pkt_type |= RTE_PTYPE_L3_IPV6; + if (BIT_ISSET_AT_POS(annotation->word4, L3_IP_1_OPT_PRESENT | + L3_IP_N_OPT_PRESENT)) + pkt_type |= RTE_PTYPE_L3_IPV6_EXT; + } else { + goto parse_done; + } + + if (BIT_ISSET_AT_POS(annotation->word4, L3_IP_1_FIRST_FRAGMENT | + L3_IP_1_MORE_FRAGMENT | + L3_IP_N_FIRST_FRAGMENT | + L3_IP_N_MORE_FRAGMENT)) { + pkt_type |= RTE_PTYPE_L4_FRAG; + goto parse_done; + } else { + pkt_type |= RTE_PTYPE_L4_NONFRAG; + } + + if (BIT_ISSET_AT_POS(annotation->word4, L3_PROTO_UDP_PRESENT)) + pkt_type |= RTE_PTYPE_L4_UDP; + + else if (BIT_ISSET_AT_POS(annotation->word4, L3_PROTO_TCP_PRESENT)) + pkt_type |= RTE_PTYPE_L4_TCP; + + else if (BIT_ISSET_AT_POS(annotation->word4, L3_PROTO_SCTP_PRESENT)) + pkt_type |= RTE_PTYPE_L4_SCTP; + + else if (BIT_ISSET_AT_POS(annotation->word4, L3_PROTO_ICMP_PRESENT)) + pkt_type |= RTE_PTYPE_L4_ICMP; + + else if (BIT_ISSET_AT_POS(annotation->word4, L3_IP_UNKNOWN_PROTOCOL)) + pkt_type |= RTE_PTYPE_UNKNOWN; + +parse_done: + return pkt_type; +} + +static inline void __attribute__((hot)) +dpaa2_dev_rx_offload(uint64_t hw_annot_addr, struct rte_mbuf *mbuf) +{ + struct dpaa2_annot_hdr *annotation = + (struct dpaa2_annot_hdr *)hw_annot_addr; + + if (BIT_ISSET_AT_POS(annotation->word3, + L2_VLAN_1_PRESENT | L2_VLAN_N_PRESENT)) + mbuf->ol_flags |= PKT_RX_VLAN; + + if (BIT_ISSET_AT_POS(annotation->word8, DPAA2_ETH_FAS_L3CE)) + mbuf->ol_flags |= PKT_RX_IP_CKSUM_BAD; + + if (BIT_ISSET_AT_POS(annotation->word8, DPAA2_ETH_FAS_L4CE)) + mbuf->ol_flags |= PKT_RX_L4_CKSUM_BAD; +} + +static inline struct rte_mbuf *__attribute__((hot)) +eth_sg_fd_to_mbuf(const struct qbman_fd *fd) +{ + struct qbman_sge *sgt, *sge; + dma_addr_t sg_addr; + int i = 0; + uint64_t fd_addr; + struct rte_mbuf *first_seg, *next_seg, *cur_seg, *temp; + + fd_addr = (uint64_t)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)); + + /* Get Scatter gather table address */ + sgt = (struct qbman_sge *)(fd_addr + DPAA2_GET_FD_OFFSET(fd)); + + sge = &sgt[i++]; + sg_addr = (uint64_t)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FLE_ADDR(sge)); + + /* First Scatter gather entry */ + first_seg = DPAA2_INLINE_MBUF_FROM_BUF(sg_addr, + rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size); + /* Prepare all the metadata for first segment */ + first_seg->buf_addr = (uint8_t *)sg_addr; + first_seg->ol_flags = 0; + first_seg->data_off = DPAA2_GET_FLE_OFFSET(sge); + first_seg->data_len = sge->length & 0x1FFFF; + first_seg->pkt_len = DPAA2_GET_FD_LEN(fd); + first_seg->nb_segs = 1; + first_seg->next = NULL; + + first_seg->packet_type = dpaa2_dev_rx_parse( + (uint64_t)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)) + + DPAA2_FD_PTA_SIZE); + dpaa2_dev_rx_offload((uint64_t)DPAA2_IOVA_TO_VADDR( + DPAA2_GET_FD_ADDR(fd)) + + DPAA2_FD_PTA_SIZE, first_seg); + rte_mbuf_refcnt_set(first_seg, 1); + cur_seg = first_seg; + while (!DPAA2_SG_IS_FINAL(sge)) { + sge = &sgt[i++]; + sg_addr = (uint64_t)DPAA2_IOVA_TO_VADDR( + DPAA2_GET_FLE_ADDR(sge)); + next_seg = DPAA2_INLINE_MBUF_FROM_BUF(sg_addr, + rte_dpaa2_bpid_info[DPAA2_GET_FLE_BPID(sge)].meta_data_size); + next_seg->buf_addr = (uint8_t *)sg_addr; + next_seg->data_off = DPAA2_GET_FLE_OFFSET(sge); + next_seg->data_len = sge->length & 0x1FFFF; + first_seg->nb_segs += 1; + rte_mbuf_refcnt_set(next_seg, 1); + cur_seg->next = next_seg; + next_seg->next = NULL; + cur_seg = next_seg; + } + temp = DPAA2_INLINE_MBUF_FROM_BUF(fd_addr, + rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size); + rte_mbuf_refcnt_set(temp, 1); + rte_pktmbuf_free_seg(temp); + + return (void *)first_seg; +} + +static inline struct rte_mbuf *__attribute__((hot)) +eth_fd_to_mbuf(const struct qbman_fd *fd) +{ + struct rte_mbuf *mbuf = DPAA2_INLINE_MBUF_FROM_BUF( + DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)), + rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size); + + /* need to repopulated some of the fields, + * as they may have changed in last transmission + */ + mbuf->nb_segs = 1; + mbuf->ol_flags = 0; + mbuf->data_off = DPAA2_GET_FD_OFFSET(fd); + mbuf->data_len = DPAA2_GET_FD_LEN(fd); + mbuf->pkt_len = mbuf->data_len; + + /* Parse the packet */ + /* parse results are after the private - sw annotation area */ + mbuf->packet_type = dpaa2_dev_rx_parse( + (uint64_t)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)) + + DPAA2_FD_PTA_SIZE); + + dpaa2_dev_rx_offload((uint64_t)DPAA2_IOVA_TO_VADDR( + DPAA2_GET_FD_ADDR(fd)) + + DPAA2_FD_PTA_SIZE, mbuf); + + mbuf->next = NULL; + rte_mbuf_refcnt_set(mbuf, 1); + + PMD_RX_LOG(DEBUG, "to mbuf - mbuf =%p, mbuf->buf_addr =%p, off = %d," + "fd_off=%d fd =%lx, meta = %d bpid =%d, len=%d\n", + mbuf, mbuf->buf_addr, mbuf->data_off, + DPAA2_GET_FD_OFFSET(fd), DPAA2_GET_FD_ADDR(fd), + rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size, + DPAA2_GET_FD_BPID(fd), DPAA2_GET_FD_LEN(fd)); + + return mbuf; +} + +static int __attribute__ ((noinline)) __attribute__((hot)) +eth_mbuf_to_sg_fd(struct rte_mbuf *mbuf, + struct qbman_fd *fd, uint16_t bpid) +{ + struct rte_mbuf *cur_seg = mbuf, *prev_seg, *mi, *temp; + struct qbman_sge *sgt, *sge = NULL; + int i; + + /* First Prepare FD to be transmited*/ + /* Resetting the buffer pool id and offset field*/ + fd->simple.bpid_offset = 0; + + temp = rte_pktmbuf_alloc(mbuf->pool); + if (temp == NULL) { + PMD_TX_LOG(ERR, "No memory to allocate S/G table"); + return -ENOMEM; + } + + DPAA2_SET_FD_ADDR(fd, DPAA2_MBUF_VADDR_TO_IOVA(temp)); + DPAA2_SET_FD_LEN(fd, mbuf->pkt_len); + DPAA2_SET_FD_OFFSET(fd, temp->data_off); + DPAA2_SET_FD_BPID(fd, bpid); + DPAA2_SET_FD_ASAL(fd, DPAA2_ASAL_VAL); + DPAA2_FD_SET_FORMAT(fd, qbman_fd_sg); + /*Set Scatter gather table and Scatter gather entries*/ + sgt = (struct qbman_sge *)( + (uint64_t)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)) + + DPAA2_GET_FD_OFFSET(fd)); + + for (i = 0; i < mbuf->nb_segs; i++) { + sge = &sgt[i]; + /*Resetting the buffer pool id and offset field*/ + sge->fin_bpid_offset = 0; + DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(cur_seg)); + DPAA2_SET_FLE_OFFSET(sge, cur_seg->data_off); + sge->length = cur_seg->data_len; + if (RTE_MBUF_DIRECT(cur_seg)) { + if (rte_mbuf_refcnt_read(cur_seg) > 1) { + /* If refcnt > 1, invalid bpid is set to ensure + * buffer is not freed by HW + */ + DPAA2_SET_FLE_IVP(sge); + rte_mbuf_refcnt_update(cur_seg, -1); + } else + DPAA2_SET_FLE_BPID(sge, + mempool_to_bpid(cur_seg->pool)); + cur_seg = cur_seg->next; + } else { + /* Get owner MBUF from indirect buffer */ + mi = rte_mbuf_from_indirect(cur_seg); + if (rte_mbuf_refcnt_read(mi) > 1) { + /* If refcnt > 1, invalid bpid is set to ensure + * owner buffer is not freed by HW + */ + DPAA2_SET_FLE_IVP(sge); + } else { + DPAA2_SET_FLE_BPID(sge, + mempool_to_bpid(mi->pool)); + rte_mbuf_refcnt_update(mi, 1); + } + prev_seg = cur_seg; + cur_seg = cur_seg->next; + prev_seg->next = NULL; + rte_pktmbuf_free(prev_seg); + } + } + DPAA2_SG_SET_FINAL(sge, true); + return 0; +} + +static void +eth_mbuf_to_fd(struct rte_mbuf *mbuf, + struct qbman_fd *fd, uint16_t bpid) __attribute__((unused)); + +static void __attribute__ ((noinline)) __attribute__((hot)) +eth_mbuf_to_fd(struct rte_mbuf *mbuf, + struct qbman_fd *fd, uint16_t bpid) +{ + /*Resetting the buffer pool id and offset field*/ + fd->simple.bpid_offset = 0; + + DPAA2_SET_FD_ADDR(fd, DPAA2_MBUF_VADDR_TO_IOVA(mbuf)); + DPAA2_SET_FD_LEN(fd, mbuf->data_len); + DPAA2_SET_FD_BPID(fd, bpid); + DPAA2_SET_FD_OFFSET(fd, mbuf->data_off); + DPAA2_SET_FD_ASAL(fd, DPAA2_ASAL_VAL); + + PMD_TX_LOG(DEBUG, "mbuf =%p, mbuf->buf_addr =%p, off = %d," + "fd_off=%d fd =%lx, meta = %d bpid =%d, len=%d\n", + mbuf, mbuf->buf_addr, mbuf->data_off, + DPAA2_GET_FD_OFFSET(fd), DPAA2_GET_FD_ADDR(fd), + rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size, + DPAA2_GET_FD_BPID(fd), DPAA2_GET_FD_LEN(fd)); + if (RTE_MBUF_DIRECT(mbuf)) { + if (rte_mbuf_refcnt_read(mbuf) > 1) { + DPAA2_SET_FD_IVP(fd); + rte_mbuf_refcnt_update(mbuf, -1); + } + } else { + struct rte_mbuf *mi; + + mi = rte_mbuf_from_indirect(mbuf); + if (rte_mbuf_refcnt_read(mi) > 1) + DPAA2_SET_FD_IVP(fd); + else + rte_mbuf_refcnt_update(mi, 1); + rte_pktmbuf_free(mbuf); + } +} + +static inline int __attribute__((hot)) +eth_copy_mbuf_to_fd(struct rte_mbuf *mbuf, + struct qbman_fd *fd, uint16_t bpid) +{ + struct rte_mbuf *m; + void *mb = NULL; + + if (rte_dpaa2_mbuf_alloc_bulk( + rte_dpaa2_bpid_info[bpid].bp_list->mp, &mb, 1)) { + PMD_TX_LOG(WARNING, "Unable to allocated DPAA2 buffer"); + return -1; + } + m = (struct rte_mbuf *)mb; + memcpy((char *)m->buf_addr + mbuf->data_off, + (void *)((char *)mbuf->buf_addr + mbuf->data_off), + mbuf->pkt_len); + + /* Copy required fields */ + m->data_off = mbuf->data_off; + m->ol_flags = mbuf->ol_flags; + m->packet_type = mbuf->packet_type; + m->tx_offload = mbuf->tx_offload; + + /*Resetting the buffer pool id and offset field*/ + fd->simple.bpid_offset = 0; + + DPAA2_SET_FD_ADDR(fd, DPAA2_MBUF_VADDR_TO_IOVA(m)); + DPAA2_SET_FD_LEN(fd, mbuf->data_len); + DPAA2_SET_FD_BPID(fd, bpid); + DPAA2_SET_FD_OFFSET(fd, mbuf->data_off); + DPAA2_SET_FD_ASAL(fd, DPAA2_ASAL_VAL); + + PMD_TX_LOG(DEBUG, " mbuf %p BMAN buf addr %p", + (void *)mbuf, mbuf->buf_addr); + + PMD_TX_LOG(DEBUG, " fdaddr =%lx bpid =%d meta =%d off =%d, len =%d", + DPAA2_GET_FD_ADDR(fd), + DPAA2_GET_FD_BPID(fd), + rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size, + DPAA2_GET_FD_OFFSET(fd), + DPAA2_GET_FD_LEN(fd)); + + return 0; +} + +uint16_t +dpaa2_dev_prefetch_rx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) +{ + /* Function receive frames for a given device and VQ*/ + struct dpaa2_queue *dpaa2_q = (struct dpaa2_queue *)queue; + struct qbman_result *dq_storage; + uint32_t fqid = dpaa2_q->fqid; + int ret, num_rx = 0; + uint8_t is_last = 0, status; + struct qbman_swp *swp; + const struct qbman_fd *fd[DPAA2_DQRR_RING_SIZE]; + struct qbman_pull_desc pulldesc; + struct queue_storage_info_t *q_storage = dpaa2_q->q_storage; + struct rte_eth_dev *dev = dpaa2_q->dev; + + if (unlikely(!DPAA2_PER_LCORE_DPIO)) { + ret = dpaa2_affine_qbman_swp(); + if (ret) { + RTE_LOG(ERR, PMD, "Failure in affining portal\n"); + return 0; + } + } + swp = DPAA2_PER_LCORE_PORTAL; + if (!q_storage->active_dqs) { + q_storage->toggle = 0; + dq_storage = q_storage->dq_storage[q_storage->toggle]; + qbman_pull_desc_clear(&pulldesc); + qbman_pull_desc_set_numframes(&pulldesc, + (nb_pkts > DPAA2_DQRR_RING_SIZE) ? + DPAA2_DQRR_RING_SIZE : nb_pkts); + qbman_pull_desc_set_fq(&pulldesc, fqid); + qbman_pull_desc_set_storage(&pulldesc, dq_storage, + (dma_addr_t)(DPAA2_VADDR_TO_IOVA(dq_storage)), 1); + if (check_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index)) { + while (!qbman_check_command_complete( + get_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index))) + ; + clear_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index); + } + while (1) { + if (qbman_swp_pull(swp, &pulldesc)) { + PMD_RX_LOG(WARNING, "VDQ command is not issued." + "QBMAN is busy\n"); + /* Portal was busy, try again */ + continue; + } + break; + } + q_storage->active_dqs = dq_storage; + q_storage->active_dpio_id = DPAA2_PER_LCORE_DPIO->index; + set_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index, dq_storage); + } + dq_storage = q_storage->active_dqs; + /* Check if the previous issued command is completed. + * Also seems like the SWP is shared between the Ethernet Driver + * and the SEC driver. + */ + while (!qbman_check_command_complete(dq_storage)) + ; + if (dq_storage == get_swp_active_dqs(q_storage->active_dpio_id)) + clear_swp_active_dqs(q_storage->active_dpio_id); + while (!is_last) { + /* Loop until the dq_storage is updated with + * new token by QBMAN + */ + while (!qbman_check_new_result(dq_storage)) + ; + rte_prefetch0((void *)((uint64_t)(dq_storage + 1))); + /* Check whether Last Pull command is Expired and + * setting Condition for Loop termination + */ + if (qbman_result_DQ_is_pull_complete(dq_storage)) { + is_last = 1; + /* Check for valid frame. */ + status = (uint8_t)qbman_result_DQ_flags(dq_storage); + if (unlikely((status & QBMAN_DQ_STAT_VALIDFRAME) == 0)) + continue; + } + fd[num_rx] = qbman_result_DQ_fd(dq_storage); + + /* Prefetch Annotation address for the parse results */ + rte_prefetch0((void *)((uint64_t)DPAA2_GET_FD_ADDR(fd[num_rx]) + + DPAA2_FD_PTA_SIZE + 16)); + + if (unlikely(DPAA2_FD_GET_FORMAT(fd[num_rx]) == qbman_fd_sg)) + bufs[num_rx] = eth_sg_fd_to_mbuf(fd[num_rx]); + else + bufs[num_rx] = eth_fd_to_mbuf(fd[num_rx]); + bufs[num_rx]->port = dev->data->port_id; + + if (dev->data->dev_conf.rxmode.hw_vlan_strip) + rte_vlan_strip(bufs[num_rx]); + + dq_storage++; + num_rx++; + } + + if (check_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index)) { + while (!qbman_check_command_complete( + get_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index))) + ; + clear_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index); + } + q_storage->toggle ^= 1; + dq_storage = q_storage->dq_storage[q_storage->toggle]; + qbman_pull_desc_clear(&pulldesc); + qbman_pull_desc_set_numframes(&pulldesc, DPAA2_DQRR_RING_SIZE); + qbman_pull_desc_set_fq(&pulldesc, fqid); + qbman_pull_desc_set_storage(&pulldesc, dq_storage, + (dma_addr_t)(DPAA2_VADDR_TO_IOVA(dq_storage)), 1); + /* Issue a volatile dequeue command. */ + while (1) { + if (qbman_swp_pull(swp, &pulldesc)) { + PMD_RX_LOG(WARNING, "VDQ command is not issued." + "QBMAN is busy\n"); + continue; + } + break; + } + q_storage->active_dqs = dq_storage; + q_storage->active_dpio_id = DPAA2_PER_LCORE_DPIO->index; + set_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index, dq_storage); + + dpaa2_q->rx_pkts += num_rx; + + /* Return the total number of packets received to DPAA2 app */ + return num_rx; +} + +void __attribute__((hot)) +dpaa2_dev_process_parallel_event(struct qbman_swp *swp, + const struct qbman_fd *fd, + const struct qbman_result *dq, + struct dpaa2_queue *rxq, + struct rte_event *ev) +{ + ev->mbuf = eth_fd_to_mbuf(fd); + + ev->flow_id = rxq->ev.flow_id; + ev->sub_event_type = rxq->ev.sub_event_type; + ev->event_type = RTE_EVENT_TYPE_ETHDEV; + ev->op = RTE_EVENT_OP_NEW; + ev->sched_type = rxq->ev.sched_type; + ev->queue_id = rxq->ev.queue_id; + ev->priority = rxq->ev.priority; + + qbman_swp_dqrr_consume(swp, dq); +} + +/* + * Callback to handle sending packets through WRIOP based interface + */ +uint16_t +dpaa2_dev_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) +{ + /* Function to transmit the frames to given device and VQ*/ + uint32_t loop, retry_count; + int32_t ret; + struct qbman_fd fd_arr[MAX_TX_RING_SLOTS]; + struct rte_mbuf *mi; + uint32_t frames_to_send; + struct rte_mempool *mp; + struct qbman_eq_desc eqdesc; + struct dpaa2_queue *dpaa2_q = (struct dpaa2_queue *)queue; + struct qbman_swp *swp; + uint16_t num_tx = 0; + uint16_t bpid; + struct rte_eth_dev *dev = dpaa2_q->dev; + struct dpaa2_dev_priv *priv = dev->data->dev_private; + + if (unlikely(!DPAA2_PER_LCORE_DPIO)) { + ret = dpaa2_affine_qbman_swp(); + if (ret) { + RTE_LOG(ERR, PMD, "Failure in affining portal\n"); + return 0; + } + } + swp = DPAA2_PER_LCORE_PORTAL; + + PMD_TX_LOG(DEBUG, "===> dev =%p, fqid =%d", dev, dpaa2_q->fqid); + + /*Prepare enqueue descriptor*/ + qbman_eq_desc_clear(&eqdesc); + qbman_eq_desc_set_no_orp(&eqdesc, DPAA2_EQ_RESP_ERR_FQ); + qbman_eq_desc_set_response(&eqdesc, 0, 0); + qbman_eq_desc_set_qd(&eqdesc, priv->qdid, + dpaa2_q->flow_id, dpaa2_q->tc_index); + + /*Clear the unused FD fields before sending*/ + while (nb_pkts) { + /*Check if the queue is congested*/ + retry_count = 0; + while (qbman_result_SCN_state(dpaa2_q->cscn)) { + retry_count++; + /* Retry for some time before giving up */ + if (retry_count > CONG_RETRY_COUNT) + goto skip_tx; + } + + frames_to_send = (nb_pkts >> 3) ? MAX_TX_RING_SLOTS : nb_pkts; + + for (loop = 0; loop < frames_to_send; loop++) { + fd_arr[loop].simple.frc = 0; + DPAA2_RESET_FD_CTRL((&fd_arr[loop])); + DPAA2_SET_FD_FLC((&fd_arr[loop]), NULL); + if (RTE_MBUF_DIRECT(*bufs)) { + mp = (*bufs)->pool; + } else { + mi = rte_mbuf_from_indirect(*bufs); + mp = mi->pool; + } + /* Not a hw_pkt pool allocated frame */ + if (unlikely(!mp || !priv->bp_list)) { + PMD_TX_LOG(ERR, "err: no bpool attached"); + goto send_n_return; + } + + if (mp->ops_index != priv->bp_list->dpaa2_ops_index) { + PMD_TX_LOG(ERR, "non hw offload bufffer "); + /* alloc should be from the default buffer pool + * attached to this interface + */ + bpid = priv->bp_list->buf_pool.bpid; + + if (unlikely((*bufs)->nb_segs > 1)) { + PMD_TX_LOG(ERR, "S/G support not added" + " for non hw offload buffer"); + goto send_n_return; + } + if (eth_copy_mbuf_to_fd(*bufs, + &fd_arr[loop], bpid)) { + goto send_n_return; + } + /* free the original packet */ + rte_pktmbuf_free(*bufs); + } else { + bpid = mempool_to_bpid(mp); + if (unlikely((*bufs)->nb_segs > 1)) { + if (eth_mbuf_to_sg_fd(*bufs, + &fd_arr[loop], bpid)) + goto send_n_return; + } else { + eth_mbuf_to_fd(*bufs, + &fd_arr[loop], bpid); + } + } + bufs++; + } + loop = 0; + while (loop < frames_to_send) { + loop += qbman_swp_enqueue_multiple(swp, &eqdesc, + &fd_arr[loop], frames_to_send - loop); + } + + num_tx += frames_to_send; + dpaa2_q->tx_pkts += frames_to_send; + nb_pkts -= frames_to_send; + } + return num_tx; + +send_n_return: + /* send any already prepared fd */ + if (loop) { + unsigned int i = 0; + + while (i < loop) { + i += qbman_swp_enqueue_multiple(swp, &eqdesc, + &fd_arr[i], loop - i); + } + num_tx += loop; + dpaa2_q->tx_pkts += loop; + } +skip_tx: + return num_tx; +} + +/** + * Dummy DPDK callback for TX. + * + * This function is used to temporarily replace the real callback during + * unsafe control operations on the queue, or in case of error. + * + * @param dpdk_txq + * Generic pointer to TX queue structure. + * @param[in] pkts + * Packets to transmit. + * @param pkts_n + * Number of packets in array. + * + * @return + * Number of packets successfully transmitted (<= pkts_n). + */ +uint16_t +dummy_dev_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) +{ + (void)queue; + (void)bufs; + (void)nb_pkts; + return 0; +} diff --git a/dpdk/drivers/net/dpaa2/mc/dpkg.c b/dpdk/drivers/net/dpaa2/mc/dpkg.c new file mode 100644 index 00000000..3f98907f --- /dev/null +++ b/dpdk/drivers/net/dpaa2/mc/dpkg.c @@ -0,0 +1,107 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2017 NXP + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#include +#include +#include + +/** + * dpkg_prepare_key_cfg() - function prepare extract parameters + * @cfg: defining a full Key Generation profile (rule) + * @key_cfg_buf: Zeroed 256 bytes of memory before mapping it to DMA + * + * This function has to be called before the following functions: + * - dpni_set_rx_tc_dist() + * - dpni_set_qos_table() + * - dpkg_prepare_key_cfg() + */ +int +dpkg_prepare_key_cfg(const struct dpkg_profile_cfg *cfg, uint8_t *key_cfg_buf) +{ + int i, j; + struct dpni_ext_set_rx_tc_dist *dpni_ext; + struct dpni_dist_extract *extr; + + if (cfg->num_extracts > DPKG_MAX_NUM_OF_EXTRACTS) + return -EINVAL; + + dpni_ext = (struct dpni_ext_set_rx_tc_dist *)key_cfg_buf; + dpni_ext->num_extracts = cfg->num_extracts; + + for (i = 0; i < cfg->num_extracts; i++) { + extr = &dpni_ext->extracts[i]; + + switch (cfg->extracts[i].type) { + case DPKG_EXTRACT_FROM_HDR: + extr->prot = cfg->extracts[i].extract.from_hdr.prot; + dpkg_set_field(extr->efh_type, EFH_TYPE, + cfg->extracts[i].extract.from_hdr.type); + extr->size = cfg->extracts[i].extract.from_hdr.size; + extr->offset = cfg->extracts[i].extract.from_hdr.offset; + extr->field = cpu_to_le32( + cfg->extracts[i].extract.from_hdr.field); + extr->hdr_index = + cfg->extracts[i].extract.from_hdr.hdr_index; + break; + case DPKG_EXTRACT_FROM_DATA: + extr->size = cfg->extracts[i].extract.from_data.size; + extr->offset = + cfg->extracts[i].extract.from_data.offset; + break; + case DPKG_EXTRACT_FROM_PARSE: + extr->size = cfg->extracts[i].extract.from_parse.size; + extr->offset = + cfg->extracts[i].extract.from_parse.offset; + break; + default: + return -EINVAL; + } + + extr->num_of_byte_masks = cfg->extracts[i].num_of_byte_masks; + dpkg_set_field(extr->extract_type, EXTRACT_TYPE, + cfg->extracts[i].type); + + for (j = 0; j < DPKG_NUM_OF_MASKS; j++) { + extr->masks[j].mask = cfg->extracts[i].masks[j].mask; + extr->masks[j].offset = + cfg->extracts[i].masks[j].offset; + } + } + + return 0; +} diff --git a/dpdk/drivers/net/dpaa2/mc/dpni.c b/dpdk/drivers/net/dpaa2/mc/dpni.c new file mode 100644 index 00000000..6f671fe0 --- /dev/null +++ b/dpdk/drivers/net/dpaa2/mc/dpni.c @@ -0,0 +1,1976 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2013-2016 Freescale Semiconductor Inc. + * Copyright 2016 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#include +#include +#include +#include + +/** + * dpni_open() - Open a control session for the specified object + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @dpni_id: DPNI unique ID + * @token: Returned token; use in subsequent API calls + * + * This function can be used to open a control session for an + * already created object; an object may have been declared in + * the DPL or by calling the dpni_create() function. + * This function returns a unique authentication token, + * associated with the specific object ID and the specific MC + * portal; this token must be used in all subsequent commands for + * this specific object. + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_open(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + int dpni_id, + uint16_t *token) +{ + struct mc_command cmd = { 0 }; + struct dpni_cmd_open *cmd_params; + + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_OPEN, + cmd_flags, + 0); + cmd_params = (struct dpni_cmd_open *)cmd.params; + cmd_params->dpni_id = cpu_to_le32(dpni_id); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + *token = mc_cmd_hdr_read_token(&cmd); + + return 0; +} + +/** + * dpni_close() - Close the control session of the object + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * + * After this function is called, no further operations are + * allowed on the object without opening a new control session. + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_close(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token) +{ + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_CLOSE, + cmd_flags, + token); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpni_create() - Create the DPNI object + * @mc_io: Pointer to MC portal's I/O object + * @dprc_token: Parent container token; '0' for default container + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @cfg: Configuration structure + * @obj_id: Returned object id + * + * Create the DPNI object, allocate required resources and + * perform required initialization. + * + * The object can be created either by declaring it in the + * DPL file, or by calling this function. + * + * The function accepts an authentication token of a parent + * container that this object should be assigned to. The token + * can be '0' so the object will be assigned to the default container. + * The newly created object can be opened with the returned + * object id and using the container's associated tokens and MC portals. + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_create(struct fsl_mc_io *mc_io, + uint16_t dprc_token, + uint32_t cmd_flags, + const struct dpni_cfg *cfg, + uint32_t *obj_id) +{ + struct dpni_cmd_create *cmd_params; + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_CREATE, + cmd_flags, + dprc_token); + cmd_params = (struct dpni_cmd_create *)cmd.params; + cmd_params->options = cpu_to_le32(cfg->options); + cmd_params->num_queues = cfg->num_queues; + cmd_params->num_tcs = cfg->num_tcs; + cmd_params->mac_filter_entries = cfg->mac_filter_entries; + cmd_params->vlan_filter_entries = cfg->vlan_filter_entries; + cmd_params->qos_entries = cfg->qos_entries; + cmd_params->fs_entries = cpu_to_le16(cfg->fs_entries); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + *obj_id = mc_cmd_read_object_id(&cmd); + + return 0; +} + +/** + * dpni_destroy() - Destroy the DPNI object and release all its resources. + * @mc_io: Pointer to MC portal's I/O object + * @dprc_token: Parent container token; '0' for default container + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @object_id: The object id; it must be a valid id within the container that + * created this object; + * + * The function accepts the authentication token of the parent container that + * created the object (not the one that currently owns the object). The object + * is searched within parent using the provided 'object_id'. + * All tokens to the object must be closed before calling destroy. + * + * Return: '0' on Success; error code otherwise. + */ +int dpni_destroy(struct fsl_mc_io *mc_io, + uint16_t dprc_token, + uint32_t cmd_flags, + uint32_t object_id) +{ + struct dpni_cmd_destroy *cmd_params; + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_DESTROY, + cmd_flags, + dprc_token); + /* set object id to destroy */ + cmd_params = (struct dpni_cmd_destroy *)cmd.params; + cmd_params->dpsw_id = cpu_to_le32(object_id); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpni_set_pools() - Set buffer pools configuration + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * @cfg: Buffer pools configuration + * + * mandatory for DPNI operation + * warning:Allowed only when DPNI is disabled + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_set_pools(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + const struct dpni_pools_cfg *cfg) +{ + struct mc_command cmd = { 0 }; + struct dpni_cmd_set_pools *cmd_params; + int i; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_SET_POOLS, + cmd_flags, + token); + cmd_params = (struct dpni_cmd_set_pools *)cmd.params; + cmd_params->num_dpbp = cfg->num_dpbp; + for (i = 0; i < DPNI_MAX_DPBP; i++) { + cmd_params->pool[i].dpbp_id = + cpu_to_le16(cfg->pools[i].dpbp_id); + cmd_params->pool[i].priority_mask = + cfg->pools[i].priority_mask; + cmd_params->buffer_size[i] = + cpu_to_le16(cfg->pools[i].buffer_size); + cmd_params->backup_pool_mask |= + DPNI_BACKUP_POOL(cfg->pools[i].backup_pool, i); + } + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpni_enable() - Enable the DPNI, allow sending and receiving frames. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_enable(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token) +{ + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_ENABLE, + cmd_flags, + token); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpni_disable() - Disable the DPNI, stop sending and receiving frames. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_disable(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token) +{ + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_DISABLE, + cmd_flags, + token); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpni_is_enabled() - Check if the DPNI is enabled. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * @en: Returns '1' if object is enabled; '0' otherwise + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_is_enabled(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + int *en) +{ + struct mc_command cmd = { 0 }; + struct dpni_rsp_is_enabled *rsp_params; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_IS_ENABLED, + cmd_flags, + token); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + rsp_params = (struct dpni_rsp_is_enabled *)cmd.params; + *en = dpni_get_field(rsp_params->enabled, ENABLE); + + return 0; +} + +/** + * dpni_reset() - Reset the DPNI, returns the object to initial state. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_reset(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token) +{ + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_RESET, + cmd_flags, + token); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpni_set_irq_enable() - Set overall interrupt state. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * @irq_index: The interrupt index to configure + * @en: Interrupt state: - enable = 1, disable = 0 + * + * Allows GPP software to control when interrupts are generated. + * Each interrupt can have up to 32 causes. The enable/disable control's the + * overall interrupt state. if the interrupt is disabled no causes will cause + * an interrupt. + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_set_irq_enable(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t irq_index, + uint8_t en) +{ + struct mc_command cmd = { 0 }; + struct dpni_cmd_set_irq_enable *cmd_params; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_SET_IRQ_ENABLE, + cmd_flags, + token); + cmd_params = (struct dpni_cmd_set_irq_enable *)cmd.params; + dpni_set_field(cmd_params->enable, ENABLE, en); + cmd_params->irq_index = irq_index; + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpni_get_irq_enable() - Get overall interrupt state + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * @irq_index: The interrupt index to configure + * @en: Returned interrupt state - enable = 1, disable = 0 + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_get_irq_enable(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t irq_index, + uint8_t *en) +{ + struct mc_command cmd = { 0 }; + struct dpni_cmd_get_irq_enable *cmd_params; + struct dpni_rsp_get_irq_enable *rsp_params; + + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_GET_IRQ_ENABLE, + cmd_flags, + token); + cmd_params = (struct dpni_cmd_get_irq_enable *)cmd.params; + cmd_params->irq_index = irq_index; + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + rsp_params = (struct dpni_rsp_get_irq_enable *)cmd.params; + *en = dpni_get_field(rsp_params->enabled, ENABLE); + + return 0; +} + +/** + * dpni_set_irq_mask() - Set interrupt mask. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * @irq_index: The interrupt index to configure + * @mask: Event mask to trigger interrupt; + * each bit: + * 0 = ignore event + * 1 = consider event for asserting IRQ + * + * Every interrupt can have up to 32 causes and the interrupt model supports + * masking/unmasking each cause independently + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_set_irq_mask(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t irq_index, + uint32_t mask) +{ + struct mc_command cmd = { 0 }; + struct dpni_cmd_set_irq_mask *cmd_params; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_SET_IRQ_MASK, + cmd_flags, + token); + cmd_params = (struct dpni_cmd_set_irq_mask *)cmd.params; + cmd_params->mask = cpu_to_le32(mask); + cmd_params->irq_index = irq_index; + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpni_get_irq_mask() - Get interrupt mask. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * @irq_index: The interrupt index to configure + * @mask: Returned event mask to trigger interrupt + * + * Every interrupt can have up to 32 causes and the interrupt model supports + * masking/unmasking each cause independently + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_get_irq_mask(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t irq_index, + uint32_t *mask) +{ + struct mc_command cmd = { 0 }; + struct dpni_cmd_get_irq_mask *cmd_params; + struct dpni_rsp_get_irq_mask *rsp_params; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_GET_IRQ_MASK, + cmd_flags, + token); + cmd_params = (struct dpni_cmd_get_irq_mask *)cmd.params; + cmd_params->irq_index = irq_index; + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + rsp_params = (struct dpni_rsp_get_irq_mask *)cmd.params; + *mask = le32_to_cpu(rsp_params->mask); + + return 0; +} + +/** + * dpni_get_irq_status() - Get the current status of any pending interrupts. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * @irq_index: The interrupt index to configure + * @status: Returned interrupts status - one bit per cause: + * 0 = no interrupt pending + * 1 = interrupt pending + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_get_irq_status(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t irq_index, + uint32_t *status) +{ + struct mc_command cmd = { 0 }; + struct dpni_cmd_get_irq_status *cmd_params; + struct dpni_rsp_get_irq_status *rsp_params; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_GET_IRQ_STATUS, + cmd_flags, + token); + cmd_params = (struct dpni_cmd_get_irq_status *)cmd.params; + cmd_params->status = cpu_to_le32(*status); + cmd_params->irq_index = irq_index; + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + rsp_params = (struct dpni_rsp_get_irq_status *)cmd.params; + *status = le32_to_cpu(rsp_params->status); + + return 0; +} + +/** + * dpni_clear_irq_status() - Clear a pending interrupt's status + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * @irq_index: The interrupt index to configure + * @status: bits to clear (W1C) - one bit per cause: + * 0 = don't change + * 1 = clear status bit + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_clear_irq_status(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t irq_index, + uint32_t status) +{ + struct mc_command cmd = { 0 }; + struct dpni_cmd_clear_irq_status *cmd_params; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_CLEAR_IRQ_STATUS, + cmd_flags, + token); + cmd_params = (struct dpni_cmd_clear_irq_status *)cmd.params; + cmd_params->irq_index = irq_index; + cmd_params->status = cpu_to_le32(status); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpni_get_attributes() - Retrieve DPNI attributes. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * @attr: Object's attributes + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_get_attributes(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + struct dpni_attr *attr) +{ + struct mc_command cmd = { 0 }; + struct dpni_rsp_get_attr *rsp_params; + + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_GET_ATTR, + cmd_flags, + token); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + rsp_params = (struct dpni_rsp_get_attr *)cmd.params; + attr->options = le32_to_cpu(rsp_params->options); + attr->num_queues = rsp_params->num_queues; + attr->num_rx_tcs = rsp_params->num_rx_tcs; + attr->num_tx_tcs = rsp_params->num_tx_tcs; + attr->mac_filter_entries = rsp_params->mac_filter_entries; + attr->vlan_filter_entries = rsp_params->vlan_filter_entries; + attr->qos_entries = rsp_params->qos_entries; + attr->fs_entries = le16_to_cpu(rsp_params->fs_entries); + attr->qos_key_size = rsp_params->qos_key_size; + attr->fs_key_size = rsp_params->fs_key_size; + attr->wriop_version = le16_to_cpu(rsp_params->wriop_version); + + return 0; +} + +/** + * dpni_set_errors_behavior() - Set errors behavior + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * @cfg: Errors configuration + * + * This function may be called numerous times with different + * error masks + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_set_errors_behavior(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + struct dpni_error_cfg *cfg) +{ + struct mc_command cmd = { 0 }; + struct dpni_cmd_set_errors_behavior *cmd_params; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_SET_ERRORS_BEHAVIOR, + cmd_flags, + token); + cmd_params = (struct dpni_cmd_set_errors_behavior *)cmd.params; + cmd_params->errors = cpu_to_le32(cfg->errors); + dpni_set_field(cmd_params->flags, ERROR_ACTION, cfg->error_action); + dpni_set_field(cmd_params->flags, FRAME_ANN, cfg->set_frame_annotation); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpni_get_buffer_layout() - Retrieve buffer layout attributes. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * @qtype: Type of queue to retrieve configuration for + * @layout: Returns buffer layout attributes + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_get_buffer_layout(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + enum dpni_queue_type qtype, + struct dpni_buffer_layout *layout) +{ + struct mc_command cmd = { 0 }; + struct dpni_cmd_get_buffer_layout *cmd_params; + struct dpni_rsp_get_buffer_layout *rsp_params; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_GET_BUFFER_LAYOUT, + cmd_flags, + token); + cmd_params = (struct dpni_cmd_get_buffer_layout *)cmd.params; + cmd_params->qtype = qtype; + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + rsp_params = (struct dpni_rsp_get_buffer_layout *)cmd.params; + layout->pass_timestamp = dpni_get_field(rsp_params->flags, PASS_TS); + layout->pass_parser_result = dpni_get_field(rsp_params->flags, PASS_PR); + layout->pass_frame_status = dpni_get_field(rsp_params->flags, PASS_FS); + layout->private_data_size = le16_to_cpu(rsp_params->private_data_size); + layout->data_align = le16_to_cpu(rsp_params->data_align); + layout->data_head_room = le16_to_cpu(rsp_params->head_room); + layout->data_tail_room = le16_to_cpu(rsp_params->tail_room); + + return 0; +} + +/** + * dpni_set_buffer_layout() - Set buffer layout configuration. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * @qtype: Type of queue this configuration applies to + * @layout: Buffer layout configuration + * + * Return: '0' on Success; Error code otherwise. + * + * @warning Allowed only when DPNI is disabled + */ +int dpni_set_buffer_layout(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + enum dpni_queue_type qtype, + const struct dpni_buffer_layout *layout) +{ + struct mc_command cmd = { 0 }; + struct dpni_cmd_set_buffer_layout *cmd_params; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_SET_BUFFER_LAYOUT, + cmd_flags, + token); + cmd_params = (struct dpni_cmd_set_buffer_layout *)cmd.params; + cmd_params->qtype = qtype; + cmd_params->options = cpu_to_le16(layout->options); + dpni_set_field(cmd_params->flags, PASS_TS, layout->pass_timestamp); + dpni_set_field(cmd_params->flags, PASS_PR, layout->pass_parser_result); + dpni_set_field(cmd_params->flags, PASS_FS, layout->pass_frame_status); + cmd_params->private_data_size = cpu_to_le16(layout->private_data_size); + cmd_params->data_align = cpu_to_le16(layout->data_align); + cmd_params->head_room = cpu_to_le16(layout->data_head_room); + cmd_params->tail_room = cpu_to_le16(layout->data_tail_room); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpni_set_offload() - Set DPNI offload configuration. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * @type: Type of DPNI offload + * @config: Offload configuration. + * For checksum offloads, non-zero value enables the offload + * + * Return: '0' on Success; Error code otherwise. + * + * @warning Allowed only when DPNI is disabled + */ + +int dpni_set_offload(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + enum dpni_offload type, + uint32_t config) +{ + struct mc_command cmd = { 0 }; + struct dpni_cmd_set_offload *cmd_params; + + cmd.header = mc_encode_cmd_header(DPNI_CMDID_SET_OFFLOAD, + cmd_flags, + token); + cmd_params = (struct dpni_cmd_set_offload *)cmd.params; + cmd_params->dpni_offload = type; + cmd_params->config = cpu_to_le32(config); + + return mc_send_command(mc_io, &cmd); +} + +/** + * dpni_get_offload() - Get DPNI offload configuration. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * @type: Type of DPNI offload + * @config: Offload configuration. + * For checksum offloads, a value of 1 indicates that the + * offload is enabled. + * + * Return: '0' on Success; Error code otherwise. + * + * @warning Allowed only when DPNI is disabled + */ +int dpni_get_offload(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + enum dpni_offload type, + uint32_t *config) +{ + struct mc_command cmd = { 0 }; + struct dpni_cmd_get_offload *cmd_params; + struct dpni_rsp_get_offload *rsp_params; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_GET_OFFLOAD, + cmd_flags, + token); + cmd_params = (struct dpni_cmd_get_offload *)cmd.params; + cmd_params->dpni_offload = type; + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + rsp_params = (struct dpni_rsp_get_offload *)cmd.params; + *config = le32_to_cpu(rsp_params->config); + + return 0; +} + +/** + * dpni_get_qdid() - Get the Queuing Destination ID (QDID) that should be used + * for enqueue operations + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * @qtype: Type of queue to receive QDID for + * @qdid: Returned virtual QDID value that should be used as an argument + * in all enqueue operations + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_get_qdid(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + enum dpni_queue_type qtype, + uint16_t *qdid) +{ + struct mc_command cmd = { 0 }; + struct dpni_cmd_get_qdid *cmd_params; + struct dpni_rsp_get_qdid *rsp_params; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_GET_QDID, + cmd_flags, + token); + cmd_params = (struct dpni_cmd_get_qdid *)cmd.params; + cmd_params->qtype = qtype; + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + rsp_params = (struct dpni_rsp_get_qdid *)cmd.params; + *qdid = le16_to_cpu(rsp_params->qdid); + + return 0; +} + +/** + * dpni_get_tx_data_offset() - Get the Tx data offset (from start of buffer) + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * @data_offset: Tx data offset (from start of buffer) + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_get_tx_data_offset(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint16_t *data_offset) +{ + struct mc_command cmd = { 0 }; + struct dpni_rsp_get_tx_data_offset *rsp_params; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_GET_TX_DATA_OFFSET, + cmd_flags, + token); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + rsp_params = (struct dpni_rsp_get_tx_data_offset *)cmd.params; + *data_offset = le16_to_cpu(rsp_params->data_offset); + + return 0; +} + +/** + * dpni_set_link_cfg() - set the link configuration. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * @cfg: Link configuration + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_set_link_cfg(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + const struct dpni_link_cfg *cfg) +{ + struct mc_command cmd = { 0 }; + struct dpni_cmd_set_link_cfg *cmd_params; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_SET_LINK_CFG, + cmd_flags, + token); + cmd_params = (struct dpni_cmd_set_link_cfg *)cmd.params; + cmd_params->rate = cpu_to_le32(cfg->rate); + cmd_params->options = cpu_to_le64(cfg->options); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpni_get_link_state() - Return the link state (either up or down) + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * @state: Returned link state; + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_get_link_state(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + struct dpni_link_state *state) +{ + struct mc_command cmd = { 0 }; + struct dpni_rsp_get_link_state *rsp_params; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_GET_LINK_STATE, + cmd_flags, + token); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + rsp_params = (struct dpni_rsp_get_link_state *)cmd.params; + state->up = dpni_get_field(rsp_params->flags, LINK_STATE); + state->rate = le32_to_cpu(rsp_params->rate); + state->options = le64_to_cpu(rsp_params->options); + + return 0; +} + +/** + * dpni_set_max_frame_length() - Set the maximum received frame length. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * @max_frame_length: Maximum received frame length (in bytes); + * frame is discarded if its length exceeds this value + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_set_max_frame_length(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint16_t max_frame_length) +{ + struct mc_command cmd = { 0 }; + struct dpni_cmd_set_max_frame_length *cmd_params; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_SET_MAX_FRAME_LENGTH, + cmd_flags, + token); + cmd_params = (struct dpni_cmd_set_max_frame_length *)cmd.params; + cmd_params->max_frame_length = cpu_to_le16(max_frame_length); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpni_get_max_frame_length() - Get the maximum received frame length. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * @max_frame_length: Maximum received frame length (in bytes); + * frame is discarded if its length exceeds this value + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_get_max_frame_length(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint16_t *max_frame_length) +{ + struct mc_command cmd = { 0 }; + struct dpni_rsp_get_max_frame_length *rsp_params; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_GET_MAX_FRAME_LENGTH, + cmd_flags, + token); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + rsp_params = (struct dpni_rsp_get_max_frame_length *)cmd.params; + *max_frame_length = le16_to_cpu(rsp_params->max_frame_length); + + return 0; +} + +/** + * dpni_set_multicast_promisc() - Enable/disable multicast promiscuous mode + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * @en: Set to '1' to enable; '0' to disable + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_set_multicast_promisc(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + int en) +{ + struct mc_command cmd = { 0 }; + struct dpni_cmd_set_multicast_promisc *cmd_params; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_SET_MCAST_PROMISC, + cmd_flags, + token); + cmd_params = (struct dpni_cmd_set_multicast_promisc *)cmd.params; + dpni_set_field(cmd_params->enable, ENABLE, en); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpni_get_multicast_promisc() - Get multicast promiscuous mode + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * @en: Returns '1' if enabled; '0' otherwise + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_get_multicast_promisc(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + int *en) +{ + struct mc_command cmd = { 0 }; + struct dpni_rsp_get_multicast_promisc *rsp_params; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_GET_MCAST_PROMISC, + cmd_flags, + token); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + rsp_params = (struct dpni_rsp_get_multicast_promisc *)cmd.params; + *en = dpni_get_field(rsp_params->enabled, ENABLE); + + return 0; +} + +/** + * dpni_set_unicast_promisc() - Enable/disable unicast promiscuous mode + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * @en: Set to '1' to enable; '0' to disable + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_set_unicast_promisc(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + int en) +{ + struct mc_command cmd = { 0 }; + struct dpni_cmd_set_unicast_promisc *cmd_params; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_SET_UNICAST_PROMISC, + cmd_flags, + token); + cmd_params = (struct dpni_cmd_set_unicast_promisc *)cmd.params; + dpni_set_field(cmd_params->enable, ENABLE, en); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpni_get_unicast_promisc() - Get unicast promiscuous mode + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * @en: Returns '1' if enabled; '0' otherwise + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_get_unicast_promisc(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + int *en) +{ + struct mc_command cmd = { 0 }; + struct dpni_rsp_get_unicast_promisc *rsp_params; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_GET_UNICAST_PROMISC, + cmd_flags, + token); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + rsp_params = (struct dpni_rsp_get_unicast_promisc *)cmd.params; + *en = dpni_get_field(rsp_params->enabled, ENABLE); + + return 0; +} + +/** + * dpni_set_primary_mac_addr() - Set the primary MAC address + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * @mac_addr: MAC address to set as primary address + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_set_primary_mac_addr(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + const uint8_t mac_addr[6]) +{ + struct mc_command cmd = { 0 }; + struct dpni_cmd_set_primary_mac_addr *cmd_params; + int i; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_SET_PRIM_MAC, + cmd_flags, + token); + cmd_params = (struct dpni_cmd_set_primary_mac_addr *)cmd.params; + for (i = 0; i < 6; i++) + cmd_params->mac_addr[i] = mac_addr[5 - i]; + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpni_get_primary_mac_addr() - Get the primary MAC address + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * @mac_addr: Returned MAC address + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_get_primary_mac_addr(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t mac_addr[6]) +{ + struct mc_command cmd = { 0 }; + struct dpni_rsp_get_primary_mac_addr *rsp_params; + int i, err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_GET_PRIM_MAC, + cmd_flags, + token); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + rsp_params = (struct dpni_rsp_get_primary_mac_addr *)cmd.params; + for (i = 0; i < 6; i++) + mac_addr[5 - i] = rsp_params->mac_addr[i]; + + return 0; +} + +/** + * dpni_add_mac_addr() - Add MAC address filter + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * @mac_addr: MAC address to add + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_add_mac_addr(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + const uint8_t mac_addr[6]) +{ + struct mc_command cmd = { 0 }; + struct dpni_cmd_add_mac_addr *cmd_params; + int i; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_ADD_MAC_ADDR, + cmd_flags, + token); + cmd_params = (struct dpni_cmd_add_mac_addr *)cmd.params; + for (i = 0; i < 6; i++) + cmd_params->mac_addr[i] = mac_addr[5 - i]; + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpni_remove_mac_addr() - Remove MAC address filter + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * @mac_addr: MAC address to remove + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_remove_mac_addr(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + const uint8_t mac_addr[6]) +{ + struct mc_command cmd = { 0 }; + struct dpni_cmd_remove_mac_addr *cmd_params; + int i; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_REMOVE_MAC_ADDR, + cmd_flags, + token); + cmd_params = (struct dpni_cmd_remove_mac_addr *)cmd.params; + for (i = 0; i < 6; i++) + cmd_params->mac_addr[i] = mac_addr[5 - i]; + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpni_clear_mac_filters() - Clear all unicast and/or multicast MAC filters + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * @unicast: Set to '1' to clear unicast addresses + * @multicast: Set to '1' to clear multicast addresses + * + * The primary MAC address is not cleared by this operation. + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_clear_mac_filters(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + int unicast, + int multicast) +{ + struct mc_command cmd = { 0 }; + struct dpni_cmd_clear_mac_filters *cmd_params; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_CLR_MAC_FILTERS, + cmd_flags, + token); + cmd_params = (struct dpni_cmd_clear_mac_filters *)cmd.params; + dpni_set_field(cmd_params->flags, UNICAST_FILTERS, unicast); + dpni_set_field(cmd_params->flags, MULTICAST_FILTERS, multicast); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpni_get_port_mac_addr() - Retrieve MAC address associated to the physical + * port the DPNI is attached to + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * @mac_addr: MAC address of the physical port, if any, otherwise 0 + * + * The primary MAC address is not cleared by this operation. + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_get_port_mac_addr(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t mac_addr[6]) +{ + struct mc_command cmd = { 0 }; + struct dpni_rsp_get_port_mac_addr *rsp_params; + int i, err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_GET_PORT_MAC_ADDR, + cmd_flags, + token); + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + rsp_params = (struct dpni_rsp_get_port_mac_addr *)cmd.params; + for (i = 0; i < 6; i++) + mac_addr[5 - i] = rsp_params->mac_addr[i]; + + return 0; +} + +/** + * dpni_enable_vlan_filter() - Enable/disable VLAN filtering mode + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * @en: Set to '1' to enable; '0' to disable + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_enable_vlan_filter(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + int en) +{ + struct dpni_cmd_enable_vlan_filter *cmd_params; + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_ENABLE_VLAN_FILTER, + cmd_flags, + token); + cmd_params = (struct dpni_cmd_enable_vlan_filter *)cmd.params; + dpni_set_field(cmd_params->en, ENABLE, en); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpni_add_vlan_id() - Add VLAN ID filter + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * @vlan_id: VLAN ID to add + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_add_vlan_id(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint16_t vlan_id) +{ + struct dpni_cmd_vlan_id *cmd_params; + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_ADD_VLAN_ID, + cmd_flags, + token); + cmd_params = (struct dpni_cmd_vlan_id *)cmd.params; + cmd_params->vlan_id = cpu_to_le16(vlan_id); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpni_remove_vlan_id() - Remove VLAN ID filter + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * @vlan_id: VLAN ID to remove + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_remove_vlan_id(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint16_t vlan_id) +{ + struct dpni_cmd_vlan_id *cmd_params; + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_REMOVE_VLAN_ID, + cmd_flags, + token); + cmd_params = (struct dpni_cmd_vlan_id *)cmd.params; + cmd_params->vlan_id = cpu_to_le16(vlan_id); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpni_clear_vlan_filters() - Clear all VLAN filters + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_clear_vlan_filters(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token) +{ + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_CLR_VLAN_FILTERS, + cmd_flags, + token); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpni_set_rx_tc_dist() - Set Rx traffic class distribution configuration + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * @tc_id: Traffic class selection (0-7) + * @cfg: Traffic class distribution configuration + * + * warning: if 'dist_mode != DPNI_DIST_MODE_NONE', call dpkg_prepare_key_cfg() + * first to prepare the key_cfg_iova parameter + * + * Return: '0' on Success; error code otherwise. + */ +int dpni_set_rx_tc_dist(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t tc_id, + const struct dpni_rx_tc_dist_cfg *cfg) +{ + struct mc_command cmd = { 0 }; + struct dpni_cmd_set_rx_tc_dist *cmd_params; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_SET_RX_TC_DIST, + cmd_flags, + token); + cmd_params = (struct dpni_cmd_set_rx_tc_dist *)cmd.params; + cmd_params->dist_size = cpu_to_le16(cfg->dist_size); + cmd_params->tc_id = tc_id; + cmd_params->default_flow_id = cpu_to_le16(cfg->fs_cfg.default_flow_id); + cmd_params->key_cfg_iova = cpu_to_le64(cfg->key_cfg_iova); + dpni_set_field(cmd_params->flags, + DIST_MODE, + cfg->dist_mode); + dpni_set_field(cmd_params->flags, + MISS_ACTION, + cfg->fs_cfg.miss_action); + dpni_set_field(cmd_params->keep_hash_key, + KEEP_HASH_KEY, + cfg->fs_cfg.keep_hash_key); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpni_set_tx_confirmation_mode() - Tx confirmation mode + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * @mode: Tx confirmation mode + * + * This function is useful only when 'DPNI_OPT_TX_CONF_DISABLED' is not + * selected at DPNI creation. + * Calling this function with 'mode' set to DPNI_CONF_DISABLE disables all + * transmit confirmation (including the private confirmation queues), regardless + * of previous settings; Note that in this case, Tx error frames are still + * enqueued to the general transmit errors queue. + * Calling this function with 'mode' set to DPNI_CONF_SINGLE switches all + * Tx confirmations to a shared Tx conf queue. 'index' field in dpni_get_queue + * command will be ignored. + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_set_tx_confirmation_mode(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + enum dpni_confirmation_mode mode) +{ + struct dpni_tx_confirmation_mode *cmd_params; + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_SET_TX_CONFIRMATION_MODE, + cmd_flags, + token); + cmd_params = (struct dpni_tx_confirmation_mode *)cmd.params; + cmd_params->confirmation_mode = mode; + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpni_set_congestion_notification() - Set traffic class congestion + * notification configuration + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * @qtype: Type of queue - Rx, Tx and Tx confirm types are supported + * @tc_id: Traffic class selection (0-7) + * @cfg: congestion notification configuration + * + * Return: '0' on Success; error code otherwise. + */ +int dpni_set_congestion_notification(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + enum dpni_queue_type qtype, + uint8_t tc_id, + const struct dpni_congestion_notification_cfg *cfg) +{ + struct dpni_cmd_set_congestion_notification *cmd_params; + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header( + DPNI_CMDID_SET_CONGESTION_NOTIFICATION, + cmd_flags, + token); + cmd_params = (struct dpni_cmd_set_congestion_notification *)cmd.params; + cmd_params->qtype = qtype; + cmd_params->tc = tc_id; + cmd_params->dest_id = cpu_to_le32(cfg->dest_cfg.dest_id); + cmd_params->notification_mode = cpu_to_le16(cfg->notification_mode); + cmd_params->dest_priority = cfg->dest_cfg.priority; + cmd_params->message_iova = cpu_to_le64(cfg->message_iova); + cmd_params->message_ctx = cpu_to_le64(cfg->message_ctx); + cmd_params->threshold_entry = cpu_to_le32(cfg->threshold_entry); + cmd_params->threshold_exit = cpu_to_le32(cfg->threshold_exit); + dpni_set_field(cmd_params->type_units, + DEST_TYPE, + cfg->dest_cfg.dest_type); + dpni_set_field(cmd_params->type_units, + CONG_UNITS, + cfg->units); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpni_get_congestion_notification() - Get traffic class congestion + * notification configuration + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * @qtype: Type of queue - Rx, Tx and Tx confirm types are supported + * @tc_id: Traffic class selection (0-7) + * @cfg: congestion notification configuration + * + * Return: '0' on Success; error code otherwise. + */ +int dpni_get_congestion_notification(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + enum dpni_queue_type qtype, + uint8_t tc_id, + struct dpni_congestion_notification_cfg *cfg) +{ + struct dpni_rsp_get_congestion_notification *rsp_params; + struct dpni_cmd_get_congestion_notification *cmd_params; + struct mc_command cmd = { 0 }; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header( + DPNI_CMDID_GET_CONGESTION_NOTIFICATION, + cmd_flags, + token); + cmd_params = (struct dpni_cmd_get_congestion_notification *)cmd.params; + cmd_params->qtype = qtype; + cmd_params->tc = tc_id; + + /* send command to mc*/ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + rsp_params = (struct dpni_rsp_get_congestion_notification *)cmd.params; + cfg->units = dpni_get_field(rsp_params->type_units, CONG_UNITS); + cfg->threshold_entry = le32_to_cpu(rsp_params->threshold_entry); + cfg->threshold_exit = le32_to_cpu(rsp_params->threshold_exit); + cfg->message_ctx = le64_to_cpu(rsp_params->message_ctx); + cfg->message_iova = le64_to_cpu(rsp_params->message_iova); + cfg->notification_mode = le16_to_cpu(rsp_params->notification_mode); + cfg->dest_cfg.dest_id = le32_to_cpu(rsp_params->dest_id); + cfg->dest_cfg.priority = rsp_params->dest_priority; + cfg->dest_cfg.dest_type = dpni_get_field(rsp_params->type_units, + DEST_TYPE); + + return 0; +} + +/** + * dpni_get_api_version() - Get Data Path Network Interface API version + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @major_ver: Major version of data path network interface API + * @minor_ver: Minor version of data path network interface API + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_get_api_version(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t *major_ver, + uint16_t *minor_ver) +{ + struct dpni_rsp_get_api_version *rsp_params; + struct mc_command cmd = { 0 }; + int err; + + cmd.header = mc_encode_cmd_header(DPNI_CMDID_GET_API_VERSION, + cmd_flags, + 0); + + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + rsp_params = (struct dpni_rsp_get_api_version *)cmd.params; + *major_ver = le16_to_cpu(rsp_params->major); + *minor_ver = le16_to_cpu(rsp_params->minor); + + return 0; +} + +/** + * dpni_set_queue() - Set queue parameters + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * @qtype: Type of queue - all queue types are supported, although + * the command is ignored for Tx + * @tc: Traffic class, in range 0 to NUM_TCS - 1 + * @index: Selects the specific queue out of the set allocated for the + * same TC. Value must be in range 0 to NUM_QUEUES - 1 + * @options: A combination of DPNI_QUEUE_OPT_ values that control what + * configuration options are set on the queue + * @queue: Queue structure + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_set_queue(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + enum dpni_queue_type qtype, + uint8_t tc, + uint8_t index, + uint8_t options, + const struct dpni_queue *queue) +{ + struct mc_command cmd = { 0 }; + struct dpni_cmd_set_queue *cmd_params; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_SET_QUEUE, + cmd_flags, + token); + cmd_params = (struct dpni_cmd_set_queue *)cmd.params; + cmd_params->qtype = qtype; + cmd_params->tc = tc; + cmd_params->index = index; + cmd_params->options = options; + cmd_params->dest_id = cpu_to_le32(queue->destination.id); + cmd_params->dest_prio = queue->destination.priority; + dpni_set_field(cmd_params->flags, DEST_TYPE, queue->destination.type); + dpni_set_field(cmd_params->flags, STASH_CTRL, queue->flc.stash_control); + dpni_set_field(cmd_params->flags, HOLD_ACTIVE, + queue->destination.hold_active); + cmd_params->flc = cpu_to_le64(queue->flc.value); + cmd_params->user_context = cpu_to_le64(queue->user_context); + + /* send command to mc */ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpni_get_queue() - Get queue parameters + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * @qtype: Type of queue - all queue types are supported + * @tc: Traffic class, in range 0 to NUM_TCS - 1 + * @index: Selects the specific queue out of the set allocated for the + * same TC. Value must be in range 0 to NUM_QUEUES - 1 + * @queue: Queue configuration structure + * @qid: Queue identification + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_get_queue(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + enum dpni_queue_type qtype, + uint8_t tc, + uint8_t index, + struct dpni_queue *queue, + struct dpni_queue_id *qid) +{ + struct mc_command cmd = { 0 }; + struct dpni_cmd_get_queue *cmd_params; + struct dpni_rsp_get_queue *rsp_params; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_GET_QUEUE, + cmd_flags, + token); + cmd_params = (struct dpni_cmd_get_queue *)cmd.params; + cmd_params->qtype = qtype; + cmd_params->tc = tc; + cmd_params->index = index; + + /* send command to mc */ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + rsp_params = (struct dpni_rsp_get_queue *)cmd.params; + queue->destination.id = le32_to_cpu(rsp_params->dest_id); + queue->destination.priority = rsp_params->dest_prio; + queue->destination.type = dpni_get_field(rsp_params->flags, + DEST_TYPE); + queue->flc.stash_control = dpni_get_field(rsp_params->flags, + STASH_CTRL); + queue->destination.hold_active = dpni_get_field(rsp_params->flags, + HOLD_ACTIVE); + queue->flc.value = le64_to_cpu(rsp_params->flc); + queue->user_context = le64_to_cpu(rsp_params->user_context); + qid->fqid = le32_to_cpu(rsp_params->fqid); + qid->qdbin = le16_to_cpu(rsp_params->qdbin); + + return 0; +} + +/** + * dpni_get_statistics() - Get DPNI statistics + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * @page: Selects the statistics page to retrieve, see + * DPNI_GET_STATISTICS output. Pages are numbered 0 to 2. + * @param: Custom parameter for some pages used to select + * a certain statistic source, for example the TC. + * @stat: Structure containing the statistics + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_get_statistics(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t page, + uint8_t param, + union dpni_statistics *stat) +{ + struct mc_command cmd = { 0 }; + struct dpni_cmd_get_statistics *cmd_params; + struct dpni_rsp_get_statistics *rsp_params; + int i, err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_GET_STATISTICS, + cmd_flags, + token); + cmd_params = (struct dpni_cmd_get_statistics *)cmd.params; + cmd_params->page_number = page; + cmd_params->param = param; + + /* send command to mc */ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + rsp_params = (struct dpni_rsp_get_statistics *)cmd.params; + for (i = 0; i < DPNI_STATISTICS_CNT; i++) + stat->raw.counter[i] = le64_to_cpu(rsp_params->counter[i]); + + return 0; +} + +/** + * dpni_reset_statistics() - Clears DPNI statistics + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_reset_statistics(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token) +{ + struct mc_command cmd = { 0 }; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_RESET_STATISTICS, + cmd_flags, + token); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpni_set_taildrop() - Set taildrop per queue or TC + * + * Setting a per-TC taildrop (cg_point = DPNI_CP_GROUP) will reset any current + * congestion notification or early drop (WRED) configuration previously applied + * to the same TC. + * + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * @cg_point: Congestion point, DPNI_CP_QUEUE is only supported in + * combination with DPNI_QUEUE_RX. + * @q_type: Queue type, can be DPNI_QUEUE_RX or DPNI_QUEUE_TX. + * @tc: Traffic class to apply this taildrop to + * @q_index: Index of the queue if the DPNI supports multiple queues for + * traffic distribution. + * Ignored if CONGESTION_POINT is not DPNI_CP_QUEUE. + * @taildrop: Taildrop structure + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_set_taildrop(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + enum dpni_congestion_point cg_point, + enum dpni_queue_type qtype, + uint8_t tc, + uint8_t index, + struct dpni_taildrop *taildrop) +{ + struct mc_command cmd = { 0 }; + struct dpni_cmd_set_taildrop *cmd_params; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_SET_TAILDROP, + cmd_flags, + token); + cmd_params = (struct dpni_cmd_set_taildrop *)cmd.params; + cmd_params->congestion_point = cg_point; + cmd_params->qtype = qtype; + cmd_params->tc = tc; + cmd_params->index = index; + cmd_params->units = taildrop->units; + cmd_params->threshold = cpu_to_le32(taildrop->threshold); + dpni_set_field(cmd_params->enable_oal_lo, ENABLE, taildrop->enable); + dpni_set_field(cmd_params->enable_oal_lo, OAL_LO, taildrop->oal); + dpni_set_field(cmd_params->oal_hi, + OAL_HI, + taildrop->oal >> DPNI_OAL_LO_SIZE); + + /* send command to mc */ + return mc_send_command(mc_io, &cmd); +} + +/** + * dpni_get_taildrop() - Get taildrop information + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPNI object + * @cg_point: Congestion point + * @q_type: Queue type on which the taildrop is configured. + * Only Rx queues are supported for now + * @tc: Traffic class to apply this taildrop to + * @q_index: Index of the queue if the DPNI supports multiple queues for + * traffic distribution. Ignored if CONGESTION_POINT is not 0. + * @taildrop: Taildrop structure + * + * Return: '0' on Success; Error code otherwise. + */ +int dpni_get_taildrop(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + enum dpni_congestion_point cg_point, + enum dpni_queue_type qtype, + uint8_t tc, + uint8_t index, + struct dpni_taildrop *taildrop) +{ + struct mc_command cmd = { 0 }; + struct dpni_cmd_get_taildrop *cmd_params; + struct dpni_rsp_get_taildrop *rsp_params; + uint8_t oal_lo, oal_hi; + int err; + + /* prepare command */ + cmd.header = mc_encode_cmd_header(DPNI_CMDID_GET_TAILDROP, + cmd_flags, + token); + cmd_params = (struct dpni_cmd_get_taildrop *)cmd.params; + cmd_params->congestion_point = cg_point; + cmd_params->qtype = qtype; + cmd_params->tc = tc; + cmd_params->index = index; + + /* send command to mc */ + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + /* retrieve response parameters */ + rsp_params = (struct dpni_rsp_get_taildrop *)cmd.params; + taildrop->enable = dpni_get_field(rsp_params->enable_oal_lo, ENABLE); + taildrop->units = rsp_params->units; + taildrop->threshold = le32_to_cpu(rsp_params->threshold); + oal_lo = dpni_get_field(rsp_params->enable_oal_lo, OAL_LO); + oal_hi = dpni_get_field(rsp_params->oal_hi, OAL_HI); + taildrop->oal = oal_hi << DPNI_OAL_LO_SIZE | oal_lo; + + /* Fill the first 4 bits, 'oal' is a 2's complement value of 12 bits */ + if (taildrop->oal >= 0x0800) + taildrop->oal |= 0xF000; + + return 0; +} diff --git a/dpdk/drivers/net/dpaa2/mc/fsl_dpkg.h b/dpdk/drivers/net/dpaa2/mc/fsl_dpkg.h new file mode 100644 index 00000000..7f46bafb --- /dev/null +++ b/dpdk/drivers/net/dpaa2/mc/fsl_dpkg.h @@ -0,0 +1,237 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2013-2015 Freescale Semiconductor Inc. + * Copyright 2016-2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef __FSL_DPKG_H_ +#define __FSL_DPKG_H_ + +#include + +/* Data Path Key Generator API + * Contains initialization APIs and runtime APIs for the Key Generator + */ + +/** Key Generator properties */ + +/** + * Number of masks per key extraction + */ +#define DPKG_NUM_OF_MASKS 4 +/** + * Number of extractions per key profile + */ +#define DPKG_MAX_NUM_OF_EXTRACTS 10 + +/** + * enum dpkg_extract_from_hdr_type - Selecting extraction by header types + * @DPKG_FROM_HDR: Extract selected bytes from header, by offset + * @DPKG_FROM_FIELD: Extract selected bytes from header, by offset from field + * @DPKG_FULL_FIELD: Extract a full field + */ +enum dpkg_extract_from_hdr_type { + DPKG_FROM_HDR = 0, + DPKG_FROM_FIELD = 1, + DPKG_FULL_FIELD = 2 +}; + +/** + * enum dpkg_extract_type - Enumeration for selecting extraction type + * @DPKG_EXTRACT_FROM_HDR: Extract from the header + * @DPKG_EXTRACT_FROM_DATA: Extract from data not in specific header + * @DPKG_EXTRACT_FROM_PARSE: Extract from parser-result; + * e.g. can be used to extract header existence; + * please refer to 'Parse Result definition' section in the parser BG + */ +enum dpkg_extract_type { + DPKG_EXTRACT_FROM_HDR = 0, + DPKG_EXTRACT_FROM_DATA = 1, + DPKG_EXTRACT_FROM_PARSE = 3 +}; + +/** + * struct dpkg_mask - A structure for defining a single extraction mask + * @mask: Byte mask for the extracted content + * @offset: Offset within the extracted content + */ +struct dpkg_mask { + uint8_t mask; + uint8_t offset; +}; + +/* Macros for accessing command fields smaller than 1byte */ +#define DPKG_MASK(field) \ + GENMASK(DPKG_##field##_SHIFT + DPKG_##field##_SIZE - 1, \ + DPKG_##field##_SHIFT) +#define dpkg_set_field(var, field, val) \ + ((var) |= (((val) << DPKG_##field##_SHIFT) & DPKG_MASK(field))) +#define dpkg_get_field(var, field) \ + (((var) & DPKG_MASK(field)) >> DPKG_##field##_SHIFT) + +/** + * struct dpkg_extract - A structure for defining a single extraction + * @type: Determines how the union below is interpreted: + * DPKG_EXTRACT_FROM_HDR: selects 'from_hdr'; + * DPKG_EXTRACT_FROM_DATA: selects 'from_data'; + * DPKG_EXTRACT_FROM_PARSE: selects 'from_parse' + * @extract: Selects extraction method + * @num_of_byte_masks: Defines the number of valid entries in the array below; + * This is also the number of bytes to be used as masks + * @masks: Masks parameters + */ +struct dpkg_extract { + enum dpkg_extract_type type; + /** + * union extract - Selects extraction method + * @from_hdr - Used when 'type = DPKG_EXTRACT_FROM_HDR' + * @from_data - Used when 'type = DPKG_EXTRACT_FROM_DATA' + * @from_parse - Used when 'type = DPKG_EXTRACT_FROM_PARSE' + */ + union { + /** + * struct from_hdr - Used when 'type = DPKG_EXTRACT_FROM_HDR' + * @prot: Any of the supported headers + * @type: Defines the type of header extraction: + * DPKG_FROM_HDR: use size & offset below; + * DPKG_FROM_FIELD: use field, size and offset below; + * DPKG_FULL_FIELD: use field below + * @field: One of the supported fields (NH_FLD_) + * + * @size: Size in bytes + * @offset: Byte offset + * @hdr_index: Clear for cases not listed below; + * Used for protocols that may have more than a single + * header, 0 indicates an outer header; + * Supported protocols (possible values): + * NET_PROT_VLAN (0, HDR_INDEX_LAST); + * NET_PROT_MPLS (0, 1, HDR_INDEX_LAST); + * NET_PROT_IP(0, HDR_INDEX_LAST); + * NET_PROT_IPv4(0, HDR_INDEX_LAST); + * NET_PROT_IPv6(0, HDR_INDEX_LAST); + */ + + struct { + enum net_prot prot; + enum dpkg_extract_from_hdr_type type; + uint32_t field; + uint8_t size; + uint8_t offset; + uint8_t hdr_index; + } from_hdr; + /** + * struct from_data + * Used when 'type = DPKG_EXTRACT_FROM_DATA' + * @size: Size in bytes + * @offset: Byte offset + */ + struct { + uint8_t size; + uint8_t offset; + } from_data; + + /** + * struct from_parse + * Used when 'type = DPKG_EXTRACT_FROM_PARSE' + * @size: Size in bytes + * @offset: Byte offset + */ + struct { + uint8_t size; + uint8_t offset; + } from_parse; + } extract; + + uint8_t num_of_byte_masks; + struct dpkg_mask masks[DPKG_NUM_OF_MASKS]; +}; + +/** + * struct dpkg_profile_cfg - A structure for defining a full Key Generation + * profile (rule) + * @num_extracts: Defines the number of valid entries in the array below + * @extracts: Array of required extractions + */ +struct dpkg_profile_cfg { + uint8_t num_extracts; + struct dpkg_extract extracts[DPKG_MAX_NUM_OF_EXTRACTS]; +}; + +/* dpni_set_rx_tc_dist extension (structure of the DMA-able memory at + * key_cfg_iova) + */ +struct dpni_mask_cfg { + uint8_t mask; + uint8_t offset; +}; + +#define DPKG_EFH_TYPE_SHIFT 0 +#define DPKG_EFH_TYPE_SIZE 4 +#define DPKG_EXTRACT_TYPE_SHIFT 0 +#define DPKG_EXTRACT_TYPE_SIZE 4 + +struct dpni_dist_extract { + /* word 0 */ + uint8_t prot; + /* EFH type stored in the 4 least significant bits */ + uint8_t efh_type; + uint8_t size; + uint8_t offset; + uint32_t field; + /* word 1 */ + uint8_t hdr_index; + uint8_t constant; + uint8_t num_of_repeats; + uint8_t num_of_byte_masks; + /* Extraction type is stored in the 4 LSBs */ + uint8_t extract_type; + uint8_t pad[3]; + /* word 2 */ + struct dpni_mask_cfg masks[4]; +}; + +struct dpni_ext_set_rx_tc_dist { + /* extension word 0 */ + uint8_t num_extracts; + uint8_t pad[7]; + /* words 1..25 */ + struct dpni_dist_extract extracts[10]; +}; + +int dpkg_prepare_key_cfg(const struct dpkg_profile_cfg *cfg, + uint8_t *key_cfg_buf); + +#endif /* __FSL_DPKG_H_ */ diff --git a/dpdk/drivers/net/dpaa2/mc/fsl_dpni.h b/dpdk/drivers/net/dpaa2/mc/fsl_dpni.h new file mode 100644 index 00000000..5227ea15 --- /dev/null +++ b/dpdk/drivers/net/dpaa2/mc/fsl_dpni.h @@ -0,0 +1,1168 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2013-2016 Freescale Semiconductor Inc. + * Copyright 2016-2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef __FSL_DPNI_H +#define __FSL_DPNI_H + +#include + +struct fsl_mc_io; + +/** + * Data Path Network Interface API + * Contains initialization APIs and runtime control APIs for DPNI + */ + +/** General DPNI macros */ + +/** + * Maximum number of traffic classes + */ +#define DPNI_MAX_TC 8 +/** + * Maximum number of buffer pools per DPNI + */ +#define DPNI_MAX_DPBP 8 +/** + * Maximum number of storage-profiles per DPNI + */ +#define DPNI_MAX_SP 2 + +/** + * All traffic classes considered; see dpni_set_queue() + */ +#define DPNI_ALL_TCS (uint8_t)(-1) +/** + * All flows within traffic class considered; see dpni_set_queue() + */ +#define DPNI_ALL_TC_FLOWS (uint16_t)(-1) + +/** + * Tx traffic is always released to a buffer pool on transmit, there are no + * resources allocated to have the frames confirmed back to the source after + * transmission. + */ +#define DPNI_OPT_TX_FRM_RELEASE 0x000001 +/** + * Disables support for MAC address filtering for addresses other than primary + * MAC address. This affects both unicast and multicast. Promiscuous mode can + * still be enabled/disabled for both unicast and multicast. If promiscuous mode + * is disabled, only traffic matching the primary MAC address will be accepted. + */ +#define DPNI_OPT_NO_MAC_FILTER 0x000002 +/** + * Allocate policers for this DPNI. They can be used to rate-limit traffic per + * traffic class (TC) basis. + */ +#define DPNI_OPT_HAS_POLICING 0x000004 +/** + * Congestion can be managed in several ways, allowing the buffer pool to + * deplete on ingress, taildrop on each queue or use congestion groups for sets + * of queues. If set, it configures a single congestion groups across all TCs. + * If reset, a congestion group is allocated for each TC. Only relevant if the + * DPNI has multiple traffic classes. + */ +#define DPNI_OPT_SHARED_CONGESTION 0x000008 +/** + * Enables TCAM for Flow Steering and QoS look-ups. If not specified, all + * look-ups are exact match. Note that TCAM is not available on LS1088 and its + * variants. Setting this bit on these SoCs will trigger an error. + */ +#define DPNI_OPT_HAS_KEY_MASKING 0x000010 +/** + * Disables the flow steering table. + */ +#define DPNI_OPT_NO_FS 0x000020 + +int dpni_open(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + int dpni_id, + uint16_t *token); + +int dpni_close(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token); + +/** + * struct dpni_cfg - Structure representing DPNI configuration + * @mac_addr: Primary MAC address + * @adv: Advanced parameters; default is all zeros; + * use this structure to change default settings + */ +struct dpni_cfg { + /** + * @options: Any combination of the following options: + * DPNI_OPT_TX_FRM_RELEASE + * DPNI_OPT_NO_MAC_FILTER + * DPNI_OPT_HAS_POLICING + * DPNI_OPT_SHARED_CONGESTION + * DPNI_OPT_HAS_KEY_MASKING + * DPNI_OPT_NO_FS + * @fs_entries: Number of entries in the flow steering table. + * This table is used to select the ingress queue for + * ingress traffic, targeting a GPP core or another. + * In addition it can be used to discard traffic that + * matches the set rule. It is either an exact match table + * or a TCAM table, depending on DPNI_OPT_ HAS_KEY_MASKING + * bit in OPTIONS field. This field is ignored if + * DPNI_OPT_NO_FS bit is set in OPTIONS field. Otherwise, + * value 0 defaults to 64. Maximum supported value is 1024. + * Note that the total number of entries is limited on the + * SoC to as low as 512 entries if TCAM is used. + * @vlan_filter_entries: Number of entries in the VLAN address filtering + * table. This is an exact match table used to filter + * ingress traffic based on VLAN IDs. Value 0 disables VLAN + * filtering. Maximum supported value is 16. + * @mac_filter_entries: Number of entries in the MAC address filtering + * table. This is an exact match table and allows both + * unicast and multicast entries. The primary MAC address + * of the network interface is not part of this table, + * this contains only entries in addition to it. This + * field is ignored if DPNI_OPT_ NO_MAC_FILTER is set in + * OPTIONS field. Otherwise, value 0 defaults to 80. + * Maximum supported value is 80. + * @num_queues: Number of Tx and Rx queues used for traffic + * distribution. This is orthogonal to QoS and is only + * used to distribute traffic to multiple GPP cores. + * This configuration affects the number of Tx queues + * (logical FQs, all associated with a single CEETM queue), + * Rx queues and Tx confirmation queues, if applicable. + * Value 0 defaults to one queue. Maximum supported value + * is 8. + * @num_tcs: Number of traffic classes (TCs), reserved for the DPNI. + * TCs can have different priority levels for the purpose + * of Tx scheduling (see DPNI_SET_TX_SELECTION), different + * BPs (DPNI_ SET_POOLS), policers. There are dedicated QM + * queues for traffic classes (including class queues on + * Tx). Value 0 defaults to one TC. Maximum supported value + * is 8. + * @qos_entries: Number of entries in the QoS classification table. This + * table is used to select the TC for ingress traffic. It + * is either an exact match or a TCAM table, depending on + * DPNI_OPT_ HAS_KEY_MASKING bit in OPTIONS field. This + * field is ignored if the DPNI has a single TC. Otherwise, + * a value of 0 defaults to 64. Maximum supported value + * is 64. + */ + uint32_t options; + uint16_t fs_entries; + uint8_t vlan_filter_entries; + uint8_t mac_filter_entries; + uint8_t num_queues; + uint8_t num_tcs; + uint8_t qos_entries; +}; + +int dpni_create(struct fsl_mc_io *mc_io, + uint16_t dprc_token, + uint32_t cmd_flags, + const struct dpni_cfg *cfg, + uint32_t *obj_id); + +int dpni_destroy(struct fsl_mc_io *mc_io, + uint16_t dprc_token, + uint32_t cmd_flags, + uint32_t object_id); + +/** + * struct dpni_pools_cfg - Structure representing buffer pools configuration + * @num_dpbp: Number of DPBPs + * @pools: Array of buffer pools parameters; The number of valid entries + * must match 'num_dpbp' value + */ +struct dpni_pools_cfg { + uint8_t num_dpbp; + /** + * struct pools - Buffer pools parameters + * @dpbp_id: DPBP object ID + * @priority: priority mask that indicates TC's used with this buffer. + * I set to 0x00 MC will assume value 0xff. + * @buffer_size: Buffer size + * @backup_pool: Backup pool + */ + struct { + int dpbp_id; + uint8_t priority_mask; + uint16_t buffer_size; + int backup_pool; + } pools[DPNI_MAX_DPBP]; +}; + +int dpni_set_pools(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + const struct dpni_pools_cfg *cfg); + +int dpni_enable(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token); + +int dpni_disable(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token); + +int dpni_is_enabled(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + int *en); + +int dpni_reset(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token); + +/** + * DPNI IRQ Index and Events + */ + +/** + * IRQ index + */ +#define DPNI_IRQ_INDEX 0 +/** + * IRQ event - indicates a change in link state + */ +#define DPNI_IRQ_EVENT_LINK_CHANGED 0x00000001 + +int dpni_set_irq_enable(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t irq_index, + uint8_t en); + +int dpni_get_irq_enable(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t irq_index, + uint8_t *en); + +int dpni_set_irq_mask(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t irq_index, + uint32_t mask); + +int dpni_get_irq_mask(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t irq_index, + uint32_t *mask); + +int dpni_get_irq_status(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t irq_index, + uint32_t *status); + +int dpni_clear_irq_status(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t irq_index, + uint32_t status); + +/** + * struct dpni_attr - Structure representing DPNI attributes + * @options: Any combination of the following options: + * DPNI_OPT_TX_FRM_RELEASE + * DPNI_OPT_NO_MAC_FILTER + * DPNI_OPT_HAS_POLICING + * DPNI_OPT_SHARED_CONGESTION + * DPNI_OPT_HAS_KEY_MASKING + * DPNI_OPT_NO_FS + * @num_queues: Number of Tx and Rx queues used for traffic distribution. + * @num_rx_tcs: Number of RX traffic classes (TCs), reserved for the DPNI. + * @num_tx_tcs: Number of TX traffic classes (TCs), reserved for the DPNI. + * @mac_filter_entries: Number of entries in the MAC address filtering + * table. + * @vlan_filter_entries: Number of entries in the VLAN address filtering + * table. + * @qos_entries: Number of entries in the QoS classification table. + * @fs_entries: Number of entries in the flow steering table. + * @qos_key_size: Size, in bytes, of the QoS look-up key. Defining a key larger + * than this when adding QoS entries will result + * in an error. + * @fs_key_size: Size, in bytes, of the flow steering look-up key. Defining a + * key larger than this when composing the hash + FS key + * will result in an error. + * @wriop_version: Version of WRIOP HW block. + * The 3 version values are stored on 6, 5, 5 bits + * respectively. + * Values returned: + * - 0x400 - WRIOP version 1.0.0, used on LS2080 and + * variants, + * - 0x421 - WRIOP version 1.1.1, used on LS2088 and + * variants, + * - 0x422 - WRIOP version 1.1.2, used on LS1088 and + * variants. + */ +struct dpni_attr { + uint32_t options; + uint8_t num_queues; + uint8_t num_rx_tcs; + uint8_t num_tx_tcs; + uint8_t mac_filter_entries; + uint8_t vlan_filter_entries; + uint8_t qos_entries; + uint16_t fs_entries; + uint8_t qos_key_size; + uint8_t fs_key_size; + uint16_t wriop_version; +}; + +int dpni_get_attributes(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + struct dpni_attr *attr); + +/** + * DPNI errors + */ + +/** + * Extract out of frame header error + */ +#define DPNI_ERROR_EOFHE 0x00020000 +/** + * Frame length error + */ +#define DPNI_ERROR_FLE 0x00002000 +/** + * Frame physical error + */ +#define DPNI_ERROR_FPE 0x00001000 +/** + * Parsing header error + */ +#define DPNI_ERROR_PHE 0x00000020 +/** + * Parser L3 checksum error + */ +#define DPNI_ERROR_L3CE 0x00000004 +/** + * Parser L3 checksum error + */ +#define DPNI_ERROR_L4CE 0x00000001 + +/** + * enum dpni_error_action - Defines DPNI behavior for errors + * @DPNI_ERROR_ACTION_DISCARD: Discard the frame + * @DPNI_ERROR_ACTION_CONTINUE: Continue with the normal flow + * @DPNI_ERROR_ACTION_SEND_TO_ERROR_QUEUE: Send the frame to the error queue + */ +enum dpni_error_action { + DPNI_ERROR_ACTION_DISCARD = 0, + DPNI_ERROR_ACTION_CONTINUE = 1, + DPNI_ERROR_ACTION_SEND_TO_ERROR_QUEUE = 2 +}; + +/** + * struct dpni_error_cfg - Structure representing DPNI errors treatment + * @errors: Errors mask; use 'DPNI_ERROR__ + * @error_action: The desired action for the errors mask + * @set_frame_annotation: Set to '1' to mark the errors in frame + * annotation status (FAS); relevant only + * for the non-discard action + */ +struct dpni_error_cfg { + uint32_t errors; + enum dpni_error_action error_action; + int set_frame_annotation; +}; + +int dpni_set_errors_behavior(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + struct dpni_error_cfg *cfg); + +/** + * DPNI buffer layout modification options + */ + +/** + * Select to modify the time-stamp setting + */ +#define DPNI_BUF_LAYOUT_OPT_TIMESTAMP 0x00000001 +/** + * Select to modify the parser-result setting; not applicable for Tx + */ +#define DPNI_BUF_LAYOUT_OPT_PARSER_RESULT 0x00000002 +/** + * Select to modify the frame-status setting + */ +#define DPNI_BUF_LAYOUT_OPT_FRAME_STATUS 0x00000004 +/** + * Select to modify the private-data-size setting + */ +#define DPNI_BUF_LAYOUT_OPT_PRIVATE_DATA_SIZE 0x00000008 +/** + * Select to modify the data-alignment setting + */ +#define DPNI_BUF_LAYOUT_OPT_DATA_ALIGN 0x00000010 +/** + * Select to modify the data-head-room setting + */ +#define DPNI_BUF_LAYOUT_OPT_DATA_HEAD_ROOM 0x00000020 +/** + * Select to modify the data-tail-room setting + */ +#define DPNI_BUF_LAYOUT_OPT_DATA_TAIL_ROOM 0x00000040 + +/** + * struct dpni_buffer_layout - Structure representing DPNI buffer layout + * @options: Flags representing the suggested modifications to the + * buffer layout; + * Use any combination of 'DPNI_BUF_LAYOUT_OPT_' flags + * @pass_timestamp: Pass timestamp value + * @pass_parser_result: Pass parser results + * @pass_frame_status: Pass frame status + * @private_data_size: Size kept for private data (in bytes) + * @data_align: Data alignment + * @data_head_room: Data head room + * @data_tail_room: Data tail room + */ +struct dpni_buffer_layout { + uint32_t options; + int pass_timestamp; + int pass_parser_result; + int pass_frame_status; + uint16_t private_data_size; + uint16_t data_align; + uint16_t data_head_room; + uint16_t data_tail_room; +}; + +/** + * enum dpni_queue_type - Identifies a type of queue targeted by the command + * @DPNI_QUEUE_RX: Rx queue + * @DPNI_QUEUE_TX: Tx queue + * @DPNI_QUEUE_TX_CONFIRM: Tx confirmation queue + * @DPNI_QUEUE_RX_ERR: Rx error queue + */ +enum dpni_queue_type { + DPNI_QUEUE_RX, + DPNI_QUEUE_TX, + DPNI_QUEUE_TX_CONFIRM, + DPNI_QUEUE_RX_ERR, +}; + +int dpni_get_buffer_layout(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + enum dpni_queue_type qtype, + struct dpni_buffer_layout *layout); + +int dpni_set_buffer_layout(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + enum dpni_queue_type qtype, + const struct dpni_buffer_layout *layout); + +/** + * enum dpni_offload - Identifies a type of offload targeted by the command + * @DPNI_OFF_RX_L3_CSUM: Rx L3 checksum validation + * @DPNI_OFF_RX_L4_CSUM: Rx L4 checksum validation + * @DPNI_OFF_TX_L3_CSUM: Tx L3 checksum generation + * @DPNI_OFF_TX_L4_CSUM: Tx L4 checksum generation + * @DPNI_OPT_FLCTYPE_HASH: flow context will be generated by WRIOP for AIOP or + * for CPU + */ +enum dpni_offload { + DPNI_OFF_RX_L3_CSUM, + DPNI_OFF_RX_L4_CSUM, + DPNI_OFF_TX_L3_CSUM, + DPNI_OFF_TX_L4_CSUM, + DPNI_FLCTYPE_HASH, +}; + +int dpni_set_offload(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + enum dpni_offload type, + uint32_t config); + +int dpni_get_offload(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + enum dpni_offload type, + uint32_t *config); + +int dpni_get_qdid(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + enum dpni_queue_type qtype, + uint16_t *qdid); + +int dpni_get_tx_data_offset(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint16_t *data_offset); + +#define DPNI_STATISTICS_CNT 7 + +union dpni_statistics { + /** + * struct page_0 - Page_0 statistics structure + * @ingress_all_frames: Ingress frame count + * @ingress_all_bytes: Ingress byte count + * @ingress_multicast_frames: Ingress multicast frame count + * @ingress_multicast_bytes: Ingress multicast byte count + * @ingress_broadcast_frames: Ingress broadcast frame count + * @ingress_broadcast_bytes: Ingress broadcast byte count + */ + struct { + uint64_t ingress_all_frames; + uint64_t ingress_all_bytes; + uint64_t ingress_multicast_frames; + uint64_t ingress_multicast_bytes; + uint64_t ingress_broadcast_frames; + uint64_t ingress_broadcast_bytes; + } page_0; + /** + * struct page_1 - Page_1 statistics structure + * @egress_all_frames: Egress frame count + * @egress_all_bytes: Egress byte count + * @egress_multicast_frames: Egress multicast frame count + * @egress_multicast_bytes: Egress multicast byte count + * @egress_broadcast_frames: Egress broadcast frame count + * @egress_broadcast_bytes: Egress broadcast byte count + */ + struct { + uint64_t egress_all_frames; + uint64_t egress_all_bytes; + uint64_t egress_multicast_frames; + uint64_t egress_multicast_bytes; + uint64_t egress_broadcast_frames; + uint64_t egress_broadcast_bytes; + } page_1; + /** + * struct page_2 - Page_2 statistics structure + * @ingress_filtered_frames: Ingress filtered frame count + * @ingress_discarded_frames: Ingress discarded frame count + * @ingress_nobuffer_discards: Ingress discarded frame count due to + * lack of buffers + * @egress_discarded_frames: Egress discarded frame count + * @egress_confirmed_frames: Egress confirmed frame count + */ + struct { + uint64_t ingress_filtered_frames; + uint64_t ingress_discarded_frames; + uint64_t ingress_nobuffer_discards; + uint64_t egress_discarded_frames; + uint64_t egress_confirmed_frames; + } page_2; + /** + * struct page_3 - Page_3 statistics structure with values for the + * selected TC + * @ceetm_dequeue_bytes: Cumulative count of the number of bytes + * dequeued + * @ceetm_dequeue_frames: Cumulative count of the number of frames + * dequeued + * @ceetm_reject_bytes: Cumulative count of the number of bytes in all + * frames whose enqueue was rejected + * @ceetm_reject_frames: Cumulative count of all frame enqueues rejected + */ + struct { + uint64_t ceetm_dequeue_bytes; + uint64_t ceetm_dequeue_frames; + uint64_t ceetm_reject_bytes; + uint64_t ceetm_reject_frames; + } page_3; + /** + * struct raw - raw statistics structure, used to index counters + */ + struct { + uint64_t counter[DPNI_STATISTICS_CNT]; + } raw; +}; + +/** + * Enable auto-negotiation + */ +#define DPNI_LINK_OPT_AUTONEG 0x0000000000000001ULL +/** + * Enable half-duplex mode + */ +#define DPNI_LINK_OPT_HALF_DUPLEX 0x0000000000000002ULL +/** + * Enable pause frames + */ +#define DPNI_LINK_OPT_PAUSE 0x0000000000000004ULL +/** + * Enable a-symmetric pause frames + */ +#define DPNI_LINK_OPT_ASYM_PAUSE 0x0000000000000008ULL +/** + * Enable priority flow control pause frames + */ +#define DPNI_LINK_OPT_PFC_PAUSE 0x0000000000000010ULL + +/** + * struct - Structure representing DPNI link configuration + * @rate: Rate + * @options: Mask of available options; use 'DPNI_LINK_OPT_' values + */ +struct dpni_link_cfg { + uint32_t rate; + uint64_t options; +}; + +int dpni_set_link_cfg(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + const struct dpni_link_cfg *cfg); + +/** + * struct dpni_link_state - Structure representing DPNI link state + * @rate: Rate + * @options: Mask of available options; use 'DPNI_LINK_OPT_' values + * @up: Link state; '0' for down, '1' for up + */ +struct dpni_link_state { + uint32_t rate; + uint64_t options; + int up; +}; + +int dpni_get_link_state(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + struct dpni_link_state *state); + +int dpni_set_max_frame_length(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint16_t max_frame_length); + +int dpni_get_max_frame_length(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint16_t *max_frame_length); + +int dpni_set_mtu(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint16_t mtu); + +int dpni_get_mtu(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint16_t *mtu); + +int dpni_set_multicast_promisc(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + int en); + +int dpni_get_multicast_promisc(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + int *en); + +int dpni_set_unicast_promisc(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + int en); + +int dpni_get_unicast_promisc(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + int *en); + +int dpni_set_primary_mac_addr(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + const uint8_t mac_addr[6]); + +int dpni_get_primary_mac_addr(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t mac_addr[6]); + +int dpni_add_mac_addr(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + const uint8_t mac_addr[6]); + +int dpni_remove_mac_addr(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + const uint8_t mac_addr[6]); + +int dpni_clear_mac_filters(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + int unicast, + int multicast); + +int dpni_get_port_mac_addr(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t mac_addr[6]); + +int dpni_enable_vlan_filter(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + int en); + +int dpni_add_vlan_id(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint16_t vlan_id); + +int dpni_remove_vlan_id(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint16_t vlan_id); + +int dpni_clear_vlan_filters(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token); + +/** + * enum dpni_dist_mode - DPNI distribution mode + * @DPNI_DIST_MODE_NONE: No distribution + * @DPNI_DIST_MODE_HASH: Use hash distribution; only relevant if + * the 'DPNI_OPT_DIST_HASH' option was set at DPNI creation + * @DPNI_DIST_MODE_FS: Use explicit flow steering; only relevant if + * the 'DPNI_OPT_DIST_FS' option was set at DPNI creation + */ +enum dpni_dist_mode { + DPNI_DIST_MODE_NONE = 0, + DPNI_DIST_MODE_HASH = 1, + DPNI_DIST_MODE_FS = 2 +}; + +/** + * enum dpni_fs_miss_action - DPNI Flow Steering miss action + * @DPNI_FS_MISS_DROP: In case of no-match, drop the frame + * @DPNI_FS_MISS_EXPLICIT_FLOWID: In case of no-match, use explicit flow-id + * @DPNI_FS_MISS_HASH: In case of no-match, distribute using hash + */ +enum dpni_fs_miss_action { + DPNI_FS_MISS_DROP = 0, + DPNI_FS_MISS_EXPLICIT_FLOWID = 1, + DPNI_FS_MISS_HASH = 2 +}; + +/** + * struct dpni_fs_tbl_cfg - Flow Steering table configuration + * @miss_action: Miss action selection + * @default_flow_id: Used when 'miss_action = DPNI_FS_MISS_EXPLICIT_FLOWID' + */ +struct dpni_fs_tbl_cfg { + enum dpni_fs_miss_action miss_action; + uint16_t default_flow_id; + char keep_hash_key; +}; + +/** + * struct dpni_rx_tc_dist_cfg - Rx traffic class distribution configuration + * @dist_size: Set the distribution size; + * supported values: 1,2,3,4,6,7,8,12,14,16,24,28,32,48,56,64,96, + * 112,128,192,224,256,384,448,512,768,896,1024 + * @dist_mode: Distribution mode + * @key_cfg_iova: I/O virtual address of 256 bytes DMA-able memory filled with + * the extractions to be used for the distribution key by calling + * dpkg_prepare_key_cfg() relevant only when + * 'dist_mode != DPNI_DIST_MODE_NONE', otherwise it can be '0' + * @fs_cfg: Flow Steering table configuration; only relevant if + * 'dist_mode = DPNI_DIST_MODE_FS' + */ +struct dpni_rx_tc_dist_cfg { + uint16_t dist_size; + enum dpni_dist_mode dist_mode; + uint64_t key_cfg_iova; + struct dpni_fs_tbl_cfg fs_cfg; +}; + +int dpni_set_rx_tc_dist(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t tc_id, + const struct dpni_rx_tc_dist_cfg *cfg); + +/** + * enum dpni_congestion_unit - DPNI congestion units + * @DPNI_CONGESTION_UNIT_BYTES: bytes units + * @DPNI_CONGESTION_UNIT_FRAMES: frames units + */ +enum dpni_congestion_unit { + DPNI_CONGESTION_UNIT_BYTES = 0, + DPNI_CONGESTION_UNIT_FRAMES +}; + + +/** + * enum dpni_dest - DPNI destination types + * @DPNI_DEST_NONE: Unassigned destination; The queue is set in parked mode and + * does not generate FQDAN notifications; user is expected to + * dequeue from the queue based on polling or other user-defined + * method + * @DPNI_DEST_DPIO: The queue is set in schedule mode and generates FQDAN + * notifications to the specified DPIO; user is expected to dequeue + * from the queue only after notification is received + * @DPNI_DEST_DPCON: The queue is set in schedule mode and does not generate + * FQDAN notifications, but is connected to the specified DPCON + * object; user is expected to dequeue from the DPCON channel + */ +enum dpni_dest { + DPNI_DEST_NONE = 0, + DPNI_DEST_DPIO = 1, + DPNI_DEST_DPCON = 2 +}; + +/** + * struct dpni_dest_cfg - Structure representing DPNI destination parameters + * @dest_type: Destination type + * @dest_id: Either DPIO ID or DPCON ID, depending on the destination type + * @priority: Priority selection within the DPIO or DPCON channel; valid values + * are 0-1 or 0-7, depending on the number of priorities in that + * channel; not relevant for 'DPNI_DEST_NONE' option + */ +struct dpni_dest_cfg { + enum dpni_dest dest_type; + int dest_id; + uint8_t priority; +}; + +/* DPNI congestion options */ + +/** + * CSCN message is written to message_iova once entering a + * congestion state (see 'threshold_entry') + */ +#define DPNI_CONG_OPT_WRITE_MEM_ON_ENTER 0x00000001 +/** + * CSCN message is written to message_iova once exiting a + * congestion state (see 'threshold_exit') + */ +#define DPNI_CONG_OPT_WRITE_MEM_ON_EXIT 0x00000002 +/** + * CSCN write will attempt to allocate into a cache (coherent write); + * valid only if 'DPNI_CONG_OPT_WRITE_MEM_' is selected + */ +#define DPNI_CONG_OPT_COHERENT_WRITE 0x00000004 +/** + * if 'dest_cfg.dest_type != DPNI_DEST_NONE' CSCN message is sent to + * DPIO/DPCON's WQ channel once entering a congestion state + * (see 'threshold_entry') + */ +#define DPNI_CONG_OPT_NOTIFY_DEST_ON_ENTER 0x00000008 +/** + * if 'dest_cfg.dest_type != DPNI_DEST_NONE' CSCN message is sent to + * DPIO/DPCON's WQ channel once exiting a congestion state + * (see 'threshold_exit') + */ +#define DPNI_CONG_OPT_NOTIFY_DEST_ON_EXIT 0x00000010 +/** + * if 'dest_cfg.dest_type != DPNI_DEST_NONE' when the CSCN is written to the + * sw-portal's DQRR, the DQRI interrupt is asserted immediately (if enabled) + */ +#define DPNI_CONG_OPT_INTR_COALESCING_DISABLED 0x00000020 +/** + * This congestion will trigger flow control or priority flow control. This + * will have effect only if flow control is enabled with dpni_set_link_cfg() + */ +#define DPNI_CONG_OPT_FLOW_CONTROL 0x00000040 + +/** + * struct dpni_congestion_notification_cfg - congestion notification + * configuration + * @units: units type + * @threshold_entry: above this threshold we enter a congestion state. + * set it to '0' to disable it + * @threshold_exit: below this threshold we exit the congestion state. + * @message_ctx: The context that will be part of the CSCN message + * @message_iova: I/O virtual address (must be in DMA-able memory), + * must be 16B aligned; valid only if 'DPNI_CONG_OPT_WRITE_MEM_' is + * contained in 'options' + * @dest_cfg: CSCN can be send to either DPIO or DPCON WQ channel + * @notification_mode: Mask of available options; use 'DPNI_CONG_OPT_' values + */ + +struct dpni_congestion_notification_cfg { + enum dpni_congestion_unit units; + uint32_t threshold_entry; + uint32_t threshold_exit; + uint64_t message_ctx; + uint64_t message_iova; + struct dpni_dest_cfg dest_cfg; + uint16_t notification_mode; +}; + +int dpni_set_congestion_notification(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + enum dpni_queue_type qtype, + uint8_t tc_id, + const struct dpni_congestion_notification_cfg *cfg); + +int dpni_get_congestion_notification(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + enum dpni_queue_type qtype, + uint8_t tc_id, + struct dpni_congestion_notification_cfg *cfg); + +/* DPNI FLC stash options */ + +/** + * stashes the whole annotation area (up to 192 bytes) + */ +#define DPNI_FLC_STASH_FRAME_ANNOTATION 0x00000001 + +/** + * struct dpni_queue - Queue structure + * @user_context: User data, presented to the user along with any frames + * from this queue. Not relevant for Tx queues. + */ +struct dpni_queue { + /** + * struct destination - Destination structure + * @id: ID of the destination, only relevant if DEST_TYPE is > 0. + * Identifies either a DPIO or a DPCON object. + * Not relevant for Tx queues. + * @type: May be one of the following: + * 0 - No destination, queue can be manually + * queried, but will not push traffic or + * notifications to a DPIO; + * 1 - The destination is a DPIO. When traffic + * becomes available in the queue a FQDAN + * (FQ data available notification) will be + * generated to selected DPIO; + * 2 - The destination is a DPCON. The queue is + * associated with a DPCON object for the + * purpose of scheduling between multiple + * queues. The DPCON may be independently + * configured to generate notifications. + * Not relevant for Tx queues. + * @hold_active: Hold active, maintains a queue scheduled for longer + * in a DPIO during dequeue to reduce spread of traffic. + * Only relevant if queues are + * not affined to a single DPIO. + */ + struct { + uint16_t id; + enum dpni_dest type; + char hold_active; + uint8_t priority; + } destination; + uint64_t user_context; + /** + * struct flc - FD FLow Context structure + * @value: Default FLC value for traffic dequeued from + * this queue. Please check description of FD + * structure for more information. + * Note that FLC values set using dpni_add_fs_entry, + * if any, take precedence over values per queue. + * @stash_control: Boolean, indicates whether the 6 lowest + * - significant bits are used for stash control. + * significant bits are used for stash control. If set, the 6 + * least significant bits in value are interpreted as follows: + * - bits 0-1: indicates the number of 64 byte units of context + * that are stashed. FLC value is interpreted as a memory address + * in this case, excluding the 6 LS bits. + * - bits 2-3: indicates the number of 64 byte units of frame + * annotation to be stashed. Annotation is placed at FD[ADDR]. + * - bits 4-5: indicates the number of 64 byte units of frame + * data to be stashed. Frame data is placed at FD[ADDR] + + * FD[OFFSET]. + * For more details check the Frame Descriptor section in the + * hardware documentation. + */ + struct { + uint64_t value; + char stash_control; + } flc; +}; + +/** + * struct dpni_queue_id - Queue identification, used for enqueue commands + * or queue control + * @fqid: FQID used for enqueueing to and/or configuration of this + * specific FQ + * @qdbin: Queueing bin, used to enqueue using QDID, DQBIN, QPRI. + * Only relevant for Tx queues. + */ +struct dpni_queue_id { + uint32_t fqid; + uint16_t qdbin; +}; + +/** + * enum dpni_confirmation_mode - Defines DPNI options supported for Tx + * confirmation + * @DPNI_CONF_AFFINE: For each Tx queue set associated with a sender there is + * an affine Tx Confirmation queue + * @DPNI_CONF_SINGLE: All Tx queues are associated with a single Tx + * confirmation queue + * @DPNI_CONF_DISABLE: Tx frames are not confirmed. This must be associated + * with proper FD set-up to have buffers release to a Buffer Pool, otherwise + * buffers will be leaked + */ +enum dpni_confirmation_mode { + DPNI_CONF_AFFINE, + DPNI_CONF_SINGLE, + DPNI_CONF_DISABLE, +}; + +int dpni_set_tx_confirmation_mode(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + enum dpni_confirmation_mode mode); + +int dpni_get_tx_confirmation_mode(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + enum dpni_confirmation_mode *mode); + +int dpni_get_api_version(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t *major_ver, + uint16_t *minor_ver); + +/** + * Set User Context + */ +#define DPNI_QUEUE_OPT_USER_CTX 0x00000001 + +/** + * Set queue destination configuration + */ +#define DPNI_QUEUE_OPT_DEST 0x00000002 + +/** + * Set FD[FLC] configuration for traffic on this queue. Note that FLC values + * set with dpni_add_fs_entry, if any, take precedence over values per queue. + */ +#define DPNI_QUEUE_OPT_FLC 0x00000004 + +/** + * Set the queue to hold active mode. This prevents the queue from being + * rescheduled between DPIOs while it carries traffic and is active on one + * DPNI. Can help reduce reordering when servicing one queue on multiple + * CPUs, but the queue is also less likely to push data to multiple CPUs + * especially when congested. + */ +#define DPNI_QUEUE_OPT_HOLD_ACTIVE 0x00000008 + +int dpni_set_queue(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + enum dpni_queue_type qtype, + uint8_t tc, + uint8_t index, + uint8_t options, + const struct dpni_queue *queue); + +int dpni_get_queue(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + enum dpni_queue_type qtype, + uint8_t tc, + uint8_t index, + struct dpni_queue *queue, + struct dpni_queue_id *qid); + +int dpni_get_statistics(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t page, + uint8_t param, + union dpni_statistics *stat); + +int dpni_reset_statistics(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token); + +/** + * enum dpni_congestion_point - Structure representing congestion point + * @DPNI_CP_QUEUE: Set taildrop per queue, identified by QUEUE_TYPE, TC and + * QUEUE_INDEX + * @DPNI_CP_GROUP: Set taildrop per queue group. Depending on options used + * to define the DPNI this can be either per + * TC (default) or per interface + * (DPNI_OPT_SHARED_CONGESTION set at DPNI create). + * QUEUE_INDEX is ignored if this type is used. + */ +enum dpni_congestion_point { + DPNI_CP_QUEUE, + DPNI_CP_GROUP, +}; + +/** + * struct dpni_taildrop - Structure representing the taildrop + * @enable: Indicates whether the taildrop is active or not. + * @units: Indicates the unit of THRESHOLD. Queue taildrop only + * supports byte units, this field is ignored and + * assumed = 0 if CONGESTION_POINT is 0. + * @threshold: Threshold value, in units identified by UNITS field. Value 0 + * cannot be used as a valid taildrop threshold, + * THRESHOLD must be > 0 if the taildrop is + * enabled. + * @oal : Overhead Accounting Length, a 12-bit, 2's complement value + * with range (-2048 to +2047) representing a fixed per-frame + * overhead to be added to the actual length of a frame when + * performing WRED and tail drop calculations and threshold + * comparisons. + */ +struct dpni_taildrop { + char enable; + enum dpni_congestion_unit units; + uint32_t threshold; + int16_t oal; +}; + +int dpni_set_taildrop(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + enum dpni_congestion_point cg_point, + enum dpni_queue_type q_type, + uint8_t tc, + uint8_t q_index, + struct dpni_taildrop *taildrop); + +int dpni_get_taildrop(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + enum dpni_congestion_point cg_point, + enum dpni_queue_type q_type, + uint8_t tc, + uint8_t q_index, + struct dpni_taildrop *taildrop); +#endif /* __FSL_DPNI_H */ diff --git a/dpdk/drivers/net/dpaa2/mc/fsl_dpni_cmd.h b/dpdk/drivers/net/dpaa2/mc/fsl_dpni_cmd.h new file mode 100644 index 00000000..1a483329 --- /dev/null +++ b/dpdk/drivers/net/dpaa2/mc/fsl_dpni_cmd.h @@ -0,0 +1,638 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2013-2016 Freescale Semiconductor Inc. + * Copyright 2016-2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef _FSL_DPNI_CMD_H +#define _FSL_DPNI_CMD_H + +/* DPNI Version */ +#define DPNI_VER_MAJOR 7 +#define DPNI_VER_MINOR 3 + +#define DPNI_CMD_BASE_VERSION 1 +#define DPNI_CMD_VERSION_2 2 +#define DPNI_CMD_ID_OFFSET 4 + +#define DPNI_CMD(id) (((id) << DPNI_CMD_ID_OFFSET) | DPNI_CMD_BASE_VERSION) +#define DPNI_CMD_V2(id) (((id) << DPNI_CMD_ID_OFFSET) | DPNI_CMD_VERSION_2) + +/* Command IDs */ +#define DPNI_CMDID_OPEN DPNI_CMD(0x801) +#define DPNI_CMDID_CLOSE DPNI_CMD(0x800) +#define DPNI_CMDID_CREATE DPNI_CMD(0x901) +#define DPNI_CMDID_DESTROY DPNI_CMD(0x981) +#define DPNI_CMDID_GET_API_VERSION DPNI_CMD(0xa01) + +#define DPNI_CMDID_ENABLE DPNI_CMD(0x002) +#define DPNI_CMDID_DISABLE DPNI_CMD(0x003) +#define DPNI_CMDID_GET_ATTR DPNI_CMD_V2(0x004) +#define DPNI_CMDID_RESET DPNI_CMD(0x005) +#define DPNI_CMDID_IS_ENABLED DPNI_CMD(0x006) + +#define DPNI_CMDID_SET_IRQ_ENABLE DPNI_CMD(0x012) +#define DPNI_CMDID_GET_IRQ_ENABLE DPNI_CMD(0x013) +#define DPNI_CMDID_SET_IRQ_MASK DPNI_CMD(0x014) +#define DPNI_CMDID_GET_IRQ_MASK DPNI_CMD(0x015) +#define DPNI_CMDID_GET_IRQ_STATUS DPNI_CMD(0x016) +#define DPNI_CMDID_CLEAR_IRQ_STATUS DPNI_CMD(0x017) + +#define DPNI_CMDID_SET_POOLS DPNI_CMD_V2(0x200) +#define DPNI_CMDID_SET_ERRORS_BEHAVIOR DPNI_CMD(0x20B) + +#define DPNI_CMDID_GET_QDID DPNI_CMD(0x210) +#define DPNI_CMDID_GET_SP_INFO DPNI_CMD(0x211) +#define DPNI_CMDID_GET_TX_DATA_OFFSET DPNI_CMD(0x212) +#define DPNI_CMDID_GET_LINK_STATE DPNI_CMD(0x215) +#define DPNI_CMDID_SET_MAX_FRAME_LENGTH DPNI_CMD(0x216) +#define DPNI_CMDID_GET_MAX_FRAME_LENGTH DPNI_CMD(0x217) +#define DPNI_CMDID_SET_LINK_CFG DPNI_CMD(0x21A) +#define DPNI_CMDID_SET_TX_SHAPING DPNI_CMD_V2(0x21B) + +#define DPNI_CMDID_SET_MCAST_PROMISC DPNI_CMD(0x220) +#define DPNI_CMDID_GET_MCAST_PROMISC DPNI_CMD(0x221) +#define DPNI_CMDID_SET_UNICAST_PROMISC DPNI_CMD(0x222) +#define DPNI_CMDID_GET_UNICAST_PROMISC DPNI_CMD(0x223) +#define DPNI_CMDID_SET_PRIM_MAC DPNI_CMD(0x224) +#define DPNI_CMDID_GET_PRIM_MAC DPNI_CMD(0x225) +#define DPNI_CMDID_ADD_MAC_ADDR DPNI_CMD(0x226) +#define DPNI_CMDID_REMOVE_MAC_ADDR DPNI_CMD(0x227) +#define DPNI_CMDID_CLR_MAC_FILTERS DPNI_CMD(0x228) + +#define DPNI_CMDID_ENABLE_VLAN_FILTER DPNI_CMD(0x230) +#define DPNI_CMDID_ADD_VLAN_ID DPNI_CMD(0x231) +#define DPNI_CMDID_REMOVE_VLAN_ID DPNI_CMD(0x232) +#define DPNI_CMDID_CLR_VLAN_FILTERS DPNI_CMD(0x233) + +#define DPNI_CMDID_SET_RX_TC_DIST DPNI_CMD_V2(0x235) + +#define DPNI_CMDID_GET_STATISTICS DPNI_CMD_V2(0x25D) +#define DPNI_CMDID_RESET_STATISTICS DPNI_CMD(0x25E) +#define DPNI_CMDID_GET_QUEUE DPNI_CMD(0x25F) +#define DPNI_CMDID_SET_QUEUE DPNI_CMD(0x260) +#define DPNI_CMDID_GET_TAILDROP DPNI_CMD_V2(0x261) +#define DPNI_CMDID_SET_TAILDROP DPNI_CMD_V2(0x262) + +#define DPNI_CMDID_GET_PORT_MAC_ADDR DPNI_CMD(0x263) + +#define DPNI_CMDID_GET_BUFFER_LAYOUT DPNI_CMD(0x264) +#define DPNI_CMDID_SET_BUFFER_LAYOUT DPNI_CMD(0x265) + +#define DPNI_CMDID_SET_CONGESTION_NOTIFICATION DPNI_CMD(0x267) +#define DPNI_CMDID_GET_CONGESTION_NOTIFICATION DPNI_CMD(0x268) +#define DPNI_CMDID_SET_EARLY_DROP DPNI_CMD_V2(0x269) +#define DPNI_CMDID_GET_EARLY_DROP DPNI_CMD_V2(0x26A) +#define DPNI_CMDID_GET_OFFLOAD DPNI_CMD(0x26B) +#define DPNI_CMDID_SET_OFFLOAD DPNI_CMD(0x26C) +#define DPNI_CMDID_SET_TX_CONFIRMATION_MODE DPNI_CMD(0x266) +#define DPNI_CMDID_GET_TX_CONFIRMATION_MODE DPNI_CMD(0x26D) + +/* Macros for accessing command fields smaller than 1byte */ +#define DPNI_MASK(field) \ + GENMASK(DPNI_##field##_SHIFT + DPNI_##field##_SIZE - 1, \ + DPNI_##field##_SHIFT) +#define dpni_set_field(var, field, val) \ + ((var) |= (((val) << DPNI_##field##_SHIFT) & DPNI_MASK(field))) +#define dpni_get_field(var, field) \ + (((var) & DPNI_MASK(field)) >> DPNI_##field##_SHIFT) + +#pragma pack(push, 1) +struct dpni_cmd_open { + uint32_t dpni_id; +}; + +struct dpni_cmd_create { + uint32_t options; + uint8_t num_queues; + uint8_t num_tcs; + uint8_t mac_filter_entries; + uint8_t pad1; + uint8_t vlan_filter_entries; + uint8_t pad2; + uint8_t qos_entries; + uint8_t pad3; + uint16_t fs_entries; +}; + +struct dpni_cmd_destroy { + uint32_t dpsw_id; +}; + +#define DPNI_BACKUP_POOL(val, order) (((val) & 0x1) << (order)) + +struct dpni_cmd_pool { + uint16_t dpbp_id; + uint8_t priority_mask; + uint8_t pad; +}; + +struct dpni_cmd_set_pools { + uint8_t num_dpbp; + uint8_t backup_pool_mask; + uint16_t pad; + struct dpni_cmd_pool pool[8]; + uint16_t buffer_size[8]; +}; + +/* The enable indication is always the least significant bit */ +#define DPNI_ENABLE_SHIFT 0 +#define DPNI_ENABLE_SIZE 1 + +struct dpni_rsp_is_enabled { + uint8_t enabled; +}; + +struct dpni_cmd_set_irq_enable { + uint8_t enable; + uint8_t pad[3]; + uint8_t irq_index; +}; + +struct dpni_cmd_get_irq_enable { + uint32_t pad; + uint8_t irq_index; +}; + +struct dpni_rsp_get_irq_enable { + uint8_t enabled; +}; + +struct dpni_cmd_set_irq_mask { + uint32_t mask; + uint8_t irq_index; +}; + +struct dpni_cmd_get_irq_mask { + uint32_t pad; + uint8_t irq_index; +}; + +struct dpni_rsp_get_irq_mask { + uint32_t mask; +}; + +struct dpni_cmd_get_irq_status { + uint32_t status; + uint8_t irq_index; +}; + +struct dpni_rsp_get_irq_status { + uint32_t status; +}; + +struct dpni_cmd_clear_irq_status { + uint32_t status; + uint8_t irq_index; +}; + +struct dpni_rsp_get_attr { + /* response word 0 */ + uint32_t options; + uint8_t num_queues; + uint8_t num_rx_tcs; + uint8_t mac_filter_entries; + uint8_t num_tx_tcs; + /* response word 1 */ + uint8_t vlan_filter_entries; + uint8_t pad1; + uint8_t qos_entries; + uint8_t pad2; + uint16_t fs_entries; + uint16_t pad3; + /* response word 2 */ + uint8_t qos_key_size; + uint8_t fs_key_size; + uint16_t wriop_version; +}; + +#define DPNI_ERROR_ACTION_SHIFT 0 +#define DPNI_ERROR_ACTION_SIZE 4 +#define DPNI_FRAME_ANN_SHIFT 4 +#define DPNI_FRAME_ANN_SIZE 1 + +struct dpni_cmd_set_errors_behavior { + uint32_t errors; + /* from least significant bit: error_action:4, set_frame_annotation:1 */ + uint8_t flags; +}; + +/* There are 3 separate commands for configuring Rx, Tx and Tx confirmation + * buffer layouts, but they all share the same parameters. + * If one of the functions changes, below structure needs to be split. + */ + +#define DPNI_PASS_TS_SHIFT 0 +#define DPNI_PASS_TS_SIZE 1 +#define DPNI_PASS_PR_SHIFT 1 +#define DPNI_PASS_PR_SIZE 1 +#define DPNI_PASS_FS_SHIFT 2 +#define DPNI_PASS_FS_SIZE 1 + +struct dpni_cmd_get_buffer_layout { + uint8_t qtype; +}; + +struct dpni_rsp_get_buffer_layout { + /* response word 0 */ + uint8_t pad0[6]; + /* from LSB: pass_timestamp:1, parser_result:1, frame_status:1 */ + uint8_t flags; + uint8_t pad1; + /* response word 1 */ + uint16_t private_data_size; + uint16_t data_align; + uint16_t head_room; + uint16_t tail_room; +}; + +struct dpni_cmd_set_buffer_layout { + /* cmd word 0 */ + uint8_t qtype; + uint8_t pad0[3]; + uint16_t options; + /* from LSB: pass_timestamp:1, parser_result:1, frame_status:1 */ + uint8_t flags; + uint8_t pad1; + /* cmd word 1 */ + uint16_t private_data_size; + uint16_t data_align; + uint16_t head_room; + uint16_t tail_room; +}; + +struct dpni_cmd_set_offload { + uint8_t pad[3]; + uint8_t dpni_offload; + uint32_t config; +}; + +struct dpni_cmd_get_offload { + uint8_t pad[3]; + uint8_t dpni_offload; +}; + +struct dpni_rsp_get_offload { + uint32_t pad; + uint32_t config; +}; + +struct dpni_cmd_get_qdid { + uint8_t qtype; +}; + +struct dpni_rsp_get_qdid { + uint16_t qdid; +}; + +struct dpni_rsp_get_sp_info { + uint16_t spids[2]; +}; + +struct dpni_rsp_get_tx_data_offset { + uint16_t data_offset; +}; + +struct dpni_cmd_get_statistics { + uint8_t page_number; + uint8_t param; +}; + +struct dpni_rsp_get_statistics { + uint64_t counter[7]; +}; + +struct dpni_cmd_set_link_cfg { + uint64_t pad0; + uint32_t rate; + uint32_t pad1; + uint64_t options; +}; + +#define DPNI_LINK_STATE_SHIFT 0 +#define DPNI_LINK_STATE_SIZE 1 + +struct dpni_rsp_get_link_state { + uint32_t pad0; + /* from LSB: up:1 */ + uint8_t flags; + uint8_t pad1[3]; + uint32_t rate; + uint32_t pad2; + uint64_t options; +}; + +struct dpni_cmd_set_max_frame_length { + uint16_t max_frame_length; +}; + +struct dpni_rsp_get_max_frame_length { + uint16_t max_frame_length; +}; + +struct dpni_cmd_set_multicast_promisc { + uint8_t enable; +}; + +struct dpni_rsp_get_multicast_promisc { + uint8_t enabled; +}; + +struct dpni_cmd_set_unicast_promisc { + uint8_t enable; +}; + +struct dpni_rsp_get_unicast_promisc { + uint8_t enabled; +}; + +struct dpni_cmd_set_primary_mac_addr { + uint16_t pad; + uint8_t mac_addr[6]; +}; + +struct dpni_rsp_get_primary_mac_addr { + uint16_t pad; + uint8_t mac_addr[6]; +}; + +struct dpni_rsp_get_port_mac_addr { + uint16_t pad; + uint8_t mac_addr[6]; +}; + +struct dpni_cmd_add_mac_addr { + uint16_t pad; + uint8_t mac_addr[6]; +}; + +struct dpni_cmd_remove_mac_addr { + uint16_t pad; + uint8_t mac_addr[6]; +}; + +#define DPNI_UNICAST_FILTERS_SHIFT 0 +#define DPNI_UNICAST_FILTERS_SIZE 1 +#define DPNI_MULTICAST_FILTERS_SHIFT 1 +#define DPNI_MULTICAST_FILTERS_SIZE 1 + +struct dpni_cmd_clear_mac_filters { + /* from LSB: unicast:1, multicast:1 */ + uint8_t flags; +}; + +struct dpni_cmd_enable_vlan_filter { + /* only the LSB */ + uint8_t en; +}; + +struct dpni_cmd_vlan_id { + uint32_t pad; + uint16_t vlan_id; +}; + +#define DPNI_SEPARATE_GRP_SHIFT 0 +#define DPNI_SEPARATE_GRP_SIZE 1 +#define DPNI_MODE_1_SHIFT 0 +#define DPNI_MODE_1_SIZE 4 +#define DPNI_MODE_2_SHIFT 4 +#define DPNI_MODE_2_SIZE 4 + +struct dpni_cmd_set_tx_priorities { + uint16_t flags; + uint8_t prio_group_A; + uint8_t prio_group_B; + uint32_t pad0; + uint8_t modes[4]; + uint32_t pad1; + uint64_t pad2; + uint16_t delta_bandwidth[8]; +}; + +#define DPNI_DIST_MODE_SHIFT 0 +#define DPNI_DIST_MODE_SIZE 4 +#define DPNI_MISS_ACTION_SHIFT 4 +#define DPNI_MISS_ACTION_SIZE 4 +#define DPNI_KEEP_HASH_KEY_SHIFT 7 +#define DPNI_KEEP_HASH_KEY_SIZE 1 + +struct dpni_cmd_set_rx_tc_dist { + uint16_t dist_size; + uint8_t tc_id; + /* from LSB: dist_mode:4, miss_action:4 */ + uint8_t flags; + uint8_t pad0; + /* only the LSB */ + uint8_t keep_hash_key; + uint16_t default_flow_id; + uint64_t pad1[5]; + uint64_t key_cfg_iova; +}; + +struct dpni_cmd_get_queue { + uint8_t qtype; + uint8_t tc; + uint8_t index; +}; + +#define DPNI_DEST_TYPE_SHIFT 0 +#define DPNI_DEST_TYPE_SIZE 4 +#define DPNI_STASH_CTRL_SHIFT 6 +#define DPNI_STASH_CTRL_SIZE 1 +#define DPNI_HOLD_ACTIVE_SHIFT 7 +#define DPNI_HOLD_ACTIVE_SIZE 1 + +struct dpni_rsp_get_queue { + /* response word 0 */ + uint64_t pad0; + /* response word 1 */ + uint32_t dest_id; + uint16_t pad1; + uint8_t dest_prio; + /* From LSB: dest_type:4, pad:2, flc_stash_ctrl:1, hold_active:1 */ + uint8_t flags; + /* response word 2 */ + uint64_t flc; + /* response word 3 */ + uint64_t user_context; + /* response word 4 */ + uint32_t fqid; + uint16_t qdbin; +}; + +struct dpni_cmd_set_queue { + /* cmd word 0 */ + uint8_t qtype; + uint8_t tc; + uint8_t index; + uint8_t options; + uint32_t pad0; + /* cmd word 1 */ + uint32_t dest_id; + uint16_t pad1; + uint8_t dest_prio; + uint8_t flags; + /* cmd word 2 */ + uint64_t flc; + /* cmd word 3 */ + uint64_t user_context; +}; + +#define DPNI_DROP_ENABLE_SHIFT 0 +#define DPNI_DROP_ENABLE_SIZE 1 +#define DPNI_DROP_UNITS_SHIFT 2 +#define DPNI_DROP_UNITS_SIZE 2 + +struct dpni_early_drop { + /* from LSB: enable:1 units:2 */ + uint8_t flags; + uint8_t pad0[3]; + uint32_t pad1; + uint8_t green_drop_probability; + uint8_t pad2[7]; + uint64_t green_max_threshold; + uint64_t green_min_threshold; + uint64_t pad3; + uint8_t yellow_drop_probability; + uint8_t pad4[7]; + uint64_t yellow_max_threshold; + uint64_t yellow_min_threshold; + uint64_t pad5; + uint8_t red_drop_probability; + uint8_t pad6[7]; + uint64_t red_max_threshold; + uint64_t red_min_threshold; +}; + +struct dpni_cmd_early_drop { + uint8_t qtype; + uint8_t tc; + uint8_t pad[6]; + uint64_t early_drop_iova; +}; + +struct dpni_rsp_get_api_version { + uint16_t major; + uint16_t minor; +}; + +struct dpni_cmd_get_taildrop { + uint8_t congestion_point; + uint8_t qtype; + uint8_t tc; + uint8_t index; +}; + +struct dpni_rsp_get_taildrop { + /* cmd word 0 */ + uint64_t pad0; + /* cmd word 1 */ + /* from LSB: enable:1 oal_lo:7 */ + uint8_t enable_oal_lo; + /* from LSB: oal_hi:5 */ + uint8_t oal_hi; + uint8_t units; + uint8_t pad2; + uint32_t threshold; +}; + +#define DPNI_OAL_LO_SHIFT 1 +#define DPNI_OAL_LO_SIZE 7 +#define DPNI_OAL_HI_SHIFT 0 +#define DPNI_OAL_HI_SIZE 5 + +struct dpni_cmd_set_taildrop { + /* cmd word 0 */ + uint8_t congestion_point; + uint8_t qtype; + uint8_t tc; + uint8_t index; + uint32_t pad0; + /* cmd word 1 */ + /* from LSB: enable:1 oal_lo:7 */ + uint8_t enable_oal_lo; + /* from LSB: oal_hi:5 */ + uint8_t oal_hi; + uint8_t units; + uint8_t pad2; + uint32_t threshold; +}; + +struct dpni_tx_confirmation_mode { + uint32_t pad; + uint8_t confirmation_mode; +}; + +#define DPNI_DEST_TYPE_SHIFT 0 +#define DPNI_DEST_TYPE_SIZE 4 +#define DPNI_CONG_UNITS_SHIFT 4 +#define DPNI_CONG_UNITS_SIZE 2 + +struct dpni_cmd_set_congestion_notification { + uint8_t qtype; + uint8_t tc; + uint8_t pad[6]; + uint32_t dest_id; + uint16_t notification_mode; + uint8_t dest_priority; + /* from LSB: dest_type: 4 units:2 */ + uint8_t type_units; + uint64_t message_iova; + uint64_t message_ctx; + uint32_t threshold_entry; + uint32_t threshold_exit; +}; + +struct dpni_cmd_get_congestion_notification { + uint8_t qtype; + uint8_t tc; +}; + +struct dpni_rsp_get_congestion_notification { + uint64_t pad; + uint32_t dest_id; + uint16_t notification_mode; + uint8_t dest_priority; + /* from LSB: dest_type: 4 units:2 */ + uint8_t type_units; + uint64_t message_iova; + uint64_t message_ctx; + uint32_t threshold_entry; + uint32_t threshold_exit; +}; + +#pragma pack(pop) +#endif /* _FSL_DPNI_CMD_H */ diff --git a/dpdk/drivers/net/dpaa2/mc/fsl_net.h b/dpdk/drivers/net/dpaa2/mc/fsl_net.h new file mode 100644 index 00000000..dbec306a --- /dev/null +++ b/dpdk/drivers/net/dpaa2/mc/fsl_net.h @@ -0,0 +1,487 @@ +/*- + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * BSD LICENSE + * + * Copyright 2013-2015 Freescale Semiconductor Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * GPL LICENSE SUMMARY + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef __FSL_NET_H +#define __FSL_NET_H + +#define LAST_HDR_INDEX 0xFFFFFFFF + +/*****************************************************************************/ +/* Protocol fields */ +/*****************************************************************************/ + +/************************* Ethernet fields *********************************/ +#define NH_FLD_ETH_DA (1) +#define NH_FLD_ETH_SA (NH_FLD_ETH_DA << 1) +#define NH_FLD_ETH_LENGTH (NH_FLD_ETH_DA << 2) +#define NH_FLD_ETH_TYPE (NH_FLD_ETH_DA << 3) +#define NH_FLD_ETH_FINAL_CKSUM (NH_FLD_ETH_DA << 4) +#define NH_FLD_ETH_PADDING (NH_FLD_ETH_DA << 5) +#define NH_FLD_ETH_ALL_FIELDS ((NH_FLD_ETH_DA << 6) - 1) + +#define NH_FLD_ETH_ADDR_SIZE 6 + +/*************************** VLAN fields ***********************************/ +#define NH_FLD_VLAN_VPRI (1) +#define NH_FLD_VLAN_CFI (NH_FLD_VLAN_VPRI << 1) +#define NH_FLD_VLAN_VID (NH_FLD_VLAN_VPRI << 2) +#define NH_FLD_VLAN_LENGTH (NH_FLD_VLAN_VPRI << 3) +#define NH_FLD_VLAN_TYPE (NH_FLD_VLAN_VPRI << 4) +#define NH_FLD_VLAN_ALL_FIELDS ((NH_FLD_VLAN_VPRI << 5) - 1) + +#define NH_FLD_VLAN_TCI (NH_FLD_VLAN_VPRI | \ + NH_FLD_VLAN_CFI | \ + NH_FLD_VLAN_VID) + +/************************ IP (generic) fields ******************************/ +#define NH_FLD_IP_VER (1) +#define NH_FLD_IP_DSCP (NH_FLD_IP_VER << 2) +#define NH_FLD_IP_ECN (NH_FLD_IP_VER << 3) +#define NH_FLD_IP_PROTO (NH_FLD_IP_VER << 4) +#define NH_FLD_IP_SRC (NH_FLD_IP_VER << 5) +#define NH_FLD_IP_DST (NH_FLD_IP_VER << 6) +#define NH_FLD_IP_TOS_TC (NH_FLD_IP_VER << 7) +#define NH_FLD_IP_ID (NH_FLD_IP_VER << 8) +#define NH_FLD_IP_ALL_FIELDS ((NH_FLD_IP_VER << 9) - 1) + +#define NH_FLD_IP_PROTO_SIZE 1 + +/***************************** IPV4 fields *********************************/ +#define NH_FLD_IPV4_VER (1) +#define NH_FLD_IPV4_HDR_LEN (NH_FLD_IPV4_VER << 1) +#define NH_FLD_IPV4_TOS (NH_FLD_IPV4_VER << 2) +#define NH_FLD_IPV4_TOTAL_LEN (NH_FLD_IPV4_VER << 3) +#define NH_FLD_IPV4_ID (NH_FLD_IPV4_VER << 4) +#define NH_FLD_IPV4_FLAG_D (NH_FLD_IPV4_VER << 5) +#define NH_FLD_IPV4_FLAG_M (NH_FLD_IPV4_VER << 6) +#define NH_FLD_IPV4_OFFSET (NH_FLD_IPV4_VER << 7) +#define NH_FLD_IPV4_TTL (NH_FLD_IPV4_VER << 8) +#define NH_FLD_IPV4_PROTO (NH_FLD_IPV4_VER << 9) +#define NH_FLD_IPV4_CKSUM (NH_FLD_IPV4_VER << 10) +#define NH_FLD_IPV4_SRC_IP (NH_FLD_IPV4_VER << 11) +#define NH_FLD_IPV4_DST_IP (NH_FLD_IPV4_VER << 12) +#define NH_FLD_IPV4_OPTS (NH_FLD_IPV4_VER << 13) +#define NH_FLD_IPV4_OPTS_COUNT (NH_FLD_IPV4_VER << 14) +#define NH_FLD_IPV4_ALL_FIELDS ((NH_FLD_IPV4_VER << 15) - 1) + +#define NH_FLD_IPV4_ADDR_SIZE 4 +#define NH_FLD_IPV4_PROTO_SIZE 1 + +/***************************** IPV6 fields *********************************/ +#define NH_FLD_IPV6_VER (1) +#define NH_FLD_IPV6_TC (NH_FLD_IPV6_VER << 1) +#define NH_FLD_IPV6_SRC_IP (NH_FLD_IPV6_VER << 2) +#define NH_FLD_IPV6_DST_IP (NH_FLD_IPV6_VER << 3) +#define NH_FLD_IPV6_NEXT_HDR (NH_FLD_IPV6_VER << 4) +#define NH_FLD_IPV6_FL (NH_FLD_IPV6_VER << 5) +#define NH_FLD_IPV6_HOP_LIMIT (NH_FLD_IPV6_VER << 6) +#define NH_FLD_IPV6_ID (NH_FLD_IPV6_VER << 7) +#define NH_FLD_IPV6_ALL_FIELDS ((NH_FLD_IPV6_VER << 8) - 1) + +#define NH_FLD_IPV6_ADDR_SIZE 16 +#define NH_FLD_IPV6_NEXT_HDR_SIZE 1 + +/***************************** ICMP fields *********************************/ +#define NH_FLD_ICMP_TYPE (1) +#define NH_FLD_ICMP_CODE (NH_FLD_ICMP_TYPE << 1) +#define NH_FLD_ICMP_CKSUM (NH_FLD_ICMP_TYPE << 2) +#define NH_FLD_ICMP_ID (NH_FLD_ICMP_TYPE << 3) +#define NH_FLD_ICMP_SQ_NUM (NH_FLD_ICMP_TYPE << 4) +#define NH_FLD_ICMP_ALL_FIELDS ((NH_FLD_ICMP_TYPE << 5) - 1) + +#define NH_FLD_ICMP_CODE_SIZE 1 +#define NH_FLD_ICMP_TYPE_SIZE 1 + +/***************************** IGMP fields *********************************/ +#define NH_FLD_IGMP_VERSION (1) +#define NH_FLD_IGMP_TYPE (NH_FLD_IGMP_VERSION << 1) +#define NH_FLD_IGMP_CKSUM (NH_FLD_IGMP_VERSION << 2) +#define NH_FLD_IGMP_DATA (NH_FLD_IGMP_VERSION << 3) +#define NH_FLD_IGMP_ALL_FIELDS ((NH_FLD_IGMP_VERSION << 4) - 1) + +/***************************** TCP fields **********************************/ +#define NH_FLD_TCP_PORT_SRC (1) +#define NH_FLD_TCP_PORT_DST (NH_FLD_TCP_PORT_SRC << 1) +#define NH_FLD_TCP_SEQ (NH_FLD_TCP_PORT_SRC << 2) +#define NH_FLD_TCP_ACK (NH_FLD_TCP_PORT_SRC << 3) +#define NH_FLD_TCP_OFFSET (NH_FLD_TCP_PORT_SRC << 4) +#define NH_FLD_TCP_FLAGS (NH_FLD_TCP_PORT_SRC << 5) +#define NH_FLD_TCP_WINDOW (NH_FLD_TCP_PORT_SRC << 6) +#define NH_FLD_TCP_CKSUM (NH_FLD_TCP_PORT_SRC << 7) +#define NH_FLD_TCP_URGPTR (NH_FLD_TCP_PORT_SRC << 8) +#define NH_FLD_TCP_OPTS (NH_FLD_TCP_PORT_SRC << 9) +#define NH_FLD_TCP_OPTS_COUNT (NH_FLD_TCP_PORT_SRC << 10) +#define NH_FLD_TCP_ALL_FIELDS ((NH_FLD_TCP_PORT_SRC << 11) - 1) + +#define NH_FLD_TCP_PORT_SIZE 2 + +/***************************** UDP fields **********************************/ +#define NH_FLD_UDP_PORT_SRC (1) +#define NH_FLD_UDP_PORT_DST (NH_FLD_UDP_PORT_SRC << 1) +#define NH_FLD_UDP_LEN (NH_FLD_UDP_PORT_SRC << 2) +#define NH_FLD_UDP_CKSUM (NH_FLD_UDP_PORT_SRC << 3) +#define NH_FLD_UDP_ALL_FIELDS ((NH_FLD_UDP_PORT_SRC << 4) - 1) + +#define NH_FLD_UDP_PORT_SIZE 2 + +/*************************** UDP-lite fields *******************************/ +#define NH_FLD_UDP_LITE_PORT_SRC (1) +#define NH_FLD_UDP_LITE_PORT_DST (NH_FLD_UDP_LITE_PORT_SRC << 1) +#define NH_FLD_UDP_LITE_ALL_FIELDS \ + ((NH_FLD_UDP_LITE_PORT_SRC << 2) - 1) + +#define NH_FLD_UDP_LITE_PORT_SIZE 2 + +/*************************** UDP-encap-ESP fields **************************/ +#define NH_FLD_UDP_ENC_ESP_PORT_SRC (1) +#define NH_FLD_UDP_ENC_ESP_PORT_DST (NH_FLD_UDP_ENC_ESP_PORT_SRC << 1) +#define NH_FLD_UDP_ENC_ESP_LEN (NH_FLD_UDP_ENC_ESP_PORT_SRC << 2) +#define NH_FLD_UDP_ENC_ESP_CKSUM (NH_FLD_UDP_ENC_ESP_PORT_SRC << 3) +#define NH_FLD_UDP_ENC_ESP_SPI (NH_FLD_UDP_ENC_ESP_PORT_SRC << 4) +#define NH_FLD_UDP_ENC_ESP_SEQUENCE_NUM (NH_FLD_UDP_ENC_ESP_PORT_SRC << 5) +#define NH_FLD_UDP_ENC_ESP_ALL_FIELDS \ + ((NH_FLD_UDP_ENC_ESP_PORT_SRC << 6) - 1) + +#define NH_FLD_UDP_ENC_ESP_PORT_SIZE 2 +#define NH_FLD_UDP_ENC_ESP_SPI_SIZE 4 + +/***************************** SCTP fields *********************************/ +#define NH_FLD_SCTP_PORT_SRC (1) +#define NH_FLD_SCTP_PORT_DST (NH_FLD_SCTP_PORT_SRC << 1) +#define NH_FLD_SCTP_VER_TAG (NH_FLD_SCTP_PORT_SRC << 2) +#define NH_FLD_SCTP_CKSUM (NH_FLD_SCTP_PORT_SRC << 3) +#define NH_FLD_SCTP_ALL_FIELDS ((NH_FLD_SCTP_PORT_SRC << 4) - 1) + +#define NH_FLD_SCTP_PORT_SIZE 2 + +/***************************** DCCP fields *********************************/ +#define NH_FLD_DCCP_PORT_SRC (1) +#define NH_FLD_DCCP_PORT_DST (NH_FLD_DCCP_PORT_SRC << 1) +#define NH_FLD_DCCP_ALL_FIELDS ((NH_FLD_DCCP_PORT_SRC << 2) - 1) + +#define NH_FLD_DCCP_PORT_SIZE 2 + +/***************************** IPHC fields *********************************/ +#define NH_FLD_IPHC_CID (1) +#define NH_FLD_IPHC_CID_TYPE (NH_FLD_IPHC_CID << 1) +#define NH_FLD_IPHC_HCINDEX (NH_FLD_IPHC_CID << 2) +#define NH_FLD_IPHC_GEN (NH_FLD_IPHC_CID << 3) +#define NH_FLD_IPHC_D_BIT (NH_FLD_IPHC_CID << 4) +#define NH_FLD_IPHC_ALL_FIELDS ((NH_FLD_IPHC_CID << 5) - 1) + +/***************************** SCTP fields *********************************/ +#define NH_FLD_SCTP_CHUNK_DATA_TYPE (1) +#define NH_FLD_SCTP_CHUNK_DATA_FLAGS (NH_FLD_SCTP_CHUNK_DATA_TYPE << 1) +#define NH_FLD_SCTP_CHUNK_DATA_LENGTH (NH_FLD_SCTP_CHUNK_DATA_TYPE << 2) +#define NH_FLD_SCTP_CHUNK_DATA_TSN (NH_FLD_SCTP_CHUNK_DATA_TYPE << 3) +#define NH_FLD_SCTP_CHUNK_DATA_STREAM_ID (NH_FLD_SCTP_CHUNK_DATA_TYPE << 4) +#define NH_FLD_SCTP_CHUNK_DATA_STREAM_SQN (NH_FLD_SCTP_CHUNK_DATA_TYPE << 5) +#define NH_FLD_SCTP_CHUNK_DATA_PAYLOAD_PID (NH_FLD_SCTP_CHUNK_DATA_TYPE << 6) +#define NH_FLD_SCTP_CHUNK_DATA_UNORDERED (NH_FLD_SCTP_CHUNK_DATA_TYPE << 7) +#define NH_FLD_SCTP_CHUNK_DATA_BEGGINNING (NH_FLD_SCTP_CHUNK_DATA_TYPE << 8) +#define NH_FLD_SCTP_CHUNK_DATA_END (NH_FLD_SCTP_CHUNK_DATA_TYPE << 9) +#define NH_FLD_SCTP_CHUNK_DATA_ALL_FIELDS \ + ((NH_FLD_SCTP_CHUNK_DATA_TYPE << 10) - 1) + +/*************************** L2TPV2 fields *********************************/ +#define NH_FLD_L2TPV2_TYPE_BIT (1) +#define NH_FLD_L2TPV2_LENGTH_BIT (NH_FLD_L2TPV2_TYPE_BIT << 1) +#define NH_FLD_L2TPV2_SEQUENCE_BIT (NH_FLD_L2TPV2_TYPE_BIT << 2) +#define NH_FLD_L2TPV2_OFFSET_BIT (NH_FLD_L2TPV2_TYPE_BIT << 3) +#define NH_FLD_L2TPV2_PRIORITY_BIT (NH_FLD_L2TPV2_TYPE_BIT << 4) +#define NH_FLD_L2TPV2_VERSION (NH_FLD_L2TPV2_TYPE_BIT << 5) +#define NH_FLD_L2TPV2_LEN (NH_FLD_L2TPV2_TYPE_BIT << 6) +#define NH_FLD_L2TPV2_TUNNEL_ID (NH_FLD_L2TPV2_TYPE_BIT << 7) +#define NH_FLD_L2TPV2_SESSION_ID (NH_FLD_L2TPV2_TYPE_BIT << 8) +#define NH_FLD_L2TPV2_NS (NH_FLD_L2TPV2_TYPE_BIT << 9) +#define NH_FLD_L2TPV2_NR (NH_FLD_L2TPV2_TYPE_BIT << 10) +#define NH_FLD_L2TPV2_OFFSET_SIZE (NH_FLD_L2TPV2_TYPE_BIT << 11) +#define NH_FLD_L2TPV2_FIRST_BYTE (NH_FLD_L2TPV2_TYPE_BIT << 12) +#define NH_FLD_L2TPV2_ALL_FIELDS \ + ((NH_FLD_L2TPV2_TYPE_BIT << 13) - 1) + +/*************************** L2TPV3 fields *********************************/ +#define NH_FLD_L2TPV3_CTRL_TYPE_BIT (1) +#define NH_FLD_L2TPV3_CTRL_LENGTH_BIT (NH_FLD_L2TPV3_CTRL_TYPE_BIT << 1) +#define NH_FLD_L2TPV3_CTRL_SEQUENCE_BIT (NH_FLD_L2TPV3_CTRL_TYPE_BIT << 2) +#define NH_FLD_L2TPV3_CTRL_VERSION (NH_FLD_L2TPV3_CTRL_TYPE_BIT << 3) +#define NH_FLD_L2TPV3_CTRL_LENGTH (NH_FLD_L2TPV3_CTRL_TYPE_BIT << 4) +#define NH_FLD_L2TPV3_CTRL_CONTROL (NH_FLD_L2TPV3_CTRL_TYPE_BIT << 5) +#define NH_FLD_L2TPV3_CTRL_SENT (NH_FLD_L2TPV3_CTRL_TYPE_BIT << 6) +#define NH_FLD_L2TPV3_CTRL_RECV (NH_FLD_L2TPV3_CTRL_TYPE_BIT << 7) +#define NH_FLD_L2TPV3_CTRL_FIRST_BYTE (NH_FLD_L2TPV3_CTRL_TYPE_BIT << 8) +#define NH_FLD_L2TPV3_CTRL_ALL_FIELDS \ + ((NH_FLD_L2TPV3_CTRL_TYPE_BIT << 9) - 1) + +#define NH_FLD_L2TPV3_SESS_TYPE_BIT (1) +#define NH_FLD_L2TPV3_SESS_VERSION (NH_FLD_L2TPV3_SESS_TYPE_BIT << 1) +#define NH_FLD_L2TPV3_SESS_ID (NH_FLD_L2TPV3_SESS_TYPE_BIT << 2) +#define NH_FLD_L2TPV3_SESS_COOKIE (NH_FLD_L2TPV3_SESS_TYPE_BIT << 3) +#define NH_FLD_L2TPV3_SESS_ALL_FIELDS \ + ((NH_FLD_L2TPV3_SESS_TYPE_BIT << 4) - 1) + +/**************************** PPP fields ***********************************/ +#define NH_FLD_PPP_PID (1) +#define NH_FLD_PPP_COMPRESSED (NH_FLD_PPP_PID << 1) +#define NH_FLD_PPP_ALL_FIELDS ((NH_FLD_PPP_PID << 2) - 1) + +/************************** PPPoE fields ***********************************/ +#define NH_FLD_PPPOE_VER (1) +#define NH_FLD_PPPOE_TYPE (NH_FLD_PPPOE_VER << 1) +#define NH_FLD_PPPOE_CODE (NH_FLD_PPPOE_VER << 2) +#define NH_FLD_PPPOE_SID (NH_FLD_PPPOE_VER << 3) +#define NH_FLD_PPPOE_LEN (NH_FLD_PPPOE_VER << 4) +#define NH_FLD_PPPOE_SESSION (NH_FLD_PPPOE_VER << 5) +#define NH_FLD_PPPOE_PID (NH_FLD_PPPOE_VER << 6) +#define NH_FLD_PPPOE_ALL_FIELDS ((NH_FLD_PPPOE_VER << 7) - 1) + +/************************* PPP-Mux fields **********************************/ +#define NH_FLD_PPPMUX_PID (1) +#define NH_FLD_PPPMUX_CKSUM (NH_FLD_PPPMUX_PID << 1) +#define NH_FLD_PPPMUX_COMPRESSED (NH_FLD_PPPMUX_PID << 2) +#define NH_FLD_PPPMUX_ALL_FIELDS ((NH_FLD_PPPMUX_PID << 3) - 1) + +/*********************** PPP-Mux sub-frame fields **************************/ +#define NH_FLD_PPPMUX_SUBFRM_PFF (1) +#define NH_FLD_PPPMUX_SUBFRM_LXT (NH_FLD_PPPMUX_SUBFRM_PFF << 1) +#define NH_FLD_PPPMUX_SUBFRM_LEN (NH_FLD_PPPMUX_SUBFRM_PFF << 2) +#define NH_FLD_PPPMUX_SUBFRM_PID (NH_FLD_PPPMUX_SUBFRM_PFF << 3) +#define NH_FLD_PPPMUX_SUBFRM_USE_PID (NH_FLD_PPPMUX_SUBFRM_PFF << 4) +#define NH_FLD_PPPMUX_SUBFRM_ALL_FIELDS \ + ((NH_FLD_PPPMUX_SUBFRM_PFF << 5) - 1) + +/*************************** LLC fields ************************************/ +#define NH_FLD_LLC_DSAP (1) +#define NH_FLD_LLC_SSAP (NH_FLD_LLC_DSAP << 1) +#define NH_FLD_LLC_CTRL (NH_FLD_LLC_DSAP << 2) +#define NH_FLD_LLC_ALL_FIELDS ((NH_FLD_LLC_DSAP << 3) - 1) + +/*************************** NLPID fields **********************************/ +#define NH_FLD_NLPID_NLPID (1) +#define NH_FLD_NLPID_ALL_FIELDS ((NH_FLD_NLPID_NLPID << 1) - 1) + +/*************************** SNAP fields ***********************************/ +#define NH_FLD_SNAP_OUI (1) +#define NH_FLD_SNAP_PID (NH_FLD_SNAP_OUI << 1) +#define NH_FLD_SNAP_ALL_FIELDS ((NH_FLD_SNAP_OUI << 2) - 1) + +/*************************** LLC SNAP fields *******************************/ +#define NH_FLD_LLC_SNAP_TYPE (1) +#define NH_FLD_LLC_SNAP_ALL_FIELDS ((NH_FLD_LLC_SNAP_TYPE << 1) - 1) + +#define NH_FLD_ARP_HTYPE (1) +#define NH_FLD_ARP_PTYPE (NH_FLD_ARP_HTYPE << 1) +#define NH_FLD_ARP_HLEN (NH_FLD_ARP_HTYPE << 2) +#define NH_FLD_ARP_PLEN (NH_FLD_ARP_HTYPE << 3) +#define NH_FLD_ARP_OPER (NH_FLD_ARP_HTYPE << 4) +#define NH_FLD_ARP_SHA (NH_FLD_ARP_HTYPE << 5) +#define NH_FLD_ARP_SPA (NH_FLD_ARP_HTYPE << 6) +#define NH_FLD_ARP_THA (NH_FLD_ARP_HTYPE << 7) +#define NH_FLD_ARP_TPA (NH_FLD_ARP_HTYPE << 8) +#define NH_FLD_ARP_ALL_FIELDS ((NH_FLD_ARP_HTYPE << 9) - 1) + +/*************************** RFC2684 fields ********************************/ +#define NH_FLD_RFC2684_LLC (1) +#define NH_FLD_RFC2684_NLPID (NH_FLD_RFC2684_LLC << 1) +#define NH_FLD_RFC2684_OUI (NH_FLD_RFC2684_LLC << 2) +#define NH_FLD_RFC2684_PID (NH_FLD_RFC2684_LLC << 3) +#define NH_FLD_RFC2684_VPN_OUI (NH_FLD_RFC2684_LLC << 4) +#define NH_FLD_RFC2684_VPN_IDX (NH_FLD_RFC2684_LLC << 5) +#define NH_FLD_RFC2684_ALL_FIELDS ((NH_FLD_RFC2684_LLC << 6) - 1) + +/*************************** User defined fields ***************************/ +#define NH_FLD_USER_DEFINED_SRCPORT (1) +#define NH_FLD_USER_DEFINED_PCDID (NH_FLD_USER_DEFINED_SRCPORT << 1) +#define NH_FLD_USER_DEFINED_ALL_FIELDS \ + ((NH_FLD_USER_DEFINED_SRCPORT << 2) - 1) + +/*************************** Payload fields ********************************/ +#define NH_FLD_PAYLOAD_BUFFER (1) +#define NH_FLD_PAYLOAD_SIZE (NH_FLD_PAYLOAD_BUFFER << 1) +#define NH_FLD_MAX_FRM_SIZE (NH_FLD_PAYLOAD_BUFFER << 2) +#define NH_FLD_MIN_FRM_SIZE (NH_FLD_PAYLOAD_BUFFER << 3) +#define NH_FLD_PAYLOAD_TYPE (NH_FLD_PAYLOAD_BUFFER << 4) +#define NH_FLD_FRAME_SIZE (NH_FLD_PAYLOAD_BUFFER << 5) +#define NH_FLD_PAYLOAD_ALL_FIELDS ((NH_FLD_PAYLOAD_BUFFER << 6) - 1) + +/*************************** GRE fields ************************************/ +#define NH_FLD_GRE_TYPE (1) +#define NH_FLD_GRE_ALL_FIELDS ((NH_FLD_GRE_TYPE << 1) - 1) + +/*************************** MINENCAP fields *******************************/ +#define NH_FLD_MINENCAP_SRC_IP (1) +#define NH_FLD_MINENCAP_DST_IP (NH_FLD_MINENCAP_SRC_IP << 1) +#define NH_FLD_MINENCAP_TYPE (NH_FLD_MINENCAP_SRC_IP << 2) +#define NH_FLD_MINENCAP_ALL_FIELDS \ + ((NH_FLD_MINENCAP_SRC_IP << 3) - 1) + +/*************************** IPSEC AH fields *******************************/ +#define NH_FLD_IPSEC_AH_SPI (1) +#define NH_FLD_IPSEC_AH_NH (NH_FLD_IPSEC_AH_SPI << 1) +#define NH_FLD_IPSEC_AH_ALL_FIELDS ((NH_FLD_IPSEC_AH_SPI << 2) - 1) + +/*************************** IPSEC ESP fields ******************************/ +#define NH_FLD_IPSEC_ESP_SPI (1) +#define NH_FLD_IPSEC_ESP_SEQUENCE_NUM (NH_FLD_IPSEC_ESP_SPI << 1) +#define NH_FLD_IPSEC_ESP_ALL_FIELDS ((NH_FLD_IPSEC_ESP_SPI << 2) - 1) + +#define NH_FLD_IPSEC_ESP_SPI_SIZE 4 + +/*************************** MPLS fields ***********************************/ +#define NH_FLD_MPLS_LABEL_STACK (1) +#define NH_FLD_MPLS_LABEL_STACK_ALL_FIELDS \ + ((NH_FLD_MPLS_LABEL_STACK << 1) - 1) + +/*************************** MACSEC fields *********************************/ +#define NH_FLD_MACSEC_SECTAG (1) +#define NH_FLD_MACSEC_ALL_FIELDS ((NH_FLD_MACSEC_SECTAG << 1) - 1) + +/*************************** GTP fields ************************************/ +#define NH_FLD_GTP_TEID (1) + +/* Protocol options */ + +/* Ethernet options */ +#define NH_OPT_ETH_BROADCAST 1 +#define NH_OPT_ETH_MULTICAST 2 +#define NH_OPT_ETH_UNICAST 3 +#define NH_OPT_ETH_BPDU 4 + +#define NH_ETH_IS_MULTICAST_ADDR(addr) (addr[0] & 0x01) +/* also applicable for broadcast */ + +/* VLAN options */ +#define NH_OPT_VLAN_CFI 1 + +/* IPV4 options */ +#define NH_OPT_IPV4_UNICAST 1 +#define NH_OPT_IPV4_MULTICAST 2 +#define NH_OPT_IPV4_BROADCAST 3 +#define NH_OPT_IPV4_OPTION 4 +#define NH_OPT_IPV4_FRAG 5 +#define NH_OPT_IPV4_INITIAL_FRAG 6 + +/* IPV6 options */ +#define NH_OPT_IPV6_UNICAST 1 +#define NH_OPT_IPV6_MULTICAST 2 +#define NH_OPT_IPV6_OPTION 3 +#define NH_OPT_IPV6_FRAG 4 +#define NH_OPT_IPV6_INITIAL_FRAG 5 + +/* General IP options (may be used for any version) */ +#define NH_OPT_IP_FRAG 1 +#define NH_OPT_IP_INITIAL_FRAG 2 +#define NH_OPT_IP_OPTION 3 + +/* Minenc. options */ +#define NH_OPT_MINENCAP_SRC_ADDR_PRESENT 1 + +/* GRE. options */ +#define NH_OPT_GRE_ROUTING_PRESENT 1 + +/* TCP options */ +#define NH_OPT_TCP_OPTIONS 1 +#define NH_OPT_TCP_CONTROL_HIGH_BITS 2 +#define NH_OPT_TCP_CONTROL_LOW_BITS 3 + +/* CAPWAP options */ +#define NH_OPT_CAPWAP_DTLS 1 + +enum net_prot { + NET_PROT_NONE = 0, + NET_PROT_PAYLOAD, + NET_PROT_ETH, + NET_PROT_VLAN, + NET_PROT_IPV4, + NET_PROT_IPV6, + NET_PROT_IP, + NET_PROT_TCP, + NET_PROT_UDP, + NET_PROT_UDP_LITE, + NET_PROT_IPHC, + NET_PROT_SCTP, + NET_PROT_SCTP_CHUNK_DATA, + NET_PROT_PPPOE, + NET_PROT_PPP, + NET_PROT_PPPMUX, + NET_PROT_PPPMUX_SUBFRM, + NET_PROT_L2TPV2, + NET_PROT_L2TPV3_CTRL, + NET_PROT_L2TPV3_SESS, + NET_PROT_LLC, + NET_PROT_LLC_SNAP, + NET_PROT_NLPID, + NET_PROT_SNAP, + NET_PROT_MPLS, + NET_PROT_IPSEC_AH, + NET_PROT_IPSEC_ESP, + NET_PROT_UDP_ENC_ESP, /* RFC 3948 */ + NET_PROT_MACSEC, + NET_PROT_GRE, + NET_PROT_MINENCAP, + NET_PROT_DCCP, + NET_PROT_ICMP, + NET_PROT_IGMP, + NET_PROT_ARP, + NET_PROT_CAPWAP_DATA, + NET_PROT_CAPWAP_CTRL, + NET_PROT_RFC2684, + NET_PROT_ICMPV6, + NET_PROT_FCOE, + NET_PROT_FIP, + NET_PROT_ISCSI, + NET_PROT_GTP, + NET_PROT_USER_DEFINED_L2, + NET_PROT_USER_DEFINED_L3, + NET_PROT_USER_DEFINED_L4, + NET_PROT_USER_DEFINED_L5, + NET_PROT_USER_DEFINED_SHIM1, + NET_PROT_USER_DEFINED_SHIM2, + + NET_PROT_DUMMY_LAST +}; + +/*! IEEE8021.Q */ +#define NH_IEEE8021Q_ETYPE 0x8100 +#define NH_IEEE8021Q_HDR(etype, pcp, dei, vlan_id) \ + ((((uint32_t)(etype & 0xFFFF)) << 16) | \ + (((uint32_t)(pcp & 0x07)) << 13) | \ + (((uint32_t)(dei & 0x01)) << 12) | \ + (((uint32_t)(vlan_id & 0xFFF)))) + +#endif /* __FSL_NET_H */ diff --git a/dpdk/drivers/net/dpaa2/rte_pmd_dpaa2_version.map b/dpdk/drivers/net/dpaa2/rte_pmd_dpaa2_version.map new file mode 100644 index 00000000..09f4364b --- /dev/null +++ b/dpdk/drivers/net/dpaa2/rte_pmd_dpaa2_version.map @@ -0,0 +1,12 @@ +DPDK_17.05 { + + local: *; +}; + +DPDK_17.11 { + global: + + dpaa2_eth_eventq_attach; + dpaa2_eth_eventq_detach; + +} DPDK_17.05; diff --git a/dpdk/drivers/net/e1000/Makefile b/dpdk/drivers/net/e1000/Makefile index 57a60f0f..3f0344b4 100644 --- a/dpdk/drivers/net/e1000/Makefile +++ b/dpdk/drivers/net/e1000/Makefile @@ -38,6 +38,9 @@ LIB = librte_pmd_e1000.a CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs +LDLIBS += -lrte_bus_pci EXPORT_MAP := rte_pmd_e1000_version.map @@ -57,6 +60,9 @@ CFLAGS_BASE_DRIVER += -Wno-unused-variable ifeq ($(CONFIG_RTE_TOOLCHAIN_GCC),y) ifeq ($(shell test $(GCC_VERSION) -ge 60 && echo 1), 1) CFLAGS_BASE_DRIVER += -Wno-misleading-indentation +ifeq ($(shell test $(GCC_VERSION) -ge 70 && echo 1), 1) +CFLAGS_BASE_DRIVER += -Wno-implicit-fallthrough +endif endif endif endif @@ -65,7 +71,7 @@ endif # Add extra flags for base driver files (also known as shared code) # to disable warnings in them # -BASE_DRIVER_OBJS=$(patsubst %.c,%.o,$(notdir $(wildcard $(SRCDIR)/base/*.c))) +BASE_DRIVER_OBJS=$(sort $(patsubst %.c,%.o,$(notdir $(wildcard $(SRCDIR)/base/*.c)))) $(foreach obj, $(BASE_DRIVER_OBJS), $(eval CFLAGS_$(obj)+=$(CFLAGS_BASE_DRIVER))) VPATH += $(SRCDIR)/base @@ -93,12 +99,8 @@ SRCS-$(CONFIG_RTE_LIBRTE_E1000_PMD) += e1000_vf.c SRCS-$(CONFIG_RTE_LIBRTE_IGB_PMD) += igb_ethdev.c SRCS-$(CONFIG_RTE_LIBRTE_IGB_PMD) += igb_rxtx.c SRCS-$(CONFIG_RTE_LIBRTE_IGB_PMD) += igb_pf.c +SRCS-$(CONFIG_RTE_LIBRTE_IGB_PMD) += igb_flow.c SRCS-$(CONFIG_RTE_LIBRTE_EM_PMD) += em_ethdev.c SRCS-$(CONFIG_RTE_LIBRTE_EM_PMD) += em_rxtx.c -# this lib depends upon: -DEPDIRS-$(CONFIG_RTE_LIBRTE_E1000_PMD) += lib/librte_eal lib/librte_ether -DEPDIRS-$(CONFIG_RTE_LIBRTE_E1000_PMD) += lib/librte_mempool lib/librte_mbuf -DEPDIRS-$(CONFIG_RTE_LIBRTE_E1000_PMD) += lib/librte_net - include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/net/e1000/base/README b/dpdk/drivers/net/e1000/base/README index 8d48135a..de1ae4cf 100644 --- a/dpdk/drivers/net/e1000/base/README +++ b/dpdk/drivers/net/e1000/base/README @@ -1,7 +1,7 @@ .. BSD LICENSE - Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + Copyright(c) 2010-2016 Intel Corporation. All rights reserved. All rights reserved. Redistribution and use in source and binary forms, with or without @@ -31,9 +31,30 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. This directory contains source code of FreeBSD em & igb drivers of version -cid-shared-code.2015.10.09 released by ND. The sub-directory of base/ +cid-shared-code.2016.11.22 released by ND. The sub-directory of base/ contains the original source package. +This driver is valid for the product(s) listed below +* Intel® Ethernet Controller 82540 +* Intel® Ethernet Controller 82545 Series +* Intel® Ethernet Controller 82546 Series +* Intel® Ethernet Controller 82571 Series +* Intel® Ethernet Controller 82572 Series +* Intel® Ethernet Controller 82573 +* Intel® Ethernet Controller 82574 +* Intel® Ethernet Controller 82583 +* Intel® Ethernet Controller I217 Series +* Intel® Ethernet Controller I218 Series +* Intel® Ethernet Controller I219 Series +* Intel® Ethernet Controller 82576 Series +* Intel® Ethernet Controller 82575 Series +* Intel® Ethernet Controller 82580 Series +* Intel® Ethernet Controller I350 Series +* Intel® Ethernet Controller I210 Series +* Intel® Ethernet Controller I211 +* Intel® Ethernet Controller I354 Series +* Intel® Ethernet Controller DH89XXCC Series + Updating the driver =================== diff --git a/dpdk/drivers/net/e1000/base/e1000_82575.c b/dpdk/drivers/net/e1000/base/e1000_82575.c index 723885d7..c6400bde 100644 --- a/dpdk/drivers/net/e1000/base/e1000_82575.c +++ b/dpdk/drivers/net/e1000/base/e1000_82575.c @@ -100,7 +100,6 @@ STATIC s32 e1000_validate_nvm_checksum_with_offset(struct e1000_hw *hw, u16 offset); STATIC s32 e1000_validate_nvm_checksum_i350(struct e1000_hw *hw); STATIC s32 e1000_update_nvm_checksum_i350(struct e1000_hw *hw); -STATIC void e1000_write_vfta_i350(struct e1000_hw *hw, u32 offset, u32 value); STATIC void e1000_clear_vfta_i350(struct e1000_hw *hw); STATIC void e1000_i2c_start(struct e1000_hw *hw); diff --git a/dpdk/drivers/net/e1000/base/e1000_82575.h b/dpdk/drivers/net/e1000/base/e1000_82575.h index c4986841..4133cdd8 100644 --- a/dpdk/drivers/net/e1000/base/e1000_82575.h +++ b/dpdk/drivers/net/e1000/base/e1000_82575.h @@ -492,6 +492,7 @@ enum e1000_promisc_type { void e1000_vfta_set_vf(struct e1000_hw *, u16, bool); void e1000_rlpml_set_vf(struct e1000_hw *, u16); s32 e1000_promisc_set_vf(struct e1000_hw *, enum e1000_promisc_type type); +void e1000_write_vfta_i350(struct e1000_hw *hw, u32 offset, u32 value); u16 e1000_rxpbs_adjust_82580(u32 data); s32 e1000_read_emi_reg(struct e1000_hw *hw, u16 addr, u16 *data); s32 e1000_set_eee_i350(struct e1000_hw *hw, bool adv1G, bool adv100M); diff --git a/dpdk/drivers/net/e1000/base/e1000_api.c b/dpdk/drivers/net/e1000/base/e1000_api.c index bbfcae88..f7cf83b6 100644 --- a/dpdk/drivers/net/e1000/base/e1000_api.c +++ b/dpdk/drivers/net/e1000/base/e1000_api.c @@ -298,6 +298,23 @@ s32 e1000_set_mac_type(struct e1000_hw *hw) case E1000_DEV_ID_PCH_I218_V3: mac->type = e1000_pch_lpt; break; + case E1000_DEV_ID_PCH_SPT_I219_LM: + case E1000_DEV_ID_PCH_SPT_I219_V: + case E1000_DEV_ID_PCH_SPT_I219_LM2: + case E1000_DEV_ID_PCH_SPT_I219_V2: + case E1000_DEV_ID_PCH_LBG_I219_LM3: + case E1000_DEV_ID_PCH_SPT_I219_LM4: + case E1000_DEV_ID_PCH_SPT_I219_V4: + case E1000_DEV_ID_PCH_SPT_I219_LM5: + case E1000_DEV_ID_PCH_SPT_I219_V5: + mac->type = e1000_pch_spt; + break; + case E1000_DEV_ID_PCH_CNP_I219_LM6: + case E1000_DEV_ID_PCH_CNP_I219_V6: + case E1000_DEV_ID_PCH_CNP_I219_LM7: + case E1000_DEV_ID_PCH_CNP_I219_V7: + mac->type = e1000_pch_cnp; + break; case E1000_DEV_ID_82575EB_COPPER: case E1000_DEV_ID_82575EB_FIBER_SERDES: case E1000_DEV_ID_82575GB_QUAD_COPPER: @@ -448,6 +465,8 @@ s32 e1000_setup_init_funcs(struct e1000_hw *hw, bool init_device) case e1000_pchlan: case e1000_pch2lan: case e1000_pch_lpt: + case e1000_pch_spt: + case e1000_pch_cnp: e1000_init_function_pointers_ich8lan(hw); break; case e1000_82575: diff --git a/dpdk/drivers/net/e1000/base/e1000_defines.h b/dpdk/drivers/net/e1000/base/e1000_defines.h index 69aa1f23..dbc2bbbe 100644 --- a/dpdk/drivers/net/e1000/base/e1000_defines.h +++ b/dpdk/drivers/net/e1000/base/e1000_defines.h @@ -198,6 +198,7 @@ POSSIBILITY OF SUCH DAMAGE. #define E1000_RCTL_DTYP_PS 0x00000400 /* Packet Split descriptor */ #define E1000_RCTL_RDMTS_HALF 0x00000000 /* Rx desc min thresh size */ #define E1000_RCTL_RDMTS_HEX 0x00010000 +#define E1000_RCTL_RDMTS1_HEX E1000_RCTL_RDMTS_HEX #define E1000_RCTL_MO_SHIFT 12 /* multicast offset shift */ #define E1000_RCTL_MO_3 0x00003000 /* multicast offset 15:4 */ #define E1000_RCTL_BAM 0x00008000 /* broadcast enable */ @@ -468,6 +469,8 @@ POSSIBILITY OF SUCH DAMAGE. #define ETHERNET_FCS_SIZE 4 #define MAX_JUMBO_FRAME_SIZE 0x3F00 +/* The datasheet maximum supported RX size is 9.5KB (9728 bytes) */ +#define MAX_RX_JUMBO_FRAME_SIZE 0x2600 #define E1000_TX_PTR_GAP 0x1F /* Extended Configuration Control and Size */ @@ -751,6 +754,12 @@ POSSIBILITY OF SUCH DAMAGE. #define E1000_TSYNCTXCTL_VALID 0x00000001 /* Tx timestamp valid */ #define E1000_TSYNCTXCTL_ENABLED 0x00000010 /* enable Tx timestamping */ +/* HH Time Sync */ +#define E1000_TSYNCTXCTL_MAX_ALLOWED_DLY_MASK 0x0000F000 /* max delay */ +#define E1000_TSYNCTXCTL_SYNC_COMP_ERR 0x20000000 /* sync err */ +#define E1000_TSYNCTXCTL_SYNC_COMP 0x40000000 /* sync complete */ +#define E1000_TSYNCTXCTL_START_SYNC 0x80000000 /* initiate sync */ + #define E1000_TSYNCRXCTL_VALID 0x00000001 /* Rx timestamp valid */ #define E1000_TSYNCRXCTL_TYPE_MASK 0x0000000E /* Rx type mask */ #define E1000_TSYNCRXCTL_TYPE_L2_V2 0x00 diff --git a/dpdk/drivers/net/e1000/base/e1000_hw.h b/dpdk/drivers/net/e1000/base/e1000_hw.h index e4e4f764..d9de9fc1 100644 --- a/dpdk/drivers/net/e1000/base/e1000_hw.h +++ b/dpdk/drivers/net/e1000/base/e1000_hw.h @@ -136,6 +136,19 @@ struct e1000_hw; #define E1000_DEV_ID_PCH_I218_V2 0x15A1 #define E1000_DEV_ID_PCH_I218_LM3 0x15A2 /* Wildcat Point PCH */ #define E1000_DEV_ID_PCH_I218_V3 0x15A3 /* Wildcat Point PCH */ +#define E1000_DEV_ID_PCH_SPT_I219_LM 0x156F /* Sunrise Point PCH */ +#define E1000_DEV_ID_PCH_SPT_I219_V 0x1570 /* Sunrise Point PCH */ +#define E1000_DEV_ID_PCH_SPT_I219_LM2 0x15B7 /* Sunrise Point-H PCH */ +#define E1000_DEV_ID_PCH_SPT_I219_V2 0x15B8 /* Sunrise Point-H PCH */ +#define E1000_DEV_ID_PCH_LBG_I219_LM3 0x15B9 /* LEWISBURG PCH */ +#define E1000_DEV_ID_PCH_SPT_I219_LM4 0x15D7 +#define E1000_DEV_ID_PCH_SPT_I219_V4 0x15D8 +#define E1000_DEV_ID_PCH_SPT_I219_LM5 0x15E3 +#define E1000_DEV_ID_PCH_SPT_I219_V5 0x15D6 +#define E1000_DEV_ID_PCH_CNP_I219_LM6 0x15BD +#define E1000_DEV_ID_PCH_CNP_I219_V6 0x15BE +#define E1000_DEV_ID_PCH_CNP_I219_LM7 0x15BB +#define E1000_DEV_ID_PCH_CNP_I219_V7 0x15BC #define E1000_DEV_ID_82576 0x10C9 #define E1000_DEV_ID_82576_FIBER 0x10E6 #define E1000_DEV_ID_82576_SERDES 0x10E7 @@ -221,6 +234,8 @@ enum e1000_mac_type { e1000_pchlan, e1000_pch2lan, e1000_pch_lpt, + e1000_pch_spt, + e1000_pch_cnp, e1000_82575, e1000_82576, e1000_82580, @@ -950,11 +965,15 @@ struct e1000_dev_spec_ich8lan { E1000_MUTEX nvm_mutex; E1000_MUTEX swflag_mutex; bool nvm_k1_enabled; + bool disable_k1_off; bool eee_disable; u16 eee_lp_ability; #ifdef ULP_SUPPORT enum e1000_ulp_state ulp_state; -#endif /* NAHUM6LP_HW && ULP_SUPPORT */ + bool ulp_capability_disabled; + bool during_suspend_flow; + bool during_dpg_exit; +#endif /* ULP_SUPPORT */ u16 lat_enc; u16 max_ltr_enc; bool smbus_disable; diff --git a/dpdk/drivers/net/e1000/base/e1000_ich8lan.c b/dpdk/drivers/net/e1000/base/e1000_ich8lan.c index 89d07e90..6dd046d2 100644 --- a/dpdk/drivers/net/e1000/base/e1000_ich8lan.c +++ b/dpdk/drivers/net/e1000/base/e1000_ich8lan.c @@ -94,10 +94,13 @@ STATIC s32 e1000_set_d3_lplu_state_ich8lan(struct e1000_hw *hw, bool active); STATIC s32 e1000_read_nvm_ich8lan(struct e1000_hw *hw, u16 offset, u16 words, u16 *data); +STATIC s32 e1000_read_nvm_spt(struct e1000_hw *hw, u16 offset, u16 words, + u16 *data); STATIC s32 e1000_write_nvm_ich8lan(struct e1000_hw *hw, u16 offset, u16 words, u16 *data); STATIC s32 e1000_validate_nvm_checksum_ich8lan(struct e1000_hw *hw); STATIC s32 e1000_update_nvm_checksum_ich8lan(struct e1000_hw *hw); +STATIC s32 e1000_update_nvm_checksum_spt(struct e1000_hw *hw); STATIC s32 e1000_valid_led_default_ich8lan(struct e1000_hw *hw, u16 *data); STATIC s32 e1000_id_led_init_pchlan(struct e1000_hw *hw); @@ -125,6 +128,14 @@ STATIC s32 e1000_read_flash_byte_ich8lan(struct e1000_hw *hw, u32 offset, u8 *data); STATIC s32 e1000_read_flash_data_ich8lan(struct e1000_hw *hw, u32 offset, u8 size, u16 *data); +STATIC s32 e1000_read_flash_data32_ich8lan(struct e1000_hw *hw, u32 offset, + u32 *data); +STATIC s32 e1000_read_flash_dword_ich8lan(struct e1000_hw *hw, + u32 offset, u32 *data); +STATIC s32 e1000_write_flash_data32_ich8lan(struct e1000_hw *hw, + u32 offset, u32 data); +STATIC s32 e1000_retry_write_flash_dword_ich8lan(struct e1000_hw *hw, + u32 offset, u32 dword); STATIC s32 e1000_read_flash_word_ich8lan(struct e1000_hw *hw, u32 offset, u16 *data); STATIC s32 e1000_retry_write_flash_byte_ich8lan(struct e1000_hw *hw, @@ -233,7 +244,7 @@ STATIC bool e1000_phy_is_accessible_pchlan(struct e1000_hw *hw) if (ret_val) return false; out: - if (hw->mac.type == e1000_pch_lpt) { + if (hw->mac.type >= e1000_pch_lpt) { /* Only unforce SMBus if ME is not active */ if (!(E1000_READ_REG(hw, E1000_FWSM) & E1000_ICH_FWSM_FW_VALID)) { @@ -277,7 +288,7 @@ STATIC void e1000_toggle_lanphypc_pch_lpt(struct e1000_hw *hw) mac_reg &= ~E1000_CTRL_LANPHYPC_VALUE; E1000_WRITE_REG(hw, E1000_CTRL, mac_reg); E1000_WRITE_FLUSH(hw); - usec_delay(10); + msec_delay(1); mac_reg &= ~E1000_CTRL_LANPHYPC_OVERRIDE; E1000_WRITE_REG(hw, E1000_CTRL, mac_reg); E1000_WRITE_FLUSH(hw); @@ -334,6 +345,8 @@ STATIC s32 e1000_init_phy_workarounds_pchlan(struct e1000_hw *hw) */ switch (hw->mac.type) { case e1000_pch_lpt: + case e1000_pch_spt: + case e1000_pch_cnp: if (e1000_phy_is_accessible_pchlan(hw)) break; @@ -481,6 +494,8 @@ STATIC s32 e1000_init_phy_params_pchlan(struct e1000_hw *hw) /* fall-through */ case e1000_pch2lan: case e1000_pch_lpt: + case e1000_pch_spt: + case e1000_pch_cnp: /* In case the PHY needs to be in mdio slow mode, * set slow mode and try to get the PHY id again. */ @@ -623,37 +638,58 @@ STATIC s32 e1000_init_nvm_params_ich8lan(struct e1000_hw *hw) struct e1000_dev_spec_ich8lan *dev_spec = &hw->dev_spec.ich8lan; u32 gfpreg, sector_base_addr, sector_end_addr; u16 i; + u32 nvm_size; DEBUGFUNC("e1000_init_nvm_params_ich8lan"); - /* Can't read flash registers if the register set isn't mapped. */ nvm->type = e1000_nvm_flash_sw; - if (!hw->flash_address) { - DEBUGOUT("ERROR: Flash registers not mapped\n"); - return -E1000_ERR_CONFIG; + + if (hw->mac.type >= e1000_pch_spt) { + /* in SPT, gfpreg doesn't exist. NVM size is taken from the + * STRAP register. This is because in SPT the GbE Flash region + * is no longer accessed through the flash registers. Instead, + * the mechanism has changed, and the Flash region access + * registers are now implemented in GbE memory space. + */ + nvm->flash_base_addr = 0; + nvm_size = + (((E1000_READ_REG(hw, E1000_STRAP) >> 1) & 0x1F) + 1) + * NVM_SIZE_MULTIPLIER; + nvm->flash_bank_size = nvm_size / 2; + /* Adjust to word count */ + nvm->flash_bank_size /= sizeof(u16); + /* Set the base address for flash register access */ + hw->flash_address = hw->hw_addr + E1000_FLASH_BASE_ADDR; + } else { + /* Can't read flash registers if register set isn't mapped. */ + if (!hw->flash_address) { + DEBUGOUT("ERROR: Flash registers not mapped\n"); + return -E1000_ERR_CONFIG; + } + + gfpreg = E1000_READ_FLASH_REG(hw, ICH_FLASH_GFPREG); + + /* sector_X_addr is a "sector"-aligned address (4096 bytes) + * Add 1 to sector_end_addr since this sector is included in + * the overall size. + */ + sector_base_addr = gfpreg & FLASH_GFPREG_BASE_MASK; + sector_end_addr = ((gfpreg >> 16) & FLASH_GFPREG_BASE_MASK) + 1; + + /* flash_base_addr is byte-aligned */ + nvm->flash_base_addr = sector_base_addr + << FLASH_SECTOR_ADDR_SHIFT; + + /* find total size of the NVM, then cut in half since the total + * size represents two separate NVM banks. + */ + nvm->flash_bank_size = ((sector_end_addr - sector_base_addr) + << FLASH_SECTOR_ADDR_SHIFT); + nvm->flash_bank_size /= 2; + /* Adjust to word count */ + nvm->flash_bank_size /= sizeof(u16); } - gfpreg = E1000_READ_FLASH_REG(hw, ICH_FLASH_GFPREG); - - /* sector_X_addr is a "sector"-aligned address (4096 bytes) - * Add 1 to sector_end_addr since this sector is included in - * the overall size. - */ - sector_base_addr = gfpreg & FLASH_GFPREG_BASE_MASK; - sector_end_addr = ((gfpreg >> 16) & FLASH_GFPREG_BASE_MASK) + 1; - - /* flash_base_addr is byte-aligned */ - nvm->flash_base_addr = sector_base_addr << FLASH_SECTOR_ADDR_SHIFT; - - /* find total size of the NVM, then cut in half since the total - * size represents two separate NVM banks. - */ - nvm->flash_bank_size = ((sector_end_addr - sector_base_addr) - << FLASH_SECTOR_ADDR_SHIFT); - nvm->flash_bank_size /= 2; - /* Adjust to word count */ - nvm->flash_bank_size /= sizeof(u16); - nvm->word_size = E1000_SHADOW_RAM_WORDS; /* Clear shadow ram */ @@ -668,8 +704,13 @@ STATIC s32 e1000_init_nvm_params_ich8lan(struct e1000_hw *hw) /* Function Pointers */ nvm->ops.acquire = e1000_acquire_nvm_ich8lan; nvm->ops.release = e1000_release_nvm_ich8lan; - nvm->ops.read = e1000_read_nvm_ich8lan; - nvm->ops.update = e1000_update_nvm_checksum_ich8lan; + if (hw->mac.type >= e1000_pch_spt) { + nvm->ops.read = e1000_read_nvm_spt; + nvm->ops.update = e1000_update_nvm_checksum_spt; + } else { + nvm->ops.read = e1000_read_nvm_ich8lan; + nvm->ops.update = e1000_update_nvm_checksum_ich8lan; + } nvm->ops.valid_led_default = e1000_valid_led_default_ich8lan; nvm->ops.validate = e1000_validate_nvm_checksum_ich8lan; nvm->ops.write = e1000_write_nvm_ich8lan; @@ -758,6 +799,8 @@ STATIC s32 e1000_init_mac_params_ich8lan(struct e1000_hw *hw) mac->ops.rar_set = e1000_rar_set_pch2lan; /* fall-through */ case e1000_pch_lpt: + case e1000_pch_spt: + case e1000_pch_cnp: #ifndef NO_NON_BLOCKING_PHY_MTA_UPDATE_SUPPORT /* multicast address update for pch2 */ mac->ops.update_mc_addr_list = @@ -768,7 +811,13 @@ STATIC s32 e1000_init_mac_params_ich8lan(struct e1000_hw *hw) #if defined(QV_RELEASE) || !defined(NO_PCH_LPT_B0_SUPPORT) /* save PCH revision_id */ e1000_read_pci_cfg(hw, E1000_PCI_REVISION_ID_REG, &pci_cfg); - hw->revision_id = (u8)(pci_cfg &= 0x000F); + /* SPT uses full byte for revision ID, + * as opposed to previous generations + */ + if (hw->mac.type >= e1000_pch_spt) + hw->revision_id = (u8)(pci_cfg &= 0x00FF); + else + hw->revision_id = (u8)(pci_cfg &= 0x000F); #endif /* QV_RELEASE || !defined(NO_PCH_LPT_B0_SUPPORT) */ /* check management mode */ mac->ops.check_mng_mode = e1000_check_mng_mode_pchlan; @@ -786,7 +835,7 @@ STATIC s32 e1000_init_mac_params_ich8lan(struct e1000_hw *hw) break; } - if (mac->type == e1000_pch_lpt) { + if (mac->type >= e1000_pch_lpt) { mac->rar_entry_count = E1000_PCH_LPT_RAR_ENTRIES; mac->ops.rar_set = e1000_rar_set_pch_lpt; mac->ops.setup_physical_interface = e1000_setup_copper_link_pch_lpt; @@ -1015,8 +1064,9 @@ release: /* clear FEXTNVM6 bit 8 on link down or 10/100 */ fextnvm6 &= ~E1000_FEXTNVM6_REQ_PLL_CLK; - if (!link || ((status & E1000_STATUS_SPEED_100) && - (status & E1000_STATUS_FD))) + if ((hw->phy.revision > 5) || !link || + ((status & E1000_STATUS_SPEED_100) && + (status & E1000_STATUS_FD))) goto update_fextnvm6; ret_val = hw->phy.ops.read_reg(hw, I217_INBAND_CTRL, ®); @@ -1068,6 +1118,7 @@ s32 e1000_enable_ulp_lpt_lp(struct e1000_hw *hw, bool to_sx) u32 mac_reg; s32 ret_val = E1000_SUCCESS; u16 phy_reg; + u16 oem_reg = 0; if ((hw->mac.type < e1000_pch_lpt) || (hw->device_id == E1000_DEV_ID_PCH_LPT_I217_LM) || @@ -1128,6 +1179,25 @@ s32 e1000_enable_ulp_lpt_lp(struct e1000_hw *hw, bool to_sx) mac_reg |= E1000_CTRL_EXT_FORCE_SMBUS; E1000_WRITE_REG(hw, E1000_CTRL_EXT, mac_reg); + /* Si workaround for ULP entry flow on i127/rev6 h/w. Enable + * LPLU and disable Gig speed when entering ULP + */ + if ((hw->phy.type == e1000_phy_i217) && (hw->phy.revision == 6)) { + ret_val = e1000_read_phy_reg_hv_locked(hw, HV_OEM_BITS, + &oem_reg); + if (ret_val) + goto release; + + phy_reg = oem_reg; + phy_reg |= HV_OEM_BITS_LPLU | HV_OEM_BITS_GBE_DIS; + + ret_val = e1000_write_phy_reg_hv_locked(hw, HV_OEM_BITS, + phy_reg); + + if (ret_val) + goto release; + } + skip_smbus: if (!to_sx) { /* Change the 'Link Status Change' interrupt to trigger @@ -1184,6 +1254,14 @@ skip_smbus: E1000_WRITE_REG(hw, E1000_TCTL, mac_reg); } + if ((hw->phy.type == e1000_phy_i217) && (hw->phy.revision == 6) && + to_sx && (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU)) { + ret_val = e1000_write_phy_reg_hv_locked(hw, HV_OEM_BITS, + oem_reg); + if (ret_val) + goto release; + } + release: hw->phy.ops.release(hw); out: @@ -1240,10 +1318,10 @@ s32 e1000_disable_ulp_lpt_lp(struct e1000_hw *hw, bool force) E1000_WRITE_REG(hw, E1000_H2ME, mac_reg); } - /* Poll up to 100msec for ME to clear ULP_CFG_DONE */ + /* Poll up to 300msec for ME to clear ULP_CFG_DONE. */ while (E1000_READ_REG(hw, E1000_FWSM) & E1000_FWSM_ULP_CFG_DONE) { - if (i++ == 10) { + if (i++ == 30) { ret_val = -E1000_ERR_PHY; goto out; } @@ -1343,6 +1421,8 @@ s32 e1000_disable_ulp_lpt_lp(struct e1000_hw *hw, bool force) I218_ULP_CONFIG1_RESET_TO_SMBUS | I218_ULP_CONFIG1_WOL_HOST | I218_ULP_CONFIG1_INBAND_EXIT | + I218_ULP_CONFIG1_EN_ULP_LANPHYPC | + I218_ULP_CONFIG1_DIS_CLR_STICKY_ON_PERST | I218_ULP_CONFIG1_DISABLE_SMB_PERST); e1000_write_phy_reg_hv_locked(hw, I218_ULP_CONFIG1, phy_reg); @@ -1360,6 +1440,8 @@ s32 e1000_disable_ulp_lpt_lp(struct e1000_hw *hw, bool force) if (hw->mac.autoneg) e1000_phy_setup_autoneg(hw); + else + e1000_setup_copper_link_generic(hw); e1000_sw_lcd_config_ich8lan(hw); @@ -1397,6 +1479,8 @@ out: } #endif /* ULP_SUPPORT */ + + /** * e1000_check_for_copper_link_ich8lan - Check for link (Copper) * @hw: pointer to the HW structure @@ -1456,8 +1540,7 @@ STATIC s32 e1000_check_for_copper_link_ich8lan(struct e1000_hw *hw) * aggressive resulting in many collisions. To avoid this, increase * the IPG and reduce Rx latency in the PHY. */ - if (((hw->mac.type == e1000_pch2lan) || - (hw->mac.type == e1000_pch_lpt)) && link) { + if ((hw->mac.type >= e1000_pch2lan) && link) { u16 speed, duplex; e1000_get_speed_and_duplex_copper_generic(hw, &speed, &duplex); @@ -1468,6 +1551,10 @@ STATIC s32 e1000_check_for_copper_link_ich8lan(struct e1000_hw *hw) tipg_reg |= 0xFF; /* Reduce Rx latency in analog PHY */ emi_val = 0; + } else if (hw->mac.type >= e1000_pch_spt && + duplex == FULL_DUPLEX && speed != SPEED_1000) { + tipg_reg |= 0xC; + emi_val = 1; } else { /* Roll back the default values */ tipg_reg |= 0x08; @@ -1486,10 +1573,78 @@ STATIC s32 e1000_check_for_copper_link_ich8lan(struct e1000_hw *hw) emi_addr = I217_RX_CONFIG; ret_val = e1000_write_emi_reg_locked(hw, emi_addr, emi_val); + + if (hw->mac.type >= e1000_pch_lpt) { + u16 phy_reg; + + hw->phy.ops.read_reg_locked(hw, I217_PLL_CLOCK_GATE_REG, + &phy_reg); + phy_reg &= ~I217_PLL_CLOCK_GATE_MASK; + if (speed == SPEED_100 || speed == SPEED_10) + phy_reg |= 0x3E8; + else + phy_reg |= 0xFA; + hw->phy.ops.write_reg_locked(hw, + I217_PLL_CLOCK_GATE_REG, + phy_reg); + + if (speed == SPEED_1000) { + hw->phy.ops.read_reg_locked(hw, HV_PM_CTRL, + &phy_reg); + + phy_reg |= HV_PM_CTRL_K1_CLK_REQ; + + hw->phy.ops.write_reg_locked(hw, HV_PM_CTRL, + phy_reg); + } + } hw->phy.ops.release(hw); if (ret_val) return ret_val; + + if (hw->mac.type >= e1000_pch_spt) { + u16 data; + u16 ptr_gap; + + if (speed == SPEED_1000) { + ret_val = hw->phy.ops.acquire(hw); + if (ret_val) + return ret_val; + + ret_val = hw->phy.ops.read_reg_locked(hw, + PHY_REG(776, 20), + &data); + if (ret_val) { + hw->phy.ops.release(hw); + return ret_val; + } + + ptr_gap = (data & (0x3FF << 2)) >> 2; + if (ptr_gap < 0x18) { + data &= ~(0x3FF << 2); + data |= (0x18 << 2); + ret_val = + hw->phy.ops.write_reg_locked(hw, + PHY_REG(776, 20), data); + } + hw->phy.ops.release(hw); + if (ret_val) + return ret_val; + } else { + ret_val = hw->phy.ops.acquire(hw); + if (ret_val) + return ret_val; + + ret_val = hw->phy.ops.write_reg_locked(hw, + PHY_REG(776, 20), + 0xC023); + hw->phy.ops.release(hw); + if (ret_val) + return ret_val; + + } + } } /* I217 Packet Loss issue: @@ -1497,7 +1652,7 @@ STATIC s32 e1000_check_for_copper_link_ich8lan(struct e1000_hw *hw) * on power up. * Set the Beacon Duration for I217 to 8 usec */ - if (hw->mac.type == e1000_pch_lpt) { + if (hw->mac.type >= e1000_pch_lpt) { u32 mac_reg; mac_reg = E1000_READ_REG(hw, E1000_FEXTNVM4); @@ -1519,10 +1674,29 @@ STATIC s32 e1000_check_for_copper_link_ich8lan(struct e1000_hw *hw) hw->dev_spec.ich8lan.eee_lp_ability = 0; /* Configure K0s minimum time */ - if (hw->mac.type == e1000_pch_lpt) { + if (hw->mac.type >= e1000_pch_lpt) { e1000_configure_k0s_lpt(hw, K1_ENTRY_LATENCY, K1_MIN_TIME); } + if (hw->mac.type >= e1000_pch_lpt) { + u32 fextnvm6 = E1000_READ_REG(hw, E1000_FEXTNVM6); + + if (hw->mac.type == e1000_pch_spt) { + /* FEXTNVM6 K1-off workaround - for SPT only */ + u32 pcieanacfg = E1000_READ_REG(hw, E1000_PCIEANACFG); + + if (pcieanacfg & E1000_FEXTNVM6_K1_OFF_ENABLE) + fextnvm6 |= E1000_FEXTNVM6_K1_OFF_ENABLE; + else + fextnvm6 &= ~E1000_FEXTNVM6_K1_OFF_ENABLE; + } + + if (hw->dev_spec.ich8lan.disable_k1_off == true) + fextnvm6 &= ~E1000_FEXTNVM6_K1_OFF_ENABLE; + + E1000_WRITE_REG(hw, E1000_FEXTNVM6, fextnvm6); + } + if (!link) return E1000_SUCCESS; /* No link detected */ @@ -1616,6 +1790,8 @@ void e1000_init_function_pointers_ich8lan(struct e1000_hw *hw) case e1000_pchlan: case e1000_pch2lan: case e1000_pch_lpt: + case e1000_pch_spt: + case e1000_pch_cnp: hw->phy.ops.init_params = e1000_init_phy_params_pchlan; break; default: @@ -2081,6 +2257,8 @@ STATIC s32 e1000_sw_lcd_config_ich8lan(struct e1000_hw *hw) case e1000_pchlan: case e1000_pch2lan: case e1000_pch_lpt: + case e1000_pch_spt: + case e1000_pch_cnp: sw_cfg_mask = E1000_FEXTNVM_SW_CONFIG_ICH8M; break; default: @@ -3204,6 +3382,41 @@ STATIC s32 e1000_valid_nvm_bank_detect_ich8lan(struct e1000_hw *hw, u32 *bank) DEBUGFUNC("e1000_valid_nvm_bank_detect_ich8lan"); switch (hw->mac.type) { + case e1000_pch_spt: + case e1000_pch_cnp: + bank1_offset = nvm->flash_bank_size; + act_offset = E1000_ICH_NVM_SIG_WORD; + + /* set bank to 0 in case flash read fails */ + *bank = 0; + + /* Check bank 0 */ + ret_val = e1000_read_flash_dword_ich8lan(hw, act_offset, + &nvm_dword); + if (ret_val) + return ret_val; + sig_byte = (u8)((nvm_dword & 0xFF00) >> 8); + if ((sig_byte & E1000_ICH_NVM_VALID_SIG_MASK) == + E1000_ICH_NVM_SIG_VALUE) { + *bank = 0; + return E1000_SUCCESS; + } + + /* Check bank 1 */ + ret_val = e1000_read_flash_dword_ich8lan(hw, act_offset + + bank1_offset, + &nvm_dword); + if (ret_val) + return ret_val; + sig_byte = (u8)((nvm_dword & 0xFF00) >> 8); + if ((sig_byte & E1000_ICH_NVM_VALID_SIG_MASK) == + E1000_ICH_NVM_SIG_VALUE) { + *bank = 1; + return E1000_SUCCESS; + } + + DEBUGOUT("ERROR: No valid NVM bank present\n"); + return -E1000_ERR_NVM; case e1000_ich8lan: case e1000_ich9lan: eecd = E1000_READ_REG(hw, E1000_EECD); @@ -3250,6 +3463,99 @@ STATIC s32 e1000_valid_nvm_bank_detect_ich8lan(struct e1000_hw *hw, u32 *bank) } } +/** + * e1000_read_nvm_spt - NVM access for SPT + * @hw: pointer to the HW structure + * @offset: The offset (in bytes) of the word(s) to read. + * @words: Size of data to read in words. + * @data: pointer to the word(s) to read at offset. + * + * Reads a word(s) from the NVM + **/ +STATIC s32 e1000_read_nvm_spt(struct e1000_hw *hw, u16 offset, u16 words, + u16 *data) +{ + struct e1000_nvm_info *nvm = &hw->nvm; + struct e1000_dev_spec_ich8lan *dev_spec = &hw->dev_spec.ich8lan; + u32 act_offset; + s32 ret_val = E1000_SUCCESS; + u32 bank = 0; + u32 dword = 0; + u16 offset_to_read; + u16 i; + + DEBUGFUNC("e1000_read_nvm_spt"); + + if ((offset >= nvm->word_size) || (words > nvm->word_size - offset) || + (words == 0)) { + DEBUGOUT("nvm parameter(s) out of bounds\n"); + ret_val = -E1000_ERR_NVM; + goto out; + } + + nvm->ops.acquire(hw); + + ret_val = e1000_valid_nvm_bank_detect_ich8lan(hw, &bank); + if (ret_val != E1000_SUCCESS) { + DEBUGOUT("Could not detect valid bank, assuming bank 0\n"); + bank = 0; + } + + act_offset = (bank) ? nvm->flash_bank_size : 0; + act_offset += offset; + + ret_val = E1000_SUCCESS; + + for (i = 0; i < words; i += 2) { + if (words - i == 1) { + if (dev_spec->shadow_ram[offset+i].modified) { + data[i] = dev_spec->shadow_ram[offset+i].value; + } else { + offset_to_read = act_offset + i - + ((act_offset + i) % 2); + ret_val = + e1000_read_flash_dword_ich8lan(hw, + offset_to_read, + &dword); + if (ret_val) + break; + if ((act_offset + i) % 2 == 0) + data[i] = (u16)(dword & 0xFFFF); + else + data[i] = (u16)((dword >> 16) & 0xFFFF); + } + } else { + offset_to_read = act_offset + i; + if (!(dev_spec->shadow_ram[offset+i].modified) || + !(dev_spec->shadow_ram[offset+i+1].modified)) { + ret_val = + e1000_read_flash_dword_ich8lan(hw, + offset_to_read, + &dword); + if (ret_val) + break; + } + if (dev_spec->shadow_ram[offset+i].modified) + data[i] = dev_spec->shadow_ram[offset+i].value; + else + data[i] = (u16) (dword & 0xFFFF); + if (dev_spec->shadow_ram[offset+i].modified) + data[i+1] = + dev_spec->shadow_ram[offset+i+1].value; + else + data[i+1] = (u16) (dword >> 16 & 0xFFFF); + } + } + + nvm->ops.release(hw); + +out: + if (ret_val) + DEBUGOUT1("NVM read error: %d\n", ret_val); + + return ret_val; +} + /** * e1000_read_nvm_ich8lan - Read word(s) from the NVM * @hw: pointer to the HW structure @@ -3337,7 +3643,11 @@ STATIC s32 e1000_flash_cycle_init_ich8lan(struct e1000_hw *hw) /* Clear FCERR and DAEL in hw status by writing 1 */ hsfsts.hsf_status.flcerr = 1; hsfsts.hsf_status.dael = 1; - E1000_WRITE_FLASH_REG16(hw, ICH_FLASH_HSFSTS, hsfsts.regval); + if (hw->mac.type >= e1000_pch_spt) + E1000_WRITE_FLASH_REG(hw, ICH_FLASH_HSFSTS, + hsfsts.regval & 0xFFFF); + else + E1000_WRITE_FLASH_REG16(hw, ICH_FLASH_HSFSTS, hsfsts.regval); /* Either we should have a hardware SPI cycle in progress * bit to check against, in order to start a new cycle or @@ -3353,7 +3663,12 @@ STATIC s32 e1000_flash_cycle_init_ich8lan(struct e1000_hw *hw) * Begin by setting Flash Cycle Done. */ hsfsts.hsf_status.flcdone = 1; - E1000_WRITE_FLASH_REG16(hw, ICH_FLASH_HSFSTS, hsfsts.regval); + if (hw->mac.type >= e1000_pch_spt) + E1000_WRITE_FLASH_REG(hw, ICH_FLASH_HSFSTS, + hsfsts.regval & 0xFFFF); + else + E1000_WRITE_FLASH_REG16(hw, ICH_FLASH_HSFSTS, + hsfsts.regval); ret_val = E1000_SUCCESS; } else { s32 i; @@ -3375,8 +3690,12 @@ STATIC s32 e1000_flash_cycle_init_ich8lan(struct e1000_hw *hw) * now set the Flash Cycle Done. */ hsfsts.hsf_status.flcdone = 1; - E1000_WRITE_FLASH_REG16(hw, ICH_FLASH_HSFSTS, - hsfsts.regval); + if (hw->mac.type >= e1000_pch_spt) + E1000_WRITE_FLASH_REG(hw, ICH_FLASH_HSFSTS, + hsfsts.regval & 0xFFFF); + else + E1000_WRITE_FLASH_REG16(hw, ICH_FLASH_HSFSTS, + hsfsts.regval); } else { DEBUGOUT("Flash controller busy, cannot get access\n"); } @@ -3401,10 +3720,17 @@ STATIC s32 e1000_flash_cycle_ich8lan(struct e1000_hw *hw, u32 timeout) DEBUGFUNC("e1000_flash_cycle_ich8lan"); /* Start a cycle by writing 1 in Flash Cycle Go in Hw Flash Control */ - hsflctl.regval = E1000_READ_FLASH_REG16(hw, ICH_FLASH_HSFCTL); + if (hw->mac.type >= e1000_pch_spt) + hsflctl.regval = E1000_READ_FLASH_REG(hw, ICH_FLASH_HSFSTS)>>16; + else + hsflctl.regval = E1000_READ_FLASH_REG16(hw, ICH_FLASH_HSFCTL); hsflctl.hsf_ctrl.flcgo = 1; - E1000_WRITE_FLASH_REG16(hw, ICH_FLASH_HSFCTL, hsflctl.regval); + if (hw->mac.type >= e1000_pch_spt) + E1000_WRITE_FLASH_REG(hw, ICH_FLASH_HSFSTS, + hsflctl.regval << 16); + else + E1000_WRITE_FLASH_REG16(hw, ICH_FLASH_HSFCTL, hsflctl.regval); /* wait till FDONE bit is set to 1 */ do { @@ -3420,6 +3746,29 @@ STATIC s32 e1000_flash_cycle_ich8lan(struct e1000_hw *hw, u32 timeout) return -E1000_ERR_NVM; } +/** + * e1000_read_flash_dword_ich8lan - Read dword from flash + * @hw: pointer to the HW structure + * @offset: offset to data location + * @data: pointer to the location for storing the data + * + * Reads the flash dword at offset into data. Offset is converted + * to bytes before read. + **/ +STATIC s32 e1000_read_flash_dword_ich8lan(struct e1000_hw *hw, u32 offset, + u32 *data) +{ + DEBUGFUNC("e1000_read_flash_dword_ich8lan"); + + if (!data) + return -E1000_ERR_NVM; + + /* Must convert word offset into bytes. */ + offset <<= 1; + + return e1000_read_flash_data32_ich8lan(hw, offset, data); +} + /** * e1000_read_flash_word_ich8lan - Read word from flash * @hw: pointer to the HW structure @@ -3457,7 +3806,13 @@ STATIC s32 e1000_read_flash_byte_ich8lan(struct e1000_hw *hw, u32 offset, s32 ret_val; u16 word = 0; - ret_val = e1000_read_flash_data_ich8lan(hw, offset, 1, &word); + /* In SPT, only 32 bits access is supported, + * so this function should not be called. + */ + if (hw->mac.type >= e1000_pch_spt) + return -E1000_ERR_NVM; + else + ret_val = e1000_read_flash_data_ich8lan(hw, offset, 1, &word); if (ret_val) return ret_val; @@ -3543,6 +3898,83 @@ STATIC s32 e1000_read_flash_data_ich8lan(struct e1000_hw *hw, u32 offset, return ret_val; } +/** + * e1000_read_flash_data32_ich8lan - Read dword from NVM + * @hw: pointer to the HW structure + * @offset: The offset (in bytes) of the dword to read. + * @data: Pointer to the dword to store the value read. + * + * Reads a byte or word from the NVM using the flash access registers. + **/ +STATIC s32 e1000_read_flash_data32_ich8lan(struct e1000_hw *hw, u32 offset, + u32 *data) +{ + union ich8_hws_flash_status hsfsts; + union ich8_hws_flash_ctrl hsflctl; + u32 flash_linear_addr; + s32 ret_val = -E1000_ERR_NVM; + u8 count = 0; + + DEBUGFUNC("e1000_read_flash_data_ich8lan"); + + if (offset > ICH_FLASH_LINEAR_ADDR_MASK || + hw->mac.type < e1000_pch_spt) + return -E1000_ERR_NVM; + flash_linear_addr = ((ICH_FLASH_LINEAR_ADDR_MASK & offset) + + hw->nvm.flash_base_addr); + + do { + usec_delay(1); + /* Steps */ + ret_val = e1000_flash_cycle_init_ich8lan(hw); + if (ret_val != E1000_SUCCESS) + break; + /* In SPT, This register is in Lan memory space, not flash. + * Therefore, only 32 bit access is supported + */ + hsflctl.regval = E1000_READ_FLASH_REG(hw, ICH_FLASH_HSFSTS)>>16; + + /* 0b/1b corresponds to 1 or 2 byte size, respectively. */ + hsflctl.hsf_ctrl.fldbcount = sizeof(u32) - 1; + hsflctl.hsf_ctrl.flcycle = ICH_CYCLE_READ; + /* In SPT, This register is in Lan memory space, not flash. + * Therefore, only 32 bit access is supported + */ + E1000_WRITE_FLASH_REG(hw, ICH_FLASH_HSFSTS, + (u32)hsflctl.regval << 16); + E1000_WRITE_FLASH_REG(hw, ICH_FLASH_FADDR, flash_linear_addr); + + ret_val = e1000_flash_cycle_ich8lan(hw, + ICH_FLASH_READ_COMMAND_TIMEOUT); + + /* Check if FCERR is set to 1, if set to 1, clear it + * and try the whole sequence a few more times, else + * read in (shift in) the Flash Data0, the order is + * least significant byte first msb to lsb + */ + if (ret_val == E1000_SUCCESS) { + *data = E1000_READ_FLASH_REG(hw, ICH_FLASH_FDATA0); + break; + } else { + /* If we've gotten here, then things are probably + * completely hosed, but if the error condition is + * detected, it won't hurt to give it another try... + * ICH_FLASH_CYCLE_REPEAT_COUNT times. + */ + hsfsts.regval = E1000_READ_FLASH_REG16(hw, + ICH_FLASH_HSFSTS); + if (hsfsts.hsf_status.flcerr) { + /* Repeat for some time before giving up. */ + continue; + } else if (!hsfsts.hsf_status.flcdone) { + DEBUGOUT("Timeout error - flash cycle did not complete.\n"); + break; + } + } + } while (count++ < ICH_FLASH_CYCLE_REPEAT_COUNT); + + return ret_val; +} /** * e1000_write_nvm_ich8lan - Write word(s) to the NVM @@ -3580,6 +4012,175 @@ STATIC s32 e1000_write_nvm_ich8lan(struct e1000_hw *hw, u16 offset, u16 words, return E1000_SUCCESS; } +/** + * e1000_update_nvm_checksum_spt - Update the checksum for NVM + * @hw: pointer to the HW structure + * + * The NVM checksum is updated by calling the generic update_nvm_checksum, + * which writes the checksum to the shadow ram. The changes in the shadow + * ram are then committed to the EEPROM by processing each bank at a time + * checking for the modified bit and writing only the pending changes. + * After a successful commit, the shadow ram is cleared and is ready for + * future writes. + **/ +STATIC s32 e1000_update_nvm_checksum_spt(struct e1000_hw *hw) +{ + struct e1000_nvm_info *nvm = &hw->nvm; + struct e1000_dev_spec_ich8lan *dev_spec = &hw->dev_spec.ich8lan; + u32 i, act_offset, new_bank_offset, old_bank_offset, bank; + s32 ret_val; + u32 dword = 0; + + DEBUGFUNC("e1000_update_nvm_checksum_spt"); + + ret_val = e1000_update_nvm_checksum_generic(hw); + if (ret_val) + goto out; + + if (nvm->type != e1000_nvm_flash_sw) + goto out; + + nvm->ops.acquire(hw); + + /* We're writing to the opposite bank so if we're on bank 1, + * write to bank 0 etc. We also need to erase the segment that + * is going to be written + */ + ret_val = e1000_valid_nvm_bank_detect_ich8lan(hw, &bank); + if (ret_val != E1000_SUCCESS) { + DEBUGOUT("Could not detect valid bank, assuming bank 0\n"); + bank = 0; + } + + if (bank == 0) { + new_bank_offset = nvm->flash_bank_size; + old_bank_offset = 0; + ret_val = e1000_erase_flash_bank_ich8lan(hw, 1); + if (ret_val) + goto release; + } else { + old_bank_offset = nvm->flash_bank_size; + new_bank_offset = 0; + ret_val = e1000_erase_flash_bank_ich8lan(hw, 0); + if (ret_val) + goto release; + } + for (i = 0; i < E1000_SHADOW_RAM_WORDS; i += 2) { + /* Determine whether to write the value stored + * in the other NVM bank or a modified value stored + * in the shadow RAM + */ + ret_val = e1000_read_flash_dword_ich8lan(hw, + i + old_bank_offset, + &dword); + + if (dev_spec->shadow_ram[i].modified) { + dword &= 0xffff0000; + dword |= (dev_spec->shadow_ram[i].value & 0xffff); + } + if (dev_spec->shadow_ram[i + 1].modified) { + dword &= 0x0000ffff; + dword |= ((dev_spec->shadow_ram[i + 1].value & 0xffff) + << 16); + } + if (ret_val) + break; + + /* If the word is 0x13, then make sure the signature bits + * (15:14) are 11b until the commit has completed. + * This will allow us to write 10b which indicates the + * signature is valid. We want to do this after the write + * has completed so that we don't mark the segment valid + * while the write is still in progress + */ + if (i == E1000_ICH_NVM_SIG_WORD - 1) + dword |= E1000_ICH_NVM_SIG_MASK << 16; + + /* Convert offset to bytes. */ + act_offset = (i + new_bank_offset) << 1; + + usec_delay(100); + + /* Write the data to the new bank. Offset in words*/ + act_offset = i + new_bank_offset; + ret_val = e1000_retry_write_flash_dword_ich8lan(hw, act_offset, + dword); + if (ret_val) + break; + } + + /* Don't bother writing the segment valid bits if sector + * programming failed. + */ + if (ret_val) { + DEBUGOUT("Flash commit failed.\n"); + goto release; + } + + /* Finally validate the new segment by setting bit 15:14 + * to 10b in word 0x13 , this can be done without an + * erase as well since these bits are 11 to start with + * and we need to change bit 14 to 0b + */ + act_offset = new_bank_offset + E1000_ICH_NVM_SIG_WORD; + + /*offset in words but we read dword*/ + --act_offset; + ret_val = e1000_read_flash_dword_ich8lan(hw, act_offset, &dword); + + if (ret_val) + goto release; + + dword &= 0xBFFFFFFF; + ret_val = e1000_retry_write_flash_dword_ich8lan(hw, act_offset, dword); + + if (ret_val) + goto release; + + /* And invalidate the previously valid segment by setting + * its signature word (0x13) high_byte to 0b. This can be + * done without an erase because flash erase sets all bits + * to 1's. We can write 1's to 0's without an erase + */ + act_offset = (old_bank_offset + E1000_ICH_NVM_SIG_WORD) * 2 + 1; + + /* offset in words but we read dword*/ + act_offset = old_bank_offset + E1000_ICH_NVM_SIG_WORD - 1; + ret_val = e1000_read_flash_dword_ich8lan(hw, act_offset, &dword); + + if (ret_val) + goto release; + + dword &= 0x00FFFFFF; + ret_val = e1000_retry_write_flash_dword_ich8lan(hw, act_offset, dword); + + if (ret_val) + goto release; + + /* Great! Everything worked, we can now clear the cached entries. */ + for (i = 0; i < E1000_SHADOW_RAM_WORDS; i++) { + dev_spec->shadow_ram[i].modified = false; + dev_spec->shadow_ram[i].value = 0xFFFF; + } + +release: + nvm->ops.release(hw); + + /* Reload the EEPROM, or else modifications will not appear + * until after the next adapter reset. + */ + if (!ret_val) { + nvm->ops.reload(hw); + msec_delay(10); + } + +out: + if (ret_val) + DEBUGOUT1("NVM update error: %d\n", ret_val); + + return ret_val; +} + /** * e1000_update_nvm_checksum_ich8lan - Update the checksum for NVM * @hw: pointer to the HW structure @@ -3757,6 +4358,8 @@ STATIC s32 e1000_validate_nvm_checksum_ich8lan(struct e1000_hw *hw) */ switch (hw->mac.type) { case e1000_pch_lpt: + case e1000_pch_spt: + case e1000_pch_cnp: word = NVM_COMPAT; valid_csum_mask = NVM_COMPAT_VALID_CSUM; break; @@ -3804,8 +4407,13 @@ STATIC s32 e1000_write_flash_data_ich8lan(struct e1000_hw *hw, u32 offset, DEBUGFUNC("e1000_write_ich8_data"); - if (size < 1 || size > 2 || offset > ICH_FLASH_LINEAR_ADDR_MASK) - return -E1000_ERR_NVM; + if (hw->mac.type >= e1000_pch_spt) { + if (size != 4 || offset > ICH_FLASH_LINEAR_ADDR_MASK) + return -E1000_ERR_NVM; + } else { + if (size < 1 || size > 2 || offset > ICH_FLASH_LINEAR_ADDR_MASK) + return -E1000_ERR_NVM; + } flash_linear_addr = ((ICH_FLASH_LINEAR_ADDR_MASK & offset) + hw->nvm.flash_base_addr); @@ -3816,12 +4424,29 @@ STATIC s32 e1000_write_flash_data_ich8lan(struct e1000_hw *hw, u32 offset, ret_val = e1000_flash_cycle_init_ich8lan(hw); if (ret_val != E1000_SUCCESS) break; - hsflctl.regval = E1000_READ_FLASH_REG16(hw, ICH_FLASH_HSFCTL); + /* In SPT, This register is in Lan memory space, not + * flash. Therefore, only 32 bit access is supported + */ + if (hw->mac.type >= e1000_pch_spt) + hsflctl.regval = + E1000_READ_FLASH_REG(hw, ICH_FLASH_HSFSTS)>>16; + else + hsflctl.regval = + E1000_READ_FLASH_REG16(hw, ICH_FLASH_HSFCTL); /* 0b/1b corresponds to 1 or 2 byte size, respectively. */ hsflctl.hsf_ctrl.fldbcount = size - 1; hsflctl.hsf_ctrl.flcycle = ICH_CYCLE_WRITE; - E1000_WRITE_FLASH_REG16(hw, ICH_FLASH_HSFCTL, hsflctl.regval); + /* In SPT, This register is in Lan memory space, + * not flash. Therefore, only 32 bit access is + * supported + */ + if (hw->mac.type >= e1000_pch_spt) + E1000_WRITE_FLASH_REG(hw, ICH_FLASH_HSFSTS, + hsflctl.regval << 16); + else + E1000_WRITE_FLASH_REG16(hw, ICH_FLASH_HSFCTL, + hsflctl.regval); E1000_WRITE_FLASH_REG(hw, ICH_FLASH_FADDR, flash_linear_addr); @@ -3859,6 +4484,94 @@ STATIC s32 e1000_write_flash_data_ich8lan(struct e1000_hw *hw, u32 offset, return ret_val; } +/** +* e1000_write_flash_data32_ich8lan - Writes 4 bytes to the NVM +* @hw: pointer to the HW structure +* @offset: The offset (in bytes) of the dwords to read. +* @data: The 4 bytes to write to the NVM. +* +* Writes one/two/four bytes to the NVM using the flash access registers. +**/ +STATIC s32 e1000_write_flash_data32_ich8lan(struct e1000_hw *hw, u32 offset, + u32 data) +{ + union ich8_hws_flash_status hsfsts; + union ich8_hws_flash_ctrl hsflctl; + u32 flash_linear_addr; + s32 ret_val; + u8 count = 0; + + DEBUGFUNC("e1000_write_flash_data32_ich8lan"); + + if (hw->mac.type >= e1000_pch_spt) { + if (offset > ICH_FLASH_LINEAR_ADDR_MASK) + return -E1000_ERR_NVM; + } + flash_linear_addr = ((ICH_FLASH_LINEAR_ADDR_MASK & offset) + + hw->nvm.flash_base_addr); + do { + usec_delay(1); + /* Steps */ + ret_val = e1000_flash_cycle_init_ich8lan(hw); + if (ret_val != E1000_SUCCESS) + break; + + /* In SPT, This register is in Lan memory space, not + * flash. Therefore, only 32 bit access is supported + */ + if (hw->mac.type >= e1000_pch_spt) + hsflctl.regval = E1000_READ_FLASH_REG(hw, + ICH_FLASH_HSFSTS) + >> 16; + else + hsflctl.regval = E1000_READ_FLASH_REG16(hw, + ICH_FLASH_HSFCTL); + + hsflctl.hsf_ctrl.fldbcount = sizeof(u32) - 1; + hsflctl.hsf_ctrl.flcycle = ICH_CYCLE_WRITE; + + /* In SPT, This register is in Lan memory space, + * not flash. Therefore, only 32 bit access is + * supported + */ + if (hw->mac.type >= e1000_pch_spt) + E1000_WRITE_FLASH_REG(hw, ICH_FLASH_HSFSTS, + hsflctl.regval << 16); + else + E1000_WRITE_FLASH_REG16(hw, ICH_FLASH_HSFCTL, + hsflctl.regval); + + E1000_WRITE_FLASH_REG(hw, ICH_FLASH_FADDR, flash_linear_addr); + + E1000_WRITE_FLASH_REG(hw, ICH_FLASH_FDATA0, data); + + /* check if FCERR is set to 1 , if set to 1, clear it + * and try the whole sequence a few more times else done + */ + ret_val = e1000_flash_cycle_ich8lan(hw, + ICH_FLASH_WRITE_COMMAND_TIMEOUT); + + if (ret_val == E1000_SUCCESS) + break; + + /* If we're here, then things are most likely + * completely hosed, but if the error condition + * is detected, it won't hurt to give it another + * try...ICH_FLASH_CYCLE_REPEAT_COUNT times. + */ + hsfsts.regval = E1000_READ_FLASH_REG16(hw, ICH_FLASH_HSFSTS); + + if (hsfsts.hsf_status.flcerr) + /* Repeat for some time before giving up. */ + continue; + if (!hsfsts.hsf_status.flcdone) { + DEBUGOUT("Timeout error - flash cycle did not complete.\n"); + break; + } + } while (count++ < ICH_FLASH_CYCLE_REPEAT_COUNT); + + return ret_val; +} /** * e1000_write_flash_byte_ich8lan - Write a single byte to NVM @@ -3878,7 +4591,42 @@ STATIC s32 e1000_write_flash_byte_ich8lan(struct e1000_hw *hw, u32 offset, return e1000_write_flash_data_ich8lan(hw, offset, 1, word); } +/** +* e1000_retry_write_flash_dword_ich8lan - Writes a dword to NVM +* @hw: pointer to the HW structure +* @offset: The offset of the word to write. +* @dword: The dword to write to the NVM. +* +* Writes a single dword to the NVM using the flash access registers. +* Goes through a retry algorithm before giving up. +**/ +STATIC s32 e1000_retry_write_flash_dword_ich8lan(struct e1000_hw *hw, + u32 offset, u32 dword) +{ + s32 ret_val; + u16 program_retries; + DEBUGFUNC("e1000_retry_write_flash_dword_ich8lan"); + + /* Must convert word offset into bytes. */ + offset <<= 1; + + ret_val = e1000_write_flash_data32_ich8lan(hw, offset, dword); + + if (!ret_val) + return ret_val; + for (program_retries = 0; program_retries < 100; program_retries++) { + DEBUGOUT2("Retrying Byte %8.8X at offset %u\n", dword, offset); + usec_delay(100); + ret_val = e1000_write_flash_data32_ich8lan(hw, offset, dword); + if (ret_val == E1000_SUCCESS) + break; + } + if (program_retries == 100) + return -E1000_ERR_NVM; + + return E1000_SUCCESS; +} /** * e1000_retry_write_flash_byte_ich8lan - Writes a single byte to NVM @@ -3988,12 +4736,22 @@ STATIC s32 e1000_erase_flash_bank_ich8lan(struct e1000_hw *hw, u32 bank) /* Write a value 11 (block Erase) in Flash * Cycle field in hw flash control */ - hsflctl.regval = - E1000_READ_FLASH_REG16(hw, ICH_FLASH_HSFCTL); + if (hw->mac.type >= e1000_pch_spt) + hsflctl.regval = + E1000_READ_FLASH_REG(hw, + ICH_FLASH_HSFSTS)>>16; + else + hsflctl.regval = + E1000_READ_FLASH_REG16(hw, + ICH_FLASH_HSFCTL); hsflctl.hsf_ctrl.flcycle = ICH_CYCLE_ERASE; - E1000_WRITE_FLASH_REG16(hw, ICH_FLASH_HSFCTL, - hsflctl.regval); + if (hw->mac.type >= e1000_pch_spt) + E1000_WRITE_FLASH_REG(hw, ICH_FLASH_HSFSTS, + hsflctl.regval << 16); + else + E1000_WRITE_FLASH_REG16(hw, ICH_FLASH_HSFCTL, + hsflctl.regval); /* Write the last 24 bits of an index within the * block into Flash Linear address field in Flash @@ -4426,7 +5184,7 @@ STATIC void e1000_initialize_hw_bits_ich8lan(struct e1000_hw *hw) E1000_WRITE_REG(hw, E1000_RFCTL, reg); /* Enable ECC on Lynxpoint */ - if (hw->mac.type == e1000_pch_lpt) { + if (hw->mac.type >= e1000_pch_lpt) { reg = E1000_READ_REG(hw, E1000_PBECCSTS); reg |= E1000_PBECCSTS_ECC_ENABLE; E1000_WRITE_REG(hw, E1000_PBECCSTS, reg); @@ -4858,7 +5616,8 @@ void e1000_suspend_workarounds_ich8lan(struct e1000_hw *hw) if ((device_id == E1000_DEV_ID_PCH_LPTLP_I218_LM) || (device_id == E1000_DEV_ID_PCH_LPTLP_I218_V) || (device_id == E1000_DEV_ID_PCH_I218_LM3) || - (device_id == E1000_DEV_ID_PCH_I218_V3)) { + (device_id == E1000_DEV_ID_PCH_I218_V3) || + (hw->mac.type >= e1000_pch_spt)) { u32 fextnvm6 = E1000_READ_REG(hw, E1000_FEXTNVM6); E1000_WRITE_REG(hw, E1000_FEXTNVM6, diff --git a/dpdk/drivers/net/e1000/base/e1000_ich8lan.h b/dpdk/drivers/net/e1000/base/e1000_ich8lan.h index 33e77fb8..bc4ed1dd 100644 --- a/dpdk/drivers/net/e1000/base/e1000_ich8lan.h +++ b/dpdk/drivers/net/e1000/base/e1000_ich8lan.h @@ -121,6 +121,18 @@ POSSIBILITY OF SUCH DAMAGE. #if !defined(EXTERNAL_RELEASE) || defined(ULP_SUPPORT) #define E1000_FEXTNVM7_DISABLE_SMB_PERST 0x00000020 #endif /* !EXTERNAL_RELEASE || ULP_SUPPORT */ +#define E1000_FEXTNVM9_IOSFSB_CLKGATE_DIS 0x00000800 +#define E1000_FEXTNVM9_IOSFSB_CLKREQ_DIS 0x00001000 +#define E1000_FEXTNVM11_DISABLE_PB_READ 0x00000200 +#define E1000_FEXTNVM11_DISABLE_MULR_FIX 0x00002000 + +/* bit24: RXDCTL thresholds granularity: 0 - cache lines, 1 - descriptors */ +#define E1000_RXDCTL_THRESH_UNIT_DESC 0x01000000 + +#define NVM_SIZE_MULTIPLIER 4096 /*multiplier for NVMS field*/ +#define E1000_FLASH_BASE_ADDR 0xE000 /*offset of NVM access regs*/ +#define E1000_CTRL_EXT_NVMVS 0x3 /*NVM valid sector */ +#define E1000_TARC0_CB_MULTIQ_3_REQ (1 << 28 | 1 << 29) #define PCIE_ICH8_SNOOP_ALL PCIE_NO_SNOOP_ALL #define E1000_ICH_RAR_ENTRIES 7 @@ -198,6 +210,10 @@ POSSIBILITY OF SUCH DAMAGE. #define I218_ULP_CONFIG1_INBAND_EXIT 0x0020 /* Inband on ULP exit */ #define I218_ULP_CONFIG1_WOL_HOST 0x0040 /* WoL Host on ULP exit */ #define I218_ULP_CONFIG1_RESET_TO_SMBUS 0x0100 /* Reset to SMBus mode */ +/* enable ULP even if when phy powered down via lanphypc */ +#define I218_ULP_CONFIG1_EN_ULP_LANPHYPC 0x0400 +/* disable clear of sticky ULP on PERST */ +#define I218_ULP_CONFIG1_DIS_CLR_STICKY_ON_PERST 0x0800 #define I218_ULP_CONFIG1_DISABLE_SMB_PERST 0x1000 /* Disable on PERST# */ #endif /* !EXTERNAL_RELEASE || ULP_SUPPORT */ @@ -234,9 +250,12 @@ POSSIBILITY OF SUCH DAMAGE. /* PHY Power Management Control */ #define HV_PM_CTRL PHY_REG(770, 17) -#define HV_PM_CTRL_PLL_STOP_IN_K1_GIGA 0x100 +#define HV_PM_CTRL_K1_CLK_REQ 0x200 #define HV_PM_CTRL_K1_ENABLE 0x4000 +#define I217_PLL_CLOCK_GATE_REG PHY_REG(772, 28) +#define I217_PLL_CLOCK_GATE_MASK 0x07FF + #define SW_FLAG_TIMEOUT 1000 /* SW Semaphore flag timeout in ms */ /* Inband Control */ diff --git a/dpdk/drivers/net/e1000/base/e1000_mbx.c b/dpdk/drivers/net/e1000/base/e1000_mbx.c index 6daf16b0..a92fd22e 100644 --- a/dpdk/drivers/net/e1000/base/e1000_mbx.c +++ b/dpdk/drivers/net/e1000/base/e1000_mbx.c @@ -430,15 +430,21 @@ STATIC s32 e1000_check_for_rst_vf(struct e1000_hw *hw, STATIC s32 e1000_obtain_mbx_lock_vf(struct e1000_hw *hw) { s32 ret_val = -E1000_ERR_MBX; + int count = 10; DEBUGFUNC("e1000_obtain_mbx_lock_vf"); - /* Take ownership of the buffer */ - E1000_WRITE_REG(hw, E1000_V2PMAILBOX(0), E1000_V2PMAILBOX_VFU); + do { + /* Take ownership of the buffer */ + E1000_WRITE_REG(hw, E1000_V2PMAILBOX(0), E1000_V2PMAILBOX_VFU); - /* reserve mailbox for vf use */ - if (e1000_read_v2p_mailbox(hw) & E1000_V2PMAILBOX_VFU) - ret_val = E1000_SUCCESS; + /* reserve mailbox for vf use */ + if (e1000_read_v2p_mailbox(hw) & E1000_V2PMAILBOX_VFU) { + ret_val = E1000_SUCCESS; + break; + } + usec_delay(1000); + } while (count-- > 0); return ret_val; } @@ -645,18 +651,26 @@ STATIC s32 e1000_obtain_mbx_lock_pf(struct e1000_hw *hw, u16 vf_number) { s32 ret_val = -E1000_ERR_MBX; u32 p2v_mailbox; + int count = 10; DEBUGFUNC("e1000_obtain_mbx_lock_pf"); - /* Take ownership of the buffer */ - E1000_WRITE_REG(hw, E1000_P2VMAILBOX(vf_number), E1000_P2VMAILBOX_PFU); + do { + /* Take ownership of the buffer */ + E1000_WRITE_REG(hw, E1000_P2VMAILBOX(vf_number), + E1000_P2VMAILBOX_PFU); - /* reserve mailbox for vf use */ - p2v_mailbox = E1000_READ_REG(hw, E1000_P2VMAILBOX(vf_number)); - if (p2v_mailbox & E1000_P2VMAILBOX_PFU) - ret_val = E1000_SUCCESS; + /* reserve mailbox for pf use */ + p2v_mailbox = E1000_READ_REG(hw, E1000_P2VMAILBOX(vf_number)); + if (p2v_mailbox & E1000_P2VMAILBOX_PFU) { + ret_val = E1000_SUCCESS; + break; + } + usec_delay(1000); + } while (count-- > 0); return ret_val; + } /** diff --git a/dpdk/drivers/net/e1000/base/e1000_nvm.c b/dpdk/drivers/net/e1000/base/e1000_nvm.c index 762acd16..75c22827 100644 --- a/dpdk/drivers/net/e1000/base/e1000_nvm.c +++ b/dpdk/drivers/net/e1000/base/e1000_nvm.c @@ -1295,6 +1295,7 @@ void e1000_get_fw_version(struct e1000_hw *hw, struct e1000_fw_version *fw_vers) case e1000_82575: case e1000_82576: case e1000_82580: + case e1000_i354: hw->nvm.ops.read(hw, NVM_ETRACK_HIWORD, 1, &etrack_test); /* Use this format, unless EETRACK ID exists, * then use alternate format diff --git a/dpdk/drivers/net/e1000/base/e1000_osdep.h b/dpdk/drivers/net/e1000/base/e1000_osdep.h index 47a19481..b8868049 100644 --- a/dpdk/drivers/net/e1000/base/e1000_osdep.h +++ b/dpdk/drivers/net/e1000/base/e1000_osdep.h @@ -44,6 +44,7 @@ #include #include #include +#include #include "../e1000_logs.h" @@ -94,17 +95,18 @@ typedef int bool; #define E1000_WRITE_FLUSH(a) E1000_READ_REG(a, E1000_STATUS) -#define E1000_PCI_REG(reg) (*((volatile uint32_t *)(reg))) +#define E1000_PCI_REG(reg) rte_read32(reg) -#define E1000_PCI_REG16(reg) (*((volatile uint16_t *)(reg))) +#define E1000_PCI_REG16(reg) rte_read16(reg) -#define E1000_PCI_REG_WRITE(reg, value) do { \ - E1000_PCI_REG((reg)) = (rte_cpu_to_le_32(value)); \ -} while (0) +#define E1000_PCI_REG_WRITE(reg, value) \ + rte_write32((rte_cpu_to_le_32(value)), reg) -#define E1000_PCI_REG_WRITE16(reg, value) do { \ - E1000_PCI_REG16((reg)) = (rte_cpu_to_le_16(value)); \ -} while (0) +#define E1000_PCI_REG_WRITE_RELAXED(reg, value) \ + rte_write32_relaxed((rte_cpu_to_le_32(value)), reg) + +#define E1000_PCI_REG_WRITE16(reg, value) \ + rte_write16((rte_cpu_to_le_16(value)), reg) #define E1000_PCI_REG_ADDR(hw, reg) \ ((volatile uint32_t *)((char *)(hw)->hw_addr + (reg))) diff --git a/dpdk/drivers/net/e1000/base/e1000_regs.h b/dpdk/drivers/net/e1000/base/e1000_regs.h index 84531a99..364a7261 100644 --- a/dpdk/drivers/net/e1000/base/e1000_regs.h +++ b/dpdk/drivers/net/e1000/base/e1000_regs.h @@ -66,6 +66,8 @@ POSSIBILITY OF SUCH DAMAGE. #define E1000_FEXTNVM4 0x00024 /* Future Extended NVM 4 - RW */ #define E1000_FEXTNVM6 0x00010 /* Future Extended NVM 6 - RW */ #define E1000_FEXTNVM7 0x000E4 /* Future Extended NVM 7 - RW */ +#define E1000_FEXTNVM9 0x5BB4 /* Future Extended NVM 9 - RW */ +#define E1000_FEXTNVM11 0x5BBC /* Future Extended NVM 11 - RW */ #define E1000_PCIEANACFG 0x00F18 /* PCIE Analog Config */ #define E1000_FCT 0x00030 /* Flow Control Type - RW */ #define E1000_CONNSW 0x00034 /* Copper/Fiber switch control - RW */ @@ -109,6 +111,7 @@ POSSIBILITY OF SUCH DAMAGE. #define E1000_PBA 0x01000 /* Packet Buffer Allocation - RW */ #define E1000_PBS 0x01008 /* Packet Buffer Size */ #define E1000_PBECCSTS 0x0100C /* Packet Buffer ECC Status - RW */ +#define E1000_IOSFPC 0x00F28 /* TX corrupted data */ #define E1000_EEMNGCTL 0x01010 /* MNG EEprom Control */ #define E1000_EEMNGCTL_I210 0x01010 /* i210 MNG EEprom Mode Control */ #define E1000_EEARBC 0x01024 /* EEPROM Auto Read Bus Control */ @@ -591,6 +594,10 @@ POSSIBILITY OF SUCH DAMAGE. #define E1000_TIMADJL 0x0B60C /* Time sync time adjustment offset Low - RW */ #define E1000_TIMADJH 0x0B610 /* Time sync time adjustment offset High - RW */ #define E1000_TSAUXC 0x0B640 /* Timesync Auxiliary Control register */ +#define E1000_SYSSTMPL 0x0B648 /* HH Timesync system stamp low register */ +#define E1000_SYSSTMPH 0x0B64C /* HH Timesync system stamp hi register */ +#define E1000_PLTSTMPL 0x0B640 /* HH Timesync platform stamp low register */ +#define E1000_PLTSTMPH 0x0B644 /* HH Timesync platform stamp hi register */ #define E1000_SYSTIMR 0x0B6F8 /* System time register Residue */ #define E1000_TSICR 0x0B66C /* Interrupt Cause Register */ #define E1000_TSIM 0x0B674 /* Interrupt Mask Register */ diff --git a/dpdk/drivers/net/e1000/base/e1000_vf.c b/dpdk/drivers/net/e1000/base/e1000_vf.c index 7845b48e..44ab0188 100644 --- a/dpdk/drivers/net/e1000/base/e1000_vf.c +++ b/dpdk/drivers/net/e1000/base/e1000_vf.c @@ -421,12 +421,13 @@ void e1000_update_mc_addr_list_vf(struct e1000_hw *hw, DEBUGOUT1("MC Addr Count = %d\n", mc_addr_count); + msgbuf[0] = E1000_VF_SET_MULTICAST; + if (mc_addr_count > 30) { msgbuf[0] |= E1000_VF_SET_MULTICAST_OVERFLOW; mc_addr_count = 30; } - msgbuf[0] = E1000_VF_SET_MULTICAST; msgbuf[0] |= mc_addr_count << E1000_VT_MSGINFO_SHIFT; for (i = 0; i < mc_addr_count; i++) { diff --git a/dpdk/drivers/net/e1000/e1000_ethdev.h b/dpdk/drivers/net/e1000/e1000_ethdev.h index 6c25c8da..5668910c 100644 --- a/dpdk/drivers/net/e1000/e1000_ethdev.h +++ b/dpdk/drivers/net/e1000/e1000_ethdev.h @@ -34,6 +34,7 @@ #ifndef _E1000_ETHDEV_H_ #define _E1000_ETHDEV_H_ #include +#include #define E1000_INTEL_VENDOR_ID 0x8086 @@ -82,7 +83,7 @@ #define E1000_MAX_FLEX_FILTER_DWDS \ (E1000_MAX_FLEX_FILTER_LEN / sizeof(uint32_t)) #define E1000_FLEX_FILTERS_MASK_SIZE \ - (E1000_MAX_FLEX_FILTER_DWDS / 4) + (E1000_MAX_FLEX_FILTER_DWDS / 2) #define E1000_FHFT_QUEUEING_LEN 0x0000007F #define E1000_FHFT_QUEUEING_QUEUE 0x00000700 #define E1000_FHFT_QUEUEING_PRIO 0x00070000 @@ -138,6 +139,24 @@ #define E1000_MISC_VEC_ID RTE_INTR_VEC_ZERO_OFFSET #define E1000_RX_VEC_START RTE_INTR_VEC_RXTX_OFFSET +#define IGB_TX_MAX_SEG UINT8_MAX +#define IGB_TX_MAX_MTU_SEG UINT8_MAX +#define EM_TX_MAX_SEG UINT8_MAX +#define EM_TX_MAX_MTU_SEG UINT8_MAX + +#define MAC_TYPE_FILTER_SUP(type) do {\ + if ((type) != e1000_82580 && (type) != e1000_i350 &&\ + (type) != e1000_82576 && (type) != e1000_i210 &&\ + (type) != e1000_i211)\ + return -ENOTSUP;\ +} while (0) + +#define MAC_TYPE_FILTER_SUP_EXT(type) do {\ + if ((type) != e1000_82580 && (type) != e1000_i350 &&\ + (type) != e1000_i210 && (type) != e1000_i211)\ + return -ENOTSUP; \ +} while (0) + /* structure for interrupt relative data */ struct e1000_interrupt { uint32_t flags; @@ -232,13 +251,19 @@ struct e1000_2tuple_filter { uint16_t queue; /* rx queue assigned to */ }; +/* ethertype filter structure */ +struct igb_ethertype_filter { + uint16_t ethertype; + uint32_t etqf; +}; + /* - * Structure to store filters' info. + * Structure to store filters'info. */ struct e1000_filter_info { uint8_t ethertype_mask; /* Bit mask for every used ethertype filter */ /* store used ethertype filters*/ - uint16_t ethertype_filters[E1000_MAX_ETQF_FILTERS]; + struct igb_ethertype_filter ethertype_filters[E1000_MAX_ETQF_FILTERS]; uint8_t flex_mask; /* Bit mask for every used flex filter */ struct e1000_flex_filter_list flex_list; /* Bit mask for every used 5tuple filter */ @@ -247,6 +272,8 @@ struct e1000_filter_info { /* Bit mask for every used 2tuple filter */ uint8_t twotuple_mask; struct e1000_2tuple_filter_list twotuple_list; + /* store the SYN filter info */ + uint32_t syn_info; }; /* @@ -286,6 +313,55 @@ struct e1000_adapter { #define E1000_DEV_PRIVATE_TO_FILTER_INFO(adapter) \ (&((struct e1000_adapter *)adapter)->filter) +struct rte_flow { + enum rte_filter_type filter_type; + void *rule; +}; + +/* ntuple filter list structure */ +struct igb_ntuple_filter_ele { + TAILQ_ENTRY(igb_ntuple_filter_ele) entries; + struct rte_eth_ntuple_filter filter_info; +}; + +/* ethertype filter list structure */ +struct igb_ethertype_filter_ele { + TAILQ_ENTRY(igb_ethertype_filter_ele) entries; + struct rte_eth_ethertype_filter filter_info; +}; + +/* syn filter list structure */ +struct igb_eth_syn_filter_ele { + TAILQ_ENTRY(igb_eth_syn_filter_ele) entries; + struct rte_eth_syn_filter filter_info; +}; + +/* flex filter list structure */ +struct igb_flex_filter_ele { + TAILQ_ENTRY(igb_flex_filter_ele) entries; + struct rte_eth_flex_filter filter_info; +}; + +/* igb_flow memory list structure */ +struct igb_flow_mem { + TAILQ_ENTRY(igb_flow_mem) entries; + struct rte_flow *flow; + struct rte_eth_dev *dev; +}; + +TAILQ_HEAD(igb_ntuple_filter_list, igb_ntuple_filter_ele); +struct igb_ntuple_filter_list igb_filter_ntuple_list; +TAILQ_HEAD(igb_ethertype_filter_list, igb_ethertype_filter_ele); +struct igb_ethertype_filter_list igb_filter_ethertype_list; +TAILQ_HEAD(igb_syn_filter_list, igb_eth_syn_filter_ele); +struct igb_syn_filter_list igb_filter_syn_list; +TAILQ_HEAD(igb_flex_filter_list, igb_flex_filter_ele); +struct igb_flex_filter_list igb_filter_flex_list; +TAILQ_HEAD(igb_flow_mem_list, igb_flow_mem); +struct igb_flow_mem_list igb_flow_list; + +extern const struct rte_flow_ops igb_flow_ops; + /* * RX/TX IGB function prototypes */ @@ -304,10 +380,15 @@ uint32_t eth_igb_rx_queue_count(struct rte_eth_dev *dev, int eth_igb_rx_descriptor_done(void *rx_queue, uint16_t offset); +int eth_igb_rx_descriptor_status(void *rx_queue, uint16_t offset); +int eth_igb_tx_descriptor_status(void *tx_queue, uint16_t offset); + int eth_igb_tx_queue_setup(struct rte_eth_dev *dev, uint16_t tx_queue_id, uint16_t nb_tx_desc, unsigned int socket_id, const struct rte_eth_txconf *tx_conf); +int eth_igb_tx_done_cleanup(void *txq, uint32_t free_cnt); + int eth_igb_rx_init(struct rte_eth_dev *dev); void eth_igb_tx_init(struct rte_eth_dev *dev); @@ -315,6 +396,9 @@ void eth_igb_tx_init(struct rte_eth_dev *dev); uint16_t eth_igb_xmit_pkts(void *txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts); +uint16_t eth_igb_prep_pkts(void *txq, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts); + uint16_t eth_igb_recv_pkts(void *rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts); @@ -365,6 +449,9 @@ uint32_t eth_em_rx_queue_count(struct rte_eth_dev *dev, int eth_em_rx_descriptor_done(void *rx_queue, uint16_t offset); +int eth_em_rx_descriptor_status(void *rx_queue, uint16_t offset); +int eth_em_tx_descriptor_status(void *tx_queue, uint16_t offset); + int eth_em_tx_queue_setup(struct rte_eth_dev *dev, uint16_t tx_queue_id, uint16_t nb_tx_desc, unsigned int socket_id, const struct rte_eth_txconf *tx_conf); @@ -376,6 +463,9 @@ void eth_em_tx_init(struct rte_eth_dev *dev); uint16_t eth_em_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts); +uint16_t eth_em_prep_pkts(void *txq, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts); + uint16_t eth_em_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts); @@ -390,4 +480,24 @@ void em_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, void igb_pf_host_uninit(struct rte_eth_dev *dev); +void igb_filterlist_flush(struct rte_eth_dev *dev); +int igb_delete_5tuple_filter_82576(struct rte_eth_dev *dev, + struct e1000_5tuple_filter *filter); +int igb_delete_2tuple_filter(struct rte_eth_dev *dev, + struct e1000_2tuple_filter *filter); +void igb_remove_flex_filter(struct rte_eth_dev *dev, + struct e1000_flex_filter *filter); +int igb_ethertype_filter_remove(struct e1000_filter_info *filter_info, + uint8_t idx); +int igb_add_del_ntuple_filter(struct rte_eth_dev *dev, + struct rte_eth_ntuple_filter *ntuple_filter, bool add); +int igb_add_del_ethertype_filter(struct rte_eth_dev *dev, + struct rte_eth_ethertype_filter *filter, + bool add); +int eth_igb_syn_filter_set(struct rte_eth_dev *dev, + struct rte_eth_syn_filter *filter, + bool add); +int eth_igb_add_del_flex_filter(struct rte_eth_dev *dev, + struct rte_eth_flex_filter *filter, + bool add); #endif /* _E1000_ETHDEV_H_ */ diff --git a/dpdk/drivers/net/e1000/em_ethdev.c b/dpdk/drivers/net/e1000/em_ethdev.c index ad104ed7..29dac6e3 100644 --- a/dpdk/drivers/net/e1000/em_ethdev.c +++ b/dpdk/drivers/net/e1000/em_ethdev.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -43,10 +43,11 @@ #include #include #include +#include #include #include +#include #include -#include #include #include #include @@ -71,7 +72,7 @@ static void eth_em_allmulticast_enable(struct rte_eth_dev *dev); static void eth_em_allmulticast_disable(struct rte_eth_dev *dev); static int eth_em_link_update(struct rte_eth_dev *dev, int wait_to_complete); -static void eth_em_stats_get(struct rte_eth_dev *dev, +static int eth_em_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats); static void eth_em_stats_reset(struct rte_eth_dev *dev); static void eth_em_infos_get(struct rte_eth_dev *dev, @@ -83,9 +84,9 @@ static int eth_em_flow_ctrl_set(struct rte_eth_dev *dev, static int eth_em_interrupt_setup(struct rte_eth_dev *dev); static int eth_em_rxq_interrupt_setup(struct rte_eth_dev *dev); static int eth_em_interrupt_get_status(struct rte_eth_dev *dev); -static int eth_em_interrupt_action(struct rte_eth_dev *dev); -static void eth_em_interrupt_handler(struct rte_intr_handle *handle, - void *param); +static int eth_em_interrupt_action(struct rte_eth_dev *dev, + struct rte_intr_handle *handle); +static void eth_em_interrupt_handler(void *param); static int em_hw_init(struct e1000_hw *hw); static int em_hardware_init(struct e1000_hw *hw); @@ -98,7 +99,7 @@ static int eth_em_mtu_set(struct rte_eth_dev *dev, uint16_t mtu); static int eth_em_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on); -static void eth_em_vlan_offload_set(struct rte_eth_dev *dev, int mask); +static int eth_em_vlan_offload_set(struct rte_eth_dev *dev, int mask); static void em_vlan_hw_filter_enable(struct rte_eth_dev *dev); static void em_vlan_hw_filter_disable(struct rte_eth_dev *dev); static void em_vlan_hw_strip_enable(struct rte_eth_dev *dev); @@ -119,8 +120,8 @@ static int eth_em_led_on(struct rte_eth_dev *dev); static int eth_em_led_off(struct rte_eth_dev *dev); static int em_get_rx_buffer_size(struct e1000_hw *hw); -static void eth_em_rar_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr, - uint32_t index, uint32_t pool); +static int eth_em_rar_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr, + uint32_t index, uint32_t pool); static void eth_em_rar_clear(struct rte_eth_dev *dev, uint32_t index); static int eth_em_set_mc_addr_list(struct rte_eth_dev *dev, @@ -168,6 +169,19 @@ static const struct rte_pci_id pci_id_em_map[] = { { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_PCH_I218_V2) }, { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_PCH_I218_LM3) }, { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_PCH_I218_V3) }, + { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_PCH_SPT_I219_LM) }, + { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_PCH_SPT_I219_V) }, + { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_PCH_SPT_I219_LM2) }, + { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_PCH_SPT_I219_V2) }, + { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_PCH_LBG_I219_LM3) }, + { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_PCH_SPT_I219_LM4) }, + { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_PCH_SPT_I219_V4) }, + { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_PCH_SPT_I219_LM5) }, + { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_PCH_SPT_I219_V5) }, + { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_PCH_CNP_I219_LM6) }, + { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_PCH_CNP_I219_V6) }, + { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_PCH_CNP_I219_LM7) }, + { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_PCH_CNP_I219_V7) }, { .vendor_id = 0, /* sentinel */ }, }; @@ -191,6 +205,8 @@ static const struct eth_dev_ops eth_em_ops = { .rx_queue_release = eth_em_rx_queue_release, .rx_queue_count = eth_em_rx_queue_count, .rx_descriptor_done = eth_em_rx_descriptor_done, + .rx_descriptor_status = eth_em_rx_descriptor_status, + .tx_descriptor_status = eth_em_tx_descriptor_status, .tx_queue_setup = eth_em_tx_queue_setup, .tx_queue_release = eth_em_tx_queue_release, .rx_queue_intr_enable = eth_em_rx_queue_intr_enable, @@ -278,6 +294,19 @@ eth_em_dev_is_ich8(struct e1000_hw *hw) case E1000_DEV_ID_PCH_I218_LM2: case E1000_DEV_ID_PCH_I218_V3: case E1000_DEV_ID_PCH_I218_LM3: + case E1000_DEV_ID_PCH_SPT_I219_LM: + case E1000_DEV_ID_PCH_SPT_I219_V: + case E1000_DEV_ID_PCH_SPT_I219_LM2: + case E1000_DEV_ID_PCH_SPT_I219_V2: + case E1000_DEV_ID_PCH_LBG_I219_LM3: + case E1000_DEV_ID_PCH_SPT_I219_LM4: + case E1000_DEV_ID_PCH_SPT_I219_V4: + case E1000_DEV_ID_PCH_SPT_I219_LM5: + case E1000_DEV_ID_PCH_SPT_I219_V5: + case E1000_DEV_ID_PCH_CNP_I219_LM6: + case E1000_DEV_ID_PCH_CNP_I219_V6: + case E1000_DEV_ID_PCH_CNP_I219_LM7: + case E1000_DEV_ID_PCH_CNP_I219_V7: return 1; default: return 0; @@ -287,7 +316,8 @@ eth_em_dev_is_ich8(struct e1000_hw *hw) static int eth_em_dev_init(struct rte_eth_dev *eth_dev) { - struct rte_pci_device *pci_dev; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); + struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; struct e1000_adapter *adapter = E1000_DEV_PRIVATE(eth_dev->data->dev_private); struct e1000_hw *hw = @@ -295,11 +325,10 @@ eth_em_dev_init(struct rte_eth_dev *eth_dev) struct e1000_vfta * shadow_vfta = E1000_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private); - pci_dev = eth_dev->pci_dev; - eth_dev->dev_ops = ð_em_ops; eth_dev->rx_pkt_burst = (eth_rx_burst_t)ð_em_recv_pkts; eth_dev->tx_pkt_burst = (eth_tx_burst_t)ð_em_xmit_pkts; + eth_dev->tx_pkt_prepare = (eth_tx_prep_t)ð_em_prep_pkts; /* for secondary processes, we don't initialise any further as primary * has already done this work. Only check we don't need a different @@ -351,8 +380,8 @@ eth_em_dev_init(struct rte_eth_dev *eth_dev) eth_dev->data->port_id, pci_dev->id.vendor_id, pci_dev->id.device_id); - rte_intr_callback_register(&(pci_dev->intr_handle), - eth_em_interrupt_handler, (void *)eth_dev); + rte_intr_callback_register(intr_handle, + eth_em_interrupt_handler, eth_dev); return 0; } @@ -360,17 +389,16 @@ eth_em_dev_init(struct rte_eth_dev *eth_dev) static int eth_em_dev_uninit(struct rte_eth_dev *eth_dev) { - struct rte_pci_device *pci_dev; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); struct e1000_adapter *adapter = E1000_DEV_PRIVATE(eth_dev->data->dev_private); + struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; PMD_INIT_FUNC_TRACE(); if (rte_eal_process_type() != RTE_PROC_PRIMARY) return -EPERM; - pci_dev = eth_dev->pci_dev; - if (adapter->stopped == 0) eth_em_close(eth_dev); @@ -382,32 +410,33 @@ eth_em_dev_uninit(struct rte_eth_dev *eth_dev) eth_dev->data->mac_addrs = NULL; /* disable uio intr before callback unregister */ - rte_intr_disable(&(pci_dev->intr_handle)); - rte_intr_callback_unregister(&(pci_dev->intr_handle), - eth_em_interrupt_handler, (void *)eth_dev); + rte_intr_disable(intr_handle); + rte_intr_callback_unregister(intr_handle, + eth_em_interrupt_handler, eth_dev); return 0; } -static struct eth_driver rte_em_pmd = { - .pci_drv = { - .name = "rte_em_pmd", - .id_table = pci_id_em_map, - .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC | - RTE_PCI_DRV_DETACHABLE, - }, - .eth_dev_init = eth_em_dev_init, - .eth_dev_uninit = eth_em_dev_uninit, - .dev_private_size = sizeof(struct e1000_adapter), -}; - -static int -rte_em_pmd_init(const char *name __rte_unused, const char *params __rte_unused) +static int eth_em_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, + struct rte_pci_device *pci_dev) { - rte_eth_driver_register(&rte_em_pmd); - return 0; + return rte_eth_dev_pci_generic_probe(pci_dev, + sizeof(struct e1000_adapter), eth_em_dev_init); } +static int eth_em_pci_remove(struct rte_pci_device *pci_dev) +{ + return rte_eth_dev_pci_generic_remove(pci_dev, eth_em_dev_uninit); +} + +static struct rte_pci_driver rte_em_pmd = { + .id_table = pci_id_em_map, + .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC | + RTE_PCI_DRV_IOVA_AS_VA, + .probe = eth_em_pci_probe, + .remove = eth_em_pci_remove, +}; + static int em_hw_init(struct e1000_hw *hw) { @@ -546,6 +575,8 @@ em_set_pba(struct e1000_hw *hw) case e1000_pchlan: case e1000_pch2lan: case e1000_pch_lpt: + case e1000_pch_spt: + case e1000_pch_cnp: pba = E1000_PBA_26K; break; default: @@ -562,7 +593,8 @@ eth_em_start(struct rte_eth_dev *dev) E1000_DEV_PRIVATE(dev->data->dev_private); struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); - struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; int ret, mask; uint32_t intr_vector = 0; uint32_t *speeds; @@ -615,7 +647,7 @@ eth_em_start(struct rte_eth_dev *dev) dev->data->nb_rx_queues * sizeof(int), 0); if (intr_handle->intr_vec == NULL) { PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues" - " intr_vec\n", dev->data->nb_rx_queues); + " intr_vec", dev->data->nb_rx_queues); return -ENOMEM; } @@ -636,7 +668,12 @@ eth_em_start(struct rte_eth_dev *dev) mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \ ETH_VLAN_EXTEND_MASK; - eth_em_vlan_offload_set(dev, mask); + ret = eth_em_vlan_offload_set(dev, mask); + if (ret) { + PMD_INIT_LOG(ERR, "Unable to update vlan offload"); + em_dev_clear_queues(dev); + return ret; + } /* Set Interrupt Throttling Rate to maximum allowed value. */ E1000_WRITE_REG(hw, E1000_ITR, UINT16_MAX); @@ -645,6 +682,7 @@ eth_em_start(struct rte_eth_dev *dev) speeds = &dev->data->dev_conf.link_speeds; if (*speeds == ETH_LINK_SPEED_AUTONEG) { hw->phy.autoneg_advertised = E1000_ALL_SPEED_DUPLEX; + hw->mac.autoneg = 1; } else { num_speeds = 0; autoneg = (*speeds & ETH_LINK_SPEED_FIXED) == 0; @@ -680,6 +718,17 @@ eth_em_start(struct rte_eth_dev *dev) } if (num_speeds == 0 || (!autoneg && (num_speeds > 1))) goto error_invalid_config; + + /* Set/reset the mac.autoneg based on the link speed, + * fixed or not + */ + if (!autoneg) { + hw->mac.autoneg = 0; + hw->mac.forced_speed_duplex = + hw->phy.autoneg_advertised; + } else { + hw->mac.autoneg = 1; + } } e1000_setup_link(hw); @@ -700,7 +749,7 @@ eth_em_start(struct rte_eth_dev *dev) (void *)dev); if (dev->data->dev_conf.intr_conf.lsc != 0) PMD_INIT_LOG(INFO, "lsc won't enable because of" - " no intr multiplex\n"); + " no intr multiplexn"); } /* check if rxq interrupt is enabled */ if (dev->data->dev_conf.intr_conf.rxq != 0) @@ -732,7 +781,8 @@ eth_em_stop(struct rte_eth_dev *dev) { struct rte_eth_link link; struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); - struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; em_rxq_intr_disable(hw); em_lsc_intr_disable(hw); @@ -847,7 +897,9 @@ em_hardware_init(struct e1000_hw *hw) hw->fc.low_water = 0x5048; hw->fc.pause_time = 0x0650; hw->fc.refresh_time = 0x0400; - } else if (hw->mac.type == e1000_pch_lpt) { + } else if (hw->mac.type == e1000_pch_lpt || + hw->mac.type == e1000_pch_spt || + hw->mac.type == e1000_pch_cnp) { hw->fc.requested_mode = e1000_fc_full; } @@ -859,7 +911,7 @@ em_hardware_init(struct e1000_hw *hw) } /* This function is based on em_update_stats_counters() in e1000/if_em.c */ -static void +static int eth_em_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats) { struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); @@ -959,7 +1011,7 @@ eth_em_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats) } if (rte_stats == NULL) - return; + return -EINVAL; /* Rx Errors */ rte_stats->imissed = stats->mpc; @@ -974,6 +1026,7 @@ eth_em_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats) rte_stats->opackets = stats->gptc; rte_stats->ibytes = stats->gorc; rte_stats->obytes = stats->gotc; + return 0; } static void @@ -993,9 +1046,11 @@ static int eth_em_rx_queue_intr_enable(struct rte_eth_dev *dev, __rte_unused uint16_t queue_id) { struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; em_rxq_intr_enable(hw); - rte_intr_enable(&dev->pci_dev->intr_handle); + rte_intr_enable(intr_handle); return 0; } @@ -1020,6 +1075,8 @@ em_get_max_pktlen(const struct e1000_hw *hw) case e1000_ich10lan: case e1000_pch2lan: case e1000_pch_lpt: + case e1000_pch_spt: + case e1000_pch_cnp: case e1000_82574: case e1000_80003es2lan: /* 9K Jumbo Frame size */ case e1000_82583: @@ -1039,9 +1096,20 @@ eth_em_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) { struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + dev_info->pci_dev = RTE_ETH_DEV_TO_PCI(dev); dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */ dev_info->max_rx_pktlen = em_get_max_pktlen(hw); dev_info->max_mac_addrs = hw->mac.rar_entry_count; + dev_info->rx_offload_capa = + DEV_RX_OFFLOAD_VLAN_STRIP | + DEV_RX_OFFLOAD_IPV4_CKSUM | + DEV_RX_OFFLOAD_UDP_CKSUM | + DEV_RX_OFFLOAD_TCP_CKSUM; + dev_info->tx_offload_capa = + DEV_TX_OFFLOAD_VLAN_INSERT | + DEV_TX_OFFLOAD_IPV4_CKSUM | + DEV_TX_OFFLOAD_UDP_CKSUM | + DEV_TX_OFFLOAD_TCP_CKSUM; /* * Starting with 631xESB hw supports 2 TX/RX queues per port. @@ -1073,6 +1141,8 @@ eth_em_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) .nb_max = E1000_MAX_RING_DESC, .nb_min = E1000_MIN_RING_DESC, .nb_align = EM_TXD_ALIGN, + .nb_seg_max = EM_TX_MAX_SEG, + .nb_mtu_seg_max = EM_TX_MAX_MTU_SEG, }; dev_info->speed_capa = ETH_LINK_SPEED_10M_HD | ETH_LINK_SPEED_10M | @@ -1139,7 +1209,7 @@ eth_em_link_update(struct rte_eth_dev *dev, int wait_to_complete) link.link_speed = 0; link.link_duplex = ETH_LINK_HALF_DUPLEX; link.link_status = ETH_LINK_DOWN; - link.link_autoneg = ETH_LINK_SPEED_FIXED; + link.link_autoneg = ETH_LINK_FIXED; } rte_em_dev_atomic_write_link_status(dev, &link); @@ -1383,7 +1453,7 @@ em_vlan_hw_strip_enable(struct rte_eth_dev *dev) E1000_WRITE_REG(hw, E1000_CTRL, reg); } -static void +static int eth_em_vlan_offload_set(struct rte_eth_dev *dev, int mask) { if(mask & ETH_VLAN_STRIP_MASK){ @@ -1399,6 +1469,8 @@ eth_em_vlan_offload_set(struct rte_eth_dev *dev, int mask) else em_vlan_hw_filter_disable(dev); } + + return 0; } /* @@ -1530,8 +1602,10 @@ eth_em_interrupt_get_status(struct rte_eth_dev *dev) * - On failure, a negative value. */ static int -eth_em_interrupt_action(struct rte_eth_dev *dev) +eth_em_interrupt_action(struct rte_eth_dev *dev, + struct rte_intr_handle *intr_handle) { + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct e1000_interrupt *intr = @@ -1544,7 +1618,7 @@ eth_em_interrupt_action(struct rte_eth_dev *dev) return -1; intr->flags &= ~E1000_FLAG_NEED_LINK_UPDATE; - rte_intr_enable(&(dev->pci_dev->intr_handle)); + rte_intr_enable(intr_handle); /* set get_link_status to check register later */ hw->mac.get_link_status = 1; @@ -1558,15 +1632,15 @@ eth_em_interrupt_action(struct rte_eth_dev *dev) rte_em_dev_atomic_read_link_status(dev, &link); if (link.link_status) { PMD_INIT_LOG(INFO, " Port %d: Link Up - speed %u Mbps - %s", - dev->data->port_id, (unsigned)link.link_speed, + dev->data->port_id, link.link_speed, link.link_duplex == ETH_LINK_FULL_DUPLEX ? "full-duplex" : "half-duplex"); } else { PMD_INIT_LOG(INFO, " Port %d: Link Down", dev->data->port_id); } PMD_INIT_LOG(DEBUG, "PCI Address: %04d:%02d:%02d:%d", - dev->pci_dev->addr.domain, dev->pci_dev->addr.bus, - dev->pci_dev->addr.devid, dev->pci_dev->addr.function); + pci_dev->addr.domain, pci_dev->addr.bus, + pci_dev->addr.devid, pci_dev->addr.function); tctl = E1000_READ_REG(hw, E1000_TCTL); rctl = E1000_READ_REG(hw, E1000_RCTL); @@ -1598,14 +1672,13 @@ eth_em_interrupt_action(struct rte_eth_dev *dev) * void */ static void -eth_em_interrupt_handler(__rte_unused struct rte_intr_handle *handle, - void *param) +eth_em_interrupt_handler(void *param) { struct rte_eth_dev *dev = (struct rte_eth_dev *)param; eth_em_interrupt_get_status(dev); - eth_em_interrupt_action(dev); - _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC); + eth_em_interrupt_action(dev, dev->intr_handle); + _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL, NULL); } static int @@ -1728,13 +1801,13 @@ eth_em_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) return -EIO; } -static void +static int eth_em_rar_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr, uint32_t index, __rte_unused uint32_t pool) { struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); - e1000_rar_set(hw, mac_addr->addr_bytes, index); + return e1000_rar_set(hw, mac_addr->addr_bytes, index); } static void @@ -1799,10 +1872,6 @@ eth_em_set_mc_addr_list(struct rte_eth_dev *dev, return 0; } -struct rte_driver em_pmd_drv = { - .type = PMD_PDEV, - .init = rte_em_pmd_init, -}; - -PMD_REGISTER_DRIVER(em_pmd_drv, em); -DRIVER_REGISTER_PCI_TABLE(em, pci_id_em_map); +RTE_PMD_REGISTER_PCI(net_e1000_em, rte_em_pmd); +RTE_PMD_REGISTER_PCI_TABLE(net_e1000_em, pci_id_em_map); +RTE_PMD_REGISTER_KMOD_DEP(net_e1000_em, "* igb_uio | uio_pci_generic | vfio-pci"); diff --git a/dpdk/drivers/net/e1000/em_rxtx.c b/dpdk/drivers/net/e1000/em_rxtx.c index 6d8750a8..1d8f0794 100644 --- a/dpdk/drivers/net/e1000/em_rxtx.c +++ b/dpdk/drivers/net/e1000/em_rxtx.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -56,7 +56,6 @@ #include #include #include -#include #include #include #include @@ -67,6 +66,7 @@ #include #include #include +#include #include #include "e1000_logs.h" @@ -78,6 +78,14 @@ #define E1000_RXDCTL_GRAN 0x01000000 /* RXDCTL Granularity */ +#define E1000_TX_OFFLOAD_MASK ( \ + PKT_TX_IP_CKSUM | \ + PKT_TX_L4_MASK | \ + PKT_TX_VLAN_PKT) + +#define E1000_TX_OFFLOAD_NOTSUP_MASK \ + (PKT_TX_OFFLOAD_MASK ^ E1000_TX_OFFLOAD_MASK) + /** * Structure associated with each descriptor of the RX ring of a RX queue. */ @@ -111,7 +119,7 @@ struct em_rx_queue { uint16_t nb_rx_hold; /**< number of held free RX desc. */ uint16_t rx_free_thresh; /**< max free RX desc to hold. */ uint16_t queue_id; /**< RX queue index. */ - uint8_t port_id; /**< Device port identifier. */ + uint16_t port_id; /**< Device port identifier. */ uint8_t pthresh; /**< Prefetch threshold register. */ uint8_t hthresh; /**< Host threshold register. */ uint8_t wthresh; /**< Write-back threshold register. */ @@ -178,7 +186,7 @@ struct em_tx_queue { /** Total number of TX descriptors ready to be allocated. */ uint16_t nb_tx_free; uint16_t queue_id; /**< TX queue index. */ - uint8_t port_id; /**< Device port identifier. */ + uint16_t port_id; /**< Device port identifier. */ uint8_t pthresh; /**< Prefetch threshold register. */ uint8_t hthresh; /**< Host threshold register. */ uint8_t wthresh; /**< Write-back threshold register. */ @@ -569,7 +577,7 @@ eth_em_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, * Set up Transmit Data Descriptor. */ slen = m_seg->data_len; - buf_dma_addr = rte_mbuf_data_dma_addr(m_seg); + buf_dma_addr = rte_mbuf_data_iova(m_seg); txd->buffer_addr = rte_cpu_to_le_64(buf_dma_addr); txd->lower.data = rte_cpu_to_le_32(cmd_type_len | slen); @@ -611,12 +619,49 @@ end_of_tx: PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_tx=%u", (unsigned) txq->port_id, (unsigned) txq->queue_id, (unsigned) tx_id, (unsigned) nb_tx); - E1000_PCI_REG_WRITE(txq->tdt_reg_addr, tx_id); + E1000_PCI_REG_WRITE_RELAXED(txq->tdt_reg_addr, tx_id); txq->tx_tail = tx_id; return nb_tx; } +/********************************************************************* + * + * TX prep functions + * + **********************************************************************/ +uint16_t +eth_em_prep_pkts(__rte_unused void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts) +{ + int i, ret; + struct rte_mbuf *m; + + for (i = 0; i < nb_pkts; i++) { + m = tx_pkts[i]; + + if (m->ol_flags & E1000_TX_OFFLOAD_NOTSUP_MASK) { + rte_errno = -ENOTSUP; + return i; + } + +#ifdef RTE_LIBRTE_ETHDEV_DEBUG + ret = rte_validate_tx_offload(m); + if (ret != 0) { + rte_errno = ret; + return i; + } +#endif + ret = rte_net_intel_cksum_prepare(m); + if (ret != 0) { + rte_errno = ret; + return i; + } + } + + return i; +} + /********************************************************************* * * RX functions @@ -630,7 +675,7 @@ rx_desc_status_to_pkt_flags(uint32_t rx_status) /* Check if VLAN present */ pkt_flags = ((rx_status & E1000_RXD_STAT_VP) ? - PKT_RX_VLAN_PKT | PKT_RX_VLAN_STRIPPED : 0); + PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED : 0); return pkt_flags; } @@ -754,7 +799,7 @@ eth_em_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, rxm = rxe->mbuf; rxe->mbuf = nmb; dma_addr = - rte_cpu_to_le_64(rte_mbuf_data_dma_addr_default(nmb)); + rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb)); rxdp->buffer_addr = dma_addr; rxdp->status = 0; @@ -785,7 +830,7 @@ eth_em_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, rxm->ol_flags = rxm->ol_flags | rx_desc_error_to_pkt_flags(rxd.errors); - /* Only valid if PKT_RX_VLAN_PKT set in pkt_flags */ + /* Only valid if PKT_RX_VLAN set in pkt_flags */ rxm->vlan_tci = rte_le_to_cpu_16(rxd.special); /* @@ -934,7 +979,7 @@ eth_em_recv_scattered_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, */ rxm = rxe->mbuf; rxe->mbuf = nmb; - dma = rte_cpu_to_le_64(rte_mbuf_data_dma_addr_default(nmb)); + dma = rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb)); rxdp->buffer_addr = dma; rxdp->status = 0; @@ -1011,7 +1056,7 @@ eth_em_recv_scattered_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, first_seg->ol_flags = first_seg->ol_flags | rx_desc_error_to_pkt_flags(rxd.errors); - /* Only valid if PKT_RX_VLAN_PKT set in pkt_flags */ + /* Only valid if PKT_RX_VLAN set in pkt_flags */ rxm->vlan_tci = rte_le_to_cpu_16(rxd.special); /* Prefetch data of first segment, if configured to do so. */ @@ -1244,7 +1289,7 @@ eth_em_tx_queue_setup(struct rte_eth_dev *dev, txq->port_id = dev->data->port_id; txq->tdt_reg_addr = E1000_PCI_REG_ADDR(hw, E1000_TDT(queue_idx)); - txq->tx_ring_phys_addr = rte_mem_phy2mch(tz->memseg_id, tz->phys_addr); + txq->tx_ring_phys_addr = tz->iova; txq->tx_ring = (struct e1000_data_desc *) tz->addr; PMD_INIT_LOG(DEBUG, "sw_ring=%p hw_ring=%p dma_addr=0x%"PRIx64, @@ -1371,7 +1416,7 @@ eth_em_rx_queue_setup(struct rte_eth_dev *dev, rxq->rdt_reg_addr = E1000_PCI_REG_ADDR(hw, E1000_RDT(queue_idx)); rxq->rdh_reg_addr = E1000_PCI_REG_ADDR(hw, E1000_RDH(queue_idx)); - rxq->rx_ring_phys_addr = rte_mem_phy2mch(rz->memseg_id, rz->phys_addr); + rxq->rx_ring_phys_addr = rz->iova; rxq->rx_ring = (struct e1000_rx_desc *) rz->addr; PMD_INIT_LOG(DEBUG, "sw_ring=%p hw_ring=%p dma_addr=0x%"PRIx64, @@ -1391,11 +1436,6 @@ eth_em_rx_queue_count(struct rte_eth_dev *dev, uint16_t rx_queue_id) struct em_rx_queue *rxq; uint32_t desc = 0; - if (rx_queue_id >= dev->data->nb_rx_queues) { - PMD_RX_LOG(DEBUG, "Invalid RX queue_id=%d", rx_queue_id); - return 0; - } - rxq = dev->data->rx_queues[rx_queue_id]; rxdp = &(rxq->rx_ring[rxq->rx_tail]); @@ -1428,6 +1468,57 @@ eth_em_rx_descriptor_done(void *rx_queue, uint16_t offset) return !!(rxdp->status & E1000_RXD_STAT_DD); } +int +eth_em_rx_descriptor_status(void *rx_queue, uint16_t offset) +{ + struct em_rx_queue *rxq = rx_queue; + volatile uint8_t *status; + uint32_t desc; + + if (unlikely(offset >= rxq->nb_rx_desc)) + return -EINVAL; + + if (offset >= rxq->nb_rx_desc - rxq->nb_rx_hold) + return RTE_ETH_RX_DESC_UNAVAIL; + + desc = rxq->rx_tail + offset; + if (desc >= rxq->nb_rx_desc) + desc -= rxq->nb_rx_desc; + + status = &rxq->rx_ring[desc].status; + if (*status & E1000_RXD_STAT_DD) + return RTE_ETH_RX_DESC_DONE; + + return RTE_ETH_RX_DESC_AVAIL; +} + +int +eth_em_tx_descriptor_status(void *tx_queue, uint16_t offset) +{ + struct em_tx_queue *txq = tx_queue; + volatile uint8_t *status; + uint32_t desc; + + if (unlikely(offset >= txq->nb_tx_desc)) + return -EINVAL; + + desc = txq->tx_tail + offset; + /* go to next desc that has the RS bit */ + desc = ((desc + txq->tx_rs_thresh - 1) / txq->tx_rs_thresh) * + txq->tx_rs_thresh; + if (desc >= txq->nb_tx_desc) { + desc -= txq->nb_tx_desc; + if (desc >= txq->nb_tx_desc) + desc -= txq->nb_tx_desc; + } + + status = &txq->tx_ring[desc].upper.fields.status; + if (*status & E1000_TXD_STAT_DD) + return RTE_ETH_TX_DESC_DONE; + + return RTE_ETH_TX_DESC_FULL; +} + void em_dev_clear_queues(struct rte_eth_dev *dev) { @@ -1561,7 +1652,7 @@ em_alloc_rx_queue_mbufs(struct em_rx_queue *rxq) } dma_addr = - rte_cpu_to_le_64(rte_mbuf_data_dma_addr_default(mbuf)); + rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf)); /* Clear HW ring memory */ rxq->rx_ring[i] = rxd_init; diff --git a/dpdk/drivers/net/e1000/igb_ethdev.c b/dpdk/drivers/net/e1000/igb_ethdev.c index fbf4d090..79d0244f 100644 --- a/dpdk/drivers/net/e1000/igb_ethdev.c +++ b/dpdk/drivers/net/e1000/igb_ethdev.c @@ -43,10 +43,11 @@ #include #include #include +#include #include #include +#include #include -#include #include #include #include @@ -111,15 +112,23 @@ static void eth_igb_allmulticast_enable(struct rte_eth_dev *dev); static void eth_igb_allmulticast_disable(struct rte_eth_dev *dev); static int eth_igb_link_update(struct rte_eth_dev *dev, int wait_to_complete); -static void eth_igb_stats_get(struct rte_eth_dev *dev, +static int eth_igb_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats); static int eth_igb_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, unsigned n); +static int eth_igb_xstats_get_by_id(struct rte_eth_dev *dev, + const uint64_t *ids, + uint64_t *values, unsigned int n); static int eth_igb_xstats_get_names(struct rte_eth_dev *dev, struct rte_eth_xstat_name *xstats_names, - unsigned limit); + unsigned int size); +static int eth_igb_xstats_get_names_by_id(struct rte_eth_dev *dev, + struct rte_eth_xstat_name *xstats_names, const uint64_t *ids, + unsigned int limit); static void eth_igb_stats_reset(struct rte_eth_dev *dev); static void eth_igb_xstats_reset(struct rte_eth_dev *dev); +static int eth_igb_fw_version_get(struct rte_eth_dev *dev, + char *fw_version, size_t fw_size); static void eth_igb_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info); static const uint32_t *eth_igb_supported_ptypes_get(struct rte_eth_dev *dev); @@ -129,12 +138,12 @@ static int eth_igb_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf); static int eth_igb_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf); -static int eth_igb_lsc_interrupt_setup(struct rte_eth_dev *dev); +static int eth_igb_lsc_interrupt_setup(struct rte_eth_dev *dev, uint8_t on); static int eth_igb_rxq_interrupt_setup(struct rte_eth_dev *dev); static int eth_igb_interrupt_get_status(struct rte_eth_dev *dev); -static int eth_igb_interrupt_action(struct rte_eth_dev *dev); -static void eth_igb_interrupt_handler(struct rte_intr_handle *handle, - void *param); +static int eth_igb_interrupt_action(struct rte_eth_dev *dev, + struct rte_intr_handle *handle); +static void eth_igb_interrupt_handler(void *param); static int igb_hardware_init(struct e1000_hw *hw); static void igb_hw_control_acquire(struct e1000_hw *hw); static void igb_hw_control_release(struct e1000_hw *hw); @@ -148,7 +157,7 @@ static int eth_igb_vlan_filter_set(struct rte_eth_dev *dev, static int eth_igb_vlan_tpid_set(struct rte_eth_dev *dev, enum rte_vlan_type vlan_type, uint16_t tpid_id); -static void eth_igb_vlan_offload_set(struct rte_eth_dev *dev, int mask); +static int eth_igb_vlan_offload_set(struct rte_eth_dev *dev, int mask); static void igb_vlan_hw_filter_enable(struct rte_eth_dev *dev); static void igb_vlan_hw_filter_disable(struct rte_eth_dev *dev); @@ -162,9 +171,9 @@ static int eth_igb_led_off(struct rte_eth_dev *dev); static void igb_intr_disable(struct e1000_hw *hw); static int igb_get_rx_buffer_size(struct e1000_hw *hw); -static void eth_igb_rar_set(struct rte_eth_dev *dev, - struct ether_addr *mac_addr, - uint32_t index, uint32_t pool); +static int eth_igb_rar_set(struct rte_eth_dev *dev, + struct ether_addr *mac_addr, + uint32_t index, uint32_t pool); static void eth_igb_rar_clear(struct rte_eth_dev *dev, uint32_t index); static void eth_igb_default_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *addr); @@ -179,7 +188,7 @@ static void igbvf_promiscuous_disable(struct rte_eth_dev *dev); static void igbvf_allmulticast_enable(struct rte_eth_dev *dev); static void igbvf_allmulticast_disable(struct rte_eth_dev *dev); static int eth_igbvf_link_update(struct e1000_hw *hw); -static void eth_igbvf_stats_get(struct rte_eth_dev *dev, +static int eth_igbvf_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats); static int eth_igbvf_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, unsigned n); @@ -204,9 +213,6 @@ static int eth_igb_rss_reta_query(struct rte_eth_dev *dev, struct rte_eth_rss_reta_entry64 *reta_conf, uint16_t reta_size); -static int eth_igb_syn_filter_set(struct rte_eth_dev *dev, - struct rte_eth_syn_filter *filter, - bool add); static int eth_igb_syn_filter_get(struct rte_eth_dev *dev, struct rte_eth_syn_filter *filter); static int eth_igb_syn_filter_handle(struct rte_eth_dev *dev, @@ -216,9 +222,6 @@ static int igb_add_2tuple_filter(struct rte_eth_dev *dev, struct rte_eth_ntuple_filter *ntuple_filter); static int igb_remove_2tuple_filter(struct rte_eth_dev *dev, struct rte_eth_ntuple_filter *ntuple_filter); -static int eth_igb_add_del_flex_filter(struct rte_eth_dev *dev, - struct rte_eth_flex_filter *filter, - bool add); static int eth_igb_get_flex_filter(struct rte_eth_dev *dev, struct rte_eth_flex_filter *filter); static int eth_igb_flex_filter_handle(struct rte_eth_dev *dev, @@ -228,17 +231,11 @@ static int igb_add_5tuple_filter_82576(struct rte_eth_dev *dev, struct rte_eth_ntuple_filter *ntuple_filter); static int igb_remove_5tuple_filter_82576(struct rte_eth_dev *dev, struct rte_eth_ntuple_filter *ntuple_filter); -static int igb_add_del_ntuple_filter(struct rte_eth_dev *dev, - struct rte_eth_ntuple_filter *filter, - bool add); static int igb_get_ntuple_filter(struct rte_eth_dev *dev, struct rte_eth_ntuple_filter *filter); static int igb_ntuple_filter_handle(struct rte_eth_dev *dev, enum rte_filter_op filter_op, void *arg); -static int igb_add_del_ethertype_filter(struct rte_eth_dev *dev, - struct rte_eth_ethertype_filter *filter, - bool add); static int igb_ethertype_filter_handle(struct rte_eth_dev *dev, enum rte_filter_op filter_op, void *arg); @@ -280,9 +277,9 @@ static void eth_igb_assign_msix_vector(struct e1000_hw *hw, int8_t direction, static void eth_igb_write_ivar(struct e1000_hw *hw, uint8_t msix_vector, uint8_t index, uint8_t offset); static void eth_igb_configure_msix_intr(struct rte_eth_dev *dev); -static void eth_igbvf_interrupt_handler(struct rte_intr_handle *handle, - void *param); +static void eth_igbvf_interrupt_handler(void *param); static void igbvf_mbx_process(struct rte_eth_dev *dev); +static int igb_filter_restore(struct rte_eth_dev *dev); /* * Define VF Stats MACRO for Non "cleared on read" register @@ -306,22 +303,59 @@ static enum e1000_fc_mode igb_fc_setting = e1000_fc_full; * The set of PCI devices this driver supports */ static const struct rte_pci_id pci_id_igb_map[] = { + { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576) }, + { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_FIBER) }, + { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_SERDES) }, + { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_QUAD_COPPER) }, + { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_QUAD_COPPER_ET2) }, + { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_NS) }, + { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_NS_SERDES) }, + { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_SERDES_QUAD) }, -#define RTE_PCI_DEV_ID_DECL_IGB(vend, dev) {RTE_PCI_DEVICE(vend, dev)}, -#include "rte_pci_dev_ids.h" + { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82575EB_COPPER) }, + { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82575EB_FIBER_SERDES) }, + { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82575GB_QUAD_COPPER) }, -{0}, + { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_COPPER) }, + { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_FIBER) }, + { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_SERDES) }, + { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_SGMII) }, + { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_COPPER_DUAL) }, + { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_QUAD_FIBER) }, + + { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_COPPER) }, + { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_FIBER) }, + { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_SERDES) }, + { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_SGMII) }, + { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_DA4) }, + { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER) }, + { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_OEM1) }, + { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_IT) }, + { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_FIBER) }, + { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SERDES) }, + { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SGMII) }, + { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_FLASHLESS) }, + { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SERDES_FLASHLESS) }, + { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I211_COPPER) }, + { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I354_BACKPLANE_1GBPS) }, + { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I354_SGMII) }, + { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I354_BACKPLANE_2_5GBPS) }, + { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SGMII) }, + { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SERDES) }, + { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_BACKPLANE) }, + { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SFP) }, + { .vendor_id = 0, /* sentinel */ }, }; /* * The set of PCI devices this driver supports (for 82576&I350 VF) */ static const struct rte_pci_id pci_id_igbvf_map[] = { - -#define RTE_PCI_DEV_ID_DECL_IGBVF(vend, dev) {RTE_PCI_DEVICE(vend, dev)}, -#include "rte_pci_dev_ids.h" - -{0}, + { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_VF) }, + { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_VF_HV) }, + { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_VF) }, + { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_VF_HV) }, + { .vendor_id = 0, /* sentinel */ }, }; static const struct rte_eth_desc_lim rx_desc_lim = { @@ -334,6 +368,8 @@ static const struct rte_eth_desc_lim tx_desc_lim = { .nb_max = E1000_MAX_RING_DESC, .nb_min = E1000_MIN_RING_DESC, .nb_align = IGB_RXD_ALIGN, + .nb_seg_max = IGB_TX_MAX_SEG, + .nb_mtu_seg_max = IGB_TX_MAX_MTU_SEG, }; static const struct eth_dev_ops eth_igb_ops = { @@ -350,9 +386,12 @@ static const struct eth_dev_ops eth_igb_ops = { .link_update = eth_igb_link_update, .stats_get = eth_igb_stats_get, .xstats_get = eth_igb_xstats_get, + .xstats_get_by_id = eth_igb_xstats_get_by_id, + .xstats_get_names_by_id = eth_igb_xstats_get_names_by_id, .xstats_get_names = eth_igb_xstats_get_names, .stats_reset = eth_igb_stats_reset, .xstats_reset = eth_igb_xstats_reset, + .fw_version_get = eth_igb_fw_version_get, .dev_infos_get = eth_igb_infos_get, .dev_supported_ptypes_get = eth_igb_supported_ptypes_get, .mtu_set = eth_igb_mtu_set, @@ -365,8 +404,11 @@ static const struct eth_dev_ops eth_igb_ops = { .rx_queue_release = eth_igb_rx_queue_release, .rx_queue_count = eth_igb_rx_queue_count, .rx_descriptor_done = eth_igb_rx_descriptor_done, + .rx_descriptor_status = eth_igb_rx_descriptor_status, + .tx_descriptor_status = eth_igb_tx_descriptor_status, .tx_queue_setup = eth_igb_tx_queue_setup, .tx_queue_release = eth_igb_tx_queue_release, + .tx_done_cleanup = eth_igb_tx_done_cleanup, .dev_led_on = eth_igb_led_on, .dev_led_off = eth_igb_led_off, .flow_ctrl_get = eth_igb_flow_ctrl_get, @@ -633,15 +675,16 @@ igb_pf_reset_hw(struct e1000_hw *hw) } static void -igb_identify_hardware(struct rte_eth_dev *dev) +igb_identify_hardware(struct rte_eth_dev *dev, struct rte_pci_device *pci_dev) { struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); - hw->vendor_id = dev->pci_dev->id.vendor_id; - hw->device_id = dev->pci_dev->id.device_id; - hw->subsystem_vendor_id = dev->pci_dev->id.subsystem_vendor_id; - hw->subsystem_device_id = dev->pci_dev->id.subsystem_device_id; + + hw->vendor_id = pci_dev->id.vendor_id; + hw->device_id = pci_dev->id.device_id; + hw->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id; + hw->subsystem_device_id = pci_dev->id.subsystem_device_id; e1000_set_mac_type(hw); @@ -704,11 +747,51 @@ igb_reset_swfw_lock(struct e1000_hw *hw) return E1000_SUCCESS; } +/* Remove all ntuple filters of the device */ +static int igb_ntuple_filter_uninit(struct rte_eth_dev *eth_dev) +{ + struct e1000_filter_info *filter_info = + E1000_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private); + struct e1000_5tuple_filter *p_5tuple; + struct e1000_2tuple_filter *p_2tuple; + + while ((p_5tuple = TAILQ_FIRST(&filter_info->fivetuple_list))) { + TAILQ_REMOVE(&filter_info->fivetuple_list, + p_5tuple, entries); + rte_free(p_5tuple); + } + filter_info->fivetuple_mask = 0; + while ((p_2tuple = TAILQ_FIRST(&filter_info->twotuple_list))) { + TAILQ_REMOVE(&filter_info->twotuple_list, + p_2tuple, entries); + rte_free(p_2tuple); + } + filter_info->twotuple_mask = 0; + + return 0; +} + +/* Remove all flex filters of the device */ +static int igb_flex_filter_uninit(struct rte_eth_dev *eth_dev) +{ + struct e1000_filter_info *filter_info = + E1000_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private); + struct e1000_flex_filter *p_flex; + + while ((p_flex = TAILQ_FIRST(&filter_info->flex_list))) { + TAILQ_REMOVE(&filter_info->flex_list, p_flex, entries); + rte_free(p_flex); + } + filter_info->flex_mask = 0; + + return 0; +} + static int eth_igb_dev_init(struct rte_eth_dev *eth_dev) { int error = 0; - struct rte_pci_device *pci_dev; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); struct e1000_vfta * shadow_vfta = @@ -720,11 +803,10 @@ eth_igb_dev_init(struct rte_eth_dev *eth_dev) uint32_t ctrl_ext; - pci_dev = eth_dev->pci_dev; - eth_dev->dev_ops = ð_igb_ops; eth_dev->rx_pkt_burst = ð_igb_recv_pkts; eth_dev->tx_pkt_burst = ð_igb_xmit_pkts; + eth_dev->tx_pkt_prepare = ð_igb_prep_pkts; /* for secondary processes, we don't initialise any further as primary * has already done this work. Only check we don't need a different @@ -739,7 +821,7 @@ eth_igb_dev_init(struct rte_eth_dev *eth_dev) hw->hw_addr= (void *)pci_dev->mem_resource[0].addr; - igb_identify_hardware(eth_dev); + igb_identify_hardware(eth_dev, pci_dev); if (e1000_setup_init_funcs(hw, FALSE) != E1000_SUCCESS) { error = -EIO; goto err_late; @@ -854,12 +936,19 @@ eth_igb_dev_init(struct rte_eth_dev *eth_dev) /* enable support intr */ igb_intr_enable(eth_dev); + /* initialize filter info */ + memset(filter_info, 0, + sizeof(struct e1000_filter_info)); + TAILQ_INIT(&filter_info->flex_list); - filter_info->flex_mask = 0; TAILQ_INIT(&filter_info->twotuple_list); - filter_info->twotuple_mask = 0; TAILQ_INIT(&filter_info->fivetuple_list); - filter_info->fivetuple_mask = 0; + + TAILQ_INIT(&igb_filter_ntuple_list); + TAILQ_INIT(&igb_filter_ethertype_list); + TAILQ_INIT(&igb_filter_syn_list); + TAILQ_INIT(&igb_filter_flex_list); + TAILQ_INIT(&igb_flow_list); return 0; @@ -873,9 +962,12 @@ static int eth_igb_dev_uninit(struct rte_eth_dev *eth_dev) { struct rte_pci_device *pci_dev; + struct rte_intr_handle *intr_handle; struct e1000_hw *hw; struct e1000_adapter *adapter = E1000_DEV_PRIVATE(eth_dev->data->dev_private); + struct e1000_filter_info *filter_info = + E1000_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private); PMD_INIT_FUNC_TRACE(); @@ -883,7 +975,8 @@ eth_igb_dev_uninit(struct rte_eth_dev *eth_dev) return -EPERM; hw = E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); - pci_dev = eth_dev->pci_dev; + pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); + intr_handle = &pci_dev->intr_handle; if (adapter->stopped == 0) eth_igb_close(eth_dev); @@ -902,9 +995,26 @@ eth_igb_dev_uninit(struct rte_eth_dev *eth_dev) igb_pf_host_uninit(eth_dev); /* disable uio intr before callback unregister */ - rte_intr_disable(&(pci_dev->intr_handle)); - rte_intr_callback_unregister(&(pci_dev->intr_handle), - eth_igb_interrupt_handler, (void *)eth_dev); + rte_intr_disable(intr_handle); + rte_intr_callback_unregister(intr_handle, + eth_igb_interrupt_handler, eth_dev); + + /* clear the SYN filter info */ + filter_info->syn_info = 0; + + /* clear the ethertype filters info */ + filter_info->ethertype_mask = 0; + memset(filter_info->ethertype_filters, 0, + E1000_MAX_ETQF_FILTERS * sizeof(struct igb_ethertype_filter)); + + /* remove all ntuple filters of the device */ + igb_ntuple_filter_uninit(eth_dev); + + /* remove all flex filters of the device */ + igb_flex_filter_uninit(eth_dev); + + /* clear all the filters list */ + igb_filterlist_flush(eth_dev); return 0; } @@ -916,6 +1026,7 @@ static int eth_igbvf_dev_init(struct rte_eth_dev *eth_dev) { struct rte_pci_device *pci_dev; + struct rte_intr_handle *intr_handle; struct e1000_adapter *adapter = E1000_DEV_PRIVATE(eth_dev->data->dev_private); struct e1000_hw *hw = @@ -928,6 +1039,7 @@ eth_igbvf_dev_init(struct rte_eth_dev *eth_dev) eth_dev->dev_ops = &igbvf_eth_dev_ops; eth_dev->rx_pkt_burst = ð_igb_recv_pkts; eth_dev->tx_pkt_burst = ð_igb_xmit_pkts; + eth_dev->tx_pkt_prepare = ð_igb_prep_pkts; /* for secondary processes, we don't initialise any further as primary * has already done this work. Only check we don't need a different @@ -938,8 +1050,7 @@ eth_igbvf_dev_init(struct rte_eth_dev *eth_dev) return 0; } - pci_dev = eth_dev->pci_dev; - + pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); rte_eth_copy_pci_info(eth_dev, pci_dev); hw->device_id = pci_dev->id.device_id; @@ -977,12 +1088,6 @@ eth_igbvf_dev_init(struct rte_eth_dev *eth_dev) /* Generate a random MAC address, if none was assigned by PF. */ if (is_zero_ether_addr(perm_addr)) { eth_random_addr(perm_addr->addr_bytes); - diag = e1000_rar_set(hw, perm_addr->addr_bytes, 0); - if (diag) { - rte_free(eth_dev->data->mac_addrs); - eth_dev->data->mac_addrs = NULL; - return diag; - } PMD_INIT_LOG(INFO, "\tVF MAC address not assigned by Host PF"); PMD_INIT_LOG(INFO, "\tAssign randomly generated MAC address " "%02x:%02x:%02x:%02x:%02x:%02x", @@ -994,6 +1099,12 @@ eth_igbvf_dev_init(struct rte_eth_dev *eth_dev) perm_addr->addr_bytes[5]); } + diag = e1000_rar_set(hw, perm_addr->addr_bytes, 0); + if (diag) { + rte_free(eth_dev->data->mac_addrs); + eth_dev->data->mac_addrs = NULL; + return diag; + } /* Copy the permanent MAC address */ ether_addr_copy((struct ether_addr *) hw->mac.perm_addr, ð_dev->data->mac_addrs[0]); @@ -1003,9 +1114,9 @@ eth_igbvf_dev_init(struct rte_eth_dev *eth_dev) eth_dev->data->port_id, pci_dev->id.vendor_id, pci_dev->id.device_id, "igb_mac_82576_vf"); - rte_intr_callback_register(&pci_dev->intr_handle, - eth_igbvf_interrupt_handler, - (void *)eth_dev); + intr_handle = &pci_dev->intr_handle; + rte_intr_callback_register(intr_handle, + eth_igbvf_interrupt_handler, eth_dev); return 0; } @@ -1015,7 +1126,7 @@ eth_igbvf_dev_uninit(struct rte_eth_dev *eth_dev) { struct e1000_adapter *adapter = E1000_DEV_PRIVATE(eth_dev->data->dev_private); - struct rte_pci_device *pci_dev = eth_dev->pci_dev; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); PMD_INIT_FUNC_TRACE(); @@ -1041,39 +1152,49 @@ eth_igbvf_dev_uninit(struct rte_eth_dev *eth_dev) return 0; } -static struct eth_driver rte_igb_pmd = { - .pci_drv = { - .name = "rte_igb_pmd", - .id_table = pci_id_igb_map, - .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC | - RTE_PCI_DRV_DETACHABLE, - }, - .eth_dev_init = eth_igb_dev_init, - .eth_dev_uninit = eth_igb_dev_uninit, - .dev_private_size = sizeof(struct e1000_adapter), +static int eth_igb_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, + struct rte_pci_device *pci_dev) +{ + return rte_eth_dev_pci_generic_probe(pci_dev, + sizeof(struct e1000_adapter), eth_igb_dev_init); +} + +static int eth_igb_pci_remove(struct rte_pci_device *pci_dev) +{ + return rte_eth_dev_pci_generic_remove(pci_dev, eth_igb_dev_uninit); +} + +static struct rte_pci_driver rte_igb_pmd = { + .id_table = pci_id_igb_map, + .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC | + RTE_PCI_DRV_IOVA_AS_VA, + .probe = eth_igb_pci_probe, + .remove = eth_igb_pci_remove, }; + +static int eth_igbvf_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, + struct rte_pci_device *pci_dev) +{ + return rte_eth_dev_pci_generic_probe(pci_dev, + sizeof(struct e1000_adapter), eth_igbvf_dev_init); +} + +static int eth_igbvf_pci_remove(struct rte_pci_device *pci_dev) +{ + return rte_eth_dev_pci_generic_remove(pci_dev, eth_igbvf_dev_uninit); +} + /* * virtual function driver struct */ -static struct eth_driver rte_igbvf_pmd = { - .pci_drv = { - .name = "rte_igbvf_pmd", - .id_table = pci_id_igbvf_map, - .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_DETACHABLE, - }, - .eth_dev_init = eth_igbvf_dev_init, - .eth_dev_uninit = eth_igbvf_dev_uninit, - .dev_private_size = sizeof(struct e1000_adapter), +static struct rte_pci_driver rte_igbvf_pmd = { + .id_table = pci_id_igbvf_map, + .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_IOVA_AS_VA, + .probe = eth_igbvf_pci_probe, + .remove = eth_igbvf_pci_remove, }; -static int -rte_igb_pmd_init(const char *name __rte_unused, const char *params __rte_unused) -{ - rte_eth_driver_register(&rte_igb_pmd); - return 0; -} - static void igb_vmdq_vlan_hw_filter_enable(struct rte_eth_dev *dev) { @@ -1085,27 +1206,13 @@ igb_vmdq_vlan_hw_filter_enable(struct rte_eth_dev *dev) E1000_WRITE_REG(hw, E1000_RCTL, rctl); } -/* - * VF Driver initialization routine. - * Invoked one at EAL init time. - * Register itself as the [Virtual Poll Mode] Driver of PCI IGB devices. - */ -static int -rte_igbvf_pmd_init(const char *name __rte_unused, const char *params __rte_unused) -{ - PMD_INIT_FUNC_TRACE(); - - rte_eth_driver_register(&rte_igbvf_pmd); - return 0; -} - static int igb_check_mq_mode(struct rte_eth_dev *dev) { enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode; enum rte_eth_tx_mq_mode tx_mq_mode = dev->data->dev_conf.txmode.mq_mode; uint16_t nb_rx_q = dev->data->nb_rx_queues; - uint16_t nb_tx_q = dev->data->nb_rx_queues; + uint16_t nb_tx_q = dev->data->nb_tx_queues; if ((rx_mq_mode & ETH_MQ_RX_DCB_FLAG) || tx_mq_mode == ETH_MQ_TX_DCB || @@ -1201,7 +1308,8 @@ eth_igb_start(struct rte_eth_dev *dev) E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct e1000_adapter *adapter = E1000_DEV_PRIVATE(dev->data->dev_private); - struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; int ret, mask; uint32_t intr_vector = 0; uint32_t ctrl_ext; @@ -1265,7 +1373,7 @@ eth_igb_start(struct rte_eth_dev *dev) dev->data->nb_rx_queues * sizeof(int), 0); if (intr_handle->intr_vec == NULL) { PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues" - " intr_vec\n", dev->data->nb_rx_queues); + " intr_vec", dev->data->nb_rx_queues); return -ENOMEM; } } @@ -1293,7 +1401,12 @@ eth_igb_start(struct rte_eth_dev *dev) */ mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \ ETH_VLAN_EXTEND_MASK; - eth_igb_vlan_offload_set(dev, mask); + ret = eth_igb_vlan_offload_set(dev, mask); + if (ret) { + PMD_INIT_LOG(ERR, "Unable to set vlan offload"); + igb_dev_clear_queues(dev); + return ret; + } if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_VMDQ_ONLY) { /* Enable VLAN filter since VMDq always use VLAN filter */ @@ -1311,6 +1424,7 @@ eth_igb_start(struct rte_eth_dev *dev) speeds = &dev->data->dev_conf.link_speeds; if (*speeds == ETH_LINK_SPEED_AUTONEG) { hw->phy.autoneg_advertised = E1000_ALL_SPEED_DUPLEX; + hw->mac.autoneg = 1; } else { num_speeds = 0; autoneg = (*speeds & ETH_LINK_SPEED_FIXED) == 0; @@ -1346,6 +1460,17 @@ eth_igb_start(struct rte_eth_dev *dev) } if (num_speeds == 0 || (!autoneg && (num_speeds > 1))) goto error_invalid_config; + + /* Set/reset the mac.autoneg based on the link speed, + * fixed or not + */ + if (!autoneg) { + hw->mac.autoneg = 0; + hw->mac.forced_speed_duplex = + hw->phy.autoneg_advertised; + } else { + hw->mac.autoneg = 1; + } } e1000_setup_link(hw); @@ -1353,14 +1478,16 @@ eth_igb_start(struct rte_eth_dev *dev) if (rte_intr_allow_others(intr_handle)) { /* check if lsc interrupt is enabled */ if (dev->data->dev_conf.intr_conf.lsc != 0) - eth_igb_lsc_interrupt_setup(dev); + eth_igb_lsc_interrupt_setup(dev, TRUE); + else + eth_igb_lsc_interrupt_setup(dev, FALSE); } else { rte_intr_callback_unregister(intr_handle, eth_igb_interrupt_handler, (void *)dev); if (dev->data->dev_conf.intr_conf.lsc != 0) PMD_INIT_LOG(INFO, "lsc won't enable because of" - " no intr multiplex\n"); + " no intr multiplex"); } /* check if rxq interrupt is enabled */ @@ -1374,6 +1501,9 @@ eth_igb_start(struct rte_eth_dev *dev) /* resume enabled intr since hw reset */ igb_intr_enable(dev); + /* restore all types filter */ + igb_filter_restore(dev); + PMD_INIT_LOG(DEBUG, "<<"); return 0; @@ -1395,13 +1525,9 @@ static void eth_igb_stop(struct rte_eth_dev *dev) { struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); - struct e1000_filter_info *filter_info = - E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); struct rte_eth_link link; - struct e1000_flex_filter *p_flex; - struct e1000_5tuple_filter *p_5tuple, *p_5tuple_next; - struct e1000_2tuple_filter *p_2tuple, *p_2tuple_next; - struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle; + struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; igb_intr_disable(hw); @@ -1429,31 +1555,6 @@ eth_igb_stop(struct rte_eth_dev *dev) memset(&link, 0, sizeof(link)); rte_igb_dev_atomic_write_link_status(dev, &link); - /* Remove all flex filters of the device */ - while ((p_flex = TAILQ_FIRST(&filter_info->flex_list))) { - TAILQ_REMOVE(&filter_info->flex_list, p_flex, entries); - rte_free(p_flex); - } - filter_info->flex_mask = 0; - - /* Remove all ntuple filters of the device */ - for (p_5tuple = TAILQ_FIRST(&filter_info->fivetuple_list); - p_5tuple != NULL; p_5tuple = p_5tuple_next) { - p_5tuple_next = TAILQ_NEXT(p_5tuple, entries); - TAILQ_REMOVE(&filter_info->fivetuple_list, - p_5tuple, entries); - rte_free(p_5tuple); - } - filter_info->fivetuple_mask = 0; - for (p_2tuple = TAILQ_FIRST(&filter_info->twotuple_list); - p_2tuple != NULL; p_2tuple = p_2tuple_next) { - p_2tuple_next = TAILQ_NEXT(p_2tuple, entries); - TAILQ_REMOVE(&filter_info->twotuple_list, - p_2tuple, entries); - rte_free(p_2tuple); - } - filter_info->twotuple_mask = 0; - if (!rte_intr_allow_others(intr_handle)) /* resume to the default handler */ rte_intr_callback_register(intr_handle, @@ -1501,7 +1602,8 @@ eth_igb_close(struct rte_eth_dev *dev) struct e1000_adapter *adapter = E1000_DEV_PRIVATE(dev->data->dev_private); struct rte_eth_link link; - struct rte_pci_device *pci_dev; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; eth_igb_stop(dev); adapter->stopped = 1; @@ -1521,10 +1623,9 @@ eth_igb_close(struct rte_eth_dev *dev) igb_dev_free_queues(dev); - pci_dev = dev->pci_dev; - if (pci_dev->intr_handle.intr_vec) { - rte_free(pci_dev->intr_handle.intr_vec); - pci_dev->intr_handle.intr_vec = NULL; + if (intr_handle->intr_vec) { + rte_free(intr_handle->intr_vec); + intr_handle->intr_vec = NULL; } memset(&link, 0, sizeof(link)); @@ -1733,7 +1834,7 @@ igb_read_stats_registers(struct e1000_hw *hw, struct e1000_hw_stats *stats) stats->tsctfc += E1000_READ_REG(hw, E1000_TSCTFC); } -static void +static int eth_igb_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats) { struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); @@ -1743,7 +1844,7 @@ eth_igb_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats) igb_read_stats_registers(hw, stats); if (rte_stats == NULL) - return; + return -EINVAL; /* Rx Errors */ rte_stats->imissed = stats->mpc; @@ -1758,6 +1859,7 @@ eth_igb_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats) rte_stats->opackets = stats->gptc; rte_stats->ibytes = stats->gorc; rte_stats->obytes = stats->gotc; + return 0; } static void @@ -1788,7 +1890,7 @@ eth_igb_xstats_reset(struct rte_eth_dev *dev) static int eth_igb_xstats_get_names(__rte_unused struct rte_eth_dev *dev, struct rte_eth_xstat_name *xstats_names, - __rte_unused unsigned limit) + __rte_unused unsigned int size) { unsigned i; @@ -1805,6 +1907,41 @@ static int eth_igb_xstats_get_names(__rte_unused struct rte_eth_dev *dev, return IGB_NB_XSTATS; } +static int eth_igb_xstats_get_names_by_id(struct rte_eth_dev *dev, + struct rte_eth_xstat_name *xstats_names, const uint64_t *ids, + unsigned int limit) +{ + unsigned int i; + + if (!ids) { + if (xstats_names == NULL) + return IGB_NB_XSTATS; + + for (i = 0; i < IGB_NB_XSTATS; i++) + snprintf(xstats_names[i].name, + sizeof(xstats_names[i].name), + "%s", rte_igb_stats_strings[i].name); + + return IGB_NB_XSTATS; + + } else { + struct rte_eth_xstat_name xstats_names_copy[IGB_NB_XSTATS]; + + eth_igb_xstats_get_names_by_id(dev, xstats_names_copy, NULL, + IGB_NB_XSTATS); + + for (i = 0; i < limit; i++) { + if (ids[i] >= IGB_NB_XSTATS) { + PMD_INIT_LOG(ERR, "id value isn't valid"); + return -1; + } + strcpy(xstats_names[i].name, + xstats_names_copy[ids[i]].name); + } + return limit; + } +} + static int eth_igb_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, unsigned n) @@ -1835,6 +1972,53 @@ eth_igb_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, return IGB_NB_XSTATS; } +static int +eth_igb_xstats_get_by_id(struct rte_eth_dev *dev, const uint64_t *ids, + uint64_t *values, unsigned int n) +{ + unsigned int i; + + if (!ids) { + struct e1000_hw *hw = + E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct e1000_hw_stats *hw_stats = + E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private); + + if (n < IGB_NB_XSTATS) + return IGB_NB_XSTATS; + + igb_read_stats_registers(hw, hw_stats); + + /* If this is a reset xstats is NULL, and we have cleared the + * registers by reading them. + */ + if (!values) + return 0; + + /* Extended stats */ + for (i = 0; i < IGB_NB_XSTATS; i++) + values[i] = *(uint64_t *)(((char *)hw_stats) + + rte_igb_stats_strings[i].offset); + + return IGB_NB_XSTATS; + + } else { + uint64_t values_copy[IGB_NB_XSTATS]; + + eth_igb_xstats_get_by_id(dev, NULL, values_copy, + IGB_NB_XSTATS); + + for (i = 0; i < n; i++) { + if (ids[i] >= IGB_NB_XSTATS) { + PMD_INIT_LOG(ERR, "id value isn't valid"); + return -1; + } + values[i] = values_copy[ids[i]]; + } + return n; + } +} + static void igbvf_read_stats_registers(struct e1000_hw *hw, struct e1000_vf_stats *hw_stats) { @@ -1916,7 +2100,7 @@ eth_igbvf_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, return IGBVF_NB_XSTATS; } -static void +static int eth_igbvf_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats) { struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); @@ -1926,12 +2110,13 @@ eth_igbvf_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats) igbvf_read_stats_registers(hw, hw_stats); if (rte_stats == NULL) - return; + return -EINVAL; rte_stats->ipackets = hw_stats->gprc; rte_stats->ibytes = hw_stats->gorc; rte_stats->opackets = hw_stats->gptc; rte_stats->obytes = hw_stats->gotc; + return 0; } static void @@ -1948,11 +2133,64 @@ eth_igbvf_stats_reset(struct rte_eth_dev *dev) offsetof(struct e1000_vf_stats, gprc)); } +static int +eth_igb_fw_version_get(struct rte_eth_dev *dev, char *fw_version, + size_t fw_size) +{ + struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct e1000_fw_version fw; + int ret; + + e1000_get_fw_version(hw, &fw); + + switch (hw->mac.type) { + case e1000_i210: + case e1000_i211: + if (!(e1000_get_flash_presence_i210(hw))) { + ret = snprintf(fw_version, fw_size, + "%2d.%2d-%d", + fw.invm_major, fw.invm_minor, + fw.invm_img_type); + break; + } + /* fall through */ + default: + /* if option rom is valid, display its version too */ + if (fw.or_valid) { + ret = snprintf(fw_version, fw_size, + "%d.%d, 0x%08x, %d.%d.%d", + fw.eep_major, fw.eep_minor, fw.etrack_id, + fw.or_major, fw.or_build, fw.or_patch); + /* no option rom */ + } else { + if (fw.etrack_id != 0X0000) { + ret = snprintf(fw_version, fw_size, + "%d.%d, 0x%08x", + fw.eep_major, fw.eep_minor, + fw.etrack_id); + } else { + ret = snprintf(fw_version, fw_size, + "%d.%d.%d", + fw.eep_major, fw.eep_minor, + fw.eep_build); + } + } + break; + } + + ret += 1; /* add the size of '\0' */ + if (fw_size < (u32)ret) + return ret; + else + return 0; +} + static void eth_igb_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) { struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + dev_info->pci_dev = RTE_ETH_DEV_TO_PCI(dev); dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */ dev_info->max_rx_pktlen = 0x3FFF; /* See RLPML register. */ dev_info->max_mac_addrs = hw->mac.rar_entry_count; @@ -2081,6 +2319,7 @@ eth_igbvf_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) { struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + dev_info->pci_dev = RTE_ETH_DEV_TO_PCI(dev); dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */ dev_info->max_rx_pktlen = 0x3FFF; /* See RLPML register. */ dev_info->max_mac_addrs = hw->mac.rar_entry_count; @@ -2196,7 +2435,7 @@ eth_igb_link_update(struct rte_eth_dev *dev, int wait_to_complete) link.link_speed = 0; link.link_duplex = ETH_LINK_HALF_DUPLEX; link.link_status = ETH_LINK_DOWN; - link.link_autoneg = ETH_LINK_SPEED_FIXED; + link.link_autoneg = ETH_LINK_FIXED; } rte_igb_dev_atomic_write_link_status(dev, &link); @@ -2484,7 +2723,7 @@ igb_vlan_hw_extend_enable(struct rte_eth_dev *dev) 2 * VLAN_TAG_SIZE); } -static void +static int eth_igb_vlan_offload_set(struct rte_eth_dev *dev, int mask) { if(mask & ETH_VLAN_STRIP_MASK){ @@ -2507,6 +2746,8 @@ eth_igb_vlan_offload_set(struct rte_eth_dev *dev, int mask) else igb_vlan_hw_extend_disable(dev); } + + return 0; } @@ -2515,18 +2756,23 @@ eth_igb_vlan_offload_set(struct rte_eth_dev *dev, int mask) * * @param dev * Pointer to struct rte_eth_dev. + * @param on + * Enable or Disable * * @return * - On success, zero. * - On failure, a negative value. */ static int -eth_igb_lsc_interrupt_setup(struct rte_eth_dev *dev) +eth_igb_lsc_interrupt_setup(struct rte_eth_dev *dev, uint8_t on) { struct e1000_interrupt *intr = E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); - intr->mask |= E1000_ICR_LSC; + if (on) + intr->mask |= E1000_ICR_LSC; + else + intr->mask &= ~E1000_ICR_LSC; return 0; } @@ -2605,12 +2851,14 @@ eth_igb_interrupt_get_status(struct rte_eth_dev *dev) * - On failure, a negative value. */ static int -eth_igb_interrupt_action(struct rte_eth_dev *dev) +eth_igb_interrupt_action(struct rte_eth_dev *dev, + struct rte_intr_handle *intr_handle) { struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct e1000_interrupt *intr = E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); uint32_t tctl, rctl; struct rte_eth_link link; int ret; @@ -2621,7 +2869,7 @@ eth_igb_interrupt_action(struct rte_eth_dev *dev) } igb_intr_enable(dev); - rte_intr_enable(&(dev->pci_dev->intr_handle)); + rte_intr_enable(intr_handle); if (intr->flags & E1000_FLAG_NEED_LINK_UPDATE) { intr->flags &= ~E1000_FLAG_NEED_LINK_UPDATE; @@ -2649,10 +2897,10 @@ eth_igb_interrupt_action(struct rte_eth_dev *dev) } PMD_INIT_LOG(DEBUG, "PCI Address: %04d:%02d:%02d:%d", - dev->pci_dev->addr.domain, - dev->pci_dev->addr.bus, - dev->pci_dev->addr.devid, - dev->pci_dev->addr.function); + pci_dev->addr.domain, + pci_dev->addr.bus, + pci_dev->addr.devid, + pci_dev->addr.function); tctl = E1000_READ_REG(hw, E1000_TCTL); rctl = E1000_READ_REG(hw, E1000_RCTL); if (link.link_status) { @@ -2667,7 +2915,8 @@ eth_igb_interrupt_action(struct rte_eth_dev *dev) E1000_WRITE_REG(hw, E1000_TCTL, tctl); E1000_WRITE_REG(hw, E1000_RCTL, rctl); E1000_WRITE_FLUSH(hw); - _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC); + _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, + NULL, NULL); } return 0; @@ -2685,13 +2934,12 @@ eth_igb_interrupt_action(struct rte_eth_dev *dev) * void */ static void -eth_igb_interrupt_handler(__rte_unused struct rte_intr_handle *handle, - void *param) +eth_igb_interrupt_handler(void *param) { struct rte_eth_dev *dev = (struct rte_eth_dev *)param; eth_igb_interrupt_get_status(dev); - eth_igb_interrupt_action(dev); + eth_igb_interrupt_action(dev, dev->intr_handle); } static int @@ -2727,11 +2975,12 @@ void igbvf_mbx_process(struct rte_eth_dev *dev) /* PF reset VF event */ if (in_msg == E1000_PF_CONTROL_MSG) - _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RESET); + _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RESET, + NULL, NULL); } static int -eth_igbvf_interrupt_action(struct rte_eth_dev *dev) +eth_igbvf_interrupt_action(struct rte_eth_dev *dev, struct rte_intr_handle *intr_handle) { struct e1000_interrupt *intr = E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); @@ -2742,19 +2991,18 @@ eth_igbvf_interrupt_action(struct rte_eth_dev *dev) } igbvf_intr_enable(dev); - rte_intr_enable(&dev->pci_dev->intr_handle); + rte_intr_enable(intr_handle); return 0; } static void -eth_igbvf_interrupt_handler(__rte_unused struct rte_intr_handle *handle, - void *param) +eth_igbvf_interrupt_handler(void *param) { struct rte_eth_dev *dev = (struct rte_eth_dev *)param; eth_igbvf_interrupt_get_status(dev); - eth_igbvf_interrupt_action(dev); + eth_igbvf_interrupt_action(dev, dev->intr_handle); } static int @@ -2878,9 +3126,9 @@ eth_igb_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) } #define E1000_RAH_POOLSEL_SHIFT (18) -static void +static int eth_igb_rar_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr, - uint32_t index, __rte_unused uint32_t pool) + uint32_t index, uint32_t pool) { struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); uint32_t rah; @@ -2889,6 +3137,7 @@ eth_igb_rar_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr, rah = E1000_READ_REG(hw, E1000_RAH(index)); rah |= (0x1 << (E1000_RAH_POOLSEL_SHIFT + pool)); E1000_WRITE_REG(hw, E1000_RAH(index), rah); + return 0; } static void @@ -3027,8 +3276,9 @@ igbvf_dev_start(struct rte_eth_dev *dev) E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct e1000_adapter *adapter = E1000_DEV_PRIVATE(dev->data->dev_private); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; int ret; - struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle; uint32_t intr_vector = 0; PMD_INIT_FUNC_TRACE(); @@ -3050,7 +3300,8 @@ igbvf_dev_start(struct rte_eth_dev *dev) } /* check and configure queue intr-vector mapping */ - if (dev->data->dev_conf.intr_conf.rxq != 0) { + if (rte_intr_cap_multiple(intr_handle) && + dev->data->dev_conf.intr_conf.rxq) { intr_vector = dev->data->nb_rx_queues; ret = rte_intr_efd_enable(intr_handle, intr_vector); if (ret) @@ -3063,7 +3314,7 @@ igbvf_dev_start(struct rte_eth_dev *dev) dev->data->nb_rx_queues * sizeof(int), 0); if (!intr_handle->intr_vec) { PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues" - " intr_vec\n", dev->data->nb_rx_queues); + " intr_vec", dev->data->nb_rx_queues); return -ENOMEM; } } @@ -3082,7 +3333,8 @@ igbvf_dev_start(struct rte_eth_dev *dev) static void igbvf_dev_stop(struct rte_eth_dev *dev) { - struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; PMD_INIT_FUNC_TRACE(); @@ -3281,7 +3533,7 @@ eth_igb_rss_reta_update(struct rte_eth_dev *dev, if (reta_size != ETH_RSS_RETA_SIZE_128) { PMD_DRV_LOG(ERR, "The size of hash lookup table configured " "(%d) doesn't match the number hardware can supported " - "(%d)\n", reta_size, ETH_RSS_RETA_SIZE_128); + "(%d)", reta_size, ETH_RSS_RETA_SIZE_128); return -EINVAL; } @@ -3322,7 +3574,7 @@ eth_igb_rss_reta_query(struct rte_eth_dev *dev, if (reta_size != ETH_RSS_RETA_SIZE_128) { PMD_DRV_LOG(ERR, "The size of hash lookup table configured " "(%d) doesn't match the number hardware can supported " - "(%d)\n", reta_size, ETH_RSS_RETA_SIZE_128); + "(%d)", reta_size, ETH_RSS_RETA_SIZE_128); return -EINVAL; } @@ -3345,18 +3597,14 @@ eth_igb_rss_reta_query(struct rte_eth_dev *dev, return 0; } -#define MAC_TYPE_FILTER_SUP(type) do {\ - if ((type) != e1000_82580 && (type) != e1000_i350 &&\ - (type) != e1000_82576)\ - return -ENOTSUP;\ -} while (0) - -static int +int eth_igb_syn_filter_set(struct rte_eth_dev *dev, struct rte_eth_syn_filter *filter, bool add) { struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct e1000_filter_info *filter_info = + E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); uint32_t synqf, rfctl; if (filter->queue >= IGB_MAX_RX_QUEUE_NUM) @@ -3384,6 +3632,7 @@ eth_igb_syn_filter_set(struct rte_eth_dev *dev, synqf = 0; } + filter_info->syn_info = synqf; E1000_WRITE_REG(hw, E1000_SYNQF(0), synqf); E1000_WRITE_FLUSH(hw); return 0; @@ -3443,7 +3692,7 @@ eth_igb_syn_filter_handle(struct rte_eth_dev *dev, (struct rte_eth_syn_filter *)arg); break; default: - PMD_DRV_LOG(ERR, "unsupported operation %u\n", filter_op); + PMD_DRV_LOG(ERR, "unsupported operation %u", filter_op); ret = -EINVAL; break; } @@ -3451,11 +3700,6 @@ eth_igb_syn_filter_handle(struct rte_eth_dev *dev, return ret; } -#define MAC_TYPE_FILTER_SUP_EXT(type) do {\ - if ((type) != e1000_82580 && (type) != e1000_i350)\ - return -ENOSYS; \ -} while (0) - /* translate elements in struct rte_eth_ntuple_filter to struct e1000_2tuple_filter_info*/ static inline int ntuple_filter_to_2tuple(struct rte_eth_ntuple_filter *filter, @@ -3518,6 +3762,54 @@ igb_2tuple_filter_lookup(struct e1000_2tuple_filter_list *filter_list, return NULL; } +/* inject a igb 2tuple filter to HW */ +static inline void +igb_inject_2uple_filter(struct rte_eth_dev *dev, + struct e1000_2tuple_filter *filter) +{ + struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + uint32_t ttqf = E1000_TTQF_DISABLE_MASK; + uint32_t imir, imir_ext = E1000_IMIREXT_SIZE_BP; + int i; + + i = filter->index; + imir = (uint32_t)(filter->filter_info.dst_port & E1000_IMIR_DSTPORT); + if (filter->filter_info.dst_port_mask == 1) /* 1b means not compare. */ + imir |= E1000_IMIR_PORT_BP; + else + imir &= ~E1000_IMIR_PORT_BP; + + imir |= filter->filter_info.priority << E1000_IMIR_PRIORITY_SHIFT; + + ttqf |= E1000_TTQF_QUEUE_ENABLE; + ttqf |= (uint32_t)(filter->queue << E1000_TTQF_QUEUE_SHIFT); + ttqf |= (uint32_t)(filter->filter_info.proto & + E1000_TTQF_PROTOCOL_MASK); + if (filter->filter_info.proto_mask == 0) + ttqf &= ~E1000_TTQF_MASK_ENABLE; + + /* tcp flags bits setting. */ + if (filter->filter_info.tcp_flags & TCP_FLAG_ALL) { + if (filter->filter_info.tcp_flags & TCP_URG_FLAG) + imir_ext |= E1000_IMIREXT_CTRL_URG; + if (filter->filter_info.tcp_flags & TCP_ACK_FLAG) + imir_ext |= E1000_IMIREXT_CTRL_ACK; + if (filter->filter_info.tcp_flags & TCP_PSH_FLAG) + imir_ext |= E1000_IMIREXT_CTRL_PSH; + if (filter->filter_info.tcp_flags & TCP_RST_FLAG) + imir_ext |= E1000_IMIREXT_CTRL_RST; + if (filter->filter_info.tcp_flags & TCP_SYN_FLAG) + imir_ext |= E1000_IMIREXT_CTRL_SYN; + if (filter->filter_info.tcp_flags & TCP_FIN_FLAG) + imir_ext |= E1000_IMIREXT_CTRL_FIN; + } else { + imir_ext |= E1000_IMIREXT_CTRL_BP; + } + E1000_WRITE_REG(hw, E1000_IMIR(i), imir); + E1000_WRITE_REG(hw, E1000_TTQF(i), ttqf); + E1000_WRITE_REG(hw, E1000_IMIREXT(i), imir_ext); +} + /* * igb_add_2tuple_filter - add a 2tuple filter * @@ -3533,12 +3825,9 @@ static int igb_add_2tuple_filter(struct rte_eth_dev *dev, struct rte_eth_ntuple_filter *ntuple_filter) { - struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct e1000_filter_info *filter_info = E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); struct e1000_2tuple_filter *filter; - uint32_t ttqf = E1000_TTQF_DISABLE_MASK; - uint32_t imir, imir_ext = E1000_IMIREXT_SIZE_BP; int i, ret; filter = rte_zmalloc("e1000_2tuple_filter", @@ -3580,39 +3869,25 @@ igb_add_2tuple_filter(struct rte_eth_dev *dev, return -ENOSYS; } - imir = (uint32_t)(filter->filter_info.dst_port & E1000_IMIR_DSTPORT); - if (filter->filter_info.dst_port_mask == 1) /* 1b means not compare. */ - imir |= E1000_IMIR_PORT_BP; - else - imir &= ~E1000_IMIR_PORT_BP; + igb_inject_2uple_filter(dev, filter); + return 0; +} - imir |= filter->filter_info.priority << E1000_IMIR_PRIORITY_SHIFT; +int +igb_delete_2tuple_filter(struct rte_eth_dev *dev, + struct e1000_2tuple_filter *filter) +{ + struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct e1000_filter_info *filter_info = + E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); - ttqf |= E1000_TTQF_QUEUE_ENABLE; - ttqf |= (uint32_t)(filter->queue << E1000_TTQF_QUEUE_SHIFT); - ttqf |= (uint32_t)(filter->filter_info.proto & E1000_TTQF_PROTOCOL_MASK); - if (filter->filter_info.proto_mask == 0) - ttqf &= ~E1000_TTQF_MASK_ENABLE; + filter_info->twotuple_mask &= ~(1 << filter->index); + TAILQ_REMOVE(&filter_info->twotuple_list, filter, entries); + rte_free(filter); - /* tcp flags bits setting. */ - if (filter->filter_info.tcp_flags & TCP_FLAG_ALL) { - if (filter->filter_info.tcp_flags & TCP_URG_FLAG) - imir_ext |= E1000_IMIREXT_CTRL_URG; - if (filter->filter_info.tcp_flags & TCP_ACK_FLAG) - imir_ext |= E1000_IMIREXT_CTRL_ACK; - if (filter->filter_info.tcp_flags & TCP_PSH_FLAG) - imir_ext |= E1000_IMIREXT_CTRL_PSH; - if (filter->filter_info.tcp_flags & TCP_RST_FLAG) - imir_ext |= E1000_IMIREXT_CTRL_RST; - if (filter->filter_info.tcp_flags & TCP_SYN_FLAG) - imir_ext |= E1000_IMIREXT_CTRL_SYN; - if (filter->filter_info.tcp_flags & TCP_FIN_FLAG) - imir_ext |= E1000_IMIREXT_CTRL_FIN; - } else - imir_ext |= E1000_IMIREXT_CTRL_BP; - E1000_WRITE_REG(hw, E1000_IMIR(i), imir); - E1000_WRITE_REG(hw, E1000_TTQF(i), ttqf); - E1000_WRITE_REG(hw, E1000_IMIREXT(i), imir_ext); + E1000_WRITE_REG(hw, E1000_TTQF(filter->index), E1000_TTQF_DISABLE_MASK); + E1000_WRITE_REG(hw, E1000_IMIR(filter->index), 0); + E1000_WRITE_REG(hw, E1000_IMIREXT(filter->index), 0); return 0; } @@ -3631,7 +3906,6 @@ static int igb_remove_2tuple_filter(struct rte_eth_dev *dev, struct rte_eth_ntuple_filter *ntuple_filter) { - struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct e1000_filter_info *filter_info = E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); struct e1000_2tuple_filter_info filter_2tuple; @@ -3651,16 +3925,50 @@ igb_remove_2tuple_filter(struct rte_eth_dev *dev, return -ENOENT; } - filter_info->twotuple_mask &= ~(1 << filter->index); - TAILQ_REMOVE(&filter_info->twotuple_list, filter, entries); - rte_free(filter); + igb_delete_2tuple_filter(dev, filter); - E1000_WRITE_REG(hw, E1000_TTQF(filter->index), E1000_TTQF_DISABLE_MASK); - E1000_WRITE_REG(hw, E1000_IMIR(filter->index), 0); - E1000_WRITE_REG(hw, E1000_IMIREXT(filter->index), 0); return 0; } +/* inject a igb flex filter to HW */ +static inline void +igb_inject_flex_filter(struct rte_eth_dev *dev, + struct e1000_flex_filter *filter) +{ + struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + uint32_t wufc, queueing; + uint32_t reg_off; + uint8_t i, j = 0; + + wufc = E1000_READ_REG(hw, E1000_WUFC); + if (filter->index < E1000_MAX_FHFT) + reg_off = E1000_FHFT(filter->index); + else + reg_off = E1000_FHFT_EXT(filter->index - E1000_MAX_FHFT); + + E1000_WRITE_REG(hw, E1000_WUFC, wufc | E1000_WUFC_FLEX_HQ | + (E1000_WUFC_FLX0 << filter->index)); + queueing = filter->filter_info.len | + (filter->queue << E1000_FHFT_QUEUEING_QUEUE_SHIFT) | + (filter->filter_info.priority << + E1000_FHFT_QUEUEING_PRIO_SHIFT); + E1000_WRITE_REG(hw, reg_off + E1000_FHFT_QUEUEING_OFFSET, + queueing); + + for (i = 0; i < E1000_FLEX_FILTERS_MASK_SIZE; i++) { + E1000_WRITE_REG(hw, reg_off, + filter->filter_info.dwords[j]); + reg_off += sizeof(uint32_t); + E1000_WRITE_REG(hw, reg_off, + filter->filter_info.dwords[++j]); + reg_off += sizeof(uint32_t); + E1000_WRITE_REG(hw, reg_off, + (uint32_t)filter->filter_info.mask[i]); + reg_off += sizeof(uint32_t) * 2; + ++j; + } +} + static inline struct e1000_flex_filter * eth_igb_flex_filter_lookup(struct e1000_flex_filter_list *filter_list, struct e1000_flex_filter_info *key) @@ -3676,18 +3984,48 @@ eth_igb_flex_filter_lookup(struct e1000_flex_filter_list *filter_list, return NULL; } -static int +/* remove a flex byte filter + * @param + * dev: Pointer to struct rte_eth_dev. + * filter: the pointer of the filter will be removed. + */ +void +igb_remove_flex_filter(struct rte_eth_dev *dev, + struct e1000_flex_filter *filter) +{ + struct e1000_filter_info *filter_info = + E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); + struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + uint32_t wufc, i; + uint32_t reg_off; + + wufc = E1000_READ_REG(hw, E1000_WUFC); + if (filter->index < E1000_MAX_FHFT) + reg_off = E1000_FHFT(filter->index); + else + reg_off = E1000_FHFT_EXT(filter->index - E1000_MAX_FHFT); + + for (i = 0; i < E1000_FHFT_SIZE_IN_DWD; i++) + E1000_WRITE_REG(hw, reg_off + i * sizeof(uint32_t), 0); + + E1000_WRITE_REG(hw, E1000_WUFC, wufc & + (~(E1000_WUFC_FLX0 << filter->index))); + + filter_info->flex_mask &= ~(1 << filter->index); + TAILQ_REMOVE(&filter_info->flex_list, filter, entries); + rte_free(filter); +} + +int eth_igb_add_del_flex_filter(struct rte_eth_dev *dev, struct rte_eth_flex_filter *filter, bool add) { - struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct e1000_filter_info *filter_info = E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); struct e1000_flex_filter *flex_filter, *it; - uint32_t wufc, queueing, mask; - uint32_t reg_off; - uint8_t shift, i, j = 0; + uint32_t mask; + uint8_t shift, i; flex_filter = rte_zmalloc("e1000_flex_filter", sizeof(struct e1000_flex_filter), 0); @@ -3707,19 +4045,20 @@ eth_igb_add_del_flex_filter(struct rte_eth_dev *dev, flex_filter->filter_info.mask[i] = mask; } - wufc = E1000_READ_REG(hw, E1000_WUFC); - if (flex_filter->index < E1000_MAX_FHFT) - reg_off = E1000_FHFT(flex_filter->index); - else - reg_off = E1000_FHFT_EXT(flex_filter->index - E1000_MAX_FHFT); + it = eth_igb_flex_filter_lookup(&filter_info->flex_list, + &flex_filter->filter_info); + if (it == NULL && !add) { + PMD_DRV_LOG(ERR, "filter doesn't exist."); + rte_free(flex_filter); + return -ENOENT; + } + if (it != NULL && add) { + PMD_DRV_LOG(ERR, "filter exists."); + rte_free(flex_filter); + return -EEXIST; + } if (add) { - if (eth_igb_flex_filter_lookup(&filter_info->flex_list, - &flex_filter->filter_info) != NULL) { - PMD_DRV_LOG(ERR, "filter exists."); - rte_free(flex_filter); - return -EEXIST; - } flex_filter->queue = filter->queue; /* * look for an unused flex filter index @@ -3741,42 +4080,10 @@ eth_igb_add_del_flex_filter(struct rte_eth_dev *dev, return -ENOSYS; } - E1000_WRITE_REG(hw, E1000_WUFC, wufc | E1000_WUFC_FLEX_HQ | - (E1000_WUFC_FLX0 << flex_filter->index)); - queueing = filter->len | - (filter->queue << E1000_FHFT_QUEUEING_QUEUE_SHIFT) | - (filter->priority << E1000_FHFT_QUEUEING_PRIO_SHIFT); - E1000_WRITE_REG(hw, reg_off + E1000_FHFT_QUEUEING_OFFSET, - queueing); - for (i = 0; i < E1000_FLEX_FILTERS_MASK_SIZE; i++) { - E1000_WRITE_REG(hw, reg_off, - flex_filter->filter_info.dwords[j]); - reg_off += sizeof(uint32_t); - E1000_WRITE_REG(hw, reg_off, - flex_filter->filter_info.dwords[++j]); - reg_off += sizeof(uint32_t); - E1000_WRITE_REG(hw, reg_off, - (uint32_t)flex_filter->filter_info.mask[i]); - reg_off += sizeof(uint32_t) * 2; - ++j; - } + igb_inject_flex_filter(dev, flex_filter); + } else { - it = eth_igb_flex_filter_lookup(&filter_info->flex_list, - &flex_filter->filter_info); - if (it == NULL) { - PMD_DRV_LOG(ERR, "filter doesn't exist."); - rte_free(flex_filter); - return -ENOENT; - } - - for (i = 0; i < E1000_FHFT_SIZE_IN_DWD; i++) - E1000_WRITE_REG(hw, reg_off + i * sizeof(uint32_t), 0); - E1000_WRITE_REG(hw, E1000_WUFC, wufc & - (~(E1000_WUFC_FLX0 << it->index))); - - filter_info->flex_mask &= ~(1 << it->index); - TAILQ_REMOVE(&filter_info->flex_list, it, entries); - rte_free(it); + igb_remove_flex_filter(dev, it); rte_free(flex_filter); } @@ -3798,7 +4105,7 @@ eth_igb_get_flex_filter(struct rte_eth_dev *dev, flex_filter.filter_info.priority = filter->priority; memcpy(flex_filter.filter_info.dwords, filter->bytes, filter->len); memcpy(flex_filter.filter_info.mask, filter->mask, - RTE_ALIGN(filter->len, sizeof(char)) / sizeof(char)); + RTE_ALIGN(filter->len, CHAR_BIT) / CHAR_BIT); it = eth_igb_flex_filter_lookup(&filter_info->flex_list, &flex_filter.filter_info); @@ -3980,6 +4287,64 @@ igb_5tuple_filter_lookup_82576(struct e1000_5tuple_filter_list *filter_list, return NULL; } +/* inject a igb 5-tuple filter to HW */ +static inline void +igb_inject_5tuple_filter_82576(struct rte_eth_dev *dev, + struct e1000_5tuple_filter *filter) +{ + struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + uint32_t ftqf = E1000_FTQF_VF_BP | E1000_FTQF_MASK; + uint32_t spqf, imir, imir_ext = E1000_IMIREXT_SIZE_BP; + uint8_t i; + + i = filter->index; + ftqf |= filter->filter_info.proto & E1000_FTQF_PROTOCOL_MASK; + if (filter->filter_info.src_ip_mask == 0) /* 0b means compare. */ + ftqf &= ~E1000_FTQF_MASK_SOURCE_ADDR_BP; + if (filter->filter_info.dst_ip_mask == 0) + ftqf &= ~E1000_FTQF_MASK_DEST_ADDR_BP; + if (filter->filter_info.src_port_mask == 0) + ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP; + if (filter->filter_info.proto_mask == 0) + ftqf &= ~E1000_FTQF_MASK_PROTO_BP; + ftqf |= (filter->queue << E1000_FTQF_QUEUE_SHIFT) & + E1000_FTQF_QUEUE_MASK; + ftqf |= E1000_FTQF_QUEUE_ENABLE; + E1000_WRITE_REG(hw, E1000_FTQF(i), ftqf); + E1000_WRITE_REG(hw, E1000_DAQF(i), filter->filter_info.dst_ip); + E1000_WRITE_REG(hw, E1000_SAQF(i), filter->filter_info.src_ip); + + spqf = filter->filter_info.src_port & E1000_SPQF_SRCPORT; + E1000_WRITE_REG(hw, E1000_SPQF(i), spqf); + + imir = (uint32_t)(filter->filter_info.dst_port & E1000_IMIR_DSTPORT); + if (filter->filter_info.dst_port_mask == 1) /* 1b means not compare. */ + imir |= E1000_IMIR_PORT_BP; + else + imir &= ~E1000_IMIR_PORT_BP; + imir |= filter->filter_info.priority << E1000_IMIR_PRIORITY_SHIFT; + + /* tcp flags bits setting. */ + if (filter->filter_info.tcp_flags & TCP_FLAG_ALL) { + if (filter->filter_info.tcp_flags & TCP_URG_FLAG) + imir_ext |= E1000_IMIREXT_CTRL_URG; + if (filter->filter_info.tcp_flags & TCP_ACK_FLAG) + imir_ext |= E1000_IMIREXT_CTRL_ACK; + if (filter->filter_info.tcp_flags & TCP_PSH_FLAG) + imir_ext |= E1000_IMIREXT_CTRL_PSH; + if (filter->filter_info.tcp_flags & TCP_RST_FLAG) + imir_ext |= E1000_IMIREXT_CTRL_RST; + if (filter->filter_info.tcp_flags & TCP_SYN_FLAG) + imir_ext |= E1000_IMIREXT_CTRL_SYN; + if (filter->filter_info.tcp_flags & TCP_FIN_FLAG) + imir_ext |= E1000_IMIREXT_CTRL_FIN; + } else { + imir_ext |= E1000_IMIREXT_CTRL_BP; + } + E1000_WRITE_REG(hw, E1000_IMIR(i), imir); + E1000_WRITE_REG(hw, E1000_IMIREXT(i), imir_ext); +} + /* * igb_add_5tuple_filter_82576 - add a 5tuple filter * @@ -3995,12 +4360,9 @@ static int igb_add_5tuple_filter_82576(struct rte_eth_dev *dev, struct rte_eth_ntuple_filter *ntuple_filter) { - struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct e1000_filter_info *filter_info = E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); struct e1000_5tuple_filter *filter; - uint32_t ftqf = E1000_FTQF_VF_BP | E1000_FTQF_MASK; - uint32_t spqf, imir, imir_ext = E1000_IMIREXT_SIZE_BP; uint8_t i; int ret; @@ -4044,50 +4406,29 @@ igb_add_5tuple_filter_82576(struct rte_eth_dev *dev, return -ENOSYS; } - ftqf |= filter->filter_info.proto & E1000_FTQF_PROTOCOL_MASK; - if (filter->filter_info.src_ip_mask == 0) /* 0b means compare. */ - ftqf &= ~E1000_FTQF_MASK_SOURCE_ADDR_BP; - if (filter->filter_info.dst_ip_mask == 0) - ftqf &= ~E1000_FTQF_MASK_DEST_ADDR_BP; - if (filter->filter_info.src_port_mask == 0) - ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP; - if (filter->filter_info.proto_mask == 0) - ftqf &= ~E1000_FTQF_MASK_PROTO_BP; - ftqf |= (filter->queue << E1000_FTQF_QUEUE_SHIFT) & - E1000_FTQF_QUEUE_MASK; - ftqf |= E1000_FTQF_QUEUE_ENABLE; - E1000_WRITE_REG(hw, E1000_FTQF(i), ftqf); - E1000_WRITE_REG(hw, E1000_DAQF(i), filter->filter_info.dst_ip); - E1000_WRITE_REG(hw, E1000_SAQF(i), filter->filter_info.src_ip); + igb_inject_5tuple_filter_82576(dev, filter); + return 0; +} - spqf = filter->filter_info.src_port & E1000_SPQF_SRCPORT; - E1000_WRITE_REG(hw, E1000_SPQF(i), spqf); +int +igb_delete_5tuple_filter_82576(struct rte_eth_dev *dev, + struct e1000_5tuple_filter *filter) +{ + struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct e1000_filter_info *filter_info = + E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); - imir = (uint32_t)(filter->filter_info.dst_port & E1000_IMIR_DSTPORT); - if (filter->filter_info.dst_port_mask == 1) /* 1b means not compare. */ - imir |= E1000_IMIR_PORT_BP; - else - imir &= ~E1000_IMIR_PORT_BP; - imir |= filter->filter_info.priority << E1000_IMIR_PRIORITY_SHIFT; + filter_info->fivetuple_mask &= ~(1 << filter->index); + TAILQ_REMOVE(&filter_info->fivetuple_list, filter, entries); + rte_free(filter); - /* tcp flags bits setting. */ - if (filter->filter_info.tcp_flags & TCP_FLAG_ALL) { - if (filter->filter_info.tcp_flags & TCP_URG_FLAG) - imir_ext |= E1000_IMIREXT_CTRL_URG; - if (filter->filter_info.tcp_flags & TCP_ACK_FLAG) - imir_ext |= E1000_IMIREXT_CTRL_ACK; - if (filter->filter_info.tcp_flags & TCP_PSH_FLAG) - imir_ext |= E1000_IMIREXT_CTRL_PSH; - if (filter->filter_info.tcp_flags & TCP_RST_FLAG) - imir_ext |= E1000_IMIREXT_CTRL_RST; - if (filter->filter_info.tcp_flags & TCP_SYN_FLAG) - imir_ext |= E1000_IMIREXT_CTRL_SYN; - if (filter->filter_info.tcp_flags & TCP_FIN_FLAG) - imir_ext |= E1000_IMIREXT_CTRL_FIN; - } else - imir_ext |= E1000_IMIREXT_CTRL_BP; - E1000_WRITE_REG(hw, E1000_IMIR(i), imir); - E1000_WRITE_REG(hw, E1000_IMIREXT(i), imir_ext); + E1000_WRITE_REG(hw, E1000_FTQF(filter->index), + E1000_FTQF_VF_BP | E1000_FTQF_MASK); + E1000_WRITE_REG(hw, E1000_DAQF(filter->index), 0); + E1000_WRITE_REG(hw, E1000_SAQF(filter->index), 0); + E1000_WRITE_REG(hw, E1000_SPQF(filter->index), 0); + E1000_WRITE_REG(hw, E1000_IMIR(filter->index), 0); + E1000_WRITE_REG(hw, E1000_IMIREXT(filter->index), 0); return 0; } @@ -4106,7 +4447,6 @@ static int igb_remove_5tuple_filter_82576(struct rte_eth_dev *dev, struct rte_eth_ntuple_filter *ntuple_filter) { - struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct e1000_filter_info *filter_info = E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); struct e1000_5tuple_filter_info filter_5tuple; @@ -4126,17 +4466,8 @@ igb_remove_5tuple_filter_82576(struct rte_eth_dev *dev, return -ENOENT; } - filter_info->fivetuple_mask &= ~(1 << filter->index); - TAILQ_REMOVE(&filter_info->fivetuple_list, filter, entries); - rte_free(filter); + igb_delete_5tuple_filter_82576(dev, filter); - E1000_WRITE_REG(hw, E1000_FTQF(filter->index), - E1000_FTQF_VF_BP | E1000_FTQF_MASK); - E1000_WRITE_REG(hw, E1000_DAQF(filter->index), 0); - E1000_WRITE_REG(hw, E1000_SAQF(filter->index), 0); - E1000_WRITE_REG(hw, E1000_SPQF(filter->index), 0); - E1000_WRITE_REG(hw, E1000_IMIR(filter->index), 0); - E1000_WRITE_REG(hw, E1000_IMIREXT(filter->index), 0); return 0; } @@ -4202,7 +4533,7 @@ eth_igb_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) * - On success, zero. * - On failure, a negative value. */ -static int +int igb_add_del_ntuple_filter(struct rte_eth_dev *dev, struct rte_eth_ntuple_filter *ntuple_filter, bool add) @@ -4224,7 +4555,9 @@ igb_add_del_ntuple_filter(struct rte_eth_dev *dev, break; case RTE_2TUPLE_FLAGS: case (RTE_2TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG): - if (hw->mac.type != e1000_82580 && hw->mac.type != e1000_i350) + if (hw->mac.type != e1000_82580 && hw->mac.type != e1000_i350 && + hw->mac.type != e1000_i210 && + hw->mac.type != e1000_i211) return -ENOTSUP; if (add) ret = igb_add_2tuple_filter(dev, ntuple_filter); @@ -4366,7 +4699,7 @@ igb_ethertype_filter_lookup(struct e1000_filter_info *filter_info, int i; for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) { - if (filter_info->ethertype_filters[i] == ethertype && + if (filter_info->ethertype_filters[i].ethertype == ethertype && (filter_info->ethertype_mask & (1 << i))) return i; } @@ -4375,33 +4708,35 @@ igb_ethertype_filter_lookup(struct e1000_filter_info *filter_info, static inline int igb_ethertype_filter_insert(struct e1000_filter_info *filter_info, - uint16_t ethertype) + uint16_t ethertype, uint32_t etqf) { int i; for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) { if (!(filter_info->ethertype_mask & (1 << i))) { filter_info->ethertype_mask |= 1 << i; - filter_info->ethertype_filters[i] = ethertype; + filter_info->ethertype_filters[i].ethertype = ethertype; + filter_info->ethertype_filters[i].etqf = etqf; return i; } } return -1; } -static inline int +int igb_ethertype_filter_remove(struct e1000_filter_info *filter_info, uint8_t idx) { if (idx >= E1000_MAX_ETQF_FILTERS) return -1; filter_info->ethertype_mask &= ~(1 << idx); - filter_info->ethertype_filters[idx] = 0; + filter_info->ethertype_filters[idx].ethertype = 0; + filter_info->ethertype_filters[idx].etqf = 0; return idx; } -static int +int igb_add_del_ethertype_filter(struct rte_eth_dev *dev, struct rte_eth_ethertype_filter *filter, bool add) @@ -4441,16 +4776,15 @@ igb_add_del_ethertype_filter(struct rte_eth_dev *dev, } if (add) { + etqf |= E1000_ETQF_FILTER_ENABLE | E1000_ETQF_QUEUE_ENABLE; + etqf |= (uint32_t)(filter->ether_type & E1000_ETQF_ETHERTYPE); + etqf |= filter->queue << E1000_ETQF_QUEUE_SHIFT; ret = igb_ethertype_filter_insert(filter_info, - filter->ether_type); + filter->ether_type, etqf); if (ret < 0) { PMD_DRV_LOG(ERR, "ethertype filters are full."); return -ENOSYS; } - - etqf |= E1000_ETQF_FILTER_ENABLE | E1000_ETQF_QUEUE_ENABLE; - etqf |= (uint32_t)(filter->ether_type & E1000_ETQF_ETHERTYPE); - etqf |= filter->queue << E1000_ETQF_QUEUE_SHIFT; } else { ret = igb_ethertype_filter_remove(filter_info, (uint8_t)ret); if (ret < 0) @@ -4545,7 +4879,7 @@ eth_igb_filter_ctrl(struct rte_eth_dev *dev, enum rte_filter_op filter_op, void *arg) { - int ret = -EINVAL; + int ret = 0; switch (filter_type) { case RTE_ETH_FILTER_NTUPLE: @@ -4560,6 +4894,11 @@ eth_igb_filter_ctrl(struct rte_eth_dev *dev, case RTE_ETH_FILTER_FLEXIBLE: ret = eth_igb_flex_filter_handle(dev, filter_op, arg); break; + case RTE_ETH_FILTER_GENERIC: + if (filter_op != RTE_ETH_FILTER_GET) + return -EINVAL; + *(const void **)arg = &igb_flow_ops; + break; default: PMD_DRV_LOG(WARNING, "Filter type (%d) not supported", filter_type); @@ -5049,22 +5388,19 @@ eth_igb_set_eeprom(struct rte_eth_dev *dev, return nvm->ops.write(hw, first, length, data); } -static struct rte_driver pmd_igb_drv = { - .type = PMD_PDEV, - .init = rte_igb_pmd_init, -}; - -static struct rte_driver pmd_igbvf_drv = { - .type = PMD_PDEV, - .init = rte_igbvf_pmd_init, -}; - static int eth_igb_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id) { struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); - uint32_t mask = 1 << queue_id; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; + uint32_t vec = E1000_MISC_VEC_ID; + + if (rte_intr_allow_others(intr_handle)) + vec = E1000_RX_VEC_START; + + uint32_t mask = 1 << (queue_id + vec); E1000_WRITE_REG(hw, E1000_EIMC, mask); E1000_WRITE_FLUSH(hw); @@ -5077,14 +5413,21 @@ eth_igb_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) { struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); - uint32_t mask = 1 << queue_id; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; + uint32_t vec = E1000_MISC_VEC_ID; + + if (rte_intr_allow_others(intr_handle)) + vec = E1000_RX_VEC_START; + + uint32_t mask = 1 << (queue_id + vec); uint32_t regval; regval = E1000_READ_REG(hw, E1000_EIMS); E1000_WRITE_REG(hw, E1000_EIMS, regval | mask); E1000_WRITE_FLUSH(hw); - rte_intr_enable(&dev->pci_dev->intr_handle); + rte_intr_enable(intr_handle); return 0; } @@ -5148,8 +5491,8 @@ eth_igb_configure_msix_intr(struct rte_eth_dev *dev) uint32_t vec = E1000_MISC_VEC_ID; uint32_t base = E1000_MISC_VEC_ID; uint32_t misc_shift = 0; - - struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; /* won't configure msix register if no mapping is done * between intr vector and event fd @@ -5220,7 +5563,87 @@ eth_igb_configure_msix_intr(struct rte_eth_dev *dev) E1000_WRITE_FLUSH(hw); } -PMD_REGISTER_DRIVER(pmd_igb_drv, igb); -DRIVER_REGISTER_PCI_TABLE(igb, pci_id_igb_map); -PMD_REGISTER_DRIVER(pmd_igbvf_drv, igbvf); -DRIVER_REGISTER_PCI_TABLE(igbvf, pci_id_igbvf_map); +/* restore n-tuple filter */ +static inline void +igb_ntuple_filter_restore(struct rte_eth_dev *dev) +{ + struct e1000_filter_info *filter_info = + E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); + struct e1000_5tuple_filter *p_5tuple; + struct e1000_2tuple_filter *p_2tuple; + + TAILQ_FOREACH(p_5tuple, &filter_info->fivetuple_list, entries) { + igb_inject_5tuple_filter_82576(dev, p_5tuple); + } + + TAILQ_FOREACH(p_2tuple, &filter_info->twotuple_list, entries) { + igb_inject_2uple_filter(dev, p_2tuple); + } +} + +/* restore SYN filter */ +static inline void +igb_syn_filter_restore(struct rte_eth_dev *dev) +{ + struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct e1000_filter_info *filter_info = + E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); + uint32_t synqf; + + synqf = filter_info->syn_info; + + if (synqf & E1000_SYN_FILTER_ENABLE) { + E1000_WRITE_REG(hw, E1000_SYNQF(0), synqf); + E1000_WRITE_FLUSH(hw); + } +} + +/* restore ethernet type filter */ +static inline void +igb_ethertype_filter_restore(struct rte_eth_dev *dev) +{ + struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct e1000_filter_info *filter_info = + E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); + int i; + + for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) { + if (filter_info->ethertype_mask & (1 << i)) { + E1000_WRITE_REG(hw, E1000_ETQF(i), + filter_info->ethertype_filters[i].etqf); + E1000_WRITE_FLUSH(hw); + } + } +} + +/* restore flex byte filter */ +static inline void +igb_flex_filter_restore(struct rte_eth_dev *dev) +{ + struct e1000_filter_info *filter_info = + E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); + struct e1000_flex_filter *flex_filter; + + TAILQ_FOREACH(flex_filter, &filter_info->flex_list, entries) { + igb_inject_flex_filter(dev, flex_filter); + } +} + +/* restore all types filter */ +static int +igb_filter_restore(struct rte_eth_dev *dev) +{ + igb_ntuple_filter_restore(dev); + igb_ethertype_filter_restore(dev); + igb_syn_filter_restore(dev); + igb_flex_filter_restore(dev); + + return 0; +} + +RTE_PMD_REGISTER_PCI(net_e1000_igb, rte_igb_pmd); +RTE_PMD_REGISTER_PCI_TABLE(net_e1000_igb, pci_id_igb_map); +RTE_PMD_REGISTER_KMOD_DEP(net_e1000_igb, "* igb_uio | uio_pci_generic | vfio-pci"); +RTE_PMD_REGISTER_PCI(net_e1000_igb_vf, rte_igbvf_pmd); +RTE_PMD_REGISTER_PCI_TABLE(net_e1000_igb_vf, pci_id_igbvf_map); +RTE_PMD_REGISTER_KMOD_DEP(net_e1000_igb_vf, "* igb_uio | vfio-pci"); diff --git a/dpdk/drivers/net/e1000/igb_flow.c b/dpdk/drivers/net/e1000/igb_flow.c new file mode 100644 index 00000000..057579b3 --- /dev/null +++ b/dpdk/drivers/net/e1000/igb_flow.c @@ -0,0 +1,1726 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "e1000_logs.h" +#include "base/e1000_api.h" +#include "e1000_ethdev.h" + +#define NEXT_ITEM_OF_PATTERN(item, pattern, index) \ + do { \ + item = (pattern) + (index); \ + while (item->type == RTE_FLOW_ITEM_TYPE_VOID) { \ + (index)++; \ + item = (pattern) + (index); \ + } \ + } while (0) + +#define NEXT_ITEM_OF_ACTION(act, actions, index) \ + do { \ + act = (actions) + (index); \ + while (act->type == RTE_FLOW_ACTION_TYPE_VOID) {\ + (index)++; \ + act = (actions) + (index); \ + } \ + } while (0) + +#define IGB_FLEX_RAW_NUM 12 + +/** + * Please aware there's an asumption for all the parsers. + * rte_flow_item is using big endian, rte_flow_attr and + * rte_flow_action are using CPU order. + * Because the pattern is used to describe the packets, + * normally the packets should use network order. + */ + +/** + * Parse the rule to see if it is a n-tuple rule. + * And get the n-tuple filter info BTW. + * pattern: + * The first not void item can be ETH or IPV4. + * The second not void item must be IPV4 if the first one is ETH. + * The third not void item must be UDP or TCP or SCTP + * The next not void item must be END. + * action: + * The first not void action should be QUEUE. + * The next not void action should be END. + * pattern example: + * ITEM Spec Mask + * ETH NULL NULL + * IPV4 src_addr 192.168.1.20 0xFFFFFFFF + * dst_addr 192.167.3.50 0xFFFFFFFF + * next_proto_id 17 0xFF + * UDP/TCP/ src_port 80 0xFFFF + * SCTP dst_port 80 0xFFFF + * END + * other members in mask and spec should set to 0x00. + * item->last should be NULL. + */ +static int +cons_parse_ntuple_filter(const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_eth_ntuple_filter *filter, + struct rte_flow_error *error) +{ + const struct rte_flow_item *item; + const struct rte_flow_action *act; + const struct rte_flow_item_ipv4 *ipv4_spec; + const struct rte_flow_item_ipv4 *ipv4_mask; + const struct rte_flow_item_tcp *tcp_spec; + const struct rte_flow_item_tcp *tcp_mask; + const struct rte_flow_item_udp *udp_spec; + const struct rte_flow_item_udp *udp_mask; + const struct rte_flow_item_sctp *sctp_spec; + const struct rte_flow_item_sctp *sctp_mask; + uint32_t index; + + if (!pattern) { + rte_flow_error_set(error, + EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_NUM, + NULL, "NULL pattern."); + return -rte_errno; + } + + if (!actions) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION_NUM, + NULL, "NULL action."); + return -rte_errno; + } + if (!attr) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR, + NULL, "NULL attribute."); + return -rte_errno; + } + + /* parse pattern */ + index = 0; + + /* the first not void item can be MAC or IPv4 */ + NEXT_ITEM_OF_PATTERN(item, pattern, index); + + if (item->type != RTE_FLOW_ITEM_TYPE_ETH && + item->type != RTE_FLOW_ITEM_TYPE_IPV4) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by ntuple filter"); + return -rte_errno; + } + /* Skip Ethernet */ + if (item->type == RTE_FLOW_ITEM_TYPE_ETH) { + /*Not supported last point for range*/ + if (item->last) { + rte_flow_error_set(error, + EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + item, "Not supported last point for range"); + return -rte_errno; + } + /* if the first item is MAC, the content should be NULL */ + if (item->spec || item->mask) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by ntuple filter"); + return -rte_errno; + } + /* check if the next not void item is IPv4 */ + index++; + NEXT_ITEM_OF_PATTERN(item, pattern, index); + if (item->type != RTE_FLOW_ITEM_TYPE_IPV4) { + rte_flow_error_set(error, + EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by ntuple filter"); + return -rte_errno; + } + } + + /* get the IPv4 info */ + if (!item->spec || !item->mask) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Invalid ntuple mask"); + return -rte_errno; + } + /* Not supported last point for range */ + if (item->last) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + item, "Not supported last point for range"); + return -rte_errno; + } + + ipv4_mask = (const struct rte_flow_item_ipv4 *)item->mask; + /** + * Only support src & dst addresses, protocol, + * others should be masked. + */ + + if (ipv4_mask->hdr.version_ihl || + ipv4_mask->hdr.type_of_service || + ipv4_mask->hdr.total_length || + ipv4_mask->hdr.packet_id || + ipv4_mask->hdr.fragment_offset || + ipv4_mask->hdr.time_to_live || + ipv4_mask->hdr.hdr_checksum) { + rte_flow_error_set(error, + EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by ntuple filter"); + return -rte_errno; + } + + filter->dst_ip_mask = ipv4_mask->hdr.dst_addr; + filter->src_ip_mask = ipv4_mask->hdr.src_addr; + filter->proto_mask = ipv4_mask->hdr.next_proto_id; + + ipv4_spec = (const struct rte_flow_item_ipv4 *)item->spec; + filter->dst_ip = ipv4_spec->hdr.dst_addr; + filter->src_ip = ipv4_spec->hdr.src_addr; + filter->proto = ipv4_spec->hdr.next_proto_id; + + /* check if the next not void item is TCP or UDP or SCTP */ + index++; + NEXT_ITEM_OF_PATTERN(item, pattern, index); + if (item->type != RTE_FLOW_ITEM_TYPE_TCP && + item->type != RTE_FLOW_ITEM_TYPE_UDP && + item->type != RTE_FLOW_ITEM_TYPE_SCTP) { + memset(filter, 0, sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by ntuple filter"); + return -rte_errno; + } + + /* Not supported last point for range */ + if (item->last) { + memset(filter, 0, sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + item, "Not supported last point for range"); + return -rte_errno; + } + + /* get the TCP/UDP/SCTP info */ + if (item->type == RTE_FLOW_ITEM_TYPE_TCP) { + if (item->spec && item->mask) { + tcp_mask = (const struct rte_flow_item_tcp *)item->mask; + + /** + * Only support src & dst ports, tcp flags, + * others should be masked. + */ + if (tcp_mask->hdr.sent_seq || + tcp_mask->hdr.recv_ack || + tcp_mask->hdr.data_off || + tcp_mask->hdr.rx_win || + tcp_mask->hdr.cksum || + tcp_mask->hdr.tcp_urp) { + memset(filter, 0, + sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by ntuple filter"); + return -rte_errno; + } + + filter->dst_port_mask = tcp_mask->hdr.dst_port; + filter->src_port_mask = tcp_mask->hdr.src_port; + if (tcp_mask->hdr.tcp_flags == 0xFF) { + filter->flags |= RTE_NTUPLE_FLAGS_TCP_FLAG; + } else if (!tcp_mask->hdr.tcp_flags) { + filter->flags &= ~RTE_NTUPLE_FLAGS_TCP_FLAG; + } else { + memset(filter, 0, + sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by ntuple filter"); + return -rte_errno; + } + + tcp_spec = (const struct rte_flow_item_tcp *)item->spec; + filter->dst_port = tcp_spec->hdr.dst_port; + filter->src_port = tcp_spec->hdr.src_port; + filter->tcp_flags = tcp_spec->hdr.tcp_flags; + } + } else if (item->type == RTE_FLOW_ITEM_TYPE_UDP) { + if (item->spec && item->mask) { + udp_mask = (const struct rte_flow_item_udp *)item->mask; + + /** + * Only support src & dst ports, + * others should be masked. + */ + if (udp_mask->hdr.dgram_len || + udp_mask->hdr.dgram_cksum) { + memset(filter, 0, + sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by ntuple filter"); + return -rte_errno; + } + + filter->dst_port_mask = udp_mask->hdr.dst_port; + filter->src_port_mask = udp_mask->hdr.src_port; + + udp_spec = (const struct rte_flow_item_udp *)item->spec; + filter->dst_port = udp_spec->hdr.dst_port; + filter->src_port = udp_spec->hdr.src_port; + } + } else { + if (item->spec && item->mask) { + sctp_mask = (const struct rte_flow_item_sctp *) + item->mask; + + /** + * Only support src & dst ports, + * others should be masked. + */ + if (sctp_mask->hdr.tag || + sctp_mask->hdr.cksum) { + memset(filter, 0, + sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by ntuple filter"); + return -rte_errno; + } + + filter->dst_port_mask = sctp_mask->hdr.dst_port; + filter->src_port_mask = sctp_mask->hdr.src_port; + + sctp_spec = (const struct rte_flow_item_sctp *) + item->spec; + filter->dst_port = sctp_spec->hdr.dst_port; + filter->src_port = sctp_spec->hdr.src_port; + } + } + /* check if the next not void item is END */ + index++; + NEXT_ITEM_OF_PATTERN(item, pattern, index); + if (item->type != RTE_FLOW_ITEM_TYPE_END) { + memset(filter, 0, sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by ntuple filter"); + return -rte_errno; + } + + /* parse action */ + index = 0; + + /** + * n-tuple only supports forwarding, + * check if the first not void action is QUEUE. + */ + NEXT_ITEM_OF_ACTION(act, actions, index); + if (act->type != RTE_FLOW_ACTION_TYPE_QUEUE) { + memset(filter, 0, sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + item, "Not supported action."); + return -rte_errno; + } + filter->queue = + ((const struct rte_flow_action_queue *)act->conf)->index; + + /* check if the next not void item is END */ + index++; + NEXT_ITEM_OF_ACTION(act, actions, index); + if (act->type != RTE_FLOW_ACTION_TYPE_END) { + memset(filter, 0, sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + act, "Not supported action."); + return -rte_errno; + } + + /* parse attr */ + /* must be input direction */ + if (!attr->ingress) { + memset(filter, 0, sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, + attr, "Only support ingress."); + return -rte_errno; + } + + /* not supported */ + if (attr->egress) { + memset(filter, 0, sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, + attr, "Not support egress."); + return -rte_errno; + } + + if (attr->priority > 0xFFFF) { + memset(filter, 0, sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, + attr, "Error priority."); + return -rte_errno; + } + filter->priority = (uint16_t)attr->priority; + + return 0; +} + +/* a specific function for igb because the flags is specific */ +static int +igb_parse_ntuple_filter(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_eth_ntuple_filter *filter, + struct rte_flow_error *error) +{ + struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + int ret; + + MAC_TYPE_FILTER_SUP(hw->mac.type); + + ret = cons_parse_ntuple_filter(attr, pattern, actions, filter, error); + + if (ret) + return ret; + + /* Igb doesn't support many priorities. */ + if (filter->priority > E1000_2TUPLE_MAX_PRI) { + memset(filter, 0, sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + NULL, "Priority not supported by ntuple filter"); + return -rte_errno; + } + + if (hw->mac.type == e1000_82576) { + if (filter->queue >= IGB_MAX_RX_QUEUE_NUM_82576) { + memset(filter, 0, sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + NULL, "queue number not " + "supported by ntuple filter"); + return -rte_errno; + } + filter->flags |= RTE_5TUPLE_FLAGS; + } else { + if (filter->src_ip_mask || filter->dst_ip_mask || + filter->src_port_mask) { + memset(filter, 0, sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + NULL, "only two tuple are " + "supported by this filter"); + return -rte_errno; + } + if (filter->queue >= IGB_MAX_RX_QUEUE_NUM) { + memset(filter, 0, sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + NULL, "queue number not " + "supported by ntuple filter"); + return -rte_errno; + } + filter->flags |= RTE_2TUPLE_FLAGS; + } + + return 0; +} + +/** + * Parse the rule to see if it is a ethertype rule. + * And get the ethertype filter info BTW. + * pattern: + * The first not void item can be ETH. + * The next not void item must be END. + * action: + * The first not void action should be QUEUE. + * The next not void action should be END. + * pattern example: + * ITEM Spec Mask + * ETH type 0x0807 0xFFFF + * END + * other members in mask and spec should set to 0x00. + * item->last should be NULL. + */ +static int +cons_parse_ethertype_filter(const struct rte_flow_attr *attr, + const struct rte_flow_item *pattern, + const struct rte_flow_action *actions, + struct rte_eth_ethertype_filter *filter, + struct rte_flow_error *error) +{ + const struct rte_flow_item *item; + const struct rte_flow_action *act; + const struct rte_flow_item_eth *eth_spec; + const struct rte_flow_item_eth *eth_mask; + const struct rte_flow_action_queue *act_q; + uint32_t index; + + if (!pattern) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM_NUM, + NULL, "NULL pattern."); + return -rte_errno; + } + + if (!actions) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION_NUM, + NULL, "NULL action."); + return -rte_errno; + } + + if (!attr) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR, + NULL, "NULL attribute."); + return -rte_errno; + } + + /* Parse pattern */ + index = 0; + + /* The first non-void item should be MAC. */ + NEXT_ITEM_OF_PATTERN(item, pattern, index); + if (item->type != RTE_FLOW_ITEM_TYPE_ETH) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by ethertype filter"); + return -rte_errno; + } + + /*Not supported last point for range*/ + if (item->last) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + item, "Not supported last point for range"); + return -rte_errno; + } + + /* Get the MAC info. */ + if (!item->spec || !item->mask) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by ethertype filter"); + return -rte_errno; + } + + eth_spec = (const struct rte_flow_item_eth *)item->spec; + eth_mask = (const struct rte_flow_item_eth *)item->mask; + + /* Mask bits of source MAC address must be full of 0. + * Mask bits of destination MAC address must be full + * of 1 or full of 0. + */ + if (!is_zero_ether_addr(ð_mask->src) || + (!is_zero_ether_addr(ð_mask->dst) && + !is_broadcast_ether_addr(ð_mask->dst))) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Invalid ether address mask"); + return -rte_errno; + } + + if ((eth_mask->type & UINT16_MAX) != UINT16_MAX) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Invalid ethertype mask"); + return -rte_errno; + } + + /* If mask bits of destination MAC address + * are full of 1, set RTE_ETHTYPE_FLAGS_MAC. + */ + if (is_broadcast_ether_addr(ð_mask->dst)) { + filter->mac_addr = eth_spec->dst; + filter->flags |= RTE_ETHTYPE_FLAGS_MAC; + } else { + filter->flags &= ~RTE_ETHTYPE_FLAGS_MAC; + } + filter->ether_type = rte_be_to_cpu_16(eth_spec->type); + + /* Check if the next non-void item is END. */ + index++; + NEXT_ITEM_OF_PATTERN(item, pattern, index); + if (item->type != RTE_FLOW_ITEM_TYPE_END) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by ethertype filter."); + return -rte_errno; + } + + /* Parse action */ + + index = 0; + /* Check if the first non-void action is QUEUE or DROP. */ + NEXT_ITEM_OF_ACTION(act, actions, index); + if (act->type != RTE_FLOW_ACTION_TYPE_QUEUE && + act->type != RTE_FLOW_ACTION_TYPE_DROP) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + act, "Not supported action."); + return -rte_errno; + } + + if (act->type == RTE_FLOW_ACTION_TYPE_QUEUE) { + act_q = (const struct rte_flow_action_queue *)act->conf; + filter->queue = act_q->index; + } else { + filter->flags |= RTE_ETHTYPE_FLAGS_DROP; + } + + /* Check if the next non-void item is END */ + index++; + NEXT_ITEM_OF_ACTION(act, actions, index); + if (act->type != RTE_FLOW_ACTION_TYPE_END) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + act, "Not supported action."); + return -rte_errno; + } + + /* Parse attr */ + /* Must be input direction */ + if (!attr->ingress) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, + attr, "Only support ingress."); + return -rte_errno; + } + + /* Not supported */ + if (attr->egress) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, + attr, "Not support egress."); + return -rte_errno; + } + + /* Not supported */ + if (attr->priority) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, + attr, "Not support priority."); + return -rte_errno; + } + + /* Not supported */ + if (attr->group) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_GROUP, + attr, "Not support group."); + return -rte_errno; + } + + return 0; +} + +static int +igb_parse_ethertype_filter(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_eth_ethertype_filter *filter, + struct rte_flow_error *error) +{ + struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + int ret; + + MAC_TYPE_FILTER_SUP(hw->mac.type); + + ret = cons_parse_ethertype_filter(attr, pattern, + actions, filter, error); + + if (ret) + return ret; + + if (hw->mac.type == e1000_82576) { + if (filter->queue >= IGB_MAX_RX_QUEUE_NUM_82576) { + memset(filter, 0, sizeof( + struct rte_eth_ethertype_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + NULL, "queue number not supported " + "by ethertype filter"); + return -rte_errno; + } + } else { + if (filter->queue >= IGB_MAX_RX_QUEUE_NUM) { + memset(filter, 0, sizeof( + struct rte_eth_ethertype_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + NULL, "queue number not supported " + "by ethertype filter"); + return -rte_errno; + } + } + + if (filter->ether_type == ETHER_TYPE_IPv4 || + filter->ether_type == ETHER_TYPE_IPv6) { + memset(filter, 0, sizeof(struct rte_eth_ethertype_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + NULL, "IPv4/IPv6 not supported by ethertype filter"); + return -rte_errno; + } + + if (filter->flags & RTE_ETHTYPE_FLAGS_MAC) { + memset(filter, 0, sizeof(struct rte_eth_ethertype_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + NULL, "mac compare is unsupported"); + return -rte_errno; + } + + if (filter->flags & RTE_ETHTYPE_FLAGS_DROP) { + memset(filter, 0, sizeof(struct rte_eth_ethertype_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + NULL, "drop option is unsupported"); + return -rte_errno; + } + + return 0; +} + +/** + * Parse the rule to see if it is a TCP SYN rule. + * And get the TCP SYN filter info BTW. + * pattern: + * The first not void item must be ETH. + * The second not void item must be IPV4 or IPV6. + * The third not void item must be TCP. + * The next not void item must be END. + * action: + * The first not void action should be QUEUE. + * The next not void action should be END. + * pattern example: + * ITEM Spec Mask + * ETH NULL NULL + * IPV4/IPV6 NULL NULL + * TCP tcp_flags 0x02 0xFF + * END + * other members in mask and spec should set to 0x00. + * item->last should be NULL. + */ +static int +cons_parse_syn_filter(const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_eth_syn_filter *filter, + struct rte_flow_error *error) +{ + const struct rte_flow_item *item; + const struct rte_flow_action *act; + const struct rte_flow_item_tcp *tcp_spec; + const struct rte_flow_item_tcp *tcp_mask; + const struct rte_flow_action_queue *act_q; + uint32_t index; + + if (!pattern) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM_NUM, + NULL, "NULL pattern."); + return -rte_errno; + } + + if (!actions) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION_NUM, + NULL, "NULL action."); + return -rte_errno; + } + + if (!attr) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR, + NULL, "NULL attribute."); + return -rte_errno; + } + + /* parse pattern */ + index = 0; + + /* the first not void item should be MAC or IPv4 or IPv6 or TCP */ + NEXT_ITEM_OF_PATTERN(item, pattern, index); + if (item->type != RTE_FLOW_ITEM_TYPE_ETH && + item->type != RTE_FLOW_ITEM_TYPE_IPV4 && + item->type != RTE_FLOW_ITEM_TYPE_IPV6 && + item->type != RTE_FLOW_ITEM_TYPE_TCP) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by syn filter"); + return -rte_errno; + } + /*Not supported last point for range*/ + if (item->last) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + item, "Not supported last point for range"); + return -rte_errno; + } + + /* Skip Ethernet */ + if (item->type == RTE_FLOW_ITEM_TYPE_ETH) { + /* if the item is MAC, the content should be NULL */ + if (item->spec || item->mask) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Invalid SYN address mask"); + return -rte_errno; + } + + /* check if the next not void item is IPv4 or IPv6 */ + index++; + NEXT_ITEM_OF_PATTERN(item, pattern, index); + if (item->type != RTE_FLOW_ITEM_TYPE_IPV4 && + item->type != RTE_FLOW_ITEM_TYPE_IPV6) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by syn filter"); + return -rte_errno; + } + } + + /* Skip IP */ + if (item->type == RTE_FLOW_ITEM_TYPE_IPV4 || + item->type == RTE_FLOW_ITEM_TYPE_IPV6) { + /* if the item is IP, the content should be NULL */ + if (item->spec || item->mask) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Invalid SYN mask"); + return -rte_errno; + } + + /* check if the next not void item is TCP */ + index++; + NEXT_ITEM_OF_PATTERN(item, pattern, index); + if (item->type != RTE_FLOW_ITEM_TYPE_TCP) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by syn filter"); + return -rte_errno; + } + } + + /* Get the TCP info. Only support SYN. */ + if (!item->spec || !item->mask) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Invalid SYN mask"); + return -rte_errno; + } + /*Not supported last point for range*/ + if (item->last) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + item, "Not supported last point for range"); + return -rte_errno; + } + + tcp_spec = (const struct rte_flow_item_tcp *)item->spec; + tcp_mask = (const struct rte_flow_item_tcp *)item->mask; + if (!(tcp_spec->hdr.tcp_flags & TCP_SYN_FLAG) || + tcp_mask->hdr.src_port || + tcp_mask->hdr.dst_port || + tcp_mask->hdr.sent_seq || + tcp_mask->hdr.recv_ack || + tcp_mask->hdr.data_off || + tcp_mask->hdr.tcp_flags != TCP_SYN_FLAG || + tcp_mask->hdr.rx_win || + tcp_mask->hdr.cksum || + tcp_mask->hdr.tcp_urp) { + memset(filter, 0, sizeof(struct rte_eth_syn_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by syn filter"); + return -rte_errno; + } + + /* check if the next not void item is END */ + index++; + NEXT_ITEM_OF_PATTERN(item, pattern, index); + if (item->type != RTE_FLOW_ITEM_TYPE_END) { + memset(filter, 0, sizeof(struct rte_eth_syn_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by syn filter"); + return -rte_errno; + } + + /* parse action */ + index = 0; + + /* check if the first not void action is QUEUE. */ + NEXT_ITEM_OF_ACTION(act, actions, index); + if (act->type != RTE_FLOW_ACTION_TYPE_QUEUE) { + memset(filter, 0, sizeof(struct rte_eth_syn_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + act, "Not supported action."); + return -rte_errno; + } + + act_q = (const struct rte_flow_action_queue *)act->conf; + filter->queue = act_q->index; + + /* check if the next not void item is END */ + index++; + NEXT_ITEM_OF_ACTION(act, actions, index); + if (act->type != RTE_FLOW_ACTION_TYPE_END) { + memset(filter, 0, sizeof(struct rte_eth_syn_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + act, "Not supported action."); + return -rte_errno; + } + + /* parse attr */ + /* must be input direction */ + if (!attr->ingress) { + memset(filter, 0, sizeof(struct rte_eth_syn_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, + attr, "Only support ingress."); + return -rte_errno; + } + + /* not supported */ + if (attr->egress) { + memset(filter, 0, sizeof(struct rte_eth_syn_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, + attr, "Not support egress."); + return -rte_errno; + } + + /* Support 2 priorities, the lowest or highest. */ + if (!attr->priority) { + filter->hig_pri = 0; + } else if (attr->priority == (uint32_t)~0U) { + filter->hig_pri = 1; + } else { + memset(filter, 0, sizeof(struct rte_eth_syn_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, + attr, "Not support priority."); + return -rte_errno; + } + + return 0; +} + +static int +igb_parse_syn_filter(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_eth_syn_filter *filter, + struct rte_flow_error *error) +{ + struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + int ret; + + MAC_TYPE_FILTER_SUP(hw->mac.type); + + ret = cons_parse_syn_filter(attr, pattern, + actions, filter, error); + + if (hw->mac.type == e1000_82576) { + if (filter->queue >= IGB_MAX_RX_QUEUE_NUM_82576) { + memset(filter, 0, sizeof(struct rte_eth_syn_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + NULL, "queue number not " + "supported by syn filter"); + return -rte_errno; + } + } else { + if (filter->queue >= IGB_MAX_RX_QUEUE_NUM) { + memset(filter, 0, sizeof(struct rte_eth_syn_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + NULL, "queue number not " + "supported by syn filter"); + return -rte_errno; + } + } + + if (ret) + return ret; + + return 0; +} + +/** + * Parse the rule to see if it is a flex byte rule. + * And get the flex byte filter info BTW. + * pattern: + * The first not void item must be RAW. + * The second not void item can be RAW or END. + * The third not void item can be RAW or END. + * The last not void item must be END. + * action: + * The first not void action should be QUEUE. + * The next not void action should be END. + * pattern example: + * ITEM Spec Mask + * RAW relative 0 0x1 + * offset 0 0xFFFFFFFF + * pattern {0x08, 0x06} {0xFF, 0xFF} + * RAW relative 1 0x1 + * offset 100 0xFFFFFFFF + * pattern {0x11, 0x22, 0x33} {0xFF, 0xFF, 0xFF} + * END + * other members in mask and spec should set to 0x00. + * item->last should be NULL. + */ +static int +cons_parse_flex_filter(const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_eth_flex_filter *filter, + struct rte_flow_error *error) +{ + const struct rte_flow_item *item; + const struct rte_flow_action *act; + const struct rte_flow_item_raw *raw_spec; + const struct rte_flow_item_raw *raw_mask; + const struct rte_flow_action_queue *act_q; + uint32_t index, i, offset, total_offset; + uint32_t max_offset = 0; + int32_t shift, j, raw_index = 0; + int32_t relative[IGB_FLEX_RAW_NUM] = {0}; + int32_t raw_offset[IGB_FLEX_RAW_NUM] = {0}; + + if (!pattern) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM_NUM, + NULL, "NULL pattern."); + return -rte_errno; + } + + if (!actions) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION_NUM, + NULL, "NULL action."); + return -rte_errno; + } + + if (!attr) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR, + NULL, "NULL attribute."); + return -rte_errno; + } + + /* parse pattern */ + index = 0; + +item_loop: + + /* the first not void item should be RAW */ + NEXT_ITEM_OF_PATTERN(item, pattern, index); + if (item->type != RTE_FLOW_ITEM_TYPE_RAW) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by flex filter"); + return -rte_errno; + } + /*Not supported last point for range*/ + if (item->last) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + item, "Not supported last point for range"); + return -rte_errno; + } + + raw_spec = (const struct rte_flow_item_raw *)item->spec; + raw_mask = (const struct rte_flow_item_raw *)item->mask; + + if (!raw_mask->length || + !raw_mask->relative) { + memset(filter, 0, sizeof(struct rte_eth_flex_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by flex filter"); + return -rte_errno; + } + + if (raw_mask->offset) + offset = raw_spec->offset; + else + offset = 0; + + for (j = 0; j < raw_spec->length; j++) { + if (raw_mask->pattern[j] != 0xFF) { + memset(filter, 0, sizeof(struct rte_eth_flex_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by flex filter"); + return -rte_errno; + } + } + + total_offset = 0; + + if (raw_spec->relative) { + for (j = raw_index; j > 0; j--) { + total_offset += raw_offset[j - 1]; + if (!relative[j - 1]) + break; + } + if (total_offset + raw_spec->length + offset > max_offset) + max_offset = total_offset + raw_spec->length + offset; + } else { + if (raw_spec->length + offset > max_offset) + max_offset = raw_spec->length + offset; + } + + if ((raw_spec->length + offset + total_offset) > + RTE_FLEX_FILTER_MAXLEN) { + memset(filter, 0, sizeof(struct rte_eth_flex_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by flex filter"); + return -rte_errno; + } + + if (raw_spec->relative == 0) { + for (j = 0; j < raw_spec->length; j++) + filter->bytes[offset + j] = + raw_spec->pattern[j]; + j = offset / CHAR_BIT; + shift = offset % CHAR_BIT; + } else { + for (j = 0; j < raw_spec->length; j++) + filter->bytes[total_offset + offset + j] = + raw_spec->pattern[j]; + j = (total_offset + offset) / CHAR_BIT; + shift = (total_offset + offset) % CHAR_BIT; + } + + i = 0; + + for ( ; shift < CHAR_BIT; shift++) { + filter->mask[j] |= (0x80 >> shift); + i++; + if (i == raw_spec->length) + break; + if (shift == (CHAR_BIT - 1)) { + j++; + shift = -1; + } + } + + relative[raw_index] = raw_spec->relative; + raw_offset[raw_index] = offset + raw_spec->length; + raw_index++; + + /* check if the next not void item is RAW */ + index++; + NEXT_ITEM_OF_PATTERN(item, pattern, index); + if (item->type != RTE_FLOW_ITEM_TYPE_RAW && + item->type != RTE_FLOW_ITEM_TYPE_END) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by flex filter"); + return -rte_errno; + } + + /* go back to parser */ + if (item->type == RTE_FLOW_ITEM_TYPE_RAW) { + /* if the item is RAW, the content should be parse */ + goto item_loop; + } + + filter->len = RTE_ALIGN(max_offset, 8); + + /* parse action */ + index = 0; + + /* check if the first not void action is QUEUE. */ + NEXT_ITEM_OF_ACTION(act, actions, index); + if (act->type != RTE_FLOW_ACTION_TYPE_QUEUE) { + memset(filter, 0, sizeof(struct rte_eth_flex_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + act, "Not supported action."); + return -rte_errno; + } + + act_q = (const struct rte_flow_action_queue *)act->conf; + filter->queue = act_q->index; + + /* check if the next not void item is END */ + index++; + NEXT_ITEM_OF_ACTION(act, actions, index); + if (act->type != RTE_FLOW_ACTION_TYPE_END) { + memset(filter, 0, sizeof(struct rte_eth_flex_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + act, "Not supported action."); + return -rte_errno; + } + + /* parse attr */ + /* must be input direction */ + if (!attr->ingress) { + memset(filter, 0, sizeof(struct rte_eth_flex_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, + attr, "Only support ingress."); + return -rte_errno; + } + + /* not supported */ + if (attr->egress) { + memset(filter, 0, sizeof(struct rte_eth_flex_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, + attr, "Not support egress."); + return -rte_errno; + } + + if (attr->priority > 0xFFFF) { + memset(filter, 0, sizeof(struct rte_eth_flex_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, + attr, "Error priority."); + return -rte_errno; + } + + filter->priority = (uint16_t)attr->priority; + + return 0; +} + +static int +igb_parse_flex_filter(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_eth_flex_filter *filter, + struct rte_flow_error *error) +{ + struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + int ret; + + MAC_TYPE_FILTER_SUP_EXT(hw->mac.type); + + ret = cons_parse_flex_filter(attr, pattern, + actions, filter, error); + + if (filter->queue >= IGB_MAX_RX_QUEUE_NUM) { + memset(filter, 0, sizeof(struct rte_eth_flex_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + NULL, "queue number not supported by flex filter"); + return -rte_errno; + } + + if (filter->len == 0 || filter->len > E1000_MAX_FLEX_FILTER_LEN || + filter->len % sizeof(uint64_t) != 0) { + PMD_DRV_LOG(ERR, "filter's length is out of range"); + return -EINVAL; + } + + if (filter->priority > E1000_MAX_FLEX_FILTER_PRI) { + PMD_DRV_LOG(ERR, "filter's priority is out of range"); + return -EINVAL; + } + + if (ret) + return ret; + + return 0; +} + +/** + * Create a flow rule. + * Theorically one rule can match more than one filters. + * We will let it use the filter which it hitt first. + * So, the sequence matters. + */ +static struct rte_flow * +igb_flow_create(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error) +{ + int ret; + struct rte_eth_ntuple_filter ntuple_filter; + struct rte_eth_ethertype_filter ethertype_filter; + struct rte_eth_syn_filter syn_filter; + struct rte_eth_flex_filter flex_filter; + struct rte_flow *flow = NULL; + struct igb_ntuple_filter_ele *ntuple_filter_ptr; + struct igb_ethertype_filter_ele *ethertype_filter_ptr; + struct igb_eth_syn_filter_ele *syn_filter_ptr; + struct igb_flex_filter_ele *flex_filter_ptr; + struct igb_flow_mem *igb_flow_mem_ptr; + + flow = rte_zmalloc("igb_rte_flow", sizeof(struct rte_flow), 0); + if (!flow) { + PMD_DRV_LOG(ERR, "failed to allocate memory"); + return (struct rte_flow *)flow; + } + igb_flow_mem_ptr = rte_zmalloc("igb_flow_mem", + sizeof(struct igb_flow_mem), 0); + if (!igb_flow_mem_ptr) { + PMD_DRV_LOG(ERR, "failed to allocate memory"); + rte_free(flow); + return NULL; + } + igb_flow_mem_ptr->flow = flow; + igb_flow_mem_ptr->dev = dev; + TAILQ_INSERT_TAIL(&igb_flow_list, + igb_flow_mem_ptr, entries); + + memset(&ntuple_filter, 0, sizeof(struct rte_eth_ntuple_filter)); + ret = igb_parse_ntuple_filter(dev, attr, pattern, + actions, &ntuple_filter, error); + if (!ret) { + ret = igb_add_del_ntuple_filter(dev, &ntuple_filter, TRUE); + if (!ret) { + ntuple_filter_ptr = rte_zmalloc("igb_ntuple_filter", + sizeof(struct igb_ntuple_filter_ele), 0); + if (!ntuple_filter_ptr) { + PMD_DRV_LOG(ERR, "failed to allocate memory"); + goto out; + } + + rte_memcpy(&ntuple_filter_ptr->filter_info, + &ntuple_filter, + sizeof(struct rte_eth_ntuple_filter)); + TAILQ_INSERT_TAIL(&igb_filter_ntuple_list, + ntuple_filter_ptr, entries); + flow->rule = ntuple_filter_ptr; + flow->filter_type = RTE_ETH_FILTER_NTUPLE; + return flow; + } + goto out; + } + + memset(ðertype_filter, 0, sizeof(struct rte_eth_ethertype_filter)); + ret = igb_parse_ethertype_filter(dev, attr, pattern, + actions, ðertype_filter, error); + if (!ret) { + ret = igb_add_del_ethertype_filter(dev, + ðertype_filter, TRUE); + if (!ret) { + ethertype_filter_ptr = rte_zmalloc( + "igb_ethertype_filter", + sizeof(struct igb_ethertype_filter_ele), 0); + if (!ethertype_filter_ptr) { + PMD_DRV_LOG(ERR, "failed to allocate memory"); + goto out; + } + + rte_memcpy(ðertype_filter_ptr->filter_info, + ðertype_filter, + sizeof(struct rte_eth_ethertype_filter)); + TAILQ_INSERT_TAIL(&igb_filter_ethertype_list, + ethertype_filter_ptr, entries); + flow->rule = ethertype_filter_ptr; + flow->filter_type = RTE_ETH_FILTER_ETHERTYPE; + return flow; + } + goto out; + } + + memset(&syn_filter, 0, sizeof(struct rte_eth_syn_filter)); + ret = igb_parse_syn_filter(dev, attr, pattern, + actions, &syn_filter, error); + if (!ret) { + ret = eth_igb_syn_filter_set(dev, &syn_filter, TRUE); + if (!ret) { + syn_filter_ptr = rte_zmalloc("igb_syn_filter", + sizeof(struct igb_eth_syn_filter_ele), 0); + if (!syn_filter_ptr) { + PMD_DRV_LOG(ERR, "failed to allocate memory"); + goto out; + } + + rte_memcpy(&syn_filter_ptr->filter_info, + &syn_filter, + sizeof(struct rte_eth_syn_filter)); + TAILQ_INSERT_TAIL(&igb_filter_syn_list, + syn_filter_ptr, + entries); + flow->rule = syn_filter_ptr; + flow->filter_type = RTE_ETH_FILTER_SYN; + return flow; + } + goto out; + } + + memset(&flex_filter, 0, sizeof(struct rte_eth_flex_filter)); + ret = igb_parse_flex_filter(dev, attr, pattern, + actions, &flex_filter, error); + if (!ret) { + ret = eth_igb_add_del_flex_filter(dev, &flex_filter, TRUE); + if (!ret) { + flex_filter_ptr = rte_zmalloc("igb_flex_filter", + sizeof(struct igb_flex_filter_ele), 0); + if (!flex_filter_ptr) { + PMD_DRV_LOG(ERR, "failed to allocate memory"); + goto out; + } + + rte_memcpy(&flex_filter_ptr->filter_info, + &flex_filter, + sizeof(struct rte_eth_flex_filter)); + TAILQ_INSERT_TAIL(&igb_filter_flex_list, + flex_filter_ptr, entries); + flow->rule = flex_filter_ptr; + flow->filter_type = RTE_ETH_FILTER_FLEXIBLE; + return flow; + } + } + +out: + TAILQ_REMOVE(&igb_flow_list, + igb_flow_mem_ptr, entries); + rte_flow_error_set(error, -ret, + RTE_FLOW_ERROR_TYPE_HANDLE, NULL, + "Failed to create flow."); + rte_free(igb_flow_mem_ptr); + rte_free(flow); + return NULL; +} + +/** + * Check if the flow rule is supported by igb. + * It only checkes the format. Don't guarantee the rule can be programmed into + * the HW. Because there can be no enough room for the rule. + */ +static int +igb_flow_validate(__rte_unused struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error) +{ + struct rte_eth_ntuple_filter ntuple_filter; + struct rte_eth_ethertype_filter ethertype_filter; + struct rte_eth_syn_filter syn_filter; + struct rte_eth_flex_filter flex_filter; + int ret; + + memset(&ntuple_filter, 0, sizeof(struct rte_eth_ntuple_filter)); + ret = igb_parse_ntuple_filter(dev, attr, pattern, + actions, &ntuple_filter, error); + if (!ret) + return 0; + + memset(ðertype_filter, 0, sizeof(struct rte_eth_ethertype_filter)); + ret = igb_parse_ethertype_filter(dev, attr, pattern, + actions, ðertype_filter, error); + if (!ret) + return 0; + + memset(&syn_filter, 0, sizeof(struct rte_eth_syn_filter)); + ret = igb_parse_syn_filter(dev, attr, pattern, + actions, &syn_filter, error); + if (!ret) + return 0; + + memset(&flex_filter, 0, sizeof(struct rte_eth_flex_filter)); + ret = igb_parse_flex_filter(dev, attr, pattern, + actions, &flex_filter, error); + + return ret; +} + +/* Destroy a flow rule on igb. */ +static int +igb_flow_destroy(struct rte_eth_dev *dev, + struct rte_flow *flow, + struct rte_flow_error *error) +{ + int ret; + struct rte_flow *pmd_flow = flow; + enum rte_filter_type filter_type = pmd_flow->filter_type; + struct igb_ntuple_filter_ele *ntuple_filter_ptr; + struct igb_ethertype_filter_ele *ethertype_filter_ptr; + struct igb_eth_syn_filter_ele *syn_filter_ptr; + struct igb_flex_filter_ele *flex_filter_ptr; + struct igb_flow_mem *igb_flow_mem_ptr; + + switch (filter_type) { + case RTE_ETH_FILTER_NTUPLE: + ntuple_filter_ptr = (struct igb_ntuple_filter_ele *) + pmd_flow->rule; + ret = igb_add_del_ntuple_filter(dev, + &ntuple_filter_ptr->filter_info, FALSE); + if (!ret) { + TAILQ_REMOVE(&igb_filter_ntuple_list, + ntuple_filter_ptr, entries); + rte_free(ntuple_filter_ptr); + } + break; + case RTE_ETH_FILTER_ETHERTYPE: + ethertype_filter_ptr = (struct igb_ethertype_filter_ele *) + pmd_flow->rule; + ret = igb_add_del_ethertype_filter(dev, + ðertype_filter_ptr->filter_info, FALSE); + if (!ret) { + TAILQ_REMOVE(&igb_filter_ethertype_list, + ethertype_filter_ptr, entries); + rte_free(ethertype_filter_ptr); + } + break; + case RTE_ETH_FILTER_SYN: + syn_filter_ptr = (struct igb_eth_syn_filter_ele *) + pmd_flow->rule; + ret = eth_igb_syn_filter_set(dev, + &syn_filter_ptr->filter_info, FALSE); + if (!ret) { + TAILQ_REMOVE(&igb_filter_syn_list, + syn_filter_ptr, entries); + rte_free(syn_filter_ptr); + } + break; + case RTE_ETH_FILTER_FLEXIBLE: + flex_filter_ptr = (struct igb_flex_filter_ele *) + pmd_flow->rule; + ret = eth_igb_add_del_flex_filter(dev, + &flex_filter_ptr->filter_info, FALSE); + if (!ret) { + TAILQ_REMOVE(&igb_filter_flex_list, + flex_filter_ptr, entries); + rte_free(flex_filter_ptr); + } + break; + default: + PMD_DRV_LOG(WARNING, "Filter type (%d) not supported", + filter_type); + ret = -EINVAL; + break; + } + + if (ret) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_HANDLE, + NULL, "Failed to destroy flow"); + return ret; + } + + TAILQ_FOREACH(igb_flow_mem_ptr, &igb_flow_list, entries) { + if (igb_flow_mem_ptr->flow == pmd_flow) { + TAILQ_REMOVE(&igb_flow_list, + igb_flow_mem_ptr, entries); + rte_free(igb_flow_mem_ptr); + } + } + rte_free(flow); + + return ret; +} + +/* remove all the n-tuple filters */ +static void +igb_clear_all_ntuple_filter(struct rte_eth_dev *dev) +{ + struct e1000_filter_info *filter_info = + E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); + struct e1000_5tuple_filter *p_5tuple; + struct e1000_2tuple_filter *p_2tuple; + + while ((p_5tuple = TAILQ_FIRST(&filter_info->fivetuple_list))) + igb_delete_5tuple_filter_82576(dev, p_5tuple); + + while ((p_2tuple = TAILQ_FIRST(&filter_info->twotuple_list))) + igb_delete_2tuple_filter(dev, p_2tuple); +} + +/* remove all the ether type filters */ +static void +igb_clear_all_ethertype_filter(struct rte_eth_dev *dev) +{ + struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct e1000_filter_info *filter_info = + E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); + int i; + + for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) { + if (filter_info->ethertype_mask & (1 << i)) { + (void)igb_ethertype_filter_remove(filter_info, + (uint8_t)i); + E1000_WRITE_REG(hw, E1000_ETQF(i), 0); + E1000_WRITE_FLUSH(hw); + } + } +} + +/* remove the SYN filter */ +static void +igb_clear_syn_filter(struct rte_eth_dev *dev) +{ + struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct e1000_filter_info *filter_info = + E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); + + if (filter_info->syn_info & E1000_SYN_FILTER_ENABLE) { + filter_info->syn_info = 0; + E1000_WRITE_REG(hw, E1000_SYNQF(0), 0); + E1000_WRITE_FLUSH(hw); + } +} + +/* remove all the flex filters */ +static void +igb_clear_all_flex_filter(struct rte_eth_dev *dev) +{ + struct e1000_filter_info *filter_info = + E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); + struct e1000_flex_filter *flex_filter; + + while ((flex_filter = TAILQ_FIRST(&filter_info->flex_list))) + igb_remove_flex_filter(dev, flex_filter); +} + +void +igb_filterlist_flush(struct rte_eth_dev *dev) +{ + struct igb_ntuple_filter_ele *ntuple_filter_ptr; + struct igb_ethertype_filter_ele *ethertype_filter_ptr; + struct igb_eth_syn_filter_ele *syn_filter_ptr; + struct igb_flex_filter_ele *flex_filter_ptr; + struct igb_flow_mem *igb_flow_mem_ptr; + enum rte_filter_type filter_type; + struct rte_flow *pmd_flow; + + TAILQ_FOREACH(igb_flow_mem_ptr, &igb_flow_list, entries) { + if (igb_flow_mem_ptr->dev == dev) { + pmd_flow = igb_flow_mem_ptr->flow; + filter_type = pmd_flow->filter_type; + + switch (filter_type) { + case RTE_ETH_FILTER_NTUPLE: + ntuple_filter_ptr = + (struct igb_ntuple_filter_ele *) + pmd_flow->rule; + TAILQ_REMOVE(&igb_filter_ntuple_list, + ntuple_filter_ptr, entries); + rte_free(ntuple_filter_ptr); + break; + case RTE_ETH_FILTER_ETHERTYPE: + ethertype_filter_ptr = + (struct igb_ethertype_filter_ele *) + pmd_flow->rule; + TAILQ_REMOVE(&igb_filter_ethertype_list, + ethertype_filter_ptr, entries); + rte_free(ethertype_filter_ptr); + break; + case RTE_ETH_FILTER_SYN: + syn_filter_ptr = + (struct igb_eth_syn_filter_ele *) + pmd_flow->rule; + TAILQ_REMOVE(&igb_filter_syn_list, + syn_filter_ptr, entries); + rte_free(syn_filter_ptr); + break; + case RTE_ETH_FILTER_FLEXIBLE: + flex_filter_ptr = + (struct igb_flex_filter_ele *) + pmd_flow->rule; + TAILQ_REMOVE(&igb_filter_flex_list, + flex_filter_ptr, entries); + rte_free(flex_filter_ptr); + break; + default: + PMD_DRV_LOG(WARNING, "Filter type" + "(%d) not supported", filter_type); + break; + } + TAILQ_REMOVE(&igb_flow_list, + igb_flow_mem_ptr, + entries); + rte_free(igb_flow_mem_ptr->flow); + rte_free(igb_flow_mem_ptr); + } + } +} + +/* Destroy all flow rules associated with a port on igb. */ +static int +igb_flow_flush(struct rte_eth_dev *dev, + __rte_unused struct rte_flow_error *error) +{ + igb_clear_all_ntuple_filter(dev); + igb_clear_all_ethertype_filter(dev); + igb_clear_syn_filter(dev); + igb_clear_all_flex_filter(dev); + igb_filterlist_flush(dev); + + return 0; +} + +const struct rte_flow_ops igb_flow_ops = { + .validate = igb_flow_validate, + .create = igb_flow_create, + .destroy = igb_flow_destroy, + .flush = igb_flow_flush, +}; diff --git a/dpdk/drivers/net/e1000/igb_pf.c b/dpdk/drivers/net/e1000/igb_pf.c index 5845bc22..cd6ae2fb 100644 --- a/dpdk/drivers/net/e1000/igb_pf.c +++ b/dpdk/drivers/net/e1000/igb_pf.c @@ -39,6 +39,7 @@ #include #include +#include #include #include #include @@ -57,7 +58,9 @@ static inline uint16_t dev_num_vf(struct rte_eth_dev *eth_dev) { - return eth_dev->pci_dev->max_vfs; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); + + return pci_dev->max_vfs; } static inline @@ -330,12 +333,16 @@ igb_vf_set_mac_addr(struct rte_eth_dev *dev, uint32_t vf, uint32_t *msgbuf) *(E1000_DEV_PRIVATE_TO_P_VFDATA(dev->data->dev_private)); int rar_entry = hw->mac.rar_entry_count - (vf + 1); uint8_t *new_mac = (uint8_t *)(&msgbuf[1]); + int rah; if (is_unicast_ether_addr((struct ether_addr *)new_mac)) { if (!is_zero_ether_addr((struct ether_addr *)new_mac)) rte_memcpy(vfinfo[vf].vf_mac_addresses, new_mac, sizeof(vfinfo[vf].vf_mac_addresses)); hw->mac.ops.rar_set(hw, new_mac, rar_entry); + rah = E1000_READ_REG(hw, E1000_RAH(rar_entry)); + rah |= (0x1 << (E1000_RAH_POOLSEL_SHIFT + vf)); + E1000_WRITE_REG(hw, E1000_RAH(rar_entry), rah); return 0; } return -1; diff --git a/dpdk/drivers/net/e1000/igb_rxtx.c b/dpdk/drivers/net/e1000/igb_rxtx.c index c5db727d..4ee12e9e 100644 --- a/dpdk/drivers/net/e1000/igb_rxtx.c +++ b/dpdk/drivers/net/e1000/igb_rxtx.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -56,7 +56,6 @@ #include #include #include -#include #include #include #include @@ -66,18 +65,28 @@ #include #include #include +#include #include #include "e1000_logs.h" #include "base/e1000_api.h" #include "e1000_ethdev.h" +#ifdef RTE_LIBRTE_IEEE1588 +#define IGB_TX_IEEE1588_TMST PKT_TX_IEEE1588_TMST +#else +#define IGB_TX_IEEE1588_TMST 0 +#endif /* Bit Mask to indicate what bits required for building TX context */ #define IGB_TX_OFFLOAD_MASK ( \ PKT_TX_VLAN_PKT | \ PKT_TX_IP_CKSUM | \ PKT_TX_L4_MASK | \ - PKT_TX_TCP_SEG) + PKT_TX_TCP_SEG | \ + IGB_TX_IEEE1588_TMST) + +#define IGB_TX_OFFLOAD_NOTSUP_MASK \ + (PKT_TX_OFFLOAD_MASK ^ IGB_TX_OFFLOAD_MASK) /** * Structure associated with each descriptor of the RX ring of a RX queue. @@ -95,6 +104,13 @@ struct igb_tx_entry { uint16_t last_id; /**< Index of last scattered descriptor. */ }; +/** + * rx queue flags + */ +enum igb_rxq_flags { + IGB_RXQ_FLAG_LB_BSWAP_VLAN = 0x01, +}; + /** * Structure associated with each RX queue. */ @@ -113,12 +129,13 @@ struct igb_rx_queue { uint16_t rx_free_thresh; /**< max free RX desc to hold. */ uint16_t queue_id; /**< RX queue index. */ uint16_t reg_idx; /**< RX queue register index. */ - uint8_t port_id; /**< Device port identifier. */ + uint16_t port_id; /**< Device port identifier. */ uint8_t pthresh; /**< Prefetch threshold register. */ uint8_t hthresh; /**< Host threshold register. */ uint8_t wthresh; /**< Write-back threshold register. */ uint8_t crc_len; /**< 0 if CRC stripped, 4 otherwise. */ uint8_t drop_en; /**< If not 0, set SRRCTL.Drop_En. */ + uint32_t flags; /**< RX flags. */ }; /** @@ -182,7 +199,7 @@ struct igb_tx_queue { /**< Index of first used TX descriptor. */ uint16_t queue_id; /**< TX queue index. */ uint16_t reg_idx; /**< TX queue register index. */ - uint8_t port_id; /**< Device port identifier. */ + uint16_t port_id; /**< Device port identifier. */ uint8_t pthresh; /**< Prefetch threshold register. */ uint8_t hthresh; /**< Host threshold register. */ uint8_t wthresh; /**< Write-back threshold register. */ @@ -580,7 +597,7 @@ eth_igb_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, * Set up transmit descriptor. */ slen = (uint16_t) m_seg->data_len; - buf_dma_addr = rte_mbuf_data_dma_addr(m_seg); + buf_dma_addr = rte_mbuf_data_iova(m_seg); txd->read.buffer_addr = rte_cpu_to_le_64(buf_dma_addr); txd->read.cmd_type_len = @@ -606,7 +623,7 @@ eth_igb_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, /* * Set the Transmit Descriptor Tail (TDT). */ - E1000_PCI_REG_WRITE(txq->tdt_reg_addr, tx_id); + E1000_PCI_REG_WRITE_RELAXED(txq->tdt_reg_addr, tx_id); PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_tx=%u", (unsigned) txq->port_id, (unsigned) txq->queue_id, (unsigned) tx_id, (unsigned) nb_tx); @@ -615,6 +632,52 @@ eth_igb_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, return nb_tx; } +/********************************************************************* + * + * TX prep functions + * + **********************************************************************/ +uint16_t +eth_igb_prep_pkts(__rte_unused void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts) +{ + int i, ret; + struct rte_mbuf *m; + + for (i = 0; i < nb_pkts; i++) { + m = tx_pkts[i]; + + /* Check some limitations for TSO in hardware */ + if (m->ol_flags & PKT_TX_TCP_SEG) + if ((m->tso_segsz > IGB_TSO_MAX_MSS) || + (m->l2_len + m->l3_len + m->l4_len > + IGB_TSO_MAX_HDRLEN)) { + rte_errno = -EINVAL; + return i; + } + + if (m->ol_flags & IGB_TX_OFFLOAD_NOTSUP_MASK) { + rte_errno = -ENOTSUP; + return i; + } + +#ifdef RTE_LIBRTE_ETHDEV_DEBUG + ret = rte_validate_tx_offload(m); + if (ret != 0) { + rte_errno = ret; + return i; + } +#endif + ret = rte_net_intel_cksum_prepare(m); + if (ret != 0) { + rte_errno = ret; + return i; + } + } + + return i; +} + /********************************************************************* * * RX functions @@ -730,7 +793,7 @@ rx_desc_status_to_pkt_flags(uint32_t rx_status) /* Check if VLAN present */ pkt_flags = ((rx_status & E1000_RXD_STAT_VP) ? - PKT_RX_VLAN_PKT | PKT_RX_VLAN_STRIPPED : 0); + PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED : 0); #if defined(RTE_LIBRTE_IEEE1588) if (rx_status & E1000_RXD_STAT_TMST) @@ -748,7 +811,9 @@ rx_desc_error_to_pkt_flags(uint32_t rx_status) */ static uint64_t error_to_pkt_flags_map[4] = { - 0, PKT_RX_L4_CKSUM_BAD, PKT_RX_IP_CKSUM_BAD, + PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD, + PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_BAD, + PKT_RX_IP_CKSUM_BAD | PKT_RX_L4_CKSUM_GOOD, PKT_RX_IP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD }; return error_to_pkt_flags_map[(rx_status >> @@ -860,7 +925,7 @@ eth_igb_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, rxm = rxe->mbuf; rxe->mbuf = nmb; dma_addr = - rte_cpu_to_le_64(rte_mbuf_data_dma_addr_default(nmb)); + rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb)); rxdp->read.hdr_addr = 0; rxdp->read.pkt_addr = dma_addr; @@ -889,9 +954,17 @@ eth_igb_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, rxm->hash.rss = rxd.wb.lower.hi_dword.rss; hlen_type_rss = rte_le_to_cpu_32(rxd.wb.lower.lo_dword.data); - /* Only valid if PKT_RX_VLAN_PKT set in pkt_flags */ - rxm->vlan_tci = rte_le_to_cpu_16(rxd.wb.upper.vlan); + /* + * The vlan_tci field is only valid when PKT_RX_VLAN is + * set in the pkt_flags field and must be in CPU byte order. + */ + if ((staterr & rte_cpu_to_le_32(E1000_RXDEXT_STATERR_LB)) && + (rxq->flags & IGB_RXQ_FLAG_LB_BSWAP_VLAN)) { + rxm->vlan_tci = rte_be_to_cpu_16(rxd.wb.upper.vlan); + } else { + rxm->vlan_tci = rte_le_to_cpu_16(rxd.wb.upper.vlan); + } pkt_flags = rx_desc_hlen_type_rss_to_pkt_flags(rxq, hlen_type_rss); pkt_flags = pkt_flags | rx_desc_status_to_pkt_flags(staterr); pkt_flags = pkt_flags | rx_desc_error_to_pkt_flags(staterr); @@ -1046,7 +1119,7 @@ eth_igb_recv_scattered_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, */ rxm = rxe->mbuf; rxe->mbuf = nmb; - dma = rte_cpu_to_le_64(rte_mbuf_data_dma_addr_default(nmb)); + dma = rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb)); rxdp->read.pkt_addr = dma; rxdp->read.hdr_addr = 0; @@ -1123,10 +1196,17 @@ eth_igb_recv_scattered_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, first_seg->hash.rss = rxd.wb.lower.hi_dword.rss; /* - * The vlan_tci field is only valid when PKT_RX_VLAN_PKT is - * set in the pkt_flags field. + * The vlan_tci field is only valid when PKT_RX_VLAN is + * set in the pkt_flags field and must be in CPU byte order. */ - first_seg->vlan_tci = rte_le_to_cpu_16(rxd.wb.upper.vlan); + if ((staterr & rte_cpu_to_le_32(E1000_RXDEXT_STATERR_LB)) && + (rxq->flags & IGB_RXQ_FLAG_LB_BSWAP_VLAN)) { + first_seg->vlan_tci = + rte_be_to_cpu_16(rxd.wb.upper.vlan); + } else { + first_seg->vlan_tci = + rte_le_to_cpu_16(rxd.wb.upper.vlan); + } hlen_type_rss = rte_le_to_cpu_32(rxd.wb.lower.lo_dword.data); pkt_flags = rx_desc_hlen_type_rss_to_pkt_flags(rxq, hlen_type_rss); pkt_flags = pkt_flags | rx_desc_status_to_pkt_flags(staterr); @@ -1226,6 +1306,132 @@ eth_igb_tx_queue_release(void *txq) igb_tx_queue_release(txq); } +static int +igb_tx_done_cleanup(struct igb_tx_queue *txq, uint32_t free_cnt) +{ + struct igb_tx_entry *sw_ring; + volatile union e1000_adv_tx_desc *txr; + uint16_t tx_first; /* First segment analyzed. */ + uint16_t tx_id; /* Current segment being processed. */ + uint16_t tx_last; /* Last segment in the current packet. */ + uint16_t tx_next; /* First segment of the next packet. */ + int count; + + if (txq != NULL) { + count = 0; + sw_ring = txq->sw_ring; + txr = txq->tx_ring; + + /* + * tx_tail is the last sent packet on the sw_ring. Goto the end + * of that packet (the last segment in the packet chain) and + * then the next segment will be the start of the oldest segment + * in the sw_ring. This is the first packet that will be + * attempted to be freed. + */ + + /* Get last segment in most recently added packet. */ + tx_first = sw_ring[txq->tx_tail].last_id; + + /* Get the next segment, which is the oldest segment in ring. */ + tx_first = sw_ring[tx_first].next_id; + + /* Set the current index to the first. */ + tx_id = tx_first; + + /* + * Loop through each packet. For each packet, verify that an + * mbuf exists and that the last segment is free. If so, free + * it and move on. + */ + while (1) { + tx_last = sw_ring[tx_id].last_id; + + if (sw_ring[tx_last].mbuf) { + if (txr[tx_last].wb.status & + E1000_TXD_STAT_DD) { + /* + * Increment the number of packets + * freed. + */ + count++; + + /* Get the start of the next packet. */ + tx_next = sw_ring[tx_last].next_id; + + /* + * Loop through all segments in a + * packet. + */ + do { + rte_pktmbuf_free_seg(sw_ring[tx_id].mbuf); + sw_ring[tx_id].mbuf = NULL; + sw_ring[tx_id].last_id = tx_id; + + /* Move to next segemnt. */ + tx_id = sw_ring[tx_id].next_id; + + } while (tx_id != tx_next); + + if (unlikely(count == (int)free_cnt)) + break; + } else + /* + * mbuf still in use, nothing left to + * free. + */ + break; + } else { + /* + * There are multiple reasons to be here: + * 1) All the packets on the ring have been + * freed - tx_id is equal to tx_first + * and some packets have been freed. + * - Done, exit + * 2) Interfaces has not sent a rings worth of + * packets yet, so the segment after tail is + * still empty. Or a previous call to this + * function freed some of the segments but + * not all so there is a hole in the list. + * Hopefully this is a rare case. + * - Walk the list and find the next mbuf. If + * there isn't one, then done. + */ + if (likely((tx_id == tx_first) && (count != 0))) + break; + + /* + * Walk the list and find the next mbuf, if any. + */ + do { + /* Move to next segemnt. */ + tx_id = sw_ring[tx_id].next_id; + + if (sw_ring[tx_id].mbuf) + break; + + } while (tx_id != tx_first); + + /* + * Determine why previous loop bailed. If there + * is not an mbuf, done. + */ + if (sw_ring[tx_id].mbuf == NULL) + break; + } + } + } else + count = -ENODEV; + + return count; +} + +int +eth_igb_tx_done_cleanup(void *txq, uint32_t free_cnt) +{ + return igb_tx_done_cleanup(txq, free_cnt); +} + static void igb_reset_tx_queue_stat(struct igb_tx_queue *txq) { @@ -1347,7 +1553,7 @@ eth_igb_tx_queue_setup(struct rte_eth_dev *dev, txq->port_id = dev->data->port_id; txq->tdt_reg_addr = E1000_PCI_REG_ADDR(hw, E1000_TDT(txq->reg_idx)); - txq->tx_ring_phys_addr = rte_mem_phy2mch(tz->memseg_id, tz->phys_addr); + txq->tx_ring_phys_addr = tz->iova; txq->tx_ring = (union e1000_adv_tx_desc *) tz->addr; /* Allocate software ring */ @@ -1363,6 +1569,7 @@ eth_igb_tx_queue_setup(struct rte_eth_dev *dev, igb_reset_tx_queue(txq, dev); dev->tx_pkt_burst = eth_igb_xmit_pkts; + dev->tx_pkt_prepare = ð_igb_prep_pkts; dev->data->tx_queues[queue_idx] = txq; return 0; @@ -1483,7 +1690,7 @@ eth_igb_rx_queue_setup(struct rte_eth_dev *dev, } rxq->rdt_reg_addr = E1000_PCI_REG_ADDR(hw, E1000_RDT(rxq->reg_idx)); rxq->rdh_reg_addr = E1000_PCI_REG_ADDR(hw, E1000_RDH(rxq->reg_idx)); - rxq->rx_ring_phys_addr = rte_mem_phy2mch(rz->memseg_id, rz->phys_addr); + rxq->rx_ring_phys_addr = rz->iova; rxq->rx_ring = (union e1000_adv_rx_desc *) rz->addr; /* Allocate software ring. */ @@ -1511,11 +1718,6 @@ eth_igb_rx_queue_count(struct rte_eth_dev *dev, uint16_t rx_queue_id) struct igb_rx_queue *rxq; uint32_t desc = 0; - if (rx_queue_id >= dev->data->nb_rx_queues) { - PMD_RX_LOG(ERR, "Invalid RX queue id=%d", rx_queue_id); - return 0; - } - rxq = dev->data->rx_queues[rx_queue_id]; rxdp = &(rxq->rx_ring[rxq->rx_tail]); @@ -1548,6 +1750,51 @@ eth_igb_rx_descriptor_done(void *rx_queue, uint16_t offset) return !!(rxdp->wb.upper.status_error & E1000_RXD_STAT_DD); } +int +eth_igb_rx_descriptor_status(void *rx_queue, uint16_t offset) +{ + struct igb_rx_queue *rxq = rx_queue; + volatile uint32_t *status; + uint32_t desc; + + if (unlikely(offset >= rxq->nb_rx_desc)) + return -EINVAL; + + if (offset >= rxq->nb_rx_desc - rxq->nb_rx_hold) + return RTE_ETH_RX_DESC_UNAVAIL; + + desc = rxq->rx_tail + offset; + if (desc >= rxq->nb_rx_desc) + desc -= rxq->nb_rx_desc; + + status = &rxq->rx_ring[desc].wb.upper.status_error; + if (*status & rte_cpu_to_le_32(E1000_RXD_STAT_DD)) + return RTE_ETH_RX_DESC_DONE; + + return RTE_ETH_RX_DESC_AVAIL; +} + +int +eth_igb_tx_descriptor_status(void *tx_queue, uint16_t offset) +{ + struct igb_tx_queue *txq = tx_queue; + volatile uint32_t *status; + uint32_t desc; + + if (unlikely(offset >= txq->nb_tx_desc)) + return -EINVAL; + + desc = txq->tx_tail + offset; + if (desc >= txq->nb_tx_desc) + desc -= txq->nb_tx_desc; + + status = &txq->tx_ring[desc].wb.status; + if (*status & rte_cpu_to_le_32(E1000_TXD_STAT_DD)) + return RTE_ETH_TX_DESC_DONE; + + return RTE_ETH_TX_DESC_FULL; +} + void igb_dev_clear_queues(struct rte_eth_dev *dev) { @@ -1956,7 +2203,7 @@ igb_alloc_rx_queue_mbufs(struct igb_rx_queue *rxq) return -ENOMEM; } dma_addr = - rte_cpu_to_le_64(rte_mbuf_data_dma_addr_default(mbuf)); + rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf)); rxd = &rxq->rx_ring[i]; rxd->read.hdr_addr = 0; rxd->read.pkt_addr = dma_addr; @@ -2054,6 +2301,17 @@ eth_igb_rx_init(struct rte_eth_dev *dev) rxq = dev->data->rx_queues[i]; + rxq->flags = 0; + /* + * i350 and i354 vlan packets have vlan tags byte swapped. + */ + if (hw->mac.type == e1000_i350 || hw->mac.type == e1000_i354) { + rxq->flags |= IGB_RXQ_FLAG_LB_BSWAP_VLAN; + PMD_INIT_LOG(DEBUG, "IGB rx vlan bswap required"); + } else { + PMD_INIT_LOG(DEBUG, "IGB rx vlan bswap not required"); + } + /* Allocate buffers for descriptor rings and set up queue */ ret = igb_alloc_rx_queue_mbufs(rxq); if (ret) @@ -2178,9 +2436,11 @@ eth_igb_rx_init(struct rte_eth_dev *dev) /* Enable both L3/L4 rx checksum offload */ if (dev->data->dev_conf.rxmode.hw_ip_checksum) - rxcsum |= (E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL); + rxcsum |= (E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL | + E1000_RXCSUM_CRCOFL); else - rxcsum &= ~(E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL); + rxcsum &= ~(E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL | + E1000_RXCSUM_CRCOFL); E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum); /* Setup the Receive Control Register. */ @@ -2331,6 +2591,17 @@ eth_igbvf_rx_init(struct rte_eth_dev *dev) rxq = dev->data->rx_queues[i]; + rxq->flags = 0; + /* + * i350VF LB vlan packets have vlan tags byte swapped. + */ + if (hw->mac.type == e1000_vfadapt_i350) { + rxq->flags |= IGB_RXQ_FLAG_LB_BSWAP_VLAN; + PMD_INIT_LOG(DEBUG, "IGB rx vlan bswap required"); + } else { + PMD_INIT_LOG(DEBUG, "IGB rx vlan bswap not required"); + } + /* Allocate buffers for descriptor rings and set up queue */ ret = igb_alloc_rx_queue_mbufs(rxq); if (ret) diff --git a/dpdk/drivers/net/ena/Makefile b/dpdk/drivers/net/ena/Makefile index a0d3358d..f9bfe053 100644 --- a/dpdk/drivers/net/ena/Makefile +++ b/dpdk/drivers/net/ena/Makefile @@ -51,11 +51,9 @@ SRCS-$(CONFIG_RTE_LIBRTE_ENA_PMD) += ena_ethdev.c SRCS-$(CONFIG_RTE_LIBRTE_ENA_PMD) += ena_com.c SRCS-$(CONFIG_RTE_LIBRTE_ENA_PMD) += ena_eth_com.c -# this lib depends upon: -DEPDIRS-$(CONFIG_RTE_LIBRTE_ENA_PMD) += lib/librte_eal lib/librte_ether -DEPDIRS-$(CONFIG_RTE_LIBRTE_ENA_PMD) += lib/librte_mempool lib/librte_mbuf -DEPDIRS-$(CONFIG_RTE_LIBRTE_ENA_PMD) += lib/librte_net - CFLAGS += $(INCLUDES) +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs +LDLIBS += -lrte_bus_pci include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/net/ena/base/ena_com.c b/dpdk/drivers/net/ena/base/ena_com.c index 88053e33..38a05877 100644 --- a/dpdk/drivers/net/ena/base/ena_com.c +++ b/dpdk/drivers/net/ena/base/ena_com.c @@ -2242,7 +2242,6 @@ int ena_com_set_hash_ctrl(struct ena_com_dev *ena_dev) { struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue; struct ena_rss *rss = &ena_dev->rss; - struct ena_admin_feature_rss_hash_control *hash_ctrl = rss->hash_ctrl; struct ena_admin_set_feat_cmd cmd; struct ena_admin_set_feat_resp resp; int ret; @@ -2269,7 +2268,8 @@ int ena_com_set_hash_ctrl(struct ena_com_dev *ena_dev) ena_trc_err("memory address set failed\n"); return ret; } - cmd.control_buffer.length = sizeof(*hash_ctrl); + cmd.control_buffer.length = + sizeof(struct ena_admin_feature_rss_hash_control); ret = ena_com_execute_admin_command(admin_queue, (struct ena_admin_aq_entry *)&cmd, @@ -2278,7 +2278,7 @@ int ena_com_set_hash_ctrl(struct ena_com_dev *ena_dev) sizeof(resp)); if (unlikely(ret)) { ena_trc_err("Failed to set hash input. error: %d\n", ret); - ret = ENA_COM_INVAL; + return ENA_COM_INVAL; } return 0; @@ -2590,19 +2590,11 @@ int ena_com_set_host_attributes(struct ena_com_dev *ena_dev) struct ena_com_admin_queue *admin_queue; struct ena_admin_set_feat_cmd cmd; struct ena_admin_set_feat_resp resp; + int ret; - int ret = 0; - - if (unlikely(!ena_dev)) { - ena_trc_err("%s : ena_dev is NULL\n", __func__); - return ENA_COM_NO_DEVICE; - } - - if (!ena_com_check_supported_feature_id(ena_dev, - ENA_ADMIN_HOST_ATTR_CONFIG)) { - ena_trc_warn("Set host attribute isn't supported\n"); - return ENA_COM_PERMISSION; - } + /* Host attribute config is called before ena_com_get_dev_attr_feat + * so ena_com can't check if the feature is supported. + */ memset(&cmd, 0x0, sizeof(cmd)); admin_queue = &ena_dev->admin_queue; diff --git a/dpdk/drivers/net/ena/base/ena_plat_dpdk.h b/dpdk/drivers/net/ena/base/ena_plat_dpdk.h index 87c3bf13..accecf51 100644 --- a/dpdk/drivers/net/ena/base/ena_plat_dpdk.h +++ b/dpdk/drivers/net/ena/base/ena_plat_dpdk.h @@ -44,6 +44,7 @@ #include #include #include +#include #include #include #include @@ -190,7 +191,7 @@ typedef uint64_t dma_addr_t; mz = rte_memzone_reserve(z_name, size, SOCKET_ID_ANY, 0); \ memset(mz->addr, 0, size); \ virt = mz->addr; \ - phys = mz->phys_addr; \ + phys = mz->iova; \ handle = mz; \ } while (0) #define ENA_MEM_FREE_COHERENT(dmadev, size, virt, phys, handle) \ @@ -206,8 +207,9 @@ typedef uint64_t dma_addr_t; snprintf(z_name, sizeof(z_name), \ "ena_alloc_%d", ena_alloc_cnt++); \ mz = rte_memzone_reserve(z_name, size, node, 0); \ + memset(mz->addr, 0, size); \ virt = mz->addr; \ - phys = mz->phys_addr; \ + phys = mz->iova; \ } while (0) #define ENA_MEM_ALLOC_NODE(dmadev, size, virt, node, dev_node) \ @@ -218,24 +220,15 @@ typedef uint64_t dma_addr_t; snprintf(z_name, sizeof(z_name), \ "ena_alloc_%d", ena_alloc_cnt++); \ mz = rte_memzone_reserve(z_name, size, node, 0); \ + memset(mz->addr, 0, size); \ virt = mz->addr; \ } while (0) #define ENA_MEM_ALLOC(dmadev, size) rte_zmalloc(NULL, size, 1) #define ENA_MEM_FREE(dmadev, ptr) ({ENA_TOUCH(dmadev); rte_free(ptr); }) -static inline void writel(u32 value, volatile void *addr) -{ - *(volatile u32 *)addr = value; -} - -static inline u32 readl(const volatile void *addr) -{ - return *(const volatile u32 *)addr; -} - -#define ENA_REG_WRITE32(value, reg) writel((value), (reg)) -#define ENA_REG_READ32(reg) readl((reg)) +#define ENA_REG_WRITE32(value, reg) rte_write32_relaxed((value), (reg)) +#define ENA_REG_READ32(reg) rte_read32_relaxed((reg)) #define ATOMIC32_INC(i32_ptr) rte_atomic32_inc(i32_ptr) #define ATOMIC32_DEC(i32_ptr) rte_atomic32_dec(i32_ptr) diff --git a/dpdk/drivers/net/ena/ena_ethdev.c b/dpdk/drivers/net/ena/ena_ethdev.c index 30581610..aa24ef36 100644 --- a/dpdk/drivers/net/ena/ena_ethdev.c +++ b/dpdk/drivers/net/ena/ena_ethdev.c @@ -33,12 +33,14 @@ #include #include +#include #include #include #include #include #include #include +#include #include "ena_ethdev.h" #include "ena_logs.h" @@ -168,7 +170,15 @@ static const struct ena_stats ena_stats_ena_com_strings[] = { #define PCI_DEVICE_ID_ENA_VF 0xEC20 #define PCI_DEVICE_ID_ENA_LLQ_VF 0xEC21 -static struct rte_pci_id pci_id_ena_map[] = { +#define ENA_TX_OFFLOAD_MASK (\ + PKT_TX_L4_MASK | \ + PKT_TX_IP_CKSUM | \ + PKT_TX_TCP_SEG) + +#define ENA_TX_OFFLOAD_NOTSUP_MASK \ + (PKT_TX_OFFLOAD_MASK ^ ENA_TX_OFFLOAD_MASK) + +static const struct rte_pci_id pci_id_ena_map[] = { { RTE_PCI_DEVICE(PCI_VENDOR_ID_AMAZON, PCI_DEVICE_ID_ENA_VF) }, { RTE_PCI_DEVICE(PCI_VENDOR_ID_AMAZON, PCI_DEVICE_ID_ENA_LLQ_VF) }, { .device_id = 0 }, @@ -179,6 +189,8 @@ static int ena_device_init(struct ena_com_dev *ena_dev, static int ena_dev_configure(struct rte_eth_dev *dev); static uint16_t eth_ena_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts); +static uint16_t eth_ena_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts); static int ena_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, uint16_t nb_desc, unsigned int socket_id, const struct rte_eth_txconf *tx_conf); @@ -193,7 +205,7 @@ static void ena_init_rings(struct ena_adapter *adapter); static int ena_mtu_set(struct rte_eth_dev *dev, uint16_t mtu); static int ena_start(struct rte_eth_dev *dev); static void ena_close(struct rte_eth_dev *dev); -static void ena_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats); +static int ena_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats); static void ena_rx_queue_release_all(struct rte_eth_dev *dev); static void ena_tx_queue_release_all(struct rte_eth_dev *dev); static void ena_rx_queue_release(void *queue); @@ -201,12 +213,12 @@ static void ena_tx_queue_release(void *queue); static void ena_rx_queue_release_bufs(struct ena_ring *ring); static void ena_tx_queue_release_bufs(struct ena_ring *ring); static int ena_link_update(struct rte_eth_dev *dev, - __rte_unused int wait_to_complete); + int wait_to_complete); static int ena_queue_restart(struct ena_ring *ring); static int ena_queue_restart_all(struct rte_eth_dev *dev, enum ena_ring_type ring_type); static void ena_stats_restart(struct rte_eth_dev *dev); -static void ena_infos_get(__rte_unused struct rte_eth_dev *dev, +static void ena_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info); static int ena_rss_reta_update(struct rte_eth_dev *dev, struct rte_eth_rss_reta_entry64 *reta_conf, @@ -216,7 +228,7 @@ static int ena_rss_reta_query(struct rte_eth_dev *dev, uint16_t reta_size); static int ena_get_sset_count(struct rte_eth_dev *dev, int sset); -static struct eth_dev_ops ena_dev_ops = { +static const struct eth_dev_ops ena_dev_ops = { .dev_configure = ena_dev_configure, .dev_infos_get = ena_infos_get, .rx_queue_setup = ena_rx_queue_setup, @@ -248,16 +260,17 @@ static inline void ena_rx_mbuf_prepare(struct rte_mbuf *mbuf, struct ena_com_rx_ctx *ena_rx_ctx) { uint64_t ol_flags = 0; + uint32_t packet_type = 0; if (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_TCP) - ol_flags |= PKT_TX_TCP_CKSUM; + packet_type |= RTE_PTYPE_L4_TCP; else if (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_UDP) - ol_flags |= PKT_TX_UDP_CKSUM; + packet_type |= RTE_PTYPE_L4_UDP; if (ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV4) - ol_flags |= PKT_TX_IPV4; + packet_type |= RTE_PTYPE_L3_IPV4; else if (ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV6) - ol_flags |= PKT_TX_IPV6; + packet_type |= RTE_PTYPE_L3_IPV6; if (unlikely(ena_rx_ctx->l4_csum_err)) ol_flags |= PKT_RX_L4_CKSUM_BAD; @@ -265,6 +278,7 @@ static inline void ena_rx_mbuf_prepare(struct rte_mbuf *mbuf, ol_flags |= PKT_RX_IP_CKSUM_BAD; mbuf->ol_flags = ol_flags; + mbuf->packet_type = packet_type; } static inline void ena_tx_mbuf_prepare(struct rte_mbuf *mbuf, @@ -357,12 +371,9 @@ static void ena_config_host_info(struct ena_com_dev *ena_dev) rc = ena_com_set_host_attributes(ena_dev); if (rc) { - if (rc == -EPERM) - RTE_LOG(ERR, PMD, "Cannot set host attributes\n"); - else - RTE_LOG(ERR, PMD, "Cannot set host attributes\n"); - - goto err; + RTE_LOG(ERR, PMD, "Cannot set host attributes\n"); + if (rc != -EPERM) + goto err; } return; @@ -413,11 +424,9 @@ static void ena_config_debug_area(struct ena_adapter *adapter) rc = ena_com_set_host_attributes(&adapter->ena_dev); if (rc) { - if (rc == -EPERM) - RTE_LOG(WARNING, PMD, "Cannot set host attributes\n"); - else - RTE_LOG(ERR, PMD, "Cannot set host attributes\n"); - goto err; + RTE_LOG(WARNING, PMD, "Cannot set host attributes\n"); + if (rc != -EPERM) + goto err; } return; @@ -674,26 +683,23 @@ static void ena_rx_queue_release_bufs(struct ena_ring *ring) ring->rx_buffer_info[ring->next_to_clean & ring_mask]; if (m) - __rte_mbuf_raw_free(m); + rte_mbuf_raw_free(m); - ring->next_to_clean = - ENA_CIRC_INC(ring->next_to_clean, 1, ring->ring_size); + ring->next_to_clean++; } } static void ena_tx_queue_release_bufs(struct ena_ring *ring) { - unsigned int ring_mask = ring->ring_size - 1; + unsigned int i; - while (ring->next_to_clean != ring->next_to_use) { - struct ena_tx_buffer *tx_buf = - &ring->tx_buffer_info[ring->next_to_clean & ring_mask]; + for (i = 0; i < ring->ring_size; ++i) { + struct ena_tx_buffer *tx_buf = &ring->tx_buffer_info[i]; if (tx_buf->mbuf) rte_pktmbuf_free(tx_buf->mbuf); - ring->next_to_clean = - ENA_CIRC_INC(ring->next_to_clean, 1, ring->ring_size); + ring->next_to_clean++; } } @@ -737,7 +743,7 @@ static int ena_queue_restart_all(struct rte_eth_dev *dev, if (rc) { PMD_INIT_LOG(ERR, - "failed to restart queue %d type(%d)\n", + "failed to restart queue %d type(%d)", i, ring_type); return -1; } @@ -763,7 +769,7 @@ static int ena_check_valid_conf(struct ena_adapter *adapter) uint32_t max_frame_len = ena_get_mtu_conf(adapter); if (max_frame_len > adapter->max_mtu) { - PMD_INIT_LOG(ERR, "Unsupported MTU of %d\n", max_frame_len); + PMD_INIT_LOG(ERR, "Unsupported MTU of %d", max_frame_len); return -1; } @@ -790,7 +796,7 @@ ena_calc_queue_size(struct ena_com_dev *ena_dev, queue_size = rte_align32pow2(queue_size >> 1); if (queue_size == 0) { - PMD_INIT_LOG(ERR, "Invalid queue size\n"); + PMD_INIT_LOG(ERR, "Invalid queue size"); return -EFAULT; } @@ -807,7 +813,7 @@ static void ena_stats_restart(struct rte_eth_dev *dev) rte_atomic64_init(&adapter->drv_stats->rx_nombuf); } -static void ena_stats_get(struct rte_eth_dev *dev, +static int ena_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) { struct ena_admin_basic_stats ena_stats; @@ -817,13 +823,13 @@ static void ena_stats_get(struct rte_eth_dev *dev, int rc; if (rte_eal_process_type() != RTE_PROC_PRIMARY) - return; + return -ENOTSUP; memset(&ena_stats, 0, sizeof(ena_stats)); rc = ena_com_get_dev_basic_stats(ena_dev, &ena_stats); if (unlikely(rc)) { RTE_LOG(ERR, PMD, "Could not retrieve statistics from ENA"); - return; + return rc; } /* Set of basic statistics from ENA */ @@ -842,6 +848,7 @@ static void ena_stats_get(struct rte_eth_dev *dev, stats->ierrors = rte_atomic64_read(&adapter->drv_stats->ierrors); stats->oerrors = rte_atomic64_read(&adapter->drv_stats->oerrors); stats->rx_nombuf = rte_atomic64_read(&adapter->drv_stats->rx_nombuf); + return 0; } static int ena_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) @@ -915,7 +922,7 @@ static int ena_start(struct rte_eth_dev *dev) static int ena_queue_restart(struct ena_ring *ring) { - int rc; + int rc, bufs_num; ena_assert_msg(ring->configured == 1, "Trying to restart unconfigured queue\n"); @@ -926,9 +933,10 @@ static int ena_queue_restart(struct ena_ring *ring) if (ring->type == ENA_RING_TYPE_TX) return 0; - rc = ena_populate_rx_queue(ring, ring->ring_size - 1); - if ((unsigned int)rc != ring->ring_size - 1) { - PMD_INIT_LOG(ERR, "Failed to populate rx ring !\n"); + bufs_num = ring->ring_size - 1; + rc = ena_populate_rx_queue(ring, bufs_num); + if (rc != bufs_num) { + PMD_INIT_LOG(ERR, "Failed to populate rx ring !"); return (-1); } @@ -962,6 +970,13 @@ static int ena_tx_queue_setup(struct rte_eth_dev *dev, return -1; } + if (!rte_is_power_of_2(nb_desc)) { + RTE_LOG(ERR, PMD, + "Unsupported size of RX queue: %d is not a power of 2.", + nb_desc); + return -EINVAL; + } + if (nb_desc > adapter->tx_ring_size) { RTE_LOG(ERR, PMD, "Unsupported size of TX queue (max size: %d)\n", @@ -1056,6 +1071,13 @@ static int ena_rx_queue_setup(struct rte_eth_dev *dev, return -1; } + if (!rte_is_power_of_2(nb_desc)) { + RTE_LOG(ERR, PMD, + "Unsupported size of TX queue: %d is not a power of 2.", + nb_desc); + return -EINVAL; + } + if (nb_desc > adapter->rx_ring_size) { RTE_LOG(ERR, PMD, "Unsupported size of RX queue (max size: %d)\n", @@ -1115,23 +1137,25 @@ static int ena_populate_rx_queue(struct ena_ring *rxq, unsigned int count) { unsigned int i; int rc; - unsigned int ring_size = rxq->ring_size; - unsigned int ring_mask = ring_size - 1; - int next_to_use = rxq->next_to_use & ring_mask; + uint16_t ring_size = rxq->ring_size; + uint16_t ring_mask = ring_size - 1; + uint16_t next_to_use = rxq->next_to_use; + uint16_t in_use; struct rte_mbuf **mbufs = &rxq->rx_buffer_info[0]; if (unlikely(!count)) return 0; - ena_assert_msg((((ENA_CIRC_COUNT(rxq->next_to_use, rxq->next_to_clean, - rxq->ring_size)) + - count) < rxq->ring_size), "bad ring state"); + in_use = rxq->next_to_use - rxq->next_to_clean; + ena_assert_msg(((in_use + count) < ring_size), "bad ring state"); - count = RTE_MIN(count, ring_size - next_to_use); + count = RTE_MIN(count, + (uint16_t)(ring_size - (next_to_use & ring_mask))); /* get resources for incoming packets */ rc = rte_mempool_get_bulk(rxq->mb_pool, - (void **)(&mbufs[next_to_use]), count); + (void **)(&mbufs[next_to_use & ring_mask]), + count); if (unlikely(rc < 0)) { rte_atomic64_inc(&rxq->adapter->drv_stats->rx_nombuf); PMD_RX_LOG(DEBUG, "there are no enough free buffers"); @@ -1139,21 +1163,24 @@ static int ena_populate_rx_queue(struct ena_ring *rxq, unsigned int count) } for (i = 0; i < count; i++) { - struct rte_mbuf *mbuf = mbufs[next_to_use]; + uint16_t next_to_use_masked = next_to_use & ring_mask; + struct rte_mbuf *mbuf = mbufs[next_to_use_masked]; struct ena_com_buf ebuf; rte_prefetch0(mbufs[((next_to_use + 4) & ring_mask)]); /* prepare physical address for DMA transaction */ - ebuf.paddr = mbuf->buf_physaddr + RTE_PKTMBUF_HEADROOM; + ebuf.paddr = mbuf->buf_iova + RTE_PKTMBUF_HEADROOM; ebuf.len = mbuf->buf_len - RTE_PKTMBUF_HEADROOM; /* pass resource to device */ rc = ena_com_add_single_rx_desc(rxq->ena_com_io_sq, - &ebuf, next_to_use); + &ebuf, next_to_use_masked); if (unlikely(rc)) { + rte_mempool_put_bulk(rxq->mb_pool, (void **)(&mbuf), + count - i); RTE_LOG(WARNING, PMD, "failed adding rx desc\n"); break; } - next_to_use = ENA_RX_RING_IDX_NEXT(next_to_use, ring_size); + next_to_use++; } /* When we submitted free recources to device... */ @@ -1213,14 +1240,14 @@ static int ena_device_init(struct ena_com_dev *ena_dev, goto err_mmio_read_less; } - ena_config_host_info(ena_dev); - /* To enable the msix interrupts the driver needs to know the number * of queues. So the driver uses polling mode to retrieve this * information. */ ena_com_set_admin_polling_mode(ena_dev, true); + ena_config_host_info(ena_dev); + /* Get Device Attributes and features */ rc = ena_com_get_dev_attr_feat(ena_dev, get_feat_ctx); if (rc) { @@ -1257,16 +1284,17 @@ static int eth_ena_dev_init(struct rte_eth_dev *eth_dev) eth_dev->dev_ops = &ena_dev_ops; eth_dev->rx_pkt_burst = ð_ena_recv_pkts; eth_dev->tx_pkt_burst = ð_ena_xmit_pkts; + eth_dev->tx_pkt_prepare = ð_ena_prep_pkts; adapter->rte_eth_dev_data = eth_dev->data; adapter->rte_dev = eth_dev; if (rte_eal_process_type() != RTE_PROC_PRIMARY) return 0; - pci_dev = eth_dev->pci_dev; + pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); adapter->pdev = pci_dev; - PMD_INIT_LOG(INFO, "Initializing %x:%x:%x.%d\n", + PMD_INIT_LOG(INFO, "Initializing %x:%x:%x.%d", pci_dev->addr.domain, pci_dev->addr.bus, pci_dev->addr.devid, @@ -1283,7 +1311,7 @@ static int eth_ena_dev_init(struct rte_eth_dev *eth_dev) else if (adapter->regs) ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; else - PMD_INIT_LOG(CRIT, "Failed to access registers BAR(%d)\n", + PMD_INIT_LOG(CRIT, "Failed to access registers BAR(%d)", ENA_REGS_BAR); ena_dev->reg_bar = adapter->regs; @@ -1297,7 +1325,7 @@ static int eth_ena_dev_init(struct rte_eth_dev *eth_dev) /* device specific initialization routine */ rc = ena_device_init(ena_dev, &get_feat_ctx); if (rc) { - PMD_INIT_LOG(CRIT, "Failed to init ENA device\n"); + PMD_INIT_LOG(CRIT, "Failed to init ENA device"); return -1; } @@ -1305,7 +1333,7 @@ static int eth_ena_dev_init(struct rte_eth_dev *eth_dev) if (get_feat_ctx.max_queues.max_llq_num == 0) { PMD_INIT_LOG(ERR, "Trying to use LLQ but llq_num is 0.\n" - "Fall back into regular queues.\n"); + "Fall back into regular queues."); ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; adapter->num_queues = @@ -1333,6 +1361,10 @@ static int eth_ena_dev_init(struct rte_eth_dev *eth_dev) /* Set max MTU for this device */ adapter->max_mtu = get_feat_ctx.dev_attr.max_mtu; + /* set device support for TSO */ + adapter->tso4_supported = get_feat_ctx.offload.tx & + ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK; + /* Copy MAC address and point DPDK to it */ eth_dev->data->mac_addrs = (struct ether_addr *)adapter->mac_addr; ether_addr_copy((struct ether_addr *)get_feat_ctx.dev_attr.mac_addr, @@ -1359,7 +1391,7 @@ static int ena_dev_configure(struct rte_eth_dev *dev) if (!(adapter->state == ENA_ADAPTER_STATE_INIT || adapter->state == ENA_ADAPTER_STATE_STOPPED)) { - PMD_INIT_LOG(ERR, "Illegal adapter state: %d\n", + PMD_INIT_LOG(ERR, "Illegal adapter state: %d", adapter->state); return -1; } @@ -1421,6 +1453,8 @@ static void ena_infos_get(struct rte_eth_dev *dev, ena_dev = &adapter->ena_dev; ena_assert_msg(ena_dev != NULL, "Uninitialized device"); + dev_info->pci_dev = RTE_ETH_DEV_TO_PCI(dev); + dev_info->speed_capa = ETH_LINK_SPEED_1G | ETH_LINK_SPEED_2_5G | @@ -1449,7 +1483,7 @@ static void ena_infos_get(struct rte_eth_dev *dev, DEV_TX_OFFLOAD_UDP_CKSUM | DEV_TX_OFFLOAD_TCP_CKSUM; - if (feat.offload.tx & + if (feat.offload.rx_supported & ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_MASK) rx_feat |= DEV_RX_OFFLOAD_IPV4_CKSUM | DEV_RX_OFFLOAD_UDP_CKSUM | @@ -1475,7 +1509,7 @@ static uint16_t eth_ena_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, unsigned int ring_size = rx_ring->ring_size; unsigned int ring_mask = ring_size - 1; uint16_t next_to_clean = rx_ring->next_to_clean; - int desc_in_use = 0; + uint16_t desc_in_use = 0; unsigned int recv_idx = 0; struct rte_mbuf *mbuf = NULL; struct rte_mbuf *mbuf_head = NULL; @@ -1493,8 +1527,7 @@ static uint16_t eth_ena_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, return 0; } - desc_in_use = ENA_CIRC_COUNT(rx_ring->next_to_use, - next_to_clean, ring_size); + desc_in_use = rx_ring->next_to_use - next_to_clean; if (unlikely(nb_pkts > desc_in_use)) nb_pkts = desc_in_use; @@ -1535,8 +1568,7 @@ static uint16_t eth_ena_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, mbuf_prev = mbuf; segments++; - next_to_clean = - ENA_RX_RING_IDX_NEXT(next_to_clean, ring_size); + next_to_clean++; } /* fill mbuf attributes if any */ @@ -1548,20 +1580,92 @@ static uint16_t eth_ena_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, recv_idx++; } - /* Burst refill to save doorbells, memory barriers, const interval */ - if (ring_size - desc_in_use - 1 > ENA_RING_DESCS_RATIO(ring_size)) - ena_populate_rx_queue(rx_ring, ring_size - desc_in_use - 1); + rx_ring->next_to_clean = next_to_clean; - rx_ring->next_to_clean = next_to_clean & ring_mask; + desc_in_use = desc_in_use - completed + 1; + /* Burst refill to save doorbells, memory barriers, const interval */ + if (ring_size - desc_in_use > ENA_RING_DESCS_RATIO(ring_size)) + ena_populate_rx_queue(rx_ring, ring_size - desc_in_use); return recv_idx; } +static uint16_t +eth_ena_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts) +{ + int32_t ret; + uint32_t i; + struct rte_mbuf *m; + struct ena_ring *tx_ring = (struct ena_ring *)(tx_queue); + struct ipv4_hdr *ip_hdr; + uint64_t ol_flags; + uint16_t frag_field; + + for (i = 0; i != nb_pkts; i++) { + m = tx_pkts[i]; + ol_flags = m->ol_flags; + + if (!(ol_flags & PKT_TX_IPV4)) + continue; + + /* If there was not L2 header length specified, assume it is + * length of the ethernet header. + */ + if (unlikely(m->l2_len == 0)) + m->l2_len = sizeof(struct ether_hdr); + + ip_hdr = rte_pktmbuf_mtod_offset(m, struct ipv4_hdr *, + m->l2_len); + frag_field = rte_be_to_cpu_16(ip_hdr->fragment_offset); + + if ((frag_field & IPV4_HDR_DF_FLAG) != 0) { + m->packet_type |= RTE_PTYPE_L4_NONFRAG; + + /* If IPv4 header has DF flag enabled and TSO support is + * disabled, partial chcecksum should not be calculated. + */ + if (!tx_ring->adapter->tso4_supported) + continue; + } + + if ((ol_flags & ENA_TX_OFFLOAD_NOTSUP_MASK) != 0 || + (ol_flags & PKT_TX_L4_MASK) == + PKT_TX_SCTP_CKSUM) { + rte_errno = -ENOTSUP; + return i; + } + +#ifdef RTE_LIBRTE_ETHDEV_DEBUG + ret = rte_validate_tx_offload(m); + if (ret != 0) { + rte_errno = ret; + return i; + } +#endif + + /* In case we are supposed to TSO and have DF not set (DF=0) + * hardware must be provided with partial checksum, otherwise + * it will take care of necessary calculations. + */ + + ret = rte_net_intel_cksum_flags_prepare(m, + ol_flags & ~PKT_TX_TCP_SEG); + if (ret != 0) { + rte_errno = ret; + return i; + } + } + + return i; +} + static uint16_t eth_ena_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { struct ena_ring *tx_ring = (struct ena_ring *)(tx_queue); - unsigned int next_to_use = tx_ring->next_to_use; + uint16_t next_to_use = tx_ring->next_to_use; + uint16_t next_to_clean = tx_ring->next_to_clean; struct rte_mbuf *mbuf; unsigned int ring_size = tx_ring->ring_size; unsigned int ring_mask = ring_size - 1; @@ -1569,7 +1673,7 @@ static uint16_t eth_ena_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, struct ena_tx_buffer *tx_info; struct ena_com_buf *ebuf; uint16_t rc, req_id, total_tx_descs = 0; - uint16_t sent_idx = 0; + uint16_t sent_idx = 0, empty_tx_reqs; int nb_hw_desc; /* Check adapter state */ @@ -1579,10 +1683,14 @@ static uint16_t eth_ena_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, return 0; } + empty_tx_reqs = ring_size - (next_to_use - next_to_clean); + if (nb_pkts > empty_tx_reqs) + nb_pkts = empty_tx_reqs; + for (sent_idx = 0; sent_idx < nb_pkts; sent_idx++) { mbuf = tx_pkts[sent_idx]; - req_id = tx_ring->empty_tx_reqs[next_to_use]; + req_id = tx_ring->empty_tx_reqs[next_to_use & ring_mask]; tx_info = &tx_ring->tx_buffer_info[req_id]; tx_info->mbuf = mbuf; tx_info->num_of_bufs = 0; @@ -1620,7 +1728,7 @@ static uint16_t eth_ena_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, * consideration pushed header */ if (mbuf->data_len > ena_tx_ctx.header_len) { - ebuf->paddr = mbuf->buf_physaddr + + ebuf->paddr = mbuf->buf_iova + mbuf->data_off + ena_tx_ctx.header_len; ebuf->len = mbuf->data_len - ena_tx_ctx.header_len; @@ -1629,7 +1737,7 @@ static uint16_t eth_ena_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, } while ((mbuf = mbuf->next) != NULL) { - ebuf->paddr = mbuf->buf_physaddr + mbuf->data_off; + ebuf->paddr = mbuf->buf_iova + mbuf->data_off; ebuf->len = mbuf->data_len; ebuf++; tx_info->num_of_bufs++; @@ -1645,7 +1753,7 @@ static uint16_t eth_ena_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, tx_info->tx_descs = nb_hw_desc; - next_to_use = ENA_TX_RING_IDX_NEXT(next_to_use, ring_size); + next_to_use++; } /* If there are ready packets to be xmitted... */ @@ -1666,12 +1774,11 @@ static uint16_t eth_ena_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, /* Free whole mbuf chain */ mbuf = tx_info->mbuf; rte_pktmbuf_free(mbuf); + tx_info->mbuf = NULL; /* Put back descriptor to the ring for reuse */ - tx_ring->empty_tx_reqs[tx_ring->next_to_clean] = req_id; - tx_ring->next_to_clean = - ENA_TX_RING_IDX_NEXT(tx_ring->next_to_clean, - tx_ring->ring_size); + tx_ring->empty_tx_reqs[next_to_clean & ring_mask] = req_id; + next_to_clean++; /* If too many descs to clean, leave it for another run */ if (unlikely(total_tx_descs > ENA_RING_DESCS_RATIO(ring_size))) @@ -1681,33 +1788,31 @@ static uint16_t eth_ena_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, if (total_tx_descs > 0) { /* acknowledge completion of sent packets */ ena_com_comp_ack(tx_ring->ena_com_io_sq, total_tx_descs); + tx_ring->next_to_clean = next_to_clean; } return sent_idx; } -static struct eth_driver rte_ena_pmd = { - { - .name = "rte_ena_pmd", - .id_table = pci_id_ena_map, - .drv_flags = RTE_PCI_DRV_NEED_MAPPING, - }, - .eth_dev_init = eth_ena_dev_init, - .dev_private_size = sizeof(struct ena_adapter), -}; - -static int -rte_ena_pmd_init(const char *name __rte_unused, - const char *params __rte_unused) +static int eth_ena_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, + struct rte_pci_device *pci_dev) { - rte_eth_driver_register(&rte_ena_pmd); - return 0; + return rte_eth_dev_pci_generic_probe(pci_dev, + sizeof(struct ena_adapter), eth_ena_dev_init); +} + +static int eth_ena_pci_remove(struct rte_pci_device *pci_dev) +{ + return rte_eth_dev_pci_generic_remove(pci_dev, NULL); +} + +static struct rte_pci_driver rte_ena_pmd = { + .id_table = pci_id_ena_map, + .drv_flags = RTE_PCI_DRV_NEED_MAPPING, + .probe = eth_ena_pci_probe, + .remove = eth_ena_pci_remove, }; -struct rte_driver ena_pmd_drv = { - .type = PMD_PDEV, - .init = rte_ena_pmd_init, -}; - -PMD_REGISTER_DRIVER(ena_pmd_drv, ena); -DRIVER_REGISTER_PCI_TABLE(ena, pci_id_ena_map); +RTE_PMD_REGISTER_PCI(net_ena, rte_ena_pmd); +RTE_PMD_REGISTER_PCI_TABLE(net_ena, pci_id_ena_map); +RTE_PMD_REGISTER_KMOD_DEP(net_ena, "* igb_uio | uio_pci_generic | vfio-pci"); diff --git a/dpdk/drivers/net/ena/ena_ethdev.h b/dpdk/drivers/net/ena/ena_ethdev.h index 61390a93..be8bc9fa 100644 --- a/dpdk/drivers/net/ena/ena_ethdev.h +++ b/dpdk/drivers/net/ena/ena_ethdev.h @@ -35,6 +35,7 @@ #define _ENA_ETHDEV_H_ #include +#include #include "ena_com.h" @@ -42,33 +43,13 @@ #define ENA_MEM_BAR 2 #define ENA_MAX_NUM_QUEUES 128 - -#define ENA_DEFAULT_TX_SW_DESCS (1024) -#define ENA_DEFAULT_TX_HW_DESCS (1024) #define ENA_DEFAULT_RING_SIZE (1024) - #define ENA_MIN_FRAME_LEN 64 - -#define ENA_NAME_MAX_LEN 20 -#define ENA_IRQNAME_SIZE 40 - -#define ENA_PKT_MAX_BUFS 17 +#define ENA_NAME_MAX_LEN 20 +#define ENA_PKT_MAX_BUFS 17 #define ENA_MMIO_DISABLE_REG_READ BIT(0) -#define ENA_CIRC_COUNT(head, tail, size) \ - (((uint16_t)((uint16_t)(head) - (uint16_t)(tail))) & ((size) - 1)) - -#define ENA_CIRC_INC(index, step, size) \ - ((uint16_t)(index) + (uint16_t)(step)) -#define ENA_CIRC_INC_WRAP(index, step, size) \ - (((uint16_t)(index) + (uint16_t)(step)) & ((size) - 1)) - -#define ENA_TX_RING_IDX_NEXT(idx, ring_size) \ - ENA_CIRC_INC_WRAP(idx, 1, ring_size) -#define ENA_RX_RING_IDX_NEXT(idx, ring_size) \ - ENA_CIRC_INC_WRAP(idx, 1, ring_size) - struct ena_adapter; enum ena_ring_type { @@ -182,6 +163,7 @@ struct ena_adapter { u16 num_queues; u16 max_mtu; + u8 tso4_supported; int id_number; char name[ENA_NAME_MAX_LEN]; diff --git a/dpdk/drivers/net/enic/Makefile b/dpdk/drivers/net/enic/Makefile index 3926b795..5191db54 100644 --- a/dpdk/drivers/net/enic/Makefile +++ b/dpdk/drivers/net/enic/Makefile @@ -45,6 +45,9 @@ CFLAGS += -I$(SRCDIR)/base/ CFLAGS += -I$(SRCDIR) CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) -Wno-strict-aliasing +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs -lrte_hash +LDLIBS += -lrte_bus_pci VPATH += $(SRCDIR)/src @@ -56,6 +59,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_main.c SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_rxtx.c SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_clsf.c SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_res.c +SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_flow.c SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += base/vnic_cq.c SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += base/vnic_wq.c SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += base/vnic_dev.c @@ -63,10 +67,4 @@ SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += base/vnic_intr.c SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += base/vnic_rq.c SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += base/vnic_rss.c -# this lib depends upon: -DEPDIRS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += lib/librte_eal lib/librte_ether -DEPDIRS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += lib/librte_mempool lib/librte_mbuf -DEPDIRS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += lib/librte_net -DEPDIRS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += lib/librte_hash - include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/net/enic/base/cq_enet_desc.h b/dpdk/drivers/net/enic/base/cq_enet_desc.h index f9822a45..e8410563 100644 --- a/dpdk/drivers/net/enic/base/cq_enet_desc.h +++ b/dpdk/drivers/net/enic/base/cq_enet_desc.h @@ -71,6 +71,19 @@ struct cq_enet_rq_desc { u8 type_color; }; +/* Completion queue descriptor: Ethernet receive queue, 16B */ +struct cq_enet_rq_clsf_desc { + __le16 completed_index_flags; + __le16 q_number_rss_type_flags; + __le16 filter_id; + __le16 lif; + __le16 bytes_written_flags; + __le16 vlan; + __le16 checksum_fcoe; + u8 flags; + u8 type_color; +}; + #define CQ_ENET_RQ_DESC_FLAGS_INGRESS_PORT (0x1 << 12) #define CQ_ENET_RQ_DESC_FLAGS_FCOE (0x1 << 13) #define CQ_ENET_RQ_DESC_FLAGS_EOP (0x1 << 14) diff --git a/dpdk/drivers/net/enic/base/vnic_cq.c b/dpdk/drivers/net/enic/base/vnic_cq.c index 2f65f357..3549fece 100644 --- a/dpdk/drivers/net/enic/base/vnic_cq.c +++ b/dpdk/drivers/net/enic/base/vnic_cq.c @@ -35,16 +35,6 @@ #include "vnic_dev.h" #include "vnic_cq.h" -int vnic_cq_mem_size(struct vnic_cq *cq, unsigned int desc_count, - unsigned int desc_size) -{ - int mem_size; - - mem_size = vnic_dev_desc_ring_size(&cq->ring, desc_count, desc_size); - - return mem_size; -} - void vnic_cq_free(struct vnic_cq *cq) { vnic_dev_free_desc_ring(cq->vdev, &cq->ring); @@ -65,11 +55,11 @@ int vnic_cq_alloc(struct vnic_dev *vdev, struct vnic_cq *cq, unsigned int index, cq->ctrl = vnic_dev_get_res(vdev, RES_TYPE_CQ, index); if (!cq->ctrl) { - pr_err("Failed to hook CQ[%d] resource\n", index); + pr_err("Failed to hook CQ[%u] resource\n", index); return -EINVAL; } - snprintf(res_name, sizeof(res_name), "%d-cq-%d", instance++, index); + snprintf(res_name, sizeof(res_name), "%d-cq-%u", instance++, index); err = vnic_dev_alloc_desc_ring(vdev, &cq->ring, desc_count, desc_size, socket_id, res_name); if (err) diff --git a/dpdk/drivers/net/enic/base/vnic_dev.c b/dpdk/drivers/net/enic/base/vnic_dev.c index 4db21a4d..9b25d219 100644 --- a/dpdk/drivers/net/enic/base/vnic_dev.c +++ b/dpdk/drivers/net/enic/base/vnic_dev.c @@ -272,7 +272,7 @@ int vnic_dev_alloc_desc_ring(struct vnic_dev *vdev, __attribute__((unused)) unsigned int socket_id, char *z_name) { - void *alloc_addr = NULL; + void *alloc_addr; dma_addr_t alloc_pa = 0; vnic_dev_desc_ring_size(ring, desc_count, desc_size); @@ -387,17 +387,24 @@ static int _vnic_dev_cmd(struct vnic_dev *vdev, enum vnic_devcmd_cmd cmd, static int vnic_dev_cmd_proxy(struct vnic_dev *vdev, enum vnic_devcmd_cmd proxy_cmd, enum vnic_devcmd_cmd cmd, - u64 *a0, u64 *a1, int wait) + u64 *args, int nargs, int wait) { u32 status; int err; + /* + * Proxy command consumes 2 arguments. One for proxy index, + * the other is for command to be proxied + */ + if (nargs > VNIC_DEVCMD_NARGS - 2) { + pr_err("number of args %d exceeds the maximum\n", nargs); + return -EINVAL; + } memset(vdev->args, 0, sizeof(vdev->args)); vdev->args[0] = vdev->proxy_index; vdev->args[1] = cmd; - vdev->args[2] = *a0; - vdev->args[3] = *a1; + memcpy(&vdev->args[2], args, nargs * sizeof(args[0])); err = _vnic_dev_cmd(vdev, proxy_cmd, wait); if (err) @@ -412,65 +419,159 @@ static int vnic_dev_cmd_proxy(struct vnic_dev *vdev, return err; } - *a0 = vdev->args[1]; - *a1 = vdev->args[2]; + memcpy(args, &vdev->args[1], nargs * sizeof(args[0])); return 0; } static int vnic_dev_cmd_no_proxy(struct vnic_dev *vdev, - enum vnic_devcmd_cmd cmd, u64 *a0, u64 *a1, int wait) + enum vnic_devcmd_cmd cmd, u64 *args, int nargs, int wait) { int err; - vdev->args[0] = *a0; - vdev->args[1] = *a1; + if (nargs > VNIC_DEVCMD_NARGS) { + pr_err("number of args %d exceeds the maximum\n", nargs); + return -EINVAL; + } + memset(vdev->args, 0, sizeof(vdev->args)); + memcpy(vdev->args, args, nargs * sizeof(args[0])); err = _vnic_dev_cmd(vdev, cmd, wait); - *a0 = vdev->args[0]; - *a1 = vdev->args[1]; + memcpy(args, vdev->args, nargs * sizeof(args[0])); return err; } -void vnic_dev_cmd_proxy_by_index_start(struct vnic_dev *vdev, u16 index) -{ - vdev->proxy = PROXY_BY_INDEX; - vdev->proxy_index = index; -} - -void vnic_dev_cmd_proxy_by_bdf_start(struct vnic_dev *vdev, u16 bdf) -{ - vdev->proxy = PROXY_BY_BDF; - vdev->proxy_index = bdf; -} - -void vnic_dev_cmd_proxy_end(struct vnic_dev *vdev) -{ - vdev->proxy = PROXY_NONE; - vdev->proxy_index = 0; -} - int vnic_dev_cmd(struct vnic_dev *vdev, enum vnic_devcmd_cmd cmd, u64 *a0, u64 *a1, int wait) { + u64 args[2]; + int err; + + args[0] = *a0; + args[1] = *a1; memset(vdev->args, 0, sizeof(vdev->args)); switch (vdev->proxy) { case PROXY_BY_INDEX: - return vnic_dev_cmd_proxy(vdev, CMD_PROXY_BY_INDEX, cmd, - a0, a1, wait); + err = vnic_dev_cmd_proxy(vdev, CMD_PROXY_BY_INDEX, cmd, + args, ARRAY_SIZE(args), wait); + break; case PROXY_BY_BDF: - return vnic_dev_cmd_proxy(vdev, CMD_PROXY_BY_BDF, cmd, - a0, a1, wait); + err = vnic_dev_cmd_proxy(vdev, CMD_PROXY_BY_BDF, cmd, + args, ARRAY_SIZE(args), wait); + break; case PROXY_NONE: default: - return vnic_dev_cmd_no_proxy(vdev, cmd, a0, a1, wait); + err = vnic_dev_cmd_no_proxy(vdev, cmd, args, 2, wait); + break; + } + + if (err == 0) { + *a0 = args[0]; + *a1 = args[1]; + } + + return err; +} + +int vnic_dev_cmd_args(struct vnic_dev *vdev, enum vnic_devcmd_cmd cmd, + u64 *args, int nargs, int wait) +{ + switch (vdev->proxy) { + case PROXY_BY_INDEX: + return vnic_dev_cmd_proxy(vdev, CMD_PROXY_BY_INDEX, cmd, + args, nargs, wait); + case PROXY_BY_BDF: + return vnic_dev_cmd_proxy(vdev, CMD_PROXY_BY_BDF, cmd, + args, nargs, wait); + case PROXY_NONE: + default: + return vnic_dev_cmd_no_proxy(vdev, cmd, args, nargs, wait); } } -static int vnic_dev_capable(struct vnic_dev *vdev, enum vnic_devcmd_cmd cmd) +static int vnic_dev_advanced_filters_cap(struct vnic_dev *vdev, u64 *args, + int nargs) +{ + memset(args, 0, nargs * sizeof(*args)); + args[0] = CMD_ADD_ADV_FILTER; + args[1] = FILTER_CAP_MODE_V1_FLAG; + return vnic_dev_cmd_args(vdev, CMD_CAPABILITY, args, nargs, 1000); +} + +int vnic_dev_capable_adv_filters(struct vnic_dev *vdev) +{ + u64 a0 = CMD_ADD_ADV_FILTER, a1 = 0; + int wait = 1000; + int err; + + err = vnic_dev_cmd(vdev, CMD_CAPABILITY, &a0, &a1, wait); + if (err) + return 0; + return (a1 >= (u32)FILTER_DPDK_1); +} + +/* Determine the "best" filtering mode VIC is capaible of. Returns one of 3 + * value or 0 on error: + * FILTER_DPDK_1- advanced filters availabile + * FILTER_USNIC_IP_FLAG - advanced filters but with the restriction that + * the IP layer must explicitly specified. I.e. cannot have a UDP + * filter that matches both IPv4 and IPv6. + * FILTER_IPV4_5TUPLE - fallback if either of the 2 above aren't available. + * all other filter types are not available. + * Retrun true in filter_tags if supported + */ +int vnic_dev_capable_filter_mode(struct vnic_dev *vdev, u32 *mode, + u8 *filter_tags) +{ + u64 args[4]; + int err; + u32 max_level = 0; + + err = vnic_dev_advanced_filters_cap(vdev, args, 4); + + /* determine if filter tags are available */ + if (err) + *filter_tags = 0; + if ((args[2] == FILTER_CAP_MODE_V1) && + (args[3] & FILTER_ACTION_FILTER_ID_FLAG)) + *filter_tags = 1; + else + *filter_tags = 0; + + if (err || ((args[0] == 1) && (args[1] == 0))) { + /* Adv filter Command not supported or adv filters available but + * not enabled. Try the normal filter capability command. + */ + args[0] = CMD_ADD_FILTER; + args[1] = 0; + err = vnic_dev_cmd_args(vdev, CMD_CAPABILITY, args, 2, 1000); + if (err) + return err; + max_level = args[1]; + goto parse_max_level; + } else if (args[2] == FILTER_CAP_MODE_V1) { + /* parse filter capability mask in args[1] */ + if (args[1] & FILTER_DPDK_1_FLAG) + *mode = FILTER_DPDK_1; + else if (args[1] & FILTER_USNIC_IP_FLAG) + *mode = FILTER_USNIC_IP; + else if (args[1] & FILTER_IPV4_5TUPLE_FLAG) + *mode = FILTER_IPV4_5TUPLE; + return 0; + } + max_level = args[1]; +parse_max_level: + if (max_level >= (u32)FILTER_USNIC_IP) + *mode = FILTER_USNIC_IP; + else + *mode = FILTER_IPV4_5TUPLE; + return 0; +} + +int vnic_dev_capable(struct vnic_dev *vdev, enum vnic_devcmd_cmd cmd) { u64 a0 = (u32)cmd, a1 = 0; int wait = 1000; @@ -531,7 +632,7 @@ int vnic_dev_stats_dump(struct vnic_dev *vdev, struct vnic_stats **stats) if (!vdev->stats) { snprintf((char *)name, sizeof(name), - "vnic_stats-%d", instance++); + "vnic_stats-%u", instance++); vdev->stats = vdev->alloc_consistent(vdev->priv, sizeof(struct vnic_stats), &vdev->stats_pa, (u8 *)name); if (!vdev->stats) @@ -553,15 +654,6 @@ int vnic_dev_close(struct vnic_dev *vdev) return vnic_dev_cmd(vdev, CMD_CLOSE, &a0, &a1, wait); } -/** Deprecated. @see vnic_dev_enable_wait */ -int vnic_dev_enable(struct vnic_dev *vdev) -{ - u64 a0 = 0, a1 = 0; - int wait = 1000; - - return vnic_dev_cmd(vdev, CMD_ENABLE, &a0, &a1, wait); -} - int vnic_dev_enable_wait(struct vnic_dev *vdev) { u64 a0 = 0, a1 = 0; @@ -606,34 +698,9 @@ int vnic_dev_open_done(struct vnic_dev *vdev, int *done) return 0; } -int vnic_dev_soft_reset(struct vnic_dev *vdev, int arg) -{ - u64 a0 = (u32)arg, a1 = 0; - int wait = 1000; - - return vnic_dev_cmd(vdev, CMD_SOFT_RESET, &a0, &a1, wait); -} - -int vnic_dev_soft_reset_done(struct vnic_dev *vdev, int *done) -{ - u64 a0 = 0, a1 = 0; - int wait = 1000; - int err; - - *done = 0; - - err = vnic_dev_cmd(vdev, CMD_SOFT_RESET_STATUS, &a0, &a1, wait); - if (err) - return err; - - *done = (a0 == 0); - - return 0; -} - int vnic_dev_get_mac_addr(struct vnic_dev *vdev, u8 *mac_addr) { - u64 a0, a1 = 0; + u64 a0 = 0, a1 = 0; int wait = 1000; int err, i; @@ -721,19 +788,6 @@ int vnic_dev_set_ig_vlan_rewrite_mode(struct vnic_dev *vdev, return 0; } -int vnic_dev_raise_intr(struct vnic_dev *vdev, u16 intr) -{ - u64 a0 = intr, a1 = 0; - int wait = 1000; - int err; - - err = vnic_dev_cmd(vdev, CMD_IAR, &a0, &a1, wait); - if (err) - pr_err("Failed to raise INTR[%d], err %d\n", intr, err); - - return err; -} - void vnic_dev_set_reset_flag(struct vnic_dev *vdev, int state) { vdev->in_reset = state; @@ -781,7 +835,7 @@ int vnic_dev_notify_set(struct vnic_dev *vdev, u16 intr) } if (!vnic_dev_in_reset(vdev)) { snprintf((char *)name, sizeof(name), - "vnic_notify-%d", instance++); + "vnic_notify-%u", instance++); notify_addr = vdev->alloc_consistent(vdev->priv, sizeof(struct vnic_devcmd_notify), ¬ify_pa, (u8 *)name); @@ -866,14 +920,6 @@ int vnic_dev_init(struct vnic_dev *vdev, int arg) return r; } -int vnic_dev_deinit(struct vnic_dev *vdev) -{ - u64 a0 = 0, a1 = 0; - int wait = 1000; - - return vnic_dev_cmd(vdev, CMD_DEINIT, &a0, &a1, wait); -} - void vnic_dev_intr_coal_timer_info_default(struct vnic_dev *vdev) { /* Default: hardware intr coal timer is in units of 1.5 usecs */ @@ -899,18 +945,6 @@ u32 vnic_dev_port_speed(struct vnic_dev *vdev) return vdev->notify_copy.port_speed; } -void vnic_dev_set_intr_mode(struct vnic_dev *vdev, - enum vnic_dev_intr_mode intr_mode) -{ - vdev->intr_mode = intr_mode; -} - -enum vnic_dev_intr_mode vnic_dev_get_intr_mode( - struct vnic_dev *vdev) -{ - return vdev->intr_mode; -} - u32 vnic_dev_intr_coal_timer_usec_to_hw(struct vnic_dev *vdev, u32 usec) { return (usec * vdev->intr_coal_timer_info.mul) / @@ -944,7 +978,7 @@ void vnic_dev_unregister(struct vnic_dev *vdev) vdev->free_consistent(vdev->priv, sizeof(struct vnic_devcmd_fw_info), vdev->fw_info, vdev->fw_info_pa); - kfree(vdev); + rte_free(vdev); } } @@ -953,7 +987,13 @@ struct vnic_dev *vnic_dev_register(struct vnic_dev *vdev, unsigned int num_bars) { if (!vdev) { - vdev = kzalloc(sizeof(struct vnic_dev), GFP_ATOMIC); + char name[NAME_MAX]; + snprintf((char *)name, sizeof(name), "%s-vnic", + pdev->device.name); + vdev = (struct vnic_dev *)rte_zmalloc_socket(name, + sizeof(struct vnic_dev), + RTE_CACHE_LINE_SIZE, + pdev->device.numa_node); if (!vdev) return NULL; } @@ -975,23 +1015,6 @@ err_out: return NULL; } -struct rte_pci_device *vnic_dev_get_pdev(struct vnic_dev *vdev) -{ - return vdev->pdev; -} - -int vnic_dev_set_mac_addr(struct vnic_dev *vdev, u8 *mac_addr) -{ - u64 a0, a1 = 0; - int wait = 1000; - int i; - - for (i = 0; i < ETH_ALEN; i++) - ((u8 *)&a0)[i] = mac_addr[i]; - - return vnic_dev_cmd(vdev, CMD_SET_MAC_ADDR, &a0, &a1, wait); -} - /* * vnic_dev_classifier: Add/Delete classifier entries * @vdev: vdev of the device @@ -1005,26 +1028,33 @@ int vnic_dev_set_mac_addr(struct vnic_dev *vdev, u8 *mac_addr) * In case of DEL filter, the caller passes the RQ number. Return * value is irrelevant. * @data: filter data + * @action: action data */ int vnic_dev_classifier(struct vnic_dev *vdev, u8 cmd, u16 *entry, - struct filter *data) + struct filter_v2 *data, struct filter_action_v2 *action_v2) { - u64 a0, a1; + u64 a0 = 0, a1 = 0; int wait = 1000; dma_addr_t tlv_pa; int ret = -EINVAL; struct filter_tlv *tlv, *tlv_va; - struct filter_action *action; u64 tlv_size; + u32 filter_size, action_size; static unsigned int unique_id; char z_name[RTE_MEMZONE_NAMESIZE]; + enum vnic_devcmd_cmd dev_cmd; if (cmd == CLSF_ADD) { - tlv_size = sizeof(struct filter) + - sizeof(struct filter_action) + + dev_cmd = (data->type >= FILTER_DPDK_1) ? + CMD_ADD_ADV_FILTER : CMD_ADD_FILTER; + + filter_size = vnic_filter_size(data); + action_size = vnic_action_size(action_v2); + + tlv_size = filter_size + action_size + 2*sizeof(struct filter_tlv); snprintf((char *)z_name, sizeof(z_name), - "vnic_clsf_%d", unique_id++); + "vnic_clsf_%u", unique_id++); tlv_va = vdev->alloc_consistent(vdev->priv, tlv_size, &tlv_pa, (u8 *)z_name); if (!tlv_va) @@ -1034,20 +1064,17 @@ int vnic_dev_classifier(struct vnic_dev *vdev, u8 cmd, u16 *entry, a1 = tlv_size; memset(tlv, 0, tlv_size); tlv->type = CLSF_TLV_FILTER; - tlv->length = sizeof(struct filter); - *(struct filter *)&tlv->val = *data; + tlv->length = filter_size; + memcpy(&tlv->val, (void *)data, filter_size); tlv = (struct filter_tlv *)((char *)tlv + sizeof(struct filter_tlv) + - sizeof(struct filter)); + filter_size); tlv->type = CLSF_TLV_ACTION; - tlv->length = sizeof(struct filter_action); - action = (struct filter_action *)&tlv->val; - action->type = FILTER_ACTION_RQ_STEERING; - action->u.rq_idx = *entry; - - ret = vnic_dev_cmd(vdev, CMD_ADD_FILTER, &a0, &a1, wait); + tlv->length = action_size; + memcpy(&tlv->val, (void *)action_v2, action_size); + ret = vnic_dev_cmd(vdev, dev_cmd, &a0, &a1, wait); *entry = (u16)a0; vdev->free_consistent(vdev->priv, tlv_size, tlv_va, tlv_pa); } else if (cmd == CLSF_DEL) { diff --git a/dpdk/drivers/net/enic/base/vnic_dev.h b/dpdk/drivers/net/enic/base/vnic_dev.h index 689442f3..c9ca25b3 100644 --- a/dpdk/drivers/net/enic/base/vnic_dev.h +++ b/dpdk/drivers/net/enic/base/vnic_dev.h @@ -35,8 +35,10 @@ #ifndef _VNIC_DEV_H_ #define _VNIC_DEV_H_ +#include +#include + #include "enic_compat.h" -#include "rte_pci.h" #include "vnic_resource.h" #include "vnic_devcmd.h" @@ -134,6 +136,10 @@ void vnic_dev_cmd_proxy_by_bdf_start(struct vnic_dev *vdev, u16 bdf); void vnic_dev_cmd_proxy_end(struct vnic_dev *vdev); int vnic_dev_fw_info(struct vnic_dev *vdev, struct vnic_devcmd_fw_info **fw_info); +int vnic_dev_capable_adv_filters(struct vnic_dev *vdev); +int vnic_dev_capable(struct vnic_dev *vdev, enum vnic_devcmd_cmd cmd); +int vnic_dev_capable_filter_mode(struct vnic_dev *vdev, u32 *mode, + u8 *filter_tags); int vnic_dev_asic_info(struct vnic_dev *vdev, u16 *asic_type, u16 *asic_rev); int vnic_dev_spec(struct vnic_dev *vdev, unsigned int offset, size_t size, void *value); @@ -201,7 +207,7 @@ int vnic_dev_enable2_done(struct vnic_dev *vdev, int *status); int vnic_dev_deinit_done(struct vnic_dev *vdev, int *status); int vnic_dev_set_mac_addr(struct vnic_dev *vdev, u8 *mac_addr); int vnic_dev_classifier(struct vnic_dev *vdev, u8 cmd, u16 *entry, - struct filter *data); + struct filter_v2 *data, struct filter_action_v2 *action_v2); #ifdef ENIC_VXLAN int vnic_dev_overlay_offload_enable_disable(struct vnic_dev *vdev, u8 overlay, u8 config); diff --git a/dpdk/drivers/net/enic/base/vnic_devcmd.h b/dpdk/drivers/net/enic/base/vnic_devcmd.h index b3d5a6cc..05d87b91 100644 --- a/dpdk/drivers/net/enic/base/vnic_devcmd.h +++ b/dpdk/drivers/net/enic/base/vnic_devcmd.h @@ -1,5 +1,5 @@ /* - * Copyright 2008-2010 Cisco Systems, Inc. All rights reserved. + * Copyright 2008-2016 Cisco Systems, Inc. All rights reserved. * Copyright 2007 Nuova Systems, Inc. All rights reserved. * * Copyright (c) 2014, Cisco Systems, Inc. @@ -92,6 +92,8 @@ #define _CMD_VTYPE(cmd) (((cmd) >> _CMD_VTYPESHIFT) & _CMD_VTYPEMASK) #define _CMD_N(cmd) (((cmd) >> _CMD_NSHIFT) & _CMD_NMASK) +#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) + enum vnic_devcmd_cmd { CMD_NONE = _CMDC(_CMD_DIR_NONE, _CMD_VTYPE_NONE, 0), @@ -126,7 +128,8 @@ enum vnic_devcmd_cmd { /* dev-specific block member: * in: (u16)a0=offset,(u8)a1=size - * out: a0=value */ + * out: a0=value + */ CMD_DEV_SPEC = _CMDC(_CMD_DIR_RW, _CMD_VTYPE_ALL, 2), /* stats clear */ @@ -146,8 +149,9 @@ enum vnic_devcmd_cmd { CMD_HANG_NOTIFY = _CMDC(_CMD_DIR_NONE, _CMD_VTYPE_ALL, 8), /* MAC address in (u48)a0 */ - CMD_GET_MAC_ADDR = _CMDC(_CMD_DIR_READ, + CMD_MAC_ADDR = _CMDC(_CMD_DIR_READ, _CMD_VTYPE_ENET | _CMD_VTYPE_FC, 9), +#define CMD_GET_MAC_ADDR CMD_MAC_ADDR /* some uses are aliased */ /* add addr from (u48)a0 */ CMD_ADDR_ADD = _CMDCNW(_CMD_DIR_WRITE, @@ -387,9 +391,8 @@ enum vnic_devcmd_cmd { * Subvnic migration from MQ <--> VF. * Enable the LIF migration from MQ to VF and vice versa. MQ and VF * indexes are statically bound at the time of initialization. - * Based on the - * direction of migration, the resources of either MQ or the VF shall - * be attached to the LIF. + * Based on the direction of migration, the resources of either MQ or + * the VF shall be attached to the LIF. * in: (u32)a0=Direction of Migration * 0=> Migrate to VF * 1=> Migrate to MQ @@ -397,7 +400,6 @@ enum vnic_devcmd_cmd { */ CMD_MIGRATE_SUBVNIC = _CMDC(_CMD_DIR_WRITE, _CMD_VTYPE_ENET, 53), - /* * Register / Deregister the notification block for MQ subvnics * in: @@ -433,6 +435,10 @@ enum vnic_devcmd_cmd { * in: (u64) a0= filter address * (u32) a1= size of filter * out: (u32) a0=filter identifier + * + * Capability query: + * out: (u64) a0= 1 if capability query supported + * (u64) a1= MAX filter type supported */ CMD_ADD_FILTER = _CMDC(_CMD_DIR_RW, _CMD_VTYPE_ENET, 58), @@ -471,25 +477,152 @@ enum vnic_devcmd_cmd { CMD_QP_STATS_CLEAR = _CMDC(_CMD_DIR_WRITE, _CMD_VTYPE_ENET, 63), /* - * Enable/Disable overlay offloads on the given vnic + * UEFI BOOT API: (u64)a0= UEFI FLS_CMD_xxx + * (ui64)a1= paddr for the info buffer + */ + CMD_FC_REQ = _CMDC(_CMD_DIR_RW, _CMD_VTYPE_FC, 64), + + /* + * Return the iSCSI config details required by the EFI Option ROM + * in: (u32) a0=0 Get Boot Info for PXE eNIC as per pxe_boot_config_t + * a0=1 Get Boot info for iSCSI enic as per + * iscsi_boot_efi_cfg_t + * in: (u64) a1=Host address where iSCSI config info is returned + */ + CMD_VNIC_BOOT_CONFIG_INFO = _CMDC(_CMD_DIR_RW, _CMD_VTYPE_ALL, 65), + + /* + * Create a Queue Pair (RoCE) + * in: (u32) a0 = Queue Pair number + * (u32) a1 = Remote QP + * (u32) a2 = RDMA-RQ + * (u16) a3 = RQ Res Group + * (u16) a4 = SQ Res Group + * (u32) a5 = Protection Domain + * (u64) a6 = Remote MAC + * (u32) a7 = start PSN + * (u16) a8 = MSS + * (u32) a9 = protocol version + */ + CMD_RDMA_QP_CREATE = _CMDC(_CMD_DIR_WRITE, _CMD_VTYPE_ENET, 66), + + /* + * Delete a Queue Pair (RoCE) + * in: (u32) a0 = Queue Pair number + */ + CMD_RDMA_QP_DELETE = _CMDC(_CMD_DIR_WRITE, _CMD_VTYPE_ENET, 67), + + /* + * Retrieve a Queue Pair's status information (RoCE) + * in: (u32) a0 = Queue Pair number + * (u64) a1 = host buffer addr for QP status struct + * (u32) a2 = length of the buffer + */ + CMD_RDMA_QP_STATUS = _CMDC(_CMD_DIR_RW, _CMD_VTYPE_ENET, 68), + + /* + * Use this devcmd for agreeing on the highest common version supported + * by both driver and fw for by features who need such a facility. + * in: (u64) a0 = feature (driver requests for the supported versions + * on this feature) + * out: (u64) a0 = bitmap of all supported versions for that feature + */ + CMD_GET_SUPP_FEATURE_VER = _CMDC(_CMD_DIR_RW, _CMD_VTYPE_ENET, 69), + + /* + * Initialize the RDMA notification work queue + * in: (u64) a0 = host buffer address + * in: (u16) a1 = number of entries in buffer + * in: (u16) a2 = resource group number + * in: (u16) a3 = CQ number to post completion + */ + CMD_RDMA_INIT_INFO_BUF = _CMDC(_CMD_DIR_WRITE, _CMD_VTYPE_ENET, 70), + + /* + * De-init the RDMA notification work queue + * in: (u64) a0=resource group number + */ + CMD_RDMA_DEINIT_INFO_BUF = _CMDC(_CMD_DIR_WRITE, _CMD_VTYPE_ENET, 71), + + /* + * Control (Enable/Disable) overlay offloads on the given vnic * in: (u8) a0 = OVERLAY_FEATURE_NVGRE : NVGRE * a0 = OVERLAY_FEATURE_VXLAN : VxLAN - * in: (u8) a1 = OVERLAY_OFFLOAD_ENABLE : Enable - * a1 = OVERLAY_OFFLOAD_DISABLE : Disable + * in: (u8) a1 = OVERLAY_OFFLOAD_ENABLE : Enable or + * a1 = OVERLAY_OFFLOAD_DISABLE : Disable or + * a1 = OVERLAY_OFFLOAD_ENABLE_V2 : Enable with version 2 */ - CMD_OVERLAY_OFFLOAD_ENABLE_DISABLE = - _CMDC(_CMD_DIR_WRITE, _CMD_VTYPE_ENET, 72), + CMD_OVERLAY_OFFLOAD_CTRL = + _CMDC(_CMD_DIR_WRITE, _CMD_VTYPE_ENET, 72), /* * Configuration of overlay offloads feature on a given vNIC - * in: (u8) a0 = DEVCMD_OVERLAY_NVGRE : NVGRE - * a0 = DEVCMD_OVERLAY_VXLAN : VxLAN - * in: (u8) a1 = VXLAN_PORT_UPDATE : VxLAN - * in: (u16) a2 = unsigned short int port information + * in: (u8) a0 = OVERLAY_CFG_VXLAN_PORT_UPDATE : VxLAN + * in: (u16) a1 = unsigned short int port information */ CMD_OVERLAY_OFFLOAD_CFG = _CMDC(_CMD_DIR_WRITE, _CMD_VTYPE_ENET, 73), + + /* + * Return the configured name for the device + * in: (u64) a0=Host address where the name is copied + * (u32) a1=Size of the buffer + */ + CMD_GET_CONFIG_NAME = _CMDC(_CMD_DIR_WRITE, _CMD_VTYPE_ALL, 74), + + /* + * Enable group interrupt for the VF + * in: (u32) a0 = GRPINTR_ENABLE : enable + * a0 = GRPINTR_DISABLE : disable + * a0 = GRPINTR_UPD_VECT: update group vector addr + * in: (u32) a1 = interrupt group count + * in: (u64) a2 = Start of host buffer address for DMAing group + * vector bitmap + * in: (u64) a3 = Stride between group vectors + */ + CMD_CONFIG_GRPINTR = _CMDC(_CMD_DIR_WRITE, _CMD_VTYPE_ENET, 75), + + /* + * Set cq arrary base and size in a list of consective wqs and + * rqs for a device + * in: (u16) a0 = the wq relative index in the device. + * -1 indicates skipping wq configuration + * in: (u16) a1 = the wcq relative index in the device + * in: (u16) a2 = the rq relative index in the device + * -1 indicates skipping rq configuration + * in: (u16) a3 = the rcq relative index in the device + */ + CMD_CONFIG_CQ_ARRAY = _CMDC(_CMD_DIR_WRITE, _CMD_VTYPE_ENET, 76), + + /* + * Add an advanced filter. + * in: (u64) a0= filter address + * (u32) a1= size of filter + * out: (u32) a0=filter identifier + * + * Capability query: + * in: (u64) a1= supported filter capability exchange modes + * out: (u64) a0= 1 if capability query supported + * if (u64) a1 = 0: a1 = MAX filter type supported + * if (u64) a1 & FILTER_CAP_MODE_V1_FLAG: + * a1 = bitmask of supported filters + * a2 = FILTER_CAP_MODE_V1 + * a3 = bitmask of supported actions + */ + CMD_ADD_ADV_FILTER = _CMDC(_CMD_DIR_RW, _CMD_VTYPE_ENET, 77), }; +/* Modes for exchanging advanced filter capabilities. The modes supported by + * the driver are passed in the CMD_ADD_ADV_FILTER capability command and the + * mode selected is returned. + * V0: the maximum filter type supported is returned + * V1: bitmasks of supported filters and actions are returned + */ +enum filter_cap_mode { + FILTER_CAP_MODE_V0 = 0, /* Must always be 0 for legacy drivers */ + FILTER_CAP_MODE_V1 = 1, +}; +#define FILTER_CAP_MODE_V1_FLAG (1 << FILTER_CAP_MODE_V1) + /* CMD_ENABLE2 flags */ #define CMD_ENABLE2_STANDBY 0x0 #define CMD_ENABLE2_ACTIVE 0x1 @@ -520,6 +653,9 @@ enum vnic_devcmd_status { STAT_NONE = 0, STAT_BUSY = 1 << 0, /* cmd in progress */ STAT_ERROR = 1 << 1, /* last cmd caused error (code in a0) */ + STAT_FAILOVER = 1 << 2, /* always set on vnics in pci standby state + * if seen a failover to the standby happened + */ }; enum vnic_devcmd_error { @@ -558,9 +694,9 @@ enum fwinfo_asic_type { FWINFO_ASIC_TYPE_UNKNOWN, FWINFO_ASIC_TYPE_PALO, FWINFO_ASIC_TYPE_SERENO, + FWINFO_ASIC_TYPE_CRUZ, }; - struct vnic_devcmd_notify { u32 csum; /* checksum over following words */ @@ -595,25 +731,16 @@ struct vnic_devcmd_provinfo { */ #define FILTER_FIELD_VALID(fld) (1 << (fld - 1)) -#define FILTER_FIELDS_USNIC (FILTER_FIELD_VALID(1) | \ - FILTER_FIELD_VALID(2) | \ - FILTER_FIELD_VALID(3) | \ - FILTER_FIELD_VALID(4)) - -#define FILTER_FIELDS_IPV4_5TUPLE (FILTER_FIELD_VALID(1) | \ - FILTER_FIELD_VALID(2) | \ - FILTER_FIELD_VALID(3) | \ - FILTER_FIELD_VALID(4) | \ - FILTER_FIELD_VALID(5)) - -#define FILTER_FIELDS_MAC_VLAN (FILTER_FIELD_VALID(1) | \ - FILTER_FIELD_VALID(2)) - #define FILTER_FIELD_USNIC_VLAN FILTER_FIELD_VALID(1) #define FILTER_FIELD_USNIC_ETHTYPE FILTER_FIELD_VALID(2) #define FILTER_FIELD_USNIC_PROTO FILTER_FIELD_VALID(3) #define FILTER_FIELD_USNIC_ID FILTER_FIELD_VALID(4) +#define FILTER_FIELDS_USNIC (FILTER_FIELD_USNIC_VLAN | \ + FILTER_FIELD_USNIC_ETHTYPE | \ + FILTER_FIELD_USNIC_PROTO | \ + FILTER_FIELD_USNIC_ID) + struct filter_usnic_id { u32 flags; u16 vlan; @@ -628,10 +755,18 @@ struct filter_usnic_id { #define FILTER_FIELD_5TUP_SRC_PT FILTER_FIELD_VALID(4) #define FILTER_FIELD_5TUP_DST_PT FILTER_FIELD_VALID(5) +#define FILTER_FIELDS_IPV4_5TUPLE (FILTER_FIELD_5TUP_PROTO | \ + FILTER_FIELD_5TUP_SRC_AD | \ + FILTER_FIELD_5TUP_DST_AD | \ + FILTER_FIELD_5TUP_SRC_PT | \ + FILTER_FIELD_5TUP_DST_PT) + /* Enums for the protocol field. */ enum protocol_e { PROTO_UDP = 0, PROTO_TCP = 1, + PROTO_IPV4 = 2, + PROTO_IPV6 = 3 }; struct filter_ipv4_5tuple { @@ -646,15 +781,82 @@ struct filter_ipv4_5tuple { #define FILTER_FIELD_VMQ_VLAN FILTER_FIELD_VALID(1) #define FILTER_FIELD_VMQ_MAC FILTER_FIELD_VALID(2) +#define FILTER_FIELDS_MAC_VLAN (FILTER_FIELD_VMQ_VLAN | \ + FILTER_FIELD_VMQ_MAC) + +#define FILTER_FIELDS_NVGRE FILTER_FIELD_VMQ_MAC + struct filter_mac_vlan { u32 flags; u16 vlan; u8 mac_addr[6]; } __attribute__((packed)); +#define FILTER_FIELD_VLAN_IP_3TUP_VLAN FILTER_FIELD_VALID(1) +#define FILTER_FIELD_VLAN_IP_3TUP_L3_PROTO FILTER_FIELD_VALID(2) +#define FILTER_FIELD_VLAN_IP_3TUP_DST_AD FILTER_FIELD_VALID(3) +#define FILTER_FIELD_VLAN_IP_3TUP_L4_PROTO FILTER_FIELD_VALID(4) +#define FILTER_FIELD_VLAN_IP_3TUP_DST_PT FILTER_FIELD_VALID(5) + +#define FILTER_FIELDS_VLAN_IP_3TUP (FILTER_FIELD_VLAN_IP_3TUP_VLAN | \ + FILTER_FIELD_VLAN_IP_3TUP_L3_PROTO | \ + FILTER_FIELD_VLAN_IP_3TUP_DST_AD | \ + FILTER_FIELD_VLAN_IP_3TUP_L4_PROTO | \ + FILTER_FIELD_VLAN_IP_3TUP_DST_PT) + +struct filter_vlan_ip_3tuple { + u32 flags; + u16 vlan; + u16 l3_protocol; + union { + u32 dst_addr_v4; + u8 dst_addr_v6[16]; + } u; + u32 l4_protocol; + u16 dst_port; +} __attribute__((packed)); + +#define FILTER_GENERIC_1_BYTES 64 + +enum filter_generic_1_layer { + FILTER_GENERIC_1_L2, + FILTER_GENERIC_1_L3, + FILTER_GENERIC_1_L4, + FILTER_GENERIC_1_L5, + FILTER_GENERIC_1_NUM_LAYERS +}; + +#define FILTER_GENERIC_1_IPV4 (1 << 0) +#define FILTER_GENERIC_1_IPV6 (1 << 1) +#define FILTER_GENERIC_1_UDP (1 << 2) +#define FILTER_GENERIC_1_TCP (1 << 3) +#define FILTER_GENERIC_1_TCP_OR_UDP (1 << 4) +#define FILTER_GENERIC_1_IP4SUM_OK (1 << 5) +#define FILTER_GENERIC_1_L4SUM_OK (1 << 6) +#define FILTER_GENERIC_1_IPFRAG (1 << 7) + +#define FILTER_GENERIC_1_KEY_LEN 64 + +/* + * Version 1 of generic filter specification + * position is only 16 bits, reserving positions > 64k to be used by firmware + */ +struct filter_generic_1 { + u16 position; /* lower position comes first */ + u32 mask_flags; + u32 val_flags; + u16 mask_vlan; + u16 val_vlan; + struct { + u8 mask[FILTER_GENERIC_1_KEY_LEN]; /* 0 bit means "don't care"*/ + u8 val[FILTER_GENERIC_1_KEY_LEN]; + } __attribute__((packed)) layer[FILTER_GENERIC_1_NUM_LAYERS]; +} __attribute__((packed)); + /* Specifies the filter_action type. */ enum { FILTER_ACTION_RQ_STEERING = 0, + FILTER_ACTION_V2 = 1, FILTER_ACTION_MAX }; @@ -665,20 +867,76 @@ struct filter_action { } u; } __attribute__((packed)); +#define FILTER_ACTION_RQ_STEERING_FLAG (1 << 0) +#define FILTER_ACTION_FILTER_ID_FLAG (1 << 1) +#define FILTER_ACTION_V2_ALL (FILTER_ACTION_RQ_STEERING_FLAG \ + | FILTER_ACTION_FILTER_ID_FLAG) + +/* Version 2 of filter action must be a strict extension of struct filter_action + * where the first fields exactly match in size and meaning. + */ +struct filter_action_v2 { + u32 type; + u32 rq_idx; + u32 flags; /* use FILTER_ACTION_XXX_FLAG defines */ + u16 filter_id; + u_int8_t reserved[32]; /* for future expansion */ +} __attribute__((packed)); + /* Specifies the filter type. */ enum filter_type { FILTER_USNIC_ID = 0, FILTER_IPV4_5TUPLE = 1, FILTER_MAC_VLAN = 2, + FILTER_VLAN_IP_3TUPLE = 3, + FILTER_NVGRE_VMQ = 4, + FILTER_USNIC_IP = 5, + FILTER_DPDK_1 = 6, FILTER_MAX }; +#define FILTER_USNIC_ID_FLAG (1 << FILTER_USNIC_ID) +#define FILTER_IPV4_5TUPLE_FLAG (1 << FILTER_IPV4_5TUPLE) +#define FILTER_MAC_VLAN_FLAG (1 << FILTER_MAC_VLAN) +#define FILTER_VLAN_IP_3TUPLE_FLAG (1 << FILTER_VLAN_IP_3TUPLE) +#define FILTER_NVGRE_VMQ_FLAG (1 << FILTER_NVGRE_VMQ) +#define FILTER_USNIC_IP_FLAG (1 << FILTER_USNIC_IP) +#define FILTER_DPDK_1_FLAG (1 << FILTER_DPDK_1) +#define FILTER_V1_ALL (FILTER_USNIC_ID_FLAG | \ + FILTER_IPV4_5TUPLE_FLAG | \ + FILTER_MAC_VLAN_FLAG | \ + FILTER_VLAN_IP_3TUPLE_FLAG | \ + FILTER_NVGRE_VMQ_FLAG | \ + FILTER_USNIC_IP_FLAG | \ + FILTER_DPDK_1_FLAG) + struct filter { u32 type; union { struct filter_usnic_id usnic; struct filter_ipv4_5tuple ipv4; struct filter_mac_vlan mac_vlan; + struct filter_vlan_ip_3tuple vlan_3tuple; + } u; +} __attribute__((packed)); + +/* + * This is a strict superset of "struct filter" and exists only + * because many drivers use "sizeof (struct filter)" in deciding TLV size. + * This new, larger struct filter would cause any code that uses that method + * to not work with older firmware, so we add filter_v2 to hold the + * new filter types. Drivers should use vnic_filter_size() to determine + * the TLV size instead of sizeof (struct fiter_v2) to guard against future + * growth. + */ +struct filter_v2 { + u32 type; + union { + struct filter_usnic_id usnic; + struct filter_ipv4_5tuple ipv4; + struct filter_mac_vlan mac_vlan; + struct filter_vlan_ip_3tuple vlan_3tuple; + struct filter_generic_1 generic_1; } u; } __attribute__((packed)); @@ -687,19 +945,84 @@ enum { CLSF_TLV_ACTION = 1, }; -#define FILTER_MAX_BUF_SIZE 100 /* Maximum size of buffer to CMD_ADD_FILTER */ - struct filter_tlv { - uint32_t type; - uint32_t length; - uint32_t val[0]; + u_int32_t type; + u_int32_t length; + u_int32_t val[0]; }; +/* Data for CMD_ADD_FILTER is 2 TLV and filter + action structs */ +#define FILTER_MAX_BUF_SIZE 100 +#define FILTER_V2_MAX_BUF_SIZE (sizeof(struct filter_v2) + \ + sizeof(struct filter_action_v2) + \ + (2 * sizeof(struct filter_tlv))) + +/* + * Compute actual structure size given filter type. To be "future-proof," + * drivers should use this instead of "sizeof (struct filter_v2)" when + * computing length for TLV. + */ +static inline u_int32_t +vnic_filter_size(struct filter_v2 *fp) +{ + u_int32_t size; + + switch (fp->type) { + case FILTER_USNIC_ID: + size = sizeof(fp->u.usnic); + break; + case FILTER_IPV4_5TUPLE: + size = sizeof(fp->u.ipv4); + break; + case FILTER_MAC_VLAN: + case FILTER_NVGRE_VMQ: + size = sizeof(fp->u.mac_vlan); + break; + case FILTER_VLAN_IP_3TUPLE: + size = sizeof(fp->u.vlan_3tuple); + break; + case FILTER_USNIC_IP: + case FILTER_DPDK_1: + size = sizeof(fp->u.generic_1); + break; + default: + size = sizeof(fp->u); + break; + } + size += sizeof(fp->type); + return size; +} + + enum { CLSF_ADD = 0, CLSF_DEL = 1, }; +/* + * Get the action structure size given action type. To be "future-proof," + * drivers should use this instead of "sizeof (struct filter_action_v2)" + * when computing length for TLV. + */ +static inline u_int32_t +vnic_action_size(struct filter_action_v2 *fap) +{ + u_int32_t size; + + switch (fap->type) { + case FILTER_ACTION_RQ_STEERING: + size = sizeof(struct filter_action); + break; + case FILTER_ACTION_V2: + size = sizeof(struct filter_action_v2); + break; + default: + size = sizeof(struct filter_action); + break; + } + return size; +} + /* * Writing cmd register causes STAT_BUSY to get set in status register. * When cmd completes, STAT_BUSY will be cleared. @@ -766,8 +1089,30 @@ typedef enum { OVERLAY_FEATURE_MAX, } overlay_feature_t; -#define OVERLAY_OFFLOAD_ENABLE 0 -#define OVERLAY_OFFLOAD_DISABLE 1 +#define OVERLAY_OFFLOAD_ENABLE 0 +#define OVERLAY_OFFLOAD_DISABLE 1 +#define OVERLAY_OFFLOAD_ENABLE_V2 2 #define OVERLAY_CFG_VXLAN_PORT_UPDATE 0 + +/* + * Use this enum to get the supported versions for each of these features + * If you need to use the devcmd_get_supported_feature_version(), add + * the new feature into this enum and install function handler in devcmd.c + */ +typedef enum { + VIC_FEATURE_VXLAN, + VIC_FEATURE_RDMA, + VIC_FEATURE_MAX, +} vic_feature_t; + +/* + * CMD_CONFIG_GRPINTR subcommands + */ +typedef enum { + GRPINTR_ENABLE = 1, + GRPINTR_DISABLE, + GRPINTR_UPD_VECT, +} grpintr_subcmd_t; + #endif /* _VNIC_DEVCMD_H_ */ diff --git a/dpdk/drivers/net/enic/base/vnic_rq.c b/dpdk/drivers/net/enic/base/vnic_rq.c index 10a40c1b..ea297eef 100644 --- a/dpdk/drivers/net/enic/base/vnic_rq.c +++ b/dpdk/drivers/net/enic/base/vnic_rq.c @@ -58,13 +58,13 @@ int vnic_rq_alloc(struct vnic_dev *vdev, struct vnic_rq *rq, unsigned int index, rq->ctrl = vnic_dev_get_res(vdev, RES_TYPE_RQ, index); if (!rq->ctrl) { - pr_err("Failed to hook RQ[%d] resource\n", index); + pr_err("Failed to hook RQ[%u] resource\n", index); return -EINVAL; } vnic_rq_disable(rq); - snprintf(res_name, sizeof(res_name), "%d-rq-%d", instance++, index); + snprintf(res_name, sizeof(res_name), "%d-rq-%u", instance++, index); rc = vnic_dev_alloc_desc_ring(vdev, &rq->ring, desc_count, desc_size, rq->socket_id, res_name); return rc; @@ -118,11 +118,6 @@ void vnic_rq_init(struct vnic_rq *rq, unsigned int cq_index, rq->pkt_last_seg = NULL; } -void vnic_rq_error_out(struct vnic_rq *rq, unsigned int error) -{ - iowrite32(error, &rq->ctrl->error_status); -} - unsigned int vnic_rq_error_status(struct vnic_rq *rq) { return ioread32(&rq->ctrl->error_status); diff --git a/dpdk/drivers/net/enic/base/vnic_rq.h b/dpdk/drivers/net/enic/base/vnic_rq.h index 2d9104c4..f3fd39f7 100644 --- a/dpdk/drivers/net/enic/base/vnic_rq.h +++ b/dpdk/drivers/net/enic/base/vnic_rq.h @@ -97,6 +97,7 @@ struct vnic_rq { struct rte_mbuf *pkt_first_seg; struct rte_mbuf *pkt_last_seg; unsigned int max_mbufs_per_pkt; + uint16_t tot_nb_desc; }; static inline unsigned int vnic_rq_desc_avail(struct vnic_rq *rq) diff --git a/dpdk/drivers/net/enic/base/vnic_rss.c b/dpdk/drivers/net/enic/base/vnic_rss.c index 1cf055b0..87d40c0d 100644 --- a/dpdk/drivers/net/enic/base/vnic_rss.c +++ b/dpdk/drivers/net/enic/base/vnic_rss.c @@ -50,35 +50,3 @@ void vnic_set_rss_key(union vnic_rss_key *rss_key, u8 *key) } } -void vnic_set_rss_cpu(union vnic_rss_cpu *rss_cpu, u8 *cpu) -{ - u32 i; - u32 *p = (u32 *)cpu; - - for (i = 0; i < 32; ++i) - iowrite32(*p++, &rss_cpu->cpu[i].b[0]); -} - -void vnic_get_rss_key(union vnic_rss_key *rss_key, u8 *key) -{ - u32 i; - u32 *p; - u16 *q; - - for (i = 0; i < 4; ++i) { - p = (u32 *)(key + (10 * i)); - *p++ = ioread32(&rss_key->key[i].b[0]); - *p++ = ioread32(&rss_key->key[i].b[4]); - q = (u16 *)p; - *q = (u16)ioread32(&rss_key->key[i].b[8]); - } -} - -void vnic_get_rss_cpu(union vnic_rss_cpu *rss_cpu, u8 *cpu) -{ - u32 i; - u32 *p = (u32 *)cpu; - - for (i = 0; i < 32; ++i) - *p++ = ioread32(&rss_cpu->cpu[i].b[0]); -} diff --git a/dpdk/drivers/net/enic/base/vnic_wq.c b/dpdk/drivers/net/enic/base/vnic_wq.c index 7c4119c3..0a1247f4 100644 --- a/dpdk/drivers/net/enic/base/vnic_wq.c +++ b/dpdk/drivers/net/enic/base/vnic_wq.c @@ -52,7 +52,7 @@ int vnic_wq_alloc_ring(struct vnic_dev *vdev, struct vnic_wq *wq, char res_name[NAME_MAX]; static int instance; - snprintf(res_name, sizeof(res_name), "%d-wq-%d", instance++, wq->index); + snprintf(res_name, sizeof(res_name), "%d-wq-%u", instance++, wq->index); return vnic_dev_alloc_desc_ring(vdev, &wq->ring, desc_count, desc_size, wq->socket_id, res_name); } @@ -145,11 +145,6 @@ void vnic_wq_init(struct vnic_wq *wq, unsigned int cq_index, wq->last_completed_index = 0; } -void vnic_wq_error_out(struct vnic_wq *wq, unsigned int error) -{ - iowrite32(error, &wq->ctrl->error_status); -} - unsigned int vnic_wq_error_status(struct vnic_wq *wq) { return ioread32(&wq->ctrl->error_status); diff --git a/dpdk/drivers/net/enic/enic.h b/dpdk/drivers/net/enic/enic.h index 8f12b435..a43fddc5 100644 --- a/dpdk/drivers/net/enic/enic.h +++ b/dpdk/drivers/net/enic/enic.h @@ -53,13 +53,7 @@ #define DRV_DESCRIPTION "Cisco VIC Ethernet NIC Poll-mode Driver" #define DRV_COPYRIGHT "Copyright 2008-2015 Cisco Systems, Inc" -#define ENIC_WQ_MAX 8 -/* With Rx scatter support, we use two RQs on VIC per RQ used by app. Both - * RQs use the same CQ. - */ -#define ENIC_RQ_MAX 16 -#define ENIC_CQ_MAX (ENIC_WQ_MAX + (ENIC_RQ_MAX / 2)) -#define ENIC_INTR_MAX (ENIC_CQ_MAX + 2) +#define ENIC_MAX_MAC_ADDR 64 #define VLAN_ETH_HLEN 18 @@ -79,6 +73,8 @@ #define PCI_DEVICE_ID_CISCO_VIC_ENET 0x0043 /* ethernet vnic */ #define PCI_DEVICE_ID_CISCO_VIC_ENET_VF 0x0071 /* enet SRIOV VF */ +/* Special Filter id for non-specific packet flagging. Don't change value */ +#define ENIC_MAGIC_FILTER_ID 0xffff #define ENICPMD_FDIR_MAX 64 @@ -92,11 +88,17 @@ struct enic_fdir { struct rte_eth_fdir_stats stats; struct rte_hash *hash; struct enic_fdir_node *nodes[ENICPMD_FDIR_MAX]; + u32 modes; + u32 types_mask; + void (*copy_fltr_fn)(struct filter_v2 *filt, + struct rte_eth_fdir_input *input, + struct rte_eth_fdir_masks *masks); }; struct enic_soft_stats { rte_atomic64_t rx_nombuf; rte_atomic64_t rx_packet_errors; + rte_atomic64_t tx_oversized; }; struct enic_memzone_entry { @@ -104,6 +106,12 @@ struct enic_memzone_entry { LIST_ENTRY(enic_memzone_entry) entries; }; +struct rte_flow { + LIST_ENTRY(rte_flow) next; + u16 enic_filter_id; + struct filter_v2 enic_filter; +}; + /* Per-instance private data structure */ struct enic { struct enic *next; @@ -128,21 +136,24 @@ struct enic { int link_status; u8 hw_ip_checksum; u16 max_mtu; + u8 adv_filters; + u32 flow_filter_mode; + u8 filter_tags; unsigned int flags; unsigned int priv_flags; - /* work queue */ - struct vnic_wq wq[ENIC_WQ_MAX]; - unsigned int wq_count; + /* work queue (len = conf_wq_count) */ + struct vnic_wq *wq; + unsigned int wq_count; /* equals eth_dev nb_tx_queues */ - /* receive queue */ - struct vnic_rq rq[ENIC_RQ_MAX]; - unsigned int rq_count; + /* receive queue (len = conf_rq_count) */ + struct vnic_rq *rq; + unsigned int rq_count; /* equals eth_dev nb_rx_queues */ - /* completion queue */ - struct vnic_cq cq[ENIC_CQ_MAX]; - unsigned int cq_count; + /* completion queue (len = conf_cq_count) */ + struct vnic_cq *cq; + unsigned int cq_count; /* equals rq_count + wq_count */ /* interrupt resource */ struct vnic_intr intr; @@ -160,7 +171,10 @@ struct enic { /* linked list storing memory allocations */ LIST_HEAD(enic_memzone_list, enic_memzone_entry) memzone_list; rte_spinlock_t memzone_list_lock; + rte_spinlock_t mtu_lock; + LIST_HEAD(enic_flows, rte_flow) flows; + rte_spinlock_t flows_lock; }; /* Get the CQ index from a Start of Packet(SOP) RQ index */ @@ -169,14 +183,22 @@ static inline unsigned int enic_sop_rq_idx_to_cq_idx(unsigned int sop_idx) return sop_idx / 2; } -static inline unsigned int enic_sop_rq(unsigned int rq) +/* Get the RTE RQ index from a Start of Packet(SOP) RQ index */ +static inline unsigned int enic_sop_rq_idx_to_rte_idx(unsigned int sop_idx) { - return rq * 2; + return sop_idx / 2; } -static inline unsigned int enic_data_rq(unsigned int rq) +/* Get the Start of Packet(SOP) RQ index from a RTE RQ index */ +static inline unsigned int enic_rte_rq_idx_to_sop_idx(unsigned int rte_idx) { - return rq * 2 + 1; + return rte_idx * 2; +} + +/* Get the Data RQ index from a RTE RQ index */ +static inline unsigned int enic_rte_rq_idx_to_data_idx(unsigned int rte_idx) +{ + return rte_idx * 2 + 1; } static inline unsigned int enic_vnic_rq_count(struct enic *enic) @@ -198,11 +220,6 @@ static inline unsigned int enic_cq_wq(struct enic *enic, unsigned int wq) return enic->rq_count + wq; } -static inline unsigned int enic_msix_err_intr(__rte_unused struct enic *enic) -{ - return 0; -} - static inline struct enic *pmd_priv(struct rte_eth_dev *eth_dev) { return (struct enic *)eth_dev->data->dev_private; @@ -258,12 +275,12 @@ extern int enic_enable(struct enic *enic); extern int enic_disable(struct enic *enic); extern void enic_remove(struct enic *enic); extern int enic_get_link_status(struct enic *enic); -extern void enic_dev_stats_get(struct enic *enic, +extern int enic_dev_stats_get(struct enic *enic, struct rte_eth_stats *r_stats); extern void enic_dev_stats_clear(struct enic *enic); extern void enic_add_packet_filter(struct enic *enic); -extern void enic_set_mac_address(struct enic *enic, uint8_t *mac_addr); -extern void enic_del_mac_address(struct enic *enic); +int enic_set_mac_address(struct enic *enic, uint8_t *mac_addr); +void enic_del_mac_address(struct enic *enic, int mac_index); extern unsigned int enic_cleanup_wq(struct enic *enic, struct vnic_wq *wq); extern void enic_send_pkt(struct enic *enic, struct vnic_wq *wq, struct rte_mbuf *tx_pkt, unsigned short len, @@ -276,7 +293,18 @@ extern int enic_clsf_init(struct enic *enic); extern void enic_clsf_destroy(struct enic *enic); uint16_t enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts); +uint16_t enic_dummy_recv_pkts(void *rx_queue, + struct rte_mbuf **rx_pkts, + uint16_t nb_pkts); uint16_t enic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts); int enic_set_mtu(struct enic *enic, uint16_t new_mtu); +int enic_link_update(struct enic *enic); +void enic_fdir_info(struct enic *enic); +void enic_fdir_info_get(struct enic *enic, struct rte_eth_fdir_info *stats); +void copy_fltr_v1(struct filter_v2 *fltr, struct rte_eth_fdir_input *input, + struct rte_eth_fdir_masks *masks); +void copy_fltr_v2(struct filter_v2 *fltr, struct rte_eth_fdir_input *input, + struct rte_eth_fdir_masks *masks); +extern const struct rte_flow_ops enic_flow_ops; #endif /* _ENIC_H_ */ diff --git a/dpdk/drivers/net/enic/enic_clsf.c b/dpdk/drivers/net/enic/enic_clsf.c index 111b1942..9b461424 100644 --- a/dpdk/drivers/net/enic/enic_clsf.c +++ b/dpdk/drivers/net/enic/enic_clsf.c @@ -38,6 +38,11 @@ #include #include #include +#include +#include +#include +#include +#include #include "enic_compat.h" #include "enic.h" @@ -52,7 +57,7 @@ #include "vnic_intr.h" #include "vnic_nic.h" -#ifdef RTE_MACHINE_CPUFLAG_SSE4_2 +#ifdef RTE_ARCH_X86 #include #define DEFAULT_HASH_FUNC rte_hash_crc #else @@ -67,6 +72,262 @@ void enic_fdir_stats_get(struct enic *enic, struct rte_eth_fdir_stats *stats) *stats = enic->fdir.stats; } +void enic_fdir_info_get(struct enic *enic, struct rte_eth_fdir_info *info) +{ + info->mode = (enum rte_fdir_mode)enic->fdir.modes; + info->flow_types_mask[0] = enic->fdir.types_mask; +} + +void enic_fdir_info(struct enic *enic) +{ + enic->fdir.modes = (u32)RTE_FDIR_MODE_PERFECT; + enic->fdir.types_mask = 1 << RTE_ETH_FLOW_NONFRAG_IPV4_UDP | + 1 << RTE_ETH_FLOW_NONFRAG_IPV4_TCP; + if (enic->adv_filters) { + enic->fdir.types_mask |= 1 << RTE_ETH_FLOW_NONFRAG_IPV4_OTHER | + 1 << RTE_ETH_FLOW_NONFRAG_IPV4_SCTP | + 1 << RTE_ETH_FLOW_NONFRAG_IPV6_UDP | + 1 << RTE_ETH_FLOW_NONFRAG_IPV6_TCP | + 1 << RTE_ETH_FLOW_NONFRAG_IPV6_SCTP | + 1 << RTE_ETH_FLOW_NONFRAG_IPV6_OTHER; + enic->fdir.copy_fltr_fn = copy_fltr_v2; + } else { + enic->fdir.copy_fltr_fn = copy_fltr_v1; + } +} + +static void +enic_set_layer(struct filter_generic_1 *gp, unsigned int flag, + enum filter_generic_1_layer layer, void *mask, void *val, + unsigned int len) +{ + gp->mask_flags |= flag; + gp->val_flags |= gp->mask_flags; + memcpy(gp->layer[layer].mask, mask, len); + memcpy(gp->layer[layer].val, val, len); +} + +/* Copy Flow Director filter to a VIC ipv4 filter (for Cisco VICs + * without advanced filter support. + */ +void +copy_fltr_v1(struct filter_v2 *fltr, struct rte_eth_fdir_input *input, + __rte_unused struct rte_eth_fdir_masks *masks) +{ + fltr->type = FILTER_IPV4_5TUPLE; + fltr->u.ipv4.src_addr = rte_be_to_cpu_32( + input->flow.ip4_flow.src_ip); + fltr->u.ipv4.dst_addr = rte_be_to_cpu_32( + input->flow.ip4_flow.dst_ip); + fltr->u.ipv4.src_port = rte_be_to_cpu_16( + input->flow.udp4_flow.src_port); + fltr->u.ipv4.dst_port = rte_be_to_cpu_16( + input->flow.udp4_flow.dst_port); + + if (input->flow_type == RTE_ETH_FLOW_NONFRAG_IPV4_TCP) + fltr->u.ipv4.protocol = PROTO_TCP; + else + fltr->u.ipv4.protocol = PROTO_UDP; + + fltr->u.ipv4.flags = FILTER_FIELDS_IPV4_5TUPLE; +} + +/* Copy Flow Director filter to a VIC generic filter (requires advanced + * filter support. + */ +void +copy_fltr_v2(struct filter_v2 *fltr, struct rte_eth_fdir_input *input, + struct rte_eth_fdir_masks *masks) +{ + struct filter_generic_1 *gp = &fltr->u.generic_1; + int i; + + fltr->type = FILTER_DPDK_1; + memset(gp, 0, sizeof(*gp)); + + if (input->flow_type == RTE_ETH_FLOW_NONFRAG_IPV4_UDP) { + struct udp_hdr udp_mask, udp_val; + memset(&udp_mask, 0, sizeof(udp_mask)); + memset(&udp_val, 0, sizeof(udp_val)); + + if (input->flow.udp4_flow.src_port) { + udp_mask.src_port = masks->src_port_mask; + udp_val.src_port = input->flow.udp4_flow.src_port; + } + if (input->flow.udp4_flow.dst_port) { + udp_mask.dst_port = masks->dst_port_mask; + udp_val.dst_port = input->flow.udp4_flow.dst_port; + } + + enic_set_layer(gp, FILTER_GENERIC_1_UDP, FILTER_GENERIC_1_L4, + &udp_mask, &udp_val, sizeof(struct udp_hdr)); + } else if (input->flow_type == RTE_ETH_FLOW_NONFRAG_IPV4_TCP) { + struct tcp_hdr tcp_mask, tcp_val; + memset(&tcp_mask, 0, sizeof(tcp_mask)); + memset(&tcp_val, 0, sizeof(tcp_val)); + + if (input->flow.tcp4_flow.src_port) { + tcp_mask.src_port = masks->src_port_mask; + tcp_val.src_port = input->flow.tcp4_flow.src_port; + } + if (input->flow.tcp4_flow.dst_port) { + tcp_mask.dst_port = masks->dst_port_mask; + tcp_val.dst_port = input->flow.tcp4_flow.dst_port; + } + + enic_set_layer(gp, FILTER_GENERIC_1_TCP, FILTER_GENERIC_1_L4, + &tcp_mask, &tcp_val, sizeof(struct tcp_hdr)); + } else if (input->flow_type == RTE_ETH_FLOW_NONFRAG_IPV4_SCTP) { + struct sctp_hdr sctp_mask, sctp_val; + memset(&sctp_mask, 0, sizeof(sctp_mask)); + memset(&sctp_val, 0, sizeof(sctp_val)); + + if (input->flow.sctp4_flow.src_port) { + sctp_mask.src_port = masks->src_port_mask; + sctp_val.src_port = input->flow.sctp4_flow.src_port; + } + if (input->flow.sctp4_flow.dst_port) { + sctp_mask.dst_port = masks->dst_port_mask; + sctp_val.dst_port = input->flow.sctp4_flow.dst_port; + } + if (input->flow.sctp4_flow.verify_tag) { + sctp_mask.tag = 0xffffffff; + sctp_val.tag = input->flow.sctp4_flow.verify_tag; + } + + /* v4 proto should be 132, override ip4_flow.proto */ + input->flow.ip4_flow.proto = 132; + + enic_set_layer(gp, 0, FILTER_GENERIC_1_L4, &sctp_mask, + &sctp_val, sizeof(struct sctp_hdr)); + } + + if (input->flow_type == RTE_ETH_FLOW_NONFRAG_IPV4_UDP || + input->flow_type == RTE_ETH_FLOW_NONFRAG_IPV4_TCP || + input->flow_type == RTE_ETH_FLOW_NONFRAG_IPV4_SCTP || + input->flow_type == RTE_ETH_FLOW_NONFRAG_IPV4_OTHER) { + struct ipv4_hdr ip4_mask, ip4_val; + memset(&ip4_mask, 0, sizeof(struct ipv4_hdr)); + memset(&ip4_val, 0, sizeof(struct ipv4_hdr)); + + if (input->flow.ip4_flow.tos) { + ip4_mask.type_of_service = masks->ipv4_mask.tos; + ip4_val.type_of_service = input->flow.ip4_flow.tos; + } + if (input->flow.ip4_flow.ttl) { + ip4_mask.time_to_live = masks->ipv4_mask.ttl; + ip4_val.time_to_live = input->flow.ip4_flow.ttl; + } + if (input->flow.ip4_flow.proto) { + ip4_mask.next_proto_id = masks->ipv4_mask.proto; + ip4_val.next_proto_id = input->flow.ip4_flow.proto; + } + if (input->flow.ip4_flow.src_ip) { + ip4_mask.src_addr = masks->ipv4_mask.src_ip; + ip4_val.src_addr = input->flow.ip4_flow.src_ip; + } + if (input->flow.ip4_flow.dst_ip) { + ip4_mask.dst_addr = masks->ipv4_mask.dst_ip; + ip4_val.dst_addr = input->flow.ip4_flow.dst_ip; + } + + enic_set_layer(gp, FILTER_GENERIC_1_IPV4, FILTER_GENERIC_1_L3, + &ip4_mask, &ip4_val, sizeof(struct ipv4_hdr)); + } + + if (input->flow_type == RTE_ETH_FLOW_NONFRAG_IPV6_UDP) { + struct udp_hdr udp_mask, udp_val; + memset(&udp_mask, 0, sizeof(udp_mask)); + memset(&udp_val, 0, sizeof(udp_val)); + + if (input->flow.udp6_flow.src_port) { + udp_mask.src_port = masks->src_port_mask; + udp_val.src_port = input->flow.udp6_flow.src_port; + } + if (input->flow.udp6_flow.dst_port) { + udp_mask.dst_port = masks->dst_port_mask; + udp_val.dst_port = input->flow.udp6_flow.dst_port; + } + enic_set_layer(gp, FILTER_GENERIC_1_UDP, FILTER_GENERIC_1_L4, + &udp_mask, &udp_val, sizeof(struct udp_hdr)); + } else if (input->flow_type == RTE_ETH_FLOW_NONFRAG_IPV6_TCP) { + struct tcp_hdr tcp_mask, tcp_val; + memset(&tcp_mask, 0, sizeof(tcp_mask)); + memset(&tcp_val, 0, sizeof(tcp_val)); + + if (input->flow.tcp6_flow.src_port) { + tcp_mask.src_port = masks->src_port_mask; + tcp_val.src_port = input->flow.tcp6_flow.src_port; + } + if (input->flow.tcp6_flow.dst_port) { + tcp_mask.dst_port = masks->dst_port_mask; + tcp_val.dst_port = input->flow.tcp6_flow.dst_port; + } + enic_set_layer(gp, FILTER_GENERIC_1_TCP, FILTER_GENERIC_1_L4, + &tcp_mask, &tcp_val, sizeof(struct tcp_hdr)); + } else if (input->flow_type == RTE_ETH_FLOW_NONFRAG_IPV6_SCTP) { + struct sctp_hdr sctp_mask, sctp_val; + memset(&sctp_mask, 0, sizeof(sctp_mask)); + memset(&sctp_val, 0, sizeof(sctp_val)); + + if (input->flow.sctp6_flow.src_port) { + sctp_mask.src_port = masks->src_port_mask; + sctp_val.src_port = input->flow.sctp6_flow.src_port; + } + if (input->flow.sctp6_flow.dst_port) { + sctp_mask.dst_port = masks->dst_port_mask; + sctp_val.dst_port = input->flow.sctp6_flow.dst_port; + } + if (input->flow.sctp6_flow.verify_tag) { + sctp_mask.tag = 0xffffffff; + sctp_val.tag = input->flow.sctp6_flow.verify_tag; + } + + /* v4 proto should be 132, override ipv6_flow.proto */ + input->flow.ipv6_flow.proto = 132; + + enic_set_layer(gp, 0, FILTER_GENERIC_1_L4, &sctp_mask, + &sctp_val, sizeof(struct sctp_hdr)); + } + + if (input->flow_type == RTE_ETH_FLOW_NONFRAG_IPV6_UDP || + input->flow_type == RTE_ETH_FLOW_NONFRAG_IPV6_TCP || + input->flow_type == RTE_ETH_FLOW_NONFRAG_IPV6_SCTP || + input->flow_type == RTE_ETH_FLOW_NONFRAG_IPV6_OTHER) { + struct ipv6_hdr ipv6_mask, ipv6_val; + memset(&ipv6_mask, 0, sizeof(struct ipv6_hdr)); + memset(&ipv6_val, 0, sizeof(struct ipv6_hdr)); + + if (input->flow.ipv6_flow.proto) { + ipv6_mask.proto = masks->ipv6_mask.proto; + ipv6_val.proto = input->flow.ipv6_flow.proto; + } + for (i = 0; i < 4; i++) { + *(uint32_t *)&ipv6_mask.src_addr[i * 4] = + masks->ipv6_mask.src_ip[i]; + *(uint32_t *)&ipv6_val.src_addr[i * 4] = + input->flow.ipv6_flow.src_ip[i]; + } + for (i = 0; i < 4; i++) { + *(uint32_t *)&ipv6_mask.dst_addr[i * 4] = + masks->ipv6_mask.src_ip[i]; + *(uint32_t *)&ipv6_val.dst_addr[i * 4] = + input->flow.ipv6_flow.dst_ip[i]; + } + if (input->flow.ipv6_flow.tc) { + ipv6_mask.vtc_flow = masks->ipv6_mask.tc << 12; + ipv6_val.vtc_flow = input->flow.ipv6_flow.tc << 12; + } + if (input->flow.ipv6_flow.hop_limits) { + ipv6_mask.hop_limits = masks->ipv6_mask.hop_limits; + ipv6_val.hop_limits = input->flow.ipv6_flow.hop_limits; + } + + enic_set_layer(gp, FILTER_GENERIC_1_IPV6, FILTER_GENERIC_1_L3, + &ipv6_mask, &ipv6_val, sizeof(struct ipv6_hdr)); + } +} + int enic_fdir_del_fltr(struct enic *enic, struct rte_eth_fdir_filter *params) { int32_t pos; @@ -84,7 +345,7 @@ int enic_fdir_del_fltr(struct enic *enic, struct rte_eth_fdir_filter *params) /* Delete the filter */ vnic_dev_classifier(enic->vdev, CLSF_DEL, - &key->fltr_id, NULL); + &key->fltr_id, NULL, NULL); rte_free(key); enic->fdir.nodes[pos] = NULL; enic->fdir.stats.free++; @@ -97,17 +358,19 @@ int enic_fdir_del_fltr(struct enic *enic, struct rte_eth_fdir_filter *params) int enic_fdir_add_fltr(struct enic *enic, struct rte_eth_fdir_filter *params) { struct enic_fdir_node *key; - struct filter fltr = {0}; + struct filter_v2 fltr; int32_t pos; u8 do_free = 0; u16 old_fltr_id = 0; u32 flowtype_supported; u16 flex_bytes; u16 queue; + struct filter_action_v2 action; - flowtype_supported = ( - (RTE_ETH_FLOW_NONFRAG_IPV4_TCP == params->input.flow_type) || - (RTE_ETH_FLOW_NONFRAG_IPV4_UDP == params->input.flow_type)); + memset(&fltr, 0, sizeof(fltr)); + memset(&action, 0, sizeof(action)); + flowtype_supported = enic->fdir.types_mask + & (1 << params->input.flow_type); flex_bytes = ((params->input.flow_ext.flexbytes[1] << 8 & 0xFF00) | (params->input.flow_ext.flexbytes[0] & 0xFF)); @@ -121,7 +384,10 @@ int enic_fdir_add_fltr(struct enic *enic, struct rte_eth_fdir_filter *params) } /* Get the enicpmd RQ from the DPDK Rx queue */ - queue = enic_sop_rq(params->action.rx_queue); + queue = enic_rte_rq_idx_to_sop_idx(params->action.rx_queue); + + if (!enic->rq[queue].in_use) + return -EINVAL; /* See if the key is already there in the table */ pos = rte_hash_del_key(enic->fdir.hash, params); @@ -175,7 +441,7 @@ int enic_fdir_add_fltr(struct enic *enic, struct rte_eth_fdir_filter *params) * Delete the filter and add the modified one later */ vnic_dev_classifier(enic->vdev, CLSF_DEL, - &key->fltr_id, NULL); + &key->fltr_id, NULL, NULL); enic->fdir.stats.free++; } @@ -185,24 +451,13 @@ int enic_fdir_add_fltr(struct enic *enic, struct rte_eth_fdir_filter *params) key->filter = *params; key->rq_index = queue; - fltr.type = FILTER_IPV4_5TUPLE; - fltr.u.ipv4.src_addr = rte_be_to_cpu_32( - params->input.flow.ip4_flow.src_ip); - fltr.u.ipv4.dst_addr = rte_be_to_cpu_32( - params->input.flow.ip4_flow.dst_ip); - fltr.u.ipv4.src_port = rte_be_to_cpu_16( - params->input.flow.udp4_flow.src_port); - fltr.u.ipv4.dst_port = rte_be_to_cpu_16( - params->input.flow.udp4_flow.dst_port); + enic->fdir.copy_fltr_fn(&fltr, ¶ms->input, + &enic->rte_dev->data->dev_conf.fdir_conf.mask); + action.type = FILTER_ACTION_RQ_STEERING; + action.rq_idx = queue; - if (RTE_ETH_FLOW_NONFRAG_IPV4_TCP == params->input.flow_type) - fltr.u.ipv4.protocol = PROTO_TCP; - else - fltr.u.ipv4.protocol = PROTO_UDP; - - fltr.u.ipv4.flags = FILTER_FIELDS_IPV4_5TUPLE; - - if (!vnic_dev_classifier(enic->vdev, CLSF_ADD, &queue, &fltr)) { + if (!vnic_dev_classifier(enic->vdev, CLSF_ADD, &queue, &fltr, + &action)) { key->fltr_id = queue; } else { dev_err(enic, "Add classifier entry failed\n"); @@ -212,7 +467,8 @@ int enic_fdir_add_fltr(struct enic *enic, struct rte_eth_fdir_filter *params) } if (do_free) - vnic_dev_classifier(enic->vdev, CLSF_DEL, &old_fltr_id, NULL); + vnic_dev_classifier(enic->vdev, CLSF_DEL, &old_fltr_id, NULL, + NULL); else{ enic->fdir.stats.free--; enic->fdir.stats.add++; @@ -238,7 +494,7 @@ void enic_clsf_destroy(struct enic *enic) key = enic->fdir.nodes[index]; if (key) { vnic_dev_classifier(enic->vdev, CLSF_DEL, - &key->fltr_id, NULL); + &key->fltr_id, NULL, NULL); rte_free(key); enic->fdir.nodes[index] = NULL; } diff --git a/dpdk/drivers/net/enic/enic_compat.h b/dpdk/drivers/net/enic/enic_compat.h index 5dbd983b..1cb5686f 100644 --- a/dpdk/drivers/net/enic/enic_compat.h +++ b/dpdk/drivers/net/enic/enic_compat.h @@ -41,6 +41,7 @@ #include #include #include +#include #define ENIC_PAGE_ALIGN 4096UL #define ENIC_ALIGN ENIC_PAGE_ALIGN @@ -95,42 +96,32 @@ typedef unsigned long long dma_addr_t; static inline uint32_t ioread32(volatile void *addr) { - return *(volatile uint32_t *)addr; -} - -static inline uint16_t ioread16(volatile void *addr) -{ - return *(volatile uint16_t *)addr; + return rte_read32(addr); } static inline uint8_t ioread8(volatile void *addr) { - return *(volatile uint8_t *)addr; + return rte_read8(addr); } static inline void iowrite32(uint32_t val, volatile void *addr) { - *(volatile uint32_t *)addr = val; + rte_write32(val, addr); } -static inline void iowrite16(uint16_t val, volatile void *addr) +static inline void iowrite32_relaxed(uint32_t val, volatile void *addr) { - *(volatile uint16_t *)addr = val; -} - -static inline void iowrite8(uint8_t val, volatile void *addr) -{ - *(volatile uint8_t *)addr = val; + rte_write32_relaxed(val, addr); } static inline unsigned int readl(volatile void __iomem *addr) { - return *(volatile unsigned int *)addr; + return rte_read32(addr); } static inline void writel(unsigned int val, volatile void __iomem *addr) { - *(volatile unsigned int *)addr = val; + rte_write32(val, addr); } #define min_t(type, x, y) ({ \ diff --git a/dpdk/drivers/net/enic/enic_ethdev.c b/dpdk/drivers/net/enic/enic_ethdev.c index 04e7ba8d..98391b00 100644 --- a/dpdk/drivers/net/enic/enic_ethdev.c +++ b/dpdk/drivers/net/enic/enic_ethdev.c @@ -37,7 +37,9 @@ #include #include +#include #include +#include #include #include "vnic_intr.h" @@ -95,10 +97,12 @@ enicpmd_fdir_ctrl_func(struct rte_eth_dev *eth_dev, break; case RTE_ETH_FILTER_FLUSH: - case RTE_ETH_FILTER_INFO: dev_warning(enic, "unsupported operation %u", filter_op); ret = -ENOTSUP; break; + case RTE_ETH_FILTER_INFO: + enic_fdir_info_get(enic, (struct rte_eth_fdir_info *)arg); + break; default: dev_err(enic, "unknown operation %u", filter_op); ret = -EINVAL; @@ -113,13 +117,25 @@ enicpmd_dev_filter_ctrl(struct rte_eth_dev *dev, enum rte_filter_op filter_op, void *arg) { - int ret = -EINVAL; + int ret = 0; - if (RTE_ETH_FILTER_FDIR == filter_type) + ENICPMD_FUNC_TRACE(); + + switch (filter_type) { + case RTE_ETH_FILTER_GENERIC: + if (filter_op != RTE_ETH_FILTER_GET) + return -EINVAL; + *(const void **)arg = &enic_flow_ops; + break; + case RTE_ETH_FILTER_FDIR: ret = enicpmd_fdir_ctrl_func(dev, filter_op, arg); - else + break; + default: dev_warning(enic, "Filter type (%d) not supported", filter_type); + ret = -EINVAL; + break; + } return ret; } @@ -127,6 +143,10 @@ enicpmd_dev_filter_ctrl(struct rte_eth_dev *dev, static void enicpmd_dev_tx_queue_release(void *txq) { ENICPMD_FUNC_TRACE(); + + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return; + enic_free_wq(txq); } @@ -152,7 +172,7 @@ static int enicpmd_dev_setup_intr(struct enic *enic) return 0; /* check start of packet (SOP) RQs only in case scatter is disabled. */ for (index = 0; index < enic->rq_count; index++) { - if (!enic->rq[enic_sop_rq(index)].ctrl) + if (!enic->rq[enic_rte_rq_idx_to_sop_idx(index)].ctrl) break; } if (enic->rq_count != index) @@ -181,14 +201,11 @@ static int enicpmd_dev_tx_queue_setup(struct rte_eth_dev *eth_dev, int ret; struct enic *enic = pmd_priv(eth_dev); - ENICPMD_FUNC_TRACE(); - if (queue_idx >= ENIC_WQ_MAX) { - dev_err(enic, - "Max number of TX queues exceeded. Max is %d\n", - ENIC_WQ_MAX); - return -EINVAL; - } + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return -E_RTE_SECONDARY; + ENICPMD_FUNC_TRACE(); + RTE_ASSERT(queue_idx < enic->conf_wq_count); eth_dev->data->tx_queues[queue_idx] = (void *)&enic->wq[queue_idx]; ret = enic_alloc_wq(enic, queue_idx, socket_id, nb_desc); @@ -257,9 +274,37 @@ static int enicpmd_dev_rx_queue_stop(struct rte_eth_dev *eth_dev, static void enicpmd_dev_rx_queue_release(void *rxq) { ENICPMD_FUNC_TRACE(); + + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return; + enic_free_rq(rxq); } +static uint32_t enicpmd_dev_rx_queue_count(struct rte_eth_dev *dev, + uint16_t rx_queue_id) +{ + struct enic *enic = pmd_priv(dev); + uint32_t queue_count = 0; + struct vnic_cq *cq; + uint32_t cq_tail; + uint16_t cq_idx; + int rq_num; + + rq_num = enic_rte_rq_idx_to_sop_idx(rx_queue_id); + cq = &enic->cq[enic_cq_rq(enic, rq_num)]; + cq_idx = cq->to_clean; + + cq_tail = ioread32(&cq->ctrl->cq_tail); + + if (cq_tail < cq_idx) + cq_tail += cq->ring.desc_count; + + queue_count = cq_tail - cq_idx; + + return queue_count; +} + static int enicpmd_dev_rx_queue_setup(struct rte_eth_dev *eth_dev, uint16_t queue_idx, uint16_t nb_desc, @@ -271,18 +316,12 @@ static int enicpmd_dev_rx_queue_setup(struct rte_eth_dev *eth_dev, struct enic *enic = pmd_priv(eth_dev); ENICPMD_FUNC_TRACE(); - /* With Rx scatter support, two RQs are now used on VIC per RQ used - * by the application. - */ - if (queue_idx * 2 >= ENIC_RQ_MAX) { - dev_err(enic, - "Max number of RX queues exceeded. Max is %d. This PMD uses 2 RQs on VIC per RQ used by DPDK.\n", - ENIC_RQ_MAX); - return -EINVAL; - } + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return -E_RTE_SECONDARY; + RTE_ASSERT(enic_rte_rq_idx_to_sop_idx(queue_idx) < enic->conf_rq_count); eth_dev->data->rx_queues[queue_idx] = - (void *)&enic->rq[enic_sop_rq(queue_idx)]; + (void *)&enic->rq[enic_rte_rq_idx_to_sop_idx(queue_idx)]; ret = enic_alloc_rq(enic, queue_idx, socket_id, mp, nb_desc, rx_conf->rx_free_thresh); @@ -308,7 +347,7 @@ static int enicpmd_vlan_filter_set(struct rte_eth_dev *eth_dev, return err; } -static void enicpmd_vlan_offload_set(struct rte_eth_dev *eth_dev, int mask) +static int enicpmd_vlan_offload_set(struct rte_eth_dev *eth_dev, int mask) { struct enic *enic = pmd_priv(eth_dev); @@ -332,6 +371,8 @@ static void enicpmd_vlan_offload_set(struct rte_eth_dev *eth_dev, int mask) dev_warning(enic, "Configuration of extended VLAN is not supported\n"); } + + return 0; } static int enicpmd_dev_configure(struct rte_eth_dev *eth_dev) @@ -339,6 +380,9 @@ static int enicpmd_dev_configure(struct rte_eth_dev *eth_dev) int ret; struct enic *enic = pmd_priv(eth_dev); + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return -E_RTE_SECONDARY; + ENICPMD_FUNC_TRACE(); ret = enic_set_vnic_res(enic); if (ret) { @@ -353,9 +397,10 @@ static int enicpmd_dev_configure(struct rte_eth_dev *eth_dev) eth_dev->data->dev_conf.rxmode.split_hdr_size); } - enicpmd_vlan_offload_set(eth_dev, ETH_VLAN_STRIP_MASK); enic->hw_ip_checksum = eth_dev->data->dev_conf.rxmode.hw_ip_checksum; - return 0; + ret = enicpmd_vlan_offload_set(eth_dev, ETH_VLAN_STRIP_MASK); + + return ret; } /* Start the device. @@ -365,6 +410,9 @@ static int enicpmd_dev_start(struct rte_eth_dev *eth_dev) { struct enic *enic = pmd_priv(eth_dev); + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return -E_RTE_SECONDARY; + ENICPMD_FUNC_TRACE(); return enic_enable(enic); } @@ -377,6 +425,9 @@ static void enicpmd_dev_stop(struct rte_eth_dev *eth_dev) struct rte_eth_link link; struct enic *enic = pmd_priv(eth_dev); + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return; + ENICPMD_FUNC_TRACE(); enic_disable(enic); memset(&link, 0, sizeof(link)); @@ -400,26 +451,18 @@ static int enicpmd_dev_link_update(struct rte_eth_dev *eth_dev, __rte_unused int wait_to_complete) { struct enic *enic = pmd_priv(eth_dev); - int ret; - int link_status = 0; ENICPMD_FUNC_TRACE(); - link_status = enic_get_link_status(enic); - ret = (link_status == enic->link_status); - enic->link_status = link_status; - eth_dev->data->dev_link.link_status = link_status; - eth_dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX; - eth_dev->data->dev_link.link_speed = vnic_dev_port_speed(enic->vdev); - return ret; + return enic_link_update(enic); } -static void enicpmd_dev_stats_get(struct rte_eth_dev *eth_dev, +static int enicpmd_dev_stats_get(struct rte_eth_dev *eth_dev, struct rte_eth_stats *stats) { struct enic *enic = pmd_priv(eth_dev); ENICPMD_FUNC_TRACE(); - enic_dev_stats_get(enic, stats); + return enic_dev_stats_get(enic, stats); } static void enicpmd_dev_stats_reset(struct rte_eth_dev *eth_dev) @@ -436,12 +479,13 @@ static void enicpmd_dev_info_get(struct rte_eth_dev *eth_dev, struct enic *enic = pmd_priv(eth_dev); ENICPMD_FUNC_TRACE(); + device_info->pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); /* Scattered Rx uses two receive queues per rx queue exposed to dpdk */ device_info->max_rx_queues = enic->conf_rq_count / 2; device_info->max_tx_queues = enic->conf_wq_count; device_info->min_rx_bufsize = ENIC_MIN_MTU; device_info->max_rx_pktlen = enic->max_mtu + ETHER_HDR_LEN + 4; - device_info->max_mac_addrs = 1; + device_info->max_mac_addrs = ENIC_MAX_MAC_ADDR; device_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP | DEV_RX_OFFLOAD_IPV4_CKSUM | @@ -451,7 +495,8 @@ static void enicpmd_dev_info_get(struct rte_eth_dev *eth_dev, DEV_TX_OFFLOAD_VLAN_INSERT | DEV_TX_OFFLOAD_IPV4_CKSUM | DEV_TX_OFFLOAD_UDP_CKSUM | - DEV_TX_OFFLOAD_TCP_CKSUM; + DEV_TX_OFFLOAD_TCP_CKSUM | + DEV_TX_OFFLOAD_TCP_TSO; device_info->default_rxconf = (struct rte_eth_rxconf) { .rx_free_thresh = ENIC_DEFAULT_RX_FREE_THRESH }; @@ -460,6 +505,8 @@ static void enicpmd_dev_info_get(struct rte_eth_dev *eth_dev, static const uint32_t *enicpmd_dev_supported_ptypes_get(struct rte_eth_dev *dev) { static const uint32_t ptypes[] = { + RTE_PTYPE_L2_ETHER, + RTE_PTYPE_L2_ETHER_VLAN, RTE_PTYPE_L3_IPV4_EXT_UNKNOWN, RTE_PTYPE_L3_IPV6_EXT_UNKNOWN, RTE_PTYPE_L4_TCP, @@ -478,7 +525,11 @@ static void enicpmd_dev_promiscuous_enable(struct rte_eth_dev *eth_dev) { struct enic *enic = pmd_priv(eth_dev); + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return; + ENICPMD_FUNC_TRACE(); + enic->promisc = 1; enic_add_packet_filter(enic); } @@ -487,6 +538,9 @@ static void enicpmd_dev_promiscuous_disable(struct rte_eth_dev *eth_dev) { struct enic *enic = pmd_priv(eth_dev); + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return; + ENICPMD_FUNC_TRACE(); enic->promisc = 0; enic_add_packet_filter(enic); @@ -496,6 +550,9 @@ static void enicpmd_dev_allmulticast_enable(struct rte_eth_dev *eth_dev) { struct enic *enic = pmd_priv(eth_dev); + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return; + ENICPMD_FUNC_TRACE(); enic->allmulti = 1; enic_add_packet_filter(enic); @@ -505,27 +562,36 @@ static void enicpmd_dev_allmulticast_disable(struct rte_eth_dev *eth_dev) { struct enic *enic = pmd_priv(eth_dev); + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return; + ENICPMD_FUNC_TRACE(); enic->allmulti = 0; enic_add_packet_filter(enic); } -static void enicpmd_add_mac_addr(struct rte_eth_dev *eth_dev, +static int enicpmd_add_mac_addr(struct rte_eth_dev *eth_dev, struct ether_addr *mac_addr, __rte_unused uint32_t index, __rte_unused uint32_t pool) { struct enic *enic = pmd_priv(eth_dev); + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return -E_RTE_SECONDARY; + ENICPMD_FUNC_TRACE(); - enic_set_mac_address(enic, mac_addr->addr_bytes); + return enic_set_mac_address(enic, mac_addr->addr_bytes); } -static void enicpmd_remove_mac_addr(struct rte_eth_dev *eth_dev, __rte_unused uint32_t index) +static void enicpmd_remove_mac_addr(struct rte_eth_dev *eth_dev, uint32_t index) { struct enic *enic = pmd_priv(eth_dev); + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return; + ENICPMD_FUNC_TRACE(); - enic_del_mac_address(enic); + enic_del_mac_address(enic, index); } static int enicpmd_mtu_set(struct rte_eth_dev *eth_dev, uint16_t mtu) @@ -564,7 +630,7 @@ static const struct eth_dev_ops enicpmd_eth_dev_ops = { .tx_queue_stop = enicpmd_dev_tx_queue_stop, .rx_queue_setup = enicpmd_dev_rx_queue_setup, .rx_queue_release = enicpmd_dev_rx_queue_release, - .rx_queue_count = NULL, + .rx_queue_count = enicpmd_dev_rx_queue_count, .rx_descriptor_done = NULL, .tx_queue_setup = enicpmd_dev_tx_queue_setup, .tx_queue_release = enicpmd_dev_tx_queue_release, @@ -596,7 +662,7 @@ static int eth_enicpmd_dev_init(struct rte_eth_dev *eth_dev) eth_dev->rx_pkt_burst = &enic_recv_pkts; eth_dev->tx_pkt_burst = &enic_xmit_pkts; - pdev = eth_dev->pci_dev; + pdev = RTE_ETH_DEV_TO_PCI(eth_dev); rte_eth_copy_pci_info(eth_dev, pdev); enic->pdev = pdev; addr = &pdev->addr; @@ -607,34 +673,25 @@ static int eth_enicpmd_dev_init(struct rte_eth_dev *eth_dev) return enic_probe(enic); } -static struct eth_driver rte_enic_pmd = { - .pci_drv = { - .name = "rte_enic_pmd", - .id_table = pci_id_enic_map, - .drv_flags = RTE_PCI_DRV_NEED_MAPPING, - }, - .eth_dev_init = eth_enicpmd_dev_init, - .dev_private_size = sizeof(struct enic), -}; - -/* Driver initialization routine. - * Invoked once at EAL init time. - * Register as the [Poll Mode] Driver of Cisco ENIC device. - */ -static int -rte_enic_pmd_init(__rte_unused const char *name, - __rte_unused const char *params) +static int eth_enic_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, + struct rte_pci_device *pci_dev) { - ENICPMD_FUNC_TRACE(); - - rte_eth_driver_register(&rte_enic_pmd); - return 0; + return rte_eth_dev_pci_generic_probe(pci_dev, sizeof(struct enic), + eth_enicpmd_dev_init); } -static struct rte_driver rte_enic_driver = { - .type = PMD_PDEV, - .init = rte_enic_pmd_init, +static int eth_enic_pci_remove(struct rte_pci_device *pci_dev) +{ + return rte_eth_dev_pci_generic_remove(pci_dev, NULL); +} + +static struct rte_pci_driver rte_enic_pmd = { + .id_table = pci_id_enic_map, + .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC, + .probe = eth_enic_pci_probe, + .remove = eth_enic_pci_remove, }; -PMD_REGISTER_DRIVER(rte_enic_driver, enic); -DRIVER_REGISTER_PCI_TABLE(enic, pci_id_enic_map); +RTE_PMD_REGISTER_PCI(net_enic, rte_enic_pmd); +RTE_PMD_REGISTER_PCI_TABLE(net_enic, pci_id_enic_map); +RTE_PMD_REGISTER_KMOD_DEP(net_enic, "* igb_uio | uio_pci_generic | vfio-pci"); diff --git a/dpdk/drivers/net/enic/enic_flow.c b/dpdk/drivers/net/enic/enic_flow.c new file mode 100644 index 00000000..a728d077 --- /dev/null +++ b/dpdk/drivers/net/enic/enic_flow.c @@ -0,0 +1,1544 @@ +/* + * Copyright (c) 2017, Cisco Systems, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "enic_compat.h" +#include "enic.h" +#include "vnic_dev.h" +#include "vnic_nic.h" + +#ifdef RTE_LIBRTE_ENIC_DEBUG_FLOW +#define FLOW_TRACE() \ + RTE_LOG(DEBUG, PMD, "%s()\n", __func__) +#define FLOW_LOG(level, fmt, args...) \ + RTE_LOG(level, PMD, fmt, ## args) +#else +#define FLOW_TRACE() do { } while (0) +#define FLOW_LOG(level, fmt, args...) do { } while (0) +#endif + +/** Info about how to copy items into enic filters. */ +struct enic_items { + /** Function for copying and validating an item. */ + int (*copy_item)(const struct rte_flow_item *item, + struct filter_v2 *enic_filter, u8 *inner_ofst); + /** List of valid previous items. */ + const enum rte_flow_item_type * const prev_items; + /** True if it's OK for this item to be the first item. For some NIC + * versions, it's invalid to start the stack above layer 3. + */ + const u8 valid_start_item; +}; + +/** Filtering capabilities for various NIC and firmware versions. */ +struct enic_filter_cap { + /** list of valid items and their handlers and attributes. */ + const struct enic_items *item_info; +}; + +/* functions for copying flow actions into enic actions */ +typedef int (copy_action_fn)(const struct rte_flow_action actions[], + struct filter_action_v2 *enic_action); + +/* functions for copying items into enic filters */ +typedef int(enic_copy_item_fn)(const struct rte_flow_item *item, + struct filter_v2 *enic_filter, u8 *inner_ofst); + +/** Action capabilities for various NICs. */ +struct enic_action_cap { + /** list of valid actions */ + const enum rte_flow_action_type *actions; + /** copy function for a particular NIC */ + int (*copy_fn)(const struct rte_flow_action actions[], + struct filter_action_v2 *enic_action); +}; + +/* Forward declarations */ +static enic_copy_item_fn enic_copy_item_ipv4_v1; +static enic_copy_item_fn enic_copy_item_udp_v1; +static enic_copy_item_fn enic_copy_item_tcp_v1; +static enic_copy_item_fn enic_copy_item_eth_v2; +static enic_copy_item_fn enic_copy_item_vlan_v2; +static enic_copy_item_fn enic_copy_item_ipv4_v2; +static enic_copy_item_fn enic_copy_item_ipv6_v2; +static enic_copy_item_fn enic_copy_item_udp_v2; +static enic_copy_item_fn enic_copy_item_tcp_v2; +static enic_copy_item_fn enic_copy_item_sctp_v2; +static enic_copy_item_fn enic_copy_item_sctp_v2; +static enic_copy_item_fn enic_copy_item_vxlan_v2; +static copy_action_fn enic_copy_action_v1; +static copy_action_fn enic_copy_action_v2; + +/** + * Legacy NICs or NICs with outdated firmware. Only 5-tuple perfect match + * is supported. + */ +static const struct enic_items enic_items_v1[] = { + [RTE_FLOW_ITEM_TYPE_IPV4] = { + .copy_item = enic_copy_item_ipv4_v1, + .valid_start_item = 1, + .prev_items = (const enum rte_flow_item_type[]) { + RTE_FLOW_ITEM_TYPE_END, + }, + }, + [RTE_FLOW_ITEM_TYPE_UDP] = { + .copy_item = enic_copy_item_udp_v1, + .valid_start_item = 0, + .prev_items = (const enum rte_flow_item_type[]) { + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_END, + }, + }, + [RTE_FLOW_ITEM_TYPE_TCP] = { + .copy_item = enic_copy_item_tcp_v1, + .valid_start_item = 0, + .prev_items = (const enum rte_flow_item_type[]) { + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_END, + }, + }, +}; + +/** + * NICs have Advanced Filters capability but they are disabled. This means + * that layer 3 must be specified. + */ +static const struct enic_items enic_items_v2[] = { + [RTE_FLOW_ITEM_TYPE_ETH] = { + .copy_item = enic_copy_item_eth_v2, + .valid_start_item = 1, + .prev_items = (const enum rte_flow_item_type[]) { + RTE_FLOW_ITEM_TYPE_VXLAN, + RTE_FLOW_ITEM_TYPE_END, + }, + }, + [RTE_FLOW_ITEM_TYPE_VLAN] = { + .copy_item = enic_copy_item_vlan_v2, + .valid_start_item = 1, + .prev_items = (const enum rte_flow_item_type[]) { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_END, + }, + }, + [RTE_FLOW_ITEM_TYPE_IPV4] = { + .copy_item = enic_copy_item_ipv4_v2, + .valid_start_item = 1, + .prev_items = (const enum rte_flow_item_type[]) { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_END, + }, + }, + [RTE_FLOW_ITEM_TYPE_IPV6] = { + .copy_item = enic_copy_item_ipv6_v2, + .valid_start_item = 1, + .prev_items = (const enum rte_flow_item_type[]) { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_END, + }, + }, + [RTE_FLOW_ITEM_TYPE_UDP] = { + .copy_item = enic_copy_item_udp_v2, + .valid_start_item = 0, + .prev_items = (const enum rte_flow_item_type[]) { + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_END, + }, + }, + [RTE_FLOW_ITEM_TYPE_TCP] = { + .copy_item = enic_copy_item_tcp_v2, + .valid_start_item = 0, + .prev_items = (const enum rte_flow_item_type[]) { + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_END, + }, + }, + [RTE_FLOW_ITEM_TYPE_SCTP] = { + .copy_item = enic_copy_item_sctp_v2, + .valid_start_item = 0, + .prev_items = (const enum rte_flow_item_type[]) { + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_END, + }, + }, + [RTE_FLOW_ITEM_TYPE_VXLAN] = { + .copy_item = enic_copy_item_vxlan_v2, + .valid_start_item = 0, + .prev_items = (const enum rte_flow_item_type[]) { + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_END, + }, + }, +}; + +/** NICs with Advanced filters enabled */ +static const struct enic_items enic_items_v3[] = { + [RTE_FLOW_ITEM_TYPE_ETH] = { + .copy_item = enic_copy_item_eth_v2, + .valid_start_item = 1, + .prev_items = (const enum rte_flow_item_type[]) { + RTE_FLOW_ITEM_TYPE_VXLAN, + RTE_FLOW_ITEM_TYPE_END, + }, + }, + [RTE_FLOW_ITEM_TYPE_VLAN] = { + .copy_item = enic_copy_item_vlan_v2, + .valid_start_item = 1, + .prev_items = (const enum rte_flow_item_type[]) { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_END, + }, + }, + [RTE_FLOW_ITEM_TYPE_IPV4] = { + .copy_item = enic_copy_item_ipv4_v2, + .valid_start_item = 1, + .prev_items = (const enum rte_flow_item_type[]) { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_END, + }, + }, + [RTE_FLOW_ITEM_TYPE_IPV6] = { + .copy_item = enic_copy_item_ipv6_v2, + .valid_start_item = 1, + .prev_items = (const enum rte_flow_item_type[]) { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_END, + }, + }, + [RTE_FLOW_ITEM_TYPE_UDP] = { + .copy_item = enic_copy_item_udp_v2, + .valid_start_item = 1, + .prev_items = (const enum rte_flow_item_type[]) { + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_END, + }, + }, + [RTE_FLOW_ITEM_TYPE_TCP] = { + .copy_item = enic_copy_item_tcp_v2, + .valid_start_item = 1, + .prev_items = (const enum rte_flow_item_type[]) { + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_END, + }, + }, + [RTE_FLOW_ITEM_TYPE_SCTP] = { + .copy_item = enic_copy_item_sctp_v2, + .valid_start_item = 1, + .prev_items = (const enum rte_flow_item_type[]) { + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_END, + }, + }, + [RTE_FLOW_ITEM_TYPE_VXLAN] = { + .copy_item = enic_copy_item_vxlan_v2, + .valid_start_item = 1, + .prev_items = (const enum rte_flow_item_type[]) { + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_END, + }, + }, +}; + +/** Filtering capabilities indexed this NICs supported filter type. */ +static const struct enic_filter_cap enic_filter_cap[] = { + [FILTER_IPV4_5TUPLE] = { + .item_info = enic_items_v1, + }, + [FILTER_USNIC_IP] = { + .item_info = enic_items_v2, + }, + [FILTER_DPDK_1] = { + .item_info = enic_items_v3, + }, +}; + +/** Supported actions for older NICs */ +static const enum rte_flow_action_type enic_supported_actions_v1[] = { + RTE_FLOW_ACTION_TYPE_QUEUE, + RTE_FLOW_ACTION_TYPE_END, +}; + +/** Supported actions for newer NICs */ +static const enum rte_flow_action_type enic_supported_actions_v2[] = { + RTE_FLOW_ACTION_TYPE_QUEUE, + RTE_FLOW_ACTION_TYPE_MARK, + RTE_FLOW_ACTION_TYPE_FLAG, + RTE_FLOW_ACTION_TYPE_END, +}; + +/** Action capabilities indexed by NIC version information */ +static const struct enic_action_cap enic_action_cap[] = { + [FILTER_ACTION_RQ_STEERING_FLAG] = { + .actions = enic_supported_actions_v1, + .copy_fn = enic_copy_action_v1, + }, + [FILTER_ACTION_V2_ALL] = { + .actions = enic_supported_actions_v2, + .copy_fn = enic_copy_action_v2, + }, +}; + +static int +mask_exact_match(const u8 *supported, const u8 *supplied, + unsigned int size) +{ + unsigned int i; + for (i = 0; i < size; i++) { + if (supported[i] != supplied[i]) + return 0; + } + return 1; +} + +/** + * Copy IPv4 item into version 1 NIC filter. + * + * @param item[in] + * Item specification. + * @param enic_filter[out] + * Partially filled in NIC filter structure. + * @param inner_ofst[in] + * Should always be 0 for version 1. + */ +static int +enic_copy_item_ipv4_v1(const struct rte_flow_item *item, + struct filter_v2 *enic_filter, u8 *inner_ofst) +{ + const struct rte_flow_item_ipv4 *spec = item->spec; + const struct rte_flow_item_ipv4 *mask = item->mask; + struct filter_ipv4_5tuple *enic_5tup = &enic_filter->u.ipv4; + struct ipv4_hdr supported_mask = { + .src_addr = 0xffffffff, + .dst_addr = 0xffffffff, + }; + + FLOW_TRACE(); + + if (*inner_ofst) + return ENOTSUP; + + if (!mask) + mask = &rte_flow_item_ipv4_mask; + + /* This is an exact match filter, both fields must be set */ + if (!spec || !spec->hdr.src_addr || !spec->hdr.dst_addr) { + FLOW_LOG(ERR, "IPv4 exact match src/dst addr"); + return ENOTSUP; + } + + /* check that the suppied mask exactly matches capabilty */ + if (!mask_exact_match((const u8 *)&supported_mask, + (const u8 *)item->mask, sizeof(*mask))) { + FLOW_LOG(ERR, "IPv4 exact match mask"); + return ENOTSUP; + } + + enic_filter->u.ipv4.flags = FILTER_FIELDS_IPV4_5TUPLE; + enic_5tup->src_addr = spec->hdr.src_addr; + enic_5tup->dst_addr = spec->hdr.dst_addr; + + return 0; +} + +/** + * Copy UDP item into version 1 NIC filter. + * + * @param item[in] + * Item specification. + * @param enic_filter[out] + * Partially filled in NIC filter structure. + * @param inner_ofst[in] + * Should always be 0 for version 1. + */ +static int +enic_copy_item_udp_v1(const struct rte_flow_item *item, + struct filter_v2 *enic_filter, u8 *inner_ofst) +{ + const struct rte_flow_item_udp *spec = item->spec; + const struct rte_flow_item_udp *mask = item->mask; + struct filter_ipv4_5tuple *enic_5tup = &enic_filter->u.ipv4; + struct udp_hdr supported_mask = { + .src_port = 0xffff, + .dst_port = 0xffff, + }; + + FLOW_TRACE(); + + if (*inner_ofst) + return ENOTSUP; + + if (!mask) + mask = &rte_flow_item_udp_mask; + + /* This is an exact match filter, both ports must be set */ + if (!spec || !spec->hdr.src_port || !spec->hdr.dst_port) { + FLOW_LOG(ERR, "UDP exact match src/dst addr"); + return ENOTSUP; + } + + /* check that the suppied mask exactly matches capabilty */ + if (!mask_exact_match((const u8 *)&supported_mask, + (const u8 *)item->mask, sizeof(*mask))) { + FLOW_LOG(ERR, "UDP exact match mask"); + return ENOTSUP; + } + + enic_filter->u.ipv4.flags = FILTER_FIELDS_IPV4_5TUPLE; + enic_5tup->src_port = spec->hdr.src_port; + enic_5tup->dst_port = spec->hdr.dst_port; + enic_5tup->protocol = PROTO_UDP; + + return 0; +} + +/** + * Copy TCP item into version 1 NIC filter. + * + * @param item[in] + * Item specification. + * @param enic_filter[out] + * Partially filled in NIC filter structure. + * @param inner_ofst[in] + * Should always be 0 for version 1. + */ +static int +enic_copy_item_tcp_v1(const struct rte_flow_item *item, + struct filter_v2 *enic_filter, u8 *inner_ofst) +{ + const struct rte_flow_item_tcp *spec = item->spec; + const struct rte_flow_item_tcp *mask = item->mask; + struct filter_ipv4_5tuple *enic_5tup = &enic_filter->u.ipv4; + struct tcp_hdr supported_mask = { + .src_port = 0xffff, + .dst_port = 0xffff, + }; + + FLOW_TRACE(); + + if (*inner_ofst) + return ENOTSUP; + + if (!mask) + mask = &rte_flow_item_tcp_mask; + + /* This is an exact match filter, both ports must be set */ + if (!spec || !spec->hdr.src_port || !spec->hdr.dst_port) { + FLOW_LOG(ERR, "TCPIPv4 exact match src/dst addr"); + return ENOTSUP; + } + + /* check that the suppied mask exactly matches capabilty */ + if (!mask_exact_match((const u8 *)&supported_mask, + (const u8 *)item->mask, sizeof(*mask))) { + FLOW_LOG(ERR, "TCP exact match mask"); + return ENOTSUP; + } + + enic_filter->u.ipv4.flags = FILTER_FIELDS_IPV4_5TUPLE; + enic_5tup->src_port = spec->hdr.src_port; + enic_5tup->dst_port = spec->hdr.dst_port; + enic_5tup->protocol = PROTO_TCP; + + return 0; +} + +/** + * Copy ETH item into version 2 NIC filter. + * + * @param item[in] + * Item specification. + * @param enic_filter[out] + * Partially filled in NIC filter structure. + * @param inner_ofst[in] + * If zero, this is an outer header. If non-zero, this is the offset into L5 + * where the header begins. + */ +static int +enic_copy_item_eth_v2(const struct rte_flow_item *item, + struct filter_v2 *enic_filter, u8 *inner_ofst) +{ + struct ether_hdr enic_spec; + struct ether_hdr enic_mask; + const struct rte_flow_item_eth *spec = item->spec; + const struct rte_flow_item_eth *mask = item->mask; + struct filter_generic_1 *gp = &enic_filter->u.generic_1; + + FLOW_TRACE(); + + /* Match all if no spec */ + if (!spec) + return 0; + + if (!mask) + mask = &rte_flow_item_eth_mask; + + memcpy(enic_spec.d_addr.addr_bytes, spec->dst.addr_bytes, + ETHER_ADDR_LEN); + memcpy(enic_spec.s_addr.addr_bytes, spec->src.addr_bytes, + ETHER_ADDR_LEN); + + memcpy(enic_mask.d_addr.addr_bytes, mask->dst.addr_bytes, + ETHER_ADDR_LEN); + memcpy(enic_mask.s_addr.addr_bytes, mask->src.addr_bytes, + ETHER_ADDR_LEN); + enic_spec.ether_type = spec->type; + enic_mask.ether_type = mask->type; + + if (*inner_ofst == 0) { + /* outer header */ + memcpy(gp->layer[FILTER_GENERIC_1_L2].mask, &enic_mask, + sizeof(struct ether_hdr)); + memcpy(gp->layer[FILTER_GENERIC_1_L2].val, &enic_spec, + sizeof(struct ether_hdr)); + } else { + /* inner header */ + if ((*inner_ofst + sizeof(struct ether_hdr)) > + FILTER_GENERIC_1_KEY_LEN) + return ENOTSUP; + /* Offset into L5 where inner Ethernet header goes */ + memcpy(&gp->layer[FILTER_GENERIC_1_L5].mask[*inner_ofst], + &enic_mask, sizeof(struct ether_hdr)); + memcpy(&gp->layer[FILTER_GENERIC_1_L5].val[*inner_ofst], + &enic_spec, sizeof(struct ether_hdr)); + *inner_ofst += sizeof(struct ether_hdr); + } + return 0; +} + +/** + * Copy VLAN item into version 2 NIC filter. + * + * @param item[in] + * Item specification. + * @param enic_filter[out] + * Partially filled in NIC filter structure. + * @param inner_ofst[in] + * If zero, this is an outer header. If non-zero, this is the offset into L5 + * where the header begins. + */ +static int +enic_copy_item_vlan_v2(const struct rte_flow_item *item, + struct filter_v2 *enic_filter, u8 *inner_ofst) +{ + const struct rte_flow_item_vlan *spec = item->spec; + const struct rte_flow_item_vlan *mask = item->mask; + struct filter_generic_1 *gp = &enic_filter->u.generic_1; + + FLOW_TRACE(); + + /* Match all if no spec */ + if (!spec) + return 0; + + /* Don't support filtering in tpid */ + if (mask) { + if (mask->tpid != 0) + return ENOTSUP; + } else { + mask = &rte_flow_item_vlan_mask; + RTE_ASSERT(mask->tpid == 0); + } + + if (*inner_ofst == 0) { + /* Outer header. Use the vlan mask/val fields */ + gp->mask_vlan = mask->tci; + gp->val_vlan = spec->tci; + } else { + /* Inner header. Mask/Val start at *inner_ofst into L5 */ + if ((*inner_ofst + sizeof(struct vlan_hdr)) > + FILTER_GENERIC_1_KEY_LEN) + return ENOTSUP; + memcpy(&gp->layer[FILTER_GENERIC_1_L5].mask[*inner_ofst], + mask, sizeof(struct vlan_hdr)); + memcpy(&gp->layer[FILTER_GENERIC_1_L5].val[*inner_ofst], + spec, sizeof(struct vlan_hdr)); + *inner_ofst += sizeof(struct vlan_hdr); + } + return 0; +} + +/** + * Copy IPv4 item into version 2 NIC filter. + * + * @param item[in] + * Item specification. + * @param enic_filter[out] + * Partially filled in NIC filter structure. + * @param inner_ofst[in] + * Must be 0. Don't support inner IPv4 filtering. + */ +static int +enic_copy_item_ipv4_v2(const struct rte_flow_item *item, + struct filter_v2 *enic_filter, u8 *inner_ofst) +{ + const struct rte_flow_item_ipv4 *spec = item->spec; + const struct rte_flow_item_ipv4 *mask = item->mask; + struct filter_generic_1 *gp = &enic_filter->u.generic_1; + + FLOW_TRACE(); + + if (*inner_ofst == 0) { + /* Match IPv4 */ + gp->mask_flags |= FILTER_GENERIC_1_IPV4; + gp->val_flags |= FILTER_GENERIC_1_IPV4; + + /* Match all if no spec */ + if (!spec) + return 0; + + if (!mask) + mask = &rte_flow_item_ipv4_mask; + + memcpy(gp->layer[FILTER_GENERIC_1_L3].mask, &mask->hdr, + sizeof(struct ipv4_hdr)); + memcpy(gp->layer[FILTER_GENERIC_1_L3].val, &spec->hdr, + sizeof(struct ipv4_hdr)); + } else { + /* Inner IPv4 header. Mask/Val start at *inner_ofst into L5 */ + if ((*inner_ofst + sizeof(struct ipv4_hdr)) > + FILTER_GENERIC_1_KEY_LEN) + return ENOTSUP; + memcpy(&gp->layer[FILTER_GENERIC_1_L5].mask[*inner_ofst], + mask, sizeof(struct ipv4_hdr)); + memcpy(&gp->layer[FILTER_GENERIC_1_L5].val[*inner_ofst], + spec, sizeof(struct ipv4_hdr)); + *inner_ofst += sizeof(struct ipv4_hdr); + } + return 0; +} + +/** + * Copy IPv6 item into version 2 NIC filter. + * + * @param item[in] + * Item specification. + * @param enic_filter[out] + * Partially filled in NIC filter structure. + * @param inner_ofst[in] + * Must be 0. Don't support inner IPv6 filtering. + */ +static int +enic_copy_item_ipv6_v2(const struct rte_flow_item *item, + struct filter_v2 *enic_filter, u8 *inner_ofst) +{ + const struct rte_flow_item_ipv6 *spec = item->spec; + const struct rte_flow_item_ipv6 *mask = item->mask; + struct filter_generic_1 *gp = &enic_filter->u.generic_1; + + FLOW_TRACE(); + + /* Match IPv6 */ + gp->mask_flags |= FILTER_GENERIC_1_IPV6; + gp->val_flags |= FILTER_GENERIC_1_IPV6; + + /* Match all if no spec */ + if (!spec) + return 0; + + if (!mask) + mask = &rte_flow_item_ipv6_mask; + + if (*inner_ofst == 0) { + memcpy(gp->layer[FILTER_GENERIC_1_L3].mask, &mask->hdr, + sizeof(struct ipv6_hdr)); + memcpy(gp->layer[FILTER_GENERIC_1_L3].val, &spec->hdr, + sizeof(struct ipv6_hdr)); + } else { + /* Inner IPv6 header. Mask/Val start at *inner_ofst into L5 */ + if ((*inner_ofst + sizeof(struct ipv6_hdr)) > + FILTER_GENERIC_1_KEY_LEN) + return ENOTSUP; + memcpy(&gp->layer[FILTER_GENERIC_1_L5].mask[*inner_ofst], + mask, sizeof(struct ipv6_hdr)); + memcpy(&gp->layer[FILTER_GENERIC_1_L5].val[*inner_ofst], + spec, sizeof(struct ipv6_hdr)); + *inner_ofst += sizeof(struct ipv6_hdr); + } + return 0; +} + +/** + * Copy UDP item into version 2 NIC filter. + * + * @param item[in] + * Item specification. + * @param enic_filter[out] + * Partially filled in NIC filter structure. + * @param inner_ofst[in] + * Must be 0. Don't support inner UDP filtering. + */ +static int +enic_copy_item_udp_v2(const struct rte_flow_item *item, + struct filter_v2 *enic_filter, u8 *inner_ofst) +{ + const struct rte_flow_item_udp *spec = item->spec; + const struct rte_flow_item_udp *mask = item->mask; + struct filter_generic_1 *gp = &enic_filter->u.generic_1; + + FLOW_TRACE(); + + /* Match UDP */ + gp->mask_flags |= FILTER_GENERIC_1_UDP; + gp->val_flags |= FILTER_GENERIC_1_UDP; + + /* Match all if no spec */ + if (!spec) + return 0; + + if (!mask) + mask = &rte_flow_item_udp_mask; + + if (*inner_ofst == 0) { + memcpy(gp->layer[FILTER_GENERIC_1_L4].mask, &mask->hdr, + sizeof(struct udp_hdr)); + memcpy(gp->layer[FILTER_GENERIC_1_L4].val, &spec->hdr, + sizeof(struct udp_hdr)); + } else { + /* Inner IPv6 header. Mask/Val start at *inner_ofst into L5 */ + if ((*inner_ofst + sizeof(struct udp_hdr)) > + FILTER_GENERIC_1_KEY_LEN) + return ENOTSUP; + memcpy(&gp->layer[FILTER_GENERIC_1_L5].mask[*inner_ofst], + mask, sizeof(struct udp_hdr)); + memcpy(&gp->layer[FILTER_GENERIC_1_L5].val[*inner_ofst], + spec, sizeof(struct udp_hdr)); + *inner_ofst += sizeof(struct udp_hdr); + } + return 0; +} + +/** + * Copy TCP item into version 2 NIC filter. + * + * @param item[in] + * Item specification. + * @param enic_filter[out] + * Partially filled in NIC filter structure. + * @param inner_ofst[in] + * Must be 0. Don't support inner TCP filtering. + */ +static int +enic_copy_item_tcp_v2(const struct rte_flow_item *item, + struct filter_v2 *enic_filter, u8 *inner_ofst) +{ + const struct rte_flow_item_tcp *spec = item->spec; + const struct rte_flow_item_tcp *mask = item->mask; + struct filter_generic_1 *gp = &enic_filter->u.generic_1; + + FLOW_TRACE(); + + /* Match TCP */ + gp->mask_flags |= FILTER_GENERIC_1_TCP; + gp->val_flags |= FILTER_GENERIC_1_TCP; + + /* Match all if no spec */ + if (!spec) + return 0; + + if (!mask) + return ENOTSUP; + + if (*inner_ofst == 0) { + memcpy(gp->layer[FILTER_GENERIC_1_L4].mask, &mask->hdr, + sizeof(struct tcp_hdr)); + memcpy(gp->layer[FILTER_GENERIC_1_L4].val, &spec->hdr, + sizeof(struct tcp_hdr)); + } else { + /* Inner IPv6 header. Mask/Val start at *inner_ofst into L5 */ + if ((*inner_ofst + sizeof(struct tcp_hdr)) > + FILTER_GENERIC_1_KEY_LEN) + return ENOTSUP; + memcpy(&gp->layer[FILTER_GENERIC_1_L5].mask[*inner_ofst], + mask, sizeof(struct tcp_hdr)); + memcpy(&gp->layer[FILTER_GENERIC_1_L5].val[*inner_ofst], + spec, sizeof(struct tcp_hdr)); + *inner_ofst += sizeof(struct tcp_hdr); + } + return 0; +} + +/** + * Copy SCTP item into version 2 NIC filter. + * + * @param item[in] + * Item specification. + * @param enic_filter[out] + * Partially filled in NIC filter structure. + * @param inner_ofst[in] + * Must be 0. Don't support inner SCTP filtering. + */ +static int +enic_copy_item_sctp_v2(const struct rte_flow_item *item, + struct filter_v2 *enic_filter, u8 *inner_ofst) +{ + const struct rte_flow_item_sctp *spec = item->spec; + const struct rte_flow_item_sctp *mask = item->mask; + struct filter_generic_1 *gp = &enic_filter->u.generic_1; + + FLOW_TRACE(); + + if (*inner_ofst) + return ENOTSUP; + + /* Match all if no spec */ + if (!spec) + return 0; + + if (!mask) + mask = &rte_flow_item_sctp_mask; + + memcpy(gp->layer[FILTER_GENERIC_1_L4].mask, &mask->hdr, + sizeof(struct sctp_hdr)); + memcpy(gp->layer[FILTER_GENERIC_1_L4].val, &spec->hdr, + sizeof(struct sctp_hdr)); + return 0; +} + +/** + * Copy UDP item into version 2 NIC filter. + * + * @param item[in] + * Item specification. + * @param enic_filter[out] + * Partially filled in NIC filter structure. + * @param inner_ofst[in] + * Must be 0. VxLAN headers always start at the beginning of L5. + */ +static int +enic_copy_item_vxlan_v2(const struct rte_flow_item *item, + struct filter_v2 *enic_filter, u8 *inner_ofst) +{ + const struct rte_flow_item_vxlan *spec = item->spec; + const struct rte_flow_item_vxlan *mask = item->mask; + struct filter_generic_1 *gp = &enic_filter->u.generic_1; + + FLOW_TRACE(); + + if (*inner_ofst) + return EINVAL; + + /* Match all if no spec */ + if (!spec) + return 0; + + if (!mask) + mask = &rte_flow_item_vxlan_mask; + + memcpy(gp->layer[FILTER_GENERIC_1_L5].mask, mask, + sizeof(struct vxlan_hdr)); + memcpy(gp->layer[FILTER_GENERIC_1_L5].val, spec, + sizeof(struct vxlan_hdr)); + + *inner_ofst = sizeof(struct vxlan_hdr); + return 0; +} + +/** + * Return 1 if current item is valid on top of the previous one. + * + * @param prev_item[in] + * The item before this one in the pattern or RTE_FLOW_ITEM_TYPE_END if this + * is the first item. + * @param item_info[in] + * Info about this item, like valid previous items. + * @param is_first[in] + * True if this the first item in the pattern. + */ +static int +item_stacking_valid(enum rte_flow_item_type prev_item, + const struct enic_items *item_info, u8 is_first_item) +{ + enum rte_flow_item_type const *allowed_items = item_info->prev_items; + + FLOW_TRACE(); + + for (; *allowed_items != RTE_FLOW_ITEM_TYPE_END; allowed_items++) { + if (prev_item == *allowed_items) + return 1; + } + + /* This is the first item in the stack. Check if that's cool */ + if (is_first_item && item_info->valid_start_item) + return 1; + + return 0; +} + +/** + * Build the intenal enic filter structure from the provided pattern. The + * pattern is validated as the items are copied. + * + * @param pattern[in] + * @param items_info[in] + * Info about this NICs item support, like valid previous items. + * @param enic_filter[out] + * NIC specfilc filters derived from the pattern. + * @param error[out] + */ +static int +enic_copy_filter(const struct rte_flow_item pattern[], + const struct enic_items *items_info, + struct filter_v2 *enic_filter, + struct rte_flow_error *error) +{ + int ret; + const struct rte_flow_item *item = pattern; + u8 inner_ofst = 0; /* If encapsulated, ofst into L5 */ + enum rte_flow_item_type prev_item; + const struct enic_items *item_info; + + u8 is_first_item = 1; + + FLOW_TRACE(); + + prev_item = 0; + + for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { + /* Get info about how to validate and copy the item. If NULL + * is returned the nic does not support the item. + */ + if (item->type == RTE_FLOW_ITEM_TYPE_VOID) + continue; + + item_info = &items_info[item->type]; + + /* check to see if item stacking is valid */ + if (!item_stacking_valid(prev_item, item_info, is_first_item)) + goto stacking_error; + + ret = item_info->copy_item(item, enic_filter, &inner_ofst); + if (ret) + goto item_not_supported; + prev_item = item->type; + is_first_item = 0; + } + return 0; + +item_not_supported: + rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_ITEM, + NULL, "enic type error"); + return -rte_errno; + +stacking_error: + rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, + item, "stacking error"); + return -rte_errno; +} + +/** + * Build the intenal version 1 NIC action structure from the provided pattern. + * The pattern is validated as the items are copied. + * + * @param actions[in] + * @param enic_action[out] + * NIC specfilc actions derived from the actions. + * @param error[out] + */ +static int +enic_copy_action_v1(const struct rte_flow_action actions[], + struct filter_action_v2 *enic_action) +{ + FLOW_TRACE(); + + for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { + if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) + continue; + + switch (actions->type) { + case RTE_FLOW_ACTION_TYPE_QUEUE: { + const struct rte_flow_action_queue *queue = + (const struct rte_flow_action_queue *) + actions->conf; + enic_action->rq_idx = + enic_rte_rq_idx_to_sop_idx(queue->index); + break; + } + default: + RTE_ASSERT(0); + break; + } + } + enic_action->type = FILTER_ACTION_RQ_STEERING; + return 0; +} + +/** + * Build the intenal version 2 NIC action structure from the provided pattern. + * The pattern is validated as the items are copied. + * + * @param actions[in] + * @param enic_action[out] + * NIC specfilc actions derived from the actions. + * @param error[out] + */ +static int +enic_copy_action_v2(const struct rte_flow_action actions[], + struct filter_action_v2 *enic_action) +{ + FLOW_TRACE(); + + for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { + switch (actions->type) { + case RTE_FLOW_ACTION_TYPE_QUEUE: { + const struct rte_flow_action_queue *queue = + (const struct rte_flow_action_queue *) + actions->conf; + enic_action->rq_idx = + enic_rte_rq_idx_to_sop_idx(queue->index); + enic_action->flags |= FILTER_ACTION_RQ_STEERING_FLAG; + break; + } + case RTE_FLOW_ACTION_TYPE_MARK: { + const struct rte_flow_action_mark *mark = + (const struct rte_flow_action_mark *) + actions->conf; + + /* ENIC_MAGIC_FILTER_ID is reserved and is the highest + * in the range of allows mark ids. + */ + if (mark->id >= ENIC_MAGIC_FILTER_ID) + return EINVAL; + enic_action->filter_id = mark->id; + enic_action->flags |= FILTER_ACTION_FILTER_ID_FLAG; + break; + } + case RTE_FLOW_ACTION_TYPE_FLAG: { + enic_action->filter_id = ENIC_MAGIC_FILTER_ID; + enic_action->flags |= FILTER_ACTION_FILTER_ID_FLAG; + break; + } + case RTE_FLOW_ACTION_TYPE_VOID: + continue; + default: + RTE_ASSERT(0); + break; + } + } + enic_action->type = FILTER_ACTION_V2; + return 0; +} + +/** Check if the action is supported */ +static int +enic_match_action(const struct rte_flow_action *action, + const enum rte_flow_action_type *supported_actions) +{ + for (; *supported_actions != RTE_FLOW_ACTION_TYPE_END; + supported_actions++) { + if (action->type == *supported_actions) + return 1; + } + return 0; +} + +/** Get the NIC filter capabilties structure */ +static const struct enic_filter_cap * +enic_get_filter_cap(struct enic *enic) +{ + if (enic->flow_filter_mode) + return &enic_filter_cap[enic->flow_filter_mode]; + + return NULL; +} + +/** Get the actions for this NIC version. */ +static const struct enic_action_cap * +enic_get_action_cap(struct enic *enic) +{ + static const struct enic_action_cap *ea; + + if (enic->filter_tags) + ea = &enic_action_cap[FILTER_ACTION_V2_ALL]; + else + ea = &enic_action_cap[FILTER_ACTION_RQ_STEERING_FLAG]; + return ea; +} + +/* Debug function to dump internal NIC action structure. */ +static void +enic_dump_actions(const struct filter_action_v2 *ea) +{ + if (ea->type == FILTER_ACTION_RQ_STEERING) { + FLOW_LOG(INFO, "Action(V1), queue: %u\n", ea->rq_idx); + } else if (ea->type == FILTER_ACTION_V2) { + FLOW_LOG(INFO, "Actions(V2)\n"); + if (ea->flags & FILTER_ACTION_RQ_STEERING_FLAG) + FLOW_LOG(INFO, "\tqueue: %u\n", + enic_sop_rq_idx_to_rte_idx(ea->rq_idx)); + if (ea->flags & FILTER_ACTION_FILTER_ID_FLAG) + FLOW_LOG(INFO, "\tfilter_id: %u\n", ea->filter_id); + } +} + +/* Debug function to dump internal NIC filter structure. */ +static void +enic_dump_filter(const struct filter_v2 *filt) +{ + const struct filter_generic_1 *gp; + int i, j, mbyte; + char buf[128], *bp; + char ip4[16], ip6[16], udp[16], tcp[16], tcpudp[16], ip4csum[16]; + char l4csum[16], ipfrag[16]; + + switch (filt->type) { + case FILTER_IPV4_5TUPLE: + FLOW_LOG(INFO, "FILTER_IPV4_5TUPLE\n"); + break; + case FILTER_USNIC_IP: + case FILTER_DPDK_1: + /* FIXME: this should be a loop */ + gp = &filt->u.generic_1; + FLOW_LOG(INFO, "Filter: vlan: 0x%04x, mask: 0x%04x\n", + gp->val_vlan, gp->mask_vlan); + + if (gp->mask_flags & FILTER_GENERIC_1_IPV4) + sprintf(ip4, "%s ", + (gp->val_flags & FILTER_GENERIC_1_IPV4) + ? "ip4(y)" : "ip4(n)"); + else + sprintf(ip4, "%s ", "ip4(x)"); + + if (gp->mask_flags & FILTER_GENERIC_1_IPV6) + sprintf(ip6, "%s ", + (gp->val_flags & FILTER_GENERIC_1_IPV4) + ? "ip6(y)" : "ip6(n)"); + else + sprintf(ip6, "%s ", "ip6(x)"); + + if (gp->mask_flags & FILTER_GENERIC_1_UDP) + sprintf(udp, "%s ", + (gp->val_flags & FILTER_GENERIC_1_UDP) + ? "udp(y)" : "udp(n)"); + else + sprintf(udp, "%s ", "udp(x)"); + + if (gp->mask_flags & FILTER_GENERIC_1_TCP) + sprintf(tcp, "%s ", + (gp->val_flags & FILTER_GENERIC_1_TCP) + ? "tcp(y)" : "tcp(n)"); + else + sprintf(tcp, "%s ", "tcp(x)"); + + if (gp->mask_flags & FILTER_GENERIC_1_TCP_OR_UDP) + sprintf(tcpudp, "%s ", + (gp->val_flags & FILTER_GENERIC_1_TCP_OR_UDP) + ? "tcpudp(y)" : "tcpudp(n)"); + else + sprintf(tcpudp, "%s ", "tcpudp(x)"); + + if (gp->mask_flags & FILTER_GENERIC_1_IP4SUM_OK) + sprintf(ip4csum, "%s ", + (gp->val_flags & FILTER_GENERIC_1_IP4SUM_OK) + ? "ip4csum(y)" : "ip4csum(n)"); + else + sprintf(ip4csum, "%s ", "ip4csum(x)"); + + if (gp->mask_flags & FILTER_GENERIC_1_L4SUM_OK) + sprintf(l4csum, "%s ", + (gp->val_flags & FILTER_GENERIC_1_L4SUM_OK) + ? "l4csum(y)" : "l4csum(n)"); + else + sprintf(l4csum, "%s ", "l4csum(x)"); + + if (gp->mask_flags & FILTER_GENERIC_1_IPFRAG) + sprintf(ipfrag, "%s ", + (gp->val_flags & FILTER_GENERIC_1_IPFRAG) + ? "ipfrag(y)" : "ipfrag(n)"); + else + sprintf(ipfrag, "%s ", "ipfrag(x)"); + FLOW_LOG(INFO, "\tFlags: %s%s%s%s%s%s%s%s\n", ip4, ip6, udp, + tcp, tcpudp, ip4csum, l4csum, ipfrag); + + for (i = 0; i < FILTER_GENERIC_1_NUM_LAYERS; i++) { + mbyte = FILTER_GENERIC_1_KEY_LEN - 1; + while (mbyte && !gp->layer[i].mask[mbyte]) + mbyte--; + if (mbyte == 0) + continue; + + bp = buf; + for (j = 0; j <= mbyte; j++) { + sprintf(bp, "%02x", + gp->layer[i].mask[j]); + bp += 2; + } + *bp = '\0'; + FLOW_LOG(INFO, "\tL%u mask: %s\n", i + 2, buf); + bp = buf; + for (j = 0; j <= mbyte; j++) { + sprintf(bp, "%02x", + gp->layer[i].val[j]); + bp += 2; + } + *bp = '\0'; + FLOW_LOG(INFO, "\tL%u val: %s\n", i + 2, buf); + } + break; + default: + FLOW_LOG(INFO, "FILTER UNKNOWN\n"); + break; + } +} + +/* Debug function to dump internal NIC flow structures. */ +static void +enic_dump_flow(const struct filter_action_v2 *ea, const struct filter_v2 *filt) +{ + enic_dump_filter(filt); + enic_dump_actions(ea); +} + + +/** + * Internal flow parse/validate function. + * + * @param dev[in] + * This device pointer. + * @param pattern[in] + * @param actions[in] + * @param error[out] + * @param enic_filter[out] + * Internal NIC filter structure pointer. + * @param enic_action[out] + * Internal NIC action structure pointer. + */ +static int +enic_flow_parse(struct rte_eth_dev *dev, + const struct rte_flow_attr *attrs, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error, + struct filter_v2 *enic_filter, + struct filter_action_v2 *enic_action) +{ + unsigned int ret = 0; + struct enic *enic = pmd_priv(dev); + const struct enic_filter_cap *enic_filter_cap; + const struct enic_action_cap *enic_action_cap; + const struct rte_flow_action *action; + + FLOW_TRACE(); + + memset(enic_filter, 0, sizeof(*enic_filter)); + memset(enic_action, 0, sizeof(*enic_action)); + + if (!pattern) { + rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_NUM, + NULL, "No pattern specified"); + return -rte_errno; + } + + if (!actions) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION_NUM, + NULL, "No action specified"); + return -rte_errno; + } + + if (attrs) { + if (attrs->group) { + rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ATTR_GROUP, + NULL, + "priority groups are not supported"); + return -rte_errno; + } else if (attrs->priority) { + rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, + NULL, + "priorities are not supported"); + return -rte_errno; + } else if (attrs->egress) { + rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, + NULL, + "egress is not supported"); + return -rte_errno; + } else if (!attrs->ingress) { + rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, + NULL, + "only ingress is supported"); + return -rte_errno; + } + + } else { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR, + NULL, "No attribute specified"); + return -rte_errno; + } + + /* Verify Actions. */ + enic_action_cap = enic_get_action_cap(enic); + for (action = &actions[0]; action->type != RTE_FLOW_ACTION_TYPE_END; + action++) { + if (action->type == RTE_FLOW_ACTION_TYPE_VOID) + continue; + else if (!enic_match_action(action, enic_action_cap->actions)) + break; + } + if (action->type != RTE_FLOW_ACTION_TYPE_END) { + rte_flow_error_set(error, EPERM, RTE_FLOW_ERROR_TYPE_ACTION, + action, "Invalid action."); + return -rte_errno; + } + ret = enic_action_cap->copy_fn(actions, enic_action); + if (ret) { + rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE, + NULL, "Unsupported action."); + return -rte_errno; + } + + /* Verify Flow items. If copying the filter from flow format to enic + * format fails, the flow is not supported + */ + enic_filter_cap = enic_get_filter_cap(enic); + if (enic_filter_cap == NULL) { + rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE, + NULL, "Flow API not available"); + return -rte_errno; + } + enic_filter->type = enic->flow_filter_mode; + ret = enic_copy_filter(pattern, enic_filter_cap->item_info, + enic_filter, error); + return ret; +} + +/** + * Push filter/action to the NIC. + * + * @param enic[in] + * Device structure pointer. + * @param enic_filter[in] + * Internal NIC filter structure pointer. + * @param enic_action[in] + * Internal NIC action structure pointer. + * @param error[out] + */ +static struct rte_flow * +enic_flow_add_filter(struct enic *enic, struct filter_v2 *enic_filter, + struct filter_action_v2 *enic_action, + struct rte_flow_error *error) +{ + struct rte_flow *flow; + int ret; + u16 entry; + + FLOW_TRACE(); + + flow = rte_calloc(__func__, 1, sizeof(*flow), 0); + if (!flow) { + rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, + NULL, "cannot allocate flow memory"); + return NULL; + } + + /* entry[in] is the queue id, entry[out] is the filter Id for delete */ + entry = enic_action->rq_idx; + ret = vnic_dev_classifier(enic->vdev, CLSF_ADD, &entry, enic_filter, + enic_action); + if (!ret) { + flow->enic_filter_id = entry; + flow->enic_filter = *enic_filter; + } else { + rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_HANDLE, + NULL, "vnic_dev_classifier error"); + rte_free(flow); + return NULL; + } + return flow; +} + +/** + * Remove filter/action from the NIC. + * + * @param enic[in] + * Device structure pointer. + * @param filter_id[in] + * Id of NIC filter. + * @param enic_action[in] + * Internal NIC action structure pointer. + * @param error[out] + */ +static int +enic_flow_del_filter(struct enic *enic, u16 filter_id, + struct rte_flow_error *error) +{ + int ret; + + FLOW_TRACE(); + + ret = vnic_dev_classifier(enic->vdev, CLSF_DEL, &filter_id, NULL, NULL); + if (!ret) + rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_HANDLE, + NULL, "vnic_dev_classifier failed"); + return ret; +} + +/* + * The following functions are callbacks for Generic flow API. + */ + +/** + * Validate a flow supported by the NIC. + * + * @see rte_flow_validate() + * @see rte_flow_ops + */ +static int +enic_flow_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attrs, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error) +{ + struct filter_v2 enic_filter; + struct filter_action_v2 enic_action; + int ret; + + FLOW_TRACE(); + + ret = enic_flow_parse(dev, attrs, pattern, actions, error, + &enic_filter, &enic_action); + if (!ret) + enic_dump_flow(&enic_action, &enic_filter); + return ret; +} + +/** + * Create a flow supported by the NIC. + * + * @see rte_flow_create() + * @see rte_flow_ops + */ +static struct rte_flow * +enic_flow_create(struct rte_eth_dev *dev, + const struct rte_flow_attr *attrs, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error) +{ + int ret; + struct filter_v2 enic_filter; + struct filter_action_v2 enic_action; + struct rte_flow *flow; + struct enic *enic = pmd_priv(dev); + + FLOW_TRACE(); + + ret = enic_flow_parse(dev, attrs, pattern, actions, error, &enic_filter, + &enic_action); + if (ret < 0) + return NULL; + + rte_spinlock_lock(&enic->flows_lock); + flow = enic_flow_add_filter(enic, &enic_filter, &enic_action, + error); + if (flow) + LIST_INSERT_HEAD(&enic->flows, flow, next); + rte_spinlock_unlock(&enic->flows_lock); + + return flow; +} + +/** + * Destroy a flow supported by the NIC. + * + * @see rte_flow_destroy() + * @see rte_flow_ops + */ +static int +enic_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow, + __rte_unused struct rte_flow_error *error) +{ + struct enic *enic = pmd_priv(dev); + + FLOW_TRACE(); + + rte_spinlock_lock(&enic->flows_lock); + enic_flow_del_filter(enic, flow->enic_filter_id, error); + LIST_REMOVE(flow, next); + rte_spinlock_unlock(&enic->flows_lock); + return 0; +} + +/** + * Flush all flows on the device. + * + * @see rte_flow_flush() + * @see rte_flow_ops + */ +static int +enic_flow_flush(struct rte_eth_dev *dev, struct rte_flow_error *error) +{ + struct rte_flow *flow; + struct enic *enic = pmd_priv(dev); + + FLOW_TRACE(); + + rte_spinlock_lock(&enic->flows_lock); + + while (!LIST_EMPTY(&enic->flows)) { + flow = LIST_FIRST(&enic->flows); + enic_flow_del_filter(enic, flow->enic_filter_id, error); + LIST_REMOVE(flow, next); + } + rte_spinlock_unlock(&enic->flows_lock); + return 0; +} + +/** + * Flow callback registration. + * + * @see rte_flow_ops + */ +const struct rte_flow_ops enic_flow_ops = { + .validate = enic_flow_validate, + .create = enic_flow_create, + .destroy = enic_flow_destroy, + .flush = enic_flow_flush, +}; diff --git a/dpdk/drivers/net/enic/enic_main.c b/dpdk/drivers/net/enic/enic_main.c index 7549c12e..1694aed1 100644 --- a/dpdk/drivers/net/enic/enic_main.c +++ b/dpdk/drivers/net/enic/enic_main.c @@ -40,6 +40,7 @@ #include #include +#include #include #include #include @@ -137,6 +138,7 @@ static void enic_clear_soft_stats(struct enic *enic) struct enic_soft_stats *soft_stats = &enic->soft_stats; rte_atomic64_clear(&soft_stats->rx_nombuf); rte_atomic64_clear(&soft_stats->rx_packet_errors); + rte_atomic64_clear(&soft_stats->tx_oversized); } static void enic_init_soft_stats(struct enic *enic) @@ -144,6 +146,7 @@ static void enic_init_soft_stats(struct enic *enic) struct enic_soft_stats *soft_stats = &enic->soft_stats; rte_atomic64_init(&soft_stats->rx_nombuf); rte_atomic64_init(&soft_stats->rx_packet_errors); + rte_atomic64_init(&soft_stats->tx_oversized); enic_clear_soft_stats(enic); } @@ -154,16 +157,17 @@ void enic_dev_stats_clear(struct enic *enic) enic_clear_soft_stats(enic); } -void enic_dev_stats_get(struct enic *enic, struct rte_eth_stats *r_stats) +int enic_dev_stats_get(struct enic *enic, struct rte_eth_stats *r_stats) { struct vnic_stats *stats; struct enic_soft_stats *soft_stats = &enic->soft_stats; int64_t rx_truncated; uint64_t rx_packet_errors; + int ret = vnic_dev_stats_dump(enic->vdev, &stats); - if (vnic_dev_stats_dump(enic->vdev, &stats)) { + if (ret) { dev_err(enic, "Error in getting stats\n"); - return; + return ret; } /* The number of truncated packets can only be calculated by @@ -183,42 +187,37 @@ void enic_dev_stats_get(struct enic *enic, struct rte_eth_stats *r_stats) r_stats->obytes = stats->tx.tx_bytes_ok; r_stats->ierrors = stats->rx.rx_errors + stats->rx.rx_drop; - r_stats->oerrors = stats->tx.tx_errors; + r_stats->oerrors = stats->tx.tx_errors + + rte_atomic64_read(&soft_stats->tx_oversized); r_stats->imissed = stats->rx.rx_no_bufs + rx_truncated; r_stats->rx_nombuf = rte_atomic64_read(&soft_stats->rx_nombuf); + return 0; } -void enic_del_mac_address(struct enic *enic) +void enic_del_mac_address(struct enic *enic, int mac_index) { - if (vnic_dev_del_addr(enic->vdev, enic->mac_addr)) + struct rte_eth_dev *eth_dev = enic->rte_dev; + uint8_t *mac_addr = eth_dev->data->mac_addrs[mac_index].addr_bytes; + + if (vnic_dev_del_addr(enic->vdev, mac_addr)) dev_err(enic, "del mac addr failed\n"); } -void enic_set_mac_address(struct enic *enic, uint8_t *mac_addr) +int enic_set_mac_address(struct enic *enic, uint8_t *mac_addr) { int err; if (!is_eth_addr_valid(mac_addr)) { dev_err(enic, "invalid mac address\n"); - return; + return -EINVAL; } - err = vnic_dev_del_addr(enic->vdev, enic->mac_addr); - if (err) { - dev_err(enic, "del mac addr failed\n"); - return; - } - - ether_addr_copy((struct ether_addr *)mac_addr, - (struct ether_addr *)enic->mac_addr); - err = vnic_dev_add_addr(enic->vdev, mac_addr); - if (err) { + if (err) dev_err(enic, "add mac addr failed\n"); - return; - } + return err; } static void @@ -228,7 +227,7 @@ enic_free_rq_buf(struct rte_mbuf **mbuf) return; rte_pktmbuf_free(*mbuf); - mbuf = NULL; + *mbuf = NULL; } void enic_init_vnic_resources(struct enic *enic) @@ -240,14 +239,14 @@ void enic_init_vnic_resources(struct enic *enic) struct vnic_rq *data_rq; for (index = 0; index < enic->rq_count; index++) { - cq_idx = enic_cq_rq(enic, enic_sop_rq(index)); + cq_idx = enic_cq_rq(enic, enic_rte_rq_idx_to_sop_idx(index)); - vnic_rq_init(&enic->rq[enic_sop_rq(index)], + vnic_rq_init(&enic->rq[enic_rte_rq_idx_to_sop_idx(index)], cq_idx, error_interrupt_enable, error_interrupt_offset); - data_rq = &enic->rq[enic_data_rq(index)]; + data_rq = &enic->rq[enic_rte_rq_idx_to_data_idx(index)]; if (data_rq->in_use) vnic_rq_init(data_rq, cq_idx, @@ -284,7 +283,7 @@ void enic_init_vnic_resources(struct enic *enic) 0 /* cq_entry_enable */, 1 /* cq_message_enable */, 0 /* interrupt offset */, - (u64)enic->wq[index].cqmsg_rz->phys_addr); + (u64)enic->wq[index].cqmsg_rz->iova); } vnic_intr_init(&enic->intr, @@ -317,7 +316,7 @@ enic_alloc_rx_queue_mbufs(struct enic *enic, struct vnic_rq *rq) } mb->data_off = RTE_PKTMBUF_HEADROOM; - dma_addr = (dma_addr_t)(mb->buf_physaddr + dma_addr = (dma_addr_t)(mb->buf_iova + RTE_PKTMBUF_HEADROOM); rq_enet_desc_enc(rqd, dma_addr, (rq->is_sop ? RQ_ENET_TYPE_ONLY_SOP @@ -363,7 +362,7 @@ enic_alloc_consistent(void *priv, size_t size, } vaddr = rz->addr; - *dma_handle = (dma_addr_t)rz->phys_addr; + *dma_handle = (dma_addr_t)rz->iova; mze = rte_malloc("enic memzone entry", sizeof(struct enic_memzone_entry), 0); @@ -372,6 +371,7 @@ enic_alloc_consistent(void *priv, size_t size, pr_err("%s : Failed to allocate memory for memzone list\n", __func__); rte_memzone_free(rz); + return NULL; } mze->rz = rz; @@ -395,7 +395,7 @@ enic_free_consistent(void *priv, rte_spinlock_lock(&enic->memzone_list_lock); LIST_FOREACH(mze, &enic->memzone_list, entries) { if (mze->rz->addr == vaddr && - mze->rz->phys_addr == dma_handle) + mze->rz->iova == dma_handle) break; } if (mze == NULL) { @@ -410,14 +410,31 @@ enic_free_consistent(void *priv, rte_free(mze); } -static void -enic_intr_handler(__rte_unused struct rte_intr_handle *handle, - void *arg) +int enic_link_update(struct enic *enic) { - struct enic *enic = pmd_priv((struct rte_eth_dev *)arg); + struct rte_eth_dev *eth_dev = enic->rte_dev; + int ret; + int link_status = 0; + + link_status = enic_get_link_status(enic); + ret = (link_status == enic->link_status); + enic->link_status = link_status; + eth_dev->data->dev_link.link_status = link_status; + eth_dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX; + eth_dev->data->dev_link.link_speed = vnic_dev_port_speed(enic->vdev); + return ret; +} + +static void +enic_intr_handler(void *arg) +{ + struct rte_eth_dev *dev = (struct rte_eth_dev *)arg; + struct enic *enic = pmd_priv(dev); vnic_intr_return_all_credits(&enic->intr); + enic_link_update(enic); + _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL, NULL); enic_log_q_error(enic); } @@ -429,7 +446,13 @@ int enic_enable(struct enic *enic) eth_dev->data->dev_link.link_speed = vnic_dev_port_speed(enic->vdev); eth_dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX; - vnic_dev_notify_set(enic->vdev, -1); /* No Intr for notify */ + + /* vnic notification of link status has already been turned on in + * enic_dev_init() which is called during probe time. Here we are + * just turning on interrupt vector 0 if needed. + */ + if (eth_dev->data->dev_conf.intr_conf.lsc) + vnic_dev_notify_set(enic->vdev, 0); if (enic_clsf_init(enic)) dev_warning(enic, "Init of hash table for clsf failed."\ @@ -437,17 +460,17 @@ int enic_enable(struct enic *enic) for (index = 0; index < enic->rq_count; index++) { err = enic_alloc_rx_queue_mbufs(enic, - &enic->rq[enic_sop_rq(index)]); + &enic->rq[enic_rte_rq_idx_to_sop_idx(index)]); if (err) { dev_err(enic, "Failed to alloc sop RX queue mbufs\n"); return err; } err = enic_alloc_rx_queue_mbufs(enic, - &enic->rq[enic_data_rq(index)]); + &enic->rq[enic_rte_rq_idx_to_data_idx(index)]); if (err) { /* release the allocated mbufs for the sop rq*/ enic_rxmbuf_queue_release(enic, - &enic->rq[enic_sop_rq(index)]); + &enic->rq[enic_rte_rq_idx_to_sop_idx(index)]); dev_err(enic, "Failed to alloc data RX queue mbufs\n"); return err; @@ -517,6 +540,9 @@ void enic_free_rq(void *rxq) vnic_rq_free(rq_data); vnic_cq_free(&enic->cq[enic_sop_rq_idx_to_cq_idx(rq_sop->index)]); + + rq_sop->in_use = 0; + rq_data->in_use = 0; } void enic_start_wq(struct enic *enic, uint16_t queue_idx) @@ -541,8 +567,10 @@ int enic_stop_wq(struct enic *enic, uint16_t queue_idx) void enic_start_rq(struct enic *enic, uint16_t queue_idx) { - struct vnic_rq *rq_sop = &enic->rq[enic_sop_rq(queue_idx)]; - struct vnic_rq *rq_data = &enic->rq[rq_sop->data_queue_idx]; + struct vnic_rq *rq_sop; + struct vnic_rq *rq_data; + rq_sop = &enic->rq[enic_rte_rq_idx_to_sop_idx(queue_idx)]; + rq_data = &enic->rq[rq_sop->data_queue_idx]; struct rte_eth_dev *eth_dev = enic->rte_dev; if (rq_data->in_use) @@ -556,8 +584,10 @@ int enic_stop_rq(struct enic *enic, uint16_t queue_idx) { int ret1 = 0, ret2 = 0; struct rte_eth_dev *eth_dev = enic->rte_dev; - struct vnic_rq *rq_sop = &enic->rq[enic_sop_rq(queue_idx)]; - struct vnic_rq *rq_data = &enic->rq[rq_sop->data_queue_idx]; + struct vnic_rq *rq_sop; + struct vnic_rq *rq_data; + rq_sop = &enic->rq[enic_rte_rq_idx_to_sop_idx(queue_idx)]; + rq_data = &enic->rq[rq_sop->data_queue_idx]; ret2 = vnic_rq_disable(rq_sop); rte_mb(); @@ -578,13 +608,14 @@ int enic_alloc_rq(struct enic *enic, uint16_t queue_idx, uint16_t nb_desc, uint16_t free_thresh) { int rc; - uint16_t sop_queue_idx = enic_sop_rq(queue_idx); - uint16_t data_queue_idx = enic_data_rq(queue_idx); + uint16_t sop_queue_idx = enic_rte_rq_idx_to_sop_idx(queue_idx); + uint16_t data_queue_idx = enic_rte_rq_idx_to_data_idx(queue_idx); struct vnic_rq *rq_sop = &enic->rq[sop_queue_idx]; struct vnic_rq *rq_data = &enic->rq[data_queue_idx]; unsigned int mbuf_size, mbufs_per_pkt; unsigned int nb_sop_desc, nb_data_desc; uint16_t min_sop, max_sop, min_data, max_data; + uint16_t mtu = enic->rte_dev->data->mtu; rq_sop->is_sop = 1; rq_sop->data_queue_idx = data_queue_idx; @@ -604,9 +635,9 @@ int enic_alloc_rq(struct enic *enic, uint16_t queue_idx, RTE_PKTMBUF_HEADROOM); if (enic->rte_dev->data->dev_conf.rxmode.enable_scatter) { - dev_info(enic, "Scatter rx mode enabled\n"); + dev_info(enic, "Rq %u Scatter rx mode enabled\n", queue_idx); /* ceil((mtu + ETHER_HDR_LEN + 4)/mbuf_size) */ - mbufs_per_pkt = ((enic->config.mtu + ETHER_HDR_LEN + 4) + + mbufs_per_pkt = ((mtu + ETHER_HDR_LEN + 4) + (mbuf_size - 1)) / mbuf_size; } else { dev_info(enic, "Scatter rx mode disabled\n"); @@ -657,7 +688,7 @@ int enic_alloc_rq(struct enic *enic, uint16_t queue_idx, } if (mbufs_per_pkt > 1) { dev_info(enic, "For mtu %d and mbuf size %d valid rx descriptor range is %d to %d\n", - enic->config.mtu, mbuf_size, min_sop + min_data, + mtu, mbuf_size, min_sop + min_data, max_sop + max_data); } dev_info(enic, "Using %d rx descriptors (sop %d, data %d)\n", @@ -708,6 +739,8 @@ int enic_alloc_rq(struct enic *enic, uint16_t queue_idx, goto err_free_sop_mbuf; } + rq_sop->tot_nb_desc = nb_desc; /* squirl away for MTU update function */ + return 0; err_free_sop_mbuf: @@ -804,6 +837,10 @@ int enic_disable(struct enic *enic) vnic_intr_mask(&enic->intr); (void)vnic_intr_masked(&enic->intr); /* flush write */ + rte_intr_disable(&enic->pdev->intr_handle); + rte_intr_callback_unregister(&enic->pdev->intr_handle, + enic_intr_handler, + (void *)enic->rte_dev); vnic_dev_disable(enic->vdev); @@ -825,8 +862,14 @@ int enic_disable(struct enic *enic) } } + /* If we were using interrupts, set the interrupt vector to -1 + * to disable interrupts. We are not disabling link notifcations, + * though, as we want the polling of link status to continue working. + */ + if (enic->rte_dev->data->dev_conf.intr_conf.lsc) + vnic_dev_notify_set(enic->vdev, -1); + vnic_dev_set_reset_flag(enic->vdev, 1); - vnic_dev_notify_unset(enic->vdev); for (i = 0; i < enic->wq_count; i++) vnic_wq_clean(&enic->wq[i], enic_free_wq_buf); @@ -928,7 +971,7 @@ static int enic_set_rsscpu(struct enic *enic, u8 rss_hash_bits) for (i = 0; i < (1 << rss_hash_bits); i++) (*rss_cpu_buf_va).cpu[i / 4].b[i % 4] = - enic_sop_rq(i % enic->rq_count); + enic_rte_rq_idx_to_sop_idx(i % enic->rq_count); err = enic_set_rss_cpu(enic, rss_cpu_buf_pa, @@ -1028,7 +1071,13 @@ static void enic_dev_deinit(struct enic *enic) { struct rte_eth_dev *eth_dev = enic->rte_dev; + /* stop link status checking */ + vnic_dev_notify_unset(enic->vdev); + rte_free(eth_dev->data->mac_addrs); + rte_free(enic->cq); + rte_free(enic->rq); + rte_free(enic->wq); } @@ -1036,27 +1085,28 @@ int enic_set_vnic_res(struct enic *enic) { struct rte_eth_dev *eth_dev = enic->rte_dev; int rc = 0; + unsigned int required_rq, required_wq, required_cq; - /* With Rx scatter support, two RQs are now used per RQ used by - * the application. - */ - if (enic->conf_rq_count < eth_dev->data->nb_rx_queues) { + /* Always use two vNIC RQs per eth_dev RQ, regardless of Rx scatter. */ + required_rq = eth_dev->data->nb_rx_queues * 2; + required_wq = eth_dev->data->nb_tx_queues; + required_cq = eth_dev->data->nb_rx_queues + eth_dev->data->nb_tx_queues; + + if (enic->conf_rq_count < required_rq) { dev_err(dev, "Not enough Receive queues. Requested:%u which uses %d RQs on VIC, Configured:%u\n", eth_dev->data->nb_rx_queues, - eth_dev->data->nb_rx_queues * 2, enic->conf_rq_count); + required_rq, enic->conf_rq_count); rc = -EINVAL; } - if (enic->conf_wq_count < eth_dev->data->nb_tx_queues) { + if (enic->conf_wq_count < required_wq) { dev_err(dev, "Not enough Transmit queues. Requested:%u, Configured:%u\n", eth_dev->data->nb_tx_queues, enic->conf_wq_count); rc = -EINVAL; } - if (enic->conf_cq_count < (eth_dev->data->nb_rx_queues + - eth_dev->data->nb_tx_queues)) { + if (enic->conf_cq_count < required_cq) { dev_err(dev, "Not enough Completion queues. Required:%u, Configured:%u\n", - (eth_dev->data->nb_rx_queues + - eth_dev->data->nb_tx_queues), enic->conf_cq_count); + required_cq, enic->conf_cq_count); rc = -EINVAL; } @@ -1069,6 +1119,57 @@ int enic_set_vnic_res(struct enic *enic) return rc; } +/* Initialize the completion queue for an RQ */ +static int +enic_reinit_rq(struct enic *enic, unsigned int rq_idx) +{ + struct vnic_rq *sop_rq, *data_rq; + unsigned int cq_idx; + int rc = 0; + + sop_rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)]; + data_rq = &enic->rq[enic_rte_rq_idx_to_data_idx(rq_idx)]; + cq_idx = rq_idx; + + vnic_cq_clean(&enic->cq[cq_idx]); + vnic_cq_init(&enic->cq[cq_idx], + 0 /* flow_control_enable */, + 1 /* color_enable */, + 0 /* cq_head */, + 0 /* cq_tail */, + 1 /* cq_tail_color */, + 0 /* interrupt_enable */, + 1 /* cq_entry_enable */, + 0 /* cq_message_enable */, + 0 /* interrupt offset */, + 0 /* cq_message_addr */); + + + vnic_rq_init_start(sop_rq, enic_cq_rq(enic, + enic_rte_rq_idx_to_sop_idx(rq_idx)), 0, + sop_rq->ring.desc_count - 1, 1, 0); + if (data_rq->in_use) { + vnic_rq_init_start(data_rq, + enic_cq_rq(enic, + enic_rte_rq_idx_to_data_idx(rq_idx)), 0, + data_rq->ring.desc_count - 1, 1, 0); + } + + rc = enic_alloc_rx_queue_mbufs(enic, sop_rq); + if (rc) + return rc; + + if (data_rq->in_use) { + rc = enic_alloc_rx_queue_mbufs(enic, data_rq); + if (rc) { + enic_rxmbuf_queue_release(enic, sop_rq); + return rc; + } + } + + return 0; +} + /* The Cisco NIC can send and receive packets up to a max packet size * determined by the NIC type and firmware. There is also an MTU * configured into the NIC via the CIMC/UCSM management interface @@ -1078,6 +1179,9 @@ int enic_set_vnic_res(struct enic *enic) */ int enic_set_mtu(struct enic *enic, uint16_t new_mtu) { + unsigned int rq_idx; + struct vnic_rq *rq; + int rc = 0; uint16_t old_mtu; /* previous setting */ uint16_t config_mtu; /* Value configured into NIC via CIMC/UCSM */ struct rte_eth_dev *eth_dev = enic->rte_dev; @@ -1085,9 +1189,8 @@ int enic_set_mtu(struct enic *enic, uint16_t new_mtu) old_mtu = eth_dev->data->mtu; config_mtu = enic->config.mtu; - /* only works with Rx scatter disabled */ - if (enic->rte_dev->data->dev_conf.rxmode.enable_scatter) - return -ENOTSUP; + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return -E_RTE_SECONDARY; if (new_mtu > enic->max_mtu) { dev_err(enic, @@ -1106,11 +1209,83 @@ int enic_set_mtu(struct enic *enic, uint16_t new_mtu) "MTU (%u) is greater than value configured in NIC (%u)\n", new_mtu, config_mtu); + /* The easy case is when scatter is disabled. However if the MTU + * becomes greater than the mbuf data size, packet drops will ensue. + */ + if (!enic->rte_dev->data->dev_conf.rxmode.enable_scatter) { + eth_dev->data->mtu = new_mtu; + goto set_mtu_done; + } + + /* Rx scatter is enabled so reconfigure RQ's on the fly. The point is to + * change Rx scatter mode if necessary for better performance. I.e. if + * MTU was greater than the mbuf size and now it's less, scatter Rx + * doesn't have to be used and vice versa. + */ + rte_spinlock_lock(&enic->mtu_lock); + + /* Stop traffic on all RQs */ + for (rq_idx = 0; rq_idx < enic->rq_count * 2; rq_idx++) { + rq = &enic->rq[rq_idx]; + if (rq->is_sop && rq->in_use) { + rc = enic_stop_rq(enic, + enic_sop_rq_idx_to_rte_idx(rq_idx)); + if (rc) { + dev_err(enic, "Failed to stop Rq %u\n", rq_idx); + goto set_mtu_done; + } + } + } + + /* replace Rx function with a no-op to avoid getting stale pkts */ + eth_dev->rx_pkt_burst = enic_dummy_recv_pkts; + rte_mb(); + + /* Allow time for threads to exit the real Rx function. */ + usleep(100000); + + /* now it is safe to reconfigure the RQs */ + /* update the mtu */ eth_dev->data->mtu = new_mtu; + /* free and reallocate RQs with the new MTU */ + for (rq_idx = 0; rq_idx < enic->rq_count; rq_idx++) { + rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)]; + + enic_free_rq(rq); + rc = enic_alloc_rq(enic, rq_idx, rq->socket_id, rq->mp, + rq->tot_nb_desc, rq->rx_free_thresh); + if (rc) { + dev_err(enic, + "Fatal MTU alloc error- No traffic will pass\n"); + goto set_mtu_done; + } + + rc = enic_reinit_rq(enic, rq_idx); + if (rc) { + dev_err(enic, + "Fatal MTU RQ reinit- No traffic will pass\n"); + goto set_mtu_done; + } + } + + /* put back the real receive function */ + rte_mb(); + eth_dev->rx_pkt_burst = enic_recv_pkts; + rte_mb(); + + /* restart Rx traffic */ + for (rq_idx = 0; rq_idx < enic->rq_count; rq_idx++) { + rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)]; + if (rq->is_sop && rq->in_use) + enic_start_rq(enic, rq_idx); + } + +set_mtu_done: dev_info(enic, "MTU changed from %u to %u\n", old_mtu, new_mtu); - return 0; + rte_spinlock_unlock(&enic->mtu_lock); + return rc; } static int enic_dev_init(struct enic *enic) @@ -1136,17 +1311,46 @@ static int enic_dev_init(struct enic *enic) dev_err(enic, "See the ENIC PMD guide for more information.\n"); return -EINVAL; } + /* Queue counts may be zeros. rte_zmalloc returns NULL in that case. */ + enic->cq = rte_zmalloc("enic_vnic_cq", sizeof(struct vnic_cq) * + enic->conf_cq_count, 8); + enic->rq = rte_zmalloc("enic_vnic_rq", sizeof(struct vnic_rq) * + enic->conf_rq_count, 8); + enic->wq = rte_zmalloc("enic_vnic_wq", sizeof(struct vnic_wq) * + enic->conf_wq_count, 8); + if (enic->conf_cq_count > 0 && enic->cq == NULL) { + dev_err(enic, "failed to allocate vnic_cq, aborting.\n"); + return -1; + } + if (enic->conf_rq_count > 0 && enic->rq == NULL) { + dev_err(enic, "failed to allocate vnic_rq, aborting.\n"); + return -1; + } + if (enic->conf_wq_count > 0 && enic->wq == NULL) { + dev_err(enic, "failed to allocate vnic_wq, aborting.\n"); + return -1; + } - eth_dev->data->mac_addrs = rte_zmalloc("enic_mac_addr", ETH_ALEN, 0); + /* Get the supported filters */ + enic_fdir_info(enic); + + eth_dev->data->mac_addrs = rte_zmalloc("enic_mac_addr", ETH_ALEN + * ENIC_MAX_MAC_ADDR, 0); if (!eth_dev->data->mac_addrs) { dev_err(enic, "mac addr storage alloc failed, aborting.\n"); return -1; } ether_addr_copy((struct ether_addr *) enic->mac_addr, - ð_dev->data->mac_addrs[0]); + eth_dev->data->mac_addrs); vnic_dev_set_reset_flag(enic->vdev, 0); + LIST_INIT(&enic->flows); + rte_spinlock_init(&enic->flows_lock); + + /* set up link status checking */ + vnic_dev_notify_set(enic->vdev, -1); /* No Intr for notify */ + return 0; } @@ -1158,6 +1362,10 @@ int enic_probe(struct enic *enic) dev_debug(enic, " Initializing ENIC PMD\n"); + /* if this is a secondary process the hardware is already initialized */ + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return 0; + enic->bar0.vaddr = (void *)pdev->mem_resource[0].addr; enic->bar0.len = pdev->mem_resource[0].len; diff --git a/dpdk/drivers/net/enic/enic_res.c b/dpdk/drivers/net/enic/enic_res.c index 84c5d336..e4b80d49 100644 --- a/dpdk/drivers/net/enic/enic_res.c +++ b/dpdk/drivers/net/enic/enic_res.c @@ -62,6 +62,7 @@ int enic_get_vnic_config(struct enic *enic) return err; } + #define GET_CONFIG(m) \ do { \ err = vnic_dev_spec(enic->vdev, \ @@ -88,16 +89,36 @@ int enic_get_vnic_config(struct enic *enic) /* max packet size is only defined in newer VIC firmware * and will be 0 for legacy firmware and VICs */ - if (c->max_pkt_size > ENIC_DEFAULT_MAX_PKT_SIZE) + if (c->max_pkt_size > ENIC_DEFAULT_RX_MAX_PKT_SIZE) enic->max_mtu = c->max_pkt_size - (ETHER_HDR_LEN + 4); else - enic->max_mtu = ENIC_DEFAULT_MAX_PKT_SIZE - (ETHER_HDR_LEN + 4); + enic->max_mtu = ENIC_DEFAULT_RX_MAX_PKT_SIZE + - (ETHER_HDR_LEN + 4); if (c->mtu == 0) c->mtu = 1500; enic->rte_dev->data->mtu = min_t(u16, enic->max_mtu, max_t(u16, ENIC_MIN_MTU, c->mtu)); + enic->adv_filters = vnic_dev_capable_adv_filters(enic->vdev); + dev_info(enic, "Advanced Filters %savailable\n", ((enic->adv_filters) + ? "" : "not ")); + + err = vnic_dev_capable_filter_mode(enic->vdev, &enic->flow_filter_mode, + &enic->filter_tags); + if (err) { + dev_err(enic_get_dev(enic), + "Error getting filter modes, %d\n", err); + return err; + } + + dev_info(enic, "Flow api filter mode: %s, Filter tagging %savailable\n", + ((enic->flow_filter_mode == FILTER_DPDK_1) ? "DPDK" : + ((enic->flow_filter_mode == FILTER_USNIC_IP) ? "USNIC" : + ((enic->flow_filter_mode == FILTER_IPV4_5TUPLE) ? "5TUPLE" : + "NONE"))), + ((enic->filter_tags) ? "" : "not ")); + c->wq_desc_count = min_t(u32, ENIC_MAX_WQ_DESCS, max_t(u32, ENIC_MIN_WQ_DESCS, diff --git a/dpdk/drivers/net/enic/enic_res.h b/dpdk/drivers/net/enic/enic_res.h index 303530ef..1135d2e1 100644 --- a/dpdk/drivers/net/enic/enic_res.h +++ b/dpdk/drivers/net/enic/enic_res.h @@ -48,7 +48,10 @@ #define ENIC_MIN_MTU 68 /* Does not include (possible) inserted VLAN tag and FCS */ -#define ENIC_DEFAULT_MAX_PKT_SIZE 9022 +#define ENIC_DEFAULT_RX_MAX_PKT_SIZE 9022 + +/* Does not include (possible) inserted VLAN tag and FCS */ +#define ENIC_TX_MAX_PKT_SIZE 9208 #define ENIC_MULTICAST_PERFECT_FILTERS 32 #define ENIC_UNICAST_PERFECT_FILTERS 32 diff --git a/dpdk/drivers/net/enic/enic_rxtx.c b/dpdk/drivers/net/enic/enic_rxtx.c index ad596136..831c90a1 100644 --- a/dpdk/drivers/net/enic/enic_rxtx.c +++ b/dpdk/drivers/net/enic/enic_rxtx.c @@ -37,6 +37,9 @@ #include "enic_compat.h" #include "rq_enet_desc.h" #include "enic.h" +#include +#include +#include #define RTE_PMD_USE_PREFETCH @@ -129,6 +132,60 @@ enic_cq_rx_desc_n_bytes(struct cq_desc *cqd) CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK; } +/* Find the offset to L5. This is needed by enic TSO implementation. + * Return 0 if not a TCP packet or can't figure out the length. + */ +static inline uint8_t tso_header_len(struct rte_mbuf *mbuf) +{ + struct ether_hdr *eh; + struct vlan_hdr *vh; + struct ipv4_hdr *ip4; + struct ipv6_hdr *ip6; + struct tcp_hdr *th; + uint8_t hdr_len; + uint16_t ether_type; + + /* offset past Ethernet header */ + eh = rte_pktmbuf_mtod(mbuf, struct ether_hdr *); + ether_type = eh->ether_type; + hdr_len = sizeof(struct ether_hdr); + if (ether_type == rte_cpu_to_be_16(ETHER_TYPE_VLAN)) { + vh = rte_pktmbuf_mtod_offset(mbuf, struct vlan_hdr *, hdr_len); + ether_type = vh->eth_proto; + hdr_len += sizeof(struct vlan_hdr); + } + + /* offset past IP header */ + switch (rte_be_to_cpu_16(ether_type)) { + case ETHER_TYPE_IPv4: + ip4 = rte_pktmbuf_mtod_offset(mbuf, struct ipv4_hdr *, hdr_len); + if (ip4->next_proto_id != IPPROTO_TCP) + return 0; + hdr_len += (ip4->version_ihl & 0xf) * 4; + break; + case ETHER_TYPE_IPv6: + ip6 = rte_pktmbuf_mtod_offset(mbuf, struct ipv6_hdr *, hdr_len); + if (ip6->proto != IPPROTO_TCP) + return 0; + hdr_len += sizeof(struct ipv6_hdr); + break; + default: + return 0; + } + + if ((hdr_len + sizeof(struct tcp_hdr)) > mbuf->pkt_len) + return 0; + + /* offset past TCP header */ + th = rte_pktmbuf_mtod_offset(mbuf, struct tcp_hdr *, hdr_len); + hdr_len += (th->data_off >> 4) * 4; + + if (hdr_len > mbuf->pkt_len) + return 0; + + return hdr_len; +} + static inline uint8_t enic_cq_rx_check_err(struct cq_desc *cqd) { @@ -149,30 +206,18 @@ enic_cq_rx_flags_to_pkt_type(struct cq_desc *cqd) uint8_t cqrd_flags = cqrd->flags; static const uint32_t cq_type_table[128] __rte_cache_aligned = { [0x00] = RTE_PTYPE_UNKNOWN, - [0x20] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN - | RTE_PTYPE_L4_NONFRAG, - [0x22] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN - | RTE_PTYPE_L4_UDP, - [0x24] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN - | RTE_PTYPE_L4_TCP, - [0x60] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN - | RTE_PTYPE_L4_FRAG, - [0x62] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN - | RTE_PTYPE_L4_UDP, - [0x64] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN - | RTE_PTYPE_L4_TCP, - [0x10] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN - | RTE_PTYPE_L4_NONFRAG, - [0x12] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN - | RTE_PTYPE_L4_UDP, - [0x14] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN - | RTE_PTYPE_L4_TCP, - [0x50] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN - | RTE_PTYPE_L4_FRAG, - [0x52] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN - | RTE_PTYPE_L4_UDP, - [0x54] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN - | RTE_PTYPE_L4_TCP, + [0x20] = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_NONFRAG, + [0x22] = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP, + [0x24] = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_TCP, + [0x60] = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_FRAG, + [0x62] = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP, + [0x64] = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_TCP, + [0x10] = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_NONFRAG, + [0x12] = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_UDP, + [0x14] = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_TCP, + [0x50] = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_FRAG, + [0x52] = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_UDP, + [0x54] = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_TCP, /* All others reserved */ }; cqrd_flags &= CQ_ENET_RQ_DESC_FLAGS_IPV4_FRAGMENT @@ -185,9 +230,10 @@ static inline void enic_cq_rx_to_pkt_flags(struct cq_desc *cqd, struct rte_mbuf *mbuf) { struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; - uint16_t ciflags, bwflags, pkt_flags = 0; + uint16_t ciflags, bwflags, pkt_flags = 0, vlan_tci; ciflags = enic_cq_rx_desc_ciflags(cqrd); bwflags = enic_cq_rx_desc_bwflags(cqrd); + vlan_tci = enic_cq_rx_desc_vlan(cqrd); mbuf->ol_flags = 0; @@ -195,31 +241,57 @@ enic_cq_rx_to_pkt_flags(struct cq_desc *cqd, struct rte_mbuf *mbuf) if (unlikely(!enic_cq_rx_desc_eop(ciflags))) goto mbuf_flags_done; - /* VLAN stripping */ + /* VLAN STRIPPED flag. The L2 packet type updated here also */ if (bwflags & CQ_ENET_RQ_DESC_FLAGS_VLAN_STRIPPED) { - pkt_flags |= PKT_RX_VLAN_PKT | PKT_RX_VLAN_STRIPPED; - mbuf->vlan_tci = enic_cq_rx_desc_vlan(cqrd); + pkt_flags |= PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED; + mbuf->packet_type |= RTE_PTYPE_L2_ETHER; } else { - mbuf->vlan_tci = 0; + if (vlan_tci != 0) + mbuf->packet_type |= RTE_PTYPE_L2_ETHER_VLAN; + else + mbuf->packet_type |= RTE_PTYPE_L2_ETHER; } + mbuf->vlan_tci = vlan_tci; - /* RSS flag */ - if (enic_cq_rx_desc_rss_type(cqrd)) { + if ((cqd->type_color & CQ_DESC_TYPE_MASK) == CQ_DESC_TYPE_CLASSIFIER) { + struct cq_enet_rq_clsf_desc *clsf_cqd; + uint16_t filter_id; + clsf_cqd = (struct cq_enet_rq_clsf_desc *)cqd; + filter_id = clsf_cqd->filter_id; + if (filter_id) { + pkt_flags |= PKT_RX_FDIR; + if (filter_id != ENIC_MAGIC_FILTER_ID) { + mbuf->hash.fdir.hi = clsf_cqd->filter_id; + pkt_flags |= PKT_RX_FDIR_ID; + } + } + } else if (enic_cq_rx_desc_rss_type(cqrd)) { + /* RSS flag */ pkt_flags |= PKT_RX_RSS_HASH; mbuf->hash.rss = enic_cq_rx_desc_rss_hash(cqrd); } /* checksum flags */ - if (!enic_cq_rx_desc_csum_not_calc(cqrd) && - (mbuf->packet_type & RTE_PTYPE_L3_IPV4)) { - uint32_t l4_flags = mbuf->packet_type & RTE_PTYPE_L4_MASK; + if (mbuf->packet_type & RTE_PTYPE_L3_IPV4) { + if (enic_cq_rx_desc_csum_not_calc(cqrd)) + pkt_flags |= (PKT_RX_IP_CKSUM_UNKNOWN & + PKT_RX_L4_CKSUM_UNKNOWN); + else { + uint32_t l4_flags; + l4_flags = mbuf->packet_type & RTE_PTYPE_L4_MASK; - if (unlikely(!enic_cq_rx_desc_ipv4_csum_ok(cqrd))) - pkt_flags |= PKT_RX_IP_CKSUM_BAD; - if (l4_flags == RTE_PTYPE_L4_UDP || - l4_flags == RTE_PTYPE_L4_TCP) { - if (unlikely(!enic_cq_rx_desc_tcp_udp_csum_ok(cqrd))) - pkt_flags |= PKT_RX_L4_CKSUM_BAD; + if (enic_cq_rx_desc_ipv4_csum_ok(cqrd)) + pkt_flags |= PKT_RX_IP_CKSUM_GOOD; + else + pkt_flags |= PKT_RX_IP_CKSUM_BAD; + + if (l4_flags == RTE_PTYPE_L4_UDP || + l4_flags == RTE_PTYPE_L4_TCP) { + if (enic_cq_rx_desc_tcp_udp_csum_ok(cqrd)) + pkt_flags |= PKT_RX_L4_CKSUM_GOOD; + else + pkt_flags |= PKT_RX_L4_CKSUM_BAD; + } } } @@ -227,6 +299,17 @@ enic_cq_rx_to_pkt_flags(struct cq_desc *cqd, struct rte_mbuf *mbuf) mbuf->ol_flags = pkt_flags; } +/* dummy receive function to replace actual function in + * order to do safe reconfiguration operations. + */ +uint16_t +enic_dummy_recv_pkts(__rte_unused void *rx_queue, + __rte_unused struct rte_mbuf **rx_pkts, + __rte_unused uint16_t nb_pkts) +{ + return 0; +} + uint16_t enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) @@ -304,7 +387,7 @@ enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, /* Push descriptor for newly allocated mbuf */ nmb->data_off = RTE_PKTMBUF_HEADROOM; - dma_addr = (dma_addr_t)(nmb->buf_physaddr + + dma_addr = (dma_addr_t)(nmb->buf_iova + RTE_PKTMBUF_HEADROOM); rq_enet_desc_enc(rqd_ptr, dma_addr, (rq->is_sop ? RQ_ENET_TYPE_ONLY_SOP @@ -316,7 +399,6 @@ enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, if (rq->is_sop) { first_seg = rxmb; - first_seg->nb_segs = 1; first_seg->pkt_len = seg_length; } else { first_seg->pkt_len = (uint16_t)(first_seg->pkt_len @@ -325,7 +407,6 @@ enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, last_seg->next = rxmb; } - rxmb->next = NULL; rxmb->port = enic->port_id; rxmb->data_len = seg_length; @@ -376,10 +457,11 @@ enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, rte_mb(); if (data_rq->in_use) - iowrite32(data_rq->posted_index, - &data_rq->ctrl->posted_index); + iowrite32_relaxed(data_rq->posted_index, + &data_rq->ctrl->posted_index); rte_compiler_barrier(); - iowrite32(sop_rq->posted_index, &sop_rq->ctrl->posted_index); + iowrite32_relaxed(sop_rq->posted_index, + &sop_rq->ctrl->posted_index); } @@ -402,7 +484,7 @@ static inline void enic_free_wq_bufs(struct vnic_wq *wq, u16 completed_index) pool = ((struct rte_mbuf *)buf->mb)->pool; for (i = 0; i < nb_to_free; i++) { buf = &wq->bufs[tail_idx]; - m = __rte_pktmbuf_prefree_seg((struct rte_mbuf *)(buf->mb)); + m = rte_pktmbuf_prefree_seg((struct rte_mbuf *)(buf->mb)); buf->mb = NULL; if (unlikely(m == NULL)) { @@ -422,7 +504,8 @@ static inline void enic_free_wq_bufs(struct vnic_wq *wq, u16 completed_index) tail_idx = enic_ring_incr(desc_count, tail_idx); } - rte_mempool_put_bulk(pool, (void **)free, nb_free); + if (nb_free > 0) + rte_mempool_put_bulk(pool, (void **)free, nb_free); wq->tail_idx = tail_idx; wq->ring.desc_avail += nb_to_free; @@ -462,44 +545,70 @@ uint16_t enic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint8_t vlan_tag_insert; uint8_t eop; uint64_t bus_addr; + uint8_t offload_mode; + uint16_t header_len; + uint64_t tso; + rte_atomic64_t *tx_oversized; enic_cleanup_wq(enic, wq); wq_desc_avail = vnic_wq_desc_avail(wq); head_idx = wq->head_idx; desc_count = wq->ring.desc_count; ol_flags_mask = PKT_TX_VLAN_PKT | PKT_TX_IP_CKSUM | PKT_TX_L4_MASK; + tx_oversized = &enic->soft_stats.tx_oversized; nb_pkts = RTE_MIN(nb_pkts, ENIC_TX_XMIT_MAX); for (index = 0; index < nb_pkts; index++) { tx_pkt = *tx_pkts++; + pkt_len = tx_pkt->pkt_len; + data_len = tx_pkt->data_len; + ol_flags = tx_pkt->ol_flags; nb_segs = tx_pkt->nb_segs; + tso = ol_flags & PKT_TX_TCP_SEG; + + /* drop packet if it's too big to send */ + if (unlikely(!tso && pkt_len > ENIC_TX_MAX_PKT_SIZE)) { + rte_pktmbuf_free(tx_pkt); + rte_atomic64_inc(tx_oversized); + continue; + } + if (nb_segs > wq_desc_avail) { if (index > 0) goto post; goto done; } - pkt_len = tx_pkt->pkt_len; - data_len = tx_pkt->data_len; - ol_flags = tx_pkt->ol_flags; mss = 0; vlan_id = 0; vlan_tag_insert = 0; bus_addr = (dma_addr_t) - (tx_pkt->buf_physaddr + tx_pkt->data_off); + (tx_pkt->buf_iova + tx_pkt->data_off); descs = (struct wq_enet_desc *)wq->ring.descs; desc_p = descs + head_idx; eop = (data_len == pkt_len); + offload_mode = WQ_ENET_OFFLOAD_MODE_CSUM; + header_len = 0; - if (ol_flags & ol_flags_mask) { - if (ol_flags & PKT_TX_VLAN_PKT) { - vlan_tag_insert = 1; - vlan_id = tx_pkt->vlan_tci; + if (tso) { + header_len = tso_header_len(tx_pkt); + + /* Drop if non-TCP packet or TSO seg size is too big */ + if (unlikely(header_len == 0 || ((tx_pkt->tso_segsz + + header_len) > ENIC_TX_MAX_PKT_SIZE))) { + rte_pktmbuf_free(tx_pkt); + rte_atomic64_inc(tx_oversized); + continue; } + offload_mode = WQ_ENET_OFFLOAD_MODE_TSO; + mss = tx_pkt->tso_segsz; + } + + if ((ol_flags & ol_flags_mask) && (header_len == 0)) { if (ol_flags & PKT_TX_IP_CKSUM) mss |= ENIC_CALC_IP_CKSUM; @@ -512,8 +621,14 @@ uint16_t enic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, } } - wq_enet_desc_enc(&desc_tmp, bus_addr, data_len, mss, 0, 0, eop, - eop, 0, vlan_tag_insert, vlan_id, 0); + if (ol_flags & PKT_TX_VLAN_PKT) { + vlan_tag_insert = 1; + vlan_id = tx_pkt->vlan_tci; + } + + wq_enet_desc_enc(&desc_tmp, bus_addr, data_len, mss, header_len, + offload_mode, eop, eop, 0, vlan_tag_insert, + vlan_id, 0); *desc_p = desc_tmp; buf = &wq->bufs[head_idx]; @@ -529,12 +644,13 @@ uint16_t enic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, if (tx_pkt->next == NULL) eop = 1; desc_p = descs + head_idx; - bus_addr = (dma_addr_t)(tx_pkt->buf_physaddr + bus_addr = (dma_addr_t)(tx_pkt->buf_iova + tx_pkt->data_off); wq_enet_desc_enc((struct wq_enet_desc *) &desc_tmp, bus_addr, data_len, - mss, 0, 0, eop, eop, 0, - vlan_tag_insert, vlan_id, 0); + mss, 0, offload_mode, eop, eop, + 0, vlan_tag_insert, vlan_id, + 0); *desc_p = desc_tmp; buf = &wq->bufs[head_idx]; @@ -546,7 +662,7 @@ uint16_t enic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, } post: rte_wmb(); - iowrite32(head_idx, &wq->ctrl->posted_index); + iowrite32_relaxed(head_idx, &wq->ctrl->posted_index); done: wq->ring.desc_avail = wq_desc_avail; wq->head_idx = head_idx; diff --git a/dpdk/drivers/net/failsafe/Makefile b/dpdk/drivers/net/failsafe/Makefile new file mode 100644 index 00000000..ea2a8fe4 --- /dev/null +++ b/dpdk/drivers/net/failsafe/Makefile @@ -0,0 +1,65 @@ +# BSD LICENSE +# +# Copyright 2017 6WIND S.A. +# Copyright 2017 Mellanox. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of 6WIND S.A. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +include $(RTE_SDK)/mk/rte.vars.mk + +# Library name +LIB = librte_pmd_failsafe.a + +EXPORT_MAP := rte_pmd_failsafe_version.map + +LIBABIVER := 1 + +# Sources are stored in SRCS-y +SRCS-$(CONFIG_RTE_LIBRTE_PMD_FAILSAFE) += failsafe.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_FAILSAFE) += failsafe_args.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_FAILSAFE) += failsafe_eal.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_FAILSAFE) += failsafe_ops.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_FAILSAFE) += failsafe_rxtx.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_FAILSAFE) += failsafe_ether.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_FAILSAFE) += failsafe_flow.c + +# No exported include files + +# Basic CFLAGS: +CFLAGS += -std=gnu99 -Wextra +CFLAGS += -O3 +CFLAGS += -I. +CFLAGS += -D_DEFAULT_SOURCE +CFLAGS += -D_XOPEN_SOURCE=700 +CFLAGS += $(WERROR_FLAGS) +CFLAGS += -Wno-strict-prototypes +CFLAGS += -pedantic +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs +LDLIBS += -lrte_bus_vdev + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/net/failsafe/failsafe.c b/dpdk/drivers/net/failsafe/failsafe.c new file mode 100644 index 00000000..8336510f --- /dev/null +++ b/dpdk/drivers/net/failsafe/failsafe.c @@ -0,0 +1,299 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of 6WIND S.A. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "failsafe_private.h" + +const char pmd_failsafe_driver_name[] = FAILSAFE_DRIVER_NAME; +static const struct rte_eth_link eth_link = { + .link_speed = ETH_SPEED_NUM_10G, + .link_duplex = ETH_LINK_FULL_DUPLEX, + .link_status = ETH_LINK_UP, + .link_autoneg = ETH_LINK_AUTONEG, +}; + +static int +fs_sub_device_alloc(struct rte_eth_dev *dev, + const char *params) +{ + uint8_t nb_subs; + int ret; + + ret = failsafe_args_count_subdevice(dev, params); + if (ret) + return ret; + if (PRIV(dev)->subs_tail > FAILSAFE_MAX_ETHPORTS) { + ERROR("Cannot allocate more than %d ports", + FAILSAFE_MAX_ETHPORTS); + return -ENOSPC; + } + nb_subs = PRIV(dev)->subs_tail; + PRIV(dev)->subs = rte_zmalloc(NULL, + sizeof(struct sub_device) * nb_subs, + RTE_CACHE_LINE_SIZE); + if (PRIV(dev)->subs == NULL) { + ERROR("Could not allocate sub_devices"); + return -ENOMEM; + } + return 0; +} + +static void +fs_sub_device_free(struct rte_eth_dev *dev) +{ + rte_free(PRIV(dev)->subs); +} + +static void fs_hotplug_alarm(void *arg); + +int +failsafe_hotplug_alarm_install(struct rte_eth_dev *dev) +{ + int ret; + + if (dev == NULL) + return -EINVAL; + if (PRIV(dev)->pending_alarm) + return 0; + ret = rte_eal_alarm_set(hotplug_poll * 1000, + fs_hotplug_alarm, + dev); + if (ret) { + ERROR("Could not set up plug-in event detection"); + return ret; + } + PRIV(dev)->pending_alarm = 1; + return 0; +} + +int +failsafe_hotplug_alarm_cancel(struct rte_eth_dev *dev) +{ + int ret = 0; + + if (PRIV(dev)->pending_alarm) { + rte_errno = 0; + rte_eal_alarm_cancel(fs_hotplug_alarm, dev); + if (rte_errno) { + ERROR("rte_eal_alarm_cancel failed (errno: %s)", + strerror(rte_errno)); + ret = -rte_errno; + } else { + PRIV(dev)->pending_alarm = 0; + } + } + return ret; +} + +static void +fs_hotplug_alarm(void *arg) +{ + struct rte_eth_dev *dev = arg; + struct sub_device *sdev; + int ret; + uint8_t i; + + if (!PRIV(dev)->pending_alarm) + return; + PRIV(dev)->pending_alarm = 0; + FOREACH_SUBDEV(sdev, i, dev) + if (sdev->state != PRIV(dev)->state) + break; + /* if we have non-probed device */ + if (i != PRIV(dev)->subs_tail) { + ret = failsafe_eth_dev_state_sync(dev); + if (ret) + ERROR("Unable to synchronize sub_device state"); + } + failsafe_dev_remove(dev); + ret = failsafe_hotplug_alarm_install(dev); + if (ret) + ERROR("Unable to set up next alarm"); +} + +static int +fs_eth_dev_create(struct rte_vdev_device *vdev) +{ + struct rte_eth_dev *dev; + struct ether_addr *mac; + struct fs_priv *priv; + struct sub_device *sdev; + const char *params; + unsigned int socket_id; + uint8_t i; + int ret; + + dev = NULL; + priv = NULL; + socket_id = rte_socket_id(); + INFO("Creating fail-safe device on NUMA socket %u", socket_id); + params = rte_vdev_device_args(vdev); + if (params == NULL) { + ERROR("This PMD requires sub-devices, none provided"); + return -1; + } + dev = rte_eth_vdev_allocate(vdev, sizeof(*priv)); + if (dev == NULL) { + ERROR("Unable to allocate rte_eth_dev"); + return -1; + } + priv = PRIV(dev); + priv->dev = dev; + dev->dev_ops = &failsafe_ops; + dev->data->mac_addrs = &PRIV(dev)->mac_addrs[0]; + dev->data->dev_link = eth_link; + PRIV(dev)->nb_mac_addr = 1; + TAILQ_INIT(&PRIV(dev)->flow_list); + dev->rx_pkt_burst = (eth_rx_burst_t)&failsafe_rx_burst; + dev->tx_pkt_burst = (eth_tx_burst_t)&failsafe_tx_burst; + ret = fs_sub_device_alloc(dev, params); + if (ret) { + ERROR("Could not allocate sub_devices"); + goto free_dev; + } + ret = failsafe_args_parse(dev, params); + if (ret) + goto free_subs; + ret = failsafe_eal_init(dev); + if (ret) + goto free_args; + ret = failsafe_hotplug_alarm_install(dev); + if (ret) { + ERROR("Could not set up plug-in event detection"); + goto free_args; + } + mac = &dev->data->mac_addrs[0]; + if (mac_from_arg) { + /* + * If MAC address was provided as a parameter, + * apply to all probed slaves. + */ + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_PROBED) { + ret = rte_eth_dev_default_mac_addr_set(PORT_ID(sdev), + mac); + if (ret) { + ERROR("Failed to set default MAC address"); + goto free_args; + } + } + } else { + /* + * Use the ether_addr from first probed + * device, either preferred or fallback. + */ + FOREACH_SUBDEV(sdev, i, dev) + if (sdev->state >= DEV_PROBED) { + ether_addr_copy(Ð(sdev)->data->mac_addrs[0], + mac); + break; + } + /* + * If no device has been probed and no ether_addr + * has been provided on the command line, use a random + * valid one. + * It will be applied during future slave state syncs to + * probed slaves. + */ + if (i == priv->subs_tail) + eth_random_addr(&mac->addr_bytes[0]); + } + INFO("MAC address is %02x:%02x:%02x:%02x:%02x:%02x", + mac->addr_bytes[0], mac->addr_bytes[1], + mac->addr_bytes[2], mac->addr_bytes[3], + mac->addr_bytes[4], mac->addr_bytes[5]); + dev->data->dev_flags |= RTE_ETH_DEV_INTR_LSC; + return 0; +free_args: + failsafe_args_free(dev); +free_subs: + fs_sub_device_free(dev); +free_dev: + rte_free(PRIV(dev)); + rte_eth_dev_release_port(dev); + return -1; +} + +static int +fs_rte_eth_free(const char *name) +{ + struct rte_eth_dev *dev; + int ret; + + dev = rte_eth_dev_allocated(name); + if (dev == NULL) + return -ENODEV; + ret = failsafe_eal_uninit(dev); + if (ret) + ERROR("Error while uninitializing sub-EAL"); + failsafe_args_free(dev); + fs_sub_device_free(dev); + rte_free(PRIV(dev)); + rte_eth_dev_release_port(dev); + return ret; +} + +static int +rte_pmd_failsafe_probe(struct rte_vdev_device *vdev) +{ + const char *name; + + name = rte_vdev_device_name(vdev); + INFO("Initializing " FAILSAFE_DRIVER_NAME " for %s", + name); + return fs_eth_dev_create(vdev); +} + +static int +rte_pmd_failsafe_remove(struct rte_vdev_device *vdev) +{ + const char *name; + + name = rte_vdev_device_name(vdev); + INFO("Uninitializing " FAILSAFE_DRIVER_NAME " for %s", name); + return fs_rte_eth_free(name); +} + +static struct rte_vdev_driver failsafe_drv = { + .probe = rte_pmd_failsafe_probe, + .remove = rte_pmd_failsafe_remove, +}; + +RTE_PMD_REGISTER_VDEV(net_failsafe, failsafe_drv); +RTE_PMD_REGISTER_PARAM_STRING(net_failsafe, PMD_FAILSAFE_PARAM_STRING); diff --git a/dpdk/drivers/net/failsafe/failsafe_args.c b/dpdk/drivers/net/failsafe/failsafe_args.c new file mode 100644 index 00000000..ec63ac97 --- /dev/null +++ b/dpdk/drivers/net/failsafe/failsafe_args.c @@ -0,0 +1,472 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of 6WIND S.A. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include + +#include +#include +#include +#include + +#include "failsafe_private.h" + +#define DEVARGS_MAXLEN 4096 + +/* Callback used when a new device is found in devargs */ +typedef int (parse_cb)(struct rte_eth_dev *dev, const char *params, + uint8_t head); + +uint64_t hotplug_poll = FAILSAFE_HOTPLUG_DEFAULT_TIMEOUT_MS; +int mac_from_arg = 0; + +const char *pmd_failsafe_init_parameters[] = { + PMD_FAILSAFE_HOTPLUG_POLL_KVARG, + PMD_FAILSAFE_MAC_KVARG, + NULL, +}; + +/* + * input: text. + * output: 0: if text[0] != '(', + * 0: if there are no corresponding ')' + * n: distance to corresponding ')' otherwise + */ +static size_t +closing_paren(const char *text) +{ + int nb_open = 0; + size_t i = 0; + + while (text[i] != '\0') { + if (text[i] == '(') + nb_open++; + if (text[i] == ')') + nb_open--; + if (nb_open == 0) + return i; + i++; + } + return 0; +} + +static int +fs_parse_device(struct sub_device *sdev, char *args) +{ + struct rte_devargs *d; + int ret; + + d = &sdev->devargs; + DEBUG("%s", args); + ret = rte_eal_devargs_parse(args, d); + if (ret) { + DEBUG("devargs parsing failed with code %d", ret); + return ret; + } + sdev->bus = d->bus; + sdev->state = DEV_PARSED; + return 0; +} + +static void +fs_sanitize_cmdline(char *args) +{ + char *nl; + + nl = strrchr(args, '\n'); + if (nl) + nl[0] = '\0'; +} + +static int +fs_execute_cmd(struct sub_device *sdev, char *cmdline) +{ + FILE *fp; + /* store possible newline as well */ + char output[DEVARGS_MAXLEN + 1]; + size_t len; + int ret; + + RTE_ASSERT(cmdline != NULL || sdev->cmdline != NULL); + if (sdev->cmdline == NULL) { + size_t i; + + len = strlen(cmdline) + 1; + sdev->cmdline = calloc(1, len); + if (sdev->cmdline == NULL) { + ERROR("Command line allocation failed"); + return -ENOMEM; + } + snprintf(sdev->cmdline, len, "%s", cmdline); + /* Replace all commas in the command line by spaces */ + for (i = 0; i < len; i++) + if (sdev->cmdline[i] == ',') + sdev->cmdline[i] = ' '; + } + DEBUG("'%s'", sdev->cmdline); + fp = popen(sdev->cmdline, "r"); + if (fp == NULL) { + ret = -errno; + ERROR("popen: %s", strerror(errno)); + return ret; + } + /* We only read one line */ + if (fgets(output, sizeof(output) - 1, fp) == NULL) { + DEBUG("Could not read command output"); + ret = -ENODEV; + goto ret_pclose; + } + fs_sanitize_cmdline(output); + if (output[0] == '\0') { + ret = -ENODEV; + goto ret_pclose; + } + ret = fs_parse_device(sdev, output); + if (ret) + ERROR("Parsing device '%s' failed", output); +ret_pclose: + if (pclose(fp) == -1) + ERROR("pclose: %s", strerror(errno)); + return ret; +} + +static int +fs_parse_device_param(struct rte_eth_dev *dev, const char *param, + uint8_t head) +{ + struct fs_priv *priv; + struct sub_device *sdev; + char *args = NULL; + size_t a, b; + int ret; + + priv = PRIV(dev); + a = 0; + b = 0; + ret = 0; + while (param[b] != '(' && + param[b] != '\0') + b++; + a = b; + b += closing_paren(¶m[b]); + if (a == b) { + ERROR("Dangling parenthesis"); + return -EINVAL; + } + a += 1; + args = strndup(¶m[a], b - a); + if (args == NULL) { + ERROR("Not enough memory for parameter parsing"); + return -ENOMEM; + } + sdev = &priv->subs[head]; + if (strncmp(param, "dev", 3) == 0) { + ret = fs_parse_device(sdev, args); + if (ret) + goto free_args; + } else if (strncmp(param, "exec", 4) == 0) { + ret = fs_execute_cmd(sdev, args); + if (ret == -ENODEV) { + DEBUG("Reading device info from command line failed"); + ret = 0; + } + if (ret) + goto free_args; + } else { + ERROR("Unrecognized device type: %.*s", (int)b, param); + return -EINVAL; + } +free_args: + free(args); + return ret; +} + +static int +fs_parse_sub_devices(parse_cb *cb, + struct rte_eth_dev *dev, const char *params) +{ + size_t a, b; + uint8_t head; + int ret; + + a = 0; + head = 0; + ret = 0; + while (params[a] != '\0') { + b = a; + while (params[b] != '(' && + params[b] != ',' && + params[b] != '\0') + b++; + if (b == a) { + ERROR("Invalid parameter"); + return -EINVAL; + } + if (params[b] == ',') { + a = b + 1; + continue; + } + if (params[b] == '(') { + size_t start = b; + + b += closing_paren(¶ms[b]); + if (b == start) { + ERROR("Dangling parenthesis"); + return -EINVAL; + } + ret = (*cb)(dev, ¶ms[a], head); + if (ret) + return ret; + head += 1; + b += 1; + if (params[b] == '\0') + return 0; + } + a = b + 1; + } + return 0; +} + +static int +fs_remove_sub_devices_definition(char params[DEVARGS_MAXLEN]) +{ + char buffer[DEVARGS_MAXLEN] = {0}; + size_t a, b; + int i; + + a = 0; + i = 0; + while (params[a] != '\0') { + b = a; + while (params[b] != '(' && + params[b] != ',' && + params[b] != '\0') + b++; + if (b == a) { + ERROR("Invalid parameter"); + return -EINVAL; + } + if (params[b] == ',' || params[b] == '\0') { + size_t len = b - a; + + if (i > 0) + len += 1; + snprintf(&buffer[i], len + 1, "%s%s", + i ? "," : "", ¶ms[a]); + i += len; + } else if (params[b] == '(') { + size_t start = b; + + b += closing_paren(¶ms[b]); + if (b == start) + return -EINVAL; + b += 1; + if (params[b] == '\0') + goto out; + } + a = b + 1; + } +out: + snprintf(params, DEVARGS_MAXLEN, "%s", buffer); + return 0; +} + +static int +fs_get_u64_arg(const char *key __rte_unused, + const char *value, void *out) +{ + uint64_t *u64 = out; + char *endptr = NULL; + + if ((value == NULL) || (out == NULL)) + return -EINVAL; + errno = 0; + *u64 = strtoull(value, &endptr, 0); + if (errno != 0) + return -errno; + if (endptr == value) + return -1; + return 0; +} + +static int +fs_get_mac_addr_arg(const char *key __rte_unused, + const char *value, void *out) +{ + struct ether_addr *ea = out; + int ret; + + if ((value == NULL) || (out == NULL)) + return -EINVAL; + ret = sscanf(value, "%hhx:%hhx:%hhx:%hhx:%hhx:%hhx", + &ea->addr_bytes[0], &ea->addr_bytes[1], + &ea->addr_bytes[2], &ea->addr_bytes[3], + &ea->addr_bytes[4], &ea->addr_bytes[5]); + return ret != ETHER_ADDR_LEN; +} + +int +failsafe_args_parse(struct rte_eth_dev *dev, const char *params) +{ + struct fs_priv *priv; + char mut_params[DEVARGS_MAXLEN] = ""; + struct rte_kvargs *kvlist = NULL; + unsigned int arg_count; + size_t n; + int ret; + + priv = PRIV(dev); + ret = 0; + priv->subs_tx = FAILSAFE_MAX_ETHPORTS; + /* default parameters */ + n = snprintf(mut_params, sizeof(mut_params), "%s", params); + if (n >= sizeof(mut_params)) { + ERROR("Parameter string too long (>=%zu)", + sizeof(mut_params)); + return -ENOMEM; + } + ret = fs_parse_sub_devices(fs_parse_device_param, + dev, params); + if (ret < 0) + return ret; + ret = fs_remove_sub_devices_definition(mut_params); + if (ret < 0) + return ret; + if (strnlen(mut_params, sizeof(mut_params)) > 0) { + kvlist = rte_kvargs_parse(mut_params, + pmd_failsafe_init_parameters); + if (kvlist == NULL) { + ERROR("Error parsing parameters, usage:\n" + PMD_FAILSAFE_PARAM_STRING); + return -1; + } + /* PLUG_IN event poll timer */ + arg_count = rte_kvargs_count(kvlist, + PMD_FAILSAFE_HOTPLUG_POLL_KVARG); + if (arg_count == 1) { + ret = rte_kvargs_process(kvlist, + PMD_FAILSAFE_HOTPLUG_POLL_KVARG, + &fs_get_u64_arg, &hotplug_poll); + if (ret < 0) + goto free_kvlist; + } + /* MAC addr */ + arg_count = rte_kvargs_count(kvlist, + PMD_FAILSAFE_MAC_KVARG); + if (arg_count > 0) { + ret = rte_kvargs_process(kvlist, + PMD_FAILSAFE_MAC_KVARG, + &fs_get_mac_addr_arg, + &dev->data->mac_addrs[0]); + if (ret < 0) + goto free_kvlist; + + mac_from_arg = 1; + } + } + PRIV(dev)->state = DEV_PARSED; +free_kvlist: + rte_kvargs_free(kvlist); + return ret; +} + +void +failsafe_args_free(struct rte_eth_dev *dev) +{ + struct sub_device *sdev; + uint8_t i; + + FOREACH_SUBDEV(sdev, i, dev) { + free(sdev->cmdline); + sdev->cmdline = NULL; + free(sdev->devargs.args); + sdev->devargs.args = NULL; + } +} + +static int +fs_count_device(struct rte_eth_dev *dev, const char *param, + uint8_t head __rte_unused) +{ + size_t b = 0; + + while (param[b] != '(' && + param[b] != '\0') + b++; + if (strncmp(param, "dev", b) != 0 && + strncmp(param, "exec", b) != 0) { + ERROR("Unrecognized device type: %.*s", (int)b, param); + return -EINVAL; + } + PRIV(dev)->subs_tail += 1; + return 0; +} + +int +failsafe_args_count_subdevice(struct rte_eth_dev *dev, + const char *params) +{ + return fs_parse_sub_devices(fs_count_device, + dev, params); +} + +static int +fs_parse_sub_device(struct sub_device *sdev) +{ + struct rte_devargs *da; + char devstr[DEVARGS_MAXLEN] = ""; + + da = &sdev->devargs; + snprintf(devstr, sizeof(devstr), "%s,%s", da->name, da->args); + return fs_parse_device(sdev, devstr); +} + +int +failsafe_args_parse_subs(struct rte_eth_dev *dev) +{ + struct sub_device *sdev; + uint8_t i; + int ret = 0; + + FOREACH_SUBDEV(sdev, i, dev) { + if (sdev->state >= DEV_PARSED) + continue; + if (sdev->cmdline) + ret = fs_execute_cmd(sdev, sdev->cmdline); + else + ret = fs_parse_sub_device(sdev); + if (ret == 0) + sdev->state = DEV_PARSED; + } + return 0; +} diff --git a/dpdk/drivers/net/failsafe/failsafe_eal.c b/dpdk/drivers/net/failsafe/failsafe_eal.c new file mode 100644 index 00000000..19d26f53 --- /dev/null +++ b/dpdk/drivers/net/failsafe/failsafe_eal.c @@ -0,0 +1,124 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of 6WIND S.A. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +#include "failsafe_private.h" + +static int +fs_bus_init(struct rte_eth_dev *dev) +{ + struct sub_device *sdev; + struct rte_devargs *da; + uint8_t i; + uint16_t j; + int ret; + + FOREACH_SUBDEV(sdev, i, dev) { + if (sdev->state != DEV_PARSED) + continue; + da = &sdev->devargs; + ret = rte_eal_hotplug_add(da->bus->name, + da->name, + da->args); + if (ret) { + ERROR("sub_device %d probe failed %s%s%s", i, + rte_errno ? "(" : "", + rte_errno ? strerror(rte_errno) : "", + rte_errno ? ")" : ""); + continue; + } + RTE_ETH_FOREACH_DEV(j) { + if (strcmp(rte_eth_devices[j].device->name, + da->name) == 0) { + ETH(sdev) = &rte_eth_devices[j]; + break; + } + } + if (ETH(sdev) == NULL) { + ERROR("sub_device %d init went wrong", i); + return -ENODEV; + } + SUB_ID(sdev) = i; + sdev->fs_dev = dev; + sdev->dev = ETH(sdev)->device; + ETH(sdev)->state = RTE_ETH_DEV_DEFERRED; + sdev->state = DEV_PROBED; + } + return 0; +} + +int +failsafe_eal_init(struct rte_eth_dev *dev) +{ + int ret; + + ret = fs_bus_init(dev); + if (ret) + return ret; + if (PRIV(dev)->state < DEV_PROBED) + PRIV(dev)->state = DEV_PROBED; + fs_switch_dev(dev, NULL); + return 0; +} + +static int +fs_bus_uninit(struct rte_eth_dev *dev) +{ + struct sub_device *sdev = NULL; + uint8_t i; + int sdev_ret; + int ret = 0; + + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_PROBED) { + sdev_ret = rte_eal_hotplug_remove(sdev->bus->name, + sdev->dev->name); + if (sdev_ret) { + ERROR("Failed to remove requested device %s (err: %d)", + sdev->dev->name, sdev_ret); + continue; + } + sdev->state = DEV_PROBED - 1; + } + return ret; +} + +int +failsafe_eal_uninit(struct rte_eth_dev *dev) +{ + int ret; + + ret = fs_bus_uninit(dev); + PRIV(dev)->state = DEV_PROBED - 1; + return ret; +} diff --git a/dpdk/drivers/net/failsafe/failsafe_ether.c b/dpdk/drivers/net/failsafe/failsafe_ether.c new file mode 100644 index 00000000..21392e5a --- /dev/null +++ b/dpdk/drivers/net/failsafe/failsafe_ether.c @@ -0,0 +1,486 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of 6WIND S.A. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +#include +#include +#include + +#include "failsafe_private.h" + +/** Print a message out of a flow error. */ +static int +fs_flow_complain(struct rte_flow_error *error) +{ + static const char *const errstrlist[] = { + [RTE_FLOW_ERROR_TYPE_NONE] = "no error", + [RTE_FLOW_ERROR_TYPE_UNSPECIFIED] = "cause unspecified", + [RTE_FLOW_ERROR_TYPE_HANDLE] = "flow rule (handle)", + [RTE_FLOW_ERROR_TYPE_ATTR_GROUP] = "group field", + [RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY] = "priority field", + [RTE_FLOW_ERROR_TYPE_ATTR_INGRESS] = "ingress field", + [RTE_FLOW_ERROR_TYPE_ATTR_EGRESS] = "egress field", + [RTE_FLOW_ERROR_TYPE_ATTR] = "attributes structure", + [RTE_FLOW_ERROR_TYPE_ITEM_NUM] = "pattern length", + [RTE_FLOW_ERROR_TYPE_ITEM] = "specific pattern item", + [RTE_FLOW_ERROR_TYPE_ACTION_NUM] = "number of actions", + [RTE_FLOW_ERROR_TYPE_ACTION] = "specific action", + }; + const char *errstr; + char buf[32]; + int err = rte_errno; + + if ((unsigned int)error->type >= RTE_DIM(errstrlist) || + !errstrlist[error->type]) + errstr = "unknown type"; + else + errstr = errstrlist[error->type]; + ERROR("Caught error type %d (%s): %s%s\n", + error->type, errstr, + error->cause ? (snprintf(buf, sizeof(buf), "cause: %p, ", + error->cause), buf) : "", + error->message ? error->message : "(no stated reason)"); + return -err; +} + +static int +eth_dev_flow_isolate_set(struct rte_eth_dev *dev, + struct sub_device *sdev) +{ + struct rte_flow_error ferror; + int ret; + + if (!PRIV(dev)->flow_isolated) { + DEBUG("Flow isolation already disabled"); + } else { + DEBUG("Enabling flow isolation"); + ret = rte_flow_isolate(PORT_ID(sdev), + PRIV(dev)->flow_isolated, + &ferror); + if (ret) { + fs_flow_complain(&ferror); + return ret; + } + } + return 0; +} + +static int +fs_eth_dev_conf_apply(struct rte_eth_dev *dev, + struct sub_device *sdev) +{ + struct rte_eth_dev *edev; + struct rte_vlan_filter_conf *vfc1; + struct rte_vlan_filter_conf *vfc2; + struct rte_flow *flow; + struct rte_flow_error ferror; + uint32_t i; + int ret; + + edev = ETH(sdev); + /* RX queue setup */ + for (i = 0; i < dev->data->nb_rx_queues; i++) { + struct rxq *rxq; + + rxq = dev->data->rx_queues[i]; + ret = rte_eth_rx_queue_setup(PORT_ID(sdev), i, + rxq->info.nb_desc, rxq->socket_id, + &rxq->info.conf, rxq->info.mp); + if (ret) { + ERROR("rx_queue_setup failed"); + return ret; + } + } + /* TX queue setup */ + for (i = 0; i < dev->data->nb_tx_queues; i++) { + struct txq *txq; + + txq = dev->data->tx_queues[i]; + ret = rte_eth_tx_queue_setup(PORT_ID(sdev), i, + txq->info.nb_desc, txq->socket_id, + &txq->info.conf); + if (ret) { + ERROR("tx_queue_setup failed"); + return ret; + } + } + /* dev_link.link_status */ + if (dev->data->dev_link.link_status != + edev->data->dev_link.link_status) { + DEBUG("Configuring link_status"); + if (dev->data->dev_link.link_status) + ret = rte_eth_dev_set_link_up(PORT_ID(sdev)); + else + ret = rte_eth_dev_set_link_down(PORT_ID(sdev)); + if (ret) { + ERROR("Failed to apply link_status"); + return ret; + } + } else { + DEBUG("link_status already set"); + } + /* promiscuous */ + if (dev->data->promiscuous != edev->data->promiscuous) { + DEBUG("Configuring promiscuous"); + if (dev->data->promiscuous) + rte_eth_promiscuous_enable(PORT_ID(sdev)); + else + rte_eth_promiscuous_disable(PORT_ID(sdev)); + } else { + DEBUG("promiscuous already set"); + } + /* all_multicast */ + if (dev->data->all_multicast != edev->data->all_multicast) { + DEBUG("Configuring all_multicast"); + if (dev->data->all_multicast) + rte_eth_allmulticast_enable(PORT_ID(sdev)); + else + rte_eth_allmulticast_disable(PORT_ID(sdev)); + } else { + DEBUG("all_multicast already set"); + } + /* MTU */ + if (dev->data->mtu != edev->data->mtu) { + DEBUG("Configuring MTU"); + ret = rte_eth_dev_set_mtu(PORT_ID(sdev), dev->data->mtu); + if (ret) { + ERROR("Failed to apply MTU"); + return ret; + } + } else { + DEBUG("MTU already set"); + } + /* default MAC */ + DEBUG("Configuring default MAC address"); + ret = rte_eth_dev_default_mac_addr_set(PORT_ID(sdev), + &dev->data->mac_addrs[0]); + if (ret) { + ERROR("Setting default MAC address failed"); + return ret; + } + /* additional MAC */ + if (PRIV(dev)->nb_mac_addr > 1) + DEBUG("Configure additional MAC address%s", + (PRIV(dev)->nb_mac_addr > 2 ? "es" : "")); + for (i = 1; i < PRIV(dev)->nb_mac_addr; i++) { + struct ether_addr *ea; + + ea = &dev->data->mac_addrs[i]; + ret = rte_eth_dev_mac_addr_add(PORT_ID(sdev), ea, + PRIV(dev)->mac_addr_pool[i]); + if (ret) { + char ea_fmt[ETHER_ADDR_FMT_SIZE]; + + ether_format_addr(ea_fmt, ETHER_ADDR_FMT_SIZE, ea); + ERROR("Adding MAC address %s failed", ea_fmt); + return ret; + } + } + /* VLAN filter */ + vfc1 = &dev->data->vlan_filter_conf; + vfc2 = &edev->data->vlan_filter_conf; + if (memcmp(vfc1, vfc2, sizeof(struct rte_vlan_filter_conf))) { + uint64_t vbit; + uint64_t ids; + size_t i; + uint16_t vlan_id; + + DEBUG("Configuring VLAN filter"); + for (i = 0; i < RTE_DIM(vfc1->ids); i++) { + if (vfc1->ids[i] == 0) + continue; + ids = vfc1->ids[i]; + while (ids) { + vlan_id = 64 * i; + /* count trailing zeroes */ + vbit = ~ids & (ids - 1); + /* clear least significant bit set */ + ids ^= (ids ^ (ids - 1)) ^ vbit; + for (; vbit; vlan_id++) + vbit >>= 1; + ret = rte_eth_dev_vlan_filter( + PORT_ID(sdev), vlan_id, 1); + if (ret) { + ERROR("Failed to apply VLAN filter %hu", + vlan_id); + return ret; + } + } + } + } else { + DEBUG("VLAN filter already set"); + } + /* rte_flow */ + if (TAILQ_EMPTY(&PRIV(dev)->flow_list)) { + DEBUG("rte_flow already set"); + } else { + DEBUG("Resetting rte_flow configuration"); + ret = rte_flow_flush(PORT_ID(sdev), &ferror); + if (ret) { + fs_flow_complain(&ferror); + return ret; + } + i = 0; + rte_errno = 0; + DEBUG("Configuring rte_flow"); + TAILQ_FOREACH(flow, &PRIV(dev)->flow_list, next) { + DEBUG("Creating flow #%" PRIu32, i++); + flow->flows[SUB_ID(sdev)] = + rte_flow_create(PORT_ID(sdev), + &flow->fd->attr, + flow->fd->items, + flow->fd->actions, + &ferror); + ret = rte_errno; + if (ret) + break; + } + if (ret) { + fs_flow_complain(&ferror); + return ret; + } + } + return 0; +} + +static void +fs_dev_remove(struct sub_device *sdev) +{ + int ret; + + if (sdev == NULL) + return; + switch (sdev->state) { + case DEV_STARTED: + rte_eth_dev_stop(PORT_ID(sdev)); + sdev->state = DEV_ACTIVE; + /* fallthrough */ + case DEV_ACTIVE: + rte_eth_dev_close(PORT_ID(sdev)); + sdev->state = DEV_PROBED; + /* fallthrough */ + case DEV_PROBED: + ret = rte_eal_hotplug_remove(sdev->bus->name, + sdev->dev->name); + if (ret) { + ERROR("Bus detach failed for sub_device %u", + SUB_ID(sdev)); + } else { + ETH(sdev)->state = RTE_ETH_DEV_UNUSED; + } + sdev->state = DEV_PARSED; + /* fallthrough */ + case DEV_PARSED: + case DEV_UNDEFINED: + sdev->state = DEV_UNDEFINED; + /* the end */ + break; + } + failsafe_hotplug_alarm_install(sdev->fs_dev); +} + +static void +fs_dev_stats_save(struct sub_device *sdev) +{ + struct rte_eth_stats stats; + int err; + + /* Attempt to read current stats. */ + err = rte_eth_stats_get(PORT_ID(sdev), &stats); + if (err) { + uint64_t timestamp = sdev->stats_snapshot.timestamp; + + WARN("Could not access latest statistics from sub-device %d.\n", + SUB_ID(sdev)); + if (timestamp != 0) + WARN("Using latest snapshot taken before %"PRIu64" seconds.\n", + (rte_rdtsc() - timestamp) / rte_get_tsc_hz()); + } + failsafe_stats_increment(&PRIV(sdev->fs_dev)->stats_accumulator, + err ? &sdev->stats_snapshot.stats : &stats); + memset(&sdev->stats_snapshot, 0, sizeof(sdev->stats_snapshot)); +} + +static inline int +fs_rxtx_clean(struct sub_device *sdev) +{ + uint16_t i; + + for (i = 0; i < ETH(sdev)->data->nb_rx_queues; i++) + if (FS_ATOMIC_RX(sdev, i)) + return 0; + for (i = 0; i < ETH(sdev)->data->nb_tx_queues; i++) + if (FS_ATOMIC_TX(sdev, i)) + return 0; + return 1; +} + +void +failsafe_dev_remove(struct rte_eth_dev *dev) +{ + struct sub_device *sdev; + uint8_t i; + + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) + if (sdev->remove && fs_rxtx_clean(sdev)) { + fs_dev_stats_save(sdev); + fs_dev_remove(sdev); + } +} + +int +failsafe_eth_dev_state_sync(struct rte_eth_dev *dev) +{ + struct sub_device *sdev; + uint32_t inactive; + int ret; + uint8_t i; + + if (PRIV(dev)->state < DEV_PARSED) + return 0; + + ret = failsafe_args_parse_subs(dev); + if (ret) + goto err_remove; + + if (PRIV(dev)->state < DEV_PROBED) + return 0; + ret = failsafe_eal_init(dev); + if (ret) + goto err_remove; + if (PRIV(dev)->state < DEV_ACTIVE) + return 0; + inactive = 0; + FOREACH_SUBDEV(sdev, i, dev) { + if (sdev->state == DEV_PROBED) { + inactive |= UINT32_C(1) << i; + ret = eth_dev_flow_isolate_set(dev, sdev); + if (ret) { + ERROR("Could not apply configuration to sub_device %d", + i); + goto err_remove; + } + } + } + ret = dev->dev_ops->dev_configure(dev); + if (ret) + goto err_remove; + FOREACH_SUBDEV(sdev, i, dev) { + if (inactive & (UINT32_C(1) << i)) { + ret = fs_eth_dev_conf_apply(dev, sdev); + if (ret) { + ERROR("Could not apply configuration to sub_device %d", + i); + goto err_remove; + } + } + } + /* + * If new devices have been configured, check if + * the link state has changed. + */ + if (inactive) + dev->dev_ops->link_update(dev, 1); + if (PRIV(dev)->state < DEV_STARTED) + return 0; + ret = dev->dev_ops->dev_start(dev); + if (ret) + goto err_remove; + return 0; +err_remove: + FOREACH_SUBDEV(sdev, i, dev) + if (sdev->state != PRIV(dev)->state) + sdev->remove = 1; + return ret; +} + +void +failsafe_stats_increment(struct rte_eth_stats *to, struct rte_eth_stats *from) +{ + uint32_t i; + + RTE_ASSERT(to != NULL && from != NULL); + to->ipackets += from->ipackets; + to->opackets += from->opackets; + to->ibytes += from->ibytes; + to->obytes += from->obytes; + to->imissed += from->imissed; + to->ierrors += from->ierrors; + to->oerrors += from->oerrors; + to->rx_nombuf += from->rx_nombuf; + for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS; i++) { + to->q_ipackets[i] += from->q_ipackets[i]; + to->q_opackets[i] += from->q_opackets[i]; + to->q_ibytes[i] += from->q_ibytes[i]; + to->q_obytes[i] += from->q_obytes[i]; + to->q_errors[i] += from->q_errors[i]; + } +} + +int +failsafe_eth_rmv_event_callback(uint16_t port_id __rte_unused, + enum rte_eth_event_type event __rte_unused, + void *cb_arg, void *out __rte_unused) +{ + struct sub_device *sdev = cb_arg; + + /* Switch as soon as possible tx_dev. */ + fs_switch_dev(sdev->fs_dev, sdev); + /* Use safe bursts in any case. */ + set_burst_fn(sdev->fs_dev, 1); + /* + * Async removal, the sub-PMD will try to unregister + * the callback at the source of the current thread context. + */ + sdev->remove = 1; + return 0; +} + +int +failsafe_eth_lsc_event_callback(uint16_t port_id __rte_unused, + enum rte_eth_event_type event __rte_unused, + void *cb_arg, void *out __rte_unused) +{ + struct rte_eth_dev *dev = cb_arg; + int ret; + + ret = dev->dev_ops->link_update(dev, 0); + /* We must pass on the LSC event */ + if (ret) + return _rte_eth_dev_callback_process(dev, + RTE_ETH_EVENT_INTR_LSC, + NULL, NULL); + else + return 0; +} diff --git a/dpdk/drivers/net/failsafe/failsafe_flow.c b/dpdk/drivers/net/failsafe/failsafe_flow.c new file mode 100644 index 00000000..153ceeed --- /dev/null +++ b/dpdk/drivers/net/failsafe/failsafe_flow.c @@ -0,0 +1,244 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of 6WIND S.A. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +#include +#include +#include +#include + +#include "failsafe_private.h" + +static struct rte_flow * +fs_flow_allocate(const struct rte_flow_attr *attr, + const struct rte_flow_item *items, + const struct rte_flow_action *actions) +{ + struct rte_flow *flow; + size_t fdsz; + + fdsz = rte_flow_copy(NULL, 0, attr, items, actions); + flow = rte_zmalloc(NULL, + sizeof(struct rte_flow) + fdsz, + RTE_CACHE_LINE_SIZE); + if (flow == NULL) { + ERROR("Could not allocate new flow"); + return NULL; + } + flow->fd = (void *)((uintptr_t)flow + sizeof(*flow)); + if (rte_flow_copy(flow->fd, fdsz, attr, items, actions) != fdsz) { + ERROR("Failed to copy flow description"); + rte_free(flow); + return NULL; + } + return flow; +} + +static void +fs_flow_release(struct rte_flow **flow) +{ + rte_free(*flow); + *flow = NULL; +} + +static int +fs_flow_validate(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item patterns[], + const struct rte_flow_action actions[], + struct rte_flow_error *error) +{ + struct sub_device *sdev; + uint8_t i; + int ret; + + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { + DEBUG("Calling rte_flow_validate on sub_device %d", i); + ret = rte_flow_validate(PORT_ID(sdev), + attr, patterns, actions, error); + if (ret) { + ERROR("Operation rte_flow_validate failed for sub_device %d" + " with error %d", i, ret); + return ret; + } + } + return 0; +} + +static struct rte_flow * +fs_flow_create(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item patterns[], + const struct rte_flow_action actions[], + struct rte_flow_error *error) +{ + struct sub_device *sdev; + struct rte_flow *flow; + uint8_t i; + + flow = fs_flow_allocate(attr, patterns, actions); + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { + flow->flows[i] = rte_flow_create(PORT_ID(sdev), + attr, patterns, actions, error); + if (flow->flows[i] == NULL) { + ERROR("Failed to create flow on sub_device %d", + i); + goto err; + } + } + TAILQ_INSERT_TAIL(&PRIV(dev)->flow_list, flow, next); + return flow; +err: + FOREACH_SUBDEV(sdev, i, dev) { + if (flow->flows[i] != NULL) + rte_flow_destroy(PORT_ID(sdev), + flow->flows[i], error); + } + fs_flow_release(&flow); + return NULL; +} + +static int +fs_flow_destroy(struct rte_eth_dev *dev, + struct rte_flow *flow, + struct rte_flow_error *error) +{ + struct sub_device *sdev; + uint8_t i; + int ret; + + if (flow == NULL) { + ERROR("Invalid flow"); + return -EINVAL; + } + ret = 0; + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { + int local_ret; + + if (flow->flows[i] == NULL) + continue; + local_ret = rte_flow_destroy(PORT_ID(sdev), + flow->flows[i], error); + if (local_ret) { + ERROR("Failed to destroy flow on sub_device %d: %d", + i, local_ret); + if (ret == 0) + ret = local_ret; + } + } + TAILQ_REMOVE(&PRIV(dev)->flow_list, flow, next); + fs_flow_release(&flow); + return ret; +} + +static int +fs_flow_flush(struct rte_eth_dev *dev, + struct rte_flow_error *error) +{ + struct sub_device *sdev; + struct rte_flow *flow; + void *tmp; + uint8_t i; + int ret; + + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { + DEBUG("Calling rte_flow_flush on sub_device %d", i); + ret = rte_flow_flush(PORT_ID(sdev), error); + if (ret) { + ERROR("Operation rte_flow_flush failed for sub_device %d" + " with error %d", i, ret); + return ret; + } + } + TAILQ_FOREACH_SAFE(flow, &PRIV(dev)->flow_list, next, tmp) { + TAILQ_REMOVE(&PRIV(dev)->flow_list, flow, next); + fs_flow_release(&flow); + } + return 0; +} + +static int +fs_flow_query(struct rte_eth_dev *dev, + struct rte_flow *flow, + enum rte_flow_action_type type, + void *arg, + struct rte_flow_error *error) +{ + struct sub_device *sdev; + + sdev = TX_SUBDEV(dev); + if (sdev != NULL) { + return rte_flow_query(PORT_ID(sdev), + flow->flows[SUB_ID(sdev)], type, arg, error); + } + WARN("No active sub_device to query about its flow"); + return -1; +} + +static int +fs_flow_isolate(struct rte_eth_dev *dev, + int set, + struct rte_flow_error *error) +{ + struct sub_device *sdev; + uint8_t i; + int ret; + + FOREACH_SUBDEV(sdev, i, dev) { + if (sdev->state < DEV_PROBED) + continue; + DEBUG("Calling rte_flow_isolate on sub_device %d", i); + if (PRIV(dev)->flow_isolated != sdev->flow_isolated) + WARN("flow isolation mode of sub_device %d in incoherent state.", + i); + ret = rte_flow_isolate(PORT_ID(sdev), set, error); + if (ret) { + ERROR("Operation rte_flow_isolate failed for sub_device %d" + " with error %d", i, ret); + return ret; + } + sdev->flow_isolated = set; + } + PRIV(dev)->flow_isolated = set; + return 0; +} + +const struct rte_flow_ops fs_flow_ops = { + .validate = fs_flow_validate, + .create = fs_flow_create, + .destroy = fs_flow_destroy, + .flush = fs_flow_flush, + .query = fs_flow_query, + .isolate = fs_flow_isolate, +}; diff --git a/dpdk/drivers/net/failsafe/failsafe_ops.c b/dpdk/drivers/net/failsafe/failsafe_ops.c new file mode 100644 index 00000000..e16a5903 --- /dev/null +++ b/dpdk/drivers/net/failsafe/failsafe_ops.c @@ -0,0 +1,770 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of 6WIND S.A. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +#include +#include +#include +#include +#include +#include + +#include "failsafe_private.h" + +static struct rte_eth_dev_info default_infos = { + /* Max possible number of elements */ + .max_rx_pktlen = UINT32_MAX, + .max_rx_queues = RTE_MAX_QUEUES_PER_PORT, + .max_tx_queues = RTE_MAX_QUEUES_PER_PORT, + .max_mac_addrs = FAILSAFE_MAX_ETHADDR, + .max_hash_mac_addrs = UINT32_MAX, + .max_vfs = UINT16_MAX, + .max_vmdq_pools = UINT16_MAX, + .rx_desc_lim = { + .nb_max = UINT16_MAX, + .nb_min = 0, + .nb_align = 1, + .nb_seg_max = UINT16_MAX, + .nb_mtu_seg_max = UINT16_MAX, + }, + .tx_desc_lim = { + .nb_max = UINT16_MAX, + .nb_min = 0, + .nb_align = 1, + .nb_seg_max = UINT16_MAX, + .nb_mtu_seg_max = UINT16_MAX, + }, + /* + * Set of capabilities that can be verified upon + * configuring a sub-device. + */ + .rx_offload_capa = + DEV_RX_OFFLOAD_VLAN_STRIP | + DEV_RX_OFFLOAD_QINQ_STRIP | + DEV_RX_OFFLOAD_IPV4_CKSUM | + DEV_RX_OFFLOAD_UDP_CKSUM | + DEV_RX_OFFLOAD_TCP_CKSUM | + DEV_RX_OFFLOAD_TCP_LRO, + .tx_offload_capa = 0x0, + .flow_type_rss_offloads = 0x0, +}; + +static int +fs_dev_configure(struct rte_eth_dev *dev) +{ + struct sub_device *sdev; + uint8_t i; + int ret; + + FOREACH_SUBDEV(sdev, i, dev) { + int rmv_interrupt = 0; + int lsc_interrupt = 0; + int lsc_enabled; + + if (sdev->state != DEV_PROBED) + continue; + + rmv_interrupt = ETH(sdev)->data->dev_flags & + RTE_ETH_DEV_INTR_RMV; + if (rmv_interrupt) { + DEBUG("Enabling RMV interrupts for sub_device %d", i); + dev->data->dev_conf.intr_conf.rmv = 1; + } else { + DEBUG("sub_device %d does not support RMV event", i); + } + lsc_enabled = dev->data->dev_conf.intr_conf.lsc; + lsc_interrupt = lsc_enabled && + (ETH(sdev)->data->dev_flags & + RTE_ETH_DEV_INTR_LSC); + if (lsc_interrupt) { + DEBUG("Enabling LSC interrupts for sub_device %d", i); + dev->data->dev_conf.intr_conf.lsc = 1; + } else if (lsc_enabled && !lsc_interrupt) { + DEBUG("Disabling LSC interrupts for sub_device %d", i); + dev->data->dev_conf.intr_conf.lsc = 0; + } + DEBUG("Configuring sub-device %d", i); + sdev->remove = 0; + ret = rte_eth_dev_configure(PORT_ID(sdev), + dev->data->nb_rx_queues, + dev->data->nb_tx_queues, + &dev->data->dev_conf); + if (ret) { + ERROR("Could not configure sub_device %d", i); + return ret; + } + if (rmv_interrupt) { + ret = rte_eth_dev_callback_register(PORT_ID(sdev), + RTE_ETH_EVENT_INTR_RMV, + failsafe_eth_rmv_event_callback, + sdev); + if (ret) + WARN("Failed to register RMV callback for sub_device %d", + SUB_ID(sdev)); + } + dev->data->dev_conf.intr_conf.rmv = 0; + if (lsc_interrupt) { + ret = rte_eth_dev_callback_register(PORT_ID(sdev), + RTE_ETH_EVENT_INTR_LSC, + failsafe_eth_lsc_event_callback, + dev); + if (ret) + WARN("Failed to register LSC callback for sub_device %d", + SUB_ID(sdev)); + } + dev->data->dev_conf.intr_conf.lsc = lsc_enabled; + sdev->state = DEV_ACTIVE; + } + if (PRIV(dev)->state < DEV_ACTIVE) + PRIV(dev)->state = DEV_ACTIVE; + return 0; +} + +static int +fs_dev_start(struct rte_eth_dev *dev) +{ + struct sub_device *sdev; + uint8_t i; + int ret; + + FOREACH_SUBDEV(sdev, i, dev) { + if (sdev->state != DEV_ACTIVE) + continue; + DEBUG("Starting sub_device %d", i); + ret = rte_eth_dev_start(PORT_ID(sdev)); + if (ret) + return ret; + sdev->state = DEV_STARTED; + } + if (PRIV(dev)->state < DEV_STARTED) + PRIV(dev)->state = DEV_STARTED; + fs_switch_dev(dev, NULL); + return 0; +} + +static void +fs_dev_stop(struct rte_eth_dev *dev) +{ + struct sub_device *sdev; + uint8_t i; + + PRIV(dev)->state = DEV_STARTED - 1; + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_STARTED) { + rte_eth_dev_stop(PORT_ID(sdev)); + sdev->state = DEV_STARTED - 1; + } +} + +static int +fs_dev_set_link_up(struct rte_eth_dev *dev) +{ + struct sub_device *sdev; + uint8_t i; + int ret; + + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { + DEBUG("Calling rte_eth_dev_set_link_up on sub_device %d", i); + ret = rte_eth_dev_set_link_up(PORT_ID(sdev)); + if (ret) { + ERROR("Operation rte_eth_dev_set_link_up failed for sub_device %d" + " with error %d", i, ret); + return ret; + } + } + return 0; +} + +static int +fs_dev_set_link_down(struct rte_eth_dev *dev) +{ + struct sub_device *sdev; + uint8_t i; + int ret; + + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { + DEBUG("Calling rte_eth_dev_set_link_down on sub_device %d", i); + ret = rte_eth_dev_set_link_down(PORT_ID(sdev)); + if (ret) { + ERROR("Operation rte_eth_dev_set_link_down failed for sub_device %d" + " with error %d", i, ret); + return ret; + } + } + return 0; +} + +static void fs_dev_free_queues(struct rte_eth_dev *dev); +static void +fs_dev_close(struct rte_eth_dev *dev) +{ + struct sub_device *sdev; + uint8_t i; + + failsafe_hotplug_alarm_cancel(dev); + if (PRIV(dev)->state == DEV_STARTED) + dev->dev_ops->dev_stop(dev); + PRIV(dev)->state = DEV_ACTIVE - 1; + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { + DEBUG("Closing sub_device %d", i); + rte_eth_dev_close(PORT_ID(sdev)); + sdev->state = DEV_ACTIVE - 1; + } + fs_dev_free_queues(dev); +} + +static void +fs_rx_queue_release(void *queue) +{ + struct rte_eth_dev *dev; + struct sub_device *sdev; + uint8_t i; + struct rxq *rxq; + + if (queue == NULL) + return; + rxq = queue; + dev = rxq->priv->dev; + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) + SUBOPS(sdev, rx_queue_release) + (ETH(sdev)->data->rx_queues[rxq->qid]); + dev->data->rx_queues[rxq->qid] = NULL; + rte_free(rxq); +} + +static int +fs_rx_queue_setup(struct rte_eth_dev *dev, + uint16_t rx_queue_id, + uint16_t nb_rx_desc, + unsigned int socket_id, + const struct rte_eth_rxconf *rx_conf, + struct rte_mempool *mb_pool) +{ + struct sub_device *sdev; + struct rxq *rxq; + uint8_t i; + int ret; + + rxq = dev->data->rx_queues[rx_queue_id]; + if (rxq != NULL) { + fs_rx_queue_release(rxq); + dev->data->rx_queues[rx_queue_id] = NULL; + } + rxq = rte_zmalloc(NULL, + sizeof(*rxq) + + sizeof(rte_atomic64_t) * PRIV(dev)->subs_tail, + RTE_CACHE_LINE_SIZE); + if (rxq == NULL) + return -ENOMEM; + FOREACH_SUBDEV(sdev, i, dev) + rte_atomic64_init(&rxq->refcnt[i]); + rxq->qid = rx_queue_id; + rxq->socket_id = socket_id; + rxq->info.mp = mb_pool; + rxq->info.conf = *rx_conf; + rxq->info.nb_desc = nb_rx_desc; + rxq->priv = PRIV(dev); + dev->data->rx_queues[rx_queue_id] = rxq; + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { + ret = rte_eth_rx_queue_setup(PORT_ID(sdev), + rx_queue_id, + nb_rx_desc, socket_id, + rx_conf, mb_pool); + if (ret) { + ERROR("RX queue setup failed for sub_device %d", i); + goto free_rxq; + } + } + return 0; +free_rxq: + fs_rx_queue_release(rxq); + return ret; +} + +static void +fs_tx_queue_release(void *queue) +{ + struct rte_eth_dev *dev; + struct sub_device *sdev; + uint8_t i; + struct txq *txq; + + if (queue == NULL) + return; + txq = queue; + dev = txq->priv->dev; + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) + SUBOPS(sdev, tx_queue_release) + (ETH(sdev)->data->tx_queues[txq->qid]); + dev->data->tx_queues[txq->qid] = NULL; + rte_free(txq); +} + +static int +fs_tx_queue_setup(struct rte_eth_dev *dev, + uint16_t tx_queue_id, + uint16_t nb_tx_desc, + unsigned int socket_id, + const struct rte_eth_txconf *tx_conf) +{ + struct sub_device *sdev; + struct txq *txq; + uint8_t i; + int ret; + + txq = dev->data->tx_queues[tx_queue_id]; + if (txq != NULL) { + fs_tx_queue_release(txq); + dev->data->tx_queues[tx_queue_id] = NULL; + } + txq = rte_zmalloc("ethdev TX queue", + sizeof(*txq) + + sizeof(rte_atomic64_t) * PRIV(dev)->subs_tail, + RTE_CACHE_LINE_SIZE); + if (txq == NULL) + return -ENOMEM; + FOREACH_SUBDEV(sdev, i, dev) + rte_atomic64_init(&txq->refcnt[i]); + txq->qid = tx_queue_id; + txq->socket_id = socket_id; + txq->info.conf = *tx_conf; + txq->info.nb_desc = nb_tx_desc; + txq->priv = PRIV(dev); + dev->data->tx_queues[tx_queue_id] = txq; + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { + ret = rte_eth_tx_queue_setup(PORT_ID(sdev), + tx_queue_id, + nb_tx_desc, socket_id, + tx_conf); + if (ret) { + ERROR("TX queue setup failed for sub_device %d", i); + goto free_txq; + } + } + return 0; +free_txq: + fs_tx_queue_release(txq); + return ret; +} + +static void +fs_dev_free_queues(struct rte_eth_dev *dev) +{ + uint16_t i; + + for (i = 0; i < dev->data->nb_rx_queues; i++) { + fs_rx_queue_release(dev->data->rx_queues[i]); + dev->data->rx_queues[i] = NULL; + } + dev->data->nb_rx_queues = 0; + for (i = 0; i < dev->data->nb_tx_queues; i++) { + fs_tx_queue_release(dev->data->tx_queues[i]); + dev->data->tx_queues[i] = NULL; + } + dev->data->nb_tx_queues = 0; +} + +static void +fs_promiscuous_enable(struct rte_eth_dev *dev) +{ + struct sub_device *sdev; + uint8_t i; + + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) + rte_eth_promiscuous_enable(PORT_ID(sdev)); +} + +static void +fs_promiscuous_disable(struct rte_eth_dev *dev) +{ + struct sub_device *sdev; + uint8_t i; + + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) + rte_eth_promiscuous_disable(PORT_ID(sdev)); +} + +static void +fs_allmulticast_enable(struct rte_eth_dev *dev) +{ + struct sub_device *sdev; + uint8_t i; + + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) + rte_eth_allmulticast_enable(PORT_ID(sdev)); +} + +static void +fs_allmulticast_disable(struct rte_eth_dev *dev) +{ + struct sub_device *sdev; + uint8_t i; + + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) + rte_eth_allmulticast_disable(PORT_ID(sdev)); +} + +static int +fs_link_update(struct rte_eth_dev *dev, + int wait_to_complete) +{ + struct sub_device *sdev; + uint8_t i; + int ret; + + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { + DEBUG("Calling link_update on sub_device %d", i); + ret = (SUBOPS(sdev, link_update))(ETH(sdev), wait_to_complete); + if (ret && ret != -1) { + ERROR("Link update failed for sub_device %d with error %d", + i, ret); + return ret; + } + } + if (TX_SUBDEV(dev)) { + struct rte_eth_link *l1; + struct rte_eth_link *l2; + + l1 = &dev->data->dev_link; + l2 = Ð(TX_SUBDEV(dev))->data->dev_link; + if (memcmp(l1, l2, sizeof(*l1))) { + *l1 = *l2; + return 0; + } + } + return -1; +} + +static int +fs_stats_get(struct rte_eth_dev *dev, + struct rte_eth_stats *stats) +{ + struct sub_device *sdev; + uint8_t i; + int ret; + + rte_memcpy(stats, &PRIV(dev)->stats_accumulator, sizeof(*stats)); + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { + struct rte_eth_stats *snapshot = &sdev->stats_snapshot.stats; + uint64_t *timestamp = &sdev->stats_snapshot.timestamp; + + ret = rte_eth_stats_get(PORT_ID(sdev), snapshot); + if (ret) { + ERROR("Operation rte_eth_stats_get failed for sub_device %d with error %d", + i, ret); + *timestamp = 0; + return ret; + } + *timestamp = rte_rdtsc(); + failsafe_stats_increment(stats, snapshot); + } + return 0; +} + +static void +fs_stats_reset(struct rte_eth_dev *dev) +{ + struct sub_device *sdev; + uint8_t i; + + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { + rte_eth_stats_reset(PORT_ID(sdev)); + memset(&sdev->stats_snapshot, 0, sizeof(struct rte_eth_stats)); + } + memset(&PRIV(dev)->stats_accumulator, 0, sizeof(struct rte_eth_stats)); +} + +/** + * Fail-safe dev_infos_get rules: + * + * No sub_device: + * Numerables: + * Use the maximum possible values for any field, so as not + * to impede any further configuration effort. + * Capabilities: + * Limits capabilities to those that are understood by the + * fail-safe PMD. This understanding stems from the fail-safe + * being capable of verifying that the related capability is + * expressed within the device configuration (struct rte_eth_conf). + * + * At least one probed sub_device: + * Numerables: + * Uses values from the active probed sub_device + * The rationale here is that if any sub_device is less capable + * (for example concerning the number of queues) than the active + * sub_device, then its subsequent configuration will fail. + * It is impossible to foresee this failure when the failing sub_device + * is supposed to be plugged-in later on, so the configuration process + * is the single point of failure and error reporting. + * Capabilities: + * Uses a logical AND of RX capabilities among + * all sub_devices and the default capabilities. + * Uses a logical AND of TX capabilities among + * the active probed sub_device and the default capabilities. + * + */ +static void +fs_dev_infos_get(struct rte_eth_dev *dev, + struct rte_eth_dev_info *infos) +{ + struct sub_device *sdev; + uint8_t i; + + sdev = TX_SUBDEV(dev); + if (sdev == NULL) { + DEBUG("No probed device, using default infos"); + rte_memcpy(&PRIV(dev)->infos, &default_infos, + sizeof(default_infos)); + } else { + uint32_t rx_offload_capa; + + rx_offload_capa = default_infos.rx_offload_capa; + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_PROBED) { + rte_eth_dev_info_get(PORT_ID(sdev), + &PRIV(dev)->infos); + rx_offload_capa &= PRIV(dev)->infos.rx_offload_capa; + } + sdev = TX_SUBDEV(dev); + rte_eth_dev_info_get(PORT_ID(sdev), &PRIV(dev)->infos); + PRIV(dev)->infos.rx_offload_capa = rx_offload_capa; + PRIV(dev)->infos.tx_offload_capa &= + default_infos.tx_offload_capa; + PRIV(dev)->infos.flow_type_rss_offloads &= + default_infos.flow_type_rss_offloads; + } + rte_memcpy(infos, &PRIV(dev)->infos, sizeof(*infos)); +} + +static const uint32_t * +fs_dev_supported_ptypes_get(struct rte_eth_dev *dev) +{ + struct sub_device *sdev; + struct rte_eth_dev *edev; + + sdev = TX_SUBDEV(dev); + if (sdev == NULL) + return NULL; + edev = ETH(sdev); + /* ENOTSUP: counts as no supported ptypes */ + if (SUBOPS(sdev, dev_supported_ptypes_get) == NULL) + return NULL; + /* + * The API does not permit to do a clean AND of all ptypes, + * It is also incomplete by design and we do not really care + * to have a best possible value in this context. + * We just return the ptypes of the device of highest + * priority, usually the PREFERRED device. + */ + return SUBOPS(sdev, dev_supported_ptypes_get)(edev); +} + +static int +fs_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) +{ + struct sub_device *sdev; + uint8_t i; + int ret; + + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { + DEBUG("Calling rte_eth_dev_set_mtu on sub_device %d", i); + ret = rte_eth_dev_set_mtu(PORT_ID(sdev), mtu); + if (ret) { + ERROR("Operation rte_eth_dev_set_mtu failed for sub_device %d with error %d", + i, ret); + return ret; + } + } + return 0; +} + +static int +fs_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) +{ + struct sub_device *sdev; + uint8_t i; + int ret; + + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { + DEBUG("Calling rte_eth_dev_vlan_filter on sub_device %d", i); + ret = rte_eth_dev_vlan_filter(PORT_ID(sdev), vlan_id, on); + if (ret) { + ERROR("Operation rte_eth_dev_vlan_filter failed for sub_device %d" + " with error %d", i, ret); + return ret; + } + } + return 0; +} + +static int +fs_flow_ctrl_get(struct rte_eth_dev *dev, + struct rte_eth_fc_conf *fc_conf) +{ + struct sub_device *sdev; + + sdev = TX_SUBDEV(dev); + if (sdev == NULL) + return 0; + if (SUBOPS(sdev, flow_ctrl_get) == NULL) + return -ENOTSUP; + return SUBOPS(sdev, flow_ctrl_get)(ETH(sdev), fc_conf); +} + +static int +fs_flow_ctrl_set(struct rte_eth_dev *dev, + struct rte_eth_fc_conf *fc_conf) +{ + struct sub_device *sdev; + uint8_t i; + int ret; + + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { + DEBUG("Calling rte_eth_dev_flow_ctrl_set on sub_device %d", i); + ret = rte_eth_dev_flow_ctrl_set(PORT_ID(sdev), fc_conf); + if (ret) { + ERROR("Operation rte_eth_dev_flow_ctrl_set failed for sub_device %d" + " with error %d", i, ret); + return ret; + } + } + return 0; +} + +static void +fs_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index) +{ + struct sub_device *sdev; + uint8_t i; + + /* No check: already done within the rte_eth_dev_mac_addr_remove + * call for the fail-safe device. + */ + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) + rte_eth_dev_mac_addr_remove(PORT_ID(sdev), + &dev->data->mac_addrs[index]); + PRIV(dev)->mac_addr_pool[index] = 0; +} + +static int +fs_mac_addr_add(struct rte_eth_dev *dev, + struct ether_addr *mac_addr, + uint32_t index, + uint32_t vmdq) +{ + struct sub_device *sdev; + int ret; + uint8_t i; + + RTE_ASSERT(index < FAILSAFE_MAX_ETHADDR); + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { + ret = rte_eth_dev_mac_addr_add(PORT_ID(sdev), mac_addr, vmdq); + if (ret) { + ERROR("Operation rte_eth_dev_mac_addr_add failed for sub_device %" + PRIu8 " with error %d", i, ret); + return ret; + } + } + if (index >= PRIV(dev)->nb_mac_addr) { + DEBUG("Growing mac_addrs array"); + PRIV(dev)->nb_mac_addr = index; + } + PRIV(dev)->mac_addr_pool[index] = vmdq; + return 0; +} + +static void +fs_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr) +{ + struct sub_device *sdev; + uint8_t i; + + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) + rte_eth_dev_default_mac_addr_set(PORT_ID(sdev), mac_addr); +} + +static int +fs_filter_ctrl(struct rte_eth_dev *dev, + enum rte_filter_type type, + enum rte_filter_op op, + void *arg) +{ + struct sub_device *sdev; + uint8_t i; + int ret; + + if (type == RTE_ETH_FILTER_GENERIC && + op == RTE_ETH_FILTER_GET) { + *(const void **)arg = &fs_flow_ops; + return 0; + } + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { + DEBUG("Calling rte_eth_dev_filter_ctrl on sub_device %d", i); + ret = rte_eth_dev_filter_ctrl(PORT_ID(sdev), type, op, arg); + if (ret) { + ERROR("Operation rte_eth_dev_filter_ctrl failed for sub_device %d" + " with error %d", i, ret); + return ret; + } + } + return 0; +} + +const struct eth_dev_ops failsafe_ops = { + .dev_configure = fs_dev_configure, + .dev_start = fs_dev_start, + .dev_stop = fs_dev_stop, + .dev_set_link_down = fs_dev_set_link_down, + .dev_set_link_up = fs_dev_set_link_up, + .dev_close = fs_dev_close, + .promiscuous_enable = fs_promiscuous_enable, + .promiscuous_disable = fs_promiscuous_disable, + .allmulticast_enable = fs_allmulticast_enable, + .allmulticast_disable = fs_allmulticast_disable, + .link_update = fs_link_update, + .stats_get = fs_stats_get, + .stats_reset = fs_stats_reset, + .dev_infos_get = fs_dev_infos_get, + .dev_supported_ptypes_get = fs_dev_supported_ptypes_get, + .mtu_set = fs_mtu_set, + .vlan_filter_set = fs_vlan_filter_set, + .rx_queue_setup = fs_rx_queue_setup, + .tx_queue_setup = fs_tx_queue_setup, + .rx_queue_release = fs_rx_queue_release, + .tx_queue_release = fs_tx_queue_release, + .flow_ctrl_get = fs_flow_ctrl_get, + .flow_ctrl_set = fs_flow_ctrl_set, + .mac_addr_remove = fs_mac_addr_remove, + .mac_addr_add = fs_mac_addr_add, + .mac_addr_set = fs_mac_addr_set, + .filter_ctrl = fs_filter_ctrl, +}; diff --git a/dpdk/drivers/net/failsafe/failsafe_private.h b/dpdk/drivers/net/failsafe/failsafe_private.h new file mode 100644 index 00000000..d81cc3ca --- /dev/null +++ b/dpdk/drivers/net/failsafe/failsafe_private.h @@ -0,0 +1,378 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of 6WIND S.A. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_ETH_FAILSAFE_PRIVATE_H_ +#define _RTE_ETH_FAILSAFE_PRIVATE_H_ + +#include + +#include +#include +#include +#include + +#define FAILSAFE_DRIVER_NAME "Fail-safe PMD" + +#define PMD_FAILSAFE_MAC_KVARG "mac" +#define PMD_FAILSAFE_HOTPLUG_POLL_KVARG "hotplug_poll" +#define PMD_FAILSAFE_PARAM_STRING \ + "dev()," \ + "exec()," \ + "mac=mac_addr," \ + "hotplug_poll=u64" \ + "" + +#define FAILSAFE_HOTPLUG_DEFAULT_TIMEOUT_MS 2000 + +#define FAILSAFE_MAX_ETHPORTS 2 +#define FAILSAFE_MAX_ETHADDR 128 + +/* TYPES */ + +struct rxq { + struct fs_priv *priv; + uint16_t qid; + /* id of last sub_device polled */ + uint8_t last_polled; + unsigned int socket_id; + struct rte_eth_rxq_info info; + rte_atomic64_t refcnt[]; +}; + +struct txq { + struct fs_priv *priv; + uint16_t qid; + unsigned int socket_id; + struct rte_eth_txq_info info; + rte_atomic64_t refcnt[]; +}; + +struct rte_flow { + TAILQ_ENTRY(rte_flow) next; + /* sub_flows */ + struct rte_flow *flows[FAILSAFE_MAX_ETHPORTS]; + /* flow description for synchronization */ + struct rte_flow_desc *fd; +}; + +enum dev_state { + DEV_UNDEFINED, + DEV_PARSED, + DEV_PROBED, + DEV_ACTIVE, + DEV_STARTED, +}; + +struct fs_stats { + struct rte_eth_stats stats; + uint64_t timestamp; +}; + +struct sub_device { + /* Exhaustive DPDK device description */ + struct rte_devargs devargs; + struct rte_bus *bus; + struct rte_device *dev; + struct rte_eth_dev *edev; + uint8_t sid; + /* Device state machine */ + enum dev_state state; + /* Last stats snapshot passed to user */ + struct fs_stats stats_snapshot; + /* Some device are defined as a command line */ + char *cmdline; + /* fail-safe device backreference */ + struct rte_eth_dev *fs_dev; + /* flag calling for recollection */ + volatile unsigned int remove:1; + /* flow isolation state */ + int flow_isolated:1; +}; + +struct fs_priv { + struct rte_eth_dev *dev; + /* + * Set of sub_devices. + * subs[0] is the preferred device + * any other is just another slave + */ + struct sub_device *subs; + uint8_t subs_head; /* if head == tail, no subs */ + uint8_t subs_tail; /* first invalid */ + uint8_t subs_tx; /* current emitting device */ + uint8_t current_probed; + /* flow mapping */ + TAILQ_HEAD(sub_flows, rte_flow) flow_list; + /* current number of mac_addr slots allocated. */ + uint32_t nb_mac_addr; + struct ether_addr mac_addrs[FAILSAFE_MAX_ETHADDR]; + uint32_t mac_addr_pool[FAILSAFE_MAX_ETHADDR]; + /* current capabilities */ + struct rte_eth_dev_info infos; + /* + * Fail-safe state machine. + * This level will be tracking state of the EAL and eth + * layer at large as defined by the user application. + * It will then steer the sub_devices toward the same + * synchronized state. + */ + enum dev_state state; + struct rte_eth_stats stats_accumulator; + unsigned int pending_alarm:1; /* An alarm is pending */ + /* flow isolation state */ + int flow_isolated:1; +}; + +/* MISC */ + +int failsafe_hotplug_alarm_install(struct rte_eth_dev *dev); +int failsafe_hotplug_alarm_cancel(struct rte_eth_dev *dev); + +/* RX / TX */ + +void set_burst_fn(struct rte_eth_dev *dev, int force_safe); + +uint16_t failsafe_rx_burst(void *rxq, + struct rte_mbuf **rx_pkts, uint16_t nb_pkts); +uint16_t failsafe_tx_burst(void *txq, + struct rte_mbuf **tx_pkts, uint16_t nb_pkts); + +uint16_t failsafe_rx_burst_fast(void *rxq, + struct rte_mbuf **rx_pkts, uint16_t nb_pkts); +uint16_t failsafe_tx_burst_fast(void *txq, + struct rte_mbuf **tx_pkts, uint16_t nb_pkts); + +/* ARGS */ + +int failsafe_args_parse(struct rte_eth_dev *dev, const char *params); +void failsafe_args_free(struct rte_eth_dev *dev); +int failsafe_args_count_subdevice(struct rte_eth_dev *dev, const char *params); +int failsafe_args_parse_subs(struct rte_eth_dev *dev); + +/* EAL */ + +int failsafe_eal_init(struct rte_eth_dev *dev); +int failsafe_eal_uninit(struct rte_eth_dev *dev); + +/* ETH_DEV */ + +int failsafe_eth_dev_state_sync(struct rte_eth_dev *dev); +void failsafe_dev_remove(struct rte_eth_dev *dev); +void failsafe_stats_increment(struct rte_eth_stats *to, + struct rte_eth_stats *from); +int failsafe_eth_rmv_event_callback(uint16_t port_id, + enum rte_eth_event_type type, + void *arg, void *out); +int failsafe_eth_lsc_event_callback(uint16_t port_id, + enum rte_eth_event_type event, + void *cb_arg, void *out); + +/* GLOBALS */ + +extern const char pmd_failsafe_driver_name[]; +extern const struct eth_dev_ops failsafe_ops; +extern const struct rte_flow_ops fs_flow_ops; +extern uint64_t hotplug_poll; +extern int mac_from_arg; + +/* HELPERS */ + +/* dev: (struct rte_eth_dev *) fail-safe device */ +#define PRIV(dev) \ + ((struct fs_priv *)(dev)->data->dev_private) + +/* sdev: (struct sub_device *) */ +#define ETH(sdev) \ + ((sdev)->edev) + +/* sdev: (struct sub_device *) */ +#define PORT_ID(sdev) \ + (ETH(sdev)->data->port_id) + +/* sdev: (struct sub_device *) */ +#define SUB_ID(sdev) \ + ((sdev)->sid) + +/** + * Stateful iterator construct over fail-safe sub-devices: + * s: (struct sub_device *), iterator + * i: (uint8_t), increment + * dev: (struct rte_eth_dev *), fail-safe ethdev + * state: (enum dev_state), minimum acceptable device state + */ +#define FOREACH_SUBDEV_STATE(s, i, dev, state) \ + for (s = fs_find_next((dev), 0, state, &i); \ + s != NULL; \ + s = fs_find_next((dev), i + 1, state, &i)) + +/** + * Iterator construct over fail-safe sub-devices: + * s: (struct sub_device *), iterator + * i: (uint8_t), increment + * dev: (struct rte_eth_dev *), fail-safe ethdev + */ +#define FOREACH_SUBDEV(s, i, dev) \ + FOREACH_SUBDEV_STATE(s, i, dev, DEV_UNDEFINED) + +/* dev: (struct rte_eth_dev *) fail-safe device */ +#define PREFERRED_SUBDEV(dev) \ + (&PRIV(dev)->subs[0]) + +/* dev: (struct rte_eth_dev *) fail-safe device */ +#define TX_SUBDEV(dev) \ + (PRIV(dev)->subs_tx >= PRIV(dev)->subs_tail ? NULL \ + : (PRIV(dev)->subs[PRIV(dev)->subs_tx].state < DEV_PROBED ? NULL \ + : &PRIV(dev)->subs[PRIV(dev)->subs_tx])) + +/** + * s: (struct sub_device *) + * ops: (struct eth_dev_ops) member + */ +#define SUBOPS(s, ops) \ + (ETH(s)->dev_ops->ops) + +/** + * Atomic guard + */ + +/** + * a: (rte_atomic64_t) + */ +#define FS_ATOMIC_P(a) \ + rte_atomic64_add(&(a), 1) + +/** + * a: (rte_atomic64_t) + */ +#define FS_ATOMIC_V(a) \ + rte_atomic64_sub(&(a), 1) + +/** + * s: (struct sub_device *) + * i: uint16_t qid + */ +#define FS_ATOMIC_RX(s, i) \ + rte_atomic64_read( \ + &((struct rxq *)((s)->fs_dev->data->rx_queues[i]))->refcnt[(s)->sid] \ + ) +/** + * s: (struct sub_device *) + * i: uint16_t qid + */ +#define FS_ATOMIC_TX(s, i) \ + rte_atomic64_read( \ + &((struct txq *)((s)->fs_dev->data->tx_queues[i]))->refcnt[(s)->sid] \ + ) + +#define LOG__(level, m, ...) \ + RTE_LOG(level, PMD, "net_failsafe: " m "%c", __VA_ARGS__) +#define LOG_(level, ...) LOG__(level, __VA_ARGS__, '\n') +#define DEBUG(...) LOG_(DEBUG, __VA_ARGS__) +#define INFO(...) LOG_(INFO, __VA_ARGS__) +#define WARN(...) LOG_(WARNING, __VA_ARGS__) +#define ERROR(...) LOG_(ERR, __VA_ARGS__) + +/* inlined functions */ + +static inline struct sub_device * +fs_find_next(struct rte_eth_dev *dev, + uint8_t sid, + enum dev_state min_state, + uint8_t *sid_out) +{ + struct sub_device *subs; + uint8_t tail; + + subs = PRIV(dev)->subs; + tail = PRIV(dev)->subs_tail; + while (sid < tail) { + if (subs[sid].state >= min_state) + break; + sid++; + } + *sid_out = sid; + if (sid >= tail) + return NULL; + return &subs[sid]; +} + +/* + * Switch emitting device. + * If banned is set, banned must not be considered for + * the role of emitting device. + */ +static inline void +fs_switch_dev(struct rte_eth_dev *dev, + struct sub_device *banned) +{ + struct sub_device *txd; + enum dev_state req_state; + + req_state = PRIV(dev)->state; + txd = TX_SUBDEV(dev); + if (PREFERRED_SUBDEV(dev)->state >= req_state && + PREFERRED_SUBDEV(dev) != banned) { + if (txd != PREFERRED_SUBDEV(dev) && + (txd == NULL || + (req_state == DEV_STARTED) || + (txd && txd->state < DEV_STARTED))) { + DEBUG("Switching tx_dev to preferred sub_device"); + PRIV(dev)->subs_tx = 0; + } + } else if ((txd && txd->state < req_state) || + txd == NULL || + txd == banned) { + struct sub_device *sdev = NULL; + uint8_t i; + + /* Using acceptable device */ + FOREACH_SUBDEV_STATE(sdev, i, dev, req_state) { + if (sdev == banned) + continue; + DEBUG("Switching tx_dev to sub_device %d", + i); + PRIV(dev)->subs_tx = i; + break; + } + if (i >= PRIV(dev)->subs_tail || sdev == NULL) { + DEBUG("No device ready, deactivating tx_dev"); + PRIV(dev)->subs_tx = PRIV(dev)->subs_tail; + } + } else { + return; + } + set_burst_fn(dev, 0); + rte_wmb(); +} + +#endif /* _RTE_ETH_FAILSAFE_PRIVATE_H_ */ diff --git a/dpdk/drivers/net/failsafe/failsafe_rxtx.c b/dpdk/drivers/net/failsafe/failsafe_rxtx.c new file mode 100644 index 00000000..178294c2 --- /dev/null +++ b/dpdk/drivers/net/failsafe/failsafe_rxtx.c @@ -0,0 +1,204 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of 6WIND S.A. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include + +#include "failsafe_private.h" + +static inline int +fs_rx_unsafe(struct sub_device *sdev) +{ + return (ETH(sdev) == NULL) || + (ETH(sdev)->rx_pkt_burst == NULL) || + (sdev->state != DEV_STARTED) || + (sdev->remove != 0); +} + +static inline int +fs_tx_unsafe(struct sub_device *sdev) +{ + return (sdev == NULL) || + (ETH(sdev) == NULL) || + (ETH(sdev)->tx_pkt_burst == NULL) || + (sdev->state != DEV_STARTED); +} + +void +set_burst_fn(struct rte_eth_dev *dev, int force_safe) +{ + struct sub_device *sdev; + uint8_t i; + int need_safe; + int safe_set; + + need_safe = force_safe; + FOREACH_SUBDEV(sdev, i, dev) + need_safe |= fs_rx_unsafe(sdev); + safe_set = (dev->rx_pkt_burst == &failsafe_rx_burst); + if (need_safe && !safe_set) { + DEBUG("Using safe RX bursts%s", + (force_safe ? " (forced)" : "")); + dev->rx_pkt_burst = &failsafe_rx_burst; + } else if (!need_safe && safe_set) { + DEBUG("Using fast RX bursts"); + dev->rx_pkt_burst = &failsafe_rx_burst_fast; + } + need_safe = force_safe || fs_tx_unsafe(TX_SUBDEV(dev)); + safe_set = (dev->tx_pkt_burst == &failsafe_tx_burst); + if (need_safe && !safe_set) { + DEBUG("Using safe TX bursts%s", + (force_safe ? " (forced)" : "")); + dev->tx_pkt_burst = &failsafe_tx_burst; + } else if (!need_safe && safe_set) { + DEBUG("Using fast TX bursts"); + dev->tx_pkt_burst = &failsafe_tx_burst_fast; + } + rte_wmb(); +} + +uint16_t +failsafe_rx_burst(void *queue, + struct rte_mbuf **rx_pkts, + uint16_t nb_pkts) +{ + struct fs_priv *priv; + struct sub_device *sdev; + struct rxq *rxq; + void *sub_rxq; + uint16_t nb_rx; + uint8_t nb_polled, nb_subs; + uint8_t i; + + rxq = queue; + priv = rxq->priv; + nb_subs = priv->subs_tail - priv->subs_head; + nb_polled = 0; + for (i = rxq->last_polled; nb_polled < nb_subs; nb_polled++) { + i++; + if (i == priv->subs_tail) + i = priv->subs_head; + sdev = &priv->subs[i]; + if (fs_rx_unsafe(sdev)) + continue; + sub_rxq = ETH(sdev)->data->rx_queues[rxq->qid]; + FS_ATOMIC_P(rxq->refcnt[sdev->sid]); + nb_rx = ETH(sdev)-> + rx_pkt_burst(sub_rxq, rx_pkts, nb_pkts); + FS_ATOMIC_V(rxq->refcnt[sdev->sid]); + if (nb_rx) { + rxq->last_polled = i; + return nb_rx; + } + } + return 0; +} + +uint16_t +failsafe_rx_burst_fast(void *queue, + struct rte_mbuf **rx_pkts, + uint16_t nb_pkts) +{ + struct fs_priv *priv; + struct sub_device *sdev; + struct rxq *rxq; + void *sub_rxq; + uint16_t nb_rx; + uint8_t nb_polled, nb_subs; + uint8_t i; + + rxq = queue; + priv = rxq->priv; + nb_subs = priv->subs_tail - priv->subs_head; + nb_polled = 0; + for (i = rxq->last_polled; nb_polled < nb_subs; nb_polled++) { + i++; + if (i == priv->subs_tail) + i = priv->subs_head; + sdev = &priv->subs[i]; + RTE_ASSERT(!fs_rx_unsafe(sdev)); + sub_rxq = ETH(sdev)->data->rx_queues[rxq->qid]; + FS_ATOMIC_P(rxq->refcnt[sdev->sid]); + nb_rx = ETH(sdev)-> + rx_pkt_burst(sub_rxq, rx_pkts, nb_pkts); + FS_ATOMIC_V(rxq->refcnt[sdev->sid]); + if (nb_rx) { + rxq->last_polled = i; + return nb_rx; + } + } + return 0; +} + +uint16_t +failsafe_tx_burst(void *queue, + struct rte_mbuf **tx_pkts, + uint16_t nb_pkts) +{ + struct sub_device *sdev; + struct txq *txq; + void *sub_txq; + uint16_t nb_tx; + + txq = queue; + sdev = TX_SUBDEV(txq->priv->dev); + if (unlikely(fs_tx_unsafe(sdev))) + return 0; + sub_txq = ETH(sdev)->data->tx_queues[txq->qid]; + FS_ATOMIC_P(txq->refcnt[sdev->sid]); + nb_tx = ETH(sdev)->tx_pkt_burst(sub_txq, tx_pkts, nb_pkts); + FS_ATOMIC_V(txq->refcnt[sdev->sid]); + return nb_tx; +} + +uint16_t +failsafe_tx_burst_fast(void *queue, + struct rte_mbuf **tx_pkts, + uint16_t nb_pkts) +{ + struct sub_device *sdev; + struct txq *txq; + void *sub_txq; + uint16_t nb_tx; + + txq = queue; + sdev = TX_SUBDEV(txq->priv->dev); + RTE_ASSERT(!fs_tx_unsafe(sdev)); + sub_txq = ETH(sdev)->data->tx_queues[txq->qid]; + FS_ATOMIC_P(txq->refcnt[sdev->sid]); + nb_tx = ETH(sdev)->tx_pkt_burst(sub_txq, tx_pkts, nb_pkts); + FS_ATOMIC_V(txq->refcnt[sdev->sid]); + return nb_tx; +} diff --git a/dpdk/drivers/net/failsafe/rte_pmd_failsafe_version.map b/dpdk/drivers/net/failsafe/rte_pmd_failsafe_version.map new file mode 100644 index 00000000..b6d2840b --- /dev/null +++ b/dpdk/drivers/net/failsafe/rte_pmd_failsafe_version.map @@ -0,0 +1,4 @@ +DPDK_17.08 { + + local: *; +}; diff --git a/dpdk/drivers/net/fm10k/Makefile b/dpdk/drivers/net/fm10k/Makefile index afcbd1d8..1129596f 100644 --- a/dpdk/drivers/net/fm10k/Makefile +++ b/dpdk/drivers/net/fm10k/Makefile @@ -71,19 +71,26 @@ CFLAGS_BASE_DRIVER += -Wno-missing-field-initializers ifeq ($(shell test $(GCC_VERSION) -ge 44 && echo 1), 1) CFLAGS += -Wno-deprecated CFLAGS_BASE_DRIVER += -Wno-unused-but-set-variable +ifeq ($(shell test $(GCC_VERSION) -ge 70 && echo 1), 1) +CFLAGS_BASE_DRIVER += -Wno-implicit-fallthrough endif endif +endif +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs -lrte_hash +LDLIBS += -lrte_bus_pci # # Add extra flags for base driver source files to disable warnings in them # -BASE_DRIVER_OBJS=$(patsubst %.c,%.o,$(notdir $(wildcard $(SRCDIR)/base/*.c))) +BASE_DRIVER_OBJS=$(sort $(patsubst %.c,%.o,$(notdir $(wildcard $(SRCDIR)/base/*.c)))) $(foreach obj, $(BASE_DRIVER_OBJS), $(eval CFLAGS_$(obj)+=$(CFLAGS_BASE_DRIVER))) VPATH += $(SRCDIR)/base # # all source are stored in SRCS-y +# base driver is based on the package of cid-fm10k.2017.01.24.tar.gz # SRCS-$(CONFIG_RTE_LIBRTE_FM10K_PMD) += fm10k_ethdev.c SRCS-$(CONFIG_RTE_LIBRTE_FM10K_PMD) += fm10k_rxtx.c @@ -96,10 +103,4 @@ SRCS-$(CONFIG_RTE_LIBRTE_FM10K_PMD) += fm10k_vf.c SRCS-$(CONFIG_RTE_LIBRTE_FM10K_PMD) += fm10k_api.c SRCS-$(CONFIG_RTE_LIBRTE_FM10K_INC_VECTOR) += fm10k_rxtx_vec.c -# this lib depends upon: -DEPDIRS-$(CONFIG_RTE_LIBRTE_FM10K_PMD) += lib/librte_eal lib/librte_ether -DEPDIRS-$(CONFIG_RTE_LIBRTE_FM10K_PMD) += lib/librte_mempool lib/librte_mbuf -DEPDIRS-$(CONFIG_RTE_LIBRTE_FM10K_PMD) += lib/librte_net -DEPDIRS-$(CONFIG_RTE_LIBRTE_FM10K_PMD) += lib/librte_kvargs - include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/net/fm10k/base/fm10k_common.c b/dpdk/drivers/net/fm10k/base/fm10k_common.c index a90d2f0b..29f35d7d 100644 --- a/dpdk/drivers/net/fm10k/base/fm10k_common.c +++ b/dpdk/drivers/net/fm10k/base/fm10k_common.c @@ -230,6 +230,9 @@ s32 fm10k_disable_queues_generic(struct fm10k_hw *hw, u16 q_cnt) /* clear tx_ready to prevent any false hits for reset */ hw->mac.tx_ready = false; + if (FM10K_REMOVED(hw->hw_addr)) + return FM10K_SUCCESS; + /* clear the enable bit for all rings */ for (i = 0; i < q_cnt; i++) { reg = FM10K_READ_REG(hw, FM10K_TXDCTL(i)); @@ -542,7 +545,7 @@ s32 fm10k_get_host_state_generic(struct fm10k_hw *hw, bool *host_ready) goto out; /* if we somehow dropped the Tx enable we should reset */ - if (hw->mac.tx_ready && !(txdctl & FM10K_TXDCTL_ENABLE)) { + if (mac->tx_ready && !(txdctl & FM10K_TXDCTL_ENABLE)) { ret_val = FM10K_ERR_RESET_REQUESTED; goto out; } @@ -558,8 +561,12 @@ s32 fm10k_get_host_state_generic(struct fm10k_hw *hw, bool *host_ready) goto out; /* interface cannot receive traffic without logical ports */ - if (mac->dglort_map == FM10K_DGLORTMAP_NONE) + if (mac->dglort_map == FM10K_DGLORTMAP_NONE) { + if (mac->ops.request_lport_map) + ret_val = mac->ops.request_lport_map(hw); + goto out; + } /* if we passed all the tests above then the switch is ready and we no * longer need to check for link diff --git a/dpdk/drivers/net/fm10k/base/fm10k_mbx.c b/dpdk/drivers/net/fm10k/base/fm10k_mbx.c index 2e704340..16ab98d3 100644 --- a/dpdk/drivers/net/fm10k/base/fm10k_mbx.c +++ b/dpdk/drivers/net/fm10k/base/fm10k_mbx.c @@ -2066,9 +2066,10 @@ STATIC void fm10k_sm_mbx_create_reply(struct fm10k_hw *hw, * function can also be used to respond to an error as the connection * resetting would also be a means of dealing with errors. **/ -STATIC void fm10k_sm_mbx_process_reset(struct fm10k_hw *hw, - struct fm10k_mbx_info *mbx) +STATIC s32 fm10k_sm_mbx_process_reset(struct fm10k_hw *hw, + struct fm10k_mbx_info *mbx) { + s32 err = FM10K_SUCCESS; const enum fm10k_mbx_state state = mbx->state; switch (state) { @@ -2081,6 +2082,7 @@ STATIC void fm10k_sm_mbx_process_reset(struct fm10k_hw *hw, case FM10K_STATE_OPEN: /* flush any incomplete work */ fm10k_sm_mbx_connect_reset(mbx); + err = FM10K_ERR_RESET_REQUESTED; break; case FM10K_STATE_CONNECT: /* Update remote value to match local value */ @@ -2090,6 +2092,8 @@ STATIC void fm10k_sm_mbx_process_reset(struct fm10k_hw *hw, } fm10k_sm_mbx_create_reply(hw, mbx, mbx->tail); + + return err; } /** @@ -2172,7 +2176,7 @@ STATIC s32 fm10k_sm_mbx_process(struct fm10k_hw *hw, switch (FM10K_MSG_HDR_FIELD_GET(mbx->mbx_hdr, SM_VER)) { case 0: - fm10k_sm_mbx_process_reset(hw, mbx); + err = fm10k_sm_mbx_process_reset(hw, mbx); break; case FM10K_SM_MBX_VERSION: err = fm10k_sm_mbx_process_version_1(hw, mbx); diff --git a/dpdk/drivers/net/fm10k/base/fm10k_mbx.h b/dpdk/drivers/net/fm10k/base/fm10k_mbx.h index edc57dff..2fac012c 100644 --- a/dpdk/drivers/net/fm10k/base/fm10k_mbx.h +++ b/dpdk/drivers/net/fm10k/base/fm10k_mbx.h @@ -54,6 +54,8 @@ struct fm10k_mbx_info; #define FM10K_MBX_ACK_INTERRUPT 0x00000010 #define FM10K_MBX_INTERRUPT_ENABLE 0x00000020 #define FM10K_MBX_INTERRUPT_DISABLE 0x00000040 +#define FM10K_MBX_GLOBAL_REQ_INTERRUPT 0x00000200 +#define FM10K_MBX_GLOBAL_ACK_INTERRUPT 0x00000400 #define FM10K_MBICR(_n) ((_n) + 0x18840) #define FM10K_GMBX 0x18842 diff --git a/dpdk/drivers/net/fm10k/base/fm10k_osdep.h b/dpdk/drivers/net/fm10k/base/fm10k_osdep.h index a21daa2a..199ebd8e 100644 --- a/dpdk/drivers/net/fm10k/base/fm10k_osdep.h +++ b/dpdk/drivers/net/fm10k/base/fm10k_osdep.h @@ -39,6 +39,8 @@ POSSIBILITY OF SUCH DAMAGE. #include #include #include +#include + #include "../fm10k_logs.h" /* TODO: this does not look like it should be used... */ @@ -88,17 +90,16 @@ typedef int bool; #endif /* offsets are WORD offsets, not BYTE offsets */ -#define FM10K_WRITE_REG(hw, reg, val) \ - ((((volatile uint32_t *)(hw)->hw_addr)[(reg)]) = ((uint32_t)(val))) -#define FM10K_READ_REG(hw, reg) \ - (((volatile uint32_t *)(hw)->hw_addr)[(reg)]) +#define FM10K_WRITE_REG(hw, reg, val) \ + rte_write32((val), ((hw)->hw_addr + (reg))) + +#define FM10K_READ_REG(hw, reg) rte_read32(((hw)->hw_addr + (reg))) + #define FM10K_WRITE_FLUSH(a) FM10K_READ_REG(a, FM10K_CTRL) -#define FM10K_PCI_REG(reg) (*((volatile uint32_t *)(reg))) +#define FM10K_PCI_REG(reg) rte_read32(reg) -#define FM10K_PCI_REG_WRITE(reg, value) do { \ - FM10K_PCI_REG((reg)) = (value); \ -} while (0) +#define FM10K_PCI_REG_WRITE(reg, value) rte_write32((value), (reg)) /* not implemented */ #define FM10K_READ_PCI_WORD(hw, reg) 0 @@ -162,23 +163,6 @@ typedef int bool; #define FM10K_RXD_PKTTYPE_MASK 0x03F0 #define FM10K_RXD_PKTTYPE_SHIFT 4 -enum fm10k_rdesc_pkt_type { - /* L3 type */ - FM10K_PKTTYPE_OTHER = 0x00, - FM10K_PKTTYPE_IPV4 = 0x01, - FM10K_PKTTYPE_IPV4_EX = 0x02, - FM10K_PKTTYPE_IPV6 = 0x03, - FM10K_PKTTYPE_IPV6_EX = 0x04, - - /* L4 type */ - FM10K_PKTTYPE_TCP = 0x08, - FM10K_PKTTYPE_UDP = 0x10, - FM10K_PKTTYPE_GRE = 0x18, - FM10K_PKTTYPE_VXLAN = 0x20, - FM10K_PKTTYPE_NVGRE = 0x28, - FM10K_PKTTYPE_GENEVE = 0x30 -}; - #define FM10K_RXD_STATUS_IPCS 0x0008 /* Indicates IPv4 csum */ #define FM10K_RXD_STATUS_HBO 0x0400 /* header buffer overrun */ diff --git a/dpdk/drivers/net/fm10k/base/fm10k_pf.c b/dpdk/drivers/net/fm10k/base/fm10k_pf.c index 105babf4..db5f4912 100644 --- a/dpdk/drivers/net/fm10k/base/fm10k_pf.c +++ b/dpdk/drivers/net/fm10k/base/fm10k_pf.c @@ -66,21 +66,21 @@ STATIC s32 fm10k_reset_hw_pf(struct fm10k_hw *hw) /* shut down all rings */ err = fm10k_disable_queues_generic(hw, FM10K_MAX_QUEUES); - if (err) + if (err == FM10K_ERR_REQUESTS_PENDING) { + hw->mac.reset_while_pending++; + goto force_reset; + } else if (err) { return err; + } /* Verify that DMA is no longer active */ reg = FM10K_READ_REG(hw, FM10K_DMA_CTRL); if (reg & (FM10K_DMA_CTRL_TX_ACTIVE | FM10K_DMA_CTRL_RX_ACTIVE)) return FM10K_ERR_DMA_PENDING; - /* verify the switch is ready for reset */ - reg = FM10K_READ_REG(hw, FM10K_DMA_CTRL2); - if (!(reg & FM10K_DMA_CTRL2_SWITCH_READY)) - goto out; - +force_reset: /* Inititate data path reset */ - reg |= FM10K_DMA_CTRL_DATAPATH_RESET; + reg = FM10K_DMA_CTRL_DATAPATH_RESET; FM10K_WRITE_REG(hw, FM10K_DMA_CTRL, reg); /* Flush write and allow 100us for reset to complete */ @@ -90,10 +90,9 @@ STATIC s32 fm10k_reset_hw_pf(struct fm10k_hw *hw) /* Verify we made it out of reset */ reg = FM10K_READ_REG(hw, FM10K_IP); if (!(reg & FM10K_IP_NOTINRESET)) - err = FM10K_ERR_RESET_FAILED; + return FM10K_ERR_RESET_FAILED; -out: - return err; + return FM10K_SUCCESS; } /** @@ -255,8 +254,8 @@ STATIC s32 fm10k_update_vlan_pf(struct fm10k_hw *hw, u32 vid, u8 vsi, bool set) /* VLAN multi-bit write: * The multi-bit write has several parts to it. - * 3 2 1 0 - * 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 + * 24 16 8 0 + * 7 6 5 4 3 2 1 0 7 6 5 4 3 2 1 0 7 6 5 4 3 2 1 0 7 6 5 4 3 2 1 0 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ * | RSVD0 | Length |C|RSVD0| VLAN ID | * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ @@ -412,7 +411,7 @@ STATIC s32 fm10k_update_uc_addr_pf(struct fm10k_hw *hw, u16 glort, DEBUGFUNC("fm10k_update_uc_addr_pf"); /* verify MAC address is valid */ - if (!FM10K_IS_VALID_ETHER_ADDR(mac)) + if (!IS_VALID_ETHER_ADDR(mac)) return FM10K_ERR_PARAM; return fm10k_update_xc_addr_pf(hw, glort, mac, vid, add, flags); @@ -435,7 +434,7 @@ STATIC s32 fm10k_update_mc_addr_pf(struct fm10k_hw *hw, u16 glort, DEBUGFUNC("fm10k_update_mc_addr_pf"); /* verify multicast address is valid */ - if (!FM10K_IS_MULTICAST_ETHER_ADDR(mac)) + if (!IS_MULTICAST_ETHER_ADDR(mac)) return FM10K_ERR_PARAM; return fm10k_update_xc_addr_pf(hw, glort, mac, vid, add, 0); @@ -536,6 +535,10 @@ STATIC s32 fm10k_update_lport_state_pf(struct fm10k_hw *hw, u16 glort, if (!fm10k_glort_valid_pf(hw, glort)) return FM10K_ERR_PARAM; + /* reset multicast mode if deleting lport */ + if (!enable) + fm10k_update_xcast_mode_pf(hw, glort, FM10K_XCAST_MODE_NONE); + /* construct the lport message from the 2 pieces of data we have */ lport_msg = ((u32)count << 16) | glort; @@ -908,13 +911,13 @@ STATIC s32 fm10k_iov_assign_default_mac_vlan_pf(struct fm10k_hw *hw, vf_q_idx = fm10k_vf_queue_index(hw, vf_idx); qmap_idx = qmap_stride * vf_idx; - /* MAP Tx queue back to 0 temporarily, and disable it */ - FM10K_WRITE_REG(hw, FM10K_TQMAP(qmap_idx), 0); - FM10K_WRITE_REG(hw, FM10K_TXDCTL(vf_q_idx), 0); - - /* determine correct default VLAN ID */ + /* Determine correct default VLAN ID. The FM10K_VLAN_OVERRIDE bit is + * used here to indicate to the VF that it will not have privilege to + * write VLAN_TABLE. All policy is enforced on the PF but this allows + * the VF to correctly report errors to userspace rqeuests. + */ if (vf_info->pf_vid) - vf_vid = vf_info->pf_vid | FM10K_VLAN_CLEAR; + vf_vid = vf_info->pf_vid | FM10K_VLAN_OVERRIDE; else vf_vid = vf_info->sw_vid; @@ -923,9 +926,35 @@ STATIC s32 fm10k_iov_assign_default_mac_vlan_pf(struct fm10k_hw *hw, fm10k_tlv_attr_put_mac_vlan(msg, FM10K_MAC_VLAN_MSG_DEFAULT_MAC, vf_info->mac, vf_vid); - /* load onto outgoing mailbox, ignore any errors on enqueue */ - if (vf_info->mbx.ops.enqueue_tx) - vf_info->mbx.ops.enqueue_tx(hw, &vf_info->mbx, msg); + /* Configure Queue control register with new VLAN ID. The TXQCTL + * register is RO from the VF, so the PF must do this even in the + * case of notifying the VF of a new VID via the mailbox. + */ + txqctl = ((u32)vf_vid << FM10K_TXQCTL_VID_SHIFT) & + FM10K_TXQCTL_VID_MASK; + txqctl |= (vf_idx << FM10K_TXQCTL_TC_SHIFT) | + FM10K_TXQCTL_VF | vf_idx; + + for (i = 0; i < queues_per_pool; i++) + FM10K_WRITE_REG(hw, FM10K_TXQCTL(vf_q_idx + i), txqctl); + + /* try loading a message onto outgoing mailbox first */ + if (vf_info->mbx.ops.enqueue_tx) { + err = vf_info->mbx.ops.enqueue_tx(hw, &vf_info->mbx, msg); + if (err != FM10K_MBX_ERR_NO_MBX) + return err; + err = FM10K_SUCCESS; + } + + /* If we aren't connected to a mailbox, this is most likely because + * the VF driver is not running. It should thus be safe to re-map + * queues and use the registers to pass the MAC address so that the VF + * driver gets correct information during its initialization. + */ + + /* MAP Tx queue back to 0 temporarily, and disable it */ + FM10K_WRITE_REG(hw, FM10K_TQMAP(qmap_idx), 0); + FM10K_WRITE_REG(hw, FM10K_TXDCTL(vf_q_idx), 0); /* verify ring has disabled before modifying base address registers */ txdctl = FM10K_READ_REG(hw, FM10K_TXDCTL(vf_q_idx)); @@ -941,7 +970,7 @@ STATIC s32 fm10k_iov_assign_default_mac_vlan_pf(struct fm10k_hw *hw, } /* Update base address registers to contain MAC address */ - if (FM10K_IS_VALID_ETHER_ADDR(vf_info->mac)) { + if (IS_VALID_ETHER_ADDR(vf_info->mac)) { tdbal = (((u32)vf_info->mac[3]) << 24) | (((u32)vf_info->mac[4]) << 16) | (((u32)vf_info->mac[5]) << 8); @@ -964,16 +993,6 @@ STATIC s32 fm10k_iov_assign_default_mac_vlan_pf(struct fm10k_hw *hw, FM10K_TDLEN_ITR_SCALE_SHIFT); err_out: - /* configure Queue control register */ - txqctl = ((u32)vf_vid << FM10K_TXQCTL_VID_SHIFT) & - FM10K_TXQCTL_VID_MASK; - txqctl |= (vf_idx << FM10K_TXQCTL_TC_SHIFT) | - FM10K_TXQCTL_VF | vf_idx; - - /* assign VLAN ID */ - for (i = 0; i < queues_per_pool; i++) - FM10K_WRITE_REG(hw, FM10K_TXQCTL(vf_q_idx + i), txqctl); - /* restore the queue back to VF ownership */ FM10K_WRITE_REG(hw, FM10K_TQMAP(qmap_idx), vf_q_idx); return err; @@ -1081,7 +1100,7 @@ STATIC s32 fm10k_iov_reset_resources_pf(struct fm10k_hw *hw, FM10K_WRITE_REG(hw, FM10K_MRQC(vf_info->vsi), 0); /* Update base address registers to contain MAC address */ - if (FM10K_IS_VALID_ETHER_ADDR(vf_info->mac)) { + if (IS_VALID_ETHER_ADDR(vf_info->mac)) { tdbal = (((u32)vf_info->mac[3]) << 24) | (((u32)vf_info->mac[4]) << 16) | (((u32)vf_info->mac[5]) << 8); @@ -1267,18 +1286,32 @@ s32 fm10k_iov_msg_mac_vlan_pf(struct fm10k_hw *hw, u32 **results, if (err) return err; - /* verify upper 16 bits are zero */ - if (vid >> 16) - return FM10K_ERR_PARAM; - set = !(vid & FM10K_VLAN_CLEAR); vid &= ~FM10K_VLAN_CLEAR; - err = fm10k_iov_select_vid(vf_info, (u16)vid); - if (err < 0) - return err; + /* if the length field has been set, this is a multi-bit + * update request. For multi-bit requests, simply disallow + * them when the pf_vid has been set. In this case, the PF + * should have already cleared the VLAN_TABLE, and if we + * allowed them, it could allow a rogue VF to receive traffic + * on a VLAN it was not assigned. In the single-bit case, we + * need to modify requests for VLAN 0 to use the default PF or + * SW vid when assigned. + */ - vid = err; + if (vid >> 16) { + /* prevent multi-bit requests when PF has + * administratively set the VLAN for this VF + */ + if (vf_info->pf_vid) + return FM10K_ERR_PARAM; + } else { + err = fm10k_iov_select_vid(vf_info, (u16)vid); + if (err < 0) + return err; + + vid = err; + } /* update VSI info for VF in regards to VLAN table */ err = hw->mac.ops.update_vlan(hw, vid, vf_info->vsi, set); @@ -1293,7 +1326,7 @@ s32 fm10k_iov_msg_mac_vlan_pf(struct fm10k_hw *hw, u32 **results, return err; /* block attempts to set MAC for a locked device */ - if (FM10K_IS_VALID_ETHER_ADDR(vf_info->mac) && + if (IS_VALID_ETHER_ADDR(vf_info->mac) && memcmp(mac, vf_info->mac, ETH_ALEN)) return FM10K_ERR_PARAM; @@ -1670,13 +1703,12 @@ STATIC s32 fm10k_request_lport_map_pf(struct fm10k_hw *hw) * @hw: pointer to hardware structure * @switch_ready: pointer to boolean value that will record switch state * - * This funciton will check the DMA_CTRL2 register and mailbox in order + * This function will check the DMA_CTRL2 register and mailbox in order * to determine if the switch is ready for the PF to begin requesting * addresses and mapping traffic to the local interface. **/ STATIC s32 fm10k_get_host_state_pf(struct fm10k_hw *hw, bool *switch_ready) { - s32 ret_val = FM10K_SUCCESS; u32 dma_ctrl2; DEBUGFUNC("fm10k_get_host_state_pf"); @@ -1684,23 +1716,16 @@ STATIC s32 fm10k_get_host_state_pf(struct fm10k_hw *hw, bool *switch_ready) /* verify the switch is ready for interaction */ dma_ctrl2 = FM10K_READ_REG(hw, FM10K_DMA_CTRL2); if (!(dma_ctrl2 & FM10K_DMA_CTRL2_SWITCH_READY)) - goto out; + return FM10K_SUCCESS; /* retrieve generic host state info */ - ret_val = fm10k_get_host_state_generic(hw, switch_ready); - if (ret_val) - goto out; - - /* interface cannot receive traffic without logical ports */ - if (hw->mac.dglort_map == FM10K_DGLORTMAP_NONE) - ret_val = fm10k_request_lport_map_pf(hw); - -out: - return ret_val; + return fm10k_get_host_state_generic(hw, switch_ready); } /* This structure defines the attibutes to be parsed below */ const struct fm10k_tlv_attr fm10k_lport_map_msg_attr[] = { + FM10K_TLV_ATTR_LE_STRUCT(FM10K_PF_ATTR_ID_ERR, + sizeof(struct fm10k_swapi_error)), FM10K_TLV_ATTR_U32(FM10K_PF_ATTR_ID_LPORT_MAP), FM10K_TLV_ATTR_LAST }; @@ -2082,6 +2107,7 @@ s32 fm10k_init_ops_pf(struct fm10k_hw *hw) mac->ops.set_dma_mask = &fm10k_set_dma_mask_pf; mac->ops.get_fault = &fm10k_get_fault_pf; mac->ops.get_host_state = &fm10k_get_host_state_pf; + mac->ops.request_lport_map = &fm10k_request_lport_map_pf; mac->ops.adjust_systime = &fm10k_adjust_systime_pf; mac->ops.notify_offset = &fm10k_notify_offset_pf; mac->ops.read_systime = &fm10k_read_systime_pf; diff --git a/dpdk/drivers/net/fm10k/base/fm10k_pf.h b/dpdk/drivers/net/fm10k/base/fm10k_pf.h index c84b1bc5..ca125c27 100644 --- a/dpdk/drivers/net/fm10k/base/fm10k_pf.h +++ b/dpdk/drivers/net/fm10k/base/fm10k_pf.h @@ -91,6 +91,8 @@ enum fm10k_pf_tlv_attr_id_v1 { #define FM10K_MSG_UPDATE_PVID_PVID_SHIFT 16 #define FM10K_MSG_UPDATE_PVID_PVID_SIZE 16 +#define FM10K_MSG_ERR_PEP_NOT_SCHEDULED 280 + /* The following data structures are overlayed directly onto TLV mailbox * messages, and must not break 4 byte alignment. Ensure the structures line * up correctly as per their TLV definition. diff --git a/dpdk/drivers/net/fm10k/base/fm10k_tlv.c b/dpdk/drivers/net/fm10k/base/fm10k_tlv.c index e6150c1d..0328ede2 100644 --- a/dpdk/drivers/net/fm10k/base/fm10k_tlv.c +++ b/dpdk/drivers/net/fm10k/base/fm10k_tlv.c @@ -520,7 +520,8 @@ STATIC s32 fm10k_tlv_attr_validate(u32 *attr, * up into an array of pointers stored in results. The function will * return FM10K_ERR_PARAM on any input or message error, * FM10K_NOT_IMPLEMENTED for any attribute that is outside of the array - * and 0 on success. + * and 0 on success. Any attributes not found in tlv_attr will be silently + * ignored. **/ static s32 fm10k_tlv_attr_parse(u32 *attr, u32 **results, const struct fm10k_tlv_attr *tlv_attr) @@ -559,14 +560,15 @@ static s32 fm10k_tlv_attr_parse(u32 *attr, u32 **results, while (offset < len) { attr_id = *attr & FM10K_TLV_ID_MASK; - if (attr_id < FM10K_TLV_RESULTS_MAX) - err = fm10k_tlv_attr_validate(attr, tlv_attr); - else - err = FM10K_NOT_IMPLEMENTED; + if (attr_id >= FM10K_TLV_RESULTS_MAX) + return FM10K_NOT_IMPLEMENTED; - if (err < 0) + err = fm10k_tlv_attr_validate(attr, tlv_attr); + if (err == FM10K_NOT_IMPLEMENTED) + ; /* silently ignore non-implemented attributes */ + else if (err) return err; - if (!err) + else results[attr_id] = attr; /* update offset */ diff --git a/dpdk/drivers/net/fm10k/base/fm10k_type.h b/dpdk/drivers/net/fm10k/base/fm10k_type.h index 3fc8f136..1f38a02c 100644 --- a/dpdk/drivers/net/fm10k/base/fm10k_type.h +++ b/dpdk/drivers/net/fm10k/base/fm10k_type.h @@ -40,6 +40,7 @@ struct fm10k_hw; #include "fm10k_osdep.h" #include "fm10k_mbx.h" +#define FM10K_INTEL_VENDOR_ID 0x8086 #define FM10K_DEV_ID_PF 0x15A4 #define FM10K_DEV_ID_VF 0x15A5 #ifdef BOULDER_RAPIDS_HW @@ -125,11 +126,15 @@ struct fm10k_hw; /* Interrupt control registers */ #define FM10K_EICR 0x0006 +#define FM10K_EICR_PCA_FAULT 0x00000001 +#define FM10K_EICR_THI_FAULT 0x00000004 +#define FM10K_EICR_FUM_FAULT 0x00000020 #define FM10K_EICR_FAULT_MASK 0x0000003F #define FM10K_EICR_MAILBOX 0x00000040 #define FM10K_EICR_SWITCHREADY 0x00000080 #define FM10K_EICR_SWITCHNOTREADY 0x00000100 #define FM10K_EICR_SWITCHINTERRUPT 0x00000200 +#define FM10K_EICR_SRAMERROR 0x00000400 #define FM10K_EICR_VFLR 0x00000800 #define FM10K_EICR_MAXHOLDTIME 0x00001000 #define FM10K_EIMR 0x0007 @@ -183,6 +188,7 @@ struct fm10k_hw; #define FM10K_DGLORTDEC_INNERRSS_ENABLE 0x08000000 #define FM10K_TUNNEL_CFG 0x0040 #define FM10K_TUNNEL_CFG_NVGRE_SHIFT 16 +#define FM10K_TUNNEL_CFG_GENEVE 0x0041 #define FM10K_SWPRI_MAP(_n) ((_n) + 0x0050) #define FM10K_SWPRI_MAX 16 #define FM10K_RSSRK(_n, _m) (((_n) * 0x10) + (_m) + 0x0800) @@ -211,6 +217,7 @@ struct fm10k_hw; #define FM10K_DMA_CTRL_RX_ENABLE 0x00000010 #define FM10K_DMA_CTRL_RX_ACTIVE 0x00000080 #define FM10K_DMA_CTRL_RX_DESC_SIZE 0x00000100 +#define FM10K_DMA_CTRL_MINMSS_SHIFT 9 #define FM10K_DMA_CTRL_MINMSS_64 0x00008000 #define FM10K_DMA_CTRL_MAX_HOLD_1US_GEN3 0x04800000 #define FM10K_DMA_CTRL_MAX_HOLD_1US_GEN2 0x04000000 @@ -283,6 +290,7 @@ struct fm10k_hw; #define FM10K_RXDCTL_WRITE_BACK_MIN_DELAY 0x00000001 #define FM10K_RXDCTL_DROP_ON_EMPTY 0x00000200 #define FM10K_RXINT(_n) ((0x40 * (_n)) + 0x4008) +#define FM10K_RXINT_TIMER_SHIFT 8 #define FM10K_SRRCTL(_n) ((0x40 * (_n)) + 0x4009) #define FM10K_SRRCTL_BSIZEPKT_SHIFT 8 /* shift _right_ */ #define FM10K_SRRCTL_LOOPBACK_SUPPRESS 0x40000000 @@ -336,6 +344,7 @@ struct fm10k_hw; #define FM10K_TXQCTL_VID_MASK 0x0FFF0000 #define FM10K_TXQCTL_UNLIMITED_BW 0x10000000 #define FM10K_TXINT(_n) ((0x40 * (_n)) + 0x8008) +#define FM10K_TXINT_TIMER_SHIFT 8 /* Tx Statistics */ #define FM10K_QPTC(_n) ((0x40 * (_n)) + 0x8009) @@ -374,6 +383,7 @@ struct fm10k_hw; /* Switch manager interrupt registers */ #define FM10K_IP 0x13000 #define FM10K_IP_NOTINRESET 0x00000100 +#define FM10K_SRAM_IP 0x13003 /* VLAN registers */ #define FM10K_VLAN_TABLE(_n, _m) ((0x80 * (_n)) + (_m) + 0x14000) @@ -384,6 +394,7 @@ struct fm10k_hw; #define FM10K_VLAN_TABLE_VSI_MAX 64 #define FM10K_VLAN_LENGTH_SHIFT 16 #define FM10K_VLAN_CLEAR BIT(15) +#define FM10K_VLAN_OVERRIDE FM10K_VLAN_CLEAR #define FM10K_VLAN_ALL \ ((FM10K_VLAN_TABLE_VID_MAX - 1) << FM10K_VLAN_LENGTH_SHIFT) @@ -422,20 +433,20 @@ struct fm10k_hw; #define ETH_ALEN 6 #endif /* ETH_ALEN */ -#ifndef FM10K_IS_ZERO_ETHER_ADDR +#ifndef IS_ZERO_ETHER_ADDR /* make certain address is not 0 */ -#define FM10K_IS_ZERO_ETHER_ADDR(addr) \ +#define IS_ZERO_ETHER_ADDR(addr) \ (!((addr)[0] | (addr)[1] | (addr)[2] | (addr)[3] | (addr)[4] | (addr)[5])) #endif -#ifndef FM10K_IS_MULTICAST_ETHER_ADDR -#define FM10K_IS_MULTICAST_ETHER_ADDR(addr) ((addr)[0] & 0x1) +#ifndef IS_MULTICAST_ETHER_ADDR +#define IS_MULTICAST_ETHER_ADDR(addr) ((addr)[0] & 0x1) #endif -#ifndef FM10K_IS_VALID_ETHER_ADDR +#ifndef IS_VALID_ETHER_ADDR /* make certain address is not multicast or 0 */ -#define FM10K_IS_VALID_ETHER_ADDR(addr) \ -(!FM10K_IS_MULTICAST_ETHER_ADDR(addr) && !FM10K_IS_ZERO_ETHER_ADDR(addr)) +#define IS_VALID_ETHER_ADDR(addr) \ +(!IS_MULTICAST_ETHER_ADDR(addr) && !IS_ZERO_ETHER_ADDR(addr)) #endif enum fm10k_int_source { @@ -587,6 +598,7 @@ struct fm10k_mac_ops { s32 (*stop_hw)(struct fm10k_hw *); s32 (*get_bus_info)(struct fm10k_hw *); s32 (*get_host_state)(struct fm10k_hw *, bool *); + s32 (*request_lport_map)(struct fm10k_hw *); #ifndef NO_IS_SLOT_APPROPRIATE_CHECK bool (*is_slot_appropriate)(struct fm10k_hw *); #endif @@ -629,6 +641,7 @@ struct fm10k_mac_info { bool tx_ready; u32 dglort_map; u8 itr_scale; + u64 reset_while_pending; }; struct fm10k_swapi_table_info { @@ -676,6 +689,9 @@ struct fm10k_vf_info { u8 vf_flags; /* flags indicating what modes * are supported for the port */ +#ifndef NO_FM10K_VF_TRUSTED_MODE + bool trusted; /* VF trust mode */ +#endif }; #define FM10K_VF_FLAG_ALLMULTI_CAPABLE (u8)(BIT(FM10K_XCAST_MODE_ALLMULTI)) @@ -812,6 +828,24 @@ enum fm10k_rdesc_rss_type { /* Reserved 0x9 - 0xF */ }; +#define FM10K_RXD_PKTTYPE_MASK 0x03F0 +#define FM10K_RXD_PKTTYPE_SHIFT 4 +enum fm10k_rdesc_pkt_type { + /* L3 type */ + FM10K_PKTTYPE_OTHER = 0x00, + FM10K_PKTTYPE_IPV4 = 0x01, + FM10K_PKTTYPE_IPV4_EX = 0x02, + FM10K_PKTTYPE_IPV6 = 0x03, + FM10K_PKTTYPE_IPV6_EX = 0x04, + + /* L4 type */ + FM10K_PKTTYPE_TCP = 0x08, + FM10K_PKTTYPE_UDP = 0x10, + FM10K_PKTTYPE_GRE = 0x18, + FM10K_PKTTYPE_VXLAN = 0x20, + FM10K_PKTTYPE_NVGRE = 0x28, + FM10K_PKTTYPE_GENEVE = 0x30 +}; #define FM10K_RXD_HDR_INFO_XC_MASK 0x0006 enum fm10k_rxdesc_xc { @@ -823,6 +857,7 @@ enum fm10k_rxdesc_xc { #define FM10K_RXD_STATUS_DD 0x0001 /* Descriptor done */ #define FM10K_RXD_STATUS_EOP 0x0002 /* End of packet */ +#define FM10K_RXD_STATUS_IPCS 0x0008 /* Indicates IPv4 csum */ #define FM10K_RXD_STATUS_L4CS 0x0010 /* Indicates an L4 csum */ #define FM10K_RXD_STATUS_L4CS2 0x0040 /* Inner header L4 csum */ #define FM10K_RXD_STATUS_L4E2 0x0800 /* Inner header L4 csum err */ diff --git a/dpdk/drivers/net/fm10k/base/fm10k_vf.c b/dpdk/drivers/net/fm10k/base/fm10k_vf.c index efbdbd1e..bd449773 100644 --- a/dpdk/drivers/net/fm10k/base/fm10k_vf.c +++ b/dpdk/drivers/net/fm10k/base/fm10k_vf.c @@ -49,11 +49,11 @@ STATIC s32 fm10k_stop_hw_vf(struct fm10k_hw *hw) /* we need to disable the queues before taking further steps */ err = fm10k_stop_hw_generic(hw); - if (err) + if (err && err != FM10K_ERR_REQUESTS_PENDING) return err; /* If permanent address is set then we need to restore it */ - if (FM10K_IS_VALID_ETHER_ADDR(perm_addr)) { + if (IS_VALID_ETHER_ADDR(perm_addr)) { bal = (((u32)perm_addr[3]) << 24) | (((u32)perm_addr[4]) << 16) | (((u32)perm_addr[5]) << 8); @@ -82,7 +82,7 @@ STATIC s32 fm10k_stop_hw_vf(struct fm10k_hw *hw) FM10K_WRITE_REG(hw, FM10K_TDLEN(i), tdlen); } - return FM10K_SUCCESS; + return err; } /** @@ -100,7 +100,9 @@ STATIC s32 fm10k_reset_hw_vf(struct fm10k_hw *hw) /* shut down queues we own and reset DMA configuration */ err = fm10k_stop_hw_vf(hw); - if (err) + if (err == FM10K_ERR_REQUESTS_PENDING) + hw->mac.reset_while_pending++; + else if (err) return err; /* Inititate VF reset */ @@ -113,9 +115,9 @@ STATIC s32 fm10k_reset_hw_vf(struct fm10k_hw *hw) /* Clear reset bit and verify it was cleared */ FM10K_WRITE_REG(hw, FM10K_VFCTRL, 0); if (FM10K_READ_REG(hw, FM10K_VFCTRL) & FM10K_VFCTRL_RST) - err = FM10K_ERR_RESET_FAILED; + return FM10K_ERR_RESET_FAILED; - return err; + return FM10K_SUCCESS; } /** @@ -225,7 +227,7 @@ STATIC s32 fm10k_update_vlan_vf(struct fm10k_hw *hw, u32 vid, u8 vsi, bool set) if (vsi) return FM10K_ERR_PARAM; - /* verify upper 4 bits of vid and length are 0 */ + /* clever trick to verify reserved bits in both vid and length */ if ((vid << 16 | vid) >> 28) return FM10K_ERR_PARAM; @@ -268,7 +270,7 @@ s32 fm10k_msg_mac_vlan_vf(struct fm10k_hw *hw, u32 **results, memcpy(hw->mac.perm_addr, perm_addr, ETH_ALEN); hw->mac.default_vid = vid & (FM10K_VLAN_TABLE_VID_MAX - 1); - hw->mac.vlan_override = !!(vid & FM10K_VLAN_CLEAR); + hw->mac.vlan_override = !!(vid & FM10K_VLAN_OVERRIDE); return FM10K_SUCCESS; } @@ -339,11 +341,11 @@ STATIC s32 fm10k_update_uc_addr_vf(struct fm10k_hw *hw, u16 glort, return FM10K_ERR_PARAM; /* verify MAC address is valid */ - if (!FM10K_IS_VALID_ETHER_ADDR(mac)) + if (!IS_VALID_ETHER_ADDR(mac)) return FM10K_ERR_PARAM; /* verify we are not locked down on the MAC address */ - if (FM10K_IS_VALID_ETHER_ADDR(hw->mac.perm_addr) && + if (IS_VALID_ETHER_ADDR(hw->mac.perm_addr) && memcmp(hw->mac.perm_addr, mac, ETH_ALEN)) return FM10K_ERR_PARAM; @@ -385,7 +387,7 @@ STATIC s32 fm10k_update_mc_addr_vf(struct fm10k_hw *hw, u16 glort, return FM10K_ERR_PARAM; /* verify multicast address is valid */ - if (!FM10K_IS_MULTICAST_ETHER_ADDR(mac)) + if (!IS_MULTICAST_ETHER_ADDR(mac)) return FM10K_ERR_PARAM; /* add bit to notify us if this is a set or clear operation */ diff --git a/dpdk/drivers/net/fm10k/fm10k.h b/dpdk/drivers/net/fm10k/fm10k.h index 05aa1a25..1273aa86 100644 --- a/dpdk/drivers/net/fm10k/fm10k.h +++ b/dpdk/drivers/net/fm10k/fm10k.h @@ -69,6 +69,9 @@ #define FM10K_MAX_RX_DESC (FM10K_MAX_RX_RING_SZ / sizeof(union fm10k_rx_desc)) #define FM10K_MAX_TX_DESC (FM10K_MAX_TX_RING_SZ / sizeof(struct fm10k_tx_desc)) +#define FM10K_TX_MAX_SEG UINT8_MAX +#define FM10K_TX_MAX_MTU_SEG UINT8_MAX + /* * byte aligment for HW RX data buffer * Datasheet requires RX buffer addresses shall either be 512-byte aligned or @@ -152,6 +155,7 @@ struct fm10k_dev_info { struct fm10k_macvlan_filter_info macvlan; /* Flag to indicate if RX vector conditions satisfied */ bool rx_vec_allowed; + bool sm_down; }; /* @@ -201,7 +205,7 @@ struct fm10k_rx_queue { uint16_t rxrearm_nb; /* number of remaining to be re-armed */ uint16_t rxrearm_start; /* the idx we start the re-arming from */ uint16_t rx_using_sse; /* indicates that vector RX is in use */ - uint8_t port_id; + uint16_t port_id; uint8_t drop_en; uint8_t rx_deferred_start; /* don't start this queue in dev start. */ uint16_t rx_ftag_en; /* indicates FTAG RX supported */ @@ -238,7 +242,7 @@ struct fm10k_tx_queue { volatile uint32_t *tail_ptr; uint32_t txq_flags; /* Holds flags for this TXq */ uint16_t nb_desc; - uint8_t port_id; + uint16_t port_id; uint8_t tx_deferred_start; /** don't start this queue in dev start. */ uint16_t queue_id; uint16_t tx_ftag_en; /* indicates FTAG TX supported */ @@ -249,11 +253,11 @@ struct fm10k_txq_ops { }; #define MBUF_DMA_ADDR(mb) \ - ((uint64_t) ((mb)->buf_physaddr + (mb)->data_off)) + ((uint64_t) ((mb)->buf_iova + (mb)->data_off)) /* enforce 512B alignment on default Rx DMA addresses */ #define MBUF_DMA_ADDR_DEFAULT(mb) \ - ((uint64_t) RTE_ALIGN(((mb)->buf_physaddr + RTE_PKTMBUF_HEADROOM),\ + ((uint64_t) RTE_ALIGN(((mb)->buf_iova + RTE_PKTMBUF_HEADROOM),\ FM10K_RX_DATABUF_ALIGN)) static inline void fifo_reset(struct fifo *fifo, uint32_t len) @@ -286,7 +290,7 @@ static inline uint16_t fifo_remove(struct fifo *fifo) } static inline void -fm10k_pktmbuf_reset(struct rte_mbuf *mb, uint8_t in_port) +fm10k_pktmbuf_reset(struct rte_mbuf *mb, uint16_t in_port) { rte_mbuf_refcnt_set(mb, 1); mb->next = NULL; @@ -356,14 +360,17 @@ fm10k_dev_rx_descriptor_done(void *rx_queue, uint16_t offset); uint16_t fm10k_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts); +uint16_t fm10k_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts); + int fm10k_rxq_vec_setup(struct fm10k_rx_queue *rxq); int fm10k_rx_vec_condition_check(struct rte_eth_dev *); void fm10k_rx_queue_release_mbufs_vec(struct fm10k_rx_queue *rxq); uint16_t fm10k_recv_pkts_vec(void *, struct rte_mbuf **, uint16_t); uint16_t fm10k_recv_scattered_pkts_vec(void *, struct rte_mbuf **, uint16_t); -uint16_t fm10k_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts, - uint16_t nb_pkts); +uint16_t fm10k_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts); void fm10k_txq_vec_setup(struct fm10k_tx_queue *txq); int fm10k_tx_vec_condition_check(struct fm10k_tx_queue *txq); diff --git a/dpdk/drivers/net/fm10k/fm10k_ethdev.c b/dpdk/drivers/net/fm10k/fm10k_ethdev.c index 35cbe086..58dac389 100644 --- a/dpdk/drivers/net/fm10k/fm10k_ethdev.c +++ b/dpdk/drivers/net/fm10k/fm10k_ethdev.c @@ -32,6 +32,7 @@ */ #include +#include #include #include #include @@ -53,13 +54,13 @@ /* Wait interval to get switch status */ #define WAIT_SWITCH_MSG_US 100000 /* A period of quiescence for switch */ -#define FM10K_SWITCH_QUIESCE_US 10000 +#define FM10K_SWITCH_QUIESCE_US 100000 /* Number of chars per uint32 type */ #define CHARS_PER_UINT32 (sizeof(uint32_t)) #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1) /* default 1:1 map from queue ID to interrupt vector ID */ -#define Q2V(dev, queue_id) (dev->pci_dev->intr_handle.intr_vec[queue_id]) +#define Q2V(pci_dev, queue_id) ((pci_dev)->intr_handle.intr_vec[queue_id]) /* First 64 Logical ports for PF/VMDQ, second 64 for Flow director */ #define MAX_LPORT_NUM 128 @@ -83,6 +84,7 @@ static void fm10k_rx_queue_release(void *queue); static void fm10k_set_rx_function(struct rte_eth_dev *dev); static void fm10k_set_tx_function(struct rte_eth_dev *dev); static int fm10k_check_ftag(struct rte_devargs *devargs); +static int fm10k_link_update(struct rte_eth_dev *dev, int wait_to_complete); struct fm10k_xstats_name_off { char name[RTE_ETH_XSTATS_NAME_SIZE]; @@ -197,9 +199,9 @@ fm10k_tx_vec_condition_check(__rte_unused struct fm10k_tx_queue *txq) } uint16_t __attribute__((weak)) -fm10k_xmit_pkts_vec(__rte_unused void *tx_queue, - __rte_unused struct rte_mbuf **tx_pkts, - __rte_unused uint16_t nb_pkts) +fm10k_xmit_fixed_burst_vec(__rte_unused void *tx_queue, + __rte_unused struct rte_mbuf **tx_pkts, + __rte_unused uint16_t nb_pkts) { return 0; } @@ -677,7 +679,7 @@ fm10k_dev_tx_init(struct rte_eth_dev *dev) /* Enable use of FTAG bit in TX descriptor, PFVTCTL * register is read-only for VF. */ - if (fm10k_check_ftag(dev->pci_dev->devargs)) { + if (fm10k_check_ftag(dev->device->devargs)) { if (hw->mac.type == fm10k_mac_pf) { FM10K_WRITE_REG(hw, FM10K_PFVTCTL(i), FM10K_PFVTCTL_FTAG_DESC_ENABLE); @@ -711,7 +713,8 @@ fm10k_dev_rx_init(struct rte_eth_dev *dev) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct fm10k_macvlan_filter_info *macvlan; - struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle; + struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev); + struct rte_intr_handle *intr_handle = &pdev->intr_handle; int i, ret; struct fm10k_rx_queue *rxq; uint64_t base_addr; @@ -725,13 +728,13 @@ fm10k_dev_rx_init(struct rte_eth_dev *dev) i = 0; if (rte_intr_dp_is_en(intr_handle)) { for (; i < dev->data->nb_rx_queues; i++) { - FM10K_WRITE_REG(hw, FM10K_RXINT(i), Q2V(dev, i)); + FM10K_WRITE_REG(hw, FM10K_RXINT(i), Q2V(pdev, i)); if (hw->mac.type == fm10k_mac_pf) - FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, i)), + FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)), FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR); else - FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, i)), + FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)), FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR); } @@ -1164,6 +1167,8 @@ fm10k_dev_start(struct rte_eth_dev *dev) if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)) fm10k_vlan_filter_set(dev, hw->mac.default_vid, true); + fm10k_link_update(dev, 0); + return 0; } @@ -1171,7 +1176,8 @@ static void fm10k_dev_stop(struct rte_eth_dev *dev) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); - struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle; + struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev); + struct rte_intr_handle *intr_handle = &pdev->intr_handle; int i; PMD_INIT_FUNC_TRACE(); @@ -1190,10 +1196,10 @@ fm10k_dev_stop(struct rte_eth_dev *dev) FM10K_WRITE_REG(hw, FM10K_RXINT(i), 3 << FM10K_RXINT_TIMER_SHIFT); if (hw->mac.type == fm10k_mac_pf) - FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, i)), + FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)), FM10K_ITR_MASK_SET); else - FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, i)), + FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)), FM10K_ITR_MASK_SET); } } @@ -1236,7 +1242,7 @@ fm10k_dev_close(struct rte_eth_dev *dev) MAX_LPORT_NUM, false); fm10k_mbx_unlock(hw); - /* allow 10ms for device to quiesce */ + /* allow 100ms for device to quiesce */ rte_delay_us(FM10K_SWITCH_QUIESCE_US); /* Stop mailbox service first */ @@ -1250,14 +1256,17 @@ static int fm10k_link_update(struct rte_eth_dev *dev, __rte_unused int wait_to_complete) { + struct fm10k_dev_info *dev_info = + FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private); PMD_INIT_FUNC_TRACE(); - /* The host-interface link is always up. The speed is ~50Gbps per Gen3 - * x8 PCIe interface. For now, we leave the speed undefined since there - * is no 50Gbps Ethernet. */ + /* The speed is ~50Gbps per Gen3 x8 PCIe interface. For now, we + * leave the speed undefined since there is no 50Gbps Ethernet. + */ dev->data->dev_link.link_speed = 0; dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX; - dev->data->dev_link.link_status = ETH_LINK_UP; + dev->data->dev_link.link_status = + dev_info->sm_down ? ETH_LINK_DOWN : ETH_LINK_UP; return 0; } @@ -1315,6 +1324,7 @@ fm10k_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, for (i = 0; i < FM10K_NB_HW_XSTATS; i++) { xstats[count].value = *(uint64_t *)(((char *)hw_stats) + fm10k_hw_stats_strings[count].offset); + xstats[count].id = count; count++; } @@ -1324,12 +1334,14 @@ fm10k_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, xstats[count].value = *(uint64_t *)(((char *)&hw_stats->q[q]) + fm10k_hw_stats_rx_q_strings[i].offset); + xstats[count].id = count; count++; } for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) { xstats[count].value = *(uint64_t *)(((char *)&hw_stats->q[q]) + fm10k_hw_stats_tx_q_strings[i].offset); + xstats[count].id = count; count++; } } @@ -1337,7 +1349,7 @@ fm10k_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, return FM10K_NB_XSTATS; } -static void +static int fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) { uint64_t ipackets, opackets, ibytes, obytes; @@ -1367,6 +1379,7 @@ fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) stats->opackets = opackets; stats->ibytes = ibytes; stats->obytes = obytes; + return 0; } static void @@ -1387,16 +1400,18 @@ fm10k_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev); PMD_INIT_FUNC_TRACE(); + dev_info->pci_dev = pdev; dev_info->min_rx_bufsize = FM10K_MIN_RX_BUF_SIZE; dev_info->max_rx_pktlen = FM10K_MAX_PKT_SIZE; dev_info->max_rx_queues = hw->mac.max_queues; dev_info->max_tx_queues = hw->mac.max_queues; dev_info->max_mac_addrs = FM10K_MAX_MACADDR_NUM; dev_info->max_hash_mac_addrs = 0; - dev_info->max_vfs = dev->pci_dev->max_vfs; + dev_info->max_vfs = pdev->max_vfs; dev_info->vmdq_pool_base = 0; dev_info->vmdq_queue_base = 0; dev_info->max_vmdq_pools = ETH_32_POOLS; @@ -1447,6 +1462,8 @@ fm10k_dev_infos_get(struct rte_eth_dev *dev, .nb_max = FM10K_MAX_TX_DESC, .nb_min = FM10K_MIN_TX_DESC, .nb_align = FM10K_MULT_TX_DESC, + .nb_seg_max = FM10K_TX_MAX_SEG, + .nb_mtu_seg_max = FM10K_TX_MAX_MTU_SEG, }; dev_info->speed_capa = ETH_LINK_SPEED_1G | ETH_LINK_SPEED_2_5G | @@ -1577,8 +1594,8 @@ fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) return 0; } -static void -fm10k_vlan_offload_set(__rte_unused struct rte_eth_dev *dev, int mask) +static int +fm10k_vlan_offload_set(struct rte_eth_dev *dev, int mask) { if (mask & ETH_VLAN_STRIP_MASK) { if (!dev->data->dev_conf.rxmode.hw_vlan_strip) @@ -1596,6 +1613,8 @@ fm10k_vlan_offload_set(__rte_unused struct rte_eth_dev *dev, int mask) if (!dev->data->dev_conf.rxmode.hw_vlan_filter) PMD_INIT_LOG(ERR, "VLAN filter is always on in fm10k"); } + + return 0; } /* Add/Remove a MAC address, and update filters to main VSI */ @@ -1680,7 +1699,7 @@ static void fm10k_MAC_filter_set(struct rte_eth_dev *dev, } /* Add a MAC address, and update filters */ -static void +static int fm10k_macaddr_add(struct rte_eth_dev *dev, struct ether_addr *mac_addr, uint32_t index, @@ -1691,6 +1710,7 @@ fm10k_macaddr_add(struct rte_eth_dev *dev, macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private); fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE, pool); macvlan->mac_vmdq_id[index] = pool; + return 0; } /* Remove a MAC address, and update filters */ @@ -1794,7 +1814,8 @@ fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id, const struct rte_eth_rxconf *conf, struct rte_mempool *mp) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); - struct fm10k_dev_info *dev_info = FM10K_DEV_PRIVATE_TO_INFO(dev); + struct fm10k_dev_info *dev_info = + FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private); struct fm10k_rx_queue *q; const struct rte_memzone *mz; @@ -1872,7 +1893,7 @@ fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id, return -ENOMEM; } q->hw_ring = mz->addr; - q->hw_ring_phys_addr = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr); + q->hw_ring_phys_addr = mz->iova; /* Check if number of descs satisfied Vector requirement */ if (!rte_is_power_of_2(nb_desc)) { @@ -2032,7 +2053,7 @@ fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id, return -ENOMEM; } q->hw_ring = mz->addr; - q->hw_ring_phys_addr = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr); + q->hw_ring_phys_addr = mz->iova; /* * allocate memory for the RS bit tracker. Enough slots to hold the @@ -2333,15 +2354,16 @@ static int fm10k_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev); /* Enable ITR */ if (hw->mac.type == fm10k_mac_pf) - FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, queue_id)), + FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)), FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR); else - FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, queue_id)), + FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)), FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR); - rte_intr_enable(&dev->pci_dev->intr_handle); + rte_intr_enable(&pdev->intr_handle); return 0; } @@ -2349,13 +2371,14 @@ static int fm10k_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev); /* Disable ITR */ if (hw->mac.type == fm10k_mac_pf) - FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, queue_id)), + FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)), FM10K_ITR_MASK_SET); else - FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, queue_id)), + FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)), FM10K_ITR_MASK_SET); return 0; } @@ -2364,7 +2387,8 @@ static int fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); - struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle; + struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev); + struct rte_intr_handle *intr_handle = &pdev->intr_handle; uint32_t intr_vector, vec; uint16_t queue_id; int result = 0; @@ -2380,7 +2404,7 @@ fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev) intr_vector = dev->data->nb_rx_queues; /* disable interrupt first */ - rte_intr_disable(&dev->pci_dev->intr_handle); + rte_intr_disable(intr_handle); if (hw->mac.type == fm10k_mac_pf) fm10k_dev_disable_intr_pf(dev); else @@ -2415,7 +2439,7 @@ fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev) fm10k_dev_enable_intr_pf(dev); else fm10k_dev_enable_intr_vf(dev); - rte_intr_enable(&dev->pci_dev->intr_handle); + rte_intr_enable(intr_handle); hw->mac.ops.update_int_moderator(hw); return result; } @@ -2529,13 +2553,15 @@ error: * void */ static void -fm10k_dev_interrupt_handler_pf( - __rte_unused struct rte_intr_handle *handle, - void *param) +fm10k_dev_interrupt_handler_pf(void *param) { struct rte_eth_dev *dev = (struct rte_eth_dev *)param; struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); uint32_t cause, status; + struct fm10k_dev_info *dev_info = + FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private); + int status_mbx; + s32 err; if (hw->mac.type != fm10k_mac_pf) return; @@ -2552,14 +2578,69 @@ fm10k_dev_interrupt_handler_pf( if (cause & FM10K_EICR_SWITCHNOTREADY) PMD_INIT_LOG(ERR, "INT: Switch is not ready"); - if (cause & FM10K_EICR_SWITCHREADY) + if (cause & FM10K_EICR_SWITCHREADY) { PMD_INIT_LOG(INFO, "INT: Switch is ready"); + if (dev_info->sm_down == 1) { + fm10k_mbx_lock(hw); + + /* For recreating logical ports */ + status_mbx = hw->mac.ops.update_lport_state(hw, + hw->mac.dglort_map, MAX_LPORT_NUM, 1); + if (status_mbx == FM10K_SUCCESS) + PMD_INIT_LOG(INFO, + "INT: Recreated Logical port"); + else + PMD_INIT_LOG(INFO, + "INT: Logical ports weren't recreated"); + + status_mbx = hw->mac.ops.update_xcast_mode(hw, + hw->mac.dglort_map, FM10K_XCAST_MODE_NONE); + if (status_mbx != FM10K_SUCCESS) + PMD_INIT_LOG(ERR, "Failed to set XCAST mode"); + + fm10k_mbx_unlock(hw); + + /* first clear the internal SW recording structure */ + if (!(dev->data->dev_conf.rxmode.mq_mode & + ETH_MQ_RX_VMDQ_FLAG)) + fm10k_vlan_filter_set(dev, hw->mac.default_vid, + false); + + fm10k_MAC_filter_set(dev, hw->mac.addr, false, + MAIN_VSI_POOL_NUMBER); + + /* + * Add default mac address and vlan for the logical + * ports that have been created, leave to the + * application to fully recover Rx filtering. + */ + fm10k_MAC_filter_set(dev, hw->mac.addr, true, + MAIN_VSI_POOL_NUMBER); + + if (!(dev->data->dev_conf.rxmode.mq_mode & + ETH_MQ_RX_VMDQ_FLAG)) + fm10k_vlan_filter_set(dev, hw->mac.default_vid, + true); + + dev_info->sm_down = 0; + _rte_eth_dev_callback_process(dev, + RTE_ETH_EVENT_INTR_LSC, + NULL, NULL); + } + } /* Handle mailbox message */ fm10k_mbx_lock(hw); - hw->mbx.ops.process(hw, &hw->mbx); + err = hw->mbx.ops.process(hw, &hw->mbx); fm10k_mbx_unlock(hw); + if (err == FM10K_ERR_RESET_REQUESTED) { + PMD_INIT_LOG(INFO, "INT: Switch is down"); + dev_info->sm_down = 1; + _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, + NULL, NULL); + } + /* Handle SRAM error */ if (cause & FM10K_EICR_SRAMERROR) { PMD_INIT_LOG(ERR, "INT: SRAM error on PEP"); @@ -2581,7 +2662,7 @@ fm10k_dev_interrupt_handler_pf( FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR); /* Re-enable interrupt from host side */ - rte_intr_enable(&(dev->pci_dev->intr_handle)); + rte_intr_enable(dev->intr_handle); } /** @@ -2596,12 +2677,15 @@ fm10k_dev_interrupt_handler_pf( * void */ static void -fm10k_dev_interrupt_handler_vf( - __rte_unused struct rte_intr_handle *handle, - void *param) +fm10k_dev_interrupt_handler_vf(void *param) { struct rte_eth_dev *dev = (struct rte_eth_dev *)param; struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct fm10k_mbx_info *mbx = &hw->mbx; + struct fm10k_dev_info *dev_info = + FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private); + const enum fm10k_mbx_state state = mbx->state; + int status_mbx; if (hw->mac.type != fm10k_mac_vf) return; @@ -2611,11 +2695,54 @@ fm10k_dev_interrupt_handler_vf( hw->mbx.ops.process(hw, &hw->mbx); fm10k_mbx_unlock(hw); + if (state == FM10K_STATE_OPEN && mbx->state == FM10K_STATE_CONNECT) { + PMD_INIT_LOG(INFO, "INT: Switch has gone down"); + + fm10k_mbx_lock(hw); + hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map, + MAX_LPORT_NUM, 1); + fm10k_mbx_unlock(hw); + + /* Setting reset flag */ + dev_info->sm_down = 1; + _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, + NULL, NULL); + } + + if (dev_info->sm_down == 1 && + hw->mac.dglort_map == FM10K_DGLORTMAP_ZERO) { + PMD_INIT_LOG(INFO, "INT: Switch has gone up"); + fm10k_mbx_lock(hw); + status_mbx = hw->mac.ops.update_xcast_mode(hw, + hw->mac.dglort_map, FM10K_XCAST_MODE_NONE); + if (status_mbx != FM10K_SUCCESS) + PMD_INIT_LOG(ERR, "Failed to set XCAST mode"); + fm10k_mbx_unlock(hw); + + /* first clear the internal SW recording structure */ + fm10k_vlan_filter_set(dev, hw->mac.default_vid, false); + fm10k_MAC_filter_set(dev, hw->mac.addr, false, + MAIN_VSI_POOL_NUMBER); + + /* + * Add default mac address and vlan for the logical ports that + * have been created, leave to the application to fully recover + * Rx filtering. + */ + fm10k_MAC_filter_set(dev, hw->mac.addr, true, + MAIN_VSI_POOL_NUMBER); + fm10k_vlan_filter_set(dev, hw->mac.default_vid, true); + + dev_info->sm_down = 0; + _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, + NULL, NULL); + } + /* Re-enable interrupt from device side */ FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR); /* Re-enable interrupt from host side */ - rte_intr_enable(&(dev->pci_dev->intr_handle)); + rte_intr_enable(dev->intr_handle); } /* Mailbox message handler in VF */ @@ -2729,6 +2856,28 @@ fm10k_check_ftag(struct rte_devargs *devargs) return 1; } +static uint16_t +fm10k_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts) +{ + uint16_t nb_tx = 0; + struct fm10k_tx_queue *txq = (struct fm10k_tx_queue *)tx_queue; + + while (nb_pkts) { + uint16_t ret, num; + + num = (uint16_t)RTE_MIN(nb_pkts, txq->rs_thresh); + ret = fm10k_xmit_fixed_burst_vec(tx_queue, &tx_pkts[nb_tx], + num); + nb_tx += ret; + nb_pkts -= ret; + if (ret < num) + break; + } + + return nb_tx; +} + static void __attribute__((cold)) fm10k_set_tx_function(struct rte_eth_dev *dev) { @@ -2737,7 +2886,22 @@ fm10k_set_tx_function(struct rte_eth_dev *dev) int use_sse = 1; uint16_t tx_ftag_en = 0; - if (fm10k_check_ftag(dev->pci_dev->devargs)) + if (rte_eal_process_type() != RTE_PROC_PRIMARY) { + /* primary process has set the ftag flag and txq_flags */ + txq = dev->data->tx_queues[0]; + if (fm10k_tx_vec_condition_check(txq)) { + dev->tx_pkt_burst = fm10k_xmit_pkts; + dev->tx_pkt_prepare = fm10k_prep_pkts; + PMD_INIT_LOG(DEBUG, "Use regular Tx func"); + } else { + PMD_INIT_LOG(DEBUG, "Use vector Tx func"); + dev->tx_pkt_burst = fm10k_xmit_pkts_vec; + dev->tx_pkt_prepare = NULL; + } + return; + } + + if (fm10k_check_ftag(dev->device->devargs)) tx_ftag_en = 1; for (i = 0; i < dev->data->nb_tx_queues; i++) { @@ -2755,8 +2919,10 @@ fm10k_set_tx_function(struct rte_eth_dev *dev) fm10k_txq_vec_setup(txq); } dev->tx_pkt_burst = fm10k_xmit_pkts_vec; + dev->tx_pkt_prepare = NULL; } else { dev->tx_pkt_burst = fm10k_xmit_pkts; + dev->tx_pkt_prepare = fm10k_prep_pkts; PMD_INIT_LOG(DEBUG, "Use regular Tx func"); } } @@ -2764,11 +2930,12 @@ fm10k_set_tx_function(struct rte_eth_dev *dev) static void __attribute__((cold)) fm10k_set_rx_function(struct rte_eth_dev *dev) { - struct fm10k_dev_info *dev_info = FM10K_DEV_PRIVATE_TO_INFO(dev); + struct fm10k_dev_info *dev_info = + FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private); uint16_t i, rx_using_sse; uint16_t rx_ftag_en = 0; - if (fm10k_check_ftag(dev->pci_dev->devargs)) + if (fm10k_check_ftag(dev->device->devargs)) rx_ftag_en = 1; /* In order to allow Vector Rx there are a few configuration @@ -2794,6 +2961,9 @@ fm10k_set_rx_function(struct rte_eth_dev *dev) else PMD_INIT_LOG(DEBUG, "Use regular Rx func"); + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return; + for (i = 0; i < dev->data->nb_rx_queues; i++) { struct fm10k_rx_queue *rxq = dev->data->rx_queues[i]; @@ -2806,7 +2976,8 @@ static void fm10k_params_init(struct rte_eth_dev *dev) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); - struct fm10k_dev_info *info = FM10K_DEV_PRIVATE_TO_INFO(dev); + struct fm10k_dev_info *info = + FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private); /* Inialize bus info. Normally we would call fm10k_get_bus_info(), but * there is no way to get link status without reading BAR4. Until this @@ -2827,6 +2998,8 @@ static int eth_fm10k_dev_init(struct rte_eth_dev *dev) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev); + struct rte_intr_handle *intr_handle = &pdev->intr_handle; int diag, i; struct fm10k_macvlan_filter_info *macvlan; @@ -2835,23 +3008,30 @@ eth_fm10k_dev_init(struct rte_eth_dev *dev) dev->dev_ops = &fm10k_eth_dev_ops; dev->rx_pkt_burst = &fm10k_recv_pkts; dev->tx_pkt_burst = &fm10k_xmit_pkts; + dev->tx_pkt_prepare = &fm10k_prep_pkts; - /* only initialize in the primary process */ - if (rte_eal_process_type() != RTE_PROC_PRIMARY) + /* + * Primary process does the whole initialization, for secondary + * processes, we just select the same Rx and Tx function as primary. + */ + if (rte_eal_process_type() != RTE_PROC_PRIMARY) { + fm10k_set_rx_function(dev); + fm10k_set_tx_function(dev); return 0; + } - rte_eth_copy_pci_info(dev, dev->pci_dev); + rte_eth_copy_pci_info(dev, pdev); macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private); memset(macvlan, 0, sizeof(*macvlan)); /* Vendor and Device ID need to be set before init of shared code */ memset(hw, 0, sizeof(*hw)); - hw->device_id = dev->pci_dev->id.device_id; - hw->vendor_id = dev->pci_dev->id.vendor_id; - hw->subsystem_device_id = dev->pci_dev->id.subsystem_device_id; - hw->subsystem_vendor_id = dev->pci_dev->id.subsystem_vendor_id; + hw->device_id = pdev->id.device_id; + hw->vendor_id = pdev->id.vendor_id; + hw->subsystem_device_id = pdev->id.subsystem_device_id; + hw->subsystem_vendor_id = pdev->id.subsystem_vendor_id; hw->revision_id = 0; - hw->hw_addr = (void *)dev->pci_dev->mem_resource[0].addr; + hw->hw_addr = (void *)pdev->mem_resource[0].addr; if (hw->hw_addr == NULL) { PMD_INIT_LOG(ERR, "Bad mem resource." " Try to blacklist unused devices."); @@ -2921,20 +3101,20 @@ eth_fm10k_dev_init(struct rte_eth_dev *dev) /*PF/VF has different interrupt handling mechanism */ if (hw->mac.type == fm10k_mac_pf) { /* register callback func to eal lib */ - rte_intr_callback_register(&(dev->pci_dev->intr_handle), + rte_intr_callback_register(intr_handle, fm10k_dev_interrupt_handler_pf, (void *)dev); /* enable MISC interrupt */ fm10k_dev_enable_intr_pf(dev); } else { /* VF */ - rte_intr_callback_register(&(dev->pci_dev->intr_handle), + rte_intr_callback_register(intr_handle, fm10k_dev_interrupt_handler_vf, (void *)dev); fm10k_dev_enable_intr_vf(dev); } /* Enable intr after callback registered */ - rte_intr_enable(&(dev->pci_dev->intr_handle)); + rte_intr_enable(intr_handle); hw->mac.ops.update_int_moderator(hw); @@ -3004,7 +3184,8 @@ static int eth_fm10k_dev_uninit(struct rte_eth_dev *dev) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); - + struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev); + struct rte_intr_handle *intr_handle = &pdev->intr_handle; PMD_INIT_FUNC_TRACE(); /* only uninitialize in the primary process */ @@ -3019,7 +3200,7 @@ eth_fm10k_dev_uninit(struct rte_eth_dev *dev) dev->tx_pkt_burst = NULL; /* disable uio/vfio intr */ - rte_intr_disable(&(dev->pci_dev->intr_handle)); + rte_intr_disable(intr_handle); /*PF/VF has different interrupt handling mechanism */ if (hw->mac.type == fm10k_mac_pf) { @@ -3027,13 +3208,13 @@ eth_fm10k_dev_uninit(struct rte_eth_dev *dev) fm10k_dev_disable_intr_pf(dev); /* unregister callback func to eal lib */ - rte_intr_callback_unregister(&(dev->pci_dev->intr_handle), + rte_intr_callback_unregister(intr_handle, fm10k_dev_interrupt_handler_pf, (void *)dev); } else { /* disable interrupt */ fm10k_dev_disable_intr_vf(dev); - rte_intr_callback_unregister(&(dev->pci_dev->intr_handle), + rte_intr_callback_unregister(intr_handle, fm10k_dev_interrupt_handler_vf, (void *)dev); } @@ -3048,6 +3229,18 @@ eth_fm10k_dev_uninit(struct rte_eth_dev *dev) return 0; } +static int eth_fm10k_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, + struct rte_pci_device *pci_dev) +{ + return rte_eth_dev_pci_generic_probe(pci_dev, + sizeof(struct fm10k_adapter), eth_fm10k_dev_init); +} + +static int eth_fm10k_pci_remove(struct rte_pci_device *pci_dev) +{ + return rte_eth_dev_pci_generic_remove(pci_dev, eth_fm10k_dev_uninit); +} + /* * The set of PCI devices this driver supports. This driver will enable both PF * and SRIOV-VF devices. @@ -3059,36 +3252,14 @@ static const struct rte_pci_id pci_id_fm10k_map[] = { { .vendor_id = 0, /* sentinel */ }, }; -static struct eth_driver rte_pmd_fm10k = { - .pci_drv = { - .name = "rte_pmd_fm10k", - .id_table = pci_id_fm10k_map, - .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC | - RTE_PCI_DRV_DETACHABLE, - }, - .eth_dev_init = eth_fm10k_dev_init, - .eth_dev_uninit = eth_fm10k_dev_uninit, - .dev_private_size = sizeof(struct fm10k_adapter), +static struct rte_pci_driver rte_pmd_fm10k = { + .id_table = pci_id_fm10k_map, + .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC | + RTE_PCI_DRV_IOVA_AS_VA, + .probe = eth_fm10k_pci_probe, + .remove = eth_fm10k_pci_remove, }; -/* - * Driver initialization routine. - * Invoked once at EAL init time. - * Register itself as the [Poll Mode] Driver of PCI FM10K devices. - */ -static int -rte_pmd_fm10k_init(__rte_unused const char *name, - __rte_unused const char *params) -{ - PMD_INIT_FUNC_TRACE(); - rte_eth_driver_register(&rte_pmd_fm10k); - return 0; -} - -static struct rte_driver rte_fm10k_driver = { - .type = PMD_PDEV, - .init = rte_pmd_fm10k_init, -}; - -PMD_REGISTER_DRIVER(rte_fm10k_driver, fm10k); -DRIVER_REGISTER_PCI_TABLE(fm10k, pci_id_fm10k_map); +RTE_PMD_REGISTER_PCI(net_fm10k, rte_pmd_fm10k); +RTE_PMD_REGISTER_PCI_TABLE(net_fm10k, pci_id_fm10k_map); +RTE_PMD_REGISTER_KMOD_DEP(net_fm10k, "* igb_uio | uio_pci_generic | vfio-pci"); diff --git a/dpdk/drivers/net/fm10k/fm10k_rxtx.c b/dpdk/drivers/net/fm10k/fm10k_rxtx.c index bf5888b0..d6081e48 100644 --- a/dpdk/drivers/net/fm10k/fm10k_rxtx.c +++ b/dpdk/drivers/net/fm10k/fm10k_rxtx.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2013-2015 Intel Corporation. All rights reserved. + * Copyright(c) 2013-2016 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -35,6 +35,7 @@ #include #include +#include #include "fm10k.h" #include "base/fm10k_type.h" @@ -65,6 +66,15 @@ static inline void dump_rxd(union fm10k_rx_desc *rxd) } #endif +#define FM10K_TX_OFFLOAD_MASK ( \ + PKT_TX_VLAN_PKT | \ + PKT_TX_IP_CKSUM | \ + PKT_TX_L4_MASK | \ + PKT_TX_TCP_SEG) + +#define FM10K_TX_OFFLOAD_NOTSUP_MASK \ + (PKT_TX_OFFLOAD_MASK ^ FM10K_TX_OFFLOAD_MASK) + /* @note: When this function is changed, make corresponding change to * fm10k_dev_supported_ptypes_get() */ @@ -101,11 +111,15 @@ rx_desc_to_ol_flags(struct rte_mbuf *m, const union fm10k_rx_desc *d) (FM10K_RXD_STATUS_IPCS | FM10K_RXD_STATUS_IPE)) == (FM10K_RXD_STATUS_IPCS | FM10K_RXD_STATUS_IPE))) m->ol_flags |= PKT_RX_IP_CKSUM_BAD; + else + m->ol_flags |= PKT_RX_IP_CKSUM_GOOD; if (unlikely((d->d.staterr & (FM10K_RXD_STATUS_L4CS | FM10K_RXD_STATUS_L4E)) == (FM10K_RXD_STATUS_L4CS | FM10K_RXD_STATUS_L4E))) m->ol_flags |= PKT_RX_L4_CKSUM_BAD; + else + m->ol_flags |= PKT_RX_L4_CKSUM_GOOD; } uint16_t @@ -144,10 +158,10 @@ fm10k_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, * Packets in fm10k device always carry at least one VLAN tag. * For those packets coming in without VLAN tag, * the port default VLAN tag will be used. - * So, always PKT_RX_VLAN_PKT flag is set and vlan_tci + * So, always PKT_RX_VLAN flag is set and vlan_tci * is valid for each RX packet's mbuf. */ - mbuf->ol_flags |= PKT_RX_VLAN_PKT; + mbuf->ol_flags |= PKT_RX_VLAN; mbuf->vlan_tci = desc.w.vlan; /** * mbuf->vlan_tci_outer is an idle field in fm10k driver, @@ -184,7 +198,7 @@ fm10k_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, q->alloc_thresh); if (unlikely(ret != 0)) { - uint8_t port = q->port_id; + uint16_t port = q->port_id; PMD_RX_LOG(ERR, "Failed to alloc mbuf"); /* * Need to restore next_dd if we cannot allocate new @@ -305,10 +319,10 @@ fm10k_recv_scattered_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, * Packets in fm10k device always carry at least one VLAN tag. * For those packets coming in without VLAN tag, * the port default VLAN tag will be used. - * So, always PKT_RX_VLAN_PKT flag is set and vlan_tci + * So, always PKT_RX_VLAN flag is set and vlan_tci * is valid for each RX packet's mbuf. */ - first_seg->ol_flags |= PKT_RX_VLAN_PKT; + first_seg->ol_flags |= PKT_RX_VLAN; first_seg->vlan_tci = desc.w.vlan; /** * mbuf->vlan_tci_outer is an idle field in fm10k driver, @@ -342,7 +356,7 @@ fm10k_recv_scattered_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, q->alloc_thresh); if (unlikely(ret != 0)) { - uint8_t port = q->port_id; + uint16_t port = q->port_id; PMD_RX_LOG(ERR, "Failed to alloc mbuf"); /* * Need to restore next_dd if we cannot allocate new @@ -420,12 +434,12 @@ static inline void tx_free_bulk_mbuf(struct rte_mbuf **txep, int num) if (unlikely(num == 0)) return; - m = __rte_pktmbuf_prefree_seg(txep[0]); + m = rte_pktmbuf_prefree_seg(txep[0]); if (likely(m != NULL)) { free[0] = m; nb_free = 1; for (i = 1; i < num; i++) { - m = __rte_pktmbuf_prefree_seg(txep[i]); + m = rte_pktmbuf_prefree_seg(txep[i]); if (likely(m != NULL)) { if (likely(m->pool == free[0]->pool)) free[nb_free++] = m; @@ -441,7 +455,7 @@ static inline void tx_free_bulk_mbuf(struct rte_mbuf **txep, int num) rte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free); } else { for (i = 1; i < num; i++) { - m = __rte_pktmbuf_prefree_seg(txep[i]); + m = rte_pktmbuf_prefree_seg(txep[i]); if (m != NULL) rte_mempool_put(m->pool, m); txep[i] = NULL; @@ -593,3 +607,41 @@ fm10k_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, return count; } + +uint16_t +fm10k_prep_pkts(__rte_unused void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts) +{ + int i, ret; + struct rte_mbuf *m; + + for (i = 0; i < nb_pkts; i++) { + m = tx_pkts[i]; + + if ((m->ol_flags & PKT_TX_TCP_SEG) && + (m->tso_segsz < FM10K_TSO_MINMSS)) { + rte_errno = -EINVAL; + return i; + } + + if (m->ol_flags & FM10K_TX_OFFLOAD_NOTSUP_MASK) { + rte_errno = -ENOTSUP; + return i; + } + +#ifdef RTE_LIBRTE_ETHDEV_DEBUG + ret = rte_validate_tx_offload(m); + if (ret != 0) { + rte_errno = ret; + return i; + } +#endif + ret = rte_net_intel_cksum_prepare(m); + if (ret != 0) { + rte_errno = ret; + return i; + } + } + + return i; +} diff --git a/dpdk/drivers/net/fm10k/fm10k_rxtx_vec.c b/dpdk/drivers/net/fm10k/fm10k_rxtx_vec.c index c9a49e36..ce042d3d 100644 --- a/dpdk/drivers/net/fm10k/fm10k_rxtx_vec.c +++ b/dpdk/drivers/net/fm10k/fm10k_rxtx_vec.c @@ -67,6 +67,8 @@ fm10k_reset_tx_queue(struct fm10k_tx_queue *txq); #define RXEFLAG_SHIFT (13) /* IPE/L4E flag shift */ #define L3L4EFLAG_SHIFT (14) +/* shift PKT_RX_L4_CKSUM_GOOD into one byte by 1 bit */ +#define CKSUM_SHIFT (1) static inline void fm10k_desc_to_olflags_v(__m128i descs[4], struct rte_mbuf **rx_pkts) @@ -79,8 +81,8 @@ fm10k_desc_to_olflags_v(__m128i descs[4], struct rte_mbuf **rx_pkts) const __m128i pkttype_msk = _mm_set_epi16( 0x0000, 0x0000, 0x0000, 0x0000, - PKT_RX_VLAN_PKT, PKT_RX_VLAN_PKT, - PKT_RX_VLAN_PKT, PKT_RX_VLAN_PKT); + PKT_RX_VLAN, PKT_RX_VLAN, + PKT_RX_VLAN, PKT_RX_VLAN); /* mask everything except rss type */ const __m128i rsstype_msk = _mm_set_epi16( @@ -92,11 +94,18 @@ fm10k_desc_to_olflags_v(__m128i descs[4], struct rte_mbuf **rx_pkts) 0x0000, 0x0000, 0x0000, 0x0000, 0x0001, 0x0001, 0x0001, 0x0001); + /* mask the lower byte of ol_flags */ + const __m128i ol_flags_msk = _mm_set_epi16( + 0x0000, 0x0000, 0x0000, 0x0000, + 0x00FF, 0x00FF, 0x00FF, 0x00FF); + const __m128i l3l4cksum_flag = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - PKT_RX_IP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD, - PKT_RX_IP_CKSUM_BAD, PKT_RX_L4_CKSUM_BAD, 0); + (PKT_RX_IP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD) >> CKSUM_SHIFT, + (PKT_RX_IP_CKSUM_BAD | PKT_RX_L4_CKSUM_GOOD) >> CKSUM_SHIFT, + (PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_BAD) >> CKSUM_SHIFT, + (PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD) >> CKSUM_SHIFT); const __m128i rxe_flag = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, @@ -139,6 +148,10 @@ fm10k_desc_to_olflags_v(__m128i descs[4], struct rte_mbuf **rx_pkts) /* Process L4/L3 checksum error flags */ cksumflag = _mm_srli_epi16(cksumflag, L3L4EFLAG_SHIFT); cksumflag = _mm_shuffle_epi8(l3l4cksum_flag, cksumflag); + + /* clean the higher byte and shift back the flag bits */ + cksumflag = _mm_and_si128(cksumflag, ol_flags_msk); + cksumflag = _mm_slli_epi16(cksumflag, CKSUM_SHIFT); vtag1 = _mm_or_si128(cksumflag, vtag1); vol.dword = _mm_cvtsi128_si64(vtag1); @@ -234,11 +247,8 @@ fm10k_rx_vec_condition_check(struct rte_eth_dev *dev) if (fconf->mode != RTE_FDIR_MODE_NONE) return -1; - /* - no csum error report support - * - no header split support - */ - if (rxmode->hw_ip_checksum == 1 || - rxmode->header_split == 1) + /* no header split support */ + if (rxmode->header_split == 1) return -1; return 0; @@ -314,16 +324,15 @@ fm10k_rxq_rearm(struct fm10k_rx_queue *rxq) /* Flush mbuf with pkt template. * Data to be rearmed is 6 bytes long. - * Though, RX will overwrite ol_flags that are coming next - * anyway. So overwrite whole 8 bytes with one load: - * 6 bytes of rearm_data plus first 2 bytes of ol_flags. */ p0 = (uintptr_t)&mb0->rearm_data; *(uint64_t *)p0 = rxq->mbuf_initializer; p1 = (uintptr_t)&mb1->rearm_data; *(uint64_t *)p1 = rxq->mbuf_initializer; - /* load buf_addr(lo 64bit) and buf_physaddr(hi 64bit) */ + /* load buf_addr(lo 64bit) and buf_iova(hi 64bit) */ + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, buf_iova) != + offsetof(struct rte_mbuf, buf_addr) + 8); vaddr0 = _mm_loadu_si128((__m128i *)&mb0->buf_addr); vaddr1 = _mm_loadu_si128((__m128i *)&mb1->buf_addr); @@ -406,7 +415,7 @@ fm10k_recv_raw_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, */ rxdp = rxq->hw_ring + next_dd; - _mm_prefetch((const void *)rxdp, _MM_HINT_T0); + rte_prefetch0(rxdp); /* See if we need to rearm the RX queue - gives the prefetch a bit * of time to act @@ -441,6 +450,19 @@ fm10k_recv_raw_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, 0xFF, 0xFF, /* skip high 16 bits pkt_type */ 0xFF, 0xFF /* Skip pkt_type field in shuffle operation */ ); + /* + * Compile-time verify the shuffle mask + * NOTE: some field positions already verified above, but duplicated + * here for completeness in case of future modifications. + */ + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, pkt_len) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 4); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_len) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 8); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, vlan_tci) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 10); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, hash) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 12); /* Cache is empty -> need to scan the buffer rings, but first move * the next 'n' mbufs into the cache @@ -460,9 +482,13 @@ fm10k_recv_raw_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, __m128i descs0[RTE_FM10K_DESCS_PER_LOOP]; __m128i pkt_mb1, pkt_mb2, pkt_mb3, pkt_mb4; __m128i zero, staterr, sterr_tmp1, sterr_tmp2; - __m128i mbp1, mbp2; /* two mbuf pointer in one XMM reg. */ + __m128i mbp1; + /* 2 64 bit or 4 32 bit mbuf pointers in one XMM reg. */ +#if defined(RTE_ARCH_X86_64) + __m128i mbp2; +#endif - /* B.1 load 1 mbuf point */ + /* B.1 load 2 (64 bit) or 4 (32 bit) mbuf points */ mbp1 = _mm_loadu_si128((__m128i *)&mbufp[pos]); /* Read desc statuses backwards to avoid race condition */ @@ -470,11 +496,13 @@ fm10k_recv_raw_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, descs0[3] = _mm_loadu_si128((__m128i *)(rxdp + 3)); rte_compiler_barrier(); - /* B.2 copy 2 mbuf point into rx_pkts */ + /* B.2 copy 2 64 bit or 4 32 bit mbuf point into rx_pkts */ _mm_storeu_si128((__m128i *)&rx_pkts[pos], mbp1); - /* B.1 load 1 mbuf point */ +#if defined(RTE_ARCH_X86_64) + /* B.1 load 2 64 bit mbuf poitns */ mbp2 = _mm_loadu_si128((__m128i *)&mbufp[pos+2]); +#endif descs0[2] = _mm_loadu_si128((__m128i *)(rxdp + 2)); rte_compiler_barrier(); @@ -483,8 +511,10 @@ fm10k_recv_raw_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, rte_compiler_barrier(); descs0[0] = _mm_loadu_si128((__m128i *)(rxdp)); +#if defined(RTE_ARCH_X86_64) /* B.2 copy 2 mbuf point into rx_pkts */ _mm_storeu_si128((__m128i *)&rx_pkts[pos+2], mbp2); +#endif /* avoid compiler reorder optimization */ rte_compiler_barrier(); @@ -723,7 +753,7 @@ vtx(volatile struct fm10k_tx_desc *txdp, vtx1(txdp, *pkt, flags); } -static inline int __attribute__((always_inline)) +static __rte_always_inline int fm10k_tx_free_bufs(struct fm10k_tx_queue *txq) { struct rte_mbuf **txep; @@ -744,12 +774,12 @@ fm10k_tx_free_bufs(struct fm10k_tx_queue *txq) * next_dd - (rs_thresh-1) */ txep = &txq->sw_ring[txq->next_dd - (n - 1)]; - m = __rte_pktmbuf_prefree_seg(txep[0]); + m = rte_pktmbuf_prefree_seg(txep[0]); if (likely(m != NULL)) { free[0] = m; nb_free = 1; for (i = 1; i < n; i++) { - m = __rte_pktmbuf_prefree_seg(txep[i]); + m = rte_pktmbuf_prefree_seg(txep[i]); if (likely(m != NULL)) { if (likely(m->pool == free[0]->pool)) free[nb_free++] = m; @@ -764,7 +794,7 @@ fm10k_tx_free_bufs(struct fm10k_tx_queue *txq) rte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free); } else { for (i = 1; i < n; i++) { - m = __rte_pktmbuf_prefree_seg(txep[i]); + m = rte_pktmbuf_prefree_seg(txep[i]); if (m != NULL) rte_mempool_put(m->pool, m); } @@ -779,7 +809,7 @@ fm10k_tx_free_bufs(struct fm10k_tx_queue *txq) return txq->rs_thresh; } -static inline void __attribute__((always_inline)) +static __rte_always_inline void tx_backlog_entry(struct rte_mbuf **txep, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { @@ -790,8 +820,8 @@ tx_backlog_entry(struct rte_mbuf **txep, } uint16_t -fm10k_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts, - uint16_t nb_pkts) +fm10k_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts) { struct fm10k_tx_queue *txq = (struct fm10k_tx_queue *)tx_queue; volatile struct fm10k_tx_desc *txdp; diff --git a/dpdk/drivers/net/i40e/Makefile b/dpdk/drivers/net/i40e/Makefile index 53fe145f..9ab8c84d 100644 --- a/dpdk/drivers/net/i40e/Makefile +++ b/dpdk/drivers/net/i40e/Makefile @@ -1,6 +1,6 @@ # BSD LICENSE # -# Copyright(c) 2010-2015 Intel Corporation. All rights reserved. +# Copyright(c) 2010-2017 Intel Corporation. All rights reserved. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -38,11 +38,14 @@ LIB = librte_pmd_i40e.a CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) -DPF_DRIVER -DVF_DRIVER -DINTEGRATED_VF -CFLAGS += -DX722_SUPPORT -DX722_A0_SUPPORT +CFLAGS += -DX722_A0_SUPPORT +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs -lrte_hash +LDLIBS += -lrte_bus_pci EXPORT_MAP := rte_pmd_i40e_version.map -LIBABIVER := 1 +LIBABIVER := 2 # # Add extra flags for base driver files (also known as shared code) @@ -78,7 +81,7 @@ endif CFLAGS_i40e_lan_hmc.o += -Wno-error endif -OBJS_BASE_DRIVER=$(patsubst %.c,%.o,$(notdir $(wildcard $(SRCDIR)/base/*.c))) +OBJS_BASE_DRIVER=$(sort $(patsubst %.c,%.o,$(notdir $(wildcard $(SRCDIR)/base/*.c)))) $(foreach obj, $(OBJS_BASE_DRIVER), $(eval CFLAGS_$(obj)+=$(CFLAGS_BASE_DRIVER))) VPATH += $(SRCDIR)/base @@ -97,21 +100,21 @@ SRCS-$(CONFIG_RTE_LIBRTE_I40E_PMD) += i40e_dcb.c SRCS-$(CONFIG_RTE_LIBRTE_I40E_PMD) += i40e_ethdev.c SRCS-$(CONFIG_RTE_LIBRTE_I40E_PMD) += i40e_rxtx.c -SRCS-$(CONFIG_RTE_LIBRTE_I40E_INC_VECTOR) += i40e_rxtx_vec.c +ifeq ($(CONFIG_RTE_ARCH_ARM64),y) +SRCS-$(CONFIG_RTE_LIBRTE_I40E_INC_VECTOR) += i40e_rxtx_vec_neon.c +else ifeq ($(CONFIG_RTE_ARCH_PPC_64),y) +SRCS-$(CONFIG_RTE_LIBRTE_I40E_INC_VECTOR) += i40e_rxtx_vec_altivec.c +else +SRCS-$(CONFIG_RTE_LIBRTE_I40E_INC_VECTOR) += i40e_rxtx_vec_sse.c +endif SRCS-$(CONFIG_RTE_LIBRTE_I40E_PMD) += i40e_ethdev_vf.c SRCS-$(CONFIG_RTE_LIBRTE_I40E_PMD) += i40e_pf.c SRCS-$(CONFIG_RTE_LIBRTE_I40E_PMD) += i40e_fdir.c +SRCS-$(CONFIG_RTE_LIBRTE_I40E_PMD) += i40e_flow.c +SRCS-$(CONFIG_RTE_LIBRTE_I40E_PMD) += rte_pmd_i40e.c +SRCS-$(CONFIG_RTE_LIBRTE_I40E_PMD) += i40e_tm.c -# vector PMD driver needs SSE4.1 support -ifeq ($(findstring RTE_MACHINE_CPUFLAG_SSE4_1,$(CFLAGS)),) -CFLAGS_i40e_rxtx_vec.o += -msse4.1 -endif - - -# this lib depends upon: -DEPDIRS-$(CONFIG_RTE_LIBRTE_I40E_PMD) += lib/librte_eal lib/librte_ether -DEPDIRS-$(CONFIG_RTE_LIBRTE_I40E_PMD) += lib/librte_mempool lib/librte_mbuf -DEPDIRS-$(CONFIG_RTE_LIBRTE_I40E_PMD) += lib/librte_net -DEPDIRS-$(CONFIG_RTE_LIBRTE_I40E_PMD) += lib/librte_kvargs +# install this header file +SYMLINK-$(CONFIG_RTE_LIBRTE_I40E_PMD)-include := rte_pmd_i40e.h include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/net/i40e/base/README b/dpdk/drivers/net/i40e/base/README new file mode 100644 index 00000000..59e76c21 --- /dev/null +++ b/dpdk/drivers/net/i40e/base/README @@ -0,0 +1,59 @@ +.. + BSD LICENSE + + Copyright(c) 2017 Intel Corporation. All rights reserved. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Intel® I40E driver +================== + +This directory contains source code of FreeBSD i40e driver of version +cid-i40e.2017.06.23.tar.gz released by the team which develops +basic drivers for any i40e NIC. The directory of base/ contains the +original source package. +This driver is valid for the product(s) listed below + +* Intel® Ethernet Converged Network Adapters X710 +* Intel® Ethernet Converged Network Adapters XL710 +* Intel® Ethernet Network Adapter XXV710 +* Intel® Ethernet Connection X722 for 10GBASE-T +* Intel® Ethernet Connection X722 for 10GbE backplane +* Intel® Ethernet Connection X722 for 10GbE SFP+ +* Intel® Ethernet Connection X722 for 1GbE +* Intel® Ethernet Controller X710 and XL710 Family +* Intel® Ethernet Controller XXV710 for 25GbE backplane +* Intel® Ethernet Controller XXV710 for 25GbE SFP28 + +Updating the driver +=================== + +NOTE: The source code in this directory should not be modified apart from +the following file(s): + + i40e_osdep.h diff --git a/dpdk/drivers/net/i40e/base/i40e_adminq.c b/dpdk/drivers/net/i40e/base/i40e_adminq.c index 0d3a83fa..8cc8c5ec 100644 --- a/dpdk/drivers/net/i40e/base/i40e_adminq.c +++ b/dpdk/drivers/net/i40e/base/i40e_adminq.c @@ -682,6 +682,18 @@ enum i40e_status_code i40e_init_adminq(struct i40e_hw *hw) &oem_lo); hw->nvm.oem_ver = ((u32)oem_hi << 16) | oem_lo; + /* The ability to RX (not drop) 802.1ad frames was added in API 1.7 */ + if ((hw->aq.api_maj_ver > 1) || + ((hw->aq.api_maj_ver == 1) && + (hw->aq.api_min_ver >= 7))) + hw->flags |= I40E_HW_FLAG_802_1AD_CAPABLE; + + if (hw->mac.type == I40E_MAC_XL710 && + hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR && + hw->aq.api_min_ver >= I40E_MINOR_VER_GET_LINK_INFO_XL710) { + hw->flags |= I40E_HW_FLAG_AQ_PHY_ACCESS_CAPABLE; + } + if (hw->aq.api_maj_ver > I40E_FW_API_VERSION_MAJOR) { ret_code = I40E_ERR_FIRMWARE_API_VERSION; goto init_adminq_free_arq; @@ -944,8 +956,8 @@ enum i40e_status_code i40e_asq_send_command(struct i40e_hw *hw, */ if (i40e_asq_done(hw)) break; - i40e_msec_delay(1); - total_delay++; + i40e_usec_delay(50); + total_delay += 50; } while (total_delay < hw->aq.asq_cmd_timeout); } @@ -1077,11 +1089,11 @@ enum i40e_status_code i40e_clean_arq_element(struct i40e_hw *hw, desc = I40E_ADMINQ_DESC(hw->aq.arq, ntc); desc_idx = ntc; + hw->aq.arq_last_status = + (enum i40e_admin_queue_err)LE16_TO_CPU(desc->retval); flags = LE16_TO_CPU(desc->flags); if (flags & I40E_AQ_FLAG_ERR) { ret_code = I40E_ERR_ADMIN_QUEUE_ERROR; - hw->aq.arq_last_status = - (enum i40e_admin_queue_err)LE16_TO_CPU(desc->retval); i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE, "AQRX: Event received with error 0x%X.\n", diff --git a/dpdk/drivers/net/i40e/base/i40e_adminq.h b/dpdk/drivers/net/i40e/base/i40e_adminq.h index 750973c5..182e40b9 100644 --- a/dpdk/drivers/net/i40e/base/i40e_adminq.h +++ b/dpdk/drivers/net/i40e/base/i40e_adminq.h @@ -158,9 +158,9 @@ STATIC INLINE int i40e_aq_rc_to_posix(int aq_ret, int aq_rc) /* general information */ #define I40E_AQ_LARGE_BUF 512 -#define I40E_ASQ_CMD_TIMEOUT 250 /* msecs */ +#define I40E_ASQ_CMD_TIMEOUT 250000 /* usecs */ #ifdef I40E_ESS_SUPPORT -#define I40E_ASQ_CMD_TIMEOUT_ESS 50000 /* msecs */ +#define I40E_ASQ_CMD_TIMEOUT_ESS 50000000 /* usecs */ #endif void i40e_fill_default_direct_cmd_desc(struct i40e_aq_desc *desc, diff --git a/dpdk/drivers/net/i40e/base/i40e_adminq_cmd.h b/dpdk/drivers/net/i40e/base/i40e_adminq_cmd.h index 2b7a7608..c36da2a3 100644 --- a/dpdk/drivers/net/i40e/base/i40e_adminq_cmd.h +++ b/dpdk/drivers/net/i40e/base/i40e_adminq_cmd.h @@ -41,7 +41,15 @@ POSSIBILITY OF SUCH DAMAGE. */ #define I40E_FW_API_VERSION_MAJOR 0x0001 -#define I40E_FW_API_VERSION_MINOR 0x0005 +#define I40E_FW_API_VERSION_MINOR_X722 0x0005 +#define I40E_FW_API_VERSION_MINOR_X710 0x0007 + +#define I40E_FW_MINOR_VERSION(_h) ((_h)->mac.type == I40E_MAC_XL710 ? \ + I40E_FW_API_VERSION_MINOR_X710 : \ + I40E_FW_API_VERSION_MINOR_X722) + +/* API version 1.7 implements additional link and PHY-specific APIs */ +#define I40E_MINOR_VER_GET_LINK_INFO_XL710 0x0007 struct i40e_aq_desc { __le16 flags; @@ -139,12 +147,10 @@ enum i40e_admin_queue_opc { i40e_aqc_opc_list_func_capabilities = 0x000A, i40e_aqc_opc_list_dev_capabilities = 0x000B, -#ifdef X722_SUPPORT /* Proxy commands */ i40e_aqc_opc_set_proxy_config = 0x0104, i40e_aqc_opc_set_ns_proxy_table_entry = 0x0105, -#endif /* LAA */ i40e_aqc_opc_mac_address_read = 0x0107, i40e_aqc_opc_mac_address_write = 0x0108, @@ -152,12 +158,11 @@ enum i40e_admin_queue_opc { /* PXE */ i40e_aqc_opc_clear_pxe_mode = 0x0110, -#ifdef X722_SUPPORT /* WoL commands */ i40e_aqc_opc_set_wol_filter = 0x0120, i40e_aqc_opc_get_wake_reason = 0x0121, + i40e_aqc_opc_clear_all_wol_filters = 0x025E, -#endif /* internal switch commands */ i40e_aqc_opc_get_switch_config = 0x0200, i40e_aqc_opc_add_statistics = 0x0201, @@ -196,10 +201,16 @@ enum i40e_admin_queue_opc { i40e_aqc_opc_remove_control_packet_filter = 0x025B, i40e_aqc_opc_add_cloud_filters = 0x025C, i40e_aqc_opc_remove_cloud_filters = 0x025D, + i40e_aqc_opc_clear_wol_switch_filters = 0x025E, + i40e_aqc_opc_replace_cloud_filters = 0x025F, i40e_aqc_opc_add_mirror_rule = 0x0260, i40e_aqc_opc_delete_mirror_rule = 0x0261, + /* Dynamic Device Personalization */ + i40e_aqc_opc_write_personalization_profile = 0x0270, + i40e_aqc_opc_get_personalization_profile_list = 0x0271, + /* DCB commands */ i40e_aqc_opc_dcb_ignore_pfc = 0x0301, i40e_aqc_opc_dcb_updated = 0x0302, @@ -223,6 +234,9 @@ enum i40e_admin_queue_opc { i40e_aqc_opc_suspend_port_tx = 0x041B, i40e_aqc_opc_resume_port_tx = 0x041C, i40e_aqc_opc_configure_partition_bw = 0x041D, + /* hmc */ + i40e_aqc_opc_query_hmc_resource_profile = 0x0500, + i40e_aqc_opc_set_hmc_resource_profile = 0x0501, /* phy commands*/ i40e_aqc_opc_get_phy_abilities = 0x0600, @@ -239,6 +253,8 @@ enum i40e_admin_queue_opc { i40e_aqc_opc_set_phy_debug = 0x0622, i40e_aqc_opc_upload_ext_phy_fm = 0x0625, i40e_aqc_opc_run_phy_activity = 0x0626, + i40e_aqc_opc_set_phy_register = 0x0628, + i40e_aqc_opc_get_phy_register = 0x0629, /* NVM commands */ i40e_aqc_opc_nvm_read = 0x0701, @@ -278,12 +294,10 @@ enum i40e_admin_queue_opc { /* Tunnel commands */ i40e_aqc_opc_add_udp_tunnel = 0x0B00, i40e_aqc_opc_del_udp_tunnel = 0x0B01, -#ifdef X722_SUPPORT i40e_aqc_opc_set_rss_key = 0x0B02, i40e_aqc_opc_set_rss_lut = 0x0B03, i40e_aqc_opc_get_rss_key = 0x0B04, i40e_aqc_opc_get_rss_lut = 0x0B05, -#endif /* Async Events */ i40e_aqc_opc_event_lan_overflow = 0x1001, @@ -471,13 +485,15 @@ I40E_CHECK_CMD_LENGTH(i40e_aqc_cppm_configuration); /* Set ARP Proxy command / response (indirect 0x0104) */ struct i40e_aqc_arp_proxy_data { __le16 command_flags; -#define I40E_AQ_ARP_INIT_IPV4 0x0008 -#define I40E_AQ_ARP_UNSUP_CTL 0x0010 -#define I40E_AQ_ARP_ENA 0x0020 -#define I40E_AQ_ARP_ADD_IPV4 0x0040 -#define I40E_AQ_ARP_DEL_IPV4 0x0080 +#define I40E_AQ_ARP_INIT_IPV4 0x0800 +#define I40E_AQ_ARP_UNSUP_CTL 0x1000 +#define I40E_AQ_ARP_ENA 0x2000 +#define I40E_AQ_ARP_ADD_IPV4 0x4000 +#define I40E_AQ_ARP_DEL_IPV4 0x8000 __le16 table_id; - __le32 pfpm_proxyfc; + __le32 enabled_offloads; +#define I40E_AQ_ARP_DIRECTED_OFFLOAD_ENABLE 0x00000020 +#define I40E_AQ_ARP_OFFLOAD_ENABLE 0x00000800 __le32 ip_addr; u8 mac_addr[6]; u8 reserved[2]; @@ -492,17 +508,19 @@ struct i40e_aqc_ns_proxy_data { __le16 table_idx_ipv6_0; __le16 table_idx_ipv6_1; __le16 control; -#define I40E_AQ_NS_PROXY_ADD_0 0x0100 -#define I40E_AQ_NS_PROXY_DEL_0 0x0200 -#define I40E_AQ_NS_PROXY_ADD_1 0x0400 -#define I40E_AQ_NS_PROXY_DEL_1 0x0800 -#define I40E_AQ_NS_PROXY_ADD_IPV6_0 0x1000 -#define I40E_AQ_NS_PROXY_DEL_IPV6_0 0x2000 -#define I40E_AQ_NS_PROXY_ADD_IPV6_1 0x4000 -#define I40E_AQ_NS_PROXY_DEL_IPV6_1 0x8000 -#define I40E_AQ_NS_PROXY_COMMAND_SEQ 0x0001 -#define I40E_AQ_NS_PROXY_INIT_IPV6_TBL 0x0002 -#define I40E_AQ_NS_PROXY_INIT_MAC_TBL 0x0004 +#define I40E_AQ_NS_PROXY_ADD_0 0x0001 +#define I40E_AQ_NS_PROXY_DEL_0 0x0002 +#define I40E_AQ_NS_PROXY_ADD_1 0x0004 +#define I40E_AQ_NS_PROXY_DEL_1 0x0008 +#define I40E_AQ_NS_PROXY_ADD_IPV6_0 0x0010 +#define I40E_AQ_NS_PROXY_DEL_IPV6_0 0x0020 +#define I40E_AQ_NS_PROXY_ADD_IPV6_1 0x0040 +#define I40E_AQ_NS_PROXY_DEL_IPV6_1 0x0080 +#define I40E_AQ_NS_PROXY_COMMAND_SEQ 0x0100 +#define I40E_AQ_NS_PROXY_INIT_IPV6_TBL 0x0200 +#define I40E_AQ_NS_PROXY_INIT_MAC_TBL 0x0400 +#define I40E_AQ_NS_PROXY_OFFLOAD_ENABLE 0x0800 +#define I40E_AQ_NS_PROXY_DIRECTED_OFFLOAD_ENABLE 0x1000 u8 mac_addr_0[6]; u8 mac_addr_1[6]; u8 local_mac_addr[6]; @@ -532,7 +550,8 @@ struct i40e_aqc_mac_address_read { #define I40E_AQC_PORT_ADDR_VALID 0x40 #define I40E_AQC_WOL_ADDR_VALID 0x80 #define I40E_AQC_MC_MAG_EN_VALID 0x100 -#define I40E_AQC_ADDR_VALID_MASK 0x1F0 +#define I40E_AQC_WOL_PRESERVE_STATUS 0x200 +#define I40E_AQC_ADDR_VALID_MASK 0x3F0 u8 reserved[6]; __le32 addr_high; __le32 addr_low; @@ -552,6 +571,8 @@ I40E_CHECK_STRUCT_LEN(24, i40e_aqc_mac_address_read_data); /* Manage MAC Address Write Command (0x0108) */ struct i40e_aqc_mac_address_write { __le16 command_flags; +#define I40E_AQC_MC_MAG_EN 0x0100 +#define I40E_AQC_WOL_PRESERVE_ON_PFR 0x0200 #define I40E_AQC_WRITE_TYPE_LAA_ONLY 0x0000 #define I40E_AQC_WRITE_TYPE_LAA_WOL 0x4000 #define I40E_AQC_WRITE_TYPE_PORT 0x8000 @@ -575,15 +596,24 @@ struct i40e_aqc_clear_pxe { I40E_CHECK_CMD_LENGTH(i40e_aqc_clear_pxe); -#ifdef X722_SUPPORT /* Set WoL Filter (0x0120) */ struct i40e_aqc_set_wol_filter { __le16 filter_index; #define I40E_AQC_MAX_NUM_WOL_FILTERS 8 +#define I40E_AQC_SET_WOL_FILTER_TYPE_MAGIC_SHIFT 15 +#define I40E_AQC_SET_WOL_FILTER_TYPE_MAGIC_MASK (0x1 << \ + I40E_AQC_SET_WOL_FILTER_TYPE_MAGIC_SHIFT) + +#define I40E_AQC_SET_WOL_FILTER_INDEX_SHIFT 0 +#define I40E_AQC_SET_WOL_FILTER_INDEX_MASK (0x7 << \ + I40E_AQC_SET_WOL_FILTER_INDEX_SHIFT) __le16 cmd_flags; #define I40E_AQC_SET_WOL_FILTER 0x8000 #define I40E_AQC_SET_WOL_FILTER_NO_TCO_WOL 0x4000 +#define I40E_AQC_SET_WOL_FILTER_WOL_PRESERVE_ON_PFR 0x2000 +#define I40E_AQC_SET_WOL_FILTER_ACTION_CLEAR 0 +#define I40E_AQC_SET_WOL_FILTER_ACTION_SET 1 __le16 valid_flags; #define I40E_AQC_SET_WOL_FILTER_ACTION_VALID 0x8000 #define I40E_AQC_SET_WOL_FILTER_NO_TCO_ACTION_VALID 0x4000 @@ -594,16 +624,6 @@ struct i40e_aqc_set_wol_filter { I40E_CHECK_CMD_LENGTH(i40e_aqc_set_wol_filter); -/* Get Wake Reason (0x0121) */ - -struct i40e_aqc_get_wake_reason_completion { - u8 reserved_1[2]; - __le16 wake_reason; - u8 reserved_2[12]; -}; - -I40E_CHECK_CMD_LENGTH(i40e_aqc_get_wake_reason_completion); - struct i40e_aqc_set_wol_filter_data { u8 filter[128]; u8 mask[16]; @@ -611,7 +631,22 @@ struct i40e_aqc_set_wol_filter_data { I40E_CHECK_STRUCT_LEN(0x90, i40e_aqc_set_wol_filter_data); -#endif /* X722_SUPPORT */ +/* Get Wake Reason (0x0121) */ + +struct i40e_aqc_get_wake_reason_completion { + u8 reserved_1[2]; + __le16 wake_reason; +#define I40E_AQC_GET_WAKE_UP_REASON_WOL_REASON_MATCHED_INDEX_SHIFT 0 +#define I40E_AQC_GET_WAKE_UP_REASON_WOL_REASON_MATCHED_INDEX_MASK (0xFF << \ + I40E_AQC_GET_WAKE_UP_REASON_WOL_REASON_MATCHED_INDEX_SHIFT) +#define I40E_AQC_GET_WAKE_UP_REASON_WOL_REASON_RESERVED_SHIFT 8 +#define I40E_AQC_GET_WAKE_UP_REASON_WOL_REASON_RESERVED_MASK (0xFF << \ + I40E_AQC_GET_WAKE_UP_REASON_WOL_REASON_RESERVED_SHIFT) + u8 reserved_2[12]; +}; + +I40E_CHECK_CMD_LENGTH(i40e_aqc_get_wake_reason_completion); + /* Switch configuration commands (0x02xx) */ /* Used by many indirect commands that only pass an seid and a buffer in the @@ -694,6 +729,8 @@ struct i40e_aqc_set_port_parameters { #define I40E_AQ_SET_P_PARAMS_PAD_SHORT_PACKETS 2 /* must set! */ #define I40E_AQ_SET_P_PARAMS_DOUBLE_VLAN_ENA 4 __le16 bad_frame_vsi; +#define I40E_AQ_SET_P_PARAMS_BFRAME_SEID_SHIFT 0x0 +#define I40E_AQ_SET_P_PARAMS_BFRAME_SEID_MASK 0x3FF __le16 default_seid; /* reserved for command */ u8 reserved[10]; }; @@ -745,10 +782,27 @@ I40E_CHECK_STRUCT_LEN(0x10, i40e_aqc_switch_resource_alloc_element_resp); /* Set Switch Configuration (direct 0x0205) */ struct i40e_aqc_set_switch_config { __le16 flags; +/* flags used for both fields below */ #define I40E_AQ_SET_SWITCH_CFG_PROMISC 0x0001 #define I40E_AQ_SET_SWITCH_CFG_L2_FILTER 0x0002 +#define I40E_AQ_SET_SWITCH_CFG_HW_ATR_EVICT 0x0004 __le16 valid_flags; - u8 reserved[12]; + /* The ethertype in switch_tag is dropped on ingress and used + * internally by the switch. Set this to zero for the default + * of 0x88a8 (802.1ad). Should be zero for firmware API + * versions lower than 1.7. + */ + __le16 switch_tag; + /* The ethertypes in first_tag and second_tag are used to + * match the outer and inner VLAN tags (respectively) when HW + * double VLAN tagging is enabled via the set port parameters + * AQ command. Otherwise these are both ignored. Set them to + * zero for their defaults of 0x8100 (802.1Q). Should be zero + * for firmware API versions lower than 1.7. + */ + __le16 first_tag; + __le16 second_tag; + u8 reserved[6]; }; I40E_CHECK_CMD_LENGTH(i40e_aqc_set_switch_config); @@ -913,16 +967,12 @@ struct i40e_aqc_vsi_properties_data { I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT) /* queueing option section */ u8 queueing_opt_flags; -#ifdef X722_SUPPORT #define I40E_AQ_VSI_QUE_OPT_MULTICAST_UDP_ENA 0x04 #define I40E_AQ_VSI_QUE_OPT_UNICAST_UDP_ENA 0x08 -#endif #define I40E_AQ_VSI_QUE_OPT_TCP_ENA 0x10 #define I40E_AQ_VSI_QUE_OPT_FCOE_ENA 0x20 -#ifdef X722_SUPPORT #define I40E_AQ_VSI_QUE_OPT_RSS_LUT_PF 0x00 #define I40E_AQ_VSI_QUE_OPT_RSS_LUT_VSI 0x40 -#endif u8 queueing_opt_reserved[3]; /* scheduler section */ u8 up_enable_bits; @@ -1305,7 +1355,9 @@ struct i40e_aqc_add_remove_cloud_filters { #define I40E_AQC_ADD_CLOUD_CMD_SEID_NUM_SHIFT 0 #define I40E_AQC_ADD_CLOUD_CMD_SEID_NUM_MASK (0x3FF << \ I40E_AQC_ADD_CLOUD_CMD_SEID_NUM_SHIFT) - u8 reserved2[4]; + u8 big_buffer_flag; +#define I40E_AQC_ADD_REM_CLOUD_CMD_BIG_BUFFER 1 + u8 reserved2[3]; __le32 addr_high; __le32 addr_low; }; @@ -1342,6 +1394,7 @@ struct i40e_aqc_add_remove_cloud_filters_element_data { #define I40E_AQC_ADD_CLOUD_FILTER_IMAC 0x000A #define I40E_AQC_ADD_CLOUD_FILTER_OMAC_TEN_ID_IMAC 0x000B #define I40E_AQC_ADD_CLOUD_FILTER_IIP 0x000C +/* 0x0010 to 0x0017 is for custom filters */ #define I40E_AQC_ADD_CLOUD_FLAGS_TO_QUEUE 0x0080 #define I40E_AQC_ADD_CLOUD_VNK_SHIFT 6 @@ -1376,6 +1429,46 @@ struct i40e_aqc_add_remove_cloud_filters_element_data { u8 response_reserved[7]; }; +/* i40e_aqc_add_rm_cloud_filt_elem_ext is used when + * I40E_AQC_ADD_REM_CLOUD_CMD_BIG_BUFFER flag is set. refer to + * DCR288 + */ +struct i40e_aqc_add_rm_cloud_filt_elem_ext { + struct i40e_aqc_add_remove_cloud_filters_element_data element; + u16 general_fields[32]; +#define I40E_AQC_ADD_CLOUD_FV_FLU_0X10_WORD0 0 +#define I40E_AQC_ADD_CLOUD_FV_FLU_0X10_WORD1 1 +#define I40E_AQC_ADD_CLOUD_FV_FLU_0X10_WORD2 2 +#define I40E_AQC_ADD_CLOUD_FV_FLU_0X11_WORD0 3 +#define I40E_AQC_ADD_CLOUD_FV_FLU_0X11_WORD1 4 +#define I40E_AQC_ADD_CLOUD_FV_FLU_0X11_WORD2 5 +#define I40E_AQC_ADD_CLOUD_FV_FLU_0X12_WORD0 6 +#define I40E_AQC_ADD_CLOUD_FV_FLU_0X12_WORD1 7 +#define I40E_AQC_ADD_CLOUD_FV_FLU_0X12_WORD2 8 +#define I40E_AQC_ADD_CLOUD_FV_FLU_0X13_WORD0 9 +#define I40E_AQC_ADD_CLOUD_FV_FLU_0X13_WORD1 10 +#define I40E_AQC_ADD_CLOUD_FV_FLU_0X13_WORD2 11 +#define I40E_AQC_ADD_CLOUD_FV_FLU_0X14_WORD0 12 +#define I40E_AQC_ADD_CLOUD_FV_FLU_0X14_WORD1 13 +#define I40E_AQC_ADD_CLOUD_FV_FLU_0X14_WORD2 14 +#define I40E_AQC_ADD_CLOUD_FV_FLU_0X16_WORD0 15 +#define I40E_AQC_ADD_CLOUD_FV_FLU_0X16_WORD1 16 +#define I40E_AQC_ADD_CLOUD_FV_FLU_0X16_WORD2 17 +#define I40E_AQC_ADD_CLOUD_FV_FLU_0X16_WORD3 18 +#define I40E_AQC_ADD_CLOUD_FV_FLU_0X16_WORD4 19 +#define I40E_AQC_ADD_CLOUD_FV_FLU_0X16_WORD5 20 +#define I40E_AQC_ADD_CLOUD_FV_FLU_0X16_WORD6 21 +#define I40E_AQC_ADD_CLOUD_FV_FLU_0X16_WORD7 22 +#define I40E_AQC_ADD_CLOUD_FV_FLU_0X17_WORD0 23 +#define I40E_AQC_ADD_CLOUD_FV_FLU_0X17_WORD1 24 +#define I40E_AQC_ADD_CLOUD_FV_FLU_0X17_WORD2 25 +#define I40E_AQC_ADD_CLOUD_FV_FLU_0X17_WORD3 26 +#define I40E_AQC_ADD_CLOUD_FV_FLU_0X17_WORD4 27 +#define I40E_AQC_ADD_CLOUD_FV_FLU_0X17_WORD5 28 +#define I40E_AQC_ADD_CLOUD_FV_FLU_0X17_WORD6 29 +#define I40E_AQC_ADD_CLOUD_FV_FLU_0X17_WORD7 30 +}; + struct i40e_aqc_remove_cloud_filters_completion { __le16 perfect_ovlan_used; __le16 perfect_ovlan_free; @@ -1387,6 +1480,54 @@ struct i40e_aqc_remove_cloud_filters_completion { I40E_CHECK_CMD_LENGTH(i40e_aqc_remove_cloud_filters_completion); +/* Replace filter Command 0x025F + * uses the i40e_aqc_replace_cloud_filters, + * and the generic indirect completion structure + */ +struct i40e_filter_data { + u8 filter_type; + u8 input[3]; +}; + +struct i40e_aqc_replace_cloud_filters_cmd { + u8 valid_flags; +#define I40E_AQC_REPLACE_L1_FILTER 0x0 +#define I40E_AQC_REPLACE_CLOUD_FILTER 0x1 +#define I40E_AQC_GET_CLOUD_FILTERS 0x2 +#define I40E_AQC_MIRROR_CLOUD_FILTER 0x4 +#define I40E_AQC_HIGH_PRIORITY_CLOUD_FILTER 0x8 + u8 old_filter_type; + u8 new_filter_type; + u8 tr_bit; + u8 reserved[4]; + __le32 addr_high; + __le32 addr_low; +}; + +struct i40e_aqc_replace_cloud_filters_cmd_buf { + u8 data[32]; +/* Filter type INPUT codes*/ +#define I40E_AQC_REPLACE_CLOUD_CMD_INPUT_ENTRIES_MAX 3 +#define I40E_AQC_REPLACE_CLOUD_CMD_INPUT_VALIDATED (1 << 7UL) + +/* Field Vector offsets */ +#define I40E_AQC_REPLACE_CLOUD_CMD_INPUT_FV_MAC_DA 0 +#define I40E_AQC_REPLACE_CLOUD_CMD_INPUT_FV_STAG_ETH 6 +#define I40E_AQC_REPLACE_CLOUD_CMD_INPUT_FV_STAG 7 +#define I40E_AQC_REPLACE_CLOUD_CMD_INPUT_FV_VLAN 8 +#define I40E_AQC_REPLACE_CLOUD_CMD_INPUT_FV_STAG_OVLAN 9 +#define I40E_AQC_REPLACE_CLOUD_CMD_INPUT_FV_STAG_IVLAN 10 +#define I40E_AQC_REPLACE_CLOUD_CMD_INPUT_FV_TUNNLE_KEY 11 +#define I40E_AQC_REPLACE_CLOUD_CMD_INPUT_FV_IMAC 12 +/* big FLU */ +#define I40E_AQC_REPLACE_CLOUD_CMD_INPUT_FV_IP_DA 14 +/* big FLU */ +#define I40E_AQC_REPLACE_CLOUD_CMD_INPUT_FV_OIP_DA 15 + +#define I40E_AQC_REPLACE_CLOUD_CMD_INPUT_FV_INNER_VLAN 37 + struct i40e_filter_data filters[8]; +}; + /* Add Mirror Rule (indirect or direct 0x0260) * Delete Mirror Rule (indirect or direct 0x0261) * note: some rule types (4,5) do not use an external buffer. @@ -1422,6 +1563,36 @@ struct i40e_aqc_add_delete_mirror_rule_completion { I40E_CHECK_CMD_LENGTH(i40e_aqc_add_delete_mirror_rule_completion); +/* Dynamic Device Personalization */ +struct i40e_aqc_write_personalization_profile { + u8 flags; + u8 reserved[3]; + __le32 profile_track_id; + __le32 addr_high; + __le32 addr_low; +}; + +I40E_CHECK_CMD_LENGTH(i40e_aqc_write_personalization_profile); + +struct i40e_aqc_write_ddp_resp { + __le32 error_offset; + __le32 error_info; + __le32 addr_high; + __le32 addr_low; +}; + +struct i40e_aqc_get_applied_profiles { + u8 flags; +#define I40E_AQC_GET_DDP_GET_CONF 0x1 +#define I40E_AQC_GET_DDP_GET_RDPU_CONF 0x2 + u8 rsv[3]; + __le32 reserved; + __le32 addr_high; + __le32 addr_low; +}; + +I40E_CHECK_CMD_LENGTH(i40e_aqc_get_applied_profiles); + /* DCB 0x03xx*/ /* PFC Ignore (direct 0x0301) @@ -1644,6 +1815,24 @@ struct i40e_aqc_configure_partition_bw_data { I40E_CHECK_STRUCT_LEN(0x22, i40e_aqc_configure_partition_bw_data); +/* Get and set the active HMC resource profile and status. + * (direct 0x0500) and (direct 0x0501) + */ +struct i40e_aq_get_set_hmc_resource_profile { + u8 pm_profile; + u8 pe_vf_enabled; + u8 reserved[14]; +}; + +I40E_CHECK_CMD_LENGTH(i40e_aq_get_set_hmc_resource_profile); + +enum i40e_aq_hmc_profile { + /* I40E_HMC_PROFILE_NO_CHANGE = 0, reserved */ + I40E_HMC_PROFILE_DEFAULT = 1, + I40E_HMC_PROFILE_FAVOR_VF = 2, + I40E_HMC_PROFILE_EQUAL = 3, +}; + /* Get PHY Abilities (indirect 0x0600) uses the generic indirect struct */ /* set in param0 for get phy abilities to report qualified modules */ @@ -1665,6 +1854,8 @@ enum i40e_aq_phy_type { I40E_PHY_TYPE_10GBASE_CR1_CU = 0xB, I40E_PHY_TYPE_10GBASE_AOC = 0xC, I40E_PHY_TYPE_40GBASE_AOC = 0xD, + I40E_PHY_TYPE_UNRECOGNIZED = 0xE, + I40E_PHY_TYPE_UNSUPPORTED = 0xF, I40E_PHY_TYPE_100BASE_TX = 0x11, I40E_PHY_TYPE_1000BASE_T = 0x12, I40E_PHY_TYPE_10GBASE_T = 0x13, @@ -1683,7 +1874,11 @@ enum i40e_aq_phy_type { I40E_PHY_TYPE_25GBASE_CR = 0x20, I40E_PHY_TYPE_25GBASE_SR = 0x21, I40E_PHY_TYPE_25GBASE_LR = 0x22, - I40E_PHY_TYPE_MAX + I40E_PHY_TYPE_25GBASE_AOC = 0x23, + I40E_PHY_TYPE_25GBASE_ACC = 0x24, + I40E_PHY_TYPE_MAX, + I40E_PHY_TYPE_EMPTY = 0xFE, + I40E_PHY_TYPE_DEFAULT = 0xFF, }; #define I40E_LINK_SPEED_100MB_SHIFT 0x1 @@ -1691,6 +1886,7 @@ enum i40e_aq_phy_type { #define I40E_LINK_SPEED_10GB_SHIFT 0x3 #define I40E_LINK_SPEED_40GB_SHIFT 0x4 #define I40E_LINK_SPEED_20GB_SHIFT 0x5 +#define I40E_LINK_SPEED_25GB_SHIFT 0x6 enum i40e_aq_link_speed { I40E_LINK_SPEED_UNKNOWN = 0, @@ -1698,7 +1894,8 @@ enum i40e_aq_link_speed { I40E_LINK_SPEED_1GB = (1 << I40E_LINK_SPEED_1000MB_SHIFT), I40E_LINK_SPEED_10GB = (1 << I40E_LINK_SPEED_10GB_SHIFT), I40E_LINK_SPEED_40GB = (1 << I40E_LINK_SPEED_40GB_SHIFT), - I40E_LINK_SPEED_20GB = (1 << I40E_LINK_SPEED_20GB_SHIFT) + I40E_LINK_SPEED_20GB = (1 << I40E_LINK_SPEED_20GB_SHIFT), + I40E_LINK_SPEED_25GB = (1 << I40E_LINK_SPEED_25GB_SHIFT), }; struct i40e_aqc_module_desc { @@ -1721,6 +1918,8 @@ struct i40e_aq_get_phy_abilities_resp { #define I40E_AQ_PHY_LINK_ENABLED 0x08 #define I40E_AQ_PHY_AN_ENABLED 0x10 #define I40E_AQ_PHY_FLAG_MODULE_QUAL 0x20 +#define I40E_AQ_PHY_FEC_ABILITY_KR 0x40 +#define I40E_AQ_PHY_FEC_ABILITY_RS 0x80 __le16 eee_capability; #define I40E_AQ_EEE_100BASE_TX 0x0002 #define I40E_AQ_EEE_1000BASE_T 0x0004 @@ -1731,7 +1930,24 @@ struct i40e_aq_get_phy_abilities_resp { __le32 eeer_val; u8 d3_lpan; #define I40E_AQ_SET_PHY_D3_LPAN_ENA 0x01 - u8 reserved[3]; + u8 phy_type_ext; +#define I40E_AQ_PHY_TYPE_EXT_25G_KR 0x01 +#define I40E_AQ_PHY_TYPE_EXT_25G_CR 0x02 +#define I40E_AQ_PHY_TYPE_EXT_25G_SR 0x04 +#define I40E_AQ_PHY_TYPE_EXT_25G_LR 0x08 +#define I40E_AQ_PHY_TYPE_EXT_25G_AOC 0x10 +#define I40E_AQ_PHY_TYPE_EXT_25G_ACC 0x20 + u8 fec_cfg_curr_mod_ext_info; +#define I40E_AQ_ENABLE_FEC_KR 0x01 +#define I40E_AQ_ENABLE_FEC_RS 0x02 +#define I40E_AQ_REQUEST_FEC_KR 0x04 +#define I40E_AQ_REQUEST_FEC_RS 0x08 +#define I40E_AQ_ENABLE_FEC_AUTO 0x10 +#define I40E_AQ_FEC +#define I40E_AQ_MODULE_TYPE_EXT_MASK 0xE0 +#define I40E_AQ_MODULE_TYPE_EXT_SHIFT 5 + + u8 ext_comp_code; u8 phy_id[4]; u8 module_type[3]; u8 qualified_module_count; @@ -1753,7 +1969,16 @@ struct i40e_aq_set_phy_config { /* same bits as above in all */ __le16 eee_capability; __le32 eeer; u8 low_power_ctrl; - u8 reserved[3]; + u8 phy_type_ext; + u8 fec_config; +#define I40E_AQ_SET_FEC_ABILITY_KR BIT(0) +#define I40E_AQ_SET_FEC_ABILITY_RS BIT(1) +#define I40E_AQ_SET_FEC_REQUEST_KR BIT(2) +#define I40E_AQ_SET_FEC_REQUEST_RS BIT(3) +#define I40E_AQ_SET_FEC_AUTO BIT(4) +#define I40E_AQ_PHY_FEC_CONFIG_SHIFT 0x0 +#define I40E_AQ_PHY_FEC_CONFIG_MASK (0x1F << I40E_AQ_PHY_FEC_CONFIG_SHIFT) + u8 reserved; }; I40E_CHECK_CMD_LENGTH(i40e_aq_set_phy_config); @@ -1833,17 +2058,39 @@ struct i40e_aqc_get_link_status { #define I40E_AQ_LINK_TX_DRAINED 0x01 #define I40E_AQ_LINK_TX_FLUSHED 0x03 #define I40E_AQ_LINK_FORCED_40G 0x10 +/* 25G Error Codes */ +#define I40E_AQ_25G_NO_ERR 0X00 +#define I40E_AQ_25G_NOT_PRESENT 0X01 +#define I40E_AQ_25G_NVM_CRC_ERR 0X02 +#define I40E_AQ_25G_SBUS_UCODE_ERR 0X03 +#define I40E_AQ_25G_SERDES_UCODE_ERR 0X04 +#define I40E_AQ_25G_NIMB_UCODE_ERR 0X05 u8 loopback; /* use defines from i40e_aqc_set_lb_mode */ +/* Since firmware API 1.7 loopback field keeps power class info as well */ +#define I40E_AQ_LOOPBACK_MASK 0x07 +#define I40E_AQ_PWR_CLASS_SHIFT_LB 6 +#define I40E_AQ_PWR_CLASS_MASK_LB (0x03 << I40E_AQ_PWR_CLASS_SHIFT_LB) __le16 max_frame_size; u8 config; +#define I40E_AQ_CONFIG_FEC_KR_ENA 0x01 +#define I40E_AQ_CONFIG_FEC_RS_ENA 0x02 #define I40E_AQ_CONFIG_CRC_ENA 0x04 #define I40E_AQ_CONFIG_PACING_MASK 0x78 - u8 external_power_ability; + union { + struct { + u8 power_desc; #define I40E_AQ_LINK_POWER_CLASS_1 0x00 #define I40E_AQ_LINK_POWER_CLASS_2 0x01 #define I40E_AQ_LINK_POWER_CLASS_3 0x02 #define I40E_AQ_LINK_POWER_CLASS_4 0x03 - u8 reserved[4]; +#define I40E_AQ_PWR_CLASS_MASK 0x03 + u8 reserved[4]; + }; + struct { + u8 link_type[4]; + u8 link_type_ext; + }; + }; }; I40E_CHECK_CMD_LENGTH(i40e_aqc_get_link_status); @@ -1926,6 +2173,22 @@ struct i40e_aqc_run_phy_activity { I40E_CHECK_CMD_LENGTH(i40e_aqc_run_phy_activity); +/* Set PHY Register command (0x0628) */ +/* Get PHY Register command (0x0629) */ +struct i40e_aqc_phy_register_access { + u8 phy_interface; +#define I40E_AQ_PHY_REG_ACCESS_INTERNAL 0 +#define I40E_AQ_PHY_REG_ACCESS_EXTERNAL 1 +#define I40E_AQ_PHY_REG_ACCESS_EXTERNAL_MODULE 2 + u8 dev_addres; + u8 reserved1[2]; + u32 reg_address; + u32 reg_value; + u8 reserved2[4]; +}; + +I40E_CHECK_CMD_LENGTH(i40e_aqc_phy_register_access); + /* NVM Read command (indirect 0x0701) * NVM Erase commands (direct 0x0702) * NVM Update commands (indirect 0x0703) @@ -2340,7 +2603,6 @@ struct i40e_aqc_del_udp_tunnel_completion { }; I40E_CHECK_CMD_LENGTH(i40e_aqc_del_udp_tunnel_completion); -#ifdef X722_SUPPORT struct i40e_aqc_get_set_rss_key { #define I40E_AQC_SET_RSS_KEY_VSI_VALID (0x1 << 15) @@ -2381,7 +2643,6 @@ struct i40e_aqc_get_set_rss_lut { }; I40E_CHECK_CMD_LENGTH(i40e_aqc_get_set_rss_lut); -#endif /* tunnel key structure 0x0B10 */ diff --git a/dpdk/drivers/net/i40e/base/i40e_common.c b/dpdk/drivers/net/i40e/base/i40e_common.c index 4407f2d3..900d379c 100644 --- a/dpdk/drivers/net/i40e/base/i40e_common.c +++ b/dpdk/drivers/net/i40e/base/i40e_common.c @@ -34,7 +34,7 @@ POSSIBILITY OF SUCH DAMAGE. #include "i40e_type.h" #include "i40e_adminq.h" #include "i40e_prototype.h" -#include "i40e_virtchnl.h" +#include "virtchnl.h" /** @@ -71,7 +71,6 @@ STATIC enum i40e_status_code i40e_set_mac_type(struct i40e_hw *hw) case I40E_DEV_ID_25G_SFP28: hw->mac.type = I40E_MAC_XL710; break; -#ifdef X722_SUPPORT #ifdef X722_A0_SUPPORT case I40E_DEV_ID_X722_A0: #endif @@ -81,24 +80,20 @@ STATIC enum i40e_status_code i40e_set_mac_type(struct i40e_hw *hw) case I40E_DEV_ID_1G_BASE_T_X722: case I40E_DEV_ID_10G_BASE_T_X722: case I40E_DEV_ID_SFP_I_X722: - case I40E_DEV_ID_QSFP_I_X722: hw->mac.type = I40E_MAC_X722; break; -#endif -#ifdef X722_SUPPORT #if defined(INTEGRATED_VF) || defined(VF_DRIVER) case I40E_DEV_ID_X722_VF: - case I40E_DEV_ID_X722_VF_HV: #ifdef X722_A0_SUPPORT case I40E_DEV_ID_X722_A0_VF: #endif hw->mac.type = I40E_MAC_X722_VF; break; #endif /* INTEGRATED_VF || VF_DRIVER */ -#endif /* X722_SUPPORT */ #if defined(INTEGRATED_VF) || defined(VF_DRIVER) case I40E_DEV_ID_VF: case I40E_DEV_ID_VF_HV: + case I40E_DEV_ID_ADAPTIVE_VF: hw->mac.type = I40E_MAC_VF; break; #endif @@ -115,7 +110,6 @@ STATIC enum i40e_status_code i40e_set_mac_type(struct i40e_hw *hw) return status; } -#ifndef I40E_NDIS_SUPPORT /** * i40e_aq_str - convert AQ err code to a string * @hw: pointer to the HW structure @@ -322,7 +316,6 @@ const char *i40e_stat_str(struct i40e_hw *hw, enum i40e_status_code stat_err) return hw->err_str; } -#endif /* I40E_NDIS_SUPPORT */ /** * i40e_debug_aq * @hw: debug mask related to admin queue @@ -337,13 +330,15 @@ void i40e_debug_aq(struct i40e_hw *hw, enum i40e_debug_mask mask, void *desc, void *buffer, u16 buf_len) { struct i40e_aq_desc *aq_desc = (struct i40e_aq_desc *)desc; - u16 len = LE16_TO_CPU(aq_desc->datalen); u8 *buf = (u8 *)buffer; + u16 len; u16 i = 0; if ((!(mask & hw->debug_mask)) || (desc == NULL)) return; + len = LE16_TO_CPU(aq_desc->datalen); + i40e_debug(hw, mask, "AQ CMD: opcode 0x%04X, flags 0x%04X, datalen 0x%04X, retval 0x%04X\n", LE16_TO_CPU(aq_desc->opcode), @@ -383,8 +378,7 @@ void i40e_debug_aq(struct i40e_hw *hw, enum i40e_debug_mask mask, void *desc, d_buf[j] = buf[i]; i40e_debug(hw, mask, "\t0x%04X %02X %02X %02X %02X %02X %02X %02X %02X %02X %02X %02X %02X %02X %02X %02X %02X\n", - i_sav, d_buf[0], d_buf[1], - d_buf[2], d_buf[3], + i_sav, d_buf[0], d_buf[1], d_buf[2], d_buf[3], d_buf[4], d_buf[5], d_buf[6], d_buf[7], d_buf[8], d_buf[9], d_buf[10], d_buf[11], d_buf[12], d_buf[13], d_buf[14], d_buf[15]); @@ -449,7 +443,6 @@ enum i40e_status_code i40e_aq_queue_shutdown(struct i40e_hw *hw, return status; } -#ifdef X722_SUPPORT /** * i40e_aq_get_set_rss_lut @@ -608,7 +601,6 @@ enum i40e_status_code i40e_aq_set_rss_key(struct i40e_hw *hw, { return i40e_aq_get_set_rss_key(hw, vsi_id, key, true); } -#endif /* X722_SUPPORT */ /* The i40e_ptype_lookup table is used to convert from the 8-bit ptype in the * hardware to a bit-field that can be used by SW to more easily determine the @@ -1024,9 +1016,7 @@ enum i40e_status_code i40e_init_shared_code(struct i40e_hw *hw) switch (hw->mac.type) { case I40E_MAC_XL710: -#ifdef X722_SUPPORT case I40E_MAC_X722: -#endif break; default: return I40E_ERR_DEVICE_NOT_SUPPORTED; @@ -1046,11 +1036,9 @@ enum i40e_status_code i40e_init_shared_code(struct i40e_hw *hw) else hw->pf_id = (u8)(func_rid & 0x7); -#ifdef X722_SUPPORT if (hw->mac.type == I40E_MAC_X722) hw->flags |= I40E_HW_FLAG_AQ_SRCTL_ACCESS_ENABLE; -#endif status = i40e_init_nvm(hw); return status; } @@ -1128,7 +1116,8 @@ enum i40e_status_code i40e_get_mac_addr(struct i40e_hw *hw, u8 *mac_addr) status = i40e_aq_mac_address_read(hw, &flags, &addrs, NULL); if (flags & I40E_AQC_LAN_ADDR_VALID) - memcpy(mac_addr, &addrs.pf_lan_mac, sizeof(addrs.pf_lan_mac)); + i40e_memcpy(mac_addr, &addrs.pf_lan_mac, sizeof(addrs.pf_lan_mac), + I40E_NONDMA_TO_NONDMA); return status; } @@ -1151,7 +1140,8 @@ enum i40e_status_code i40e_get_port_mac_addr(struct i40e_hw *hw, u8 *mac_addr) return status; if (flags & I40E_AQC_PORT_ADDR_VALID) - memcpy(mac_addr, &addrs.port_mac, sizeof(addrs.port_mac)); + i40e_memcpy(mac_addr, &addrs.port_mac, sizeof(addrs.port_mac), + I40E_NONDMA_TO_NONDMA); else status = I40E_ERR_INVALID_MAC_ADDR; @@ -1190,6 +1180,33 @@ void i40e_pre_tx_queue_cfg(struct i40e_hw *hw, u32 queue, bool enable) wr32(hw, I40E_GLLAN_TXPRE_QDIS(reg_block), reg_val); } +/** + * i40e_get_san_mac_addr - get SAN MAC address + * @hw: pointer to the HW structure + * @mac_addr: pointer to SAN MAC address + * + * Reads the adapter's SAN MAC address from NVM + **/ +enum i40e_status_code i40e_get_san_mac_addr(struct i40e_hw *hw, + u8 *mac_addr) +{ + struct i40e_aqc_mac_address_read_data addrs; + enum i40e_status_code status; + u16 flags = 0; + + status = i40e_aq_mac_address_read(hw, &flags, &addrs, NULL); + if (status) + return status; + + if (flags & I40E_AQC_SAN_ADDR_VALID) + i40e_memcpy(mac_addr, &addrs.pf_san_mac, sizeof(addrs.pf_san_mac), + I40E_NONDMA_TO_NONDMA); + else + status = I40E_ERR_INVALID_MAC_ADDR; + + return status; +} + /** * i40e_read_pba_string - Reads part number string from EEPROM * @hw: pointer to hardware structure @@ -1264,6 +1281,8 @@ STATIC enum i40e_media_type i40e_get_media_type(struct i40e_hw *hw) case I40E_PHY_TYPE_1000BASE_LX: case I40E_PHY_TYPE_40GBASE_SR4: case I40E_PHY_TYPE_40GBASE_LR4: + case I40E_PHY_TYPE_25GBASE_LR: + case I40E_PHY_TYPE_25GBASE_SR: media = I40E_MEDIA_TYPE_FIBER; break; case I40E_PHY_TYPE_100BASE_TX: @@ -1278,6 +1297,9 @@ STATIC enum i40e_media_type i40e_get_media_type(struct i40e_hw *hw) case I40E_PHY_TYPE_10GBASE_SFPP_CU: case I40E_PHY_TYPE_40GBASE_AOC: case I40E_PHY_TYPE_10GBASE_AOC: + case I40E_PHY_TYPE_25GBASE_CR: + case I40E_PHY_TYPE_25GBASE_AOC: + case I40E_PHY_TYPE_25GBASE_ACC: media = I40E_MEDIA_TYPE_DA; break; case I40E_PHY_TYPE_1000BASE_KX: @@ -1285,6 +1307,7 @@ STATIC enum i40e_media_type i40e_get_media_type(struct i40e_hw *hw) case I40E_PHY_TYPE_10GBASE_KR: case I40E_PHY_TYPE_40GBASE_KR4: case I40E_PHY_TYPE_20GBASE_KR2: + case I40E_PHY_TYPE_25GBASE_KR: media = I40E_MEDIA_TYPE_BACKPLANE; break; case I40E_PHY_TYPE_SGMII: @@ -1359,6 +1382,8 @@ enum i40e_status_code i40e_pf_reset(struct i40e_hw *hw) * we don't need to do the PF Reset */ if (!cnt) { + u32 reg2 = 0; + reg = rd32(hw, I40E_PFGEN_CTRL); wr32(hw, I40E_PFGEN_CTRL, (reg | I40E_PFGEN_CTRL_PFSWR_MASK)); @@ -1366,6 +1391,12 @@ enum i40e_status_code i40e_pf_reset(struct i40e_hw *hw) reg = rd32(hw, I40E_PFGEN_CTRL); if (!(reg & I40E_PFGEN_CTRL_PFSWR_MASK)) break; + reg2 = rd32(hw, I40E_GLGEN_RSTAT); + if (reg2 & I40E_GLGEN_RSTAT_DEVSTATE_MASK) { + DEBUGOUT("Core reset upcoming.\n"); + DEBUGOUT1("I40E_GLGEN_RSTAT = 0x%x\n", reg2); + return I40E_ERR_NOT_READY; + } i40e_msec_delay(1); } if (reg & I40E_PFGEN_CTRL_PFSWR_MASK) { @@ -1670,8 +1701,17 @@ enum i40e_status_code i40e_aq_get_phy_capabilities(struct i40e_hw *hw, if (hw->aq.asq_last_status == I40E_AQ_RC_EIO) status = I40E_ERR_UNKNOWN_PHY; - if (report_init) - hw->phy.phy_types = LE32_TO_CPU(abilities->phy_type); + if (report_init) { + if (hw->mac.type == I40E_MAC_XL710 && + hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR && + hw->aq.api_min_ver >= I40E_MINOR_VER_GET_LINK_INFO_XL710) { + status = i40e_aq_get_link_info(hw, true, NULL, NULL); + } else { + hw->phy.phy_types = LE32_TO_CPU(abilities->phy_type); + hw->phy.phy_types |= + ((u64)abilities->phy_type_ext << 32); + } + } return status; } @@ -1763,10 +1803,13 @@ enum i40e_status_code i40e_set_fc(struct i40e_hw *hw, u8 *aq_failures, config.abilities |= I40E_AQ_PHY_ENABLE_ATOMIC_LINK; /* Copy over all the old settings */ config.phy_type = abilities.phy_type; + config.phy_type_ext = abilities.phy_type_ext; config.link_speed = abilities.link_speed; config.eee_capability = abilities.eee_capability; config.eeer = abilities.eeer_val; config.low_power_ctrl = abilities.d3_lpan; + config.fec_config = abilities.fec_cfg_curr_mod_ext_info & + I40E_AQ_PHY_FEC_CONFIG_MASK; status = i40e_aq_set_phy_config(hw, &config, NULL); if (status) @@ -1926,8 +1969,10 @@ enum i40e_status_code i40e_aq_get_link_info(struct i40e_hw *hw, hw_link_info->link_speed = (enum i40e_aq_link_speed)resp->link_speed; hw_link_info->link_info = resp->link_info; hw_link_info->an_info = resp->an_info; + hw_link_info->fec_info = resp->config & (I40E_AQ_CONFIG_FEC_KR_ENA | + I40E_AQ_CONFIG_FEC_RS_ENA); hw_link_info->ext_info = resp->ext_info; - hw_link_info->loopback = resp->loopback; + hw_link_info->loopback = resp->loopback & I40E_AQ_LOOPBACK_MASK; hw_link_info->max_frame_size = LE16_TO_CPU(resp->max_frame_size); hw_link_info->pacing = resp->config & I40E_AQ_CONFIG_PACING_MASK; @@ -1948,15 +1993,22 @@ enum i40e_status_code i40e_aq_get_link_info(struct i40e_hw *hw, else hw_link_info->crc_enable = false; - if (resp->command_flags & CPU_TO_LE16(I40E_AQ_LSE_ENABLE)) + if (resp->command_flags & CPU_TO_LE16(I40E_AQ_LSE_IS_ENABLED)) hw_link_info->lse_enable = true; else hw_link_info->lse_enable = false; - if ((hw->aq.fw_maj_ver < 4 || (hw->aq.fw_maj_ver == 4 && + if ((hw->mac.type == I40E_MAC_XL710) && + (hw->aq.fw_maj_ver < 4 || (hw->aq.fw_maj_ver == 4 && hw->aq.fw_min_ver < 40)) && hw_link_info->phy_type == 0xE) hw_link_info->phy_type = I40E_PHY_TYPE_10GBASE_SFPP_CU; + if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR && + hw->aq.api_min_ver >= 7) { + hw->phy.phy_types = LE32_TO_CPU(*(__le32 *)resp->link_type); + hw->phy.phy_types |= ((u64)resp->link_type_ext << 32); + } + /* save link status information */ if (link) i40e_memcpy(link, hw_link_info, sizeof(*hw_link_info), @@ -2214,6 +2266,34 @@ enum i40e_status_code i40e_aq_set_default_vsi(struct i40e_hw *hw, return status; } +/** + * i40e_aq_clear_default_vsi + * @hw: pointer to the hw struct + * @seid: vsi number + * @cmd_details: pointer to command details structure or NULL + **/ +enum i40e_status_code i40e_aq_clear_default_vsi(struct i40e_hw *hw, + u16 seid, + struct i40e_asq_cmd_details *cmd_details) +{ + struct i40e_aq_desc desc; + struct i40e_aqc_set_vsi_promiscuous_modes *cmd = + (struct i40e_aqc_set_vsi_promiscuous_modes *) + &desc.params.raw; + enum i40e_status_code status; + + i40e_fill_default_direct_cmd_desc(&desc, + i40e_aqc_opc_set_vsi_promiscuous_modes); + + cmd->promiscuous_flags = CPU_TO_LE16(0); + cmd->valid_flags = CPU_TO_LE16(I40E_AQC_SET_VSI_DEFAULT); + cmd->seid = CPU_TO_LE16(seid); + + status = i40e_asq_send_command(hw, &desc, NULL, 0, cmd_details); + + return status; +} + /** * i40e_aq_set_vsi_unicast_promiscuous * @hw: pointer to the hw struct @@ -2289,6 +2369,43 @@ enum i40e_status_code i40e_aq_set_vsi_multicast_promiscuous(struct i40e_hw *hw, return status; } +/** +* i40e_aq_set_vsi_full_promiscuous +* @hw: pointer to the hw struct +* @seid: VSI number +* @set: set promiscuous enable/disable +* @cmd_details: pointer to command details structure or NULL +**/ +enum i40e_status_code i40e_aq_set_vsi_full_promiscuous(struct i40e_hw *hw, + u16 seid, bool set, + struct i40e_asq_cmd_details *cmd_details) +{ + struct i40e_aq_desc desc; + struct i40e_aqc_set_vsi_promiscuous_modes *cmd = + (struct i40e_aqc_set_vsi_promiscuous_modes *)&desc.params.raw; + enum i40e_status_code status; + u16 flags = 0; + + i40e_fill_default_direct_cmd_desc(&desc, + i40e_aqc_opc_set_vsi_promiscuous_modes); + + if (set) + flags = I40E_AQC_SET_VSI_PROMISC_UNICAST | + I40E_AQC_SET_VSI_PROMISC_MULTICAST | + I40E_AQC_SET_VSI_PROMISC_BROADCAST; + + cmd->promiscuous_flags = CPU_TO_LE16(flags); + + cmd->valid_flags = CPU_TO_LE16(I40E_AQC_SET_VSI_PROMISC_UNICAST | + I40E_AQC_SET_VSI_PROMISC_MULTICAST | + I40E_AQC_SET_VSI_PROMISC_BROADCAST); + + cmd->seid = CPU_TO_LE16(seid); + status = i40e_asq_send_command(hw, &desc, NULL, 0, cmd_details); + + return status; +} + /** * i40e_aq_set_vsi_mc_promisc_on_vlan * @hw: pointer to the hw struct @@ -2357,6 +2474,40 @@ enum i40e_status_code i40e_aq_set_vsi_uc_promisc_on_vlan(struct i40e_hw *hw, return status; } +/** + * i40e_aq_set_vsi_bc_promisc_on_vlan + * @hw: pointer to the hw struct + * @seid: vsi number + * @enable: set broadcast promiscuous enable/disable for a given VLAN + * @vid: The VLAN tag filter - capture any broadcast packet with this VLAN tag + * @cmd_details: pointer to command details structure or NULL + **/ +enum i40e_status_code i40e_aq_set_vsi_bc_promisc_on_vlan(struct i40e_hw *hw, + u16 seid, bool enable, u16 vid, + struct i40e_asq_cmd_details *cmd_details) +{ + struct i40e_aq_desc desc; + struct i40e_aqc_set_vsi_promiscuous_modes *cmd = + (struct i40e_aqc_set_vsi_promiscuous_modes *)&desc.params.raw; + enum i40e_status_code status; + u16 flags = 0; + + i40e_fill_default_direct_cmd_desc(&desc, + i40e_aqc_opc_set_vsi_promiscuous_modes); + + if (enable) + flags |= I40E_AQC_SET_VSI_PROMISC_BROADCAST; + + cmd->promiscuous_flags = CPU_TO_LE16(flags); + cmd->valid_flags = CPU_TO_LE16(I40E_AQC_SET_VSI_PROMISC_BROADCAST); + cmd->seid = CPU_TO_LE16(seid); + cmd->vlan_tag = CPU_TO_LE16(vid | I40E_AQC_SET_VSI_VLAN_VALID); + + status = i40e_asq_send_command(hw, &desc, NULL, 0, cmd_details); + + return status; +} + /** * i40e_aq_set_vsi_broadcast * @hw: pointer to the hw struct @@ -2554,7 +2705,11 @@ enum i40e_status_code i40e_aq_set_switch_config(struct i40e_hw *hw, i40e_aqc_opc_set_switch_config); scfg->flags = CPU_TO_LE16(flags); scfg->valid_flags = CPU_TO_LE16(valid_flags); - + if (hw->flags & I40E_HW_FLAG_802_1AD_CAPABLE) { + scfg->switch_tag = CPU_TO_LE16(hw->switch_tag); + scfg->first_tag = CPU_TO_LE16(hw->first_tag); + scfg->second_tag = CPU_TO_LE16(hw->second_tag); + } status = i40e_asq_send_command(hw, &desc, NULL, 0, cmd_details); return status; @@ -2691,14 +2846,21 @@ enum i40e_status_code i40e_update_link_info(struct i40e_hw *hw) if (status) return status; - if (hw->phy.link_info.link_info & I40E_AQ_MEDIA_AVAILABLE) { + /* extra checking needed to ensure link info to user is timely */ + if ((hw->phy.link_info.link_info & I40E_AQ_MEDIA_AVAILABLE) && + ((hw->phy.link_info.link_info & I40E_AQ_LINK_UP) || + !(hw->phy.link_info_old.link_info & I40E_AQ_LINK_UP))) { status = i40e_aq_get_phy_capabilities(hw, false, false, &abilities, NULL); if (status) return status; - memcpy(hw->phy.link_info.module_type, &abilities.module_type, - sizeof(hw->phy.link_info.module_type)); + hw->phy.link_info.req_fec_info = + abilities.fec_cfg_curr_mod_ext_info & + (I40E_AQ_REQUEST_FEC_KR | I40E_AQ_REQUEST_FEC_RS); + + i40e_memcpy(hw->phy.link_info.module_type, &abilities.module_type, + sizeof(hw->phy.link_info.module_type), I40E_NONDMA_TO_NONDMA); } return status; } @@ -3549,6 +3711,14 @@ STATIC void i40e_parse_discover_capabilities(struct i40e_hw *hw, void *buff, break; case I40E_AQ_CAP_ID_MNG_MODE: p->management_mode = number; + if (major_rev > 1) { + p->mng_protocols_over_mctp = logical_id; + i40e_debug(hw, I40E_DEBUG_INIT, + "HW Capability: Protocols over MCTP = %d\n", + p->mng_protocols_over_mctp); + } else { + p->mng_protocols_over_mctp = 0; + } i40e_debug(hw, I40E_DEBUG_INIT, "HW Capability: Management Mode = %d\n", p->management_mode); @@ -3768,7 +3938,6 @@ STATIC void i40e_parse_discover_capabilities(struct i40e_hw *hw, void *buff, if (number & I40E_NVM_MGMT_UPDATE_DISABLED) p->update_disabled = true; break; -#ifdef X722_SUPPORT case I40E_AQ_CAP_ID_WOL_AND_PROXY: hw->num_wol_proxy_filters = (u16)number; hw->wol_proxy_vsi_seid = (u16)logical_id; @@ -3778,12 +3947,10 @@ STATIC void i40e_parse_discover_capabilities(struct i40e_hw *hw, void *buff, else p->acpi_prog_method = I40E_ACPI_PROGRAMMING_METHOD_HW_FVL; p->proxy_support = (phys_id & I40E_PROXY_SUPPORT_MASK) ? 1 : 0; - p->proxy_support = p->proxy_support; i40e_debug(hw, I40E_DEBUG_INIT, "HW Capability: WOL proxy filters = %d\n", hw->num_wol_proxy_filters); break; -#endif default: break; } @@ -3792,16 +3959,8 @@ STATIC void i40e_parse_discover_capabilities(struct i40e_hw *hw, void *buff, if (p->fcoe) i40e_debug(hw, I40E_DEBUG_ALL, "device is FCoE capable\n"); -#ifdef I40E_FCOE_ENA - /* Software override ensuring FCoE is disabled if npar or mfp - * mode because it is not supported in these modes. - */ - if (p->npar_enable || p->flex10_enable) - p->fcoe = false; -#else /* Always disable FCoE if compiled without the I40E_FCOE_ENA flag */ p->fcoe = false; -#endif /* count the enabled ports (aka the "not disabled" ports) */ hw->num_ports = 0; @@ -3828,8 +3987,10 @@ STATIC void i40e_parse_discover_capabilities(struct i40e_hw *hw, void *buff, /* partition id is 1-based, and functions are evenly spread * across the ports as partitions */ - hw->partition_id = (hw->pf_id / hw->num_ports) + 1; - hw->num_partitions = num_functions / hw->num_ports; + if (hw->num_ports != 0) { + hw->partition_id = (hw->pf_id / hw->num_ports) + 1; + hw->num_partitions = num_functions / hw->num_ports; + } /* additional HW specific goodies that might * someday be HW version specific @@ -4314,11 +4475,15 @@ enum i40e_status_code i40e_aq_start_stop_dcbx(struct i40e_hw *hw, /** * i40e_aq_add_udp_tunnel * @hw: pointer to the hw struct - * @udp_port: the UDP port to add + * @udp_port: the UDP port to add in Host byte order * @header_len: length of the tunneling header length in DWords * @protocol_index: protocol index type * @filter_index: pointer to filter index * @cmd_details: pointer to command details structure or NULL + * + * Note: Firmware expects the udp_port value to be in Little Endian format, + * and this function will call CPU_TO_LE16 to convert from Host byte order to + * Little Endian order. **/ enum i40e_status_code i40e_aq_add_udp_tunnel(struct i40e_hw *hw, u16 udp_port, u8 protocol_index, @@ -5392,7 +5557,7 @@ enum i40e_status_code i40e_aq_add_rem_control_packet_filter(struct i40e_hw *hw, } if (mac_addr) - i40e_memcpy(cmd->mac, mac_addr, I40E_ETH_LENGTH_OF_ADDRESS, + i40e_memcpy(cmd->mac, mac_addr, ETH_ALEN, I40E_NONDMA_TO_NONDMA); cmd->etype = CPU_TO_LE16(ethtype); @@ -5452,12 +5617,12 @@ STATIC void i40e_fix_up_geneve_vni( u16 tnl_type; u32 ti; - tnl_type = (le16_to_cpu(f[i].flags) & + tnl_type = (LE16_TO_CPU(f[i].flags) & I40E_AQC_ADD_CLOUD_TNL_TYPE_MASK) >> I40E_AQC_ADD_CLOUD_TNL_TYPE_SHIFT; if (tnl_type == I40E_AQC_ADD_CLOUD_TNL_TYPE_GENEVE) { - ti = le32_to_cpu(f[i].tenant_id); - f[i].tenant_id = cpu_to_le32(ti << 8); + ti = LE32_TO_CPU(f[i].tenant_id); + f[i].tenant_id = CPU_TO_LE32(ti << 8); } } } @@ -5501,6 +5666,59 @@ enum i40e_status_code i40e_aq_add_cloud_filters(struct i40e_hw *hw, return status; } +/** + * i40e_aq_add_cloud_filters_big_buffer + * @hw: pointer to the hardware structure + * @seid: VSI seid to add cloud filters from + * @filters: Buffer which contains the filters in big buffer to be added + * @filter_count: number of filters contained in the buffer + * + * Set the cloud filters for a given VSI. The contents of the + * i40e_aqc_add_rm_cloud_filt_elem_ext are filled in by the caller of + * the function. + * + **/ +enum i40e_status_code i40e_aq_add_cloud_filters_big_buffer(struct i40e_hw *hw, + u16 seid, + struct i40e_aqc_add_rm_cloud_filt_elem_ext *filters, + u8 filter_count) +{ + struct i40e_aq_desc desc; + struct i40e_aqc_add_remove_cloud_filters *cmd = + (struct i40e_aqc_add_remove_cloud_filters *)&desc.params.raw; + enum i40e_status_code status; + u16 buff_len; + int i; + + i40e_fill_default_direct_cmd_desc(&desc, + i40e_aqc_opc_add_cloud_filters); + + buff_len = filter_count * sizeof(*filters); + desc.datalen = CPU_TO_LE16(buff_len); + desc.flags |= CPU_TO_LE16((u16)(I40E_AQ_FLAG_BUF | I40E_AQ_FLAG_RD)); + cmd->num_filters = filter_count; + cmd->seid = CPU_TO_LE16(seid); + cmd->big_buffer_flag = I40E_AQC_ADD_REM_CLOUD_CMD_BIG_BUFFER; + + /* adjust Geneve VNI for HW issue */ + for (i = 0; i < filter_count; i++) { + u16 tnl_type; + u32 ti; + + tnl_type = (LE16_TO_CPU(filters[i].element.flags) & + I40E_AQC_ADD_CLOUD_TNL_TYPE_MASK) >> + I40E_AQC_ADD_CLOUD_TNL_TYPE_SHIFT; + if (tnl_type == I40E_AQC_ADD_CLOUD_TNL_TYPE_GENEVE) { + ti = LE32_TO_CPU(filters[i].element.tenant_id); + filters[i].element.tenant_id = CPU_TO_LE32(ti << 8); + } + } + + status = i40e_asq_send_command(hw, &desc, filters, buff_len, NULL); + + return status; +} + /** * i40e_aq_remove_cloud_filters * @hw: pointer to the hardware structure @@ -5514,9 +5732,9 @@ enum i40e_status_code i40e_aq_add_cloud_filters(struct i40e_hw *hw, * **/ enum i40e_status_code i40e_aq_remove_cloud_filters(struct i40e_hw *hw, - u16 seid, - struct i40e_aqc_add_remove_cloud_filters_element_data *filters, - u8 filter_count) + u16 seid, + struct i40e_aqc_add_remove_cloud_filters_element_data *filters, + u8 filter_count) { struct i40e_aq_desc desc; struct i40e_aqc_add_remove_cloud_filters *cmd = @@ -5540,6 +5758,103 @@ enum i40e_status_code i40e_aq_remove_cloud_filters(struct i40e_hw *hw, return status; } +/** + * i40e_aq_remove_cloud_filters_big_buffer + * @hw: pointer to the hardware structure + * @seid: VSI seid to remove cloud filters from + * @filters: Buffer which contains the filters in big buffer to be removed + * @filter_count: number of filters contained in the buffer + * + * Remove the cloud filters for a given VSI. The contents of the + * i40e_aqc_add_rm_cloud_filt_elem_ext are filled in by the caller of + * the function. + * + **/ +enum i40e_status_code i40e_aq_remove_cloud_filters_big_buffer( + struct i40e_hw *hw, + u16 seid, + struct i40e_aqc_add_rm_cloud_filt_elem_ext *filters, + u8 filter_count) +{ + struct i40e_aq_desc desc; + struct i40e_aqc_add_remove_cloud_filters *cmd = + (struct i40e_aqc_add_remove_cloud_filters *)&desc.params.raw; + enum i40e_status_code status; + u16 buff_len; + int i; + + i40e_fill_default_direct_cmd_desc(&desc, + i40e_aqc_opc_remove_cloud_filters); + + buff_len = filter_count * sizeof(*filters); + desc.datalen = CPU_TO_LE16(buff_len); + desc.flags |= CPU_TO_LE16((u16)(I40E_AQ_FLAG_BUF | I40E_AQ_FLAG_RD)); + cmd->num_filters = filter_count; + cmd->seid = CPU_TO_LE16(seid); + cmd->big_buffer_flag = I40E_AQC_ADD_REM_CLOUD_CMD_BIG_BUFFER; + + /* adjust Geneve VNI for HW issue */ + for (i = 0; i < filter_count; i++) { + u16 tnl_type; + u32 ti; + + tnl_type = (LE16_TO_CPU(filters[i].element.flags) & + I40E_AQC_ADD_CLOUD_TNL_TYPE_MASK) >> + I40E_AQC_ADD_CLOUD_TNL_TYPE_SHIFT; + if (tnl_type == I40E_AQC_ADD_CLOUD_TNL_TYPE_GENEVE) { + ti = LE32_TO_CPU(filters[i].element.tenant_id); + filters[i].element.tenant_id = CPU_TO_LE32(ti << 8); + } + } + + status = i40e_asq_send_command(hw, &desc, filters, buff_len, NULL); + + return status; +} + +/** + * i40e_aq_replace_cloud_filters - Replace cloud filter command + * @hw: pointer to the hw struct + * @filters: pointer to the i40e_aqc_replace_cloud_filter_cmd struct + * @cmd_buf: pointer to the i40e_aqc_replace_cloud_filter_cmd_buf struct + * + **/ +enum +i40e_status_code i40e_aq_replace_cloud_filters(struct i40e_hw *hw, + struct i40e_aqc_replace_cloud_filters_cmd *filters, + struct i40e_aqc_replace_cloud_filters_cmd_buf *cmd_buf) +{ + struct i40e_aq_desc desc; + struct i40e_aqc_replace_cloud_filters_cmd *cmd = + (struct i40e_aqc_replace_cloud_filters_cmd *)&desc.params.raw; + enum i40e_status_code status = I40E_SUCCESS; + int i = 0; + + i40e_fill_default_direct_cmd_desc(&desc, + i40e_aqc_opc_replace_cloud_filters); + + desc.datalen = CPU_TO_LE16(32); + desc.flags |= CPU_TO_LE16((u16)(I40E_AQ_FLAG_BUF | I40E_AQ_FLAG_RD)); + cmd->old_filter_type = filters->old_filter_type; + cmd->new_filter_type = filters->new_filter_type; + cmd->valid_flags = filters->valid_flags; + cmd->tr_bit = filters->tr_bit; + + status = i40e_asq_send_command(hw, &desc, cmd_buf, + sizeof(struct i40e_aqc_replace_cloud_filters_cmd_buf), NULL); + + /* for get cloud filters command */ + for (i = 0; i < 32; i += 4) { + cmd_buf->filters[i / 4].filter_type = cmd_buf->data[i]; + cmd_buf->filters[i / 4].input[0] = cmd_buf->data[i + 1]; + cmd_buf->filters[i / 4].input[1] = cmd_buf->data[i + 2]; + cmd_buf->filters[i / 4].input[2] = cmd_buf->data[i + 3]; + } + + return status; +} + + /** * i40e_aq_alternate_write * @hw: pointer to the hardware structure @@ -5961,9 +6276,6 @@ enum i40e_status_code i40e_aq_configure_partition_bw(struct i40e_hw *hw, desc.flags |= CPU_TO_LE16((u16)I40E_AQ_FLAG_BUF); desc.flags |= CPU_TO_LE16((u16)I40E_AQ_FLAG_RD); - if (bwd_size > I40E_AQ_LARGE_BUF) - desc.flags |= CPU_TO_LE16((u16)I40E_AQ_FLAG_LB); - desc.datalen = CPU_TO_LE16(bwd_size); status = i40e_asq_send_command(hw, &desc, bw_data, bwd_size, cmd_details); @@ -5972,7 +6284,92 @@ enum i40e_status_code i40e_aq_configure_partition_bw(struct i40e_hw *hw, } /** - * i40e_read_phy_register + * i40e_read_phy_register_clause22 + * @hw: pointer to the HW structure + * @reg: register address in the page + * @phy_adr: PHY address on MDIO interface + * @value: PHY register value + * + * Reads specified PHY register value + **/ +enum i40e_status_code i40e_read_phy_register_clause22(struct i40e_hw *hw, + u16 reg, u8 phy_addr, u16 *value) +{ + enum i40e_status_code status = I40E_ERR_TIMEOUT; + u8 port_num = (u8)hw->func_caps.mdio_port_num; + u32 command = 0; + u16 retry = 1000; + + command = (reg << I40E_GLGEN_MSCA_DEVADD_SHIFT) | + (phy_addr << I40E_GLGEN_MSCA_PHYADD_SHIFT) | + (I40E_MDIO_CLAUSE22_OPCODE_READ_MASK) | + (I40E_MDIO_CLAUSE22_STCODE_MASK) | + (I40E_GLGEN_MSCA_MDICMD_MASK); + wr32(hw, I40E_GLGEN_MSCA(port_num), command); + do { + command = rd32(hw, I40E_GLGEN_MSCA(port_num)); + if (!(command & I40E_GLGEN_MSCA_MDICMD_MASK)) { + status = I40E_SUCCESS; + break; + } + i40e_usec_delay(10); + retry--; + } while (retry); + + if (status) { + i40e_debug(hw, I40E_DEBUG_PHY, + "PHY: Can't write command to external PHY.\n"); + } else { + command = rd32(hw, I40E_GLGEN_MSRWD(port_num)); + *value = (command & I40E_GLGEN_MSRWD_MDIRDDATA_MASK) >> + I40E_GLGEN_MSRWD_MDIRDDATA_SHIFT; + } + + return status; +} + +/** + * i40e_write_phy_register_clause22 + * @hw: pointer to the HW structure + * @reg: register address in the page + * @phy_adr: PHY address on MDIO interface + * @value: PHY register value + * + * Writes specified PHY register value + **/ +enum i40e_status_code i40e_write_phy_register_clause22(struct i40e_hw *hw, + u16 reg, u8 phy_addr, u16 value) +{ + enum i40e_status_code status = I40E_ERR_TIMEOUT; + u8 port_num = (u8)hw->func_caps.mdio_port_num; + u32 command = 0; + u16 retry = 1000; + + command = value << I40E_GLGEN_MSRWD_MDIWRDATA_SHIFT; + wr32(hw, I40E_GLGEN_MSRWD(port_num), command); + + command = (reg << I40E_GLGEN_MSCA_DEVADD_SHIFT) | + (phy_addr << I40E_GLGEN_MSCA_PHYADD_SHIFT) | + (I40E_MDIO_CLAUSE22_OPCODE_WRITE_MASK) | + (I40E_MDIO_CLAUSE22_STCODE_MASK) | + (I40E_GLGEN_MSCA_MDICMD_MASK); + + wr32(hw, I40E_GLGEN_MSCA(port_num), command); + do { + command = rd32(hw, I40E_GLGEN_MSCA(port_num)); + if (!(command & I40E_GLGEN_MSCA_MDICMD_MASK)) { + status = I40E_SUCCESS; + break; + } + i40e_usec_delay(10); + retry--; + } while (retry); + + return status; +} + +/** + * i40e_read_phy_register_clause45 * @hw: pointer to the HW structure * @page: registers page number * @reg: register address in the page @@ -5981,9 +6378,8 @@ enum i40e_status_code i40e_aq_configure_partition_bw(struct i40e_hw *hw, * * Reads specified PHY register value **/ -enum i40e_status_code i40e_read_phy_register(struct i40e_hw *hw, - u8 page, u16 reg, u8 phy_addr, - u16 *value) +enum i40e_status_code i40e_read_phy_register_clause45(struct i40e_hw *hw, + u8 page, u16 reg, u8 phy_addr, u16 *value) { enum i40e_status_code status = I40E_ERR_TIMEOUT; u32 command = 0; @@ -5993,8 +6389,8 @@ enum i40e_status_code i40e_read_phy_register(struct i40e_hw *hw, command = (reg << I40E_GLGEN_MSCA_MDIADD_SHIFT) | (page << I40E_GLGEN_MSCA_DEVADD_SHIFT) | (phy_addr << I40E_GLGEN_MSCA_PHYADD_SHIFT) | - (I40E_MDIO_OPCODE_ADDRESS) | - (I40E_MDIO_STCODE) | + (I40E_MDIO_CLAUSE45_OPCODE_ADDRESS_MASK) | + (I40E_MDIO_CLAUSE45_STCODE_MASK) | (I40E_GLGEN_MSCA_MDICMD_MASK) | (I40E_GLGEN_MSCA_MDIINPROGEN_MASK); wr32(hw, I40E_GLGEN_MSCA(port_num), command); @@ -6016,8 +6412,8 @@ enum i40e_status_code i40e_read_phy_register(struct i40e_hw *hw, command = (page << I40E_GLGEN_MSCA_DEVADD_SHIFT) | (phy_addr << I40E_GLGEN_MSCA_PHYADD_SHIFT) | - (I40E_MDIO_OPCODE_READ) | - (I40E_MDIO_STCODE) | + (I40E_MDIO_CLAUSE45_OPCODE_READ_MASK) | + (I40E_MDIO_CLAUSE45_STCODE_MASK) | (I40E_GLGEN_MSCA_MDICMD_MASK) | (I40E_GLGEN_MSCA_MDIINPROGEN_MASK); status = I40E_ERR_TIMEOUT; @@ -6047,7 +6443,7 @@ phy_read_end: } /** - * i40e_write_phy_register + * i40e_write_phy_register_clause45 * @hw: pointer to the HW structure * @page: registers page number * @reg: register address in the page @@ -6056,9 +6452,8 @@ phy_read_end: * * Writes value to specified PHY register **/ -enum i40e_status_code i40e_write_phy_register(struct i40e_hw *hw, - u8 page, u16 reg, u8 phy_addr, - u16 value) +enum i40e_status_code i40e_write_phy_register_clause45(struct i40e_hw *hw, + u8 page, u16 reg, u8 phy_addr, u16 value) { enum i40e_status_code status = I40E_ERR_TIMEOUT; u32 command = 0; @@ -6068,8 +6463,8 @@ enum i40e_status_code i40e_write_phy_register(struct i40e_hw *hw, command = (reg << I40E_GLGEN_MSCA_MDIADD_SHIFT) | (page << I40E_GLGEN_MSCA_DEVADD_SHIFT) | (phy_addr << I40E_GLGEN_MSCA_PHYADD_SHIFT) | - (I40E_MDIO_OPCODE_ADDRESS) | - (I40E_MDIO_STCODE) | + (I40E_MDIO_CLAUSE45_OPCODE_ADDRESS_MASK) | + (I40E_MDIO_CLAUSE45_STCODE_MASK) | (I40E_GLGEN_MSCA_MDICMD_MASK) | (I40E_GLGEN_MSCA_MDIINPROGEN_MASK); wr32(hw, I40E_GLGEN_MSCA(port_num), command); @@ -6093,8 +6488,8 @@ enum i40e_status_code i40e_write_phy_register(struct i40e_hw *hw, command = (page << I40E_GLGEN_MSCA_DEVADD_SHIFT) | (phy_addr << I40E_GLGEN_MSCA_PHYADD_SHIFT) | - (I40E_MDIO_OPCODE_WRITE) | - (I40E_MDIO_STCODE) | + (I40E_MDIO_CLAUSE45_OPCODE_WRITE_MASK) | + (I40E_MDIO_CLAUSE45_STCODE_MASK) | (I40E_GLGEN_MSCA_MDICMD_MASK) | (I40E_GLGEN_MSCA_MDIINPROGEN_MASK); status = I40E_ERR_TIMEOUT; @@ -6114,6 +6509,78 @@ phy_write_end: return status; } +/** + * i40e_write_phy_register + * @hw: pointer to the HW structure + * @page: registers page number + * @reg: register address in the page + * @phy_adr: PHY address on MDIO interface + * @value: PHY register value + * + * Writes value to specified PHY register + **/ +enum i40e_status_code i40e_write_phy_register(struct i40e_hw *hw, + u8 page, u16 reg, u8 phy_addr, u16 value) +{ + enum i40e_status_code status; + + switch (hw->device_id) { + case I40E_DEV_ID_1G_BASE_T_X722: + status = i40e_write_phy_register_clause22(hw, + reg, phy_addr, value); + break; + case I40E_DEV_ID_10G_BASE_T: + case I40E_DEV_ID_10G_BASE_T4: + case I40E_DEV_ID_10G_BASE_T_X722: + case I40E_DEV_ID_25G_B: + case I40E_DEV_ID_25G_SFP28: + status = i40e_write_phy_register_clause45(hw, + page, reg, phy_addr, value); + break; + default: + status = I40E_ERR_UNKNOWN_PHY; + break; + } + + return status; +} + +/** + * i40e_read_phy_register + * @hw: pointer to the HW structure + * @page: registers page number + * @reg: register address in the page + * @phy_adr: PHY address on MDIO interface + * @value: PHY register value + * + * Reads specified PHY register value + **/ +enum i40e_status_code i40e_read_phy_register(struct i40e_hw *hw, + u8 page, u16 reg, u8 phy_addr, u16 *value) +{ + enum i40e_status_code status; + + switch (hw->device_id) { + case I40E_DEV_ID_1G_BASE_T_X722: + status = i40e_read_phy_register_clause22(hw, reg, phy_addr, + value); + break; + case I40E_DEV_ID_10G_BASE_T: + case I40E_DEV_ID_10G_BASE_T4: + case I40E_DEV_ID_10G_BASE_T_X722: + case I40E_DEV_ID_25G_B: + case I40E_DEV_ID_25G_SFP28: + status = i40e_read_phy_register_clause45(hw, page, reg, + phy_addr, value); + break; + default: + status = I40E_ERR_UNKNOWN_PHY; + break; + } + + return status; +} + /** * i40e_get_phy_address * @hw: pointer to the HW structure @@ -6156,14 +6623,16 @@ enum i40e_status_code i40e_blink_phy_link_led(struct i40e_hw *hw, for (gpio_led_port = 0; gpio_led_port < 3; gpio_led_port++, led_addr++) { - status = i40e_read_phy_register(hw, I40E_PHY_COM_REG_PAGE, - led_addr, phy_addr, &led_reg); + status = i40e_read_phy_register_clause45(hw, + I40E_PHY_COM_REG_PAGE, + led_addr, phy_addr, + &led_reg); if (status) goto phy_blinking_end; led_ctl = led_reg; if (led_reg & I40E_PHY_LED_LINK_MODE_MASK) { led_reg = 0; - status = i40e_write_phy_register(hw, + status = i40e_write_phy_register_clause45(hw, I40E_PHY_COM_REG_PAGE, led_addr, phy_addr, led_reg); @@ -6175,20 +6644,18 @@ enum i40e_status_code i40e_blink_phy_link_led(struct i40e_hw *hw, if (time > 0 && interval > 0) { for (i = 0; i < time * 1000; i += interval) { - status = i40e_read_phy_register(hw, - I40E_PHY_COM_REG_PAGE, - led_addr, phy_addr, - &led_reg); + status = i40e_read_phy_register_clause45(hw, + I40E_PHY_COM_REG_PAGE, + led_addr, phy_addr, &led_reg); if (status) goto restore_config; if (led_reg & I40E_PHY_LED_MANUAL_ON) led_reg = 0; else led_reg = I40E_PHY_LED_MANUAL_ON; - status = i40e_write_phy_register(hw, - I40E_PHY_COM_REG_PAGE, - led_addr, phy_addr, - led_reg); + status = i40e_write_phy_register_clause45(hw, + I40E_PHY_COM_REG_PAGE, + led_addr, phy_addr, led_reg); if (status) goto restore_config; i40e_msec_delay(interval); @@ -6196,8 +6663,9 @@ enum i40e_status_code i40e_blink_phy_link_led(struct i40e_hw *hw, } restore_config: - status = i40e_write_phy_register(hw, I40E_PHY_COM_REG_PAGE, led_addr, - phy_addr, led_ctl); + status = i40e_write_phy_register_clause45(hw, + I40E_PHY_COM_REG_PAGE, + led_addr, phy_addr, led_ctl); phy_blinking_end: return status; @@ -6220,22 +6688,38 @@ enum i40e_status_code i40e_led_get_phy(struct i40e_hw *hw, u16 *led_addr, u16 temp_addr; u8 port_num; u32 i; + u32 reg_val_aq; - temp_addr = I40E_PHY_LED_PROV_REG_1; - i = rd32(hw, I40E_PFGEN_PORTNUM); - port_num = (u8)(i & I40E_PFGEN_PORTNUM_PORT_NUM_MASK); - phy_addr = i40e_get_phy_address(hw, port_num); - - for (gpio_led_port = 0; gpio_led_port < 3; gpio_led_port++, - temp_addr++) { - status = i40e_read_phy_register(hw, I40E_PHY_COM_REG_PAGE, - temp_addr, phy_addr, ®_val); + if (hw->flags & I40E_HW_FLAG_AQ_PHY_ACCESS_CAPABLE) { + status = + i40e_aq_get_phy_register(hw, + I40E_AQ_PHY_REG_ACCESS_EXTERNAL, + I40E_PHY_COM_REG_PAGE, + I40E_PHY_LED_PROV_REG_1, + ®_val_aq, NULL); if (status) return status; - *val = reg_val; - if (reg_val & I40E_PHY_LED_LINK_MODE_MASK) { - *led_addr = temp_addr; - break; + *val = (u16)reg_val_aq; + } else { + temp_addr = I40E_PHY_LED_PROV_REG_1; + i = rd32(hw, I40E_PFGEN_PORTNUM); + port_num = (u8)(i & I40E_PFGEN_PORTNUM_PORT_NUM_MASK); + phy_addr = i40e_get_phy_address(hw, port_num); + + for (gpio_led_port = 0; gpio_led_port < 3; gpio_led_port++, + temp_addr++) { + status = + i40e_read_phy_register_clause45(hw, + I40E_PHY_COM_REG_PAGE, + temp_addr, phy_addr, + ®_val); + if (status) + return status; + *val = reg_val; + if (reg_val & I40E_PHY_LED_LINK_MODE_MASK) { + *led_addr = temp_addr; + break; + } } } return status; @@ -6253,50 +6737,115 @@ enum i40e_status_code i40e_led_set_phy(struct i40e_hw *hw, bool on, u16 led_addr, u32 mode) { enum i40e_status_code status = I40E_SUCCESS; - u16 led_ctl = 0; - u16 led_reg = 0; + u32 led_ctl = 0; + u32 led_reg = 0; u8 phy_addr = 0; u8 port_num; u32 i; - i = rd32(hw, I40E_PFGEN_PORTNUM); - port_num = (u8)(i & I40E_PFGEN_PORTNUM_PORT_NUM_MASK); - phy_addr = i40e_get_phy_address(hw, port_num); - - status = i40e_read_phy_register(hw, I40E_PHY_COM_REG_PAGE, led_addr, - phy_addr, &led_reg); + if (hw->flags & I40E_HW_FLAG_AQ_PHY_ACCESS_CAPABLE) { + status = + i40e_aq_get_phy_register(hw, + I40E_AQ_PHY_REG_ACCESS_EXTERNAL, + I40E_PHY_COM_REG_PAGE, + I40E_PHY_LED_PROV_REG_1, + &led_reg, NULL); + } else { + i = rd32(hw, I40E_PFGEN_PORTNUM); + port_num = (u8)(i & I40E_PFGEN_PORTNUM_PORT_NUM_MASK); + phy_addr = i40e_get_phy_address(hw, port_num); + status = i40e_read_phy_register_clause45(hw, + I40E_PHY_COM_REG_PAGE, + led_addr, phy_addr, + (u16 *)&led_reg); + } if (status) return status; led_ctl = led_reg; if (led_reg & I40E_PHY_LED_LINK_MODE_MASK) { led_reg = 0; - status = i40e_write_phy_register(hw, I40E_PHY_COM_REG_PAGE, - led_addr, phy_addr, led_reg); + if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR && + hw->aq.api_min_ver >= I40E_MINOR_VER_GET_LINK_INFO_XL710) { + status = i40e_aq_set_phy_register(hw, + I40E_AQ_PHY_REG_ACCESS_EXTERNAL, + I40E_PHY_COM_REG_PAGE, + I40E_PHY_LED_PROV_REG_1, + led_reg, NULL); + } else { + status = i40e_write_phy_register_clause45(hw, + I40E_PHY_COM_REG_PAGE, + led_addr, phy_addr, + (u16)led_reg); + } if (status) return status; } - status = i40e_read_phy_register(hw, I40E_PHY_COM_REG_PAGE, - led_addr, phy_addr, &led_reg); + if (hw->flags & I40E_HW_FLAG_AQ_PHY_ACCESS_CAPABLE) { + status = + i40e_aq_get_phy_register(hw, + I40E_AQ_PHY_REG_ACCESS_EXTERNAL, + I40E_PHY_COM_REG_PAGE, + I40E_PHY_LED_PROV_REG_1, + &led_reg, NULL); + } else { + status = i40e_read_phy_register_clause45(hw, + I40E_PHY_COM_REG_PAGE, + led_addr, phy_addr, + (u16 *)&led_reg); + } if (status) goto restore_config; if (on) led_reg = I40E_PHY_LED_MANUAL_ON; else led_reg = 0; - status = i40e_write_phy_register(hw, I40E_PHY_COM_REG_PAGE, - led_addr, phy_addr, led_reg); + + if (hw->flags & I40E_HW_FLAG_AQ_PHY_ACCESS_CAPABLE) { + status = + i40e_aq_set_phy_register(hw, + I40E_AQ_PHY_REG_ACCESS_EXTERNAL, + I40E_PHY_COM_REG_PAGE, + I40E_PHY_LED_PROV_REG_1, + led_reg, NULL); + } else { + status = + i40e_write_phy_register_clause45(hw, I40E_PHY_COM_REG_PAGE, + led_addr, phy_addr, + (u16)led_reg); + } if (status) goto restore_config; if (mode & I40E_PHY_LED_MODE_ORIG) { led_ctl = (mode & I40E_PHY_LED_MODE_MASK); - status = i40e_write_phy_register(hw, - I40E_PHY_COM_REG_PAGE, - led_addr, phy_addr, led_ctl); + if (hw->flags & I40E_HW_FLAG_AQ_PHY_ACCESS_CAPABLE) { + status = i40e_aq_set_phy_register(hw, + I40E_AQ_PHY_REG_ACCESS_EXTERNAL, + I40E_PHY_COM_REG_PAGE, + I40E_PHY_LED_PROV_REG_1, + led_ctl, NULL); + } else { + status = i40e_write_phy_register_clause45(hw, + I40E_PHY_COM_REG_PAGE, + led_addr, phy_addr, + (u16)led_ctl); + } } return status; restore_config: - status = i40e_write_phy_register(hw, I40E_PHY_COM_REG_PAGE, led_addr, - phy_addr, led_ctl); + if (hw->flags & I40E_HW_FLAG_AQ_PHY_ACCESS_CAPABLE) { + status = + i40e_aq_set_phy_register(hw, + I40E_AQ_PHY_REG_ACCESS_EXTERNAL, + I40E_PHY_COM_REG_PAGE, + I40E_PHY_LED_PROV_REG_1, + led_ctl, NULL); + } else { + status = + i40e_write_phy_register_clause45(hw, + I40E_PHY_COM_REG_PAGE, + led_addr, phy_addr, + (u16)led_ctl); + } return status; } #endif /* PF_DRIVER */ @@ -6347,7 +6896,9 @@ u32 i40e_read_rx_ctl(struct i40e_hw *hw, u32 reg_addr) int retry = 5; u32 val = 0; - use_register = (hw->aq.api_maj_ver == 1) && (hw->aq.api_min_ver < 5); + use_register = (((hw->aq.api_maj_ver == 1) && + (hw->aq.api_min_ver < 5)) || + (hw->mac.type == I40E_MAC_X722)); if (!use_register) { do_retry: status = i40e_aq_rx_ctl_read_register(hw, reg_addr, &val, NULL); @@ -6406,7 +6957,9 @@ void i40e_write_rx_ctl(struct i40e_hw *hw, u32 reg_addr, u32 reg_val) bool use_register; int retry = 5; - use_register = (hw->aq.api_maj_ver == 1) && (hw->aq.api_min_ver < 5); + use_register = (((hw->aq.api_maj_ver == 1) && + (hw->aq.api_min_ver < 5)) || + (hw->mac.type == I40E_MAC_X722)); if (!use_register) { do_retry: status = i40e_aq_rx_ctl_write_register(hw, reg_addr, @@ -6422,6 +6975,76 @@ do_retry: if (status || use_register) wr32(hw, reg_addr, reg_val); } + +/** + * i40e_aq_set_phy_register + * @hw: pointer to the hw struct + * @phy_select: select which phy should be accessed + * @dev_addr: PHY device address + * @reg_addr: PHY register address + * @reg_val: new register value + * @cmd_details: pointer to command details structure or NULL + * + * Write the external PHY register. + **/ +enum i40e_status_code i40e_aq_set_phy_register(struct i40e_hw *hw, + u8 phy_select, u8 dev_addr, + u32 reg_addr, u32 reg_val, + struct i40e_asq_cmd_details *cmd_details) +{ + struct i40e_aq_desc desc; + struct i40e_aqc_phy_register_access *cmd = + (struct i40e_aqc_phy_register_access *)&desc.params.raw; + enum i40e_status_code status; + + i40e_fill_default_direct_cmd_desc(&desc, + i40e_aqc_opc_set_phy_register); + + cmd->phy_interface = phy_select; + cmd->dev_addres = dev_addr; + cmd->reg_address = reg_addr; + cmd->reg_value = reg_val; + + status = i40e_asq_send_command(hw, &desc, NULL, 0, cmd_details); + + return status; +} + +/** + * i40e_aq_get_phy_register + * @hw: pointer to the hw struct + * @phy_select: select which phy should be accessed + * @dev_addr: PHY device address + * @reg_addr: PHY register address + * @reg_val: read register value + * @cmd_details: pointer to command details structure or NULL + * + * Read the external PHY register. + **/ +enum i40e_status_code i40e_aq_get_phy_register(struct i40e_hw *hw, + u8 phy_select, u8 dev_addr, + u32 reg_addr, u32 *reg_val, + struct i40e_asq_cmd_details *cmd_details) +{ + struct i40e_aq_desc desc; + struct i40e_aqc_phy_register_access *cmd = + (struct i40e_aqc_phy_register_access *)&desc.params.raw; + enum i40e_status_code status; + + i40e_fill_default_direct_cmd_desc(&desc, + i40e_aqc_opc_get_phy_register); + + cmd->phy_interface = phy_select; + cmd->dev_addres = dev_addr; + cmd->reg_address = reg_addr; + + status = i40e_asq_send_command(hw, &desc, NULL, 0, cmd_details); + if (!status) + *reg_val = cmd->reg_value; + + return status; +} + #ifdef VF_DRIVER /** @@ -6438,7 +7061,7 @@ do_retry: * completion before returning. **/ enum i40e_status_code i40e_aq_send_msg_to_pf(struct i40e_hw *hw, - enum i40e_virtchnl_ops v_opcode, + enum virtchnl_ops v_opcode, enum i40e_status_code v_retval, u8 *msg, u16 msglen, struct i40e_asq_cmd_details *cmd_details) @@ -6477,9 +7100,9 @@ enum i40e_status_code i40e_aq_send_msg_to_pf(struct i40e_hw *hw, * with appropriate information. **/ void i40e_vf_parse_hw_config(struct i40e_hw *hw, - struct i40e_virtchnl_vf_resource *msg) + struct virtchnl_vf_resource *msg) { - struct i40e_virtchnl_vsi_resource *vsi_res; + struct virtchnl_vsi_resource *vsi_res; int i; vsi_res = &msg->vsi_res[0]; @@ -6489,19 +7112,17 @@ void i40e_vf_parse_hw_config(struct i40e_hw *hw, hw->dev_caps.num_tx_qp = msg->num_queue_pairs; hw->dev_caps.num_msix_vectors_vf = msg->max_vectors; hw->dev_caps.dcb = msg->vf_offload_flags & - I40E_VIRTCHNL_VF_OFFLOAD_L2; - hw->dev_caps.fcoe = (msg->vf_offload_flags & - I40E_VIRTCHNL_VF_OFFLOAD_FCOE) ? 1 : 0; + VIRTCHNL_VF_OFFLOAD_L2; hw->dev_caps.iwarp = (msg->vf_offload_flags & - I40E_VIRTCHNL_VF_OFFLOAD_IWARP) ? 1 : 0; + VIRTCHNL_VF_OFFLOAD_IWARP) ? 1 : 0; for (i = 0; i < msg->num_vsis; i++) { - if (vsi_res->vsi_type == I40E_VSI_SRIOV) { + if (vsi_res->vsi_type == VIRTCHNL_VSI_SRIOV) { i40e_memcpy(hw->mac.perm_addr, vsi_res->default_mac_addr, - I40E_ETH_LENGTH_OF_ADDRESS, + ETH_ALEN, I40E_NONDMA_TO_NONDMA); i40e_memcpy(hw->mac.addr, vsi_res->default_mac_addr, - I40E_ETH_LENGTH_OF_ADDRESS, + ETH_ALEN, I40E_NONDMA_TO_NONDMA); } vsi_res++; @@ -6518,11 +7139,10 @@ void i40e_vf_parse_hw_config(struct i40e_hw *hw, **/ enum i40e_status_code i40e_vf_reset(struct i40e_hw *hw) { - return i40e_aq_send_msg_to_pf(hw, I40E_VIRTCHNL_OP_RESET_VF, + return i40e_aq_send_msg_to_pf(hw, VIRTCHNL_OP_RESET_VF, I40E_SUCCESS, NULL, 0, NULL); } #endif /* VF_DRIVER */ -#ifdef X722_SUPPORT /** * i40e_aq_set_arp_proxy_config @@ -6545,10 +7165,13 @@ enum i40e_status_code i40e_aq_set_arp_proxy_config(struct i40e_hw *hw, i40e_fill_default_direct_cmd_desc(&desc, i40e_aqc_opc_set_proxy_config); + desc.flags |= CPU_TO_LE16((u16)I40E_AQ_FLAG_BUF); + desc.flags |= CPU_TO_LE16((u16)I40E_AQ_FLAG_RD); desc.params.external.addr_high = CPU_TO_LE32(I40E_HI_DWORD((u64)proxy_config)); desc.params.external.addr_low = CPU_TO_LE32(I40E_LO_DWORD((u64)proxy_config)); + desc.datalen = CPU_TO_LE16(sizeof(struct i40e_aqc_arp_proxy_data)); status = i40e_asq_send_command(hw, &desc, proxy_config, sizeof(struct i40e_aqc_arp_proxy_data), @@ -6579,10 +7202,13 @@ enum i40e_status_code i40e_aq_set_ns_proxy_table_entry(struct i40e_hw *hw, i40e_fill_default_direct_cmd_desc(&desc, i40e_aqc_opc_set_ns_proxy_table_entry); + desc.flags |= CPU_TO_LE16((u16)I40E_AQ_FLAG_BUF); + desc.flags |= CPU_TO_LE16((u16)I40E_AQ_FLAG_RD); desc.params.external.addr_high = CPU_TO_LE32(I40E_HI_DWORD((u64)ns_proxy_table_entry)); desc.params.external.addr_low = CPU_TO_LE32(I40E_LO_DWORD((u64)ns_proxy_table_entry)); + desc.datalen = CPU_TO_LE16(sizeof(struct i40e_aqc_ns_proxy_data)); status = i40e_asq_send_command(hw, &desc, ns_proxy_table_entry, sizeof(struct i40e_aqc_ns_proxy_data), @@ -6629,9 +7255,11 @@ enum i40e_status_code i40e_aq_set_clear_wol_filter(struct i40e_hw *hw, if (set_filter) { if (!filter) return I40E_ERR_PARAM; + cmd_flags |= I40E_AQC_SET_WOL_FILTER; - buff_len = sizeof(*filter); + cmd_flags |= I40E_AQC_SET_WOL_FILTER_WOL_PRESERVE_ON_PFR; } + if (no_wol_tco) cmd_flags |= I40E_AQC_SET_WOL_FILTER_NO_TCO_WOL; cmd->cmd_flags = CPU_TO_LE16(cmd_flags); @@ -6642,6 +7270,12 @@ enum i40e_status_code i40e_aq_set_clear_wol_filter(struct i40e_hw *hw, valid_flags |= I40E_AQC_SET_WOL_FILTER_NO_TCO_ACTION_VALID; cmd->valid_flags = CPU_TO_LE16(valid_flags); + buff_len = sizeof(*filter); + desc.datalen = CPU_TO_LE16(buff_len); + + desc.flags |= CPU_TO_LE16((u16)I40E_AQ_FLAG_BUF); + desc.flags |= CPU_TO_LE16((u16)I40E_AQ_FLAG_RD); + cmd->address_high = CPU_TO_LE32(I40E_HI_DWORD((u64)filter)); cmd->address_low = CPU_TO_LE32(I40E_LO_DWORD((u64)filter)); @@ -6678,4 +7312,443 @@ enum i40e_status_code i40e_aq_get_wake_event_reason(struct i40e_hw *hw, return status; } -#endif /* X722_SUPPORT */ +/** +* i40e_aq_clear_all_wol_filters +* @hw: pointer to the hw struct +* @cmd_details: pointer to command details structure or NULL +* +* Get information for the reason of a Wake Up event +**/ +enum i40e_status_code i40e_aq_clear_all_wol_filters(struct i40e_hw *hw, + struct i40e_asq_cmd_details *cmd_details) +{ + struct i40e_aq_desc desc; + enum i40e_status_code status; + + i40e_fill_default_direct_cmd_desc(&desc, + i40e_aqc_opc_clear_all_wol_filters); + + status = i40e_asq_send_command(hw, &desc, NULL, 0, cmd_details); + + return status; +} + + +/** + * i40e_aq_write_ddp - Write dynamic device personalization (ddp) + * @hw: pointer to the hw struct + * @buff: command buffer (size in bytes = buff_size) + * @buff_size: buffer size in bytes + * @track_id: package tracking id + * @error_offset: returns error offset + * @error_info: returns error information + * @cmd_details: pointer to command details structure or NULL + **/ +enum +i40e_status_code i40e_aq_write_ddp(struct i40e_hw *hw, void *buff, + u16 buff_size, u32 track_id, + u32 *error_offset, u32 *error_info, + struct i40e_asq_cmd_details *cmd_details) +{ + struct i40e_aq_desc desc; + struct i40e_aqc_write_personalization_profile *cmd = + (struct i40e_aqc_write_personalization_profile *) + &desc.params.raw; + struct i40e_aqc_write_ddp_resp *resp; + enum i40e_status_code status; + + i40e_fill_default_direct_cmd_desc(&desc, + i40e_aqc_opc_write_personalization_profile); + + desc.flags |= CPU_TO_LE16(I40E_AQ_FLAG_BUF | I40E_AQ_FLAG_RD); + if (buff_size > I40E_AQ_LARGE_BUF) + desc.flags |= CPU_TO_LE16((u16)I40E_AQ_FLAG_LB); + + desc.datalen = CPU_TO_LE16(buff_size); + + cmd->profile_track_id = CPU_TO_LE32(track_id); + + status = i40e_asq_send_command(hw, &desc, buff, buff_size, cmd_details); + if (!status) { + resp = (struct i40e_aqc_write_ddp_resp *)&desc.params.raw; + if (error_offset) + *error_offset = LE32_TO_CPU(resp->error_offset); + if (error_info) + *error_info = LE32_TO_CPU(resp->error_info); + } + + return status; +} + +/** + * i40e_aq_get_ddp_list - Read dynamic device personalization (ddp) + * @hw: pointer to the hw struct + * @buff: command buffer (size in bytes = buff_size) + * @buff_size: buffer size in bytes + * @cmd_details: pointer to command details structure or NULL + **/ +enum +i40e_status_code i40e_aq_get_ddp_list(struct i40e_hw *hw, void *buff, + u16 buff_size, u8 flags, + struct i40e_asq_cmd_details *cmd_details) +{ + struct i40e_aq_desc desc; + struct i40e_aqc_get_applied_profiles *cmd = + (struct i40e_aqc_get_applied_profiles *)&desc.params.raw; + enum i40e_status_code status; + + i40e_fill_default_direct_cmd_desc(&desc, + i40e_aqc_opc_get_personalization_profile_list); + + desc.flags |= CPU_TO_LE16((u16)I40E_AQ_FLAG_BUF); + if (buff_size > I40E_AQ_LARGE_BUF) + desc.flags |= CPU_TO_LE16((u16)I40E_AQ_FLAG_LB); + desc.datalen = CPU_TO_LE16(buff_size); + + cmd->flags = flags; + + status = i40e_asq_send_command(hw, &desc, buff, buff_size, cmd_details); + + return status; +} + +/** + * i40e_find_segment_in_package + * @segment_type: the segment type to search for (i.e., SEGMENT_TYPE_I40E) + * @pkg_hdr: pointer to the package header to be searched + * + * This function searches a package file for a particular segment type. On + * success it returns a pointer to the segment header, otherwise it will + * return NULL. + **/ +struct i40e_generic_seg_header * +i40e_find_segment_in_package(u32 segment_type, + struct i40e_package_header *pkg_hdr) +{ + struct i40e_generic_seg_header *segment; + u32 i; + + /* Search all package segments for the requested segment type */ + for (i = 0; i < pkg_hdr->segment_count; i++) { + segment = + (struct i40e_generic_seg_header *)((u8 *)pkg_hdr + + pkg_hdr->segment_offset[i]); + + if (segment->type == segment_type) + return segment; + } + + return NULL; +} + +/* Get section table in profile */ +#define I40E_SECTION_TABLE(profile, sec_tbl) \ + do { \ + struct i40e_profile_segment *p = (profile); \ + u32 count; \ + u32 *nvm; \ + count = p->device_table_count; \ + nvm = (u32 *)&p->device_table[count]; \ + sec_tbl = (struct i40e_section_table *)&nvm[nvm[0] + 1]; \ + } while (0) + +/* Get section header in profile */ +#define I40E_SECTION_HEADER(profile, offset) \ + (struct i40e_profile_section_header *)((u8 *)(profile) + (offset)) + +/** + * i40e_find_section_in_profile + * @section_type: the section type to search for (i.e., SECTION_TYPE_NOTE) + * @profile: pointer to the i40e segment header to be searched + * + * This function searches i40e segment for a particular section type. On + * success it returns a pointer to the section header, otherwise it will + * return NULL. + **/ +struct i40e_profile_section_header * +i40e_find_section_in_profile(u32 section_type, + struct i40e_profile_segment *profile) +{ + struct i40e_profile_section_header *sec; + struct i40e_section_table *sec_tbl; + u32 sec_off; + u32 i; + + if (profile->header.type != SEGMENT_TYPE_I40E) + return NULL; + + I40E_SECTION_TABLE(profile, sec_tbl); + + for (i = 0; i < sec_tbl->section_count; i++) { + sec_off = sec_tbl->section_offset[i]; + sec = I40E_SECTION_HEADER(profile, sec_off); + if (sec->section.type == section_type) + return sec; + } + + return NULL; +} + +/** + * i40e_ddp_exec_aq_section - Execute generic AQ for DDP + * @hw: pointer to the hw struct + * @aq: command buffer containing all data to execute AQ + **/ +STATIC enum +i40e_status_code i40e_ddp_exec_aq_section(struct i40e_hw *hw, + struct i40e_profile_aq_section *aq) +{ + enum i40e_status_code status; + struct i40e_aq_desc desc; + u8 *msg = NULL; + u16 msglen; + + i40e_fill_default_direct_cmd_desc(&desc, aq->opcode); + desc.flags |= CPU_TO_LE16(aq->flags); + i40e_memcpy(desc.params.raw, aq->param, sizeof(desc.params.raw), + I40E_NONDMA_TO_NONDMA); + + msglen = aq->datalen; + if (msglen) { + desc.flags |= CPU_TO_LE16((u16)(I40E_AQ_FLAG_BUF | + I40E_AQ_FLAG_RD)); + if (msglen > I40E_AQ_LARGE_BUF) + desc.flags |= CPU_TO_LE16((u16)I40E_AQ_FLAG_LB); + desc.datalen = CPU_TO_LE16(msglen); + msg = &aq->data[0]; + } + + status = i40e_asq_send_command(hw, &desc, msg, msglen, NULL); + + if (status != I40E_SUCCESS) { + i40e_debug(hw, I40E_DEBUG_PACKAGE, + "unable to exec DDP AQ opcode %u, error %d\n", + aq->opcode, status); + return status; + } + + /* copy returned desc to aq_buf */ + i40e_memcpy(aq->param, desc.params.raw, sizeof(desc.params.raw), + I40E_NONDMA_TO_NONDMA); + + return I40E_SUCCESS; +} + +/** + * i40e_validate_profile + * @hw: pointer to the hardware structure + * @profile: pointer to the profile segment of the package to be validated + * @track_id: package tracking id + * @rollback: flag if the profile is for rollback. + * + * Validates supported devices and profile's sections. + */ +STATIC enum i40e_status_code +i40e_validate_profile(struct i40e_hw *hw, struct i40e_profile_segment *profile, + u32 track_id, bool rollback) +{ + struct i40e_profile_section_header *sec = NULL; + enum i40e_status_code status = I40E_SUCCESS; + struct i40e_section_table *sec_tbl; + u32 vendor_dev_id; + u32 dev_cnt; + u32 sec_off; + u32 i; + + if (track_id == I40E_DDP_TRACKID_INVALID) { + i40e_debug(hw, I40E_DEBUG_PACKAGE, "Invalid track_id\n"); + return I40E_NOT_SUPPORTED; + } + + dev_cnt = profile->device_table_count; + for (i = 0; i < dev_cnt; i++) { + vendor_dev_id = profile->device_table[i].vendor_dev_id; + if ((vendor_dev_id >> 16) == I40E_INTEL_VENDOR_ID && + hw->device_id == (vendor_dev_id & 0xFFFF)) + break; + } + if (dev_cnt && (i == dev_cnt)) { + i40e_debug(hw, I40E_DEBUG_PACKAGE, + "Device doesn't support DDP\n"); + return I40E_ERR_DEVICE_NOT_SUPPORTED; + } + + I40E_SECTION_TABLE(profile, sec_tbl); + + /* Validate sections types */ + for (i = 0; i < sec_tbl->section_count; i++) { + sec_off = sec_tbl->section_offset[i]; + sec = I40E_SECTION_HEADER(profile, sec_off); + if (rollback) { + if (sec->section.type == SECTION_TYPE_MMIO || + sec->section.type == SECTION_TYPE_AQ || + sec->section.type == SECTION_TYPE_RB_AQ) { + i40e_debug(hw, I40E_DEBUG_PACKAGE, + "Not a roll-back package\n"); + return I40E_NOT_SUPPORTED; + } + } else { + if (sec->section.type == SECTION_TYPE_RB_AQ || + sec->section.type == SECTION_TYPE_RB_MMIO) { + i40e_debug(hw, I40E_DEBUG_PACKAGE, + "Not an original package\n"); + return I40E_NOT_SUPPORTED; + } + } + } + + return status; +} + +/** + * i40e_write_profile + * @hw: pointer to the hardware structure + * @profile: pointer to the profile segment of the package to be downloaded + * @track_id: package tracking id + * + * Handles the download of a complete package. + */ +enum i40e_status_code +i40e_write_profile(struct i40e_hw *hw, struct i40e_profile_segment *profile, + u32 track_id) +{ + enum i40e_status_code status = I40E_SUCCESS; + struct i40e_section_table *sec_tbl; + struct i40e_profile_section_header *sec = NULL; + struct i40e_profile_aq_section *ddp_aq; + u32 section_size = 0; + u32 offset = 0, info = 0; + u32 sec_off; + u32 i; + + status = i40e_validate_profile(hw, profile, track_id, false); + if (status) + return status; + + I40E_SECTION_TABLE(profile, sec_tbl); + + for (i = 0; i < sec_tbl->section_count; i++) { + sec_off = sec_tbl->section_offset[i]; + sec = I40E_SECTION_HEADER(profile, sec_off); + /* Process generic admin command */ + if (sec->section.type == SECTION_TYPE_AQ) { + ddp_aq = (struct i40e_profile_aq_section *)&sec[1]; + status = i40e_ddp_exec_aq_section(hw, ddp_aq); + if (status) { + i40e_debug(hw, I40E_DEBUG_PACKAGE, + "Failed to execute aq: section %d, opcode %u\n", + i, ddp_aq->opcode); + break; + } + sec->section.type = SECTION_TYPE_RB_AQ; + } + + /* Skip any non-mmio sections */ + if (sec->section.type != SECTION_TYPE_MMIO) + continue; + + section_size = sec->section.size + + sizeof(struct i40e_profile_section_header); + + /* Write MMIO section */ + status = i40e_aq_write_ddp(hw, (void *)sec, (u16)section_size, + track_id, &offset, &info, NULL); + if (status) { + i40e_debug(hw, I40E_DEBUG_PACKAGE, + "Failed to write profile: section %d, offset %d, info %d\n", + i, offset, info); + break; + } + } + return status; +} + +/** + * i40e_rollback_profile + * @hw: pointer to the hardware structure + * @profile: pointer to the profile segment of the package to be removed + * @track_id: package tracking id + * + * Rolls back previously loaded package. + */ +enum i40e_status_code +i40e_rollback_profile(struct i40e_hw *hw, struct i40e_profile_segment *profile, + u32 track_id) +{ + struct i40e_profile_section_header *sec = NULL; + enum i40e_status_code status = I40E_SUCCESS; + struct i40e_section_table *sec_tbl; + u32 offset = 0, info = 0; + u32 section_size = 0; + u32 sec_off; + int i; + + status = i40e_validate_profile(hw, profile, track_id, true); + if (status) + return status; + + I40E_SECTION_TABLE(profile, sec_tbl); + + /* For rollback write sections in reverse */ + for (i = sec_tbl->section_count - 1; i >= 0; i--) { + sec_off = sec_tbl->section_offset[i]; + sec = I40E_SECTION_HEADER(profile, sec_off); + + /* Skip any non-rollback sections */ + if (sec->section.type != SECTION_TYPE_RB_MMIO) + continue; + + section_size = sec->section.size + + sizeof(struct i40e_profile_section_header); + + /* Write roll-back MMIO section */ + status = i40e_aq_write_ddp(hw, (void *)sec, (u16)section_size, + track_id, &offset, &info, NULL); + if (status) { + i40e_debug(hw, I40E_DEBUG_PACKAGE, + "Failed to write profile: section %d, offset %d, info %d\n", + i, offset, info); + break; + } + } + return status; +} + +/** + * i40e_add_pinfo_to_list + * @hw: pointer to the hardware structure + * @profile: pointer to the profile segment of the package + * @profile_info_sec: buffer for information section + * @track_id: package tracking id + * + * Register a profile to the list of loaded profiles. + */ +enum i40e_status_code +i40e_add_pinfo_to_list(struct i40e_hw *hw, + struct i40e_profile_segment *profile, + u8 *profile_info_sec, u32 track_id) +{ + enum i40e_status_code status = I40E_SUCCESS; + struct i40e_profile_section_header *sec = NULL; + struct i40e_profile_info *pinfo; + u32 offset = 0, info = 0; + + sec = (struct i40e_profile_section_header *)profile_info_sec; + sec->tbl_size = 1; + sec->data_end = sizeof(struct i40e_profile_section_header) + + sizeof(struct i40e_profile_info); + sec->section.type = SECTION_TYPE_INFO; + sec->section.offset = sizeof(struct i40e_profile_section_header); + sec->section.size = sizeof(struct i40e_profile_info); + pinfo = (struct i40e_profile_info *)(profile_info_sec + + sec->section.offset); + pinfo->track_id = track_id; + pinfo->version = profile->version; + pinfo->op = I40E_DDP_ADD_TRACKID; + i40e_memcpy(pinfo->name, profile->name, I40E_DDP_NAME_SIZE, + I40E_NONDMA_TO_NONDMA); + + status = i40e_aq_write_ddp(hw, (void *)sec, sec->data_end, + track_id, &offset, &info, NULL); + return status; +} diff --git a/dpdk/drivers/net/i40e/base/i40e_dcb.c b/dpdk/drivers/net/i40e/base/i40e_dcb.c index 26c344fd..9b5405db 100644 --- a/dpdk/drivers/net/i40e/base/i40e_dcb.c +++ b/dpdk/drivers/net/i40e/base/i40e_dcb.c @@ -396,6 +396,8 @@ static void i40e_parse_cee_app_tlv(struct i40e_cee_feat_tlv *tlv, dcbcfg->numapps = length / sizeof(*app); if (!dcbcfg->numapps) return; + if (dcbcfg->numapps > I40E_DCBX_MAX_APPS) + dcbcfg->numapps = I40E_DCBX_MAX_APPS; for (i = 0; i < dcbcfg->numapps; i++) { u8 up, selector; diff --git a/dpdk/drivers/net/i40e/base/i40e_devids.h b/dpdk/drivers/net/i40e/base/i40e_devids.h index ed73e1d2..f4a87842 100644 --- a/dpdk/drivers/net/i40e/base/i40e_devids.h +++ b/dpdk/drivers/net/i40e/base/i40e_devids.h @@ -54,8 +54,8 @@ POSSIBILITY OF SUCH DAMAGE. #if defined(INTEGRATED_VF) || defined(VF_DRIVER) || defined(I40E_NDIS_SUPPORT) #define I40E_DEV_ID_VF 0x154C #define I40E_DEV_ID_VF_HV 0x1571 +#define I40E_DEV_ID_ADAPTIVE_VF 0x1889 #endif /* VF_DRIVER */ -#ifdef X722_SUPPORT #ifdef X722_A0_SUPPORT #define I40E_DEV_ID_X722_A0 0x374C #if defined(INTEGRATED_VF) || defined(VF_DRIVER) @@ -68,12 +68,9 @@ POSSIBILITY OF SUCH DAMAGE. #define I40E_DEV_ID_1G_BASE_T_X722 0x37D1 #define I40E_DEV_ID_10G_BASE_T_X722 0x37D2 #define I40E_DEV_ID_SFP_I_X722 0x37D3 -#define I40E_DEV_ID_QSFP_I_X722 0x37D4 #if defined(INTEGRATED_VF) || defined(VF_DRIVER) || defined(I40E_NDIS_SUPPORT) #define I40E_DEV_ID_X722_VF 0x37CD -#define I40E_DEV_ID_X722_VF_HV 0x37D9 #endif /* VF_DRIVER */ -#endif /* X722_SUPPORT */ #define i40e_is_40G_device(d) ((d) == I40E_DEV_ID_QSFP_A || \ (d) == I40E_DEV_ID_QSFP_B || \ diff --git a/dpdk/drivers/net/i40e/base/i40e_lan_hmc.c b/dpdk/drivers/net/i40e/base/i40e_lan_hmc.c index 22606484..f03f3813 100644 --- a/dpdk/drivers/net/i40e/base/i40e_lan_hmc.c +++ b/dpdk/drivers/net/i40e/base/i40e_lan_hmc.c @@ -1239,11 +1239,6 @@ enum i40e_status_code i40e_hmc_get_object_va(struct i40e_hw *hw, u64 obj_offset_in_fpm; u32 sd_idx, sd_lmt; - if (NULL == hmc_info) { - ret_code = I40E_ERR_BAD_PTR; - DEBUGOUT("i40e_hmc_get_object_va: bad hmc_info ptr\n"); - goto exit; - } if (NULL == hmc_info->hmc_obj) { ret_code = I40E_ERR_BAD_PTR; DEBUGOUT("i40e_hmc_get_object_va: bad hmc_info->hmc_obj ptr\n"); diff --git a/dpdk/drivers/net/i40e/base/i40e_nvm.c b/dpdk/drivers/net/i40e/base/i40e_nvm.c index 4fa1220b..a1e78300 100644 --- a/dpdk/drivers/net/i40e/base/i40e_nvm.c +++ b/dpdk/drivers/net/i40e/base/i40e_nvm.c @@ -219,19 +219,15 @@ enum i40e_status_code i40e_read_nvm_word(struct i40e_hw *hw, u16 offset, { enum i40e_status_code ret_code = I40E_SUCCESS; -#ifdef X722_SUPPORT - if (hw->flags & I40E_HW_FLAG_AQ_SRCTL_ACCESS_ENABLE) { - ret_code = i40e_acquire_nvm(hw, I40E_RESOURCE_READ); - if (!ret_code) { + ret_code = i40e_acquire_nvm(hw, I40E_RESOURCE_READ); + if (!ret_code) { + if (hw->flags & I40E_HW_FLAG_AQ_SRCTL_ACCESS_ENABLE) { ret_code = i40e_read_nvm_word_aq(hw, offset, data); - i40e_release_nvm(hw); + } else { + ret_code = i40e_read_nvm_word_srctl(hw, offset, data); } - } else { - ret_code = i40e_read_nvm_word_srctl(hw, offset, data); + i40e_release_nvm(hw); } -#else - ret_code = i40e_read_nvm_word_srctl(hw, offset, data); -#endif return ret_code; } @@ -249,14 +245,10 @@ enum i40e_status_code __i40e_read_nvm_word(struct i40e_hw *hw, { enum i40e_status_code ret_code = I40E_SUCCESS; -#ifdef X722_SUPPORT if (hw->flags & I40E_HW_FLAG_AQ_SRCTL_ACCESS_ENABLE) ret_code = i40e_read_nvm_word_aq(hw, offset, data); else ret_code = i40e_read_nvm_word_srctl(hw, offset, data); -#else - ret_code = i40e_read_nvm_word_srctl(hw, offset, data); -#endif return ret_code; } @@ -348,14 +340,10 @@ enum i40e_status_code __i40e_read_nvm_buffer(struct i40e_hw *hw, { enum i40e_status_code ret_code = I40E_SUCCESS; -#ifdef X722_SUPPORT if (hw->flags & I40E_HW_FLAG_AQ_SRCTL_ACCESS_ENABLE) ret_code = i40e_read_nvm_buffer_aq(hw, offset, words, data); else ret_code = i40e_read_nvm_buffer_srctl(hw, offset, words, data); -#else - ret_code = i40e_read_nvm_buffer_srctl(hw, offset, words, data); -#endif return ret_code; } @@ -375,7 +363,6 @@ enum i40e_status_code i40e_read_nvm_buffer(struct i40e_hw *hw, u16 offset, { enum i40e_status_code ret_code = I40E_SUCCESS; -#ifdef X722_SUPPORT if (hw->flags & I40E_HW_FLAG_AQ_SRCTL_ACCESS_ENABLE) { ret_code = i40e_acquire_nvm(hw, I40E_RESOURCE_READ); if (!ret_code) { @@ -386,9 +373,6 @@ enum i40e_status_code i40e_read_nvm_buffer(struct i40e_hw *hw, u16 offset, } else { ret_code = i40e_read_nvm_buffer_srctl(hw, offset, words, data); } -#else - ret_code = i40e_read_nvm_buffer_srctl(hw, offset, words, data); -#endif return ret_code; } @@ -765,12 +749,18 @@ enum i40e_status_code i40e_validate_nvm_checksum(struct i40e_hw *hw, DEBUGFUNC("i40e_validate_nvm_checksum"); - if (hw->flags & I40E_HW_FLAG_AQ_SRCTL_ACCESS_ENABLE) - ret_code = i40e_acquire_nvm(hw, I40E_RESOURCE_READ); + /* acquire_nvm provides exclusive NVM lock to synchronize access across + * PFs. X710 uses i40e_read_nvm_word_srctl which polls for done bit + * twice (first time to be able to write address to I40E_GLNVM_SRCTL + * register, second to read data from I40E_GLNVM_SRDATA. One PF can see + * done bit and try to write address, while another one will interpret + * it as a good time to read data. It will cause invalid data to be + * read. + */ + ret_code = i40e_acquire_nvm(hw, I40E_RESOURCE_READ); if (!ret_code) { ret_code = i40e_calc_nvm_checksum(hw, &checksum_local); - if (hw->flags & I40E_HW_FLAG_AQ_SRCTL_ACCESS_ENABLE) - i40e_release_nvm(hw); + i40e_release_nvm(hw); if (ret_code != I40E_SUCCESS) goto i40e_validate_nvm_checksum_exit; } else { @@ -901,9 +891,25 @@ enum i40e_status_code i40e_nvmupd_command(struct i40e_hw *hw, *((u16 *)&bytes[2]) = hw->nvm_wait_opcode; } + /* Clear error status on read */ + if (hw->nvmupd_state == I40E_NVMUPD_STATE_ERROR) + hw->nvmupd_state = I40E_NVMUPD_STATE_INIT; + return I40E_SUCCESS; } + /* Clear status even it is not read and log */ + if (hw->nvmupd_state == I40E_NVMUPD_STATE_ERROR) { + i40e_debug(hw, I40E_DEBUG_NVM, + "Clearing I40E_NVMUPD_STATE_ERROR state without reading\n"); + hw->nvmupd_state = I40E_NVMUPD_STATE_INIT; + } + + /* Acquire lock to prevent race condition where adminq_task + * can execute after i40e_nvmupd_nvm_read/write but before state + * variables (nvm_wait_opcode, nvm_release_on_done) are updated + */ + i40e_acquire_spinlock(&hw->aq.arq_spinlock); switch (hw->nvmupd_state) { case I40E_NVMUPD_STATE_INIT: status = i40e_nvmupd_state_init(hw, cmd, bytes, perrno); @@ -939,6 +945,7 @@ enum i40e_status_code i40e_nvmupd_command(struct i40e_hw *hw, *perrno = -ESRCH; break; } + i40e_release_spinlock(&hw->aq.arq_spinlock); return status; } @@ -1253,6 +1260,7 @@ retry: void i40e_nvmupd_check_wait_event(struct i40e_hw *hw, u16 opcode) { if (opcode == hw->nvm_wait_opcode) { + i40e_debug(hw, I40E_DEBUG_NVM, "NVMUPD: clearing wait on opcode 0x%04x\n", opcode); if (hw->nvm_release_on_done) { @@ -1261,6 +1269,11 @@ void i40e_nvmupd_check_wait_event(struct i40e_hw *hw, u16 opcode) } hw->nvm_wait_opcode = 0; + if (hw->aq.arq_last_status) { + hw->nvmupd_state = I40E_NVMUPD_STATE_ERROR; + return; + } + switch (hw->nvmupd_state) { case I40E_NVMUPD_STATE_INIT_WAIT: hw->nvmupd_state = I40E_NVMUPD_STATE_INIT; @@ -1423,7 +1436,8 @@ STATIC enum i40e_status_code i40e_nvmupd_exec_aq(struct i40e_hw *hw, if (hw->nvm_buff.va) { buff = hw->nvm_buff.va; - memcpy(buff, &bytes[aq_desc_len], aq_data_len); + i40e_memcpy(buff, &bytes[aq_desc_len], aq_data_len, + I40E_NONDMA_TO_NONDMA); } } @@ -1496,7 +1510,7 @@ STATIC enum i40e_status_code i40e_nvmupd_get_aq_result(struct i40e_hw *hw, __func__, cmd->offset, cmd->offset + len); buff = ((u8 *)&hw->nvm_wb_desc) + cmd->offset; - memcpy(bytes, buff, len); + i40e_memcpy(bytes, buff, len, I40E_NONDMA_TO_NONDMA); bytes += len; remainder -= len; @@ -1510,7 +1524,7 @@ STATIC enum i40e_status_code i40e_nvmupd_get_aq_result(struct i40e_hw *hw, i40e_debug(hw, I40E_DEBUG_NVM, "%s: databuf bytes %d to %d\n", __func__, start_byte, start_byte + remainder); - memcpy(bytes, buff, remainder); + i40e_memcpy(bytes, buff, remainder, I40E_NONDMA_TO_NONDMA); } return I40E_SUCCESS; diff --git a/dpdk/drivers/net/i40e/base/i40e_osdep.h b/dpdk/drivers/net/i40e/base/i40e_osdep.h index 38e7ba5b..8e5c593c 100644 --- a/dpdk/drivers/net/i40e/base/i40e_osdep.h +++ b/dpdk/drivers/net/i40e/base/i40e_osdep.h @@ -35,6 +35,7 @@ #include #include +#include #include #include @@ -44,6 +45,7 @@ #include #include #include +#include #include "../i40e_logs.h" @@ -56,7 +58,6 @@ typedef uint16_t u16; typedef uint32_t u32; typedef int32_t s32; typedef uint64_t u64; -typedef int bool; typedef enum i40e_status_code i40e_status; #define __iomem @@ -98,7 +99,6 @@ typedef enum i40e_status_code i40e_status; #define max(a,b) RTE_MAX(a,b) #define FIELD_SIZEOF(t, f) (sizeof(((t*)0)->f)) -#define ASSERT(x) if(!(x)) rte_panic("IXGBE: x") #define DEBUGOUT(S) PMD_DRV_LOG_RAW(DEBUG, S) #define DEBUGOUT1(S, A...) PMD_DRV_LOG_RAW(DEBUG, S, ##A) @@ -153,15 +153,18 @@ do { \ * I40E_PRTQF_FD_MSK */ -#define I40E_PCI_REG(reg) (*((volatile uint32_t *)(reg))) +#define I40E_PCI_REG(reg) rte_read32(reg) #define I40E_PCI_REG_ADDR(a, reg) \ ((volatile uint32_t *)((char *)(a)->hw_addr + (reg))) static inline uint32_t i40e_read_addr(volatile void *addr) { return rte_le_to_cpu_32(I40E_PCI_REG(addr)); } -#define I40E_PCI_REG_WRITE(reg, value) \ - do { I40E_PCI_REG((reg)) = rte_cpu_to_le_32(value); } while (0) + +#define I40E_PCI_REG_WRITE(reg, value) \ + rte_write32((rte_cpu_to_le_32(value)), reg) +#define I40E_PCI_REG_WRITE_RELAXED(reg, value) \ + rte_write32_relaxed((rte_cpu_to_le_32(value)), reg) #define I40E_WRITE_FLUSH(a) I40E_READ_REG(a, I40E_GLGEN_STAT) #define I40EVF_WRITE_FLUSH(a) I40E_READ_REG(a, I40E_VFGEN_RSTAT) diff --git a/dpdk/drivers/net/i40e/base/i40e_prototype.h b/dpdk/drivers/net/i40e/base/i40e_prototype.h index 03dda937..acb2023f 100644 --- a/dpdk/drivers/net/i40e/base/i40e_prototype.h +++ b/dpdk/drivers/net/i40e/base/i40e_prototype.h @@ -36,7 +36,7 @@ POSSIBILITY OF SUCH DAMAGE. #include "i40e_type.h" #include "i40e_alloc.h" -#include "i40e_virtchnl.h" +#include "virtchnl.h" /* Prototypes for shared code functions that are not in * the standard function pointer structures. These are @@ -78,7 +78,6 @@ void i40e_debug_aq(struct i40e_hw *hw, enum i40e_debug_mask mask, void i40e_idle_aq(struct i40e_hw *hw); bool i40e_check_asq_alive(struct i40e_hw *hw); enum i40e_status_code i40e_aq_queue_shutdown(struct i40e_hw *hw, bool unloading); -#ifdef X722_SUPPORT enum i40e_status_code i40e_aq_get_rss_lut(struct i40e_hw *hw, u16 seid, bool pf_lut, u8 *lut, u16 lut_size); @@ -90,11 +89,8 @@ enum i40e_status_code i40e_aq_get_rss_key(struct i40e_hw *hw, enum i40e_status_code i40e_aq_set_rss_key(struct i40e_hw *hw, u16 seid, struct i40e_aqc_get_set_rss_key_data *key); -#endif -#ifndef I40E_NDIS_SUPPORT const char *i40e_aq_str(struct i40e_hw *hw, enum i40e_admin_queue_err aq_err); const char *i40e_stat_str(struct i40e_hw *hw, enum i40e_status_code stat_err); -#endif /* I40E_NDIS_SUPPORT */ #ifdef PF_DRIVER @@ -124,6 +120,8 @@ enum i40e_status_code i40e_aq_set_phy_debug(struct i40e_hw *hw, u8 cmd_flags, struct i40e_asq_cmd_details *cmd_details); enum i40e_status_code i40e_aq_set_default_vsi(struct i40e_hw *hw, u16 vsi_id, struct i40e_asq_cmd_details *cmd_details); +enum i40e_status_code i40e_aq_clear_default_vsi(struct i40e_hw *hw, u16 vsi_id, + struct i40e_asq_cmd_details *cmd_details); enum i40e_status_code i40e_aq_get_phy_capabilities(struct i40e_hw *hw, bool qualified_modules, bool report_init, struct i40e_aq_get_phy_abilities_resp *abilities, @@ -170,12 +168,18 @@ enum i40e_status_code i40e_aq_set_vsi_unicast_promiscuous(struct i40e_hw *hw, bool rx_only_promisc); enum i40e_status_code i40e_aq_set_vsi_multicast_promiscuous(struct i40e_hw *hw, u16 vsi_id, bool set, struct i40e_asq_cmd_details *cmd_details); +enum i40e_status_code i40e_aq_set_vsi_full_promiscuous(struct i40e_hw *hw, + u16 seid, bool set, + struct i40e_asq_cmd_details *cmd_details); enum i40e_status_code i40e_aq_set_vsi_mc_promisc_on_vlan(struct i40e_hw *hw, u16 seid, bool enable, u16 vid, struct i40e_asq_cmd_details *cmd_details); enum i40e_status_code i40e_aq_set_vsi_uc_promisc_on_vlan(struct i40e_hw *hw, u16 seid, bool enable, u16 vid, struct i40e_asq_cmd_details *cmd_details); +enum i40e_status_code i40e_aq_set_vsi_bc_promisc_on_vlan(struct i40e_hw *hw, + u16 seid, bool enable, u16 vid, + struct i40e_asq_cmd_details *cmd_details); enum i40e_status_code i40e_aq_set_vsi_vlan_promisc(struct i40e_hw *hw, u16 seid, bool enable, struct i40e_asq_cmd_details *cmd_details); @@ -400,11 +404,21 @@ enum i40e_status_code i40e_aq_add_cloud_filters(struct i40e_hw *hw, u16 vsi, struct i40e_aqc_add_remove_cloud_filters_element_data *filters, u8 filter_count); - +enum i40e_status_code i40e_aq_add_cloud_filters_big_buffer(struct i40e_hw *hw, + u16 seid, + struct i40e_aqc_add_rm_cloud_filt_elem_ext *filters, + u8 filter_count); enum i40e_status_code i40e_aq_remove_cloud_filters(struct i40e_hw *hw, u16 vsi, struct i40e_aqc_add_remove_cloud_filters_element_data *filters, u8 filter_count); +enum i40e_status_code i40e_aq_remove_cloud_filters_big_buffer( + struct i40e_hw *hw, u16 seid, + struct i40e_aqc_add_rm_cloud_filt_elem_ext *filters, + u8 filter_count); +enum i40e_status_code i40e_aq_replace_cloud_filters(struct i40e_hw *hw, + struct i40e_aqc_replace_cloud_filters_cmd *filters, + struct i40e_aqc_replace_cloud_filters_cmd_buf *cmd_buf); enum i40e_status_code i40e_aq_alternate_read(struct i40e_hw *hw, u32 reg_addr0, u32 *reg_val0, u32 reg_addr1, u32 *reg_val1); @@ -438,6 +452,7 @@ enum i40e_status_code i40e_get_port_mac_addr(struct i40e_hw *hw, u8 *mac_addr); enum i40e_status_code i40e_read_pba_string(struct i40e_hw *hw, u8 *pba_num, u32 pba_num_size); void i40e_pre_tx_queue_cfg(struct i40e_hw *hw, u32 queue, bool enable); +enum i40e_status_code i40e_get_san_mac_addr(struct i40e_hw *hw, u8 *mac_addr); enum i40e_aq_link_speed i40e_get_link_speed(struct i40e_hw *hw); /* prototype for functions used for NVM access */ enum i40e_status_code i40e_init_nvm(struct i40e_hw *hw); @@ -489,10 +504,10 @@ void i40e_destroy_spinlock(struct i40e_spinlock *sp); /* i40e_common for VF drivers*/ void i40e_vf_parse_hw_config(struct i40e_hw *hw, - struct i40e_virtchnl_vf_resource *msg); + struct virtchnl_vf_resource *msg); enum i40e_status_code i40e_vf_reset(struct i40e_hw *hw); enum i40e_status_code i40e_aq_send_msg_to_pf(struct i40e_hw *hw, - enum i40e_virtchnl_ops v_opcode, + enum virtchnl_ops v_opcode, enum i40e_status_code v_retval, u8 *msg, u16 msglen, struct i40e_asq_cmd_details *cmd_details); @@ -518,7 +533,15 @@ enum i40e_status_code i40e_aq_rx_ctl_write_register(struct i40e_hw *hw, u32 reg_addr, u32 reg_val, struct i40e_asq_cmd_details *cmd_details); void i40e_write_rx_ctl(struct i40e_hw *hw, u32 reg_addr, u32 reg_val); -#ifdef X722_SUPPORT +enum i40e_status_code i40e_aq_set_phy_register(struct i40e_hw *hw, + u8 phy_select, u8 dev_addr, + u32 reg_addr, u32 reg_val, + struct i40e_asq_cmd_details *cmd_details); +enum i40e_status_code i40e_aq_get_phy_register(struct i40e_hw *hw, + u8 phy_select, u8 dev_addr, + u32 reg_addr, u32 *reg_val, + struct i40e_asq_cmd_details *cmd_details); + enum i40e_status_code i40e_aq_set_arp_proxy_config(struct i40e_hw *hw, struct i40e_aqc_arp_proxy_data *proxy_config, struct i40e_asq_cmd_details *cmd_details); @@ -534,12 +557,46 @@ enum i40e_status_code i40e_aq_set_clear_wol_filter(struct i40e_hw *hw, enum i40e_status_code i40e_aq_get_wake_event_reason(struct i40e_hw *hw, u16 *wake_reason, struct i40e_asq_cmd_details *cmd_details); -#endif -enum i40e_status_code i40e_read_phy_register(struct i40e_hw *hw, u8 page, - u16 reg, u8 phy_addr, u16 *value); -enum i40e_status_code i40e_write_phy_register(struct i40e_hw *hw, u8 page, - u16 reg, u8 phy_addr, u16 value); +enum i40e_status_code i40e_aq_clear_all_wol_filters(struct i40e_hw *hw, + struct i40e_asq_cmd_details *cmd_details); +enum i40e_status_code i40e_read_phy_register_clause22(struct i40e_hw *hw, + u16 reg, u8 phy_addr, u16 *value); +enum i40e_status_code i40e_write_phy_register_clause22(struct i40e_hw *hw, + u16 reg, u8 phy_addr, u16 value); +enum i40e_status_code i40e_read_phy_register_clause45(struct i40e_hw *hw, + u8 page, u16 reg, u8 phy_addr, u16 *value); +enum i40e_status_code i40e_write_phy_register_clause45(struct i40e_hw *hw, + u8 page, u16 reg, u8 phy_addr, u16 value); +enum i40e_status_code i40e_read_phy_register(struct i40e_hw *hw, + u8 page, u16 reg, u8 phy_addr, u16 *value); +enum i40e_status_code i40e_write_phy_register(struct i40e_hw *hw, + u8 page, u16 reg, u8 phy_addr, u16 value); u8 i40e_get_phy_address(struct i40e_hw *hw, u8 dev_num); enum i40e_status_code i40e_blink_phy_link_led(struct i40e_hw *hw, u32 time, u32 interval); +enum i40e_status_code i40e_aq_write_ddp(struct i40e_hw *hw, void *buff, + u16 buff_size, u32 track_id, + u32 *error_offset, u32 *error_info, + struct i40e_asq_cmd_details * + cmd_details); +enum i40e_status_code i40e_aq_get_ddp_list(struct i40e_hw *hw, void *buff, + u16 buff_size, u8 flags, + struct i40e_asq_cmd_details * + cmd_details); +struct i40e_generic_seg_header * +i40e_find_segment_in_package(u32 segment_type, + struct i40e_package_header *pkg_header); +struct i40e_profile_section_header * +i40e_find_section_in_profile(u32 section_type, + struct i40e_profile_segment *profile); +enum i40e_status_code +i40e_write_profile(struct i40e_hw *hw, struct i40e_profile_segment *i40e_seg, + u32 track_id); +enum i40e_status_code +i40e_rollback_profile(struct i40e_hw *hw, struct i40e_profile_segment *i40e_seg, + u32 track_id); +enum i40e_status_code +i40e_add_pinfo_to_list(struct i40e_hw *hw, + struct i40e_profile_segment *profile, + u8 *profile_info_sec, u32 track_id); #endif /* _I40E_PROTOTYPE_H_ */ diff --git a/dpdk/drivers/net/i40e/base/i40e_register.h b/dpdk/drivers/net/i40e/base/i40e_register.h index fd0a7230..a482ab90 100644 --- a/dpdk/drivers/net/i40e/base/i40e_register.h +++ b/dpdk/drivers/net/i40e/base/i40e_register.h @@ -2805,7 +2805,7 @@ POSSIBILITY OF SUCH DAMAGE. #define I40E_GLV_RUPP_MAX_INDEX 383 #define I40E_GLV_RUPP_RUPP_SHIFT 0 #define I40E_GLV_RUPP_RUPP_MASK I40E_MASK(0xFFFFFFFF, I40E_GLV_RUPP_RUPP_SHIFT) -#define I40E_GLV_TEPC(_VSI) (0x00344000 + ((_VSI) * 4)) /* _i=0...383 */ /* Reset: CORER */ +#define I40E_GLV_TEPC(_i) (0x00344000 + ((_i) * 8)) /* _i=0...383 */ /* Reset: CORER */ #define I40E_GLV_TEPC_MAX_INDEX 383 #define I40E_GLV_TEPC_TEPC_SHIFT 0 #define I40E_GLV_TEPC_TEPC_MASK I40E_MASK(0xFFFFFFFF, I40E_GLV_TEPC_TEPC_SHIFT) @@ -3401,7 +3401,6 @@ POSSIBILITY OF SUCH DAMAGE. #define I40E_VFQF_HREGION_OVERRIDE_ENA_7_MASK I40E_MASK(0x1, I40E_VFQF_HREGION_OVERRIDE_ENA_7_SHIFT) #define I40E_VFQF_HREGION_REGION_7_SHIFT 29 #define I40E_VFQF_HREGION_REGION_7_MASK I40E_MASK(0x7, I40E_VFQF_HREGION_REGION_7_SHIFT) -#ifdef X722_SUPPORT #ifdef PF_DRIVER #define I40E_MNGSB_FDCRC 0x000B7050 /* Reset: POR */ @@ -5366,5 +5365,4 @@ POSSIBILITY OF SUCH DAMAGE. #define I40E_VFPE_WQEALLOC1_WQE_DESC_INDEX_SHIFT 20 #define I40E_VFPE_WQEALLOC1_WQE_DESC_INDEX_MASK I40E_MASK(0xFFF, I40E_VFPE_WQEALLOC1_WQE_DESC_INDEX_SHIFT) -#endif /* X722_SUPPORT */ #endif /* _I40E_REGISTER_H_ */ diff --git a/dpdk/drivers/net/i40e/base/i40e_type.h b/dpdk/drivers/net/i40e/base/i40e_type.h index 5349419f..dca725af 100644 --- a/dpdk/drivers/net/i40e/base/i40e_type.h +++ b/dpdk/drivers/net/i40e/base/i40e_type.h @@ -92,7 +92,9 @@ POSSIBILITY OF SUCH DAMAGE. struct i40e_hw; typedef void (*I40E_ADMINQ_CALLBACK)(struct i40e_hw *, struct i40e_aq_desc *); -#define I40E_ETH_LENGTH_OF_ADDRESS 6 +#ifndef ETH_ALEN +#define ETH_ALEN 6 +#endif /* Data type manipulation macros. */ #define I40E_HI_DWORD(x) ((u32)((((x) >> 16) >> 16) & 0xFFFFFFFF)) #define I40E_LO_DWORD(x) ((u32)((x) & 0xFFFFFFFF)) @@ -133,6 +135,7 @@ enum i40e_debug_mask { I40E_DEBUG_DCB = 0x00000400, I40E_DEBUG_DIAG = 0x00000800, I40E_DEBUG_FD = 0x00001000, + I40E_DEBUG_PACKAGE = 0x00002000, I40E_DEBUG_AQ_MESSAGE = 0x01000000, I40E_DEBUG_AQ_DESCRIPTOR = 0x02000000, @@ -157,13 +160,22 @@ enum i40e_debug_mask { #define I40E_PCI_LINK_SPEED_5000 0x2 #define I40E_PCI_LINK_SPEED_8000 0x3 -#define I40E_MDIO_STCODE 0 -#define I40E_MDIO_OPCODE_ADDRESS 0 -#define I40E_MDIO_OPCODE_WRITE I40E_MASK(1, \ +#define I40E_MDIO_CLAUSE22_STCODE_MASK I40E_MASK(1, \ + I40E_GLGEN_MSCA_STCODE_SHIFT) +#define I40E_MDIO_CLAUSE22_OPCODE_WRITE_MASK I40E_MASK(1, \ I40E_GLGEN_MSCA_OPCODE_SHIFT) -#define I40E_MDIO_OPCODE_READ_INC_ADDR I40E_MASK(2, \ +#define I40E_MDIO_CLAUSE22_OPCODE_READ_MASK I40E_MASK(2, \ I40E_GLGEN_MSCA_OPCODE_SHIFT) -#define I40E_MDIO_OPCODE_READ I40E_MASK(3, \ + +#define I40E_MDIO_CLAUSE45_STCODE_MASK I40E_MASK(0, \ + I40E_GLGEN_MSCA_STCODE_SHIFT) +#define I40E_MDIO_CLAUSE45_OPCODE_ADDRESS_MASK I40E_MASK(0, \ + I40E_GLGEN_MSCA_OPCODE_SHIFT) +#define I40E_MDIO_CLAUSE45_OPCODE_WRITE_MASK I40E_MASK(1, \ + I40E_GLGEN_MSCA_OPCODE_SHIFT) +#define I40E_MDIO_CLAUSE45_OPCODE_READ_INC_ADDR_MASK I40E_MASK(2, \ + I40E_GLGEN_MSCA_OPCODE_SHIFT) +#define I40E_MDIO_CLAUSE45_OPCODE_READ_MASK I40E_MASK(3, \ I40E_GLGEN_MSCA_OPCODE_SHIFT) #define I40E_PHY_COM_REG_PAGE 0x1E @@ -187,12 +199,6 @@ enum i40e_memcpy_type { I40E_DMA_TO_NONDMA }; -#ifdef X722_SUPPORT -#define I40E_FW_API_VERSION_MINOR_X722 0x0005 -#endif -#define I40E_FW_API_VERSION_MINOR_X710 0x0005 - - /* These are structs for managing the hardware information and the operations. * The structures of function pointers are filled out at init time when we * know for sure exactly which hardware we're working with. This gives us the @@ -203,13 +209,10 @@ enum i40e_memcpy_type { */ enum i40e_mac_type { I40E_MAC_UNKNOWN = 0, - I40E_MAC_X710, I40E_MAC_XL710, I40E_MAC_VF, -#ifdef X722_SUPPORT I40E_MAC_X722, I40E_MAC_X722_VF, -#endif I40E_MAC_GENERIC, }; @@ -264,6 +267,8 @@ struct i40e_link_status { enum i40e_aq_link_speed link_speed; u8 link_info; u8 an_info; + u8 req_fec_info; + u8 fec_info; u8 ext_info; u8 loopback; /* is Link Status Event notification to SW enabled */ @@ -292,61 +297,73 @@ struct i40e_link_status { #define I40E_MODULE_TYPE_1000BASE_T 0x08 }; -enum i40e_aq_capabilities_phy_type { - I40E_CAP_PHY_TYPE_SGMII = BIT(I40E_PHY_TYPE_SGMII), - I40E_CAP_PHY_TYPE_1000BASE_KX = BIT(I40E_PHY_TYPE_1000BASE_KX), - I40E_CAP_PHY_TYPE_10GBASE_KX4 = BIT(I40E_PHY_TYPE_10GBASE_KX4), - I40E_CAP_PHY_TYPE_10GBASE_KR = BIT(I40E_PHY_TYPE_10GBASE_KR), - I40E_CAP_PHY_TYPE_40GBASE_KR4 = BIT(I40E_PHY_TYPE_40GBASE_KR4), - I40E_CAP_PHY_TYPE_XAUI = BIT(I40E_PHY_TYPE_XAUI), - I40E_CAP_PHY_TYPE_XFI = BIT(I40E_PHY_TYPE_XFI), - I40E_CAP_PHY_TYPE_SFI = BIT(I40E_PHY_TYPE_SFI), - I40E_CAP_PHY_TYPE_XLAUI = BIT(I40E_PHY_TYPE_XLAUI), - I40E_CAP_PHY_TYPE_XLPPI = BIT(I40E_PHY_TYPE_XLPPI), - I40E_CAP_PHY_TYPE_40GBASE_CR4_CU = BIT(I40E_PHY_TYPE_40GBASE_CR4_CU), - I40E_CAP_PHY_TYPE_10GBASE_CR1_CU = BIT(I40E_PHY_TYPE_10GBASE_CR1_CU), - I40E_CAP_PHY_TYPE_10GBASE_AOC = BIT(I40E_PHY_TYPE_10GBASE_AOC), - I40E_CAP_PHY_TYPE_40GBASE_AOC = BIT(I40E_PHY_TYPE_40GBASE_AOC), - I40E_CAP_PHY_TYPE_100BASE_TX = BIT(I40E_PHY_TYPE_100BASE_TX), - I40E_CAP_PHY_TYPE_1000BASE_T = BIT(I40E_PHY_TYPE_1000BASE_T), - I40E_CAP_PHY_TYPE_10GBASE_T = BIT(I40E_PHY_TYPE_10GBASE_T), - I40E_CAP_PHY_TYPE_10GBASE_SR = BIT(I40E_PHY_TYPE_10GBASE_SR), - I40E_CAP_PHY_TYPE_10GBASE_LR = BIT(I40E_PHY_TYPE_10GBASE_LR), - I40E_CAP_PHY_TYPE_10GBASE_SFPP_CU = BIT(I40E_PHY_TYPE_10GBASE_SFPP_CU), - I40E_CAP_PHY_TYPE_10GBASE_CR1 = BIT(I40E_PHY_TYPE_10GBASE_CR1), - I40E_CAP_PHY_TYPE_40GBASE_CR4 = BIT(I40E_PHY_TYPE_40GBASE_CR4), - I40E_CAP_PHY_TYPE_40GBASE_SR4 = BIT(I40E_PHY_TYPE_40GBASE_SR4), - I40E_CAP_PHY_TYPE_40GBASE_LR4 = BIT(I40E_PHY_TYPE_40GBASE_LR4), - I40E_CAP_PHY_TYPE_1000BASE_SX = BIT(I40E_PHY_TYPE_1000BASE_SX), - I40E_CAP_PHY_TYPE_1000BASE_LX = BIT(I40E_PHY_TYPE_1000BASE_LX), - I40E_CAP_PHY_TYPE_1000BASE_T_OPTICAL = BIT(I40E_PHY_TYPE_1000BASE_T_OPTICAL), - I40E_CAP_PHY_TYPE_20GBASE_KR2 = BIT(I40E_PHY_TYPE_20GBASE_KR2) -}; - struct i40e_phy_info { struct i40e_link_status link_info; struct i40e_link_status link_info_old; bool get_link_info; enum i40e_media_type media_type; /* all the phy types the NVM is capable of */ - u32 phy_types; + u64 phy_types; }; +#define I40E_CAP_PHY_TYPE_SGMII BIT_ULL(I40E_PHY_TYPE_SGMII) +#define I40E_CAP_PHY_TYPE_1000BASE_KX BIT_ULL(I40E_PHY_TYPE_1000BASE_KX) +#define I40E_CAP_PHY_TYPE_10GBASE_KX4 BIT_ULL(I40E_PHY_TYPE_10GBASE_KX4) +#define I40E_CAP_PHY_TYPE_10GBASE_KR BIT_ULL(I40E_PHY_TYPE_10GBASE_KR) +#define I40E_CAP_PHY_TYPE_40GBASE_KR4 BIT_ULL(I40E_PHY_TYPE_40GBASE_KR4) +#define I40E_CAP_PHY_TYPE_XAUI BIT_ULL(I40E_PHY_TYPE_XAUI) +#define I40E_CAP_PHY_TYPE_XFI BIT_ULL(I40E_PHY_TYPE_XFI) +#define I40E_CAP_PHY_TYPE_SFI BIT_ULL(I40E_PHY_TYPE_SFI) +#define I40E_CAP_PHY_TYPE_XLAUI BIT_ULL(I40E_PHY_TYPE_XLAUI) +#define I40E_CAP_PHY_TYPE_XLPPI BIT_ULL(I40E_PHY_TYPE_XLPPI) +#define I40E_CAP_PHY_TYPE_40GBASE_CR4_CU BIT_ULL(I40E_PHY_TYPE_40GBASE_CR4_CU) +#define I40E_CAP_PHY_TYPE_10GBASE_CR1_CU BIT_ULL(I40E_PHY_TYPE_10GBASE_CR1_CU) +#define I40E_CAP_PHY_TYPE_10GBASE_AOC BIT_ULL(I40E_PHY_TYPE_10GBASE_AOC) +#define I40E_CAP_PHY_TYPE_40GBASE_AOC BIT_ULL(I40E_PHY_TYPE_40GBASE_AOC) +#define I40E_CAP_PHY_TYPE_100BASE_TX BIT_ULL(I40E_PHY_TYPE_100BASE_TX) +#define I40E_CAP_PHY_TYPE_1000BASE_T BIT_ULL(I40E_PHY_TYPE_1000BASE_T) +#define I40E_CAP_PHY_TYPE_10GBASE_T BIT_ULL(I40E_PHY_TYPE_10GBASE_T) +#define I40E_CAP_PHY_TYPE_10GBASE_SR BIT_ULL(I40E_PHY_TYPE_10GBASE_SR) +#define I40E_CAP_PHY_TYPE_10GBASE_LR BIT_ULL(I40E_PHY_TYPE_10GBASE_LR) +#define I40E_CAP_PHY_TYPE_10GBASE_SFPP_CU BIT_ULL(I40E_PHY_TYPE_10GBASE_SFPP_CU) +#define I40E_CAP_PHY_TYPE_10GBASE_CR1 BIT_ULL(I40E_PHY_TYPE_10GBASE_CR1) +#define I40E_CAP_PHY_TYPE_40GBASE_CR4 BIT_ULL(I40E_PHY_TYPE_40GBASE_CR4) +#define I40E_CAP_PHY_TYPE_40GBASE_SR4 BIT_ULL(I40E_PHY_TYPE_40GBASE_SR4) +#define I40E_CAP_PHY_TYPE_40GBASE_LR4 BIT_ULL(I40E_PHY_TYPE_40GBASE_LR4) +#define I40E_CAP_PHY_TYPE_1000BASE_SX BIT_ULL(I40E_PHY_TYPE_1000BASE_SX) +#define I40E_CAP_PHY_TYPE_1000BASE_LX BIT_ULL(I40E_PHY_TYPE_1000BASE_LX) +#define I40E_CAP_PHY_TYPE_1000BASE_T_OPTICAL \ + BIT_ULL(I40E_PHY_TYPE_1000BASE_T_OPTICAL) +#define I40E_CAP_PHY_TYPE_20GBASE_KR2 BIT_ULL(I40E_PHY_TYPE_20GBASE_KR2) +/* + * Defining the macro I40E_TYPE_OFFSET to implement a bit shift for some + * PHY types. There is an unused bit (31) in the I40E_CAP_PHY_TYPE_* bit + * fields but no corresponding gap in the i40e_aq_phy_type enumeration. So, + * a shift is needed to adjust for this with values larger than 31. The + * only affected values are I40E_PHY_TYPE_25GBASE_*. + */ +#define I40E_PHY_TYPE_OFFSET 1 +#define I40E_CAP_PHY_TYPE_25GBASE_KR BIT_ULL(I40E_PHY_TYPE_25GBASE_KR + \ + I40E_PHY_TYPE_OFFSET) +#define I40E_CAP_PHY_TYPE_25GBASE_CR BIT_ULL(I40E_PHY_TYPE_25GBASE_CR + \ + I40E_PHY_TYPE_OFFSET) +#define I40E_CAP_PHY_TYPE_25GBASE_SR BIT_ULL(I40E_PHY_TYPE_25GBASE_SR + \ + I40E_PHY_TYPE_OFFSET) +#define I40E_CAP_PHY_TYPE_25GBASE_LR BIT_ULL(I40E_PHY_TYPE_25GBASE_LR + \ + I40E_PHY_TYPE_OFFSET) #define I40E_HW_CAP_MAX_GPIO 30 #define I40E_HW_CAP_MDIO_PORT_MODE_MDIO 0 #define I40E_HW_CAP_MDIO_PORT_MODE_I2C 1 -#ifdef X722_SUPPORT enum i40e_acpi_programming_method { I40E_ACPI_PROGRAMMING_METHOD_HW_FVL = 0, I40E_ACPI_PROGRAMMING_METHOD_AQC_FPK = 1 }; -#define I40E_WOL_SUPPORT_MASK 1 -#define I40E_ACPI_PROGRAMMING_METHOD_MASK (1 << 1) -#define I40E_PROXY_SUPPORT_MASK (1 << 2) +#define I40E_WOL_SUPPORT_MASK 0x1 +#define I40E_ACPI_PROGRAMMING_METHOD_MASK 0x2 +#define I40E_PROXY_SUPPORT_MASK 0x4 -#endif /* Capabilities of a PF or a VF or the whole device */ struct i40e_hw_capabilities { u32 switch_mode; @@ -355,6 +372,10 @@ struct i40e_hw_capabilities { #define I40E_NVM_IMAGE_TYPE_UDP_CLOUD 0x3 u32 management_mode; + u32 mng_protocols_over_mctp; +#define I40E_MNG_PROTOCOL_PLDM 0x2 +#define I40E_MNG_PROTOCOL_OEM_COMMANDS 0x4 +#define I40E_MNG_PROTOCOL_NCSI 0x8 u32 npar_enable; u32 os2bmc; u32 valid_functions; @@ -410,19 +431,17 @@ struct i40e_hw_capabilities { u32 enabled_tcmap; u32 maxtc; u64 wr_csr_prot; -#ifdef X722_SUPPORT bool apm_wol_support; enum i40e_acpi_programming_method acpi_prog_method; bool proxy_support; -#endif }; struct i40e_mac_info { enum i40e_mac_type type; - u8 addr[I40E_ETH_LENGTH_OF_ADDRESS]; - u8 perm_addr[I40E_ETH_LENGTH_OF_ADDRESS]; - u8 san_addr[I40E_ETH_LENGTH_OF_ADDRESS]; - u8 port_addr[I40E_ETH_LENGTH_OF_ADDRESS]; + u8 addr[ETH_ALEN]; + u8 perm_addr[ETH_ALEN]; + u8 san_addr[ETH_ALEN]; + u8 port_addr[ETH_ALEN]; u16 max_fcoeq; }; @@ -472,6 +491,7 @@ enum i40e_nvmupd_state { I40E_NVMUPD_STATE_WRITING, I40E_NVMUPD_STATE_INIT_WAIT, I40E_NVMUPD_STATE_WRITE_WAIT, + I40E_NVMUPD_STATE_ERROR }; /* nvm_access definition and its masks/shifts need to be accessible to @@ -550,6 +570,7 @@ struct i40e_bus_info { u16 func; u16 device; u16 lan_id; + u16 bus_id; }; /* Flow control (FC) parameters */ @@ -674,30 +695,29 @@ struct i40e_hw { struct i40e_dcbx_config remote_dcbx_config; /* Peer Cfg */ struct i40e_dcbx_config desired_dcbx_config; /* CEE Desired Cfg */ -#ifdef X722_SUPPORT /* WoL and proxy support */ u16 num_wol_proxy_filters; u16 wol_proxy_vsi_seid; -#endif #define I40E_HW_FLAG_AQ_SRCTL_ACCESS_ENABLE BIT_ULL(0) +#define I40E_HW_FLAG_802_1AD_CAPABLE BIT_ULL(1) +#define I40E_HW_FLAG_AQ_PHY_ACCESS_CAPABLE BIT_ULL(2) u64 flags; + /* Used in set switch config AQ command */ + u16 switch_tag; + u16 first_tag; + u16 second_tag; + /* debug mask */ u32 debug_mask; -#ifndef I40E_NDIS_SUPPORT char err_str[16]; -#endif /* I40E_NDIS_SUPPORT */ }; STATIC INLINE bool i40e_is_vf(struct i40e_hw *hw) { -#ifdef X722_SUPPORT return (hw->mac.type == I40E_MAC_VF || hw->mac.type == I40E_MAC_X722_VF); -#else - return hw->mac.type == I40E_MAC_VF; -#endif } struct i40e_driver_version { @@ -801,11 +821,7 @@ enum i40e_rx_desc_status_bits { I40E_RX_DESC_STATUS_CRCP_SHIFT = 4, I40E_RX_DESC_STATUS_TSYNINDX_SHIFT = 5, /* 2 BITS */ I40E_RX_DESC_STATUS_TSYNVALID_SHIFT = 7, -#ifdef X722_SUPPORT I40E_RX_DESC_STATUS_EXT_UDP_0_SHIFT = 8, -#else - I40E_RX_DESC_STATUS_RESERVED1_SHIFT = 8, -#endif I40E_RX_DESC_STATUS_UMBCAST_SHIFT = 9, /* 2 BITS */ I40E_RX_DESC_STATUS_FLM_SHIFT = 11, @@ -813,11 +829,7 @@ enum i40e_rx_desc_status_bits { I40E_RX_DESC_STATUS_LPBK_SHIFT = 14, I40E_RX_DESC_STATUS_IPV6EXADD_SHIFT = 15, I40E_RX_DESC_STATUS_RESERVED2_SHIFT = 16, /* 2 BITS */ -#ifdef X722_SUPPORT I40E_RX_DESC_STATUS_INT_UDP_0_SHIFT = 18, -#else - I40E_RX_DESC_STATUS_UDP_0_SHIFT = 18, -#endif I40E_RX_DESC_STATUS_LAST /* this entry must be last!!! */ }; @@ -1195,10 +1207,8 @@ enum i40e_tx_ctx_desc_eipt_offload { #define I40E_TXD_CTX_QW0_DECTTL_MASK (0xFULL << \ I40E_TXD_CTX_QW0_DECTTL_SHIFT) -#ifdef X722_SUPPORT #define I40E_TXD_CTX_QW0_L4T_CS_SHIFT 23 #define I40E_TXD_CTX_QW0_L4T_CS_MASK BIT_ULL(I40E_TXD_CTX_QW0_L4T_CS_SHIFT) -#endif struct i40e_nop_desc { __le64 rsvd; __le64 dtype_cmd; @@ -1235,38 +1245,24 @@ struct i40e_filter_program_desc { /* Packet Classifier Types for filters */ enum i40e_filter_pctype { -#ifdef X722_SUPPORT /* Note: Values 0-28 are reserved for future use. * Value 29, 30, 32 are not supported on XL710 and X710. */ I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP = 29, I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP = 30, -#else - /* Note: Values 0-30 are reserved for future use */ -#endif I40E_FILTER_PCTYPE_NONF_IPV4_UDP = 31, -#ifdef X722_SUPPORT I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK = 32, -#else - /* Note: Value 32 is reserved for future use */ -#endif I40E_FILTER_PCTYPE_NONF_IPV4_TCP = 33, I40E_FILTER_PCTYPE_NONF_IPV4_SCTP = 34, I40E_FILTER_PCTYPE_NONF_IPV4_OTHER = 35, I40E_FILTER_PCTYPE_FRAG_IPV4 = 36, -#ifdef X722_SUPPORT /* Note: Values 37-38 are reserved for future use. * Value 39, 40, 42 are not supported on XL710 and X710. */ I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP = 39, I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP = 40, -#else - /* Note: Values 37-40 are reserved for future use */ -#endif I40E_FILTER_PCTYPE_NONF_IPV6_UDP = 41, -#ifdef X722_SUPPORT I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK = 42, -#endif I40E_FILTER_PCTYPE_NONF_IPV6_TCP = 43, I40E_FILTER_PCTYPE_NONF_IPV6_SCTP = 44, I40E_FILTER_PCTYPE_NONF_IPV6_OTHER = 45, @@ -1321,12 +1317,10 @@ enum i40e_filter_program_desc_pcmd { I40E_TXD_FLTR_QW1_CMD_SHIFT) #define I40E_TXD_FLTR_QW1_FD_STATUS_MASK (0x3ULL << \ I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT) -#ifdef X722_SUPPORT #define I40E_TXD_FLTR_QW1_ATR_SHIFT (0xEULL + \ I40E_TXD_FLTR_QW1_CMD_SHIFT) #define I40E_TXD_FLTR_QW1_ATR_MASK BIT_ULL(I40E_TXD_FLTR_QW1_ATR_SHIFT) -#endif #define I40E_TXD_FLTR_QW1_CNTINDEX_SHIFT 20 #define I40E_TXD_FLTR_QW1_CNTINDEX_MASK (0x1FFUL << \ @@ -1388,6 +1382,23 @@ struct i40e_veb_tc_stats { u64 tc_tx_bytes[I40E_MAX_TRAFFIC_CLASS]; }; +/* Statistics collected per function for FCoE */ +struct i40e_fcoe_stats { + u64 rx_fcoe_packets; /* fcoeprc */ + u64 rx_fcoe_dwords; /* focedwrc */ + u64 rx_fcoe_dropped; /* fcoerpdc */ + u64 tx_fcoe_packets; /* fcoeptc */ + u64 tx_fcoe_dwords; /* focedwtc */ + u64 fcoe_bad_fccrc; /* fcoecrc */ + u64 fcoe_last_error; /* fcoelast */ + u64 fcoe_ddp_count; /* fcoeddpc */ +}; + +/* offset to per function FCoE statistics block */ +#define I40E_FCOE_VF_STAT_OFFSET 0 +#define I40E_FCOE_PF_STAT_OFFSET 128 +#define I40E_FCOE_STAT_MAX (I40E_FCOE_PF_STAT_OFFSET + I40E_MAX_PF) + /* Statistics collected by the MAC */ struct i40e_hw_port_stats { /* eth stats collected by the port */ @@ -1481,6 +1492,7 @@ struct i40e_hw_port_stats { #define I40E_SR_EMPR_REGS_AUTO_LOAD_PTR 0x3A #define I40E_SR_GLOBR_REGS_AUTO_LOAD_PTR 0x3B #define I40E_SR_CORER_REGS_AUTO_LOAD_PTR 0x3C +#define I40E_SR_PHY_ACTIVITY_LIST_PTR 0x3D #define I40E_SR_PCIE_ALT_AUTO_LOAD_PTR 0x3E #define I40E_SR_SW_CHECKSUM_WORD 0x3F #define I40E_SR_1ST_FREE_PROVISION_AREA_PTR 0x40 @@ -1509,6 +1521,208 @@ struct i40e_hw_port_stats { #define I40E_SRRD_SRCTL_ATTEMPTS 100000 +/* FCoE Tx context descriptor - Use the i40e_tx_context_desc struct */ + +enum i40E_fcoe_tx_ctx_desc_cmd_bits { + I40E_FCOE_TX_CTX_DESC_OPCODE_SINGLE_SEND = 0x00, /* 4 BITS */ + I40E_FCOE_TX_CTX_DESC_OPCODE_TSO_FC_CLASS2 = 0x01, /* 4 BITS */ + I40E_FCOE_TX_CTX_DESC_OPCODE_TSO_FC_CLASS3 = 0x05, /* 4 BITS */ + I40E_FCOE_TX_CTX_DESC_OPCODE_ETSO_FC_CLASS2 = 0x02, /* 4 BITS */ + I40E_FCOE_TX_CTX_DESC_OPCODE_ETSO_FC_CLASS3 = 0x06, /* 4 BITS */ + I40E_FCOE_TX_CTX_DESC_OPCODE_DWO_FC_CLASS2 = 0x03, /* 4 BITS */ + I40E_FCOE_TX_CTX_DESC_OPCODE_DWO_FC_CLASS3 = 0x07, /* 4 BITS */ + I40E_FCOE_TX_CTX_DESC_OPCODE_DDP_CTX_INVL = 0x08, /* 4 BITS */ + I40E_FCOE_TX_CTX_DESC_OPCODE_DWO_CTX_INVL = 0x09, /* 4 BITS */ + I40E_FCOE_TX_CTX_DESC_RELOFF = 0x10, + I40E_FCOE_TX_CTX_DESC_CLRSEQ = 0x20, + I40E_FCOE_TX_CTX_DESC_DIFENA = 0x40, + I40E_FCOE_TX_CTX_DESC_IL2TAG2 = 0x80 +}; + +/* FCoE DIF/DIX Context descriptor */ +struct i40e_fcoe_difdix_context_desc { + __le64 flags_buff0_buff1_ref; + __le64 difapp_msk_bias; +}; + +#define I40E_FCOE_DIFDIX_CTX_QW0_FLAGS_SHIFT 0 +#define I40E_FCOE_DIFDIX_CTX_QW0_FLAGS_MASK (0xFFFULL << \ + I40E_FCOE_DIFDIX_CTX_QW0_FLAGS_SHIFT) + +enum i40e_fcoe_difdix_ctx_desc_flags_bits { + /* 2 BITS */ + I40E_FCOE_DIFDIX_CTX_DESC_RSVD = 0x0000, + /* 1 BIT */ + I40E_FCOE_DIFDIX_CTX_DESC_APPTYPE_TAGCHK = 0x0000, + /* 1 BIT */ + I40E_FCOE_DIFDIX_CTX_DESC_APPTYPE_TAGNOTCHK = 0x0004, + /* 2 BITS */ + I40E_FCOE_DIFDIX_CTX_DESC_GTYPE_OPAQUE = 0x0000, + /* 2 BITS */ + I40E_FCOE_DIFDIX_CTX_DESC_GTYPE_CHKINTEGRITY = 0x0008, + /* 2 BITS */ + I40E_FCOE_DIFDIX_CTX_DESC_GTYPE_CHKINTEGRITY_APPTAG = 0x0010, + /* 2 BITS */ + I40E_FCOE_DIFDIX_CTX_DESC_GTYPE_CHKINTEGRITY_APPREFTAG = 0x0018, + /* 2 BITS */ + I40E_FCOE_DIFDIX_CTX_DESC_REFTYPE_CNST = 0x0000, + /* 2 BITS */ + I40E_FCOE_DIFDIX_CTX_DESC_REFTYPE_INC1BLK = 0x0020, + /* 2 BITS */ + I40E_FCOE_DIFDIX_CTX_DESC_REFTYPE_APPTAG = 0x0040, + /* 2 BITS */ + I40E_FCOE_DIFDIX_CTX_DESC_REFTYPE_RSVD = 0x0060, + /* 1 BIT */ + I40E_FCOE_DIFDIX_CTX_DESC_DIXMODE_XSUM = 0x0000, + /* 1 BIT */ + I40E_FCOE_DIFDIX_CTX_DESC_DIXMODE_CRC = 0x0080, + /* 2 BITS */ + I40E_FCOE_DIFDIX_CTX_DESC_DIFHOST_UNTAG = 0x0000, + /* 2 BITS */ + I40E_FCOE_DIFDIX_CTX_DESC_DIFHOST_BUF = 0x0100, + /* 2 BITS */ + I40E_FCOE_DIFDIX_CTX_DESC_DIFHOST_RSVD = 0x0200, + /* 2 BITS */ + I40E_FCOE_DIFDIX_CTX_DESC_DIFHOST_EMBDTAGS = 0x0300, + /* 1 BIT */ + I40E_FCOE_DIFDIX_CTX_DESC_DIFLAN_UNTAG = 0x0000, + /* 1 BIT */ + I40E_FCOE_DIFDIX_CTX_DESC_DIFLAN_TAG = 0x0400, + /* 1 BIT */ + I40E_FCOE_DIFDIX_CTX_DESC_DIFBLK_512B = 0x0000, + /* 1 BIT */ + I40E_FCOE_DIFDIX_CTX_DESC_DIFBLK_4K = 0x0800 +}; + +#define I40E_FCOE_DIFDIX_CTX_QW0_BUFF0_SHIFT 12 +#define I40E_FCOE_DIFDIX_CTX_QW0_BUFF0_MASK (0x3FFULL << \ + I40E_FCOE_DIFDIX_CTX_QW0_BUFF0_SHIFT) + +#define I40E_FCOE_DIFDIX_CTX_QW0_BUFF1_SHIFT 22 +#define I40E_FCOE_DIFDIX_CTX_QW0_BUFF1_MASK (0x3FFULL << \ + I40E_FCOE_DIFDIX_CTX_QW0_BUFF1_SHIFT) + +#define I40E_FCOE_DIFDIX_CTX_QW0_REF_SHIFT 32 +#define I40E_FCOE_DIFDIX_CTX_QW0_REF_MASK (0xFFFFFFFFULL << \ + I40E_FCOE_DIFDIX_CTX_QW0_REF_SHIFT) + +#define I40E_FCOE_DIFDIX_CTX_QW1_APP_SHIFT 0 +#define I40E_FCOE_DIFDIX_CTX_QW1_APP_MASK (0xFFFFULL << \ + I40E_FCOE_DIFDIX_CTX_QW1_APP_SHIFT) + +#define I40E_FCOE_DIFDIX_CTX_QW1_APP_MSK_SHIFT 16 +#define I40E_FCOE_DIFDIX_CTX_QW1_APP_MSK_MASK (0xFFFFULL << \ + I40E_FCOE_DIFDIX_CTX_QW1_APP_MSK_SHIFT) + +#define I40E_FCOE_DIFDIX_CTX_QW1_REF_BIAS_SHIFT 32 +#define I40E_FCOE_DIFDIX_CTX_QW0_REF_BIAS_MASK (0xFFFFFFFFULL << \ + I40E_FCOE_DIFDIX_CTX_QW1_REF_BIAS_SHIFT) + +/* FCoE DIF/DIX Buffers descriptor */ +struct i40e_fcoe_difdix_buffers_desc { + __le64 buff_addr0; + __le64 buff_addr1; +}; + +/* FCoE DDP Context descriptor */ +struct i40e_fcoe_ddp_context_desc { + __le64 rsvd; + __le64 type_cmd_foff_lsize; +}; + +#define I40E_FCOE_DDP_CTX_QW1_DTYPE_SHIFT 0 +#define I40E_FCOE_DDP_CTX_QW1_DTYPE_MASK (0xFULL << \ + I40E_FCOE_DDP_CTX_QW1_DTYPE_SHIFT) + +#define I40E_FCOE_DDP_CTX_QW1_CMD_SHIFT 4 +#define I40E_FCOE_DDP_CTX_QW1_CMD_MASK (0xFULL << \ + I40E_FCOE_DDP_CTX_QW1_CMD_SHIFT) + +enum i40e_fcoe_ddp_ctx_desc_cmd_bits { + I40E_FCOE_DDP_CTX_DESC_BSIZE_512B = 0x00, /* 2 BITS */ + I40E_FCOE_DDP_CTX_DESC_BSIZE_4K = 0x01, /* 2 BITS */ + I40E_FCOE_DDP_CTX_DESC_BSIZE_8K = 0x02, /* 2 BITS */ + I40E_FCOE_DDP_CTX_DESC_BSIZE_16K = 0x03, /* 2 BITS */ + I40E_FCOE_DDP_CTX_DESC_DIFENA = 0x04, /* 1 BIT */ + I40E_FCOE_DDP_CTX_DESC_LASTSEQH = 0x08, /* 1 BIT */ +}; + +#define I40E_FCOE_DDP_CTX_QW1_FOFF_SHIFT 16 +#define I40E_FCOE_DDP_CTX_QW1_FOFF_MASK (0x3FFFULL << \ + I40E_FCOE_DDP_CTX_QW1_FOFF_SHIFT) + +#define I40E_FCOE_DDP_CTX_QW1_LSIZE_SHIFT 32 +#define I40E_FCOE_DDP_CTX_QW1_LSIZE_MASK (0x3FFFULL << \ + I40E_FCOE_DDP_CTX_QW1_LSIZE_SHIFT) + +/* FCoE DDP/DWO Queue Context descriptor */ +struct i40e_fcoe_queue_context_desc { + __le64 dmaindx_fbase; /* 0:11 DMAINDX, 12:63 FBASE */ + __le64 flen_tph; /* 0:12 FLEN, 13:15 TPH */ +}; + +#define I40E_FCOE_QUEUE_CTX_QW0_DMAINDX_SHIFT 0 +#define I40E_FCOE_QUEUE_CTX_QW0_DMAINDX_MASK (0xFFFULL << \ + I40E_FCOE_QUEUE_CTX_QW0_DMAINDX_SHIFT) + +#define I40E_FCOE_QUEUE_CTX_QW0_FBASE_SHIFT 12 +#define I40E_FCOE_QUEUE_CTX_QW0_FBASE_MASK (0xFFFFFFFFFFFFFULL << \ + I40E_FCOE_QUEUE_CTX_QW0_FBASE_SHIFT) + +#define I40E_FCOE_QUEUE_CTX_QW1_FLEN_SHIFT 0 +#define I40E_FCOE_QUEUE_CTX_QW1_FLEN_MASK (0x1FFFULL << \ + I40E_FCOE_QUEUE_CTX_QW1_FLEN_SHIFT) + +#define I40E_FCOE_QUEUE_CTX_QW1_TPH_SHIFT 13 +#define I40E_FCOE_QUEUE_CTX_QW1_TPH_MASK (0x7ULL << \ + I40E_FCOE_QUEUE_CTX_QW1_FLEN_SHIFT) + +enum i40e_fcoe_queue_ctx_desc_tph_bits { + I40E_FCOE_QUEUE_CTX_DESC_TPHRDESC = 0x1, + I40E_FCOE_QUEUE_CTX_DESC_TPHDATA = 0x2 +}; + +#define I40E_FCOE_QUEUE_CTX_QW1_RECIPE_SHIFT 30 +#define I40E_FCOE_QUEUE_CTX_QW1_RECIPE_MASK (0x3ULL << \ + I40E_FCOE_QUEUE_CTX_QW1_RECIPE_SHIFT) + +/* FCoE DDP/DWO Filter Context descriptor */ +struct i40e_fcoe_filter_context_desc { + __le32 param; + __le16 seqn; + + /* 48:51(0:3) RSVD, 52:63(4:15) DMAINDX */ + __le16 rsvd_dmaindx; + + /* 0:7 FLAGS, 8:52 RSVD, 53:63 LANQ */ + __le64 flags_rsvd_lanq; +}; + +#define I40E_FCOE_FILTER_CTX_QW0_DMAINDX_SHIFT 4 +#define I40E_FCOE_FILTER_CTX_QW0_DMAINDX_MASK (0xFFF << \ + I40E_FCOE_FILTER_CTX_QW0_DMAINDX_SHIFT) + +enum i40e_fcoe_filter_ctx_desc_flags_bits { + I40E_FCOE_FILTER_CTX_DESC_CTYP_DDP = 0x00, + I40E_FCOE_FILTER_CTX_DESC_CTYP_DWO = 0x01, + I40E_FCOE_FILTER_CTX_DESC_ENODE_INIT = 0x00, + I40E_FCOE_FILTER_CTX_DESC_ENODE_RSP = 0x02, + I40E_FCOE_FILTER_CTX_DESC_FC_CLASS2 = 0x00, + I40E_FCOE_FILTER_CTX_DESC_FC_CLASS3 = 0x04 +}; + +#define I40E_FCOE_FILTER_CTX_QW1_FLAGS_SHIFT 0 +#define I40E_FCOE_FILTER_CTX_QW1_FLAGS_MASK (0xFFULL << \ + I40E_FCOE_FILTER_CTX_QW1_FLAGS_SHIFT) + +#define I40E_FCOE_FILTER_CTX_QW1_PCTYPE_SHIFT 8 +#define I40E_FCOE_FILTER_CTX_QW1_PCTYPE_MASK (0x3FULL << \ + I40E_FCOE_FILTER_CTX_QW1_PCTYPE_SHIFT) + +#define I40E_FCOE_FILTER_CTX_QW1_LANQINDX_SHIFT 53 +#define I40E_FCOE_FILTER_CTX_QW1_LANQINDX_MASK (0x7FFULL << \ + I40E_FCOE_FILTER_CTX_QW1_LANQINDX_SHIFT) + enum i40e_switch_element_types { I40E_SWITCH_ELEMENT_TYPE_MAC = 1, I40E_SWITCH_ELEMENT_TYPE_PF = 2, @@ -1671,4 +1885,106 @@ struct i40e_lldp_variables { #define I40E_FLEX_56_MASK (0x1ULL << I40E_FLEX_56_SHIFT) #define I40E_FLEX_57_SHIFT 6 #define I40E_FLEX_57_MASK (0x1ULL << I40E_FLEX_57_SHIFT) + +/* Version format for Dynamic Device Personalization(DDP) */ +struct i40e_ddp_version { + u8 major; + u8 minor; + u8 update; + u8 draft; +}; + +#define I40E_DDP_NAME_SIZE 32 + +/* Package header */ +struct i40e_package_header { + struct i40e_ddp_version version; + u32 segment_count; + u32 segment_offset[1]; +}; + +/* Generic segment header */ +struct i40e_generic_seg_header { +#define SEGMENT_TYPE_METADATA 0x00000001 +#define SEGMENT_TYPE_NOTES 0x00000002 +#define SEGMENT_TYPE_I40E 0x00000011 +#define SEGMENT_TYPE_X722 0x00000012 + u32 type; + struct i40e_ddp_version version; + u32 size; + char name[I40E_DDP_NAME_SIZE]; +}; + +struct i40e_metadata_segment { + struct i40e_generic_seg_header header; + struct i40e_ddp_version version; +#define I40E_DDP_TRACKID_RDONLY 0 +#define I40E_DDP_TRACKID_INVALID 0xFFFFFFFF + u32 track_id; + char name[I40E_DDP_NAME_SIZE]; +}; + +struct i40e_device_id_entry { + u32 vendor_dev_id; + u32 sub_vendor_dev_id; +}; + +struct i40e_profile_segment { + struct i40e_generic_seg_header header; + struct i40e_ddp_version version; + char name[I40E_DDP_NAME_SIZE]; + u32 device_table_count; + struct i40e_device_id_entry device_table[1]; +}; + +struct i40e_section_table { + u32 section_count; + u32 section_offset[1]; +}; + +struct i40e_profile_section_header { + u16 tbl_size; + u16 data_end; + struct { +#define SECTION_TYPE_INFO 0x00000010 +#define SECTION_TYPE_MMIO 0x00000800 +#define SECTION_TYPE_RB_MMIO 0x00001800 +#define SECTION_TYPE_AQ 0x00000801 +#define SECTION_TYPE_RB_AQ 0x00001801 +#define SECTION_TYPE_NOTE 0x80000000 +#define SECTION_TYPE_NAME 0x80000001 +#define SECTION_TYPE_PROTO 0x80000002 +#define SECTION_TYPE_PCTYPE 0x80000003 +#define SECTION_TYPE_PTYPE 0x80000004 + u32 type; + u32 offset; + u32 size; + } section; +}; + +struct i40e_profile_tlv_section_record { + u8 rtype; + u8 type; + u16 len; + u8 data[12]; +}; + +/* Generic AQ section in proflie */ +struct i40e_profile_aq_section { + u16 opcode; + u16 flags; + u8 param[16]; + u16 datalen; + u8 data[1]; +}; + +struct i40e_profile_info { + u32 track_id; + struct i40e_ddp_version version; + u8 op; +#define I40E_DDP_ADD_TRACKID 0x01 +#define I40E_DDP_REMOVE_TRACKID 0x02 + u8 reserved[7]; + u8 name[I40E_DDP_NAME_SIZE]; +}; #endif /* _I40E_TYPE_H_ */ diff --git a/dpdk/drivers/net/i40e/base/i40e_virtchnl.h b/dpdk/drivers/net/i40e/base/i40e_virtchnl.h deleted file mode 100644 index fd51ec32..00000000 --- a/dpdk/drivers/net/i40e/base/i40e_virtchnl.h +++ /dev/null @@ -1,438 +0,0 @@ -/******************************************************************************* - -Copyright (c) 2013 - 2015, Intel Corporation -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - 3. Neither the name of the Intel Corporation nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. - -***************************************************************************/ - -#ifndef _I40E_VIRTCHNL_H_ -#define _I40E_VIRTCHNL_H_ - -#include "i40e_type.h" - -/* Description: - * This header file describes the VF-PF communication protocol used - * by the various i40e drivers. - * - * Admin queue buffer usage: - * desc->opcode is always i40e_aqc_opc_send_msg_to_pf - * flags, retval, datalen, and data addr are all used normally. - * Firmware copies the cookie fields when sending messages between the PF and - * VF, but uses all other fields internally. Due to this limitation, we - * must send all messages as "indirect", i.e. using an external buffer. - * - * All the vsi indexes are relative to the VF. Each VF can have maximum of - * three VSIs. All the queue indexes are relative to the VSI. Each VF can - * have a maximum of sixteen queues for all of its VSIs. - * - * The PF is required to return a status code in v_retval for all messages - * except RESET_VF, which does not require any response. The return value is of - * i40e_status_code type, defined in the i40e_type.h. - * - * In general, VF driver initialization should roughly follow the order of these - * opcodes. The VF driver must first validate the API version of the PF driver, - * then request a reset, then get resources, then configure queues and - * interrupts. After these operations are complete, the VF driver may start - * its queues, optionally add MAC and VLAN filters, and process traffic. - */ - -/* Opcodes for VF-PF communication. These are placed in the v_opcode field - * of the virtchnl_msg structure. - */ -enum i40e_virtchnl_ops { -/* The PF sends status change events to VFs using - * the I40E_VIRTCHNL_OP_EVENT opcode. - * VFs send requests to the PF using the other ops. - */ - I40E_VIRTCHNL_OP_UNKNOWN = 0, - I40E_VIRTCHNL_OP_VERSION = 1, /* must ALWAYS be 1 */ - I40E_VIRTCHNL_OP_RESET_VF = 2, - I40E_VIRTCHNL_OP_GET_VF_RESOURCES = 3, - I40E_VIRTCHNL_OP_CONFIG_TX_QUEUE = 4, - I40E_VIRTCHNL_OP_CONFIG_RX_QUEUE = 5, - I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES = 6, - I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP = 7, - I40E_VIRTCHNL_OP_ENABLE_QUEUES = 8, - I40E_VIRTCHNL_OP_DISABLE_QUEUES = 9, - I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS = 10, - I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS = 11, - I40E_VIRTCHNL_OP_ADD_VLAN = 12, - I40E_VIRTCHNL_OP_DEL_VLAN = 13, - I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE = 14, - I40E_VIRTCHNL_OP_GET_STATS = 15, - I40E_VIRTCHNL_OP_FCOE = 16, - I40E_VIRTCHNL_OP_EVENT = 17, /* must ALWAYS be 17 */ -#ifdef I40E_SOL_VF_SUPPORT - I40E_VIRTCHNL_OP_GET_ADDNL_SOL_CONFIG = 19, -#endif - I40E_VIRTCHNL_OP_CONFIG_RSS_KEY = 23, - I40E_VIRTCHNL_OP_CONFIG_RSS_LUT = 24, - I40E_VIRTCHNL_OP_GET_RSS_HENA_CAPS = 25, - I40E_VIRTCHNL_OP_SET_RSS_HENA = 26, - -}; - -/* Virtual channel message descriptor. This overlays the admin queue - * descriptor. All other data is passed in external buffers. - */ - -struct i40e_virtchnl_msg { - u8 pad[8]; /* AQ flags/opcode/len/retval fields */ - enum i40e_virtchnl_ops v_opcode; /* avoid confusion with desc->opcode */ - enum i40e_status_code v_retval; /* ditto for desc->retval */ - u32 vfid; /* used by PF when sending to VF */ -}; - -/* Message descriptions and data structures.*/ - -/* I40E_VIRTCHNL_OP_VERSION - * VF posts its version number to the PF. PF responds with its version number - * in the same format, along with a return code. - * Reply from PF has its major/minor versions also in param0 and param1. - * If there is a major version mismatch, then the VF cannot operate. - * If there is a minor version mismatch, then the VF can operate but should - * add a warning to the system log. - * - * This enum element MUST always be specified as == 1, regardless of other - * changes in the API. The PF must always respond to this message without - * error regardless of version mismatch. - */ -#define I40E_VIRTCHNL_VERSION_MAJOR 1 -#define I40E_VIRTCHNL_VERSION_MINOR 1 -#define I40E_VIRTCHNL_VERSION_MINOR_NO_VF_CAPS 0 - -struct i40e_virtchnl_version_info { - u32 major; - u32 minor; -}; - -/* I40E_VIRTCHNL_OP_RESET_VF - * VF sends this request to PF with no parameters - * PF does NOT respond! VF driver must delay then poll VFGEN_RSTAT register - * until reset completion is indicated. The admin queue must be reinitialized - * after this operation. - * - * When reset is complete, PF must ensure that all queues in all VSIs associated - * with the VF are stopped, all queue configurations in the HMC are set to 0, - * and all MAC and VLAN filters (except the default MAC address) on all VSIs - * are cleared. - */ - -/* I40E_VIRTCHNL_OP_GET_VF_RESOURCES - * Version 1.0 VF sends this request to PF with no parameters - * Version 1.1 VF sends this request to PF with u32 bitmap of its capabilities - * PF responds with an indirect message containing - * i40e_virtchnl_vf_resource and one or more - * i40e_virtchnl_vsi_resource structures. - */ - -struct i40e_virtchnl_vsi_resource { - u16 vsi_id; - u16 num_queue_pairs; - enum i40e_vsi_type vsi_type; - u16 qset_handle; - u8 default_mac_addr[I40E_ETH_LENGTH_OF_ADDRESS]; -}; -/* VF offload flags */ -#define I40E_VIRTCHNL_VF_OFFLOAD_L2 0x00000001 -#define I40E_VIRTCHNL_VF_OFFLOAD_IWARP 0x00000002 -#define I40E_VIRTCHNL_VF_OFFLOAD_FCOE 0x00000004 -#define I40E_VIRTCHNL_VF_OFFLOAD_RSS_AQ 0x00000008 -#define I40E_VIRTCHNL_VF_OFFLOAD_RSS_REG 0x00000010 -#define I40E_VIRTCHNL_VF_OFFLOAD_WB_ON_ITR 0x00000020 -#define I40E_VIRTCHNL_VF_OFFLOAD_VLAN 0x00010000 -#define I40E_VIRTCHNL_VF_OFFLOAD_RX_POLLING 0x00020000 -#define I40E_VIRTCHNL_VF_OFFLOAD_RSS_PCTYPE_V2 0x00040000 -#define I40E_VIRTCHNL_VF_OFFLOAD_RSS_PF 0X00080000 - -struct i40e_virtchnl_vf_resource { - u16 num_vsis; - u16 num_queue_pairs; - u16 max_vectors; - u16 max_mtu; - - u32 vf_offload_flags; - u32 rss_key_size; - u32 rss_lut_size; - - struct i40e_virtchnl_vsi_resource vsi_res[1]; -}; - -/* I40E_VIRTCHNL_OP_CONFIG_TX_QUEUE - * VF sends this message to set up parameters for one TX queue. - * External data buffer contains one instance of i40e_virtchnl_txq_info. - * PF configures requested queue and returns a status code. - */ - -/* Tx queue config info */ -struct i40e_virtchnl_txq_info { - u16 vsi_id; - u16 queue_id; - u16 ring_len; /* number of descriptors, multiple of 8 */ - u16 headwb_enabled; - u64 dma_ring_addr; - u64 dma_headwb_addr; -}; - -/* I40E_VIRTCHNL_OP_CONFIG_RX_QUEUE - * VF sends this message to set up parameters for one RX queue. - * External data buffer contains one instance of i40e_virtchnl_rxq_info. - * PF configures requested queue and returns a status code. - */ - -/* Rx queue config info */ -struct i40e_virtchnl_rxq_info { - u16 vsi_id; - u16 queue_id; - u32 ring_len; /* number of descriptors, multiple of 32 */ - u16 hdr_size; - u16 splithdr_enabled; - u32 databuffer_size; - u32 max_pkt_size; - u64 dma_ring_addr; - enum i40e_hmc_obj_rx_hsplit_0 rx_split_pos; -}; - -/* I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES - * VF sends this message to set parameters for all active TX and RX queues - * associated with the specified VSI. - * PF configures queues and returns status. - * If the number of queues specified is greater than the number of queues - * associated with the VSI, an error is returned and no queues are configured. - */ -struct i40e_virtchnl_queue_pair_info { - /* NOTE: vsi_id and queue_id should be identical for both queues. */ - struct i40e_virtchnl_txq_info txq; - struct i40e_virtchnl_rxq_info rxq; -}; - -struct i40e_virtchnl_vsi_queue_config_info { - u16 vsi_id; - u16 num_queue_pairs; - struct i40e_virtchnl_queue_pair_info qpair[1]; -}; - -/* I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP - * VF uses this message to map vectors to queues. - * The rxq_map and txq_map fields are bitmaps used to indicate which queues - * are to be associated with the specified vector. - * The "other" causes are always mapped to vector 0. - * PF configures interrupt mapping and returns status. - */ -struct i40e_virtchnl_vector_map { - u16 vsi_id; - u16 vector_id; - u16 rxq_map; - u16 txq_map; - u16 rxitr_idx; - u16 txitr_idx; -}; - -struct i40e_virtchnl_irq_map_info { - u16 num_vectors; - struct i40e_virtchnl_vector_map vecmap[1]; -}; - -/* I40E_VIRTCHNL_OP_ENABLE_QUEUES - * I40E_VIRTCHNL_OP_DISABLE_QUEUES - * VF sends these message to enable or disable TX/RX queue pairs. - * The queues fields are bitmaps indicating which queues to act upon. - * (Currently, we only support 16 queues per VF, but we make the field - * u32 to allow for expansion.) - * PF performs requested action and returns status. - */ -struct i40e_virtchnl_queue_select { - u16 vsi_id; - u16 pad; - u32 rx_queues; - u32 tx_queues; -}; - -/* I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS - * VF sends this message in order to add one or more unicast or multicast - * address filters for the specified VSI. - * PF adds the filters and returns status. - */ - -/* I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS - * VF sends this message in order to remove one or more unicast or multicast - * filters for the specified VSI. - * PF removes the filters and returns status. - */ - -struct i40e_virtchnl_ether_addr { - u8 addr[I40E_ETH_LENGTH_OF_ADDRESS]; - u8 pad[2]; -}; - -struct i40e_virtchnl_ether_addr_list { - u16 vsi_id; - u16 num_elements; - struct i40e_virtchnl_ether_addr list[1]; -}; - -#ifdef I40E_SOL_VF_SUPPORT -/* I40E_VIRTCHNL_OP_GET_ADDNL_SOL_CONFIG - * VF sends this message to get the default MTU and list of additional ethernet - * addresses it is allowed to use. - * PF responds with an indirect message containing - * i40e_virtchnl_addnl_solaris_config with zero or more - * i40e_virtchnl_ether_addr structures. - * - * It is expected that this operation will only ever be needed for Solaris VFs - * running under a Solaris PF. - */ -struct i40e_virtchnl_addnl_solaris_config { - u16 default_mtu; - struct i40e_virtchnl_ether_addr_list al; -}; - -#endif -/* I40E_VIRTCHNL_OP_ADD_VLAN - * VF sends this message to add one or more VLAN tag filters for receives. - * PF adds the filters and returns status. - * If a port VLAN is configured by the PF, this operation will return an - * error to the VF. - */ - -/* I40E_VIRTCHNL_OP_DEL_VLAN - * VF sends this message to remove one or more VLAN tag filters for receives. - * PF removes the filters and returns status. - * If a port VLAN is configured by the PF, this operation will return an - * error to the VF. - */ - -struct i40e_virtchnl_vlan_filter_list { - u16 vsi_id; - u16 num_elements; - u16 vlan_id[1]; -}; - -/* I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE - * VF sends VSI id and flags. - * PF returns status code in retval. - * Note: we assume that broadcast accept mode is always enabled. - */ -struct i40e_virtchnl_promisc_info { - u16 vsi_id; - u16 flags; -}; - -#define I40E_FLAG_VF_UNICAST_PROMISC 0x00000001 -#define I40E_FLAG_VF_MULTICAST_PROMISC 0x00000002 - -/* I40E_VIRTCHNL_OP_GET_STATS - * VF sends this message to request stats for the selected VSI. VF uses - * the i40e_virtchnl_queue_select struct to specify the VSI. The queue_id - * field is ignored by the PF. - * - * PF replies with struct i40e_eth_stats in an external buffer. - */ - -/* I40E_VIRTCHNL_OP_CONFIG_RSS_KEY - * I40E_VIRTCHNL_OP_CONFIG_RSS_LUT - * VF sends these messages to configure RSS. Only supported if both PF - * and VF drivers set the I40E_VIRTCHNL_VF_OFFLOAD_RSS_PF bit during - * configuration negotiation. If this is the case, then the rss fields in - * the vf resource struct are valid. - * Both the key and LUT are initialized to 0 by the PF, meaning that - * RSS is effectively disabled until set up by the VF. - */ -struct i40e_virtchnl_rss_key { - u16 vsi_id; - u16 key_len; - u8 key[1]; /* RSS hash key, packed bytes */ -}; - -struct i40e_virtchnl_rss_lut { - u16 vsi_id; - u16 lut_entries; - u8 lut[1]; /* RSS lookup table*/ -}; - -/* I40E_VIRTCHNL_OP_GET_RSS_HENA_CAPS - * I40E_VIRTCHNL_OP_SET_RSS_HENA - * VF sends these messages to get and set the hash filter enable bits for RSS. - * By default, the PF sets these to all possible traffic types that the - * hardware supports. The VF can query this value if it wants to change the - * traffic types that are hashed by the hardware. - * Traffic types are defined in the i40e_filter_pctype enum in i40e_type.h - */ -struct i40e_virtchnl_rss_hena { - u64 hena; -}; - -/* I40E_VIRTCHNL_OP_EVENT - * PF sends this message to inform the VF driver of events that may affect it. - * No direct response is expected from the VF, though it may generate other - * messages in response to this one. - */ -enum i40e_virtchnl_event_codes { - I40E_VIRTCHNL_EVENT_UNKNOWN = 0, - I40E_VIRTCHNL_EVENT_LINK_CHANGE, - I40E_VIRTCHNL_EVENT_RESET_IMPENDING, - I40E_VIRTCHNL_EVENT_PF_DRIVER_CLOSE, -}; -#define I40E_PF_EVENT_SEVERITY_INFO 0 -#define I40E_PF_EVENT_SEVERITY_ATTENTION 1 -#define I40E_PF_EVENT_SEVERITY_ACTION_REQUIRED 2 -#define I40E_PF_EVENT_SEVERITY_CERTAIN_DOOM 255 - -struct i40e_virtchnl_pf_event { - enum i40e_virtchnl_event_codes event; - union { - struct { - enum i40e_aq_link_speed link_speed; - bool link_status; - } link_event; - } event_data; - - int severity; -}; - -/* VF reset states - these are written into the RSTAT register: - * I40E_VFGEN_RSTAT1 on the PF - * I40E_VFGEN_RSTAT on the VF - * When the PF initiates a reset, it writes 0 - * When the reset is complete, it writes 1 - * When the PF detects that the VF has recovered, it writes 2 - * VF checks this register periodically to determine if a reset has occurred, - * then polls it to know when the reset is complete. - * If either the PF or VF reads the register while the hardware - * is in a reset state, it will return DEADBEEF, which, when masked - * will result in 3. - */ -enum i40e_vfr_states { - I40E_VFR_INPROGRESS = 0, - I40E_VFR_COMPLETED, - I40E_VFR_VFACTIVE, - I40E_VFR_UNKNOWN, -}; - -#endif /* _I40E_VIRTCHNL_H_ */ diff --git a/dpdk/drivers/net/i40e/base/virtchnl.h b/dpdk/drivers/net/i40e/base/virtchnl.h new file mode 100644 index 00000000..f00dd360 --- /dev/null +++ b/dpdk/drivers/net/i40e/base/virtchnl.h @@ -0,0 +1,772 @@ +/******************************************************************************* + +Copyright (c) 2013 - 2015, Intel Corporation +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. Neither the name of the Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +***************************************************************************/ + +#ifndef _VIRTCHNL_H_ +#define _VIRTCHNL_H_ + +/* Description: + * This header file describes the VF-PF communication protocol used + * by the drivers for all devices starting from our 40G product line + * + * Admin queue buffer usage: + * desc->opcode is always aqc_opc_send_msg_to_pf + * flags, retval, datalen, and data addr are all used normally. + * The Firmware copies the cookie fields when sending messages between the + * PF and VF, but uses all other fields internally. Due to this limitation, + * we must send all messages as "indirect", i.e. using an external buffer. + * + * All the VSI indexes are relative to the VF. Each VF can have maximum of + * three VSIs. All the queue indexes are relative to the VSI. Each VF can + * have a maximum of sixteen queues for all of its VSIs. + * + * The PF is required to return a status code in v_retval for all messages + * except RESET_VF, which does not require any response. The return value + * is of status_code type, defined in the shared type.h. + * + * In general, VF driver initialization should roughly follow the order of + * these opcodes. The VF driver must first validate the API version of the + * PF driver, then request a reset, then get resources, then configure + * queues and interrupts. After these operations are complete, the VF + * driver may start its queues, optionally add MAC and VLAN filters, and + * process traffic. + */ + +/* START GENERIC DEFINES + * Need to ensure the following enums and defines hold the same meaning and + * value in current and future projects + */ + +/* Error Codes */ +enum virtchnl_status_code { + VIRTCHNL_STATUS_SUCCESS = 0, + VIRTCHNL_ERR_PARAM = -5, + VIRTCHNL_STATUS_ERR_OPCODE_MISMATCH = -38, + VIRTCHNL_STATUS_ERR_CQP_COMPL_ERROR = -39, + VIRTCHNL_STATUS_ERR_INVALID_VF_ID = -40, + VIRTCHNL_STATUS_NOT_SUPPORTED = -64, +}; + +#define VIRTCHNL_LINK_SPEED_100MB_SHIFT 0x1 +#define VIRTCHNL_LINK_SPEED_1000MB_SHIFT 0x2 +#define VIRTCHNL_LINK_SPEED_10GB_SHIFT 0x3 +#define VIRTCHNL_LINK_SPEED_40GB_SHIFT 0x4 +#define VIRTCHNL_LINK_SPEED_20GB_SHIFT 0x5 +#define VIRTCHNL_LINK_SPEED_25GB_SHIFT 0x6 + +enum virtchnl_link_speed { + VIRTCHNL_LINK_SPEED_UNKNOWN = 0, + VIRTCHNL_LINK_SPEED_100MB = BIT(VIRTCHNL_LINK_SPEED_100MB_SHIFT), + VIRTCHNL_LINK_SPEED_1GB = BIT(VIRTCHNL_LINK_SPEED_1000MB_SHIFT), + VIRTCHNL_LINK_SPEED_10GB = BIT(VIRTCHNL_LINK_SPEED_10GB_SHIFT), + VIRTCHNL_LINK_SPEED_40GB = BIT(VIRTCHNL_LINK_SPEED_40GB_SHIFT), + VIRTCHNL_LINK_SPEED_20GB = BIT(VIRTCHNL_LINK_SPEED_20GB_SHIFT), + VIRTCHNL_LINK_SPEED_25GB = BIT(VIRTCHNL_LINK_SPEED_25GB_SHIFT), +}; + +/* for hsplit_0 field of Rx HMC context */ +/* deprecated with AVF 1.0 */ +enum virtchnl_rx_hsplit { + VIRTCHNL_RX_HSPLIT_NO_SPLIT = 0, + VIRTCHNL_RX_HSPLIT_SPLIT_L2 = 1, + VIRTCHNL_RX_HSPLIT_SPLIT_IP = 2, + VIRTCHNL_RX_HSPLIT_SPLIT_TCP_UDP = 4, + VIRTCHNL_RX_HSPLIT_SPLIT_SCTP = 8, +}; + +#define VIRTCHNL_ETH_LENGTH_OF_ADDRESS 6 +/* END GENERIC DEFINES */ + +/* Opcodes for VF-PF communication. These are placed in the v_opcode field + * of the virtchnl_msg structure. + */ +enum virtchnl_ops { +/* The PF sends status change events to VFs using + * the VIRTCHNL_OP_EVENT opcode. + * VFs send requests to the PF using the other ops. + * Use of "advanced opcode" features must be negotiated as part of capabilities + * exchange and are not considered part of base mode feature set. + */ + VIRTCHNL_OP_UNKNOWN = 0, + VIRTCHNL_OP_VERSION = 1, /* must ALWAYS be 1 */ + VIRTCHNL_OP_RESET_VF = 2, + VIRTCHNL_OP_GET_VF_RESOURCES = 3, + VIRTCHNL_OP_CONFIG_TX_QUEUE = 4, + VIRTCHNL_OP_CONFIG_RX_QUEUE = 5, + VIRTCHNL_OP_CONFIG_VSI_QUEUES = 6, + VIRTCHNL_OP_CONFIG_IRQ_MAP = 7, + VIRTCHNL_OP_ENABLE_QUEUES = 8, + VIRTCHNL_OP_DISABLE_QUEUES = 9, + VIRTCHNL_OP_ADD_ETH_ADDR = 10, + VIRTCHNL_OP_DEL_ETH_ADDR = 11, + VIRTCHNL_OP_ADD_VLAN = 12, + VIRTCHNL_OP_DEL_VLAN = 13, + VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE = 14, + VIRTCHNL_OP_GET_STATS = 15, + VIRTCHNL_OP_RSVD = 16, + VIRTCHNL_OP_EVENT = 17, /* must ALWAYS be 17 */ +#ifdef VIRTCHNL_SOL_VF_SUPPORT + VIRTCHNL_OP_GET_ADDNL_SOL_CONFIG = 19, +#endif +#ifdef VIRTCHNL_IWARP + VIRTCHNL_OP_IWARP = 20, /* advanced opcode */ + VIRTCHNL_OP_CONFIG_IWARP_IRQ_MAP = 21, /* advanced opcode */ + VIRTCHNL_OP_RELEASE_IWARP_IRQ_MAP = 22, /* advanced opcode */ +#endif + VIRTCHNL_OP_CONFIG_RSS_KEY = 23, + VIRTCHNL_OP_CONFIG_RSS_LUT = 24, + VIRTCHNL_OP_GET_RSS_HENA_CAPS = 25, + VIRTCHNL_OP_SET_RSS_HENA = 26, + VIRTCHNL_OP_ENABLE_VLAN_STRIPPING = 27, + VIRTCHNL_OP_DISABLE_VLAN_STRIPPING = 28, + VIRTCHNL_OP_REQUEST_QUEUES = 29, + +}; + +/* This macro is used to generate a compilation error if a structure + * is not exactly the correct length. It gives a divide by zero error if the + * structure is not of the correct size, otherwise it creates an enum that is + * never used. + */ +#define VIRTCHNL_CHECK_STRUCT_LEN(n, X) enum virtchnl_static_assert_enum_##X \ + {virtchnl_static_assert_##X = (n) / ((sizeof(struct X) == (n)) ? 1 : 0)} + +/* Virtual channel message descriptor. This overlays the admin queue + * descriptor. All other data is passed in external buffers. + */ + +struct virtchnl_msg { + u8 pad[8]; /* AQ flags/opcode/len/retval fields */ + enum virtchnl_ops v_opcode; /* avoid confusion with desc->opcode */ + enum virtchnl_status_code v_retval; /* ditto for desc->retval */ + u32 vfid; /* used by PF when sending to VF */ +}; + +VIRTCHNL_CHECK_STRUCT_LEN(20, virtchnl_msg); + +/* Message descriptions and data structures.*/ + +/* VIRTCHNL_OP_VERSION + * VF posts its version number to the PF. PF responds with its version number + * in the same format, along with a return code. + * Reply from PF has its major/minor versions also in param0 and param1. + * If there is a major version mismatch, then the VF cannot operate. + * If there is a minor version mismatch, then the VF can operate but should + * add a warning to the system log. + * + * This enum element MUST always be specified as == 1, regardless of other + * changes in the API. The PF must always respond to this message without + * error regardless of version mismatch. + */ +#define VIRTCHNL_VERSION_MAJOR 1 +#define VIRTCHNL_VERSION_MINOR 1 +#define VIRTCHNL_VERSION_MINOR_NO_VF_CAPS 0 + +struct virtchnl_version_info { + u32 major; + u32 minor; +}; + +VIRTCHNL_CHECK_STRUCT_LEN(8, virtchnl_version_info); + +#define VF_IS_V10(_v) (((_v)->major == 1) && ((_v)->minor == 0)) +#define VF_IS_V11(_ver) (((_ver)->major == 1) && ((_ver)->minor == 1)) + +/* VIRTCHNL_OP_RESET_VF + * VF sends this request to PF with no parameters + * PF does NOT respond! VF driver must delay then poll VFGEN_RSTAT register + * until reset completion is indicated. The admin queue must be reinitialized + * after this operation. + * + * When reset is complete, PF must ensure that all queues in all VSIs associated + * with the VF are stopped, all queue configurations in the HMC are set to 0, + * and all MAC and VLAN filters (except the default MAC address) on all VSIs + * are cleared. + */ + +/* VSI types that use VIRTCHNL interface for VF-PF communication. VSI_SRIOV + * vsi_type should always be 6 for backward compatibility. Add other fields + * as needed. + */ +enum virtchnl_vsi_type { + VIRTCHNL_VSI_TYPE_INVALID = 0, + VIRTCHNL_VSI_SRIOV = 6, +}; + +/* VIRTCHNL_OP_GET_VF_RESOURCES + * Version 1.0 VF sends this request to PF with no parameters + * Version 1.1 VF sends this request to PF with u32 bitmap of its capabilities + * PF responds with an indirect message containing + * virtchnl_vf_resource and one or more + * virtchnl_vsi_resource structures. + */ + +struct virtchnl_vsi_resource { + u16 vsi_id; + u16 num_queue_pairs; + enum virtchnl_vsi_type vsi_type; + u16 qset_handle; + u8 default_mac_addr[VIRTCHNL_ETH_LENGTH_OF_ADDRESS]; +}; + +VIRTCHNL_CHECK_STRUCT_LEN(16, virtchnl_vsi_resource); + +/* VF offload flags + * VIRTCHNL_VF_OFFLOAD_L2 flag is inclusive of base mode L2 offloads including + * TX/RX Checksum offloading and TSO for non-tunnelled packets. + */ +#define VIRTCHNL_VF_OFFLOAD_L2 0x00000001 +#define VIRTCHNL_VF_OFFLOAD_IWARP 0x00000002 +#define VIRTCHNL_VF_OFFLOAD_RSVD 0x00000004 +#define VIRTCHNL_VF_OFFLOAD_RSS_AQ 0x00000008 +#define VIRTCHNL_VF_OFFLOAD_RSS_REG 0x00000010 +#define VIRTCHNL_VF_OFFLOAD_WB_ON_ITR 0x00000020 +#define VIRTCHNL_VF_OFFLOAD_REQ_QUEUES 0x00000040 +#define VIRTCHNL_VF_OFFLOAD_VLAN 0x00010000 +#define VIRTCHNL_VF_OFFLOAD_RX_POLLING 0x00020000 +#define VIRTCHNL_VF_OFFLOAD_RSS_PCTYPE_V2 0x00040000 +#define VIRTCHNL_VF_OFFLOAD_RSS_PF 0X00080000 +#define VIRTCHNL_VF_OFFLOAD_ENCAP 0X00100000 +#define VIRTCHNL_VF_OFFLOAD_ENCAP_CSUM 0X00200000 +#define VIRTCHNL_VF_OFFLOAD_RX_ENCAP_CSUM 0X00400000 + +#define VF_BASE_MODE_OFFLOADS (VIRTCHNL_VF_OFFLOAD_L2 | \ + VIRTCHNL_VF_OFFLOAD_VLAN | \ + VIRTCHNL_VF_OFFLOAD_RSS_PF) + +struct virtchnl_vf_resource { + u16 num_vsis; + u16 num_queue_pairs; + u16 max_vectors; + u16 max_mtu; + + u32 vf_offload_flags; + u32 rss_key_size; + u32 rss_lut_size; + + struct virtchnl_vsi_resource vsi_res[1]; +}; + +VIRTCHNL_CHECK_STRUCT_LEN(36, virtchnl_vf_resource); + +/* VIRTCHNL_OP_CONFIG_TX_QUEUE + * VF sends this message to set up parameters for one TX queue. + * External data buffer contains one instance of virtchnl_txq_info. + * PF configures requested queue and returns a status code. + */ + +/* Tx queue config info */ +struct virtchnl_txq_info { + u16 vsi_id; + u16 queue_id; + u16 ring_len; /* number of descriptors, multiple of 8 */ + u16 headwb_enabled; /* deprecated with AVF 1.0 */ + u64 dma_ring_addr; + u64 dma_headwb_addr; /* deprecated with AVF 1.0 */ +}; + +VIRTCHNL_CHECK_STRUCT_LEN(24, virtchnl_txq_info); + +/* VIRTCHNL_OP_CONFIG_RX_QUEUE + * VF sends this message to set up parameters for one RX queue. + * External data buffer contains one instance of virtchnl_rxq_info. + * PF configures requested queue and returns a status code. + */ + +/* Rx queue config info */ +struct virtchnl_rxq_info { + u16 vsi_id; + u16 queue_id; + u32 ring_len; /* number of descriptors, multiple of 32 */ + u16 hdr_size; + u16 splithdr_enabled; /* deprecated with AVF 1.0 */ + u32 databuffer_size; + u32 max_pkt_size; + u32 pad1; + u64 dma_ring_addr; + enum virtchnl_rx_hsplit rx_split_pos; /* deprecated with AVF 1.0 */ + u32 pad2; +}; + +VIRTCHNL_CHECK_STRUCT_LEN(40, virtchnl_rxq_info); + +/* VIRTCHNL_OP_CONFIG_VSI_QUEUES + * VF sends this message to set parameters for all active TX and RX queues + * associated with the specified VSI. + * PF configures queues and returns status. + * If the number of queues specified is greater than the number of queues + * associated with the VSI, an error is returned and no queues are configured. + */ +struct virtchnl_queue_pair_info { + /* NOTE: vsi_id and queue_id should be identical for both queues. */ + struct virtchnl_txq_info txq; + struct virtchnl_rxq_info rxq; +}; + +VIRTCHNL_CHECK_STRUCT_LEN(64, virtchnl_queue_pair_info); + +struct virtchnl_vsi_queue_config_info { + u16 vsi_id; + u16 num_queue_pairs; + u32 pad; + struct virtchnl_queue_pair_info qpair[1]; +}; + +VIRTCHNL_CHECK_STRUCT_LEN(72, virtchnl_vsi_queue_config_info); + +/* VIRTCHNL_OP_REQUEST_QUEUES + * VF sends this message to request the PF to allocate additional queues to + * this VF. Each VF gets a guaranteed number of queues on init but asking for + * additional queues must be negotiated. This is a best effort request as it + * is possible the PF does not have enough queues left to support the request. + * If the PF cannot support the number requested it will respond with the + * maximum number it is able to support; otherwise it will respond with the + * number requested. + */ + +/* VF resource request */ +struct virtchnl_vf_res_request { + u16 num_queue_pairs; +}; + +/* VIRTCHNL_OP_CONFIG_IRQ_MAP + * VF uses this message to map vectors to queues. + * The rxq_map and txq_map fields are bitmaps used to indicate which queues + * are to be associated with the specified vector. + * The "other" causes are always mapped to vector 0. + * PF configures interrupt mapping and returns status. + */ +struct virtchnl_vector_map { + u16 vsi_id; + u16 vector_id; + u16 rxq_map; + u16 txq_map; + u16 rxitr_idx; + u16 txitr_idx; +}; + +VIRTCHNL_CHECK_STRUCT_LEN(12, virtchnl_vector_map); + +struct virtchnl_irq_map_info { + u16 num_vectors; + struct virtchnl_vector_map vecmap[1]; +}; + +VIRTCHNL_CHECK_STRUCT_LEN(14, virtchnl_irq_map_info); + +/* VIRTCHNL_OP_ENABLE_QUEUES + * VIRTCHNL_OP_DISABLE_QUEUES + * VF sends these message to enable or disable TX/RX queue pairs. + * The queues fields are bitmaps indicating which queues to act upon. + * (Currently, we only support 16 queues per VF, but we make the field + * u32 to allow for expansion.) + * PF performs requested action and returns status. + */ +struct virtchnl_queue_select { + u16 vsi_id; + u16 pad; + u32 rx_queues; + u32 tx_queues; +}; + +VIRTCHNL_CHECK_STRUCT_LEN(12, virtchnl_queue_select); + +/* VIRTCHNL_OP_ADD_ETH_ADDR + * VF sends this message in order to add one or more unicast or multicast + * address filters for the specified VSI. + * PF adds the filters and returns status. + */ + +/* VIRTCHNL_OP_DEL_ETH_ADDR + * VF sends this message in order to remove one or more unicast or multicast + * filters for the specified VSI. + * PF removes the filters and returns status. + */ + +struct virtchnl_ether_addr { + u8 addr[VIRTCHNL_ETH_LENGTH_OF_ADDRESS]; + u8 pad[2]; +}; + +VIRTCHNL_CHECK_STRUCT_LEN(8, virtchnl_ether_addr); + +struct virtchnl_ether_addr_list { + u16 vsi_id; + u16 num_elements; + struct virtchnl_ether_addr list[1]; +}; + +VIRTCHNL_CHECK_STRUCT_LEN(12, virtchnl_ether_addr_list); + +#ifdef VIRTCHNL_SOL_VF_SUPPORT +/* VIRTCHNL_OP_GET_ADDNL_SOL_CONFIG + * VF sends this message to get the default MTU and list of additional ethernet + * addresses it is allowed to use. + * PF responds with an indirect message containing + * virtchnl_addnl_solaris_config with zero or more + * virtchnl_ether_addr structures. + * + * It is expected that this operation will only ever be needed for Solaris VFs + * running under a Solaris PF. + */ +struct virtchnl_addnl_solaris_config { + u16 default_mtu; + struct virtchnl_ether_addr_list al; +}; + +#endif +/* VIRTCHNL_OP_ADD_VLAN + * VF sends this message to add one or more VLAN tag filters for receives. + * PF adds the filters and returns status. + * If a port VLAN is configured by the PF, this operation will return an + * error to the VF. + */ + +/* VIRTCHNL_OP_DEL_VLAN + * VF sends this message to remove one or more VLAN tag filters for receives. + * PF removes the filters and returns status. + * If a port VLAN is configured by the PF, this operation will return an + * error to the VF. + */ + +struct virtchnl_vlan_filter_list { + u16 vsi_id; + u16 num_elements; + u16 vlan_id[1]; +}; + +VIRTCHNL_CHECK_STRUCT_LEN(6, virtchnl_vlan_filter_list); + +/* VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE + * VF sends VSI id and flags. + * PF returns status code in retval. + * Note: we assume that broadcast accept mode is always enabled. + */ +struct virtchnl_promisc_info { + u16 vsi_id; + u16 flags; +}; + +VIRTCHNL_CHECK_STRUCT_LEN(4, virtchnl_promisc_info); + +#define FLAG_VF_UNICAST_PROMISC 0x00000001 +#define FLAG_VF_MULTICAST_PROMISC 0x00000002 + +/* VIRTCHNL_OP_GET_STATS + * VF sends this message to request stats for the selected VSI. VF uses + * the virtchnl_queue_select struct to specify the VSI. The queue_id + * field is ignored by the PF. + * + * PF replies with struct eth_stats in an external buffer. + */ + +/* VIRTCHNL_OP_CONFIG_RSS_KEY + * VIRTCHNL_OP_CONFIG_RSS_LUT + * VF sends these messages to configure RSS. Only supported if both PF + * and VF drivers set the VIRTCHNL_VF_OFFLOAD_RSS_PF bit during + * configuration negotiation. If this is the case, then the RSS fields in + * the VF resource struct are valid. + * Both the key and LUT are initialized to 0 by the PF, meaning that + * RSS is effectively disabled until set up by the VF. + */ +struct virtchnl_rss_key { + u16 vsi_id; + u16 key_len; + u8 key[1]; /* RSS hash key, packed bytes */ +}; + +VIRTCHNL_CHECK_STRUCT_LEN(6, virtchnl_rss_key); + +struct virtchnl_rss_lut { + u16 vsi_id; + u16 lut_entries; + u8 lut[1]; /* RSS lookup table*/ +}; + +VIRTCHNL_CHECK_STRUCT_LEN(6, virtchnl_rss_lut); + +/* VIRTCHNL_OP_GET_RSS_HENA_CAPS + * VIRTCHNL_OP_SET_RSS_HENA + * VF sends these messages to get and set the hash filter enable bits for RSS. + * By default, the PF sets these to all possible traffic types that the + * hardware supports. The VF can query this value if it wants to change the + * traffic types that are hashed by the hardware. + */ +struct virtchnl_rss_hena { + u64 hena; +}; + +VIRTCHNL_CHECK_STRUCT_LEN(8, virtchnl_rss_hena); + +/* VIRTCHNL_OP_EVENT + * PF sends this message to inform the VF driver of events that may affect it. + * No direct response is expected from the VF, though it may generate other + * messages in response to this one. + */ +enum virtchnl_event_codes { + VIRTCHNL_EVENT_UNKNOWN = 0, + VIRTCHNL_EVENT_LINK_CHANGE, + VIRTCHNL_EVENT_RESET_IMPENDING, + VIRTCHNL_EVENT_PF_DRIVER_CLOSE, +}; + +#define PF_EVENT_SEVERITY_INFO 0 +#define PF_EVENT_SEVERITY_ATTENTION 1 +#define PF_EVENT_SEVERITY_ACTION_REQUIRED 2 +#define PF_EVENT_SEVERITY_CERTAIN_DOOM 255 + +struct virtchnl_pf_event { + enum virtchnl_event_codes event; + union { + struct { + enum virtchnl_link_speed link_speed; + bool link_status; + } link_event; + } event_data; + + int severity; +}; + +VIRTCHNL_CHECK_STRUCT_LEN(16, virtchnl_pf_event); + +#ifdef VIRTCHNL_IWARP + +/* VIRTCHNL_OP_CONFIG_IWARP_IRQ_MAP + * VF uses this message to request PF to map IWARP vectors to IWARP queues. + * The request for this originates from the VF IWARP driver through + * a client interface between VF LAN and VF IWARP driver. + * A vector could have an AEQ and CEQ attached to it although + * there is a single AEQ per VF IWARP instance in which case + * most vectors will have an INVALID_IDX for aeq and valid idx for ceq. + * There will never be a case where there will be multiple CEQs attached + * to a single vector. + * PF configures interrupt mapping and returns status. + */ + +/* HW does not define a type value for AEQ; only for RX/TX and CEQ. + * In order for us to keep the interface simple, SW will define a + * unique type value for AEQ. + */ +#define QUEUE_TYPE_PE_AEQ 0x80 +#define QUEUE_INVALID_IDX 0xFFFF + +struct virtchnl_iwarp_qv_info { + u32 v_idx; /* msix_vector */ + u16 ceq_idx; + u16 aeq_idx; + u8 itr_idx; +}; + +VIRTCHNL_CHECK_STRUCT_LEN(12, virtchnl_iwarp_qv_info); + +struct virtchnl_iwarp_qvlist_info { + u32 num_vectors; + struct virtchnl_iwarp_qv_info qv_info[1]; +}; + +VIRTCHNL_CHECK_STRUCT_LEN(16, virtchnl_iwarp_qvlist_info); + +#endif + +/* VF reset states - these are written into the RSTAT register: + * VFGEN_RSTAT on the VF + * When the PF initiates a reset, it writes 0 + * When the reset is complete, it writes 1 + * When the PF detects that the VF has recovered, it writes 2 + * VF checks this register periodically to determine if a reset has occurred, + * then polls it to know when the reset is complete. + * If either the PF or VF reads the register while the hardware + * is in a reset state, it will return DEADBEEF, which, when masked + * will result in 3. + */ +enum virtchnl_vfr_states { + VIRTCHNL_VFR_INPROGRESS = 0, + VIRTCHNL_VFR_COMPLETED, + VIRTCHNL_VFR_VFACTIVE, +}; + +/** + * virtchnl_vc_validate_vf_msg + * @ver: Virtchnl version info + * @v_opcode: Opcode for the message + * @msg: pointer to the msg buffer + * @msglen: msg length + * + * validate msg format against struct for each opcode + */ +static inline int +virtchnl_vc_validate_vf_msg(struct virtchnl_version_info *ver, u32 v_opcode, + u8 *msg, u16 msglen) +{ + bool err_msg_format = false; + int valid_len = 0; + + /* Validate message length. */ + switch (v_opcode) { + case VIRTCHNL_OP_VERSION: + valid_len = sizeof(struct virtchnl_version_info); + break; + case VIRTCHNL_OP_RESET_VF: + break; + case VIRTCHNL_OP_GET_VF_RESOURCES: + if (VF_IS_V11(ver)) + valid_len = sizeof(u32); + break; + case VIRTCHNL_OP_CONFIG_TX_QUEUE: + valid_len = sizeof(struct virtchnl_txq_info); + break; + case VIRTCHNL_OP_CONFIG_RX_QUEUE: + valid_len = sizeof(struct virtchnl_rxq_info); + break; + case VIRTCHNL_OP_CONFIG_VSI_QUEUES: + valid_len = sizeof(struct virtchnl_vsi_queue_config_info); + if (msglen >= valid_len) { + struct virtchnl_vsi_queue_config_info *vqc = + (struct virtchnl_vsi_queue_config_info *)msg; + valid_len += (vqc->num_queue_pairs * + sizeof(struct + virtchnl_queue_pair_info)); + if (vqc->num_queue_pairs == 0) + err_msg_format = true; + } + break; + case VIRTCHNL_OP_CONFIG_IRQ_MAP: + valid_len = sizeof(struct virtchnl_irq_map_info); + if (msglen >= valid_len) { + struct virtchnl_irq_map_info *vimi = + (struct virtchnl_irq_map_info *)msg; + valid_len += (vimi->num_vectors * + sizeof(struct virtchnl_vector_map)); + if (vimi->num_vectors == 0) + err_msg_format = true; + } + break; + case VIRTCHNL_OP_ENABLE_QUEUES: + case VIRTCHNL_OP_DISABLE_QUEUES: + valid_len = sizeof(struct virtchnl_queue_select); + break; + case VIRTCHNL_OP_ADD_ETH_ADDR: + case VIRTCHNL_OP_DEL_ETH_ADDR: + valid_len = sizeof(struct virtchnl_ether_addr_list); + if (msglen >= valid_len) { + struct virtchnl_ether_addr_list *veal = + (struct virtchnl_ether_addr_list *)msg; + valid_len += veal->num_elements * + sizeof(struct virtchnl_ether_addr); + if (veal->num_elements == 0) + err_msg_format = true; + } + break; + case VIRTCHNL_OP_ADD_VLAN: + case VIRTCHNL_OP_DEL_VLAN: + valid_len = sizeof(struct virtchnl_vlan_filter_list); + if (msglen >= valid_len) { + struct virtchnl_vlan_filter_list *vfl = + (struct virtchnl_vlan_filter_list *)msg; + valid_len += vfl->num_elements * sizeof(u16); + if (vfl->num_elements == 0) + err_msg_format = true; + } + break; + case VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE: + valid_len = sizeof(struct virtchnl_promisc_info); + break; + case VIRTCHNL_OP_GET_STATS: + valid_len = sizeof(struct virtchnl_queue_select); + break; +#ifdef VIRTCHNL_IWARP + case VIRTCHNL_OP_IWARP: + /* These messages are opaque to us and will be validated in + * the RDMA client code. We just need to check for nonzero + * length. The firmware will enforce max length restrictions. + */ + if (msglen) + valid_len = msglen; + else + err_msg_format = true; + break; + case VIRTCHNL_OP_RELEASE_IWARP_IRQ_MAP: + break; + case VIRTCHNL_OP_CONFIG_IWARP_IRQ_MAP: + valid_len = sizeof(struct virtchnl_iwarp_qvlist_info); + if (msglen >= valid_len) { + struct virtchnl_iwarp_qvlist_info *qv = + (struct virtchnl_iwarp_qvlist_info *)msg; + if (qv->num_vectors == 0) { + err_msg_format = true; + break; + } + valid_len += ((qv->num_vectors - 1) * + sizeof(struct virtchnl_iwarp_qv_info)); + } + break; +#endif + case VIRTCHNL_OP_CONFIG_RSS_KEY: + valid_len = sizeof(struct virtchnl_rss_key); + if (msglen >= valid_len) { + struct virtchnl_rss_key *vrk = + (struct virtchnl_rss_key *)msg; + valid_len += vrk->key_len - 1; + } + break; + case VIRTCHNL_OP_CONFIG_RSS_LUT: + valid_len = sizeof(struct virtchnl_rss_lut); + if (msglen >= valid_len) { + struct virtchnl_rss_lut *vrl = + (struct virtchnl_rss_lut *)msg; + valid_len += vrl->lut_entries - 1; + } + break; + case VIRTCHNL_OP_GET_RSS_HENA_CAPS: + break; + case VIRTCHNL_OP_SET_RSS_HENA: + valid_len = sizeof(struct virtchnl_rss_hena); + break; + case VIRTCHNL_OP_ENABLE_VLAN_STRIPPING: + case VIRTCHNL_OP_DISABLE_VLAN_STRIPPING: + break; + case VIRTCHNL_OP_REQUEST_QUEUES: + valid_len = sizeof(struct virtchnl_vf_res_request); + break; + /* These are always errors coming from the VF. */ + case VIRTCHNL_OP_EVENT: + case VIRTCHNL_OP_UNKNOWN: + default: + return VIRTCHNL_ERR_PARAM; + } + /* few more checks */ + if ((valid_len != msglen) || (err_msg_format)) + return VIRTCHNL_STATUS_ERR_OPCODE_MISMATCH; + + return 0; +} +#endif /* _VIRTCHNL_H_ */ diff --git a/dpdk/drivers/net/i40e/i40e_ethdev.c b/dpdk/drivers/net/i40e/i40e_ethdev.c index 13068cc4..290ef249 100644 --- a/dpdk/drivers/net/i40e/i40e_ethdev.c +++ b/dpdk/drivers/net/i40e/i40e_ethdev.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. + * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -40,10 +40,13 @@ #include #include +#include #include #include +#include #include #include +#include #include #include #include @@ -51,6 +54,7 @@ #include #include #include +#include #include "i40e_logs.h" #include "base/i40e_prototype.h" @@ -62,6 +66,7 @@ #include "i40e_rxtx.h" #include "i40e_pf.h" #include "i40e_regs.h" +#include "rte_pmd_i40e.h" #define ETH_I40E_FLOATING_VEB_ARG "enable_floating_veb" #define ETH_I40E_FLOATING_VEB_LIST_ARG "floating_veb_list" @@ -71,7 +76,7 @@ /* Maximun number of capability elements */ #define I40E_MAX_CAP_ELE_NUM 128 -/* Wait count and inteval */ +/* Wait count and interval */ #define I40E_CHK_Q_ENA_COUNT 1000 #define I40E_CHK_Q_ENA_INTERVAL_US 1000 @@ -83,12 +88,6 @@ /* Flow control default timer */ #define I40E_DEFAULT_PAUSE_TIME 0xFFFFU -/* Flow control default high water */ -#define I40E_DEFAULT_HIGH_WATER (0x1C40/1024) - -/* Flow control default low water */ -#define I40E_DEFAULT_LOW_WATER (0x1A40/1024) - /* Flow control enable fwd bit */ #define I40E_PRTMAC_FWD_CTRL 0x00000001 @@ -98,6 +97,12 @@ /* Kilobytes shift */ #define I40E_KILOSHIFT 10 +/* Flow control default high water */ +#define I40E_DEFAULT_HIGH_WATER (0xF2000 >> I40E_KILOSHIFT) + +/* Flow control default low water */ +#define I40E_DEFAULT_LOW_WATER (0xF2000 >> I40E_KILOSHIFT) + /* Receive Average Packet Size in Byte*/ #define I40E_PACKET_AVERAGE_SIZE 128 @@ -134,64 +139,6 @@ #define I40E_PRTTSYN_TSYNTYPE 0x0e000000 #define I40E_CYCLECOUNTER_MASK 0xffffffffffffffffULL -#define I40E_MAX_PERCENT 100 -#define I40E_DEFAULT_DCB_APP_NUM 1 -#define I40E_DEFAULT_DCB_APP_PRIO 3 - -#define I40E_INSET_NONE 0x00000000000000000ULL - -/* bit0 ~ bit 7 */ -#define I40E_INSET_DMAC 0x0000000000000001ULL -#define I40E_INSET_SMAC 0x0000000000000002ULL -#define I40E_INSET_VLAN_OUTER 0x0000000000000004ULL -#define I40E_INSET_VLAN_INNER 0x0000000000000008ULL -#define I40E_INSET_VLAN_TUNNEL 0x0000000000000010ULL - -/* bit 8 ~ bit 15 */ -#define I40E_INSET_IPV4_SRC 0x0000000000000100ULL -#define I40E_INSET_IPV4_DST 0x0000000000000200ULL -#define I40E_INSET_IPV6_SRC 0x0000000000000400ULL -#define I40E_INSET_IPV6_DST 0x0000000000000800ULL -#define I40E_INSET_SRC_PORT 0x0000000000001000ULL -#define I40E_INSET_DST_PORT 0x0000000000002000ULL -#define I40E_INSET_SCTP_VT 0x0000000000004000ULL - -/* bit 16 ~ bit 31 */ -#define I40E_INSET_IPV4_TOS 0x0000000000010000ULL -#define I40E_INSET_IPV4_PROTO 0x0000000000020000ULL -#define I40E_INSET_IPV4_TTL 0x0000000000040000ULL -#define I40E_INSET_IPV6_TC 0x0000000000080000ULL -#define I40E_INSET_IPV6_FLOW 0x0000000000100000ULL -#define I40E_INSET_IPV6_NEXT_HDR 0x0000000000200000ULL -#define I40E_INSET_IPV6_HOP_LIMIT 0x0000000000400000ULL -#define I40E_INSET_TCP_FLAGS 0x0000000000800000ULL - -/* bit 32 ~ bit 47, tunnel fields */ -#define I40E_INSET_TUNNEL_IPV4_DST 0x0000000100000000ULL -#define I40E_INSET_TUNNEL_IPV6_DST 0x0000000200000000ULL -#define I40E_INSET_TUNNEL_DMAC 0x0000000400000000ULL -#define I40E_INSET_TUNNEL_SRC_PORT 0x0000000800000000ULL -#define I40E_INSET_TUNNEL_DST_PORT 0x0000001000000000ULL -#define I40E_INSET_TUNNEL_ID 0x0000002000000000ULL - -/* bit 48 ~ bit 55 */ -#define I40E_INSET_LAST_ETHER_TYPE 0x0001000000000000ULL - -/* bit 56 ~ bit 63, Flex Payload */ -#define I40E_INSET_FLEX_PAYLOAD_W1 0x0100000000000000ULL -#define I40E_INSET_FLEX_PAYLOAD_W2 0x0200000000000000ULL -#define I40E_INSET_FLEX_PAYLOAD_W3 0x0400000000000000ULL -#define I40E_INSET_FLEX_PAYLOAD_W4 0x0800000000000000ULL -#define I40E_INSET_FLEX_PAYLOAD_W5 0x1000000000000000ULL -#define I40E_INSET_FLEX_PAYLOAD_W6 0x2000000000000000ULL -#define I40E_INSET_FLEX_PAYLOAD_W7 0x4000000000000000ULL -#define I40E_INSET_FLEX_PAYLOAD_W8 0x8000000000000000ULL -#define I40E_INSET_FLEX_PAYLOAD \ - (I40E_INSET_FLEX_PAYLOAD_W1 | I40E_INSET_FLEX_PAYLOAD_W2 | \ - I40E_INSET_FLEX_PAYLOAD_W3 | I40E_INSET_FLEX_PAYLOAD_W4 | \ - I40E_INSET_FLEX_PAYLOAD_W5 | I40E_INSET_FLEX_PAYLOAD_W6 | \ - I40E_INSET_FLEX_PAYLOAD_W7 | I40E_INSET_FLEX_PAYLOAD_W8) - /** * Below are values for writing un-exposed registers suggested * by silicon experts @@ -284,11 +231,6 @@ #define I40E_INSET_IPV6_HOP_LIMIT_MASK 0x000CFF00UL #define I40E_INSET_IPV6_NEXT_HDR_MASK 0x000C00FFUL -#define I40E_GL_SWT_L2TAGCTRL(_i) (0x001C0A70 + ((_i) * 4)) -#define I40E_GL_SWT_L2TAGCTRL_ETHERTYPE_SHIFT 16 -#define I40E_GL_SWT_L2TAGCTRL_ETHERTYPE_MASK \ - I40E_MASK(0xFFFF, I40E_GL_SWT_L2TAGCTRL_ETHERTYPE_SHIFT) - /* PCI offset for querying capability */ #define PCI_DEV_CAP_REG 0xA4 /* PCI offset for enabling/disabling Extended Tag */ @@ -306,13 +248,14 @@ static int i40e_dev_configure(struct rte_eth_dev *dev); static int i40e_dev_start(struct rte_eth_dev *dev); static void i40e_dev_stop(struct rte_eth_dev *dev); static void i40e_dev_close(struct rte_eth_dev *dev); +static int i40e_dev_reset(struct rte_eth_dev *dev); static void i40e_dev_promiscuous_enable(struct rte_eth_dev *dev); static void i40e_dev_promiscuous_disable(struct rte_eth_dev *dev); static void i40e_dev_allmulticast_enable(struct rte_eth_dev *dev); static void i40e_dev_allmulticast_disable(struct rte_eth_dev *dev); static int i40e_dev_set_link_up(struct rte_eth_dev *dev); static int i40e_dev_set_link_down(struct rte_eth_dev *dev); -static void i40e_dev_stats_get(struct rte_eth_dev *dev, +static int i40e_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats); static int i40e_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, unsigned n); @@ -324,6 +267,8 @@ static int i40e_dev_queue_stats_mapping_set(struct rte_eth_dev *dev, uint16_t queue_id, uint8_t stat_idx, uint8_t is_rx); +static int i40e_fw_version_get(struct rte_eth_dev *dev, + char *fw_version, size_t fw_size); static void i40e_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info); static int i40e_vlan_filter_set(struct rte_eth_dev *dev, @@ -332,7 +277,7 @@ static int i40e_vlan_filter_set(struct rte_eth_dev *dev, static int i40e_vlan_tpid_set(struct rte_eth_dev *dev, enum rte_vlan_type vlan_type, uint16_t tpid); -static void i40e_vlan_offload_set(struct rte_eth_dev *dev, int mask); +static int i40e_vlan_offload_set(struct rte_eth_dev *dev, int mask); static void i40e_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on); @@ -345,10 +290,10 @@ static int i40e_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf); static int i40e_priority_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_pfc_conf *pfc_conf); -static void i40e_macaddr_add(struct rte_eth_dev *dev, - struct ether_addr *mac_addr, - uint32_t index, - uint32_t pool); +static int i40e_macaddr_add(struct rte_eth_dev *dev, + struct ether_addr *mac_addr, + uint32_t index, + uint32_t pool); static void i40e_macaddr_remove(struct rte_eth_dev *dev, uint32_t index); static int i40e_dev_rss_reta_update(struct rte_eth_dev *dev, struct rte_eth_rss_reta_entry64 *reta_conf, @@ -362,7 +307,6 @@ static int i40e_pf_parameter_init(struct rte_eth_dev *dev); static int i40e_pf_setup(struct i40e_pf *pf); static int i40e_dev_rxtx_init(struct i40e_pf *pf); static int i40e_vmdq_setup(struct rte_eth_dev *dev); -static int i40e_dcb_init_configure(struct rte_eth_dev *dev, bool sw_dcb); static int i40e_dcb_setup(struct rte_eth_dev *dev); static void i40e_stat_update_32(struct i40e_hw *hw, uint32_t reg, bool offset_loaded, uint64_t *offset, uint64_t *stat); @@ -373,8 +317,7 @@ static void i40e_stat_update_48(struct i40e_hw *hw, uint64_t *offset, uint64_t *stat); static void i40e_pf_config_irq0(struct i40e_hw *hw, bool no_queue); -static void i40e_dev_interrupt_handler( - __rte_unused struct rte_intr_handle *handle, void *param); +static void i40e_dev_interrupt_handler(void *param); static int i40e_res_pool_init(struct i40e_res_pool_info *pool, uint32_t base, uint32_t num); static void i40e_res_pool_destroy(struct i40e_res_pool_info *pool); @@ -388,10 +331,6 @@ static struct i40e_veb *i40e_veb_setup(struct i40e_pf *pf, struct i40e_vsi *vsi); static int i40e_pf_config_mq_rx(struct i40e_pf *pf); static int i40e_vsi_config_double_vlan(struct i40e_vsi *vsi, int on); -static inline int i40e_find_all_vlan_for_mac(struct i40e_vsi *vsi, - struct i40e_macvlan_filter *mv_f, - int num, - struct ether_addr *addr); static inline int i40e_find_all_mac_for_vlan(struct i40e_vsi *vsi, struct i40e_macvlan_filter *mv_f, int num, @@ -406,9 +345,6 @@ static int i40e_dev_udp_tunnel_port_add(struct rte_eth_dev *dev, static int i40e_dev_udp_tunnel_port_del(struct rte_eth_dev *dev, struct rte_eth_udp_tunnel *udp_tunnel); static void i40e_filter_input_set_init(struct i40e_pf *pf); -static int i40e_ethertype_filter_set(struct i40e_pf *pf, - struct rte_eth_ethertype_filter *filter, - bool add); static int i40e_ethertype_filter_handle(struct rte_eth_dev *dev, enum rte_filter_op filter_op, void *arg); @@ -418,9 +354,16 @@ static int i40e_dev_filter_ctrl(struct rte_eth_dev *dev, void *arg); static int i40e_dev_get_dcb_info(struct rte_eth_dev *dev, struct rte_eth_dcb_info *dcb_info); +static int i40e_dev_sync_phy_type(struct i40e_hw *hw); static void i40e_configure_registers(struct i40e_hw *hw); static void i40e_hw_init(struct rte_eth_dev *dev); static int i40e_config_qinq(struct i40e_hw *hw, struct i40e_vsi *vsi); +static enum i40e_status_code i40e_aq_del_mirror_rule(struct i40e_hw *hw, + uint16_t seid, + uint16_t rule_type, + uint16_t *entries, + uint16_t count, + uint16_t rule_id); static int i40e_mirror_rule_set(struct rte_eth_dev *dev, struct rte_eth_mirror_conf *mirror_conf, uint8_t sw_id, uint8_t on); @@ -460,6 +403,27 @@ static void i40e_set_default_mac_addr(struct rte_eth_dev *dev, static int i40e_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu); +static int i40e_ethertype_filter_convert( + const struct rte_eth_ethertype_filter *input, + struct i40e_ethertype_filter *filter); +static int i40e_sw_ethertype_filter_insert(struct i40e_pf *pf, + struct i40e_ethertype_filter *filter); + +static int i40e_tunnel_filter_convert( + struct i40e_aqc_add_rm_cloud_filt_elem_ext *cld_filter, + struct i40e_tunnel_filter *tunnel_filter); +static int i40e_sw_tunnel_filter_insert(struct i40e_pf *pf, + struct i40e_tunnel_filter *tunnel_filter); +static int i40e_cloud_filter_qinq_create(struct i40e_pf *pf); + +static void i40e_ethertype_filter_restore(struct i40e_pf *pf); +static void i40e_tunnel_filter_restore(struct i40e_pf *pf); +static void i40e_filter_restore(struct i40e_pf *pf); +static void i40e_notify_all_vfs_link_status(struct rte_eth_dev *dev); + +int i40e_logtype_init; +int i40e_logtype_driver; + static const struct rte_pci_id pci_id_i40e_map[] = { { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_SFP_XL710) }, { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QEMU) }, @@ -481,7 +445,6 @@ static const struct rte_pci_id pci_id_i40e_map[] = { { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_1G_BASE_T_X722) }, { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T_X722) }, { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_SFP_I_X722) }, - { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_I_X722) }, { .vendor_id = 0, /* sentinel */ }, }; @@ -490,6 +453,7 @@ static const struct eth_dev_ops i40e_eth_dev_ops = { .dev_start = i40e_dev_start, .dev_stop = i40e_dev_stop, .dev_close = i40e_dev_close, + .dev_reset = i40e_dev_reset, .promiscuous_enable = i40e_dev_promiscuous_enable, .promiscuous_disable = i40e_dev_promiscuous_disable, .allmulticast_enable = i40e_dev_allmulticast_enable, @@ -503,6 +467,7 @@ static const struct eth_dev_ops i40e_eth_dev_ops = { .stats_reset = i40e_dev_stats_reset, .xstats_reset = i40e_dev_stats_reset, .queue_stats_mapping_set = i40e_dev_queue_stats_mapping_set, + .fw_version_get = i40e_fw_version_get, .dev_infos_get = i40e_dev_info_get, .dev_supported_ptypes_get = i40e_dev_supported_ptypes_get, .vlan_filter_set = i40e_vlan_filter_set, @@ -520,6 +485,8 @@ static const struct eth_dev_ops i40e_eth_dev_ops = { .rx_queue_release = i40e_dev_rx_queue_release, .rx_queue_count = i40e_dev_rx_queue_count, .rx_descriptor_done = i40e_dev_rx_descriptor_done, + .rx_descriptor_status = i40e_dev_rx_descriptor_status, + .tx_descriptor_status = i40e_dev_tx_descriptor_status, .tx_queue_setup = i40e_dev_tx_queue_setup, .tx_queue_release = i40e_dev_tx_queue_release, .dev_led_on = i40e_dev_led_on, @@ -553,6 +520,7 @@ static const struct eth_dev_ops i40e_eth_dev_ops = { .get_eeprom = i40e_get_eeprom, .mac_addr_set = i40e_set_default_mac_addr, .mtu_set = i40e_dev_mtu_set, + .tm_ops_get = i40e_tm_ops_get, }; /* store statistics names and its offset in stats structure */ @@ -668,16 +636,24 @@ static const struct rte_i40e_xstats_name_off rte_i40e_txq_prio_strings[] = { #define I40E_NB_TXQ_PRIO_XSTATS (sizeof(rte_i40e_txq_prio_strings) / \ sizeof(rte_i40e_txq_prio_strings[0])) -static struct eth_driver rte_i40e_pmd = { - .pci_drv = { - .name = "rte_i40e_pmd", - .id_table = pci_id_i40e_map, - .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC | - RTE_PCI_DRV_DETACHABLE, - }, - .eth_dev_init = eth_i40e_dev_init, - .eth_dev_uninit = eth_i40e_dev_uninit, - .dev_private_size = sizeof(struct i40e_adapter), +static int eth_i40e_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, + struct rte_pci_device *pci_dev) +{ + return rte_eth_dev_pci_generic_probe(pci_dev, + sizeof(struct i40e_adapter), eth_i40e_dev_init); +} + +static int eth_i40e_pci_remove(struct rte_pci_device *pci_dev) +{ + return rte_eth_dev_pci_generic_remove(pci_dev, eth_i40e_dev_uninit); +} + +static struct rte_pci_driver rte_i40e_pmd = { + .id_table = pci_id_i40e_map, + .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC | + RTE_PCI_DRV_IOVA_AS_VA, + .probe = eth_i40e_pci_probe, + .remove = eth_i40e_pci_remove, }; static inline int @@ -708,28 +684,18 @@ rte_i40e_dev_atomic_write_link_status(struct rte_eth_dev *dev, return 0; } -/* - * Driver initialization routine. - * Invoked once at EAL init time. - * Register itself as the [Poll Mode] Driver of PCI IXGBE devices. - */ -static int -rte_i40e_pmd_init(const char *name __rte_unused, - const char *params __rte_unused) +static inline void +i40e_write_global_rx_ctl(struct i40e_hw *hw, u32 reg_addr, u32 reg_val) { - PMD_INIT_FUNC_TRACE(); - rte_eth_driver_register(&rte_i40e_pmd); - - return 0; + i40e_write_rx_ctl(hw, reg_addr, reg_val); + PMD_DRV_LOG(DEBUG, "Global register 0x%08x is modified " + "with value 0x%08x", + reg_addr, reg_val); } -static struct rte_driver rte_i40e_driver = { - .type = PMD_PDEV, - .init = rte_i40e_pmd_init, -}; - -PMD_REGISTER_DRIVER(rte_i40e_driver, i40e); -DRIVER_REGISTER_PCI_TABLE(i40e, pci_id_i40e_map); +RTE_PMD_REGISTER_PCI(net_i40e, rte_i40e_pmd); +RTE_PMD_REGISTER_PCI_TABLE(net_i40e, pci_id_i40e_map); +RTE_PMD_REGISTER_KMOD_DEP(net_i40e, "* igb_uio | uio_pci_generic | vfio-pci"); #ifndef I40E_GLQF_ORT #define I40E_GLQF_ORT(_i) (0x00268900 + ((_i) * 4)) @@ -737,29 +703,50 @@ DRIVER_REGISTER_PCI_TABLE(i40e, pci_id_i40e_map); #ifndef I40E_GLQF_PIT #define I40E_GLQF_PIT(_i) (0x00268C80 + ((_i) * 4)) #endif +#ifndef I40E_GLQF_L3_MAP +#define I40E_GLQF_L3_MAP(_i) (0x0026C700 + ((_i) * 4)) +#endif static inline void i40e_GLQF_reg_init(struct i40e_hw *hw) { /* - * Initialize registers for flexible payload, which should be set by NVM. - * This should be removed from code once it is fixed in NVM. + * Force global configuration for flexible payload + * to the first 16 bytes of the corresponding L2/L3/L4 paylod. + * This should be removed from code once proper + * configuration API is added to avoid configuration conflicts + * between ports of the same device. */ - I40E_WRITE_REG(hw, I40E_GLQF_ORT(18), 0x00000030); - I40E_WRITE_REG(hw, I40E_GLQF_ORT(19), 0x00000030); - I40E_WRITE_REG(hw, I40E_GLQF_ORT(26), 0x0000002B); - I40E_WRITE_REG(hw, I40E_GLQF_ORT(30), 0x0000002B); - I40E_WRITE_REG(hw, I40E_GLQF_ORT(33), 0x000000E0); - I40E_WRITE_REG(hw, I40E_GLQF_ORT(34), 0x000000E3); - I40E_WRITE_REG(hw, I40E_GLQF_ORT(35), 0x000000E6); - I40E_WRITE_REG(hw, I40E_GLQF_ORT(20), 0x00000031); - I40E_WRITE_REG(hw, I40E_GLQF_ORT(23), 0x00000031); - I40E_WRITE_REG(hw, I40E_GLQF_ORT(63), 0x0000002D); - I40E_WRITE_REG(hw, I40E_GLQF_PIT(16), 0x00007480); - I40E_WRITE_REG(hw, I40E_GLQF_PIT(17), 0x00007440); + I40E_WRITE_GLB_REG(hw, I40E_GLQF_ORT(33), 0x000000E0); + I40E_WRITE_GLB_REG(hw, I40E_GLQF_ORT(34), 0x000000E3); + I40E_WRITE_GLB_REG(hw, I40E_GLQF_ORT(35), 0x000000E6); + i40e_global_cfg_warning(I40E_WARNING_ENA_FLX_PLD); - /* Initialize registers for parsing packet type of QinQ */ - I40E_WRITE_REG(hw, I40E_GLQF_ORT(40), 0x00000029); - I40E_WRITE_REG(hw, I40E_GLQF_PIT(9), 0x00009420); + /* + * Initialize registers for parsing packet type of QinQ + * This should be removed from code once proper + * configuration API is added to avoid configuration conflicts + * between ports of the same device. + */ + I40E_WRITE_GLB_REG(hw, I40E_GLQF_ORT(40), 0x00000029); + I40E_WRITE_GLB_REG(hw, I40E_GLQF_PIT(9), 0x00009420); + i40e_global_cfg_warning(I40E_WARNING_QINQ_PARSER); +} + +static inline void i40e_config_automask(struct i40e_pf *pf) +{ + struct i40e_hw *hw = I40E_PF_TO_HW(pf); + uint32_t val; + + /* INTENA flag is not auto-cleared for interrupt */ + val = I40E_READ_REG(hw, I40E_GLINT_CTL); + val |= I40E_GLINT_CTL_DIS_AUTOMASK_PF0_MASK | + I40E_GLINT_CTL_DIS_AUTOMASK_VF0_MASK; + + /* If support multi-driver, PF will use INT0. */ + if (!pf->support_multi_driver) + val |= I40E_GLINT_CTL_DIS_AUTOMASK_N_MASK; + + I40E_WRITE_REG(hw, I40E_GLINT_CTL, val); } #define I40E_FLOW_CONTROL_ETHERTYPE 0x8808 @@ -782,8 +769,8 @@ i40e_add_tx_flow_control_drop_filter(struct i40e_pf *pf) pf->main_vsi_seid, 0, TRUE, NULL, NULL); if (ret) - PMD_INIT_LOG(ERR, "Failed to add filter to drop flow control " - " frames from VSIs."); + PMD_INIT_LOG(ERR, + "Failed to add filter to drop flow control frames from VSIs."); } static int @@ -926,15 +913,17 @@ is_floating_veb_supported(struct rte_devargs *devargs) static void config_floating_veb(struct rte_eth_dev *dev) { - struct rte_pci_device *pci_dev = dev->pci_dev; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); memset(pf->floating_veb_list, 0, sizeof(pf->floating_veb_list)); if (hw->aq.fw_maj_ver >= FLOATING_VEB_SUPPORTED_FW_MAJ) { - pf->floating_veb = is_floating_veb_supported(pci_dev->devargs); - config_vf_floating_veb(pci_dev->devargs, pf->floating_veb, + pf->floating_veb = + is_floating_veb_supported(pci_dev->device.devargs); + config_vf_floating_veb(pci_dev->device.devargs, + pf->floating_veb, pf->floating_veb_list); } else { pf->floating_veb = false; @@ -944,10 +933,236 @@ config_floating_veb(struct rte_eth_dev *dev) #define I40E_L2_TAGS_S_TAG_SHIFT 1 #define I40E_L2_TAGS_S_TAG_MASK I40E_MASK(0x1, I40E_L2_TAGS_S_TAG_SHIFT) +static int +i40e_init_ethtype_filter_list(struct rte_eth_dev *dev) +{ + struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + struct i40e_ethertype_rule *ethertype_rule = &pf->ethertype; + char ethertype_hash_name[RTE_HASH_NAMESIZE]; + int ret; + + struct rte_hash_parameters ethertype_hash_params = { + .name = ethertype_hash_name, + .entries = I40E_MAX_ETHERTYPE_FILTER_NUM, + .key_len = sizeof(struct i40e_ethertype_filter_input), + .hash_func = rte_hash_crc, + .hash_func_init_val = 0, + .socket_id = rte_socket_id(), + }; + + /* Initialize ethertype filter rule list and hash */ + TAILQ_INIT(ðertype_rule->ethertype_list); + snprintf(ethertype_hash_name, RTE_HASH_NAMESIZE, + "ethertype_%s", dev->device->name); + ethertype_rule->hash_table = rte_hash_create(ðertype_hash_params); + if (!ethertype_rule->hash_table) { + PMD_INIT_LOG(ERR, "Failed to create ethertype hash table!"); + return -EINVAL; + } + ethertype_rule->hash_map = rte_zmalloc("i40e_ethertype_hash_map", + sizeof(struct i40e_ethertype_filter *) * + I40E_MAX_ETHERTYPE_FILTER_NUM, + 0); + if (!ethertype_rule->hash_map) { + PMD_INIT_LOG(ERR, + "Failed to allocate memory for ethertype hash map!"); + ret = -ENOMEM; + goto err_ethertype_hash_map_alloc; + } + + return 0; + +err_ethertype_hash_map_alloc: + rte_hash_free(ethertype_rule->hash_table); + + return ret; +} + +static int +i40e_init_tunnel_filter_list(struct rte_eth_dev *dev) +{ + struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + struct i40e_tunnel_rule *tunnel_rule = &pf->tunnel; + char tunnel_hash_name[RTE_HASH_NAMESIZE]; + int ret; + + struct rte_hash_parameters tunnel_hash_params = { + .name = tunnel_hash_name, + .entries = I40E_MAX_TUNNEL_FILTER_NUM, + .key_len = sizeof(struct i40e_tunnel_filter_input), + .hash_func = rte_hash_crc, + .hash_func_init_val = 0, + .socket_id = rte_socket_id(), + }; + + /* Initialize tunnel filter rule list and hash */ + TAILQ_INIT(&tunnel_rule->tunnel_list); + snprintf(tunnel_hash_name, RTE_HASH_NAMESIZE, + "tunnel_%s", dev->device->name); + tunnel_rule->hash_table = rte_hash_create(&tunnel_hash_params); + if (!tunnel_rule->hash_table) { + PMD_INIT_LOG(ERR, "Failed to create tunnel hash table!"); + return -EINVAL; + } + tunnel_rule->hash_map = rte_zmalloc("i40e_tunnel_hash_map", + sizeof(struct i40e_tunnel_filter *) * + I40E_MAX_TUNNEL_FILTER_NUM, + 0); + if (!tunnel_rule->hash_map) { + PMD_INIT_LOG(ERR, + "Failed to allocate memory for tunnel hash map!"); + ret = -ENOMEM; + goto err_tunnel_hash_map_alloc; + } + + return 0; + +err_tunnel_hash_map_alloc: + rte_hash_free(tunnel_rule->hash_table); + + return ret; +} + +static int +i40e_init_fdir_filter_list(struct rte_eth_dev *dev) +{ + struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + struct i40e_fdir_info *fdir_info = &pf->fdir; + char fdir_hash_name[RTE_HASH_NAMESIZE]; + int ret; + + struct rte_hash_parameters fdir_hash_params = { + .name = fdir_hash_name, + .entries = I40E_MAX_FDIR_FILTER_NUM, + .key_len = sizeof(struct i40e_fdir_input), + .hash_func = rte_hash_crc, + .hash_func_init_val = 0, + .socket_id = rte_socket_id(), + }; + + /* Initialize flow director filter rule list and hash */ + TAILQ_INIT(&fdir_info->fdir_list); + snprintf(fdir_hash_name, RTE_HASH_NAMESIZE, + "fdir_%s", dev->device->name); + fdir_info->hash_table = rte_hash_create(&fdir_hash_params); + if (!fdir_info->hash_table) { + PMD_INIT_LOG(ERR, "Failed to create fdir hash table!"); + return -EINVAL; + } + fdir_info->hash_map = rte_zmalloc("i40e_fdir_hash_map", + sizeof(struct i40e_fdir_filter *) * + I40E_MAX_FDIR_FILTER_NUM, + 0); + if (!fdir_info->hash_map) { + PMD_INIT_LOG(ERR, + "Failed to allocate memory for fdir hash map!"); + ret = -ENOMEM; + goto err_fdir_hash_map_alloc; + } + return 0; + +err_fdir_hash_map_alloc: + rte_hash_free(fdir_info->hash_table); + + return ret; +} + +static void +i40e_init_customized_info(struct i40e_pf *pf) +{ + int i; + + /* Initialize customized pctype */ + for (i = I40E_CUSTOMIZED_GTPC; i < I40E_CUSTOMIZED_MAX; i++) { + pf->customized_pctype[i].index = i; + pf->customized_pctype[i].pctype = I40E_FILTER_PCTYPE_INVALID; + pf->customized_pctype[i].valid = false; + } + + pf->gtp_support = false; +} + +void +i40e_init_queue_region_conf(struct rte_eth_dev *dev) +{ + struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + struct i40e_queue_regions *info = &pf->queue_region; + uint16_t i; + + for (i = 0; i < I40E_PFQF_HREGION_MAX_INDEX; i++) + i40e_write_rx_ctl(hw, I40E_PFQF_HREGION(i), 0); + + memset(info, 0, sizeof(struct i40e_queue_regions)); +} + +#define ETH_I40E_SUPPORT_MULTI_DRIVER "support-multi-driver" + +static int +i40e_parse_multi_drv_handler(__rte_unused const char *key, + const char *value, + void *opaque) +{ + struct i40e_pf *pf; + unsigned long support_multi_driver; + char *end; + + pf = (struct i40e_pf *)opaque; + + errno = 0; + support_multi_driver = strtoul(value, &end, 10); + if (errno != 0 || end == value || *end != 0) { + PMD_DRV_LOG(WARNING, "Wrong global configuration"); + return -(EINVAL); + } + + if (support_multi_driver == 1 || support_multi_driver == 0) + pf->support_multi_driver = (bool)support_multi_driver; + else + PMD_DRV_LOG(WARNING, "%s must be 1 or 0,", + "enable global configuration by default." + ETH_I40E_SUPPORT_MULTI_DRIVER); + return 0; +} + +static int +i40e_support_multi_driver(struct rte_eth_dev *dev) +{ + struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + static const char *const valid_keys[] = { + ETH_I40E_SUPPORT_MULTI_DRIVER, NULL}; + struct rte_kvargs *kvlist; + + /* Enable global configuration by default */ + pf->support_multi_driver = false; + + if (!dev->device->devargs) + return 0; + + kvlist = rte_kvargs_parse(dev->device->devargs->args, valid_keys); + if (!kvlist) + return -EINVAL; + + if (rte_kvargs_count(kvlist, ETH_I40E_SUPPORT_MULTI_DRIVER) > 1) + PMD_DRV_LOG(WARNING, "More than one argument \"%s\" and only " + "the first invalid or last valid one is used !", + ETH_I40E_SUPPORT_MULTI_DRIVER); + + if (rte_kvargs_process(kvlist, ETH_I40E_SUPPORT_MULTI_DRIVER, + i40e_parse_multi_drv_handler, pf) < 0) { + rte_kvargs_free(kvlist); + return -EINVAL; + } + + rte_kvargs_free(kvlist); + return 0; +} + static int eth_i40e_dev_init(struct rte_eth_dev *dev) { struct rte_pci_device *pci_dev; + struct rte_intr_handle *intr_handle; struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct i40e_vsi *vsi; @@ -960,6 +1175,7 @@ eth_i40e_dev_init(struct rte_eth_dev *dev) dev->dev_ops = &i40e_eth_dev_ops; dev->rx_pkt_burst = i40e_recv_pkts; dev->tx_pkt_burst = i40e_xmit_pkts; + dev->tx_pkt_prepare = i40e_prep_pkts; /* for secondary processes, we don't initialise any further as primary * has already done this work. Only check we don't need a different @@ -969,7 +1185,9 @@ eth_i40e_dev_init(struct rte_eth_dev *dev) i40e_set_tx_function(dev); return 0; } - pci_dev = dev->pci_dev; + i40e_set_default_ptype_table(dev); + pci_dev = RTE_ETH_DEV_TO_PCI(dev); + intr_handle = &pci_dev->intr_handle; rte_eth_copy_pci_info(dev, pci_dev); @@ -980,8 +1198,8 @@ eth_i40e_dev_init(struct rte_eth_dev *dev) hw->back = I40E_PF_TO_ADAPTER(pf); hw->hw_addr = (uint8_t *)(pci_dev->mem_resource[0].addr); if (!hw->hw_addr) { - PMD_INIT_LOG(ERR, "Hardware is not available, " - "as address is NULL"); + PMD_INIT_LOG(ERR, + "Hardware is not available, as address is NULL"); return -ENODEV; } @@ -993,6 +1211,9 @@ eth_i40e_dev_init(struct rte_eth_dev *dev) hw->bus.func = pci_dev->addr.function; hw->adapter_stopped = 0; + /* Check if need to support multi-driver */ + i40e_support_multi_driver(dev); + /* Make sure all is clean before doing PF reset */ i40e_clear_hw(hw); @@ -1013,13 +1234,18 @@ eth_i40e_dev_init(struct rte_eth_dev *dev) return ret; } + i40e_config_automask(pf); + + i40e_set_default_pctype_table(dev); + /* * To work around the NVM issue, initialize registers * for flexible payload and packet type of QinQ by * software. It should be removed once issues are fixed * in NVM. */ - i40e_GLQF_reg_init(hw); + if (!pf->support_multi_driver) + i40e_GLQF_reg_init(hw); /* Initialize the input set for filters (hash and fd) to default value */ i40e_filter_input_set_init(pf); @@ -1038,10 +1264,24 @@ eth_i40e_dev_init(struct rte_eth_dev *dev) ((hw->nvm.version >> 4) & 0xff), (hw->nvm.version & 0xf), hw->nvm.eetrack); + /* initialise the L3_MAP register */ + if (!pf->support_multi_driver) { + ret = i40e_aq_debug_write_register(hw, I40E_GLQF_L3_MAP(40), + 0x00000028, NULL); + if (ret) + PMD_INIT_LOG(ERR, "Failed to write L3 MAP register %d", + ret); + PMD_INIT_LOG(DEBUG, + "Global register 0x%08x is changed with 0x28", + I40E_GLQF_L3_MAP(40)); + i40e_global_cfg_warning(I40E_WARNING_QINQ_CLOUD_FILTER); + } + /* Need the special FW version to support floating VEB */ config_floating_veb(dev); /* Clear PXE mode */ i40e_clear_pxe_mode(hw); + i40e_dev_sync_phy_type(hw); /* * On X710, performance number is far from the expectation on recent @@ -1111,11 +1351,15 @@ eth_i40e_dev_init(struct rte_eth_dev *dev) i40e_set_fc(hw, &aq_fail, TRUE); /* Set the global registers with default ether type value */ - ret = i40e_vlan_tpid_set(dev, ETH_VLAN_TYPE_OUTER, ETHER_TYPE_VLAN); - if (ret != I40E_SUCCESS) { - PMD_INIT_LOG(ERR, "Failed to set the default outer " - "VLAN ether type"); - goto err_setup_pf_switch; + if (!pf->support_multi_driver) { + ret = i40e_vlan_tpid_set(dev, ETH_VLAN_TYPE_OUTER, + ETHER_TYPE_VLAN); + if (ret != I40E_SUCCESS) { + PMD_INIT_LOG(ERR, + "Failed to set the default outer " + "VLAN ether type"); + goto err_setup_pf_switch; + } } /* PF setup, which includes VSI setup */ @@ -1150,26 +1394,35 @@ eth_i40e_dev_init(struct rte_eth_dev *dev) /* Should be after VSI initialized */ dev->data->mac_addrs = rte_zmalloc("i40e", len, 0); if (!dev->data->mac_addrs) { - PMD_INIT_LOG(ERR, "Failed to allocated memory " - "for storing mac address"); + PMD_INIT_LOG(ERR, + "Failed to allocated memory for storing mac address"); goto err_mac_alloc; } ether_addr_copy((struct ether_addr *)hw->mac.perm_addr, &dev->data->mac_addrs[0]); + /* Init dcb to sw mode by default */ + ret = i40e_dcb_init_configure(dev, TRUE); + if (ret != I40E_SUCCESS) { + PMD_INIT_LOG(INFO, "Failed to init dcb."); + pf->flags &= ~I40E_FLAG_DCB; + } + /* Update HW struct after DCB configuration */ + i40e_get_cap(hw); + /* initialize pf host driver to setup SRIOV resource if applicable */ i40e_pf_host_init(dev); /* register callback func to eal lib */ - rte_intr_callback_register(&(pci_dev->intr_handle), - i40e_dev_interrupt_handler, (void *)dev); + rte_intr_callback_register(intr_handle, + i40e_dev_interrupt_handler, dev); /* configure and enable device interrupt */ i40e_pf_config_irq0(hw, TRUE); i40e_pf_enable_irq0(hw); /* enable uio intr after callback register */ - rte_intr_enable(&(pci_dev->intr_handle)); + rte_intr_enable(intr_handle); /* * Add an ethertype filter to drop all flow control frames transmitted * from VSIs. By doing so, we stop VF from sending out PAUSE or PFC @@ -1185,15 +1438,35 @@ eth_i40e_dev_init(struct rte_eth_dev *dev) /* initialize mirror rule list */ TAILQ_INIT(&pf->mirror_list); - /* Init dcb to sw mode by default */ - ret = i40e_dcb_init_configure(dev, TRUE); - if (ret != I40E_SUCCESS) { - PMD_INIT_LOG(INFO, "Failed to init dcb."); - pf->flags &= ~I40E_FLAG_DCB; - } + /* initialize Traffic Manager configuration */ + i40e_tm_conf_init(dev); + + /* Initialize customized information */ + i40e_init_customized_info(pf); + + ret = i40e_init_ethtype_filter_list(dev); + if (ret < 0) + goto err_init_ethtype_filter_list; + ret = i40e_init_tunnel_filter_list(dev); + if (ret < 0) + goto err_init_tunnel_filter_list; + ret = i40e_init_fdir_filter_list(dev); + if (ret < 0) + goto err_init_fdir_filter_list; + + /* initialize queue region configuration */ + i40e_init_queue_region_conf(dev); return 0; +err_init_fdir_filter_list: + rte_free(pf->tunnel.hash_table); + rte_free(pf->tunnel.hash_map); +err_init_tunnel_filter_list: + rte_free(pf->ethertype.hash_table); + rte_free(pf->ethertype.hash_map); +err_init_ethtype_filter_list: + rte_free(dev->data->mac_addrs); err_mac_alloc: i40e_vsi_release(pf->main_vsi); err_setup_pf_switch: @@ -1212,12 +1485,73 @@ err_get_capabilities: return ret; } +static void +i40e_rm_ethtype_filter_list(struct i40e_pf *pf) +{ + struct i40e_ethertype_filter *p_ethertype; + struct i40e_ethertype_rule *ethertype_rule; + + ethertype_rule = &pf->ethertype; + /* Remove all ethertype filter rules and hash */ + if (ethertype_rule->hash_map) + rte_free(ethertype_rule->hash_map); + if (ethertype_rule->hash_table) + rte_hash_free(ethertype_rule->hash_table); + + while ((p_ethertype = TAILQ_FIRST(ðertype_rule->ethertype_list))) { + TAILQ_REMOVE(ðertype_rule->ethertype_list, + p_ethertype, rules); + rte_free(p_ethertype); + } +} + +static void +i40e_rm_tunnel_filter_list(struct i40e_pf *pf) +{ + struct i40e_tunnel_filter *p_tunnel; + struct i40e_tunnel_rule *tunnel_rule; + + tunnel_rule = &pf->tunnel; + /* Remove all tunnel director rules and hash */ + if (tunnel_rule->hash_map) + rte_free(tunnel_rule->hash_map); + if (tunnel_rule->hash_table) + rte_hash_free(tunnel_rule->hash_table); + + while ((p_tunnel = TAILQ_FIRST(&tunnel_rule->tunnel_list))) { + TAILQ_REMOVE(&tunnel_rule->tunnel_list, p_tunnel, rules); + rte_free(p_tunnel); + } +} + +static void +i40e_rm_fdir_filter_list(struct i40e_pf *pf) +{ + struct i40e_fdir_filter *p_fdir; + struct i40e_fdir_info *fdir_info; + + fdir_info = &pf->fdir; + /* Remove all flow director rules and hash */ + if (fdir_info->hash_map) + rte_free(fdir_info->hash_map); + if (fdir_info->hash_table) + rte_hash_free(fdir_info->hash_table); + + while ((p_fdir = TAILQ_FIRST(&fdir_info->fdir_list))) { + TAILQ_REMOVE(&fdir_info->fdir_list, p_fdir, rules); + rte_free(p_fdir); + } +} + static int eth_i40e_dev_uninit(struct rte_eth_dev *dev) { + struct i40e_pf *pf; struct rte_pci_device *pci_dev; + struct rte_intr_handle *intr_handle; struct i40e_hw *hw; struct i40e_filter_control_settings settings; + struct rte_flow *p_flow; int ret; uint8_t aq_fail = 0; @@ -1226,8 +1560,10 @@ eth_i40e_dev_uninit(struct rte_eth_dev *dev) if (rte_eal_process_type() != RTE_PROC_PRIMARY) return 0; + pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); - pci_dev = dev->pci_dev; + pci_dev = RTE_ETH_DEV_TO_PCI(dev); + intr_handle = &pci_dev->intr_handle; if (hw->adapter_stopped == 0) i40e_dev_close(dev); @@ -1257,11 +1593,24 @@ eth_i40e_dev_uninit(struct rte_eth_dev *dev) dev->data->mac_addrs = NULL; /* disable uio intr before callback unregister */ - rte_intr_disable(&(pci_dev->intr_handle)); + rte_intr_disable(intr_handle); /* register callback func to eal lib */ - rte_intr_callback_unregister(&(pci_dev->intr_handle), - i40e_dev_interrupt_handler, (void *)dev); + rte_intr_callback_unregister(intr_handle, + i40e_dev_interrupt_handler, dev); + + i40e_rm_ethtype_filter_list(pf); + i40e_rm_tunnel_filter_list(pf); + i40e_rm_fdir_filter_list(pf); + + /* Remove all flows */ + while ((p_flow = TAILQ_FIRST(&pf->flow_list))) { + TAILQ_REMOVE(&pf->flow_list, p_flow, node); + rte_free(p_flow); + } + + /* Remove all Traffic Manager configuration */ + i40e_tm_conf_uninit(dev); return 0; } @@ -1272,9 +1621,14 @@ i40e_dev_configure(struct rte_eth_dev *dev) struct i40e_adapter *ad = I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); enum rte_eth_rx_mq_mode mq_mode = dev->data->dev_conf.rxmode.mq_mode; int i, ret; + ret = i40e_dev_sync_phy_type(hw); + if (ret) + return ret; + /* Initialize to TRUE. If any of Rx queues doesn't meet the * bulk allocation or vector Rx preconditions we will reset it. */ @@ -1327,6 +1681,8 @@ i40e_dev_configure(struct rte_eth_dev *dev) } } + TAILQ_INIT(&pf->flow_list); + return 0; err_dcb: @@ -1347,7 +1703,8 @@ void i40e_vsi_queues_unbind_intr(struct i40e_vsi *vsi) { struct rte_eth_dev *dev = vsi->adapter->eth_dev; - struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; struct i40e_hw *hw = I40E_VSI_TO_HW(vsi); uint16_t msix_vect = vsi->msix_intr; uint16_t i; @@ -1385,16 +1742,18 @@ i40e_vsi_queues_unbind_intr(struct i40e_vsi *vsi) static void __vsi_queues_bind_intr(struct i40e_vsi *vsi, uint16_t msix_vect, - int base_queue, int nb_queue) + int base_queue, int nb_queue, + uint16_t itr_idx) { int i; uint32_t val; struct i40e_hw *hw = I40E_VSI_TO_HW(vsi); + struct i40e_pf *pf = I40E_VSI_TO_PF(vsi); /* Bind all RX queues to allocated MSIX interrupt */ for (i = 0; i < nb_queue; i++) { val = (msix_vect << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | - I40E_QINT_RQCTL_ITR_INDX_MASK | + itr_idx << I40E_QINT_RQCTL_ITR_INDX_SHIFT | ((base_queue + i + 1) << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | (0 << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT) | @@ -1408,7 +1767,8 @@ __vsi_queues_bind_intr(struct i40e_vsi *vsi, uint16_t msix_vect, /* Write first RX queue to Link list register as the head element */ if (vsi->type != I40E_VSI_SRIOV) { uint16_t interval = - i40e_calc_itr_interval(RTE_LIBRTE_I40E_ITR_INTERVAL); + i40e_calc_itr_interval(RTE_LIBRTE_I40E_ITR_INTERVAL, + pf->support_multi_driver); if (msix_vect == I40E_MISC_VEC_ID) { I40E_WRITE_REG(hw, I40E_PFINT_LNKLST0, @@ -1457,16 +1817,16 @@ __vsi_queues_bind_intr(struct i40e_vsi *vsi, uint16_t msix_vect, } void -i40e_vsi_queues_bind_intr(struct i40e_vsi *vsi) +i40e_vsi_queues_bind_intr(struct i40e_vsi *vsi, uint16_t itr_idx) { struct rte_eth_dev *dev = vsi->adapter->eth_dev; - struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; struct i40e_hw *hw = I40E_VSI_TO_HW(vsi); uint16_t msix_vect = vsi->msix_intr; uint16_t nb_msix = RTE_MIN(vsi->nb_msix, intr_handle->nb_efd); uint16_t queue_idx = 0; int record = 0; - uint32_t val; int i; for (i = 0; i < vsi->nb_qps; i++) { @@ -1474,17 +1834,11 @@ i40e_vsi_queues_bind_intr(struct i40e_vsi *vsi) I40E_WRITE_REG(hw, I40E_QINT_RQCTL(vsi->base_queue + i), 0); } - /* INTENA flag is not auto-cleared for interrupt */ - val = I40E_READ_REG(hw, I40E_GLINT_CTL); - val |= I40E_GLINT_CTL_DIS_AUTOMASK_PF0_MASK | - I40E_GLINT_CTL_DIS_AUTOMASK_N_MASK | - I40E_GLINT_CTL_DIS_AUTOMASK_VF0_MASK; - I40E_WRITE_REG(hw, I40E_GLINT_CTL, val); - /* VF bind interrupt */ if (vsi->type == I40E_VSI_SRIOV) { __vsi_queues_bind_intr(vsi, msix_vect, - vsi->base_queue, vsi->nb_qps); + vsi->base_queue, vsi->nb_qps, + itr_idx); return; } @@ -1510,7 +1864,8 @@ i40e_vsi_queues_bind_intr(struct i40e_vsi *vsi) /* no enough msix_vect, map all to one */ __vsi_queues_bind_intr(vsi, msix_vect, vsi->base_queue + i, - vsi->nb_used_qps - i); + vsi->nb_used_qps - i, + itr_idx); for (; !!record && i < vsi->nb_used_qps; i++) intr_handle->intr_vec[queue_idx + i] = msix_vect; @@ -1518,7 +1873,8 @@ i40e_vsi_queues_bind_intr(struct i40e_vsi *vsi) } /* 1:1 queue/msix_vect mapping */ __vsi_queues_bind_intr(vsi, msix_vect, - vsi->base_queue + i, 1); + vsi->base_queue + i, 1, + itr_idx); if (!!record) intr_handle->intr_vec[queue_idx + i] = msix_vect; @@ -1531,29 +1887,25 @@ static void i40e_vsi_enable_queues_intr(struct i40e_vsi *vsi) { struct rte_eth_dev *dev = vsi->adapter->eth_dev; - struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; struct i40e_hw *hw = I40E_VSI_TO_HW(vsi); - uint16_t interval = i40e_calc_itr_interval(\ - RTE_LIBRTE_I40E_ITR_INTERVAL); + struct i40e_pf *pf = I40E_VSI_TO_PF(vsi); uint16_t msix_intr, i; - if (rte_intr_allow_others(intr_handle)) + if (rte_intr_allow_others(intr_handle) && !pf->support_multi_driver) for (i = 0; i < vsi->nb_msix; i++) { msix_intr = vsi->msix_intr + i; I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTLN(msix_intr - 1), I40E_PFINT_DYN_CTLN_INTENA_MASK | I40E_PFINT_DYN_CTLN_CLEARPBA_MASK | - (0 << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT) | - (interval << - I40E_PFINT_DYN_CTLN_INTERVAL_SHIFT)); + I40E_PFINT_DYN_CTLN_ITR_INDX_MASK); } else I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTL0, I40E_PFINT_DYN_CTL0_INTENA_MASK | I40E_PFINT_DYN_CTL0_CLEARPBA_MASK | - (0 << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT) | - (interval << - I40E_PFINT_DYN_CTL0_INTERVAL_SHIFT)); + I40E_PFINT_DYN_CTL0_ITR_INDX_MASK); I40E_WRITE_FLUSH(hw); } @@ -1562,18 +1914,21 @@ static void i40e_vsi_disable_queues_intr(struct i40e_vsi *vsi) { struct rte_eth_dev *dev = vsi->adapter->eth_dev; - struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; struct i40e_hw *hw = I40E_VSI_TO_HW(vsi); + struct i40e_pf *pf = I40E_VSI_TO_PF(vsi); uint16_t msix_intr, i; - if (rte_intr_allow_others(intr_handle)) + if (rte_intr_allow_others(intr_handle) && !pf->support_multi_driver) for (i = 0; i < vsi->nb_msix; i++) { msix_intr = vsi->msix_intr + i; I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTLN(msix_intr - 1), - 0); + I40E_PFINT_DYN_CTLN_ITR_INDX_MASK); } else - I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTL0, 0); + I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTL0, + I40E_PFINT_DYN_CTL0_ITR_INDX_MASK); I40E_WRITE_FLUSH(hw); } @@ -1585,6 +1940,8 @@ i40e_parse_link_speeds(uint16_t link_speeds) if (link_speeds & ETH_LINK_SPEED_40G) link_speed |= I40E_LINK_SPEED_40GB; + if (link_speeds & ETH_LINK_SPEED_25G) + link_speed |= I40E_LINK_SPEED_25GB; if (link_speeds & ETH_LINK_SPEED_20G) link_speed |= I40E_LINK_SPEED_20GB; if (link_speeds & ETH_LINK_SPEED_10G) @@ -1600,16 +1957,21 @@ i40e_parse_link_speeds(uint16_t link_speeds) static int i40e_phy_conf_link(struct i40e_hw *hw, uint8_t abilities, - uint8_t force_speed) + uint8_t force_speed, + bool is_up) { enum i40e_status_code status; struct i40e_aq_get_phy_abilities_resp phy_ab; struct i40e_aq_set_phy_config phy_conf; + enum i40e_aq_phy_type cnt; + uint32_t phy_type_mask = 0; + const uint8_t mask = I40E_AQ_PHY_FLAG_PAUSE_TX | I40E_AQ_PHY_FLAG_PAUSE_RX | I40E_AQ_PHY_FLAG_PAUSE_RX | I40E_AQ_PHY_FLAG_LOW_POWER; const uint8_t advt = I40E_LINK_SPEED_40GB | + I40E_LINK_SPEED_25GB | I40E_LINK_SPEED_10GB | I40E_LINK_SPEED_1GB | I40E_LINK_SPEED_100MB; @@ -1621,6 +1983,10 @@ i40e_phy_conf_link(struct i40e_hw *hw, if (status) return ret; + /* If link already up, no need to set up again */ + if (is_up && phy_ab.phy_type != 0) + return I40E_SUCCESS; + memset(&phy_conf, 0, sizeof(phy_conf)); /* bits 0-2 use the values from get_phy_abilities_resp */ @@ -1631,12 +1997,22 @@ i40e_phy_conf_link(struct i40e_hw *hw, if (abilities & I40E_AQ_PHY_AN_ENABLED) phy_conf.link_speed = advt; else - phy_conf.link_speed = force_speed; + phy_conf.link_speed = is_up ? force_speed : phy_ab.link_speed; phy_conf.abilities = abilities; + + + /* To enable link, phy_type mask needs to include each type */ + for (cnt = I40E_PHY_TYPE_SGMII; cnt < I40E_PHY_TYPE_MAX; cnt++) + phy_type_mask |= 1 << cnt; + /* use get_phy_abilities_resp value for the rest */ - phy_conf.phy_type = phy_ab.phy_type; + phy_conf.phy_type = is_up ? cpu_to_le32(phy_type_mask) : 0; + phy_conf.phy_type_ext = is_up ? (I40E_AQ_PHY_TYPE_EXT_25G_KR | + I40E_AQ_PHY_TYPE_EXT_25G_CR | I40E_AQ_PHY_TYPE_EXT_25G_SR | + I40E_AQ_PHY_TYPE_EXT_25G_LR) : 0; + phy_conf.fec_config = phy_ab.fec_cfg_curr_mod_ext_info; phy_conf.eee_capability = phy_ab.eee_capability; phy_conf.eeer = phy_ab.eeer_val; phy_conf.low_power_ctrl = phy_ab.d3_lpan; @@ -1667,13 +2043,7 @@ i40e_apply_link_speed(struct rte_eth_dev *dev) abilities |= I40E_AQ_PHY_AN_ENABLED; abilities |= I40E_AQ_PHY_LINK_ENABLED; - /* Skip changing speed on 40G interfaces, FW does not support */ - if (i40e_is_40G_device(hw->device_id)) { - speed = I40E_LINK_SPEED_UNKNOWN; - abilities |= I40E_AQ_PHY_AN_ENABLED; - } - - return i40e_phy_conf_link(hw, abilities, speed); + return i40e_phy_conf_link(hw, abilities, speed, true); } static int @@ -1683,14 +2053,17 @@ i40e_dev_start(struct rte_eth_dev *dev) struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct i40e_vsi *main_vsi = pf->main_vsi; int ret, i; - struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; uint32_t intr_vector = 0; + struct i40e_vsi *vsi; hw->adapter_stopped = 0; if (dev->data->dev_conf.link_speeds & ETH_LINK_SPEED_FIXED) { - PMD_INIT_LOG(ERR, "Invalid link_speeds for port %hhu; autonegotiation disabled", - dev->data->port_id); + PMD_INIT_LOG(ERR, + "Invalid link_speeds for port %u, autonegotiation disabled", + dev->data->port_id); return -EINVAL; } @@ -1700,8 +2073,9 @@ i40e_dev_start(struct rte_eth_dev *dev) !RTE_ETH_DEV_SRIOV(dev).active) && dev->data->dev_conf.intr_conf.rxq != 0) { intr_vector = dev->data->nb_rx_queues; - if (rte_intr_efd_enable(intr_handle, intr_vector)) - return -1; + ret = rte_intr_efd_enable(intr_handle, intr_vector); + if (ret) + return ret; } if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) { @@ -1710,8 +2084,9 @@ i40e_dev_start(struct rte_eth_dev *dev) dev->data->nb_rx_queues * sizeof(int), 0); if (!intr_handle->intr_vec) { - PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues" - " intr_vec\n", dev->data->nb_rx_queues); + PMD_INIT_LOG(ERR, + "Failed to allocate %d rx_queues intr_vec", + dev->data->nb_rx_queues); return -ENOMEM; } } @@ -1726,19 +2101,21 @@ i40e_dev_start(struct rte_eth_dev *dev) /* Map queues with MSIX interrupt */ main_vsi->nb_used_qps = dev->data->nb_rx_queues - pf->nb_cfg_vmdq_vsi * RTE_LIBRTE_I40E_QUEUE_NUM_PER_VM; - i40e_vsi_queues_bind_intr(main_vsi); + i40e_vsi_queues_bind_intr(main_vsi, I40E_ITR_INDEX_DEFAULT); i40e_vsi_enable_queues_intr(main_vsi); /* Map VMDQ VSI queues with MSIX interrupt */ for (i = 0; i < pf->nb_cfg_vmdq_vsi; i++) { pf->vmdq[i].vsi->nb_used_qps = RTE_LIBRTE_I40E_QUEUE_NUM_PER_VM; - i40e_vsi_queues_bind_intr(pf->vmdq[i].vsi); + i40e_vsi_queues_bind_intr(pf->vmdq[i].vsi, + I40E_ITR_INDEX_DEFAULT); i40e_vsi_enable_queues_intr(pf->vmdq[i].vsi); } /* enable FDIR MSIX interrupt */ if (pf->fdir.fdir_vsi) { - i40e_vsi_queues_bind_intr(pf->fdir.fdir_vsi); + i40e_vsi_queues_bind_intr(pf->fdir.fdir_vsi, + I40E_ITR_INDEX_NONE); i40e_vsi_enable_queues_intr(pf->fdir.fdir_vsi); } @@ -1761,10 +2138,20 @@ i40e_dev_start(struct rte_eth_dev *dev) PMD_DRV_LOG(INFO, "fail to set vsi broadcast"); } + /* Enable the VLAN promiscuous mode. */ + if (pf->vfs) { + for (i = 0; i < pf->vf_num; i++) { + vsi = pf->vfs[i].vsi; + i40e_aq_set_vsi_vlan_promisc(hw, vsi->seid, + true, NULL); + } + } + /* Apply link configure */ if (dev->data->dev_conf.link_speeds & ~(ETH_LINK_SPEED_100M | ETH_LINK_SPEED_1G | ETH_LINK_SPEED_10G | - ETH_LINK_SPEED_20G | ETH_LINK_SPEED_40G)) { + ETH_LINK_SPEED_20G | ETH_LINK_SPEED_25G | + ETH_LINK_SPEED_40G)) { PMD_DRV_LOG(ERR, "Invalid link setting"); goto err_up; } @@ -1783,9 +2170,9 @@ i40e_dev_start(struct rte_eth_dev *dev) i40e_pf_enable_irq0(hw); if (dev->data->dev_conf.intr_conf.lsc != 0) - PMD_INIT_LOG(INFO, "lsc won't enable because of" - " no intr multiplex\n"); - } else if (dev->data->dev_conf.intr_conf.lsc != 0) { + PMD_INIT_LOG(INFO, + "lsc won't enable because of no intr multiplex"); + } else { ret = i40e_aq_set_phy_int_mask(hw, ~(I40E_AQ_EVENT_LINK_UPDOWN | I40E_AQ_EVENT_MODULE_QUAL_FAIL | @@ -1793,13 +2180,20 @@ i40e_dev_start(struct rte_eth_dev *dev) if (ret != I40E_SUCCESS) PMD_DRV_LOG(WARNING, "Fail to set phy mask"); - /* Call get_link_info aq commond to enable LSE */ + /* Call get_link_info aq commond to enable/disable LSE */ i40e_dev_link_update(dev, 0); } /* enable uio intr after callback register */ rte_intr_enable(intr_handle); + i40e_filter_restore(pf); + + if (pf->tm_conf.root && !pf->tm_conf.committed) + PMD_DRV_LOG(WARNING, + "please call hierarchy_commit() " + "before starting the port"); + return I40E_SUCCESS; err_up: @@ -1813,11 +2207,14 @@ static void i40e_dev_stop(struct rte_eth_dev *dev) { struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct i40e_vsi *main_vsi = pf->main_vsi; - struct i40e_mirror_rule *p_mirror; - struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; int i; + if (hw->adapter_stopped == 1) + return; /* Disable all queues */ i40e_dev_switch_queues(pf, FALSE); @@ -1840,13 +2237,6 @@ i40e_dev_stop(struct rte_eth_dev *dev) /* Set link down */ i40e_dev_set_link_down(dev); - /* Remove all mirror rules */ - while ((p_mirror = TAILQ_FIRST(&pf->mirror_list))) { - TAILQ_REMOVE(&pf->mirror_list, p_mirror, rules); - rte_free(p_mirror); - } - pf->nb_mirror_rule = 0; - if (!rte_intr_allow_others(intr_handle)) /* resume to the default handler */ rte_intr_callback_register(intr_handle, @@ -1859,6 +2249,11 @@ i40e_dev_stop(struct rte_eth_dev *dev) rte_free(intr_handle->intr_vec); intr_handle->intr_vec = NULL; } + + /* reset hierarchy commit */ + pf->tm_conf.committed = false; + + hw->adapter_stopped = 1; } static void @@ -1866,34 +2261,56 @@ i40e_dev_close(struct rte_eth_dev *dev) { struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; + struct i40e_mirror_rule *p_mirror; uint32_t reg; int i; + int ret; PMD_INIT_FUNC_TRACE(); i40e_dev_stop(dev); - hw->adapter_stopped = 1; + + /* Remove all mirror rules */ + while ((p_mirror = TAILQ_FIRST(&pf->mirror_list))) { + ret = i40e_aq_del_mirror_rule(hw, + pf->main_vsi->veb->seid, + p_mirror->rule_type, + p_mirror->entries, + p_mirror->num_entries, + p_mirror->id); + if (ret < 0) + PMD_DRV_LOG(ERR, "failed to remove mirror rule: " + "status = %d, aq_err = %d.", ret, + hw->aq.asq_last_status); + + /* remove mirror software resource anyway */ + TAILQ_REMOVE(&pf->mirror_list, p_mirror, rules); + rte_free(p_mirror); + pf->nb_mirror_rule--; + } + i40e_dev_free_queues(dev); /* Disable interrupt */ i40e_pf_disable_irq0(hw); - rte_intr_disable(&(dev->pci_dev->intr_handle)); + rte_intr_disable(intr_handle); /* shutdown and destroy the HMC */ i40e_shutdown_lan_hmc(hw); - /* release all the existing VSIs and VEBs */ - i40e_fdir_teardown(pf); - i40e_vsi_release(pf->main_vsi); - for (i = 0; i < pf->nb_cfg_vmdq_vsi; i++) { i40e_vsi_release(pf->vmdq[i].vsi); pf->vmdq[i].vsi = NULL; } - rte_free(pf->vmdq); pf->vmdq = NULL; + /* release all the existing VSIs and VEBs */ + i40e_fdir_teardown(pf); + i40e_vsi_release(pf->main_vsi); + /* shutdown the adminq */ i40e_aq_queue_shutdown(hw, true); i40e_shutdown_adminq(hw); @@ -1908,6 +2325,32 @@ i40e_dev_close(struct rte_eth_dev *dev) I40E_WRITE_FLUSH(hw); } +/* + * Reset PF device only to re-initialize resources in PMD layer + */ +static int +i40e_dev_reset(struct rte_eth_dev *dev) +{ + int ret; + + /* When a DPDK PMD PF begin to reset PF port, it should notify all + * its VF to make them align with it. The detailed notification + * mechanism is PMD specific. As to i40e PF, it is rather complex. + * To avoid unexpected behavior in VF, currently reset of PF with + * SR-IOV activation is not supported. It might be supported later. + */ + if (dev->data->sriov.active) + return -ENOTSUP; + + ret = eth_i40e_dev_uninit(dev); + if (ret) + return ret; + + ret = eth_i40e_dev_init(dev); + + return ret; +} + static void i40e_dev_promiscuous_enable(struct rte_eth_dev *dev) { @@ -1994,10 +2437,11 @@ static int i40e_dev_set_link_down(struct rte_eth_dev *dev) { uint8_t speed = I40E_LINK_SPEED_UNKNOWN; - uint8_t abilities = I40E_AQ_PHY_ENABLE_ATOMIC_LINK; + uint8_t abilities = 0; struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); - return i40e_phy_conf_link(hw, abilities, speed); + abilities = I40E_AQ_PHY_ENABLE_ATOMIC_LINK; + return i40e_phy_conf_link(hw, abilities, speed, false); } int @@ -2030,11 +2474,11 @@ i40e_dev_link_update(struct rte_eth_dev *dev, } link.link_status = link_status.link_info & I40E_AQ_LINK_UP; - if (!wait_to_complete) + if (!wait_to_complete || link.link_status) break; rte_delay_ms(CHECK_INTERVAL); - } while (!link.link_status && rep_cnt--); + } while (--rep_cnt); if (!link.link_status) goto out; @@ -2056,6 +2500,9 @@ i40e_dev_link_update(struct rte_eth_dev *dev, case I40E_LINK_SPEED_20GB: link.link_speed = ETH_SPEED_NUM_20G; break; + case I40E_LINK_SPEED_25GB: + link.link_speed = ETH_SPEED_NUM_25G; + break; case I40E_LINK_SPEED_40GB: link.link_speed = ETH_SPEED_NUM_40G; break; @@ -2072,6 +2519,8 @@ out: if (link.link_status == old.link_status) return -1; + i40e_notify_all_vfs_link_status(dev); + return 0; } @@ -2096,6 +2545,10 @@ i40e_update_vsi_stats(struct i40e_vsi *vsi) i40e_stat_update_48(hw, I40E_GLV_BPRCH(idx), I40E_GLV_BPRCL(idx), vsi->offset_loaded, &oes->rx_broadcast, &nes->rx_broadcast); + /* exclude CRC bytes */ + nes->rx_bytes -= (nes->rx_unicast + nes->rx_multicast + + nes->rx_broadcast) * ETHER_CRC_LEN; + i40e_stat_update_32(hw, I40E_GLV_RDPC(idx), vsi->offset_loaded, &oes->rx_discards, &nes->rx_discards); /* GLV_REPC not supported */ @@ -2146,6 +2599,56 @@ i40e_read_stats_registers(struct i40e_pf *pf, struct i40e_hw *hw) struct i40e_hw_port_stats *ns = &pf->stats; /* new stats */ struct i40e_hw_port_stats *os = &pf->stats_offset; /* old stats */ + /* Get rx/tx bytes of internal transfer packets */ + i40e_stat_update_48(hw, I40E_GLV_GORCH(hw->port), + I40E_GLV_GORCL(hw->port), + pf->offset_loaded, + &pf->internal_stats_offset.rx_bytes, + &pf->internal_stats.rx_bytes); + + i40e_stat_update_48(hw, I40E_GLV_GOTCH(hw->port), + I40E_GLV_GOTCL(hw->port), + pf->offset_loaded, + &pf->internal_stats_offset.tx_bytes, + &pf->internal_stats.tx_bytes); + /* Get total internal rx packet count */ + i40e_stat_update_48(hw, I40E_GLV_UPRCH(hw->port), + I40E_GLV_UPRCL(hw->port), + pf->offset_loaded, + &pf->internal_stats_offset.rx_unicast, + &pf->internal_stats.rx_unicast); + i40e_stat_update_48(hw, I40E_GLV_MPRCH(hw->port), + I40E_GLV_MPRCL(hw->port), + pf->offset_loaded, + &pf->internal_stats_offset.rx_multicast, + &pf->internal_stats.rx_multicast); + i40e_stat_update_48(hw, I40E_GLV_BPRCH(hw->port), + I40E_GLV_BPRCL(hw->port), + pf->offset_loaded, + &pf->internal_stats_offset.rx_broadcast, + &pf->internal_stats.rx_broadcast); + /* Get total internal tx packet count */ + i40e_stat_update_48(hw, I40E_GLV_UPTCH(hw->port), + I40E_GLV_UPTCL(hw->port), + pf->offset_loaded, + &pf->internal_stats_offset.tx_unicast, + &pf->internal_stats.tx_unicast); + i40e_stat_update_48(hw, I40E_GLV_MPTCH(hw->port), + I40E_GLV_MPTCL(hw->port), + pf->offset_loaded, + &pf->internal_stats_offset.tx_multicast, + &pf->internal_stats.tx_multicast); + i40e_stat_update_48(hw, I40E_GLV_BPTCH(hw->port), + I40E_GLV_BPTCL(hw->port), + pf->offset_loaded, + &pf->internal_stats_offset.tx_broadcast, + &pf->internal_stats.tx_broadcast); + + /* exclude CRC size */ + pf->internal_stats.rx_bytes -= (pf->internal_stats.rx_unicast + + pf->internal_stats.rx_multicast + + pf->internal_stats.rx_broadcast) * ETHER_CRC_LEN; + /* Get statistics of struct i40e_eth_stats */ i40e_stat_update_48(hw, I40E_GLPRT_GORCH(hw->port), I40E_GLPRT_GORCL(hw->port), @@ -2169,6 +2672,32 @@ i40e_read_stats_registers(struct i40e_pf *pf, struct i40e_hw *hw) ns->eth.rx_bytes -= (ns->eth.rx_unicast + ns->eth.rx_multicast + ns->eth.rx_broadcast) * ETHER_CRC_LEN; + /* exclude internal rx bytes + * Workaround: it is possible I40E_GLV_GORCH[H/L] is updated before + * I40E_GLPRT_GORCH[H/L], so there is a small window that cause negative + * value. + * same to I40E_GLV_UPRC[H/L], I40E_GLV_MPRC[H/L], I40E_GLV_BPRC[H/L]. + */ + if (ns->eth.rx_bytes < pf->internal_stats.rx_bytes) + ns->eth.rx_bytes = 0; + else + ns->eth.rx_bytes -= pf->internal_stats.rx_bytes; + + if (ns->eth.rx_unicast < pf->internal_stats.rx_unicast) + ns->eth.rx_unicast = 0; + else + ns->eth.rx_unicast -= pf->internal_stats.rx_unicast; + + if (ns->eth.rx_multicast < pf->internal_stats.rx_multicast) + ns->eth.rx_multicast = 0; + else + ns->eth.rx_multicast -= pf->internal_stats.rx_multicast; + + if (ns->eth.rx_broadcast < pf->internal_stats.rx_broadcast) + ns->eth.rx_broadcast = 0; + else + ns->eth.rx_broadcast -= pf->internal_stats.rx_broadcast; + i40e_stat_update_32(hw, I40E_GLPRT_RDPC(hw->port), pf->offset_loaded, &os->eth.rx_discards, &ns->eth.rx_discards); @@ -2196,6 +2725,33 @@ i40e_read_stats_registers(struct i40e_pf *pf, struct i40e_hw *hw) &ns->eth.tx_broadcast); ns->eth.tx_bytes -= (ns->eth.tx_unicast + ns->eth.tx_multicast + ns->eth.tx_broadcast) * ETHER_CRC_LEN; + + /* exclude internal tx bytes + * Workaround: it is possible I40E_GLV_GOTCH[H/L] is updated before + * I40E_GLPRT_GOTCH[H/L], so there is a small window that cause negative + * value. + * same to I40E_GLV_UPTC[H/L], I40E_GLV_MPTC[H/L], I40E_GLV_BPTC[H/L]. + */ + if (ns->eth.tx_bytes < pf->internal_stats.tx_bytes) + ns->eth.tx_bytes = 0; + else + ns->eth.tx_bytes -= pf->internal_stats.tx_bytes; + + if (ns->eth.tx_unicast < pf->internal_stats.tx_unicast) + ns->eth.tx_unicast = 0; + else + ns->eth.tx_unicast -= pf->internal_stats.tx_unicast; + + if (ns->eth.tx_multicast < pf->internal_stats.tx_multicast) + ns->eth.tx_multicast = 0; + else + ns->eth.tx_multicast -= pf->internal_stats.tx_multicast; + + if (ns->eth.tx_broadcast < pf->internal_stats.tx_broadcast) + ns->eth.tx_broadcast = 0; + else + ns->eth.tx_broadcast -= pf->internal_stats.tx_broadcast; + /* GLPRT_TEPC not supported */ /* additional port specific stats */ @@ -2335,7 +2891,7 @@ i40e_read_stats_registers(struct i40e_pf *pf, struct i40e_hw *hw) } /* Get all statistics of a port */ -static void +static int i40e_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) { struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); @@ -2346,13 +2902,14 @@ i40e_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) /* call read registers - updates values, now write them to struct */ i40e_read_stats_registers(pf, hw); - stats->ipackets = pf->main_vsi->eth_stats.rx_unicast + - pf->main_vsi->eth_stats.rx_multicast + - pf->main_vsi->eth_stats.rx_broadcast - + stats->ipackets = ns->eth.rx_unicast + + ns->eth.rx_multicast + + ns->eth.rx_broadcast - + ns->eth.rx_discards - pf->main_vsi->eth_stats.rx_discards; - stats->opackets = pf->main_vsi->eth_stats.tx_unicast + - pf->main_vsi->eth_stats.tx_multicast + - pf->main_vsi->eth_stats.tx_broadcast; + stats->opackets = ns->eth.tx_unicast + + ns->eth.tx_multicast + + ns->eth.tx_broadcast; stats->ibytes = ns->eth.rx_bytes; stats->obytes = ns->eth.tx_bytes; stats->oerrors = ns->eth.tx_errors + @@ -2434,6 +2991,7 @@ i40e_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) ns->checksum_error); PMD_DRV_LOG(DEBUG, "fdir_match: %"PRIu64"", ns->fd_sb_match); PMD_DRV_LOG(DEBUG, "***************** PF stats end ********************"); + return 0; } /* Reset the statistics */ @@ -2534,6 +3092,7 @@ i40e_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, for (i = 0; i < I40E_NB_ETH_XSTATS; i++) { xstats[count].value = *(uint64_t *)(((char *)&hw_stats->eth) + rte_i40e_stats_strings[i].offset); + xstats[count].id = count; count++; } @@ -2541,6 +3100,7 @@ i40e_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, for (i = 0; i < I40E_NB_HW_PORT_XSTATS; i++) { xstats[count].value = *(uint64_t *)(((char *)hw_stats) + rte_i40e_hw_port_strings[i].offset); + xstats[count].id = count; count++; } @@ -2550,6 +3110,7 @@ i40e_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, *(uint64_t *)(((char *)hw_stats) + rte_i40e_rxq_prio_strings[i].offset + (sizeof(uint64_t) * prio)); + xstats[count].id = count; count++; } } @@ -2560,6 +3121,7 @@ i40e_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, *(uint64_t *)(((char *)hw_stats) + rte_i40e_txq_prio_strings[i].offset + (sizeof(uint64_t) * prio)); + xstats[count].id = count; count++; } } @@ -2578,19 +3140,49 @@ i40e_dev_queue_stats_mapping_set(__rte_unused struct rte_eth_dev *dev, return -ENOSYS; } +static int +i40e_fw_version_get(struct rte_eth_dev *dev, char *fw_version, size_t fw_size) +{ + struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); + u32 full_ver; + u8 ver, patch; + u16 build; + int ret; + + full_ver = hw->nvm.oem_ver; + ver = (u8)(full_ver >> 24); + build = (u16)((full_ver >> 8) & 0xffff); + patch = (u8)(full_ver & 0xff); + + ret = snprintf(fw_version, fw_size, + "%d.%d%d 0x%08x %d.%d.%d", + ((hw->nvm.version >> 12) & 0xf), + ((hw->nvm.version >> 4) & 0xff), + (hw->nvm.version & 0xf), hw->nvm.eetrack, + ver, build, patch); + + ret += 1; /* add the size of '\0' */ + if (fw_size < (u32)ret) + return ret; + else + return 0; +} + static void i40e_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) { struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct i40e_vsi *vsi = pf->main_vsi; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + dev_info->pci_dev = pci_dev; dev_info->max_rx_queues = vsi->nb_qps; dev_info->max_tx_queues = vsi->nb_qps; dev_info->min_rx_bufsize = I40E_BUF_SIZE_MIN; dev_info->max_rx_pktlen = I40E_FRAME_SIZE_MAX; dev_info->max_mac_addrs = vsi->max_macaddrs; - dev_info->max_vfs = dev->pci_dev->max_vfs; + dev_info->max_vfs = pci_dev->max_vfs; dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP | DEV_RX_OFFLOAD_QINQ_STRIP | @@ -2605,11 +3197,15 @@ i40e_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) DEV_TX_OFFLOAD_TCP_CKSUM | DEV_TX_OFFLOAD_SCTP_CKSUM | DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM | - DEV_TX_OFFLOAD_TCP_TSO; + DEV_TX_OFFLOAD_TCP_TSO | + DEV_TX_OFFLOAD_VXLAN_TNL_TSO | + DEV_TX_OFFLOAD_GRE_TNL_TSO | + DEV_TX_OFFLOAD_IPIP_TNL_TSO | + DEV_TX_OFFLOAD_GENEVE_TNL_TSO; dev_info->hash_key_size = (I40E_PFQF_HKEY_MAX_INDEX + 1) * sizeof(uint32_t); dev_info->reta_size = pf->hash_lut_size; - dev_info->flow_type_rss_offloads = I40E_RSS_OFFLOAD_ALL; + dev_info->flow_type_rss_offloads = pf->adapter->flow_types_mask; dev_info->default_rxconf = (struct rte_eth_rxconf) { .rx_thresh = { @@ -2643,6 +3239,8 @@ i40e_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) .nb_max = I40E_MAX_RING_DESC, .nb_min = I40E_MIN_RING_DESC, .nb_align = I40E_ALIGN_RING_DESC, + .nb_seg_max = I40E_TX_MAX_SEG, + .nb_mtu_seg_max = I40E_TX_MAX_MTU_SEG, }; if (pf->flags & I40E_FLAG_VMDQ) { @@ -2655,9 +3253,12 @@ i40e_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) dev_info->max_tx_queues += dev_info->vmdq_queue_num; } - if (i40e_is_40G_device(hw->device_id)) + if (I40E_PHY_TYPE_SUPPORT_40G(hw->phy.phy_types)) /* For XL710 */ dev_info->speed_capa = ETH_LINK_SPEED_40G; + else if (I40E_PHY_TYPE_SUPPORT_25G(hw->phy.phy_types)) + /* For XXV710 */ + dev_info->speed_capa = ETH_LINK_SPEED_25G; else /* For X710 */ dev_info->speed_capa = ETH_LINK_SPEED_1G | ETH_LINK_SPEED_10G; @@ -2677,72 +3278,106 @@ i40e_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) } static int -i40e_vlan_tpid_set(struct rte_eth_dev *dev, - enum rte_vlan_type vlan_type, - uint16_t tpid) +i40e_vlan_tpid_set_by_registers(struct rte_eth_dev *dev, + enum rte_vlan_type vlan_type, + uint16_t tpid, int qinq) { struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); - uint64_t reg_r = 0, reg_w = 0; - uint16_t reg_id = 0; - int ret = 0; - int qinq = dev->data->dev_conf.rxmode.hw_vlan_extend; + uint64_t reg_r = 0; + uint64_t reg_w = 0; + uint16_t reg_id = 3; + int ret; - switch (vlan_type) { - case ETH_VLAN_TYPE_OUTER: - if (qinq) + if (qinq) { + if (vlan_type == ETH_VLAN_TYPE_OUTER) reg_id = 2; - else - reg_id = 3; - break; - case ETH_VLAN_TYPE_INNER: - if (qinq) - reg_id = 3; - else { - ret = -EINVAL; - PMD_DRV_LOG(ERR, - "Unsupported vlan type in single vlan.\n"); - return ret; - } - break; - default: - ret = -EINVAL; - PMD_DRV_LOG(ERR, "Unsupported vlan type %d", vlan_type); - return ret; } + ret = i40e_aq_debug_read_register(hw, I40E_GL_SWT_L2TAGCTRL(reg_id), ®_r, NULL); if (ret != I40E_SUCCESS) { - PMD_DRV_LOG(ERR, "Fail to debug read from " - "I40E_GL_SWT_L2TAGCTRL[%d]", reg_id); - ret = -EIO; - return ret; + PMD_DRV_LOG(ERR, + "Fail to debug read from I40E_GL_SWT_L2TAGCTRL[%d]", + reg_id); + return -EIO; } - PMD_DRV_LOG(DEBUG, "Debug read from I40E_GL_SWT_L2TAGCTRL[%d]: " - "0x%08"PRIx64"", reg_id, reg_r); + PMD_DRV_LOG(DEBUG, + "Debug read from I40E_GL_SWT_L2TAGCTRL[%d]: 0x%08"PRIx64, + reg_id, reg_r); reg_w = reg_r & (~(I40E_GL_SWT_L2TAGCTRL_ETHERTYPE_MASK)); reg_w |= ((uint64_t)tpid << I40E_GL_SWT_L2TAGCTRL_ETHERTYPE_SHIFT); if (reg_r == reg_w) { - ret = 0; PMD_DRV_LOG(DEBUG, "No need to write"); - return ret; + return 0; } ret = i40e_aq_debug_write_register(hw, I40E_GL_SWT_L2TAGCTRL(reg_id), reg_w, NULL); if (ret != I40E_SUCCESS) { - ret = -EIO; - PMD_DRV_LOG(ERR, "Fail to debug write to " - "I40E_GL_SWT_L2TAGCTRL[%d]", reg_id); - return ret; + PMD_DRV_LOG(ERR, + "Fail to debug write to I40E_GL_SWT_L2TAGCTRL[%d]", + reg_id); + return -EIO; } - PMD_DRV_LOG(DEBUG, "Debug write 0x%08"PRIx64" to " - "I40E_GL_SWT_L2TAGCTRL[%d]", reg_w, reg_id); + PMD_DRV_LOG(DEBUG, + "Global register 0x%08x is changed with value 0x%08x", + I40E_GL_SWT_L2TAGCTRL(reg_id), (uint32_t)reg_w); + + return 0; +} + +static int +i40e_vlan_tpid_set(struct rte_eth_dev *dev, + enum rte_vlan_type vlan_type, + uint16_t tpid) +{ + struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + int qinq = dev->data->dev_conf.rxmode.hw_vlan_extend; + int ret = 0; + + if ((vlan_type != ETH_VLAN_TYPE_INNER && + vlan_type != ETH_VLAN_TYPE_OUTER) || + (!qinq && vlan_type == ETH_VLAN_TYPE_INNER)) { + PMD_DRV_LOG(ERR, + "Unsupported vlan type."); + return -EINVAL; + } + + if (pf->support_multi_driver) { + PMD_DRV_LOG(ERR, "Setting TPID is not supported."); + return -ENOTSUP; + } + + /* 802.1ad frames ability is added in NVM API 1.7*/ + if (hw->flags & I40E_HW_FLAG_802_1AD_CAPABLE) { + if (qinq) { + if (vlan_type == ETH_VLAN_TYPE_OUTER) + hw->first_tag = rte_cpu_to_le_16(tpid); + else if (vlan_type == ETH_VLAN_TYPE_INNER) + hw->second_tag = rte_cpu_to_le_16(tpid); + } else { + if (vlan_type == ETH_VLAN_TYPE_OUTER) + hw->second_tag = rte_cpu_to_le_16(tpid); + } + ret = i40e_aq_set_switch_config(hw, 0, 0, NULL); + if (ret != I40E_SUCCESS) { + PMD_DRV_LOG(ERR, + "Set switch config failed aq_err: %d", + hw->aq.asq_last_status); + ret = -EIO; + } + } else + /* If NVM API < 1.7, keep the register setting */ + ret = i40e_vlan_tpid_set_by_registers(dev, vlan_type, + tpid, qinq); + i40e_global_cfg_warning(I40E_WARNING_TPID); return ret; } -static void +static int i40e_vlan_offload_set(struct rte_eth_dev *dev, int mask) { struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); @@ -2766,7 +3401,7 @@ i40e_vlan_offload_set(struct rte_eth_dev *dev, int mask) if (mask & ETH_VLAN_EXTEND_MASK) { if (dev->data->dev_conf.rxmode.hw_vlan_extend) { i40e_vsi_config_double_vlan(vsi, TRUE); - /* Set global registers with default ether type value */ + /* Set global registers with default ethertype. */ i40e_vlan_tpid_set(dev, ETH_VLAN_TYPE_OUTER, ETHER_TYPE_VLAN); i40e_vlan_tpid_set(dev, ETH_VLAN_TYPE_INNER, @@ -2775,6 +3410,8 @@ i40e_vlan_offload_set(struct rte_eth_dev *dev, int mask) else i40e_vsi_config_double_vlan(vsi, FALSE); } + + return 0; } static void @@ -2838,6 +3475,13 @@ i40e_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); fc_conf->pause_time = pf->fc_conf.pause_time; + + /* read out from register, in case they are modified by other port */ + pf->fc_conf.high_water[I40E_MAX_TRAFFIC_CLASS] = + I40E_READ_REG(hw, I40E_GLRPB_GHW) >> I40E_KILOSHIFT; + pf->fc_conf.low_water[I40E_MAX_TRAFFIC_CLASS] = + I40E_READ_REG(hw, I40E_GLRPB_GLW) >> I40E_KILOSHIFT; + fc_conf->high_water = pf->fc_conf.high_water[I40E_MAX_TRAFFIC_CLASS]; fc_conf->low_water = pf->fc_conf.low_water[I40E_MAX_TRAFFIC_CLASS]; @@ -2881,8 +3525,9 @@ i40e_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) max_high_water = I40E_RXPBSIZE >> I40E_KILOSHIFT; if ((fc_conf->high_water > max_high_water) || (fc_conf->high_water < fc_conf->low_water)) { - PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB, " - "High_water must <= %d.", max_high_water); + PMD_INIT_LOG(ERR, + "Invalid high/low water setup value in KB, High_water must be <= %d.", + max_high_water); return -EINVAL; } @@ -2903,7 +3548,7 @@ i40e_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) if (err < 0) return -ENOSYS; - if (i40e_is_40G_device(hw->device_id)) { + if (I40E_PHY_TYPE_SUPPORT_40G(hw->phy.phy_types)) { /* Configure flow control refresh threshold, * the value for stat_tx_pause_refresh_timer[8] * is used for global pause operation. @@ -2957,19 +3602,25 @@ i40e_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) I40E_WRITE_REG(hw, I40E_PRTDCB_MFLCN, mflcn_reg); } - /* config the water marker both based on the packets and bytes */ - I40E_WRITE_REG(hw, I40E_GLRPB_PHW, - (pf->fc_conf.high_water[I40E_MAX_TRAFFIC_CLASS] - << I40E_KILOSHIFT) / I40E_PACKET_AVERAGE_SIZE); - I40E_WRITE_REG(hw, I40E_GLRPB_PLW, - (pf->fc_conf.low_water[I40E_MAX_TRAFFIC_CLASS] - << I40E_KILOSHIFT) / I40E_PACKET_AVERAGE_SIZE); - I40E_WRITE_REG(hw, I40E_GLRPB_GHW, - pf->fc_conf.high_water[I40E_MAX_TRAFFIC_CLASS] - << I40E_KILOSHIFT); - I40E_WRITE_REG(hw, I40E_GLRPB_GLW, - pf->fc_conf.low_water[I40E_MAX_TRAFFIC_CLASS] - << I40E_KILOSHIFT); + if (!pf->support_multi_driver) { + /* config water marker both based on the packets and bytes */ + I40E_WRITE_GLB_REG(hw, I40E_GLRPB_PHW, + (pf->fc_conf.high_water[I40E_MAX_TRAFFIC_CLASS] + << I40E_KILOSHIFT) / I40E_PACKET_AVERAGE_SIZE); + I40E_WRITE_GLB_REG(hw, I40E_GLRPB_PLW, + (pf->fc_conf.low_water[I40E_MAX_TRAFFIC_CLASS] + << I40E_KILOSHIFT) / I40E_PACKET_AVERAGE_SIZE); + I40E_WRITE_GLB_REG(hw, I40E_GLRPB_GHW, + pf->fc_conf.high_water[I40E_MAX_TRAFFIC_CLASS] + << I40E_KILOSHIFT); + I40E_WRITE_GLB_REG(hw, I40E_GLRPB_GLW, + pf->fc_conf.low_water[I40E_MAX_TRAFFIC_CLASS] + << I40E_KILOSHIFT); + i40e_global_cfg_warning(I40E_WARNING_FLOW_CTL); + } else { + PMD_DRV_LOG(ERR, + "Water marker configuration is not supported."); + } I40E_WRITE_FLUSH(hw); @@ -2986,7 +3637,7 @@ i40e_priority_flow_ctrl_set(__rte_unused struct rte_eth_dev *dev, } /* Add a MAC address, and update filters */ -static void +static int i40e_macaddr_add(struct rte_eth_dev *dev, struct ether_addr *mac_addr, __rte_unused uint32_t index, @@ -3003,16 +3654,16 @@ i40e_macaddr_add(struct rte_eth_dev *dev, PMD_DRV_LOG(ERR, "VMDQ not %s, can't set mac to pool %u", pf->flags & I40E_FLAG_VMDQ ? "configured" : "enabled", pool); - return; + return -ENOTSUP; } if (pool > pf->nb_cfg_vmdq_vsi) { PMD_DRV_LOG(ERR, "Pool number %u invalid. Max pool is %u", pool, pf->nb_cfg_vmdq_vsi); - return; + return -EINVAL; } - (void)rte_memcpy(&mac_filter.mac_addr, mac_addr, ETHER_ADDR_LEN); + rte_memcpy(&mac_filter.mac_addr, mac_addr, ETHER_ADDR_LEN); if (dev->data->dev_conf.rxmode.hw_vlan_filter) mac_filter.filter_type = RTE_MACVLAN_PERFECT_MATCH; else @@ -3026,8 +3677,9 @@ i40e_macaddr_add(struct rte_eth_dev *dev, ret = i40e_vsi_add_mac(vsi, &mac_filter); if (ret != I40E_SUCCESS) { PMD_DRV_LOG(ERR, "Failed to add MACVLAN filter"); - return; + return -ENODEV; } + return 0; } /* Remove a MAC address, and update filters */ @@ -3054,8 +3706,8 @@ i40e_macaddr_remove(struct rte_eth_dev *dev, uint32_t index) /* No VMDQ pool enabled or configured */ if (!(pf->flags & I40E_FLAG_VMDQ) || (i > pf->nb_cfg_vmdq_vsi)) { - PMD_DRV_LOG(ERR, "No VMDQ pool enabled" - "/configured"); + PMD_DRV_LOG(ERR, + "No VMDQ pool enabled/configured"); return; } vsi = pf->vmdq[i - 1].vsi; @@ -3116,10 +3768,10 @@ i40e_vf_mac_filter_set(struct i40e_pf *pf, } if (add) { - (void)rte_memcpy(&old_mac, hw->mac.addr, ETHER_ADDR_LEN); - (void)rte_memcpy(hw->mac.addr, new_mac->addr_bytes, + rte_memcpy(&old_mac, hw->mac.addr, ETHER_ADDR_LEN); + rte_memcpy(hw->mac.addr, new_mac->addr_bytes, ETHER_ADDR_LEN); - (void)rte_memcpy(&mac_filter.mac_addr, &filter->mac_addr, + rte_memcpy(&mac_filter.mac_addr, &filter->mac_addr, ETHER_ADDR_LEN); mac_filter.filter_type = filter->filter_type; @@ -3130,7 +3782,7 @@ i40e_vf_mac_filter_set(struct i40e_pf *pf, } ether_addr_copy(new_mac, &pf->dev_addr); } else { - (void)rte_memcpy(hw->mac.addr, hw->mac.perm_addr, + rte_memcpy(hw->mac.addr, hw->mac.perm_addr, ETHER_ADDR_LEN); ret = i40e_vsi_delete_mac(vf->vsi, &filter->mac_addr); if (ret != I40E_SUCCESS) { @@ -3256,9 +3908,9 @@ i40e_dev_rss_reta_update(struct rte_eth_dev *dev, if (reta_size != lut_size || reta_size > ETH_RSS_RETA_SIZE_512) { - PMD_DRV_LOG(ERR, "The size of hash lookup table configured " - "(%d) doesn't match the number hardware can supported " - "(%d)\n", reta_size, lut_size); + PMD_DRV_LOG(ERR, + "The size of hash lookup table configured (%d) doesn't match the number hardware can supported (%d)", + reta_size, lut_size); return -EINVAL; } @@ -3297,9 +3949,9 @@ i40e_dev_rss_reta_query(struct rte_eth_dev *dev, if (reta_size != lut_size || reta_size > ETH_RSS_RETA_SIZE_512) { - PMD_DRV_LOG(ERR, "The size of hash lookup table configured " - "(%d) doesn't match the number hardware can supported " - "(%d)\n", reta_size, lut_size); + PMD_DRV_LOG(ERR, + "The size of hash lookup table configured (%d) doesn't match the number hardware can supported (%d)", + reta_size, lut_size); return -EINVAL; } @@ -3352,10 +4004,11 @@ i40e_allocate_dma_mem_d(__attribute__((unused)) struct i40e_hw *hw, mem->size = size; mem->va = mz->addr; - mem->pa = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr); + mem->pa = mz->iova; mem->zone = (const void *)mz; - PMD_DRV_LOG(DEBUG, "memzone %s allocated with physical address: " - "%"PRIu64, mz->name, mem->pa); + PMD_DRV_LOG(DEBUG, + "memzone %s allocated with physical address: %"PRIu64, + mz->name, mem->pa); return I40E_SUCCESS; } @@ -3372,9 +4025,9 @@ i40e_free_dma_mem_d(__attribute__((unused)) struct i40e_hw *hw, if (!mem) return I40E_ERR_PARAM; - PMD_DRV_LOG(DEBUG, "memzone %s to be freed with physical address: " - "%"PRIu64, ((const struct rte_memzone *)mem->zone)->name, - mem->pa); + PMD_DRV_LOG(DEBUG, + "memzone %s to be freed with physical address: %"PRIu64, + ((const struct rte_memzone *)mem->zone)->name, mem->pa); rte_memzone_free((const struct rte_memzone *)mem->zone); mem->zone = NULL; mem->va = NULL; @@ -3485,9 +4138,10 @@ i40e_pf_parameter_init(struct rte_eth_dev *dev) { struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); struct i40e_hw *hw = I40E_PF_TO_HW(pf); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); uint16_t qp_count = 0, vsi_count = 0; - if (dev->pci_dev->max_vfs && !hw->func_caps.sr_iov_1_1) { + if (pci_dev->max_vfs && !hw->func_caps.sr_iov_1_1) { PMD_INIT_LOG(ERR, "HW configuration doesn't support SRIOV"); return -EINVAL; } @@ -3528,13 +4182,13 @@ i40e_pf_parameter_init(struct rte_eth_dev *dev) /* VF queue/VSI allocation */ pf->vf_qp_offset = pf->lan_qp_offset + pf->lan_nb_qps; - if (hw->func_caps.sr_iov_1_1 && dev->pci_dev->max_vfs) { + if (hw->func_caps.sr_iov_1_1 && pci_dev->max_vfs) { pf->flags |= I40E_FLAG_SRIOV; pf->vf_nb_qps = RTE_LIBRTE_I40E_QUEUE_NUM_PER_VF; - pf->vf_num = dev->pci_dev->max_vfs; - PMD_DRV_LOG(DEBUG, "%u VF VSIs, %u queues per VF VSI, " - "in total %u queues", pf->vf_num, pf->vf_nb_qps, - pf->vf_nb_qps * pf->vf_num); + pf->vf_num = pci_dev->max_vfs; + PMD_DRV_LOG(DEBUG, + "%u VF VSIs, %u queues per VF VSI, in total %u queues", + pf->vf_num, pf->vf_nb_qps, pf->vf_nb_qps * pf->vf_num); } else { pf->vf_nb_qps = 0; pf->vf_num = 0; @@ -3562,14 +4216,13 @@ i40e_pf_parameter_init(struct rte_eth_dev *dev) if (pf->max_nb_vmdq_vsi) { pf->flags |= I40E_FLAG_VMDQ; pf->vmdq_nb_qps = pf->vmdq_nb_qp_max; - PMD_DRV_LOG(DEBUG, "%u VMDQ VSIs, %u queues " - "per VMDQ VSI, in total %u queues", - pf->max_nb_vmdq_vsi, - pf->vmdq_nb_qps, pf->vmdq_nb_qps * - pf->max_nb_vmdq_vsi); + PMD_DRV_LOG(DEBUG, + "%u VMDQ VSIs, %u queues per VMDQ VSI, in total %u queues", + pf->max_nb_vmdq_vsi, pf->vmdq_nb_qps, + pf->vmdq_nb_qps * pf->max_nb_vmdq_vsi); } else { - PMD_DRV_LOG(INFO, "No enough queues left for " - "VMDq"); + PMD_DRV_LOG(INFO, + "No enough queues left for VMDq"); } } else { PMD_DRV_LOG(INFO, "No queue or VSI left for VMDq"); @@ -3582,15 +4235,15 @@ i40e_pf_parameter_init(struct rte_eth_dev *dev) pf->flags |= I40E_FLAG_DCB; if (qp_count > hw->func_caps.num_tx_qp) { - PMD_DRV_LOG(ERR, "Failed to allocate %u queues, which exceeds " - "the hardware maximum %u", qp_count, - hw->func_caps.num_tx_qp); + PMD_DRV_LOG(ERR, + "Failed to allocate %u queues, which exceeds the hardware maximum %u", + qp_count, hw->func_caps.num_tx_qp); return -EINVAL; } if (vsi_count > hw->func_caps.num_vsis) { - PMD_DRV_LOG(ERR, "Failed to allocate %u VSIs, which exceeds " - "the hardware maximum %u", vsi_count, - hw->func_caps.num_vsis); + PMD_DRV_LOG(ERR, + "Failed to allocate %u VSIs, which exceeds the hardware maximum %u", + vsi_count, hw->func_caps.num_vsis); return -EINVAL; } @@ -3836,8 +4489,8 @@ i40e_res_pool_alloc(struct i40e_res_pool_info *pool, */ entry = rte_zmalloc("res_pool", sizeof(*entry), 0); if (entry == NULL) { - PMD_DRV_LOG(ERR, "Failed to allocate memory for " - "resource pool"); + PMD_DRV_LOG(ERR, + "Failed to allocate memory for resource pool"); return -ENOMEM; } entry->base = valid_entry->base; @@ -3877,9 +4530,9 @@ validate_tcmap_parameter(struct i40e_vsi *vsi, uint8_t enabled_tcmap) } if (!bitmap_is_subset(hw->func_caps.enabled_tcmap, enabled_tcmap)) { - PMD_DRV_LOG(ERR, "Enabled TC map 0x%x not applicable to " - "HW support 0x%x", hw->func_caps.enabled_tcmap, - enabled_tcmap); + PMD_DRV_LOG(ERR, + "Enabled TC map 0x%x not applicable to HW support 0x%x", + hw->func_caps.enabled_tcmap, enabled_tcmap); return I40E_NOT_SUPPORTED; } return I40E_SUCCESS; @@ -3921,7 +4574,7 @@ i40e_vsi_vlan_pvid_set(struct i40e_vsi *vsi, vsi->info.valid_sections = rte_cpu_to_le_16(I40E_AQ_VSI_PROP_VLAN_VALID); memset(&ctxt, 0, sizeof(ctxt)); - (void)rte_memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info)); + rte_memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info)); ctxt.seid = vsi->seid; hw = I40E_VSI_TO_HW(vsi); @@ -3960,7 +4613,7 @@ i40e_vsi_update_tc_bandwidth(struct i40e_vsi *vsi, uint8_t enabled_tcmap) return ret; } - (void)rte_memcpy(vsi->info.qs_handle, tc_bw_data.qs_handles, + rte_memcpy(vsi->info.qs_handle, tc_bw_data.qs_handles, sizeof(vsi->info.qs_handle)); return I40E_SUCCESS; } @@ -3981,6 +4634,8 @@ i40e_vsi_config_tc_queue_mapping(struct i40e_vsi *vsi, for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) if (enabled_tcmap & (1 << i)) total_tc++; + if (total_tc == 0) + total_tc = 1; vsi->enabled_tc = enabled_tcmap; /* Number of queues per enabled TC */ @@ -4097,12 +4752,13 @@ i40e_veb_setup(struct i40e_pf *pf, struct i40e_vsi *vsi) hw->aq.asq_last_status); goto fail; } + veb->enabled_tc = I40E_DEFAULT_TCMAP; /* get statistics index */ ret = i40e_aq_get_veb_parameters(hw, veb->seid, NULL, NULL, &veb->stats_idx, NULL, NULL, NULL); if (ret != I40E_SUCCESS) { - PMD_DRV_LOG(ERR, "Get veb statics index failed, aq_err: %d", + PMD_DRV_LOG(ERR, "Get veb statistics index failed, aq_err: %d", hw->aq.asq_last_status); goto fail; } @@ -4131,6 +4787,9 @@ i40e_vsi_release(struct i40e_vsi *vsi) if (!vsi) return I40E_SUCCESS; + if (!vsi->adapter) + return -EFAULT; + user_param = vsi->user_param; pf = I40E_VSI_TO_PF(vsi); @@ -4211,7 +4870,7 @@ i40e_update_default_filter_setting(struct i40e_vsi *vsi) if (vsi->type != I40E_VSI_MAIN) return I40E_ERR_CONFIG; memset(&def_filter, 0, sizeof(def_filter)); - (void)rte_memcpy(def_filter.mac_addr, hw->mac.perm_addr, + rte_memcpy(def_filter.mac_addr, hw->mac.perm_addr, ETH_ADDR_LEN); def_filter.vlan_tag = 0; def_filter.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH | @@ -4221,8 +4880,8 @@ i40e_update_default_filter_setting(struct i40e_vsi *vsi) struct i40e_mac_filter *f; struct ether_addr *mac; - PMD_DRV_LOG(WARNING, "Cannot remove the default " - "macvlan filter"); + PMD_DRV_LOG(DEBUG, + "Cannot remove the default macvlan filter"); /* It needs to add the permanent mac into mac list */ f = rte_zmalloc("macv_filter", sizeof(*f), 0); if (f == NULL) { @@ -4230,7 +4889,7 @@ i40e_update_default_filter_setting(struct i40e_vsi *vsi) return I40E_ERR_NO_MEMORY; } mac = &f->mac_info.mac_addr; - (void)rte_memcpy(&mac->addr_bytes, hw->mac.perm_addr, + rte_memcpy(&mac->addr_bytes, hw->mac.perm_addr, ETH_ADDR_LEN); f->mac_info.filter_type = RTE_MACVLAN_PERFECT_MATCH; TAILQ_INSERT_TAIL(&vsi->mac_list, f, next); @@ -4238,7 +4897,7 @@ i40e_update_default_filter_setting(struct i40e_vsi *vsi) return ret; } - (void)rte_memcpy(&filter.mac_addr, + rte_memcpy(&filter.mac_addr, (struct ether_addr *)(hw->mac.perm_addr), ETH_ADDR_LEN); filter.filter_type = RTE_MACVLAN_PERFECT_MATCH; return i40e_vsi_add_mac(vsi, &filter); @@ -4272,8 +4931,9 @@ i40e_vsi_get_bw_config(struct i40e_vsi *vsi) ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid, &ets_sla_config, NULL); if (ret != I40E_SUCCESS) { - PMD_DRV_LOG(ERR, "VSI failed to get TC bandwdith " - "configuration %u", hw->aq.asq_last_status); + PMD_DRV_LOG(ERR, + "VSI failed to get TC bandwdith configuration %u", + hw->aq.asq_last_status); return ret; } @@ -4340,7 +5000,7 @@ i40e_enable_pf_lb(struct i40e_pf *pf) ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); if (ret) - PMD_DRV_LOG(ERR, "update vsi switch failed, aq_err=%d\n", + PMD_DRV_LOG(ERR, "update vsi switch failed, aq_err=%d", hw->aq.asq_last_status); } @@ -4361,14 +5021,14 @@ i40e_vsi_setup(struct i40e_pf *pf, if (type != I40E_VSI_MAIN && type != I40E_VSI_SRIOV && uplink_vsi == NULL) { - PMD_DRV_LOG(ERR, "VSI setup failed, " - "VSI link shouldn't be NULL"); + PMD_DRV_LOG(ERR, + "VSI setup failed, VSI link shouldn't be NULL"); return NULL; } if (type == I40E_VSI_MAIN && uplink_vsi != NULL) { - PMD_DRV_LOG(ERR, "VSI setup failed, MAIN VSI " - "uplink VSI should be NULL"); + PMD_DRV_LOG(ERR, + "VSI setup failed, MAIN VSI uplink VSI should be NULL"); return NULL; } @@ -4412,6 +5072,8 @@ i40e_vsi_setup(struct i40e_pf *pf, vsi->max_macaddrs = I40E_NUM_MACADDR_MAX; vsi->parent_vsi = uplink_vsi ? uplink_vsi : pf->main_vsi; vsi->user_param = user_param; + vsi->vlan_anti_spoof_on = 0; + vsi->vlan_filter_on = 0; /* Allocate queues */ switch (vsi->type) { case I40E_VSI_MAIN : @@ -4450,16 +5112,28 @@ i40e_vsi_setup(struct i40e_pf *pf, /* VF has MSIX interrupt in VF range, don't allocate here */ if (type == I40E_VSI_MAIN) { - ret = i40e_res_pool_alloc(&pf->msix_pool, - RTE_MIN(vsi->nb_qps, - RTE_MAX_RXTX_INTR_VEC_ID)); - if (ret < 0) { - PMD_DRV_LOG(ERR, "VSI MAIN %d get heap failed %d", - vsi->seid, ret); - goto fail_queue_alloc; + if (pf->support_multi_driver) { + /* If support multi-driver, need to use INT0 instead of + * allocating from msix pool. The Msix pool is init from + * INT1, so it's OK just set msix_intr to 0 and nb_msix + * to 1 without calling i40e_res_pool_alloc. + */ + vsi->msix_intr = 0; + vsi->nb_msix = 1; + } else { + ret = i40e_res_pool_alloc(&pf->msix_pool, + RTE_MIN(vsi->nb_qps, + RTE_MAX_RXTX_INTR_VEC_ID)); + if (ret < 0) { + PMD_DRV_LOG(ERR, + "VSI MAIN %d get heap failed %d", + vsi->seid, ret); + goto fail_queue_alloc; + } + vsi->msix_intr = ret; + vsi->nb_msix = RTE_MIN(vsi->nb_qps, + RTE_MAX_RXTX_INTR_VEC_ID); } - vsi->msix_intr = ret; - vsi->nb_msix = RTE_MIN(vsi->nb_qps, RTE_MAX_RXTX_INTR_VEC_ID); } else if (type != I40E_VSI_SRIOV) { ret = i40e_res_pool_alloc(&pf->msix_pool, 1); if (ret < 0) { @@ -4496,7 +5170,7 @@ i40e_vsi_setup(struct i40e_pf *pf, PMD_DRV_LOG(ERR, "Failed to get VSI params"); goto fail_msix_alloc; } - (void)rte_memcpy(&vsi->info, &ctxt.info, + rte_memcpy(&vsi->info, &ctxt.info, sizeof(struct i40e_aqc_vsi_properties_data)); vsi->vsi_id = ctxt.vsi_number; vsi->info.valid_sections = 0; @@ -4514,13 +5188,13 @@ i40e_vsi_setup(struct i40e_pf *pf, rte_cpu_to_le_16(I40E_AQ_VSI_PROP_VLAN_VALID); vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL | I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH; - (void)rte_memcpy(&ctxt.info, &vsi->info, + rte_memcpy(&ctxt.info, &vsi->info, sizeof(struct i40e_aqc_vsi_properties_data)); ret = i40e_vsi_config_tc_queue_mapping(vsi, &ctxt.info, I40E_DEFAULT_TCMAP); if (ret != I40E_SUCCESS) { - PMD_DRV_LOG(ERR, "Failed to configure " - "TC queue mapping"); + PMD_DRV_LOG(ERR, + "Failed to configure TC queue mapping"); goto fail_msix_alloc; } ctxt.seid = vsi->seid; @@ -4535,15 +5209,15 @@ i40e_vsi_setup(struct i40e_pf *pf, goto fail_msix_alloc; } - (void)rte_memcpy(&vsi->info.tc_mapping, &ctxt.info.tc_mapping, + rte_memcpy(&vsi->info.tc_mapping, &ctxt.info.tc_mapping, sizeof(vsi->info.tc_mapping)); - (void)rte_memcpy(&vsi->info.queue_mapping, + rte_memcpy(&vsi->info.queue_mapping, &ctxt.info.queue_mapping, sizeof(vsi->info.queue_mapping)); vsi->info.mapping_flags = ctxt.info.mapping_flags; vsi->info.valid_sections = 0; - (void)rte_memcpy(pf->dev_addr.addr_bytes, hw->mac.perm_addr, + rte_memcpy(pf->dev_addr.addr_bytes, hw->mac.perm_addr, ETH_ADDR_LEN); /** @@ -4588,13 +5262,14 @@ i40e_vsi_setup(struct i40e_pf *pf, rte_cpu_to_le_16(I40E_AQ_VSI_PROP_VLAN_VALID); ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL; ret = i40e_vsi_config_tc_queue_mapping(vsi, &ctxt.info, - I40E_DEFAULT_TCMAP); + hw->func_caps.enabled_tcmap); if (ret != I40E_SUCCESS) { - PMD_DRV_LOG(ERR, "Failed to configure " - "TC queue mapping"); + PMD_DRV_LOG(ERR, + "Failed to configure TC queue mapping"); goto fail_msix_alloc; } - ctxt.info.up_enable_bits = I40E_DEFAULT_TCMAP; + + ctxt.info.up_enable_bits = hw->func_caps.enabled_tcmap; ctxt.info.valid_sections |= rte_cpu_to_le_16(I40E_AQ_VSI_PROP_SCHED_VALID); /** @@ -4633,8 +5308,8 @@ i40e_vsi_setup(struct i40e_pf *pf, ret = i40e_vsi_config_tc_queue_mapping(vsi, &ctxt.info, I40E_DEFAULT_TCMAP); if (ret != I40E_SUCCESS) { - PMD_DRV_LOG(ERR, "Failed to configure " - "TC queue mapping"); + PMD_DRV_LOG(ERR, + "Failed to configure TC queue mapping"); goto fail_msix_alloc; } ctxt.info.up_enable_bits = I40E_DEFAULT_TCMAP; @@ -4651,8 +5326,8 @@ i40e_vsi_setup(struct i40e_pf *pf, ret = i40e_vsi_config_tc_queue_mapping(vsi, &ctxt.info, I40E_DEFAULT_TCMAP); if (ret != I40E_SUCCESS) { - PMD_DRV_LOG(ERR, "Failed to configure " - "TC queue mapping."); + PMD_DRV_LOG(ERR, + "Failed to configure TC queue mapping."); goto fail_msix_alloc; } ctxt.info.up_enable_bits = I40E_DEFAULT_TCMAP; @@ -4685,7 +5360,7 @@ i40e_vsi_setup(struct i40e_pf *pf, } /* MAC/VLAN configuration */ - (void)rte_memcpy(&filter.mac_addr, &broadcast, ETHER_ADDR_LEN); + rte_memcpy(&filter.mac_addr, &broadcast, ETHER_ADDR_LEN); filter.filter_type = RTE_MACVLAN_PERFECT_MATCH; ret = i40e_vsi_add_mac(vsi, &filter); @@ -4797,7 +5472,7 @@ i40e_vsi_config_vlan_stripping(struct i40e_vsi *vsi, bool on) vsi->info.port_vlan_flags &= ~(I40E_AQ_VSI_PVLAN_EMOD_MASK); vsi->info.port_vlan_flags |= vlan_flags; ctxt.seid = vsi->seid; - (void)rte_memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info)); + rte_memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info)); ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); if (ret) PMD_DRV_LOG(INFO, "Update VSI failed to %s vlan stripping", @@ -4814,10 +5489,14 @@ i40e_dev_init_vlan(struct rte_eth_dev *dev) int mask = 0; /* Apply vlan offload setting */ - mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK; - i40e_vlan_offload_set(dev, mask); - - /* Apply double-vlan setting, not implemented yet */ + mask = ETH_VLAN_STRIP_MASK | + ETH_VLAN_FILTER_MASK | + ETH_VLAN_EXTEND_MASK; + ret = i40e_vlan_offload_set(dev, mask); + if (ret) { + PMD_DRV_LOG(INFO, "Failed to update vlan offload"); + return ret; + } /* Apply pvid setting */ ret = i40e_vlan_pvid_set(dev, data->dev_conf.txmode.pvid, @@ -4905,6 +5584,8 @@ i40e_pf_setup(struct i40e_pf *pf) pf->offset_loaded = FALSE; memset(&pf->stats, 0, sizeof(struct i40e_hw_port_stats)); memset(&pf->stats_offset, 0, sizeof(struct i40e_hw_port_stats)); + memset(&pf->internal_stats, 0, sizeof(struct i40e_eth_stats)); + memset(&pf->internal_stats_offset, 0, sizeof(struct i40e_eth_stats)); ret = i40e_pf_get_switch_config(pf); if (ret != I40E_SUCCESS) { @@ -4915,8 +5596,9 @@ i40e_pf_setup(struct i40e_pf *pf) /* make queue allocated first, let FDIR use queue pair 0*/ ret = i40e_res_pool_alloc(&pf->qp_pool, I40E_DEFAULT_QP_NUM_FDIR); if (ret != I40E_FDIR_QUEUE_ID) { - PMD_DRV_LOG(ERR, "queue allocation fails for FDIR :" - " ret =%d", ret); + PMD_DRV_LOG(ERR, + "queue allocation fails for FDIR: ret =%d", + ret); pf->flags &= ~I40E_FLAG_FDIR; } } @@ -4935,12 +5617,12 @@ i40e_pf_setup(struct i40e_pf *pf) else if (hw->func_caps.rss_table_size == ETH_RSS_RETA_SIZE_512) settings.hash_lut_size = I40E_HASH_LUT_SIZE_512; else { - PMD_DRV_LOG(ERR, "Hash lookup table size (%u) not supported\n", - hw->func_caps.rss_table_size); + PMD_DRV_LOG(ERR, "Hash lookup table size (%u) not supported", + hw->func_caps.rss_table_size); return I40E_ERR_PARAM; } - PMD_DRV_LOG(INFO, "Hardware capability of hash lookup table " - "size: %u\n", hw->func_caps.rss_table_size); + PMD_DRV_LOG(INFO, "Hardware capability of hash lookup table size: %u", + hw->func_caps.rss_table_size); pf->hash_lut_size = hw->func_caps.rss_table_size; /* Enable ethtype and macvlan filters */ @@ -5190,8 +5872,8 @@ i40e_dev_rx_init(struct i40e_pf *pf) ret = i40e_rx_queue_init(rxq); if (ret != I40E_SUCCESS) { - PMD_DRV_LOG(ERR, "Failed to do RX queue " - "initialization"); + PMD_DRV_LOG(ERR, + "Failed to do RX queue initialization"); break; } } @@ -5368,7 +6050,8 @@ void i40e_pf_disable_irq0(struct i40e_hw *hw) { /* Disable all interrupt types */ - I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTL0, 0); + I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTL0, + I40E_PFINT_DYN_CTL0_ITR_INDX_MASK); I40E_WRITE_FLUSH(hw); } @@ -5418,16 +6101,16 @@ i40e_dev_handle_vfr_event(struct rte_eth_dev *dev) index = abs_vf_id / I40E_UINT32_BIT_SIZE; offset = abs_vf_id % I40E_UINT32_BIT_SIZE; val = I40E_READ_REG(hw, I40E_GLGEN_VFLRSTAT(index)); - /* VFR event occured */ + /* VFR event occurred */ if (val & (0x1 << offset)) { int ret; /* Clear the event first */ I40E_WRITE_REG(hw, I40E_GLGEN_VFLRSTAT(index), (0x1 << offset)); - PMD_DRV_LOG(INFO, "VF %u reset occured", abs_vf_id); + PMD_DRV_LOG(INFO, "VF %u reset occurred", abs_vf_id); /** - * Only notify a VF reset event occured, + * Only notify a VF reset event occurred, * don't trigger another SW reset */ ret = i40e_pf_host_vf_reset(&pf->vfs[i], 0); @@ -5441,18 +6124,10 @@ static void i40e_notify_all_vfs_link_status(struct rte_eth_dev *dev) { struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); - struct i40e_virtchnl_pf_event event; int i; - event.event = I40E_VIRTCHNL_EVENT_LINK_CHANGE; - event.event_data.link_event.link_status = - dev->data->dev_link.link_status; - event.event_data.link_event.link_speed = - (enum i40e_aq_link_speed)dev->data->dev_link.link_speed; - for (i = 0; i < pf->vf_num; i++) - i40e_pf_host_send_msg_to_vf(&pf->vfs[i], I40E_VIRTCHNL_OP_EVENT, - I40E_SUCCESS, (uint8_t *)&event, sizeof(event)); + i40e_notify_vf_link_status(dev, &pf->vfs[i]); } static void @@ -5475,8 +6150,9 @@ i40e_dev_handle_aq_msg(struct rte_eth_dev *dev) ret = i40e_clean_arq_element(hw, &info, &pending); if (ret != I40E_SUCCESS) { - PMD_DRV_LOG(INFO, "Failed to read msg from AdminQ, " - "aq_err: %u", hw->aq.asq_last_status); + PMD_DRV_LOG(INFO, + "Failed to read msg from AdminQ, aq_err: %u", + hw->aq.asq_last_status); break; } opcode = rte_le_to_cpu_16(info.desc.opcode); @@ -5493,14 +6169,12 @@ i40e_dev_handle_aq_msg(struct rte_eth_dev *dev) break; case i40e_aqc_opc_get_link_status: ret = i40e_dev_link_update(dev, 0); - if (!ret) { - i40e_notify_all_vfs_link_status(dev); + if (!ret) _rte_eth_dev_callback_process(dev, - RTE_ETH_EVENT_INTR_LSC); - } + RTE_ETH_EVENT_INTR_LSC, NULL, NULL); break; default: - PMD_DRV_LOG(ERR, "Request %u is not supported yet", + PMD_DRV_LOG(DEBUG, "Request %u is not supported yet", opcode); break; } @@ -5521,8 +6195,7 @@ i40e_dev_handle_aq_msg(struct rte_eth_dev *dev) * void */ static void -i40e_dev_interrupt_handler(__rte_unused struct rte_intr_handle *handle, - void *param) +i40e_dev_interrupt_handler(void *param) { struct rte_eth_dev *dev = (struct rte_eth_dev *)param; struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); @@ -5539,7 +6212,6 @@ i40e_dev_interrupt_handler(__rte_unused struct rte_intr_handle *handle, PMD_DRV_LOG(INFO, "No interrupt event"); goto done; } -#ifdef RTE_LIBRTE_I40E_DEBUG_DRIVER if (icr0 & I40E_PFINT_ICR0_ECC_ERR_MASK) PMD_DRV_LOG(ERR, "ICR0: unrecoverable ECC error"); if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK) @@ -5554,7 +6226,6 @@ i40e_dev_interrupt_handler(__rte_unused struct rte_intr_handle *handle, PMD_DRV_LOG(ERR, "ICR0: HMC error"); if (icr0 & I40E_PFINT_ICR0_PE_CRITERR_MASK) PMD_DRV_LOG(ERR, "ICR0: protocol engine critical error"); -#endif /* RTE_LIBRTE_I40E_DEBUG_DRIVER */ if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) { PMD_DRV_LOG(INFO, "ICR0: VF reset detected"); @@ -5564,13 +6235,14 @@ i40e_dev_interrupt_handler(__rte_unused struct rte_intr_handle *handle, PMD_DRV_LOG(INFO, "ICR0: adminq event"); i40e_dev_handle_aq_msg(dev); } + done: /* Enable interrupt */ i40e_pf_enable_irq0(hw); - rte_intr_enable(&(dev->pci_dev->intr_handle)); + rte_intr_enable(dev->intr_handle); } -static int +int i40e_add_macvlan_filters(struct i40e_vsi *vsi, struct i40e_macvlan_filter *filter, int total) @@ -5599,7 +6271,7 @@ i40e_add_macvlan_filters(struct i40e_vsi *vsi, memset(req_list, 0, ele_buff_size); for (i = 0; i < actual_num; i++) { - (void)rte_memcpy(req_list[i].mac_addr, + rte_memcpy(req_list[i].mac_addr, &filter[num + i].macaddr, ETH_ADDR_LEN); req_list[i].vlan_tag = rte_cpu_to_le_16(filter[num + i].vlan_id); @@ -5620,7 +6292,7 @@ i40e_add_macvlan_filters(struct i40e_vsi *vsi, flags = I40E_AQC_MACVLAN_ADD_HASH_MATCH; break; default: - PMD_DRV_LOG(ERR, "Invalid MAC match type\n"); + PMD_DRV_LOG(ERR, "Invalid MAC match type"); ret = I40E_ERR_PARAM; goto DONE; } @@ -5644,7 +6316,7 @@ DONE: return ret; } -static int +int i40e_remove_macvlan_filters(struct i40e_vsi *vsi, struct i40e_macvlan_filter *filter, int total) @@ -5674,7 +6346,7 @@ i40e_remove_macvlan_filters(struct i40e_vsi *vsi, memset(req_list, 0, ele_buff_size); for (i = 0; i < actual_num; i++) { - (void)rte_memcpy(req_list[i].mac_addr, + rte_memcpy(req_list[i].mac_addr, &filter[num + i].macaddr, ETH_ADDR_LEN); req_list[i].vlan_tag = rte_cpu_to_le_16(filter[num + i].vlan_id); @@ -5695,7 +6367,7 @@ i40e_remove_macvlan_filters(struct i40e_vsi *vsi, flags = I40E_AQC_MACVLAN_DEL_HASH_MATCH; break; default: - PMD_DRV_LOG(ERR, "Invalid MAC filter type\n"); + PMD_DRV_LOG(ERR, "Invalid MAC filter type"); ret = I40E_ERR_PARAM; goto DONE; } @@ -5750,14 +6422,11 @@ i40e_find_vlan_filter(struct i40e_vsi *vsi, } static void -i40e_set_vlan_filter(struct i40e_vsi *vsi, - uint16_t vlan_id, bool on) +i40e_store_vlan_filter(struct i40e_vsi *vsi, + uint16_t vlan_id, bool on) { uint32_t vid_idx, vid_bit; - if (vlan_id > ETH_VLAN_ID_MAX) - return; - vid_idx = I40E_VFTA_IDX(vlan_id); vid_bit = I40E_VFTA_BIT(vlan_id); @@ -5767,11 +6436,43 @@ i40e_set_vlan_filter(struct i40e_vsi *vsi, vsi->vfta[vid_idx] &= ~vid_bit; } +void +i40e_set_vlan_filter(struct i40e_vsi *vsi, + uint16_t vlan_id, bool on) +{ + struct i40e_hw *hw = I40E_VSI_TO_HW(vsi); + struct i40e_aqc_add_remove_vlan_element_data vlan_data = {0}; + int ret; + + if (vlan_id > ETH_VLAN_ID_MAX) + return; + + i40e_store_vlan_filter(vsi, vlan_id, on); + + if ((!vsi->vlan_anti_spoof_on && !vsi->vlan_filter_on) || !vlan_id) + return; + + vlan_data.vlan_tag = rte_cpu_to_le_16(vlan_id); + + if (on) { + ret = i40e_aq_add_vlan(hw, vsi->seid, + &vlan_data, 1, NULL); + if (ret != I40E_SUCCESS) + PMD_DRV_LOG(ERR, "Failed to add vlan filter"); + } else { + ret = i40e_aq_remove_vlan(hw, vsi->seid, + &vlan_data, 1, NULL); + if (ret != I40E_SUCCESS) + PMD_DRV_LOG(ERR, + "Failed to remove vlan filter"); + } +} + /** * Find all vlan options for specific mac addr, * return with actual vlan found. */ -static inline int +int i40e_find_all_vlan_for_mac(struct i40e_vsi *vsi, struct i40e_macvlan_filter *mv_f, int num, struct ether_addr *addr) @@ -5792,11 +6493,11 @@ i40e_find_all_vlan_for_mac(struct i40e_vsi *vsi, for (k = 0; k < I40E_UINT32_BIT_SIZE; k++) { if (vsi->vfta[j] & (1 << k)) { if (i > num - 1) { - PMD_DRV_LOG(ERR, "vlan number " - "not match"); + PMD_DRV_LOG(ERR, + "vlan number doesn't match"); return I40E_ERR_PARAM; } - (void)rte_memcpy(&mv_f[i].macaddr, + rte_memcpy(&mv_f[i].macaddr, addr, ETH_ADDR_LEN); mv_f[i].vlan_id = j * I40E_UINT32_BIT_SIZE + k; @@ -5825,7 +6526,7 @@ i40e_find_all_mac_for_vlan(struct i40e_vsi *vsi, PMD_DRV_LOG(ERR, "buffer number not match"); return I40E_ERR_PARAM; } - (void)rte_memcpy(&mv_f[i].macaddr, &f->mac_info.mac_addr, + rte_memcpy(&mv_f[i].macaddr, &f->mac_info.mac_addr, ETH_ADDR_LEN); mv_f[i].vlan_id = vlan; mv_f[i].filter_type = f->mac_info.filter_type; @@ -5838,7 +6539,7 @@ i40e_find_all_mac_for_vlan(struct i40e_vsi *vsi, static int i40e_vsi_remove_all_macvlan_filter(struct i40e_vsi *vsi) { - int i, num; + int i, j, num; struct i40e_mac_filter *f; struct i40e_macvlan_filter *mv_f; int ret = I40E_SUCCESS; @@ -5861,8 +6562,9 @@ i40e_vsi_remove_all_macvlan_filter(struct i40e_vsi *vsi) i = 0; if (vsi->vlan_num == 0) { TAILQ_FOREACH(f, &vsi->mac_list, next) { - (void)rte_memcpy(&mv_f[i].macaddr, + rte_memcpy(&mv_f[i].macaddr, &f->mac_info.mac_addr, ETH_ADDR_LEN); + mv_f[i].filter_type = f->mac_info.filter_type; mv_f[i].vlan_id = 0; i++; } @@ -5872,6 +6574,8 @@ i40e_vsi_remove_all_macvlan_filter(struct i40e_vsi *vsi) vsi->vlan_num, &f->mac_info.mac_addr); if (ret != I40E_SUCCESS) goto DONE; + for (j = i; j < i + vsi->vlan_num; j++) + mv_f[j].filter_type = f->mac_info.filter_type; i += vsi->vlan_num; } } @@ -6028,7 +6732,7 @@ i40e_vsi_add_mac(struct i40e_vsi *vsi, struct i40e_mac_filter_info *mac_filter) for (i = 0; i < vlan_num; i++) { mv_f[i].filter_type = mac_filter->filter_type; - (void)rte_memcpy(&mv_f[i].macaddr, &mac_filter->mac_addr, + rte_memcpy(&mv_f[i].macaddr, &mac_filter->mac_addr, ETH_ADDR_LEN); } @@ -6051,7 +6755,7 @@ i40e_vsi_add_mac(struct i40e_vsi *vsi, struct i40e_mac_filter_info *mac_filter) ret = I40E_ERR_NO_MEMORY; goto DONE; } - (void)rte_memcpy(&f->mac_info.mac_addr, &mac_filter->mac_addr, + rte_memcpy(&f->mac_info.mac_addr, &mac_filter->mac_addr, ETH_ADDR_LEN); f->mac_info.filter_type = mac_filter->filter_type; TAILQ_INSERT_TAIL(&vsi->mac_list, f, next); @@ -6083,7 +6787,7 @@ i40e_vsi_delete_mac(struct i40e_vsi *vsi, struct ether_addr *addr) if (filter_type == RTE_MACVLAN_PERFECT_MATCH || filter_type == RTE_MACVLAN_HASH_MATCH) { if (vlan_num == 0) { - PMD_DRV_LOG(ERR, "VLAN number shouldn't be 0\n"); + PMD_DRV_LOG(ERR, "VLAN number shouldn't be 0"); return I40E_ERR_PARAM; } } else if (filter_type == RTE_MAC_PERFECT_MATCH || @@ -6098,7 +6802,7 @@ i40e_vsi_delete_mac(struct i40e_vsi *vsi, struct ether_addr *addr) for (i = 0; i < vlan_num; i++) { mv_f[i].filter_type = filter_type; - (void)rte_memcpy(&mv_f[i].macaddr, &f->mac_info.mac_addr, + rte_memcpy(&mv_f[i].macaddr, &f->mac_info.mac_addr, ETH_ADDR_LEN); } if (filter_type == RTE_MACVLAN_PERFECT_MATCH || @@ -6125,70 +6829,36 @@ DONE: /* Configure hash enable flags for RSS */ uint64_t -i40e_config_hena(uint64_t flags) +i40e_config_hena(const struct i40e_adapter *adapter, uint64_t flags) { uint64_t hena = 0; + int i; if (!flags) return hena; - if (flags & ETH_RSS_FRAG_IPV4) - hena |= 1ULL << I40E_FILTER_PCTYPE_FRAG_IPV4; - if (flags & ETH_RSS_NONFRAG_IPV4_TCP) - hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_TCP; - if (flags & ETH_RSS_NONFRAG_IPV4_UDP) - hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_UDP; - if (flags & ETH_RSS_NONFRAG_IPV4_SCTP) - hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_SCTP; - if (flags & ETH_RSS_NONFRAG_IPV4_OTHER) - hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER; - if (flags & ETH_RSS_FRAG_IPV6) - hena |= 1ULL << I40E_FILTER_PCTYPE_FRAG_IPV6; - if (flags & ETH_RSS_NONFRAG_IPV6_TCP) - hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_TCP; - if (flags & ETH_RSS_NONFRAG_IPV6_UDP) - hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_UDP; - if (flags & ETH_RSS_NONFRAG_IPV6_SCTP) - hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_SCTP; - if (flags & ETH_RSS_NONFRAG_IPV6_OTHER) - hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER; - if (flags & ETH_RSS_L2_PAYLOAD) - hena |= 1ULL << I40E_FILTER_PCTYPE_L2_PAYLOAD; + for (i = RTE_ETH_FLOW_UNKNOWN + 1; i < I40E_FLOW_TYPE_MAX; i++) { + if (flags & (1ULL << i)) + hena |= adapter->pctypes_tbl[i]; + } return hena; } /* Parse the hash enable flags */ uint64_t -i40e_parse_hena(uint64_t flags) +i40e_parse_hena(const struct i40e_adapter *adapter, uint64_t flags) { uint64_t rss_hf = 0; if (!flags) return rss_hf; - if (flags & (1ULL << I40E_FILTER_PCTYPE_FRAG_IPV4)) - rss_hf |= ETH_RSS_FRAG_IPV4; - if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_TCP)) - rss_hf |= ETH_RSS_NONFRAG_IPV4_TCP; - if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_UDP)) - rss_hf |= ETH_RSS_NONFRAG_IPV4_UDP; - if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_SCTP)) - rss_hf |= ETH_RSS_NONFRAG_IPV4_SCTP; - if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER)) - rss_hf |= ETH_RSS_NONFRAG_IPV4_OTHER; - if (flags & (1ULL << I40E_FILTER_PCTYPE_FRAG_IPV6)) - rss_hf |= ETH_RSS_FRAG_IPV6; - if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_TCP)) - rss_hf |= ETH_RSS_NONFRAG_IPV6_TCP; - if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_UDP)) - rss_hf |= ETH_RSS_NONFRAG_IPV6_UDP; - if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_SCTP)) - rss_hf |= ETH_RSS_NONFRAG_IPV6_SCTP; - if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER)) - rss_hf |= ETH_RSS_NONFRAG_IPV6_OTHER; - if (flags & (1ULL << I40E_FILTER_PCTYPE_L2_PAYLOAD)) - rss_hf |= ETH_RSS_L2_PAYLOAD; + int i; + for (i = RTE_ETH_FLOW_UNKNOWN + 1; i < I40E_FLOW_TYPE_MAX; i++) { + if (flags & adapter->pctypes_tbl[i]) + rss_hf |= (1ULL << i); + } return rss_hf; } @@ -6197,13 +6867,9 @@ static void i40e_pf_disable_rss(struct i40e_pf *pf) { struct i40e_hw *hw = I40E_PF_TO_HW(pf); - uint64_t hena; - hena = (uint64_t)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)); - hena |= ((uint64_t)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1))) << 32; - hena &= ~I40E_RSS_HENA_ALL; - i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (uint32_t)hena); - i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (uint32_t)(hena >> 32)); + i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), 0); + i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), 0); I40E_WRITE_FLUSH(hw); } @@ -6229,8 +6895,7 @@ i40e_set_rss_key(struct i40e_vsi *vsi, uint8_t *key, uint8_t key_len) ret = i40e_aq_set_rss_key(hw, vsi->vsi_id, key_dw); if (ret) - PMD_INIT_LOG(ERR, "Failed to configure RSS key " - "via AQ"); + PMD_INIT_LOG(ERR, "Failed to configure RSS key via AQ"); } else { uint32_t *hash_key = (uint32_t *)key; uint16_t i; @@ -6276,7 +6941,6 @@ static int i40e_hw_rss_hash_set(struct i40e_pf *pf, struct rte_eth_rss_conf *rss_conf) { struct i40e_hw *hw = I40E_PF_TO_HW(pf); - uint64_t rss_hf; uint64_t hena; int ret; @@ -6285,11 +6949,7 @@ i40e_hw_rss_hash_set(struct i40e_pf *pf, struct rte_eth_rss_conf *rss_conf) if (ret) return ret; - rss_hf = rss_conf->rss_hf; - hena = (uint64_t)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)); - hena |= ((uint64_t)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1))) << 32; - hena &= ~I40E_RSS_HENA_ALL; - hena |= i40e_config_hena(rss_hf); + hena = i40e_config_hena(pf->adapter, rss_conf->rss_hf); i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (uint32_t)hena); i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (uint32_t)(hena >> 32)); I40E_WRITE_FLUSH(hw); @@ -6303,12 +6963,13 @@ i40e_dev_rss_hash_update(struct rte_eth_dev *dev, { struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); - uint64_t rss_hf = rss_conf->rss_hf & I40E_RSS_OFFLOAD_ALL; + uint64_t rss_hf = rss_conf->rss_hf & pf->adapter->flow_types_mask; uint64_t hena; hena = (uint64_t)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)); hena |= ((uint64_t)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1))) << 32; - if (!(hena & I40E_RSS_HENA_ALL)) { /* RSS disabled */ + + if (!(hena & pf->adapter->pctypes_mask)) { /* RSS disabled */ if (rss_hf != 0) /* Enable RSS */ return -EINVAL; return 0; /* Nothing to do */ @@ -6333,7 +6994,7 @@ i40e_dev_rss_hash_conf_get(struct rte_eth_dev *dev, hena = (uint64_t)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)); hena |= ((uint64_t)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1))) << 32; - rss_conf->rss_hf = i40e_parse_hena(hena); + rss_conf->rss_hf = i40e_parse_hena(pf->adapter, hena); return 0; } @@ -6371,50 +7032,146 @@ i40e_dev_get_filter_type(uint16_t filter_type, uint16_t *flag) return 0; } +/* Convert tunnel filter structure */ static int +i40e_tunnel_filter_convert( + struct i40e_aqc_add_rm_cloud_filt_elem_ext *cld_filter, + struct i40e_tunnel_filter *tunnel_filter) +{ + ether_addr_copy((struct ether_addr *)&cld_filter->element.outer_mac, + (struct ether_addr *)&tunnel_filter->input.outer_mac); + ether_addr_copy((struct ether_addr *)&cld_filter->element.inner_mac, + (struct ether_addr *)&tunnel_filter->input.inner_mac); + tunnel_filter->input.inner_vlan = cld_filter->element.inner_vlan; + if ((rte_le_to_cpu_16(cld_filter->element.flags) & + I40E_AQC_ADD_CLOUD_FLAGS_IPV6) == + I40E_AQC_ADD_CLOUD_FLAGS_IPV6) + tunnel_filter->input.ip_type = I40E_TUNNEL_IPTYPE_IPV6; + else + tunnel_filter->input.ip_type = I40E_TUNNEL_IPTYPE_IPV4; + tunnel_filter->input.flags = cld_filter->element.flags; + tunnel_filter->input.tenant_id = cld_filter->element.tenant_id; + tunnel_filter->queue = cld_filter->element.queue_number; + rte_memcpy(tunnel_filter->input.general_fields, + cld_filter->general_fields, + sizeof(cld_filter->general_fields)); + + return 0; +} + +/* Check if there exists the tunnel filter */ +struct i40e_tunnel_filter * +i40e_sw_tunnel_filter_lookup(struct i40e_tunnel_rule *tunnel_rule, + const struct i40e_tunnel_filter_input *input) +{ + int ret; + + ret = rte_hash_lookup(tunnel_rule->hash_table, (const void *)input); + if (ret < 0) + return NULL; + + return tunnel_rule->hash_map[ret]; +} + +/* Add a tunnel filter into the SW list */ +static int +i40e_sw_tunnel_filter_insert(struct i40e_pf *pf, + struct i40e_tunnel_filter *tunnel_filter) +{ + struct i40e_tunnel_rule *rule = &pf->tunnel; + int ret; + + ret = rte_hash_add_key(rule->hash_table, &tunnel_filter->input); + if (ret < 0) { + PMD_DRV_LOG(ERR, + "Failed to insert tunnel filter to hash table %d!", + ret); + return ret; + } + rule->hash_map[ret] = tunnel_filter; + + TAILQ_INSERT_TAIL(&rule->tunnel_list, tunnel_filter, rules); + + return 0; +} + +/* Delete a tunnel filter from the SW list */ +int +i40e_sw_tunnel_filter_del(struct i40e_pf *pf, + struct i40e_tunnel_filter_input *input) +{ + struct i40e_tunnel_rule *rule = &pf->tunnel; + struct i40e_tunnel_filter *tunnel_filter; + int ret; + + ret = rte_hash_del_key(rule->hash_table, input); + if (ret < 0) { + PMD_DRV_LOG(ERR, + "Failed to delete tunnel filter to hash table %d!", + ret); + return ret; + } + tunnel_filter = rule->hash_map[ret]; + rule->hash_map[ret] = NULL; + + TAILQ_REMOVE(&rule->tunnel_list, tunnel_filter, rules); + rte_free(tunnel_filter); + + return 0; +} + +int i40e_dev_tunnel_filter_set(struct i40e_pf *pf, struct rte_eth_tunnel_filter_conf *tunnel_filter, uint8_t add) { uint16_t ip_type; - uint32_t ipv4_addr; + uint32_t ipv4_addr, ipv4_addr_le; uint8_t i, tun_type = 0; /* internal varialbe to convert ipv6 byte order */ uint32_t convert_ipv6[4]; int val, ret = 0; struct i40e_hw *hw = I40E_PF_TO_HW(pf); struct i40e_vsi *vsi = pf->main_vsi; - struct i40e_aqc_add_remove_cloud_filters_element_data *cld_filter; - struct i40e_aqc_add_remove_cloud_filters_element_data *pfilter; + struct i40e_aqc_add_rm_cloud_filt_elem_ext *cld_filter; + struct i40e_aqc_add_rm_cloud_filt_elem_ext *pfilter; + struct i40e_tunnel_rule *tunnel_rule = &pf->tunnel; + struct i40e_tunnel_filter *tunnel, *node; + struct i40e_tunnel_filter check_filter; /* Check if filter exists */ cld_filter = rte_zmalloc("tunnel_filter", - sizeof(struct i40e_aqc_add_remove_cloud_filters_element_data), - 0); + sizeof(struct i40e_aqc_add_rm_cloud_filt_elem_ext), + 0); if (NULL == cld_filter) { PMD_DRV_LOG(ERR, "Failed to alloc memory."); - return -EINVAL; + return -ENOMEM; } pfilter = cld_filter; - ether_addr_copy(&tunnel_filter->outer_mac, (struct ether_addr*)&pfilter->outer_mac); - ether_addr_copy(&tunnel_filter->inner_mac, (struct ether_addr*)&pfilter->inner_mac); + ether_addr_copy(&tunnel_filter->outer_mac, + (struct ether_addr *)&pfilter->element.outer_mac); + ether_addr_copy(&tunnel_filter->inner_mac, + (struct ether_addr *)&pfilter->element.inner_mac); - pfilter->inner_vlan = rte_cpu_to_le_16(tunnel_filter->inner_vlan); + pfilter->element.inner_vlan = + rte_cpu_to_le_16(tunnel_filter->inner_vlan); if (tunnel_filter->ip_type == RTE_TUNNEL_IPTYPE_IPV4) { ip_type = I40E_AQC_ADD_CLOUD_FLAGS_IPV4; ipv4_addr = rte_be_to_cpu_32(tunnel_filter->ip_addr.ipv4_addr); - rte_memcpy(&pfilter->ipaddr.v4.data, - &rte_cpu_to_le_32(ipv4_addr), - sizeof(pfilter->ipaddr.v4.data)); + ipv4_addr_le = rte_cpu_to_le_32(ipv4_addr); + rte_memcpy(&pfilter->element.ipaddr.v4.data, + &ipv4_addr_le, + sizeof(pfilter->element.ipaddr.v4.data)); } else { ip_type = I40E_AQC_ADD_CLOUD_FLAGS_IPV6; for (i = 0; i < 4; i++) { convert_ipv6[i] = rte_cpu_to_le_32(rte_be_to_cpu_32(tunnel_filter->ip_addr.ipv6_addr[i])); } - rte_memcpy(&pfilter->ipaddr.v6.data, &convert_ipv6, - sizeof(pfilter->ipaddr.v6.data)); + rte_memcpy(&pfilter->element.ipaddr.v6.data, + &convert_ipv6, + sizeof(pfilter->element.ipaddr.v6.data)); } /* check tunneled type */ @@ -6436,23 +7193,596 @@ i40e_dev_tunnel_filter_set(struct i40e_pf *pf, } val = i40e_dev_get_filter_type(tunnel_filter->filter_type, - &pfilter->flags); + &pfilter->element.flags); if (val < 0) { rte_free(cld_filter); return -EINVAL; } - pfilter->flags |= rte_cpu_to_le_16( + pfilter->element.flags |= rte_cpu_to_le_16( I40E_AQC_ADD_CLOUD_FLAGS_TO_QUEUE | ip_type | (tun_type << I40E_AQC_ADD_CLOUD_TNL_TYPE_SHIFT)); - pfilter->tenant_id = rte_cpu_to_le_32(tunnel_filter->tenant_id); - pfilter->queue_number = rte_cpu_to_le_16(tunnel_filter->queue_id); + pfilter->element.tenant_id = rte_cpu_to_le_32(tunnel_filter->tenant_id); + pfilter->element.queue_number = + rte_cpu_to_le_16(tunnel_filter->queue_id); - if (add) - ret = i40e_aq_add_cloud_filters(hw, vsi->seid, cld_filter, 1); - else + /* Check if there is the filter in SW list */ + memset(&check_filter, 0, sizeof(check_filter)); + i40e_tunnel_filter_convert(cld_filter, &check_filter); + node = i40e_sw_tunnel_filter_lookup(tunnel_rule, &check_filter.input); + if (add && node) { + PMD_DRV_LOG(ERR, "Conflict with existing tunnel rules!"); + rte_free(cld_filter); + return -EINVAL; + } + + if (!add && !node) { + PMD_DRV_LOG(ERR, "There's no corresponding tunnel filter!"); + rte_free(cld_filter); + return -EINVAL; + } + + if (add) { + ret = i40e_aq_add_cloud_filters(hw, + vsi->seid, &cld_filter->element, 1); + if (ret < 0) { + PMD_DRV_LOG(ERR, "Failed to add a tunnel filter."); + rte_free(cld_filter); + return -ENOTSUP; + } + tunnel = rte_zmalloc("tunnel_filter", sizeof(*tunnel), 0); + if (tunnel == NULL) { + PMD_DRV_LOG(ERR, "Failed to alloc memory."); + rte_free(cld_filter); + return -ENOMEM; + } + + rte_memcpy(tunnel, &check_filter, sizeof(check_filter)); + ret = i40e_sw_tunnel_filter_insert(pf, tunnel); + if (ret < 0) + rte_free(tunnel); + } else { ret = i40e_aq_remove_cloud_filters(hw, vsi->seid, - cld_filter, 1); + &cld_filter->element, 1); + if (ret < 0) { + PMD_DRV_LOG(ERR, "Failed to delete a tunnel filter."); + rte_free(cld_filter); + return -ENOTSUP; + } + ret = i40e_sw_tunnel_filter_del(pf, &node->input); + } + + rte_free(cld_filter); + return ret; +} + +#define I40E_AQC_REPLACE_CLOUD_CMD_INPUT_TR_WORD0 0x48 +#define I40E_TR_VXLAN_GRE_KEY_MASK 0x4 +#define I40E_TR_GENEVE_KEY_MASK 0x8 +#define I40E_TR_GENERIC_UDP_TUNNEL_MASK 0x40 +#define I40E_TR_GRE_KEY_MASK 0x400 +#define I40E_TR_GRE_KEY_WITH_XSUM_MASK 0x800 +#define I40E_TR_GRE_NO_KEY_MASK 0x8000 + +static enum +i40e_status_code i40e_replace_mpls_l1_filter(struct i40e_pf *pf) +{ + struct i40e_aqc_replace_cloud_filters_cmd filter_replace; + struct i40e_aqc_replace_cloud_filters_cmd_buf filter_replace_buf; + struct i40e_hw *hw = I40E_PF_TO_HW(pf); + enum i40e_status_code status = I40E_SUCCESS; + + if (pf->support_multi_driver) { + PMD_DRV_LOG(ERR, "Replace l1 filter is not supported."); + return I40E_NOT_SUPPORTED; + } + + memset(&filter_replace, 0, + sizeof(struct i40e_aqc_replace_cloud_filters_cmd)); + memset(&filter_replace_buf, 0, + sizeof(struct i40e_aqc_replace_cloud_filters_cmd_buf)); + + /* create L1 filter */ + filter_replace.old_filter_type = + I40E_AQC_REPLACE_CLOUD_CMD_INPUT_FV_IMAC; + filter_replace.new_filter_type = I40E_AQC_ADD_L1_FILTER_0X11; + filter_replace.tr_bit = 0; + + /* Prepare the buffer, 3 entries */ + filter_replace_buf.data[0] = + I40E_AQC_REPLACE_CLOUD_CMD_INPUT_FV_TEID_WORD0; + filter_replace_buf.data[0] |= + I40E_AQC_REPLACE_CLOUD_CMD_INPUT_VALIDATED; + filter_replace_buf.data[2] = 0xFF; + filter_replace_buf.data[3] = 0xFF; + filter_replace_buf.data[4] = + I40E_AQC_REPLACE_CLOUD_CMD_INPUT_FV_TEID_WORD1; + filter_replace_buf.data[4] |= + I40E_AQC_REPLACE_CLOUD_CMD_INPUT_VALIDATED; + filter_replace_buf.data[7] = 0xF0; + filter_replace_buf.data[8] + = I40E_AQC_REPLACE_CLOUD_CMD_INPUT_TR_WORD0; + filter_replace_buf.data[8] |= + I40E_AQC_REPLACE_CLOUD_CMD_INPUT_VALIDATED; + filter_replace_buf.data[10] = I40E_TR_VXLAN_GRE_KEY_MASK | + I40E_TR_GENEVE_KEY_MASK | + I40E_TR_GENERIC_UDP_TUNNEL_MASK; + filter_replace_buf.data[11] = (I40E_TR_GRE_KEY_MASK | + I40E_TR_GRE_KEY_WITH_XSUM_MASK | + I40E_TR_GRE_NO_KEY_MASK) >> 8; + + status = i40e_aq_replace_cloud_filters(hw, &filter_replace, + &filter_replace_buf); + if (!status) + i40e_global_cfg_warning(I40E_WARNING_RPL_CLD_FILTER); + return status; +} + +static enum +i40e_status_code i40e_replace_mpls_cloud_filter(struct i40e_pf *pf) +{ + struct i40e_aqc_replace_cloud_filters_cmd filter_replace; + struct i40e_aqc_replace_cloud_filters_cmd_buf filter_replace_buf; + struct i40e_hw *hw = I40E_PF_TO_HW(pf); + enum i40e_status_code status = I40E_SUCCESS; + + if (pf->support_multi_driver) { + PMD_DRV_LOG(ERR, "Replace cloud filter is not supported."); + return I40E_NOT_SUPPORTED; + } + + /* For MPLSoUDP */ + memset(&filter_replace, 0, + sizeof(struct i40e_aqc_replace_cloud_filters_cmd)); + memset(&filter_replace_buf, 0, + sizeof(struct i40e_aqc_replace_cloud_filters_cmd_buf)); + filter_replace.valid_flags = I40E_AQC_REPLACE_CLOUD_FILTER | + I40E_AQC_MIRROR_CLOUD_FILTER; + filter_replace.old_filter_type = I40E_AQC_ADD_CLOUD_FILTER_IIP; + filter_replace.new_filter_type = + I40E_AQC_ADD_CLOUD_FILTER_0X11; + /* Prepare the buffer, 2 entries */ + filter_replace_buf.data[0] = I40E_AQC_REPLACE_CLOUD_CMD_INPUT_FV_STAG; + filter_replace_buf.data[0] |= + I40E_AQC_REPLACE_CLOUD_CMD_INPUT_VALIDATED; + filter_replace_buf.data[4] = I40E_AQC_ADD_L1_FILTER_0X11; + filter_replace_buf.data[4] |= + I40E_AQC_REPLACE_CLOUD_CMD_INPUT_VALIDATED; + status = i40e_aq_replace_cloud_filters(hw, &filter_replace, + &filter_replace_buf); + if (status < 0) + return status; + + /* For MPLSoGRE */ + memset(&filter_replace, 0, + sizeof(struct i40e_aqc_replace_cloud_filters_cmd)); + memset(&filter_replace_buf, 0, + sizeof(struct i40e_aqc_replace_cloud_filters_cmd_buf)); + + filter_replace.valid_flags = I40E_AQC_REPLACE_CLOUD_FILTER | + I40E_AQC_MIRROR_CLOUD_FILTER; + filter_replace.old_filter_type = I40E_AQC_ADD_CLOUD_FILTER_IMAC; + filter_replace.new_filter_type = + I40E_AQC_ADD_CLOUD_FILTER_0X12; + /* Prepare the buffer, 2 entries */ + filter_replace_buf.data[0] = I40E_AQC_REPLACE_CLOUD_CMD_INPUT_FV_STAG; + filter_replace_buf.data[0] |= + I40E_AQC_REPLACE_CLOUD_CMD_INPUT_VALIDATED; + filter_replace_buf.data[4] = I40E_AQC_ADD_L1_FILTER_0X11; + filter_replace_buf.data[4] |= + I40E_AQC_REPLACE_CLOUD_CMD_INPUT_VALIDATED; + + status = i40e_aq_replace_cloud_filters(hw, &filter_replace, + &filter_replace_buf); + if (!status) + i40e_global_cfg_warning(I40E_WARNING_RPL_CLD_FILTER); + return status; +} + +static enum i40e_status_code +i40e_replace_gtp_l1_filter(struct i40e_pf *pf) +{ + struct i40e_aqc_replace_cloud_filters_cmd filter_replace; + struct i40e_aqc_replace_cloud_filters_cmd_buf filter_replace_buf; + struct i40e_hw *hw = I40E_PF_TO_HW(pf); + enum i40e_status_code status = I40E_SUCCESS; + + if (pf->support_multi_driver) { + PMD_DRV_LOG(ERR, "Replace l1 filter is not supported."); + return I40E_NOT_SUPPORTED; + } + + /* For GTP-C */ + memset(&filter_replace, 0, + sizeof(struct i40e_aqc_replace_cloud_filters_cmd)); + memset(&filter_replace_buf, 0, + sizeof(struct i40e_aqc_replace_cloud_filters_cmd_buf)); + /* create L1 filter */ + filter_replace.old_filter_type = + I40E_AQC_REPLACE_CLOUD_CMD_INPUT_FV_IMAC; + filter_replace.new_filter_type = I40E_AQC_ADD_L1_FILTER_0X12; + filter_replace.tr_bit = I40E_AQC_NEW_TR_22 | + I40E_AQC_REPLACE_CLOUD_CMD_INPUT_VALIDATED; + /* Prepare the buffer, 2 entries */ + filter_replace_buf.data[0] = + I40E_AQC_REPLACE_CLOUD_CMD_INPUT_FV_TEID_WORD0; + filter_replace_buf.data[0] |= + I40E_AQC_REPLACE_CLOUD_CMD_INPUT_VALIDATED; + filter_replace_buf.data[2] = 0xFF; + filter_replace_buf.data[3] = 0xFF; + filter_replace_buf.data[4] = + I40E_AQC_REPLACE_CLOUD_CMD_INPUT_FV_TEID_WORD1; + filter_replace_buf.data[4] |= + I40E_AQC_REPLACE_CLOUD_CMD_INPUT_VALIDATED; + filter_replace_buf.data[6] = 0xFF; + filter_replace_buf.data[7] = 0xFF; + status = i40e_aq_replace_cloud_filters(hw, &filter_replace, + &filter_replace_buf); + if (status < 0) + return status; + PMD_DRV_LOG(DEBUG, "Global configuration modification: " + "cloud l1 type is changed from 0x%x to 0x%x", + filter_replace.old_filter_type, + filter_replace.new_filter_type); + + /* for GTP-U */ + memset(&filter_replace, 0, + sizeof(struct i40e_aqc_replace_cloud_filters_cmd)); + memset(&filter_replace_buf, 0, + sizeof(struct i40e_aqc_replace_cloud_filters_cmd_buf)); + /* create L1 filter */ + filter_replace.old_filter_type = + I40E_AQC_REPLACE_CLOUD_CMD_INPUT_FV_TUNNLE_KEY; + filter_replace.new_filter_type = I40E_AQC_ADD_L1_FILTER_0X13; + filter_replace.tr_bit = I40E_AQC_NEW_TR_21 | + I40E_AQC_REPLACE_CLOUD_CMD_INPUT_VALIDATED; + /* Prepare the buffer, 2 entries */ + filter_replace_buf.data[0] = + I40E_AQC_REPLACE_CLOUD_CMD_INPUT_FV_TEID_WORD0; + filter_replace_buf.data[0] |= + I40E_AQC_REPLACE_CLOUD_CMD_INPUT_VALIDATED; + filter_replace_buf.data[2] = 0xFF; + filter_replace_buf.data[3] = 0xFF; + filter_replace_buf.data[4] = + I40E_AQC_REPLACE_CLOUD_CMD_INPUT_FV_TEID_WORD1; + filter_replace_buf.data[4] |= + I40E_AQC_REPLACE_CLOUD_CMD_INPUT_VALIDATED; + filter_replace_buf.data[6] = 0xFF; + filter_replace_buf.data[7] = 0xFF; + + status = i40e_aq_replace_cloud_filters(hw, &filter_replace, + &filter_replace_buf); + if (!status) { + i40e_global_cfg_warning(I40E_WARNING_RPL_CLD_FILTER); + PMD_DRV_LOG(DEBUG, "Global configuration modification: " + "cloud l1 type is changed from 0x%x to 0x%x", + filter_replace.old_filter_type, + filter_replace.new_filter_type); + } + return status; +} + +static enum +i40e_status_code i40e_replace_gtp_cloud_filter(struct i40e_pf *pf) +{ + struct i40e_aqc_replace_cloud_filters_cmd filter_replace; + struct i40e_aqc_replace_cloud_filters_cmd_buf filter_replace_buf; + struct i40e_hw *hw = I40E_PF_TO_HW(pf); + enum i40e_status_code status = I40E_SUCCESS; + + if (pf->support_multi_driver) { + PMD_DRV_LOG(ERR, "Replace cloud filter is not supported."); + return I40E_NOT_SUPPORTED; + } + + /* for GTP-C */ + memset(&filter_replace, 0, + sizeof(struct i40e_aqc_replace_cloud_filters_cmd)); + memset(&filter_replace_buf, 0, + sizeof(struct i40e_aqc_replace_cloud_filters_cmd_buf)); + filter_replace.valid_flags = I40E_AQC_REPLACE_CLOUD_FILTER; + filter_replace.old_filter_type = I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN; + filter_replace.new_filter_type = + I40E_AQC_ADD_CLOUD_FILTER_0X11; + /* Prepare the buffer, 2 entries */ + filter_replace_buf.data[0] = I40E_AQC_ADD_L1_FILTER_0X12; + filter_replace_buf.data[0] |= + I40E_AQC_REPLACE_CLOUD_CMD_INPUT_VALIDATED; + filter_replace_buf.data[4] = I40E_AQC_REPLACE_CLOUD_CMD_INPUT_FV_STAG; + filter_replace_buf.data[4] |= + I40E_AQC_REPLACE_CLOUD_CMD_INPUT_VALIDATED; + status = i40e_aq_replace_cloud_filters(hw, &filter_replace, + &filter_replace_buf); + if (status < 0) + return status; + PMD_DRV_LOG(DEBUG, "Global configuration modification: " + "cloud filter type is changed from 0x%x to 0x%x", + filter_replace.old_filter_type, + filter_replace.new_filter_type); + + /* for GTP-U */ + memset(&filter_replace, 0, + sizeof(struct i40e_aqc_replace_cloud_filters_cmd)); + memset(&filter_replace_buf, 0, + sizeof(struct i40e_aqc_replace_cloud_filters_cmd_buf)); + filter_replace.valid_flags = I40E_AQC_REPLACE_CLOUD_FILTER; + filter_replace.old_filter_type = + I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN_TEN_ID; + filter_replace.new_filter_type = + I40E_AQC_ADD_CLOUD_FILTER_0X12; + /* Prepare the buffer, 2 entries */ + filter_replace_buf.data[0] = I40E_AQC_ADD_L1_FILTER_0X13; + filter_replace_buf.data[0] |= + I40E_AQC_REPLACE_CLOUD_CMD_INPUT_VALIDATED; + filter_replace_buf.data[4] = I40E_AQC_REPLACE_CLOUD_CMD_INPUT_FV_STAG; + filter_replace_buf.data[4] |= + I40E_AQC_REPLACE_CLOUD_CMD_INPUT_VALIDATED; + + status = i40e_aq_replace_cloud_filters(hw, &filter_replace, + &filter_replace_buf); + if (!status) { + i40e_global_cfg_warning(I40E_WARNING_RPL_CLD_FILTER); + PMD_DRV_LOG(DEBUG, "Global configuration modification: " + "cloud filter type is changed from 0x%x to 0x%x", + filter_replace.old_filter_type, + filter_replace.new_filter_type); + } + return status; +} + +int +i40e_dev_consistent_tunnel_filter_set(struct i40e_pf *pf, + struct i40e_tunnel_filter_conf *tunnel_filter, + uint8_t add) +{ + uint16_t ip_type; + uint32_t ipv4_addr, ipv4_addr_le; + uint8_t i, tun_type = 0; + /* internal variable to convert ipv6 byte order */ + uint32_t convert_ipv6[4]; + int val, ret = 0; + struct i40e_pf_vf *vf = NULL; + struct i40e_hw *hw = I40E_PF_TO_HW(pf); + struct i40e_vsi *vsi; + struct i40e_aqc_add_rm_cloud_filt_elem_ext *cld_filter; + struct i40e_aqc_add_rm_cloud_filt_elem_ext *pfilter; + struct i40e_tunnel_rule *tunnel_rule = &pf->tunnel; + struct i40e_tunnel_filter *tunnel, *node; + struct i40e_tunnel_filter check_filter; /* Check if filter exists */ + uint32_t teid_le; + bool big_buffer = 0; + + cld_filter = rte_zmalloc("tunnel_filter", + sizeof(struct i40e_aqc_add_rm_cloud_filt_elem_ext), + 0); + + if (cld_filter == NULL) { + PMD_DRV_LOG(ERR, "Failed to alloc memory."); + return -ENOMEM; + } + pfilter = cld_filter; + + ether_addr_copy(&tunnel_filter->outer_mac, + (struct ether_addr *)&pfilter->element.outer_mac); + ether_addr_copy(&tunnel_filter->inner_mac, + (struct ether_addr *)&pfilter->element.inner_mac); + + pfilter->element.inner_vlan = + rte_cpu_to_le_16(tunnel_filter->inner_vlan); + if (tunnel_filter->ip_type == I40E_TUNNEL_IPTYPE_IPV4) { + ip_type = I40E_AQC_ADD_CLOUD_FLAGS_IPV4; + ipv4_addr = rte_be_to_cpu_32(tunnel_filter->ip_addr.ipv4_addr); + ipv4_addr_le = rte_cpu_to_le_32(ipv4_addr); + rte_memcpy(&pfilter->element.ipaddr.v4.data, + &ipv4_addr_le, + sizeof(pfilter->element.ipaddr.v4.data)); + } else { + ip_type = I40E_AQC_ADD_CLOUD_FLAGS_IPV6; + for (i = 0; i < 4; i++) { + convert_ipv6[i] = + rte_cpu_to_le_32(rte_be_to_cpu_32( + tunnel_filter->ip_addr.ipv6_addr[i])); + } + rte_memcpy(&pfilter->element.ipaddr.v6.data, + &convert_ipv6, + sizeof(pfilter->element.ipaddr.v6.data)); + } + + /* check tunneled type */ + switch (tunnel_filter->tunnel_type) { + case I40E_TUNNEL_TYPE_VXLAN: + tun_type = I40E_AQC_ADD_CLOUD_TNL_TYPE_VXLAN; + break; + case I40E_TUNNEL_TYPE_NVGRE: + tun_type = I40E_AQC_ADD_CLOUD_TNL_TYPE_NVGRE_OMAC; + break; + case I40E_TUNNEL_TYPE_IP_IN_GRE: + tun_type = I40E_AQC_ADD_CLOUD_TNL_TYPE_IP; + break; + case I40E_TUNNEL_TYPE_MPLSoUDP: + if (!pf->mpls_replace_flag) { + i40e_replace_mpls_l1_filter(pf); + i40e_replace_mpls_cloud_filter(pf); + pf->mpls_replace_flag = 1; + } + teid_le = rte_cpu_to_le_32(tunnel_filter->tenant_id); + pfilter->general_fields[I40E_AQC_ADD_CLOUD_FV_FLU_0X11_WORD0] = + teid_le >> 4; + pfilter->general_fields[I40E_AQC_ADD_CLOUD_FV_FLU_0X11_WORD1] = + (teid_le & 0xF) << 12; + pfilter->general_fields[I40E_AQC_ADD_CLOUD_FV_FLU_0X11_WORD2] = + 0x40; + big_buffer = 1; + tun_type = I40E_AQC_ADD_CLOUD_TNL_TYPE_MPLSOUDP; + break; + case I40E_TUNNEL_TYPE_MPLSoGRE: + if (!pf->mpls_replace_flag) { + i40e_replace_mpls_l1_filter(pf); + i40e_replace_mpls_cloud_filter(pf); + pf->mpls_replace_flag = 1; + } + teid_le = rte_cpu_to_le_32(tunnel_filter->tenant_id); + pfilter->general_fields[I40E_AQC_ADD_CLOUD_FV_FLU_0X11_WORD0] = + teid_le >> 4; + pfilter->general_fields[I40E_AQC_ADD_CLOUD_FV_FLU_0X11_WORD1] = + (teid_le & 0xF) << 12; + pfilter->general_fields[I40E_AQC_ADD_CLOUD_FV_FLU_0X11_WORD2] = + 0x0; + big_buffer = 1; + tun_type = I40E_AQC_ADD_CLOUD_TNL_TYPE_MPLSOGRE; + break; + case I40E_TUNNEL_TYPE_GTPC: + if (!pf->gtp_replace_flag) { + i40e_replace_gtp_l1_filter(pf); + i40e_replace_gtp_cloud_filter(pf); + pf->gtp_replace_flag = 1; + } + teid_le = rte_cpu_to_le_32(tunnel_filter->tenant_id); + pfilter->general_fields[I40E_AQC_ADD_CLOUD_FV_FLU_0X12_WORD0] = + (teid_le >> 16) & 0xFFFF; + pfilter->general_fields[I40E_AQC_ADD_CLOUD_FV_FLU_0X12_WORD1] = + teid_le & 0xFFFF; + pfilter->general_fields[I40E_AQC_ADD_CLOUD_FV_FLU_0X12_WORD2] = + 0x0; + big_buffer = 1; + break; + case I40E_TUNNEL_TYPE_GTPU: + if (!pf->gtp_replace_flag) { + i40e_replace_gtp_l1_filter(pf); + i40e_replace_gtp_cloud_filter(pf); + pf->gtp_replace_flag = 1; + } + teid_le = rte_cpu_to_le_32(tunnel_filter->tenant_id); + pfilter->general_fields[I40E_AQC_ADD_CLOUD_FV_FLU_0X13_WORD0] = + (teid_le >> 16) & 0xFFFF; + pfilter->general_fields[I40E_AQC_ADD_CLOUD_FV_FLU_0X13_WORD1] = + teid_le & 0xFFFF; + pfilter->general_fields[I40E_AQC_ADD_CLOUD_FV_FLU_0X13_WORD2] = + 0x0; + big_buffer = 1; + break; + case I40E_TUNNEL_TYPE_QINQ: + if (!pf->qinq_replace_flag) { + ret = i40e_cloud_filter_qinq_create(pf); + if (ret < 0) + PMD_DRV_LOG(DEBUG, + "QinQ tunnel filter already created."); + pf->qinq_replace_flag = 1; + } + /* Add in the General fields the values of + * the Outer and Inner VLAN + * Big Buffer should be set, see changes in + * i40e_aq_add_cloud_filters + */ + pfilter->general_fields[0] = tunnel_filter->inner_vlan; + pfilter->general_fields[1] = tunnel_filter->outer_vlan; + big_buffer = 1; + break; + default: + /* Other tunnel types is not supported. */ + PMD_DRV_LOG(ERR, "tunnel type is not supported."); + rte_free(cld_filter); + return -EINVAL; + } + + if (tunnel_filter->tunnel_type == I40E_TUNNEL_TYPE_MPLSoUDP) + pfilter->element.flags = + I40E_AQC_ADD_CLOUD_FILTER_0X11; + else if (tunnel_filter->tunnel_type == I40E_TUNNEL_TYPE_MPLSoGRE) + pfilter->element.flags = + I40E_AQC_ADD_CLOUD_FILTER_0X12; + else if (tunnel_filter->tunnel_type == I40E_TUNNEL_TYPE_GTPC) + pfilter->element.flags = + I40E_AQC_ADD_CLOUD_FILTER_0X11; + else if (tunnel_filter->tunnel_type == I40E_TUNNEL_TYPE_GTPU) + pfilter->element.flags = + I40E_AQC_ADD_CLOUD_FILTER_0X12; + else if (tunnel_filter->tunnel_type == I40E_TUNNEL_TYPE_QINQ) + pfilter->element.flags |= + I40E_AQC_ADD_CLOUD_FILTER_0X10; + else { + val = i40e_dev_get_filter_type(tunnel_filter->filter_type, + &pfilter->element.flags); + if (val < 0) { + rte_free(cld_filter); + return -EINVAL; + } + } + + pfilter->element.flags |= rte_cpu_to_le_16( + I40E_AQC_ADD_CLOUD_FLAGS_TO_QUEUE | + ip_type | (tun_type << I40E_AQC_ADD_CLOUD_TNL_TYPE_SHIFT)); + pfilter->element.tenant_id = rte_cpu_to_le_32(tunnel_filter->tenant_id); + pfilter->element.queue_number = + rte_cpu_to_le_16(tunnel_filter->queue_id); + + if (!tunnel_filter->is_to_vf) + vsi = pf->main_vsi; + else { + if (tunnel_filter->vf_id >= pf->vf_num) { + PMD_DRV_LOG(ERR, "Invalid argument."); + rte_free(cld_filter); + return -EINVAL; + } + vf = &pf->vfs[tunnel_filter->vf_id]; + vsi = vf->vsi; + } + + /* Check if there is the filter in SW list */ + memset(&check_filter, 0, sizeof(check_filter)); + i40e_tunnel_filter_convert(cld_filter, &check_filter); + check_filter.is_to_vf = tunnel_filter->is_to_vf; + check_filter.vf_id = tunnel_filter->vf_id; + node = i40e_sw_tunnel_filter_lookup(tunnel_rule, &check_filter.input); + if (add && node) { + PMD_DRV_LOG(ERR, "Conflict with existing tunnel rules!"); + rte_free(cld_filter); + return -EINVAL; + } + + if (!add && !node) { + PMD_DRV_LOG(ERR, "There's no corresponding tunnel filter!"); + rte_free(cld_filter); + return -EINVAL; + } + + if (add) { + if (big_buffer) + ret = i40e_aq_add_cloud_filters_big_buffer(hw, + vsi->seid, cld_filter, 1); + else + ret = i40e_aq_add_cloud_filters(hw, + vsi->seid, &cld_filter->element, 1); + if (ret < 0) { + PMD_DRV_LOG(ERR, "Failed to add a tunnel filter."); + rte_free(cld_filter); + return -ENOTSUP; + } + tunnel = rte_zmalloc("tunnel_filter", sizeof(*tunnel), 0); + if (tunnel == NULL) { + PMD_DRV_LOG(ERR, "Failed to alloc memory."); + rte_free(cld_filter); + return -ENOMEM; + } + + rte_memcpy(tunnel, &check_filter, sizeof(check_filter)); + ret = i40e_sw_tunnel_filter_insert(pf, tunnel); + if (ret < 0) + rte_free(tunnel); + } else { + if (big_buffer) + ret = i40e_aq_remove_cloud_filters_big_buffer( + hw, vsi->seid, cld_filter, 1); + else + ret = i40e_aq_remove_cloud_filters(hw, vsi->seid, + &cld_filter->element, 1); + if (ret < 0) { + PMD_DRV_LOG(ERR, "Failed to delete a tunnel filter."); + rte_free(cld_filter); + return -ENOTSUP; + } + ret = i40e_sw_tunnel_filter_del(pf, &node->input); + } rte_free(cld_filter); return ret; @@ -6489,8 +7819,9 @@ i40e_add_vxlan_port(struct i40e_pf *pf, uint16_t port) /* Now check if there is space to add the new port */ idx = i40e_get_vxlan_port_idx(pf, 0); if (idx < 0) { - PMD_DRV_LOG(ERR, "Maximum number of UDP ports reached," - "not adding port %d", port); + PMD_DRV_LOG(ERR, + "Maximum number of UDP ports reached, not adding port %d", + port); return -ENOSPC; } @@ -6640,7 +7971,7 @@ i40e_pf_config_rss(struct i40e_pf *pf) /* * If both VMDQ and RSS enabled, not all of PF queues are configured. - * It's necessary to calulate the actual PF queues that are configured. + * It's necessary to calculate the actual PF queues that are configured. */ if (pf->dev_data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG) num = i40e_pf_calc_configured_queues_num(pf); @@ -6666,7 +7997,7 @@ i40e_pf_config_rss(struct i40e_pf *pf) } rss_conf = pf->dev_data->dev_conf.rx_adv_conf.rss_conf; - if ((rss_conf.rss_hf & I40E_RSS_OFFLOAD_ALL) == 0) { + if ((rss_conf.rss_hf & pf->adapter->flow_types_mask) == 0) { i40e_pf_disable_rss(pf); return 0; } @@ -6725,11 +8056,17 @@ i40e_tunnel_filter_param_check(struct i40e_pf *pf, static int i40e_dev_set_gre_key_len(struct i40e_hw *hw, uint8_t len) { + struct i40e_pf *pf = &((struct i40e_adapter *)hw->back)->pf; uint32_t val, reg; int ret = -EINVAL; + if (pf->support_multi_driver) { + PMD_DRV_LOG(ERR, "GRE key length configuration is unsupported"); + return -ENOTSUP; + } + val = I40E_READ_REG(hw, I40E_GL_PRS_FVBM(2)); - PMD_DRV_LOG(DEBUG, "Read original GL_PRS_FVBM with 0x%08x\n", val); + PMD_DRV_LOG(DEBUG, "Read original GL_PRS_FVBM with 0x%08x", val); if (len == 3) { reg = val | I40E_GL_PRS_FVBM_MSK_ENA; @@ -6745,10 +8082,14 @@ i40e_dev_set_gre_key_len(struct i40e_hw *hw, uint8_t len) reg, NULL); if (ret != 0) return ret; + PMD_DRV_LOG(DEBUG, "Global register 0x%08x is changed " + "with value 0x%08x", + I40E_GL_PRS_FVBM(2), reg); + i40e_global_cfg_warning(I40E_WARNING_GRE_KEY_LEN); } else { ret = 0; } - PMD_DRV_LOG(DEBUG, "Read modified GL_PRS_FVBM with 0x%08x\n", + PMD_DRV_LOG(DEBUG, "Read modified GL_PRS_FVBM with 0x%08x", I40E_READ_REG(hw, I40E_GL_PRS_FVBM(2))); return ret; @@ -6861,15 +8202,15 @@ i40e_set_symmetric_hash_enable_per_port(struct i40e_hw *hw, uint8_t enable) if (enable > 0) { if (reg & I40E_PRTQF_CTL_0_HSYM_ENA_MASK) { - PMD_DRV_LOG(INFO, "Symmetric hash has already " - "been enabled"); + PMD_DRV_LOG(INFO, + "Symmetric hash has already been enabled"); return; } reg |= I40E_PRTQF_CTL_0_HSYM_ENA_MASK; } else { if (!(reg & I40E_PRTQF_CTL_0_HSYM_ENA_MASK)) { - PMD_DRV_LOG(INFO, "Symmetric hash has already " - "been disabled"); + PMD_DRV_LOG(INFO, + "Symmetric hash has already been disabled"); return; } reg &= ~I40E_PRTQF_CTL_0_HSYM_ENA_MASK; @@ -6887,9 +8228,9 @@ static int i40e_get_hash_filter_global_config(struct i40e_hw *hw, struct rte_eth_hash_global_conf *g_cfg) { - uint32_t reg, mask = I40E_FLOW_TYPES; - uint16_t i; - enum i40e_filter_pctype pctype; + struct i40e_adapter *adapter = (struct i40e_adapter *)hw->back; + uint32_t reg; + uint16_t i, j; memset(g_cfg, 0, sizeof(*g_cfg)); reg = i40e_read_rx_ctl(hw, I40E_GLQF_CTL); @@ -6900,29 +8241,38 @@ i40e_get_hash_filter_global_config(struct i40e_hw *hw, PMD_DRV_LOG(DEBUG, "Hash function is %s", (reg & I40E_GLQF_CTL_HTOEP_MASK) ? "Toeplitz" : "Simple XOR"); - for (i = 0; mask && i < RTE_ETH_FLOW_MAX; i++) { - if (!(mask & (1UL << i))) + /* + * We work only with lowest 32 bits which is not correct, but to work + * properly the valid_bit_mask size should be increased up to 64 bits + * and this will brake ABI. This modification will be done in next + * release + */ + g_cfg->valid_bit_mask[0] = (uint32_t)adapter->flow_types_mask; + + for (i = RTE_ETH_FLOW_UNKNOWN + 1; i < UINT32_BIT; i++) { + if (!adapter->pctypes_tbl[i]) continue; - mask &= ~(1UL << i); - /* Bit set indicats the coresponding flow type is supported */ - g_cfg->valid_bit_mask[0] |= (1UL << i); - /* if flowtype is invalid, continue */ - if (!I40E_VALID_FLOW(i)) - continue; - pctype = i40e_flowtype_to_pctype(i); - reg = i40e_read_rx_ctl(hw, I40E_GLQF_HSYM(pctype)); - if (reg & I40E_GLQF_HSYM_SYMH_ENA_MASK) - g_cfg->sym_hash_enable_mask[0] |= (1UL << i); + for (j = I40E_FILTER_PCTYPE_INVALID + 1; + j < I40E_FILTER_PCTYPE_MAX; j++) { + if (adapter->pctypes_tbl[i] & (1ULL << j)) { + reg = i40e_read_rx_ctl(hw, I40E_GLQF_HSYM(j)); + if (reg & I40E_GLQF_HSYM_SYMH_ENA_MASK) { + g_cfg->sym_hash_enable_mask[0] |= + (1UL << i); + } + } + } } return 0; } static int -i40e_hash_global_config_check(struct rte_eth_hash_global_conf *g_cfg) +i40e_hash_global_config_check(const struct i40e_adapter *adapter, + const struct rte_eth_hash_global_conf *g_cfg) { uint32_t i; - uint32_t mask0, i40e_mask = I40E_FLOW_TYPES; + uint32_t mask0, i40e_mask = adapter->flow_types_mask; if (g_cfg->hash_func != RTE_ETH_HASH_FUNCTION_TOEPLITZ && g_cfg->hash_func != RTE_ETH_HASH_FUNCTION_SIMPLE_XOR && @@ -6965,44 +8315,60 @@ static int i40e_set_hash_filter_global_config(struct i40e_hw *hw, struct rte_eth_hash_global_conf *g_cfg) { + struct i40e_adapter *adapter = (struct i40e_adapter *)hw->back; + struct i40e_pf *pf = &((struct i40e_adapter *)hw->back)->pf; int ret; - uint16_t i; + uint16_t i, j; uint32_t reg; - uint32_t mask0 = g_cfg->valid_bit_mask[0]; - enum i40e_filter_pctype pctype; + /* + * We work only with lowest 32 bits which is not correct, but to work + * properly the valid_bit_mask size should be increased up to 64 bits + * and this will brake ABI. This modification will be done in next + * release + */ + uint32_t mask0 = g_cfg->valid_bit_mask[0] & + (uint32_t)adapter->flow_types_mask; + + if (pf->support_multi_driver) { + PMD_DRV_LOG(ERR, "Hash global configuration is not supported."); + return -ENOTSUP; + } /* Check the input parameters */ - ret = i40e_hash_global_config_check(g_cfg); + ret = i40e_hash_global_config_check(adapter, g_cfg); if (ret < 0) return ret; - for (i = 0; mask0 && i < UINT32_BIT; i++) { - if (!(mask0 & (1UL << i))) - continue; - mask0 &= ~(1UL << i); - /* if flowtype is invalid, continue */ - if (!I40E_VALID_FLOW(i)) - continue; - pctype = i40e_flowtype_to_pctype(i); - reg = (g_cfg->sym_hash_enable_mask[0] & (1UL << i)) ? - I40E_GLQF_HSYM_SYMH_ENA_MASK : 0; - i40e_write_rx_ctl(hw, I40E_GLQF_HSYM(pctype), reg); + for (i = RTE_ETH_FLOW_UNKNOWN + 1; mask0 && i < UINT32_BIT; i++) { + if (mask0 & (1UL << i)) { + reg = (g_cfg->sym_hash_enable_mask[0] & (1UL << i)) ? + I40E_GLQF_HSYM_SYMH_ENA_MASK : 0; + + for (j = I40E_FILTER_PCTYPE_INVALID + 1; + j < I40E_FILTER_PCTYPE_MAX; j++) { + if (adapter->pctypes_tbl[i] & (1ULL << j)) + i40e_write_global_rx_ctl(hw, + I40E_GLQF_HSYM(j), + reg); + } + i40e_global_cfg_warning(I40E_WARNING_HSYM); + } } reg = i40e_read_rx_ctl(hw, I40E_GLQF_CTL); if (g_cfg->hash_func == RTE_ETH_HASH_FUNCTION_TOEPLITZ) { /* Toeplitz */ if (reg & I40E_GLQF_CTL_HTOEP_MASK) { - PMD_DRV_LOG(DEBUG, "Hash function already set to " - "Toeplitz"); + PMD_DRV_LOG(DEBUG, + "Hash function already set to Toeplitz"); goto out; } reg |= I40E_GLQF_CTL_HTOEP_MASK; } else if (g_cfg->hash_func == RTE_ETH_HASH_FUNCTION_SIMPLE_XOR) { /* Simple XOR */ if (!(reg & I40E_GLQF_CTL_HTOEP_MASK)) { - PMD_DRV_LOG(DEBUG, "Hash function already set to " - "Simple XOR"); + PMD_DRV_LOG(DEBUG, + "Hash function already set to Simple XOR"); goto out; } reg &= ~I40E_GLQF_CTL_HTOEP_MASK; @@ -7010,7 +8376,8 @@ i40e_set_hash_filter_global_config(struct i40e_hw *hw, /* Use the default, and keep it as it is */ goto out; - i40e_write_rx_ctl(hw, I40E_GLQF_CTL, reg); + i40e_write_global_rx_ctl(hw, I40E_GLQF_CTL, reg); + i40e_global_cfg_warning(I40E_WARNING_QF_CTL); out: I40E_WRITE_FLUSH(hw); @@ -7045,6 +8412,24 @@ i40e_get_valid_input_set(enum i40e_filter_pctype pctype, I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST | I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT | I40E_INSET_FLEX_PAYLOAD, + [I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP] = + I40E_INSET_DMAC | I40E_INSET_SMAC | + I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER | + I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV4_TOS | + I40E_INSET_IPV4_PROTO | I40E_INSET_IPV4_TTL | + I40E_INSET_TUNNEL_DMAC | I40E_INSET_TUNNEL_ID | + I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST | + I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT | + I40E_INSET_FLEX_PAYLOAD, + [I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP] = + I40E_INSET_DMAC | I40E_INSET_SMAC | + I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER | + I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV4_TOS | + I40E_INSET_IPV4_PROTO | I40E_INSET_IPV4_TTL | + I40E_INSET_TUNNEL_DMAC | I40E_INSET_TUNNEL_ID | + I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST | + I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT | + I40E_INSET_FLEX_PAYLOAD, [I40E_FILTER_PCTYPE_NONF_IPV4_TCP] = I40E_INSET_DMAC | I40E_INSET_SMAC | I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER | @@ -7054,6 +8439,15 @@ i40e_get_valid_input_set(enum i40e_filter_pctype pctype, I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST | I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT | I40E_INSET_TCP_FLAGS | I40E_INSET_FLEX_PAYLOAD, + [I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK] = + I40E_INSET_DMAC | I40E_INSET_SMAC | + I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER | + I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV4_TOS | + I40E_INSET_IPV4_PROTO | I40E_INSET_IPV4_TTL | + I40E_INSET_TUNNEL_DMAC | I40E_INSET_TUNNEL_ID | + I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST | + I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT | + I40E_INSET_TCP_FLAGS | I40E_INSET_FLEX_PAYLOAD, [I40E_FILTER_PCTYPE_NONF_IPV4_SCTP] = I40E_INSET_DMAC | I40E_INSET_SMAC | I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER | @@ -7087,6 +8481,24 @@ i40e_get_valid_input_set(enum i40e_filter_pctype pctype, I40E_INSET_IPV6_HOP_LIMIT | I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST | I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT | I40E_INSET_FLEX_PAYLOAD, + [I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP] = + I40E_INSET_DMAC | I40E_INSET_SMAC | + I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER | + I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV6_TC | + I40E_INSET_IPV6_FLOW | I40E_INSET_IPV6_NEXT_HDR | + I40E_INSET_IPV6_HOP_LIMIT | I40E_INSET_IPV6_SRC | + I40E_INSET_IPV6_DST | I40E_INSET_SRC_PORT | + I40E_INSET_DST_PORT | I40E_INSET_TCP_FLAGS | + I40E_INSET_FLEX_PAYLOAD, + [I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP] = + I40E_INSET_DMAC | I40E_INSET_SMAC | + I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER | + I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV6_TC | + I40E_INSET_IPV6_FLOW | I40E_INSET_IPV6_NEXT_HDR | + I40E_INSET_IPV6_HOP_LIMIT | I40E_INSET_IPV6_SRC | + I40E_INSET_IPV6_DST | I40E_INSET_SRC_PORT | + I40E_INSET_DST_PORT | I40E_INSET_TCP_FLAGS | + I40E_INSET_FLEX_PAYLOAD, [I40E_FILTER_PCTYPE_NONF_IPV6_TCP] = I40E_INSET_DMAC | I40E_INSET_SMAC | I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER | @@ -7096,6 +8508,15 @@ i40e_get_valid_input_set(enum i40e_filter_pctype pctype, I40E_INSET_IPV6_DST | I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT | I40E_INSET_TCP_FLAGS | I40E_INSET_FLEX_PAYLOAD, + [I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK] = + I40E_INSET_DMAC | I40E_INSET_SMAC | + I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER | + I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV6_TC | + I40E_INSET_IPV6_FLOW | I40E_INSET_IPV6_NEXT_HDR | + I40E_INSET_IPV6_HOP_LIMIT | I40E_INSET_IPV6_SRC | + I40E_INSET_IPV6_DST | I40E_INSET_SRC_PORT | + I40E_INSET_DST_PORT | I40E_INSET_TCP_FLAGS | + I40E_INSET_FLEX_PAYLOAD, [I40E_FILTER_PCTYPE_NONF_IPV6_SCTP] = I40E_INSET_DMAC | I40E_INSET_SMAC | I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER | @@ -7135,11 +8556,26 @@ i40e_get_valid_input_set(enum i40e_filter_pctype pctype, I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST | I40E_INSET_IPV4_TOS | I40E_INSET_IPV4_TTL | I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT, + [I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP] = + I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER | + I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST | + I40E_INSET_IPV4_TOS | I40E_INSET_IPV4_TTL | + I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT, + [I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP] = + I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER | + I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST | + I40E_INSET_IPV4_TOS | I40E_INSET_IPV4_TTL | + I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT, [I40E_FILTER_PCTYPE_NONF_IPV4_TCP] = I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER | I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST | I40E_INSET_IPV4_TOS | I40E_INSET_IPV4_TTL | I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT, + [I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK] = + I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER | + I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST | + I40E_INSET_IPV4_TOS | I40E_INSET_IPV4_TTL | + I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT, [I40E_FILTER_PCTYPE_NONF_IPV4_SCTP] = I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER | I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST | @@ -7161,11 +8597,26 @@ i40e_get_valid_input_set(enum i40e_filter_pctype pctype, I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST | I40E_INSET_IPV6_TC | I40E_INSET_IPV6_HOP_LIMIT | I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT, + [I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP] = + I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER | + I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST | + I40E_INSET_IPV6_TC | I40E_INSET_IPV6_HOP_LIMIT | + I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT, + [I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP] = + I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER | + I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST | + I40E_INSET_IPV6_TC | I40E_INSET_IPV6_HOP_LIMIT | + I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT, [I40E_FILTER_PCTYPE_NONF_IPV6_TCP] = I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER | I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST | I40E_INSET_IPV6_TC | I40E_INSET_IPV6_HOP_LIMIT | I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT, + [I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK] = + I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER | + I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST | + I40E_INSET_IPV6_TC | I40E_INSET_IPV6_HOP_LIMIT | + I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT, [I40E_FILTER_PCTYPE_NONF_IPV6_SCTP] = I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER | I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST | @@ -7195,7 +8646,7 @@ i40e_get_valid_input_set(enum i40e_filter_pctype pctype, /** * Validate if the input set is allowed for a specific PCTYPE */ -static int +int i40e_validate_input_set(enum i40e_filter_pctype pctype, enum rte_filter_type filter, uint64_t inset) { @@ -7209,7 +8660,7 @@ i40e_validate_input_set(enum i40e_filter_pctype pctype, } /* default input set fields combination per pctype */ -static uint64_t +uint64_t i40e_get_default_input_set(uint16_t pctype) { static const uint64_t default_inset_table[] = { @@ -7218,9 +8669,18 @@ i40e_get_default_input_set(uint16_t pctype) [I40E_FILTER_PCTYPE_NONF_IPV4_UDP] = I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST | I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT, + [I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP] = + I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST | + I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT, + [I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP] = + I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST | + I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT, [I40E_FILTER_PCTYPE_NONF_IPV4_TCP] = I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST | I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT, + [I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK] = + I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST | + I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT, [I40E_FILTER_PCTYPE_NONF_IPV4_SCTP] = I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST | I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT | @@ -7232,9 +8692,18 @@ i40e_get_default_input_set(uint16_t pctype) [I40E_FILTER_PCTYPE_NONF_IPV6_UDP] = I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST | I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT, + [I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP] = + I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST | + I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT, + [I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP] = + I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST | + I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT, [I40E_FILTER_PCTYPE_NONF_IPV6_TCP] = I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST | I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT, + [I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK] = + I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST | + I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT, [I40E_FILTER_PCTYPE_NONF_IPV6_SCTP] = I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST | I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT | @@ -7352,7 +8821,7 @@ i40e_parse_input_set(uint64_t *inset, * Translate the input set from bit masks to register aware bit masks * and vice versa */ -static uint64_t +uint64_t i40e_translate_input_set_reg(enum i40e_mac_type type, uint64_t input) { uint64_t val = 0; @@ -7437,7 +8906,7 @@ i40e_translate_input_set_reg(enum i40e_mac_type type, uint64_t input) return val; } -static int +int i40e_generate_inset_mask_reg(uint64_t inset, uint32_t *mask, uint8_t nb_elem) { uint8_t i, idx = 0; @@ -7485,15 +8954,27 @@ i40e_generate_inset_mask_reg(uint64_t inset, uint32_t *mask, uint8_t nb_elem) return idx; } -static void +void i40e_check_write_reg(struct i40e_hw *hw, uint32_t addr, uint32_t val) { uint32_t reg = i40e_read_rx_ctl(hw, addr); - PMD_DRV_LOG(DEBUG, "[0x%08x] original: 0x%08x\n", addr, reg); + PMD_DRV_LOG(DEBUG, "[0x%08x] original: 0x%08x", addr, reg); if (reg != val) i40e_write_rx_ctl(hw, addr, val); - PMD_DRV_LOG(DEBUG, "[0x%08x] after: 0x%08x\n", addr, + PMD_DRV_LOG(DEBUG, "[0x%08x] after: 0x%08x", addr, + (uint32_t)i40e_read_rx_ctl(hw, addr)); +} + +void +i40e_check_write_global_reg(struct i40e_hw *hw, uint32_t addr, uint32_t val) +{ + uint32_t reg = i40e_read_rx_ctl(hw, addr); + + PMD_DRV_LOG(DEBUG, "[0x%08x] original: 0x%08x", addr, reg); + if (reg != val) + i40e_write_global_rx_ctl(hw, addr, val); + PMD_DRV_LOG(DEBUG, "[0x%08x] after: 0x%08x", addr, (uint32_t)i40e_read_rx_ctl(hw, addr)); } @@ -7505,17 +8986,25 @@ i40e_filter_input_set_init(struct i40e_pf *pf) uint64_t input_set, inset_reg; uint32_t mask_reg[I40E_INSET_MASK_NUM_REG] = {0}; int num, i; + uint16_t flow_type; for (pctype = I40E_FILTER_PCTYPE_NONF_IPV4_UDP; pctype <= I40E_FILTER_PCTYPE_L2_PAYLOAD; pctype++) { - if (!I40E_VALID_PCTYPE(pctype)) + flow_type = i40e_pctype_to_flowtype(pf->adapter, pctype); + + if (flow_type == RTE_ETH_FLOW_UNKNOWN) continue; + input_set = i40e_get_default_input_set(pctype); num = i40e_generate_inset_mask_reg(input_set, mask_reg, I40E_INSET_MASK_NUM_REG); if (num < 0) return; + if (pf->support_multi_driver && num > 0) { + PMD_DRV_LOG(ERR, "Input set setting is not supported."); + return; + } inset_reg = i40e_translate_input_set_reg(hw->mac.type, input_set); @@ -7524,31 +9013,48 @@ i40e_filter_input_set_init(struct i40e_pf *pf) i40e_check_write_reg(hw, I40E_PRTQF_FD_INSET(pctype, 1), (uint32_t)((inset_reg >> I40E_32_BIT_WIDTH) & UINT32_MAX)); - i40e_check_write_reg(hw, I40E_GLQF_HASH_INSET(0, pctype), - (uint32_t)(inset_reg & UINT32_MAX)); - i40e_check_write_reg(hw, I40E_GLQF_HASH_INSET(1, pctype), - (uint32_t)((inset_reg >> - I40E_32_BIT_WIDTH) & UINT32_MAX)); + if (!pf->support_multi_driver) { + i40e_check_write_global_reg(hw, + I40E_GLQF_HASH_INSET(0, pctype), + (uint32_t)(inset_reg & UINT32_MAX)); + i40e_check_write_global_reg(hw, + I40E_GLQF_HASH_INSET(1, pctype), + (uint32_t)((inset_reg >> + I40E_32_BIT_WIDTH) & UINT32_MAX)); - for (i = 0; i < num; i++) { - i40e_check_write_reg(hw, I40E_GLQF_FD_MSK(i, pctype), - mask_reg[i]); - i40e_check_write_reg(hw, I40E_GLQF_HASH_MSK(i, pctype), - mask_reg[i]); - } - /*clear unused mask registers of the pctype */ - for (i = num; i < I40E_INSET_MASK_NUM_REG; i++) { - i40e_check_write_reg(hw, I40E_GLQF_FD_MSK(i, pctype), - 0); - i40e_check_write_reg(hw, I40E_GLQF_HASH_MSK(i, pctype), - 0); + for (i = 0; i < num; i++) { + i40e_check_write_global_reg(hw, + I40E_GLQF_FD_MSK(i, pctype), + mask_reg[i]); + i40e_check_write_global_reg(hw, + I40E_GLQF_HASH_MSK(i, pctype), + mask_reg[i]); + } + /*clear unused mask registers of the pctype */ + for (i = num; i < I40E_INSET_MASK_NUM_REG; i++) { + i40e_check_write_global_reg(hw, + I40E_GLQF_FD_MSK(i, pctype), + 0); + i40e_check_write_global_reg(hw, + I40E_GLQF_HASH_MSK(i, pctype), + 0); + } + } else { + PMD_DRV_LOG(ERR, "Input set setting is not supported."); } I40E_WRITE_FLUSH(hw); /* store the default input set */ - pf->hash_input_set[pctype] = input_set; + if (!pf->support_multi_driver) + pf->hash_input_set[pctype] = input_set; pf->fdir.input_set[pctype] = input_set; } + + if (!pf->support_multi_driver) { + i40e_global_cfg_warning(I40E_WARNING_HASH_INSET); + i40e_global_cfg_warning(I40E_WARNING_FD_MSK); + i40e_global_cfg_warning(I40E_WARNING_HASH_MSK); + } } int @@ -7571,22 +9077,30 @@ i40e_hash_filter_inset_select(struct i40e_hw *hw, return -EINVAL; } - if (!I40E_VALID_FLOW(conf->flow_type)) { + if (pf->support_multi_driver) { + PMD_DRV_LOG(ERR, "Hash input set setting is not supported."); + return -ENOTSUP; + } + + pctype = i40e_flowtype_to_pctype(pf->adapter, conf->flow_type); + if (pctype == I40E_FILTER_PCTYPE_INVALID) { PMD_DRV_LOG(ERR, "invalid flow_type input."); return -EINVAL; } - pctype = i40e_flowtype_to_pctype(conf->flow_type); + + if (hw->mac.type == I40E_MAC_X722) { + /* get translated pctype value in fd pctype register */ + pctype = (enum i40e_filter_pctype)i40e_read_rx_ctl(hw, + I40E_GLQF_FD_PCTYPES((int)pctype)); + } + ret = i40e_parse_input_set(&input_set, pctype, conf->field, conf->inset_size); if (ret) { PMD_DRV_LOG(ERR, "Failed to parse input set"); return -EINVAL; } - if (i40e_validate_input_set(pctype, RTE_ETH_FILTER_HASH, - input_set) != 0) { - PMD_DRV_LOG(ERR, "Invalid input set"); - return -EINVAL; - } + if (conf->op == RTE_ETH_INPUT_SET_ADD) { /* get inset value in register */ inset_reg = i40e_read_rx_ctl(hw, I40E_GLQF_HASH_INSET(1, pctype)); @@ -7601,19 +9115,21 @@ i40e_hash_filter_inset_select(struct i40e_hw *hw, inset_reg |= i40e_translate_input_set_reg(hw->mac.type, input_set); - i40e_check_write_reg(hw, I40E_GLQF_HASH_INSET(0, pctype), - (uint32_t)(inset_reg & UINT32_MAX)); - i40e_check_write_reg(hw, I40E_GLQF_HASH_INSET(1, pctype), - (uint32_t)((inset_reg >> - I40E_32_BIT_WIDTH) & UINT32_MAX)); + i40e_check_write_global_reg(hw, I40E_GLQF_HASH_INSET(0, pctype), + (uint32_t)(inset_reg & UINT32_MAX)); + i40e_check_write_global_reg(hw, I40E_GLQF_HASH_INSET(1, pctype), + (uint32_t)((inset_reg >> + I40E_32_BIT_WIDTH) & UINT32_MAX)); + i40e_global_cfg_warning(I40E_WARNING_HASH_INSET); for (i = 0; i < num; i++) - i40e_check_write_reg(hw, I40E_GLQF_HASH_MSK(i, pctype), - mask_reg[i]); + i40e_check_write_global_reg(hw, I40E_GLQF_HASH_MSK(i, pctype), + mask_reg[i]); /*clear unused mask registers of the pctype */ for (i = num; i < I40E_INSET_MASK_NUM_REG; i++) - i40e_check_write_reg(hw, I40E_GLQF_HASH_MSK(i, pctype), - 0); + i40e_check_write_global_reg(hw, I40E_GLQF_HASH_MSK(i, pctype), + 0); + i40e_global_cfg_warning(I40E_WARNING_HASH_MSK); I40E_WRITE_FLUSH(hw); pf->hash_input_set[pctype] = input_set; @@ -7640,22 +9156,19 @@ i40e_fdir_filter_inset_select(struct i40e_pf *pf, return -EINVAL; } - if (!I40E_VALID_FLOW(conf->flow_type)) { + pctype = i40e_flowtype_to_pctype(pf->adapter, conf->flow_type); + + if (pctype == I40E_FILTER_PCTYPE_INVALID) { PMD_DRV_LOG(ERR, "invalid flow_type input."); return -EINVAL; } - pctype = i40e_flowtype_to_pctype(conf->flow_type); + ret = i40e_parse_input_set(&input_set, pctype, conf->field, conf->inset_size); if (ret) { PMD_DRV_LOG(ERR, "Failed to parse input set"); return -EINVAL; } - if (i40e_validate_input_set(pctype, RTE_ETH_FILTER_FDIR, - input_set) != 0) { - PMD_DRV_LOG(ERR, "Invalid input set"); - return -EINVAL; - } /* get inset value in register */ inset_reg = i40e_read_rx_ctl(hw, I40E_PRTQF_FD_INSET(pctype, 1)); @@ -7674,6 +9187,10 @@ i40e_fdir_filter_inset_select(struct i40e_pf *pf, I40E_INSET_MASK_NUM_REG); if (num < 0) return -EINVAL; + if (pf->support_multi_driver && num > 0) { + PMD_DRV_LOG(ERR, "FDIR bit mask is not supported."); + return -ENOTSUP; + } inset_reg |= i40e_translate_input_set_reg(hw->mac.type, input_set); @@ -7683,13 +9200,20 @@ i40e_fdir_filter_inset_select(struct i40e_pf *pf, (uint32_t)((inset_reg >> I40E_32_BIT_WIDTH) & UINT32_MAX)); - for (i = 0; i < num; i++) - i40e_check_write_reg(hw, I40E_GLQF_FD_MSK(i, pctype), - mask_reg[i]); - /*clear unused mask registers of the pctype */ - for (i = num; i < I40E_INSET_MASK_NUM_REG; i++) - i40e_check_write_reg(hw, I40E_GLQF_FD_MSK(i, pctype), - 0); + if (!pf->support_multi_driver) { + for (i = 0; i < num; i++) + i40e_check_write_global_reg(hw, + I40E_GLQF_FD_MSK(i, pctype), + mask_reg[i]); + /*clear unused mask registers of the pctype */ + for (i = num; i < I40E_INSET_MASK_NUM_REG; i++) + i40e_check_write_global_reg(hw, + I40E_GLQF_FD_MSK(i, pctype), + 0); + i40e_global_cfg_warning(I40E_WARNING_FD_MSK); + } else { + PMD_DRV_LOG(ERR, "FDIR bit mask is not supported."); + } I40E_WRITE_FLUSH(hw); pf->fdir.input_set[pctype] = input_set; @@ -7788,16 +9312,95 @@ i40e_hash_filter_ctrl(struct rte_eth_dev *dev, return ret; } +/* Convert ethertype filter structure */ +static int +i40e_ethertype_filter_convert(const struct rte_eth_ethertype_filter *input, + struct i40e_ethertype_filter *filter) +{ + rte_memcpy(&filter->input.mac_addr, &input->mac_addr, ETHER_ADDR_LEN); + filter->input.ether_type = input->ether_type; + filter->flags = input->flags; + filter->queue = input->queue; + + return 0; +} + +/* Check if there exists the ehtertype filter */ +struct i40e_ethertype_filter * +i40e_sw_ethertype_filter_lookup(struct i40e_ethertype_rule *ethertype_rule, + const struct i40e_ethertype_filter_input *input) +{ + int ret; + + ret = rte_hash_lookup(ethertype_rule->hash_table, (const void *)input); + if (ret < 0) + return NULL; + + return ethertype_rule->hash_map[ret]; +} + +/* Add ethertype filter in SW list */ +static int +i40e_sw_ethertype_filter_insert(struct i40e_pf *pf, + struct i40e_ethertype_filter *filter) +{ + struct i40e_ethertype_rule *rule = &pf->ethertype; + int ret; + + ret = rte_hash_add_key(rule->hash_table, &filter->input); + if (ret < 0) { + PMD_DRV_LOG(ERR, + "Failed to insert ethertype filter" + " to hash table %d!", + ret); + return ret; + } + rule->hash_map[ret] = filter; + + TAILQ_INSERT_TAIL(&rule->ethertype_list, filter, rules); + + return 0; +} + +/* Delete ethertype filter in SW list */ +int +i40e_sw_ethertype_filter_del(struct i40e_pf *pf, + struct i40e_ethertype_filter_input *input) +{ + struct i40e_ethertype_rule *rule = &pf->ethertype; + struct i40e_ethertype_filter *filter; + int ret; + + ret = rte_hash_del_key(rule->hash_table, input); + if (ret < 0) { + PMD_DRV_LOG(ERR, + "Failed to delete ethertype filter" + " to hash table %d!", + ret); + return ret; + } + filter = rule->hash_map[ret]; + rule->hash_map[ret] = NULL; + + TAILQ_REMOVE(&rule->ethertype_list, filter, rules); + rte_free(filter); + + return 0; +} + /* * Configure ethertype filter, which can director packet by filtering * with mac address and ether_type or only ether_type */ -static int +int i40e_ethertype_filter_set(struct i40e_pf *pf, struct rte_eth_ethertype_filter *filter, bool add) { struct i40e_hw *hw = I40E_PF_TO_HW(pf); + struct i40e_ethertype_rule *ethertype_rule = &pf->ethertype; + struct i40e_ethertype_filter *ethertype_filter, *node; + struct i40e_ethertype_filter check_filter; struct i40e_control_filter_stats stats; uint16_t flags = 0; int ret; @@ -7808,13 +9411,29 @@ i40e_ethertype_filter_set(struct i40e_pf *pf, } if (filter->ether_type == ETHER_TYPE_IPv4 || filter->ether_type == ETHER_TYPE_IPv6) { - PMD_DRV_LOG(ERR, "unsupported ether_type(0x%04x) in" - " control packet filter.", filter->ether_type); + PMD_DRV_LOG(ERR, + "unsupported ether_type(0x%04x) in control packet filter.", + filter->ether_type); return -EINVAL; } if (filter->ether_type == ETHER_TYPE_VLAN) - PMD_DRV_LOG(WARNING, "filter vlan ether_type in first tag is" - " not supported."); + PMD_DRV_LOG(WARNING, + "filter vlan ether_type in first tag is not supported."); + + /* Check if there is the filter in SW list */ + memset(&check_filter, 0, sizeof(check_filter)); + i40e_ethertype_filter_convert(filter, &check_filter); + node = i40e_sw_ethertype_filter_lookup(ethertype_rule, + &check_filter.input); + if (add && node) { + PMD_DRV_LOG(ERR, "Conflict with existing ethertype rules!"); + return -EINVAL; + } + + if (!add && !node) { + PMD_DRV_LOG(ERR, "There's no corresponding ethertype filter!"); + return -EINVAL; + } if (!(filter->flags & RTE_ETHTYPE_FLAGS_MAC)) flags |= I40E_AQC_ADD_CONTROL_PACKET_FLAGS_IGNORE_MAC; @@ -7829,14 +9448,32 @@ i40e_ethertype_filter_set(struct i40e_pf *pf, pf->main_vsi->seid, filter->queue, add, &stats, NULL); - PMD_DRV_LOG(INFO, "add/rem control packet filter, return %d," - " mac_etype_used = %u, etype_used = %u," - " mac_etype_free = %u, etype_free = %u\n", - ret, stats.mac_etype_used, stats.etype_used, - stats.mac_etype_free, stats.etype_free); + PMD_DRV_LOG(INFO, + "add/rem control packet filter, return %d, mac_etype_used = %u, etype_used = %u, mac_etype_free = %u, etype_free = %u", + ret, stats.mac_etype_used, stats.etype_used, + stats.mac_etype_free, stats.etype_free); if (ret < 0) return -ENOSYS; - return 0; + + /* Add or delete a filter in SW list */ + if (add) { + ethertype_filter = rte_zmalloc("ethertype_filter", + sizeof(*ethertype_filter), 0); + if (ethertype_filter == NULL) { + PMD_DRV_LOG(ERR, "Failed to alloc memory."); + return -ENOMEM; + } + + rte_memcpy(ethertype_filter, &check_filter, + sizeof(check_filter)); + ret = i40e_sw_ethertype_filter_insert(pf, ethertype_filter); + if (ret < 0) + rte_free(ethertype_filter); + } else { + ret = i40e_sw_ethertype_filter_del(pf, &node->input); + } + + return ret; } /* @@ -7871,7 +9508,7 @@ i40e_ethertype_filter_handle(struct rte_eth_dev *dev, FALSE); break; default: - PMD_DRV_LOG(ERR, "unsupported operation %u\n", filter_op); + PMD_DRV_LOG(ERR, "unsupported operation %u", filter_op); ret = -ENOSYS; break; } @@ -7909,6 +9546,11 @@ i40e_dev_filter_ctrl(struct rte_eth_dev *dev, case RTE_ETH_FILTER_FDIR: ret = i40e_fdir_ctrl_func(dev, filter_op, arg); break; + case RTE_ETH_FILTER_GENERIC: + if (filter_op != RTE_ETH_FILTER_GET) + return -EINVAL; + *(const void **)arg = &i40e_flow_ops; + break; default: PMD_DRV_LOG(WARNING, "Filter type (%d) not supported", filter_type); @@ -7926,10 +9568,11 @@ i40e_dev_filter_ctrl(struct rte_eth_dev *dev, static void i40e_enable_extended_tag(struct rte_eth_dev *dev) { + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); uint32_t buf = 0; int ret; - ret = rte_eal_pci_read_config(dev->pci_dev, &buf, sizeof(buf), + ret = rte_pci_read_config(pci_dev, &buf, sizeof(buf), PCI_DEV_CAP_REG); if (ret < 0) { PMD_DRV_LOG(ERR, "Failed to read PCI offset 0x%x", @@ -7942,7 +9585,7 @@ i40e_enable_extended_tag(struct rte_eth_dev *dev) } buf = 0; - ret = rte_eal_pci_read_config(dev->pci_dev, &buf, sizeof(buf), + ret = rte_pci_read_config(pci_dev, &buf, sizeof(buf), PCI_DEV_CTRL_REG); if (ret < 0) { PMD_DRV_LOG(ERR, "Failed to read PCI offset 0x%x", @@ -7954,7 +9597,7 @@ i40e_enable_extended_tag(struct rte_eth_dev *dev) return; } buf |= PCI_DEV_CTRL_EXT_TAG_MASK; - ret = rte_eal_pci_write_config(dev->pci_dev, &buf, sizeof(buf), + ret = rte_pci_write_config(pci_dev, &buf, sizeof(buf), PCI_DEV_CTRL_REG); if (ret < 0) { PMD_DRV_LOG(ERR, "Failed to write PCI offset 0x%x", @@ -7982,60 +9625,42 @@ i40e_hw_init(struct rte_eth_dev *dev) i40e_set_symmetric_hash_enable_per_port(hw, 0); } +/* + * For X722 it is possible to have multiple pctypes mapped to the same flowtype + * however this function will return only one highest pctype index, + * which is not quite correct. This is known problem of i40e driver + * and needs to be fixed later. + */ enum i40e_filter_pctype -i40e_flowtype_to_pctype(uint16_t flow_type) +i40e_flowtype_to_pctype(const struct i40e_adapter *adapter, uint16_t flow_type) { - static const enum i40e_filter_pctype pctype_table[] = { - [RTE_ETH_FLOW_FRAG_IPV4] = I40E_FILTER_PCTYPE_FRAG_IPV4, - [RTE_ETH_FLOW_NONFRAG_IPV4_UDP] = - I40E_FILTER_PCTYPE_NONF_IPV4_UDP, - [RTE_ETH_FLOW_NONFRAG_IPV4_TCP] = - I40E_FILTER_PCTYPE_NONF_IPV4_TCP, - [RTE_ETH_FLOW_NONFRAG_IPV4_SCTP] = - I40E_FILTER_PCTYPE_NONF_IPV4_SCTP, - [RTE_ETH_FLOW_NONFRAG_IPV4_OTHER] = - I40E_FILTER_PCTYPE_NONF_IPV4_OTHER, - [RTE_ETH_FLOW_FRAG_IPV6] = I40E_FILTER_PCTYPE_FRAG_IPV6, - [RTE_ETH_FLOW_NONFRAG_IPV6_UDP] = - I40E_FILTER_PCTYPE_NONF_IPV6_UDP, - [RTE_ETH_FLOW_NONFRAG_IPV6_TCP] = - I40E_FILTER_PCTYPE_NONF_IPV6_TCP, - [RTE_ETH_FLOW_NONFRAG_IPV6_SCTP] = - I40E_FILTER_PCTYPE_NONF_IPV6_SCTP, - [RTE_ETH_FLOW_NONFRAG_IPV6_OTHER] = - I40E_FILTER_PCTYPE_NONF_IPV6_OTHER, - [RTE_ETH_FLOW_L2_PAYLOAD] = I40E_FILTER_PCTYPE_L2_PAYLOAD, - }; + int i; + uint64_t pctype_mask; - return pctype_table[flow_type]; + if (flow_type < I40E_FLOW_TYPE_MAX) { + pctype_mask = adapter->pctypes_tbl[flow_type]; + for (i = I40E_FILTER_PCTYPE_MAX - 1; i > 0; i--) { + if (pctype_mask & (1ULL << i)) + return (enum i40e_filter_pctype)i; + } + } + return I40E_FILTER_PCTYPE_INVALID; } uint16_t -i40e_pctype_to_flowtype(enum i40e_filter_pctype pctype) +i40e_pctype_to_flowtype(const struct i40e_adapter *adapter, + enum i40e_filter_pctype pctype) { - static const uint16_t flowtype_table[] = { - [I40E_FILTER_PCTYPE_FRAG_IPV4] = RTE_ETH_FLOW_FRAG_IPV4, - [I40E_FILTER_PCTYPE_NONF_IPV4_UDP] = - RTE_ETH_FLOW_NONFRAG_IPV4_UDP, - [I40E_FILTER_PCTYPE_NONF_IPV4_TCP] = - RTE_ETH_FLOW_NONFRAG_IPV4_TCP, - [I40E_FILTER_PCTYPE_NONF_IPV4_SCTP] = - RTE_ETH_FLOW_NONFRAG_IPV4_SCTP, - [I40E_FILTER_PCTYPE_NONF_IPV4_OTHER] = - RTE_ETH_FLOW_NONFRAG_IPV4_OTHER, - [I40E_FILTER_PCTYPE_FRAG_IPV6] = RTE_ETH_FLOW_FRAG_IPV6, - [I40E_FILTER_PCTYPE_NONF_IPV6_UDP] = - RTE_ETH_FLOW_NONFRAG_IPV6_UDP, - [I40E_FILTER_PCTYPE_NONF_IPV6_TCP] = - RTE_ETH_FLOW_NONFRAG_IPV6_TCP, - [I40E_FILTER_PCTYPE_NONF_IPV6_SCTP] = - RTE_ETH_FLOW_NONFRAG_IPV6_SCTP, - [I40E_FILTER_PCTYPE_NONF_IPV6_OTHER] = - RTE_ETH_FLOW_NONFRAG_IPV6_OTHER, - [I40E_FILTER_PCTYPE_L2_PAYLOAD] = RTE_ETH_FLOW_L2_PAYLOAD, - }; + uint16_t flowtype; + uint64_t pctype_mask = 1ULL << pctype; - return flowtype_table[pctype]; + for (flowtype = RTE_ETH_FLOW_UNKNOWN + 1; flowtype < I40E_FLOW_TYPE_MAX; + flowtype++) { + if (adapter->pctypes_tbl[flowtype] & pctype_mask) + return flowtype; + } + + return RTE_ETH_FLOW_UNKNOWN; } /* @@ -8051,18 +9676,48 @@ i40e_pctype_to_flowtype(enum i40e_filter_pctype pctype) */ /* For both X710 and XL710 */ -#define I40E_GL_SWR_PRI_JOIN_MAP_0_VALUE 0x10000200 -#define I40E_GL_SWR_PRI_JOIN_MAP_0 0x26CE00 +#define I40E_GL_SWR_PRI_JOIN_MAP_0_VALUE_1 0x10000200 +#define I40E_GL_SWR_PRI_JOIN_MAP_0_VALUE_2 0x203F0200 +#define I40E_GL_SWR_PRI_JOIN_MAP_0 0x26CE00 #define I40E_GL_SWR_PRI_JOIN_MAP_2_VALUE 0x011f0200 #define I40E_GL_SWR_PRI_JOIN_MAP_2 0x26CE08 +/* For X722 */ +#define I40E_X722_GL_SWR_PRI_JOIN_MAP_0_VALUE 0x20000200 +#define I40E_X722_GL_SWR_PRI_JOIN_MAP_2_VALUE 0x013F0200 + /* For X710 */ #define I40E_GL_SWR_PM_UP_THR_EF_VALUE 0x03030303 /* For XL710 */ #define I40E_GL_SWR_PM_UP_THR_SF_VALUE 0x06060606 #define I40E_GL_SWR_PM_UP_THR 0x269FBC +static int +i40e_dev_sync_phy_type(struct i40e_hw *hw) +{ + enum i40e_status_code status; + struct i40e_aq_get_phy_abilities_resp phy_ab; + int ret = -ENOTSUP; + int retries = 0; + + status = i40e_aq_get_phy_capabilities(hw, false, true, &phy_ab, + NULL); + + while (status) { + PMD_INIT_LOG(WARNING, "Failed to sync phy type: status=%d", + status); + retries++; + rte_delay_us(100000); + if (retries < 5) + status = i40e_aq_get_phy_capabilities(hw, false, + true, &phy_ab, NULL); + else + return ret; + } + return 0; +} + static void i40e_configure_registers(struct i40e_hw *hw) { @@ -8070,8 +9725,8 @@ i40e_configure_registers(struct i40e_hw *hw) uint32_t addr; uint64_t val; } reg_table[] = { - {I40E_GL_SWR_PRI_JOIN_MAP_0, I40E_GL_SWR_PRI_JOIN_MAP_0_VALUE}, - {I40E_GL_SWR_PRI_JOIN_MAP_2, I40E_GL_SWR_PRI_JOIN_MAP_2_VALUE}, + {I40E_GL_SWR_PRI_JOIN_MAP_0, 0}, + {I40E_GL_SWR_PRI_JOIN_MAP_2, 0}, {I40E_GL_SWR_PM_UP_THR, 0}, /* Compute value dynamically */ }; uint64_t reg; @@ -8079,8 +9734,31 @@ i40e_configure_registers(struct i40e_hw *hw) int ret; for (i = 0; i < RTE_DIM(reg_table); i++) { + if (reg_table[i].addr == I40E_GL_SWR_PRI_JOIN_MAP_0) { + if (hw->mac.type == I40E_MAC_X722) /* For X722 */ + reg_table[i].val = + I40E_X722_GL_SWR_PRI_JOIN_MAP_0_VALUE; + else /* For X710/XL710/XXV710 */ + if (hw->aq.fw_maj_ver < 6) + reg_table[i].val = + I40E_GL_SWR_PRI_JOIN_MAP_0_VALUE_1; + else + reg_table[i].val = + I40E_GL_SWR_PRI_JOIN_MAP_0_VALUE_2; + } + + if (reg_table[i].addr == I40E_GL_SWR_PRI_JOIN_MAP_2) { + if (hw->mac.type == I40E_MAC_X722) /* For X722 */ + reg_table[i].val = + I40E_X722_GL_SWR_PRI_JOIN_MAP_2_VALUE; + else /* For X710/XL710/XXV710 */ + reg_table[i].val = + I40E_GL_SWR_PRI_JOIN_MAP_2_VALUE; + } + if (reg_table[i].addr == I40E_GL_SWR_PM_UP_THR) { - if (i40e_is_40G_device(hw->device_id)) /* For XL710 */ + if (I40E_PHY_TYPE_SUPPORT_40G(hw->phy.phy_types) || /* For XL710 */ + I40E_PHY_TYPE_SUPPORT_25G(hw->phy.phy_types)) /* For XXV710 */ reg_table[i].val = I40E_GL_SWR_PM_UP_THR_SF_VALUE; else /* For X710 */ @@ -8103,9 +9781,9 @@ i40e_configure_registers(struct i40e_hw *hw) ret = i40e_aq_debug_write_register(hw, reg_table[i].addr, reg_table[i].val, NULL); if (ret < 0) { - PMD_DRV_LOG(ERR, "Failed to write 0x%"PRIx64" to the " - "address of 0x%"PRIx32, reg_table[i].val, - reg_table[i].addr); + PMD_DRV_LOG(ERR, + "Failed to write 0x%"PRIx64" to the address of 0x%"PRIx32, + reg_table[i].val, reg_table[i].addr); break; } PMD_DRV_LOG(DEBUG, "Write 0x%"PRIx64" to the address of " @@ -8150,8 +9828,9 @@ i40e_config_qinq(struct i40e_hw *hw, struct i40e_vsi *vsi) I40E_VSI_L2TAGSTXVALID( vsi->vsi_id), reg, NULL); if (ret < 0) { - PMD_DRV_LOG(ERR, "Failed to update " - "VSI_L2TAGSTXVALID[%d]", vsi->vsi_id); + PMD_DRV_LOG(ERR, + "Failed to update VSI_L2TAGSTXVALID[%d]", + vsi->vsi_id); return I40E_ERR_CONFIG; } } @@ -8202,11 +9881,10 @@ i40e_aq_add_mirror_rule(struct i40e_hw *hw, rte_memcpy(&desc.params.raw, &cmd, sizeof(cmd)); status = i40e_asq_send_command(hw, &desc, entries, buff_len, NULL); - PMD_DRV_LOG(INFO, "i40e_aq_add_mirror_rule, aq_status %d," - "rule_id = %u" - " mirror_rules_used = %u, mirror_rules_free = %u,", - hw->aq.asq_last_status, resp->rule_id, - resp->mirror_rules_used, resp->mirror_rules_free); + PMD_DRV_LOG(INFO, + "i40e_aq_add_mirror_rule, aq_status %d, rule_id = %u mirror_rules_used = %u, mirror_rules_free = %u,", + hw->aq.asq_last_status, resp->rule_id, + resp->mirror_rules_used, resp->mirror_rules_free); *rule_id = rte_le_to_cpu_16(resp->rule_id); return status; @@ -8284,8 +9962,8 @@ i40e_mirror_rule_set(struct rte_eth_dev *dev, PMD_DRV_LOG(DEBUG, "i40e_mirror_rule_set: sw_id = %d.", sw_id); if (pf->main_vsi->veb == NULL || pf->vfs == NULL) { - PMD_DRV_LOG(ERR, "mirror rule can not be configured" - " without veb or vfs."); + PMD_DRV_LOG(ERR, + "mirror rule can not be configured without veb or vfs."); return -ENOSYS; } if (pf->nb_mirror_rule > I40E_MAX_MIRROR_RULES) { @@ -8317,9 +9995,9 @@ i40e_mirror_rule_set(struct rte_eth_dev *dev, mirr_rule->entries, mirr_rule->num_entries, mirr_rule->id); if (ret < 0) { - PMD_DRV_LOG(ERR, "failed to remove mirror rule:" - " ret = %d, aq_err = %d.", - ret, hw->aq.asq_last_status); + PMD_DRV_LOG(ERR, + "failed to remove mirror rule: ret = %d, aq_err = %d.", + ret, hw->aq.asq_last_status); return -ENOSYS; } TAILQ_REMOVE(&pf->mirror_list, mirr_rule, rules); @@ -8408,9 +10086,9 @@ i40e_mirror_rule_set(struct rte_eth_dev *dev, mirr_rule->rule_type, mirr_rule->entries, j, &rule_id); if (ret < 0) { - PMD_DRV_LOG(ERR, "failed to add mirror rule:" - " ret = %d, aq_err = %d.", - ret, hw->aq.asq_last_status); + PMD_DRV_LOG(ERR, + "failed to add mirror rule: ret = %d, aq_err = %d.", + ret, hw->aq.asq_last_status); rte_free(mirr_rule); return -ENOSYS; } @@ -8462,9 +10140,9 @@ i40e_mirror_rule_reset(struct rte_eth_dev *dev, uint8_t sw_id) mirr_rule->entries, mirr_rule->num_entries, mirr_rule->id); if (ret < 0) { - PMD_DRV_LOG(ERR, "failed to remove mirror rule:" - " status = %d, aq_err = %d.", - ret, hw->aq.asq_last_status); + PMD_DRV_LOG(ERR, + "failed to remove mirror rule: status = %d, aq_err = %d.", + ret, hw->aq.asq_last_status); return -ENOSYS; } TAILQ_REMOVE(&pf->mirror_list, mirr_rule, rules); @@ -8896,9 +10574,9 @@ i40e_config_switch_comp_tc(struct i40e_veb *veb, uint8_t tc_map) ret = i40e_aq_config_switch_comp_bw_config(hw, veb->seid, &veb_bw, NULL); if (ret) { - PMD_INIT_LOG(ERR, "AQ command Config switch_comp BW allocation" - " per TC failed = %d", - hw->aq.asq_last_status); + PMD_INIT_LOG(ERR, + "AQ command Config switch_comp BW allocation per TC failed = %d", + hw->aq.asq_last_status); return ret; } @@ -8906,16 +10584,18 @@ i40e_config_switch_comp_tc(struct i40e_veb *veb, uint8_t tc_map) ret = i40e_aq_query_switch_comp_ets_config(hw, veb->seid, &ets_query, NULL); if (ret != I40E_SUCCESS) { - PMD_DRV_LOG(ERR, "Failed to get switch_comp ETS" - " configuration %u", hw->aq.asq_last_status); + PMD_DRV_LOG(ERR, + "Failed to get switch_comp ETS configuration %u", + hw->aq.asq_last_status); return ret; } memset(&bw_query, 0, sizeof(bw_query)); ret = i40e_aq_query_switch_comp_bw_config(hw, veb->seid, &bw_query, NULL); if (ret != I40E_SUCCESS) { - PMD_DRV_LOG(ERR, "Failed to get switch_comp bandwidth" - " configuration %u", hw->aq.asq_last_status); + PMD_DRV_LOG(ERR, + "Failed to get switch_comp bandwidth configuration %u", + hw->aq.asq_last_status); return ret; } @@ -8980,8 +10660,8 @@ i40e_vsi_config_tc(struct i40e_vsi *vsi, uint8_t tc_map) } ret = i40e_aq_config_vsi_tc_bw(hw, vsi->seid, &bw_data, NULL); if (ret) { - PMD_INIT_LOG(ERR, "AQ command Config VSI BW allocation" - " per TC failed = %d", + PMD_INIT_LOG(ERR, + "AQ command Config VSI BW allocation per TC failed = %d", hw->aq.asq_last_status); goto out; } @@ -9002,15 +10682,14 @@ i40e_vsi_config_tc(struct i40e_vsi *vsi, uint8_t tc_map) /* Update the VSI after updating the VSI queue-mapping information */ ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); if (ret) { - PMD_INIT_LOG(ERR, "Failed to configure " - "TC queue mapping = %d", - hw->aq.asq_last_status); + PMD_INIT_LOG(ERR, "Failed to configure TC queue mapping = %d", + hw->aq.asq_last_status); goto out; } /* update the local VSI info with updated queue map */ - (void)rte_memcpy(&vsi->info.tc_mapping, &ctxt.info.tc_mapping, + rte_memcpy(&vsi->info.tc_mapping, &ctxt.info.tc_mapping, sizeof(vsi->info.tc_mapping)); - (void)rte_memcpy(&vsi->info.queue_mapping, + rte_memcpy(&vsi->info.queue_mapping, &ctxt.info.queue_mapping, sizeof(vsi->info.queue_mapping)); vsi->info.mapping_flags = ctxt.info.mapping_flags; @@ -9056,8 +10735,8 @@ i40e_dcb_hw_configure(struct i40e_pf *pf, /* Use the FW API if FW > v4.4*/ if (!(((hw->aq.fw_maj_ver == 4) && (hw->aq.fw_min_ver >= 4)) || (hw->aq.fw_maj_ver >= 5))) { - PMD_INIT_LOG(ERR, "FW < v4.4, can not use FW LLDP API" - " to configure DCB"); + PMD_INIT_LOG(ERR, + "FW < v4.4, can not use FW LLDP API to configure DCB"); return I40E_ERR_FIRMWARE_API_VERSION; } @@ -9072,8 +10751,7 @@ i40e_dcb_hw_configure(struct i40e_pf *pf, old_cfg->etsrec = old_cfg->etscfg; ret = i40e_set_dcb_config(hw); if (ret) { - PMD_INIT_LOG(ERR, - "Set DCB Config failed, err %s aq_err %s\n", + PMD_INIT_LOG(ERR, "Set DCB Config failed, err %s aq_err %s", i40e_stat_str(hw, ret), i40e_aq_str(hw, hw->aq.asq_last_status)); return ret; @@ -9105,7 +10783,7 @@ i40e_dcb_hw_configure(struct i40e_pf *pf, ret = i40e_config_switch_comp_tc(main_vsi->veb, tc_map); if (ret) PMD_INIT_LOG(WARNING, - "Failed configuring TC for VEB seid=%d\n", + "Failed configuring TC for VEB seid=%d", main_vsi->veb->seid); } /* Update each VSI */ @@ -9123,8 +10801,8 @@ i40e_dcb_hw_configure(struct i40e_pf *pf, I40E_DEFAULT_TCMAP); if (ret) PMD_INIT_LOG(WARNING, - "Failed configuring TC for VSI seid=%d\n", - vsi_list->vsi->seid); + "Failed configuring TC for VSI seid=%d", + vsi_list->vsi->seid); /* continue */ } } @@ -9138,12 +10816,12 @@ i40e_dcb_hw_configure(struct i40e_pf *pf, * * Returns 0 on success, negative value on failure */ -static int +int i40e_dcb_init_configure(struct rte_eth_dev *dev, bool sw_dcb) { struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); - int ret = 0; + int i, ret = 0; if ((pf->flags & I40E_FLAG_DCB) == 0) { PMD_INIT_LOG(ERR, "HW doesn't support DCB"); @@ -9170,6 +10848,9 @@ i40e_dcb_init_configure(struct rte_eth_dev *dev, bool sw_dcb) hw->local_dcbx_config.etscfg.tcbwtable[0] = 100; hw->local_dcbx_config.etscfg.tsatable[0] = I40E_IEEE_TSA_ETS; + /* all UPs mapping to TC0 */ + for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) + hw->local_dcbx_config.etscfg.prioritytable[i] = 0; hw->local_dcbx_config.etsrec = hw->local_dcbx_config.etscfg; hw->local_dcbx_config.pfc.willing = 0; @@ -9184,15 +10865,15 @@ i40e_dcb_init_configure(struct rte_eth_dev *dev, bool sw_dcb) I40E_APP_PROTOID_FCOE; ret = i40e_set_dcb_config(hw); if (ret) { - PMD_INIT_LOG(ERR, "default dcb config fails." - " err = %d, aq_err = %d.", ret, - hw->aq.asq_last_status); + PMD_INIT_LOG(ERR, + "default dcb config fails. err = %d, aq_err = %d.", + ret, hw->aq.asq_last_status); return -ENOSYS; } } else { - PMD_INIT_LOG(ERR, "DCB initialization in FW fails," - " err = %d, aq_err = %d.", ret, - hw->aq.asq_last_status); + PMD_INIT_LOG(ERR, + "DCB initialization in FW fails, err = %d, aq_err = %d.", + ret, hw->aq.asq_last_status); return -ENOTSUP; } } else { @@ -9203,14 +10884,14 @@ i40e_dcb_init_configure(struct rte_eth_dev *dev, bool sw_dcb) ret = i40e_init_dcb(hw); if (!ret) { if (hw->dcbx_status == I40E_DCBX_STATUS_DISABLED) { - PMD_INIT_LOG(ERR, "HW doesn't support" - " DCBX offload."); + PMD_INIT_LOG(ERR, + "HW doesn't support DCBX offload."); return -ENOTSUP; } } else { - PMD_INIT_LOG(ERR, "DCBX configuration failed, err = %d," - " aq_err = %d.", ret, - hw->aq.asq_last_status); + PMD_INIT_LOG(ERR, + "DCBX configuration failed, err = %d, aq_err = %d.", + ret, hw->aq.asq_last_status); return -ENOTSUP; } } @@ -9319,32 +11000,27 @@ i40e_dev_get_dcb_info(struct rte_eth_dev *dev, static int i40e_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) { - struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); - uint16_t interval = - i40e_calc_itr_interval(RTE_LIBRTE_I40E_ITR_INTERVAL); uint16_t msix_intr; msix_intr = intr_handle->intr_vec[queue_id]; if (msix_intr == I40E_MISC_VEC_ID) I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTL0, - I40E_PFINT_DYN_CTLN_INTENA_MASK | - I40E_PFINT_DYN_CTLN_CLEARPBA_MASK | - (0 << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT) | - (interval << - I40E_PFINT_DYN_CTLN_INTERVAL_SHIFT)); + I40E_PFINT_DYN_CTL0_INTENA_MASK | + I40E_PFINT_DYN_CTL0_CLEARPBA_MASK | + I40E_PFINT_DYN_CTL0_ITR_INDX_MASK); else I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTLN(msix_intr - I40E_RX_VEC_START), I40E_PFINT_DYN_CTLN_INTENA_MASK | I40E_PFINT_DYN_CTLN_CLEARPBA_MASK | - (0 << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT) | - (interval << - I40E_PFINT_DYN_CTLN_INTERVAL_SHIFT)); + I40E_PFINT_DYN_CTLN_ITR_INDX_MASK); I40E_WRITE_FLUSH(hw); - rte_intr_enable(&dev->pci_dev->intr_handle); + rte_intr_enable(&pci_dev->intr_handle); return 0; } @@ -9352,18 +11028,20 @@ i40e_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) static int i40e_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id) { - struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); uint16_t msix_intr; msix_intr = intr_handle->intr_vec[queue_id]; if (msix_intr == I40E_MISC_VEC_ID) - I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTL0, 0); + I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTL0, + I40E_PFINT_DYN_CTL0_ITR_INDX_MASK); else I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTLN(msix_intr - I40E_RX_VEC_START), - 0); + I40E_PFINT_DYN_CTLN_ITR_INDX_MASK); I40E_WRITE_FLUSH(hw); return 0; @@ -9459,14 +11137,43 @@ static void i40e_set_default_mac_addr(struct rte_eth_dev *dev, struct ether_addr *mac_addr) { struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + struct i40e_vsi *vsi = pf->main_vsi; + struct i40e_mac_filter_info mac_filter; + struct i40e_mac_filter *f; + int ret; if (!is_valid_assigned_ether_addr(mac_addr)) { PMD_DRV_LOG(ERR, "Tried to set invalid MAC address."); return; } - /* Flags: 0x3 updates port address */ - i40e_aq_mac_address_write(hw, 0x3, mac_addr->addr_bytes, NULL); + TAILQ_FOREACH(f, &vsi->mac_list, next) { + if (is_same_ether_addr(&pf->dev_addr, &f->mac_info.mac_addr)) + break; + } + + if (f == NULL) { + PMD_DRV_LOG(ERR, "Failed to find filter for default mac"); + return; + } + + mac_filter = f->mac_info; + ret = i40e_vsi_delete_mac(vsi, &mac_filter.mac_addr); + if (ret != I40E_SUCCESS) { + PMD_DRV_LOG(ERR, "Failed to delete mac filter"); + return; + } + memcpy(&mac_filter.mac_addr, mac_addr, ETH_ADDR_LEN); + ret = i40e_vsi_add_mac(vsi, &mac_filter); + if (ret != I40E_SUCCESS) { + PMD_DRV_LOG(ERR, "Failed to add mac filter"); + return; + } + memcpy(&pf->dev_addr, mac_addr, ETH_ADDR_LEN); + + i40e_aq_mac_address_write(hw, I40E_AQC_WRITE_TYPE_LAA_WOL, + mac_addr->addr_bytes, NULL); } static int @@ -9474,8 +11181,7 @@ i40e_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) { struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); struct rte_eth_dev_data *dev_data = pf->dev_data; - uint32_t frame_size = mtu + ETHER_HDR_LEN - + ETHER_CRC_LEN + I40E_VLAN_TAG_SIZE; + uint32_t frame_size = mtu + I40E_ETH_OVERHEAD; int ret = 0; /* check if mtu is within the allowed range */ @@ -9484,8 +11190,7 @@ i40e_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) /* mtu setting is forbidden if port is start */ if (dev_data->dev_started) { - PMD_DRV_LOG(ERR, - "port %d must be stopped before configuration\n", + PMD_DRV_LOG(ERR, "port %d must be stopped before configuration", dev_data->port_id); return -EBUSY; } @@ -9499,3 +11204,543 @@ i40e_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) return ret; } + +/* Restore ethertype filter */ +static void +i40e_ethertype_filter_restore(struct i40e_pf *pf) +{ + struct i40e_hw *hw = I40E_PF_TO_HW(pf); + struct i40e_ethertype_filter_list + *ethertype_list = &pf->ethertype.ethertype_list; + struct i40e_ethertype_filter *f; + struct i40e_control_filter_stats stats; + uint16_t flags; + + TAILQ_FOREACH(f, ethertype_list, rules) { + flags = 0; + if (!(f->flags & RTE_ETHTYPE_FLAGS_MAC)) + flags |= I40E_AQC_ADD_CONTROL_PACKET_FLAGS_IGNORE_MAC; + if (f->flags & RTE_ETHTYPE_FLAGS_DROP) + flags |= I40E_AQC_ADD_CONTROL_PACKET_FLAGS_DROP; + flags |= I40E_AQC_ADD_CONTROL_PACKET_FLAGS_TO_QUEUE; + + memset(&stats, 0, sizeof(stats)); + i40e_aq_add_rem_control_packet_filter(hw, + f->input.mac_addr.addr_bytes, + f->input.ether_type, + flags, pf->main_vsi->seid, + f->queue, 1, &stats, NULL); + } + PMD_DRV_LOG(INFO, "Ethertype filter:" + " mac_etype_used = %u, etype_used = %u," + " mac_etype_free = %u, etype_free = %u", + stats.mac_etype_used, stats.etype_used, + stats.mac_etype_free, stats.etype_free); +} + +/* Restore tunnel filter */ +static void +i40e_tunnel_filter_restore(struct i40e_pf *pf) +{ + struct i40e_hw *hw = I40E_PF_TO_HW(pf); + struct i40e_vsi *vsi; + struct i40e_pf_vf *vf; + struct i40e_tunnel_filter_list + *tunnel_list = &pf->tunnel.tunnel_list; + struct i40e_tunnel_filter *f; + struct i40e_aqc_add_rm_cloud_filt_elem_ext cld_filter; + bool big_buffer = 0; + + TAILQ_FOREACH(f, tunnel_list, rules) { + if (!f->is_to_vf) + vsi = pf->main_vsi; + else { + vf = &pf->vfs[f->vf_id]; + vsi = vf->vsi; + } + memset(&cld_filter, 0, sizeof(cld_filter)); + ether_addr_copy((struct ether_addr *)&f->input.outer_mac, + (struct ether_addr *)&cld_filter.element.outer_mac); + ether_addr_copy((struct ether_addr *)&f->input.inner_mac, + (struct ether_addr *)&cld_filter.element.inner_mac); + cld_filter.element.inner_vlan = f->input.inner_vlan; + cld_filter.element.flags = f->input.flags; + cld_filter.element.tenant_id = f->input.tenant_id; + cld_filter.element.queue_number = f->queue; + rte_memcpy(cld_filter.general_fields, + f->input.general_fields, + sizeof(f->input.general_fields)); + + if (((f->input.flags & + I40E_AQC_ADD_CLOUD_FILTER_0X11) == + I40E_AQC_ADD_CLOUD_FILTER_0X11) || + ((f->input.flags & + I40E_AQC_ADD_CLOUD_FILTER_0X12) == + I40E_AQC_ADD_CLOUD_FILTER_0X12) || + ((f->input.flags & + I40E_AQC_ADD_CLOUD_FILTER_0X10) == + I40E_AQC_ADD_CLOUD_FILTER_0X10)) + big_buffer = 1; + + if (big_buffer) + i40e_aq_add_cloud_filters_big_buffer(hw, + vsi->seid, &cld_filter, 1); + else + i40e_aq_add_cloud_filters(hw, vsi->seid, + &cld_filter.element, 1); + } +} + +static void +i40e_filter_restore(struct i40e_pf *pf) +{ + i40e_ethertype_filter_restore(pf); + i40e_tunnel_filter_restore(pf); + i40e_fdir_filter_restore(pf); +} + +static bool +is_device_supported(struct rte_eth_dev *dev, struct rte_pci_driver *drv) +{ + if (strcmp(dev->device->driver->name, drv->driver.name)) + return false; + + return true; +} + +bool +is_i40e_supported(struct rte_eth_dev *dev) +{ + return is_device_supported(dev, &rte_i40e_pmd); +} + +struct i40e_customized_pctype* +i40e_find_customized_pctype(struct i40e_pf *pf, uint8_t index) +{ + int i; + + for (i = 0; i < I40E_CUSTOMIZED_MAX; i++) { + if (pf->customized_pctype[i].index == index) + return &pf->customized_pctype[i]; + } + return NULL; +} + +static int +i40e_update_customized_pctype(struct rte_eth_dev *dev, uint8_t *pkg, + uint32_t pkg_size, uint32_t proto_num, + struct rte_pmd_i40e_proto_info *proto) +{ + struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + uint32_t pctype_num; + struct rte_pmd_i40e_ptype_info *pctype; + uint32_t buff_size; + struct i40e_customized_pctype *new_pctype = NULL; + uint8_t proto_id; + uint8_t pctype_value; + char name[64]; + uint32_t i, j, n; + int ret; + + ret = rte_pmd_i40e_get_ddp_info(pkg, pkg_size, + (uint8_t *)&pctype_num, sizeof(pctype_num), + RTE_PMD_I40E_PKG_INFO_PCTYPE_NUM); + if (ret) { + PMD_DRV_LOG(ERR, "Failed to get pctype number"); + return -1; + } + if (!pctype_num) { + PMD_DRV_LOG(INFO, "No new pctype added"); + return -1; + } + + buff_size = pctype_num * sizeof(struct rte_pmd_i40e_proto_info); + pctype = rte_zmalloc("new_pctype", buff_size, 0); + if (!pctype) { + PMD_DRV_LOG(ERR, "Failed to allocate memory"); + return -1; + } + /* get information about new pctype list */ + ret = rte_pmd_i40e_get_ddp_info(pkg, pkg_size, + (uint8_t *)pctype, buff_size, + RTE_PMD_I40E_PKG_INFO_PCTYPE_LIST); + if (ret) { + PMD_DRV_LOG(ERR, "Failed to get pctype list"); + rte_free(pctype); + return -1; + } + + /* Update customized pctype. */ + for (i = 0; i < pctype_num; i++) { + pctype_value = pctype[i].ptype_id; + memset(name, 0, sizeof(name)); + for (j = 0; j < RTE_PMD_I40E_PROTO_NUM; j++) { + proto_id = pctype[i].protocols[j]; + if (proto_id == RTE_PMD_I40E_PROTO_UNUSED) + continue; + for (n = 0; n < proto_num; n++) { + if (proto[n].proto_id != proto_id) + continue; + strcat(name, proto[n].name); + strcat(name, "_"); + break; + } + } + name[strlen(name) - 1] = '\0'; + if (!strcmp(name, "GTPC")) + new_pctype = + i40e_find_customized_pctype(pf, + I40E_CUSTOMIZED_GTPC); + else if (!strcmp(name, "GTPU_IPV4")) + new_pctype = + i40e_find_customized_pctype(pf, + I40E_CUSTOMIZED_GTPU_IPV4); + else if (!strcmp(name, "GTPU_IPV6")) + new_pctype = + i40e_find_customized_pctype(pf, + I40E_CUSTOMIZED_GTPU_IPV6); + else if (!strcmp(name, "GTPU")) + new_pctype = + i40e_find_customized_pctype(pf, + I40E_CUSTOMIZED_GTPU); + if (new_pctype) { + new_pctype->pctype = pctype_value; + new_pctype->valid = true; + } + } + + rte_free(pctype); + return 0; +} + +static int +i40e_update_customized_ptype(struct rte_eth_dev *dev, uint8_t *pkg, + uint32_t pkg_size, uint32_t proto_num, + struct rte_pmd_i40e_proto_info *proto) +{ + struct rte_pmd_i40e_ptype_mapping *ptype_mapping; + uint16_t port_id = dev->data->port_id; + uint32_t ptype_num; + struct rte_pmd_i40e_ptype_info *ptype; + uint32_t buff_size; + uint8_t proto_id; + char name[RTE_PMD_I40E_DDP_NAME_SIZE]; + uint32_t i, j, n; + bool inner_ip; + int ret; + + /* get information about new ptype num */ + ret = rte_pmd_i40e_get_ddp_info(pkg, pkg_size, + (uint8_t *)&ptype_num, sizeof(ptype_num), + RTE_PMD_I40E_PKG_INFO_PTYPE_NUM); + if (ret) { + PMD_DRV_LOG(ERR, "Failed to get ptype number"); + return ret; + } + if (!ptype_num) { + PMD_DRV_LOG(INFO, "No new ptype added"); + return -1; + } + + buff_size = ptype_num * sizeof(struct rte_pmd_i40e_ptype_info); + ptype = rte_zmalloc("new_ptype", buff_size, 0); + if (!ptype) { + PMD_DRV_LOG(ERR, "Failed to allocate memory"); + return -1; + } + + /* get information about new ptype list */ + ret = rte_pmd_i40e_get_ddp_info(pkg, pkg_size, + (uint8_t *)ptype, buff_size, + RTE_PMD_I40E_PKG_INFO_PTYPE_LIST); + if (ret) { + PMD_DRV_LOG(ERR, "Failed to get ptype list"); + rte_free(ptype); + return ret; + } + + buff_size = ptype_num * sizeof(struct rte_pmd_i40e_ptype_mapping); + ptype_mapping = rte_zmalloc("ptype_mapping", buff_size, 0); + if (!ptype_mapping) { + PMD_DRV_LOG(ERR, "Failed to allocate memory"); + rte_free(ptype); + return -1; + } + + /* Update ptype mapping table. */ + for (i = 0; i < ptype_num; i++) { + ptype_mapping[i].hw_ptype = ptype[i].ptype_id; + ptype_mapping[i].sw_ptype = 0; + inner_ip = false; + for (j = 0; j < RTE_PMD_I40E_PROTO_NUM; j++) { + proto_id = ptype[i].protocols[j]; + if (proto_id == RTE_PMD_I40E_PROTO_UNUSED) + continue; + for (n = 0; n < proto_num; n++) { + if (proto[n].proto_id != proto_id) + continue; + memset(name, 0, sizeof(name)); + strcpy(name, proto[n].name); + if (!strncmp(name, "IPV4", 4) && !inner_ip) { + ptype_mapping[i].sw_ptype |= + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN; + inner_ip = true; + } else if (!strncmp(name, "IPV4FRAG", 8) && + inner_ip) { + ptype_mapping[i].sw_ptype |= + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN; + ptype_mapping[i].sw_ptype |= + RTE_PTYPE_INNER_L4_FRAG; + } else if (!strncmp(name, "IPV4", 4) && + inner_ip) + ptype_mapping[i].sw_ptype |= + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN; + else if (!strncmp(name, "IPV6", 4) && + !inner_ip) { + ptype_mapping[i].sw_ptype |= + RTE_PTYPE_L3_IPV6_EXT_UNKNOWN; + inner_ip = true; + } else if (!strncmp(name, "IPV6FRAG", 8) && + inner_ip) { + ptype_mapping[i].sw_ptype |= + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN; + ptype_mapping[i].sw_ptype |= + RTE_PTYPE_INNER_L4_FRAG; + } else if (!strncmp(name, "IPV6", 4) && + inner_ip) + ptype_mapping[i].sw_ptype |= + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN; + else if (!strncmp(name, "GTPC", 4)) + ptype_mapping[i].sw_ptype |= + RTE_PTYPE_TUNNEL_GTPC; + else if (!strncmp(name, "GTPU", 4)) + ptype_mapping[i].sw_ptype |= + RTE_PTYPE_TUNNEL_GTPU; + else if (!strncmp(name, "UDP", 3)) + ptype_mapping[i].sw_ptype |= + RTE_PTYPE_INNER_L4_UDP; + else if (!strncmp(name, "TCP", 3)) + ptype_mapping[i].sw_ptype |= + RTE_PTYPE_INNER_L4_TCP; + else if (!strncmp(name, "SCTP", 4)) + ptype_mapping[i].sw_ptype |= + RTE_PTYPE_INNER_L4_SCTP; + else if (!strncmp(name, "ICMP", 4) || + !strncmp(name, "ICMPV6", 6)) + ptype_mapping[i].sw_ptype |= + RTE_PTYPE_INNER_L4_ICMP; + + break; + } + } + } + + ret = rte_pmd_i40e_ptype_mapping_update(port_id, ptype_mapping, + ptype_num, 0); + if (ret) + PMD_DRV_LOG(ERR, "Failed to update mapping table."); + + rte_free(ptype_mapping); + rte_free(ptype); + return ret; +} + +void +i40e_update_customized_info(struct rte_eth_dev *dev, uint8_t *pkg, + uint32_t pkg_size) +{ + struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + uint32_t proto_num; + struct rte_pmd_i40e_proto_info *proto; + uint32_t buff_size; + uint32_t i; + int ret; + + /* get information about protocol number */ + ret = rte_pmd_i40e_get_ddp_info(pkg, pkg_size, + (uint8_t *)&proto_num, sizeof(proto_num), + RTE_PMD_I40E_PKG_INFO_PROTOCOL_NUM); + if (ret) { + PMD_DRV_LOG(ERR, "Failed to get protocol number"); + return; + } + if (!proto_num) { + PMD_DRV_LOG(INFO, "No new protocol added"); + return; + } + + buff_size = proto_num * sizeof(struct rte_pmd_i40e_proto_info); + proto = rte_zmalloc("new_proto", buff_size, 0); + if (!proto) { + PMD_DRV_LOG(ERR, "Failed to allocate memory"); + return; + } + + /* get information about protocol list */ + ret = rte_pmd_i40e_get_ddp_info(pkg, pkg_size, + (uint8_t *)proto, buff_size, + RTE_PMD_I40E_PKG_INFO_PROTOCOL_LIST); + if (ret) { + PMD_DRV_LOG(ERR, "Failed to get protocol list"); + rte_free(proto); + return; + } + + /* Check if GTP is supported. */ + for (i = 0; i < proto_num; i++) { + if (!strncmp(proto[i].name, "GTP", 3)) { + pf->gtp_support = true; + break; + } + } + + /* Update customized pctype info */ + ret = i40e_update_customized_pctype(dev, pkg, pkg_size, + proto_num, proto); + if (ret) + PMD_DRV_LOG(INFO, "No pctype is updated."); + + /* Update customized ptype info */ + ret = i40e_update_customized_ptype(dev, pkg, pkg_size, + proto_num, proto); + if (ret) + PMD_DRV_LOG(INFO, "No ptype is updated."); + + rte_free(proto); +} + +/* Create a QinQ cloud filter + * + * The Fortville NIC has limited resources for tunnel filters, + * so we can only reuse existing filters. + * + * In step 1 we define which Field Vector fields can be used for + * filter types. + * As we do not have the inner tag defined as a field, + * we have to define it first, by reusing one of L1 entries. + * + * In step 2 we are replacing one of existing filter types with + * a new one for QinQ. + * As we reusing L1 and replacing L2, some of the default filter + * types will disappear,which depends on L1 and L2 entries we reuse. + * + * Step 1: Create L1 filter of outer vlan (12b) + inner vlan (12b) + * + * 1. Create L1 filter of outer vlan (12b) which will be in use + * later when we define the cloud filter. + * a. Valid_flags.replace_cloud = 0 + * b. Old_filter = 10 (Stag_Inner_Vlan) + * c. New_filter = 0x10 + * d. TR bit = 0xff (optional, not used here) + * e. Buffer – 2 entries: + * i. Byte 0 = 8 (outer vlan FV index). + * Byte 1 = 0 (rsv) + * Byte 2-3 = 0x0fff + * ii. Byte 0 = 37 (inner vlan FV index). + * Byte 1 =0 (rsv) + * Byte 2-3 = 0x0fff + * + * Step 2: + * 2. Create cloud filter using two L1 filters entries: stag and + * new filter(outer vlan+ inner vlan) + * a. Valid_flags.replace_cloud = 1 + * b. Old_filter = 1 (instead of outer IP) + * c. New_filter = 0x10 + * d. Buffer – 2 entries: + * i. Byte 0 = 0x80 | 7 (valid | Stag). + * Byte 1-3 = 0 (rsv) + * ii. Byte 8 = 0x80 | 0x10 (valid | new l1 filter step1) + * Byte 9-11 = 0 (rsv) + */ +static int +i40e_cloud_filter_qinq_create(struct i40e_pf *pf) +{ + int ret = -ENOTSUP; + struct i40e_aqc_replace_cloud_filters_cmd filter_replace; + struct i40e_aqc_replace_cloud_filters_cmd_buf filter_replace_buf; + struct i40e_hw *hw = I40E_PF_TO_HW(pf); + + if (pf->support_multi_driver) { + PMD_DRV_LOG(ERR, "Replace cloud filter is not supported."); + return ret; + } + + /* Init */ + memset(&filter_replace, 0, + sizeof(struct i40e_aqc_replace_cloud_filters_cmd)); + memset(&filter_replace_buf, 0, + sizeof(struct i40e_aqc_replace_cloud_filters_cmd_buf)); + + /* create L1 filter */ + filter_replace.old_filter_type = + I40E_AQC_REPLACE_CLOUD_CMD_INPUT_FV_STAG_IVLAN; + filter_replace.new_filter_type = I40E_AQC_ADD_CLOUD_FILTER_0X10; + filter_replace.tr_bit = 0; + + /* Prepare the buffer, 2 entries */ + filter_replace_buf.data[0] = I40E_AQC_REPLACE_CLOUD_CMD_INPUT_FV_VLAN; + filter_replace_buf.data[0] |= + I40E_AQC_REPLACE_CLOUD_CMD_INPUT_VALIDATED; + /* Field Vector 12b mask */ + filter_replace_buf.data[2] = 0xff; + filter_replace_buf.data[3] = 0x0f; + filter_replace_buf.data[4] = + I40E_AQC_REPLACE_CLOUD_CMD_INPUT_FV_INNER_VLAN; + filter_replace_buf.data[4] |= + I40E_AQC_REPLACE_CLOUD_CMD_INPUT_VALIDATED; + /* Field Vector 12b mask */ + filter_replace_buf.data[6] = 0xff; + filter_replace_buf.data[7] = 0x0f; + ret = i40e_aq_replace_cloud_filters(hw, &filter_replace, + &filter_replace_buf); + if (ret != I40E_SUCCESS) + return ret; + PMD_DRV_LOG(DEBUG, "Global configuration modification: " + "cloud l1 type is changed from 0x%x to 0x%x", + filter_replace.old_filter_type, + filter_replace.new_filter_type); + + /* Apply the second L2 cloud filter */ + memset(&filter_replace, 0, + sizeof(struct i40e_aqc_replace_cloud_filters_cmd)); + memset(&filter_replace_buf, 0, + sizeof(struct i40e_aqc_replace_cloud_filters_cmd_buf)); + + /* create L2 filter, input for L2 filter will be L1 filter */ + filter_replace.valid_flags = I40E_AQC_REPLACE_CLOUD_FILTER; + filter_replace.old_filter_type = I40E_AQC_ADD_CLOUD_FILTER_OIP; + filter_replace.new_filter_type = I40E_AQC_ADD_CLOUD_FILTER_0X10; + + /* Prepare the buffer, 2 entries */ + filter_replace_buf.data[0] = I40E_AQC_REPLACE_CLOUD_CMD_INPUT_FV_STAG; + filter_replace_buf.data[0] |= + I40E_AQC_REPLACE_CLOUD_CMD_INPUT_VALIDATED; + filter_replace_buf.data[4] = I40E_AQC_ADD_CLOUD_FILTER_0X10; + filter_replace_buf.data[4] |= + I40E_AQC_REPLACE_CLOUD_CMD_INPUT_VALIDATED; + ret = i40e_aq_replace_cloud_filters(hw, &filter_replace, + &filter_replace_buf); + if (!ret) { + i40e_global_cfg_warning(I40E_WARNING_RPL_CLD_FILTER); + PMD_DRV_LOG(DEBUG, "Global configuration modification: " + "cloud filter type is changed from 0x%x to 0x%x", + filter_replace.old_filter_type, + filter_replace.new_filter_type); + } + return ret; +} + +RTE_INIT(i40e_init_log); +static void +i40e_init_log(void) +{ + i40e_logtype_init = rte_log_register("pmd.i40e.init"); + if (i40e_logtype_init >= 0) + rte_log_set_level(i40e_logtype_init, RTE_LOG_NOTICE); + i40e_logtype_driver = rte_log_register("pmd.i40e.driver"); + if (i40e_logtype_driver >= 0) + rte_log_set_level(i40e_logtype_driver, RTE_LOG_NOTICE); +} + +RTE_PMD_REGISTER_PARAM_STRING(net_i40e, + ETH_I40E_SUPPORT_MULTI_DRIVER "=1"); diff --git a/dpdk/drivers/net/i40e/i40e_ethdev.h b/dpdk/drivers/net/i40e/i40e_ethdev.h index 61dfa932..229c974e 100644 --- a/dpdk/drivers/net/i40e/i40e_ethdev.h +++ b/dpdk/drivers/net/i40e/i40e_ethdev.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -37,6 +37,9 @@ #include #include #include +#include +#include +#include #define I40E_VLAN_TAG_SIZE 4 @@ -100,9 +103,18 @@ /* Linux PF host with virtchnl version 1.1 */ #define PF_IS_V11(vf) \ - (((vf)->version_major == I40E_VIRTCHNL_VERSION_MAJOR) && \ + (((vf)->version_major == VIRTCHNL_VERSION_MAJOR) && \ ((vf)->version_minor == 1)) +#define I40E_WRITE_GLB_REG(hw, reg, value) \ + do { \ + I40E_PCI_REG_WRITE(I40E_PCI_REG_ADDR((hw), \ + (reg)), (value)); \ + PMD_DRV_LOG(DEBUG, "Global register 0x%08x is modified " \ + "with value 0x%08x", \ + (reg), (value)); \ + } while (0) + /* index flex payload per layer */ enum i40e_flxpld_layer_idx { I40E_FLXPLD_L2_IDX = 0, @@ -126,6 +138,7 @@ enum i40e_flxpld_layer_idx { #define I40E_FLAG_FDIR (1ULL << 6) #define I40E_FLAG_VXLAN (1ULL << 7) #define I40E_FLAG_RSS_AQ_CAPABLE (1ULL << 8) +#define I40E_FLAG_VF_MAC_BY_PF (1ULL << 9) #define I40E_FLAG_ALL (I40E_FLAG_RSS | \ I40E_FLAG_DCB | \ I40E_FLAG_VMDQ | \ @@ -134,7 +147,8 @@ enum i40e_flxpld_layer_idx { I40E_FLAG_HEADER_SPLIT_ENABLED | \ I40E_FLAG_FDIR | \ I40E_FLAG_VXLAN | \ - I40E_FLAG_RSS_AQ_CAPABLE) + I40E_FLAG_RSS_AQ_CAPABLE | \ + I40E_FLAG_VF_MAC_BY_PF) #define I40E_RSS_OFFLOAD_ALL ( \ ETH_RSS_FRAG_IPV4 | \ @@ -149,6 +163,16 @@ enum i40e_flxpld_layer_idx { ETH_RSS_NONFRAG_IPV6_OTHER | \ ETH_RSS_L2_PAYLOAD) +/* All bits of RSS hash enable for X722*/ +#define I40E_RSS_HENA_ALL_X722 ( \ + (1ULL << I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP) | \ + (1ULL << I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP) | \ + (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK) | \ + (1ULL << I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP) | \ + (1ULL << I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP) | \ + (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK) | \ + I40E_RSS_HENA_ALL) + /* All bits of RSS hash enable */ #define I40E_RSS_HENA_ALL ( \ (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_UDP) | \ @@ -171,13 +195,94 @@ enum i40e_flxpld_layer_idx { /* Default queue interrupt throttling time in microseconds */ #define I40E_ITR_INDEX_DEFAULT 0 +#define I40E_ITR_INDEX_NONE 3 #define I40E_QUEUE_ITR_INTERVAL_DEFAULT 32 /* 32 us */ #define I40E_QUEUE_ITR_INTERVAL_MAX 8160 /* 8160 us */ - /* Special FW support this floating VEB feature */ #define FLOATING_VEB_SUPPORTED_FW_MAJ 5 #define FLOATING_VEB_SUPPORTED_FW_MIN 0 +#define I40E_GL_SWT_L2TAGCTRL(_i) (0x001C0A70 + ((_i) * 4)) +#define I40E_GL_SWT_L2TAGCTRL_ETHERTYPE_SHIFT 16 +#define I40E_GL_SWT_L2TAGCTRL_ETHERTYPE_MASK \ + I40E_MASK(0xFFFF, I40E_GL_SWT_L2TAGCTRL_ETHERTYPE_SHIFT) + +#define I40E_INSET_NONE 0x00000000000000000ULL + +/* bit0 ~ bit 7 */ +#define I40E_INSET_DMAC 0x0000000000000001ULL +#define I40E_INSET_SMAC 0x0000000000000002ULL +#define I40E_INSET_VLAN_OUTER 0x0000000000000004ULL +#define I40E_INSET_VLAN_INNER 0x0000000000000008ULL +#define I40E_INSET_VLAN_TUNNEL 0x0000000000000010ULL + +/* bit 8 ~ bit 15 */ +#define I40E_INSET_IPV4_SRC 0x0000000000000100ULL +#define I40E_INSET_IPV4_DST 0x0000000000000200ULL +#define I40E_INSET_IPV6_SRC 0x0000000000000400ULL +#define I40E_INSET_IPV6_DST 0x0000000000000800ULL +#define I40E_INSET_SRC_PORT 0x0000000000001000ULL +#define I40E_INSET_DST_PORT 0x0000000000002000ULL +#define I40E_INSET_SCTP_VT 0x0000000000004000ULL + +/* bit 16 ~ bit 31 */ +#define I40E_INSET_IPV4_TOS 0x0000000000010000ULL +#define I40E_INSET_IPV4_PROTO 0x0000000000020000ULL +#define I40E_INSET_IPV4_TTL 0x0000000000040000ULL +#define I40E_INSET_IPV6_TC 0x0000000000080000ULL +#define I40E_INSET_IPV6_FLOW 0x0000000000100000ULL +#define I40E_INSET_IPV6_NEXT_HDR 0x0000000000200000ULL +#define I40E_INSET_IPV6_HOP_LIMIT 0x0000000000400000ULL +#define I40E_INSET_TCP_FLAGS 0x0000000000800000ULL + +/* bit 32 ~ bit 47, tunnel fields */ +#define I40E_INSET_TUNNEL_IPV4_DST 0x0000000100000000ULL +#define I40E_INSET_TUNNEL_IPV6_DST 0x0000000200000000ULL +#define I40E_INSET_TUNNEL_DMAC 0x0000000400000000ULL +#define I40E_INSET_TUNNEL_SRC_PORT 0x0000000800000000ULL +#define I40E_INSET_TUNNEL_DST_PORT 0x0000001000000000ULL +#define I40E_INSET_TUNNEL_ID 0x0000002000000000ULL + +/* bit 48 ~ bit 55 */ +#define I40E_INSET_LAST_ETHER_TYPE 0x0001000000000000ULL + +/* bit 56 ~ bit 63, Flex Payload */ +#define I40E_INSET_FLEX_PAYLOAD_W1 0x0100000000000000ULL +#define I40E_INSET_FLEX_PAYLOAD_W2 0x0200000000000000ULL +#define I40E_INSET_FLEX_PAYLOAD_W3 0x0400000000000000ULL +#define I40E_INSET_FLEX_PAYLOAD_W4 0x0800000000000000ULL +#define I40E_INSET_FLEX_PAYLOAD_W5 0x1000000000000000ULL +#define I40E_INSET_FLEX_PAYLOAD_W6 0x2000000000000000ULL +#define I40E_INSET_FLEX_PAYLOAD_W7 0x4000000000000000ULL +#define I40E_INSET_FLEX_PAYLOAD_W8 0x8000000000000000ULL +#define I40E_INSET_FLEX_PAYLOAD \ + (I40E_INSET_FLEX_PAYLOAD_W1 | I40E_INSET_FLEX_PAYLOAD_W2 | \ + I40E_INSET_FLEX_PAYLOAD_W3 | I40E_INSET_FLEX_PAYLOAD_W4 | \ + I40E_INSET_FLEX_PAYLOAD_W5 | I40E_INSET_FLEX_PAYLOAD_W6 | \ + I40E_INSET_FLEX_PAYLOAD_W7 | I40E_INSET_FLEX_PAYLOAD_W8) + +/* The max bandwidth of i40e is 40Gbps. */ +#define I40E_QOS_BW_MAX 40000 +/* The bandwidth should be the multiple of 50Mbps. */ +#define I40E_QOS_BW_GRANULARITY 50 +/* The min bandwidth weight is 1. */ +#define I40E_QOS_BW_WEIGHT_MIN 1 +/* The max bandwidth weight is 127. */ +#define I40E_QOS_BW_WEIGHT_MAX 127 +/* The max queue region index is 7. */ +#define I40E_REGION_MAX_INDEX 7 + +#define I40E_MAX_PERCENT 100 +#define I40E_DEFAULT_DCB_APP_NUM 1 +#define I40E_DEFAULT_DCB_APP_PRIO 3 + +/** + * The overhead from MTU to max frame size. + * Considering QinQ packet, the VLAN tag needs to be counted twice. + */ +#define I40E_ETH_OVERHEAD \ + (ETHER_HDR_LEN + ETHER_CRC_LEN + I40E_VLAN_TAG_SIZE * 2) + struct i40e_adapter; /** @@ -217,7 +322,7 @@ struct i40e_bw_info { /* Relative credits within same TC with respect to other VSIs or Comps */ uint8_t bw_ets_share_credits[I40E_MAX_TRAFFIC_CLASS]; /* Bandwidth limit per TC */ - uint8_t bw_ets_credits[I40E_MAX_TRAFFIC_CLASS]; + uint16_t bw_ets_credits[I40E_MAX_TRAFFIC_CLASS]; /* Max bandwidth limit per TC */ uint8_t bw_ets_max[I40E_MAX_TRAFFIC_CLASS]; }; @@ -232,6 +337,7 @@ struct i40e_veb { uint16_t stats_idx; struct i40e_eth_stats stats; uint8_t enabled_tc; /* The traffic class enabled */ + uint8_t strict_prio_tc; /* bit map of TCs set to strict priority mode */ struct i40e_bw_info bw_info; /* VEB bandwidth information */ }; @@ -290,6 +396,8 @@ struct i40e_vsi { uint16_t msix_intr; /* The MSIX interrupt binds to VSI */ uint16_t nb_msix; /* The max number of msix vector */ uint8_t enabled_tc; /* The traffic class enabled */ + uint8_t vlan_anti_spoof_on; /* The VLAN anti-spoofing enabled */ + uint8_t vlan_filter_on; /* The VLAN filter enabled */ struct i40e_bw_info bw_info; /* VSI bandwidth information */ }; @@ -322,7 +430,7 @@ enum I40E_VF_STATE { struct i40e_pf_vf { struct i40e_pf *pf; struct i40e_vsi *vsi; - enum I40E_VF_STATE state; /* The number of queue pairs availiable */ + enum I40E_VF_STATE state; /* The number of queue pairs available */ uint16_t vf_idx; /* VF index in pf->vfs */ uint16_t lan_nb_qps; /* Actual queues allocated */ uint16_t reset_cnt; /* Total vf reset times */ @@ -348,6 +456,138 @@ struct i40e_vmdq_info { struct i40e_vsi *vsi; }; +#define I40E_FDIR_MAX_FLEXLEN 16 /**< Max length of flexbytes. */ +#define I40E_MAX_FLX_SOURCE_OFF 480 +#define NONUSE_FLX_PIT_DEST_OFF 63 +#define NONUSE_FLX_PIT_FSIZE 1 +#define I40E_FLX_OFFSET_IN_FIELD_VECTOR 50 +#define MK_FLX_PIT(src_offset, fsize, dst_offset) ( \ + (((src_offset) << I40E_PRTQF_FLX_PIT_SOURCE_OFF_SHIFT) & \ + I40E_PRTQF_FLX_PIT_SOURCE_OFF_MASK) | \ + (((fsize) << I40E_PRTQF_FLX_PIT_FSIZE_SHIFT) & \ + I40E_PRTQF_FLX_PIT_FSIZE_MASK) | \ + ((((dst_offset) == NONUSE_FLX_PIT_DEST_OFF ? \ + NONUSE_FLX_PIT_DEST_OFF : \ + ((dst_offset) + I40E_FLX_OFFSET_IN_FIELD_VECTOR)) << \ + I40E_PRTQF_FLX_PIT_DEST_OFF_SHIFT) & \ + I40E_PRTQF_FLX_PIT_DEST_OFF_MASK)) +#define I40E_WORD(hi, lo) (uint16_t)((((hi) << 8) & 0xFF00) | ((lo) & 0xFF)) +#define I40E_FLEX_WORD_MASK(off) (0x80 >> (off)) +#define I40E_FDIR_IPv6_TC_OFFSET 20 + +/* A structure used to define the input for GTP flow */ +struct i40e_gtp_flow { + struct rte_eth_udpv4_flow udp; /* IPv4 UDP fields to match. */ + uint8_t msg_type; /* Message type. */ + uint32_t teid; /* TEID in big endian. */ +}; + +/* A structure used to define the input for GTP IPV4 flow */ +struct i40e_gtp_ipv4_flow { + struct i40e_gtp_flow gtp; + struct rte_eth_ipv4_flow ip4; +}; + +/* A structure used to define the input for GTP IPV6 flow */ +struct i40e_gtp_ipv6_flow { + struct i40e_gtp_flow gtp; + struct rte_eth_ipv6_flow ip6; +}; + +/* A structure used to define the input for raw type flow */ +struct i40e_raw_flow { + uint16_t pctype; + void *packet; + uint32_t length; +}; + +/* + * A union contains the inputs for all types of flow + * items in flows need to be in big endian + */ +union i40e_fdir_flow { + struct rte_eth_l2_flow l2_flow; + struct rte_eth_udpv4_flow udp4_flow; + struct rte_eth_tcpv4_flow tcp4_flow; + struct rte_eth_sctpv4_flow sctp4_flow; + struct rte_eth_ipv4_flow ip4_flow; + struct rte_eth_udpv6_flow udp6_flow; + struct rte_eth_tcpv6_flow tcp6_flow; + struct rte_eth_sctpv6_flow sctp6_flow; + struct rte_eth_ipv6_flow ipv6_flow; + struct i40e_gtp_flow gtp_flow; + struct i40e_gtp_ipv4_flow gtp_ipv4_flow; + struct i40e_gtp_ipv6_flow gtp_ipv6_flow; + struct i40e_raw_flow raw_flow; +}; + +enum i40e_fdir_ip_type { + I40E_FDIR_IPTYPE_IPV4, + I40E_FDIR_IPTYPE_IPV6, +}; + +/* A structure used to contain extend input of flow */ +struct i40e_fdir_flow_ext { + uint16_t vlan_tci; + uint8_t flexbytes[RTE_ETH_FDIR_MAX_FLEXLEN]; + /* It is filled by the flexible payload to match. */ + uint8_t is_vf; /* 1 for VF, 0 for port dev */ + uint16_t dst_id; /* VF ID, available when is_vf is 1*/ + bool inner_ip; /* If there is inner ip */ + enum i40e_fdir_ip_type iip_type; /* ip type for inner ip */ + bool customized_pctype; /* If customized pctype is used */ + bool pkt_template; /* If raw packet template is used */ +}; + +/* A structure used to define the input for a flow director filter entry */ +struct i40e_fdir_input { + enum i40e_filter_pctype pctype; + union i40e_fdir_flow flow; + /* Flow fields to match, dependent on flow_type */ + struct i40e_fdir_flow_ext flow_ext; + /* Additional fields to match */ +}; + +/* Behavior will be taken if FDIR match */ +enum i40e_fdir_behavior { + I40E_FDIR_ACCEPT = 0, + I40E_FDIR_REJECT, + I40E_FDIR_PASSTHRU, +}; + +/* Flow director report status + * It defines what will be reported if FDIR entry is matched. + */ +enum i40e_fdir_status { + I40E_FDIR_NO_REPORT_STATUS = 0, /* Report nothing. */ + I40E_FDIR_REPORT_ID, /* Only report FD ID. */ + I40E_FDIR_REPORT_ID_FLEX_4, /* Report FD ID and 4 flex bytes. */ + I40E_FDIR_REPORT_FLEX_8, /* Report 8 flex bytes. */ +}; + +/* A structure used to define an action when match FDIR packet filter. */ +struct i40e_fdir_action { + uint16_t rx_queue; /* Queue assigned to if FDIR match. */ + enum i40e_fdir_behavior behavior; /* Behavior will be taken */ + enum i40e_fdir_status report_status; /* Status report option */ + /* If report_status is I40E_FDIR_REPORT_ID_FLEX_4 or + * I40E_FDIR_REPORT_FLEX_8, flex_off specifies where the reported + * flex bytes start from in flexible payload. + */ + uint8_t flex_off; +}; + +/* A structure used to define the flow director filter entry by filter_ctrl API + * It supports RTE_ETH_FILTER_FDIR with RTE_ETH_FILTER_ADD and + * RTE_ETH_FILTER_DELETE operations. + */ +struct i40e_fdir_filter_conf { + uint32_t soft_id; + /* ID, an unique value is required when deal with FDIR entry */ + struct i40e_fdir_input input; /* Input set */ + struct i40e_fdir_action action; /* Action taken when match */ +}; + /* * Structure to store flex pit for flow diretor. */ @@ -359,13 +599,23 @@ struct i40e_fdir_flex_pit { struct i40e_fdir_flex_mask { uint8_t word_mask; /**< Bit i enables word i of flexible payload */ + uint8_t nb_bitmask; struct { uint8_t offset; uint16_t mask; } bitmask[I40E_FDIR_BITMASK_NUM_WORD]; }; -#define I40E_FILTER_PCTYPE_MAX 64 +#define I40E_FILTER_PCTYPE_INVALID 0 +#define I40E_FILTER_PCTYPE_MAX 64 +#define I40E_MAX_FDIR_FILTER_NUM (1024 * 8) + +struct i40e_fdir_filter { + TAILQ_ENTRY(i40e_fdir_filter) rules; + struct i40e_fdir_filter_conf fdir; +}; + +TAILQ_HEAD(i40e_fdir_filter_list, i40e_fdir_filter); /* * A structure used to define fields of a FDIR related info. */ @@ -384,6 +634,162 @@ struct i40e_fdir_info { */ struct i40e_fdir_flex_pit flex_set[I40E_MAX_FLXPLD_LAYER * I40E_MAX_FLXPLD_FIED]; struct i40e_fdir_flex_mask flex_mask[I40E_FILTER_PCTYPE_MAX]; + + struct i40e_fdir_filter_list fdir_list; + struct i40e_fdir_filter **hash_map; + struct rte_hash *hash_table; + + /* Mark if flex pit and mask is set */ + bool flex_pit_flag[I40E_MAX_FLXPLD_LAYER]; + bool flex_mask_flag[I40E_FILTER_PCTYPE_MAX]; + + bool inset_flag[I40E_FILTER_PCTYPE_MAX]; /* Mark if input set is set */ +}; + +/* Ethertype filter number HW supports */ +#define I40E_MAX_ETHERTYPE_FILTER_NUM 768 + +/* Ethertype filter struct */ +struct i40e_ethertype_filter_input { + struct ether_addr mac_addr; /* Mac address to match */ + uint16_t ether_type; /* Ether type to match */ +}; + +struct i40e_ethertype_filter { + TAILQ_ENTRY(i40e_ethertype_filter) rules; + struct i40e_ethertype_filter_input input; + uint16_t flags; /* Flags from RTE_ETHTYPE_FLAGS_* */ + uint16_t queue; /* Queue assigned to when match */ +}; + +TAILQ_HEAD(i40e_ethertype_filter_list, i40e_ethertype_filter); + +struct i40e_ethertype_rule { + struct i40e_ethertype_filter_list ethertype_list; + struct i40e_ethertype_filter **hash_map; + struct rte_hash *hash_table; +}; + +/* queue region info */ +struct i40e_queue_region_info { + /* the region id for this configuration */ + uint8_t region_id; + /* the start queue index for this region */ + uint8_t queue_start_index; + /* the total queue number of this queue region */ + uint8_t queue_num; + /* the total number of user priority for this region */ + uint8_t user_priority_num; + /* the packet's user priority for this region */ + uint8_t user_priority[I40E_MAX_USER_PRIORITY]; + /* the total number of flowtype for this region */ + uint8_t flowtype_num; + /** + * the pctype or hardware flowtype of packet, + * the specific index for each type has been defined + * in file i40e_type.h as enum i40e_filter_pctype. + */ + uint8_t hw_flowtype[I40E_FILTER_PCTYPE_MAX]; +}; + +struct i40e_queue_regions { + /* the total number of queue region for this port */ + uint16_t queue_region_number; + struct i40e_queue_region_info region[I40E_REGION_MAX_INDEX + 1]; +}; + +/* Tunnel filter number HW supports */ +#define I40E_MAX_TUNNEL_FILTER_NUM 400 + +#define I40E_AQC_REPLACE_CLOUD_CMD_INPUT_FV_TEID_WORD0 44 +#define I40E_AQC_REPLACE_CLOUD_CMD_INPUT_FV_TEID_WORD1 45 +#define I40E_AQC_ADD_CLOUD_TNL_TYPE_MPLSOUDP 8 +#define I40E_AQC_ADD_CLOUD_TNL_TYPE_MPLSOGRE 9 +#define I40E_AQC_ADD_CLOUD_FILTER_0X10 0x10 +#define I40E_AQC_ADD_CLOUD_FILTER_0X11 0x11 +#define I40E_AQC_ADD_CLOUD_FILTER_0X12 0x12 +#define I40E_AQC_ADD_L1_FILTER_0X11 0x11 +#define I40E_AQC_ADD_L1_FILTER_0X12 0x12 +#define I40E_AQC_ADD_L1_FILTER_0X13 0x13 +#define I40E_AQC_NEW_TR_21 21 +#define I40E_AQC_NEW_TR_22 22 + +enum i40e_tunnel_iptype { + I40E_TUNNEL_IPTYPE_IPV4, + I40E_TUNNEL_IPTYPE_IPV6, +}; + +/* Tunnel filter struct */ +struct i40e_tunnel_filter_input { + uint8_t outer_mac[6]; /* Outer mac address to match */ + uint8_t inner_mac[6]; /* Inner mac address to match */ + uint16_t inner_vlan; /* Inner vlan address to match */ + enum i40e_tunnel_iptype ip_type; + uint16_t flags; /* Filter type flag */ + uint32_t tenant_id; /* Tenant id to match */ + uint16_t general_fields[32]; /* Big buffer */ +}; + +struct i40e_tunnel_filter { + TAILQ_ENTRY(i40e_tunnel_filter) rules; + struct i40e_tunnel_filter_input input; + uint8_t is_to_vf; /* 0 - to PF, 1 - to VF */ + uint16_t vf_id; /* VF id, avaiblable when is_to_vf is 1. */ + uint16_t queue; /* Queue assigned to when match */ +}; + +TAILQ_HEAD(i40e_tunnel_filter_list, i40e_tunnel_filter); + +struct i40e_tunnel_rule { + struct i40e_tunnel_filter_list tunnel_list; + struct i40e_tunnel_filter **hash_map; + struct rte_hash *hash_table; +}; + +/** + * Tunnel type. + */ +enum i40e_tunnel_type { + I40E_TUNNEL_TYPE_NONE = 0, + I40E_TUNNEL_TYPE_VXLAN, + I40E_TUNNEL_TYPE_GENEVE, + I40E_TUNNEL_TYPE_TEREDO, + I40E_TUNNEL_TYPE_NVGRE, + I40E_TUNNEL_TYPE_IP_IN_GRE, + I40E_L2_TUNNEL_TYPE_E_TAG, + I40E_TUNNEL_TYPE_MPLSoUDP, + I40E_TUNNEL_TYPE_MPLSoGRE, + I40E_TUNNEL_TYPE_QINQ, + I40E_TUNNEL_TYPE_GTPC, + I40E_TUNNEL_TYPE_GTPU, + I40E_TUNNEL_TYPE_MAX, +}; + +/** + * Tunneling Packet filter configuration. + */ +struct i40e_tunnel_filter_conf { + struct ether_addr outer_mac; /**< Outer MAC address to match. */ + struct ether_addr inner_mac; /**< Inner MAC address to match. */ + uint16_t inner_vlan; /**< Inner VLAN to match. */ + uint32_t outer_vlan; /**< Outer VLAN to match */ + enum i40e_tunnel_iptype ip_type; /**< IP address type. */ + /** + * Outer destination IP address to match if ETH_TUNNEL_FILTER_OIP + * is set in filter_type, or inner destination IP address to match + * if ETH_TUNNEL_FILTER_IIP is set in filter_type. + */ + union { + uint32_t ipv4_addr; /**< IPv4 address in big endian. */ + uint32_t ipv6_addr[4]; /**< IPv6 address in big endian. */ + } ip_addr; + /** Flags from ETH_TUNNEL_FILTER_XX - see above. */ + uint16_t filter_type; + enum i40e_tunnel_type tunnel_type; /**< Tunnel Type. */ + uint32_t tenant_id; /**< Tenant ID to match. VNI, GRE key... */ + uint16_t queue_id; /**< Queue assigned to if match. */ + uint8_t is_to_vf; /**< 0 - to PF, 1 - to VF */ + uint16_t vf_id; /**< VF id, avaiblable when is_to_vf is 1. */ }; #define I40E_MIRROR_MAX_ENTRIES_PER_RULE 64 @@ -407,6 +813,93 @@ struct i40e_mirror_rule { TAILQ_HEAD(i40e_mirror_rule_list, i40e_mirror_rule); +/* + * Struct to store flow created. + */ +struct rte_flow { + TAILQ_ENTRY(rte_flow) node; + enum rte_filter_type filter_type; + void *rule; +}; + +TAILQ_HEAD(i40e_flow_list, rte_flow); + +/* Struct to store Traffic Manager shaper profile. */ +struct i40e_tm_shaper_profile { + TAILQ_ENTRY(i40e_tm_shaper_profile) node; + uint32_t shaper_profile_id; + uint32_t reference_count; + struct rte_tm_shaper_params profile; +}; + +TAILQ_HEAD(i40e_shaper_profile_list, i40e_tm_shaper_profile); + +/* node type of Traffic Manager */ +enum i40e_tm_node_type { + I40E_TM_NODE_TYPE_PORT, + I40E_TM_NODE_TYPE_TC, + I40E_TM_NODE_TYPE_QUEUE, + I40E_TM_NODE_TYPE_MAX, +}; + +/* Struct to store Traffic Manager node configuration. */ +struct i40e_tm_node { + TAILQ_ENTRY(i40e_tm_node) node; + uint32_t id; + uint32_t priority; + uint32_t weight; + uint32_t reference_count; + struct i40e_tm_node *parent; + struct i40e_tm_shaper_profile *shaper_profile; + struct rte_tm_node_params params; +}; + +TAILQ_HEAD(i40e_tm_node_list, i40e_tm_node); + +/* Struct to store all the Traffic Manager configuration. */ +struct i40e_tm_conf { + struct i40e_shaper_profile_list shaper_profile_list; + struct i40e_tm_node *root; /* root node - port */ + struct i40e_tm_node_list tc_list; /* node list for all the TCs */ + struct i40e_tm_node_list queue_list; /* node list for all the queues */ + /** + * The number of added TC nodes. + * It should be no more than the TC number of this port. + */ + uint32_t nb_tc_node; + /** + * The number of added queue nodes. + * It should be no more than the queue number of this port. + */ + uint32_t nb_queue_node; + /** + * This flag is used to check if APP can change the TM node + * configuration. + * When it's true, means the configuration is applied to HW, + * APP should not change the configuration. + * As we don't support on-the-fly configuration, when starting + * the port, APP should call the hierarchy_commit API to set this + * flag to true. When stopping the port, this flag should be set + * to false. + */ + bool committed; +}; + +enum i40e_new_pctype { + I40E_CUSTOMIZED_GTPC = 0, + I40E_CUSTOMIZED_GTPU_IPV4, + I40E_CUSTOMIZED_GTPU_IPV6, + I40E_CUSTOMIZED_GTPU, + I40E_CUSTOMIZED_MAX, +}; + +#define I40E_FILTER_PCTYPE_INVALID 0 +struct i40e_customized_pctype { + enum i40e_new_pctype index; /* Indicate which customized pctype */ + uint8_t pctype; /* New pctype value */ + bool valid; /* Check if it's valid */ +}; + /* * Structure to store private data specific for PF instance. */ @@ -421,6 +914,9 @@ struct i40e_pf { struct i40e_hw_port_stats stats_offset; struct i40e_hw_port_stats stats; + /* internal packet statistics, it should be excluded from the total */ + struct i40e_eth_stats internal_stats_offset; + struct i40e_eth_stats internal_stats; bool offset_loaded; struct rte_eth_dev_data *dev_data; /* Pointer to the device data */ @@ -456,12 +952,26 @@ struct i40e_pf { struct i40e_vmdq_info *vmdq; struct i40e_fdir_info fdir; /* flow director info */ + struct i40e_ethertype_rule ethertype; /* Ethertype filter rule */ + struct i40e_tunnel_rule tunnel; /* Tunnel filter rule */ + struct i40e_queue_regions queue_region; /* queue region info */ struct i40e_fc_conf fc_conf; /* Flow control conf */ struct i40e_mirror_rule_list mirror_list; uint16_t nb_mirror_rule; /* The number of mirror rules */ bool floating_veb; /* The flag to use the floating VEB */ /* The floating enable flag for the specific VF */ bool floating_veb_list[I40E_MAX_VF]; + struct i40e_flow_list flow_list; + bool mpls_replace_flag; /* 1 - MPLS filter replace is done */ + bool gtp_replace_flag; /* 1 - GTP-C/U filter replace is done */ + bool qinq_replace_flag; /* QINQ filter replace is done */ + struct i40e_tm_conf tm_conf; + bool support_multi_driver; /* 1 - support multiple driver */ + + /* Dynamic Device Personalization */ + bool gtp_support; /* 1 - support GTP-C and GTP-U */ + /* customer customized pctype */ + struct i40e_customized_pctype customized_pctype[I40E_CUSTOMIZED_MAX]; }; enum pending_msg { @@ -514,20 +1024,23 @@ struct i40e_vf { /* Event from pf */ bool dev_closed; bool link_up; - enum i40e_aq_link_speed link_speed; + enum virtchnl_link_speed link_speed; bool vf_reset; volatile uint32_t pend_cmd; /* pending command not finished yet */ - uint32_t cmd_retval; /* return value of the cmd response from PF */ + int32_t cmd_retval; /* return value of the cmd response from PF */ u16 pend_msg; /* flags indicates events from pf not handled yet */ uint8_t *aq_resp; /* buffer to store the adminq response from PF */ /* VSI info */ - struct i40e_virtchnl_vf_resource *vf_res; /* All VSIs */ - struct i40e_virtchnl_vsi_resource *vsi_res; /* LAN VSI */ + struct virtchnl_vf_resource *vf_res; /* All VSIs */ + struct virtchnl_vsi_resource *vsi_res; /* LAN VSI */ struct i40e_vsi vsi; uint64_t flags; }; +#define I40E_MAX_PKT_TYPE 256 +#define I40E_FLOW_TYPE_MAX 64 + /* * Structure to store private data for each PF/VF instance. */ @@ -552,6 +1065,49 @@ struct i40e_adapter { struct rte_timecounter systime_tc; struct rte_timecounter rx_tstamp_tc; struct rte_timecounter tx_tstamp_tc; + + /* ptype mapping table */ + uint32_t ptype_tbl[I40E_MAX_PKT_TYPE] __rte_cache_min_aligned; + /* flow type to pctype mapping table */ + uint64_t pctypes_tbl[I40E_FLOW_TYPE_MAX] __rte_cache_min_aligned; + uint64_t flow_types_mask; + uint64_t pctypes_mask; +}; + +extern const struct rte_flow_ops i40e_flow_ops; + +union i40e_filter_t { + struct rte_eth_ethertype_filter ethertype_filter; + struct i40e_fdir_filter_conf fdir_filter; + struct rte_eth_tunnel_filter_conf tunnel_filter; + struct i40e_tunnel_filter_conf consistent_tunnel_filter; +}; + +typedef int (*parse_filter_t)(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error, + union i40e_filter_t *filter); +struct i40e_valid_pattern { + enum rte_flow_item_type *items; + parse_filter_t parse_filter; +}; + +enum I40E_WARNING_IDX { + I40E_WARNING_DIS_FLX_PLD, + I40E_WARNING_ENA_FLX_PLD, + I40E_WARNING_QINQ_PARSER, + I40E_WARNING_QINQ_CLOUD_FILTER, + I40E_WARNING_TPID, + I40E_WARNING_FLOW_CTL, + I40E_WARNING_GRE_KEY_LEN, + I40E_WARNING_QF_CTL, + I40E_WARNING_HASH_INSET, + I40E_WARNING_HSYM, + I40E_WARNING_HASH_MSK, + I40E_WARNING_FD_MSK, + I40E_WARNING_RPL_CLD_FILTER, }; int i40e_dev_switch_queues(struct i40e_pf *pf, bool on); @@ -569,16 +1125,15 @@ int i40e_vsi_delete_mac(struct i40e_vsi *vsi, struct ether_addr *addr); void i40e_update_vsi_stats(struct i40e_vsi *vsi); void i40e_pf_disable_irq0(struct i40e_hw *hw); void i40e_pf_enable_irq0(struct i40e_hw *hw); -int i40e_dev_link_update(struct rte_eth_dev *dev, - __rte_unused int wait_to_complete); -void i40e_vsi_queues_bind_intr(struct i40e_vsi *vsi); +int i40e_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete); +void i40e_vsi_queues_bind_intr(struct i40e_vsi *vsi, uint16_t itr_idx); void i40e_vsi_queues_unbind_intr(struct i40e_vsi *vsi); int i40e_vsi_vlan_pvid_set(struct i40e_vsi *vsi, struct i40e_vsi_vlan_pvid_info *info); int i40e_vsi_config_vlan_stripping(struct i40e_vsi *vsi, bool on); int i40e_vsi_config_vlan_filter(struct i40e_vsi *vsi, bool on); -uint64_t i40e_config_hena(uint64_t flags); -uint64_t i40e_parse_hena(uint64_t flags); +uint64_t i40e_config_hena(const struct i40e_adapter *adapter, uint64_t flags); +uint64_t i40e_parse_hena(const struct i40e_adapter *adapter, uint64_t flags); enum i40e_status_code i40e_fdir_setup_tx_resources(struct i40e_pf *pf); enum i40e_status_code i40e_fdir_setup_rx_resources(struct i40e_pf *pf); int i40e_fdir_setup(struct i40e_pf *pf); @@ -587,14 +1142,18 @@ const struct rte_memzone *i40e_memzone_reserve(const char *name, int socket_id); int i40e_fdir_configure(struct rte_eth_dev *dev); void i40e_fdir_teardown(struct i40e_pf *pf); -enum i40e_filter_pctype i40e_flowtype_to_pctype(uint16_t flow_type); -uint16_t i40e_pctype_to_flowtype(enum i40e_filter_pctype pctype); +enum i40e_filter_pctype + i40e_flowtype_to_pctype(const struct i40e_adapter *adapter, + uint16_t flow_type); +uint16_t i40e_pctype_to_flowtype(const struct i40e_adapter *adapter, + enum i40e_filter_pctype pctype); int i40e_fdir_ctrl_func(struct rte_eth_dev *dev, enum rte_filter_op filter_op, void *arg); int i40e_select_filter_input_set(struct i40e_hw *hw, struct rte_eth_input_set_conf *conf, enum rte_filter_type filter); +void i40e_fdir_filter_restore(struct i40e_pf *pf); int i40e_hash_filter_inset_select(struct i40e_hw *hw, struct rte_eth_input_set_conf *conf); int i40e_fdir_filter_inset_select(struct i40e_pf *pf, @@ -606,6 +1165,70 @@ void i40e_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, struct rte_eth_rxq_info *qinfo); void i40e_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, struct rte_eth_txq_info *qinfo); +struct i40e_ethertype_filter * +i40e_sw_ethertype_filter_lookup(struct i40e_ethertype_rule *ethertype_rule, + const struct i40e_ethertype_filter_input *input); +int i40e_sw_ethertype_filter_del(struct i40e_pf *pf, + struct i40e_ethertype_filter_input *input); +int i40e_sw_fdir_filter_del(struct i40e_pf *pf, + struct i40e_fdir_input *input); +struct i40e_tunnel_filter * +i40e_sw_tunnel_filter_lookup(struct i40e_tunnel_rule *tunnel_rule, + const struct i40e_tunnel_filter_input *input); +int i40e_sw_tunnel_filter_del(struct i40e_pf *pf, + struct i40e_tunnel_filter_input *input); +uint64_t i40e_get_default_input_set(uint16_t pctype); +int i40e_ethertype_filter_set(struct i40e_pf *pf, + struct rte_eth_ethertype_filter *filter, + bool add); +int i40e_add_del_fdir_filter(struct rte_eth_dev *dev, + const struct rte_eth_fdir_filter *filter, + bool add); +int i40e_flow_add_del_fdir_filter(struct rte_eth_dev *dev, + const struct i40e_fdir_filter_conf *filter, + bool add); +int i40e_dev_tunnel_filter_set(struct i40e_pf *pf, + struct rte_eth_tunnel_filter_conf *tunnel_filter, + uint8_t add); +int i40e_dev_consistent_tunnel_filter_set(struct i40e_pf *pf, + struct i40e_tunnel_filter_conf *tunnel_filter, + uint8_t add); +int i40e_fdir_flush(struct rte_eth_dev *dev); +int i40e_find_all_vlan_for_mac(struct i40e_vsi *vsi, + struct i40e_macvlan_filter *mv_f, + int num, struct ether_addr *addr); +int i40e_remove_macvlan_filters(struct i40e_vsi *vsi, + struct i40e_macvlan_filter *filter, + int total); +void i40e_set_vlan_filter(struct i40e_vsi *vsi, uint16_t vlan_id, bool on); +int i40e_add_macvlan_filters(struct i40e_vsi *vsi, + struct i40e_macvlan_filter *filter, + int total); +bool is_i40e_supported(struct rte_eth_dev *dev); + +int i40e_validate_input_set(enum i40e_filter_pctype pctype, + enum rte_filter_type filter, uint64_t inset); +int i40e_generate_inset_mask_reg(uint64_t inset, uint32_t *mask, + uint8_t nb_elem); +uint64_t i40e_translate_input_set_reg(enum i40e_mac_type type, uint64_t input); +void i40e_check_write_reg(struct i40e_hw *hw, uint32_t addr, uint32_t val); +void i40e_check_write_global_reg(struct i40e_hw *hw, + uint32_t addr, uint32_t val); + +int i40e_tm_ops_get(struct rte_eth_dev *dev, void *ops); +void i40e_tm_conf_init(struct rte_eth_dev *dev); +void i40e_tm_conf_uninit(struct rte_eth_dev *dev); +struct i40e_customized_pctype* +i40e_find_customized_pctype(struct i40e_pf *pf, uint8_t index); +void i40e_update_customized_info(struct rte_eth_dev *dev, uint8_t *pkg, + uint32_t pkg_size); +int i40e_dcb_init_configure(struct rte_eth_dev *dev, bool sw_dcb); +int i40e_flush_queue_region_all_conf(struct rte_eth_dev *dev, + struct i40e_hw *hw, struct i40e_pf *pf, uint16_t on); +void i40e_init_queue_region_conf(struct rte_eth_dev *dev); + +#define I40E_DEV_TO_PCI(eth_dev) \ + RTE_DEV_TO_PCI((eth_dev)->device) /* I40E_DEV_PRIVATE_TO */ #define I40E_DEV_PRIVATE_TO_PF(adapter) \ @@ -679,15 +1302,44 @@ i40e_align_floor(int n) } static inline uint16_t -i40e_calc_itr_interval(int16_t interval) +i40e_calc_itr_interval(int16_t interval, bool is_multi_drv) { - if (interval < 0 || interval > I40E_QUEUE_ITR_INTERVAL_MAX) + if (is_multi_drv) + interval = I40E_QUEUE_ITR_INTERVAL_MAX; + else if (interval < 0 || interval > I40E_QUEUE_ITR_INTERVAL_MAX) interval = I40E_QUEUE_ITR_INTERVAL_DEFAULT; /* Convert to hardware count, as writing each 1 represents 2 us */ return interval / 2; } +static inline void +i40e_global_cfg_warning(enum I40E_WARNING_IDX idx) +{ + const char *warning; + static const char *const warning_list[] = { + [I40E_WARNING_DIS_FLX_PLD] = "disable FDIR flexible payload", + [I40E_WARNING_ENA_FLX_PLD] = "enable FDIR flexible payload", + [I40E_WARNING_QINQ_PARSER] = "support QinQ parser", + [I40E_WARNING_QINQ_CLOUD_FILTER] = "support QinQ cloud filter", + [I40E_WARNING_TPID] = "support TPID configuration", + [I40E_WARNING_FLOW_CTL] = "configure water marker", + [I40E_WARNING_GRE_KEY_LEN] = "support GRE key length setting", + [I40E_WARNING_QF_CTL] = "support hash function setting", + [I40E_WARNING_HASH_INSET] = "configure hash input set", + [I40E_WARNING_HSYM] = "set symmetric hash", + [I40E_WARNING_HASH_MSK] = "configure hash mask", + [I40E_WARNING_FD_MSK] = "configure fdir mask", + [I40E_WARNING_RPL_CLD_FILTER] = "replace cloud filter", + }; + + warning = warning_list[idx]; + + RTE_LOG(WARNING, PMD, + "Global register is changed during %s\n", + warning); +} + #define I40E_VALID_FLOW(flow_type) \ ((flow_type) == RTE_ETH_FLOW_FRAG_IPV4 || \ (flow_type) == RTE_ETH_FLOW_NONFRAG_IPV4_TCP || \ @@ -701,6 +1353,25 @@ i40e_calc_itr_interval(int16_t interval) (flow_type) == RTE_ETH_FLOW_NONFRAG_IPV6_OTHER || \ (flow_type) == RTE_ETH_FLOW_L2_PAYLOAD) +#define I40E_VALID_PCTYPE_X722(pctype) \ + ((pctype) == I40E_FILTER_PCTYPE_FRAG_IPV4 || \ + (pctype) == I40E_FILTER_PCTYPE_NONF_IPV4_TCP || \ + (pctype) == I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK || \ + (pctype) == I40E_FILTER_PCTYPE_NONF_IPV4_UDP || \ + (pctype) == I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP || \ + (pctype) == I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP || \ + (pctype) == I40E_FILTER_PCTYPE_NONF_IPV4_SCTP || \ + (pctype) == I40E_FILTER_PCTYPE_NONF_IPV4_OTHER || \ + (pctype) == I40E_FILTER_PCTYPE_FRAG_IPV6 || \ + (pctype) == I40E_FILTER_PCTYPE_NONF_IPV6_UDP || \ + (pctype) == I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP || \ + (pctype) == I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP || \ + (pctype) == I40E_FILTER_PCTYPE_NONF_IPV6_TCP || \ + (pctype) == I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK || \ + (pctype) == I40E_FILTER_PCTYPE_NONF_IPV6_SCTP || \ + (pctype) == I40E_FILTER_PCTYPE_NONF_IPV6_OTHER || \ + (pctype) == I40E_FILTER_PCTYPE_L2_PAYLOAD) + #define I40E_VALID_PCTYPE(pctype) \ ((pctype) == I40E_FILTER_PCTYPE_FRAG_IPV4 || \ (pctype) == I40E_FILTER_PCTYPE_NONF_IPV4_TCP || \ @@ -714,4 +1385,18 @@ i40e_calc_itr_interval(int16_t interval) (pctype) == I40E_FILTER_PCTYPE_NONF_IPV6_OTHER || \ (pctype) == I40E_FILTER_PCTYPE_L2_PAYLOAD) +#define I40E_PHY_TYPE_SUPPORT_40G(phy_type) \ + (((phy_type) & I40E_CAP_PHY_TYPE_40GBASE_KR4) || \ + ((phy_type) & I40E_CAP_PHY_TYPE_40GBASE_CR4_CU) || \ + ((phy_type) & I40E_CAP_PHY_TYPE_40GBASE_AOC) || \ + ((phy_type) & I40E_CAP_PHY_TYPE_40GBASE_CR4) || \ + ((phy_type) & I40E_CAP_PHY_TYPE_40GBASE_SR4) || \ + ((phy_type) & I40E_CAP_PHY_TYPE_40GBASE_LR4)) + +#define I40E_PHY_TYPE_SUPPORT_25G(phy_type) \ + (((phy_type) & I40E_CAP_PHY_TYPE_25GBASE_KR) || \ + ((phy_type) & I40E_CAP_PHY_TYPE_25GBASE_CR) || \ + ((phy_type) & I40E_CAP_PHY_TYPE_25GBASE_SR) || \ + ((phy_type) & I40E_CAP_PHY_TYPE_25GBASE_LR)) + #endif /* _I40E_ETHDEV_H_ */ diff --git a/dpdk/drivers/net/i40e/i40e_ethdev_vf.c b/dpdk/drivers/net/i40e/i40e_ethdev_vf.c index ba63a7f1..b36ba9f8 100644 --- a/dpdk/drivers/net/i40e/i40e_ethdev_vf.c +++ b/dpdk/drivers/net/i40e/i40e_ethdev_vf.c @@ -47,15 +47,15 @@ #include #include #include +#include #include #include #include -#include #include #include #include #include -#include +#include #include #include @@ -67,8 +67,6 @@ #include "i40e_rxtx.h" #include "i40e_ethdev.h" #include "i40e_pf.h" -#define I40EVF_VSI_DEFAULT_MSIX_INTR 1 -#define I40EVF_VSI_DEFAULT_MSIX_INTR_LNX 0 /* busy wait delay in msec */ #define I40EVF_BUSY_WAIT_DELAY 10 @@ -76,7 +74,7 @@ #define MAX_RESET_WAIT_CNT 20 struct i40evf_arq_msg_info { - enum i40e_virtchnl_ops ops; + enum virtchnl_ops ops; enum i40e_status_code result; uint16_t buf_len; uint16_t msg_len; @@ -84,7 +82,7 @@ struct i40evf_arq_msg_info { }; struct vf_cmd_info { - enum i40e_virtchnl_ops ops; + enum virtchnl_ops ops; uint8_t *in_args; uint32_t in_args_size; uint8_t *out_buffer; @@ -107,8 +105,8 @@ static void i40evf_dev_stop(struct rte_eth_dev *dev); static void i40evf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info); static int i40evf_dev_link_update(struct rte_eth_dev *dev, - __rte_unused int wait_to_complete); -static void i40evf_dev_stats_get(struct rte_eth_dev *dev, + int wait_to_complete); +static int i40evf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats); static int i40evf_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, unsigned n); @@ -118,10 +116,9 @@ static int i40evf_dev_xstats_get_names(struct rte_eth_dev *dev, static void i40evf_dev_xstats_reset(struct rte_eth_dev *dev); static int i40evf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on); -static void i40evf_vlan_offload_set(struct rte_eth_dev *dev, int mask); -static int i40evf_vlan_pvid_set(struct rte_eth_dev *dev, uint16_t pvid, - int on); +static int i40evf_vlan_offload_set(struct rte_eth_dev *dev, int mask); static void i40evf_dev_close(struct rte_eth_dev *dev); +static int i40evf_dev_reset(struct rte_eth_dev *dev); static void i40evf_dev_promiscuous_enable(struct rte_eth_dev *dev); static void i40evf_dev_promiscuous_disable(struct rte_eth_dev *dev); static void i40evf_dev_allmulticast_enable(struct rte_eth_dev *dev); @@ -135,10 +132,10 @@ static int i40evf_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id); static int i40evf_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id); -static void i40evf_add_mac_addr(struct rte_eth_dev *dev, - struct ether_addr *addr, - uint32_t index, - uint32_t pool); +static int i40evf_add_mac_addr(struct rte_eth_dev *dev, + struct ether_addr *addr, + uint32_t index, + uint32_t pool); static void i40evf_del_mac_addr(struct rte_eth_dev *dev, uint32_t index); static int i40evf_dev_rss_reta_update(struct rte_eth_dev *dev, struct rte_eth_rss_reta_entry64 *reta_conf, @@ -151,11 +148,14 @@ static int i40evf_dev_rss_hash_update(struct rte_eth_dev *dev, struct rte_eth_rss_conf *rss_conf); static int i40evf_dev_rss_hash_conf_get(struct rte_eth_dev *dev, struct rte_eth_rss_conf *rss_conf); +static int i40evf_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu); +static void i40evf_set_default_mac_addr(struct rte_eth_dev *dev, + struct ether_addr *mac_addr); static int i40evf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id); static int i40evf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id); -static void i40evf_handle_pf_event(__rte_unused struct rte_eth_dev *dev, +static void i40evf_handle_pf_event(struct rte_eth_dev *dev, uint8_t *msg, uint16_t msglen); @@ -176,11 +176,11 @@ static const struct rte_i40evf_xstats_name_off rte_i40evf_stats_strings[] = { {"rx_unknown_protocol_packets", offsetof(struct i40e_eth_stats, rx_unknown_protocol)}, {"tx_bytes", offsetof(struct i40e_eth_stats, tx_bytes)}, - {"tx_unicast_packets", offsetof(struct i40e_eth_stats, tx_bytes)}, - {"tx_multicast_packets", offsetof(struct i40e_eth_stats, tx_bytes)}, - {"tx_broadcast_packets", offsetof(struct i40e_eth_stats, tx_bytes)}, - {"tx_dropped_packets", offsetof(struct i40e_eth_stats, tx_bytes)}, - {"tx_error_packets", offsetof(struct i40e_eth_stats, tx_bytes)}, + {"tx_unicast_packets", offsetof(struct i40e_eth_stats, tx_unicast)}, + {"tx_multicast_packets", offsetof(struct i40e_eth_stats, tx_multicast)}, + {"tx_broadcast_packets", offsetof(struct i40e_eth_stats, tx_broadcast)}, + {"tx_dropped_packets", offsetof(struct i40e_eth_stats, tx_discards)}, + {"tx_error_packets", offsetof(struct i40e_eth_stats, tx_errors)}, }; #define I40EVF_NB_XSTATS (sizeof(rte_i40evf_stats_strings) / \ @@ -196,15 +196,16 @@ static const struct eth_dev_ops i40evf_eth_dev_ops = { .allmulticast_disable = i40evf_dev_allmulticast_disable, .link_update = i40evf_dev_link_update, .stats_get = i40evf_dev_stats_get, + .stats_reset = i40evf_dev_xstats_reset, .xstats_get = i40evf_dev_xstats_get, .xstats_get_names = i40evf_dev_xstats_get_names, .xstats_reset = i40evf_dev_xstats_reset, .dev_close = i40evf_dev_close, + .dev_reset = i40evf_dev_reset, .dev_infos_get = i40evf_dev_info_get, .dev_supported_ptypes_get = i40e_dev_supported_ptypes_get, .vlan_filter_set = i40evf_vlan_filter_set, .vlan_offload_set = i40evf_vlan_offload_set, - .vlan_pvid_set = i40evf_vlan_pvid_set, .rx_queue_start = i40evf_dev_rx_queue_start, .rx_queue_stop = i40evf_dev_rx_queue_stop, .tx_queue_start = i40evf_dev_tx_queue_start, @@ -214,6 +215,8 @@ static const struct eth_dev_ops i40evf_eth_dev_ops = { .rx_queue_intr_enable = i40evf_dev_rx_queue_intr_enable, .rx_queue_intr_disable = i40evf_dev_rx_queue_intr_disable, .rx_descriptor_done = i40e_dev_rx_descriptor_done, + .rx_descriptor_status = i40e_dev_rx_descriptor_status, + .tx_descriptor_status = i40e_dev_tx_descriptor_status, .tx_queue_setup = i40e_dev_tx_queue_setup, .tx_queue_release = i40e_dev_tx_queue_release, .rx_queue_count = i40e_dev_rx_queue_count, @@ -225,6 +228,8 @@ static const struct eth_dev_ops i40evf_eth_dev_ops = { .reta_query = i40evf_dev_rss_reta_query, .rss_hash_update = i40evf_dev_rss_hash_update, .rss_hash_conf_get = i40evf_dev_rss_hash_conf_get, + .mtu_set = i40evf_dev_mtu_set, + .mac_addr_set = i40evf_set_default_mac_addr, }; /* @@ -236,7 +241,7 @@ i40evf_read_pfmsg(struct rte_eth_dev *dev, struct i40evf_arq_msg_info *data) struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); struct i40e_arq_event_info event; - enum i40e_virtchnl_ops opcode; + enum virtchnl_ops opcode; enum i40e_status_code retval; int ret; enum i40evf_aq_result result = I40EVF_MSG_NON; @@ -251,16 +256,16 @@ i40evf_read_pfmsg(struct rte_eth_dev *dev, struct i40evf_arq_msg_info *data) return result; } - opcode = (enum i40e_virtchnl_ops)rte_le_to_cpu_32(event.desc.cookie_high); + opcode = (enum virtchnl_ops)rte_le_to_cpu_32(event.desc.cookie_high); retval = (enum i40e_status_code)rte_le_to_cpu_32(event.desc.cookie_low); /* pf sys event */ - if (opcode == I40E_VIRTCHNL_OP_EVENT) { - struct i40e_virtchnl_pf_event *vpe = - (struct i40e_virtchnl_pf_event *)event.msg_buf; + if (opcode == VIRTCHNL_OP_EVENT) { + struct virtchnl_pf_event *vpe = + (struct virtchnl_pf_event *)event.msg_buf; result = I40EVF_MSG_SYS; switch (vpe->event) { - case I40E_VIRTCHNL_EVENT_LINK_CHANGE: + case VIRTCHNL_EVENT_LINK_CHANGE: vf->link_up = vpe->event_data.link_event.link_status; vf->link_speed = @@ -269,12 +274,12 @@ i40evf_read_pfmsg(struct rte_eth_dev *dev, struct i40evf_arq_msg_info *data) PMD_DRV_LOG(INFO, "Link status update:%s", vf->link_up ? "up" : "down"); break; - case I40E_VIRTCHNL_EVENT_RESET_IMPENDING: + case VIRTCHNL_EVENT_RESET_IMPENDING: vf->vf_reset = true; vf->pend_msg |= PFMSG_RESET_IMPENDING; PMD_DRV_LOG(INFO, "vf is reseting"); break; - case I40E_VIRTCHNL_EVENT_PF_DRIVER_CLOSE: + case VIRTCHNL_EVENT_PF_DRIVER_CLOSE: vf->dev_closed = true; vf->pend_msg |= PFMSG_DRIVER_CLOSE; PMD_DRV_LOG(INFO, "PF driver closed"); @@ -304,17 +309,17 @@ static inline void _clear_cmd(struct i40e_vf *vf) { rte_wmb(); - vf->pend_cmd = I40E_VIRTCHNL_OP_UNKNOWN; + vf->pend_cmd = VIRTCHNL_OP_UNKNOWN; } /* * Check there is pending cmd in execution. If none, set new command. */ static inline int -_atomic_set_cmd(struct i40e_vf *vf, enum i40e_virtchnl_ops ops) +_atomic_set_cmd(struct i40e_vf *vf, enum virtchnl_ops ops) { int ret = rte_atomic32_cmpset(&vf->pend_cmd, - I40E_VIRTCHNL_OP_UNKNOWN, ops); + VIRTCHNL_OP_UNKNOWN, ops); if (!ret) PMD_DRV_LOG(ERR, "There is incomplete cmd %d", vf->pend_cmd); @@ -339,7 +344,7 @@ i40evf_execute_vf_cmd(struct rte_eth_dev *dev, struct vf_cmd_info *args) info.msg = args->out_buffer; info.buf_len = args->out_size; - info.ops = I40E_VIRTCHNL_OP_UNKNOWN; + info.ops = VIRTCHNL_OP_UNKNOWN; info.result = I40E_SUCCESS; err = i40e_aq_send_msg_to_pf(hw, args->ops, I40E_SUCCESS, @@ -351,16 +356,17 @@ i40evf_execute_vf_cmd(struct rte_eth_dev *dev, struct vf_cmd_info *args) } switch (args->ops) { - case I40E_VIRTCHNL_OP_RESET_VF: + case VIRTCHNL_OP_RESET_VF: /*no need to process in this function */ err = 0; break; - case I40E_VIRTCHNL_OP_VERSION: - case I40E_VIRTCHNL_OP_GET_VF_RESOURCES: + case VIRTCHNL_OP_VERSION: + case VIRTCHNL_OP_GET_VF_RESOURCES: /* for init adminq commands, need to poll the response */ err = -1; do { ret = i40evf_read_pfmsg(dev, &info); + vf->cmd_retval = info.result; if (ret == I40EVF_MSG_CMD) { err = 0; break; @@ -376,13 +382,18 @@ i40evf_execute_vf_cmd(struct rte_eth_dev *dev, struct vf_cmd_info *args) /* for other adminq in running time, waiting the cmd done flag */ err = -1; do { - if (vf->pend_cmd == I40E_VIRTCHNL_OP_UNKNOWN) { + if (vf->pend_cmd == VIRTCHNL_OP_UNKNOWN) { err = 0; break; } rte_delay_ms(ASQ_DELAY_MS); /* If don't read msg or read sys event, continue */ } while (i++ < MAX_TRY_TIMES); + /* If there's no response is received, clear command */ + if (i >= MAX_TRY_TIMES) { + PMD_DRV_LOG(WARNING, "No response for %d", args->ops); + _clear_cmd(vf); + } break; } @@ -395,15 +406,15 @@ i40evf_execute_vf_cmd(struct rte_eth_dev *dev, struct vf_cmd_info *args) static int i40evf_check_api_version(struct rte_eth_dev *dev) { - struct i40e_virtchnl_version_info version, *pver; + struct virtchnl_version_info version, *pver; int err; struct vf_cmd_info args; struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); - version.major = I40E_VIRTCHNL_VERSION_MAJOR; - version.minor = I40E_VIRTCHNL_VERSION_MINOR; + version.major = VIRTCHNL_VERSION_MAJOR; + version.minor = VIRTCHNL_VERSION_MINOR; - args.ops = I40E_VIRTCHNL_OP_VERSION; + args.ops = VIRTCHNL_OP_VERSION; args.in_args = (uint8_t *)&version; args.in_args_size = sizeof(version); args.out_buffer = vf->aq_resp; @@ -415,19 +426,17 @@ i40evf_check_api_version(struct rte_eth_dev *dev) return err; } - pver = (struct i40e_virtchnl_version_info *)args.out_buffer; + pver = (struct virtchnl_version_info *)args.out_buffer; vf->version_major = pver->major; vf->version_minor = pver->minor; - if (vf->version_major == I40E_DPDK_VERSION_MAJOR) - PMD_DRV_LOG(INFO, "Peer is DPDK PF host"); - else if ((vf->version_major == I40E_VIRTCHNL_VERSION_MAJOR) && - (vf->version_minor <= I40E_VIRTCHNL_VERSION_MINOR)) + if ((vf->version_major == VIRTCHNL_VERSION_MAJOR) && + (vf->version_minor <= VIRTCHNL_VERSION_MINOR)) PMD_DRV_LOG(INFO, "Peer is Linux PF host"); else { PMD_INIT_LOG(ERR, "PF/VF API version mismatch:(%u.%u)-(%u.%u)", vf->version_major, vf->version_minor, - I40E_VIRTCHNL_VERSION_MAJOR, - I40E_VIRTCHNL_VERSION_MINOR); + VIRTCHNL_VERSION_MAJOR, + VIRTCHNL_VERSION_MINOR); return -1; } @@ -443,15 +452,15 @@ i40evf_get_vf_resource(struct rte_eth_dev *dev) struct vf_cmd_info args; uint32_t caps, len; - args.ops = I40E_VIRTCHNL_OP_GET_VF_RESOURCES; + args.ops = VIRTCHNL_OP_GET_VF_RESOURCES; args.out_buffer = vf->aq_resp; args.out_size = I40E_AQ_BUF_SZ; if (PF_IS_V11(vf)) { - caps = I40E_VIRTCHNL_VF_OFFLOAD_L2 | - I40E_VIRTCHNL_VF_OFFLOAD_RSS_AQ | - I40E_VIRTCHNL_VF_OFFLOAD_RSS_REG | - I40E_VIRTCHNL_VF_OFFLOAD_VLAN | - I40E_VIRTCHNL_VF_OFFLOAD_RX_POLLING; + caps = VIRTCHNL_VF_OFFLOAD_L2 | + VIRTCHNL_VF_OFFLOAD_RSS_AQ | + VIRTCHNL_VF_OFFLOAD_RSS_REG | + VIRTCHNL_VF_OFFLOAD_VLAN | + VIRTCHNL_VF_OFFLOAD_RX_POLLING; args.in_args = (uint8_t *)∩︀ args.in_args_size = sizeof(caps); } else { @@ -465,10 +474,10 @@ i40evf_get_vf_resource(struct rte_eth_dev *dev) return err; } - len = sizeof(struct i40e_virtchnl_vf_resource) + - I40E_MAX_VF_VSI * sizeof(struct i40e_virtchnl_vsi_resource); + len = sizeof(struct virtchnl_vf_resource) + + I40E_MAX_VF_VSI * sizeof(struct virtchnl_vsi_resource); - (void)rte_memcpy(vf->vf_res, args.out_buffer, + rte_memcpy(vf->vf_res, args.out_buffer, RTE_MIN(args.out_size, len)); i40e_vf_parse_hw_config(hw, vf->vf_res); @@ -483,18 +492,18 @@ i40evf_config_promisc(struct rte_eth_dev *dev, struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); int err; struct vf_cmd_info args; - struct i40e_virtchnl_promisc_info promisc; + struct virtchnl_promisc_info promisc; promisc.flags = 0; promisc.vsi_id = vf->vsi_res->vsi_id; if (enable_unicast) - promisc.flags |= I40E_FLAG_VF_UNICAST_PROMISC; + promisc.flags |= FLAG_VF_UNICAST_PROMISC; if (enable_multicast) - promisc.flags |= I40E_FLAG_VF_MULTICAST_PROMISC; + promisc.flags |= FLAG_VF_MULTICAST_PROMISC; - args.ops = I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE; + args.ops = VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE; args.in_args = (uint8_t *)&promisc; args.in_args_size = sizeof(promisc); args.out_buffer = vf->aq_resp; @@ -508,65 +517,50 @@ i40evf_config_promisc(struct rte_eth_dev *dev, return err; } -/* Configure vlan and double vlan offload. Use flag to specify which part to configure */ static int -i40evf_config_vlan_offload(struct rte_eth_dev *dev, - bool enable_vlan_strip) +i40evf_enable_vlan_strip(struct rte_eth_dev *dev) { struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); - int err; struct vf_cmd_info args; - struct i40e_virtchnl_vlan_offload_info offload; + int ret; - offload.vsi_id = vf->vsi_res->vsi_id; - offload.enable_vlan_strip = enable_vlan_strip; - - args.ops = (enum i40e_virtchnl_ops)I40E_VIRTCHNL_OP_CFG_VLAN_OFFLOAD; - args.in_args = (uint8_t *)&offload; - args.in_args_size = sizeof(offload); + memset(&args, 0, sizeof(args)); + args.ops = VIRTCHNL_OP_ENABLE_VLAN_STRIPPING; + args.in_args = NULL; + args.in_args_size = 0; args.out_buffer = vf->aq_resp; args.out_size = I40E_AQ_BUF_SZ; + ret = i40evf_execute_vf_cmd(dev, &args); + if (ret) + PMD_DRV_LOG(ERR, "Failed to execute command of " + "VIRTCHNL_OP_ENABLE_VLAN_STRIPPING"); - err = i40evf_execute_vf_cmd(dev, &args); - if (err) - PMD_DRV_LOG(ERR, "fail to execute command CFG_VLAN_OFFLOAD"); - - return err; + return ret; } static int -i40evf_config_vlan_pvid(struct rte_eth_dev *dev, - struct i40e_vsi_vlan_pvid_info *info) +i40evf_disable_vlan_strip(struct rte_eth_dev *dev) { struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); - int err; struct vf_cmd_info args; - struct i40e_virtchnl_pvid_info tpid_info; + int ret; - if (info == NULL) { - PMD_DRV_LOG(ERR, "invalid parameters"); - return I40E_ERR_PARAM; - } - - memset(&tpid_info, 0, sizeof(tpid_info)); - tpid_info.vsi_id = vf->vsi_res->vsi_id; - (void)rte_memcpy(&tpid_info.info, info, sizeof(*info)); - - args.ops = (enum i40e_virtchnl_ops)I40E_VIRTCHNL_OP_CFG_VLAN_PVID; - args.in_args = (uint8_t *)&tpid_info; - args.in_args_size = sizeof(tpid_info); + memset(&args, 0, sizeof(args)); + args.ops = VIRTCHNL_OP_DISABLE_VLAN_STRIPPING; + args.in_args = NULL; + args.in_args_size = 0; args.out_buffer = vf->aq_resp; args.out_size = I40E_AQ_BUF_SZ; + ret = i40evf_execute_vf_cmd(dev, &args); + if (ret) + PMD_DRV_LOG(ERR, "Failed to execute command of " + "VIRTCHNL_OP_DISABLE_VLAN_STRIPPING"); - err = i40evf_execute_vf_cmd(dev, &args); - if (err) - PMD_DRV_LOG(ERR, "fail to execute command CFG_VLAN_PVID"); - - return err; + return ret; } static void -i40evf_fill_virtchnl_vsi_txq_info(struct i40e_virtchnl_txq_info *txq_info, +i40evf_fill_virtchnl_vsi_txq_info(struct virtchnl_txq_info *txq_info, uint16_t vsi_id, uint16_t queue_id, uint16_t nb_txq, @@ -581,7 +575,7 @@ i40evf_fill_virtchnl_vsi_txq_info(struct i40e_virtchnl_txq_info *txq_info, } static void -i40evf_fill_virtchnl_vsi_rxq_info(struct i40e_virtchnl_rxq_info *rxq_info, +i40evf_fill_virtchnl_vsi_rxq_info(struct virtchnl_rxq_info *rxq_info, uint16_t vsi_id, uint16_t queue_id, uint16_t nb_rxq, @@ -600,7 +594,6 @@ i40evf_fill_virtchnl_vsi_rxq_info(struct i40e_virtchnl_rxq_info *rxq_info, } } -/* It configures VSI queues to co-work with Linux PF host */ static int i40evf_configure_vsi_queues(struct rte_eth_dev *dev) { @@ -609,8 +602,8 @@ i40evf_configure_vsi_queues(struct rte_eth_dev *dev) (struct i40e_rx_queue **)dev->data->rx_queues; struct i40e_tx_queue **txq = (struct i40e_tx_queue **)dev->data->tx_queues; - struct i40e_virtchnl_vsi_queue_config_info *vc_vqci; - struct i40e_virtchnl_queue_pair_info *vc_qpi; + struct virtchnl_vsi_queue_config_info *vc_vqci; + struct virtchnl_queue_pair_info *vc_qpi; struct vf_cmd_info args; uint16_t i, nb_qp = vf->num_queue_pairs; const uint32_t size = @@ -619,7 +612,7 @@ i40evf_configure_vsi_queues(struct rte_eth_dev *dev) int ret; memset(buff, 0, sizeof(buff)); - vc_vqci = (struct i40e_virtchnl_vsi_queue_config_info *)buff; + vc_vqci = (struct virtchnl_vsi_queue_config_info *)buff; vc_vqci->vsi_id = vf->vsi_res->vsi_id; vc_vqci->num_queue_pairs = nb_qp; @@ -631,7 +624,7 @@ i40evf_configure_vsi_queues(struct rte_eth_dev *dev) vf->max_pkt_len, rxq[i]); } memset(&args, 0, sizeof(args)); - args.ops = I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES; + args.ops = VIRTCHNL_OP_CONFIG_VSI_QUEUES; args.in_args = (uint8_t *)vc_vqci; args.in_args_size = size; args.out_buffer = vf->aq_resp; @@ -639,99 +632,31 @@ i40evf_configure_vsi_queues(struct rte_eth_dev *dev) ret = i40evf_execute_vf_cmd(dev, &args); if (ret) PMD_DRV_LOG(ERR, "Failed to execute command of " - "I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES\n"); + "VIRTCHNL_OP_CONFIG_VSI_QUEUES"); return ret; } -/* It configures VSI queues to co-work with DPDK PF host */ -static int -i40evf_configure_vsi_queues_ext(struct rte_eth_dev *dev) -{ - struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); - struct i40e_rx_queue **rxq = - (struct i40e_rx_queue **)dev->data->rx_queues; - struct i40e_tx_queue **txq = - (struct i40e_tx_queue **)dev->data->tx_queues; - struct i40e_virtchnl_vsi_queue_config_ext_info *vc_vqcei; - struct i40e_virtchnl_queue_pair_ext_info *vc_qpei; - struct vf_cmd_info args; - uint16_t i, nb_qp = vf->num_queue_pairs; - const uint32_t size = - I40E_VIRTCHNL_CONFIG_VSI_QUEUES_SIZE(vc_vqcei, nb_qp); - uint8_t buff[size]; - int ret; - - memset(buff, 0, sizeof(buff)); - vc_vqcei = (struct i40e_virtchnl_vsi_queue_config_ext_info *)buff; - vc_vqcei->vsi_id = vf->vsi_res->vsi_id; - vc_vqcei->num_queue_pairs = nb_qp; - vc_qpei = vc_vqcei->qpair; - for (i = 0; i < nb_qp; i++, vc_qpei++) { - i40evf_fill_virtchnl_vsi_txq_info(&vc_qpei->txq, - vc_vqcei->vsi_id, i, dev->data->nb_tx_queues, txq[i]); - i40evf_fill_virtchnl_vsi_rxq_info(&vc_qpei->rxq, - vc_vqcei->vsi_id, i, dev->data->nb_rx_queues, - vf->max_pkt_len, rxq[i]); - if (i < dev->data->nb_rx_queues) - /* - * It adds extra info for configuring VSI queues, which - * is needed to enable the configurable crc stripping - * in VF. - */ - vc_qpei->rxq_ext.crcstrip = - dev->data->dev_conf.rxmode.hw_strip_crc; - } - memset(&args, 0, sizeof(args)); - args.ops = - (enum i40e_virtchnl_ops)I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES_EXT; - args.in_args = (uint8_t *)vc_vqcei; - args.in_args_size = size; - args.out_buffer = vf->aq_resp; - args.out_size = I40E_AQ_BUF_SZ; - ret = i40evf_execute_vf_cmd(dev, &args); - if (ret) - PMD_DRV_LOG(ERR, "Failed to execute command of " - "I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES_EXT\n"); - - return ret; -} - -static int -i40evf_configure_queues(struct rte_eth_dev *dev) -{ - struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); - - if (vf->version_major == I40E_DPDK_VERSION_MAJOR) - /* To support DPDK PF host */ - return i40evf_configure_vsi_queues_ext(dev); - else - /* To support Linux PF host */ - return i40evf_configure_vsi_queues(dev); -} - static int i40evf_config_irq_map(struct rte_eth_dev *dev) { struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); struct vf_cmd_info args; - uint8_t cmd_buffer[sizeof(struct i40e_virtchnl_irq_map_info) + \ - sizeof(struct i40e_virtchnl_vector_map)]; - struct i40e_virtchnl_irq_map_info *map_info; - struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle; + uint8_t cmd_buffer[sizeof(struct virtchnl_irq_map_info) + \ + sizeof(struct virtchnl_vector_map)]; + struct virtchnl_irq_map_info *map_info; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; uint32_t vector_id; int i, err; - if (rte_intr_allow_others(intr_handle)) { - if (vf->version_major == I40E_DPDK_VERSION_MAJOR) - vector_id = I40EVF_VSI_DEFAULT_MSIX_INTR; - else - vector_id = I40EVF_VSI_DEFAULT_MSIX_INTR_LNX; - } else { + if (dev->data->dev_conf.intr_conf.rxq != 0 && + rte_intr_allow_others(intr_handle)) + vector_id = I40E_RX_VEC_START; + else vector_id = I40E_MISC_VEC_ID; - } - map_info = (struct i40e_virtchnl_irq_map_info *)cmd_buffer; + map_info = (struct virtchnl_irq_map_info *)cmd_buffer; map_info->num_vectors = 1; map_info->vecmap[0].rxitr_idx = I40E_ITR_INDEX_DEFAULT; map_info->vecmap[0].vsi_id = vf->vsi_res->vsi_id; @@ -746,7 +671,7 @@ i40evf_config_irq_map(struct rte_eth_dev *dev) intr_handle->intr_vec[i] = vector_id; } - args.ops = I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP; + args.ops = VIRTCHNL_OP_CONFIG_IRQ_MAP; args.in_args = (u8 *)cmd_buffer; args.in_args_size = sizeof(cmd_buffer); args.out_buffer = vf->aq_resp; @@ -763,7 +688,7 @@ i40evf_switch_queue(struct rte_eth_dev *dev, bool isrx, uint16_t qid, bool on) { struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); - struct i40e_virtchnl_queue_select queue_select; + struct virtchnl_queue_select queue_select; int err; struct vf_cmd_info args; memset(&queue_select, 0, sizeof(queue_select)); @@ -775,9 +700,9 @@ i40evf_switch_queue(struct rte_eth_dev *dev, bool isrx, uint16_t qid, queue_select.tx_queues |= 1 << qid; if (on) - args.ops = I40E_VIRTCHNL_OP_ENABLE_QUEUES; + args.ops = VIRTCHNL_OP_ENABLE_QUEUES; else - args.ops = I40E_VIRTCHNL_OP_DISABLE_QUEUES; + args.ops = VIRTCHNL_OP_DISABLE_QUEUES; args.in_args = (u8 *)&queue_select; args.in_args_size = sizeof(queue_select); args.out_buffer = vf->aq_resp; @@ -845,34 +770,34 @@ i40evf_stop_queues(struct rte_eth_dev *dev) return 0; } -static void +static int i40evf_add_mac_addr(struct rte_eth_dev *dev, struct ether_addr *addr, __rte_unused uint32_t index, __rte_unused uint32_t pool) { - struct i40e_virtchnl_ether_addr_list *list; + struct virtchnl_ether_addr_list *list; struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); - uint8_t cmd_buffer[sizeof(struct i40e_virtchnl_ether_addr_list) + \ - sizeof(struct i40e_virtchnl_ether_addr)]; + uint8_t cmd_buffer[sizeof(struct virtchnl_ether_addr_list) + \ + sizeof(struct virtchnl_ether_addr)]; int err; struct vf_cmd_info args; - if (i40e_validate_mac_addr(addr->addr_bytes) != I40E_SUCCESS) { + if (is_zero_ether_addr(addr)) { PMD_DRV_LOG(ERR, "Invalid mac:%x:%x:%x:%x:%x:%x", addr->addr_bytes[0], addr->addr_bytes[1], addr->addr_bytes[2], addr->addr_bytes[3], addr->addr_bytes[4], addr->addr_bytes[5]); - return; + return I40E_ERR_INVALID_MAC_ADDR; } - list = (struct i40e_virtchnl_ether_addr_list *)cmd_buffer; + list = (struct virtchnl_ether_addr_list *)cmd_buffer; list->vsi_id = vf->vsi_res->vsi_id; list->num_elements = 1; - (void)rte_memcpy(list->list[0].addr, addr->addr_bytes, + rte_memcpy(list->list[0].addr, addr->addr_bytes, sizeof(addr->addr_bytes)); - args.ops = I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS; + args.ops = VIRTCHNL_OP_ADD_ETH_ADDR; args.in_args = cmd_buffer; args.in_args_size = sizeof(cmd_buffer); args.out_buffer = vf->aq_resp; @@ -881,24 +806,23 @@ i40evf_add_mac_addr(struct rte_eth_dev *dev, if (err) PMD_DRV_LOG(ERR, "fail to execute command " "OP_ADD_ETHER_ADDRESS"); + else + vf->vsi.mac_num++; - return; + return err; } static void -i40evf_del_mac_addr(struct rte_eth_dev *dev, uint32_t index) +i40evf_del_mac_addr_by_addr(struct rte_eth_dev *dev, + struct ether_addr *addr) { - struct i40e_virtchnl_ether_addr_list *list; + struct virtchnl_ether_addr_list *list; struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); - struct rte_eth_dev_data *data = dev->data; - struct ether_addr *addr; - uint8_t cmd_buffer[sizeof(struct i40e_virtchnl_ether_addr_list) + \ - sizeof(struct i40e_virtchnl_ether_addr)]; + uint8_t cmd_buffer[sizeof(struct virtchnl_ether_addr_list) + \ + sizeof(struct virtchnl_ether_addr)]; int err; struct vf_cmd_info args; - addr = &(data->mac_addrs[index]); - if (i40e_validate_mac_addr(addr->addr_bytes) != I40E_SUCCESS) { PMD_DRV_LOG(ERR, "Invalid mac:%x-%x-%x-%x-%x-%x", addr->addr_bytes[0], addr->addr_bytes[1], @@ -907,13 +831,13 @@ i40evf_del_mac_addr(struct rte_eth_dev *dev, uint32_t index) return; } - list = (struct i40e_virtchnl_ether_addr_list *)cmd_buffer; + list = (struct virtchnl_ether_addr_list *)cmd_buffer; list->vsi_id = vf->vsi_res->vsi_id; list->num_elements = 1; - (void)rte_memcpy(list->list[0].addr, addr->addr_bytes, + rte_memcpy(list->list[0].addr, addr->addr_bytes, sizeof(addr->addr_bytes)); - args.ops = I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS; + args.ops = VIRTCHNL_OP_DEL_ETH_ADDR; args.in_args = cmd_buffer; args.in_args_size = sizeof(cmd_buffer); args.out_buffer = vf->aq_resp; @@ -922,20 +846,33 @@ i40evf_del_mac_addr(struct rte_eth_dev *dev, uint32_t index) if (err) PMD_DRV_LOG(ERR, "fail to execute command " "OP_DEL_ETHER_ADDRESS"); + else + vf->vsi.mac_num--; return; } +static void +i40evf_del_mac_addr(struct rte_eth_dev *dev, uint32_t index) +{ + struct rte_eth_dev_data *data = dev->data; + struct ether_addr *addr; + + addr = &data->mac_addrs[index]; + + i40evf_del_mac_addr_by_addr(dev, addr); +} + static int -i40evf_update_stats(struct rte_eth_dev *dev, struct i40e_eth_stats **pstats) +i40evf_query_stats(struct rte_eth_dev *dev, struct i40e_eth_stats **pstats) { struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); - struct i40e_virtchnl_queue_select q_stats; + struct virtchnl_queue_select q_stats; int err; struct vf_cmd_info args; memset(&q_stats, 0, sizeof(q_stats)); q_stats.vsi_id = vf->vsi_res->vsi_id; - args.ops = I40E_VIRTCHNL_OP_GET_STATS; + args.ops = VIRTCHNL_OP_GET_STATS; args.in_args = (u8 *)&q_stats; args.in_args_size = sizeof(q_stats); args.out_buffer = vf->aq_resp; @@ -951,39 +888,73 @@ i40evf_update_stats(struct rte_eth_dev *dev, struct i40e_eth_stats **pstats) return 0; } -static int -i40evf_get_statics(struct rte_eth_dev *dev, struct rte_eth_stats *stats) +static void +i40evf_stat_update_48(uint64_t *offset, + uint64_t *stat) { - int ret; - struct i40e_eth_stats *pstats = NULL; + if (*stat >= *offset) + *stat = *stat - *offset; + else + *stat = (uint64_t)((*stat + + ((uint64_t)1 << I40E_48_BIT_WIDTH)) - *offset); - ret = i40evf_update_stats(dev, &pstats); - if (ret != 0) - return 0; + *stat &= I40E_48_BIT_MASK; +} - stats->ipackets = pstats->rx_unicast + pstats->rx_multicast + - pstats->rx_broadcast; - stats->opackets = pstats->tx_broadcast + pstats->tx_multicast + - pstats->tx_unicast; - stats->ierrors = pstats->rx_discards; - stats->oerrors = pstats->tx_errors + pstats->tx_discards; - stats->ibytes = pstats->rx_bytes; - stats->obytes = pstats->tx_bytes; +static void +i40evf_stat_update_32(uint64_t *offset, + uint64_t *stat) +{ + if (*stat >= *offset) + *stat = (uint64_t)(*stat - *offset); + else + *stat = (uint64_t)((*stat + + ((uint64_t)1 << I40E_32_BIT_WIDTH)) - *offset); +} - return 0; +static void +i40evf_update_stats(struct i40e_vsi *vsi, + struct i40e_eth_stats *nes) +{ + struct i40e_eth_stats *oes = &vsi->eth_stats_offset; + + i40evf_stat_update_48(&oes->rx_bytes, + &nes->rx_bytes); + i40evf_stat_update_48(&oes->rx_unicast, + &nes->rx_unicast); + i40evf_stat_update_48(&oes->rx_multicast, + &nes->rx_multicast); + i40evf_stat_update_48(&oes->rx_broadcast, + &nes->rx_broadcast); + i40evf_stat_update_32(&oes->rx_discards, + &nes->rx_discards); + i40evf_stat_update_32(&oes->rx_unknown_protocol, + &nes->rx_unknown_protocol); + i40evf_stat_update_48(&oes->tx_bytes, + &nes->tx_bytes); + i40evf_stat_update_48(&oes->tx_unicast, + &nes->tx_unicast); + i40evf_stat_update_48(&oes->tx_multicast, + &nes->tx_multicast); + i40evf_stat_update_48(&oes->tx_broadcast, + &nes->tx_broadcast); + i40evf_stat_update_32(&oes->tx_errors, &nes->tx_errors); + i40evf_stat_update_32(&oes->tx_discards, &nes->tx_discards); } static void i40evf_dev_xstats_reset(struct rte_eth_dev *dev) { + int ret; struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); struct i40e_eth_stats *pstats = NULL; /* read stat values to clear hardware registers */ - i40evf_update_stats(dev, &pstats); + ret = i40evf_query_stats(dev, &pstats); /* set stats offset base on current values */ - vf->vsi.eth_stats_offset = vf->vsi.eth_stats; + if (ret == 0) + vf->vsi.eth_stats_offset = *pstats; } static int i40evf_dev_xstats_get_names(__rte_unused struct rte_eth_dev *dev, @@ -1007,17 +978,21 @@ static int i40evf_dev_xstats_get(struct rte_eth_dev *dev, int ret; unsigned i; struct i40e_eth_stats *pstats = NULL; + struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); + struct i40e_vsi *vsi = &vf->vsi; if (n < I40EVF_NB_XSTATS) return I40EVF_NB_XSTATS; - ret = i40evf_update_stats(dev, &pstats); + ret = i40evf_query_stats(dev, &pstats); if (ret != 0) return 0; if (!xstats) return 0; + i40evf_update_stats(vsi, pstats); + /* loop over xstats array and values from pstats */ for (i = 0; i < I40EVF_NB_XSTATS; i++) { xstats[i].id = i; @@ -1032,18 +1007,18 @@ static int i40evf_add_vlan(struct rte_eth_dev *dev, uint16_t vlanid) { struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); - struct i40e_virtchnl_vlan_filter_list *vlan_list; - uint8_t cmd_buffer[sizeof(struct i40e_virtchnl_vlan_filter_list) + + struct virtchnl_vlan_filter_list *vlan_list; + uint8_t cmd_buffer[sizeof(struct virtchnl_vlan_filter_list) + sizeof(uint16_t)]; int err; struct vf_cmd_info args; - vlan_list = (struct i40e_virtchnl_vlan_filter_list *)cmd_buffer; + vlan_list = (struct virtchnl_vlan_filter_list *)cmd_buffer; vlan_list->vsi_id = vf->vsi_res->vsi_id; vlan_list->num_elements = 1; vlan_list->vlan_id[0] = vlanid; - args.ops = I40E_VIRTCHNL_OP_ADD_VLAN; + args.ops = VIRTCHNL_OP_ADD_VLAN; args.in_args = (u8 *)&cmd_buffer; args.in_args_size = sizeof(cmd_buffer); args.out_buffer = vf->aq_resp; @@ -1059,18 +1034,18 @@ static int i40evf_del_vlan(struct rte_eth_dev *dev, uint16_t vlanid) { struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); - struct i40e_virtchnl_vlan_filter_list *vlan_list; - uint8_t cmd_buffer[sizeof(struct i40e_virtchnl_vlan_filter_list) + + struct virtchnl_vlan_filter_list *vlan_list; + uint8_t cmd_buffer[sizeof(struct virtchnl_vlan_filter_list) + sizeof(uint16_t)]; int err; struct vf_cmd_info args; - vlan_list = (struct i40e_virtchnl_vlan_filter_list *)cmd_buffer; + vlan_list = (struct virtchnl_vlan_filter_list *)cmd_buffer; vlan_list->vsi_id = vf->vsi_res->vsi_id; vlan_list->num_elements = 1; vlan_list->vlan_id[0] = vlanid; - args.ops = I40E_VIRTCHNL_OP_DEL_VLAN; + args.ops = VIRTCHNL_OP_DEL_VLAN; args.in_args = (u8 *)&cmd_buffer; args.in_args_size = sizeof(cmd_buffer); args.out_buffer = vf->aq_resp; @@ -1087,7 +1062,6 @@ static const struct rte_pci_id pci_id_i40evf_map[] = { { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_VF_HV) }, { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_X722_A0_VF) }, { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_X722_VF) }, - { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_X722_VF_HV) }, { .vendor_id = 0, /* sentinel */ }, }; @@ -1138,10 +1112,30 @@ i40evf_enable_irq0(struct i40e_hw *hw) } static int -i40evf_reset_vf(struct i40e_hw *hw) +i40evf_check_vf_reset_done(struct i40e_hw *hw) { int i, reset; + for (i = 0; i < MAX_RESET_WAIT_CNT; i++) { + reset = I40E_READ_REG(hw, I40E_VFGEN_RSTAT) & + I40E_VFGEN_RSTAT_VFR_STATE_MASK; + reset = reset >> I40E_VFGEN_RSTAT_VFR_STATE_SHIFT; + if (reset == VIRTCHNL_VFR_VFACTIVE || + reset == VIRTCHNL_VFR_COMPLETED) + break; + rte_delay_ms(50); + } + + if (i >= MAX_RESET_WAIT_CNT) + return -1; + + return 0; +} +static int +i40evf_reset_vf(struct i40e_hw *hw) +{ + int ret; + if (i40e_vf_reset(hw) != I40E_SUCCESS) { PMD_INIT_LOG(ERR, "Reset VF NIC failed"); return -1; @@ -1157,19 +1151,10 @@ i40evf_reset_vf(struct i40e_hw *hw) */ rte_delay_ms(200); - for (i = 0; i < MAX_RESET_WAIT_CNT; i++) { - reset = rd32(hw, I40E_VFGEN_RSTAT) & - I40E_VFGEN_RSTAT_VFR_STATE_MASK; - reset = reset >> I40E_VFGEN_RSTAT_VFR_STATE_SHIFT; - if (I40E_VFR_COMPLETED == reset || I40E_VFR_VFACTIVE == reset) - break; - else - rte_delay_ms(50); - } - - if (i >= MAX_RESET_WAIT_CNT) { - PMD_INIT_LOG(ERR, "Reset VF NIC failed"); - return -1; + ret = i40evf_check_vf_reset_done(hw); + if (ret) { + PMD_INIT_LOG(ERR, "VF is still resetting"); + return ret; } return 0; @@ -1181,9 +1166,8 @@ i40evf_init_vf(struct rte_eth_dev *dev) int i, err, bufsz; struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); - struct ether_addr *p_mac_addr; uint16_t interval = - i40e_calc_itr_interval(I40E_QUEUE_ITR_INTERVAL_MAX); + i40e_calc_itr_interval(I40E_QUEUE_ITR_INTERVAL_MAX, 0); vf->adapter = I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); vf->dev_data = dev->data; @@ -1193,6 +1177,10 @@ i40evf_init_vf(struct rte_eth_dev *dev) goto err; } + err = i40evf_check_vf_reset_done(hw); + if (err) + goto err; + i40e_init_adminq_parameter(hw); err = i40e_init_adminq(hw); if (err) { @@ -1209,29 +1197,30 @@ i40evf_init_vf(struct rte_eth_dev *dev) /* VF reset, shutdown admin queue and initialize again */ if (i40e_shutdown_adminq(hw) != I40E_SUCCESS) { PMD_INIT_LOG(ERR, "i40e_shutdown_adminq failed"); - return -1; + goto err; } i40e_init_adminq_parameter(hw); if (i40e_init_adminq(hw) != I40E_SUCCESS) { PMD_INIT_LOG(ERR, "init_adminq failed"); - return -1; + goto err; } + vf->aq_resp = rte_zmalloc("vf_aq_resp", I40E_AQ_BUF_SZ, 0); if (!vf->aq_resp) { PMD_INIT_LOG(ERR, "unable to allocate vf_aq_resp memory"); - goto err_aq; + goto err_aq; } if (i40evf_check_api_version(dev) != 0) { PMD_INIT_LOG(ERR, "check_api version failed"); - goto err_aq; + goto err_api; } - bufsz = sizeof(struct i40e_virtchnl_vf_resource) + - (I40E_MAX_VF_VSI * sizeof(struct i40e_virtchnl_vsi_resource)); + bufsz = sizeof(struct virtchnl_vf_resource) + + (I40E_MAX_VF_VSI * sizeof(struct virtchnl_vsi_resource)); vf->vf_res = rte_zmalloc("vf_res", bufsz, 0); if (!vf->vf_res) { PMD_INIT_LOG(ERR, "unable to allocate vf_res memory"); - goto err_aq; + goto err_api; } if (i40evf_get_vf_resource(dev) != 0) { @@ -1241,7 +1230,7 @@ i40evf_init_vf(struct rte_eth_dev *dev) /* got VF config message back from PF, now we can parse it */ for (i = 0; i < vf->vf_res->num_vsis; i++) { - if (vf->vf_res->vsi_res[i].vsi_type == I40E_VSI_SRIOV) + if (vf->vf_res->vsi_res[i].vsi_type == VIRTCHNL_VSI_SRIOV) vf->vsi_res = &vf->vf_res->vsi_res[i]; } @@ -1253,31 +1242,38 @@ i40evf_init_vf(struct rte_eth_dev *dev) if (hw->mac.type == I40E_MAC_X722_VF) vf->flags = I40E_FLAG_RSS_AQ_CAPABLE; vf->vsi.vsi_id = vf->vsi_res->vsi_id; - vf->vsi.type = vf->vsi_res->vsi_type; + + switch (vf->vsi_res->vsi_type) { + case VIRTCHNL_VSI_SRIOV: + vf->vsi.type = I40E_VSI_SRIOV; + break; + default: + vf->vsi.type = I40E_VSI_TYPE_UNKNOWN; + break; + } vf->vsi.nb_qps = vf->vsi_res->num_queue_pairs; vf->vsi.adapter = I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); /* Store the MAC address configured by host, or generate random one */ - p_mac_addr = (struct ether_addr *)(vf->vsi_res->default_mac_addr); - if (is_valid_assigned_ether_addr(p_mac_addr)) /* Configured by host */ - ether_addr_copy(p_mac_addr, (struct ether_addr *)hw->mac.addr); + if (is_valid_assigned_ether_addr((struct ether_addr *)hw->mac.addr)) + vf->flags |= I40E_FLAG_VF_MAC_BY_PF; else eth_random_addr(hw->mac.addr); /* Generate a random one */ - /* If the PF host is not DPDK, set the interval of ITR0 to max*/ - if (vf->version_major != I40E_DPDK_VERSION_MAJOR) { - I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01, - (I40E_ITR_INDEX_DEFAULT << - I40E_VFINT_DYN_CTL0_ITR_INDX_SHIFT) | - (interval << - I40E_VFINT_DYN_CTL0_INTERVAL_SHIFT)); - I40EVF_WRITE_FLUSH(hw); - } + I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01, + (I40E_ITR_INDEX_DEFAULT << + I40E_VFINT_DYN_CTL0_ITR_INDX_SHIFT) | + (interval << + I40E_VFINT_DYN_CTL0_INTERVAL_SHIFT)); + I40EVF_WRITE_FLUSH(hw); return 0; err_alloc: rte_free(vf->vf_res); + vf->vsi_res = NULL; +err_api: + rte_free(vf->aq_resp); err_aq: i40e_shutdown_adminq(hw); /* ignore error */ err: @@ -1303,26 +1299,26 @@ i40evf_uninit_vf(struct rte_eth_dev *dev) } static void -i40evf_handle_pf_event(__rte_unused struct rte_eth_dev *dev, - uint8_t *msg, - __rte_unused uint16_t msglen) +i40evf_handle_pf_event(struct rte_eth_dev *dev, uint8_t *msg, + __rte_unused uint16_t msglen) { - struct i40e_virtchnl_pf_event *pf_msg = - (struct i40e_virtchnl_pf_event *)msg; + struct virtchnl_pf_event *pf_msg = + (struct virtchnl_pf_event *)msg; struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); switch (pf_msg->event) { - case I40E_VIRTCHNL_EVENT_RESET_IMPENDING: - PMD_DRV_LOG(DEBUG, "VIRTCHNL_EVENT_RESET_IMPENDING event\n"); - _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RESET); + case VIRTCHNL_EVENT_RESET_IMPENDING: + PMD_DRV_LOG(DEBUG, "VIRTCHNL_EVENT_RESET_IMPENDING event"); + _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RESET, + NULL, NULL); break; - case I40E_VIRTCHNL_EVENT_LINK_CHANGE: - PMD_DRV_LOG(DEBUG, "VIRTCHNL_EVENT_LINK_CHANGE event\n"); + case VIRTCHNL_EVENT_LINK_CHANGE: + PMD_DRV_LOG(DEBUG, "VIRTCHNL_EVENT_LINK_CHANGE event"); vf->link_up = pf_msg->event_data.link_event.link_status; vf->link_speed = pf_msg->event_data.link_event.link_speed; break; - case I40E_VIRTCHNL_EVENT_PF_DRIVER_CLOSE: - PMD_DRV_LOG(DEBUG, "VIRTCHNL_EVENT_PF_DRIVER_CLOSE event\n"); + case VIRTCHNL_EVENT_PF_DRIVER_CLOSE: + PMD_DRV_LOG(DEBUG, "VIRTCHNL_EVENT_PF_DRIVER_CLOSE event"); break; default: PMD_DRV_LOG(ERR, " unknown event received %u", pf_msg->event); @@ -1336,8 +1332,9 @@ i40evf_handle_aq_msg(struct rte_eth_dev *dev) struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); struct i40e_arq_event_info info; - struct i40e_virtchnl_msg *v_msg; - uint16_t pending, opcode; + uint16_t pending, aq_opc; + enum virtchnl_ops msg_opc; + enum i40e_status_code msg_ret; int ret; info.buf_len = I40E_AQ_BUF_SZ; @@ -1346,7 +1343,6 @@ i40evf_handle_aq_msg(struct rte_eth_dev *dev) return; } info.msg_buf = vf->aq_resp; - v_msg = (struct i40e_virtchnl_msg *)&info.desc; pending = 1; while (pending) { @@ -1357,32 +1353,39 @@ i40evf_handle_aq_msg(struct rte_eth_dev *dev) "ret: %d", ret); break; } - opcode = rte_le_to_cpu_16(info.desc.opcode); - - switch (opcode) { + aq_opc = rte_le_to_cpu_16(info.desc.opcode); + /* For the message sent from pf to vf, opcode is stored in + * cookie_high of struct i40e_aq_desc, while return error code + * are stored in cookie_low, Which is done by + * i40e_aq_send_msg_to_vf in PF driver.*/ + msg_opc = (enum virtchnl_ops)rte_le_to_cpu_32( + info.desc.cookie_high); + msg_ret = (enum i40e_status_code)rte_le_to_cpu_32( + info.desc.cookie_low); + switch (aq_opc) { case i40e_aqc_opc_send_msg_to_vf: - if (v_msg->v_opcode == I40E_VIRTCHNL_OP_EVENT) + if (msg_opc == VIRTCHNL_OP_EVENT) /* process event*/ i40evf_handle_pf_event(dev, info.msg_buf, info.msg_len); else { /* read message and it's expected one */ - if (v_msg->v_opcode == vf->pend_cmd) { - vf->cmd_retval = v_msg->v_retval; + if (msg_opc == vf->pend_cmd) { + vf->cmd_retval = msg_ret; /* prevent compiler reordering */ rte_compiler_barrier(); _clear_cmd(vf); } else PMD_DRV_LOG(ERR, "command mismatch," "expect %u, get %u", - vf->pend_cmd, v_msg->v_opcode); + vf->pend_cmd, msg_opc); PMD_DRV_LOG(DEBUG, "adminq response is received," - " opcode = %d\n", v_msg->v_opcode); + " opcode = %d", msg_opc); } break; default: PMD_DRV_LOG(ERR, "Request %u is not supported yet", - opcode); + aq_opc); break; } } @@ -1401,8 +1404,7 @@ i40evf_handle_aq_msg(struct rte_eth_dev *dev) * void */ static void -i40evf_dev_interrupt_handler(__rte_unused struct rte_intr_handle *handle, - void *param) +i40evf_dev_interrupt_handler(void *param) { struct rte_eth_dev *dev = (struct rte_eth_dev *)param; struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); @@ -1415,31 +1417,30 @@ i40evf_dev_interrupt_handler(__rte_unused struct rte_intr_handle *handle, /* No interrupt event indicated */ if (!(icr0 & I40E_VFINT_ICR01_INTEVENT_MASK)) { - PMD_DRV_LOG(DEBUG, "No interrupt event, nothing to do\n"); + PMD_DRV_LOG(DEBUG, "No interrupt event, nothing to do"); goto done; } if (icr0 & I40E_VFINT_ICR01_ADMINQ_MASK) { - PMD_DRV_LOG(DEBUG, "ICR01_ADMINQ is reported\n"); + PMD_DRV_LOG(DEBUG, "ICR01_ADMINQ is reported"); i40evf_handle_aq_msg(dev); } /* Link Status Change interrupt */ if (icr0 & I40E_VFINT_ICR01_LINK_STAT_CHANGE_MASK) PMD_DRV_LOG(DEBUG, "LINK_STAT_CHANGE is reported," - " do nothing\n"); + " do nothing"); done: i40evf_enable_irq0(hw); - rte_intr_enable(&dev->pci_dev->intr_handle); } static int i40evf_dev_init(struct rte_eth_dev *eth_dev) { - struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(\ - eth_dev->data->dev_private); - struct rte_pci_device *pci_dev = eth_dev->pci_dev; + struct i40e_hw *hw + = I40E_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); PMD_INIT_FUNC_TRACE(); @@ -1457,16 +1458,17 @@ i40evf_dev_init(struct rte_eth_dev *eth_dev) i40e_set_tx_function(eth_dev); return 0; } + i40e_set_default_ptype_table(eth_dev); + i40e_set_default_pctype_table(eth_dev); + rte_eth_copy_pci_info(eth_dev, pci_dev); - rte_eth_copy_pci_info(eth_dev, eth_dev->pci_dev); - - hw->vendor_id = eth_dev->pci_dev->id.vendor_id; - hw->device_id = eth_dev->pci_dev->id.device_id; - hw->subsystem_vendor_id = eth_dev->pci_dev->id.subsystem_vendor_id; - hw->subsystem_device_id = eth_dev->pci_dev->id.subsystem_device_id; - hw->bus.device = eth_dev->pci_dev->addr.devid; - hw->bus.func = eth_dev->pci_dev->addr.function; - hw->hw_addr = (void *)eth_dev->pci_dev->mem_resource[0].addr; + hw->vendor_id = pci_dev->id.vendor_id; + hw->device_id = pci_dev->id.device_id; + hw->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id; + hw->subsystem_device_id = pci_dev->id.subsystem_device_id; + hw->bus.device = pci_dev->addr.devid; + hw->bus.func = pci_dev->addr.function; + hw->hw_addr = (void *)pci_dev->mem_resource[0].addr; hw->adapter_stopped = 0; if(i40evf_init_vf(eth_dev) != 0) { @@ -1522,43 +1524,32 @@ i40evf_dev_uninit(struct rte_eth_dev *eth_dev) return 0; } + +static int eth_i40evf_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, + struct rte_pci_device *pci_dev) +{ + return rte_eth_dev_pci_generic_probe(pci_dev, + sizeof(struct i40e_adapter), i40evf_dev_init); +} + +static int eth_i40evf_pci_remove(struct rte_pci_device *pci_dev) +{ + return rte_eth_dev_pci_generic_remove(pci_dev, i40evf_dev_uninit); +} + /* * virtual function driver struct */ -static struct eth_driver rte_i40evf_pmd = { - .pci_drv = { - .name = "rte_i40evf_pmd", - .id_table = pci_id_i40evf_map, - .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_DETACHABLE, - }, - .eth_dev_init = i40evf_dev_init, - .eth_dev_uninit = i40evf_dev_uninit, - .dev_private_size = sizeof(struct i40e_adapter), +static struct rte_pci_driver rte_i40evf_pmd = { + .id_table = pci_id_i40evf_map, + .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_IOVA_AS_VA, + .probe = eth_i40evf_pci_probe, + .remove = eth_i40evf_pci_remove, }; -/* - * VF Driver initialization routine. - * Invoked one at EAL init time. - * Register itself as the [Virtual Poll Mode] Driver of PCI Fortville devices. - */ -static int -rte_i40evf_pmd_init(const char *name __rte_unused, - const char *params __rte_unused) -{ - PMD_INIT_FUNC_TRACE(); - - rte_eth_driver_register(&rte_i40evf_pmd); - - return 0; -} - -static struct rte_driver rte_i40evf_driver = { - .type = PMD_PDEV, - .init = rte_i40evf_pmd_init, -}; - -PMD_REGISTER_DRIVER(rte_i40evf_driver, i40evf); -DRIVER_REGISTER_PCI_TABLE(i40evf, pci_id_i40evf_map); +RTE_PMD_REGISTER_PCI(net_i40e_vf, rte_i40evf_pmd); +RTE_PMD_REGISTER_PCI_TABLE(net_i40e_vf, pci_id_i40evf_map); +RTE_PMD_REGISTER_KMOD_DEP(net_i40e_vf, "* igb_uio | vfio-pci"); static int i40evf_dev_configure(struct rte_eth_dev *dev) @@ -1581,8 +1572,8 @@ i40evf_dev_configure(struct rte_eth_dev *dev) */ if (!conf->rxmode.hw_strip_crc) { vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); - if ((vf->version_major == I40E_VIRTCHNL_VERSION_MAJOR) && - (vf->version_minor <= I40E_VIRTCHNL_VERSION_MINOR)) { + if ((vf->version_major == VIRTCHNL_VERSION_MAJOR) && + (vf->version_minor <= VIRTCHNL_VERSION_MINOR)) { /* Peer is running non-DPDK PF driver. */ PMD_INIT_LOG(ERR, "VF can't disable HW CRC Strip"); return -EINVAL; @@ -1595,61 +1586,28 @@ i40evf_dev_configure(struct rte_eth_dev *dev) static int i40evf_init_vlan(struct rte_eth_dev *dev) { - struct rte_eth_dev_data *data = dev->data; - int ret; - /* Apply vlan offload setting */ i40evf_vlan_offload_set(dev, ETH_VLAN_STRIP_MASK); - /* Apply pvid setting */ - ret = i40evf_vlan_pvid_set(dev, data->dev_conf.txmode.pvid, - data->dev_conf.txmode.hw_vlan_insert_pvid); - return ret; -} - -static void -i40evf_vlan_offload_set(struct rte_eth_dev *dev, int mask) -{ - bool enable_vlan_strip = 0; - struct rte_eth_conf *dev_conf = &dev->data->dev_conf; - struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); - - /* Linux pf host doesn't support vlan offload yet */ - if (vf->version_major == I40E_DPDK_VERSION_MAJOR) { - /* Vlan stripping setting */ - if (mask & ETH_VLAN_STRIP_MASK) { - /* Enable or disable VLAN stripping */ - if (dev_conf->rxmode.hw_vlan_strip) - enable_vlan_strip = 1; - else - enable_vlan_strip = 0; - - i40evf_config_vlan_offload(dev, enable_vlan_strip); - } - } + return 0; } static int -i40evf_vlan_pvid_set(struct rte_eth_dev *dev, uint16_t pvid, int on) +i40evf_vlan_offload_set(struct rte_eth_dev *dev, int mask) { struct rte_eth_conf *dev_conf = &dev->data->dev_conf; - struct i40e_vsi_vlan_pvid_info info; struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); - memset(&info, 0, sizeof(info)); - info.on = on; + if (!(vf->vf_res->vf_offload_flags & VIRTCHNL_VF_OFFLOAD_VLAN)) + return -ENOTSUP; - /* Linux pf host don't support vlan offload yet */ - if (vf->version_major == I40E_DPDK_VERSION_MAJOR) { - if (info.on) - info.config.pvid = pvid; - else { - info.config.reject.tagged = - dev_conf->txmode.hw_vlan_reject_tagged; - info.config.reject.untagged = - dev_conf->txmode.hw_vlan_reject_untagged; - } - return i40evf_config_vlan_pvid(dev, &info); + /* Vlan stripping setting */ + if (mask & ETH_VLAN_STRIP_MASK) { + /* Enable or disable VLAN stripping */ + if (dev_conf->rxmode.hw_vlan_strip) + i40evf_enable_vlan_strip(dev); + else + i40evf_disable_vlan_strip(dev); } return 0; @@ -1871,9 +1829,9 @@ i40evf_tx_init(struct rte_eth_dev *dev) static inline void i40evf_enable_queues_intr(struct rte_eth_dev *dev) { - struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); - struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; if (!rte_intr_allow_others(intr_handle)) { I40E_WRITE_REG(hw, @@ -1885,27 +1843,15 @@ i40evf_enable_queues_intr(struct rte_eth_dev *dev) return; } - if (vf->version_major == I40E_DPDK_VERSION_MAJOR) - /* To support DPDK PF host */ - I40E_WRITE_REG(hw, - I40E_VFINT_DYN_CTLN1(I40EVF_VSI_DEFAULT_MSIX_INTR - 1), - I40E_VFINT_DYN_CTLN1_INTENA_MASK | - I40E_VFINT_DYN_CTLN_CLEARPBA_MASK); - /* If host driver is kernel driver, do nothing. - * Interrupt 0 is used for rx packets, but don't set - * I40E_VFINT_DYN_CTL01, - * because it is already done in i40evf_enable_irq0. - */ - I40EVF_WRITE_FLUSH(hw); } static inline void i40evf_disable_queues_intr(struct rte_eth_dev *dev) { - struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); - struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; if (!rte_intr_allow_others(intr_handle)) { I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01, @@ -1914,27 +1860,17 @@ i40evf_disable_queues_intr(struct rte_eth_dev *dev) return; } - if (vf->version_major == I40E_DPDK_VERSION_MAJOR) - I40E_WRITE_REG(hw, - I40E_VFINT_DYN_CTLN1(I40EVF_VSI_DEFAULT_MSIX_INTR - - 1), - 0); - /* If host driver is kernel driver, do nothing. - * Interrupt 0 is used for rx packets, but don't zero - * I40E_VFINT_DYN_CTL01, - * because interrupt 0 is also used for adminq processing. - */ - I40EVF_WRITE_FLUSH(hw); } static int i40evf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) { - struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); uint16_t interval = - i40e_calc_itr_interval(RTE_LIBRTE_I40E_ITR_INTERVAL); + i40e_calc_itr_interval(RTE_LIBRTE_I40E_ITR_INTERVAL, 0); uint16_t msix_intr; msix_intr = intr_handle->intr_vec[queue_id]; @@ -1957,7 +1893,7 @@ i40evf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) I40EVF_WRITE_FLUSH(hw); - rte_intr_enable(&dev->pci_dev->intr_handle); + rte_intr_enable(&pci_dev->intr_handle); return 0; } @@ -1965,7 +1901,8 @@ i40evf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) static int i40evf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id) { - struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); uint16_t msix_intr; @@ -1986,7 +1923,7 @@ i40evf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id) static void i40evf_add_del_all_mac_addr(struct rte_eth_dev *dev, bool add) { - struct i40e_virtchnl_ether_addr_list *list; + struct virtchnl_ether_addr_list *list; struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); int err, i, j; int next_begin = 0; @@ -1997,11 +1934,11 @@ i40evf_add_del_all_mac_addr(struct rte_eth_dev *dev, bool add) do { j = 0; - len = sizeof(struct i40e_virtchnl_ether_addr_list); + len = sizeof(struct virtchnl_ether_addr_list); for (i = begin; i < I40E_NUM_MACADDR_MAX; i++, next_begin++) { if (is_zero_ether_addr(&dev->data->mac_addrs[i])) continue; - len += sizeof(struct i40e_virtchnl_ether_addr); + len += sizeof(struct virtchnl_ether_addr); if (len >= I40E_AQ_BUF_SZ) { next_begin = i + 1; break; @@ -2009,12 +1946,16 @@ i40evf_add_del_all_mac_addr(struct rte_eth_dev *dev, bool add) } list = rte_zmalloc("i40evf_del_mac_buffer", len, 0); + if (!list) { + PMD_DRV_LOG(ERR, "fail to allocate memory"); + return; + } for (i = begin; i < next_begin; i++) { addr = &dev->data->mac_addrs[i]; if (is_zero_ether_addr(addr)) continue; - (void)rte_memcpy(list->list[j].addr, addr->addr_bytes, + rte_memcpy(list->list[j].addr, addr->addr_bytes, sizeof(addr->addr_bytes)); PMD_DRV_LOG(DEBUG, "add/rm mac:%x:%x:%x:%x:%x:%x", addr->addr_bytes[0], addr->addr_bytes[1], @@ -2024,17 +1965,23 @@ i40evf_add_del_all_mac_addr(struct rte_eth_dev *dev, bool add) } list->vsi_id = vf->vsi_res->vsi_id; list->num_elements = j; - args.ops = add ? I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS : - I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS; + args.ops = add ? VIRTCHNL_OP_ADD_ETH_ADDR : + VIRTCHNL_OP_DEL_ETH_ADDR; args.in_args = (uint8_t *)list; args.in_args_size = len; args.out_buffer = vf->aq_resp; args.out_size = I40E_AQ_BUF_SZ; err = i40evf_execute_vf_cmd(dev, &args); - if (err) + if (err) { PMD_DRV_LOG(ERR, "fail to execute command %s", add ? "OP_ADD_ETHER_ADDRESS" : "OP_DEL_ETHER_ADDRESS"); + } else { + if (add) + vf->vsi.mac_num++; + else + vf->vsi.mac_num--; + } rte_free(list); begin = next_begin; } while (begin < I40E_NUM_MACADDR_MAX); @@ -2045,7 +1992,8 @@ i40evf_dev_start(struct rte_eth_dev *dev) { struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); - struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; uint32_t intr_vector = 0; PMD_INIT_FUNC_TRACE(); @@ -2057,7 +2005,8 @@ i40evf_dev_start(struct rte_eth_dev *dev) dev->data->nb_tx_queues); /* check and configure queue intr-vector mapping */ - if (dev->data->dev_conf.intr_conf.rxq != 0) { + if (rte_intr_cap_multiple(intr_handle) && + dev->data->dev_conf.intr_conf.rxq) { intr_vector = dev->data->nb_rx_queues; if (rte_intr_efd_enable(intr_handle, intr_vector)) return -1; @@ -2069,7 +2018,7 @@ i40evf_dev_start(struct rte_eth_dev *dev) dev->data->nb_rx_queues * sizeof(int), 0); if (!intr_handle->intr_vec) { PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues" - " intr_vec\n", dev->data->nb_rx_queues); + " intr_vec", dev->data->nb_rx_queues); return -ENOMEM; } } @@ -2081,7 +2030,7 @@ i40evf_dev_start(struct rte_eth_dev *dev) i40evf_tx_init(dev); - if (i40evf_configure_queues(dev) != 0) { + if (i40evf_configure_vsi_queues(dev) != 0) { PMD_DRV_LOG(ERR, "configure queues failed"); goto err_queue; } @@ -2098,7 +2047,20 @@ i40evf_dev_start(struct rte_eth_dev *dev) goto err_mac; } + /* When a VF port is bound to VFIO-PCI, only miscellaneous interrupt + * is mapped to VFIO vector 0 in i40evf_dev_init( ). + * If previous VFIO interrupt mapping set in i40evf_dev_init( ) is + * not cleared, it will fail when rte_intr_enable( ) tries to map Rx + * queue interrupt to other VFIO vectors. + * So clear uio/vfio intr/evevnfd first to avoid failure. + */ + if (dev->data->dev_conf.intr_conf.rxq != 0) { + rte_intr_disable(intr_handle); + rte_intr_enable(intr_handle); + } + i40evf_enable_queues_intr(dev); + return 0; err_mac: @@ -2110,10 +2072,14 @@ err_queue: static void i40evf_dev_stop(struct rte_eth_dev *dev) { - struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; + struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); PMD_INIT_FUNC_TRACE(); + if (hw->adapter_stopped == 1) + return; i40evf_stop_queues(dev); i40evf_disable_queues_intr(dev); i40e_dev_clear_queues(dev); @@ -2126,6 +2092,7 @@ i40evf_dev_stop(struct rte_eth_dev *dev) } /* remove all mac addrs */ i40evf_add_del_all_mac_addr(dev, FALSE); + hw->adapter_stopped = 1; } @@ -2154,6 +2121,9 @@ i40evf_dev_link_update(struct rte_eth_dev *dev, case I40E_LINK_SPEED_20GB: new_link.link_speed = ETH_SPEED_NUM_20G; break; + case I40E_LINK_SPEED_25GB: + new_link.link_speed = ETH_SPEED_NUM_25G; + break; case I40E_LINK_SPEED_40GB: new_link.link_speed = ETH_SPEED_NUM_40G; break; @@ -2165,6 +2135,8 @@ i40evf_dev_link_update(struct rte_eth_dev *dev, new_link.link_duplex = ETH_LINK_FULL_DUPLEX; new_link.link_status = vf->link_up ? ETH_LINK_UP : ETH_LINK_DOWN; + new_link.link_autoneg = + dev->data->dev_conf.link_speeds & ETH_LINK_SPEED_FIXED; i40evf_dev_atomic_write_link_status(dev, &new_link); @@ -2237,13 +2209,14 @@ i40evf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); memset(dev_info, 0, sizeof(*dev_info)); + dev_info->pci_dev = RTE_ETH_DEV_TO_PCI(dev); dev_info->max_rx_queues = vf->vsi_res->num_queue_pairs; dev_info->max_tx_queues = vf->vsi_res->num_queue_pairs; dev_info->min_rx_bufsize = I40E_BUF_SIZE_MIN; dev_info->max_rx_pktlen = I40E_FRAME_SIZE_MAX; dev_info->hash_key_size = (I40E_VFQF_HKEY_MAX_INDEX + 1) * sizeof(uint32_t); dev_info->reta_size = ETH_RSS_RETA_SIZE_64; - dev_info->flow_type_rss_offloads = I40E_RSS_OFFLOAD_ALL; + dev_info->flow_type_rss_offloads = vf->adapter->flow_types_mask; dev_info->max_mac_addrs = I40E_NUM_MACADDR_MAX; dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP | @@ -2294,33 +2267,69 @@ i40evf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) }; } -static void +static int i40evf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) { - if (i40evf_get_statics(dev, stats)) - PMD_DRV_LOG(ERR, "Get statics failed"); + int ret; + struct i40e_eth_stats *pstats = NULL; + struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); + struct i40e_vsi *vsi = &vf->vsi; + + ret = i40evf_query_stats(dev, &pstats); + if (ret == 0) { + i40evf_update_stats(vsi, pstats); + + stats->ipackets = pstats->rx_unicast + pstats->rx_multicast + + pstats->rx_broadcast; + stats->opackets = pstats->tx_broadcast + pstats->tx_multicast + + pstats->tx_unicast; + stats->imissed = pstats->rx_discards; + stats->oerrors = pstats->tx_errors + pstats->tx_discards; + stats->ibytes = pstats->rx_bytes; + stats->obytes = pstats->tx_bytes; + } else { + PMD_DRV_LOG(ERR, "Get statistics failed"); + } + return ret; } static void i40evf_dev_close(struct rte_eth_dev *dev) { struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); - struct rte_pci_device *pci_dev = dev->pci_dev; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; i40evf_dev_stop(dev); - hw->adapter_stopped = 1; i40e_dev_free_queues(dev); i40evf_reset_vf(hw); i40e_shutdown_adminq(hw); /* disable uio intr before callback unregister */ - rte_intr_disable(&pci_dev->intr_handle); + rte_intr_disable(intr_handle); /* unregister callback func from eal lib */ - rte_intr_callback_unregister(&pci_dev->intr_handle, - i40evf_dev_interrupt_handler, (void *)dev); + rte_intr_callback_unregister(intr_handle, + i40evf_dev_interrupt_handler, dev); i40evf_disable_irq0(hw); } +/* + * Reset VF device only to re-initialize resources in PMD layer + */ +static int +i40evf_dev_reset(struct rte_eth_dev *dev) +{ + int ret; + + ret = i40evf_dev_uninit(dev); + if (ret) + return ret; + + ret = i40evf_dev_init(dev); + + return ret; +} + static int i40evf_get_rss_lut(struct i40e_vsi *vsi, uint8_t *lut, uint16_t lut_size) { @@ -2394,7 +2403,7 @@ i40evf_dev_rss_reta_update(struct rte_eth_dev *dev, if (reta_size != ETH_RSS_RETA_SIZE_64) { PMD_DRV_LOG(ERR, "The size of hash lookup table configured " "(%d) doesn't match the number of hardware can " - "support (%d)\n", reta_size, ETH_RSS_RETA_SIZE_64); + "support (%d)", reta_size, ETH_RSS_RETA_SIZE_64); return -EINVAL; } @@ -2433,7 +2442,7 @@ i40evf_dev_rss_reta_query(struct rte_eth_dev *dev, if (reta_size != ETH_RSS_RETA_SIZE_64) { PMD_DRV_LOG(ERR, "The size of hash lookup table configured " "(%d) doesn't match the number of hardware can " - "support (%d)\n", reta_size, ETH_RSS_RETA_SIZE_64); + "support (%d)", reta_size, ETH_RSS_RETA_SIZE_64); return -EINVAL; } @@ -2528,7 +2537,7 @@ static int i40evf_hw_rss_hash_set(struct i40e_vf *vf, struct rte_eth_rss_conf *rss_conf) { struct i40e_hw *hw = I40E_VF_TO_HW(vf); - uint64_t rss_hf, hena; + uint64_t hena; int ret; ret = i40evf_set_rss_key(&vf->vsi, rss_conf->rss_key, @@ -2536,11 +2545,7 @@ i40evf_hw_rss_hash_set(struct i40e_vf *vf, struct rte_eth_rss_conf *rss_conf) if (ret) return ret; - rss_hf = rss_conf->rss_hf; - hena = (uint64_t)i40e_read_rx_ctl(hw, I40E_VFQF_HENA(0)); - hena |= ((uint64_t)i40e_read_rx_ctl(hw, I40E_VFQF_HENA(1))) << 32; - hena &= ~I40E_RSS_HENA_ALL; - hena |= i40e_config_hena(rss_hf); + hena = i40e_config_hena(vf->adapter, rss_conf->rss_hf); i40e_write_rx_ctl(hw, I40E_VFQF_HENA(0), (uint32_t)hena); i40e_write_rx_ctl(hw, I40E_VFQF_HENA(1), (uint32_t)(hena >> 32)); I40EVF_WRITE_FLUSH(hw); @@ -2552,13 +2557,9 @@ static void i40evf_disable_rss(struct i40e_vf *vf) { struct i40e_hw *hw = I40E_VF_TO_HW(vf); - uint64_t hena; - hena = (uint64_t)i40e_read_rx_ctl(hw, I40E_VFQF_HENA(0)); - hena |= ((uint64_t)i40e_read_rx_ctl(hw, I40E_VFQF_HENA(1))) << 32; - hena &= ~I40E_RSS_HENA_ALL; - i40e_write_rx_ctl(hw, I40E_VFQF_HENA(0), (uint32_t)hena); - i40e_write_rx_ctl(hw, I40E_VFQF_HENA(1), (uint32_t)(hena >> 32)); + i40e_write_rx_ctl(hw, I40E_VFQF_HENA(0), 0); + i40e_write_rx_ctl(hw, I40E_VFQF_HENA(1), 0); I40EVF_WRITE_FLUSH(hw); } @@ -2572,7 +2573,7 @@ i40evf_config_rss(struct i40e_vf *vf) if (vf->dev_data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_RSS) { i40evf_disable_rss(vf); - PMD_DRV_LOG(DEBUG, "RSS not configured\n"); + PMD_DRV_LOG(DEBUG, "RSS not configured"); return 0; } @@ -2587,9 +2588,9 @@ i40evf_config_rss(struct i40e_vf *vf) } rss_conf = vf->dev_data->dev_conf.rx_adv_conf.rss_conf; - if ((rss_conf.rss_hf & I40E_RSS_OFFLOAD_ALL) == 0) { + if ((rss_conf.rss_hf & vf->adapter->flow_types_mask) == 0) { i40evf_disable_rss(vf); - PMD_DRV_LOG(DEBUG, "No hash flag is set\n"); + PMD_DRV_LOG(DEBUG, "No hash flag is set"); return 0; } @@ -2612,12 +2613,13 @@ i40evf_dev_rss_hash_update(struct rte_eth_dev *dev, { struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); - uint64_t rss_hf = rss_conf->rss_hf & I40E_RSS_OFFLOAD_ALL; + uint64_t rss_hf = rss_conf->rss_hf & vf->adapter->flow_types_mask; uint64_t hena; hena = (uint64_t)i40e_read_rx_ctl(hw, I40E_VFQF_HENA(0)); hena |= ((uint64_t)i40e_read_rx_ctl(hw, I40E_VFQF_HENA(1))) << 32; - if (!(hena & I40E_RSS_HENA_ALL)) { /* RSS disabled */ + + if (!(hena & vf->adapter->pctypes_mask)) { /* RSS disabled */ if (rss_hf != 0) /* Enable RSS */ return -EINVAL; return 0; @@ -2643,7 +2645,58 @@ i40evf_dev_rss_hash_conf_get(struct rte_eth_dev *dev, hena = (uint64_t)i40e_read_rx_ctl(hw, I40E_VFQF_HENA(0)); hena |= ((uint64_t)i40e_read_rx_ctl(hw, I40E_VFQF_HENA(1))) << 32; - rss_conf->rss_hf = i40e_parse_hena(hena); + rss_conf->rss_hf = i40e_parse_hena(vf->adapter, hena); return 0; } + +static int +i40evf_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) +{ + struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); + struct rte_eth_dev_data *dev_data = vf->dev_data; + uint32_t frame_size = mtu + I40E_ETH_OVERHEAD; + int ret = 0; + + /* check if mtu is within the allowed range */ + if ((mtu < ETHER_MIN_MTU) || (frame_size > I40E_FRAME_SIZE_MAX)) + return -EINVAL; + + /* mtu setting is forbidden if port is start */ + if (dev_data->dev_started) { + PMD_DRV_LOG(ERR, "port %d must be stopped before configuration", + dev_data->port_id); + return -EBUSY; + } + + if (frame_size > ETHER_MAX_LEN) + dev_data->dev_conf.rxmode.jumbo_frame = 1; + else + dev_data->dev_conf.rxmode.jumbo_frame = 0; + + dev_data->dev_conf.rxmode.max_rx_pkt_len = frame_size; + + return ret; +} + +static void +i40evf_set_default_mac_addr(struct rte_eth_dev *dev, + struct ether_addr *mac_addr) +{ + struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); + struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); + + if (!is_valid_assigned_ether_addr(mac_addr)) { + PMD_DRV_LOG(ERR, "Tried to set invalid MAC address."); + return; + } + + if (vf->flags & I40E_FLAG_VF_MAC_BY_PF) + return; + + i40evf_del_mac_addr_by_addr(dev, (struct ether_addr *)hw->mac.addr); + + i40evf_add_mac_addr(dev, mac_addr, 0, 0); + + ether_addr_copy(mac_addr, (struct ether_addr *)hw->mac.addr); +} diff --git a/dpdk/drivers/net/i40e/i40e_fdir.c b/dpdk/drivers/net/i40e/i40e_fdir.c index f65c4110..6802995f 100644 --- a/dpdk/drivers/net/i40e/i40e_fdir.c +++ b/dpdk/drivers/net/i40e/i40e_fdir.c @@ -49,6 +49,7 @@ #include #include #include +#include #include "i40e_logs.h" #include "base/i40e_type.h" @@ -67,15 +68,23 @@ #define I40E_FDIR_IP_DEFAULT_VERSION_IHL 0x45 #define I40E_FDIR_TCP_DEFAULT_DATAOFF 0x50 #define I40E_FDIR_IPv6_DEFAULT_VTC_FLOW 0x60000000 -#define I40E_FDIR_IPv6_TC_OFFSET 20 #define I40E_FDIR_IPv6_DEFAULT_HOP_LIMITS 0xFF #define I40E_FDIR_IPv6_PAYLOAD_LEN 380 #define I40E_FDIR_UDP_DEFAULT_LEN 400 +#define I40E_FDIR_GTP_DEFAULT_LEN 384 +#define I40E_FDIR_INNER_IP_DEFAULT_LEN 384 +#define I40E_FDIR_INNER_IPV6_DEFAULT_LEN 344 -/* Wait count and interval for fdir filter programming */ -#define I40E_FDIR_WAIT_COUNT 10 -#define I40E_FDIR_WAIT_INTERVAL_US 1000 +#define I40E_FDIR_GTPC_DST_PORT 2123 +#define I40E_FDIR_GTPU_DST_PORT 2152 +#define I40E_FDIR_GTP_VER_FLAG_0X30 0x30 +#define I40E_FDIR_GTP_VER_FLAG_0X32 0x32 +#define I40E_FDIR_GTP_MSG_TYPE_0X01 0x01 +#define I40E_FDIR_GTP_MSG_TYPE_0XFF 0xFF + +/* Wait time for fdir filter programming */ +#define I40E_FDIR_MAX_WAIT_US 10000 /* Wait count and interval for fdir filter flush */ #define I40E_FDIR_FLUSH_RETRY 50 @@ -84,21 +93,6 @@ #define I40E_COUNTER_PF 2 /* Statistic counter index for one pf */ #define I40E_COUNTER_INDEX_FDIR(pf_id) (0 + (pf_id) * I40E_COUNTER_PF) -#define I40E_MAX_FLX_SOURCE_OFF 480 -#define I40E_FLX_OFFSET_IN_FIELD_VECTOR 50 - -#define NONUSE_FLX_PIT_DEST_OFF 63 -#define NONUSE_FLX_PIT_FSIZE 1 -#define MK_FLX_PIT(src_offset, fsize, dst_offset) ( \ - (((src_offset) << I40E_PRTQF_FLX_PIT_SOURCE_OFF_SHIFT) & \ - I40E_PRTQF_FLX_PIT_SOURCE_OFF_MASK) | \ - (((fsize) << I40E_PRTQF_FLX_PIT_FSIZE_SHIFT) & \ - I40E_PRTQF_FLX_PIT_FSIZE_MASK) | \ - ((((dst_offset) == NONUSE_FLX_PIT_DEST_OFF ? \ - NONUSE_FLX_PIT_DEST_OFF : \ - ((dst_offset) + I40E_FLX_OFFSET_IN_FIELD_VECTOR)) << \ - I40E_PRTQF_FLX_PIT_DEST_OFF_SHIFT) & \ - I40E_PRTQF_FLX_PIT_DEST_OFF_MASK)) #define I40E_FDIR_FLOWS ( \ (1 << RTE_ETH_FLOW_FRAG_IPV4) | \ @@ -113,13 +107,22 @@ (1 << RTE_ETH_FLOW_NONFRAG_IPV6_OTHER) | \ (1 << RTE_ETH_FLOW_L2_PAYLOAD)) -#define I40E_FLEX_WORD_MASK(off) (0x80 >> (off)) - static int i40e_fdir_filter_programming(struct i40e_pf *pf, enum i40e_filter_pctype pctype, const struct rte_eth_fdir_filter *filter, bool add); -static int i40e_fdir_flush(struct rte_eth_dev *dev); +static int i40e_fdir_filter_convert(const struct i40e_fdir_filter_conf *input, + struct i40e_fdir_filter *filter); +static struct i40e_fdir_filter * +i40e_sw_fdir_filter_lookup(struct i40e_fdir_info *fdir_info, + const struct i40e_fdir_input *input); +static int i40e_sw_fdir_filter_insert(struct i40e_pf *pf, + struct i40e_fdir_filter *filter); +static int +i40e_flow_fdir_filter_programming(struct i40e_pf *pf, + enum i40e_filter_pctype pctype, + const struct i40e_fdir_filter_conf *filter, + bool add); static int i40e_fdir_rx_queue_init(struct i40e_rx_queue *rxq) @@ -165,7 +168,6 @@ i40e_fdir_rx_queue_init(struct i40e_rx_queue *rxq) rte_wmb(); /* Init the RX tail regieter. */ - I40E_PCI_REG_WRITE(rxq->qrx_tail, 0); I40E_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1); return err; @@ -251,7 +253,7 @@ i40e_fdir_setup(struct i40e_pf *pf) /* reserve memory for the fdir programming packet */ snprintf(z_name, sizeof(z_name), "%s_%s_%d", - eth_dev->driver->pci_drv.name, + eth_dev->device->driver->name, I40E_FDIR_MZ_NAME, eth_dev->data->port_id); mz = i40e_memzone_reserve(z_name, I40E_FDIR_PKT_LEN, SOCKET_ID_ANY); @@ -262,7 +264,7 @@ i40e_fdir_setup(struct i40e_pf *pf) goto fail_mem; } pf->fdir.prg_pkt = mz->addr; - pf->fdir.dma_addr = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr); + pf->fdir.dma_addr = mz->iova; pf->fdir.match_counter_index = I40E_COUNTER_INDEX_FDIR(hw->pf_id); PMD_DRV_LOG(INFO, "FDIR setup successfully, with programming queue %u.", @@ -294,8 +296,12 @@ i40e_fdir_teardown(struct i40e_pf *pf) vsi = pf->fdir.fdir_vsi; if (!vsi) return; - i40e_switch_tx_queue(hw, vsi->base_queue, FALSE); - i40e_switch_rx_queue(hw, vsi->base_queue, FALSE); + int err = i40e_switch_tx_queue(hw, vsi->base_queue, FALSE); + if (err) + PMD_DRV_LOG(DEBUG, "Failed to do FDIR TX switch off"); + err = i40e_switch_rx_queue(hw, vsi->base_queue, FALSE); + if (err) + PMD_DRV_LOG(DEBUG, "Failed to do FDIR RX switch off"); i40e_dev_rx_queue_release(pf->fdir.rxq); pf->fdir.rxq = NULL; i40e_dev_tx_queue_release(pf->fdir.txq); @@ -332,6 +338,7 @@ i40e_init_flx_pld(struct i40e_pf *pf) struct i40e_hw *hw = I40E_PF_TO_HW(pf); uint8_t pctype; int i, index; + uint16_t flow_type; /* * Define the bytes stream extracted as flexible payload in @@ -353,7 +360,9 @@ i40e_init_flx_pld(struct i40e_pf *pf) /* initialize the masks */ for (pctype = I40E_FILTER_PCTYPE_NONF_IPV4_UDP; pctype <= I40E_FILTER_PCTYPE_L2_PAYLOAD; pctype++) { - if (!I40E_VALID_PCTYPE((enum i40e_filter_pctype)pctype)) + flow_type = i40e_pctype_to_flowtype(pf->adapter, pctype); + + if (flow_type == RTE_ETH_FLOW_UNKNOWN) continue; pf->fdir.flex_mask[pctype].word_mask = 0; i40e_write_rx_ctl(hw, I40E_PRTQF_FD_FLXINSET(pctype), 0); @@ -365,8 +374,6 @@ i40e_init_flx_pld(struct i40e_pf *pf) } } -#define I40E_WORD(hi, lo) (uint16_t)((((hi) << 8) & 0xFF00) | ((lo) & 0xFF)) - #define I40E_VALIDATE_FLEX_PIT(flex_pit1, flex_pit2) do { \ if ((flex_pit2).src_offset < \ (flex_pit1).src_offset + (flex_pit1).size) { \ @@ -453,7 +460,8 @@ i40e_check_fdir_flex_payload(const struct rte_eth_flex_payload_cfg *flex_cfg) * arguments are valid */ static int -i40e_check_fdir_flex_conf(const struct rte_eth_fdir_flex_conf *conf) +i40e_check_fdir_flex_conf(const struct i40e_adapter *adapter, + const struct rte_eth_fdir_flex_conf *conf) { const struct rte_eth_flex_payload_cfg *flex_cfg; const struct rte_eth_fdir_flex_mask *flex_mask; @@ -461,6 +469,7 @@ i40e_check_fdir_flex_conf(const struct rte_eth_fdir_flex_conf *conf) uint8_t nb_bitmask; uint16_t i, j; int ret = 0; + enum i40e_filter_pctype pctype; if (conf == NULL) { PMD_DRV_LOG(INFO, "NULL pointer."); @@ -491,7 +500,8 @@ i40e_check_fdir_flex_conf(const struct rte_eth_fdir_flex_conf *conf) } for (i = 0; i < conf->nb_flexmasks; i++) { flex_mask = &conf->flex_mask[i]; - if (!I40E_VALID_FLOW(flex_mask->flow_type)) { + pctype = i40e_flowtype_to_pctype(adapter, flex_mask->flow_type); + if (pctype == I40E_FILTER_PCTYPE_INVALID) { PMD_DRV_LOG(WARNING, "invalid flow type."); return -EINVAL; } @@ -654,18 +664,36 @@ i40e_fdir_configure(struct rte_eth_dev *dev) i40e_init_flx_pld(pf); /* set flex config to default value */ conf = &dev->data->dev_conf.fdir_conf.flex_conf; - ret = i40e_check_fdir_flex_conf(conf); + ret = i40e_check_fdir_flex_conf(pf->adapter, conf); if (ret < 0) { PMD_DRV_LOG(ERR, " invalid configuration arguments."); return -EINVAL; } - /* configure flex payload */ - for (i = 0; i < conf->nb_payloads; i++) - i40e_set_flx_pld_cfg(pf, &conf->flex_set[i]); - /* configure flex mask*/ - for (i = 0; i < conf->nb_flexmasks; i++) { - pctype = i40e_flowtype_to_pctype(conf->flex_mask[i].flow_type); - i40e_set_flex_mask_on_pctype(pf, pctype, &conf->flex_mask[i]); + + if (!pf->support_multi_driver) { + /* configure flex payload */ + for (i = 0; i < conf->nb_payloads; i++) + i40e_set_flx_pld_cfg(pf, &conf->flex_set[i]); + /* configure flex mask*/ + for (i = 0; i < conf->nb_flexmasks; i++) { + if (hw->mac.type == I40E_MAC_X722) { + /* get pctype value in fd pctype register */ + pctype = (enum i40e_filter_pctype) + i40e_read_rx_ctl(hw, + I40E_GLQF_FD_PCTYPES( + (int)i40e_flowtype_to_pctype( + pf->adapter, + conf->flex_mask[i].flow_type))); + } else { + pctype = i40e_flowtype_to_pctype(pf->adapter, + conf->flex_mask[i].flow_type); + } + + i40e_set_flex_mask_on_pctype(pf, pctype, + &conf->flex_mask[i]); + } + } else { + PMD_DRV_LOG(ERR, "Not support flexible payload."); } return ret; @@ -921,6 +949,358 @@ i40e_fdir_construct_pkt(struct i40e_pf *pf, dst = pf->fdir.flex_set[pit_idx].dst_offset * sizeof(uint16_t); ptr = payload + pf->fdir.flex_set[pit_idx].src_offset * sizeof(uint16_t); + rte_memcpy(ptr, + &fdir_input->flow_ext.flexbytes[dst], + size * sizeof(uint16_t)); + } + + return 0; +} + +static struct i40e_customized_pctype * +i40e_flow_fdir_find_customized_pctype(struct i40e_pf *pf, uint8_t pctype) +{ + struct i40e_customized_pctype *cus_pctype; + enum i40e_new_pctype i = I40E_CUSTOMIZED_GTPC; + + for (; i < I40E_CUSTOMIZED_MAX; i++) { + cus_pctype = &pf->customized_pctype[i]; + if (pctype == cus_pctype->pctype) + return cus_pctype; + } + return NULL; +} + +static inline int +i40e_flow_fdir_fill_eth_ip_head(struct i40e_pf *pf, + const struct i40e_fdir_input *fdir_input, + unsigned char *raw_pkt, + bool vlan) +{ + struct i40e_customized_pctype *cus_pctype = NULL; + static uint8_t vlan_frame[] = {0x81, 0, 0, 0}; + uint16_t *ether_type; + uint8_t len = 2 * sizeof(struct ether_addr); + struct ipv4_hdr *ip; + struct ipv6_hdr *ip6; + uint8_t pctype = fdir_input->pctype; + bool is_customized_pctype = fdir_input->flow_ext.customized_pctype; + static const uint8_t next_proto[] = { + [I40E_FILTER_PCTYPE_FRAG_IPV4] = IPPROTO_IP, + [I40E_FILTER_PCTYPE_NONF_IPV4_TCP] = IPPROTO_TCP, + [I40E_FILTER_PCTYPE_NONF_IPV4_UDP] = IPPROTO_UDP, + [I40E_FILTER_PCTYPE_NONF_IPV4_SCTP] = IPPROTO_SCTP, + [I40E_FILTER_PCTYPE_NONF_IPV4_OTHER] = IPPROTO_IP, + [I40E_FILTER_PCTYPE_FRAG_IPV6] = IPPROTO_NONE, + [I40E_FILTER_PCTYPE_NONF_IPV6_TCP] = IPPROTO_TCP, + [I40E_FILTER_PCTYPE_NONF_IPV6_UDP] = IPPROTO_UDP, + [I40E_FILTER_PCTYPE_NONF_IPV6_SCTP] = IPPROTO_SCTP, + [I40E_FILTER_PCTYPE_NONF_IPV6_OTHER] = IPPROTO_NONE, + }; + + raw_pkt += 2 * sizeof(struct ether_addr); + if (vlan && fdir_input->flow_ext.vlan_tci) { + rte_memcpy(raw_pkt, vlan_frame, sizeof(vlan_frame)); + rte_memcpy(raw_pkt + sizeof(uint16_t), + &fdir_input->flow_ext.vlan_tci, + sizeof(uint16_t)); + raw_pkt += sizeof(vlan_frame); + len += sizeof(vlan_frame); + } + ether_type = (uint16_t *)raw_pkt; + raw_pkt += sizeof(uint16_t); + len += sizeof(uint16_t); + + if (is_customized_pctype) { + cus_pctype = i40e_flow_fdir_find_customized_pctype(pf, pctype); + if (!cus_pctype) { + PMD_DRV_LOG(ERR, "unknown pctype %u.", + fdir_input->pctype); + return -1; + } + } + + if (pctype == I40E_FILTER_PCTYPE_L2_PAYLOAD) + *ether_type = fdir_input->flow.l2_flow.ether_type; + else if (pctype == I40E_FILTER_PCTYPE_NONF_IPV4_TCP || + pctype == I40E_FILTER_PCTYPE_NONF_IPV4_UDP || + pctype == I40E_FILTER_PCTYPE_NONF_IPV4_SCTP || + pctype == I40E_FILTER_PCTYPE_NONF_IPV4_OTHER || + pctype == I40E_FILTER_PCTYPE_FRAG_IPV4 || + is_customized_pctype) { + ip = (struct ipv4_hdr *)raw_pkt; + + *ether_type = rte_cpu_to_be_16(ETHER_TYPE_IPv4); + ip->version_ihl = I40E_FDIR_IP_DEFAULT_VERSION_IHL; + /* set len to by default */ + ip->total_length = rte_cpu_to_be_16(I40E_FDIR_IP_DEFAULT_LEN); + ip->time_to_live = fdir_input->flow.ip4_flow.ttl ? + fdir_input->flow.ip4_flow.ttl : + I40E_FDIR_IP_DEFAULT_TTL; + ip->type_of_service = fdir_input->flow.ip4_flow.tos; + /** + * The source and destination fields in the transmitted packet + * need to be presented in a reversed order with respect + * to the expected received packets. + */ + ip->src_addr = fdir_input->flow.ip4_flow.dst_ip; + ip->dst_addr = fdir_input->flow.ip4_flow.src_ip; + + if (!is_customized_pctype) + ip->next_proto_id = fdir_input->flow.ip4_flow.proto ? + fdir_input->flow.ip4_flow.proto : + next_proto[fdir_input->pctype]; + else if (cus_pctype->index == I40E_CUSTOMIZED_GTPC || + cus_pctype->index == I40E_CUSTOMIZED_GTPU_IPV4 || + cus_pctype->index == I40E_CUSTOMIZED_GTPU_IPV6 || + cus_pctype->index == I40E_CUSTOMIZED_GTPU) + ip->next_proto_id = IPPROTO_UDP; + len += sizeof(struct ipv4_hdr); + } else if (pctype == I40E_FILTER_PCTYPE_NONF_IPV6_TCP || + pctype == I40E_FILTER_PCTYPE_NONF_IPV6_UDP || + pctype == I40E_FILTER_PCTYPE_NONF_IPV6_SCTP || + pctype == I40E_FILTER_PCTYPE_NONF_IPV6_OTHER || + pctype == I40E_FILTER_PCTYPE_FRAG_IPV6) { + ip6 = (struct ipv6_hdr *)raw_pkt; + + *ether_type = rte_cpu_to_be_16(ETHER_TYPE_IPv6); + ip6->vtc_flow = + rte_cpu_to_be_32(I40E_FDIR_IPv6_DEFAULT_VTC_FLOW | + (fdir_input->flow.ipv6_flow.tc << + I40E_FDIR_IPv6_TC_OFFSET)); + ip6->payload_len = + rte_cpu_to_be_16(I40E_FDIR_IPv6_PAYLOAD_LEN); + ip6->proto = fdir_input->flow.ipv6_flow.proto ? + fdir_input->flow.ipv6_flow.proto : + next_proto[fdir_input->pctype]; + ip6->hop_limits = fdir_input->flow.ipv6_flow.hop_limits ? + fdir_input->flow.ipv6_flow.hop_limits : + I40E_FDIR_IPv6_DEFAULT_HOP_LIMITS; + /** + * The source and destination fields in the transmitted packet + * need to be presented in a reversed order with respect + * to the expected received packets. + */ + rte_memcpy(&ip6->src_addr, + &fdir_input->flow.ipv6_flow.dst_ip, + IPV6_ADDR_LEN); + rte_memcpy(&ip6->dst_addr, + &fdir_input->flow.ipv6_flow.src_ip, + IPV6_ADDR_LEN); + len += sizeof(struct ipv6_hdr); + } else { + PMD_DRV_LOG(ERR, "unknown pctype %u.", + fdir_input->pctype); + return -1; + } + + return len; +} + +/** + * i40e_flow_fdir_construct_pkt - construct packet based on fields in input + * @pf: board private structure + * @fdir_input: input set of the flow director entry + * @raw_pkt: a packet to be constructed + */ +static int +i40e_flow_fdir_construct_pkt(struct i40e_pf *pf, + const struct i40e_fdir_input *fdir_input, + unsigned char *raw_pkt) +{ + unsigned char *payload = NULL; + unsigned char *ptr; + struct udp_hdr *udp; + struct tcp_hdr *tcp; + struct sctp_hdr *sctp; + struct rte_flow_item_gtp *gtp; + struct ipv4_hdr *gtp_ipv4; + struct ipv6_hdr *gtp_ipv6; + uint8_t size, dst = 0; + uint8_t i, pit_idx, set_idx = I40E_FLXPLD_L4_IDX; /* use l4 by default*/ + int len; + uint8_t pctype = fdir_input->pctype; + struct i40e_customized_pctype *cus_pctype; + + /* raw pcket template - just copy contents of the raw packet */ + if (fdir_input->flow_ext.pkt_template) { + memcpy(raw_pkt, fdir_input->flow.raw_flow.packet, + fdir_input->flow.raw_flow.length); + return 0; + } + + /* fill the ethernet and IP head */ + len = i40e_flow_fdir_fill_eth_ip_head(pf, fdir_input, raw_pkt, + !!fdir_input->flow_ext.vlan_tci); + if (len < 0) + return -EINVAL; + + /* fill the L4 head */ + if (pctype == I40E_FILTER_PCTYPE_NONF_IPV4_UDP) { + udp = (struct udp_hdr *)(raw_pkt + len); + payload = (unsigned char *)udp + sizeof(struct udp_hdr); + /** + * The source and destination fields in the transmitted packet + * need to be presented in a reversed order with respect + * to the expected received packets. + */ + udp->src_port = fdir_input->flow.udp4_flow.dst_port; + udp->dst_port = fdir_input->flow.udp4_flow.src_port; + udp->dgram_len = rte_cpu_to_be_16(I40E_FDIR_UDP_DEFAULT_LEN); + } else if (pctype == I40E_FILTER_PCTYPE_NONF_IPV4_TCP) { + tcp = (struct tcp_hdr *)(raw_pkt + len); + payload = (unsigned char *)tcp + sizeof(struct tcp_hdr); + /** + * The source and destination fields in the transmitted packet + * need to be presented in a reversed order with respect + * to the expected received packets. + */ + tcp->src_port = fdir_input->flow.tcp4_flow.dst_port; + tcp->dst_port = fdir_input->flow.tcp4_flow.src_port; + tcp->data_off = I40E_FDIR_TCP_DEFAULT_DATAOFF; + } else if (pctype == I40E_FILTER_PCTYPE_NONF_IPV4_SCTP) { + sctp = (struct sctp_hdr *)(raw_pkt + len); + payload = (unsigned char *)sctp + sizeof(struct sctp_hdr); + /** + * The source and destination fields in the transmitted packet + * need to be presented in a reversed order with respect + * to the expected received packets. + */ + sctp->src_port = fdir_input->flow.sctp4_flow.dst_port; + sctp->dst_port = fdir_input->flow.sctp4_flow.src_port; + sctp->tag = fdir_input->flow.sctp4_flow.verify_tag; + } else if (pctype == I40E_FILTER_PCTYPE_NONF_IPV4_OTHER || + pctype == I40E_FILTER_PCTYPE_FRAG_IPV4) { + payload = raw_pkt + len; + set_idx = I40E_FLXPLD_L3_IDX; + } else if (pctype == I40E_FILTER_PCTYPE_NONF_IPV6_UDP) { + udp = (struct udp_hdr *)(raw_pkt + len); + payload = (unsigned char *)udp + sizeof(struct udp_hdr); + /** + * The source and destination fields in the transmitted packet + * need to be presented in a reversed order with respect + * to the expected received packets. + */ + udp->src_port = fdir_input->flow.udp6_flow.dst_port; + udp->dst_port = fdir_input->flow.udp6_flow.src_port; + udp->dgram_len = rte_cpu_to_be_16(I40E_FDIR_IPv6_PAYLOAD_LEN); + } else if (pctype == I40E_FILTER_PCTYPE_NONF_IPV6_TCP) { + tcp = (struct tcp_hdr *)(raw_pkt + len); + payload = (unsigned char *)tcp + sizeof(struct tcp_hdr); + /** + * The source and destination fields in the transmitted packet + * need to be presented in a reversed order with respect + * to the expected received packets. + */ + tcp->data_off = I40E_FDIR_TCP_DEFAULT_DATAOFF; + tcp->src_port = fdir_input->flow.udp6_flow.dst_port; + tcp->dst_port = fdir_input->flow.udp6_flow.src_port; + } else if (pctype == I40E_FILTER_PCTYPE_NONF_IPV6_SCTP) { + sctp = (struct sctp_hdr *)(raw_pkt + len); + payload = (unsigned char *)sctp + sizeof(struct sctp_hdr); + /** + * The source and destination fields in the transmitted packet + * need to be presented in a reversed order with respect + * to the expected received packets. + */ + sctp->src_port = fdir_input->flow.sctp6_flow.dst_port; + sctp->dst_port = fdir_input->flow.sctp6_flow.src_port; + sctp->tag = fdir_input->flow.sctp6_flow.verify_tag; + } else if (pctype == I40E_FILTER_PCTYPE_NONF_IPV6_OTHER || + pctype == I40E_FILTER_PCTYPE_FRAG_IPV6) { + payload = raw_pkt + len; + set_idx = I40E_FLXPLD_L3_IDX; + } else if (pctype == I40E_FILTER_PCTYPE_L2_PAYLOAD) { + payload = raw_pkt + len; + /** + * ARP packet is a special case on which the payload + * starts after the whole ARP header + */ + if (fdir_input->flow.l2_flow.ether_type == + rte_cpu_to_be_16(ETHER_TYPE_ARP)) + payload += sizeof(struct arp_hdr); + set_idx = I40E_FLXPLD_L2_IDX; + } else if (fdir_input->flow_ext.customized_pctype) { + /* If customized pctype is used */ + cus_pctype = i40e_flow_fdir_find_customized_pctype(pf, pctype); + if (cus_pctype->index == I40E_CUSTOMIZED_GTPC || + cus_pctype->index == I40E_CUSTOMIZED_GTPU_IPV4 || + cus_pctype->index == I40E_CUSTOMIZED_GTPU_IPV6 || + cus_pctype->index == I40E_CUSTOMIZED_GTPU) { + udp = (struct udp_hdr *)(raw_pkt + len); + udp->dgram_len = + rte_cpu_to_be_16(I40E_FDIR_UDP_DEFAULT_LEN); + + gtp = (struct rte_flow_item_gtp *) + ((unsigned char *)udp + sizeof(struct udp_hdr)); + gtp->msg_len = + rte_cpu_to_be_16(I40E_FDIR_GTP_DEFAULT_LEN); + gtp->teid = fdir_input->flow.gtp_flow.teid; + gtp->msg_type = I40E_FDIR_GTP_MSG_TYPE_0X01; + + /* GTP-C message type is not supported. */ + if (cus_pctype->index == I40E_CUSTOMIZED_GTPC) { + udp->dst_port = + rte_cpu_to_be_16(I40E_FDIR_GTPC_DST_PORT); + gtp->v_pt_rsv_flags = + I40E_FDIR_GTP_VER_FLAG_0X32; + } else { + udp->dst_port = + rte_cpu_to_be_16(I40E_FDIR_GTPU_DST_PORT); + gtp->v_pt_rsv_flags = + I40E_FDIR_GTP_VER_FLAG_0X30; + } + + if (cus_pctype->index == I40E_CUSTOMIZED_GTPU_IPV4) { + gtp->msg_type = I40E_FDIR_GTP_MSG_TYPE_0XFF; + gtp_ipv4 = (struct ipv4_hdr *) + ((unsigned char *)gtp + + sizeof(struct rte_flow_item_gtp)); + gtp_ipv4->version_ihl = + I40E_FDIR_IP_DEFAULT_VERSION_IHL; + gtp_ipv4->next_proto_id = IPPROTO_IP; + gtp_ipv4->total_length = + rte_cpu_to_be_16( + I40E_FDIR_INNER_IP_DEFAULT_LEN); + payload = (unsigned char *)gtp_ipv4 + + sizeof(struct ipv4_hdr); + } else if (cus_pctype->index == + I40E_CUSTOMIZED_GTPU_IPV6) { + gtp->msg_type = I40E_FDIR_GTP_MSG_TYPE_0XFF; + gtp_ipv6 = (struct ipv6_hdr *) + ((unsigned char *)gtp + + sizeof(struct rte_flow_item_gtp)); + gtp_ipv6->vtc_flow = + rte_cpu_to_be_32( + I40E_FDIR_IPv6_DEFAULT_VTC_FLOW | + (0 << I40E_FDIR_IPv6_TC_OFFSET)); + gtp_ipv6->proto = IPPROTO_NONE; + gtp_ipv6->payload_len = + rte_cpu_to_be_16( + I40E_FDIR_INNER_IPV6_DEFAULT_LEN); + gtp_ipv6->hop_limits = + I40E_FDIR_IPv6_DEFAULT_HOP_LIMITS; + payload = (unsigned char *)gtp_ipv6 + + sizeof(struct ipv6_hdr); + } else + payload = (unsigned char *)gtp + + sizeof(struct rte_flow_item_gtp); + } + } else { + PMD_DRV_LOG(ERR, "unknown pctype %u.", + fdir_input->pctype); + return -1; + } + + /* fill the flexbytes to payload */ + for (i = 0; i < I40E_MAX_FLXPLD_FIED; i++) { + pit_idx = set_idx * I40E_MAX_FLXPLD_FIED + i; + size = pf->fdir.flex_set[pit_idx].size; + if (size == 0) + continue; + dst = pf->fdir.flex_set[pit_idx].dst_offset * sizeof(uint16_t); + ptr = payload + + pf->fdir.flex_set[pit_idx].src_offset * sizeof(uint16_t); (void)rte_memcpy(ptr, &fdir_input->flow_ext.flexbytes[dst], size * sizeof(uint16_t)); @@ -991,27 +1371,124 @@ i40e_check_fdir_programming_status(struct i40e_rx_queue *rxq) PMD_DRV_LOG(ERR, "invalid programming status" " reported, error = %u.", error); } else - PMD_DRV_LOG(ERR, "unknown programming status" + PMD_DRV_LOG(INFO, "unknown programming status" " reported, len = %d, id = %u.", len, id); rxdp->wb.qword1.status_error_len = 0; rxq->rx_tail++; if (unlikely(rxq->rx_tail == rxq->nb_rx_desc)) rxq->rx_tail = 0; + if (rxq->rx_tail == 0) + I40E_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1); + else + I40E_PCI_REG_WRITE(rxq->qrx_tail, rxq->rx_tail - 1); } + return ret; } +static int +i40e_fdir_filter_convert(const struct i40e_fdir_filter_conf *input, + struct i40e_fdir_filter *filter) +{ + rte_memcpy(&filter->fdir, input, sizeof(struct i40e_fdir_filter_conf)); + if (input->input.flow_ext.pkt_template) { + filter->fdir.input.flow.raw_flow.packet = NULL; + filter->fdir.input.flow.raw_flow.length = + rte_hash_crc(input->input.flow.raw_flow.packet, + input->input.flow.raw_flow.length, + input->input.flow.raw_flow.pctype); + } + return 0; +} + +/* Check if there exists the flow director filter */ +static struct i40e_fdir_filter * +i40e_sw_fdir_filter_lookup(struct i40e_fdir_info *fdir_info, + const struct i40e_fdir_input *input) +{ + int ret; + + if (input->flow_ext.pkt_template) + ret = rte_hash_lookup_with_hash(fdir_info->hash_table, + (const void *)input, + input->flow.raw_flow.length); + else + ret = rte_hash_lookup(fdir_info->hash_table, + (const void *)input); + if (ret < 0) + return NULL; + + return fdir_info->hash_map[ret]; +} + +/* Add a flow director filter into the SW list */ +static int +i40e_sw_fdir_filter_insert(struct i40e_pf *pf, struct i40e_fdir_filter *filter) +{ + struct i40e_fdir_info *fdir_info = &pf->fdir; + int ret; + + if (filter->fdir.input.flow_ext.pkt_template) + ret = rte_hash_add_key_with_hash(fdir_info->hash_table, + &filter->fdir.input, + filter->fdir.input.flow.raw_flow.length); + else + ret = rte_hash_add_key(fdir_info->hash_table, + &filter->fdir.input); + if (ret < 0) { + PMD_DRV_LOG(ERR, + "Failed to insert fdir filter to hash table %d!", + ret); + return ret; + } + fdir_info->hash_map[ret] = filter; + + TAILQ_INSERT_TAIL(&fdir_info->fdir_list, filter, rules); + + return 0; +} + +/* Delete a flow director filter from the SW list */ +int +i40e_sw_fdir_filter_del(struct i40e_pf *pf, struct i40e_fdir_input *input) +{ + struct i40e_fdir_info *fdir_info = &pf->fdir; + struct i40e_fdir_filter *filter; + int ret; + + if (input->flow_ext.pkt_template) + ret = rte_hash_del_key_with_hash(fdir_info->hash_table, + input, + input->flow.raw_flow.length); + else + ret = rte_hash_del_key(fdir_info->hash_table, input); + if (ret < 0) { + PMD_DRV_LOG(ERR, + "Failed to delete fdir filter to hash table %d!", + ret); + return ret; + } + filter = fdir_info->hash_map[ret]; + fdir_info->hash_map[ret] = NULL; + + TAILQ_REMOVE(&fdir_info->fdir_list, filter, rules); + rte_free(filter); + + return 0; +} + /* * i40e_add_del_fdir_filter - add or remove a flow director filter. * @pf: board private structure * @filter: fdir filter entry * @add: 0 - delete, 1 - add */ -static int +int i40e_add_del_fdir_filter(struct rte_eth_dev *dev, - const struct rte_eth_fdir_filter *filter, - bool add) + const struct rte_eth_fdir_filter *filter, + bool add) { + struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); unsigned char *pkt = (unsigned char *)pf->fdir.prg_pkt; enum i40e_filter_pctype pctype; @@ -1023,7 +1500,8 @@ i40e_add_del_fdir_filter(struct rte_eth_dev *dev, return -ENOTSUP; } - if (!I40E_VALID_FLOW(filter->input.flow_type)) { + pctype = i40e_flowtype_to_pctype(pf->adapter, filter->input.flow_type); + if (pctype == I40E_FILTER_PCTYPE_INVALID) { PMD_DRV_LOG(ERR, "invalid flow_type input."); return -EINVAL; } @@ -1044,13 +1522,122 @@ i40e_add_del_fdir_filter(struct rte_eth_dev *dev, PMD_DRV_LOG(ERR, "construct packet for fdir fails."); return ret; } - pctype = i40e_flowtype_to_pctype(filter->input.flow_type); + + if (hw->mac.type == I40E_MAC_X722) { + /* get translated pctype value in fd pctype register */ + pctype = (enum i40e_filter_pctype)i40e_read_rx_ctl( + hw, I40E_GLQF_FD_PCTYPES((int)pctype)); + } + ret = i40e_fdir_filter_programming(pf, pctype, filter, add); if (ret < 0) { PMD_DRV_LOG(ERR, "fdir programming fails for PCTYPE(%u).", pctype); return ret; } + + return ret; +} + +/** + * i40e_flow_add_del_fdir_filter - add or remove a flow director filter. + * @pf: board private structure + * @filter: fdir filter entry + * @add: 0 - delete, 1 - add + */ +int +i40e_flow_add_del_fdir_filter(struct rte_eth_dev *dev, + const struct i40e_fdir_filter_conf *filter, + bool add) +{ + struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + unsigned char *pkt = (unsigned char *)pf->fdir.prg_pkt; + enum i40e_filter_pctype pctype; + struct i40e_fdir_info *fdir_info = &pf->fdir; + struct i40e_fdir_filter *fdir_filter, *node; + struct i40e_fdir_filter check_filter; /* Check if the filter exists */ + int ret = 0; + + if (dev->data->dev_conf.fdir_conf.mode != RTE_FDIR_MODE_PERFECT) { + PMD_DRV_LOG(ERR, "FDIR is not enabled, please check the mode in fdir_conf."); + return -ENOTSUP; + } + + if (filter->action.rx_queue >= pf->dev_data->nb_rx_queues) { + PMD_DRV_LOG(ERR, "Invalid queue ID"); + return -EINVAL; + } + if (filter->input.flow_ext.is_vf && + filter->input.flow_ext.dst_id >= pf->vf_num) { + PMD_DRV_LOG(ERR, "Invalid VF ID"); + return -EINVAL; + } + if (filter->input.flow_ext.pkt_template) { + if (filter->input.flow.raw_flow.length > I40E_FDIR_PKT_LEN || + !filter->input.flow.raw_flow.packet) { + PMD_DRV_LOG(ERR, "Invalid raw packet template" + " flow filter parameters!"); + return -EINVAL; + } + pctype = filter->input.flow.raw_flow.pctype; + } else { + pctype = filter->input.pctype; + } + + /* Check if there is the filter in SW list */ + memset(&check_filter, 0, sizeof(check_filter)); + i40e_fdir_filter_convert(filter, &check_filter); + node = i40e_sw_fdir_filter_lookup(fdir_info, &check_filter.fdir.input); + if (add && node) { + PMD_DRV_LOG(ERR, + "Conflict with existing flow director rules!"); + return -EINVAL; + } + + if (!add && !node) { + PMD_DRV_LOG(ERR, + "There's no corresponding flow firector filter!"); + return -EINVAL; + } + + memset(pkt, 0, I40E_FDIR_PKT_LEN); + + ret = i40e_flow_fdir_construct_pkt(pf, &filter->input, pkt); + if (ret < 0) { + PMD_DRV_LOG(ERR, "construct packet for fdir fails."); + return ret; + } + + if (hw->mac.type == I40E_MAC_X722) { + /* get translated pctype value in fd pctype register */ + pctype = (enum i40e_filter_pctype)i40e_read_rx_ctl( + hw, I40E_GLQF_FD_PCTYPES((int)pctype)); + } + + ret = i40e_flow_fdir_filter_programming(pf, pctype, filter, add); + if (ret < 0) { + PMD_DRV_LOG(ERR, "fdir programming fails for PCTYPE(%u).", + pctype); + return ret; + } + + if (add) { + fdir_filter = rte_zmalloc("fdir_filter", + sizeof(*fdir_filter), 0); + if (fdir_filter == NULL) { + PMD_DRV_LOG(ERR, "Failed to alloc memory."); + return -ENOMEM; + } + + rte_memcpy(fdir_filter, &check_filter, sizeof(check_filter)); + ret = i40e_sw_fdir_filter_insert(pf, fdir_filter); + if (ret < 0) + rte_free(fdir_filter); + } else { + ret = i40e_sw_fdir_filter_del(pf, &node->fdir.input); + } + return ret; } @@ -1166,25 +1753,158 @@ i40e_fdir_filter_programming(struct i40e_pf *pf, /* Update the tx tail register */ rte_wmb(); I40E_PCI_REG_WRITE(txq->qtx_tail, txq->tx_tail); - - for (i = 0; i < I40E_FDIR_WAIT_COUNT; i++) { - rte_delay_us(I40E_FDIR_WAIT_INTERVAL_US); + for (i = 0; i < I40E_FDIR_MAX_WAIT_US; i++) { if ((txdp->cmd_type_offset_bsz & rte_cpu_to_le_64(I40E_TXD_QW1_DTYPE_MASK)) == rte_cpu_to_le_64(I40E_TX_DESC_DTYPE_DESC_DONE)) break; + rte_delay_us(1); } - if (i >= I40E_FDIR_WAIT_COUNT) { + if (i >= I40E_FDIR_MAX_WAIT_US) { PMD_DRV_LOG(ERR, "Failed to program FDIR filter:" " time out to get DD on tx queue."); return -ETIMEDOUT; } /* totally delay 10 ms to check programming status*/ - rte_delay_us((I40E_FDIR_WAIT_COUNT - i) * I40E_FDIR_WAIT_INTERVAL_US); + for (; i < I40E_FDIR_MAX_WAIT_US; i++) { + if (i40e_check_fdir_programming_status(rxq) >= 0) + return 0; + rte_delay_us(1); + } + PMD_DRV_LOG(ERR, + "Failed to program FDIR filter: programming status reported."); + return -ETIMEDOUT; +} + +/* + * i40e_flow_fdir_filter_programming - Program a flow director filter rule. + * Is done by Flow Director Programming Descriptor followed by packet + * structure that contains the filter fields need to match. + * @pf: board private structure + * @pctype: pctype + * @filter: fdir filter entry + * @add: 0 - delete, 1 - add + */ +static int +i40e_flow_fdir_filter_programming(struct i40e_pf *pf, + enum i40e_filter_pctype pctype, + const struct i40e_fdir_filter_conf *filter, + bool add) +{ + struct i40e_tx_queue *txq = pf->fdir.txq; + struct i40e_rx_queue *rxq = pf->fdir.rxq; + const struct i40e_fdir_action *fdir_action = &filter->action; + volatile struct i40e_tx_desc *txdp; + volatile struct i40e_filter_program_desc *fdirdp; + uint32_t td_cmd; + uint16_t vsi_id, i; + uint8_t dest; + + PMD_DRV_LOG(INFO, "filling filter programming descriptor."); + fdirdp = (volatile struct i40e_filter_program_desc *) + (&txq->tx_ring[txq->tx_tail]); + + fdirdp->qindex_flex_ptype_vsi = + rte_cpu_to_le_32((fdir_action->rx_queue << + I40E_TXD_FLTR_QW0_QINDEX_SHIFT) & + I40E_TXD_FLTR_QW0_QINDEX_MASK); + + fdirdp->qindex_flex_ptype_vsi |= + rte_cpu_to_le_32((fdir_action->flex_off << + I40E_TXD_FLTR_QW0_FLEXOFF_SHIFT) & + I40E_TXD_FLTR_QW0_FLEXOFF_MASK); + + fdirdp->qindex_flex_ptype_vsi |= + rte_cpu_to_le_32((pctype << + I40E_TXD_FLTR_QW0_PCTYPE_SHIFT) & + I40E_TXD_FLTR_QW0_PCTYPE_MASK); + + if (filter->input.flow_ext.is_vf) + vsi_id = pf->vfs[filter->input.flow_ext.dst_id].vsi->vsi_id; + else + /* Use LAN VSI Id by default */ + vsi_id = pf->main_vsi->vsi_id; + fdirdp->qindex_flex_ptype_vsi |= + rte_cpu_to_le_32(((uint32_t)vsi_id << + I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT) & + I40E_TXD_FLTR_QW0_DEST_VSI_MASK); + + fdirdp->dtype_cmd_cntindex = + rte_cpu_to_le_32(I40E_TX_DESC_DTYPE_FILTER_PROG); + + if (add) + fdirdp->dtype_cmd_cntindex |= rte_cpu_to_le_32( + I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE << + I40E_TXD_FLTR_QW1_PCMD_SHIFT); + else + fdirdp->dtype_cmd_cntindex |= rte_cpu_to_le_32( + I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE << + I40E_TXD_FLTR_QW1_PCMD_SHIFT); + + if (fdir_action->behavior == I40E_FDIR_REJECT) + dest = I40E_FILTER_PROGRAM_DESC_DEST_DROP_PACKET; + else if (fdir_action->behavior == I40E_FDIR_ACCEPT) + dest = I40E_FILTER_PROGRAM_DESC_DEST_DIRECT_PACKET_QINDEX; + else if (fdir_action->behavior == I40E_FDIR_PASSTHRU) + dest = I40E_FILTER_PROGRAM_DESC_DEST_DIRECT_PACKET_OTHER; + else { + PMD_DRV_LOG(ERR, "Failed to program FDIR filter: unsupported fdir behavior."); + return -EINVAL; + } + + fdirdp->dtype_cmd_cntindex |= rte_cpu_to_le_32((dest << + I40E_TXD_FLTR_QW1_DEST_SHIFT) & + I40E_TXD_FLTR_QW1_DEST_MASK); + + fdirdp->dtype_cmd_cntindex |= + rte_cpu_to_le_32((fdir_action->report_status << + I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT) & + I40E_TXD_FLTR_QW1_FD_STATUS_MASK); + + fdirdp->dtype_cmd_cntindex |= + rte_cpu_to_le_32(I40E_TXD_FLTR_QW1_CNT_ENA_MASK); + fdirdp->dtype_cmd_cntindex |= + rte_cpu_to_le_32( + ((uint32_t)pf->fdir.match_counter_index << + I40E_TXD_FLTR_QW1_CNTINDEX_SHIFT) & + I40E_TXD_FLTR_QW1_CNTINDEX_MASK); + + fdirdp->fd_id = rte_cpu_to_le_32(filter->soft_id); + + PMD_DRV_LOG(INFO, "filling transmit descriptor."); + txdp = &txq->tx_ring[txq->tx_tail + 1]; + txdp->buffer_addr = rte_cpu_to_le_64(pf->fdir.dma_addr); + td_cmd = I40E_TX_DESC_CMD_EOP | + I40E_TX_DESC_CMD_RS | + I40E_TX_DESC_CMD_DUMMY; + + txdp->cmd_type_offset_bsz = + i40e_build_ctob(td_cmd, 0, I40E_FDIR_PKT_LEN, 0); + + txq->tx_tail += 2; /* set 2 descriptors above, fdirdp and txdp */ + if (txq->tx_tail >= txq->nb_tx_desc) + txq->tx_tail = 0; + /* Update the tx tail register */ + rte_wmb(); + I40E_PCI_REG_WRITE(txq->qtx_tail, txq->tx_tail); + for (i = 0; i < I40E_FDIR_MAX_WAIT_US; i++) { + if ((txdp->cmd_type_offset_bsz & + rte_cpu_to_le_64(I40E_TXD_QW1_DTYPE_MASK)) == + rte_cpu_to_le_64(I40E_TX_DESC_DTYPE_DESC_DONE)) + break; + rte_delay_us(1); + } + if (i >= I40E_FDIR_MAX_WAIT_US) { + PMD_DRV_LOG(ERR, + "Failed to program FDIR filter: time out to get DD on tx queue."); + return -ETIMEDOUT; + } + /* totally delay 10 ms to check programming status*/ + rte_delay_us(I40E_FDIR_MAX_WAIT_US); if (i40e_check_fdir_programming_status(rxq) < 0) { - PMD_DRV_LOG(ERR, "Failed to program FDIR filter:" - " programming status reported."); - return -ENOSYS; + PMD_DRV_LOG(ERR, + "Failed to program FDIR filter: programming status reported."); + return -ETIMEDOUT; } return 0; @@ -1194,7 +1914,7 @@ i40e_fdir_filter_programming(struct i40e_pf *pf, * i40e_fdir_flush - clear all filters of Flow Director table * @pf: board private structure */ -static int +int i40e_fdir_flush(struct rte_eth_dev *dev) { struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); @@ -1281,9 +2001,11 @@ i40e_fdir_info_get_flex_mask(struct i40e_pf *pf, i <= I40E_FILTER_PCTYPE_L2_PAYLOAD; i++) { mask = &pf->fdir.flex_mask[i]; - if (!I40E_VALID_PCTYPE((enum i40e_filter_pctype)i)) + flow_type = i40e_pctype_to_flowtype(pf->adapter, + (enum i40e_filter_pctype)i); + if (flow_type == RTE_ETH_FLOW_UNKNOWN) continue; - flow_type = i40e_pctype_to_flowtype((enum i40e_filter_pctype)i); + for (j = 0; j < I40E_FDIR_MAX_FLEXWORD_NUM; j++) { if (mask->word_mask & I40E_FLEX_WORD_MASK(j)) { ptr->mask[j * sizeof(uint16_t)] = UINT8_MAX; @@ -1449,3 +2171,30 @@ i40e_fdir_ctrl_func(struct rte_eth_dev *dev, } return ret; } + +/* Restore flow director filter */ +void +i40e_fdir_filter_restore(struct i40e_pf *pf) +{ + struct rte_eth_dev *dev = I40E_VSI_TO_ETH_DEV(pf->main_vsi); + struct i40e_fdir_filter_list *fdir_list = &pf->fdir.fdir_list; + struct i40e_fdir_filter *f; + struct i40e_hw *hw = I40E_PF_TO_HW(pf); + uint32_t fdstat; + uint32_t guarant_cnt; /**< Number of filters in guaranteed spaces. */ + uint32_t best_cnt; /**< Number of filters in best effort spaces. */ + + TAILQ_FOREACH(f, fdir_list, rules) + i40e_flow_add_del_fdir_filter(dev, &f->fdir, TRUE); + + fdstat = I40E_READ_REG(hw, I40E_PFQF_FDSTAT); + guarant_cnt = + (uint32_t)((fdstat & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) >> + I40E_PFQF_FDSTAT_GUARANT_CNT_SHIFT); + best_cnt = + (uint32_t)((fdstat & I40E_PFQF_FDSTAT_BEST_CNT_MASK) >> + I40E_PFQF_FDSTAT_BEST_CNT_SHIFT); + + PMD_DRV_LOG(INFO, "FDIR: Guarant count: %d, Best count: %d", + guarant_cnt, best_cnt); +} diff --git a/dpdk/drivers/net/i40e/i40e_flow.c b/dpdk/drivers/net/i40e/i40e_flow.c new file mode 100644 index 00000000..37380e6e --- /dev/null +++ b/dpdk/drivers/net/i40e/i40e_flow.c @@ -0,0 +1,4537 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2016-2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "i40e_logs.h" +#include "base/i40e_type.h" +#include "base/i40e_prototype.h" +#include "i40e_ethdev.h" + +#define I40E_IPV6_TC_MASK (0xFF << I40E_FDIR_IPv6_TC_OFFSET) +#define I40E_IPV6_FRAG_HEADER 44 +#define I40E_TENANT_ARRAY_NUM 3 +#define I40E_TCI_MASK 0xFFFF + +static int i40e_flow_validate(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error); +static struct rte_flow *i40e_flow_create(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error); +static int i40e_flow_destroy(struct rte_eth_dev *dev, + struct rte_flow *flow, + struct rte_flow_error *error); +static int i40e_flow_flush(struct rte_eth_dev *dev, + struct rte_flow_error *error); +static int +i40e_flow_parse_ethertype_pattern(struct rte_eth_dev *dev, + const struct rte_flow_item *pattern, + struct rte_flow_error *error, + struct rte_eth_ethertype_filter *filter); +static int i40e_flow_parse_ethertype_action(struct rte_eth_dev *dev, + const struct rte_flow_action *actions, + struct rte_flow_error *error, + struct rte_eth_ethertype_filter *filter); +static int i40e_flow_parse_fdir_pattern(struct rte_eth_dev *dev, + const struct rte_flow_item *pattern, + struct rte_flow_error *error, + struct i40e_fdir_filter_conf *filter); +static int i40e_flow_parse_fdir_action(struct rte_eth_dev *dev, + const struct rte_flow_action *actions, + struct rte_flow_error *error, + struct i40e_fdir_filter_conf *filter); +static int i40e_flow_parse_tunnel_action(struct rte_eth_dev *dev, + const struct rte_flow_action *actions, + struct rte_flow_error *error, + struct i40e_tunnel_filter_conf *filter); +static int i40e_flow_parse_attr(const struct rte_flow_attr *attr, + struct rte_flow_error *error); +static int i40e_flow_parse_ethertype_filter(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error, + union i40e_filter_t *filter); +static int i40e_flow_parse_fdir_filter(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error, + union i40e_filter_t *filter); +static int i40e_flow_parse_vxlan_filter(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error, + union i40e_filter_t *filter); +static int i40e_flow_parse_nvgre_filter(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error, + union i40e_filter_t *filter); +static int i40e_flow_parse_mpls_filter(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error, + union i40e_filter_t *filter); +static int i40e_flow_parse_gtp_filter(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error, + union i40e_filter_t *filter); +static int i40e_flow_destroy_ethertype_filter(struct i40e_pf *pf, + struct i40e_ethertype_filter *filter); +static int i40e_flow_destroy_tunnel_filter(struct i40e_pf *pf, + struct i40e_tunnel_filter *filter); +static int i40e_flow_flush_fdir_filter(struct i40e_pf *pf); +static int i40e_flow_flush_ethertype_filter(struct i40e_pf *pf); +static int i40e_flow_flush_tunnel_filter(struct i40e_pf *pf); +static int +i40e_flow_parse_qinq_filter(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error, + union i40e_filter_t *filter); +static int +i40e_flow_parse_qinq_pattern(struct rte_eth_dev *dev, + const struct rte_flow_item *pattern, + struct rte_flow_error *error, + struct i40e_tunnel_filter_conf *filter); + +const struct rte_flow_ops i40e_flow_ops = { + .validate = i40e_flow_validate, + .create = i40e_flow_create, + .destroy = i40e_flow_destroy, + .flush = i40e_flow_flush, +}; + +union i40e_filter_t cons_filter; +enum rte_filter_type cons_filter_type = RTE_ETH_FILTER_NONE; + +/* Pattern matched ethertype filter */ +static enum rte_flow_item_type pattern_ethertype[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_END, +}; + +/* Pattern matched flow director filter */ +static enum rte_flow_item_type pattern_fdir_ipv4[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_udp[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_tcp[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_sctp[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_gtpc[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_GTPC, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_gtpu[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_GTPU, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_gtpu_ipv4[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_GTPU, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_gtpu_ipv6[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_GTPU, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_udp[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_tcp[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_sctp[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_gtpc[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_GTPC, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_gtpu[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_GTPU, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_gtpu_ipv4[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_GTPU, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_gtpu_ipv6[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_GTPU, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ethertype_raw_1[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ethertype_raw_2[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ethertype_raw_3[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_raw_1[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_raw_2[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_raw_3[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_udp_raw_1[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_udp_raw_2[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_udp_raw_3[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_tcp_raw_1[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_tcp_raw_2[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_tcp_raw_3[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_sctp_raw_1[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_sctp_raw_2[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_sctp_raw_3[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_raw_1[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_raw_2[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_raw_3[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_udp_raw_1[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_udp_raw_2[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_udp_raw_3[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_tcp_raw_1[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_tcp_raw_2[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_tcp_raw_3[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_sctp_raw_1[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_sctp_raw_2[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_sctp_raw_3[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ethertype_vlan[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_udp[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_tcp[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_sctp[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_udp[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_tcp[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_sctp[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ethertype_vlan_raw_1[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ethertype_vlan_raw_2[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ethertype_vlan_raw_3[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_raw_1[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_raw_2[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_raw_3[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_udp_raw_1[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_udp_raw_2[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_udp_raw_3[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_tcp_raw_1[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_tcp_raw_2[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_tcp_raw_3[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_sctp_raw_1[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_sctp_raw_2[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_sctp_raw_3[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_raw_1[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_raw_2[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_raw_3[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_udp_raw_1[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_udp_raw_2[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_udp_raw_3[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_tcp_raw_1[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_tcp_raw_2[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_tcp_raw_3[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_sctp_raw_1[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_sctp_raw_2[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_sctp_raw_3[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_udp_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_tcp_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_sctp_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_udp_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_tcp_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_sctp_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ethertype_raw_1_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ethertype_raw_2_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ethertype_raw_3_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_raw_1_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_raw_2_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_raw_3_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_udp_raw_1_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_udp_raw_2_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_udp_raw_3_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_tcp_raw_1_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_tcp_raw_2_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_tcp_raw_3_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_sctp_raw_1_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_sctp_raw_2_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv4_sctp_raw_3_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_raw_1_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_raw_2_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_raw_3_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_udp_raw_1_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_udp_raw_2_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_udp_raw_3_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_tcp_raw_1_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_tcp_raw_2_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_tcp_raw_3_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_sctp_raw_1_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_sctp_raw_2_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ipv6_sctp_raw_3_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ethertype_vlan_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_udp_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_tcp_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_sctp_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_udp_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_tcp_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_sctp_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ethertype_vlan_raw_1_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ethertype_vlan_raw_2_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_ethertype_vlan_raw_3_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_raw_1_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_raw_2_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_raw_3_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_udp_raw_1_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_udp_raw_2_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_udp_raw_3_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_tcp_raw_1_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_tcp_raw_2_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_tcp_raw_3_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_sctp_raw_1_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_sctp_raw_2_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv4_sctp_raw_3_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_raw_1_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_raw_2_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_raw_3_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_udp_raw_1_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_udp_raw_2_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_udp_raw_3_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_tcp_raw_1_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_tcp_raw_2_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_tcp_raw_3_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_sctp_raw_1_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_sctp_raw_2_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_fdir_vlan_ipv6_sctp_raw_3_vf[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_RAW, + RTE_FLOW_ITEM_TYPE_VF, + RTE_FLOW_ITEM_TYPE_END, +}; + +/* Pattern matched tunnel filter */ +static enum rte_flow_item_type pattern_vxlan_1[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_VXLAN, + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_vxlan_2[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_VXLAN, + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_vxlan_3[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_VXLAN, + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_vxlan_4[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_VXLAN, + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_nvgre_1[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_NVGRE, + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_nvgre_2[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_NVGRE, + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_nvgre_3[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_NVGRE, + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_nvgre_4[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_NVGRE, + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_mpls_1[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_MPLS, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_mpls_2[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_MPLS, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_mpls_3[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_GRE, + RTE_FLOW_ITEM_TYPE_MPLS, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_mpls_4[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV6, + RTE_FLOW_ITEM_TYPE_GRE, + RTE_FLOW_ITEM_TYPE_MPLS, + RTE_FLOW_ITEM_TYPE_END, +}; + +static enum rte_flow_item_type pattern_qinq_1[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_END, +}; + +static struct i40e_valid_pattern i40e_supported_patterns[] = { + /* Ethertype */ + { pattern_ethertype, i40e_flow_parse_ethertype_filter }, + /* FDIR - support default flow type without flexible payload*/ + { pattern_ethertype, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_udp, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_tcp, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_sctp, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_gtpc, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_gtpu, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_gtpu_ipv4, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_gtpu_ipv6, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_udp, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_tcp, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_sctp, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_gtpc, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_gtpu, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_gtpu_ipv4, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_gtpu_ipv6, i40e_flow_parse_fdir_filter }, + /* FDIR - support default flow type with flexible payload */ + { pattern_fdir_ethertype_raw_1, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ethertype_raw_2, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ethertype_raw_3, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_raw_1, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_raw_2, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_raw_3, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_udp_raw_1, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_udp_raw_2, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_udp_raw_3, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_tcp_raw_1, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_tcp_raw_2, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_tcp_raw_3, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_sctp_raw_1, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_sctp_raw_2, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_sctp_raw_3, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_raw_1, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_raw_2, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_raw_3, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_udp_raw_1, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_udp_raw_2, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_udp_raw_3, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_tcp_raw_1, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_tcp_raw_2, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_tcp_raw_3, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_sctp_raw_1, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_sctp_raw_2, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_sctp_raw_3, i40e_flow_parse_fdir_filter }, + /* FDIR - support single vlan input set */ + { pattern_fdir_ethertype_vlan, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_udp, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_tcp, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_sctp, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_udp, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_tcp, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_sctp, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ethertype_vlan_raw_1, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ethertype_vlan_raw_2, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ethertype_vlan_raw_3, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_raw_1, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_raw_2, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_raw_3, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_udp_raw_1, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_udp_raw_2, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_udp_raw_3, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_tcp_raw_1, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_tcp_raw_2, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_tcp_raw_3, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_sctp_raw_1, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_sctp_raw_2, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_sctp_raw_3, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_raw_1, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_raw_2, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_raw_3, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_udp_raw_1, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_udp_raw_2, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_udp_raw_3, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_tcp_raw_1, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_tcp_raw_2, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_tcp_raw_3, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_sctp_raw_1, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_sctp_raw_2, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_sctp_raw_3, i40e_flow_parse_fdir_filter }, + /* FDIR - support VF item */ + { pattern_fdir_ipv4_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_udp_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_tcp_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_sctp_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_udp_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_tcp_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_sctp_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ethertype_raw_1_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ethertype_raw_2_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ethertype_raw_3_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_raw_1_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_raw_2_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_raw_3_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_udp_raw_1_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_udp_raw_2_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_udp_raw_3_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_tcp_raw_1_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_tcp_raw_2_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_tcp_raw_3_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_sctp_raw_1_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_sctp_raw_2_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv4_sctp_raw_3_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_raw_1_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_raw_2_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_raw_3_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_udp_raw_1_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_udp_raw_2_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_udp_raw_3_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_tcp_raw_1_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_tcp_raw_2_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_tcp_raw_3_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_sctp_raw_1_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_sctp_raw_2_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ipv6_sctp_raw_3_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ethertype_vlan_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_udp_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_tcp_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_sctp_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_udp_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_tcp_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_sctp_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ethertype_vlan_raw_1_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ethertype_vlan_raw_2_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_ethertype_vlan_raw_3_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_raw_1_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_raw_2_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_raw_3_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_udp_raw_1_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_udp_raw_2_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_udp_raw_3_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_tcp_raw_1_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_tcp_raw_2_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_tcp_raw_3_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_sctp_raw_1_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_sctp_raw_2_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv4_sctp_raw_3_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_raw_1_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_raw_2_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_raw_3_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_udp_raw_1_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_udp_raw_2_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_udp_raw_3_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_tcp_raw_1_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_tcp_raw_2_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_tcp_raw_3_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_sctp_raw_1_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_sctp_raw_2_vf, i40e_flow_parse_fdir_filter }, + { pattern_fdir_vlan_ipv6_sctp_raw_3_vf, i40e_flow_parse_fdir_filter }, + /* VXLAN */ + { pattern_vxlan_1, i40e_flow_parse_vxlan_filter }, + { pattern_vxlan_2, i40e_flow_parse_vxlan_filter }, + { pattern_vxlan_3, i40e_flow_parse_vxlan_filter }, + { pattern_vxlan_4, i40e_flow_parse_vxlan_filter }, + /* NVGRE */ + { pattern_nvgre_1, i40e_flow_parse_nvgre_filter }, + { pattern_nvgre_2, i40e_flow_parse_nvgre_filter }, + { pattern_nvgre_3, i40e_flow_parse_nvgre_filter }, + { pattern_nvgre_4, i40e_flow_parse_nvgre_filter }, + /* MPLSoUDP & MPLSoGRE */ + { pattern_mpls_1, i40e_flow_parse_mpls_filter }, + { pattern_mpls_2, i40e_flow_parse_mpls_filter }, + { pattern_mpls_3, i40e_flow_parse_mpls_filter }, + { pattern_mpls_4, i40e_flow_parse_mpls_filter }, + /* GTP-C & GTP-U */ + { pattern_fdir_ipv4_gtpc, i40e_flow_parse_gtp_filter }, + { pattern_fdir_ipv4_gtpu, i40e_flow_parse_gtp_filter }, + { pattern_fdir_ipv6_gtpc, i40e_flow_parse_gtp_filter }, + { pattern_fdir_ipv6_gtpu, i40e_flow_parse_gtp_filter }, + /* QINQ */ + { pattern_qinq_1, i40e_flow_parse_qinq_filter }, +}; + +#define NEXT_ITEM_OF_ACTION(act, actions, index) \ + do { \ + act = actions + index; \ + while (act->type == RTE_FLOW_ACTION_TYPE_VOID) { \ + index++; \ + act = actions + index; \ + } \ + } while (0) + +/* Find the first VOID or non-VOID item pointer */ +static const struct rte_flow_item * +i40e_find_first_item(const struct rte_flow_item *item, bool is_void) +{ + bool is_find; + + while (item->type != RTE_FLOW_ITEM_TYPE_END) { + if (is_void) + is_find = item->type == RTE_FLOW_ITEM_TYPE_VOID; + else + is_find = item->type != RTE_FLOW_ITEM_TYPE_VOID; + if (is_find) + break; + item++; + } + return item; +} + +/* Skip all VOID items of the pattern */ +static void +i40e_pattern_skip_void_item(struct rte_flow_item *items, + const struct rte_flow_item *pattern) +{ + uint32_t cpy_count = 0; + const struct rte_flow_item *pb = pattern, *pe = pattern; + + for (;;) { + /* Find a non-void item first */ + pb = i40e_find_first_item(pb, false); + if (pb->type == RTE_FLOW_ITEM_TYPE_END) { + pe = pb; + break; + } + + /* Find a void item */ + pe = i40e_find_first_item(pb + 1, true); + + cpy_count = pe - pb; + rte_memcpy(items, pb, sizeof(struct rte_flow_item) * cpy_count); + + items += cpy_count; + + if (pe->type == RTE_FLOW_ITEM_TYPE_END) { + pb = pe; + break; + } + + pb = pe + 1; + } + /* Copy the END item. */ + rte_memcpy(items, pe, sizeof(struct rte_flow_item)); +} + +/* Check if the pattern matches a supported item type array */ +static bool +i40e_match_pattern(enum rte_flow_item_type *item_array, + struct rte_flow_item *pattern) +{ + struct rte_flow_item *item = pattern; + + while ((*item_array == item->type) && + (*item_array != RTE_FLOW_ITEM_TYPE_END)) { + item_array++; + item++; + } + + return (*item_array == RTE_FLOW_ITEM_TYPE_END && + item->type == RTE_FLOW_ITEM_TYPE_END); +} + +/* Find if there's parse filter function matched */ +static parse_filter_t +i40e_find_parse_filter_func(struct rte_flow_item *pattern, uint32_t *idx) +{ + parse_filter_t parse_filter = NULL; + uint8_t i = *idx; + + for (; i < RTE_DIM(i40e_supported_patterns); i++) { + if (i40e_match_pattern(i40e_supported_patterns[i].items, + pattern)) { + parse_filter = i40e_supported_patterns[i].parse_filter; + break; + } + } + + *idx = ++i; + + return parse_filter; +} + +/* Parse attributes */ +static int +i40e_flow_parse_attr(const struct rte_flow_attr *attr, + struct rte_flow_error *error) +{ + /* Must be input direction */ + if (!attr->ingress) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, + attr, "Only support ingress."); + return -rte_errno; + } + + /* Not supported */ + if (attr->egress) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, + attr, "Not support egress."); + return -rte_errno; + } + + /* Not supported */ + if (attr->priority) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, + attr, "Not support priority."); + return -rte_errno; + } + + /* Not supported */ + if (attr->group) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_GROUP, + attr, "Not support group."); + return -rte_errno; + } + + return 0; +} + +static uint16_t +i40e_get_outer_vlan(struct rte_eth_dev *dev) +{ + struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); + int qinq = dev->data->dev_conf.rxmode.hw_vlan_extend; + uint64_t reg_r = 0; + uint16_t reg_id; + uint16_t tpid; + + if (qinq) + reg_id = 2; + else + reg_id = 3; + + i40e_aq_debug_read_register(hw, I40E_GL_SWT_L2TAGCTRL(reg_id), + ®_r, NULL); + + tpid = (reg_r >> I40E_GL_SWT_L2TAGCTRL_ETHERTYPE_SHIFT) & 0xFFFF; + + return tpid; +} + +/* 1. Last in item should be NULL as range is not supported. + * 2. Supported filter types: MAC_ETHTYPE and ETHTYPE. + * 3. SRC mac_addr mask should be 00:00:00:00:00:00. + * 4. DST mac_addr mask should be 00:00:00:00:00:00 or + * FF:FF:FF:FF:FF:FF + * 5. Ether_type mask should be 0xFFFF. + */ +static int +i40e_flow_parse_ethertype_pattern(struct rte_eth_dev *dev, + const struct rte_flow_item *pattern, + struct rte_flow_error *error, + struct rte_eth_ethertype_filter *filter) +{ + const struct rte_flow_item *item = pattern; + const struct rte_flow_item_eth *eth_spec; + const struct rte_flow_item_eth *eth_mask; + enum rte_flow_item_type item_type; + uint16_t outer_tpid; + + outer_tpid = i40e_get_outer_vlan(dev); + + for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { + if (item->last) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Not support range"); + return -rte_errno; + } + item_type = item->type; + switch (item_type) { + case RTE_FLOW_ITEM_TYPE_ETH: + eth_spec = (const struct rte_flow_item_eth *)item->spec; + eth_mask = (const struct rte_flow_item_eth *)item->mask; + /* Get the MAC info. */ + if (!eth_spec || !eth_mask) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "NULL ETH spec/mask"); + return -rte_errno; + } + + /* Mask bits of source MAC address must be full of 0. + * Mask bits of destination MAC address must be full + * of 1 or full of 0. + */ + if (!is_zero_ether_addr(ð_mask->src) || + (!is_zero_ether_addr(ð_mask->dst) && + !is_broadcast_ether_addr(ð_mask->dst))) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid MAC_addr mask"); + return -rte_errno; + } + + if ((eth_mask->type & UINT16_MAX) != UINT16_MAX) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid ethertype mask"); + return -rte_errno; + } + + /* If mask bits of destination MAC address + * are full of 1, set RTE_ETHTYPE_FLAGS_MAC. + */ + if (is_broadcast_ether_addr(ð_mask->dst)) { + filter->mac_addr = eth_spec->dst; + filter->flags |= RTE_ETHTYPE_FLAGS_MAC; + } else { + filter->flags &= ~RTE_ETHTYPE_FLAGS_MAC; + } + filter->ether_type = rte_be_to_cpu_16(eth_spec->type); + + if (filter->ether_type == ETHER_TYPE_IPv4 || + filter->ether_type == ETHER_TYPE_IPv6 || + filter->ether_type == ETHER_TYPE_LLDP || + filter->ether_type == outer_tpid) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Unsupported ether_type in" + " control packet filter."); + return -rte_errno; + } + break; + default: + break; + } + } + + return 0; +} + +/* Ethertype action only supports QUEUE or DROP. */ +static int +i40e_flow_parse_ethertype_action(struct rte_eth_dev *dev, + const struct rte_flow_action *actions, + struct rte_flow_error *error, + struct rte_eth_ethertype_filter *filter) +{ + struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + const struct rte_flow_action *act; + const struct rte_flow_action_queue *act_q; + uint32_t index = 0; + + /* Check if the first non-void action is QUEUE or DROP. */ + NEXT_ITEM_OF_ACTION(act, actions, index); + if (act->type != RTE_FLOW_ACTION_TYPE_QUEUE && + act->type != RTE_FLOW_ACTION_TYPE_DROP) { + rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, + act, "Not supported action."); + return -rte_errno; + } + + if (act->type == RTE_FLOW_ACTION_TYPE_QUEUE) { + act_q = (const struct rte_flow_action_queue *)act->conf; + filter->queue = act_q->index; + if (filter->queue >= pf->dev_data->nb_rx_queues) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + act, "Invalid queue ID for" + " ethertype_filter."); + return -rte_errno; + } + } else { + filter->flags |= RTE_ETHTYPE_FLAGS_DROP; + } + + /* Check if the next non-void item is END */ + index++; + NEXT_ITEM_OF_ACTION(act, actions, index); + if (act->type != RTE_FLOW_ACTION_TYPE_END) { + rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, + act, "Not supported action."); + return -rte_errno; + } + + return 0; +} + +static int +i40e_flow_parse_ethertype_filter(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error, + union i40e_filter_t *filter) +{ + struct rte_eth_ethertype_filter *ethertype_filter = + &filter->ethertype_filter; + int ret; + + ret = i40e_flow_parse_ethertype_pattern(dev, pattern, error, + ethertype_filter); + if (ret) + return ret; + + ret = i40e_flow_parse_ethertype_action(dev, actions, error, + ethertype_filter); + if (ret) + return ret; + + ret = i40e_flow_parse_attr(attr, error); + if (ret) + return ret; + + cons_filter_type = RTE_ETH_FILTER_ETHERTYPE; + + return ret; +} + +static int +i40e_flow_check_raw_item(const struct rte_flow_item *item, + const struct rte_flow_item_raw *raw_spec, + struct rte_flow_error *error) +{ + if (!raw_spec->relative) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Relative should be 1."); + return -rte_errno; + } + + if (raw_spec->offset % sizeof(uint16_t)) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Offset should be even."); + return -rte_errno; + } + + if (raw_spec->search || raw_spec->limit) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "search or limit is not supported."); + return -rte_errno; + } + + if (raw_spec->offset < 0) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Offset should be non-negative."); + return -rte_errno; + } + return 0; +} + +static int +i40e_flow_store_flex_pit(struct i40e_pf *pf, + struct i40e_fdir_flex_pit *flex_pit, + enum i40e_flxpld_layer_idx layer_idx, + uint8_t raw_id) +{ + uint8_t field_idx; + + field_idx = layer_idx * I40E_MAX_FLXPLD_FIED + raw_id; + /* Check if the configuration is conflicted */ + if (pf->fdir.flex_pit_flag[layer_idx] && + (pf->fdir.flex_set[field_idx].src_offset != flex_pit->src_offset || + pf->fdir.flex_set[field_idx].size != flex_pit->size || + pf->fdir.flex_set[field_idx].dst_offset != flex_pit->dst_offset)) + return -1; + + /* Check if the configuration exists. */ + if (pf->fdir.flex_pit_flag[layer_idx] && + (pf->fdir.flex_set[field_idx].src_offset == flex_pit->src_offset && + pf->fdir.flex_set[field_idx].size == flex_pit->size && + pf->fdir.flex_set[field_idx].dst_offset == flex_pit->dst_offset)) + return 1; + + pf->fdir.flex_set[field_idx].src_offset = + flex_pit->src_offset; + pf->fdir.flex_set[field_idx].size = + flex_pit->size; + pf->fdir.flex_set[field_idx].dst_offset = + flex_pit->dst_offset; + + return 0; +} + +static int +i40e_flow_store_flex_mask(struct i40e_pf *pf, + enum i40e_filter_pctype pctype, + uint8_t *mask) +{ + struct i40e_fdir_flex_mask flex_mask; + uint16_t mask_tmp; + uint8_t i, nb_bitmask = 0; + + memset(&flex_mask, 0, sizeof(struct i40e_fdir_flex_mask)); + for (i = 0; i < I40E_FDIR_MAX_FLEX_LEN; i += sizeof(uint16_t)) { + mask_tmp = I40E_WORD(mask[i], mask[i + 1]); + if (mask_tmp) { + flex_mask.word_mask |= + I40E_FLEX_WORD_MASK(i / sizeof(uint16_t)); + if (mask_tmp != UINT16_MAX) { + flex_mask.bitmask[nb_bitmask].mask = ~mask_tmp; + flex_mask.bitmask[nb_bitmask].offset = + i / sizeof(uint16_t); + nb_bitmask++; + if (nb_bitmask > I40E_FDIR_BITMASK_NUM_WORD) + return -1; + } + } + } + flex_mask.nb_bitmask = nb_bitmask; + + if (pf->fdir.flex_mask_flag[pctype] && + (memcmp(&flex_mask, &pf->fdir.flex_mask[pctype], + sizeof(struct i40e_fdir_flex_mask)))) + return -2; + else if (pf->fdir.flex_mask_flag[pctype] && + !(memcmp(&flex_mask, &pf->fdir.flex_mask[pctype], + sizeof(struct i40e_fdir_flex_mask)))) + return 1; + + memcpy(&pf->fdir.flex_mask[pctype], &flex_mask, + sizeof(struct i40e_fdir_flex_mask)); + return 0; +} + +static void +i40e_flow_set_fdir_flex_pit(struct i40e_pf *pf, + enum i40e_flxpld_layer_idx layer_idx, + uint8_t raw_id) +{ + struct i40e_hw *hw = I40E_PF_TO_HW(pf); + uint32_t flx_pit; + uint8_t field_idx; + uint16_t min_next_off = 0; /* in words */ + uint8_t i; + + /* Set flex pit */ + for (i = 0; i < raw_id; i++) { + field_idx = layer_idx * I40E_MAX_FLXPLD_FIED + i; + flx_pit = MK_FLX_PIT(pf->fdir.flex_set[field_idx].src_offset, + pf->fdir.flex_set[field_idx].size, + pf->fdir.flex_set[field_idx].dst_offset); + + I40E_WRITE_REG(hw, I40E_PRTQF_FLX_PIT(field_idx), flx_pit); + min_next_off = pf->fdir.flex_set[field_idx].src_offset + + pf->fdir.flex_set[field_idx].size; + } + + for (; i < I40E_MAX_FLXPLD_FIED; i++) { + /* set the non-used register obeying register's constrain */ + field_idx = layer_idx * I40E_MAX_FLXPLD_FIED + i; + flx_pit = MK_FLX_PIT(min_next_off, NONUSE_FLX_PIT_FSIZE, + NONUSE_FLX_PIT_DEST_OFF); + I40E_WRITE_REG(hw, I40E_PRTQF_FLX_PIT(field_idx), flx_pit); + min_next_off++; + } + + pf->fdir.flex_pit_flag[layer_idx] = 1; +} + +static void +i40e_flow_set_fdir_flex_msk(struct i40e_pf *pf, + enum i40e_filter_pctype pctype) +{ + struct i40e_hw *hw = I40E_PF_TO_HW(pf); + struct i40e_fdir_flex_mask *flex_mask; + uint32_t flxinset, fd_mask; + uint8_t i; + + /* Set flex mask */ + flex_mask = &pf->fdir.flex_mask[pctype]; + flxinset = (flex_mask->word_mask << + I40E_PRTQF_FD_FLXINSET_INSET_SHIFT) & + I40E_PRTQF_FD_FLXINSET_INSET_MASK; + i40e_write_rx_ctl(hw, I40E_PRTQF_FD_FLXINSET(pctype), flxinset); + + for (i = 0; i < flex_mask->nb_bitmask; i++) { + fd_mask = (flex_mask->bitmask[i].mask << + I40E_PRTQF_FD_MSK_MASK_SHIFT) & + I40E_PRTQF_FD_MSK_MASK_MASK; + fd_mask |= ((flex_mask->bitmask[i].offset + + I40E_FLX_OFFSET_IN_FIELD_VECTOR) << + I40E_PRTQF_FD_MSK_OFFSET_SHIFT) & + I40E_PRTQF_FD_MSK_OFFSET_MASK; + i40e_write_rx_ctl(hw, I40E_PRTQF_FD_MSK(pctype, i), fd_mask); + } + + pf->fdir.flex_mask_flag[pctype] = 1; +} + +static int +i40e_flow_set_fdir_inset(struct i40e_pf *pf, + enum i40e_filter_pctype pctype, + uint64_t input_set) +{ + struct i40e_hw *hw = I40E_PF_TO_HW(pf); + uint64_t inset_reg = 0; + uint32_t mask_reg[I40E_INSET_MASK_NUM_REG] = {0}; + int i, num; + + /* Check if the input set is valid */ + if (i40e_validate_input_set(pctype, RTE_ETH_FILTER_FDIR, + input_set) != 0) { + PMD_DRV_LOG(ERR, "Invalid input set"); + return -EINVAL; + } + + /* Check if the configuration is conflicted */ + if (pf->fdir.inset_flag[pctype] && + memcmp(&pf->fdir.input_set[pctype], &input_set, sizeof(uint64_t))) + return -1; + + if (pf->fdir.inset_flag[pctype] && + !memcmp(&pf->fdir.input_set[pctype], &input_set, sizeof(uint64_t))) + return 0; + + num = i40e_generate_inset_mask_reg(input_set, mask_reg, + I40E_INSET_MASK_NUM_REG); + if (num < 0) + return -EINVAL; + + inset_reg |= i40e_translate_input_set_reg(hw->mac.type, input_set); + + i40e_check_write_reg(hw, I40E_PRTQF_FD_INSET(pctype, 0), + (uint32_t)(inset_reg & UINT32_MAX)); + i40e_check_write_reg(hw, I40E_PRTQF_FD_INSET(pctype, 1), + (uint32_t)((inset_reg >> + I40E_32_BIT_WIDTH) & UINT32_MAX)); + + for (i = 0; i < num; i++) + i40e_check_write_reg(hw, I40E_GLQF_FD_MSK(i, pctype), + mask_reg[i]); + + /*clear unused mask registers of the pctype */ + for (i = num; i < I40E_INSET_MASK_NUM_REG; i++) + i40e_check_write_reg(hw, I40E_GLQF_FD_MSK(i, pctype), 0); + I40E_WRITE_FLUSH(hw); + + pf->fdir.input_set[pctype] = input_set; + pf->fdir.inset_flag[pctype] = 1; + return 0; +} + +static uint8_t +i40e_flow_fdir_get_pctype_value(struct i40e_pf *pf, + enum rte_flow_item_type item_type, + struct i40e_fdir_filter_conf *filter) +{ + struct i40e_customized_pctype *cus_pctype = NULL; + + switch (item_type) { + case RTE_FLOW_ITEM_TYPE_GTPC: + cus_pctype = i40e_find_customized_pctype(pf, + I40E_CUSTOMIZED_GTPC); + break; + case RTE_FLOW_ITEM_TYPE_GTPU: + if (!filter->input.flow_ext.inner_ip) + cus_pctype = i40e_find_customized_pctype(pf, + I40E_CUSTOMIZED_GTPU); + else if (filter->input.flow_ext.iip_type == + I40E_FDIR_IPTYPE_IPV4) + cus_pctype = i40e_find_customized_pctype(pf, + I40E_CUSTOMIZED_GTPU_IPV4); + else if (filter->input.flow_ext.iip_type == + I40E_FDIR_IPTYPE_IPV6) + cus_pctype = i40e_find_customized_pctype(pf, + I40E_CUSTOMIZED_GTPU_IPV6); + break; + default: + PMD_DRV_LOG(ERR, "Unsupported item type"); + break; + } + + if (cus_pctype) + return cus_pctype->pctype; + + return I40E_FILTER_PCTYPE_INVALID; +} + +/* 1. Last in item should be NULL as range is not supported. + * 2. Supported patterns: refer to array i40e_supported_patterns. + * 3. Default supported flow type and input set: refer to array + * valid_fdir_inset_table in i40e_ethdev.c. + * 4. Mask of fields which need to be matched should be + * filled with 1. + * 5. Mask of fields which needn't to be matched should be + * filled with 0. + * 6. GTP profile supports GTPv1 only. + * 7. GTP-C response message ('source_port' = 2123) is not supported. + */ +static int +i40e_flow_parse_fdir_pattern(struct rte_eth_dev *dev, + const struct rte_flow_item *pattern, + struct rte_flow_error *error, + struct i40e_fdir_filter_conf *filter) +{ + struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + const struct rte_flow_item *item = pattern; + const struct rte_flow_item_eth *eth_spec, *eth_mask; + const struct rte_flow_item_vlan *vlan_spec, *vlan_mask; + const struct rte_flow_item_ipv4 *ipv4_spec, *ipv4_mask; + const struct rte_flow_item_ipv6 *ipv6_spec, *ipv6_mask; + const struct rte_flow_item_tcp *tcp_spec, *tcp_mask; + const struct rte_flow_item_udp *udp_spec, *udp_mask; + const struct rte_flow_item_sctp *sctp_spec, *sctp_mask; + const struct rte_flow_item_gtp *gtp_spec, *gtp_mask; + const struct rte_flow_item_raw *raw_spec, *raw_mask; + const struct rte_flow_item_vf *vf_spec; + + uint8_t pctype = 0; + uint64_t input_set = I40E_INSET_NONE; + uint16_t frag_off; + enum rte_flow_item_type item_type; + enum rte_flow_item_type l3 = RTE_FLOW_ITEM_TYPE_END; + enum rte_flow_item_type cus_proto = RTE_FLOW_ITEM_TYPE_END; + uint32_t i, j; + uint8_t ipv6_addr_mask[16] = { + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF }; + enum i40e_flxpld_layer_idx layer_idx = I40E_FLXPLD_L2_IDX; + uint8_t raw_id = 0; + int32_t off_arr[I40E_MAX_FLXPLD_FIED]; + uint16_t len_arr[I40E_MAX_FLXPLD_FIED]; + struct i40e_fdir_flex_pit flex_pit; + uint8_t next_dst_off = 0; + uint8_t flex_mask[I40E_FDIR_MAX_FLEX_LEN]; + uint16_t flex_size; + bool cfg_flex_pit = true; + bool cfg_flex_msk = true; + uint16_t outer_tpid; + uint16_t ether_type; + uint32_t vtc_flow_cpu; + bool outer_ip = true; + int ret; + + memset(off_arr, 0, sizeof(off_arr)); + memset(len_arr, 0, sizeof(len_arr)); + memset(flex_mask, 0, I40E_FDIR_MAX_FLEX_LEN); + outer_tpid = i40e_get_outer_vlan(dev); + filter->input.flow_ext.customized_pctype = false; + for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { + if (item->last) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Not support range"); + return -rte_errno; + } + item_type = item->type; + switch (item_type) { + case RTE_FLOW_ITEM_TYPE_ETH: + eth_spec = (const struct rte_flow_item_eth *)item->spec; + eth_mask = (const struct rte_flow_item_eth *)item->mask; + + if (eth_spec && eth_mask) { + if (!is_zero_ether_addr(ð_mask->src) || + !is_zero_ether_addr(ð_mask->dst)) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid MAC_addr mask."); + return -rte_errno; + } + + if ((eth_mask->type & UINT16_MAX) == + UINT16_MAX) { + input_set |= I40E_INSET_LAST_ETHER_TYPE; + filter->input.flow.l2_flow.ether_type = + eth_spec->type; + } + + ether_type = rte_be_to_cpu_16(eth_spec->type); + if (ether_type == ETHER_TYPE_IPv4 || + ether_type == ETHER_TYPE_IPv6 || + ether_type == ETHER_TYPE_ARP || + ether_type == outer_tpid) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Unsupported ether_type."); + return -rte_errno; + } + } + + pctype = I40E_FILTER_PCTYPE_L2_PAYLOAD; + layer_idx = I40E_FLXPLD_L2_IDX; + + break; + case RTE_FLOW_ITEM_TYPE_VLAN: + vlan_spec = + (const struct rte_flow_item_vlan *)item->spec; + vlan_mask = + (const struct rte_flow_item_vlan *)item->mask; + if (vlan_spec && vlan_mask) { + if (vlan_mask->tci == + rte_cpu_to_be_16(I40E_TCI_MASK)) { + input_set |= I40E_INSET_VLAN_INNER; + filter->input.flow_ext.vlan_tci = + vlan_spec->tci; + } + } + + pctype = I40E_FILTER_PCTYPE_L2_PAYLOAD; + layer_idx = I40E_FLXPLD_L2_IDX; + + break; + case RTE_FLOW_ITEM_TYPE_IPV4: + l3 = RTE_FLOW_ITEM_TYPE_IPV4; + ipv4_spec = + (const struct rte_flow_item_ipv4 *)item->spec; + ipv4_mask = + (const struct rte_flow_item_ipv4 *)item->mask; + pctype = I40E_FILTER_PCTYPE_NONF_IPV4_OTHER; + layer_idx = I40E_FLXPLD_L3_IDX; + + if (ipv4_spec && ipv4_mask && outer_ip) { + /* Check IPv4 mask and update input set */ + if (ipv4_mask->hdr.version_ihl || + ipv4_mask->hdr.total_length || + ipv4_mask->hdr.packet_id || + ipv4_mask->hdr.fragment_offset || + ipv4_mask->hdr.hdr_checksum) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid IPv4 mask."); + return -rte_errno; + } + + if (ipv4_mask->hdr.src_addr == UINT32_MAX) + input_set |= I40E_INSET_IPV4_SRC; + if (ipv4_mask->hdr.dst_addr == UINT32_MAX) + input_set |= I40E_INSET_IPV4_DST; + if (ipv4_mask->hdr.type_of_service == UINT8_MAX) + input_set |= I40E_INSET_IPV4_TOS; + if (ipv4_mask->hdr.time_to_live == UINT8_MAX) + input_set |= I40E_INSET_IPV4_TTL; + if (ipv4_mask->hdr.next_proto_id == UINT8_MAX) + input_set |= I40E_INSET_IPV4_PROTO; + + /* Check if it is fragment. */ + frag_off = ipv4_spec->hdr.fragment_offset; + frag_off = rte_be_to_cpu_16(frag_off); + if (frag_off & IPV4_HDR_OFFSET_MASK || + frag_off & IPV4_HDR_MF_FLAG) + pctype = I40E_FILTER_PCTYPE_FRAG_IPV4; + + /* Get the filter info */ + filter->input.flow.ip4_flow.proto = + ipv4_spec->hdr.next_proto_id; + filter->input.flow.ip4_flow.tos = + ipv4_spec->hdr.type_of_service; + filter->input.flow.ip4_flow.ttl = + ipv4_spec->hdr.time_to_live; + filter->input.flow.ip4_flow.src_ip = + ipv4_spec->hdr.src_addr; + filter->input.flow.ip4_flow.dst_ip = + ipv4_spec->hdr.dst_addr; + } else if (!ipv4_spec && !ipv4_mask && !outer_ip) { + filter->input.flow_ext.inner_ip = true; + filter->input.flow_ext.iip_type = + I40E_FDIR_IPTYPE_IPV4; + } else if ((ipv4_spec || ipv4_mask) && !outer_ip) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid inner IPv4 mask."); + return -rte_errno; + } + + if (outer_ip) + outer_ip = false; + + break; + case RTE_FLOW_ITEM_TYPE_IPV6: + l3 = RTE_FLOW_ITEM_TYPE_IPV6; + ipv6_spec = + (const struct rte_flow_item_ipv6 *)item->spec; + ipv6_mask = + (const struct rte_flow_item_ipv6 *)item->mask; + pctype = I40E_FILTER_PCTYPE_NONF_IPV6_OTHER; + layer_idx = I40E_FLXPLD_L3_IDX; + + if (ipv6_spec && ipv6_mask && outer_ip) { + /* Check IPv6 mask and update input set */ + if (ipv6_mask->hdr.payload_len) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid IPv6 mask"); + return -rte_errno; + } + + if (!memcmp(ipv6_mask->hdr.src_addr, + ipv6_addr_mask, + RTE_DIM(ipv6_mask->hdr.src_addr))) + input_set |= I40E_INSET_IPV6_SRC; + if (!memcmp(ipv6_mask->hdr.dst_addr, + ipv6_addr_mask, + RTE_DIM(ipv6_mask->hdr.dst_addr))) + input_set |= I40E_INSET_IPV6_DST; + + if ((ipv6_mask->hdr.vtc_flow & + rte_cpu_to_be_32(I40E_IPV6_TC_MASK)) + == rte_cpu_to_be_32(I40E_IPV6_TC_MASK)) + input_set |= I40E_INSET_IPV6_TC; + if (ipv6_mask->hdr.proto == UINT8_MAX) + input_set |= I40E_INSET_IPV6_NEXT_HDR; + if (ipv6_mask->hdr.hop_limits == UINT8_MAX) + input_set |= I40E_INSET_IPV6_HOP_LIMIT; + + /* Get filter info */ + vtc_flow_cpu = + rte_be_to_cpu_32(ipv6_spec->hdr.vtc_flow); + filter->input.flow.ipv6_flow.tc = + (uint8_t)(vtc_flow_cpu >> + I40E_FDIR_IPv6_TC_OFFSET); + filter->input.flow.ipv6_flow.proto = + ipv6_spec->hdr.proto; + filter->input.flow.ipv6_flow.hop_limits = + ipv6_spec->hdr.hop_limits; + + rte_memcpy(filter->input.flow.ipv6_flow.src_ip, + ipv6_spec->hdr.src_addr, 16); + rte_memcpy(filter->input.flow.ipv6_flow.dst_ip, + ipv6_spec->hdr.dst_addr, 16); + + /* Check if it is fragment. */ + if (ipv6_spec->hdr.proto == + I40E_IPV6_FRAG_HEADER) + pctype = I40E_FILTER_PCTYPE_FRAG_IPV6; + } else if (!ipv6_spec && !ipv6_mask && !outer_ip) { + filter->input.flow_ext.inner_ip = true; + filter->input.flow_ext.iip_type = + I40E_FDIR_IPTYPE_IPV6; + } else if ((ipv6_spec || ipv6_mask) && !outer_ip) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid inner IPv6 mask"); + return -rte_errno; + } + + if (outer_ip) + outer_ip = false; + break; + case RTE_FLOW_ITEM_TYPE_TCP: + tcp_spec = (const struct rte_flow_item_tcp *)item->spec; + tcp_mask = (const struct rte_flow_item_tcp *)item->mask; + + if (l3 == RTE_FLOW_ITEM_TYPE_IPV4) + pctype = + I40E_FILTER_PCTYPE_NONF_IPV4_TCP; + else if (l3 == RTE_FLOW_ITEM_TYPE_IPV6) + pctype = + I40E_FILTER_PCTYPE_NONF_IPV6_TCP; + if (tcp_spec && tcp_mask) { + /* Check TCP mask and update input set */ + if (tcp_mask->hdr.sent_seq || + tcp_mask->hdr.recv_ack || + tcp_mask->hdr.data_off || + tcp_mask->hdr.tcp_flags || + tcp_mask->hdr.rx_win || + tcp_mask->hdr.cksum || + tcp_mask->hdr.tcp_urp) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid TCP mask"); + return -rte_errno; + } + + if (tcp_mask->hdr.src_port == UINT16_MAX) + input_set |= I40E_INSET_SRC_PORT; + if (tcp_mask->hdr.dst_port == UINT16_MAX) + input_set |= I40E_INSET_DST_PORT; + + /* Get filter info */ + if (l3 == RTE_FLOW_ITEM_TYPE_IPV4) { + filter->input.flow.tcp4_flow.src_port = + tcp_spec->hdr.src_port; + filter->input.flow.tcp4_flow.dst_port = + tcp_spec->hdr.dst_port; + } else if (l3 == RTE_FLOW_ITEM_TYPE_IPV6) { + filter->input.flow.tcp6_flow.src_port = + tcp_spec->hdr.src_port; + filter->input.flow.tcp6_flow.dst_port = + tcp_spec->hdr.dst_port; + } + } + + layer_idx = I40E_FLXPLD_L4_IDX; + + break; + case RTE_FLOW_ITEM_TYPE_UDP: + udp_spec = (const struct rte_flow_item_udp *)item->spec; + udp_mask = (const struct rte_flow_item_udp *)item->mask; + + if (l3 == RTE_FLOW_ITEM_TYPE_IPV4) + pctype = + I40E_FILTER_PCTYPE_NONF_IPV4_UDP; + else if (l3 == RTE_FLOW_ITEM_TYPE_IPV6) + pctype = + I40E_FILTER_PCTYPE_NONF_IPV6_UDP; + + if (udp_spec && udp_mask) { + /* Check UDP mask and update input set*/ + if (udp_mask->hdr.dgram_len || + udp_mask->hdr.dgram_cksum) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid UDP mask"); + return -rte_errno; + } + + if (udp_mask->hdr.src_port == UINT16_MAX) + input_set |= I40E_INSET_SRC_PORT; + if (udp_mask->hdr.dst_port == UINT16_MAX) + input_set |= I40E_INSET_DST_PORT; + + /* Get filter info */ + if (l3 == RTE_FLOW_ITEM_TYPE_IPV4) { + filter->input.flow.udp4_flow.src_port = + udp_spec->hdr.src_port; + filter->input.flow.udp4_flow.dst_port = + udp_spec->hdr.dst_port; + } else if (l3 == RTE_FLOW_ITEM_TYPE_IPV6) { + filter->input.flow.udp6_flow.src_port = + udp_spec->hdr.src_port; + filter->input.flow.udp6_flow.dst_port = + udp_spec->hdr.dst_port; + } + } + + layer_idx = I40E_FLXPLD_L4_IDX; + + break; + case RTE_FLOW_ITEM_TYPE_GTPC: + case RTE_FLOW_ITEM_TYPE_GTPU: + if (!pf->gtp_support) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Unsupported protocol"); + return -rte_errno; + } + + gtp_spec = (const struct rte_flow_item_gtp *)item->spec; + gtp_mask = (const struct rte_flow_item_gtp *)item->mask; + + if (gtp_spec && gtp_mask) { + if (gtp_mask->v_pt_rsv_flags || + gtp_mask->msg_type || + gtp_mask->msg_len || + gtp_mask->teid != UINT32_MAX) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid GTP mask"); + return -rte_errno; + } + + filter->input.flow.gtp_flow.teid = + gtp_spec->teid; + filter->input.flow_ext.customized_pctype = true; + cus_proto = item_type; + } + break; + case RTE_FLOW_ITEM_TYPE_SCTP: + sctp_spec = + (const struct rte_flow_item_sctp *)item->spec; + sctp_mask = + (const struct rte_flow_item_sctp *)item->mask; + + if (l3 == RTE_FLOW_ITEM_TYPE_IPV4) + pctype = + I40E_FILTER_PCTYPE_NONF_IPV4_SCTP; + else if (l3 == RTE_FLOW_ITEM_TYPE_IPV6) + pctype = + I40E_FILTER_PCTYPE_NONF_IPV6_SCTP; + + if (sctp_spec && sctp_mask) { + /* Check SCTP mask and update input set */ + if (sctp_mask->hdr.cksum) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid UDP mask"); + return -rte_errno; + } + + if (sctp_mask->hdr.src_port == UINT16_MAX) + input_set |= I40E_INSET_SRC_PORT; + if (sctp_mask->hdr.dst_port == UINT16_MAX) + input_set |= I40E_INSET_DST_PORT; + if (sctp_mask->hdr.tag == UINT32_MAX) + input_set |= I40E_INSET_SCTP_VT; + + /* Get filter info */ + if (l3 == RTE_FLOW_ITEM_TYPE_IPV4) { + filter->input.flow.sctp4_flow.src_port = + sctp_spec->hdr.src_port; + filter->input.flow.sctp4_flow.dst_port = + sctp_spec->hdr.dst_port; + filter->input.flow.sctp4_flow.verify_tag + = sctp_spec->hdr.tag; + } else if (l3 == RTE_FLOW_ITEM_TYPE_IPV6) { + filter->input.flow.sctp6_flow.src_port = + sctp_spec->hdr.src_port; + filter->input.flow.sctp6_flow.dst_port = + sctp_spec->hdr.dst_port; + filter->input.flow.sctp6_flow.verify_tag + = sctp_spec->hdr.tag; + } + } + + layer_idx = I40E_FLXPLD_L4_IDX; + + break; + case RTE_FLOW_ITEM_TYPE_RAW: + raw_spec = (const struct rte_flow_item_raw *)item->spec; + raw_mask = (const struct rte_flow_item_raw *)item->mask; + + if (!raw_spec || !raw_mask) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "NULL RAW spec/mask"); + return -rte_errno; + } + + if (pf->support_multi_driver) { + rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Unsupported flexible payload."); + return -rte_errno; + } + + ret = i40e_flow_check_raw_item(item, raw_spec, error); + if (ret < 0) + return ret; + + off_arr[raw_id] = raw_spec->offset; + len_arr[raw_id] = raw_spec->length; + + flex_size = 0; + memset(&flex_pit, 0, sizeof(struct i40e_fdir_flex_pit)); + flex_pit.size = + raw_spec->length / sizeof(uint16_t); + flex_pit.dst_offset = + next_dst_off / sizeof(uint16_t); + + for (i = 0; i <= raw_id; i++) { + if (i == raw_id) + flex_pit.src_offset += + raw_spec->offset / + sizeof(uint16_t); + else + flex_pit.src_offset += + (off_arr[i] + len_arr[i]) / + sizeof(uint16_t); + flex_size += len_arr[i]; + } + if (((flex_pit.src_offset + flex_pit.size) >= + I40E_MAX_FLX_SOURCE_OFF / sizeof(uint16_t)) || + flex_size > I40E_FDIR_MAX_FLEXLEN) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Exceeds maxmial payload limit."); + return -rte_errno; + } + + /* Store flex pit to SW */ + ret = i40e_flow_store_flex_pit(pf, &flex_pit, + layer_idx, raw_id); + if (ret < 0) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Conflict with the first flexible rule."); + return -rte_errno; + } else if (ret > 0) + cfg_flex_pit = false; + + for (i = 0; i < raw_spec->length; i++) { + j = i + next_dst_off; + filter->input.flow_ext.flexbytes[j] = + raw_spec->pattern[i]; + flex_mask[j] = raw_mask->pattern[i]; + } + + next_dst_off += raw_spec->length; + raw_id++; + break; + case RTE_FLOW_ITEM_TYPE_VF: + vf_spec = (const struct rte_flow_item_vf *)item->spec; + filter->input.flow_ext.is_vf = 1; + filter->input.flow_ext.dst_id = vf_spec->id; + if (filter->input.flow_ext.is_vf && + filter->input.flow_ext.dst_id >= pf->vf_num) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid VF ID for FDIR."); + return -rte_errno; + } + break; + default: + break; + } + } + + /* Get customized pctype value */ + if (filter->input.flow_ext.customized_pctype) { + pctype = i40e_flow_fdir_get_pctype_value(pf, cus_proto, filter); + if (pctype == I40E_FILTER_PCTYPE_INVALID) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Unsupported pctype"); + return -rte_errno; + } + } + + /* If customized pctype is not used, set fdir configuration.*/ + if (!filter->input.flow_ext.customized_pctype) { + ret = i40e_flow_set_fdir_inset(pf, pctype, input_set); + if (ret == -1) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, item, + "Conflict with the first rule's input set."); + return -rte_errno; + } else if (ret == -EINVAL) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, item, + "Invalid pattern mask."); + return -rte_errno; + } + + /* Store flex mask to SW */ + ret = i40e_flow_store_flex_mask(pf, pctype, flex_mask); + if (ret == -1) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Exceed maximal number of bitmasks"); + return -rte_errno; + } else if (ret == -2) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Conflict with the first flexible rule"); + return -rte_errno; + } else if (ret > 0) + cfg_flex_msk = false; + + if (cfg_flex_pit) + i40e_flow_set_fdir_flex_pit(pf, layer_idx, raw_id); + + if (cfg_flex_msk) + i40e_flow_set_fdir_flex_msk(pf, pctype); + } + + filter->input.pctype = pctype; + + return 0; +} + +/* Parse to get the action info of a FDIR filter. + * FDIR action supports QUEUE or (QUEUE + MARK). + */ +static int +i40e_flow_parse_fdir_action(struct rte_eth_dev *dev, + const struct rte_flow_action *actions, + struct rte_flow_error *error, + struct i40e_fdir_filter_conf *filter) +{ + struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + const struct rte_flow_action *act; + const struct rte_flow_action_queue *act_q; + const struct rte_flow_action_mark *mark_spec; + uint32_t index = 0; + + /* Check if the first non-void action is QUEUE or DROP or PASSTHRU. */ + NEXT_ITEM_OF_ACTION(act, actions, index); + switch (act->type) { + case RTE_FLOW_ACTION_TYPE_QUEUE: + act_q = (const struct rte_flow_action_queue *)act->conf; + filter->action.rx_queue = act_q->index; + if ((!filter->input.flow_ext.is_vf && + filter->action.rx_queue >= pf->dev_data->nb_rx_queues) || + (filter->input.flow_ext.is_vf && + filter->action.rx_queue >= pf->vf_nb_qps)) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, act, + "Invalid queue ID for FDIR."); + return -rte_errno; + } + filter->action.behavior = I40E_FDIR_ACCEPT; + break; + case RTE_FLOW_ACTION_TYPE_DROP: + filter->action.behavior = I40E_FDIR_REJECT; + break; + case RTE_FLOW_ACTION_TYPE_PASSTHRU: + filter->action.behavior = I40E_FDIR_PASSTHRU; + break; + default: + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, act, + "Invalid action."); + return -rte_errno; + } + + /* Check if the next non-void item is MARK or FLAG or END. */ + index++; + NEXT_ITEM_OF_ACTION(act, actions, index); + switch (act->type) { + case RTE_FLOW_ACTION_TYPE_MARK: + mark_spec = (const struct rte_flow_action_mark *)act->conf; + filter->action.report_status = I40E_FDIR_REPORT_ID; + filter->soft_id = mark_spec->id; + break; + case RTE_FLOW_ACTION_TYPE_FLAG: + filter->action.report_status = I40E_FDIR_NO_REPORT_STATUS; + break; + case RTE_FLOW_ACTION_TYPE_END: + return 0; + default: + rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, + act, "Invalid action."); + return -rte_errno; + } + + /* Check if the next non-void item is END */ + index++; + NEXT_ITEM_OF_ACTION(act, actions, index); + if (act->type != RTE_FLOW_ACTION_TYPE_END) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + act, "Invalid action."); + return -rte_errno; + } + + return 0; +} + +static int +i40e_flow_parse_fdir_filter(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error, + union i40e_filter_t *filter) +{ + struct i40e_fdir_filter_conf *fdir_filter = + &filter->fdir_filter; + int ret; + + ret = i40e_flow_parse_fdir_pattern(dev, pattern, error, fdir_filter); + if (ret) + return ret; + + ret = i40e_flow_parse_fdir_action(dev, actions, error, fdir_filter); + if (ret) + return ret; + + ret = i40e_flow_parse_attr(attr, error); + if (ret) + return ret; + + cons_filter_type = RTE_ETH_FILTER_FDIR; + + if (dev->data->dev_conf.fdir_conf.mode != + RTE_FDIR_MODE_PERFECT) { + rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, + "Check the mode in fdir_conf."); + return -rte_errno; + } + + return 0; +} + +/* Parse to get the action info of a tunnel filter + * Tunnel action only supports PF, VF and QUEUE. + */ +static int +i40e_flow_parse_tunnel_action(struct rte_eth_dev *dev, + const struct rte_flow_action *actions, + struct rte_flow_error *error, + struct i40e_tunnel_filter_conf *filter) +{ + struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + const struct rte_flow_action *act; + const struct rte_flow_action_queue *act_q; + const struct rte_flow_action_vf *act_vf; + uint32_t index = 0; + + /* Check if the first non-void action is PF or VF. */ + NEXT_ITEM_OF_ACTION(act, actions, index); + if (act->type != RTE_FLOW_ACTION_TYPE_PF && + act->type != RTE_FLOW_ACTION_TYPE_VF) { + rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, + act, "Not supported action."); + return -rte_errno; + } + + if (act->type == RTE_FLOW_ACTION_TYPE_VF) { + act_vf = (const struct rte_flow_action_vf *)act->conf; + filter->vf_id = act_vf->id; + filter->is_to_vf = 1; + if (filter->vf_id >= pf->vf_num) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + act, "Invalid VF ID for tunnel filter"); + return -rte_errno; + } + } + + /* Check if the next non-void item is QUEUE */ + index++; + NEXT_ITEM_OF_ACTION(act, actions, index); + if (act->type == RTE_FLOW_ACTION_TYPE_QUEUE) { + act_q = (const struct rte_flow_action_queue *)act->conf; + filter->queue_id = act_q->index; + if ((!filter->is_to_vf) && + (filter->queue_id >= pf->dev_data->nb_rx_queues)) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + act, "Invalid queue ID for tunnel filter"); + return -rte_errno; + } else if (filter->is_to_vf && + (filter->queue_id >= pf->vf_nb_qps)) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + act, "Invalid queue ID for tunnel filter"); + return -rte_errno; + } + } + + /* Check if the next non-void item is END */ + index++; + NEXT_ITEM_OF_ACTION(act, actions, index); + if (act->type != RTE_FLOW_ACTION_TYPE_END) { + rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, + act, "Not supported action."); + return -rte_errno; + } + + return 0; +} + +static uint16_t i40e_supported_tunnel_filter_types[] = { + ETH_TUNNEL_FILTER_IMAC | ETH_TUNNEL_FILTER_TENID | + ETH_TUNNEL_FILTER_IVLAN, + ETH_TUNNEL_FILTER_IMAC | ETH_TUNNEL_FILTER_IVLAN, + ETH_TUNNEL_FILTER_IMAC | ETH_TUNNEL_FILTER_TENID, + ETH_TUNNEL_FILTER_OMAC | ETH_TUNNEL_FILTER_TENID | + ETH_TUNNEL_FILTER_IMAC, + ETH_TUNNEL_FILTER_IMAC, +}; + +static int +i40e_check_tunnel_filter_type(uint8_t filter_type) +{ + uint8_t i; + + for (i = 0; i < RTE_DIM(i40e_supported_tunnel_filter_types); i++) { + if (filter_type == i40e_supported_tunnel_filter_types[i]) + return 0; + } + + return -1; +} + +/* 1. Last in item should be NULL as range is not supported. + * 2. Supported filter types: IMAC_IVLAN_TENID, IMAC_IVLAN, + * IMAC_TENID, OMAC_TENID_IMAC and IMAC. + * 3. Mask of fields which need to be matched should be + * filled with 1. + * 4. Mask of fields which needn't to be matched should be + * filled with 0. + */ +static int +i40e_flow_parse_vxlan_pattern(__rte_unused struct rte_eth_dev *dev, + const struct rte_flow_item *pattern, + struct rte_flow_error *error, + struct i40e_tunnel_filter_conf *filter) +{ + const struct rte_flow_item *item = pattern; + const struct rte_flow_item_eth *eth_spec; + const struct rte_flow_item_eth *eth_mask; + const struct rte_flow_item_vxlan *vxlan_spec; + const struct rte_flow_item_vxlan *vxlan_mask; + const struct rte_flow_item_vlan *vlan_spec; + const struct rte_flow_item_vlan *vlan_mask; + uint8_t filter_type = 0; + bool is_vni_masked = 0; + uint8_t vni_mask[] = {0xFF, 0xFF, 0xFF}; + enum rte_flow_item_type item_type; + bool vxlan_flag = 0; + uint32_t tenant_id_be = 0; + int ret; + + for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { + if (item->last) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Not support range"); + return -rte_errno; + } + item_type = item->type; + switch (item_type) { + case RTE_FLOW_ITEM_TYPE_ETH: + eth_spec = (const struct rte_flow_item_eth *)item->spec; + eth_mask = (const struct rte_flow_item_eth *)item->mask; + + /* Check if ETH item is used for place holder. + * If yes, both spec and mask should be NULL. + * If no, both spec and mask shouldn't be NULL. + */ + if ((!eth_spec && eth_mask) || + (eth_spec && !eth_mask)) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid ether spec/mask"); + return -rte_errno; + } + + if (eth_spec && eth_mask) { + /* DST address of inner MAC shouldn't be masked. + * SRC address of Inner MAC should be masked. + */ + if (!is_broadcast_ether_addr(ð_mask->dst) || + !is_zero_ether_addr(ð_mask->src) || + eth_mask->type) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid ether spec/mask"); + return -rte_errno; + } + + if (!vxlan_flag) { + rte_memcpy(&filter->outer_mac, + ð_spec->dst, + ETHER_ADDR_LEN); + filter_type |= ETH_TUNNEL_FILTER_OMAC; + } else { + rte_memcpy(&filter->inner_mac, + ð_spec->dst, + ETHER_ADDR_LEN); + filter_type |= ETH_TUNNEL_FILTER_IMAC; + } + } + break; + case RTE_FLOW_ITEM_TYPE_VLAN: + vlan_spec = + (const struct rte_flow_item_vlan *)item->spec; + vlan_mask = + (const struct rte_flow_item_vlan *)item->mask; + if (!(vlan_spec && vlan_mask)) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid vlan item"); + return -rte_errno; + } + + if (vlan_spec && vlan_mask) { + if (vlan_mask->tci == + rte_cpu_to_be_16(I40E_TCI_MASK)) + filter->inner_vlan = + rte_be_to_cpu_16(vlan_spec->tci) & + I40E_TCI_MASK; + filter_type |= ETH_TUNNEL_FILTER_IVLAN; + } + break; + case RTE_FLOW_ITEM_TYPE_IPV4: + filter->ip_type = I40E_TUNNEL_IPTYPE_IPV4; + /* IPv4 is used to describe protocol, + * spec and mask should be NULL. + */ + if (item->spec || item->mask) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid IPv4 item"); + return -rte_errno; + } + break; + case RTE_FLOW_ITEM_TYPE_IPV6: + filter->ip_type = I40E_TUNNEL_IPTYPE_IPV6; + /* IPv6 is used to describe protocol, + * spec and mask should be NULL. + */ + if (item->spec || item->mask) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid IPv6 item"); + return -rte_errno; + } + break; + case RTE_FLOW_ITEM_TYPE_UDP: + /* UDP is used to describe protocol, + * spec and mask should be NULL. + */ + if (item->spec || item->mask) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid UDP item"); + return -rte_errno; + } + break; + case RTE_FLOW_ITEM_TYPE_VXLAN: + vxlan_spec = + (const struct rte_flow_item_vxlan *)item->spec; + vxlan_mask = + (const struct rte_flow_item_vxlan *)item->mask; + /* Check if VXLAN item is used to describe protocol. + * If yes, both spec and mask should be NULL. + * If no, both spec and mask shouldn't be NULL. + */ + if ((!vxlan_spec && vxlan_mask) || + (vxlan_spec && !vxlan_mask)) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid VXLAN item"); + return -rte_errno; + } + + /* Check if VNI is masked. */ + if (vxlan_spec && vxlan_mask) { + is_vni_masked = + !!memcmp(vxlan_mask->vni, vni_mask, + RTE_DIM(vni_mask)); + if (is_vni_masked) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid VNI mask"); + return -rte_errno; + } + + rte_memcpy(((uint8_t *)&tenant_id_be + 1), + vxlan_spec->vni, 3); + filter->tenant_id = + rte_be_to_cpu_32(tenant_id_be); + filter_type |= ETH_TUNNEL_FILTER_TENID; + } + + vxlan_flag = 1; + break; + default: + break; + } + } + + ret = i40e_check_tunnel_filter_type(filter_type); + if (ret < 0) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + NULL, + "Invalid filter type"); + return -rte_errno; + } + filter->filter_type = filter_type; + + filter->tunnel_type = I40E_TUNNEL_TYPE_VXLAN; + + return 0; +} + +static int +i40e_flow_parse_vxlan_filter(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error, + union i40e_filter_t *filter) +{ + struct i40e_tunnel_filter_conf *tunnel_filter = + &filter->consistent_tunnel_filter; + int ret; + + ret = i40e_flow_parse_vxlan_pattern(dev, pattern, + error, tunnel_filter); + if (ret) + return ret; + + ret = i40e_flow_parse_tunnel_action(dev, actions, error, tunnel_filter); + if (ret) + return ret; + + ret = i40e_flow_parse_attr(attr, error); + if (ret) + return ret; + + cons_filter_type = RTE_ETH_FILTER_TUNNEL; + + return ret; +} + +/* 1. Last in item should be NULL as range is not supported. + * 2. Supported filter types: IMAC_IVLAN_TENID, IMAC_IVLAN, + * IMAC_TENID, OMAC_TENID_IMAC and IMAC. + * 3. Mask of fields which need to be matched should be + * filled with 1. + * 4. Mask of fields which needn't to be matched should be + * filled with 0. + */ +static int +i40e_flow_parse_nvgre_pattern(__rte_unused struct rte_eth_dev *dev, + const struct rte_flow_item *pattern, + struct rte_flow_error *error, + struct i40e_tunnel_filter_conf *filter) +{ + const struct rte_flow_item *item = pattern; + const struct rte_flow_item_eth *eth_spec; + const struct rte_flow_item_eth *eth_mask; + const struct rte_flow_item_nvgre *nvgre_spec; + const struct rte_flow_item_nvgre *nvgre_mask; + const struct rte_flow_item_vlan *vlan_spec; + const struct rte_flow_item_vlan *vlan_mask; + enum rte_flow_item_type item_type; + uint8_t filter_type = 0; + bool is_tni_masked = 0; + uint8_t tni_mask[] = {0xFF, 0xFF, 0xFF}; + bool nvgre_flag = 0; + uint32_t tenant_id_be = 0; + int ret; + + for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { + if (item->last) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Not support range"); + return -rte_errno; + } + item_type = item->type; + switch (item_type) { + case RTE_FLOW_ITEM_TYPE_ETH: + eth_spec = (const struct rte_flow_item_eth *)item->spec; + eth_mask = (const struct rte_flow_item_eth *)item->mask; + + /* Check if ETH item is used for place holder. + * If yes, both spec and mask should be NULL. + * If no, both spec and mask shouldn't be NULL. + */ + if ((!eth_spec && eth_mask) || + (eth_spec && !eth_mask)) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid ether spec/mask"); + return -rte_errno; + } + + if (eth_spec && eth_mask) { + /* DST address of inner MAC shouldn't be masked. + * SRC address of Inner MAC should be masked. + */ + if (!is_broadcast_ether_addr(ð_mask->dst) || + !is_zero_ether_addr(ð_mask->src) || + eth_mask->type) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid ether spec/mask"); + return -rte_errno; + } + + if (!nvgre_flag) { + rte_memcpy(&filter->outer_mac, + ð_spec->dst, + ETHER_ADDR_LEN); + filter_type |= ETH_TUNNEL_FILTER_OMAC; + } else { + rte_memcpy(&filter->inner_mac, + ð_spec->dst, + ETHER_ADDR_LEN); + filter_type |= ETH_TUNNEL_FILTER_IMAC; + } + } + + break; + case RTE_FLOW_ITEM_TYPE_VLAN: + vlan_spec = + (const struct rte_flow_item_vlan *)item->spec; + vlan_mask = + (const struct rte_flow_item_vlan *)item->mask; + if (!(vlan_spec && vlan_mask)) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid vlan item"); + return -rte_errno; + } + + if (vlan_spec && vlan_mask) { + if (vlan_mask->tci == + rte_cpu_to_be_16(I40E_TCI_MASK)) + filter->inner_vlan = + rte_be_to_cpu_16(vlan_spec->tci) & + I40E_TCI_MASK; + filter_type |= ETH_TUNNEL_FILTER_IVLAN; + } + break; + case RTE_FLOW_ITEM_TYPE_IPV4: + filter->ip_type = I40E_TUNNEL_IPTYPE_IPV4; + /* IPv4 is used to describe protocol, + * spec and mask should be NULL. + */ + if (item->spec || item->mask) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid IPv4 item"); + return -rte_errno; + } + break; + case RTE_FLOW_ITEM_TYPE_IPV6: + filter->ip_type = I40E_TUNNEL_IPTYPE_IPV6; + /* IPv6 is used to describe protocol, + * spec and mask should be NULL. + */ + if (item->spec || item->mask) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid IPv6 item"); + return -rte_errno; + } + break; + case RTE_FLOW_ITEM_TYPE_NVGRE: + nvgre_spec = + (const struct rte_flow_item_nvgre *)item->spec; + nvgre_mask = + (const struct rte_flow_item_nvgre *)item->mask; + /* Check if NVGRE item is used to describe protocol. + * If yes, both spec and mask should be NULL. + * If no, both spec and mask shouldn't be NULL. + */ + if ((!nvgre_spec && nvgre_mask) || + (nvgre_spec && !nvgre_mask)) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid NVGRE item"); + return -rte_errno; + } + + if (nvgre_spec && nvgre_mask) { + is_tni_masked = + !!memcmp(nvgre_mask->tni, tni_mask, + RTE_DIM(tni_mask)); + if (is_tni_masked) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid TNI mask"); + return -rte_errno; + } + if (nvgre_mask->protocol && + nvgre_mask->protocol != 0xFFFF) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid NVGRE item"); + return -rte_errno; + } + if (nvgre_mask->c_k_s_rsvd0_ver && + nvgre_mask->c_k_s_rsvd0_ver != + rte_cpu_to_be_16(0xFFFF)) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid NVGRE item"); + return -rte_errno; + } + if (nvgre_spec->c_k_s_rsvd0_ver != + rte_cpu_to_be_16(0x2000) && + nvgre_mask->c_k_s_rsvd0_ver) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid NVGRE item"); + return -rte_errno; + } + if (nvgre_mask->protocol && + nvgre_spec->protocol != + rte_cpu_to_be_16(0x6558)) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid NVGRE item"); + return -rte_errno; + } + rte_memcpy(((uint8_t *)&tenant_id_be + 1), + nvgre_spec->tni, 3); + filter->tenant_id = + rte_be_to_cpu_32(tenant_id_be); + filter_type |= ETH_TUNNEL_FILTER_TENID; + } + + nvgre_flag = 1; + break; + default: + break; + } + } + + ret = i40e_check_tunnel_filter_type(filter_type); + if (ret < 0) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + NULL, + "Invalid filter type"); + return -rte_errno; + } + filter->filter_type = filter_type; + + filter->tunnel_type = I40E_TUNNEL_TYPE_NVGRE; + + return 0; +} + +static int +i40e_flow_parse_nvgre_filter(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error, + union i40e_filter_t *filter) +{ + struct i40e_tunnel_filter_conf *tunnel_filter = + &filter->consistent_tunnel_filter; + int ret; + + ret = i40e_flow_parse_nvgre_pattern(dev, pattern, + error, tunnel_filter); + if (ret) + return ret; + + ret = i40e_flow_parse_tunnel_action(dev, actions, error, tunnel_filter); + if (ret) + return ret; + + ret = i40e_flow_parse_attr(attr, error); + if (ret) + return ret; + + cons_filter_type = RTE_ETH_FILTER_TUNNEL; + + return ret; +} + +/* 1. Last in item should be NULL as range is not supported. + * 2. Supported filter types: MPLS label. + * 3. Mask of fields which need to be matched should be + * filled with 1. + * 4. Mask of fields which needn't to be matched should be + * filled with 0. + */ +static int +i40e_flow_parse_mpls_pattern(__rte_unused struct rte_eth_dev *dev, + const struct rte_flow_item *pattern, + struct rte_flow_error *error, + struct i40e_tunnel_filter_conf *filter) +{ + const struct rte_flow_item *item = pattern; + const struct rte_flow_item_mpls *mpls_spec; + const struct rte_flow_item_mpls *mpls_mask; + enum rte_flow_item_type item_type; + bool is_mplsoudp = 0; /* 1 - MPLSoUDP, 0 - MPLSoGRE */ + const uint8_t label_mask[3] = {0xFF, 0xFF, 0xF0}; + uint32_t label_be = 0; + + for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { + if (item->last) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Not support range"); + return -rte_errno; + } + item_type = item->type; + switch (item_type) { + case RTE_FLOW_ITEM_TYPE_ETH: + if (item->spec || item->mask) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid ETH item"); + return -rte_errno; + } + break; + case RTE_FLOW_ITEM_TYPE_IPV4: + filter->ip_type = I40E_TUNNEL_IPTYPE_IPV4; + /* IPv4 is used to describe protocol, + * spec and mask should be NULL. + */ + if (item->spec || item->mask) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid IPv4 item"); + return -rte_errno; + } + break; + case RTE_FLOW_ITEM_TYPE_IPV6: + filter->ip_type = I40E_TUNNEL_IPTYPE_IPV6; + /* IPv6 is used to describe protocol, + * spec and mask should be NULL. + */ + if (item->spec || item->mask) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid IPv6 item"); + return -rte_errno; + } + break; + case RTE_FLOW_ITEM_TYPE_UDP: + /* UDP is used to describe protocol, + * spec and mask should be NULL. + */ + if (item->spec || item->mask) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid UDP item"); + return -rte_errno; + } + is_mplsoudp = 1; + break; + case RTE_FLOW_ITEM_TYPE_GRE: + /* GRE is used to describe protocol, + * spec and mask should be NULL. + */ + if (item->spec || item->mask) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid GRE item"); + return -rte_errno; + } + break; + case RTE_FLOW_ITEM_TYPE_MPLS: + mpls_spec = + (const struct rte_flow_item_mpls *)item->spec; + mpls_mask = + (const struct rte_flow_item_mpls *)item->mask; + + if (!mpls_spec || !mpls_mask) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid MPLS item"); + return -rte_errno; + } + + if (memcmp(mpls_mask->label_tc_s, label_mask, 3)) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid MPLS label mask"); + return -rte_errno; + } + rte_memcpy(((uint8_t *)&label_be + 1), + mpls_spec->label_tc_s, 3); + filter->tenant_id = rte_be_to_cpu_32(label_be) >> 4; + break; + default: + break; + } + } + + if (is_mplsoudp) + filter->tunnel_type = I40E_TUNNEL_TYPE_MPLSoUDP; + else + filter->tunnel_type = I40E_TUNNEL_TYPE_MPLSoGRE; + + return 0; +} + +static int +i40e_flow_parse_mpls_filter(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error, + union i40e_filter_t *filter) +{ + struct i40e_tunnel_filter_conf *tunnel_filter = + &filter->consistent_tunnel_filter; + int ret; + + ret = i40e_flow_parse_mpls_pattern(dev, pattern, + error, tunnel_filter); + if (ret) + return ret; + + ret = i40e_flow_parse_tunnel_action(dev, actions, error, tunnel_filter); + if (ret) + return ret; + + ret = i40e_flow_parse_attr(attr, error); + if (ret) + return ret; + + cons_filter_type = RTE_ETH_FILTER_TUNNEL; + + return ret; +} + +/* 1. Last in item should be NULL as range is not supported. + * 2. Supported filter types: GTP TEID. + * 3. Mask of fields which need to be matched should be + * filled with 1. + * 4. Mask of fields which needn't to be matched should be + * filled with 0. + * 5. GTP profile supports GTPv1 only. + * 6. GTP-C response message ('source_port' = 2123) is not supported. + */ +static int +i40e_flow_parse_gtp_pattern(struct rte_eth_dev *dev, + const struct rte_flow_item *pattern, + struct rte_flow_error *error, + struct i40e_tunnel_filter_conf *filter) +{ + struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + const struct rte_flow_item *item = pattern; + const struct rte_flow_item_gtp *gtp_spec; + const struct rte_flow_item_gtp *gtp_mask; + enum rte_flow_item_type item_type; + + if (!pf->gtp_support) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "GTP is not supported by default."); + return -rte_errno; + } + + for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { + if (item->last) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Not support range"); + return -rte_errno; + } + item_type = item->type; + switch (item_type) { + case RTE_FLOW_ITEM_TYPE_ETH: + if (item->spec || item->mask) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid ETH item"); + return -rte_errno; + } + break; + case RTE_FLOW_ITEM_TYPE_IPV4: + filter->ip_type = I40E_TUNNEL_IPTYPE_IPV4; + /* IPv4 is used to describe protocol, + * spec and mask should be NULL. + */ + if (item->spec || item->mask) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid IPv4 item"); + return -rte_errno; + } + break; + case RTE_FLOW_ITEM_TYPE_UDP: + if (item->spec || item->mask) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid UDP item"); + return -rte_errno; + } + break; + case RTE_FLOW_ITEM_TYPE_GTPC: + case RTE_FLOW_ITEM_TYPE_GTPU: + gtp_spec = + (const struct rte_flow_item_gtp *)item->spec; + gtp_mask = + (const struct rte_flow_item_gtp *)item->mask; + + if (!gtp_spec || !gtp_mask) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid GTP item"); + return -rte_errno; + } + + if (gtp_mask->v_pt_rsv_flags || + gtp_mask->msg_type || + gtp_mask->msg_len || + gtp_mask->teid != UINT32_MAX) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid GTP mask"); + return -rte_errno; + } + + if (item_type == RTE_FLOW_ITEM_TYPE_GTPC) + filter->tunnel_type = I40E_TUNNEL_TYPE_GTPC; + else if (item_type == RTE_FLOW_ITEM_TYPE_GTPU) + filter->tunnel_type = I40E_TUNNEL_TYPE_GTPU; + + filter->tenant_id = rte_be_to_cpu_32(gtp_spec->teid); + + break; + default: + break; + } + } + + return 0; +} + +static int +i40e_flow_parse_gtp_filter(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error, + union i40e_filter_t *filter) +{ + struct i40e_tunnel_filter_conf *tunnel_filter = + &filter->consistent_tunnel_filter; + int ret; + + ret = i40e_flow_parse_gtp_pattern(dev, pattern, + error, tunnel_filter); + if (ret) + return ret; + + ret = i40e_flow_parse_tunnel_action(dev, actions, error, tunnel_filter); + if (ret) + return ret; + + ret = i40e_flow_parse_attr(attr, error); + if (ret) + return ret; + + cons_filter_type = RTE_ETH_FILTER_TUNNEL; + + return ret; +} + +/* 1. Last in item should be NULL as range is not supported. + * 2. Supported filter types: QINQ. + * 3. Mask of fields which need to be matched should be + * filled with 1. + * 4. Mask of fields which needn't to be matched should be + * filled with 0. + */ +static int +i40e_flow_parse_qinq_pattern(__rte_unused struct rte_eth_dev *dev, + const struct rte_flow_item *pattern, + struct rte_flow_error *error, + struct i40e_tunnel_filter_conf *filter) +{ + const struct rte_flow_item *item = pattern; + const struct rte_flow_item_vlan *vlan_spec = NULL; + const struct rte_flow_item_vlan *vlan_mask = NULL; + const struct rte_flow_item_vlan *i_vlan_spec = NULL; + const struct rte_flow_item_vlan *i_vlan_mask = NULL; + const struct rte_flow_item_vlan *o_vlan_spec = NULL; + const struct rte_flow_item_vlan *o_vlan_mask = NULL; + + enum rte_flow_item_type item_type; + bool vlan_flag = 0; + + for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { + if (item->last) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Not support range"); + return -rte_errno; + } + item_type = item->type; + switch (item_type) { + case RTE_FLOW_ITEM_TYPE_ETH: + if (item->spec || item->mask) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid ETH item"); + return -rte_errno; + } + break; + case RTE_FLOW_ITEM_TYPE_VLAN: + vlan_spec = + (const struct rte_flow_item_vlan *)item->spec; + vlan_mask = + (const struct rte_flow_item_vlan *)item->mask; + + if (!(vlan_spec && vlan_mask)) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Invalid vlan item"); + return -rte_errno; + } + + if (!vlan_flag) { + o_vlan_spec = vlan_spec; + o_vlan_mask = vlan_mask; + vlan_flag = 1; + } else { + i_vlan_spec = vlan_spec; + i_vlan_mask = vlan_mask; + vlan_flag = 0; + } + break; + + default: + break; + } + } + + /* Get filter specification */ + if ((o_vlan_mask != NULL) && (o_vlan_mask->tci == + rte_cpu_to_be_16(I40E_TCI_MASK)) && + (i_vlan_mask != NULL) && + (i_vlan_mask->tci == rte_cpu_to_be_16(I40E_TCI_MASK))) { + filter->outer_vlan = rte_be_to_cpu_16(o_vlan_spec->tci) + & I40E_TCI_MASK; + filter->inner_vlan = rte_be_to_cpu_16(i_vlan_spec->tci) + & I40E_TCI_MASK; + } else { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + NULL, + "Invalid filter type"); + return -rte_errno; + } + + filter->tunnel_type = I40E_TUNNEL_TYPE_QINQ; + return 0; +} + +static int +i40e_flow_parse_qinq_filter(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error, + union i40e_filter_t *filter) +{ + struct i40e_tunnel_filter_conf *tunnel_filter = + &filter->consistent_tunnel_filter; + int ret; + + ret = i40e_flow_parse_qinq_pattern(dev, pattern, + error, tunnel_filter); + if (ret) + return ret; + + ret = i40e_flow_parse_tunnel_action(dev, actions, error, tunnel_filter); + if (ret) + return ret; + + ret = i40e_flow_parse_attr(attr, error); + if (ret) + return ret; + + cons_filter_type = RTE_ETH_FILTER_TUNNEL; + + return ret; +} + +static int +i40e_flow_validate(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error) +{ + struct rte_flow_item *items; /* internal pattern w/o VOID items */ + parse_filter_t parse_filter; + uint32_t item_num = 0; /* non-void item number of pattern*/ + uint32_t i = 0; + bool flag = false; + int ret = I40E_NOT_SUPPORTED; + + if (!pattern) { + rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_NUM, + NULL, "NULL pattern."); + return -rte_errno; + } + + if (!actions) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION_NUM, + NULL, "NULL action."); + return -rte_errno; + } + + if (!attr) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR, + NULL, "NULL attribute."); + return -rte_errno; + } + + memset(&cons_filter, 0, sizeof(cons_filter)); + + /* Get the non-void item number of pattern */ + while ((pattern + i)->type != RTE_FLOW_ITEM_TYPE_END) { + if ((pattern + i)->type != RTE_FLOW_ITEM_TYPE_VOID) + item_num++; + i++; + } + item_num++; + + items = rte_zmalloc("i40e_pattern", + item_num * sizeof(struct rte_flow_item), 0); + if (!items) { + rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_ITEM_NUM, + NULL, "No memory for PMD internal items."); + return -ENOMEM; + } + + i40e_pattern_skip_void_item(items, pattern); + + i = 0; + do { + parse_filter = i40e_find_parse_filter_func(items, &i); + if (!parse_filter && !flag) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + pattern, "Unsupported pattern"); + rte_free(items); + return -rte_errno; + } + if (parse_filter) + ret = parse_filter(dev, attr, items, actions, + error, &cons_filter); + flag = true; + } while ((ret < 0) && (i < RTE_DIM(i40e_supported_patterns))); + + rte_free(items); + + return ret; +} + +static struct rte_flow * +i40e_flow_create(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error) +{ + struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + struct rte_flow *flow; + int ret; + + flow = rte_zmalloc("i40e_flow", sizeof(struct rte_flow), 0); + if (!flow) { + rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_HANDLE, NULL, + "Failed to allocate memory"); + return flow; + } + + ret = i40e_flow_validate(dev, attr, pattern, actions, error); + if (ret < 0) + return NULL; + + switch (cons_filter_type) { + case RTE_ETH_FILTER_ETHERTYPE: + ret = i40e_ethertype_filter_set(pf, + &cons_filter.ethertype_filter, 1); + if (ret) + goto free_flow; + flow->rule = TAILQ_LAST(&pf->ethertype.ethertype_list, + i40e_ethertype_filter_list); + break; + case RTE_ETH_FILTER_FDIR: + ret = i40e_flow_add_del_fdir_filter(dev, + &cons_filter.fdir_filter, 1); + if (ret) + goto free_flow; + flow->rule = TAILQ_LAST(&pf->fdir.fdir_list, + i40e_fdir_filter_list); + break; + case RTE_ETH_FILTER_TUNNEL: + ret = i40e_dev_consistent_tunnel_filter_set(pf, + &cons_filter.consistent_tunnel_filter, 1); + if (ret) + goto free_flow; + flow->rule = TAILQ_LAST(&pf->tunnel.tunnel_list, + i40e_tunnel_filter_list); + break; + default: + goto free_flow; + } + + flow->filter_type = cons_filter_type; + TAILQ_INSERT_TAIL(&pf->flow_list, flow, node); + return flow; + +free_flow: + rte_flow_error_set(error, -ret, + RTE_FLOW_ERROR_TYPE_HANDLE, NULL, + "Failed to create flow."); + rte_free(flow); + return NULL; +} + +static int +i40e_flow_destroy(struct rte_eth_dev *dev, + struct rte_flow *flow, + struct rte_flow_error *error) +{ + struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + enum rte_filter_type filter_type = flow->filter_type; + int ret = 0; + + switch (filter_type) { + case RTE_ETH_FILTER_ETHERTYPE: + ret = i40e_flow_destroy_ethertype_filter(pf, + (struct i40e_ethertype_filter *)flow->rule); + break; + case RTE_ETH_FILTER_TUNNEL: + ret = i40e_flow_destroy_tunnel_filter(pf, + (struct i40e_tunnel_filter *)flow->rule); + break; + case RTE_ETH_FILTER_FDIR: + ret = i40e_flow_add_del_fdir_filter(dev, + &((struct i40e_fdir_filter *)flow->rule)->fdir, 0); + break; + default: + PMD_DRV_LOG(WARNING, "Filter type (%d) not supported", + filter_type); + ret = -EINVAL; + break; + } + + if (!ret) { + TAILQ_REMOVE(&pf->flow_list, flow, node); + rte_free(flow); + } else + rte_flow_error_set(error, -ret, + RTE_FLOW_ERROR_TYPE_HANDLE, NULL, + "Failed to destroy flow."); + + return ret; +} + +static int +i40e_flow_destroy_ethertype_filter(struct i40e_pf *pf, + struct i40e_ethertype_filter *filter) +{ + struct i40e_hw *hw = I40E_PF_TO_HW(pf); + struct i40e_ethertype_rule *ethertype_rule = &pf->ethertype; + struct i40e_ethertype_filter *node; + struct i40e_control_filter_stats stats; + uint16_t flags = 0; + int ret = 0; + + if (!(filter->flags & RTE_ETHTYPE_FLAGS_MAC)) + flags |= I40E_AQC_ADD_CONTROL_PACKET_FLAGS_IGNORE_MAC; + if (filter->flags & RTE_ETHTYPE_FLAGS_DROP) + flags |= I40E_AQC_ADD_CONTROL_PACKET_FLAGS_DROP; + flags |= I40E_AQC_ADD_CONTROL_PACKET_FLAGS_TO_QUEUE; + + memset(&stats, 0, sizeof(stats)); + ret = i40e_aq_add_rem_control_packet_filter(hw, + filter->input.mac_addr.addr_bytes, + filter->input.ether_type, + flags, pf->main_vsi->seid, + filter->queue, 0, &stats, NULL); + if (ret < 0) + return ret; + + node = i40e_sw_ethertype_filter_lookup(ethertype_rule, &filter->input); + if (!node) + return -EINVAL; + + ret = i40e_sw_ethertype_filter_del(pf, &node->input); + + return ret; +} + +static int +i40e_flow_destroy_tunnel_filter(struct i40e_pf *pf, + struct i40e_tunnel_filter *filter) +{ + struct i40e_hw *hw = I40E_PF_TO_HW(pf); + struct i40e_vsi *vsi; + struct i40e_pf_vf *vf; + struct i40e_aqc_add_rm_cloud_filt_elem_ext cld_filter; + struct i40e_tunnel_rule *tunnel_rule = &pf->tunnel; + struct i40e_tunnel_filter *node; + bool big_buffer = 0; + int ret = 0; + + memset(&cld_filter, 0, sizeof(cld_filter)); + ether_addr_copy((struct ether_addr *)&filter->input.outer_mac, + (struct ether_addr *)&cld_filter.element.outer_mac); + ether_addr_copy((struct ether_addr *)&filter->input.inner_mac, + (struct ether_addr *)&cld_filter.element.inner_mac); + cld_filter.element.inner_vlan = filter->input.inner_vlan; + cld_filter.element.flags = filter->input.flags; + cld_filter.element.tenant_id = filter->input.tenant_id; + cld_filter.element.queue_number = filter->queue; + rte_memcpy(cld_filter.general_fields, + filter->input.general_fields, + sizeof(cld_filter.general_fields)); + + if (!filter->is_to_vf) + vsi = pf->main_vsi; + else { + vf = &pf->vfs[filter->vf_id]; + vsi = vf->vsi; + } + + if (((filter->input.flags & I40E_AQC_ADD_CLOUD_FILTER_0X11) == + I40E_AQC_ADD_CLOUD_FILTER_0X11) || + ((filter->input.flags & I40E_AQC_ADD_CLOUD_FILTER_0X12) == + I40E_AQC_ADD_CLOUD_FILTER_0X12) || + ((filter->input.flags & I40E_AQC_ADD_CLOUD_FILTER_0X10) == + I40E_AQC_ADD_CLOUD_FILTER_0X10)) + big_buffer = 1; + + if (big_buffer) + ret = i40e_aq_remove_cloud_filters_big_buffer(hw, vsi->seid, + &cld_filter, 1); + else + ret = i40e_aq_remove_cloud_filters(hw, vsi->seid, + &cld_filter.element, 1); + if (ret < 0) + return -ENOTSUP; + + node = i40e_sw_tunnel_filter_lookup(tunnel_rule, &filter->input); + if (!node) + return -EINVAL; + + ret = i40e_sw_tunnel_filter_del(pf, &node->input); + + return ret; +} + +static int +i40e_flow_flush(struct rte_eth_dev *dev, struct rte_flow_error *error) +{ + struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + int ret; + + ret = i40e_flow_flush_fdir_filter(pf); + if (ret) { + rte_flow_error_set(error, -ret, + RTE_FLOW_ERROR_TYPE_HANDLE, NULL, + "Failed to flush FDIR flows."); + return -rte_errno; + } + + ret = i40e_flow_flush_ethertype_filter(pf); + if (ret) { + rte_flow_error_set(error, -ret, + RTE_FLOW_ERROR_TYPE_HANDLE, NULL, + "Failed to ethertype flush flows."); + return -rte_errno; + } + + ret = i40e_flow_flush_tunnel_filter(pf); + if (ret) { + rte_flow_error_set(error, -ret, + RTE_FLOW_ERROR_TYPE_HANDLE, NULL, + "Failed to flush tunnel flows."); + return -rte_errno; + } + + return ret; +} + +static int +i40e_flow_flush_fdir_filter(struct i40e_pf *pf) +{ + struct rte_eth_dev *dev = pf->adapter->eth_dev; + struct i40e_fdir_info *fdir_info = &pf->fdir; + struct i40e_fdir_filter *fdir_filter; + enum i40e_filter_pctype pctype; + struct rte_flow *flow; + void *temp; + int ret; + + ret = i40e_fdir_flush(dev); + if (!ret) { + /* Delete FDIR filters in FDIR list. */ + while ((fdir_filter = TAILQ_FIRST(&fdir_info->fdir_list))) { + ret = i40e_sw_fdir_filter_del(pf, + &fdir_filter->fdir.input); + if (ret < 0) + return ret; + } + + /* Delete FDIR flows in flow list. */ + TAILQ_FOREACH_SAFE(flow, &pf->flow_list, node, temp) { + if (flow->filter_type == RTE_ETH_FILTER_FDIR) { + TAILQ_REMOVE(&pf->flow_list, flow, node); + rte_free(flow); + } + } + + for (pctype = I40E_FILTER_PCTYPE_NONF_IPV4_UDP; + pctype <= I40E_FILTER_PCTYPE_L2_PAYLOAD; pctype++) + pf->fdir.inset_flag[pctype] = 0; + } + + return ret; +} + +/* Flush all ethertype filters */ +static int +i40e_flow_flush_ethertype_filter(struct i40e_pf *pf) +{ + struct i40e_ethertype_filter_list + *ethertype_list = &pf->ethertype.ethertype_list; + struct i40e_ethertype_filter *filter; + struct rte_flow *flow; + void *temp; + int ret = 0; + + while ((filter = TAILQ_FIRST(ethertype_list))) { + ret = i40e_flow_destroy_ethertype_filter(pf, filter); + if (ret) + return ret; + } + + /* Delete ethertype flows in flow list. */ + TAILQ_FOREACH_SAFE(flow, &pf->flow_list, node, temp) { + if (flow->filter_type == RTE_ETH_FILTER_ETHERTYPE) { + TAILQ_REMOVE(&pf->flow_list, flow, node); + rte_free(flow); + } + } + + return ret; +} + +/* Flush all tunnel filters */ +static int +i40e_flow_flush_tunnel_filter(struct i40e_pf *pf) +{ + struct i40e_tunnel_filter_list + *tunnel_list = &pf->tunnel.tunnel_list; + struct i40e_tunnel_filter *filter; + struct rte_flow *flow; + void *temp; + int ret = 0; + + while ((filter = TAILQ_FIRST(tunnel_list))) { + ret = i40e_flow_destroy_tunnel_filter(pf, filter); + if (ret) + return ret; + } + + /* Delete tunnel flows in flow list. */ + TAILQ_FOREACH_SAFE(flow, &pf->flow_list, node, temp) { + if (flow->filter_type == RTE_ETH_FILTER_TUNNEL) { + TAILQ_REMOVE(&pf->flow_list, flow, node); + rte_free(flow); + } + } + + return ret; +} diff --git a/dpdk/drivers/net/i40e/i40e_logs.h b/dpdk/drivers/net/i40e/i40e_logs.h index e042e242..8e99cd52 100644 --- a/dpdk/drivers/net/i40e/i40e_logs.h +++ b/dpdk/drivers/net/i40e/i40e_logs.h @@ -34,14 +34,11 @@ #ifndef _I40E_LOGS_H_ #define _I40E_LOGS_H_ +extern int i40e_logtype_init; #define PMD_INIT_LOG(level, fmt, args...) \ - RTE_LOG(level, PMD, "%s(): " fmt "\n", __func__, ##args) - -#ifdef RTE_LIBRTE_I40E_DEBUG_INIT + rte_log(RTE_LOG_ ## level, i40e_logtype_init, "%s(): " fmt "\n", \ + __func__, ##args) #define PMD_INIT_FUNC_TRACE() PMD_INIT_LOG(DEBUG, " >>") -#else -#define PMD_INIT_FUNC_TRACE() do { } while(0) -#endif #ifdef RTE_LIBRTE_I40E_DEBUG_RX #define PMD_RX_LOG(level, fmt, args...) \ @@ -64,12 +61,10 @@ #define PMD_TX_FREE_LOG(level, fmt, args...) do { } while(0) #endif -#ifdef RTE_LIBRTE_I40E_DEBUG_DRIVER +extern int i40e_logtype_driver; #define PMD_DRV_LOG_RAW(level, fmt, args...) \ - RTE_LOG(level, PMD, "%s(): " fmt, __func__, ## args) -#else -#define PMD_DRV_LOG_RAW(level, fmt, args...) do { } while (0) -#endif + rte_log(RTE_LOG_ ## level, i40e_logtype_driver, "%s(): " fmt, \ + __func__, ## args) #define PMD_DRV_LOG(level, fmt, args...) \ PMD_DRV_LOG_RAW(level, fmt "\n", ## args) diff --git a/dpdk/drivers/net/i40e/i40e_pf.c b/dpdk/drivers/net/i40e/i40e_pf.c index 4e2f6b63..94bb0cfd 100644 --- a/dpdk/drivers/net/i40e/i40e_pf.c +++ b/dpdk/drivers/net/i40e/i40e_pf.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -44,7 +44,6 @@ #include #include #include -#include #include #include @@ -55,12 +54,13 @@ #include "i40e_ethdev.h" #include "i40e_rxtx.h" #include "i40e_pf.h" +#include "rte_pmd_i40e.h" #define I40E_CFG_CRCSTRIP_DEFAULT 1 static int i40e_pf_host_switch_queues(struct i40e_pf_vf *vf, - struct i40e_virtchnl_queue_select *qsel, + struct virtchnl_queue_select *qsel, bool on); /** @@ -127,7 +127,7 @@ i40e_pf_host_vf_reset(struct i40e_pf_vf *vf, bool do_hw_reset) struct i40e_pf *pf; uint16_t vf_id, abs_vf_id, vf_msix_num; int ret; - struct i40e_virtchnl_queue_select qsel; + struct virtchnl_queue_select qsel; if (vf == NULL) return -EINVAL; @@ -138,7 +138,7 @@ i40e_pf_host_vf_reset(struct i40e_pf_vf *vf, bool do_hw_reset) abs_vf_id = vf_id + hw->func_caps.vf_base_id; /* Notify VF that we are in VFR progress */ - I40E_WRITE_REG(hw, I40E_VFGEN_RSTAT1(vf_id), I40E_PF_VFR_INPROGRESS); + I40E_WRITE_REG(hw, I40E_VFGEN_RSTAT1(vf_id), VIRTCHNL_VFR_INPROGRESS); /* * If require a SW VF reset, a VFLR interrupt will be generated, @@ -166,7 +166,6 @@ i40e_pf_host_vf_reset(struct i40e_pf_vf *vf, bool do_hw_reset) PMD_DRV_LOG(ERR, "VF reset timeout"); return -ETIMEDOUT; } - /* This is not first time to do reset, do cleanup job first */ if (vf->vsi) { /* Disable queues */ @@ -219,7 +218,7 @@ i40e_pf_host_vf_reset(struct i40e_pf_vf *vf, bool do_hw_reset) } /* Reset done, Set COMPLETE flag and clear reset bit */ - I40E_WRITE_REG(hw, I40E_VFGEN_RSTAT1(vf_id), I40E_PF_VFR_COMPLETED); + I40E_WRITE_REG(hw, I40E_VFGEN_RSTAT1(vf_id), VIRTCHNL_VFR_COMPLETED); val = I40E_READ_REG(hw, I40E_VPGEN_VFRTRIG(vf_id)); val &= ~I40E_VPGEN_VFRTRIG_VFSWR_MASK; I40E_WRITE_REG(hw, I40E_VPGEN_VFRTRIG(vf_id), val); @@ -247,6 +246,8 @@ i40e_pf_host_vf_reset(struct i40e_pf_vf *vf, bool do_hw_reset) return -EFAULT; } + I40E_WRITE_REG(hw, I40E_VFGEN_RSTAT1(vf_id), VIRTCHNL_VFR_VFACTIVE); + return ret; } @@ -272,14 +273,30 @@ i40e_pf_host_send_msg_to_vf(struct i40e_pf_vf *vf, } static void -i40e_pf_host_process_cmd_version(struct i40e_pf_vf *vf) +i40e_pf_host_process_cmd_version(struct i40e_pf_vf *vf, bool b_op) { - struct i40e_virtchnl_version_info info; + struct virtchnl_version_info info; - info.major = I40E_DPDK_VERSION_MAJOR; - info.minor = I40E_DPDK_VERSION_MINOR; - i40e_pf_host_send_msg_to_vf(vf, I40E_VIRTCHNL_OP_VERSION, - I40E_SUCCESS, (uint8_t *)&info, sizeof(info)); + /* Respond like a Linux PF host in order to support both DPDK VF and + * Linux VF driver. The expense is original DPDK host specific feature + * like CFG_VLAN_PVID and CONFIG_VSI_QUEUES_EXT will not available. + * + * DPDK VF also can't identify host driver by version number returned. + * It always assume talking with Linux PF. + */ + info.major = VIRTCHNL_VERSION_MAJOR; + info.minor = VIRTCHNL_VERSION_MINOR_NO_VF_CAPS; + + if (b_op) + i40e_pf_host_send_msg_to_vf(vf, VIRTCHNL_OP_VERSION, + I40E_SUCCESS, + (uint8_t *)&info, + sizeof(info)); + else + i40e_pf_host_send_msg_to_vf(vf, VIRTCHNL_OP_VERSION, + I40E_NOT_SUPPORTED, + (uint8_t *)&info, + sizeof(info)); } static int @@ -292,17 +309,24 @@ i40e_pf_host_process_cmd_reset_vf(struct i40e_pf_vf *vf) } static int -i40e_pf_host_process_cmd_get_vf_resource(struct i40e_pf_vf *vf) +i40e_pf_host_process_cmd_get_vf_resource(struct i40e_pf_vf *vf, bool b_op) { - struct i40e_virtchnl_vf_resource *vf_res = NULL; + struct virtchnl_vf_resource *vf_res = NULL; struct i40e_hw *hw = I40E_PF_TO_HW(vf->pf); uint32_t len = 0; int ret = I40E_SUCCESS; + if (!b_op) { + i40e_pf_host_send_msg_to_vf(vf, + VIRTCHNL_OP_GET_VF_RESOURCES, + I40E_NOT_SUPPORTED, NULL, 0); + return ret; + } + /* only have 1 VSI by default */ - len = sizeof(struct i40e_virtchnl_vf_resource) + + len = sizeof(struct virtchnl_vf_resource) + I40E_DEFAULT_VF_VSI_NUM * - sizeof(struct i40e_virtchnl_vsi_resource); + sizeof(struct virtchnl_vsi_resource); vf_res = rte_zmalloc("i40e_vf_res", len, 0); if (vf_res == NULL) { @@ -313,22 +337,21 @@ i40e_pf_host_process_cmd_get_vf_resource(struct i40e_pf_vf *vf) goto send_msg; } - vf_res->vf_offload_flags = I40E_VIRTCHNL_VF_OFFLOAD_L2 | - I40E_VIRTCHNL_VF_OFFLOAD_VLAN; + vf_res->vf_offload_flags = VIRTCHNL_VF_OFFLOAD_L2 | + VIRTCHNL_VF_OFFLOAD_VLAN; vf_res->max_vectors = hw->func_caps.num_msix_vectors_vf; vf_res->num_queue_pairs = vf->vsi->nb_qps; vf_res->num_vsis = I40E_DEFAULT_VF_VSI_NUM; /* Change below setting if PF host can support more VSIs for VF */ - vf_res->vsi_res[0].vsi_type = I40E_VSI_SRIOV; - /* As assume Vf only has single VSI now, always return 0 */ - vf_res->vsi_res[0].vsi_id = 0; + vf_res->vsi_res[0].vsi_type = VIRTCHNL_VSI_SRIOV; + vf_res->vsi_res[0].vsi_id = vf->vsi->vsi_id; vf_res->vsi_res[0].num_queue_pairs = vf->vsi->nb_qps; ether_addr_copy(&vf->mac_addr, (struct ether_addr *)vf_res->vsi_res[0].default_mac_addr); send_msg: - i40e_pf_host_send_msg_to_vf(vf, I40E_VIRTCHNL_OP_GET_VF_RESOURCES, + i40e_pf_host_send_msg_to_vf(vf, VIRTCHNL_OP_GET_VF_RESOURCES, ret, (uint8_t *)vf_res, len); rte_free(vf_res); @@ -338,7 +361,7 @@ send_msg: static int i40e_pf_host_hmc_config_rxq(struct i40e_hw *hw, struct i40e_pf_vf *vf, - struct i40e_virtchnl_rxq_info *rxq, + struct virtchnl_rxq_info *rxq, uint8_t crcstrip) { int err = I40E_SUCCESS; @@ -380,23 +403,50 @@ i40e_pf_host_hmc_config_rxq(struct i40e_hw *hw, return err; } +static inline uint8_t +i40e_vsi_get_tc_of_queue(struct i40e_vsi *vsi, + uint16_t queue_id) +{ + struct i40e_aqc_vsi_properties_data *info = &vsi->info; + uint16_t bsf, qp_idx; + uint8_t i; + + for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { + if (vsi->enabled_tc & (1 << i)) { + qp_idx = rte_le_to_cpu_16((info->tc_mapping[i] & + I40E_AQ_VSI_TC_QUE_OFFSET_MASK) >> + I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT); + bsf = rte_le_to_cpu_16((info->tc_mapping[i] & + I40E_AQ_VSI_TC_QUE_NUMBER_MASK) >> + I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT); + if (queue_id >= qp_idx && queue_id < qp_idx + (1 << bsf)) + return i; + } + } + return 0; +} + static int i40e_pf_host_hmc_config_txq(struct i40e_hw *hw, struct i40e_pf_vf *vf, - struct i40e_virtchnl_txq_info *txq) + struct virtchnl_txq_info *txq) { int err = I40E_SUCCESS; struct i40e_hmc_obj_txq tx_ctx; + struct i40e_vsi *vsi = vf->vsi; uint32_t qtx_ctl; - uint16_t abs_queue_id = vf->vsi->base_queue + txq->queue_id; - + uint16_t abs_queue_id = vsi->base_queue + txq->queue_id; + uint8_t dcb_tc; /* clear the context structure first */ memset(&tx_ctx, 0, sizeof(tx_ctx)); - tx_ctx.new_context = 1; tx_ctx.base = txq->dma_ring_addr / I40E_QUEUE_BASE_ADDR_UNIT; tx_ctx.qlen = txq->ring_len; - tx_ctx.rdylist = rte_le_to_cpu_16(vf->vsi->info.qs_handle[0]); + dcb_tc = i40e_vsi_get_tc_of_queue(vsi, txq->queue_id); + tx_ctx.rdylist = rte_le_to_cpu_16(vsi->info.qs_handle[dcb_tc]); + tx_ctx.head_wb_ena = txq->headwb_enabled; + tx_ctx.head_wb_addr = txq->dma_headwb_addr; + err = i40e_clear_lan_tx_queue_context(hw, abs_queue_id); if (err != I40E_SUCCESS) return err; @@ -423,20 +473,28 @@ i40e_pf_host_hmc_config_txq(struct i40e_hw *hw, static int i40e_pf_host_process_cmd_config_vsi_queues(struct i40e_pf_vf *vf, uint8_t *msg, - uint16_t msglen) + uint16_t msglen, + bool b_op) { struct i40e_hw *hw = I40E_PF_TO_HW(vf->pf); struct i40e_vsi *vsi = vf->vsi; - struct i40e_virtchnl_vsi_queue_config_info *vc_vqci = - (struct i40e_virtchnl_vsi_queue_config_info *)msg; - struct i40e_virtchnl_queue_pair_info *vc_qpi; + struct virtchnl_vsi_queue_config_info *vc_vqci = + (struct virtchnl_vsi_queue_config_info *)msg; + struct virtchnl_queue_pair_info *vc_qpi; int i, ret = I40E_SUCCESS; + if (!b_op) { + i40e_pf_host_send_msg_to_vf(vf, + VIRTCHNL_OP_CONFIG_VSI_QUEUES, + I40E_NOT_SUPPORTED, NULL, 0); + return ret; + } + if (!msg || vc_vqci->num_queue_pairs > vsi->nb_qps || vc_vqci->num_queue_pairs > I40E_MAX_VSI_QP || msglen < I40E_VIRTCHNL_CONFIG_VSI_QUEUES_SIZE(vc_vqci, vc_vqci->num_queue_pairs)) { - PMD_DRV_LOG(ERR, "vsi_queue_config_info argument wrong\n"); + PMD_DRV_LOG(ERR, "vsi_queue_config_info argument wrong"); ret = I40E_ERR_PARAM; goto send_msg; } @@ -451,9 +509,9 @@ i40e_pf_host_process_cmd_config_vsi_queues(struct i40e_pf_vf *vf, /* * Apply VF RX queue setting to HMC. - * If the opcode is I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES_EXT, + * If the opcode is VIRTCHNL_OP_CONFIG_VSI_QUEUES_EXT, * then the extra information of - * 'struct i40e_virtchnl_queue_pair_extra_info' is needed, + * 'struct virtchnl_queue_pair_extra_info' is needed, * otherwise set the last parameter to NULL. */ if (i40e_pf_host_hmc_config_rxq(hw, vf, &vc_qpi[i].rxq, @@ -473,103 +531,181 @@ i40e_pf_host_process_cmd_config_vsi_queues(struct i40e_pf_vf *vf, } send_msg: - i40e_pf_host_send_msg_to_vf(vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, + i40e_pf_host_send_msg_to_vf(vf, VIRTCHNL_OP_CONFIG_VSI_QUEUES, ret, NULL, 0); return ret; } -static int -i40e_pf_host_process_cmd_config_vsi_queues_ext(struct i40e_pf_vf *vf, - uint8_t *msg, - uint16_t msglen) +static void +i40e_pf_config_irq_link_list(struct i40e_pf_vf *vf, + struct virtchnl_vector_map *vvm) { +#define BITS_PER_CHAR 8 + uint64_t linklistmap = 0, tempmap; struct i40e_hw *hw = I40E_PF_TO_HW(vf->pf); - struct i40e_vsi *vsi = vf->vsi; - struct i40e_virtchnl_vsi_queue_config_ext_info *vc_vqcei = - (struct i40e_virtchnl_vsi_queue_config_ext_info *)msg; - struct i40e_virtchnl_queue_pair_ext_info *vc_qpei; - int i, ret = I40E_SUCCESS; + uint16_t qid; + bool b_first_q = true; + enum i40e_queue_type qtype; + uint16_t vector_id; + uint32_t reg, reg_idx; + uint16_t itr_idx = 0, i; - if (!msg || vc_vqcei->num_queue_pairs > vsi->nb_qps || - vc_vqcei->num_queue_pairs > I40E_MAX_VSI_QP || - msglen < I40E_VIRTCHNL_CONFIG_VSI_QUEUES_SIZE(vc_vqcei, - vc_vqcei->num_queue_pairs)) { - PMD_DRV_LOG(ERR, "vsi_queue_config_ext_info argument wrong\n"); - ret = I40E_ERR_PARAM; - goto send_msg; + vector_id = vvm->vector_id; + /* setup the head */ + if (!vector_id) + reg_idx = I40E_VPINT_LNKLST0(vf->vf_idx); + else + reg_idx = I40E_VPINT_LNKLSTN( + ((hw->func_caps.num_msix_vectors_vf - 1) * vf->vf_idx) + + (vector_id - 1)); + + if (vvm->rxq_map == 0 && vvm->txq_map == 0) { + I40E_WRITE_REG(hw, reg_idx, + I40E_VPINT_LNKLST0_FIRSTQ_INDX_MASK); + goto cfg_irq_done; } - vc_qpei = vc_vqcei->qpair; - for (i = 0; i < vc_vqcei->num_queue_pairs; i++) { - if (vc_qpei[i].rxq.queue_id > vsi->nb_qps - 1 || - vc_qpei[i].txq.queue_id > vsi->nb_qps - 1) { - ret = I40E_ERR_PARAM; - goto send_msg; - } - /* - * Apply VF RX queue setting to HMC. - * If the opcode is I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES_EXT, - * then the extra information of - * 'struct i40e_virtchnl_queue_pair_ext_info' is needed, - * otherwise set the last parameter to NULL. - */ - if (i40e_pf_host_hmc_config_rxq(hw, vf, &vc_qpei[i].rxq, - vc_qpei[i].rxq_ext.crcstrip) != I40E_SUCCESS) { - PMD_DRV_LOG(ERR, "Configure RX queue HMC failed"); - ret = I40E_ERR_PARAM; - goto send_msg; - } - - /* Apply VF TX queue setting to HMC */ - if (i40e_pf_host_hmc_config_txq(hw, vf, &vc_qpei[i].txq) != - I40E_SUCCESS) { - PMD_DRV_LOG(ERR, "Configure TX queue HMC failed"); - ret = I40E_ERR_PARAM; - goto send_msg; - } + /* sort all rx and tx queues */ + tempmap = vvm->rxq_map; + for (i = 0; i < sizeof(vvm->rxq_map) * BITS_PER_CHAR; i++) { + if (tempmap & 0x1) + linklistmap |= (1 << (2 * i)); + tempmap >>= 1; } -send_msg: - i40e_pf_host_send_msg_to_vf(vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES_EXT, - ret, NULL, 0); + tempmap = vvm->txq_map; + for (i = 0; i < sizeof(vvm->txq_map) * BITS_PER_CHAR; i++) { + if (tempmap & 0x1) + linklistmap |= (1 << (2 * i + 1)); + tempmap >>= 1; + } - return ret; + /* Link all rx and tx queues into a chained list */ + tempmap = linklistmap; + i = 0; + b_first_q = true; + do { + if (tempmap & 0x1) { + qtype = (enum i40e_queue_type)(i % 2); + qid = vf->vsi->base_queue + i / 2; + if (b_first_q) { + /* This is header */ + b_first_q = false; + reg = ((qtype << + I40E_VPINT_LNKLSTN_FIRSTQ_TYPE_SHIFT) + | qid); + } else { + /* element in the link list */ + reg = (vector_id) | + (qtype << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT) | + (qid << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | + BIT(I40E_QINT_RQCTL_CAUSE_ENA_SHIFT) | + (itr_idx << I40E_QINT_RQCTL_ITR_INDX_SHIFT); + } + I40E_WRITE_REG(hw, reg_idx, reg); + /* find next register to program */ + switch (qtype) { + case I40E_QUEUE_TYPE_RX: + reg_idx = I40E_QINT_RQCTL(qid); + itr_idx = vvm->rxitr_idx; + break; + case I40E_QUEUE_TYPE_TX: + reg_idx = I40E_QINT_TQCTL(qid); + itr_idx = vvm->txitr_idx; + break; + default: + break; + } + } + i++; + tempmap >>= 1; + } while (tempmap); + + /* Terminate the link list */ + reg = (vector_id) | + (0 << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT) | + (0x7FF << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | + BIT(I40E_QINT_RQCTL_CAUSE_ENA_SHIFT) | + (itr_idx << I40E_QINT_RQCTL_ITR_INDX_SHIFT); + I40E_WRITE_REG(hw, reg_idx, reg); + +cfg_irq_done: + I40E_WRITE_FLUSH(hw); } static int i40e_pf_host_process_cmd_config_irq_map(struct i40e_pf_vf *vf, - uint8_t *msg, uint16_t msglen) + uint8_t *msg, uint16_t msglen, + bool b_op) { int ret = I40E_SUCCESS; - struct i40e_virtchnl_irq_map_info *irqmap = - (struct i40e_virtchnl_irq_map_info *)msg; + struct i40e_pf *pf = vf->pf; + struct i40e_hw *hw = I40E_PF_TO_HW(vf->pf); + struct virtchnl_irq_map_info *irqmap = + (struct virtchnl_irq_map_info *)msg; + struct virtchnl_vector_map *map; + int i; + uint16_t vector_id, itr_idx; + unsigned long qbit_max; - if (msg == NULL || msglen < sizeof(struct i40e_virtchnl_irq_map_info)) { + if (!b_op) { + i40e_pf_host_send_msg_to_vf( + vf, + VIRTCHNL_OP_CONFIG_IRQ_MAP, + I40E_NOT_SUPPORTED, NULL, 0); + return ret; + } + + if (msg == NULL || msglen < sizeof(struct virtchnl_irq_map_info)) { PMD_DRV_LOG(ERR, "buffer too short"); ret = I40E_ERR_PARAM; goto send_msg; } - /* Assume VF only have 1 vector to bind all queues */ - if (irqmap->num_vectors != 1) { - PMD_DRV_LOG(ERR, "DKDK host only support 1 vector"); - ret = I40E_ERR_PARAM; + /* PF host will support both DPDK VF or Linux VF driver, identify by + * number of vectors requested. + */ + + /* DPDK VF only requires single vector */ + if (irqmap->num_vectors == 1) { + /* This MSIX intr store the intr in VF range */ + vf->vsi->msix_intr = irqmap->vecmap[0].vector_id; + vf->vsi->nb_msix = irqmap->num_vectors; + vf->vsi->nb_used_qps = vf->vsi->nb_qps; + itr_idx = irqmap->vecmap[0].rxitr_idx; + + /* Don't care how the TX/RX queue mapping with this vector. + * Link all VF RX queues together. Only did mapping work. + * VF can disable/enable the intr by itself. + */ + i40e_vsi_queues_bind_intr(vf->vsi, itr_idx); goto send_msg; } - /* This MSIX intr store the intr in VF range */ - vf->vsi->msix_intr = irqmap->vecmap[0].vector_id; - vf->vsi->nb_msix = irqmap->num_vectors; - vf->vsi->nb_used_qps = vf->vsi->nb_qps; + /* Then, it's Linux VF driver */ + qbit_max = 1 << pf->vf_nb_qp_max; + for (i = 0; i < irqmap->num_vectors; i++) { + map = &irqmap->vecmap[i]; + + vector_id = map->vector_id; + /* validate msg params */ + if (vector_id >= hw->func_caps.num_msix_vectors_vf) { + ret = I40E_ERR_PARAM; + goto send_msg; + } + + if ((map->rxq_map < qbit_max) && (map->txq_map < qbit_max)) { + i40e_pf_config_irq_link_list(vf, map); + } else { + /* configured queue size excceed limit */ + ret = I40E_ERR_PARAM; + goto send_msg; + } + } - /* Don't care how the TX/RX queue mapping with this vector. - * Link all VF RX queues together. Only did mapping work. - * VF can disable/enable the intr by itself. - */ - i40e_vsi_queues_bind_intr(vf->vsi); send_msg: - i40e_pf_host_send_msg_to_vf(vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, + i40e_pf_host_send_msg_to_vf(vf, VIRTCHNL_OP_CONFIG_IRQ_MAP, ret, NULL, 0); return ret; @@ -577,7 +713,7 @@ send_msg: static int i40e_pf_host_switch_queues(struct i40e_pf_vf *vf, - struct i40e_virtchnl_queue_select *qsel, + struct virtchnl_queue_select *qsel, bool on) { int ret = I40E_SUCCESS; @@ -627,8 +763,8 @@ i40e_pf_host_process_cmd_enable_queues(struct i40e_pf_vf *vf, uint16_t msglen) { int ret = I40E_SUCCESS; - struct i40e_virtchnl_queue_select *q_sel = - (struct i40e_virtchnl_queue_select *)msg; + struct virtchnl_queue_select *q_sel = + (struct virtchnl_queue_select *)msg; if (msg == NULL || msglen != sizeof(*q_sel)) { ret = I40E_ERR_PARAM; @@ -637,7 +773,7 @@ i40e_pf_host_process_cmd_enable_queues(struct i40e_pf_vf *vf, ret = i40e_pf_host_switch_queues(vf, q_sel, true); send_msg: - i40e_pf_host_send_msg_to_vf(vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES, + i40e_pf_host_send_msg_to_vf(vf, VIRTCHNL_OP_ENABLE_QUEUES, ret, NULL, 0); return ret; @@ -646,11 +782,20 @@ send_msg: static int i40e_pf_host_process_cmd_disable_queues(struct i40e_pf_vf *vf, uint8_t *msg, - uint16_t msglen) + uint16_t msglen, + bool b_op) { int ret = I40E_SUCCESS; - struct i40e_virtchnl_queue_select *q_sel = - (struct i40e_virtchnl_queue_select *)msg; + struct virtchnl_queue_select *q_sel = + (struct virtchnl_queue_select *)msg; + + if (!b_op) { + i40e_pf_host_send_msg_to_vf( + vf, + VIRTCHNL_OP_DISABLE_QUEUES, + I40E_NOT_SUPPORTED, NULL, 0); + return ret; + } if (msg == NULL || msglen != sizeof(*q_sel)) { ret = I40E_ERR_PARAM; @@ -659,7 +804,7 @@ i40e_pf_host_process_cmd_disable_queues(struct i40e_pf_vf *vf, ret = i40e_pf_host_switch_queues(vf, q_sel, false); send_msg: - i40e_pf_host_send_msg_to_vf(vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES, + i40e_pf_host_send_msg_to_vf(vf, VIRTCHNL_OP_DISABLE_QUEUES, ret, NULL, 0); return ret; @@ -669,15 +814,24 @@ send_msg: static int i40e_pf_host_process_cmd_add_ether_address(struct i40e_pf_vf *vf, uint8_t *msg, - uint16_t msglen) + uint16_t msglen, + bool b_op) { int ret = I40E_SUCCESS; - struct i40e_virtchnl_ether_addr_list *addr_list = - (struct i40e_virtchnl_ether_addr_list *)msg; + struct virtchnl_ether_addr_list *addr_list = + (struct virtchnl_ether_addr_list *)msg; struct i40e_mac_filter_info filter; int i; struct ether_addr *mac; + if (!b_op) { + i40e_pf_host_send_msg_to_vf( + vf, + VIRTCHNL_OP_ADD_ETH_ADDR, + I40E_NOT_SUPPORTED, NULL, 0); + return ret; + } + memset(&filter, 0 , sizeof(struct i40e_mac_filter_info)); if (msg == NULL || msglen <= sizeof(*addr_list)) { @@ -688,17 +842,17 @@ i40e_pf_host_process_cmd_add_ether_address(struct i40e_pf_vf *vf, for (i = 0; i < addr_list->num_elements; i++) { mac = (struct ether_addr *)(addr_list->list[i].addr); - (void)rte_memcpy(&filter.mac_addr, mac, ETHER_ADDR_LEN); + rte_memcpy(&filter.mac_addr, mac, ETHER_ADDR_LEN); filter.filter_type = RTE_MACVLAN_PERFECT_MATCH; - if(!is_valid_assigned_ether_addr(mac) || - i40e_vsi_add_mac(vf->vsi, &filter)) { + if (is_zero_ether_addr(mac) || + i40e_vsi_add_mac(vf->vsi, &filter)) { ret = I40E_ERR_INVALID_MAC_ADDR; goto send_msg; } } send_msg: - i40e_pf_host_send_msg_to_vf(vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, + i40e_pf_host_send_msg_to_vf(vf, VIRTCHNL_OP_ADD_ETH_ADDR, ret, NULL, 0); return ret; @@ -707,14 +861,23 @@ send_msg: static int i40e_pf_host_process_cmd_del_ether_address(struct i40e_pf_vf *vf, uint8_t *msg, - uint16_t msglen) + uint16_t msglen, + bool b_op) { int ret = I40E_SUCCESS; - struct i40e_virtchnl_ether_addr_list *addr_list = - (struct i40e_virtchnl_ether_addr_list *)msg; + struct virtchnl_ether_addr_list *addr_list = + (struct virtchnl_ether_addr_list *)msg; int i; struct ether_addr *mac; + if (!b_op) { + i40e_pf_host_send_msg_to_vf( + vf, + VIRTCHNL_OP_DEL_ETH_ADDR, + I40E_NOT_SUPPORTED, NULL, 0); + return ret; + } + if (msg == NULL || msglen <= sizeof(*addr_list)) { PMD_DRV_LOG(ERR, "delete_ether_address argument too short"); ret = I40E_ERR_PARAM; @@ -723,7 +886,7 @@ i40e_pf_host_process_cmd_del_ether_address(struct i40e_pf_vf *vf, for (i = 0; i < addr_list->num_elements; i++) { mac = (struct ether_addr *)(addr_list->list[i].addr); - if(!is_valid_assigned_ether_addr(mac) || + if(is_zero_ether_addr(mac) || i40e_vsi_delete_mac(vf->vsi, mac)) { ret = I40E_ERR_INVALID_MAC_ADDR; goto send_msg; @@ -731,7 +894,7 @@ i40e_pf_host_process_cmd_del_ether_address(struct i40e_pf_vf *vf, } send_msg: - i40e_pf_host_send_msg_to_vf(vf, I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS, + i40e_pf_host_send_msg_to_vf(vf, VIRTCHNL_OP_DEL_ETH_ADDR, ret, NULL, 0); return ret; @@ -739,14 +902,23 @@ send_msg: static int i40e_pf_host_process_cmd_add_vlan(struct i40e_pf_vf *vf, - uint8_t *msg, uint16_t msglen) + uint8_t *msg, uint16_t msglen, + bool b_op) { int ret = I40E_SUCCESS; - struct i40e_virtchnl_vlan_filter_list *vlan_filter_list = - (struct i40e_virtchnl_vlan_filter_list *)msg; + struct virtchnl_vlan_filter_list *vlan_filter_list = + (struct virtchnl_vlan_filter_list *)msg; int i; uint16_t *vid; + if (!b_op) { + i40e_pf_host_send_msg_to_vf( + vf, + VIRTCHNL_OP_ADD_VLAN, + I40E_NOT_SUPPORTED, NULL, 0); + return ret; + } + if (msg == NULL || msglen <= sizeof(*vlan_filter_list)) { PMD_DRV_LOG(ERR, "add_vlan argument too short"); ret = I40E_ERR_PARAM; @@ -762,7 +934,7 @@ i40e_pf_host_process_cmd_add_vlan(struct i40e_pf_vf *vf, } send_msg: - i40e_pf_host_send_msg_to_vf(vf, I40E_VIRTCHNL_OP_ADD_VLAN, + i40e_pf_host_send_msg_to_vf(vf, VIRTCHNL_OP_ADD_VLAN, ret, NULL, 0); return ret; @@ -771,14 +943,23 @@ send_msg: static int i40e_pf_host_process_cmd_del_vlan(struct i40e_pf_vf *vf, uint8_t *msg, - uint16_t msglen) + uint16_t msglen, + bool b_op) { int ret = I40E_SUCCESS; - struct i40e_virtchnl_vlan_filter_list *vlan_filter_list = - (struct i40e_virtchnl_vlan_filter_list *)msg; + struct virtchnl_vlan_filter_list *vlan_filter_list = + (struct virtchnl_vlan_filter_list *)msg; int i; uint16_t *vid; + if (!b_op) { + i40e_pf_host_send_msg_to_vf( + vf, + VIRTCHNL_OP_DEL_VLAN, + I40E_NOT_SUPPORTED, NULL, 0); + return ret; + } + if (msg == NULL || msglen <= sizeof(*vlan_filter_list)) { PMD_DRV_LOG(ERR, "delete_vlan argument too short"); ret = I40E_ERR_PARAM; @@ -793,7 +974,7 @@ i40e_pf_host_process_cmd_del_vlan(struct i40e_pf_vf *vf, } send_msg: - i40e_pf_host_send_msg_to_vf(vf, I40E_VIRTCHNL_OP_DEL_VLAN, + i40e_pf_host_send_msg_to_vf(vf, VIRTCHNL_OP_DEL_VLAN, ret, NULL, 0); return ret; @@ -803,111 +984,158 @@ static int i40e_pf_host_process_cmd_config_promisc_mode( struct i40e_pf_vf *vf, uint8_t *msg, - uint16_t msglen) + uint16_t msglen, + bool b_op) { int ret = I40E_SUCCESS; - struct i40e_virtchnl_promisc_info *promisc = - (struct i40e_virtchnl_promisc_info *)msg; + struct virtchnl_promisc_info *promisc = + (struct virtchnl_promisc_info *)msg; struct i40e_hw *hw = I40E_PF_TO_HW(vf->pf); bool unicast = FALSE, multicast = FALSE; + if (!b_op) { + i40e_pf_host_send_msg_to_vf( + vf, + VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, + I40E_NOT_SUPPORTED, NULL, 0); + return ret; + } + if (msg == NULL || msglen != sizeof(*promisc)) { ret = I40E_ERR_PARAM; goto send_msg; } - if (promisc->flags & I40E_FLAG_VF_UNICAST_PROMISC) + if (promisc->flags & FLAG_VF_UNICAST_PROMISC) unicast = TRUE; ret = i40e_aq_set_vsi_unicast_promiscuous(hw, vf->vsi->seid, unicast, NULL, true); if (ret != I40E_SUCCESS) goto send_msg; - if (promisc->flags & I40E_FLAG_VF_MULTICAST_PROMISC) + if (promisc->flags & FLAG_VF_MULTICAST_PROMISC) multicast = TRUE; ret = i40e_aq_set_vsi_multicast_promiscuous(hw, vf->vsi->seid, multicast, NULL); send_msg: i40e_pf_host_send_msg_to_vf(vf, - I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, ret, NULL, 0); + VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, ret, NULL, 0); return ret; } static int -i40e_pf_host_process_cmd_get_stats(struct i40e_pf_vf *vf) +i40e_pf_host_process_cmd_get_stats(struct i40e_pf_vf *vf, bool b_op) { i40e_update_vsi_stats(vf->vsi); - i40e_pf_host_send_msg_to_vf(vf, I40E_VIRTCHNL_OP_GET_STATS, - I40E_SUCCESS, (uint8_t *)&vf->vsi->eth_stats, - sizeof(vf->vsi->eth_stats)); + if (b_op) + i40e_pf_host_send_msg_to_vf(vf, VIRTCHNL_OP_GET_STATS, + I40E_SUCCESS, + (uint8_t *)&vf->vsi->eth_stats, + sizeof(vf->vsi->eth_stats)); + else + i40e_pf_host_send_msg_to_vf(vf, VIRTCHNL_OP_GET_STATS, + I40E_NOT_SUPPORTED, + (uint8_t *)&vf->vsi->eth_stats, + sizeof(vf->vsi->eth_stats)); return I40E_SUCCESS; } static int -i40e_pf_host_process_cmd_cfg_vlan_offload( - struct i40e_pf_vf *vf, - uint8_t *msg, - uint16_t msglen) +i40e_pf_host_process_cmd_enable_vlan_strip(struct i40e_pf_vf *vf, bool b_op) { int ret = I40E_SUCCESS; - struct i40e_virtchnl_vlan_offload_info *offload = - (struct i40e_virtchnl_vlan_offload_info *)msg; - if (msg == NULL || msglen != sizeof(*offload)) { - ret = I40E_ERR_PARAM; - goto send_msg; + if (!b_op) { + i40e_pf_host_send_msg_to_vf( + vf, + VIRTCHNL_OP_ENABLE_VLAN_STRIPPING, + I40E_NOT_SUPPORTED, NULL, 0); + return ret; } - ret = i40e_vsi_config_vlan_stripping(vf->vsi, - !!offload->enable_vlan_strip); + ret = i40e_vsi_config_vlan_stripping(vf->vsi, TRUE); if (ret != 0) - PMD_DRV_LOG(ERR, "Failed to configure vlan stripping"); + PMD_DRV_LOG(ERR, "Failed to enable vlan stripping"); -send_msg: - i40e_pf_host_send_msg_to_vf(vf, I40E_VIRTCHNL_OP_CFG_VLAN_OFFLOAD, - ret, NULL, 0); + i40e_pf_host_send_msg_to_vf(vf, VIRTCHNL_OP_ENABLE_VLAN_STRIPPING, + ret, NULL, 0); return ret; } static int -i40e_pf_host_process_cmd_cfg_pvid(struct i40e_pf_vf *vf, - uint8_t *msg, - uint16_t msglen) +i40e_pf_host_process_cmd_disable_vlan_strip(struct i40e_pf_vf *vf, bool b_op) { int ret = I40E_SUCCESS; - struct i40e_virtchnl_pvid_info *tpid_info = - (struct i40e_virtchnl_pvid_info *)msg; - if (msg == NULL || msglen != sizeof(*tpid_info)) { - ret = I40E_ERR_PARAM; - goto send_msg; + if (!b_op) { + i40e_pf_host_send_msg_to_vf( + vf, + VIRTCHNL_OP_DISABLE_VLAN_STRIPPING, + I40E_NOT_SUPPORTED, NULL, 0); + return ret; } - ret = i40e_vsi_vlan_pvid_set(vf->vsi, &tpid_info->info); + ret = i40e_vsi_config_vlan_stripping(vf->vsi, FALSE); + if (ret != 0) + PMD_DRV_LOG(ERR, "Failed to disable vlan stripping"); -send_msg: - i40e_pf_host_send_msg_to_vf(vf, I40E_VIRTCHNL_OP_CFG_VLAN_PVID, - ret, NULL, 0); + i40e_pf_host_send_msg_to_vf(vf, VIRTCHNL_OP_DISABLE_VLAN_STRIPPING, + ret, NULL, 0); return ret; } -static void +void i40e_notify_vf_link_status(struct rte_eth_dev *dev, struct i40e_pf_vf *vf) { - struct i40e_virtchnl_pf_event event; + struct i40e_hw *hw = I40E_PF_TO_HW(vf->pf); + struct virtchnl_pf_event event; + uint16_t vf_id = vf->vf_idx; + uint32_t tval, rval; - event.event = I40E_VIRTCHNL_EVENT_LINK_CHANGE; + event.event = VIRTCHNL_EVENT_LINK_CHANGE; event.event_data.link_event.link_status = - dev->data->dev_link.link_status; - event.event_data.link_event.link_speed = - (enum i40e_aq_link_speed)dev->data->dev_link.link_speed; - i40e_pf_host_send_msg_to_vf(vf, I40E_VIRTCHNL_OP_EVENT, + dev->data->dev_link.link_status; + + /* need to convert the ETH_SPEED_xxx into VIRTCHNL_LINK_SPEED_xxx */ + switch (dev->data->dev_link.link_speed) { + case ETH_SPEED_NUM_100M: + event.event_data.link_event.link_speed = VIRTCHNL_LINK_SPEED_100MB; + break; + case ETH_SPEED_NUM_1G: + event.event_data.link_event.link_speed = VIRTCHNL_LINK_SPEED_1GB; + break; + case ETH_SPEED_NUM_10G: + event.event_data.link_event.link_speed = VIRTCHNL_LINK_SPEED_10GB; + break; + case ETH_SPEED_NUM_20G: + event.event_data.link_event.link_speed = VIRTCHNL_LINK_SPEED_20GB; + break; + case ETH_SPEED_NUM_25G: + event.event_data.link_event.link_speed = VIRTCHNL_LINK_SPEED_25GB; + break; + case ETH_SPEED_NUM_40G: + event.event_data.link_event.link_speed = VIRTCHNL_LINK_SPEED_40GB; + break; + default: + event.event_data.link_event.link_speed = + VIRTCHNL_LINK_SPEED_UNKNOWN; + break; + } + + tval = I40E_READ_REG(hw, I40E_VF_ATQLEN(vf_id)); + rval = I40E_READ_REG(hw, I40E_VF_ARQLEN(vf_id)); + + if (tval & I40E_VF_ATQLEN_ATQLEN_MASK || + tval & I40E_VF_ATQLEN_ATQENABLE_MASK || + rval & I40E_VF_ARQLEN_ARQLEN_MASK || + rval & I40E_VF_ARQLEN_ARQENABLE_MASK) + i40e_pf_host_send_msg_to_vf(vf, VIRTCHNL_OP_EVENT, I40E_SUCCESS, (uint8_t *)&event, sizeof(event)); } @@ -923,6 +1151,8 @@ i40e_pf_host_handle_vf_msg(struct rte_eth_dev *dev, struct i40e_pf_vf *vf; /* AdminQ will pass absolute VF id, transfer to internal vf id */ uint16_t vf_id = abs_vf_id - hw->func_caps.vf_base_id; + struct rte_pmd_i40e_mb_event_param ret_param; + bool b_op = TRUE; if (vf_id > pf->vf_num - 1 || !pf->vfs) { PMD_DRV_LOG(ERR, "invalid argument"); @@ -937,72 +1167,103 @@ i40e_pf_host_handle_vf_msg(struct rte_eth_dev *dev, return; } + /** + * initialise structure to send to user application + * will return response from user in retval field + */ + ret_param.retval = RTE_PMD_I40E_MB_EVENT_PROCEED; + ret_param.vfid = vf_id; + ret_param.msg_type = opcode; + ret_param.msg = (void *)msg; + ret_param.msglen = msglen; + + /** + * Ask user application if we're allowed to perform those functions. + * If we get ret_param.retval == RTE_PMD_I40E_MB_EVENT_PROCEED, + * then business as usual. + * If RTE_PMD_I40E_MB_EVENT_NOOP_ACK or RTE_PMD_I40E_MB_EVENT_NOOP_NACK, + * do nothing and send not_supported to VF. As PF must send a response + * to VF and ACK/NACK is not defined. + */ + _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_VF_MBOX, + NULL, &ret_param); + if (ret_param.retval != RTE_PMD_I40E_MB_EVENT_PROCEED) { + PMD_DRV_LOG(WARNING, "VF to PF message(%d) is not permitted!", + opcode); + b_op = FALSE; + } + switch (opcode) { - case I40E_VIRTCHNL_OP_VERSION : + case VIRTCHNL_OP_VERSION: PMD_DRV_LOG(INFO, "OP_VERSION received"); - i40e_pf_host_process_cmd_version(vf); + i40e_pf_host_process_cmd_version(vf, b_op); break; - case I40E_VIRTCHNL_OP_RESET_VF : + case VIRTCHNL_OP_RESET_VF: PMD_DRV_LOG(INFO, "OP_RESET_VF received"); i40e_pf_host_process_cmd_reset_vf(vf); break; - case I40E_VIRTCHNL_OP_GET_VF_RESOURCES: + case VIRTCHNL_OP_GET_VF_RESOURCES: PMD_DRV_LOG(INFO, "OP_GET_VF_RESOURCES received"); - i40e_pf_host_process_cmd_get_vf_resource(vf); + i40e_pf_host_process_cmd_get_vf_resource(vf, b_op); break; - case I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES: + case VIRTCHNL_OP_CONFIG_VSI_QUEUES: PMD_DRV_LOG(INFO, "OP_CONFIG_VSI_QUEUES received"); - i40e_pf_host_process_cmd_config_vsi_queues(vf, msg, msglen); + i40e_pf_host_process_cmd_config_vsi_queues(vf, msg, + msglen, b_op); break; - case I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES_EXT: - PMD_DRV_LOG(INFO, "OP_CONFIG_VSI_QUEUES_EXT received"); - i40e_pf_host_process_cmd_config_vsi_queues_ext(vf, msg, - msglen); - break; - case I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP: + case VIRTCHNL_OP_CONFIG_IRQ_MAP: PMD_DRV_LOG(INFO, "OP_CONFIG_IRQ_MAP received"); - i40e_pf_host_process_cmd_config_irq_map(vf, msg, msglen); + i40e_pf_host_process_cmd_config_irq_map(vf, msg, msglen, b_op); break; - case I40E_VIRTCHNL_OP_ENABLE_QUEUES: + case VIRTCHNL_OP_ENABLE_QUEUES: PMD_DRV_LOG(INFO, "OP_ENABLE_QUEUES received"); - i40e_pf_host_process_cmd_enable_queues(vf, msg, msglen); - i40e_notify_vf_link_status(dev, vf); + if (b_op) { + i40e_pf_host_process_cmd_enable_queues(vf, msg, msglen); + i40e_notify_vf_link_status(dev, vf); + } else { + i40e_pf_host_send_msg_to_vf( + vf, VIRTCHNL_OP_ENABLE_QUEUES, + I40E_NOT_SUPPORTED, NULL, 0); + } break; - case I40E_VIRTCHNL_OP_DISABLE_QUEUES: + case VIRTCHNL_OP_DISABLE_QUEUES: PMD_DRV_LOG(INFO, "OP_DISABLE_QUEUE received"); - i40e_pf_host_process_cmd_disable_queues(vf, msg, msglen); + i40e_pf_host_process_cmd_disable_queues(vf, msg, msglen, b_op); break; - case I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS: + case VIRTCHNL_OP_ADD_ETH_ADDR: PMD_DRV_LOG(INFO, "OP_ADD_ETHER_ADDRESS received"); - i40e_pf_host_process_cmd_add_ether_address(vf, msg, msglen); + i40e_pf_host_process_cmd_add_ether_address(vf, msg, + msglen, b_op); break; - case I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS: + case VIRTCHNL_OP_DEL_ETH_ADDR: PMD_DRV_LOG(INFO, "OP_DEL_ETHER_ADDRESS received"); - i40e_pf_host_process_cmd_del_ether_address(vf, msg, msglen); + i40e_pf_host_process_cmd_del_ether_address(vf, msg, + msglen, b_op); break; - case I40E_VIRTCHNL_OP_ADD_VLAN: + case VIRTCHNL_OP_ADD_VLAN: PMD_DRV_LOG(INFO, "OP_ADD_VLAN received"); - i40e_pf_host_process_cmd_add_vlan(vf, msg, msglen); + i40e_pf_host_process_cmd_add_vlan(vf, msg, msglen, b_op); break; - case I40E_VIRTCHNL_OP_DEL_VLAN: + case VIRTCHNL_OP_DEL_VLAN: PMD_DRV_LOG(INFO, "OP_DEL_VLAN received"); - i40e_pf_host_process_cmd_del_vlan(vf, msg, msglen); + i40e_pf_host_process_cmd_del_vlan(vf, msg, msglen, b_op); break; - case I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE: + case VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE: PMD_DRV_LOG(INFO, "OP_CONFIG_PROMISCUOUS_MODE received"); - i40e_pf_host_process_cmd_config_promisc_mode(vf, msg, msglen); + i40e_pf_host_process_cmd_config_promisc_mode(vf, msg, + msglen, b_op); break; - case I40E_VIRTCHNL_OP_GET_STATS: + case VIRTCHNL_OP_GET_STATS: PMD_DRV_LOG(INFO, "OP_GET_STATS received"); - i40e_pf_host_process_cmd_get_stats(vf); + i40e_pf_host_process_cmd_get_stats(vf, b_op); break; - case I40E_VIRTCHNL_OP_CFG_VLAN_OFFLOAD: - PMD_DRV_LOG(INFO, "OP_CFG_VLAN_OFFLOAD received"); - i40e_pf_host_process_cmd_cfg_vlan_offload(vf, msg, msglen); + case VIRTCHNL_OP_ENABLE_VLAN_STRIPPING: + PMD_DRV_LOG(INFO, "OP_ENABLE_VLAN_STRIPPING received"); + i40e_pf_host_process_cmd_enable_vlan_strip(vf, b_op); break; - case I40E_VIRTCHNL_OP_CFG_VLAN_PVID: - PMD_DRV_LOG(INFO, "OP_CFG_VLAN_PVID received"); - i40e_pf_host_process_cmd_cfg_pvid(vf, msg, msglen); + case VIRTCHNL_OP_DISABLE_VLAN_STRIPPING: + PMD_DRV_LOG(INFO, "OP_DISABLE_VLAN_STRIPPING received"); + i40e_pf_host_process_cmd_disable_vlan_strip(vf, b_op); break; /* Don't add command supported below, which will * return an error code. @@ -1053,9 +1314,9 @@ i40e_pf_host_init(struct rte_eth_dev *dev) ret = i40e_pf_host_vf_reset(&pf->vfs[i], 0); if (ret != I40E_SUCCESS) goto fail; - eth_random_addr(pf->vfs[i].mac_addr.addr_bytes); } + RTE_ETH_DEV_SRIOV(dev).active = pf->vf_num; /* restore irq0 */ i40e_pf_enable_irq0(hw); diff --git a/dpdk/drivers/net/i40e/i40e_pf.h b/dpdk/drivers/net/i40e/i40e_pf.h index cddc45cf..04116637 100644 --- a/dpdk/drivers/net/i40e/i40e_pf.h +++ b/dpdk/drivers/net/i40e/i40e_pf.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -34,65 +34,10 @@ #ifndef _I40E_PF_H_ #define _I40E_PF_H_ -/* VERSION info to exchange between VF and PF host. In case VF works with - * ND kernel driver, it reads I40E_VIRTCHNL_VERSION_MAJOR/MINOR. In - * case works with DPDK host, it reads version below. Then VF realize who it - * is talking to and use proper language to communicate. - * */ -#define I40E_DPDK_SIGNATURE ('D' << 24 | 'P' << 16 | 'D' << 8 | 'K') -#define I40E_DPDK_VERSION_MAJOR I40E_DPDK_SIGNATURE -#define I40E_DPDK_VERSION_MINOR 0 - /* Default setting on number of VSIs that VF can contain */ #define I40E_DEFAULT_VF_VSI_NUM 1 -#define I40E_DPDK_OFFSET 0x100 - -enum i40e_pf_vfr_state { - I40E_PF_VFR_INPROGRESS = 0, - I40E_PF_VFR_COMPLETED = 1, -}; - -/* DPDK pf driver specific command to VF */ -enum i40e_virtchnl_ops_dpdk { - /* - * Keep some gap between Linux PF commands and - * DPDK PF extended commands. - */ - I40E_VIRTCHNL_OP_CFG_VLAN_OFFLOAD = I40E_VIRTCHNL_OP_VERSION + - I40E_DPDK_OFFSET, - I40E_VIRTCHNL_OP_CFG_VLAN_PVID, - I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES_EXT, -}; - -/* A structure to support extended info of a receive queue. */ -struct i40e_virtchnl_rxq_ext_info { - uint8_t crcstrip; -}; - -/* - * A structure to support extended info of queue pairs, an additional field - * is added, comparing to original 'struct i40e_virtchnl_queue_pair_info'. - */ -struct i40e_virtchnl_queue_pair_ext_info { - /* vsi_id and queue_id should be identical for both rx and tx queues.*/ - struct i40e_virtchnl_txq_info txq; - struct i40e_virtchnl_rxq_info rxq; - struct i40e_virtchnl_rxq_ext_info rxq_ext; -}; - -/* - * A structure to support extended info of VSI queue pairs, - * 'struct i40e_virtchnl_queue_pair_ext_info' is used, see its original - * of 'struct i40e_virtchnl_queue_pair_info'. - */ -struct i40e_virtchnl_vsi_queue_config_ext_info { - uint16_t vsi_id; - uint16_t num_queue_pairs; - struct i40e_virtchnl_queue_pair_ext_info qpair[0]; -}; - -struct i40e_virtchnl_vlan_offload_info { +struct virtchnl_vlan_offload_info { uint16_t vsi_id; uint8_t enable_vlan_strip; uint8_t reserved; @@ -105,23 +50,14 @@ struct i40e_virtchnl_vlan_offload_info { #define I40E_VIRTCHNL_CONFIG_VSI_QUEUES_SIZE(x, n) \ (sizeof(*(x)) + sizeof((x)->qpair[0]) * (n)) -/* - * I40E_VIRTCHNL_OP_CFG_VLAN_PVID - * VF sends this message to enable/disable pvid. If it's - * enable op, needs to specify the pvid. PF returns status - * code in retval. - */ -struct i40e_virtchnl_pvid_info { - uint16_t vsi_id; - struct i40e_vsi_vlan_pvid_info info; -}; - int i40e_pf_host_vf_reset(struct i40e_pf_vf *vf, bool do_hw_reset); void i40e_pf_host_handle_vf_msg(struct rte_eth_dev *dev, uint16_t abs_vf_id, uint32_t opcode, - __rte_unused uint32_t retval, + uint32_t retval, uint8_t *msg, uint16_t msglen); int i40e_pf_host_init(struct rte_eth_dev *dev); int i40e_pf_host_uninit(struct rte_eth_dev *dev); +void i40e_notify_vf_link_status(struct rte_eth_dev *dev, + struct i40e_pf_vf *vf); #endif /* _I40E_PF_H_ */ diff --git a/dpdk/drivers/net/i40e/i40e_rxtx.c b/dpdk/drivers/net/i40e/i40e_rxtx.c index 0556a4d4..078b405a 100644 --- a/dpdk/drivers/net/i40e/i40e_rxtx.c +++ b/dpdk/drivers/net/i40e/i40e_rxtx.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -50,6 +50,8 @@ #include #include #include +#include +#include #include "i40e_logs.h" #include "base/i40e_prototype.h" @@ -59,7 +61,6 @@ #define DEFAULT_TX_RS_THRESH 32 #define DEFAULT_TX_FREE_THRESH 32 -#define I40E_MAX_PKT_TYPE 256 #define I40E_TX_MAX_BURST 32 @@ -73,12 +74,31 @@ #define I40E_TXD_CMD (I40E_TX_DESC_CMD_EOP | I40E_TX_DESC_CMD_RS) +#ifdef RTE_LIBRTE_IEEE1588 +#define I40E_TX_IEEE1588_TMST PKT_TX_IEEE1588_TMST +#else +#define I40E_TX_IEEE1588_TMST 0 +#endif + #define I40E_TX_CKSUM_OFFLOAD_MASK ( \ PKT_TX_IP_CKSUM | \ PKT_TX_L4_MASK | \ PKT_TX_TCP_SEG | \ PKT_TX_OUTER_IP_CKSUM) +#define I40E_TX_OFFLOAD_MASK ( \ + PKT_TX_IP_CKSUM | \ + PKT_TX_L4_MASK | \ + PKT_TX_OUTER_IP_CKSUM | \ + PKT_TX_TCP_SEG | \ + PKT_TX_QINQ_PKT | \ + PKT_TX_VLAN_PKT | \ + PKT_TX_TUNNEL_MASK | \ + I40E_TX_IEEE1588_TMST) + +#define I40E_TX_OFFLOAD_NOTSUP_MASK \ + (PKT_TX_OFFLOAD_MASK ^ I40E_TX_OFFLOAD_MASK) + static uint16_t i40e_xmit_pkts_simple(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts); @@ -88,7 +108,7 @@ i40e_rxd_to_vlan_tci(struct rte_mbuf *mb, volatile union i40e_rx_desc *rxdp) { if (rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len) & (1 << I40E_RX_DESC_STATUS_L2TAG1P_SHIFT)) { - mb->ol_flags |= PKT_RX_VLAN_PKT | PKT_RX_VLAN_STRIPPED; + mb->ol_flags |= PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED; mb->vlan_tci = rte_le_to_cpu_16(rxdp->wb.qword0.lo_dword.l2tag1); PMD_RX_LOG(DEBUG, "Descriptor l2tag1: %u", @@ -138,12 +158,21 @@ i40e_rxd_error_to_pkt_flags(uint64_t qword) uint64_t error_bits = (qword >> I40E_RXD_QW1_ERROR_SHIFT); #define I40E_RX_ERR_BITS 0x3f - if (likely((error_bits & I40E_RX_ERR_BITS) == 0)) + if (likely((error_bits & I40E_RX_ERR_BITS) == 0)) { + flags |= (PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD); return flags; + } + if (unlikely(error_bits & (1 << I40E_RX_DESC_ERROR_IPE_SHIFT))) flags |= PKT_RX_IP_CKSUM_BAD; + else + flags |= PKT_RX_IP_CKSUM_GOOD; + if (unlikely(error_bits & (1 << I40E_RX_DESC_ERROR_L4E_SHIFT))) flags |= PKT_RX_L4_CKSUM_BAD; + else + flags |= PKT_RX_L4_CKSUM_GOOD; + if (unlikely(error_bits & (1 << I40E_RX_DESC_ERROR_EIPE_SHIFT))) flags |= PKT_RX_EIP_CKSUM_BAD; @@ -174,569 +203,6 @@ i40e_get_iee15888_flags(struct rte_mbuf *mb, uint64_t qword) } #endif -/* For each value it means, datasheet of hardware can tell more details - * - * @note: fix i40e_dev_supported_ptypes_get() if any change here. - */ -static inline uint32_t -i40e_rxd_pkt_type_mapping(uint8_t ptype) -{ - static const uint32_t type_table[UINT8_MAX + 1] __rte_cache_aligned = { - /* L2 types */ - /* [0] reserved */ - [1] = RTE_PTYPE_L2_ETHER, - [2] = RTE_PTYPE_L2_ETHER_TIMESYNC, - /* [3] - [5] reserved */ - [6] = RTE_PTYPE_L2_ETHER_LLDP, - /* [7] - [10] reserved */ - [11] = RTE_PTYPE_L2_ETHER_ARP, - /* [12] - [21] reserved */ - - /* Non tunneled IPv4 */ - [22] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_L4_FRAG, - [23] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_L4_NONFRAG, - [24] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_L4_UDP, - /* [25] reserved */ - [26] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_L4_TCP, - [27] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_L4_SCTP, - [28] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_L4_ICMP, - - /* IPv4 --> IPv4 */ - [29] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_IP | - RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_FRAG, - [30] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_IP | - RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_NONFRAG, - [31] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_IP | - RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_UDP, - /* [32] reserved */ - [33] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_IP | - RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_TCP, - [34] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_IP | - RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_SCTP, - [35] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_IP | - RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_ICMP, - - /* IPv4 --> IPv6 */ - [36] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_IP | - RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_FRAG, - [37] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_IP | - RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_NONFRAG, - [38] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_IP | - RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_UDP, - /* [39] reserved */ - [40] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_IP | - RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_TCP, - [41] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_IP | - RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_SCTP, - [42] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_IP | - RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_ICMP, - - /* IPv4 --> GRE/Teredo/VXLAN */ - [43] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT, - - /* IPv4 --> GRE/Teredo/VXLAN --> IPv4 */ - [44] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | - RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_FRAG, - [45] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | - RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_NONFRAG, - [46] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | - RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_UDP, - /* [47] reserved */ - [48] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | - RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_TCP, - [49] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | - RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_SCTP, - [50] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | - RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_ICMP, - - /* IPv4 --> GRE/Teredo/VXLAN --> IPv6 */ - [51] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | - RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_FRAG, - [52] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | - RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_NONFRAG, - [53] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | - RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_UDP, - /* [54] reserved */ - [55] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | - RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_TCP, - [56] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | - RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_SCTP, - [57] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | - RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_ICMP, - - /* IPv4 --> GRE/Teredo/VXLAN --> MAC */ - [58] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER, - - /* IPv4 --> GRE/Teredo/VXLAN --> MAC --> IPv4 */ - [59] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_FRAG, - [60] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_NONFRAG, - [61] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_UDP, - /* [62] reserved */ - [63] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_TCP, - [64] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_SCTP, - [65] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_ICMP, - - /* IPv4 --> GRE/Teredo/VXLAN --> MAC --> IPv6 */ - [66] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_FRAG, - [67] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_NONFRAG, - [68] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_UDP, - /* [69] reserved */ - [70] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_TCP, - [71] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_SCTP, - [72] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_ICMP, - - /* IPv4 --> GRE/Teredo/VXLAN --> MAC/VLAN */ - [73] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | - RTE_PTYPE_INNER_L2_ETHER_VLAN, - - /* IPv4 --> GRE/Teredo/VXLAN --> MAC/VLAN --> IPv4 */ - [74] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | - RTE_PTYPE_INNER_L2_ETHER_VLAN | - RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_FRAG, - [75] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | - RTE_PTYPE_INNER_L2_ETHER_VLAN | - RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_NONFRAG, - [76] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | - RTE_PTYPE_INNER_L2_ETHER_VLAN | - RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_UDP, - /* [77] reserved */ - [78] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | - RTE_PTYPE_INNER_L2_ETHER_VLAN | - RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_TCP, - [79] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | - RTE_PTYPE_INNER_L2_ETHER_VLAN | - RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_SCTP, - [80] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | - RTE_PTYPE_INNER_L2_ETHER_VLAN | - RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_ICMP, - - /* IPv4 --> GRE/Teredo/VXLAN --> MAC/VLAN --> IPv6 */ - [81] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | - RTE_PTYPE_INNER_L2_ETHER_VLAN | - RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_FRAG, - [82] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | - RTE_PTYPE_INNER_L2_ETHER_VLAN | - RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_NONFRAG, - [83] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | - RTE_PTYPE_INNER_L2_ETHER_VLAN | - RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_UDP, - /* [84] reserved */ - [85] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | - RTE_PTYPE_INNER_L2_ETHER_VLAN | - RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_TCP, - [86] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | - RTE_PTYPE_INNER_L2_ETHER_VLAN | - RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_SCTP, - [87] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | - RTE_PTYPE_INNER_L2_ETHER_VLAN | - RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_ICMP, - - /* Non tunneled IPv6 */ - [88] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_L4_FRAG, - [89] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_L4_NONFRAG, - [90] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_L4_UDP, - /* [91] reserved */ - [92] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_L4_TCP, - [93] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_L4_SCTP, - [94] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_L4_ICMP, - - /* IPv6 --> IPv4 */ - [95] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_IP | - RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_FRAG, - [96] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_IP | - RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_NONFRAG, - [97] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_IP | - RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_UDP, - /* [98] reserved */ - [99] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_IP | - RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_TCP, - [100] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_IP | - RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_SCTP, - [101] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_IP | - RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_ICMP, - - /* IPv6 --> IPv6 */ - [102] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_IP | - RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_FRAG, - [103] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_IP | - RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_NONFRAG, - [104] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_IP | - RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_UDP, - /* [105] reserved */ - [106] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_IP | - RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_TCP, - [107] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_IP | - RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_SCTP, - [108] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_IP | - RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_ICMP, - - /* IPv6 --> GRE/Teredo/VXLAN */ - [109] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT, - - /* IPv6 --> GRE/Teredo/VXLAN --> IPv4 */ - [110] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | - RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_FRAG, - [111] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | - RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_NONFRAG, - [112] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | - RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_UDP, - /* [113] reserved */ - [114] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | - RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_TCP, - [115] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | - RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_SCTP, - [116] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | - RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_ICMP, - - /* IPv6 --> GRE/Teredo/VXLAN --> IPv6 */ - [117] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | - RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_FRAG, - [118] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | - RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_NONFRAG, - [119] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | - RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_UDP, - /* [120] reserved */ - [121] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | - RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_TCP, - [122] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | - RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_SCTP, - [123] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | - RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_ICMP, - - /* IPv6 --> GRE/Teredo/VXLAN --> MAC */ - [124] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER, - - /* IPv6 --> GRE/Teredo/VXLAN --> MAC --> IPv4 */ - [125] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_FRAG, - [126] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_NONFRAG, - [127] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_UDP, - /* [128] reserved */ - [129] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_TCP, - [130] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_SCTP, - [131] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_ICMP, - - /* IPv6 --> GRE/Teredo/VXLAN --> MAC --> IPv6 */ - [132] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_FRAG, - [133] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_NONFRAG, - [134] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_UDP, - /* [135] reserved */ - [136] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_TCP, - [137] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_SCTP, - [138] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_ICMP, - - /* IPv6 --> GRE/Teredo/VXLAN --> MAC/VLAN */ - [139] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | - RTE_PTYPE_INNER_L2_ETHER_VLAN, - - /* IPv6 --> GRE/Teredo/VXLAN --> MAC/VLAN --> IPv4 */ - [140] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | - RTE_PTYPE_INNER_L2_ETHER_VLAN | - RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_FRAG, - [141] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | - RTE_PTYPE_INNER_L2_ETHER_VLAN | - RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_NONFRAG, - [142] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | - RTE_PTYPE_INNER_L2_ETHER_VLAN | - RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_UDP, - /* [143] reserved */ - [144] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | - RTE_PTYPE_INNER_L2_ETHER_VLAN | - RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_TCP, - [145] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | - RTE_PTYPE_INNER_L2_ETHER_VLAN | - RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_SCTP, - [146] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | - RTE_PTYPE_INNER_L2_ETHER_VLAN | - RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_ICMP, - - /* IPv6 --> GRE/Teredo/VXLAN --> MAC/VLAN --> IPv6 */ - [147] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | - RTE_PTYPE_INNER_L2_ETHER_VLAN | - RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_FRAG, - [148] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | - RTE_PTYPE_INNER_L2_ETHER_VLAN | - RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_NONFRAG, - [149] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | - RTE_PTYPE_INNER_L2_ETHER_VLAN | - RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_UDP, - /* [150] reserved */ - [151] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | - RTE_PTYPE_INNER_L2_ETHER_VLAN | - RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_TCP, - [152] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | - RTE_PTYPE_INNER_L2_ETHER_VLAN | - RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_SCTP, - [153] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRENAT | - RTE_PTYPE_INNER_L2_ETHER_VLAN | - RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_INNER_L4_ICMP, - - /* L2 NSH packet type */ - [154] = RTE_PTYPE_L2_ETHER_NSH, - [155] = RTE_PTYPE_L2_ETHER_NSH | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_L4_FRAG, - [156] = RTE_PTYPE_L2_ETHER_NSH | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_L4_NONFRAG, - [157] = RTE_PTYPE_L2_ETHER_NSH | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_L4_UDP, - [158] = RTE_PTYPE_L2_ETHER_NSH | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_L4_TCP, - [159] = RTE_PTYPE_L2_ETHER_NSH | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_L4_SCTP, - [160] = RTE_PTYPE_L2_ETHER_NSH | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_L4_ICMP, - [161] = RTE_PTYPE_L2_ETHER_NSH | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_L4_FRAG, - [162] = RTE_PTYPE_L2_ETHER_NSH | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_L4_NONFRAG, - [163] = RTE_PTYPE_L2_ETHER_NSH | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_L4_UDP, - [164] = RTE_PTYPE_L2_ETHER_NSH | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_L4_TCP, - [165] = RTE_PTYPE_L2_ETHER_NSH | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_L4_SCTP, - [166] = RTE_PTYPE_L2_ETHER_NSH | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - RTE_PTYPE_L4_ICMP, - - /* All others reserved */ - }; - - return type_table[ptype]; -} - #define I40E_RX_DESC_EXT_STATUS_FLEXBH_MASK 0x03 #define I40E_RX_DESC_EXT_STATUS_FLEXBH_FD_ID 0x01 #define I40E_RX_DESC_EXT_STATUS_FLEXBH_FLEX 0x02 @@ -779,33 +245,65 @@ i40e_rxd_build_fdir(volatile union i40e_rx_desc *rxdp, struct rte_mbuf *mb) #endif return flags; } + +static inline void +i40e_parse_tunneling_params(uint64_t ol_flags, + union i40e_tx_offload tx_offload, + uint32_t *cd_tunneling) +{ + /* EIPT: External (outer) IP header type */ + if (ol_flags & PKT_TX_OUTER_IP_CKSUM) + *cd_tunneling |= I40E_TX_CTX_EXT_IP_IPV4; + else if (ol_flags & PKT_TX_OUTER_IPV4) + *cd_tunneling |= I40E_TX_CTX_EXT_IP_IPV4_NO_CSUM; + else if (ol_flags & PKT_TX_OUTER_IPV6) + *cd_tunneling |= I40E_TX_CTX_EXT_IP_IPV6; + + /* EIPLEN: External (outer) IP header length, in DWords */ + *cd_tunneling |= (tx_offload.outer_l3_len >> 2) << + I40E_TXD_CTX_QW0_EXT_IPLEN_SHIFT; + + /* L4TUNT: L4 Tunneling Type */ + switch (ol_flags & PKT_TX_TUNNEL_MASK) { + case PKT_TX_TUNNEL_IPIP: + /* for non UDP / GRE tunneling, set to 00b */ + break; + case PKT_TX_TUNNEL_VXLAN: + case PKT_TX_TUNNEL_GENEVE: + *cd_tunneling |= I40E_TXD_CTX_UDP_TUNNELING; + break; + case PKT_TX_TUNNEL_GRE: + *cd_tunneling |= I40E_TXD_CTX_GRE_TUNNELING; + break; + default: + PMD_TX_LOG(ERR, "Tunnel type not supported"); + return; + } + + /* L4TUNLEN: L4 Tunneling Length, in Words + * + * We depend on app to set rte_mbuf.l2_len correctly. + * For IP in GRE it should be set to the length of the GRE + * header; + * for MAC in GRE or MAC in UDP it should be set to the length + * of the GRE or UDP headers plus the inner MAC up to including + * its last Ethertype. + */ + *cd_tunneling |= (tx_offload.l2_len >> 1) << + I40E_TXD_CTX_QW0_NATLEN_SHIFT; +} + static inline void i40e_txd_enable_checksum(uint64_t ol_flags, uint32_t *td_cmd, uint32_t *td_offset, - union i40e_tx_offload tx_offload, - uint32_t *cd_tunneling) + union i40e_tx_offload tx_offload) { - /* UDP tunneling packet TX checksum offload */ - if (ol_flags & PKT_TX_OUTER_IP_CKSUM) { - + /* Set MACLEN */ + if (ol_flags & PKT_TX_TUNNEL_MASK) *td_offset |= (tx_offload.outer_l2_len >> 1) << I40E_TX_DESC_LENGTH_MACLEN_SHIFT; - - if (ol_flags & PKT_TX_OUTER_IP_CKSUM) - *cd_tunneling |= I40E_TX_CTX_EXT_IP_IPV4; - else if (ol_flags & PKT_TX_OUTER_IPV4) - *cd_tunneling |= I40E_TX_CTX_EXT_IP_IPV4_NO_CSUM; - else if (ol_flags & PKT_TX_OUTER_IPV6) - *cd_tunneling |= I40E_TX_CTX_EXT_IP_IPV6; - - /* Now set the ctx descriptor fields */ - *cd_tunneling |= (tx_offload.outer_l3_len >> 2) << - I40E_TXD_CTX_QW0_EXT_IPLEN_SHIFT | - (tx_offload.l2_len >> 1) << - I40E_TXD_CTX_QW0_NATLEN_SHIFT; - - } else + else *td_offset |= (tx_offload.l2_len >> 1) << I40E_TX_DESC_LENGTH_MACLEN_SHIFT; @@ -934,15 +432,6 @@ check_rx_burst_bulk_alloc_preconditions(__rte_unused struct i40e_rx_queue *rxq) "rxq->rx_free_thresh=%d", rxq->nb_rx_desc, rxq->rx_free_thresh); ret = -EINVAL; - } else if (!(rxq->nb_rx_desc < (I40E_MAX_RING_DESC - - RTE_PMD_I40E_RX_MAX_BURST))) { - PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: " - "rxq->nb_rx_desc=%d, " - "I40E_MAX_RING_DESC=%d, " - "RTE_PMD_I40E_RX_MAX_BURST=%d", - rxq->nb_rx_desc, I40E_MAX_RING_DESC, - RTE_PMD_I40E_RX_MAX_BURST); - ret = -EINVAL; } #else ret = -EINVAL; @@ -968,6 +457,7 @@ i40e_rx_scan_hw_ring(struct i40e_rx_queue *rxq) int32_t s[I40E_LOOK_AHEAD], nb_dd; int32_t i, j, nb_rx = 0; uint64_t pkt_flags; + uint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl; rxdp = &rxq->rx_ring[rxq->rx_tail]; rxep = &rxq->sw_ring[rxq->rx_tail]; @@ -994,6 +484,8 @@ i40e_rx_scan_hw_ring(struct i40e_rx_queue *rxq) I40E_RXD_QW1_STATUS_SHIFT; } + rte_smp_rmb(); + /* Compute how many status bits were set */ for (j = 0, nb_dd = 0; j < I40E_LOOK_AHEAD; j++) nb_dd += s[j] & (1 << I40E_RX_DESC_STATUS_DD_SHIFT); @@ -1014,9 +506,9 @@ i40e_rx_scan_hw_ring(struct i40e_rx_queue *rxq) pkt_flags = i40e_rxd_status_to_pkt_flags(qword1); pkt_flags |= i40e_rxd_error_to_pkt_flags(qword1); mb->packet_type = - i40e_rxd_pkt_type_mapping((uint8_t)((qword1 & - I40E_RXD_QW1_PTYPE_MASK) >> - I40E_RXD_QW1_PTYPE_SHIFT)); + ptype_tbl[(uint8_t)((qword1 & + I40E_RXD_QW1_PTYPE_MASK) >> + I40E_RXD_QW1_PTYPE_SHIFT)]; if (pkt_flags & PKT_RX_RSS_HASH) mb->hash.rss = rte_le_to_cpu_32(\ rxdp[j].wb.qword0.hi_dword.rss); @@ -1097,14 +589,14 @@ i40e_rx_alloc_bufs(struct i40e_rx_queue *rxq) mb->nb_segs = 1; mb->port = rxq->port_id; dma_addr = rte_cpu_to_le_64(\ - rte_mbuf_data_dma_addr_default(mb)); + rte_mbuf_data_iova_default(mb)); rxdp[i].read.hdr_addr = 0; rxdp[i].read.pkt_addr = dma_addr; } /* Update rx tail regsiter */ rte_wmb(); - I40E_PCI_REG_WRITE(rxq->qrx_tail, rxq->rx_free_trigger); + I40E_PCI_REG_WRITE_RELAXED(rxq->qrx_tail, rxq->rx_free_trigger); rxq->rx_free_trigger = (uint16_t)(rxq->rx_free_trigger + rxq->rx_free_thresh); @@ -1118,6 +610,7 @@ static inline uint16_t rx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) { struct i40e_rx_queue *rxq = (struct i40e_rx_queue *)rx_queue; + struct rte_eth_dev *dev; uint16_t nb_rx = 0; if (!nb_pkts) @@ -1135,9 +628,10 @@ rx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) if (i40e_rx_alloc_bufs(rxq) != 0) { uint16_t i, j; - PMD_RX_LOG(DEBUG, "Rx mbuf alloc failed for " - "port_id=%u, queue_id=%u", - rxq->port_id, rxq->queue_id); + dev = I40E_VSI_TO_ETH_DEV(rxq->vsi); + dev->data->rx_mbuf_alloc_failed += + rxq->rx_free_thresh; + rxq->rx_nb_avail = 0; rxq->rx_tail = (uint16_t)(rxq->rx_tail - nb_rx); for (i = 0, j = rxq->rx_tail; i < nb_rx; i++, j++) @@ -1199,6 +693,7 @@ i40e_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) union i40e_rx_desc rxd; struct i40e_rx_entry *sw_ring; struct i40e_rx_entry *rxe; + struct rte_eth_dev *dev; struct rte_mbuf *rxm; struct rte_mbuf *nmb; uint16_t nb_rx; @@ -1208,6 +703,7 @@ i40e_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) uint16_t rx_id, nb_hold; uint64_t dma_addr; uint64_t pkt_flags; + uint32_t *ptype_tbl; nb_rx = 0; nb_hold = 0; @@ -1215,6 +711,7 @@ i40e_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) rx_id = rxq->rx_tail; rx_ring = rxq->rx_ring; sw_ring = rxq->sw_ring; + ptype_tbl = rxq->vsi->adapter->ptype_tbl; while (nb_rx < nb_pkts) { rxdp = &rx_ring[rx_id]; @@ -1227,10 +724,13 @@ i40e_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) break; nmb = rte_mbuf_raw_alloc(rxq->mp); - if (unlikely(!nmb)) + if (unlikely(!nmb)) { + dev = I40E_VSI_TO_ETH_DEV(rxq->vsi); + dev->data->rx_mbuf_alloc_failed++; break; - rxd = *rxdp; + } + rxd = *rxdp; nb_hold++; rxe = &sw_ring[rx_id]; rx_id++; @@ -1252,7 +752,7 @@ i40e_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) rxm = rxe->mbuf; rxe->mbuf = nmb; dma_addr = - rte_cpu_to_le_64(rte_mbuf_data_dma_addr_default(nmb)); + rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb)); rxdp->read.hdr_addr = 0; rxdp->read.pkt_addr = dma_addr; @@ -1271,8 +771,8 @@ i40e_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) pkt_flags = i40e_rxd_status_to_pkt_flags(qword1); pkt_flags |= i40e_rxd_error_to_pkt_flags(qword1); rxm->packet_type = - i40e_rxd_pkt_type_mapping((uint8_t)((qword1 & - I40E_RXD_QW1_PTYPE_MASK) >> I40E_RXD_QW1_PTYPE_SHIFT)); + ptype_tbl[(uint8_t)((qword1 & + I40E_RXD_QW1_PTYPE_MASK) >> I40E_RXD_QW1_PTYPE_SHIFT)]; if (pkt_flags & PKT_RX_RSS_HASH) rxm->hash.rss = rte_le_to_cpu_32(rxd.wb.qword0.hi_dword.rss); @@ -1322,10 +822,12 @@ i40e_recv_scattered_pkts(void *rx_queue, struct rte_mbuf *nmb, *rxm; uint16_t rx_id = rxq->rx_tail; uint16_t nb_rx = 0, nb_hold = 0, rx_packet_len; + struct rte_eth_dev *dev; uint32_t rx_status; uint64_t qword1; uint64_t dma_addr; uint64_t pkt_flags; + uint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl; while (nb_rx < nb_pkts) { rxdp = &rx_ring[rx_id]; @@ -1338,8 +840,12 @@ i40e_recv_scattered_pkts(void *rx_queue, break; nmb = rte_mbuf_raw_alloc(rxq->mp); - if (unlikely(!nmb)) + if (unlikely(!nmb)) { + dev = I40E_VSI_TO_ETH_DEV(rxq->vsi); + dev->data->rx_mbuf_alloc_failed++; break; + } + rxd = *rxdp; nb_hold++; rxe = &sw_ring[rx_id]; @@ -1363,7 +869,7 @@ i40e_recv_scattered_pkts(void *rx_queue, rxm = rxe->mbuf; rxe->mbuf = nmb; dma_addr = - rte_cpu_to_le_64(rte_mbuf_data_dma_addr_default(nmb)); + rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb)); /* Set data buffer address and data length of the mbuf */ rxdp->read.hdr_addr = 0; @@ -1433,8 +939,8 @@ i40e_recv_scattered_pkts(void *rx_queue, pkt_flags = i40e_rxd_status_to_pkt_flags(qword1); pkt_flags |= i40e_rxd_error_to_pkt_flags(qword1); first_seg->packet_type = - i40e_rxd_pkt_type_mapping((uint8_t)((qword1 & - I40E_RXD_QW1_PTYPE_MASK) >> I40E_RXD_QW1_PTYPE_SHIFT)); + ptype_tbl[(uint8_t)((qword1 & + I40E_RXD_QW1_PTYPE_MASK) >> I40E_RXD_QW1_PTYPE_SHIFT)]; if (pkt_flags & PKT_RX_RSS_HASH) first_seg->hash.rss = rte_le_to_cpu_32(rxd.wb.qword0.hi_dword.rss); @@ -1484,7 +990,8 @@ i40e_calc_context_desc(uint64_t flags) { static uint64_t mask = PKT_TX_OUTER_IP_CKSUM | PKT_TX_TCP_SEG | - PKT_TX_QINQ_PKT; + PKT_TX_QINQ_PKT | + PKT_TX_TUNNEL_MASK; #ifdef RTE_LIBRTE_IEEE1588 mask |= PKT_TX_IEEE1588_TMST; @@ -1506,7 +1013,7 @@ i40e_set_tso_ctx(struct rte_mbuf *mbuf, union i40e_tx_offload tx_offload) } /** - * in case of tunneling packet, the outer_l2_len and + * in case of non tunneling packet, the outer_l2_len and * outer_l3_len must be 0. */ hdr_len = tx_offload.outer_l2_len + @@ -1541,7 +1048,6 @@ i40e_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) uint16_t nb_tx; uint32_t td_cmd; uint32_t td_offset; - uint32_t tx_flags; uint32_t td_tag; uint64_t ol_flags; uint16_t nb_used; @@ -1565,7 +1071,6 @@ i40e_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) td_cmd = 0; td_tag = 0; td_offset = 0; - tx_flags = 0; tx_pkt = *tx_pkts++; RTE_MBUF_PREFETCH_TO_FREE(txe->mbuf); @@ -1612,23 +1117,22 @@ i40e_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) /* Descriptor based VLAN insertion */ if (ol_flags & (PKT_TX_VLAN_PKT | PKT_TX_QINQ_PKT)) { - tx_flags |= tx_pkt->vlan_tci << - I40E_TX_FLAG_L2TAG1_SHIFT; - tx_flags |= I40E_TX_FLAG_INSERT_VLAN; td_cmd |= I40E_TX_DESC_CMD_IL2TAG1; - td_tag = (tx_flags & I40E_TX_FLAG_L2TAG1_MASK) >> - I40E_TX_FLAG_L2TAG1_SHIFT; + td_tag = tx_pkt->vlan_tci; } /* Always enable CRC offload insertion */ td_cmd |= I40E_TX_DESC_CMD_ICRC; - /* Enable checksum offloading */ + /* Fill in tunneling parameters if necessary */ cd_tunneling_params = 0; - if (ol_flags & I40E_TX_CKSUM_OFFLOAD_MASK) { - i40e_txd_enable_checksum(ol_flags, &td_cmd, &td_offset, - tx_offload, &cd_tunneling_params); - } + if (ol_flags & PKT_TX_TUNNEL_MASK) + i40e_parse_tunneling_params(ol_flags, tx_offload, + &cd_tunneling_params); + /* Enable checksum offloading */ + if (ol_flags & I40E_TX_CKSUM_OFFLOAD_MASK) + i40e_txd_enable_checksum(ol_flags, &td_cmd, + &td_offset, tx_offload); if (nb_ctx) { /* Setup TX context descriptor if required */ @@ -1698,7 +1202,7 @@ i40e_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) /* Setup TX Descriptor */ slen = m_seg->data_len; - buf_dma_addr = rte_mbuf_data_dma_addr(m_seg); + buf_dma_addr = rte_mbuf_data_iova(m_seg); PMD_TX_LOG(DEBUG, "mbuf: %p, TDD[%u]:\n" "buf_dma_addr: %#"PRIx64";\n" @@ -1747,13 +1251,13 @@ end_of_tx: (unsigned) txq->port_id, (unsigned) txq->queue_id, (unsigned) tx_id, (unsigned) nb_tx); - I40E_PCI_REG_WRITE(txq->qtx_tail, tx_id); + I40E_PCI_REG_WRITE_RELAXED(txq->qtx_tail, tx_id); txq->tx_tail = tx_id; return nb_tx; } -static inline int __attribute__((always_inline)) +static __rte_always_inline int i40e_tx_free_bufs(struct i40e_tx_queue *txq) { struct i40e_tx_entry *txep; @@ -1797,7 +1301,7 @@ tx4(volatile struct i40e_tx_desc *txdp, struct rte_mbuf **pkts) uint32_t i; for (i = 0; i < 4; i++, txdp++, pkts++) { - dma_addr = rte_mbuf_data_dma_addr(*pkts); + dma_addr = rte_mbuf_data_iova(*pkts); txdp->buffer_addr = rte_cpu_to_le_64(dma_addr); txdp->cmd_type_offset_bsz = i40e_build_ctob((uint32_t)I40E_TD_CMD, 0, @@ -1811,7 +1315,7 @@ tx1(volatile struct i40e_tx_desc *txdp, struct rte_mbuf **pkts) { uint64_t dma_addr; - dma_addr = rte_mbuf_data_dma_addr(*pkts); + dma_addr = rte_mbuf_data_iova(*pkts); txdp->buffer_addr = rte_cpu_to_le_64(dma_addr); txdp->cmd_type_offset_bsz = i40e_build_ctob((uint32_t)I40E_TD_CMD, 0, @@ -1899,7 +1403,7 @@ tx_xmit_pkts(struct i40e_tx_queue *txq, /* Update the tx tail register */ rte_wmb(); - I40E_PCI_REG_WRITE(txq->qtx_tail, txq->tx_tail); + I40E_PCI_REG_WRITE_RELAXED(txq->qtx_tail, txq->tx_tail); return nb_pkts; } @@ -1930,6 +1434,82 @@ i40e_xmit_pkts_simple(void *tx_queue, return nb_tx; } +static uint16_t +i40e_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts) +{ + uint16_t nb_tx = 0; + struct i40e_tx_queue *txq = (struct i40e_tx_queue *)tx_queue; + + while (nb_pkts) { + uint16_t ret, num; + + num = (uint16_t)RTE_MIN(nb_pkts, txq->tx_rs_thresh); + ret = i40e_xmit_fixed_burst_vec(tx_queue, &tx_pkts[nb_tx], + num); + nb_tx += ret; + nb_pkts -= ret; + if (ret < num) + break; + } + + return nb_tx; +} + +/********************************************************************* + * + * TX prep functions + * + **********************************************************************/ +uint16_t +i40e_prep_pkts(__rte_unused void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts) +{ + int i, ret; + uint64_t ol_flags; + struct rte_mbuf *m; + + for (i = 0; i < nb_pkts; i++) { + m = tx_pkts[i]; + ol_flags = m->ol_flags; + + /* Check for m->nb_segs to not exceed the limits. */ + if (!(ol_flags & PKT_TX_TCP_SEG)) { + if (m->nb_segs > I40E_TX_MAX_SEG || + m->nb_segs > I40E_TX_MAX_MTU_SEG) { + rte_errno = -EINVAL; + return i; + } + } else if ((m->tso_segsz < I40E_MIN_TSO_MSS) || + (m->tso_segsz > I40E_MAX_TSO_MSS)) { + /* MSS outside the range (256B - 9674B) are considered + * malicious + */ + rte_errno = -EINVAL; + return i; + } + + if (ol_flags & I40E_TX_OFFLOAD_NOTSUP_MASK) { + rte_errno = -ENOTSUP; + return i; + } + +#ifdef RTE_LIBRTE_ETHDEV_DEBUG + ret = rte_validate_tx_offload(m); + if (ret != 0) { + rte_errno = ret; + return i; + } +#endif + ret = rte_net_intel_cksum_prepare(m); + if (ret != 0) { + rte_errno = ret; + return i; + } + } + return i; +} + /* * Find the VSI the queue belongs to. 'queue_idx' is the queue index * application used, which assume having sequential ones. But from driver's @@ -2025,7 +1605,7 @@ i40e_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id) rxq = dev->data->rx_queues[rx_queue_id]; /* - * rx_queue_id is queue id aplication refers to, while + * rx_queue_id is queue id application refers to, while * rxq->reg_idx is the real queue index. */ err = i40e_switch_rx_queue(hw, rxq->reg_idx, FALSE); @@ -2056,7 +1636,7 @@ i40e_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id) txq = dev->data->tx_queues[tx_queue_id]; /* - * tx_queue_id is queue id aplication refers to, while + * tx_queue_id is queue id application refers to, while * rxq->reg_idx is the real queue index. */ err = i40e_switch_tx_queue(hw, txq->reg_idx, TRUE); @@ -2081,7 +1661,7 @@ i40e_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id) txq = dev->data->tx_queues[tx_queue_id]; /* - * tx_queue_id is queue id aplication refers to, while + * tx_queue_id is queue id application refers to, while * txq->reg_idx is the real queue index. */ err = i40e_switch_tx_queue(hw, txq->reg_idx, FALSE); @@ -2136,7 +1716,9 @@ i40e_dev_supported_ptypes_get(struct rte_eth_dev *dev) #ifdef RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC dev->rx_pkt_burst == i40e_recv_pkts_bulk_alloc || #endif - dev->rx_pkt_burst == i40e_recv_scattered_pkts) + dev->rx_pkt_burst == i40e_recv_scattered_pkts || + dev->rx_pkt_burst == i40e_recv_scattered_pkts_vec || + dev->rx_pkt_burst == i40e_recv_pkts_vec) return ptypes; return NULL; } @@ -2149,36 +1731,42 @@ i40e_dev_rx_queue_setup(struct rte_eth_dev *dev, const struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp) { - struct i40e_vsi *vsi; struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); - struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); struct i40e_adapter *ad = I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); + struct i40e_vsi *vsi; + struct i40e_pf *pf = NULL; + struct i40e_vf *vf = NULL; struct i40e_rx_queue *rxq; const struct rte_memzone *rz; uint32_t ring_size; uint16_t len, i; - uint16_t base, bsf, tc_mapping; - int use_def_burst_func = 1; + uint16_t reg_idx, base, bsf, tc_mapping; + int q_offset, use_def_burst_func = 1; if (hw->mac.type == I40E_MAC_VF || hw->mac.type == I40E_MAC_X722_VF) { - struct i40e_vf *vf = - I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); + vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); vsi = &vf->vsi; - } else + if (!vsi) + return -EINVAL; + reg_idx = queue_idx; + } else { + pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); vsi = i40e_pf_get_vsi_by_qindex(pf, queue_idx); - - if (vsi == NULL) { - PMD_DRV_LOG(ERR, "VSI not available or queue " - "index exceeds the maximum"); - return I40E_ERR_PARAM; + if (!vsi) + return -EINVAL; + q_offset = i40e_get_queue_offset_by_qindex(pf, queue_idx); + if (q_offset < 0) + return -EINVAL; + reg_idx = vsi->base_queue + q_offset; } + if (nb_desc % I40E_ALIGN_RING_DESC != 0 || - (nb_desc > I40E_MAX_RING_DESC) || - (nb_desc < I40E_MIN_RING_DESC)) { + (nb_desc > I40E_MAX_RING_DESC) || + (nb_desc < I40E_MIN_RING_DESC)) { PMD_DRV_LOG(ERR, "Number (%u) of receive descriptors is " "invalid", nb_desc); - return I40E_ERR_PARAM; + return -EINVAL; } /* Free memory if needed */ @@ -2201,12 +1789,7 @@ i40e_dev_rx_queue_setup(struct rte_eth_dev *dev, rxq->nb_rx_desc = nb_desc; rxq->rx_free_thresh = rx_conf->rx_free_thresh; rxq->queue_id = queue_idx; - if (hw->mac.type == I40E_MAC_VF || hw->mac.type == I40E_MAC_X722_VF) - rxq->reg_idx = queue_idx; - else /* PF device */ - rxq->reg_idx = vsi->base_queue + - i40e_get_queue_offset_by_qindex(pf, queue_idx); - + rxq->reg_idx = reg_idx; rxq->port_id = dev->data->port_id; rxq->crc_len = (uint8_t) ((dev->data->dev_conf.rxmode.hw_strip_crc) ? 0 : ETHER_CRC_LEN); @@ -2215,8 +1798,17 @@ i40e_dev_rx_queue_setup(struct rte_eth_dev *dev, rxq->rx_deferred_start = rx_conf->rx_deferred_start; /* Allocate the maximun number of RX ring hardware descriptor. */ - ring_size = sizeof(union i40e_rx_desc) * I40E_MAX_RING_DESC; - ring_size = RTE_ALIGN(ring_size, I40E_DMA_MEM_ALIGN); + len = I40E_MAX_RING_DESC; + + /** + * Allocating a little more memory because vectorized/bulk_alloc Rx + * functions doesn't check boundaries each time. + */ + len += RTE_PMD_I40E_RX_MAX_BURST; + + ring_size = RTE_ALIGN(len * sizeof(union i40e_rx_desc), + I40E_DMA_MEM_ALIGN); + rz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_idx, ring_size, I40E_RING_BASE_ALIGN, socket_id); if (!rz) { @@ -2228,14 +1820,10 @@ i40e_dev_rx_queue_setup(struct rte_eth_dev *dev, /* Zero all the descriptors in the ring. */ memset(rz->addr, 0, ring_size); - rxq->rx_ring_phys_addr = rte_mem_phy2mch(rz->memseg_id, rz->phys_addr); + rxq->rx_ring_phys_addr = rz->iova; rxq->rx_ring = (union i40e_rx_desc *)rz->addr; -#ifdef RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC len = (uint16_t)(nb_desc + RTE_PMD_I40E_RX_MAX_BURST); -#else - len = nb_desc; -#endif /* Allocate the software ring. */ rxq->sw_ring = @@ -2310,11 +1898,6 @@ i40e_dev_rx_queue_count(struct rte_eth_dev *dev, uint16_t rx_queue_id) struct i40e_rx_queue *rxq; uint16_t desc = 0; - if (unlikely(rx_queue_id >= dev->data->nb_rx_queues)) { - PMD_DRV_LOG(ERR, "Invalid RX queue id %u", rx_queue_id); - return 0; - } - rxq = dev->data->rx_queues[rx_queue_id]; rxdp = &(rxq->rx_ring[rxq->rx_tail]); while ((desc < rxq->nb_rx_desc) && @@ -2345,7 +1928,7 @@ i40e_dev_rx_descriptor_done(void *rx_queue, uint16_t offset) int ret; if (unlikely(offset >= rxq->nb_rx_desc)) { - PMD_DRV_LOG(ERR, "Invalid RX queue id %u", offset); + PMD_DRV_LOG(ERR, "Invalid RX descriptor id %u", offset); return 0; } @@ -2362,6 +1945,64 @@ i40e_dev_rx_descriptor_done(void *rx_queue, uint16_t offset) return ret; } +int +i40e_dev_rx_descriptor_status(void *rx_queue, uint16_t offset) +{ + struct i40e_rx_queue *rxq = rx_queue; + volatile uint64_t *status; + uint64_t mask; + uint32_t desc; + + if (unlikely(offset >= rxq->nb_rx_desc)) + return -EINVAL; + + if (offset >= rxq->nb_rx_desc - rxq->nb_rx_hold) + return RTE_ETH_RX_DESC_UNAVAIL; + + desc = rxq->rx_tail + offset; + if (desc >= rxq->nb_rx_desc) + desc -= rxq->nb_rx_desc; + + status = &rxq->rx_ring[desc].wb.qword1.status_error_len; + mask = rte_le_to_cpu_64((1ULL << I40E_RX_DESC_STATUS_DD_SHIFT) + << I40E_RXD_QW1_STATUS_SHIFT); + if (*status & mask) + return RTE_ETH_RX_DESC_DONE; + + return RTE_ETH_RX_DESC_AVAIL; +} + +int +i40e_dev_tx_descriptor_status(void *tx_queue, uint16_t offset) +{ + struct i40e_tx_queue *txq = tx_queue; + volatile uint64_t *status; + uint64_t mask, expect; + uint32_t desc; + + if (unlikely(offset >= txq->nb_tx_desc)) + return -EINVAL; + + desc = txq->tx_tail + offset; + /* go to next desc that has the RS bit */ + desc = ((desc + txq->tx_rs_thresh - 1) / txq->tx_rs_thresh) * + txq->tx_rs_thresh; + if (desc >= txq->nb_tx_desc) { + desc -= txq->nb_tx_desc; + if (desc >= txq->nb_tx_desc) + desc -= txq->nb_tx_desc; + } + + status = &txq->tx_ring[desc].cmd_type_offset_bsz; + mask = rte_le_to_cpu_64(I40E_TXD_QW1_DTYPE_MASK); + expect = rte_cpu_to_le_64( + I40E_TX_DESC_DTYPE_DESC_DONE << I40E_TXD_QW1_DTYPE_SHIFT); + if ((*status & mask) == expect) + return RTE_ETH_TX_DESC_DONE; + + return RTE_ETH_TX_DESC_FULL; +} + int i40e_dev_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, @@ -2369,34 +2010,40 @@ i40e_dev_tx_queue_setup(struct rte_eth_dev *dev, unsigned int socket_id, const struct rte_eth_txconf *tx_conf) { - struct i40e_vsi *vsi; struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); - struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + struct i40e_vsi *vsi; + struct i40e_pf *pf = NULL; + struct i40e_vf *vf = NULL; struct i40e_tx_queue *txq; const struct rte_memzone *tz; uint32_t ring_size; uint16_t tx_rs_thresh, tx_free_thresh; - uint16_t i, base, bsf, tc_mapping; + uint16_t reg_idx, i, base, bsf, tc_mapping; + int q_offset; if (hw->mac.type == I40E_MAC_VF || hw->mac.type == I40E_MAC_X722_VF) { - struct i40e_vf *vf = - I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); + vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); vsi = &vf->vsi; - } else + if (!vsi) + return -EINVAL; + reg_idx = queue_idx; + } else { + pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); vsi = i40e_pf_get_vsi_by_qindex(pf, queue_idx); - - if (vsi == NULL) { - PMD_DRV_LOG(ERR, "VSI is NULL, or queue index (%u) " - "exceeds the maximum", queue_idx); - return I40E_ERR_PARAM; + if (!vsi) + return -EINVAL; + q_offset = i40e_get_queue_offset_by_qindex(pf, queue_idx); + if (q_offset < 0) + return -EINVAL; + reg_idx = vsi->base_queue + q_offset; } if (nb_desc % I40E_ALIGN_RING_DESC != 0 || - (nb_desc > I40E_MAX_RING_DESC) || - (nb_desc < I40E_MIN_RING_DESC)) { + (nb_desc > I40E_MAX_RING_DESC) || + (nb_desc < I40E_MIN_RING_DESC)) { PMD_DRV_LOG(ERR, "Number (%u) of transmit descriptors is " "invalid", nb_desc); - return I40E_ERR_PARAM; + return -EINVAL; } /** @@ -2433,8 +2080,7 @@ i40e_dev_tx_queue_setup(struct rte_eth_dev *dev, return I40E_ERR_PARAM; } if (tx_free_thresh >= (nb_desc - 3)) { - PMD_INIT_LOG(ERR, "tx_rs_thresh must be less than the " - "tx_free_thresh must be less than the " + PMD_INIT_LOG(ERR, "tx_free_thresh must be less than the " "number of TX descriptors minus 3. " "(tx_free_thresh=%u port=%d queue=%d)", (unsigned int)tx_free_thresh, @@ -2506,18 +2152,13 @@ i40e_dev_tx_queue_setup(struct rte_eth_dev *dev, txq->hthresh = tx_conf->tx_thresh.hthresh; txq->wthresh = tx_conf->tx_thresh.wthresh; txq->queue_id = queue_idx; - if (hw->mac.type == I40E_MAC_VF || hw->mac.type == I40E_MAC_X722_VF) - txq->reg_idx = queue_idx; - else /* PF device */ - txq->reg_idx = vsi->base_queue + - i40e_get_queue_offset_by_qindex(pf, queue_idx); - + txq->reg_idx = reg_idx; txq->port_id = dev->data->port_id; txq->txq_flags = tx_conf->txq_flags; txq->vsi = vsi; txq->tx_deferred_start = tx_conf->tx_deferred_start; - txq->tx_ring_phys_addr = rte_mem_phy2mch(tz->memseg_id, tz->phys_addr); + txq->tx_ring_phys_addr = tz->iova; txq->tx_ring = (struct i40e_tx_desc *)tz->addr; /* Allocate software ring */ @@ -2579,12 +2220,8 @@ i40e_memzone_reserve(const char *name, uint32_t len, int socket_id) if (mz) return mz; - if (rte_xen_dom0_supported()) - mz = rte_memzone_reserve_bounded(name, len, - socket_id, 0, I40E_RING_BASE_ALIGN, RTE_PGSIZE_2M); - else - mz = rte_memzone_reserve_aligned(name, len, - socket_id, 0, I40E_RING_BASE_ALIGN); + mz = rte_memzone_reserve_aligned(name, len, + socket_id, 0, I40E_RING_BASE_ALIGN); return mz; } @@ -2644,11 +2281,11 @@ i40e_reset_rx_queue(struct i40e_rx_queue *rxq) for (i = 0; i < len * sizeof(union i40e_rx_desc); i++) ((volatile char *)rxq->rx_ring)[i] = 0; -#ifdef RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC memset(&rxq->fake_mbuf, 0x0, sizeof(rxq->fake_mbuf)); for (i = 0; i < RTE_PMD_I40E_RX_MAX_BURST; ++i) rxq->sw_ring[rxq->nb_rx_desc + i].mbuf = &rxq->fake_mbuf; +#ifdef RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC rxq->rx_nb_avail = 0; rxq->rx_next_avail = 0; rxq->rx_free_trigger = (uint16_t)(rxq->rx_free_thresh - 1); @@ -2665,18 +2302,40 @@ i40e_reset_rx_queue(struct i40e_rx_queue *rxq) void i40e_tx_queue_release_mbufs(struct i40e_tx_queue *txq) { + struct rte_eth_dev *dev; uint16_t i; + dev = &rte_eth_devices[txq->port_id]; + if (!txq || !txq->sw_ring) { PMD_DRV_LOG(DEBUG, "Pointer to rxq or sw_ring is NULL"); return; } - for (i = 0; i < txq->nb_tx_desc; i++) { - if (txq->sw_ring[i].mbuf) { + /** + * vPMD tx will not set sw_ring's mbuf to NULL after free, + * so need to free remains more carefully. + */ + if (dev->tx_pkt_burst == i40e_xmit_pkts_vec) { + i = txq->tx_next_dd - txq->tx_rs_thresh + 1; + if (txq->tx_tail < i) { + for (; i < txq->nb_tx_desc; i++) { + rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf); + txq->sw_ring[i].mbuf = NULL; + } + i = 0; + } + for (; i < txq->tx_tail; i++) { rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf); txq->sw_ring[i].mbuf = NULL; } + } else { + for (i = 0; i < txq->nb_tx_desc; i++) { + if (txq->sw_ring[i].mbuf) { + rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf); + txq->sw_ring[i].mbuf = NULL; + } + } } } @@ -2789,7 +2448,7 @@ i40e_alloc_rx_queue_mbufs(struct i40e_rx_queue *rxq) mbuf->port = rxq->port_id; dma_addr = - rte_cpu_to_le_64(rte_mbuf_data_dma_addr_default(mbuf)); + rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf)); rxd = &rxq->rx_ring[i]; rxd->read.pkt_addr = dma_addr; @@ -2832,7 +2491,7 @@ i40e_rx_queue_config(struct i40e_rx_queue *rxq) case I40E_FLAG_HEADER_SPLIT_DISABLED: default: rxq->rx_hdr_len = 0; - rxq->rx_buf_len = RTE_ALIGN(buf_size, + rxq->rx_buf_len = RTE_ALIGN_FLOOR(buf_size, (1 << I40E_RXQ_CTX_DBUFF_SHIFT)); rxq->hs_mode = i40e_header_split_none; break; @@ -3033,7 +2692,7 @@ i40e_fdir_setup_tx_resources(struct i40e_pf *pf) txq->reg_idx = pf->fdir.fdir_vsi->base_queue; txq->vsi = pf->fdir.fdir_vsi; - txq->tx_ring_phys_addr = rte_mem_phy2mch(tz->memseg_id, tz->phys_addr); + txq->tx_ring_phys_addr = tz->iova; txq->tx_ring = (struct i40e_tx_desc *)tz->addr; /* * don't need to allocate software ring and reset for the fdir @@ -3089,7 +2748,8 @@ i40e_fdir_setup_rx_resources(struct i40e_pf *pf) rxq->reg_idx = pf->fdir.fdir_vsi->base_queue; rxq->vsi = pf->fdir.fdir_vsi; - rxq->rx_ring_phys_addr = rte_mem_phy2mch(rz->memseg_id, rz->phys_addr); + rxq->rx_ring_phys_addr = rz->iova; + memset(rz->addr, 0, I40E_FDIR_NUM_RX_DESC * sizeof(union i40e_rx_desc)); rxq->rx_ring = (union i40e_rx_desc *)rz->addr; /* @@ -3280,9 +2940,80 @@ i40e_set_tx_function(struct rte_eth_dev *dev) PMD_INIT_LOG(DEBUG, "Simple tx finally be used."); dev->tx_pkt_burst = i40e_xmit_pkts_simple; } + dev->tx_pkt_prepare = NULL; } else { PMD_INIT_LOG(DEBUG, "Xmit tx finally be used."); dev->tx_pkt_burst = i40e_xmit_pkts; + dev->tx_pkt_prepare = i40e_prep_pkts; + } +} + +void __attribute__((cold)) +i40e_set_default_ptype_table(struct rte_eth_dev *dev) +{ + struct i40e_adapter *ad = + I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); + int i; + + for (i = 0; i < I40E_MAX_PKT_TYPE; i++) + ad->ptype_tbl[i] = i40e_get_default_pkt_type(i); +} + +void __attribute__((cold)) +i40e_set_default_pctype_table(struct rte_eth_dev *dev) +{ + struct i40e_adapter *ad = + I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); + struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); + int i; + + for (i = 0; i < I40E_FLOW_TYPE_MAX; i++) + ad->pctypes_tbl[i] = 0ULL; + ad->flow_types_mask = 0ULL; + ad->pctypes_mask = 0ULL; + + ad->pctypes_tbl[RTE_ETH_FLOW_FRAG_IPV4] = + (1ULL << I40E_FILTER_PCTYPE_FRAG_IPV4); + ad->pctypes_tbl[RTE_ETH_FLOW_NONFRAG_IPV4_UDP] = + (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_UDP); + ad->pctypes_tbl[RTE_ETH_FLOW_NONFRAG_IPV4_TCP] = + (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_TCP); + ad->pctypes_tbl[RTE_ETH_FLOW_NONFRAG_IPV4_SCTP] = + (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_SCTP); + ad->pctypes_tbl[RTE_ETH_FLOW_NONFRAG_IPV4_OTHER] = + (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER); + ad->pctypes_tbl[RTE_ETH_FLOW_FRAG_IPV6] = + (1ULL << I40E_FILTER_PCTYPE_FRAG_IPV6); + ad->pctypes_tbl[RTE_ETH_FLOW_NONFRAG_IPV6_UDP] = + (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_UDP); + ad->pctypes_tbl[RTE_ETH_FLOW_NONFRAG_IPV6_TCP] = + (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_TCP); + ad->pctypes_tbl[RTE_ETH_FLOW_NONFRAG_IPV6_SCTP] = + (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_SCTP); + ad->pctypes_tbl[RTE_ETH_FLOW_NONFRAG_IPV6_OTHER] = + (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER); + ad->pctypes_tbl[RTE_ETH_FLOW_L2_PAYLOAD] = + (1ULL << I40E_FILTER_PCTYPE_L2_PAYLOAD); + + if (hw->mac.type == I40E_MAC_X722) { + ad->pctypes_tbl[RTE_ETH_FLOW_NONFRAG_IPV4_UDP] |= + (1ULL << I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP); + ad->pctypes_tbl[RTE_ETH_FLOW_NONFRAG_IPV4_UDP] |= + (1ULL << I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP); + ad->pctypes_tbl[RTE_ETH_FLOW_NONFRAG_IPV4_TCP] |= + (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK); + ad->pctypes_tbl[RTE_ETH_FLOW_NONFRAG_IPV6_UDP] |= + (1ULL << I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP); + ad->pctypes_tbl[RTE_ETH_FLOW_NONFRAG_IPV6_UDP] |= + (1ULL << I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP); + ad->pctypes_tbl[RTE_ETH_FLOW_NONFRAG_IPV6_TCP] |= + (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK); + } + + for (i = 0; i < I40E_FLOW_TYPE_MAX; i++) { + if (ad->pctypes_tbl[i]) + ad->flow_types_mask |= (1ULL << i); + ad->pctypes_mask |= ad->pctypes_tbl[i]; } } @@ -3330,9 +3061,9 @@ i40e_rx_queue_release_mbufs_vec(struct i40e_rx_queue __rte_unused*rxq) } uint16_t __attribute__((weak)) -i40e_xmit_pkts_vec(void __rte_unused *tx_queue, - struct rte_mbuf __rte_unused **tx_pkts, - uint16_t __rte_unused nb_pkts) +i40e_xmit_fixed_burst_vec(void __rte_unused * tx_queue, + struct rte_mbuf __rte_unused **tx_pkts, + uint16_t __rte_unused nb_pkts) { return 0; } diff --git a/dpdk/drivers/net/i40e/i40e_rxtx.h b/dpdk/drivers/net/i40e/i40e_rxtx.h index 98179f00..06c6a659 100644 --- a/dpdk/drivers/net/i40e/i40e_rxtx.h +++ b/dpdk/drivers/net/i40e/i40e_rxtx.h @@ -34,16 +34,6 @@ #ifndef _I40E_RXTX_H_ #define _I40E_RXTX_H_ -/** - * 32 bits tx flags, high 16 bits for L2TAG1 (VLAN), - * low 16 bits for others. - */ -#define I40E_TX_FLAG_L2TAG1_SHIFT 16 -#define I40E_TX_FLAG_L2TAG1_MASK 0xffff0000 -#define I40E_TX_FLAG_CSUM ((uint32_t)(1 << 0)) -#define I40E_TX_FLAG_INSERT_VLAN ((uint32_t)(1 << 1)) -#define I40E_TX_FLAG_TSYN ((uint32_t)(1 << 2)) - #define RTE_PMD_I40E_RX_MAX_BURST 32 #define RTE_PMD_I40E_TX_MAX_BURST 32 @@ -63,6 +53,12 @@ #define I40E_MIN_RING_DESC 64 #define I40E_MAX_RING_DESC 4096 +#define I40E_MIN_TSO_MSS 256 +#define I40E_MAX_TSO_MSS 9674 + +#define I40E_TX_MAX_SEG UINT8_MAX +#define I40E_TX_MAX_MTU_SEG 8 + #undef container_of #define container_of(ptr, type, member) ({ \ typeof(((type *)0)->member)(*__mptr) = (ptr); \ @@ -113,11 +109,11 @@ struct i40e_rx_queue { uint16_t nb_rx_hold; /**< number of held free RX desc */ struct rte_mbuf *pkt_first_seg; /**< first segment of current packet */ struct rte_mbuf *pkt_last_seg; /**< last segment of current packet */ + struct rte_mbuf fake_mbuf; /**< dummy mbuf */ #ifdef RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC uint16_t rx_nb_avail; /**< number of staged packets ready */ uint16_t rx_next_avail; /**< index of next staged packets */ uint16_t rx_free_trigger; /**< triggers rx buffer allocation */ - struct rte_mbuf fake_mbuf; /**< dummy mbuf */ struct rte_mbuf *rx_stage[RTE_PMD_I40E_RX_MAX_BURST * 2]; #endif @@ -125,7 +121,7 @@ struct i40e_rx_queue { uint16_t rxrearm_start; /**< the idx we start the re-arming from */ uint64_t mbuf_initializer; /**< value to init mbufs */ - uint8_t port_id; /**< device port ID */ + uint16_t port_id; /**< device port ID */ uint8_t crc_len; /**< 0 if CRC stripped, 4 otherwise */ uint16_t queue_id; /**< RX queue index */ uint16_t reg_idx; /**< RX queue register index */ @@ -171,7 +167,7 @@ struct i40e_tx_queue { uint8_t pthresh; /**< Prefetch threshold register. */ uint8_t hthresh; /**< Host threshold register. */ uint8_t wthresh; /**< Write-back threshold reg. */ - uint8_t port_id; /**< Device port identifier. */ + uint16_t port_id; /**< Device port identifier. */ uint16_t queue_id; /**< TX queue index. */ uint16_t reg_idx; uint32_t txq_flags; @@ -223,6 +219,8 @@ uint16_t i40e_recv_scattered_pkts(void *rx_queue, uint16_t i40e_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts); +uint16_t i40e_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts); int i40e_tx_queue_init(struct i40e_tx_queue *txq); int i40e_rx_queue_init(struct i40e_rx_queue *rxq); void i40e_free_tx_resources(struct i40e_tx_queue *txq); @@ -238,6 +236,8 @@ void i40e_rx_queue_release_mbufs(struct i40e_rx_queue *rxq); uint32_t i40e_dev_rx_queue_count(struct rte_eth_dev *dev, uint16_t rx_queue_id); int i40e_dev_rx_descriptor_done(void *rx_queue, uint16_t offset); +int i40e_dev_rx_descriptor_status(void *rx_queue, uint16_t offset); +int i40e_dev_tx_descriptor_status(void *tx_queue, uint16_t offset); uint16_t i40e_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts); @@ -248,11 +248,576 @@ int i40e_rx_vec_dev_conf_condition_check(struct rte_eth_dev *dev); int i40e_rxq_vec_setup(struct i40e_rx_queue *rxq); int i40e_txq_vec_setup(struct i40e_tx_queue *txq); void i40e_rx_queue_release_mbufs_vec(struct i40e_rx_queue *rxq); -uint16_t i40e_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts, - uint16_t nb_pkts); +uint16_t i40e_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts); void i40e_set_rx_function(struct rte_eth_dev *dev); void i40e_set_tx_function_flag(struct rte_eth_dev *dev, struct i40e_tx_queue *txq); void i40e_set_tx_function(struct rte_eth_dev *dev); +void i40e_set_default_ptype_table(struct rte_eth_dev *dev); +void i40e_set_default_pctype_table(struct rte_eth_dev *dev); + +/* For each value it means, datasheet of hardware can tell more details + * + * @note: fix i40e_dev_supported_ptypes_get() if any change here. + */ +static inline uint32_t +i40e_get_default_pkt_type(uint8_t ptype) +{ + static const uint32_t type_table[UINT8_MAX + 1] __rte_cache_aligned = { + /* L2 types */ + /* [0] reserved */ + [1] = RTE_PTYPE_L2_ETHER, + [2] = RTE_PTYPE_L2_ETHER_TIMESYNC, + /* [3] - [5] reserved */ + [6] = RTE_PTYPE_L2_ETHER_LLDP, + /* [7] - [10] reserved */ + [11] = RTE_PTYPE_L2_ETHER_ARP, + /* [12] - [21] reserved */ + + /* Non tunneled IPv4 */ + [22] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_FRAG, + [23] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_NONFRAG, + [24] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_UDP, + /* [25] reserved */ + [26] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_TCP, + [27] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_SCTP, + [28] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_ICMP, + + /* IPv4 --> IPv4 */ + [29] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_FRAG, + [30] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_NONFRAG, + [31] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_UDP, + /* [32] reserved */ + [33] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_TCP, + [34] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_SCTP, + [35] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_ICMP, + + /* IPv4 --> IPv6 */ + [36] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_FRAG, + [37] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_NONFRAG, + [38] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_UDP, + /* [39] reserved */ + [40] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_TCP, + [41] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_SCTP, + [42] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_ICMP, + + /* IPv4 --> GRE/Teredo/VXLAN */ + [43] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT, + + /* IPv4 --> GRE/Teredo/VXLAN --> IPv4 */ + [44] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_FRAG, + [45] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_NONFRAG, + [46] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_UDP, + /* [47] reserved */ + [48] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_TCP, + [49] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_SCTP, + [50] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_ICMP, + + /* IPv4 --> GRE/Teredo/VXLAN --> IPv6 */ + [51] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_FRAG, + [52] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_NONFRAG, + [53] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_UDP, + /* [54] reserved */ + [55] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_TCP, + [56] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_SCTP, + [57] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_ICMP, + + /* IPv4 --> GRE/Teredo/VXLAN --> MAC */ + [58] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER, + + /* IPv4 --> GRE/Teredo/VXLAN --> MAC --> IPv4 */ + [59] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_FRAG, + [60] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_NONFRAG, + [61] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_UDP, + /* [62] reserved */ + [63] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_TCP, + [64] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_SCTP, + [65] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_ICMP, + + /* IPv4 --> GRE/Teredo/VXLAN --> MAC --> IPv6 */ + [66] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_FRAG, + [67] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_NONFRAG, + [68] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_UDP, + /* [69] reserved */ + [70] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_TCP, + [71] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_SCTP, + [72] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_ICMP, + + /* IPv4 --> GRE/Teredo/VXLAN --> MAC/VLAN */ + [73] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L2_ETHER_VLAN, + + /* IPv4 --> GRE/Teredo/VXLAN --> MAC/VLAN --> IPv4 */ + [74] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L2_ETHER_VLAN | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_FRAG, + [75] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L2_ETHER_VLAN | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_NONFRAG, + [76] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L2_ETHER_VLAN | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_UDP, + /* [77] reserved */ + [78] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L2_ETHER_VLAN | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_TCP, + [79] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L2_ETHER_VLAN | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_SCTP, + [80] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L2_ETHER_VLAN | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_ICMP, + + /* IPv4 --> GRE/Teredo/VXLAN --> MAC/VLAN --> IPv6 */ + [81] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L2_ETHER_VLAN | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_FRAG, + [82] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L2_ETHER_VLAN | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_NONFRAG, + [83] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L2_ETHER_VLAN | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_UDP, + /* [84] reserved */ + [85] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L2_ETHER_VLAN | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_TCP, + [86] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L2_ETHER_VLAN | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_SCTP, + [87] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L2_ETHER_VLAN | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_ICMP, + + /* Non tunneled IPv6 */ + [88] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_L4_FRAG, + [89] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_L4_NONFRAG, + [90] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_L4_UDP, + /* [91] reserved */ + [92] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_L4_TCP, + [93] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_L4_SCTP, + [94] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_L4_ICMP, + + /* IPv6 --> IPv4 */ + [95] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_FRAG, + [96] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_NONFRAG, + [97] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_UDP, + /* [98] reserved */ + [99] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_TCP, + [100] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_SCTP, + [101] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_ICMP, + + /* IPv6 --> IPv6 */ + [102] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_FRAG, + [103] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_NONFRAG, + [104] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_UDP, + /* [105] reserved */ + [106] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_TCP, + [107] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_SCTP, + [108] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_ICMP, + + /* IPv6 --> GRE/Teredo/VXLAN */ + [109] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT, + + /* IPv6 --> GRE/Teredo/VXLAN --> IPv4 */ + [110] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_FRAG, + [111] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_NONFRAG, + [112] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_UDP, + /* [113] reserved */ + [114] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_TCP, + [115] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_SCTP, + [116] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_ICMP, + + /* IPv6 --> GRE/Teredo/VXLAN --> IPv6 */ + [117] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_FRAG, + [118] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_NONFRAG, + [119] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_UDP, + /* [120] reserved */ + [121] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_TCP, + [122] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_SCTP, + [123] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_ICMP, + + /* IPv6 --> GRE/Teredo/VXLAN --> MAC */ + [124] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER, + + /* IPv6 --> GRE/Teredo/VXLAN --> MAC --> IPv4 */ + [125] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_FRAG, + [126] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_NONFRAG, + [127] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_UDP, + /* [128] reserved */ + [129] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_TCP, + [130] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_SCTP, + [131] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_ICMP, + + /* IPv6 --> GRE/Teredo/VXLAN --> MAC --> IPv6 */ + [132] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_FRAG, + [133] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_NONFRAG, + [134] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_UDP, + /* [135] reserved */ + [136] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_TCP, + [137] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_SCTP, + [138] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_ICMP, + + /* IPv6 --> GRE/Teredo/VXLAN --> MAC/VLAN */ + [139] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L2_ETHER_VLAN, + + /* IPv6 --> GRE/Teredo/VXLAN --> MAC/VLAN --> IPv4 */ + [140] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L2_ETHER_VLAN | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_FRAG, + [141] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L2_ETHER_VLAN | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_NONFRAG, + [142] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L2_ETHER_VLAN | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_UDP, + /* [143] reserved */ + [144] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L2_ETHER_VLAN | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_TCP, + [145] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L2_ETHER_VLAN | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_SCTP, + [146] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L2_ETHER_VLAN | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_ICMP, + + /* IPv6 --> GRE/Teredo/VXLAN --> MAC/VLAN --> IPv6 */ + [147] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L2_ETHER_VLAN | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_FRAG, + [148] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L2_ETHER_VLAN | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_NONFRAG, + [149] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L2_ETHER_VLAN | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_UDP, + /* [150] reserved */ + [151] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L2_ETHER_VLAN | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_TCP, + [152] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L2_ETHER_VLAN | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_SCTP, + [153] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRENAT | + RTE_PTYPE_INNER_L2_ETHER_VLAN | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_ICMP, + + /* L2 NSH packet type */ + [154] = RTE_PTYPE_L2_ETHER_NSH, + [155] = RTE_PTYPE_L2_ETHER_NSH | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_FRAG, + [156] = RTE_PTYPE_L2_ETHER_NSH | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_NONFRAG, + [157] = RTE_PTYPE_L2_ETHER_NSH | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_UDP, + [158] = RTE_PTYPE_L2_ETHER_NSH | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_TCP, + [159] = RTE_PTYPE_L2_ETHER_NSH | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_SCTP, + [160] = RTE_PTYPE_L2_ETHER_NSH | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_ICMP, + [161] = RTE_PTYPE_L2_ETHER_NSH | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_L4_FRAG, + [162] = RTE_PTYPE_L2_ETHER_NSH | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_L4_NONFRAG, + [163] = RTE_PTYPE_L2_ETHER_NSH | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_L4_UDP, + [164] = RTE_PTYPE_L2_ETHER_NSH | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_L4_TCP, + [165] = RTE_PTYPE_L2_ETHER_NSH | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_L4_SCTP, + [166] = RTE_PTYPE_L2_ETHER_NSH | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_L4_ICMP, + + /* All others reserved */ + }; + + return type_table[ptype]; +} #endif /* _I40E_RXTX_H_ */ diff --git a/dpdk/drivers/net/i40e/i40e_rxtx_vec.c b/dpdk/drivers/net/i40e/i40e_rxtx_vec.c deleted file mode 100644 index a9649d35..00000000 --- a/dpdk/drivers/net/i40e/i40e_rxtx_vec.c +++ /dev/null @@ -1,775 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include -#include -#include - -#include "base/i40e_prototype.h" -#include "base/i40e_type.h" -#include "i40e_ethdev.h" -#include "i40e_rxtx.h" - -#include - -#ifndef __INTEL_COMPILER -#pragma GCC diagnostic ignored "-Wcast-qual" -#endif - -static inline void -i40e_rxq_rearm(struct i40e_rx_queue *rxq) -{ - int i; - uint16_t rx_id; - volatile union i40e_rx_desc *rxdp; - struct i40e_rx_entry *rxep = &rxq->sw_ring[rxq->rxrearm_start]; - struct rte_mbuf *mb0, *mb1; - __m128i hdr_room = _mm_set_epi64x(RTE_PKTMBUF_HEADROOM, - RTE_PKTMBUF_HEADROOM); - __m128i dma_addr0, dma_addr1; - - rxdp = rxq->rx_ring + rxq->rxrearm_start; - - /* Pull 'n' more MBUFs into the software ring */ - if (rte_mempool_get_bulk(rxq->mp, - (void *)rxep, - RTE_I40E_RXQ_REARM_THRESH) < 0) { - if (rxq->rxrearm_nb + RTE_I40E_RXQ_REARM_THRESH >= - rxq->nb_rx_desc) { - dma_addr0 = _mm_setzero_si128(); - for (i = 0; i < RTE_I40E_DESCS_PER_LOOP; i++) { - rxep[i].mbuf = &rxq->fake_mbuf; - _mm_store_si128((__m128i *)&rxdp[i].read, - dma_addr0); - } - } - rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed += - RTE_I40E_RXQ_REARM_THRESH; - return; - } - - /* Initialize the mbufs in vector, process 2 mbufs in one loop */ - for (i = 0; i < RTE_I40E_RXQ_REARM_THRESH; i += 2, rxep += 2) { - __m128i vaddr0, vaddr1; - uintptr_t p0, p1; - - mb0 = rxep[0].mbuf; - mb1 = rxep[1].mbuf; - - /* Flush mbuf with pkt template. - * Data to be rearmed is 6 bytes long. - * Though, RX will overwrite ol_flags that are coming next - * anyway. So overwrite whole 8 bytes with one load: - * 6 bytes of rearm_data plus first 2 bytes of ol_flags. - */ - p0 = (uintptr_t)&mb0->rearm_data; - *(uint64_t *)p0 = rxq->mbuf_initializer; - p1 = (uintptr_t)&mb1->rearm_data; - *(uint64_t *)p1 = rxq->mbuf_initializer; - - /* load buf_addr(lo 64bit) and buf_physaddr(hi 64bit) */ - vaddr0 = _mm_loadu_si128((__m128i *)&mb0->buf_addr); - vaddr1 = _mm_loadu_si128((__m128i *)&mb1->buf_addr); - - /* convert pa to dma_addr hdr/data */ - dma_addr0 = _mm_unpackhi_epi64(vaddr0, vaddr0); - dma_addr1 = _mm_unpackhi_epi64(vaddr1, vaddr1); - - /* add headroom to pa values */ - dma_addr0 = _mm_add_epi64(dma_addr0, hdr_room); - dma_addr1 = _mm_add_epi64(dma_addr1, hdr_room); - - /* flush desc with pa dma_addr */ - _mm_store_si128((__m128i *)&rxdp++->read, dma_addr0); - _mm_store_si128((__m128i *)&rxdp++->read, dma_addr1); - } - - rxq->rxrearm_start += RTE_I40E_RXQ_REARM_THRESH; - if (rxq->rxrearm_start >= rxq->nb_rx_desc) - rxq->rxrearm_start = 0; - - rxq->rxrearm_nb -= RTE_I40E_RXQ_REARM_THRESH; - - rx_id = (uint16_t)((rxq->rxrearm_start == 0) ? - (rxq->nb_rx_desc - 1) : (rxq->rxrearm_start - 1)); - - /* Update the tail pointer on the NIC */ - I40E_PCI_REG_WRITE(rxq->qrx_tail, rx_id); -} - -/* Handling the offload flags (olflags) field takes computation - * time when receiving packets. Therefore we provide a flag to disable - * the processing of the olflags field when they are not needed. This - * gives improved performance, at the cost of losing the offload info - * in the received packet - */ -#ifdef RTE_LIBRTE_I40E_RX_OLFLAGS_ENABLE - -static inline void -desc_to_olflags_v(__m128i descs[4], struct rte_mbuf **rx_pkts) -{ - __m128i vlan0, vlan1, rss; - union { - uint16_t e[4]; - uint64_t dword; - } vol; - - /* mask everything except RSS, flow director and VLAN flags - * bit2 is for VLAN tag, bit11 for flow director indication - * bit13:12 for RSS indication. - */ - const __m128i rss_vlan_msk = _mm_set_epi16( - 0x0000, 0x0000, 0x0000, 0x0000, - 0x3804, 0x3804, 0x3804, 0x3804); - - /* map rss and vlan type to rss hash and vlan flag */ - const __m128i vlan_flags = _mm_set_epi8(0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, PKT_RX_VLAN_PKT | PKT_RX_VLAN_STRIPPED, - 0, 0, 0, 0); - - const __m128i rss_flags = _mm_set_epi8(0, 0, 0, 0, - 0, 0, 0, 0, - PKT_RX_RSS_HASH | PKT_RX_FDIR, PKT_RX_RSS_HASH, 0, 0, - 0, 0, PKT_RX_FDIR, 0); - - vlan0 = _mm_unpackhi_epi16(descs[0], descs[1]); - vlan1 = _mm_unpackhi_epi16(descs[2], descs[3]); - vlan0 = _mm_unpacklo_epi32(vlan0, vlan1); - - vlan1 = _mm_and_si128(vlan0, rss_vlan_msk); - vlan0 = _mm_shuffle_epi8(vlan_flags, vlan1); - - rss = _mm_srli_epi16(vlan1, 11); - rss = _mm_shuffle_epi8(rss_flags, rss); - - vlan0 = _mm_or_si128(vlan0, rss); - vol.dword = _mm_cvtsi128_si64(vlan0); - - rx_pkts[0]->ol_flags = vol.e[0]; - rx_pkts[1]->ol_flags = vol.e[1]; - rx_pkts[2]->ol_flags = vol.e[2]; - rx_pkts[3]->ol_flags = vol.e[3]; -} -#else -#define desc_to_olflags_v(desc, rx_pkts) do {} while (0) -#endif - -#define PKTLEN_SHIFT 10 - - /* - * Notice: - * - nb_pkts < RTE_I40E_DESCS_PER_LOOP, just return no packet - * - nb_pkts > RTE_I40E_VPMD_RX_BURST, only scan RTE_I40E_VPMD_RX_BURST - * numbers of DD bits - */ -static inline uint16_t -_recv_raw_pkts_vec(struct i40e_rx_queue *rxq, struct rte_mbuf **rx_pkts, - uint16_t nb_pkts, uint8_t *split_packet) -{ - volatile union i40e_rx_desc *rxdp; - struct i40e_rx_entry *sw_ring; - uint16_t nb_pkts_recd; - int pos; - uint64_t var; - __m128i shuf_msk; - - __m128i crc_adjust = _mm_set_epi16( - 0, 0, 0, /* ignore non-length fields */ - -rxq->crc_len, /* sub crc on data_len */ - 0, /* ignore high-16bits of pkt_len */ - -rxq->crc_len, /* sub crc on pkt_len */ - 0, 0 /* ignore pkt_type field */ - ); - __m128i dd_check, eop_check; - - /* nb_pkts shall be less equal than RTE_I40E_MAX_RX_BURST */ - nb_pkts = RTE_MIN(nb_pkts, RTE_I40E_MAX_RX_BURST); - - /* nb_pkts has to be floor-aligned to RTE_I40E_DESCS_PER_LOOP */ - nb_pkts = RTE_ALIGN_FLOOR(nb_pkts, RTE_I40E_DESCS_PER_LOOP); - - /* Just the act of getting into the function from the application is - * going to cost about 7 cycles - */ - rxdp = rxq->rx_ring + rxq->rx_tail; - - _mm_prefetch((const void *)rxdp, _MM_HINT_T0); - - /* See if we need to rearm the RX queue - gives the prefetch a bit - * of time to act - */ - if (rxq->rxrearm_nb > RTE_I40E_RXQ_REARM_THRESH) - i40e_rxq_rearm(rxq); - - /* Before we start moving massive data around, check to see if - * there is actually a packet available - */ - if (!(rxdp->wb.qword1.status_error_len & - rte_cpu_to_le_32(1 << I40E_RX_DESC_STATUS_DD_SHIFT))) - return 0; - - /* 4 packets DD mask */ - dd_check = _mm_set_epi64x(0x0000000100000001LL, 0x0000000100000001LL); - - /* 4 packets EOP mask */ - eop_check = _mm_set_epi64x(0x0000000200000002LL, 0x0000000200000002LL); - - /* mask to shuffle from desc. to mbuf */ - shuf_msk = _mm_set_epi8( - 7, 6, 5, 4, /* octet 4~7, 32bits rss */ - 3, 2, /* octet 2~3, low 16 bits vlan_macip */ - 15, 14, /* octet 15~14, 16 bits data_len */ - 0xFF, 0xFF, /* skip high 16 bits pkt_len, zero out */ - 15, 14, /* octet 15~14, low 16 bits pkt_len */ - 0xFF, 0xFF, /* pkt_type set as unknown */ - 0xFF, 0xFF /*pkt_type set as unknown */ - ); - - /* Cache is empty -> need to scan the buffer rings, but first move - * the next 'n' mbufs into the cache - */ - sw_ring = &rxq->sw_ring[rxq->rx_tail]; - - /* A. load 4 packet in one loop - * [A*. mask out 4 unused dirty field in desc] - * B. copy 4 mbuf point from swring to rx_pkts - * C. calc the number of DD bits among the 4 packets - * [C*. extract the end-of-packet bit, if requested] - * D. fill info. from desc to mbuf - */ - - for (pos = 0, nb_pkts_recd = 0; pos < nb_pkts; - pos += RTE_I40E_DESCS_PER_LOOP, - rxdp += RTE_I40E_DESCS_PER_LOOP) { - __m128i descs[RTE_I40E_DESCS_PER_LOOP]; - __m128i pkt_mb1, pkt_mb2, pkt_mb3, pkt_mb4; - __m128i zero, staterr, sterr_tmp1, sterr_tmp2; - __m128i mbp1, mbp2; /* two mbuf pointer in one XMM reg. */ - - /* B.1 load 1 mbuf point */ - mbp1 = _mm_loadu_si128((__m128i *)&sw_ring[pos]); - /* Read desc statuses backwards to avoid race condition */ - /* A.1 load 4 pkts desc */ - descs[3] = _mm_loadu_si128((__m128i *)(rxdp + 3)); - rte_compiler_barrier(); - /* B.2 copy 2 mbuf point into rx_pkts */ - _mm_storeu_si128((__m128i *)&rx_pkts[pos], mbp1); - - /* B.1 load 1 mbuf point */ - mbp2 = _mm_loadu_si128((__m128i *)&sw_ring[pos+2]); - - descs[2] = _mm_loadu_si128((__m128i *)(rxdp + 2)); - rte_compiler_barrier(); - /* B.1 load 2 mbuf point */ - descs[1] = _mm_loadu_si128((__m128i *)(rxdp + 1)); - rte_compiler_barrier(); - descs[0] = _mm_loadu_si128((__m128i *)(rxdp)); - - /* B.2 copy 2 mbuf point into rx_pkts */ - _mm_storeu_si128((__m128i *)&rx_pkts[pos+2], mbp2); - - if (split_packet) { - rte_mbuf_prefetch_part2(rx_pkts[pos]); - rte_mbuf_prefetch_part2(rx_pkts[pos + 1]); - rte_mbuf_prefetch_part2(rx_pkts[pos + 2]); - rte_mbuf_prefetch_part2(rx_pkts[pos + 3]); - } - - /* avoid compiler reorder optimization */ - rte_compiler_barrier(); - - /* pkt 3,4 shift the pktlen field to be 16-bit aligned*/ - const __m128i len3 = _mm_slli_epi32(descs[3], PKTLEN_SHIFT); - const __m128i len2 = _mm_slli_epi32(descs[2], PKTLEN_SHIFT); - - /* merge the now-aligned packet length fields back in */ - descs[3] = _mm_blend_epi16(descs[3], len3, 0x80); - descs[2] = _mm_blend_epi16(descs[2], len2, 0x80); - - /* D.1 pkt 3,4 convert format from desc to pktmbuf */ - pkt_mb4 = _mm_shuffle_epi8(descs[3], shuf_msk); - pkt_mb3 = _mm_shuffle_epi8(descs[2], shuf_msk); - - /* C.1 4=>2 filter staterr info only */ - sterr_tmp2 = _mm_unpackhi_epi32(descs[3], descs[2]); - /* C.1 4=>2 filter staterr info only */ - sterr_tmp1 = _mm_unpackhi_epi32(descs[1], descs[0]); - - desc_to_olflags_v(descs, &rx_pkts[pos]); - - /* D.2 pkt 3,4 set in_port/nb_seg and remove crc */ - pkt_mb4 = _mm_add_epi16(pkt_mb4, crc_adjust); - pkt_mb3 = _mm_add_epi16(pkt_mb3, crc_adjust); - - /* pkt 1,2 shift the pktlen field to be 16-bit aligned*/ - const __m128i len1 = _mm_slli_epi32(descs[1], PKTLEN_SHIFT); - const __m128i len0 = _mm_slli_epi32(descs[0], PKTLEN_SHIFT); - - /* merge the now-aligned packet length fields back in */ - descs[1] = _mm_blend_epi16(descs[1], len1, 0x80); - descs[0] = _mm_blend_epi16(descs[0], len0, 0x80); - - /* D.1 pkt 1,2 convert format from desc to pktmbuf */ - pkt_mb2 = _mm_shuffle_epi8(descs[1], shuf_msk); - pkt_mb1 = _mm_shuffle_epi8(descs[0], shuf_msk); - - /* C.2 get 4 pkts staterr value */ - zero = _mm_xor_si128(dd_check, dd_check); - staterr = _mm_unpacklo_epi32(sterr_tmp1, sterr_tmp2); - - /* D.3 copy final 3,4 data to rx_pkts */ - _mm_storeu_si128((void *)&rx_pkts[pos+3]->rx_descriptor_fields1, - pkt_mb4); - _mm_storeu_si128((void *)&rx_pkts[pos+2]->rx_descriptor_fields1, - pkt_mb3); - - /* D.2 pkt 1,2 set in_port/nb_seg and remove crc */ - pkt_mb2 = _mm_add_epi16(pkt_mb2, crc_adjust); - pkt_mb1 = _mm_add_epi16(pkt_mb1, crc_adjust); - - /* C* extract and record EOP bit */ - if (split_packet) { - __m128i eop_shuf_mask = _mm_set_epi8( - 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, - 0x04, 0x0C, 0x00, 0x08 - ); - - /* and with mask to extract bits, flipping 1-0 */ - __m128i eop_bits = _mm_andnot_si128(staterr, eop_check); - /* the staterr values are not in order, as the count - * count of dd bits doesn't care. However, for end of - * packet tracking, we do care, so shuffle. This also - * compresses the 32-bit values to 8-bit - */ - eop_bits = _mm_shuffle_epi8(eop_bits, eop_shuf_mask); - /* store the resulting 32-bit value */ - *(int *)split_packet = _mm_cvtsi128_si32(eop_bits); - split_packet += RTE_I40E_DESCS_PER_LOOP; - - /* zero-out next pointers */ - rx_pkts[pos]->next = NULL; - rx_pkts[pos + 1]->next = NULL; - rx_pkts[pos + 2]->next = NULL; - rx_pkts[pos + 3]->next = NULL; - } - - /* C.3 calc available number of desc */ - staterr = _mm_and_si128(staterr, dd_check); - staterr = _mm_packs_epi32(staterr, zero); - - /* D.3 copy final 1,2 data to rx_pkts */ - _mm_storeu_si128((void *)&rx_pkts[pos+1]->rx_descriptor_fields1, - pkt_mb2); - _mm_storeu_si128((void *)&rx_pkts[pos]->rx_descriptor_fields1, - pkt_mb1); - /* C.4 calc avaialbe number of desc */ - var = __builtin_popcountll(_mm_cvtsi128_si64(staterr)); - nb_pkts_recd += var; - if (likely(var != RTE_I40E_DESCS_PER_LOOP)) - break; - } - - /* Update our internal tail pointer */ - rxq->rx_tail = (uint16_t)(rxq->rx_tail + nb_pkts_recd); - rxq->rx_tail = (uint16_t)(rxq->rx_tail & (rxq->nb_rx_desc - 1)); - rxq->rxrearm_nb = (uint16_t)(rxq->rxrearm_nb + nb_pkts_recd); - - return nb_pkts_recd; -} - - /* - * Notice: - * - nb_pkts < RTE_I40E_DESCS_PER_LOOP, just return no packet - * - nb_pkts > RTE_I40E_VPMD_RX_BURST, only scan RTE_I40E_VPMD_RX_BURST - * numbers of DD bits - */ -uint16_t -i40e_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, - uint16_t nb_pkts) -{ - return _recv_raw_pkts_vec(rx_queue, rx_pkts, nb_pkts, NULL); -} - -static inline uint16_t -reassemble_packets(struct i40e_rx_queue *rxq, struct rte_mbuf **rx_bufs, - uint16_t nb_bufs, uint8_t *split_flags) -{ - struct rte_mbuf *pkts[RTE_I40E_VPMD_RX_BURST]; /*finished pkts*/ - struct rte_mbuf *start = rxq->pkt_first_seg; - struct rte_mbuf *end = rxq->pkt_last_seg; - unsigned pkt_idx, buf_idx; - - for (buf_idx = 0, pkt_idx = 0; buf_idx < nb_bufs; buf_idx++) { - if (end != NULL) { - /* processing a split packet */ - end->next = rx_bufs[buf_idx]; - rx_bufs[buf_idx]->data_len += rxq->crc_len; - - start->nb_segs++; - start->pkt_len += rx_bufs[buf_idx]->data_len; - end = end->next; - - if (!split_flags[buf_idx]) { - /* it's the last packet of the set */ - start->hash = end->hash; - start->ol_flags = end->ol_flags; - /* we need to strip crc for the whole packet */ - start->pkt_len -= rxq->crc_len; - if (end->data_len > rxq->crc_len) { - end->data_len -= rxq->crc_len; - } else { - /* free up last mbuf */ - struct rte_mbuf *secondlast = start; - - while (secondlast->next != end) - secondlast = secondlast->next; - secondlast->data_len -= (rxq->crc_len - - end->data_len); - secondlast->next = NULL; - rte_pktmbuf_free_seg(end); - end = secondlast; - } - pkts[pkt_idx++] = start; - start = end = NULL; - } - } else { - /* not processing a split packet */ - if (!split_flags[buf_idx]) { - /* not a split packet, save and skip */ - pkts[pkt_idx++] = rx_bufs[buf_idx]; - continue; - } - end = start = rx_bufs[buf_idx]; - rx_bufs[buf_idx]->data_len += rxq->crc_len; - rx_bufs[buf_idx]->pkt_len += rxq->crc_len; - } - } - - /* save the partial packet for next time */ - rxq->pkt_first_seg = start; - rxq->pkt_last_seg = end; - memcpy(rx_bufs, pkts, pkt_idx * (sizeof(*pkts))); - return pkt_idx; -} - - /* vPMD receive routine that reassembles scattered packets - * Notice: - * - nb_pkts < RTE_I40E_DESCS_PER_LOOP, just return no packet - * - nb_pkts > RTE_I40E_VPMD_RX_BURST, only scan RTE_I40E_VPMD_RX_BURST - * numbers of DD bits - */ -uint16_t -i40e_recv_scattered_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, - uint16_t nb_pkts) -{ - - struct i40e_rx_queue *rxq = rx_queue; - uint8_t split_flags[RTE_I40E_VPMD_RX_BURST] = {0}; - - /* get some new buffers */ - uint16_t nb_bufs = _recv_raw_pkts_vec(rxq, rx_pkts, nb_pkts, - split_flags); - if (nb_bufs == 0) - return 0; - - /* happy day case, full burst + no packets to be joined */ - const uint64_t *split_fl64 = (uint64_t *)split_flags; - - if (rxq->pkt_first_seg == NULL && - split_fl64[0] == 0 && split_fl64[1] == 0 && - split_fl64[2] == 0 && split_fl64[3] == 0) - return nb_bufs; - - /* reassemble any packets that need reassembly*/ - unsigned i = 0; - - if (rxq->pkt_first_seg == NULL) { - /* find the first split flag, and only reassemble then*/ - while (i < nb_bufs && !split_flags[i]) - i++; - if (i == nb_bufs) - return nb_bufs; - } - return i + reassemble_packets(rxq, &rx_pkts[i], nb_bufs - i, - &split_flags[i]); -} - -static inline void -vtx1(volatile struct i40e_tx_desc *txdp, - struct rte_mbuf *pkt, uint64_t flags) -{ - uint64_t high_qw = (I40E_TX_DESC_DTYPE_DATA | - ((uint64_t)flags << I40E_TXD_QW1_CMD_SHIFT) | - ((uint64_t)pkt->data_len << I40E_TXD_QW1_TX_BUF_SZ_SHIFT)); - - __m128i descriptor = _mm_set_epi64x(high_qw, - pkt->buf_physaddr + pkt->data_off); - _mm_store_si128((__m128i *)txdp, descriptor); -} - -static inline void -vtx(volatile struct i40e_tx_desc *txdp, - struct rte_mbuf **pkt, uint16_t nb_pkts, uint64_t flags) -{ - int i; - - for (i = 0; i < nb_pkts; ++i, ++txdp, ++pkt) - vtx1(txdp, *pkt, flags); -} - -static inline int __attribute__((always_inline)) -i40e_tx_free_bufs(struct i40e_tx_queue *txq) -{ - struct i40e_tx_entry *txep; - uint32_t n; - uint32_t i; - int nb_free = 0; - struct rte_mbuf *m, *free[RTE_I40E_TX_MAX_FREE_BUF_SZ]; - - /* check DD bits on threshold descriptor */ - if ((txq->tx_ring[txq->tx_next_dd].cmd_type_offset_bsz & - rte_cpu_to_le_64(I40E_TXD_QW1_DTYPE_MASK)) != - rte_cpu_to_le_64(I40E_TX_DESC_DTYPE_DESC_DONE)) - return 0; - - n = txq->tx_rs_thresh; - - /* first buffer to free from S/W ring is at index - * tx_next_dd - (tx_rs_thresh-1) - */ - txep = &txq->sw_ring[txq->tx_next_dd - (n - 1)]; - m = __rte_pktmbuf_prefree_seg(txep[0].mbuf); - if (likely(m != NULL)) { - free[0] = m; - nb_free = 1; - for (i = 1; i < n; i++) { - m = __rte_pktmbuf_prefree_seg(txep[i].mbuf); - if (likely(m != NULL)) { - if (likely(m->pool == free[0]->pool)) { - free[nb_free++] = m; - } else { - rte_mempool_put_bulk(free[0]->pool, - (void *)free, - nb_free); - free[0] = m; - nb_free = 1; - } - } - } - rte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free); - } else { - for (i = 1; i < n; i++) { - m = __rte_pktmbuf_prefree_seg(txep[i].mbuf); - if (m != NULL) - rte_mempool_put(m->pool, m); - } - } - - /* buffers were freed, update counters */ - txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_rs_thresh); - txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_rs_thresh); - if (txq->tx_next_dd >= txq->nb_tx_desc) - txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1); - - return txq->tx_rs_thresh; -} - -static inline void __attribute__((always_inline)) -tx_backlog_entry(struct i40e_tx_entry *txep, - struct rte_mbuf **tx_pkts, uint16_t nb_pkts) -{ - int i; - - for (i = 0; i < (int)nb_pkts; ++i) - txep[i].mbuf = tx_pkts[i]; -} - -uint16_t -i40e_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts, - uint16_t nb_pkts) -{ - struct i40e_tx_queue *txq = (struct i40e_tx_queue *)tx_queue; - volatile struct i40e_tx_desc *txdp; - struct i40e_tx_entry *txep; - uint16_t n, nb_commit, tx_id; - uint64_t flags = I40E_TD_CMD; - uint64_t rs = I40E_TX_DESC_CMD_RS | I40E_TD_CMD; - int i; - - /* cross rx_thresh boundary is not allowed */ - nb_pkts = RTE_MIN(nb_pkts, txq->tx_rs_thresh); - - if (txq->nb_tx_free < txq->tx_free_thresh) - i40e_tx_free_bufs(txq); - - nb_commit = nb_pkts = (uint16_t)RTE_MIN(txq->nb_tx_free, nb_pkts); - if (unlikely(nb_pkts == 0)) - return 0; - - tx_id = txq->tx_tail; - txdp = &txq->tx_ring[tx_id]; - txep = &txq->sw_ring[tx_id]; - - txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_pkts); - - n = (uint16_t)(txq->nb_tx_desc - tx_id); - if (nb_commit >= n) { - tx_backlog_entry(txep, tx_pkts, n); - - for (i = 0; i < n - 1; ++i, ++tx_pkts, ++txdp) - vtx1(txdp, *tx_pkts, flags); - - vtx1(txdp, *tx_pkts++, rs); - - nb_commit = (uint16_t)(nb_commit - n); - - tx_id = 0; - txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1); - - /* avoid reach the end of ring */ - txdp = &txq->tx_ring[tx_id]; - txep = &txq->sw_ring[tx_id]; - } - - tx_backlog_entry(txep, tx_pkts, nb_commit); - - vtx(txdp, tx_pkts, nb_commit, flags); - - tx_id = (uint16_t)(tx_id + nb_commit); - if (tx_id > txq->tx_next_rs) { - txq->tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |= - rte_cpu_to_le_64(((uint64_t)I40E_TX_DESC_CMD_RS) << - I40E_TXD_QW1_CMD_SHIFT); - txq->tx_next_rs = - (uint16_t)(txq->tx_next_rs + txq->tx_rs_thresh); - } - - txq->tx_tail = tx_id; - - I40E_PCI_REG_WRITE(txq->qtx_tail, txq->tx_tail); - - return nb_pkts; -} - -void __attribute__((cold)) -i40e_rx_queue_release_mbufs_vec(struct i40e_rx_queue *rxq) -{ - const unsigned mask = rxq->nb_rx_desc - 1; - unsigned i; - - if (rxq->sw_ring == NULL || rxq->rxrearm_nb >= rxq->nb_rx_desc) - return; - - /* free all mbufs that are valid in the ring */ - if (rxq->rxrearm_nb == 0) { - for (i = 0; i < rxq->nb_rx_desc; i++) { - if (rxq->sw_ring[i].mbuf != NULL) - rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf); - } - } else { - for (i = rxq->rx_tail; - i != rxq->rxrearm_start; - i = (i + 1) & mask) { - if (rxq->sw_ring[i].mbuf != NULL) - rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf); - } - } - - rxq->rxrearm_nb = rxq->nb_rx_desc; - - /* set all entries to NULL */ - memset(rxq->sw_ring, 0, sizeof(rxq->sw_ring[0]) * rxq->nb_rx_desc); -} - -int __attribute__((cold)) -i40e_rxq_vec_setup(struct i40e_rx_queue *rxq) -{ - uintptr_t p; - struct rte_mbuf mb_def = { .buf_addr = 0 }; /* zeroed mbuf */ - - mb_def.nb_segs = 1; - mb_def.data_off = RTE_PKTMBUF_HEADROOM; - mb_def.port = rxq->port_id; - rte_mbuf_refcnt_set(&mb_def, 1); - - /* prevent compiler reordering: rearm_data covers previous fields */ - rte_compiler_barrier(); - p = (uintptr_t)&mb_def.rearm_data; - rxq->mbuf_initializer = *(uint64_t *)p; - return 0; -} - -int __attribute__((cold)) -i40e_txq_vec_setup(struct i40e_tx_queue __rte_unused *txq) -{ - return 0; -} - -int __attribute__((cold)) -i40e_rx_vec_dev_conf_condition_check(struct rte_eth_dev *dev) -{ -#ifndef RTE_LIBRTE_IEEE1588 - struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode; - struct rte_fdir_conf *fconf = &dev->data->dev_conf.fdir_conf; - - /* need SSE4.1 support */ - if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_SSE4_1)) - return -1; - -#ifndef RTE_LIBRTE_I40E_RX_OLFLAGS_ENABLE - /* whithout rx ol_flags, no VP flag report */ - if (rxmode->hw_vlan_strip != 0 || - rxmode->hw_vlan_extend != 0) - return -1; -#endif - - /* no fdir support */ - if (fconf->mode != RTE_FDIR_MODE_NONE) - return -1; - - /* - no csum error report support - * - no header split support - */ - if (rxmode->hw_ip_checksum == 1 || - rxmode->header_split == 1) - return -1; - - return 0; -#else - RTE_SET_USED(dev); - return -1; -#endif -} diff --git a/dpdk/drivers/net/i40e/i40e_rxtx_vec_altivec.c b/dpdk/drivers/net/i40e/i40e_rxtx_vec_altivec.c new file mode 100644 index 00000000..5e4e472a --- /dev/null +++ b/dpdk/drivers/net/i40e/i40e_rxtx_vec_altivec.c @@ -0,0 +1,645 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * Copyright(c) 2017 IBM Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include + +#include "base/i40e_prototype.h" +#include "base/i40e_type.h" +#include "i40e_ethdev.h" +#include "i40e_rxtx.h" +#include "i40e_rxtx_vec_common.h" + +#include + +#pragma GCC diagnostic ignored "-Wcast-qual" + +static inline void +i40e_rxq_rearm(struct i40e_rx_queue *rxq) +{ + int i; + uint16_t rx_id; + volatile union i40e_rx_desc *rxdp; + + struct i40e_rx_entry *rxep = &rxq->sw_ring[rxq->rxrearm_start]; + struct rte_mbuf *mb0, *mb1; + + vector unsigned long hdr_room = (vector unsigned long){ + RTE_PKTMBUF_HEADROOM, + RTE_PKTMBUF_HEADROOM}; + vector unsigned long dma_addr0, dma_addr1; + + rxdp = rxq->rx_ring + rxq->rxrearm_start; + + /* Pull 'n' more MBUFs into the software ring */ + if (rte_mempool_get_bulk(rxq->mp, + (void *)rxep, + RTE_I40E_RXQ_REARM_THRESH) < 0) { + if (rxq->rxrearm_nb + RTE_I40E_RXQ_REARM_THRESH >= + rxq->nb_rx_desc) { + dma_addr0 = (vector unsigned long){}; + for (i = 0; i < RTE_I40E_DESCS_PER_LOOP; i++) { + rxep[i].mbuf = &rxq->fake_mbuf; + vec_st(dma_addr0, 0, + (vector unsigned long *)&rxdp[i].read); + } + } + rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed += + RTE_I40E_RXQ_REARM_THRESH; + return; + } + + /* Initialize the mbufs in vector, process 2 mbufs in one loop */ + for (i = 0; i < RTE_I40E_RXQ_REARM_THRESH; i += 2, rxep += 2) { + vector unsigned long vaddr0, vaddr1; + uintptr_t p0, p1; + + mb0 = rxep[0].mbuf; + mb1 = rxep[1].mbuf; + + /* Flush mbuf with pkt template. + * Data to be rearmed is 6 bytes long. + * Though, RX will overwrite ol_flags that are coming next + * anyway. So overwrite whole 8 bytes with one load: + * 6 bytes of rearm_data plus first 2 bytes of ol_flags. + */ + p0 = (uintptr_t)&mb0->rearm_data; + *(uint64_t *)p0 = rxq->mbuf_initializer; + p1 = (uintptr_t)&mb1->rearm_data; + *(uint64_t *)p1 = rxq->mbuf_initializer; + + /* load buf_addr(lo 64bit) and buf_iova(hi 64bit) */ + vaddr0 = vec_ld(0, (vector unsigned long *)&mb0->buf_addr); + vaddr1 = vec_ld(0, (vector unsigned long *)&mb1->buf_addr); + + /* convert pa to dma_addr hdr/data */ + dma_addr0 = vec_mergel(vaddr0, vaddr0); + dma_addr1 = vec_mergel(vaddr1, vaddr1); + + /* add headroom to pa values */ + dma_addr0 = vec_add(dma_addr0, hdr_room); + dma_addr1 = vec_add(dma_addr1, hdr_room); + + /* flush desc with pa dma_addr */ + vec_st(dma_addr0, 0, (vector unsigned long *)&rxdp++->read); + vec_st(dma_addr1, 0, (vector unsigned long *)&rxdp++->read); + } + + rxq->rxrearm_start += RTE_I40E_RXQ_REARM_THRESH; + if (rxq->rxrearm_start >= rxq->nb_rx_desc) + rxq->rxrearm_start = 0; + + rxq->rxrearm_nb -= RTE_I40E_RXQ_REARM_THRESH; + + rx_id = (uint16_t)((rxq->rxrearm_start == 0) ? + (rxq->nb_rx_desc - 1) : (rxq->rxrearm_start - 1)); + + /* Update the tail pointer on the NIC */ + I40E_PCI_REG_WRITE(rxq->qrx_tail, rx_id); +} + +static inline void +desc_to_olflags_v(vector unsigned long descs[4], struct rte_mbuf **rx_pkts) +{ + vector unsigned int vlan0, vlan1, rss, l3_l4e; + + /* mask everything except RSS, flow director and VLAN flags + * bit2 is for VLAN tag, bit11 for flow director indication + * bit13:12 for RSS indication. + */ + const vector unsigned int rss_vlan_msk = (vector unsigned int){ + (int32_t)0x1c03804, (int32_t)0x1c03804, + (int32_t)0x1c03804, (int32_t)0x1c03804}; + + /* map rss and vlan type to rss hash and vlan flag */ + const vector unsigned char vlan_flags = (vector unsigned char){ + 0, 0, 0, 0, + PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0}; + + const vector unsigned char rss_flags = (vector unsigned char){ + 0, PKT_RX_FDIR, 0, 0, + 0, 0, PKT_RX_RSS_HASH, PKT_RX_RSS_HASH | PKT_RX_FDIR, + 0, 0, 0, 0, + 0, 0, 0, 0}; + + const vector unsigned char l3_l4e_flags = (vector unsigned char){ + 0, + PKT_RX_IP_CKSUM_BAD, + PKT_RX_L4_CKSUM_BAD, + PKT_RX_L4_CKSUM_BAD | PKT_RX_IP_CKSUM_BAD, + PKT_RX_EIP_CKSUM_BAD, + PKT_RX_EIP_CKSUM_BAD | PKT_RX_IP_CKSUM_BAD, + PKT_RX_EIP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD, + PKT_RX_EIP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD + | PKT_RX_IP_CKSUM_BAD, + 0, 0, 0, 0, 0, 0, 0, 0}; + + vlan0 = (vector unsigned int)vec_mergel(descs[0], descs[1]); + vlan1 = (vector unsigned int)vec_mergel(descs[2], descs[3]); + vlan0 = (vector unsigned int)vec_mergeh(vlan0, vlan1); + + vlan1 = vec_and(vlan0, rss_vlan_msk); + vlan0 = (vector unsigned int)vec_perm(vlan_flags, + (vector unsigned char){}, + *(vector unsigned char *)&vlan1); + + rss = vec_sr(vlan1, (vector unsigned int){11, 11, 11, 11}); + rss = (vector unsigned int)vec_perm(rss_flags, (vector unsigned char){}, + *(vector unsigned char *)&rss); + + l3_l4e = vec_sr(vlan1, (vector unsigned int){22, 22, 22, 22}); + l3_l4e = (vector unsigned int)vec_perm(l3_l4e_flags, + (vector unsigned char){}, + *(vector unsigned char *)&l3_l4e); + + vlan0 = vec_or(vlan0, rss); + vlan0 = vec_or(vlan0, l3_l4e); + + rx_pkts[0]->ol_flags = (uint64_t)vlan0[2]; + rx_pkts[1]->ol_flags = (uint64_t)vlan0[3]; + rx_pkts[2]->ol_flags = (uint64_t)vlan0[0]; + rx_pkts[3]->ol_flags = (uint64_t)vlan0[1]; +} + +#define PKTLEN_SHIFT 10 + +static inline void +desc_to_ptype_v(vector unsigned long descs[4], struct rte_mbuf **rx_pkts, + uint32_t *ptype_tbl) +{ + vector unsigned long ptype0 = vec_mergel(descs[0], descs[1]); + vector unsigned long ptype1 = vec_mergel(descs[2], descs[3]); + + ptype0 = vec_sr(ptype0, (vector unsigned long){30, 30}); + ptype1 = vec_sr(ptype1, (vector unsigned long){30, 30}); + + rx_pkts[0]->packet_type = + ptype_tbl[(*(vector unsigned char *)&ptype0)[0]]; + rx_pkts[1]->packet_type = + ptype_tbl[(*(vector unsigned char *)&ptype0)[8]]; + rx_pkts[2]->packet_type = + ptype_tbl[(*(vector unsigned char *)&ptype1)[0]]; + rx_pkts[3]->packet_type = + ptype_tbl[(*(vector unsigned char *)&ptype1)[8]]; +} + + /* Notice: + * - nb_pkts < RTE_I40E_DESCS_PER_LOOP, just return no packet + * - nb_pkts > RTE_I40E_VPMD_RX_BURST, only scan RTE_I40E_VPMD_RX_BURST + * numbers of DD bits + */ +static inline uint16_t +_recv_raw_pkts_vec(struct i40e_rx_queue *rxq, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts, uint8_t *split_packet) +{ + volatile union i40e_rx_desc *rxdp; + struct i40e_rx_entry *sw_ring; + uint16_t nb_pkts_recd; + int pos; + uint64_t var; + vector unsigned char shuf_msk; + uint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl; + + vector unsigned short crc_adjust = (vector unsigned short){ + 0, 0, /* ignore pkt_type field */ + rxq->crc_len, /* sub crc on pkt_len */ + 0, /* ignore high-16bits of pkt_len */ + rxq->crc_len, /* sub crc on data_len */ + 0, 0, 0 /* ignore non-length fields */ + }; + vector unsigned long dd_check, eop_check; + + /* nb_pkts shall be less equal than RTE_I40E_MAX_RX_BURST */ + nb_pkts = RTE_MIN(nb_pkts, RTE_I40E_MAX_RX_BURST); + + /* nb_pkts has to be floor-aligned to RTE_I40E_DESCS_PER_LOOP */ + nb_pkts = RTE_ALIGN_FLOOR(nb_pkts, RTE_I40E_DESCS_PER_LOOP); + + /* Just the act of getting into the function from the application is + * going to cost about 7 cycles + */ + rxdp = rxq->rx_ring + rxq->rx_tail; + + rte_prefetch0(rxdp); + + /* See if we need to rearm the RX queue - gives the prefetch a bit + * of time to act + */ + if (rxq->rxrearm_nb > RTE_I40E_RXQ_REARM_THRESH) + i40e_rxq_rearm(rxq); + + /* Before we start moving massive data around, check to see if + * there is actually a packet available + */ + if (!(rxdp->wb.qword1.status_error_len & + rte_cpu_to_le_32(1 << I40E_RX_DESC_STATUS_DD_SHIFT))) + return 0; + + /* 4 packets DD mask */ + dd_check = (vector unsigned long){0x0000000100000001ULL, + 0x0000000100000001ULL}; + + /* 4 packets EOP mask */ + eop_check = (vector unsigned long){0x0000000200000002ULL, + 0x0000000200000002ULL}; + + /* mask to shuffle from desc. to mbuf */ + shuf_msk = (vector unsigned char){ + 0xFF, 0xFF, /* pkt_type set as unknown */ + 0xFF, 0xFF, /* pkt_type set as unknown */ + 14, 15, /* octet 15~14, low 16 bits pkt_len */ + 0xFF, 0xFF, /* skip high 16 bits pkt_len, zero out */ + 14, 15, /* octet 15~14, 16 bits data_len */ + 2, 3, /* octet 2~3, low 16 bits vlan_macip */ + 4, 5, 6, 7 /* octet 4~7, 32bits rss */ + }; + + /* Cache is empty -> need to scan the buffer rings, but first move + * the next 'n' mbufs into the cache + */ + sw_ring = &rxq->sw_ring[rxq->rx_tail]; + + /* A. load 4 packet in one loop + * [A*. mask out 4 unused dirty field in desc] + * B. copy 4 mbuf point from swring to rx_pkts + * C. calc the number of DD bits among the 4 packets + * [C*. extract the end-of-packet bit, if requested] + * D. fill info. from desc to mbuf + */ + + for (pos = 0, nb_pkts_recd = 0; pos < nb_pkts; + pos += RTE_I40E_DESCS_PER_LOOP, + rxdp += RTE_I40E_DESCS_PER_LOOP) { + vector unsigned long descs[RTE_I40E_DESCS_PER_LOOP]; + vector unsigned char pkt_mb1, pkt_mb2, pkt_mb3, pkt_mb4; + vector unsigned short staterr, sterr_tmp1, sterr_tmp2; + vector unsigned long mbp1, mbp2; /* two mbuf pointer + * in one XMM reg. + */ + + /* B.1 load 1 mbuf point */ + mbp1 = *(vector unsigned long *)&sw_ring[pos]; + /* Read desc statuses backwards to avoid race condition */ + /* A.1 load 4 pkts desc */ + descs[3] = *(vector unsigned long *)(rxdp + 3); + rte_compiler_barrier(); + + /* B.2 copy 2 mbuf point into rx_pkts */ + *(vector unsigned long *)&rx_pkts[pos] = mbp1; + + /* B.1 load 1 mbuf point */ + mbp2 = *(vector unsigned long *)&sw_ring[pos + 2]; + + descs[2] = *(vector unsigned long *)(rxdp + 2); + rte_compiler_barrier(); + /* B.1 load 2 mbuf point */ + descs[1] = *(vector unsigned long *)(rxdp + 1); + rte_compiler_barrier(); + descs[0] = *(vector unsigned long *)(rxdp); + + /* B.2 copy 2 mbuf point into rx_pkts */ + *(vector unsigned long *)&rx_pkts[pos + 2] = mbp2; + + if (split_packet) { + rte_mbuf_prefetch_part2(rx_pkts[pos]); + rte_mbuf_prefetch_part2(rx_pkts[pos + 1]); + rte_mbuf_prefetch_part2(rx_pkts[pos + 2]); + rte_mbuf_prefetch_part2(rx_pkts[pos + 3]); + } + + /* avoid compiler reorder optimization */ + rte_compiler_barrier(); + + /* pkt 3,4 shift the pktlen field to be 16-bit aligned*/ + const vector unsigned int len3 = vec_sl( + vec_ld(0, (vector unsigned int *)&descs[3]), + (vector unsigned int){0, 0, 0, PKTLEN_SHIFT}); + + const vector unsigned int len2 = vec_sl( + vec_ld(0, (vector unsigned int *)&descs[2]), + (vector unsigned int){0, 0, 0, PKTLEN_SHIFT}); + + /* merge the now-aligned packet length fields back in */ + descs[3] = (vector unsigned long)len3; + descs[2] = (vector unsigned long)len2; + + /* D.1 pkt 3,4 convert format from desc to pktmbuf */ + pkt_mb4 = vec_perm((vector unsigned char)descs[3], + (vector unsigned char){}, shuf_msk); + pkt_mb3 = vec_perm((vector unsigned char)descs[2], + (vector unsigned char){}, shuf_msk); + + /* C.1 4=>2 filter staterr info only */ + sterr_tmp2 = vec_mergel((vector unsigned short)descs[3], + (vector unsigned short)descs[2]); + /* C.1 4=>2 filter staterr info only */ + sterr_tmp1 = vec_mergel((vector unsigned short)descs[1], + (vector unsigned short)descs[0]); + /* D.2 pkt 3,4 set in_port/nb_seg and remove crc */ + pkt_mb4 = (vector unsigned char)vec_sub( + (vector unsigned short)pkt_mb4, crc_adjust); + pkt_mb3 = (vector unsigned char)vec_sub( + (vector unsigned short)pkt_mb3, crc_adjust); + + /* pkt 1,2 shift the pktlen field to be 16-bit aligned*/ + const vector unsigned int len1 = vec_sl( + vec_ld(0, (vector unsigned int *)&descs[1]), + (vector unsigned int){0, 0, 0, PKTLEN_SHIFT}); + const vector unsigned int len0 = vec_sl( + vec_ld(0, (vector unsigned int *)&descs[0]), + (vector unsigned int){0, 0, 0, PKTLEN_SHIFT}); + + /* merge the now-aligned packet length fields back in */ + descs[1] = (vector unsigned long)len1; + descs[0] = (vector unsigned long)len0; + + /* D.1 pkt 1,2 convert format from desc to pktmbuf */ + pkt_mb2 = vec_perm((vector unsigned char)descs[1], + (vector unsigned char){}, shuf_msk); + pkt_mb1 = vec_perm((vector unsigned char)descs[0], + (vector unsigned char){}, shuf_msk); + + /* C.2 get 4 pkts staterr value */ + staterr = (vector unsigned short)vec_mergeh( + sterr_tmp1, sterr_tmp2); + + /* D.3 copy final 3,4 data to rx_pkts */ + vec_st(pkt_mb4, 0, + (vector unsigned char *)&rx_pkts[pos + 3] + ->rx_descriptor_fields1 + ); + vec_st(pkt_mb3, 0, + (vector unsigned char *)&rx_pkts[pos + 2] + ->rx_descriptor_fields1 + ); + + /* D.2 pkt 1,2 set in_port/nb_seg and remove crc */ + pkt_mb2 = (vector unsigned char)vec_sub( + (vector unsigned short)pkt_mb2, crc_adjust); + pkt_mb1 = (vector unsigned char)vec_sub( + (vector unsigned short)pkt_mb1, crc_adjust); + + /* C* extract and record EOP bit */ + if (split_packet) { + vector unsigned char eop_shuf_mask = + (vector unsigned char){ + 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, + 0x04, 0x0C, 0x00, 0x08 + }; + + /* and with mask to extract bits, flipping 1-0 */ + vector unsigned char eop_bits = vec_and( + (vector unsigned char)vec_nor(staterr, staterr), + (vector unsigned char)eop_check); + /* the staterr values are not in order, as the count + * count of dd bits doesn't care. However, for end of + * packet tracking, we do care, so shuffle. This also + * compresses the 32-bit values to 8-bit + */ + eop_bits = vec_perm(eop_bits, (vector unsigned char){}, + eop_shuf_mask); + /* store the resulting 32-bit value */ + *split_packet = (vec_ld(0, + (vector unsigned int *)&eop_bits))[0]; + split_packet += RTE_I40E_DESCS_PER_LOOP; + + /* zero-out next pointers */ + rx_pkts[pos]->next = NULL; + rx_pkts[pos + 1]->next = NULL; + rx_pkts[pos + 2]->next = NULL; + rx_pkts[pos + 3]->next = NULL; + } + + /* C.3 calc available number of desc */ + staterr = vec_and(staterr, (vector unsigned short)dd_check); + + /* D.3 copy final 1,2 data to rx_pkts */ + vec_st(pkt_mb2, 0, + (vector unsigned char *)&rx_pkts[pos + 1] + ->rx_descriptor_fields1 + ); + vec_st(pkt_mb1, 0, + (vector unsigned char *)&rx_pkts[pos]->rx_descriptor_fields1 + ); + desc_to_ptype_v(descs, &rx_pkts[pos], ptype_tbl); + desc_to_olflags_v(descs, &rx_pkts[pos]); + + /* C.4 calc avaialbe number of desc */ + var = __builtin_popcountll((vec_ld(0, + (vector unsigned long *)&staterr)[0])); + nb_pkts_recd += var; + if (likely(var != RTE_I40E_DESCS_PER_LOOP)) + break; + } + + /* Update our internal tail pointer */ + rxq->rx_tail = (uint16_t)(rxq->rx_tail + nb_pkts_recd); + rxq->rx_tail = (uint16_t)(rxq->rx_tail & (rxq->nb_rx_desc - 1)); + rxq->rxrearm_nb = (uint16_t)(rxq->rxrearm_nb + nb_pkts_recd); + + return nb_pkts_recd; +} + + /* Notice: + * - nb_pkts < RTE_I40E_DESCS_PER_LOOP, just return no packet + * - nb_pkts > RTE_I40E_VPMD_RX_BURST, only scan RTE_I40E_VPMD_RX_BURST + * numbers of DD bits + */ +uint16_t +i40e_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts) +{ + return _recv_raw_pkts_vec(rx_queue, rx_pkts, nb_pkts, NULL); +} + + /* vPMD receive routine that reassembles scattered packets + * Notice: + * - nb_pkts < RTE_I40E_DESCS_PER_LOOP, just return no packet + * - nb_pkts > RTE_I40E_VPMD_RX_BURST, only scan RTE_I40E_VPMD_RX_BURST + * numbers of DD bits + */ +uint16_t +i40e_recv_scattered_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts) +{ + struct i40e_rx_queue *rxq = rx_queue; + uint8_t split_flags[RTE_I40E_VPMD_RX_BURST] = {0}; + + /* get some new buffers */ + uint16_t nb_bufs = _recv_raw_pkts_vec(rxq, rx_pkts, nb_pkts, + split_flags); + if (nb_bufs == 0) + return 0; + + /* happy day case, full burst + no packets to be joined */ + const uint64_t *split_fl64 = (uint64_t *)split_flags; + + if (rxq->pkt_first_seg == NULL && + split_fl64[0] == 0 && split_fl64[1] == 0 && + split_fl64[2] == 0 && split_fl64[3] == 0) + return nb_bufs; + + /* reassemble any packets that need reassembly*/ + unsigned int i = 0; + + if (!rxq->pkt_first_seg) { + /* find the first split flag, and only reassemble then*/ + while (i < nb_bufs && !split_flags[i]) + i++; + if (i == nb_bufs) + return nb_bufs; + } + return i + reassemble_packets(rxq, &rx_pkts[i], nb_bufs - i, + &split_flags[i]); +} + +static inline void +vtx1(volatile struct i40e_tx_desc *txdp, + struct rte_mbuf *pkt, uint64_t flags) +{ + uint64_t high_qw = (I40E_TX_DESC_DTYPE_DATA | + ((uint64_t)flags << I40E_TXD_QW1_CMD_SHIFT) | + ((uint64_t)pkt->data_len << I40E_TXD_QW1_TX_BUF_SZ_SHIFT)); + + vector unsigned long descriptor = (vector unsigned long){ + pkt->buf_iova + pkt->data_off, high_qw}; + *(vector unsigned long *)txdp = descriptor; +} + +static inline void +vtx(volatile struct i40e_tx_desc *txdp, + struct rte_mbuf **pkt, uint16_t nb_pkts, uint64_t flags) +{ + int i; + + for (i = 0; i < nb_pkts; ++i, ++txdp, ++pkt) + vtx1(txdp, *pkt, flags); +} + +uint16_t +i40e_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts) +{ + struct i40e_tx_queue *txq = (struct i40e_tx_queue *)tx_queue; + volatile struct i40e_tx_desc *txdp; + struct i40e_tx_entry *txep; + uint16_t n, nb_commit, tx_id; + uint64_t flags = I40E_TD_CMD; + uint64_t rs = I40E_TX_DESC_CMD_RS | I40E_TD_CMD; + int i; + + /* cross rx_thresh boundary is not allowed */ + nb_pkts = RTE_MIN(nb_pkts, txq->tx_rs_thresh); + + if (txq->nb_tx_free < txq->tx_free_thresh) + i40e_tx_free_bufs(txq); + + nb_pkts = (uint16_t)RTE_MIN(txq->nb_tx_free, nb_pkts); + nb_commit = nb_pkts; + if (unlikely(nb_pkts == 0)) + return 0; + + tx_id = txq->tx_tail; + txdp = &txq->tx_ring[tx_id]; + txep = &txq->sw_ring[tx_id]; + + txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_pkts); + + n = (uint16_t)(txq->nb_tx_desc - tx_id); + if (nb_commit >= n) { + tx_backlog_entry(txep, tx_pkts, n); + + for (i = 0; i < n - 1; ++i, ++tx_pkts, ++txdp) + vtx1(txdp, *tx_pkts, flags); + + vtx1(txdp, *tx_pkts++, rs); + + nb_commit = (uint16_t)(nb_commit - n); + + tx_id = 0; + txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1); + + /* avoid reach the end of ring */ + txdp = &txq->tx_ring[tx_id]; + txep = &txq->sw_ring[tx_id]; + } + + tx_backlog_entry(txep, tx_pkts, nb_commit); + + vtx(txdp, tx_pkts, nb_commit, flags); + + tx_id = (uint16_t)(tx_id + nb_commit); + if (tx_id > txq->tx_next_rs) { + txq->tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |= + rte_cpu_to_le_64(((uint64_t)I40E_TX_DESC_CMD_RS) << + I40E_TXD_QW1_CMD_SHIFT); + txq->tx_next_rs = + (uint16_t)(txq->tx_next_rs + txq->tx_rs_thresh); + } + + txq->tx_tail = tx_id; + + I40E_PCI_REG_WRITE(txq->qtx_tail, txq->tx_tail); + + return nb_pkts; +} + +void __attribute__((cold)) +i40e_rx_queue_release_mbufs_vec(struct i40e_rx_queue *rxq) +{ + _i40e_rx_queue_release_mbufs_vec(rxq); +} + +int __attribute__((cold)) +i40e_rxq_vec_setup(struct i40e_rx_queue *rxq) +{ + return i40e_rxq_vec_setup_default(rxq); +} + +int __attribute__((cold)) +i40e_txq_vec_setup(struct i40e_tx_queue __rte_unused * txq) +{ + return 0; +} + +int __attribute__((cold)) +i40e_rx_vec_dev_conf_condition_check(struct rte_eth_dev *dev) +{ + return i40e_rx_vec_dev_conf_condition_check_default(dev); +} diff --git a/dpdk/drivers/net/i40e/i40e_rxtx_vec_common.h b/dpdk/drivers/net/i40e/i40e_rxtx_vec_common.h new file mode 100644 index 00000000..39a6da06 --- /dev/null +++ b/dpdk/drivers/net/i40e/i40e_rxtx_vec_common.h @@ -0,0 +1,247 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _I40E_RXTX_VEC_COMMON_H_ +#define _I40E_RXTX_VEC_COMMON_H_ +#include +#include +#include + +#include "i40e_ethdev.h" +#include "i40e_rxtx.h" + +static inline uint16_t +reassemble_packets(struct i40e_rx_queue *rxq, struct rte_mbuf **rx_bufs, + uint16_t nb_bufs, uint8_t *split_flags) +{ + struct rte_mbuf *pkts[RTE_I40E_VPMD_RX_BURST]; /*finished pkts*/ + struct rte_mbuf *start = rxq->pkt_first_seg; + struct rte_mbuf *end = rxq->pkt_last_seg; + unsigned pkt_idx, buf_idx; + + for (buf_idx = 0, pkt_idx = 0; buf_idx < nb_bufs; buf_idx++) { + if (end != NULL) { + /* processing a split packet */ + end->next = rx_bufs[buf_idx]; + rx_bufs[buf_idx]->data_len += rxq->crc_len; + + start->nb_segs++; + start->pkt_len += rx_bufs[buf_idx]->data_len; + end = end->next; + + if (!split_flags[buf_idx]) { + /* it's the last packet of the set */ + start->hash = end->hash; + start->ol_flags = end->ol_flags; + /* we need to strip crc for the whole packet */ + start->pkt_len -= rxq->crc_len; + if (end->data_len > rxq->crc_len) + end->data_len -= rxq->crc_len; + else { + /* free up last mbuf */ + struct rte_mbuf *secondlast = start; + + start->nb_segs--; + while (secondlast->next != end) + secondlast = secondlast->next; + secondlast->data_len -= (rxq->crc_len - + end->data_len); + secondlast->next = NULL; + rte_pktmbuf_free_seg(end); + } + pkts[pkt_idx++] = start; + start = end = NULL; + } + } else { + /* not processing a split packet */ + if (!split_flags[buf_idx]) { + /* not a split packet, save and skip */ + pkts[pkt_idx++] = rx_bufs[buf_idx]; + continue; + } + end = start = rx_bufs[buf_idx]; + rx_bufs[buf_idx]->data_len += rxq->crc_len; + rx_bufs[buf_idx]->pkt_len += rxq->crc_len; + } + } + + /* save the partial packet for next time */ + rxq->pkt_first_seg = start; + rxq->pkt_last_seg = end; + memcpy(rx_bufs, pkts, pkt_idx * (sizeof(*pkts))); + return pkt_idx; +} + +static __rte_always_inline int +i40e_tx_free_bufs(struct i40e_tx_queue *txq) +{ + struct i40e_tx_entry *txep; + uint32_t n; + uint32_t i; + int nb_free = 0; + struct rte_mbuf *m, *free[RTE_I40E_TX_MAX_FREE_BUF_SZ]; + + /* check DD bits on threshold descriptor */ + if ((txq->tx_ring[txq->tx_next_dd].cmd_type_offset_bsz & + rte_cpu_to_le_64(I40E_TXD_QW1_DTYPE_MASK)) != + rte_cpu_to_le_64(I40E_TX_DESC_DTYPE_DESC_DONE)) + return 0; + + n = txq->tx_rs_thresh; + + /* first buffer to free from S/W ring is at index + * tx_next_dd - (tx_rs_thresh-1) + */ + txep = &txq->sw_ring[txq->tx_next_dd - (n - 1)]; + m = rte_pktmbuf_prefree_seg(txep[0].mbuf); + if (likely(m != NULL)) { + free[0] = m; + nb_free = 1; + for (i = 1; i < n; i++) { + m = rte_pktmbuf_prefree_seg(txep[i].mbuf); + if (likely(m != NULL)) { + if (likely(m->pool == free[0]->pool)) { + free[nb_free++] = m; + } else { + rte_mempool_put_bulk(free[0]->pool, + (void *)free, + nb_free); + free[0] = m; + nb_free = 1; + } + } + } + rte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free); + } else { + for (i = 1; i < n; i++) { + m = rte_pktmbuf_prefree_seg(txep[i].mbuf); + if (m != NULL) + rte_mempool_put(m->pool, m); + } + } + + /* buffers were freed, update counters */ + txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_rs_thresh); + txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_rs_thresh); + if (txq->tx_next_dd >= txq->nb_tx_desc) + txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1); + + return txq->tx_rs_thresh; +} + +static __rte_always_inline void +tx_backlog_entry(struct i40e_tx_entry *txep, + struct rte_mbuf **tx_pkts, uint16_t nb_pkts) +{ + int i; + + for (i = 0; i < (int)nb_pkts; ++i) + txep[i].mbuf = tx_pkts[i]; +} + +static inline void +_i40e_rx_queue_release_mbufs_vec(struct i40e_rx_queue *rxq) +{ + const unsigned mask = rxq->nb_rx_desc - 1; + unsigned i; + + if (rxq->sw_ring == NULL || rxq->rxrearm_nb >= rxq->nb_rx_desc) + return; + + /* free all mbufs that are valid in the ring */ + if (rxq->rxrearm_nb == 0) { + for (i = 0; i < rxq->nb_rx_desc; i++) { + if (rxq->sw_ring[i].mbuf != NULL) + rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf); + } + } else { + for (i = rxq->rx_tail; + i != rxq->rxrearm_start; + i = (i + 1) & mask) { + if (rxq->sw_ring[i].mbuf != NULL) + rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf); + } + } + + rxq->rxrearm_nb = rxq->nb_rx_desc; + + /* set all entries to NULL */ + memset(rxq->sw_ring, 0, sizeof(rxq->sw_ring[0]) * rxq->nb_rx_desc); +} + +static inline int +i40e_rxq_vec_setup_default(struct i40e_rx_queue *rxq) +{ + uintptr_t p; + struct rte_mbuf mb_def = { .buf_addr = 0 }; /* zeroed mbuf */ + + mb_def.nb_segs = 1; + mb_def.data_off = RTE_PKTMBUF_HEADROOM; + mb_def.port = rxq->port_id; + rte_mbuf_refcnt_set(&mb_def, 1); + + /* prevent compiler reordering: rearm_data covers previous fields */ + rte_compiler_barrier(); + p = (uintptr_t)&mb_def.rearm_data; + rxq->mbuf_initializer = *(uint64_t *)p; + return 0; +} + +static inline int +i40e_rx_vec_dev_conf_condition_check_default(struct rte_eth_dev *dev) +{ +#ifndef RTE_LIBRTE_IEEE1588 + struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode; + struct rte_fdir_conf *fconf = &dev->data->dev_conf.fdir_conf; + + /* no fdir support */ + if (fconf->mode != RTE_FDIR_MODE_NONE) + return -1; + + /* - no csum error report support + * - no header split support + */ + if (rxmode->header_split == 1) + return -1; + + /* no QinQ support */ + if (rxmode->hw_vlan_extend == 1) + return -1; + + return 0; +#else + RTE_SET_USED(dev); + return -1; +#endif +} +#endif diff --git a/dpdk/drivers/net/i40e/i40e_rxtx_vec_neon.c b/dpdk/drivers/net/i40e/i40e_rxtx_vec_neon.c new file mode 100644 index 00000000..b5685e2b --- /dev/null +++ b/dpdk/drivers/net/i40e/i40e_rxtx_vec_neon.c @@ -0,0 +1,626 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * Copyright(c) 2016, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include + +#include "base/i40e_prototype.h" +#include "base/i40e_type.h" +#include "i40e_ethdev.h" +#include "i40e_rxtx.h" +#include "i40e_rxtx_vec_common.h" + +#include + +#pragma GCC diagnostic ignored "-Wcast-qual" + +static inline void +i40e_rxq_rearm(struct i40e_rx_queue *rxq) +{ + int i; + uint16_t rx_id; + volatile union i40e_rx_desc *rxdp; + struct i40e_rx_entry *rxep = &rxq->sw_ring[rxq->rxrearm_start]; + struct rte_mbuf *mb0, *mb1; + uint64x2_t dma_addr0, dma_addr1; + uint64x2_t zero = vdupq_n_u64(0); + uint64_t paddr; + + rxdp = rxq->rx_ring + rxq->rxrearm_start; + + /* Pull 'n' more MBUFs into the software ring */ + if (unlikely(rte_mempool_get_bulk(rxq->mp, + (void *)rxep, + RTE_I40E_RXQ_REARM_THRESH) < 0)) { + if (rxq->rxrearm_nb + RTE_I40E_RXQ_REARM_THRESH >= + rxq->nb_rx_desc) { + for (i = 0; i < RTE_I40E_DESCS_PER_LOOP; i++) { + rxep[i].mbuf = &rxq->fake_mbuf; + vst1q_u64((uint64_t *)&rxdp[i].read, zero); + } + } + rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed += + RTE_I40E_RXQ_REARM_THRESH; + return; + } + + /* Initialize the mbufs in vector, process 2 mbufs in one loop */ + for (i = 0; i < RTE_I40E_RXQ_REARM_THRESH; i += 2, rxep += 2) { + mb0 = rxep[0].mbuf; + mb1 = rxep[1].mbuf; + + paddr = mb0->buf_iova + RTE_PKTMBUF_HEADROOM; + dma_addr0 = vdupq_n_u64(paddr); + + /* flush desc with pa dma_addr */ + vst1q_u64((uint64_t *)&rxdp++->read, dma_addr0); + + paddr = mb1->buf_iova + RTE_PKTMBUF_HEADROOM; + dma_addr1 = vdupq_n_u64(paddr); + vst1q_u64((uint64_t *)&rxdp++->read, dma_addr1); + } + + rxq->rxrearm_start += RTE_I40E_RXQ_REARM_THRESH; + if (rxq->rxrearm_start >= rxq->nb_rx_desc) + rxq->rxrearm_start = 0; + + rxq->rxrearm_nb -= RTE_I40E_RXQ_REARM_THRESH; + + rx_id = (uint16_t)((rxq->rxrearm_start == 0) ? + (rxq->nb_rx_desc - 1) : (rxq->rxrearm_start - 1)); + + /* Update the tail pointer on the NIC */ + I40E_PCI_REG_WRITE(rxq->qrx_tail, rx_id); +} + +static inline void +desc_to_olflags_v(struct i40e_rx_queue *rxq, uint64x2_t descs[4], + struct rte_mbuf **rx_pkts) +{ + uint32x4_t vlan0, vlan1, rss, l3_l4e; + const uint64x2_t mbuf_init = {rxq->mbuf_initializer, 0}; + uint64x2_t rearm0, rearm1, rearm2, rearm3; + + /* mask everything except RSS, flow director and VLAN flags + * bit2 is for VLAN tag, bit11 for flow director indication + * bit13:12 for RSS indication. + */ + const uint32x4_t rss_vlan_msk = { + 0x1c03804, 0x1c03804, 0x1c03804, 0x1c03804}; + + const uint32x4_t cksum_mask = { + PKT_RX_IP_CKSUM_GOOD | PKT_RX_IP_CKSUM_BAD | + PKT_RX_L4_CKSUM_GOOD | PKT_RX_L4_CKSUM_BAD | + PKT_RX_EIP_CKSUM_BAD, + PKT_RX_IP_CKSUM_GOOD | PKT_RX_IP_CKSUM_BAD | + PKT_RX_L4_CKSUM_GOOD | PKT_RX_L4_CKSUM_BAD | + PKT_RX_EIP_CKSUM_BAD, + PKT_RX_IP_CKSUM_GOOD | PKT_RX_IP_CKSUM_BAD | + PKT_RX_L4_CKSUM_GOOD | PKT_RX_L4_CKSUM_BAD | + PKT_RX_EIP_CKSUM_BAD, + PKT_RX_IP_CKSUM_GOOD | PKT_RX_IP_CKSUM_BAD | + PKT_RX_L4_CKSUM_GOOD | PKT_RX_L4_CKSUM_BAD | + PKT_RX_EIP_CKSUM_BAD}; + + /* map rss and vlan type to rss hash and vlan flag */ + const uint8x16_t vlan_flags = { + 0, 0, 0, 0, + PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0}; + + const uint8x16_t rss_flags = { + 0, PKT_RX_FDIR, 0, 0, + 0, 0, PKT_RX_RSS_HASH, PKT_RX_RSS_HASH | PKT_RX_FDIR, + 0, 0, 0, 0, + 0, 0, 0, 0}; + + const uint8x16_t l3_l4e_flags = { + (PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD) >> 1, + PKT_RX_IP_CKSUM_BAD >> 1, + (PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_BAD) >> 1, + (PKT_RX_L4_CKSUM_BAD | PKT_RX_IP_CKSUM_BAD) >> 1, + (PKT_RX_IP_CKSUM_GOOD | PKT_RX_EIP_CKSUM_BAD) >> 1, + (PKT_RX_EIP_CKSUM_BAD | PKT_RX_IP_CKSUM_BAD) >> 1, + (PKT_RX_IP_CKSUM_GOOD | PKT_RX_EIP_CKSUM_BAD | + PKT_RX_L4_CKSUM_BAD) >> 1, + (PKT_RX_EIP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD | + PKT_RX_IP_CKSUM_BAD) >> 1, + 0, 0, 0, 0, 0, 0, 0, 0}; + + vlan0 = vzipq_u32(vreinterpretq_u32_u64(descs[0]), + vreinterpretq_u32_u64(descs[2])).val[1]; + vlan1 = vzipq_u32(vreinterpretq_u32_u64(descs[1]), + vreinterpretq_u32_u64(descs[3])).val[1]; + vlan0 = vzipq_u32(vlan0, vlan1).val[0]; + + vlan1 = vandq_u32(vlan0, rss_vlan_msk); + vlan0 = vreinterpretq_u32_u8(vqtbl1q_u8(vlan_flags, + vreinterpretq_u8_u32(vlan1))); + + rss = vshrq_n_u32(vlan1, 11); + rss = vreinterpretq_u32_u8(vqtbl1q_u8(rss_flags, + vreinterpretq_u8_u32(rss))); + + l3_l4e = vshrq_n_u32(vlan1, 22); + l3_l4e = vreinterpretq_u32_u8(vqtbl1q_u8(l3_l4e_flags, + vreinterpretq_u8_u32(l3_l4e))); + /* then we shift left 1 bit */ + l3_l4e = vshlq_n_u32(l3_l4e, 1); + /* we need to mask out the reduntant bits */ + l3_l4e = vandq_u32(l3_l4e, cksum_mask); + + vlan0 = vorrq_u32(vlan0, rss); + vlan0 = vorrq_u32(vlan0, l3_l4e); + + rearm0 = vsetq_lane_u64(vgetq_lane_u32(vlan0, 0), mbuf_init, 1); + rearm1 = vsetq_lane_u64(vgetq_lane_u32(vlan0, 1), mbuf_init, 1); + rearm2 = vsetq_lane_u64(vgetq_lane_u32(vlan0, 2), mbuf_init, 1); + rearm3 = vsetq_lane_u64(vgetq_lane_u32(vlan0, 3), mbuf_init, 1); + + vst1q_u64((uint64_t *)&rx_pkts[0]->rearm_data, rearm0); + vst1q_u64((uint64_t *)&rx_pkts[1]->rearm_data, rearm1); + vst1q_u64((uint64_t *)&rx_pkts[2]->rearm_data, rearm2); + vst1q_u64((uint64_t *)&rx_pkts[3]->rearm_data, rearm3); +} + +#define PKTLEN_SHIFT 10 +#define I40E_UINT16_BIT (CHAR_BIT * sizeof(uint16_t)) + +static inline void +desc_to_ptype_v(uint64x2_t descs[4], struct rte_mbuf **rx_pkts, + uint32_t *ptype_tbl) +{ + int i; + uint8_t ptype; + uint8x16_t tmp; + + for (i = 0; i < 4; i++) { + tmp = vreinterpretq_u8_u64(vshrq_n_u64(descs[i], 30)); + ptype = vgetq_lane_u8(tmp, 8); + rx_pkts[i]->packet_type = ptype_tbl[ptype]; + } + +} + + /* + * Notice: + * - nb_pkts < RTE_I40E_DESCS_PER_LOOP, just return no packet + * - nb_pkts > RTE_I40E_VPMD_RX_BURST, only scan RTE_I40E_VPMD_RX_BURST + * numbers of DD bits + */ +static inline uint16_t +_recv_raw_pkts_vec(struct i40e_rx_queue *rxq, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts, uint8_t *split_packet) +{ + volatile union i40e_rx_desc *rxdp; + struct i40e_rx_entry *sw_ring; + uint16_t nb_pkts_recd; + int pos; + uint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl; + + /* mask to shuffle from desc. to mbuf */ + uint8x16_t shuf_msk = { + 0xFF, 0xFF, /* pkt_type set as unknown */ + 0xFF, 0xFF, /* pkt_type set as unknown */ + 14, 15, /* octet 15~14, low 16 bits pkt_len */ + 0xFF, 0xFF, /* skip high 16 bits pkt_len, zero out */ + 14, 15, /* octet 15~14, 16 bits data_len */ + 2, 3, /* octet 2~3, low 16 bits vlan_macip */ + 4, 5, 6, 7 /* octet 4~7, 32bits rss */ + }; + + uint8x16_t eop_check = { + 0x02, 0x00, 0x02, 0x00, + 0x02, 0x00, 0x02, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00 + }; + + uint16x8_t crc_adjust = { + 0, 0, /* ignore pkt_type field */ + rxq->crc_len, /* sub crc on pkt_len */ + 0, /* ignore high-16bits of pkt_len */ + rxq->crc_len, /* sub crc on data_len */ + 0, 0, 0 /* ignore non-length fields */ + }; + + /* nb_pkts shall be less equal than RTE_I40E_MAX_RX_BURST */ + nb_pkts = RTE_MIN(nb_pkts, RTE_I40E_MAX_RX_BURST); + + /* nb_pkts has to be floor-aligned to RTE_I40E_DESCS_PER_LOOP */ + nb_pkts = RTE_ALIGN_FLOOR(nb_pkts, RTE_I40E_DESCS_PER_LOOP); + + /* Just the act of getting into the function from the application is + * going to cost about 7 cycles + */ + rxdp = rxq->rx_ring + rxq->rx_tail; + + rte_prefetch_non_temporal(rxdp); + + /* See if we need to rearm the RX queue - gives the prefetch a bit + * of time to act + */ + if (rxq->rxrearm_nb > RTE_I40E_RXQ_REARM_THRESH) + i40e_rxq_rearm(rxq); + + /* Before we start moving massive data around, check to see if + * there is actually a packet available + */ + if (!(rxdp->wb.qword1.status_error_len & + rte_cpu_to_le_32(1 << I40E_RX_DESC_STATUS_DD_SHIFT))) + return 0; + + /* Cache is empty -> need to scan the buffer rings, but first move + * the next 'n' mbufs into the cache + */ + sw_ring = &rxq->sw_ring[rxq->rx_tail]; + + /* A. load 4 packet in one loop + * [A*. mask out 4 unused dirty field in desc] + * B. copy 4 mbuf point from swring to rx_pkts + * C. calc the number of DD bits among the 4 packets + * [C*. extract the end-of-packet bit, if requested] + * D. fill info. from desc to mbuf + */ + + for (pos = 0, nb_pkts_recd = 0; pos < nb_pkts; + pos += RTE_I40E_DESCS_PER_LOOP, + rxdp += RTE_I40E_DESCS_PER_LOOP) { + uint64x2_t descs[RTE_I40E_DESCS_PER_LOOP]; + uint8x16_t pkt_mb1, pkt_mb2, pkt_mb3, pkt_mb4; + uint16x8x2_t sterr_tmp1, sterr_tmp2; + uint64x2_t mbp1, mbp2; + uint16x8_t staterr; + uint16x8_t tmp; + uint64_t stat; + + int32x4_t len_shl = {0, 0, 0, PKTLEN_SHIFT}; + + /* B.1 load 1 mbuf point */ + mbp1 = vld1q_u64((uint64_t *)&sw_ring[pos]); + /* Read desc statuses backwards to avoid race condition */ + /* A.1 load 4 pkts desc */ + descs[3] = vld1q_u64((uint64_t *)(rxdp + 3)); + rte_rmb(); + + /* B.2 copy 2 mbuf point into rx_pkts */ + vst1q_u64((uint64_t *)&rx_pkts[pos], mbp1); + + /* B.1 load 1 mbuf point */ + mbp2 = vld1q_u64((uint64_t *)&sw_ring[pos + 2]); + + descs[2] = vld1q_u64((uint64_t *)(rxdp + 2)); + /* B.1 load 2 mbuf point */ + descs[1] = vld1q_u64((uint64_t *)(rxdp + 1)); + descs[0] = vld1q_u64((uint64_t *)(rxdp)); + + /* B.2 copy 2 mbuf point into rx_pkts */ + vst1q_u64((uint64_t *)&rx_pkts[pos + 2], mbp2); + + if (split_packet) { + rte_mbuf_prefetch_part2(rx_pkts[pos]); + rte_mbuf_prefetch_part2(rx_pkts[pos + 1]); + rte_mbuf_prefetch_part2(rx_pkts[pos + 2]); + rte_mbuf_prefetch_part2(rx_pkts[pos + 3]); + } + + /* avoid compiler reorder optimization */ + rte_compiler_barrier(); + + /* pkt 3,4 shift the pktlen field to be 16-bit aligned*/ + uint32x4_t len3 = vshlq_u32(vreinterpretq_u32_u64(descs[3]), + len_shl); + descs[3] = vreinterpretq_u64_u32(len3); + uint32x4_t len2 = vshlq_u32(vreinterpretq_u32_u64(descs[2]), + len_shl); + descs[2] = vreinterpretq_u64_u32(len2); + + /* D.1 pkt 3,4 convert format from desc to pktmbuf */ + pkt_mb4 = vqtbl1q_u8(vreinterpretq_u8_u64(descs[3]), shuf_msk); + pkt_mb3 = vqtbl1q_u8(vreinterpretq_u8_u64(descs[2]), shuf_msk); + + /* C.1 4=>2 filter staterr info only */ + sterr_tmp2 = vzipq_u16(vreinterpretq_u16_u64(descs[1]), + vreinterpretq_u16_u64(descs[3])); + /* C.1 4=>2 filter staterr info only */ + sterr_tmp1 = vzipq_u16(vreinterpretq_u16_u64(descs[0]), + vreinterpretq_u16_u64(descs[2])); + + /* C.2 get 4 pkts staterr value */ + staterr = vzipq_u16(sterr_tmp1.val[1], + sterr_tmp2.val[1]).val[0]; + + desc_to_olflags_v(rxq, descs, &rx_pkts[pos]); + + /* D.2 pkt 3,4 set in_port/nb_seg and remove crc */ + tmp = vsubq_u16(vreinterpretq_u16_u8(pkt_mb4), crc_adjust); + pkt_mb4 = vreinterpretq_u8_u16(tmp); + tmp = vsubq_u16(vreinterpretq_u16_u8(pkt_mb3), crc_adjust); + pkt_mb3 = vreinterpretq_u8_u16(tmp); + + /* pkt 1,2 shift the pktlen field to be 16-bit aligned*/ + uint32x4_t len1 = vshlq_u32(vreinterpretq_u32_u64(descs[1]), + len_shl); + descs[1] = vreinterpretq_u64_u32(len1); + uint32x4_t len0 = vshlq_u32(vreinterpretq_u32_u64(descs[0]), + len_shl); + descs[0] = vreinterpretq_u64_u32(len0); + + /* D.1 pkt 1,2 convert format from desc to pktmbuf */ + pkt_mb2 = vqtbl1q_u8(vreinterpretq_u8_u64(descs[1]), shuf_msk); + pkt_mb1 = vqtbl1q_u8(vreinterpretq_u8_u64(descs[0]), shuf_msk); + + /* D.3 copy final 3,4 data to rx_pkts */ + vst1q_u8((void *)&rx_pkts[pos + 3]->rx_descriptor_fields1, + pkt_mb4); + vst1q_u8((void *)&rx_pkts[pos + 2]->rx_descriptor_fields1, + pkt_mb3); + + /* D.2 pkt 1,2 set in_port/nb_seg and remove crc */ + tmp = vsubq_u16(vreinterpretq_u16_u8(pkt_mb2), crc_adjust); + pkt_mb2 = vreinterpretq_u8_u16(tmp); + tmp = vsubq_u16(vreinterpretq_u16_u8(pkt_mb1), crc_adjust); + pkt_mb1 = vreinterpretq_u8_u16(tmp); + + /* C* extract and record EOP bit */ + if (split_packet) { + uint8x16_t eop_shuf_mask = { + 0x00, 0x02, 0x04, 0x06, + 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF}; + uint8x16_t eop_bits; + + /* and with mask to extract bits, flipping 1-0 */ + eop_bits = vmvnq_u8(vreinterpretq_u8_u16(staterr)); + eop_bits = vandq_u8(eop_bits, eop_check); + /* the staterr values are not in order, as the count + * count of dd bits doesn't care. However, for end of + * packet tracking, we do care, so shuffle. This also + * compresses the 32-bit values to 8-bit + */ + eop_bits = vqtbl1q_u8(eop_bits, eop_shuf_mask); + + /* store the resulting 32-bit value */ + vst1q_lane_u32((uint32_t *)split_packet, + vreinterpretq_u32_u8(eop_bits), 0); + split_packet += RTE_I40E_DESCS_PER_LOOP; + + /* zero-out next pointers */ + rx_pkts[pos]->next = NULL; + rx_pkts[pos + 1]->next = NULL; + rx_pkts[pos + 2]->next = NULL; + rx_pkts[pos + 3]->next = NULL; + } + + staterr = vshlq_n_u16(staterr, I40E_UINT16_BIT - 1); + staterr = vreinterpretq_u16_s16( + vshrq_n_s16(vreinterpretq_s16_u16(staterr), + I40E_UINT16_BIT - 1)); + stat = ~vgetq_lane_u64(vreinterpretq_u64_u16(staterr), 0); + + rte_prefetch_non_temporal(rxdp + RTE_I40E_DESCS_PER_LOOP); + + /* D.3 copy final 1,2 data to rx_pkts */ + vst1q_u8((void *)&rx_pkts[pos + 1]->rx_descriptor_fields1, + pkt_mb2); + vst1q_u8((void *)&rx_pkts[pos]->rx_descriptor_fields1, + pkt_mb1); + desc_to_ptype_v(descs, &rx_pkts[pos], ptype_tbl); + /* C.4 calc avaialbe number of desc */ + if (unlikely(stat == 0)) { + nb_pkts_recd += RTE_I40E_DESCS_PER_LOOP; + } else { + nb_pkts_recd += __builtin_ctzl(stat) / I40E_UINT16_BIT; + break; + } + } + + /* Update our internal tail pointer */ + rxq->rx_tail = (uint16_t)(rxq->rx_tail + nb_pkts_recd); + rxq->rx_tail = (uint16_t)(rxq->rx_tail & (rxq->nb_rx_desc - 1)); + rxq->rxrearm_nb = (uint16_t)(rxq->rxrearm_nb + nb_pkts_recd); + + return nb_pkts_recd; +} + + /* + * Notice: + * - nb_pkts < RTE_I40E_DESCS_PER_LOOP, just return no packet + * - nb_pkts > RTE_I40E_VPMD_RX_BURST, only scan RTE_I40E_VPMD_RX_BURST + * numbers of DD bits + */ +uint16_t +i40e_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts) +{ + return _recv_raw_pkts_vec(rx_queue, rx_pkts, nb_pkts, NULL); +} + + /* vPMD receive routine that reassembles scattered packets + * Notice: + * - nb_pkts < RTE_I40E_DESCS_PER_LOOP, just return no packet + * - nb_pkts > RTE_I40E_VPMD_RX_BURST, only scan RTE_I40E_VPMD_RX_BURST + * numbers of DD bits + */ +uint16_t +i40e_recv_scattered_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts) +{ + + struct i40e_rx_queue *rxq = rx_queue; + uint8_t split_flags[RTE_I40E_VPMD_RX_BURST] = {0}; + + /* get some new buffers */ + uint16_t nb_bufs = _recv_raw_pkts_vec(rxq, rx_pkts, nb_pkts, + split_flags); + if (nb_bufs == 0) + return 0; + + /* happy day case, full burst + no packets to be joined */ + const uint64_t *split_fl64 = (uint64_t *)split_flags; + + if (rxq->pkt_first_seg == NULL && + split_fl64[0] == 0 && split_fl64[1] == 0 && + split_fl64[2] == 0 && split_fl64[3] == 0) + return nb_bufs; + + /* reassemble any packets that need reassembly*/ + unsigned i = 0; + + if (rxq->pkt_first_seg == NULL) { + /* find the first split flag, and only reassemble then*/ + while (i < nb_bufs && !split_flags[i]) + i++; + if (i == nb_bufs) + return nb_bufs; + } + return i + reassemble_packets(rxq, &rx_pkts[i], nb_bufs - i, + &split_flags[i]); +} + +static inline void +vtx1(volatile struct i40e_tx_desc *txdp, + struct rte_mbuf *pkt, uint64_t flags) +{ + uint64_t high_qw = (I40E_TX_DESC_DTYPE_DATA | + ((uint64_t)flags << I40E_TXD_QW1_CMD_SHIFT) | + ((uint64_t)pkt->data_len << I40E_TXD_QW1_TX_BUF_SZ_SHIFT)); + + uint64x2_t descriptor = {pkt->buf_iova + pkt->data_off, high_qw}; + vst1q_u64((uint64_t *)txdp, descriptor); +} + +static inline void +vtx(volatile struct i40e_tx_desc *txdp, + struct rte_mbuf **pkt, uint16_t nb_pkts, uint64_t flags) +{ + int i; + + for (i = 0; i < nb_pkts; ++i, ++txdp, ++pkt) + vtx1(txdp, *pkt, flags); +} + +uint16_t +i40e_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts) +{ + struct i40e_tx_queue *txq = (struct i40e_tx_queue *)tx_queue; + volatile struct i40e_tx_desc *txdp; + struct i40e_tx_entry *txep; + uint16_t n, nb_commit, tx_id; + uint64_t flags = I40E_TD_CMD; + uint64_t rs = I40E_TX_DESC_CMD_RS | I40E_TD_CMD; + int i; + + /* cross rx_thresh boundary is not allowed */ + nb_pkts = RTE_MIN(nb_pkts, txq->tx_rs_thresh); + + if (txq->nb_tx_free < txq->tx_free_thresh) + i40e_tx_free_bufs(txq); + + nb_commit = nb_pkts = (uint16_t)RTE_MIN(txq->nb_tx_free, nb_pkts); + if (unlikely(nb_pkts == 0)) + return 0; + + tx_id = txq->tx_tail; + txdp = &txq->tx_ring[tx_id]; + txep = &txq->sw_ring[tx_id]; + + txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_pkts); + + n = (uint16_t)(txq->nb_tx_desc - tx_id); + if (nb_commit >= n) { + tx_backlog_entry(txep, tx_pkts, n); + + for (i = 0; i < n - 1; ++i, ++tx_pkts, ++txdp) + vtx1(txdp, *tx_pkts, flags); + + vtx1(txdp, *tx_pkts++, rs); + + nb_commit = (uint16_t)(nb_commit - n); + + tx_id = 0; + txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1); + + /* avoid reach the end of ring */ + txdp = &txq->tx_ring[tx_id]; + txep = &txq->sw_ring[tx_id]; + } + + tx_backlog_entry(txep, tx_pkts, nb_commit); + + vtx(txdp, tx_pkts, nb_commit, flags); + + tx_id = (uint16_t)(tx_id + nb_commit); + if (tx_id > txq->tx_next_rs) { + txq->tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |= + rte_cpu_to_le_64(((uint64_t)I40E_TX_DESC_CMD_RS) << + I40E_TXD_QW1_CMD_SHIFT); + txq->tx_next_rs = + (uint16_t)(txq->tx_next_rs + txq->tx_rs_thresh); + } + + txq->tx_tail = tx_id; + + I40E_PCI_REG_WRITE(txq->qtx_tail, txq->tx_tail); + + return nb_pkts; +} + +void __attribute__((cold)) +i40e_rx_queue_release_mbufs_vec(struct i40e_rx_queue *rxq) +{ + _i40e_rx_queue_release_mbufs_vec(rxq); +} + +int __attribute__((cold)) +i40e_rxq_vec_setup(struct i40e_rx_queue *rxq) +{ + return i40e_rxq_vec_setup_default(rxq); +} + +int __attribute__((cold)) +i40e_txq_vec_setup(struct i40e_tx_queue __rte_unused *txq) +{ + return 0; +} + +int __attribute__((cold)) +i40e_rx_vec_dev_conf_condition_check(struct rte_eth_dev *dev) +{ + return i40e_rx_vec_dev_conf_condition_check_default(dev); +} diff --git a/dpdk/drivers/net/i40e/i40e_rxtx_vec_sse.c b/dpdk/drivers/net/i40e/i40e_rxtx_vec_sse.c new file mode 100644 index 00000000..9d2d1f83 --- /dev/null +++ b/dpdk/drivers/net/i40e/i40e_rxtx_vec_sse.c @@ -0,0 +1,655 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include + +#include "base/i40e_prototype.h" +#include "base/i40e_type.h" +#include "i40e_ethdev.h" +#include "i40e_rxtx.h" +#include "i40e_rxtx_vec_common.h" + +#include + +#ifndef __INTEL_COMPILER +#pragma GCC diagnostic ignored "-Wcast-qual" +#endif + +static inline void +i40e_rxq_rearm(struct i40e_rx_queue *rxq) +{ + int i; + uint16_t rx_id; + volatile union i40e_rx_desc *rxdp; + struct i40e_rx_entry *rxep = &rxq->sw_ring[rxq->rxrearm_start]; + struct rte_mbuf *mb0, *mb1; + __m128i hdr_room = _mm_set_epi64x(RTE_PKTMBUF_HEADROOM, + RTE_PKTMBUF_HEADROOM); + __m128i dma_addr0, dma_addr1; + + rxdp = rxq->rx_ring + rxq->rxrearm_start; + + /* Pull 'n' more MBUFs into the software ring */ + if (rte_mempool_get_bulk(rxq->mp, + (void *)rxep, + RTE_I40E_RXQ_REARM_THRESH) < 0) { + if (rxq->rxrearm_nb + RTE_I40E_RXQ_REARM_THRESH >= + rxq->nb_rx_desc) { + dma_addr0 = _mm_setzero_si128(); + for (i = 0; i < RTE_I40E_DESCS_PER_LOOP; i++) { + rxep[i].mbuf = &rxq->fake_mbuf; + _mm_store_si128((__m128i *)&rxdp[i].read, + dma_addr0); + } + } + rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed += + RTE_I40E_RXQ_REARM_THRESH; + return; + } + + /* Initialize the mbufs in vector, process 2 mbufs in one loop */ + for (i = 0; i < RTE_I40E_RXQ_REARM_THRESH; i += 2, rxep += 2) { + __m128i vaddr0, vaddr1; + + mb0 = rxep[0].mbuf; + mb1 = rxep[1].mbuf; + + /* load buf_addr(lo 64bit) and buf_iova(hi 64bit) */ + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, buf_iova) != + offsetof(struct rte_mbuf, buf_addr) + 8); + vaddr0 = _mm_loadu_si128((__m128i *)&mb0->buf_addr); + vaddr1 = _mm_loadu_si128((__m128i *)&mb1->buf_addr); + + /* convert pa to dma_addr hdr/data */ + dma_addr0 = _mm_unpackhi_epi64(vaddr0, vaddr0); + dma_addr1 = _mm_unpackhi_epi64(vaddr1, vaddr1); + + /* add headroom to pa values */ + dma_addr0 = _mm_add_epi64(dma_addr0, hdr_room); + dma_addr1 = _mm_add_epi64(dma_addr1, hdr_room); + + /* flush desc with pa dma_addr */ + _mm_store_si128((__m128i *)&rxdp++->read, dma_addr0); + _mm_store_si128((__m128i *)&rxdp++->read, dma_addr1); + } + + rxq->rxrearm_start += RTE_I40E_RXQ_REARM_THRESH; + if (rxq->rxrearm_start >= rxq->nb_rx_desc) + rxq->rxrearm_start = 0; + + rxq->rxrearm_nb -= RTE_I40E_RXQ_REARM_THRESH; + + rx_id = (uint16_t)((rxq->rxrearm_start == 0) ? + (rxq->nb_rx_desc - 1) : (rxq->rxrearm_start - 1)); + + /* Update the tail pointer on the NIC */ + I40E_PCI_REG_WRITE(rxq->qrx_tail, rx_id); +} + +static inline void +desc_to_olflags_v(struct i40e_rx_queue *rxq, __m128i descs[4], + struct rte_mbuf **rx_pkts) +{ + const __m128i mbuf_init = _mm_set_epi64x(0, rxq->mbuf_initializer); + __m128i rearm0, rearm1, rearm2, rearm3; + + __m128i vlan0, vlan1, rss, l3_l4e; + + /* mask everything except RSS, flow director and VLAN flags + * bit2 is for VLAN tag, bit11 for flow director indication + * bit13:12 for RSS indication. + */ + const __m128i rss_vlan_msk = _mm_set_epi32( + 0x1c03804, 0x1c03804, 0x1c03804, 0x1c03804); + + const __m128i cksum_mask = _mm_set_epi32( + PKT_RX_IP_CKSUM_GOOD | PKT_RX_IP_CKSUM_BAD | + PKT_RX_L4_CKSUM_GOOD | PKT_RX_L4_CKSUM_BAD | + PKT_RX_EIP_CKSUM_BAD, + PKT_RX_IP_CKSUM_GOOD | PKT_RX_IP_CKSUM_BAD | + PKT_RX_L4_CKSUM_GOOD | PKT_RX_L4_CKSUM_BAD | + PKT_RX_EIP_CKSUM_BAD, + PKT_RX_IP_CKSUM_GOOD | PKT_RX_IP_CKSUM_BAD | + PKT_RX_L4_CKSUM_GOOD | PKT_RX_L4_CKSUM_BAD | + PKT_RX_EIP_CKSUM_BAD, + PKT_RX_IP_CKSUM_GOOD | PKT_RX_IP_CKSUM_BAD | + PKT_RX_L4_CKSUM_GOOD | PKT_RX_L4_CKSUM_BAD | + PKT_RX_EIP_CKSUM_BAD); + + /* map rss and vlan type to rss hash and vlan flag */ + const __m128i vlan_flags = _mm_set_epi8(0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED, + 0, 0, 0, 0); + + const __m128i rss_flags = _mm_set_epi8(0, 0, 0, 0, + 0, 0, 0, 0, + PKT_RX_RSS_HASH | PKT_RX_FDIR, PKT_RX_RSS_HASH, 0, 0, + 0, 0, PKT_RX_FDIR, 0); + + const __m128i l3_l4e_flags = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, + /* shift right 1 bit to make sure it not exceed 255 */ + (PKT_RX_EIP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD | + PKT_RX_IP_CKSUM_BAD) >> 1, + (PKT_RX_IP_CKSUM_GOOD | PKT_RX_EIP_CKSUM_BAD | + PKT_RX_L4_CKSUM_BAD) >> 1, + (PKT_RX_EIP_CKSUM_BAD | PKT_RX_IP_CKSUM_BAD) >> 1, + (PKT_RX_IP_CKSUM_GOOD | PKT_RX_EIP_CKSUM_BAD) >> 1, + (PKT_RX_L4_CKSUM_BAD | PKT_RX_IP_CKSUM_BAD) >> 1, + (PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_BAD) >> 1, + PKT_RX_IP_CKSUM_BAD >> 1, + (PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD) >> 1); + + vlan0 = _mm_unpackhi_epi32(descs[0], descs[1]); + vlan1 = _mm_unpackhi_epi32(descs[2], descs[3]); + vlan0 = _mm_unpacklo_epi64(vlan0, vlan1); + + vlan1 = _mm_and_si128(vlan0, rss_vlan_msk); + vlan0 = _mm_shuffle_epi8(vlan_flags, vlan1); + + rss = _mm_srli_epi32(vlan1, 11); + rss = _mm_shuffle_epi8(rss_flags, rss); + + l3_l4e = _mm_srli_epi32(vlan1, 22); + l3_l4e = _mm_shuffle_epi8(l3_l4e_flags, l3_l4e); + /* then we shift left 1 bit */ + l3_l4e = _mm_slli_epi32(l3_l4e, 1); + /* we need to mask out the reduntant bits */ + l3_l4e = _mm_and_si128(l3_l4e, cksum_mask); + + vlan0 = _mm_or_si128(vlan0, rss); + vlan0 = _mm_or_si128(vlan0, l3_l4e); + + /* + * At this point, we have the 4 sets of flags in the low 16-bits + * of each 32-bit value in vlan0. + * We want to extract these, and merge them with the mbuf init data + * so we can do a single 16-byte write to the mbuf to set the flags + * and all the other initialization fields. Extracting the + * appropriate flags means that we have to do a shift and blend for + * each mbuf before we do the write. + */ + rearm0 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(vlan0, 8), 0x10); + rearm1 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(vlan0, 4), 0x10); + rearm2 = _mm_blend_epi16(mbuf_init, vlan0, 0x10); + rearm3 = _mm_blend_epi16(mbuf_init, _mm_srli_si128(vlan0, 4), 0x10); + + /* write the rearm data and the olflags in one write */ + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, ol_flags) != + offsetof(struct rte_mbuf, rearm_data) + 8); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, rearm_data) != + RTE_ALIGN(offsetof(struct rte_mbuf, rearm_data), 16)); + _mm_store_si128((__m128i *)&rx_pkts[0]->rearm_data, rearm0); + _mm_store_si128((__m128i *)&rx_pkts[1]->rearm_data, rearm1); + _mm_store_si128((__m128i *)&rx_pkts[2]->rearm_data, rearm2); + _mm_store_si128((__m128i *)&rx_pkts[3]->rearm_data, rearm3); +} + +#define PKTLEN_SHIFT 10 + +static inline void +desc_to_ptype_v(__m128i descs[4], struct rte_mbuf **rx_pkts, + uint32_t *ptype_tbl) +{ + __m128i ptype0 = _mm_unpackhi_epi64(descs[0], descs[1]); + __m128i ptype1 = _mm_unpackhi_epi64(descs[2], descs[3]); + + ptype0 = _mm_srli_epi64(ptype0, 30); + ptype1 = _mm_srli_epi64(ptype1, 30); + + rx_pkts[0]->packet_type = ptype_tbl[_mm_extract_epi8(ptype0, 0)]; + rx_pkts[1]->packet_type = ptype_tbl[_mm_extract_epi8(ptype0, 8)]; + rx_pkts[2]->packet_type = ptype_tbl[_mm_extract_epi8(ptype1, 0)]; + rx_pkts[3]->packet_type = ptype_tbl[_mm_extract_epi8(ptype1, 8)]; +} + + /* + * Notice: + * - nb_pkts < RTE_I40E_DESCS_PER_LOOP, just return no packet + * - nb_pkts > RTE_I40E_VPMD_RX_BURST, only scan RTE_I40E_VPMD_RX_BURST + * numbers of DD bits + */ +static inline uint16_t +_recv_raw_pkts_vec(struct i40e_rx_queue *rxq, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts, uint8_t *split_packet) +{ + volatile union i40e_rx_desc *rxdp; + struct i40e_rx_entry *sw_ring; + uint16_t nb_pkts_recd; + int pos; + uint64_t var; + __m128i shuf_msk; + uint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl; + + __m128i crc_adjust = _mm_set_epi16( + 0, 0, 0, /* ignore non-length fields */ + -rxq->crc_len, /* sub crc on data_len */ + 0, /* ignore high-16bits of pkt_len */ + -rxq->crc_len, /* sub crc on pkt_len */ + 0, 0 /* ignore pkt_type field */ + ); + /* + * compile-time check the above crc_adjust layout is correct. + * NOTE: the first field (lowest address) is given last in set_epi16 + * call above. + */ + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, pkt_len) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 4); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_len) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 8); + __m128i dd_check, eop_check; + + /* nb_pkts shall be less equal than RTE_I40E_MAX_RX_BURST */ + nb_pkts = RTE_MIN(nb_pkts, RTE_I40E_MAX_RX_BURST); + + /* nb_pkts has to be floor-aligned to RTE_I40E_DESCS_PER_LOOP */ + nb_pkts = RTE_ALIGN_FLOOR(nb_pkts, RTE_I40E_DESCS_PER_LOOP); + + /* Just the act of getting into the function from the application is + * going to cost about 7 cycles + */ + rxdp = rxq->rx_ring + rxq->rx_tail; + + rte_prefetch0(rxdp); + + /* See if we need to rearm the RX queue - gives the prefetch a bit + * of time to act + */ + if (rxq->rxrearm_nb > RTE_I40E_RXQ_REARM_THRESH) + i40e_rxq_rearm(rxq); + + /* Before we start moving massive data around, check to see if + * there is actually a packet available + */ + if (!(rxdp->wb.qword1.status_error_len & + rte_cpu_to_le_32(1 << I40E_RX_DESC_STATUS_DD_SHIFT))) + return 0; + + /* 4 packets DD mask */ + dd_check = _mm_set_epi64x(0x0000000100000001LL, 0x0000000100000001LL); + + /* 4 packets EOP mask */ + eop_check = _mm_set_epi64x(0x0000000200000002LL, 0x0000000200000002LL); + + /* mask to shuffle from desc. to mbuf */ + shuf_msk = _mm_set_epi8( + 7, 6, 5, 4, /* octet 4~7, 32bits rss */ + 3, 2, /* octet 2~3, low 16 bits vlan_macip */ + 15, 14, /* octet 15~14, 16 bits data_len */ + 0xFF, 0xFF, /* skip high 16 bits pkt_len, zero out */ + 15, 14, /* octet 15~14, low 16 bits pkt_len */ + 0xFF, 0xFF, /* pkt_type set as unknown */ + 0xFF, 0xFF /*pkt_type set as unknown */ + ); + /* + * Compile-time verify the shuffle mask + * NOTE: some field positions already verified above, but duplicated + * here for completeness in case of future modifications. + */ + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, pkt_len) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 4); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_len) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 8); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, vlan_tci) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 10); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, hash) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 12); + + /* Cache is empty -> need to scan the buffer rings, but first move + * the next 'n' mbufs into the cache + */ + sw_ring = &rxq->sw_ring[rxq->rx_tail]; + + /* A. load 4 packet in one loop + * [A*. mask out 4 unused dirty field in desc] + * B. copy 4 mbuf point from swring to rx_pkts + * C. calc the number of DD bits among the 4 packets + * [C*. extract the end-of-packet bit, if requested] + * D. fill info. from desc to mbuf + */ + + for (pos = 0, nb_pkts_recd = 0; pos < nb_pkts; + pos += RTE_I40E_DESCS_PER_LOOP, + rxdp += RTE_I40E_DESCS_PER_LOOP) { + __m128i descs[RTE_I40E_DESCS_PER_LOOP]; + __m128i pkt_mb1, pkt_mb2, pkt_mb3, pkt_mb4; + __m128i zero, staterr, sterr_tmp1, sterr_tmp2; + /* 2 64 bit or 4 32 bit mbuf pointers in one XMM reg. */ + __m128i mbp1; +#if defined(RTE_ARCH_X86_64) + __m128i mbp2; +#endif + + /* B.1 load 2 (64 bit) or 4 (32 bit) mbuf points */ + mbp1 = _mm_loadu_si128((__m128i *)&sw_ring[pos]); + /* Read desc statuses backwards to avoid race condition */ + /* A.1 load 4 pkts desc */ + descs[3] = _mm_loadu_si128((__m128i *)(rxdp + 3)); + rte_compiler_barrier(); + + /* B.2 copy 2 64 bit or 4 32 bit mbuf point into rx_pkts */ + _mm_storeu_si128((__m128i *)&rx_pkts[pos], mbp1); + +#if defined(RTE_ARCH_X86_64) + /* B.1 load 2 64 bit mbuf points */ + mbp2 = _mm_loadu_si128((__m128i *)&sw_ring[pos+2]); +#endif + + descs[2] = _mm_loadu_si128((__m128i *)(rxdp + 2)); + rte_compiler_barrier(); + /* B.1 load 2 mbuf point */ + descs[1] = _mm_loadu_si128((__m128i *)(rxdp + 1)); + rte_compiler_barrier(); + descs[0] = _mm_loadu_si128((__m128i *)(rxdp)); + +#if defined(RTE_ARCH_X86_64) + /* B.2 copy 2 mbuf point into rx_pkts */ + _mm_storeu_si128((__m128i *)&rx_pkts[pos+2], mbp2); +#endif + + if (split_packet) { + rte_mbuf_prefetch_part2(rx_pkts[pos]); + rte_mbuf_prefetch_part2(rx_pkts[pos + 1]); + rte_mbuf_prefetch_part2(rx_pkts[pos + 2]); + rte_mbuf_prefetch_part2(rx_pkts[pos + 3]); + } + + /* avoid compiler reorder optimization */ + rte_compiler_barrier(); + + /* pkt 3,4 shift the pktlen field to be 16-bit aligned*/ + const __m128i len3 = _mm_slli_epi32(descs[3], PKTLEN_SHIFT); + const __m128i len2 = _mm_slli_epi32(descs[2], PKTLEN_SHIFT); + + /* merge the now-aligned packet length fields back in */ + descs[3] = _mm_blend_epi16(descs[3], len3, 0x80); + descs[2] = _mm_blend_epi16(descs[2], len2, 0x80); + + /* D.1 pkt 3,4 convert format from desc to pktmbuf */ + pkt_mb4 = _mm_shuffle_epi8(descs[3], shuf_msk); + pkt_mb3 = _mm_shuffle_epi8(descs[2], shuf_msk); + + /* C.1 4=>2 filter staterr info only */ + sterr_tmp2 = _mm_unpackhi_epi32(descs[3], descs[2]); + /* C.1 4=>2 filter staterr info only */ + sterr_tmp1 = _mm_unpackhi_epi32(descs[1], descs[0]); + + desc_to_olflags_v(rxq, descs, &rx_pkts[pos]); + + /* D.2 pkt 3,4 set in_port/nb_seg and remove crc */ + pkt_mb4 = _mm_add_epi16(pkt_mb4, crc_adjust); + pkt_mb3 = _mm_add_epi16(pkt_mb3, crc_adjust); + + /* pkt 1,2 shift the pktlen field to be 16-bit aligned*/ + const __m128i len1 = _mm_slli_epi32(descs[1], PKTLEN_SHIFT); + const __m128i len0 = _mm_slli_epi32(descs[0], PKTLEN_SHIFT); + + /* merge the now-aligned packet length fields back in */ + descs[1] = _mm_blend_epi16(descs[1], len1, 0x80); + descs[0] = _mm_blend_epi16(descs[0], len0, 0x80); + + /* D.1 pkt 1,2 convert format from desc to pktmbuf */ + pkt_mb2 = _mm_shuffle_epi8(descs[1], shuf_msk); + pkt_mb1 = _mm_shuffle_epi8(descs[0], shuf_msk); + + /* C.2 get 4 pkts staterr value */ + zero = _mm_xor_si128(dd_check, dd_check); + staterr = _mm_unpacklo_epi32(sterr_tmp1, sterr_tmp2); + + /* D.3 copy final 3,4 data to rx_pkts */ + _mm_storeu_si128((void *)&rx_pkts[pos+3]->rx_descriptor_fields1, + pkt_mb4); + _mm_storeu_si128((void *)&rx_pkts[pos+2]->rx_descriptor_fields1, + pkt_mb3); + + /* D.2 pkt 1,2 set in_port/nb_seg and remove crc */ + pkt_mb2 = _mm_add_epi16(pkt_mb2, crc_adjust); + pkt_mb1 = _mm_add_epi16(pkt_mb1, crc_adjust); + + /* C* extract and record EOP bit */ + if (split_packet) { + __m128i eop_shuf_mask = _mm_set_epi8( + 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, + 0x04, 0x0C, 0x00, 0x08 + ); + + /* and with mask to extract bits, flipping 1-0 */ + __m128i eop_bits = _mm_andnot_si128(staterr, eop_check); + /* the staterr values are not in order, as the count + * count of dd bits doesn't care. However, for end of + * packet tracking, we do care, so shuffle. This also + * compresses the 32-bit values to 8-bit + */ + eop_bits = _mm_shuffle_epi8(eop_bits, eop_shuf_mask); + /* store the resulting 32-bit value */ + *(int *)split_packet = _mm_cvtsi128_si32(eop_bits); + split_packet += RTE_I40E_DESCS_PER_LOOP; + } + + /* C.3 calc available number of desc */ + staterr = _mm_and_si128(staterr, dd_check); + staterr = _mm_packs_epi32(staterr, zero); + + /* D.3 copy final 1,2 data to rx_pkts */ + _mm_storeu_si128((void *)&rx_pkts[pos+1]->rx_descriptor_fields1, + pkt_mb2); + _mm_storeu_si128((void *)&rx_pkts[pos]->rx_descriptor_fields1, + pkt_mb1); + desc_to_ptype_v(descs, &rx_pkts[pos], ptype_tbl); + /* C.4 calc avaialbe number of desc */ + var = __builtin_popcountll(_mm_cvtsi128_si64(staterr)); + nb_pkts_recd += var; + if (likely(var != RTE_I40E_DESCS_PER_LOOP)) + break; + } + + /* Update our internal tail pointer */ + rxq->rx_tail = (uint16_t)(rxq->rx_tail + nb_pkts_recd); + rxq->rx_tail = (uint16_t)(rxq->rx_tail & (rxq->nb_rx_desc - 1)); + rxq->rxrearm_nb = (uint16_t)(rxq->rxrearm_nb + nb_pkts_recd); + + return nb_pkts_recd; +} + + /* + * Notice: + * - nb_pkts < RTE_I40E_DESCS_PER_LOOP, just return no packet + * - nb_pkts > RTE_I40E_VPMD_RX_BURST, only scan RTE_I40E_VPMD_RX_BURST + * numbers of DD bits + */ +uint16_t +i40e_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts) +{ + return _recv_raw_pkts_vec(rx_queue, rx_pkts, nb_pkts, NULL); +} + + /* vPMD receive routine that reassembles scattered packets + * Notice: + * - nb_pkts < RTE_I40E_DESCS_PER_LOOP, just return no packet + * - nb_pkts > RTE_I40E_VPMD_RX_BURST, only scan RTE_I40E_VPMD_RX_BURST + * numbers of DD bits + */ +uint16_t +i40e_recv_scattered_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts) +{ + + struct i40e_rx_queue *rxq = rx_queue; + uint8_t split_flags[RTE_I40E_VPMD_RX_BURST] = {0}; + + /* get some new buffers */ + uint16_t nb_bufs = _recv_raw_pkts_vec(rxq, rx_pkts, nb_pkts, + split_flags); + if (nb_bufs == 0) + return 0; + + /* happy day case, full burst + no packets to be joined */ + const uint64_t *split_fl64 = (uint64_t *)split_flags; + + if (rxq->pkt_first_seg == NULL && + split_fl64[0] == 0 && split_fl64[1] == 0 && + split_fl64[2] == 0 && split_fl64[3] == 0) + return nb_bufs; + + /* reassemble any packets that need reassembly*/ + unsigned i = 0; + + if (rxq->pkt_first_seg == NULL) { + /* find the first split flag, and only reassemble then*/ + while (i < nb_bufs && !split_flags[i]) + i++; + if (i == nb_bufs) + return nb_bufs; + } + return i + reassemble_packets(rxq, &rx_pkts[i], nb_bufs - i, + &split_flags[i]); +} + +static inline void +vtx1(volatile struct i40e_tx_desc *txdp, + struct rte_mbuf *pkt, uint64_t flags) +{ + uint64_t high_qw = (I40E_TX_DESC_DTYPE_DATA | + ((uint64_t)flags << I40E_TXD_QW1_CMD_SHIFT) | + ((uint64_t)pkt->data_len << I40E_TXD_QW1_TX_BUF_SZ_SHIFT)); + + __m128i descriptor = _mm_set_epi64x(high_qw, + pkt->buf_iova + pkt->data_off); + _mm_store_si128((__m128i *)txdp, descriptor); +} + +static inline void +vtx(volatile struct i40e_tx_desc *txdp, + struct rte_mbuf **pkt, uint16_t nb_pkts, uint64_t flags) +{ + int i; + + for (i = 0; i < nb_pkts; ++i, ++txdp, ++pkt) + vtx1(txdp, *pkt, flags); +} + +uint16_t +i40e_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts) +{ + struct i40e_tx_queue *txq = (struct i40e_tx_queue *)tx_queue; + volatile struct i40e_tx_desc *txdp; + struct i40e_tx_entry *txep; + uint16_t n, nb_commit, tx_id; + uint64_t flags = I40E_TD_CMD; + uint64_t rs = I40E_TX_DESC_CMD_RS | I40E_TD_CMD; + int i; + + /* cross rx_thresh boundary is not allowed */ + nb_pkts = RTE_MIN(nb_pkts, txq->tx_rs_thresh); + + if (txq->nb_tx_free < txq->tx_free_thresh) + i40e_tx_free_bufs(txq); + + nb_commit = nb_pkts = (uint16_t)RTE_MIN(txq->nb_tx_free, nb_pkts); + if (unlikely(nb_pkts == 0)) + return 0; + + tx_id = txq->tx_tail; + txdp = &txq->tx_ring[tx_id]; + txep = &txq->sw_ring[tx_id]; + + txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_pkts); + + n = (uint16_t)(txq->nb_tx_desc - tx_id); + if (nb_commit >= n) { + tx_backlog_entry(txep, tx_pkts, n); + + for (i = 0; i < n - 1; ++i, ++tx_pkts, ++txdp) + vtx1(txdp, *tx_pkts, flags); + + vtx1(txdp, *tx_pkts++, rs); + + nb_commit = (uint16_t)(nb_commit - n); + + tx_id = 0; + txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1); + + /* avoid reach the end of ring */ + txdp = &txq->tx_ring[tx_id]; + txep = &txq->sw_ring[tx_id]; + } + + tx_backlog_entry(txep, tx_pkts, nb_commit); + + vtx(txdp, tx_pkts, nb_commit, flags); + + tx_id = (uint16_t)(tx_id + nb_commit); + if (tx_id > txq->tx_next_rs) { + txq->tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |= + rte_cpu_to_le_64(((uint64_t)I40E_TX_DESC_CMD_RS) << + I40E_TXD_QW1_CMD_SHIFT); + txq->tx_next_rs = + (uint16_t)(txq->tx_next_rs + txq->tx_rs_thresh); + } + + txq->tx_tail = tx_id; + + I40E_PCI_REG_WRITE(txq->qtx_tail, txq->tx_tail); + + return nb_pkts; +} + +void __attribute__((cold)) +i40e_rx_queue_release_mbufs_vec(struct i40e_rx_queue *rxq) +{ + _i40e_rx_queue_release_mbufs_vec(rxq); +} + +int __attribute__((cold)) +i40e_rxq_vec_setup(struct i40e_rx_queue *rxq) +{ + return i40e_rxq_vec_setup_default(rxq); +} + +int __attribute__((cold)) +i40e_txq_vec_setup(struct i40e_tx_queue __rte_unused *txq) +{ + return 0; +} + +int __attribute__((cold)) +i40e_rx_vec_dev_conf_condition_check(struct rte_eth_dev *dev) +{ + return i40e_rx_vec_dev_conf_condition_check_default(dev); +} diff --git a/dpdk/drivers/net/i40e/i40e_tm.c b/dpdk/drivers/net/i40e/i40e_tm.c new file mode 100644 index 00000000..44316f64 --- /dev/null +++ b/dpdk/drivers/net/i40e/i40e_tm.c @@ -0,0 +1,1000 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +#include "base/i40e_prototype.h" +#include "i40e_ethdev.h" + +static int i40e_tm_capabilities_get(struct rte_eth_dev *dev, + struct rte_tm_capabilities *cap, + struct rte_tm_error *error); +static int i40e_shaper_profile_add(struct rte_eth_dev *dev, + uint32_t shaper_profile_id, + struct rte_tm_shaper_params *profile, + struct rte_tm_error *error); +static int i40e_shaper_profile_del(struct rte_eth_dev *dev, + uint32_t shaper_profile_id, + struct rte_tm_error *error); +static int i40e_node_add(struct rte_eth_dev *dev, uint32_t node_id, + uint32_t parent_node_id, uint32_t priority, + uint32_t weight, uint32_t level_id, + struct rte_tm_node_params *params, + struct rte_tm_error *error); +static int i40e_node_delete(struct rte_eth_dev *dev, uint32_t node_id, + struct rte_tm_error *error); +static int i40e_node_type_get(struct rte_eth_dev *dev, uint32_t node_id, + int *is_leaf, struct rte_tm_error *error); +static int i40e_level_capabilities_get(struct rte_eth_dev *dev, + uint32_t level_id, + struct rte_tm_level_capabilities *cap, + struct rte_tm_error *error); +static int i40e_node_capabilities_get(struct rte_eth_dev *dev, + uint32_t node_id, + struct rte_tm_node_capabilities *cap, + struct rte_tm_error *error); +static int i40e_hierarchy_commit(struct rte_eth_dev *dev, + int clear_on_fail, + struct rte_tm_error *error); + +const struct rte_tm_ops i40e_tm_ops = { + .capabilities_get = i40e_tm_capabilities_get, + .shaper_profile_add = i40e_shaper_profile_add, + .shaper_profile_delete = i40e_shaper_profile_del, + .node_add = i40e_node_add, + .node_delete = i40e_node_delete, + .node_type_get = i40e_node_type_get, + .level_capabilities_get = i40e_level_capabilities_get, + .node_capabilities_get = i40e_node_capabilities_get, + .hierarchy_commit = i40e_hierarchy_commit, +}; + +int +i40e_tm_ops_get(struct rte_eth_dev *dev __rte_unused, + void *arg) +{ + if (!arg) + return -EINVAL; + + *(const void **)arg = &i40e_tm_ops; + + return 0; +} + +void +i40e_tm_conf_init(struct rte_eth_dev *dev) +{ + struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + + /* initialize shaper profile list */ + TAILQ_INIT(&pf->tm_conf.shaper_profile_list); + + /* initialize node configuration */ + pf->tm_conf.root = NULL; + TAILQ_INIT(&pf->tm_conf.tc_list); + TAILQ_INIT(&pf->tm_conf.queue_list); + pf->tm_conf.nb_tc_node = 0; + pf->tm_conf.nb_queue_node = 0; + pf->tm_conf.committed = false; +} + +void +i40e_tm_conf_uninit(struct rte_eth_dev *dev) +{ + struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + struct i40e_tm_shaper_profile *shaper_profile; + struct i40e_tm_node *tm_node; + + /* clear node configuration */ + while ((tm_node = TAILQ_FIRST(&pf->tm_conf.queue_list))) { + TAILQ_REMOVE(&pf->tm_conf.queue_list, tm_node, node); + rte_free(tm_node); + } + pf->tm_conf.nb_queue_node = 0; + while ((tm_node = TAILQ_FIRST(&pf->tm_conf.tc_list))) { + TAILQ_REMOVE(&pf->tm_conf.tc_list, tm_node, node); + rte_free(tm_node); + } + pf->tm_conf.nb_tc_node = 0; + if (pf->tm_conf.root) { + rte_free(pf->tm_conf.root); + pf->tm_conf.root = NULL; + } + + /* Remove all shaper profiles */ + while ((shaper_profile = + TAILQ_FIRST(&pf->tm_conf.shaper_profile_list))) { + TAILQ_REMOVE(&pf->tm_conf.shaper_profile_list, + shaper_profile, node); + rte_free(shaper_profile); + } +} + +static inline uint16_t +i40e_tc_nb_get(struct rte_eth_dev *dev) +{ + struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + struct i40e_vsi *main_vsi = pf->main_vsi; + uint16_t sum = 0; + int i; + + for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { + if (main_vsi->enabled_tc & BIT_ULL(i)) + sum++; + } + + return sum; +} + +static int +i40e_tm_capabilities_get(struct rte_eth_dev *dev, + struct rte_tm_capabilities *cap, + struct rte_tm_error *error) +{ + struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); + uint16_t tc_nb = i40e_tc_nb_get(dev); + + if (!cap || !error) + return -EINVAL; + + if (tc_nb > hw->func_caps.num_tx_qp) + return -EINVAL; + + error->type = RTE_TM_ERROR_TYPE_NONE; + + /* set all the parameters to 0 first. */ + memset(cap, 0, sizeof(struct rte_tm_capabilities)); + + /** + * support port + TCs + queues + * here shows the max capability not the current configuration. + */ + cap->n_nodes_max = 1 + I40E_MAX_TRAFFIC_CLASS + hw->func_caps.num_tx_qp; + cap->n_levels_max = 3; /* port, TC, queue */ + cap->non_leaf_nodes_identical = 1; + cap->leaf_nodes_identical = 1; + cap->shaper_n_max = cap->n_nodes_max; + cap->shaper_private_n_max = cap->n_nodes_max; + cap->shaper_private_dual_rate_n_max = 0; + cap->shaper_private_rate_min = 0; + /* 40Gbps -> 5GBps */ + cap->shaper_private_rate_max = 5000000000ull; + cap->shaper_shared_n_max = 0; + cap->shaper_shared_n_nodes_per_shaper_max = 0; + cap->shaper_shared_n_shapers_per_node_max = 0; + cap->shaper_shared_dual_rate_n_max = 0; + cap->shaper_shared_rate_min = 0; + cap->shaper_shared_rate_max = 0; + cap->sched_n_children_max = hw->func_caps.num_tx_qp; + /** + * HW supports SP. But no plan to support it now. + * So, all the nodes should have the same priority. + */ + cap->sched_sp_n_priorities_max = 1; + cap->sched_wfq_n_children_per_group_max = 0; + cap->sched_wfq_n_groups_max = 0; + /** + * SW only supports fair round robin now. + * So, all the nodes should have the same weight. + */ + cap->sched_wfq_weight_max = 1; + cap->cman_head_drop_supported = 0; + cap->dynamic_update_mask = 0; + cap->shaper_pkt_length_adjust_min = RTE_TM_ETH_FRAMING_OVERHEAD; + cap->shaper_pkt_length_adjust_max = RTE_TM_ETH_FRAMING_OVERHEAD_FCS; + cap->cman_wred_context_n_max = 0; + cap->cman_wred_context_private_n_max = 0; + cap->cman_wred_context_shared_n_max = 0; + cap->cman_wred_context_shared_n_nodes_per_context_max = 0; + cap->cman_wred_context_shared_n_contexts_per_node_max = 0; + cap->stats_mask = 0; + + return 0; +} + +static inline struct i40e_tm_shaper_profile * +i40e_shaper_profile_search(struct rte_eth_dev *dev, + uint32_t shaper_profile_id) +{ + struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + struct i40e_shaper_profile_list *shaper_profile_list = + &pf->tm_conf.shaper_profile_list; + struct i40e_tm_shaper_profile *shaper_profile; + + TAILQ_FOREACH(shaper_profile, shaper_profile_list, node) { + if (shaper_profile_id == shaper_profile->shaper_profile_id) + return shaper_profile; + } + + return NULL; +} + +static int +i40e_shaper_profile_param_check(struct rte_tm_shaper_params *profile, + struct rte_tm_error *error) +{ + /* min rate not supported */ + if (profile->committed.rate) { + error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_COMMITTED_RATE; + error->message = "committed rate not supported"; + return -EINVAL; + } + /* min bucket size not supported */ + if (profile->committed.size) { + error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_COMMITTED_SIZE; + error->message = "committed bucket size not supported"; + return -EINVAL; + } + /* max bucket size not supported */ + if (profile->peak.size) { + error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_PEAK_SIZE; + error->message = "peak bucket size not supported"; + return -EINVAL; + } + /* length adjustment not supported */ + if (profile->pkt_length_adjust) { + error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_PKT_ADJUST_LEN; + error->message = "packet length adjustment not supported"; + return -EINVAL; + } + + return 0; +} + +static int +i40e_shaper_profile_add(struct rte_eth_dev *dev, + uint32_t shaper_profile_id, + struct rte_tm_shaper_params *profile, + struct rte_tm_error *error) +{ + struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + struct i40e_tm_shaper_profile *shaper_profile; + int ret; + + if (!profile || !error) + return -EINVAL; + + ret = i40e_shaper_profile_param_check(profile, error); + if (ret) + return ret; + + shaper_profile = i40e_shaper_profile_search(dev, shaper_profile_id); + + if (shaper_profile) { + error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_ID; + error->message = "profile ID exist"; + return -EINVAL; + } + + shaper_profile = rte_zmalloc("i40e_tm_shaper_profile", + sizeof(struct i40e_tm_shaper_profile), + 0); + if (!shaper_profile) + return -ENOMEM; + shaper_profile->shaper_profile_id = shaper_profile_id; + rte_memcpy(&shaper_profile->profile, profile, + sizeof(struct rte_tm_shaper_params)); + TAILQ_INSERT_TAIL(&pf->tm_conf.shaper_profile_list, + shaper_profile, node); + + return 0; +} + +static int +i40e_shaper_profile_del(struct rte_eth_dev *dev, + uint32_t shaper_profile_id, + struct rte_tm_error *error) +{ + struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + struct i40e_tm_shaper_profile *shaper_profile; + + if (!error) + return -EINVAL; + + shaper_profile = i40e_shaper_profile_search(dev, shaper_profile_id); + + if (!shaper_profile) { + error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_ID; + error->message = "profile ID not exist"; + return -EINVAL; + } + + /* don't delete a profile if it's used by one or several nodes */ + if (shaper_profile->reference_count) { + error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE; + error->message = "profile in use"; + return -EINVAL; + } + + TAILQ_REMOVE(&pf->tm_conf.shaper_profile_list, shaper_profile, node); + rte_free(shaper_profile); + + return 0; +} + +static inline struct i40e_tm_node * +i40e_tm_node_search(struct rte_eth_dev *dev, + uint32_t node_id, enum i40e_tm_node_type *node_type) +{ + struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + struct i40e_tm_node_list *queue_list = &pf->tm_conf.queue_list; + struct i40e_tm_node_list *tc_list = &pf->tm_conf.tc_list; + struct i40e_tm_node *tm_node; + + if (pf->tm_conf.root && pf->tm_conf.root->id == node_id) { + *node_type = I40E_TM_NODE_TYPE_PORT; + return pf->tm_conf.root; + } + + TAILQ_FOREACH(tm_node, tc_list, node) { + if (tm_node->id == node_id) { + *node_type = I40E_TM_NODE_TYPE_TC; + return tm_node; + } + } + + TAILQ_FOREACH(tm_node, queue_list, node) { + if (tm_node->id == node_id) { + *node_type = I40E_TM_NODE_TYPE_QUEUE; + return tm_node; + } + } + + return NULL; +} + +static int +i40e_node_param_check(struct rte_eth_dev *dev, uint32_t node_id, + uint32_t priority, uint32_t weight, + struct rte_tm_node_params *params, + struct rte_tm_error *error) +{ + struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); + + if (node_id == RTE_TM_NODE_ID_NULL) { + error->type = RTE_TM_ERROR_TYPE_NODE_ID; + error->message = "invalid node id"; + return -EINVAL; + } + + if (priority) { + error->type = RTE_TM_ERROR_TYPE_NODE_PRIORITY; + error->message = "priority should be 0"; + return -EINVAL; + } + + if (weight != 1) { + error->type = RTE_TM_ERROR_TYPE_NODE_WEIGHT; + error->message = "weight must be 1"; + return -EINVAL; + } + + /* not support shared shaper */ + if (params->shared_shaper_id) { + error->type = RTE_TM_ERROR_TYPE_NODE_PARAMS_SHARED_SHAPER_ID; + error->message = "shared shaper not supported"; + return -EINVAL; + } + if (params->n_shared_shapers) { + error->type = RTE_TM_ERROR_TYPE_NODE_PARAMS_N_SHARED_SHAPERS; + error->message = "shared shaper not supported"; + return -EINVAL; + } + + /* for non-leaf node */ + if (node_id >= hw->func_caps.num_tx_qp) { + if (params->nonleaf.wfq_weight_mode) { + error->type = + RTE_TM_ERROR_TYPE_NODE_PARAMS_WFQ_WEIGHT_MODE; + error->message = "WFQ not supported"; + return -EINVAL; + } + if (params->nonleaf.n_sp_priorities != 1) { + error->type = + RTE_TM_ERROR_TYPE_NODE_PARAMS_N_SP_PRIORITIES; + error->message = "SP priority not supported"; + return -EINVAL; + } else if (params->nonleaf.wfq_weight_mode && + !(*params->nonleaf.wfq_weight_mode)) { + error->type = + RTE_TM_ERROR_TYPE_NODE_PARAMS_WFQ_WEIGHT_MODE; + error->message = "WFP should be byte mode"; + return -EINVAL; + } + + return 0; + } + + /* for leaf node */ + if (params->leaf.cman) { + error->type = RTE_TM_ERROR_TYPE_NODE_PARAMS_CMAN; + error->message = "Congestion management not supported"; + return -EINVAL; + } + if (params->leaf.wred.wred_profile_id != + RTE_TM_WRED_PROFILE_ID_NONE) { + error->type = + RTE_TM_ERROR_TYPE_NODE_PARAMS_WRED_PROFILE_ID; + error->message = "WRED not supported"; + return -EINVAL; + } + if (params->leaf.wred.shared_wred_context_id) { + error->type = + RTE_TM_ERROR_TYPE_NODE_PARAMS_SHARED_WRED_CONTEXT_ID; + error->message = "WRED not supported"; + return -EINVAL; + } + if (params->leaf.wred.n_shared_wred_contexts) { + error->type = + RTE_TM_ERROR_TYPE_NODE_PARAMS_N_SHARED_WRED_CONTEXTS; + error->message = "WRED not supported"; + return -EINVAL; + } + + return 0; +} + +/** + * Now the TC and queue configuration is controlled by DCB. + * We need check if the node configuration follows the DCB configuration. + * In the future, we may use TM to cover DCB. + */ +static int +i40e_node_add(struct rte_eth_dev *dev, uint32_t node_id, + uint32_t parent_node_id, uint32_t priority, + uint32_t weight, uint32_t level_id, + struct rte_tm_node_params *params, + struct rte_tm_error *error) +{ + struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + enum i40e_tm_node_type node_type = I40E_TM_NODE_TYPE_MAX; + enum i40e_tm_node_type parent_node_type = I40E_TM_NODE_TYPE_MAX; + struct i40e_tm_shaper_profile *shaper_profile = NULL; + struct i40e_tm_node *tm_node; + struct i40e_tm_node *parent_node; + uint16_t tc_nb = 0; + int ret; + + if (!params || !error) + return -EINVAL; + + /* if already committed */ + if (pf->tm_conf.committed) { + error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED; + error->message = "already committed"; + return -EINVAL; + } + + ret = i40e_node_param_check(dev, node_id, priority, weight, + params, error); + if (ret) + return ret; + + /* check if the node ID is already used */ + if (i40e_tm_node_search(dev, node_id, &node_type)) { + error->type = RTE_TM_ERROR_TYPE_NODE_ID; + error->message = "node id already used"; + return -EINVAL; + } + + /* check the shaper profile id */ + if (params->shaper_profile_id != RTE_TM_SHAPER_PROFILE_ID_NONE) { + shaper_profile = i40e_shaper_profile_search( + dev, params->shaper_profile_id); + if (!shaper_profile) { + error->type = + RTE_TM_ERROR_TYPE_NODE_PARAMS_SHAPER_PROFILE_ID; + error->message = "shaper profile not exist"; + return -EINVAL; + } + } + + /* root node if not have a parent */ + if (parent_node_id == RTE_TM_NODE_ID_NULL) { + /* check level */ + if (level_id != RTE_TM_NODE_LEVEL_ID_ANY && + level_id > I40E_TM_NODE_TYPE_PORT) { + error->type = RTE_TM_ERROR_TYPE_NODE_PARAMS; + error->message = "Wrong level"; + return -EINVAL; + } + + /* obviously no more than one root */ + if (pf->tm_conf.root) { + error->type = RTE_TM_ERROR_TYPE_NODE_PARENT_NODE_ID; + error->message = "already have a root"; + return -EINVAL; + } + + /* add the root node */ + tm_node = rte_zmalloc("i40e_tm_node", + sizeof(struct i40e_tm_node), + 0); + if (!tm_node) + return -ENOMEM; + tm_node->id = node_id; + tm_node->priority = priority; + tm_node->weight = weight; + tm_node->reference_count = 0; + tm_node->parent = NULL; + tm_node->shaper_profile = shaper_profile; + rte_memcpy(&tm_node->params, params, + sizeof(struct rte_tm_node_params)); + pf->tm_conf.root = tm_node; + + /* increase the reference counter of the shaper profile */ + if (shaper_profile) + shaper_profile->reference_count++; + + return 0; + } + + /* TC or queue node */ + /* check the parent node */ + parent_node = i40e_tm_node_search(dev, parent_node_id, + &parent_node_type); + if (!parent_node) { + error->type = RTE_TM_ERROR_TYPE_NODE_PARENT_NODE_ID; + error->message = "parent not exist"; + return -EINVAL; + } + if (parent_node_type != I40E_TM_NODE_TYPE_PORT && + parent_node_type != I40E_TM_NODE_TYPE_TC) { + error->type = RTE_TM_ERROR_TYPE_NODE_PARENT_NODE_ID; + error->message = "parent is not port or TC"; + return -EINVAL; + } + /* check level */ + if (level_id != RTE_TM_NODE_LEVEL_ID_ANY && + level_id != parent_node_type + 1) { + error->type = RTE_TM_ERROR_TYPE_NODE_PARAMS; + error->message = "Wrong level"; + return -EINVAL; + } + + /* check the node number */ + if (parent_node_type == I40E_TM_NODE_TYPE_PORT) { + /* check the TC number */ + tc_nb = i40e_tc_nb_get(dev); + if (pf->tm_conf.nb_tc_node >= tc_nb) { + error->type = RTE_TM_ERROR_TYPE_NODE_ID; + error->message = "too many TCs"; + return -EINVAL; + } + } else { + /* check the queue number */ + if (pf->tm_conf.nb_queue_node >= hw->func_caps.num_tx_qp) { + error->type = RTE_TM_ERROR_TYPE_NODE_ID; + error->message = "too many queues"; + return -EINVAL; + } + + /** + * check the node id. + * For queue, the node id means queue id. + */ + if (node_id >= hw->func_caps.num_tx_qp) { + error->type = RTE_TM_ERROR_TYPE_NODE_ID; + error->message = "too large queue id"; + return -EINVAL; + } + } + + /* add the TC or queue node */ + tm_node = rte_zmalloc("i40e_tm_node", + sizeof(struct i40e_tm_node), + 0); + if (!tm_node) + return -ENOMEM; + tm_node->id = node_id; + tm_node->priority = priority; + tm_node->weight = weight; + tm_node->reference_count = 0; + tm_node->parent = parent_node; + tm_node->shaper_profile = shaper_profile; + rte_memcpy(&tm_node->params, params, + sizeof(struct rte_tm_node_params)); + if (parent_node_type == I40E_TM_NODE_TYPE_PORT) { + TAILQ_INSERT_TAIL(&pf->tm_conf.tc_list, + tm_node, node); + pf->tm_conf.nb_tc_node++; + } else { + TAILQ_INSERT_TAIL(&pf->tm_conf.queue_list, + tm_node, node); + pf->tm_conf.nb_queue_node++; + } + tm_node->parent->reference_count++; + + /* increase the reference counter of the shaper profile */ + if (shaper_profile) + shaper_profile->reference_count++; + + return 0; +} + +static int +i40e_node_delete(struct rte_eth_dev *dev, uint32_t node_id, + struct rte_tm_error *error) +{ + struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + enum i40e_tm_node_type node_type = I40E_TM_NODE_TYPE_MAX; + struct i40e_tm_node *tm_node; + + if (!error) + return -EINVAL; + + /* if already committed */ + if (pf->tm_conf.committed) { + error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED; + error->message = "already committed"; + return -EINVAL; + } + + if (node_id == RTE_TM_NODE_ID_NULL) { + error->type = RTE_TM_ERROR_TYPE_NODE_ID; + error->message = "invalid node id"; + return -EINVAL; + } + + /* check if the node id exists */ + tm_node = i40e_tm_node_search(dev, node_id, &node_type); + if (!tm_node) { + error->type = RTE_TM_ERROR_TYPE_NODE_ID; + error->message = "no such node"; + return -EINVAL; + } + + /* the node should have no child */ + if (tm_node->reference_count) { + error->type = RTE_TM_ERROR_TYPE_NODE_ID; + error->message = + "cannot delete a node which has children"; + return -EINVAL; + } + + /* root node */ + if (node_type == I40E_TM_NODE_TYPE_PORT) { + if (tm_node->shaper_profile) + tm_node->shaper_profile->reference_count--; + rte_free(tm_node); + pf->tm_conf.root = NULL; + return 0; + } + + /* TC or queue node */ + if (tm_node->shaper_profile) + tm_node->shaper_profile->reference_count--; + tm_node->parent->reference_count--; + if (node_type == I40E_TM_NODE_TYPE_TC) { + TAILQ_REMOVE(&pf->tm_conf.tc_list, tm_node, node); + pf->tm_conf.nb_tc_node--; + } else { + TAILQ_REMOVE(&pf->tm_conf.queue_list, tm_node, node); + pf->tm_conf.nb_queue_node--; + } + rte_free(tm_node); + + return 0; +} + +static int +i40e_node_type_get(struct rte_eth_dev *dev, uint32_t node_id, + int *is_leaf, struct rte_tm_error *error) +{ + enum i40e_tm_node_type node_type = I40E_TM_NODE_TYPE_MAX; + struct i40e_tm_node *tm_node; + + if (!is_leaf || !error) + return -EINVAL; + + if (node_id == RTE_TM_NODE_ID_NULL) { + error->type = RTE_TM_ERROR_TYPE_NODE_ID; + error->message = "invalid node id"; + return -EINVAL; + } + + /* check if the node id exists */ + tm_node = i40e_tm_node_search(dev, node_id, &node_type); + if (!tm_node) { + error->type = RTE_TM_ERROR_TYPE_NODE_ID; + error->message = "no such node"; + return -EINVAL; + } + + if (node_type == I40E_TM_NODE_TYPE_QUEUE) + *is_leaf = true; + else + *is_leaf = false; + + return 0; +} + +static int +i40e_level_capabilities_get(struct rte_eth_dev *dev, + uint32_t level_id, + struct rte_tm_level_capabilities *cap, + struct rte_tm_error *error) +{ + struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); + + if (!cap || !error) + return -EINVAL; + + if (level_id >= I40E_TM_NODE_TYPE_MAX) { + error->type = RTE_TM_ERROR_TYPE_LEVEL_ID; + error->message = "too deep level"; + return -EINVAL; + } + + /* root node */ + if (level_id == I40E_TM_NODE_TYPE_PORT) { + cap->n_nodes_max = 1; + cap->n_nodes_nonleaf_max = 1; + cap->n_nodes_leaf_max = 0; + } else if (level_id == I40E_TM_NODE_TYPE_TC) { + /* TC */ + cap->n_nodes_max = I40E_MAX_TRAFFIC_CLASS; + cap->n_nodes_nonleaf_max = I40E_MAX_TRAFFIC_CLASS; + cap->n_nodes_leaf_max = 0; + } else { + /* queue */ + cap->n_nodes_max = hw->func_caps.num_tx_qp; + cap->n_nodes_nonleaf_max = 0; + cap->n_nodes_leaf_max = hw->func_caps.num_tx_qp; + } + + cap->non_leaf_nodes_identical = true; + cap->leaf_nodes_identical = true; + + if (level_id != I40E_TM_NODE_TYPE_QUEUE) { + cap->nonleaf.shaper_private_supported = true; + cap->nonleaf.shaper_private_dual_rate_supported = false; + cap->nonleaf.shaper_private_rate_min = 0; + /* 40Gbps -> 5GBps */ + cap->nonleaf.shaper_private_rate_max = 5000000000ull; + cap->nonleaf.shaper_shared_n_max = 0; + if (level_id == I40E_TM_NODE_TYPE_PORT) + cap->nonleaf.sched_n_children_max = + I40E_MAX_TRAFFIC_CLASS; + else + cap->nonleaf.sched_n_children_max = + hw->func_caps.num_tx_qp; + cap->nonleaf.sched_sp_n_priorities_max = 1; + cap->nonleaf.sched_wfq_n_children_per_group_max = 0; + cap->nonleaf.sched_wfq_n_groups_max = 0; + cap->nonleaf.sched_wfq_weight_max = 1; + cap->nonleaf.stats_mask = 0; + + return 0; + } + + /* queue node */ + cap->leaf.shaper_private_supported = true; + cap->leaf.shaper_private_dual_rate_supported = false; + cap->leaf.shaper_private_rate_min = 0; + /* 40Gbps -> 5GBps */ + cap->leaf.shaper_private_rate_max = 5000000000ull; + cap->leaf.shaper_shared_n_max = 0; + cap->leaf.cman_head_drop_supported = false; + cap->leaf.cman_wred_context_private_supported = true; + cap->leaf.cman_wred_context_shared_n_max = 0; + cap->leaf.stats_mask = 0; + + return 0; +} + +static int +i40e_node_capabilities_get(struct rte_eth_dev *dev, + uint32_t node_id, + struct rte_tm_node_capabilities *cap, + struct rte_tm_error *error) +{ + struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); + enum i40e_tm_node_type node_type; + struct i40e_tm_node *tm_node; + + if (!cap || !error) + return -EINVAL; + + if (node_id == RTE_TM_NODE_ID_NULL) { + error->type = RTE_TM_ERROR_TYPE_NODE_ID; + error->message = "invalid node id"; + return -EINVAL; + } + + /* check if the node id exists */ + tm_node = i40e_tm_node_search(dev, node_id, &node_type); + if (!tm_node) { + error->type = RTE_TM_ERROR_TYPE_NODE_ID; + error->message = "no such node"; + return -EINVAL; + } + + cap->shaper_private_supported = true; + cap->shaper_private_dual_rate_supported = false; + cap->shaper_private_rate_min = 0; + /* 40Gbps -> 5GBps */ + cap->shaper_private_rate_max = 5000000000ull; + cap->shaper_shared_n_max = 0; + + if (node_type == I40E_TM_NODE_TYPE_QUEUE) { + cap->leaf.cman_head_drop_supported = false; + cap->leaf.cman_wred_context_private_supported = true; + cap->leaf.cman_wred_context_shared_n_max = 0; + } else { + if (node_type == I40E_TM_NODE_TYPE_PORT) + cap->nonleaf.sched_n_children_max = + I40E_MAX_TRAFFIC_CLASS; + else + cap->nonleaf.sched_n_children_max = + hw->func_caps.num_tx_qp; + cap->nonleaf.sched_sp_n_priorities_max = 1; + cap->nonleaf.sched_wfq_n_children_per_group_max = 0; + cap->nonleaf.sched_wfq_n_groups_max = 0; + cap->nonleaf.sched_wfq_weight_max = 1; + } + + cap->stats_mask = 0; + + return 0; +} + +static int +i40e_hierarchy_commit(struct rte_eth_dev *dev, + int clear_on_fail, + struct rte_tm_error *error) +{ + struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + struct i40e_tm_node_list *tc_list = &pf->tm_conf.tc_list; + struct i40e_tm_node_list *queue_list = &pf->tm_conf.queue_list; + struct i40e_tm_node *tm_node; + struct i40e_vsi *vsi; + struct i40e_hw *hw; + struct i40e_aqc_configure_vsi_ets_sla_bw_data tc_bw; + uint64_t bw; + uint8_t tc_map; + int ret; + int i; + + if (!error) + return -EINVAL; + + /* check the setting */ + if (!pf->tm_conf.root) + goto done; + + vsi = pf->main_vsi; + hw = I40E_VSI_TO_HW(vsi); + + /** + * Don't support bandwidth control for port and TCs in parallel. + * If the port has a max bandwidth, the TCs should have none. + */ + /* port */ + if (pf->tm_conf.root->shaper_profile) + bw = pf->tm_conf.root->shaper_profile->profile.peak.rate; + else + bw = 0; + if (bw) { + /* check if any TC has a max bandwidth */ + TAILQ_FOREACH(tm_node, tc_list, node) { + if (tm_node->shaper_profile && + tm_node->shaper_profile->profile.peak.rate) { + error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE; + error->message = "no port and TC max bandwidth" + " in parallel"; + goto fail_clear; + } + } + + /* interpret Bps to 50Mbps */ + bw = bw * 8 / 1000 / 1000 / I40E_QOS_BW_GRANULARITY; + + /* set the max bandwidth */ + ret = i40e_aq_config_vsi_bw_limit(hw, vsi->seid, + (uint16_t)bw, 0, NULL); + if (ret) { + error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE; + error->message = "fail to set port max bandwidth"; + goto fail_clear; + } + + goto done; + } + + /* TC */ + memset(&tc_bw, 0, sizeof(tc_bw)); + tc_bw.tc_valid_bits = vsi->enabled_tc; + tc_map = vsi->enabled_tc; + TAILQ_FOREACH(tm_node, tc_list, node) { + if (!tm_node->reference_count) { + error->type = RTE_TM_ERROR_TYPE_NODE_PARAMS; + error->message = "TC without queue assigned"; + goto fail_clear; + } + + i = 0; + while (i < I40E_MAX_TRAFFIC_CLASS && !(tc_map & BIT_ULL(i))) + i++; + if (i >= I40E_MAX_TRAFFIC_CLASS) { + error->type = RTE_TM_ERROR_TYPE_NODE_PARAMS; + error->message = "cannot find the TC"; + goto fail_clear; + } + tc_map &= ~BIT_ULL(i); + + if (tm_node->shaper_profile) + bw = tm_node->shaper_profile->profile.peak.rate; + else + bw = 0; + if (!bw) + continue; + + /* interpret Bps to 50Mbps */ + bw = bw * 8 / 1000 / 1000 / I40E_QOS_BW_GRANULARITY; + + tc_bw.tc_bw_credits[i] = rte_cpu_to_le_16((uint16_t)bw); + } + + TAILQ_FOREACH(tm_node, queue_list, node) { + if (tm_node->shaper_profile) + bw = tm_node->shaper_profile->profile.peak.rate; + else + bw = 0; + if (bw) { + error->type = RTE_TM_ERROR_TYPE_NODE_PARAMS; + error->message = "not support queue QoS"; + goto fail_clear; + } + } + + ret = i40e_aq_config_vsi_ets_sla_bw_limit(hw, vsi->seid, &tc_bw, NULL); + if (ret) { + error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE; + error->message = "fail to set TC max bandwidth"; + goto fail_clear; + } + +done: + pf->tm_conf.committed = true; + return 0; + +fail_clear: + /* clear all the traffic manager configuration */ + if (clear_on_fail) { + i40e_tm_conf_uninit(dev); + i40e_tm_conf_init(dev); + } + return -EINVAL; +} diff --git a/dpdk/drivers/net/i40e/rte_pmd_i40e.c b/dpdk/drivers/net/i40e/rte_pmd_i40e.c new file mode 100644 index 00000000..f726a9cc --- /dev/null +++ b/dpdk/drivers/net/i40e/rte_pmd_i40e.c @@ -0,0 +1,3020 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include + +#include "base/i40e_prototype.h" +#include "base/i40e_dcb.h" +#include "i40e_ethdev.h" +#include "i40e_pf.h" +#include "i40e_rxtx.h" +#include "rte_pmd_i40e.h" + +int +rte_pmd_i40e_ping_vfs(uint16_t port, uint16_t vf) +{ + struct rte_eth_dev *dev; + struct i40e_pf *pf; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + dev = &rte_eth_devices[port]; + + if (!is_i40e_supported(dev)) + return -ENOTSUP; + + pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + + if (vf >= pf->vf_num || !pf->vfs) { + PMD_DRV_LOG(ERR, "Invalid argument."); + return -EINVAL; + } + + i40e_notify_vf_link_status(dev, &pf->vfs[vf]); + + return 0; +} + +int +rte_pmd_i40e_set_vf_mac_anti_spoof(uint16_t port, uint16_t vf_id, uint8_t on) +{ + struct rte_eth_dev *dev; + struct i40e_pf *pf; + struct i40e_vsi *vsi; + struct i40e_hw *hw; + struct i40e_vsi_context ctxt; + int ret; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + dev = &rte_eth_devices[port]; + + if (!is_i40e_supported(dev)) + return -ENOTSUP; + + pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + + if (vf_id >= pf->vf_num || !pf->vfs) { + PMD_DRV_LOG(ERR, "Invalid argument."); + return -EINVAL; + } + + vsi = pf->vfs[vf_id].vsi; + if (!vsi) { + PMD_DRV_LOG(ERR, "Invalid VSI."); + return -EINVAL; + } + + /* Check if it has been already on or off */ + if (vsi->info.valid_sections & + rte_cpu_to_le_16(I40E_AQ_VSI_PROP_SECURITY_VALID)) { + if (on) { + if ((vsi->info.sec_flags & + I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK) == + I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK) + return 0; /* already on */ + } else { + if ((vsi->info.sec_flags & + I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK) == 0) + return 0; /* already off */ + } + } + + vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SECURITY_VALID); + if (on) + vsi->info.sec_flags |= I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK; + else + vsi->info.sec_flags &= ~I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK; + + memset(&ctxt, 0, sizeof(ctxt)); + rte_memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info)); + ctxt.seid = vsi->seid; + + hw = I40E_VSI_TO_HW(vsi); + ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); + if (ret != I40E_SUCCESS) { + ret = -ENOTSUP; + PMD_DRV_LOG(ERR, "Failed to update VSI params"); + } + + return ret; +} + +static int +i40e_add_rm_all_vlan_filter(struct i40e_vsi *vsi, uint8_t add) +{ + uint32_t j, k; + uint16_t vlan_id; + struct i40e_hw *hw = I40E_VSI_TO_HW(vsi); + struct i40e_aqc_add_remove_vlan_element_data vlan_data = {0}; + int ret; + + for (j = 0; j < I40E_VFTA_SIZE; j++) { + if (!vsi->vfta[j]) + continue; + + for (k = 0; k < I40E_UINT32_BIT_SIZE; k++) { + if (!(vsi->vfta[j] & (1 << k))) + continue; + + vlan_id = j * I40E_UINT32_BIT_SIZE + k; + if (!vlan_id) + continue; + + vlan_data.vlan_tag = rte_cpu_to_le_16(vlan_id); + if (add) + ret = i40e_aq_add_vlan(hw, vsi->seid, + &vlan_data, 1, NULL); + else + ret = i40e_aq_remove_vlan(hw, vsi->seid, + &vlan_data, 1, NULL); + if (ret != I40E_SUCCESS) { + PMD_DRV_LOG(ERR, + "Failed to add/rm vlan filter"); + return ret; + } + } + } + + return I40E_SUCCESS; +} + +int +rte_pmd_i40e_set_vf_vlan_anti_spoof(uint16_t port, uint16_t vf_id, uint8_t on) +{ + struct rte_eth_dev *dev; + struct i40e_pf *pf; + struct i40e_vsi *vsi; + struct i40e_hw *hw; + struct i40e_vsi_context ctxt; + int ret; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + dev = &rte_eth_devices[port]; + + if (!is_i40e_supported(dev)) + return -ENOTSUP; + + pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + + if (vf_id >= pf->vf_num || !pf->vfs) { + PMD_DRV_LOG(ERR, "Invalid argument."); + return -EINVAL; + } + + vsi = pf->vfs[vf_id].vsi; + if (!vsi) { + PMD_DRV_LOG(ERR, "Invalid VSI."); + return -EINVAL; + } + + /* Check if it has been already on or off */ + if (vsi->vlan_anti_spoof_on == on) + return 0; /* already on or off */ + + vsi->vlan_anti_spoof_on = on; + if (!vsi->vlan_filter_on) { + ret = i40e_add_rm_all_vlan_filter(vsi, on); + if (ret) { + PMD_DRV_LOG(ERR, "Failed to add/remove VLAN filters."); + return -ENOTSUP; + } + } + + vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SECURITY_VALID); + if (on) + vsi->info.sec_flags |= I40E_AQ_VSI_SEC_FLAG_ENABLE_VLAN_CHK; + else + vsi->info.sec_flags &= ~I40E_AQ_VSI_SEC_FLAG_ENABLE_VLAN_CHK; + + memset(&ctxt, 0, sizeof(ctxt)); + rte_memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info)); + ctxt.seid = vsi->seid; + + hw = I40E_VSI_TO_HW(vsi); + ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); + if (ret != I40E_SUCCESS) { + ret = -ENOTSUP; + PMD_DRV_LOG(ERR, "Failed to update VSI params"); + } + + return ret; +} + +static int +i40e_vsi_rm_mac_filter(struct i40e_vsi *vsi) +{ + struct i40e_mac_filter *f; + struct i40e_macvlan_filter *mv_f; + int i, vlan_num; + enum rte_mac_filter_type filter_type; + int ret = I40E_SUCCESS; + void *temp; + + /* remove all the MACs */ + TAILQ_FOREACH_SAFE(f, &vsi->mac_list, next, temp) { + vlan_num = vsi->vlan_num; + filter_type = f->mac_info.filter_type; + if (filter_type == RTE_MACVLAN_PERFECT_MATCH || + filter_type == RTE_MACVLAN_HASH_MATCH) { + if (vlan_num == 0) { + PMD_DRV_LOG(ERR, "VLAN number shouldn't be 0"); + return I40E_ERR_PARAM; + } + } else if (filter_type == RTE_MAC_PERFECT_MATCH || + filter_type == RTE_MAC_HASH_MATCH) + vlan_num = 1; + + mv_f = rte_zmalloc("macvlan_data", vlan_num * sizeof(*mv_f), 0); + if (!mv_f) { + PMD_DRV_LOG(ERR, "failed to allocate memory"); + return I40E_ERR_NO_MEMORY; + } + + for (i = 0; i < vlan_num; i++) { + mv_f[i].filter_type = filter_type; + rte_memcpy(&mv_f[i].macaddr, + &f->mac_info.mac_addr, + ETH_ADDR_LEN); + } + if (filter_type == RTE_MACVLAN_PERFECT_MATCH || + filter_type == RTE_MACVLAN_HASH_MATCH) { + ret = i40e_find_all_vlan_for_mac(vsi, mv_f, vlan_num, + &f->mac_info.mac_addr); + if (ret != I40E_SUCCESS) { + rte_free(mv_f); + return ret; + } + } + + ret = i40e_remove_macvlan_filters(vsi, mv_f, vlan_num); + if (ret != I40E_SUCCESS) { + rte_free(mv_f); + return ret; + } + + rte_free(mv_f); + ret = I40E_SUCCESS; + } + + return ret; +} + +static int +i40e_vsi_restore_mac_filter(struct i40e_vsi *vsi) +{ + struct i40e_mac_filter *f; + struct i40e_macvlan_filter *mv_f; + int i, vlan_num = 0; + int ret = I40E_SUCCESS; + void *temp; + + /* restore all the MACs */ + TAILQ_FOREACH_SAFE(f, &vsi->mac_list, next, temp) { + if ((f->mac_info.filter_type == RTE_MACVLAN_PERFECT_MATCH) || + (f->mac_info.filter_type == RTE_MACVLAN_HASH_MATCH)) { + /** + * If vlan_num is 0, that's the first time to add mac, + * set mask for vlan_id 0. + */ + if (vsi->vlan_num == 0) { + i40e_set_vlan_filter(vsi, 0, 1); + vsi->vlan_num = 1; + } + vlan_num = vsi->vlan_num; + } else if ((f->mac_info.filter_type == RTE_MAC_PERFECT_MATCH) || + (f->mac_info.filter_type == RTE_MAC_HASH_MATCH)) + vlan_num = 1; + + mv_f = rte_zmalloc("macvlan_data", vlan_num * sizeof(*mv_f), 0); + if (!mv_f) { + PMD_DRV_LOG(ERR, "failed to allocate memory"); + return I40E_ERR_NO_MEMORY; + } + + for (i = 0; i < vlan_num; i++) { + mv_f[i].filter_type = f->mac_info.filter_type; + rte_memcpy(&mv_f[i].macaddr, + &f->mac_info.mac_addr, + ETH_ADDR_LEN); + } + + if (f->mac_info.filter_type == RTE_MACVLAN_PERFECT_MATCH || + f->mac_info.filter_type == RTE_MACVLAN_HASH_MATCH) { + ret = i40e_find_all_vlan_for_mac(vsi, mv_f, vlan_num, + &f->mac_info.mac_addr); + if (ret != I40E_SUCCESS) { + rte_free(mv_f); + return ret; + } + } + + ret = i40e_add_macvlan_filters(vsi, mv_f, vlan_num); + if (ret != I40E_SUCCESS) { + rte_free(mv_f); + return ret; + } + + rte_free(mv_f); + ret = I40E_SUCCESS; + } + + return ret; +} + +static int +i40e_vsi_set_tx_loopback(struct i40e_vsi *vsi, uint8_t on) +{ + struct i40e_vsi_context ctxt; + struct i40e_hw *hw; + int ret; + + if (!vsi) + return -EINVAL; + + hw = I40E_VSI_TO_HW(vsi); + + /* Use the FW API if FW >= v5.0 */ + if (hw->aq.fw_maj_ver < 5) { + PMD_INIT_LOG(ERR, "FW < v5.0, cannot enable loopback"); + return -ENOTSUP; + } + + /* Check if it has been already on or off */ + if (vsi->info.valid_sections & + rte_cpu_to_le_16(I40E_AQ_VSI_PROP_SWITCH_VALID)) { + if (on) { + if ((vsi->info.switch_id & + I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB) == + I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB) + return 0; /* already on */ + } else { + if ((vsi->info.switch_id & + I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB) == 0) + return 0; /* already off */ + } + } + + /* remove all the MAC and VLAN first */ + ret = i40e_vsi_rm_mac_filter(vsi); + if (ret) { + PMD_INIT_LOG(ERR, "Failed to remove MAC filters."); + return ret; + } + if (vsi->vlan_anti_spoof_on || vsi->vlan_filter_on) { + ret = i40e_add_rm_all_vlan_filter(vsi, 0); + if (ret) { + PMD_INIT_LOG(ERR, "Failed to remove VLAN filters."); + return ret; + } + } + + vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); + if (on) + vsi->info.switch_id |= I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB; + else + vsi->info.switch_id &= ~I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB; + + memset(&ctxt, 0, sizeof(ctxt)); + rte_memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info)); + ctxt.seid = vsi->seid; + + ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); + if (ret != I40E_SUCCESS) { + PMD_DRV_LOG(ERR, "Failed to update VSI params"); + return ret; + } + + /* add all the MAC and VLAN back */ + ret = i40e_vsi_restore_mac_filter(vsi); + if (ret) + return ret; + if (vsi->vlan_anti_spoof_on || vsi->vlan_filter_on) { + ret = i40e_add_rm_all_vlan_filter(vsi, 1); + if (ret) + return ret; + } + + return ret; +} + +int +rte_pmd_i40e_set_tx_loopback(uint16_t port, uint8_t on) +{ + struct rte_eth_dev *dev; + struct i40e_pf *pf; + struct i40e_pf_vf *vf; + struct i40e_vsi *vsi; + uint16_t vf_id; + int ret; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + dev = &rte_eth_devices[port]; + + if (!is_i40e_supported(dev)) + return -ENOTSUP; + + pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + + /* setup PF TX loopback */ + vsi = pf->main_vsi; + ret = i40e_vsi_set_tx_loopback(vsi, on); + if (ret) + return -ENOTSUP; + + /* setup TX loopback for all the VFs */ + if (!pf->vfs) { + /* if no VF, do nothing. */ + return 0; + } + + for (vf_id = 0; vf_id < pf->vf_num; vf_id++) { + vf = &pf->vfs[vf_id]; + vsi = vf->vsi; + + ret = i40e_vsi_set_tx_loopback(vsi, on); + if (ret) + return -ENOTSUP; + } + + return ret; +} + +int +rte_pmd_i40e_set_vf_unicast_promisc(uint16_t port, uint16_t vf_id, uint8_t on) +{ + struct rte_eth_dev *dev; + struct i40e_pf *pf; + struct i40e_vsi *vsi; + struct i40e_hw *hw; + int ret; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + dev = &rte_eth_devices[port]; + + if (!is_i40e_supported(dev)) + return -ENOTSUP; + + pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + + if (vf_id >= pf->vf_num || !pf->vfs) { + PMD_DRV_LOG(ERR, "Invalid argument."); + return -EINVAL; + } + + vsi = pf->vfs[vf_id].vsi; + if (!vsi) { + PMD_DRV_LOG(ERR, "Invalid VSI."); + return -EINVAL; + } + + hw = I40E_VSI_TO_HW(vsi); + + ret = i40e_aq_set_vsi_unicast_promiscuous(hw, vsi->seid, + on, NULL, true); + if (ret != I40E_SUCCESS) { + ret = -ENOTSUP; + PMD_DRV_LOG(ERR, "Failed to set unicast promiscuous mode"); + } + + return ret; +} + +int +rte_pmd_i40e_set_vf_multicast_promisc(uint16_t port, uint16_t vf_id, uint8_t on) +{ + struct rte_eth_dev *dev; + struct i40e_pf *pf; + struct i40e_vsi *vsi; + struct i40e_hw *hw; + int ret; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + dev = &rte_eth_devices[port]; + + if (!is_i40e_supported(dev)) + return -ENOTSUP; + + pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + + if (vf_id >= pf->vf_num || !pf->vfs) { + PMD_DRV_LOG(ERR, "Invalid argument."); + return -EINVAL; + } + + vsi = pf->vfs[vf_id].vsi; + if (!vsi) { + PMD_DRV_LOG(ERR, "Invalid VSI."); + return -EINVAL; + } + + hw = I40E_VSI_TO_HW(vsi); + + ret = i40e_aq_set_vsi_multicast_promiscuous(hw, vsi->seid, + on, NULL); + if (ret != I40E_SUCCESS) { + ret = -ENOTSUP; + PMD_DRV_LOG(ERR, "Failed to set multicast promiscuous mode"); + } + + return ret; +} + +int +rte_pmd_i40e_set_vf_mac_addr(uint16_t port, uint16_t vf_id, + struct ether_addr *mac_addr) +{ + struct i40e_mac_filter *f; + struct rte_eth_dev *dev; + struct i40e_pf_vf *vf; + struct i40e_vsi *vsi; + struct i40e_pf *pf; + void *temp; + + if (i40e_validate_mac_addr((u8 *)mac_addr) != I40E_SUCCESS) + return -EINVAL; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + dev = &rte_eth_devices[port]; + + if (!is_i40e_supported(dev)) + return -ENOTSUP; + + pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + + if (vf_id >= pf->vf_num || !pf->vfs) + return -EINVAL; + + vf = &pf->vfs[vf_id]; + vsi = vf->vsi; + if (!vsi) { + PMD_DRV_LOG(ERR, "Invalid VSI."); + return -EINVAL; + } + + ether_addr_copy(mac_addr, &vf->mac_addr); + + /* Remove all existing mac */ + TAILQ_FOREACH_SAFE(f, &vsi->mac_list, next, temp) + if (i40e_vsi_delete_mac(vsi, &f->mac_info.mac_addr) + != I40E_SUCCESS) + PMD_DRV_LOG(WARNING, "Delete MAC failed"); + + return 0; +} + +/* Set vlan strip on/off for specific VF from host */ +int +rte_pmd_i40e_set_vf_vlan_stripq(uint16_t port, uint16_t vf_id, uint8_t on) +{ + struct rte_eth_dev *dev; + struct i40e_pf *pf; + struct i40e_vsi *vsi; + int ret; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + dev = &rte_eth_devices[port]; + + if (!is_i40e_supported(dev)) + return -ENOTSUP; + + pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + + if (vf_id >= pf->vf_num || !pf->vfs) { + PMD_DRV_LOG(ERR, "Invalid argument."); + return -EINVAL; + } + + vsi = pf->vfs[vf_id].vsi; + + if (!vsi) + return -EINVAL; + + ret = i40e_vsi_config_vlan_stripping(vsi, !!on); + if (ret != I40E_SUCCESS) { + ret = -ENOTSUP; + PMD_DRV_LOG(ERR, "Failed to set VLAN stripping!"); + } + + return ret; +} + +int rte_pmd_i40e_set_vf_vlan_insert(uint16_t port, uint16_t vf_id, + uint16_t vlan_id) +{ + struct rte_eth_dev *dev; + struct i40e_pf *pf; + struct i40e_hw *hw; + struct i40e_vsi *vsi; + struct i40e_vsi_context ctxt; + int ret; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + if (vlan_id > ETHER_MAX_VLAN_ID) { + PMD_DRV_LOG(ERR, "Invalid VLAN ID."); + return -EINVAL; + } + + dev = &rte_eth_devices[port]; + + if (!is_i40e_supported(dev)) + return -ENOTSUP; + + pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + hw = I40E_PF_TO_HW(pf); + + /** + * return -ENODEV if SRIOV not enabled, VF number not configured + * or no queue assigned. + */ + if (!hw->func_caps.sr_iov_1_1 || pf->vf_num == 0 || + pf->vf_nb_qps == 0) + return -ENODEV; + + if (vf_id >= pf->vf_num || !pf->vfs) { + PMD_DRV_LOG(ERR, "Invalid VF ID."); + return -EINVAL; + } + + vsi = pf->vfs[vf_id].vsi; + if (!vsi) { + PMD_DRV_LOG(ERR, "Invalid VSI."); + return -EINVAL; + } + + vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); + vsi->info.pvid = vlan_id; + if (vlan_id > 0) + vsi->info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_INSERT_PVID; + else + vsi->info.port_vlan_flags &= ~I40E_AQ_VSI_PVLAN_INSERT_PVID; + + memset(&ctxt, 0, sizeof(ctxt)); + rte_memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info)); + ctxt.seid = vsi->seid; + + hw = I40E_VSI_TO_HW(vsi); + ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); + if (ret != I40E_SUCCESS) { + ret = -ENOTSUP; + PMD_DRV_LOG(ERR, "Failed to update VSI params"); + } + + return ret; +} + +int rte_pmd_i40e_set_vf_broadcast(uint16_t port, uint16_t vf_id, + uint8_t on) +{ + struct rte_eth_dev *dev; + struct i40e_pf *pf; + struct i40e_vsi *vsi; + struct i40e_hw *hw; + struct i40e_mac_filter_info filter; + struct ether_addr broadcast = { + .addr_bytes = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff} }; + int ret; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + if (on > 1) { + PMD_DRV_LOG(ERR, "on should be 0 or 1."); + return -EINVAL; + } + + dev = &rte_eth_devices[port]; + + if (!is_i40e_supported(dev)) + return -ENOTSUP; + + pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + hw = I40E_PF_TO_HW(pf); + + if (vf_id >= pf->vf_num || !pf->vfs) { + PMD_DRV_LOG(ERR, "Invalid VF ID."); + return -EINVAL; + } + + /** + * return -ENODEV if SRIOV not enabled, VF number not configured + * or no queue assigned. + */ + if (!hw->func_caps.sr_iov_1_1 || pf->vf_num == 0 || + pf->vf_nb_qps == 0) { + PMD_DRV_LOG(ERR, "SRIOV is not enabled or no queue."); + return -ENODEV; + } + + vsi = pf->vfs[vf_id].vsi; + if (!vsi) { + PMD_DRV_LOG(ERR, "Invalid VSI."); + return -EINVAL; + } + + if (on) { + rte_memcpy(&filter.mac_addr, &broadcast, ETHER_ADDR_LEN); + filter.filter_type = RTE_MACVLAN_PERFECT_MATCH; + ret = i40e_vsi_add_mac(vsi, &filter); + } else { + ret = i40e_vsi_delete_mac(vsi, &broadcast); + } + + if (ret != I40E_SUCCESS && ret != I40E_ERR_PARAM) { + ret = -ENOTSUP; + PMD_DRV_LOG(ERR, "Failed to set VSI broadcast"); + } else { + ret = 0; + } + + return ret; +} + +int rte_pmd_i40e_set_vf_vlan_tag(uint16_t port, uint16_t vf_id, uint8_t on) +{ + struct rte_eth_dev *dev; + struct i40e_pf *pf; + struct i40e_hw *hw; + struct i40e_vsi *vsi; + struct i40e_vsi_context ctxt; + int ret; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + if (on > 1) { + PMD_DRV_LOG(ERR, "on should be 0 or 1."); + return -EINVAL; + } + + dev = &rte_eth_devices[port]; + + if (!is_i40e_supported(dev)) + return -ENOTSUP; + + pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + hw = I40E_PF_TO_HW(pf); + + /** + * return -ENODEV if SRIOV not enabled, VF number not configured + * or no queue assigned. + */ + if (!hw->func_caps.sr_iov_1_1 || pf->vf_num == 0 || + pf->vf_nb_qps == 0) { + PMD_DRV_LOG(ERR, "SRIOV is not enabled or no queue."); + return -ENODEV; + } + + if (vf_id >= pf->vf_num || !pf->vfs) { + PMD_DRV_LOG(ERR, "Invalid VF ID."); + return -EINVAL; + } + + vsi = pf->vfs[vf_id].vsi; + if (!vsi) { + PMD_DRV_LOG(ERR, "Invalid VSI."); + return -EINVAL; + } + + vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); + if (on) { + vsi->info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_TAGGED; + vsi->info.port_vlan_flags &= ~I40E_AQ_VSI_PVLAN_MODE_UNTAGGED; + } else { + vsi->info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_UNTAGGED; + vsi->info.port_vlan_flags &= ~I40E_AQ_VSI_PVLAN_MODE_TAGGED; + } + + memset(&ctxt, 0, sizeof(ctxt)); + rte_memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info)); + ctxt.seid = vsi->seid; + + hw = I40E_VSI_TO_HW(vsi); + ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); + if (ret != I40E_SUCCESS) { + ret = -ENOTSUP; + PMD_DRV_LOG(ERR, "Failed to update VSI params"); + } + + return ret; +} + +static int +i40e_vlan_filter_count(struct i40e_vsi *vsi) +{ + uint32_t j, k; + uint16_t vlan_id; + int count = 0; + + for (j = 0; j < I40E_VFTA_SIZE; j++) { + if (!vsi->vfta[j]) + continue; + + for (k = 0; k < I40E_UINT32_BIT_SIZE; k++) { + if (!(vsi->vfta[j] & (1 << k))) + continue; + + vlan_id = j * I40E_UINT32_BIT_SIZE + k; + if (!vlan_id) + continue; + + count++; + } + } + + return count; +} + +int rte_pmd_i40e_set_vf_vlan_filter(uint16_t port, uint16_t vlan_id, + uint64_t vf_mask, uint8_t on) +{ + struct rte_eth_dev *dev; + struct i40e_pf *pf; + struct i40e_hw *hw; + struct i40e_vsi *vsi; + uint16_t vf_idx; + int ret = I40E_SUCCESS; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + dev = &rte_eth_devices[port]; + + if (!is_i40e_supported(dev)) + return -ENOTSUP; + + if (vlan_id > ETHER_MAX_VLAN_ID || !vlan_id) { + PMD_DRV_LOG(ERR, "Invalid VLAN ID."); + return -EINVAL; + } + + if (vf_mask == 0) { + PMD_DRV_LOG(ERR, "No VF."); + return -EINVAL; + } + + if (on > 1) { + PMD_DRV_LOG(ERR, "on is should be 0 or 1."); + return -EINVAL; + } + + pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + hw = I40E_PF_TO_HW(pf); + + /** + * return -ENODEV if SRIOV not enabled, VF number not configured + * or no queue assigned. + */ + if (!hw->func_caps.sr_iov_1_1 || pf->vf_num == 0 || + pf->vf_nb_qps == 0) { + PMD_DRV_LOG(ERR, "SRIOV is not enabled or no queue."); + return -ENODEV; + } + + for (vf_idx = 0; vf_idx < pf->vf_num && ret == I40E_SUCCESS; vf_idx++) { + if (vf_mask & ((uint64_t)(1ULL << vf_idx))) { + vsi = pf->vfs[vf_idx].vsi; + if (on) { + if (!vsi->vlan_filter_on) { + vsi->vlan_filter_on = true; + i40e_aq_set_vsi_vlan_promisc(hw, + vsi->seid, + false, + NULL); + if (!vsi->vlan_anti_spoof_on) + i40e_add_rm_all_vlan_filter( + vsi, true); + } + ret = i40e_vsi_add_vlan(vsi, vlan_id); + } else { + ret = i40e_vsi_delete_vlan(vsi, vlan_id); + + if (!i40e_vlan_filter_count(vsi)) { + vsi->vlan_filter_on = false; + i40e_aq_set_vsi_vlan_promisc(hw, + vsi->seid, + true, + NULL); + } + } + } + } + + if (ret != I40E_SUCCESS) { + ret = -ENOTSUP; + PMD_DRV_LOG(ERR, "Failed to set VF VLAN filter, on = %d", on); + } + + return ret; +} + +int +rte_pmd_i40e_get_vf_stats(uint16_t port, + uint16_t vf_id, + struct rte_eth_stats *stats) +{ + struct rte_eth_dev *dev; + struct i40e_pf *pf; + struct i40e_vsi *vsi; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + dev = &rte_eth_devices[port]; + + if (!is_i40e_supported(dev)) + return -ENOTSUP; + + pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + + if (vf_id >= pf->vf_num || !pf->vfs) { + PMD_DRV_LOG(ERR, "Invalid VF ID."); + return -EINVAL; + } + + vsi = pf->vfs[vf_id].vsi; + if (!vsi) { + PMD_DRV_LOG(ERR, "Invalid VSI."); + return -EINVAL; + } + + i40e_update_vsi_stats(vsi); + + stats->ipackets = vsi->eth_stats.rx_unicast + + vsi->eth_stats.rx_multicast + + vsi->eth_stats.rx_broadcast; + stats->opackets = vsi->eth_stats.tx_unicast + + vsi->eth_stats.tx_multicast + + vsi->eth_stats.tx_broadcast; + stats->ibytes = vsi->eth_stats.rx_bytes; + stats->obytes = vsi->eth_stats.tx_bytes; + stats->ierrors = vsi->eth_stats.rx_discards; + stats->oerrors = vsi->eth_stats.tx_errors + vsi->eth_stats.tx_discards; + + return 0; +} + +int +rte_pmd_i40e_reset_vf_stats(uint16_t port, + uint16_t vf_id) +{ + struct rte_eth_dev *dev; + struct i40e_pf *pf; + struct i40e_vsi *vsi; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + dev = &rte_eth_devices[port]; + + if (!is_i40e_supported(dev)) + return -ENOTSUP; + + pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + + if (vf_id >= pf->vf_num || !pf->vfs) { + PMD_DRV_LOG(ERR, "Invalid VF ID."); + return -EINVAL; + } + + vsi = pf->vfs[vf_id].vsi; + if (!vsi) { + PMD_DRV_LOG(ERR, "Invalid VSI."); + return -EINVAL; + } + + vsi->offset_loaded = false; + i40e_update_vsi_stats(vsi); + + return 0; +} + +int +rte_pmd_i40e_set_vf_max_bw(uint16_t port, uint16_t vf_id, uint32_t bw) +{ + struct rte_eth_dev *dev; + struct i40e_pf *pf; + struct i40e_vsi *vsi; + struct i40e_hw *hw; + int ret = 0; + int i; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + dev = &rte_eth_devices[port]; + + if (!is_i40e_supported(dev)) + return -ENOTSUP; + + pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + + if (vf_id >= pf->vf_num || !pf->vfs) { + PMD_DRV_LOG(ERR, "Invalid VF ID."); + return -EINVAL; + } + + vsi = pf->vfs[vf_id].vsi; + if (!vsi) { + PMD_DRV_LOG(ERR, "Invalid VSI."); + return -EINVAL; + } + + if (bw > I40E_QOS_BW_MAX) { + PMD_DRV_LOG(ERR, "Bandwidth should not be larger than %dMbps.", + I40E_QOS_BW_MAX); + return -EINVAL; + } + + if (bw % I40E_QOS_BW_GRANULARITY) { + PMD_DRV_LOG(ERR, "Bandwidth should be the multiple of %dMbps.", + I40E_QOS_BW_GRANULARITY); + return -EINVAL; + } + + bw /= I40E_QOS_BW_GRANULARITY; + + hw = I40E_VSI_TO_HW(vsi); + + /* No change. */ + if (bw == vsi->bw_info.bw_limit) { + PMD_DRV_LOG(INFO, + "No change for VF max bandwidth. Nothing to do."); + return 0; + } + + /** + * VF bandwidth limitation and TC bandwidth limitation cannot be + * enabled in parallel, quit if TC bandwidth limitation is enabled. + * + * If bw is 0, means disable bandwidth limitation. Then no need to + * check TC bandwidth limitation. + */ + if (bw) { + for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { + if ((vsi->enabled_tc & BIT_ULL(i)) && + vsi->bw_info.bw_ets_credits[i]) + break; + } + if (i != I40E_MAX_TRAFFIC_CLASS) { + PMD_DRV_LOG(ERR, + "TC max bandwidth has been set on this VF," + " please disable it first."); + return -EINVAL; + } + } + + ret = i40e_aq_config_vsi_bw_limit(hw, vsi->seid, (uint16_t)bw, 0, NULL); + if (ret) { + PMD_DRV_LOG(ERR, + "Failed to set VF %d bandwidth, err(%d).", + vf_id, ret); + return -EINVAL; + } + + /* Store the configuration. */ + vsi->bw_info.bw_limit = (uint16_t)bw; + vsi->bw_info.bw_max = 0; + + return 0; +} + +int +rte_pmd_i40e_set_vf_tc_bw_alloc(uint16_t port, uint16_t vf_id, + uint8_t tc_num, uint8_t *bw_weight) +{ + struct rte_eth_dev *dev; + struct i40e_pf *pf; + struct i40e_vsi *vsi; + struct i40e_hw *hw; + struct i40e_aqc_configure_vsi_tc_bw_data tc_bw; + int ret = 0; + int i, j; + uint16_t sum; + bool b_change = false; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + dev = &rte_eth_devices[port]; + + if (!is_i40e_supported(dev)) + return -ENOTSUP; + + pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + + if (vf_id >= pf->vf_num || !pf->vfs) { + PMD_DRV_LOG(ERR, "Invalid VF ID."); + return -EINVAL; + } + + vsi = pf->vfs[vf_id].vsi; + if (!vsi) { + PMD_DRV_LOG(ERR, "Invalid VSI."); + return -EINVAL; + } + + if (tc_num > I40E_MAX_TRAFFIC_CLASS) { + PMD_DRV_LOG(ERR, "TCs should be no more than %d.", + I40E_MAX_TRAFFIC_CLASS); + return -EINVAL; + } + + sum = 0; + for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { + if (vsi->enabled_tc & BIT_ULL(i)) + sum++; + } + if (sum != tc_num) { + PMD_DRV_LOG(ERR, + "Weight should be set for all %d enabled TCs.", + sum); + return -EINVAL; + } + + sum = 0; + for (i = 0; i < tc_num; i++) { + if (!bw_weight[i]) { + PMD_DRV_LOG(ERR, + "The weight should be 1 at least."); + return -EINVAL; + } + sum += bw_weight[i]; + } + if (sum != 100) { + PMD_DRV_LOG(ERR, + "The summary of the TC weight should be 100."); + return -EINVAL; + } + + /** + * Create the configuration for all the TCs. + */ + memset(&tc_bw, 0, sizeof(tc_bw)); + tc_bw.tc_valid_bits = vsi->enabled_tc; + j = 0; + for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { + if (vsi->enabled_tc & BIT_ULL(i)) { + if (bw_weight[j] != + vsi->bw_info.bw_ets_share_credits[i]) + b_change = true; + + tc_bw.tc_bw_credits[i] = bw_weight[j]; + j++; + } + } + + /* No change. */ + if (!b_change) { + PMD_DRV_LOG(INFO, + "No change for TC allocated bandwidth." + " Nothing to do."); + return 0; + } + + hw = I40E_VSI_TO_HW(vsi); + + ret = i40e_aq_config_vsi_tc_bw(hw, vsi->seid, &tc_bw, NULL); + if (ret) { + PMD_DRV_LOG(ERR, + "Failed to set VF %d TC bandwidth weight, err(%d).", + vf_id, ret); + return -EINVAL; + } + + /* Store the configuration. */ + j = 0; + for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { + if (vsi->enabled_tc & BIT_ULL(i)) { + vsi->bw_info.bw_ets_share_credits[i] = bw_weight[j]; + j++; + } + } + + return 0; +} + +int +rte_pmd_i40e_set_vf_tc_max_bw(uint16_t port, uint16_t vf_id, + uint8_t tc_no, uint32_t bw) +{ + struct rte_eth_dev *dev; + struct i40e_pf *pf; + struct i40e_vsi *vsi; + struct i40e_hw *hw; + struct i40e_aqc_configure_vsi_ets_sla_bw_data tc_bw; + int ret = 0; + int i; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + dev = &rte_eth_devices[port]; + + if (!is_i40e_supported(dev)) + return -ENOTSUP; + + pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + + if (vf_id >= pf->vf_num || !pf->vfs) { + PMD_DRV_LOG(ERR, "Invalid VF ID."); + return -EINVAL; + } + + vsi = pf->vfs[vf_id].vsi; + if (!vsi) { + PMD_DRV_LOG(ERR, "Invalid VSI."); + return -EINVAL; + } + + if (bw > I40E_QOS_BW_MAX) { + PMD_DRV_LOG(ERR, "Bandwidth should not be larger than %dMbps.", + I40E_QOS_BW_MAX); + return -EINVAL; + } + + if (bw % I40E_QOS_BW_GRANULARITY) { + PMD_DRV_LOG(ERR, "Bandwidth should be the multiple of %dMbps.", + I40E_QOS_BW_GRANULARITY); + return -EINVAL; + } + + bw /= I40E_QOS_BW_GRANULARITY; + + if (tc_no >= I40E_MAX_TRAFFIC_CLASS) { + PMD_DRV_LOG(ERR, "TC No. should be less than %d.", + I40E_MAX_TRAFFIC_CLASS); + return -EINVAL; + } + + hw = I40E_VSI_TO_HW(vsi); + + if (!(vsi->enabled_tc & BIT_ULL(tc_no))) { + PMD_DRV_LOG(ERR, "VF %d TC %d isn't enabled.", + vf_id, tc_no); + return -EINVAL; + } + + /* No change. */ + if (bw == vsi->bw_info.bw_ets_credits[tc_no]) { + PMD_DRV_LOG(INFO, + "No change for TC max bandwidth. Nothing to do."); + return 0; + } + + /** + * VF bandwidth limitation and TC bandwidth limitation cannot be + * enabled in parallel, disable VF bandwidth limitation if it's + * enabled. + * If bw is 0, means disable bandwidth limitation. Then no need to + * care about VF bandwidth limitation configuration. + */ + if (bw && vsi->bw_info.bw_limit) { + ret = i40e_aq_config_vsi_bw_limit(hw, vsi->seid, 0, 0, NULL); + if (ret) { + PMD_DRV_LOG(ERR, + "Failed to disable VF(%d)" + " bandwidth limitation, err(%d).", + vf_id, ret); + return -EINVAL; + } + + PMD_DRV_LOG(INFO, + "VF max bandwidth is disabled according" + " to TC max bandwidth setting."); + } + + /** + * Get all the TCs' info to create a whole picture. + * Because the incremental change isn't permitted. + */ + memset(&tc_bw, 0, sizeof(tc_bw)); + tc_bw.tc_valid_bits = vsi->enabled_tc; + for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { + if (vsi->enabled_tc & BIT_ULL(i)) { + tc_bw.tc_bw_credits[i] = + rte_cpu_to_le_16( + vsi->bw_info.bw_ets_credits[i]); + } + } + tc_bw.tc_bw_credits[tc_no] = rte_cpu_to_le_16((uint16_t)bw); + + ret = i40e_aq_config_vsi_ets_sla_bw_limit(hw, vsi->seid, &tc_bw, NULL); + if (ret) { + PMD_DRV_LOG(ERR, + "Failed to set VF %d TC %d max bandwidth, err(%d).", + vf_id, tc_no, ret); + return -EINVAL; + } + + /* Store the configuration. */ + vsi->bw_info.bw_ets_credits[tc_no] = (uint16_t)bw; + + return 0; +} + +int +rte_pmd_i40e_set_tc_strict_prio(uint16_t port, uint8_t tc_map) +{ + struct rte_eth_dev *dev; + struct i40e_pf *pf; + struct i40e_vsi *vsi; + struct i40e_veb *veb; + struct i40e_hw *hw; + struct i40e_aqc_configure_switching_comp_ets_data ets_data; + int i; + int ret; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + dev = &rte_eth_devices[port]; + + if (!is_i40e_supported(dev)) + return -ENOTSUP; + + pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + + vsi = pf->main_vsi; + if (!vsi) { + PMD_DRV_LOG(ERR, "Invalid VSI."); + return -EINVAL; + } + + veb = vsi->veb; + if (!veb) { + PMD_DRV_LOG(ERR, "Invalid VEB."); + return -EINVAL; + } + + if ((tc_map & veb->enabled_tc) != tc_map) { + PMD_DRV_LOG(ERR, + "TC bitmap isn't the subset of enabled TCs 0x%x.", + veb->enabled_tc); + return -EINVAL; + } + + if (tc_map == veb->strict_prio_tc) { + PMD_DRV_LOG(INFO, "No change for TC bitmap. Nothing to do."); + return 0; + } + + hw = I40E_VSI_TO_HW(vsi); + + /* Disable DCBx if it's the first time to set strict priority. */ + if (!veb->strict_prio_tc) { + ret = i40e_aq_stop_lldp(hw, true, NULL); + if (ret) + PMD_DRV_LOG(INFO, + "Failed to disable DCBx as it's already" + " disabled."); + else + PMD_DRV_LOG(INFO, + "DCBx is disabled according to strict" + " priority setting."); + } + + memset(&ets_data, 0, sizeof(ets_data)); + ets_data.tc_valid_bits = veb->enabled_tc; + ets_data.seepage = I40E_AQ_ETS_SEEPAGE_EN_MASK; + ets_data.tc_strict_priority_flags = tc_map; + /* Get all TCs' bandwidth. */ + for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { + if (veb->enabled_tc & BIT_ULL(i)) { + /* For rubust, if bandwidth is 0, use 1 instead. */ + if (veb->bw_info.bw_ets_share_credits[i]) + ets_data.tc_bw_share_credits[i] = + veb->bw_info.bw_ets_share_credits[i]; + else + ets_data.tc_bw_share_credits[i] = + I40E_QOS_BW_WEIGHT_MIN; + } + } + + if (!veb->strict_prio_tc) + ret = i40e_aq_config_switch_comp_ets( + hw, veb->uplink_seid, + &ets_data, i40e_aqc_opc_enable_switching_comp_ets, + NULL); + else if (tc_map) + ret = i40e_aq_config_switch_comp_ets( + hw, veb->uplink_seid, + &ets_data, i40e_aqc_opc_modify_switching_comp_ets, + NULL); + else + ret = i40e_aq_config_switch_comp_ets( + hw, veb->uplink_seid, + &ets_data, i40e_aqc_opc_disable_switching_comp_ets, + NULL); + + if (ret) { + PMD_DRV_LOG(ERR, + "Failed to set TCs' strict priority mode." + " err (%d)", ret); + return -EINVAL; + } + + veb->strict_prio_tc = tc_map; + + /* Enable DCBx again, if all the TCs' strict priority disabled. */ + if (!tc_map) { + ret = i40e_aq_start_lldp(hw, NULL); + if (ret) { + PMD_DRV_LOG(ERR, + "Failed to enable DCBx, err(%d).", ret); + return -EINVAL; + } + + PMD_DRV_LOG(INFO, + "DCBx is enabled again according to strict" + " priority setting."); + } + + return ret; +} + +#define I40E_PROFILE_INFO_SIZE sizeof(struct rte_pmd_i40e_profile_info) +#define I40E_MAX_PROFILE_NUM 16 + +static void +i40e_generate_profile_info_sec(char *name, struct i40e_ddp_version *version, + uint32_t track_id, uint8_t *profile_info_sec, + bool add) +{ + struct i40e_profile_section_header *sec = NULL; + struct i40e_profile_info *pinfo; + + sec = (struct i40e_profile_section_header *)profile_info_sec; + sec->tbl_size = 1; + sec->data_end = sizeof(struct i40e_profile_section_header) + + sizeof(struct i40e_profile_info); + sec->section.type = SECTION_TYPE_INFO; + sec->section.offset = sizeof(struct i40e_profile_section_header); + sec->section.size = sizeof(struct i40e_profile_info); + pinfo = (struct i40e_profile_info *)(profile_info_sec + + sec->section.offset); + pinfo->track_id = track_id; + memcpy(pinfo->name, name, I40E_DDP_NAME_SIZE); + memcpy(&pinfo->version, version, sizeof(struct i40e_ddp_version)); + if (add) + pinfo->op = I40E_DDP_ADD_TRACKID; + else + pinfo->op = I40E_DDP_REMOVE_TRACKID; +} + +static enum i40e_status_code +i40e_add_rm_profile_info(struct i40e_hw *hw, uint8_t *profile_info_sec) +{ + enum i40e_status_code status = I40E_SUCCESS; + struct i40e_profile_section_header *sec; + uint32_t track_id; + uint32_t offset = 0; + uint32_t info = 0; + + sec = (struct i40e_profile_section_header *)profile_info_sec; + track_id = ((struct i40e_profile_info *)(profile_info_sec + + sec->section.offset))->track_id; + + status = i40e_aq_write_ddp(hw, (void *)sec, sec->data_end, + track_id, &offset, &info, NULL); + if (status) + PMD_DRV_LOG(ERR, "Failed to add/remove profile info: " + "offset %d, info %d", + offset, info); + + return status; +} + +/* Check if the profile info exists */ +static int +i40e_check_profile_info(uint16_t port, uint8_t *profile_info_sec) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port]; + struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); + uint8_t *buff; + struct rte_pmd_i40e_profile_list *p_list; + struct rte_pmd_i40e_profile_info *pinfo, *p; + uint32_t i; + int ret; + static const uint32_t group_mask = 0x00ff0000; + + pinfo = (struct rte_pmd_i40e_profile_info *)(profile_info_sec + + sizeof(struct i40e_profile_section_header)); + if (pinfo->track_id == 0) { + PMD_DRV_LOG(INFO, "Read-only profile."); + return 0; + } + buff = rte_zmalloc("pinfo_list", + (I40E_PROFILE_INFO_SIZE * I40E_MAX_PROFILE_NUM + 4), + 0); + if (!buff) { + PMD_DRV_LOG(ERR, "failed to allocate memory"); + return -1; + } + + ret = i40e_aq_get_ddp_list( + hw, (void *)buff, + (I40E_PROFILE_INFO_SIZE * I40E_MAX_PROFILE_NUM + 4), + 0, NULL); + if (ret) { + PMD_DRV_LOG(ERR, "Failed to get profile info list."); + rte_free(buff); + return -1; + } + p_list = (struct rte_pmd_i40e_profile_list *)buff; + for (i = 0; i < p_list->p_count; i++) { + p = &p_list->p_info[i]; + if (pinfo->track_id == p->track_id) { + PMD_DRV_LOG(INFO, "Profile exists."); + rte_free(buff); + return 1; + } + } + for (i = 0; i < p_list->p_count; i++) { + p = &p_list->p_info[i]; + if ((p->track_id & group_mask) == 0) { + PMD_DRV_LOG(INFO, "Profile of the group 0 exists."); + rte_free(buff); + return 2; + } + } + for (i = 0; i < p_list->p_count; i++) { + p = &p_list->p_info[i]; + if ((pinfo->track_id & group_mask) != + (p->track_id & group_mask)) { + PMD_DRV_LOG(INFO, "Profile of different group exists."); + rte_free(buff); + return 3; + } + } + + rte_free(buff); + return 0; +} + +int +rte_pmd_i40e_process_ddp_package(uint16_t port, uint8_t *buff, + uint32_t size, + enum rte_pmd_i40e_package_op op) +{ + struct rte_eth_dev *dev; + struct i40e_hw *hw; + struct i40e_package_header *pkg_hdr; + struct i40e_generic_seg_header *profile_seg_hdr; + struct i40e_generic_seg_header *metadata_seg_hdr; + uint32_t track_id; + uint8_t *profile_info_sec; + int is_exist; + enum i40e_status_code status = I40E_SUCCESS; + static const uint32_t type_mask = 0xff000000; + + if (op != RTE_PMD_I40E_PKG_OP_WR_ADD && + op != RTE_PMD_I40E_PKG_OP_WR_ONLY && + op != RTE_PMD_I40E_PKG_OP_WR_DEL) { + PMD_DRV_LOG(ERR, "Operation not supported."); + return -ENOTSUP; + } + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + dev = &rte_eth_devices[port]; + + if (!is_i40e_supported(dev)) + return -ENOTSUP; + + hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); + + if (size < (sizeof(struct i40e_package_header) + + sizeof(struct i40e_metadata_segment) + + sizeof(uint32_t) * 2)) { + PMD_DRV_LOG(ERR, "Buff is invalid."); + return -EINVAL; + } + + pkg_hdr = (struct i40e_package_header *)buff; + + if (!pkg_hdr) { + PMD_DRV_LOG(ERR, "Failed to fill the package structure"); + return -EINVAL; + } + + if (pkg_hdr->segment_count < 2) { + PMD_DRV_LOG(ERR, "Segment_count should be 2 at least."); + return -EINVAL; + } + + i40e_update_customized_info(dev, buff, size); + + /* Find metadata segment */ + metadata_seg_hdr = i40e_find_segment_in_package(SEGMENT_TYPE_METADATA, + pkg_hdr); + if (!metadata_seg_hdr) { + PMD_DRV_LOG(ERR, "Failed to find metadata segment header"); + return -EINVAL; + } + track_id = ((struct i40e_metadata_segment *)metadata_seg_hdr)->track_id; + if (track_id == I40E_DDP_TRACKID_INVALID) { + PMD_DRV_LOG(ERR, "Invalid track_id"); + return -EINVAL; + } + + /* force read-only track_id for type 0 */ + if ((track_id & type_mask) == 0) + track_id = 0; + + /* Find profile segment */ + profile_seg_hdr = i40e_find_segment_in_package(SEGMENT_TYPE_I40E, + pkg_hdr); + if (!profile_seg_hdr) { + PMD_DRV_LOG(ERR, "Failed to find profile segment header"); + return -EINVAL; + } + + profile_info_sec = rte_zmalloc( + "i40e_profile_info", + sizeof(struct i40e_profile_section_header) + + sizeof(struct i40e_profile_info), + 0); + if (!profile_info_sec) { + PMD_DRV_LOG(ERR, "Failed to allocate memory"); + return -EINVAL; + } + + /* Check if the profile already loaded */ + i40e_generate_profile_info_sec( + ((struct i40e_profile_segment *)profile_seg_hdr)->name, + &((struct i40e_profile_segment *)profile_seg_hdr)->version, + track_id, profile_info_sec, + op == RTE_PMD_I40E_PKG_OP_WR_ADD); + is_exist = i40e_check_profile_info(port, profile_info_sec); + if (is_exist < 0) { + PMD_DRV_LOG(ERR, "Failed to check profile."); + rte_free(profile_info_sec); + return -EINVAL; + } + + if (op == RTE_PMD_I40E_PKG_OP_WR_ADD) { + if (is_exist) { + if (is_exist == 1) + PMD_DRV_LOG(ERR, "Profile already exists."); + else if (is_exist == 2) + PMD_DRV_LOG(ERR, "Profile of group 0 already exists."); + else if (is_exist == 3) + PMD_DRV_LOG(ERR, "Profile of different group already exists"); + rte_free(profile_info_sec); + return -EEXIST; + } + } else if (op == RTE_PMD_I40E_PKG_OP_WR_DEL) { + if (is_exist != 1) { + PMD_DRV_LOG(ERR, "Profile does not exist."); + rte_free(profile_info_sec); + return -EACCES; + } + } + + if (op == RTE_PMD_I40E_PKG_OP_WR_DEL) { + status = i40e_rollback_profile( + hw, + (struct i40e_profile_segment *)profile_seg_hdr, + track_id); + if (status) { + PMD_DRV_LOG(ERR, "Failed to write profile for delete."); + rte_free(profile_info_sec); + return status; + } + } else { + status = i40e_write_profile( + hw, + (struct i40e_profile_segment *)profile_seg_hdr, + track_id); + if (status) { + if (op == RTE_PMD_I40E_PKG_OP_WR_ADD) + PMD_DRV_LOG(ERR, "Failed to write profile for add."); + else + PMD_DRV_LOG(ERR, "Failed to write profile."); + rte_free(profile_info_sec); + return status; + } + } + + if (track_id && (op != RTE_PMD_I40E_PKG_OP_WR_ONLY)) { + /* Modify loaded profiles info list */ + status = i40e_add_rm_profile_info(hw, profile_info_sec); + if (status) { + if (op == RTE_PMD_I40E_PKG_OP_WR_ADD) + PMD_DRV_LOG(ERR, "Failed to add profile to info list."); + else + PMD_DRV_LOG(ERR, "Failed to delete profile from info list."); + } + } + + rte_free(profile_info_sec); + return status; +} + +/* Get number of tvl records in the section */ +static unsigned int +i40e_get_tlv_section_size(struct i40e_profile_section_header *sec) +{ + unsigned int i, nb_rec, nb_tlv = 0; + struct i40e_profile_tlv_section_record *tlv; + + if (!sec) + return nb_tlv; + + /* get number of records in the section */ + nb_rec = sec->section.size / + sizeof(struct i40e_profile_tlv_section_record); + for (i = 0; i < nb_rec; ) { + tlv = (struct i40e_profile_tlv_section_record *)&sec[1 + i]; + i += tlv->len; + nb_tlv++; + } + return nb_tlv; +} + +int rte_pmd_i40e_get_ddp_info(uint8_t *pkg_buff, uint32_t pkg_size, + uint8_t *info_buff, uint32_t info_size, + enum rte_pmd_i40e_package_info type) +{ + uint32_t ret_size; + struct i40e_package_header *pkg_hdr; + struct i40e_generic_seg_header *i40e_seg_hdr; + struct i40e_generic_seg_header *note_seg_hdr; + struct i40e_generic_seg_header *metadata_seg_hdr; + + if (!info_buff) { + PMD_DRV_LOG(ERR, "Output info buff is invalid."); + return -EINVAL; + } + + if (!pkg_buff || pkg_size < (sizeof(struct i40e_package_header) + + sizeof(struct i40e_metadata_segment) + + sizeof(uint32_t) * 2)) { + PMD_DRV_LOG(ERR, "Package buff is invalid."); + return -EINVAL; + } + + pkg_hdr = (struct i40e_package_header *)pkg_buff; + if (pkg_hdr->segment_count < 2) { + PMD_DRV_LOG(ERR, "Segment_count should be 2 at least."); + return -EINVAL; + } + + /* Find metadata segment */ + metadata_seg_hdr = i40e_find_segment_in_package(SEGMENT_TYPE_METADATA, + pkg_hdr); + + /* Find global notes segment */ + note_seg_hdr = i40e_find_segment_in_package(SEGMENT_TYPE_NOTES, + pkg_hdr); + + /* Find i40e profile segment */ + i40e_seg_hdr = i40e_find_segment_in_package(SEGMENT_TYPE_I40E, pkg_hdr); + + /* get global header info */ + if (type == RTE_PMD_I40E_PKG_INFO_GLOBAL_HEADER) { + struct rte_pmd_i40e_profile_info *info = + (struct rte_pmd_i40e_profile_info *)info_buff; + + if (info_size < sizeof(struct rte_pmd_i40e_profile_info)) { + PMD_DRV_LOG(ERR, "Output info buff size is invalid."); + return -EINVAL; + } + + if (!metadata_seg_hdr) { + PMD_DRV_LOG(ERR, "Failed to find metadata segment header"); + return -EINVAL; + } + + memset(info, 0, sizeof(struct rte_pmd_i40e_profile_info)); + info->owner = RTE_PMD_I40E_DDP_OWNER_UNKNOWN; + info->track_id = + ((struct i40e_metadata_segment *)metadata_seg_hdr)->track_id; + + memcpy(info->name, + ((struct i40e_metadata_segment *)metadata_seg_hdr)->name, + I40E_DDP_NAME_SIZE); + memcpy(&info->version, + &((struct i40e_metadata_segment *)metadata_seg_hdr)->version, + sizeof(struct i40e_ddp_version)); + return I40E_SUCCESS; + } + + /* get global note size */ + if (type == RTE_PMD_I40E_PKG_INFO_GLOBAL_NOTES_SIZE) { + if (info_size < sizeof(uint32_t)) { + PMD_DRV_LOG(ERR, "Invalid information buffer size"); + return -EINVAL; + } + if (note_seg_hdr == NULL) + ret_size = 0; + else + ret_size = note_seg_hdr->size; + *(uint32_t *)info_buff = ret_size; + return I40E_SUCCESS; + } + + /* get global note */ + if (type == RTE_PMD_I40E_PKG_INFO_GLOBAL_NOTES) { + if (note_seg_hdr == NULL) + return -ENOTSUP; + if (info_size < note_seg_hdr->size) { + PMD_DRV_LOG(ERR, "Information buffer size is too small"); + return -EINVAL; + } + memcpy(info_buff, ¬e_seg_hdr[1], note_seg_hdr->size); + return I40E_SUCCESS; + } + + /* get i40e segment header info */ + if (type == RTE_PMD_I40E_PKG_INFO_HEADER) { + struct rte_pmd_i40e_profile_info *info = + (struct rte_pmd_i40e_profile_info *)info_buff; + + if (info_size < sizeof(struct rte_pmd_i40e_profile_info)) { + PMD_DRV_LOG(ERR, "Output info buff size is invalid."); + return -EINVAL; + } + + if (!metadata_seg_hdr) { + PMD_DRV_LOG(ERR, "Failed to find metadata segment header"); + return -EINVAL; + } + + if (!i40e_seg_hdr) { + PMD_DRV_LOG(ERR, "Failed to find i40e segment header"); + return -EINVAL; + } + + memset(info, 0, sizeof(struct rte_pmd_i40e_profile_info)); + info->owner = RTE_PMD_I40E_DDP_OWNER_UNKNOWN; + info->track_id = + ((struct i40e_metadata_segment *)metadata_seg_hdr)->track_id; + + memcpy(info->name, + ((struct i40e_profile_segment *)i40e_seg_hdr)->name, + I40E_DDP_NAME_SIZE); + memcpy(&info->version, + &((struct i40e_profile_segment *)i40e_seg_hdr)->version, + sizeof(struct i40e_ddp_version)); + return I40E_SUCCESS; + } + + /* get number of devices */ + if (type == RTE_PMD_I40E_PKG_INFO_DEVID_NUM) { + if (info_size < sizeof(uint32_t)) { + PMD_DRV_LOG(ERR, "Invalid information buffer size"); + return -EINVAL; + } + *(uint32_t *)info_buff = + ((struct i40e_profile_segment *)i40e_seg_hdr)->device_table_count; + return I40E_SUCCESS; + } + + /* get list of devices */ + if (type == RTE_PMD_I40E_PKG_INFO_DEVID_LIST) { + uint32_t dev_num; + dev_num = + ((struct i40e_profile_segment *)i40e_seg_hdr)->device_table_count; + if (info_size < sizeof(struct rte_pmd_i40e_ddp_device_id) * dev_num) { + PMD_DRV_LOG(ERR, "Invalid information buffer size"); + return -EINVAL; + } + memcpy(info_buff, + ((struct i40e_profile_segment *)i40e_seg_hdr)->device_table, + sizeof(struct rte_pmd_i40e_ddp_device_id) * dev_num); + return I40E_SUCCESS; + } + + /* get number of protocols */ + if (type == RTE_PMD_I40E_PKG_INFO_PROTOCOL_NUM) { + struct i40e_profile_section_header *proto; + + if (info_size < sizeof(uint32_t)) { + PMD_DRV_LOG(ERR, "Invalid information buffer size"); + return -EINVAL; + } + proto = i40e_find_section_in_profile(SECTION_TYPE_PROTO, + (struct i40e_profile_segment *)i40e_seg_hdr); + *(uint32_t *)info_buff = i40e_get_tlv_section_size(proto); + return I40E_SUCCESS; + } + + /* get list of protocols */ + if (type == RTE_PMD_I40E_PKG_INFO_PROTOCOL_LIST) { + uint32_t i, j, nb_tlv, nb_rec, nb_proto_info; + struct rte_pmd_i40e_proto_info *pinfo; + struct i40e_profile_section_header *proto; + struct i40e_profile_tlv_section_record *tlv; + + pinfo = (struct rte_pmd_i40e_proto_info *)info_buff; + nb_proto_info = info_size / + sizeof(struct rte_pmd_i40e_proto_info); + for (i = 0; i < nb_proto_info; i++) { + pinfo[i].proto_id = RTE_PMD_I40E_PROTO_UNUSED; + memset(pinfo[i].name, 0, RTE_PMD_I40E_DDP_NAME_SIZE); + } + proto = i40e_find_section_in_profile(SECTION_TYPE_PROTO, + (struct i40e_profile_segment *)i40e_seg_hdr); + nb_tlv = i40e_get_tlv_section_size(proto); + if (nb_tlv == 0) + return I40E_SUCCESS; + if (nb_proto_info < nb_tlv) { + PMD_DRV_LOG(ERR, "Invalid information buffer size"); + return -EINVAL; + } + /* get number of records in the section */ + nb_rec = proto->section.size / + sizeof(struct i40e_profile_tlv_section_record); + tlv = (struct i40e_profile_tlv_section_record *)&proto[1]; + for (i = j = 0; i < nb_rec; j++) { + pinfo[j].proto_id = tlv->data[0]; + snprintf(pinfo[j].name, I40E_DDP_NAME_SIZE, "%s", + (const char *)&tlv->data[1]); + i += tlv->len; + tlv = &tlv[tlv->len]; + } + return I40E_SUCCESS; + } + + /* get number of packet classification types */ + if (type == RTE_PMD_I40E_PKG_INFO_PCTYPE_NUM) { + struct i40e_profile_section_header *pctype; + + if (info_size < sizeof(uint32_t)) { + PMD_DRV_LOG(ERR, "Invalid information buffer size"); + return -EINVAL; + } + pctype = i40e_find_section_in_profile(SECTION_TYPE_PCTYPE, + (struct i40e_profile_segment *)i40e_seg_hdr); + *(uint32_t *)info_buff = i40e_get_tlv_section_size(pctype); + return I40E_SUCCESS; + } + + /* get list of packet classification types */ + if (type == RTE_PMD_I40E_PKG_INFO_PCTYPE_LIST) { + uint32_t i, j, nb_tlv, nb_rec, nb_proto_info; + struct rte_pmd_i40e_ptype_info *pinfo; + struct i40e_profile_section_header *pctype; + struct i40e_profile_tlv_section_record *tlv; + + pinfo = (struct rte_pmd_i40e_ptype_info *)info_buff; + nb_proto_info = info_size / + sizeof(struct rte_pmd_i40e_ptype_info); + for (i = 0; i < nb_proto_info; i++) + memset(&pinfo[i], RTE_PMD_I40E_PROTO_UNUSED, + sizeof(struct rte_pmd_i40e_ptype_info)); + pctype = i40e_find_section_in_profile(SECTION_TYPE_PCTYPE, + (struct i40e_profile_segment *)i40e_seg_hdr); + nb_tlv = i40e_get_tlv_section_size(pctype); + if (nb_tlv == 0) + return I40E_SUCCESS; + if (nb_proto_info < nb_tlv) { + PMD_DRV_LOG(ERR, "Invalid information buffer size"); + return -EINVAL; + } + + /* get number of records in the section */ + nb_rec = pctype->section.size / + sizeof(struct i40e_profile_tlv_section_record); + tlv = (struct i40e_profile_tlv_section_record *)&pctype[1]; + for (i = j = 0; i < nb_rec; j++) { + memcpy(&pinfo[j], tlv->data, + sizeof(struct rte_pmd_i40e_ptype_info)); + i += tlv->len; + tlv = &tlv[tlv->len]; + } + return I40E_SUCCESS; + } + + /* get number of packet types */ + if (type == RTE_PMD_I40E_PKG_INFO_PTYPE_NUM) { + struct i40e_profile_section_header *ptype; + + if (info_size < sizeof(uint32_t)) { + PMD_DRV_LOG(ERR, "Invalid information buffer size"); + return -EINVAL; + } + ptype = i40e_find_section_in_profile(SECTION_TYPE_PTYPE, + (struct i40e_profile_segment *)i40e_seg_hdr); + *(uint32_t *)info_buff = i40e_get_tlv_section_size(ptype); + return I40E_SUCCESS; + } + + /* get list of packet types */ + if (type == RTE_PMD_I40E_PKG_INFO_PTYPE_LIST) { + uint32_t i, j, nb_tlv, nb_rec, nb_proto_info; + struct rte_pmd_i40e_ptype_info *pinfo; + struct i40e_profile_section_header *ptype; + struct i40e_profile_tlv_section_record *tlv; + + pinfo = (struct rte_pmd_i40e_ptype_info *)info_buff; + nb_proto_info = info_size / + sizeof(struct rte_pmd_i40e_ptype_info); + for (i = 0; i < nb_proto_info; i++) + memset(&pinfo[i], RTE_PMD_I40E_PROTO_UNUSED, + sizeof(struct rte_pmd_i40e_ptype_info)); + ptype = i40e_find_section_in_profile(SECTION_TYPE_PTYPE, + (struct i40e_profile_segment *)i40e_seg_hdr); + nb_tlv = i40e_get_tlv_section_size(ptype); + if (nb_tlv == 0) + return I40E_SUCCESS; + if (nb_proto_info < nb_tlv) { + PMD_DRV_LOG(ERR, "Invalid information buffer size"); + return -EINVAL; + } + /* get number of records in the section */ + nb_rec = ptype->section.size / + sizeof(struct i40e_profile_tlv_section_record); + for (i = j = 0; i < nb_rec; j++) { + tlv = (struct i40e_profile_tlv_section_record *) + &ptype[1 + i]; + memcpy(&pinfo[j], tlv->data, + sizeof(struct rte_pmd_i40e_ptype_info)); + i += tlv->len; + } + return I40E_SUCCESS; + } + + PMD_DRV_LOG(ERR, "Info type %u is invalid.", type); + return -EINVAL; +} + +int +rte_pmd_i40e_get_ddp_list(uint16_t port, uint8_t *buff, uint32_t size) +{ + struct rte_eth_dev *dev; + struct i40e_hw *hw; + enum i40e_status_code status = I40E_SUCCESS; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + dev = &rte_eth_devices[port]; + + if (!is_i40e_supported(dev)) + return -ENOTSUP; + + if (size < (I40E_PROFILE_INFO_SIZE * I40E_MAX_PROFILE_NUM + 4)) + return -EINVAL; + + hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); + + status = i40e_aq_get_ddp_list(hw, (void *)buff, + size, 0, NULL); + + return status; +} + +static int check_invalid_pkt_type(uint32_t pkt_type) +{ + uint32_t l2, l3, l4, tnl, il2, il3, il4; + + l2 = pkt_type & RTE_PTYPE_L2_MASK; + l3 = pkt_type & RTE_PTYPE_L3_MASK; + l4 = pkt_type & RTE_PTYPE_L4_MASK; + tnl = pkt_type & RTE_PTYPE_TUNNEL_MASK; + il2 = pkt_type & RTE_PTYPE_INNER_L2_MASK; + il3 = pkt_type & RTE_PTYPE_INNER_L3_MASK; + il4 = pkt_type & RTE_PTYPE_INNER_L4_MASK; + + if (l2 && + l2 != RTE_PTYPE_L2_ETHER && + l2 != RTE_PTYPE_L2_ETHER_TIMESYNC && + l2 != RTE_PTYPE_L2_ETHER_ARP && + l2 != RTE_PTYPE_L2_ETHER_LLDP && + l2 != RTE_PTYPE_L2_ETHER_NSH && + l2 != RTE_PTYPE_L2_ETHER_VLAN && + l2 != RTE_PTYPE_L2_ETHER_QINQ) + return -1; + + if (l3 && + l3 != RTE_PTYPE_L3_IPV4 && + l3 != RTE_PTYPE_L3_IPV4_EXT && + l3 != RTE_PTYPE_L3_IPV6 && + l3 != RTE_PTYPE_L3_IPV4_EXT_UNKNOWN && + l3 != RTE_PTYPE_L3_IPV6_EXT && + l3 != RTE_PTYPE_L3_IPV6_EXT_UNKNOWN) + return -1; + + if (l4 && + l4 != RTE_PTYPE_L4_TCP && + l4 != RTE_PTYPE_L4_UDP && + l4 != RTE_PTYPE_L4_FRAG && + l4 != RTE_PTYPE_L4_SCTP && + l4 != RTE_PTYPE_L4_ICMP && + l4 != RTE_PTYPE_L4_NONFRAG) + return -1; + + if (tnl && + tnl != RTE_PTYPE_TUNNEL_IP && + tnl != RTE_PTYPE_TUNNEL_GRENAT && + tnl != RTE_PTYPE_TUNNEL_VXLAN && + tnl != RTE_PTYPE_TUNNEL_NVGRE && + tnl != RTE_PTYPE_TUNNEL_GENEVE && + tnl != RTE_PTYPE_TUNNEL_GRENAT && + tnl != RTE_PTYPE_TUNNEL_GTPC && + tnl != RTE_PTYPE_TUNNEL_GTPU) + return -1; + + if (il2 && + il2 != RTE_PTYPE_INNER_L2_ETHER && + il2 != RTE_PTYPE_INNER_L2_ETHER_VLAN && + il2 != RTE_PTYPE_INNER_L2_ETHER_QINQ) + return -1; + + if (il3 && + il3 != RTE_PTYPE_INNER_L3_IPV4 && + il3 != RTE_PTYPE_INNER_L3_IPV4_EXT && + il3 != RTE_PTYPE_INNER_L3_IPV6 && + il3 != RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN && + il3 != RTE_PTYPE_INNER_L3_IPV6_EXT && + il3 != RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN) + return -1; + + if (il4 && + il4 != RTE_PTYPE_INNER_L4_TCP && + il4 != RTE_PTYPE_INNER_L4_UDP && + il4 != RTE_PTYPE_INNER_L4_FRAG && + il4 != RTE_PTYPE_INNER_L4_SCTP && + il4 != RTE_PTYPE_INNER_L4_ICMP && + il4 != RTE_PTYPE_INNER_L4_NONFRAG) + return -1; + + return 0; +} + +static int check_invalid_ptype_mapping( + struct rte_pmd_i40e_ptype_mapping *mapping_table, + uint16_t count) +{ + int i; + + for (i = 0; i < count; i++) { + uint16_t ptype = mapping_table[i].hw_ptype; + uint32_t pkt_type = mapping_table[i].sw_ptype; + + if (ptype >= I40E_MAX_PKT_TYPE) + return -1; + + if (pkt_type == RTE_PTYPE_UNKNOWN) + continue; + + if (pkt_type & RTE_PMD_I40E_PTYPE_USER_DEFINE_MASK) + continue; + + if (check_invalid_pkt_type(pkt_type)) + return -1; + } + + return 0; +} + +int +rte_pmd_i40e_ptype_mapping_update( + uint16_t port, + struct rte_pmd_i40e_ptype_mapping *mapping_items, + uint16_t count, + uint8_t exclusive) +{ + struct rte_eth_dev *dev; + struct i40e_adapter *ad; + int i; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + dev = &rte_eth_devices[port]; + + if (!is_i40e_supported(dev)) + return -ENOTSUP; + + if (count > I40E_MAX_PKT_TYPE) + return -EINVAL; + + if (check_invalid_ptype_mapping(mapping_items, count)) + return -EINVAL; + + ad = I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); + + if (exclusive) { + for (i = 0; i < I40E_MAX_PKT_TYPE; i++) + ad->ptype_tbl[i] = RTE_PTYPE_UNKNOWN; + } + + for (i = 0; i < count; i++) + ad->ptype_tbl[mapping_items[i].hw_ptype] + = mapping_items[i].sw_ptype; + + return 0; +} + +int rte_pmd_i40e_ptype_mapping_reset(uint16_t port) +{ + struct rte_eth_dev *dev; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + dev = &rte_eth_devices[port]; + + if (!is_i40e_supported(dev)) + return -ENOTSUP; + + i40e_set_default_ptype_table(dev); + + return 0; +} + +int rte_pmd_i40e_ptype_mapping_get( + uint16_t port, + struct rte_pmd_i40e_ptype_mapping *mapping_items, + uint16_t size, + uint16_t *count, + uint8_t valid_only) +{ + struct rte_eth_dev *dev; + struct i40e_adapter *ad; + int n = 0; + uint16_t i; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + dev = &rte_eth_devices[port]; + + if (!is_i40e_supported(dev)) + return -ENOTSUP; + + ad = I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); + + for (i = 0; i < I40E_MAX_PKT_TYPE; i++) { + if (n >= size) + break; + if (valid_only && ad->ptype_tbl[i] == RTE_PTYPE_UNKNOWN) + continue; + mapping_items[n].hw_ptype = i; + mapping_items[n].sw_ptype = ad->ptype_tbl[i]; + n++; + } + + *count = n; + return 0; +} + +int rte_pmd_i40e_ptype_mapping_replace(uint16_t port, + uint32_t target, + uint8_t mask, + uint32_t pkt_type) +{ + struct rte_eth_dev *dev; + struct i40e_adapter *ad; + uint16_t i; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + dev = &rte_eth_devices[port]; + + if (!is_i40e_supported(dev)) + return -ENOTSUP; + + if (!mask && check_invalid_pkt_type(target)) + return -EINVAL; + + if (check_invalid_pkt_type(pkt_type)) + return -EINVAL; + + ad = I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); + + for (i = 0; i < I40E_MAX_PKT_TYPE; i++) { + if (mask) { + if ((target | ad->ptype_tbl[i]) == target && + (target & ad->ptype_tbl[i])) + ad->ptype_tbl[i] = pkt_type; + } else { + if (ad->ptype_tbl[i] == target) + ad->ptype_tbl[i] = pkt_type; + } + } + + return 0; +} + +int +rte_pmd_i40e_add_vf_mac_addr(uint16_t port, uint16_t vf_id, + struct ether_addr *mac_addr) +{ + struct rte_eth_dev *dev; + struct i40e_pf_vf *vf; + struct i40e_vsi *vsi; + struct i40e_pf *pf; + struct i40e_mac_filter_info mac_filter; + int ret; + + if (i40e_validate_mac_addr((u8 *)mac_addr) != I40E_SUCCESS) + return -EINVAL; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + dev = &rte_eth_devices[port]; + + if (!is_i40e_supported(dev)) + return -ENOTSUP; + + pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + + if (vf_id >= pf->vf_num || !pf->vfs) + return -EINVAL; + + vf = &pf->vfs[vf_id]; + vsi = vf->vsi; + if (!vsi) { + PMD_DRV_LOG(ERR, "Invalid VSI."); + return -EINVAL; + } + + mac_filter.filter_type = RTE_MACVLAN_PERFECT_MATCH; + ether_addr_copy(mac_addr, &mac_filter.mac_addr); + ret = i40e_vsi_add_mac(vsi, &mac_filter); + if (ret != I40E_SUCCESS) { + PMD_DRV_LOG(ERR, "Failed to add MAC filter."); + return -1; + } + + return 0; +} + +int rte_pmd_i40e_flow_type_mapping_reset(uint16_t port) +{ + struct rte_eth_dev *dev; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + dev = &rte_eth_devices[port]; + + if (!is_i40e_supported(dev)) + return -ENOTSUP; + + i40e_set_default_pctype_table(dev); + + return 0; +} + +int rte_pmd_i40e_flow_type_mapping_get( + uint16_t port, + struct rte_pmd_i40e_flow_type_mapping *mapping_items) +{ + struct rte_eth_dev *dev; + struct i40e_adapter *ad; + uint16_t i; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + dev = &rte_eth_devices[port]; + + if (!is_i40e_supported(dev)) + return -ENOTSUP; + + ad = I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); + + for (i = 0; i < I40E_FLOW_TYPE_MAX; i++) { + mapping_items[i].flow_type = i; + mapping_items[i].pctype = ad->pctypes_tbl[i]; + } + + return 0; +} + +int +rte_pmd_i40e_flow_type_mapping_update( + uint16_t port, + struct rte_pmd_i40e_flow_type_mapping *mapping_items, + uint16_t count, + uint8_t exclusive) +{ + struct rte_eth_dev *dev; + struct i40e_adapter *ad; + int i; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + dev = &rte_eth_devices[port]; + + if (!is_i40e_supported(dev)) + return -ENOTSUP; + + if (count > I40E_FLOW_TYPE_MAX) + return -EINVAL; + + for (i = 0; i < count; i++) + if (mapping_items[i].flow_type >= I40E_FLOW_TYPE_MAX || + mapping_items[i].flow_type == RTE_ETH_FLOW_UNKNOWN || + (mapping_items[i].pctype & + (1ULL << I40E_FILTER_PCTYPE_INVALID))) + return -EINVAL; + + ad = I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); + + if (exclusive) { + for (i = 0; i < I40E_FLOW_TYPE_MAX; i++) + ad->pctypes_tbl[i] = 0ULL; + ad->flow_types_mask = 0ULL; + } + + for (i = 0; i < count; i++) { + ad->pctypes_tbl[mapping_items[i].flow_type] = + mapping_items[i].pctype; + if (mapping_items[i].pctype) + ad->flow_types_mask |= + (1ULL << mapping_items[i].flow_type); + else + ad->flow_types_mask &= + ~(1ULL << mapping_items[i].flow_type); + } + + for (i = 0, ad->pctypes_mask = 0ULL; i < I40E_FLOW_TYPE_MAX; i++) + ad->pctypes_mask |= ad->pctypes_tbl[i]; + + return 0; +} + +int +rte_pmd_i40e_query_vfid_by_mac(uint16_t port, const struct ether_addr *vf_mac) +{ + struct rte_eth_dev *dev; + struct ether_addr *mac; + struct i40e_pf *pf; + int vf_id; + struct i40e_pf_vf *vf; + uint16_t vf_num; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + dev = &rte_eth_devices[port]; + + if (!is_i40e_supported(dev)) + return -ENOTSUP; + + pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + vf_num = pf->vf_num; + + for (vf_id = 0; vf_id < vf_num; vf_id++) { + vf = &pf->vfs[vf_id]; + mac = &vf->mac_addr; + + if (is_same_ether_addr(mac, vf_mac)) + return vf_id; + } + + return -EINVAL; +} + +static int +i40e_vsi_update_queue_region_mapping(struct i40e_hw *hw, + struct i40e_pf *pf) +{ + uint16_t i; + struct i40e_vsi *vsi = pf->main_vsi; + uint16_t queue_offset, bsf, tc_index; + struct i40e_vsi_context ctxt; + struct i40e_aqc_vsi_properties_data *vsi_info; + struct i40e_queue_regions *region_info = + &pf->queue_region; + int32_t ret = -EINVAL; + + if (!region_info->queue_region_number) { + PMD_INIT_LOG(ERR, "there is no that region id been set before"); + return ret; + } + + memset(&ctxt, 0, sizeof(struct i40e_vsi_context)); + + /* Update Queue Pairs Mapping for currently enabled UPs */ + ctxt.seid = vsi->seid; + ctxt.pf_num = hw->pf_id; + ctxt.vf_num = 0; + ctxt.uplink_seid = vsi->uplink_seid; + ctxt.info = vsi->info; + vsi_info = &ctxt.info; + + memset(vsi_info->tc_mapping, 0, sizeof(uint16_t) * 8); + memset(vsi_info->queue_mapping, 0, sizeof(uint16_t) * 16); + + /* Configure queue region and queue mapping parameters, + * for enabled queue region, allocate queues to this region. + */ + + for (i = 0; i < region_info->queue_region_number; i++) { + tc_index = region_info->region[i].region_id; + bsf = rte_bsf32(region_info->region[i].queue_num); + queue_offset = region_info->region[i].queue_start_index; + vsi_info->tc_mapping[tc_index] = rte_cpu_to_le_16( + (queue_offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | + (bsf << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT)); + } + + /* Associate queue number with VSI, Keep vsi->nb_qps unchanged */ + vsi_info->mapping_flags |= + rte_cpu_to_le_16(I40E_AQ_VSI_QUE_MAP_CONTIG); + vsi_info->queue_mapping[0] = rte_cpu_to_le_16(vsi->base_queue); + vsi_info->valid_sections |= + rte_cpu_to_le_16(I40E_AQ_VSI_PROP_QUEUE_MAP_VALID); + + /* Update the VSI after updating the VSI queue-mapping information */ + ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); + if (ret) { + PMD_DRV_LOG(ERR, "Failed to configure queue region mapping = %d ", + hw->aq.asq_last_status); + return ret; + } + /* update the local VSI info with updated queue map */ + rte_memcpy(&vsi->info.tc_mapping, &ctxt.info.tc_mapping, + sizeof(vsi->info.tc_mapping)); + rte_memcpy(&vsi->info.queue_mapping, + &ctxt.info.queue_mapping, + sizeof(vsi->info.queue_mapping)); + vsi->info.mapping_flags = ctxt.info.mapping_flags; + vsi->info.valid_sections = 0; + + return 0; +} + + +static int +i40e_queue_region_set_region(struct i40e_pf *pf, + struct rte_pmd_i40e_queue_region_conf *conf_ptr) +{ + uint16_t i; + struct i40e_vsi *main_vsi = pf->main_vsi; + struct i40e_queue_regions *info = &pf->queue_region; + int32_t ret = -EINVAL; + + if (!((rte_is_power_of_2(conf_ptr->queue_num)) && + conf_ptr->queue_num <= 64)) { + PMD_DRV_LOG(ERR, "The region sizes should be any of the following values: 1, 2, 4, 8, 16, 32, 64 as long as the " + "total number of queues do not exceed the VSI allocation"); + return ret; + } + + if (conf_ptr->region_id > I40E_REGION_MAX_INDEX) { + PMD_DRV_LOG(ERR, "the queue region max index is 7"); + return ret; + } + + if ((conf_ptr->queue_start_index + conf_ptr->queue_num) + > main_vsi->nb_used_qps) { + PMD_DRV_LOG(ERR, "the queue index exceeds the VSI range"); + return ret; + } + + for (i = 0; i < info->queue_region_number; i++) + if (conf_ptr->region_id == info->region[i].region_id) + break; + + if (i == info->queue_region_number && + i <= I40E_REGION_MAX_INDEX) { + info->region[i].region_id = conf_ptr->region_id; + info->region[i].queue_num = conf_ptr->queue_num; + info->region[i].queue_start_index = + conf_ptr->queue_start_index; + info->queue_region_number++; + } else { + PMD_DRV_LOG(ERR, "queue region number exceeds maxnum 8 or the queue region id has been set before"); + return ret; + } + + return 0; +} + +static int +i40e_queue_region_set_flowtype(struct i40e_pf *pf, + struct rte_pmd_i40e_queue_region_conf *rss_region_conf) +{ + int32_t ret = -EINVAL; + struct i40e_queue_regions *info = &pf->queue_region; + uint16_t i, j; + uint16_t region_index, flowtype_index; + + /* For the pctype or hardware flowtype of packet, + * the specific index for each type has been defined + * in file i40e_type.h as enum i40e_filter_pctype. + */ + + if (rss_region_conf->region_id > I40E_PFQF_HREGION_MAX_INDEX) { + PMD_DRV_LOG(ERR, "the queue region max index is 7"); + return ret; + } + + if (rss_region_conf->hw_flowtype >= I40E_FILTER_PCTYPE_MAX) { + PMD_DRV_LOG(ERR, "the hw_flowtype or PCTYPE max index is 63"); + return ret; + } + + + for (i = 0; i < info->queue_region_number; i++) + if (rss_region_conf->region_id == info->region[i].region_id) + break; + + if (i == info->queue_region_number) { + PMD_DRV_LOG(ERR, "that region id has not been set before"); + ret = -EINVAL; + return ret; + } + region_index = i; + + for (i = 0; i < info->queue_region_number; i++) { + for (j = 0; j < info->region[i].flowtype_num; j++) { + if (rss_region_conf->hw_flowtype == + info->region[i].hw_flowtype[j]) { + PMD_DRV_LOG(ERR, "that hw_flowtype has been set before"); + return 0; + } + } + } + + flowtype_index = info->region[region_index].flowtype_num; + info->region[region_index].hw_flowtype[flowtype_index] = + rss_region_conf->hw_flowtype; + info->region[region_index].flowtype_num++; + + return 0; +} + +static void +i40e_queue_region_pf_flowtype_conf(struct i40e_hw *hw, + struct i40e_pf *pf) +{ + uint8_t hw_flowtype; + uint32_t pfqf_hregion; + uint16_t i, j, index; + struct i40e_queue_regions *info = &pf->queue_region; + + /* For the pctype or hardware flowtype of packet, + * the specific index for each type has been defined + * in file i40e_type.h as enum i40e_filter_pctype. + */ + + for (i = 0; i < info->queue_region_number; i++) { + for (j = 0; j < info->region[i].flowtype_num; j++) { + hw_flowtype = info->region[i].hw_flowtype[j]; + index = hw_flowtype >> 3; + pfqf_hregion = + i40e_read_rx_ctl(hw, I40E_PFQF_HREGION(index)); + + if ((hw_flowtype & 0x7) == 0) { + pfqf_hregion |= info->region[i].region_id << + I40E_PFQF_HREGION_REGION_0_SHIFT; + pfqf_hregion |= 1 << + I40E_PFQF_HREGION_OVERRIDE_ENA_0_SHIFT; + } else if ((hw_flowtype & 0x7) == 1) { + pfqf_hregion |= info->region[i].region_id << + I40E_PFQF_HREGION_REGION_1_SHIFT; + pfqf_hregion |= 1 << + I40E_PFQF_HREGION_OVERRIDE_ENA_1_SHIFT; + } else if ((hw_flowtype & 0x7) == 2) { + pfqf_hregion |= info->region[i].region_id << + I40E_PFQF_HREGION_REGION_2_SHIFT; + pfqf_hregion |= 1 << + I40E_PFQF_HREGION_OVERRIDE_ENA_2_SHIFT; + } else if ((hw_flowtype & 0x7) == 3) { + pfqf_hregion |= info->region[i].region_id << + I40E_PFQF_HREGION_REGION_3_SHIFT; + pfqf_hregion |= 1 << + I40E_PFQF_HREGION_OVERRIDE_ENA_3_SHIFT; + } else if ((hw_flowtype & 0x7) == 4) { + pfqf_hregion |= info->region[i].region_id << + I40E_PFQF_HREGION_REGION_4_SHIFT; + pfqf_hregion |= 1 << + I40E_PFQF_HREGION_OVERRIDE_ENA_4_SHIFT; + } else if ((hw_flowtype & 0x7) == 5) { + pfqf_hregion |= info->region[i].region_id << + I40E_PFQF_HREGION_REGION_5_SHIFT; + pfqf_hregion |= 1 << + I40E_PFQF_HREGION_OVERRIDE_ENA_5_SHIFT; + } else if ((hw_flowtype & 0x7) == 6) { + pfqf_hregion |= info->region[i].region_id << + I40E_PFQF_HREGION_REGION_6_SHIFT; + pfqf_hregion |= 1 << + I40E_PFQF_HREGION_OVERRIDE_ENA_6_SHIFT; + } else { + pfqf_hregion |= info->region[i].region_id << + I40E_PFQF_HREGION_REGION_7_SHIFT; + pfqf_hregion |= 1 << + I40E_PFQF_HREGION_OVERRIDE_ENA_7_SHIFT; + } + + i40e_write_rx_ctl(hw, I40E_PFQF_HREGION(index), + pfqf_hregion); + } + } +} + +static int +i40e_queue_region_set_user_priority(struct i40e_pf *pf, + struct rte_pmd_i40e_queue_region_conf *rss_region_conf) +{ + struct i40e_queue_regions *info = &pf->queue_region; + int32_t ret = -EINVAL; + uint16_t i, j, region_index; + + if (rss_region_conf->user_priority >= I40E_MAX_USER_PRIORITY) { + PMD_DRV_LOG(ERR, "the queue region max index is 7"); + return ret; + } + + if (rss_region_conf->region_id > I40E_REGION_MAX_INDEX) { + PMD_DRV_LOG(ERR, "the region_id max index is 7"); + return ret; + } + + for (i = 0; i < info->queue_region_number; i++) + if (rss_region_conf->region_id == info->region[i].region_id) + break; + + if (i == info->queue_region_number) { + PMD_DRV_LOG(ERR, "that region id has not been set before"); + ret = -EINVAL; + return ret; + } + + region_index = i; + + for (i = 0; i < info->queue_region_number; i++) { + for (j = 0; j < info->region[i].user_priority_num; j++) { + if (info->region[i].user_priority[j] == + rss_region_conf->user_priority) { + PMD_DRV_LOG(ERR, "that user priority has been set before"); + return 0; + } + } + } + + j = info->region[region_index].user_priority_num; + info->region[region_index].user_priority[j] = + rss_region_conf->user_priority; + info->region[region_index].user_priority_num++; + + return 0; +} + +static int +i40e_queue_region_dcb_configure(struct i40e_hw *hw, + struct i40e_pf *pf) +{ + struct i40e_dcbx_config dcb_cfg_local; + struct i40e_dcbx_config *dcb_cfg; + struct i40e_queue_regions *info = &pf->queue_region; + struct i40e_dcbx_config *old_cfg = &hw->local_dcbx_config; + int32_t ret = -EINVAL; + uint16_t i, j, prio_index, region_index; + uint8_t tc_map, tc_bw, bw_lf; + + if (!info->queue_region_number) { + PMD_DRV_LOG(ERR, "No queue region been set before"); + return ret; + } + + dcb_cfg = &dcb_cfg_local; + memset(dcb_cfg, 0, sizeof(struct i40e_dcbx_config)); + + /* assume each tc has the same bw */ + tc_bw = I40E_MAX_PERCENT / info->queue_region_number; + for (i = 0; i < info->queue_region_number; i++) + dcb_cfg->etscfg.tcbwtable[i] = tc_bw; + /* to ensure the sum of tcbw is equal to 100 */ + bw_lf = I40E_MAX_PERCENT % info->queue_region_number; + for (i = 0; i < bw_lf; i++) + dcb_cfg->etscfg.tcbwtable[i]++; + + /* assume each tc has the same Transmission Selection Algorithm */ + for (i = 0; i < info->queue_region_number; i++) + dcb_cfg->etscfg.tsatable[i] = I40E_IEEE_TSA_ETS; + + for (i = 0; i < info->queue_region_number; i++) { + for (j = 0; j < info->region[i].user_priority_num; j++) { + prio_index = info->region[i].user_priority[j]; + region_index = info->region[i].region_id; + dcb_cfg->etscfg.prioritytable[prio_index] = + region_index; + } + } + + /* FW needs one App to configure HW */ + dcb_cfg->numapps = I40E_DEFAULT_DCB_APP_NUM; + dcb_cfg->app[0].selector = I40E_APP_SEL_ETHTYPE; + dcb_cfg->app[0].priority = I40E_DEFAULT_DCB_APP_PRIO; + dcb_cfg->app[0].protocolid = I40E_APP_PROTOID_FCOE; + + tc_map = RTE_LEN2MASK(info->queue_region_number, uint8_t); + + dcb_cfg->pfc.willing = 0; + dcb_cfg->pfc.pfccap = I40E_MAX_TRAFFIC_CLASS; + dcb_cfg->pfc.pfcenable = tc_map; + + /* Copy the new config to the current config */ + *old_cfg = *dcb_cfg; + old_cfg->etsrec = old_cfg->etscfg; + ret = i40e_set_dcb_config(hw); + + if (ret) { + PMD_DRV_LOG(ERR, "Set queue region DCB Config failed, err %s aq_err %s", + i40e_stat_str(hw, ret), + i40e_aq_str(hw, hw->aq.asq_last_status)); + return ret; + } + + return 0; +} + +int +i40e_flush_queue_region_all_conf(struct rte_eth_dev *dev, + struct i40e_hw *hw, struct i40e_pf *pf, uint16_t on) +{ + int32_t ret = -EINVAL; + struct i40e_queue_regions *info = &pf->queue_region; + struct i40e_vsi *main_vsi = pf->main_vsi; + + if (on) { + i40e_queue_region_pf_flowtype_conf(hw, pf); + + ret = i40e_vsi_update_queue_region_mapping(hw, pf); + if (ret != I40E_SUCCESS) { + PMD_DRV_LOG(INFO, "Failed to flush queue region mapping."); + return ret; + } + + ret = i40e_queue_region_dcb_configure(hw, pf); + if (ret != I40E_SUCCESS) { + PMD_DRV_LOG(INFO, "Failed to flush dcb."); + return ret; + } + + return 0; + } + + if (info->queue_region_number) { + info->queue_region_number = 1; + info->region[0].queue_num = main_vsi->nb_used_qps; + info->region[0].queue_start_index = 0; + + ret = i40e_vsi_update_queue_region_mapping(hw, pf); + if (ret != I40E_SUCCESS) + PMD_DRV_LOG(INFO, "Failed to flush queue region mapping."); + + ret = i40e_dcb_init_configure(dev, TRUE); + if (ret != I40E_SUCCESS) { + PMD_DRV_LOG(INFO, "Failed to flush dcb."); + pf->flags &= ~I40E_FLAG_DCB; + } + + i40e_init_queue_region_conf(dev); + } + return 0; +} + +static int +i40e_queue_region_pf_check_rss(struct i40e_pf *pf) +{ + struct i40e_hw *hw = I40E_PF_TO_HW(pf); + uint64_t hena; + + hena = (uint64_t)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)); + hena |= ((uint64_t)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1))) << 32; + + if (!hena) + return -ENOTSUP; + + return 0; +} + +static int +i40e_queue_region_get_all_info(struct i40e_pf *pf, + struct i40e_queue_regions *regions_ptr) +{ + struct i40e_queue_regions *info = &pf->queue_region; + + rte_memcpy(regions_ptr, info, + sizeof(struct i40e_queue_regions)); + + return 0; +} + +int rte_pmd_i40e_rss_queue_region_conf(uint16_t port_id, + enum rte_pmd_i40e_queue_region_op op_type, void *arg) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); + int32_t ret; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); + + if (!is_i40e_supported(dev)) + return -ENOTSUP; + + if (!(!i40e_queue_region_pf_check_rss(pf))) + return -ENOTSUP; + + /* This queue region feature only support pf by now. It should + * be called after dev_start, and will be clear after dev_stop. + * "RTE_PMD_I40E_RSS_QUEUE_REGION_ALL_FLUSH_ON" + * is just an enable function which server for other configuration, + * it is for all configuration about queue region from up layer, + * at first will only keep in DPDK softwarestored in driver, + * only after "FLUSH_ON", it commit all configuration to HW. + * Because PMD had to set hardware configuration at a time, so + * it will record all up layer command at first. + * "RTE_PMD_I40E_RSS_QUEUE_REGION_ALL_FLUSH_OFF" is + * just clean all configuration about queue region just now, + * and restore all to DPDK i40e driver default + * config when start up. + */ + + switch (op_type) { + case RTE_PMD_I40E_RSS_QUEUE_REGION_SET: + ret = i40e_queue_region_set_region(pf, + (struct rte_pmd_i40e_queue_region_conf *)arg); + break; + case RTE_PMD_I40E_RSS_QUEUE_REGION_FLOWTYPE_SET: + ret = i40e_queue_region_set_flowtype(pf, + (struct rte_pmd_i40e_queue_region_conf *)arg); + break; + case RTE_PMD_I40E_RSS_QUEUE_REGION_USER_PRIORITY_SET: + ret = i40e_queue_region_set_user_priority(pf, + (struct rte_pmd_i40e_queue_region_conf *)arg); + break; + case RTE_PMD_I40E_RSS_QUEUE_REGION_ALL_FLUSH_ON: + ret = i40e_flush_queue_region_all_conf(dev, hw, pf, 1); + break; + case RTE_PMD_I40E_RSS_QUEUE_REGION_ALL_FLUSH_OFF: + ret = i40e_flush_queue_region_all_conf(dev, hw, pf, 0); + break; + case RTE_PMD_I40E_RSS_QUEUE_REGION_INFO_GET: + ret = i40e_queue_region_get_all_info(pf, + (struct i40e_queue_regions *)arg); + break; + default: + PMD_DRV_LOG(WARNING, "op type (%d) not supported", + op_type); + ret = -EINVAL; + } + + I40E_WRITE_FLUSH(hw); + + return ret; +} + +int rte_pmd_i40e_flow_add_del_packet_template( + uint16_t port, + const struct rte_pmd_i40e_pkt_template_conf *conf, + uint8_t add) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port]; + struct i40e_fdir_filter_conf filter_conf; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + if (!is_i40e_supported(dev)) + return -ENOTSUP; + + memset(&filter_conf, 0, sizeof(filter_conf)); + filter_conf.soft_id = conf->soft_id; + filter_conf.input.flow.raw_flow.pctype = conf->input.pctype; + filter_conf.input.flow.raw_flow.packet = conf->input.packet; + filter_conf.input.flow.raw_flow.length = conf->input.length; + filter_conf.input.flow_ext.pkt_template = true; + + filter_conf.action.rx_queue = conf->action.rx_queue; + filter_conf.action.behavior = + (enum i40e_fdir_behavior)conf->action.behavior; + filter_conf.action.report_status = + (enum i40e_fdir_status)conf->action.report_status; + filter_conf.action.flex_off = conf->action.flex_off; + + return i40e_flow_add_del_fdir_filter(dev, &filter_conf, add); +} diff --git a/dpdk/drivers/net/i40e/rte_pmd_i40e.h b/dpdk/drivers/net/i40e/rte_pmd_i40e.h new file mode 100644 index 00000000..49f4427a --- /dev/null +++ b/dpdk/drivers/net/i40e/rte_pmd_i40e.h @@ -0,0 +1,936 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _PMD_I40E_H_ +#define _PMD_I40E_H_ + +/** + * @file rte_pmd_i40e.h + * + * i40e PMD specific functions. + * + * @b EXPERIMENTAL: this API may change, or be removed, without prior notice + * + */ + +#include + +/** + * Response sent back to i40e driver from user app after callback + */ +enum rte_pmd_i40e_mb_event_rsp { + RTE_PMD_I40E_MB_EVENT_NOOP_ACK, /**< skip mbox request and ACK */ + RTE_PMD_I40E_MB_EVENT_NOOP_NACK, /**< skip mbox request and NACK */ + RTE_PMD_I40E_MB_EVENT_PROCEED, /**< proceed with mbox request */ + RTE_PMD_I40E_MB_EVENT_MAX /**< max value of this enum */ +}; + +/** + * Data sent to the user application when the callback is executed. + */ +struct rte_pmd_i40e_mb_event_param { + uint16_t vfid; /**< Virtual Function number */ + uint16_t msg_type; /**< VF to PF message type, see virtchnl_ops */ + uint16_t retval; /**< return value */ + void *msg; /**< pointer to message */ + uint16_t msglen; /**< length of the message */ +}; + +/** + * Option of package processing. + */ +enum rte_pmd_i40e_package_op { + RTE_PMD_I40E_PKG_OP_UNDEFINED = 0, + RTE_PMD_I40E_PKG_OP_WR_ADD, /**< load package and add to info list */ + RTE_PMD_I40E_PKG_OP_WR_DEL, /**< load package and delete from info list */ + RTE_PMD_I40E_PKG_OP_WR_ONLY, /**< load package without modifying info list */ + RTE_PMD_I40E_PKG_OP_MAX = 32 +}; + +/** + * Types of package information. + */ +enum rte_pmd_i40e_package_info { + RTE_PMD_I40E_PKG_INFO_UNDEFINED = 0, + RTE_PMD_I40E_PKG_INFO_GLOBAL_HEADER, + RTE_PMD_I40E_PKG_INFO_GLOBAL_NOTES_SIZE, + RTE_PMD_I40E_PKG_INFO_GLOBAL_NOTES, + RTE_PMD_I40E_PKG_INFO_GLOBAL_MAX = 1024, + RTE_PMD_I40E_PKG_INFO_HEADER, + RTE_PMD_I40E_PKG_INFO_DEVID_NUM, + RTE_PMD_I40E_PKG_INFO_DEVID_LIST, + RTE_PMD_I40E_PKG_INFO_PROTOCOL_NUM, + RTE_PMD_I40E_PKG_INFO_PROTOCOL_LIST, + RTE_PMD_I40E_PKG_INFO_PCTYPE_NUM, + RTE_PMD_I40E_PKG_INFO_PCTYPE_LIST, + RTE_PMD_I40E_PKG_INFO_PTYPE_NUM, + RTE_PMD_I40E_PKG_INFO_PTYPE_LIST, + RTE_PMD_I40E_PKG_INFO_MAX = (int)0xFFFFFFFF +}; + +/** + * Option types of queue region. + */ +enum rte_pmd_i40e_queue_region_op { + RTE_PMD_I40E_RSS_QUEUE_REGION_UNDEFINED, + /** add queue region set */ + RTE_PMD_I40E_RSS_QUEUE_REGION_SET, + /** add PF region pctype set */ + RTE_PMD_I40E_RSS_QUEUE_REGION_FLOWTYPE_SET, + /** add queue region user priority set */ + RTE_PMD_I40E_RSS_QUEUE_REGION_USER_PRIORITY_SET, + /** + * ALL configuration about queue region from up layer + * at first will only keep in DPDK software stored in driver, + * only after " FLUSH_ON ", it commit all configuration to HW. + * Because PMD had to set hardware configuration at a time, so + * it will record all up layer command at first. + */ + RTE_PMD_I40E_RSS_QUEUE_REGION_ALL_FLUSH_ON, + /** + * "FLUSH_OFF " is just clean all configuration about queue + * region just now, and restore all to DPDK i40e driver default + * config when start up. + */ + RTE_PMD_I40E_RSS_QUEUE_REGION_ALL_FLUSH_OFF, + RTE_PMD_I40E_RSS_QUEUE_REGION_INFO_GET, + RTE_PMD_I40E_RSS_QUEUE_REGION_OP_MAX +}; + +#define RTE_PMD_I40E_DDP_NAME_SIZE 32 +#define RTE_PMD_I40E_PCTYPE_MAX 64 +#define RTE_PMD_I40E_REGION_MAX_NUM 8 +#define RTE_PMD_I40E_MAX_USER_PRIORITY 8 + +/** + * Version for dynamic device personalization. + * Version in "major.minor.update.draft" format. + */ +struct rte_pmd_i40e_ddp_version { + uint8_t major; + uint8_t minor; + uint8_t update; + uint8_t draft; +}; + +/** + * Device ID for dynamic device personalization. + */ +struct rte_pmd_i40e_ddp_device_id { + uint32_t vendor_dev_id; + uint32_t sub_vendor_dev_id; +}; + +/** + * Profile information in profile info list. + */ +struct rte_pmd_i40e_profile_info { + uint32_t track_id; + struct rte_pmd_i40e_ddp_version version; + uint8_t owner; + uint8_t reserved[7]; + uint8_t name[RTE_PMD_I40E_DDP_NAME_SIZE]; +}; + +#define RTE_PMD_I40E_DDP_OWNER_UNKNOWN 0xFF + +/** + * Profile information list returned from HW. + */ +struct rte_pmd_i40e_profile_list { + uint32_t p_count; + struct rte_pmd_i40e_profile_info p_info[1]; +}; + +#define RTE_PMD_I40E_PROTO_NUM 6 +#define RTE_PMD_I40E_PROTO_UNUSED 0xFF + +/** + * Protocols information stored in profile + */ +struct rte_pmd_i40e_proto_info { + uint8_t proto_id; + char name[RTE_PMD_I40E_DDP_NAME_SIZE]; +}; + +/** + * Packet classification/ packet type information stored in profile + */ +struct rte_pmd_i40e_ptype_info { + uint8_t ptype_id; + uint8_t protocols[RTE_PMD_I40E_PROTO_NUM]; +}; + +/** + * ptype mapping table only accept RTE_PTYPE_XXX or "user defined" ptype. + * A ptype with MSB set will be regarded as a user defined ptype. + * Below macro help to create a user defined ptype. + */ +#define RTE_PMD_I40E_PTYPE_USER_DEFINE_MASK 0x80000000 + +struct rte_pmd_i40e_ptype_mapping { + uint16_t hw_ptype; /**< hardware defined packet type*/ + uint32_t sw_ptype; /**< software defined packet type */ +}; + +/** + * Queue region related information. + */ +struct rte_pmd_i40e_queue_region_conf { + /** the region id for this configuration */ + uint8_t region_id; + /** the pctype or hardware flowtype of packet, + * the specific index for each type has been defined + * in file i40e_type.h as enum i40e_filter_pctype. + */ + uint8_t hw_flowtype; + /** the start queue index for this region */ + uint8_t queue_start_index; + /** the total queue number of this queue region */ + uint8_t queue_num; + /** the packet's user priority for this region */ + uint8_t user_priority; +}; + +/* queue region info */ +struct rte_pmd_i40e_queue_region_info { + /** the region id for this configuration */ + uint8_t region_id; + /** the start queue index for this region */ + uint8_t queue_start_index; + /** the total queue number of this queue region */ + uint8_t queue_num; + /** the total number of user priority for this region */ + uint8_t user_priority_num; + /** the packet's user priority for this region */ + uint8_t user_priority[RTE_PMD_I40E_MAX_USER_PRIORITY]; + /** the total number of flowtype for this region */ + uint8_t flowtype_num; + /** + * the pctype or hardware flowtype of packet, + * the specific index for each type has been defined + * in file i40e_type.h as enum i40e_filter_pctype. + */ + uint8_t hw_flowtype[RTE_PMD_I40E_PCTYPE_MAX]; +}; + +struct rte_pmd_i40e_queue_regions { + /** the total number of queue region for this port */ + uint16_t queue_region_number; + struct rte_pmd_i40e_queue_region_info + region[RTE_PMD_I40E_REGION_MAX_NUM]; +}; + +/** + * Behavior will be taken if raw packet template is matched. + */ +enum rte_pmd_i40e_pkt_template_behavior { + RTE_PMD_I40E_PKT_TEMPLATE_ACCEPT, + RTE_PMD_I40E_PKT_TEMPLATE_REJECT, + RTE_PMD_I40E_PKT_TEMPLATE_PASSTHRU, +}; + +/** + * Flow director report status + * It defines what will be reported if raw packet template is matched. + */ +enum rte_pmd_i40e_pkt_template_status { + /** report nothing */ + RTE_PMD_I40E_PKT_TEMPLATE_NO_REPORT_STATUS, + /** only report FD ID */ + RTE_PMD_I40E_PKT_TEMPLATE_REPORT_ID, + /** report FD ID and 4 flex bytes */ + RTE_PMD_I40E_PKT_TEMPLATE_REPORT_ID_FLEX_4, + /** report 8 flex bytes */ + RTE_PMD_I40E_PKT_TEMPLATE_REPORT_FLEX_8, +}; + +/** + * A structure used to define an action when raw packet template is matched. + */ +struct rte_pmd_i40e_pkt_template_action { + /** queue assigned to if raw packet template match */ + uint16_t rx_queue; + /** behavior will be taken */ + enum rte_pmd_i40e_pkt_template_behavior behavior; + /** status report option */ + enum rte_pmd_i40e_pkt_template_status report_status; + /** + * If report_status is RTE_PMD_I40E_PKT_TEMPLATE_REPORT_ID_FLEX_4 or + * RTE_PMD_I40E_PKT_TEMPLATE_REPORT_FLEX_8, flex_off specifies + * where the reported flex bytes start from in flexible payload. + */ + uint8_t flex_off; +}; + +/** + * A structure used to define the input for raw packet template. + */ +struct rte_pmd_i40e_pkt_template_input { + /** the pctype used for raw packet template */ + uint16_t pctype; + /** the buffer conatining raw packet template */ + void *packet; + /** the length of buffer with raw packet template */ + uint32_t length; +}; + +/** + * A structure used to define the configuration parameters + * for raw packet template. + */ +struct rte_pmd_i40e_pkt_template_conf { + /** the input for raw packet template. */ + struct rte_pmd_i40e_pkt_template_input input; + /** the action to be taken when raw packet template is matched */ + struct rte_pmd_i40e_pkt_template_action action; + /** ID, an unique software index for the raw packet template filter */ + uint32_t soft_id; +}; + +/** + * Add or remove raw packet template filter to Flow Director. + * + * @param port + * The port identifier of the Ethernet device. + * @param conf + * Specifies configuration parameters of raw packet template filter. + * @param add + * Speicifes an action to be taken - add or remove raw packet template filter. + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-EINVAL) if *conf* invalid. + * - (-ENOTSUP) not supported by firmware. + */ +int rte_pmd_i40e_flow_add_del_packet_template( + uint16_t port, + const struct rte_pmd_i40e_pkt_template_conf *conf, + uint8_t add); + +/** + * Notify VF when PF link status changes. + * + * @param port + * The port identifier of the Ethernet device. + * @param vf + * VF id. + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-EINVAL) if *vf* invalid. + */ +int rte_pmd_i40e_ping_vfs(uint16_t port, uint16_t vf); + +/** + * Enable/Disable VF MAC anti spoofing. + * + * @param port + * The port identifier of the Ethernet device. + * @param vf_id + * VF on which to set MAC anti spoofing. + * @param on + * 1 - Enable VFs MAC anti spoofing. + * 0 - Disable VFs MAC anti spoofing. + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-EINVAL) if bad parameter. + */ +int rte_pmd_i40e_set_vf_mac_anti_spoof(uint16_t port, + uint16_t vf_id, + uint8_t on); + +/** + * Enable/Disable VF VLAN anti spoofing. + * + * @param port + * The port identifier of the Ethernet device. + * @param vf_id + * VF on which to set VLAN anti spoofing. + * @param on + * 1 - Enable VFs VLAN anti spoofing. + * 0 - Disable VFs VLAN anti spoofing. + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-EINVAL) if bad parameter. + */ +int rte_pmd_i40e_set_vf_vlan_anti_spoof(uint16_t port, + uint16_t vf_id, + uint8_t on); + +/** + * Enable/Disable TX loopback on all the PF and VFs. + * + * @param port + * The port identifier of the Ethernet device. + * @param on + * 1 - Enable TX loopback. + * 0 - Disable TX loopback. + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-EINVAL) if bad parameter. + */ +int rte_pmd_i40e_set_tx_loopback(uint16_t port, + uint8_t on); + +/** + * Enable/Disable VF unicast promiscuous mode. + * + * @param port + * The port identifier of the Ethernet device. + * @param vf_id + * VF on which to set. + * @param on + * 1 - Enable. + * 0 - Disable. + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-EINVAL) if bad parameter. + */ +int rte_pmd_i40e_set_vf_unicast_promisc(uint16_t port, + uint16_t vf_id, + uint8_t on); + +/** + * Enable/Disable VF multicast promiscuous mode. + * + * @param port + * The port identifier of the Ethernet device. + * @param vf_id + * VF on which to set. + * @param on + * 1 - Enable. + * 0 - Disable. + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-EINVAL) if bad parameter. + */ +int rte_pmd_i40e_set_vf_multicast_promisc(uint16_t port, + uint16_t vf_id, + uint8_t on); + +/** + * Set the VF MAC address. + * + * PF should set MAC address before VF initialized, if PF sets the MAC + * address after VF initialized, new MAC address won't be effective until + * VF reinitialize. + * + * This will remove all existing MAC filters. + * + * @param port + * The port identifier of the Ethernet device. + * @param vf_id + * VF id. + * @param mac_addr + * VF MAC address. + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-EINVAL) if *vf* or *mac_addr* is invalid. + */ +int rte_pmd_i40e_set_vf_mac_addr(uint16_t port, uint16_t vf_id, + struct ether_addr *mac_addr); + +/** + * Enable/Disable vf vlan strip for all queues in a pool + * + * @param port + * The port identifier of the Ethernet device. + * @param vf + * ID specifying VF. + * @param on + * 1 - Enable VF's vlan strip on RX queues. + * 0 - Disable VF's vlan strip on RX queues. + * + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-EINVAL) if bad parameter. + */ +int +rte_pmd_i40e_set_vf_vlan_stripq(uint16_t port, uint16_t vf, uint8_t on); + +/** + * Enable/Disable vf vlan insert + * + * @param port + * The port identifier of the Ethernet device. + * @param vf_id + * ID specifying VF. + * @param vlan_id + * 0 - Disable VF's vlan insert. + * n - Enable; n is inserted as the vlan id. + * + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-EINVAL) if bad parameter. + */ +int rte_pmd_i40e_set_vf_vlan_insert(uint16_t port, uint16_t vf_id, + uint16_t vlan_id); + +/** + * Enable/Disable vf broadcast mode + * + * @param port + * The port identifier of the Ethernet device. + * @param vf_id + * ID specifying VF. + * @param on + * 0 - Disable broadcast. + * 1 - Enable broadcast. + * + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-EINVAL) if bad parameter. + */ +int rte_pmd_i40e_set_vf_broadcast(uint16_t port, uint16_t vf_id, + uint8_t on); + +/** + * Enable/Disable vf vlan tag + * + * @param port + * The port identifier of the Ethernet device. + * @param vf_id + * ID specifying VF. + * @param on + * 0 - Disable VF's vlan tag. + * n - Enable VF's vlan tag. + * + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-EINVAL) if bad parameter. + */ +int rte_pmd_i40e_set_vf_vlan_tag(uint16_t port, uint16_t vf_id, uint8_t on); + +/** + * Enable/Disable VF VLAN filter + * + * @param port + * The port identifier of the Ethernet device. + * @param vlan_id + * ID specifying VLAN + * @param vf_mask + * Mask to filter VF's + * @param on + * 0 - Disable VF's VLAN filter. + * 1 - Enable VF's VLAN filter. + * + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-EINVAL) if bad parameter. + * - (-ENOTSUP) not supported by firmware. + */ +int rte_pmd_i40e_set_vf_vlan_filter(uint16_t port, uint16_t vlan_id, + uint64_t vf_mask, uint8_t on); + +/** + * Get VF's statistics + * + * @param port + * The port identifier of the Ethernet device. + * @param vf_id + * VF on which to get. + * @param stats + * A pointer to a structure of type *rte_eth_stats* to be filled with + * the values of device counters for the following set of statistics: + * - *ipackets* with the total of successfully received packets. + * - *opackets* with the total of successfully transmitted packets. + * - *ibytes* with the total of successfully received bytes. + * - *obytes* with the total of successfully transmitted bytes. + * - *ierrors* with the total of erroneous received packets. + * - *oerrors* with the total of failed transmitted packets. + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-EINVAL) if bad parameter. + */ + +int rte_pmd_i40e_get_vf_stats(uint16_t port, + uint16_t vf_id, + struct rte_eth_stats *stats); + +/** + * Clear VF's statistics + * + * @param port + * The port identifier of the Ethernet device. + * @param vf_id + * VF on which to get. + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-EINVAL) if bad parameter. + */ +int rte_pmd_i40e_reset_vf_stats(uint16_t port, + uint16_t vf_id); + +/** + * Set VF's max bandwidth. + * + * Per VF bandwidth limitation and per TC bandwidth limitation cannot + * be enabled in parallel. If per TC bandwidth is enabled, this function + * will disable it. + * + * @param port + * The port identifier of the Ethernet device. + * @param vf_id + * ID specifying VF. + * @param bw + * Bandwidth for this VF. + * The value should be an absolute bandwidth in Mbps. + * The bandwidth is a L2 bandwidth counting the bytes of ethernet packets. + * Not count the bytes added by physical layer. + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-EINVAL) if bad parameter. + * - (-ENOTSUP) not supported by firmware. + */ +int rte_pmd_i40e_set_vf_max_bw(uint16_t port, + uint16_t vf_id, + uint32_t bw); + +/** + * Set all the TCs' bandwidth weight on a specific VF. + * + * The bw_weight means the percentage occupied by the TC. + * It can be taken as the relative min bandwidth setting. + * + * @param port + * The port identifier of the Ethernet device. + * @param vf_id + * ID specifying VF. + * @param tc_num + * Number of TCs. + * @param bw_weight + * An array of relative bandwidth weight for all the TCs. + * The summary of the bw_weight should be 100. + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-EINVAL) if bad parameter. + * - (-ENOTSUP) not supported by firmware. + */ +int rte_pmd_i40e_set_vf_tc_bw_alloc(uint16_t port, + uint16_t vf_id, + uint8_t tc_num, + uint8_t *bw_weight); + +/** + * Set a specific TC's max bandwidth on a specific VF. + * + * @param port + * The port identifier of the Ethernet device. + * @param vf_id + * ID specifying VF. + * @param tc_no + * Number specifying TC. + * @param bw + * Max bandwidth for this TC. + * The value should be an absolute bandwidth in Mbps. + * The bandwidth is a L2 bandwidth counting the bytes of ethernet packets. + * Not count the bytes added by physical layer. + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-EINVAL) if bad parameter. + * - (-ENOTSUP) not supported by firmware. + */ +int rte_pmd_i40e_set_vf_tc_max_bw(uint16_t port, + uint16_t vf_id, + uint8_t tc_no, + uint32_t bw); + +/** + * Set some TCs to strict priority mode on a physical port. + * + * @param port + * The port identifier of the Ethernet device. + * @param tc_map + * A bit map for the TCs. + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-EINVAL) if bad parameter. + * - (-ENOTSUP) not supported by firmware. + */ +int rte_pmd_i40e_set_tc_strict_prio(uint16_t port, uint8_t tc_map); + +/** + * Load/Unload a ddp package + * + * @param port + * The port identifier of the Ethernet device. + * @param buff + * buffer of package. + * @param size + * size of buffer. + * @param op + * Operation of package processing + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-EINVAL) if bad parameter. + * - (-EEXIST) if profile exists. + * - (-EACCES) if profile does not exist. + * - (-ENOTSUP) if operation not supported. + */ +int rte_pmd_i40e_process_ddp_package(uint16_t port, uint8_t *buff, + uint32_t size, + enum rte_pmd_i40e_package_op op); + +/** + * rte_pmd_i40e_get_ddp_info - Get profile's info + * @param pkg + * buffer of package. + * @param pkg_size + * package buffer size + * @param info + * buffer for response + * @param size + * response buffer size + * @param type + * type of information requested + * @return + * - (0) if successful. + * - (-ENOTSUP) if information type not supported by the profile. + * - (-EINVAL) if bad parameter. + */ +int rte_pmd_i40e_get_ddp_info(uint8_t *pkg, uint32_t pkg_size, + uint8_t *info, uint32_t size, + enum rte_pmd_i40e_package_info type); + +/** + * rte_pmd_i40e_get_ddp_list - Get loaded profile list + * @param port + * port id + * @param buff + * buffer for response + * @param size + * buffer size + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-EINVAL) if bad parameter. + */ +int rte_pmd_i40e_get_ddp_list(uint16_t port, uint8_t *buff, uint32_t size); + +/** + * Update hardware defined ptype to software defined packet type + * mapping table. + * + * @param port + * pointer to port identifier of the device. + * @param mapping_items + * the base address of the mapping items array. + * @param count + * number of mapping items. + * @param exclusive + * the flag indicate different ptype mapping update method. + * -(0) only overwrite referred PTYPE mapping, + * keep other PTYPEs mapping unchanged. + * -(!0) overwrite referred PTYPE mapping, + * set other PTYPEs maps to PTYPE_UNKNOWN. + */ +int rte_pmd_i40e_ptype_mapping_update( + uint16_t port, + struct rte_pmd_i40e_ptype_mapping *mapping_items, + uint16_t count, + uint8_t exclusive); + +/** + * Reset hardware defined ptype to software defined ptype + * mapping table to default. + * + * @param port + * pointer to port identifier of the device + */ +int rte_pmd_i40e_ptype_mapping_reset(uint16_t port); + +/** + * Get hardware defined ptype to software defined ptype + * mapping items. + * + * @param port + * pointer to port identifier of the device. + * @param mapping_items + * the base address of the array to store returned items. + * @param size + * the size of the input array. + * @param count + * the place to store the number of returned items. + * @param valid_only + * -(0) return full mapping table. + * -(!0) only return mapping items which packet_type != RTE_PTYPE_UNKNOWN. + */ +int rte_pmd_i40e_ptype_mapping_get( + uint16_t port, + struct rte_pmd_i40e_ptype_mapping *mapping_items, + uint16_t size, + uint16_t *count, + uint8_t valid_only); + +/** + * Replace a specific or a group of software defined ptypes + * with a new one + * + * @param port + * pointer to port identifier of the device + * @param target + * the packet type to be replaced + * @param mask + * -(0) target represent a specific software defined ptype. + * -(!0) target is a mask to represent a group of software defined ptypes. + * @param pkt_type + * the new packet type to overwrite + */ +int rte_pmd_i40e_ptype_mapping_replace(uint16_t port, + uint32_t target, + uint8_t mask, + uint32_t pkt_type); + +/** + * Add a VF MAC address. + * + * Add more MAC address for VF. The existing MAC addresses + * are still effective. + * + * @param port + * The port identifier of the Ethernet device. + * @param vf_id + * VF id. + * @param mac_addr + * VF MAC address. + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-EINVAL) if *vf* or *mac_addr* is invalid. + */ +int rte_pmd_i40e_add_vf_mac_addr(uint16_t port, uint16_t vf_id, + struct ether_addr *mac_addr); + +#define RTE_PMD_I40E_PCTYPE_MAX 64 +#define RTE_PMD_I40E_FLOW_TYPE_MAX 64 + +struct rte_pmd_i40e_flow_type_mapping { + uint16_t flow_type; /**< software defined flow type*/ + uint64_t pctype; /**< hardware defined pctype */ +}; + +/** + * Update hardware defined pctype to software defined flow type + * mapping table. + * + * @param port + * pointer to port identifier of the device. + * @param mapping_items + * the base address of the mapping items array. + * @param count + * number of mapping items. + * @param exclusive + * the flag indicate different pctype mapping update method. + * -(0) only overwrite referred PCTYPE mapping, + * keep other PCTYPEs mapping unchanged. + * -(!0) overwrite referred PCTYPE mapping, + * set other PCTYPEs maps to PCTYPE_INVALID. + */ +int rte_pmd_i40e_flow_type_mapping_update( + uint16_t port, + struct rte_pmd_i40e_flow_type_mapping *mapping_items, + uint16_t count, + uint8_t exclusive); + +/** + * Get software defined flow type to hardware defined pctype + * mapping items. + * + * @param port + * pointer to port identifier of the device. + * @param mapping_items + * the base address of the array to store returned items. + * array should be allocated by caller with minimum size of + * RTE_PMD_I40E_FLOW_TYPE_MAX items + */ +int rte_pmd_i40e_flow_type_mapping_get( + uint16_t port, + struct rte_pmd_i40e_flow_type_mapping *mapping_items); + +/** + * Reset hardware defined pctype to software defined flow type + * mapping table to default. + * + * @param port + * pointer to port identifier of the device + */ +int rte_pmd_i40e_flow_type_mapping_reset(uint16_t port); + +/** + * On the PF, find VF index based on VF MAC address + * + * @param port + * pointer to port identifier of the device + * @param vf_mac + * the mac address of the vf to determine index of + * @return + * The index of vfid If successful. + * -EINVAL: vf mac address does not exist for this port + * -ENOTSUP: i40e not supported for this port. + */ +int rte_pmd_i40e_query_vfid_by_mac(uint16_t port, + const struct ether_addr *vf_mac); + +/** + * Do RSS queue region configuration for that port as + * the command option type + * + * @param port_id + * The port identifier of the Ethernet device. + * @param op_type + * Queue region operation type + * @param arg + * Queue region operation type specific data + */ +int rte_pmd_i40e_rss_queue_region_conf(uint16_t port_id, + enum rte_pmd_i40e_queue_region_op op_type, void *arg); + +#endif /* _PMD_I40E_H_ */ diff --git a/dpdk/drivers/net/i40e/rte_pmd_i40e_version.map b/dpdk/drivers/net/i40e/rte_pmd_i40e_version.map index ef353984..ebbd24e0 100644 --- a/dpdk/drivers/net/i40e/rte_pmd_i40e_version.map +++ b/dpdk/drivers/net/i40e/rte_pmd_i40e_version.map @@ -2,3 +2,59 @@ DPDK_2.0 { local: *; }; + +DPDK_17.02 { + global: + + rte_pmd_i40e_get_vf_stats; + rte_pmd_i40e_ping_vfs; + rte_pmd_i40e_ptype_mapping_get; + rte_pmd_i40e_ptype_mapping_replace; + rte_pmd_i40e_ptype_mapping_reset; + rte_pmd_i40e_ptype_mapping_update; + rte_pmd_i40e_reset_vf_stats; + rte_pmd_i40e_set_tx_loopback; + rte_pmd_i40e_set_vf_broadcast; + rte_pmd_i40e_set_vf_mac_addr; + rte_pmd_i40e_set_vf_mac_anti_spoof; + rte_pmd_i40e_set_vf_multicast_promisc; + rte_pmd_i40e_set_vf_unicast_promisc; + rte_pmd_i40e_set_vf_vlan_anti_spoof; + rte_pmd_i40e_set_vf_vlan_filter; + rte_pmd_i40e_set_vf_vlan_insert; + rte_pmd_i40e_set_vf_vlan_stripq; + rte_pmd_i40e_set_vf_vlan_tag; + +} DPDK_2.0; + +DPDK_17.05 { + global: + + rte_pmd_i40e_set_tc_strict_prio; + rte_pmd_i40e_set_vf_max_bw; + rte_pmd_i40e_set_vf_tc_bw_alloc; + rte_pmd_i40e_set_vf_tc_max_bw; + rte_pmd_i40e_process_ddp_package; + rte_pmd_i40e_get_ddp_list; + +} DPDK_17.02; + +DPDK_17.08 { + global: + + rte_pmd_i40e_get_ddp_info; + +} DPDK_17.05; + +DPDK_17.11 { + global: + + rte_pmd_i40e_add_vf_mac_addr; + rte_pmd_i40e_flow_add_del_packet_template; + rte_pmd_i40e_flow_type_mapping_update; + rte_pmd_i40e_flow_type_mapping_get; + rte_pmd_i40e_flow_type_mapping_reset; + rte_pmd_i40e_query_vfid_by_mac; + rte_pmd_i40e_rss_queue_region_conf; + +} DPDK_17.08; diff --git a/dpdk/drivers/net/ixgbe/Makefile b/dpdk/drivers/net/ixgbe/Makefile index a6c71f34..511a64eb 100644 --- a/dpdk/drivers/net/ixgbe/Makefile +++ b/dpdk/drivers/net/ixgbe/Makefile @@ -1,6 +1,6 @@ # BSD LICENSE # -# Copyright(c) 2010-2015 Intel Corporation. All rights reserved. +# Copyright(c) 2010-2016 Intel Corporation. All rights reserved. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -41,7 +41,7 @@ CFLAGS += $(WERROR_FLAGS) EXPORT_MAP := rte_pmd_ixgbe_version.map -LIBABIVER := 1 +LIBABIVER := 2 ifeq ($(CONFIG_RTE_TOOLCHAIN_ICC),y) # @@ -76,15 +76,21 @@ endif ifeq ($(shell test $(GCC_VERSION) -ge 50 && echo 1), 1) CFLAGS_ixgbe_common.o += -Wno-logical-not-parentheses +ifeq ($(shell test $(GCC_VERSION) -ge 70 && echo 1), 1) +CFLAGS_BASE_DRIVER += -Wno-implicit-fallthrough +endif endif endif +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs -lrte_hash +LDLIBS += -lrte_bus_pci # # Add extra flags for base driver files (also known as shared code) # to disable warnings in them # -BASE_DRIVER_OBJS=$(patsubst %.c,%.o,$(notdir $(wildcard $(SRCDIR)/base/*.c))) +BASE_DRIVER_OBJS=$(sort $(patsubst %.c,%.o,$(notdir $(wildcard $(SRCDIR)/base/*.c)))) $(foreach obj, $(BASE_DRIVER_OBJS), $(eval CFLAGS_$(obj)+=$(CFLAGS_BASE_DRIVER))) VPATH += $(SRCDIR)/base @@ -100,6 +106,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_IXGBE_PMD) += ixgbe_x550.c SRCS-$(CONFIG_RTE_LIBRTE_IXGBE_PMD) += ixgbe_phy.c SRCS-$(CONFIG_RTE_LIBRTE_IXGBE_PMD) += ixgbe_api.c SRCS-$(CONFIG_RTE_LIBRTE_IXGBE_PMD) += ixgbe_vf.c +SRCS-$(CONFIG_RTE_LIBRTE_IXGBE_PMD) += ixgbe_hv_vf.c SRCS-$(CONFIG_RTE_LIBRTE_IXGBE_PMD) += ixgbe_dcb.c SRCS-$(CONFIG_RTE_LIBRTE_IXGBE_PMD) += ixgbe_dcb_82599.c SRCS-$(CONFIG_RTE_LIBRTE_IXGBE_PMD) += ixgbe_dcb_82598.c @@ -108,21 +115,23 @@ SRCS-$(CONFIG_RTE_LIBRTE_IXGBE_PMD) += ixgbe_rxtx.c SRCS-$(CONFIG_RTE_LIBRTE_IXGBE_PMD) += ixgbe_ethdev.c SRCS-$(CONFIG_RTE_LIBRTE_IXGBE_PMD) += ixgbe_fdir.c SRCS-$(CONFIG_RTE_LIBRTE_IXGBE_PMD) += ixgbe_pf.c +SRCS-$(CONFIG_RTE_LIBRTE_IXGBE_PMD) += ixgbe_flow.c ifeq ($(CONFIG_RTE_ARCH_ARM64),y) SRCS-$(CONFIG_RTE_IXGBE_INC_VECTOR) += ixgbe_rxtx_vec_neon.c else SRCS-$(CONFIG_RTE_IXGBE_INC_VECTOR) += ixgbe_rxtx_vec_sse.c endif - -ifeq ($(CONFIG_RTE_NIC_BYPASS),y) +ifeq ($(CONFIG_RTE_LIBRTE_IXGBE_BYPASS),y) SRCS-$(CONFIG_RTE_LIBRTE_IXGBE_PMD) += ixgbe_bypass.c SRCS-$(CONFIG_RTE_LIBRTE_IXGBE_PMD) += ixgbe_82599_bypass.c endif +ifeq ($(CONFIG_RTE_LIBRTE_SECURITY),y) +SRCS-$(CONFIG_RTE_LIBRTE_IXGBE_PMD) += ixgbe_ipsec.c +endif +SRCS-$(CONFIG_RTE_LIBRTE_IXGBE_PMD) += rte_pmd_ixgbe.c +SRCS-$(CONFIG_RTE_LIBRTE_IXGBE_PMD) += ixgbe_tm.c - -# this lib depends upon: -DEPDIRS-$(CONFIG_RTE_LIBRTE_IXGBE_PMD) += lib/librte_eal lib/librte_ether -DEPDIRS-$(CONFIG_RTE_LIBRTE_IXGBE_PMD) += lib/librte_mempool lib/librte_mbuf -DEPDIRS-$(CONFIG_RTE_LIBRTE_IXGBE_PMD) += lib/librte_net +# install this header file +SYMLINK-$(CONFIG_RTE_LIBRTE_IXGBE_PMD)-include := rte_pmd_ixgbe.h include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/net/ixgbe/base/README b/dpdk/drivers/net/ixgbe/base/README index 76e78051..8c833b44 100644 --- a/dpdk/drivers/net/ixgbe/base/README +++ b/dpdk/drivers/net/ixgbe/base/README @@ -1,7 +1,7 @@ .. BSD LICENSE - Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + Copyright(c) 2010-2017 Intel Corporation. All rights reserved. All rights reserved. Redistribution and use in source and binary forms, with or without @@ -34,7 +34,8 @@ Intel® IXGBE driver =================== This directory contains source code of FreeBSD ixgbe driver of version -cid-10g-shared-code.2016.04.12 released by ND. The sub-directory of base/ +cid-10g-shared-code.2017.05.16 released by the team which develop +basic drivers for any ixgbe NIC. The sub-directory of base/ contains the original source package. This driver is valid for the product(s) listed below diff --git a/dpdk/drivers/net/ixgbe/base/ixgbe_82598.c b/dpdk/drivers/net/ixgbe/base/ixgbe_82598.c index db808801..d64abb2e 100644 --- a/dpdk/drivers/net/ixgbe/base/ixgbe_82598.c +++ b/dpdk/drivers/net/ixgbe/base/ixgbe_82598.c @@ -995,19 +995,19 @@ STATIC s32 ixgbe_clear_vmdq_82598(struct ixgbe_hw *hw, u32 rar, u32 vmdq) * @vlan: VLAN id to write to VLAN filter * @vind: VMDq output index that maps queue to VLAN id in VFTA * @vlan_on: boolean flag to turn on/off VLAN in VFTA - * @bypass_vlvf: boolean flag - unused + * @vlvf_bypass: boolean flag - unused * * Turn on/off specified VLAN in the VLAN filter table. **/ s32 ixgbe_set_vfta_82598(struct ixgbe_hw *hw, u32 vlan, u32 vind, - bool vlan_on, bool bypass_vlvf) + bool vlan_on, bool vlvf_bypass) { u32 regindex; u32 bitindex; u32 bits; u32 vftabyte; - UNREFERENCED_1PARAMETER(bypass_vlvf); + UNREFERENCED_1PARAMETER(vlvf_bypass); DEBUGFUNC("ixgbe_set_vfta_82598"); @@ -1222,9 +1222,9 @@ STATIC s32 ixgbe_read_i2c_sff8472_82598(struct ixgbe_hw *hw, u8 byte_offset, * * Determines physical layer capabilities of the current configuration. **/ -u32 ixgbe_get_supported_physical_layer_82598(struct ixgbe_hw *hw) +u64 ixgbe_get_supported_physical_layer_82598(struct ixgbe_hw *hw) { - u32 physical_layer = IXGBE_PHYSICAL_LAYER_UNKNOWN; + u64 physical_layer = IXGBE_PHYSICAL_LAYER_UNKNOWN; u32 autoc = IXGBE_READ_REG(hw, IXGBE_AUTOC); u32 pma_pmd_10g = autoc & IXGBE_AUTOC_10G_PMA_PMD_MASK; u32 pma_pmd_1g = autoc & IXGBE_AUTOC_1G_PMA_PMD_MASK; diff --git a/dpdk/drivers/net/ixgbe/base/ixgbe_82598.h b/dpdk/drivers/net/ixgbe/base/ixgbe_82598.h index 0326e70b..20aab9fc 100644 --- a/dpdk/drivers/net/ixgbe/base/ixgbe_82598.h +++ b/dpdk/drivers/net/ixgbe/base/ixgbe_82598.h @@ -45,7 +45,7 @@ s32 ixgbe_read_analog_reg8_82598(struct ixgbe_hw *hw, u32 reg, u8 *val); s32 ixgbe_write_analog_reg8_82598(struct ixgbe_hw *hw, u32 reg, u8 val); s32 ixgbe_read_i2c_eeprom_82598(struct ixgbe_hw *hw, u8 byte_offset, u8 *eeprom_data); -u32 ixgbe_get_supported_physical_layer_82598(struct ixgbe_hw *hw); +u64 ixgbe_get_supported_physical_layer_82598(struct ixgbe_hw *hw); s32 ixgbe_init_phy_ops_82598(struct ixgbe_hw *hw); void ixgbe_set_lan_id_multi_port_pcie_82598(struct ixgbe_hw *hw); void ixgbe_set_pcie_completion_timeout(struct ixgbe_hw *hw); diff --git a/dpdk/drivers/net/ixgbe/base/ixgbe_82599.c b/dpdk/drivers/net/ixgbe/base/ixgbe_82599.c index 5bc7c2b9..d9d11a8e 100644 --- a/dpdk/drivers/net/ixgbe/base/ixgbe_82599.c +++ b/dpdk/drivers/net/ixgbe/base/ixgbe_82599.c @@ -1178,6 +1178,7 @@ mac_reset_top: if (ixgbe_validate_mac_addr(hw->mac.san_addr) == 0) { /* Save the SAN MAC RAR index */ hw->mac.san_mac_rar_index = hw->mac.num_rar_entries - 1; + hw->mac.ops.set_rar(hw, hw->mac.san_mac_rar_index, hw->mac.san_addr, 0, IXGBE_RAH_AV); @@ -1809,14 +1810,23 @@ s32 ixgbe_fdir_set_input_mask_82599(struct ixgbe_hw *hw, } IXGBE_WRITE_REG_BE32(hw, IXGBE_FDIRIP6M, fdirip6m); - /* Set all bits in FDIRTCPM, FDIRUDPM, FDIRSIP4M and - * FDIRDIP4M in cloud mode to allow L3/L3 packets to - * tunnel. + /* Set all bits in FDIRTCPM, FDIRUDPM, FDIRSCTPM, + * FDIRSIP4M and FDIRDIP4M in cloud mode to allow + * L3/L3 packets to tunnel. */ IXGBE_WRITE_REG(hw, IXGBE_FDIRTCPM, 0xFFFFFFFF); IXGBE_WRITE_REG(hw, IXGBE_FDIRUDPM, 0xFFFFFFFF); IXGBE_WRITE_REG_BE32(hw, IXGBE_FDIRDIP4M, 0xFFFFFFFF); IXGBE_WRITE_REG_BE32(hw, IXGBE_FDIRSIP4M, 0xFFFFFFFF); + switch (hw->mac.type) { + case ixgbe_mac_X550: + case ixgbe_mac_X550EM_x: + case ixgbe_mac_X550EM_a: + IXGBE_WRITE_REG(hw, IXGBE_FDIRSCTPM, 0xFFFFFFFF); + break; + default: + break; + } } /* Now mask VM pool and destination IPv6 - bits 5 and 2 */ @@ -2159,9 +2169,9 @@ s32 ixgbe_identify_phy_82599(struct ixgbe_hw *hw) * * Determines physical layer capabilities of the current configuration. **/ -u32 ixgbe_get_supported_physical_layer_82599(struct ixgbe_hw *hw) +u64 ixgbe_get_supported_physical_layer_82599(struct ixgbe_hw *hw) { - u32 physical_layer = IXGBE_PHYSICAL_LAYER_UNKNOWN; + u64 physical_layer = IXGBE_PHYSICAL_LAYER_UNKNOWN; u32 autoc = IXGBE_READ_REG(hw, IXGBE_AUTOC); u32 autoc2 = IXGBE_READ_REG(hw, IXGBE_AUTOC2); u32 pma_pmd_10g_serial = autoc2 & IXGBE_AUTOC2_10G_SERIAL_PMA_PMD_MASK; diff --git a/dpdk/drivers/net/ixgbe/base/ixgbe_82599.h b/dpdk/drivers/net/ixgbe/base/ixgbe_82599.h index c034d3d9..d555dbce 100644 --- a/dpdk/drivers/net/ixgbe/base/ixgbe_82599.h +++ b/dpdk/drivers/net/ixgbe/base/ixgbe_82599.h @@ -57,7 +57,7 @@ s32 ixgbe_write_analog_reg8_82599(struct ixgbe_hw *hw, u32 reg, u8 val); s32 ixgbe_start_hw_82599(struct ixgbe_hw *hw); s32 ixgbe_identify_phy_82599(struct ixgbe_hw *hw); s32 ixgbe_init_phy_ops_82599(struct ixgbe_hw *hw); -u32 ixgbe_get_supported_physical_layer_82599(struct ixgbe_hw *hw); +u64 ixgbe_get_supported_physical_layer_82599(struct ixgbe_hw *hw); s32 ixgbe_enable_rx_dma_82599(struct ixgbe_hw *hw, u32 regval); s32 prot_autoc_read_82599(struct ixgbe_hw *hw, bool *locked, u32 *reg_val); s32 prot_autoc_write_82599(struct ixgbe_hw *hw, u32 reg_val, bool locked); diff --git a/dpdk/drivers/net/ixgbe/base/ixgbe_api.c b/dpdk/drivers/net/ixgbe/base/ixgbe_api.c index 17868676..4117fb01 100644 --- a/dpdk/drivers/net/ixgbe/base/ixgbe_api.c +++ b/dpdk/drivers/net/ixgbe/base/ixgbe_api.c @@ -106,8 +106,10 @@ s32 ixgbe_init_shared_code(struct ixgbe_hw *hw) status = ixgbe_init_ops_X550(hw); break; case ixgbe_mac_X550EM_x: + status = ixgbe_init_ops_X550EM_x(hw); + break; case ixgbe_mac_X550EM_a: - status = ixgbe_init_ops_X550EM(hw); + status = ixgbe_init_ops_X550EM_a(hw); break; case ixgbe_mac_82599_vf: case ixgbe_mac_X540_vf: @@ -203,6 +205,7 @@ s32 ixgbe_set_mac_type(struct ixgbe_hw *hw) case IXGBE_DEV_ID_X550EM_X_10G_T: case IXGBE_DEV_ID_X550EM_X_1G_T: case IXGBE_DEV_ID_X550EM_X_SFP: + case IXGBE_DEV_ID_X550EM_X_XFI: hw->mac.type = ixgbe_mac_X550EM_x; hw->mvals = ixgbe_mvals_X550EM_x; break; @@ -1090,7 +1093,7 @@ s32 ixgbe_set_vfta(struct ixgbe_hw *hw, u32 vlan, u32 vind, bool vlan_on, bool vlvf_bypass) { return ixgbe_call_func(hw, hw->mac.ops.set_vfta, (hw, vlan, vind, - vlan_on, vlvf_bypass), IXGBE_NOT_IMPLEMENTED); + vlan_on, vlvf_bypass), IXGBE_NOT_IMPLEMENTED); } /** @@ -1100,7 +1103,7 @@ s32 ixgbe_set_vfta(struct ixgbe_hw *hw, u32 vlan, u32 vind, bool vlan_on, * @vind: VMDq output index that maps queue to VLAN id in VLVFB * @vlan_on: boolean flag to turn on/off VLAN in VLVF * @vfta_delta: pointer to the difference between the current value of VFTA - * and the desired value + * and the desired value * @vfta: the desired value of the VFTA * @vlvf_bypass: boolean flag indicating updating the default pool is okay * @@ -1110,7 +1113,7 @@ s32 ixgbe_set_vlvf(struct ixgbe_hw *hw, u32 vlan, u32 vind, bool vlan_on, u32 *vfta_delta, u32 vfta, bool vlvf_bypass) { return ixgbe_call_func(hw, hw->mac.ops.set_vlvf, (hw, vlan, vind, - vlan_on, vfta_delta, vfta, vlvf_bypass), + vlan_on, vfta_delta, vfta, vlvf_bypass), IXGBE_NOT_IMPLEMENTED); } @@ -1145,12 +1148,15 @@ s32 ixgbe_setup_fc(struct ixgbe_hw *hw) * @min: driver minor number to be sent to firmware * @build: driver build number to be sent to firmware * @ver: driver version number to be sent to firmware + * @len: length of driver_ver string + * @driver_ver: driver string **/ s32 ixgbe_set_fw_drv_ver(struct ixgbe_hw *hw, u8 maj, u8 min, u8 build, - u8 ver) + u8 ver, u16 len, char *driver_ver) { return ixgbe_call_func(hw, hw->mac.ops.set_fw_drv_ver, (hw, maj, min, - build, ver), IXGBE_NOT_IMPLEMENTED); + build, ver, len, driver_ver), + IXGBE_NOT_IMPLEMENTED); } @@ -1573,7 +1579,7 @@ s32 ixgbe_read_i2c_eeprom(struct ixgbe_hw *hw, u8 byte_offset, u8 *eeprom_data) * * Determines physical layer capabilities of the current configuration. **/ -u32 ixgbe_get_supported_physical_layer(struct ixgbe_hw *hw) +u64 ixgbe_get_supported_physical_layer(struct ixgbe_hw *hw) { return ixgbe_call_func(hw, hw->mac.ops.get_supported_physical_layer, (hw), IXGBE_PHYSICAL_LAYER_UNKNOWN); @@ -1659,6 +1665,7 @@ void ixgbe_init_swfw_semaphore(struct ixgbe_hw *hw) hw->mac.ops.init_swfw_sync(hw); } + void ixgbe_disable_rx(struct ixgbe_hw *hw) { if (hw->mac.ops.disable_rx) diff --git a/dpdk/drivers/net/ixgbe/base/ixgbe_api.h b/dpdk/drivers/net/ixgbe/base/ixgbe_api.h index 3aad1da7..2f532aa8 100644 --- a/dpdk/drivers/net/ixgbe/base/ixgbe_api.h +++ b/dpdk/drivers/net/ixgbe/base/ixgbe_api.h @@ -45,6 +45,8 @@ extern s32 ixgbe_init_ops_82599(struct ixgbe_hw *hw); extern s32 ixgbe_init_ops_X540(struct ixgbe_hw *hw); extern s32 ixgbe_init_ops_X550(struct ixgbe_hw *hw); extern s32 ixgbe_init_ops_X550EM(struct ixgbe_hw *hw); +extern s32 ixgbe_init_ops_X550EM_x(struct ixgbe_hw *hw); +extern s32 ixgbe_init_ops_X550EM_a(struct ixgbe_hw *hw); extern s32 ixgbe_init_ops_vf(struct ixgbe_hw *hw); s32 ixgbe_set_mac_type(struct ixgbe_hw *hw); @@ -131,7 +133,7 @@ s32 ixgbe_set_vlvf(struct ixgbe_hw *hw, u32 vlan, u32 vind, s32 ixgbe_fc_enable(struct ixgbe_hw *hw); s32 ixgbe_setup_fc(struct ixgbe_hw *hw); s32 ixgbe_set_fw_drv_ver(struct ixgbe_hw *hw, u8 maj, u8 min, u8 build, - u8 ver); + u8 ver, u16 len, char *driver_ver); s32 ixgbe_get_thermal_sensor_data(struct ixgbe_hw *hw); s32 ixgbe_init_thermal_sensor_thresh(struct ixgbe_hw *hw); void ixgbe_set_mta(struct ixgbe_hw *hw, u8 *mc_addr); @@ -141,7 +143,7 @@ s32 ixgbe_read_analog_reg8(struct ixgbe_hw *hw, u32 reg, u8 *val); s32 ixgbe_write_analog_reg8(struct ixgbe_hw *hw, u32 reg, u8 val); s32 ixgbe_init_uta_tables(struct ixgbe_hw *hw); s32 ixgbe_read_i2c_eeprom(struct ixgbe_hw *hw, u8 byte_offset, u8 *eeprom_data); -u32 ixgbe_get_supported_physical_layer(struct ixgbe_hw *hw); +u64 ixgbe_get_supported_physical_layer(struct ixgbe_hw *hw); s32 ixgbe_enable_rx_dma(struct ixgbe_hw *hw, u32 regval); s32 ixgbe_disable_sec_rx_path(struct ixgbe_hw *hw); s32 ixgbe_enable_sec_rx_path(struct ixgbe_hw *hw); diff --git a/dpdk/drivers/net/ixgbe/base/ixgbe_common.c b/dpdk/drivers/net/ixgbe/base/ixgbe_common.c index 1c5cb913..5e6ad952 100644 --- a/dpdk/drivers/net/ixgbe/base/ixgbe_common.c +++ b/dpdk/drivers/net/ixgbe/base/ixgbe_common.c @@ -113,6 +113,7 @@ s32 ixgbe_init_ops_generic(struct ixgbe_hw *hw) mac->ops.led_off = ixgbe_led_off_generic; mac->ops.blink_led_start = ixgbe_blink_led_start_generic; mac->ops.blink_led_stop = ixgbe_blink_led_stop_generic; + mac->ops.init_led_link_act = ixgbe_init_led_link_act_generic; /* RAR, Multicast, VLAN */ mac->ops.set_rar = ixgbe_set_rar_generic; @@ -168,16 +169,30 @@ bool ixgbe_device_supports_autoneg_fc(struct ixgbe_hw *hw) switch (hw->phy.media_type) { case ixgbe_media_type_fiber_qsfp: case ixgbe_media_type_fiber: - hw->mac.ops.check_link(hw, &speed, &link_up, false); - /* if link is down, assume supported */ - if (link_up) - supported = speed == IXGBE_LINK_SPEED_1GB_FULL ? + /* flow control autoneg black list */ + switch (hw->device_id) { + case IXGBE_DEV_ID_X550EM_A_SFP: + case IXGBE_DEV_ID_X550EM_A_SFP_N: + case IXGBE_DEV_ID_X550EM_A_QSFP: + case IXGBE_DEV_ID_X550EM_A_QSFP_N: + supported = false; + break; + default: + hw->mac.ops.check_link(hw, &speed, &link_up, false); + /* if link is down, assume supported */ + if (link_up) + supported = speed == IXGBE_LINK_SPEED_1GB_FULL ? true : false; - else - supported = true; + else + supported = true; + } + break; case ixgbe_media_type_backplane: - supported = true; + if (hw->device_id == IXGBE_DEV_ID_X550EM_X_XFI) + supported = false; + else + supported = true; break; case ixgbe_media_type_copper: /* only some copper devices support flow control autoneg */ @@ -188,6 +203,9 @@ bool ixgbe_device_supports_autoneg_fc(struct ixgbe_hw *hw) case IXGBE_DEV_ID_X550T: case IXGBE_DEV_ID_X550T1: case IXGBE_DEV_ID_X550EM_X_10G_T: + case IXGBE_DEV_ID_X550EM_A_10G_T: + case IXGBE_DEV_ID_X550EM_A_1G_T: + case IXGBE_DEV_ID_X550EM_A_1G_T_L: supported = true; break; default: @@ -197,9 +215,10 @@ bool ixgbe_device_supports_autoneg_fc(struct ixgbe_hw *hw) break; } - ERROR_REPORT2(IXGBE_ERROR_UNSUPPORTED, - "Device %x does not support flow control autoneg", - hw->device_id); + if (!supported) + ERROR_REPORT2(IXGBE_ERROR_UNSUPPORTED, + "Device %x does not support flow control autoneg", + hw->device_id); return supported; } @@ -371,6 +390,7 @@ s32 ixgbe_start_hw_generic(struct ixgbe_hw *hw) { s32 ret_val; u32 ctrl_ext; + u16 device_caps; DEBUGFUNC("ixgbe_start_hw_generic"); @@ -393,14 +413,31 @@ s32 ixgbe_start_hw_generic(struct ixgbe_hw *hw) /* Setup flow control */ ret_val = ixgbe_setup_fc(hw); - if (ret_val != IXGBE_SUCCESS) - goto out; + if (ret_val != IXGBE_SUCCESS && ret_val != IXGBE_NOT_IMPLEMENTED) { + DEBUGOUT1("Flow control setup failed, returning %d\n", ret_val); + return ret_val; + } + + /* Cache bit indicating need for crosstalk fix */ + switch (hw->mac.type) { + case ixgbe_mac_82599EB: + case ixgbe_mac_X550EM_x: + case ixgbe_mac_X550EM_a: + hw->mac.ops.get_device_caps(hw, &device_caps); + if (device_caps & IXGBE_DEVICE_CAPS_NO_CROSSTALK_WR) + hw->need_crosstalk_fix = false; + else + hw->need_crosstalk_fix = true; + break; + default: + hw->need_crosstalk_fix = false; + break; + } /* Clear adapter stopped flag */ hw->adapter_stopped = false; -out: - return ret_val; + return IXGBE_SUCCESS; } /** @@ -461,11 +498,18 @@ s32 ixgbe_init_hw_generic(struct ixgbe_hw *hw) /* Reset the hardware */ status = hw->mac.ops.reset_hw(hw); - if (status == IXGBE_SUCCESS) { + if (status == IXGBE_SUCCESS || status == IXGBE_ERR_SFP_NOT_PRESENT) { /* Start the HW */ status = hw->mac.ops.start_hw(hw); } + /* Initialize the LED link active for LED blink support */ + if (hw->mac.ops.init_led_link_act) + hw->mac.ops.init_led_link_act(hw); + + if (status != IXGBE_SUCCESS) + DEBUGOUT1("Failed to initialize HW, STATUS = %d\n", status); + return status; } @@ -1046,7 +1090,7 @@ void ixgbe_set_lan_id_multi_port_pcie(struct ixgbe_hw *hw) if (hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP) { hw->eeprom.ops.read(hw, IXGBE_EEPROM_CTRL_4, &ee_ctrl_4); bus->instance_id = (ee_ctrl_4 & IXGBE_EE_CTRL_4_INST_ID) >> - IXGBE_EE_CTRL_4_INST_ID_SHIFT; + IXGBE_EE_CTRL_4_INST_ID_SHIFT; } } @@ -1104,6 +1148,47 @@ s32 ixgbe_stop_adapter_generic(struct ixgbe_hw *hw) return ixgbe_disable_pcie_master(hw); } +/** + * ixgbe_init_led_link_act_generic - Store the LED index link/activity. + * @hw: pointer to hardware structure + * + * Store the index for the link active LED. This will be used to support + * blinking the LED. + **/ +s32 ixgbe_init_led_link_act_generic(struct ixgbe_hw *hw) +{ + struct ixgbe_mac_info *mac = &hw->mac; + u32 led_reg, led_mode; + u8 i; + + led_reg = IXGBE_READ_REG(hw, IXGBE_LEDCTL); + + /* Get LED link active from the LEDCTL register */ + for (i = 0; i < 4; i++) { + led_mode = led_reg >> IXGBE_LED_MODE_SHIFT(i); + + if ((led_mode & IXGBE_LED_MODE_MASK_BASE) == + IXGBE_LED_LINK_ACTIVE) { + mac->led_link_act = i; + return IXGBE_SUCCESS; + } + } + + /* + * If LEDCTL register does not have the LED link active set, then use + * known MAC defaults. + */ + switch (hw->mac.type) { + case ixgbe_mac_X550EM_a: + case ixgbe_mac_X550EM_x: + mac->led_link_act = 1; + break; + default: + mac->led_link_act = 2; + } + return IXGBE_SUCCESS; +} + /** * ixgbe_led_on_generic - Turns on the software controllable LEDs. * @hw: pointer to hardware structure @@ -1115,6 +1200,9 @@ s32 ixgbe_led_on_generic(struct ixgbe_hw *hw, u32 index) DEBUGFUNC("ixgbe_led_on_generic"); + if (index > 3) + return IXGBE_ERR_PARAM; + /* To turn on the LED, set mode to ON. */ led_reg &= ~IXGBE_LED_MODE_MASK(index); led_reg |= IXGBE_LED_ON << IXGBE_LED_MODE_SHIFT(index); @@ -1135,6 +1223,9 @@ s32 ixgbe_led_off_generic(struct ixgbe_hw *hw, u32 index) DEBUGFUNC("ixgbe_led_off_generic"); + if (index > 3) + return IXGBE_ERR_PARAM; + /* To turn off the LED, set mode to OFF. */ led_reg &= ~IXGBE_LED_MODE_MASK(index); led_reg |= IXGBE_LED_OFF << IXGBE_LED_MODE_SHIFT(index); @@ -2851,7 +2942,7 @@ out: * advertised settings **/ s32 ixgbe_negotiate_fc(struct ixgbe_hw *hw, u32 adv_reg, u32 lp_reg, - u32 adv_sym, u32 adv_asm, u32 lp_sym, u32 lp_asm) + u32 adv_sym, u32 adv_asm, u32 lp_sym, u32 lp_asm) { if ((!(adv_reg)) || (!(lp_reg))) { ERROR_REPORT3(IXGBE_ERROR_UNSUPPORTED, @@ -3323,7 +3414,7 @@ s32 prot_autoc_write_generic(struct ixgbe_hw *hw, u32 reg_val, bool locked) **/ s32 ixgbe_enable_sec_rx_path_generic(struct ixgbe_hw *hw) { - int secrxreg; + u32 secrxreg; DEBUGFUNC("ixgbe_enable_sec_rx_path_generic"); @@ -3370,6 +3461,9 @@ s32 ixgbe_blink_led_start_generic(struct ixgbe_hw *hw, u32 index) DEBUGFUNC("ixgbe_blink_led_start_generic"); + if (index > 3) + return IXGBE_ERR_PARAM; + /* * Link must be up to auto-blink the LEDs; * Force it if link is down. @@ -3415,6 +3509,10 @@ s32 ixgbe_blink_led_stop_generic(struct ixgbe_hw *hw, u32 index) DEBUGFUNC("ixgbe_blink_led_stop_generic"); + if (index > 3) + return IXGBE_ERR_PARAM; + + ret_val = hw->mac.ops.prot_autoc_read(hw, &locked, &autoc_reg); if (ret_val != IXGBE_SUCCESS) goto out; @@ -3720,7 +3818,8 @@ s32 ixgbe_clear_vmdq_generic(struct ixgbe_hw *hw, u32 rar, u32 vmdq) } /* was that the last pool using this rar? */ - if (mpsar_lo == 0 && mpsar_hi == 0 && rar != 0) + if (mpsar_lo == 0 && mpsar_hi == 0 && + rar != 0 && rar != hw->mac.san_mac_rar_index) hw->mac.ops.clear_rar(hw, rar); done: return IXGBE_SUCCESS; @@ -3887,7 +3986,8 @@ s32 ixgbe_set_vfta_generic(struct ixgbe_hw *hw, u32 vlan, u32 vind, vfta_delta = 1 << (vlan % 32); vfta = IXGBE_READ_REG(hw, IXGBE_VFTA(regidx)); - /* vfta_delta represents the difference between the current value + /* + * vfta_delta represents the difference between the current value * of vfta and the value we want in the register. Since the diff * is an XOR mask we can just update the vfta using an XOR */ @@ -3920,7 +4020,7 @@ vfta_update: * @vind: VMDq output index that maps queue to VLAN id in VLVFB * @vlan_on: boolean flag to turn on/off VLAN in VLVF * @vfta_delta: pointer to the difference between the current value of VFTA - * and the desired value + * and the desired value * @vfta: the desired value of the VFTA * @vlvf_bypass: boolean flag indicating updating default pool is okay * @@ -3947,6 +4047,7 @@ s32 ixgbe_set_vlvf_generic(struct ixgbe_hw *hw, u32 vlan, u32 vind, */ if (!(IXGBE_READ_REG(hw, IXGBE_VT_CTL) & IXGBE_VT_CTL_VT_ENABLE)) return IXGBE_SUCCESS; + vlvf_index = ixgbe_find_vlvf_slot(hw, vlan, vlvf_bypass); if (vlvf_index < 0) return vlvf_index; @@ -3976,6 +4077,7 @@ s32 ixgbe_set_vlvf_generic(struct ixgbe_hw *hw, u32 vlan, u32 vind, return IXGBE_SUCCESS; } + /* If there are still bits set in the VLVFB registers * for the VLAN ID indicated we need to see if the * caller is requesting that we clear the VFTA entry bit. @@ -4024,6 +4126,32 @@ s32 ixgbe_clear_vfta_generic(struct ixgbe_hw *hw) return IXGBE_SUCCESS; } +/** + * ixgbe_need_crosstalk_fix - Determine if we need to do cross talk fix + * @hw: pointer to hardware structure + * + * Contains the logic to identify if we need to verify link for the + * crosstalk fix + **/ +static bool ixgbe_need_crosstalk_fix(struct ixgbe_hw *hw) +{ + + /* Does FW say we need the fix */ + if (!hw->need_crosstalk_fix) + return false; + + /* Only consider SFP+ PHYs i.e. media type fiber */ + switch (hw->mac.ops.get_media_type(hw)) { + case ixgbe_media_type_fiber: + case ixgbe_media_type_fiber_qsfp: + break; + default: + return false; + } + + return true; +} + /** * ixgbe_check_mac_link_generic - Determine link and speed status * @hw: pointer to hardware structure @@ -4041,6 +4169,35 @@ s32 ixgbe_check_mac_link_generic(struct ixgbe_hw *hw, ixgbe_link_speed *speed, DEBUGFUNC("ixgbe_check_mac_link_generic"); + /* If Crosstalk fix enabled do the sanity check of making sure + * the SFP+ cage is full. + */ + if (ixgbe_need_crosstalk_fix(hw)) { + u32 sfp_cage_full; + + switch (hw->mac.type) { + case ixgbe_mac_82599EB: + sfp_cage_full = IXGBE_READ_REG(hw, IXGBE_ESDP) & + IXGBE_ESDP_SDP2; + break; + case ixgbe_mac_X550EM_x: + case ixgbe_mac_X550EM_a: + sfp_cage_full = IXGBE_READ_REG(hw, IXGBE_ESDP) & + IXGBE_ESDP_SDP0; + break; + default: + /* sanity check - No SFP+ devices here */ + sfp_cage_full = false; + break; + } + + if (!sfp_cage_full) { + *link_up = false; + *speed = IXGBE_LINK_SPEED_UNKNOWN; + return IXGBE_SUCCESS; + } + } + /* clear the old state */ links_orig = IXGBE_READ_REG(hw, IXGBE_LINKS); @@ -4082,11 +4239,18 @@ s32 ixgbe_check_mac_link_generic(struct ixgbe_hw *hw, ixgbe_link_speed *speed, break; case IXGBE_LINKS_SPEED_100_82599: *speed = IXGBE_LINK_SPEED_100_FULL; - if (hw->mac.type >= ixgbe_mac_X550) { + if (hw->mac.type == ixgbe_mac_X550) { if (links_reg & IXGBE_LINKS_SPEED_NON_STD) *speed = IXGBE_LINK_SPEED_5GB_FULL; } break; + case IXGBE_LINKS_SPEED_10_X550EM_A: + *speed = IXGBE_LINK_SPEED_UNKNOWN; + if (hw->device_id == IXGBE_DEV_ID_X550EM_A_1G_T || + hw->device_id == IXGBE_DEV_ID_X550EM_A_1G_T_L) { + *speed = IXGBE_LINK_SPEED_10_FULL; + } + break; default: *speed = IXGBE_LINK_SPEED_UNKNOWN; } @@ -4443,7 +4607,7 @@ s32 ixgbe_host_interface_command(struct ixgbe_hw *hw, u32 *buffer, /* first pull in the header so we know the buffer length */ for (bi = 0; bi < dword_len; bi++) { buffer[bi] = IXGBE_READ_REG_ARRAY(hw, IXGBE_FLEX_MNG, bi); - IXGBE_LE32_TO_CPUS(&buffer[bi]); + IXGBE_LE32_TO_CPUS((uintptr_t)&buffer[bi]); } /* If there is any thing in data position pull it in */ @@ -4463,7 +4627,7 @@ s32 ixgbe_host_interface_command(struct ixgbe_hw *hw, u32 *buffer, /* Pull in the rest of the buffer (bi is where we left off) */ for (; bi <= dword_len; bi++) { buffer[bi] = IXGBE_READ_REG_ARRAY(hw, IXGBE_FLEX_MNG, bi); - IXGBE_LE32_TO_CPUS(&buffer[bi]); + IXGBE_LE32_TO_CPUS((uintptr_t)&buffer[bi]); } rel_out: @@ -4486,13 +4650,15 @@ rel_out: * semaphore or IXGBE_ERR_HOST_INTERFACE_COMMAND when command fails. **/ s32 ixgbe_set_fw_drv_ver_generic(struct ixgbe_hw *hw, u8 maj, u8 min, - u8 build, u8 sub) + u8 build, u8 sub, u16 len, + const char *driver_ver) { struct ixgbe_hic_drv_info fw_cmd; int i; s32 ret_val = IXGBE_SUCCESS; DEBUGFUNC("ixgbe_set_fw_drv_ver_generic"); + UNREFERENCED_2PARAMETER(len, driver_ver); fw_cmd.hdr.cmd = FW_CEM_CMD_DRIVER_INFO; fw_cmd.hdr.buf_len = FW_CEM_CMD_DRIVER_INFO_LEN; diff --git a/dpdk/drivers/net/ixgbe/base/ixgbe_common.h b/dpdk/drivers/net/ixgbe/base/ixgbe_common.h index cd042375..903f34d5 100644 --- a/dpdk/drivers/net/ixgbe/base/ixgbe_common.h +++ b/dpdk/drivers/net/ixgbe/base/ixgbe_common.h @@ -72,6 +72,7 @@ s32 ixgbe_stop_adapter_generic(struct ixgbe_hw *hw); s32 ixgbe_led_on_generic(struct ixgbe_hw *hw, u32 index); s32 ixgbe_led_off_generic(struct ixgbe_hw *hw, u32 index); +s32 ixgbe_init_led_link_act_generic(struct ixgbe_hw *hw); s32 ixgbe_init_eeprom_params_generic(struct ixgbe_hw *hw); s32 ixgbe_write_eeprom_generic(struct ixgbe_hw *hw, u16 offset, u16 data); @@ -133,7 +134,7 @@ s32 ixgbe_clear_vmdq_generic(struct ixgbe_hw *hw, u32 rar, u32 vmdq); s32 ixgbe_insert_mac_addr_generic(struct ixgbe_hw *hw, u8 *addr, u32 vmdq); s32 ixgbe_init_uta_tables_generic(struct ixgbe_hw *hw); s32 ixgbe_set_vfta_generic(struct ixgbe_hw *hw, u32 vlan, - u32 vind, bool vlan_on, bool vlvf_bypass); + u32 vind, bool vlan_on, bool vlvf_bypass); s32 ixgbe_set_vlvf_generic(struct ixgbe_hw *hw, u32 vlan, u32 vind, bool vlan_on, u32 *vfta_delta, u32 vfta, bool vlvf_bypass); @@ -155,12 +156,14 @@ void ixgbe_set_rxpba_generic(struct ixgbe_hw *hw, int num_pb, u32 headroom, int strategy); void ixgbe_enable_relaxed_ordering_gen2(struct ixgbe_hw *hw); s32 ixgbe_set_fw_drv_ver_generic(struct ixgbe_hw *hw, u8 maj, u8 min, - u8 build, u8 ver); + u8 build, u8 ver, u16 len, const char *str); u8 ixgbe_calculate_checksum(u8 *buffer, u32 length); s32 ixgbe_host_interface_command(struct ixgbe_hw *hw, u32 *buffer, u32 length, u32 timeout, bool return_data); s32 ixgbe_hic_unlocked(struct ixgbe_hw *, u32 *buffer, u32 length, u32 timeout); - +s32 ixgbe_shutdown_fw_phy(struct ixgbe_hw *); +s32 ixgbe_fw_phy_activity(struct ixgbe_hw *, u16 activity, + u32 (*data)[FW_PHY_ACT_DATA_COUNT]); void ixgbe_clear_tx_pending(struct ixgbe_hw *hw); extern s32 ixgbe_reset_pipeline_82599(struct ixgbe_hw *hw); diff --git a/dpdk/drivers/net/ixgbe/base/ixgbe_hv_vf.c b/dpdk/drivers/net/ixgbe/base/ixgbe_hv_vf.c new file mode 100644 index 00000000..47143a26 --- /dev/null +++ b/dpdk/drivers/net/ixgbe/base/ixgbe_hv_vf.c @@ -0,0 +1,240 @@ +/******************************************************************************* + +Copyright (c) 2001-2015, Intel Corporation +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. Neither the name of the Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +***************************************************************************/ + +#include "ixgbe_vf.h" +#include "ixgbe_hv_vf.h" + +/** + * Hyper-V variant - just a stub. + */ +static s32 ixgbevf_hv_update_mc_addr_list_vf(struct ixgbe_hw *hw, u8 *mc_addr_list, + u32 mc_addr_count, ixgbe_mc_addr_itr next, + bool clear) +{ + UNREFERENCED_5PARAMETER(hw, mc_addr_list, mc_addr_count, next, clear); + + return IXGBE_ERR_FEATURE_NOT_SUPPORTED; +} + +/** + * Hyper-V variant - just a stub. + */ +static s32 ixgbevf_hv_update_xcast_mode(struct ixgbe_hw *hw, int xcast_mode) +{ + UNREFERENCED_2PARAMETER(hw, xcast_mode); + + return IXGBE_ERR_FEATURE_NOT_SUPPORTED; +} + +/** + * Hyper-V variant - just a stub. + */ +static s32 ixgbevf_hv_set_vfta_vf(struct ixgbe_hw *hw, u32 vlan, u32 vind, + bool vlan_on, bool vlvf_bypass) +{ + UNREFERENCED_5PARAMETER(hw, vlan, vind, vlan_on, vlvf_bypass); + + return IXGBE_ERR_FEATURE_NOT_SUPPORTED; +} + +static s32 ixgbevf_hv_set_uc_addr_vf(struct ixgbe_hw *hw, u32 index, u8 *addr) +{ + UNREFERENCED_3PARAMETER(hw, index, addr); + + return IXGBE_ERR_FEATURE_NOT_SUPPORTED; +} + +/** + * Hyper-V variant - just a stub. + */ +static s32 ixgbevf_hv_reset_hw_vf(struct ixgbe_hw *hw) +{ + UNREFERENCED_PARAMETER(hw); + + return IXGBE_ERR_FEATURE_NOT_SUPPORTED; +} + +/** + * Hyper-V variant - just a stub. + */ +static s32 ixgbevf_hv_set_rar_vf(struct ixgbe_hw *hw, u32 index, u8 *addr, u32 vlan, u32 vind) +{ + UNREFERENCED_5PARAMETER(hw, index, addr, vlan, vind); + + return IXGBE_ERR_FEATURE_NOT_SUPPORTED; +} + +/** + * Hyper-V variant; there is no mailbox communication. + */ +static s32 ixgbevf_hv_check_mac_link_vf(struct ixgbe_hw *hw, + ixgbe_link_speed *speed, + bool *link_up, + bool autoneg_wait_to_complete) +{ + struct ixgbe_mbx_info *mbx = &hw->mbx; + struct ixgbe_mac_info *mac = &hw->mac; + u32 links_reg; + UNREFERENCED_1PARAMETER(autoneg_wait_to_complete); + + /* If we were hit with a reset drop the link */ + if (!mbx->ops.check_for_rst(hw, 0) || !mbx->timeout) + mac->get_link_status = true; + + if (!mac->get_link_status) + goto out; + + /* if link status is down no point in checking to see if pf is up */ + links_reg = IXGBE_READ_REG(hw, IXGBE_VFLINKS); + if (!(links_reg & IXGBE_LINKS_UP)) + goto out; + + /* for SFP+ modules and DA cables on 82599 it can take up to 500usecs + * before the link status is correct + */ + if (mac->type == ixgbe_mac_82599_vf) { + int i; + + for (i = 0; i < 5; i++) { + DELAY(100); + links_reg = IXGBE_READ_REG(hw, IXGBE_VFLINKS); + + if (!(links_reg & IXGBE_LINKS_UP)) + goto out; + } + } + + switch (links_reg & IXGBE_LINKS_SPEED_82599) { + case IXGBE_LINKS_SPEED_10G_82599: + *speed = IXGBE_LINK_SPEED_10GB_FULL; + if (hw->mac.type >= ixgbe_mac_X550) { + if (links_reg & IXGBE_LINKS_SPEED_NON_STD) + *speed = IXGBE_LINK_SPEED_2_5GB_FULL; + } + break; + case IXGBE_LINKS_SPEED_1G_82599: + *speed = IXGBE_LINK_SPEED_1GB_FULL; + break; + case IXGBE_LINKS_SPEED_100_82599: + *speed = IXGBE_LINK_SPEED_100_FULL; + if (hw->mac.type == ixgbe_mac_X550) { + if (links_reg & IXGBE_LINKS_SPEED_NON_STD) + *speed = IXGBE_LINK_SPEED_5GB_FULL; + } + break; + case IXGBE_LINKS_SPEED_10_X550EM_A: + *speed = IXGBE_LINK_SPEED_UNKNOWN; + /* Reserved for pre-x550 devices */ + if (hw->mac.type >= ixgbe_mac_X550) + *speed = IXGBE_LINK_SPEED_10_FULL; + break; + default: + *speed = IXGBE_LINK_SPEED_UNKNOWN; + } + + /* if we passed all the tests above then the link is up and we no + * longer need to check for link + */ + mac->get_link_status = false; + +out: + *link_up = !mac->get_link_status; + return IXGBE_SUCCESS; +} + +/** + * ixgbevf_hv_set_rlpml_vf - Set the maximum receive packet length + * @hw: pointer to the HW structure + * @max_size: value to assign to max frame size + * Hyper-V variant. + **/ +static s32 ixgbevf_hv_set_rlpml_vf(struct ixgbe_hw *hw, u16 max_size) +{ + u32 reg; + + /* If we are on Hyper-V, we implement this functionality + * differently. + */ + reg = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(0)); + /* CRC == 4 */ + reg |= ((max_size + 4) | IXGBE_RXDCTL_RLPML_EN); + IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(0), reg); + + return IXGBE_SUCCESS; +} + +/** + * ixgbevf_hv_negotiate_api_version_vf - Negotiate supported API version + * @hw: pointer to the HW structure + * @api: integer containing requested API version + * Hyper-V version - only ixgbe_mbox_api_10 supported. + **/ +static int ixgbevf_hv_negotiate_api_version_vf(struct ixgbe_hw *hw, int api) +{ + UNREFERENCED_1PARAMETER(hw); + + /* Hyper-V only supports api version ixgbe_mbox_api_10 */ + if (api != ixgbe_mbox_api_10) + return IXGBE_ERR_INVALID_ARGUMENT; + + return IXGBE_SUCCESS; +} + +/** + * ixgbevf_hv_init_ops_vf - Initialize the pointers for vf + * @hw: pointer to hardware structure + * + * This will assign function pointers, adapter-specific functions can + * override the assignment of generic function pointers by assigning + * their own adapter-specific function pointers. + * Does not touch the hardware. + **/ +s32 ixgbevf_hv_init_ops_vf(struct ixgbe_hw *hw) +{ + /* Set defaults for VF then override applicable Hyper-V + * specific functions + */ + ixgbe_init_ops_vf(hw); + + hw->mac.ops.reset_hw = ixgbevf_hv_reset_hw_vf; + hw->mac.ops.check_link = ixgbevf_hv_check_mac_link_vf; + hw->mac.ops.negotiate_api_version = ixgbevf_hv_negotiate_api_version_vf; + hw->mac.ops.set_rar = ixgbevf_hv_set_rar_vf; + hw->mac.ops.update_mc_addr_list = ixgbevf_hv_update_mc_addr_list_vf; + hw->mac.ops.update_xcast_mode = ixgbevf_hv_update_xcast_mode; + hw->mac.ops.set_uc_addr = ixgbevf_hv_set_uc_addr_vf; + hw->mac.ops.set_vfta = ixgbevf_hv_set_vfta_vf; + hw->mac.ops.set_rlpml = ixgbevf_hv_set_rlpml_vf; + + return IXGBE_SUCCESS; +} diff --git a/dpdk/drivers/net/ixgbe/base/ixgbe_hv_vf.h b/dpdk/drivers/net/ixgbe/base/ixgbe_hv_vf.h new file mode 100644 index 00000000..9119f29f --- /dev/null +++ b/dpdk/drivers/net/ixgbe/base/ixgbe_hv_vf.h @@ -0,0 +1,41 @@ +/******************************************************************************* + +Copyright (c) 2001-2016, Intel Corporation +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. Neither the name of the Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +***************************************************************************/ + +#ifndef _IXGBE_HV_VF_H_ +#define _IXGBE_HV_VF_H_ + +#include "ixgbe_type.h" + +s32 ixgbevf_hv_init_ops_vf(struct ixgbe_hw *hw); + +#endif /* _IXGBE_HV_VF_H_ */ diff --git a/dpdk/drivers/net/ixgbe/base/ixgbe_mbx.h b/dpdk/drivers/net/ixgbe/base/ixgbe_mbx.h index d775142d..bde50a51 100644 --- a/dpdk/drivers/net/ixgbe/base/ixgbe_mbx.h +++ b/dpdk/drivers/net/ixgbe/base/ixgbe_mbx.h @@ -90,6 +90,7 @@ enum ixgbe_pfvf_api_rev { ixgbe_mbox_api_20, /* API version 2.0, solaris Phase1 VF driver */ ixgbe_mbox_api_11, /* API version 1.1, linux/freebsd VF driver */ ixgbe_mbox_api_12, /* API version 1.2, linux/freebsd VF driver */ + ixgbe_mbox_api_13, /* API version 1.3, linux/freebsd VF driver */ /* This value should always be last */ ixgbe_mbox_api_unknown, /* indicates that API version is not known */ }; @@ -109,9 +110,17 @@ enum ixgbe_pfvf_api_rev { #define IXGBE_VF_GET_QUEUES 0x09 /* get queue configuration */ /* mailbox API, version 1.2 VF requests */ -#define IXGBE_VF_GET_RETA 0x0a /* VF request for RETA */ -#define IXGBE_VF_GET_RSS_KEY 0x0b /* get RSS key */ -#define IXGBE_VF_UPDATE_XCAST_MODE 0x0C +#define IXGBE_VF_GET_RETA 0x0a /* VF request for RETA */ +#define IXGBE_VF_GET_RSS_KEY 0x0b /* get RSS key */ +#define IXGBE_VF_UPDATE_XCAST_MODE 0x0c + +/* mode choices for IXGBE_VF_UPDATE_XCAST_MODE */ +enum ixgbevf_xcast_modes { + IXGBEVF_XCAST_MODE_NONE = 0, + IXGBEVF_XCAST_MODE_MULTI, + IXGBEVF_XCAST_MODE_ALLMULTI, + IXGBEVF_XCAST_MODE_PROMISC, +}; /* GET_QUEUES return data indices within the mailbox */ #define IXGBE_VF_TX_QUEUES 1 /* number of Tx queues supported */ diff --git a/dpdk/drivers/net/ixgbe/base/ixgbe_osdep.h b/dpdk/drivers/net/ixgbe/base/ixgbe_osdep.h index 06d1ee1c..bb5dfd2a 100644 --- a/dpdk/drivers/net/ixgbe/base/ixgbe_osdep.h +++ b/dpdk/drivers/net/ixgbe/base/ixgbe_osdep.h @@ -44,6 +44,7 @@ #include #include #include +#include #include "../ixgbe_logs.h" #include "../ixgbe_bypass_defines.h" @@ -81,6 +82,7 @@ #define UNREFERENCED_2PARAMETER(_p, _q) #define UNREFERENCED_3PARAMETER(_p, _q, _r) #define UNREFERENCED_4PARAMETER(_p, _q, _r, _s) +#define UNREFERENCED_5PARAMETER(_p, _q, _r, _s, _t) /* Shared code error reporting */ enum { @@ -95,11 +97,13 @@ enum { #define STATIC static #define IXGBE_NTOHL(_i) rte_be_to_cpu_32(_i) #define IXGBE_NTOHS(_i) rte_be_to_cpu_16(_i) +#define IXGBE_CPU_TO_LE16(_i) rte_cpu_to_le_16(_i) #define IXGBE_CPU_TO_LE32(_i) rte_cpu_to_le_32(_i) -#define IXGBE_LE32_TO_CPU(_i) rte_le_to_cpu_32(_i) +#define IXGBE_LE32_TO_CPU(_i) rte_le_to_cpu_32(_i) #define IXGBE_LE32_TO_CPUS(_i) rte_le_to_cpu_32(_i) #define IXGBE_CPU_TO_BE16(_i) rte_cpu_to_be_16(_i) #define IXGBE_CPU_TO_BE32(_i) rte_cpu_to_be_32(_i) +#define IXGBE_BE32_TO_CPU(_i) rte_be_to_cpu_32(_i) typedef uint8_t u8; typedef int8_t s8; @@ -120,16 +124,18 @@ typedef int bool; #define prefetch(x) rte_prefetch0(x) -#define IXGBE_PCI_REG(reg) (*((volatile uint32_t *)(reg))) +#define IXGBE_PCI_REG(reg) rte_read32(reg) static inline uint32_t ixgbe_read_addr(volatile void* addr) { return rte_le_to_cpu_32(IXGBE_PCI_REG(addr)); } -#define IXGBE_PCI_REG_WRITE(reg, value) do { \ - IXGBE_PCI_REG((reg)) = (rte_cpu_to_le_32(value)); \ -} while(0) +#define IXGBE_PCI_REG_WRITE(reg, value) \ + rte_write32((rte_cpu_to_le_32(value)), reg) + +#define IXGBE_PCI_REG_WRITE_RELAXED(reg, value) \ + rte_write32_relaxed((rte_cpu_to_le_32(value)), reg) #define IXGBE_PCI_REG_ADDR(hw, reg) \ ((volatile uint32_t *)((char *)(hw)->hw_addr + (reg))) @@ -155,4 +161,12 @@ static inline uint32_t ixgbe_read_addr(volatile void* addr) #define IXGBE_WRITE_REG_ARRAY(hw, reg, index, value) \ IXGBE_PCI_REG_WRITE(IXGBE_PCI_REG_ARRAY_ADDR((hw), (reg), (index)), (value)) +#define IXGBE_WRITE_REG_THEN_POLL_MASK(hw, reg, val, mask, poll_ms) \ +do { \ + uint32_t cnt = poll_ms; \ + IXGBE_WRITE_REG(hw, (reg), (val)); \ + while (((IXGBE_READ_REG(hw, (reg))) & (mask)) && (cnt--)) \ + rte_delay_ms(1); \ +} while (0) + #endif /* _IXGBE_OS_H_ */ diff --git a/dpdk/drivers/net/ixgbe/base/ixgbe_phy.c b/dpdk/drivers/net/ixgbe/base/ixgbe_phy.c index ed1b14f3..62c3080a 100644 --- a/dpdk/drivers/net/ixgbe/base/ixgbe_phy.c +++ b/dpdk/drivers/net/ixgbe/base/ixgbe_phy.c @@ -113,7 +113,7 @@ s32 ixgbe_read_i2c_combined_generic_int(struct ixgbe_hw *hw, u8 addr, u16 reg, u16 *val, bool lock) { u32 swfw_mask = hw->phy.phy_semaphore_mask; - int max_retry = 10; + int max_retry = 3; int retry = 0; u8 csum_byte; u8 high_bits; @@ -121,8 +121,6 @@ s32 ixgbe_read_i2c_combined_generic_int(struct ixgbe_hw *hw, u8 addr, u16 reg, u8 reg_high; u8 csum; - if (hw->mac.type >= ixgbe_mac_X550) - max_retry = 3; reg_high = ((reg >> 7) & 0xFE) | 1; /* Indicate read combined */ csum = ixgbe_ones_comp_byte_add(reg_high, reg & 0xFF); csum = ~csum; @@ -282,6 +280,42 @@ s32 ixgbe_init_phy_ops_generic(struct ixgbe_hw *hw) return IXGBE_SUCCESS; } +/** + * ixgbe_probe_phy - Probe a single address for a PHY + * @hw: pointer to hardware structure + * @phy_addr: PHY address to probe + * + * Returns true if PHY found + */ +static bool ixgbe_probe_phy(struct ixgbe_hw *hw, u16 phy_addr) +{ + u16 ext_ability = 0; + + if (!ixgbe_validate_phy_addr(hw, phy_addr)) { + DEBUGOUT1("Unable to validate PHY address 0x%04X\n", + phy_addr); + return false; + } + + if (ixgbe_get_phy_id(hw)) + return false; + + hw->phy.type = ixgbe_get_phy_type_from_id(hw->phy.id); + + if (hw->phy.type == ixgbe_phy_unknown) { + hw->phy.ops.read_reg(hw, IXGBE_MDIO_PHY_EXT_ABILITY, + IXGBE_MDIO_PMA_PMD_DEV_TYPE, &ext_ability); + if (ext_ability & + (IXGBE_MDIO_PHY_10GBASET_ABILITY | + IXGBE_MDIO_PHY_1000BASET_ABILITY)) + hw->phy.type = ixgbe_phy_cu_unknown; + else + hw->phy.type = ixgbe_phy_generic; + } + + return true; +} + /** * ixgbe_identify_phy_generic - Get physical layer module * @hw: pointer to hardware structure @@ -291,8 +325,7 @@ s32 ixgbe_init_phy_ops_generic(struct ixgbe_hw *hw) s32 ixgbe_identify_phy_generic(struct ixgbe_hw *hw) { s32 status = IXGBE_ERR_PHY_ADDR_INVALID; - u32 phy_addr; - u16 ext_ability = 0; + u16 phy_addr; DEBUGFUNC("ixgbe_identify_phy_generic"); @@ -303,45 +336,33 @@ s32 ixgbe_identify_phy_generic(struct ixgbe_hw *hw) hw->phy.phy_semaphore_mask = IXGBE_GSSR_PHY0_SM; } - if (hw->phy.type == ixgbe_phy_unknown) { - for (phy_addr = 0; phy_addr < IXGBE_MAX_PHY_ADDR; phy_addr++) { - if (ixgbe_validate_phy_addr(hw, phy_addr)) { - hw->phy.addr = phy_addr; - ixgbe_get_phy_id(hw); - hw->phy.type = - ixgbe_get_phy_type_from_id(hw->phy.id); + if (hw->phy.type != ixgbe_phy_unknown) + return IXGBE_SUCCESS; - if (hw->phy.type == ixgbe_phy_unknown) { - hw->phy.ops.read_reg(hw, - IXGBE_MDIO_PHY_EXT_ABILITY, - IXGBE_MDIO_PMA_PMD_DEV_TYPE, - &ext_ability); - if (ext_ability & - (IXGBE_MDIO_PHY_10GBASET_ABILITY | - IXGBE_MDIO_PHY_1000BASET_ABILITY)) - hw->phy.type = - ixgbe_phy_cu_unknown; - else - hw->phy.type = - ixgbe_phy_generic; - } - - status = IXGBE_SUCCESS; - break; - } - } - - /* Certain media types do not have a phy so an address will not - * be found and the code will take this path. Caller has to - * decide if it is an error or not. - */ - if (status != IXGBE_SUCCESS) { - hw->phy.addr = 0; - } - } else { - status = IXGBE_SUCCESS; + if (hw->phy.nw_mng_if_sel) { + phy_addr = (hw->phy.nw_mng_if_sel & + IXGBE_NW_MNG_IF_SEL_MDIO_PHY_ADD) >> + IXGBE_NW_MNG_IF_SEL_MDIO_PHY_ADD_SHIFT; + if (ixgbe_probe_phy(hw, phy_addr)) + return IXGBE_SUCCESS; + else + return IXGBE_ERR_PHY_ADDR_INVALID; } + for (phy_addr = 0; phy_addr < IXGBE_MAX_PHY_ADDR; phy_addr++) { + if (ixgbe_probe_phy(hw, phy_addr)) { + status = IXGBE_SUCCESS; + break; + } + } + + /* Certain media types do not have a phy so an address will not + * be found and the code will take this path. Caller has to + * decide if it is an error or not. + */ + if (status != IXGBE_SUCCESS) + hw->phy.addr = 0; + return status; } @@ -393,6 +414,8 @@ bool ixgbe_validate_phy_addr(struct ixgbe_hw *hw, u32 phy_addr) if (phy_id != 0xFFFF && phy_id != 0x0) valid = true; + DEBUGOUT1("PHY ID HIGH is 0x%04X\n", phy_id); + return valid; } @@ -421,6 +444,9 @@ s32 ixgbe_get_phy_id(struct ixgbe_hw *hw) hw->phy.id |= (u32)(phy_id_low & IXGBE_PHY_REVISION_MASK); hw->phy.revision = (u32)(phy_id_low & ~IXGBE_PHY_REVISION_MASK); } + DEBUGOUT2("PHY_ID_HIGH 0x%04X, PHY_ID_LOW 0x%04X\n", + phy_id_high, phy_id_low); + return status; } @@ -439,7 +465,6 @@ enum ixgbe_phy_type ixgbe_get_phy_type_from_id(u32 phy_id) case TN1010_PHY_ID: phy_type = ixgbe_phy_tn; break; - case X550_PHY_ID1: case X550_PHY_ID2: case X550_PHY_ID3: case X540_PHY_ID: @@ -452,10 +477,12 @@ enum ixgbe_phy_type ixgbe_get_phy_type_from_id(u32 phy_id) phy_type = ixgbe_phy_nl; break; case X557_PHY_ID: + case X557_PHY_ID2: phy_type = ixgbe_phy_x550em_ext_t; break; case IXGBE_M88E1500_E_PHY_ID: - phy_type = ixgbe_phy_m88; + case IXGBE_M88E1543_E_PHY_ID: + phy_type = ixgbe_phy_ext_1g_t; break; default: phy_type = ixgbe_phy_unknown; @@ -506,11 +533,30 @@ s32 ixgbe_reset_phy_generic(struct ixgbe_hw *hw) */ for (i = 0; i < 30; i++) { msec_delay(100); - hw->phy.ops.read_reg(hw, IXGBE_MDIO_PHY_XS_CONTROL, - IXGBE_MDIO_PHY_XS_DEV_TYPE, &ctrl); - if (!(ctrl & IXGBE_MDIO_PHY_XS_RESET)) { - usec_delay(2); - break; + if (hw->phy.type == ixgbe_phy_x550em_ext_t) { + status = hw->phy.ops.read_reg(hw, + IXGBE_MDIO_TX_VENDOR_ALARMS_3, + IXGBE_MDIO_PMA_PMD_DEV_TYPE, + &ctrl); + if (status != IXGBE_SUCCESS) + return status; + + if (ctrl & IXGBE_MDIO_TX_VENDOR_ALARMS_3_RST_MASK) { + usec_delay(2); + break; + } + } else { + status = hw->phy.ops.read_reg(hw, + IXGBE_MDIO_PHY_XS_CONTROL, + IXGBE_MDIO_PHY_XS_DEV_TYPE, + &ctrl); + if (status != IXGBE_SUCCESS) + return status; + + if (!(ctrl & IXGBE_MDIO_PHY_XS_RESET)) { + usec_delay(2); + break; + } } } @@ -532,7 +578,7 @@ out: * @phy_data: Pointer to read data from PHY register **/ s32 ixgbe_read_phy_reg_mdi(struct ixgbe_hw *hw, u32 reg_addr, u32 device_type, - u16 *phy_data) + u16 *phy_data) { u32 i, data, command; @@ -554,12 +600,13 @@ s32 ixgbe_read_phy_reg_mdi(struct ixgbe_hw *hw, u32 reg_addr, u32 device_type, command = IXGBE_READ_REG(hw, IXGBE_MSCA); if ((command & IXGBE_MSCA_MDI_COMMAND) == 0) - break; + break; } if ((command & IXGBE_MSCA_MDI_COMMAND) != 0) { ERROR_REPORT1(IXGBE_ERROR_POLLING, "PHY address command did not complete.\n"); + DEBUGOUT("PHY address command did not complete, returning IXGBE_ERR_PHY\n"); return IXGBE_ERR_PHY; } @@ -589,6 +636,7 @@ s32 ixgbe_read_phy_reg_mdi(struct ixgbe_hw *hw, u32 reg_addr, u32 device_type, if ((command & IXGBE_MSCA_MDI_COMMAND) != 0) { ERROR_REPORT1(IXGBE_ERROR_POLLING, "PHY read command didn't complete\n"); + DEBUGOUT("PHY read command didn't complete, returning IXGBE_ERR_PHY\n"); return IXGBE_ERR_PHY; } @@ -719,7 +767,7 @@ s32 ixgbe_write_phy_reg_generic(struct ixgbe_hw *hw, u32 reg_addr, DEBUGFUNC("ixgbe_write_phy_reg_generic"); if (hw->mac.ops.acquire_swfw_sync(hw, gssr) == IXGBE_SUCCESS) { - status = ixgbe_write_phy_reg_mdi(hw, reg_addr, device_type, + status = hw->phy.ops.write_reg_mdi(hw, reg_addr, device_type, phy_data); hw->mac.ops.release_swfw_sync(hw, gssr); } else { @@ -746,91 +794,63 @@ s32 ixgbe_setup_phy_link_generic(struct ixgbe_hw *hw) ixgbe_get_copper_link_capabilities_generic(hw, &speed, &autoneg); - if (speed & IXGBE_LINK_SPEED_10GB_FULL) { - /* Set or unset auto-negotiation 10G advertisement */ - hw->phy.ops.read_reg(hw, IXGBE_MII_10GBASE_T_AUTONEG_CTRL_REG, - IXGBE_MDIO_AUTO_NEG_DEV_TYPE, - &autoneg_reg); + /* Set or unset auto-negotiation 10G advertisement */ + hw->phy.ops.read_reg(hw, IXGBE_MII_10GBASE_T_AUTONEG_CTRL_REG, + IXGBE_MDIO_AUTO_NEG_DEV_TYPE, + &autoneg_reg); - autoneg_reg &= ~IXGBE_MII_10GBASE_T_ADVERTISE; - if (hw->phy.autoneg_advertised & IXGBE_LINK_SPEED_10GB_FULL) - autoneg_reg |= IXGBE_MII_10GBASE_T_ADVERTISE; + autoneg_reg &= ~IXGBE_MII_10GBASE_T_ADVERTISE; + if ((hw->phy.autoneg_advertised & IXGBE_LINK_SPEED_10GB_FULL) && + (speed & IXGBE_LINK_SPEED_10GB_FULL)) + autoneg_reg |= IXGBE_MII_10GBASE_T_ADVERTISE; - hw->phy.ops.write_reg(hw, IXGBE_MII_10GBASE_T_AUTONEG_CTRL_REG, - IXGBE_MDIO_AUTO_NEG_DEV_TYPE, - autoneg_reg); - } + hw->phy.ops.write_reg(hw, IXGBE_MII_10GBASE_T_AUTONEG_CTRL_REG, + IXGBE_MDIO_AUTO_NEG_DEV_TYPE, + autoneg_reg); + + hw->phy.ops.read_reg(hw, IXGBE_MII_AUTONEG_VENDOR_PROVISION_1_REG, + IXGBE_MDIO_AUTO_NEG_DEV_TYPE, + &autoneg_reg); if (hw->mac.type == ixgbe_mac_X550) { - if (speed & IXGBE_LINK_SPEED_5GB_FULL) { - /* Set or unset auto-negotiation 5G advertisement */ - hw->phy.ops.read_reg(hw, - IXGBE_MII_AUTONEG_VENDOR_PROVISION_1_REG, - IXGBE_MDIO_AUTO_NEG_DEV_TYPE, - &autoneg_reg); + /* Set or unset auto-negotiation 5G advertisement */ + autoneg_reg &= ~IXGBE_MII_5GBASE_T_ADVERTISE; + if ((hw->phy.autoneg_advertised & IXGBE_LINK_SPEED_5GB_FULL) && + (speed & IXGBE_LINK_SPEED_5GB_FULL)) + autoneg_reg |= IXGBE_MII_5GBASE_T_ADVERTISE; - autoneg_reg &= ~IXGBE_MII_5GBASE_T_ADVERTISE; - if (hw->phy.autoneg_advertised & - IXGBE_LINK_SPEED_5GB_FULL) - autoneg_reg |= IXGBE_MII_5GBASE_T_ADVERTISE; - - hw->phy.ops.write_reg(hw, - IXGBE_MII_AUTONEG_VENDOR_PROVISION_1_REG, - IXGBE_MDIO_AUTO_NEG_DEV_TYPE, - autoneg_reg); - } - - if (speed & IXGBE_LINK_SPEED_2_5GB_FULL) { - /* Set or unset auto-negotiation 2.5G advertisement */ - hw->phy.ops.read_reg(hw, - IXGBE_MII_AUTONEG_VENDOR_PROVISION_1_REG, - IXGBE_MDIO_AUTO_NEG_DEV_TYPE, - &autoneg_reg); - - autoneg_reg &= ~IXGBE_MII_2_5GBASE_T_ADVERTISE; - if (hw->phy.autoneg_advertised & - IXGBE_LINK_SPEED_2_5GB_FULL) - autoneg_reg |= IXGBE_MII_2_5GBASE_T_ADVERTISE; - - hw->phy.ops.write_reg(hw, - IXGBE_MII_AUTONEG_VENDOR_PROVISION_1_REG, - IXGBE_MDIO_AUTO_NEG_DEV_TYPE, - autoneg_reg); - } + /* Set or unset auto-negotiation 2.5G advertisement */ + autoneg_reg &= ~IXGBE_MII_2_5GBASE_T_ADVERTISE; + if ((hw->phy.autoneg_advertised & + IXGBE_LINK_SPEED_2_5GB_FULL) && + (speed & IXGBE_LINK_SPEED_2_5GB_FULL)) + autoneg_reg |= IXGBE_MII_2_5GBASE_T_ADVERTISE; } - if (speed & IXGBE_LINK_SPEED_1GB_FULL) { - /* Set or unset auto-negotiation 1G advertisement */ - hw->phy.ops.read_reg(hw, - IXGBE_MII_AUTONEG_VENDOR_PROVISION_1_REG, - IXGBE_MDIO_AUTO_NEG_DEV_TYPE, - &autoneg_reg); + /* Set or unset auto-negotiation 1G advertisement */ + autoneg_reg &= ~IXGBE_MII_1GBASE_T_ADVERTISE; + if ((hw->phy.autoneg_advertised & IXGBE_LINK_SPEED_1GB_FULL) && + (speed & IXGBE_LINK_SPEED_1GB_FULL)) + autoneg_reg |= IXGBE_MII_1GBASE_T_ADVERTISE; - autoneg_reg &= ~IXGBE_MII_1GBASE_T_ADVERTISE; - if (hw->phy.autoneg_advertised & IXGBE_LINK_SPEED_1GB_FULL) - autoneg_reg |= IXGBE_MII_1GBASE_T_ADVERTISE; + hw->phy.ops.write_reg(hw, IXGBE_MII_AUTONEG_VENDOR_PROVISION_1_REG, + IXGBE_MDIO_AUTO_NEG_DEV_TYPE, + autoneg_reg); - hw->phy.ops.write_reg(hw, - IXGBE_MII_AUTONEG_VENDOR_PROVISION_1_REG, - IXGBE_MDIO_AUTO_NEG_DEV_TYPE, - autoneg_reg); - } + /* Set or unset auto-negotiation 100M advertisement */ + hw->phy.ops.read_reg(hw, IXGBE_MII_AUTONEG_ADVERTISE_REG, + IXGBE_MDIO_AUTO_NEG_DEV_TYPE, + &autoneg_reg); - if (speed & IXGBE_LINK_SPEED_100_FULL) { - /* Set or unset auto-negotiation 100M advertisement */ - hw->phy.ops.read_reg(hw, IXGBE_MII_AUTONEG_ADVERTISE_REG, - IXGBE_MDIO_AUTO_NEG_DEV_TYPE, - &autoneg_reg); + autoneg_reg &= ~(IXGBE_MII_100BASE_T_ADVERTISE | + IXGBE_MII_100BASE_T_ADVERTISE_HALF); + if ((hw->phy.autoneg_advertised & IXGBE_LINK_SPEED_100_FULL) && + (speed & IXGBE_LINK_SPEED_100_FULL)) + autoneg_reg |= IXGBE_MII_100BASE_T_ADVERTISE; - autoneg_reg &= ~(IXGBE_MII_100BASE_T_ADVERTISE | - IXGBE_MII_100BASE_T_ADVERTISE_HALF); - if (hw->phy.autoneg_advertised & IXGBE_LINK_SPEED_100_FULL) - autoneg_reg |= IXGBE_MII_100BASE_T_ADVERTISE; - - hw->phy.ops.write_reg(hw, IXGBE_MII_AUTONEG_ADVERTISE_REG, - IXGBE_MDIO_AUTO_NEG_DEV_TYPE, - autoneg_reg); - } + hw->phy.ops.write_reg(hw, IXGBE_MII_AUTONEG_ADVERTISE_REG, + IXGBE_MDIO_AUTO_NEG_DEV_TYPE, + autoneg_reg); /* Blocked by MNG FW so don't reset PHY */ if (ixgbe_check_reset_blocked(hw)) @@ -882,6 +902,9 @@ s32 ixgbe_setup_phy_link_speed_generic(struct ixgbe_hw *hw, if (speed & IXGBE_LINK_SPEED_100_FULL) hw->phy.autoneg_advertised |= IXGBE_LINK_SPEED_100_FULL; + if (speed & IXGBE_LINK_SPEED_10_FULL) + hw->phy.autoneg_advertised |= IXGBE_LINK_SPEED_10_FULL; + /* Setup link based on the new speed settings */ ixgbe_setup_phy_link(hw); @@ -919,6 +942,7 @@ static s32 ixgbe_get_copper_speeds_supported(struct ixgbe_hw *hw) hw->phy.speeds_supported |= IXGBE_LINK_SPEED_5GB_FULL; break; case ixgbe_mac_X550EM_x: + case ixgbe_mac_X550EM_a: hw->phy.speeds_supported &= ~IXGBE_LINK_SPEED_100_FULL; break; default: @@ -1516,16 +1540,10 @@ s32 ixgbe_identify_sfp_module_generic(struct ixgbe_hw *hw) status = IXGBE_SUCCESS; } else { if (hw->allow_unsupported_sfp == true) { - EWARN(hw, "WARNING: Intel (R) Network " - "Connections are quality tested " - "using Intel (R) Ethernet Optics." - " Using untested modules is not " - "supported and may cause unstable" - " operation or damage to the " - "module or the adapter. Intel " - "Corporation is not responsible " - "for any harm caused by using " - "untested modules.\n", status); + EWARN(hw, + "WARNING: Intel (R) Network Connections are quality tested using Intel (R) Ethernet Optics. " + "Using untested modules is not supported and may cause unstable operation or damage to the module or the adapter. " + "Intel Corporation is not responsible for any harm caused by using untested modules.\n"); status = IXGBE_SUCCESS; } else { DEBUGOUT("SFP+ module not supported\n"); @@ -1557,9 +1575,9 @@ err_read_i2c_eeprom: * * Determines physical layer capabilities of the current SFP. */ -s32 ixgbe_get_supported_phy_sfp_layer_generic(struct ixgbe_hw *hw) +u64 ixgbe_get_supported_phy_sfp_layer_generic(struct ixgbe_hw *hw) { - u32 physical_layer = IXGBE_PHYSICAL_LAYER_UNKNOWN; + u64 physical_layer = IXGBE_PHYSICAL_LAYER_UNKNOWN; u8 comp_codes_10g = 0; u8 comp_codes_1g = 0; @@ -1778,16 +1796,10 @@ s32 ixgbe_identify_qsfp_module_generic(struct ixgbe_hw *hw) status = IXGBE_SUCCESS; } else { if (hw->allow_unsupported_sfp == true) { - EWARN(hw, "WARNING: Intel (R) Network " - "Connections are quality tested " - "using Intel (R) Ethernet Optics." - " Using untested modules is not " - "supported and may cause unstable" - " operation or damage to the " - "module or the adapter. Intel " - "Corporation is not responsible " - "for any harm caused by using " - "untested modules.\n", status); + EWARN(hw, + "WARNING: Intel (R) Network Connections are quality tested using Intel (R) Ethernet Optics. " + "Using untested modules is not supported and may cause unstable operation or damage to the module or the adapter. " + "Intel Corporation is not responsible for any harm caused by using untested modules.\n"); status = IXGBE_SUCCESS; } else { DEBUGOUT("QSFP module not supported\n"); @@ -1812,7 +1824,6 @@ err_read_i2c_eeprom: return IXGBE_ERR_SFP_NOT_PRESENT; } - /** * ixgbe_get_sfp_init_sequence_offsets - Provides offset of PHY init sequence * @hw: pointer to hardware structure diff --git a/dpdk/drivers/net/ixgbe/base/ixgbe_phy.h b/dpdk/drivers/net/ixgbe/base/ixgbe_phy.h index 281f9faf..cf8cadd9 100644 --- a/dpdk/drivers/net/ixgbe/base/ixgbe_phy.h +++ b/dpdk/drivers/net/ixgbe/base/ixgbe_phy.h @@ -92,8 +92,9 @@ POSSIBILITY OF SUCH DAMAGE. #define IXGBE_CS4227_GLOBAL_ID_MSB 1 #define IXGBE_CS4227_SCRATCH 2 #define IXGBE_CS4227_GLOBAL_ID_VALUE 0x03E5 -#define IXGBE_CS4223_PHY_ID 0x7003/* Quad port */ -#define IXGBE_CS4227_PHY_ID 0x3003/* Dual port */ +#define IXGBE_CS4227_EFUSE_PDF_SKU 0x19F +#define IXGBE_CS4223_SKU_ID 0x0010 /* Quad port */ +#define IXGBE_CS4227_SKU_ID 0x0014 /* Dual port */ #define IXGBE_CS4227_RESET_PENDING 0x1357 #define IXGBE_CS4227_RESET_COMPLETE 0x5AA5 #define IXGBE_CS4227_RETRIES 15 @@ -191,7 +192,7 @@ s32 ixgbe_reset_phy_nl(struct ixgbe_hw *hw); s32 ixgbe_set_copper_phy_power(struct ixgbe_hw *hw, bool on); s32 ixgbe_identify_module_generic(struct ixgbe_hw *hw); s32 ixgbe_identify_sfp_module_generic(struct ixgbe_hw *hw); -s32 ixgbe_get_supported_phy_sfp_layer_generic(struct ixgbe_hw *hw); +u64 ixgbe_get_supported_phy_sfp_layer_generic(struct ixgbe_hw *hw); s32 ixgbe_identify_qsfp_module_generic(struct ixgbe_hw *hw); s32 ixgbe_get_sfp_init_sequence_offsets(struct ixgbe_hw *hw, u16 *list_offset, diff --git a/dpdk/drivers/net/ixgbe/base/ixgbe_type.h b/dpdk/drivers/net/ixgbe/base/ixgbe_type.h index 83818a96..bda85589 100644 --- a/dpdk/drivers/net/ixgbe/base/ixgbe_type.h +++ b/dpdk/drivers/net/ixgbe/base/ixgbe_type.h @@ -105,11 +105,11 @@ POSSIBILITY OF SUCH DAMAGE. #define IXGBE_SUBDEV_ID_82599_560FLR 0x17D0 #define IXGBE_SUBDEV_ID_82599_ECNA_DP 0x0470 #define IXGBE_SUBDEV_ID_82599_SP_560FLR 0x211B -#define IXGBE_SUBDEV_ID_82599_LOM_SFP 0x8976 #define IXGBE_SUBDEV_ID_82599_LOM_SNAP6 0x2159 #define IXGBE_SUBDEV_ID_82599_SFP_1OCP 0x000D #define IXGBE_SUBDEV_ID_82599_SFP_2OCP 0x0008 -#define IXGBE_SUBDEV_ID_82599_SFP_LOM 0x06EE +#define IXGBE_SUBDEV_ID_82599_SFP_LOM_OEM1 0x8976 +#define IXGBE_SUBDEV_ID_82599_SFP_LOM_OEM2 0x06EE #define IXGBE_DEV_ID_82599_BACKPLANE_FCOE 0x152A #define IXGBE_DEV_ID_82599_SFP_FCOE 0x1529 #define IXGBE_DEV_ID_82599_SFP_EM 0x1507 @@ -146,6 +146,7 @@ POSSIBILITY OF SUCH DAMAGE. #define IXGBE_DEV_ID_X550EM_X_SFP 0x15AC #define IXGBE_DEV_ID_X550EM_X_10G_T 0x15AD #define IXGBE_DEV_ID_X550EM_X_1G_T 0x15AE +#define IXGBE_DEV_ID_X550EM_X_XFI 0x15B0 #define IXGBE_DEV_ID_X550_VF_HV 0x1564 #define IXGBE_DEV_ID_X550_VF 0x1565 #define IXGBE_DEV_ID_X550EM_A_VF 0x15C5 @@ -565,6 +566,13 @@ struct ixgbe_thermal_sensor_data { #define IXGBE_PROXYFC 0x05F64 /* Proxying Filter Control Register */ #define IXGBE_VXLANCTRL 0x0000507C /* Rx filter VXLAN UDPPORT Register */ +/* masks for accessing VXLAN and GENEVE UDP ports */ +#define IXGBE_VXLANCTRL_VXLAN_UDPPORT_MASK 0x0000ffff /* VXLAN port */ +#define IXGBE_VXLANCTRL_GENEVE_UDPPORT_MASK 0xffff0000 /* GENEVE port */ +#define IXGBE_VXLANCTRL_ALL_UDPPORT_MASK 0xffffffff /* GENEVE/VXLAN */ + +#define IXGBE_VXLANCTRL_GENEVE_UDPPORT_SHIFT 16 + #define IXGBE_FHFT(_n) (0x09000 + ((_n) * 0x100)) /* Flex host filter table */ /* Ext Flexible Host Filter Table */ #define IXGBE_FHFT_EXT(_n) (0x09800 + ((_n) * 0x100)) @@ -1038,7 +1046,7 @@ struct ixgbe_dmac_config { #define IXGBE_FTFT 0x09400 /* 0x9400-0x97FC */ #define IXGBE_METF(_i) (0x05190 + ((_i) * 4)) /* 4 of these (0-3) */ #define IXGBE_MDEF_EXT(_i) (0x05160 + ((_i) * 4)) /* 8 of these (0-7) */ -#define IXGBE_LSWFW 0x15014 +#define IXGBE_LSWFW 0x15F14 #define IXGBE_BMCIP(_i) (0x05050 + ((_i) * 4)) /* 0x5050-0x505C */ #define IXGBE_BMCIPVAL 0x05060 #define IXGBE_BMCIP_IPADDR_TYPE 0x00000001 @@ -1640,17 +1648,17 @@ struct ixgbe_dmac_config { #define TN1010_PHY_ID 0x00A19410 #define TNX_FW_REV 0xB #define X540_PHY_ID 0x01540200 -#define X550_PHY_ID1 0x01540220 #define X550_PHY_ID2 0x01540223 #define X550_PHY_ID3 0x01540221 #define X557_PHY_ID 0x01540240 +#define X557_PHY_ID2 0x01540250 #define AQ_FW_REV 0x20 #define QT2022_PHY_ID 0x0043A400 #define ATH_PHY_ID 0x03429050 /* PHY Types */ -#define IXGBE_M88E1500_E_PHY_ID 0x01410DD0 -#define IXGBE_M88E1543_E_PHY_ID 0x01410EA0 +#define IXGBE_M88E1500_E_PHY_ID 0x01410DD0 +#define IXGBE_M88E1543_E_PHY_ID 0x01410EA0 /* Special PHY Init Routine */ #define IXGBE_PHY_INIT_OFFSET_NL 0x002B @@ -1765,6 +1773,8 @@ enum { #define IXGBE_VT_CTL_POOL_MASK (0x3F << IXGBE_VT_CTL_POOL_SHIFT) /* VMOLR bitmasks */ +#define IXGBE_VMOLR_UPE 0x00400000 /* unicast promiscuous */ +#define IXGBE_VMOLR_VPE 0x00800000 /* VLAN promiscuous */ #define IXGBE_VMOLR_AUPE 0x01000000 /* accept untagged packets */ #define IXGBE_VMOLR_ROMPE 0x02000000 /* accept packets in MTA tbl */ #define IXGBE_VMOLR_ROPE 0x04000000 /* accept packets in UC tbl */ @@ -2203,6 +2213,7 @@ enum { #define IXGBE_LINKS_SPEED_10G_82599 0x30000000 #define IXGBE_LINKS_SPEED_1G_82599 0x20000000 #define IXGBE_LINKS_SPEED_100_82599 0x10000000 +#define IXGBE_LINKS_SPEED_10_X550EM_A 0x00000000 #define IXGBE_LINK_UP_TIME 90 /* 9.0 Seconds */ #define IXGBE_AUTO_NEG_TIME 45 /* 4.5 Seconds */ @@ -2335,7 +2346,9 @@ enum { #define IXGBE_SAN_MAC_ADDR_PTR 0x28 #define IXGBE_DEVICE_CAPS 0x2C -#define IXGBE_SERIAL_NUMBER_MAC_ADDR 0x11 +#define IXGBE_82599_SERIAL_NUMBER_MAC_ADDR 0x11 +#define IXGBE_X550_SERIAL_NUMBER_MAC_ADDR 0x04 + #define IXGBE_PCIE_MSIX_82599_CAPS 0x72 #define IXGBE_MAX_MSIX_VECTORS_82599 0x40 #define IXGBE_PCIE_MSIX_82598_CAPS 0x62 @@ -2609,6 +2622,7 @@ enum { #define IXGBE_MRQC_VMDQRSS64EN 0x0000000B /* VMDq2 64 pools w/ RSS */ #define IXGBE_MRQC_VMDQRT8TCEN 0x0000000C /* VMDq2/RT 16 pool 8 TC */ #define IXGBE_MRQC_VMDQRT4TCEN 0x0000000D /* VMDq2/RT 32 pool 4 TC */ +#define IXGBE_MRQC_L3L4TXSWEN 0x00008000 /* Enable L3/L4 Tx switch */ #define IXGBE_MRQC_RSS_FIELD_MASK 0xFFFF0000 #define IXGBE_MRQC_RSS_FIELD_IPV4_TCP 0x00010000 #define IXGBE_MRQC_RSS_FIELD_IPV4 0x00020000 @@ -2780,6 +2794,7 @@ enum { #define IXGBE_RXDADV_PKTTYPE_UDP 0x00000200 /* UDP hdr present */ #define IXGBE_RXDADV_PKTTYPE_SCTP 0x00000400 /* SCTP hdr present */ #define IXGBE_RXDADV_PKTTYPE_NFS 0x00000800 /* NFS hdr present */ +#define IXGBE_RXDADV_PKTTYPE_GENEVE 0x00000800 /* GENEVE hdr present */ #define IXGBE_RXDADV_PKTTYPE_VXLAN 0x00000800 /* VXLAN hdr present */ #define IXGBE_RXDADV_PKTTYPE_TUNNEL 0x00010000 /* Tunnel type */ #define IXGBE_RXDADV_PKTTYPE_IPSEC_ESP 0x00001000 /* IPSec ESP */ @@ -3023,6 +3038,7 @@ enum ixgbe_fdir_pballoc_type { #define FW_CEM_UNUSED_VER 0x0 #define FW_CEM_MAX_RETRIES 3 #define FW_CEM_RESP_STATUS_SUCCESS 0x1 +#define FW_CEM_DRIVER_VERSION_SIZE 39 /* +9 would send 48 bytes to fw */ #define FW_READ_SHADOW_RAM_CMD 0x31 #define FW_READ_SHADOW_RAM_LEN 0x6 #define FW_WRITE_SHADOW_RAM_CMD 0x33 @@ -3048,13 +3064,66 @@ enum ixgbe_fdir_pballoc_type { #define FW_INT_PHY_REQ_LEN 10 #define FW_INT_PHY_REQ_READ 0 #define FW_INT_PHY_REQ_WRITE 1 +#define FW_PHY_ACT_REQ_CMD 5 +#define FW_PHY_ACT_DATA_COUNT 4 +#define FW_PHY_ACT_REQ_LEN (4 + 4 * FW_PHY_ACT_DATA_COUNT) +#define FW_PHY_ACT_INIT_PHY 1 +#define FW_PHY_ACT_SETUP_LINK 2 +#define FW_PHY_ACT_LINK_SPEED_10 (1u << 0) +#define FW_PHY_ACT_LINK_SPEED_100 (1u << 1) +#define FW_PHY_ACT_LINK_SPEED_1G (1u << 2) +#define FW_PHY_ACT_LINK_SPEED_2_5G (1u << 3) +#define FW_PHY_ACT_LINK_SPEED_5G (1u << 4) +#define FW_PHY_ACT_LINK_SPEED_10G (1u << 5) +#define FW_PHY_ACT_LINK_SPEED_20G (1u << 6) +#define FW_PHY_ACT_LINK_SPEED_25G (1u << 7) +#define FW_PHY_ACT_LINK_SPEED_40G (1u << 8) +#define FW_PHY_ACT_LINK_SPEED_50G (1u << 9) +#define FW_PHY_ACT_LINK_SPEED_100G (1u << 10) +#define FW_PHY_ACT_SETUP_LINK_PAUSE_SHIFT 16 +#define FW_PHY_ACT_SETUP_LINK_PAUSE_MASK (3u << \ + FW_PHY_ACT_SETUP_LINK_PAUSE_SHIFT) +#define FW_PHY_ACT_SETUP_LINK_PAUSE_NONE 0u +#define FW_PHY_ACT_SETUP_LINK_PAUSE_TX 1u +#define FW_PHY_ACT_SETUP_LINK_PAUSE_RX 2u +#define FW_PHY_ACT_SETUP_LINK_PAUSE_RXTX 3u +#define FW_PHY_ACT_SETUP_LINK_LP (1u << 18) +#define FW_PHY_ACT_SETUP_LINK_HP (1u << 19) +#define FW_PHY_ACT_SETUP_LINK_EEE (1u << 20) +#define FW_PHY_ACT_SETUP_LINK_AN (1u << 22) +#define FW_PHY_ACT_SETUP_LINK_RSP_DOWN (1u << 0) +#define FW_PHY_ACT_GET_LINK_INFO 3 +#define FW_PHY_ACT_GET_LINK_INFO_EEE (1u << 19) +#define FW_PHY_ACT_GET_LINK_INFO_FC_TX (1u << 20) +#define FW_PHY_ACT_GET_LINK_INFO_FC_RX (1u << 21) +#define FW_PHY_ACT_GET_LINK_INFO_POWER (1u << 22) +#define FW_PHY_ACT_GET_LINK_INFO_AN_COMPLETE (1u << 24) +#define FW_PHY_ACT_GET_LINK_INFO_TEMP (1u << 25) +#define FW_PHY_ACT_GET_LINK_INFO_LP_FC_TX (1u << 28) +#define FW_PHY_ACT_GET_LINK_INFO_LP_FC_RX (1u << 29) +#define FW_PHY_ACT_FORCE_LINK_DOWN 4 +#define FW_PHY_ACT_FORCE_LINK_DOWN_OFF (1u << 0) +#define FW_PHY_ACT_PHY_SW_RESET 5 +#define FW_PHY_ACT_PHY_HW_RESET 6 +#define FW_PHY_ACT_GET_PHY_INFO 7 +#define FW_PHY_ACT_UD_2 0x1002 +#define FW_PHY_ACT_UD_2_10G_KR_EEE (1u << 6) +#define FW_PHY_ACT_UD_2_10G_KX4_EEE (1u << 5) +#define FW_PHY_ACT_UD_2_1G_KX_EEE (1u << 4) +#define FW_PHY_ACT_UD_2_10G_T_EEE (1u << 3) +#define FW_PHY_ACT_UD_2_1G_T_EEE (1u << 2) +#define FW_PHY_ACT_UD_2_100M_TX_EEE (1u << 1) +#define FW_PHY_ACT_RETRIES 50 +#define FW_PHY_INFO_SPEED_MASK 0xFFFu +#define FW_PHY_INFO_ID_HI_MASK 0xFFFF0000u +#define FW_PHY_INFO_ID_LO_MASK 0x0000FFFFu /* Host Interface Command Structures */ #ifdef C99 #pragma pack(push, 1) #else -#pragma pack(1) +#pragma pack (1) #endif /* C99 */ struct ixgbe_hic_hdr { @@ -3097,6 +3166,16 @@ struct ixgbe_hic_drv_info { u16 pad2; /* end spacing to ensure length is mult. of dword2 */ }; +struct ixgbe_hic_drv_info2 { + struct ixgbe_hic_hdr hdr; + u8 port_num; + u8 ver_sub; + u8 ver_build; + u8 ver_min; + u8 ver_maj; + char driver_string[FW_CEM_DRIVER_VERSION_SIZE]; +}; + /* These need to be dword aligned */ struct ixgbe_hic_read_shadow_ram { union ixgbe_hic_hdr2 hdr; @@ -3136,13 +3215,26 @@ struct ixgbe_hic_internal_phy_req { u8 command_type; __be16 address; u16 rsv1; - __le32 write_data; + __be32 write_data; u16 pad; }; struct ixgbe_hic_internal_phy_resp { struct ixgbe_hic_hdr hdr; - __le32 read_data; + __be32 read_data; +}; + +struct ixgbe_hic_phy_activity_req { + struct ixgbe_hic_hdr hdr; + u8 port_number; + u8 pad; + __le16 activity_id; + __be32 data[FW_PHY_ACT_DATA_COUNT]; +}; + +struct ixgbe_hic_phy_activity_resp { + struct ixgbe_hic_hdr hdr; + __be32 data[FW_PHY_ACT_DATA_COUNT]; }; #ifdef C99 @@ -3305,7 +3397,7 @@ typedef u32 ixgbe_autoneg_advertised; /* Link speed */ typedef u32 ixgbe_link_speed; #define IXGBE_LINK_SPEED_UNKNOWN 0 -#define IXGBE_LINK_SPEED_10_FULL 0x0004 +#define IXGBE_LINK_SPEED_10_FULL 0x0002 #define IXGBE_LINK_SPEED_100_FULL 0x0008 #define IXGBE_LINK_SPEED_1GB_FULL 0x0020 #define IXGBE_LINK_SPEED_2_5GB_FULL 0x0400 @@ -3318,23 +3410,25 @@ typedef u32 ixgbe_link_speed; IXGBE_LINK_SPEED_10GB_FULL) /* Physical layer type */ -typedef u32 ixgbe_physical_layer; +typedef u64 ixgbe_physical_layer; #define IXGBE_PHYSICAL_LAYER_UNKNOWN 0 -#define IXGBE_PHYSICAL_LAYER_10GBASE_T 0x0001 -#define IXGBE_PHYSICAL_LAYER_1000BASE_T 0x0002 -#define IXGBE_PHYSICAL_LAYER_100BASE_TX 0x0004 -#define IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU 0x0008 -#define IXGBE_PHYSICAL_LAYER_10GBASE_LR 0x0010 -#define IXGBE_PHYSICAL_LAYER_10GBASE_LRM 0x0020 -#define IXGBE_PHYSICAL_LAYER_10GBASE_SR 0x0040 -#define IXGBE_PHYSICAL_LAYER_10GBASE_KX4 0x0080 -#define IXGBE_PHYSICAL_LAYER_10GBASE_CX4 0x0100 -#define IXGBE_PHYSICAL_LAYER_1000BASE_KX 0x0200 -#define IXGBE_PHYSICAL_LAYER_1000BASE_BX 0x0400 -#define IXGBE_PHYSICAL_LAYER_10GBASE_KR 0x0800 -#define IXGBE_PHYSICAL_LAYER_10GBASE_XAUI 0x1000 -#define IXGBE_PHYSICAL_LAYER_SFP_ACTIVE_DA 0x2000 -#define IXGBE_PHYSICAL_LAYER_1000BASE_SX 0x4000 +#define IXGBE_PHYSICAL_LAYER_10GBASE_T 0x00001 +#define IXGBE_PHYSICAL_LAYER_1000BASE_T 0x00002 +#define IXGBE_PHYSICAL_LAYER_100BASE_TX 0x00004 +#define IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU 0x00008 +#define IXGBE_PHYSICAL_LAYER_10GBASE_LR 0x00010 +#define IXGBE_PHYSICAL_LAYER_10GBASE_LRM 0x00020 +#define IXGBE_PHYSICAL_LAYER_10GBASE_SR 0x00040 +#define IXGBE_PHYSICAL_LAYER_10GBASE_KX4 0x00080 +#define IXGBE_PHYSICAL_LAYER_10GBASE_CX4 0x00100 +#define IXGBE_PHYSICAL_LAYER_1000BASE_KX 0x00200 +#define IXGBE_PHYSICAL_LAYER_1000BASE_BX 0x00400 +#define IXGBE_PHYSICAL_LAYER_10GBASE_KR 0x00800 +#define IXGBE_PHYSICAL_LAYER_10GBASE_XAUI 0x01000 +#define IXGBE_PHYSICAL_LAYER_SFP_ACTIVE_DA 0x02000 +#define IXGBE_PHYSICAL_LAYER_1000BASE_SX 0x04000 +#define IXGBE_PHYSICAL_LAYER_10BASE_T 0x08000 +#define IXGBE_PHYSICAL_LAYER_2500BASE_KX 0x10000 /* Flow Control Data Sheet defined values * Calculation and defines taken from 802.1bb Annex O @@ -3553,7 +3647,9 @@ enum ixgbe_phy_type { ixgbe_phy_aq, ixgbe_phy_x550em_kr, ixgbe_phy_x550em_kx4, + ixgbe_phy_x550em_xfi, ixgbe_phy_x550em_ext_t, + ixgbe_phy_ext_1g_t, ixgbe_phy_cu_unknown, ixgbe_phy_qt, ixgbe_phy_xaui, @@ -3572,7 +3668,7 @@ enum ixgbe_phy_type { ixgbe_phy_qsfp_unknown, ixgbe_phy_sfp_unsupported, /*Enforce bit set with unsupported module*/ ixgbe_phy_sgmii, - ixgbe_phy_m88, + ixgbe_phy_fw, ixgbe_phy_generic }; @@ -3629,14 +3725,6 @@ enum ixgbe_fc_mode { ixgbe_fc_default }; -/* Master/slave control */ -enum ixgbe_ms_type { - ixgbe_ms_hw_default = 0, - ixgbe_ms_force_master, - ixgbe_ms_force_slave, - ixgbe_ms_auto -}; - /* Smart Speed Settings */ #define IXGBE_SMARTSPEED_MAX_RETRIES 3 enum ixgbe_smart_speed { @@ -3819,7 +3907,7 @@ struct ixgbe_mac_operations { s32 (*clear_hw_cntrs)(struct ixgbe_hw *); void (*enable_relaxed_ordering)(struct ixgbe_hw *); enum ixgbe_media_type (*get_media_type)(struct ixgbe_hw *); - u32 (*get_supported_physical_layer)(struct ixgbe_hw *); + u64 (*get_supported_physical_layer)(struct ixgbe_hw *); s32 (*get_mac_addr)(struct ixgbe_hw *, u8 *); s32 (*get_san_mac_addr)(struct ixgbe_hw *, u8 *); s32 (*set_san_mac_addr)(struct ixgbe_hw *, u8 *); @@ -3840,6 +3928,7 @@ struct ixgbe_mac_operations { void (*init_swfw_sync)(struct ixgbe_hw *); s32 (*prot_autoc_read)(struct ixgbe_hw *, bool *, u32 *); s32 (*prot_autoc_write)(struct ixgbe_hw *, u32, bool); + s32 (*negotiate_api_version)(struct ixgbe_hw *hw, int api); /* Link */ void (*disable_tx_laser)(struct ixgbe_hw *); @@ -3860,6 +3949,7 @@ struct ixgbe_mac_operations { s32 (*led_off)(struct ixgbe_hw *, u32); s32 (*blink_led_start)(struct ixgbe_hw *, u32); s32 (*blink_led_stop)(struct ixgbe_hw *, u32); + s32 (*init_led_link_act)(struct ixgbe_hw *); /* RAR, Multicast, VLAN */ s32 (*set_rar)(struct ixgbe_hw *, u32, u8 *, u32, u32); @@ -3883,6 +3973,8 @@ struct ixgbe_mac_operations { s32 (*init_uta_tables)(struct ixgbe_hw *); void (*set_mac_anti_spoofing)(struct ixgbe_hw *, bool, int); void (*set_vlan_anti_spoofing)(struct ixgbe_hw *, bool, int); + s32 (*update_xcast_mode)(struct ixgbe_hw *, int); + s32 (*set_rlpml)(struct ixgbe_hw *, u16); /* Flow Control */ s32 (*fc_enable)(struct ixgbe_hw *); @@ -3890,7 +3982,8 @@ struct ixgbe_mac_operations { void (*fc_autoneg)(struct ixgbe_hw *); /* Manageability interface */ - s32 (*set_fw_drv_ver)(struct ixgbe_hw *, u8, u8, u8, u8); + s32 (*set_fw_drv_ver)(struct ixgbe_hw *, u8, u8, u8, u8, u16, + const char *); s32 (*get_thermal_sensor_data)(struct ixgbe_hw *); s32 (*init_thermal_sensor_thresh)(struct ixgbe_hw *hw); void (*get_rtrup2tc)(struct ixgbe_hw *hw, u8 *map); @@ -4000,6 +4093,7 @@ struct ixgbe_mac_info { struct ixgbe_dmac_config dmac_config; bool set_lben; u32 max_link_up_time; + u8 led_link_act; }; struct ixgbe_phy_info { @@ -4015,8 +4109,8 @@ struct ixgbe_phy_info { bool reset_disable; ixgbe_autoneg_advertised autoneg_advertised; ixgbe_link_speed speeds_supported; - enum ixgbe_ms_type ms_type; - enum ixgbe_ms_type original_ms_type; + ixgbe_link_speed eee_speeds_supported; + ixgbe_link_speed eee_speeds_advertised; enum ixgbe_smart_speed smart_speed; bool smart_speed_active; bool multispeed_fiber; @@ -4078,6 +4172,7 @@ struct ixgbe_hw { bool force_full_reset; bool allow_unsupported_sfp; bool wol_enabled; + bool need_crosstalk_fix; }; #define ixgbe_call_func(hw, func, params, error) \ @@ -4136,16 +4231,35 @@ struct ixgbe_hw { #define IXGBE_KRM_LINK_S1(P) ((P) ? 0x8200 : 0x4200) #define IXGBE_KRM_LINK_CTRL_1(P) ((P) ? 0x820C : 0x420C) #define IXGBE_KRM_AN_CNTL_1(P) ((P) ? 0x822C : 0x422C) +#define IXGBE_KRM_AN_CNTL_4(P) ((P) ? 0x8238 : 0x4238) #define IXGBE_KRM_AN_CNTL_8(P) ((P) ? 0x8248 : 0x4248) +#define IXGBE_KRM_PCS_KX_AN(P) ((P) ? 0x9918 : 0x5918) +#define IXGBE_KRM_PCS_KX_AN_LP(P) ((P) ? 0x991C : 0x591C) #define IXGBE_KRM_SGMII_CTRL(P) ((P) ? 0x82A0 : 0x42A0) #define IXGBE_KRM_LP_BASE_PAGE_HIGH(P) ((P) ? 0x836C : 0x436C) #define IXGBE_KRM_DSP_TXFFE_STATE_4(P) ((P) ? 0x8634 : 0x4634) #define IXGBE_KRM_DSP_TXFFE_STATE_5(P) ((P) ? 0x8638 : 0x4638) #define IXGBE_KRM_RX_TRN_LINKUP_CTRL(P) ((P) ? 0x8B00 : 0x4B00) #define IXGBE_KRM_PMD_DFX_BURNIN(P) ((P) ? 0x8E00 : 0x4E00) +#define IXGBE_KRM_PMD_FLX_MASK_ST20(P) ((P) ? 0x9054 : 0x5054) #define IXGBE_KRM_TX_COEFF_CTRL_1(P) ((P) ? 0x9520 : 0x5520) #define IXGBE_KRM_RX_ANA_CTL(P) ((P) ? 0x9A00 : 0x5A00) +#define IXGBE_KRM_PMD_FLX_MASK_ST20_SFI_10G_DA ~(0x3 << 20) +#define IXGBE_KRM_PMD_FLX_MASK_ST20_SFI_10G_SR (1u << 20) +#define IXGBE_KRM_PMD_FLX_MASK_ST20_SFI_10G_LR (0x2 << 20) +#define IXGBE_KRM_PMD_FLX_MASK_ST20_SGMII_EN (1u << 25) +#define IXGBE_KRM_PMD_FLX_MASK_ST20_AN37_EN (1u << 26) +#define IXGBE_KRM_PMD_FLX_MASK_ST20_AN_EN (1u << 27) +#define IXGBE_KRM_PMD_FLX_MASK_ST20_SPEED_10M ~(0x7 << 28) +#define IXGBE_KRM_PMD_FLX_MASK_ST20_SPEED_100M (1u << 28) +#define IXGBE_KRM_PMD_FLX_MASK_ST20_SPEED_1G (0x2 << 28) +#define IXGBE_KRM_PMD_FLX_MASK_ST20_SPEED_10G (0x3 << 28) +#define IXGBE_KRM_PMD_FLX_MASK_ST20_SPEED_AN (0x4 << 28) +#define IXGBE_KRM_PMD_FLX_MASK_ST20_SPEED_2_5G (0x7 << 28) +#define IXGBE_KRM_PMD_FLX_MASK_ST20_SPEED_MASK (0x7 << 28) +#define IXGBE_KRM_PMD_FLX_MASK_ST20_FW_AN_RESTART (1u << 31) + #define IXGBE_KRM_PORT_CAR_GEN_CTRL_NELB_32B (1 << 9) #define IXGBE_KRM_PORT_CAR_GEN_CTRL_NELB_KRPCS (1 << 11) @@ -4166,9 +4280,14 @@ struct ixgbe_hw { #define IXGBE_KRM_AN_CNTL_1_SYM_PAUSE (1 << 28) #define IXGBE_KRM_AN_CNTL_1_ASM_PAUSE (1 << 29) - +#define IXGBE_KRM_PCS_KX_AN_SYM_PAUSE (1 << 1) +#define IXGBE_KRM_PCS_KX_AN_ASM_PAUSE (1 << 2) +#define IXGBE_KRM_PCS_KX_AN_LP_SYM_PAUSE (1 << 2) +#define IXGBE_KRM_PCS_KX_AN_LP_ASM_PAUSE (1 << 3) +#define IXGBE_KRM_AN_CNTL_4_ECSR_AN37_OVER_73 (1 << 29) #define IXGBE_KRM_AN_CNTL_8_LINEAR (1 << 0) #define IXGBE_KRM_AN_CNTL_8_LIMITING (1 << 1) + #define IXGBE_KRM_LP_BASE_PAGE_HIGH_SYM_PAUSE (1 << 10) #define IXGBE_KRM_LP_BASE_PAGE_HIGH_ASM_PAUSE (1 << 11) @@ -4207,11 +4326,18 @@ struct ixgbe_hw { #define IXGBE_SB_IOSF_TARGET_KR_PHY 0 #define IXGBE_NW_MNG_IF_SEL 0x00011178 -#define IXGBE_NW_MNG_IF_SEL_MDIO_ACT (1 << 1) -#define IXGBE_NW_MNG_IF_SEL_ENABLE_10_100M (1 << 23) -#define IXGBE_NW_MNG_IF_SEL_INT_PHY_MODE (1 << 24) +#define IXGBE_NW_MNG_IF_SEL_MDIO_ACT (1u << 1) +#define IXGBE_NW_MNG_IF_SEL_MDIO_IF_MODE (1u << 2) +#define IXGBE_NW_MNG_IF_SEL_EN_SHARED_MDIO (1u << 13) +#define IXGBE_NW_MNG_IF_SEL_PHY_SPEED_10M (1u << 17) +#define IXGBE_NW_MNG_IF_SEL_PHY_SPEED_100M (1u << 18) +#define IXGBE_NW_MNG_IF_SEL_PHY_SPEED_1G (1u << 19) +#define IXGBE_NW_MNG_IF_SEL_PHY_SPEED_2_5G (1u << 20) +#define IXGBE_NW_MNG_IF_SEL_PHY_SPEED_10G (1u << 21) +#define IXGBE_NW_MNG_IF_SEL_SGMII_ENABLE (1u << 25) +#define IXGBE_NW_MNG_IF_SEL_INT_PHY_MODE (1 << 24) /* X552 reg field only */ #define IXGBE_NW_MNG_IF_SEL_MDIO_PHY_ADD_SHIFT 3 #define IXGBE_NW_MNG_IF_SEL_MDIO_PHY_ADD \ - (0x1F << IXGBE_NW_MNG_IF_SEL_MDIO_PHY_ADD_SHIFT) + (0x1F << IXGBE_NW_MNG_IF_SEL_MDIO_PHY_ADD_SHIFT) #endif /* _IXGBE_TYPE_H_ */ diff --git a/dpdk/drivers/net/ixgbe/base/ixgbe_vf.c b/dpdk/drivers/net/ixgbe/base/ixgbe_vf.c index 26c0d81c..b513190a 100644 --- a/dpdk/drivers/net/ixgbe/base/ixgbe_vf.c +++ b/dpdk/drivers/net/ixgbe/base/ixgbe_vf.c @@ -64,6 +64,7 @@ s32 ixgbe_init_ops_vf(struct ixgbe_hw *hw) hw->mac.ops.get_mac_addr = ixgbe_get_mac_addr_vf; hw->mac.ops.stop_adapter = ixgbe_stop_adapter_vf; hw->mac.ops.get_bus_info = NULL; + hw->mac.ops.negotiate_api_version = ixgbevf_negotiate_api_version; /* Link */ hw->mac.ops.setup_link = ixgbe_setup_mac_link_vf; @@ -75,10 +76,12 @@ s32 ixgbe_init_ops_vf(struct ixgbe_hw *hw) hw->mac.ops.set_uc_addr = ixgbevf_set_uc_addr_vf; hw->mac.ops.init_rx_addrs = NULL; hw->mac.ops.update_mc_addr_list = ixgbe_update_mc_addr_list_vf; + hw->mac.ops.update_xcast_mode = ixgbevf_update_xcast_mode; hw->mac.ops.enable_mc = NULL; hw->mac.ops.disable_mc = NULL; hw->mac.ops.clear_vfta = NULL; hw->mac.ops.set_vfta = ixgbe_set_vfta_vf; + hw->mac.ops.set_rlpml = ixgbevf_rlpml_set_vf; hw->mac.max_tx_queues = 1; hw->mac.max_rx_queues = 1; @@ -322,15 +325,16 @@ STATIC s32 ixgbe_mta_vector(struct ixgbe_hw *hw, u8 *mc_addr) return vector; } -STATIC void ixgbevf_write_msg_read_ack(struct ixgbe_hw *hw, - u32 *msg, u16 size) +STATIC s32 ixgbevf_write_msg_read_ack(struct ixgbe_hw *hw, u32 *msg, + u32 *retmsg, u16 size) { struct ixgbe_mbx_info *mbx = &hw->mbx; - u32 retmsg[IXGBE_VFMAILBOX_SIZE]; s32 retval = mbx->ops.write_posted(hw, msg, size, 0); - if (!retval) - mbx->ops.read_posted(hw, retmsg, size, 0); + if (retval) + return retval; + + return mbx->ops.read_posted(hw, retmsg, size, 0); } /** @@ -344,7 +348,6 @@ STATIC void ixgbevf_write_msg_read_ack(struct ixgbe_hw *hw, s32 ixgbe_set_rar_vf(struct ixgbe_hw *hw, u32 index, u8 *addr, u32 vmdq, u32 enable_addr) { - struct ixgbe_mbx_info *mbx = &hw->mbx; u32 msgbuf[3]; u8 *msg_addr = (u8 *)(&msgbuf[1]); s32 ret_val; @@ -353,10 +356,7 @@ s32 ixgbe_set_rar_vf(struct ixgbe_hw *hw, u32 index, u8 *addr, u32 vmdq, memset(msgbuf, 0, 12); msgbuf[0] = IXGBE_VF_SET_MAC_ADDR; memcpy(msg_addr, addr, 6); - ret_val = mbx->ops.write_posted(hw, msgbuf, 3, 0); - - if (!ret_val) - ret_val = mbx->ops.read_posted(hw, msgbuf, 3, 0); + ret_val = ixgbevf_write_msg_read_ack(hw, msgbuf, msgbuf, 3); msgbuf[0] &= ~IXGBE_VT_MSGTYPE_CTS; @@ -418,6 +418,43 @@ s32 ixgbe_update_mc_addr_list_vf(struct ixgbe_hw *hw, u8 *mc_addr_list, return mbx->ops.write_posted(hw, msgbuf, IXGBE_VFMAILBOX_SIZE, 0); } +/** + * ixgbevf_update_xcast_mode - Update Multicast mode + * @hw: pointer to the HW structure + * @xcast_mode: new multicast mode + * + * Updates the Multicast Mode of VF. + **/ +s32 ixgbevf_update_xcast_mode(struct ixgbe_hw *hw, int xcast_mode) +{ + u32 msgbuf[2]; + s32 err; + + switch (hw->api_version) { + case ixgbe_mbox_api_12: + /* New modes were introduced in 1.3 version */ + if (xcast_mode > IXGBEVF_XCAST_MODE_ALLMULTI) + return IXGBE_ERR_FEATURE_NOT_SUPPORTED; + /* Fall through */ + case ixgbe_mbox_api_13: + break; + default: + return IXGBE_ERR_FEATURE_NOT_SUPPORTED; + } + + msgbuf[0] = IXGBE_VF_UPDATE_XCAST_MODE; + msgbuf[1] = xcast_mode; + + err = ixgbevf_write_msg_read_ack(hw, msgbuf, msgbuf, 2); + if (err) + return err; + + msgbuf[0] &= ~IXGBE_VT_MSGTYPE_CTS; + if (msgbuf[0] == (IXGBE_VF_UPDATE_XCAST_MODE | IXGBE_VT_MSGTYPE_NACK)) + return IXGBE_ERR_FEATURE_NOT_SUPPORTED; + return IXGBE_SUCCESS; +} + /** * ixgbe_set_vfta_vf - Set/Unset vlan filter table address * @hw: pointer to the HW structure @@ -425,11 +462,12 @@ s32 ixgbe_update_mc_addr_list_vf(struct ixgbe_hw *hw, u8 *mc_addr_list, * @vind: unused by VF drivers * @vlan_on: if true then set bit, else clear bit * @vlvf_bypass: boolean flag indicating updating default pool is okay + * + * Turn on/off specified VLAN in the VLAN filter table. **/ s32 ixgbe_set_vfta_vf(struct ixgbe_hw *hw, u32 vlan, u32 vind, bool vlan_on, bool vlvf_bypass) { - struct ixgbe_mbx_info *mbx = &hw->mbx; u32 msgbuf[2]; s32 ret_val; UNREFERENCED_2PARAMETER(vind, vlvf_bypass); @@ -439,10 +477,7 @@ s32 ixgbe_set_vfta_vf(struct ixgbe_hw *hw, u32 vlan, u32 vind, /* Setting the 8 bit field MSG INFO to TRUE indicates "add" */ msgbuf[0] |= vlan_on << IXGBE_VT_MSGINFO_SHIFT; - ret_val = mbx->ops.write_posted(hw, msgbuf, 2, 0); - if (!ret_val) - ret_val = mbx->ops.read_posted(hw, msgbuf, 1, 0); - + ret_val = ixgbevf_write_msg_read_ack(hw, msgbuf, msgbuf, 2); if (!ret_val && (msgbuf[0] & IXGBE_VT_MSGTYPE_ACK)) return IXGBE_SUCCESS; @@ -489,7 +524,6 @@ s32 ixgbe_get_mac_addr_vf(struct ixgbe_hw *hw, u8 *mac_addr) s32 ixgbevf_set_uc_addr_vf(struct ixgbe_hw *hw, u32 index, u8 *addr) { - struct ixgbe_mbx_info *mbx = &hw->mbx; u32 msgbuf[3], msgbuf_chk; u8 *msg_addr = (u8 *)(&msgbuf[1]); s32 ret_val; @@ -506,10 +540,8 @@ s32 ixgbevf_set_uc_addr_vf(struct ixgbe_hw *hw, u32 index, u8 *addr) msgbuf_chk = msgbuf[0]; if (addr) memcpy(msg_addr, addr, 6); - ret_val = mbx->ops.write_posted(hw, msgbuf, 3, 0); - if (!ret_val) - ret_val = mbx->ops.read_posted(hw, msgbuf, 3, 0); + ret_val = ixgbevf_write_msg_read_ack(hw, msgbuf, msgbuf, 3); if (!ret_val) { msgbuf[0] &= ~IXGBE_VT_MSGTYPE_CTS; @@ -585,13 +617,29 @@ s32 ixgbe_check_mac_link_vf(struct ixgbe_hw *hw, ixgbe_link_speed *speed, switch (links_reg & IXGBE_LINKS_SPEED_82599) { case IXGBE_LINKS_SPEED_10G_82599: *speed = IXGBE_LINK_SPEED_10GB_FULL; + if (hw->mac.type >= ixgbe_mac_X550) { + if (links_reg & IXGBE_LINKS_SPEED_NON_STD) + *speed = IXGBE_LINK_SPEED_2_5GB_FULL; + } break; case IXGBE_LINKS_SPEED_1G_82599: *speed = IXGBE_LINK_SPEED_1GB_FULL; break; case IXGBE_LINKS_SPEED_100_82599: *speed = IXGBE_LINK_SPEED_100_FULL; + if (hw->mac.type == ixgbe_mac_X550) { + if (links_reg & IXGBE_LINKS_SPEED_NON_STD) + *speed = IXGBE_LINK_SPEED_5GB_FULL; + } break; + case IXGBE_LINKS_SPEED_10_X550EM_A: + *speed = IXGBE_LINK_SPEED_UNKNOWN; + /* Since Reserved in older MAC's */ + if (hw->mac.type >= ixgbe_mac_X550) + *speed = IXGBE_LINK_SPEED_10_FULL; + break; + default: + *speed = IXGBE_LINK_SPEED_UNKNOWN; } /* if the read failed it could just be a mailbox collision, best wait @@ -628,13 +676,22 @@ out: * @hw: pointer to the HW structure * @max_size: value to assign to max frame size **/ -void ixgbevf_rlpml_set_vf(struct ixgbe_hw *hw, u16 max_size) +s32 ixgbevf_rlpml_set_vf(struct ixgbe_hw *hw, u16 max_size) { u32 msgbuf[2]; + s32 retval; msgbuf[0] = IXGBE_VF_SET_LPE; msgbuf[1] = max_size; - ixgbevf_write_msg_read_ack(hw, msgbuf, 2); + + retval = ixgbevf_write_msg_read_ack(hw, msgbuf, msgbuf, 2); + if (retval) + return retval; + if ((msgbuf[0] & IXGBE_VF_SET_LPE) && + (msgbuf[0] & IXGBE_VT_MSGTYPE_NACK)) + return IXGBE_ERR_MBX; + + return 0; } /** @@ -651,11 +708,8 @@ int ixgbevf_negotiate_api_version(struct ixgbe_hw *hw, int api) msg[0] = IXGBE_VF_API_NEGOTIATE; msg[1] = api; msg[2] = 0; - err = hw->mbx.ops.write_posted(hw, msg, 3, 0); - - if (!err) - err = hw->mbx.ops.read_posted(hw, msg, 3, 0); + err = ixgbevf_write_msg_read_ack(hw, msg, msg, 3); if (!err) { msg[0] &= ~IXGBE_VT_MSGTYPE_CTS; @@ -681,6 +735,7 @@ int ixgbevf_get_queues(struct ixgbe_hw *hw, unsigned int *num_tcs, switch (hw->api_version) { case ixgbe_mbox_api_11: case ixgbe_mbox_api_12: + case ixgbe_mbox_api_13: break; default: return 0; @@ -689,11 +744,8 @@ int ixgbevf_get_queues(struct ixgbe_hw *hw, unsigned int *num_tcs, /* Fetch queue configuration from the PF */ msg[0] = IXGBE_VF_GET_QUEUES; msg[1] = msg[2] = msg[3] = msg[4] = 0; - err = hw->mbx.ops.write_posted(hw, msg, 5, 0); - - if (!err) - err = hw->mbx.ops.read_posted(hw, msg, 5, 0); + err = ixgbevf_write_msg_read_ack(hw, msg, msg, 5); if (!err) { msg[0] &= ~IXGBE_VT_MSGTYPE_CTS; diff --git a/dpdk/drivers/net/ixgbe/base/ixgbe_vf.h b/dpdk/drivers/net/ixgbe/base/ixgbe_vf.h index 8851cb82..3efffe82 100644 --- a/dpdk/drivers/net/ixgbe/base/ixgbe_vf.h +++ b/dpdk/drivers/net/ixgbe/base/ixgbe_vf.h @@ -34,6 +34,8 @@ POSSIBILITY OF SUCH DAMAGE. #ifndef _IXGBE_VF_H_ #define _IXGBE_VF_H_ +#include "ixgbe_type.h" + #define IXGBE_VF_IRQ_CLEAR_MASK 7 #define IXGBE_VF_MAX_TX_QUEUES 8 #define IXGBE_VF_MAX_RX_QUEUES 8 @@ -114,6 +116,7 @@ struct ixgbevf_hw_stats { u64 saved_reset_vfmprc; }; +s32 ixgbe_init_ops_vf(struct ixgbe_hw *hw); s32 ixgbe_init_hw_vf(struct ixgbe_hw *hw); s32 ixgbe_start_hw_vf(struct ixgbe_hw *hw); s32 ixgbe_reset_hw_vf(struct ixgbe_hw *hw); @@ -131,9 +134,10 @@ s32 ixgbevf_set_uc_addr_vf(struct ixgbe_hw *hw, u32 index, u8 *addr); s32 ixgbe_update_mc_addr_list_vf(struct ixgbe_hw *hw, u8 *mc_addr_list, u32 mc_addr_count, ixgbe_mc_addr_itr, bool clear); +s32 ixgbevf_update_xcast_mode(struct ixgbe_hw *hw, int xcast_mode); s32 ixgbe_set_vfta_vf(struct ixgbe_hw *hw, u32 vlan, u32 vind, bool vlan_on, bool vlvf_bypass); -void ixgbevf_rlpml_set_vf(struct ixgbe_hw *hw, u16 max_size); +s32 ixgbevf_rlpml_set_vf(struct ixgbe_hw *hw, u16 max_size); int ixgbevf_negotiate_api_version(struct ixgbe_hw *hw, int api); int ixgbevf_get_queues(struct ixgbe_hw *hw, unsigned int *num_tcs, unsigned int *default_tc); diff --git a/dpdk/drivers/net/ixgbe/base/ixgbe_x540.c b/dpdk/drivers/net/ixgbe/base/ixgbe_x540.c index 31dead0d..0e51813b 100644 --- a/dpdk/drivers/net/ixgbe/base/ixgbe_x540.c +++ b/dpdk/drivers/net/ixgbe/base/ixgbe_x540.c @@ -208,6 +208,7 @@ s32 ixgbe_reset_hw_X540(struct ixgbe_hw *hw) { s32 status; u32 ctrl, i; + u32 swfw_mask = hw->phy.phy_semaphore_mask; DEBUGFUNC("ixgbe_reset_hw_X540"); @@ -220,10 +221,17 @@ s32 ixgbe_reset_hw_X540(struct ixgbe_hw *hw) ixgbe_clear_tx_pending(hw); mac_reset_top: + status = hw->mac.ops.acquire_swfw_sync(hw, swfw_mask); + if (status != IXGBE_SUCCESS) { + ERROR_REPORT2(IXGBE_ERROR_CAUTION, + "semaphore failed with %d", status); + return IXGBE_ERR_SWFW_SYNC; + } ctrl = IXGBE_CTRL_RST; ctrl |= IXGBE_READ_REG(hw, IXGBE_CTRL); IXGBE_WRITE_REG(hw, IXGBE_CTRL, ctrl); IXGBE_WRITE_FLUSH(hw); + hw->mac.ops.release_swfw_sync(hw, swfw_mask); /* Poll for reset bit to self-clear indicating reset is complete */ for (i = 0; i < 10; i++) { @@ -271,6 +279,7 @@ mac_reset_top: if (ixgbe_validate_mac_addr(hw->mac.san_addr) == 0) { /* Save the SAN MAC RAR index */ hw->mac.san_mac_rar_index = hw->mac.num_rar_entries - 1; + hw->mac.ops.set_rar(hw, hw->mac.san_mac_rar_index, hw->mac.san_addr, 0, IXGBE_RAH_AV); @@ -320,9 +329,9 @@ out: * * Determines physical layer capabilities of the current configuration. **/ -u32 ixgbe_get_supported_physical_layer_X540(struct ixgbe_hw *hw) +u64 ixgbe_get_supported_physical_layer_X540(struct ixgbe_hw *hw) { - u32 physical_layer = IXGBE_PHYSICAL_LAYER_UNKNOWN; + u64 physical_layer = IXGBE_PHYSICAL_LAYER_UNKNOWN; u16 ext_ability = 0; DEBUGFUNC("ixgbe_get_supported_physical_layer_X540"); @@ -490,7 +499,6 @@ s32 ixgbe_calc_eeprom_checksum_X540(struct ixgbe_hw *hw) u16 length = 0; u16 pointer = 0; u16 word = 0; - u16 checksum_last_word = IXGBE_EEPROM_CHECKSUM; u16 ptr_start = IXGBE_PCIE_ANALOG_PTR; /* Do not use hw->eeprom.ops.read because we do not want to take @@ -500,14 +508,15 @@ s32 ixgbe_calc_eeprom_checksum_X540(struct ixgbe_hw *hw) DEBUGFUNC("ixgbe_calc_eeprom_checksum_X540"); - /* Include 0x0-0x3F in the checksum */ - for (i = 0; i <= checksum_last_word; i++) { + /* Include 0x0 up to IXGBE_EEPROM_CHECKSUM; do not include the + * checksum itself + */ + for (i = 0; i < IXGBE_EEPROM_CHECKSUM; i++) { if (ixgbe_read_eerd_generic(hw, i, &word)) { DEBUGOUT("EEPROM read failed\n"); return IXGBE_ERR_EEPROM; } - if (i != IXGBE_EEPROM_CHECKSUM) - checksum += word; + checksum += word; } /* Include all data from pointers 0x3, 0x6-0xE. This excludes the @@ -774,8 +783,10 @@ s32 ixgbe_acquire_swfw_sync_X540(struct ixgbe_hw *hw, u32 mask) /* SW NVM semaphore bit is used for access to all * SW_FW_SYNC bits (not just NVM) */ - if (ixgbe_get_swfw_sync_semaphore(hw)) + if (ixgbe_get_swfw_sync_semaphore(hw)) { + DEBUGOUT("Failed to get NVM access and register semaphore, returning IXGBE_ERR_SWFW_SYNC\n"); return IXGBE_ERR_SWFW_SYNC; + } swfw_sync = IXGBE_READ_REG(hw, IXGBE_SWFW_SYNC_BY_MAC(hw)); if (!(swfw_sync & (fwmask | swmask | hwmask))) { @@ -783,7 +794,6 @@ s32 ixgbe_acquire_swfw_sync_X540(struct ixgbe_hw *hw, u32 mask) IXGBE_WRITE_REG(hw, IXGBE_SWFW_SYNC_BY_MAC(hw), swfw_sync); ixgbe_release_swfw_sync_semaphore(hw); - msec_delay(5); return IXGBE_SUCCESS; } /* Firmware currently using resource (fwmask), hardware @@ -798,6 +808,7 @@ s32 ixgbe_acquire_swfw_sync_X540(struct ixgbe_hw *hw, u32 mask) if (swmask == IXGBE_GSSR_SW_MNG_SM) { ERROR_REPORT1(IXGBE_ERROR_POLLING, "Failed to get SW only semaphore"); + DEBUGOUT("Failed to get SW only semaphore, returning IXGBE_ERR_SWFW_SYNC\n"); return IXGBE_ERR_SWFW_SYNC; } @@ -806,8 +817,10 @@ s32 ixgbe_acquire_swfw_sync_X540(struct ixgbe_hw *hw, u32 mask) * of the requested resource(s) while ignoring the corresponding FW/HW * bits in the SW_FW_SYNC register. */ - if (ixgbe_get_swfw_sync_semaphore(hw)) + if (ixgbe_get_swfw_sync_semaphore(hw)) { + DEBUGOUT("Failed to get NVM sempahore and register semaphore while forcefully ignoring FW sempahore bit(s) and setting SW semaphore bit(s), returning IXGBE_ERR_SWFW_SYNC\n"); return IXGBE_ERR_SWFW_SYNC; + } swfw_sync = IXGBE_READ_REG(hw, IXGBE_SWFW_SYNC_BY_MAC(hw)); if (swfw_sync & (fwmask | hwmask)) { swfw_sync |= swmask; @@ -829,9 +842,11 @@ s32 ixgbe_acquire_swfw_sync_X540(struct ixgbe_hw *hw, u32 mask) rmask |= IXGBE_GSSR_I2C_MASK; ixgbe_release_swfw_sync_X540(hw, rmask); ixgbe_release_swfw_sync_semaphore(hw); + DEBUGOUT("Resource not released by other SW, returning IXGBE_ERR_SWFW_SYNC\n"); return IXGBE_ERR_SWFW_SYNC; } ixgbe_release_swfw_sync_semaphore(hw); + DEBUGOUT("Returning error IXGBE_ERR_SWFW_SYNC\n"); return IXGBE_ERR_SWFW_SYNC; } @@ -860,7 +875,7 @@ void ixgbe_release_swfw_sync_X540(struct ixgbe_hw *hw, u32 mask) IXGBE_WRITE_REG(hw, IXGBE_SWFW_SYNC_BY_MAC(hw), swfw_sync); ixgbe_release_swfw_sync_semaphore(hw); - msec_delay(5); + msec_delay(2); } /** @@ -982,6 +997,9 @@ s32 ixgbe_blink_led_start_X540(struct ixgbe_hw *hw, u32 index) DEBUGFUNC("ixgbe_blink_led_start_X540"); + if (index > 3) + return IXGBE_ERR_PARAM; + /* * Link should be up in order for the blink bit in the LED control * register to work. Force link and speed in the MAC if link is down. @@ -1016,6 +1034,9 @@ s32 ixgbe_blink_led_stop_X540(struct ixgbe_hw *hw, u32 index) u32 macc_reg; u32 ledctl_reg; + if (index > 3) + return IXGBE_ERR_PARAM; + DEBUGFUNC("ixgbe_blink_led_stop_X540"); /* Restore the LED to its default value. */ diff --git a/dpdk/drivers/net/ixgbe/base/ixgbe_x540.h b/dpdk/drivers/net/ixgbe/base/ixgbe_x540.h index e4baf6ff..8a19ae2e 100644 --- a/dpdk/drivers/net/ixgbe/base/ixgbe_x540.h +++ b/dpdk/drivers/net/ixgbe/base/ixgbe_x540.h @@ -43,7 +43,7 @@ s32 ixgbe_setup_mac_link_X540(struct ixgbe_hw *hw, ixgbe_link_speed speed, bool link_up_wait_to_complete); s32 ixgbe_reset_hw_X540(struct ixgbe_hw *hw); s32 ixgbe_start_hw_X540(struct ixgbe_hw *hw); -u32 ixgbe_get_supported_physical_layer_X540(struct ixgbe_hw *hw); +u64 ixgbe_get_supported_physical_layer_X540(struct ixgbe_hw *hw); s32 ixgbe_init_eeprom_params_X540(struct ixgbe_hw *hw); s32 ixgbe_read_eerd_X540(struct ixgbe_hw *hw, u16 offset, u16 *data); diff --git a/dpdk/drivers/net/ixgbe/base/ixgbe_x550.c b/dpdk/drivers/net/ixgbe/base/ixgbe_x550.c index e78c9c2c..9862391b 100644 --- a/dpdk/drivers/net/ixgbe/base/ixgbe_x550.c +++ b/dpdk/drivers/net/ixgbe/base/ixgbe_x550.c @@ -41,6 +41,7 @@ POSSIBILITY OF SUCH DAMAGE. STATIC s32 ixgbe_setup_ixfi_x550em(struct ixgbe_hw *hw, ixgbe_link_speed *speed); STATIC s32 ixgbe_acquire_swfw_sync_X550a(struct ixgbe_hw *, u32 mask); STATIC void ixgbe_release_swfw_sync_X550a(struct ixgbe_hw *, u32 mask); +STATIC s32 ixgbe_read_mng_if_sel_x550em(struct ixgbe_hw *hw); /** * ixgbe_init_ops_X550 - Inits func ptrs and MAC type @@ -61,7 +62,7 @@ s32 ixgbe_init_ops_X550(struct ixgbe_hw *hw) mac->ops.dmac_config = ixgbe_dmac_config_X550; mac->ops.dmac_config_tcs = ixgbe_dmac_config_tcs_X550; mac->ops.dmac_update_tcs = ixgbe_dmac_update_tcs_X550; - mac->ops.setup_eee = ixgbe_setup_eee_X550; + mac->ops.setup_eee = NULL; mac->ops.set_source_address_pruning = ixgbe_set_source_address_pruning_X550; mac->ops.set_ethertype_anti_spoofing = @@ -82,7 +83,13 @@ s32 ixgbe_init_ops_X550(struct ixgbe_hw *hw) mac->ops.mdd_event = ixgbe_mdd_event_X550; mac->ops.restore_mdd_vf = ixgbe_restore_mdd_vf_X550; mac->ops.disable_rx = ixgbe_disable_rx_x550; + /* Manageability interface */ + mac->ops.set_fw_drv_ver = ixgbe_set_fw_drv_ver_x550; switch (hw->device_id) { + case IXGBE_DEV_ID_X550EM_X_1G_T: + hw->mac.ops.led_on = NULL; + hw->mac.ops.led_off = NULL; + break; case IXGBE_DEV_ID_X550EM_X_10G_T: case IXGBE_DEV_ID_X550EM_A_10G_T: hw->mac.ops.led_on = ixgbe_led_on_t_X550em; @@ -342,11 +349,10 @@ STATIC s32 ixgbe_read_phy_reg_mdi_22(struct ixgbe_hw *hw, u32 reg_addr, UNREFERENCED_1PARAMETER(dev_type); /* Setup and write the read command */ - command = (reg_addr << IXGBE_MSCA_NP_ADDR_SHIFT) | - (reg_addr << IXGBE_MSCA_DEV_TYPE_SHIFT) | - (hw->phy.addr << IXGBE_MSCA_PHY_ADDR_SHIFT) | - IXGBE_MSCA_OLD_PROTOCOL | IXGBE_MSCA_READ | - IXGBE_MSCA_MDI_COMMAND; + command = (reg_addr << IXGBE_MSCA_DEV_TYPE_SHIFT) | + (hw->phy.addr << IXGBE_MSCA_PHY_ADDR_SHIFT) | + IXGBE_MSCA_OLD_PROTOCOL | IXGBE_MSCA_READ_AUTOINC | + IXGBE_MSCA_MDI_COMMAND; IXGBE_WRITE_REG(hw, IXGBE_MSCA, command); @@ -393,11 +399,10 @@ STATIC s32 ixgbe_write_phy_reg_mdi_22(struct ixgbe_hw *hw, u32 reg_addr, IXGBE_WRITE_REG(hw, IXGBE_MSRWD, (u32)phy_data); /* Setup and write the write command */ - command = (reg_addr << IXGBE_MSCA_NP_ADDR_SHIFT) | - (reg_addr << IXGBE_MSCA_DEV_TYPE_SHIFT) | - (hw->phy.addr << IXGBE_MSCA_PHY_ADDR_SHIFT) | - IXGBE_MSCA_OLD_PROTOCOL | IXGBE_MSCA_WRITE | - IXGBE_MSCA_MDI_COMMAND; + command = (reg_addr << IXGBE_MSCA_DEV_TYPE_SHIFT) | + (hw->phy.addr << IXGBE_MSCA_PHY_ADDR_SHIFT) | + IXGBE_MSCA_OLD_PROTOCOL | IXGBE_MSCA_WRITE | + IXGBE_MSCA_MDI_COMMAND; IXGBE_WRITE_REG(hw, IXGBE_MSCA, command); @@ -422,43 +427,6 @@ STATIC s32 ixgbe_write_phy_reg_mdi_22(struct ixgbe_hw *hw, u32 reg_addr, return IXGBE_SUCCESS; } -/** - * ixgbe_identify_phy_1g - Get 1g PHY type based on device id - * @hw: pointer to hardware structure - * - * Returns error code - */ -STATIC s32 ixgbe_identify_phy_1g(struct ixgbe_hw *hw) -{ - u32 swfw_mask = hw->phy.phy_semaphore_mask; - u16 phy_id_high; - u16 phy_id_low; - s32 rc; - - rc = hw->mac.ops.acquire_swfw_sync(hw, swfw_mask); - if (rc) - return rc; - - rc = ixgbe_read_phy_reg_mdi_22(hw, IXGBE_MDIO_PHY_ID_HIGH, 0, - &phy_id_high); - if (rc) - goto rel_out; - - rc = ixgbe_read_phy_reg_mdi_22(hw, IXGBE_MDIO_PHY_ID_LOW, 0, - &phy_id_low); - if (rc) - goto rel_out; - - hw->phy.id = (u32)phy_id_high << 16; - hw->phy.id |= phy_id_low & IXGBE_PHY_REVISION_MASK; - hw->phy.revision = (u32)phy_id_low & ~IXGBE_PHY_REVISION_MASK; - -rel_out: - hw->mac.ops.release_swfw_sync(hw, swfw_mask); - - return rc; -} - /** * ixgbe_identify_phy_x550em - Get PHY type based on device id * @hw: pointer to hardware structure @@ -467,18 +435,15 @@ rel_out: */ STATIC s32 ixgbe_identify_phy_x550em(struct ixgbe_hw *hw) { + hw->mac.ops.set_lan_id(hw); + + ixgbe_read_mng_if_sel_x550em(hw); + switch (hw->device_id) { case IXGBE_DEV_ID_X550EM_A_SFP: - hw->phy.ops.read_reg = ixgbe_read_phy_reg_x550a; - hw->phy.ops.write_reg = ixgbe_write_phy_reg_x550a; - if (hw->bus.lan_id) - hw->phy.phy_semaphore_mask |= IXGBE_GSSR_PHY1_SM; - else - hw->phy.phy_semaphore_mask |= IXGBE_GSSR_PHY0_SM; return ixgbe_identify_module_generic(hw); case IXGBE_DEV_ID_X550EM_X_SFP: /* set up for CS4227 usage */ - hw->phy.phy_semaphore_mask = IXGBE_GSSR_SHARED_I2C_SM; ixgbe_setup_mux_ctl(hw); ixgbe_check_cs4227(hw); /* Fallthrough */ @@ -489,30 +454,165 @@ STATIC s32 ixgbe_identify_phy_x550em(struct ixgbe_hw *hw) case IXGBE_DEV_ID_X550EM_X_KX4: hw->phy.type = ixgbe_phy_x550em_kx4; break; + case IXGBE_DEV_ID_X550EM_X_XFI: + hw->phy.type = ixgbe_phy_x550em_xfi; + break; case IXGBE_DEV_ID_X550EM_X_KR: case IXGBE_DEV_ID_X550EM_A_KR: case IXGBE_DEV_ID_X550EM_A_KR_L: hw->phy.type = ixgbe_phy_x550em_kr; break; - case IXGBE_DEV_ID_X550EM_X_1G_T: - case IXGBE_DEV_ID_X550EM_X_10G_T: case IXGBE_DEV_ID_X550EM_A_10G_T: + case IXGBE_DEV_ID_X550EM_X_10G_T: return ixgbe_identify_phy_generic(hw); + case IXGBE_DEV_ID_X550EM_X_1G_T: + hw->phy.type = ixgbe_phy_ext_1g_t; + hw->phy.ops.read_reg = NULL; + hw->phy.ops.write_reg = NULL; + break; case IXGBE_DEV_ID_X550EM_A_1G_T: case IXGBE_DEV_ID_X550EM_A_1G_T_L: - hw->phy.ops.read_reg = ixgbe_read_phy_reg_x550a; - hw->phy.ops.write_reg = ixgbe_write_phy_reg_x550a; + hw->phy.type = ixgbe_phy_fw; + hw->phy.ops.read_reg = NULL; + hw->phy.ops.write_reg = NULL; if (hw->bus.lan_id) hw->phy.phy_semaphore_mask |= IXGBE_GSSR_PHY1_SM; else hw->phy.phy_semaphore_mask |= IXGBE_GSSR_PHY0_SM; - return ixgbe_identify_phy_1g(hw); + break; default: break; } return IXGBE_SUCCESS; } +/** + * ixgbe_fw_phy_activity - Perform an activity on a PHY + * @hw: pointer to hardware structure + * @activity: activity to perform + * @data: Pointer to 4 32-bit words of data + */ +s32 ixgbe_fw_phy_activity(struct ixgbe_hw *hw, u16 activity, + u32 (*data)[FW_PHY_ACT_DATA_COUNT]) +{ + union { + struct ixgbe_hic_phy_activity_req cmd; + struct ixgbe_hic_phy_activity_resp rsp; + } hic; + u16 retries = FW_PHY_ACT_RETRIES; + s32 rc; + u16 i; + + do { + memset(&hic, 0, sizeof(hic)); + hic.cmd.hdr.cmd = FW_PHY_ACT_REQ_CMD; + hic.cmd.hdr.buf_len = FW_PHY_ACT_REQ_LEN; + hic.cmd.hdr.checksum = FW_DEFAULT_CHECKSUM; + hic.cmd.port_number = hw->bus.lan_id; + hic.cmd.activity_id = IXGBE_CPU_TO_LE16(activity); + for (i = 0; i < FW_PHY_ACT_DATA_COUNT; ++i) + hic.cmd.data[i] = IXGBE_CPU_TO_BE32((*data)[i]); + + rc = ixgbe_host_interface_command(hw, (u32 *)&hic.cmd, + sizeof(hic.cmd), + IXGBE_HI_COMMAND_TIMEOUT, + true); + if (rc != IXGBE_SUCCESS) + return rc; + if (hic.rsp.hdr.cmd_or_resp.ret_status == + FW_CEM_RESP_STATUS_SUCCESS) { + for (i = 0; i < FW_PHY_ACT_DATA_COUNT; ++i) + (*data)[i] = IXGBE_BE32_TO_CPU(hic.rsp.data[i]); + return IXGBE_SUCCESS; + } + usec_delay(20); + --retries; + } while (retries > 0); + + return IXGBE_ERR_HOST_INTERFACE_COMMAND; +} + +static const struct { + u16 fw_speed; + ixgbe_link_speed phy_speed; +} ixgbe_fw_map[] = { + { FW_PHY_ACT_LINK_SPEED_10, IXGBE_LINK_SPEED_10_FULL }, + { FW_PHY_ACT_LINK_SPEED_100, IXGBE_LINK_SPEED_100_FULL }, + { FW_PHY_ACT_LINK_SPEED_1G, IXGBE_LINK_SPEED_1GB_FULL }, + { FW_PHY_ACT_LINK_SPEED_2_5G, IXGBE_LINK_SPEED_2_5GB_FULL }, + { FW_PHY_ACT_LINK_SPEED_5G, IXGBE_LINK_SPEED_5GB_FULL }, + { FW_PHY_ACT_LINK_SPEED_10G, IXGBE_LINK_SPEED_10GB_FULL }, +}; + +/** + * ixgbe_get_phy_id_fw - Get the phy ID via firmware command + * @hw: pointer to hardware structure + * + * Returns error code + */ +static s32 ixgbe_get_phy_id_fw(struct ixgbe_hw *hw) +{ + u32 info[FW_PHY_ACT_DATA_COUNT] = { 0 }; + u16 phy_speeds; + u16 phy_id_lo; + s32 rc; + u16 i; + + rc = ixgbe_fw_phy_activity(hw, FW_PHY_ACT_GET_PHY_INFO, &info); + if (rc) + return rc; + + hw->phy.speeds_supported = 0; + phy_speeds = info[0] & FW_PHY_INFO_SPEED_MASK; + for (i = 0; i < sizeof(ixgbe_fw_map) / sizeof(ixgbe_fw_map[0]); ++i) { + if (phy_speeds & ixgbe_fw_map[i].fw_speed) + hw->phy.speeds_supported |= ixgbe_fw_map[i].phy_speed; + } + if (!hw->phy.autoneg_advertised) + hw->phy.autoneg_advertised = hw->phy.speeds_supported; + + hw->phy.id = info[0] & FW_PHY_INFO_ID_HI_MASK; + phy_id_lo = info[1] & FW_PHY_INFO_ID_LO_MASK; + hw->phy.id |= phy_id_lo & IXGBE_PHY_REVISION_MASK; + hw->phy.revision = phy_id_lo & ~IXGBE_PHY_REVISION_MASK; + if (!hw->phy.id || hw->phy.id == IXGBE_PHY_REVISION_MASK) + return IXGBE_ERR_PHY_ADDR_INVALID; + return IXGBE_SUCCESS; +} + +/** + * ixgbe_identify_phy_fw - Get PHY type based on firmware command + * @hw: pointer to hardware structure + * + * Returns error code + */ +static s32 ixgbe_identify_phy_fw(struct ixgbe_hw *hw) +{ + if (hw->bus.lan_id) + hw->phy.phy_semaphore_mask = IXGBE_GSSR_PHY1_SM; + else + hw->phy.phy_semaphore_mask = IXGBE_GSSR_PHY0_SM; + + hw->phy.type = ixgbe_phy_fw; + hw->phy.ops.read_reg = NULL; + hw->phy.ops.write_reg = NULL; + return ixgbe_get_phy_id_fw(hw); +} + +/** + * ixgbe_shutdown_fw_phy - Shutdown a firmware-controlled PHY + * @hw: pointer to hardware structure + * + * Returns error code + */ +s32 ixgbe_shutdown_fw_phy(struct ixgbe_hw *hw) +{ + u32 setup[FW_PHY_ACT_DATA_COUNT] = { 0 }; + + setup[0] = FW_PHY_ACT_FORCE_LINK_DOWN_OFF; + return ixgbe_fw_phy_activity(hw, FW_PHY_ACT_FORCE_LINK_DOWN, &setup); +} + STATIC s32 ixgbe_read_phy_reg_x550em(struct ixgbe_hw *hw, u32 reg_addr, u32 device_type, u16 *phy_data) { @@ -601,7 +701,6 @@ s32 ixgbe_init_ops_X550EM(struct ixgbe_hw *hw) struct ixgbe_mac_info *mac = &hw->mac; struct ixgbe_eeprom_info *eeprom = &hw->eeprom; struct ixgbe_phy_info *phy = &hw->phy; - struct ixgbe_link_info *link = &hw->link; s32 ret_val; DEBUGFUNC("ixgbe_init_ops_X550EM"); @@ -637,25 +736,6 @@ s32 ixgbe_init_ops_X550EM(struct ixgbe_hw *hw) hw->bus.type = ixgbe_bus_type_internal; mac->ops.get_bus_info = ixgbe_get_bus_info_X550em; - if (hw->mac.type == ixgbe_mac_X550EM_x) { - mac->ops.read_iosf_sb_reg = ixgbe_read_iosf_sb_reg_x550; - mac->ops.write_iosf_sb_reg = ixgbe_write_iosf_sb_reg_x550; - mac->ops.acquire_swfw_sync = ixgbe_acquire_swfw_sync_X550em; - mac->ops.release_swfw_sync = ixgbe_release_swfw_sync_X550em; - link->ops.read_link = ixgbe_read_i2c_combined_generic; - link->ops.read_link_unlocked = - ixgbe_read_i2c_combined_generic_unlocked; - link->ops.write_link = ixgbe_write_i2c_combined_generic; - link->ops.write_link_unlocked = - ixgbe_write_i2c_combined_generic_unlocked; - link->addr = IXGBE_CS4227; - } - if (hw->mac.type == ixgbe_mac_X550EM_a) { - mac->ops.read_iosf_sb_reg = ixgbe_read_iosf_sb_reg_x550; - mac->ops.write_iosf_sb_reg = ixgbe_write_iosf_sb_reg_x550; - mac->ops.acquire_swfw_sync = ixgbe_acquire_swfw_sync_X550a; - mac->ops.release_swfw_sync = ixgbe_release_swfw_sync_X550a; - } mac->ops.get_media_type = ixgbe_get_media_type_X550em; mac->ops.setup_sfp = ixgbe_setup_sfp_modules_X550em; @@ -666,25 +746,28 @@ s32 ixgbe_init_ops_X550EM(struct ixgbe_hw *hw) if (mac->ops.get_media_type(hw) == ixgbe_media_type_copper) mac->ops.setup_fc = ixgbe_setup_fc_generic; - else if (hw->mac.type == ixgbe_mac_X550EM_a) { - mac->ops.setup_fc = ixgbe_setup_fc_x550a; - mac->ops.fc_autoneg = ixgbe_fc_autoneg_x550a; - } else mac->ops.setup_fc = ixgbe_setup_fc_X550em; - switch (hw->device_id) { - case IXGBE_DEV_ID_X550EM_X_KR: - case IXGBE_DEV_ID_X550EM_A_KR: - case IXGBE_DEV_ID_X550EM_A_KR_L: - break; - default: - mac->ops.setup_eee = NULL; - } - /* PHY */ phy->ops.init = ixgbe_init_phy_ops_X550em; - phy->ops.identify = ixgbe_identify_phy_x550em; + switch (hw->device_id) { + case IXGBE_DEV_ID_X550EM_A_1G_T: + case IXGBE_DEV_ID_X550EM_A_1G_T_L: + mac->ops.setup_fc = NULL; + phy->ops.identify = ixgbe_identify_phy_fw; + phy->ops.set_phy_power = NULL; + phy->ops.get_firmware_version = NULL; + break; + case IXGBE_DEV_ID_X550EM_X_1G_T: + mac->ops.setup_fc = NULL; + phy->ops.identify = ixgbe_identify_phy_x550em; + phy->ops.set_phy_power = NULL; + break; + default: + phy->ops.identify = ixgbe_identify_phy_x550em; + } + if (mac->ops.get_media_type(hw) != ixgbe_media_type_copper) phy->ops.set_phy_power = NULL; @@ -702,6 +785,188 @@ s32 ixgbe_init_ops_X550EM(struct ixgbe_hw *hw) return ret_val; } +/** + * ixgbe_setup_fw_link - Setup firmware-controlled PHYs + * @hw: pointer to hardware structure + */ +static s32 ixgbe_setup_fw_link(struct ixgbe_hw *hw) +{ + u32 setup[FW_PHY_ACT_DATA_COUNT] = { 0 }; + s32 rc; + u16 i; + + if (hw->phy.reset_disable || ixgbe_check_reset_blocked(hw)) + return 0; + + if (hw->fc.strict_ieee && hw->fc.requested_mode == ixgbe_fc_rx_pause) { + ERROR_REPORT1(IXGBE_ERROR_UNSUPPORTED, + "ixgbe_fc_rx_pause not valid in strict IEEE mode\n"); + return IXGBE_ERR_INVALID_LINK_SETTINGS; + } + + switch (hw->fc.requested_mode) { + case ixgbe_fc_full: + setup[0] |= FW_PHY_ACT_SETUP_LINK_PAUSE_RXTX << + FW_PHY_ACT_SETUP_LINK_PAUSE_SHIFT; + break; + case ixgbe_fc_rx_pause: + setup[0] |= FW_PHY_ACT_SETUP_LINK_PAUSE_RX << + FW_PHY_ACT_SETUP_LINK_PAUSE_SHIFT; + break; + case ixgbe_fc_tx_pause: + setup[0] |= FW_PHY_ACT_SETUP_LINK_PAUSE_TX << + FW_PHY_ACT_SETUP_LINK_PAUSE_SHIFT; + break; + default: + break; + } + + for (i = 0; i < sizeof(ixgbe_fw_map) / sizeof(ixgbe_fw_map[0]); ++i) { + if (hw->phy.autoneg_advertised & ixgbe_fw_map[i].phy_speed) + setup[0] |= ixgbe_fw_map[i].fw_speed; + } + setup[0] |= FW_PHY_ACT_SETUP_LINK_HP | FW_PHY_ACT_SETUP_LINK_AN; + + if (hw->phy.eee_speeds_advertised) + setup[0] |= FW_PHY_ACT_SETUP_LINK_EEE; + + rc = ixgbe_fw_phy_activity(hw, FW_PHY_ACT_SETUP_LINK, &setup); + if (rc) + return rc; + if (setup[0] == FW_PHY_ACT_SETUP_LINK_RSP_DOWN) + return IXGBE_ERR_OVERTEMP; + return IXGBE_SUCCESS; +} + +/** + * ixgbe_fc_autoneg_fw _ Set up flow control for FW-controlled PHYs + * @hw: pointer to hardware structure + * + * Called at init time to set up flow control. + */ +static s32 ixgbe_fc_autoneg_fw(struct ixgbe_hw *hw) +{ + if (hw->fc.requested_mode == ixgbe_fc_default) + hw->fc.requested_mode = ixgbe_fc_full; + + return ixgbe_setup_fw_link(hw); +} + +/** + * ixgbe_setup_eee_fw - Enable/disable EEE support + * @hw: pointer to the HW structure + * @enable_eee: boolean flag to enable EEE + * + * Enable/disable EEE based on enable_eee flag. + * This function controls EEE for firmware-based PHY implementations. + */ +static s32 ixgbe_setup_eee_fw(struct ixgbe_hw *hw, bool enable_eee) +{ + if (!!hw->phy.eee_speeds_advertised == enable_eee) + return IXGBE_SUCCESS; + if (enable_eee) + hw->phy.eee_speeds_advertised = hw->phy.eee_speeds_supported; + else + hw->phy.eee_speeds_advertised = 0; + return hw->phy.ops.setup_link(hw); +} + +/** +* ixgbe_init_ops_X550EM_a - Inits func ptrs and MAC type +* @hw: pointer to hardware structure +* +* Initialize the function pointers and for MAC type X550EM_a. +* Does not touch the hardware. +**/ +s32 ixgbe_init_ops_X550EM_a(struct ixgbe_hw *hw) +{ + struct ixgbe_mac_info *mac = &hw->mac; + s32 ret_val; + + DEBUGFUNC("ixgbe_init_ops_X550EM_a"); + + /* Start with generic X550EM init */ + ret_val = ixgbe_init_ops_X550EM(hw); + + if (hw->device_id == IXGBE_DEV_ID_X550EM_A_SGMII || + hw->device_id == IXGBE_DEV_ID_X550EM_A_SGMII_L) { + mac->ops.read_iosf_sb_reg = ixgbe_read_iosf_sb_reg_x550; + mac->ops.write_iosf_sb_reg = ixgbe_write_iosf_sb_reg_x550; + } else { + mac->ops.read_iosf_sb_reg = ixgbe_read_iosf_sb_reg_x550a; + mac->ops.write_iosf_sb_reg = ixgbe_write_iosf_sb_reg_x550a; + } + mac->ops.acquire_swfw_sync = ixgbe_acquire_swfw_sync_X550a; + mac->ops.release_swfw_sync = ixgbe_release_swfw_sync_X550a; + + switch (mac->ops.get_media_type(hw)) { + case ixgbe_media_type_fiber: + mac->ops.setup_fc = NULL; + mac->ops.fc_autoneg = ixgbe_fc_autoneg_fiber_x550em_a; + break; + case ixgbe_media_type_backplane: + mac->ops.fc_autoneg = ixgbe_fc_autoneg_backplane_x550em_a; + mac->ops.setup_fc = ixgbe_setup_fc_backplane_x550em_a; + break; + default: + break; + } + + switch (hw->device_id) { + case IXGBE_DEV_ID_X550EM_A_1G_T: + case IXGBE_DEV_ID_X550EM_A_1G_T_L: + mac->ops.fc_autoneg = ixgbe_fc_autoneg_sgmii_x550em_a; + mac->ops.setup_fc = ixgbe_fc_autoneg_fw; + mac->ops.setup_eee = ixgbe_setup_eee_fw; + hw->phy.eee_speeds_supported = IXGBE_LINK_SPEED_100_FULL | + IXGBE_LINK_SPEED_1GB_FULL; + hw->phy.eee_speeds_advertised = hw->phy.eee_speeds_supported; + break; + default: + break; + } + + return ret_val; +} + +/** +* ixgbe_init_ops_X550EM_x - Inits func ptrs and MAC type +* @hw: pointer to hardware structure +* +* Initialize the function pointers and for MAC type X550EM_x. +* Does not touch the hardware. +**/ +s32 ixgbe_init_ops_X550EM_x(struct ixgbe_hw *hw) +{ + struct ixgbe_mac_info *mac = &hw->mac; + struct ixgbe_link_info *link = &hw->link; + s32 ret_val; + + DEBUGFUNC("ixgbe_init_ops_X550EM_x"); + + /* Start with generic X550EM init */ + ret_val = ixgbe_init_ops_X550EM(hw); + + mac->ops.read_iosf_sb_reg = ixgbe_read_iosf_sb_reg_x550; + mac->ops.write_iosf_sb_reg = ixgbe_write_iosf_sb_reg_x550; + mac->ops.acquire_swfw_sync = ixgbe_acquire_swfw_sync_X550em; + mac->ops.release_swfw_sync = ixgbe_release_swfw_sync_X550em; + link->ops.read_link = ixgbe_read_i2c_combined_generic; + link->ops.read_link_unlocked = ixgbe_read_i2c_combined_generic_unlocked; + link->ops.write_link = ixgbe_write_i2c_combined_generic; + link->ops.write_link_unlocked = + ixgbe_write_i2c_combined_generic_unlocked; + link->addr = IXGBE_CS4227; + + if (hw->device_id == IXGBE_DEV_ID_X550EM_X_1G_T) { + mac->ops.setup_fc = NULL; + mac->ops.setup_eee = NULL; + mac->ops.init_led_link_act = NULL; + } + + return ret_val; +} + /** * ixgbe_dmac_config_X550 * @hw: pointer to hardware structure @@ -765,6 +1030,7 @@ s32 ixgbe_dmac_config_tcs_X550(struct ixgbe_hw *hw) /* Configure DMA coalescing enabled */ switch (hw->mac.dmac_config.link_speed) { + case IXGBE_LINK_SPEED_10_FULL: case IXGBE_LINK_SPEED_100_FULL: pb_headroom = IXGBE_DMACRXT_100M; break; @@ -864,158 +1130,6 @@ s32 ixgbe_init_eeprom_params_X550(struct ixgbe_hw *hw) return IXGBE_SUCCESS; } -/** - * ixgbe_enable_eee_x550 - Enable EEE support - * @hw: pointer to hardware structure - */ -STATIC s32 ixgbe_enable_eee_x550(struct ixgbe_hw *hw) -{ - u16 autoneg_eee_reg; - u32 link_reg; - s32 status; - - if (hw->mac.type == ixgbe_mac_X550) { - /* Advertise EEE capability */ - hw->phy.ops.read_reg(hw, IXGBE_MDIO_AUTO_NEG_EEE_ADVT, - IXGBE_MDIO_AUTO_NEG_DEV_TYPE, - &autoneg_eee_reg); - - autoneg_eee_reg |= (IXGBE_AUTO_NEG_10GBASE_EEE_ADVT | - IXGBE_AUTO_NEG_1000BASE_EEE_ADVT | - IXGBE_AUTO_NEG_100BASE_EEE_ADVT); - - hw->phy.ops.write_reg(hw, IXGBE_MDIO_AUTO_NEG_EEE_ADVT, - IXGBE_MDIO_AUTO_NEG_DEV_TYPE, - autoneg_eee_reg); - return IXGBE_SUCCESS; - } - - switch (hw->device_id) { - case IXGBE_DEV_ID_X550EM_X_KR: - case IXGBE_DEV_ID_X550EM_A_KR: - case IXGBE_DEV_ID_X550EM_A_KR_L: - status = hw->mac.ops.read_iosf_sb_reg(hw, - IXGBE_KRM_LINK_CTRL_1(hw->bus.lan_id), - IXGBE_SB_IOSF_TARGET_KR_PHY, &link_reg); - if (status != IXGBE_SUCCESS) - return status; - - link_reg |= IXGBE_KRM_LINK_CTRL_1_TETH_EEE_CAP_KR | - IXGBE_KRM_LINK_CTRL_1_TETH_EEE_CAP_KX; - - /* Don't advertise FEC capability when EEE enabled. */ - link_reg &= ~IXGBE_KRM_LINK_CTRL_1_TETH_AN_CAP_FEC; - - status = hw->mac.ops.write_iosf_sb_reg(hw, - IXGBE_KRM_LINK_CTRL_1(hw->bus.lan_id), - IXGBE_SB_IOSF_TARGET_KR_PHY, link_reg); - if (status != IXGBE_SUCCESS) - return status; - break; - default: - break; - } - - return IXGBE_SUCCESS; -} - -/** - * ixgbe_disable_eee_x550 - Disable EEE support - * @hw: pointer to hardware structure - */ -STATIC s32 ixgbe_disable_eee_x550(struct ixgbe_hw *hw) -{ - u16 autoneg_eee_reg; - u32 link_reg; - s32 status; - - if (hw->mac.type == ixgbe_mac_X550) { - /* Disable advertised EEE capability */ - hw->phy.ops.read_reg(hw, IXGBE_MDIO_AUTO_NEG_EEE_ADVT, - IXGBE_MDIO_AUTO_NEG_DEV_TYPE, - &autoneg_eee_reg); - - autoneg_eee_reg &= ~(IXGBE_AUTO_NEG_10GBASE_EEE_ADVT | - IXGBE_AUTO_NEG_1000BASE_EEE_ADVT | - IXGBE_AUTO_NEG_100BASE_EEE_ADVT); - - hw->phy.ops.write_reg(hw, IXGBE_MDIO_AUTO_NEG_EEE_ADVT, - IXGBE_MDIO_AUTO_NEG_DEV_TYPE, - autoneg_eee_reg); - return IXGBE_SUCCESS; - } - - switch (hw->device_id) { - case IXGBE_DEV_ID_X550EM_X_KR: - case IXGBE_DEV_ID_X550EM_A_KR: - case IXGBE_DEV_ID_X550EM_A_KR_L: - status = hw->mac.ops.read_iosf_sb_reg(hw, - IXGBE_KRM_LINK_CTRL_1(hw->bus.lan_id), - IXGBE_SB_IOSF_TARGET_KR_PHY, &link_reg); - if (status != IXGBE_SUCCESS) - return status; - - link_reg &= ~(IXGBE_KRM_LINK_CTRL_1_TETH_EEE_CAP_KR | - IXGBE_KRM_LINK_CTRL_1_TETH_EEE_CAP_KX); - - /* Advertise FEC capability when EEE is disabled. */ - link_reg |= IXGBE_KRM_LINK_CTRL_1_TETH_AN_CAP_FEC; - - status = hw->mac.ops.write_iosf_sb_reg(hw, - IXGBE_KRM_LINK_CTRL_1(hw->bus.lan_id), - IXGBE_SB_IOSF_TARGET_KR_PHY, link_reg); - if (status != IXGBE_SUCCESS) - return status; - break; - default: - break; - } - - return IXGBE_SUCCESS; -} - -/** - * ixgbe_setup_eee_X550 - Enable/disable EEE support - * @hw: pointer to the HW structure - * @enable_eee: boolean flag to enable EEE - * - * Enable/disable EEE based on enable_eee flag. - * Auto-negotiation must be started after BASE-T EEE bits in PHY register 7.3C - * are modified. - * - **/ -s32 ixgbe_setup_eee_X550(struct ixgbe_hw *hw, bool enable_eee) -{ - s32 status; - u32 eeer; - - DEBUGFUNC("ixgbe_setup_eee_X550"); - - eeer = IXGBE_READ_REG(hw, IXGBE_EEER); - /* Enable or disable EEE per flag */ - if (enable_eee) { - eeer |= (IXGBE_EEER_TX_LPI_EN | IXGBE_EEER_RX_LPI_EN); - - /* Not supported on first revision of X550EM_x. */ - if ((hw->mac.type == ixgbe_mac_X550EM_x) && - !(IXGBE_FUSES0_REV_MASK & - IXGBE_READ_REG(hw, IXGBE_FUSES0_GROUP(0)))) - return IXGBE_SUCCESS; - status = ixgbe_enable_eee_x550(hw); - if (status) - return status; - } else { - eeer &= ~(IXGBE_EEER_TX_LPI_EN | IXGBE_EEER_RX_LPI_EN); - - status = ixgbe_disable_eee_x550(hw); - if (status) - return status; - } - IXGBE_WRITE_REG(hw, IXGBE_EEER, eeer); - - return IXGBE_SUCCESS; -} - /** * ixgbe_set_source_address_pruning_X550 - Enable/Disbale source address pruning * @hw: pointer to hardware structure @@ -1102,8 +1216,8 @@ STATIC s32 ixgbe_iosf_wait(struct ixgbe_hw *hw, u32 *ctrl) } /** - * ixgbe_write_iosf_sb_reg_x550 - Writes a value to specified register of the IOSF - * device + * ixgbe_write_iosf_sb_reg_x550 - Writes a value to specified register + * of the IOSF device * @hw: pointer to hardware structure * @reg_addr: 32 bit PHY register to write * @device_type: 3 bit device type @@ -1149,12 +1263,11 @@ out: } /** - * ixgbe_read_iosf_sb_reg_x550 - Writes a value to specified register of the IOSF - * device + * ixgbe_read_iosf_sb_reg_x550 - Reads specified register of the IOSF device * @hw: pointer to hardware structure * @reg_addr: 32 bit PHY register to write * @device_type: 3 bit device type - * @phy_data: Pointer to read data from the register + * @data: Pointer to read data from the register **/ s32 ixgbe_read_iosf_sb_reg_x550(struct ixgbe_hw *hw, u32 reg_addr, u32 device_type, u32 *data) @@ -1216,13 +1329,20 @@ s32 ixgbe_get_phy_token(struct ixgbe_hw *hw) sizeof(token_cmd), IXGBE_HI_COMMAND_TIMEOUT, true); - if (status) + if (status) { + DEBUGOUT1("Issuing host interface command failed with Status = %d\n", + status); return status; + } if (token_cmd.hdr.cmd_or_resp.ret_status == FW_PHY_TOKEN_OK) return IXGBE_SUCCESS; - if (token_cmd.hdr.cmd_or_resp.ret_status != FW_PHY_TOKEN_RETRY) + if (token_cmd.hdr.cmd_or_resp.ret_status != FW_PHY_TOKEN_RETRY) { + DEBUGOUT1("Host interface command returned 0x%08x , returning IXGBE_ERR_FW_RESP_INVALID\n", + token_cmd.hdr.cmd_or_resp.ret_status); return IXGBE_ERR_FW_RESP_INVALID; + } + DEBUGOUT("Returning IXGBE_ERR_TOKEN_RETRY\n"); return IXGBE_ERR_TOKEN_RETRY; } @@ -1278,7 +1398,7 @@ s32 ixgbe_write_iosf_sb_reg_x550a(struct ixgbe_hw *hw, u32 reg_addr, write_cmd.port_number = hw->bus.lan_id; write_cmd.command_type = FW_INT_PHY_REQ_WRITE; write_cmd.address = IXGBE_CPU_TO_BE16(reg_addr); - write_cmd.write_data = IXGBE_CPU_TO_LE32(data); + write_cmd.write_data = IXGBE_CPU_TO_BE32(data); status = ixgbe_host_interface_command(hw, (u32 *)&write_cmd, sizeof(write_cmd), @@ -1288,8 +1408,7 @@ s32 ixgbe_write_iosf_sb_reg_x550a(struct ixgbe_hw *hw, u32 reg_addr, } /** - * ixgbe_read_iosf_sb_reg_x550a - Writes a value to specified register - * of the IOSF device. + * ixgbe_read_iosf_sb_reg_x550a - Reads specified register of the IOSF device * @hw: pointer to hardware structure * @reg_addr: 32 bit PHY register to write * @device_type: 3 bit device type @@ -1318,7 +1437,7 @@ s32 ixgbe_read_iosf_sb_reg_x550a(struct ixgbe_hw *hw, u32 reg_addr, IXGBE_HI_COMMAND_TIMEOUT, true); /* Extract the register value from the response. */ - *data = IXGBE_LE32_TO_CPU(hic.rsp.read_data); + *data = IXGBE_BE32_TO_CPU(hic.rsp.read_data); return status; } @@ -1482,6 +1601,7 @@ enum ixgbe_media_type ixgbe_get_media_type_X550em(struct ixgbe_hw *hw) switch (hw->device_id) { case IXGBE_DEV_ID_X550EM_X_KR: case IXGBE_DEV_ID_X550EM_X_KX4: + case IXGBE_DEV_ID_X550EM_X_XFI: case IXGBE_DEV_ID_X550EM_A_KR: case IXGBE_DEV_ID_X550EM_A_KR_L: media_type = ixgbe_media_type_backplane; @@ -1506,7 +1626,6 @@ enum ixgbe_media_type ixgbe_get_media_type_X550em(struct ixgbe_hw *hw) case IXGBE_DEV_ID_X550EM_A_1G_T: case IXGBE_DEV_ID_X550EM_A_1G_T_L: media_type = ixgbe_media_type_copper; - hw->phy.type = ixgbe_phy_m88; break; default: media_type = ixgbe_media_type_unknown; @@ -1598,17 +1717,63 @@ s32 ixgbe_setup_sfp_modules_X550em(struct ixgbe_hw *hw) return IXGBE_SUCCESS; } +/** +* ixgbe_restart_an_internal_phy_x550em - restart autonegotiation for the +* internal PHY +* @hw: pointer to hardware structure +**/ +STATIC s32 ixgbe_restart_an_internal_phy_x550em(struct ixgbe_hw *hw) +{ + s32 status; + u32 link_ctrl; + + /* Restart auto-negotiation. */ + status = hw->mac.ops.read_iosf_sb_reg(hw, + IXGBE_KRM_LINK_CTRL_1(hw->bus.lan_id), + IXGBE_SB_IOSF_TARGET_KR_PHY, &link_ctrl); + + if (status) { + DEBUGOUT("Auto-negotiation did not complete\n"); + return status; + } + + link_ctrl |= IXGBE_KRM_LINK_CTRL_1_TETH_AN_RESTART; + status = hw->mac.ops.write_iosf_sb_reg(hw, + IXGBE_KRM_LINK_CTRL_1(hw->bus.lan_id), + IXGBE_SB_IOSF_TARGET_KR_PHY, link_ctrl); + + if (hw->mac.type == ixgbe_mac_X550EM_a) { + u32 flx_mask_st20; + + /* Indicate to FW that AN restart has been asserted */ + status = hw->mac.ops.read_iosf_sb_reg(hw, + IXGBE_KRM_PMD_FLX_MASK_ST20(hw->bus.lan_id), + IXGBE_SB_IOSF_TARGET_KR_PHY, &flx_mask_st20); + + if (status) { + DEBUGOUT("Auto-negotiation did not complete\n"); + return status; + } + + flx_mask_st20 |= IXGBE_KRM_PMD_FLX_MASK_ST20_FW_AN_RESTART; + status = hw->mac.ops.write_iosf_sb_reg(hw, + IXGBE_KRM_PMD_FLX_MASK_ST20(hw->bus.lan_id), + IXGBE_SB_IOSF_TARGET_KR_PHY, flx_mask_st20); + } + + return status; +} + /** * ixgbe_setup_sgmii - Set up link for sgmii * @hw: pointer to hardware structure */ STATIC s32 ixgbe_setup_sgmii(struct ixgbe_hw *hw, ixgbe_link_speed speed, - bool autoneg_wait_to_complete) + bool autoneg_wait) { struct ixgbe_mac_info *mac = &hw->mac; - u32 lval, sval; + u32 lval, sval, flx_val; s32 rc; - UNREFERENCED_2PARAMETER(speed, autoneg_wait_to_complete); rc = mac->ops.read_iosf_sb_reg(hw, IXGBE_KRM_LINK_CTRL_1(hw->bus.lan_id), @@ -1641,12 +1806,100 @@ STATIC s32 ixgbe_setup_sgmii(struct ixgbe_hw *hw, ixgbe_link_speed speed, if (rc) return rc; - lval |= IXGBE_KRM_LINK_CTRL_1_TETH_AN_RESTART; + rc = mac->ops.read_iosf_sb_reg(hw, + IXGBE_KRM_PMD_FLX_MASK_ST20(hw->bus.lan_id), + IXGBE_SB_IOSF_TARGET_KR_PHY, &flx_val); + if (rc) + return rc; + + flx_val &= ~IXGBE_KRM_PMD_FLX_MASK_ST20_SPEED_MASK; + flx_val |= IXGBE_KRM_PMD_FLX_MASK_ST20_SPEED_1G; + flx_val &= ~IXGBE_KRM_PMD_FLX_MASK_ST20_AN_EN; + flx_val |= IXGBE_KRM_PMD_FLX_MASK_ST20_SGMII_EN; + flx_val |= IXGBE_KRM_PMD_FLX_MASK_ST20_AN37_EN; + + rc = mac->ops.write_iosf_sb_reg(hw, + IXGBE_KRM_PMD_FLX_MASK_ST20(hw->bus.lan_id), + IXGBE_SB_IOSF_TARGET_KR_PHY, flx_val); + if (rc) + return rc; + + rc = ixgbe_restart_an_internal_phy_x550em(hw); + if (rc) + return rc; + + return hw->phy.ops.setup_link_speed(hw, speed, autoneg_wait); +} + +/** + * ixgbe_setup_sgmii_fw - Set up link for sgmii with firmware-controlled PHYs + * @hw: pointer to hardware structure + */ +STATIC s32 ixgbe_setup_sgmii_fw(struct ixgbe_hw *hw, ixgbe_link_speed speed, + bool autoneg_wait) +{ + struct ixgbe_mac_info *mac = &hw->mac; + u32 lval, sval, flx_val; + s32 rc; + + rc = mac->ops.read_iosf_sb_reg(hw, + IXGBE_KRM_LINK_CTRL_1(hw->bus.lan_id), + IXGBE_SB_IOSF_TARGET_KR_PHY, &lval); + if (rc) + return rc; + + lval &= ~IXGBE_KRM_LINK_CTRL_1_TETH_AN_ENABLE; + lval &= ~IXGBE_KRM_LINK_CTRL_1_TETH_FORCE_SPEED_MASK; + lval |= IXGBE_KRM_LINK_CTRL_1_TETH_AN_SGMII_EN; + lval |= IXGBE_KRM_LINK_CTRL_1_TETH_AN_CLAUSE_37_EN; + lval &= ~IXGBE_KRM_LINK_CTRL_1_TETH_FORCE_SPEED_1G; rc = mac->ops.write_iosf_sb_reg(hw, IXGBE_KRM_LINK_CTRL_1(hw->bus.lan_id), IXGBE_SB_IOSF_TARGET_KR_PHY, lval); + if (rc) + return rc; - return rc; + rc = mac->ops.read_iosf_sb_reg(hw, + IXGBE_KRM_SGMII_CTRL(hw->bus.lan_id), + IXGBE_SB_IOSF_TARGET_KR_PHY, &sval); + if (rc) + return rc; + + sval &= ~IXGBE_KRM_SGMII_CTRL_MAC_TAR_FORCE_10_D; + sval &= ~IXGBE_KRM_SGMII_CTRL_MAC_TAR_FORCE_100_D; + rc = mac->ops.write_iosf_sb_reg(hw, + IXGBE_KRM_SGMII_CTRL(hw->bus.lan_id), + IXGBE_SB_IOSF_TARGET_KR_PHY, sval); + if (rc) + return rc; + + rc = mac->ops.write_iosf_sb_reg(hw, + IXGBE_KRM_LINK_CTRL_1(hw->bus.lan_id), + IXGBE_SB_IOSF_TARGET_KR_PHY, lval); + if (rc) + return rc; + + rc = mac->ops.read_iosf_sb_reg(hw, + IXGBE_KRM_PMD_FLX_MASK_ST20(hw->bus.lan_id), + IXGBE_SB_IOSF_TARGET_KR_PHY, &flx_val); + if (rc) + return rc; + + flx_val &= ~IXGBE_KRM_PMD_FLX_MASK_ST20_SPEED_MASK; + flx_val |= IXGBE_KRM_PMD_FLX_MASK_ST20_SPEED_AN; + flx_val &= ~IXGBE_KRM_PMD_FLX_MASK_ST20_AN_EN; + flx_val |= IXGBE_KRM_PMD_FLX_MASK_ST20_SGMII_EN; + flx_val |= IXGBE_KRM_PMD_FLX_MASK_ST20_AN37_EN; + + rc = mac->ops.write_iosf_sb_reg(hw, + IXGBE_KRM_PMD_FLX_MASK_ST20(hw->bus.lan_id), + IXGBE_SB_IOSF_TARGET_KR_PHY, flx_val); + if (rc) + return rc; + + rc = ixgbe_restart_an_internal_phy_x550em(hw); + + return hw->phy.ops.setup_link_speed(hw, speed, autoneg_wait); } /** @@ -1670,17 +1923,32 @@ void ixgbe_init_mac_link_ops_X550em(struct ixgbe_hw *hw) mac->ops.setup_link = ixgbe_setup_mac_link_multispeed_fiber; mac->ops.set_rate_select_speed = ixgbe_set_soft_rate_select_speed; + if ((hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP_N) || (hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP)) mac->ops.setup_mac_link = - ixgbe_setup_mac_link_sfp_x550a; + ixgbe_setup_mac_link_sfp_x550a; else mac->ops.setup_mac_link = - ixgbe_setup_mac_link_sfp_x550em; + ixgbe_setup_mac_link_sfp_x550em; break; case ixgbe_media_type_copper: - mac->ops.setup_link = ixgbe_setup_mac_link_t_X550em; - mac->ops.check_link = ixgbe_check_link_t_X550em; + if (hw->device_id == IXGBE_DEV_ID_X550EM_X_1G_T) + break; + if (hw->mac.type == ixgbe_mac_X550EM_a) { + if (hw->device_id == IXGBE_DEV_ID_X550EM_A_1G_T || + hw->device_id == IXGBE_DEV_ID_X550EM_A_1G_T_L) { + mac->ops.setup_link = ixgbe_setup_sgmii_fw; + mac->ops.check_link = + ixgbe_check_mac_link_generic; + } else { + mac->ops.setup_link = + ixgbe_setup_mac_link_t_X550em; + } + } else { + mac->ops.setup_link = ixgbe_setup_mac_link_t_X550em; + mac->ops.check_link = ixgbe_check_link_t_X550em; + } break; case ixgbe_media_type_backplane: if (hw->device_id == IXGBE_DEV_ID_X550EM_A_SGMII || @@ -1704,6 +1972,13 @@ s32 ixgbe_get_link_capabilities_X550em(struct ixgbe_hw *hw, { DEBUGFUNC("ixgbe_get_link_capabilities_X550em"); + + if (hw->phy.type == ixgbe_phy_fw) { + *autoneg = true; + *speed = hw->phy.speeds_supported; + return 0; + } + /* SFP */ if (hw->phy.media_type == ixgbe_media_type_fiber) { @@ -1727,13 +2002,24 @@ s32 ixgbe_get_link_capabilities_X550em(struct ixgbe_hw *hw, *speed = IXGBE_LINK_SPEED_10GB_FULL; } else { switch (hw->phy.type) { - case ixgbe_phy_m88: - *speed = IXGBE_LINK_SPEED_100_FULL | - IXGBE_LINK_SPEED_1GB_FULL; - break; + case ixgbe_phy_ext_1g_t: case ixgbe_phy_sgmii: *speed = IXGBE_LINK_SPEED_1GB_FULL; break; + case ixgbe_phy_x550em_kr: + if (hw->mac.type == ixgbe_mac_X550EM_a) { + /* check different backplane modes */ + if (hw->phy.nw_mng_if_sel & + IXGBE_NW_MNG_IF_SEL_PHY_SPEED_2_5G) { + *speed = IXGBE_LINK_SPEED_2_5GB_FULL; + break; + } else if (hw->device_id == + IXGBE_DEV_ID_X550EM_A_KR_L) { + *speed = IXGBE_LINK_SPEED_1GB_FULL; + break; + } + } + /* fall through */ default: *speed = IXGBE_LINK_SPEED_10GB_FULL | IXGBE_LINK_SPEED_1GB_FULL; @@ -1854,19 +2140,32 @@ STATIC s32 ixgbe_enable_lasi_ext_t_x550em(struct ixgbe_hw *hw) status = ixgbe_get_lasi_ext_t_x550em(hw, &lsc); /* Enable link status change alarm */ - status = hw->phy.ops.read_reg(hw, IXGBE_MDIO_PMA_TX_VEN_LASI_INT_MASK, - IXGBE_MDIO_AUTO_NEG_DEV_TYPE, ®); - if (status != IXGBE_SUCCESS) - return status; + /* Enable the LASI interrupts on X552 devices to receive notifications + * of the link configurations of the external PHY and correspondingly + * support the configuration of the internal iXFI link, since iXFI does + * not support auto-negotiation. This is not required for X553 devices + * having KR support, which performs auto-negotiations and which is used + * as the internal link to the external PHY. Hence adding a check here + * to avoid enabling LASI interrupts for X553 devices. + */ + if (hw->mac.type != ixgbe_mac_X550EM_a) { + status = hw->phy.ops.read_reg(hw, + IXGBE_MDIO_PMA_TX_VEN_LASI_INT_MASK, + IXGBE_MDIO_AUTO_NEG_DEV_TYPE, ®); - reg |= IXGBE_MDIO_PMA_TX_VEN_LASI_INT_EN; + if (status != IXGBE_SUCCESS) + return status; - status = hw->phy.ops.write_reg(hw, IXGBE_MDIO_PMA_TX_VEN_LASI_INT_MASK, - IXGBE_MDIO_AUTO_NEG_DEV_TYPE, reg); + reg |= IXGBE_MDIO_PMA_TX_VEN_LASI_INT_EN; - if (status != IXGBE_SUCCESS) - return status; + status = hw->phy.ops.write_reg(hw, + IXGBE_MDIO_PMA_TX_VEN_LASI_INT_MASK, + IXGBE_MDIO_AUTO_NEG_DEV_TYPE, reg); + + if (status != IXGBE_SUCCESS) + return status; + } /* Enable high temperature failure and global fault alarms */ status = hw->phy.ops.read_reg(hw, IXGBE_MDIO_GLOBAL_INT_MASK, @@ -1935,8 +2234,8 @@ STATIC s32 ixgbe_setup_kr_speed_x550em(struct ixgbe_hw *hw, u32 reg_val; status = hw->mac.ops.read_iosf_sb_reg(hw, - IXGBE_KRM_LINK_CTRL_1(hw->bus.lan_id), - IXGBE_SB_IOSF_TARGET_KR_PHY, ®_val); + IXGBE_KRM_LINK_CTRL_1(hw->bus.lan_id), + IXGBE_SB_IOSF_TARGET_KR_PHY, ®_val); if (status) return status; @@ -1952,182 +2251,75 @@ STATIC s32 ixgbe_setup_kr_speed_x550em(struct ixgbe_hw *hw, if (speed & IXGBE_LINK_SPEED_1GB_FULL) reg_val |= IXGBE_KRM_LINK_CTRL_1_TETH_AN_CAP_KX; - /* Restart auto-negotiation. */ - reg_val |= IXGBE_KRM_LINK_CTRL_1_TETH_AN_RESTART; status = hw->mac.ops.write_iosf_sb_reg(hw, - IXGBE_KRM_LINK_CTRL_1(hw->bus.lan_id), - IXGBE_SB_IOSF_TARGET_KR_PHY, reg_val); + IXGBE_KRM_LINK_CTRL_1(hw->bus.lan_id), + IXGBE_SB_IOSF_TARGET_KR_PHY, reg_val); - return status; -} + if (hw->mac.type == ixgbe_mac_X550EM_a) { + /* Set lane mode to KR auto negotiation */ + status = hw->mac.ops.read_iosf_sb_reg(hw, + IXGBE_KRM_PMD_FLX_MASK_ST20(hw->bus.lan_id), + IXGBE_SB_IOSF_TARGET_KR_PHY, ®_val); -/** - * ixgbe_set_master_slave_mode - Set up PHY for master/slave mode - * @hw: pointer to hardware structure - * - * Must be called while holding the PHY semaphore and token - */ -STATIC s32 ixgbe_set_master_slave_mode(struct ixgbe_hw *hw) -{ - u16 phy_data; - s32 rc; + if (status) + return status; - /* Resolve master/slave mode */ - rc = ixgbe_read_phy_reg_mdi_22(hw, IXGBE_M88E1500_1000T_CTRL, 0, - &phy_data); - if (rc) - return rc; + reg_val &= ~IXGBE_KRM_PMD_FLX_MASK_ST20_SPEED_MASK; + reg_val |= IXGBE_KRM_PMD_FLX_MASK_ST20_SPEED_AN; + reg_val |= IXGBE_KRM_PMD_FLX_MASK_ST20_AN_EN; + reg_val &= ~IXGBE_KRM_PMD_FLX_MASK_ST20_AN37_EN; + reg_val &= ~IXGBE_KRM_PMD_FLX_MASK_ST20_SGMII_EN; - /* load defaults for future use */ - if (phy_data & IXGBE_M88E1500_1000T_CTRL_MS_ENABLE) { - if (phy_data & IXGBE_M88E1500_1000T_CTRL_MS_VALUE) - hw->phy.original_ms_type = ixgbe_ms_force_master; - else - hw->phy.original_ms_type = ixgbe_ms_force_slave; - } else { - hw->phy.original_ms_type = ixgbe_ms_auto; + status = hw->mac.ops.write_iosf_sb_reg(hw, + IXGBE_KRM_PMD_FLX_MASK_ST20(hw->bus.lan_id), + IXGBE_SB_IOSF_TARGET_KR_PHY, reg_val); } - switch (hw->phy.ms_type) { - case ixgbe_ms_force_master: - phy_data |= IXGBE_M88E1500_1000T_CTRL_MS_ENABLE; - phy_data |= IXGBE_M88E1500_1000T_CTRL_MS_VALUE; - break; - case ixgbe_ms_force_slave: - phy_data |= IXGBE_M88E1500_1000T_CTRL_MS_ENABLE; - phy_data &= ~IXGBE_M88E1500_1000T_CTRL_MS_VALUE; - break; - case ixgbe_ms_auto: - phy_data &= ~IXGBE_M88E1500_1000T_CTRL_MS_ENABLE; - break; - default: - break; - } - - return ixgbe_write_phy_reg_mdi_22(hw, IXGBE_M88E1500_1000T_CTRL, 0, - phy_data); + return ixgbe_restart_an_internal_phy_x550em(hw); } /** - * ixgbe_reset_phy_m88_nolock - Reset m88 PHY without locking - * @hw: pointer to hardware structure - * - * Must be called while holding the PHY semaphore and token - */ -STATIC s32 ixgbe_reset_phy_m88_nolock(struct ixgbe_hw *hw) -{ - s32 rc; - - rc = ixgbe_write_phy_reg_mdi_22(hw, IXGBE_M88E1500_PAGE_ADDR, 0, 1); - if (rc) - return rc; - - rc = ixgbe_write_phy_reg_mdi_22(hw, IXGBE_M88E1500_FIBER_CTRL, 0, - IXGBE_M88E1500_FIBER_CTRL_RESET | - IXGBE_M88E1500_FIBER_CTRL_DUPLEX_FULL | - IXGBE_M88E1500_FIBER_CTRL_SPEED_MSB); - if (rc) - goto res_out; - - rc = ixgbe_write_phy_reg_mdi_22(hw, IXGBE_M88E1500_PAGE_ADDR, 0, 18); - if (rc) - goto res_out; - - rc = ixgbe_write_phy_reg_mdi_22(hw, IXGBE_M88E1500_GEN_CTRL, 0, - IXGBE_M88E1500_GEN_CTRL_RESET | - IXGBE_M88E1500_GEN_CTRL_SGMII_COPPER); - if (rc) - goto res_out; - - rc = ixgbe_write_phy_reg_mdi_22(hw, IXGBE_M88E1500_PAGE_ADDR, 0, 0); - if (rc) - goto res_out; - - rc = ixgbe_write_phy_reg_mdi_22(hw, IXGBE_M88E1500_COPPER_CTRL, 0, - IXGBE_M88E1500_COPPER_CTRL_RESET | - IXGBE_M88E1500_COPPER_CTRL_AN_EN | - IXGBE_M88E1500_COPPER_CTRL_RESTART_AN | - IXGBE_M88E1500_COPPER_CTRL_FULL_DUPLEX | - IXGBE_M88E1500_COPPER_CTRL_SPEED_MSB); - -res_out: - ixgbe_write_phy_reg_mdi_22(hw, IXGBE_M88E1500_PAGE_ADDR, 0, 0); - return rc; -} - -/** - * ixgbe_reset_phy_m88 - Reset m88 PHY + * ixgbe_reset_phy_fw - Reset firmware-controlled PHYs * @hw: pointer to hardware structure */ -STATIC s32 ixgbe_reset_phy_m88(struct ixgbe_hw *hw) +static s32 ixgbe_reset_phy_fw(struct ixgbe_hw *hw) { - u32 swfw_mask = hw->phy.phy_semaphore_mask; + u32 store[FW_PHY_ACT_DATA_COUNT] = { 0 }; s32 rc; if (hw->phy.reset_disable || ixgbe_check_reset_blocked(hw)) return IXGBE_SUCCESS; - rc = hw->mac.ops.acquire_swfw_sync(hw, swfw_mask); + rc = ixgbe_fw_phy_activity(hw, FW_PHY_ACT_PHY_SW_RESET, &store); + if (rc) + return rc; + memset(store, 0, sizeof(store)); + + rc = ixgbe_fw_phy_activity(hw, FW_PHY_ACT_INIT_PHY, &store); if (rc) return rc; - rc = ixgbe_reset_phy_m88_nolock(hw); - - hw->mac.ops.release_swfw_sync(hw, swfw_mask); - return rc; + return ixgbe_setup_fw_link(hw); } /** - * ixgbe_setup_m88 - setup m88 PHY + * ixgbe_check_overtemp_fw - Check firmware-controlled PHYs for overtemp * @hw: pointer to hardware structure */ -STATIC s32 ixgbe_setup_m88(struct ixgbe_hw *hw) +static s32 ixgbe_check_overtemp_fw(struct ixgbe_hw *hw) { - u32 swfw_mask = hw->phy.phy_semaphore_mask; - struct ixgbe_phy_info *phy = &hw->phy; - u16 phy_data; + u32 store[FW_PHY_ACT_DATA_COUNT] = { 0 }; s32 rc; - if (phy->reset_disable || ixgbe_check_reset_blocked(hw)) - return IXGBE_SUCCESS; - - rc = hw->mac.ops.acquire_swfw_sync(hw, swfw_mask); + rc = ixgbe_fw_phy_activity(hw, FW_PHY_ACT_GET_LINK_INFO, &store); if (rc) return rc; - rc = ixgbe_read_phy_reg_mdi_22(hw, IXGBE_M88E1500_PHY_SPEC_CTRL, 0, - &phy_data); - if (rc) - goto rel_out; - - /* Enable downshift and setting it to X6 */ - phy_data &= ~IXGBE_M88E1500_PSCR_DOWNSHIFT_ENABLE; - phy_data |= IXGBE_M88E1500_PSCR_DOWNSHIFT_6X; - phy_data |= IXGBE_M88E1500_PSCR_DOWNSHIFT_ENABLE; - rc = ixgbe_write_phy_reg_mdi_22(hw, - IXGBE_M88E1500_PHY_SPEC_CTRL, 0, - phy_data); - if (rc) - goto rel_out; - - ixgbe_write_phy_reg_mdi_22(hw, IXGBE_M88E1500_PAGE_ADDR, 0, 0); - - /* Commit the changes */ - rc = ixgbe_reset_phy_m88_nolock(hw); - if (rc) { - DEBUGOUT("Error committing the PHY changes\n"); - goto rel_out; + if (store[0] & FW_PHY_ACT_GET_LINK_INFO_TEMP) { + ixgbe_shutdown_fw_phy(hw); + return IXGBE_ERR_OVERTEMP; } - - rc = ixgbe_set_master_slave_mode(hw); - - hw->mac.ops.release_swfw_sync(hw, swfw_mask); - return rc; - -rel_out: - ixgbe_write_phy_reg_mdi_22(hw, IXGBE_M88E1500_PAGE_ADDR, 0, 0); - hw->mac.ops.release_swfw_sync(hw, swfw_mask); - return rc; + return IXGBE_SUCCESS; } /** @@ -2151,7 +2343,7 @@ STATIC s32 ixgbe_read_mng_if_sel_x550em(struct ixgbe_hw *hw) hw->phy.nw_mng_if_sel & IXGBE_NW_MNG_IF_SEL_MDIO_ACT) { hw->phy.addr = (hw->phy.nw_mng_if_sel & IXGBE_NW_MNG_IF_SEL_MDIO_PHY_ADD) >> - IXGBE_NW_MNG_IF_SEL_MDIO_PHY_ADD_SHIFT; + IXGBE_NW_MNG_IF_SEL_MDIO_PHY_ADD_SHIFT; } return IXGBE_SUCCESS; @@ -2173,7 +2365,6 @@ s32 ixgbe_init_phy_ops_X550em(struct ixgbe_hw *hw) DEBUGFUNC("ixgbe_init_phy_ops_X550em"); hw->mac.ops.set_lan_id(hw); - ixgbe_read_mng_if_sel_x550em(hw); if (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_fiber) { @@ -2182,9 +2373,41 @@ s32 ixgbe_init_phy_ops_X550em(struct ixgbe_hw *hw) phy->ops.identify_sfp = ixgbe_identify_sfp_module_X550em; } + switch (hw->device_id) { + case IXGBE_DEV_ID_X550EM_A_1G_T: + case IXGBE_DEV_ID_X550EM_A_1G_T_L: + phy->ops.read_reg_mdi = ixgbe_read_phy_reg_mdi_22; + phy->ops.write_reg_mdi = ixgbe_write_phy_reg_mdi_22; + hw->phy.ops.read_reg = ixgbe_read_phy_reg_x550a; + hw->phy.ops.write_reg = ixgbe_write_phy_reg_x550a; + phy->ops.check_overtemp = ixgbe_check_overtemp_fw; + if (hw->bus.lan_id) + hw->phy.phy_semaphore_mask |= IXGBE_GSSR_PHY1_SM; + else + hw->phy.phy_semaphore_mask |= IXGBE_GSSR_PHY0_SM; + + break; + case IXGBE_DEV_ID_X550EM_A_10G_T: + case IXGBE_DEV_ID_X550EM_A_SFP: + hw->phy.ops.read_reg = ixgbe_read_phy_reg_x550a; + hw->phy.ops.write_reg = ixgbe_write_phy_reg_x550a; + if (hw->bus.lan_id) + hw->phy.phy_semaphore_mask |= IXGBE_GSSR_PHY1_SM; + else + hw->phy.phy_semaphore_mask |= IXGBE_GSSR_PHY0_SM; + break; + case IXGBE_DEV_ID_X550EM_X_SFP: + /* set up for CS4227 usage */ + hw->phy.phy_semaphore_mask = IXGBE_GSSR_SHARED_I2C_SM; + break; + default: + break; + } + /* Identify the PHY or SFP module */ ret_val = phy->ops.identify(hw); - if (ret_val == IXGBE_ERR_SFP_NOT_SUPPORTED) + if (ret_val == IXGBE_ERR_SFP_NOT_SUPPORTED || + ret_val == IXGBE_ERR_PHY_ADDR_INVALID) return ret_val; /* Setup function pointers based on detected hardware */ @@ -2204,6 +2427,17 @@ s32 ixgbe_init_phy_ops_X550em(struct ixgbe_hw *hw) phy->ops.read_reg = ixgbe_read_phy_reg_x550em; phy->ops.write_reg = ixgbe_write_phy_reg_x550em; break; + case ixgbe_phy_ext_1g_t: + /* link is managed by FW */ + phy->ops.setup_link = NULL; + phy->ops.reset = NULL; + break; + case ixgbe_phy_x550em_xfi: + /* link is managed by HW */ + phy->ops.setup_link = NULL; + phy->ops.read_reg = ixgbe_read_phy_reg_x550em; + phy->ops.write_reg = ixgbe_write_phy_reg_x550em; + break; case ixgbe_phy_x550em_ext_t: /* If internal link mode is XFI, then setup iXFI internal link, * else setup KR now. @@ -2223,11 +2457,9 @@ s32 ixgbe_init_phy_ops_X550em(struct ixgbe_hw *hw) case ixgbe_phy_sgmii: phy->ops.setup_link = NULL; break; - case ixgbe_phy_m88: - phy->ops.setup_link = ixgbe_setup_m88; - phy->ops.read_reg_mdi = ixgbe_read_phy_reg_mdi_22; - phy->ops.write_reg_mdi = ixgbe_write_phy_reg_mdi_22; - phy->ops.reset = ixgbe_reset_phy_m88; + case ixgbe_phy_fw: + phy->ops.setup_link = ixgbe_setup_fw_link; + phy->ops.reset = ixgbe_reset_phy_fw; break; default: break; @@ -2247,8 +2479,6 @@ STATIC void ixgbe_set_mdio_speed(struct ixgbe_hw *hw) case IXGBE_DEV_ID_X550EM_X_10G_T: case IXGBE_DEV_ID_X550EM_A_SGMII: case IXGBE_DEV_ID_X550EM_A_SGMII_L: - case IXGBE_DEV_ID_X550EM_A_1G_T: - case IXGBE_DEV_ID_X550EM_A_1G_T_L: case IXGBE_DEV_ID_X550EM_A_10G_T: case IXGBE_DEV_ID_X550EM_A_SFP: case IXGBE_DEV_ID_X550EM_A_QSFP: @@ -2257,6 +2487,13 @@ STATIC void ixgbe_set_mdio_speed(struct ixgbe_hw *hw) hlreg0 &= ~IXGBE_HLREG0_MDCSPD; IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0); break; + case IXGBE_DEV_ID_X550EM_A_1G_T: + case IXGBE_DEV_ID_X550EM_A_1G_T_L: + /* Select fast MDIO clock speed for these devices */ + hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0); + hlreg0 |= IXGBE_HLREG0_MDCSPD; + IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0); + break; default: break; } @@ -2277,14 +2514,16 @@ s32 ixgbe_reset_hw_X550em(struct ixgbe_hw *hw) u32 ctrl = 0; u32 i; bool link_up = false; + u32 swfw_mask = hw->phy.phy_semaphore_mask; DEBUGFUNC("ixgbe_reset_hw_X550em"); /* Call adapter stop to disable Tx/Rx and clear interrupts */ status = hw->mac.ops.stop_adapter(hw); - if (status != IXGBE_SUCCESS) + if (status != IXGBE_SUCCESS) { + DEBUGOUT1("Failed to stop adapter, STATUS = %d\n", status); return status; - + } /* flush pending Tx transactions */ ixgbe_clear_tx_pending(hw); @@ -2293,14 +2532,23 @@ s32 ixgbe_reset_hw_X550em(struct ixgbe_hw *hw) /* PHY ops must be identified and initialized prior to reset */ status = hw->phy.ops.init(hw); - if (status == IXGBE_ERR_SFP_NOT_SUPPORTED) + if (status) + DEBUGOUT1("Failed to initialize PHY ops, STATUS = %d\n", + status); + + if (status == IXGBE_ERR_SFP_NOT_SUPPORTED) { + DEBUGOUT("Returning from reset HW due to PHY init failure\n"); return status; + } /* start the external PHY */ if (hw->phy.type == ixgbe_phy_x550em_ext_t) { status = ixgbe_init_ext_t_x550em(hw); - if (status) + if (status) { + DEBUGOUT1("Failed to start the external PHY, STATUS = %d\n", + status); return status; + } } /* Setup SFP module if there is one present. */ @@ -2313,8 +2561,10 @@ s32 ixgbe_reset_hw_X550em(struct ixgbe_hw *hw) return status; /* Reset PHY */ - if (!hw->phy.reset_disable && hw->phy.ops.reset) - hw->phy.ops.reset(hw); + if (!hw->phy.reset_disable && hw->phy.ops.reset) { + if (hw->phy.ops.reset(hw) == IXGBE_ERR_OVERTEMP) + return IXGBE_ERR_OVERTEMP; + } mac_reset_top: /* Issue global reset to the MAC. Needs to be SW reset if link is up. @@ -2329,9 +2579,16 @@ mac_reset_top: ctrl = IXGBE_CTRL_RST; } + status = hw->mac.ops.acquire_swfw_sync(hw, swfw_mask); + if (status != IXGBE_SUCCESS) { + ERROR_REPORT2(IXGBE_ERROR_CAUTION, + "semaphore failed with %d", status); + return IXGBE_ERR_SWFW_SYNC; + } ctrl |= IXGBE_READ_REG(hw, IXGBE_CTRL); IXGBE_WRITE_REG(hw, IXGBE_CTRL, ctrl); IXGBE_WRITE_FLUSH(hw); + hw->mac.ops.release_swfw_sync(hw, swfw_mask); /* Poll for reset bit to self-clear meaning reset is complete */ for (i = 0; i < 10; i++) { @@ -2372,6 +2629,9 @@ mac_reset_top: if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP) ixgbe_setup_mux_ctl(hw); + if (status != IXGBE_SUCCESS) + DEBUGOUT1("Reset HW failed, STATUS = %d\n", status); + return status; } @@ -2421,14 +2681,16 @@ s32 ixgbe_init_ext_t_x550em(struct ixgbe_hw *hw) /** * ixgbe_setup_kr_x550em - Configure the KR PHY. * @hw: pointer to hardware structure - * - * Configures the integrated KR PHY for X550EM_x. **/ s32 ixgbe_setup_kr_x550em(struct ixgbe_hw *hw) { - if (hw->mac.type != ixgbe_mac_X550EM_x) + /* leave link alone for 2.5G */ + if (hw->phy.autoneg_advertised & IXGBE_LINK_SPEED_2_5GB_FULL) return IXGBE_SUCCESS; + if (ixgbe_check_reset_blocked(hw)) + return 0; + return ixgbe_setup_kr_speed_x550em(hw, hw->phy.autoneg_advertised); } @@ -2460,56 +2722,70 @@ s32 ixgbe_setup_mac_link_sfp_x550em(struct ixgbe_hw *hw, if (ret_val != IXGBE_SUCCESS) return ret_val; - if (!(hw->phy.nw_mng_if_sel & IXGBE_NW_MNG_IF_SEL_INT_PHY_MODE)) { - /* Configure CS4227 LINE side to 10G SR. */ - reg_slice = IXGBE_CS4227_LINE_SPARE22_MSB + - (hw->bus.lan_id << 12); - reg_val = IXGBE_CS4227_SPEED_10G; - ret_val = hw->link.ops.write_link(hw, hw->link.addr, reg_slice, - reg_val); + /* Configure internal PHY for KR/KX. */ + ixgbe_setup_kr_speed_x550em(hw, speed); - reg_slice = IXGBE_CS4227_LINE_SPARE24_LSB + - (hw->bus.lan_id << 12); + /* Configure CS4227 LINE side to proper mode. */ + reg_slice = IXGBE_CS4227_LINE_SPARE24_LSB + + (hw->bus.lan_id << 12); + if (setup_linear) + reg_val = (IXGBE_CS4227_EDC_MODE_CX1 << 1) | 0x1; + else reg_val = (IXGBE_CS4227_EDC_MODE_SR << 1) | 0x1; - ret_val = hw->link.ops.write_link(hw, hw->link.addr, reg_slice, - reg_val); - - /* Configure CS4227 for HOST connection rate then type. */ - reg_slice = IXGBE_CS4227_HOST_SPARE22_MSB + - (hw->bus.lan_id << 12); - reg_val = (speed & IXGBE_LINK_SPEED_10GB_FULL) ? - IXGBE_CS4227_SPEED_10G : IXGBE_CS4227_SPEED_1G; - ret_val = hw->link.ops.write_link(hw, hw->link.addr, reg_slice, - reg_val); - - reg_slice = IXGBE_CS4227_HOST_SPARE24_LSB + - (hw->bus.lan_id << 12); - if (setup_linear) - reg_val = (IXGBE_CS4227_EDC_MODE_CX1 << 1) | 0x1; - else - reg_val = (IXGBE_CS4227_EDC_MODE_SR << 1) | 0x1; - ret_val = hw->link.ops.write_link(hw, hw->link.addr, reg_slice, - reg_val); - - /* Setup XFI internal link. */ - ret_val = ixgbe_setup_ixfi_x550em(hw, &speed); - } else { - /* Configure internal PHY for KR/KX. */ - ixgbe_setup_kr_speed_x550em(hw, speed); - - /* Configure CS4227 LINE side to proper mode. */ - reg_slice = IXGBE_CS4227_LINE_SPARE24_LSB + - (hw->bus.lan_id << 12); - if (setup_linear) - reg_val = (IXGBE_CS4227_EDC_MODE_CX1 << 1) | 0x1; - else - reg_val = (IXGBE_CS4227_EDC_MODE_SR << 1) | 0x1; - ret_val = hw->link.ops.write_link(hw, hw->link.addr, reg_slice, - reg_val); - } + ret_val = hw->link.ops.write_link(hw, hw->link.addr, reg_slice, + reg_val); return ret_val; } +/** + * ixgbe_setup_sfi_x550a - Configure the internal PHY for native SFI mode + * @hw: pointer to hardware structure + * @speed: the link speed to force + * + * Configures the integrated PHY for native SFI mode. Used to connect the + * internal PHY directly to an SFP cage, without autonegotiation. + **/ +STATIC s32 ixgbe_setup_sfi_x550a(struct ixgbe_hw *hw, ixgbe_link_speed *speed) +{ + struct ixgbe_mac_info *mac = &hw->mac; + s32 status; + u32 reg_val; + + /* Disable all AN and force speed to 10G Serial. */ + status = mac->ops.read_iosf_sb_reg(hw, + IXGBE_KRM_PMD_FLX_MASK_ST20(hw->bus.lan_id), + IXGBE_SB_IOSF_TARGET_KR_PHY, ®_val); + if (status != IXGBE_SUCCESS) + return status; + + reg_val &= ~IXGBE_KRM_PMD_FLX_MASK_ST20_AN_EN; + reg_val &= ~IXGBE_KRM_PMD_FLX_MASK_ST20_AN37_EN; + reg_val &= ~IXGBE_KRM_PMD_FLX_MASK_ST20_SGMII_EN; + reg_val &= ~IXGBE_KRM_PMD_FLX_MASK_ST20_SPEED_MASK; + + /* Select forced link speed for internal PHY. */ + switch (*speed) { + case IXGBE_LINK_SPEED_10GB_FULL: + reg_val |= IXGBE_KRM_PMD_FLX_MASK_ST20_SPEED_10G; + break; + case IXGBE_LINK_SPEED_1GB_FULL: + reg_val |= IXGBE_KRM_PMD_FLX_MASK_ST20_SPEED_1G; + break; + default: + /* Other link speeds are not supported by internal PHY. */ + return IXGBE_ERR_LINK_SETUP; + } + + status = mac->ops.write_iosf_sb_reg(hw, + IXGBE_KRM_PMD_FLX_MASK_ST20(hw->bus.lan_id), + IXGBE_SB_IOSF_TARGET_KR_PHY, reg_val); + + /* Toggle port SW reset by AN reset. */ + status = ixgbe_restart_an_internal_phy_x550em(hw); + + return status; +} + /** * ixgbe_setup_mac_link_sfp_x550a - Setup internal PHY for SFP * @hw: pointer to hardware structure @@ -2517,8 +2793,8 @@ s32 ixgbe_setup_mac_link_sfp_x550em(struct ixgbe_hw *hw, * Configure the the integrated PHY for SFP support. **/ s32 ixgbe_setup_mac_link_sfp_x550a(struct ixgbe_hw *hw, - ixgbe_link_speed speed, - bool autoneg_wait_to_complete) + ixgbe_link_speed speed, + bool autoneg_wait_to_complete) { s32 ret_val; u16 reg_phy_ext; @@ -2540,31 +2816,27 @@ s32 ixgbe_setup_mac_link_sfp_x550a(struct ixgbe_hw *hw, return ret_val; if (hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP_N) { - /* Configure internal PHY for native SFI */ + /* Configure internal PHY for native SFI based on module type */ ret_val = hw->mac.ops.read_iosf_sb_reg(hw, - IXGBE_KRM_AN_CNTL_8(hw->bus.lan_id), - IXGBE_SB_IOSF_TARGET_KR_PHY, ®_phy_int); + IXGBE_KRM_PMD_FLX_MASK_ST20(hw->bus.lan_id), + IXGBE_SB_IOSF_TARGET_KR_PHY, ®_phy_int); if (ret_val != IXGBE_SUCCESS) return ret_val; - if (setup_linear) { - reg_phy_int &= ~IXGBE_KRM_AN_CNTL_8_LIMITING; - reg_phy_int |= IXGBE_KRM_AN_CNTL_8_LINEAR; - } else { - reg_phy_int |= IXGBE_KRM_AN_CNTL_8_LIMITING; - reg_phy_int &= ~IXGBE_KRM_AN_CNTL_8_LINEAR; - } + reg_phy_int &= IXGBE_KRM_PMD_FLX_MASK_ST20_SFI_10G_DA; + if (!setup_linear) + reg_phy_int |= IXGBE_KRM_PMD_FLX_MASK_ST20_SFI_10G_SR; ret_val = hw->mac.ops.write_iosf_sb_reg(hw, - IXGBE_KRM_AN_CNTL_8(hw->bus.lan_id), - IXGBE_SB_IOSF_TARGET_KR_PHY, reg_phy_int); + IXGBE_KRM_PMD_FLX_MASK_ST20(hw->bus.lan_id), + IXGBE_SB_IOSF_TARGET_KR_PHY, reg_phy_int); if (ret_val != IXGBE_SUCCESS) return ret_val; - /* Setup XFI/SFI internal link. */ - ret_val = ixgbe_setup_ixfi_x550em(hw, &speed); + /* Setup SFI internal link. */ + ret_val = ixgbe_setup_sfi_x550a(hw, &speed); } else { /* Configure internal PHY for KR/KX. */ ixgbe_setup_kr_speed_x550em(hw, speed); @@ -2575,9 +2847,9 @@ s32 ixgbe_setup_mac_link_sfp_x550a(struct ixgbe_hw *hw, return IXGBE_ERR_PHY_ADDR_INVALID; } - /* Get external PHY device id */ - ret_val = hw->phy.ops.read_reg(hw, IXGBE_CS4227_GLOBAL_ID_MSB, - IXGBE_MDIO_ZERO_DEV_TYPE, ®_phy_ext); + /* Get external PHY SKU id */ + ret_val = hw->phy.ops.read_reg(hw, IXGBE_CS4227_EFUSE_PDF_SKU, + IXGBE_MDIO_ZERO_DEV_TYPE, ®_phy_ext); if (ret_val != IXGBE_SUCCESS) return ret_val; @@ -2585,7 +2857,7 @@ s32 ixgbe_setup_mac_link_sfp_x550a(struct ixgbe_hw *hw, /* When configuring quad port CS4223, the MAC instance is part * of the slice offset. */ - if (reg_phy_ext == IXGBE_CS4223_PHY_ID) + if (reg_phy_ext == IXGBE_CS4223_SKU_ID) slice_offset = (hw->bus.lan_id + (hw->bus.instance_id << 1)) << 12; else @@ -2593,12 +2865,26 @@ s32 ixgbe_setup_mac_link_sfp_x550a(struct ixgbe_hw *hw, /* Configure CS4227/CS4223 LINE side to proper mode. */ reg_slice = IXGBE_CS4227_LINE_SPARE24_LSB + slice_offset; + + ret_val = hw->phy.ops.read_reg(hw, reg_slice, + IXGBE_MDIO_ZERO_DEV_TYPE, ®_phy_ext); + + if (ret_val != IXGBE_SUCCESS) + return ret_val; + + reg_phy_ext &= ~((IXGBE_CS4227_EDC_MODE_CX1 << 1) | + (IXGBE_CS4227_EDC_MODE_SR << 1)); + if (setup_linear) reg_phy_ext = (IXGBE_CS4227_EDC_MODE_CX1 << 1) | 0x1; else reg_phy_ext = (IXGBE_CS4227_EDC_MODE_SR << 1) | 0x1; ret_val = hw->phy.ops.write_reg(hw, reg_slice, - IXGBE_MDIO_ZERO_DEV_TYPE, reg_phy_ext); + IXGBE_MDIO_ZERO_DEV_TYPE, reg_phy_ext); + + /* Flush previous write with a read */ + ret_val = hw->phy.ops.read_reg(hw, reg_slice, + IXGBE_MDIO_ZERO_DEV_TYPE, ®_phy_ext); } return ret_val; } @@ -2611,24 +2897,25 @@ s32 ixgbe_setup_mac_link_sfp_x550a(struct ixgbe_hw *hw, **/ STATIC s32 ixgbe_setup_ixfi_x550em_x(struct ixgbe_hw *hw) { + struct ixgbe_mac_info *mac = &hw->mac; s32 status; u32 reg_val; /* Disable training protocol FSM. */ - status = ixgbe_read_iosf_sb_reg_x550(hw, + status = mac->ops.read_iosf_sb_reg(hw, IXGBE_KRM_RX_TRN_LINKUP_CTRL(hw->bus.lan_id), IXGBE_SB_IOSF_TARGET_KR_PHY, ®_val); if (status != IXGBE_SUCCESS) return status; reg_val |= IXGBE_KRM_RX_TRN_LINKUP_CTRL_CONV_WO_PROTOCOL; - status = ixgbe_write_iosf_sb_reg_x550(hw, + status = mac->ops.write_iosf_sb_reg(hw, IXGBE_KRM_RX_TRN_LINKUP_CTRL(hw->bus.lan_id), IXGBE_SB_IOSF_TARGET_KR_PHY, reg_val); if (status != IXGBE_SUCCESS) return status; /* Disable Flex from training TXFFE. */ - status = ixgbe_read_iosf_sb_reg_x550(hw, + status = mac->ops.read_iosf_sb_reg(hw, IXGBE_KRM_DSP_TXFFE_STATE_4(hw->bus.lan_id), IXGBE_SB_IOSF_TARGET_KR_PHY, ®_val); if (status != IXGBE_SUCCESS) @@ -2636,12 +2923,12 @@ STATIC s32 ixgbe_setup_ixfi_x550em_x(struct ixgbe_hw *hw) reg_val &= ~IXGBE_KRM_DSP_TXFFE_STATE_C0_EN; reg_val &= ~IXGBE_KRM_DSP_TXFFE_STATE_CP1_CN1_EN; reg_val &= ~IXGBE_KRM_DSP_TXFFE_STATE_CO_ADAPT_EN; - status = ixgbe_write_iosf_sb_reg_x550(hw, + status = mac->ops.write_iosf_sb_reg(hw, IXGBE_KRM_DSP_TXFFE_STATE_4(hw->bus.lan_id), IXGBE_SB_IOSF_TARGET_KR_PHY, reg_val); if (status != IXGBE_SUCCESS) return status; - status = ixgbe_read_iosf_sb_reg_x550(hw, + status = mac->ops.read_iosf_sb_reg(hw, IXGBE_KRM_DSP_TXFFE_STATE_5(hw->bus.lan_id), IXGBE_SB_IOSF_TARGET_KR_PHY, ®_val); if (status != IXGBE_SUCCESS) @@ -2649,14 +2936,14 @@ STATIC s32 ixgbe_setup_ixfi_x550em_x(struct ixgbe_hw *hw) reg_val &= ~IXGBE_KRM_DSP_TXFFE_STATE_C0_EN; reg_val &= ~IXGBE_KRM_DSP_TXFFE_STATE_CP1_CN1_EN; reg_val &= ~IXGBE_KRM_DSP_TXFFE_STATE_CO_ADAPT_EN; - status = ixgbe_write_iosf_sb_reg_x550(hw, + status = mac->ops.write_iosf_sb_reg(hw, IXGBE_KRM_DSP_TXFFE_STATE_5(hw->bus.lan_id), IXGBE_SB_IOSF_TARGET_KR_PHY, reg_val); if (status != IXGBE_SUCCESS) return status; /* Enable override for coefficients. */ - status = ixgbe_read_iosf_sb_reg_x550(hw, + status = mac->ops.read_iosf_sb_reg(hw, IXGBE_KRM_TX_COEFF_CTRL_1(hw->bus.lan_id), IXGBE_SB_IOSF_TARGET_KR_PHY, ®_val); if (status != IXGBE_SUCCESS) @@ -2665,7 +2952,7 @@ STATIC s32 ixgbe_setup_ixfi_x550em_x(struct ixgbe_hw *hw) reg_val |= IXGBE_KRM_TX_COEFF_CTRL_1_CZERO_EN; reg_val |= IXGBE_KRM_TX_COEFF_CTRL_1_CPLUS1_OVRRD_EN; reg_val |= IXGBE_KRM_TX_COEFF_CTRL_1_CMINUS1_OVRRD_EN; - status = ixgbe_write_iosf_sb_reg_x550(hw, + status = mac->ops.write_iosf_sb_reg(hw, IXGBE_KRM_TX_COEFF_CTRL_1(hw->bus.lan_id), IXGBE_SB_IOSF_TARGET_KR_PHY, reg_val); return status; @@ -2681,11 +2968,16 @@ STATIC s32 ixgbe_setup_ixfi_x550em_x(struct ixgbe_hw *hw) **/ STATIC s32 ixgbe_setup_ixfi_x550em(struct ixgbe_hw *hw, ixgbe_link_speed *speed) { + struct ixgbe_mac_info *mac = &hw->mac; s32 status; u32 reg_val; + /* iXFI is only supported with X552 */ + if (mac->type != ixgbe_mac_X550EM_x) + return IXGBE_ERR_LINK_SETUP; + /* Disable AN and force speed to 10G Serial. */ - status = ixgbe_read_iosf_sb_reg_x550(hw, + status = mac->ops.read_iosf_sb_reg(hw, IXGBE_KRM_LINK_CTRL_1(hw->bus.lan_id), IXGBE_SB_IOSF_TARGET_KR_PHY, ®_val); if (status != IXGBE_SUCCESS) @@ -2707,7 +2999,7 @@ STATIC s32 ixgbe_setup_ixfi_x550em(struct ixgbe_hw *hw, ixgbe_link_speed *speed) return IXGBE_ERR_LINK_SETUP; } - status = ixgbe_write_iosf_sb_reg_x550(hw, + status = mac->ops.write_iosf_sb_reg(hw, IXGBE_KRM_LINK_CTRL_1(hw->bus.lan_id), IXGBE_SB_IOSF_TARGET_KR_PHY, reg_val); if (status != IXGBE_SUCCESS) @@ -2721,15 +3013,7 @@ STATIC s32 ixgbe_setup_ixfi_x550em(struct ixgbe_hw *hw, ixgbe_link_speed *speed) } /* Toggle port SW reset by AN reset. */ - status = ixgbe_read_iosf_sb_reg_x550(hw, - IXGBE_KRM_LINK_CTRL_1(hw->bus.lan_id), - IXGBE_SB_IOSF_TARGET_KR_PHY, ®_val); - if (status != IXGBE_SUCCESS) - return status; - reg_val |= IXGBE_KRM_LINK_CTRL_1_TETH_AN_RESTART; - status = ixgbe_write_iosf_sb_reg_x550(hw, - IXGBE_KRM_LINK_CTRL_1(hw->bus.lan_id), - IXGBE_SB_IOSF_TARGET_KR_PHY, reg_val); + status = ixgbe_restart_an_internal_phy_x550em(hw); return status; } @@ -2788,7 +3072,8 @@ s32 ixgbe_setup_internal_phy_t_x550em(struct ixgbe_hw *hw) if (hw->mac.ops.get_media_type(hw) != ixgbe_media_type_copper) return IXGBE_ERR_CONFIG; - if (!(hw->phy.nw_mng_if_sel & IXGBE_NW_MNG_IF_SEL_INT_PHY_MODE)) { + if (hw->mac.type == ixgbe_mac_X550EM_x && + !(hw->phy.nw_mng_if_sel & IXGBE_NW_MNG_IF_SEL_INT_PHY_MODE)) { /* If link is down, there is no setup necessary so return */ status = ixgbe_ext_phy_t_x550em_get_link(hw, &link_up); if (status != IXGBE_SUCCESS) @@ -2847,56 +3132,56 @@ s32 ixgbe_setup_phy_loopback_x550em(struct ixgbe_hw *hw) /* Disable AN and force speed to 10G Serial. */ status = hw->mac.ops.read_iosf_sb_reg(hw, - IXGBE_KRM_LINK_CTRL_1(hw->bus.lan_id), - IXGBE_SB_IOSF_TARGET_KR_PHY, ®_val); + IXGBE_KRM_LINK_CTRL_1(hw->bus.lan_id), + IXGBE_SB_IOSF_TARGET_KR_PHY, ®_val); if (status != IXGBE_SUCCESS) return status; reg_val &= ~IXGBE_KRM_LINK_CTRL_1_TETH_AN_ENABLE; reg_val &= ~IXGBE_KRM_LINK_CTRL_1_TETH_FORCE_SPEED_MASK; reg_val |= IXGBE_KRM_LINK_CTRL_1_TETH_FORCE_SPEED_10G; status = hw->mac.ops.write_iosf_sb_reg(hw, - IXGBE_KRM_LINK_CTRL_1(hw->bus.lan_id), - IXGBE_SB_IOSF_TARGET_KR_PHY, reg_val); + IXGBE_KRM_LINK_CTRL_1(hw->bus.lan_id), + IXGBE_SB_IOSF_TARGET_KR_PHY, reg_val); if (status != IXGBE_SUCCESS) return status; /* Set near-end loopback clocks. */ status = hw->mac.ops.read_iosf_sb_reg(hw, - IXGBE_KRM_PORT_CAR_GEN_CTRL(hw->bus.lan_id), - IXGBE_SB_IOSF_TARGET_KR_PHY, ®_val); + IXGBE_KRM_PORT_CAR_GEN_CTRL(hw->bus.lan_id), + IXGBE_SB_IOSF_TARGET_KR_PHY, ®_val); if (status != IXGBE_SUCCESS) return status; reg_val |= IXGBE_KRM_PORT_CAR_GEN_CTRL_NELB_32B; reg_val |= IXGBE_KRM_PORT_CAR_GEN_CTRL_NELB_KRPCS; status = hw->mac.ops.write_iosf_sb_reg(hw, - IXGBE_KRM_PORT_CAR_GEN_CTRL(hw->bus.lan_id), - IXGBE_SB_IOSF_TARGET_KR_PHY, reg_val); + IXGBE_KRM_PORT_CAR_GEN_CTRL(hw->bus.lan_id), + IXGBE_SB_IOSF_TARGET_KR_PHY, reg_val); if (status != IXGBE_SUCCESS) return status; /* Set loopback enable. */ status = hw->mac.ops.read_iosf_sb_reg(hw, - IXGBE_KRM_PMD_DFX_BURNIN(hw->bus.lan_id), - IXGBE_SB_IOSF_TARGET_KR_PHY, ®_val); + IXGBE_KRM_PMD_DFX_BURNIN(hw->bus.lan_id), + IXGBE_SB_IOSF_TARGET_KR_PHY, ®_val); if (status != IXGBE_SUCCESS) return status; reg_val |= IXGBE_KRM_PMD_DFX_BURNIN_TX_RX_KR_LB_MASK; status = hw->mac.ops.write_iosf_sb_reg(hw, - IXGBE_KRM_PMD_DFX_BURNIN(hw->bus.lan_id), - IXGBE_SB_IOSF_TARGET_KR_PHY, reg_val); + IXGBE_KRM_PMD_DFX_BURNIN(hw->bus.lan_id), + IXGBE_SB_IOSF_TARGET_KR_PHY, reg_val); if (status != IXGBE_SUCCESS) return status; /* Training bypass. */ status = hw->mac.ops.read_iosf_sb_reg(hw, - IXGBE_KRM_RX_TRN_LINKUP_CTRL(hw->bus.lan_id), - IXGBE_SB_IOSF_TARGET_KR_PHY, ®_val); + IXGBE_KRM_RX_TRN_LINKUP_CTRL(hw->bus.lan_id), + IXGBE_SB_IOSF_TARGET_KR_PHY, ®_val); if (status != IXGBE_SUCCESS) return status; reg_val |= IXGBE_KRM_RX_TRN_LINKUP_CTRL_PROTOCOL_BYPASS; status = hw->mac.ops.write_iosf_sb_reg(hw, - IXGBE_KRM_RX_TRN_LINKUP_CTRL(hw->bus.lan_id), - IXGBE_SB_IOSF_TARGET_KR_PHY, reg_val); + IXGBE_KRM_RX_TRN_LINKUP_CTRL(hw->bus.lan_id), + IXGBE_SB_IOSF_TARGET_KR_PHY, reg_val); return status; } @@ -2969,6 +3254,7 @@ s32 ixgbe_read_ee_hostif_buffer_X550(struct ixgbe_hw *hw, DEBUGOUT("EEPROM read buffer - semaphore failed\n"); return status; } + while (words) { if (words > FW_MAX_READ_BUFFER_SIZE / 2) words_to_read = FW_MAX_READ_BUFFER_SIZE / 2; @@ -3403,9 +3689,9 @@ s32 ixgbe_update_flash_X550(struct ixgbe_hw *hw) * * Determines physical layer capabilities of the current configuration. **/ -u32 ixgbe_get_supported_physical_layer_X550em(struct ixgbe_hw *hw) +u64 ixgbe_get_supported_physical_layer_X550em(struct ixgbe_hw *hw) { - u32 physical_layer = IXGBE_PHYSICAL_LAYER_UNKNOWN; + u64 physical_layer = IXGBE_PHYSICAL_LAYER_UNKNOWN; u16 ext_ability = 0; DEBUGFUNC("ixgbe_get_supported_physical_layer_X550em"); @@ -3414,6 +3700,21 @@ u32 ixgbe_get_supported_physical_layer_X550em(struct ixgbe_hw *hw) switch (hw->phy.type) { case ixgbe_phy_x550em_kr: + if (hw->mac.type == ixgbe_mac_X550EM_a) { + if (hw->phy.nw_mng_if_sel & + IXGBE_NW_MNG_IF_SEL_PHY_SPEED_2_5G) { + physical_layer = + IXGBE_PHYSICAL_LAYER_2500BASE_KX; + break; + } else if (hw->device_id == + IXGBE_DEV_ID_X550EM_A_KR_L) { + physical_layer = + IXGBE_PHYSICAL_LAYER_1000BASE_KX; + break; + } + } + /* fall through */ + case ixgbe_phy_x550em_xfi: physical_layer = IXGBE_PHYSICAL_LAYER_10GBASE_KR | IXGBE_PHYSICAL_LAYER_1000BASE_KX; break; @@ -3430,6 +3731,20 @@ u32 ixgbe_get_supported_physical_layer_X550em(struct ixgbe_hw *hw) if (ext_ability & IXGBE_MDIO_PHY_1000BASET_ABILITY) physical_layer |= IXGBE_PHYSICAL_LAYER_1000BASE_T; break; + case ixgbe_phy_fw: + if (hw->phy.speeds_supported & IXGBE_LINK_SPEED_1GB_FULL) + physical_layer |= IXGBE_PHYSICAL_LAYER_1000BASE_T; + if (hw->phy.speeds_supported & IXGBE_LINK_SPEED_100_FULL) + physical_layer |= IXGBE_PHYSICAL_LAYER_100BASE_TX; + if (hw->phy.speeds_supported & IXGBE_LINK_SPEED_10_FULL) + physical_layer |= IXGBE_PHYSICAL_LAYER_10BASE_T; + break; + case ixgbe_phy_sgmii: + physical_layer = IXGBE_PHYSICAL_LAYER_1000BASE_KX; + break; + case ixgbe_phy_ext_1g_t: + physical_layer |= IXGBE_PHYSICAL_LAYER_1000BASE_T; + break; default: break; } @@ -3712,23 +4027,26 @@ s32 ixgbe_setup_fc_X550em(struct ixgbe_hw *hw) case IXGBE_DEV_ID_X550EM_A_KR: case IXGBE_DEV_ID_X550EM_A_KR_L: ret_val = hw->mac.ops.read_iosf_sb_reg(hw, - IXGBE_KRM_AN_CNTL_1(hw->bus.lan_id), - IXGBE_SB_IOSF_TARGET_KR_PHY, ®_val); + IXGBE_KRM_AN_CNTL_1(hw->bus.lan_id), + IXGBE_SB_IOSF_TARGET_KR_PHY, ®_val); if (ret_val != IXGBE_SUCCESS) goto out; reg_val &= ~(IXGBE_KRM_AN_CNTL_1_SYM_PAUSE | - IXGBE_KRM_AN_CNTL_1_ASM_PAUSE); + IXGBE_KRM_AN_CNTL_1_ASM_PAUSE); if (pause) reg_val |= IXGBE_KRM_AN_CNTL_1_SYM_PAUSE; if (asm_dir) reg_val |= IXGBE_KRM_AN_CNTL_1_ASM_PAUSE; ret_val = hw->mac.ops.write_iosf_sb_reg(hw, - IXGBE_KRM_AN_CNTL_1(hw->bus.lan_id), - IXGBE_SB_IOSF_TARGET_KR_PHY, reg_val); + IXGBE_KRM_AN_CNTL_1(hw->bus.lan_id), + IXGBE_SB_IOSF_TARGET_KR_PHY, reg_val); /* This device does not fully support AN. */ hw->fc.disable_fc_autoneg = true; break; + case IXGBE_DEV_ID_X550EM_X_XFI: + hw->fc.disable_fc_autoneg = true; + break; default: break; } @@ -3738,12 +4056,12 @@ out: } /** - * ixgbe_fc_autoneg_x550a - Enable flow control IEEE clause 37 + * ixgbe_fc_autoneg_backplane_x550em_a - Enable flow control IEEE clause 37 * @hw: pointer to hardware structure * * Enable flow control according to IEEE clause 37. **/ -void ixgbe_fc_autoneg_x550a(struct ixgbe_hw *hw) +void ixgbe_fc_autoneg_backplane_x550em_a(struct ixgbe_hw *hw) { u32 link_s1, lp_an_page_low, an_cntl_1; s32 status = IXGBE_ERR_FC_NOT_NEGOTIATED; @@ -3757,7 +4075,7 @@ void ixgbe_fc_autoneg_x550a(struct ixgbe_hw *hw) */ if (hw->fc.disable_fc_autoneg) { ERROR_REPORT1(IXGBE_ERROR_UNSUPPORTED, - "Flow control autoneg is disabled"); + "Flow control autoneg is disabled"); goto out; } @@ -3769,12 +4087,13 @@ void ixgbe_fc_autoneg_x550a(struct ixgbe_hw *hw) /* Check at auto-negotiation has completed */ status = hw->mac.ops.read_iosf_sb_reg(hw, - IXGBE_KRM_LINK_S1(hw->bus.lan_id), - IXGBE_SB_IOSF_TARGET_KR_PHY, &link_s1); + IXGBE_KRM_LINK_S1(hw->bus.lan_id), + IXGBE_SB_IOSF_TARGET_KR_PHY, &link_s1); if (status != IXGBE_SUCCESS || (link_s1 & IXGBE_KRM_LINK_S1_MAC_AN_COMPLETE) == 0) { DEBUGOUT("Auto-Negotiation did not complete\n"); + status = IXGBE_ERR_FC_NOT_NEGOTIATED; goto out; } @@ -3782,8 +4101,8 @@ void ixgbe_fc_autoneg_x550a(struct ixgbe_hw *hw) * local flow control settings accordingly */ status = hw->mac.ops.read_iosf_sb_reg(hw, - IXGBE_KRM_AN_CNTL_1(hw->bus.lan_id), - IXGBE_SB_IOSF_TARGET_KR_PHY, &an_cntl_1); + IXGBE_KRM_AN_CNTL_1(hw->bus.lan_id), + IXGBE_SB_IOSF_TARGET_KR_PHY, &an_cntl_1); if (status != IXGBE_SUCCESS) { DEBUGOUT("Auto-Negotiation did not complete\n"); @@ -3791,8 +4110,8 @@ void ixgbe_fc_autoneg_x550a(struct ixgbe_hw *hw) } status = hw->mac.ops.read_iosf_sb_reg(hw, - IXGBE_KRM_LP_BASE_PAGE_HIGH(hw->bus.lan_id), - IXGBE_SB_IOSF_TARGET_KR_PHY, &lp_an_page_low); + IXGBE_KRM_LP_BASE_PAGE_HIGH(hw->bus.lan_id), + IXGBE_SB_IOSF_TARGET_KR_PHY, &lp_an_page_low); if (status != IXGBE_SUCCESS) { DEBUGOUT("Auto-Negotiation did not complete\n"); @@ -3815,22 +4134,88 @@ out: } /** - * ixgbe_setup_fc_x550em - Set up flow control + * ixgbe_fc_autoneg_fiber_x550em_a - passthrough FC settings + * @hw: pointer to hardware structure + * + **/ +void ixgbe_fc_autoneg_fiber_x550em_a(struct ixgbe_hw *hw) +{ + hw->fc.fc_was_autonegged = false; + hw->fc.current_mode = hw->fc.requested_mode; +} + +/** + * ixgbe_fc_autoneg_sgmii_x550em_a - Enable flow control IEEE clause 37 + * @hw: pointer to hardware structure + * + * Enable flow control according to IEEE clause 37. + **/ +void ixgbe_fc_autoneg_sgmii_x550em_a(struct ixgbe_hw *hw) +{ + s32 status = IXGBE_ERR_FC_NOT_NEGOTIATED; + u32 info[FW_PHY_ACT_DATA_COUNT] = { 0 }; + ixgbe_link_speed speed; + bool link_up; + + /* AN should have completed when the cable was plugged in. + * Look for reasons to bail out. Bail out if: + * - FC autoneg is disabled, or if + * - link is not up. + */ + if (hw->fc.disable_fc_autoneg) { + ERROR_REPORT1(IXGBE_ERROR_UNSUPPORTED, + "Flow control autoneg is disabled"); + goto out; + } + + hw->mac.ops.check_link(hw, &speed, &link_up, false); + if (!link_up) { + ERROR_REPORT1(IXGBE_ERROR_SOFTWARE, "The link is down"); + goto out; + } + + /* Check if auto-negotiation has completed */ + status = ixgbe_fw_phy_activity(hw, FW_PHY_ACT_GET_LINK_INFO, &info); + if (status != IXGBE_SUCCESS || + !(info[0] & FW_PHY_ACT_GET_LINK_INFO_AN_COMPLETE)) { + DEBUGOUT("Auto-Negotiation did not complete\n"); + status = IXGBE_ERR_FC_NOT_NEGOTIATED; + goto out; + } + + /* Negotiate the flow control */ + status = ixgbe_negotiate_fc(hw, info[0], info[0], + FW_PHY_ACT_GET_LINK_INFO_FC_RX, + FW_PHY_ACT_GET_LINK_INFO_FC_TX, + FW_PHY_ACT_GET_LINK_INFO_LP_FC_RX, + FW_PHY_ACT_GET_LINK_INFO_LP_FC_TX); + +out: + if (status == IXGBE_SUCCESS) { + hw->fc.fc_was_autonegged = true; + } else { + hw->fc.fc_was_autonegged = false; + hw->fc.current_mode = hw->fc.requested_mode; + } +} + +/** + * ixgbe_setup_fc_backplane_x550em_a - Set up flow control * @hw: pointer to hardware structure * * Called at init time to set up flow control. **/ -s32 ixgbe_setup_fc_x550a(struct ixgbe_hw *hw) +s32 ixgbe_setup_fc_backplane_x550em_a(struct ixgbe_hw *hw) { s32 status = IXGBE_SUCCESS; - u32 an_cntl, link_ctrl = 0; + u32 an_cntl = 0; - DEBUGFUNC("ixgbe_setup_fc_x550em"); + DEBUGFUNC("ixgbe_setup_fc_backplane_x550em_a"); /* Validate the requested mode */ if (hw->fc.strict_ieee && hw->fc.requested_mode == ixgbe_fc_rx_pause) { ERROR_REPORT1(IXGBE_ERROR_UNSUPPORTED, - "ixgbe_fc_rx_pause not valid in strict IEEE mode\n"); + "ixgbe_fc_rx_pause not valid in strict IEEE mode\n"); return IXGBE_ERR_INVALID_LINK_SETTINGS; } @@ -3842,8 +4227,8 @@ s32 ixgbe_setup_fc_x550a(struct ixgbe_hw *hw) * we link at 10G, the 1G advertisement is harmless and vice versa. */ status = hw->mac.ops.read_iosf_sb_reg(hw, - IXGBE_KRM_AN_CNTL_1(hw->bus.lan_id), - IXGBE_SB_IOSF_TARGET_KR_PHY, &an_cntl); + IXGBE_KRM_AN_CNTL_1(hw->bus.lan_id), + IXGBE_SB_IOSF_TARGET_KR_PHY, &an_cntl); if (status != IXGBE_SUCCESS) { DEBUGOUT("Auto-Negotiation did not complete\n"); @@ -3884,7 +4269,7 @@ s32 ixgbe_setup_fc_x550a(struct ixgbe_hw *hw) case ixgbe_fc_full: /* Flow control (both Rx and Tx) is enabled by SW override. */ an_cntl |= IXGBE_KRM_AN_CNTL_1_SYM_PAUSE | - IXGBE_KRM_AN_CNTL_1_ASM_PAUSE; + IXGBE_KRM_AN_CNTL_1_ASM_PAUSE; break; default: ERROR_REPORT1(IXGBE_ERROR_ARGUMENT, @@ -3893,23 +4278,11 @@ s32 ixgbe_setup_fc_x550a(struct ixgbe_hw *hw) } status = hw->mac.ops.write_iosf_sb_reg(hw, - IXGBE_KRM_AN_CNTL_1(hw->bus.lan_id), - IXGBE_SB_IOSF_TARGET_KR_PHY, an_cntl); + IXGBE_KRM_AN_CNTL_1(hw->bus.lan_id), + IXGBE_SB_IOSF_TARGET_KR_PHY, an_cntl); /* Restart auto-negotiation. */ - status = hw->mac.ops.read_iosf_sb_reg(hw, - IXGBE_KRM_LINK_CTRL_1(hw->bus.lan_id), - IXGBE_SB_IOSF_TARGET_KR_PHY, &link_ctrl); - - if (status != IXGBE_SUCCESS) { - DEBUGOUT("Auto-Negotiation did not complete\n"); - return status; - } - - link_ctrl |= IXGBE_KRM_LINK_CTRL_1_TETH_AN_RESTART; - status = hw->mac.ops.write_iosf_sb_reg(hw, - IXGBE_KRM_LINK_CTRL_1(hw->bus.lan_id), - IXGBE_SB_IOSF_TARGET_KR_PHY, link_ctrl); + status = ixgbe_restart_an_internal_phy_x550em(hw); return status; } @@ -3993,22 +4366,34 @@ STATIC s32 ixgbe_acquire_swfw_sync_X550a(struct ixgbe_hw *hw, u32 mask) status = IXGBE_SUCCESS; if (hmask) status = ixgbe_acquire_swfw_sync_X540(hw, hmask); - if (status) + if (status) { + DEBUGOUT1("Could not acquire SWFW semaphore, Status = %d\n", + status); return status; + } if (!(mask & IXGBE_GSSR_TOKEN_SM)) return IXGBE_SUCCESS; status = ixgbe_get_phy_token(hw); + if (status == IXGBE_ERR_TOKEN_RETRY) + DEBUGOUT1("Could not acquire PHY token, Status = %d\n", + status); + if (status == IXGBE_SUCCESS) return IXGBE_SUCCESS; if (hmask) ixgbe_release_swfw_sync_X540(hw, hmask); - if (status != IXGBE_ERR_TOKEN_RETRY) + + if (status != IXGBE_ERR_TOKEN_RETRY) { + DEBUGOUT1("Unable to retry acquiring the PHY token, Status = %d\n", + status); return status; - msec_delay(FW_PHY_TOKEN_DELAY); + } } + DEBUGOUT1("Semaphore acquisition retries failed!: PHY ID = 0x%08X\n", + hw->phy.id); return status; } @@ -4043,7 +4428,7 @@ STATIC void ixgbe_release_swfw_sync_X550a(struct ixgbe_hw *hw, u32 mask) * instances. **/ s32 ixgbe_read_phy_reg_x550a(struct ixgbe_hw *hw, u32 reg_addr, - u32 device_type, u16 *phy_data) + u32 device_type, u16 *phy_data) { s32 status; u32 mask = hw->phy.phy_semaphore_mask | IXGBE_GSSR_TOKEN_SM; @@ -4071,7 +4456,7 @@ s32 ixgbe_read_phy_reg_x550a(struct ixgbe_hw *hw, u32 reg_addr, * The PHY Token is needed since the MDIO is shared between to MAC instances. **/ s32 ixgbe_write_phy_reg_x550a(struct ixgbe_hw *hw, u32 reg_addr, - u32 device_type, u16 phy_data) + u32 device_type, u16 phy_data) { s32 status; u32 mask = hw->phy.phy_semaphore_mask | IXGBE_GSSR_TOKEN_SM; @@ -4079,7 +4464,7 @@ s32 ixgbe_write_phy_reg_x550a(struct ixgbe_hw *hw, u32 reg_addr, DEBUGFUNC("ixgbe_write_phy_reg_x550a"); if (hw->mac.ops.acquire_swfw_sync(hw, mask) == IXGBE_SUCCESS) { - status = ixgbe_write_phy_reg_mdi(hw, reg_addr, device_type, + status = hw->phy.ops.write_reg_mdi(hw, reg_addr, device_type, phy_data); hw->mac.ops.release_swfw_sync(hw, mask); } else { @@ -4144,8 +4529,10 @@ s32 ixgbe_setup_mac_link_t_X550em(struct ixgbe_hw *hw, else force_speed = IXGBE_LINK_SPEED_1GB_FULL; - /* If internal link mode is XFI, then setup XFI internal link. */ - if (!(hw->phy.nw_mng_if_sel & IXGBE_NW_MNG_IF_SEL_INT_PHY_MODE)) { + /* If X552 and internal link mode is XFI, then setup XFI internal link. + */ + if (hw->mac.type == ixgbe_mac_X550EM_x && + !(hw->phy.nw_mng_if_sel & IXGBE_NW_MNG_IF_SEL_INT_PHY_MODE)) { status = ixgbe_setup_ixfi_x550em(hw, &force_speed); if (status != IXGBE_SUCCESS) @@ -4168,7 +4555,7 @@ s32 ixgbe_check_link_t_X550em(struct ixgbe_hw *hw, ixgbe_link_speed *speed, bool *link_up, bool link_up_wait_to_complete) { u32 status; - u16 autoneg_status; + u16 i, autoneg_status = 0; if (hw->mac.ops.get_media_type(hw) != ixgbe_media_type_copper) return IXGBE_ERR_CONFIG; @@ -4181,21 +4568,18 @@ s32 ixgbe_check_link_t_X550em(struct ixgbe_hw *hw, ixgbe_link_speed *speed, return status; /* MAC link is up, so check external PHY link. - * Read this twice back to back to indicate current status. + * X557 PHY. Link status is latching low, and can only be used to detect + * link drop, and not the current status of the link without performing + * back-to-back reads. */ - status = hw->phy.ops.read_reg(hw, IXGBE_MDIO_AUTO_NEG_STATUS, - IXGBE_MDIO_AUTO_NEG_DEV_TYPE, - &autoneg_status); + for (i = 0; i < 2; i++) { + status = hw->phy.ops.read_reg(hw, IXGBE_MDIO_AUTO_NEG_STATUS, + IXGBE_MDIO_AUTO_NEG_DEV_TYPE, + &autoneg_status); - if (status != IXGBE_SUCCESS) - return status; - - status = hw->phy.ops.read_reg(hw, IXGBE_MDIO_AUTO_NEG_STATUS, - IXGBE_MDIO_AUTO_NEG_DEV_TYPE, - &autoneg_status); - - if (status != IXGBE_SUCCESS) - return status; + if (status != IXGBE_SUCCESS) + return status; + } /* If external PHY link is not up, then indicate link not up */ if (!(autoneg_status & IXGBE_MDIO_AUTO_NEG_LINK_STATUS)) @@ -4242,7 +4626,8 @@ s32 ixgbe_led_on_t_X550em(struct ixgbe_hw *hw, u32 led_idx) ixgbe_write_phy_reg(hw, IXGBE_X557_LED_PROVISIONING + led_idx, IXGBE_MDIO_VENDOR_SPECIFIC_1_DEV_TYPE, phy_data); - return IXGBE_SUCCESS; + /* Some designs have the LEDs wired to the MAC */ + return ixgbe_led_on_generic(hw, led_idx); } /** @@ -4266,6 +4651,67 @@ s32 ixgbe_led_off_t_X550em(struct ixgbe_hw *hw, u32 led_idx) ixgbe_write_phy_reg(hw, IXGBE_X557_LED_PROVISIONING + led_idx, IXGBE_MDIO_VENDOR_SPECIFIC_1_DEV_TYPE, phy_data); - return IXGBE_SUCCESS; + /* Some designs have the LEDs wired to the MAC */ + return ixgbe_led_off_generic(hw, led_idx); } +/** + * ixgbe_set_fw_drv_ver_x550 - Sends driver version to firmware + * @hw: pointer to the HW structure + * @maj: driver version major number + * @min: driver version minor number + * @build: driver version build number + * @sub: driver version sub build number + * @len: length of driver_ver string + * @driver_ver: driver string + * + * Sends driver version number to firmware through the manageability + * block. On success return IXGBE_SUCCESS + * else returns IXGBE_ERR_SWFW_SYNC when encountering an error acquiring + * semaphore or IXGBE_ERR_HOST_INTERFACE_COMMAND when command fails. + **/ +s32 ixgbe_set_fw_drv_ver_x550(struct ixgbe_hw *hw, u8 maj, u8 min, + u8 build, u8 sub, u16 len, const char *driver_ver) +{ + struct ixgbe_hic_drv_info2 fw_cmd; + s32 ret_val = IXGBE_SUCCESS; + int i; + + DEBUGFUNC("ixgbe_set_fw_drv_ver_x550"); + + if ((len == 0) || (driver_ver == NULL) || + (len > sizeof(fw_cmd.driver_string))) + return IXGBE_ERR_INVALID_ARGUMENT; + + fw_cmd.hdr.cmd = FW_CEM_CMD_DRIVER_INFO; + fw_cmd.hdr.buf_len = FW_CEM_CMD_DRIVER_INFO_LEN + len; + fw_cmd.hdr.cmd_or_resp.cmd_resv = FW_CEM_CMD_RESERVED; + fw_cmd.port_num = (u8)hw->bus.func; + fw_cmd.ver_maj = maj; + fw_cmd.ver_min = min; + fw_cmd.ver_build = build; + fw_cmd.ver_sub = sub; + fw_cmd.hdr.checksum = 0; + memcpy(fw_cmd.driver_string, driver_ver, len); + fw_cmd.hdr.checksum = ixgbe_calculate_checksum((u8 *)&fw_cmd, + (FW_CEM_HDR_LEN + fw_cmd.hdr.buf_len)); + + for (i = 0; i <= FW_CEM_MAX_RETRIES; i++) { + ret_val = ixgbe_host_interface_command(hw, (u32 *)&fw_cmd, + sizeof(fw_cmd), + IXGBE_HI_COMMAND_TIMEOUT, + true); + if (ret_val != IXGBE_SUCCESS) + continue; + + if (fw_cmd.hdr.cmd_or_resp.ret_status == + FW_CEM_RESP_STATUS_SUCCESS) + ret_val = IXGBE_SUCCESS; + else + ret_val = IXGBE_ERR_HOST_INTERFACE_COMMAND; + + break; + } + + return ret_val; +} diff --git a/dpdk/drivers/net/ixgbe/base/ixgbe_x550.h b/dpdk/drivers/net/ixgbe/base/ixgbe_x550.h index 1d4b290c..6d188741 100644 --- a/dpdk/drivers/net/ixgbe/base/ixgbe_x550.h +++ b/dpdk/drivers/net/ixgbe/base/ixgbe_x550.h @@ -36,49 +36,6 @@ POSSIBILITY OF SUCH DAMAGE. #include "ixgbe_type.h" -/* More phy definitions */ -#define IXGBE_M88E1500_COPPER_CTRL 0x0/* Page 0 reg */ -#define IXGBE_M88E1500_COPPER_CTRL_RESET 0x8000 -#define IXGBE_M88E1500_COPPER_CTRL_AN_EN 0x1000 -#define IXGBE_M88E1500_COPPER_CTRL_RESTART_AN 0x0200 -#define IXGBE_M88E1500_COPPER_CTRL_FULL_DUPLEX 0x0100 -#define IXGBE_M88E1500_COPPER_CTRL_SPEED_MSB 0x0040 -#define IXGBE_M88E1500_1000T_CTRL 0x09 /* 1000Base-T Ctrl Reg */ -/* 1=Configure PHY as Master 0=Configure PHY as Slave */ -#define IXGBE_M88E1500_1000T_CTRL_MS_VALUE 0x0800 -/* 1=Master/Slave manual config value 0=Automatic Master/Slave config */ -#define IXGBE_M88E1500_1000T_CTRL_MS_ENABLE 0x1000 -#define IXGBE_M88E1500_1000T_STATUS 0x0A /* 1000Base-T Status Reg */ -#define IXGBE_M88E1500_AUTO_COPPER_SGMII 0x2 -#define IXGBE_M88E1500_AUTO_COPPER_BASEX 0x3 -#define IXGBE_M88E1500_STATUS_LINK 0x0004 /* Interface Link Bit */ -#define IXGBE_M88E1500_MAC_CTRL_1 0x10 -#define IXGBE_M88E1500_MAC_CTRL_1_MODE_MASK 0x0380 /* Mode Select */ -#define IXGBE_M88E1500_CFG_REG_1 0x0010 -#define IXGBE_M88E1500_CFG_REG_2 0x0011 -#define IXGBE_M88E1500_CFG_REG_3 0x0007 -#define IXGBE_M88E1500_MODE 0x0014 -#define IXGBE_M88E1500_PAGE_ADDR 0x16/* Page Offset reg */ -#define IXGBE_M88E1500_FIBER_CTRL 0x0/* Page 1 reg */ -#define IXGBE_M88E1500_FIBER_CTRL_RESET 0x8000 -#define IXGBE_M88E1500_FIBER_CTRL_SPEED_LSB 0x2000 -#define IXGBE_M88E1500_FIBER_CTRL_POWER_DOWN 0x0800 -#define IXGBE_M88E1500_FIBER_CTRL_DUPLEX_FULL 0x0100 -#define IXGBE_M88E1500_FIBER_CTRL_SPEED_MSB 0x0040 -#define IXGBE_M88E1500_EEE_CTRL_1 0x0/* Page 18 reg */ -#define IXGBE_M88E1500_EEE_CTRL_1_MS 0x0001/* EEE Master/Slave */ -#define IXGBE_M88E1500_GEN_CTRL 0x14/* Page 18 reg */ -#define IXGBE_M88E1500_GEN_CTRL_RESET 0x8000 -#define IXGBE_M88E1500_GEN_CTRL_SGMII_COPPER 0x0001/* Mode bits 0-2 */ - -/* M88E1500 Specific Registers */ -#define IXGBE_M88E1500_PHY_SPEC_CTRL 0x10 /* PHY Specific Ctrl Reg */ -#define IXGBE_M88E1500_PHY_SPEC_STATUS 0x11 /* PHY Specific Stat Reg */ - -#define IXGBE_M88E1500_PSCR_DOWNSHIFT_ENABLE 0x0800 -#define IXGBE_M88E1500_PSCR_DOWNSHIFT_MASK 0x7000 -#define IXGBE_M88E1500_PSCR_DOWNSHIFT_6X 0x5000 - s32 ixgbe_dmac_config_X550(struct ixgbe_hw *hw); s32 ixgbe_dmac_config_tcs_X550(struct ixgbe_hw *hw); s32 ixgbe_dmac_update_tcs_X550(struct ixgbe_hw *hw); @@ -100,8 +57,6 @@ s32 ixgbe_read_ee_hostif_X550(struct ixgbe_hw *hw, u16 offset, u16 *data); s32 ixgbe_write_ee_hostif_data_X550(struct ixgbe_hw *hw, u16 offset, u16 data); -s32 ixgbe_set_eee_X550(struct ixgbe_hw *hw, bool enable_eee); -s32 ixgbe_setup_eee_X550(struct ixgbe_hw *hw, bool enable_eee); void ixgbe_set_source_address_pruning_X550(struct ixgbe_hw *hw, bool enable, unsigned int pool); void ixgbe_set_ethertype_anti_spoofing_X550(struct ixgbe_hw *hw, @@ -110,6 +65,8 @@ s32 ixgbe_write_iosf_sb_reg_x550(struct ixgbe_hw *hw, u32 reg_addr, u32 device_type, u32 data); s32 ixgbe_read_iosf_sb_reg_x550(struct ixgbe_hw *hw, u32 reg_addr, u32 device_type, u32 *data); +s32 ixgbe_set_fw_drv_ver_x550(struct ixgbe_hw *hw, u8 maj, u8 min, + u8 build, u8 ver, u16 len, const char *str); s32 ixgbe_get_phy_token(struct ixgbe_hw *); s32 ixgbe_put_phy_token(struct ixgbe_hw *); s32 ixgbe_write_iosf_sb_reg_x550a(struct ixgbe_hw *hw, u32 reg_addr, @@ -131,7 +88,7 @@ s32 ixgbe_setup_kr_x550em(struct ixgbe_hw *hw); s32 ixgbe_init_ext_t_x550em(struct ixgbe_hw *hw); s32 ixgbe_setup_internal_phy_t_x550em(struct ixgbe_hw *hw); s32 ixgbe_setup_phy_loopback_x550em(struct ixgbe_hw *hw); -u32 ixgbe_get_supported_physical_layer_X550em(struct ixgbe_hw *hw); +u64 ixgbe_get_supported_physical_layer_X550em(struct ixgbe_hw *hw); void ixgbe_disable_rx_x550(struct ixgbe_hw *hw); s32 ixgbe_get_lcd_t_x550em(struct ixgbe_hw *hw, ixgbe_link_speed *lcd_speed); s32 ixgbe_enter_lplu_t_x550em(struct ixgbe_hw *hw); @@ -142,14 +99,18 @@ s32 ixgbe_setup_mac_link_sfp_x550em(struct ixgbe_hw *hw, ixgbe_link_speed speed, bool autoneg_wait_to_complete); s32 ixgbe_setup_mac_link_sfp_x550a(struct ixgbe_hw *hw, - ixgbe_link_speed speed, - bool autoneg_wait_to_complete); + ixgbe_link_speed speed, + bool autoneg_wait_to_complete); s32 ixgbe_read_phy_reg_x550a(struct ixgbe_hw *hw, u32 reg_addr, - u32 device_type, u16 *phy_data); + u32 device_type, u16 *phy_data); s32 ixgbe_write_phy_reg_x550a(struct ixgbe_hw *hw, u32 reg_addr, - u32 device_type, u16 phy_data); -s32 ixgbe_setup_fc_x550a(struct ixgbe_hw *hw); -void ixgbe_fc_autoneg_x550a(struct ixgbe_hw *hw); + u32 device_type, u16 phy_data); +s32 ixgbe_setup_fc_fiber_x550em_a(struct ixgbe_hw *hw); +s32 ixgbe_setup_fc_backplane_x550em_a(struct ixgbe_hw *hw); +s32 ixgbe_setup_fc_sgmii_x550em_a(struct ixgbe_hw *hw); +void ixgbe_fc_autoneg_fiber_x550em_a(struct ixgbe_hw *hw); +void ixgbe_fc_autoneg_backplane_x550em_a(struct ixgbe_hw *hw); +void ixgbe_fc_autoneg_sgmii_x550em_a(struct ixgbe_hw *hw); s32 ixgbe_handle_lasi_ext_t_x550em(struct ixgbe_hw *hw); s32 ixgbe_setup_mac_link_t_X550em(struct ixgbe_hw *hw, ixgbe_link_speed speed, diff --git a/dpdk/drivers/net/ixgbe/ixgbe_bypass.c b/dpdk/drivers/net/ixgbe/ixgbe_bypass.c index 70069284..38a44936 100644 --- a/dpdk/drivers/net/ixgbe/ixgbe_bypass.c +++ b/dpdk/drivers/net/ixgbe/ixgbe_bypass.c @@ -36,6 +36,7 @@ #include #include "ixgbe_ethdev.h" #include "ixgbe_bypass_api.h" +#include "rte_pmd_ixgbe.h" #define BYPASS_STATUS_OFF_MASK 3 @@ -284,7 +285,7 @@ ixgbe_bypass_wd_timeout_store(struct rte_eth_dev *dev, u32 timeout) FUNC_PTR_OR_ERR_RET(adapter->bps.ops.bypass_set, -ENOTSUP); /* disable the timer with timeout of zero */ - if (timeout == RTE_BYPASS_TMT_OFF) { + if (timeout == RTE_PMD_IXGBE_BYPASS_TMT_OFF) { status = 0x0; /* WDG enable off */ mask = BYPASS_WDT_ENABLE_M; } else { @@ -355,7 +356,7 @@ ixgbe_bypass_wd_timeout_show(struct rte_eth_dev *dev, u32 *wd_timeout) wdg = by_ctl & BYPASS_WDT_ENABLE_M; if (!wdg) - *wd_timeout = RTE_BYPASS_TMT_OFF; + *wd_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF; else *wd_timeout = (by_ctl >> BYPASS_WDT_TIME_SHIFT) & BYPASS_WDT_MASK; diff --git a/dpdk/drivers/net/ixgbe/ixgbe_bypass.h b/dpdk/drivers/net/ixgbe/ixgbe_bypass.h index 5f5c63e3..09155bb3 100644 --- a/dpdk/drivers/net/ixgbe/ixgbe_bypass.h +++ b/dpdk/drivers/net/ixgbe/ixgbe_bypass.h @@ -34,7 +34,7 @@ #ifndef _IXGBE_BYPASS_H_ #define _IXGBE_BYPASS_H_ -#ifdef RTE_NIC_BYPASS +#ifdef RTE_LIBRTE_IXGBE_BYPASS struct ixgbe_bypass_mac_ops { s32 (*bypass_rw)(struct ixgbe_hw *hw, u32 cmd, u32 *status); @@ -63,6 +63,6 @@ s32 ixgbe_bypass_wd_reset(struct rte_eth_dev *dev); s32 ixgbe_bypass_init_shared_code(struct ixgbe_hw *hw); s32 ixgbe_bypass_init_hw(struct ixgbe_hw *hw); -#endif /* RTE_NIC_BYPASS */ +#endif /* RTE_LIBRTE_IXGBE_BYPASS */ #endif /* _IXGBE_BYPASS_H_ */ diff --git a/dpdk/drivers/net/ixgbe/ixgbe_bypass_api.h b/dpdk/drivers/net/ixgbe/ixgbe_bypass_api.h index aec8f1ec..d52fde04 100644 --- a/dpdk/drivers/net/ixgbe/ixgbe_bypass_api.h +++ b/dpdk/drivers/net/ixgbe/ixgbe_bypass_api.h @@ -34,7 +34,7 @@ #ifndef _IXGBE_BYPASS_API_H_ #define _IXGBE_BYPASS_API_H_ -#ifdef RTE_NIC_BYPASS +#ifdef RTE_LIBRTE_IXGBE_BYPASS #include "ixgbe_bypass_defines.h" /** @@ -295,6 +295,6 @@ static s32 ixgbe_bypass_rd_eep_generic(struct ixgbe_hw *hw, u32 addr, u8 *value) return 0; } -#endif /* RTE_NIC_BYPASS */ +#endif /* RTE_LIBRTE_IXGBE_BYPASS */ #endif /* _IXGBE_BYPASS_API_H_ */ diff --git a/dpdk/drivers/net/ixgbe/ixgbe_bypass_defines.h b/dpdk/drivers/net/ixgbe/ixgbe_bypass_defines.h index cafcb278..d12c2714 100644 --- a/dpdk/drivers/net/ixgbe/ixgbe_bypass_defines.h +++ b/dpdk/drivers/net/ixgbe/ixgbe_bypass_defines.h @@ -34,7 +34,7 @@ #ifndef _IXGBE_BYPASS_DEFINES_H_ #define _IXGBE_BYPASS_DEFINES_H_ -#ifdef RTE_NIC_BYPASS +#ifdef RTE_LIBRTE_IXGBE_BYPASS #define msleep(x) rte_delay_us(x*1000) #define usleep_range(min, max) rte_delay_us(min) @@ -155,6 +155,6 @@ enum ixgbe_state_t { #define IXGBE_BYPASS_FW_WRITE_FAILURE -35 -#endif /* RTE_NIC_BYPASS */ +#endif /* RTE_LIBRTE_IXGBE_BYPASS */ #endif /* _IXGBE_BYPASS_DEFINES_H_ */ diff --git a/dpdk/drivers/net/ixgbe/ixgbe_ethdev.c b/dpdk/drivers/net/ixgbe/ixgbe_ethdev.c index e1029301..f2198660 100644 --- a/dpdk/drivers/net/ixgbe/ixgbe_ethdev.c +++ b/dpdk/drivers/net/ixgbe/ixgbe_ethdev.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. + * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -48,18 +48,22 @@ #include #include #include +#include #include #include #include -#include #include #include #include #include -#include +#include #include #include #include +#include +#ifdef RTE_LIBRTE_SECURITY +#include +#endif #include "ixgbe_logs.h" #include "base/ixgbe_api.h" @@ -91,6 +95,9 @@ /* Timer value included in XOFF frames. */ #define IXGBE_FC_PAUSE 0x680 +/*Default value of Max Rx Queue*/ +#define IXGBE_MAX_RX_QUEUE_NUM 128 + #define IXGBE_LINK_DOWN_CHECK_TIMEOUT 4000 /* ms */ #define IXGBE_LINK_UP_CHECK_TIMEOUT 1000 /* ms */ #define IXGBE_VMDQ_NUM_UC_MAC 4096 /* Maximum nb. of UC MAC addr. */ @@ -152,45 +159,56 @@ #define IXGBE_QDE_STRIP_TAG 0x00000004 #define IXGBE_VTEICR_MASK 0x07 -enum ixgbevf_xcast_modes { - IXGBEVF_XCAST_MODE_NONE = 0, - IXGBEVF_XCAST_MODE_MULTI, - IXGBEVF_XCAST_MODE_ALLMULTI, -}; - #define IXGBE_EXVET_VET_EXT_SHIFT 16 #define IXGBE_DMATXCTL_VT_MASK 0xFFFF0000 static int eth_ixgbe_dev_init(struct rte_eth_dev *eth_dev); static int eth_ixgbe_dev_uninit(struct rte_eth_dev *eth_dev); +static int ixgbe_fdir_filter_init(struct rte_eth_dev *eth_dev); +static int ixgbe_fdir_filter_uninit(struct rte_eth_dev *eth_dev); +static int ixgbe_l2_tn_filter_init(struct rte_eth_dev *eth_dev); +static int ixgbe_l2_tn_filter_uninit(struct rte_eth_dev *eth_dev); +static int ixgbe_ntuple_filter_uninit(struct rte_eth_dev *eth_dev); static int ixgbe_dev_configure(struct rte_eth_dev *dev); static int ixgbe_dev_start(struct rte_eth_dev *dev); static void ixgbe_dev_stop(struct rte_eth_dev *dev); static int ixgbe_dev_set_link_up(struct rte_eth_dev *dev); static int ixgbe_dev_set_link_down(struct rte_eth_dev *dev); static void ixgbe_dev_close(struct rte_eth_dev *dev); +static int ixgbe_dev_reset(struct rte_eth_dev *dev); static void ixgbe_dev_promiscuous_enable(struct rte_eth_dev *dev); static void ixgbe_dev_promiscuous_disable(struct rte_eth_dev *dev); static void ixgbe_dev_allmulticast_enable(struct rte_eth_dev *dev); static void ixgbe_dev_allmulticast_disable(struct rte_eth_dev *dev); static int ixgbe_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete); -static void ixgbe_dev_stats_get(struct rte_eth_dev *dev, +static int ixgbe_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats); static int ixgbe_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, unsigned n); static int ixgbevf_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, unsigned n); +static int +ixgbe_dev_xstats_get_by_id(struct rte_eth_dev *dev, const uint64_t *ids, + uint64_t *values, unsigned int n); static void ixgbe_dev_stats_reset(struct rte_eth_dev *dev); static void ixgbe_dev_xstats_reset(struct rte_eth_dev *dev); -static int ixgbe_dev_xstats_get_names(__rte_unused struct rte_eth_dev *dev, - struct rte_eth_xstat_name *xstats_names, __rte_unused unsigned limit); -static int ixgbevf_dev_xstats_get_names(__rte_unused struct rte_eth_dev *dev, - struct rte_eth_xstat_name *xstats_names, __rte_unused unsigned limit); +static int ixgbe_dev_xstats_get_names(struct rte_eth_dev *dev, + struct rte_eth_xstat_name *xstats_names, + unsigned int size); +static int ixgbevf_dev_xstats_get_names(struct rte_eth_dev *dev, + struct rte_eth_xstat_name *xstats_names, unsigned limit); +static int ixgbe_dev_xstats_get_names_by_id( + struct rte_eth_dev *dev, + struct rte_eth_xstat_name *xstats_names, + const uint64_t *ids, + unsigned int limit); static int ixgbe_dev_queue_stats_mapping_set(struct rte_eth_dev *eth_dev, uint16_t queue_id, uint8_t stat_idx, uint8_t is_rx); +static int ixgbe_fw_version_get(struct rte_eth_dev *dev, char *fw_version, + size_t fw_size); static void ixgbe_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info); static const uint32_t *ixgbe_dev_supported_ptypes_get(struct rte_eth_dev *dev); @@ -207,7 +225,7 @@ static void ixgbe_vlan_hw_strip_bitmap_set(struct rte_eth_dev *dev, uint16_t queue, bool on); static void ixgbe_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on); -static void ixgbe_vlan_offload_set(struct rte_eth_dev *dev, int mask); +static int ixgbe_vlan_offload_set(struct rte_eth_dev *dev, int mask); static void ixgbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue); static void ixgbe_vlan_hw_strip_disable(struct rte_eth_dev *dev, uint16_t queue); static void ixgbe_vlan_hw_extend_enable(struct rte_eth_dev *dev); @@ -228,37 +246,43 @@ static int ixgbe_dev_rss_reta_query(struct rte_eth_dev *dev, struct rte_eth_rss_reta_entry64 *reta_conf, uint16_t reta_size); static void ixgbe_dev_link_status_print(struct rte_eth_dev *dev); -static int ixgbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev); +static int ixgbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev, uint8_t on); +static int ixgbe_dev_macsec_interrupt_setup(struct rte_eth_dev *dev); static int ixgbe_dev_rxq_interrupt_setup(struct rte_eth_dev *dev); static int ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev); -static int ixgbe_dev_interrupt_action(struct rte_eth_dev *dev); -static void ixgbe_dev_interrupt_handler(struct rte_intr_handle *handle, - void *param); +static int ixgbe_dev_interrupt_action(struct rte_eth_dev *dev, + struct rte_intr_handle *handle); +static void ixgbe_dev_interrupt_handler(void *param); static void ixgbe_dev_interrupt_delayed_handler(void *param); -static void ixgbe_add_rar(struct rte_eth_dev *dev, struct ether_addr *mac_addr, - uint32_t index, uint32_t pool); +static int ixgbe_add_rar(struct rte_eth_dev *dev, struct ether_addr *mac_addr, + uint32_t index, uint32_t pool); static void ixgbe_remove_rar(struct rte_eth_dev *dev, uint32_t index); static void ixgbe_set_default_mac_addr(struct rte_eth_dev *dev, struct ether_addr *mac_addr); static void ixgbe_dcb_init(struct ixgbe_hw *hw, struct ixgbe_dcb_config *dcb_config); +static bool is_device_supported(struct rte_eth_dev *dev, + struct rte_pci_driver *drv); /* For Virtual Function support */ static int eth_ixgbevf_dev_init(struct rte_eth_dev *eth_dev); static int eth_ixgbevf_dev_uninit(struct rte_eth_dev *eth_dev); static int ixgbevf_dev_configure(struct rte_eth_dev *dev); static int ixgbevf_dev_start(struct rte_eth_dev *dev); +static int ixgbevf_dev_link_update(struct rte_eth_dev *dev, + int wait_to_complete); static void ixgbevf_dev_stop(struct rte_eth_dev *dev); static void ixgbevf_dev_close(struct rte_eth_dev *dev); +static int ixgbevf_dev_reset(struct rte_eth_dev *dev); static void ixgbevf_intr_disable(struct ixgbe_hw *hw); static void ixgbevf_intr_enable(struct ixgbe_hw *hw); -static void ixgbevf_dev_stats_get(struct rte_eth_dev *dev, +static int ixgbevf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats); static void ixgbevf_dev_stats_reset(struct rte_eth_dev *dev); static int ixgbevf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on); static void ixgbevf_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on); -static void ixgbevf_vlan_offload_set(struct rte_eth_dev *dev, int mask); +static int ixgbevf_vlan_offload_set(struct rte_eth_dev *dev, int mask); static void ixgbevf_set_vfta_all(struct rte_eth_dev *dev, bool on); static int ixgbevf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id); @@ -274,12 +298,6 @@ static void ixgbevf_dev_allmulticast_disable(struct rte_eth_dev *dev); static int ixgbe_uc_hash_table_set(struct rte_eth_dev *dev, struct ether_addr * mac_addr, uint8_t on); static int ixgbe_uc_all_hash_table_set(struct rte_eth_dev *dev, uint8_t on); -static int ixgbe_set_pool_rx_mode(struct rte_eth_dev *dev, uint16_t pool, - uint16_t rx_mask, uint8_t on); -static int ixgbe_set_pool_rx(struct rte_eth_dev *dev, uint16_t pool, uint8_t on); -static int ixgbe_set_pool_tx(struct rte_eth_dev *dev, uint16_t pool, uint8_t on); -static int ixgbe_set_pool_vlan_filter(struct rte_eth_dev *dev, uint16_t vlan, - uint64_t pool_mask, uint8_t vlan_on); static int ixgbe_mirror_rule_set(struct rte_eth_dev *dev, struct rte_eth_mirror_conf *mirror_conf, uint8_t rule_id, uint8_t on); @@ -293,20 +311,12 @@ static void ixgbe_set_ivar_map(struct ixgbe_hw *hw, int8_t direction, uint8_t queue, uint8_t msix_vector); static void ixgbe_configure_msix(struct rte_eth_dev *dev); -static int ixgbe_set_queue_rate_limit(struct rte_eth_dev *dev, - uint16_t queue_idx, uint16_t tx_rate); -static int ixgbe_set_vf_rate_limit(struct rte_eth_dev *dev, uint16_t vf, - uint16_t tx_rate, uint64_t q_msk); - -static void ixgbevf_add_mac_addr(struct rte_eth_dev *dev, - struct ether_addr *mac_addr, - uint32_t index, uint32_t pool); +static int ixgbevf_add_mac_addr(struct rte_eth_dev *dev, + struct ether_addr *mac_addr, + uint32_t index, uint32_t pool); static void ixgbevf_remove_mac_addr(struct rte_eth_dev *dev, uint32_t index); static void ixgbevf_set_default_mac_addr(struct rte_eth_dev *dev, struct ether_addr *mac_addr); -static int ixgbe_syn_filter_set(struct rte_eth_dev *dev, - struct rte_eth_syn_filter *filter, - bool add); static int ixgbe_syn_filter_get(struct rte_eth_dev *dev, struct rte_eth_syn_filter *filter); static int ixgbe_syn_filter_handle(struct rte_eth_dev *dev, @@ -316,17 +326,11 @@ static int ixgbe_add_5tuple_filter(struct rte_eth_dev *dev, struct ixgbe_5tuple_filter *filter); static void ixgbe_remove_5tuple_filter(struct rte_eth_dev *dev, struct ixgbe_5tuple_filter *filter); -static int ixgbe_add_del_ntuple_filter(struct rte_eth_dev *dev, - struct rte_eth_ntuple_filter *filter, - bool add); static int ixgbe_ntuple_filter_handle(struct rte_eth_dev *dev, enum rte_filter_op filter_op, void *arg); static int ixgbe_get_ntuple_filter(struct rte_eth_dev *dev, struct rte_eth_ntuple_filter *filter); -static int ixgbe_add_del_ethertype_filter(struct rte_eth_dev *dev, - struct rte_eth_ethertype_filter *filter, - bool add); static int ixgbe_ethertype_filter_handle(struct rte_eth_dev *dev, enum rte_filter_op filter_op, void *arg); @@ -369,8 +373,7 @@ static int ixgbe_timesync_read_time(struct rte_eth_dev *dev, struct timespec *timestamp); static int ixgbe_timesync_write_time(struct rte_eth_dev *dev, const struct timespec *timestamp); -static void ixgbevf_dev_interrupt_handler(struct rte_intr_handle *handle, - void *param); +static void ixgbevf_dev_interrupt_handler(void *param); static int ixgbe_dev_l2_tunnel_eth_type_conf (struct rte_eth_dev *dev, struct rte_eth_l2_tunnel_conf *l2_tunnel); @@ -387,6 +390,8 @@ static int ixgbe_dev_udp_tunnel_port_add(struct rte_eth_dev *dev, struct rte_eth_udp_tunnel *udp_tunnel); static int ixgbe_dev_udp_tunnel_port_del(struct rte_eth_dev *dev, struct rte_eth_udp_tunnel *udp_tunnel); +static int ixgbe_filter_restore(struct rte_eth_dev *dev); +static void ixgbe_l2_tunnel_conf(struct rte_eth_dev *dev); /* * Define VF Stats MACRO for Non "cleared on read" register @@ -429,23 +434,75 @@ static int ixgbe_dev_udp_tunnel_port_del(struct rte_eth_dev *dev, * The set of PCI devices this driver supports */ static const struct rte_pci_id pci_id_ixgbe_map[] = { - -#define RTE_PCI_DEV_ID_DECL_IXGBE(vend, dev) {RTE_PCI_DEVICE(vend, dev)}, -#include "rte_pci_dev_ids.h" - -{ .vendor_id = 0, /* sentinel */ }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598) }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_BX) }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT) }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT) }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT) }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2) }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM) }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4) }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT) }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT) }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM) }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR) }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4) }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ) }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KR) }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE) }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4) }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP) }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE) }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE) }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_EM) }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF2) }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF_QP) }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_QSFP_SF_QP) }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP) }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM) }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM) }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_LS) }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T) }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T1) }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_SFP) }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_10G_T) }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_1G_T) }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550T) }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550T1) }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_KR) }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_KR_L) }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SFP_N) }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SGMII) }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SGMII_L) }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_10G_T) }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_QSFP) }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_QSFP_N) }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SFP) }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_1G_T) }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_1G_T_L) }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_KX4) }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_KR) }, +#ifdef RTE_LIBRTE_IXGBE_BYPASS + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BYPASS) }, +#endif + { .vendor_id = 0, /* sentinel */ }, }; - /* * The set of PCI devices this driver supports (for 82599 VF) */ static const struct rte_pci_id pci_id_ixgbevf_map[] = { - -#define RTE_PCI_DEV_ID_DECL_IXGBEVF(vend, dev) {RTE_PCI_DEVICE(vend, dev)}, -#include "rte_pci_dev_ids.h" -{ .vendor_id = 0, /* sentinel */ }, - + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_VF) }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_VF_HV) }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_VF) }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_VF_HV) }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550_VF_HV) }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550_VF) }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_VF) }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_VF_HV) }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_VF) }, + { RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_VF_HV) }, + { .vendor_id = 0, /* sentinel */ }, }; static const struct rte_eth_desc_lim rx_desc_lim = { @@ -458,6 +515,8 @@ static const struct rte_eth_desc_lim tx_desc_lim = { .nb_max = IXGBE_MAX_RING_DESC, .nb_min = IXGBE_MIN_RING_DESC, .nb_align = IXGBE_TXD_ALIGN, + .nb_seg_max = IXGBE_TX_MAX_SEG, + .nb_mtu_seg_max = IXGBE_TX_MAX_SEG, }; static const struct eth_dev_ops ixgbe_eth_dev_ops = { @@ -467,6 +526,7 @@ static const struct eth_dev_ops ixgbe_eth_dev_ops = { .dev_set_link_up = ixgbe_dev_set_link_up, .dev_set_link_down = ixgbe_dev_set_link_down, .dev_close = ixgbe_dev_close, + .dev_reset = ixgbe_dev_reset, .promiscuous_enable = ixgbe_dev_promiscuous_enable, .promiscuous_disable = ixgbe_dev_promiscuous_disable, .allmulticast_enable = ixgbe_dev_allmulticast_enable, @@ -474,10 +534,13 @@ static const struct eth_dev_ops ixgbe_eth_dev_ops = { .link_update = ixgbe_dev_link_update, .stats_get = ixgbe_dev_stats_get, .xstats_get = ixgbe_dev_xstats_get, + .xstats_get_by_id = ixgbe_dev_xstats_get_by_id, .stats_reset = ixgbe_dev_stats_reset, .xstats_reset = ixgbe_dev_xstats_reset, .xstats_get_names = ixgbe_dev_xstats_get_names, + .xstats_get_names_by_id = ixgbe_dev_xstats_get_names_by_id, .queue_stats_mapping_set = ixgbe_dev_queue_stats_mapping_set, + .fw_version_get = ixgbe_fw_version_get, .dev_infos_get = ixgbe_dev_info_get, .dev_supported_ptypes_get = ixgbe_dev_supported_ptypes_get, .mtu_set = ixgbe_dev_mtu_set, @@ -495,6 +558,8 @@ static const struct eth_dev_ops ixgbe_eth_dev_ops = { .rx_queue_release = ixgbe_dev_rx_queue_release, .rx_queue_count = ixgbe_dev_rx_queue_count, .rx_descriptor_done = ixgbe_dev_rx_descriptor_done, + .rx_descriptor_status = ixgbe_dev_rx_descriptor_status, + .tx_descriptor_status = ixgbe_dev_tx_descriptor_status, .tx_queue_setup = ixgbe_dev_tx_queue_setup, .tx_queue_release = ixgbe_dev_tx_queue_release, .dev_led_on = ixgbe_dev_led_on, @@ -509,25 +574,9 @@ static const struct eth_dev_ops ixgbe_eth_dev_ops = { .uc_all_hash_table_set = ixgbe_uc_all_hash_table_set, .mirror_rule_set = ixgbe_mirror_rule_set, .mirror_rule_reset = ixgbe_mirror_rule_reset, - .set_vf_rx_mode = ixgbe_set_pool_rx_mode, - .set_vf_rx = ixgbe_set_pool_rx, - .set_vf_tx = ixgbe_set_pool_tx, - .set_vf_vlan_filter = ixgbe_set_pool_vlan_filter, .set_queue_rate_limit = ixgbe_set_queue_rate_limit, - .set_vf_rate_limit = ixgbe_set_vf_rate_limit, .reta_update = ixgbe_dev_rss_reta_update, .reta_query = ixgbe_dev_rss_reta_query, -#ifdef RTE_NIC_BYPASS - .bypass_init = ixgbe_bypass_init, - .bypass_state_set = ixgbe_bypass_state_store, - .bypass_state_show = ixgbe_bypass_state_show, - .bypass_event_set = ixgbe_bypass_event_store, - .bypass_event_show = ixgbe_bypass_event_show, - .bypass_wd_timeout_set = ixgbe_bypass_wd_timeout_store, - .bypass_wd_timeout_show = ixgbe_bypass_wd_timeout_show, - .bypass_ver_show = ixgbe_bypass_ver_show, - .bypass_wd_reset = ixgbe_bypass_wd_reset, -#endif /* RTE_NIC_BYPASS */ .rss_hash_update = ixgbe_dev_rss_hash_update, .rss_hash_conf_get = ixgbe_dev_rss_hash_conf_get, .filter_ctrl = ixgbe_dev_filter_ctrl, @@ -550,6 +599,7 @@ static const struct eth_dev_ops ixgbe_eth_dev_ops = { .l2_tunnel_offload_set = ixgbe_dev_l2_tunnel_offload_set, .udp_tunnel_port_add = ixgbe_dev_udp_tunnel_port_add, .udp_tunnel_port_del = ixgbe_dev_udp_tunnel_port_del, + .tm_ops_get = ixgbe_tm_ops_get, }; /* @@ -560,13 +610,14 @@ static const struct eth_dev_ops ixgbevf_eth_dev_ops = { .dev_configure = ixgbevf_dev_configure, .dev_start = ixgbevf_dev_start, .dev_stop = ixgbevf_dev_stop, - .link_update = ixgbe_dev_link_update, + .link_update = ixgbevf_dev_link_update, .stats_get = ixgbevf_dev_stats_get, .xstats_get = ixgbevf_dev_xstats_get, .stats_reset = ixgbevf_dev_stats_reset, .xstats_reset = ixgbevf_dev_stats_reset, .xstats_get_names = ixgbevf_dev_xstats_get_names, .dev_close = ixgbevf_dev_close, + .dev_reset = ixgbevf_dev_reset, .allmulticast_enable = ixgbevf_dev_allmulticast_enable, .allmulticast_disable = ixgbevf_dev_allmulticast_disable, .dev_infos_get = ixgbevf_dev_info_get, @@ -578,6 +629,8 @@ static const struct eth_dev_ops ixgbevf_eth_dev_ops = { .rx_queue_setup = ixgbe_dev_rx_queue_setup, .rx_queue_release = ixgbe_dev_rx_queue_release, .rx_descriptor_done = ixgbe_dev_rx_descriptor_done, + .rx_descriptor_status = ixgbe_dev_rx_descriptor_status, + .tx_descriptor_status = ixgbe_dev_tx_descriptor_status, .tx_queue_setup = ixgbe_dev_tx_queue_setup, .tx_queue_release = ixgbe_dev_tx_queue_release, .rx_queue_intr_enable = ixgbevf_dev_rx_queue_intr_enable, @@ -685,6 +738,51 @@ static const struct rte_ixgbe_xstats_name_off rte_ixgbe_stats_strings[] = { #define IXGBE_NB_HW_STATS (sizeof(rte_ixgbe_stats_strings) / \ sizeof(rte_ixgbe_stats_strings[0])) +/* MACsec statistics */ +static const struct rte_ixgbe_xstats_name_off rte_ixgbe_macsec_strings[] = { + {"out_pkts_untagged", offsetof(struct ixgbe_macsec_stats, + out_pkts_untagged)}, + {"out_pkts_encrypted", offsetof(struct ixgbe_macsec_stats, + out_pkts_encrypted)}, + {"out_pkts_protected", offsetof(struct ixgbe_macsec_stats, + out_pkts_protected)}, + {"out_octets_encrypted", offsetof(struct ixgbe_macsec_stats, + out_octets_encrypted)}, + {"out_octets_protected", offsetof(struct ixgbe_macsec_stats, + out_octets_protected)}, + {"in_pkts_untagged", offsetof(struct ixgbe_macsec_stats, + in_pkts_untagged)}, + {"in_pkts_badtag", offsetof(struct ixgbe_macsec_stats, + in_pkts_badtag)}, + {"in_pkts_nosci", offsetof(struct ixgbe_macsec_stats, + in_pkts_nosci)}, + {"in_pkts_unknownsci", offsetof(struct ixgbe_macsec_stats, + in_pkts_unknownsci)}, + {"in_octets_decrypted", offsetof(struct ixgbe_macsec_stats, + in_octets_decrypted)}, + {"in_octets_validated", offsetof(struct ixgbe_macsec_stats, + in_octets_validated)}, + {"in_pkts_unchecked", offsetof(struct ixgbe_macsec_stats, + in_pkts_unchecked)}, + {"in_pkts_delayed", offsetof(struct ixgbe_macsec_stats, + in_pkts_delayed)}, + {"in_pkts_late", offsetof(struct ixgbe_macsec_stats, + in_pkts_late)}, + {"in_pkts_ok", offsetof(struct ixgbe_macsec_stats, + in_pkts_ok)}, + {"in_pkts_invalid", offsetof(struct ixgbe_macsec_stats, + in_pkts_invalid)}, + {"in_pkts_notvalid", offsetof(struct ixgbe_macsec_stats, + in_pkts_notvalid)}, + {"in_pkts_unusedsa", offsetof(struct ixgbe_macsec_stats, + in_pkts_unusedsa)}, + {"in_pkts_notusingsa", offsetof(struct ixgbe_macsec_stats, + in_pkts_notusingsa)}, +}; + +#define IXGBE_NB_MACSEC_STATS (sizeof(rte_ixgbe_macsec_strings) / \ + sizeof(rte_ixgbe_macsec_strings[0])) + /* Per-queue statistics */ static const struct rte_ixgbe_xstats_name_off rte_ixgbe_rxq_strings[] = { {"mbuf_allocation_errors", offsetof(struct ixgbe_hw_stats, rnbc)}, @@ -800,6 +898,8 @@ ixgbe_pf_reset_hw(struct ixgbe_hw *hw) IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); IXGBE_WRITE_FLUSH(hw); + if (status == IXGBE_ERR_SFP_NOT_PRESENT) + status = IXGBE_SUCCESS; return status; } @@ -1024,7 +1124,8 @@ ixgbe_swfw_lock_reset(struct ixgbe_hw *hw) static int eth_ixgbe_dev_init(struct rte_eth_dev *eth_dev) { - struct rte_pci_device *pci_dev; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); + struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); struct ixgbe_vfta *shadow_vfta = @@ -1035,6 +1136,8 @@ eth_ixgbe_dev_init(struct rte_eth_dev *eth_dev) IXGBE_DEV_PRIVATE_TO_DCB_CFG(eth_dev->data->dev_private); struct ixgbe_filter_info *filter_info = IXGBE_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private); + struct ixgbe_bw_conf *bw_conf = + IXGBE_DEV_PRIVATE_TO_BW_CONF(eth_dev->data->dev_private); uint32_t ctrl_ext; uint16_t csum; int diag, i; @@ -1044,6 +1147,7 @@ eth_ixgbe_dev_init(struct rte_eth_dev *eth_dev) eth_dev->dev_ops = &ixgbe_eth_dev_ops; eth_dev->rx_pkt_burst = &ixgbe_recv_pkts; eth_dev->tx_pkt_burst = &ixgbe_xmit_pkts; + eth_dev->tx_pkt_prepare = &ixgbe_prep_pkts; /* * For secondary processes, we don't initialise any further as primary @@ -1068,7 +1172,13 @@ eth_ixgbe_dev_init(struct rte_eth_dev *eth_dev) return 0; } - pci_dev = eth_dev->pci_dev; + +#ifdef RTE_LIBRTE_SECURITY + /* Initialize security_ctx only for primary process*/ + eth_dev->security_ctx = ixgbe_ipsec_ctx_create(eth_dev); + if (eth_dev->security_ctx == NULL) + return -ENOMEM; +#endif rte_eth_copy_pci_info(eth_dev, pci_dev); @@ -1079,11 +1189,11 @@ eth_ixgbe_dev_init(struct rte_eth_dev *eth_dev) hw->allow_unsupported_sfp = 1; /* Initialize the shared code (base driver) */ -#ifdef RTE_NIC_BYPASS +#ifdef RTE_LIBRTE_IXGBE_BYPASS diag = ixgbe_bypass_init_shared_code(hw); #else diag = ixgbe_init_shared_code(hw); -#endif /* RTE_NIC_BYPASS */ +#endif /* RTE_LIBRTE_IXGBE_BYPASS */ if (diag != IXGBE_SUCCESS) { PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag); @@ -1116,11 +1226,11 @@ eth_ixgbe_dev_init(struct rte_eth_dev *eth_dev) return -EIO; } -#ifdef RTE_NIC_BYPASS +#ifdef RTE_LIBRTE_IXGBE_BYPASS diag = ixgbe_bypass_init_hw(hw); #else diag = ixgbe_init_hw(hw); -#endif /* RTE_NIC_BYPASS */ +#endif /* RTE_LIBRTE_IXGBE_BYPASS */ /* * Devices with copper phys will fail to initialise if ixgbe_init_hw() @@ -1137,6 +1247,9 @@ eth_ixgbe_dev_init(struct rte_eth_dev *eth_dev) diag = ixgbe_init_hw(hw); } + if (diag == IXGBE_ERR_SFP_NOT_PRESENT) + diag = IXGBE_SUCCESS; + if (diag == IXGBE_ERR_EEPROM_VERSION) { PMD_INIT_LOG(ERR, "This device is a pre-production adapter/" "LOM. Please be aware there may be issues associated " @@ -1213,20 +1326,36 @@ eth_ixgbe_dev_init(struct rte_eth_dev *eth_dev) eth_dev->data->port_id, pci_dev->id.vendor_id, pci_dev->id.device_id); - rte_intr_callback_register(&pci_dev->intr_handle, - ixgbe_dev_interrupt_handler, - (void *)eth_dev); + rte_intr_callback_register(intr_handle, + ixgbe_dev_interrupt_handler, eth_dev); /* enable uio/vfio intr/eventfd mapping */ - rte_intr_enable(&pci_dev->intr_handle); + rte_intr_enable(intr_handle); /* enable support intr */ ixgbe_enable_intr(eth_dev); + /* initialize filter info */ + memset(filter_info, 0, + sizeof(struct ixgbe_filter_info)); + /* initialize 5tuple filter list */ TAILQ_INIT(&filter_info->fivetuple_list); - memset(filter_info->fivetuple_mask, 0, - sizeof(uint32_t) * IXGBE_5TUPLE_ARRAY_SIZE); + + /* initialize flow director filter list & hash */ + ixgbe_fdir_filter_init(eth_dev); + + /* initialize l2 tunnel filter list & hash */ + ixgbe_l2_tn_filter_init(eth_dev); + + /* initialize flow filter lists */ + ixgbe_filterlist_init(); + + /* initialize bandwidth configuration info */ + memset(bw_conf, 0, sizeof(struct ixgbe_bw_conf)); + + /* initialize Traffic Manager configuration */ + ixgbe_tm_conf_init(eth_dev); return 0; } @@ -1234,7 +1363,8 @@ eth_ixgbe_dev_init(struct rte_eth_dev *eth_dev) static int eth_ixgbe_dev_uninit(struct rte_eth_dev *eth_dev) { - struct rte_pci_device *pci_dev; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); + struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; struct ixgbe_hw *hw; PMD_INIT_FUNC_TRACE(); @@ -1243,7 +1373,6 @@ eth_ixgbe_dev_uninit(struct rte_eth_dev *eth_dev) return -EPERM; hw = IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); - pci_dev = eth_dev->pci_dev; if (hw->adapter_stopped == 0) ixgbe_dev_close(eth_dev); @@ -1256,9 +1385,9 @@ eth_ixgbe_dev_uninit(struct rte_eth_dev *eth_dev) ixgbe_swfw_lock_reset(hw); /* disable uio intr before callback unregister */ - rte_intr_disable(&(pci_dev->intr_handle)); - rte_intr_callback_unregister(&(pci_dev->intr_handle), - ixgbe_dev_interrupt_handler, (void *)eth_dev); + rte_intr_disable(intr_handle); + rte_intr_callback_unregister(intr_handle, + ixgbe_dev_interrupt_handler, eth_dev); /* uninitialize PF if max_vfs not zero */ ixgbe_pf_host_uninit(eth_dev); @@ -1269,9 +1398,161 @@ eth_ixgbe_dev_uninit(struct rte_eth_dev *eth_dev) rte_free(eth_dev->data->hash_mac_addrs); eth_dev->data->hash_mac_addrs = NULL; + /* remove all the fdir filters & hash */ + ixgbe_fdir_filter_uninit(eth_dev); + + /* remove all the L2 tunnel filters & hash */ + ixgbe_l2_tn_filter_uninit(eth_dev); + + /* Remove all ntuple filters of the device */ + ixgbe_ntuple_filter_uninit(eth_dev); + + /* clear all the filters list */ + ixgbe_filterlist_flush(); + + /* Remove all Traffic Manager configuration */ + ixgbe_tm_conf_uninit(eth_dev); + +#ifdef RTE_LIBRTE_SECURITY + rte_free(eth_dev->security_ctx); +#endif + return 0; } +static int ixgbe_ntuple_filter_uninit(struct rte_eth_dev *eth_dev) +{ + struct ixgbe_filter_info *filter_info = + IXGBE_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private); + struct ixgbe_5tuple_filter *p_5tuple; + + while ((p_5tuple = TAILQ_FIRST(&filter_info->fivetuple_list))) { + TAILQ_REMOVE(&filter_info->fivetuple_list, + p_5tuple, + entries); + rte_free(p_5tuple); + } + memset(filter_info->fivetuple_mask, 0, + sizeof(uint32_t) * IXGBE_5TUPLE_ARRAY_SIZE); + + return 0; +} + +static int ixgbe_fdir_filter_uninit(struct rte_eth_dev *eth_dev) +{ + struct ixgbe_hw_fdir_info *fdir_info = + IXGBE_DEV_PRIVATE_TO_FDIR_INFO(eth_dev->data->dev_private); + struct ixgbe_fdir_filter *fdir_filter; + + if (fdir_info->hash_map) + rte_free(fdir_info->hash_map); + if (fdir_info->hash_handle) + rte_hash_free(fdir_info->hash_handle); + + while ((fdir_filter = TAILQ_FIRST(&fdir_info->fdir_list))) { + TAILQ_REMOVE(&fdir_info->fdir_list, + fdir_filter, + entries); + rte_free(fdir_filter); + } + + return 0; +} + +static int ixgbe_l2_tn_filter_uninit(struct rte_eth_dev *eth_dev) +{ + struct ixgbe_l2_tn_info *l2_tn_info = + IXGBE_DEV_PRIVATE_TO_L2_TN_INFO(eth_dev->data->dev_private); + struct ixgbe_l2_tn_filter *l2_tn_filter; + + if (l2_tn_info->hash_map) + rte_free(l2_tn_info->hash_map); + if (l2_tn_info->hash_handle) + rte_hash_free(l2_tn_info->hash_handle); + + while ((l2_tn_filter = TAILQ_FIRST(&l2_tn_info->l2_tn_list))) { + TAILQ_REMOVE(&l2_tn_info->l2_tn_list, + l2_tn_filter, + entries); + rte_free(l2_tn_filter); + } + + return 0; +} + +static int ixgbe_fdir_filter_init(struct rte_eth_dev *eth_dev) +{ + struct ixgbe_hw_fdir_info *fdir_info = + IXGBE_DEV_PRIVATE_TO_FDIR_INFO(eth_dev->data->dev_private); + char fdir_hash_name[RTE_HASH_NAMESIZE]; + struct rte_hash_parameters fdir_hash_params = { + .name = fdir_hash_name, + .entries = IXGBE_MAX_FDIR_FILTER_NUM, + .key_len = sizeof(union ixgbe_atr_input), + .hash_func = rte_hash_crc, + .hash_func_init_val = 0, + .socket_id = rte_socket_id(), + }; + + TAILQ_INIT(&fdir_info->fdir_list); + snprintf(fdir_hash_name, RTE_HASH_NAMESIZE, + "fdir_%s", eth_dev->device->name); + fdir_info->hash_handle = rte_hash_create(&fdir_hash_params); + if (!fdir_info->hash_handle) { + PMD_INIT_LOG(ERR, "Failed to create fdir hash table!"); + return -EINVAL; + } + fdir_info->hash_map = rte_zmalloc("ixgbe", + sizeof(struct ixgbe_fdir_filter *) * + IXGBE_MAX_FDIR_FILTER_NUM, + 0); + if (!fdir_info->hash_map) { + PMD_INIT_LOG(ERR, + "Failed to allocate memory for fdir hash map!"); + return -ENOMEM; + } + fdir_info->mask_added = FALSE; + + return 0; +} + +static int ixgbe_l2_tn_filter_init(struct rte_eth_dev *eth_dev) +{ + struct ixgbe_l2_tn_info *l2_tn_info = + IXGBE_DEV_PRIVATE_TO_L2_TN_INFO(eth_dev->data->dev_private); + char l2_tn_hash_name[RTE_HASH_NAMESIZE]; + struct rte_hash_parameters l2_tn_hash_params = { + .name = l2_tn_hash_name, + .entries = IXGBE_MAX_L2_TN_FILTER_NUM, + .key_len = sizeof(struct ixgbe_l2_tn_key), + .hash_func = rte_hash_crc, + .hash_func_init_val = 0, + .socket_id = rte_socket_id(), + }; + + TAILQ_INIT(&l2_tn_info->l2_tn_list); + snprintf(l2_tn_hash_name, RTE_HASH_NAMESIZE, + "l2_tn_%s", eth_dev->device->name); + l2_tn_info->hash_handle = rte_hash_create(&l2_tn_hash_params); + if (!l2_tn_info->hash_handle) { + PMD_INIT_LOG(ERR, "Failed to create L2 TN hash table!"); + return -EINVAL; + } + l2_tn_info->hash_map = rte_zmalloc("ixgbe", + sizeof(struct ixgbe_l2_tn_filter *) * + IXGBE_MAX_L2_TN_FILTER_NUM, + 0); + if (!l2_tn_info->hash_map) { + PMD_INIT_LOG(ERR, + "Failed to allocate memory for L2 TN hash map!"); + return -ENOMEM; + } + l2_tn_info->e_tag_en = FALSE; + l2_tn_info->e_tag_fwd_en = FALSE; + l2_tn_info->e_tag_ether_type = DEFAULT_ETAG_ETYPE; + + return 0; +} /* * Negotiate mailbox API version with the PF. * After reset API version is always set to the basic one (ixgbe_mbox_api_10). @@ -1322,7 +1603,8 @@ eth_ixgbevf_dev_init(struct rte_eth_dev *eth_dev) { int diag; uint32_t tc, tcs; - struct rte_pci_device *pci_dev; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); + struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); struct ixgbe_vfta *shadow_vfta = @@ -1360,8 +1642,6 @@ eth_ixgbevf_dev_init(struct rte_eth_dev *eth_dev) return 0; } - pci_dev = eth_dev->pci_dev; - rte_eth_copy_pci_info(eth_dev, pci_dev); hw->device_id = pci_dev->id.device_id; @@ -1454,10 +1734,9 @@ eth_ixgbevf_dev_init(struct rte_eth_dev *eth_dev) return -EIO; } - rte_intr_callback_register(&pci_dev->intr_handle, - ixgbevf_dev_interrupt_handler, - (void *)eth_dev); - rte_intr_enable(&pci_dev->intr_handle); + rte_intr_callback_register(intr_handle, + ixgbevf_dev_interrupt_handler, eth_dev); + rte_intr_enable(intr_handle); ixgbevf_intr_enable(hw); PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x mac.type=%s", @@ -1472,8 +1751,9 @@ eth_ixgbevf_dev_init(struct rte_eth_dev *eth_dev) static int eth_ixgbevf_dev_uninit(struct rte_eth_dev *eth_dev) { + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); + struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; struct ixgbe_hw *hw; - struct rte_pci_device *pci_dev = eth_dev->pci_dev; PMD_INIT_FUNC_TRACE(); @@ -1495,68 +1775,55 @@ eth_ixgbevf_dev_uninit(struct rte_eth_dev *eth_dev) rte_free(eth_dev->data->mac_addrs); eth_dev->data->mac_addrs = NULL; - rte_intr_disable(&pci_dev->intr_handle); - rte_intr_callback_unregister(&pci_dev->intr_handle, - ixgbevf_dev_interrupt_handler, - (void *)eth_dev); + rte_intr_disable(intr_handle); + rte_intr_callback_unregister(intr_handle, + ixgbevf_dev_interrupt_handler, eth_dev); return 0; } -static struct eth_driver rte_ixgbe_pmd = { - .pci_drv = { - .name = "rte_ixgbe_pmd", - .id_table = pci_id_ixgbe_map, - .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC | - RTE_PCI_DRV_DETACHABLE, - }, - .eth_dev_init = eth_ixgbe_dev_init, - .eth_dev_uninit = eth_ixgbe_dev_uninit, - .dev_private_size = sizeof(struct ixgbe_adapter), +static int eth_ixgbe_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, + struct rte_pci_device *pci_dev) +{ + return rte_eth_dev_pci_generic_probe(pci_dev, + sizeof(struct ixgbe_adapter), eth_ixgbe_dev_init); +} + +static int eth_ixgbe_pci_remove(struct rte_pci_device *pci_dev) +{ + return rte_eth_dev_pci_generic_remove(pci_dev, eth_ixgbe_dev_uninit); +} + +static struct rte_pci_driver rte_ixgbe_pmd = { + .id_table = pci_id_ixgbe_map, + .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC | + RTE_PCI_DRV_IOVA_AS_VA, + .probe = eth_ixgbe_pci_probe, + .remove = eth_ixgbe_pci_remove, }; +static int eth_ixgbevf_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, + struct rte_pci_device *pci_dev) +{ + return rte_eth_dev_pci_generic_probe(pci_dev, + sizeof(struct ixgbe_adapter), eth_ixgbevf_dev_init); +} + +static int eth_ixgbevf_pci_remove(struct rte_pci_device *pci_dev) +{ + return rte_eth_dev_pci_generic_remove(pci_dev, eth_ixgbevf_dev_uninit); +} + /* * virtual function driver struct */ -static struct eth_driver rte_ixgbevf_pmd = { - .pci_drv = { - .name = "rte_ixgbevf_pmd", - .id_table = pci_id_ixgbevf_map, - .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_DETACHABLE, - }, - .eth_dev_init = eth_ixgbevf_dev_init, - .eth_dev_uninit = eth_ixgbevf_dev_uninit, - .dev_private_size = sizeof(struct ixgbe_adapter), +static struct rte_pci_driver rte_ixgbevf_pmd = { + .id_table = pci_id_ixgbevf_map, + .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_IOVA_AS_VA, + .probe = eth_ixgbevf_pci_probe, + .remove = eth_ixgbevf_pci_remove, }; -/* - * Driver initialization routine. - * Invoked once at EAL init time. - * Register itself as the [Poll Mode] Driver of PCI IXGBE devices. - */ -static int -rte_ixgbe_pmd_init(const char *name __rte_unused, const char *params __rte_unused) -{ - PMD_INIT_FUNC_TRACE(); - - rte_eth_driver_register(&rte_ixgbe_pmd); - return 0; -} - -/* - * VF Driver initialization routine. - * Invoked one at EAL init time. - * Register itself as the [Virtual Poll Mode] Driver of PCI niantic devices. - */ -static int -rte_ixgbevf_pmd_init(const char *name __rte_unused, const char *param __rte_unused) -{ - PMD_INIT_FUNC_TRACE(); - - rte_eth_driver_register(&rte_ixgbevf_pmd); - return 0; -} - static int ixgbe_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) { @@ -1708,9 +1975,9 @@ ixgbe_vlan_hw_strip_bitmap_set(struct rte_eth_dev *dev, uint16_t queue, bool on) rxq = dev->data->rx_queues[queue]; if (on) - rxq->vlan_flags = PKT_RX_VLAN_PKT | PKT_RX_VLAN_STRIPPED; + rxq->vlan_flags = PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED; else - rxq->vlan_flags = PKT_RX_VLAN_PKT; + rxq->vlan_flags = PKT_RX_VLAN; } static void @@ -1874,7 +2141,7 @@ ixgbe_vlan_hw_extend_enable(struct rte_eth_dev *dev) */ } -static void +static int ixgbe_vlan_offload_set(struct rte_eth_dev *dev, int mask) { if (mask & ETH_VLAN_STRIP_MASK) { @@ -1897,6 +2164,8 @@ ixgbe_vlan_offload_set(struct rte_eth_dev *dev, int mask) else ixgbe_vlan_hw_extend_disable(dev); } + + return 0; } static void @@ -1914,6 +2183,8 @@ ixgbe_vmdq_vlan_hw_filter_enable(struct rte_eth_dev *dev) static int ixgbe_check_vf_rss_rxq_num(struct rte_eth_dev *dev, uint16_t nb_rx_q) { + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + switch (nb_rx_q) { case 1: case 2: @@ -1926,9 +2197,10 @@ ixgbe_check_vf_rss_rxq_num(struct rte_eth_dev *dev, uint16_t nb_rx_q) return -EINVAL; } - RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool = nb_rx_q; - RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx = dev->pci_dev->max_vfs * nb_rx_q; - + RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool = + IXGBE_MAX_RX_QUEUE_NUM / RTE_ETH_DEV_SRIOV(dev).active; + RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx = + pci_dev->max_vfs * RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool; return 0; } @@ -1944,6 +2216,8 @@ ixgbe_check_mq_mode(struct rte_eth_dev *dev) /* check multi-queue mode */ switch (dev_conf->rxmode.mq_mode) { case ETH_MQ_RX_VMDQ_DCB: + PMD_INIT_LOG(INFO, "ETH_MQ_RX_VMDQ_DCB mode supported in SRIOV"); + break; case ETH_MQ_RX_VMDQ_DCB_RSS: /* DCB/RSS VMDQ in SRIOV mode, not implement yet */ PMD_INIT_LOG(ERR, "SRIOV active," @@ -1966,8 +2240,6 @@ ixgbe_check_mq_mode(struct rte_eth_dev *dev) case ETH_MQ_RX_NONE: /* if nothing mq mode configure, use default scheme */ dev->data->dev_conf.rxmode.mq_mode = ETH_MQ_RX_VMDQ_ONLY; - if (RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool > 1) - RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool = 1; break; default: /* ETH_MQ_RX_DCB, ETH_MQ_RX_DCB_RSS or ETH_MQ_TX_DCB*/ /* SRIOV only works in VMDq enable mode */ @@ -1979,11 +2251,9 @@ ixgbe_check_mq_mode(struct rte_eth_dev *dev) switch (dev_conf->txmode.mq_mode) { case ETH_MQ_TX_VMDQ_DCB: - /* DCB VMDQ in SRIOV mode, not implement yet */ - PMD_INIT_LOG(ERR, "SRIOV is active," - " unsupported VMDQ mq_mode tx %d.", - dev_conf->txmode.mq_mode); - return -EINVAL; + PMD_INIT_LOG(INFO, "ETH_MQ_TX_VMDQ_DCB mode supported in SRIOV"); + dev->data->dev_conf.txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB; + break; default: /* ETH_MQ_TX_VMDQ_ONLY or ETH_MQ_TX_NONE */ dev->data->dev_conf.txmode.mq_mode = ETH_MQ_TX_VMDQ_ONLY; break; @@ -2147,6 +2417,80 @@ ixgbe_dev_phy_intr_setup(struct rte_eth_dev *dev) } } +int +ixgbe_set_vf_rate_limit(struct rte_eth_dev *dev, uint16_t vf, + uint16_t tx_rate, uint64_t q_msk) +{ + struct ixgbe_hw *hw; + struct ixgbe_vf_info *vfinfo; + struct rte_eth_link link; + uint8_t nb_q_per_pool; + uint32_t queue_stride; + uint32_t queue_idx, idx = 0, vf_idx; + uint32_t queue_end; + uint16_t total_rate = 0; + struct rte_pci_device *pci_dev; + + pci_dev = RTE_ETH_DEV_TO_PCI(dev); + rte_eth_link_get_nowait(dev->data->port_id, &link); + + if (vf >= pci_dev->max_vfs) + return -EINVAL; + + if (tx_rate > link.link_speed) + return -EINVAL; + + if (q_msk == 0) + return 0; + + hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + vfinfo = *(IXGBE_DEV_PRIVATE_TO_P_VFDATA(dev->data->dev_private)); + nb_q_per_pool = RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool; + queue_stride = IXGBE_MAX_RX_QUEUE_NUM / RTE_ETH_DEV_SRIOV(dev).active; + queue_idx = vf * queue_stride; + queue_end = queue_idx + nb_q_per_pool - 1; + if (queue_end >= hw->mac.max_tx_queues) + return -EINVAL; + + if (vfinfo) { + for (vf_idx = 0; vf_idx < pci_dev->max_vfs; vf_idx++) { + if (vf_idx == vf) + continue; + for (idx = 0; idx < RTE_DIM(vfinfo[vf_idx].tx_rate); + idx++) + total_rate += vfinfo[vf_idx].tx_rate[idx]; + } + } else { + return -EINVAL; + } + + /* Store tx_rate for this vf. */ + for (idx = 0; idx < nb_q_per_pool; idx++) { + if (((uint64_t)0x1 << idx) & q_msk) { + if (vfinfo[vf].tx_rate[idx] != tx_rate) + vfinfo[vf].tx_rate[idx] = tx_rate; + total_rate += tx_rate; + } + } + + if (total_rate > dev->data->dev_link.link_speed) { + /* Reset stored TX rate of the VF if it causes exceed + * link speed. + */ + memset(vfinfo[vf].tx_rate, 0, sizeof(vfinfo[vf].tx_rate)); + return -EINVAL; + } + + /* Set RTTBCNRC of each queue/pool for vf X */ + for (; queue_idx <= queue_end; queue_idx++) { + if (0x1 & q_msk) + ixgbe_set_queue_rate_limit(dev, queue_idx, tx_rate); + q_msk = q_msk >> 1; + } + + return 0; +} + /* * Configure device link speed and setup link. * It returns 0 on success. @@ -2158,7 +2502,8 @@ ixgbe_dev_start(struct rte_eth_dev *dev) IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct ixgbe_vf_info *vfinfo = *IXGBE_DEV_PRIVATE_TO_P_VFDATA(dev->data->dev_private); - struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; uint32_t intr_vector = 0; int err, link_up = 0, negotiate = 0; uint32_t speed = 0; @@ -2166,6 +2511,8 @@ ixgbe_dev_start(struct rte_eth_dev *dev) int status; uint16_t vf, idx; uint32_t *link_speeds; + struct ixgbe_tm_conf *tm_conf = + IXGBE_DEV_PRIVATE_TO_TM_CONF(dev->data->dev_private); PMD_INIT_FUNC_TRACE(); @@ -2174,8 +2521,9 @@ ixgbe_dev_start(struct rte_eth_dev *dev) * - fixed speed: TODO implement */ if (dev->data->dev_conf.link_speeds & ETH_LINK_SPEED_FIXED) { - PMD_INIT_LOG(ERR, "Invalid link_speeds for port %hhu; fix speed not supported", - dev->data->port_id); + PMD_INIT_LOG(ERR, + "Invalid link_speeds for port %u, fix speed not supported", + dev->data->port_id); return -EINVAL; } @@ -2220,7 +2568,7 @@ ixgbe_dev_start(struct rte_eth_dev *dev) dev->data->nb_rx_queues * sizeof(int), 0); if (intr_handle->intr_vec == NULL) { PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues" - " intr_vec\n", dev->data->nb_rx_queues); + " intr_vec", dev->data->nb_rx_queues); return -ENOMEM; } } @@ -2238,6 +2586,41 @@ ixgbe_dev_start(struct rte_eth_dev *dev) goto error; } + mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | + ETH_VLAN_EXTEND_MASK; + err = ixgbe_vlan_offload_set(dev, mask); + if (err) { + PMD_INIT_LOG(ERR, "Unable to set VLAN offload"); + goto error; + } + + if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_VMDQ_ONLY) { + /* Enable vlan filtering for VMDq */ + ixgbe_vmdq_vlan_hw_filter_enable(dev); + } + + /* Configure DCB hw */ + ixgbe_configure_dcb(dev); + + if (dev->data->dev_conf.fdir_conf.mode != RTE_FDIR_MODE_NONE) { + err = ixgbe_fdir_configure(dev); + if (err) + goto error; + } + + /* Restore vf rate limit */ + if (vfinfo != NULL) { + for (vf = 0; vf < pci_dev->max_vfs; vf++) + for (idx = 0; idx < IXGBE_MAX_QUEUE_NUM_PER_VF; idx++) + if (vfinfo[vf].tx_rate[idx] != 0) + ixgbe_set_vf_rate_limit( + dev, vf, + vfinfo[vf].tx_rate[idx], + 1 << idx); + } + + ixgbe_restore_statistics_mapping(dev); + err = ixgbe_dev_rxtx_start(dev); if (err < 0) { PMD_INIT_LOG(ERR, "Unable to start rxtx queues"); @@ -2281,9 +2664,22 @@ ixgbe_dev_start(struct rte_eth_dev *dev) speed = 0x0; if (*link_speeds == ETH_LINK_SPEED_AUTONEG) { - speed = (hw->mac.type != ixgbe_mac_82598EB) ? - IXGBE_LINK_SPEED_82599_AUTONEG : - IXGBE_LINK_SPEED_82598_AUTONEG; + switch (hw->mac.type) { + case ixgbe_mac_82598EB: + speed = IXGBE_LINK_SPEED_82598_AUTONEG; + break; + case ixgbe_mac_82599EB: + case ixgbe_mac_X540: + speed = IXGBE_LINK_SPEED_82599_AUTONEG; + break; + case ixgbe_mac_X550: + case ixgbe_mac_X550EM_x: + case ixgbe_mac_X550EM_a: + speed = IXGBE_LINK_SPEED_X550_AUTONEG; + break; + default: + speed = IXGBE_LINK_SPEED_82599_AUTONEG; + } } else { if (*link_speeds & ETH_LINK_SPEED_10G) speed |= IXGBE_LINK_SPEED_10GB_FULL; @@ -2302,14 +2698,16 @@ skip_link_setup: if (rte_intr_allow_others(intr_handle)) { /* check if lsc interrupt is enabled */ if (dev->data->dev_conf.intr_conf.lsc != 0) - ixgbe_dev_lsc_interrupt_setup(dev); + ixgbe_dev_lsc_interrupt_setup(dev, TRUE); + else + ixgbe_dev_lsc_interrupt_setup(dev, FALSE); + ixgbe_dev_macsec_interrupt_setup(dev); } else { rte_intr_callback_unregister(intr_handle, - ixgbe_dev_interrupt_handler, - (void *)dev); + ixgbe_dev_interrupt_handler, dev); if (dev->data->dev_conf.intr_conf.lsc != 0) PMD_INIT_LOG(INFO, "lsc won't enable because of" - " no intr multiplex\n"); + " no intr multiplex"); } /* check if rxq interrupt is enabled */ @@ -2322,36 +2720,13 @@ skip_link_setup: /* resume enabled intr since hw reset */ ixgbe_enable_intr(dev); + ixgbe_l2_tunnel_conf(dev); + ixgbe_filter_restore(dev); - mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | - ETH_VLAN_EXTEND_MASK; - ixgbe_vlan_offload_set(dev, mask); - - if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_VMDQ_ONLY) { - /* Enable vlan filtering for VMDq */ - ixgbe_vmdq_vlan_hw_filter_enable(dev); - } - - /* Configure DCB hw */ - ixgbe_configure_dcb(dev); - - if (dev->data->dev_conf.fdir_conf.mode != RTE_FDIR_MODE_NONE) { - err = ixgbe_fdir_configure(dev); - if (err) - goto error; - } - - /* Restore vf rate limit */ - if (vfinfo != NULL) { - for (vf = 0; vf < dev->pci_dev->max_vfs; vf++) - for (idx = 0; idx < IXGBE_MAX_QUEUE_NUM_PER_VF; idx++) - if (vfinfo[vf].tx_rate[idx] != 0) - ixgbe_set_vf_rate_limit(dev, vf, - vfinfo[vf].tx_rate[idx], - 1 << idx); - } - - ixgbe_restore_statistics_mapping(dev); + if (tm_conf->root && !tm_conf->committed) + PMD_DRV_LOG(WARNING, + "please call hierarchy_commit() " + "before starting the port"); return 0; @@ -2372,11 +2747,11 @@ ixgbe_dev_stop(struct rte_eth_dev *dev) IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct ixgbe_vf_info *vfinfo = *IXGBE_DEV_PRIVATE_TO_P_VFDATA(dev->data->dev_private); - struct ixgbe_filter_info *filter_info = - IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); - struct ixgbe_5tuple_filter *p_5tuple, *p_5tuple_next; - struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; int vf; + struct ixgbe_tm_conf *tm_conf = + IXGBE_DEV_PRIVATE_TO_TM_CONF(dev->data->dev_private); PMD_INIT_FUNC_TRACE(); @@ -2390,8 +2765,7 @@ ixgbe_dev_stop(struct rte_eth_dev *dev) /* stop adapter */ ixgbe_stop_adapter(hw); - for (vf = 0; vfinfo != NULL && - vf < dev->pci_dev->max_vfs; vf++) + for (vf = 0; vfinfo != NULL && vf < pci_dev->max_vfs; vf++) vfinfo[vf].clear_to_send = false; if (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper) { @@ -2412,17 +2786,6 @@ ixgbe_dev_stop(struct rte_eth_dev *dev) memset(&link, 0, sizeof(link)); rte_ixgbe_dev_atomic_write_link_status(dev, &link); - /* Remove all ntuple filters of the device */ - for (p_5tuple = TAILQ_FIRST(&filter_info->fivetuple_list); - p_5tuple != NULL; p_5tuple = p_5tuple_next) { - p_5tuple_next = TAILQ_NEXT(p_5tuple, entries); - TAILQ_REMOVE(&filter_info->fivetuple_list, - p_5tuple, entries); - rte_free(p_5tuple); - } - memset(filter_info->fivetuple_mask, 0, - sizeof(uint32_t) * IXGBE_5TUPLE_ARRAY_SIZE); - if (!rte_intr_allow_others(intr_handle)) /* resume to the default handler */ rte_intr_callback_register(intr_handle, @@ -2435,6 +2798,9 @@ ixgbe_dev_stop(struct rte_eth_dev *dev) rte_free(intr_handle->intr_vec); intr_handle->intr_vec = NULL; } + + /* reset hierarchy commit */ + tm_conf->committed = false; } /* @@ -2446,7 +2812,7 @@ ixgbe_dev_set_link_up(struct rte_eth_dev *dev) struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); if (hw->mac.type == ixgbe_mac_82599EB) { -#ifdef RTE_NIC_BYPASS +#ifdef RTE_LIBRTE_IXGBE_BYPASS if (hw->device_id == IXGBE_DEV_ID_82599_BYPASS) { /* Not suported in bypass mode */ PMD_INIT_LOG(ERR, "Set link up is not supported " @@ -2476,7 +2842,7 @@ ixgbe_dev_set_link_down(struct rte_eth_dev *dev) struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); if (hw->mac.type == ixgbe_mac_82599EB) { -#ifdef RTE_NIC_BYPASS +#ifdef RTE_LIBRTE_IXGBE_BYPASS if (hw->device_id == IXGBE_DEV_ID_82599_BYPASS) { /* Not suported in bypass mode */ PMD_INIT_LOG(ERR, "Set link down is not supported " @@ -2498,7 +2864,7 @@ ixgbe_dev_set_link_down(struct rte_eth_dev *dev) } /* - * Reest and stop device. + * Reset and stop device. */ static void ixgbe_dev_close(struct rte_eth_dev *dev) @@ -2521,9 +2887,36 @@ ixgbe_dev_close(struct rte_eth_dev *dev) ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); } +/* + * Reset PF device. + */ +static int +ixgbe_dev_reset(struct rte_eth_dev *dev) +{ + int ret; + + /* When a DPDK PMD PF begin to reset PF port, it should notify all + * its VF to make them align with it. The detailed notification + * mechanism is PMD specific. As to ixgbe PF, it is rather complex. + * To avoid unexpected behavior in VF, currently reset of PF with + * SR-IOV activation is not supported. It might be supported later. + */ + if (dev->data->sriov.active) + return -ENOTSUP; + + ret = eth_ixgbe_dev_uninit(dev); + if (ret) + return ret; + + ret = eth_ixgbe_dev_init(dev); + + return ret; +} + static void ixgbe_read_stats_registers(struct ixgbe_hw *hw, struct ixgbe_hw_stats *hw_stats, + struct ixgbe_macsec_stats *macsec_stats, uint64_t *total_missed_rx, uint64_t *total_qbrc, uint64_t *total_qprc, uint64_t *total_qprdc) { @@ -2531,9 +2924,9 @@ ixgbe_read_stats_registers(struct ixgbe_hw *hw, uint32_t delta_gprc = 0; unsigned i; /* Workaround for RX byte count not including CRC bytes when CRC -+ * strip is enabled. CRC bytes are removed from counters when crc_strip + * strip is enabled. CRC bytes are removed from counters when crc_strip * is disabled. -+ */ + */ int crc_strip = (IXGBE_READ_REG(hw, IXGBE_HLREG0) & IXGBE_HLREG0_RXCRCSTRP); @@ -2693,18 +3086,55 @@ ixgbe_read_stats_registers(struct ixgbe_hw *hw, /* Flow Director Stats registers */ hw_stats->fdirmatch += IXGBE_READ_REG(hw, IXGBE_FDIRMATCH); hw_stats->fdirmiss += IXGBE_READ_REG(hw, IXGBE_FDIRMISS); + + /* MACsec Stats registers */ + macsec_stats->out_pkts_untagged += IXGBE_READ_REG(hw, IXGBE_LSECTXUT); + macsec_stats->out_pkts_encrypted += + IXGBE_READ_REG(hw, IXGBE_LSECTXPKTE); + macsec_stats->out_pkts_protected += + IXGBE_READ_REG(hw, IXGBE_LSECTXPKTP); + macsec_stats->out_octets_encrypted += + IXGBE_READ_REG(hw, IXGBE_LSECTXOCTE); + macsec_stats->out_octets_protected += + IXGBE_READ_REG(hw, IXGBE_LSECTXOCTP); + macsec_stats->in_pkts_untagged += IXGBE_READ_REG(hw, IXGBE_LSECRXUT); + macsec_stats->in_pkts_badtag += IXGBE_READ_REG(hw, IXGBE_LSECRXBAD); + macsec_stats->in_pkts_nosci += IXGBE_READ_REG(hw, IXGBE_LSECRXNOSCI); + macsec_stats->in_pkts_unknownsci += + IXGBE_READ_REG(hw, IXGBE_LSECRXUNSCI); + macsec_stats->in_octets_decrypted += + IXGBE_READ_REG(hw, IXGBE_LSECRXOCTD); + macsec_stats->in_octets_validated += + IXGBE_READ_REG(hw, IXGBE_LSECRXOCTV); + macsec_stats->in_pkts_unchecked += IXGBE_READ_REG(hw, IXGBE_LSECRXUNCH); + macsec_stats->in_pkts_delayed += IXGBE_READ_REG(hw, IXGBE_LSECRXDELAY); + macsec_stats->in_pkts_late += IXGBE_READ_REG(hw, IXGBE_LSECRXLATE); + for (i = 0; i < 2; i++) { + macsec_stats->in_pkts_ok += + IXGBE_READ_REG(hw, IXGBE_LSECRXOK(i)); + macsec_stats->in_pkts_invalid += + IXGBE_READ_REG(hw, IXGBE_LSECRXINV(i)); + macsec_stats->in_pkts_notvalid += + IXGBE_READ_REG(hw, IXGBE_LSECRXNV(i)); + } + macsec_stats->in_pkts_unusedsa += IXGBE_READ_REG(hw, IXGBE_LSECRXUNSA); + macsec_stats->in_pkts_notusingsa += + IXGBE_READ_REG(hw, IXGBE_LSECRXNUSA); } /* * This function is based on ixgbe_update_stats_counters() in ixgbe/ixgbe.c */ -static void +static int ixgbe_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) { struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct ixgbe_hw_stats *hw_stats = IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private); + struct ixgbe_macsec_stats *macsec_stats = + IXGBE_DEV_PRIVATE_TO_MACSEC_STATS( + dev->data->dev_private); uint64_t total_missed_rx, total_qbrc, total_qprc, total_qprdc; unsigned i; @@ -2713,11 +3143,11 @@ ixgbe_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) total_qprc = 0; total_qprdc = 0; - ixgbe_read_stats_registers(hw, hw_stats, &total_missed_rx, &total_qbrc, - &total_qprc, &total_qprdc); + ixgbe_read_stats_registers(hw, hw_stats, macsec_stats, &total_missed_rx, + &total_qbrc, &total_qprc, &total_qprdc); if (stats == NULL) - return; + return -EINVAL; /* Fill out the rte_eth_stats statistics structure */ stats->ipackets = total_qprc; @@ -2748,6 +3178,7 @@ ixgbe_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) /* Tx Errors */ stats->oerrors = 0; + return 0; } static void @@ -2766,13 +3197,13 @@ ixgbe_dev_stats_reset(struct rte_eth_dev *dev) /* This function calculates the number of xstats based on the current config */ static unsigned ixgbe_xstats_calc_num(void) { - return IXGBE_NB_HW_STATS + + return IXGBE_NB_HW_STATS + IXGBE_NB_MACSEC_STATS + (IXGBE_NB_RXQ_PRIO_STATS * IXGBE_NB_RXQ_PRIO_VALUES) + (IXGBE_NB_TXQ_PRIO_STATS * IXGBE_NB_TXQ_PRIO_VALUES); } static int ixgbe_dev_xstats_get_names(__rte_unused struct rte_eth_dev *dev, - struct rte_eth_xstat_name *xstats_names, __rte_unused unsigned limit) + struct rte_eth_xstat_name *xstats_names, __rte_unused unsigned int size) { const unsigned cnt_stats = ixgbe_xstats_calc_num(); unsigned stat, i, count; @@ -2793,6 +3224,15 @@ static int ixgbe_dev_xstats_get_names(__rte_unused struct rte_eth_dev *dev, count++; } + /* MACsec Stats */ + for (i = 0; i < IXGBE_NB_MACSEC_STATS; i++) { + snprintf(xstats_names[count].name, + sizeof(xstats_names[count].name), + "%s", + rte_ixgbe_macsec_strings[i].name); + count++; + } + /* RX Priority Stats */ for (stat = 0; stat < IXGBE_NB_RXQ_PRIO_STATS; stat++) { for (i = 0; i < IXGBE_NB_RXQ_PRIO_VALUES; i++) { @@ -2818,6 +3258,84 @@ static int ixgbe_dev_xstats_get_names(__rte_unused struct rte_eth_dev *dev, return cnt_stats; } +static int ixgbe_dev_xstats_get_names_by_id( + struct rte_eth_dev *dev, + struct rte_eth_xstat_name *xstats_names, + const uint64_t *ids, + unsigned int limit) +{ + if (!ids) { + const unsigned int cnt_stats = ixgbe_xstats_calc_num(); + unsigned int stat, i, count; + + if (xstats_names != NULL) { + count = 0; + + /* Note: limit >= cnt_stats checked upstream + * in rte_eth_xstats_names() + */ + + /* Extended stats from ixgbe_hw_stats */ + for (i = 0; i < IXGBE_NB_HW_STATS; i++) { + snprintf(xstats_names[count].name, + sizeof(xstats_names[count].name), + "%s", + rte_ixgbe_stats_strings[i].name); + count++; + } + + /* MACsec Stats */ + for (i = 0; i < IXGBE_NB_MACSEC_STATS; i++) { + snprintf(xstats_names[count].name, + sizeof(xstats_names[count].name), + "%s", + rte_ixgbe_macsec_strings[i].name); + count++; + } + + /* RX Priority Stats */ + for (stat = 0; stat < IXGBE_NB_RXQ_PRIO_STATS; stat++) { + for (i = 0; i < IXGBE_NB_RXQ_PRIO_VALUES; i++) { + snprintf(xstats_names[count].name, + sizeof(xstats_names[count].name), + "rx_priority%u_%s", i, + rte_ixgbe_rxq_strings[stat].name); + count++; + } + } + + /* TX Priority Stats */ + for (stat = 0; stat < IXGBE_NB_TXQ_PRIO_STATS; stat++) { + for (i = 0; i < IXGBE_NB_TXQ_PRIO_VALUES; i++) { + snprintf(xstats_names[count].name, + sizeof(xstats_names[count].name), + "tx_priority%u_%s", i, + rte_ixgbe_txq_strings[stat].name); + count++; + } + } + } + return cnt_stats; + } + + uint16_t i; + uint16_t size = ixgbe_xstats_calc_num(); + struct rte_eth_xstat_name xstats_names_copy[size]; + + ixgbe_dev_xstats_get_names_by_id(dev, xstats_names_copy, NULL, + size); + + for (i = 0; i < limit; i++) { + if (ids[i] >= size) { + PMD_INIT_LOG(ERR, "id value isn't valid"); + return -1; + } + strcpy(xstats_names[i].name, + xstats_names_copy[ids[i]].name); + } + return limit; +} + static int ixgbevf_dev_xstats_get_names(__rte_unused struct rte_eth_dev *dev, struct rte_eth_xstat_name *xstats_names, unsigned limit) { @@ -2842,6 +3360,9 @@ ixgbe_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct ixgbe_hw_stats *hw_stats = IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private); + struct ixgbe_macsec_stats *macsec_stats = + IXGBE_DEV_PRIVATE_TO_MACSEC_STATS( + dev->data->dev_private); uint64_t total_missed_rx, total_qbrc, total_qprc, total_qprdc; unsigned i, stat, count = 0; @@ -2855,8 +3376,8 @@ ixgbe_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, total_qprc = 0; total_qprdc = 0; - ixgbe_read_stats_registers(hw, hw_stats, &total_missed_rx, &total_qbrc, - &total_qprc, &total_qprdc); + ixgbe_read_stats_registers(hw, hw_stats, macsec_stats, &total_missed_rx, + &total_qbrc, &total_qprc, &total_qprdc); /* If this is a reset xstats is NULL, and we have cleared the * registers by reading them. @@ -2869,6 +3390,15 @@ ixgbe_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, for (i = 0; i < IXGBE_NB_HW_STATS; i++) { xstats[count].value = *(uint64_t *)(((char *)hw_stats) + rte_ixgbe_stats_strings[i].offset); + xstats[count].id = count; + count++; + } + + /* MACsec Stats */ + for (i = 0; i < IXGBE_NB_MACSEC_STATS; i++) { + xstats[count].value = *(uint64_t *)(((char *)macsec_stats) + + rte_ixgbe_macsec_strings[i].offset); + xstats[count].id = count; count++; } @@ -2878,6 +3408,7 @@ ixgbe_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, xstats[count].value = *(uint64_t *)(((char *)hw_stats) + rte_ixgbe_rxq_strings[stat].offset + (sizeof(uint64_t) * i)); + xstats[count].id = count; count++; } } @@ -2888,17 +3419,112 @@ ixgbe_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, xstats[count].value = *(uint64_t *)(((char *)hw_stats) + rte_ixgbe_txq_strings[stat].offset + (sizeof(uint64_t) * i)); + xstats[count].id = count; count++; } } return count; } +static int +ixgbe_dev_xstats_get_by_id(struct rte_eth_dev *dev, const uint64_t *ids, + uint64_t *values, unsigned int n) +{ + if (!ids) { + struct ixgbe_hw *hw = + IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct ixgbe_hw_stats *hw_stats = + IXGBE_DEV_PRIVATE_TO_STATS( + dev->data->dev_private); + struct ixgbe_macsec_stats *macsec_stats = + IXGBE_DEV_PRIVATE_TO_MACSEC_STATS( + dev->data->dev_private); + uint64_t total_missed_rx, total_qbrc, total_qprc, total_qprdc; + unsigned int i, stat, count = 0; + + count = ixgbe_xstats_calc_num(); + + if (!ids && n < count) + return count; + + total_missed_rx = 0; + total_qbrc = 0; + total_qprc = 0; + total_qprdc = 0; + + ixgbe_read_stats_registers(hw, hw_stats, macsec_stats, + &total_missed_rx, &total_qbrc, &total_qprc, + &total_qprdc); + + /* If this is a reset xstats is NULL, and we have cleared the + * registers by reading them. + */ + if (!ids && !values) + return 0; + + /* Extended stats from ixgbe_hw_stats */ + count = 0; + for (i = 0; i < IXGBE_NB_HW_STATS; i++) { + values[count] = *(uint64_t *)(((char *)hw_stats) + + rte_ixgbe_stats_strings[i].offset); + count++; + } + + /* MACsec Stats */ + for (i = 0; i < IXGBE_NB_MACSEC_STATS; i++) { + values[count] = *(uint64_t *)(((char *)macsec_stats) + + rte_ixgbe_macsec_strings[i].offset); + count++; + } + + /* RX Priority Stats */ + for (stat = 0; stat < IXGBE_NB_RXQ_PRIO_STATS; stat++) { + for (i = 0; i < IXGBE_NB_RXQ_PRIO_VALUES; i++) { + values[count] = + *(uint64_t *)(((char *)hw_stats) + + rte_ixgbe_rxq_strings[stat].offset + + (sizeof(uint64_t) * i)); + count++; + } + } + + /* TX Priority Stats */ + for (stat = 0; stat < IXGBE_NB_TXQ_PRIO_STATS; stat++) { + for (i = 0; i < IXGBE_NB_TXQ_PRIO_VALUES; i++) { + values[count] = + *(uint64_t *)(((char *)hw_stats) + + rte_ixgbe_txq_strings[stat].offset + + (sizeof(uint64_t) * i)); + count++; + } + } + return count; + } + + uint16_t i; + uint16_t size = ixgbe_xstats_calc_num(); + uint64_t values_copy[size]; + + ixgbe_dev_xstats_get_by_id(dev, NULL, values_copy, size); + + for (i = 0; i < n; i++) { + if (ids[i] >= size) { + PMD_INIT_LOG(ERR, "id value isn't valid"); + return -1; + } + values[i] = values_copy[ids[i]]; + } + return n; +} + static void ixgbe_dev_xstats_reset(struct rte_eth_dev *dev) { struct ixgbe_hw_stats *stats = IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private); + struct ixgbe_macsec_stats *macsec_stats = + IXGBE_DEV_PRIVATE_TO_MACSEC_STATS( + dev->data->dev_private); unsigned count = ixgbe_xstats_calc_num(); @@ -2907,6 +3533,7 @@ ixgbe_dev_xstats_reset(struct rte_eth_dev *dev) /* Reset software totals */ memset(stats, 0, sizeof(*stats)); + memset(macsec_stats, 0, sizeof(*macsec_stats)); } static void @@ -2955,6 +3582,7 @@ ixgbevf_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, /* Extended stats */ for (i = 0; i < IXGBEVF_NB_XSTATS; i++) { + xstats[i].id = i; xstats[i].value = *(uint64_t *)(((char *)hw_stats) + rte_ixgbevf_stats_strings[i].offset); } @@ -2962,7 +3590,7 @@ ixgbevf_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, return IXGBEVF_NB_XSTATS; } -static void +static int ixgbevf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) { struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats *) @@ -2971,12 +3599,13 @@ ixgbevf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) ixgbevf_update_stats(dev); if (stats == NULL) - return; + return -EINVAL; stats->ipackets = hw_stats->vfgprc; stats->ibytes = hw_stats->vfgorc; stats->opackets = hw_stats->vfgptc; stats->obytes = hw_stats->vfgotc; + return 0; } static void @@ -2995,12 +3624,35 @@ ixgbevf_dev_stats_reset(struct rte_eth_dev *dev) hw_stats->vfgotc = 0; } +static int +ixgbe_fw_version_get(struct rte_eth_dev *dev, char *fw_version, size_t fw_size) +{ + struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + u16 eeprom_verh, eeprom_verl; + u32 etrack_id; + int ret; + + ixgbe_read_eeprom(hw, 0x2e, &eeprom_verh); + ixgbe_read_eeprom(hw, 0x2d, &eeprom_verl); + + etrack_id = (eeprom_verh << 16) | eeprom_verl; + ret = snprintf(fw_version, fw_size, "0x%08x", etrack_id); + + ret += 1; /* add the size of '\0' */ + if (fw_size < (u32)ret) + return ret; + else + return 0; +} + static void ixgbe_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) { + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct rte_eth_conf *dev_conf = &dev->data->dev_conf; + dev_info->pci_dev = pci_dev; dev_info->max_rx_queues = (uint16_t)hw->mac.max_rx_queues; dev_info->max_tx_queues = (uint16_t)hw->mac.max_tx_queues; if (RTE_ETH_DEV_SRIOV(dev).active == 0) { @@ -3016,7 +3668,7 @@ ixgbe_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) dev_info->max_rx_pktlen = 15872; /* includes CRC, cf MAXFRS register */ dev_info->max_mac_addrs = hw->mac.num_rar_entries; dev_info->max_hash_mac_addrs = IXGBE_VMDQ_NUM_UC_MAC; - dev_info->max_vfs = dev->pci_dev->max_vfs; + dev_info->max_vfs = pci_dev->max_vfs; if (hw->mac.type == ixgbe_mac_82598EB) dev_info->max_vmdq_pools = ETH_16_POOLS; else @@ -3037,6 +3689,10 @@ ixgbe_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) !RTE_ETH_DEV_SRIOV(dev).active) dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_TCP_LRO; + if (hw->mac.type == ixgbe_mac_82599EB || + hw->mac.type == ixgbe_mac_X540) + dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_MACSEC_STRIP; + if (hw->mac.type == ixgbe_mac_X550 || hw->mac.type == ixgbe_mac_X550EM_x || hw->mac.type == ixgbe_mac_X550EM_a) @@ -3050,11 +3706,20 @@ ixgbe_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) DEV_TX_OFFLOAD_SCTP_CKSUM | DEV_TX_OFFLOAD_TCP_TSO; + if (hw->mac.type == ixgbe_mac_82599EB || + hw->mac.type == ixgbe_mac_X540) + dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_MACSEC_INSERT; + if (hw->mac.type == ixgbe_mac_X550 || hw->mac.type == ixgbe_mac_X550EM_x || hw->mac.type == ixgbe_mac_X550EM_a) dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM; +#ifdef RTE_LIBRTE_SECURITY + dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_SECURITY; + dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_SECURITY; +#endif + dev_info->default_rxconf = (struct rte_eth_rxconf) { .rx_thresh = { .pthresh = IXGBE_DEFAULT_RX_PTHRESH, @@ -3091,6 +3756,10 @@ ixgbe_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) hw->mac.type == ixgbe_mac_X550_vf) { dev_info->speed_capa |= ETH_LINK_SPEED_100M; } + if (hw->mac.type == ixgbe_mac_X550) { + dev_info->speed_capa |= ETH_LINK_SPEED_2_5G; + dev_info->speed_capa |= ETH_LINK_SPEED_5G; + } } static const uint32_t * @@ -3123,6 +3792,12 @@ ixgbe_dev_supported_ptypes_get(struct rte_eth_dev *dev) dev->rx_pkt_burst == ixgbe_recv_pkts_lro_bulk_alloc || dev->rx_pkt_burst == ixgbe_recv_pkts_bulk_alloc) return ptypes; + +#if defined(RTE_ARCH_X86) + if (dev->rx_pkt_burst == ixgbe_recv_pkts_vec || + dev->rx_pkt_burst == ixgbe_recv_scattered_pkts_vec) + return ptypes; +#endif return NULL; } @@ -3130,15 +3805,17 @@ static void ixgbevf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) { + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + dev_info->pci_dev = pci_dev; dev_info->max_rx_queues = (uint16_t)hw->mac.max_rx_queues; dev_info->max_tx_queues = (uint16_t)hw->mac.max_tx_queues; dev_info->min_rx_bufsize = 1024; /* cf BSIZEPACKET in SRRCTL reg */ - dev_info->max_rx_pktlen = 15872; /* includes CRC, cf MAXFRS reg */ + dev_info->max_rx_pktlen = 9728; /* includes CRC, cf MAXFRS reg */ dev_info->max_mac_addrs = hw->mac.num_rar_entries; dev_info->max_hash_mac_addrs = IXGBE_VMDQ_NUM_UC_MAC; - dev_info->max_vfs = dev->pci_dev->max_vfs; + dev_info->max_vfs = pci_dev->max_vfs; if (hw->mac.type == ixgbe_mac_82598EB) dev_info->max_vmdq_pools = ETH_16_POOLS; else @@ -3180,29 +3857,153 @@ ixgbevf_dev_info_get(struct rte_eth_dev *dev, dev_info->tx_desc_lim = tx_desc_lim; } +static int +ixgbevf_check_link(struct ixgbe_hw *hw, ixgbe_link_speed *speed, + int *link_up, int wait_to_complete) +{ + /** + * for a quick link status checking, wait_to_compelet == 0, + * skip PF link status checking + */ + bool no_pflink_check = wait_to_complete == 0; + struct ixgbe_mbx_info *mbx = &hw->mbx; + struct ixgbe_mac_info *mac = &hw->mac; + uint32_t links_reg, in_msg; + int ret_val = 0; + + /* If we were hit with a reset drop the link */ + if (!mbx->ops.check_for_rst(hw, 0) || !mbx->timeout) + mac->get_link_status = true; + + if (!mac->get_link_status) + goto out; + + /* if link status is down no point in checking to see if pf is up */ + links_reg = IXGBE_READ_REG(hw, IXGBE_VFLINKS); + if (!(links_reg & IXGBE_LINKS_UP)) + goto out; + + /* for SFP+ modules and DA cables on 82599 it can take up to 500usecs + * before the link status is correct + */ + if (mac->type == ixgbe_mac_82599_vf) { + int i; + + for (i = 0; i < 5; i++) { + rte_delay_us(100); + links_reg = IXGBE_READ_REG(hw, IXGBE_VFLINKS); + + if (!(links_reg & IXGBE_LINKS_UP)) + goto out; + } + } + + switch (links_reg & IXGBE_LINKS_SPEED_82599) { + case IXGBE_LINKS_SPEED_10G_82599: + *speed = IXGBE_LINK_SPEED_10GB_FULL; + if (hw->mac.type >= ixgbe_mac_X550) { + if (links_reg & IXGBE_LINKS_SPEED_NON_STD) + *speed = IXGBE_LINK_SPEED_2_5GB_FULL; + } + break; + case IXGBE_LINKS_SPEED_1G_82599: + *speed = IXGBE_LINK_SPEED_1GB_FULL; + break; + case IXGBE_LINKS_SPEED_100_82599: + *speed = IXGBE_LINK_SPEED_100_FULL; + if (hw->mac.type == ixgbe_mac_X550) { + if (links_reg & IXGBE_LINKS_SPEED_NON_STD) + *speed = IXGBE_LINK_SPEED_5GB_FULL; + } + break; + case IXGBE_LINKS_SPEED_10_X550EM_A: + *speed = IXGBE_LINK_SPEED_UNKNOWN; + /* Since Reserved in older MAC's */ + if (hw->mac.type >= ixgbe_mac_X550) + *speed = IXGBE_LINK_SPEED_10_FULL; + break; + default: + *speed = IXGBE_LINK_SPEED_UNKNOWN; + } + + if (no_pflink_check) { + if (*speed == IXGBE_LINK_SPEED_UNKNOWN) + mac->get_link_status = true; + else + mac->get_link_status = false; + + goto out; + } + /* if the read failed it could just be a mailbox collision, best wait + * until we are called again and don't report an error + */ + if (mbx->ops.read(hw, &in_msg, 1, 0)) + goto out; + + if (!(in_msg & IXGBE_VT_MSGTYPE_CTS)) { + /* msg is not CTS and is NACK we must have lost CTS status */ + if (in_msg & IXGBE_VT_MSGTYPE_NACK) + ret_val = -1; + goto out; + } + + /* the pf is talking, if we timed out in the past we reinit */ + if (!mbx->timeout) { + ret_val = -1; + goto out; + } + + /* if we passed all the tests above then the link is up and we no + * longer need to check for link + */ + mac->get_link_status = false; + +out: + *link_up = !mac->get_link_status; + return ret_val; +} + /* return 0 means link status changed, -1 means not changed */ static int -ixgbe_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete) +ixgbe_dev_link_update_share(struct rte_eth_dev *dev, + int wait_to_complete, int vf) { struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct rte_eth_link link, old; ixgbe_link_speed link_speed = IXGBE_LINK_SPEED_UNKNOWN; + struct ixgbe_interrupt *intr = + IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); int link_up; int diag; + u32 speed = 0; + int wait = 1; + bool autoneg = false; link.link_status = ETH_LINK_DOWN; link.link_speed = 0; link.link_duplex = ETH_LINK_HALF_DUPLEX; + link.link_autoneg = ETH_LINK_AUTONEG; memset(&old, 0, sizeof(old)); rte_ixgbe_dev_atomic_read_link_status(dev, &old); hw->mac.get_link_status = true; + if ((intr->flags & IXGBE_FLAG_NEED_LINK_CONFIG) && + ixgbe_get_media_type(hw) == ixgbe_media_type_fiber) { + speed = hw->phy.autoneg_advertised; + if (!speed) + ixgbe_get_link_capabilities(hw, &speed, &autoneg); + ixgbe_setup_link(hw, speed, true); + } + /* check if it needs to wait to complete, if lsc interrupt is enabled */ if (wait_to_complete == 0 || dev->data->dev_conf.intr_conf.lsc != 0) - diag = ixgbe_check_link(hw, &link_speed, &link_up, 0); + wait = 0; + + if (vf) + diag = ixgbevf_check_link(hw, &link_speed, &link_up, wait); else - diag = ixgbe_check_link(hw, &link_speed, &link_up, 1); + diag = ixgbe_check_link(hw, &link_speed, &link_up, wait); if (diag != 0) { link.link_speed = ETH_SPEED_NUM_100M; @@ -3215,10 +4016,12 @@ ixgbe_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete) if (link_up == 0) { rte_ixgbe_dev_atomic_write_link_status(dev, &link); + intr->flags |= IXGBE_FLAG_NEED_LINK_CONFIG; if (link.link_status == old.link_status) return -1; return 0; } + intr->flags &= ~IXGBE_FLAG_NEED_LINK_CONFIG; link.link_status = ETH_LINK_UP; link.link_duplex = ETH_LINK_FULL_DUPLEX; @@ -3237,6 +4040,14 @@ ixgbe_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete) link.link_speed = ETH_SPEED_NUM_1G; break; + case IXGBE_LINK_SPEED_2_5GB_FULL: + link.link_speed = ETH_SPEED_NUM_2_5G; + break; + + case IXGBE_LINK_SPEED_5GB_FULL: + link.link_speed = ETH_SPEED_NUM_5G; + break; + case IXGBE_LINK_SPEED_10GB_FULL: link.link_speed = ETH_SPEED_NUM_10G; break; @@ -3249,6 +4060,18 @@ ixgbe_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete) return 0; } +static int +ixgbe_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete) +{ + return ixgbe_dev_link_update_share(dev, wait_to_complete, 0); +} + +static int +ixgbevf_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete) +{ + return ixgbe_dev_link_update_share(dev, wait_to_complete, 1); +} + static void ixgbe_dev_promiscuous_enable(struct rte_eth_dev *dev) { @@ -3306,19 +4129,24 @@ ixgbe_dev_allmulticast_disable(struct rte_eth_dev *dev) * * @param dev * Pointer to struct rte_eth_dev. + * @param on + * Enable or Disable. * * @return * - On success, zero. * - On failure, a negative value. */ static int -ixgbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev) +ixgbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev, uint8_t on) { struct ixgbe_interrupt *intr = IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); ixgbe_dev_link_status_print(dev); - intr->mask |= IXGBE_EICR_LSC; + if (on) + intr->mask |= IXGBE_EICR_LSC; + else + intr->mask &= ~IXGBE_EICR_LSC; return 0; } @@ -3345,6 +4173,28 @@ ixgbe_dev_rxq_interrupt_setup(struct rte_eth_dev *dev) return 0; } +/** + * It clears the interrupt causes and enables the interrupt. + * It will be called once only during nic initialized. + * + * @param dev + * Pointer to struct rte_eth_dev. + * + * @return + * - On success, zero. + * - On failure, a negative value. + */ +static int +ixgbe_dev_macsec_interrupt_setup(struct rte_eth_dev *dev) +{ + struct ixgbe_interrupt *intr = + IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); + + intr->mask |= IXGBE_EICR_LINKSEC; + + return 0; +} + /* * It reads ICR and sets flag (IXGBE_EICR_LSC) for the link_update. * @@ -3379,6 +4229,9 @@ ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev) if (eicr & IXGBE_EICR_MAILBOX) intr->flags |= IXGBE_FLAG_MAILBOX; + if (eicr & IXGBE_EICR_LINKSEC) + intr->flags |= IXGBE_FLAG_MACSEC; + if (hw->mac.type == ixgbe_mac_X550EM_x && hw->phy.type == ixgbe_phy_x550em_ext_t && (eicr & IXGBE_EICR_GPI_SDP0_X550EM_x)) @@ -3400,6 +4253,7 @@ ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev) static void ixgbe_dev_link_status_print(struct rte_eth_dev *dev) { + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); struct rte_eth_link link; memset(&link, 0, sizeof(link)); @@ -3414,11 +4268,11 @@ ixgbe_dev_link_status_print(struct rte_eth_dev *dev) PMD_INIT_LOG(INFO, " Port %d: Link Down", (int)(dev->data->port_id)); } - PMD_INIT_LOG(DEBUG, "PCI Address: %04d:%02d:%02d:%d", - dev->pci_dev->addr.domain, - dev->pci_dev->addr.bus, - dev->pci_dev->addr.devid, - dev->pci_dev->addr.function); + PMD_INIT_LOG(DEBUG, "PCI Address: " PCI_PRI_FMT, + pci_dev->addr.domain, + pci_dev->addr.bus, + pci_dev->addr.devid, + pci_dev->addr.function); } /* @@ -3432,13 +4286,13 @@ ixgbe_dev_link_status_print(struct rte_eth_dev *dev) * - On failure, a negative value. */ static int -ixgbe_dev_interrupt_action(struct rte_eth_dev *dev) +ixgbe_dev_interrupt_action(struct rte_eth_dev *dev, + struct rte_intr_handle *intr_handle) { struct ixgbe_interrupt *intr = IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); int64_t timeout; struct rte_eth_link link; - int intr_enable_delay = false; struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); @@ -3471,20 +4325,20 @@ ixgbe_dev_interrupt_action(struct rte_eth_dev *dev) timeout = IXGBE_LINK_DOWN_CHECK_TIMEOUT; ixgbe_dev_link_status_print(dev); - - intr_enable_delay = true; - } - - if (intr_enable_delay) { if (rte_eal_alarm_set(timeout * 1000, ixgbe_dev_interrupt_delayed_handler, (void *)dev) < 0) PMD_DRV_LOG(ERR, "Error setting alarm"); - } else { - PMD_DRV_LOG(DEBUG, "enable intr immediately"); - ixgbe_enable_intr(dev); - rte_intr_enable(&(dev->pci_dev->intr_handle)); + else { + /* remember original mask */ + intr->mask_original = intr->mask; + /* only disable lsc interrupt */ + intr->mask &= ~IXGBE_EIMS_LSC; + } } + PMD_DRV_LOG(DEBUG, "enable intr immediately"); + ixgbe_enable_intr(dev); + rte_intr_enable(intr_handle); return 0; } @@ -3507,12 +4361,16 @@ static void ixgbe_dev_interrupt_delayed_handler(void *param) { struct rte_eth_dev *dev = (struct rte_eth_dev *)param; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; struct ixgbe_interrupt *intr = IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); uint32_t eicr; + ixgbe_disable_intr(hw); + eicr = IXGBE_READ_REG(hw, IXGBE_EICR); if (eicr & IXGBE_EICR_MAILBOX) ixgbe_pf_mbx_process(dev); @@ -3526,12 +4384,23 @@ ixgbe_dev_interrupt_delayed_handler(void *param) ixgbe_dev_link_update(dev, 0); intr->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE; ixgbe_dev_link_status_print(dev); - _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC); + _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, + NULL, NULL); } + if (intr->flags & IXGBE_FLAG_MACSEC) { + _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_MACSEC, + NULL, NULL); + intr->flags &= ~IXGBE_FLAG_MACSEC; + } + + /* restore original mask */ + intr->mask = intr->mask_original; + intr->mask_original = 0; + PMD_DRV_LOG(DEBUG, "enable intr in delayed handler S[%08x]", eicr); ixgbe_enable_intr(dev); - rte_intr_enable(&(dev->pci_dev->intr_handle)); + rte_intr_enable(intr_handle); } /** @@ -3547,13 +4416,12 @@ ixgbe_dev_interrupt_delayed_handler(void *param) * void */ static void -ixgbe_dev_interrupt_handler(__rte_unused struct rte_intr_handle *handle, - void *param) +ixgbe_dev_interrupt_handler(void *param) { struct rte_eth_dev *dev = (struct rte_eth_dev *)param; ixgbe_dev_interrupt_get_status(dev); - ixgbe_dev_interrupt_action(dev); + ixgbe_dev_interrupt_action(dev, dev->intr_handle); } static int @@ -3917,7 +4785,7 @@ ixgbe_dev_rss_reta_update(struct rte_eth_dev *dev, if (reta_size != sp_reta_size) { PMD_DRV_LOG(ERR, "The size of hash lookup table configured " "(%d) doesn't match the number hardware can supported " - "(%d)\n", reta_size, sp_reta_size); + "(%d)", reta_size, sp_reta_size); return -EINVAL; } @@ -3964,7 +4832,7 @@ ixgbe_dev_rss_reta_query(struct rte_eth_dev *dev, if (reta_size != sp_reta_size) { PMD_DRV_LOG(ERR, "The size of hash lookup table configured " "(%d) doesn't match the number hardware can supported " - "(%d)\n", reta_size, sp_reta_size); + "(%d)", reta_size, sp_reta_size); return -EINVAL; } @@ -3989,14 +4857,15 @@ ixgbe_dev_rss_reta_query(struct rte_eth_dev *dev, return 0; } -static void +static int ixgbe_add_rar(struct rte_eth_dev *dev, struct ether_addr *mac_addr, uint32_t index, uint32_t pool) { struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); uint32_t enable_addr = 1; - ixgbe_set_rar(hw, index, mac_addr->addr_bytes, pool, enable_addr); + return ixgbe_set_rar(hw, index, mac_addr->addr_bytes, + pool, enable_addr); } static void @@ -4010,9 +4879,26 @@ ixgbe_remove_rar(struct rte_eth_dev *dev, uint32_t index) static void ixgbe_set_default_mac_addr(struct rte_eth_dev *dev, struct ether_addr *addr) { + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + ixgbe_remove_rar(dev, 0); - ixgbe_add_rar(dev, addr, 0, 0); + ixgbe_add_rar(dev, addr, 0, pci_dev->max_vfs); +} + +static bool +is_device_supported(struct rte_eth_dev *dev, struct rte_pci_driver *drv) +{ + if (strcmp(dev->device->driver->name, drv->driver.name)) + return false; + + return true; +} + +bool +is_ixgbe_supported(struct rte_eth_dev *dev) +{ + return is_device_supported(dev, &rte_ixgbe_pmd); } static int @@ -4023,6 +4909,7 @@ ixgbe_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) struct ixgbe_hw *hw; struct rte_eth_dev_info dev_info; uint32_t frame_size = mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; + struct rte_eth_dev_data *dev_data = dev->data; ixgbe_dev_info_get(dev, &dev_info); @@ -4030,13 +4917,15 @@ ixgbe_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) if ((mtu < ETHER_MIN_MTU) || (frame_size > dev_info.max_rx_pktlen)) return -EINVAL; - /* refuse mtu that requires the support of scattered packets when this - * feature has not been enabled before. + /* If device is started, refuse mtu that requires the support of + * scattered packets when this feature has not been enabled before. */ - if (!dev->data->scattered_rx && + if (dev_data->dev_started && !dev_data->scattered_rx && (frame_size + 2 * IXGBE_VLAN_TAG_SIZE > - dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM)) + dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM)) { + PMD_INIT_LOG(ERR, "Stop port first."); return -EINVAL; + } hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0); @@ -4131,13 +5020,18 @@ ixgbevf_dev_start(struct rte_eth_dev *dev) struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); uint32_t intr_vector = 0; - struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; int err, mask = 0; PMD_INIT_FUNC_TRACE(); - hw->mac.ops.reset_hw(hw); + err = hw->mac.ops.reset_hw(hw); + if (err) { + PMD_INIT_LOG(ERR, "Unable to reset vf hardware (%d)", err); + return err; + } hw->mac.get_link_status = true; /* negotiate mailbox API version to use with the PF. */ @@ -4159,13 +5053,22 @@ ixgbevf_dev_start(struct rte_eth_dev *dev) /* Set HW strip */ mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | ETH_VLAN_EXTEND_MASK; - ixgbevf_vlan_offload_set(dev, mask); + err = ixgbevf_vlan_offload_set(dev, mask); + if (err) { + PMD_INIT_LOG(ERR, "Unable to set VLAN offload (%d)", err); + ixgbe_dev_clear_queues(dev); + return err; + } ixgbevf_dev_rxtx_start(dev); /* check and configure queue intr-vector mapping */ - if (dev->data->dev_conf.intr_conf.rxq != 0) { - intr_vector = dev->data->nb_rx_queues; + if (rte_intr_cap_multiple(intr_handle) && + dev->data->dev_conf.intr_conf.rxq) { + /* According to datasheet, only vector 0/1/2 can be used, + * now only one vector is used for Rx queue + */ + intr_vector = 1; if (rte_intr_efd_enable(intr_handle, intr_vector)) return -1; } @@ -4176,12 +5079,21 @@ ixgbevf_dev_start(struct rte_eth_dev *dev) dev->data->nb_rx_queues * sizeof(int), 0); if (intr_handle->intr_vec == NULL) { PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues" - " intr_vec\n", dev->data->nb_rx_queues); + " intr_vec", dev->data->nb_rx_queues); return -ENOMEM; } } ixgbevf_configure_msix(dev); + /* When a VF port is bound to VFIO-PCI, only miscellaneous interrupt + * is mapped to VFIO vector 0 in eth_ixgbevf_dev_init( ). + * If previous VFIO interrupt mapping setting in eth_ixgbevf_dev_init( ) + * is not cleared, it will fail when following rte_intr_enable( ) tries + * to map Rx queue interrupt to other VFIO vectors. + * So clear uio/vfio intr/evevnfd first to avoid failure. + */ + rte_intr_disable(intr_handle); + rte_intr_enable(intr_handle); /* Re-enable interrupt for VF */ @@ -4194,7 +5106,8 @@ static void ixgbevf_dev_stop(struct rte_eth_dev *dev) { struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); - struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; PMD_INIT_FUNC_TRACE(); @@ -4243,6 +5156,23 @@ ixgbevf_dev_close(struct rte_eth_dev *dev) ixgbevf_remove_mac_addr(dev, 0); } +/* + * Reset VF device + */ +static int +ixgbevf_dev_reset(struct rte_eth_dev *dev) +{ + int ret; + + ret = eth_ixgbevf_dev_uninit(dev); + if (ret) + return ret; + + ret = eth_ixgbevf_dev_init(dev); + + return ret; +} + static void ixgbevf_set_vfta_all(struct rte_eth_dev *dev, bool on) { struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); @@ -4318,7 +5248,7 @@ ixgbevf_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on) ixgbe_vlan_hw_strip_bitmap_set(dev, queue, on); } -static void +static int ixgbevf_vlan_offload_set(struct rte_eth_dev *dev, int mask) { struct ixgbe_hw *hw = @@ -4333,17 +5263,19 @@ ixgbevf_vlan_offload_set(struct rte_eth_dev *dev, int mask) for (i = 0; i < hw->mac.max_rx_queues; i++) ixgbevf_vlan_strip_queue_set(dev, i, on); } + + return 0; } -static int -ixgbe_vmdq_mode_check(struct ixgbe_hw *hw) +int +ixgbe_vt_check(struct ixgbe_hw *hw) { uint32_t reg_val; - /* we only need to do this if VMDq is enabled */ + /* if Virtualization Technology is enabled */ reg_val = IXGBE_READ_REG(hw, IXGBE_VT_CTL); if (!(reg_val & IXGBE_VT_CTL_VT_ENABLE)) { - PMD_INIT_LOG(ERR, "VMDq must be enabled for this setting"); + PMD_INIT_LOG(ERR, "VT must be enabled for this setting"); return -1; } @@ -4481,133 +5413,6 @@ ixgbe_convert_vm_rx_mask_to_val(uint16_t rx_mask, uint32_t orig_val) return new_val; } -static int -ixgbe_set_pool_rx_mode(struct rte_eth_dev *dev, uint16_t pool, - uint16_t rx_mask, uint8_t on) -{ - int val = 0; - - struct ixgbe_hw *hw = - IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); - uint32_t vmolr = IXGBE_READ_REG(hw, IXGBE_VMOLR(pool)); - - if (hw->mac.type == ixgbe_mac_82598EB) { - PMD_INIT_LOG(ERR, "setting VF receive mode set should be done" - " on 82599 hardware and newer"); - return -ENOTSUP; - } - if (ixgbe_vmdq_mode_check(hw) < 0) - return -ENOTSUP; - - val = ixgbe_convert_vm_rx_mask_to_val(rx_mask, val); - - if (on) - vmolr |= val; - else - vmolr &= ~val; - - IXGBE_WRITE_REG(hw, IXGBE_VMOLR(pool), vmolr); - - return 0; -} - -static int -ixgbe_set_pool_rx(struct rte_eth_dev *dev, uint16_t pool, uint8_t on) -{ - uint32_t reg, addr; - uint32_t val; - const uint8_t bit1 = 0x1; - - struct ixgbe_hw *hw = - IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); - - if (ixgbe_vmdq_mode_check(hw) < 0) - return -ENOTSUP; - - if (pool >= ETH_64_POOLS) - return -EINVAL; - - /* for pool >= 32, set bit in PFVFRE[1], otherwise PFVFRE[0] */ - if (pool >= 32) { - addr = IXGBE_VFRE(1); - val = bit1 << (pool - 32); - } else { - addr = IXGBE_VFRE(0); - val = bit1 << pool; - } - - reg = IXGBE_READ_REG(hw, addr); - - if (on) - reg |= val; - else - reg &= ~val; - - IXGBE_WRITE_REG(hw, addr, reg); - - return 0; -} - -static int -ixgbe_set_pool_tx(struct rte_eth_dev *dev, uint16_t pool, uint8_t on) -{ - uint32_t reg, addr; - uint32_t val; - const uint8_t bit1 = 0x1; - - struct ixgbe_hw *hw = - IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); - - if (ixgbe_vmdq_mode_check(hw) < 0) - return -ENOTSUP; - - if (pool >= ETH_64_POOLS) - return -EINVAL; - - /* for pool >= 32, set bit in PFVFTE[1], otherwise PFVFTE[0] */ - if (pool >= 32) { - addr = IXGBE_VFTE(1); - val = bit1 << (pool - 32); - } else { - addr = IXGBE_VFTE(0); - val = bit1 << pool; - } - - reg = IXGBE_READ_REG(hw, addr); - - if (on) - reg |= val; - else - reg &= ~val; - - IXGBE_WRITE_REG(hw, addr, reg); - - return 0; -} - -static int -ixgbe_set_pool_vlan_filter(struct rte_eth_dev *dev, uint16_t vlan, - uint64_t pool_mask, uint8_t vlan_on) -{ - int ret = 0; - uint16_t pool_idx; - struct ixgbe_hw *hw = - IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); - - if (ixgbe_vmdq_mode_check(hw) < 0) - return -ENOTSUP; - for (pool_idx = 0; pool_idx < ETH_64_POOLS; pool_idx++) { - if (pool_mask & ((uint64_t)(1ULL << pool_idx))) { - ret = hw->mac.ops.set_vfta(hw, vlan, pool_idx, - vlan_on, false); - if (ret < 0) - return ret; - } - } - - return ret; -} - #define IXGBE_MRCTL_VPME 0x01 /* Virtual Pool Mirroring. */ #define IXGBE_MRCTL_UPME 0x02 /* Uplink Port Mirroring. */ #define IXGBE_MRCTL_DPME 0x04 /* Downlink Port Mirroring. */ @@ -4618,8 +5423,8 @@ ixgbe_set_pool_vlan_filter(struct rte_eth_dev *dev, uint16_t vlan, static int ixgbe_mirror_rule_set(struct rte_eth_dev *dev, - struct rte_eth_mirror_conf *mirror_conf, - uint8_t rule_id, uint8_t on) + struct rte_eth_mirror_conf *mirror_conf, + uint8_t rule_id, uint8_t on) { uint32_t mr_ctl, vlvf; uint32_t mp_lsb = 0; @@ -4642,7 +5447,7 @@ ixgbe_mirror_rule_set(struct rte_eth_dev *dev, IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); uint8_t mirror_type = 0; - if (ixgbe_vmdq_mode_check(hw) < 0) + if (ixgbe_vt_check(hw) < 0) return -ENOTSUP; if (rule_id >= IXGBE_MAX_MIRROR_RULES) @@ -4650,22 +5455,28 @@ ixgbe_mirror_rule_set(struct rte_eth_dev *dev, if (IXGBE_INVALID_MIRROR_TYPE(mirror_conf->rule_type)) { PMD_DRV_LOG(ERR, "unsupported mirror type 0x%x.", - mirror_conf->rule_type); + mirror_conf->rule_type); return -EINVAL; } if (mirror_conf->rule_type & ETH_MIRROR_VLAN) { mirror_type |= IXGBE_MRCTL_VLME; - /* Check if vlan id is valid and find conresponding VLAN ID index in VLVF */ + /* Check if vlan id is valid and find conresponding VLAN ID + * index in VLVF + */ for (i = 0; i < IXGBE_VLVF_ENTRIES; i++) { if (mirror_conf->vlan.vlan_mask & (1ULL << i)) { - /* search vlan id related pool vlan filter index */ - reg_index = ixgbe_find_vlvf_slot(hw, - mirror_conf->vlan.vlan_id[i], - false); + /* search vlan id related pool vlan filter + * index + */ + reg_index = ixgbe_find_vlvf_slot( + hw, + mirror_conf->vlan.vlan_id[i], + false); if (reg_index < 0) return -EINVAL; - vlvf = IXGBE_READ_REG(hw, IXGBE_VLVF(reg_index)); + vlvf = IXGBE_READ_REG(hw, + IXGBE_VLVF(reg_index)); if ((vlvf & IXGBE_VLVF_VIEN) && ((vlvf & IXGBE_VLVF_VLANID_MASK) == mirror_conf->vlan.vlan_id[i])) @@ -4695,7 +5506,7 @@ ixgbe_mirror_rule_set(struct rte_eth_dev *dev, } } - /* + /** * if enable pool mirror, write related pool mask register,if disable * pool mirror, clear PFMRVM register */ @@ -4725,8 +5536,9 @@ ixgbe_mirror_rule_set(struct rte_eth_dev *dev, mr_ctl |= mirror_type; mr_ctl &= mirror_rule_mask; mr_ctl |= mirror_conf->dst_pool << dst_pool_offset; - } else + } else { mr_ctl &= ~(mirror_conf->rule_type & mirror_rule_mask); + } mr_info->mr_conf[rule_id].rule_type = mirror_conf->rule_type; mr_info->mr_conf[rule_id].dst_pool = mirror_conf->dst_pool; @@ -4735,13 +5547,13 @@ ixgbe_mirror_rule_set(struct rte_eth_dev *dev, IXGBE_WRITE_REG(hw, IXGBE_MRCTL(rule_id), mr_ctl); /* write pool mirrror control register */ - if (mirror_conf->rule_type == ETH_MIRROR_VIRTUAL_POOL_UP) { + if (mirror_conf->rule_type & ETH_MIRROR_VIRTUAL_POOL_UP) { IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id), mp_lsb); IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id + rule_mr_offset), mp_msb); } /* write VLAN mirrror control register */ - if (mirror_conf->rule_type == ETH_MIRROR_VLAN) { + if (mirror_conf->rule_type & ETH_MIRROR_VLAN) { IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id), mv_lsb); IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id + rule_mr_offset), mv_msb); @@ -4763,11 +5575,14 @@ ixgbe_mirror_rule_reset(struct rte_eth_dev *dev, uint8_t rule_id) struct ixgbe_mirror_info *mr_info = (IXGBE_DEV_PRIVATE_TO_PFDATA(dev->data->dev_private)); - if (ixgbe_vmdq_mode_check(hw) < 0) + if (ixgbe_vt_check(hw) < 0) return -ENOTSUP; + if (rule_id >= IXGBE_MAX_MIRROR_RULES) + return -EINVAL; + memset(&mr_info->mr_conf[rule_id], 0, - sizeof(struct rte_eth_mirror_conf)); + sizeof(struct rte_eth_mirror_conf)); /* clear PFVMCTL register */ IXGBE_WRITE_REG(hw, IXGBE_MRCTL(rule_id), mr_ctl); @@ -4786,16 +5601,21 @@ ixgbe_mirror_rule_reset(struct rte_eth_dev *dev, uint8_t rule_id) static int ixgbevf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) { + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; uint32_t mask; struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + uint32_t vec = IXGBE_MISC_VEC_ID; mask = IXGBE_READ_REG(hw, IXGBE_VTEIMS); - mask |= (1 << IXGBE_MISC_VEC_ID); + if (rte_intr_allow_others(intr_handle)) + vec = IXGBE_RX_VEC_START; + mask |= (1 << vec); RTE_SET_USED(queue_id); IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask); - rte_intr_enable(&dev->pci_dev->intr_handle); + rte_intr_enable(intr_handle); return 0; } @@ -4806,9 +5626,14 @@ ixgbevf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id) uint32_t mask; struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; + uint32_t vec = IXGBE_MISC_VEC_ID; mask = IXGBE_READ_REG(hw, IXGBE_VTEIMS); - mask &= ~(1 << IXGBE_MISC_VEC_ID); + if (rte_intr_allow_others(intr_handle)) + vec = IXGBE_RX_VEC_START; + mask &= ~(1 << vec); RTE_SET_USED(queue_id); IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask); @@ -4818,6 +5643,8 @@ ixgbevf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id) static int ixgbe_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) { + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; uint32_t mask; struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); @@ -4837,7 +5664,7 @@ ixgbe_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) mask &= (1 << (queue_id - 32)); IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask); } - rte_intr_enable(&dev->pci_dev->intr_handle); + rte_intr_enable(intr_handle); return 0; } @@ -4919,7 +5746,8 @@ ixgbe_set_ivar_map(struct ixgbe_hw *hw, int8_t direction, tmp |= (msix_vector << (8 * (queue & 0x3))); IXGBE_WRITE_REG(hw, IXGBE_IVAR(idx), tmp); } else if ((hw->mac.type == ixgbe_mac_82599EB) || - (hw->mac.type == ixgbe_mac_X540)) { + (hw->mac.type == ixgbe_mac_X540) || + (hw->mac.type == ixgbe_mac_X550)) { if (direction == -1) { /* other causes */ idx = ((queue & 1) * 8); @@ -4941,11 +5769,13 @@ ixgbe_set_ivar_map(struct ixgbe_hw *hw, int8_t direction, static void ixgbevf_configure_msix(struct rte_eth_dev *dev) { - struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); uint32_t q_idx; uint32_t vector_idx = IXGBE_MISC_VEC_ID; + uint32_t base = IXGBE_MISC_VEC_ID; /* Configure VF other cause ivar */ ixgbevf_set_ivar_map(hw, -1, 1, vector_idx); @@ -4956,6 +5786,11 @@ ixgbevf_configure_msix(struct rte_eth_dev *dev) if (!rte_intr_dp_is_en(intr_handle)) return; + if (rte_intr_allow_others(intr_handle)) { + base = IXGBE_RX_VEC_START; + vector_idx = IXGBE_RX_VEC_START; + } + /* Configure all RX queues of VF */ for (q_idx = 0; q_idx < dev->data->nb_rx_queues; q_idx++) { /* Force all queue use vector 0, @@ -4963,6 +5798,8 @@ ixgbevf_configure_msix(struct rte_eth_dev *dev) */ ixgbevf_set_ivar_map(hw, 0, q_idx, vector_idx); intr_handle->intr_vec[q_idx] = vector_idx; + if (vector_idx < base + intr_handle->nb_efd - 1) + vector_idx++; } } @@ -4974,7 +5811,8 @@ ixgbevf_configure_msix(struct rte_eth_dev *dev) static void ixgbe_configure_msix(struct rte_eth_dev *dev) { - struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); uint32_t queue_id, base = IXGBE_MISC_VEC_ID; @@ -5025,6 +5863,7 @@ ixgbe_configure_msix(struct rte_eth_dev *dev) break; case ixgbe_mac_82599EB: case ixgbe_mac_X540: + case ixgbe_mac_X550: ixgbe_set_ivar_map(hw, -1, 1, IXGBE_MISC_VEC_ID); break; default: @@ -5042,8 +5881,9 @@ ixgbe_configure_msix(struct rte_eth_dev *dev) IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask); } -static int ixgbe_set_queue_rate_limit(struct rte_eth_dev *dev, - uint16_t queue_idx, uint16_t tx_rate) +int +ixgbe_set_queue_rate_limit(struct rte_eth_dev *dev, + uint16_t queue_idx, uint16_t tx_rate) { struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); uint32_t rf_dec, rf_int; @@ -5089,62 +5929,7 @@ static int ixgbe_set_queue_rate_limit(struct rte_eth_dev *dev, return 0; } -static int ixgbe_set_vf_rate_limit(struct rte_eth_dev *dev, uint16_t vf, - uint16_t tx_rate, uint64_t q_msk) -{ - struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); - struct ixgbe_vf_info *vfinfo = - *(IXGBE_DEV_PRIVATE_TO_P_VFDATA(dev->data->dev_private)); - uint8_t nb_q_per_pool = RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool; - uint32_t queue_stride = - IXGBE_MAX_RX_QUEUE_NUM / RTE_ETH_DEV_SRIOV(dev).active; - uint32_t queue_idx = vf * queue_stride, idx = 0, vf_idx; - uint32_t queue_end = queue_idx + nb_q_per_pool - 1; - uint16_t total_rate = 0; - - if (queue_end >= hw->mac.max_tx_queues) - return -EINVAL; - - if (vfinfo != NULL) { - for (vf_idx = 0; vf_idx < dev->pci_dev->max_vfs; vf_idx++) { - if (vf_idx == vf) - continue; - for (idx = 0; idx < RTE_DIM(vfinfo[vf_idx].tx_rate); - idx++) - total_rate += vfinfo[vf_idx].tx_rate[idx]; - } - } else - return -EINVAL; - - /* Store tx_rate for this vf. */ - for (idx = 0; idx < nb_q_per_pool; idx++) { - if (((uint64_t)0x1 << idx) & q_msk) { - if (vfinfo[vf].tx_rate[idx] != tx_rate) - vfinfo[vf].tx_rate[idx] = tx_rate; - total_rate += tx_rate; - } - } - - if (total_rate > dev->data->dev_link.link_speed) { - /* - * Reset stored TX rate of the VF if it causes exceed - * link speed. - */ - memset(vfinfo[vf].tx_rate, 0, sizeof(vfinfo[vf].tx_rate)); - return -EINVAL; - } - - /* Set RTTBCNRC of each queue/pool for vf X */ - for (; queue_idx <= queue_end; queue_idx++) { - if (0x1 & q_msk) - ixgbe_set_queue_rate_limit(dev, queue_idx, tx_rate); - q_msk = q_msk >> 1; - } - - return 0; -} - -static void +static int ixgbevf_add_mac_addr(struct rte_eth_dev *dev, struct ether_addr *mac_addr, __attribute__((unused)) uint32_t index, __attribute__((unused)) uint32_t pool) @@ -5158,11 +5943,19 @@ ixgbevf_add_mac_addr(struct rte_eth_dev *dev, struct ether_addr *mac_addr, * set of PF resources used to store VF MAC addresses. */ if (memcmp(hw->mac.perm_addr, mac_addr, sizeof(struct ether_addr)) == 0) - return; + return -1; diag = ixgbevf_set_uc_addr_vf(hw, 2, mac_addr->addr_bytes); - if (diag == 0) - return; - PMD_DRV_LOG(ERR, "Unable to add MAC address - diag=%d", diag); + if (diag != 0) + PMD_DRV_LOG(ERR, "Unable to add MAC address " + "%02x:%02x:%02x:%02x:%02x:%02x - diag=%d", + mac_addr->addr_bytes[0], + mac_addr->addr_bytes[1], + mac_addr->addr_bytes[2], + mac_addr->addr_bytes[3], + mac_addr->addr_bytes[4], + mac_addr->addr_bytes[5], + diag); + return diag; } static void @@ -5221,28 +6014,24 @@ ixgbevf_set_default_mac_addr(struct rte_eth_dev *dev, struct ether_addr *addr) hw->mac.ops.set_rar(hw, 0, (void *)addr, 0, 0); } -#define MAC_TYPE_FILTER_SUP(type) do {\ - if ((type) != ixgbe_mac_82599EB && (type) != ixgbe_mac_X540 &&\ - (type) != ixgbe_mac_X550 && (type) != ixgbe_mac_X550EM_x &&\ - (type) != ixgbe_mac_X550EM_a)\ - return -ENOTSUP;\ -} while (0) - -static int +int ixgbe_syn_filter_set(struct rte_eth_dev *dev, struct rte_eth_syn_filter *filter, bool add) { struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct ixgbe_filter_info *filter_info = + IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); + uint32_t syn_info; uint32_t synqf; if (filter->queue >= IXGBE_MAX_RX_QUEUE_NUM) return -EINVAL; - synqf = IXGBE_READ_REG(hw, IXGBE_SYNQF); + syn_info = filter_info->syn_info; if (add) { - if (synqf & IXGBE_SYN_FILTER_ENABLE) + if (syn_info & IXGBE_SYN_FILTER_ENABLE) return -EINVAL; synqf = (uint32_t)(((filter->queue << IXGBE_SYN_FILTER_QUEUE_SHIFT) & IXGBE_SYN_FILTER_QUEUE) | IXGBE_SYN_FILTER_ENABLE); @@ -5252,10 +6041,13 @@ ixgbe_syn_filter_set(struct rte_eth_dev *dev, else synqf &= ~IXGBE_SYN_FILTER_SYNQFP; } else { - if (!(synqf & IXGBE_SYN_FILTER_ENABLE)) + synqf = IXGBE_READ_REG(hw, IXGBE_SYNQF); + if (!(syn_info & IXGBE_SYN_FILTER_ENABLE)) return -ENOENT; synqf &= ~(IXGBE_SYN_FILTER_QUEUE | IXGBE_SYN_FILTER_ENABLE); } + + filter_info->syn_info = synqf; IXGBE_WRITE_REG(hw, IXGBE_SYNQF, synqf); IXGBE_WRITE_FLUSH(hw); return 0; @@ -5311,7 +6103,7 @@ ixgbe_syn_filter_handle(struct rte_eth_dev *dev, (struct rte_eth_syn_filter *)arg); break; default: - PMD_DRV_LOG(ERR, "unsupported operation %u\n", filter_op); + PMD_DRV_LOG(ERR, "unsupported operation %u", filter_op); ret = -EINVAL; break; } @@ -5333,51 +6125,18 @@ convert_protocol_type(uint8_t protocol_value) return IXGBE_FILTER_PROTOCOL_NONE; } -/* - * add a 5tuple filter - * - * @param - * dev: Pointer to struct rte_eth_dev. - * index: the index the filter allocates. - * filter: ponter to the filter that will be added. - * rx_queue: the queue id the filter assigned to. - * - * @return - * - On success, zero. - * - On failure, a negative value. - */ -static int -ixgbe_add_5tuple_filter(struct rte_eth_dev *dev, - struct ixgbe_5tuple_filter *filter) +/* inject a 5-tuple filter to HW */ +static inline void +ixgbe_inject_5tuple_filter(struct rte_eth_dev *dev, + struct ixgbe_5tuple_filter *filter) { struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); - struct ixgbe_filter_info *filter_info = - IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); - int i, idx, shift; + int i; uint32_t ftqf, sdpqf; uint32_t l34timir = 0; uint8_t mask = 0xff; - /* - * look for an unused 5tuple filter index, - * and insert the filter to list. - */ - for (i = 0; i < IXGBE_MAX_FTQF_FILTERS; i++) { - idx = i / (sizeof(uint32_t) * NBBY); - shift = i % (sizeof(uint32_t) * NBBY); - if (!(filter_info->fivetuple_mask[idx] & (1 << shift))) { - filter_info->fivetuple_mask[idx] |= 1 << shift; - filter->index = i; - TAILQ_INSERT_TAIL(&filter_info->fivetuple_list, - filter, - entries); - break; - } - } - if (i >= IXGBE_MAX_FTQF_FILTERS) { - PMD_DRV_LOG(ERR, "5tuple filters are full."); - return -ENOSYS; - } + i = filter->index; sdpqf = (uint32_t)(filter->filter_info.dst_port << IXGBE_SDPQF_DSTPORT_SHIFT); @@ -5410,6 +6169,52 @@ ixgbe_add_5tuple_filter(struct rte_eth_dev *dev, l34timir |= (uint32_t)(filter->queue << IXGBE_L34T_IMIR_QUEUE_SHIFT); IXGBE_WRITE_REG(hw, IXGBE_L34T_IMIR(i), l34timir); +} + +/* + * add a 5tuple filter + * + * @param + * dev: Pointer to struct rte_eth_dev. + * index: the index the filter allocates. + * filter: ponter to the filter that will be added. + * rx_queue: the queue id the filter assigned to. + * + * @return + * - On success, zero. + * - On failure, a negative value. + */ +static int +ixgbe_add_5tuple_filter(struct rte_eth_dev *dev, + struct ixgbe_5tuple_filter *filter) +{ + struct ixgbe_filter_info *filter_info = + IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); + int i, idx, shift; + + /* + * look for an unused 5tuple filter index, + * and insert the filter to list. + */ + for (i = 0; i < IXGBE_MAX_FTQF_FILTERS; i++) { + idx = i / (sizeof(uint32_t) * NBBY); + shift = i % (sizeof(uint32_t) * NBBY); + if (!(filter_info->fivetuple_mask[idx] & (1 << shift))) { + filter_info->fivetuple_mask[idx] |= 1 << shift; + filter->index = i; + TAILQ_INSERT_TAIL(&filter_info->fivetuple_list, + filter, + entries); + break; + } + } + if (i >= IXGBE_MAX_FTQF_FILTERS) { + PMD_DRV_LOG(ERR, "5tuple filters are full."); + return -ENOSYS; + } + + ixgbe_inject_5tuple_filter(dev, filter); + return 0; } @@ -5446,6 +6251,7 @@ ixgbevf_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu) { struct ixgbe_hw *hw; uint32_t max_frame = mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; + struct rte_eth_rxmode *rx_conf = &dev->data->dev_conf.rxmode; hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); @@ -5455,7 +6261,7 @@ ixgbevf_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu) /* refuse mtu that requires the support of scattered packets when this * feature has not been enabled before. */ - if (!dev->data->scattered_rx && + if (!rx_conf->enable_scatter && (max_frame + 2 * IXGBE_VLAN_TAG_SIZE > dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM)) return -EINVAL; @@ -5476,11 +6282,6 @@ ixgbevf_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu) return 0; } -#define MAC_TYPE_FILTER_SUP_EXT(type) do {\ - if ((type) != ixgbe_mac_82599EB && (type) != ixgbe_mac_X540)\ - return -ENOTSUP;\ -} while (0) - static inline struct ixgbe_5tuple_filter * ixgbe_5tuple_filter_lookup(struct ixgbe_5tuple_filter_list *filter_list, struct ixgbe_5tuple_filter_info *key) @@ -5588,7 +6389,7 @@ ntuple_filter_to_5tuple(struct rte_eth_ntuple_filter *filter, * - On success, zero. * - On failure, a negative value. */ -static int +int ixgbe_add_del_ntuple_filter(struct rte_eth_dev *dev, struct rte_eth_ntuple_filter *ntuple_filter, bool add) @@ -5625,7 +6426,7 @@ ixgbe_add_del_ntuple_filter(struct rte_eth_dev *dev, sizeof(struct ixgbe_5tuple_filter), 0); if (filter == NULL) return -ENOMEM; - (void)rte_memcpy(&filter->filter_info, + rte_memcpy(&filter->filter_info, &filter_5tuple, sizeof(struct ixgbe_5tuple_filter_info)); filter->queue = ntuple_filter->queue; @@ -5733,48 +6534,7 @@ ixgbe_ntuple_filter_handle(struct rte_eth_dev *dev, return ret; } -static inline int -ixgbe_ethertype_filter_lookup(struct ixgbe_filter_info *filter_info, - uint16_t ethertype) -{ - int i; - - for (i = 0; i < IXGBE_MAX_ETQF_FILTERS; i++) { - if (filter_info->ethertype_filters[i] == ethertype && - (filter_info->ethertype_mask & (1 << i))) - return i; - } - return -1; -} - -static inline int -ixgbe_ethertype_filter_insert(struct ixgbe_filter_info *filter_info, - uint16_t ethertype) -{ - int i; - - for (i = 0; i < IXGBE_MAX_ETQF_FILTERS; i++) { - if (!(filter_info->ethertype_mask & (1 << i))) { - filter_info->ethertype_mask |= 1 << i; - filter_info->ethertype_filters[i] = ethertype; - return i; - } - } - return -1; -} - -static inline int -ixgbe_ethertype_filter_remove(struct ixgbe_filter_info *filter_info, - uint8_t idx) -{ - if (idx >= IXGBE_MAX_ETQF_FILTERS) - return -1; - filter_info->ethertype_mask &= ~(1 << idx); - filter_info->ethertype_filters[idx] = 0; - return idx; -} - -static int +int ixgbe_add_del_ethertype_filter(struct rte_eth_dev *dev, struct rte_eth_ethertype_filter *filter, bool add) @@ -5785,6 +6545,7 @@ ixgbe_add_del_ethertype_filter(struct rte_eth_dev *dev, uint32_t etqf = 0; uint32_t etqs = 0; int ret; + struct ixgbe_ethertype_filter ethertype_filter; if (filter->queue >= IXGBE_MAX_RX_QUEUE_NUM) return -EINVAL; @@ -5818,18 +6579,23 @@ ixgbe_add_del_ethertype_filter(struct rte_eth_dev *dev, } if (add) { - ret = ixgbe_ethertype_filter_insert(filter_info, - filter->ether_type); - if (ret < 0) { - PMD_DRV_LOG(ERR, "ethertype filters are full."); - return -ENOSYS; - } etqf = IXGBE_ETQF_FILTER_EN; etqf |= (uint32_t)filter->ether_type; etqs |= (uint32_t)((filter->queue << IXGBE_ETQS_RX_QUEUE_SHIFT) & IXGBE_ETQS_RX_QUEUE); etqs |= IXGBE_ETQS_QUEUE_EN; + + ethertype_filter.ethertype = filter->ether_type; + ethertype_filter.etqf = etqf; + ethertype_filter.etqs = etqs; + ethertype_filter.conf = FALSE; + ret = ixgbe_ethertype_filter_insert(filter_info, + ðertype_filter); + if (ret < 0) { + PMD_DRV_LOG(ERR, "ethertype filters are full."); + return -ENOSPC; + } } else { ret = ixgbe_ethertype_filter_remove(filter_info, (uint8_t)ret); if (ret < 0) @@ -5925,7 +6691,7 @@ ixgbe_dev_filter_ctrl(struct rte_eth_dev *dev, enum rte_filter_op filter_op, void *arg) { - int ret = -EINVAL; + int ret = 0; switch (filter_type) { case RTE_ETH_FILTER_NTUPLE: @@ -5943,9 +6709,15 @@ ixgbe_dev_filter_ctrl(struct rte_eth_dev *dev, case RTE_ETH_FILTER_L2_TUNNEL: ret = ixgbe_dev_l2_tunnel_filter_handle(dev, filter_op, arg); break; + case RTE_ETH_FILTER_GENERIC: + if (filter_op != RTE_ETH_FILTER_GET) + return -EINVAL; + *(const void **)arg = &ixgbe_flow_ops; + break; default: PMD_DRV_LOG(WARNING, "Filter type (%d) not supported", filter_type); + ret = -EINVAL; break; } @@ -6494,6 +7266,8 @@ ixgbe_dev_get_dcb_info(struct rte_eth_dev *dev, struct ixgbe_dcb_config *dcb_config = IXGBE_DEV_PRIVATE_TO_DCB_CFG(dev->data->dev_private); struct ixgbe_dcb_tc_config *tc; + struct rte_eth_dcb_tc_queue_mapping *tc_queue; + uint8_t nb_tcs; uint8_t i, j; if (dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_DCB_FLAG) @@ -6501,19 +7275,31 @@ ixgbe_dev_get_dcb_info(struct rte_eth_dev *dev, else dcb_info->nb_tcs = 1; + tc_queue = &dcb_info->tc_queue; + nb_tcs = dcb_info->nb_tcs; + if (dcb_config->vt_mode) { /* vt is enabled*/ struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_dcb_conf; for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) dcb_info->prio_tc[i] = vmdq_rx_conf->dcb_tc[i]; - for (i = 0; i < vmdq_rx_conf->nb_queue_pools; i++) { - for (j = 0; j < dcb_info->nb_tcs; j++) { - dcb_info->tc_queue.tc_rxq[i][j].base = - i * dcb_info->nb_tcs + j; - dcb_info->tc_queue.tc_rxq[i][j].nb_queue = 1; - dcb_info->tc_queue.tc_txq[i][j].base = - i * dcb_info->nb_tcs + j; - dcb_info->tc_queue.tc_txq[i][j].nb_queue = 1; + if (RTE_ETH_DEV_SRIOV(dev).active > 0) { + for (j = 0; j < nb_tcs; j++) { + tc_queue->tc_rxq[0][j].base = j; + tc_queue->tc_rxq[0][j].nb_queue = 1; + tc_queue->tc_txq[0][j].base = j; + tc_queue->tc_txq[0][j].nb_queue = 1; + } + } else { + for (i = 0; i < vmdq_rx_conf->nb_queue_pools; i++) { + for (j = 0; j < nb_tcs; j++) { + tc_queue->tc_rxq[i][j].base = + i * nb_tcs + j; + tc_queue->tc_rxq[i][j].nb_queue = 1; + tc_queue->tc_txq[i][j].base = + i * nb_tcs + j; + tc_queue->tc_txq[i][j].nb_queue = 1; + } } } } else { /* vt is disabled*/ @@ -6593,12 +7379,15 @@ ixgbe_dev_l2_tunnel_eth_type_conf(struct rte_eth_dev *dev, { int ret = 0; struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct ixgbe_l2_tn_info *l2_tn_info = + IXGBE_DEV_PRIVATE_TO_L2_TN_INFO(dev->data->dev_private); if (l2_tunnel == NULL) return -EINVAL; switch (l2_tunnel->l2_tunnel_type) { case RTE_L2_TUNNEL_TYPE_E_TAG: + l2_tn_info->e_tag_ether_type = l2_tunnel->ether_type; ret = ixgbe_update_e_tag_eth_type(hw, l2_tunnel->ether_type); break; default: @@ -6637,9 +7426,12 @@ ixgbe_dev_l2_tunnel_enable(struct rte_eth_dev *dev, { int ret = 0; struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct ixgbe_l2_tn_info *l2_tn_info = + IXGBE_DEV_PRIVATE_TO_L2_TN_INFO(dev->data->dev_private); switch (l2_tunnel_type) { case RTE_L2_TUNNEL_TYPE_E_TAG: + l2_tn_info->e_tag_en = TRUE; ret = ixgbe_e_tag_enable(hw); break; default: @@ -6678,9 +7470,12 @@ ixgbe_dev_l2_tunnel_disable(struct rte_eth_dev *dev, { int ret = 0; struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct ixgbe_l2_tn_info *l2_tn_info = + IXGBE_DEV_PRIVATE_TO_L2_TN_INFO(dev->data->dev_private); switch (l2_tunnel_type) { case RTE_L2_TUNNEL_TYPE_E_TAG: + l2_tn_info->e_tag_en = FALSE; ret = ixgbe_e_tag_disable(hw); break; default: @@ -6769,12 +7564,108 @@ ixgbe_e_tag_filter_add(struct rte_eth_dev *dev, return -EINVAL; } -/* Add l2 tunnel filter */ -static int -ixgbe_dev_l2_tunnel_filter_add(struct rte_eth_dev *dev, - struct rte_eth_l2_tunnel_conf *l2_tunnel) +static inline struct ixgbe_l2_tn_filter * +ixgbe_l2_tn_filter_lookup(struct ixgbe_l2_tn_info *l2_tn_info, + struct ixgbe_l2_tn_key *key) { - int ret = 0; + int ret; + + ret = rte_hash_lookup(l2_tn_info->hash_handle, (const void *)key); + if (ret < 0) + return NULL; + + return l2_tn_info->hash_map[ret]; +} + +static inline int +ixgbe_insert_l2_tn_filter(struct ixgbe_l2_tn_info *l2_tn_info, + struct ixgbe_l2_tn_filter *l2_tn_filter) +{ + int ret; + + ret = rte_hash_add_key(l2_tn_info->hash_handle, + &l2_tn_filter->key); + + if (ret < 0) { + PMD_DRV_LOG(ERR, + "Failed to insert L2 tunnel filter" + " to hash table %d!", + ret); + return ret; + } + + l2_tn_info->hash_map[ret] = l2_tn_filter; + + TAILQ_INSERT_TAIL(&l2_tn_info->l2_tn_list, l2_tn_filter, entries); + + return 0; +} + +static inline int +ixgbe_remove_l2_tn_filter(struct ixgbe_l2_tn_info *l2_tn_info, + struct ixgbe_l2_tn_key *key) +{ + int ret; + struct ixgbe_l2_tn_filter *l2_tn_filter; + + ret = rte_hash_del_key(l2_tn_info->hash_handle, key); + + if (ret < 0) { + PMD_DRV_LOG(ERR, + "No such L2 tunnel filter to delete %d!", + ret); + return ret; + } + + l2_tn_filter = l2_tn_info->hash_map[ret]; + l2_tn_info->hash_map[ret] = NULL; + + TAILQ_REMOVE(&l2_tn_info->l2_tn_list, l2_tn_filter, entries); + rte_free(l2_tn_filter); + + return 0; +} + +/* Add l2 tunnel filter */ +int +ixgbe_dev_l2_tunnel_filter_add(struct rte_eth_dev *dev, + struct rte_eth_l2_tunnel_conf *l2_tunnel, + bool restore) +{ + int ret; + struct ixgbe_l2_tn_info *l2_tn_info = + IXGBE_DEV_PRIVATE_TO_L2_TN_INFO(dev->data->dev_private); + struct ixgbe_l2_tn_key key; + struct ixgbe_l2_tn_filter *node; + + if (!restore) { + key.l2_tn_type = l2_tunnel->l2_tunnel_type; + key.tn_id = l2_tunnel->tunnel_id; + + node = ixgbe_l2_tn_filter_lookup(l2_tn_info, &key); + + if (node) { + PMD_DRV_LOG(ERR, + "The L2 tunnel filter already exists!"); + return -EINVAL; + } + + node = rte_zmalloc("ixgbe_l2_tn", + sizeof(struct ixgbe_l2_tn_filter), + 0); + if (!node) + return -ENOMEM; + + rte_memcpy(&node->key, + &key, + sizeof(struct ixgbe_l2_tn_key)); + node->pool = l2_tunnel->pool; + ret = ixgbe_insert_l2_tn_filter(l2_tn_info, node); + if (ret < 0) { + rte_free(node); + return ret; + } + } switch (l2_tunnel->l2_tunnel_type) { case RTE_L2_TUNNEL_TYPE_E_TAG: @@ -6786,15 +7677,27 @@ ixgbe_dev_l2_tunnel_filter_add(struct rte_eth_dev *dev, break; } + if ((!restore) && (ret < 0)) + (void)ixgbe_remove_l2_tn_filter(l2_tn_info, &key); + return ret; } /* Delete l2 tunnel filter */ -static int +int ixgbe_dev_l2_tunnel_filter_del(struct rte_eth_dev *dev, struct rte_eth_l2_tunnel_conf *l2_tunnel) { - int ret = 0; + int ret; + struct ixgbe_l2_tn_info *l2_tn_info = + IXGBE_DEV_PRIVATE_TO_L2_TN_INFO(dev->data->dev_private); + struct ixgbe_l2_tn_key key; + + key.l2_tn_type = l2_tunnel->l2_tunnel_type; + key.tn_id = l2_tunnel->tunnel_id; + ret = ixgbe_remove_l2_tn_filter(l2_tn_info, &key); + if (ret < 0) + return ret; switch (l2_tunnel->l2_tunnel_type) { case RTE_L2_TUNNEL_TYPE_E_TAG: @@ -6820,7 +7723,7 @@ ixgbe_dev_l2_tunnel_filter_handle(struct rte_eth_dev *dev, enum rte_filter_op filter_op, void *arg) { - int ret = 0; + int ret; if (filter_op == RTE_ETH_FILTER_NOP) return 0; @@ -6835,7 +7738,8 @@ ixgbe_dev_l2_tunnel_filter_handle(struct rte_eth_dev *dev, case RTE_ETH_FILTER_ADD: ret = ixgbe_dev_l2_tunnel_filter_add (dev, - (struct rte_eth_l2_tunnel_conf *)arg); + (struct rte_eth_l2_tunnel_conf *)arg, + FALSE); break; case RTE_ETH_FILTER_DELETE: ret = ixgbe_dev_l2_tunnel_filter_del @@ -6878,10 +7782,13 @@ ixgbe_dev_l2_tunnel_forwarding_enable (struct rte_eth_dev *dev, enum rte_eth_tunnel_type l2_tunnel_type) { + struct ixgbe_l2_tn_info *l2_tn_info = + IXGBE_DEV_PRIVATE_TO_L2_TN_INFO(dev->data->dev_private); int ret = 0; switch (l2_tunnel_type) { case RTE_L2_TUNNEL_TYPE_E_TAG: + l2_tn_info->e_tag_fwd_en = TRUE; ret = ixgbe_e_tag_forwarding_en_dis(dev, 1); break; default: @@ -6899,10 +7806,13 @@ ixgbe_dev_l2_tunnel_forwarding_disable (struct rte_eth_dev *dev, enum rte_eth_tunnel_type l2_tunnel_type) { + struct ixgbe_l2_tn_info *l2_tn_info = + IXGBE_DEV_PRIVATE_TO_L2_TN_INFO(dev->data->dev_private); int ret = 0; switch (l2_tunnel_type) { case RTE_L2_TUNNEL_TYPE_E_TAG: + l2_tn_info->e_tag_fwd_en = FALSE; ret = ixgbe_e_tag_forwarding_en_dis(dev, 0); break; default: @@ -6919,15 +7829,16 @@ ixgbe_e_tag_insertion_en_dis(struct rte_eth_dev *dev, struct rte_eth_l2_tunnel_conf *l2_tunnel, bool en) { + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); int ret = 0; uint32_t vmtir, vmvir; struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); - if (l2_tunnel->vf_id >= dev->pci_dev->max_vfs) { + if (l2_tunnel->vf_id >= pci_dev->max_vfs) { PMD_DRV_LOG(ERR, "VF id %u should be less than %u", l2_tunnel->vf_id, - dev->pci_dev->max_vfs); + pci_dev->max_vfs); return -EINVAL; } @@ -7240,51 +8151,12 @@ ixgbe_dev_udp_tunnel_port_del(struct rte_eth_dev *dev, return ret; } -/* ixgbevf_update_xcast_mode - Update Multicast mode - * @hw: pointer to the HW structure - * @netdev: pointer to net device structure - * @xcast_mode: new multicast mode - * - * Updates the Multicast Mode of VF. - */ -static int ixgbevf_update_xcast_mode(struct ixgbe_hw *hw, - int xcast_mode) -{ - struct ixgbe_mbx_info *mbx = &hw->mbx; - u32 msgbuf[2]; - s32 err; - - switch (hw->api_version) { - case ixgbe_mbox_api_12: - break; - default: - return -EOPNOTSUPP; - } - - msgbuf[0] = IXGBE_VF_UPDATE_XCAST_MODE; - msgbuf[1] = xcast_mode; - - err = mbx->ops.write_posted(hw, msgbuf, 2, 0); - if (err) - return err; - - err = mbx->ops.read_posted(hw, msgbuf, 2, 0); - if (err) - return err; - - msgbuf[0] &= ~IXGBE_VT_MSGTYPE_CTS; - if (msgbuf[0] == (IXGBE_VF_UPDATE_XCAST_MODE | IXGBE_VT_MSGTYPE_NACK)) - return -EPERM; - - return 0; -} - static void ixgbevf_dev_allmulticast_enable(struct rte_eth_dev *dev) { struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); - ixgbevf_update_xcast_mode(hw, IXGBEVF_XCAST_MODE_ALLMULTI); + hw->mac.ops.update_xcast_mode(hw, IXGBEVF_XCAST_MODE_ALLMULTI); } static void @@ -7292,7 +8164,7 @@ ixgbevf_dev_allmulticast_disable(struct rte_eth_dev *dev) { struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); - ixgbevf_update_xcast_mode(hw, IXGBEVF_XCAST_MODE_NONE); + hw->mac.ops.update_xcast_mode(hw, IXGBEVF_XCAST_MODE_MULTI); } static void ixgbevf_mbx_process(struct rte_eth_dev *dev) @@ -7305,7 +8177,8 @@ static void ixgbevf_mbx_process(struct rte_eth_dev *dev) /* PF reset VF event */ if (in_msg == IXGBE_PF_CONTROL_MSG) - _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RESET); + _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RESET, + NULL, NULL); } static int @@ -7347,8 +8220,7 @@ ixgbevf_dev_interrupt_action(struct rte_eth_dev *dev) } static void -ixgbevf_dev_interrupt_handler(__rte_unused struct rte_intr_handle *handle, - void *param) +ixgbevf_dev_interrupt_handler(void *param) { struct rte_eth_dev *dev = (struct rte_eth_dev *)param; @@ -7356,17 +8228,226 @@ ixgbevf_dev_interrupt_handler(__rte_unused struct rte_intr_handle *handle, ixgbevf_dev_interrupt_action(dev); } -static struct rte_driver rte_ixgbe_driver = { - .type = PMD_PDEV, - .init = rte_ixgbe_pmd_init, -}; +/** + * ixgbe_disable_sec_tx_path_generic - Stops the transmit data path + * @hw: pointer to hardware structure + * + * Stops the transmit data path and waits for the HW to internally empty + * the Tx security block + **/ +int ixgbe_disable_sec_tx_path_generic(struct ixgbe_hw *hw) +{ +#define IXGBE_MAX_SECTX_POLL 40 -static struct rte_driver rte_ixgbevf_driver = { - .type = PMD_PDEV, - .init = rte_ixgbevf_pmd_init, -}; + int i; + int sectxreg; -PMD_REGISTER_DRIVER(rte_ixgbe_driver, ixgbe); -DRIVER_REGISTER_PCI_TABLE(ixgbe, pci_id_ixgbe_map); -PMD_REGISTER_DRIVER(rte_ixgbevf_driver, ixgbevf); -DRIVER_REGISTER_PCI_TABLE(ixgbevf, pci_id_ixgbevf_map); + sectxreg = IXGBE_READ_REG(hw, IXGBE_SECTXCTRL); + sectxreg |= IXGBE_SECTXCTRL_TX_DIS; + IXGBE_WRITE_REG(hw, IXGBE_SECTXCTRL, sectxreg); + for (i = 0; i < IXGBE_MAX_SECTX_POLL; i++) { + sectxreg = IXGBE_READ_REG(hw, IXGBE_SECTXSTAT); + if (sectxreg & IXGBE_SECTXSTAT_SECTX_RDY) + break; + /* Use interrupt-safe sleep just in case */ + usec_delay(1000); + } + + /* For informational purposes only */ + if (i >= IXGBE_MAX_SECTX_POLL) + PMD_DRV_LOG(DEBUG, "Tx unit being enabled before security " + "path fully disabled. Continuing with init."); + + return IXGBE_SUCCESS; +} + +/** + * ixgbe_enable_sec_tx_path_generic - Enables the transmit data path + * @hw: pointer to hardware structure + * + * Enables the transmit data path. + **/ +int ixgbe_enable_sec_tx_path_generic(struct ixgbe_hw *hw) +{ + uint32_t sectxreg; + + sectxreg = IXGBE_READ_REG(hw, IXGBE_SECTXCTRL); + sectxreg &= ~IXGBE_SECTXCTRL_TX_DIS; + IXGBE_WRITE_REG(hw, IXGBE_SECTXCTRL, sectxreg); + IXGBE_WRITE_FLUSH(hw); + + return IXGBE_SUCCESS; +} + +/* restore n-tuple filter */ +static inline void +ixgbe_ntuple_filter_restore(struct rte_eth_dev *dev) +{ + struct ixgbe_filter_info *filter_info = + IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); + struct ixgbe_5tuple_filter *node; + + TAILQ_FOREACH(node, &filter_info->fivetuple_list, entries) { + ixgbe_inject_5tuple_filter(dev, node); + } +} + +/* restore ethernet type filter */ +static inline void +ixgbe_ethertype_filter_restore(struct rte_eth_dev *dev) +{ + struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct ixgbe_filter_info *filter_info = + IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); + int i; + + for (i = 0; i < IXGBE_MAX_ETQF_FILTERS; i++) { + if (filter_info->ethertype_mask & (1 << i)) { + IXGBE_WRITE_REG(hw, IXGBE_ETQF(i), + filter_info->ethertype_filters[i].etqf); + IXGBE_WRITE_REG(hw, IXGBE_ETQS(i), + filter_info->ethertype_filters[i].etqs); + IXGBE_WRITE_FLUSH(hw); + } + } +} + +/* restore SYN filter */ +static inline void +ixgbe_syn_filter_restore(struct rte_eth_dev *dev) +{ + struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct ixgbe_filter_info *filter_info = + IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); + uint32_t synqf; + + synqf = filter_info->syn_info; + + if (synqf & IXGBE_SYN_FILTER_ENABLE) { + IXGBE_WRITE_REG(hw, IXGBE_SYNQF, synqf); + IXGBE_WRITE_FLUSH(hw); + } +} + +/* restore L2 tunnel filter */ +static inline void +ixgbe_l2_tn_filter_restore(struct rte_eth_dev *dev) +{ + struct ixgbe_l2_tn_info *l2_tn_info = + IXGBE_DEV_PRIVATE_TO_L2_TN_INFO(dev->data->dev_private); + struct ixgbe_l2_tn_filter *node; + struct rte_eth_l2_tunnel_conf l2_tn_conf; + + TAILQ_FOREACH(node, &l2_tn_info->l2_tn_list, entries) { + l2_tn_conf.l2_tunnel_type = node->key.l2_tn_type; + l2_tn_conf.tunnel_id = node->key.tn_id; + l2_tn_conf.pool = node->pool; + (void)ixgbe_dev_l2_tunnel_filter_add(dev, &l2_tn_conf, TRUE); + } +} + +static int +ixgbe_filter_restore(struct rte_eth_dev *dev) +{ + ixgbe_ntuple_filter_restore(dev); + ixgbe_ethertype_filter_restore(dev); + ixgbe_syn_filter_restore(dev); + ixgbe_fdir_filter_restore(dev); + ixgbe_l2_tn_filter_restore(dev); + + return 0; +} + +static void +ixgbe_l2_tunnel_conf(struct rte_eth_dev *dev) +{ + struct ixgbe_l2_tn_info *l2_tn_info = + IXGBE_DEV_PRIVATE_TO_L2_TN_INFO(dev->data->dev_private); + struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + + if (l2_tn_info->e_tag_en) + (void)ixgbe_e_tag_enable(hw); + + if (l2_tn_info->e_tag_fwd_en) + (void)ixgbe_e_tag_forwarding_en_dis(dev, 1); + + (void)ixgbe_update_e_tag_eth_type(hw, l2_tn_info->e_tag_ether_type); +} + +/* remove all the n-tuple filters */ +void +ixgbe_clear_all_ntuple_filter(struct rte_eth_dev *dev) +{ + struct ixgbe_filter_info *filter_info = + IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); + struct ixgbe_5tuple_filter *p_5tuple; + + while ((p_5tuple = TAILQ_FIRST(&filter_info->fivetuple_list))) + ixgbe_remove_5tuple_filter(dev, p_5tuple); +} + +/* remove all the ether type filters */ +void +ixgbe_clear_all_ethertype_filter(struct rte_eth_dev *dev) +{ + struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct ixgbe_filter_info *filter_info = + IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); + int i; + + for (i = 0; i < IXGBE_MAX_ETQF_FILTERS; i++) { + if (filter_info->ethertype_mask & (1 << i) && + !filter_info->ethertype_filters[i].conf) { + (void)ixgbe_ethertype_filter_remove(filter_info, + (uint8_t)i); + IXGBE_WRITE_REG(hw, IXGBE_ETQF(i), 0); + IXGBE_WRITE_REG(hw, IXGBE_ETQS(i), 0); + IXGBE_WRITE_FLUSH(hw); + } + } +} + +/* remove the SYN filter */ +void +ixgbe_clear_syn_filter(struct rte_eth_dev *dev) +{ + struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct ixgbe_filter_info *filter_info = + IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); + + if (filter_info->syn_info & IXGBE_SYN_FILTER_ENABLE) { + filter_info->syn_info = 0; + + IXGBE_WRITE_REG(hw, IXGBE_SYNQF, 0); + IXGBE_WRITE_FLUSH(hw); + } +} + +/* remove all the L2 tunnel filters */ +int +ixgbe_clear_all_l2_tn_filter(struct rte_eth_dev *dev) +{ + struct ixgbe_l2_tn_info *l2_tn_info = + IXGBE_DEV_PRIVATE_TO_L2_TN_INFO(dev->data->dev_private); + struct ixgbe_l2_tn_filter *l2_tn_filter; + struct rte_eth_l2_tunnel_conf l2_tn_conf; + int ret = 0; + + while ((l2_tn_filter = TAILQ_FIRST(&l2_tn_info->l2_tn_list))) { + l2_tn_conf.l2_tunnel_type = l2_tn_filter->key.l2_tn_type; + l2_tn_conf.tunnel_id = l2_tn_filter->key.tn_id; + l2_tn_conf.pool = l2_tn_filter->pool; + ret = ixgbe_dev_l2_tunnel_filter_del(dev, &l2_tn_conf); + if (ret < 0) + return ret; + } + + return 0; +} + +RTE_PMD_REGISTER_PCI(net_ixgbe, rte_ixgbe_pmd); +RTE_PMD_REGISTER_PCI_TABLE(net_ixgbe, pci_id_ixgbe_map); +RTE_PMD_REGISTER_KMOD_DEP(net_ixgbe, "* igb_uio | uio_pci_generic | vfio-pci"); +RTE_PMD_REGISTER_PCI(net_ixgbe_vf, rte_ixgbevf_pmd); +RTE_PMD_REGISTER_PCI_TABLE(net_ixgbe_vf, pci_id_ixgbevf_map); +RTE_PMD_REGISTER_KMOD_DEP(net_ixgbe_vf, "* igb_uio | vfio-pci"); diff --git a/dpdk/drivers/net/ixgbe/ixgbe_ethdev.h b/dpdk/drivers/net/ixgbe/ixgbe_ethdev.h index 4ff6338e..51ddcfd4 100644 --- a/dpdk/drivers/net/ixgbe/ixgbe_ethdev.h +++ b/dpdk/drivers/net/ixgbe/ixgbe_ethdev.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -33,16 +33,26 @@ #ifndef _IXGBE_ETHDEV_H_ #define _IXGBE_ETHDEV_H_ +#include "base/ixgbe_type.h" #include "base/ixgbe_dcb.h" #include "base/ixgbe_dcb_82599.h" #include "base/ixgbe_dcb_82598.h" #include "ixgbe_bypass.h" +#ifdef RTE_LIBRTE_SECURITY +#include "ixgbe_ipsec.h" +#endif #include +#include +#include +#include +#include /* need update link, bit flag */ #define IXGBE_FLAG_NEED_LINK_UPDATE (uint32_t)(1 << 0) #define IXGBE_FLAG_MAILBOX (uint32_t)(1 << 1) #define IXGBE_FLAG_PHY_INTERRUPT (uint32_t)(1 << 2) +#define IXGBE_FLAG_MACSEC (uint32_t)(1 << 3) +#define IXGBE_FLAG_NEED_LINK_CONFIG (uint32_t)(1 << 4) /* * Defines that were not part of ixgbe_type.h as they are not used by the @@ -69,7 +79,7 @@ #endif #define IXGBE_HWSTRIP_BITMAP_SIZE (IXGBE_MAX_RX_QUEUE_NUM / (sizeof(uint32_t) * NBBY)) -/* EITR Inteval is in 2048ns uinits for 1G and 10G link */ +/* EITR Interval is in 2048ns uinits for 1G and 10G link */ #define IXGBE_EITR_INTERVAL_UNIT_NS 2048 #define IXGBE_EITR_ITR_INT_SHIFT 3 #define IXGBE_EITR_INTERVAL_US(us) \ @@ -130,10 +140,35 @@ #define IXGBE_MISC_VEC_ID RTE_INTR_VEC_ZERO_OFFSET #define IXGBE_RX_VEC_START RTE_INTR_VEC_RXTX_OFFSET +#define IXGBE_SECTX_MINSECIFG_MASK 0x0000000F + +#define IXGBE_MACSEC_PNTHRSH 0xFFFFFE00 + +#define IXGBE_MAX_FDIR_FILTER_NUM (1024 * 32) +#define IXGBE_MAX_L2_TN_FILTER_NUM 128 + +#define MAC_TYPE_FILTER_SUP_EXT(type) do {\ + if ((type) != ixgbe_mac_82599EB && (type) != ixgbe_mac_X540)\ + return -ENOTSUP;\ +} while (0) + +#define MAC_TYPE_FILTER_SUP(type) do {\ + if ((type) != ixgbe_mac_82599EB && (type) != ixgbe_mac_X540 &&\ + (type) != ixgbe_mac_X550 && (type) != ixgbe_mac_X550EM_x &&\ + (type) != ixgbe_mac_X550EM_a)\ + return -ENOTSUP;\ +} while (0) + +/* Link speed for X550 auto negotiation */ +#define IXGBE_LINK_SPEED_X550_AUTONEG (IXGBE_LINK_SPEED_100_FULL | \ + IXGBE_LINK_SPEED_1GB_FULL | \ + IXGBE_LINK_SPEED_2_5GB_FULL | \ + IXGBE_LINK_SPEED_5GB_FULL | \ + IXGBE_LINK_SPEED_10GB_FULL) + /* * Information about the fdir mode. */ - struct ixgbe_hw_fdir_mask { uint16_t vlan_tci_mask; uint32_t src_ipv4_mask; @@ -148,6 +183,29 @@ struct ixgbe_hw_fdir_mask { uint8_t tunnel_type_mask; }; +struct ixgbe_fdir_filter { + TAILQ_ENTRY(ixgbe_fdir_filter) entries; + union ixgbe_atr_input ixgbe_fdir; /* key of fdir filter*/ + uint32_t fdirflags; /* drop or forward */ + uint32_t fdirhash; /* hash value for fdir */ + uint8_t queue; /* assigned rx queue */ +}; + +/* list of fdir filters */ +TAILQ_HEAD(ixgbe_fdir_filter_list, ixgbe_fdir_filter); + +struct ixgbe_fdir_rule { + struct ixgbe_hw_fdir_mask mask; + union ixgbe_atr_input ixgbe_fdir; /* key of fdir filter*/ + bool b_spec; /* If TRUE, ixgbe_fdir, fdirflags, queue have meaning. */ + bool b_mask; /* If TRUE, mask has meaning. */ + enum rte_fdir_mode mode; /* IP, MAC VLAN, Tunnel */ + uint32_t fdirflags; /* drop or forward */ + uint32_t soft_id; /* an unique value for this rule */ + uint8_t queue; /* assigned rx queue */ + uint8_t flex_bytes_offset; +}; + struct ixgbe_hw_fdir_info { struct ixgbe_hw_fdir_mask mask; uint8_t flex_bytes_offset; @@ -159,12 +217,19 @@ struct ixgbe_hw_fdir_info { uint64_t remove; uint64_t f_add; uint64_t f_remove; + struct ixgbe_fdir_filter_list fdir_list; /* filter list*/ + /* store the pointers of the filters, index is the hash value. */ + struct ixgbe_fdir_filter **hash_map; + struct rte_hash *hash_handle; /* cuckoo hash handler */ + bool mask_added; /* If already got mask from consistent filter */ }; /* structure for interrupt relative data */ struct ixgbe_interrupt { uint32_t flags; uint32_t mask; + /*to save original mask during delayed handler */ + uint32_t mask_original; }; struct ixgbe_stat_mapping_registers { @@ -252,16 +317,155 @@ struct ixgbe_5tuple_filter { (RTE_ALIGN(IXGBE_MAX_FTQF_FILTERS, (sizeof(uint32_t) * NBBY)) / \ (sizeof(uint32_t) * NBBY)) +struct ixgbe_ethertype_filter { + uint16_t ethertype; + uint32_t etqf; + uint32_t etqs; + /** + * If this filter is added by configuration, + * it should not be removed. + */ + bool conf; +}; + /* * Structure to store filters' info. */ struct ixgbe_filter_info { uint8_t ethertype_mask; /* Bit mask for every used ethertype filter */ /* store used ethertype filters*/ - uint16_t ethertype_filters[IXGBE_MAX_ETQF_FILTERS]; + struct ixgbe_ethertype_filter ethertype_filters[IXGBE_MAX_ETQF_FILTERS]; /* Bit mask for every used 5tuple filter */ uint32_t fivetuple_mask[IXGBE_5TUPLE_ARRAY_SIZE]; struct ixgbe_5tuple_filter_list fivetuple_list; + /* store the SYN filter info */ + uint32_t syn_info; +}; + +struct ixgbe_l2_tn_key { + enum rte_eth_tunnel_type l2_tn_type; + uint32_t tn_id; +}; + +struct ixgbe_l2_tn_filter { + TAILQ_ENTRY(ixgbe_l2_tn_filter) entries; + struct ixgbe_l2_tn_key key; + uint32_t pool; +}; + +TAILQ_HEAD(ixgbe_l2_tn_filter_list, ixgbe_l2_tn_filter); + +struct ixgbe_l2_tn_info { + struct ixgbe_l2_tn_filter_list l2_tn_list; + struct ixgbe_l2_tn_filter **hash_map; + struct rte_hash *hash_handle; + bool e_tag_en; /* e-tag enabled */ + bool e_tag_fwd_en; /* e-tag based forwarding enabled */ + bool e_tag_ether_type; /* ether type for e-tag */ +}; + +struct rte_flow { + enum rte_filter_type filter_type; + void *rule; +}; + +/* + * Statistics counters collected by the MACsec + */ +struct ixgbe_macsec_stats { + /* TX port statistics */ + uint64_t out_pkts_untagged; + uint64_t out_pkts_encrypted; + uint64_t out_pkts_protected; + uint64_t out_octets_encrypted; + uint64_t out_octets_protected; + + /* RX port statistics */ + uint64_t in_pkts_untagged; + uint64_t in_pkts_badtag; + uint64_t in_pkts_nosci; + uint64_t in_pkts_unknownsci; + uint64_t in_octets_decrypted; + uint64_t in_octets_validated; + + /* RX SC statistics */ + uint64_t in_pkts_unchecked; + uint64_t in_pkts_delayed; + uint64_t in_pkts_late; + + /* RX SA statistics */ + uint64_t in_pkts_ok; + uint64_t in_pkts_invalid; + uint64_t in_pkts_notvalid; + uint64_t in_pkts_unusedsa; + uint64_t in_pkts_notusingsa; +}; + +/* The configuration of bandwidth */ +struct ixgbe_bw_conf { + uint8_t tc_num; /* Number of TCs. */ +}; + +/* Struct to store Traffic Manager shaper profile. */ +struct ixgbe_tm_shaper_profile { + TAILQ_ENTRY(ixgbe_tm_shaper_profile) node; + uint32_t shaper_profile_id; + uint32_t reference_count; + struct rte_tm_shaper_params profile; +}; + +TAILQ_HEAD(ixgbe_shaper_profile_list, ixgbe_tm_shaper_profile); + +/* node type of Traffic Manager */ +enum ixgbe_tm_node_type { + IXGBE_TM_NODE_TYPE_PORT, + IXGBE_TM_NODE_TYPE_TC, + IXGBE_TM_NODE_TYPE_QUEUE, + IXGBE_TM_NODE_TYPE_MAX, +}; + +/* Struct to store Traffic Manager node configuration. */ +struct ixgbe_tm_node { + TAILQ_ENTRY(ixgbe_tm_node) node; + uint32_t id; + uint32_t priority; + uint32_t weight; + uint32_t reference_count; + uint16_t no; + struct ixgbe_tm_node *parent; + struct ixgbe_tm_shaper_profile *shaper_profile; + struct rte_tm_node_params params; +}; + +TAILQ_HEAD(ixgbe_tm_node_list, ixgbe_tm_node); + +/* The configuration of Traffic Manager */ +struct ixgbe_tm_conf { + struct ixgbe_shaper_profile_list shaper_profile_list; + struct ixgbe_tm_node *root; /* root node - port */ + struct ixgbe_tm_node_list tc_list; /* node list for all the TCs */ + struct ixgbe_tm_node_list queue_list; /* node list for all the queues */ + /** + * The number of added TC nodes. + * It should be no more than the TC number of this port. + */ + uint32_t nb_tc_node; + /** + * The number of added queue nodes. + * It should be no more than the queue number of this port. + */ + uint32_t nb_queue_node; + /** + * This flag is used to check if APP can change the TM node + * configuration. + * When it's true, means the configuration is applied to HW, + * APP should not change the configuration. + * As we don't support on-the-fly configuration, when starting + * the port, APP should call the hierarchy_commit API to set this + * flag to true. When stopping the port, this flag should be set + * to false. + */ + bool committed; }; /* @@ -270,6 +474,7 @@ struct ixgbe_filter_info { struct ixgbe_adapter { struct ixgbe_hw hw; struct ixgbe_hw_stats stats; + struct ixgbe_macsec_stats macsec_stats; struct ixgbe_hw_fdir_info fdir; struct ixgbe_interrupt intr; struct ixgbe_stat_mapping_registers stat_mappings; @@ -279,16 +484,21 @@ struct ixgbe_adapter { struct ixgbe_mirror_info mr_data; struct ixgbe_vf_info *vfdata; struct ixgbe_uta_info uta_info; -#ifdef RTE_NIC_BYPASS +#ifdef RTE_LIBRTE_IXGBE_BYPASS struct ixgbe_bypass_info bps; -#endif /* RTE_NIC_BYPASS */ +#endif /* RTE_LIBRTE_IXGBE_BYPASS */ struct ixgbe_filter_info filter; - + struct ixgbe_l2_tn_info l2_tn; + struct ixgbe_bw_conf bw_conf; +#ifdef RTE_LIBRTE_SECURITY + struct ixgbe_ipsec ipsec; +#endif bool rx_bulk_alloc_allowed; bool rx_vec_allowed; struct rte_timecounter systime_tc; struct rte_timecounter rx_tstamp_tc; struct rte_timecounter tx_tstamp_tc; + struct ixgbe_tm_conf tm_conf; }; #define IXGBE_DEV_PRIVATE_TO_HW(adapter)\ @@ -297,6 +507,9 @@ struct ixgbe_adapter { #define IXGBE_DEV_PRIVATE_TO_STATS(adapter) \ (&((struct ixgbe_adapter *)adapter)->stats) +#define IXGBE_DEV_PRIVATE_TO_MACSEC_STATS(adapter) \ + (&((struct ixgbe_adapter *)adapter)->macsec_stats) + #define IXGBE_DEV_PRIVATE_TO_INTR(adapter) \ (&((struct ixgbe_adapter *)adapter)->intr) @@ -327,6 +540,18 @@ struct ixgbe_adapter { #define IXGBE_DEV_PRIVATE_TO_FILTER_INFO(adapter) \ (&((struct ixgbe_adapter *)adapter)->filter) +#define IXGBE_DEV_PRIVATE_TO_L2_TN_INFO(adapter) \ + (&((struct ixgbe_adapter *)adapter)->l2_tn) + +#define IXGBE_DEV_PRIVATE_TO_BW_CONF(adapter) \ + (&((struct ixgbe_adapter *)adapter)->bw_conf) + +#define IXGBE_DEV_PRIVATE_TO_TM_CONF(adapter) \ + (&((struct ixgbe_adapter *)adapter)->tm_conf) + +#define IXGBE_DEV_PRIVATE_TO_IPSEC(adapter)\ + (&((struct ixgbe_adapter *)adapter)->ipsec) + /* * RX/TX function prototypes */ @@ -351,7 +576,9 @@ uint32_t ixgbe_dev_rx_queue_count(struct rte_eth_dev *dev, uint16_t rx_queue_id); int ixgbe_dev_rx_descriptor_done(void *rx_queue, uint16_t offset); -int ixgbevf_dev_rx_descriptor_done(void *rx_queue, uint16_t offset); + +int ixgbe_dev_rx_descriptor_status(void *rx_queue, uint16_t offset); +int ixgbe_dev_tx_descriptor_status(void *tx_queue, uint16_t offset); int ixgbe_dev_rx_init(struct rte_eth_dev *dev); @@ -396,6 +623,9 @@ uint16_t ixgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t ixgbe_xmit_pkts_simple(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts); +uint16_t ixgbe_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts); + int ixgbe_dev_rss_hash_update(struct rte_eth_dev *dev, struct rte_eth_rss_conf *rss_conf); @@ -412,10 +642,34 @@ uint32_t ixgbe_rssrk_reg_get(enum ixgbe_mac_type mac_type, uint8_t i); bool ixgbe_rss_update_sp(enum ixgbe_mac_type mac_type); +int ixgbe_add_del_ntuple_filter(struct rte_eth_dev *dev, + struct rte_eth_ntuple_filter *filter, + bool add); +int ixgbe_add_del_ethertype_filter(struct rte_eth_dev *dev, + struct rte_eth_ethertype_filter *filter, + bool add); +int ixgbe_syn_filter_set(struct rte_eth_dev *dev, + struct rte_eth_syn_filter *filter, + bool add); +int +ixgbe_dev_l2_tunnel_filter_add(struct rte_eth_dev *dev, + struct rte_eth_l2_tunnel_conf *l2_tunnel, + bool restore); +int +ixgbe_dev_l2_tunnel_filter_del(struct rte_eth_dev *dev, + struct rte_eth_l2_tunnel_conf *l2_tunnel); +void ixgbe_filterlist_init(void); +void ixgbe_filterlist_flush(void); /* * Flow director function prototypes */ int ixgbe_fdir_configure(struct rte_eth_dev *dev); +int ixgbe_fdir_set_input_mask(struct rte_eth_dev *dev); +int ixgbe_fdir_set_flexbytes_offset(struct rte_eth_dev *dev, + uint16_t offset); +int ixgbe_fdir_filter_program(struct rte_eth_dev *dev, + struct ixgbe_fdir_rule *rule, + bool del, bool update); void ixgbe_configure_dcb(struct rte_eth_dev *dev); @@ -442,4 +696,79 @@ uint32_t ixgbe_convert_vm_rx_mask_to_val(uint16_t rx_mask, uint32_t orig_val); int ixgbe_fdir_ctrl_func(struct rte_eth_dev *dev, enum rte_filter_op filter_op, void *arg); +void ixgbe_fdir_filter_restore(struct rte_eth_dev *dev); +int ixgbe_clear_all_fdir_filter(struct rte_eth_dev *dev); + +extern const struct rte_flow_ops ixgbe_flow_ops; + +void ixgbe_clear_all_ethertype_filter(struct rte_eth_dev *dev); +void ixgbe_clear_all_ntuple_filter(struct rte_eth_dev *dev); +void ixgbe_clear_syn_filter(struct rte_eth_dev *dev); +int ixgbe_clear_all_l2_tn_filter(struct rte_eth_dev *dev); + +int ixgbe_disable_sec_tx_path_generic(struct ixgbe_hw *hw); + +int ixgbe_enable_sec_tx_path_generic(struct ixgbe_hw *hw); + +int ixgbe_vt_check(struct ixgbe_hw *hw); +int ixgbe_set_vf_rate_limit(struct rte_eth_dev *dev, uint16_t vf, + uint16_t tx_rate, uint64_t q_msk); +bool is_ixgbe_supported(struct rte_eth_dev *dev); +int ixgbe_tm_ops_get(struct rte_eth_dev *dev, void *ops); +void ixgbe_tm_conf_init(struct rte_eth_dev *dev); +void ixgbe_tm_conf_uninit(struct rte_eth_dev *dev); +int ixgbe_set_queue_rate_limit(struct rte_eth_dev *dev, uint16_t queue_idx, + uint16_t tx_rate); + +static inline int +ixgbe_ethertype_filter_lookup(struct ixgbe_filter_info *filter_info, + uint16_t ethertype) +{ + int i; + + for (i = 0; i < IXGBE_MAX_ETQF_FILTERS; i++) { + if (filter_info->ethertype_filters[i].ethertype == ethertype && + (filter_info->ethertype_mask & (1 << i))) + return i; + } + return -1; +} + +static inline int +ixgbe_ethertype_filter_insert(struct ixgbe_filter_info *filter_info, + struct ixgbe_ethertype_filter *ethertype_filter) +{ + int i; + + for (i = 0; i < IXGBE_MAX_ETQF_FILTERS; i++) { + if (!(filter_info->ethertype_mask & (1 << i))) { + filter_info->ethertype_mask |= 1 << i; + filter_info->ethertype_filters[i].ethertype = + ethertype_filter->ethertype; + filter_info->ethertype_filters[i].etqf = + ethertype_filter->etqf; + filter_info->ethertype_filters[i].etqs = + ethertype_filter->etqs; + filter_info->ethertype_filters[i].conf = + ethertype_filter->conf; + return i; + } + } + return -1; +} + +static inline int +ixgbe_ethertype_filter_remove(struct ixgbe_filter_info *filter_info, + uint8_t idx) +{ + if (idx >= IXGBE_MAX_ETQF_FILTERS) + return -1; + filter_info->ethertype_mask &= ~(1 << idx); + filter_info->ethertype_filters[idx].ethertype = 0; + filter_info->ethertype_filters[idx].etqf = 0; + filter_info->ethertype_filters[idx].etqs = 0; + filter_info->ethertype_filters[idx].etqs = FALSE; + return idx; +} + #endif /* _IXGBE_ETHDEV_H_ */ diff --git a/dpdk/drivers/net/ixgbe/ixgbe_fdir.c b/dpdk/drivers/net/ixgbe/ixgbe_fdir.c index 4b81ee37..c1176472 100644 --- a/dpdk/drivers/net/ixgbe/ixgbe_fdir.c +++ b/dpdk/drivers/net/ixgbe/ixgbe_fdir.c @@ -43,6 +43,7 @@ #include #include #include +#include #include "ixgbe_logs.h" #include "base/ixgbe_api.h" @@ -111,10 +112,8 @@ static int fdir_erase_filter_82599(struct ixgbe_hw *hw, uint32_t fdirhash); static int fdir_set_input_mask(struct rte_eth_dev *dev, const struct rte_eth_fdir_masks *input_mask); -static int fdir_set_input_mask_82599(struct rte_eth_dev *dev, - const struct rte_eth_fdir_masks *input_mask); -static int fdir_set_input_mask_x550(struct rte_eth_dev *dev, - const struct rte_eth_fdir_masks *input_mask); +static int fdir_set_input_mask_82599(struct rte_eth_dev *dev); +static int fdir_set_input_mask_x550(struct rte_eth_dev *dev); static int ixgbe_set_fdir_flex_conf(struct rte_eth_dev *dev, const struct rte_eth_fdir_flex_conf *conf, uint32_t *fdirctrl); static int fdir_enable_82599(struct ixgbe_hw *hw, uint32_t fdirctrl); @@ -294,8 +293,7 @@ reverse_fdir_bitmasks(uint16_t hi_dword, uint16_t lo_dword) * but makes use of the rte_fdir_masks structure to see which bits to set. */ static int -fdir_set_input_mask_82599(struct rte_eth_dev *dev, - const struct rte_eth_fdir_masks *input_mask) +fdir_set_input_mask_82599(struct rte_eth_dev *dev) { struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct ixgbe_hw_fdir_info *info = @@ -304,11 +302,9 @@ fdir_set_input_mask_82599(struct rte_eth_dev *dev, * mask VM pool and DIPv6 since there are currently not supported * mask FLEX byte, it will be set in flex_conf */ - uint32_t fdirm = IXGBE_FDIRM_POOL | IXGBE_FDIRM_DIPv6 | IXGBE_FDIRM_FLEX; + uint32_t fdirm = IXGBE_FDIRM_POOL | IXGBE_FDIRM_DIPv6; uint32_t fdirtcpm; /* TCP source and destination port masks. */ uint32_t fdiripv6m; /* IPv6 source and destination masks. */ - uint16_t dst_ipv6m = 0; - uint16_t src_ipv6m = 0; volatile uint32_t *reg; PMD_INIT_FUNC_TRACE(); @@ -319,31 +315,34 @@ fdir_set_input_mask_82599(struct rte_eth_dev *dev, * a VLAN of 0 is unspecified, so mask that out as well. L4type * cannot be masked out in this implementation. */ - if (input_mask->dst_port_mask == 0 && input_mask->src_port_mask == 0) + if (info->mask.dst_port_mask == 0 && info->mask.src_port_mask == 0) /* use the L4 protocol mask for raw IPv4/IPv6 traffic */ fdirm |= IXGBE_FDIRM_L4P; - if (input_mask->vlan_tci_mask == rte_cpu_to_be_16(0x0FFF)) + if (info->mask.vlan_tci_mask == rte_cpu_to_be_16(0x0FFF)) /* mask VLAN Priority */ fdirm |= IXGBE_FDIRM_VLANP; - else if (input_mask->vlan_tci_mask == rte_cpu_to_be_16(0xE000)) + else if (info->mask.vlan_tci_mask == rte_cpu_to_be_16(0xE000)) /* mask VLAN ID */ fdirm |= IXGBE_FDIRM_VLANID; - else if (input_mask->vlan_tci_mask == 0) + else if (info->mask.vlan_tci_mask == 0) /* mask VLAN ID and Priority */ fdirm |= IXGBE_FDIRM_VLANID | IXGBE_FDIRM_VLANP; - else if (input_mask->vlan_tci_mask != rte_cpu_to_be_16(0xEFFF)) { + else if (info->mask.vlan_tci_mask != rte_cpu_to_be_16(0xEFFF)) { PMD_INIT_LOG(ERR, "invalid vlan_tci_mask"); return -EINVAL; } - info->mask.vlan_tci_mask = input_mask->vlan_tci_mask; + + /* flex byte mask */ + if (info->mask.flex_bytes_mask == 0) + fdirm |= IXGBE_FDIRM_FLEX; IXGBE_WRITE_REG(hw, IXGBE_FDIRM, fdirm); /* store the TCP/UDP port masks, bit reversed from port layout */ fdirtcpm = reverse_fdir_bitmasks( - rte_be_to_cpu_16(input_mask->dst_port_mask), - rte_be_to_cpu_16(input_mask->src_port_mask)); + rte_be_to_cpu_16(info->mask.dst_port_mask), + rte_be_to_cpu_16(info->mask.src_port_mask)); /* write all the same so that UDP, TCP and SCTP use the same mask * (little-endian) @@ -351,30 +350,23 @@ fdir_set_input_mask_82599(struct rte_eth_dev *dev, IXGBE_WRITE_REG(hw, IXGBE_FDIRTCPM, ~fdirtcpm); IXGBE_WRITE_REG(hw, IXGBE_FDIRUDPM, ~fdirtcpm); IXGBE_WRITE_REG(hw, IXGBE_FDIRSCTPM, ~fdirtcpm); - info->mask.src_port_mask = input_mask->src_port_mask; - info->mask.dst_port_mask = input_mask->dst_port_mask; /* Store source and destination IPv4 masks (big-endian), * can not use IXGBE_WRITE_REG. */ reg = IXGBE_PCI_REG_ADDR(hw, IXGBE_FDIRSIP4M); - *reg = ~(input_mask->ipv4_mask.src_ip); + *reg = ~(info->mask.src_ipv4_mask); reg = IXGBE_PCI_REG_ADDR(hw, IXGBE_FDIRDIP4M); - *reg = ~(input_mask->ipv4_mask.dst_ip); - info->mask.src_ipv4_mask = input_mask->ipv4_mask.src_ip; - info->mask.dst_ipv4_mask = input_mask->ipv4_mask.dst_ip; + *reg = ~(info->mask.dst_ipv4_mask); if (dev->data->dev_conf.fdir_conf.mode == RTE_FDIR_MODE_SIGNATURE) { /* * Store source and destination IPv6 masks (bit reversed) */ - IPV6_ADDR_TO_MASK(input_mask->ipv6_mask.src_ip, src_ipv6m); - IPV6_ADDR_TO_MASK(input_mask->ipv6_mask.dst_ip, dst_ipv6m); - fdiripv6m = (dst_ipv6m << 16) | src_ipv6m; + fdiripv6m = (info->mask.dst_ipv6_mask << 16) | + info->mask.src_ipv6_mask; IXGBE_WRITE_REG(hw, IXGBE_FDIRIP6M, ~fdiripv6m); - info->mask.src_ipv6_mask = src_ipv6m; - info->mask.dst_ipv6_mask = dst_ipv6m; } return IXGBE_SUCCESS; @@ -385,8 +377,7 @@ fdir_set_input_mask_82599(struct rte_eth_dev *dev, * but makes use of the rte_fdir_masks structure to see which bits to set. */ static int -fdir_set_input_mask_x550(struct rte_eth_dev *dev, - const struct rte_eth_fdir_masks *input_mask) +fdir_set_input_mask_x550(struct rte_eth_dev *dev) { struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct ixgbe_hw_fdir_info *info = @@ -409,20 +400,19 @@ fdir_set_input_mask_x550(struct rte_eth_dev *dev, /* some bits must be set for mac vlan or tunnel mode */ fdirm |= IXGBE_FDIRM_L4P | IXGBE_FDIRM_L3P; - if (input_mask->vlan_tci_mask == rte_cpu_to_be_16(0x0FFF)) + if (info->mask.vlan_tci_mask == rte_cpu_to_be_16(0x0FFF)) /* mask VLAN Priority */ fdirm |= IXGBE_FDIRM_VLANP; - else if (input_mask->vlan_tci_mask == rte_cpu_to_be_16(0xE000)) + else if (info->mask.vlan_tci_mask == rte_cpu_to_be_16(0xE000)) /* mask VLAN ID */ fdirm |= IXGBE_FDIRM_VLANID; - else if (input_mask->vlan_tci_mask == 0) + else if (info->mask.vlan_tci_mask == 0) /* mask VLAN ID and Priority */ fdirm |= IXGBE_FDIRM_VLANID | IXGBE_FDIRM_VLANP; - else if (input_mask->vlan_tci_mask != rte_cpu_to_be_16(0xEFFF)) { + else if (info->mask.vlan_tci_mask != rte_cpu_to_be_16(0xEFFF)) { PMD_INIT_LOG(ERR, "invalid vlan_tci_mask"); return -EINVAL; } - info->mask.vlan_tci_mask = input_mask->vlan_tci_mask; IXGBE_WRITE_REG(hw, IXGBE_FDIRM, fdirm); @@ -433,12 +423,11 @@ fdir_set_input_mask_x550(struct rte_eth_dev *dev, IXGBE_FDIRIP6M_TNI_VNI; if (mode == RTE_FDIR_MODE_PERFECT_TUNNEL) { - mac_mask = input_mask->mac_addr_byte_mask; + mac_mask = info->mask.mac_addr_byte_mask; fdiripv6m |= (mac_mask << IXGBE_FDIRIP6M_INNER_MAC_SHIFT) & IXGBE_FDIRIP6M_INNER_MAC; - info->mask.mac_addr_byte_mask = input_mask->mac_addr_byte_mask; - switch (input_mask->tunnel_type_mask) { + switch (info->mask.tunnel_type_mask) { case 0: /* Mask turnnel type */ fdiripv6m |= IXGBE_FDIRIP6M_TUNNEL_TYPE; @@ -449,10 +438,8 @@ fdir_set_input_mask_x550(struct rte_eth_dev *dev, PMD_INIT_LOG(ERR, "invalid tunnel_type_mask"); return -EINVAL; } - info->mask.tunnel_type_mask = - input_mask->tunnel_type_mask; - switch (rte_be_to_cpu_32(input_mask->tunnel_id_mask)) { + switch (rte_be_to_cpu_32(info->mask.tunnel_id_mask)) { case 0x0: /* Mask vxlan id */ fdiripv6m |= IXGBE_FDIRIP6M_TNI_VNI; @@ -466,8 +453,6 @@ fdir_set_input_mask_x550(struct rte_eth_dev *dev, PMD_INIT_LOG(ERR, "invalid tunnel_id_mask"); return -EINVAL; } - info->mask.tunnel_id_mask = - input_mask->tunnel_id_mask; } IXGBE_WRITE_REG(hw, IXGBE_FDIRIP6M, fdiripv6m); @@ -481,22 +466,115 @@ fdir_set_input_mask_x550(struct rte_eth_dev *dev, } static int -fdir_set_input_mask(struct rte_eth_dev *dev, - const struct rte_eth_fdir_masks *input_mask) +ixgbe_fdir_store_input_mask_82599(struct rte_eth_dev *dev, + const struct rte_eth_fdir_masks *input_mask) +{ + struct ixgbe_hw_fdir_info *info = + IXGBE_DEV_PRIVATE_TO_FDIR_INFO(dev->data->dev_private); + uint16_t dst_ipv6m = 0; + uint16_t src_ipv6m = 0; + + memset(&info->mask, 0, sizeof(struct ixgbe_hw_fdir_mask)); + info->mask.vlan_tci_mask = input_mask->vlan_tci_mask; + info->mask.src_port_mask = input_mask->src_port_mask; + info->mask.dst_port_mask = input_mask->dst_port_mask; + info->mask.src_ipv4_mask = input_mask->ipv4_mask.src_ip; + info->mask.dst_ipv4_mask = input_mask->ipv4_mask.dst_ip; + IPV6_ADDR_TO_MASK(input_mask->ipv6_mask.src_ip, src_ipv6m); + IPV6_ADDR_TO_MASK(input_mask->ipv6_mask.dst_ip, dst_ipv6m); + info->mask.src_ipv6_mask = src_ipv6m; + info->mask.dst_ipv6_mask = dst_ipv6m; + + return IXGBE_SUCCESS; +} + +static int +ixgbe_fdir_store_input_mask_x550(struct rte_eth_dev *dev, + const struct rte_eth_fdir_masks *input_mask) +{ + struct ixgbe_hw_fdir_info *info = + IXGBE_DEV_PRIVATE_TO_FDIR_INFO(dev->data->dev_private); + + memset(&info->mask, 0, sizeof(struct ixgbe_hw_fdir_mask)); + info->mask.vlan_tci_mask = input_mask->vlan_tci_mask; + info->mask.mac_addr_byte_mask = input_mask->mac_addr_byte_mask; + info->mask.tunnel_type_mask = input_mask->tunnel_type_mask; + info->mask.tunnel_id_mask = input_mask->tunnel_id_mask; + + return IXGBE_SUCCESS; +} + +static int +ixgbe_fdir_store_input_mask(struct rte_eth_dev *dev, + const struct rte_eth_fdir_masks *input_mask) { enum rte_fdir_mode mode = dev->data->dev_conf.fdir_conf.mode; if (mode >= RTE_FDIR_MODE_SIGNATURE && mode <= RTE_FDIR_MODE_PERFECT) - return fdir_set_input_mask_82599(dev, input_mask); + return ixgbe_fdir_store_input_mask_82599(dev, input_mask); else if (mode >= RTE_FDIR_MODE_PERFECT_MAC_VLAN && mode <= RTE_FDIR_MODE_PERFECT_TUNNEL) - return fdir_set_input_mask_x550(dev, input_mask); + return ixgbe_fdir_store_input_mask_x550(dev, input_mask); PMD_DRV_LOG(ERR, "Not supported fdir mode - %d!", mode); return -ENOTSUP; } +int +ixgbe_fdir_set_input_mask(struct rte_eth_dev *dev) +{ + enum rte_fdir_mode mode = dev->data->dev_conf.fdir_conf.mode; + + if (mode >= RTE_FDIR_MODE_SIGNATURE && + mode <= RTE_FDIR_MODE_PERFECT) + return fdir_set_input_mask_82599(dev); + else if (mode >= RTE_FDIR_MODE_PERFECT_MAC_VLAN && + mode <= RTE_FDIR_MODE_PERFECT_TUNNEL) + return fdir_set_input_mask_x550(dev); + + PMD_DRV_LOG(ERR, "Not supported fdir mode - %d!", mode); + return -ENOTSUP; +} + +int +ixgbe_fdir_set_flexbytes_offset(struct rte_eth_dev *dev, + uint16_t offset) +{ + struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + uint32_t fdirctrl; + int i; + + fdirctrl = IXGBE_READ_REG(hw, IXGBE_FDIRCTRL); + + fdirctrl &= ~IXGBE_FDIRCTRL_FLEX_MASK; + fdirctrl |= ((offset >> 1) /* convert to word offset */ + << IXGBE_FDIRCTRL_FLEX_SHIFT); + + IXGBE_WRITE_REG(hw, IXGBE_FDIRCTRL, fdirctrl); + IXGBE_WRITE_FLUSH(hw); + for (i = 0; i < IXGBE_FDIR_INIT_DONE_POLL; i++) { + if (IXGBE_READ_REG(hw, IXGBE_FDIRCTRL) & + IXGBE_FDIRCTRL_INIT_DONE) + break; + msec_delay(1); + } + return 0; +} + +static int +fdir_set_input_mask(struct rte_eth_dev *dev, + const struct rte_eth_fdir_masks *input_mask) +{ + int ret; + + ret = ixgbe_fdir_store_input_mask(dev, input_mask); + if (ret) + return ret; + + return ixgbe_fdir_set_input_mask(dev); +} + /* * ixgbe_check_fdir_flex_conf -check if the flex payload and mask configuration * arguments are valid @@ -605,7 +683,7 @@ ixgbe_fdir_configure(struct rte_eth_dev *dev) /* * The defaults in the HW for RX PB 1-7 are not zero and so should be - * intialized to zero for non DCB mode otherwise actual total RX PB + * initialized to zero for non DCB mode otherwise actual total RX PB * would be bigger than programmed and filter space would run into * the PB 0 region. */ @@ -681,6 +759,7 @@ ixgbe_fdir_filter_to_atr_input(const struct rte_eth_fdir_filter *fdir_filter, fdir_filter->input.flow.udp4_flow.src_port; input->formatted.dst_port = fdir_filter->input.flow.udp4_flow.dst_port; + /* fall-through */ /*for SCTP flow type, port and verify_tag are meaningless in ixgbe.*/ case RTE_ETH_FLOW_NONFRAG_IPV4_SCTP: case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: @@ -696,6 +775,7 @@ ixgbe_fdir_filter_to_atr_input(const struct rte_eth_fdir_filter *fdir_filter, fdir_filter->input.flow.udp6_flow.src_port; input->formatted.dst_port = fdir_filter->input.flow.udp6_flow.dst_port; + /* fall-through */ /*for SCTP flow type, port and verify_tag are meaningless in ixgbe.*/ case RTE_ETH_FLOW_NONFRAG_IPV6_SCTP: case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: @@ -1075,8 +1155,236 @@ fdir_erase_filter_82599(struct ixgbe_hw *hw, uint32_t fdirhash) } -/* - * ixgbe_add_del_fdir_filter - add or remove a flow diretor filter. +static inline struct ixgbe_fdir_filter * +ixgbe_fdir_filter_lookup(struct ixgbe_hw_fdir_info *fdir_info, + union ixgbe_atr_input *key) +{ + int ret; + + ret = rte_hash_lookup(fdir_info->hash_handle, (const void *)key); + if (ret < 0) + return NULL; + + return fdir_info->hash_map[ret]; +} + +static inline int +ixgbe_insert_fdir_filter(struct ixgbe_hw_fdir_info *fdir_info, + struct ixgbe_fdir_filter *fdir_filter) +{ + int ret; + + ret = rte_hash_add_key(fdir_info->hash_handle, + &fdir_filter->ixgbe_fdir); + + if (ret < 0) { + PMD_DRV_LOG(ERR, + "Failed to insert fdir filter to hash table %d!", + ret); + return ret; + } + + fdir_info->hash_map[ret] = fdir_filter; + + TAILQ_INSERT_TAIL(&fdir_info->fdir_list, fdir_filter, entries); + + return 0; +} + +static inline int +ixgbe_remove_fdir_filter(struct ixgbe_hw_fdir_info *fdir_info, + union ixgbe_atr_input *key) +{ + int ret; + struct ixgbe_fdir_filter *fdir_filter; + + ret = rte_hash_del_key(fdir_info->hash_handle, key); + + if (ret < 0) { + PMD_DRV_LOG(ERR, "No such fdir filter to delete %d!", ret); + return ret; + } + + fdir_filter = fdir_info->hash_map[ret]; + fdir_info->hash_map[ret] = NULL; + + TAILQ_REMOVE(&fdir_info->fdir_list, fdir_filter, entries); + rte_free(fdir_filter); + + return 0; +} + +static int +ixgbe_interpret_fdir_filter(struct rte_eth_dev *dev, + const struct rte_eth_fdir_filter *fdir_filter, + struct ixgbe_fdir_rule *rule) +{ + enum rte_fdir_mode fdir_mode = dev->data->dev_conf.fdir_conf.mode; + int err; + + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + + err = ixgbe_fdir_filter_to_atr_input(fdir_filter, + &rule->ixgbe_fdir, + fdir_mode); + if (err) + return err; + + rule->mode = fdir_mode; + if (fdir_filter->action.behavior == RTE_ETH_FDIR_REJECT) + rule->fdirflags = IXGBE_FDIRCMD_DROP; + rule->queue = fdir_filter->action.rx_queue; + rule->soft_id = fdir_filter->soft_id; + + return 0; +} + +int +ixgbe_fdir_filter_program(struct rte_eth_dev *dev, + struct ixgbe_fdir_rule *rule, + bool del, + bool update) +{ + struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + uint32_t fdircmd_flags; + uint32_t fdirhash; + uint8_t queue; + bool is_perfect = FALSE; + int err; + struct ixgbe_hw_fdir_info *info = + IXGBE_DEV_PRIVATE_TO_FDIR_INFO(dev->data->dev_private); + enum rte_fdir_mode fdir_mode = dev->data->dev_conf.fdir_conf.mode; + struct ixgbe_fdir_filter *node; + bool add_node = FALSE; + + if (fdir_mode == RTE_FDIR_MODE_NONE || + fdir_mode != rule->mode) + return -ENOTSUP; + + /* + * Sanity check for x550. + * When adding a new filter with flow type set to IPv4, + * the flow director mask should be configed before, + * and the L4 protocol and ports are masked. + */ + if ((!del) && + (hw->mac.type == ixgbe_mac_X550 || + hw->mac.type == ixgbe_mac_X550EM_x || + hw->mac.type == ixgbe_mac_X550EM_a) && + (rule->ixgbe_fdir.formatted.flow_type == + IXGBE_ATR_FLOW_TYPE_IPV4 || + rule->ixgbe_fdir.formatted.flow_type == + IXGBE_ATR_FLOW_TYPE_IPV6) && + (info->mask.src_port_mask != 0 || + info->mask.dst_port_mask != 0) && + (rule->mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN && + rule->mode != RTE_FDIR_MODE_PERFECT_TUNNEL)) { + PMD_DRV_LOG(ERR, "By this device," + " IPv4 is not supported without" + " L4 protocol and ports masked!"); + return -ENOTSUP; + } + + if (fdir_mode >= RTE_FDIR_MODE_PERFECT && + fdir_mode <= RTE_FDIR_MODE_PERFECT_TUNNEL) + is_perfect = TRUE; + + if (is_perfect) { + if (rule->ixgbe_fdir.formatted.flow_type & + IXGBE_ATR_L4TYPE_IPV6_MASK) { + PMD_DRV_LOG(ERR, "IPv6 is not supported in" + " perfect mode!"); + return -ENOTSUP; + } + fdirhash = atr_compute_perfect_hash_82599(&rule->ixgbe_fdir, + dev->data->dev_conf.fdir_conf.pballoc); + fdirhash |= rule->soft_id << + IXGBE_FDIRHASH_SIG_SW_INDEX_SHIFT; + } else + fdirhash = atr_compute_sig_hash_82599(&rule->ixgbe_fdir, + dev->data->dev_conf.fdir_conf.pballoc); + + if (del) { + err = ixgbe_remove_fdir_filter(info, &rule->ixgbe_fdir); + if (err < 0) + return err; + + err = fdir_erase_filter_82599(hw, fdirhash); + if (err < 0) + PMD_DRV_LOG(ERR, "Fail to delete FDIR filter!"); + else + PMD_DRV_LOG(DEBUG, "Success to delete FDIR filter!"); + return err; + } + /* add or update an fdir filter*/ + fdircmd_flags = (update) ? IXGBE_FDIRCMD_FILTER_UPDATE : 0; + if (rule->fdirflags & IXGBE_FDIRCMD_DROP) { + if (is_perfect) { + queue = dev->data->dev_conf.fdir_conf.drop_queue; + fdircmd_flags |= IXGBE_FDIRCMD_DROP; + } else { + PMD_DRV_LOG(ERR, "Drop option is not supported in" + " signature mode."); + return -EINVAL; + } + } else if (rule->queue < IXGBE_MAX_RX_QUEUE_NUM) + queue = (uint8_t)rule->queue; + else + return -EINVAL; + + node = ixgbe_fdir_filter_lookup(info, &rule->ixgbe_fdir); + if (node) { + if (update) { + node->fdirflags = fdircmd_flags; + node->fdirhash = fdirhash; + node->queue = queue; + } else { + PMD_DRV_LOG(ERR, "Conflict with existing fdir filter!"); + return -EINVAL; + } + } else { + add_node = TRUE; + node = rte_zmalloc("ixgbe_fdir", + sizeof(struct ixgbe_fdir_filter), + 0); + if (!node) + return -ENOMEM; + rte_memcpy(&node->ixgbe_fdir, + &rule->ixgbe_fdir, + sizeof(union ixgbe_atr_input)); + node->fdirflags = fdircmd_flags; + node->fdirhash = fdirhash; + node->queue = queue; + + err = ixgbe_insert_fdir_filter(info, node); + if (err < 0) { + rte_free(node); + return err; + } + } + + if (is_perfect) { + err = fdir_write_perfect_filter_82599(hw, &rule->ixgbe_fdir, + queue, fdircmd_flags, + fdirhash, fdir_mode); + } else { + err = fdir_add_signature_filter_82599(hw, &rule->ixgbe_fdir, + queue, fdircmd_flags, + fdirhash); + } + if (err < 0) { + PMD_DRV_LOG(ERR, "Fail to add FDIR filter!"); + + if (add_node) + (void)ixgbe_remove_fdir_filter(info, &rule->ixgbe_fdir); + } else { + PMD_DRV_LOG(DEBUG, "Success to add FDIR filter"); + } + + return err; +} + +/* ixgbe_add_del_fdir_filter - add or remove a flow diretor filter. * @dev: pointer to the structure rte_eth_dev * @fdir_filter: fdir filter entry * @del: 1 - delete, 0 - add @@ -1088,104 +1396,15 @@ ixgbe_add_del_fdir_filter(struct rte_eth_dev *dev, bool del, bool update) { - struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); - uint32_t fdircmd_flags; - uint32_t fdirhash; - union ixgbe_atr_input input; - uint8_t queue; - bool is_perfect = FALSE; + struct ixgbe_fdir_rule rule; int err; - struct ixgbe_hw_fdir_info *info = - IXGBE_DEV_PRIVATE_TO_FDIR_INFO(dev->data->dev_private); - enum rte_fdir_mode fdir_mode = dev->data->dev_conf.fdir_conf.mode; - if (fdir_mode == RTE_FDIR_MODE_NONE) - return -ENOTSUP; + err = ixgbe_interpret_fdir_filter(dev, fdir_filter, &rule); - /* - * Sanity check for x550. - * When adding a new filter with flow type set to IPv4-other, - * the flow director mask should be configed before, - * and the L4 protocol and ports are masked. - */ - if ((!del) && - (hw->mac.type == ixgbe_mac_X550 || - hw->mac.type == ixgbe_mac_X550EM_x || - hw->mac.type == ixgbe_mac_X550EM_a) && - (fdir_filter->input.flow_type == - RTE_ETH_FLOW_NONFRAG_IPV4_OTHER) && - (info->mask.src_port_mask != 0 || - info->mask.dst_port_mask != 0)) { - PMD_DRV_LOG(ERR, "By this device," - " IPv4-other is not supported without" - " L4 protocol and ports masked!"); - return -ENOTSUP; - } - - if (fdir_mode >= RTE_FDIR_MODE_PERFECT && - fdir_mode <= RTE_FDIR_MODE_PERFECT_TUNNEL) - is_perfect = TRUE; - - memset(&input, 0, sizeof(input)); - - err = ixgbe_fdir_filter_to_atr_input(fdir_filter, &input, - fdir_mode); if (err) return err; - if (is_perfect) { - if (input.formatted.flow_type & IXGBE_ATR_L4TYPE_IPV6_MASK) { - PMD_DRV_LOG(ERR, "IPv6 is not supported in" - " perfect mode!"); - return -ENOTSUP; - } - fdirhash = atr_compute_perfect_hash_82599(&input, - dev->data->dev_conf.fdir_conf.pballoc); - fdirhash |= fdir_filter->soft_id << - IXGBE_FDIRHASH_SIG_SW_INDEX_SHIFT; - } else - fdirhash = atr_compute_sig_hash_82599(&input, - dev->data->dev_conf.fdir_conf.pballoc); - - if (del) { - err = fdir_erase_filter_82599(hw, fdirhash); - if (err < 0) - PMD_DRV_LOG(ERR, "Fail to delete FDIR filter!"); - else - PMD_DRV_LOG(DEBUG, "Success to delete FDIR filter!"); - return err; - } - /* add or update an fdir filter*/ - fdircmd_flags = (update) ? IXGBE_FDIRCMD_FILTER_UPDATE : 0; - if (fdir_filter->action.behavior == RTE_ETH_FDIR_REJECT) { - if (is_perfect) { - queue = dev->data->dev_conf.fdir_conf.drop_queue; - fdircmd_flags |= IXGBE_FDIRCMD_DROP; - } else { - PMD_DRV_LOG(ERR, "Drop option is not supported in" - " signature mode."); - return -EINVAL; - } - } else if (fdir_filter->action.behavior == RTE_ETH_FDIR_ACCEPT && - fdir_filter->action.rx_queue < IXGBE_MAX_RX_QUEUE_NUM) - queue = (uint8_t)fdir_filter->action.rx_queue; - else - return -EINVAL; - - if (is_perfect) { - err = fdir_write_perfect_filter_82599(hw, &input, queue, - fdircmd_flags, fdirhash, - fdir_mode); - } else { - err = fdir_add_signature_filter_82599(hw, &input, queue, - fdircmd_flags, fdirhash); - } - if (err < 0) - PMD_DRV_LOG(ERR, "Fail to add FDIR filter!"); - else - PMD_DRV_LOG(DEBUG, "Success to add FDIR filter"); - - return err; + return ixgbe_fdir_filter_program(dev, &rule, del, update); } static int @@ -1378,3 +1597,66 @@ ixgbe_fdir_ctrl_func(struct rte_eth_dev *dev, } return ret; } + +/* restore flow director filter */ +void +ixgbe_fdir_filter_restore(struct rte_eth_dev *dev) +{ + struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct ixgbe_hw_fdir_info *fdir_info = + IXGBE_DEV_PRIVATE_TO_FDIR_INFO(dev->data->dev_private); + struct ixgbe_fdir_filter *node; + bool is_perfect = FALSE; + enum rte_fdir_mode fdir_mode = dev->data->dev_conf.fdir_conf.mode; + + if (fdir_mode >= RTE_FDIR_MODE_PERFECT && + fdir_mode <= RTE_FDIR_MODE_PERFECT_TUNNEL) + is_perfect = TRUE; + + if (is_perfect) { + TAILQ_FOREACH(node, &fdir_info->fdir_list, entries) { + (void)fdir_write_perfect_filter_82599(hw, + &node->ixgbe_fdir, + node->queue, + node->fdirflags, + node->fdirhash, + fdir_mode); + } + } else { + TAILQ_FOREACH(node, &fdir_info->fdir_list, entries) { + (void)fdir_add_signature_filter_82599(hw, + &node->ixgbe_fdir, + node->queue, + node->fdirflags, + node->fdirhash); + } + } +} + +/* remove all the flow director filters */ +int +ixgbe_clear_all_fdir_filter(struct rte_eth_dev *dev) +{ + struct ixgbe_hw_fdir_info *fdir_info = + IXGBE_DEV_PRIVATE_TO_FDIR_INFO(dev->data->dev_private); + struct ixgbe_fdir_filter *fdir_filter; + struct ixgbe_fdir_filter *filter_flag; + int ret = 0; + + /* flush flow director */ + rte_hash_reset(fdir_info->hash_handle); + memset(fdir_info->hash_map, 0, + sizeof(struct ixgbe_fdir_filter *) * IXGBE_MAX_FDIR_FILTER_NUM); + filter_flag = TAILQ_FIRST(&fdir_info->fdir_list); + while ((fdir_filter = TAILQ_FIRST(&fdir_info->fdir_list))) { + TAILQ_REMOVE(&fdir_info->fdir_list, + fdir_filter, + entries); + rte_free(fdir_filter); + } + + if (filter_flag != NULL) + ret = ixgbe_fdir_flush(dev); + + return ret; +} diff --git a/dpdk/drivers/net/ixgbe/ixgbe_flow.c b/dpdk/drivers/net/ixgbe/ixgbe_flow.c new file mode 100644 index 00000000..07abb343 --- /dev/null +++ b/dpdk/drivers/net/ixgbe/ixgbe_flow.c @@ -0,0 +1,3223 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ixgbe_logs.h" +#include "base/ixgbe_api.h" +#include "base/ixgbe_vf.h" +#include "base/ixgbe_common.h" +#include "ixgbe_ethdev.h" +#include "ixgbe_bypass.h" +#include "ixgbe_rxtx.h" +#include "base/ixgbe_type.h" +#include "base/ixgbe_phy.h" +#include "rte_pmd_ixgbe.h" + + +#define IXGBE_MIN_N_TUPLE_PRIO 1 +#define IXGBE_MAX_N_TUPLE_PRIO 7 +#define IXGBE_MAX_FLX_SOURCE_OFF 62 + +/* ntuple filter list structure */ +struct ixgbe_ntuple_filter_ele { + TAILQ_ENTRY(ixgbe_ntuple_filter_ele) entries; + struct rte_eth_ntuple_filter filter_info; +}; +/* ethertype filter list structure */ +struct ixgbe_ethertype_filter_ele { + TAILQ_ENTRY(ixgbe_ethertype_filter_ele) entries; + struct rte_eth_ethertype_filter filter_info; +}; +/* syn filter list structure */ +struct ixgbe_eth_syn_filter_ele { + TAILQ_ENTRY(ixgbe_eth_syn_filter_ele) entries; + struct rte_eth_syn_filter filter_info; +}; +/* fdir filter list structure */ +struct ixgbe_fdir_rule_ele { + TAILQ_ENTRY(ixgbe_fdir_rule_ele) entries; + struct ixgbe_fdir_rule filter_info; +}; +/* l2_tunnel filter list structure */ +struct ixgbe_eth_l2_tunnel_conf_ele { + TAILQ_ENTRY(ixgbe_eth_l2_tunnel_conf_ele) entries; + struct rte_eth_l2_tunnel_conf filter_info; +}; +/* ixgbe_flow memory list structure */ +struct ixgbe_flow_mem { + TAILQ_ENTRY(ixgbe_flow_mem) entries; + struct rte_flow *flow; +}; + +TAILQ_HEAD(ixgbe_ntuple_filter_list, ixgbe_ntuple_filter_ele); +TAILQ_HEAD(ixgbe_ethertype_filter_list, ixgbe_ethertype_filter_ele); +TAILQ_HEAD(ixgbe_syn_filter_list, ixgbe_eth_syn_filter_ele); +TAILQ_HEAD(ixgbe_fdir_rule_filter_list, ixgbe_fdir_rule_ele); +TAILQ_HEAD(ixgbe_l2_tunnel_filter_list, ixgbe_eth_l2_tunnel_conf_ele); +TAILQ_HEAD(ixgbe_flow_mem_list, ixgbe_flow_mem); + +static struct ixgbe_ntuple_filter_list filter_ntuple_list; +static struct ixgbe_ethertype_filter_list filter_ethertype_list; +static struct ixgbe_syn_filter_list filter_syn_list; +static struct ixgbe_fdir_rule_filter_list filter_fdir_list; +static struct ixgbe_l2_tunnel_filter_list filter_l2_tunnel_list; +static struct ixgbe_flow_mem_list ixgbe_flow_list; + +/** + * Endless loop will never happen with below assumption + * 1. there is at least one no-void item(END) + * 2. cur is before END. + */ +static inline +const struct rte_flow_item *next_no_void_pattern( + const struct rte_flow_item pattern[], + const struct rte_flow_item *cur) +{ + const struct rte_flow_item *next = + cur ? cur + 1 : &pattern[0]; + while (1) { + if (next->type != RTE_FLOW_ITEM_TYPE_VOID) + return next; + next++; + } +} + +static inline +const struct rte_flow_action *next_no_void_action( + const struct rte_flow_action actions[], + const struct rte_flow_action *cur) +{ + const struct rte_flow_action *next = + cur ? cur + 1 : &actions[0]; + while (1) { + if (next->type != RTE_FLOW_ACTION_TYPE_VOID) + return next; + next++; + } +} + +/** + * Please aware there's an asumption for all the parsers. + * rte_flow_item is using big endian, rte_flow_attr and + * rte_flow_action are using CPU order. + * Because the pattern is used to describe the packets, + * normally the packets should use network order. + */ + +/** + * Parse the rule to see if it is a n-tuple rule. + * And get the n-tuple filter info BTW. + * pattern: + * The first not void item can be ETH or IPV4. + * The second not void item must be IPV4 if the first one is ETH. + * The third not void item must be UDP or TCP. + * The next not void item must be END. + * action: + * The first not void action should be QUEUE. + * The next not void action should be END. + * pattern example: + * ITEM Spec Mask + * ETH NULL NULL + * IPV4 src_addr 192.168.1.20 0xFFFFFFFF + * dst_addr 192.167.3.50 0xFFFFFFFF + * next_proto_id 17 0xFF + * UDP/TCP/ src_port 80 0xFFFF + * SCTP dst_port 80 0xFFFF + * END + * other members in mask and spec should set to 0x00. + * item->last should be NULL. + * + * Special case for flow action type RTE_FLOW_ACTION_TYPE_SECURITY. + * + */ +static int +cons_parse_ntuple_filter(const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_eth_ntuple_filter *filter, + struct rte_flow_error *error) +{ + const struct rte_flow_item *item; + const struct rte_flow_action *act; + const struct rte_flow_item_ipv4 *ipv4_spec; + const struct rte_flow_item_ipv4 *ipv4_mask; + const struct rte_flow_item_tcp *tcp_spec; + const struct rte_flow_item_tcp *tcp_mask; + const struct rte_flow_item_udp *udp_spec; + const struct rte_flow_item_udp *udp_mask; + const struct rte_flow_item_sctp *sctp_spec; + const struct rte_flow_item_sctp *sctp_mask; + + if (!pattern) { + rte_flow_error_set(error, + EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_NUM, + NULL, "NULL pattern."); + return -rte_errno; + } + + if (!actions) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION_NUM, + NULL, "NULL action."); + return -rte_errno; + } + if (!attr) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR, + NULL, "NULL attribute."); + return -rte_errno; + } + +#ifdef RTE_LIBRTE_SECURITY + /** + * Special case for flow action type RTE_FLOW_ACTION_TYPE_SECURITY + */ + act = next_no_void_action(actions, NULL); + if (act->type == RTE_FLOW_ACTION_TYPE_SECURITY) { + const void *conf = act->conf; + /* check if the next not void item is END */ + act = next_no_void_action(actions, act); + if (act->type != RTE_FLOW_ACTION_TYPE_END) { + memset(filter, 0, sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + act, "Not supported action."); + return -rte_errno; + } + + /* get the IP pattern*/ + item = next_no_void_pattern(pattern, NULL); + while (item->type != RTE_FLOW_ITEM_TYPE_IPV4 && + item->type != RTE_FLOW_ITEM_TYPE_IPV6) { + if (item->last || + item->type == RTE_FLOW_ITEM_TYPE_END) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "IP pattern missing."); + return -rte_errno; + } + item = next_no_void_pattern(pattern, item); + } + + filter->proto = IPPROTO_ESP; + return ixgbe_crypto_add_ingress_sa_from_flow(conf, item->spec, + item->type == RTE_FLOW_ITEM_TYPE_IPV6); + } +#endif + + /* the first not void item can be MAC or IPv4 */ + item = next_no_void_pattern(pattern, NULL); + + if (item->type != RTE_FLOW_ITEM_TYPE_ETH && + item->type != RTE_FLOW_ITEM_TYPE_IPV4) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by ntuple filter"); + return -rte_errno; + } + /* Skip Ethernet */ + if (item->type == RTE_FLOW_ITEM_TYPE_ETH) { + /*Not supported last point for range*/ + if (item->last) { + rte_flow_error_set(error, + EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + item, "Not supported last point for range"); + return -rte_errno; + + } + /* if the first item is MAC, the content should be NULL */ + if (item->spec || item->mask) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by ntuple filter"); + return -rte_errno; + } + /* check if the next not void item is IPv4 */ + item = next_no_void_pattern(pattern, item); + if (item->type != RTE_FLOW_ITEM_TYPE_IPV4) { + rte_flow_error_set(error, + EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by ntuple filter"); + return -rte_errno; + } + } + + /* get the IPv4 info */ + if (!item->spec || !item->mask) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Invalid ntuple mask"); + return -rte_errno; + } + /*Not supported last point for range*/ + if (item->last) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + item, "Not supported last point for range"); + return -rte_errno; + + } + + ipv4_mask = (const struct rte_flow_item_ipv4 *)item->mask; + /** + * Only support src & dst addresses, protocol, + * others should be masked. + */ + if (ipv4_mask->hdr.version_ihl || + ipv4_mask->hdr.type_of_service || + ipv4_mask->hdr.total_length || + ipv4_mask->hdr.packet_id || + ipv4_mask->hdr.fragment_offset || + ipv4_mask->hdr.time_to_live || + ipv4_mask->hdr.hdr_checksum) { + rte_flow_error_set(error, + EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by ntuple filter"); + return -rte_errno; + } + + filter->dst_ip_mask = ipv4_mask->hdr.dst_addr; + filter->src_ip_mask = ipv4_mask->hdr.src_addr; + filter->proto_mask = ipv4_mask->hdr.next_proto_id; + + ipv4_spec = (const struct rte_flow_item_ipv4 *)item->spec; + filter->dst_ip = ipv4_spec->hdr.dst_addr; + filter->src_ip = ipv4_spec->hdr.src_addr; + filter->proto = ipv4_spec->hdr.next_proto_id; + + /* check if the next not void item is TCP or UDP */ + item = next_no_void_pattern(pattern, item); + if (item->type != RTE_FLOW_ITEM_TYPE_TCP && + item->type != RTE_FLOW_ITEM_TYPE_UDP && + item->type != RTE_FLOW_ITEM_TYPE_SCTP && + item->type != RTE_FLOW_ITEM_TYPE_END) { + memset(filter, 0, sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by ntuple filter"); + return -rte_errno; + } + + /* get the TCP/UDP info */ + if ((item->type != RTE_FLOW_ITEM_TYPE_END) && + (!item->spec || !item->mask)) { + memset(filter, 0, sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Invalid ntuple mask"); + return -rte_errno; + } + + /*Not supported last point for range*/ + if (item->last) { + memset(filter, 0, sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + item, "Not supported last point for range"); + return -rte_errno; + + } + + if (item->type == RTE_FLOW_ITEM_TYPE_TCP) { + tcp_mask = (const struct rte_flow_item_tcp *)item->mask; + + /** + * Only support src & dst ports, tcp flags, + * others should be masked. + */ + if (tcp_mask->hdr.sent_seq || + tcp_mask->hdr.recv_ack || + tcp_mask->hdr.data_off || + tcp_mask->hdr.rx_win || + tcp_mask->hdr.cksum || + tcp_mask->hdr.tcp_urp) { + memset(filter, 0, + sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by ntuple filter"); + return -rte_errno; + } + + filter->dst_port_mask = tcp_mask->hdr.dst_port; + filter->src_port_mask = tcp_mask->hdr.src_port; + if (tcp_mask->hdr.tcp_flags == 0xFF) { + filter->flags |= RTE_NTUPLE_FLAGS_TCP_FLAG; + } else if (!tcp_mask->hdr.tcp_flags) { + filter->flags &= ~RTE_NTUPLE_FLAGS_TCP_FLAG; + } else { + memset(filter, 0, sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by ntuple filter"); + return -rte_errno; + } + + tcp_spec = (const struct rte_flow_item_tcp *)item->spec; + filter->dst_port = tcp_spec->hdr.dst_port; + filter->src_port = tcp_spec->hdr.src_port; + filter->tcp_flags = tcp_spec->hdr.tcp_flags; + } else if (item->type == RTE_FLOW_ITEM_TYPE_UDP) { + udp_mask = (const struct rte_flow_item_udp *)item->mask; + + /** + * Only support src & dst ports, + * others should be masked. + */ + if (udp_mask->hdr.dgram_len || + udp_mask->hdr.dgram_cksum) { + memset(filter, 0, + sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by ntuple filter"); + return -rte_errno; + } + + filter->dst_port_mask = udp_mask->hdr.dst_port; + filter->src_port_mask = udp_mask->hdr.src_port; + + udp_spec = (const struct rte_flow_item_udp *)item->spec; + filter->dst_port = udp_spec->hdr.dst_port; + filter->src_port = udp_spec->hdr.src_port; + } else if (item->type == RTE_FLOW_ITEM_TYPE_SCTP) { + sctp_mask = (const struct rte_flow_item_sctp *)item->mask; + + /** + * Only support src & dst ports, + * others should be masked. + */ + if (sctp_mask->hdr.tag || + sctp_mask->hdr.cksum) { + memset(filter, 0, + sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by ntuple filter"); + return -rte_errno; + } + + filter->dst_port_mask = sctp_mask->hdr.dst_port; + filter->src_port_mask = sctp_mask->hdr.src_port; + + sctp_spec = (const struct rte_flow_item_sctp *)item->spec; + filter->dst_port = sctp_spec->hdr.dst_port; + filter->src_port = sctp_spec->hdr.src_port; + } else { + goto action; + } + + /* check if the next not void item is END */ + item = next_no_void_pattern(pattern, item); + if (item->type != RTE_FLOW_ITEM_TYPE_END) { + memset(filter, 0, sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by ntuple filter"); + return -rte_errno; + } + +action: + + /** + * n-tuple only supports forwarding, + * check if the first not void action is QUEUE. + */ + act = next_no_void_action(actions, NULL); + if (act->type != RTE_FLOW_ACTION_TYPE_QUEUE) { + memset(filter, 0, sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + item, "Not supported action."); + return -rte_errno; + } + filter->queue = + ((const struct rte_flow_action_queue *)act->conf)->index; + + /* check if the next not void item is END */ + act = next_no_void_action(actions, act); + if (act->type != RTE_FLOW_ACTION_TYPE_END) { + memset(filter, 0, sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + act, "Not supported action."); + return -rte_errno; + } + + /* parse attr */ + /* must be input direction */ + if (!attr->ingress) { + memset(filter, 0, sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, + attr, "Only support ingress."); + return -rte_errno; + } + + /* not supported */ + if (attr->egress) { + memset(filter, 0, sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, + attr, "Not support egress."); + return -rte_errno; + } + + if (attr->priority > 0xFFFF) { + memset(filter, 0, sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, + attr, "Error priority."); + return -rte_errno; + } + filter->priority = (uint16_t)attr->priority; + if (attr->priority < IXGBE_MIN_N_TUPLE_PRIO || + attr->priority > IXGBE_MAX_N_TUPLE_PRIO) + filter->priority = 1; + + return 0; +} + +/* a specific function for ixgbe because the flags is specific */ +static int +ixgbe_parse_ntuple_filter(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_eth_ntuple_filter *filter, + struct rte_flow_error *error) +{ + int ret; + struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + + MAC_TYPE_FILTER_SUP_EXT(hw->mac.type); + + ret = cons_parse_ntuple_filter(attr, pattern, actions, filter, error); + + if (ret) + return ret; + +#ifdef RTE_LIBRTE_SECURITY + /* ESP flow not really a flow*/ + if (filter->proto == IPPROTO_ESP) + return 0; +#endif + + /* Ixgbe doesn't support tcp flags. */ + if (filter->flags & RTE_NTUPLE_FLAGS_TCP_FLAG) { + memset(filter, 0, sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + NULL, "Not supported by ntuple filter"); + return -rte_errno; + } + + /* Ixgbe doesn't support many priorities. */ + if (filter->priority < IXGBE_MIN_N_TUPLE_PRIO || + filter->priority > IXGBE_MAX_N_TUPLE_PRIO) { + memset(filter, 0, sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + NULL, "Priority not supported by ntuple filter"); + return -rte_errno; + } + + if (filter->queue >= dev->data->nb_rx_queues) + return -rte_errno; + + /* fixed value for ixgbe */ + filter->flags = RTE_5TUPLE_FLAGS; + return 0; +} + +/** + * Parse the rule to see if it is a ethertype rule. + * And get the ethertype filter info BTW. + * pattern: + * The first not void item can be ETH. + * The next not void item must be END. + * action: + * The first not void action should be QUEUE. + * The next not void action should be END. + * pattern example: + * ITEM Spec Mask + * ETH type 0x0807 0xFFFF + * END + * other members in mask and spec should set to 0x00. + * item->last should be NULL. + */ +static int +cons_parse_ethertype_filter(const struct rte_flow_attr *attr, + const struct rte_flow_item *pattern, + const struct rte_flow_action *actions, + struct rte_eth_ethertype_filter *filter, + struct rte_flow_error *error) +{ + const struct rte_flow_item *item; + const struct rte_flow_action *act; + const struct rte_flow_item_eth *eth_spec; + const struct rte_flow_item_eth *eth_mask; + const struct rte_flow_action_queue *act_q; + + if (!pattern) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM_NUM, + NULL, "NULL pattern."); + return -rte_errno; + } + + if (!actions) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION_NUM, + NULL, "NULL action."); + return -rte_errno; + } + + if (!attr) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR, + NULL, "NULL attribute."); + return -rte_errno; + } + + item = next_no_void_pattern(pattern, NULL); + /* The first non-void item should be MAC. */ + if (item->type != RTE_FLOW_ITEM_TYPE_ETH) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by ethertype filter"); + return -rte_errno; + } + + /*Not supported last point for range*/ + if (item->last) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + item, "Not supported last point for range"); + return -rte_errno; + } + + /* Get the MAC info. */ + if (!item->spec || !item->mask) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by ethertype filter"); + return -rte_errno; + } + + eth_spec = (const struct rte_flow_item_eth *)item->spec; + eth_mask = (const struct rte_flow_item_eth *)item->mask; + + /* Mask bits of source MAC address must be full of 0. + * Mask bits of destination MAC address must be full + * of 1 or full of 0. + */ + if (!is_zero_ether_addr(ð_mask->src) || + (!is_zero_ether_addr(ð_mask->dst) && + !is_broadcast_ether_addr(ð_mask->dst))) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Invalid ether address mask"); + return -rte_errno; + } + + if ((eth_mask->type & UINT16_MAX) != UINT16_MAX) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Invalid ethertype mask"); + return -rte_errno; + } + + /* If mask bits of destination MAC address + * are full of 1, set RTE_ETHTYPE_FLAGS_MAC. + */ + if (is_broadcast_ether_addr(ð_mask->dst)) { + filter->mac_addr = eth_spec->dst; + filter->flags |= RTE_ETHTYPE_FLAGS_MAC; + } else { + filter->flags &= ~RTE_ETHTYPE_FLAGS_MAC; + } + filter->ether_type = rte_be_to_cpu_16(eth_spec->type); + + /* Check if the next non-void item is END. */ + item = next_no_void_pattern(pattern, item); + if (item->type != RTE_FLOW_ITEM_TYPE_END) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by ethertype filter."); + return -rte_errno; + } + + /* Parse action */ + + act = next_no_void_action(actions, NULL); + if (act->type != RTE_FLOW_ACTION_TYPE_QUEUE && + act->type != RTE_FLOW_ACTION_TYPE_DROP) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + act, "Not supported action."); + return -rte_errno; + } + + if (act->type == RTE_FLOW_ACTION_TYPE_QUEUE) { + act_q = (const struct rte_flow_action_queue *)act->conf; + filter->queue = act_q->index; + } else { + filter->flags |= RTE_ETHTYPE_FLAGS_DROP; + } + + /* Check if the next non-void item is END */ + act = next_no_void_action(actions, act); + if (act->type != RTE_FLOW_ACTION_TYPE_END) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + act, "Not supported action."); + return -rte_errno; + } + + /* Parse attr */ + /* Must be input direction */ + if (!attr->ingress) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, + attr, "Only support ingress."); + return -rte_errno; + } + + /* Not supported */ + if (attr->egress) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, + attr, "Not support egress."); + return -rte_errno; + } + + /* Not supported */ + if (attr->priority) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, + attr, "Not support priority."); + return -rte_errno; + } + + /* Not supported */ + if (attr->group) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_GROUP, + attr, "Not support group."); + return -rte_errno; + } + + return 0; +} + +static int +ixgbe_parse_ethertype_filter(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_eth_ethertype_filter *filter, + struct rte_flow_error *error) +{ + int ret; + struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + + MAC_TYPE_FILTER_SUP(hw->mac.type); + + ret = cons_parse_ethertype_filter(attr, pattern, + actions, filter, error); + + if (ret) + return ret; + + /* Ixgbe doesn't support MAC address. */ + if (filter->flags & RTE_ETHTYPE_FLAGS_MAC) { + memset(filter, 0, sizeof(struct rte_eth_ethertype_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + NULL, "Not supported by ethertype filter"); + return -rte_errno; + } + + if (filter->queue >= dev->data->nb_rx_queues) { + memset(filter, 0, sizeof(struct rte_eth_ethertype_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + NULL, "queue index much too big"); + return -rte_errno; + } + + if (filter->ether_type == ETHER_TYPE_IPv4 || + filter->ether_type == ETHER_TYPE_IPv6) { + memset(filter, 0, sizeof(struct rte_eth_ethertype_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + NULL, "IPv4/IPv6 not supported by ethertype filter"); + return -rte_errno; + } + + if (filter->flags & RTE_ETHTYPE_FLAGS_MAC) { + memset(filter, 0, sizeof(struct rte_eth_ethertype_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + NULL, "mac compare is unsupported"); + return -rte_errno; + } + + if (filter->flags & RTE_ETHTYPE_FLAGS_DROP) { + memset(filter, 0, sizeof(struct rte_eth_ethertype_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + NULL, "drop option is unsupported"); + return -rte_errno; + } + + return 0; +} + +/** + * Parse the rule to see if it is a TCP SYN rule. + * And get the TCP SYN filter info BTW. + * pattern: + * The first not void item must be ETH. + * The second not void item must be IPV4 or IPV6. + * The third not void item must be TCP. + * The next not void item must be END. + * action: + * The first not void action should be QUEUE. + * The next not void action should be END. + * pattern example: + * ITEM Spec Mask + * ETH NULL NULL + * IPV4/IPV6 NULL NULL + * TCP tcp_flags 0x02 0xFF + * END + * other members in mask and spec should set to 0x00. + * item->last should be NULL. + */ +static int +cons_parse_syn_filter(const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_eth_syn_filter *filter, + struct rte_flow_error *error) +{ + const struct rte_flow_item *item; + const struct rte_flow_action *act; + const struct rte_flow_item_tcp *tcp_spec; + const struct rte_flow_item_tcp *tcp_mask; + const struct rte_flow_action_queue *act_q; + + if (!pattern) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM_NUM, + NULL, "NULL pattern."); + return -rte_errno; + } + + if (!actions) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION_NUM, + NULL, "NULL action."); + return -rte_errno; + } + + if (!attr) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR, + NULL, "NULL attribute."); + return -rte_errno; + } + + + /* the first not void item should be MAC or IPv4 or IPv6 or TCP */ + item = next_no_void_pattern(pattern, NULL); + if (item->type != RTE_FLOW_ITEM_TYPE_ETH && + item->type != RTE_FLOW_ITEM_TYPE_IPV4 && + item->type != RTE_FLOW_ITEM_TYPE_IPV6 && + item->type != RTE_FLOW_ITEM_TYPE_TCP) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by syn filter"); + return -rte_errno; + } + /*Not supported last point for range*/ + if (item->last) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + item, "Not supported last point for range"); + return -rte_errno; + } + + /* Skip Ethernet */ + if (item->type == RTE_FLOW_ITEM_TYPE_ETH) { + /* if the item is MAC, the content should be NULL */ + if (item->spec || item->mask) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Invalid SYN address mask"); + return -rte_errno; + } + + /* check if the next not void item is IPv4 or IPv6 */ + item = next_no_void_pattern(pattern, item); + if (item->type != RTE_FLOW_ITEM_TYPE_IPV4 && + item->type != RTE_FLOW_ITEM_TYPE_IPV6) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by syn filter"); + return -rte_errno; + } + } + + /* Skip IP */ + if (item->type == RTE_FLOW_ITEM_TYPE_IPV4 || + item->type == RTE_FLOW_ITEM_TYPE_IPV6) { + /* if the item is IP, the content should be NULL */ + if (item->spec || item->mask) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Invalid SYN mask"); + return -rte_errno; + } + + /* check if the next not void item is TCP */ + item = next_no_void_pattern(pattern, item); + if (item->type != RTE_FLOW_ITEM_TYPE_TCP) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by syn filter"); + return -rte_errno; + } + } + + /* Get the TCP info. Only support SYN. */ + if (!item->spec || !item->mask) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Invalid SYN mask"); + return -rte_errno; + } + /*Not supported last point for range*/ + if (item->last) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + item, "Not supported last point for range"); + return -rte_errno; + } + + tcp_spec = (const struct rte_flow_item_tcp *)item->spec; + tcp_mask = (const struct rte_flow_item_tcp *)item->mask; + if (!(tcp_spec->hdr.tcp_flags & TCP_SYN_FLAG) || + tcp_mask->hdr.src_port || + tcp_mask->hdr.dst_port || + tcp_mask->hdr.sent_seq || + tcp_mask->hdr.recv_ack || + tcp_mask->hdr.data_off || + tcp_mask->hdr.tcp_flags != TCP_SYN_FLAG || + tcp_mask->hdr.rx_win || + tcp_mask->hdr.cksum || + tcp_mask->hdr.tcp_urp) { + memset(filter, 0, sizeof(struct rte_eth_syn_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by syn filter"); + return -rte_errno; + } + + /* check if the next not void item is END */ + item = next_no_void_pattern(pattern, item); + if (item->type != RTE_FLOW_ITEM_TYPE_END) { + memset(filter, 0, sizeof(struct rte_eth_syn_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by syn filter"); + return -rte_errno; + } + + /* check if the first not void action is QUEUE. */ + act = next_no_void_action(actions, NULL); + if (act->type != RTE_FLOW_ACTION_TYPE_QUEUE) { + memset(filter, 0, sizeof(struct rte_eth_syn_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + act, "Not supported action."); + return -rte_errno; + } + + act_q = (const struct rte_flow_action_queue *)act->conf; + filter->queue = act_q->index; + if (filter->queue >= IXGBE_MAX_RX_QUEUE_NUM) { + memset(filter, 0, sizeof(struct rte_eth_syn_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + act, "Not supported action."); + return -rte_errno; + } + + /* check if the next not void item is END */ + act = next_no_void_action(actions, act); + if (act->type != RTE_FLOW_ACTION_TYPE_END) { + memset(filter, 0, sizeof(struct rte_eth_syn_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + act, "Not supported action."); + return -rte_errno; + } + + /* parse attr */ + /* must be input direction */ + if (!attr->ingress) { + memset(filter, 0, sizeof(struct rte_eth_syn_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, + attr, "Only support ingress."); + return -rte_errno; + } + + /* not supported */ + if (attr->egress) { + memset(filter, 0, sizeof(struct rte_eth_syn_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, + attr, "Not support egress."); + return -rte_errno; + } + + /* Support 2 priorities, the lowest or highest. */ + if (!attr->priority) { + filter->hig_pri = 0; + } else if (attr->priority == (uint32_t)~0U) { + filter->hig_pri = 1; + } else { + memset(filter, 0, sizeof(struct rte_eth_syn_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, + attr, "Not support priority."); + return -rte_errno; + } + + return 0; +} + +static int +ixgbe_parse_syn_filter(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_eth_syn_filter *filter, + struct rte_flow_error *error) +{ + int ret; + struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + + MAC_TYPE_FILTER_SUP(hw->mac.type); + + ret = cons_parse_syn_filter(attr, pattern, + actions, filter, error); + + if (filter->queue >= dev->data->nb_rx_queues) + return -rte_errno; + + if (ret) + return ret; + + return 0; +} + +/** + * Parse the rule to see if it is a L2 tunnel rule. + * And get the L2 tunnel filter info BTW. + * Only support E-tag now. + * pattern: + * The first not void item can be E_TAG. + * The next not void item must be END. + * action: + * The first not void action should be VF or PF. + * The next not void action should be END. + * pattern example: + * ITEM Spec Mask + * E_TAG grp 0x1 0x3 + e_cid_base 0x309 0xFFF + * END + * other members in mask and spec should set to 0x00. + * item->last should be NULL. + */ +static int +cons_parse_l2_tn_filter(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_eth_l2_tunnel_conf *filter, + struct rte_flow_error *error) +{ + const struct rte_flow_item *item; + const struct rte_flow_item_e_tag *e_tag_spec; + const struct rte_flow_item_e_tag *e_tag_mask; + const struct rte_flow_action *act; + const struct rte_flow_action_vf *act_vf; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + + if (!pattern) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM_NUM, + NULL, "NULL pattern."); + return -rte_errno; + } + + if (!actions) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION_NUM, + NULL, "NULL action."); + return -rte_errno; + } + + if (!attr) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR, + NULL, "NULL attribute."); + return -rte_errno; + } + + /* The first not void item should be e-tag. */ + item = next_no_void_pattern(pattern, NULL); + if (item->type != RTE_FLOW_ITEM_TYPE_E_TAG) { + memset(filter, 0, sizeof(struct rte_eth_l2_tunnel_conf)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by L2 tunnel filter"); + return -rte_errno; + } + + if (!item->spec || !item->mask) { + memset(filter, 0, sizeof(struct rte_eth_l2_tunnel_conf)); + rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by L2 tunnel filter"); + return -rte_errno; + } + + /*Not supported last point for range*/ + if (item->last) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + item, "Not supported last point for range"); + return -rte_errno; + } + + e_tag_spec = (const struct rte_flow_item_e_tag *)item->spec; + e_tag_mask = (const struct rte_flow_item_e_tag *)item->mask; + + /* Only care about GRP and E cid base. */ + if (e_tag_mask->epcp_edei_in_ecid_b || + e_tag_mask->in_ecid_e || + e_tag_mask->ecid_e || + e_tag_mask->rsvd_grp_ecid_b != rte_cpu_to_be_16(0x3FFF)) { + memset(filter, 0, sizeof(struct rte_eth_l2_tunnel_conf)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by L2 tunnel filter"); + return -rte_errno; + } + + filter->l2_tunnel_type = RTE_L2_TUNNEL_TYPE_E_TAG; + /** + * grp and e_cid_base are bit fields and only use 14 bits. + * e-tag id is taken as little endian by HW. + */ + filter->tunnel_id = rte_be_to_cpu_16(e_tag_spec->rsvd_grp_ecid_b); + + /* check if the next not void item is END */ + item = next_no_void_pattern(pattern, item); + if (item->type != RTE_FLOW_ITEM_TYPE_END) { + memset(filter, 0, sizeof(struct rte_eth_l2_tunnel_conf)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by L2 tunnel filter"); + return -rte_errno; + } + + /* parse attr */ + /* must be input direction */ + if (!attr->ingress) { + memset(filter, 0, sizeof(struct rte_eth_l2_tunnel_conf)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, + attr, "Only support ingress."); + return -rte_errno; + } + + /* not supported */ + if (attr->egress) { + memset(filter, 0, sizeof(struct rte_eth_l2_tunnel_conf)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, + attr, "Not support egress."); + return -rte_errno; + } + + /* not supported */ + if (attr->priority) { + memset(filter, 0, sizeof(struct rte_eth_l2_tunnel_conf)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, + attr, "Not support priority."); + return -rte_errno; + } + + /* check if the first not void action is VF or PF. */ + act = next_no_void_action(actions, NULL); + if (act->type != RTE_FLOW_ACTION_TYPE_VF && + act->type != RTE_FLOW_ACTION_TYPE_PF) { + memset(filter, 0, sizeof(struct rte_eth_l2_tunnel_conf)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + act, "Not supported action."); + return -rte_errno; + } + + if (act->type == RTE_FLOW_ACTION_TYPE_VF) { + act_vf = (const struct rte_flow_action_vf *)act->conf; + filter->pool = act_vf->id; + } else { + filter->pool = pci_dev->max_vfs; + } + + /* check if the next not void item is END */ + act = next_no_void_action(actions, act); + if (act->type != RTE_FLOW_ACTION_TYPE_END) { + memset(filter, 0, sizeof(struct rte_eth_l2_tunnel_conf)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + act, "Not supported action."); + return -rte_errno; + } + + return 0; +} + +static int +ixgbe_parse_l2_tn_filter(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_eth_l2_tunnel_conf *l2_tn_filter, + struct rte_flow_error *error) +{ + int ret = 0; + struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + uint16_t vf_num; + + ret = cons_parse_l2_tn_filter(dev, attr, pattern, + actions, l2_tn_filter, error); + + if (hw->mac.type != ixgbe_mac_X550 && + hw->mac.type != ixgbe_mac_X550EM_x && + hw->mac.type != ixgbe_mac_X550EM_a) { + memset(l2_tn_filter, 0, sizeof(struct rte_eth_l2_tunnel_conf)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + NULL, "Not supported by L2 tunnel filter"); + return -rte_errno; + } + + vf_num = pci_dev->max_vfs; + + if (l2_tn_filter->pool > vf_num) + return -rte_errno; + + return ret; +} + +/* Parse to get the attr and action info of flow director rule. */ +static int +ixgbe_parse_fdir_act_attr(const struct rte_flow_attr *attr, + const struct rte_flow_action actions[], + struct ixgbe_fdir_rule *rule, + struct rte_flow_error *error) +{ + const struct rte_flow_action *act; + const struct rte_flow_action_queue *act_q; + const struct rte_flow_action_mark *mark; + + /* parse attr */ + /* must be input direction */ + if (!attr->ingress) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, + attr, "Only support ingress."); + return -rte_errno; + } + + /* not supported */ + if (attr->egress) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, + attr, "Not support egress."); + return -rte_errno; + } + + /* not supported */ + if (attr->priority) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, + attr, "Not support priority."); + return -rte_errno; + } + + /* check if the first not void action is QUEUE or DROP. */ + act = next_no_void_action(actions, NULL); + if (act->type != RTE_FLOW_ACTION_TYPE_QUEUE && + act->type != RTE_FLOW_ACTION_TYPE_DROP) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + act, "Not supported action."); + return -rte_errno; + } + + if (act->type == RTE_FLOW_ACTION_TYPE_QUEUE) { + act_q = (const struct rte_flow_action_queue *)act->conf; + rule->queue = act_q->index; + } else { /* drop */ + /* signature mode does not support drop action. */ + if (rule->mode == RTE_FDIR_MODE_SIGNATURE) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + act, "Not supported action."); + return -rte_errno; + } + rule->fdirflags = IXGBE_FDIRCMD_DROP; + } + + /* check if the next not void item is MARK */ + act = next_no_void_action(actions, act); + if ((act->type != RTE_FLOW_ACTION_TYPE_MARK) && + (act->type != RTE_FLOW_ACTION_TYPE_END)) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + act, "Not supported action."); + return -rte_errno; + } + + rule->soft_id = 0; + + if (act->type == RTE_FLOW_ACTION_TYPE_MARK) { + mark = (const struct rte_flow_action_mark *)act->conf; + rule->soft_id = mark->id; + act = next_no_void_action(actions, act); + } + + /* check if the next not void item is END */ + if (act->type != RTE_FLOW_ACTION_TYPE_END) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + act, "Not supported action."); + return -rte_errno; + } + + return 0; +} + +/* search next no void pattern and skip fuzzy */ +static inline +const struct rte_flow_item *next_no_fuzzy_pattern( + const struct rte_flow_item pattern[], + const struct rte_flow_item *cur) +{ + const struct rte_flow_item *next = + next_no_void_pattern(pattern, cur); + while (1) { + if (next->type != RTE_FLOW_ITEM_TYPE_FUZZY) + return next; + next = next_no_void_pattern(pattern, next); + } +} + +static inline uint8_t signature_match(const struct rte_flow_item pattern[]) +{ + const struct rte_flow_item_fuzzy *spec, *last, *mask; + const struct rte_flow_item *item; + uint32_t sh, lh, mh; + int i = 0; + + while (1) { + item = pattern + i; + if (item->type == RTE_FLOW_ITEM_TYPE_END) + break; + + if (item->type == RTE_FLOW_ITEM_TYPE_FUZZY) { + spec = + (const struct rte_flow_item_fuzzy *)item->spec; + last = + (const struct rte_flow_item_fuzzy *)item->last; + mask = + (const struct rte_flow_item_fuzzy *)item->mask; + + if (!spec || !mask) + return 0; + + sh = spec->thresh; + + if (!last) + lh = sh; + else + lh = last->thresh; + + mh = mask->thresh; + sh = sh & mh; + lh = lh & mh; + + if (!sh || sh > lh) + return 0; + + return 1; + } + + i++; + } + + return 0; +} + +/** + * Parse the rule to see if it is a IP or MAC VLAN flow director rule. + * And get the flow director filter info BTW. + * UDP/TCP/SCTP PATTERN: + * The first not void item can be ETH or IPV4 or IPV6 + * The second not void item must be IPV4 or IPV6 if the first one is ETH. + * The next not void item could be UDP or TCP or SCTP (optional) + * The next not void item could be RAW (for flexbyte, optional) + * The next not void item must be END. + * A Fuzzy Match pattern can appear at any place before END. + * Fuzzy Match is optional for IPV4 but is required for IPV6 + * MAC VLAN PATTERN: + * The first not void item must be ETH. + * The second not void item must be MAC VLAN. + * The next not void item must be END. + * ACTION: + * The first not void action should be QUEUE or DROP. + * The second not void optional action should be MARK, + * mark_id is a uint32_t number. + * The next not void action should be END. + * UDP/TCP/SCTP pattern example: + * ITEM Spec Mask + * ETH NULL NULL + * IPV4 src_addr 192.168.1.20 0xFFFFFFFF + * dst_addr 192.167.3.50 0xFFFFFFFF + * UDP/TCP/SCTP src_port 80 0xFFFF + * dst_port 80 0xFFFF + * FLEX relative 0 0x1 + * search 0 0x1 + * reserved 0 0 + * offset 12 0xFFFFFFFF + * limit 0 0xFFFF + * length 2 0xFFFF + * pattern[0] 0x86 0xFF + * pattern[1] 0xDD 0xFF + * END + * MAC VLAN pattern example: + * ITEM Spec Mask + * ETH dst_addr + {0xAC, 0x7B, 0xA1, {0xFF, 0xFF, 0xFF, + 0x2C, 0x6D, 0x36} 0xFF, 0xFF, 0xFF} + * MAC VLAN tci 0x2016 0xEFFF + * END + * Other members in mask and spec should set to 0x00. + * Item->last should be NULL. + */ +static int +ixgbe_parse_fdir_filter_normal(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct ixgbe_fdir_rule *rule, + struct rte_flow_error *error) +{ + const struct rte_flow_item *item; + const struct rte_flow_item_eth *eth_spec; + const struct rte_flow_item_eth *eth_mask; + const struct rte_flow_item_ipv4 *ipv4_spec; + const struct rte_flow_item_ipv4 *ipv4_mask; + const struct rte_flow_item_ipv6 *ipv6_spec; + const struct rte_flow_item_ipv6 *ipv6_mask; + const struct rte_flow_item_tcp *tcp_spec; + const struct rte_flow_item_tcp *tcp_mask; + const struct rte_flow_item_udp *udp_spec; + const struct rte_flow_item_udp *udp_mask; + const struct rte_flow_item_sctp *sctp_spec; + const struct rte_flow_item_sctp *sctp_mask; + const struct rte_flow_item_vlan *vlan_spec; + const struct rte_flow_item_vlan *vlan_mask; + const struct rte_flow_item_raw *raw_mask; + const struct rte_flow_item_raw *raw_spec; + uint8_t j; + + struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + + if (!pattern) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM_NUM, + NULL, "NULL pattern."); + return -rte_errno; + } + + if (!actions) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION_NUM, + NULL, "NULL action."); + return -rte_errno; + } + + if (!attr) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR, + NULL, "NULL attribute."); + return -rte_errno; + } + + /** + * Some fields may not be provided. Set spec to 0 and mask to default + * value. So, we need not do anything for the not provided fields later. + */ + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + memset(&rule->mask, 0xFF, sizeof(struct ixgbe_hw_fdir_mask)); + rule->mask.vlan_tci_mask = 0; + rule->mask.flex_bytes_mask = 0; + + /** + * The first not void item should be + * MAC or IPv4 or TCP or UDP or SCTP. + */ + item = next_no_fuzzy_pattern(pattern, NULL); + if (item->type != RTE_FLOW_ITEM_TYPE_ETH && + item->type != RTE_FLOW_ITEM_TYPE_IPV4 && + item->type != RTE_FLOW_ITEM_TYPE_IPV6 && + item->type != RTE_FLOW_ITEM_TYPE_TCP && + item->type != RTE_FLOW_ITEM_TYPE_UDP && + item->type != RTE_FLOW_ITEM_TYPE_SCTP) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + + if (signature_match(pattern)) + rule->mode = RTE_FDIR_MODE_SIGNATURE; + else + rule->mode = RTE_FDIR_MODE_PERFECT; + + /*Not supported last point for range*/ + if (item->last) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + item, "Not supported last point for range"); + return -rte_errno; + } + + /* Get the MAC info. */ + if (item->type == RTE_FLOW_ITEM_TYPE_ETH) { + /** + * Only support vlan and dst MAC address, + * others should be masked. + */ + if (item->spec && !item->mask) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + + if (item->spec) { + rule->b_spec = TRUE; + eth_spec = (const struct rte_flow_item_eth *)item->spec; + + /* Get the dst MAC. */ + for (j = 0; j < ETHER_ADDR_LEN; j++) { + rule->ixgbe_fdir.formatted.inner_mac[j] = + eth_spec->dst.addr_bytes[j]; + } + } + + + if (item->mask) { + + rule->b_mask = TRUE; + eth_mask = (const struct rte_flow_item_eth *)item->mask; + + /* Ether type should be masked. */ + if (eth_mask->type || + rule->mode == RTE_FDIR_MODE_SIGNATURE) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + + /* If ethernet has meaning, it means MAC VLAN mode. */ + rule->mode = RTE_FDIR_MODE_PERFECT_MAC_VLAN; + + /** + * src MAC address must be masked, + * and don't support dst MAC address mask. + */ + for (j = 0; j < ETHER_ADDR_LEN; j++) { + if (eth_mask->src.addr_bytes[j] || + eth_mask->dst.addr_bytes[j] != 0xFF) { + memset(rule, 0, + sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + } + + /* When no VLAN, considered as full mask. */ + rule->mask.vlan_tci_mask = rte_cpu_to_be_16(0xEFFF); + } + /*** If both spec and mask are item, + * it means don't care about ETH. + * Do nothing. + */ + + /** + * Check if the next not void item is vlan or ipv4. + * IPv6 is not supported. + */ + item = next_no_fuzzy_pattern(pattern, item); + if (rule->mode == RTE_FDIR_MODE_PERFECT_MAC_VLAN) { + if (item->type != RTE_FLOW_ITEM_TYPE_VLAN) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + } else { + if (item->type != RTE_FLOW_ITEM_TYPE_IPV4) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + } + } + + if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) { + if (!(item->spec && item->mask)) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + + /*Not supported last point for range*/ + if (item->last) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + item, "Not supported last point for range"); + return -rte_errno; + } + + vlan_spec = (const struct rte_flow_item_vlan *)item->spec; + vlan_mask = (const struct rte_flow_item_vlan *)item->mask; + + rule->ixgbe_fdir.formatted.vlan_id = vlan_spec->tci; + + rule->mask.vlan_tci_mask = vlan_mask->tci; + rule->mask.vlan_tci_mask &= rte_cpu_to_be_16(0xEFFF); + /* More than one tags are not supported. */ + + /* Next not void item must be END */ + item = next_no_fuzzy_pattern(pattern, item); + if (item->type != RTE_FLOW_ITEM_TYPE_END) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + } + + /* Get the IPV4 info. */ + if (item->type == RTE_FLOW_ITEM_TYPE_IPV4) { + /** + * Set the flow type even if there's no content + * as we must have a flow type. + */ + rule->ixgbe_fdir.formatted.flow_type = + IXGBE_ATR_FLOW_TYPE_IPV4; + /*Not supported last point for range*/ + if (item->last) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + item, "Not supported last point for range"); + return -rte_errno; + } + /** + * Only care about src & dst addresses, + * others should be masked. + */ + if (!item->mask) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + rule->b_mask = TRUE; + ipv4_mask = + (const struct rte_flow_item_ipv4 *)item->mask; + if (ipv4_mask->hdr.version_ihl || + ipv4_mask->hdr.type_of_service || + ipv4_mask->hdr.total_length || + ipv4_mask->hdr.packet_id || + ipv4_mask->hdr.fragment_offset || + ipv4_mask->hdr.time_to_live || + ipv4_mask->hdr.next_proto_id || + ipv4_mask->hdr.hdr_checksum) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + rule->mask.dst_ipv4_mask = ipv4_mask->hdr.dst_addr; + rule->mask.src_ipv4_mask = ipv4_mask->hdr.src_addr; + + if (item->spec) { + rule->b_spec = TRUE; + ipv4_spec = + (const struct rte_flow_item_ipv4 *)item->spec; + rule->ixgbe_fdir.formatted.dst_ip[0] = + ipv4_spec->hdr.dst_addr; + rule->ixgbe_fdir.formatted.src_ip[0] = + ipv4_spec->hdr.src_addr; + } + + /** + * Check if the next not void item is + * TCP or UDP or SCTP or END. + */ + item = next_no_fuzzy_pattern(pattern, item); + if (item->type != RTE_FLOW_ITEM_TYPE_TCP && + item->type != RTE_FLOW_ITEM_TYPE_UDP && + item->type != RTE_FLOW_ITEM_TYPE_SCTP && + item->type != RTE_FLOW_ITEM_TYPE_END && + item->type != RTE_FLOW_ITEM_TYPE_RAW) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + } + + /* Get the IPV6 info. */ + if (item->type == RTE_FLOW_ITEM_TYPE_IPV6) { + /** + * Set the flow type even if there's no content + * as we must have a flow type. + */ + rule->ixgbe_fdir.formatted.flow_type = + IXGBE_ATR_FLOW_TYPE_IPV6; + + /** + * 1. must signature match + * 2. not support last + * 3. mask must not null + */ + if (rule->mode != RTE_FDIR_MODE_SIGNATURE || + item->last || + !item->mask) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + item, "Not supported last point for range"); + return -rte_errno; + } + + rule->b_mask = TRUE; + ipv6_mask = + (const struct rte_flow_item_ipv6 *)item->mask; + if (ipv6_mask->hdr.vtc_flow || + ipv6_mask->hdr.payload_len || + ipv6_mask->hdr.proto || + ipv6_mask->hdr.hop_limits) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + + /* check src addr mask */ + for (j = 0; j < 16; j++) { + if (ipv6_mask->hdr.src_addr[j] == UINT8_MAX) { + rule->mask.src_ipv6_mask |= 1 << j; + } else if (ipv6_mask->hdr.src_addr[j] != 0) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + } + + /* check dst addr mask */ + for (j = 0; j < 16; j++) { + if (ipv6_mask->hdr.dst_addr[j] == UINT8_MAX) { + rule->mask.dst_ipv6_mask |= 1 << j; + } else if (ipv6_mask->hdr.dst_addr[j] != 0) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + } + + if (item->spec) { + rule->b_spec = TRUE; + ipv6_spec = + (const struct rte_flow_item_ipv6 *)item->spec; + rte_memcpy(rule->ixgbe_fdir.formatted.src_ip, + ipv6_spec->hdr.src_addr, 16); + rte_memcpy(rule->ixgbe_fdir.formatted.dst_ip, + ipv6_spec->hdr.dst_addr, 16); + } + + /** + * Check if the next not void item is + * TCP or UDP or SCTP or END. + */ + item = next_no_fuzzy_pattern(pattern, item); + if (item->type != RTE_FLOW_ITEM_TYPE_TCP && + item->type != RTE_FLOW_ITEM_TYPE_UDP && + item->type != RTE_FLOW_ITEM_TYPE_SCTP && + item->type != RTE_FLOW_ITEM_TYPE_END && + item->type != RTE_FLOW_ITEM_TYPE_RAW) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + } + + /* Get the TCP info. */ + if (item->type == RTE_FLOW_ITEM_TYPE_TCP) { + /** + * Set the flow type even if there's no content + * as we must have a flow type. + */ + rule->ixgbe_fdir.formatted.flow_type |= + IXGBE_ATR_L4TYPE_TCP; + /*Not supported last point for range*/ + if (item->last) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + item, "Not supported last point for range"); + return -rte_errno; + } + /** + * Only care about src & dst ports, + * others should be masked. + */ + if (!item->mask) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + rule->b_mask = TRUE; + tcp_mask = (const struct rte_flow_item_tcp *)item->mask; + if (tcp_mask->hdr.sent_seq || + tcp_mask->hdr.recv_ack || + tcp_mask->hdr.data_off || + tcp_mask->hdr.tcp_flags || + tcp_mask->hdr.rx_win || + tcp_mask->hdr.cksum || + tcp_mask->hdr.tcp_urp) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + rule->mask.src_port_mask = tcp_mask->hdr.src_port; + rule->mask.dst_port_mask = tcp_mask->hdr.dst_port; + + if (item->spec) { + rule->b_spec = TRUE; + tcp_spec = (const struct rte_flow_item_tcp *)item->spec; + rule->ixgbe_fdir.formatted.src_port = + tcp_spec->hdr.src_port; + rule->ixgbe_fdir.formatted.dst_port = + tcp_spec->hdr.dst_port; + } + + item = next_no_fuzzy_pattern(pattern, item); + if (item->type != RTE_FLOW_ITEM_TYPE_RAW && + item->type != RTE_FLOW_ITEM_TYPE_END) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + + } + + /* Get the UDP info */ + if (item->type == RTE_FLOW_ITEM_TYPE_UDP) { + /** + * Set the flow type even if there's no content + * as we must have a flow type. + */ + rule->ixgbe_fdir.formatted.flow_type |= + IXGBE_ATR_L4TYPE_UDP; + /*Not supported last point for range*/ + if (item->last) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + item, "Not supported last point for range"); + return -rte_errno; + } + /** + * Only care about src & dst ports, + * others should be masked. + */ + if (!item->mask) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + rule->b_mask = TRUE; + udp_mask = (const struct rte_flow_item_udp *)item->mask; + if (udp_mask->hdr.dgram_len || + udp_mask->hdr.dgram_cksum) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + rule->mask.src_port_mask = udp_mask->hdr.src_port; + rule->mask.dst_port_mask = udp_mask->hdr.dst_port; + + if (item->spec) { + rule->b_spec = TRUE; + udp_spec = (const struct rte_flow_item_udp *)item->spec; + rule->ixgbe_fdir.formatted.src_port = + udp_spec->hdr.src_port; + rule->ixgbe_fdir.formatted.dst_port = + udp_spec->hdr.dst_port; + } + + item = next_no_fuzzy_pattern(pattern, item); + if (item->type != RTE_FLOW_ITEM_TYPE_RAW && + item->type != RTE_FLOW_ITEM_TYPE_END) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + + } + + /* Get the SCTP info */ + if (item->type == RTE_FLOW_ITEM_TYPE_SCTP) { + /** + * Set the flow type even if there's no content + * as we must have a flow type. + */ + rule->ixgbe_fdir.formatted.flow_type |= + IXGBE_ATR_L4TYPE_SCTP; + /*Not supported last point for range*/ + if (item->last) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + item, "Not supported last point for range"); + return -rte_errno; + } + + /* only x550 family only support sctp port */ + if (hw->mac.type == ixgbe_mac_X550 || + hw->mac.type == ixgbe_mac_X550EM_x || + hw->mac.type == ixgbe_mac_X550EM_a) { + /** + * Only care about src & dst ports, + * others should be masked. + */ + if (!item->mask) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + rule->b_mask = TRUE; + sctp_mask = + (const struct rte_flow_item_sctp *)item->mask; + if (sctp_mask->hdr.tag || + sctp_mask->hdr.cksum) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + rule->mask.src_port_mask = sctp_mask->hdr.src_port; + rule->mask.dst_port_mask = sctp_mask->hdr.dst_port; + + if (item->spec) { + rule->b_spec = TRUE; + sctp_spec = + (const struct rte_flow_item_sctp *)item->spec; + rule->ixgbe_fdir.formatted.src_port = + sctp_spec->hdr.src_port; + rule->ixgbe_fdir.formatted.dst_port = + sctp_spec->hdr.dst_port; + } + /* others even sctp port is not supported */ + } else { + sctp_mask = + (const struct rte_flow_item_sctp *)item->mask; + if (sctp_mask && + (sctp_mask->hdr.src_port || + sctp_mask->hdr.dst_port || + sctp_mask->hdr.tag || + sctp_mask->hdr.cksum)) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + } + + item = next_no_fuzzy_pattern(pattern, item); + if (item->type != RTE_FLOW_ITEM_TYPE_RAW && + item->type != RTE_FLOW_ITEM_TYPE_END) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + } + + /* Get the flex byte info */ + if (item->type == RTE_FLOW_ITEM_TYPE_RAW) { + /* Not supported last point for range*/ + if (item->last) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + item, "Not supported last point for range"); + return -rte_errno; + } + /* mask should not be null */ + if (!item->mask || !item->spec) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + + raw_mask = (const struct rte_flow_item_raw *)item->mask; + + /* check mask */ + if (raw_mask->relative != 0x1 || + raw_mask->search != 0x1 || + raw_mask->reserved != 0x0 || + (uint32_t)raw_mask->offset != 0xffffffff || + raw_mask->limit != 0xffff || + raw_mask->length != 0xffff) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + + raw_spec = (const struct rte_flow_item_raw *)item->spec; + + /* check spec */ + if (raw_spec->relative != 0 || + raw_spec->search != 0 || + raw_spec->reserved != 0 || + raw_spec->offset > IXGBE_MAX_FLX_SOURCE_OFF || + raw_spec->offset % 2 || + raw_spec->limit != 0 || + raw_spec->length != 2 || + /* pattern can't be 0xffff */ + (raw_spec->pattern[0] == 0xff && + raw_spec->pattern[1] == 0xff)) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + + /* check pattern mask */ + if (raw_mask->pattern[0] != 0xff || + raw_mask->pattern[1] != 0xff) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + + rule->mask.flex_bytes_mask = 0xffff; + rule->ixgbe_fdir.formatted.flex_bytes = + (((uint16_t)raw_spec->pattern[1]) << 8) | + raw_spec->pattern[0]; + rule->flex_bytes_offset = raw_spec->offset; + } + + if (item->type != RTE_FLOW_ITEM_TYPE_END) { + /* check if the next not void item is END */ + item = next_no_fuzzy_pattern(pattern, item); + if (item->type != RTE_FLOW_ITEM_TYPE_END) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + } + + return ixgbe_parse_fdir_act_attr(attr, actions, rule, error); +} + +#define NVGRE_PROTOCOL 0x6558 + +/** + * Parse the rule to see if it is a VxLAN or NVGRE flow director rule. + * And get the flow director filter info BTW. + * VxLAN PATTERN: + * The first not void item must be ETH. + * The second not void item must be IPV4/ IPV6. + * The third not void item must be NVGRE. + * The next not void item must be END. + * NVGRE PATTERN: + * The first not void item must be ETH. + * The second not void item must be IPV4/ IPV6. + * The third not void item must be NVGRE. + * The next not void item must be END. + * ACTION: + * The first not void action should be QUEUE or DROP. + * The second not void optional action should be MARK, + * mark_id is a uint32_t number. + * The next not void action should be END. + * VxLAN pattern example: + * ITEM Spec Mask + * ETH NULL NULL + * IPV4/IPV6 NULL NULL + * UDP NULL NULL + * VxLAN vni{0x00, 0x32, 0x54} {0xFF, 0xFF, 0xFF} + * MAC VLAN tci 0x2016 0xEFFF + * END + * NEGRV pattern example: + * ITEM Spec Mask + * ETH NULL NULL + * IPV4/IPV6 NULL NULL + * NVGRE protocol 0x6558 0xFFFF + * tni{0x00, 0x32, 0x54} {0xFF, 0xFF, 0xFF} + * MAC VLAN tci 0x2016 0xEFFF + * END + * other members in mask and spec should set to 0x00. + * item->last should be NULL. + */ +static int +ixgbe_parse_fdir_filter_tunnel(const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct ixgbe_fdir_rule *rule, + struct rte_flow_error *error) +{ + const struct rte_flow_item *item; + const struct rte_flow_item_vxlan *vxlan_spec; + const struct rte_flow_item_vxlan *vxlan_mask; + const struct rte_flow_item_nvgre *nvgre_spec; + const struct rte_flow_item_nvgre *nvgre_mask; + const struct rte_flow_item_eth *eth_spec; + const struct rte_flow_item_eth *eth_mask; + const struct rte_flow_item_vlan *vlan_spec; + const struct rte_flow_item_vlan *vlan_mask; + uint32_t j; + + if (!pattern) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM_NUM, + NULL, "NULL pattern."); + return -rte_errno; + } + + if (!actions) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION_NUM, + NULL, "NULL action."); + return -rte_errno; + } + + if (!attr) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR, + NULL, "NULL attribute."); + return -rte_errno; + } + + /** + * Some fields may not be provided. Set spec to 0 and mask to default + * value. So, we need not do anything for the not provided fields later. + */ + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + memset(&rule->mask, 0xFF, sizeof(struct ixgbe_hw_fdir_mask)); + rule->mask.vlan_tci_mask = 0; + + /** + * The first not void item should be + * MAC or IPv4 or IPv6 or UDP or VxLAN. + */ + item = next_no_void_pattern(pattern, NULL); + if (item->type != RTE_FLOW_ITEM_TYPE_ETH && + item->type != RTE_FLOW_ITEM_TYPE_IPV4 && + item->type != RTE_FLOW_ITEM_TYPE_IPV6 && + item->type != RTE_FLOW_ITEM_TYPE_UDP && + item->type != RTE_FLOW_ITEM_TYPE_VXLAN && + item->type != RTE_FLOW_ITEM_TYPE_NVGRE) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + + rule->mode = RTE_FDIR_MODE_PERFECT_TUNNEL; + + /* Skip MAC. */ + if (item->type == RTE_FLOW_ITEM_TYPE_ETH) { + /* Only used to describe the protocol stack. */ + if (item->spec || item->mask) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + /* Not supported last point for range*/ + if (item->last) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + item, "Not supported last point for range"); + return -rte_errno; + } + + /* Check if the next not void item is IPv4 or IPv6. */ + item = next_no_void_pattern(pattern, item); + if (item->type != RTE_FLOW_ITEM_TYPE_IPV4 && + item->type != RTE_FLOW_ITEM_TYPE_IPV6) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + } + + /* Skip IP. */ + if (item->type == RTE_FLOW_ITEM_TYPE_IPV4 || + item->type == RTE_FLOW_ITEM_TYPE_IPV6) { + /* Only used to describe the protocol stack. */ + if (item->spec || item->mask) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + /*Not supported last point for range*/ + if (item->last) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + item, "Not supported last point for range"); + return -rte_errno; + } + + /* Check if the next not void item is UDP or NVGRE. */ + item = next_no_void_pattern(pattern, item); + if (item->type != RTE_FLOW_ITEM_TYPE_UDP && + item->type != RTE_FLOW_ITEM_TYPE_NVGRE) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + } + + /* Skip UDP. */ + if (item->type == RTE_FLOW_ITEM_TYPE_UDP) { + /* Only used to describe the protocol stack. */ + if (item->spec || item->mask) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + /*Not supported last point for range*/ + if (item->last) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + item, "Not supported last point for range"); + return -rte_errno; + } + + /* Check if the next not void item is VxLAN. */ + item = next_no_void_pattern(pattern, item); + if (item->type != RTE_FLOW_ITEM_TYPE_VXLAN) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + } + + /* Get the VxLAN info */ + if (item->type == RTE_FLOW_ITEM_TYPE_VXLAN) { + rule->ixgbe_fdir.formatted.tunnel_type = + RTE_FDIR_TUNNEL_TYPE_VXLAN; + + /* Only care about VNI, others should be masked. */ + if (!item->mask) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + /*Not supported last point for range*/ + if (item->last) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + item, "Not supported last point for range"); + return -rte_errno; + } + rule->b_mask = TRUE; + + /* Tunnel type is always meaningful. */ + rule->mask.tunnel_type_mask = 1; + + vxlan_mask = + (const struct rte_flow_item_vxlan *)item->mask; + if (vxlan_mask->flags) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + /* VNI must be totally masked or not. */ + if ((vxlan_mask->vni[0] || vxlan_mask->vni[1] || + vxlan_mask->vni[2]) && + ((vxlan_mask->vni[0] != 0xFF) || + (vxlan_mask->vni[1] != 0xFF) || + (vxlan_mask->vni[2] != 0xFF))) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + + rte_memcpy(&rule->mask.tunnel_id_mask, vxlan_mask->vni, + RTE_DIM(vxlan_mask->vni)); + + if (item->spec) { + rule->b_spec = TRUE; + vxlan_spec = (const struct rte_flow_item_vxlan *) + item->spec; + rte_memcpy(((uint8_t *) + &rule->ixgbe_fdir.formatted.tni_vni + 1), + vxlan_spec->vni, RTE_DIM(vxlan_spec->vni)); + rule->ixgbe_fdir.formatted.tni_vni = rte_be_to_cpu_32( + rule->ixgbe_fdir.formatted.tni_vni); + } + } + + /* Get the NVGRE info */ + if (item->type == RTE_FLOW_ITEM_TYPE_NVGRE) { + rule->ixgbe_fdir.formatted.tunnel_type = + RTE_FDIR_TUNNEL_TYPE_NVGRE; + + /** + * Only care about flags0, flags1, protocol and TNI, + * others should be masked. + */ + if (!item->mask) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + /*Not supported last point for range*/ + if (item->last) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + item, "Not supported last point for range"); + return -rte_errno; + } + rule->b_mask = TRUE; + + /* Tunnel type is always meaningful. */ + rule->mask.tunnel_type_mask = 1; + + nvgre_mask = + (const struct rte_flow_item_nvgre *)item->mask; + if (nvgre_mask->flow_id) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + if (nvgre_mask->protocol && + nvgre_mask->protocol != 0xFFFF) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + if (nvgre_mask->c_k_s_rsvd0_ver && + nvgre_mask->c_k_s_rsvd0_ver != + rte_cpu_to_be_16(0xFFFF)) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + /* TNI must be totally masked or not. */ + if (nvgre_mask->tni[0] && + ((nvgre_mask->tni[0] != 0xFF) || + (nvgre_mask->tni[1] != 0xFF) || + (nvgre_mask->tni[2] != 0xFF))) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + /* tni is a 24-bits bit field */ + rte_memcpy(&rule->mask.tunnel_id_mask, nvgre_mask->tni, + RTE_DIM(nvgre_mask->tni)); + rule->mask.tunnel_id_mask <<= 8; + + if (item->spec) { + rule->b_spec = TRUE; + nvgre_spec = + (const struct rte_flow_item_nvgre *)item->spec; + if (nvgre_spec->c_k_s_rsvd0_ver != + rte_cpu_to_be_16(0x2000) && + nvgre_mask->c_k_s_rsvd0_ver) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + if (nvgre_mask->protocol && + nvgre_spec->protocol != + rte_cpu_to_be_16(NVGRE_PROTOCOL)) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + /* tni is a 24-bits bit field */ + rte_memcpy(&rule->ixgbe_fdir.formatted.tni_vni, + nvgre_spec->tni, RTE_DIM(nvgre_spec->tni)); + rule->ixgbe_fdir.formatted.tni_vni <<= 8; + } + } + + /* check if the next not void item is MAC */ + item = next_no_void_pattern(pattern, item); + if (item->type != RTE_FLOW_ITEM_TYPE_ETH) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + + /** + * Only support vlan and dst MAC address, + * others should be masked. + */ + + if (!item->mask) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + /*Not supported last point for range*/ + if (item->last) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + item, "Not supported last point for range"); + return -rte_errno; + } + rule->b_mask = TRUE; + eth_mask = (const struct rte_flow_item_eth *)item->mask; + + /* Ether type should be masked. */ + if (eth_mask->type) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + + /* src MAC address should be masked. */ + for (j = 0; j < ETHER_ADDR_LEN; j++) { + if (eth_mask->src.addr_bytes[j]) { + memset(rule, 0, + sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + } + rule->mask.mac_addr_byte_mask = 0; + for (j = 0; j < ETHER_ADDR_LEN; j++) { + /* It's a per byte mask. */ + if (eth_mask->dst.addr_bytes[j] == 0xFF) { + rule->mask.mac_addr_byte_mask |= 0x1 << j; + } else if (eth_mask->dst.addr_bytes[j]) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + } + + /* When no vlan, considered as full mask. */ + rule->mask.vlan_tci_mask = rte_cpu_to_be_16(0xEFFF); + + if (item->spec) { + rule->b_spec = TRUE; + eth_spec = (const struct rte_flow_item_eth *)item->spec; + + /* Get the dst MAC. */ + for (j = 0; j < ETHER_ADDR_LEN; j++) { + rule->ixgbe_fdir.formatted.inner_mac[j] = + eth_spec->dst.addr_bytes[j]; + } + } + + /** + * Check if the next not void item is vlan or ipv4. + * IPv6 is not supported. + */ + item = next_no_void_pattern(pattern, item); + if ((item->type != RTE_FLOW_ITEM_TYPE_VLAN) && + (item->type != RTE_FLOW_ITEM_TYPE_IPV4)) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + /*Not supported last point for range*/ + if (item->last) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + item, "Not supported last point for range"); + return -rte_errno; + } + + if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) { + if (!(item->spec && item->mask)) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + + vlan_spec = (const struct rte_flow_item_vlan *)item->spec; + vlan_mask = (const struct rte_flow_item_vlan *)item->mask; + + rule->ixgbe_fdir.formatted.vlan_id = vlan_spec->tci; + + rule->mask.vlan_tci_mask = vlan_mask->tci; + rule->mask.vlan_tci_mask &= rte_cpu_to_be_16(0xEFFF); + /* More than one tags are not supported. */ + + /* check if the next not void item is END */ + item = next_no_void_pattern(pattern, item); + + if (item->type != RTE_FLOW_ITEM_TYPE_END) { + memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by fdir filter"); + return -rte_errno; + } + } + + /** + * If the tags is 0, it means don't care about the VLAN. + * Do nothing. + */ + + return ixgbe_parse_fdir_act_attr(attr, actions, rule, error); +} + +static int +ixgbe_parse_fdir_filter(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct ixgbe_fdir_rule *rule, + struct rte_flow_error *error) +{ + int ret; + struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + enum rte_fdir_mode fdir_mode = dev->data->dev_conf.fdir_conf.mode; + + if (hw->mac.type != ixgbe_mac_82599EB && + hw->mac.type != ixgbe_mac_X540 && + hw->mac.type != ixgbe_mac_X550 && + hw->mac.type != ixgbe_mac_X550EM_x && + hw->mac.type != ixgbe_mac_X550EM_a) + return -ENOTSUP; + + ret = ixgbe_parse_fdir_filter_normal(dev, attr, pattern, + actions, rule, error); + + if (!ret) + goto step_next; + + ret = ixgbe_parse_fdir_filter_tunnel(attr, pattern, + actions, rule, error); + + if (ret) + return ret; + +step_next: + + if (hw->mac.type == ixgbe_mac_82599EB && + rule->fdirflags == IXGBE_FDIRCMD_DROP && + (rule->ixgbe_fdir.formatted.src_port != 0 || + rule->ixgbe_fdir.formatted.dst_port != 0)) + return -ENOTSUP; + + if (fdir_mode == RTE_FDIR_MODE_NONE || + fdir_mode != rule->mode) + return -ENOTSUP; + + if (rule->queue >= dev->data->nb_rx_queues) + return -ENOTSUP; + + return ret; +} + +void +ixgbe_filterlist_init(void) +{ + TAILQ_INIT(&filter_ntuple_list); + TAILQ_INIT(&filter_ethertype_list); + TAILQ_INIT(&filter_syn_list); + TAILQ_INIT(&filter_fdir_list); + TAILQ_INIT(&filter_l2_tunnel_list); + TAILQ_INIT(&ixgbe_flow_list); +} + +void +ixgbe_filterlist_flush(void) +{ + struct ixgbe_ntuple_filter_ele *ntuple_filter_ptr; + struct ixgbe_ethertype_filter_ele *ethertype_filter_ptr; + struct ixgbe_eth_syn_filter_ele *syn_filter_ptr; + struct ixgbe_eth_l2_tunnel_conf_ele *l2_tn_filter_ptr; + struct ixgbe_fdir_rule_ele *fdir_rule_ptr; + struct ixgbe_flow_mem *ixgbe_flow_mem_ptr; + + while ((ntuple_filter_ptr = TAILQ_FIRST(&filter_ntuple_list))) { + TAILQ_REMOVE(&filter_ntuple_list, + ntuple_filter_ptr, + entries); + rte_free(ntuple_filter_ptr); + } + + while ((ethertype_filter_ptr = TAILQ_FIRST(&filter_ethertype_list))) { + TAILQ_REMOVE(&filter_ethertype_list, + ethertype_filter_ptr, + entries); + rte_free(ethertype_filter_ptr); + } + + while ((syn_filter_ptr = TAILQ_FIRST(&filter_syn_list))) { + TAILQ_REMOVE(&filter_syn_list, + syn_filter_ptr, + entries); + rte_free(syn_filter_ptr); + } + + while ((l2_tn_filter_ptr = TAILQ_FIRST(&filter_l2_tunnel_list))) { + TAILQ_REMOVE(&filter_l2_tunnel_list, + l2_tn_filter_ptr, + entries); + rte_free(l2_tn_filter_ptr); + } + + while ((fdir_rule_ptr = TAILQ_FIRST(&filter_fdir_list))) { + TAILQ_REMOVE(&filter_fdir_list, + fdir_rule_ptr, + entries); + rte_free(fdir_rule_ptr); + } + + while ((ixgbe_flow_mem_ptr = TAILQ_FIRST(&ixgbe_flow_list))) { + TAILQ_REMOVE(&ixgbe_flow_list, + ixgbe_flow_mem_ptr, + entries); + rte_free(ixgbe_flow_mem_ptr->flow); + rte_free(ixgbe_flow_mem_ptr); + } +} + +/** + * Create or destroy a flow rule. + * Theorically one rule can match more than one filters. + * We will let it use the filter which it hitt first. + * So, the sequence matters. + */ +static struct rte_flow * +ixgbe_flow_create(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error) +{ + int ret; + struct rte_eth_ntuple_filter ntuple_filter; + struct rte_eth_ethertype_filter ethertype_filter; + struct rte_eth_syn_filter syn_filter; + struct ixgbe_fdir_rule fdir_rule; + struct rte_eth_l2_tunnel_conf l2_tn_filter; + struct ixgbe_hw_fdir_info *fdir_info = + IXGBE_DEV_PRIVATE_TO_FDIR_INFO(dev->data->dev_private); + struct rte_flow *flow = NULL; + struct ixgbe_ntuple_filter_ele *ntuple_filter_ptr; + struct ixgbe_ethertype_filter_ele *ethertype_filter_ptr; + struct ixgbe_eth_syn_filter_ele *syn_filter_ptr; + struct ixgbe_eth_l2_tunnel_conf_ele *l2_tn_filter_ptr; + struct ixgbe_fdir_rule_ele *fdir_rule_ptr; + struct ixgbe_flow_mem *ixgbe_flow_mem_ptr; + uint8_t first_mask = FALSE; + + flow = rte_zmalloc("ixgbe_rte_flow", sizeof(struct rte_flow), 0); + if (!flow) { + PMD_DRV_LOG(ERR, "failed to allocate memory"); + return (struct rte_flow *)flow; + } + ixgbe_flow_mem_ptr = rte_zmalloc("ixgbe_flow_mem", + sizeof(struct ixgbe_flow_mem), 0); + if (!ixgbe_flow_mem_ptr) { + PMD_DRV_LOG(ERR, "failed to allocate memory"); + rte_free(flow); + return NULL; + } + ixgbe_flow_mem_ptr->flow = flow; + TAILQ_INSERT_TAIL(&ixgbe_flow_list, + ixgbe_flow_mem_ptr, entries); + + memset(&ntuple_filter, 0, sizeof(struct rte_eth_ntuple_filter)); + ret = ixgbe_parse_ntuple_filter(dev, attr, pattern, + actions, &ntuple_filter, error); + +#ifdef RTE_LIBRTE_SECURITY + /* ESP flow not really a flow*/ + if (ntuple_filter.proto == IPPROTO_ESP) + return flow; +#endif + + if (!ret) { + ret = ixgbe_add_del_ntuple_filter(dev, &ntuple_filter, TRUE); + if (!ret) { + ntuple_filter_ptr = rte_zmalloc("ixgbe_ntuple_filter", + sizeof(struct ixgbe_ntuple_filter_ele), 0); + if (!ntuple_filter_ptr) { + PMD_DRV_LOG(ERR, "failed to allocate memory"); + goto out; + } + rte_memcpy(&ntuple_filter_ptr->filter_info, + &ntuple_filter, + sizeof(struct rte_eth_ntuple_filter)); + TAILQ_INSERT_TAIL(&filter_ntuple_list, + ntuple_filter_ptr, entries); + flow->rule = ntuple_filter_ptr; + flow->filter_type = RTE_ETH_FILTER_NTUPLE; + return flow; + } + goto out; + } + + memset(ðertype_filter, 0, sizeof(struct rte_eth_ethertype_filter)); + ret = ixgbe_parse_ethertype_filter(dev, attr, pattern, + actions, ðertype_filter, error); + if (!ret) { + ret = ixgbe_add_del_ethertype_filter(dev, + ðertype_filter, TRUE); + if (!ret) { + ethertype_filter_ptr = rte_zmalloc( + "ixgbe_ethertype_filter", + sizeof(struct ixgbe_ethertype_filter_ele), 0); + if (!ethertype_filter_ptr) { + PMD_DRV_LOG(ERR, "failed to allocate memory"); + goto out; + } + rte_memcpy(ðertype_filter_ptr->filter_info, + ðertype_filter, + sizeof(struct rte_eth_ethertype_filter)); + TAILQ_INSERT_TAIL(&filter_ethertype_list, + ethertype_filter_ptr, entries); + flow->rule = ethertype_filter_ptr; + flow->filter_type = RTE_ETH_FILTER_ETHERTYPE; + return flow; + } + goto out; + } + + memset(&syn_filter, 0, sizeof(struct rte_eth_syn_filter)); + ret = ixgbe_parse_syn_filter(dev, attr, pattern, + actions, &syn_filter, error); + if (!ret) { + ret = ixgbe_syn_filter_set(dev, &syn_filter, TRUE); + if (!ret) { + syn_filter_ptr = rte_zmalloc("ixgbe_syn_filter", + sizeof(struct ixgbe_eth_syn_filter_ele), 0); + if (!syn_filter_ptr) { + PMD_DRV_LOG(ERR, "failed to allocate memory"); + goto out; + } + rte_memcpy(&syn_filter_ptr->filter_info, + &syn_filter, + sizeof(struct rte_eth_syn_filter)); + TAILQ_INSERT_TAIL(&filter_syn_list, + syn_filter_ptr, + entries); + flow->rule = syn_filter_ptr; + flow->filter_type = RTE_ETH_FILTER_SYN; + return flow; + } + goto out; + } + + memset(&fdir_rule, 0, sizeof(struct ixgbe_fdir_rule)); + ret = ixgbe_parse_fdir_filter(dev, attr, pattern, + actions, &fdir_rule, error); + if (!ret) { + /* A mask cannot be deleted. */ + if (fdir_rule.b_mask) { + if (!fdir_info->mask_added) { + /* It's the first time the mask is set. */ + rte_memcpy(&fdir_info->mask, + &fdir_rule.mask, + sizeof(struct ixgbe_hw_fdir_mask)); + fdir_info->flex_bytes_offset = + fdir_rule.flex_bytes_offset; + + if (fdir_rule.mask.flex_bytes_mask) + ixgbe_fdir_set_flexbytes_offset(dev, + fdir_rule.flex_bytes_offset); + + ret = ixgbe_fdir_set_input_mask(dev); + if (ret) + goto out; + + fdir_info->mask_added = TRUE; + first_mask = TRUE; + } else { + /** + * Only support one global mask, + * all the masks should be the same. + */ + ret = memcmp(&fdir_info->mask, + &fdir_rule.mask, + sizeof(struct ixgbe_hw_fdir_mask)); + if (ret) + goto out; + + if (fdir_info->flex_bytes_offset != + fdir_rule.flex_bytes_offset) + goto out; + } + } + + if (fdir_rule.b_spec) { + ret = ixgbe_fdir_filter_program(dev, &fdir_rule, + FALSE, FALSE); + if (!ret) { + fdir_rule_ptr = rte_zmalloc("ixgbe_fdir_filter", + sizeof(struct ixgbe_fdir_rule_ele), 0); + if (!fdir_rule_ptr) { + PMD_DRV_LOG(ERR, "failed to allocate memory"); + goto out; + } + rte_memcpy(&fdir_rule_ptr->filter_info, + &fdir_rule, + sizeof(struct ixgbe_fdir_rule)); + TAILQ_INSERT_TAIL(&filter_fdir_list, + fdir_rule_ptr, entries); + flow->rule = fdir_rule_ptr; + flow->filter_type = RTE_ETH_FILTER_FDIR; + + return flow; + } + + if (ret) { + /** + * clean the mask_added flag if fail to + * program + **/ + if (first_mask) + fdir_info->mask_added = FALSE; + goto out; + } + } + + goto out; + } + + memset(&l2_tn_filter, 0, sizeof(struct rte_eth_l2_tunnel_conf)); + ret = ixgbe_parse_l2_tn_filter(dev, attr, pattern, + actions, &l2_tn_filter, error); + if (!ret) { + ret = ixgbe_dev_l2_tunnel_filter_add(dev, &l2_tn_filter, FALSE); + if (!ret) { + l2_tn_filter_ptr = rte_zmalloc("ixgbe_l2_tn_filter", + sizeof(struct ixgbe_eth_l2_tunnel_conf_ele), 0); + if (!l2_tn_filter_ptr) { + PMD_DRV_LOG(ERR, "failed to allocate memory"); + goto out; + } + rte_memcpy(&l2_tn_filter_ptr->filter_info, + &l2_tn_filter, + sizeof(struct rte_eth_l2_tunnel_conf)); + TAILQ_INSERT_TAIL(&filter_l2_tunnel_list, + l2_tn_filter_ptr, entries); + flow->rule = l2_tn_filter_ptr; + flow->filter_type = RTE_ETH_FILTER_L2_TUNNEL; + return flow; + } + } + +out: + TAILQ_REMOVE(&ixgbe_flow_list, + ixgbe_flow_mem_ptr, entries); + rte_flow_error_set(error, -ret, + RTE_FLOW_ERROR_TYPE_HANDLE, NULL, + "Failed to create flow."); + rte_free(ixgbe_flow_mem_ptr); + rte_free(flow); + return NULL; +} + +/** + * Check if the flow rule is supported by ixgbe. + * It only checkes the format. Don't guarantee the rule can be programmed into + * the HW. Because there can be no enough room for the rule. + */ +static int +ixgbe_flow_validate(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error) +{ + struct rte_eth_ntuple_filter ntuple_filter; + struct rte_eth_ethertype_filter ethertype_filter; + struct rte_eth_syn_filter syn_filter; + struct rte_eth_l2_tunnel_conf l2_tn_filter; + struct ixgbe_fdir_rule fdir_rule; + int ret; + + memset(&ntuple_filter, 0, sizeof(struct rte_eth_ntuple_filter)); + ret = ixgbe_parse_ntuple_filter(dev, attr, pattern, + actions, &ntuple_filter, error); + if (!ret) + return 0; + + memset(ðertype_filter, 0, sizeof(struct rte_eth_ethertype_filter)); + ret = ixgbe_parse_ethertype_filter(dev, attr, pattern, + actions, ðertype_filter, error); + if (!ret) + return 0; + + memset(&syn_filter, 0, sizeof(struct rte_eth_syn_filter)); + ret = ixgbe_parse_syn_filter(dev, attr, pattern, + actions, &syn_filter, error); + if (!ret) + return 0; + + memset(&fdir_rule, 0, sizeof(struct ixgbe_fdir_rule)); + ret = ixgbe_parse_fdir_filter(dev, attr, pattern, + actions, &fdir_rule, error); + if (!ret) + return 0; + + memset(&l2_tn_filter, 0, sizeof(struct rte_eth_l2_tunnel_conf)); + ret = ixgbe_parse_l2_tn_filter(dev, attr, pattern, + actions, &l2_tn_filter, error); + + return ret; +} + +/* Destroy a flow rule on ixgbe. */ +static int +ixgbe_flow_destroy(struct rte_eth_dev *dev, + struct rte_flow *flow, + struct rte_flow_error *error) +{ + int ret; + struct rte_flow *pmd_flow = flow; + enum rte_filter_type filter_type = pmd_flow->filter_type; + struct rte_eth_ntuple_filter ntuple_filter; + struct rte_eth_ethertype_filter ethertype_filter; + struct rte_eth_syn_filter syn_filter; + struct ixgbe_fdir_rule fdir_rule; + struct rte_eth_l2_tunnel_conf l2_tn_filter; + struct ixgbe_ntuple_filter_ele *ntuple_filter_ptr; + struct ixgbe_ethertype_filter_ele *ethertype_filter_ptr; + struct ixgbe_eth_syn_filter_ele *syn_filter_ptr; + struct ixgbe_eth_l2_tunnel_conf_ele *l2_tn_filter_ptr; + struct ixgbe_fdir_rule_ele *fdir_rule_ptr; + struct ixgbe_flow_mem *ixgbe_flow_mem_ptr; + struct ixgbe_hw_fdir_info *fdir_info = + IXGBE_DEV_PRIVATE_TO_FDIR_INFO(dev->data->dev_private); + + switch (filter_type) { + case RTE_ETH_FILTER_NTUPLE: + ntuple_filter_ptr = (struct ixgbe_ntuple_filter_ele *) + pmd_flow->rule; + rte_memcpy(&ntuple_filter, + &ntuple_filter_ptr->filter_info, + sizeof(struct rte_eth_ntuple_filter)); + ret = ixgbe_add_del_ntuple_filter(dev, &ntuple_filter, FALSE); + if (!ret) { + TAILQ_REMOVE(&filter_ntuple_list, + ntuple_filter_ptr, entries); + rte_free(ntuple_filter_ptr); + } + break; + case RTE_ETH_FILTER_ETHERTYPE: + ethertype_filter_ptr = (struct ixgbe_ethertype_filter_ele *) + pmd_flow->rule; + rte_memcpy(ðertype_filter, + ðertype_filter_ptr->filter_info, + sizeof(struct rte_eth_ethertype_filter)); + ret = ixgbe_add_del_ethertype_filter(dev, + ðertype_filter, FALSE); + if (!ret) { + TAILQ_REMOVE(&filter_ethertype_list, + ethertype_filter_ptr, entries); + rte_free(ethertype_filter_ptr); + } + break; + case RTE_ETH_FILTER_SYN: + syn_filter_ptr = (struct ixgbe_eth_syn_filter_ele *) + pmd_flow->rule; + rte_memcpy(&syn_filter, + &syn_filter_ptr->filter_info, + sizeof(struct rte_eth_syn_filter)); + ret = ixgbe_syn_filter_set(dev, &syn_filter, FALSE); + if (!ret) { + TAILQ_REMOVE(&filter_syn_list, + syn_filter_ptr, entries); + rte_free(syn_filter_ptr); + } + break; + case RTE_ETH_FILTER_FDIR: + fdir_rule_ptr = (struct ixgbe_fdir_rule_ele *)pmd_flow->rule; + rte_memcpy(&fdir_rule, + &fdir_rule_ptr->filter_info, + sizeof(struct ixgbe_fdir_rule)); + ret = ixgbe_fdir_filter_program(dev, &fdir_rule, TRUE, FALSE); + if (!ret) { + TAILQ_REMOVE(&filter_fdir_list, + fdir_rule_ptr, entries); + rte_free(fdir_rule_ptr); + if (TAILQ_EMPTY(&filter_fdir_list)) + fdir_info->mask_added = false; + } + break; + case RTE_ETH_FILTER_L2_TUNNEL: + l2_tn_filter_ptr = (struct ixgbe_eth_l2_tunnel_conf_ele *) + pmd_flow->rule; + rte_memcpy(&l2_tn_filter, &l2_tn_filter_ptr->filter_info, + sizeof(struct rte_eth_l2_tunnel_conf)); + ret = ixgbe_dev_l2_tunnel_filter_del(dev, &l2_tn_filter); + if (!ret) { + TAILQ_REMOVE(&filter_l2_tunnel_list, + l2_tn_filter_ptr, entries); + rte_free(l2_tn_filter_ptr); + } + break; + default: + PMD_DRV_LOG(WARNING, "Filter type (%d) not supported", + filter_type); + ret = -EINVAL; + break; + } + + if (ret) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_HANDLE, + NULL, "Failed to destroy flow"); + return ret; + } + + TAILQ_FOREACH(ixgbe_flow_mem_ptr, &ixgbe_flow_list, entries) { + if (ixgbe_flow_mem_ptr->flow == pmd_flow) { + TAILQ_REMOVE(&ixgbe_flow_list, + ixgbe_flow_mem_ptr, entries); + rte_free(ixgbe_flow_mem_ptr); + } + } + rte_free(flow); + + return ret; +} + +/* Destroy all flow rules associated with a port on ixgbe. */ +static int +ixgbe_flow_flush(struct rte_eth_dev *dev, + struct rte_flow_error *error) +{ + int ret = 0; + + ixgbe_clear_all_ntuple_filter(dev); + ixgbe_clear_all_ethertype_filter(dev); + ixgbe_clear_syn_filter(dev); + + ret = ixgbe_clear_all_fdir_filter(dev); + if (ret < 0) { + rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_HANDLE, + NULL, "Failed to flush rule"); + return ret; + } + + ret = ixgbe_clear_all_l2_tn_filter(dev); + if (ret < 0) { + rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_HANDLE, + NULL, "Failed to flush rule"); + return ret; + } + + ixgbe_filterlist_flush(); + + return 0; +} + +const struct rte_flow_ops ixgbe_flow_ops = { + .validate = ixgbe_flow_validate, + .create = ixgbe_flow_create, + .destroy = ixgbe_flow_destroy, + .flush = ixgbe_flow_flush, +}; diff --git a/dpdk/drivers/net/ixgbe/ixgbe_ipsec.c b/dpdk/drivers/net/ixgbe/ixgbe_ipsec.c new file mode 100644 index 00000000..105da11a --- /dev/null +++ b/dpdk/drivers/net/ixgbe/ixgbe_ipsec.c @@ -0,0 +1,737 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "base/ixgbe_type.h" +#include "base/ixgbe_api.h" +#include "ixgbe_ethdev.h" +#include "ixgbe_ipsec.h" + +#define RTE_IXGBE_REGISTER_POLL_WAIT_5_MS 5 + +#define IXGBE_WAIT_RREAD \ + IXGBE_WRITE_REG_THEN_POLL_MASK(hw, IXGBE_IPSRXIDX, reg_val, \ + IPSRXIDX_READ, RTE_IXGBE_REGISTER_POLL_WAIT_5_MS) +#define IXGBE_WAIT_RWRITE \ + IXGBE_WRITE_REG_THEN_POLL_MASK(hw, IXGBE_IPSRXIDX, reg_val, \ + IPSRXIDX_WRITE, RTE_IXGBE_REGISTER_POLL_WAIT_5_MS) +#define IXGBE_WAIT_TREAD \ + IXGBE_WRITE_REG_THEN_POLL_MASK(hw, IXGBE_IPSTXIDX, reg_val, \ + IPSRXIDX_READ, RTE_IXGBE_REGISTER_POLL_WAIT_5_MS) +#define IXGBE_WAIT_TWRITE \ + IXGBE_WRITE_REG_THEN_POLL_MASK(hw, IXGBE_IPSTXIDX, reg_val, \ + IPSRXIDX_WRITE, RTE_IXGBE_REGISTER_POLL_WAIT_5_MS) + +#define CMP_IP(a, b) (\ + (a).ipv6[0] == (b).ipv6[0] && \ + (a).ipv6[1] == (b).ipv6[1] && \ + (a).ipv6[2] == (b).ipv6[2] && \ + (a).ipv6[3] == (b).ipv6[3]) + + +static void +ixgbe_crypto_clear_ipsec_tables(struct rte_eth_dev *dev) +{ + struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + int i = 0; + + /* clear Rx IP table*/ + for (i = 0; i < IPSEC_MAX_RX_IP_COUNT; i++) { + uint16_t index = i << 3; + uint32_t reg_val = IPSRXIDX_WRITE | IPSRXIDX_TABLE_IP | index; + IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(0), 0); + IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(1), 0); + IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(2), 0); + IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(3), 0); + IXGBE_WAIT_RWRITE; + } + + /* clear Rx SPI and Rx/Tx SA tables*/ + for (i = 0; i < IPSEC_MAX_SA_COUNT; i++) { + uint32_t index = i << 3; + uint32_t reg_val = IPSRXIDX_WRITE | IPSRXIDX_TABLE_SPI | index; + IXGBE_WRITE_REG(hw, IXGBE_IPSRXSPI, 0); + IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPIDX, 0); + IXGBE_WAIT_RWRITE; + reg_val = IPSRXIDX_WRITE | IPSRXIDX_TABLE_KEY | index; + IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(0), 0); + IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(1), 0); + IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(2), 0); + IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(3), 0); + IXGBE_WRITE_REG(hw, IXGBE_IPSRXSALT, 0); + IXGBE_WRITE_REG(hw, IXGBE_IPSRXMOD, 0); + IXGBE_WAIT_RWRITE; + reg_val = IPSRXIDX_WRITE | index; + IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(0), 0); + IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(1), 0); + IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(2), 0); + IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(3), 0); + IXGBE_WRITE_REG(hw, IXGBE_IPSTXSALT, 0); + IXGBE_WAIT_TWRITE; + } +} + +static int +ixgbe_crypto_add_sa(struct ixgbe_crypto_session *ic_session) +{ + struct rte_eth_dev *dev = ic_session->dev; + struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct ixgbe_ipsec *priv = IXGBE_DEV_PRIVATE_TO_IPSEC( + dev->data->dev_private); + uint32_t reg_val; + int sa_index = -1; + + if (ic_session->op == IXGBE_OP_AUTHENTICATED_DECRYPTION) { + int i, ip_index = -1; + + /* Find a match in the IP table*/ + for (i = 0; i < IPSEC_MAX_RX_IP_COUNT; i++) { + if (CMP_IP(priv->rx_ip_tbl[i].ip, + ic_session->dst_ip)) { + ip_index = i; + break; + } + } + /* If no match, find a free entry in the IP table*/ + if (ip_index < 0) { + for (i = 0; i < IPSEC_MAX_RX_IP_COUNT; i++) { + if (priv->rx_ip_tbl[i].ref_count == 0) { + ip_index = i; + break; + } + } + } + + /* Fail if no match and no free entries*/ + if (ip_index < 0) { + PMD_DRV_LOG(ERR, + "No free entry left in the Rx IP table\n"); + return -1; + } + + /* Find a free entry in the SA table*/ + for (i = 0; i < IPSEC_MAX_SA_COUNT; i++) { + if (priv->rx_sa_tbl[i].used == 0) { + sa_index = i; + break; + } + } + /* Fail if no free entries*/ + if (sa_index < 0) { + PMD_DRV_LOG(ERR, + "No free entry left in the Rx SA table\n"); + return -1; + } + + priv->rx_ip_tbl[ip_index].ip.ipv6[0] = + ic_session->dst_ip.ipv6[0]; + priv->rx_ip_tbl[ip_index].ip.ipv6[1] = + ic_session->dst_ip.ipv6[1]; + priv->rx_ip_tbl[ip_index].ip.ipv6[2] = + ic_session->dst_ip.ipv6[2]; + priv->rx_ip_tbl[ip_index].ip.ipv6[3] = + ic_session->dst_ip.ipv6[3]; + priv->rx_ip_tbl[ip_index].ref_count++; + + priv->rx_sa_tbl[sa_index].spi = + rte_cpu_to_be_32(ic_session->spi); + priv->rx_sa_tbl[sa_index].ip_index = ip_index; + priv->rx_sa_tbl[sa_index].key[3] = + rte_cpu_to_be_32(*(uint32_t *)&ic_session->key[0]); + priv->rx_sa_tbl[sa_index].key[2] = + rte_cpu_to_be_32(*(uint32_t *)&ic_session->key[4]); + priv->rx_sa_tbl[sa_index].key[1] = + rte_cpu_to_be_32(*(uint32_t *)&ic_session->key[8]); + priv->rx_sa_tbl[sa_index].key[0] = + rte_cpu_to_be_32(*(uint32_t *)&ic_session->key[12]); + priv->rx_sa_tbl[sa_index].salt = + rte_cpu_to_be_32(ic_session->salt); + priv->rx_sa_tbl[sa_index].mode = IPSRXMOD_VALID; + if (ic_session->op == IXGBE_OP_AUTHENTICATED_DECRYPTION) + priv->rx_sa_tbl[sa_index].mode |= + (IPSRXMOD_PROTO | IPSRXMOD_DECRYPT); + if (ic_session->dst_ip.type == IPv6) + priv->rx_sa_tbl[sa_index].mode |= IPSRXMOD_IPV6; + priv->rx_sa_tbl[sa_index].used = 1; + + /* write IP table entry*/ + reg_val = IPSRXIDX_RX_EN | IPSRXIDX_WRITE | + IPSRXIDX_TABLE_IP | (ip_index << 3); + if (priv->rx_ip_tbl[ip_index].ip.type == IPv4) { + IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(0), 0); + IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(1), 0); + IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(2), 0); + IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(3), + priv->rx_ip_tbl[ip_index].ip.ipv4); + } else { + IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(0), + priv->rx_ip_tbl[ip_index].ip.ipv6[0]); + IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(1), + priv->rx_ip_tbl[ip_index].ip.ipv6[1]); + IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(2), + priv->rx_ip_tbl[ip_index].ip.ipv6[2]); + IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(3), + priv->rx_ip_tbl[ip_index].ip.ipv6[3]); + } + IXGBE_WAIT_RWRITE; + + /* write SPI table entry*/ + reg_val = IPSRXIDX_RX_EN | IPSRXIDX_WRITE | + IPSRXIDX_TABLE_SPI | (sa_index << 3); + IXGBE_WRITE_REG(hw, IXGBE_IPSRXSPI, + priv->rx_sa_tbl[sa_index].spi); + IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPIDX, + priv->rx_sa_tbl[sa_index].ip_index); + IXGBE_WAIT_RWRITE; + + /* write Key table entry*/ + reg_val = IPSRXIDX_RX_EN | IPSRXIDX_WRITE | + IPSRXIDX_TABLE_KEY | (sa_index << 3); + IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(0), + priv->rx_sa_tbl[sa_index].key[0]); + IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(1), + priv->rx_sa_tbl[sa_index].key[1]); + IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(2), + priv->rx_sa_tbl[sa_index].key[2]); + IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(3), + priv->rx_sa_tbl[sa_index].key[3]); + IXGBE_WRITE_REG(hw, IXGBE_IPSRXSALT, + priv->rx_sa_tbl[sa_index].salt); + IXGBE_WRITE_REG(hw, IXGBE_IPSRXMOD, + priv->rx_sa_tbl[sa_index].mode); + IXGBE_WAIT_RWRITE; + + } else { /* sess->dir == RTE_CRYPTO_OUTBOUND */ + int i; + + /* Find a free entry in the SA table*/ + for (i = 0; i < IPSEC_MAX_SA_COUNT; i++) { + if (priv->tx_sa_tbl[i].used == 0) { + sa_index = i; + break; + } + } + /* Fail if no free entries*/ + if (sa_index < 0) { + PMD_DRV_LOG(ERR, + "No free entry left in the Tx SA table\n"); + return -1; + } + + priv->tx_sa_tbl[sa_index].spi = + rte_cpu_to_be_32(ic_session->spi); + priv->tx_sa_tbl[sa_index].key[3] = + rte_cpu_to_be_32(*(uint32_t *)&ic_session->key[0]); + priv->tx_sa_tbl[sa_index].key[2] = + rte_cpu_to_be_32(*(uint32_t *)&ic_session->key[4]); + priv->tx_sa_tbl[sa_index].key[1] = + rte_cpu_to_be_32(*(uint32_t *)&ic_session->key[8]); + priv->tx_sa_tbl[sa_index].key[0] = + rte_cpu_to_be_32(*(uint32_t *)&ic_session->key[12]); + priv->tx_sa_tbl[sa_index].salt = + rte_cpu_to_be_32(ic_session->salt); + + reg_val = IPSRXIDX_RX_EN | IPSRXIDX_WRITE | (sa_index << 3); + IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(0), + priv->tx_sa_tbl[sa_index].key[0]); + IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(1), + priv->tx_sa_tbl[sa_index].key[1]); + IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(2), + priv->tx_sa_tbl[sa_index].key[2]); + IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(3), + priv->tx_sa_tbl[sa_index].key[3]); + IXGBE_WRITE_REG(hw, IXGBE_IPSTXSALT, + priv->tx_sa_tbl[sa_index].salt); + IXGBE_WAIT_TWRITE; + + priv->tx_sa_tbl[i].used = 1; + ic_session->sa_index = sa_index; + } + + return 0; +} + +static int +ixgbe_crypto_remove_sa(struct rte_eth_dev *dev, + struct ixgbe_crypto_session *ic_session) +{ + struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct ixgbe_ipsec *priv = + IXGBE_DEV_PRIVATE_TO_IPSEC(dev->data->dev_private); + uint32_t reg_val; + int sa_index = -1; + + if (ic_session->op == IXGBE_OP_AUTHENTICATED_DECRYPTION) { + int i, ip_index = -1; + + /* Find a match in the IP table*/ + for (i = 0; i < IPSEC_MAX_RX_IP_COUNT; i++) { + if (CMP_IP(priv->rx_ip_tbl[i].ip, ic_session->dst_ip)) { + ip_index = i; + break; + } + } + + /* Fail if no match*/ + if (ip_index < 0) { + PMD_DRV_LOG(ERR, + "Entry not found in the Rx IP table\n"); + return -1; + } + + /* Find a free entry in the SA table*/ + for (i = 0; i < IPSEC_MAX_SA_COUNT; i++) { + if (priv->rx_sa_tbl[i].spi == + rte_cpu_to_be_32(ic_session->spi)) { + sa_index = i; + break; + } + } + /* Fail if no match*/ + if (sa_index < 0) { + PMD_DRV_LOG(ERR, + "Entry not found in the Rx SA table\n"); + return -1; + } + + /* Disable and clear Rx SPI and key table table entryes*/ + reg_val = IPSRXIDX_WRITE | IPSRXIDX_TABLE_SPI | (sa_index << 3); + IXGBE_WRITE_REG(hw, IXGBE_IPSRXSPI, 0); + IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPIDX, 0); + IXGBE_WAIT_RWRITE; + reg_val = IPSRXIDX_WRITE | IPSRXIDX_TABLE_KEY | (sa_index << 3); + IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(0), 0); + IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(1), 0); + IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(2), 0); + IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(3), 0); + IXGBE_WRITE_REG(hw, IXGBE_IPSRXSALT, 0); + IXGBE_WRITE_REG(hw, IXGBE_IPSRXMOD, 0); + IXGBE_WAIT_RWRITE; + priv->rx_sa_tbl[sa_index].used = 0; + + /* If last used then clear the IP table entry*/ + priv->rx_ip_tbl[ip_index].ref_count--; + if (priv->rx_ip_tbl[ip_index].ref_count == 0) { + reg_val = IPSRXIDX_WRITE | IPSRXIDX_TABLE_IP | + (ip_index << 3); + IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(0), 0); + IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(1), 0); + IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(2), 0); + IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(3), 0); + } + } else { /* session->dir == RTE_CRYPTO_OUTBOUND */ + int i; + + /* Find a match in the SA table*/ + for (i = 0; i < IPSEC_MAX_SA_COUNT; i++) { + if (priv->tx_sa_tbl[i].spi == + rte_cpu_to_be_32(ic_session->spi)) { + sa_index = i; + break; + } + } + /* Fail if no match entries*/ + if (sa_index < 0) { + PMD_DRV_LOG(ERR, + "Entry not found in the Tx SA table\n"); + return -1; + } + reg_val = IPSRXIDX_WRITE | (sa_index << 3); + IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(0), 0); + IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(1), 0); + IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(2), 0); + IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(3), 0); + IXGBE_WRITE_REG(hw, IXGBE_IPSTXSALT, 0); + IXGBE_WAIT_TWRITE; + + priv->tx_sa_tbl[sa_index].used = 0; + } + + return 0; +} + +static int +ixgbe_crypto_create_session(void *device, + struct rte_security_session_conf *conf, + struct rte_security_session *session, + struct rte_mempool *mempool) +{ + struct rte_eth_dev *eth_dev = (struct rte_eth_dev *)device; + struct ixgbe_crypto_session *ic_session = NULL; + struct rte_crypto_aead_xform *aead_xform; + struct rte_eth_conf *dev_conf = ð_dev->data->dev_conf; + + if (rte_mempool_get(mempool, (void **)&ic_session)) { + PMD_DRV_LOG(ERR, "Cannot get object from ic_session mempool"); + return -ENOMEM; + } + + if (conf->crypto_xform->type != RTE_CRYPTO_SYM_XFORM_AEAD || + conf->crypto_xform->aead.algo != + RTE_CRYPTO_AEAD_AES_GCM) { + PMD_DRV_LOG(ERR, "Unsupported crypto transformation mode\n"); + return -ENOTSUP; + } + aead_xform = &conf->crypto_xform->aead; + + if (conf->ipsec.direction == RTE_SECURITY_IPSEC_SA_DIR_INGRESS) { + if (dev_conf->rxmode.offloads & DEV_RX_OFFLOAD_SECURITY) { + ic_session->op = IXGBE_OP_AUTHENTICATED_DECRYPTION; + } else { + PMD_DRV_LOG(ERR, "IPsec decryption not enabled\n"); + return -ENOTSUP; + } + } else { + if (dev_conf->txmode.offloads & DEV_TX_OFFLOAD_SECURITY) { + ic_session->op = IXGBE_OP_AUTHENTICATED_ENCRYPTION; + } else { + PMD_DRV_LOG(ERR, "IPsec encryption not enabled\n"); + return -ENOTSUP; + } + } + + ic_session->key = aead_xform->key.data; + memcpy(&ic_session->salt, + &aead_xform->key.data[aead_xform->key.length], 4); + ic_session->spi = conf->ipsec.spi; + ic_session->dev = eth_dev; + + set_sec_session_private_data(session, ic_session); + + if (ic_session->op == IXGBE_OP_AUTHENTICATED_ENCRYPTION) { + if (ixgbe_crypto_add_sa(ic_session)) { + PMD_DRV_LOG(ERR, "Failed to add SA\n"); + return -EPERM; + } + } + + return 0; +} + +static int +ixgbe_crypto_remove_session(void *device, + struct rte_security_session *session) +{ + struct rte_eth_dev *eth_dev = device; + struct ixgbe_crypto_session *ic_session = + (struct ixgbe_crypto_session *) + get_sec_session_private_data(session); + struct rte_mempool *mempool = rte_mempool_from_obj(ic_session); + + if (eth_dev != ic_session->dev) { + PMD_DRV_LOG(ERR, "Session not bound to this device\n"); + return -ENODEV; + } + + if (ixgbe_crypto_remove_sa(eth_dev, ic_session)) { + PMD_DRV_LOG(ERR, "Failed to remove session\n"); + return -EFAULT; + } + + rte_mempool_put(mempool, (void *)ic_session); + + return 0; +} + +static inline uint8_t +ixgbe_crypto_compute_pad_len(struct rte_mbuf *m) +{ + if (m->nb_segs == 1) { + /* 16 bytes ICV + 2 bytes ESP trailer + payload padding size + * payload padding size is stored at + */ + uint8_t *esp_pad_len = rte_pktmbuf_mtod_offset(m, uint8_t *, + rte_pktmbuf_pkt_len(m) - + (ESP_TRAILER_SIZE + ESP_ICV_SIZE)); + return *esp_pad_len + ESP_TRAILER_SIZE + ESP_ICV_SIZE; + } + return 0; +} + +static int +ixgbe_crypto_update_mb(void *device __rte_unused, + struct rte_security_session *session, + struct rte_mbuf *m, void *params __rte_unused) +{ + struct ixgbe_crypto_session *ic_session = + get_sec_session_private_data(session); + if (ic_session->op == IXGBE_OP_AUTHENTICATED_ENCRYPTION) { + union ixgbe_crypto_tx_desc_md *mdata = + (union ixgbe_crypto_tx_desc_md *)&m->udata64; + mdata->enc = 1; + mdata->sa_idx = ic_session->sa_index; + mdata->pad_len = ixgbe_crypto_compute_pad_len(m); + } + return 0; +} + + +static const struct rte_security_capability * +ixgbe_crypto_capabilities_get(void *device __rte_unused) +{ + static const struct rte_cryptodev_capabilities + aes_gcm_gmac_crypto_capabilities[] = { + { /* AES GMAC (128-bit) */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_AES_GMAC, + .block_size = 16, + .key_size = { + .min = 16, + .max = 16, + .increment = 0 + }, + .digest_size = { + .min = 16, + .max = 16, + .increment = 0 + }, + .iv_size = { + .min = 12, + .max = 12, + .increment = 0 + } + }, } + }, } + }, + { /* AES GCM (128-bit) */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD, + {.aead = { + .algo = RTE_CRYPTO_AEAD_AES_GCM, + .block_size = 16, + .key_size = { + .min = 16, + .max = 16, + .increment = 0 + }, + .digest_size = { + .min = 16, + .max = 16, + .increment = 0 + }, + .aad_size = { + .min = 0, + .max = 65535, + .increment = 1 + }, + .iv_size = { + .min = 12, + .max = 12, + .increment = 0 + } + }, } + }, } + }, + { + .op = RTE_CRYPTO_OP_TYPE_UNDEFINED, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_NOT_SPECIFIED + }, } + }, + }; + + static const struct rte_security_capability + ixgbe_security_capabilities[] = { + { /* IPsec Inline Crypto ESP Transport Egress */ + .action = RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO, + .protocol = RTE_SECURITY_PROTOCOL_IPSEC, + {.ipsec = { + .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP, + .mode = RTE_SECURITY_IPSEC_SA_MODE_TRANSPORT, + .direction = RTE_SECURITY_IPSEC_SA_DIR_EGRESS, + .options = { 0 } + } }, + .crypto_capabilities = aes_gcm_gmac_crypto_capabilities, + .ol_flags = RTE_SECURITY_TX_OLOAD_NEED_MDATA + }, + { /* IPsec Inline Crypto ESP Transport Ingress */ + .action = RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO, + .protocol = RTE_SECURITY_PROTOCOL_IPSEC, + {.ipsec = { + .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP, + .mode = RTE_SECURITY_IPSEC_SA_MODE_TRANSPORT, + .direction = RTE_SECURITY_IPSEC_SA_DIR_INGRESS, + .options = { 0 } + } }, + .crypto_capabilities = aes_gcm_gmac_crypto_capabilities, + .ol_flags = 0 + }, + { /* IPsec Inline Crypto ESP Tunnel Egress */ + .action = RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO, + .protocol = RTE_SECURITY_PROTOCOL_IPSEC, + {.ipsec = { + .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP, + .mode = RTE_SECURITY_IPSEC_SA_MODE_TUNNEL, + .direction = RTE_SECURITY_IPSEC_SA_DIR_EGRESS, + .options = { 0 } + } }, + .crypto_capabilities = aes_gcm_gmac_crypto_capabilities, + .ol_flags = RTE_SECURITY_TX_OLOAD_NEED_MDATA + }, + { /* IPsec Inline Crypto ESP Tunnel Ingress */ + .action = RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO, + .protocol = RTE_SECURITY_PROTOCOL_IPSEC, + {.ipsec = { + .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP, + .mode = RTE_SECURITY_IPSEC_SA_MODE_TUNNEL, + .direction = RTE_SECURITY_IPSEC_SA_DIR_INGRESS, + .options = { 0 } + } }, + .crypto_capabilities = aes_gcm_gmac_crypto_capabilities, + .ol_flags = 0 + }, + { + .action = RTE_SECURITY_ACTION_TYPE_NONE + } + }; + + return ixgbe_security_capabilities; +} + + +int +ixgbe_crypto_enable_ipsec(struct rte_eth_dev *dev) +{ + struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + uint32_t reg; + + /* sanity checks */ + if (dev->data->dev_conf.rxmode.enable_lro) { + PMD_DRV_LOG(ERR, "RSC and IPsec not supported"); + return -1; + } + if (!dev->data->dev_conf.rxmode.hw_strip_crc) { + PMD_DRV_LOG(ERR, "HW CRC strip needs to be enabled for IPsec"); + return -1; + } + + + /* Set IXGBE_SECTXBUFFAF to 0x15 as required in the datasheet*/ + IXGBE_WRITE_REG(hw, IXGBE_SECTXBUFFAF, 0x15); + + /* IFG needs to be set to 3 when we are using security. Otherwise a Tx + * hang will occur with heavy traffic. + */ + reg = IXGBE_READ_REG(hw, IXGBE_SECTXMINIFG); + reg = (reg & 0xFFFFFFF0) | 0x3; + IXGBE_WRITE_REG(hw, IXGBE_SECTXMINIFG, reg); + + reg = IXGBE_READ_REG(hw, IXGBE_HLREG0); + reg |= IXGBE_HLREG0_TXCRCEN | IXGBE_HLREG0_RXCRCSTRP; + IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg); + + if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_SECURITY) { + IXGBE_WRITE_REG(hw, IXGBE_SECRXCTRL, 0); + reg = IXGBE_READ_REG(hw, IXGBE_SECRXCTRL); + if (reg != 0) { + PMD_DRV_LOG(ERR, "Error enabling Rx Crypto"); + return -1; + } + } + if (dev->data->dev_conf.txmode.offloads & DEV_TX_OFFLOAD_SECURITY) { + IXGBE_WRITE_REG(hw, IXGBE_SECTXCTRL, + IXGBE_SECTXCTRL_STORE_FORWARD); + reg = IXGBE_READ_REG(hw, IXGBE_SECTXCTRL); + if (reg != IXGBE_SECTXCTRL_STORE_FORWARD) { + PMD_DRV_LOG(ERR, "Error enabling Rx Crypto"); + return -1; + } + } + + ixgbe_crypto_clear_ipsec_tables(dev); + + return 0; +} + +int +ixgbe_crypto_add_ingress_sa_from_flow(const void *sess, + const void *ip_spec, + uint8_t is_ipv6) +{ + struct ixgbe_crypto_session *ic_session + = get_sec_session_private_data(sess); + + if (ic_session->op == IXGBE_OP_AUTHENTICATED_DECRYPTION) { + if (is_ipv6) { + const struct rte_flow_item_ipv6 *ipv6 = ip_spec; + ic_session->src_ip.type = IPv6; + ic_session->dst_ip.type = IPv6; + rte_memcpy(ic_session->src_ip.ipv6, + ipv6->hdr.src_addr, 16); + rte_memcpy(ic_session->dst_ip.ipv6, + ipv6->hdr.dst_addr, 16); + } else { + const struct rte_flow_item_ipv4 *ipv4 = ip_spec; + ic_session->src_ip.type = IPv4; + ic_session->dst_ip.type = IPv4; + ic_session->src_ip.ipv4 = ipv4->hdr.src_addr; + ic_session->dst_ip.ipv4 = ipv4->hdr.dst_addr; + } + return ixgbe_crypto_add_sa(ic_session); + } + + return 0; +} + +static struct rte_security_ops ixgbe_security_ops = { + .session_create = ixgbe_crypto_create_session, + .session_update = NULL, + .session_stats_get = NULL, + .session_destroy = ixgbe_crypto_remove_session, + .set_pkt_metadata = ixgbe_crypto_update_mb, + .capabilities_get = ixgbe_crypto_capabilities_get +}; + +struct rte_security_ctx * +ixgbe_ipsec_ctx_create(struct rte_eth_dev *dev) +{ + struct rte_security_ctx *ctx = rte_malloc("rte_security_instances_ops", + sizeof(struct rte_security_ctx), 0); + if (ctx) { + ctx->device = (void *)dev; + ctx->ops = &ixgbe_security_ops; + ctx->sess_cnt = 0; + } + return ctx; +} diff --git a/dpdk/drivers/net/ixgbe/ixgbe_ipsec.h b/dpdk/drivers/net/ixgbe/ixgbe_ipsec.h new file mode 100644 index 00000000..fb8fefc8 --- /dev/null +++ b/dpdk/drivers/net/ixgbe/ixgbe_ipsec.h @@ -0,0 +1,151 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef IXGBE_IPSEC_H_ +#define IXGBE_IPSEC_H_ + +#include + +#define IPSRXIDX_RX_EN 0x00000001 +#define IPSRXIDX_TABLE_IP 0x00000002 +#define IPSRXIDX_TABLE_SPI 0x00000004 +#define IPSRXIDX_TABLE_KEY 0x00000006 +#define IPSRXIDX_WRITE 0x80000000 +#define IPSRXIDX_READ 0x40000000 +#define IPSRXMOD_VALID 0x00000001 +#define IPSRXMOD_PROTO 0x00000004 +#define IPSRXMOD_DECRYPT 0x00000008 +#define IPSRXMOD_IPV6 0x00000010 +#define IXGBE_ADVTXD_POPTS_IPSEC 0x00000400 +#define IXGBE_ADVTXD_TUCMD_IPSEC_TYPE_ESP 0x00002000 +#define IXGBE_ADVTXD_TUCMD_IPSEC_ENCRYPT_EN 0x00004000 +#define IXGBE_RXDADV_IPSEC_STATUS_SECP 0x00020000 +#define IXGBE_RXDADV_IPSEC_ERROR_BIT_MASK 0x18000000 +#define IXGBE_RXDADV_IPSEC_ERROR_INVALID_PROTOCOL 0x08000000 +#define IXGBE_RXDADV_IPSEC_ERROR_INVALID_LENGTH 0x10000000 +#define IXGBE_RXDADV_IPSEC_ERROR_AUTHENTICATION_FAILED 0x18000000 + +#define IPSEC_MAX_RX_IP_COUNT 128 +#define IPSEC_MAX_SA_COUNT 1024 + +#define ESP_ICV_SIZE 16 +#define ESP_TRAILER_SIZE 2 + +enum ixgbe_operation { + IXGBE_OP_AUTHENTICATED_ENCRYPTION, + IXGBE_OP_AUTHENTICATED_DECRYPTION +}; + +enum ixgbe_gcm_key { + IXGBE_GCM_KEY_128, + IXGBE_GCM_KEY_256 +}; + +/** + * Generic IP address structure + * TODO: Find better location for this rte_net.h possibly. + **/ +struct ipaddr { + enum ipaddr_type { + IPv4, + IPv6 + } type; + /**< IP Address Type - IPv4/IPv6 */ + + union { + uint32_t ipv4; + uint32_t ipv6[4]; + }; +}; + +/** inline crypto crypto private session structure */ +struct ixgbe_crypto_session { + enum ixgbe_operation op; + uint8_t *key; + uint32_t salt; + uint32_t sa_index; + uint32_t spi; + struct ipaddr src_ip; + struct ipaddr dst_ip; + struct rte_eth_dev *dev; +} __rte_cache_aligned; + +struct ixgbe_crypto_rx_ip_table { + struct ipaddr ip; + uint16_t ref_count; +}; +struct ixgbe_crypto_rx_sa_table { + uint32_t spi; + uint32_t ip_index; + uint32_t key[4]; + uint32_t salt; + uint8_t mode; + uint8_t used; +}; + +struct ixgbe_crypto_tx_sa_table { + uint32_t spi; + uint32_t key[4]; + uint32_t salt; + uint8_t used; +}; + +union ixgbe_crypto_tx_desc_md { + uint64_t data; + struct { + /**< SA table index */ + uint32_t sa_idx; + /**< ICV and ESP trailer length */ + uint8_t pad_len; + /**< enable encryption */ + uint8_t enc; + }; +}; + +struct ixgbe_ipsec { + struct ixgbe_crypto_rx_ip_table rx_ip_tbl[IPSEC_MAX_RX_IP_COUNT]; + struct ixgbe_crypto_rx_sa_table rx_sa_tbl[IPSEC_MAX_SA_COUNT]; + struct ixgbe_crypto_tx_sa_table tx_sa_tbl[IPSEC_MAX_SA_COUNT]; +}; + + +struct rte_security_ctx * +ixgbe_ipsec_ctx_create(struct rte_eth_dev *dev); +int ixgbe_crypto_enable_ipsec(struct rte_eth_dev *dev); +int ixgbe_crypto_add_ingress_sa_from_flow(const void *sess, + const void *ip_spec, + uint8_t is_ipv6); + + + +#endif /*IXGBE_IPSEC_H_*/ diff --git a/dpdk/drivers/net/ixgbe/ixgbe_pf.c b/dpdk/drivers/net/ixgbe/ixgbe_pf.c index 56393ff2..01146941 100644 --- a/dpdk/drivers/net/ixgbe/ixgbe_pf.c +++ b/dpdk/drivers/net/ixgbe/ixgbe_pf.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -51,6 +51,7 @@ #include "base/ixgbe_common.h" #include "ixgbe_ethdev.h" +#include "rte_pmd_ixgbe.h" #define IXGBE_MAX_VFTA (128) #define IXGBE_VF_MSG_SIZE_DEFAULT 1 @@ -60,7 +61,9 @@ static inline uint16_t dev_num_vf(struct rte_eth_dev *eth_dev) { - return eth_dev->pci_dev->max_vfs; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); + + return pci_dev->max_vfs; } static inline @@ -175,6 +178,7 @@ ixgbe_add_tx_flow_control_drop_filter(struct rte_eth_dev *eth_dev) IXGBE_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private); uint16_t vf_num; int i; + struct ixgbe_ethertype_filter ethertype_filter; if (!hw->mac.ops.set_ethertype_anti_spoofing) { RTE_LOG(INFO, PMD, "ether type anti-spoofing is not" @@ -182,16 +186,23 @@ ixgbe_add_tx_flow_control_drop_filter(struct rte_eth_dev *eth_dev) return; } - /* occupy an entity of ether type filter */ - for (i = 0; i < IXGBE_MAX_ETQF_FILTERS; i++) { - if (!(filter_info->ethertype_mask & (1 << i))) { - filter_info->ethertype_mask |= 1 << i; - filter_info->ethertype_filters[i] = - IXGBE_ETHERTYPE_FLOW_CTRL; - break; - } + i = ixgbe_ethertype_filter_lookup(filter_info, + IXGBE_ETHERTYPE_FLOW_CTRL); + if (i >= 0) { + RTE_LOG(ERR, PMD, "A ether type filter" + " entity for flow control already exists!\n"); + return; } - if (i == IXGBE_MAX_ETQF_FILTERS) { + + ethertype_filter.ethertype = IXGBE_ETHERTYPE_FLOW_CTRL; + ethertype_filter.etqf = IXGBE_ETQF_FILTER_EN | + IXGBE_ETQF_TX_ANTISPOOF | + IXGBE_ETHERTYPE_FLOW_CTRL; + ethertype_filter.etqs = 0; + ethertype_filter.conf = TRUE; + i = ixgbe_ethertype_filter_insert(filter_info, + ðertype_filter); + if (i < 0) { RTE_LOG(ERR, PMD, "Cannot find an unused ether type filter" " entity for flow control.\n"); return; @@ -262,7 +273,7 @@ int ixgbe_pf_host_configure(struct rte_eth_dev *eth_dev) gpie = IXGBE_READ_REG(hw, IXGBE_GPIE); gpie &= ~IXGBE_GPIE_VTMODE_MASK; - gpie |= IXGBE_GPIE_MSIX_MODE; + gpie |= IXGBE_GPIE_MSIX_MODE | IXGBE_GPIE_PBA_SUPPORT; switch (RTE_ETH_DEV_SRIOV(eth_dev).active) { case ETH_64_POOLS: @@ -386,15 +397,27 @@ ixgbe_vf_reset_msg(struct rte_eth_dev *dev, uint16_t vf) uint32_t reg_offset, vf_shift; const uint8_t VFRE_SHIFT = 5; /* VFRE 32 bits per slot */ const uint8_t VFRE_MASK = (uint8_t)((1U << VFRE_SHIFT) - 1); + uint8_t nb_q_per_pool; + int i; vf_shift = vf & VFRE_MASK; reg_offset = (vf >> VFRE_SHIFT) > 0 ? 1 : 0; - /* enable transmit and receive for vf */ + /* enable transmit for vf */ reg = IXGBE_READ_REG(hw, IXGBE_VFTE(reg_offset)); reg |= (reg | (1 << vf_shift)); IXGBE_WRITE_REG(hw, IXGBE_VFTE(reg_offset), reg); + /* enable all queue drop for IOV */ + nb_q_per_pool = RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool; + for (i = vf * nb_q_per_pool; i < (vf + 1) * nb_q_per_pool; i++) { + IXGBE_WRITE_FLUSH(hw); + reg = IXGBE_QDE_ENABLE | IXGBE_QDE_WRITE; + reg |= i << IXGBE_QDE_IDX_SHIFT; + IXGBE_WRITE_REG(hw, IXGBE_QDE, reg); + } + + /* enable receive for vf */ reg = IXGBE_READ_REG(hw, IXGBE_VFRE(reg_offset)); reg |= (reg | (1 << vf_shift)); IXGBE_WRITE_REG(hw, IXGBE_VFRE(reg_offset), reg); @@ -488,7 +511,7 @@ ixgbe_vf_set_mac_addr(struct rte_eth_dev *dev, uint32_t vf, uint32_t *msgbuf) } static int -ixgbe_vf_set_multicast(struct rte_eth_dev *dev, __rte_unused uint32_t vf, uint32_t *msgbuf) +ixgbe_vf_set_multicast(struct rte_eth_dev *dev, uint32_t vf, uint32_t *msgbuf) { struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct ixgbe_vf_info *vfinfo = @@ -604,6 +627,18 @@ ixgbe_get_vf_queues(struct rte_eth_dev *dev, uint32_t vf, uint32_t *msgbuf) struct ixgbe_vf_info *vfinfo = *IXGBE_DEV_PRIVATE_TO_P_VFDATA(dev->data->dev_private); uint32_t default_q = vf * RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool; + struct rte_eth_conf *eth_conf; + struct rte_eth_vmdq_dcb_tx_conf *vmdq_dcb_tx_conf; + u8 num_tcs; + struct ixgbe_hw *hw; + u32 vmvir; +#define IXGBE_VMVIR_VLANA_MASK 0xC0000000 +#define IXGBE_VMVIR_VLAN_VID_MASK 0x00000FFF +#define IXGBE_VMVIR_VLAN_UP_MASK 0x0000E000 +#define VLAN_PRIO_SHIFT 13 + u32 vlana; + u32 vid; + u32 user_priority; /* Verify if the PF supports the mbox APIs version or not */ switch (vfinfo[vf].api_version) { @@ -622,10 +657,51 @@ ixgbe_get_vf_queues(struct rte_eth_dev *dev, uint32_t vf, uint32_t *msgbuf) /* Notify VF of default queue */ msgbuf[IXGBE_VF_DEF_QUEUE] = default_q; - /* - * FIX ME if it needs fill msgbuf[IXGBE_VF_TRANS_VLAN] - * for VLAN strip or VMDQ_DCB or VMDQ_DCB_RSS - */ + /* Notify VF of number of DCB traffic classes */ + eth_conf = &dev->data->dev_conf; + switch (eth_conf->txmode.mq_mode) { + case ETH_MQ_TX_NONE: + case ETH_MQ_TX_DCB: + RTE_LOG(ERR, PMD, "PF must work with virtualization for VF %u" + ", but its tx mode = %d\n", vf, + eth_conf->txmode.mq_mode); + return -1; + + case ETH_MQ_TX_VMDQ_DCB: + vmdq_dcb_tx_conf = ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; + switch (vmdq_dcb_tx_conf->nb_queue_pools) { + case ETH_16_POOLS: + num_tcs = ETH_8_TCS; + break; + case ETH_32_POOLS: + num_tcs = ETH_4_TCS; + break; + default: + return -1; + } + break; + + /* ETH_MQ_TX_VMDQ_ONLY, DCB not enabled */ + case ETH_MQ_TX_VMDQ_ONLY: + hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + vmvir = IXGBE_READ_REG(hw, IXGBE_VMVIR(vf)); + vlana = vmvir & IXGBE_VMVIR_VLANA_MASK; + vid = vmvir & IXGBE_VMVIR_VLAN_VID_MASK; + user_priority = + (vmvir & IXGBE_VMVIR_VLAN_UP_MASK) >> VLAN_PRIO_SHIFT; + if ((vlana == IXGBE_VMVIR_VLANA_DEFAULT) && + ((vid != 0) || (user_priority != 0))) + num_tcs = 1; + else + num_tcs = 0; + break; + + default: + RTE_LOG(ERR, PMD, "PF work with invalid mode = %d\n", + eth_conf->txmode.mq_mode); + return -1; + } + msgbuf[IXGBE_VF_TRANS_VLAN] = num_tcs; return 0; } @@ -660,6 +736,7 @@ ixgbe_rcv_msg_from_vf(struct rte_eth_dev *dev, uint16_t vf) struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct ixgbe_vf_info *vfinfo = *IXGBE_DEV_PRIVATE_TO_P_VFDATA(dev->data->dev_private); + struct rte_pmd_ixgbe_mb_event_param ret_param; retval = ixgbe_read_mbx(hw, msgbuf, mbx_size, vf); if (retval) { @@ -674,27 +751,56 @@ ixgbe_rcv_msg_from_vf(struct rte_eth_dev *dev, uint16_t vf) /* flush the ack before we write any messages back */ IXGBE_WRITE_FLUSH(hw); + /** + * initialise structure to send to user application + * will return response from user in retval field + */ + ret_param.retval = RTE_PMD_IXGBE_MB_EVENT_PROCEED; + ret_param.vfid = vf; + ret_param.msg_type = msgbuf[0] & 0xFFFF; + ret_param.msg = (void *)msgbuf; + /* perform VF reset */ if (msgbuf[0] == IXGBE_VF_RESET) { int ret = ixgbe_vf_reset(dev, vf, msgbuf); vfinfo[vf].clear_to_send = true; + + /* notify application about VF reset */ + _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_VF_MBOX, + NULL, &ret_param); return ret; } + /** + * ask user application if we allowed to perform those functions + * if we get ret_param.retval == RTE_PMD_IXGBE_MB_EVENT_PROCEED + * then business as usual, + * if 0, do nothing and send ACK to VF + * if ret_param.retval > 1, do nothing and send NAK to VF + */ + _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_VF_MBOX, + NULL, &ret_param); + + retval = ret_param.retval; + /* check & process VF to PF mailbox message */ switch ((msgbuf[0] & 0xFFFF)) { case IXGBE_VF_SET_MAC_ADDR: - retval = ixgbe_vf_set_mac_addr(dev, vf, msgbuf); + if (retval == RTE_PMD_IXGBE_MB_EVENT_PROCEED) + retval = ixgbe_vf_set_mac_addr(dev, vf, msgbuf); break; case IXGBE_VF_SET_MULTICAST: - retval = ixgbe_vf_set_multicast(dev, vf, msgbuf); + if (retval == RTE_PMD_IXGBE_MB_EVENT_PROCEED) + retval = ixgbe_vf_set_multicast(dev, vf, msgbuf); break; case IXGBE_VF_SET_LPE: - retval = ixgbe_set_vf_lpe(dev, vf, msgbuf); + if (retval == RTE_PMD_IXGBE_MB_EVENT_PROCEED) + retval = ixgbe_set_vf_lpe(dev, vf, msgbuf); break; case IXGBE_VF_SET_VLAN: - retval = ixgbe_vf_set_vlan(dev, vf, msgbuf); + if (retval == RTE_PMD_IXGBE_MB_EVENT_PROCEED) + retval = ixgbe_vf_set_vlan(dev, vf, msgbuf); break; case IXGBE_VF_API_NEGOTIATE: retval = ixgbe_negotiate_vf_api(dev, vf, msgbuf); @@ -704,7 +810,8 @@ ixgbe_rcv_msg_from_vf(struct rte_eth_dev *dev, uint16_t vf) msg_size = IXGBE_VF_GET_QUEUE_MSG_SIZE; break; case IXGBE_VF_UPDATE_XCAST_MODE: - retval = ixgbe_set_vf_mc_promisc(dev, vf, msgbuf); + if (retval == RTE_PMD_IXGBE_MB_EVENT_PROCEED) + retval = ixgbe_set_vf_mc_promisc(dev, vf, msgbuf); break; default: PMD_DRV_LOG(DEBUG, "Unhandled Msg %8.8x", (unsigned)msgbuf[0]); diff --git a/dpdk/drivers/net/ixgbe/ixgbe_regs.h b/dpdk/drivers/net/ixgbe/ixgbe_regs.h index 773e1693..2aa48201 100644 --- a/dpdk/drivers/net/ixgbe/ixgbe_regs.h +++ b/dpdk/drivers/net/ixgbe/ixgbe_regs.h @@ -41,7 +41,7 @@ struct reg_info { uint32_t count; uint32_t stride; const char *name; -} reg_info; +}; static const struct reg_info ixgbe_regs_general[] = { {IXGBE_CTRL, 1, 1, "IXGBE_CTRL"}, diff --git a/dpdk/drivers/net/ixgbe/ixgbe_rxtx.c b/dpdk/drivers/net/ixgbe/ixgbe_rxtx.c index 8a306b06..9bc84624 100644 --- a/dpdk/drivers/net/ixgbe/ixgbe_rxtx.c +++ b/dpdk/drivers/net/ixgbe/ixgbe_rxtx.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. * Copyright 2014 6WIND S.A. * All rights reserved. * @@ -58,7 +58,6 @@ #include #include #include -#include #include #include #include @@ -71,6 +70,7 @@ #include #include #include +#include #include "ixgbe_logs.h" #include "base/ixgbe_api.h" @@ -80,13 +80,24 @@ #include "base/ixgbe_common.h" #include "ixgbe_rxtx.h" +#ifdef RTE_LIBRTE_IEEE1588 +#define IXGBE_TX_IEEE1588_TMST PKT_TX_IEEE1588_TMST +#else +#define IXGBE_TX_IEEE1588_TMST 0 +#endif /* Bit Mask to indicate what bits required for building TX context */ #define IXGBE_TX_OFFLOAD_MASK ( \ PKT_TX_VLAN_PKT | \ PKT_TX_IP_CKSUM | \ PKT_TX_L4_MASK | \ PKT_TX_TCP_SEG | \ - PKT_TX_OUTER_IP_CKSUM) + PKT_TX_MACSEC | \ + PKT_TX_OUTER_IP_CKSUM | \ + PKT_TX_SEC_OFFLOAD | \ + IXGBE_TX_IEEE1588_TMST) + +#define IXGBE_TX_OFFLOAD_NOTSUP_MASK \ + (PKT_TX_OFFLOAD_MASK ^ IXGBE_TX_OFFLOAD_MASK) #if 1 #define RTE_PMD_USE_PREFETCH @@ -101,6 +112,11 @@ #define rte_ixgbe_prefetch(p) do {} while (0) #endif +#ifdef RTE_IXGBE_INC_VECTOR +uint16_t ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts); +#endif + /********************************************************************* * * TX functions @@ -111,7 +127,7 @@ * Check for descriptors with their DD bit set and free mbufs. * Return the total number of buffers freed. */ -static inline int __attribute__((always_inline)) +static __rte_always_inline int ixgbe_tx_free_bufs(struct ixgbe_tx_queue *txq) { struct ixgbe_tx_entry *txep; @@ -132,7 +148,7 @@ ixgbe_tx_free_bufs(struct ixgbe_tx_queue *txq) for (i = 0; i < txq->tx_rs_thresh; ++i, ++txep) { /* free buffers one at a time */ - m = __rte_pktmbuf_prefree_seg(txep->mbuf); + m = rte_pktmbuf_prefree_seg(txep->mbuf); txep->mbuf = NULL; if (unlikely(m == NULL)) @@ -169,7 +185,7 @@ tx4(volatile union ixgbe_adv_tx_desc *txdp, struct rte_mbuf **pkts) int i; for (i = 0; i < 4; ++i, ++txdp, ++pkts) { - buf_dma_addr = rte_mbuf_data_dma_addr(*pkts); + buf_dma_addr = rte_mbuf_data_iova(*pkts); pkt_len = (*pkts)->data_len; /* write data to descriptor */ @@ -192,7 +208,7 @@ tx1(volatile union ixgbe_adv_tx_desc *txdp, struct rte_mbuf **pkts) uint64_t buf_dma_addr; uint32_t pkt_len; - buf_dma_addr = rte_mbuf_data_dma_addr(*pkts); + buf_dma_addr = rte_mbuf_data_iova(*pkts); pkt_len = (*pkts)->data_len; /* write data to descriptor */ @@ -322,7 +338,7 @@ tx_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, /* update tail pointer */ rte_wmb(); - IXGBE_PCI_REG_WRITE(txq->tdt_reg_addr, txq->tx_tail); + IXGBE_PCI_REG_WRITE_RELAXED(txq->tdt_reg_addr, txq->tx_tail); return nb_pkts; } @@ -353,10 +369,35 @@ ixgbe_xmit_pkts_simple(void *tx_queue, struct rte_mbuf **tx_pkts, return nb_tx; } +#ifdef RTE_IXGBE_INC_VECTOR +static uint16_t +ixgbe_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts) +{ + uint16_t nb_tx = 0; + struct ixgbe_tx_queue *txq = (struct ixgbe_tx_queue *)tx_queue; + + while (nb_pkts) { + uint16_t ret, num; + + num = (uint16_t)RTE_MIN(nb_pkts, txq->tx_rs_thresh); + ret = ixgbe_xmit_fixed_burst_vec(tx_queue, &tx_pkts[nb_tx], + num); + nb_tx += ret; + nb_pkts -= ret; + if (ret < num) + break; + } + + return nb_tx; +} +#endif + static inline void ixgbe_set_xmit_ctx(struct ixgbe_tx_queue *txq, volatile struct ixgbe_adv_tx_context_desc *ctx_txd, - uint64_t ol_flags, union ixgbe_tx_offload tx_offload) + uint64_t ol_flags, union ixgbe_tx_offload tx_offload, + __rte_unused uint64_t *mdata) { uint32_t type_tucmd_mlhl; uint32_t mss_l4len_idx = 0; @@ -440,6 +481,21 @@ ixgbe_set_xmit_ctx(struct ixgbe_tx_queue *txq, seqnum_seed |= tx_offload.l2_len << IXGBE_ADVTXD_TUNNEL_LEN; } +#ifdef RTE_LIBRTE_SECURITY + if (ol_flags & PKT_TX_SEC_OFFLOAD) { + union ixgbe_crypto_tx_desc_md *md = + (union ixgbe_crypto_tx_desc_md *)mdata; + seqnum_seed |= + (IXGBE_ADVTXD_IPSEC_SA_INDEX_MASK & md->sa_idx); + type_tucmd_mlhl |= md->enc ? + (IXGBE_ADVTXD_TUCMD_IPSEC_TYPE_ESP | + IXGBE_ADVTXD_TUCMD_IPSEC_ENCRYPT_EN) : 0; + type_tucmd_mlhl |= + (md->pad_len & IXGBE_ADVTXD_IPSEC_ESP_LEN_MASK); + tx_offload_mask.sa_idx |= ~0; + tx_offload_mask.sec_pad_len |= ~0; + } +#endif txq->ctx_cache[ctx_idx].flags = ol_flags; txq->ctx_cache[ctx_idx].tx_offload.data[0] = @@ -520,6 +576,8 @@ tx_desc_ol_flags_to_cmdtype(uint64_t ol_flags) cmdtype |= IXGBE_ADVTXD_DCMD_TSE; if (ol_flags & PKT_TX_OUTER_IP_CKSUM) cmdtype |= (1 << IXGBE_ADVTXD_OUTERIPCS_SHIFT); + if (ol_flags & PKT_TX_MACSEC) + cmdtype |= IXGBE_ADVTXD_MAC_LINKSEC; return cmdtype; } @@ -616,6 +674,9 @@ ixgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint32_t ctx = 0; uint32_t new_ctx; union ixgbe_tx_offload tx_offload; +#ifdef RTE_LIBRTE_SECURITY + uint8_t use_ipsec; +#endif tx_offload.data[0] = 0; tx_offload.data[1] = 0; @@ -643,6 +704,9 @@ ixgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, * are needed for offload functionality. */ ol_flags = tx_pkt->ol_flags; +#ifdef RTE_LIBRTE_SECURITY + use_ipsec = txq->using_ipsec && (ol_flags & PKT_TX_SEC_OFFLOAD); +#endif /* If hardware offload required */ tx_ol_req = ol_flags & IXGBE_TX_OFFLOAD_MASK; @@ -654,6 +718,15 @@ ixgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, tx_offload.tso_segsz = tx_pkt->tso_segsz; tx_offload.outer_l2_len = tx_pkt->outer_l2_len; tx_offload.outer_l3_len = tx_pkt->outer_l3_len; +#ifdef RTE_LIBRTE_SECURITY + if (use_ipsec) { + union ixgbe_crypto_tx_desc_md *ipsec_mdata = + (union ixgbe_crypto_tx_desc_md *) + &tx_pkt->udata64; + tx_offload.sa_idx = ipsec_mdata->sa_idx; + tx_offload.sec_pad_len = ipsec_mdata->pad_len; + } +#endif /* If new context need be built or reuse the exist ctx. */ ctx = what_advctx_update(txq, tx_ol_req, @@ -814,7 +887,7 @@ ixgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, } ixgbe_set_xmit_ctx(txq, ctx_txd, tx_ol_req, - tx_offload); + tx_offload, &tx_pkt->udata64); txe->last_id = tx_last; tx_id = txe->next_id; @@ -832,6 +905,10 @@ ixgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, } olinfo_status |= (pkt_len << IXGBE_ADVTXD_PAYLEN_SHIFT); +#ifdef RTE_LIBRTE_SECURITY + if (use_ipsec) + olinfo_status |= IXGBE_ADVTXD_POPTS_IPSEC; +#endif m_seg = tx_pkt; do { @@ -847,7 +924,7 @@ ixgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, * Set up Transmit Data Descriptor. */ slen = m_seg->data_len; - buf_dma_addr = rte_mbuf_data_dma_addr(m_seg); + buf_dma_addr = rte_mbuf_data_iova(m_seg); txd->read.buffer_addr = rte_cpu_to_le_64(buf_dma_addr); txd->read.cmd_type_len = @@ -898,12 +975,63 @@ end_of_tx: PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_tx=%u", (unsigned) txq->port_id, (unsigned) txq->queue_id, (unsigned) tx_id, (unsigned) nb_tx); - IXGBE_PCI_REG_WRITE(txq->tdt_reg_addr, tx_id); + IXGBE_PCI_REG_WRITE_RELAXED(txq->tdt_reg_addr, tx_id); txq->tx_tail = tx_id; return nb_tx; } +/********************************************************************* + * + * TX prep functions + * + **********************************************************************/ +uint16_t +ixgbe_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) +{ + int i, ret; + uint64_t ol_flags; + struct rte_mbuf *m; + struct ixgbe_tx_queue *txq = (struct ixgbe_tx_queue *)tx_queue; + + for (i = 0; i < nb_pkts; i++) { + m = tx_pkts[i]; + ol_flags = m->ol_flags; + + /** + * Check if packet meets requirements for number of segments + * + * NOTE: for ixgbe it's always (40 - WTHRESH) for both TSO and + * non-TSO + */ + + if (m->nb_segs > IXGBE_TX_MAX_SEG - txq->wthresh) { + rte_errno = -EINVAL; + return i; + } + + if (ol_flags & IXGBE_TX_OFFLOAD_NOTSUP_MASK) { + rte_errno = -ENOTSUP; + return i; + } + +#ifdef RTE_LIBRTE_ETHDEV_DEBUG + ret = rte_validate_tx_offload(m); + if (ret != 0) { + rte_errno = ret; + return i; + } +#endif + ret = rte_net_intel_cksum_prepare(m); + if (ret != 0) { + rte_errno = ret; + return i; + } + } + + return i; +} + /********************************************************************* * * RX functions @@ -992,282 +1120,279 @@ end_of_tx: #define IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT_TCP 0X9D #define IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT_UDP 0XAD -#define IXGBE_PACKET_TYPE_MAX 0X80 -#define IXGBE_PACKET_TYPE_TN_MAX 0X100 -#define IXGBE_PACKET_TYPE_SHIFT 0X04 +/** + * Use 2 different table for normal packet and tunnel packet + * to save the space. + */ +const uint32_t + ptype_table[IXGBE_PACKET_TYPE_MAX] __rte_cache_aligned = { + [IXGBE_PACKET_TYPE_ETHER] = RTE_PTYPE_L2_ETHER, + [IXGBE_PACKET_TYPE_IPV4] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4, + [IXGBE_PACKET_TYPE_IPV4_TCP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_TCP, + [IXGBE_PACKET_TYPE_IPV4_UDP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_UDP, + [IXGBE_PACKET_TYPE_IPV4_SCTP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_SCTP, + [IXGBE_PACKET_TYPE_IPV4_EXT] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT, + [IXGBE_PACKET_TYPE_IPV4_EXT_TCP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_TCP, + [IXGBE_PACKET_TYPE_IPV4_EXT_UDP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_UDP, + [IXGBE_PACKET_TYPE_IPV4_EXT_SCTP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_SCTP, + [IXGBE_PACKET_TYPE_IPV6] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV6, + [IXGBE_PACKET_TYPE_IPV6_TCP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_TCP, + [IXGBE_PACKET_TYPE_IPV6_UDP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_UDP, + [IXGBE_PACKET_TYPE_IPV6_SCTP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_SCTP, + [IXGBE_PACKET_TYPE_IPV6_EXT] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV6_EXT, + [IXGBE_PACKET_TYPE_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_TCP, + [IXGBE_PACKET_TYPE_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_UDP, + [IXGBE_PACKET_TYPE_IPV6_EXT_SCTP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_SCTP, + [IXGBE_PACKET_TYPE_IPV4_IPV6] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV6, + [IXGBE_PACKET_TYPE_IPV4_IPV6_TCP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_TCP, + [IXGBE_PACKET_TYPE_IPV4_IPV6_UDP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_UDP, + [IXGBE_PACKET_TYPE_IPV4_IPV6_SCTP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_SCTP, + [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV6, + [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_TCP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_TCP, + [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_UDP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_UDP, + [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_SCTP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_SCTP, + [IXGBE_PACKET_TYPE_IPV4_IPV6_EXT] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV6_EXT, + [IXGBE_PACKET_TYPE_IPV4_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_TCP, + [IXGBE_PACKET_TYPE_IPV4_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_UDP, + [IXGBE_PACKET_TYPE_IPV4_IPV6_EXT_SCTP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_SCTP, + [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV6_EXT, + [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_TCP, + [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_UDP, + [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT_SCTP] = + RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP | + RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_SCTP, +}; + +const uint32_t + ptype_table_tn[IXGBE_PACKET_TYPE_TN_MAX] __rte_cache_aligned = { + [IXGBE_PACKET_TYPE_NVGRE] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | + RTE_PTYPE_INNER_L2_ETHER, + [IXGBE_PACKET_TYPE_NVGRE_IPV4] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | + RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4, + [IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | + RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4_EXT, + [IXGBE_PACKET_TYPE_NVGRE_IPV6] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | + RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6, + [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | + RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4, + [IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | + RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6_EXT, + [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_EXT] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | + RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4, + [IXGBE_PACKET_TYPE_NVGRE_IPV4_TCP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | + RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4 | + RTE_PTYPE_INNER_L4_TCP, + [IXGBE_PACKET_TYPE_NVGRE_IPV6_TCP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | + RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6 | + RTE_PTYPE_INNER_L4_TCP, + [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_TCP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | + RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4, + [IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | + RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6_EXT | + RTE_PTYPE_INNER_L4_TCP, + [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_EXT_TCP] = + RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRE | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV4, + [IXGBE_PACKET_TYPE_NVGRE_IPV4_UDP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | + RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4 | + RTE_PTYPE_INNER_L4_UDP, + [IXGBE_PACKET_TYPE_NVGRE_IPV6_UDP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | + RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6 | + RTE_PTYPE_INNER_L4_UDP, + [IXGBE_PACKET_TYPE_NVGRE_IPV6_SCTP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | + RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6 | + RTE_PTYPE_INNER_L4_SCTP, + [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_UDP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | + RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4, + [IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | + RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6_EXT | + RTE_PTYPE_INNER_L4_UDP, + [IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT_SCTP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | + RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6_EXT | + RTE_PTYPE_INNER_L4_SCTP, + [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_EXT_UDP] = + RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_GRE | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV4, + [IXGBE_PACKET_TYPE_NVGRE_IPV4_SCTP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | + RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4 | + RTE_PTYPE_INNER_L4_SCTP, + [IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT_SCTP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | + RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4_EXT | + RTE_PTYPE_INNER_L4_SCTP, + [IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT_TCP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | + RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4_EXT | + RTE_PTYPE_INNER_L4_TCP, + [IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT_UDP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | + RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4_EXT | + RTE_PTYPE_INNER_L4_UDP, + + [IXGBE_PACKET_TYPE_VXLAN] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | + RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER, + [IXGBE_PACKET_TYPE_VXLAN_IPV4] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | + RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV4, + [IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | + RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV4_EXT, + [IXGBE_PACKET_TYPE_VXLAN_IPV6] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | + RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV6, + [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | + RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV4, + [IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | + RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV6_EXT, + [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | + RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV4, + [IXGBE_PACKET_TYPE_VXLAN_IPV4_TCP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | + RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV4 | RTE_PTYPE_INNER_L4_TCP, + [IXGBE_PACKET_TYPE_VXLAN_IPV6_TCP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | + RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_TCP, + [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_TCP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | + RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV4, + [IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | + RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_TCP, + [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT_TCP] = + RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_VXLAN | + RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4, + [IXGBE_PACKET_TYPE_VXLAN_IPV4_UDP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | + RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV4 | RTE_PTYPE_INNER_L4_UDP, + [IXGBE_PACKET_TYPE_VXLAN_IPV6_UDP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | + RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_UDP, + [IXGBE_PACKET_TYPE_VXLAN_IPV6_SCTP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | + RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_SCTP, + [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_UDP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | + RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV4, + [IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | + RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_UDP, + [IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT_SCTP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | + RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_SCTP, + [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT_UDP] = + RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_VXLAN | + RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4, + [IXGBE_PACKET_TYPE_VXLAN_IPV4_SCTP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | + RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV4 | RTE_PTYPE_INNER_L4_SCTP, + [IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT_SCTP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | + RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_SCTP, + [IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT_TCP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | + RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_TCP, + [IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT_UDP] = RTE_PTYPE_L2_ETHER | + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | + RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | + RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_UDP, +}; /* @note: fix ixgbe_dev_supported_ptypes_get() if any change here. */ static inline uint32_t ixgbe_rxd_pkt_info_to_pkt_type(uint32_t pkt_info, uint16_t ptype_mask) { - /** - * Use 2 different table for normal packet and tunnel packet - * to save the space. - */ - static const uint32_t - ptype_table[IXGBE_PACKET_TYPE_MAX] __rte_cache_aligned = { - [IXGBE_PACKET_TYPE_ETHER] = RTE_PTYPE_L2_ETHER, - [IXGBE_PACKET_TYPE_IPV4] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4, - [IXGBE_PACKET_TYPE_IPV4_TCP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_TCP, - [IXGBE_PACKET_TYPE_IPV4_UDP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_UDP, - [IXGBE_PACKET_TYPE_IPV4_SCTP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_SCTP, - [IXGBE_PACKET_TYPE_IPV4_EXT] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT, - [IXGBE_PACKET_TYPE_IPV4_EXT_TCP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_TCP, - [IXGBE_PACKET_TYPE_IPV4_EXT_UDP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_UDP, - [IXGBE_PACKET_TYPE_IPV4_EXT_SCTP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_SCTP, - [IXGBE_PACKET_TYPE_IPV6] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV6, - [IXGBE_PACKET_TYPE_IPV6_TCP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_TCP, - [IXGBE_PACKET_TYPE_IPV6_UDP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_UDP, - [IXGBE_PACKET_TYPE_IPV6_SCTP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_SCTP, - [IXGBE_PACKET_TYPE_IPV6_EXT] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV6_EXT, - [IXGBE_PACKET_TYPE_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_TCP, - [IXGBE_PACKET_TYPE_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_UDP, - [IXGBE_PACKET_TYPE_IPV6_EXT_SCTP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_SCTP, - [IXGBE_PACKET_TYPE_IPV4_IPV6] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP | - RTE_PTYPE_INNER_L3_IPV6, - [IXGBE_PACKET_TYPE_IPV4_IPV6_TCP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP | - RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_TCP, - [IXGBE_PACKET_TYPE_IPV4_IPV6_UDP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP | - RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_UDP, - [IXGBE_PACKET_TYPE_IPV4_IPV6_SCTP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP | - RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_SCTP, - [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP | - RTE_PTYPE_INNER_L3_IPV6, - [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_TCP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP | - RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_TCP, - [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_UDP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP | - RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_UDP, - [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_SCTP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP | - RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_SCTP, - [IXGBE_PACKET_TYPE_IPV4_IPV6_EXT] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP | - RTE_PTYPE_INNER_L3_IPV6_EXT, - [IXGBE_PACKET_TYPE_IPV4_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP | - RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_TCP, - [IXGBE_PACKET_TYPE_IPV4_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP | - RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_UDP, - [IXGBE_PACKET_TYPE_IPV4_IPV6_EXT_SCTP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP | - RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_SCTP, - [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP | - RTE_PTYPE_INNER_L3_IPV6_EXT, - [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP | - RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_TCP, - [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP | - RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_UDP, - [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT_SCTP] = - RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP | - RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_SCTP, - }; - - static const uint32_t - ptype_table_tn[IXGBE_PACKET_TYPE_TN_MAX] __rte_cache_aligned = { - [IXGBE_PACKET_TYPE_NVGRE] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | - RTE_PTYPE_INNER_L2_ETHER, - [IXGBE_PACKET_TYPE_NVGRE_IPV4] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | - RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4, - [IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | - RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4_EXT, - [IXGBE_PACKET_TYPE_NVGRE_IPV6] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | - RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6, - [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | - RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4, - [IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | - RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6_EXT, - [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_EXT] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | - RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4, - [IXGBE_PACKET_TYPE_NVGRE_IPV4_TCP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | - RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4 | - RTE_PTYPE_INNER_L4_TCP, - [IXGBE_PACKET_TYPE_NVGRE_IPV6_TCP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | - RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6 | - RTE_PTYPE_INNER_L4_TCP, - [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_TCP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | - RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4, - [IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | - RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6_EXT | - RTE_PTYPE_INNER_L4_TCP, - [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_EXT_TCP] = - RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRE | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV4, - [IXGBE_PACKET_TYPE_NVGRE_IPV4_UDP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | - RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4 | - RTE_PTYPE_INNER_L4_UDP, - [IXGBE_PACKET_TYPE_NVGRE_IPV6_UDP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | - RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6 | - RTE_PTYPE_INNER_L4_UDP, - [IXGBE_PACKET_TYPE_NVGRE_IPV6_SCTP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | - RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6 | - RTE_PTYPE_INNER_L4_SCTP, - [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_UDP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | - RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4, - [IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | - RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6_EXT | - RTE_PTYPE_INNER_L4_UDP, - [IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT_SCTP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | - RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6_EXT | - RTE_PTYPE_INNER_L4_SCTP, - [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_EXT_UDP] = - RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_TUNNEL_GRE | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV4, - [IXGBE_PACKET_TYPE_NVGRE_IPV4_SCTP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | - RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4 | - RTE_PTYPE_INNER_L4_SCTP, - [IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT_SCTP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | - RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4_EXT | - RTE_PTYPE_INNER_L4_SCTP, - [IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT_TCP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | - RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4_EXT | - RTE_PTYPE_INNER_L4_TCP, - [IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT_UDP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | - RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4_EXT | - RTE_PTYPE_INNER_L4_UDP, - - [IXGBE_PACKET_TYPE_VXLAN] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | - RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER, - [IXGBE_PACKET_TYPE_VXLAN_IPV4] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | - RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV4, - [IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | - RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV4_EXT, - [IXGBE_PACKET_TYPE_VXLAN_IPV6] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | - RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV6, - [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | - RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV4, - [IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | - RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV6_EXT, - [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | - RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV4, - [IXGBE_PACKET_TYPE_VXLAN_IPV4_TCP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | - RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV4 | RTE_PTYPE_INNER_L4_TCP, - [IXGBE_PACKET_TYPE_VXLAN_IPV6_TCP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | - RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_TCP, - [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_TCP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | - RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV4, - [IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | - RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_TCP, - [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT_TCP] = - RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_VXLAN | - RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4, - [IXGBE_PACKET_TYPE_VXLAN_IPV4_UDP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | - RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV4 | RTE_PTYPE_INNER_L4_UDP, - [IXGBE_PACKET_TYPE_VXLAN_IPV6_UDP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | - RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_UDP, - [IXGBE_PACKET_TYPE_VXLAN_IPV6_SCTP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | - RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_SCTP, - [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_UDP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | - RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV4, - [IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | - RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_UDP, - [IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT_SCTP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | - RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_SCTP, - [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT_UDP] = - RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_VXLAN | - RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4, - [IXGBE_PACKET_TYPE_VXLAN_IPV4_SCTP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | - RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV4 | RTE_PTYPE_INNER_L4_SCTP, - [IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT_SCTP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | - RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_SCTP, - [IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT_TCP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | - RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_TCP, - [IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT_UDP] = RTE_PTYPE_L2_ETHER | - RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | - RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | - RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_UDP, - }; if (unlikely(pkt_info & IXGBE_RXDADV_PKTTYPE_ETQF)) return RTE_PTYPE_UNKNOWN; @@ -1345,7 +1470,9 @@ rx_desc_error_to_pkt_flags(uint32_t rx_status) * Bit 30: L4I, L4I integrity error */ static uint64_t error_to_pkt_flags_map[4] = { - 0, PKT_RX_L4_CKSUM_BAD, PKT_RX_IP_CKSUM_BAD, + PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD, + PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_BAD, + PKT_RX_IP_CKSUM_BAD | PKT_RX_L4_CKSUM_GOOD, PKT_RX_IP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD }; pkt_flags = error_to_pkt_flags_map[(rx_status >> @@ -1356,6 +1483,14 @@ rx_desc_error_to_pkt_flags(uint32_t rx_status) pkt_flags |= PKT_RX_EIP_CKSUM_BAD; } +#ifdef RTE_LIBRTE_SECURITY + if (rx_status & IXGBE_RXD_STAT_SECP) { + pkt_flags |= PKT_RX_SEC_OFFLOAD; + if (rx_status & IXGBE_RXDADV_LNKSEC_ERROR_BAD_SIG) + pkt_flags |= PKT_RX_SEC_OFFLOAD_FAILED; + } +#endif + return pkt_flags; } @@ -1401,17 +1536,19 @@ ixgbe_rx_scan_hw_ring(struct ixgbe_rx_queue *rxq) for (i = 0; i < RTE_PMD_IXGBE_RX_MAX_BURST; i += LOOK_AHEAD, rxdp += LOOK_AHEAD, rxep += LOOK_AHEAD) { /* Read desc statuses backwards to avoid race condition */ - for (j = LOOK_AHEAD-1; j >= 0; --j) + for (j = 0; j < LOOK_AHEAD; j++) s[j] = rte_le_to_cpu_32(rxdp[j].wb.upper.status_error); - for (j = LOOK_AHEAD - 1; j >= 0; --j) - pkt_info[j] = rte_le_to_cpu_32(rxdp[j].wb.lower. - lo_dword.data); + rte_smp_rmb(); /* Compute how many status bits were set */ - nb_dd = 0; - for (j = 0; j < LOOK_AHEAD; ++j) - nb_dd += s[j] & IXGBE_RXDADV_STAT_DD; + for (nb_dd = 0; nb_dd < LOOK_AHEAD && + (s[nb_dd] & IXGBE_RXDADV_STAT_DD); nb_dd++) + ; + + for (j = 0; j < nb_dd; j++) + pkt_info[j] = rte_le_to_cpu_32(rxdp[j].wb.lower. + lo_dword.data); nb_rx += nb_dd; @@ -1489,8 +1626,6 @@ ixgbe_rx_alloc_bufs(struct ixgbe_rx_queue *rxq, bool reset_mbuf) /* populate the static rte mbuf fields */ mb = rxep[i].mbuf; if (reset_mbuf) { - mb->next = NULL; - mb->nb_segs = 1; mb->port = rxq->port_id; } @@ -1498,7 +1633,7 @@ ixgbe_rx_alloc_bufs(struct ixgbe_rx_queue *rxq, bool reset_mbuf) mb->data_off = RTE_PKTMBUF_HEADROOM; /* populate the descriptors */ - dma_addr = rte_cpu_to_le_64(rte_mbuf_data_dma_addr_default(mb)); + dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(mb)); rxdp[i].read.hdr_addr = 0; rxdp[i].read.pkt_addr = dma_addr; } @@ -1580,7 +1715,8 @@ rx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, /* update tail pointer */ rte_wmb(); - IXGBE_PCI_REG_WRITE(rxq->rdt_reg_addr, cur_free_trigger); + IXGBE_PCI_REG_WRITE_RELAXED(rxq->rdt_reg_addr, + cur_free_trigger); } if (rxq->rx_tail >= rxq->nb_rx_desc) @@ -1729,7 +1865,7 @@ ixgbe_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, rxm = rxe->mbuf; rxe->mbuf = nmb; dma_addr = - rte_cpu_to_le_64(rte_mbuf_data_dma_addr_default(nmb)); + rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb)); rxdp->read.hdr_addr = 0; rxdp->read.pkt_addr = dma_addr; @@ -1757,7 +1893,7 @@ ixgbe_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, rxm->port = rxq->port_id; pkt_info = rte_le_to_cpu_32(rxd.wb.lower.lo_dword.data); - /* Only valid if PKT_RX_VLAN_PKT set in pkt_flags */ + /* Only valid if PKT_RX_VLAN set in pkt_flags */ rxm->vlan_tci = rte_le_to_cpu_16(rxd.wb.upper.vlan); pkt_flags = rx_desc_status_to_pkt_flags(staterr, vlan_flags); @@ -1848,7 +1984,7 @@ ixgbe_fill_cluster_head_buf( head->port = rxq->port_id; - /* The vlan_tci field is only valid when PKT_RX_VLAN_PKT is + /* The vlan_tci field is only valid when PKT_RX_VLAN is * set in the pkt_flags field. */ head->vlan_tci = rte_le_to_cpu_16(desc->wb.upper.vlan); @@ -1984,8 +2120,8 @@ next_desc: if (!ixgbe_rx_alloc_bufs(rxq, false)) { rte_wmb(); - IXGBE_PCI_REG_WRITE(rxq->rdt_reg_addr, - next_rdt); + IXGBE_PCI_REG_WRITE_RELAXED(rxq->rdt_reg_addr, + next_rdt); nb_hold -= rxq->rx_free_thresh; } else { PMD_RX_LOG(DEBUG, "RX bulk alloc failed " @@ -2023,7 +2159,7 @@ next_desc: if (!bulk_alloc) { __le64 dma = - rte_cpu_to_le_64(rte_mbuf_data_dma_addr_default(nmb)); + rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb)); /* * Update RX descriptor with the physical address of the * new data buffer of the new allocated mbuf. @@ -2097,12 +2233,6 @@ next_desc: goto next_desc; } - /* - * This is the last buffer of the received packet - return - * the current cluster to the user. - */ - rxm->next = NULL; - /* Initialize the first mbuf of the returned packet */ ixgbe_fill_cluster_head_buf(first_seg, &rxd, rxq, staterr); @@ -2156,7 +2286,7 @@ next_desc: rxq->port_id, rxq->queue_id, rx_id, nb_hold, nb_rx); rte_wmb(); - IXGBE_PCI_REG_WRITE(rxq->rdt_reg_addr, prev_id); + IXGBE_PCI_REG_WRITE_RELAXED(rxq->rdt_reg_addr, prev_id); nb_hold = 0; } @@ -2278,9 +2408,13 @@ void __attribute__((cold)) ixgbe_set_tx_function(struct rte_eth_dev *dev, struct ixgbe_tx_queue *txq) { /* Use a simple Tx queue (no offloads, no multi segs) if possible */ - if (((txq->txq_flags & IXGBE_SIMPLE_FLAGS) == IXGBE_SIMPLE_FLAGS) - && (txq->tx_rs_thresh >= RTE_PMD_IXGBE_TX_MAX_BURST)) { + if (((txq->txq_flags & IXGBE_SIMPLE_FLAGS) == IXGBE_SIMPLE_FLAGS) && +#ifdef RTE_LIBRTE_SECURITY + !(txq->using_ipsec) && +#endif + (txq->tx_rs_thresh >= RTE_PMD_IXGBE_TX_MAX_BURST)) { PMD_INIT_LOG(DEBUG, "Using simple tx code path"); + dev->tx_pkt_prepare = NULL; #ifdef RTE_IXGBE_INC_VECTOR if (txq->tx_rs_thresh <= RTE_IXGBE_TX_MAX_FREE_BUF_SZ && (rte_eal_process_type() != RTE_PROC_PRIMARY || @@ -2301,6 +2435,7 @@ ixgbe_set_tx_function(struct rte_eth_dev *dev, struct ixgbe_tx_queue *txq) (unsigned long)txq->tx_rs_thresh, (unsigned long)RTE_PMD_IXGBE_TX_MAX_BURST); dev->tx_pkt_burst = ixgbe_xmit_pkts; + dev->tx_pkt_prepare = ixgbe_prep_pkts; } } @@ -2447,6 +2582,10 @@ ixgbe_dev_tx_queue_setup(struct rte_eth_dev *dev, txq->txq_flags = tx_conf->txq_flags; txq->ops = &def_txq_ops; txq->tx_deferred_start = tx_conf->tx_deferred_start; +#ifdef RTE_LIBRTE_SECURITY + txq->using_ipsec = !!(dev->data->dev_conf.txmode.offloads & + DEV_TX_OFFLOAD_SECURITY); +#endif /* * Modification to set VFTDT for virtual function if vf is detected @@ -2460,7 +2599,7 @@ ixgbe_dev_tx_queue_setup(struct rte_eth_dev *dev, else txq->tdt_reg_addr = IXGBE_PCI_REG_ADDR(hw, IXGBE_TDT(txq->reg_idx)); - txq->tx_ring_phys_addr = rte_mem_phy2mch(tz->memseg_id, tz->phys_addr); + txq->tx_ring_phys_addr = tz->iova; txq->tx_ring = (union ixgbe_adv_tx_desc *) tz->addr; /* Allocate software ring */ @@ -2500,7 +2639,7 @@ ixgbe_dev_tx_queue_setup(struct rte_eth_dev *dev, static void __attribute__((cold)) ixgbe_free_sc_cluster(struct rte_mbuf *m) { - uint8_t i, nb_segs = m->nb_segs; + uint16_t i, nb_segs = m->nb_segs; struct rte_mbuf *next_seg; for (i = 0; i < nb_segs; i++) { @@ -2584,7 +2723,6 @@ check_rx_burst_bulk_alloc_preconditions(struct ixgbe_rx_queue *rxq) * rxq->rx_free_thresh >= RTE_PMD_IXGBE_RX_MAX_BURST * rxq->rx_free_thresh < rxq->nb_rx_desc * (rxq->nb_rx_desc % rxq->rx_free_thresh) == 0 - * rxq->nb_rx_desc<(IXGBE_MAX_RING_DESC-RTE_PMD_IXGBE_RX_MAX_BURST) * Scattered packets are not supported. This should be checked * outside of this function. */ @@ -2606,15 +2744,6 @@ check_rx_burst_bulk_alloc_preconditions(struct ixgbe_rx_queue *rxq) "rxq->rx_free_thresh=%d", rxq->nb_rx_desc, rxq->rx_free_thresh); ret = -EINVAL; - } else if (!(rxq->nb_rx_desc < - (IXGBE_MAX_RING_DESC - RTE_PMD_IXGBE_RX_MAX_BURST))) { - PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: " - "rxq->nb_rx_desc=%d, " - "IXGBE_MAX_RING_DESC=%d, " - "RTE_PMD_IXGBE_RX_MAX_BURST=%d", - rxq->nb_rx_desc, IXGBE_MAX_RING_DESC, - RTE_PMD_IXGBE_RX_MAX_BURST); - ret = -EINVAL; } return ret; @@ -2631,12 +2760,7 @@ ixgbe_reset_rx_queue(struct ixgbe_adapter *adapter, struct ixgbe_rx_queue *rxq) /* * By default, the Rx queue setup function allocates enough memory for * IXGBE_MAX_RING_DESC. The Rx Burst bulk allocation function requires - * extra memory at the end of the descriptor ring to be zero'd out. A - * pre-condition for using the Rx burst bulk alloc function is that the - * number of descriptors is less than or equal to - * (IXGBE_MAX_RING_DESC - RTE_PMD_IXGBE_RX_MAX_BURST). Check all the - * constraints here to see if we need to zero out memory after the end - * of the H/W descriptor ring. + * extra memory at the end of the descriptor ring to be zero'd out. */ if (adapter->rx_bulk_alloc_allowed) /* zero out extra memory */ @@ -2777,7 +2901,7 @@ ixgbe_dev_rx_queue_setup(struct rte_eth_dev *dev, IXGBE_PCI_REG_ADDR(hw, IXGBE_RDH(rxq->reg_idx)); } - rxq->rx_ring_phys_addr = rte_mem_phy2mch(rz->memseg_id, rz->phys_addr); + rxq->rx_ring_phys_addr = rz->iova; rxq->rx_ring = (union ixgbe_adv_rx_desc *) rz->addr; /* @@ -2856,11 +2980,6 @@ ixgbe_dev_rx_queue_count(struct rte_eth_dev *dev, uint16_t rx_queue_id) struct ixgbe_rx_queue *rxq; uint32_t desc = 0; - if (rx_queue_id >= dev->data->nb_rx_queues) { - PMD_RX_LOG(ERR, "Invalid RX queue id=%d", rx_queue_id); - return 0; - } - rxq = dev->data->rx_queues[rx_queue_id]; rxdp = &(rxq->rx_ring[rxq->rx_tail]); @@ -2895,6 +3014,63 @@ ixgbe_dev_rx_descriptor_done(void *rx_queue, uint16_t offset) rte_cpu_to_le_32(IXGBE_RXDADV_STAT_DD)); } +int +ixgbe_dev_rx_descriptor_status(void *rx_queue, uint16_t offset) +{ + struct ixgbe_rx_queue *rxq = rx_queue; + volatile uint32_t *status; + uint32_t nb_hold, desc; + + if (unlikely(offset >= rxq->nb_rx_desc)) + return -EINVAL; + +#ifdef RTE_IXGBE_INC_VECTOR + if (rxq->rx_using_sse) + nb_hold = rxq->rxrearm_nb; + else +#endif + nb_hold = rxq->nb_rx_hold; + if (offset >= rxq->nb_rx_desc - nb_hold) + return RTE_ETH_RX_DESC_UNAVAIL; + + desc = rxq->rx_tail + offset; + if (desc >= rxq->nb_rx_desc) + desc -= rxq->nb_rx_desc; + + status = &rxq->rx_ring[desc].wb.upper.status_error; + if (*status & rte_cpu_to_le_32(IXGBE_RXDADV_STAT_DD)) + return RTE_ETH_RX_DESC_DONE; + + return RTE_ETH_RX_DESC_AVAIL; +} + +int +ixgbe_dev_tx_descriptor_status(void *tx_queue, uint16_t offset) +{ + struct ixgbe_tx_queue *txq = tx_queue; + volatile uint32_t *status; + uint32_t desc; + + if (unlikely(offset >= txq->nb_tx_desc)) + return -EINVAL; + + desc = txq->tx_tail + offset; + /* go to next desc that has the RS bit */ + desc = ((desc + txq->tx_rs_thresh - 1) / txq->tx_rs_thresh) * + txq->tx_rs_thresh; + if (desc >= txq->nb_tx_desc) { + desc -= txq->nb_tx_desc; + if (desc >= txq->nb_tx_desc) + desc -= txq->nb_tx_desc; + } + + status = &txq->tx_ring[desc].wb.status; + if (*status & rte_cpu_to_le_32(IXGBE_ADVTXD_STAT_DD)) + return RTE_ETH_TX_DESC_DONE; + + return RTE_ETH_TX_DESC_FULL; +} + void __attribute__((cold)) ixgbe_dev_clear_queues(struct rte_eth_dev *dev) { @@ -3312,15 +3488,15 @@ ixgbe_vmdq_dcb_configure(struct rte_eth_dev *dev) /** * ixgbe_dcb_config_tx_hw_config - Configure general DCB TX parameters - * @hw: pointer to hardware structure + * @dev: pointer to eth_dev structure * @dcb_config: pointer to ixgbe_dcb_config structure */ static void -ixgbe_dcb_tx_hw_config(struct ixgbe_hw *hw, +ixgbe_dcb_tx_hw_config(struct rte_eth_dev *dev, struct ixgbe_dcb_config *dcb_config) { uint32_t reg; - uint32_t q; + struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); PMD_INIT_FUNC_TRACE(); if (hw->mac.type != ixgbe_mac_82598EB) { @@ -3339,11 +3515,6 @@ ixgbe_dcb_tx_hw_config(struct ixgbe_hw *hw, reg |= IXGBE_MTQC_VT_ENA; IXGBE_WRITE_REG(hw, IXGBE_MTQC, reg); - /* Disable drop for all queues */ - for (q = 0; q < 128; q++) - IXGBE_WRITE_REG(hw, IXGBE_QDE, - (IXGBE_QDE_WRITE | (q << IXGBE_QDE_IDX_SHIFT))); - /* Enable the Tx desc arbiter */ reg = IXGBE_READ_REG(hw, IXGBE_RTTDCS); reg &= ~IXGBE_RTTDCS_ARBDIS; @@ -3377,7 +3548,7 @@ ixgbe_vmdq_dcb_hw_tx_config(struct rte_eth_dev *dev, vmdq_tx_conf->nb_queue_pools == ETH_16_POOLS ? 0xFFFF : 0xFFFFFFFF); /*Configure general DCB TX parameters*/ - ixgbe_dcb_tx_hw_config(hw, dcb_config); + ixgbe_dcb_tx_hw_config(dev, dcb_config); } static void @@ -3397,12 +3568,19 @@ ixgbe_vmdq_dcb_rx_config(struct rte_eth_dev *dev, dcb_config->num_tcs.pg_tcs = ETH_4_TCS; dcb_config->num_tcs.pfc_tcs = ETH_4_TCS; } + + /* Initialize User Priority to Traffic Class mapping */ + for (j = 0; j < IXGBE_DCB_MAX_TRAFFIC_CLASS; j++) { + tc = &dcb_config->tc_config[j]; + tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = 0; + } + /* User Priority to Traffic Class mapping */ for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { j = vmdq_rx_conf->dcb_tc[i]; tc = &dcb_config->tc_config[j]; - tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = - (uint8_t)(1 << j); + tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap |= + (uint8_t)(1 << i); } } @@ -3424,12 +3602,18 @@ ixgbe_dcb_vt_tx_config(struct rte_eth_dev *dev, dcb_config->num_tcs.pfc_tcs = ETH_4_TCS; } + /* Initialize User Priority to Traffic Class mapping */ + for (j = 0; j < IXGBE_DCB_MAX_TRAFFIC_CLASS; j++) { + tc = &dcb_config->tc_config[j]; + tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = 0; + } + /* User Priority to Traffic Class mapping */ for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { j = vmdq_tx_conf->dcb_tc[i]; tc = &dcb_config->tc_config[j]; - tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = - (uint8_t)(1 << j); + tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap |= + (uint8_t)(1 << i); } } @@ -3445,12 +3629,18 @@ ixgbe_dcb_rx_config(struct rte_eth_dev *dev, dcb_config->num_tcs.pg_tcs = (uint8_t)rx_conf->nb_tcs; dcb_config->num_tcs.pfc_tcs = (uint8_t)rx_conf->nb_tcs; + /* Initialize User Priority to Traffic Class mapping */ + for (j = 0; j < IXGBE_DCB_MAX_TRAFFIC_CLASS; j++) { + tc = &dcb_config->tc_config[j]; + tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = 0; + } + /* User Priority to Traffic Class mapping */ for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { j = rx_conf->dcb_tc[i]; tc = &dcb_config->tc_config[j]; - tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = - (uint8_t)(1 << j); + tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap |= + (uint8_t)(1 << i); } } @@ -3466,27 +3656,35 @@ ixgbe_dcb_tx_config(struct rte_eth_dev *dev, dcb_config->num_tcs.pg_tcs = (uint8_t)tx_conf->nb_tcs; dcb_config->num_tcs.pfc_tcs = (uint8_t)tx_conf->nb_tcs; + /* Initialize User Priority to Traffic Class mapping */ + for (j = 0; j < IXGBE_DCB_MAX_TRAFFIC_CLASS; j++) { + tc = &dcb_config->tc_config[j]; + tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = 0; + } + /* User Priority to Traffic Class mapping */ for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { j = tx_conf->dcb_tc[i]; tc = &dcb_config->tc_config[j]; - tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = - (uint8_t)(1 << j); + tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap |= + (uint8_t)(1 << i); } } /** * ixgbe_dcb_rx_hw_config - Configure general DCB RX HW parameters - * @hw: pointer to hardware structure + * @dev: pointer to eth_dev structure * @dcb_config: pointer to ixgbe_dcb_config structure */ static void -ixgbe_dcb_rx_hw_config(struct ixgbe_hw *hw, - struct ixgbe_dcb_config *dcb_config) +ixgbe_dcb_rx_hw_config(struct rte_eth_dev *dev, + struct ixgbe_dcb_config *dcb_config) { uint32_t reg; uint32_t vlanctrl; uint8_t i; + uint32_t q; + struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); PMD_INIT_FUNC_TRACE(); /* @@ -3524,6 +3722,21 @@ ixgbe_dcb_rx_hw_config(struct ixgbe_hw *hw, } IXGBE_WRITE_REG(hw, IXGBE_MRQC, reg); + + if (RTE_ETH_DEV_SRIOV(dev).active == 0) { + /* Disable drop for all queues in VMDQ mode*/ + for (q = 0; q < IXGBE_MAX_RX_QUEUE_NUM; q++) + IXGBE_WRITE_REG(hw, IXGBE_QDE, + (IXGBE_QDE_WRITE | + (q << IXGBE_QDE_IDX_SHIFT))); + } else { + /* Enable drop for all queues in SRIOV mode */ + for (q = 0; q < IXGBE_MAX_RX_QUEUE_NUM; q++) + IXGBE_WRITE_REG(hw, IXGBE_QDE, + (IXGBE_QDE_WRITE | + (q << IXGBE_QDE_IDX_SHIFT) | + IXGBE_QDE_ENABLE)); + } } /* VLNCTRL: enable vlan filtering and allow all vlan tags through */ @@ -3614,6 +3827,8 @@ ixgbe_dcb_hw_configure(struct rte_eth_dev *dev, uint32_t max_frame = dev->data->mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct ixgbe_bw_conf *bw_conf = + IXGBE_DEV_PRIVATE_TO_BW_CONF(dev->data->dev_private); switch (dev->data->dev_conf.rxmode.mq_mode) { case ETH_MQ_RX_VMDQ_DCB: @@ -3636,7 +3851,7 @@ ixgbe_dcb_hw_configure(struct rte_eth_dev *dev, /* Get dcb TX configuration parameters from rte_eth_conf */ ixgbe_dcb_rx_config(dev, dcb_config); /*Configure general DCB RX parameters*/ - ixgbe_dcb_rx_hw_config(hw, dcb_config); + ixgbe_dcb_rx_hw_config(dev, dcb_config); break; default: PMD_INIT_LOG(ERR, "Incorrect DCB RX mode configuration"); @@ -3660,7 +3875,7 @@ ixgbe_dcb_hw_configure(struct rte_eth_dev *dev, /*get DCB TX configuration parameters from rte_eth_conf*/ ixgbe_dcb_tx_config(dev, dcb_config); /*Configure general DCB TX parameters*/ - ixgbe_dcb_tx_hw_config(hw, dcb_config); + ixgbe_dcb_tx_hw_config(dev, dcb_config); break; default: PMD_INIT_LOG(ERR, "Incorrect DCB TX mode configuration"); @@ -3685,8 +3900,9 @@ ixgbe_dcb_hw_configure(struct rte_eth_dev *dev, /* Re-configure 4 TCs BW */ for (i = 0; i < nb_tcs; i++) { tc = &dcb_config->tc_config[i]; - tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent = - (uint8_t)(100 / nb_tcs); + if (bw_conf->tc_num != nb_tcs) + tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent = + (uint8_t)(100 / nb_tcs); tc->path[IXGBE_DCB_RX_CONFIG].bwg_percent = (uint8_t)(100 / nb_tcs); } @@ -3695,6 +3911,16 @@ ixgbe_dcb_hw_configure(struct rte_eth_dev *dev, tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent = 0; tc->path[IXGBE_DCB_RX_CONFIG].bwg_percent = 0; } + } else { + /* Re-configure 8 TCs BW */ + for (i = 0; i < nb_tcs; i++) { + tc = &dcb_config->tc_config[i]; + if (bw_conf->tc_num != nb_tcs) + tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent = + (uint8_t)(100 / nb_tcs + (i & 1)); + tc->path[IXGBE_DCB_RX_CONFIG].bwg_percent = + (uint8_t)(100 / nb_tcs + (i & 1)); + } } switch (hw->mac.type) { @@ -3809,7 +4035,7 @@ void ixgbe_configure_dcb(struct rte_eth_dev *dev) (dev_conf->rxmode.mq_mode != ETH_MQ_RX_DCB_RSS)) return; - if (dev->data->nb_rx_queues != ETH_DCB_NUM_QUEUES) + if (dev->data->nb_rx_queues > ETH_DCB_NUM_QUEUES) return; /** Configure DCB hardware **/ @@ -3958,14 +4184,11 @@ ixgbe_alloc_rx_queue_mbufs(struct ixgbe_rx_queue *rxq) return -ENOMEM; } - rte_mbuf_refcnt_set(mbuf, 1); - mbuf->next = NULL; mbuf->data_off = RTE_PKTMBUF_HEADROOM; - mbuf->nb_segs = 1; mbuf->port = rxq->port_id; dma_addr = - rte_cpu_to_le_64(rte_mbuf_data_dma_addr_default(mbuf)); + rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf)); rxd = &rxq->rx_ring[i]; rxd->read.hdr_addr = 0; rxd->read.pkt_addr = dma_addr; @@ -4072,21 +4295,24 @@ ixgbe_dev_mq_rx_configure(struct rte_eth_dev *dev) break; } } else { - /* - * SRIOV active scheme - * Support RSS together with VMDq & SRIOV + /* SRIOV active scheme + * Support RSS together with SRIOV. */ switch (dev->data->dev_conf.rxmode.mq_mode) { case ETH_MQ_RX_RSS: case ETH_MQ_RX_VMDQ_RSS: ixgbe_config_vf_rss(dev); break; - - /* FIXME if support DCB/RSS together with VMDq & SRIOV */ case ETH_MQ_RX_VMDQ_DCB: + case ETH_MQ_RX_DCB: + /* In SRIOV, the configuration is the same as VMDq case */ + ixgbe_vmdq_dcb_configure(dev); + break; + /* DCB/RSS together with SRIOV is not supported */ case ETH_MQ_RX_VMDQ_DCB_RSS: + case ETH_MQ_RX_DCB_RSS: PMD_INIT_LOG(ERR, - "Could not support DCB with VMDq & SRIOV"); + "Could not support DCB/RSS with VMDq & SRIOV"); return -1; default: ixgbe_config_vf_default(dev); @@ -4344,6 +4570,10 @@ ixgbe_set_rx_function(struct rte_eth_dev *dev) struct ixgbe_rx_queue *rxq = dev->data->rx_queues[i]; rxq->rx_using_sse = rx_using_sse; +#ifdef RTE_LIBRTE_SECURITY + rxq->using_ipsec = !!(dev->data->dev_conf.rxmode.offloads & + DEV_RX_OFFLOAD_SECURITY); +#endif } } @@ -4366,6 +4596,7 @@ ixgbe_set_rsc(struct rte_eth_dev *dev) bool rsc_capable = false; uint16_t i; uint32_t rdrxctl; + uint32_t rfctl; /* Sanity check */ dev->dev_ops->dev_infos_get(dev, &dev_info); @@ -4393,22 +4624,18 @@ ixgbe_set_rsc(struct rte_eth_dev *dev) } /* RFCTL configuration */ - if (rsc_capable) { - uint32_t rfctl = IXGBE_READ_REG(hw, IXGBE_RFCTL); - - if (rx_conf->enable_lro) - /* - * Since NFS packets coalescing is not supported - clear - * RFCTL.NFSW_DIS and RFCTL.NFSR_DIS when RSC is - * enabled. - */ - rfctl &= ~(IXGBE_RFCTL_RSC_DIS | IXGBE_RFCTL_NFSW_DIS | - IXGBE_RFCTL_NFSR_DIS); - else - rfctl |= IXGBE_RFCTL_RSC_DIS; - - IXGBE_WRITE_REG(hw, IXGBE_RFCTL, rfctl); - } + rfctl = IXGBE_READ_REG(hw, IXGBE_RFCTL); + if ((rsc_capable) && (rx_conf->enable_lro)) + /* + * Since NFS packets coalescing is not supported - clear + * RFCTL.NFSW_DIS and RFCTL.NFSR_DIS when RSC is + * enabled. + */ + rfctl &= ~(IXGBE_RFCTL_RSC_DIS | IXGBE_RFCTL_NFSW_DIS | + IXGBE_RFCTL_NFSR_DIS); + else + rfctl |= IXGBE_RFCTL_RSC_DIS; + IXGBE_WRITE_REG(hw, IXGBE_RFCTL, rfctl); /* If LRO hasn't been requested - we are done here. */ if (!rx_conf->enable_lro) @@ -4834,6 +5061,21 @@ ixgbe_dev_rxtx_start(struct rte_eth_dev *dev) dev->data->dev_conf.lpbk_mode == IXGBE_LPBK_82599_TX_RX) ixgbe_setup_loopback_link_82599(hw); +#ifdef RTE_LIBRTE_SECURITY + if ((dev->data->dev_conf.rxmode.offloads & + DEV_RX_OFFLOAD_SECURITY) || + (dev->data->dev_conf.txmode.offloads & + DEV_TX_OFFLOAD_SECURITY)) { + ret = ixgbe_crypto_enable_ipsec(dev); + if (ret != 0) { + PMD_DRV_LOG(ERR, + "ixgbe_crypto_enable_ipsec fails with %d.", + ret); + return ret; + } + } +#endif + return 0; } diff --git a/dpdk/drivers/net/ixgbe/ixgbe_rxtx.h b/dpdk/drivers/net/ixgbe/ixgbe_rxtx.h index 2608b364..cc7c8288 100644 --- a/dpdk/drivers/net/ixgbe/ixgbe_rxtx.h +++ b/dpdk/drivers/net/ixgbe/ixgbe_rxtx.h @@ -67,7 +67,7 @@ #define RTE_IXGBE_MAX_RX_BURST RTE_IXGBE_RXQ_REARM_THRESH #endif -#define RX_RING_SZ ((IXGBE_MAX_RING_DESC + RTE_IXGBE_DESCS_PER_LOOP - 1) * \ +#define RX_RING_SZ ((IXGBE_MAX_RING_DESC + RTE_PMD_IXGBE_RX_MAX_BURST) * \ sizeof(union ixgbe_adv_rx_desc)) #ifdef RTE_PMD_PACKET_PREFETCH @@ -80,11 +80,17 @@ #define RTE_IXGBE_WAIT_100_US 100 #define RTE_IXGBE_VMTXSW_REGISTER_COUNT 2 +#define IXGBE_TX_MAX_SEG 40 + #define IXGBE_PACKET_TYPE_MASK_82599 0X7F #define IXGBE_PACKET_TYPE_MASK_X550 0X10FF #define IXGBE_PACKET_TYPE_MASK_TUNNEL 0XFF #define IXGBE_PACKET_TYPE_TUNNEL_BIT 0X1000 +#define IXGBE_PACKET_TYPE_MAX 0X80 +#define IXGBE_PACKET_TYPE_TN_MAX 0X100 +#define IXGBE_PACKET_TYPE_SHIFT 0X04 + /** * Structure associated with each descriptor of the RX ring of a RX queue. */ @@ -132,8 +138,12 @@ struct ixgbe_rx_queue { uint16_t rx_nb_avail; /**< nr of staged pkts ready to ret to app */ uint16_t rx_next_avail; /**< idx of next staged pkt to ret to app */ uint16_t rx_free_trigger; /**< triggers rx buffer allocation */ - uint16_t rx_using_sse; + uint8_t rx_using_sse; /**< indicates that vector RX is in use */ +#ifdef RTE_LIBRTE_SECURITY + uint8_t using_ipsec; + /**< indicates that IPsec RX feature is in use */ +#endif #ifdef RTE_IXGBE_INC_VECTOR uint16_t rxrearm_nb; /**< number of remaining to be re-armed */ uint16_t rxrearm_start; /**< the idx we start the re-arming from */ @@ -142,7 +152,7 @@ struct ixgbe_rx_queue { uint16_t queue_id; /**< RX queue index. */ uint16_t reg_idx; /**< RX queue register index. */ uint16_t pkt_type_mask; /**< Packet type mask for different NICs. */ - uint8_t port_id; /**< Device port identifier. */ + uint16_t port_id; /**< Device port identifier. */ uint8_t crc_len; /**< 0 if CRC stripped, 4 otherwise. */ uint8_t drop_en; /**< If not 0, set SRRCTL.Drop_En. */ uint8_t rx_deferred_start; /**< not in global dev start. */ @@ -177,6 +187,11 @@ union ixgbe_tx_offload { /* fields for TX offloading of tunnels */ uint64_t outer_l3_len:8; /**< Outer L3 (IP) Hdr Length. */ uint64_t outer_l2_len:8; /**< Outer L2 (MAC) Hdr Length. */ +#ifdef RTE_LIBRTE_SECURITY + /* inline ipsec related*/ + uint64_t sa_idx:8; /**< TX SA database entry index */ + uint64_t sec_pad_len:4; /**< padding length */ +#endif }; }; @@ -231,7 +246,7 @@ struct ixgbe_tx_queue { uint16_t tx_next_rs; /**< next desc to set RS bit */ uint16_t queue_id; /**< TX queue index. */ uint16_t reg_idx; /**< TX queue register index. */ - uint8_t port_id; /**< Device port identifier. */ + uint16_t port_id; /**< Device port identifier. */ uint8_t pthresh; /**< Prefetch threshold register. */ uint8_t hthresh; /**< Host threshold register. */ uint8_t wthresh; /**< Write-back threshold reg. */ @@ -241,6 +256,10 @@ struct ixgbe_tx_queue { struct ixgbe_advctx_info ctx_cache[IXGBE_CTX_NUM]; const struct ixgbe_txq_ops *ops; /**< txq ops */ uint8_t tx_deferred_start; /**< not in global dev start. */ +#ifdef RTE_LIBRTE_SECURITY + uint8_t using_ipsec; + /**< indicates that IPsec TX feature is in use */ +#endif }; struct ixgbe_txq_ops { @@ -307,10 +326,13 @@ int ixgbe_rx_vec_dev_conf_condition_check(struct rte_eth_dev *dev); int ixgbe_rxq_vec_setup(struct ixgbe_rx_queue *rxq); void ixgbe_rx_queue_release_mbufs_vec(struct ixgbe_rx_queue *rxq); +extern const uint32_t ptype_table[IXGBE_PACKET_TYPE_MAX]; +extern const uint32_t ptype_table_tn[IXGBE_PACKET_TYPE_TN_MAX]; + #ifdef RTE_IXGBE_INC_VECTOR -uint16_t ixgbe_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts, - uint16_t nb_pkts); +uint16_t ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts); int ixgbe_txq_vec_setup(struct ixgbe_tx_queue *txq); #endif /* RTE_IXGBE_INC_VECTOR */ diff --git a/dpdk/drivers/net/ixgbe/ixgbe_rxtx_vec_common.h b/dpdk/drivers/net/ixgbe/ixgbe_rxtx_vec_common.h index 3c3c0095..9fc112b1 100644 --- a/dpdk/drivers/net/ixgbe/ixgbe_rxtx_vec_common.h +++ b/dpdk/drivers/net/ixgbe/ixgbe_rxtx_vec_common.h @@ -101,7 +101,7 @@ reassemble_packets(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_bufs, return pkt_idx; } -static inline int __attribute__((always_inline)) +static __rte_always_inline int ixgbe_tx_free_bufs(struct ixgbe_tx_queue *txq) { struct ixgbe_tx_entry_v *txep; @@ -123,12 +123,12 @@ ixgbe_tx_free_bufs(struct ixgbe_tx_queue *txq) * tx_next_dd - (tx_rs_thresh-1) */ txep = &txq->sw_ring_v[txq->tx_next_dd - (n - 1)]; - m = __rte_pktmbuf_prefree_seg(txep[0].mbuf); + m = rte_pktmbuf_prefree_seg(txep[0].mbuf); if (likely(m != NULL)) { free[0] = m; nb_free = 1; for (i = 1; i < n; i++) { - m = __rte_pktmbuf_prefree_seg(txep[i].mbuf); + m = rte_pktmbuf_prefree_seg(txep[i].mbuf); if (likely(m != NULL)) { if (likely(m->pool == free[0]->pool)) free[nb_free++] = m; @@ -143,7 +143,7 @@ ixgbe_tx_free_bufs(struct ixgbe_tx_queue *txq) rte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free); } else { for (i = 1; i < n; i++) { - m = __rte_pktmbuf_prefree_seg(txep[i].mbuf); + m = rte_pktmbuf_prefree_seg(txep[i].mbuf); if (m != NULL) rte_mempool_put(m->pool, m); } @@ -158,7 +158,7 @@ ixgbe_tx_free_bufs(struct ixgbe_tx_queue *txq) return txq->tx_rs_thresh; } -static inline void __attribute__((always_inline)) +static __rte_always_inline void tx_backlog_entry(struct ixgbe_tx_entry_v *txep, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { @@ -310,23 +310,12 @@ ixgbe_rx_vec_dev_conf_condition_check_default(struct rte_eth_dev *dev) struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode; struct rte_fdir_conf *fconf = &dev->data->dev_conf.fdir_conf; -#ifndef RTE_IXGBE_RX_OLFLAGS_ENABLE - /* whithout rx ol_flags, no VP flag report */ - if (rxmode->hw_vlan_strip != 0 || - rxmode->hw_vlan_extend != 0) - return -1; -#endif - /* no fdir support */ if (fconf->mode != RTE_FDIR_MODE_NONE) return -1; - /* - * - no csum error report support - * - no header split support - */ - if (rxmode->hw_ip_checksum == 1 || - rxmode->header_split == 1) + /* no header split support */ + if (rxmode->header_split == 1) return -1; return 0; diff --git a/dpdk/drivers/net/ixgbe/ixgbe_rxtx_vec_neon.c b/dpdk/drivers/net/ixgbe/ixgbe_rxtx_vec_neon.c index 64a329ea..2e87ffa0 100644 --- a/dpdk/drivers/net/ixgbe/ixgbe_rxtx_vec_neon.c +++ b/dpdk/drivers/net/ixgbe/ixgbe_rxtx_vec_neon.c @@ -85,18 +85,15 @@ ixgbe_rxq_rearm(struct ixgbe_rx_queue *rxq) /* * Flush mbuf with pkt template. * Data to be rearmed is 6 bytes long. - * Though, RX will overwrite ol_flags that are coming next - * anyway. So overwrite whole 8 bytes with one load: - * 6 bytes of rearm_data plus first 2 bytes of ol_flags. */ vst1_u8((uint8_t *)&mb0->rearm_data, p); - paddr = mb0->buf_physaddr + RTE_PKTMBUF_HEADROOM; + paddr = mb0->buf_iova + RTE_PKTMBUF_HEADROOM; dma_addr0 = vsetq_lane_u64(paddr, zero, 0); /* flush desc with pa dma_addr */ vst1q_u64((uint64_t *)&rxdp++->read, dma_addr0); vst1_u8((uint8_t *)&mb1->rearm_data, p); - paddr = mb1->buf_physaddr + RTE_PKTMBUF_HEADROOM; + paddr = mb1->buf_iova + RTE_PKTMBUF_HEADROOM; dma_addr1 = vsetq_lane_u64(paddr, zero, 0); vst1q_u64((uint64_t *)&rxdp++->read, dma_addr1); } @@ -114,14 +111,6 @@ ixgbe_rxq_rearm(struct ixgbe_rx_queue *rxq) IXGBE_PCI_REG_WRITE(rxq->rdt_reg_addr, rx_id); } -/* Handling the offload flags (olflags) field takes computation - * time when receiving packets. Therefore we provide a flag to disable - * the processing of the olflags field when they are not needed. This - * gives improved performance, at the cost of losing the offload info - * in the received packet - */ -#ifdef RTE_IXGBE_RX_OLFLAGS_ENABLE - #define VTAG_SHIFT (3) static inline void @@ -137,8 +126,8 @@ desc_to_olflags_v(uint8x16x2_t sterr_tmp1, uint8x16x2_t sterr_tmp2, } vol; const uint8x16_t pkttype_msk = { - PKT_RX_VLAN_PKT, PKT_RX_VLAN_PKT, - PKT_RX_VLAN_PKT, PKT_RX_VLAN_PKT, + PKT_RX_VLAN, PKT_RX_VLAN, + PKT_RX_VLAN, PKT_RX_VLAN, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; @@ -170,9 +159,6 @@ desc_to_olflags_v(uint8x16x2_t sterr_tmp1, uint8x16x2_t sterr_tmp2, rx_pkts[2]->ol_flags = vol.e[2]; rx_pkts[3]->ol_flags = vol.e[3]; } -#else -#define desc_to_olflags_v(sterr_tmp1, sterr_tmp2, staterr, rx_pkts) -#endif /* * vPMD raw receive routine, only accept(nb_pkts >= RTE_IXGBE_DESCS_PER_LOOP) @@ -196,7 +182,6 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, struct ixgbe_rx_entry *sw_ring; uint16_t nb_pkts_recd; int pos; - uint64_t var; uint8x16_t shuf_msk = { 0xFF, 0xFF, 0xFF, 0xFF, /* skip 32 bits pkt_type */ @@ -255,26 +240,24 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, uint64x2_t mbp1, mbp2; uint8x16_t staterr; uint16x8_t tmp; + uint32_t var = 0; uint32_t stat; /* B.1 load 1 mbuf point */ mbp1 = vld1q_u64((uint64_t *)&sw_ring[pos]); - /* Read desc statuses backwards to avoid race condition */ - /* A.1 load 4 pkts desc */ - descs[3] = vld1q_u64((uint64_t *)(rxdp + 3)); - rte_rmb(); - /* B.2 copy 2 mbuf point into rx_pkts */ vst1q_u64((uint64_t *)&rx_pkts[pos], mbp1); /* B.1 load 1 mbuf point */ mbp2 = vld1q_u64((uint64_t *)&sw_ring[pos + 2]); - descs[2] = vld1q_u64((uint64_t *)(rxdp + 2)); - /* B.1 load 2 mbuf point */ - descs[1] = vld1q_u64((uint64_t *)(rxdp + 1)); + /* A. load 4 pkts descs */ descs[0] = vld1q_u64((uint64_t *)(rxdp)); + descs[1] = vld1q_u64((uint64_t *)(rxdp + 1)); + descs[2] = vld1q_u64((uint64_t *)(rxdp + 2)); + descs[3] = vld1q_u64((uint64_t *)(rxdp + 3)); + rte_smp_rmb(); /* B.2 copy 2 mbuf point into rx_pkts */ vst1q_u64((uint64_t *)&rx_pkts[pos + 2], mbp2); @@ -333,12 +316,6 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, *(int *)split_packet = ~stat & IXGBE_VPMD_DESC_EOP_MASK; split_packet += RTE_IXGBE_DESCS_PER_LOOP; - - /* zero-out next pointers */ - rx_pkts[pos]->next = NULL; - rx_pkts[pos + 1]->next = NULL; - rx_pkts[pos + 2]->next = NULL; - rx_pkts[pos + 3]->next = NULL; } rte_prefetch_non_temporal(rxdp + RTE_IXGBE_DESCS_PER_LOOP); @@ -349,11 +326,19 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, vst1q_u8((uint8_t *)&rx_pkts[pos]->rx_descriptor_fields1, pkt_mb1); + stat &= IXGBE_VPMD_DESC_DD_MASK; + /* C.4 calc avaialbe number of desc */ - var = __builtin_popcount(stat & IXGBE_VPMD_DESC_DD_MASK); - nb_pkts_recd += var; - if (likely(var != RTE_IXGBE_DESCS_PER_LOOP)) + if (likely(stat != IXGBE_VPMD_DESC_DD_MASK)) { + while (stat & 0x01) { + ++var; + stat = stat >> 8; + } + nb_pkts_recd += var; break; + } else { + nb_pkts_recd += RTE_IXGBE_DESCS_PER_LOOP; + } } /* Update our internal tail pointer */ @@ -429,7 +414,7 @@ vtx1(volatile union ixgbe_adv_tx_desc *txdp, struct rte_mbuf *pkt, uint64_t flags) { uint64x2_t descriptor = { - pkt->buf_physaddr + pkt->data_off, + pkt->buf_iova + pkt->data_off, (uint64_t)pkt->pkt_len << 46 | flags | pkt->data_len}; vst1q_u64((uint64_t *)&txdp->read, descriptor); @@ -446,8 +431,8 @@ vtx(volatile union ixgbe_adv_tx_desc *txdp, } uint16_t -ixgbe_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts, - uint16_t nb_pkts) +ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts) { struct ixgbe_tx_queue *txq = (struct ixgbe_tx_queue *)tx_queue; volatile union ixgbe_adv_tx_desc *txdp; @@ -556,5 +541,11 @@ ixgbe_txq_vec_setup(struct ixgbe_tx_queue *txq) int __attribute__((cold)) ixgbe_rx_vec_dev_conf_condition_check(struct rte_eth_dev *dev) { + struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode; + + /* no csum error report support */ + if (rxmode->hw_ip_checksum == 1) + return -1; + return ixgbe_rx_vec_dev_conf_condition_check_default(dev); } diff --git a/dpdk/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c b/dpdk/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c index 7fb155a4..486239ba 100644 --- a/dpdk/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c +++ b/dpdk/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c @@ -82,24 +82,13 @@ ixgbe_rxq_rearm(struct ixgbe_rx_queue *rxq) /* Initialize the mbufs in vector, process 2 mbufs in one loop */ for (i = 0; i < RTE_IXGBE_RXQ_REARM_THRESH; i += 2, rxep += 2) { __m128i vaddr0, vaddr1; - uintptr_t p0, p1; mb0 = rxep[0].mbuf; mb1 = rxep[1].mbuf; - /* - * Flush mbuf with pkt template. - * Data to be rearmed is 6 bytes long. - * Though, RX will overwrite ol_flags that are coming next - * anyway. So overwrite whole 8 bytes with one load: - * 6 bytes of rearm_data plus first 2 bytes of ol_flags. - */ - p0 = (uintptr_t)&mb0->rearm_data; - *(uint64_t *)p0 = rxq->mbuf_initializer; - p1 = (uintptr_t)&mb1->rearm_data; - *(uint64_t *)p1 = rxq->mbuf_initializer; - - /* load buf_addr(lo 64bit) and buf_physaddr(hi 64bit) */ + /* load buf_addr(lo 64bit) and buf_iova(hi 64bit) */ + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, buf_iova) != + offsetof(struct rte_mbuf, buf_addr) + 8); vaddr0 = _mm_loadu_si128((__m128i *)&(mb0->buf_addr)); vaddr1 = _mm_loadu_si128((__m128i *)&(mb1->buf_addr)); @@ -133,47 +122,94 @@ ixgbe_rxq_rearm(struct ixgbe_rx_queue *rxq) IXGBE_PCI_REG_WRITE(rxq->rdt_reg_addr, rx_id); } -/* Handling the offload flags (olflags) field takes computation - * time when receiving packets. Therefore we provide a flag to disable - * the processing of the olflags field when they are not needed. This - * gives improved performance, at the cost of losing the offload info - * in the received packet - */ -#ifdef RTE_IXGBE_RX_OLFLAGS_ENABLE +#ifdef RTE_LIBRTE_SECURITY +static inline void +desc_to_olflags_v_ipsec(__m128i descs[4], struct rte_mbuf **rx_pkts) +{ + __m128i sterr, rearm, tmp_e, tmp_p; + uint32_t *rearm0 = (uint32_t *)rx_pkts[0]->rearm_data + 2; + uint32_t *rearm1 = (uint32_t *)rx_pkts[1]->rearm_data + 2; + uint32_t *rearm2 = (uint32_t *)rx_pkts[2]->rearm_data + 2; + uint32_t *rearm3 = (uint32_t *)rx_pkts[3]->rearm_data + 2; + const __m128i ipsec_sterr_msk = + _mm_set1_epi32(IXGBE_RXDADV_IPSEC_STATUS_SECP | + IXGBE_RXDADV_IPSEC_ERROR_AUTH_FAILED); + const __m128i ipsec_proc_msk = + _mm_set1_epi32(IXGBE_RXDADV_IPSEC_STATUS_SECP); + const __m128i ipsec_err_flag = + _mm_set1_epi32(PKT_RX_SEC_OFFLOAD_FAILED | + PKT_RX_SEC_OFFLOAD); + const __m128i ipsec_proc_flag = _mm_set1_epi32(PKT_RX_SEC_OFFLOAD); + + rearm = _mm_set_epi32(*rearm3, *rearm2, *rearm1, *rearm0); + sterr = _mm_set_epi32(_mm_extract_epi32(descs[3], 2), + _mm_extract_epi32(descs[2], 2), + _mm_extract_epi32(descs[1], 2), + _mm_extract_epi32(descs[0], 2)); + sterr = _mm_and_si128(sterr, ipsec_sterr_msk); + tmp_e = _mm_cmpeq_epi32(sterr, ipsec_sterr_msk); + tmp_p = _mm_cmpeq_epi32(sterr, ipsec_proc_msk); + sterr = _mm_or_si128(_mm_and_si128(tmp_e, ipsec_err_flag), + _mm_and_si128(tmp_p, ipsec_proc_flag)); + rearm = _mm_or_si128(rearm, sterr); + *rearm0 = _mm_extract_epi32(rearm, 0); + *rearm1 = _mm_extract_epi32(rearm, 1); + *rearm2 = _mm_extract_epi32(rearm, 2); + *rearm3 = _mm_extract_epi32(rearm, 3); +} +#endif static inline void -desc_to_olflags_v(__m128i descs[4], uint8_t vlan_flags, +desc_to_olflags_v(__m128i descs[4], __m128i mbuf_init, uint8_t vlan_flags, struct rte_mbuf **rx_pkts) { - __m128i ptype0, ptype1, vtag0, vtag1; - union { - uint16_t e[4]; - uint64_t dword; - } vol; + __m128i ptype0, ptype1, vtag0, vtag1, csum; + __m128i rearm0, rearm1, rearm2, rearm3; /* mask everything except rss type */ const __m128i rsstype_msk = _mm_set_epi16( 0x0000, 0x0000, 0x0000, 0x0000, 0x000F, 0x000F, 0x000F, 0x000F); + /* mask the lower byte of ol_flags */ + const __m128i ol_flags_msk = _mm_set_epi16( + 0x0000, 0x0000, 0x0000, 0x0000, + 0x00FF, 0x00FF, 0x00FF, 0x00FF); + /* map rss type to rss hash flag */ const __m128i rss_flags = _mm_set_epi8(PKT_RX_FDIR, 0, 0, 0, 0, 0, 0, PKT_RX_RSS_HASH, PKT_RX_RSS_HASH, 0, PKT_RX_RSS_HASH, 0, PKT_RX_RSS_HASH, PKT_RX_RSS_HASH, PKT_RX_RSS_HASH, 0); - /* mask everything except vlan present bit */ - const __m128i vlan_msk = _mm_set_epi16( - 0x0000, 0x0000, - 0x0000, 0x0000, - IXGBE_RXD_STAT_VP, IXGBE_RXD_STAT_VP, - IXGBE_RXD_STAT_VP, IXGBE_RXD_STAT_VP); - /* map vlan present (0x8) to ol_flags */ - const __m128i vlan_map = _mm_set_epi8( + /* mask everything except vlan present and l4/ip csum error */ + const __m128i vlan_csum_msk = _mm_set_epi16( + (IXGBE_RXDADV_ERR_TCPE | IXGBE_RXDADV_ERR_IPE) >> 16, + (IXGBE_RXDADV_ERR_TCPE | IXGBE_RXDADV_ERR_IPE) >> 16, + (IXGBE_RXDADV_ERR_TCPE | IXGBE_RXDADV_ERR_IPE) >> 16, + (IXGBE_RXDADV_ERR_TCPE | IXGBE_RXDADV_ERR_IPE) >> 16, + IXGBE_RXD_STAT_VP, IXGBE_RXD_STAT_VP, + IXGBE_RXD_STAT_VP, IXGBE_RXD_STAT_VP); + /* map vlan present (0x8), IPE (0x2), L4E (0x1) to ol_flags */ + const __m128i vlan_csum_map_lo = _mm_set_epi8( 0, 0, 0, 0, - 0, 0, 0, vlan_flags, + vlan_flags | PKT_RX_IP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD, + vlan_flags | PKT_RX_IP_CKSUM_BAD, + vlan_flags | PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_BAD, + vlan_flags | PKT_RX_IP_CKSUM_GOOD, 0, 0, 0, 0, - 0, 0, 0, 0); + PKT_RX_IP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD, + PKT_RX_IP_CKSUM_BAD, + PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_BAD, + PKT_RX_IP_CKSUM_GOOD); + + const __m128i vlan_csum_map_hi = _mm_set_epi8( + 0, 0, 0, 0, + 0, PKT_RX_L4_CKSUM_GOOD >> sizeof(uint8_t), 0, + PKT_RX_L4_CKSUM_GOOD >> sizeof(uint8_t), + 0, 0, 0, 0, + 0, PKT_RX_L4_CKSUM_GOOD >> sizeof(uint8_t), 0, + PKT_RX_L4_CKSUM_GOOD >> sizeof(uint8_t)); ptype0 = _mm_unpacklo_epi16(descs[0], descs[1]); ptype1 = _mm_unpacklo_epi16(descs[2], descs[3]); @@ -185,22 +221,115 @@ desc_to_olflags_v(__m128i descs[4], uint8_t vlan_flags, ptype0 = _mm_shuffle_epi8(rss_flags, ptype0); vtag1 = _mm_unpacklo_epi32(vtag0, vtag1); - vtag1 = _mm_and_si128(vtag1, vlan_msk); - vtag1 = _mm_shuffle_epi8(vlan_map, vtag1); + vtag1 = _mm_and_si128(vtag1, vlan_csum_msk); + + /* csum bits are in the most significant, to use shuffle we need to + * shift them. Change mask to 0xc000 to 0x0003. + */ + csum = _mm_srli_epi16(vtag1, 14); + + /* now or the most significant 64 bits containing the checksum + * flags with the vlan present flags. + */ + csum = _mm_srli_si128(csum, 8); + vtag1 = _mm_or_si128(csum, vtag1); + + /* convert VP, IPE, L4E to ol_flags */ + vtag0 = _mm_shuffle_epi8(vlan_csum_map_hi, vtag1); + vtag0 = _mm_slli_epi16(vtag0, sizeof(uint8_t)); + + vtag1 = _mm_shuffle_epi8(vlan_csum_map_lo, vtag1); + vtag1 = _mm_and_si128(vtag1, ol_flags_msk); + vtag1 = _mm_or_si128(vtag0, vtag1); vtag1 = _mm_or_si128(ptype0, vtag1); - vol.dword = _mm_cvtsi128_si64(vtag1); - rx_pkts[0]->ol_flags = vol.e[0]; - rx_pkts[1]->ol_flags = vol.e[1]; - rx_pkts[2]->ol_flags = vol.e[2]; - rx_pkts[3]->ol_flags = vol.e[3]; + /* + * At this point, we have the 4 sets of flags in the low 64-bits + * of vtag1 (4x16). + * We want to extract these, and merge them with the mbuf init data + * so we can do a single 16-byte write to the mbuf to set the flags + * and all the other initialization fields. Extracting the + * appropriate flags means that we have to do a shift and blend for + * each mbuf before we do the write. + */ + rearm0 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(vtag1, 8), 0x10); + rearm1 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(vtag1, 6), 0x10); + rearm2 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(vtag1, 4), 0x10); + rearm3 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(vtag1, 2), 0x10); + + /* write the rearm data and the olflags in one write */ + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, ol_flags) != + offsetof(struct rte_mbuf, rearm_data) + 8); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, rearm_data) != + RTE_ALIGN(offsetof(struct rte_mbuf, rearm_data), 16)); + _mm_store_si128((__m128i *)&rx_pkts[0]->rearm_data, rearm0); + _mm_store_si128((__m128i *)&rx_pkts[1]->rearm_data, rearm1); + _mm_store_si128((__m128i *)&rx_pkts[2]->rearm_data, rearm2); + _mm_store_si128((__m128i *)&rx_pkts[3]->rearm_data, rearm3); +} + +static inline uint32_t get_packet_type(int index, + uint32_t pkt_info, + uint32_t etqf_check, + uint32_t tunnel_check) +{ + if (etqf_check & (0x02 << (index * RTE_IXGBE_DESCS_PER_LOOP))) + return RTE_PTYPE_UNKNOWN; + + if (tunnel_check & (0x02 << (index * RTE_IXGBE_DESCS_PER_LOOP))) { + pkt_info &= IXGBE_PACKET_TYPE_MASK_TUNNEL; + return ptype_table_tn[pkt_info]; + } + + pkt_info &= IXGBE_PACKET_TYPE_MASK_82599; + return ptype_table[pkt_info]; +} + +static inline void +desc_to_ptype_v(__m128i descs[4], uint16_t pkt_type_mask, + struct rte_mbuf **rx_pkts) +{ + __m128i etqf_mask = _mm_set_epi64x(0x800000008000LL, 0x800000008000LL); + __m128i ptype_mask = _mm_set_epi32( + pkt_type_mask, pkt_type_mask, pkt_type_mask, pkt_type_mask); + __m128i tunnel_mask = + _mm_set_epi64x(0x100000001000LL, 0x100000001000LL); + + uint32_t etqf_check, tunnel_check, pkt_info; + + __m128i ptype0 = _mm_unpacklo_epi32(descs[0], descs[2]); + __m128i ptype1 = _mm_unpacklo_epi32(descs[1], descs[3]); + + /* interleave low 32 bits, + * now we have 4 ptypes in a XMM register + */ + ptype0 = _mm_unpacklo_epi32(ptype0, ptype1); + + /* create a etqf bitmask based on the etqf bit. */ + etqf_check = _mm_movemask_epi8(_mm_and_si128(ptype0, etqf_mask)); + + /* shift left by IXGBE_PACKET_TYPE_SHIFT, and apply ptype mask */ + ptype0 = _mm_and_si128(_mm_srli_epi32(ptype0, IXGBE_PACKET_TYPE_SHIFT), + ptype_mask); + + /* create a tunnel bitmask based on the tunnel bit */ + tunnel_check = _mm_movemask_epi8( + _mm_slli_epi32(_mm_and_si128(ptype0, tunnel_mask), 0x3)); + + pkt_info = _mm_extract_epi32(ptype0, 0); + rx_pkts[0]->packet_type = + get_packet_type(0, pkt_info, etqf_check, tunnel_check); + pkt_info = _mm_extract_epi32(ptype0, 1); + rx_pkts[1]->packet_type = + get_packet_type(1, pkt_info, etqf_check, tunnel_check); + pkt_info = _mm_extract_epi32(ptype0, 2); + rx_pkts[2]->packet_type = + get_packet_type(2, pkt_info, etqf_check, tunnel_check); + pkt_info = _mm_extract_epi32(ptype0, 3); + rx_pkts[3]->packet_type = + get_packet_type(3, pkt_info, etqf_check, tunnel_check); } -#else -#define desc_to_olflags_v(desc, vlan_flags, rx_pkts) do { \ - RTE_SET_USED(vlan_flags); \ - } while (0) -#endif /* * vPMD raw receive routine, only accept(nb_pkts >= RTE_IXGBE_DESCS_PER_LOOP) @@ -210,7 +339,6 @@ desc_to_olflags_v(__m128i descs[4], uint8_t vlan_flags, * - nb_pkts > RTE_IXGBE_MAX_RX_BURST, only scan RTE_IXGBE_MAX_RX_BURST * numbers of DD bit * - floor align nb_pkts to a RTE_IXGBE_DESC_PER_LOOP power-of-two - * - don't support ol_flags for rss and csum err */ static inline uint16_t _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, @@ -219,6 +347,9 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, volatile union ixgbe_adv_rx_desc *rxdp; struct ixgbe_rx_entry *sw_ring; uint16_t nb_pkts_recd; +#ifdef RTE_LIBRTE_SECURITY + uint8_t use_ipsec = rxq->using_ipsec; +#endif int pos; uint64_t var; __m128i shuf_msk; @@ -229,7 +360,17 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, -rxq->crc_len, /* sub crc on pkt_len */ 0, 0 /* ignore pkt_type field */ ); + /* + * compile-time check the above crc_adjust layout is correct. + * NOTE: the first field (lowest address) is given last in set_epi16 + * call above. + */ + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, pkt_len) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 4); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_len) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 8); __m128i dd_check, eop_check; + __m128i mbuf_init; uint8_t vlan_flags; /* nb_pkts shall be less equal than RTE_IXGBE_MAX_RX_BURST */ @@ -243,7 +384,7 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, */ rxdp = rxq->rx_ring + rxq->rx_tail; - _mm_prefetch((const void *)rxdp, _MM_HINT_T0); + rte_prefetch0(rxdp); /* See if we need to rearm the RX queue - gives the prefetch a bit * of time to act @@ -274,6 +415,21 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, 0xFF, 0xFF, /* skip 32 bit pkt_type */ 0xFF, 0xFF ); + /* + * Compile-time verify the shuffle mask + * NOTE: some field positions already verified above, but duplicated + * here for completeness in case of future modifications. + */ + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, pkt_len) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 4); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_len) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 8); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, vlan_tci) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 10); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, hash) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 12); + + mbuf_init = _mm_set_epi64x(0, rxq->mbuf_initializer); /* Cache is empty -> need to scan the buffer rings, but first move * the next 'n' mbufs into the cache @@ -281,7 +437,7 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, sw_ring = &rxq->sw_ring[rxq->rx_tail]; /* ensure these 2 flags are in the lower 8 bits */ - RTE_BUILD_BUG_ON((PKT_RX_VLAN_PKT | PKT_RX_VLAN_STRIPPED) > UINT8_MAX); + RTE_BUILD_BUG_ON((PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED) > UINT8_MAX); vlan_flags = rxq->vlan_flags & UINT8_MAX; /* A. load 4 packet in one loop @@ -297,9 +453,13 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, __m128i descs[RTE_IXGBE_DESCS_PER_LOOP]; __m128i pkt_mb1, pkt_mb2, pkt_mb3, pkt_mb4; __m128i zero, staterr, sterr_tmp1, sterr_tmp2; - __m128i mbp1, mbp2; /* two mbuf pointer in one XMM reg. */ + /* 2 64 bit or 4 32 bit mbuf pointers in one XMM reg. */ + __m128i mbp1; +#if defined(RTE_ARCH_X86_64) + __m128i mbp2; +#endif - /* B.1 load 1 mbuf point */ + /* B.1 load 2 (64 bit) or 4 (32 bit) mbuf points */ mbp1 = _mm_loadu_si128((__m128i *)&sw_ring[pos]); /* Read desc statuses backwards to avoid race condition */ @@ -307,11 +467,13 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, descs[3] = _mm_loadu_si128((__m128i *)(rxdp + 3)); rte_compiler_barrier(); - /* B.2 copy 2 mbuf point into rx_pkts */ + /* B.2 copy 2 64 bit or 4 32 bit mbuf point into rx_pkts */ _mm_storeu_si128((__m128i *)&rx_pkts[pos], mbp1); - /* B.1 load 1 mbuf point */ +#if defined(RTE_ARCH_X86_64) + /* B.1 load 2 64 bit mbuf points */ mbp2 = _mm_loadu_si128((__m128i *)&sw_ring[pos+2]); +#endif descs[2] = _mm_loadu_si128((__m128i *)(rxdp + 2)); rte_compiler_barrier(); @@ -320,8 +482,10 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, rte_compiler_barrier(); descs[0] = _mm_loadu_si128((__m128i *)(rxdp)); +#if defined(RTE_ARCH_X86_64) /* B.2 copy 2 mbuf point into rx_pkts */ _mm_storeu_si128((__m128i *)&rx_pkts[pos+2], mbp2); +#endif if (split_packet) { rte_mbuf_prefetch_part2(rx_pkts[pos]); @@ -347,7 +511,12 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, sterr_tmp1 = _mm_unpackhi_epi32(descs[1], descs[0]); /* set ol_flags with vlan packet type */ - desc_to_olflags_v(descs, vlan_flags, &rx_pkts[pos]); + desc_to_olflags_v(descs, mbuf_init, vlan_flags, &rx_pkts[pos]); + +#ifdef RTE_LIBRTE_SECURITY + if (unlikely(use_ipsec)) + desc_to_olflags_v_ipsec(descs, &rx_pkts[pos]); +#endif /* D.2 pkt 3,4 set in_port/nb_seg and remove crc */ pkt_mb4 = _mm_add_epi16(pkt_mb4, crc_adjust); @@ -387,12 +556,6 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, /* store the resulting 32-bit value */ *(int *)split_packet = _mm_cvtsi128_si32(eop_bits); split_packet += RTE_IXGBE_DESCS_PER_LOOP; - - /* zero-out next pointers */ - rx_pkts[pos]->next = NULL; - rx_pkts[pos + 1]->next = NULL; - rx_pkts[pos + 2]->next = NULL; - rx_pkts[pos + 3]->next = NULL; } /* C.3 calc available number of desc */ @@ -405,6 +568,8 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, _mm_storeu_si128((void *)&rx_pkts[pos]->rx_descriptor_fields1, pkt_mb1); + desc_to_ptype_v(descs, rxq->pkt_type_mask, &rx_pkts[pos]); + /* C.4 calc avaialbe number of desc */ var = __builtin_popcountll(_mm_cvtsi128_si64(staterr)); nb_pkts_recd += var; @@ -428,7 +593,6 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, * - nb_pkts > RTE_IXGBE_MAX_RX_BURST, only scan RTE_IXGBE_MAX_RX_BURST * numbers of DD bit * - floor align nb_pkts to a RTE_IXGBE_DESC_PER_LOOP power-of-two - * - don't support ol_flags for rss and csum err */ uint16_t ixgbe_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, @@ -441,7 +605,6 @@ ixgbe_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, * vPMD receive routine that reassembles scattered packets * * Notice: - * - don't support ol_flags for rss and csum err * - nb_pkts < RTE_IXGBE_DESCS_PER_LOOP, just return no packet * - nb_pkts > RTE_IXGBE_MAX_RX_BURST, only scan RTE_IXGBE_MAX_RX_BURST * numbers of DD bit @@ -486,7 +649,7 @@ vtx1(volatile union ixgbe_adv_tx_desc *txdp, { __m128i descriptor = _mm_set_epi64x((uint64_t)pkt->pkt_len << 46 | flags | pkt->data_len, - pkt->buf_physaddr + pkt->data_off); + pkt->buf_iova + pkt->data_off); _mm_store_si128((__m128i *)&txdp->read, descriptor); } @@ -501,8 +664,8 @@ vtx(volatile union ixgbe_adv_tx_desc *txdp, } uint16_t -ixgbe_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts, - uint16_t nb_pkts) +ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts) { struct ixgbe_tx_queue *txq = (struct ixgbe_tx_queue *)tx_queue; volatile union ixgbe_adv_tx_desc *txdp; diff --git a/dpdk/drivers/net/ixgbe/ixgbe_tm.c b/dpdk/drivers/net/ixgbe/ixgbe_tm.c new file mode 100644 index 00000000..ca4182c9 --- /dev/null +++ b/dpdk/drivers/net/ixgbe/ixgbe_tm.c @@ -0,0 +1,1060 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +#include "ixgbe_ethdev.h" + +static int ixgbe_tm_capabilities_get(struct rte_eth_dev *dev, + struct rte_tm_capabilities *cap, + struct rte_tm_error *error); +static int ixgbe_shaper_profile_add(struct rte_eth_dev *dev, + uint32_t shaper_profile_id, + struct rte_tm_shaper_params *profile, + struct rte_tm_error *error); +static int ixgbe_shaper_profile_del(struct rte_eth_dev *dev, + uint32_t shaper_profile_id, + struct rte_tm_error *error); +static int ixgbe_node_add(struct rte_eth_dev *dev, uint32_t node_id, + uint32_t parent_node_id, uint32_t priority, + uint32_t weight, uint32_t level_id, + struct rte_tm_node_params *params, + struct rte_tm_error *error); +static int ixgbe_node_delete(struct rte_eth_dev *dev, uint32_t node_id, + struct rte_tm_error *error); +static int ixgbe_node_type_get(struct rte_eth_dev *dev, uint32_t node_id, + int *is_leaf, struct rte_tm_error *error); +static int ixgbe_level_capabilities_get(struct rte_eth_dev *dev, + uint32_t level_id, + struct rte_tm_level_capabilities *cap, + struct rte_tm_error *error); +static int ixgbe_node_capabilities_get(struct rte_eth_dev *dev, + uint32_t node_id, + struct rte_tm_node_capabilities *cap, + struct rte_tm_error *error); +static int ixgbe_hierarchy_commit(struct rte_eth_dev *dev, + int clear_on_fail, + struct rte_tm_error *error); + +const struct rte_tm_ops ixgbe_tm_ops = { + .capabilities_get = ixgbe_tm_capabilities_get, + .shaper_profile_add = ixgbe_shaper_profile_add, + .shaper_profile_delete = ixgbe_shaper_profile_del, + .node_add = ixgbe_node_add, + .node_delete = ixgbe_node_delete, + .node_type_get = ixgbe_node_type_get, + .level_capabilities_get = ixgbe_level_capabilities_get, + .node_capabilities_get = ixgbe_node_capabilities_get, + .hierarchy_commit = ixgbe_hierarchy_commit, +}; + +int +ixgbe_tm_ops_get(struct rte_eth_dev *dev __rte_unused, + void *arg) +{ + if (!arg) + return -EINVAL; + + *(const void **)arg = &ixgbe_tm_ops; + + return 0; +} + +void +ixgbe_tm_conf_init(struct rte_eth_dev *dev) +{ + struct ixgbe_tm_conf *tm_conf = + IXGBE_DEV_PRIVATE_TO_TM_CONF(dev->data->dev_private); + + /* initialize shaper profile list */ + TAILQ_INIT(&tm_conf->shaper_profile_list); + + /* initialize node configuration */ + tm_conf->root = NULL; + TAILQ_INIT(&tm_conf->queue_list); + TAILQ_INIT(&tm_conf->tc_list); + tm_conf->nb_tc_node = 0; + tm_conf->nb_queue_node = 0; + tm_conf->committed = false; +} + +void +ixgbe_tm_conf_uninit(struct rte_eth_dev *dev) +{ + struct ixgbe_tm_conf *tm_conf = + IXGBE_DEV_PRIVATE_TO_TM_CONF(dev->data->dev_private); + struct ixgbe_tm_shaper_profile *shaper_profile; + struct ixgbe_tm_node *tm_node; + + /* clear node configuration */ + while ((tm_node = TAILQ_FIRST(&tm_conf->queue_list))) { + TAILQ_REMOVE(&tm_conf->queue_list, tm_node, node); + rte_free(tm_node); + } + tm_conf->nb_queue_node = 0; + while ((tm_node = TAILQ_FIRST(&tm_conf->tc_list))) { + TAILQ_REMOVE(&tm_conf->tc_list, tm_node, node); + rte_free(tm_node); + } + tm_conf->nb_tc_node = 0; + if (tm_conf->root) { + rte_free(tm_conf->root); + tm_conf->root = NULL; + } + + /* Remove all shaper profiles */ + while ((shaper_profile = + TAILQ_FIRST(&tm_conf->shaper_profile_list))) { + TAILQ_REMOVE(&tm_conf->shaper_profile_list, + shaper_profile, node); + rte_free(shaper_profile); + } +} + +static inline uint8_t +ixgbe_tc_nb_get(struct rte_eth_dev *dev) +{ + struct rte_eth_conf *eth_conf; + uint8_t nb_tcs = 0; + + eth_conf = &dev->data->dev_conf; + if (eth_conf->txmode.mq_mode == ETH_MQ_TX_DCB) { + nb_tcs = eth_conf->tx_adv_conf.dcb_tx_conf.nb_tcs; + } else if (eth_conf->txmode.mq_mode == ETH_MQ_TX_VMDQ_DCB) { + if (eth_conf->tx_adv_conf.vmdq_dcb_tx_conf.nb_queue_pools == + ETH_32_POOLS) + nb_tcs = ETH_4_TCS; + else + nb_tcs = ETH_8_TCS; + } else { + nb_tcs = 1; + } + + return nb_tcs; +} + +static int +ixgbe_tm_capabilities_get(struct rte_eth_dev *dev, + struct rte_tm_capabilities *cap, + struct rte_tm_error *error) +{ + struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + uint8_t tc_nb = ixgbe_tc_nb_get(dev); + + if (!cap || !error) + return -EINVAL; + + if (tc_nb > hw->mac.max_tx_queues) + return -EINVAL; + + error->type = RTE_TM_ERROR_TYPE_NONE; + + /* set all the parameters to 0 first. */ + memset(cap, 0, sizeof(struct rte_tm_capabilities)); + + /** + * here is the max capability not the current configuration. + */ + /* port + TCs + queues */ + cap->n_nodes_max = 1 + IXGBE_DCB_MAX_TRAFFIC_CLASS + + hw->mac.max_tx_queues; + cap->n_levels_max = 3; + cap->non_leaf_nodes_identical = 1; + cap->leaf_nodes_identical = 1; + cap->shaper_n_max = cap->n_nodes_max; + cap->shaper_private_n_max = cap->n_nodes_max; + cap->shaper_private_dual_rate_n_max = 0; + cap->shaper_private_rate_min = 0; + /* 10Gbps -> 1.25GBps */ + cap->shaper_private_rate_max = 1250000000ull; + cap->shaper_shared_n_max = 0; + cap->shaper_shared_n_nodes_per_shaper_max = 0; + cap->shaper_shared_n_shapers_per_node_max = 0; + cap->shaper_shared_dual_rate_n_max = 0; + cap->shaper_shared_rate_min = 0; + cap->shaper_shared_rate_max = 0; + cap->sched_n_children_max = hw->mac.max_tx_queues; + /** + * HW supports SP. But no plan to support it now. + * So, all the nodes should have the same priority. + */ + cap->sched_sp_n_priorities_max = 1; + cap->sched_wfq_n_children_per_group_max = 0; + cap->sched_wfq_n_groups_max = 0; + /** + * SW only supports fair round robin now. + * So, all the nodes should have the same weight. + */ + cap->sched_wfq_weight_max = 1; + cap->cman_head_drop_supported = 0; + cap->dynamic_update_mask = 0; + cap->shaper_pkt_length_adjust_min = RTE_TM_ETH_FRAMING_OVERHEAD; + cap->shaper_pkt_length_adjust_max = RTE_TM_ETH_FRAMING_OVERHEAD_FCS; + cap->cman_wred_context_n_max = 0; + cap->cman_wred_context_private_n_max = 0; + cap->cman_wred_context_shared_n_max = 0; + cap->cman_wred_context_shared_n_nodes_per_context_max = 0; + cap->cman_wred_context_shared_n_contexts_per_node_max = 0; + cap->stats_mask = 0; + + return 0; +} + +static inline struct ixgbe_tm_shaper_profile * +ixgbe_shaper_profile_search(struct rte_eth_dev *dev, + uint32_t shaper_profile_id) +{ + struct ixgbe_tm_conf *tm_conf = + IXGBE_DEV_PRIVATE_TO_TM_CONF(dev->data->dev_private); + struct ixgbe_shaper_profile_list *shaper_profile_list = + &tm_conf->shaper_profile_list; + struct ixgbe_tm_shaper_profile *shaper_profile; + + TAILQ_FOREACH(shaper_profile, shaper_profile_list, node) { + if (shaper_profile_id == shaper_profile->shaper_profile_id) + return shaper_profile; + } + + return NULL; +} + +static int +ixgbe_shaper_profile_param_check(struct rte_tm_shaper_params *profile, + struct rte_tm_error *error) +{ + /* min rate not supported */ + if (profile->committed.rate) { + error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_COMMITTED_RATE; + error->message = "committed rate not supported"; + return -EINVAL; + } + /* min bucket size not supported */ + if (profile->committed.size) { + error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_COMMITTED_SIZE; + error->message = "committed bucket size not supported"; + return -EINVAL; + } + /* max bucket size not supported */ + if (profile->peak.size) { + error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_PEAK_SIZE; + error->message = "peak bucket size not supported"; + return -EINVAL; + } + /* length adjustment not supported */ + if (profile->pkt_length_adjust) { + error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_PKT_ADJUST_LEN; + error->message = "packet length adjustment not supported"; + return -EINVAL; + } + + return 0; +} + +static int +ixgbe_shaper_profile_add(struct rte_eth_dev *dev, + uint32_t shaper_profile_id, + struct rte_tm_shaper_params *profile, + struct rte_tm_error *error) +{ + struct ixgbe_tm_conf *tm_conf = + IXGBE_DEV_PRIVATE_TO_TM_CONF(dev->data->dev_private); + struct ixgbe_tm_shaper_profile *shaper_profile; + int ret; + + if (!profile || !error) + return -EINVAL; + + ret = ixgbe_shaper_profile_param_check(profile, error); + if (ret) + return ret; + + shaper_profile = ixgbe_shaper_profile_search(dev, shaper_profile_id); + + if (shaper_profile) { + error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_ID; + error->message = "profile ID exist"; + return -EINVAL; + } + + shaper_profile = rte_zmalloc("ixgbe_tm_shaper_profile", + sizeof(struct ixgbe_tm_shaper_profile), + 0); + if (!shaper_profile) + return -ENOMEM; + shaper_profile->shaper_profile_id = shaper_profile_id; + rte_memcpy(&shaper_profile->profile, profile, + sizeof(struct rte_tm_shaper_params)); + TAILQ_INSERT_TAIL(&tm_conf->shaper_profile_list, + shaper_profile, node); + + return 0; +} + +static int +ixgbe_shaper_profile_del(struct rte_eth_dev *dev, + uint32_t shaper_profile_id, + struct rte_tm_error *error) +{ + struct ixgbe_tm_conf *tm_conf = + IXGBE_DEV_PRIVATE_TO_TM_CONF(dev->data->dev_private); + struct ixgbe_tm_shaper_profile *shaper_profile; + + if (!error) + return -EINVAL; + + shaper_profile = ixgbe_shaper_profile_search(dev, shaper_profile_id); + + if (!shaper_profile) { + error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_ID; + error->message = "profile ID not exist"; + return -EINVAL; + } + + /* don't delete a profile if it's used by one or several nodes */ + if (shaper_profile->reference_count) { + error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE; + error->message = "profile in use"; + return -EINVAL; + } + + TAILQ_REMOVE(&tm_conf->shaper_profile_list, shaper_profile, node); + rte_free(shaper_profile); + + return 0; +} + +static inline struct ixgbe_tm_node * +ixgbe_tm_node_search(struct rte_eth_dev *dev, uint32_t node_id, + enum ixgbe_tm_node_type *node_type) +{ + struct ixgbe_tm_conf *tm_conf = + IXGBE_DEV_PRIVATE_TO_TM_CONF(dev->data->dev_private); + struct ixgbe_tm_node *tm_node; + + if (tm_conf->root && tm_conf->root->id == node_id) { + *node_type = IXGBE_TM_NODE_TYPE_PORT; + return tm_conf->root; + } + + TAILQ_FOREACH(tm_node, &tm_conf->tc_list, node) { + if (tm_node->id == node_id) { + *node_type = IXGBE_TM_NODE_TYPE_TC; + return tm_node; + } + } + + TAILQ_FOREACH(tm_node, &tm_conf->queue_list, node) { + if (tm_node->id == node_id) { + *node_type = IXGBE_TM_NODE_TYPE_QUEUE; + return tm_node; + } + } + + return NULL; +} + +static void +ixgbe_queue_base_nb_get(struct rte_eth_dev *dev, uint16_t tc_node_no, + uint16_t *base, uint16_t *nb) +{ + uint8_t nb_tcs = ixgbe_tc_nb_get(dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + uint16_t vf_num = pci_dev->max_vfs; + + *base = 0; + *nb = 0; + + /* VT on */ + if (vf_num) { + /* no DCB */ + if (nb_tcs == 1) { + if (vf_num >= ETH_32_POOLS) { + *nb = 2; + *base = vf_num * 2; + } else if (vf_num >= ETH_16_POOLS) { + *nb = 4; + *base = vf_num * 4; + } else { + *nb = 8; + *base = vf_num * 8; + } + } else { + /* DCB */ + *nb = 1; + *base = vf_num * nb_tcs + tc_node_no; + } + } else { + /* VT off */ + if (nb_tcs == ETH_8_TCS) { + switch (tc_node_no) { + case 0: + *base = 0; + *nb = 32; + break; + case 1: + *base = 32; + *nb = 32; + break; + case 2: + *base = 64; + *nb = 16; + break; + case 3: + *base = 80; + *nb = 16; + break; + case 4: + *base = 96; + *nb = 8; + break; + case 5: + *base = 104; + *nb = 8; + break; + case 6: + *base = 112; + *nb = 8; + break; + case 7: + *base = 120; + *nb = 8; + break; + default: + return; + } + } else { + switch (tc_node_no) { + /** + * If no VF and no DCB, only 64 queues can be used. + * This case also be covered by this "case 0". + */ + case 0: + *base = 0; + *nb = 64; + break; + case 1: + *base = 64; + *nb = 32; + break; + case 2: + *base = 96; + *nb = 16; + break; + case 3: + *base = 112; + *nb = 16; + break; + default: + return; + } + } + } +} + +static int +ixgbe_node_param_check(struct rte_eth_dev *dev, uint32_t node_id, + uint32_t priority, uint32_t weight, + struct rte_tm_node_params *params, + struct rte_tm_error *error) +{ + if (node_id == RTE_TM_NODE_ID_NULL) { + error->type = RTE_TM_ERROR_TYPE_NODE_ID; + error->message = "invalid node id"; + return -EINVAL; + } + + if (priority) { + error->type = RTE_TM_ERROR_TYPE_NODE_PRIORITY; + error->message = "priority should be 0"; + return -EINVAL; + } + + if (weight != 1) { + error->type = RTE_TM_ERROR_TYPE_NODE_WEIGHT; + error->message = "weight must be 1"; + return -EINVAL; + } + + /* not support shared shaper */ + if (params->shared_shaper_id) { + error->type = RTE_TM_ERROR_TYPE_NODE_PARAMS_SHARED_SHAPER_ID; + error->message = "shared shaper not supported"; + return -EINVAL; + } + if (params->n_shared_shapers) { + error->type = RTE_TM_ERROR_TYPE_NODE_PARAMS_N_SHARED_SHAPERS; + error->message = "shared shaper not supported"; + return -EINVAL; + } + + /* for non-leaf node */ + if (node_id >= dev->data->nb_tx_queues) { + /* check the unsupported parameters */ + if (params->nonleaf.wfq_weight_mode) { + error->type = + RTE_TM_ERROR_TYPE_NODE_PARAMS_WFQ_WEIGHT_MODE; + error->message = "WFQ not supported"; + return -EINVAL; + } + if (params->nonleaf.n_sp_priorities != 1) { + error->type = + RTE_TM_ERROR_TYPE_NODE_PARAMS_N_SP_PRIORITIES; + error->message = "SP priority not supported"; + return -EINVAL; + } else if (params->nonleaf.wfq_weight_mode && + !(*params->nonleaf.wfq_weight_mode)) { + error->type = + RTE_TM_ERROR_TYPE_NODE_PARAMS_WFQ_WEIGHT_MODE; + error->message = "WFP should be byte mode"; + return -EINVAL; + } + + return 0; + } + + /* for leaf node */ + /* check the unsupported parameters */ + if (params->leaf.cman) { + error->type = RTE_TM_ERROR_TYPE_NODE_PARAMS_CMAN; + error->message = "Congestion management not supported"; + return -EINVAL; + } + if (params->leaf.wred.wred_profile_id != + RTE_TM_WRED_PROFILE_ID_NONE) { + error->type = + RTE_TM_ERROR_TYPE_NODE_PARAMS_WRED_PROFILE_ID; + error->message = "WRED not supported"; + return -EINVAL; + } + if (params->leaf.wred.shared_wred_context_id) { + error->type = + RTE_TM_ERROR_TYPE_NODE_PARAMS_SHARED_WRED_CONTEXT_ID; + error->message = "WRED not supported"; + return -EINVAL; + } + if (params->leaf.wred.n_shared_wred_contexts) { + error->type = + RTE_TM_ERROR_TYPE_NODE_PARAMS_N_SHARED_WRED_CONTEXTS; + error->message = "WRED not supported"; + return -EINVAL; + } + + return 0; +} + +/** + * Now the TC and queue configuration is controlled by DCB. + * We need check if the node configuration follows the DCB configuration. + * In the future, we may use TM to cover DCB. + */ +static int +ixgbe_node_add(struct rte_eth_dev *dev, uint32_t node_id, + uint32_t parent_node_id, uint32_t priority, + uint32_t weight, uint32_t level_id, + struct rte_tm_node_params *params, + struct rte_tm_error *error) +{ + struct ixgbe_tm_conf *tm_conf = + IXGBE_DEV_PRIVATE_TO_TM_CONF(dev->data->dev_private); + enum ixgbe_tm_node_type node_type = IXGBE_TM_NODE_TYPE_MAX; + enum ixgbe_tm_node_type parent_node_type = IXGBE_TM_NODE_TYPE_MAX; + struct ixgbe_tm_shaper_profile *shaper_profile = NULL; + struct ixgbe_tm_node *tm_node; + struct ixgbe_tm_node *parent_node; + uint8_t nb_tcs; + uint16_t q_base = 0; + uint16_t q_nb = 0; + int ret; + + if (!params || !error) + return -EINVAL; + + /* if already committed */ + if (tm_conf->committed) { + error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED; + error->message = "already committed"; + return -EINVAL; + } + + ret = ixgbe_node_param_check(dev, node_id, priority, weight, + params, error); + if (ret) + return ret; + + /* check if the node ID is already used */ + if (ixgbe_tm_node_search(dev, node_id, &node_type)) { + error->type = RTE_TM_ERROR_TYPE_NODE_ID; + error->message = "node id already used"; + return -EINVAL; + } + + /* check the shaper profile id */ + if (params->shaper_profile_id != RTE_TM_SHAPER_PROFILE_ID_NONE) { + shaper_profile = ixgbe_shaper_profile_search( + dev, params->shaper_profile_id); + if (!shaper_profile) { + error->type = + RTE_TM_ERROR_TYPE_NODE_PARAMS_SHAPER_PROFILE_ID; + error->message = "shaper profile not exist"; + return -EINVAL; + } + } + + /* root node if not have a parent */ + if (parent_node_id == RTE_TM_NODE_ID_NULL) { + /* check level */ + if (level_id != RTE_TM_NODE_LEVEL_ID_ANY && + level_id > IXGBE_TM_NODE_TYPE_PORT) { + error->type = RTE_TM_ERROR_TYPE_NODE_PARAMS; + error->message = "Wrong level"; + return -EINVAL; + } + + /* obviously no more than one root */ + if (tm_conf->root) { + error->type = RTE_TM_ERROR_TYPE_NODE_PARENT_NODE_ID; + error->message = "already have a root"; + return -EINVAL; + } + + /* add the root node */ + tm_node = rte_zmalloc("ixgbe_tm_node", + sizeof(struct ixgbe_tm_node), + 0); + if (!tm_node) + return -ENOMEM; + tm_node->id = node_id; + tm_node->priority = priority; + tm_node->weight = weight; + tm_node->reference_count = 0; + tm_node->no = 0; + tm_node->parent = NULL; + tm_node->shaper_profile = shaper_profile; + rte_memcpy(&tm_node->params, params, + sizeof(struct rte_tm_node_params)); + tm_conf->root = tm_node; + + /* increase the reference counter of the shaper profile */ + if (shaper_profile) + shaper_profile->reference_count++; + + return 0; + } + + /* TC or queue node */ + /* check the parent node */ + parent_node = ixgbe_tm_node_search(dev, parent_node_id, + &parent_node_type); + if (!parent_node) { + error->type = RTE_TM_ERROR_TYPE_NODE_PARENT_NODE_ID; + error->message = "parent not exist"; + return -EINVAL; + } + if (parent_node_type != IXGBE_TM_NODE_TYPE_PORT && + parent_node_type != IXGBE_TM_NODE_TYPE_TC) { + error->type = RTE_TM_ERROR_TYPE_NODE_PARENT_NODE_ID; + error->message = "parent is not port or TC"; + return -EINVAL; + } + /* check level */ + if (level_id != RTE_TM_NODE_LEVEL_ID_ANY && + level_id != parent_node_type + 1) { + error->type = RTE_TM_ERROR_TYPE_NODE_PARAMS; + error->message = "Wrong level"; + return -EINVAL; + } + + /* check the node number */ + if (parent_node_type == IXGBE_TM_NODE_TYPE_PORT) { + /* check TC number */ + nb_tcs = ixgbe_tc_nb_get(dev); + if (tm_conf->nb_tc_node >= nb_tcs) { + error->type = RTE_TM_ERROR_TYPE_NODE_ID; + error->message = "too many TCs"; + return -EINVAL; + } + } else { + /* check queue number */ + if (tm_conf->nb_queue_node >= dev->data->nb_tx_queues) { + error->type = RTE_TM_ERROR_TYPE_NODE_ID; + error->message = "too many queues"; + return -EINVAL; + } + + ixgbe_queue_base_nb_get(dev, parent_node->no, &q_base, &q_nb); + if (parent_node->reference_count >= q_nb) { + error->type = RTE_TM_ERROR_TYPE_NODE_ID; + error->message = "too many queues than TC supported"; + return -EINVAL; + } + + /** + * check the node id. + * For queue, the node id means queue id. + */ + if (node_id >= dev->data->nb_tx_queues) { + error->type = RTE_TM_ERROR_TYPE_NODE_ID; + error->message = "too large queue id"; + return -EINVAL; + } + } + + /* add the TC or queue node */ + tm_node = rte_zmalloc("ixgbe_tm_node", + sizeof(struct ixgbe_tm_node), + 0); + if (!tm_node) + return -ENOMEM; + tm_node->id = node_id; + tm_node->priority = priority; + tm_node->weight = weight; + tm_node->reference_count = 0; + tm_node->parent = parent_node; + tm_node->shaper_profile = shaper_profile; + rte_memcpy(&tm_node->params, params, + sizeof(struct rte_tm_node_params)); + if (parent_node_type == IXGBE_TM_NODE_TYPE_PORT) { + tm_node->no = parent_node->reference_count; + TAILQ_INSERT_TAIL(&tm_conf->tc_list, + tm_node, node); + tm_conf->nb_tc_node++; + } else { + tm_node->no = q_base + parent_node->reference_count; + TAILQ_INSERT_TAIL(&tm_conf->queue_list, + tm_node, node); + tm_conf->nb_queue_node++; + } + tm_node->parent->reference_count++; + + /* increase the reference counter of the shaper profile */ + if (shaper_profile) + shaper_profile->reference_count++; + + return 0; +} + +static int +ixgbe_node_delete(struct rte_eth_dev *dev, uint32_t node_id, + struct rte_tm_error *error) +{ + struct ixgbe_tm_conf *tm_conf = + IXGBE_DEV_PRIVATE_TO_TM_CONF(dev->data->dev_private); + enum ixgbe_tm_node_type node_type = IXGBE_TM_NODE_TYPE_MAX; + struct ixgbe_tm_node *tm_node; + + if (!error) + return -EINVAL; + + /* if already committed */ + if (tm_conf->committed) { + error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED; + error->message = "already committed"; + return -EINVAL; + } + + if (node_id == RTE_TM_NODE_ID_NULL) { + error->type = RTE_TM_ERROR_TYPE_NODE_ID; + error->message = "invalid node id"; + return -EINVAL; + } + + /* check the if the node id exists */ + tm_node = ixgbe_tm_node_search(dev, node_id, &node_type); + if (!tm_node) { + error->type = RTE_TM_ERROR_TYPE_NODE_ID; + error->message = "no such node"; + return -EINVAL; + } + + /* the node should have no child */ + if (tm_node->reference_count) { + error->type = RTE_TM_ERROR_TYPE_NODE_ID; + error->message = + "cannot delete a node which has children"; + return -EINVAL; + } + + /* root node */ + if (node_type == IXGBE_TM_NODE_TYPE_PORT) { + if (tm_node->shaper_profile) + tm_node->shaper_profile->reference_count--; + rte_free(tm_node); + tm_conf->root = NULL; + return 0; + } + + /* TC or queue node */ + if (tm_node->shaper_profile) + tm_node->shaper_profile->reference_count--; + tm_node->parent->reference_count--; + if (node_type == IXGBE_TM_NODE_TYPE_TC) { + TAILQ_REMOVE(&tm_conf->tc_list, tm_node, node); + tm_conf->nb_tc_node--; + } else { + TAILQ_REMOVE(&tm_conf->queue_list, tm_node, node); + tm_conf->nb_queue_node--; + } + rte_free(tm_node); + + return 0; +} + +static int +ixgbe_node_type_get(struct rte_eth_dev *dev, uint32_t node_id, + int *is_leaf, struct rte_tm_error *error) +{ + enum ixgbe_tm_node_type node_type = IXGBE_TM_NODE_TYPE_MAX; + struct ixgbe_tm_node *tm_node; + + if (!is_leaf || !error) + return -EINVAL; + + if (node_id == RTE_TM_NODE_ID_NULL) { + error->type = RTE_TM_ERROR_TYPE_NODE_ID; + error->message = "invalid node id"; + return -EINVAL; + } + + /* check if the node id exists */ + tm_node = ixgbe_tm_node_search(dev, node_id, &node_type); + if (!tm_node) { + error->type = RTE_TM_ERROR_TYPE_NODE_ID; + error->message = "no such node"; + return -EINVAL; + } + + if (node_type == IXGBE_TM_NODE_TYPE_QUEUE) + *is_leaf = true; + else + *is_leaf = false; + + return 0; +} + +static int +ixgbe_level_capabilities_get(struct rte_eth_dev *dev, + uint32_t level_id, + struct rte_tm_level_capabilities *cap, + struct rte_tm_error *error) +{ + struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + + if (!cap || !error) + return -EINVAL; + + if (level_id >= IXGBE_TM_NODE_TYPE_MAX) { + error->type = RTE_TM_ERROR_TYPE_LEVEL_ID; + error->message = "too deep level"; + return -EINVAL; + } + + /* root node */ + if (level_id == IXGBE_TM_NODE_TYPE_PORT) { + cap->n_nodes_max = 1; + cap->n_nodes_nonleaf_max = 1; + cap->n_nodes_leaf_max = 0; + } else if (level_id == IXGBE_TM_NODE_TYPE_TC) { + /* TC */ + cap->n_nodes_max = IXGBE_DCB_MAX_TRAFFIC_CLASS; + cap->n_nodes_nonleaf_max = IXGBE_DCB_MAX_TRAFFIC_CLASS; + cap->n_nodes_leaf_max = 0; + } else { + /* queue */ + cap->n_nodes_max = hw->mac.max_tx_queues; + cap->n_nodes_nonleaf_max = 0; + cap->n_nodes_leaf_max = hw->mac.max_tx_queues; + } + + cap->non_leaf_nodes_identical = true; + cap->leaf_nodes_identical = true; + + if (level_id != IXGBE_TM_NODE_TYPE_QUEUE) { + cap->nonleaf.shaper_private_supported = true; + cap->nonleaf.shaper_private_dual_rate_supported = false; + cap->nonleaf.shaper_private_rate_min = 0; + /* 10Gbps -> 1.25GBps */ + cap->nonleaf.shaper_private_rate_max = 1250000000ull; + cap->nonleaf.shaper_shared_n_max = 0; + if (level_id == IXGBE_TM_NODE_TYPE_PORT) + cap->nonleaf.sched_n_children_max = + IXGBE_DCB_MAX_TRAFFIC_CLASS; + else + cap->nonleaf.sched_n_children_max = + hw->mac.max_tx_queues; + cap->nonleaf.sched_sp_n_priorities_max = 1; + cap->nonleaf.sched_wfq_n_children_per_group_max = 0; + cap->nonleaf.sched_wfq_n_groups_max = 0; + cap->nonleaf.sched_wfq_weight_max = 1; + cap->nonleaf.stats_mask = 0; + + return 0; + } + + /* queue node */ + cap->leaf.shaper_private_supported = true; + cap->leaf.shaper_private_dual_rate_supported = false; + cap->leaf.shaper_private_rate_min = 0; + /* 10Gbps -> 1.25GBps */ + cap->leaf.shaper_private_rate_max = 1250000000ull; + cap->leaf.shaper_shared_n_max = 0; + cap->leaf.cman_head_drop_supported = false; + cap->leaf.cman_wred_context_private_supported = true; + cap->leaf.cman_wred_context_shared_n_max = 0; + cap->leaf.stats_mask = 0; + + return 0; +} + +static int +ixgbe_node_capabilities_get(struct rte_eth_dev *dev, + uint32_t node_id, + struct rte_tm_node_capabilities *cap, + struct rte_tm_error *error) +{ + struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + enum ixgbe_tm_node_type node_type = IXGBE_TM_NODE_TYPE_MAX; + struct ixgbe_tm_node *tm_node; + + if (!cap || !error) + return -EINVAL; + + if (node_id == RTE_TM_NODE_ID_NULL) { + error->type = RTE_TM_ERROR_TYPE_NODE_ID; + error->message = "invalid node id"; + return -EINVAL; + } + + /* check if the node id exists */ + tm_node = ixgbe_tm_node_search(dev, node_id, &node_type); + if (!tm_node) { + error->type = RTE_TM_ERROR_TYPE_NODE_ID; + error->message = "no such node"; + return -EINVAL; + } + + cap->shaper_private_supported = true; + cap->shaper_private_dual_rate_supported = false; + cap->shaper_private_rate_min = 0; + /* 10Gbps -> 1.25GBps */ + cap->shaper_private_rate_max = 1250000000ull; + cap->shaper_shared_n_max = 0; + + if (node_type == IXGBE_TM_NODE_TYPE_QUEUE) { + cap->leaf.cman_head_drop_supported = false; + cap->leaf.cman_wred_context_private_supported = true; + cap->leaf.cman_wred_context_shared_n_max = 0; + } else { + if (node_type == IXGBE_TM_NODE_TYPE_PORT) + cap->nonleaf.sched_n_children_max = + IXGBE_DCB_MAX_TRAFFIC_CLASS; + else + cap->nonleaf.sched_n_children_max = + hw->mac.max_tx_queues; + cap->nonleaf.sched_sp_n_priorities_max = 1; + cap->nonleaf.sched_wfq_n_children_per_group_max = 0; + cap->nonleaf.sched_wfq_n_groups_max = 0; + cap->nonleaf.sched_wfq_weight_max = 1; + } + + cap->stats_mask = 0; + + return 0; +} + +static int +ixgbe_hierarchy_commit(struct rte_eth_dev *dev, + int clear_on_fail, + struct rte_tm_error *error) +{ + struct ixgbe_tm_conf *tm_conf = + IXGBE_DEV_PRIVATE_TO_TM_CONF(dev->data->dev_private); + struct ixgbe_tm_node *tm_node; + uint64_t bw; + int ret; + + if (!error) + return -EINVAL; + + /* check the setting */ + if (!tm_conf->root) + goto done; + + /* not support port max bandwidth yet */ + if (tm_conf->root->shaper_profile && + tm_conf->root->shaper_profile->profile.peak.rate) { + error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE; + error->message = "no port max bandwidth"; + goto fail_clear; + } + + /* HW not support TC max bandwidth */ + TAILQ_FOREACH(tm_node, &tm_conf->tc_list, node) { + if (tm_node->shaper_profile && + tm_node->shaper_profile->profile.peak.rate) { + error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE; + error->message = "no TC max bandwidth"; + goto fail_clear; + } + } + + /* queue max bandwidth */ + TAILQ_FOREACH(tm_node, &tm_conf->queue_list, node) { + if (tm_node->shaper_profile) + bw = tm_node->shaper_profile->profile.peak.rate; + else + bw = 0; + if (bw) { + /* interpret Bps to Mbps */ + bw = bw * 8 / 1000 / 1000; + ret = ixgbe_set_queue_rate_limit(dev, tm_node->no, bw); + if (ret) { + error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE; + error->message = + "failed to set queue max bandwidth"; + goto fail_clear; + } + } + } + +done: + tm_conf->committed = true; + return 0; + +fail_clear: + /* clear all the traffic manager configuration */ + if (clear_on_fail) { + ixgbe_tm_conf_uninit(dev); + ixgbe_tm_conf_init(dev); + } + return -EINVAL; +} diff --git a/dpdk/drivers/net/ixgbe/rte_pmd_ixgbe.c b/dpdk/drivers/net/ixgbe/rte_pmd_ixgbe.c new file mode 100644 index 00000000..f1273785 --- /dev/null +++ b/dpdk/drivers/net/ixgbe/rte_pmd_ixgbe.c @@ -0,0 +1,1043 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +#include "base/ixgbe_api.h" +#include "ixgbe_ethdev.h" +#include "rte_pmd_ixgbe.h" + +int +rte_pmd_ixgbe_set_vf_mac_addr(uint16_t port, uint16_t vf, + struct ether_addr *mac_addr) +{ + struct ixgbe_hw *hw; + struct ixgbe_vf_info *vfinfo; + int rar_entry; + uint8_t *new_mac = (uint8_t *)(mac_addr); + struct rte_eth_dev *dev; + struct rte_pci_device *pci_dev; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + dev = &rte_eth_devices[port]; + pci_dev = RTE_ETH_DEV_TO_PCI(dev); + + if (!is_ixgbe_supported(dev)) + return -ENOTSUP; + + if (vf >= pci_dev->max_vfs) + return -EINVAL; + + hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + vfinfo = *(IXGBE_DEV_PRIVATE_TO_P_VFDATA(dev->data->dev_private)); + rar_entry = hw->mac.num_rar_entries - (vf + 1); + + if (is_valid_assigned_ether_addr((struct ether_addr *)new_mac)) { + rte_memcpy(vfinfo[vf].vf_mac_addresses, new_mac, + ETHER_ADDR_LEN); + return hw->mac.ops.set_rar(hw, rar_entry, new_mac, vf, + IXGBE_RAH_AV); + } + return -EINVAL; +} + +int +rte_pmd_ixgbe_ping_vf(uint16_t port, uint16_t vf) +{ + struct ixgbe_hw *hw; + struct ixgbe_vf_info *vfinfo; + struct rte_eth_dev *dev; + struct rte_pci_device *pci_dev; + uint32_t ctrl; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + dev = &rte_eth_devices[port]; + pci_dev = RTE_ETH_DEV_TO_PCI(dev); + + if (!is_ixgbe_supported(dev)) + return -ENOTSUP; + + if (vf >= pci_dev->max_vfs) + return -EINVAL; + + hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + vfinfo = *(IXGBE_DEV_PRIVATE_TO_P_VFDATA(dev->data->dev_private)); + + ctrl = IXGBE_PF_CONTROL_MSG; + if (vfinfo[vf].clear_to_send) + ctrl |= IXGBE_VT_MSGTYPE_CTS; + + ixgbe_write_mbx(hw, &ctrl, 1, vf); + + return 0; +} + +int +rte_pmd_ixgbe_set_vf_vlan_anti_spoof(uint16_t port, uint16_t vf, uint8_t on) +{ + struct ixgbe_hw *hw; + struct ixgbe_mac_info *mac; + struct rte_eth_dev *dev; + struct rte_pci_device *pci_dev; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + dev = &rte_eth_devices[port]; + pci_dev = RTE_ETH_DEV_TO_PCI(dev); + + if (!is_ixgbe_supported(dev)) + return -ENOTSUP; + + if (vf >= pci_dev->max_vfs) + return -EINVAL; + + if (on > 1) + return -EINVAL; + + hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + mac = &hw->mac; + + mac->ops.set_vlan_anti_spoofing(hw, on, vf); + + return 0; +} + +int +rte_pmd_ixgbe_set_vf_mac_anti_spoof(uint16_t port, uint16_t vf, uint8_t on) +{ + struct ixgbe_hw *hw; + struct ixgbe_mac_info *mac; + struct rte_eth_dev *dev; + struct rte_pci_device *pci_dev; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + dev = &rte_eth_devices[port]; + pci_dev = RTE_ETH_DEV_TO_PCI(dev); + + if (!is_ixgbe_supported(dev)) + return -ENOTSUP; + + if (vf >= pci_dev->max_vfs) + return -EINVAL; + + if (on > 1) + return -EINVAL; + + hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + mac = &hw->mac; + mac->ops.set_mac_anti_spoofing(hw, on, vf); + + return 0; +} + +int +rte_pmd_ixgbe_set_vf_vlan_insert(uint16_t port, uint16_t vf, uint16_t vlan_id) +{ + struct ixgbe_hw *hw; + uint32_t ctrl; + struct rte_eth_dev *dev; + struct rte_pci_device *pci_dev; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + dev = &rte_eth_devices[port]; + pci_dev = RTE_ETH_DEV_TO_PCI(dev); + + if (!is_ixgbe_supported(dev)) + return -ENOTSUP; + + if (vf >= pci_dev->max_vfs) + return -EINVAL; + + if (vlan_id > ETHER_MAX_VLAN_ID) + return -EINVAL; + + hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + ctrl = IXGBE_READ_REG(hw, IXGBE_VMVIR(vf)); + if (vlan_id) { + ctrl = vlan_id; + ctrl |= IXGBE_VMVIR_VLANA_DEFAULT; + } else { + ctrl = 0; + } + + IXGBE_WRITE_REG(hw, IXGBE_VMVIR(vf), ctrl); + + return 0; +} + +int +rte_pmd_ixgbe_set_tx_loopback(uint16_t port, uint8_t on) +{ + struct ixgbe_hw *hw; + uint32_t ctrl; + struct rte_eth_dev *dev; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + dev = &rte_eth_devices[port]; + + if (!is_ixgbe_supported(dev)) + return -ENOTSUP; + + if (on > 1) + return -EINVAL; + + hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + ctrl = IXGBE_READ_REG(hw, IXGBE_PFDTXGSWC); + /* enable or disable VMDQ loopback */ + if (on) + ctrl |= IXGBE_PFDTXGSWC_VT_LBEN; + else + ctrl &= ~IXGBE_PFDTXGSWC_VT_LBEN; + + IXGBE_WRITE_REG(hw, IXGBE_PFDTXGSWC, ctrl); + + return 0; +} + +int +rte_pmd_ixgbe_set_all_queues_drop_en(uint16_t port, uint8_t on) +{ + struct ixgbe_hw *hw; + uint32_t reg_value; + int i; + int num_queues = (int)(IXGBE_QDE_IDX_MASK >> IXGBE_QDE_IDX_SHIFT); + struct rte_eth_dev *dev; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + dev = &rte_eth_devices[port]; + + if (!is_ixgbe_supported(dev)) + return -ENOTSUP; + + if (on > 1) + return -EINVAL; + + hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + for (i = 0; i <= num_queues; i++) { + reg_value = IXGBE_QDE_WRITE | + (i << IXGBE_QDE_IDX_SHIFT) | + (on & IXGBE_QDE_ENABLE); + IXGBE_WRITE_REG(hw, IXGBE_QDE, reg_value); + } + + return 0; +} + +int +rte_pmd_ixgbe_set_vf_split_drop_en(uint16_t port, uint16_t vf, uint8_t on) +{ + struct ixgbe_hw *hw; + uint32_t reg_value; + struct rte_eth_dev *dev; + struct rte_pci_device *pci_dev; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + dev = &rte_eth_devices[port]; + pci_dev = RTE_ETH_DEV_TO_PCI(dev); + + if (!is_ixgbe_supported(dev)) + return -ENOTSUP; + + /* only support VF's 0 to 63 */ + if ((vf >= pci_dev->max_vfs) || (vf > 63)) + return -EINVAL; + + if (on > 1) + return -EINVAL; + + hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + reg_value = IXGBE_READ_REG(hw, IXGBE_SRRCTL(vf)); + if (on) + reg_value |= IXGBE_SRRCTL_DROP_EN; + else + reg_value &= ~IXGBE_SRRCTL_DROP_EN; + + IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(vf), reg_value); + + return 0; +} + +int +rte_pmd_ixgbe_set_vf_vlan_stripq(uint16_t port, uint16_t vf, uint8_t on) +{ + struct rte_eth_dev *dev; + struct rte_pci_device *pci_dev; + struct ixgbe_hw *hw; + uint16_t queues_per_pool; + uint32_t q; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + dev = &rte_eth_devices[port]; + pci_dev = RTE_ETH_DEV_TO_PCI(dev); + hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + + if (!is_ixgbe_supported(dev)) + return -ENOTSUP; + + if (vf >= pci_dev->max_vfs) + return -EINVAL; + + if (on > 1) + return -EINVAL; + + RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_strip_queue_set, -ENOTSUP); + + /* The PF has 128 queue pairs and in SRIOV configuration + * those queues will be assigned to VF's, so RXDCTL + * registers will be dealing with queues which will be + * assigned to VF's. + * Let's say we have SRIOV configured with 31 VF's then the + * first 124 queues 0-123 will be allocated to VF's and only + * the last 4 queues 123-127 will be assigned to the PF. + */ + if (hw->mac.type == ixgbe_mac_82598EB) + queues_per_pool = (uint16_t)hw->mac.max_rx_queues / + ETH_16_POOLS; + else + queues_per_pool = (uint16_t)hw->mac.max_rx_queues / + ETH_64_POOLS; + + for (q = 0; q < queues_per_pool; q++) + (*dev->dev_ops->vlan_strip_queue_set)(dev, + q + vf * queues_per_pool, on); + return 0; +} + +int +rte_pmd_ixgbe_set_vf_rxmode(uint16_t port, uint16_t vf, + uint16_t rx_mask, uint8_t on) +{ + int val = 0; + struct rte_eth_dev *dev; + struct rte_pci_device *pci_dev; + struct ixgbe_hw *hw; + uint32_t vmolr; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + dev = &rte_eth_devices[port]; + pci_dev = RTE_ETH_DEV_TO_PCI(dev); + + if (!is_ixgbe_supported(dev)) + return -ENOTSUP; + + if (vf >= pci_dev->max_vfs) + return -EINVAL; + + if (on > 1) + return -EINVAL; + + hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + vmolr = IXGBE_READ_REG(hw, IXGBE_VMOLR(vf)); + + if (hw->mac.type == ixgbe_mac_82598EB) { + PMD_INIT_LOG(ERR, "setting VF receive mode set should be done" + " on 82599 hardware and newer"); + return -ENOTSUP; + } + if (ixgbe_vt_check(hw) < 0) + return -ENOTSUP; + + val = ixgbe_convert_vm_rx_mask_to_val(rx_mask, val); + + if (on) + vmolr |= val; + else + vmolr &= ~val; + + IXGBE_WRITE_REG(hw, IXGBE_VMOLR(vf), vmolr); + + return 0; +} + +int +rte_pmd_ixgbe_set_vf_rx(uint16_t port, uint16_t vf, uint8_t on) +{ + struct rte_eth_dev *dev; + struct rte_pci_device *pci_dev; + uint32_t reg, addr; + uint32_t val; + const uint8_t bit1 = 0x1; + struct ixgbe_hw *hw; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + dev = &rte_eth_devices[port]; + pci_dev = RTE_ETH_DEV_TO_PCI(dev); + + if (!is_ixgbe_supported(dev)) + return -ENOTSUP; + + if (vf >= pci_dev->max_vfs) + return -EINVAL; + + if (on > 1) + return -EINVAL; + + hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + + if (ixgbe_vt_check(hw) < 0) + return -ENOTSUP; + + /* for vf >= 32, set bit in PFVFRE[1], otherwise PFVFRE[0] */ + if (vf >= 32) { + addr = IXGBE_VFRE(1); + val = bit1 << (vf - 32); + } else { + addr = IXGBE_VFRE(0); + val = bit1 << vf; + } + + reg = IXGBE_READ_REG(hw, addr); + + if (on) + reg |= val; + else + reg &= ~val; + + IXGBE_WRITE_REG(hw, addr, reg); + + return 0; +} + +int +rte_pmd_ixgbe_set_vf_tx(uint16_t port, uint16_t vf, uint8_t on) +{ + struct rte_eth_dev *dev; + struct rte_pci_device *pci_dev; + uint32_t reg, addr; + uint32_t val; + const uint8_t bit1 = 0x1; + + struct ixgbe_hw *hw; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + dev = &rte_eth_devices[port]; + pci_dev = RTE_ETH_DEV_TO_PCI(dev); + + if (!is_ixgbe_supported(dev)) + return -ENOTSUP; + + if (vf >= pci_dev->max_vfs) + return -EINVAL; + + if (on > 1) + return -EINVAL; + + hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + if (ixgbe_vt_check(hw) < 0) + return -ENOTSUP; + + /* for vf >= 32, set bit in PFVFTE[1], otherwise PFVFTE[0] */ + if (vf >= 32) { + addr = IXGBE_VFTE(1); + val = bit1 << (vf - 32); + } else { + addr = IXGBE_VFTE(0); + val = bit1 << vf; + } + + reg = IXGBE_READ_REG(hw, addr); + + if (on) + reg |= val; + else + reg &= ~val; + + IXGBE_WRITE_REG(hw, addr, reg); + + return 0; +} + +int +rte_pmd_ixgbe_set_vf_vlan_filter(uint16_t port, uint16_t vlan, + uint64_t vf_mask, uint8_t vlan_on) +{ + struct rte_eth_dev *dev; + int ret = 0; + uint16_t vf_idx; + struct ixgbe_hw *hw; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + dev = &rte_eth_devices[port]; + + if (!is_ixgbe_supported(dev)) + return -ENOTSUP; + + if ((vlan > ETHER_MAX_VLAN_ID) || (vf_mask == 0)) + return -EINVAL; + + hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + if (ixgbe_vt_check(hw) < 0) + return -ENOTSUP; + + for (vf_idx = 0; vf_idx < 64; vf_idx++) { + if (vf_mask & ((uint64_t)(1ULL << vf_idx))) { + ret = hw->mac.ops.set_vfta(hw, vlan, vf_idx, + vlan_on, false); + if (ret < 0) + return ret; + } + } + + return ret; +} + +int +rte_pmd_ixgbe_set_vf_rate_limit(uint16_t port, uint16_t vf, + uint16_t tx_rate, uint64_t q_msk) +{ + struct rte_eth_dev *dev; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + dev = &rte_eth_devices[port]; + + if (!is_ixgbe_supported(dev)) + return -ENOTSUP; + + return ixgbe_set_vf_rate_limit(dev, vf, tx_rate, q_msk); +} + +int +rte_pmd_ixgbe_macsec_enable(uint16_t port, uint8_t en, uint8_t rp) +{ + struct ixgbe_hw *hw; + struct rte_eth_dev *dev; + uint32_t ctrl; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + dev = &rte_eth_devices[port]; + + if (!is_ixgbe_supported(dev)) + return -ENOTSUP; + + hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + + /* Stop the data paths */ + if (ixgbe_disable_sec_rx_path(hw) != IXGBE_SUCCESS) + return -ENOTSUP; + /** + * Workaround: + * As no ixgbe_disable_sec_rx_path equivalent is + * implemented for tx in the base code, and we are + * not allowed to modify the base code in DPDK, so + * just call the hand-written one directly for now. + * The hardware support has been checked by + * ixgbe_disable_sec_rx_path(). + */ + ixgbe_disable_sec_tx_path_generic(hw); + + /* Enable Ethernet CRC (required by MACsec offload) */ + ctrl = IXGBE_READ_REG(hw, IXGBE_HLREG0); + ctrl |= IXGBE_HLREG0_TXCRCEN | IXGBE_HLREG0_RXCRCSTRP; + IXGBE_WRITE_REG(hw, IXGBE_HLREG0, ctrl); + + /* Enable the TX and RX crypto engines */ + ctrl = IXGBE_READ_REG(hw, IXGBE_SECTXCTRL); + ctrl &= ~IXGBE_SECTXCTRL_SECTX_DIS; + IXGBE_WRITE_REG(hw, IXGBE_SECTXCTRL, ctrl); + + ctrl = IXGBE_READ_REG(hw, IXGBE_SECRXCTRL); + ctrl &= ~IXGBE_SECRXCTRL_SECRX_DIS; + IXGBE_WRITE_REG(hw, IXGBE_SECRXCTRL, ctrl); + + ctrl = IXGBE_READ_REG(hw, IXGBE_SECTXMINIFG); + ctrl &= ~IXGBE_SECTX_MINSECIFG_MASK; + ctrl |= 0x3; + IXGBE_WRITE_REG(hw, IXGBE_SECTXMINIFG, ctrl); + + /* Enable SA lookup */ + ctrl = IXGBE_READ_REG(hw, IXGBE_LSECTXCTRL); + ctrl &= ~IXGBE_LSECTXCTRL_EN_MASK; + ctrl |= en ? IXGBE_LSECTXCTRL_AUTH_ENCRYPT : + IXGBE_LSECTXCTRL_AUTH; + ctrl |= IXGBE_LSECTXCTRL_AISCI; + ctrl &= ~IXGBE_LSECTXCTRL_PNTHRSH_MASK; + ctrl |= IXGBE_MACSEC_PNTHRSH & IXGBE_LSECTXCTRL_PNTHRSH_MASK; + IXGBE_WRITE_REG(hw, IXGBE_LSECTXCTRL, ctrl); + + ctrl = IXGBE_READ_REG(hw, IXGBE_LSECRXCTRL); + ctrl &= ~IXGBE_LSECRXCTRL_EN_MASK; + ctrl |= IXGBE_LSECRXCTRL_STRICT << IXGBE_LSECRXCTRL_EN_SHIFT; + ctrl &= ~IXGBE_LSECRXCTRL_PLSH; + if (rp) + ctrl |= IXGBE_LSECRXCTRL_RP; + else + ctrl &= ~IXGBE_LSECRXCTRL_RP; + IXGBE_WRITE_REG(hw, IXGBE_LSECRXCTRL, ctrl); + + /* Start the data paths */ + ixgbe_enable_sec_rx_path(hw); + /** + * Workaround: + * As no ixgbe_enable_sec_rx_path equivalent is + * implemented for tx in the base code, and we are + * not allowed to modify the base code in DPDK, so + * just call the hand-written one directly for now. + */ + ixgbe_enable_sec_tx_path_generic(hw); + + return 0; +} + +int +rte_pmd_ixgbe_macsec_disable(uint16_t port) +{ + struct ixgbe_hw *hw; + struct rte_eth_dev *dev; + uint32_t ctrl; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + dev = &rte_eth_devices[port]; + + if (!is_ixgbe_supported(dev)) + return -ENOTSUP; + + hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + + /* Stop the data paths */ + if (ixgbe_disable_sec_rx_path(hw) != IXGBE_SUCCESS) + return -ENOTSUP; + /** + * Workaround: + * As no ixgbe_disable_sec_rx_path equivalent is + * implemented for tx in the base code, and we are + * not allowed to modify the base code in DPDK, so + * just call the hand-written one directly for now. + * The hardware support has been checked by + * ixgbe_disable_sec_rx_path(). + */ + ixgbe_disable_sec_tx_path_generic(hw); + + /* Disable the TX and RX crypto engines */ + ctrl = IXGBE_READ_REG(hw, IXGBE_SECTXCTRL); + ctrl |= IXGBE_SECTXCTRL_SECTX_DIS; + IXGBE_WRITE_REG(hw, IXGBE_SECTXCTRL, ctrl); + + ctrl = IXGBE_READ_REG(hw, IXGBE_SECRXCTRL); + ctrl |= IXGBE_SECRXCTRL_SECRX_DIS; + IXGBE_WRITE_REG(hw, IXGBE_SECRXCTRL, ctrl); + + /* Disable SA lookup */ + ctrl = IXGBE_READ_REG(hw, IXGBE_LSECTXCTRL); + ctrl &= ~IXGBE_LSECTXCTRL_EN_MASK; + ctrl |= IXGBE_LSECTXCTRL_DISABLE; + IXGBE_WRITE_REG(hw, IXGBE_LSECTXCTRL, ctrl); + + ctrl = IXGBE_READ_REG(hw, IXGBE_LSECRXCTRL); + ctrl &= ~IXGBE_LSECRXCTRL_EN_MASK; + ctrl |= IXGBE_LSECRXCTRL_DISABLE << IXGBE_LSECRXCTRL_EN_SHIFT; + IXGBE_WRITE_REG(hw, IXGBE_LSECRXCTRL, ctrl); + + /* Start the data paths */ + ixgbe_enable_sec_rx_path(hw); + /** + * Workaround: + * As no ixgbe_enable_sec_rx_path equivalent is + * implemented for tx in the base code, and we are + * not allowed to modify the base code in DPDK, so + * just call the hand-written one directly for now. + */ + ixgbe_enable_sec_tx_path_generic(hw); + + return 0; +} + +int +rte_pmd_ixgbe_macsec_config_txsc(uint16_t port, uint8_t *mac) +{ + struct ixgbe_hw *hw; + struct rte_eth_dev *dev; + uint32_t ctrl; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + dev = &rte_eth_devices[port]; + + if (!is_ixgbe_supported(dev)) + return -ENOTSUP; + + hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + + ctrl = mac[0] | (mac[1] << 8) | (mac[2] << 16) | (mac[3] << 24); + IXGBE_WRITE_REG(hw, IXGBE_LSECTXSCL, ctrl); + + ctrl = mac[4] | (mac[5] << 8); + IXGBE_WRITE_REG(hw, IXGBE_LSECTXSCH, ctrl); + + return 0; +} + +int +rte_pmd_ixgbe_macsec_config_rxsc(uint16_t port, uint8_t *mac, uint16_t pi) +{ + struct ixgbe_hw *hw; + struct rte_eth_dev *dev; + uint32_t ctrl; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + dev = &rte_eth_devices[port]; + + if (!is_ixgbe_supported(dev)) + return -ENOTSUP; + + hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + + ctrl = mac[0] | (mac[1] << 8) | (mac[2] << 16) | (mac[3] << 24); + IXGBE_WRITE_REG(hw, IXGBE_LSECRXSCL, ctrl); + + pi = rte_cpu_to_be_16(pi); + ctrl = mac[4] | (mac[5] << 8) | (pi << 16); + IXGBE_WRITE_REG(hw, IXGBE_LSECRXSCH, ctrl); + + return 0; +} + +int +rte_pmd_ixgbe_macsec_select_txsa(uint16_t port, uint8_t idx, uint8_t an, + uint32_t pn, uint8_t *key) +{ + struct ixgbe_hw *hw; + struct rte_eth_dev *dev; + uint32_t ctrl, i; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + dev = &rte_eth_devices[port]; + + if (!is_ixgbe_supported(dev)) + return -ENOTSUP; + + hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + + if (idx != 0 && idx != 1) + return -EINVAL; + + if (an >= 4) + return -EINVAL; + + hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + + /* Set the PN and key */ + pn = rte_cpu_to_be_32(pn); + if (idx == 0) { + IXGBE_WRITE_REG(hw, IXGBE_LSECTXPN0, pn); + + for (i = 0; i < 4; i++) { + ctrl = (key[i * 4 + 0] << 0) | + (key[i * 4 + 1] << 8) | + (key[i * 4 + 2] << 16) | + (key[i * 4 + 3] << 24); + IXGBE_WRITE_REG(hw, IXGBE_LSECTXKEY0(i), ctrl); + } + } else { + IXGBE_WRITE_REG(hw, IXGBE_LSECTXPN1, pn); + + for (i = 0; i < 4; i++) { + ctrl = (key[i * 4 + 0] << 0) | + (key[i * 4 + 1] << 8) | + (key[i * 4 + 2] << 16) | + (key[i * 4 + 3] << 24); + IXGBE_WRITE_REG(hw, IXGBE_LSECTXKEY1(i), ctrl); + } + } + + /* Set AN and select the SA */ + ctrl = (an << idx * 2) | (idx << 4); + IXGBE_WRITE_REG(hw, IXGBE_LSECTXSA, ctrl); + + return 0; +} + +int +rte_pmd_ixgbe_macsec_select_rxsa(uint16_t port, uint8_t idx, uint8_t an, + uint32_t pn, uint8_t *key) +{ + struct ixgbe_hw *hw; + struct rte_eth_dev *dev; + uint32_t ctrl, i; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + dev = &rte_eth_devices[port]; + + if (!is_ixgbe_supported(dev)) + return -ENOTSUP; + + hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + + if (idx != 0 && idx != 1) + return -EINVAL; + + if (an >= 4) + return -EINVAL; + + /* Set the PN */ + pn = rte_cpu_to_be_32(pn); + IXGBE_WRITE_REG(hw, IXGBE_LSECRXPN(idx), pn); + + /* Set the key */ + for (i = 0; i < 4; i++) { + ctrl = (key[i * 4 + 0] << 0) | + (key[i * 4 + 1] << 8) | + (key[i * 4 + 2] << 16) | + (key[i * 4 + 3] << 24); + IXGBE_WRITE_REG(hw, IXGBE_LSECRXKEY(idx, i), ctrl); + } + + /* Set the AN and validate the SA */ + ctrl = an | (1 << 2); + IXGBE_WRITE_REG(hw, IXGBE_LSECRXSA(idx), ctrl); + + return 0; +} + +int +rte_pmd_ixgbe_set_tc_bw_alloc(uint16_t port, + uint8_t tc_num, + uint8_t *bw_weight) +{ + struct rte_eth_dev *dev; + struct ixgbe_dcb_config *dcb_config; + struct ixgbe_dcb_tc_config *tc; + struct rte_eth_conf *eth_conf; + struct ixgbe_bw_conf *bw_conf; + uint8_t i; + uint8_t nb_tcs; + uint16_t sum; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV); + + dev = &rte_eth_devices[port]; + + if (!is_ixgbe_supported(dev)) + return -ENOTSUP; + + if (tc_num > IXGBE_DCB_MAX_TRAFFIC_CLASS) { + PMD_DRV_LOG(ERR, "TCs should be no more than %d.", + IXGBE_DCB_MAX_TRAFFIC_CLASS); + return -EINVAL; + } + + dcb_config = IXGBE_DEV_PRIVATE_TO_DCB_CFG(dev->data->dev_private); + bw_conf = IXGBE_DEV_PRIVATE_TO_BW_CONF(dev->data->dev_private); + eth_conf = &dev->data->dev_conf; + + if (eth_conf->txmode.mq_mode == ETH_MQ_TX_DCB) { + nb_tcs = eth_conf->tx_adv_conf.dcb_tx_conf.nb_tcs; + } else if (eth_conf->txmode.mq_mode == ETH_MQ_TX_VMDQ_DCB) { + if (eth_conf->tx_adv_conf.vmdq_dcb_tx_conf.nb_queue_pools == + ETH_32_POOLS) + nb_tcs = ETH_4_TCS; + else + nb_tcs = ETH_8_TCS; + } else { + nb_tcs = 1; + } + + if (nb_tcs != tc_num) { + PMD_DRV_LOG(ERR, + "Weight should be set for all %d enabled TCs.", + nb_tcs); + return -EINVAL; + } + + sum = 0; + for (i = 0; i < nb_tcs; i++) + sum += bw_weight[i]; + if (sum != 100) { + PMD_DRV_LOG(ERR, + "The summary of the TC weight should be 100."); + return -EINVAL; + } + + for (i = 0; i < nb_tcs; i++) { + tc = &dcb_config->tc_config[i]; + tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent = bw_weight[i]; + } + for (; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) { + tc = &dcb_config->tc_config[i]; + tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent = 0; + } + + bw_conf->tc_num = nb_tcs; + + return 0; +} + +#ifdef RTE_LIBRTE_IXGBE_BYPASS +int +rte_pmd_ixgbe_bypass_init(uint16_t port_id) +{ + struct rte_eth_dev *dev; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); + + dev = &rte_eth_devices[port_id]; + if (!is_ixgbe_supported(dev)) + return -ENOTSUP; + + ixgbe_bypass_init(dev); + return 0; +} + +int +rte_pmd_ixgbe_bypass_state_show(uint16_t port_id, uint32_t *state) +{ + struct rte_eth_dev *dev; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); + + dev = &rte_eth_devices[port_id]; + if (!is_ixgbe_supported(dev)) + return -ENOTSUP; + + return ixgbe_bypass_state_show(dev, state); +} + +int +rte_pmd_ixgbe_bypass_state_set(uint16_t port_id, uint32_t *new_state) +{ + struct rte_eth_dev *dev; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); + + dev = &rte_eth_devices[port_id]; + if (!is_ixgbe_supported(dev)) + return -ENOTSUP; + + return ixgbe_bypass_state_store(dev, new_state); +} + +int +rte_pmd_ixgbe_bypass_event_show(uint16_t port_id, + uint32_t event, + uint32_t *state) +{ + struct rte_eth_dev *dev; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); + + dev = &rte_eth_devices[port_id]; + if (!is_ixgbe_supported(dev)) + return -ENOTSUP; + + return ixgbe_bypass_event_show(dev, event, state); +} + +int +rte_pmd_ixgbe_bypass_event_store(uint16_t port_id, + uint32_t event, + uint32_t state) +{ + struct rte_eth_dev *dev; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); + + dev = &rte_eth_devices[port_id]; + if (!is_ixgbe_supported(dev)) + return -ENOTSUP; + + return ixgbe_bypass_event_store(dev, event, state); +} + +int +rte_pmd_ixgbe_bypass_wd_timeout_store(uint16_t port_id, uint32_t timeout) +{ + struct rte_eth_dev *dev; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); + + dev = &rte_eth_devices[port_id]; + if (!is_ixgbe_supported(dev)) + return -ENOTSUP; + + return ixgbe_bypass_wd_timeout_store(dev, timeout); +} + +int +rte_pmd_ixgbe_bypass_ver_show(uint16_t port_id, uint32_t *ver) +{ + struct rte_eth_dev *dev; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); + + dev = &rte_eth_devices[port_id]; + if (!is_ixgbe_supported(dev)) + return -ENOTSUP; + + return ixgbe_bypass_ver_show(dev, ver); +} + +int +rte_pmd_ixgbe_bypass_wd_timeout_show(uint16_t port_id, uint32_t *wd_timeout) +{ + struct rte_eth_dev *dev; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); + + dev = &rte_eth_devices[port_id]; + if (!is_ixgbe_supported(dev)) + return -ENOTSUP; + + return ixgbe_bypass_wd_timeout_show(dev, wd_timeout); +} + +int +rte_pmd_ixgbe_bypass_wd_reset(uint16_t port_id) +{ + struct rte_eth_dev *dev; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); + + dev = &rte_eth_devices[port_id]; + if (!is_ixgbe_supported(dev)) + return -ENOTSUP; + + return ixgbe_bypass_wd_reset(dev); +} +#endif diff --git a/dpdk/drivers/net/ixgbe/rte_pmd_ixgbe.h b/dpdk/drivers/net/ixgbe/rte_pmd_ixgbe.h new file mode 100644 index 00000000..81b18f87 --- /dev/null +++ b/dpdk/drivers/net/ixgbe/rte_pmd_ixgbe.h @@ -0,0 +1,668 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2016 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** + * @file rte_pmd_ixgbe.h + * ixgbe PMD specific functions. + * + **/ + +#ifndef _PMD_IXGBE_H_ +#define _PMD_IXGBE_H_ + +#include + +/** + * Notify VF when PF link status changes. + * + * @param port + * The port identifier of the Ethernet device. + * @param vf + * VF id. + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-EINVAL) if *vf* invalid. + */ +int rte_pmd_ixgbe_ping_vf(uint16_t port, uint16_t vf); + +/** + * Set the VF MAC address. + * + * @param port + * The port identifier of the Ethernet device. + * @param vf + * VF id. + * @param mac_addr + * VF MAC address. + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-EINVAL) if *vf* or *mac_addr* is invalid. + */ +int rte_pmd_ixgbe_set_vf_mac_addr(uint16_t port, uint16_t vf, + struct ether_addr *mac_addr); + +/** + * Enable/Disable VF VLAN anti spoofing. + * + * @param port + * The port identifier of the Ethernet device. + * @param vf + * VF on which to set VLAN anti spoofing. + * @param on + * 1 - Enable VFs VLAN anti spoofing. + * 0 - Disable VFs VLAN anti spoofing. + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-EINVAL) if bad parameter. + */ +int rte_pmd_ixgbe_set_vf_vlan_anti_spoof(uint16_t port, uint16_t vf, + uint8_t on); + +/** + * Enable/Disable VF MAC anti spoofing. + * + * @param port + * The port identifier of the Ethernet device. + * @param vf + * VF on which to set MAC anti spoofing. + * @param on + * 1 - Enable VFs MAC anti spoofing. + * 0 - Disable VFs MAC anti spoofing. + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-EINVAL) if bad parameter. + */ +int rte_pmd_ixgbe_set_vf_mac_anti_spoof(uint16_t port, uint16_t vf, uint8_t on); + +/** + * Enable/Disable vf vlan insert + * + * @param port + * The port identifier of the Ethernet device. + * @param vf + * ID specifying VF. + * @param vlan_id + * 0 - Disable VF's vlan insert. + * n - Enable; n is inserted as the vlan id. + * + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-EINVAL) if bad parameter. + */ +int rte_pmd_ixgbe_set_vf_vlan_insert(uint16_t port, uint16_t vf, + uint16_t vlan_id); + +/** + * Enable/Disable tx loopback + * + * @param port + * The port identifier of the Ethernet device. + * @param on + * 1 - Enable tx loopback. + * 0 - Disable tx loopback. + * + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-EINVAL) if bad parameter. + */ +int rte_pmd_ixgbe_set_tx_loopback(uint16_t port, uint8_t on); + +/** + * set all queues drop enable bit + * + * @param port + * The port identifier of the Ethernet device. + * @param on + * 1 - set the queue drop enable bit for all pools. + * 0 - reset the queue drop enable bit for all pools. + * + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-EINVAL) if bad parameter. + */ +int rte_pmd_ixgbe_set_all_queues_drop_en(uint16_t port, uint8_t on); + +/** + * set drop enable bit in the VF split rx control register + * + * @param port + * The port identifier of the Ethernet device. + * @param vf + * ID specifying VF. + * @param on + * 1 - set the drop enable bit in the split rx control register. + * 0 - reset the drop enable bit in the split rx control register. + * + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-EINVAL) if bad parameter. + */ + +int rte_pmd_ixgbe_set_vf_split_drop_en(uint16_t port, uint16_t vf, uint8_t on); + +/** + * Enable/Disable vf vlan strip for all queues in a pool + * + * @param port + * The port identifier of the Ethernet device. + * @param vf + * ID specifying VF. + * @param on + * 1 - Enable VF's vlan strip on RX queues. + * 0 - Disable VF's vlan strip on RX queues. + * + * @return + * - (0) if successful. + * - (-ENOTSUP) if hardware doesn't support this feature. + * - (-ENODEV) if *port* invalid. + * - (-EINVAL) if bad parameter. + */ +int +rte_pmd_ixgbe_set_vf_vlan_stripq(uint16_t port, uint16_t vf, uint8_t on); + +/** + * Enable MACsec offload. + * + * @param port + * The port identifier of the Ethernet device. + * @param en + * 1 - Enable encryption (encrypt and add integrity signature). + * 0 - Disable encryption (only add integrity signature). + * @param rp + * 1 - Enable replay protection. + * 0 - Disable replay protection. + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-ENOTSUP) if hardware doesn't support this feature. + */ +int rte_pmd_ixgbe_macsec_enable(uint16_t port, uint8_t en, uint8_t rp); + +/** + * Disable MACsec offload. + * + * @param port + * The port identifier of the Ethernet device. + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-ENOTSUP) if hardware doesn't support this feature. + */ +int rte_pmd_ixgbe_macsec_disable(uint16_t port); + +/** + * Configure Tx SC (Secure Connection). + * + * @param port + * The port identifier of the Ethernet device. + * @param mac + * The MAC address on the local side. + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-ENOTSUP) if hardware doesn't support this feature. + */ +int rte_pmd_ixgbe_macsec_config_txsc(uint16_t port, uint8_t *mac); + +/** + * Configure Rx SC (Secure Connection). + * + * @param port + * The port identifier of the Ethernet device. + * @param mac + * The MAC address on the remote side. + * @param pi + * The PI (port identifier) on the remote side. + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-ENOTSUP) if hardware doesn't support this feature. + */ +int rte_pmd_ixgbe_macsec_config_rxsc(uint16_t port, uint8_t *mac, uint16_t pi); + +/** + * Enable Tx SA (Secure Association). + * + * @param port + * The port identifier of the Ethernet device. + * @param idx + * The SA to be enabled (0 or 1). + * @param an + * The association number on the local side. + * @param pn + * The packet number on the local side. + * @param key + * The key on the local side. + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-ENOTSUP) if hardware doesn't support this feature. + * - (-EINVAL) if bad parameter. + */ +int rte_pmd_ixgbe_macsec_select_txsa(uint16_t port, uint8_t idx, uint8_t an, + uint32_t pn, uint8_t *key); + +/** + * Enable Rx SA (Secure Association). + * + * @param port + * The port identifier of the Ethernet device. + * @param idx + * The SA to be enabled (0 or 1) + * @param an + * The association number on the remote side. + * @param pn + * The packet number on the remote side. + * @param key + * The key on the remote side. + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-ENOTSUP) if hardware doesn't support this feature. + * - (-EINVAL) if bad parameter. + */ +int rte_pmd_ixgbe_macsec_select_rxsa(uint16_t port, uint8_t idx, uint8_t an, + uint32_t pn, uint8_t *key); + +/** +* Set RX L2 Filtering mode of a VF of an Ethernet device. +* +* @param port +* The port identifier of the Ethernet device. +* @param vf +* VF id. +* @param rx_mask +* The RX mode mask, which is one or more of accepting Untagged Packets, +* packets that match the PFUTA table, Broadcast and Multicast Promiscuous. +* ETH_VMDQ_ACCEPT_UNTAG,ETH_VMDQ_ACCEPT_HASH_UC, +* ETH_VMDQ_ACCEPT_BROADCAST and ETH_VMDQ_ACCEPT_MULTICAST will be used +* in rx_mode. +* @param on +* 1 - Enable a VF RX mode. +* 0 - Disable a VF RX mode. +* @return +* - (0) if successful. +* - (-ENOTSUP) if hardware doesn't support. +* - (-ENODEV) if *port_id* invalid. +* - (-EINVAL) if bad parameter. +*/ +int +rte_pmd_ixgbe_set_vf_rxmode(uint16_t port, uint16_t vf, uint16_t rx_mask, + uint8_t on); + +/** +* Enable or disable a VF traffic receive of an Ethernet device. +* +* @param port +* The port identifier of the Ethernet device. +* @param vf +* VF id. +* @param on +* 1 - Enable a VF traffic receive. +* 0 - Disable a VF traffic receive. +* @return +* - (0) if successful. +* - (-ENOTSUP) if hardware doesn't support. +* - (-ENODEV) if *port_id* invalid. +* - (-EINVAL) if bad parameter. +*/ +int +rte_pmd_ixgbe_set_vf_rx(uint16_t port, uint16_t vf, uint8_t on); + +/** +* Enable or disable a VF traffic transmit of the Ethernet device. +* +* @param port +* The port identifier of the Ethernet device. +* @param vf +* VF id. +* @param on +* 1 - Enable a VF traffic transmit. +* 0 - Disable a VF traffic transmit. +* @return +* - (0) if successful. +* - (-ENODEV) if *port_id* invalid. +* - (-ENOTSUP) if hardware doesn't support. +* - (-EINVAL) if bad parameter. +*/ +int +rte_pmd_ixgbe_set_vf_tx(uint16_t port, uint16_t vf, uint8_t on); + +/** +* Enable/Disable hardware VF VLAN filtering by an Ethernet device of +* received VLAN packets tagged with a given VLAN Tag Identifier. +* +* @param port +* The port identifier of the Ethernet device. +* @param vlan +* The VLAN Tag Identifier whose filtering must be enabled or disabled. +* @param vf_mask +* Bitmap listing which VFs participate in the VLAN filtering. +* @param vlan_on +* 1 - Enable VFs VLAN filtering. +* 0 - Disable VFs VLAN filtering. +* @return +* - (0) if successful. +* - (-ENOTSUP) if hardware doesn't support. +* - (-ENODEV) if *port_id* invalid. +* - (-EINVAL) if bad parameter. +*/ +int +rte_pmd_ixgbe_set_vf_vlan_filter(uint16_t port, uint16_t vlan, + uint64_t vf_mask, uint8_t vlan_on); + +/** + * Set the rate limitation for a vf on an Ethernet device. + * + * @param port + * The port identifier of the Ethernet device. + * @param vf + * VF id. + * @param tx_rate + * The tx rate allocated from the total link speed for this VF id. + * @param q_msk + * The queue mask which need to set the rate. + * @return + * - (0) if successful. + * - (-ENOTSUP) if hardware doesn't support this feature. + * - (-ENODEV) if *port_id* invalid. + * - (-EINVAL) if bad parameter. + */ +int rte_pmd_ixgbe_set_vf_rate_limit(uint16_t port, uint16_t vf, + uint16_t tx_rate, uint64_t q_msk); + +/** + * Set all the TCs' bandwidth weight. + * + * The bw_weight means the percentage occupied by the TC. + * It can be taken as the relative min bandwidth setting. + * + * @param port + * The port identifier of the Ethernet device. + * @param tc_num + * Number of TCs. + * @param bw_weight + * An array of relative bandwidth weight for all the TCs. + * The summary of the bw_weight should be 100. + * @return + * - (0) if successful. + * - (-ENODEV) if *port* invalid. + * - (-EINVAL) if bad parameter. + * - (-ENOTSUP) not supported by firmware. + */ +int rte_pmd_ixgbe_set_tc_bw_alloc(uint16_t port, + uint8_t tc_num, + uint8_t *bw_weight); + + +/** + * Initialize bypass logic. This function needs to be called before + * executing any other bypass API. + * + * @param port + * The port identifier of the Ethernet device. + * @return + * - (0) if successful. + * - (-ENOTSUP) if hardware doesn't support. + * - (-EINVAL) if bad parameter. + */ +int rte_pmd_ixgbe_bypass_init(uint16_t port); + +/** + * Return bypass state. + * + * @param port + * The port identifier of the Ethernet device. + * @param state + * The return bypass state. + * - (1) Normal mode + * - (2) Bypass mode + * - (3) Isolate mode + * @return + * - (0) if successful. + * - (-ENOTSUP) if hardware doesn't support. + * - (-EINVAL) if bad parameter. + */ +int rte_pmd_ixgbe_bypass_state_show(uint16_t port, uint32_t *state); + +/** + * Set bypass state + * + * @param port + * The port identifier of the Ethernet device. + * @param new_state + * The current bypass state. + * - (1) Normal mode + * - (2) Bypass mode + * - (3) Isolate mode + * @return + * - (0) if successful. + * - (-ENOTSUP) if hardware doesn't support. + * - (-EINVAL) if bad parameter. + */ +int rte_pmd_ixgbe_bypass_state_set(uint16_t port, uint32_t *new_state); + +/** + * Return bypass state when given event occurs. + * + * @param port + * The port identifier of the Ethernet device. + * @param event + * The bypass event + * - (1) Main power on (power button is pushed) + * - (2) Auxiliary power on (power supply is being plugged) + * - (3) Main power off (system shutdown and power supply is left plugged in) + * - (4) Auxiliary power off (power supply is being unplugged) + * - (5) Display or set the watchdog timer + * @param state + * The bypass state when given event occurred. + * - (1) Normal mode + * - (2) Bypass mode + * - (3) Isolate mode + * @return + * - (0) if successful. + * - (-ENOTSUP) if hardware doesn't support. + * - (-EINVAL) if bad parameter. + */ +int rte_pmd_ixgbe_bypass_event_show(uint16_t port, + uint32_t event, + uint32_t *state); + +/** + * Set bypass state when given event occurs. + * + * @param port + * The port identifier of the Ethernet device. + * @param event + * The bypass event + * - (1) Main power on (power button is pushed) + * - (2) Auxiliary power on (power supply is being plugged) + * - (3) Main power off (system shutdown and power supply is left plugged in) + * - (4) Auxiliary power off (power supply is being unplugged) + * - (5) Display or set the watchdog timer + * @param state + * The assigned state when given event occurs. + * - (1) Normal mode + * - (2) Bypass mode + * - (3) Isolate mode + * @return + * - (0) if successful. + * - (-ENOTSUP) if hardware doesn't support. + * - (-EINVAL) if bad parameter. + */ +int rte_pmd_ixgbe_bypass_event_store(uint16_t port, + uint32_t event, + uint32_t state); + +/** + * Set bypass watchdog timeout count. + * + * @param port + * The port identifier of the Ethernet device. + * @param timeout + * The timeout to be set. + * - (0) 0 seconds (timer is off) + * - (1) 1.5 seconds + * - (2) 2 seconds + * - (3) 3 seconds + * - (4) 4 seconds + * - (5) 8 seconds + * - (6) 16 seconds + * - (7) 32 seconds + * @return + * - (0) if successful. + * - (-ENOTSUP) if hardware doesn't support. + * - (-EINVAL) if bad parameter. + */ +int rte_pmd_ixgbe_bypass_wd_timeout_store(uint16_t port, uint32_t timeout); + +/** + * Get bypass firmware version. + * + * @param port + * The port identifier of the Ethernet device. + * @param ver + * The firmware version + * @return + * - (0) if successful. + * - (-ENOTSUP) if hardware doesn't support. + * - (-EINVAL) if bad parameter. + */ +int rte_pmd_ixgbe_bypass_ver_show(uint16_t port, uint32_t *ver); + +/** + * Return bypass watchdog timeout in seconds + * + * @param port + * The port identifier of the Ethernet device. + * @param wd_timeout + * The return watchdog timeout. "0" represents timer expired + * - (0) 0 seconds (timer is off) + * - (1) 1.5 seconds + * - (2) 2 seconds + * - (3) 3 seconds + * - (4) 4 seconds + * - (5) 8 seconds + * - (6) 16 seconds + * - (7) 32 seconds + * @return + * - (0) if successful. + * - (-ENOTSUP) if hardware doesn't support. + * - (-EINVAL) if bad parameter. + */ +int rte_pmd_ixgbe_bypass_wd_timeout_show(uint16_t port, uint32_t *wd_timeout); + +/** + * Reset bypass watchdog timer + * + * @param port + * The port identifier of the Ethernet device. + * @return + * - (0) if successful. + * - (-ENOTSUP) if hardware doesn't support. + * - (-EINVAL) if bad parameter. + */ +int rte_pmd_ixgbe_bypass_wd_reset(uint16_t port); + + +/** + * Response sent back to ixgbe driver from user app after callback + */ +enum rte_pmd_ixgbe_mb_event_rsp { + RTE_PMD_IXGBE_MB_EVENT_NOOP_ACK, /**< skip mbox request and ACK */ + RTE_PMD_IXGBE_MB_EVENT_NOOP_NACK, /**< skip mbox request and NACK */ + RTE_PMD_IXGBE_MB_EVENT_PROCEED, /**< proceed with mbox request */ + RTE_PMD_IXGBE_MB_EVENT_MAX /**< max value of this enum */ +}; + +/** + * Data sent to the user application when the callback is executed. + */ +struct rte_pmd_ixgbe_mb_event_param { + uint16_t vfid; /**< Virtual Function number */ + uint16_t msg_type; /**< VF to PF message type, defined in ixgbe_mbx.h */ + uint16_t retval; /**< return value */ + void *msg; /**< pointer to message */ +}; +enum { + RTE_PMD_IXGBE_BYPASS_MODE_NONE, + RTE_PMD_IXGBE_BYPASS_MODE_NORMAL, + RTE_PMD_IXGBE_BYPASS_MODE_BYPASS, + RTE_PMD_IXGBE_BYPASS_MODE_ISOLATE, + RTE_PMD_IXGBE_BYPASS_MODE_NUM, +}; + +#define RTE_PMD_IXGBE_BYPASS_MODE_VALID(x) \ + ((x) > RTE_PMD_IXGBE_BYPASS_MODE_NONE && \ + (x) < RTE_PMD_IXGBE_BYPASS_MODE_NUM) + +enum { + RTE_PMD_IXGBE_BYPASS_EVENT_NONE, + RTE_PMD_IXGBE_BYPASS_EVENT_START, + RTE_PMD_IXGBE_BYPASS_EVENT_OS_ON = RTE_PMD_IXGBE_BYPASS_EVENT_START, + RTE_PMD_IXGBE_BYPASS_EVENT_POWER_ON, + RTE_PMD_IXGBE_BYPASS_EVENT_OS_OFF, + RTE_PMD_IXGBE_BYPASS_EVENT_POWER_OFF, + RTE_PMD_IXGBE_BYPASS_EVENT_TIMEOUT, + RTE_PMD_IXGBE_BYPASS_EVENT_NUM +}; + +#define RTE_PMD_IXGBE_BYPASS_EVENT_VALID(x) \ + ((x) > RTE_PMD_IXGBE_BYPASS_EVENT_NONE && \ + (x) < RTE_PMD_IXGBE_BYPASS_MODE_NUM) + +enum { + RTE_PMD_IXGBE_BYPASS_TMT_OFF, /* timeout disabled. */ + RTE_PMD_IXGBE_BYPASS_TMT_1_5_SEC, /* timeout for 1.5 seconds */ + RTE_PMD_IXGBE_BYPASS_TMT_2_SEC, /* timeout for 2 seconds */ + RTE_PMD_IXGBE_BYPASS_TMT_3_SEC, /* timeout for 3 seconds */ + RTE_PMD_IXGBE_BYPASS_TMT_4_SEC, /* timeout for 4 seconds */ + RTE_PMD_IXGBE_BYPASS_TMT_8_SEC, /* timeout for 8 seconds */ + RTE_PMD_IXGBE_BYPASS_TMT_16_SEC, /* timeout for 16 seconds */ + RTE_PMD_IXGBE_BYPASS_TMT_32_SEC, /* timeout for 32 seconds */ + RTE_PMD_IXGBE_BYPASS_TMT_NUM +}; + +#define RTE_PMD_IXGBE_BYPASS_TMT_VALID(x) \ + ((x) == RTE_PMD_IXGBE_BYPASS_TMT_OFF || \ + ((x) > RTE_PMD_IXGBE_BYPASS_TMT_OFF && \ + (x) < RTE_PMD_IXGBE_BYPASS_TMT_NUM)) + +#endif /* _PMD_IXGBE_H_ */ diff --git a/dpdk/drivers/net/ixgbe/rte_pmd_ixgbe_version.map b/dpdk/drivers/net/ixgbe/rte_pmd_ixgbe_version.map index ef353984..bf776742 100644 --- a/dpdk/drivers/net/ixgbe/rte_pmd_ixgbe_version.map +++ b/dpdk/drivers/net/ixgbe/rte_pmd_ixgbe_version.map @@ -2,3 +2,53 @@ DPDK_2.0 { local: *; }; + +DPDK_16.11 { + global: + + rte_pmd_ixgbe_set_all_queues_drop_en; + rte_pmd_ixgbe_set_tx_loopback; + rte_pmd_ixgbe_set_vf_mac_addr; + rte_pmd_ixgbe_set_vf_mac_anti_spoof; + rte_pmd_ixgbe_set_vf_split_drop_en; + rte_pmd_ixgbe_set_vf_vlan_anti_spoof; + rte_pmd_ixgbe_set_vf_vlan_insert; + rte_pmd_ixgbe_set_vf_vlan_stripq; +} DPDK_2.0; + +DPDK_17.02 { + global: + + rte_pmd_ixgbe_macsec_config_rxsc; + rte_pmd_ixgbe_macsec_config_txsc; + rte_pmd_ixgbe_macsec_disable; + rte_pmd_ixgbe_macsec_enable; + rte_pmd_ixgbe_macsec_select_rxsa; + rte_pmd_ixgbe_macsec_select_txsa; + rte_pmd_ixgbe_set_vf_rate_limit; + rte_pmd_ixgbe_set_vf_rx; + rte_pmd_ixgbe_set_vf_rxmode; + rte_pmd_ixgbe_set_vf_tx; + rte_pmd_ixgbe_set_vf_vlan_filter; +} DPDK_16.11; + +DPDK_17.05 { + global: + + rte_pmd_ixgbe_ping_vf; + rte_pmd_ixgbe_set_tc_bw_alloc; +} DPDK_17.02; + +DPDK_17.08 { + global: + + rte_pmd_ixgbe_bypass_event_show; + rte_pmd_ixgbe_bypass_event_store; + rte_pmd_ixgbe_bypass_init; + rte_pmd_ixgbe_bypass_state_set; + rte_pmd_ixgbe_bypass_state_show; + rte_pmd_ixgbe_bypass_ver_show; + rte_pmd_ixgbe_bypass_wd_reset; + rte_pmd_ixgbe_bypass_wd_timeout_show; + rte_pmd_ixgbe_bypass_wd_timeout_store; +} DPDK_17.05; diff --git a/dpdk/drivers/net/kni/Makefile b/dpdk/drivers/net/kni/Makefile new file mode 100644 index 00000000..a3f51f92 --- /dev/null +++ b/dpdk/drivers/net/kni/Makefile @@ -0,0 +1,59 @@ +# BSD LICENSE +# +# Copyright(c) 2017 Intel Corporation. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +include $(RTE_SDK)/mk/rte.vars.mk + +# +# library name +# +LIB = librte_pmd_kni.a + +CFLAGS += -O3 +CFLAGS += $(WERROR_FLAGS) +LDLIBS += -lpthread +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs -lrte_kni +LDLIBS += -lrte_bus_vdev + +EXPORT_MAP := rte_pmd_kni_version.map + +LIBABIVER := 1 + +# +# all source are stored in SRCS-y +# +SRCS-$(CONFIG_RTE_LIBRTE_PMD_KNI) += rte_eth_kni.c + +# +# Export include files +# +SYMLINK-y-include += + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/net/kni/rte_eth_kni.c b/dpdk/drivers/net/kni/rte_eth_kni.c new file mode 100644 index 00000000..c1a2ea51 --- /dev/null +++ b/dpdk/drivers/net/kni/rte_eth_kni.c @@ -0,0 +1,508 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +/* Only single queue supported */ +#define KNI_MAX_QUEUE_PER_PORT 1 + +#define MAX_PACKET_SZ 2048 +#define MAX_KNI_PORTS 8 + +#define ETH_KNI_NO_REQUEST_THREAD_ARG "no_request_thread" +static const char * const valid_arguments[] = { + ETH_KNI_NO_REQUEST_THREAD_ARG, + NULL +}; + +struct eth_kni_args { + int no_request_thread; +}; + +struct pmd_queue_stats { + uint64_t pkts; + uint64_t bytes; + uint64_t err_pkts; +}; + +struct pmd_queue { + struct pmd_internals *internals; + struct rte_mempool *mb_pool; + + struct pmd_queue_stats rx; + struct pmd_queue_stats tx; +}; + +struct pmd_internals { + struct rte_kni *kni; + int is_kni_started; + + pthread_t thread; + int stop_thread; + int no_request_thread; + + struct ether_addr eth_addr; + + struct pmd_queue rx_queues[KNI_MAX_QUEUE_PER_PORT]; + struct pmd_queue tx_queues[KNI_MAX_QUEUE_PER_PORT]; +}; + +static const struct rte_eth_link pmd_link = { + .link_speed = ETH_SPEED_NUM_10G, + .link_duplex = ETH_LINK_FULL_DUPLEX, + .link_status = ETH_LINK_DOWN, + .link_autoneg = ETH_LINK_AUTONEG, +}; +static int is_kni_initialized; + +static uint16_t +eth_kni_rx(void *q, struct rte_mbuf **bufs, uint16_t nb_bufs) +{ + struct pmd_queue *kni_q = q; + struct rte_kni *kni = kni_q->internals->kni; + uint16_t nb_pkts; + + nb_pkts = rte_kni_rx_burst(kni, bufs, nb_bufs); + + kni_q->rx.pkts += nb_pkts; + kni_q->rx.err_pkts += nb_bufs - nb_pkts; + + return nb_pkts; +} + +static uint16_t +eth_kni_tx(void *q, struct rte_mbuf **bufs, uint16_t nb_bufs) +{ + struct pmd_queue *kni_q = q; + struct rte_kni *kni = kni_q->internals->kni; + uint16_t nb_pkts; + + nb_pkts = rte_kni_tx_burst(kni, bufs, nb_bufs); + + kni_q->tx.pkts += nb_pkts; + kni_q->tx.err_pkts += nb_bufs - nb_pkts; + + return nb_pkts; +} + +static void * +kni_handle_request(void *param) +{ + struct pmd_internals *internals = param; +#define MS 1000 + + while (!internals->stop_thread) { + rte_kni_handle_request(internals->kni); + usleep(500 * MS); + } + + return param; +} + +static int +eth_kni_start(struct rte_eth_dev *dev) +{ + struct pmd_internals *internals = dev->data->dev_private; + uint16_t port_id = dev->data->port_id; + struct rte_mempool *mb_pool; + struct rte_kni_conf conf; + const char *name = dev->device->name + 4; /* remove net_ */ + + snprintf(conf.name, RTE_KNI_NAMESIZE, "%s", name); + conf.force_bind = 0; + conf.group_id = port_id; + conf.mbuf_size = MAX_PACKET_SZ; + mb_pool = internals->rx_queues[0].mb_pool; + + internals->kni = rte_kni_alloc(mb_pool, &conf, NULL); + if (internals->kni == NULL) { + RTE_LOG(ERR, PMD, + "Fail to create kni interface for port: %d\n", + port_id); + return -1; + } + + return 0; +} + +static int +eth_kni_dev_start(struct rte_eth_dev *dev) +{ + struct pmd_internals *internals = dev->data->dev_private; + int ret; + + if (internals->is_kni_started == 0) { + ret = eth_kni_start(dev); + if (ret) + return -1; + internals->is_kni_started = 1; + } + + if (internals->no_request_thread == 0) { + ret = pthread_create(&internals->thread, NULL, + kni_handle_request, internals); + if (ret) { + RTE_LOG(ERR, PMD, + "Fail to create kni request thread\n"); + return -1; + } + } + + dev->data->dev_link.link_status = 1; + + return 0; +} + +static void +eth_kni_dev_stop(struct rte_eth_dev *dev) +{ + struct pmd_internals *internals = dev->data->dev_private; + int ret; + + if (internals->no_request_thread == 0) { + internals->stop_thread = 1; + + ret = pthread_cancel(internals->thread); + if (ret) + RTE_LOG(ERR, PMD, "Can't cancel the thread\n"); + + ret = pthread_join(internals->thread, NULL); + if (ret) + RTE_LOG(ERR, PMD, "Can't join the thread\n"); + + internals->stop_thread = 0; + } + + dev->data->dev_link.link_status = 0; +} + +static int +eth_kni_dev_configure(struct rte_eth_dev *dev __rte_unused) +{ + return 0; +} + +static void +eth_kni_dev_info(struct rte_eth_dev *dev __rte_unused, + struct rte_eth_dev_info *dev_info) +{ + dev_info->max_mac_addrs = 1; + dev_info->max_rx_pktlen = UINT32_MAX; + dev_info->max_rx_queues = KNI_MAX_QUEUE_PER_PORT; + dev_info->max_tx_queues = KNI_MAX_QUEUE_PER_PORT; + dev_info->min_rx_bufsize = 0; + dev_info->pci_dev = NULL; +} + +static int +eth_kni_rx_queue_setup(struct rte_eth_dev *dev, + uint16_t rx_queue_id, + uint16_t nb_rx_desc __rte_unused, + unsigned int socket_id __rte_unused, + const struct rte_eth_rxconf *rx_conf __rte_unused, + struct rte_mempool *mb_pool) +{ + struct pmd_internals *internals = dev->data->dev_private; + struct pmd_queue *q; + + q = &internals->rx_queues[rx_queue_id]; + q->internals = internals; + q->mb_pool = mb_pool; + + dev->data->rx_queues[rx_queue_id] = q; + + return 0; +} + +static int +eth_kni_tx_queue_setup(struct rte_eth_dev *dev, + uint16_t tx_queue_id, + uint16_t nb_tx_desc __rte_unused, + unsigned int socket_id __rte_unused, + const struct rte_eth_txconf *tx_conf __rte_unused) +{ + struct pmd_internals *internals = dev->data->dev_private; + struct pmd_queue *q; + + q = &internals->tx_queues[tx_queue_id]; + q->internals = internals; + + dev->data->tx_queues[tx_queue_id] = q; + + return 0; +} + +static void +eth_kni_queue_release(void *q __rte_unused) +{ +} + +static int +eth_kni_link_update(struct rte_eth_dev *dev __rte_unused, + int wait_to_complete __rte_unused) +{ + return 0; +} + +static int +eth_kni_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) +{ + unsigned long rx_packets_total = 0, rx_bytes_total = 0; + unsigned long tx_packets_total = 0, tx_bytes_total = 0; + struct rte_eth_dev_data *data = dev->data; + unsigned long tx_packets_err_total = 0; + unsigned int i, num_stats; + struct pmd_queue *q; + + num_stats = RTE_MIN((unsigned int)RTE_ETHDEV_QUEUE_STAT_CNTRS, + data->nb_rx_queues); + for (i = 0; i < num_stats; i++) { + q = data->rx_queues[i]; + stats->q_ipackets[i] = q->rx.pkts; + stats->q_ibytes[i] = q->rx.bytes; + rx_packets_total += stats->q_ipackets[i]; + rx_bytes_total += stats->q_ibytes[i]; + } + + num_stats = RTE_MIN((unsigned int)RTE_ETHDEV_QUEUE_STAT_CNTRS, + data->nb_tx_queues); + for (i = 0; i < num_stats; i++) { + q = data->tx_queues[i]; + stats->q_opackets[i] = q->tx.pkts; + stats->q_obytes[i] = q->tx.bytes; + stats->q_errors[i] = q->tx.err_pkts; + tx_packets_total += stats->q_opackets[i]; + tx_bytes_total += stats->q_obytes[i]; + tx_packets_err_total += stats->q_errors[i]; + } + + stats->ipackets = rx_packets_total; + stats->ibytes = rx_bytes_total; + stats->opackets = tx_packets_total; + stats->obytes = tx_bytes_total; + stats->oerrors = tx_packets_err_total; + + return 0; +} + +static void +eth_kni_stats_reset(struct rte_eth_dev *dev) +{ + struct rte_eth_dev_data *data = dev->data; + struct pmd_queue *q; + unsigned int i; + + for (i = 0; i < data->nb_rx_queues; i++) { + q = data->rx_queues[i]; + q->rx.pkts = 0; + q->rx.bytes = 0; + } + for (i = 0; i < data->nb_tx_queues; i++) { + q = data->tx_queues[i]; + q->tx.pkts = 0; + q->tx.bytes = 0; + q->tx.err_pkts = 0; + } +} + +static const struct eth_dev_ops eth_kni_ops = { + .dev_start = eth_kni_dev_start, + .dev_stop = eth_kni_dev_stop, + .dev_configure = eth_kni_dev_configure, + .dev_infos_get = eth_kni_dev_info, + .rx_queue_setup = eth_kni_rx_queue_setup, + .tx_queue_setup = eth_kni_tx_queue_setup, + .rx_queue_release = eth_kni_queue_release, + .tx_queue_release = eth_kni_queue_release, + .link_update = eth_kni_link_update, + .stats_get = eth_kni_stats_get, + .stats_reset = eth_kni_stats_reset, +}; + +static struct rte_eth_dev * +eth_kni_create(struct rte_vdev_device *vdev, + struct eth_kni_args *args, + unsigned int numa_node) +{ + struct pmd_internals *internals; + struct rte_eth_dev_data *data; + struct rte_eth_dev *eth_dev; + const char *name; + + RTE_LOG(INFO, PMD, "Creating kni ethdev on numa socket %u\n", + numa_node); + + name = rte_vdev_device_name(vdev); + data = rte_zmalloc_socket(name, sizeof(*data), 0, numa_node); + if (data == NULL) + return NULL; + + /* reserve an ethdev entry */ + eth_dev = rte_eth_vdev_allocate(vdev, sizeof(*internals)); + if (eth_dev == NULL) { + rte_free(data); + return NULL; + } + + internals = eth_dev->data->dev_private; + rte_memcpy(data, eth_dev->data, sizeof(*data)); + data->nb_rx_queues = 1; + data->nb_tx_queues = 1; + data->dev_link = pmd_link; + data->mac_addrs = &internals->eth_addr; + + eth_random_addr(internals->eth_addr.addr_bytes); + + eth_dev->data = data; + eth_dev->dev_ops = ð_kni_ops; + + internals->no_request_thread = args->no_request_thread; + + return eth_dev; +} + +static int +kni_init(void) +{ + if (is_kni_initialized == 0) + rte_kni_init(MAX_KNI_PORTS); + + is_kni_initialized++; + + return 0; +} + +static int +eth_kni_kvargs_process(struct eth_kni_args *args, const char *params) +{ + struct rte_kvargs *kvlist; + + kvlist = rte_kvargs_parse(params, valid_arguments); + if (kvlist == NULL) + return -1; + + memset(args, 0, sizeof(struct eth_kni_args)); + + if (rte_kvargs_count(kvlist, ETH_KNI_NO_REQUEST_THREAD_ARG) == 1) + args->no_request_thread = 1; + + rte_kvargs_free(kvlist); + + return 0; +} + +static int +eth_kni_probe(struct rte_vdev_device *vdev) +{ + struct rte_eth_dev *eth_dev; + struct eth_kni_args args; + const char *name; + const char *params; + int ret; + + name = rte_vdev_device_name(vdev); + params = rte_vdev_device_args(vdev); + RTE_LOG(INFO, PMD, "Initializing eth_kni for %s\n", name); + + ret = eth_kni_kvargs_process(&args, params); + if (ret < 0) + return ret; + + ret = kni_init(); + if (ret < 0) + return ret; + + eth_dev = eth_kni_create(vdev, &args, rte_socket_id()); + if (eth_dev == NULL) + goto kni_uninit; + + eth_dev->rx_pkt_burst = eth_kni_rx; + eth_dev->tx_pkt_burst = eth_kni_tx; + + return 0; + +kni_uninit: + is_kni_initialized--; + if (is_kni_initialized == 0) + rte_kni_close(); + return -1; +} + +static int +eth_kni_remove(struct rte_vdev_device *vdev) +{ + struct rte_eth_dev *eth_dev; + struct pmd_internals *internals; + const char *name; + + name = rte_vdev_device_name(vdev); + RTE_LOG(INFO, PMD, "Un-Initializing eth_kni for %s\n", name); + + /* find the ethdev entry */ + eth_dev = rte_eth_dev_allocated(name); + if (eth_dev == NULL) + return -1; + + eth_kni_dev_stop(eth_dev); + + internals = eth_dev->data->dev_private; + rte_kni_release(internals->kni); + + rte_free(internals); + rte_free(eth_dev->data); + + rte_eth_dev_release_port(eth_dev); + + is_kni_initialized--; + if (is_kni_initialized == 0) + rte_kni_close(); + + return 0; +} + +static struct rte_vdev_driver eth_kni_drv = { + .probe = eth_kni_probe, + .remove = eth_kni_remove, +}; + +RTE_PMD_REGISTER_VDEV(net_kni, eth_kni_drv); +RTE_PMD_REGISTER_PARAM_STRING(net_kni, ETH_KNI_NO_REQUEST_THREAD_ARG "="); diff --git a/dpdk/drivers/net/kni/rte_pmd_kni_version.map b/dpdk/drivers/net/kni/rte_pmd_kni_version.map new file mode 100644 index 00000000..8591cc0b --- /dev/null +++ b/dpdk/drivers/net/kni/rte_pmd_kni_version.map @@ -0,0 +1,4 @@ +DPDK_17.05 { + + local: *; +}; diff --git a/dpdk/drivers/net/liquidio/Makefile b/dpdk/drivers/net/liquidio/Makefile new file mode 100644 index 00000000..5110099f --- /dev/null +++ b/dpdk/drivers/net/liquidio/Makefile @@ -0,0 +1,61 @@ +# +# BSD LICENSE +# +# Copyright(c) 2017 Cavium, Inc.. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Cavium, Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +include $(RTE_SDK)/mk/rte.vars.mk + +# +# library name +# +LIB = librte_pmd_lio.a + +CFLAGS += -O3 +CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR)/base -I$(SRCDIR) +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs +LDLIBS += -lrte_bus_pci + +EXPORT_MAP := rte_pmd_lio_version.map + +LIBABIVER := 1 + +VPATH += $(RTE_SDK)/drivers/net/liquidio/base + +# +# all source are stored in SRCS-y +# +SRCS-$(CONFIG_RTE_LIBRTE_LIO_PMD) += lio_ethdev.c +SRCS-$(CONFIG_RTE_LIBRTE_LIO_PMD) += lio_rxtx.c +SRCS-$(CONFIG_RTE_LIBRTE_LIO_PMD) += lio_23xx_vf.c +SRCS-$(CONFIG_RTE_LIBRTE_LIO_PMD) += lio_mbox.c + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/net/liquidio/base/lio_23xx_reg.h b/dpdk/drivers/net/liquidio/base/lio_23xx_reg.h new file mode 100644 index 00000000..794bc2ca --- /dev/null +++ b/dpdk/drivers/net/liquidio/base/lio_23xx_reg.h @@ -0,0 +1,194 @@ +/* + * BSD LICENSE + * + * Copyright(c) 2017 Cavium, Inc.. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _LIO_23XX_REG_H_ +#define _LIO_23XX_REG_H_ + +/* ###################### REQUEST QUEUE ######################### */ + +/* 64 registers for Input Queues Start Addr - SLI_PKT(0..63)_INSTR_BADDR */ +#define CN23XX_SLI_PKT_INSTR_BADDR_START64 0x10010 + +/* 64 registers for Input Doorbell - SLI_PKT(0..63)_INSTR_BAOFF_DBELL */ +#define CN23XX_SLI_PKT_INSTR_BADDR_DBELL_START 0x10020 + +/* 64 registers for Input Queue size - SLI_PKT(0..63)_INSTR_FIFO_RSIZE */ +#define CN23XX_SLI_PKT_INSTR_FIFO_RSIZE_START 0x10030 + +/* 64 registers for Input Queue Instr Count - SLI_PKT_IN_DONE(0..63)_CNTS */ +#define CN23XX_SLI_PKT_IN_DONE_CNTS_START64 0x10040 + +/* 64 registers (64-bit) - ES, RO, NS, Arbitration for Input Queue Data & + * gather list fetches. SLI_PKT(0..63)_INPUT_CONTROL. + */ +#define CN23XX_SLI_PKT_INPUT_CONTROL_START64 0x10000 + +/* ------- Request Queue Macros --------- */ + +/* Each Input Queue register is at a 16-byte Offset in BAR0 */ +#define CN23XX_IQ_OFFSET 0x20000 + +#define CN23XX_SLI_IQ_PKT_CONTROL64(iq) \ + (CN23XX_SLI_PKT_INPUT_CONTROL_START64 + ((iq) * CN23XX_IQ_OFFSET)) + +#define CN23XX_SLI_IQ_BASE_ADDR64(iq) \ + (CN23XX_SLI_PKT_INSTR_BADDR_START64 + ((iq) * CN23XX_IQ_OFFSET)) + +#define CN23XX_SLI_IQ_SIZE(iq) \ + (CN23XX_SLI_PKT_INSTR_FIFO_RSIZE_START + ((iq) * CN23XX_IQ_OFFSET)) + +#define CN23XX_SLI_IQ_DOORBELL(iq) \ + (CN23XX_SLI_PKT_INSTR_BADDR_DBELL_START + ((iq) * CN23XX_IQ_OFFSET)) + +#define CN23XX_SLI_IQ_INSTR_COUNT64(iq) \ + (CN23XX_SLI_PKT_IN_DONE_CNTS_START64 + ((iq) * CN23XX_IQ_OFFSET)) + +/* Number of instructions to be read in one MAC read request. + * setting to Max value(4) + */ +#define CN23XX_PKT_INPUT_CTL_RDSIZE (3 << 25) +#define CN23XX_PKT_INPUT_CTL_IS_64B (1 << 24) +#define CN23XX_PKT_INPUT_CTL_RST (1 << 23) +#define CN23XX_PKT_INPUT_CTL_QUIET (1 << 28) +#define CN23XX_PKT_INPUT_CTL_RING_ENB (1 << 22) +#define CN23XX_PKT_INPUT_CTL_DATA_ES_64B_SWAP (1 << 6) +#define CN23XX_PKT_INPUT_CTL_USE_CSR (1 << 4) +#define CN23XX_PKT_INPUT_CTL_GATHER_ES_64B_SWAP (2) + +/* These bits[47:44] select the Physical function number within the MAC */ +#define CN23XX_PKT_INPUT_CTL_PF_NUM_POS 45 +/* These bits[43:32] select the function number within the PF */ +#define CN23XX_PKT_INPUT_CTL_VF_NUM_POS 32 + +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN +#define CN23XX_PKT_INPUT_CTL_MASK \ + (CN23XX_PKT_INPUT_CTL_RDSIZE | \ + CN23XX_PKT_INPUT_CTL_DATA_ES_64B_SWAP | \ + CN23XX_PKT_INPUT_CTL_USE_CSR) +#elif RTE_BYTE_ORDER == RTE_BIG_ENDIAN +#define CN23XX_PKT_INPUT_CTL_MASK \ + (CN23XX_PKT_INPUT_CTL_RDSIZE | \ + CN23XX_PKT_INPUT_CTL_DATA_ES_64B_SWAP | \ + CN23XX_PKT_INPUT_CTL_USE_CSR | \ + CN23XX_PKT_INPUT_CTL_GATHER_ES_64B_SWAP) +#endif + +/* ############################ OUTPUT QUEUE ######################### */ + +/* 64 registers for Output queue control - SLI_PKT(0..63)_OUTPUT_CONTROL */ +#define CN23XX_SLI_PKT_OUTPUT_CONTROL_START 0x10050 + +/* 64 registers for Output queue buffer and info size + * SLI_PKT(0..63)_OUT_SIZE + */ +#define CN23XX_SLI_PKT_OUT_SIZE 0x10060 + +/* 64 registers for Output Queue Start Addr - SLI_PKT(0..63)_SLIST_BADDR */ +#define CN23XX_SLI_SLIST_BADDR_START64 0x10070 + +/* 64 registers for Output Queue Packet Credits + * SLI_PKT(0..63)_SLIST_BAOFF_DBELL + */ +#define CN23XX_SLI_PKT_SLIST_BAOFF_DBELL_START 0x10080 + +/* 64 registers for Output Queue size - SLI_PKT(0..63)_SLIST_FIFO_RSIZE */ +#define CN23XX_SLI_PKT_SLIST_FIFO_RSIZE_START 0x10090 + +/* 64 registers for Output Queue Packet Count - SLI_PKT(0..63)_CNTS */ +#define CN23XX_SLI_PKT_CNTS_START 0x100B0 + +/* Each Output Queue register is at a 16-byte Offset in BAR0 */ +#define CN23XX_OQ_OFFSET 0x20000 + +/* ------- Output Queue Macros --------- */ + +#define CN23XX_SLI_OQ_PKT_CONTROL(oq) \ + (CN23XX_SLI_PKT_OUTPUT_CONTROL_START + ((oq) * CN23XX_OQ_OFFSET)) + +#define CN23XX_SLI_OQ_BASE_ADDR64(oq) \ + (CN23XX_SLI_SLIST_BADDR_START64 + ((oq) * CN23XX_OQ_OFFSET)) + +#define CN23XX_SLI_OQ_SIZE(oq) \ + (CN23XX_SLI_PKT_SLIST_FIFO_RSIZE_START + ((oq) * CN23XX_OQ_OFFSET)) + +#define CN23XX_SLI_OQ_BUFF_INFO_SIZE(oq) \ + (CN23XX_SLI_PKT_OUT_SIZE + ((oq) * CN23XX_OQ_OFFSET)) + +#define CN23XX_SLI_OQ_PKTS_SENT(oq) \ + (CN23XX_SLI_PKT_CNTS_START + ((oq) * CN23XX_OQ_OFFSET)) + +#define CN23XX_SLI_OQ_PKTS_CREDIT(oq) \ + (CN23XX_SLI_PKT_SLIST_BAOFF_DBELL_START + ((oq) * CN23XX_OQ_OFFSET)) + +/* ------------------ Masks ---------------- */ +#define CN23XX_PKT_OUTPUT_CTL_IPTR (1 << 11) +#define CN23XX_PKT_OUTPUT_CTL_ES (1 << 9) +#define CN23XX_PKT_OUTPUT_CTL_NSR (1 << 8) +#define CN23XX_PKT_OUTPUT_CTL_ROR (1 << 7) +#define CN23XX_PKT_OUTPUT_CTL_DPTR (1 << 6) +#define CN23XX_PKT_OUTPUT_CTL_BMODE (1 << 5) +#define CN23XX_PKT_OUTPUT_CTL_ES_P (1 << 3) +#define CN23XX_PKT_OUTPUT_CTL_NSR_P (1 << 2) +#define CN23XX_PKT_OUTPUT_CTL_ROR_P (1 << 1) +#define CN23XX_PKT_OUTPUT_CTL_RING_ENB (1 << 0) + +/* Rings per Virtual Function [RO] */ +#define CN23XX_PKT_INPUT_CTL_RPVF_MASK 0x3F +#define CN23XX_PKT_INPUT_CTL_RPVF_POS 48 + +/* These bits[47:44][RO] give the Physical function + * number info within the MAC + */ +#define CN23XX_PKT_INPUT_CTL_PF_NUM_MASK 0x7 + +/* These bits[43:32][RO] give the virtual function + * number info within the PF + */ +#define CN23XX_PKT_INPUT_CTL_VF_NUM_MASK 0x1FFF + +/* ######################### Mailbox Reg Macros ######################## */ +#define CN23XX_SLI_PKT_PF_VF_MBOX_SIG_START 0x10200 +#define CN23XX_VF_SLI_PKT_MBOX_INT_START 0x10210 + +#define CN23XX_SLI_MBOX_OFFSET 0x20000 +#define CN23XX_SLI_MBOX_SIG_IDX_OFFSET 0x8 + +#define CN23XX_SLI_PKT_PF_VF_MBOX_SIG(q, idx) \ + (CN23XX_SLI_PKT_PF_VF_MBOX_SIG_START + \ + ((q) * CN23XX_SLI_MBOX_OFFSET + \ + (idx) * CN23XX_SLI_MBOX_SIG_IDX_OFFSET)) + +#define CN23XX_VF_SLI_PKT_MBOX_INT(q) \ + (CN23XX_VF_SLI_PKT_MBOX_INT_START + ((q) * CN23XX_SLI_MBOX_OFFSET)) + +#endif /* _LIO_23XX_REG_H_ */ diff --git a/dpdk/drivers/net/liquidio/base/lio_23xx_vf.c b/dpdk/drivers/net/liquidio/base/lio_23xx_vf.c new file mode 100644 index 00000000..e30c20dc --- /dev/null +++ b/dpdk/drivers/net/liquidio/base/lio_23xx_vf.c @@ -0,0 +1,588 @@ +/* + * BSD LICENSE + * + * Copyright(c) 2017 Cavium, Inc.. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +#include +#include +#include + +#include "lio_logs.h" +#include "lio_23xx_vf.h" +#include "lio_23xx_reg.h" +#include "lio_mbox.h" + +static int +cn23xx_vf_reset_io_queues(struct lio_device *lio_dev, uint32_t num_queues) +{ + uint32_t loop = CN23XX_VF_BUSY_READING_REG_LOOP_COUNT; + uint64_t d64, q_no; + int ret_val = 0; + + PMD_INIT_FUNC_TRACE(); + + for (q_no = 0; q_no < num_queues; q_no++) { + /* set RST bit to 1. This bit applies to both IQ and OQ */ + d64 = lio_read_csr64(lio_dev, + CN23XX_SLI_IQ_PKT_CONTROL64(q_no)); + d64 = d64 | CN23XX_PKT_INPUT_CTL_RST; + lio_write_csr64(lio_dev, CN23XX_SLI_IQ_PKT_CONTROL64(q_no), + d64); + } + + /* wait until the RST bit is clear or the RST and QUIET bits are set */ + for (q_no = 0; q_no < num_queues; q_no++) { + volatile uint64_t reg_val; + + reg_val = lio_read_csr64(lio_dev, + CN23XX_SLI_IQ_PKT_CONTROL64(q_no)); + while ((reg_val & CN23XX_PKT_INPUT_CTL_RST) && + !(reg_val & CN23XX_PKT_INPUT_CTL_QUIET) && + loop) { + reg_val = lio_read_csr64( + lio_dev, + CN23XX_SLI_IQ_PKT_CONTROL64(q_no)); + loop = loop - 1; + } + + if (loop == 0) { + lio_dev_err(lio_dev, + "clearing the reset reg failed or setting the quiet reg failed for qno: %lu\n", + (unsigned long)q_no); + return -1; + } + + reg_val = reg_val & ~CN23XX_PKT_INPUT_CTL_RST; + lio_write_csr64(lio_dev, CN23XX_SLI_IQ_PKT_CONTROL64(q_no), + reg_val); + + reg_val = lio_read_csr64( + lio_dev, CN23XX_SLI_IQ_PKT_CONTROL64(q_no)); + if (reg_val & CN23XX_PKT_INPUT_CTL_RST) { + lio_dev_err(lio_dev, + "clearing the reset failed for qno: %lu\n", + (unsigned long)q_no); + ret_val = -1; + } + } + + return ret_val; +} + +static int +cn23xx_vf_setup_global_input_regs(struct lio_device *lio_dev) +{ + uint64_t q_no; + uint64_t d64; + + PMD_INIT_FUNC_TRACE(); + + if (cn23xx_vf_reset_io_queues(lio_dev, + lio_dev->sriov_info.rings_per_vf)) + return -1; + + for (q_no = 0; q_no < (lio_dev->sriov_info.rings_per_vf); q_no++) { + lio_write_csr64(lio_dev, CN23XX_SLI_IQ_DOORBELL(q_no), + 0xFFFFFFFF); + + d64 = lio_read_csr64(lio_dev, + CN23XX_SLI_IQ_INSTR_COUNT64(q_no)); + + d64 &= 0xEFFFFFFFFFFFFFFFL; + + lio_write_csr64(lio_dev, CN23XX_SLI_IQ_INSTR_COUNT64(q_no), + d64); + + /* Select ES, RO, NS, RDSIZE,DPTR Fomat#0 for + * the Input Queues + */ + lio_write_csr64(lio_dev, CN23XX_SLI_IQ_PKT_CONTROL64(q_no), + CN23XX_PKT_INPUT_CTL_MASK); + } + + return 0; +} + +static void +cn23xx_vf_setup_global_output_regs(struct lio_device *lio_dev) +{ + uint32_t reg_val; + uint32_t q_no; + + PMD_INIT_FUNC_TRACE(); + + for (q_no = 0; q_no < lio_dev->sriov_info.rings_per_vf; q_no++) { + lio_write_csr(lio_dev, CN23XX_SLI_OQ_PKTS_CREDIT(q_no), + 0xFFFFFFFF); + + reg_val = + lio_read_csr(lio_dev, CN23XX_SLI_OQ_PKTS_SENT(q_no)); + + reg_val &= 0xEFFFFFFFFFFFFFFFL; + + reg_val = + lio_read_csr(lio_dev, CN23XX_SLI_OQ_PKT_CONTROL(q_no)); + + /* set IPTR & DPTR */ + reg_val |= + (CN23XX_PKT_OUTPUT_CTL_IPTR | CN23XX_PKT_OUTPUT_CTL_DPTR); + + /* reset BMODE */ + reg_val &= ~(CN23XX_PKT_OUTPUT_CTL_BMODE); + + /* No Relaxed Ordering, No Snoop, 64-bit Byte swap + * for Output Queue Scatter List + * reset ROR_P, NSR_P + */ + reg_val &= ~(CN23XX_PKT_OUTPUT_CTL_ROR_P); + reg_val &= ~(CN23XX_PKT_OUTPUT_CTL_NSR_P); + +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + reg_val &= ~(CN23XX_PKT_OUTPUT_CTL_ES_P); +#elif RTE_BYTE_ORDER == RTE_BIG_ENDIAN + reg_val |= (CN23XX_PKT_OUTPUT_CTL_ES_P); +#endif + /* No Relaxed Ordering, No Snoop, 64-bit Byte swap + * for Output Queue Data + * reset ROR, NSR + */ + reg_val &= ~(CN23XX_PKT_OUTPUT_CTL_ROR); + reg_val &= ~(CN23XX_PKT_OUTPUT_CTL_NSR); + /* set the ES bit */ + reg_val |= (CN23XX_PKT_OUTPUT_CTL_ES); + + /* write all the selected settings */ + lio_write_csr(lio_dev, CN23XX_SLI_OQ_PKT_CONTROL(q_no), + reg_val); + } +} + +static int +cn23xx_vf_setup_device_regs(struct lio_device *lio_dev) +{ + PMD_INIT_FUNC_TRACE(); + + if (cn23xx_vf_setup_global_input_regs(lio_dev)) + return -1; + + cn23xx_vf_setup_global_output_regs(lio_dev); + + return 0; +} + +static void +cn23xx_vf_setup_iq_regs(struct lio_device *lio_dev, uint32_t iq_no) +{ + struct lio_instr_queue *iq = lio_dev->instr_queue[iq_no]; + uint64_t pkt_in_done = 0; + + PMD_INIT_FUNC_TRACE(); + + /* Write the start of the input queue's ring and its size */ + lio_write_csr64(lio_dev, CN23XX_SLI_IQ_BASE_ADDR64(iq_no), + iq->base_addr_dma); + lio_write_csr(lio_dev, CN23XX_SLI_IQ_SIZE(iq_no), iq->max_count); + + /* Remember the doorbell & instruction count register addr + * for this queue + */ + iq->doorbell_reg = (uint8_t *)lio_dev->hw_addr + + CN23XX_SLI_IQ_DOORBELL(iq_no); + iq->inst_cnt_reg = (uint8_t *)lio_dev->hw_addr + + CN23XX_SLI_IQ_INSTR_COUNT64(iq_no); + lio_dev_dbg(lio_dev, "InstQ[%d]:dbell reg @ 0x%p instcnt_reg @ 0x%p\n", + iq_no, iq->doorbell_reg, iq->inst_cnt_reg); + + /* Store the current instruction counter (used in flush_iq + * calculation) + */ + pkt_in_done = rte_read64(iq->inst_cnt_reg); + + /* Clear the count by writing back what we read, but don't + * enable data traffic here + */ + rte_write64(pkt_in_done, iq->inst_cnt_reg); +} + +static void +cn23xx_vf_setup_oq_regs(struct lio_device *lio_dev, uint32_t oq_no) +{ + struct lio_droq *droq = lio_dev->droq[oq_no]; + + PMD_INIT_FUNC_TRACE(); + + lio_write_csr64(lio_dev, CN23XX_SLI_OQ_BASE_ADDR64(oq_no), + droq->desc_ring_dma); + lio_write_csr(lio_dev, CN23XX_SLI_OQ_SIZE(oq_no), droq->max_count); + + lio_write_csr(lio_dev, CN23XX_SLI_OQ_BUFF_INFO_SIZE(oq_no), + (droq->buffer_size | (OCTEON_RH_SIZE << 16))); + + /* Get the mapped address of the pkt_sent and pkts_credit regs */ + droq->pkts_sent_reg = (uint8_t *)lio_dev->hw_addr + + CN23XX_SLI_OQ_PKTS_SENT(oq_no); + droq->pkts_credit_reg = (uint8_t *)lio_dev->hw_addr + + CN23XX_SLI_OQ_PKTS_CREDIT(oq_no); +} + +static void +cn23xx_vf_free_mbox(struct lio_device *lio_dev) +{ + PMD_INIT_FUNC_TRACE(); + + rte_free(lio_dev->mbox[0]); + lio_dev->mbox[0] = NULL; + + rte_free(lio_dev->mbox); + lio_dev->mbox = NULL; +} + +static int +cn23xx_vf_setup_mbox(struct lio_device *lio_dev) +{ + struct lio_mbox *mbox; + + PMD_INIT_FUNC_TRACE(); + + if (lio_dev->mbox == NULL) { + lio_dev->mbox = rte_zmalloc(NULL, sizeof(void *), 0); + if (lio_dev->mbox == NULL) + return -ENOMEM; + } + + mbox = rte_zmalloc(NULL, sizeof(struct lio_mbox), 0); + if (mbox == NULL) { + rte_free(lio_dev->mbox); + lio_dev->mbox = NULL; + return -ENOMEM; + } + + rte_spinlock_init(&mbox->lock); + + mbox->lio_dev = lio_dev; + + mbox->q_no = 0; + + mbox->state = LIO_MBOX_STATE_IDLE; + + /* VF mbox interrupt reg */ + mbox->mbox_int_reg = (uint8_t *)lio_dev->hw_addr + + CN23XX_VF_SLI_PKT_MBOX_INT(0); + /* VF reads from SIG0 reg */ + mbox->mbox_read_reg = (uint8_t *)lio_dev->hw_addr + + CN23XX_SLI_PKT_PF_VF_MBOX_SIG(0, 0); + /* VF writes into SIG1 reg */ + mbox->mbox_write_reg = (uint8_t *)lio_dev->hw_addr + + CN23XX_SLI_PKT_PF_VF_MBOX_SIG(0, 1); + + lio_dev->mbox[0] = mbox; + + rte_write64(LIO_PFVFSIG, mbox->mbox_read_reg); + + return 0; +} + +static int +cn23xx_vf_enable_io_queues(struct lio_device *lio_dev) +{ + uint32_t q_no; + + PMD_INIT_FUNC_TRACE(); + + for (q_no = 0; q_no < lio_dev->num_iqs; q_no++) { + uint64_t reg_val; + + /* set the corresponding IQ IS_64B bit */ + if (lio_dev->io_qmask.iq64B & (1ULL << q_no)) { + reg_val = lio_read_csr64( + lio_dev, + CN23XX_SLI_IQ_PKT_CONTROL64(q_no)); + reg_val = reg_val | CN23XX_PKT_INPUT_CTL_IS_64B; + lio_write_csr64(lio_dev, + CN23XX_SLI_IQ_PKT_CONTROL64(q_no), + reg_val); + } + + /* set the corresponding IQ ENB bit */ + if (lio_dev->io_qmask.iq & (1ULL << q_no)) { + reg_val = lio_read_csr64( + lio_dev, + CN23XX_SLI_IQ_PKT_CONTROL64(q_no)); + reg_val = reg_val | CN23XX_PKT_INPUT_CTL_RING_ENB; + lio_write_csr64(lio_dev, + CN23XX_SLI_IQ_PKT_CONTROL64(q_no), + reg_val); + } + } + for (q_no = 0; q_no < lio_dev->num_oqs; q_no++) { + uint32_t reg_val; + + /* set the corresponding OQ ENB bit */ + if (lio_dev->io_qmask.oq & (1ULL << q_no)) { + reg_val = lio_read_csr( + lio_dev, + CN23XX_SLI_OQ_PKT_CONTROL(q_no)); + reg_val = reg_val | CN23XX_PKT_OUTPUT_CTL_RING_ENB; + lio_write_csr(lio_dev, + CN23XX_SLI_OQ_PKT_CONTROL(q_no), + reg_val); + } + } + + return 0; +} + +static void +cn23xx_vf_disable_io_queues(struct lio_device *lio_dev) +{ + uint32_t num_queues; + + PMD_INIT_FUNC_TRACE(); + + /* per HRM, rings can only be disabled via reset operation, + * NOT via SLI_PKT()_INPUT/OUTPUT_CONTROL[ENB] + */ + num_queues = lio_dev->num_iqs; + if (num_queues < lio_dev->num_oqs) + num_queues = lio_dev->num_oqs; + + cn23xx_vf_reset_io_queues(lio_dev, num_queues); +} + +void +cn23xx_vf_ask_pf_to_do_flr(struct lio_device *lio_dev) +{ + struct lio_mbox_cmd mbox_cmd; + + memset(&mbox_cmd, 0, sizeof(struct lio_mbox_cmd)); + mbox_cmd.msg.s.type = LIO_MBOX_REQUEST; + mbox_cmd.msg.s.resp_needed = 0; + mbox_cmd.msg.s.cmd = LIO_VF_FLR_REQUEST; + mbox_cmd.msg.s.len = 1; + mbox_cmd.q_no = 0; + mbox_cmd.recv_len = 0; + mbox_cmd.recv_status = 0; + mbox_cmd.fn = NULL; + mbox_cmd.fn_arg = 0; + + lio_mbox_write(lio_dev, &mbox_cmd); +} + +static void +cn23xx_pfvf_hs_callback(struct lio_device *lio_dev, + struct lio_mbox_cmd *cmd, void *arg) +{ + uint32_t major = 0; + + PMD_INIT_FUNC_TRACE(); + + rte_memcpy((uint8_t *)&lio_dev->pfvf_hsword, cmd->msg.s.params, 6); + if (cmd->recv_len > 1) { + struct lio_version *lio_ver = (struct lio_version *)cmd->data; + + major = lio_ver->major; + major = major << 16; + } + + rte_atomic64_set((rte_atomic64_t *)arg, major | 1); +} + +int +cn23xx_pfvf_handshake(struct lio_device *lio_dev) +{ + struct lio_mbox_cmd mbox_cmd; + struct lio_version *lio_ver = (struct lio_version *)&mbox_cmd.data[0]; + uint32_t q_no, count = 0; + rte_atomic64_t status; + uint32_t pfmajor; + uint32_t vfmajor; + uint32_t ret; + + PMD_INIT_FUNC_TRACE(); + + /* Sending VF_ACTIVE indication to the PF driver */ + lio_dev_dbg(lio_dev, "requesting info from PF\n"); + + mbox_cmd.msg.mbox_msg64 = 0; + mbox_cmd.msg.s.type = LIO_MBOX_REQUEST; + mbox_cmd.msg.s.resp_needed = 1; + mbox_cmd.msg.s.cmd = LIO_VF_ACTIVE; + mbox_cmd.msg.s.len = 2; + mbox_cmd.data[0] = 0; + lio_ver->major = LIO_BASE_MAJOR_VERSION; + lio_ver->minor = LIO_BASE_MINOR_VERSION; + lio_ver->micro = LIO_BASE_MICRO_VERSION; + mbox_cmd.q_no = 0; + mbox_cmd.recv_len = 0; + mbox_cmd.recv_status = 0; + mbox_cmd.fn = (lio_mbox_callback)cn23xx_pfvf_hs_callback; + mbox_cmd.fn_arg = (void *)&status; + + if (lio_mbox_write(lio_dev, &mbox_cmd)) { + lio_dev_err(lio_dev, "Write to mailbox failed\n"); + return -1; + } + + rte_atomic64_set(&status, 0); + + do { + rte_delay_ms(1); + } while ((rte_atomic64_read(&status) == 0) && (count++ < 10000)); + + ret = rte_atomic64_read(&status); + if (ret == 0) { + lio_dev_err(lio_dev, "cn23xx_pfvf_handshake timeout\n"); + return -1; + } + + for (q_no = 0; q_no < lio_dev->num_iqs; q_no++) + lio_dev->instr_queue[q_no]->txpciq.s.pkind = + lio_dev->pfvf_hsword.pkind; + + vfmajor = LIO_BASE_MAJOR_VERSION; + pfmajor = ret >> 16; + if (pfmajor != vfmajor) { + lio_dev_err(lio_dev, + "VF LiquidIO driver (major version %d) is not compatible with LiquidIO PF driver (major version %d)\n", + vfmajor, pfmajor); + ret = -EPERM; + } else { + lio_dev_dbg(lio_dev, + "VF LiquidIO driver (major version %d), LiquidIO PF driver (major version %d)\n", + vfmajor, pfmajor); + ret = 0; + } + + lio_dev_dbg(lio_dev, "got data from PF pkind is %d\n", + lio_dev->pfvf_hsword.pkind); + + return ret; +} + +void +cn23xx_vf_handle_mbox(struct lio_device *lio_dev) +{ + uint64_t mbox_int_val; + + /* read and clear by writing 1 */ + mbox_int_val = rte_read64(lio_dev->mbox[0]->mbox_int_reg); + rte_write64(mbox_int_val, lio_dev->mbox[0]->mbox_int_reg); + if (lio_mbox_read(lio_dev->mbox[0])) + lio_mbox_process_message(lio_dev->mbox[0]); +} + +int +cn23xx_vf_setup_device(struct lio_device *lio_dev) +{ + uint64_t reg_val; + + PMD_INIT_FUNC_TRACE(); + + /* INPUT_CONTROL[RPVF] gives the VF IOq count */ + reg_val = lio_read_csr64(lio_dev, CN23XX_SLI_IQ_PKT_CONTROL64(0)); + + lio_dev->pf_num = (reg_val >> CN23XX_PKT_INPUT_CTL_PF_NUM_POS) & + CN23XX_PKT_INPUT_CTL_PF_NUM_MASK; + lio_dev->vf_num = (reg_val >> CN23XX_PKT_INPUT_CTL_VF_NUM_POS) & + CN23XX_PKT_INPUT_CTL_VF_NUM_MASK; + + reg_val = reg_val >> CN23XX_PKT_INPUT_CTL_RPVF_POS; + + lio_dev->sriov_info.rings_per_vf = + reg_val & CN23XX_PKT_INPUT_CTL_RPVF_MASK; + + lio_dev->default_config = lio_get_conf(lio_dev); + if (lio_dev->default_config == NULL) + return -1; + + lio_dev->fn_list.setup_iq_regs = cn23xx_vf_setup_iq_regs; + lio_dev->fn_list.setup_oq_regs = cn23xx_vf_setup_oq_regs; + lio_dev->fn_list.setup_mbox = cn23xx_vf_setup_mbox; + lio_dev->fn_list.free_mbox = cn23xx_vf_free_mbox; + + lio_dev->fn_list.setup_device_regs = cn23xx_vf_setup_device_regs; + + lio_dev->fn_list.enable_io_queues = cn23xx_vf_enable_io_queues; + lio_dev->fn_list.disable_io_queues = cn23xx_vf_disable_io_queues; + + return 0; +} + +int +cn23xx_vf_set_io_queues_off(struct lio_device *lio_dev) +{ + uint32_t loop = CN23XX_VF_BUSY_READING_REG_LOOP_COUNT; + uint64_t q_no; + + /* Disable the i/p and o/p queues for this Octeon. + * IOQs will already be in reset. + * If RST bit is set, wait for Quiet bit to be set + * Once Quiet bit is set, clear the RST bit + */ + PMD_INIT_FUNC_TRACE(); + + for (q_no = 0; q_no < lio_dev->sriov_info.rings_per_vf; q_no++) { + volatile uint64_t reg_val; + + reg_val = lio_read_csr64(lio_dev, + CN23XX_SLI_IQ_PKT_CONTROL64(q_no)); + while ((reg_val & CN23XX_PKT_INPUT_CTL_RST) && !(reg_val & + CN23XX_PKT_INPUT_CTL_QUIET) && loop) { + reg_val = lio_read_csr64( + lio_dev, + CN23XX_SLI_IQ_PKT_CONTROL64(q_no)); + loop = loop - 1; + } + + if (loop == 0) { + lio_dev_err(lio_dev, + "clearing the reset reg failed or setting the quiet reg failed for qno %lu\n", + (unsigned long)q_no); + return -1; + } + + reg_val = reg_val & ~CN23XX_PKT_INPUT_CTL_RST; + lio_write_csr64(lio_dev, CN23XX_SLI_IQ_PKT_CONTROL64(q_no), + reg_val); + + reg_val = lio_read_csr64(lio_dev, + CN23XX_SLI_IQ_PKT_CONTROL64(q_no)); + if (reg_val & CN23XX_PKT_INPUT_CTL_RST) { + lio_dev_err(lio_dev, "unable to reset qno %lu\n", + (unsigned long)q_no); + return -1; + } + } + + return 0; +} diff --git a/dpdk/drivers/net/liquidio/base/lio_23xx_vf.h b/dpdk/drivers/net/liquidio/base/lio_23xx_vf.h new file mode 100644 index 00000000..ad8db0df --- /dev/null +++ b/dpdk/drivers/net/liquidio/base/lio_23xx_vf.h @@ -0,0 +1,97 @@ +/* + * BSD LICENSE + * + * Copyright(c) 2017 Cavium, Inc.. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _LIO_23XX_VF_H_ +#define _LIO_23XX_VF_H_ + +#include + +#include "lio_struct.h" + +static const struct lio_config default_cn23xx_conf = { + .card_type = LIO_23XX, + .card_name = LIO_23XX_NAME, + /** IQ attributes */ + .iq = { + .max_iqs = CN23XX_CFG_IO_QUEUES, + .pending_list_size = + (CN23XX_MAX_IQ_DESCRIPTORS * CN23XX_CFG_IO_QUEUES), + .instr_type = OCTEON_64BYTE_INSTR, + }, + + /** OQ attributes */ + .oq = { + .max_oqs = CN23XX_CFG_IO_QUEUES, + .info_ptr = OCTEON_OQ_INFOPTR_MODE, + .refill_threshold = CN23XX_OQ_REFIL_THRESHOLD, + }, + + .num_nic_ports = CN23XX_DEFAULT_NUM_PORTS, + .num_def_rx_descs = CN23XX_MAX_OQ_DESCRIPTORS, + .num_def_tx_descs = CN23XX_MAX_IQ_DESCRIPTORS, + .def_rx_buf_size = CN23XX_OQ_BUF_SIZE, +}; + +static inline const struct lio_config * +lio_get_conf(struct lio_device *lio_dev) +{ + const struct lio_config *default_lio_conf = NULL; + + /* check the LIO Device model & return the corresponding lio + * configuration + */ + default_lio_conf = &default_cn23xx_conf; + + if (default_lio_conf == NULL) { + lio_dev_err(lio_dev, "Configuration verification failed\n"); + return NULL; + } + + return default_lio_conf; +} + +/** Turns off the input and output queues for the device + * @param lio_dev which device io queues to disable + */ +int cn23xx_vf_set_io_queues_off(struct lio_device *lio_dev); + +#define CN23XX_VF_BUSY_READING_REG_LOOP_COUNT 100000 + +void cn23xx_vf_ask_pf_to_do_flr(struct lio_device *lio_dev); + +int cn23xx_pfvf_handshake(struct lio_device *lio_dev); + +int cn23xx_vf_setup_device(struct lio_device *lio_dev); + +void cn23xx_vf_handle_mbox(struct lio_device *lio_dev); +#endif /* _LIO_23XX_VF_H_ */ diff --git a/dpdk/drivers/net/liquidio/base/lio_hw_defs.h b/dpdk/drivers/net/liquidio/base/lio_hw_defs.h new file mode 100644 index 00000000..fe5c3bbb --- /dev/null +++ b/dpdk/drivers/net/liquidio/base/lio_hw_defs.h @@ -0,0 +1,266 @@ +/* + * BSD LICENSE + * + * Copyright(c) 2017 Cavium, Inc.. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _LIO_HW_DEFS_H_ +#define _LIO_HW_DEFS_H_ + +#include + +#ifndef PCI_VENDOR_ID_CAVIUM +#define PCI_VENDOR_ID_CAVIUM 0x177D +#endif + +#define LIO_CN23XX_VF_VID 0x9712 + +/* CN23xx subsystem device ids */ +#define PCI_SUBSYS_DEV_ID_CN2350_210 0x0004 +#define PCI_SUBSYS_DEV_ID_CN2360_210 0x0005 +#define PCI_SUBSYS_DEV_ID_CN2360_225 0x0006 +#define PCI_SUBSYS_DEV_ID_CN2350_225 0x0007 +#define PCI_SUBSYS_DEV_ID_CN2350_210SVPN3 0x0008 +#define PCI_SUBSYS_DEV_ID_CN2360_210SVPN3 0x0009 +#define PCI_SUBSYS_DEV_ID_CN2350_210SVPT 0x000a +#define PCI_SUBSYS_DEV_ID_CN2360_210SVPT 0x000b + +/* --------------------------CONFIG VALUES------------------------ */ + +/* CN23xx IQ configuration macros */ +#define CN23XX_MAX_RINGS_PER_PF 64 +#define CN23XX_MAX_RINGS_PER_VF 8 + +#define CN23XX_MAX_INPUT_QUEUES CN23XX_MAX_RINGS_PER_PF +#define CN23XX_MAX_IQ_DESCRIPTORS 512 +#define CN23XX_MIN_IQ_DESCRIPTORS 128 + +#define CN23XX_MAX_OUTPUT_QUEUES CN23XX_MAX_RINGS_PER_PF +#define CN23XX_MAX_OQ_DESCRIPTORS 512 +#define CN23XX_MIN_OQ_DESCRIPTORS 128 +#define CN23XX_OQ_BUF_SIZE 1536 + +#define CN23XX_OQ_REFIL_THRESHOLD 16 + +#define CN23XX_DEFAULT_NUM_PORTS 1 + +#define CN23XX_CFG_IO_QUEUES CN23XX_MAX_RINGS_PER_PF + +/* common OCTEON configuration macros */ +#define OCTEON_64BYTE_INSTR 64 +#define OCTEON_OQ_INFOPTR_MODE 1 + +/* Max IOQs per LIO Link */ +#define LIO_MAX_IOQS_PER_IF 64 + +/* Wait time in milliseconds for FLR */ +#define LIO_PCI_FLR_WAIT 100 + +enum lio_card_type { + LIO_23XX /* 23xx */ +}; + +#define LIO_23XX_NAME "23xx" + +#define LIO_DEV_RUNNING 0xc + +#define LIO_OQ_REFILL_THRESHOLD_CFG(cfg) \ + ((cfg)->default_config->oq.refill_threshold) +#define LIO_NUM_DEF_TX_DESCS_CFG(cfg) \ + ((cfg)->default_config->num_def_tx_descs) + +#define LIO_IQ_INSTR_TYPE(cfg) ((cfg)->default_config->iq.instr_type) + +/* The following config values are fixed and should not be modified. */ + +/* Maximum number of Instruction queues */ +#define LIO_MAX_INSTR_QUEUES(lio_dev) CN23XX_MAX_RINGS_PER_VF + +#define LIO_MAX_POSSIBLE_INSTR_QUEUES CN23XX_MAX_INPUT_QUEUES +#define LIO_MAX_POSSIBLE_OUTPUT_QUEUES CN23XX_MAX_OUTPUT_QUEUES + +#define LIO_DEVICE_NAME_LEN 32 +#define LIO_BASE_MAJOR_VERSION 1 +#define LIO_BASE_MINOR_VERSION 5 +#define LIO_BASE_MICRO_VERSION 1 + +#define LIO_FW_VERSION_LENGTH 32 + +#define LIO_VF_TRUST_MIN_VERSION "1.7.1" + +/** Tag types used by Octeon cores in its work. */ +enum octeon_tag_type { + OCTEON_ORDERED_TAG = 0, + OCTEON_ATOMIC_TAG = 1, +}; + +/* pre-defined host->NIC tag values */ +#define LIO_CONTROL (0x11111110) +#define LIO_DATA(i) (0x11111111 + (i)) + +/* used for NIC operations */ +#define LIO_OPCODE 1 + +/* Subcodes are used by host driver/apps to identify the sub-operation + * for the core. They only need to by unique for a given subsystem. + */ +#define LIO_OPCODE_SUBCODE(op, sub) \ + ((((op) & 0x0f) << 8) | ((sub) & 0x7f)) + +/** LIO_OPCODE subcodes */ +/* This subcode is sent by core PCI driver to indicate cores are ready. */ +#define LIO_OPCODE_NW_DATA 0x02 /* network packet data */ +#define LIO_OPCODE_CMD 0x03 +#define LIO_OPCODE_INFO 0x04 +#define LIO_OPCODE_PORT_STATS 0x05 +#define LIO_OPCODE_IF_CFG 0x09 + +#define LIO_MIN_RX_BUF_SIZE 64 +#define LIO_MAX_RX_PKTLEN (64 * 1024) + +/* NIC Command types */ +#define LIO_CMD_CHANGE_MTU 0x1 +#define LIO_CMD_CHANGE_DEVFLAGS 0x3 +#define LIO_CMD_RX_CTL 0x4 +#define LIO_CMD_CLEAR_STATS 0x6 +#define LIO_CMD_SET_RSS 0xD +#define LIO_CMD_TNL_RX_CSUM_CTL 0x10 +#define LIO_CMD_TNL_TX_CSUM_CTL 0x11 +#define LIO_CMD_ADD_VLAN_FILTER 0x17 +#define LIO_CMD_DEL_VLAN_FILTER 0x18 +#define LIO_CMD_VXLAN_PORT_CONFIG 0x19 + +#define LIO_CMD_VXLAN_PORT_ADD 0x0 +#define LIO_CMD_VXLAN_PORT_DEL 0x1 +#define LIO_CMD_RXCSUM_ENABLE 0x0 +#define LIO_CMD_TXCSUM_ENABLE 0x0 + +/* RX(packets coming from wire) Checksum verification flags */ +/* TCP/UDP csum */ +#define LIO_L4_CSUM_VERIFIED 0x1 +#define LIO_IP_CSUM_VERIFIED 0x2 + +/* RSS */ +#define LIO_RSS_PARAM_DISABLE_RSS 0x10 +#define LIO_RSS_PARAM_HASH_KEY_UNCHANGED 0x08 +#define LIO_RSS_PARAM_ITABLE_UNCHANGED 0x04 +#define LIO_RSS_PARAM_HASH_INFO_UNCHANGED 0x02 + +#define LIO_RSS_HASH_IPV4 0x100 +#define LIO_RSS_HASH_TCP_IPV4 0x200 +#define LIO_RSS_HASH_IPV6 0x400 +#define LIO_RSS_HASH_TCP_IPV6 0x1000 +#define LIO_RSS_HASH_IPV6_EX 0x800 +#define LIO_RSS_HASH_TCP_IPV6_EX 0x2000 + +#define LIO_RSS_OFFLOAD_ALL ( \ + LIO_RSS_HASH_IPV4 | \ + LIO_RSS_HASH_TCP_IPV4 | \ + LIO_RSS_HASH_IPV6 | \ + LIO_RSS_HASH_TCP_IPV6 | \ + LIO_RSS_HASH_IPV6_EX | \ + LIO_RSS_HASH_TCP_IPV6_EX) + +#define LIO_RSS_MAX_TABLE_SZ 128 +#define LIO_RSS_MAX_KEY_SZ 40 +#define LIO_RSS_PARAM_SIZE 16 + +/* Interface flags communicated between host driver and core app. */ +enum lio_ifflags { + LIO_IFFLAG_PROMISC = 0x01, + LIO_IFFLAG_ALLMULTI = 0x02, + LIO_IFFLAG_UNICAST = 0x10 +}; + +/* Routines for reading and writing CSRs */ +#ifdef RTE_LIBRTE_LIO_DEBUG_REGS +#define lio_write_csr(lio_dev, reg_off, value) \ + do { \ + typeof(lio_dev) _dev = lio_dev; \ + typeof(reg_off) _reg_off = reg_off; \ + typeof(value) _value = value; \ + PMD_REGS_LOG(_dev, \ + "Write32: Reg: 0x%08lx Val: 0x%08lx\n", \ + (unsigned long)_reg_off, \ + (unsigned long)_value); \ + rte_write32(_value, _dev->hw_addr + _reg_off); \ + } while (0) + +#define lio_write_csr64(lio_dev, reg_off, val64) \ + do { \ + typeof(lio_dev) _dev = lio_dev; \ + typeof(reg_off) _reg_off = reg_off; \ + typeof(val64) _val64 = val64; \ + PMD_REGS_LOG( \ + _dev, \ + "Write64: Reg: 0x%08lx Val: 0x%016llx\n", \ + (unsigned long)_reg_off, \ + (unsigned long long)_val64); \ + rte_write64(_val64, _dev->hw_addr + _reg_off); \ + } while (0) + +#define lio_read_csr(lio_dev, reg_off) \ + ({ \ + typeof(lio_dev) _dev = lio_dev; \ + typeof(reg_off) _reg_off = reg_off; \ + uint32_t val = rte_read32(_dev->hw_addr + _reg_off); \ + PMD_REGS_LOG(_dev, \ + "Read32: Reg: 0x%08lx Val: 0x%08lx\n", \ + (unsigned long)_reg_off, \ + (unsigned long)val); \ + val; \ + }) + +#define lio_read_csr64(lio_dev, reg_off) \ + ({ \ + typeof(lio_dev) _dev = lio_dev; \ + typeof(reg_off) _reg_off = reg_off; \ + uint64_t val64 = rte_read64(_dev->hw_addr + _reg_off); \ + PMD_REGS_LOG( \ + _dev, \ + "Read64: Reg: 0x%08lx Val: 0x%016llx\n", \ + (unsigned long)_reg_off, \ + (unsigned long long)val64); \ + val64; \ + }) +#else +#define lio_write_csr(lio_dev, reg_off, value) \ + rte_write32(value, (lio_dev)->hw_addr + (reg_off)) + +#define lio_write_csr64(lio_dev, reg_off, val64) \ + rte_write64(val64, (lio_dev)->hw_addr + (reg_off)) + +#define lio_read_csr(lio_dev, reg_off) \ + rte_read32((lio_dev)->hw_addr + (reg_off)) + +#define lio_read_csr64(lio_dev, reg_off) \ + rte_read64((lio_dev)->hw_addr + (reg_off)) +#endif +#endif /* _LIO_HW_DEFS_H_ */ diff --git a/dpdk/drivers/net/liquidio/base/lio_mbox.c b/dpdk/drivers/net/liquidio/base/lio_mbox.c new file mode 100644 index 00000000..b4abc623 --- /dev/null +++ b/dpdk/drivers/net/liquidio/base/lio_mbox.c @@ -0,0 +1,275 @@ +/* + * BSD LICENSE + * + * Copyright(c) 2017 Cavium, Inc.. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include + +#include "lio_logs.h" +#include "lio_struct.h" +#include "lio_mbox.h" + +/** + * lio_mbox_read: + * @mbox: Pointer mailbox + * + * Reads the 8-bytes of data from the mbox register + * Writes back the acknowledgment indicating completion of read + */ +int +lio_mbox_read(struct lio_mbox *mbox) +{ + union lio_mbox_message msg; + int ret = 0; + + msg.mbox_msg64 = rte_read64(mbox->mbox_read_reg); + + if ((msg.mbox_msg64 == LIO_PFVFACK) || (msg.mbox_msg64 == LIO_PFVFSIG)) + return 0; + + if (mbox->state & LIO_MBOX_STATE_REQ_RECEIVING) { + mbox->mbox_req.data[mbox->mbox_req.recv_len - 1] = + msg.mbox_msg64; + mbox->mbox_req.recv_len++; + } else { + if (mbox->state & LIO_MBOX_STATE_RES_RECEIVING) { + mbox->mbox_resp.data[mbox->mbox_resp.recv_len - 1] = + msg.mbox_msg64; + mbox->mbox_resp.recv_len++; + } else { + if ((mbox->state & LIO_MBOX_STATE_IDLE) && + (msg.s.type == LIO_MBOX_REQUEST)) { + mbox->state &= ~LIO_MBOX_STATE_IDLE; + mbox->state |= LIO_MBOX_STATE_REQ_RECEIVING; + mbox->mbox_req.msg.mbox_msg64 = msg.mbox_msg64; + mbox->mbox_req.q_no = mbox->q_no; + mbox->mbox_req.recv_len = 1; + } else { + if ((mbox->state & + LIO_MBOX_STATE_RES_PENDING) && + (msg.s.type == LIO_MBOX_RESPONSE)) { + mbox->state &= + ~LIO_MBOX_STATE_RES_PENDING; + mbox->state |= + LIO_MBOX_STATE_RES_RECEIVING; + mbox->mbox_resp.msg.mbox_msg64 = + msg.mbox_msg64; + mbox->mbox_resp.q_no = mbox->q_no; + mbox->mbox_resp.recv_len = 1; + } else { + rte_write64(LIO_PFVFERR, + mbox->mbox_read_reg); + mbox->state |= LIO_MBOX_STATE_ERROR; + return -1; + } + } + } + } + + if (mbox->state & LIO_MBOX_STATE_REQ_RECEIVING) { + if (mbox->mbox_req.recv_len < msg.s.len) { + ret = 0; + } else { + mbox->state &= ~LIO_MBOX_STATE_REQ_RECEIVING; + mbox->state |= LIO_MBOX_STATE_REQ_RECEIVED; + ret = 1; + } + } else { + if (mbox->state & LIO_MBOX_STATE_RES_RECEIVING) { + if (mbox->mbox_resp.recv_len < msg.s.len) { + ret = 0; + } else { + mbox->state &= ~LIO_MBOX_STATE_RES_RECEIVING; + mbox->state |= LIO_MBOX_STATE_RES_RECEIVED; + ret = 1; + } + } else { + RTE_ASSERT(0); + } + } + + rte_write64(LIO_PFVFACK, mbox->mbox_read_reg); + + return ret; +} + +/** + * lio_mbox_write: + * @lio_dev: Pointer lio device + * @mbox_cmd: Cmd to send to mailbox. + * + * Populates the queue specific mbox structure + * with cmd information. + * Write the cmd to mbox register + */ +int +lio_mbox_write(struct lio_device *lio_dev, + struct lio_mbox_cmd *mbox_cmd) +{ + struct lio_mbox *mbox = lio_dev->mbox[mbox_cmd->q_no]; + uint32_t count, i, ret = LIO_MBOX_STATUS_SUCCESS; + + if ((mbox_cmd->msg.s.type == LIO_MBOX_RESPONSE) && + !(mbox->state & LIO_MBOX_STATE_REQ_RECEIVED)) + return LIO_MBOX_STATUS_FAILED; + + if ((mbox_cmd->msg.s.type == LIO_MBOX_REQUEST) && + !(mbox->state & LIO_MBOX_STATE_IDLE)) + return LIO_MBOX_STATUS_BUSY; + + if (mbox_cmd->msg.s.type == LIO_MBOX_REQUEST) { + rte_memcpy(&mbox->mbox_resp, mbox_cmd, + sizeof(struct lio_mbox_cmd)); + mbox->state = LIO_MBOX_STATE_RES_PENDING; + } + + count = 0; + + while (rte_read64(mbox->mbox_write_reg) != LIO_PFVFSIG) { + rte_delay_ms(1); + if (count++ == 1000) { + ret = LIO_MBOX_STATUS_FAILED; + break; + } + } + + if (ret == LIO_MBOX_STATUS_SUCCESS) { + rte_write64(mbox_cmd->msg.mbox_msg64, mbox->mbox_write_reg); + for (i = 0; i < (uint32_t)(mbox_cmd->msg.s.len - 1); i++) { + count = 0; + while (rte_read64(mbox->mbox_write_reg) != + LIO_PFVFACK) { + rte_delay_ms(1); + if (count++ == 1000) { + ret = LIO_MBOX_STATUS_FAILED; + break; + } + } + rte_write64(mbox_cmd->data[i], mbox->mbox_write_reg); + } + } + + if (mbox_cmd->msg.s.type == LIO_MBOX_RESPONSE) { + mbox->state = LIO_MBOX_STATE_IDLE; + rte_write64(LIO_PFVFSIG, mbox->mbox_read_reg); + } else { + if ((!mbox_cmd->msg.s.resp_needed) || + (ret == LIO_MBOX_STATUS_FAILED)) { + mbox->state &= ~LIO_MBOX_STATE_RES_PENDING; + if (!(mbox->state & (LIO_MBOX_STATE_REQ_RECEIVING | + LIO_MBOX_STATE_REQ_RECEIVED))) + mbox->state = LIO_MBOX_STATE_IDLE; + } + } + + return ret; +} + +/** + * lio_mbox_process_cmd: + * @mbox: Pointer mailbox + * @mbox_cmd: Pointer to command received + * + * Process the cmd received in mbox + */ +static int +lio_mbox_process_cmd(struct lio_mbox *mbox, + struct lio_mbox_cmd *mbox_cmd) +{ + struct lio_device *lio_dev = mbox->lio_dev; + + if (mbox_cmd->msg.s.cmd == LIO_CORES_CRASHED) + lio_dev_err(lio_dev, "Octeon core(s) crashed or got stuck!\n"); + + return 0; +} + +/** + * Process the received mbox message. + */ +int +lio_mbox_process_message(struct lio_mbox *mbox) +{ + struct lio_mbox_cmd mbox_cmd; + + if (mbox->state & LIO_MBOX_STATE_ERROR) { + if (mbox->state & (LIO_MBOX_STATE_RES_PENDING | + LIO_MBOX_STATE_RES_RECEIVING)) { + rte_memcpy(&mbox_cmd, &mbox->mbox_resp, + sizeof(struct lio_mbox_cmd)); + mbox->state = LIO_MBOX_STATE_IDLE; + rte_write64(LIO_PFVFSIG, mbox->mbox_read_reg); + mbox_cmd.recv_status = 1; + if (mbox_cmd.fn) + mbox_cmd.fn(mbox->lio_dev, &mbox_cmd, + mbox_cmd.fn_arg); + + return 0; + } + + mbox->state = LIO_MBOX_STATE_IDLE; + rte_write64(LIO_PFVFSIG, mbox->mbox_read_reg); + + return 0; + } + + if (mbox->state & LIO_MBOX_STATE_RES_RECEIVED) { + rte_memcpy(&mbox_cmd, &mbox->mbox_resp, + sizeof(struct lio_mbox_cmd)); + mbox->state = LIO_MBOX_STATE_IDLE; + rte_write64(LIO_PFVFSIG, mbox->mbox_read_reg); + mbox_cmd.recv_status = 0; + if (mbox_cmd.fn) + mbox_cmd.fn(mbox->lio_dev, &mbox_cmd, mbox_cmd.fn_arg); + + return 0; + } + + if (mbox->state & LIO_MBOX_STATE_REQ_RECEIVED) { + rte_memcpy(&mbox_cmd, &mbox->mbox_req, + sizeof(struct lio_mbox_cmd)); + if (!mbox_cmd.msg.s.resp_needed) { + mbox->state &= ~LIO_MBOX_STATE_REQ_RECEIVED; + if (!(mbox->state & LIO_MBOX_STATE_RES_PENDING)) + mbox->state = LIO_MBOX_STATE_IDLE; + rte_write64(LIO_PFVFSIG, mbox->mbox_read_reg); + } + + lio_mbox_process_cmd(mbox, &mbox_cmd); + + return 0; + } + + RTE_ASSERT(0); + + return 0; +} diff --git a/dpdk/drivers/net/liquidio/base/lio_mbox.h b/dpdk/drivers/net/liquidio/base/lio_mbox.h new file mode 100644 index 00000000..b0875d64 --- /dev/null +++ b/dpdk/drivers/net/liquidio/base/lio_mbox.h @@ -0,0 +1,131 @@ +/* + * BSD LICENSE + * + * Copyright(c) 2017 Cavium, Inc.. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _LIO_MBOX_H_ +#define _LIO_MBOX_H_ + +#include + +#include + +/* Macros for Mail Box Communication */ + +#define LIO_MBOX_DATA_MAX 32 + +#define LIO_VF_ACTIVE 0x1 +#define LIO_VF_FLR_REQUEST 0x2 +#define LIO_CORES_CRASHED 0x3 + +/* Macro for Read acknowledgment */ +#define LIO_PFVFACK 0xffffffffffffffff +#define LIO_PFVFSIG 0x1122334455667788 +#define LIO_PFVFERR 0xDEADDEADDEADDEAD + +enum lio_mbox_cmd_status { + LIO_MBOX_STATUS_SUCCESS = 0, + LIO_MBOX_STATUS_FAILED = 1, + LIO_MBOX_STATUS_BUSY = 2 +}; + +enum lio_mbox_message_type { + LIO_MBOX_REQUEST = 0, + LIO_MBOX_RESPONSE = 1 +}; + +union lio_mbox_message { + uint64_t mbox_msg64; + struct { + uint16_t type : 1; + uint16_t resp_needed : 1; + uint16_t cmd : 6; + uint16_t len : 8; + uint8_t params[6]; + } s; +}; + +typedef void (*lio_mbox_callback)(void *, void *, void *); + +struct lio_mbox_cmd { + union lio_mbox_message msg; + uint64_t data[LIO_MBOX_DATA_MAX]; + uint32_t q_no; + uint32_t recv_len; + uint32_t recv_status; + lio_mbox_callback fn; + void *fn_arg; +}; + +enum lio_mbox_state { + LIO_MBOX_STATE_IDLE = 1, + LIO_MBOX_STATE_REQ_RECEIVING = 2, + LIO_MBOX_STATE_REQ_RECEIVED = 4, + LIO_MBOX_STATE_RES_PENDING = 8, + LIO_MBOX_STATE_RES_RECEIVING = 16, + LIO_MBOX_STATE_RES_RECEIVED = 16, + LIO_MBOX_STATE_ERROR = 32 +}; + +struct lio_mbox { + /* A spinlock to protect access to this q_mbox. */ + rte_spinlock_t lock; + + struct lio_device *lio_dev; + + uint32_t q_no; + + enum lio_mbox_state state; + + /* SLI_MAC_PF_MBOX_INT for PF, SLI_PKT_MBOX_INT for VF. */ + void *mbox_int_reg; + + /* SLI_PKT_PF_VF_MBOX_SIG(0) for PF, + * SLI_PKT_PF_VF_MBOX_SIG(1) for VF. + */ + void *mbox_write_reg; + + /* SLI_PKT_PF_VF_MBOX_SIG(1) for PF, + * SLI_PKT_PF_VF_MBOX_SIG(0) for VF. + */ + void *mbox_read_reg; + + struct lio_mbox_cmd mbox_req; + + struct lio_mbox_cmd mbox_resp; + +}; + +int lio_mbox_read(struct lio_mbox *mbox); +int lio_mbox_write(struct lio_device *lio_dev, + struct lio_mbox_cmd *mbox_cmd); +int lio_mbox_process_message(struct lio_mbox *mbox); +#endif /* _LIO_MBOX_H_ */ diff --git a/dpdk/drivers/net/liquidio/lio_ethdev.c b/dpdk/drivers/net/liquidio/lio_ethdev.c new file mode 100644 index 00000000..84b8a328 --- /dev/null +++ b/dpdk/drivers/net/liquidio/lio_ethdev.c @@ -0,0 +1,2182 @@ +/* + * BSD LICENSE + * + * Copyright(c) 2017 Cavium, Inc.. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include + +#include "lio_logs.h" +#include "lio_23xx_vf.h" +#include "lio_ethdev.h" +#include "lio_rxtx.h" + +/* Default RSS key in use */ +static uint8_t lio_rss_key[40] = { + 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2, + 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0, + 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4, + 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C, + 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA, +}; + +static const struct rte_eth_desc_lim lio_rx_desc_lim = { + .nb_max = CN23XX_MAX_OQ_DESCRIPTORS, + .nb_min = CN23XX_MIN_OQ_DESCRIPTORS, + .nb_align = 1, +}; + +static const struct rte_eth_desc_lim lio_tx_desc_lim = { + .nb_max = CN23XX_MAX_IQ_DESCRIPTORS, + .nb_min = CN23XX_MIN_IQ_DESCRIPTORS, + .nb_align = 1, +}; + +/* Wait for control command to reach nic. */ +static uint16_t +lio_wait_for_ctrl_cmd(struct lio_device *lio_dev, + struct lio_dev_ctrl_cmd *ctrl_cmd) +{ + uint16_t timeout = LIO_MAX_CMD_TIMEOUT; + + while ((ctrl_cmd->cond == 0) && --timeout) { + lio_flush_iq(lio_dev, lio_dev->instr_queue[0]); + rte_delay_ms(1); + } + + return !timeout; +} + +/** + * \brief Send Rx control command + * @param eth_dev Pointer to the structure rte_eth_dev + * @param start_stop whether to start or stop + */ +static int +lio_send_rx_ctrl_cmd(struct rte_eth_dev *eth_dev, int start_stop) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + struct lio_dev_ctrl_cmd ctrl_cmd; + struct lio_ctrl_pkt ctrl_pkt; + + /* flush added to prevent cmd failure + * incase the queue is full + */ + lio_flush_iq(lio_dev, lio_dev->instr_queue[0]); + + memset(&ctrl_pkt, 0, sizeof(struct lio_ctrl_pkt)); + memset(&ctrl_cmd, 0, sizeof(struct lio_dev_ctrl_cmd)); + + ctrl_cmd.eth_dev = eth_dev; + ctrl_cmd.cond = 0; + + ctrl_pkt.ncmd.s.cmd = LIO_CMD_RX_CTL; + ctrl_pkt.ncmd.s.param1 = start_stop; + ctrl_pkt.ctrl_cmd = &ctrl_cmd; + + if (lio_send_ctrl_pkt(lio_dev, &ctrl_pkt)) { + lio_dev_err(lio_dev, "Failed to send RX Control message\n"); + return -1; + } + + if (lio_wait_for_ctrl_cmd(lio_dev, &ctrl_cmd)) { + lio_dev_err(lio_dev, "RX Control command timed out\n"); + return -1; + } + + return 0; +} + +/* store statistics names and its offset in stats structure */ +struct rte_lio_xstats_name_off { + char name[RTE_ETH_XSTATS_NAME_SIZE]; + unsigned int offset; +}; + +static const struct rte_lio_xstats_name_off rte_lio_stats_strings[] = { + {"rx_pkts", offsetof(struct octeon_rx_stats, total_rcvd)}, + {"rx_bytes", offsetof(struct octeon_rx_stats, bytes_rcvd)}, + {"rx_broadcast_pkts", offsetof(struct octeon_rx_stats, total_bcst)}, + {"rx_multicast_pkts", offsetof(struct octeon_rx_stats, total_mcst)}, + {"rx_flow_ctrl_pkts", offsetof(struct octeon_rx_stats, ctl_rcvd)}, + {"rx_fifo_err", offsetof(struct octeon_rx_stats, fifo_err)}, + {"rx_dmac_drop", offsetof(struct octeon_rx_stats, dmac_drop)}, + {"rx_fcs_err", offsetof(struct octeon_rx_stats, fcs_err)}, + {"rx_jabber_err", offsetof(struct octeon_rx_stats, jabber_err)}, + {"rx_l2_err", offsetof(struct octeon_rx_stats, l2_err)}, + {"rx_vxlan_pkts", offsetof(struct octeon_rx_stats, fw_rx_vxlan)}, + {"rx_vxlan_err", offsetof(struct octeon_rx_stats, fw_rx_vxlan_err)}, + {"rx_lro_pkts", offsetof(struct octeon_rx_stats, fw_lro_pkts)}, + {"tx_pkts", (offsetof(struct octeon_tx_stats, total_pkts_sent)) + + sizeof(struct octeon_rx_stats)}, + {"tx_bytes", (offsetof(struct octeon_tx_stats, total_bytes_sent)) + + sizeof(struct octeon_rx_stats)}, + {"tx_broadcast_pkts", + (offsetof(struct octeon_tx_stats, bcast_pkts_sent)) + + sizeof(struct octeon_rx_stats)}, + {"tx_multicast_pkts", + (offsetof(struct octeon_tx_stats, mcast_pkts_sent)) + + sizeof(struct octeon_rx_stats)}, + {"tx_flow_ctrl_pkts", (offsetof(struct octeon_tx_stats, ctl_sent)) + + sizeof(struct octeon_rx_stats)}, + {"tx_fifo_err", (offsetof(struct octeon_tx_stats, fifo_err)) + + sizeof(struct octeon_rx_stats)}, + {"tx_total_collisions", (offsetof(struct octeon_tx_stats, + total_collisions)) + + sizeof(struct octeon_rx_stats)}, + {"tx_tso", (offsetof(struct octeon_tx_stats, fw_tso)) + + sizeof(struct octeon_rx_stats)}, + {"tx_vxlan_pkts", (offsetof(struct octeon_tx_stats, fw_tx_vxlan)) + + sizeof(struct octeon_rx_stats)}, +}; + +#define LIO_NB_XSTATS RTE_DIM(rte_lio_stats_strings) + +/* Get hw stats of the port */ +static int +lio_dev_xstats_get(struct rte_eth_dev *eth_dev, struct rte_eth_xstat *xstats, + unsigned int n) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + uint16_t timeout = LIO_MAX_CMD_TIMEOUT; + struct octeon_link_stats *hw_stats; + struct lio_link_stats_resp *resp; + struct lio_soft_command *sc; + uint32_t resp_size; + unsigned int i; + int retval; + + if (!lio_dev->intf_open) { + lio_dev_err(lio_dev, "Port %d down\n", + lio_dev->port_id); + return -EINVAL; + } + + if (n < LIO_NB_XSTATS) + return LIO_NB_XSTATS; + + resp_size = sizeof(struct lio_link_stats_resp); + sc = lio_alloc_soft_command(lio_dev, 0, resp_size, 0); + if (sc == NULL) + return -ENOMEM; + + resp = (struct lio_link_stats_resp *)sc->virtrptr; + lio_prepare_soft_command(lio_dev, sc, LIO_OPCODE, + LIO_OPCODE_PORT_STATS, 0, 0, 0); + + /* Setting wait time in seconds */ + sc->wait_time = LIO_MAX_CMD_TIMEOUT / 1000; + + retval = lio_send_soft_command(lio_dev, sc); + if (retval == LIO_IQ_SEND_FAILED) { + lio_dev_err(lio_dev, "failed to get port stats from firmware. status: %x\n", + retval); + goto get_stats_fail; + } + + while ((*sc->status_word == LIO_COMPLETION_WORD_INIT) && --timeout) { + lio_flush_iq(lio_dev, lio_dev->instr_queue[sc->iq_no]); + lio_process_ordered_list(lio_dev); + rte_delay_ms(1); + } + + retval = resp->status; + if (retval) { + lio_dev_err(lio_dev, "failed to get port stats from firmware\n"); + goto get_stats_fail; + } + + lio_swap_8B_data((uint64_t *)(&resp->link_stats), + sizeof(struct octeon_link_stats) >> 3); + + hw_stats = &resp->link_stats; + + for (i = 0; i < LIO_NB_XSTATS; i++) { + xstats[i].id = i; + xstats[i].value = + *(uint64_t *)(((char *)hw_stats) + + rte_lio_stats_strings[i].offset); + } + + lio_free_soft_command(sc); + + return LIO_NB_XSTATS; + +get_stats_fail: + lio_free_soft_command(sc); + + return -1; +} + +static int +lio_dev_xstats_get_names(struct rte_eth_dev *eth_dev, + struct rte_eth_xstat_name *xstats_names, + unsigned limit __rte_unused) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + unsigned int i; + + if (!lio_dev->intf_open) { + lio_dev_err(lio_dev, "Port %d down\n", + lio_dev->port_id); + return -EINVAL; + } + + if (xstats_names == NULL) + return LIO_NB_XSTATS; + + /* Note: limit checked in rte_eth_xstats_names() */ + + for (i = 0; i < LIO_NB_XSTATS; i++) { + snprintf(xstats_names[i].name, sizeof(xstats_names[i].name), + "%s", rte_lio_stats_strings[i].name); + } + + return LIO_NB_XSTATS; +} + +/* Reset hw stats for the port */ +static void +lio_dev_xstats_reset(struct rte_eth_dev *eth_dev) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + struct lio_dev_ctrl_cmd ctrl_cmd; + struct lio_ctrl_pkt ctrl_pkt; + + if (!lio_dev->intf_open) { + lio_dev_err(lio_dev, "Port %d down\n", + lio_dev->port_id); + return; + } + + /* flush added to prevent cmd failure + * incase the queue is full + */ + lio_flush_iq(lio_dev, lio_dev->instr_queue[0]); + + memset(&ctrl_pkt, 0, sizeof(struct lio_ctrl_pkt)); + memset(&ctrl_cmd, 0, sizeof(struct lio_dev_ctrl_cmd)); + + ctrl_cmd.eth_dev = eth_dev; + ctrl_cmd.cond = 0; + + ctrl_pkt.ncmd.s.cmd = LIO_CMD_CLEAR_STATS; + ctrl_pkt.ctrl_cmd = &ctrl_cmd; + + if (lio_send_ctrl_pkt(lio_dev, &ctrl_pkt)) { + lio_dev_err(lio_dev, "Failed to send clear stats command\n"); + return; + } + + if (lio_wait_for_ctrl_cmd(lio_dev, &ctrl_cmd)) { + lio_dev_err(lio_dev, "Clear stats command timed out\n"); + return; + } + + /* clear stored per queue stats */ + RTE_FUNC_PTR_OR_RET(*eth_dev->dev_ops->stats_reset); + (*eth_dev->dev_ops->stats_reset)(eth_dev); +} + +/* Retrieve the device statistics (# packets in/out, # bytes in/out, etc */ +static int +lio_dev_stats_get(struct rte_eth_dev *eth_dev, + struct rte_eth_stats *stats) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + struct lio_droq_stats *oq_stats; + struct lio_iq_stats *iq_stats; + struct lio_instr_queue *txq; + struct lio_droq *droq; + int i, iq_no, oq_no; + uint64_t bytes = 0; + uint64_t pkts = 0; + uint64_t drop = 0; + + for (i = 0; i < eth_dev->data->nb_tx_queues; i++) { + iq_no = lio_dev->linfo.txpciq[i].s.q_no; + txq = lio_dev->instr_queue[iq_no]; + if (txq != NULL) { + iq_stats = &txq->stats; + pkts += iq_stats->tx_done; + drop += iq_stats->tx_dropped; + bytes += iq_stats->tx_tot_bytes; + } + } + + stats->opackets = pkts; + stats->obytes = bytes; + stats->oerrors = drop; + + pkts = 0; + drop = 0; + bytes = 0; + + for (i = 0; i < eth_dev->data->nb_rx_queues; i++) { + oq_no = lio_dev->linfo.rxpciq[i].s.q_no; + droq = lio_dev->droq[oq_no]; + if (droq != NULL) { + oq_stats = &droq->stats; + pkts += oq_stats->rx_pkts_received; + drop += (oq_stats->rx_dropped + + oq_stats->dropped_toomany + + oq_stats->dropped_nomem); + bytes += oq_stats->rx_bytes_received; + } + } + stats->ibytes = bytes; + stats->ipackets = pkts; + stats->ierrors = drop; + + return 0; +} + +static void +lio_dev_stats_reset(struct rte_eth_dev *eth_dev) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + struct lio_droq_stats *oq_stats; + struct lio_iq_stats *iq_stats; + struct lio_instr_queue *txq; + struct lio_droq *droq; + int i, iq_no, oq_no; + + for (i = 0; i < eth_dev->data->nb_tx_queues; i++) { + iq_no = lio_dev->linfo.txpciq[i].s.q_no; + txq = lio_dev->instr_queue[iq_no]; + if (txq != NULL) { + iq_stats = &txq->stats; + memset(iq_stats, 0, sizeof(struct lio_iq_stats)); + } + } + + for (i = 0; i < eth_dev->data->nb_rx_queues; i++) { + oq_no = lio_dev->linfo.rxpciq[i].s.q_no; + droq = lio_dev->droq[oq_no]; + if (droq != NULL) { + oq_stats = &droq->stats; + memset(oq_stats, 0, sizeof(struct lio_droq_stats)); + } + } +} + +static void +lio_dev_info_get(struct rte_eth_dev *eth_dev, + struct rte_eth_dev_info *devinfo) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); + + devinfo->pci_dev = pci_dev; + + switch (pci_dev->id.subsystem_device_id) { + /* CN23xx 10G cards */ + case PCI_SUBSYS_DEV_ID_CN2350_210: + case PCI_SUBSYS_DEV_ID_CN2360_210: + case PCI_SUBSYS_DEV_ID_CN2350_210SVPN3: + case PCI_SUBSYS_DEV_ID_CN2360_210SVPN3: + case PCI_SUBSYS_DEV_ID_CN2350_210SVPT: + case PCI_SUBSYS_DEV_ID_CN2360_210SVPT: + devinfo->speed_capa = ETH_LINK_SPEED_10G; + break; + /* CN23xx 25G cards */ + case PCI_SUBSYS_DEV_ID_CN2350_225: + case PCI_SUBSYS_DEV_ID_CN2360_225: + devinfo->speed_capa = ETH_LINK_SPEED_25G; + break; + default: + devinfo->speed_capa = ETH_LINK_SPEED_10G; + lio_dev_err(lio_dev, + "Unknown CN23XX subsystem device id. Setting 10G as default link speed.\n"); + } + + devinfo->max_rx_queues = lio_dev->max_rx_queues; + devinfo->max_tx_queues = lio_dev->max_tx_queues; + + devinfo->min_rx_bufsize = LIO_MIN_RX_BUF_SIZE; + devinfo->max_rx_pktlen = LIO_MAX_RX_PKTLEN; + + devinfo->max_mac_addrs = 1; + + devinfo->rx_offload_capa = (DEV_RX_OFFLOAD_IPV4_CKSUM | + DEV_RX_OFFLOAD_UDP_CKSUM | + DEV_RX_OFFLOAD_TCP_CKSUM | + DEV_RX_OFFLOAD_VLAN_STRIP); + devinfo->tx_offload_capa = (DEV_TX_OFFLOAD_IPV4_CKSUM | + DEV_TX_OFFLOAD_UDP_CKSUM | + DEV_TX_OFFLOAD_TCP_CKSUM | + DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM); + + devinfo->rx_desc_lim = lio_rx_desc_lim; + devinfo->tx_desc_lim = lio_tx_desc_lim; + + devinfo->reta_size = LIO_RSS_MAX_TABLE_SZ; + devinfo->hash_key_size = LIO_RSS_MAX_KEY_SZ; + devinfo->flow_type_rss_offloads = (ETH_RSS_IPV4 | + ETH_RSS_NONFRAG_IPV4_TCP | + ETH_RSS_IPV6 | + ETH_RSS_NONFRAG_IPV6_TCP | + ETH_RSS_IPV6_EX | + ETH_RSS_IPV6_TCP_EX); +} + +static int +lio_dev_mtu_set(struct rte_eth_dev *eth_dev, uint16_t mtu) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + uint16_t pf_mtu = lio_dev->linfo.link.s.mtu; + uint32_t frame_len = mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; + struct lio_dev_ctrl_cmd ctrl_cmd; + struct lio_ctrl_pkt ctrl_pkt; + + PMD_INIT_FUNC_TRACE(); + + if (!lio_dev->intf_open) { + lio_dev_err(lio_dev, "Port %d down, can't set MTU\n", + lio_dev->port_id); + return -EINVAL; + } + + /* check if VF MTU is within allowed range. + * New value should not exceed PF MTU. + */ + if ((mtu < ETHER_MIN_MTU) || (mtu > pf_mtu)) { + lio_dev_err(lio_dev, "VF MTU should be >= %d and <= %d\n", + ETHER_MIN_MTU, pf_mtu); + return -EINVAL; + } + + /* flush added to prevent cmd failure + * incase the queue is full + */ + lio_flush_iq(lio_dev, lio_dev->instr_queue[0]); + + memset(&ctrl_pkt, 0, sizeof(struct lio_ctrl_pkt)); + memset(&ctrl_cmd, 0, sizeof(struct lio_dev_ctrl_cmd)); + + ctrl_cmd.eth_dev = eth_dev; + ctrl_cmd.cond = 0; + + ctrl_pkt.ncmd.s.cmd = LIO_CMD_CHANGE_MTU; + ctrl_pkt.ncmd.s.param1 = mtu; + ctrl_pkt.ctrl_cmd = &ctrl_cmd; + + if (lio_send_ctrl_pkt(lio_dev, &ctrl_pkt)) { + lio_dev_err(lio_dev, "Failed to send command to change MTU\n"); + return -1; + } + + if (lio_wait_for_ctrl_cmd(lio_dev, &ctrl_cmd)) { + lio_dev_err(lio_dev, "Command to change MTU timed out\n"); + return -1; + } + + if (frame_len > ETHER_MAX_LEN) + eth_dev->data->dev_conf.rxmode.jumbo_frame = 1; + else + eth_dev->data->dev_conf.rxmode.jumbo_frame = 0; + + eth_dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_len; + eth_dev->data->mtu = mtu; + + return 0; +} + +static int +lio_dev_rss_reta_update(struct rte_eth_dev *eth_dev, + struct rte_eth_rss_reta_entry64 *reta_conf, + uint16_t reta_size) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + struct lio_rss_ctx *rss_state = &lio_dev->rss_state; + struct lio_rss_set *rss_param; + struct lio_dev_ctrl_cmd ctrl_cmd; + struct lio_ctrl_pkt ctrl_pkt; + int i, j, index; + + if (!lio_dev->intf_open) { + lio_dev_err(lio_dev, "Port %d down, can't update reta\n", + lio_dev->port_id); + return -EINVAL; + } + + if (reta_size != LIO_RSS_MAX_TABLE_SZ) { + lio_dev_err(lio_dev, + "The size of hash lookup table configured (%d) doesn't match the number hardware can supported (%d)\n", + reta_size, LIO_RSS_MAX_TABLE_SZ); + return -EINVAL; + } + + /* flush added to prevent cmd failure + * incase the queue is full + */ + lio_flush_iq(lio_dev, lio_dev->instr_queue[0]); + + memset(&ctrl_pkt, 0, sizeof(struct lio_ctrl_pkt)); + memset(&ctrl_cmd, 0, sizeof(struct lio_dev_ctrl_cmd)); + + rss_param = (struct lio_rss_set *)&ctrl_pkt.udd[0]; + + ctrl_cmd.eth_dev = eth_dev; + ctrl_cmd.cond = 0; + + ctrl_pkt.ncmd.s.cmd = LIO_CMD_SET_RSS; + ctrl_pkt.ncmd.s.more = sizeof(struct lio_rss_set) >> 3; + ctrl_pkt.ctrl_cmd = &ctrl_cmd; + + rss_param->param.flags = 0xF; + rss_param->param.flags &= ~LIO_RSS_PARAM_ITABLE_UNCHANGED; + rss_param->param.itablesize = LIO_RSS_MAX_TABLE_SZ; + + for (i = 0; i < (reta_size / RTE_RETA_GROUP_SIZE); i++) { + for (j = 0; j < RTE_RETA_GROUP_SIZE; j++) { + if ((reta_conf[i].mask) & ((uint64_t)1 << j)) { + index = (i * RTE_RETA_GROUP_SIZE) + j; + rss_state->itable[index] = reta_conf[i].reta[j]; + } + } + } + + rss_state->itable_size = LIO_RSS_MAX_TABLE_SZ; + memcpy(rss_param->itable, rss_state->itable, rss_state->itable_size); + + lio_swap_8B_data((uint64_t *)rss_param, LIO_RSS_PARAM_SIZE >> 3); + + if (lio_send_ctrl_pkt(lio_dev, &ctrl_pkt)) { + lio_dev_err(lio_dev, "Failed to set rss hash\n"); + return -1; + } + + if (lio_wait_for_ctrl_cmd(lio_dev, &ctrl_cmd)) { + lio_dev_err(lio_dev, "Set rss hash timed out\n"); + return -1; + } + + return 0; +} + +static int +lio_dev_rss_reta_query(struct rte_eth_dev *eth_dev, + struct rte_eth_rss_reta_entry64 *reta_conf, + uint16_t reta_size) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + struct lio_rss_ctx *rss_state = &lio_dev->rss_state; + int i, num; + + if (reta_size != LIO_RSS_MAX_TABLE_SZ) { + lio_dev_err(lio_dev, + "The size of hash lookup table configured (%d) doesn't match the number hardware can supported (%d)\n", + reta_size, LIO_RSS_MAX_TABLE_SZ); + return -EINVAL; + } + + num = reta_size / RTE_RETA_GROUP_SIZE; + + for (i = 0; i < num; i++) { + memcpy(reta_conf->reta, + &rss_state->itable[i * RTE_RETA_GROUP_SIZE], + RTE_RETA_GROUP_SIZE); + reta_conf++; + } + + return 0; +} + +static int +lio_dev_rss_hash_conf_get(struct rte_eth_dev *eth_dev, + struct rte_eth_rss_conf *rss_conf) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + struct lio_rss_ctx *rss_state = &lio_dev->rss_state; + uint8_t *hash_key = NULL; + uint64_t rss_hf = 0; + + if (rss_state->hash_disable) { + lio_dev_info(lio_dev, "RSS disabled in nic\n"); + rss_conf->rss_hf = 0; + return 0; + } + + /* Get key value */ + hash_key = rss_conf->rss_key; + if (hash_key != NULL) + memcpy(hash_key, rss_state->hash_key, rss_state->hash_key_size); + + if (rss_state->ip) + rss_hf |= ETH_RSS_IPV4; + if (rss_state->tcp_hash) + rss_hf |= ETH_RSS_NONFRAG_IPV4_TCP; + if (rss_state->ipv6) + rss_hf |= ETH_RSS_IPV6; + if (rss_state->ipv6_tcp_hash) + rss_hf |= ETH_RSS_NONFRAG_IPV6_TCP; + if (rss_state->ipv6_ex) + rss_hf |= ETH_RSS_IPV6_EX; + if (rss_state->ipv6_tcp_ex_hash) + rss_hf |= ETH_RSS_IPV6_TCP_EX; + + rss_conf->rss_hf = rss_hf; + + return 0; +} + +static int +lio_dev_rss_hash_update(struct rte_eth_dev *eth_dev, + struct rte_eth_rss_conf *rss_conf) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + struct lio_rss_ctx *rss_state = &lio_dev->rss_state; + struct lio_rss_set *rss_param; + struct lio_dev_ctrl_cmd ctrl_cmd; + struct lio_ctrl_pkt ctrl_pkt; + + if (!lio_dev->intf_open) { + lio_dev_err(lio_dev, "Port %d down, can't update hash\n", + lio_dev->port_id); + return -EINVAL; + } + + /* flush added to prevent cmd failure + * incase the queue is full + */ + lio_flush_iq(lio_dev, lio_dev->instr_queue[0]); + + memset(&ctrl_pkt, 0, sizeof(struct lio_ctrl_pkt)); + memset(&ctrl_cmd, 0, sizeof(struct lio_dev_ctrl_cmd)); + + rss_param = (struct lio_rss_set *)&ctrl_pkt.udd[0]; + + ctrl_cmd.eth_dev = eth_dev; + ctrl_cmd.cond = 0; + + ctrl_pkt.ncmd.s.cmd = LIO_CMD_SET_RSS; + ctrl_pkt.ncmd.s.more = sizeof(struct lio_rss_set) >> 3; + ctrl_pkt.ctrl_cmd = &ctrl_cmd; + + rss_param->param.flags = 0xF; + + if (rss_conf->rss_key) { + rss_param->param.flags &= ~LIO_RSS_PARAM_HASH_KEY_UNCHANGED; + rss_state->hash_key_size = LIO_RSS_MAX_KEY_SZ; + rss_param->param.hashkeysize = LIO_RSS_MAX_KEY_SZ; + memcpy(rss_state->hash_key, rss_conf->rss_key, + rss_state->hash_key_size); + memcpy(rss_param->key, rss_state->hash_key, + rss_state->hash_key_size); + } + + if ((rss_conf->rss_hf & LIO_RSS_OFFLOAD_ALL) == 0) { + /* Can't disable rss through hash flags, + * if it is enabled by default during init + */ + if (!rss_state->hash_disable) + return -EINVAL; + + /* This is for --disable-rss during testpmd launch */ + rss_param->param.flags |= LIO_RSS_PARAM_DISABLE_RSS; + } else { + uint32_t hashinfo = 0; + + /* Can't enable rss if disabled by default during init */ + if (rss_state->hash_disable) + return -EINVAL; + + if (rss_conf->rss_hf & ETH_RSS_IPV4) { + hashinfo |= LIO_RSS_HASH_IPV4; + rss_state->ip = 1; + } else { + rss_state->ip = 0; + } + + if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) { + hashinfo |= LIO_RSS_HASH_TCP_IPV4; + rss_state->tcp_hash = 1; + } else { + rss_state->tcp_hash = 0; + } + + if (rss_conf->rss_hf & ETH_RSS_IPV6) { + hashinfo |= LIO_RSS_HASH_IPV6; + rss_state->ipv6 = 1; + } else { + rss_state->ipv6 = 0; + } + + if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV6_TCP) { + hashinfo |= LIO_RSS_HASH_TCP_IPV6; + rss_state->ipv6_tcp_hash = 1; + } else { + rss_state->ipv6_tcp_hash = 0; + } + + if (rss_conf->rss_hf & ETH_RSS_IPV6_EX) { + hashinfo |= LIO_RSS_HASH_IPV6_EX; + rss_state->ipv6_ex = 1; + } else { + rss_state->ipv6_ex = 0; + } + + if (rss_conf->rss_hf & ETH_RSS_IPV6_TCP_EX) { + hashinfo |= LIO_RSS_HASH_TCP_IPV6_EX; + rss_state->ipv6_tcp_ex_hash = 1; + } else { + rss_state->ipv6_tcp_ex_hash = 0; + } + + rss_param->param.flags &= ~LIO_RSS_PARAM_HASH_INFO_UNCHANGED; + rss_param->param.hashinfo = hashinfo; + } + + lio_swap_8B_data((uint64_t *)rss_param, LIO_RSS_PARAM_SIZE >> 3); + + if (lio_send_ctrl_pkt(lio_dev, &ctrl_pkt)) { + lio_dev_err(lio_dev, "Failed to set rss hash\n"); + return -1; + } + + if (lio_wait_for_ctrl_cmd(lio_dev, &ctrl_cmd)) { + lio_dev_err(lio_dev, "Set rss hash timed out\n"); + return -1; + } + + return 0; +} + +/** + * Add vxlan dest udp port for an interface. + * + * @param eth_dev + * Pointer to the structure rte_eth_dev + * @param udp_tnl + * udp tunnel conf + * + * @return + * On success return 0 + * On failure return -1 + */ +static int +lio_dev_udp_tunnel_add(struct rte_eth_dev *eth_dev, + struct rte_eth_udp_tunnel *udp_tnl) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + struct lio_dev_ctrl_cmd ctrl_cmd; + struct lio_ctrl_pkt ctrl_pkt; + + if (udp_tnl == NULL) + return -EINVAL; + + if (udp_tnl->prot_type != RTE_TUNNEL_TYPE_VXLAN) { + lio_dev_err(lio_dev, "Unsupported tunnel type\n"); + return -1; + } + + /* flush added to prevent cmd failure + * incase the queue is full + */ + lio_flush_iq(lio_dev, lio_dev->instr_queue[0]); + + memset(&ctrl_pkt, 0, sizeof(struct lio_ctrl_pkt)); + memset(&ctrl_cmd, 0, sizeof(struct lio_dev_ctrl_cmd)); + + ctrl_cmd.eth_dev = eth_dev; + ctrl_cmd.cond = 0; + + ctrl_pkt.ncmd.s.cmd = LIO_CMD_VXLAN_PORT_CONFIG; + ctrl_pkt.ncmd.s.param1 = udp_tnl->udp_port; + ctrl_pkt.ncmd.s.more = LIO_CMD_VXLAN_PORT_ADD; + ctrl_pkt.ctrl_cmd = &ctrl_cmd; + + if (lio_send_ctrl_pkt(lio_dev, &ctrl_pkt)) { + lio_dev_err(lio_dev, "Failed to send VXLAN_PORT_ADD command\n"); + return -1; + } + + if (lio_wait_for_ctrl_cmd(lio_dev, &ctrl_cmd)) { + lio_dev_err(lio_dev, "VXLAN_PORT_ADD command timed out\n"); + return -1; + } + + return 0; +} + +/** + * Remove vxlan dest udp port for an interface. + * + * @param eth_dev + * Pointer to the structure rte_eth_dev + * @param udp_tnl + * udp tunnel conf + * + * @return + * On success return 0 + * On failure return -1 + */ +static int +lio_dev_udp_tunnel_del(struct rte_eth_dev *eth_dev, + struct rte_eth_udp_tunnel *udp_tnl) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + struct lio_dev_ctrl_cmd ctrl_cmd; + struct lio_ctrl_pkt ctrl_pkt; + + if (udp_tnl == NULL) + return -EINVAL; + + if (udp_tnl->prot_type != RTE_TUNNEL_TYPE_VXLAN) { + lio_dev_err(lio_dev, "Unsupported tunnel type\n"); + return -1; + } + + /* flush added to prevent cmd failure + * incase the queue is full + */ + lio_flush_iq(lio_dev, lio_dev->instr_queue[0]); + + memset(&ctrl_pkt, 0, sizeof(struct lio_ctrl_pkt)); + memset(&ctrl_cmd, 0, sizeof(struct lio_dev_ctrl_cmd)); + + ctrl_cmd.eth_dev = eth_dev; + ctrl_cmd.cond = 0; + + ctrl_pkt.ncmd.s.cmd = LIO_CMD_VXLAN_PORT_CONFIG; + ctrl_pkt.ncmd.s.param1 = udp_tnl->udp_port; + ctrl_pkt.ncmd.s.more = LIO_CMD_VXLAN_PORT_DEL; + ctrl_pkt.ctrl_cmd = &ctrl_cmd; + + if (lio_send_ctrl_pkt(lio_dev, &ctrl_pkt)) { + lio_dev_err(lio_dev, "Failed to send VXLAN_PORT_DEL command\n"); + return -1; + } + + if (lio_wait_for_ctrl_cmd(lio_dev, &ctrl_cmd)) { + lio_dev_err(lio_dev, "VXLAN_PORT_DEL command timed out\n"); + return -1; + } + + return 0; +} + +static int +lio_dev_vlan_filter_set(struct rte_eth_dev *eth_dev, uint16_t vlan_id, int on) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + struct lio_dev_ctrl_cmd ctrl_cmd; + struct lio_ctrl_pkt ctrl_pkt; + + if (lio_dev->linfo.vlan_is_admin_assigned) + return -EPERM; + + /* flush added to prevent cmd failure + * incase the queue is full + */ + lio_flush_iq(lio_dev, lio_dev->instr_queue[0]); + + memset(&ctrl_pkt, 0, sizeof(struct lio_ctrl_pkt)); + memset(&ctrl_cmd, 0, sizeof(struct lio_dev_ctrl_cmd)); + + ctrl_cmd.eth_dev = eth_dev; + ctrl_cmd.cond = 0; + + ctrl_pkt.ncmd.s.cmd = on ? + LIO_CMD_ADD_VLAN_FILTER : LIO_CMD_DEL_VLAN_FILTER; + ctrl_pkt.ncmd.s.param1 = vlan_id; + ctrl_pkt.ctrl_cmd = &ctrl_cmd; + + if (lio_send_ctrl_pkt(lio_dev, &ctrl_pkt)) { + lio_dev_err(lio_dev, "Failed to %s VLAN port\n", + on ? "add" : "remove"); + return -1; + } + + if (lio_wait_for_ctrl_cmd(lio_dev, &ctrl_cmd)) { + lio_dev_err(lio_dev, "Command to %s VLAN port timed out\n", + on ? "add" : "remove"); + return -1; + } + + return 0; +} + +/** + * Atomically writes the link status information into global + * structure rte_eth_dev. + * + * @param eth_dev + * - Pointer to the structure rte_eth_dev to read from. + * - Pointer to the buffer to be saved with the link status. + * + * @return + * - On success, zero. + * - On failure, negative value. + */ +static inline int +lio_dev_atomic_write_link_status(struct rte_eth_dev *eth_dev, + struct rte_eth_link *link) +{ + struct rte_eth_link *dst = ð_dev->data->dev_link; + struct rte_eth_link *src = link; + + if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst, + *(uint64_t *)src) == 0) + return -1; + + return 0; +} + +static uint64_t +lio_hweight64(uint64_t w) +{ + uint64_t res = w - ((w >> 1) & 0x5555555555555555ul); + + res = + (res & 0x3333333333333333ul) + ((res >> 2) & 0x3333333333333333ul); + res = (res + (res >> 4)) & 0x0F0F0F0F0F0F0F0Ful; + res = res + (res >> 8); + res = res + (res >> 16); + + return (res + (res >> 32)) & 0x00000000000000FFul; +} + +static int +lio_dev_link_update(struct rte_eth_dev *eth_dev, + int wait_to_complete __rte_unused) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + struct rte_eth_link link, old; + + /* Initialize */ + link.link_status = ETH_LINK_DOWN; + link.link_speed = ETH_SPEED_NUM_NONE; + link.link_duplex = ETH_LINK_HALF_DUPLEX; + link.link_autoneg = ETH_LINK_AUTONEG; + memset(&old, 0, sizeof(old)); + + /* Return what we found */ + if (lio_dev->linfo.link.s.link_up == 0) { + /* Interface is down */ + if (lio_dev_atomic_write_link_status(eth_dev, &link)) + return -1; + if (link.link_status == old.link_status) + return -1; + return 0; + } + + link.link_status = ETH_LINK_UP; /* Interface is up */ + link.link_duplex = ETH_LINK_FULL_DUPLEX; + switch (lio_dev->linfo.link.s.speed) { + case LIO_LINK_SPEED_10000: + link.link_speed = ETH_SPEED_NUM_10G; + break; + case LIO_LINK_SPEED_25000: + link.link_speed = ETH_SPEED_NUM_25G; + break; + default: + link.link_speed = ETH_SPEED_NUM_NONE; + link.link_duplex = ETH_LINK_HALF_DUPLEX; + } + + if (lio_dev_atomic_write_link_status(eth_dev, &link)) + return -1; + + if (link.link_status == old.link_status) + return -1; + + return 0; +} + +/** + * \brief Net device enable, disable allmulticast + * @param eth_dev Pointer to the structure rte_eth_dev + */ +static void +lio_change_dev_flag(struct rte_eth_dev *eth_dev) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + struct lio_dev_ctrl_cmd ctrl_cmd; + struct lio_ctrl_pkt ctrl_pkt; + + /* flush added to prevent cmd failure + * incase the queue is full + */ + lio_flush_iq(lio_dev, lio_dev->instr_queue[0]); + + memset(&ctrl_pkt, 0, sizeof(struct lio_ctrl_pkt)); + memset(&ctrl_cmd, 0, sizeof(struct lio_dev_ctrl_cmd)); + + ctrl_cmd.eth_dev = eth_dev; + ctrl_cmd.cond = 0; + + /* Create a ctrl pkt command to be sent to core app. */ + ctrl_pkt.ncmd.s.cmd = LIO_CMD_CHANGE_DEVFLAGS; + ctrl_pkt.ncmd.s.param1 = lio_dev->ifflags; + ctrl_pkt.ctrl_cmd = &ctrl_cmd; + + if (lio_send_ctrl_pkt(lio_dev, &ctrl_pkt)) { + lio_dev_err(lio_dev, "Failed to send change flag message\n"); + return; + } + + if (lio_wait_for_ctrl_cmd(lio_dev, &ctrl_cmd)) + lio_dev_err(lio_dev, "Change dev flag command timed out\n"); +} + +static void +lio_dev_promiscuous_enable(struct rte_eth_dev *eth_dev) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + + if (strcmp(lio_dev->firmware_version, LIO_VF_TRUST_MIN_VERSION) < 0) { + lio_dev_err(lio_dev, "Require firmware version >= %s\n", + LIO_VF_TRUST_MIN_VERSION); + return; + } + + if (!lio_dev->intf_open) { + lio_dev_err(lio_dev, "Port %d down, can't enable promiscuous\n", + lio_dev->port_id); + return; + } + + lio_dev->ifflags |= LIO_IFFLAG_PROMISC; + lio_change_dev_flag(eth_dev); +} + +static void +lio_dev_promiscuous_disable(struct rte_eth_dev *eth_dev) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + + if (strcmp(lio_dev->firmware_version, LIO_VF_TRUST_MIN_VERSION) < 0) { + lio_dev_err(lio_dev, "Require firmware version >= %s\n", + LIO_VF_TRUST_MIN_VERSION); + return; + } + + if (!lio_dev->intf_open) { + lio_dev_err(lio_dev, "Port %d down, can't disable promiscuous\n", + lio_dev->port_id); + return; + } + + lio_dev->ifflags &= ~LIO_IFFLAG_PROMISC; + lio_change_dev_flag(eth_dev); +} + +static void +lio_dev_allmulticast_enable(struct rte_eth_dev *eth_dev) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + + if (!lio_dev->intf_open) { + lio_dev_err(lio_dev, "Port %d down, can't enable multicast\n", + lio_dev->port_id); + return; + } + + lio_dev->ifflags |= LIO_IFFLAG_ALLMULTI; + lio_change_dev_flag(eth_dev); +} + +static void +lio_dev_allmulticast_disable(struct rte_eth_dev *eth_dev) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + + if (!lio_dev->intf_open) { + lio_dev_err(lio_dev, "Port %d down, can't disable multicast\n", + lio_dev->port_id); + return; + } + + lio_dev->ifflags &= ~LIO_IFFLAG_ALLMULTI; + lio_change_dev_flag(eth_dev); +} + +static void +lio_dev_rss_configure(struct rte_eth_dev *eth_dev) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + struct lio_rss_ctx *rss_state = &lio_dev->rss_state; + struct rte_eth_rss_reta_entry64 reta_conf[8]; + struct rte_eth_rss_conf rss_conf; + uint16_t i; + + /* Configure the RSS key and the RSS protocols used to compute + * the RSS hash of input packets. + */ + rss_conf = eth_dev->data->dev_conf.rx_adv_conf.rss_conf; + if ((rss_conf.rss_hf & LIO_RSS_OFFLOAD_ALL) == 0) { + rss_state->hash_disable = 1; + lio_dev_rss_hash_update(eth_dev, &rss_conf); + return; + } + + if (rss_conf.rss_key == NULL) + rss_conf.rss_key = lio_rss_key; /* Default hash key */ + + lio_dev_rss_hash_update(eth_dev, &rss_conf); + + memset(reta_conf, 0, sizeof(reta_conf)); + for (i = 0; i < LIO_RSS_MAX_TABLE_SZ; i++) { + uint8_t q_idx, conf_idx, reta_idx; + + q_idx = (uint8_t)((eth_dev->data->nb_rx_queues > 1) ? + i % eth_dev->data->nb_rx_queues : 0); + conf_idx = i / RTE_RETA_GROUP_SIZE; + reta_idx = i % RTE_RETA_GROUP_SIZE; + reta_conf[conf_idx].reta[reta_idx] = q_idx; + reta_conf[conf_idx].mask |= ((uint64_t)1 << reta_idx); + } + + lio_dev_rss_reta_update(eth_dev, reta_conf, LIO_RSS_MAX_TABLE_SZ); +} + +static void +lio_dev_mq_rx_configure(struct rte_eth_dev *eth_dev) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + struct lio_rss_ctx *rss_state = &lio_dev->rss_state; + struct rte_eth_rss_conf rss_conf; + + switch (eth_dev->data->dev_conf.rxmode.mq_mode) { + case ETH_MQ_RX_RSS: + lio_dev_rss_configure(eth_dev); + break; + case ETH_MQ_RX_NONE: + /* if mq_mode is none, disable rss mode. */ + default: + memset(&rss_conf, 0, sizeof(rss_conf)); + rss_state->hash_disable = 1; + lio_dev_rss_hash_update(eth_dev, &rss_conf); + } +} + +/** + * Setup our receive queue/ringbuffer. This is the + * queue the Octeon uses to send us packets and + * responses. We are given a memory pool for our + * packet buffers that are used to populate the receive + * queue. + * + * @param eth_dev + * Pointer to the structure rte_eth_dev + * @param q_no + * Queue number + * @param num_rx_descs + * Number of entries in the queue + * @param socket_id + * Where to allocate memory + * @param rx_conf + * Pointer to the struction rte_eth_rxconf + * @param mp + * Pointer to the packet pool + * + * @return + * - On success, return 0 + * - On failure, return -1 + */ +static int +lio_dev_rx_queue_setup(struct rte_eth_dev *eth_dev, uint16_t q_no, + uint16_t num_rx_descs, unsigned int socket_id, + const struct rte_eth_rxconf *rx_conf __rte_unused, + struct rte_mempool *mp) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + struct rte_pktmbuf_pool_private *mbp_priv; + uint32_t fw_mapped_oq; + uint16_t buf_size; + + if (q_no >= lio_dev->nb_rx_queues) { + lio_dev_err(lio_dev, "Invalid rx queue number %u\n", q_no); + return -EINVAL; + } + + lio_dev_dbg(lio_dev, "setting up rx queue %u\n", q_no); + + fw_mapped_oq = lio_dev->linfo.rxpciq[q_no].s.q_no; + + if ((lio_dev->droq[fw_mapped_oq]) && + (num_rx_descs != lio_dev->droq[fw_mapped_oq]->max_count)) { + lio_dev_err(lio_dev, + "Reconfiguring Rx descs not supported. Configure descs to same value %u or restart application\n", + lio_dev->droq[fw_mapped_oq]->max_count); + return -ENOTSUP; + } + + mbp_priv = rte_mempool_get_priv(mp); + buf_size = mbp_priv->mbuf_data_room_size - RTE_PKTMBUF_HEADROOM; + + if (lio_setup_droq(lio_dev, fw_mapped_oq, num_rx_descs, buf_size, mp, + socket_id)) { + lio_dev_err(lio_dev, "droq allocation failed\n"); + return -1; + } + + eth_dev->data->rx_queues[q_no] = lio_dev->droq[fw_mapped_oq]; + + return 0; +} + +/** + * Release the receive queue/ringbuffer. Called by + * the upper layers. + * + * @param rxq + * Opaque pointer to the receive queue to release + * + * @return + * - nothing + */ +void +lio_dev_rx_queue_release(void *rxq) +{ + struct lio_droq *droq = rxq; + int oq_no; + + if (droq) { + /* Run time queue deletion not supported */ + if (droq->lio_dev->port_configured) + return; + + oq_no = droq->q_no; + lio_delete_droq_queue(droq->lio_dev, oq_no); + } +} + +/** + * Allocate and initialize SW ring. Initialize associated HW registers. + * + * @param eth_dev + * Pointer to structure rte_eth_dev + * + * @param q_no + * Queue number + * + * @param num_tx_descs + * Number of ringbuffer descriptors + * + * @param socket_id + * NUMA socket id, used for memory allocations + * + * @param tx_conf + * Pointer to the structure rte_eth_txconf + * + * @return + * - On success, return 0 + * - On failure, return -errno value + */ +static int +lio_dev_tx_queue_setup(struct rte_eth_dev *eth_dev, uint16_t q_no, + uint16_t num_tx_descs, unsigned int socket_id, + const struct rte_eth_txconf *tx_conf __rte_unused) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + int fw_mapped_iq = lio_dev->linfo.txpciq[q_no].s.q_no; + int retval; + + if (q_no >= lio_dev->nb_tx_queues) { + lio_dev_err(lio_dev, "Invalid tx queue number %u\n", q_no); + return -EINVAL; + } + + lio_dev_dbg(lio_dev, "setting up tx queue %u\n", q_no); + + if ((lio_dev->instr_queue[fw_mapped_iq] != NULL) && + (num_tx_descs != lio_dev->instr_queue[fw_mapped_iq]->max_count)) { + lio_dev_err(lio_dev, + "Reconfiguring Tx descs not supported. Configure descs to same value %u or restart application\n", + lio_dev->instr_queue[fw_mapped_iq]->max_count); + return -ENOTSUP; + } + + retval = lio_setup_iq(lio_dev, q_no, lio_dev->linfo.txpciq[q_no], + num_tx_descs, lio_dev, socket_id); + + if (retval) { + lio_dev_err(lio_dev, "Runtime IQ(TxQ) creation failed.\n"); + return retval; + } + + retval = lio_setup_sglists(lio_dev, q_no, fw_mapped_iq, + lio_dev->instr_queue[fw_mapped_iq]->max_count, + socket_id); + + if (retval) { + lio_delete_instruction_queue(lio_dev, fw_mapped_iq); + return retval; + } + + eth_dev->data->tx_queues[q_no] = lio_dev->instr_queue[fw_mapped_iq]; + + return 0; +} + +/** + * Release the transmit queue/ringbuffer. Called by + * the upper layers. + * + * @param txq + * Opaque pointer to the transmit queue to release + * + * @return + * - nothing + */ +void +lio_dev_tx_queue_release(void *txq) +{ + struct lio_instr_queue *tq = txq; + uint32_t fw_mapped_iq_no; + + + if (tq) { + /* Run time queue deletion not supported */ + if (tq->lio_dev->port_configured) + return; + + /* Free sg_list */ + lio_delete_sglist(tq); + + fw_mapped_iq_no = tq->txpciq.s.q_no; + lio_delete_instruction_queue(tq->lio_dev, fw_mapped_iq_no); + } +} + +/** + * Api to check link state. + */ +static void +lio_dev_get_link_status(struct rte_eth_dev *eth_dev) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + uint16_t timeout = LIO_MAX_CMD_TIMEOUT; + struct lio_link_status_resp *resp; + union octeon_link_status *ls; + struct lio_soft_command *sc; + uint32_t resp_size; + + if (!lio_dev->intf_open) + return; + + resp_size = sizeof(struct lio_link_status_resp); + sc = lio_alloc_soft_command(lio_dev, 0, resp_size, 0); + if (sc == NULL) + return; + + resp = (struct lio_link_status_resp *)sc->virtrptr; + lio_prepare_soft_command(lio_dev, sc, LIO_OPCODE, + LIO_OPCODE_INFO, 0, 0, 0); + + /* Setting wait time in seconds */ + sc->wait_time = LIO_MAX_CMD_TIMEOUT / 1000; + + if (lio_send_soft_command(lio_dev, sc) == LIO_IQ_SEND_FAILED) + goto get_status_fail; + + while ((*sc->status_word == LIO_COMPLETION_WORD_INIT) && --timeout) { + lio_flush_iq(lio_dev, lio_dev->instr_queue[sc->iq_no]); + rte_delay_ms(1); + } + + if (resp->status) + goto get_status_fail; + + ls = &resp->link_info.link; + + lio_swap_8B_data((uint64_t *)ls, sizeof(union octeon_link_status) >> 3); + + if (lio_dev->linfo.link.link_status64 != ls->link_status64) { + if (ls->s.mtu < eth_dev->data->mtu) { + lio_dev_info(lio_dev, "Lowered VF MTU to %d as PF MTU dropped\n", + ls->s.mtu); + eth_dev->data->mtu = ls->s.mtu; + } + lio_dev->linfo.link.link_status64 = ls->link_status64; + lio_dev_link_update(eth_dev, 0); + } + + lio_free_soft_command(sc); + + return; + +get_status_fail: + lio_free_soft_command(sc); +} + +/* This function will be invoked every LSC_TIMEOUT ns (100ms) + * and will update link state if it changes. + */ +static void +lio_sync_link_state_check(void *eth_dev) +{ + struct lio_device *lio_dev = + (((struct rte_eth_dev *)eth_dev)->data->dev_private); + + if (lio_dev->port_configured) + lio_dev_get_link_status(eth_dev); + + /* Schedule periodic link status check. + * Stop check if interface is close and start again while opening. + */ + if (lio_dev->intf_open) + rte_eal_alarm_set(LIO_LSC_TIMEOUT, lio_sync_link_state_check, + eth_dev); +} + +static int +lio_dev_start(struct rte_eth_dev *eth_dev) +{ + uint16_t mtu; + uint32_t frame_len = eth_dev->data->dev_conf.rxmode.max_rx_pkt_len; + struct lio_device *lio_dev = LIO_DEV(eth_dev); + uint16_t timeout = LIO_MAX_CMD_TIMEOUT; + int ret = 0; + + lio_dev_info(lio_dev, "Starting port %d\n", eth_dev->data->port_id); + + if (lio_dev->fn_list.enable_io_queues(lio_dev)) + return -1; + + if (lio_send_rx_ctrl_cmd(eth_dev, 1)) + return -1; + + /* Ready for link status updates */ + lio_dev->intf_open = 1; + rte_mb(); + + /* Configure RSS if device configured with multiple RX queues. */ + lio_dev_mq_rx_configure(eth_dev); + + /* start polling for lsc */ + ret = rte_eal_alarm_set(LIO_LSC_TIMEOUT, + lio_sync_link_state_check, + eth_dev); + if (ret) { + lio_dev_err(lio_dev, + "link state check handler creation failed\n"); + goto dev_lsc_handle_error; + } + + while ((lio_dev->linfo.link.link_status64 == 0) && (--timeout)) + rte_delay_ms(1); + + if (lio_dev->linfo.link.link_status64 == 0) { + ret = -1; + goto dev_mtu_set_error; + } + + mtu = (uint16_t)(frame_len - ETHER_HDR_LEN - ETHER_CRC_LEN); + if (mtu < ETHER_MIN_MTU) + mtu = ETHER_MIN_MTU; + + if (eth_dev->data->mtu != mtu) { + ret = lio_dev_mtu_set(eth_dev, mtu); + if (ret) + goto dev_mtu_set_error; + } + + return 0; + +dev_mtu_set_error: + rte_eal_alarm_cancel(lio_sync_link_state_check, eth_dev); + +dev_lsc_handle_error: + lio_dev->intf_open = 0; + lio_send_rx_ctrl_cmd(eth_dev, 0); + + return ret; +} + +/* Stop device and disable input/output functions */ +static void +lio_dev_stop(struct rte_eth_dev *eth_dev) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + + lio_dev_info(lio_dev, "Stopping port %d\n", eth_dev->data->port_id); + lio_dev->intf_open = 0; + rte_mb(); + + /* Cancel callback if still running. */ + rte_eal_alarm_cancel(lio_sync_link_state_check, eth_dev); + + lio_send_rx_ctrl_cmd(eth_dev, 0); + + /* Clear recorded link status */ + lio_dev->linfo.link.link_status64 = 0; +} + +static int +lio_dev_set_link_up(struct rte_eth_dev *eth_dev) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + + if (!lio_dev->intf_open) { + lio_dev_info(lio_dev, "Port is stopped, Start the port first\n"); + return 0; + } + + if (lio_dev->linfo.link.s.link_up) { + lio_dev_info(lio_dev, "Link is already UP\n"); + return 0; + } + + if (lio_send_rx_ctrl_cmd(eth_dev, 1)) { + lio_dev_err(lio_dev, "Unable to set Link UP\n"); + return -1; + } + + lio_dev->linfo.link.s.link_up = 1; + eth_dev->data->dev_link.link_status = ETH_LINK_UP; + + return 0; +} + +static int +lio_dev_set_link_down(struct rte_eth_dev *eth_dev) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + + if (!lio_dev->intf_open) { + lio_dev_info(lio_dev, "Port is stopped, Start the port first\n"); + return 0; + } + + if (!lio_dev->linfo.link.s.link_up) { + lio_dev_info(lio_dev, "Link is already DOWN\n"); + return 0; + } + + lio_dev->linfo.link.s.link_up = 0; + eth_dev->data->dev_link.link_status = ETH_LINK_DOWN; + + if (lio_send_rx_ctrl_cmd(eth_dev, 0)) { + lio_dev->linfo.link.s.link_up = 1; + eth_dev->data->dev_link.link_status = ETH_LINK_UP; + lio_dev_err(lio_dev, "Unable to set Link Down\n"); + return -1; + } + + return 0; +} + +/** + * Reset and stop the device. This occurs on the first + * call to this routine. Subsequent calls will simply + * return. NB: This will require the NIC to be rebooted. + * + * @param eth_dev + * Pointer to the structure rte_eth_dev + * + * @return + * - nothing + */ +static void +lio_dev_close(struct rte_eth_dev *eth_dev) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + uint32_t i; + + lio_dev_info(lio_dev, "closing port %d\n", eth_dev->data->port_id); + + if (lio_dev->intf_open) + lio_dev_stop(eth_dev); + + lio_wait_for_instr_fetch(lio_dev); + + lio_dev->fn_list.disable_io_queues(lio_dev); + + cn23xx_vf_set_io_queues_off(lio_dev); + + /* Reset iq regs (IQ_DBELL). + * Clear sli_pktx_cnts (OQ_PKTS_SENT). + */ + for (i = 0; i < lio_dev->nb_rx_queues; i++) { + struct lio_droq *droq = lio_dev->droq[i]; + + if (droq == NULL) + break; + + uint32_t pkt_count = rte_read32(droq->pkts_sent_reg); + + lio_dev_dbg(lio_dev, + "pending oq count %u\n", pkt_count); + rte_write32(pkt_count, droq->pkts_sent_reg); + } + + if (lio_dev->pci_dev->kdrv == RTE_KDRV_IGB_UIO) { + cn23xx_vf_ask_pf_to_do_flr(lio_dev); + rte_delay_ms(LIO_PCI_FLR_WAIT); + } + + /* lio_free_mbox */ + lio_dev->fn_list.free_mbox(lio_dev); + + /* Free glist resources */ + rte_free(lio_dev->glist_head); + rte_free(lio_dev->glist_lock); + lio_dev->glist_head = NULL; + lio_dev->glist_lock = NULL; + + lio_dev->port_configured = 0; + + /* Delete all queues */ + lio_dev_clear_queues(eth_dev); +} + +/** + * Enable tunnel rx checksum verification from firmware. + */ +static void +lio_enable_hw_tunnel_rx_checksum(struct rte_eth_dev *eth_dev) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + struct lio_dev_ctrl_cmd ctrl_cmd; + struct lio_ctrl_pkt ctrl_pkt; + + /* flush added to prevent cmd failure + * incase the queue is full + */ + lio_flush_iq(lio_dev, lio_dev->instr_queue[0]); + + memset(&ctrl_pkt, 0, sizeof(struct lio_ctrl_pkt)); + memset(&ctrl_cmd, 0, sizeof(struct lio_dev_ctrl_cmd)); + + ctrl_cmd.eth_dev = eth_dev; + ctrl_cmd.cond = 0; + + ctrl_pkt.ncmd.s.cmd = LIO_CMD_TNL_RX_CSUM_CTL; + ctrl_pkt.ncmd.s.param1 = LIO_CMD_RXCSUM_ENABLE; + ctrl_pkt.ctrl_cmd = &ctrl_cmd; + + if (lio_send_ctrl_pkt(lio_dev, &ctrl_pkt)) { + lio_dev_err(lio_dev, "Failed to send TNL_RX_CSUM command\n"); + return; + } + + if (lio_wait_for_ctrl_cmd(lio_dev, &ctrl_cmd)) + lio_dev_err(lio_dev, "TNL_RX_CSUM command timed out\n"); +} + +/** + * Enable checksum calculation for inner packet in a tunnel. + */ +static void +lio_enable_hw_tunnel_tx_checksum(struct rte_eth_dev *eth_dev) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + struct lio_dev_ctrl_cmd ctrl_cmd; + struct lio_ctrl_pkt ctrl_pkt; + + /* flush added to prevent cmd failure + * incase the queue is full + */ + lio_flush_iq(lio_dev, lio_dev->instr_queue[0]); + + memset(&ctrl_pkt, 0, sizeof(struct lio_ctrl_pkt)); + memset(&ctrl_cmd, 0, sizeof(struct lio_dev_ctrl_cmd)); + + ctrl_cmd.eth_dev = eth_dev; + ctrl_cmd.cond = 0; + + ctrl_pkt.ncmd.s.cmd = LIO_CMD_TNL_TX_CSUM_CTL; + ctrl_pkt.ncmd.s.param1 = LIO_CMD_TXCSUM_ENABLE; + ctrl_pkt.ctrl_cmd = &ctrl_cmd; + + if (lio_send_ctrl_pkt(lio_dev, &ctrl_pkt)) { + lio_dev_err(lio_dev, "Failed to send TNL_TX_CSUM command\n"); + return; + } + + if (lio_wait_for_ctrl_cmd(lio_dev, &ctrl_cmd)) + lio_dev_err(lio_dev, "TNL_TX_CSUM command timed out\n"); +} + +static int lio_dev_configure(struct rte_eth_dev *eth_dev) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + uint16_t timeout = LIO_MAX_CMD_TIMEOUT; + int retval, num_iqueues, num_oqueues; + uint8_t mac[ETHER_ADDR_LEN], i; + struct lio_if_cfg_resp *resp; + struct lio_soft_command *sc; + union lio_if_cfg if_cfg; + uint32_t resp_size; + + PMD_INIT_FUNC_TRACE(); + + /* Re-configuring firmware not supported. + * Can't change tx/rx queues per port from initial value. + */ + if (lio_dev->port_configured) { + if ((lio_dev->nb_rx_queues != eth_dev->data->nb_rx_queues) || + (lio_dev->nb_tx_queues != eth_dev->data->nb_tx_queues)) { + lio_dev_err(lio_dev, + "rxq/txq re-conf not supported. Restart application with new value.\n"); + return -ENOTSUP; + } + return 0; + } + + lio_dev->nb_rx_queues = eth_dev->data->nb_rx_queues; + lio_dev->nb_tx_queues = eth_dev->data->nb_tx_queues; + + resp_size = sizeof(struct lio_if_cfg_resp); + sc = lio_alloc_soft_command(lio_dev, 0, resp_size, 0); + if (sc == NULL) + return -ENOMEM; + + resp = (struct lio_if_cfg_resp *)sc->virtrptr; + + /* Firmware doesn't have capability to reconfigure the queues, + * Claim all queues, and use as many required + */ + if_cfg.if_cfg64 = 0; + if_cfg.s.num_iqueues = lio_dev->nb_tx_queues; + if_cfg.s.num_oqueues = lio_dev->nb_rx_queues; + if_cfg.s.base_queue = 0; + + if_cfg.s.gmx_port_id = lio_dev->pf_num; + + lio_prepare_soft_command(lio_dev, sc, LIO_OPCODE, + LIO_OPCODE_IF_CFG, 0, + if_cfg.if_cfg64, 0); + + /* Setting wait time in seconds */ + sc->wait_time = LIO_MAX_CMD_TIMEOUT / 1000; + + retval = lio_send_soft_command(lio_dev, sc); + if (retval == LIO_IQ_SEND_FAILED) { + lio_dev_err(lio_dev, "iq/oq config failed status: %x\n", + retval); + /* Soft instr is freed by driver in case of failure. */ + goto nic_config_fail; + } + + /* Sleep on a wait queue till the cond flag indicates that the + * response arrived or timed-out. + */ + while ((*sc->status_word == LIO_COMPLETION_WORD_INIT) && --timeout) { + lio_flush_iq(lio_dev, lio_dev->instr_queue[sc->iq_no]); + lio_process_ordered_list(lio_dev); + rte_delay_ms(1); + } + + retval = resp->status; + if (retval) { + lio_dev_err(lio_dev, "iq/oq config failed\n"); + goto nic_config_fail; + } + + snprintf(lio_dev->firmware_version, LIO_FW_VERSION_LENGTH, "%s", + resp->cfg_info.lio_firmware_version); + + lio_swap_8B_data((uint64_t *)(&resp->cfg_info), + sizeof(struct octeon_if_cfg_info) >> 3); + + num_iqueues = lio_hweight64(resp->cfg_info.iqmask); + num_oqueues = lio_hweight64(resp->cfg_info.oqmask); + + if (!(num_iqueues) || !(num_oqueues)) { + lio_dev_err(lio_dev, + "Got bad iqueues (%016lx) or oqueues (%016lx) from firmware.\n", + (unsigned long)resp->cfg_info.iqmask, + (unsigned long)resp->cfg_info.oqmask); + goto nic_config_fail; + } + + lio_dev_dbg(lio_dev, + "interface %d, iqmask %016lx, oqmask %016lx, numiqueues %d, numoqueues %d\n", + eth_dev->data->port_id, + (unsigned long)resp->cfg_info.iqmask, + (unsigned long)resp->cfg_info.oqmask, + num_iqueues, num_oqueues); + + lio_dev->linfo.num_rxpciq = num_oqueues; + lio_dev->linfo.num_txpciq = num_iqueues; + + for (i = 0; i < num_oqueues; i++) { + lio_dev->linfo.rxpciq[i].rxpciq64 = + resp->cfg_info.linfo.rxpciq[i].rxpciq64; + lio_dev_dbg(lio_dev, "index %d OQ %d\n", + i, lio_dev->linfo.rxpciq[i].s.q_no); + } + + for (i = 0; i < num_iqueues; i++) { + lio_dev->linfo.txpciq[i].txpciq64 = + resp->cfg_info.linfo.txpciq[i].txpciq64; + lio_dev_dbg(lio_dev, "index %d IQ %d\n", + i, lio_dev->linfo.txpciq[i].s.q_no); + } + + lio_dev->linfo.hw_addr = resp->cfg_info.linfo.hw_addr; + lio_dev->linfo.gmxport = resp->cfg_info.linfo.gmxport; + lio_dev->linfo.link.link_status64 = + resp->cfg_info.linfo.link.link_status64; + + /* 64-bit swap required on LE machines */ + lio_swap_8B_data(&lio_dev->linfo.hw_addr, 1); + for (i = 0; i < ETHER_ADDR_LEN; i++) + mac[i] = *((uint8_t *)(((uint8_t *)&lio_dev->linfo.hw_addr) + + 2 + i)); + + /* Copy the permanent MAC address */ + ether_addr_copy((struct ether_addr *)mac, ð_dev->data->mac_addrs[0]); + + /* enable firmware checksum support for tunnel packets */ + lio_enable_hw_tunnel_rx_checksum(eth_dev); + lio_enable_hw_tunnel_tx_checksum(eth_dev); + + lio_dev->glist_lock = + rte_zmalloc(NULL, sizeof(*lio_dev->glist_lock) * num_iqueues, 0); + if (lio_dev->glist_lock == NULL) + return -ENOMEM; + + lio_dev->glist_head = + rte_zmalloc(NULL, sizeof(*lio_dev->glist_head) * num_iqueues, + 0); + if (lio_dev->glist_head == NULL) { + rte_free(lio_dev->glist_lock); + lio_dev->glist_lock = NULL; + return -ENOMEM; + } + + lio_dev_link_update(eth_dev, 0); + + lio_dev->port_configured = 1; + + lio_free_soft_command(sc); + + /* Disable iq_0 for reconf */ + lio_dev->fn_list.disable_io_queues(lio_dev); + + /* Reset ioq regs */ + lio_dev->fn_list.setup_device_regs(lio_dev); + + /* Free iq_0 used during init */ + lio_free_instr_queue0(lio_dev); + + return 0; + +nic_config_fail: + lio_dev_err(lio_dev, "Failed retval %d\n", retval); + lio_free_soft_command(sc); + lio_free_instr_queue0(lio_dev); + + return -ENODEV; +} + +/* Define our ethernet definitions */ +static const struct eth_dev_ops liovf_eth_dev_ops = { + .dev_configure = lio_dev_configure, + .dev_start = lio_dev_start, + .dev_stop = lio_dev_stop, + .dev_set_link_up = lio_dev_set_link_up, + .dev_set_link_down = lio_dev_set_link_down, + .dev_close = lio_dev_close, + .promiscuous_enable = lio_dev_promiscuous_enable, + .promiscuous_disable = lio_dev_promiscuous_disable, + .allmulticast_enable = lio_dev_allmulticast_enable, + .allmulticast_disable = lio_dev_allmulticast_disable, + .link_update = lio_dev_link_update, + .stats_get = lio_dev_stats_get, + .xstats_get = lio_dev_xstats_get, + .xstats_get_names = lio_dev_xstats_get_names, + .stats_reset = lio_dev_stats_reset, + .xstats_reset = lio_dev_xstats_reset, + .dev_infos_get = lio_dev_info_get, + .vlan_filter_set = lio_dev_vlan_filter_set, + .rx_queue_setup = lio_dev_rx_queue_setup, + .rx_queue_release = lio_dev_rx_queue_release, + .tx_queue_setup = lio_dev_tx_queue_setup, + .tx_queue_release = lio_dev_tx_queue_release, + .reta_update = lio_dev_rss_reta_update, + .reta_query = lio_dev_rss_reta_query, + .rss_hash_conf_get = lio_dev_rss_hash_conf_get, + .rss_hash_update = lio_dev_rss_hash_update, + .udp_tunnel_port_add = lio_dev_udp_tunnel_add, + .udp_tunnel_port_del = lio_dev_udp_tunnel_del, + .mtu_set = lio_dev_mtu_set, +}; + +static void +lio_check_pf_hs_response(void *lio_dev) +{ + struct lio_device *dev = lio_dev; + + /* check till response arrives */ + if (dev->pfvf_hsword.coproc_tics_per_us) + return; + + cn23xx_vf_handle_mbox(dev); + + rte_eal_alarm_set(1, lio_check_pf_hs_response, lio_dev); +} + +/** + * \brief Identify the LIO device and to map the BAR address space + * @param lio_dev lio device + */ +static int +lio_chip_specific_setup(struct lio_device *lio_dev) +{ + struct rte_pci_device *pdev = lio_dev->pci_dev; + uint32_t dev_id = pdev->id.device_id; + const char *s; + int ret = 1; + + switch (dev_id) { + case LIO_CN23XX_VF_VID: + lio_dev->chip_id = LIO_CN23XX_VF_VID; + ret = cn23xx_vf_setup_device(lio_dev); + s = "CN23XX VF"; + break; + default: + s = "?"; + lio_dev_err(lio_dev, "Unsupported Chip\n"); + } + + if (!ret) + lio_dev_info(lio_dev, "DEVICE : %s\n", s); + + return ret; +} + +static int +lio_first_time_init(struct lio_device *lio_dev, + struct rte_pci_device *pdev) +{ + int dpdk_queues; + + PMD_INIT_FUNC_TRACE(); + + /* set dpdk specific pci device pointer */ + lio_dev->pci_dev = pdev; + + /* Identify the LIO type and set device ops */ + if (lio_chip_specific_setup(lio_dev)) { + lio_dev_err(lio_dev, "Chip specific setup failed\n"); + return -1; + } + + /* Initialize soft command buffer pool */ + if (lio_setup_sc_buffer_pool(lio_dev)) { + lio_dev_err(lio_dev, "sc buffer pool allocation failed\n"); + return -1; + } + + /* Initialize lists to manage the requests of different types that + * arrive from applications for this lio device. + */ + lio_setup_response_list(lio_dev); + + if (lio_dev->fn_list.setup_mbox(lio_dev)) { + lio_dev_err(lio_dev, "Mailbox setup failed\n"); + goto error; + } + + /* Check PF response */ + lio_check_pf_hs_response((void *)lio_dev); + + /* Do handshake and exit if incompatible PF driver */ + if (cn23xx_pfvf_handshake(lio_dev)) + goto error; + + /* Request and wait for device reset. */ + if (pdev->kdrv == RTE_KDRV_IGB_UIO) { + cn23xx_vf_ask_pf_to_do_flr(lio_dev); + /* FLR wait time doubled as a precaution. */ + rte_delay_ms(LIO_PCI_FLR_WAIT * 2); + } + + if (cn23xx_vf_set_io_queues_off(lio_dev)) { + lio_dev_err(lio_dev, "Setting io queues off failed\n"); + goto error; + } + + if (lio_dev->fn_list.setup_device_regs(lio_dev)) { + lio_dev_err(lio_dev, "Failed to configure device registers\n"); + goto error; + } + + if (lio_setup_instr_queue0(lio_dev)) { + lio_dev_err(lio_dev, "Failed to setup instruction queue 0\n"); + goto error; + } + + dpdk_queues = (int)lio_dev->sriov_info.rings_per_vf; + + lio_dev->max_tx_queues = dpdk_queues; + lio_dev->max_rx_queues = dpdk_queues; + + /* Enable input and output queues for this device */ + if (lio_dev->fn_list.enable_io_queues(lio_dev)) + goto error; + + return 0; + +error: + lio_free_sc_buffer_pool(lio_dev); + if (lio_dev->mbox[0]) + lio_dev->fn_list.free_mbox(lio_dev); + if (lio_dev->instr_queue[0]) + lio_free_instr_queue0(lio_dev); + + return -1; +} + +static int +lio_eth_dev_uninit(struct rte_eth_dev *eth_dev) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + + PMD_INIT_FUNC_TRACE(); + + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return -EPERM; + + /* lio_free_sc_buffer_pool */ + lio_free_sc_buffer_pool(lio_dev); + + rte_free(eth_dev->data->mac_addrs); + eth_dev->data->mac_addrs = NULL; + + eth_dev->dev_ops = NULL; + eth_dev->rx_pkt_burst = NULL; + eth_dev->tx_pkt_burst = NULL; + + return 0; +} + +static int +lio_eth_dev_init(struct rte_eth_dev *eth_dev) +{ + struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(eth_dev); + struct lio_device *lio_dev = LIO_DEV(eth_dev); + + PMD_INIT_FUNC_TRACE(); + + eth_dev->rx_pkt_burst = &lio_dev_recv_pkts; + eth_dev->tx_pkt_burst = &lio_dev_xmit_pkts; + + /* Primary does the initialization. */ + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return 0; + + rte_eth_copy_pci_info(eth_dev, pdev); + + if (pdev->mem_resource[0].addr) { + lio_dev->hw_addr = pdev->mem_resource[0].addr; + } else { + PMD_INIT_LOG(ERR, "ERROR: Failed to map BAR0\n"); + return -ENODEV; + } + + lio_dev->eth_dev = eth_dev; + /* set lio device print string */ + snprintf(lio_dev->dev_string, sizeof(lio_dev->dev_string), + "%s[%02x:%02x.%x]", pdev->driver->driver.name, + pdev->addr.bus, pdev->addr.devid, pdev->addr.function); + + lio_dev->port_id = eth_dev->data->port_id; + + if (lio_first_time_init(lio_dev, pdev)) { + lio_dev_err(lio_dev, "Device init failed\n"); + return -EINVAL; + } + + eth_dev->dev_ops = &liovf_eth_dev_ops; + eth_dev->data->mac_addrs = rte_zmalloc("lio", ETHER_ADDR_LEN, 0); + if (eth_dev->data->mac_addrs == NULL) { + lio_dev_err(lio_dev, + "MAC addresses memory allocation failed\n"); + eth_dev->dev_ops = NULL; + eth_dev->rx_pkt_burst = NULL; + eth_dev->tx_pkt_burst = NULL; + return -ENOMEM; + } + + rte_atomic64_set(&lio_dev->status, LIO_DEV_RUNNING); + rte_wmb(); + + lio_dev->port_configured = 0; + /* Always allow unicast packets */ + lio_dev->ifflags |= LIO_IFFLAG_UNICAST; + + return 0; +} + +static int +lio_eth_dev_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, + struct rte_pci_device *pci_dev) +{ + struct rte_eth_dev *eth_dev; + int ret; + + eth_dev = rte_eth_dev_pci_allocate(pci_dev, + sizeof(struct lio_device)); + if (eth_dev == NULL) + return -ENOMEM; + + ret = lio_eth_dev_init(eth_dev); + if (ret) + rte_eth_dev_pci_release(eth_dev); + + return ret; +} + +static int +lio_eth_dev_pci_remove(struct rte_pci_device *pci_dev) +{ + return rte_eth_dev_pci_generic_remove(pci_dev, + lio_eth_dev_uninit); +} + +/* Set of PCI devices this driver supports */ +static const struct rte_pci_id pci_id_liovf_map[] = { + { RTE_PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, LIO_CN23XX_VF_VID) }, + { .vendor_id = 0, /* sentinel */ } +}; + +static struct rte_pci_driver rte_liovf_pmd = { + .id_table = pci_id_liovf_map, + .drv_flags = RTE_PCI_DRV_NEED_MAPPING, + .probe = lio_eth_dev_pci_probe, + .remove = lio_eth_dev_pci_remove, +}; + +RTE_PMD_REGISTER_PCI(net_liovf, rte_liovf_pmd); +RTE_PMD_REGISTER_PCI_TABLE(net_liovf, pci_id_liovf_map); +RTE_PMD_REGISTER_KMOD_DEP(net_liovf, "* igb_uio | vfio-pci"); diff --git a/dpdk/drivers/net/liquidio/lio_ethdev.h b/dpdk/drivers/net/liquidio/lio_ethdev.h new file mode 100644 index 00000000..655c2011 --- /dev/null +++ b/dpdk/drivers/net/liquidio/lio_ethdev.h @@ -0,0 +1,205 @@ +/* + * BSD LICENSE + * + * Copyright(c) 2017 Cavium, Inc.. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _LIO_ETHDEV_H_ +#define _LIO_ETHDEV_H_ + +#include + +#include "lio_struct.h" + +/* timeout to check link state updates from firmware in us */ +#define LIO_LSC_TIMEOUT 100000 /* 100000us (100ms) */ +#define LIO_MAX_CMD_TIMEOUT 10000 /* 10000ms (10s) */ + +#define LIO_DEV(_eth_dev) ((_eth_dev)->data->dev_private) + +/* LIO Response condition variable */ +struct lio_dev_ctrl_cmd { + struct rte_eth_dev *eth_dev; + uint64_t cond; +}; + +enum lio_bus_speed { + LIO_LINK_SPEED_UNKNOWN = 0, + LIO_LINK_SPEED_10000 = 10000, + LIO_LINK_SPEED_25000 = 25000 +}; + +struct octeon_if_cfg_info { + uint64_t iqmask; /** mask for IQs enabled for the port */ + uint64_t oqmask; /** mask for OQs enabled for the port */ + struct octeon_link_info linfo; /** initial link information */ + char lio_firmware_version[LIO_FW_VERSION_LENGTH]; +}; + +/** Stats for each NIC port in RX direction. */ +struct octeon_rx_stats { + /* link-level stats */ + uint64_t total_rcvd; + uint64_t bytes_rcvd; + uint64_t total_bcst; + uint64_t total_mcst; + uint64_t runts; + uint64_t ctl_rcvd; + uint64_t fifo_err; /* Accounts for over/under-run of buffers */ + uint64_t dmac_drop; + uint64_t fcs_err; + uint64_t jabber_err; + uint64_t l2_err; + uint64_t frame_err; + + /* firmware stats */ + uint64_t fw_total_rcvd; + uint64_t fw_total_fwd; + uint64_t fw_total_fwd_bytes; + uint64_t fw_err_pko; + uint64_t fw_err_link; + uint64_t fw_err_drop; + uint64_t fw_rx_vxlan; + uint64_t fw_rx_vxlan_err; + + /* LRO */ + uint64_t fw_lro_pkts; /* Number of packets that are LROed */ + uint64_t fw_lro_octs; /* Number of octets that are LROed */ + uint64_t fw_total_lro; /* Number of LRO packets formed */ + uint64_t fw_lro_aborts; /* Number of times lRO of packet aborted */ + uint64_t fw_lro_aborts_port; + uint64_t fw_lro_aborts_seq; + uint64_t fw_lro_aborts_tsval; + uint64_t fw_lro_aborts_timer; + /* intrmod: packet forward rate */ + uint64_t fwd_rate; +}; + +/** Stats for each NIC port in RX direction. */ +struct octeon_tx_stats { + /* link-level stats */ + uint64_t total_pkts_sent; + uint64_t total_bytes_sent; + uint64_t mcast_pkts_sent; + uint64_t bcast_pkts_sent; + uint64_t ctl_sent; + uint64_t one_collision_sent; /* Packets sent after one collision */ + /* Packets sent after multiple collision */ + uint64_t multi_collision_sent; + /* Packets not sent due to max collisions */ + uint64_t max_collision_fail; + /* Packets not sent due to max deferrals */ + uint64_t max_deferral_fail; + /* Accounts for over/under-run of buffers */ + uint64_t fifo_err; + uint64_t runts; + uint64_t total_collisions; /* Total number of collisions detected */ + + /* firmware stats */ + uint64_t fw_total_sent; + uint64_t fw_total_fwd; + uint64_t fw_total_fwd_bytes; + uint64_t fw_err_pko; + uint64_t fw_err_link; + uint64_t fw_err_drop; + uint64_t fw_err_tso; + uint64_t fw_tso; /* number of tso requests */ + uint64_t fw_tso_fwd; /* number of packets segmented in tso */ + uint64_t fw_tx_vxlan; +}; + +struct octeon_link_stats { + struct octeon_rx_stats fromwire; + struct octeon_tx_stats fromhost; +}; + +union lio_if_cfg { + uint64_t if_cfg64; + struct { +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + uint64_t base_queue : 16; + uint64_t num_iqueues : 16; + uint64_t num_oqueues : 16; + uint64_t gmx_port_id : 8; + uint64_t vf_id : 8; +#else + uint64_t vf_id : 8; + uint64_t gmx_port_id : 8; + uint64_t num_oqueues : 16; + uint64_t num_iqueues : 16; + uint64_t base_queue : 16; +#endif + } s; +}; + +struct lio_if_cfg_resp { + uint64_t rh; + struct octeon_if_cfg_info cfg_info; + uint64_t status; +}; + +struct lio_link_stats_resp { + uint64_t rh; + struct octeon_link_stats link_stats; + uint64_t status; +}; + +struct lio_link_status_resp { + uint64_t rh; + struct octeon_link_info link_info; + uint64_t status; +}; + +struct lio_rss_set { + struct param { +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + uint64_t flags : 16; + uint64_t hashinfo : 32; + uint64_t itablesize : 16; + uint64_t hashkeysize : 16; + uint64_t reserved : 48; +#elif RTE_BYTE_ORDER == RTE_BIG_ENDIAN + uint64_t itablesize : 16; + uint64_t hashinfo : 32; + uint64_t flags : 16; + uint64_t reserved : 48; + uint64_t hashkeysize : 16; +#endif + } param; + + uint8_t itable[LIO_RSS_MAX_TABLE_SZ]; + uint8_t key[LIO_RSS_MAX_KEY_SZ]; +}; + +void lio_dev_rx_queue_release(void *rxq); + +void lio_dev_tx_queue_release(void *txq); + +#endif /* _LIO_ETHDEV_H_ */ diff --git a/dpdk/drivers/net/liquidio/lio_logs.h b/dpdk/drivers/net/liquidio/lio_logs.h new file mode 100644 index 00000000..a4c9ca4d --- /dev/null +++ b/dpdk/drivers/net/liquidio/lio_logs.h @@ -0,0 +1,91 @@ +/* + * BSD LICENSE + * + * Copyright(c) 2017 Cavium, Inc.. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _LIO_LOGS_H_ +#define _LIO_LOGS_H_ + +#define lio_dev_printf(lio_dev, level, fmt, args...) \ + RTE_LOG(level, PMD, "%s" fmt, (lio_dev)->dev_string, ##args) + +#define lio_dev_info(lio_dev, fmt, args...) \ + lio_dev_printf(lio_dev, INFO, "INFO: " fmt, ##args) + +#define lio_dev_err(lio_dev, fmt, args...) \ + lio_dev_printf(lio_dev, ERR, "ERROR: %s() " fmt, __func__, ##args) + +#define PMD_INIT_LOG(level, fmt, args...) RTE_LOG(level, PMD, fmt, ## args) + +/* Enable these through config options */ + +#ifdef RTE_LIBRTE_LIO_DEBUG_INIT +#define PMD_INIT_FUNC_TRACE() PMD_INIT_LOG(DEBUG, "%s() >>\n", __func__) +#else /* !RTE_LIBRTE_LIO_DEBUG_INIT */ +#define PMD_INIT_FUNC_TRACE() do { } while (0) +#endif /* RTE_LIBRTE_LIO_DEBUG_INIT */ + +#ifdef RTE_LIBRTE_LIO_DEBUG_DRIVER +#define lio_dev_dbg(lio_dev, fmt, args...) \ + lio_dev_printf(lio_dev, DEBUG, "DEBUG: %s() " fmt, __func__, ##args) +#else /* !RTE_LIBRTE_LIO_DEBUG_DRIVER */ +#define lio_dev_dbg(lio_dev, fmt, args...) do { } while (0) +#endif /* RTE_LIBRTE_LIO_DEBUG_DRIVER */ + +#ifdef RTE_LIBRTE_LIO_DEBUG_RX +#define PMD_RX_LOG(lio_dev, level, fmt, args...) \ + lio_dev_printf(lio_dev, level, "RX: %s() " fmt, __func__, ##args) +#else /* !RTE_LIBRTE_LIO_DEBUG_RX */ +#define PMD_RX_LOG(lio_dev, level, fmt, args...) do { } while (0) +#endif /* RTE_LIBRTE_LIO_DEBUG_RX */ + +#ifdef RTE_LIBRTE_LIO_DEBUG_TX +#define PMD_TX_LOG(lio_dev, level, fmt, args...) \ + lio_dev_printf(lio_dev, level, "TX: %s() " fmt, __func__, ##args) +#else /* !RTE_LIBRTE_LIO_DEBUG_TX */ +#define PMD_TX_LOG(lio_dev, level, fmt, args...) do { } while (0) +#endif /* RTE_LIBRTE_LIO_DEBUG_TX */ + +#ifdef RTE_LIBRTE_LIO_DEBUG_MBOX +#define PMD_MBOX_LOG(lio_dev, level, fmt, args...) \ + lio_dev_printf(lio_dev, level, "MBOX: %s() " fmt, __func__, ##args) +#else /* !RTE_LIBRTE_LIO_DEBUG_MBOX */ +#define PMD_MBOX_LOG(level, fmt, args...) do { } while (0) +#endif /* RTE_LIBRTE_LIO_DEBUG_MBOX */ + +#ifdef RTE_LIBRTE_LIO_DEBUG_REGS +#define PMD_REGS_LOG(lio_dev, fmt, args...) \ + lio_dev_printf(lio_dev, DEBUG, "REGS: " fmt, ##args) +#else /* !RTE_LIBRTE_LIO_DEBUG_REGS */ +#define PMD_REGS_LOG(level, fmt, args...) do { } while (0) +#endif /* RTE_LIBRTE_LIO_DEBUG_REGS */ + +#endif /* _LIO_LOGS_H_ */ diff --git a/dpdk/drivers/net/liquidio/lio_rxtx.c b/dpdk/drivers/net/liquidio/lio_rxtx.c new file mode 100644 index 00000000..376893ac --- /dev/null +++ b/dpdk/drivers/net/liquidio/lio_rxtx.c @@ -0,0 +1,1856 @@ +/* + * BSD LICENSE + * + * Copyright(c) 2017 Cavium, Inc.. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include + +#include "lio_logs.h" +#include "lio_struct.h" +#include "lio_ethdev.h" +#include "lio_rxtx.h" + +#define LIO_MAX_SG 12 +/* Flush iq if available tx_desc fall below LIO_FLUSH_WM */ +#define LIO_FLUSH_WM(_iq) ((_iq)->max_count / 2) +#define LIO_PKT_IN_DONE_CNT_MASK 0x00000000FFFFFFFFULL + +static void +lio_droq_compute_max_packet_bufs(struct lio_droq *droq) +{ + uint32_t count = 0; + + do { + count += droq->buffer_size; + } while (count < LIO_MAX_RX_PKTLEN); +} + +static void +lio_droq_reset_indices(struct lio_droq *droq) +{ + droq->read_idx = 0; + droq->write_idx = 0; + droq->refill_idx = 0; + droq->refill_count = 0; + rte_atomic64_set(&droq->pkts_pending, 0); +} + +static void +lio_droq_destroy_ring_buffers(struct lio_droq *droq) +{ + uint32_t i; + + for (i = 0; i < droq->max_count; i++) { + if (droq->recv_buf_list[i].buffer) { + rte_pktmbuf_free((struct rte_mbuf *) + droq->recv_buf_list[i].buffer); + droq->recv_buf_list[i].buffer = NULL; + } + } + + lio_droq_reset_indices(droq); +} + +static int +lio_droq_setup_ring_buffers(struct lio_device *lio_dev, + struct lio_droq *droq) +{ + struct lio_droq_desc *desc_ring = droq->desc_ring; + uint32_t i; + void *buf; + + for (i = 0; i < droq->max_count; i++) { + buf = rte_pktmbuf_alloc(droq->mpool); + if (buf == NULL) { + lio_dev_err(lio_dev, "buffer alloc failed\n"); + droq->stats.rx_alloc_failure++; + lio_droq_destroy_ring_buffers(droq); + return -ENOMEM; + } + + droq->recv_buf_list[i].buffer = buf; + droq->info_list[i].length = 0; + + /* map ring buffers into memory */ + desc_ring[i].info_ptr = lio_map_ring_info(droq, i); + desc_ring[i].buffer_ptr = + lio_map_ring(droq->recv_buf_list[i].buffer); + } + + lio_droq_reset_indices(droq); + + lio_droq_compute_max_packet_bufs(droq); + + return 0; +} + +static void +lio_dma_zone_free(struct lio_device *lio_dev, const struct rte_memzone *mz) +{ + const struct rte_memzone *mz_tmp; + int ret = 0; + + if (mz == NULL) { + lio_dev_err(lio_dev, "Memzone NULL\n"); + return; + } + + mz_tmp = rte_memzone_lookup(mz->name); + if (mz_tmp == NULL) { + lio_dev_err(lio_dev, "Memzone %s Not Found\n", mz->name); + return; + } + + ret = rte_memzone_free(mz); + if (ret) + lio_dev_err(lio_dev, "Memzone free Failed ret %d\n", ret); +} + +/** + * Frees the space for descriptor ring for the droq. + * + * @param lio_dev - pointer to the lio device structure + * @param q_no - droq no. + */ +static void +lio_delete_droq(struct lio_device *lio_dev, uint32_t q_no) +{ + struct lio_droq *droq = lio_dev->droq[q_no]; + + lio_dev_dbg(lio_dev, "OQ[%d]\n", q_no); + + lio_droq_destroy_ring_buffers(droq); + rte_free(droq->recv_buf_list); + droq->recv_buf_list = NULL; + lio_dma_zone_free(lio_dev, droq->info_mz); + lio_dma_zone_free(lio_dev, droq->desc_ring_mz); + + memset(droq, 0, LIO_DROQ_SIZE); +} + +static void * +lio_alloc_info_buffer(struct lio_device *lio_dev, + struct lio_droq *droq, unsigned int socket_id) +{ + droq->info_mz = rte_eth_dma_zone_reserve(lio_dev->eth_dev, + "info_list", droq->q_no, + (droq->max_count * + LIO_DROQ_INFO_SIZE), + RTE_CACHE_LINE_SIZE, + socket_id); + + if (droq->info_mz == NULL) + return NULL; + + droq->info_list_dma = droq->info_mz->iova; + droq->info_alloc_size = droq->info_mz->len; + droq->info_base_addr = (size_t)droq->info_mz->addr; + + return droq->info_mz->addr; +} + +/** + * Allocates space for the descriptor ring for the droq and + * sets the base addr, num desc etc in Octeon registers. + * + * @param lio_dev - pointer to the lio device structure + * @param q_no - droq no. + * @param app_ctx - pointer to application context + * @return Success: 0 Failure: -1 + */ +static int +lio_init_droq(struct lio_device *lio_dev, uint32_t q_no, + uint32_t num_descs, uint32_t desc_size, + struct rte_mempool *mpool, unsigned int socket_id) +{ + uint32_t c_refill_threshold; + uint32_t desc_ring_size; + struct lio_droq *droq; + + lio_dev_dbg(lio_dev, "OQ[%d]\n", q_no); + + droq = lio_dev->droq[q_no]; + droq->lio_dev = lio_dev; + droq->q_no = q_no; + droq->mpool = mpool; + + c_refill_threshold = LIO_OQ_REFILL_THRESHOLD_CFG(lio_dev); + + droq->max_count = num_descs; + droq->buffer_size = desc_size; + + desc_ring_size = droq->max_count * LIO_DROQ_DESC_SIZE; + droq->desc_ring_mz = rte_eth_dma_zone_reserve(lio_dev->eth_dev, + "droq", q_no, + desc_ring_size, + RTE_CACHE_LINE_SIZE, + socket_id); + + if (droq->desc_ring_mz == NULL) { + lio_dev_err(lio_dev, + "Output queue %d ring alloc failed\n", q_no); + return -1; + } + + droq->desc_ring_dma = droq->desc_ring_mz->iova; + droq->desc_ring = (struct lio_droq_desc *)droq->desc_ring_mz->addr; + + lio_dev_dbg(lio_dev, "droq[%d]: desc_ring: virt: 0x%p, dma: %lx\n", + q_no, droq->desc_ring, (unsigned long)droq->desc_ring_dma); + lio_dev_dbg(lio_dev, "droq[%d]: num_desc: %d\n", q_no, + droq->max_count); + + droq->info_list = lio_alloc_info_buffer(lio_dev, droq, socket_id); + if (droq->info_list == NULL) { + lio_dev_err(lio_dev, "Cannot allocate memory for info list.\n"); + goto init_droq_fail; + } + + droq->recv_buf_list = rte_zmalloc_socket("recv_buf_list", + (droq->max_count * + LIO_DROQ_RECVBUF_SIZE), + RTE_CACHE_LINE_SIZE, + socket_id); + if (droq->recv_buf_list == NULL) { + lio_dev_err(lio_dev, + "Output queue recv buf list alloc failed\n"); + goto init_droq_fail; + } + + if (lio_droq_setup_ring_buffers(lio_dev, droq)) + goto init_droq_fail; + + droq->refill_threshold = c_refill_threshold; + + rte_spinlock_init(&droq->lock); + + lio_dev->fn_list.setup_oq_regs(lio_dev, q_no); + + lio_dev->io_qmask.oq |= (1ULL << q_no); + + return 0; + +init_droq_fail: + lio_delete_droq(lio_dev, q_no); + + return -1; +} + +int +lio_setup_droq(struct lio_device *lio_dev, int oq_no, int num_descs, + int desc_size, struct rte_mempool *mpool, unsigned int socket_id) +{ + struct lio_droq *droq; + + PMD_INIT_FUNC_TRACE(); + + if (lio_dev->droq[oq_no]) { + lio_dev_dbg(lio_dev, "Droq %d in use\n", oq_no); + return 0; + } + + /* Allocate the DS for the new droq. */ + droq = rte_zmalloc_socket("ethdev RX queue", sizeof(*droq), + RTE_CACHE_LINE_SIZE, socket_id); + if (droq == NULL) + return -ENOMEM; + + lio_dev->droq[oq_no] = droq; + + /* Initialize the Droq */ + if (lio_init_droq(lio_dev, oq_no, num_descs, desc_size, mpool, + socket_id)) { + lio_dev_err(lio_dev, "Droq[%u] Initialization Failed\n", oq_no); + rte_free(lio_dev->droq[oq_no]); + lio_dev->droq[oq_no] = NULL; + return -ENOMEM; + } + + lio_dev->num_oqs++; + + lio_dev_dbg(lio_dev, "Total number of OQ: %d\n", lio_dev->num_oqs); + + /* Send credit for octeon output queues. credits are always + * sent after the output queue is enabled. + */ + rte_write32(lio_dev->droq[oq_no]->max_count, + lio_dev->droq[oq_no]->pkts_credit_reg); + rte_wmb(); + + return 0; +} + +static inline uint32_t +lio_droq_get_bufcount(uint32_t buf_size, uint32_t total_len) +{ + uint32_t buf_cnt = 0; + + while (total_len > (buf_size * buf_cnt)) + buf_cnt++; + + return buf_cnt; +} + +/* If we were not able to refill all buffers, try to move around + * the buffers that were not dispatched. + */ +static inline uint32_t +lio_droq_refill_pullup_descs(struct lio_droq *droq, + struct lio_droq_desc *desc_ring) +{ + uint32_t refill_index = droq->refill_idx; + uint32_t desc_refilled = 0; + + while (refill_index != droq->read_idx) { + if (droq->recv_buf_list[refill_index].buffer) { + droq->recv_buf_list[droq->refill_idx].buffer = + droq->recv_buf_list[refill_index].buffer; + desc_ring[droq->refill_idx].buffer_ptr = + desc_ring[refill_index].buffer_ptr; + droq->recv_buf_list[refill_index].buffer = NULL; + desc_ring[refill_index].buffer_ptr = 0; + do { + droq->refill_idx = lio_incr_index( + droq->refill_idx, 1, + droq->max_count); + desc_refilled++; + droq->refill_count--; + } while (droq->recv_buf_list[droq->refill_idx].buffer); + } + refill_index = lio_incr_index(refill_index, 1, + droq->max_count); + } /* while */ + + return desc_refilled; +} + +/* lio_droq_refill + * + * @param droq - droq in which descriptors require new buffers. + * + * Description: + * Called during normal DROQ processing in interrupt mode or by the poll + * thread to refill the descriptors from which buffers were dispatched + * to upper layers. Attempts to allocate new buffers. If that fails, moves + * up buffers (that were not dispatched) to form a contiguous ring. + * + * Returns: + * No of descriptors refilled. + * + * Locks: + * This routine is called with droq->lock held. + */ +static uint32_t +lio_droq_refill(struct lio_droq *droq) +{ + struct lio_droq_desc *desc_ring; + uint32_t desc_refilled = 0; + void *buf = NULL; + + desc_ring = droq->desc_ring; + + while (droq->refill_count && (desc_refilled < droq->max_count)) { + /* If a valid buffer exists (happens if there is no dispatch), + * reuse the buffer, else allocate. + */ + if (droq->recv_buf_list[droq->refill_idx].buffer == NULL) { + buf = rte_pktmbuf_alloc(droq->mpool); + /* If a buffer could not be allocated, no point in + * continuing + */ + if (buf == NULL) { + droq->stats.rx_alloc_failure++; + break; + } + + droq->recv_buf_list[droq->refill_idx].buffer = buf; + } + + desc_ring[droq->refill_idx].buffer_ptr = + lio_map_ring(droq->recv_buf_list[droq->refill_idx].buffer); + /* Reset any previous values in the length field. */ + droq->info_list[droq->refill_idx].length = 0; + + droq->refill_idx = lio_incr_index(droq->refill_idx, 1, + droq->max_count); + desc_refilled++; + droq->refill_count--; + } + + if (droq->refill_count) + desc_refilled += lio_droq_refill_pullup_descs(droq, desc_ring); + + /* if droq->refill_count + * The refill count would not change in pass two. We only moved buffers + * to close the gap in the ring, but we would still have the same no. of + * buffers to refill. + */ + return desc_refilled; +} + +static int +lio_droq_fast_process_packet(struct lio_device *lio_dev, + struct lio_droq *droq, + struct rte_mbuf **rx_pkts) +{ + struct rte_mbuf *nicbuf = NULL; + struct lio_droq_info *info; + uint32_t total_len = 0; + int data_total_len = 0; + uint32_t pkt_len = 0; + union octeon_rh *rh; + int data_pkts = 0; + + info = &droq->info_list[droq->read_idx]; + lio_swap_8B_data((uint64_t *)info, 2); + + if (!info->length) + return -1; + + /* Len of resp hdr in included in the received data len. */ + info->length -= OCTEON_RH_SIZE; + rh = &info->rh; + + total_len += (uint32_t)info->length; + + if (lio_opcode_slow_path(rh)) { + uint32_t buf_cnt; + + buf_cnt = lio_droq_get_bufcount(droq->buffer_size, + (uint32_t)info->length); + droq->read_idx = lio_incr_index(droq->read_idx, buf_cnt, + droq->max_count); + droq->refill_count += buf_cnt; + } else { + if (info->length <= droq->buffer_size) { + if (rh->r_dh.has_hash) + pkt_len = (uint32_t)(info->length - 8); + else + pkt_len = (uint32_t)info->length; + + nicbuf = droq->recv_buf_list[droq->read_idx].buffer; + droq->recv_buf_list[droq->read_idx].buffer = NULL; + droq->read_idx = lio_incr_index( + droq->read_idx, 1, + droq->max_count); + droq->refill_count++; + + if (likely(nicbuf != NULL)) { + /* We don't have a way to pass flags yet */ + nicbuf->ol_flags = 0; + if (rh->r_dh.has_hash) { + uint64_t *hash_ptr; + + nicbuf->ol_flags |= PKT_RX_RSS_HASH; + hash_ptr = rte_pktmbuf_mtod(nicbuf, + uint64_t *); + lio_swap_8B_data(hash_ptr, 1); + nicbuf->hash.rss = (uint32_t)*hash_ptr; + nicbuf->data_off += 8; + } + + nicbuf->pkt_len = pkt_len; + nicbuf->data_len = pkt_len; + nicbuf->port = lio_dev->port_id; + /* Store the mbuf */ + rx_pkts[data_pkts++] = nicbuf; + data_total_len += pkt_len; + } + + /* Prefetch buffer pointers when on a cache line + * boundary + */ + if ((droq->read_idx & 3) == 0) { + rte_prefetch0( + &droq->recv_buf_list[droq->read_idx]); + rte_prefetch0( + &droq->info_list[droq->read_idx]); + } + } else { + struct rte_mbuf *first_buf = NULL; + struct rte_mbuf *last_buf = NULL; + + while (pkt_len < info->length) { + int cpy_len = 0; + + cpy_len = ((pkt_len + droq->buffer_size) > + info->length) + ? ((uint32_t)info->length - + pkt_len) + : droq->buffer_size; + + nicbuf = + droq->recv_buf_list[droq->read_idx].buffer; + droq->recv_buf_list[droq->read_idx].buffer = + NULL; + + if (likely(nicbuf != NULL)) { + /* Note the first seg */ + if (!pkt_len) + first_buf = nicbuf; + + nicbuf->port = lio_dev->port_id; + /* We don't have a way to pass + * flags yet + */ + nicbuf->ol_flags = 0; + if ((!pkt_len) && (rh->r_dh.has_hash)) { + uint64_t *hash_ptr; + + nicbuf->ol_flags |= + PKT_RX_RSS_HASH; + hash_ptr = rte_pktmbuf_mtod( + nicbuf, uint64_t *); + lio_swap_8B_data(hash_ptr, 1); + nicbuf->hash.rss = + (uint32_t)*hash_ptr; + nicbuf->data_off += 8; + nicbuf->pkt_len = cpy_len - 8; + nicbuf->data_len = cpy_len - 8; + } else { + nicbuf->pkt_len = cpy_len; + nicbuf->data_len = cpy_len; + } + + if (pkt_len) + first_buf->nb_segs++; + + if (last_buf) + last_buf->next = nicbuf; + + last_buf = nicbuf; + } else { + PMD_RX_LOG(lio_dev, ERR, "no buf\n"); + } + + pkt_len += cpy_len; + droq->read_idx = lio_incr_index( + droq->read_idx, + 1, droq->max_count); + droq->refill_count++; + + /* Prefetch buffer pointers when on a + * cache line boundary + */ + if ((droq->read_idx & 3) == 0) { + rte_prefetch0(&droq->recv_buf_list + [droq->read_idx]); + + rte_prefetch0( + &droq->info_list[droq->read_idx]); + } + } + rx_pkts[data_pkts++] = first_buf; + if (rh->r_dh.has_hash) + data_total_len += (pkt_len - 8); + else + data_total_len += pkt_len; + } + + /* Inform upper layer about packet checksum verification */ + struct rte_mbuf *m = rx_pkts[data_pkts - 1]; + + if (rh->r_dh.csum_verified & LIO_IP_CSUM_VERIFIED) + m->ol_flags |= PKT_RX_IP_CKSUM_GOOD; + + if (rh->r_dh.csum_verified & LIO_L4_CSUM_VERIFIED) + m->ol_flags |= PKT_RX_L4_CKSUM_GOOD; + } + + if (droq->refill_count >= droq->refill_threshold) { + int desc_refilled = lio_droq_refill(droq); + + /* Flush the droq descriptor data to memory to be sure + * that when we update the credits the data in memory is + * accurate. + */ + rte_wmb(); + rte_write32(desc_refilled, droq->pkts_credit_reg); + /* make sure mmio write completes */ + rte_wmb(); + } + + info->length = 0; + info->rh.rh64 = 0; + + droq->stats.pkts_received++; + droq->stats.rx_pkts_received += data_pkts; + droq->stats.rx_bytes_received += data_total_len; + droq->stats.bytes_received += total_len; + + return data_pkts; +} + +static uint32_t +lio_droq_fast_process_packets(struct lio_device *lio_dev, + struct lio_droq *droq, + struct rte_mbuf **rx_pkts, + uint32_t pkts_to_process) +{ + int ret, data_pkts = 0; + uint32_t pkt; + + for (pkt = 0; pkt < pkts_to_process; pkt++) { + ret = lio_droq_fast_process_packet(lio_dev, droq, + &rx_pkts[data_pkts]); + if (ret < 0) { + lio_dev_err(lio_dev, "Port[%d] DROQ[%d] idx: %d len:0, pkt_cnt: %d\n", + lio_dev->port_id, droq->q_no, + droq->read_idx, pkts_to_process); + break; + } + data_pkts += ret; + } + + rte_atomic64_sub(&droq->pkts_pending, pkt); + + return data_pkts; +} + +static inline uint32_t +lio_droq_check_hw_for_pkts(struct lio_droq *droq) +{ + uint32_t last_count; + uint32_t pkt_count; + + pkt_count = rte_read32(droq->pkts_sent_reg); + + last_count = pkt_count - droq->pkt_count; + droq->pkt_count = pkt_count; + + if (last_count) + rte_atomic64_add(&droq->pkts_pending, last_count); + + return last_count; +} + +uint16_t +lio_dev_recv_pkts(void *rx_queue, + struct rte_mbuf **rx_pkts, + uint16_t budget) +{ + struct lio_droq *droq = rx_queue; + struct lio_device *lio_dev = droq->lio_dev; + uint32_t pkts_processed = 0; + uint32_t pkt_count = 0; + + lio_droq_check_hw_for_pkts(droq); + + pkt_count = rte_atomic64_read(&droq->pkts_pending); + if (!pkt_count) + return 0; + + if (pkt_count > budget) + pkt_count = budget; + + /* Grab the lock */ + rte_spinlock_lock(&droq->lock); + pkts_processed = lio_droq_fast_process_packets(lio_dev, + droq, rx_pkts, + pkt_count); + + if (droq->pkt_count) { + rte_write32(droq->pkt_count, droq->pkts_sent_reg); + droq->pkt_count = 0; + } + + /* Release the spin lock */ + rte_spinlock_unlock(&droq->lock); + + return pkts_processed; +} + +void +lio_delete_droq_queue(struct lio_device *lio_dev, + int oq_no) +{ + lio_delete_droq(lio_dev, oq_no); + lio_dev->num_oqs--; + rte_free(lio_dev->droq[oq_no]); + lio_dev->droq[oq_no] = NULL; +} + +/** + * lio_init_instr_queue() + * @param lio_dev - pointer to the lio device structure. + * @param txpciq - queue to be initialized. + * + * Called at driver init time for each input queue. iq_conf has the + * configuration parameters for the queue. + * + * @return Success: 0 Failure: -1 + */ +static int +lio_init_instr_queue(struct lio_device *lio_dev, + union octeon_txpciq txpciq, + uint32_t num_descs, unsigned int socket_id) +{ + uint32_t iq_no = (uint32_t)txpciq.s.q_no; + struct lio_instr_queue *iq; + uint32_t instr_type; + uint32_t q_size; + + instr_type = LIO_IQ_INSTR_TYPE(lio_dev); + + q_size = instr_type * num_descs; + iq = lio_dev->instr_queue[iq_no]; + iq->iq_mz = rte_eth_dma_zone_reserve(lio_dev->eth_dev, + "instr_queue", iq_no, q_size, + RTE_CACHE_LINE_SIZE, + socket_id); + if (iq->iq_mz == NULL) { + lio_dev_err(lio_dev, "Cannot allocate memory for instr queue %d\n", + iq_no); + return -1; + } + + iq->base_addr_dma = iq->iq_mz->iova; + iq->base_addr = (uint8_t *)iq->iq_mz->addr; + + iq->max_count = num_descs; + + /* Initialize a list to holds requests that have been posted to Octeon + * but has yet to be fetched by octeon + */ + iq->request_list = rte_zmalloc_socket("request_list", + sizeof(*iq->request_list) * + num_descs, + RTE_CACHE_LINE_SIZE, + socket_id); + if (iq->request_list == NULL) { + lio_dev_err(lio_dev, "Alloc failed for IQ[%d] nr free list\n", + iq_no); + lio_dma_zone_free(lio_dev, iq->iq_mz); + return -1; + } + + lio_dev_dbg(lio_dev, "IQ[%d]: base: %p basedma: %lx count: %d\n", + iq_no, iq->base_addr, (unsigned long)iq->base_addr_dma, + iq->max_count); + + iq->lio_dev = lio_dev; + iq->txpciq.txpciq64 = txpciq.txpciq64; + iq->fill_cnt = 0; + iq->host_write_index = 0; + iq->lio_read_index = 0; + iq->flush_index = 0; + + rte_atomic64_set(&iq->instr_pending, 0); + + /* Initialize the spinlock for this instruction queue */ + rte_spinlock_init(&iq->lock); + rte_spinlock_init(&iq->post_lock); + + rte_atomic64_clear(&iq->iq_flush_running); + + lio_dev->io_qmask.iq |= (1ULL << iq_no); + + /* Set the 32B/64B mode for each input queue */ + lio_dev->io_qmask.iq64B |= ((instr_type == 64) << iq_no); + iq->iqcmd_64B = (instr_type == 64); + + lio_dev->fn_list.setup_iq_regs(lio_dev, iq_no); + + return 0; +} + +int +lio_setup_instr_queue0(struct lio_device *lio_dev) +{ + union octeon_txpciq txpciq; + uint32_t num_descs = 0; + uint32_t iq_no = 0; + + num_descs = LIO_NUM_DEF_TX_DESCS_CFG(lio_dev); + + lio_dev->num_iqs = 0; + + lio_dev->instr_queue[0] = rte_zmalloc(NULL, + sizeof(struct lio_instr_queue), 0); + if (lio_dev->instr_queue[0] == NULL) + return -ENOMEM; + + lio_dev->instr_queue[0]->q_index = 0; + lio_dev->instr_queue[0]->app_ctx = (void *)(size_t)0; + txpciq.txpciq64 = 0; + txpciq.s.q_no = iq_no; + txpciq.s.pkind = lio_dev->pfvf_hsword.pkind; + txpciq.s.use_qpg = 0; + txpciq.s.qpg = 0; + if (lio_init_instr_queue(lio_dev, txpciq, num_descs, SOCKET_ID_ANY)) { + rte_free(lio_dev->instr_queue[0]); + lio_dev->instr_queue[0] = NULL; + return -1; + } + + lio_dev->num_iqs++; + + return 0; +} + +/** + * lio_delete_instr_queue() + * @param lio_dev - pointer to the lio device structure. + * @param iq_no - queue to be deleted. + * + * Called at driver unload time for each input queue. Deletes all + * allocated resources for the input queue. + */ +static void +lio_delete_instr_queue(struct lio_device *lio_dev, uint32_t iq_no) +{ + struct lio_instr_queue *iq = lio_dev->instr_queue[iq_no]; + + rte_free(iq->request_list); + iq->request_list = NULL; + lio_dma_zone_free(lio_dev, iq->iq_mz); +} + +void +lio_free_instr_queue0(struct lio_device *lio_dev) +{ + lio_delete_instr_queue(lio_dev, 0); + rte_free(lio_dev->instr_queue[0]); + lio_dev->instr_queue[0] = NULL; + lio_dev->num_iqs--; +} + +/* Return 0 on success, -1 on failure */ +int +lio_setup_iq(struct lio_device *lio_dev, int q_index, + union octeon_txpciq txpciq, uint32_t num_descs, void *app_ctx, + unsigned int socket_id) +{ + uint32_t iq_no = (uint32_t)txpciq.s.q_no; + + if (lio_dev->instr_queue[iq_no]) { + lio_dev_dbg(lio_dev, "IQ is in use. Cannot create the IQ: %d again\n", + iq_no); + lio_dev->instr_queue[iq_no]->txpciq.txpciq64 = txpciq.txpciq64; + lio_dev->instr_queue[iq_no]->app_ctx = app_ctx; + return 0; + } + + lio_dev->instr_queue[iq_no] = rte_zmalloc_socket("ethdev TX queue", + sizeof(struct lio_instr_queue), + RTE_CACHE_LINE_SIZE, socket_id); + if (lio_dev->instr_queue[iq_no] == NULL) + return -1; + + lio_dev->instr_queue[iq_no]->q_index = q_index; + lio_dev->instr_queue[iq_no]->app_ctx = app_ctx; + + if (lio_init_instr_queue(lio_dev, txpciq, num_descs, socket_id)) + goto release_lio_iq; + + lio_dev->num_iqs++; + if (lio_dev->fn_list.enable_io_queues(lio_dev)) + goto delete_lio_iq; + + return 0; + +delete_lio_iq: + lio_delete_instr_queue(lio_dev, iq_no); + lio_dev->num_iqs--; +release_lio_iq: + rte_free(lio_dev->instr_queue[iq_no]); + lio_dev->instr_queue[iq_no] = NULL; + + return -1; +} + +int +lio_wait_for_instr_fetch(struct lio_device *lio_dev) +{ + int pending, instr_cnt; + int i, retry = 1000; + + do { + instr_cnt = 0; + + for (i = 0; i < LIO_MAX_INSTR_QUEUES(lio_dev); i++) { + if (!(lio_dev->io_qmask.iq & (1ULL << i))) + continue; + + if (lio_dev->instr_queue[i] == NULL) + break; + + pending = rte_atomic64_read( + &lio_dev->instr_queue[i]->instr_pending); + if (pending) + lio_flush_iq(lio_dev, lio_dev->instr_queue[i]); + + instr_cnt += pending; + } + + if (instr_cnt == 0) + break; + + rte_delay_ms(1); + + } while (retry-- && instr_cnt); + + return instr_cnt; +} + +static inline void +lio_ring_doorbell(struct lio_device *lio_dev, + struct lio_instr_queue *iq) +{ + if (rte_atomic64_read(&lio_dev->status) == LIO_DEV_RUNNING) { + rte_write32(iq->fill_cnt, iq->doorbell_reg); + /* make sure doorbell write goes through */ + rte_wmb(); + iq->fill_cnt = 0; + } +} + +static inline void +copy_cmd_into_iq(struct lio_instr_queue *iq, uint8_t *cmd) +{ + uint8_t *iqptr, cmdsize; + + cmdsize = ((iq->iqcmd_64B) ? 64 : 32); + iqptr = iq->base_addr + (cmdsize * iq->host_write_index); + + rte_memcpy(iqptr, cmd, cmdsize); +} + +static inline struct lio_iq_post_status +post_command2(struct lio_instr_queue *iq, uint8_t *cmd) +{ + struct lio_iq_post_status st; + + st.status = LIO_IQ_SEND_OK; + + /* This ensures that the read index does not wrap around to the same + * position if queue gets full before Octeon could fetch any instr. + */ + if (rte_atomic64_read(&iq->instr_pending) >= + (int32_t)(iq->max_count - 1)) { + st.status = LIO_IQ_SEND_FAILED; + st.index = -1; + return st; + } + + if (rte_atomic64_read(&iq->instr_pending) >= + (int32_t)(iq->max_count - 2)) + st.status = LIO_IQ_SEND_STOP; + + copy_cmd_into_iq(iq, cmd); + + /* "index" is returned, host_write_index is modified. */ + st.index = iq->host_write_index; + iq->host_write_index = lio_incr_index(iq->host_write_index, 1, + iq->max_count); + iq->fill_cnt++; + + /* Flush the command into memory. We need to be sure the data is in + * memory before indicating that the instruction is pending. + */ + rte_wmb(); + + rte_atomic64_inc(&iq->instr_pending); + + return st; +} + +static inline void +lio_add_to_request_list(struct lio_instr_queue *iq, + int idx, void *buf, int reqtype) +{ + iq->request_list[idx].buf = buf; + iq->request_list[idx].reqtype = reqtype; +} + +static inline void +lio_free_netsgbuf(void *buf) +{ + struct lio_buf_free_info *finfo = buf; + struct lio_device *lio_dev = finfo->lio_dev; + struct rte_mbuf *m = finfo->mbuf; + struct lio_gather *g = finfo->g; + uint8_t iq = finfo->iq_no; + + /* This will take care of multiple segments also */ + rte_pktmbuf_free(m); + + rte_spinlock_lock(&lio_dev->glist_lock[iq]); + STAILQ_INSERT_TAIL(&lio_dev->glist_head[iq], &g->list, entries); + rte_spinlock_unlock(&lio_dev->glist_lock[iq]); + rte_free(finfo); +} + +/* Can only run in process context */ +static int +lio_process_iq_request_list(struct lio_device *lio_dev, + struct lio_instr_queue *iq) +{ + struct octeon_instr_irh *irh = NULL; + uint32_t old = iq->flush_index; + struct lio_soft_command *sc; + uint32_t inst_count = 0; + int reqtype; + void *buf; + + while (old != iq->lio_read_index) { + reqtype = iq->request_list[old].reqtype; + buf = iq->request_list[old].buf; + + if (reqtype == LIO_REQTYPE_NONE) + goto skip_this; + + switch (reqtype) { + case LIO_REQTYPE_NORESP_NET: + rte_pktmbuf_free((struct rte_mbuf *)buf); + break; + case LIO_REQTYPE_NORESP_NET_SG: + lio_free_netsgbuf(buf); + break; + case LIO_REQTYPE_SOFT_COMMAND: + sc = buf; + irh = (struct octeon_instr_irh *)&sc->cmd.cmd3.irh; + if (irh->rflag) { + /* We're expecting a response from Octeon. + * It's up to lio_process_ordered_list() to + * process sc. Add sc to the ordered soft + * command response list because we expect + * a response from Octeon. + */ + rte_spinlock_lock(&lio_dev->response_list.lock); + rte_atomic64_inc( + &lio_dev->response_list.pending_req_count); + STAILQ_INSERT_TAIL( + &lio_dev->response_list.head, + &sc->node, entries); + rte_spinlock_unlock( + &lio_dev->response_list.lock); + } else { + if (sc->callback) { + /* This callback must not sleep */ + sc->callback(LIO_REQUEST_DONE, + sc->callback_arg); + } + } + break; + default: + lio_dev_err(lio_dev, + "Unknown reqtype: %d buf: %p at idx %d\n", + reqtype, buf, old); + } + + iq->request_list[old].buf = NULL; + iq->request_list[old].reqtype = 0; + +skip_this: + inst_count++; + old = lio_incr_index(old, 1, iq->max_count); + } + + iq->flush_index = old; + + return inst_count; +} + +static void +lio_update_read_index(struct lio_instr_queue *iq) +{ + uint32_t pkt_in_done = rte_read32(iq->inst_cnt_reg); + uint32_t last_done; + + last_done = pkt_in_done - iq->pkt_in_done; + iq->pkt_in_done = pkt_in_done; + + /* Add last_done and modulo with the IQ size to get new index */ + iq->lio_read_index = (iq->lio_read_index + + (uint32_t)(last_done & LIO_PKT_IN_DONE_CNT_MASK)) % + iq->max_count; +} + +int +lio_flush_iq(struct lio_device *lio_dev, struct lio_instr_queue *iq) +{ + uint32_t tot_inst_processed = 0; + uint32_t inst_processed = 0; + int tx_done = 1; + + if (rte_atomic64_test_and_set(&iq->iq_flush_running) == 0) + return tx_done; + + rte_spinlock_lock(&iq->lock); + + lio_update_read_index(iq); + + do { + /* Process any outstanding IQ packets. */ + if (iq->flush_index == iq->lio_read_index) + break; + + inst_processed = lio_process_iq_request_list(lio_dev, iq); + + if (inst_processed) { + rte_atomic64_sub(&iq->instr_pending, inst_processed); + iq->stats.instr_processed += inst_processed; + } + + tot_inst_processed += inst_processed; + inst_processed = 0; + + } while (1); + + rte_spinlock_unlock(&iq->lock); + + rte_atomic64_clear(&iq->iq_flush_running); + + return tx_done; +} + +static int +lio_send_command(struct lio_device *lio_dev, uint32_t iq_no, void *cmd, + void *buf, uint32_t datasize, uint32_t reqtype) +{ + struct lio_instr_queue *iq = lio_dev->instr_queue[iq_no]; + struct lio_iq_post_status st; + + rte_spinlock_lock(&iq->post_lock); + + st = post_command2(iq, cmd); + + if (st.status != LIO_IQ_SEND_FAILED) { + lio_add_to_request_list(iq, st.index, buf, reqtype); + LIO_INCR_INSTRQUEUE_PKT_COUNT(lio_dev, iq_no, bytes_sent, + datasize); + LIO_INCR_INSTRQUEUE_PKT_COUNT(lio_dev, iq_no, instr_posted, 1); + + lio_ring_doorbell(lio_dev, iq); + } else { + LIO_INCR_INSTRQUEUE_PKT_COUNT(lio_dev, iq_no, instr_dropped, 1); + } + + rte_spinlock_unlock(&iq->post_lock); + + return st.status; +} + +void +lio_prepare_soft_command(struct lio_device *lio_dev, + struct lio_soft_command *sc, uint8_t opcode, + uint8_t subcode, uint32_t irh_ossp, uint64_t ossp0, + uint64_t ossp1) +{ + struct octeon_instr_pki_ih3 *pki_ih3; + struct octeon_instr_ih3 *ih3; + struct octeon_instr_irh *irh; + struct octeon_instr_rdp *rdp; + + RTE_ASSERT(opcode <= 15); + RTE_ASSERT(subcode <= 127); + + ih3 = (struct octeon_instr_ih3 *)&sc->cmd.cmd3.ih3; + + ih3->pkind = lio_dev->instr_queue[sc->iq_no]->txpciq.s.pkind; + + pki_ih3 = (struct octeon_instr_pki_ih3 *)&sc->cmd.cmd3.pki_ih3; + + pki_ih3->w = 1; + pki_ih3->raw = 1; + pki_ih3->utag = 1; + pki_ih3->uqpg = lio_dev->instr_queue[sc->iq_no]->txpciq.s.use_qpg; + pki_ih3->utt = 1; + + pki_ih3->tag = LIO_CONTROL; + pki_ih3->tagtype = OCTEON_ATOMIC_TAG; + pki_ih3->qpg = lio_dev->instr_queue[sc->iq_no]->txpciq.s.qpg; + pki_ih3->pm = 0x7; + pki_ih3->sl = 8; + + if (sc->datasize) + ih3->dlengsz = sc->datasize; + + irh = (struct octeon_instr_irh *)&sc->cmd.cmd3.irh; + irh->opcode = opcode; + irh->subcode = subcode; + + /* opcode/subcode specific parameters (ossp) */ + irh->ossp = irh_ossp; + sc->cmd.cmd3.ossp[0] = ossp0; + sc->cmd.cmd3.ossp[1] = ossp1; + + if (sc->rdatasize) { + rdp = (struct octeon_instr_rdp *)&sc->cmd.cmd3.rdp; + rdp->pcie_port = lio_dev->pcie_port; + rdp->rlen = sc->rdatasize; + irh->rflag = 1; + /* PKI IH3 */ + ih3->fsz = OCTEON_SOFT_CMD_RESP_IH3; + } else { + irh->rflag = 0; + /* PKI IH3 */ + ih3->fsz = OCTEON_PCI_CMD_O3; + } +} + +int +lio_send_soft_command(struct lio_device *lio_dev, + struct lio_soft_command *sc) +{ + struct octeon_instr_ih3 *ih3; + struct octeon_instr_irh *irh; + uint32_t len = 0; + + ih3 = (struct octeon_instr_ih3 *)&sc->cmd.cmd3.ih3; + if (ih3->dlengsz) { + RTE_ASSERT(sc->dmadptr); + sc->cmd.cmd3.dptr = sc->dmadptr; + } + + irh = (struct octeon_instr_irh *)&sc->cmd.cmd3.irh; + if (irh->rflag) { + RTE_ASSERT(sc->dmarptr); + RTE_ASSERT(sc->status_word != NULL); + *sc->status_word = LIO_COMPLETION_WORD_INIT; + sc->cmd.cmd3.rptr = sc->dmarptr; + } + + len = (uint32_t)ih3->dlengsz; + + if (sc->wait_time) + sc->timeout = lio_uptime + sc->wait_time; + + return lio_send_command(lio_dev, sc->iq_no, &sc->cmd, sc, len, + LIO_REQTYPE_SOFT_COMMAND); +} + +int +lio_setup_sc_buffer_pool(struct lio_device *lio_dev) +{ + char sc_pool_name[RTE_MEMPOOL_NAMESIZE]; + uint16_t buf_size; + + buf_size = LIO_SOFT_COMMAND_BUFFER_SIZE + RTE_PKTMBUF_HEADROOM; + snprintf(sc_pool_name, sizeof(sc_pool_name), + "lio_sc_pool_%u", lio_dev->port_id); + lio_dev->sc_buf_pool = rte_pktmbuf_pool_create(sc_pool_name, + LIO_MAX_SOFT_COMMAND_BUFFERS, + 0, 0, buf_size, SOCKET_ID_ANY); + return 0; +} + +void +lio_free_sc_buffer_pool(struct lio_device *lio_dev) +{ + rte_mempool_free(lio_dev->sc_buf_pool); +} + +struct lio_soft_command * +lio_alloc_soft_command(struct lio_device *lio_dev, uint32_t datasize, + uint32_t rdatasize, uint32_t ctxsize) +{ + uint32_t offset = sizeof(struct lio_soft_command); + struct lio_soft_command *sc; + struct rte_mbuf *m; + uint64_t dma_addr; + + RTE_ASSERT((offset + datasize + rdatasize + ctxsize) <= + LIO_SOFT_COMMAND_BUFFER_SIZE); + + m = rte_pktmbuf_alloc(lio_dev->sc_buf_pool); + if (m == NULL) { + lio_dev_err(lio_dev, "Cannot allocate mbuf for sc\n"); + return NULL; + } + + /* set rte_mbuf data size and there is only 1 segment */ + m->pkt_len = LIO_SOFT_COMMAND_BUFFER_SIZE; + m->data_len = LIO_SOFT_COMMAND_BUFFER_SIZE; + + /* use rte_mbuf buffer for soft command */ + sc = rte_pktmbuf_mtod(m, struct lio_soft_command *); + memset(sc, 0, LIO_SOFT_COMMAND_BUFFER_SIZE); + sc->size = LIO_SOFT_COMMAND_BUFFER_SIZE; + sc->dma_addr = rte_mbuf_data_iova(m); + sc->mbuf = m; + + dma_addr = sc->dma_addr; + + if (ctxsize) { + sc->ctxptr = (uint8_t *)sc + offset; + sc->ctxsize = ctxsize; + } + + /* Start data at 128 byte boundary */ + offset = (offset + ctxsize + 127) & 0xffffff80; + + if (datasize) { + sc->virtdptr = (uint8_t *)sc + offset; + sc->dmadptr = dma_addr + offset; + sc->datasize = datasize; + } + + /* Start rdata at 128 byte boundary */ + offset = (offset + datasize + 127) & 0xffffff80; + + if (rdatasize) { + RTE_ASSERT(rdatasize >= 16); + sc->virtrptr = (uint8_t *)sc + offset; + sc->dmarptr = dma_addr + offset; + sc->rdatasize = rdatasize; + sc->status_word = (uint64_t *)((uint8_t *)(sc->virtrptr) + + rdatasize - 8); + } + + return sc; +} + +void +lio_free_soft_command(struct lio_soft_command *sc) +{ + rte_pktmbuf_free(sc->mbuf); +} + +void +lio_setup_response_list(struct lio_device *lio_dev) +{ + STAILQ_INIT(&lio_dev->response_list.head); + rte_spinlock_init(&lio_dev->response_list.lock); + rte_atomic64_set(&lio_dev->response_list.pending_req_count, 0); +} + +int +lio_process_ordered_list(struct lio_device *lio_dev) +{ + int resp_to_process = LIO_MAX_ORD_REQS_TO_PROCESS; + struct lio_response_list *ordered_sc_list; + struct lio_soft_command *sc; + int request_complete = 0; + uint64_t status64; + uint32_t status; + + ordered_sc_list = &lio_dev->response_list; + + do { + rte_spinlock_lock(&ordered_sc_list->lock); + + if (STAILQ_EMPTY(&ordered_sc_list->head)) { + /* ordered_sc_list is empty; there is + * nothing to process + */ + rte_spinlock_unlock(&ordered_sc_list->lock); + return -1; + } + + sc = LIO_STQUEUE_FIRST_ENTRY(&ordered_sc_list->head, + struct lio_soft_command, node); + + status = LIO_REQUEST_PENDING; + + /* check if octeon has finished DMA'ing a response + * to where rptr is pointing to + */ + status64 = *sc->status_word; + + if (status64 != LIO_COMPLETION_WORD_INIT) { + /* This logic ensures that all 64b have been written. + * 1. check byte 0 for non-FF + * 2. if non-FF, then swap result from BE to host order + * 3. check byte 7 (swapped to 0) for non-FF + * 4. if non-FF, use the low 32-bit status code + * 5. if either byte 0 or byte 7 is FF, don't use status + */ + if ((status64 & 0xff) != 0xff) { + lio_swap_8B_data(&status64, 1); + if (((status64 & 0xff) != 0xff)) { + /* retrieve 16-bit firmware status */ + status = (uint32_t)(status64 & + 0xffffULL); + if (status) { + status = + LIO_FIRMWARE_STATUS_CODE( + status); + } else { + /* i.e. no error */ + status = LIO_REQUEST_DONE; + } + } + } + } else if ((sc->timeout && lio_check_timeout(lio_uptime, + sc->timeout))) { + lio_dev_err(lio_dev, + "cmd failed, timeout (%ld, %ld)\n", + (long)lio_uptime, (long)sc->timeout); + status = LIO_REQUEST_TIMEOUT; + } + + if (status != LIO_REQUEST_PENDING) { + /* we have received a response or we have timed out. + * remove node from linked list + */ + STAILQ_REMOVE(&ordered_sc_list->head, + &sc->node, lio_stailq_node, entries); + rte_atomic64_dec( + &lio_dev->response_list.pending_req_count); + rte_spinlock_unlock(&ordered_sc_list->lock); + + if (sc->callback) + sc->callback(status, sc->callback_arg); + + request_complete++; + } else { + /* no response yet */ + request_complete = 0; + rte_spinlock_unlock(&ordered_sc_list->lock); + } + + /* If we hit the Max Ordered requests to process every loop, + * we quit and let this function be invoked the next time + * the poll thread runs to process the remaining requests. + * This function can take up the entire CPU if there is + * no upper limit to the requests processed. + */ + if (request_complete >= resp_to_process) + break; + } while (request_complete); + + return 0; +} + +static inline struct lio_stailq_node * +list_delete_first_node(struct lio_stailq_head *head) +{ + struct lio_stailq_node *node; + + if (STAILQ_EMPTY(head)) + node = NULL; + else + node = STAILQ_FIRST(head); + + if (node) + STAILQ_REMOVE(head, node, lio_stailq_node, entries); + + return node; +} + +void +lio_delete_sglist(struct lio_instr_queue *txq) +{ + struct lio_device *lio_dev = txq->lio_dev; + int iq_no = txq->q_index; + struct lio_gather *g; + + if (lio_dev->glist_head == NULL) + return; + + do { + g = (struct lio_gather *)list_delete_first_node( + &lio_dev->glist_head[iq_no]); + if (g) { + if (g->sg) + rte_free( + (void *)((unsigned long)g->sg - g->adjust)); + rte_free(g); + } + } while (g); +} + +/** + * \brief Setup gather lists + * @param lio per-network private data + */ +int +lio_setup_sglists(struct lio_device *lio_dev, int iq_no, + int fw_mapped_iq, int num_descs, unsigned int socket_id) +{ + struct lio_gather *g; + int i; + + rte_spinlock_init(&lio_dev->glist_lock[iq_no]); + + STAILQ_INIT(&lio_dev->glist_head[iq_no]); + + for (i = 0; i < num_descs; i++) { + g = rte_zmalloc_socket(NULL, sizeof(*g), RTE_CACHE_LINE_SIZE, + socket_id); + if (g == NULL) { + lio_dev_err(lio_dev, + "lio_gather memory allocation failed for qno %d\n", + iq_no); + break; + } + + g->sg_size = + ((ROUNDUP4(LIO_MAX_SG) >> 2) * LIO_SG_ENTRY_SIZE); + + g->sg = rte_zmalloc_socket(NULL, g->sg_size + 8, + RTE_CACHE_LINE_SIZE, socket_id); + if (g->sg == NULL) { + lio_dev_err(lio_dev, + "sg list memory allocation failed for qno %d\n", + iq_no); + rte_free(g); + break; + } + + /* The gather component should be aligned on 64-bit boundary */ + if (((unsigned long)g->sg) & 7) { + g->adjust = 8 - (((unsigned long)g->sg) & 7); + g->sg = + (struct lio_sg_entry *)((unsigned long)g->sg + + g->adjust); + } + + STAILQ_INSERT_TAIL(&lio_dev->glist_head[iq_no], &g->list, + entries); + } + + if (i != num_descs) { + lio_delete_sglist(lio_dev->instr_queue[fw_mapped_iq]); + return -ENOMEM; + } + + return 0; +} + +void +lio_delete_instruction_queue(struct lio_device *lio_dev, int iq_no) +{ + lio_delete_instr_queue(lio_dev, iq_no); + rte_free(lio_dev->instr_queue[iq_no]); + lio_dev->instr_queue[iq_no] = NULL; + lio_dev->num_iqs--; +} + +static inline uint32_t +lio_iq_get_available(struct lio_device *lio_dev, uint32_t q_no) +{ + return ((lio_dev->instr_queue[q_no]->max_count - 1) - + (uint32_t)rte_atomic64_read( + &lio_dev->instr_queue[q_no]->instr_pending)); +} + +static inline int +lio_iq_is_full(struct lio_device *lio_dev, uint32_t q_no) +{ + return ((uint32_t)rte_atomic64_read( + &lio_dev->instr_queue[q_no]->instr_pending) >= + (lio_dev->instr_queue[q_no]->max_count - 2)); +} + +static int +lio_dev_cleanup_iq(struct lio_device *lio_dev, int iq_no) +{ + struct lio_instr_queue *iq = lio_dev->instr_queue[iq_no]; + uint32_t count = 10000; + + while ((lio_iq_get_available(lio_dev, iq_no) < LIO_FLUSH_WM(iq)) && + --count) + lio_flush_iq(lio_dev, iq); + + return count ? 0 : 1; +} + +static void +lio_ctrl_cmd_callback(uint32_t status __rte_unused, void *sc_ptr) +{ + struct lio_soft_command *sc = sc_ptr; + struct lio_dev_ctrl_cmd *ctrl_cmd; + struct lio_ctrl_pkt *ctrl_pkt; + + ctrl_pkt = (struct lio_ctrl_pkt *)sc->ctxptr; + ctrl_cmd = ctrl_pkt->ctrl_cmd; + ctrl_cmd->cond = 1; + + lio_free_soft_command(sc); +} + +static inline struct lio_soft_command * +lio_alloc_ctrl_pkt_sc(struct lio_device *lio_dev, + struct lio_ctrl_pkt *ctrl_pkt) +{ + struct lio_soft_command *sc = NULL; + uint32_t uddsize, datasize; + uint32_t rdatasize; + uint8_t *data; + + uddsize = (uint32_t)(ctrl_pkt->ncmd.s.more * 8); + + datasize = OCTEON_CMD_SIZE + uddsize; + rdatasize = (ctrl_pkt->wait_time) ? 16 : 0; + + sc = lio_alloc_soft_command(lio_dev, datasize, + rdatasize, sizeof(struct lio_ctrl_pkt)); + if (sc == NULL) + return NULL; + + rte_memcpy(sc->ctxptr, ctrl_pkt, sizeof(struct lio_ctrl_pkt)); + + data = (uint8_t *)sc->virtdptr; + + rte_memcpy(data, &ctrl_pkt->ncmd, OCTEON_CMD_SIZE); + + lio_swap_8B_data((uint64_t *)data, OCTEON_CMD_SIZE >> 3); + + if (uddsize) { + /* Endian-Swap for UDD should have been done by caller. */ + rte_memcpy(data + OCTEON_CMD_SIZE, ctrl_pkt->udd, uddsize); + } + + sc->iq_no = (uint32_t)ctrl_pkt->iq_no; + + lio_prepare_soft_command(lio_dev, sc, + LIO_OPCODE, LIO_OPCODE_CMD, + 0, 0, 0); + + sc->callback = lio_ctrl_cmd_callback; + sc->callback_arg = sc; + sc->wait_time = ctrl_pkt->wait_time; + + return sc; +} + +int +lio_send_ctrl_pkt(struct lio_device *lio_dev, struct lio_ctrl_pkt *ctrl_pkt) +{ + struct lio_soft_command *sc = NULL; + int retval; + + sc = lio_alloc_ctrl_pkt_sc(lio_dev, ctrl_pkt); + if (sc == NULL) { + lio_dev_err(lio_dev, "soft command allocation failed\n"); + return -1; + } + + retval = lio_send_soft_command(lio_dev, sc); + if (retval == LIO_IQ_SEND_FAILED) { + lio_free_soft_command(sc); + lio_dev_err(lio_dev, "Port: %d soft command: %d send failed status: %x\n", + lio_dev->port_id, ctrl_pkt->ncmd.s.cmd, retval); + return -1; + } + + return retval; +} + +/** Send data packet to the device + * @param lio_dev - lio device pointer + * @param ndata - control structure with queueing, and buffer information + * + * @returns IQ_FAILED if it failed to add to the input queue. IQ_STOP if it the + * queue should be stopped, and LIO_IQ_SEND_OK if it sent okay. + */ +static inline int +lio_send_data_pkt(struct lio_device *lio_dev, struct lio_data_pkt *ndata) +{ + return lio_send_command(lio_dev, ndata->q_no, &ndata->cmd, + ndata->buf, ndata->datasize, ndata->reqtype); +} + +uint16_t +lio_dev_xmit_pkts(void *tx_queue, struct rte_mbuf **pkts, uint16_t nb_pkts) +{ + struct lio_instr_queue *txq = tx_queue; + union lio_cmd_setup cmdsetup; + struct lio_device *lio_dev; + struct lio_iq_stats *stats; + struct lio_data_pkt ndata; + int i, processed = 0; + struct rte_mbuf *m; + uint32_t tag = 0; + int status = 0; + int iq_no; + + lio_dev = txq->lio_dev; + iq_no = txq->txpciq.s.q_no; + stats = &lio_dev->instr_queue[iq_no]->stats; + + if (!lio_dev->intf_open || !lio_dev->linfo.link.s.link_up) { + PMD_TX_LOG(lio_dev, ERR, "Transmit failed link_status : %d\n", + lio_dev->linfo.link.s.link_up); + goto xmit_failed; + } + + lio_dev_cleanup_iq(lio_dev, iq_no); + + for (i = 0; i < nb_pkts; i++) { + uint32_t pkt_len = 0; + + m = pkts[i]; + + /* Prepare the attributes for the data to be passed to BASE. */ + memset(&ndata, 0, sizeof(struct lio_data_pkt)); + + ndata.buf = m; + + ndata.q_no = iq_no; + if (lio_iq_is_full(lio_dev, ndata.q_no)) { + stats->tx_iq_busy++; + if (lio_dev_cleanup_iq(lio_dev, iq_no)) { + PMD_TX_LOG(lio_dev, ERR, + "Transmit failed iq:%d full\n", + ndata.q_no); + break; + } + } + + cmdsetup.cmd_setup64 = 0; + cmdsetup.s.iq_no = iq_no; + + /* check checksum offload flags to form cmd */ + if (m->ol_flags & PKT_TX_IP_CKSUM) + cmdsetup.s.ip_csum = 1; + + if (m->ol_flags & PKT_TX_OUTER_IP_CKSUM) + cmdsetup.s.tnl_csum = 1; + else if ((m->ol_flags & PKT_TX_TCP_CKSUM) || + (m->ol_flags & PKT_TX_UDP_CKSUM)) + cmdsetup.s.transport_csum = 1; + + if (m->nb_segs == 1) { + pkt_len = rte_pktmbuf_data_len(m); + cmdsetup.s.u.datasize = pkt_len; + lio_prepare_pci_cmd(lio_dev, &ndata.cmd, + &cmdsetup, tag); + ndata.cmd.cmd3.dptr = rte_mbuf_data_iova(m); + ndata.reqtype = LIO_REQTYPE_NORESP_NET; + } else { + struct lio_buf_free_info *finfo; + struct lio_gather *g; + rte_iova_t phyaddr; + int i, frags; + + finfo = (struct lio_buf_free_info *)rte_malloc(NULL, + sizeof(*finfo), 0); + if (finfo == NULL) { + PMD_TX_LOG(lio_dev, ERR, + "free buffer alloc failed\n"); + goto xmit_failed; + } + + rte_spinlock_lock(&lio_dev->glist_lock[iq_no]); + g = (struct lio_gather *)list_delete_first_node( + &lio_dev->glist_head[iq_no]); + rte_spinlock_unlock(&lio_dev->glist_lock[iq_no]); + if (g == NULL) { + PMD_TX_LOG(lio_dev, ERR, + "Transmit scatter gather: glist null!\n"); + goto xmit_failed; + } + + cmdsetup.s.gather = 1; + cmdsetup.s.u.gatherptrs = m->nb_segs; + lio_prepare_pci_cmd(lio_dev, &ndata.cmd, + &cmdsetup, tag); + + memset(g->sg, 0, g->sg_size); + g->sg[0].ptr[0] = rte_mbuf_data_iova(m); + lio_add_sg_size(&g->sg[0], m->data_len, 0); + pkt_len = m->data_len; + finfo->mbuf = m; + + /* First seg taken care above */ + frags = m->nb_segs - 1; + i = 1; + m = m->next; + while (frags--) { + g->sg[(i >> 2)].ptr[(i & 3)] = + rte_mbuf_data_iova(m); + lio_add_sg_size(&g->sg[(i >> 2)], + m->data_len, (i & 3)); + pkt_len += m->data_len; + i++; + m = m->next; + } + + phyaddr = rte_mem_virt2iova(g->sg); + if (phyaddr == RTE_BAD_IOVA) { + PMD_TX_LOG(lio_dev, ERR, "bad phys addr\n"); + goto xmit_failed; + } + + ndata.cmd.cmd3.dptr = phyaddr; + ndata.reqtype = LIO_REQTYPE_NORESP_NET_SG; + + finfo->g = g; + finfo->lio_dev = lio_dev; + finfo->iq_no = (uint64_t)iq_no; + ndata.buf = finfo; + } + + ndata.datasize = pkt_len; + + status = lio_send_data_pkt(lio_dev, &ndata); + + if (unlikely(status == LIO_IQ_SEND_FAILED)) { + PMD_TX_LOG(lio_dev, ERR, "send failed\n"); + break; + } + + if (unlikely(status == LIO_IQ_SEND_STOP)) { + PMD_TX_LOG(lio_dev, DEBUG, "iq full\n"); + /* create space as iq is full */ + lio_dev_cleanup_iq(lio_dev, iq_no); + } + + stats->tx_done++; + stats->tx_tot_bytes += pkt_len; + processed++; + } + +xmit_failed: + stats->tx_dropped += (nb_pkts - processed); + + return processed; +} + +void +lio_dev_clear_queues(struct rte_eth_dev *eth_dev) +{ + struct lio_instr_queue *txq; + struct lio_droq *rxq; + uint16_t i; + + for (i = 0; i < eth_dev->data->nb_tx_queues; i++) { + txq = eth_dev->data->tx_queues[i]; + if (txq != NULL) { + lio_dev_tx_queue_release(txq); + eth_dev->data->tx_queues[i] = NULL; + } + } + + for (i = 0; i < eth_dev->data->nb_rx_queues; i++) { + rxq = eth_dev->data->rx_queues[i]; + if (rxq != NULL) { + lio_dev_rx_queue_release(rxq); + eth_dev->data->rx_queues[i] = NULL; + } + } +} diff --git a/dpdk/drivers/net/liquidio/lio_rxtx.h b/dpdk/drivers/net/liquidio/lio_rxtx.h new file mode 100644 index 00000000..ef033735 --- /dev/null +++ b/dpdk/drivers/net/liquidio/lio_rxtx.h @@ -0,0 +1,769 @@ +/* + * BSD LICENSE + * + * Copyright(c) 2017 Cavium, Inc.. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _LIO_RXTX_H_ +#define _LIO_RXTX_H_ + +#include +#include + +#include +#include + +#include "lio_struct.h" + +#ifndef ROUNDUP4 +#define ROUNDUP4(val) (((val) + 3) & 0xfffffffc) +#endif + +#define LIO_STQUEUE_FIRST_ENTRY(ptr, type, elem) \ + (type *)((char *)((ptr)->stqh_first) - offsetof(type, elem)) + +#define lio_check_timeout(cur_time, chk_time) ((cur_time) > (chk_time)) + +#define lio_uptime \ + (size_t)(rte_get_timer_cycles() / rte_get_timer_hz()) + +/** Descriptor format. + * The descriptor ring is made of descriptors which have 2 64-bit values: + * -# Physical (bus) address of the data buffer. + * -# Physical (bus) address of a lio_droq_info structure. + * The device DMA's incoming packets and its information at the address + * given by these descriptor fields. + */ +struct lio_droq_desc { + /** The buffer pointer */ + uint64_t buffer_ptr; + + /** The Info pointer */ + uint64_t info_ptr; +}; + +#define LIO_DROQ_DESC_SIZE (sizeof(struct lio_droq_desc)) + +/** Information about packet DMA'ed by Octeon. + * The format of the information available at Info Pointer after Octeon + * has posted a packet. Not all descriptors have valid information. Only + * the Info field of the first descriptor for a packet has information + * about the packet. + */ +struct lio_droq_info { + /** The Output Receive Header. */ + union octeon_rh rh; + + /** The Length of the packet. */ + uint64_t length; +}; + +#define LIO_DROQ_INFO_SIZE (sizeof(struct lio_droq_info)) + +/** Pointer to data buffer. + * Driver keeps a pointer to the data buffer that it made available to + * the Octeon device. Since the descriptor ring keeps physical (bus) + * addresses, this field is required for the driver to keep track of + * the virtual address pointers. + */ +struct lio_recv_buffer { + /** Packet buffer, including meta data. */ + void *buffer; + + /** Data in the packet buffer. */ + uint8_t *data; + +}; + +#define LIO_DROQ_RECVBUF_SIZE (sizeof(struct lio_recv_buffer)) + +#define LIO_DROQ_SIZE (sizeof(struct lio_droq)) + +#define LIO_IQ_SEND_OK 0 +#define LIO_IQ_SEND_STOP 1 +#define LIO_IQ_SEND_FAILED -1 + +/* conditions */ +#define LIO_REQTYPE_NONE 0 +#define LIO_REQTYPE_NORESP_NET 1 +#define LIO_REQTYPE_NORESP_NET_SG 2 +#define LIO_REQTYPE_SOFT_COMMAND 3 + +struct lio_request_list { + uint32_t reqtype; + void *buf; +}; + +/*---------------------- INSTRUCTION FORMAT ----------------------------*/ + +struct lio_instr3_64B { + /** Pointer where the input data is available. */ + uint64_t dptr; + + /** Instruction Header. */ + uint64_t ih3; + + /** Instruction Header. */ + uint64_t pki_ih3; + + /** Input Request Header. */ + uint64_t irh; + + /** opcode/subcode specific parameters */ + uint64_t ossp[2]; + + /** Return Data Parameters */ + uint64_t rdp; + + /** Pointer where the response for a RAW mode packet will be written + * by Octeon. + */ + uint64_t rptr; + +}; + +union lio_instr_64B { + struct lio_instr3_64B cmd3; +}; + +/** The size of each buffer in soft command buffer pool */ +#define LIO_SOFT_COMMAND_BUFFER_SIZE 1536 + +/** Maximum number of buffers to allocate into soft command buffer pool */ +#define LIO_MAX_SOFT_COMMAND_BUFFERS 255 + +struct lio_soft_command { + /** Soft command buffer info. */ + struct lio_stailq_node node; + uint64_t dma_addr; + uint32_t size; + + /** Command and return status */ + union lio_instr_64B cmd; + +#define LIO_COMPLETION_WORD_INIT 0xffffffffffffffffULL + uint64_t *status_word; + + /** Data buffer info */ + void *virtdptr; + uint64_t dmadptr; + uint32_t datasize; + + /** Return buffer info */ + void *virtrptr; + uint64_t dmarptr; + uint32_t rdatasize; + + /** Context buffer info */ + void *ctxptr; + uint32_t ctxsize; + + /** Time out and callback */ + size_t wait_time; + size_t timeout; + uint32_t iq_no; + void (*callback)(uint32_t, void *); + void *callback_arg; + struct rte_mbuf *mbuf; +}; + +struct lio_iq_post_status { + int status; + int index; +}; + +/* wqe + * --------------- 0 + * | wqe word0-3 | + * --------------- 32 + * | PCI IH | + * --------------- 40 + * | RPTR | + * --------------- 48 + * | PCI IRH | + * --------------- 56 + * | OCTEON_CMD | + * --------------- 64 + * | Addtl 8-BData | + * | | + * --------------- + */ + +union octeon_cmd { + uint64_t cmd64; + + struct { +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + uint64_t cmd : 5; + + uint64_t more : 6; /* How many udd words follow the command */ + + uint64_t reserved : 29; + + uint64_t param1 : 16; + + uint64_t param2 : 8; + +#elif RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + + uint64_t param2 : 8; + + uint64_t param1 : 16; + + uint64_t reserved : 29; + + uint64_t more : 6; + + uint64_t cmd : 5; + +#endif + } s; +}; + +#define OCTEON_CMD_SIZE (sizeof(union octeon_cmd)) + +/* Maximum number of 8-byte words can be + * sent in a NIC control message. + */ +#define LIO_MAX_NCTRL_UDD 32 + +/* Structure of control information passed by driver to the BASE + * layer when sending control commands to Octeon device software. + */ +struct lio_ctrl_pkt { + /** Command to be passed to the Octeon device software. */ + union octeon_cmd ncmd; + + /** Send buffer */ + void *data; + uint64_t dmadata; + + /** Response buffer */ + void *rdata; + uint64_t dmardata; + + /** Additional data that may be needed by some commands. */ + uint64_t udd[LIO_MAX_NCTRL_UDD]; + + /** Input queue to use to send this command. */ + uint64_t iq_no; + + /** Time to wait for Octeon software to respond to this control command. + * If wait_time is 0, BASE assumes no response is expected. + */ + size_t wait_time; + + struct lio_dev_ctrl_cmd *ctrl_cmd; +}; + +/** Structure of data information passed by driver to the BASE + * layer when forwarding data to Octeon device software. + */ +struct lio_data_pkt { + /** Pointer to information maintained by NIC module for this packet. The + * BASE layer passes this as-is to the driver. + */ + void *buf; + + /** Type of buffer passed in "buf" above. */ + uint32_t reqtype; + + /** Total data bytes to be transferred in this command. */ + uint32_t datasize; + + /** Command to be passed to the Octeon device software. */ + union lio_instr_64B cmd; + + /** Input queue to use to send this command. */ + uint32_t q_no; +}; + +/** Structure passed by driver to BASE layer to prepare a command to send + * network data to Octeon. + */ +union lio_cmd_setup { + struct { + uint32_t iq_no : 8; + uint32_t gather : 1; + uint32_t timestamp : 1; + uint32_t ip_csum : 1; + uint32_t transport_csum : 1; + uint32_t tnl_csum : 1; + uint32_t rsvd : 19; + + union { + uint32_t datasize; + uint32_t gatherptrs; + } u; + } s; + + uint64_t cmd_setup64; +}; + +/* Instruction Header */ +struct octeon_instr_ih3 { +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + + /** Reserved3 */ + uint64_t reserved3 : 1; + + /** Gather indicator 1=gather*/ + uint64_t gather : 1; + + /** Data length OR no. of entries in gather list */ + uint64_t dlengsz : 14; + + /** Front Data size */ + uint64_t fsz : 6; + + /** Reserved2 */ + uint64_t reserved2 : 4; + + /** PKI port kind - PKIND */ + uint64_t pkind : 6; + + /** Reserved1 */ + uint64_t reserved1 : 32; + +#elif RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + /** Reserved1 */ + uint64_t reserved1 : 32; + + /** PKI port kind - PKIND */ + uint64_t pkind : 6; + + /** Reserved2 */ + uint64_t reserved2 : 4; + + /** Front Data size */ + uint64_t fsz : 6; + + /** Data length OR no. of entries in gather list */ + uint64_t dlengsz : 14; + + /** Gather indicator 1=gather*/ + uint64_t gather : 1; + + /** Reserved3 */ + uint64_t reserved3 : 1; + +#endif +}; + +/* PKI Instruction Header(PKI IH) */ +struct octeon_instr_pki_ih3 { +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + + /** Wider bit */ + uint64_t w : 1; + + /** Raw mode indicator 1 = RAW */ + uint64_t raw : 1; + + /** Use Tag */ + uint64_t utag : 1; + + /** Use QPG */ + uint64_t uqpg : 1; + + /** Reserved2 */ + uint64_t reserved2 : 1; + + /** Parse Mode */ + uint64_t pm : 3; + + /** Skip Length */ + uint64_t sl : 8; + + /** Use Tag Type */ + uint64_t utt : 1; + + /** Tag type */ + uint64_t tagtype : 2; + + /** Reserved1 */ + uint64_t reserved1 : 2; + + /** QPG Value */ + uint64_t qpg : 11; + + /** Tag Value */ + uint64_t tag : 32; + +#elif RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + + /** Tag Value */ + uint64_t tag : 32; + + /** QPG Value */ + uint64_t qpg : 11; + + /** Reserved1 */ + uint64_t reserved1 : 2; + + /** Tag type */ + uint64_t tagtype : 2; + + /** Use Tag Type */ + uint64_t utt : 1; + + /** Skip Length */ + uint64_t sl : 8; + + /** Parse Mode */ + uint64_t pm : 3; + + /** Reserved2 */ + uint64_t reserved2 : 1; + + /** Use QPG */ + uint64_t uqpg : 1; + + /** Use Tag */ + uint64_t utag : 1; + + /** Raw mode indicator 1 = RAW */ + uint64_t raw : 1; + + /** Wider bit */ + uint64_t w : 1; +#endif +}; + +/** Input Request Header */ +struct octeon_instr_irh { +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + uint64_t opcode : 4; + uint64_t rflag : 1; + uint64_t subcode : 7; + uint64_t vlan : 12; + uint64_t priority : 3; + uint64_t reserved : 5; + uint64_t ossp : 32; /* opcode/subcode specific parameters */ +#elif RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + uint64_t ossp : 32; /* opcode/subcode specific parameters */ + uint64_t reserved : 5; + uint64_t priority : 3; + uint64_t vlan : 12; + uint64_t subcode : 7; + uint64_t rflag : 1; + uint64_t opcode : 4; +#endif +}; + +/* pkiih3 + irh + ossp[0] + ossp[1] + rdp + rptr = 40 bytes */ +#define OCTEON_SOFT_CMD_RESP_IH3 (40 + 8) +/* pki_h3 + irh + ossp[0] + ossp[1] = 32 bytes */ +#define OCTEON_PCI_CMD_O3 (24 + 8) + +/** Return Data Parameters */ +struct octeon_instr_rdp { +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + uint64_t reserved : 49; + uint64_t pcie_port : 3; + uint64_t rlen : 12; +#elif RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + uint64_t rlen : 12; + uint64_t pcie_port : 3; + uint64_t reserved : 49; +#endif +}; + +union octeon_packet_params { + uint32_t pkt_params32; + struct { +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + uint32_t reserved : 24; + uint32_t ip_csum : 1; /* Perform IP header checksum(s) */ + /* Perform Outer transport header checksum */ + uint32_t transport_csum : 1; + /* Find tunnel, and perform transport csum. */ + uint32_t tnl_csum : 1; + uint32_t tsflag : 1; /* Timestamp this packet */ + uint32_t ipsec_ops : 4; /* IPsec operation */ +#else + uint32_t ipsec_ops : 4; + uint32_t tsflag : 1; + uint32_t tnl_csum : 1; + uint32_t transport_csum : 1; + uint32_t ip_csum : 1; + uint32_t reserved : 7; +#endif + } s; +}; + +/** Utility function to prepare a 64B NIC instruction based on a setup command + * @param cmd - pointer to instruction to be filled in. + * @param setup - pointer to the setup structure + * @param q_no - which queue for back pressure + * + * Assumes the cmd instruction is pre-allocated, but no fields are filled in. + */ +static inline void +lio_prepare_pci_cmd(struct lio_device *lio_dev, + union lio_instr_64B *cmd, + union lio_cmd_setup *setup, + uint32_t tag) +{ + union octeon_packet_params packet_params; + struct octeon_instr_pki_ih3 *pki_ih3; + struct octeon_instr_irh *irh; + struct octeon_instr_ih3 *ih3; + int port; + + memset(cmd, 0, sizeof(union lio_instr_64B)); + + ih3 = (struct octeon_instr_ih3 *)&cmd->cmd3.ih3; + pki_ih3 = (struct octeon_instr_pki_ih3 *)&cmd->cmd3.pki_ih3; + + /* assume that rflag is cleared so therefore front data will only have + * irh and ossp[1] and ossp[2] for a total of 24 bytes + */ + ih3->pkind = lio_dev->instr_queue[setup->s.iq_no]->txpciq.s.pkind; + /* PKI IH */ + ih3->fsz = OCTEON_PCI_CMD_O3; + + if (!setup->s.gather) { + ih3->dlengsz = setup->s.u.datasize; + } else { + ih3->gather = 1; + ih3->dlengsz = setup->s.u.gatherptrs; + } + + pki_ih3->w = 1; + pki_ih3->raw = 0; + pki_ih3->utag = 0; + pki_ih3->utt = 1; + pki_ih3->uqpg = lio_dev->instr_queue[setup->s.iq_no]->txpciq.s.use_qpg; + + port = (int)lio_dev->instr_queue[setup->s.iq_no]->txpciq.s.port; + + if (tag) + pki_ih3->tag = tag; + else + pki_ih3->tag = LIO_DATA(port); + + pki_ih3->tagtype = OCTEON_ORDERED_TAG; + pki_ih3->qpg = lio_dev->instr_queue[setup->s.iq_no]->txpciq.s.qpg; + pki_ih3->pm = 0x0; /* parse from L2 */ + pki_ih3->sl = 32; /* sl will be sizeof(pki_ih3) + irh + ossp0 + ossp1*/ + + irh = (struct octeon_instr_irh *)&cmd->cmd3.irh; + + irh->opcode = LIO_OPCODE; + irh->subcode = LIO_OPCODE_NW_DATA; + + packet_params.pkt_params32 = 0; + packet_params.s.ip_csum = setup->s.ip_csum; + packet_params.s.transport_csum = setup->s.transport_csum; + packet_params.s.tnl_csum = setup->s.tnl_csum; + packet_params.s.tsflag = setup->s.timestamp; + + irh->ossp = packet_params.pkt_params32; +} + +int lio_setup_sc_buffer_pool(struct lio_device *lio_dev); +void lio_free_sc_buffer_pool(struct lio_device *lio_dev); + +struct lio_soft_command * +lio_alloc_soft_command(struct lio_device *lio_dev, + uint32_t datasize, uint32_t rdatasize, + uint32_t ctxsize); +void lio_prepare_soft_command(struct lio_device *lio_dev, + struct lio_soft_command *sc, + uint8_t opcode, uint8_t subcode, + uint32_t irh_ossp, uint64_t ossp0, + uint64_t ossp1); +int lio_send_soft_command(struct lio_device *lio_dev, + struct lio_soft_command *sc); +void lio_free_soft_command(struct lio_soft_command *sc); + +/** Send control packet to the device + * @param lio_dev - lio device pointer + * @param nctrl - control structure with command, timeout, and callback info + * + * @returns IQ_FAILED if it failed to add to the input queue. IQ_STOP if it the + * queue should be stopped, and LIO_IQ_SEND_OK if it sent okay. + */ +int lio_send_ctrl_pkt(struct lio_device *lio_dev, + struct lio_ctrl_pkt *ctrl_pkt); + +/** Maximum ordered requests to process in every invocation of + * lio_process_ordered_list(). The function will continue to process requests + * as long as it can find one that has finished processing. If it keeps + * finding requests that have completed, the function can run for ever. The + * value defined here sets an upper limit on the number of requests it can + * process before it returns control to the poll thread. + */ +#define LIO_MAX_ORD_REQS_TO_PROCESS 4096 + +/** Error codes used in Octeon Host-Core communication. + * + * 31 16 15 0 + * ---------------------------- + * | | | + * ---------------------------- + * Error codes are 32-bit wide. The upper 16-bits, called Major Error Number, + * are reserved to identify the group to which the error code belongs. The + * lower 16-bits, called Minor Error Number, carry the actual code. + * + * So error codes are (MAJOR NUMBER << 16)| MINOR_NUMBER. + */ +/** Status for a request. + * If the request is successfully queued, the driver will return + * a LIO_REQUEST_PENDING status. LIO_REQUEST_TIMEOUT is only returned by + * the driver if the response for request failed to arrive before a + * time-out period or if the request processing * got interrupted due to + * a signal respectively. + */ +enum { + /** A value of 0x00000000 indicates no error i.e. success */ + LIO_REQUEST_DONE = 0x00000000, + /** (Major number: 0x0000; Minor Number: 0x0001) */ + LIO_REQUEST_PENDING = 0x00000001, + LIO_REQUEST_TIMEOUT = 0x00000003, + +}; + +/*------ Error codes used by firmware (bits 15..0 set by firmware */ +#define LIO_FIRMWARE_MAJOR_ERROR_CODE 0x0001 +#define LIO_FIRMWARE_STATUS_CODE(status) \ + ((LIO_FIRMWARE_MAJOR_ERROR_CODE << 16) | (status)) + +/** Initialize the response lists. The number of response lists to create is + * given by count. + * @param lio_dev - the lio device structure. + */ +void lio_setup_response_list(struct lio_device *lio_dev); + +/** Check the status of first entry in the ordered list. If the instruction at + * that entry finished processing or has timed-out, the entry is cleaned. + * @param lio_dev - the lio device structure. + * @return 1 if the ordered list is empty, 0 otherwise. + */ +int lio_process_ordered_list(struct lio_device *lio_dev); + +#define LIO_INCR_INSTRQUEUE_PKT_COUNT(lio_dev, iq_no, field, count) \ + (((lio_dev)->instr_queue[iq_no]->stats.field) += count) + +static inline void +lio_swap_8B_data(uint64_t *data, uint32_t blocks) +{ + while (blocks) { + *data = rte_cpu_to_be_64(*data); + blocks--; + data++; + } +} + +static inline uint64_t +lio_map_ring(void *buf) +{ + rte_iova_t dma_addr; + + dma_addr = rte_mbuf_data_iova_default(((struct rte_mbuf *)buf)); + + return (uint64_t)dma_addr; +} + +static inline uint64_t +lio_map_ring_info(struct lio_droq *droq, uint32_t i) +{ + rte_iova_t dma_addr; + + dma_addr = droq->info_list_dma + (i * LIO_DROQ_INFO_SIZE); + + return (uint64_t)dma_addr; +} + +static inline int +lio_opcode_slow_path(union octeon_rh *rh) +{ + uint16_t subcode1, subcode2; + + subcode1 = LIO_OPCODE_SUBCODE(rh->r.opcode, rh->r.subcode); + subcode2 = LIO_OPCODE_SUBCODE(LIO_OPCODE, LIO_OPCODE_NW_DATA); + + return subcode2 != subcode1; +} + +static inline void +lio_add_sg_size(struct lio_sg_entry *sg_entry, + uint16_t size, uint32_t pos) +{ +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + sg_entry->u.size[pos] = size; +#elif RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + sg_entry->u.size[3 - pos] = size; +#endif +} + +/* Macro to increment index. + * Index is incremented by count; if the sum exceeds + * max, index is wrapped-around to the start. + */ +static inline uint32_t +lio_incr_index(uint32_t index, uint32_t count, uint32_t max) +{ + if ((index + count) >= max) + index = index + count - max; + else + index += count; + + return index; +} + +int lio_setup_droq(struct lio_device *lio_dev, int q_no, int num_descs, + int desc_size, struct rte_mempool *mpool, + unsigned int socket_id); +uint16_t lio_dev_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, + uint16_t budget); +void lio_delete_droq_queue(struct lio_device *lio_dev, int oq_no); + +void lio_delete_sglist(struct lio_instr_queue *txq); +int lio_setup_sglists(struct lio_device *lio_dev, int iq_no, + int fw_mapped_iq, int num_descs, unsigned int socket_id); +uint16_t lio_dev_xmit_pkts(void *tx_queue, struct rte_mbuf **pkts, + uint16_t nb_pkts); +int lio_wait_for_instr_fetch(struct lio_device *lio_dev); +int lio_setup_iq(struct lio_device *lio_dev, int q_index, + union octeon_txpciq iq_no, uint32_t num_descs, void *app_ctx, + unsigned int socket_id); +int lio_flush_iq(struct lio_device *lio_dev, struct lio_instr_queue *iq); +void lio_delete_instruction_queue(struct lio_device *lio_dev, int iq_no); +/** Setup instruction queue zero for the device + * @param lio_dev which lio device to setup + * + * @return 0 if success. -1 if fails + */ +int lio_setup_instr_queue0(struct lio_device *lio_dev); +void lio_free_instr_queue0(struct lio_device *lio_dev); +void lio_dev_clear_queues(struct rte_eth_dev *eth_dev); +#endif /* _LIO_RXTX_H_ */ diff --git a/dpdk/drivers/net/liquidio/lio_struct.h b/dpdk/drivers/net/liquidio/lio_struct.h new file mode 100644 index 00000000..10e3976a --- /dev/null +++ b/dpdk/drivers/net/liquidio/lio_struct.h @@ -0,0 +1,690 @@ +/* + * BSD LICENSE + * + * Copyright(c) 2017 Cavium, Inc.. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _LIO_STRUCT_H_ +#define _LIO_STRUCT_H_ + +#include +#include +#include + +#include +#include + +#include "lio_hw_defs.h" + +struct lio_stailq_node { + STAILQ_ENTRY(lio_stailq_node) entries; +}; + +STAILQ_HEAD(lio_stailq_head, lio_stailq_node); + +struct lio_version { + uint16_t major; + uint16_t minor; + uint16_t micro; + uint16_t reserved; +}; + +/** Input Queue statistics. Each input queue has four stats fields. */ +struct lio_iq_stats { + uint64_t instr_posted; /**< Instructions posted to this queue. */ + uint64_t instr_processed; /**< Instructions processed in this queue. */ + uint64_t instr_dropped; /**< Instructions that could not be processed */ + uint64_t bytes_sent; /**< Bytes sent through this queue. */ + uint64_t tx_done; /**< Num of packets sent to network. */ + uint64_t tx_iq_busy; /**< Num of times this iq was found to be full. */ + uint64_t tx_dropped; /**< Num of pkts dropped due to xmitpath errors. */ + uint64_t tx_tot_bytes; /**< Total count of bytes sent to network. */ +}; + +/** Output Queue statistics. Each output queue has four stats fields. */ +struct lio_droq_stats { + /** Number of packets received in this queue. */ + uint64_t pkts_received; + + /** Bytes received by this queue. */ + uint64_t bytes_received; + + /** Packets dropped due to no memory available. */ + uint64_t dropped_nomem; + + /** Packets dropped due to large number of pkts to process. */ + uint64_t dropped_toomany; + + /** Number of packets sent to stack from this queue. */ + uint64_t rx_pkts_received; + + /** Number of Bytes sent to stack from this queue. */ + uint64_t rx_bytes_received; + + /** Num of Packets dropped due to receive path failures. */ + uint64_t rx_dropped; + + /** Num of vxlan packets received; */ + uint64_t rx_vxlan; + + /** Num of failures of rte_pktmbuf_alloc() */ + uint64_t rx_alloc_failure; + +}; + +/** The Descriptor Ring Output Queue structure. + * This structure has all the information required to implement a + * DROQ. + */ +struct lio_droq { + /** A spinlock to protect access to this ring. */ + rte_spinlock_t lock; + + uint32_t q_no; + + uint32_t pkt_count; + + struct lio_device *lio_dev; + + /** The 8B aligned descriptor ring starts at this address. */ + struct lio_droq_desc *desc_ring; + + /** Index in the ring where the driver should read the next packet */ + uint32_t read_idx; + + /** Index in the ring where Octeon will write the next packet */ + uint32_t write_idx; + + /** Index in the ring where the driver will refill the descriptor's + * buffer + */ + uint32_t refill_idx; + + /** Packets pending to be processed */ + rte_atomic64_t pkts_pending; + + /** Number of descriptors in this ring. */ + uint32_t max_count; + + /** The number of descriptors pending refill. */ + uint32_t refill_count; + + uint32_t refill_threshold; + + /** The 8B aligned info ptrs begin from this address. */ + struct lio_droq_info *info_list; + + /** The receive buffer list. This list has the virtual addresses of the + * buffers. + */ + struct lio_recv_buffer *recv_buf_list; + + /** The size of each buffer pointed by the buffer pointer. */ + uint32_t buffer_size; + + /** Pointer to the mapped packet credit register. + * Host writes number of info/buffer ptrs available to this register + */ + void *pkts_credit_reg; + + /** Pointer to the mapped packet sent register. + * Octeon writes the number of packets DMA'ed to host memory + * in this register. + */ + void *pkts_sent_reg; + + /** Statistics for this DROQ. */ + struct lio_droq_stats stats; + + /** DMA mapped address of the DROQ descriptor ring. */ + size_t desc_ring_dma; + + /** Info ptr list are allocated at this virtual address. */ + size_t info_base_addr; + + /** DMA mapped address of the info list */ + size_t info_list_dma; + + /** Allocated size of info list. */ + uint32_t info_alloc_size; + + /** Memory zone **/ + const struct rte_memzone *desc_ring_mz; + const struct rte_memzone *info_mz; + struct rte_mempool *mpool; +}; + +/** Receive Header */ +union octeon_rh { +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + uint64_t rh64; + struct { + uint64_t opcode : 4; + uint64_t subcode : 8; + uint64_t len : 3; /** additional 64-bit words */ + uint64_t reserved : 17; + uint64_t ossp : 32; /** opcode/subcode specific parameters */ + } r; + struct { + uint64_t opcode : 4; + uint64_t subcode : 8; + uint64_t len : 3; /** additional 64-bit words */ + uint64_t extra : 28; + uint64_t vlan : 12; + uint64_t priority : 3; + uint64_t csum_verified : 3; /** checksum verified. */ + uint64_t has_hwtstamp : 1; /** Has hardware timestamp.1 = yes.*/ + uint64_t encap_on : 1; + uint64_t has_hash : 1; /** Has hash (rth or rss). 1 = yes. */ + } r_dh; + struct { + uint64_t opcode : 4; + uint64_t subcode : 8; + uint64_t len : 3; /** additional 64-bit words */ + uint64_t reserved : 8; + uint64_t extra : 25; + uint64_t gmxport : 16; + } r_nic_info; +#else + uint64_t rh64; + struct { + uint64_t ossp : 32; /** opcode/subcode specific parameters */ + uint64_t reserved : 17; + uint64_t len : 3; /** additional 64-bit words */ + uint64_t subcode : 8; + uint64_t opcode : 4; + } r; + struct { + uint64_t has_hash : 1; /** Has hash (rth or rss). 1 = yes. */ + uint64_t encap_on : 1; + uint64_t has_hwtstamp : 1; /** 1 = has hwtstamp */ + uint64_t csum_verified : 3; /** checksum verified. */ + uint64_t priority : 3; + uint64_t vlan : 12; + uint64_t extra : 28; + uint64_t len : 3; /** additional 64-bit words */ + uint64_t subcode : 8; + uint64_t opcode : 4; + } r_dh; + struct { + uint64_t gmxport : 16; + uint64_t extra : 25; + uint64_t reserved : 8; + uint64_t len : 3; /** additional 64-bit words */ + uint64_t subcode : 8; + uint64_t opcode : 4; + } r_nic_info; +#endif +}; + +#define OCTEON_RH_SIZE (sizeof(union octeon_rh)) + +/** The txpciq info passed to host from the firmware */ +union octeon_txpciq { + uint64_t txpciq64; + + struct { +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + uint64_t q_no : 8; + uint64_t port : 8; + uint64_t pkind : 6; + uint64_t use_qpg : 1; + uint64_t qpg : 11; + uint64_t aura_num : 10; + uint64_t reserved : 20; +#else + uint64_t reserved : 20; + uint64_t aura_num : 10; + uint64_t qpg : 11; + uint64_t use_qpg : 1; + uint64_t pkind : 6; + uint64_t port : 8; + uint64_t q_no : 8; +#endif + } s; +}; + +/** The instruction (input) queue. + * The input queue is used to post raw (instruction) mode data or packet + * data to Octeon device from the host. Each input queue for + * a LIO device has one such structure to represent it. + */ +struct lio_instr_queue { + /** A spinlock to protect access to the input ring. */ + rte_spinlock_t lock; + + rte_spinlock_t post_lock; + + struct lio_device *lio_dev; + + uint32_t pkt_in_done; + + rte_atomic64_t iq_flush_running; + + /** Flag that indicates if the queue uses 64 byte commands. */ + uint32_t iqcmd_64B:1; + + /** Queue info. */ + union octeon_txpciq txpciq; + + uint32_t rsvd:17; + + uint32_t status:8; + + /** Maximum no. of instructions in this queue. */ + uint32_t max_count; + + /** Index in input ring where the driver should write the next packet */ + uint32_t host_write_index; + + /** Index in input ring where Octeon is expected to read the next + * packet. + */ + uint32_t lio_read_index; + + /** This index aids in finding the window in the queue where Octeon + * has read the commands. + */ + uint32_t flush_index; + + /** This field keeps track of the instructions pending in this queue. */ + rte_atomic64_t instr_pending; + + /** Pointer to the Virtual Base addr of the input ring. */ + uint8_t *base_addr; + + struct lio_request_list *request_list; + + /** Octeon doorbell register for the ring. */ + void *doorbell_reg; + + /** Octeon instruction count register for this ring. */ + void *inst_cnt_reg; + + /** Number of instructions pending to be posted to Octeon. */ + uint32_t fill_cnt; + + /** Statistics for this input queue. */ + struct lio_iq_stats stats; + + /** DMA mapped base address of the input descriptor ring. */ + uint64_t base_addr_dma; + + /** Application context */ + void *app_ctx; + + /* network stack queue index */ + int q_index; + + /* Memory zone */ + const struct rte_memzone *iq_mz; +}; + +/** This structure is used by driver to store information required + * to free the mbuff when the packet has been fetched by Octeon. + * Bytes offset below assume worst-case of a 64-bit system. + */ +struct lio_buf_free_info { + /** Bytes 1-8. Pointer to network device private structure. */ + struct lio_device *lio_dev; + + /** Bytes 9-16. Pointer to mbuff. */ + struct rte_mbuf *mbuf; + + /** Bytes 17-24. Pointer to gather list. */ + struct lio_gather *g; + + /** Bytes 25-32. Physical address of mbuf->data or gather list. */ + uint64_t dptr; + + /** Bytes 33-47. Piggybacked soft command, if any */ + struct lio_soft_command *sc; + + /** Bytes 48-63. iq no */ + uint64_t iq_no; +}; + +/* The Scatter-Gather List Entry. The scatter or gather component used with + * input instruction has this format. + */ +struct lio_sg_entry { + /** The first 64 bit gives the size of data in each dptr. */ + union { + uint16_t size[4]; + uint64_t size64; + } u; + + /** The 4 dptr pointers for this entry. */ + uint64_t ptr[4]; +}; + +#define LIO_SG_ENTRY_SIZE (sizeof(struct lio_sg_entry)) + +/** Structure of a node in list of gather components maintained by + * driver for each network device. + */ +struct lio_gather { + /** List manipulation. Next and prev pointers. */ + struct lio_stailq_node list; + + /** Size of the gather component at sg in bytes. */ + int sg_size; + + /** Number of bytes that sg was adjusted to make it 8B-aligned. */ + int adjust; + + /** Gather component that can accommodate max sized fragment list + * received from the IP layer. + */ + struct lio_sg_entry *sg; +}; + +struct lio_rss_ctx { + uint16_t hash_key_size; + uint8_t hash_key[LIO_RSS_MAX_KEY_SZ]; + /* Ideally a factor of number of queues */ + uint8_t itable[LIO_RSS_MAX_TABLE_SZ]; + uint8_t itable_size; + uint8_t ip; + uint8_t tcp_hash; + uint8_t ipv6; + uint8_t ipv6_tcp_hash; + uint8_t ipv6_ex; + uint8_t ipv6_tcp_ex_hash; + uint8_t hash_disable; +}; + +struct lio_io_enable { + uint64_t iq; + uint64_t oq; + uint64_t iq64B; +}; + +struct lio_fn_list { + void (*setup_iq_regs)(struct lio_device *, uint32_t); + void (*setup_oq_regs)(struct lio_device *, uint32_t); + + int (*setup_mbox)(struct lio_device *); + void (*free_mbox)(struct lio_device *); + + int (*setup_device_regs)(struct lio_device *); + int (*enable_io_queues)(struct lio_device *); + void (*disable_io_queues)(struct lio_device *); +}; + +struct lio_pf_vf_hs_word { +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + /** PKIND value assigned for the DPI interface */ + uint64_t pkind : 8; + + /** OCTEON core clock multiplier */ + uint64_t core_tics_per_us : 16; + + /** OCTEON coprocessor clock multiplier */ + uint64_t coproc_tics_per_us : 16; + + /** app that currently running on OCTEON */ + uint64_t app_mode : 8; + + /** RESERVED */ + uint64_t reserved : 16; + +#elif RTE_BYTE_ORDER == RTE_BIG_ENDIAN + + /** RESERVED */ + uint64_t reserved : 16; + + /** app that currently running on OCTEON */ + uint64_t app_mode : 8; + + /** OCTEON coprocessor clock multiplier */ + uint64_t coproc_tics_per_us : 16; + + /** OCTEON core clock multiplier */ + uint64_t core_tics_per_us : 16; + + /** PKIND value assigned for the DPI interface */ + uint64_t pkind : 8; +#endif +}; + +struct lio_sriov_info { + /** Number of rings assigned to VF */ + uint32_t rings_per_vf; + + /** Number of VF devices enabled */ + uint32_t num_vfs; +}; + +/* Head of a response list */ +struct lio_response_list { + /** List structure to add delete pending entries to */ + struct lio_stailq_head head; + + /** A lock for this response list */ + rte_spinlock_t lock; + + rte_atomic64_t pending_req_count; +}; + +/* Structure to define the configuration attributes for each Input queue. */ +struct lio_iq_config { + /* Max number of IQs available */ + uint8_t max_iqs; + + /** Pending list size (usually set to the sum of the size of all Input + * queues) + */ + uint32_t pending_list_size; + + /** Command size - 32 or 64 bytes */ + uint32_t instr_type; +}; + +/* Structure to define the configuration attributes for each Output queue. */ +struct lio_oq_config { + /* Max number of OQs available */ + uint8_t max_oqs; + + /** If set, the Output queue uses info-pointer mode. (Default: 1 ) */ + uint32_t info_ptr; + + /** The number of buffers that were consumed during packet processing by + * the driver on this Output queue before the driver attempts to + * replenish the descriptor ring with new buffers. + */ + uint32_t refill_threshold; +}; + +/* Structure to define the configuration. */ +struct lio_config { + uint16_t card_type; + const char *card_name; + + /** Input Queue attributes. */ + struct lio_iq_config iq; + + /** Output Queue attributes. */ + struct lio_oq_config oq; + + int num_nic_ports; + + int num_def_tx_descs; + + /* Num of desc for rx rings */ + int num_def_rx_descs; + + int def_rx_buf_size; +}; + +/** Status of a RGMII Link on Octeon as seen by core driver. */ +union octeon_link_status { + uint64_t link_status64; + + struct { +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + uint64_t duplex : 8; + uint64_t mtu : 16; + uint64_t speed : 16; + uint64_t link_up : 1; + uint64_t autoneg : 1; + uint64_t if_mode : 5; + uint64_t pause : 1; + uint64_t flashing : 1; + uint64_t reserved : 15; +#else + uint64_t reserved : 15; + uint64_t flashing : 1; + uint64_t pause : 1; + uint64_t if_mode : 5; + uint64_t autoneg : 1; + uint64_t link_up : 1; + uint64_t speed : 16; + uint64_t mtu : 16; + uint64_t duplex : 8; +#endif + } s; +}; + +/** The rxpciq info passed to host from the firmware */ +union octeon_rxpciq { + uint64_t rxpciq64; + + struct { +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + uint64_t q_no : 8; + uint64_t reserved : 56; +#else + uint64_t reserved : 56; + uint64_t q_no : 8; +#endif + } s; +}; + +/** Information for a OCTEON ethernet interface shared between core & host. */ +struct octeon_link_info { + union octeon_link_status link; + uint64_t hw_addr; + +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + uint64_t gmxport : 16; + uint64_t macaddr_is_admin_assigned : 1; + uint64_t vlan_is_admin_assigned : 1; + uint64_t rsvd : 30; + uint64_t num_txpciq : 8; + uint64_t num_rxpciq : 8; +#else + uint64_t num_rxpciq : 8; + uint64_t num_txpciq : 8; + uint64_t rsvd : 30; + uint64_t vlan_is_admin_assigned : 1; + uint64_t macaddr_is_admin_assigned : 1; + uint64_t gmxport : 16; +#endif + + union octeon_txpciq txpciq[LIO_MAX_IOQS_PER_IF]; + union octeon_rxpciq rxpciq[LIO_MAX_IOQS_PER_IF]; +}; + +/* ----------------------- THE LIO DEVICE --------------------------- */ +/** The lio device. + * Each lio device has this structure to represent all its + * components. + */ +struct lio_device { + /** PCI device pointer */ + struct rte_pci_device *pci_dev; + + /** Octeon Chip type */ + uint16_t chip_id; + uint16_t pf_num; + uint16_t vf_num; + + /** This device's PCIe port used for traffic. */ + uint16_t pcie_port; + + /** The state of this device */ + rte_atomic64_t status; + + uint8_t intf_open; + + struct octeon_link_info linfo; + + uint8_t *hw_addr; + + struct lio_fn_list fn_list; + + uint32_t num_iqs; + + /** Guards each glist */ + rte_spinlock_t *glist_lock; + /** Array of gather component linked lists */ + struct lio_stailq_head *glist_head; + + /* The pool containing pre allocated buffers used for soft commands */ + struct rte_mempool *sc_buf_pool; + + /** The input instruction queues */ + struct lio_instr_queue *instr_queue[LIO_MAX_POSSIBLE_INSTR_QUEUES]; + + /** The singly-linked tail queues of instruction response */ + struct lio_response_list response_list; + + uint32_t num_oqs; + + /** The DROQ output queues */ + struct lio_droq *droq[LIO_MAX_POSSIBLE_OUTPUT_QUEUES]; + + struct lio_io_enable io_qmask; + + struct lio_sriov_info sriov_info; + + struct lio_pf_vf_hs_word pfvf_hsword; + + /** Mail Box details of each lio queue. */ + struct lio_mbox **mbox; + + char dev_string[LIO_DEVICE_NAME_LEN]; /* Device print string */ + + const struct lio_config *default_config; + + struct rte_eth_dev *eth_dev; + + uint64_t ifflags; + uint8_t max_rx_queues; + uint8_t max_tx_queues; + uint8_t nb_rx_queues; + uint8_t nb_tx_queues; + uint8_t port_configured; + struct lio_rss_ctx rss_state; + uint16_t port_id; + char firmware_version[LIO_FW_VERSION_LENGTH]; +}; +#endif /* _LIO_STRUCT_H_ */ diff --git a/dpdk/drivers/net/liquidio/rte_pmd_lio_version.map b/dpdk/drivers/net/liquidio/rte_pmd_lio_version.map new file mode 100644 index 00000000..8591cc0b --- /dev/null +++ b/dpdk/drivers/net/liquidio/rte_pmd_lio_version.map @@ -0,0 +1,4 @@ +DPDK_17.05 { + + local: *; +}; diff --git a/dpdk/drivers/net/mlx4/Makefile b/dpdk/drivers/net/mlx4/Makefile index efed953e..1f95e0df 100644 --- a/dpdk/drivers/net/mlx4/Makefile +++ b/dpdk/drivers/net/mlx4/Makefile @@ -1,7 +1,7 @@ # BSD LICENSE # -# Copyright 2012-2015 6WIND S.A. -# Copyright 2012 Mellanox. +# Copyright 2012 6WIND S.A. +# Copyright 2012 Mellanox # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -36,23 +36,28 @@ LIB = librte_pmd_mlx4.a # Sources. SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4.c - -# Dependencies. -DEPDIRS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += lib/librte_ether -DEPDIRS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += lib/librte_mbuf -DEPDIRS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += lib/librte_eal -DEPDIRS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += lib/librte_mempool +SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4_ethdev.c +SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4_flow.c +SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4_intr.c +SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4_mr.c +SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4_rxq.c +SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4_rxtx.c +SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4_txq.c +SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += mlx4_utils.c # Basic CFLAGS. CFLAGS += -O3 -CFLAGS += -std=gnu99 -Wall -Wextra +CFLAGS += -std=c11 -Wall -Wextra CFLAGS += -g CFLAGS += -I. CFLAGS += -D_BSD_SOURCE CFLAGS += -D_DEFAULT_SOURCE CFLAGS += -D_XOPEN_SOURCE=600 CFLAGS += $(WERROR_FLAGS) -LDLIBS += -libverbs +LDLIBS += -libverbs -lmlx4 +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs +LDLIBS += -lrte_bus_pci # A few warnings cannot be avoided in external headers. CFLAGS += -Wno-error=cast-qual @@ -73,22 +78,10 @@ else CFLAGS += -DNDEBUG -UPEDANTIC endif -ifdef CONFIG_RTE_LIBRTE_MLX4_SGE_WR_N -CFLAGS += -DMLX4_PMD_SGE_WR_N=$(CONFIG_RTE_LIBRTE_MLX4_SGE_WR_N) -endif - -ifdef CONFIG_RTE_LIBRTE_MLX4_MAX_INLINE -CFLAGS += -DMLX4_PMD_MAX_INLINE=$(CONFIG_RTE_LIBRTE_MLX4_MAX_INLINE) -endif - ifdef CONFIG_RTE_LIBRTE_MLX4_TX_MP_CACHE CFLAGS += -DMLX4_PMD_TX_MP_CACHE=$(CONFIG_RTE_LIBRTE_MLX4_TX_MP_CACHE) endif -ifdef CONFIG_RTE_LIBRTE_MLX4_SOFT_COUNTERS -CFLAGS += -DMLX4_PMD_SOFT_COUNTERS=$(CONFIG_RTE_LIBRTE_MLX4_SOFT_COUNTERS) -endif - include $(RTE_SDK)/mk/rte.lib.mk # Generate and clean-up mlx4_autoconf.h. @@ -102,25 +95,9 @@ endif mlx4_autoconf.h.new: FORCE -mlx4_autoconf.h.new: $(RTE_SDK)/scripts/auto-config-h.sh +mlx4_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh $Q $(RM) -f -- '$@' - $Q sh -- '$<' '$@' \ - RSS_SUPPORT \ - infiniband/verbs.h \ - enum IBV_EXP_DEVICE_UD_RSS $(AUTOCONF_OUTPUT) - $Q sh -- '$<' '$@' \ - INLINE_RECV \ - infiniband/verbs.h \ - enum IBV_EXP_DEVICE_ATTR_INLINE_RECV_SZ $(AUTOCONF_OUTPUT) - $Q sh -- '$<' '$@' \ - HAVE_EXP_QUERY_DEVICE \ - infiniband/verbs.h \ - type 'struct ibv_exp_device_attr' $(AUTOCONF_OUTPUT) - $Q sh -- '$<' '$@' \ - HAVE_EXP_QP_BURST_CREATE_DISABLE_ETH_LOOPBACK \ - infiniband/verbs.h \ - enum IBV_EXP_QP_BURST_CREATE_DISABLE_ETH_LOOPBACK \ - $(AUTOCONF_OUTPUT) + $Q : > '$@' # Create mlx4_autoconf.h or update it in case it differs from the new one. @@ -129,7 +106,7 @@ mlx4_autoconf.h: mlx4_autoconf.h.new cmp '$<' '$@' $(AUTOCONF_OUTPUT) || \ mv '$<' '$@' -mlx4.o: mlx4_autoconf.h +$(SRCS-$(CONFIG_RTE_LIBRTE_MLX4_PMD):.c=.o): mlx4_autoconf.h clean_mlx4: FORCE $Q rm -f -- mlx4_autoconf.h mlx4_autoconf.h.new diff --git a/dpdk/drivers/net/mlx4/mlx4.c b/dpdk/drivers/net/mlx4/mlx4.c index 9f276192..97dac643 100644 --- a/dpdk/drivers/net/mlx4/mlx4.c +++ b/dpdk/drivers/net/mlx4/mlx4.c @@ -1,8 +1,8 @@ /*- * BSD LICENSE * - * Copyright 2012-2015 6WIND S.A. - * Copyright 2012 Mellanox. + * Copyright 2012 6WIND S.A. + * Copyright 2012 Mellanox * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -31,35 +31,21 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -/* - * Known limitations: - * - RSS hash key and options cannot be modified. - * - Hardware counters aren't implemented. +/** + * @file + * mlx4 driver initialization. */ -/* System headers. */ +#include +#include +#include #include +#include #include #include -#include -#include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -/* Verbs header. */ -/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ +/* Verbs headers do not support -pedantic. */ #ifdef PEDANTIC #pragma GCC diagnostic ignored "-Wpedantic" #endif @@ -68,732 +54,37 @@ #pragma GCC diagnostic error "-Wpedantic" #endif -/* DPDK headers don't like -pedantic. */ -#ifdef PEDANTIC -#pragma GCC diagnostic ignored "-Wpedantic" -#endif -#include -#include +#include #include -#include #include -#include -#include +#include +#include +#include +#include +#include +#include #include -#include -#include -#include -#include -#include -#include -#ifdef PEDANTIC -#pragma GCC diagnostic error "-Wpedantic" -#endif +#include -/* Generated configuration header. */ -#include "mlx4_autoconf.h" - -/* PMD header. */ #include "mlx4.h" +#include "mlx4_flow.h" +#include "mlx4_rxtx.h" +#include "mlx4_utils.h" -/* Runtime logging through RTE_LOG() is enabled when not in debugging mode. - * Intermediate LOG_*() macros add the required end-of-line characters. */ -#ifndef NDEBUG -#define INFO(...) DEBUG(__VA_ARGS__) -#define WARN(...) DEBUG(__VA_ARGS__) -#define ERROR(...) DEBUG(__VA_ARGS__) -#else -#define LOG__(level, m, ...) \ - RTE_LOG(level, PMD, MLX4_DRIVER_NAME ": " m "%c", __VA_ARGS__) -#define LOG_(level, ...) LOG__(level, __VA_ARGS__, '\n') -#define INFO(...) LOG_(INFO, __VA_ARGS__) -#define WARN(...) LOG_(WARNING, __VA_ARGS__) -#define ERROR(...) LOG_(ERR, __VA_ARGS__) -#endif - -/* Convenience macros for accessing mbuf fields. */ -#define NEXT(m) ((m)->next) -#define DATA_LEN(m) ((m)->data_len) -#define PKT_LEN(m) ((m)->pkt_len) -#define DATA_OFF(m) ((m)->data_off) -#define SET_DATA_OFF(m, o) ((m)->data_off = (o)) -#define NB_SEGS(m) ((m)->nb_segs) -#define PORT(m) ((m)->port) - -/* Work Request ID data type (64 bit). */ -typedef union { +/** Configuration structure for device arguments. */ +struct mlx4_conf { struct { - uint32_t id; - uint16_t offset; - } data; - uint64_t raw; -} wr_id_t; - -#define WR_ID(o) (((wr_id_t *)&(o))->data) - -/* Transpose flags. Useful to convert IBV to DPDK flags. */ -#define TRANSPOSE(val, from, to) \ - (((from) >= (to)) ? \ - (((val) & (from)) / ((from) / (to))) : \ - (((val) & (from)) * ((to) / (from)))) - -struct mlx4_rxq_stats { - unsigned int idx; /**< Mapping index. */ -#ifdef MLX4_PMD_SOFT_COUNTERS - uint64_t ipackets; /**< Total of successfully received packets. */ - uint64_t ibytes; /**< Total of successfully received bytes. */ -#endif - uint64_t idropped; /**< Total of packets dropped when RX ring full. */ - uint64_t rx_nombuf; /**< Total of RX mbuf allocation failures. */ + uint32_t present; /**< Bit-field for existing ports. */ + uint32_t enabled; /**< Bit-field for user-enabled ports. */ + } ports; }; -struct mlx4_txq_stats { - unsigned int idx; /**< Mapping index. */ -#ifdef MLX4_PMD_SOFT_COUNTERS - uint64_t opackets; /**< Total of successfully sent packets. */ - uint64_t obytes; /**< Total of successfully sent bytes. */ -#endif - uint64_t odropped; /**< Total of packets not sent when TX ring full. */ +/* Available parameters list. */ +const char *pmd_mlx4_init_params[] = { + MLX4_PMD_PORT_KVARG, + NULL, }; -/* RX element (scattered packets). */ -struct rxq_elt_sp { - struct ibv_recv_wr wr; /* Work Request. */ - struct ibv_sge sges[MLX4_PMD_SGE_WR_N]; /* Scatter/Gather Elements. */ - struct rte_mbuf *bufs[MLX4_PMD_SGE_WR_N]; /* SGEs buffers. */ -}; - -/* RX element. */ -struct rxq_elt { - struct ibv_recv_wr wr; /* Work Request. */ - struct ibv_sge sge; /* Scatter/Gather Element. */ - /* mbuf pointer is derived from WR_ID(wr.wr_id).offset. */ -}; - -/* RX queue descriptor. */ -struct rxq { - struct priv *priv; /* Back pointer to private data. */ - struct rte_mempool *mp; /* Memory Pool for allocations. */ - struct ibv_mr *mr; /* Memory Region (for mp). */ - struct ibv_cq *cq; /* Completion Queue. */ - struct ibv_qp *qp; /* Queue Pair. */ - struct ibv_exp_qp_burst_family *if_qp; /* QP burst interface. */ - struct ibv_exp_cq_family *if_cq; /* CQ interface. */ - /* - * Each VLAN ID requires a separate flow steering rule. - */ - BITFIELD_DECLARE(mac_configured, uint32_t, MLX4_MAX_MAC_ADDRESSES); - struct ibv_flow *mac_flow[MLX4_MAX_MAC_ADDRESSES][MLX4_MAX_VLAN_IDS]; - struct ibv_flow *promisc_flow; /* Promiscuous flow. */ - struct ibv_flow *allmulti_flow; /* Multicast flow. */ - unsigned int port_id; /* Port ID for incoming packets. */ - unsigned int elts_n; /* (*elts)[] length. */ - unsigned int elts_head; /* Current index in (*elts)[]. */ - union { - struct rxq_elt_sp (*sp)[]; /* Scattered RX elements. */ - struct rxq_elt (*no_sp)[]; /* RX elements. */ - } elts; - unsigned int sp:1; /* Use scattered RX elements. */ - unsigned int csum:1; /* Enable checksum offloading. */ - unsigned int csum_l2tun:1; /* Same for L2 tunnels. */ - struct mlx4_rxq_stats stats; /* RX queue counters. */ - unsigned int socket; /* CPU socket ID for allocations. */ - struct ibv_exp_res_domain *rd; /* Resource Domain. */ -}; - -/* TX element. */ -struct txq_elt { - struct rte_mbuf *buf; -}; - -/* Linear buffer type. It is used when transmitting buffers with too many - * segments that do not fit the hardware queue (see max_send_sge). - * Extra segments are copied (linearized) in such buffers, replacing the - * last SGE during TX. - * The size is arbitrary but large enough to hold a jumbo frame with - * 8 segments considering mbuf.buf_len is about 2048 bytes. */ -typedef uint8_t linear_t[16384]; - -/* TX queue descriptor. */ -struct txq { - struct priv *priv; /* Back pointer to private data. */ - struct { - const struct rte_mempool *mp; /* Cached Memory Pool. */ - struct ibv_mr *mr; /* Memory Region (for mp). */ - uint32_t lkey; /* mr->lkey */ - } mp2mr[MLX4_PMD_TX_MP_CACHE]; /* MP to MR translation table. */ - struct ibv_cq *cq; /* Completion Queue. */ - struct ibv_qp *qp; /* Queue Pair. */ - struct ibv_exp_qp_burst_family *if_qp; /* QP burst interface. */ - struct ibv_exp_cq_family *if_cq; /* CQ interface. */ -#if MLX4_PMD_MAX_INLINE > 0 - uint32_t max_inline; /* Max inline send size <= MLX4_PMD_MAX_INLINE. */ -#endif - unsigned int elts_n; /* (*elts)[] length. */ - struct txq_elt (*elts)[]; /* TX elements. */ - unsigned int elts_head; /* Current index in (*elts)[]. */ - unsigned int elts_tail; /* First element awaiting completion. */ - unsigned int elts_comp; /* Number of completion requests. */ - unsigned int elts_comp_cd; /* Countdown for next completion request. */ - unsigned int elts_comp_cd_init; /* Initial value for countdown. */ - struct mlx4_txq_stats stats; /* TX queue counters. */ - linear_t (*elts_linear)[]; /* Linearized buffers. */ - struct ibv_mr *mr_linear; /* Memory Region for linearized buffers. */ - unsigned int socket; /* CPU socket ID for allocations. */ - struct ibv_exp_res_domain *rd; /* Resource Domain. */ -}; - -struct priv { - struct rte_eth_dev *dev; /* Ethernet device. */ - struct ibv_context *ctx; /* Verbs context. */ - struct ibv_device_attr device_attr; /* Device properties. */ - struct ibv_pd *pd; /* Protection Domain. */ - /* - * MAC addresses array and configuration bit-field. - * An extra entry that cannot be modified by the DPDK is reserved - * for broadcast frames (destination MAC address ff:ff:ff:ff:ff:ff). - */ - struct ether_addr mac[MLX4_MAX_MAC_ADDRESSES]; - BITFIELD_DECLARE(mac_configured, uint32_t, MLX4_MAX_MAC_ADDRESSES); - /* VLAN filters. */ - struct { - unsigned int enabled:1; /* If enabled. */ - unsigned int id:12; /* VLAN ID (0-4095). */ - } vlan_filter[MLX4_MAX_VLAN_IDS]; /* VLAN filters table. */ - /* Device properties. */ - uint16_t mtu; /* Configured MTU. */ - uint8_t port; /* Physical port number. */ - unsigned int started:1; /* Device started, flows enabled. */ - unsigned int promisc:1; /* Device in promiscuous mode. */ - unsigned int allmulti:1; /* Device receives all multicast packets. */ - unsigned int hw_qpg:1; /* QP groups are supported. */ - unsigned int hw_tss:1; /* TSS is supported. */ - unsigned int hw_rss:1; /* RSS is supported. */ - unsigned int hw_csum:1; /* Checksum offload is supported. */ - unsigned int hw_csum_l2tun:1; /* Same for L2 tunnels. */ - unsigned int rss:1; /* RSS is enabled. */ - unsigned int vf:1; /* This is a VF device. */ - unsigned int pending_alarm:1; /* An alarm is pending. */ -#ifdef INLINE_RECV - unsigned int inl_recv_size; /* Inline recv size */ -#endif - unsigned int max_rss_tbl_sz; /* Maximum number of RSS queues. */ - /* RX/TX queues. */ - struct rxq rxq_parent; /* Parent queue when RSS is enabled. */ - unsigned int rxqs_n; /* RX queues array size. */ - unsigned int txqs_n; /* TX queues array size. */ - struct rxq *(*rxqs)[]; /* RX queues. */ - struct txq *(*txqs)[]; /* TX queues. */ - struct rte_intr_handle intr_handle; /* Interrupt handler. */ - rte_spinlock_t lock; /* Lock for control functions. */ -}; - -/* Local storage for secondary process data. */ -struct mlx4_secondary_data { - struct rte_eth_dev_data data; /* Local device data. */ - struct priv *primary_priv; /* Private structure from primary. */ - struct rte_eth_dev_data *shared_dev_data; /* Shared device data. */ - rte_spinlock_t lock; /* Port configuration lock. */ -} mlx4_secondary_data[RTE_MAX_ETHPORTS]; - -/** - * Check if running as a secondary process. - * - * @return - * Nonzero if running as a secondary process. - */ -static inline int -mlx4_is_secondary(void) -{ - return rte_eal_process_type() != RTE_PROC_PRIMARY; -} - -/** - * Return private structure associated with an Ethernet device. - * - * @param dev - * Pointer to Ethernet device structure. - * - * @return - * Pointer to private structure. - */ -static struct priv * -mlx4_get_priv(struct rte_eth_dev *dev) -{ - struct mlx4_secondary_data *sd; - - if (!mlx4_is_secondary()) - return dev->data->dev_private; - sd = &mlx4_secondary_data[dev->data->port_id]; - return sd->data.dev_private; -} - -/** - * Lock private structure to protect it from concurrent access in the - * control path. - * - * @param priv - * Pointer to private structure. - */ -static void -priv_lock(struct priv *priv) -{ - rte_spinlock_lock(&priv->lock); -} - -/** - * Unlock private structure. - * - * @param priv - * Pointer to private structure. - */ -static void -priv_unlock(struct priv *priv) -{ - rte_spinlock_unlock(&priv->lock); -} - -/* Allocate a buffer on the stack and fill it with a printf format string. */ -#define MKSTR(name, ...) \ - char name[snprintf(NULL, 0, __VA_ARGS__) + 1]; \ - \ - snprintf(name, sizeof(name), __VA_ARGS__) - -/** - * Get interface name from private structure. - * - * @param[in] priv - * Pointer to private structure. - * @param[out] ifname - * Interface name output buffer. - * - * @return - * 0 on success, -1 on failure and errno is set. - */ -static int -priv_get_ifname(const struct priv *priv, char (*ifname)[IF_NAMESIZE]) -{ - DIR *dir; - struct dirent *dent; - unsigned int dev_type = 0; - unsigned int dev_port_prev = ~0u; - char match[IF_NAMESIZE] = ""; - - { - MKSTR(path, "%s/device/net", priv->ctx->device->ibdev_path); - - dir = opendir(path); - if (dir == NULL) - return -1; - } - while ((dent = readdir(dir)) != NULL) { - char *name = dent->d_name; - FILE *file; - unsigned int dev_port; - int r; - - if ((name[0] == '.') && - ((name[1] == '\0') || - ((name[1] == '.') && (name[2] == '\0')))) - continue; - - MKSTR(path, "%s/device/net/%s/%s", - priv->ctx->device->ibdev_path, name, - (dev_type ? "dev_id" : "dev_port")); - - file = fopen(path, "rb"); - if (file == NULL) { - if (errno != ENOENT) - continue; - /* - * Switch to dev_id when dev_port does not exist as - * is the case with Linux kernel versions < 3.15. - */ -try_dev_id: - match[0] = '\0'; - if (dev_type) - break; - dev_type = 1; - dev_port_prev = ~0u; - rewinddir(dir); - continue; - } - r = fscanf(file, (dev_type ? "%x" : "%u"), &dev_port); - fclose(file); - if (r != 1) - continue; - /* - * Switch to dev_id when dev_port returns the same value for - * all ports. May happen when using a MOFED release older than - * 3.0 with a Linux kernel >= 3.15. - */ - if (dev_port == dev_port_prev) - goto try_dev_id; - dev_port_prev = dev_port; - if (dev_port == (priv->port - 1u)) - snprintf(match, sizeof(match), "%s", name); - } - closedir(dir); - if (match[0] == '\0') - return -1; - strncpy(*ifname, match, sizeof(*ifname)); - return 0; -} - -/** - * Read from sysfs entry. - * - * @param[in] priv - * Pointer to private structure. - * @param[in] entry - * Entry name relative to sysfs path. - * @param[out] buf - * Data output buffer. - * @param size - * Buffer size. - * - * @return - * 0 on success, -1 on failure and errno is set. - */ -static int -priv_sysfs_read(const struct priv *priv, const char *entry, - char *buf, size_t size) -{ - char ifname[IF_NAMESIZE]; - FILE *file; - int ret; - int err; - - if (priv_get_ifname(priv, &ifname)) - return -1; - - MKSTR(path, "%s/device/net/%s/%s", priv->ctx->device->ibdev_path, - ifname, entry); - - file = fopen(path, "rb"); - if (file == NULL) - return -1; - ret = fread(buf, 1, size, file); - err = errno; - if (((size_t)ret < size) && (ferror(file))) - ret = -1; - else - ret = size; - fclose(file); - errno = err; - return ret; -} - -/** - * Write to sysfs entry. - * - * @param[in] priv - * Pointer to private structure. - * @param[in] entry - * Entry name relative to sysfs path. - * @param[in] buf - * Data buffer. - * @param size - * Buffer size. - * - * @return - * 0 on success, -1 on failure and errno is set. - */ -static int -priv_sysfs_write(const struct priv *priv, const char *entry, - char *buf, size_t size) -{ - char ifname[IF_NAMESIZE]; - FILE *file; - int ret; - int err; - - if (priv_get_ifname(priv, &ifname)) - return -1; - - MKSTR(path, "%s/device/net/%s/%s", priv->ctx->device->ibdev_path, - ifname, entry); - - file = fopen(path, "wb"); - if (file == NULL) - return -1; - ret = fwrite(buf, 1, size, file); - err = errno; - if (((size_t)ret < size) || (ferror(file))) - ret = -1; - else - ret = size; - fclose(file); - errno = err; - return ret; -} - -/** - * Get unsigned long sysfs property. - * - * @param priv - * Pointer to private structure. - * @param[in] name - * Entry name relative to sysfs path. - * @param[out] value - * Value output buffer. - * - * @return - * 0 on success, -1 on failure and errno is set. - */ -static int -priv_get_sysfs_ulong(struct priv *priv, const char *name, unsigned long *value) -{ - int ret; - unsigned long value_ret; - char value_str[32]; - - ret = priv_sysfs_read(priv, name, value_str, (sizeof(value_str) - 1)); - if (ret == -1) { - DEBUG("cannot read %s value from sysfs: %s", - name, strerror(errno)); - return -1; - } - value_str[ret] = '\0'; - errno = 0; - value_ret = strtoul(value_str, NULL, 0); - if (errno) { - DEBUG("invalid %s value `%s': %s", name, value_str, - strerror(errno)); - return -1; - } - *value = value_ret; - return 0; -} - -/** - * Set unsigned long sysfs property. - * - * @param priv - * Pointer to private structure. - * @param[in] name - * Entry name relative to sysfs path. - * @param value - * Value to set. - * - * @return - * 0 on success, -1 on failure and errno is set. - */ -static int -priv_set_sysfs_ulong(struct priv *priv, const char *name, unsigned long value) -{ - int ret; - MKSTR(value_str, "%lu", value); - - ret = priv_sysfs_write(priv, name, value_str, (sizeof(value_str) - 1)); - if (ret == -1) { - DEBUG("cannot write %s `%s' (%lu) to sysfs: %s", - name, value_str, value, strerror(errno)); - return -1; - } - return 0; -} - -/** - * Perform ifreq ioctl() on associated Ethernet device. - * - * @param[in] priv - * Pointer to private structure. - * @param req - * Request number to pass to ioctl(). - * @param[out] ifr - * Interface request structure output buffer. - * - * @return - * 0 on success, -1 on failure and errno is set. - */ -static int -priv_ifreq(const struct priv *priv, int req, struct ifreq *ifr) -{ - int sock = socket(PF_INET, SOCK_DGRAM, IPPROTO_IP); - int ret = -1; - - if (sock == -1) - return ret; - if (priv_get_ifname(priv, &ifr->ifr_name) == 0) - ret = ioctl(sock, req, ifr); - close(sock); - return ret; -} - -/** - * Get device MTU. - * - * @param priv - * Pointer to private structure. - * @param[out] mtu - * MTU value output buffer. - * - * @return - * 0 on success, -1 on failure and errno is set. - */ -static int -priv_get_mtu(struct priv *priv, uint16_t *mtu) -{ - unsigned long ulong_mtu; - - if (priv_get_sysfs_ulong(priv, "mtu", &ulong_mtu) == -1) - return -1; - *mtu = ulong_mtu; - return 0; -} - -/** - * Set device MTU. - * - * @param priv - * Pointer to private structure. - * @param mtu - * MTU value to set. - * - * @return - * 0 on success, -1 on failure and errno is set. - */ -static int -priv_set_mtu(struct priv *priv, uint16_t mtu) -{ - uint16_t new_mtu; - - if (priv_set_sysfs_ulong(priv, "mtu", mtu) || - priv_get_mtu(priv, &new_mtu)) - return -1; - if (new_mtu == mtu) - return 0; - errno = EINVAL; - return -1; -} - -/** - * Set device flags. - * - * @param priv - * Pointer to private structure. - * @param keep - * Bitmask for flags that must remain untouched. - * @param flags - * Bitmask for flags to modify. - * - * @return - * 0 on success, -1 on failure and errno is set. - */ -static int -priv_set_flags(struct priv *priv, unsigned int keep, unsigned int flags) -{ - unsigned long tmp; - - if (priv_get_sysfs_ulong(priv, "flags", &tmp) == -1) - return -1; - tmp &= keep; - tmp |= (flags & (~keep)); - return priv_set_sysfs_ulong(priv, "flags", tmp); -} - -/* Device configuration. */ - -static int -txq_setup(struct rte_eth_dev *dev, struct txq *txq, uint16_t desc, - unsigned int socket, const struct rte_eth_txconf *conf); - -static void -txq_cleanup(struct txq *txq); - -static int -rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc, - unsigned int socket, int inactive, const struct rte_eth_rxconf *conf, - struct rte_mempool *mp); - -static void -rxq_cleanup(struct rxq *rxq); - -/** - * Ethernet device configuration. - * - * Prepare the driver for a given number of TX and RX queues. - * Allocate parent RSS queue when several RX queues are requested. - * - * @param dev - * Pointer to Ethernet device structure. - * - * @return - * 0 on success, errno value on failure. - */ -static int -dev_configure(struct rte_eth_dev *dev) -{ - struct priv *priv = dev->data->dev_private; - unsigned int rxqs_n = dev->data->nb_rx_queues; - unsigned int txqs_n = dev->data->nb_tx_queues; - unsigned int tmp; - int ret; - - priv->rxqs = (void *)dev->data->rx_queues; - priv->txqs = (void *)dev->data->tx_queues; - if (txqs_n != priv->txqs_n) { - INFO("%p: TX queues number update: %u -> %u", - (void *)dev, priv->txqs_n, txqs_n); - priv->txqs_n = txqs_n; - } - if (rxqs_n == priv->rxqs_n) - return 0; - if (!rte_is_power_of_2(rxqs_n)) { - unsigned n_active; - - n_active = rte_align32pow2(rxqs_n + 1) >> 1; - WARN("%p: number of RX queues must be a power" - " of 2: %u queues among %u will be active", - (void *)dev, n_active, rxqs_n); - } - - INFO("%p: RX queues number update: %u -> %u", - (void *)dev, priv->rxqs_n, rxqs_n); - /* If RSS is enabled, disable it first. */ - if (priv->rss) { - unsigned int i; - - /* Only if there are no remaining child RX queues. */ - for (i = 0; (i != priv->rxqs_n); ++i) - if ((*priv->rxqs)[i] != NULL) - return EINVAL; - rxq_cleanup(&priv->rxq_parent); - priv->rss = 0; - priv->rxqs_n = 0; - } - if (rxqs_n <= 1) { - /* Nothing else to do. */ - priv->rxqs_n = rxqs_n; - return 0; - } - /* Allocate a new RSS parent queue if supported by hardware. */ - if (!priv->hw_rss) { - ERROR("%p: only a single RX queue can be configured when" - " hardware doesn't support RSS", - (void *)dev); - return EINVAL; - } - /* Fail if hardware doesn't support that many RSS queues. */ - if (rxqs_n >= priv->max_rss_tbl_sz) { - ERROR("%p: only %u RX queues can be configured for RSS", - (void *)dev, priv->max_rss_tbl_sz); - return EINVAL; - } - priv->rss = 1; - tmp = priv->rxqs_n; - priv->rxqs_n = rxqs_n; - ret = rxq_setup(dev, &priv->rxq_parent, 0, 0, 0, NULL, NULL); - if (!ret) - return 0; - /* Failure, rollback. */ - priv->rss = 0; - priv->rxqs_n = tmp; - assert(ret > 0); - return ret; -} - /** * DPDK callback for Ethernet device configuration. * @@ -801,3366 +92,78 @@ dev_configure(struct rte_eth_dev *dev) * Pointer to Ethernet device structure. * * @return - * 0 on success, negative errno value on failure. + * 0 on success, negative errno value otherwise and rte_errno is set. */ static int mlx4_dev_configure(struct rte_eth_dev *dev) { struct priv *priv = dev->data->dev_private; + struct rte_flow_error error; int ret; - if (mlx4_is_secondary()) - return -E_RTE_SECONDARY; - priv_lock(priv); - ret = dev_configure(dev); - assert(ret >= 0); - priv_unlock(priv); - return -ret; -} - -static uint16_t mlx4_tx_burst(void *, struct rte_mbuf **, uint16_t); -static uint16_t removed_rx_burst(void *, struct rte_mbuf **, uint16_t); - -/** - * Configure secondary process queues from a private data pointer (primary - * or secondary) and update burst callbacks. Can take place only once. - * - * All queues must have been previously created by the primary process to - * avoid undefined behavior. - * - * @param priv - * Private data pointer from either primary or secondary process. - * - * @return - * Private data pointer from secondary process, NULL in case of error. - */ -static struct priv * -mlx4_secondary_data_setup(struct priv *priv) -{ - unsigned int port_id = 0; - struct mlx4_secondary_data *sd; - void **tx_queues; - void **rx_queues; - unsigned int nb_tx_queues; - unsigned int nb_rx_queues; - unsigned int i; - - /* priv must be valid at this point. */ - assert(priv != NULL); - /* priv->dev must also be valid but may point to local memory from - * another process, possibly with the same address and must not - * be dereferenced yet. */ - assert(priv->dev != NULL); - /* Determine port ID by finding out where priv comes from. */ - while (1) { - sd = &mlx4_secondary_data[port_id]; - rte_spinlock_lock(&sd->lock); - /* Primary process? */ - if (sd->primary_priv == priv) - break; - /* Secondary process? */ - if (sd->data.dev_private == priv) - break; - rte_spinlock_unlock(&sd->lock); - if (++port_id == RTE_DIM(mlx4_secondary_data)) - port_id = 0; + /* Prepare internal flow rules. */ + ret = mlx4_flow_sync(priv, &error); + if (ret) { + ERROR("cannot set up internal flow rules (code %d, \"%s\")," + " flow error type %d, cause %p, message: %s", + -ret, strerror(-ret), error.type, error.cause, + error.message ? error.message : "(unspecified)"); } - /* Switch to secondary private structure. If private data has already - * been updated by another thread, there is nothing else to do. */ - priv = sd->data.dev_private; - if (priv->dev->data == &sd->data) - goto end; - /* Sanity checks. Secondary private structure is supposed to point - * to local eth_dev, itself still pointing to the shared device data - * structure allocated by the primary process. */ - assert(sd->shared_dev_data != &sd->data); - assert(sd->data.nb_tx_queues == 0); - assert(sd->data.tx_queues == NULL); - assert(sd->data.nb_rx_queues == 0); - assert(sd->data.rx_queues == NULL); - assert(priv != sd->primary_priv); - assert(priv->dev->data == sd->shared_dev_data); - assert(priv->txqs_n == 0); - assert(priv->txqs == NULL); - assert(priv->rxqs_n == 0); - assert(priv->rxqs == NULL); - nb_tx_queues = sd->shared_dev_data->nb_tx_queues; - nb_rx_queues = sd->shared_dev_data->nb_rx_queues; - /* Allocate local storage for queues. */ - tx_queues = rte_zmalloc("secondary ethdev->tx_queues", - sizeof(sd->data.tx_queues[0]) * nb_tx_queues, - RTE_CACHE_LINE_SIZE); - rx_queues = rte_zmalloc("secondary ethdev->rx_queues", - sizeof(sd->data.rx_queues[0]) * nb_rx_queues, - RTE_CACHE_LINE_SIZE); - if (tx_queues == NULL || rx_queues == NULL) - goto error; - /* Lock to prevent control operations during setup. */ - priv_lock(priv); - /* TX queues. */ - for (i = 0; i != nb_tx_queues; ++i) { - struct txq *primary_txq = (*sd->primary_priv->txqs)[i]; - struct txq *txq; - - if (primary_txq == NULL) - continue; - txq = rte_calloc_socket("TXQ", 1, sizeof(*txq), 0, - primary_txq->socket); - if (txq != NULL) { - if (txq_setup(priv->dev, - txq, - primary_txq->elts_n * MLX4_PMD_SGE_WR_N, - primary_txq->socket, - NULL) == 0) { - txq->stats.idx = primary_txq->stats.idx; - tx_queues[i] = txq; - continue; - } - rte_free(txq); - } - while (i) { - txq = tx_queues[--i]; - txq_cleanup(txq); - rte_free(txq); - } - goto error; - } - /* RX queues. */ - for (i = 0; i != nb_rx_queues; ++i) { - struct rxq *primary_rxq = (*sd->primary_priv->rxqs)[i]; - - if (primary_rxq == NULL) - continue; - /* Not supported yet. */ - rx_queues[i] = NULL; - } - /* Update everything. */ - priv->txqs = (void *)tx_queues; - priv->txqs_n = nb_tx_queues; - priv->rxqs = (void *)rx_queues; - priv->rxqs_n = nb_rx_queues; - sd->data.rx_queues = rx_queues; - sd->data.tx_queues = tx_queues; - sd->data.nb_rx_queues = nb_rx_queues; - sd->data.nb_tx_queues = nb_tx_queues; - sd->data.dev_link = sd->shared_dev_data->dev_link; - sd->data.mtu = sd->shared_dev_data->mtu; - memcpy(sd->data.rx_queue_state, sd->shared_dev_data->rx_queue_state, - sizeof(sd->data.rx_queue_state)); - memcpy(sd->data.tx_queue_state, sd->shared_dev_data->tx_queue_state, - sizeof(sd->data.tx_queue_state)); - sd->data.dev_flags = sd->shared_dev_data->dev_flags; - /* Use local data from now on. */ - rte_mb(); - priv->dev->data = &sd->data; - rte_mb(); - priv->dev->tx_pkt_burst = mlx4_tx_burst; - priv->dev->rx_pkt_burst = removed_rx_burst; - priv_unlock(priv); -end: - /* More sanity checks. */ - assert(priv->dev->tx_pkt_burst == mlx4_tx_burst); - assert(priv->dev->rx_pkt_burst == removed_rx_burst); - assert(priv->dev->data == &sd->data); - rte_spinlock_unlock(&sd->lock); - return priv; -error: - priv_unlock(priv); - rte_free(tx_queues); - rte_free(rx_queues); - rte_spinlock_unlock(&sd->lock); - return NULL; -} - -/* TX queues handling. */ - -/** - * Allocate TX queue elements. - * - * @param txq - * Pointer to TX queue structure. - * @param elts_n - * Number of elements to allocate. - * - * @return - * 0 on success, errno value on failure. - */ -static int -txq_alloc_elts(struct txq *txq, unsigned int elts_n) -{ - unsigned int i; - struct txq_elt (*elts)[elts_n] = - rte_calloc_socket("TXQ", 1, sizeof(*elts), 0, txq->socket); - linear_t (*elts_linear)[elts_n] = - rte_calloc_socket("TXQ", 1, sizeof(*elts_linear), 0, - txq->socket); - struct ibv_mr *mr_linear = NULL; - int ret = 0; - - if ((elts == NULL) || (elts_linear == NULL)) { - ERROR("%p: can't allocate packets array", (void *)txq); - ret = ENOMEM; - goto error; - } - mr_linear = - ibv_reg_mr(txq->priv->pd, elts_linear, sizeof(*elts_linear), - IBV_ACCESS_LOCAL_WRITE); - if (mr_linear == NULL) { - ERROR("%p: unable to configure MR, ibv_reg_mr() failed", - (void *)txq); - ret = EINVAL; - goto error; - } - for (i = 0; (i != elts_n); ++i) { - struct txq_elt *elt = &(*elts)[i]; - - elt->buf = NULL; - } - DEBUG("%p: allocated and configured %u WRs", (void *)txq, elts_n); - txq->elts_n = elts_n; - txq->elts = elts; - txq->elts_head = 0; - txq->elts_tail = 0; - txq->elts_comp = 0; - /* Request send completion every MLX4_PMD_TX_PER_COMP_REQ packets or - * at least 4 times per ring. */ - txq->elts_comp_cd_init = - ((MLX4_PMD_TX_PER_COMP_REQ < (elts_n / 4)) ? - MLX4_PMD_TX_PER_COMP_REQ : (elts_n / 4)); - txq->elts_comp_cd = txq->elts_comp_cd_init; - txq->elts_linear = elts_linear; - txq->mr_linear = mr_linear; - assert(ret == 0); - return 0; -error: - if (mr_linear != NULL) - claim_zero(ibv_dereg_mr(mr_linear)); - - rte_free(elts_linear); - rte_free(elts); - - DEBUG("%p: failed, freed everything", (void *)txq); - assert(ret > 0); return ret; } -/** - * Free TX queue elements. - * - * @param txq - * Pointer to TX queue structure. - */ -static void -txq_free_elts(struct txq *txq) -{ - unsigned int elts_n = txq->elts_n; - unsigned int elts_head = txq->elts_head; - unsigned int elts_tail = txq->elts_tail; - struct txq_elt (*elts)[elts_n] = txq->elts; - linear_t (*elts_linear)[elts_n] = txq->elts_linear; - struct ibv_mr *mr_linear = txq->mr_linear; - - DEBUG("%p: freeing WRs", (void *)txq); - txq->elts_n = 0; - txq->elts_head = 0; - txq->elts_tail = 0; - txq->elts_comp = 0; - txq->elts_comp_cd = 0; - txq->elts_comp_cd_init = 0; - txq->elts = NULL; - txq->elts_linear = NULL; - txq->mr_linear = NULL; - if (mr_linear != NULL) - claim_zero(ibv_dereg_mr(mr_linear)); - - rte_free(elts_linear); - if (elts == NULL) - return; - while (elts_tail != elts_head) { - struct txq_elt *elt = &(*elts)[elts_tail]; - - assert(elt->buf != NULL); - rte_pktmbuf_free(elt->buf); -#ifndef NDEBUG - /* Poisoning. */ - memset(elt, 0x77, sizeof(*elt)); -#endif - if (++elts_tail == elts_n) - elts_tail = 0; - } - rte_free(elts); -} - - -/** - * Clean up a TX queue. - * - * Destroy objects, free allocated memory and reset the structure for reuse. - * - * @param txq - * Pointer to TX queue structure. - */ -static void -txq_cleanup(struct txq *txq) -{ - struct ibv_exp_release_intf_params params; - size_t i; - - DEBUG("cleaning up %p", (void *)txq); - txq_free_elts(txq); - if (txq->if_qp != NULL) { - assert(txq->priv != NULL); - assert(txq->priv->ctx != NULL); - assert(txq->qp != NULL); - params = (struct ibv_exp_release_intf_params){ - .comp_mask = 0, - }; - claim_zero(ibv_exp_release_intf(txq->priv->ctx, - txq->if_qp, - ¶ms)); - } - if (txq->if_cq != NULL) { - assert(txq->priv != NULL); - assert(txq->priv->ctx != NULL); - assert(txq->cq != NULL); - params = (struct ibv_exp_release_intf_params){ - .comp_mask = 0, - }; - claim_zero(ibv_exp_release_intf(txq->priv->ctx, - txq->if_cq, - ¶ms)); - } - if (txq->qp != NULL) - claim_zero(ibv_destroy_qp(txq->qp)); - if (txq->cq != NULL) - claim_zero(ibv_destroy_cq(txq->cq)); - if (txq->rd != NULL) { - struct ibv_exp_destroy_res_domain_attr attr = { - .comp_mask = 0, - }; - - assert(txq->priv != NULL); - assert(txq->priv->ctx != NULL); - claim_zero(ibv_exp_destroy_res_domain(txq->priv->ctx, - txq->rd, - &attr)); - } - for (i = 0; (i != elemof(txq->mp2mr)); ++i) { - if (txq->mp2mr[i].mp == NULL) - break; - assert(txq->mp2mr[i].mr != NULL); - claim_zero(ibv_dereg_mr(txq->mp2mr[i].mr)); - } - memset(txq, 0, sizeof(*txq)); -} - -/** - * Manage TX completions. - * - * When sending a burst, mlx4_tx_burst() posts several WRs. - * To improve performance, a completion event is only required once every - * MLX4_PMD_TX_PER_COMP_REQ sends. Doing so discards completion information - * for other WRs, but this information would not be used anyway. - * - * @param txq - * Pointer to TX queue structure. - * - * @return - * 0 on success, -1 on failure. - */ -static int -txq_complete(struct txq *txq) -{ - unsigned int elts_comp = txq->elts_comp; - unsigned int elts_tail = txq->elts_tail; - const unsigned int elts_n = txq->elts_n; - int wcs_n; - - if (unlikely(elts_comp == 0)) - return 0; -#ifdef DEBUG_SEND - DEBUG("%p: processing %u work requests completions", - (void *)txq, elts_comp); -#endif - wcs_n = txq->if_cq->poll_cnt(txq->cq, elts_comp); - if (unlikely(wcs_n == 0)) - return 0; - if (unlikely(wcs_n < 0)) { - DEBUG("%p: ibv_poll_cq() failed (wcs_n=%d)", - (void *)txq, wcs_n); - return -1; - } - elts_comp -= wcs_n; - assert(elts_comp <= txq->elts_comp); - /* - * Assume WC status is successful as nothing can be done about it - * anyway. - */ - elts_tail += wcs_n * txq->elts_comp_cd_init; - if (elts_tail >= elts_n) - elts_tail -= elts_n; - txq->elts_tail = elts_tail; - txq->elts_comp = elts_comp; - return 0; -} - -struct mlx4_check_mempool_data { - int ret; - char *start; - char *end; -}; - -/* Called by mlx4_check_mempool() when iterating the memory chunks. */ -static void mlx4_check_mempool_cb(struct rte_mempool *mp, - void *opaque, struct rte_mempool_memhdr *memhdr, - unsigned mem_idx) -{ - struct mlx4_check_mempool_data *data = opaque; - - (void)mp; - (void)mem_idx; - - /* It already failed, skip the next chunks. */ - if (data->ret != 0) - return; - /* It is the first chunk. */ - if (data->start == NULL && data->end == NULL) { - data->start = memhdr->addr; - data->end = data->start + memhdr->len; - return; - } - if (data->end == memhdr->addr) { - data->end += memhdr->len; - return; - } - if (data->start == (char *)memhdr->addr + memhdr->len) { - data->start -= memhdr->len; - return; - } - /* Error, mempool is not virtually contigous. */ - data->ret = -1; -} - -/** - * Check if a mempool can be used: it must be virtually contiguous. - * - * @param[in] mp - * Pointer to memory pool. - * @param[out] start - * Pointer to the start address of the mempool virtual memory area - * @param[out] end - * Pointer to the end address of the mempool virtual memory area - * - * @return - * 0 on success (mempool is virtually contiguous), -1 on error. - */ -static int mlx4_check_mempool(struct rte_mempool *mp, uintptr_t *start, - uintptr_t *end) -{ - struct mlx4_check_mempool_data data; - - memset(&data, 0, sizeof(data)); - rte_mempool_mem_iter(mp, mlx4_check_mempool_cb, &data); - *start = (uintptr_t)data.start; - *end = (uintptr_t)data.end; - - return data.ret; -} - -/* For best performance, this function should not be inlined. */ -static struct ibv_mr *mlx4_mp2mr(struct ibv_pd *, struct rte_mempool *) - __attribute__((noinline)); - -/** - * Register mempool as a memory region. - * - * @param pd - * Pointer to protection domain. - * @param mp - * Pointer to memory pool. - * - * @return - * Memory region pointer, NULL in case of error. - */ -static struct ibv_mr * -mlx4_mp2mr(struct ibv_pd *pd, struct rte_mempool *mp) -{ - const struct rte_memseg *ms = rte_eal_get_physmem_layout(); - uintptr_t start; - uintptr_t end; - unsigned int i; - - if (mlx4_check_mempool(mp, &start, &end) != 0) { - ERROR("mempool %p: not virtually contiguous", - (void *)mp); - return NULL; - } - - DEBUG("mempool %p area start=%p end=%p size=%zu", - (void *)mp, (void *)start, (void *)end, - (size_t)(end - start)); - /* Round start and end to page boundary if found in memory segments. */ - for (i = 0; (i < RTE_MAX_MEMSEG) && (ms[i].addr != NULL); ++i) { - uintptr_t addr = (uintptr_t)ms[i].addr; - size_t len = ms[i].len; - unsigned int align = ms[i].hugepage_sz; - - if ((start > addr) && (start < addr + len)) - start = RTE_ALIGN_FLOOR(start, align); - if ((end > addr) && (end < addr + len)) - end = RTE_ALIGN_CEIL(end, align); - } - DEBUG("mempool %p using start=%p end=%p size=%zu for MR", - (void *)mp, (void *)start, (void *)end, - (size_t)(end - start)); - return ibv_reg_mr(pd, - (void *)start, - end - start, - IBV_ACCESS_LOCAL_WRITE); -} - -/** - * Get Memory Pool (MP) from mbuf. If mbuf is indirect, the pool from which - * the cloned mbuf is allocated is returned instead. - * - * @param buf - * Pointer to mbuf. - * - * @return - * Memory pool where data is located for given mbuf. - */ -static struct rte_mempool * -txq_mb2mp(struct rte_mbuf *buf) -{ - if (unlikely(RTE_MBUF_INDIRECT(buf))) - return rte_mbuf_from_indirect(buf)->pool; - return buf->pool; -} - -/** - * Get Memory Region (MR) <-> Memory Pool (MP) association from txq->mp2mr[]. - * Add MP to txq->mp2mr[] if it's not registered yet. If mp2mr[] is full, - * remove an entry first. - * - * @param txq - * Pointer to TX queue structure. - * @param[in] mp - * Memory Pool for which a Memory Region lkey must be returned. - * - * @return - * mr->lkey on success, (uint32_t)-1 on failure. - */ -static uint32_t -txq_mp2mr(struct txq *txq, struct rte_mempool *mp) -{ - unsigned int i; - struct ibv_mr *mr; - - for (i = 0; (i != elemof(txq->mp2mr)); ++i) { - if (unlikely(txq->mp2mr[i].mp == NULL)) { - /* Unknown MP, add a new MR for it. */ - break; - } - if (txq->mp2mr[i].mp == mp) { - assert(txq->mp2mr[i].lkey != (uint32_t)-1); - assert(txq->mp2mr[i].mr->lkey == txq->mp2mr[i].lkey); - return txq->mp2mr[i].lkey; - } - } - /* Add a new entry, register MR first. */ - DEBUG("%p: discovered new memory pool \"%s\" (%p)", - (void *)txq, mp->name, (void *)mp); - mr = mlx4_mp2mr(txq->priv->pd, mp); - if (unlikely(mr == NULL)) { - DEBUG("%p: unable to configure MR, ibv_reg_mr() failed.", - (void *)txq); - return (uint32_t)-1; - } - if (unlikely(i == elemof(txq->mp2mr))) { - /* Table is full, remove oldest entry. */ - DEBUG("%p: MR <-> MP table full, dropping oldest entry.", - (void *)txq); - --i; - claim_zero(ibv_dereg_mr(txq->mp2mr[0].mr)); - memmove(&txq->mp2mr[0], &txq->mp2mr[1], - (sizeof(txq->mp2mr) - sizeof(txq->mp2mr[0]))); - } - /* Store the new entry. */ - txq->mp2mr[i].mp = mp; - txq->mp2mr[i].mr = mr; - txq->mp2mr[i].lkey = mr->lkey; - DEBUG("%p: new MR lkey for MP \"%s\" (%p): 0x%08" PRIu32, - (void *)txq, mp->name, (void *)mp, txq->mp2mr[i].lkey); - return txq->mp2mr[i].lkey; -} - -struct txq_mp2mr_mbuf_check_data { - int ret; -}; - -/** - * Callback function for rte_mempool_obj_iter() to check whether a given - * mempool object looks like a mbuf. - * - * @param[in] mp - * The mempool pointer - * @param[in] arg - * Context data (struct txq_mp2mr_mbuf_check_data). Contains the - * return value. - * @param[in] obj - * Object address. - * @param index - * Object index, unused. - */ -static void -txq_mp2mr_mbuf_check(struct rte_mempool *mp, void *arg, void *obj, - uint32_t index __rte_unused) -{ - struct txq_mp2mr_mbuf_check_data *data = arg; - struct rte_mbuf *buf = obj; - - /* Check whether mbuf structure fits element size and whether mempool - * pointer is valid. */ - if (sizeof(*buf) > mp->elt_size || buf->pool != mp) - data->ret = -1; -} - -/** - * Iterator function for rte_mempool_walk() to register existing mempools and - * fill the MP to MR cache of a TX queue. - * - * @param[in] mp - * Memory Pool to register. - * @param *arg - * Pointer to TX queue structure. - */ -static void -txq_mp2mr_iter(struct rte_mempool *mp, void *arg) -{ - struct txq *txq = arg; - struct txq_mp2mr_mbuf_check_data data = { - .ret = 0, - }; - - /* Register mempool only if the first element looks like a mbuf. */ - if (rte_mempool_obj_iter(mp, txq_mp2mr_mbuf_check, &data) == 0 || - data.ret == -1) - return; - txq_mp2mr(txq, mp); -} - -#if MLX4_PMD_SGE_WR_N > 1 - -/** - * Copy scattered mbuf contents to a single linear buffer. - * - * @param[out] linear - * Linear output buffer. - * @param[in] buf - * Scattered input buffer. - * - * @return - * Number of bytes copied to the output buffer or 0 if not large enough. - */ -static unsigned int -linearize_mbuf(linear_t *linear, struct rte_mbuf *buf) -{ - unsigned int size = 0; - unsigned int offset; - - do { - unsigned int len = DATA_LEN(buf); - - offset = size; - size += len; - if (unlikely(size > sizeof(*linear))) - return 0; - memcpy(&(*linear)[offset], - rte_pktmbuf_mtod(buf, uint8_t *), - len); - buf = NEXT(buf); - } while (buf != NULL); - return size; -} - -/** - * Handle scattered buffers for mlx4_tx_burst(). - * - * @param txq - * TX queue structure. - * @param segs - * Number of segments in buf. - * @param elt - * TX queue element to fill. - * @param[in] buf - * Buffer to process. - * @param elts_head - * Index of the linear buffer to use if necessary (normally txq->elts_head). - * @param[out] sges - * Array filled with SGEs on success. - * - * @return - * A structure containing the processed packet size in bytes and the - * number of SGEs. Both fields are set to (unsigned int)-1 in case of - * failure. - */ -static struct tx_burst_sg_ret { - unsigned int length; - unsigned int num; -} -tx_burst_sg(struct txq *txq, unsigned int segs, struct txq_elt *elt, - struct rte_mbuf *buf, unsigned int elts_head, - struct ibv_sge (*sges)[MLX4_PMD_SGE_WR_N]) -{ - unsigned int sent_size = 0; - unsigned int j; - int linearize = 0; - - /* When there are too many segments, extra segments are - * linearized in the last SGE. */ - if (unlikely(segs > elemof(*sges))) { - segs = (elemof(*sges) - 1); - linearize = 1; - } - /* Update element. */ - elt->buf = buf; - /* Register segments as SGEs. */ - for (j = 0; (j != segs); ++j) { - struct ibv_sge *sge = &(*sges)[j]; - uint32_t lkey; - - /* Retrieve Memory Region key for this memory pool. */ - lkey = txq_mp2mr(txq, txq_mb2mp(buf)); - if (unlikely(lkey == (uint32_t)-1)) { - /* MR does not exist. */ - DEBUG("%p: unable to get MP <-> MR association", - (void *)txq); - /* Clean up TX element. */ - elt->buf = NULL; - goto stop; - } - /* Update SGE. */ - sge->addr = rte_pktmbuf_mtod(buf, uintptr_t); - if (txq->priv->vf) - rte_prefetch0((volatile void *) - (uintptr_t)sge->addr); - sge->length = DATA_LEN(buf); - sge->lkey = lkey; - sent_size += sge->length; - buf = NEXT(buf); - } - /* If buf is not NULL here and is not going to be linearized, - * nb_segs is not valid. */ - assert(j == segs); - assert((buf == NULL) || (linearize)); - /* Linearize extra segments. */ - if (linearize) { - struct ibv_sge *sge = &(*sges)[segs]; - linear_t *linear = &(*txq->elts_linear)[elts_head]; - unsigned int size = linearize_mbuf(linear, buf); - - assert(segs == (elemof(*sges) - 1)); - if (size == 0) { - /* Invalid packet. */ - DEBUG("%p: packet too large to be linearized.", - (void *)txq); - /* Clean up TX element. */ - elt->buf = NULL; - goto stop; - } - /* If MLX4_PMD_SGE_WR_N is 1, free mbuf immediately. */ - if (elemof(*sges) == 1) { - do { - struct rte_mbuf *next = NEXT(buf); - - rte_pktmbuf_free_seg(buf); - buf = next; - } while (buf != NULL); - elt->buf = NULL; - } - /* Update SGE. */ - sge->addr = (uintptr_t)&(*linear)[0]; - sge->length = size; - sge->lkey = txq->mr_linear->lkey; - sent_size += size; - /* Include last segment. */ - segs++; - } - return (struct tx_burst_sg_ret){ - .length = sent_size, - .num = segs, - }; -stop: - return (struct tx_burst_sg_ret){ - .length = -1, - .num = -1, - }; -} - -#endif /* MLX4_PMD_SGE_WR_N > 1 */ - -/** - * DPDK callback for TX. - * - * @param dpdk_txq - * Generic pointer to TX queue structure. - * @param[in] pkts - * Packets to transmit. - * @param pkts_n - * Number of packets in array. - * - * @return - * Number of packets successfully transmitted (<= pkts_n). - */ -static uint16_t -mlx4_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) -{ - struct txq *txq = (struct txq *)dpdk_txq; - unsigned int elts_head = txq->elts_head; - const unsigned int elts_n = txq->elts_n; - unsigned int elts_comp_cd = txq->elts_comp_cd; - unsigned int elts_comp = 0; - unsigned int i; - unsigned int max; - int err; - - assert(elts_comp_cd != 0); - txq_complete(txq); - max = (elts_n - (elts_head - txq->elts_tail)); - if (max > elts_n) - max -= elts_n; - assert(max >= 1); - assert(max <= elts_n); - /* Always leave one free entry in the ring. */ - --max; - if (max == 0) - return 0; - if (max > pkts_n) - max = pkts_n; - for (i = 0; (i != max); ++i) { - struct rte_mbuf *buf = pkts[i]; - unsigned int elts_head_next = - (((elts_head + 1) == elts_n) ? 0 : elts_head + 1); - struct txq_elt *elt_next = &(*txq->elts)[elts_head_next]; - struct txq_elt *elt = &(*txq->elts)[elts_head]; - unsigned int segs = NB_SEGS(buf); -#ifdef MLX4_PMD_SOFT_COUNTERS - unsigned int sent_size = 0; -#endif - uint32_t send_flags = 0; - - /* Clean up old buffer. */ - if (likely(elt->buf != NULL)) { - struct rte_mbuf *tmp = elt->buf; - -#ifndef NDEBUG - /* Poisoning. */ - memset(elt, 0x66, sizeof(*elt)); -#endif - /* Faster than rte_pktmbuf_free(). */ - do { - struct rte_mbuf *next = NEXT(tmp); - - rte_pktmbuf_free_seg(tmp); - tmp = next; - } while (tmp != NULL); - } - /* Request TX completion. */ - if (unlikely(--elts_comp_cd == 0)) { - elts_comp_cd = txq->elts_comp_cd_init; - ++elts_comp; - send_flags |= IBV_EXP_QP_BURST_SIGNALED; - } - /* Should we enable HW CKSUM offload */ - if (buf->ol_flags & - (PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM)) { - send_flags |= IBV_EXP_QP_BURST_IP_CSUM; - /* HW does not support checksum offloads at arbitrary - * offsets but automatically recognizes the packet - * type. For inner L3/L4 checksums, only VXLAN (UDP) - * tunnels are currently supported. */ - if (RTE_ETH_IS_TUNNEL_PKT(buf->packet_type)) - send_flags |= IBV_EXP_QP_BURST_TUNNEL; - } - if (likely(segs == 1)) { - uintptr_t addr; - uint32_t length; - uint32_t lkey; - - /* Retrieve buffer information. */ - addr = rte_pktmbuf_mtod(buf, uintptr_t); - length = DATA_LEN(buf); - /* Retrieve Memory Region key for this memory pool. */ - lkey = txq_mp2mr(txq, txq_mb2mp(buf)); - if (unlikely(lkey == (uint32_t)-1)) { - /* MR does not exist. */ - DEBUG("%p: unable to get MP <-> MR" - " association", (void *)txq); - /* Clean up TX element. */ - elt->buf = NULL; - goto stop; - } - /* Update element. */ - elt->buf = buf; - if (txq->priv->vf) - rte_prefetch0((volatile void *) - (uintptr_t)addr); - RTE_MBUF_PREFETCH_TO_FREE(elt_next->buf); - /* Put packet into send queue. */ -#if MLX4_PMD_MAX_INLINE > 0 - if (length <= txq->max_inline) - err = txq->if_qp->send_pending_inline - (txq->qp, - (void *)addr, - length, - send_flags); - else -#endif - err = txq->if_qp->send_pending - (txq->qp, - addr, - length, - lkey, - send_flags); - if (unlikely(err)) - goto stop; -#ifdef MLX4_PMD_SOFT_COUNTERS - sent_size += length; -#endif - } else { -#if MLX4_PMD_SGE_WR_N > 1 - struct ibv_sge sges[MLX4_PMD_SGE_WR_N]; - struct tx_burst_sg_ret ret; - - ret = tx_burst_sg(txq, segs, elt, buf, elts_head, - &sges); - if (ret.length == (unsigned int)-1) - goto stop; - RTE_MBUF_PREFETCH_TO_FREE(elt_next->buf); - /* Put SG list into send queue. */ - err = txq->if_qp->send_pending_sg_list - (txq->qp, - sges, - ret.num, - send_flags); - if (unlikely(err)) - goto stop; -#ifdef MLX4_PMD_SOFT_COUNTERS - sent_size += ret.length; -#endif -#else /* MLX4_PMD_SGE_WR_N > 1 */ - DEBUG("%p: TX scattered buffers support not" - " compiled in", (void *)txq); - goto stop; -#endif /* MLX4_PMD_SGE_WR_N > 1 */ - } - elts_head = elts_head_next; -#ifdef MLX4_PMD_SOFT_COUNTERS - /* Increment sent bytes counter. */ - txq->stats.obytes += sent_size; -#endif - } -stop: - /* Take a shortcut if nothing must be sent. */ - if (unlikely(i == 0)) - return 0; -#ifdef MLX4_PMD_SOFT_COUNTERS - /* Increment sent packets counter. */ - txq->stats.opackets += i; -#endif - /* Ring QP doorbell. */ - err = txq->if_qp->send_flush(txq->qp); - if (unlikely(err)) { - /* A nonzero value is not supposed to be returned. - * Nothing can be done about it. */ - DEBUG("%p: send_flush() failed with error %d", - (void *)txq, err); - } - txq->elts_head = elts_head; - txq->elts_comp += elts_comp; - txq->elts_comp_cd = elts_comp_cd; - return i; -} - -/** - * DPDK callback for TX in secondary processes. - * - * This function configures all queues from primary process information - * if necessary before reverting to the normal TX burst callback. - * - * @param dpdk_txq - * Generic pointer to TX queue structure. - * @param[in] pkts - * Packets to transmit. - * @param pkts_n - * Number of packets in array. - * - * @return - * Number of packets successfully transmitted (<= pkts_n). - */ -static uint16_t -mlx4_tx_burst_secondary_setup(void *dpdk_txq, struct rte_mbuf **pkts, - uint16_t pkts_n) -{ - struct txq *txq = dpdk_txq; - struct priv *priv = mlx4_secondary_data_setup(txq->priv); - struct priv *primary_priv; - unsigned int index; - - if (priv == NULL) - return 0; - primary_priv = - mlx4_secondary_data[priv->dev->data->port_id].primary_priv; - /* Look for queue index in both private structures. */ - for (index = 0; index != priv->txqs_n; ++index) - if (((*primary_priv->txqs)[index] == txq) || - ((*priv->txqs)[index] == txq)) - break; - if (index == priv->txqs_n) - return 0; - txq = (*priv->txqs)[index]; - return priv->dev->tx_pkt_burst(txq, pkts, pkts_n); -} - -/** - * Configure a TX queue. - * - * @param dev - * Pointer to Ethernet device structure. - * @param txq - * Pointer to TX queue structure. - * @param desc - * Number of descriptors to configure in queue. - * @param socket - * NUMA socket on which memory must be allocated. - * @param[in] conf - * Thresholds parameters. - * - * @return - * 0 on success, errno value on failure. - */ -static int -txq_setup(struct rte_eth_dev *dev, struct txq *txq, uint16_t desc, - unsigned int socket, const struct rte_eth_txconf *conf) -{ - struct priv *priv = mlx4_get_priv(dev); - struct txq tmpl = { - .priv = priv, - .socket = socket - }; - union { - struct ibv_exp_query_intf_params params; - struct ibv_exp_qp_init_attr init; - struct ibv_exp_res_domain_init_attr rd; - struct ibv_exp_cq_init_attr cq; - struct ibv_exp_qp_attr mod; - } attr; - enum ibv_exp_query_intf_status status; - int ret = 0; - - (void)conf; /* Thresholds configuration (ignored). */ - if (priv == NULL) - return EINVAL; - if ((desc == 0) || (desc % MLX4_PMD_SGE_WR_N)) { - ERROR("%p: invalid number of TX descriptors (must be a" - " multiple of %d)", (void *)dev, MLX4_PMD_SGE_WR_N); - return EINVAL; - } - desc /= MLX4_PMD_SGE_WR_N; - /* MRs will be registered in mp2mr[] later. */ - attr.rd = (struct ibv_exp_res_domain_init_attr){ - .comp_mask = (IBV_EXP_RES_DOMAIN_THREAD_MODEL | - IBV_EXP_RES_DOMAIN_MSG_MODEL), - .thread_model = IBV_EXP_THREAD_SINGLE, - .msg_model = IBV_EXP_MSG_HIGH_BW, - }; - tmpl.rd = ibv_exp_create_res_domain(priv->ctx, &attr.rd); - if (tmpl.rd == NULL) { - ret = ENOMEM; - ERROR("%p: RD creation failure: %s", - (void *)dev, strerror(ret)); - goto error; - } - attr.cq = (struct ibv_exp_cq_init_attr){ - .comp_mask = IBV_EXP_CQ_INIT_ATTR_RES_DOMAIN, - .res_domain = tmpl.rd, - }; - tmpl.cq = ibv_exp_create_cq(priv->ctx, desc, NULL, NULL, 0, &attr.cq); - if (tmpl.cq == NULL) { - ret = ENOMEM; - ERROR("%p: CQ creation failure: %s", - (void *)dev, strerror(ret)); - goto error; - } - DEBUG("priv->device_attr.max_qp_wr is %d", - priv->device_attr.max_qp_wr); - DEBUG("priv->device_attr.max_sge is %d", - priv->device_attr.max_sge); - attr.init = (struct ibv_exp_qp_init_attr){ - /* CQ to be associated with the send queue. */ - .send_cq = tmpl.cq, - /* CQ to be associated with the receive queue. */ - .recv_cq = tmpl.cq, - .cap = { - /* Max number of outstanding WRs. */ - .max_send_wr = ((priv->device_attr.max_qp_wr < desc) ? - priv->device_attr.max_qp_wr : - desc), - /* Max number of scatter/gather elements in a WR. */ - .max_send_sge = ((priv->device_attr.max_sge < - MLX4_PMD_SGE_WR_N) ? - priv->device_attr.max_sge : - MLX4_PMD_SGE_WR_N), -#if MLX4_PMD_MAX_INLINE > 0 - .max_inline_data = MLX4_PMD_MAX_INLINE, -#endif - }, - .qp_type = IBV_QPT_RAW_PACKET, - /* Do *NOT* enable this, completions events are managed per - * TX burst. */ - .sq_sig_all = 0, - .pd = priv->pd, - .res_domain = tmpl.rd, - .comp_mask = (IBV_EXP_QP_INIT_ATTR_PD | - IBV_EXP_QP_INIT_ATTR_RES_DOMAIN), - }; - tmpl.qp = ibv_exp_create_qp(priv->ctx, &attr.init); - if (tmpl.qp == NULL) { - ret = (errno ? errno : EINVAL); - ERROR("%p: QP creation failure: %s", - (void *)dev, strerror(ret)); - goto error; - } -#if MLX4_PMD_MAX_INLINE > 0 - /* ibv_create_qp() updates this value. */ - tmpl.max_inline = attr.init.cap.max_inline_data; -#endif - attr.mod = (struct ibv_exp_qp_attr){ - /* Move the QP to this state. */ - .qp_state = IBV_QPS_INIT, - /* Primary port number. */ - .port_num = priv->port - }; - ret = ibv_exp_modify_qp(tmpl.qp, &attr.mod, - (IBV_EXP_QP_STATE | IBV_EXP_QP_PORT)); - if (ret) { - ERROR("%p: QP state to IBV_QPS_INIT failed: %s", - (void *)dev, strerror(ret)); - goto error; - } - ret = txq_alloc_elts(&tmpl, desc); - if (ret) { - ERROR("%p: TXQ allocation failed: %s", - (void *)dev, strerror(ret)); - goto error; - } - attr.mod = (struct ibv_exp_qp_attr){ - .qp_state = IBV_QPS_RTR - }; - ret = ibv_exp_modify_qp(tmpl.qp, &attr.mod, IBV_EXP_QP_STATE); - if (ret) { - ERROR("%p: QP state to IBV_QPS_RTR failed: %s", - (void *)dev, strerror(ret)); - goto error; - } - attr.mod.qp_state = IBV_QPS_RTS; - ret = ibv_exp_modify_qp(tmpl.qp, &attr.mod, IBV_EXP_QP_STATE); - if (ret) { - ERROR("%p: QP state to IBV_QPS_RTS failed: %s", - (void *)dev, strerror(ret)); - goto error; - } - attr.params = (struct ibv_exp_query_intf_params){ - .intf_scope = IBV_EXP_INTF_GLOBAL, - .intf = IBV_EXP_INTF_CQ, - .obj = tmpl.cq, - }; - tmpl.if_cq = ibv_exp_query_intf(priv->ctx, &attr.params, &status); - if (tmpl.if_cq == NULL) { - ERROR("%p: CQ interface family query failed with status %d", - (void *)dev, status); - goto error; - } - attr.params = (struct ibv_exp_query_intf_params){ - .intf_scope = IBV_EXP_INTF_GLOBAL, - .intf = IBV_EXP_INTF_QP_BURST, - .obj = tmpl.qp, -#ifdef HAVE_EXP_QP_BURST_CREATE_DISABLE_ETH_LOOPBACK - /* MC loopback must be disabled when not using a VF. */ - .family_flags = - (!priv->vf ? - IBV_EXP_QP_BURST_CREATE_DISABLE_ETH_LOOPBACK : - 0), -#endif - }; - tmpl.if_qp = ibv_exp_query_intf(priv->ctx, &attr.params, &status); - if (tmpl.if_qp == NULL) { - ERROR("%p: QP interface family query failed with status %d", - (void *)dev, status); - goto error; - } - /* Clean up txq in case we're reinitializing it. */ - DEBUG("%p: cleaning-up old txq just in case", (void *)txq); - txq_cleanup(txq); - *txq = tmpl; - DEBUG("%p: txq updated with %p", (void *)txq, (void *)&tmpl); - /* Pre-register known mempools. */ - rte_mempool_walk(txq_mp2mr_iter, txq); - assert(ret == 0); - return 0; -error: - txq_cleanup(&tmpl); - assert(ret > 0); - return ret; -} - -/** - * DPDK callback to configure a TX queue. - * - * @param dev - * Pointer to Ethernet device structure. - * @param idx - * TX queue index. - * @param desc - * Number of descriptors to configure in queue. - * @param socket - * NUMA socket on which memory must be allocated. - * @param[in] conf - * Thresholds parameters. - * - * @return - * 0 on success, negative errno value on failure. - */ -static int -mlx4_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, - unsigned int socket, const struct rte_eth_txconf *conf) -{ - struct priv *priv = dev->data->dev_private; - struct txq *txq = (*priv->txqs)[idx]; - int ret; - - if (mlx4_is_secondary()) - return -E_RTE_SECONDARY; - priv_lock(priv); - DEBUG("%p: configuring queue %u for %u descriptors", - (void *)dev, idx, desc); - if (idx >= priv->txqs_n) { - ERROR("%p: queue index out of range (%u >= %u)", - (void *)dev, idx, priv->txqs_n); - priv_unlock(priv); - return -EOVERFLOW; - } - if (txq != NULL) { - DEBUG("%p: reusing already allocated queue index %u (%p)", - (void *)dev, idx, (void *)txq); - if (priv->started) { - priv_unlock(priv); - return -EEXIST; - } - (*priv->txqs)[idx] = NULL; - txq_cleanup(txq); - } else { - txq = rte_calloc_socket("TXQ", 1, sizeof(*txq), 0, socket); - if (txq == NULL) { - ERROR("%p: unable to allocate queue index %u", - (void *)dev, idx); - priv_unlock(priv); - return -ENOMEM; - } - } - ret = txq_setup(dev, txq, desc, socket, conf); - if (ret) - rte_free(txq); - else { - txq->stats.idx = idx; - DEBUG("%p: adding TX queue %p to list", - (void *)dev, (void *)txq); - (*priv->txqs)[idx] = txq; - /* Update send callback. */ - dev->tx_pkt_burst = mlx4_tx_burst; - } - priv_unlock(priv); - return -ret; -} - -/** - * DPDK callback to release a TX queue. - * - * @param dpdk_txq - * Generic TX queue pointer. - */ -static void -mlx4_tx_queue_release(void *dpdk_txq) -{ - struct txq *txq = (struct txq *)dpdk_txq; - struct priv *priv; - unsigned int i; - - if (mlx4_is_secondary()) - return; - if (txq == NULL) - return; - priv = txq->priv; - priv_lock(priv); - for (i = 0; (i != priv->txqs_n); ++i) - if ((*priv->txqs)[i] == txq) { - DEBUG("%p: removing TX queue %p from list", - (void *)priv->dev, (void *)txq); - (*priv->txqs)[i] = NULL; - break; - } - txq_cleanup(txq); - rte_free(txq); - priv_unlock(priv); -} - -/* RX queues handling. */ - -/** - * Allocate RX queue elements with scattered packets support. - * - * @param rxq - * Pointer to RX queue structure. - * @param elts_n - * Number of elements to allocate. - * @param[in] pool - * If not NULL, fetch buffers from this array instead of allocating them - * with rte_pktmbuf_alloc(). - * - * @return - * 0 on success, errno value on failure. - */ -static int -rxq_alloc_elts_sp(struct rxq *rxq, unsigned int elts_n, - struct rte_mbuf **pool) -{ - unsigned int i; - struct rxq_elt_sp (*elts)[elts_n] = - rte_calloc_socket("RXQ elements", 1, sizeof(*elts), 0, - rxq->socket); - int ret = 0; - - if (elts == NULL) { - ERROR("%p: can't allocate packets array", (void *)rxq); - ret = ENOMEM; - goto error; - } - /* For each WR (packet). */ - for (i = 0; (i != elts_n); ++i) { - unsigned int j; - struct rxq_elt_sp *elt = &(*elts)[i]; - struct ibv_recv_wr *wr = &elt->wr; - struct ibv_sge (*sges)[(elemof(elt->sges))] = &elt->sges; - - /* These two arrays must have the same size. */ - assert(elemof(elt->sges) == elemof(elt->bufs)); - /* Configure WR. */ - wr->wr_id = i; - wr->next = &(*elts)[(i + 1)].wr; - wr->sg_list = &(*sges)[0]; - wr->num_sge = elemof(*sges); - /* For each SGE (segment). */ - for (j = 0; (j != elemof(elt->bufs)); ++j) { - struct ibv_sge *sge = &(*sges)[j]; - struct rte_mbuf *buf; - - if (pool != NULL) { - buf = *(pool++); - assert(buf != NULL); - rte_pktmbuf_reset(buf); - } else - buf = rte_pktmbuf_alloc(rxq->mp); - if (buf == NULL) { - assert(pool == NULL); - ERROR("%p: empty mbuf pool", (void *)rxq); - ret = ENOMEM; - goto error; - } - elt->bufs[j] = buf; - /* Headroom is reserved by rte_pktmbuf_alloc(). */ - assert(DATA_OFF(buf) == RTE_PKTMBUF_HEADROOM); - /* Buffer is supposed to be empty. */ - assert(rte_pktmbuf_data_len(buf) == 0); - assert(rte_pktmbuf_pkt_len(buf) == 0); - /* sge->addr must be able to store a pointer. */ - assert(sizeof(sge->addr) >= sizeof(uintptr_t)); - if (j == 0) { - /* The first SGE keeps its headroom. */ - sge->addr = rte_pktmbuf_mtod(buf, uintptr_t); - sge->length = (buf->buf_len - - RTE_PKTMBUF_HEADROOM); - } else { - /* Subsequent SGEs lose theirs. */ - assert(DATA_OFF(buf) == RTE_PKTMBUF_HEADROOM); - SET_DATA_OFF(buf, 0); - sge->addr = (uintptr_t)buf->buf_addr; - sge->length = buf->buf_len; - } - sge->lkey = rxq->mr->lkey; - /* Redundant check for tailroom. */ - assert(sge->length == rte_pktmbuf_tailroom(buf)); - } - } - /* The last WR pointer must be NULL. */ - (*elts)[(i - 1)].wr.next = NULL; - DEBUG("%p: allocated and configured %u WRs (%zu segments)", - (void *)rxq, elts_n, (elts_n * elemof((*elts)[0].sges))); - rxq->elts_n = elts_n; - rxq->elts_head = 0; - rxq->elts.sp = elts; - assert(ret == 0); - return 0; -error: - if (elts != NULL) { - assert(pool == NULL); - for (i = 0; (i != elemof(*elts)); ++i) { - unsigned int j; - struct rxq_elt_sp *elt = &(*elts)[i]; - - for (j = 0; (j != elemof(elt->bufs)); ++j) { - struct rte_mbuf *buf = elt->bufs[j]; - - if (buf != NULL) - rte_pktmbuf_free_seg(buf); - } - } - rte_free(elts); - } - DEBUG("%p: failed, freed everything", (void *)rxq); - assert(ret > 0); - return ret; -} - -/** - * Free RX queue elements with scattered packets support. - * - * @param rxq - * Pointer to RX queue structure. - */ -static void -rxq_free_elts_sp(struct rxq *rxq) -{ - unsigned int i; - unsigned int elts_n = rxq->elts_n; - struct rxq_elt_sp (*elts)[elts_n] = rxq->elts.sp; - - DEBUG("%p: freeing WRs", (void *)rxq); - rxq->elts_n = 0; - rxq->elts.sp = NULL; - if (elts == NULL) - return; - for (i = 0; (i != elemof(*elts)); ++i) { - unsigned int j; - struct rxq_elt_sp *elt = &(*elts)[i]; - - for (j = 0; (j != elemof(elt->bufs)); ++j) { - struct rte_mbuf *buf = elt->bufs[j]; - - if (buf != NULL) - rte_pktmbuf_free_seg(buf); - } - } - rte_free(elts); -} - -/** - * Allocate RX queue elements. - * - * @param rxq - * Pointer to RX queue structure. - * @param elts_n - * Number of elements to allocate. - * @param[in] pool - * If not NULL, fetch buffers from this array instead of allocating them - * with rte_pktmbuf_alloc(). - * - * @return - * 0 on success, errno value on failure. - */ -static int -rxq_alloc_elts(struct rxq *rxq, unsigned int elts_n, struct rte_mbuf **pool) -{ - unsigned int i; - struct rxq_elt (*elts)[elts_n] = - rte_calloc_socket("RXQ elements", 1, sizeof(*elts), 0, - rxq->socket); - int ret = 0; - - if (elts == NULL) { - ERROR("%p: can't allocate packets array", (void *)rxq); - ret = ENOMEM; - goto error; - } - /* For each WR (packet). */ - for (i = 0; (i != elts_n); ++i) { - struct rxq_elt *elt = &(*elts)[i]; - struct ibv_recv_wr *wr = &elt->wr; - struct ibv_sge *sge = &(*elts)[i].sge; - struct rte_mbuf *buf; - - if (pool != NULL) { - buf = *(pool++); - assert(buf != NULL); - rte_pktmbuf_reset(buf); - } else - buf = rte_pktmbuf_alloc(rxq->mp); - if (buf == NULL) { - assert(pool == NULL); - ERROR("%p: empty mbuf pool", (void *)rxq); - ret = ENOMEM; - goto error; - } - /* Configure WR. Work request ID contains its own index in - * the elts array and the offset between SGE buffer header and - * its data. */ - WR_ID(wr->wr_id).id = i; - WR_ID(wr->wr_id).offset = - (((uintptr_t)buf->buf_addr + RTE_PKTMBUF_HEADROOM) - - (uintptr_t)buf); - wr->next = &(*elts)[(i + 1)].wr; - wr->sg_list = sge; - wr->num_sge = 1; - /* Headroom is reserved by rte_pktmbuf_alloc(). */ - assert(DATA_OFF(buf) == RTE_PKTMBUF_HEADROOM); - /* Buffer is supposed to be empty. */ - assert(rte_pktmbuf_data_len(buf) == 0); - assert(rte_pktmbuf_pkt_len(buf) == 0); - /* sge->addr must be able to store a pointer. */ - assert(sizeof(sge->addr) >= sizeof(uintptr_t)); - /* SGE keeps its headroom. */ - sge->addr = (uintptr_t) - ((uint8_t *)buf->buf_addr + RTE_PKTMBUF_HEADROOM); - sge->length = (buf->buf_len - RTE_PKTMBUF_HEADROOM); - sge->lkey = rxq->mr->lkey; - /* Redundant check for tailroom. */ - assert(sge->length == rte_pktmbuf_tailroom(buf)); - /* Make sure elts index and SGE mbuf pointer can be deduced - * from WR ID. */ - if ((WR_ID(wr->wr_id).id != i) || - ((void *)((uintptr_t)sge->addr - - WR_ID(wr->wr_id).offset) != buf)) { - ERROR("%p: cannot store index and offset in WR ID", - (void *)rxq); - sge->addr = 0; - rte_pktmbuf_free(buf); - ret = EOVERFLOW; - goto error; - } - } - /* The last WR pointer must be NULL. */ - (*elts)[(i - 1)].wr.next = NULL; - DEBUG("%p: allocated and configured %u single-segment WRs", - (void *)rxq, elts_n); - rxq->elts_n = elts_n; - rxq->elts_head = 0; - rxq->elts.no_sp = elts; - assert(ret == 0); - return 0; -error: - if (elts != NULL) { - assert(pool == NULL); - for (i = 0; (i != elemof(*elts)); ++i) { - struct rxq_elt *elt = &(*elts)[i]; - struct rte_mbuf *buf; - - if (elt->sge.addr == 0) - continue; - assert(WR_ID(elt->wr.wr_id).id == i); - buf = (void *)((uintptr_t)elt->sge.addr - - WR_ID(elt->wr.wr_id).offset); - rte_pktmbuf_free_seg(buf); - } - rte_free(elts); - } - DEBUG("%p: failed, freed everything", (void *)rxq); - assert(ret > 0); - return ret; -} - -/** - * Free RX queue elements. - * - * @param rxq - * Pointer to RX queue structure. - */ -static void -rxq_free_elts(struct rxq *rxq) -{ - unsigned int i; - unsigned int elts_n = rxq->elts_n; - struct rxq_elt (*elts)[elts_n] = rxq->elts.no_sp; - - DEBUG("%p: freeing WRs", (void *)rxq); - rxq->elts_n = 0; - rxq->elts.no_sp = NULL; - if (elts == NULL) - return; - for (i = 0; (i != elemof(*elts)); ++i) { - struct rxq_elt *elt = &(*elts)[i]; - struct rte_mbuf *buf; - - if (elt->sge.addr == 0) - continue; - assert(WR_ID(elt->wr.wr_id).id == i); - buf = (void *)((uintptr_t)elt->sge.addr - - WR_ID(elt->wr.wr_id).offset); - rte_pktmbuf_free_seg(buf); - } - rte_free(elts); -} - -/** - * Delete flow steering rule. - * - * @param rxq - * Pointer to RX queue structure. - * @param mac_index - * MAC address index. - * @param vlan_index - * VLAN index. - */ -static void -rxq_del_flow(struct rxq *rxq, unsigned int mac_index, unsigned int vlan_index) -{ -#ifndef NDEBUG - struct priv *priv = rxq->priv; - const uint8_t (*mac)[ETHER_ADDR_LEN] = - (const uint8_t (*)[ETHER_ADDR_LEN]) - priv->mac[mac_index].addr_bytes; -#endif - assert(rxq->mac_flow[mac_index][vlan_index] != NULL); - DEBUG("%p: removing MAC address %02x:%02x:%02x:%02x:%02x:%02x index %u" - " (VLAN ID %" PRIu16 ")", - (void *)rxq, - (*mac)[0], (*mac)[1], (*mac)[2], (*mac)[3], (*mac)[4], (*mac)[5], - mac_index, priv->vlan_filter[vlan_index].id); - claim_zero(ibv_destroy_flow(rxq->mac_flow[mac_index][vlan_index])); - rxq->mac_flow[mac_index][vlan_index] = NULL; -} - -/** - * Unregister a MAC address from a RX queue. - * - * @param rxq - * Pointer to RX queue structure. - * @param mac_index - * MAC address index. - */ -static void -rxq_mac_addr_del(struct rxq *rxq, unsigned int mac_index) -{ - struct priv *priv = rxq->priv; - unsigned int i; - unsigned int vlans = 0; - - assert(mac_index < elemof(priv->mac)); - if (!BITFIELD_ISSET(rxq->mac_configured, mac_index)) - return; - for (i = 0; (i != elemof(priv->vlan_filter)); ++i) { - if (!priv->vlan_filter[i].enabled) - continue; - rxq_del_flow(rxq, mac_index, i); - vlans++; - } - if (!vlans) { - rxq_del_flow(rxq, mac_index, 0); - } - BITFIELD_RESET(rxq->mac_configured, mac_index); -} - -/** - * Unregister all MAC addresses from a RX queue. - * - * @param rxq - * Pointer to RX queue structure. - */ -static void -rxq_mac_addrs_del(struct rxq *rxq) -{ - struct priv *priv = rxq->priv; - unsigned int i; - - for (i = 0; (i != elemof(priv->mac)); ++i) - rxq_mac_addr_del(rxq, i); -} - -static int rxq_promiscuous_enable(struct rxq *); -static void rxq_promiscuous_disable(struct rxq *); - -/** - * Add single flow steering rule. - * - * @param rxq - * Pointer to RX queue structure. - * @param mac_index - * MAC address index to register. - * @param vlan_index - * VLAN index. Use -1 for a flow without VLAN. - * - * @return - * 0 on success, errno value on failure. - */ -static int -rxq_add_flow(struct rxq *rxq, unsigned int mac_index, unsigned int vlan_index) -{ - struct ibv_flow *flow; - struct priv *priv = rxq->priv; - const uint8_t (*mac)[ETHER_ADDR_LEN] = - (const uint8_t (*)[ETHER_ADDR_LEN]) - priv->mac[mac_index].addr_bytes; - - /* Allocate flow specification on the stack. */ - struct __attribute__((packed)) { - struct ibv_flow_attr attr; - struct ibv_flow_spec_eth spec; - } data; - struct ibv_flow_attr *attr = &data.attr; - struct ibv_flow_spec_eth *spec = &data.spec; - - assert(mac_index < elemof(priv->mac)); - assert((vlan_index < elemof(priv->vlan_filter)) || (vlan_index == -1u)); - /* - * No padding must be inserted by the compiler between attr and spec. - * This layout is expected by libibverbs. - */ - assert(((uint8_t *)attr + sizeof(*attr)) == (uint8_t *)spec); - *attr = (struct ibv_flow_attr){ - .type = IBV_FLOW_ATTR_NORMAL, - .num_of_specs = 1, - .port = priv->port, - .flags = 0 - }; - *spec = (struct ibv_flow_spec_eth){ - .type = IBV_FLOW_SPEC_ETH, - .size = sizeof(*spec), - .val = { - .dst_mac = { - (*mac)[0], (*mac)[1], (*mac)[2], - (*mac)[3], (*mac)[4], (*mac)[5] - }, - .vlan_tag = ((vlan_index != -1u) ? - htons(priv->vlan_filter[vlan_index].id) : - 0), - }, - .mask = { - .dst_mac = "\xff\xff\xff\xff\xff\xff", - .vlan_tag = ((vlan_index != -1u) ? htons(0xfff) : 0), - } - }; - DEBUG("%p: adding MAC address %02x:%02x:%02x:%02x:%02x:%02x index %u" - " (VLAN %s %" PRIu16 ")", - (void *)rxq, - (*mac)[0], (*mac)[1], (*mac)[2], (*mac)[3], (*mac)[4], (*mac)[5], - mac_index, - ((vlan_index != -1u) ? "ID" : "index"), - ((vlan_index != -1u) ? priv->vlan_filter[vlan_index].id : -1u)); - /* Create related flow. */ - errno = 0; - flow = ibv_create_flow(rxq->qp, attr); - if (flow == NULL) { - /* It's not clear whether errno is always set in this case. */ - ERROR("%p: flow configuration failed, errno=%d: %s", - (void *)rxq, errno, - (errno ? strerror(errno) : "Unknown error")); - if (errno) - return errno; - return EINVAL; - } - if (vlan_index == -1u) - vlan_index = 0; - assert(rxq->mac_flow[mac_index][vlan_index] == NULL); - rxq->mac_flow[mac_index][vlan_index] = flow; - return 0; -} - -/** - * Register a MAC address in a RX queue. - * - * @param rxq - * Pointer to RX queue structure. - * @param mac_index - * MAC address index to register. - * - * @return - * 0 on success, errno value on failure. - */ -static int -rxq_mac_addr_add(struct rxq *rxq, unsigned int mac_index) -{ - struct priv *priv = rxq->priv; - unsigned int i; - unsigned int vlans = 0; - int ret; - - assert(mac_index < elemof(priv->mac)); - if (BITFIELD_ISSET(rxq->mac_configured, mac_index)) - rxq_mac_addr_del(rxq, mac_index); - /* Fill VLAN specifications. */ - for (i = 0; (i != elemof(priv->vlan_filter)); ++i) { - if (!priv->vlan_filter[i].enabled) - continue; - /* Create related flow. */ - ret = rxq_add_flow(rxq, mac_index, i); - if (!ret) { - vlans++; - continue; - } - /* Failure, rollback. */ - while (i != 0) - if (priv->vlan_filter[--i].enabled) - rxq_del_flow(rxq, mac_index, i); - assert(ret > 0); - return ret; - } - /* In case there is no VLAN filter. */ - if (!vlans) { - ret = rxq_add_flow(rxq, mac_index, -1); - if (ret) - return ret; - } - BITFIELD_SET(rxq->mac_configured, mac_index); - return 0; -} - -/** - * Register all MAC addresses in a RX queue. - * - * @param rxq - * Pointer to RX queue structure. - * - * @return - * 0 on success, errno value on failure. - */ -static int -rxq_mac_addrs_add(struct rxq *rxq) -{ - struct priv *priv = rxq->priv; - unsigned int i; - int ret; - - for (i = 0; (i != elemof(priv->mac)); ++i) { - if (!BITFIELD_ISSET(priv->mac_configured, i)) - continue; - ret = rxq_mac_addr_add(rxq, i); - if (!ret) - continue; - /* Failure, rollback. */ - while (i != 0) - rxq_mac_addr_del(rxq, --i); - assert(ret > 0); - return ret; - } - return 0; -} - -/** - * Unregister a MAC address. - * - * In RSS mode, the MAC address is unregistered from the parent queue, - * otherwise it is unregistered from each queue directly. - * - * @param priv - * Pointer to private structure. - * @param mac_index - * MAC address index. - */ -static void -priv_mac_addr_del(struct priv *priv, unsigned int mac_index) -{ - unsigned int i; - - assert(mac_index < elemof(priv->mac)); - if (!BITFIELD_ISSET(priv->mac_configured, mac_index)) - return; - if (priv->rss) { - rxq_mac_addr_del(&priv->rxq_parent, mac_index); - goto end; - } - for (i = 0; (i != priv->dev->data->nb_rx_queues); ++i) - rxq_mac_addr_del((*priv->rxqs)[i], mac_index); -end: - BITFIELD_RESET(priv->mac_configured, mac_index); -} - -/** - * Register a MAC address. - * - * In RSS mode, the MAC address is registered in the parent queue, - * otherwise it is registered in each queue directly. - * - * @param priv - * Pointer to private structure. - * @param mac_index - * MAC address index to use. - * @param mac - * MAC address to register. - * - * @return - * 0 on success, errno value on failure. - */ -static int -priv_mac_addr_add(struct priv *priv, unsigned int mac_index, - const uint8_t (*mac)[ETHER_ADDR_LEN]) -{ - unsigned int i; - int ret; - - assert(mac_index < elemof(priv->mac)); - /* First, make sure this address isn't already configured. */ - for (i = 0; (i != elemof(priv->mac)); ++i) { - /* Skip this index, it's going to be reconfigured. */ - if (i == mac_index) - continue; - if (!BITFIELD_ISSET(priv->mac_configured, i)) - continue; - if (memcmp(priv->mac[i].addr_bytes, *mac, sizeof(*mac))) - continue; - /* Address already configured elsewhere, return with error. */ - return EADDRINUSE; - } - if (BITFIELD_ISSET(priv->mac_configured, mac_index)) - priv_mac_addr_del(priv, mac_index); - priv->mac[mac_index] = (struct ether_addr){ - { - (*mac)[0], (*mac)[1], (*mac)[2], - (*mac)[3], (*mac)[4], (*mac)[5] - } - }; - /* If device isn't started, this is all we need to do. */ - if (!priv->started) { -#ifndef NDEBUG - /* Verify that all queues have this index disabled. */ - for (i = 0; (i != priv->rxqs_n); ++i) { - if ((*priv->rxqs)[i] == NULL) - continue; - assert(!BITFIELD_ISSET - ((*priv->rxqs)[i]->mac_configured, mac_index)); - } -#endif - goto end; - } - if (priv->rss) { - ret = rxq_mac_addr_add(&priv->rxq_parent, mac_index); - if (ret) - return ret; - goto end; - } - for (i = 0; (i != priv->rxqs_n); ++i) { - if ((*priv->rxqs)[i] == NULL) - continue; - ret = rxq_mac_addr_add((*priv->rxqs)[i], mac_index); - if (!ret) - continue; - /* Failure, rollback. */ - while (i != 0) - if ((*priv->rxqs)[(--i)] != NULL) - rxq_mac_addr_del((*priv->rxqs)[i], mac_index); - return ret; - } -end: - BITFIELD_SET(priv->mac_configured, mac_index); - return 0; -} - -/** - * Enable allmulti mode in a RX queue. - * - * @param rxq - * Pointer to RX queue structure. - * - * @return - * 0 on success, errno value on failure. - */ -static int -rxq_allmulticast_enable(struct rxq *rxq) -{ - struct ibv_flow *flow; - struct ibv_flow_attr attr = { - .type = IBV_FLOW_ATTR_MC_DEFAULT, - .num_of_specs = 0, - .port = rxq->priv->port, - .flags = 0 - }; - - DEBUG("%p: enabling allmulticast mode", (void *)rxq); - if (rxq->allmulti_flow != NULL) - return EBUSY; - errno = 0; - flow = ibv_create_flow(rxq->qp, &attr); - if (flow == NULL) { - /* It's not clear whether errno is always set in this case. */ - ERROR("%p: flow configuration failed, errno=%d: %s", - (void *)rxq, errno, - (errno ? strerror(errno) : "Unknown error")); - if (errno) - return errno; - return EINVAL; - } - rxq->allmulti_flow = flow; - DEBUG("%p: allmulticast mode enabled", (void *)rxq); - return 0; -} - -/** - * Disable allmulti mode in a RX queue. - * - * @param rxq - * Pointer to RX queue structure. - */ -static void -rxq_allmulticast_disable(struct rxq *rxq) -{ - DEBUG("%p: disabling allmulticast mode", (void *)rxq); - if (rxq->allmulti_flow == NULL) - return; - claim_zero(ibv_destroy_flow(rxq->allmulti_flow)); - rxq->allmulti_flow = NULL; - DEBUG("%p: allmulticast mode disabled", (void *)rxq); -} - -/** - * Enable promiscuous mode in a RX queue. - * - * @param rxq - * Pointer to RX queue structure. - * - * @return - * 0 on success, errno value on failure. - */ -static int -rxq_promiscuous_enable(struct rxq *rxq) -{ - struct ibv_flow *flow; - struct ibv_flow_attr attr = { - .type = IBV_FLOW_ATTR_ALL_DEFAULT, - .num_of_specs = 0, - .port = rxq->priv->port, - .flags = 0 - }; - - if (rxq->priv->vf) - return 0; - DEBUG("%p: enabling promiscuous mode", (void *)rxq); - if (rxq->promisc_flow != NULL) - return EBUSY; - errno = 0; - flow = ibv_create_flow(rxq->qp, &attr); - if (flow == NULL) { - /* It's not clear whether errno is always set in this case. */ - ERROR("%p: flow configuration failed, errno=%d: %s", - (void *)rxq, errno, - (errno ? strerror(errno) : "Unknown error")); - if (errno) - return errno; - return EINVAL; - } - rxq->promisc_flow = flow; - DEBUG("%p: promiscuous mode enabled", (void *)rxq); - return 0; -} - -/** - * Disable promiscuous mode in a RX queue. - * - * @param rxq - * Pointer to RX queue structure. - */ -static void -rxq_promiscuous_disable(struct rxq *rxq) -{ - if (rxq->priv->vf) - return; - DEBUG("%p: disabling promiscuous mode", (void *)rxq); - if (rxq->promisc_flow == NULL) - return; - claim_zero(ibv_destroy_flow(rxq->promisc_flow)); - rxq->promisc_flow = NULL; - DEBUG("%p: promiscuous mode disabled", (void *)rxq); -} - -/** - * Clean up a RX queue. - * - * Destroy objects, free allocated memory and reset the structure for reuse. - * - * @param rxq - * Pointer to RX queue structure. - */ -static void -rxq_cleanup(struct rxq *rxq) -{ - struct ibv_exp_release_intf_params params; - - DEBUG("cleaning up %p", (void *)rxq); - if (rxq->sp) - rxq_free_elts_sp(rxq); - else - rxq_free_elts(rxq); - if (rxq->if_qp != NULL) { - assert(rxq->priv != NULL); - assert(rxq->priv->ctx != NULL); - assert(rxq->qp != NULL); - params = (struct ibv_exp_release_intf_params){ - .comp_mask = 0, - }; - claim_zero(ibv_exp_release_intf(rxq->priv->ctx, - rxq->if_qp, - ¶ms)); - } - if (rxq->if_cq != NULL) { - assert(rxq->priv != NULL); - assert(rxq->priv->ctx != NULL); - assert(rxq->cq != NULL); - params = (struct ibv_exp_release_intf_params){ - .comp_mask = 0, - }; - claim_zero(ibv_exp_release_intf(rxq->priv->ctx, - rxq->if_cq, - ¶ms)); - } - if (rxq->qp != NULL) { - rxq_promiscuous_disable(rxq); - rxq_allmulticast_disable(rxq); - rxq_mac_addrs_del(rxq); - claim_zero(ibv_destroy_qp(rxq->qp)); - } - if (rxq->cq != NULL) - claim_zero(ibv_destroy_cq(rxq->cq)); - if (rxq->rd != NULL) { - struct ibv_exp_destroy_res_domain_attr attr = { - .comp_mask = 0, - }; - - assert(rxq->priv != NULL); - assert(rxq->priv->ctx != NULL); - claim_zero(ibv_exp_destroy_res_domain(rxq->priv->ctx, - rxq->rd, - &attr)); - } - if (rxq->mr != NULL) - claim_zero(ibv_dereg_mr(rxq->mr)); - memset(rxq, 0, sizeof(*rxq)); -} - -/** - * Translate RX completion flags to packet type. - * - * @param flags - * RX completion flags returned by poll_length_flags(). - * - * @note: fix mlx4_dev_supported_ptypes_get() if any change here. - * - * @return - * Packet type for struct rte_mbuf. - */ -static inline uint32_t -rxq_cq_to_pkt_type(uint32_t flags) -{ - uint32_t pkt_type; - - if (flags & IBV_EXP_CQ_RX_TUNNEL_PACKET) - pkt_type = - TRANSPOSE(flags, - IBV_EXP_CQ_RX_OUTER_IPV4_PACKET, RTE_PTYPE_L3_IPV4) | - TRANSPOSE(flags, - IBV_EXP_CQ_RX_OUTER_IPV6_PACKET, RTE_PTYPE_L3_IPV6) | - TRANSPOSE(flags, - IBV_EXP_CQ_RX_IPV4_PACKET, RTE_PTYPE_INNER_L3_IPV4) | - TRANSPOSE(flags, - IBV_EXP_CQ_RX_IPV6_PACKET, RTE_PTYPE_INNER_L3_IPV6); - else - pkt_type = - TRANSPOSE(flags, - IBV_EXP_CQ_RX_IPV4_PACKET, RTE_PTYPE_L3_IPV4) | - TRANSPOSE(flags, - IBV_EXP_CQ_RX_IPV6_PACKET, RTE_PTYPE_L3_IPV6); - return pkt_type; -} - -/** - * Translate RX completion flags to offload flags. - * - * @param[in] rxq - * Pointer to RX queue structure. - * @param flags - * RX completion flags returned by poll_length_flags(). - * - * @return - * Offload flags (ol_flags) for struct rte_mbuf. - */ -static inline uint32_t -rxq_cq_to_ol_flags(const struct rxq *rxq, uint32_t flags) -{ - uint32_t ol_flags = 0; - - if (rxq->csum) - ol_flags |= - TRANSPOSE(~flags, - IBV_EXP_CQ_RX_IP_CSUM_OK, - PKT_RX_IP_CKSUM_BAD) | - TRANSPOSE(~flags, - IBV_EXP_CQ_RX_TCP_UDP_CSUM_OK, - PKT_RX_L4_CKSUM_BAD); - /* - * PKT_RX_IP_CKSUM_BAD and PKT_RX_L4_CKSUM_BAD are used in place - * of PKT_RX_EIP_CKSUM_BAD because the latter is not functional - * (its value is 0). - */ - if ((flags & IBV_EXP_CQ_RX_TUNNEL_PACKET) && (rxq->csum_l2tun)) - ol_flags |= - TRANSPOSE(~flags, - IBV_EXP_CQ_RX_OUTER_IP_CSUM_OK, - PKT_RX_IP_CKSUM_BAD) | - TRANSPOSE(~flags, - IBV_EXP_CQ_RX_OUTER_TCP_UDP_CSUM_OK, - PKT_RX_L4_CKSUM_BAD); - return ol_flags; -} - -static uint16_t -mlx4_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n); - -/** - * DPDK callback for RX with scattered packets support. - * - * @param dpdk_rxq - * Generic pointer to RX queue structure. - * @param[out] pkts - * Array to store received packets. - * @param pkts_n - * Maximum number of packets in array. - * - * @return - * Number of packets successfully received (<= pkts_n). - */ -static uint16_t -mlx4_rx_burst_sp(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) -{ - struct rxq *rxq = (struct rxq *)dpdk_rxq; - struct rxq_elt_sp (*elts)[rxq->elts_n] = rxq->elts.sp; - const unsigned int elts_n = rxq->elts_n; - unsigned int elts_head = rxq->elts_head; - struct ibv_recv_wr head; - struct ibv_recv_wr **next = &head.next; - struct ibv_recv_wr *bad_wr; - unsigned int i; - unsigned int pkts_ret = 0; - int ret; - - if (unlikely(!rxq->sp)) - return mlx4_rx_burst(dpdk_rxq, pkts, pkts_n); - if (unlikely(elts == NULL)) /* See RTE_DEV_CMD_SET_MTU. */ - return 0; - for (i = 0; (i != pkts_n); ++i) { - struct rxq_elt_sp *elt = &(*elts)[elts_head]; - struct ibv_recv_wr *wr = &elt->wr; - uint64_t wr_id = wr->wr_id; - unsigned int len; - unsigned int pkt_buf_len; - struct rte_mbuf *pkt_buf = NULL; /* Buffer returned in pkts. */ - struct rte_mbuf **pkt_buf_next = &pkt_buf; - unsigned int seg_headroom = RTE_PKTMBUF_HEADROOM; - unsigned int j = 0; - uint32_t flags; - - /* Sanity checks. */ -#ifdef NDEBUG - (void)wr_id; -#endif - assert(wr_id < rxq->elts_n); - assert(wr->sg_list == elt->sges); - assert(wr->num_sge == elemof(elt->sges)); - assert(elts_head < rxq->elts_n); - assert(rxq->elts_head < rxq->elts_n); - ret = rxq->if_cq->poll_length_flags(rxq->cq, NULL, NULL, - &flags); - if (unlikely(ret < 0)) { - struct ibv_wc wc; - int wcs_n; - - DEBUG("rxq=%p, poll_length() failed (ret=%d)", - (void *)rxq, ret); - /* ibv_poll_cq() must be used in case of failure. */ - wcs_n = ibv_poll_cq(rxq->cq, 1, &wc); - if (unlikely(wcs_n == 0)) - break; - if (unlikely(wcs_n < 0)) { - DEBUG("rxq=%p, ibv_poll_cq() failed (wcs_n=%d)", - (void *)rxq, wcs_n); - break; - } - assert(wcs_n == 1); - if (unlikely(wc.status != IBV_WC_SUCCESS)) { - /* Whatever, just repost the offending WR. */ - DEBUG("rxq=%p, wr_id=%" PRIu64 ": bad work" - " completion status (%d): %s", - (void *)rxq, wc.wr_id, wc.status, - ibv_wc_status_str(wc.status)); -#ifdef MLX4_PMD_SOFT_COUNTERS - /* Increment dropped packets counter. */ - ++rxq->stats.idropped; -#endif - /* Link completed WRs together for repost. */ - *next = wr; - next = &wr->next; - goto repost; - } - ret = wc.byte_len; - } - if (ret == 0) - break; - len = ret; - pkt_buf_len = len; - /* Link completed WRs together for repost. */ - *next = wr; - next = &wr->next; - /* - * Replace spent segments with new ones, concatenate and - * return them as pkt_buf. - */ - while (1) { - struct ibv_sge *sge = &elt->sges[j]; - struct rte_mbuf *seg = elt->bufs[j]; - struct rte_mbuf *rep; - unsigned int seg_tailroom; - - /* - * Fetch initial bytes of packet descriptor into a - * cacheline while allocating rep. - */ - rte_prefetch0(seg); - rep = rte_mbuf_raw_alloc(rxq->mp); - if (unlikely(rep == NULL)) { - /* - * Unable to allocate a replacement mbuf, - * repost WR. - */ - DEBUG("rxq=%p, wr_id=%" PRIu64 ":" - " can't allocate a new mbuf", - (void *)rxq, wr_id); - if (pkt_buf != NULL) { - *pkt_buf_next = NULL; - rte_pktmbuf_free(pkt_buf); - } - /* Increase out of memory counters. */ - ++rxq->stats.rx_nombuf; - ++rxq->priv->dev->data->rx_mbuf_alloc_failed; - goto repost; - } -#ifndef NDEBUG - /* Poison user-modifiable fields in rep. */ - NEXT(rep) = (void *)((uintptr_t)-1); - SET_DATA_OFF(rep, 0xdead); - DATA_LEN(rep) = 0xd00d; - PKT_LEN(rep) = 0xdeadd00d; - NB_SEGS(rep) = 0x2a; - PORT(rep) = 0x2a; - rep->ol_flags = -1; -#endif - assert(rep->buf_len == seg->buf_len); - /* Reconfigure sge to use rep instead of seg. */ - assert(sge->lkey == rxq->mr->lkey); - sge->addr = ((uintptr_t)rep->buf_addr + seg_headroom); - elt->bufs[j] = rep; - ++j; - /* Update pkt_buf if it's the first segment, or link - * seg to the previous one and update pkt_buf_next. */ - *pkt_buf_next = seg; - pkt_buf_next = &NEXT(seg); - /* Update seg information. */ - seg_tailroom = (seg->buf_len - seg_headroom); - assert(sge->length == seg_tailroom); - SET_DATA_OFF(seg, seg_headroom); - if (likely(len <= seg_tailroom)) { - /* Last segment. */ - DATA_LEN(seg) = len; - PKT_LEN(seg) = len; - /* Sanity check. */ - assert(rte_pktmbuf_headroom(seg) == - seg_headroom); - assert(rte_pktmbuf_tailroom(seg) == - (seg_tailroom - len)); - break; - } - DATA_LEN(seg) = seg_tailroom; - PKT_LEN(seg) = seg_tailroom; - /* Sanity check. */ - assert(rte_pktmbuf_headroom(seg) == seg_headroom); - assert(rte_pktmbuf_tailroom(seg) == 0); - /* Fix len and clear headroom for next segments. */ - len -= seg_tailroom; - seg_headroom = 0; - } - /* Update head and tail segments. */ - *pkt_buf_next = NULL; - assert(pkt_buf != NULL); - assert(j != 0); - NB_SEGS(pkt_buf) = j; - PORT(pkt_buf) = rxq->port_id; - PKT_LEN(pkt_buf) = pkt_buf_len; - pkt_buf->packet_type = rxq_cq_to_pkt_type(flags); - pkt_buf->ol_flags = rxq_cq_to_ol_flags(rxq, flags); - - /* Return packet. */ - *(pkts++) = pkt_buf; - ++pkts_ret; -#ifdef MLX4_PMD_SOFT_COUNTERS - /* Increase bytes counter. */ - rxq->stats.ibytes += pkt_buf_len; -#endif -repost: - if (++elts_head >= elts_n) - elts_head = 0; - continue; - } - if (unlikely(i == 0)) - return 0; - *next = NULL; - /* Repost WRs. */ -#ifdef DEBUG_RECV - DEBUG("%p: reposting %d WRs", (void *)rxq, i); -#endif - ret = ibv_post_recv(rxq->qp, head.next, &bad_wr); - if (unlikely(ret)) { - /* Inability to repost WRs is fatal. */ - DEBUG("%p: ibv_post_recv(): failed for WR %p: %s", - (void *)rxq->priv, - (void *)bad_wr, - strerror(ret)); - abort(); - } - rxq->elts_head = elts_head; -#ifdef MLX4_PMD_SOFT_COUNTERS - /* Increase packets counter. */ - rxq->stats.ipackets += pkts_ret; -#endif - return pkts_ret; -} - -/** - * DPDK callback for RX. - * - * The following function is the same as mlx4_rx_burst_sp(), except it doesn't - * manage scattered packets. Improves performance when MRU is lower than the - * size of the first segment. - * - * @param dpdk_rxq - * Generic pointer to RX queue structure. - * @param[out] pkts - * Array to store received packets. - * @param pkts_n - * Maximum number of packets in array. - * - * @return - * Number of packets successfully received (<= pkts_n). - */ -static uint16_t -mlx4_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) -{ - struct rxq *rxq = (struct rxq *)dpdk_rxq; - struct rxq_elt (*elts)[rxq->elts_n] = rxq->elts.no_sp; - const unsigned int elts_n = rxq->elts_n; - unsigned int elts_head = rxq->elts_head; - struct ibv_sge sges[pkts_n]; - unsigned int i; - unsigned int pkts_ret = 0; - int ret; - - if (unlikely(rxq->sp)) - return mlx4_rx_burst_sp(dpdk_rxq, pkts, pkts_n); - for (i = 0; (i != pkts_n); ++i) { - struct rxq_elt *elt = &(*elts)[elts_head]; - struct ibv_recv_wr *wr = &elt->wr; - uint64_t wr_id = wr->wr_id; - unsigned int len; - struct rte_mbuf *seg = (void *)((uintptr_t)elt->sge.addr - - WR_ID(wr_id).offset); - struct rte_mbuf *rep; - uint32_t flags; - - /* Sanity checks. */ - assert(WR_ID(wr_id).id < rxq->elts_n); - assert(wr->sg_list == &elt->sge); - assert(wr->num_sge == 1); - assert(elts_head < rxq->elts_n); - assert(rxq->elts_head < rxq->elts_n); - /* - * Fetch initial bytes of packet descriptor into a - * cacheline while allocating rep. - */ - rte_mbuf_prefetch_part1(seg); - rte_mbuf_prefetch_part2(seg); - ret = rxq->if_cq->poll_length_flags(rxq->cq, NULL, NULL, - &flags); - if (unlikely(ret < 0)) { - struct ibv_wc wc; - int wcs_n; - - DEBUG("rxq=%p, poll_length() failed (ret=%d)", - (void *)rxq, ret); - /* ibv_poll_cq() must be used in case of failure. */ - wcs_n = ibv_poll_cq(rxq->cq, 1, &wc); - if (unlikely(wcs_n == 0)) - break; - if (unlikely(wcs_n < 0)) { - DEBUG("rxq=%p, ibv_poll_cq() failed (wcs_n=%d)", - (void *)rxq, wcs_n); - break; - } - assert(wcs_n == 1); - if (unlikely(wc.status != IBV_WC_SUCCESS)) { - /* Whatever, just repost the offending WR. */ - DEBUG("rxq=%p, wr_id=%" PRIu64 ": bad work" - " completion status (%d): %s", - (void *)rxq, wc.wr_id, wc.status, - ibv_wc_status_str(wc.status)); -#ifdef MLX4_PMD_SOFT_COUNTERS - /* Increment dropped packets counter. */ - ++rxq->stats.idropped; -#endif - /* Add SGE to array for repost. */ - sges[i] = elt->sge; - goto repost; - } - ret = wc.byte_len; - } - if (ret == 0) - break; - len = ret; - rep = rte_mbuf_raw_alloc(rxq->mp); - if (unlikely(rep == NULL)) { - /* - * Unable to allocate a replacement mbuf, - * repost WR. - */ - DEBUG("rxq=%p, wr_id=%" PRIu32 ":" - " can't allocate a new mbuf", - (void *)rxq, WR_ID(wr_id).id); - /* Increase out of memory counters. */ - ++rxq->stats.rx_nombuf; - ++rxq->priv->dev->data->rx_mbuf_alloc_failed; - goto repost; - } - - /* Reconfigure sge to use rep instead of seg. */ - elt->sge.addr = (uintptr_t)rep->buf_addr + RTE_PKTMBUF_HEADROOM; - assert(elt->sge.lkey == rxq->mr->lkey); - WR_ID(wr->wr_id).offset = - (((uintptr_t)rep->buf_addr + RTE_PKTMBUF_HEADROOM) - - (uintptr_t)rep); - assert(WR_ID(wr->wr_id).id == WR_ID(wr_id).id); - - /* Add SGE to array for repost. */ - sges[i] = elt->sge; - - /* Update seg information. */ - SET_DATA_OFF(seg, RTE_PKTMBUF_HEADROOM); - NB_SEGS(seg) = 1; - PORT(seg) = rxq->port_id; - NEXT(seg) = NULL; - PKT_LEN(seg) = len; - DATA_LEN(seg) = len; - seg->packet_type = rxq_cq_to_pkt_type(flags); - seg->ol_flags = rxq_cq_to_ol_flags(rxq, flags); - - /* Return packet. */ - *(pkts++) = seg; - ++pkts_ret; -#ifdef MLX4_PMD_SOFT_COUNTERS - /* Increase bytes counter. */ - rxq->stats.ibytes += len; -#endif -repost: - if (++elts_head >= elts_n) - elts_head = 0; - continue; - } - if (unlikely(i == 0)) - return 0; - /* Repost WRs. */ -#ifdef DEBUG_RECV - DEBUG("%p: reposting %u WRs", (void *)rxq, i); -#endif - ret = rxq->if_qp->recv_burst(rxq->qp, sges, i); - if (unlikely(ret)) { - /* Inability to repost WRs is fatal. */ - DEBUG("%p: recv_burst(): failed (ret=%d)", - (void *)rxq->priv, - ret); - abort(); - } - rxq->elts_head = elts_head; -#ifdef MLX4_PMD_SOFT_COUNTERS - /* Increase packets counter. */ - rxq->stats.ipackets += pkts_ret; -#endif - return pkts_ret; -} - -/** - * DPDK callback for RX in secondary processes. - * - * This function configures all queues from primary process information - * if necessary before reverting to the normal RX burst callback. - * - * @param dpdk_rxq - * Generic pointer to RX queue structure. - * @param[out] pkts - * Array to store received packets. - * @param pkts_n - * Maximum number of packets in array. - * - * @return - * Number of packets successfully received (<= pkts_n). - */ -static uint16_t -mlx4_rx_burst_secondary_setup(void *dpdk_rxq, struct rte_mbuf **pkts, - uint16_t pkts_n) -{ - struct rxq *rxq = dpdk_rxq; - struct priv *priv = mlx4_secondary_data_setup(rxq->priv); - struct priv *primary_priv; - unsigned int index; - - if (priv == NULL) - return 0; - primary_priv = - mlx4_secondary_data[priv->dev->data->port_id].primary_priv; - /* Look for queue index in both private structures. */ - for (index = 0; index != priv->rxqs_n; ++index) - if (((*primary_priv->rxqs)[index] == rxq) || - ((*priv->rxqs)[index] == rxq)) - break; - if (index == priv->rxqs_n) - return 0; - rxq = (*priv->rxqs)[index]; - return priv->dev->rx_pkt_burst(rxq, pkts, pkts_n); -} - -/** - * Allocate a Queue Pair. - * Optionally setup inline receive if supported. - * - * @param priv - * Pointer to private structure. - * @param cq - * Completion queue to associate with QP. - * @param desc - * Number of descriptors in QP (hint only). - * - * @return - * QP pointer or NULL in case of error. - */ -static struct ibv_qp * -rxq_setup_qp(struct priv *priv, struct ibv_cq *cq, uint16_t desc, - struct ibv_exp_res_domain *rd) -{ - struct ibv_exp_qp_init_attr attr = { - /* CQ to be associated with the send queue. */ - .send_cq = cq, - /* CQ to be associated with the receive queue. */ - .recv_cq = cq, - .cap = { - /* Max number of outstanding WRs. */ - .max_recv_wr = ((priv->device_attr.max_qp_wr < desc) ? - priv->device_attr.max_qp_wr : - desc), - /* Max number of scatter/gather elements in a WR. */ - .max_recv_sge = ((priv->device_attr.max_sge < - MLX4_PMD_SGE_WR_N) ? - priv->device_attr.max_sge : - MLX4_PMD_SGE_WR_N), - }, - .qp_type = IBV_QPT_RAW_PACKET, - .comp_mask = (IBV_EXP_QP_INIT_ATTR_PD | - IBV_EXP_QP_INIT_ATTR_RES_DOMAIN), - .pd = priv->pd, - .res_domain = rd, - }; - -#ifdef INLINE_RECV - attr.max_inl_recv = priv->inl_recv_size; - attr.comp_mask |= IBV_EXP_QP_INIT_ATTR_INL_RECV; -#endif - return ibv_exp_create_qp(priv->ctx, &attr); -} - -#ifdef RSS_SUPPORT - -/** - * Allocate a RSS Queue Pair. - * Optionally setup inline receive if supported. - * - * @param priv - * Pointer to private structure. - * @param cq - * Completion queue to associate with QP. - * @param desc - * Number of descriptors in QP (hint only). - * @param parent - * If nonzero, create a parent QP, otherwise a child. - * - * @return - * QP pointer or NULL in case of error. - */ -static struct ibv_qp * -rxq_setup_qp_rss(struct priv *priv, struct ibv_cq *cq, uint16_t desc, - int parent, struct ibv_exp_res_domain *rd) -{ - struct ibv_exp_qp_init_attr attr = { - /* CQ to be associated with the send queue. */ - .send_cq = cq, - /* CQ to be associated with the receive queue. */ - .recv_cq = cq, - .cap = { - /* Max number of outstanding WRs. */ - .max_recv_wr = ((priv->device_attr.max_qp_wr < desc) ? - priv->device_attr.max_qp_wr : - desc), - /* Max number of scatter/gather elements in a WR. */ - .max_recv_sge = ((priv->device_attr.max_sge < - MLX4_PMD_SGE_WR_N) ? - priv->device_attr.max_sge : - MLX4_PMD_SGE_WR_N), - }, - .qp_type = IBV_QPT_RAW_PACKET, - .comp_mask = (IBV_EXP_QP_INIT_ATTR_PD | - IBV_EXP_QP_INIT_ATTR_RES_DOMAIN | - IBV_EXP_QP_INIT_ATTR_QPG), - .pd = priv->pd, - .res_domain = rd, - }; - -#ifdef INLINE_RECV - attr.max_inl_recv = priv->inl_recv_size, - attr.comp_mask |= IBV_EXP_QP_INIT_ATTR_INL_RECV; -#endif - if (parent) { - attr.qpg.qpg_type = IBV_EXP_QPG_PARENT; - /* TSS isn't necessary. */ - attr.qpg.parent_attrib.tss_child_count = 0; - attr.qpg.parent_attrib.rss_child_count = - rte_align32pow2(priv->rxqs_n + 1) >> 1; - DEBUG("initializing parent RSS queue"); - } else { - attr.qpg.qpg_type = IBV_EXP_QPG_CHILD_RX; - attr.qpg.qpg_parent = priv->rxq_parent.qp; - DEBUG("initializing child RSS queue"); - } - return ibv_exp_create_qp(priv->ctx, &attr); -} - -#endif /* RSS_SUPPORT */ - -/** - * Reconfigure a RX queue with new parameters. - * - * rxq_rehash() does not allocate mbufs, which, if not done from the right - * thread (such as a control thread), may corrupt the pool. - * In case of failure, the queue is left untouched. - * - * @param dev - * Pointer to Ethernet device structure. - * @param rxq - * RX queue pointer. - * - * @return - * 0 on success, errno value on failure. - */ -static int -rxq_rehash(struct rte_eth_dev *dev, struct rxq *rxq) -{ - struct priv *priv = rxq->priv; - struct rxq tmpl = *rxq; - unsigned int mbuf_n; - unsigned int desc_n; - struct rte_mbuf **pool; - unsigned int i, k; - struct ibv_exp_qp_attr mod; - struct ibv_recv_wr *bad_wr; - unsigned int mb_len; - int err; - int parent = (rxq == &priv->rxq_parent); - - if (parent) { - ERROR("%p: cannot rehash parent queue %p", - (void *)dev, (void *)rxq); - return EINVAL; - } - mb_len = rte_pktmbuf_data_room_size(rxq->mp); - DEBUG("%p: rehashing queue %p", (void *)dev, (void *)rxq); - /* Number of descriptors and mbufs currently allocated. */ - desc_n = (tmpl.elts_n * (tmpl.sp ? MLX4_PMD_SGE_WR_N : 1)); - mbuf_n = desc_n; - /* Toggle RX checksum offload if hardware supports it. */ - if (priv->hw_csum) { - tmpl.csum = !!dev->data->dev_conf.rxmode.hw_ip_checksum; - rxq->csum = tmpl.csum; - } - if (priv->hw_csum_l2tun) { - tmpl.csum_l2tun = !!dev->data->dev_conf.rxmode.hw_ip_checksum; - rxq->csum_l2tun = tmpl.csum_l2tun; - } - /* Enable scattered packets support for this queue if necessary. */ - assert(mb_len >= RTE_PKTMBUF_HEADROOM); - if ((dev->data->dev_conf.rxmode.jumbo_frame) && - (dev->data->dev_conf.rxmode.max_rx_pkt_len > - (mb_len - RTE_PKTMBUF_HEADROOM))) { - tmpl.sp = 1; - desc_n /= MLX4_PMD_SGE_WR_N; - } else - tmpl.sp = 0; - DEBUG("%p: %s scattered packets support (%u WRs)", - (void *)dev, (tmpl.sp ? "enabling" : "disabling"), desc_n); - /* If scatter mode is the same as before, nothing to do. */ - if (tmpl.sp == rxq->sp) { - DEBUG("%p: nothing to do", (void *)dev); - return 0; - } - /* Remove attached flows if RSS is disabled (no parent queue). */ - if (!priv->rss) { - rxq_allmulticast_disable(&tmpl); - rxq_promiscuous_disable(&tmpl); - rxq_mac_addrs_del(&tmpl); - /* Update original queue in case of failure. */ - rxq->allmulti_flow = tmpl.allmulti_flow; - rxq->promisc_flow = tmpl.promisc_flow; - memcpy(rxq->mac_configured, tmpl.mac_configured, - sizeof(rxq->mac_configured)); - memcpy(rxq->mac_flow, tmpl.mac_flow, sizeof(rxq->mac_flow)); - } - /* From now on, any failure will render the queue unusable. - * Reinitialize QP. */ - mod = (struct ibv_exp_qp_attr){ .qp_state = IBV_QPS_RESET }; - err = ibv_exp_modify_qp(tmpl.qp, &mod, IBV_EXP_QP_STATE); - if (err) { - ERROR("%p: cannot reset QP: %s", (void *)dev, strerror(err)); - assert(err > 0); - return err; - } - err = ibv_resize_cq(tmpl.cq, desc_n); - if (err) { - ERROR("%p: cannot resize CQ: %s", (void *)dev, strerror(err)); - assert(err > 0); - return err; - } - mod = (struct ibv_exp_qp_attr){ - /* Move the QP to this state. */ - .qp_state = IBV_QPS_INIT, - /* Primary port number. */ - .port_num = priv->port - }; - err = ibv_exp_modify_qp(tmpl.qp, &mod, - (IBV_EXP_QP_STATE | -#ifdef RSS_SUPPORT - (parent ? IBV_EXP_QP_GROUP_RSS : 0) | -#endif /* RSS_SUPPORT */ - IBV_EXP_QP_PORT)); - if (err) { - ERROR("%p: QP state to IBV_QPS_INIT failed: %s", - (void *)dev, strerror(err)); - assert(err > 0); - return err; - }; - /* Reconfigure flows. Do not care for errors. */ - if (!priv->rss) { - rxq_mac_addrs_add(&tmpl); - if (priv->promisc) - rxq_promiscuous_enable(&tmpl); - if (priv->allmulti) - rxq_allmulticast_enable(&tmpl); - /* Update original queue in case of failure. */ - rxq->allmulti_flow = tmpl.allmulti_flow; - rxq->promisc_flow = tmpl.promisc_flow; - memcpy(rxq->mac_configured, tmpl.mac_configured, - sizeof(rxq->mac_configured)); - memcpy(rxq->mac_flow, tmpl.mac_flow, sizeof(rxq->mac_flow)); - } - /* Allocate pool. */ - pool = rte_malloc(__func__, (mbuf_n * sizeof(*pool)), 0); - if (pool == NULL) { - ERROR("%p: cannot allocate memory", (void *)dev); - return ENOBUFS; - } - /* Snatch mbufs from original queue. */ - k = 0; - if (rxq->sp) { - struct rxq_elt_sp (*elts)[rxq->elts_n] = rxq->elts.sp; - - for (i = 0; (i != elemof(*elts)); ++i) { - struct rxq_elt_sp *elt = &(*elts)[i]; - unsigned int j; - - for (j = 0; (j != elemof(elt->bufs)); ++j) { - assert(elt->bufs[j] != NULL); - pool[k++] = elt->bufs[j]; - } - } - } else { - struct rxq_elt (*elts)[rxq->elts_n] = rxq->elts.no_sp; - - for (i = 0; (i != elemof(*elts)); ++i) { - struct rxq_elt *elt = &(*elts)[i]; - struct rte_mbuf *buf = (void *) - ((uintptr_t)elt->sge.addr - - WR_ID(elt->wr.wr_id).offset); - - assert(WR_ID(elt->wr.wr_id).id == i); - pool[k++] = buf; - } - } - assert(k == mbuf_n); - tmpl.elts_n = 0; - tmpl.elts.sp = NULL; - assert((void *)&tmpl.elts.sp == (void *)&tmpl.elts.no_sp); - err = ((tmpl.sp) ? - rxq_alloc_elts_sp(&tmpl, desc_n, pool) : - rxq_alloc_elts(&tmpl, desc_n, pool)); - if (err) { - ERROR("%p: cannot reallocate WRs, aborting", (void *)dev); - rte_free(pool); - assert(err > 0); - return err; - } - assert(tmpl.elts_n == desc_n); - assert(tmpl.elts.sp != NULL); - rte_free(pool); - /* Clean up original data. */ - rxq->elts_n = 0; - rte_free(rxq->elts.sp); - rxq->elts.sp = NULL; - /* Post WRs. */ - err = ibv_post_recv(tmpl.qp, - (tmpl.sp ? - &(*tmpl.elts.sp)[0].wr : - &(*tmpl.elts.no_sp)[0].wr), - &bad_wr); - if (err) { - ERROR("%p: ibv_post_recv() failed for WR %p: %s", - (void *)dev, - (void *)bad_wr, - strerror(err)); - goto skip_rtr; - } - mod = (struct ibv_exp_qp_attr){ - .qp_state = IBV_QPS_RTR - }; - err = ibv_exp_modify_qp(tmpl.qp, &mod, IBV_EXP_QP_STATE); - if (err) - ERROR("%p: QP state to IBV_QPS_RTR failed: %s", - (void *)dev, strerror(err)); -skip_rtr: - *rxq = tmpl; - assert(err >= 0); - return err; -} - -/** - * Configure a RX queue. - * - * @param dev - * Pointer to Ethernet device structure. - * @param rxq - * Pointer to RX queue structure. - * @param desc - * Number of descriptors to configure in queue. - * @param socket - * NUMA socket on which memory must be allocated. - * @param inactive - * If true, the queue is disabled because its index is higher or - * equal to the real number of queues, which must be a power of 2. - * @param[in] conf - * Thresholds parameters. - * @param mp - * Memory pool for buffer allocations. - * - * @return - * 0 on success, errno value on failure. - */ -static int -rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc, - unsigned int socket, int inactive, const struct rte_eth_rxconf *conf, - struct rte_mempool *mp) -{ - struct priv *priv = dev->data->dev_private; - struct rxq tmpl = { - .priv = priv, - .mp = mp, - .socket = socket - }; - struct ibv_exp_qp_attr mod; - union { - struct ibv_exp_query_intf_params params; - struct ibv_exp_cq_init_attr cq; - struct ibv_exp_res_domain_init_attr rd; - } attr; - enum ibv_exp_query_intf_status status; - struct ibv_recv_wr *bad_wr; - unsigned int mb_len; - int ret = 0; - int parent = (rxq == &priv->rxq_parent); - - (void)conf; /* Thresholds configuration (ignored). */ - /* - * If this is a parent queue, hardware must support RSS and - * RSS must be enabled. - */ - assert((!parent) || ((priv->hw_rss) && (priv->rss))); - if (parent) { - /* Even if unused, ibv_create_cq() requires at least one - * descriptor. */ - desc = 1; - goto skip_mr; - } - mb_len = rte_pktmbuf_data_room_size(mp); - if ((desc == 0) || (desc % MLX4_PMD_SGE_WR_N)) { - ERROR("%p: invalid number of RX descriptors (must be a" - " multiple of %d)", (void *)dev, MLX4_PMD_SGE_WR_N); - return EINVAL; - } - /* Toggle RX checksum offload if hardware supports it. */ - if (priv->hw_csum) - tmpl.csum = !!dev->data->dev_conf.rxmode.hw_ip_checksum; - if (priv->hw_csum_l2tun) - tmpl.csum_l2tun = !!dev->data->dev_conf.rxmode.hw_ip_checksum; - /* Enable scattered packets support for this queue if necessary. */ - assert(mb_len >= RTE_PKTMBUF_HEADROOM); - if ((dev->data->dev_conf.rxmode.jumbo_frame) && - (dev->data->dev_conf.rxmode.max_rx_pkt_len > - (mb_len - RTE_PKTMBUF_HEADROOM))) { - tmpl.sp = 1; - desc /= MLX4_PMD_SGE_WR_N; - } - DEBUG("%p: %s scattered packets support (%u WRs)", - (void *)dev, (tmpl.sp ? "enabling" : "disabling"), desc); - /* Use the entire RX mempool as the memory region. */ - tmpl.mr = mlx4_mp2mr(priv->pd, mp); - if (tmpl.mr == NULL) { - ret = EINVAL; - ERROR("%p: MR creation failure: %s", - (void *)dev, strerror(ret)); - goto error; - } -skip_mr: - attr.rd = (struct ibv_exp_res_domain_init_attr){ - .comp_mask = (IBV_EXP_RES_DOMAIN_THREAD_MODEL | - IBV_EXP_RES_DOMAIN_MSG_MODEL), - .thread_model = IBV_EXP_THREAD_SINGLE, - .msg_model = IBV_EXP_MSG_HIGH_BW, - }; - tmpl.rd = ibv_exp_create_res_domain(priv->ctx, &attr.rd); - if (tmpl.rd == NULL) { - ret = ENOMEM; - ERROR("%p: RD creation failure: %s", - (void *)dev, strerror(ret)); - goto error; - } - attr.cq = (struct ibv_exp_cq_init_attr){ - .comp_mask = IBV_EXP_CQ_INIT_ATTR_RES_DOMAIN, - .res_domain = tmpl.rd, - }; - tmpl.cq = ibv_exp_create_cq(priv->ctx, desc, NULL, NULL, 0, &attr.cq); - if (tmpl.cq == NULL) { - ret = ENOMEM; - ERROR("%p: CQ creation failure: %s", - (void *)dev, strerror(ret)); - goto error; - } - DEBUG("priv->device_attr.max_qp_wr is %d", - priv->device_attr.max_qp_wr); - DEBUG("priv->device_attr.max_sge is %d", - priv->device_attr.max_sge); -#ifdef RSS_SUPPORT - if (priv->rss && !inactive) - tmpl.qp = rxq_setup_qp_rss(priv, tmpl.cq, desc, parent, - tmpl.rd); - else -#endif /* RSS_SUPPORT */ - tmpl.qp = rxq_setup_qp(priv, tmpl.cq, desc, tmpl.rd); - if (tmpl.qp == NULL) { - ret = (errno ? errno : EINVAL); - ERROR("%p: QP creation failure: %s", - (void *)dev, strerror(ret)); - goto error; - } - mod = (struct ibv_exp_qp_attr){ - /* Move the QP to this state. */ - .qp_state = IBV_QPS_INIT, - /* Primary port number. */ - .port_num = priv->port - }; - ret = ibv_exp_modify_qp(tmpl.qp, &mod, - (IBV_EXP_QP_STATE | -#ifdef RSS_SUPPORT - (parent ? IBV_EXP_QP_GROUP_RSS : 0) | -#endif /* RSS_SUPPORT */ - IBV_EXP_QP_PORT)); - if (ret) { - ERROR("%p: QP state to IBV_QPS_INIT failed: %s", - (void *)dev, strerror(ret)); - goto error; - } - if ((parent) || (!priv->rss)) { - /* Configure MAC and broadcast addresses. */ - ret = rxq_mac_addrs_add(&tmpl); - if (ret) { - ERROR("%p: QP flow attachment failed: %s", - (void *)dev, strerror(ret)); - goto error; - } - } - /* Allocate descriptors for RX queues, except for the RSS parent. */ - if (parent) - goto skip_alloc; - if (tmpl.sp) - ret = rxq_alloc_elts_sp(&tmpl, desc, NULL); - else - ret = rxq_alloc_elts(&tmpl, desc, NULL); - if (ret) { - ERROR("%p: RXQ allocation failed: %s", - (void *)dev, strerror(ret)); - goto error; - } - ret = ibv_post_recv(tmpl.qp, - (tmpl.sp ? - &(*tmpl.elts.sp)[0].wr : - &(*tmpl.elts.no_sp)[0].wr), - &bad_wr); - if (ret) { - ERROR("%p: ibv_post_recv() failed for WR %p: %s", - (void *)dev, - (void *)bad_wr, - strerror(ret)); - goto error; - } -skip_alloc: - mod = (struct ibv_exp_qp_attr){ - .qp_state = IBV_QPS_RTR - }; - ret = ibv_exp_modify_qp(tmpl.qp, &mod, IBV_EXP_QP_STATE); - if (ret) { - ERROR("%p: QP state to IBV_QPS_RTR failed: %s", - (void *)dev, strerror(ret)); - goto error; - } - /* Save port ID. */ - tmpl.port_id = dev->data->port_id; - DEBUG("%p: RTE port ID: %u", (void *)rxq, tmpl.port_id); - attr.params = (struct ibv_exp_query_intf_params){ - .intf_scope = IBV_EXP_INTF_GLOBAL, - .intf = IBV_EXP_INTF_CQ, - .obj = tmpl.cq, - }; - tmpl.if_cq = ibv_exp_query_intf(priv->ctx, &attr.params, &status); - if (tmpl.if_cq == NULL) { - ERROR("%p: CQ interface family query failed with status %d", - (void *)dev, status); - goto error; - } - attr.params = (struct ibv_exp_query_intf_params){ - .intf_scope = IBV_EXP_INTF_GLOBAL, - .intf = IBV_EXP_INTF_QP_BURST, - .obj = tmpl.qp, - }; - tmpl.if_qp = ibv_exp_query_intf(priv->ctx, &attr.params, &status); - if (tmpl.if_qp == NULL) { - ERROR("%p: QP interface family query failed with status %d", - (void *)dev, status); - goto error; - } - /* Clean up rxq in case we're reinitializing it. */ - DEBUG("%p: cleaning-up old rxq just in case", (void *)rxq); - rxq_cleanup(rxq); - *rxq = tmpl; - DEBUG("%p: rxq updated with %p", (void *)rxq, (void *)&tmpl); - assert(ret == 0); - return 0; -error: - rxq_cleanup(&tmpl); - assert(ret > 0); - return ret; -} - -/** - * DPDK callback to configure a RX queue. - * - * @param dev - * Pointer to Ethernet device structure. - * @param idx - * RX queue index. - * @param desc - * Number of descriptors to configure in queue. - * @param socket - * NUMA socket on which memory must be allocated. - * @param[in] conf - * Thresholds parameters. - * @param mp - * Memory pool for buffer allocations. - * - * @return - * 0 on success, negative errno value on failure. - */ -static int -mlx4_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, - unsigned int socket, const struct rte_eth_rxconf *conf, - struct rte_mempool *mp) -{ - struct priv *priv = dev->data->dev_private; - struct rxq *rxq = (*priv->rxqs)[idx]; - int inactive = 0; - int ret; - - if (mlx4_is_secondary()) - return -E_RTE_SECONDARY; - priv_lock(priv); - DEBUG("%p: configuring queue %u for %u descriptors", - (void *)dev, idx, desc); - if (idx >= priv->rxqs_n) { - ERROR("%p: queue index out of range (%u >= %u)", - (void *)dev, idx, priv->rxqs_n); - priv_unlock(priv); - return -EOVERFLOW; - } - if (rxq != NULL) { - DEBUG("%p: reusing already allocated queue index %u (%p)", - (void *)dev, idx, (void *)rxq); - if (priv->started) { - priv_unlock(priv); - return -EEXIST; - } - (*priv->rxqs)[idx] = NULL; - rxq_cleanup(rxq); - } else { - rxq = rte_calloc_socket("RXQ", 1, sizeof(*rxq), 0, socket); - if (rxq == NULL) { - ERROR("%p: unable to allocate queue index %u", - (void *)dev, idx); - priv_unlock(priv); - return -ENOMEM; - } - } - if (idx >= rte_align32pow2(priv->rxqs_n + 1) >> 1) - inactive = 1; - ret = rxq_setup(dev, rxq, desc, socket, inactive, conf, mp); - if (ret) - rte_free(rxq); - else { - rxq->stats.idx = idx; - DEBUG("%p: adding RX queue %p to list", - (void *)dev, (void *)rxq); - (*priv->rxqs)[idx] = rxq; - /* Update receive callback. */ - if (rxq->sp) - dev->rx_pkt_burst = mlx4_rx_burst_sp; - else - dev->rx_pkt_burst = mlx4_rx_burst; - } - priv_unlock(priv); - return -ret; -} - -/** - * DPDK callback to release a RX queue. - * - * @param dpdk_rxq - * Generic RX queue pointer. - */ -static void -mlx4_rx_queue_release(void *dpdk_rxq) -{ - struct rxq *rxq = (struct rxq *)dpdk_rxq; - struct priv *priv; - unsigned int i; - - if (mlx4_is_secondary()) - return; - if (rxq == NULL) - return; - priv = rxq->priv; - priv_lock(priv); - assert(rxq != &priv->rxq_parent); - for (i = 0; (i != priv->rxqs_n); ++i) - if ((*priv->rxqs)[i] == rxq) { - DEBUG("%p: removing RX queue %p from list", - (void *)priv->dev, (void *)rxq); - (*priv->rxqs)[i] = NULL; - break; - } - rxq_cleanup(rxq); - rte_free(rxq); - priv_unlock(priv); -} - -static void -priv_dev_interrupt_handler_install(struct priv *, struct rte_eth_dev *); - /** * DPDK callback to start the device. * - * Simulate device start by attaching all configured flows. + * Simulate device start by initializing common RSS resources and attaching + * all configured flows. * * @param dev * Pointer to Ethernet device structure. * * @return - * 0 on success, negative errno value on failure. + * 0 on success, negative errno value otherwise and rte_errno is set. */ static int mlx4_dev_start(struct rte_eth_dev *dev) { struct priv *priv = dev->data->dev_private; - unsigned int i = 0; - unsigned int r; - struct rxq *rxq; + struct rte_flow_error error; + int ret; - if (mlx4_is_secondary()) - return -E_RTE_SECONDARY; - priv_lock(priv); - if (priv->started) { - priv_unlock(priv); + if (priv->started) return 0; - } DEBUG("%p: attaching configured flows to all RX queues", (void *)dev); priv->started = 1; - if (priv->rss) { - rxq = &priv->rxq_parent; - r = 1; - } else { - rxq = (*priv->rxqs)[0]; - r = priv->rxqs_n; + ret = mlx4_rss_init(priv); + if (ret) { + ERROR("%p: cannot initialize RSS resources: %s", + (void *)dev, strerror(-ret)); + goto err; } - /* Iterate only once when RSS is enabled. */ - do { - int ret; - - /* Ignore nonexistent RX queues. */ - if (rxq == NULL) - continue; - ret = rxq_mac_addrs_add(rxq); - if (!ret && priv->promisc) - ret = rxq_promiscuous_enable(rxq); - if (!ret && priv->allmulti) - ret = rxq_allmulticast_enable(rxq); - if (!ret) - continue; - WARN("%p: QP flow attachment failed: %s", - (void *)dev, strerror(ret)); - /* Rollback. */ - while (i != 0) { - rxq = (*priv->rxqs)[--i]; - if (rxq != NULL) { - rxq_allmulticast_disable(rxq); - rxq_promiscuous_disable(rxq); - rxq_mac_addrs_del(rxq); - } - } - priv->started = 0; - priv_unlock(priv); - return -ret; - } while ((--r) && ((rxq = (*priv->rxqs)[++i]), i)); - priv_dev_interrupt_handler_install(priv, dev); - priv_unlock(priv); + ret = mlx4_intr_install(priv); + if (ret) { + ERROR("%p: interrupt handler installation failed", + (void *)dev); + goto err; + } + ret = mlx4_flow_sync(priv, &error); + if (ret) { + ERROR("%p: cannot attach flow rules (code %d, \"%s\")," + " flow error type %d, cause %p, message: %s", + (void *)dev, + -ret, strerror(-ret), error.type, error.cause, + error.message ? error.message : "(unspecified)"); + goto err; + } + rte_wmb(); + dev->tx_pkt_burst = mlx4_tx_burst; + dev->rx_pkt_burst = mlx4_rx_burst; return 0; +err: + /* Rollback. */ + priv->started = 0; + return ret; } /** @@ -4175,91 +178,19 @@ static void mlx4_dev_stop(struct rte_eth_dev *dev) { struct priv *priv = dev->data->dev_private; - unsigned int i = 0; - unsigned int r; - struct rxq *rxq; - if (mlx4_is_secondary()) + if (!priv->started) return; - priv_lock(priv); - if (!priv->started) { - priv_unlock(priv); - return; - } DEBUG("%p: detaching flows from all RX queues", (void *)dev); priv->started = 0; - if (priv->rss) { - rxq = &priv->rxq_parent; - r = 1; - } else { - rxq = (*priv->rxqs)[0]; - r = priv->rxqs_n; - } - /* Iterate only once when RSS is enabled. */ - do { - /* Ignore nonexistent RX queues. */ - if (rxq == NULL) - continue; - rxq_allmulticast_disable(rxq); - rxq_promiscuous_disable(rxq); - rxq_mac_addrs_del(rxq); - } while ((--r) && ((rxq = (*priv->rxqs)[++i]), i)); - priv_unlock(priv); + dev->tx_pkt_burst = mlx4_tx_burst_removed; + dev->rx_pkt_burst = mlx4_rx_burst_removed; + rte_wmb(); + mlx4_flow_sync(priv, NULL); + mlx4_intr_uninstall(priv); + mlx4_rss_deinit(priv); } -/** - * Dummy DPDK callback for TX. - * - * This function is used to temporarily replace the real callback during - * unsafe control operations on the queue, or in case of error. - * - * @param dpdk_txq - * Generic pointer to TX queue structure. - * @param[in] pkts - * Packets to transmit. - * @param pkts_n - * Number of packets in array. - * - * @return - * Number of packets successfully transmitted (<= pkts_n). - */ -static uint16_t -removed_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) -{ - (void)dpdk_txq; - (void)pkts; - (void)pkts_n; - return 0; -} - -/** - * Dummy DPDK callback for RX. - * - * This function is used to temporarily replace the real callback during - * unsafe control operations on the queue, or in case of error. - * - * @param dpdk_rxq - * Generic pointer to RX queue structure. - * @param[out] pkts - * Array to store received packets. - * @param pkts_n - * Maximum number of packets in array. - * - * @return - * Number of packets successfully received (<= pkts_n). - */ -static uint16_t -removed_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) -{ - (void)dpdk_rxq; - (void)pkts; - (void)pkts_n; - return 0; -} - -static void -priv_dev_interrupt_handler_uninstall(struct priv *, struct rte_eth_dev *); - /** * DPDK callback to close the device. * @@ -4271,983 +202,60 @@ priv_dev_interrupt_handler_uninstall(struct priv *, struct rte_eth_dev *); static void mlx4_dev_close(struct rte_eth_dev *dev) { - struct priv *priv = mlx4_get_priv(dev); - void *tmp; + struct priv *priv = dev->data->dev_private; unsigned int i; - if (priv == NULL) - return; - priv_lock(priv); DEBUG("%p: closing device \"%s\"", (void *)dev, ((priv->ctx != NULL) ? priv->ctx->device->name : "")); - /* Prevent crashes when queues are still in use. This is unfortunately - * still required for DPDK 1.3 because some programs (such as testpmd) - * never release them before closing the device. */ - dev->rx_pkt_burst = removed_rx_burst; - dev->tx_pkt_burst = removed_tx_burst; - if (priv->rxqs != NULL) { - /* XXX race condition if mlx4_rx_burst() is still running. */ - usleep(1000); - for (i = 0; (i != priv->rxqs_n); ++i) { - tmp = (*priv->rxqs)[i]; - if (tmp == NULL) - continue; - (*priv->rxqs)[i] = NULL; - rxq_cleanup(tmp); - rte_free(tmp); - } - priv->rxqs_n = 0; - priv->rxqs = NULL; - } - if (priv->txqs != NULL) { - /* XXX race condition if mlx4_tx_burst() is still running. */ - usleep(1000); - for (i = 0; (i != priv->txqs_n); ++i) { - tmp = (*priv->txqs)[i]; - if (tmp == NULL) - continue; - (*priv->txqs)[i] = NULL; - txq_cleanup(tmp); - rte_free(tmp); - } - priv->txqs_n = 0; - priv->txqs = NULL; - } - if (priv->rss) - rxq_cleanup(&priv->rxq_parent); + dev->rx_pkt_burst = mlx4_rx_burst_removed; + dev->tx_pkt_burst = mlx4_tx_burst_removed; + rte_wmb(); + mlx4_flow_clean(priv); + for (i = 0; i != dev->data->nb_rx_queues; ++i) + mlx4_rx_queue_release(dev->data->rx_queues[i]); + for (i = 0; i != dev->data->nb_tx_queues; ++i) + mlx4_tx_queue_release(dev->data->tx_queues[i]); if (priv->pd != NULL) { assert(priv->ctx != NULL); claim_zero(ibv_dealloc_pd(priv->pd)); claim_zero(ibv_close_device(priv->ctx)); } else assert(priv->ctx == NULL); - priv_dev_interrupt_handler_uninstall(priv, dev); - priv_unlock(priv); + mlx4_intr_uninstall(priv); memset(priv, 0, sizeof(*priv)); } -/** - * Change the link state (UP / DOWN). - * - * @param priv - * Pointer to Ethernet device private data. - * @param up - * Nonzero for link up, otherwise link down. - * - * @return - * 0 on success, errno value on failure. - */ -static int -priv_set_link(struct priv *priv, int up) -{ - struct rte_eth_dev *dev = priv->dev; - int err; - unsigned int i; - - if (up) { - err = priv_set_flags(priv, ~IFF_UP, IFF_UP); - if (err) - return err; - for (i = 0; i < priv->rxqs_n; i++) - if ((*priv->rxqs)[i]->sp) - break; - /* Check if an sp queue exists. - * Note: Some old frames might be received. - */ - if (i == priv->rxqs_n) - dev->rx_pkt_burst = mlx4_rx_burst; - else - dev->rx_pkt_burst = mlx4_rx_burst_sp; - dev->tx_pkt_burst = mlx4_tx_burst; - } else { - err = priv_set_flags(priv, ~IFF_UP, ~IFF_UP); - if (err) - return err; - dev->rx_pkt_burst = removed_rx_burst; - dev->tx_pkt_burst = removed_tx_burst; - } - return 0; -} - -/** - * DPDK callback to bring the link DOWN. - * - * @param dev - * Pointer to Ethernet device structure. - * - * @return - * 0 on success, errno value on failure. - */ -static int -mlx4_set_link_down(struct rte_eth_dev *dev) -{ - struct priv *priv = dev->data->dev_private; - int err; - - priv_lock(priv); - err = priv_set_link(priv, 0); - priv_unlock(priv); - return err; -} - -/** - * DPDK callback to bring the link UP. - * - * @param dev - * Pointer to Ethernet device structure. - * - * @return - * 0 on success, errno value on failure. - */ -static int -mlx4_set_link_up(struct rte_eth_dev *dev) -{ - struct priv *priv = dev->data->dev_private; - int err; - - priv_lock(priv); - err = priv_set_link(priv, 1); - priv_unlock(priv); - return err; -} -/** - * DPDK callback to get information about the device. - * - * @param dev - * Pointer to Ethernet device structure. - * @param[out] info - * Info structure output buffer. - */ -static void -mlx4_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info) -{ - struct priv *priv = mlx4_get_priv(dev); - unsigned int max; - char ifname[IF_NAMESIZE]; - - if (priv == NULL) - return; - priv_lock(priv); - /* FIXME: we should ask the device for these values. */ - info->min_rx_bufsize = 32; - info->max_rx_pktlen = 65536; - /* - * Since we need one CQ per QP, the limit is the minimum number - * between the two values. - */ - max = ((priv->device_attr.max_cq > priv->device_attr.max_qp) ? - priv->device_attr.max_qp : priv->device_attr.max_cq); - /* If max >= 65535 then max = 0, max_rx_queues is uint16_t. */ - if (max >= 65535) - max = 65535; - info->max_rx_queues = max; - info->max_tx_queues = max; - /* Last array entry is reserved for broadcast. */ - info->max_mac_addrs = (elemof(priv->mac) - 1); - info->rx_offload_capa = - (priv->hw_csum ? - (DEV_RX_OFFLOAD_IPV4_CKSUM | - DEV_RX_OFFLOAD_UDP_CKSUM | - DEV_RX_OFFLOAD_TCP_CKSUM) : - 0); - info->tx_offload_capa = - (priv->hw_csum ? - (DEV_TX_OFFLOAD_IPV4_CKSUM | - DEV_TX_OFFLOAD_UDP_CKSUM | - DEV_TX_OFFLOAD_TCP_CKSUM) : - 0); - if (priv_get_ifname(priv, &ifname) == 0) - info->if_index = if_nametoindex(ifname); - info->speed_capa = - ETH_LINK_SPEED_1G | - ETH_LINK_SPEED_10G | - ETH_LINK_SPEED_20G | - ETH_LINK_SPEED_40G | - ETH_LINK_SPEED_56G; - priv_unlock(priv); -} - -static const uint32_t * -mlx4_dev_supported_ptypes_get(struct rte_eth_dev *dev) -{ - static const uint32_t ptypes[] = { - /* refers to rxq_cq_to_pkt_type() */ - RTE_PTYPE_L3_IPV4, - RTE_PTYPE_L3_IPV6, - RTE_PTYPE_INNER_L3_IPV4, - RTE_PTYPE_INNER_L3_IPV6, - RTE_PTYPE_UNKNOWN - }; - - if (dev->rx_pkt_burst == mlx4_rx_burst || - dev->rx_pkt_burst == mlx4_rx_burst_sp) - return ptypes; - return NULL; -} - -/** - * DPDK callback to get device statistics. - * - * @param dev - * Pointer to Ethernet device structure. - * @param[out] stats - * Stats structure output buffer. - */ -static void -mlx4_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) -{ - struct priv *priv = mlx4_get_priv(dev); - struct rte_eth_stats tmp = {0}; - unsigned int i; - unsigned int idx; - - if (priv == NULL) - return; - priv_lock(priv); - /* Add software counters. */ - for (i = 0; (i != priv->rxqs_n); ++i) { - struct rxq *rxq = (*priv->rxqs)[i]; - - if (rxq == NULL) - continue; - idx = rxq->stats.idx; - if (idx < RTE_ETHDEV_QUEUE_STAT_CNTRS) { -#ifdef MLX4_PMD_SOFT_COUNTERS - tmp.q_ipackets[idx] += rxq->stats.ipackets; - tmp.q_ibytes[idx] += rxq->stats.ibytes; -#endif - tmp.q_errors[idx] += (rxq->stats.idropped + - rxq->stats.rx_nombuf); - } -#ifdef MLX4_PMD_SOFT_COUNTERS - tmp.ipackets += rxq->stats.ipackets; - tmp.ibytes += rxq->stats.ibytes; -#endif - tmp.ierrors += rxq->stats.idropped; - tmp.rx_nombuf += rxq->stats.rx_nombuf; - } - for (i = 0; (i != priv->txqs_n); ++i) { - struct txq *txq = (*priv->txqs)[i]; - - if (txq == NULL) - continue; - idx = txq->stats.idx; - if (idx < RTE_ETHDEV_QUEUE_STAT_CNTRS) { -#ifdef MLX4_PMD_SOFT_COUNTERS - tmp.q_opackets[idx] += txq->stats.opackets; - tmp.q_obytes[idx] += txq->stats.obytes; -#endif - tmp.q_errors[idx] += txq->stats.odropped; - } -#ifdef MLX4_PMD_SOFT_COUNTERS - tmp.opackets += txq->stats.opackets; - tmp.obytes += txq->stats.obytes; -#endif - tmp.oerrors += txq->stats.odropped; - } -#ifndef MLX4_PMD_SOFT_COUNTERS - /* FIXME: retrieve and add hardware counters. */ -#endif - *stats = tmp; - priv_unlock(priv); -} - -/** - * DPDK callback to clear device statistics. - * - * @param dev - * Pointer to Ethernet device structure. - */ -static void -mlx4_stats_reset(struct rte_eth_dev *dev) -{ - struct priv *priv = mlx4_get_priv(dev); - unsigned int i; - unsigned int idx; - - if (priv == NULL) - return; - priv_lock(priv); - for (i = 0; (i != priv->rxqs_n); ++i) { - if ((*priv->rxqs)[i] == NULL) - continue; - idx = (*priv->rxqs)[i]->stats.idx; - (*priv->rxqs)[i]->stats = - (struct mlx4_rxq_stats){ .idx = idx }; - } - for (i = 0; (i != priv->txqs_n); ++i) { - if ((*priv->txqs)[i] == NULL) - continue; - idx = (*priv->txqs)[i]->stats.idx; - (*priv->txqs)[i]->stats = - (struct mlx4_txq_stats){ .idx = idx }; - } -#ifndef MLX4_PMD_SOFT_COUNTERS - /* FIXME: reset hardware counters. */ -#endif - priv_unlock(priv); -} - -/** - * DPDK callback to remove a MAC address. - * - * @param dev - * Pointer to Ethernet device structure. - * @param index - * MAC address index. - */ -static void -mlx4_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index) -{ - struct priv *priv = dev->data->dev_private; - - if (mlx4_is_secondary()) - return; - priv_lock(priv); - DEBUG("%p: removing MAC address from index %" PRIu32, - (void *)dev, index); - /* Last array entry is reserved for broadcast. */ - if (index >= (elemof(priv->mac) - 1)) - goto end; - priv_mac_addr_del(priv, index); -end: - priv_unlock(priv); -} - -/** - * DPDK callback to add a MAC address. - * - * @param dev - * Pointer to Ethernet device structure. - * @param mac_addr - * MAC address to register. - * @param index - * MAC address index. - * @param vmdq - * VMDq pool index to associate address with (ignored). - */ -static void -mlx4_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac_addr, - uint32_t index, uint32_t vmdq) -{ - struct priv *priv = dev->data->dev_private; - - if (mlx4_is_secondary()) - return; - (void)vmdq; - priv_lock(priv); - DEBUG("%p: adding MAC address at index %" PRIu32, - (void *)dev, index); - /* Last array entry is reserved for broadcast. */ - if (index >= (elemof(priv->mac) - 1)) - goto end; - priv_mac_addr_add(priv, index, - (const uint8_t (*)[ETHER_ADDR_LEN]) - mac_addr->addr_bytes); -end: - priv_unlock(priv); -} - -/** - * DPDK callback to set the primary MAC address. - * - * @param dev - * Pointer to Ethernet device structure. - * @param mac_addr - * MAC address to register. - */ -static void -mlx4_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr) -{ - DEBUG("%p: setting primary MAC address", (void *)dev); - mlx4_mac_addr_remove(dev, 0); - mlx4_mac_addr_add(dev, mac_addr, 0, 0); -} - -/** - * DPDK callback to enable promiscuous mode. - * - * @param dev - * Pointer to Ethernet device structure. - */ -static void -mlx4_promiscuous_enable(struct rte_eth_dev *dev) -{ - struct priv *priv = dev->data->dev_private; - unsigned int i; - int ret; - - if (mlx4_is_secondary()) - return; - priv_lock(priv); - if (priv->promisc) { - priv_unlock(priv); - return; - } - /* If device isn't started, this is all we need to do. */ - if (!priv->started) - goto end; - if (priv->rss) { - ret = rxq_promiscuous_enable(&priv->rxq_parent); - if (ret) { - priv_unlock(priv); - return; - } - goto end; - } - for (i = 0; (i != priv->rxqs_n); ++i) { - if ((*priv->rxqs)[i] == NULL) - continue; - ret = rxq_promiscuous_enable((*priv->rxqs)[i]); - if (!ret) - continue; - /* Failure, rollback. */ - while (i != 0) - if ((*priv->rxqs)[--i] != NULL) - rxq_promiscuous_disable((*priv->rxqs)[i]); - priv_unlock(priv); - return; - } -end: - priv->promisc = 1; - priv_unlock(priv); -} - -/** - * DPDK callback to disable promiscuous mode. - * - * @param dev - * Pointer to Ethernet device structure. - */ -static void -mlx4_promiscuous_disable(struct rte_eth_dev *dev) -{ - struct priv *priv = dev->data->dev_private; - unsigned int i; - - if (mlx4_is_secondary()) - return; - priv_lock(priv); - if (!priv->promisc) { - priv_unlock(priv); - return; - } - if (priv->rss) { - rxq_promiscuous_disable(&priv->rxq_parent); - goto end; - } - for (i = 0; (i != priv->rxqs_n); ++i) - if ((*priv->rxqs)[i] != NULL) - rxq_promiscuous_disable((*priv->rxqs)[i]); -end: - priv->promisc = 0; - priv_unlock(priv); -} - -/** - * DPDK callback to enable allmulti mode. - * - * @param dev - * Pointer to Ethernet device structure. - */ -static void -mlx4_allmulticast_enable(struct rte_eth_dev *dev) -{ - struct priv *priv = dev->data->dev_private; - unsigned int i; - int ret; - - if (mlx4_is_secondary()) - return; - priv_lock(priv); - if (priv->allmulti) { - priv_unlock(priv); - return; - } - /* If device isn't started, this is all we need to do. */ - if (!priv->started) - goto end; - if (priv->rss) { - ret = rxq_allmulticast_enable(&priv->rxq_parent); - if (ret) { - priv_unlock(priv); - return; - } - goto end; - } - for (i = 0; (i != priv->rxqs_n); ++i) { - if ((*priv->rxqs)[i] == NULL) - continue; - ret = rxq_allmulticast_enable((*priv->rxqs)[i]); - if (!ret) - continue; - /* Failure, rollback. */ - while (i != 0) - if ((*priv->rxqs)[--i] != NULL) - rxq_allmulticast_disable((*priv->rxqs)[i]); - priv_unlock(priv); - return; - } -end: - priv->allmulti = 1; - priv_unlock(priv); -} - -/** - * DPDK callback to disable allmulti mode. - * - * @param dev - * Pointer to Ethernet device structure. - */ -static void -mlx4_allmulticast_disable(struct rte_eth_dev *dev) -{ - struct priv *priv = dev->data->dev_private; - unsigned int i; - - if (mlx4_is_secondary()) - return; - priv_lock(priv); - if (!priv->allmulti) { - priv_unlock(priv); - return; - } - if (priv->rss) { - rxq_allmulticast_disable(&priv->rxq_parent); - goto end; - } - for (i = 0; (i != priv->rxqs_n); ++i) - if ((*priv->rxqs)[i] != NULL) - rxq_allmulticast_disable((*priv->rxqs)[i]); -end: - priv->allmulti = 0; - priv_unlock(priv); -} - -/** - * DPDK callback to retrieve physical link information (unlocked version). - * - * @param dev - * Pointer to Ethernet device structure. - * @param wait_to_complete - * Wait for request completion (ignored). - */ -static int -mlx4_link_update_unlocked(struct rte_eth_dev *dev, int wait_to_complete) -{ - struct priv *priv = mlx4_get_priv(dev); - struct ethtool_cmd edata = { - .cmd = ETHTOOL_GSET - }; - struct ifreq ifr; - struct rte_eth_link dev_link; - int link_speed = 0; - - if (priv == NULL) - return -EINVAL; - (void)wait_to_complete; - if (priv_ifreq(priv, SIOCGIFFLAGS, &ifr)) { - WARN("ioctl(SIOCGIFFLAGS) failed: %s", strerror(errno)); - return -1; - } - memset(&dev_link, 0, sizeof(dev_link)); - dev_link.link_status = ((ifr.ifr_flags & IFF_UP) && - (ifr.ifr_flags & IFF_RUNNING)); - ifr.ifr_data = (void *)&edata; - if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { - WARN("ioctl(SIOCETHTOOL, ETHTOOL_GSET) failed: %s", - strerror(errno)); - return -1; - } - link_speed = ethtool_cmd_speed(&edata); - if (link_speed == -1) - dev_link.link_speed = 0; - else - dev_link.link_speed = link_speed; - dev_link.link_duplex = ((edata.duplex == DUPLEX_HALF) ? - ETH_LINK_HALF_DUPLEX : ETH_LINK_FULL_DUPLEX); - dev_link.link_autoneg = !(dev->data->dev_conf.link_speeds & - ETH_LINK_SPEED_FIXED); - if (memcmp(&dev_link, &dev->data->dev_link, sizeof(dev_link))) { - /* Link status changed. */ - dev->data->dev_link = dev_link; - return 0; - } - /* Link status is still the same. */ - return -1; -} - -/** - * DPDK callback to retrieve physical link information. - * - * @param dev - * Pointer to Ethernet device structure. - * @param wait_to_complete - * Wait for request completion (ignored). - */ -static int -mlx4_link_update(struct rte_eth_dev *dev, int wait_to_complete) -{ - struct priv *priv = mlx4_get_priv(dev); - int ret; - - if (priv == NULL) - return -EINVAL; - priv_lock(priv); - ret = mlx4_link_update_unlocked(dev, wait_to_complete); - priv_unlock(priv); - return ret; -} - -/** - * DPDK callback to change the MTU. - * - * Setting the MTU affects hardware MRU (packets larger than the MTU cannot be - * received). Use this as a hint to enable/disable scattered packets support - * and improve performance when not needed. - * Since failure is not an option, reconfiguring queues on the fly is not - * recommended. - * - * @param dev - * Pointer to Ethernet device structure. - * @param in_mtu - * New MTU. - * - * @return - * 0 on success, negative errno value on failure. - */ -static int -mlx4_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu) -{ - struct priv *priv = dev->data->dev_private; - int ret = 0; - unsigned int i; - uint16_t (*rx_func)(void *, struct rte_mbuf **, uint16_t) = - mlx4_rx_burst; - - if (mlx4_is_secondary()) - return -E_RTE_SECONDARY; - priv_lock(priv); - /* Set kernel interface MTU first. */ - if (priv_set_mtu(priv, mtu)) { - ret = errno; - WARN("cannot set port %u MTU to %u: %s", priv->port, mtu, - strerror(ret)); - goto out; - } else - DEBUG("adapter port %u MTU set to %u", priv->port, mtu); - priv->mtu = mtu; - /* Temporarily replace RX handler with a fake one, assuming it has not - * been copied elsewhere. */ - dev->rx_pkt_burst = removed_rx_burst; - /* Make sure everyone has left mlx4_rx_burst() and uses - * removed_rx_burst() instead. */ - rte_wmb(); - usleep(1000); - /* Reconfigure each RX queue. */ - for (i = 0; (i != priv->rxqs_n); ++i) { - struct rxq *rxq = (*priv->rxqs)[i]; - unsigned int mb_len; - unsigned int max_frame_len; - int sp; - - if (rxq == NULL) - continue; - /* Calculate new maximum frame length according to MTU and - * toggle scattered support (sp) if necessary. */ - max_frame_len = (priv->mtu + ETHER_HDR_LEN + - (ETHER_MAX_VLAN_FRAME_LEN - ETHER_MAX_LEN)); - mb_len = rte_pktmbuf_data_room_size(rxq->mp); - assert(mb_len >= RTE_PKTMBUF_HEADROOM); - sp = (max_frame_len > (mb_len - RTE_PKTMBUF_HEADROOM)); - /* Provide new values to rxq_setup(). */ - dev->data->dev_conf.rxmode.jumbo_frame = sp; - dev->data->dev_conf.rxmode.max_rx_pkt_len = max_frame_len; - ret = rxq_rehash(dev, rxq); - if (ret) { - /* Force SP RX if that queue requires it and abort. */ - if (rxq->sp) - rx_func = mlx4_rx_burst_sp; - break; - } - /* Reenable non-RSS queue attributes. No need to check - * for errors at this stage. */ - if (!priv->rss) { - rxq_mac_addrs_add(rxq); - if (priv->promisc) - rxq_promiscuous_enable(rxq); - if (priv->allmulti) - rxq_allmulticast_enable(rxq); - } - /* Scattered burst function takes priority. */ - if (rxq->sp) - rx_func = mlx4_rx_burst_sp; - } - /* Burst functions can now be called again. */ - rte_wmb(); - dev->rx_pkt_burst = rx_func; -out: - priv_unlock(priv); - assert(ret >= 0); - return -ret; -} - -/** - * DPDK callback to get flow control status. - * - * @param dev - * Pointer to Ethernet device structure. - * @param[out] fc_conf - * Flow control output buffer. - * - * @return - * 0 on success, negative errno value on failure. - */ -static int -mlx4_dev_get_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) -{ - struct priv *priv = dev->data->dev_private; - struct ifreq ifr; - struct ethtool_pauseparam ethpause = { - .cmd = ETHTOOL_GPAUSEPARAM - }; - int ret; - - if (mlx4_is_secondary()) - return -E_RTE_SECONDARY; - ifr.ifr_data = (void *)ðpause; - priv_lock(priv); - if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { - ret = errno; - WARN("ioctl(SIOCETHTOOL, ETHTOOL_GPAUSEPARAM)" - " failed: %s", - strerror(ret)); - goto out; - } - - fc_conf->autoneg = ethpause.autoneg; - if (ethpause.rx_pause && ethpause.tx_pause) - fc_conf->mode = RTE_FC_FULL; - else if (ethpause.rx_pause) - fc_conf->mode = RTE_FC_RX_PAUSE; - else if (ethpause.tx_pause) - fc_conf->mode = RTE_FC_TX_PAUSE; - else - fc_conf->mode = RTE_FC_NONE; - ret = 0; - -out: - priv_unlock(priv); - assert(ret >= 0); - return -ret; -} - -/** - * DPDK callback to modify flow control parameters. - * - * @param dev - * Pointer to Ethernet device structure. - * @param[in] fc_conf - * Flow control parameters. - * - * @return - * 0 on success, negative errno value on failure. - */ -static int -mlx4_dev_set_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) -{ - struct priv *priv = dev->data->dev_private; - struct ifreq ifr; - struct ethtool_pauseparam ethpause = { - .cmd = ETHTOOL_SPAUSEPARAM - }; - int ret; - - if (mlx4_is_secondary()) - return -E_RTE_SECONDARY; - ifr.ifr_data = (void *)ðpause; - ethpause.autoneg = fc_conf->autoneg; - if (((fc_conf->mode & RTE_FC_FULL) == RTE_FC_FULL) || - (fc_conf->mode & RTE_FC_RX_PAUSE)) - ethpause.rx_pause = 1; - else - ethpause.rx_pause = 0; - - if (((fc_conf->mode & RTE_FC_FULL) == RTE_FC_FULL) || - (fc_conf->mode & RTE_FC_TX_PAUSE)) - ethpause.tx_pause = 1; - else - ethpause.tx_pause = 0; - - priv_lock(priv); - if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { - ret = errno; - WARN("ioctl(SIOCETHTOOL, ETHTOOL_SPAUSEPARAM)" - " failed: %s", - strerror(ret)); - goto out; - } - ret = 0; - -out: - priv_unlock(priv); - assert(ret >= 0); - return -ret; -} - -/** - * Configure a VLAN filter. - * - * @param dev - * Pointer to Ethernet device structure. - * @param vlan_id - * VLAN ID to filter. - * @param on - * Toggle filter. - * - * @return - * 0 on success, errno value on failure. - */ -static int -vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) -{ - struct priv *priv = dev->data->dev_private; - unsigned int i; - unsigned int j = -1; - - DEBUG("%p: %s VLAN filter ID %" PRIu16, - (void *)dev, (on ? "enable" : "disable"), vlan_id); - for (i = 0; (i != elemof(priv->vlan_filter)); ++i) { - if (!priv->vlan_filter[i].enabled) { - /* Unused index, remember it. */ - j = i; - continue; - } - if (priv->vlan_filter[i].id != vlan_id) - continue; - /* This VLAN ID is already known, use its index. */ - j = i; - break; - } - /* Check if there's room for another VLAN filter. */ - if (j == (unsigned int)-1) - return ENOMEM; - /* - * VLAN filters apply to all configured MAC addresses, flow - * specifications must be reconfigured accordingly. - */ - priv->vlan_filter[j].id = vlan_id; - if ((on) && (!priv->vlan_filter[j].enabled)) { - /* - * Filter is disabled, enable it. - * Rehashing flows in all RX queues is necessary. - */ - if (priv->rss) - rxq_mac_addrs_del(&priv->rxq_parent); - else - for (i = 0; (i != priv->rxqs_n); ++i) - if ((*priv->rxqs)[i] != NULL) - rxq_mac_addrs_del((*priv->rxqs)[i]); - priv->vlan_filter[j].enabled = 1; - if (priv->started) { - if (priv->rss) - rxq_mac_addrs_add(&priv->rxq_parent); - else - for (i = 0; (i != priv->rxqs_n); ++i) { - if ((*priv->rxqs)[i] == NULL) - continue; - rxq_mac_addrs_add((*priv->rxqs)[i]); - } - } - } else if ((!on) && (priv->vlan_filter[j].enabled)) { - /* - * Filter is enabled, disable it. - * Rehashing flows in all RX queues is necessary. - */ - if (priv->rss) - rxq_mac_addrs_del(&priv->rxq_parent); - else - for (i = 0; (i != priv->rxqs_n); ++i) - if ((*priv->rxqs)[i] != NULL) - rxq_mac_addrs_del((*priv->rxqs)[i]); - priv->vlan_filter[j].enabled = 0; - if (priv->started) { - if (priv->rss) - rxq_mac_addrs_add(&priv->rxq_parent); - else - for (i = 0; (i != priv->rxqs_n); ++i) { - if ((*priv->rxqs)[i] == NULL) - continue; - rxq_mac_addrs_add((*priv->rxqs)[i]); - } - } - } - return 0; -} - -/** - * DPDK callback to configure a VLAN filter. - * - * @param dev - * Pointer to Ethernet device structure. - * @param vlan_id - * VLAN ID to filter. - * @param on - * Toggle filter. - * - * @return - * 0 on success, negative errno value on failure. - */ -static int -mlx4_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) -{ - struct priv *priv = dev->data->dev_private; - int ret; - - if (mlx4_is_secondary()) - return -E_RTE_SECONDARY; - priv_lock(priv); - ret = vlan_filter_set(dev, vlan_id, on); - priv_unlock(priv); - assert(ret >= 0); - return -ret; -} - static const struct eth_dev_ops mlx4_dev_ops = { .dev_configure = mlx4_dev_configure, .dev_start = mlx4_dev_start, .dev_stop = mlx4_dev_stop, - .dev_set_link_down = mlx4_set_link_down, - .dev_set_link_up = mlx4_set_link_up, + .dev_set_link_down = mlx4_dev_set_link_down, + .dev_set_link_up = mlx4_dev_set_link_up, .dev_close = mlx4_dev_close, + .link_update = mlx4_link_update, .promiscuous_enable = mlx4_promiscuous_enable, .promiscuous_disable = mlx4_promiscuous_disable, .allmulticast_enable = mlx4_allmulticast_enable, .allmulticast_disable = mlx4_allmulticast_disable, - .link_update = mlx4_link_update, + .mac_addr_remove = mlx4_mac_addr_remove, + .mac_addr_add = mlx4_mac_addr_add, + .mac_addr_set = mlx4_mac_addr_set, .stats_get = mlx4_stats_get, .stats_reset = mlx4_stats_reset, - .queue_stats_mapping_set = NULL, .dev_infos_get = mlx4_dev_infos_get, .dev_supported_ptypes_get = mlx4_dev_supported_ptypes_get, .vlan_filter_set = mlx4_vlan_filter_set, - .vlan_tpid_set = NULL, - .vlan_strip_queue_set = NULL, - .vlan_offload_set = NULL, .rx_queue_setup = mlx4_rx_queue_setup, .tx_queue_setup = mlx4_tx_queue_setup, .rx_queue_release = mlx4_rx_queue_release, .tx_queue_release = mlx4_tx_queue_release, - .dev_led_on = NULL, - .dev_led_off = NULL, - .flow_ctrl_get = mlx4_dev_get_flow_ctrl, - .flow_ctrl_set = mlx4_dev_set_flow_ctrl, - .priority_flow_ctrl_set = NULL, - .mac_addr_remove = mlx4_mac_addr_remove, - .mac_addr_add = mlx4_mac_addr_add, - .mac_addr_set = mlx4_mac_addr_set, - .mtu_set = mlx4_dev_set_mtu, + .flow_ctrl_get = mlx4_flow_ctrl_get, + .flow_ctrl_set = mlx4_flow_ctrl_set, + .mtu_set = mlx4_mtu_set, + .filter_ctrl = mlx4_filter_ctrl, + .rx_queue_intr_enable = mlx4_rx_intr_enable, + .rx_queue_intr_disable = mlx4_rx_intr_disable, }; /** @@ -5259,7 +267,7 @@ static const struct eth_dev_ops mlx4_dev_ops = { * PCI bus address output buffer. * * @return - * 0 on success, -1 on failure and errno is set. + * 0 on success, negative errno value otherwise and rte_errno is set. */ static int mlx4_ibv_device_to_pci_addr(const struct ibv_device *device, @@ -5270,8 +278,10 @@ mlx4_ibv_device_to_pci_addr(const struct ibv_device *device, MKSTR(path, "%s/device/uevent", device->ibdev_path); file = fopen(path, "rb"); - if (file == NULL) - return -1; + if (file == NULL) { + rte_errno = errno; + return -rte_errno; + } while (fgets(line, sizeof(line), file) == line) { size_t len = strlen(line); int ret; @@ -5287,7 +297,7 @@ mlx4_ibv_device_to_pci_addr(const struct ibv_device *device, /* Extract information. */ if (sscanf(line, "PCI_SLOT_NAME=" - "%" SCNx16 ":%" SCNx8 ":%" SCNx8 ".%" SCNx8 "\n", + "%" SCNx32 ":%" SCNx8 ":%" SCNx8 ".%" SCNx8 "\n", &pci_addr->domain, &pci_addr->bus, &pci_addr->devid, @@ -5301,233 +311,98 @@ mlx4_ibv_device_to_pci_addr(const struct ibv_device *device, } /** - * Get MAC address by querying netdevice. + * Verify and store value for device argument. * - * @param[in] priv - * struct priv for the requested device. - * @param[out] mac - * MAC address output buffer. + * @param[in] key + * Key argument to verify. + * @param[in] val + * Value associated with key. + * @param[in, out] conf + * Shared configuration data. * * @return - * 0 on success, -1 on failure and errno is set. + * 0 on success, negative errno value otherwise and rte_errno is set. */ static int -priv_get_mac(struct priv *priv, uint8_t (*mac)[ETHER_ADDR_LEN]) +mlx4_arg_parse(const char *key, const char *val, struct mlx4_conf *conf) { - struct ifreq request; + unsigned long tmp; - if (priv_ifreq(priv, SIOCGIFHWADDR, &request)) - return -1; - memcpy(mac, request.ifr_hwaddr.sa_data, ETHER_ADDR_LEN); + errno = 0; + tmp = strtoul(val, NULL, 0); + if (errno) { + rte_errno = errno; + WARN("%s: \"%s\" is not a valid integer", key, val); + return -rte_errno; + } + if (strcmp(MLX4_PMD_PORT_KVARG, key) == 0) { + uint32_t ports = rte_log2_u32(conf->ports.present); + + if (tmp >= ports) { + ERROR("port index %lu outside range [0,%" PRIu32 ")", + tmp, ports); + return -EINVAL; + } + if (!(conf->ports.present & (1 << tmp))) { + rte_errno = EINVAL; + ERROR("invalid port index %lu", tmp); + return -rte_errno; + } + conf->ports.enabled |= 1 << tmp; + } else { + rte_errno = EINVAL; + WARN("%s: unknown parameter", key); + return -rte_errno; + } return 0; } -/* Support up to 32 adapters. */ -static struct { - struct rte_pci_addr pci_addr; /* associated PCI address */ - uint32_t ports; /* physical ports bitfield. */ -} mlx4_dev[32]; - /** - * Get device index in mlx4_dev[] from PCI bus address. + * Parse device parameters. * - * @param[in] pci_addr - * PCI bus address to look for. + * @param devargs + * Device arguments structure. * * @return - * mlx4_dev[] index on success, -1 on failure. + * 0 on success, negative errno value otherwise and rte_errno is set. */ static int -mlx4_dev_idx(struct rte_pci_addr *pci_addr) +mlx4_args(struct rte_devargs *devargs, struct mlx4_conf *conf) { - unsigned int i; - int ret = -1; - - assert(pci_addr != NULL); - for (i = 0; (i != elemof(mlx4_dev)); ++i) { - if ((mlx4_dev[i].pci_addr.domain == pci_addr->domain) && - (mlx4_dev[i].pci_addr.bus == pci_addr->bus) && - (mlx4_dev[i].pci_addr.devid == pci_addr->devid) && - (mlx4_dev[i].pci_addr.function == pci_addr->function)) - return i; - if ((mlx4_dev[i].ports == 0) && (ret == -1)) - ret = i; - } - return ret; -} - -/** - * Retrieve integer value from environment variable. - * - * @param[in] name - * Environment variable name. - * - * @return - * Integer value, 0 if the variable is not set. - */ -static int -mlx4_getenv_int(const char *name) -{ - const char *val = getenv(name); - - if (val == NULL) - return 0; - return atoi(val); -} - -static void -mlx4_dev_link_status_handler(void *); -static void -mlx4_dev_interrupt_handler(struct rte_intr_handle *, void *); - -/** - * Link status handler. - * - * @param priv - * Pointer to private structure. - * @param dev - * Pointer to the rte_eth_dev structure. - * - * @return - * Nonzero if the callback process can be called immediately. - */ -static int -priv_dev_link_status_handler(struct priv *priv, struct rte_eth_dev *dev) -{ - struct ibv_async_event event; - int port_change = 0; + struct rte_kvargs *kvlist; + unsigned int arg_count; int ret = 0; + int i; - /* Read all message and acknowledge them. */ - for (;;) { - if (ibv_get_async_event(priv->ctx, &event)) - break; - - if (event.event_type == IBV_EVENT_PORT_ACTIVE || - event.event_type == IBV_EVENT_PORT_ERR) - port_change = 1; - else - DEBUG("event type %d on port %d not handled", - event.event_type, event.element.port_num); - ibv_ack_async_event(&event); + if (devargs == NULL) + return 0; + kvlist = rte_kvargs_parse(devargs->args, pmd_mlx4_init_params); + if (kvlist == NULL) { + rte_errno = EINVAL; + ERROR("failed to parse kvargs"); + return -rte_errno; } - - if (port_change ^ priv->pending_alarm) { - struct rte_eth_link *link = &dev->data->dev_link; - - priv->pending_alarm = 0; - mlx4_link_update_unlocked(dev, 0); - if (((link->link_speed == 0) && link->link_status) || - ((link->link_speed != 0) && !link->link_status)) { - /* Inconsistent status, check again later. */ - priv->pending_alarm = 1; - rte_eal_alarm_set(MLX4_ALARM_TIMEOUT_US, - mlx4_dev_link_status_handler, - dev); - } else - ret = 1; + /* Process parameters. */ + for (i = 0; pmd_mlx4_init_params[i]; ++i) { + arg_count = rte_kvargs_count(kvlist, MLX4_PMD_PORT_KVARG); + while (arg_count-- > 0) { + ret = rte_kvargs_process(kvlist, + MLX4_PMD_PORT_KVARG, + (int (*)(const char *, + const char *, + void *)) + mlx4_arg_parse, + conf); + if (ret != 0) + goto free_kvlist; + } } +free_kvlist: + rte_kvargs_free(kvlist); return ret; } -/** - * Handle delayed link status event. - * - * @param arg - * Registered argument. - */ -static void -mlx4_dev_link_status_handler(void *arg) -{ - struct rte_eth_dev *dev = arg; - struct priv *priv = dev->data->dev_private; - int ret; - - priv_lock(priv); - assert(priv->pending_alarm == 1); - ret = priv_dev_link_status_handler(priv, dev); - priv_unlock(priv); - if (ret) - _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC); -} - -/** - * Handle interrupts from the NIC. - * - * @param[in] intr_handle - * Interrupt handler. - * @param cb_arg - * Callback argument. - */ -static void -mlx4_dev_interrupt_handler(struct rte_intr_handle *intr_handle, void *cb_arg) -{ - struct rte_eth_dev *dev = cb_arg; - struct priv *priv = dev->data->dev_private; - int ret; - - (void)intr_handle; - priv_lock(priv); - ret = priv_dev_link_status_handler(priv, dev); - priv_unlock(priv); - if (ret) - _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC); -} - -/** - * Uninstall interrupt handler. - * - * @param priv - * Pointer to private structure. - * @param dev - * Pointer to the rte_eth_dev structure. - */ -static void -priv_dev_interrupt_handler_uninstall(struct priv *priv, struct rte_eth_dev *dev) -{ - if (!dev->data->dev_conf.intr_conf.lsc) - return; - rte_intr_callback_unregister(&priv->intr_handle, - mlx4_dev_interrupt_handler, - dev); - if (priv->pending_alarm) - rte_eal_alarm_cancel(mlx4_dev_link_status_handler, dev); - priv->pending_alarm = 0; - priv->intr_handle.fd = 0; - priv->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN; -} - -/** - * Install interrupt handler. - * - * @param priv - * Pointer to private structure. - * @param dev - * Pointer to the rte_eth_dev structure. - */ -static void -priv_dev_interrupt_handler_install(struct priv *priv, struct rte_eth_dev *dev) -{ - int rc, flags; - - if (!dev->data->dev_conf.intr_conf.lsc) - return; - assert(priv->ctx->async_fd > 0); - flags = fcntl(priv->ctx->async_fd, F_GETFL); - rc = fcntl(priv->ctx->async_fd, F_SETFL, flags | O_NONBLOCK); - if (rc < 0) { - INFO("failed to change file descriptor async event queue"); - dev->data->dev_conf.intr_conf.lsc = 0; - } else { - priv->intr_handle.fd = priv->ctx->async_fd; - priv->intr_handle.type = RTE_INTR_HANDLE_EXT; - rte_intr_callback_register(&priv->intr_handle, - mlx4_dev_interrupt_handler, - dev); - } -} - -static struct eth_driver mlx4_driver; +static struct rte_pci_driver mlx4_driver; /** * DPDK callback to register a PCI device. @@ -5541,40 +416,31 @@ static struct eth_driver mlx4_driver; * PCI device information. * * @return - * 0 on success, negative errno value on failure. + * 0 on success, negative errno value otherwise and rte_errno is set. */ static int -mlx4_pci_devinit(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) +mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) { struct ibv_device **list; struct ibv_device *ibv_dev; int err = 0; struct ibv_context *attr_ctx = NULL; struct ibv_device_attr device_attr; + struct mlx4_conf conf = { + .ports.present = 0, + }; unsigned int vf; - int idx; int i; (void)pci_drv; - assert(pci_drv == &mlx4_driver.pci_drv); - /* Get mlx4_dev[] index. */ - idx = mlx4_dev_idx(&pci_dev->addr); - if (idx == -1) { - ERROR("this driver cannot support any more adapters"); - return -ENOMEM; - } - DEBUG("using driver device index %d", idx); - - /* Save PCI address. */ - mlx4_dev[idx].pci_addr = pci_dev->addr; + assert(pci_drv == &mlx4_driver); list = ibv_get_device_list(&i); if (list == NULL) { - assert(errno); - if (errno == ENOSYS) { - WARN("cannot list devices, is ib_uverbs loaded?"); - return 0; - } - return -errno; + rte_errno = errno; + assert(rte_errno); + if (rte_errno == ENOSYS) + ERROR("cannot list devices, is ib_uverbs loaded?"); + return -rte_errno; } assert(i >= 0); /* @@ -5605,169 +471,112 @@ mlx4_pci_devinit(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) ibv_free_device_list(list); switch (err) { case 0: - WARN("cannot access device, is mlx4_ib loaded?"); - return 0; + rte_errno = ENODEV; + ERROR("cannot access device, is mlx4_ib loaded?"); + return -rte_errno; case EINVAL: - WARN("cannot use device, are drivers up to date?"); - return 0; + rte_errno = EINVAL; + ERROR("cannot use device, are drivers up to date?"); + return -rte_errno; } assert(err > 0); - return -err; + rte_errno = err; + return -rte_errno; } ibv_dev = list[i]; - DEBUG("device opened"); - if (ibv_query_device(attr_ctx, &device_attr)) + if (ibv_query_device(attr_ctx, &device_attr)) { + rte_errno = ENODEV; goto error; + } INFO("%u port(s) detected", device_attr.phys_port_cnt); - + conf.ports.present |= (UINT64_C(1) << device_attr.phys_port_cnt) - 1; + if (mlx4_args(pci_dev->device.devargs, &conf)) { + ERROR("failed to process device arguments"); + rte_errno = EINVAL; + goto error; + } + /* Use all ports when none are defined */ + if (!conf.ports.enabled) + conf.ports.enabled = conf.ports.present; for (i = 0; i < device_attr.phys_port_cnt; i++) { uint32_t port = i + 1; /* ports are indexed from one */ - uint32_t test = (1 << i); struct ibv_context *ctx = NULL; struct ibv_port_attr port_attr; struct ibv_pd *pd = NULL; struct priv *priv = NULL; struct rte_eth_dev *eth_dev = NULL; -#ifdef HAVE_EXP_QUERY_DEVICE - struct ibv_exp_device_attr exp_device_attr; -#endif /* HAVE_EXP_QUERY_DEVICE */ struct ether_addr mac; -#ifdef HAVE_EXP_QUERY_DEVICE - exp_device_attr.comp_mask = IBV_EXP_DEVICE_ATTR_EXP_CAP_FLAGS; -#ifdef RSS_SUPPORT - exp_device_attr.comp_mask |= IBV_EXP_DEVICE_ATTR_RSS_TBL_SZ; -#endif /* RSS_SUPPORT */ -#endif /* HAVE_EXP_QUERY_DEVICE */ - - DEBUG("using port %u (%08" PRIx32 ")", port, test); - + /* If port is not enabled, skip. */ + if (!(conf.ports.enabled & (1 << i))) + continue; + DEBUG("using port %u", port); ctx = ibv_open_device(ibv_dev); - if (ctx == NULL) + if (ctx == NULL) { + rte_errno = ENODEV; goto port_error; - + } /* Check port status. */ err = ibv_query_port(ctx, port, &port_attr); if (err) { - ERROR("port query failed: %s", strerror(err)); + rte_errno = err; + ERROR("port query failed: %s", strerror(rte_errno)); goto port_error; } - if (port_attr.link_layer != IBV_LINK_LAYER_ETHERNET) { + rte_errno = ENOTSUP; ERROR("port %d is not configured in Ethernet mode", port); goto port_error; } - if (port_attr.state != IBV_PORT_ACTIVE) DEBUG("port %d is not active: \"%s\" (%d)", port, ibv_port_state_str(port_attr.state), port_attr.state); - + /* Make asynchronous FD non-blocking to handle interrupts. */ + if (mlx4_fd_set_non_blocking(ctx->async_fd) < 0) { + ERROR("cannot make asynchronous FD non-blocking: %s", + strerror(rte_errno)); + goto port_error; + } /* Allocate protection domain. */ pd = ibv_alloc_pd(ctx); if (pd == NULL) { + rte_errno = ENOMEM; ERROR("PD allocation failure"); - err = ENOMEM; goto port_error; } - - mlx4_dev[idx].ports |= test; - /* from rte_ethdev.c */ priv = rte_zmalloc("ethdev private structure", sizeof(*priv), RTE_CACHE_LINE_SIZE); if (priv == NULL) { + rte_errno = ENOMEM; ERROR("priv allocation failure"); - err = ENOMEM; goto port_error; } - priv->ctx = ctx; priv->device_attr = device_attr; priv->port = port; priv->pd = pd; priv->mtu = ETHER_MTU; -#ifdef HAVE_EXP_QUERY_DEVICE - if (ibv_exp_query_device(ctx, &exp_device_attr)) { - ERROR("ibv_exp_query_device() failed"); - goto port_error; - } -#ifdef RSS_SUPPORT - if ((exp_device_attr.exp_device_cap_flags & - IBV_EXP_DEVICE_QPG) && - (exp_device_attr.exp_device_cap_flags & - IBV_EXP_DEVICE_UD_RSS) && - (exp_device_attr.comp_mask & - IBV_EXP_DEVICE_ATTR_RSS_TBL_SZ) && - (exp_device_attr.max_rss_tbl_sz > 0)) { - priv->hw_qpg = 1; - priv->hw_rss = 1; - priv->max_rss_tbl_sz = exp_device_attr.max_rss_tbl_sz; - } else { - priv->hw_qpg = 0; - priv->hw_rss = 0; - priv->max_rss_tbl_sz = 0; - } - priv->hw_tss = !!(exp_device_attr.exp_device_cap_flags & - IBV_EXP_DEVICE_UD_TSS); - DEBUG("device flags: %s%s%s", - (priv->hw_qpg ? "IBV_DEVICE_QPG " : ""), - (priv->hw_tss ? "IBV_DEVICE_TSS " : ""), - (priv->hw_rss ? "IBV_DEVICE_RSS " : "")); - if (priv->hw_rss) - DEBUG("maximum RSS indirection table size: %u", - exp_device_attr.max_rss_tbl_sz); -#endif /* RSS_SUPPORT */ - - priv->hw_csum = - ((exp_device_attr.exp_device_cap_flags & - IBV_EXP_DEVICE_RX_CSUM_TCP_UDP_PKT) && - (exp_device_attr.exp_device_cap_flags & - IBV_EXP_DEVICE_RX_CSUM_IP_PKT)); + priv->vf = vf; + priv->hw_csum = !!(device_attr.device_cap_flags & + IBV_DEVICE_RAW_IP_CSUM); DEBUG("checksum offloading is %ssupported", (priv->hw_csum ? "" : "not ")); - - priv->hw_csum_l2tun = !!(exp_device_attr.exp_device_cap_flags & - IBV_EXP_DEVICE_VXLAN_SUPPORT); + /* Only ConnectX-3 Pro supports tunneling. */ + priv->hw_csum_l2tun = + priv->hw_csum && + (device_attr.vendor_part_id == + PCI_DEVICE_ID_MELLANOX_CONNECTX3PRO); DEBUG("L2 tunnel checksum offloads are %ssupported", (priv->hw_csum_l2tun ? "" : "not ")); - -#ifdef INLINE_RECV - priv->inl_recv_size = mlx4_getenv_int("MLX4_INLINE_RECV_SIZE"); - - if (priv->inl_recv_size) { - exp_device_attr.comp_mask = - IBV_EXP_DEVICE_ATTR_INLINE_RECV_SZ; - if (ibv_exp_query_device(ctx, &exp_device_attr)) { - INFO("Couldn't query device for inline-receive" - " capabilities."); - priv->inl_recv_size = 0; - } else { - if ((unsigned)exp_device_attr.inline_recv_sz < - priv->inl_recv_size) { - INFO("Max inline-receive (%d) <" - " requested inline-receive (%u)", - exp_device_attr.inline_recv_sz, - priv->inl_recv_size); - priv->inl_recv_size = - exp_device_attr.inline_recv_sz; - } - } - INFO("Set inline receive size to %u", - priv->inl_recv_size); - } -#endif /* INLINE_RECV */ -#endif /* HAVE_EXP_QUERY_DEVICE */ - - (void)mlx4_getenv_int; - priv->vf = vf; /* Configure the first MAC address by default. */ - if (priv_get_mac(priv, &mac.addr_bytes)) { + if (mlx4_get_mac(priv, &mac.addr_bytes)) { ERROR("cannot get MAC address, is mlx4_en loaded?" - " (errno: %s)", strerror(errno)); + " (rte_errno: %s)", strerror(rte_errno)); goto port_error; } INFO("port %u MAC address is %02x:%02x:%02x:%02x:%02x:%02x", @@ -5775,18 +584,13 @@ mlx4_pci_devinit(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) mac.addr_bytes[0], mac.addr_bytes[1], mac.addr_bytes[2], mac.addr_bytes[3], mac.addr_bytes[4], mac.addr_bytes[5]); - /* Register MAC and broadcast addresses. */ - claim_zero(priv_mac_addr_add(priv, 0, - (const uint8_t (*)[ETHER_ADDR_LEN]) - mac.addr_bytes)); - claim_zero(priv_mac_addr_add(priv, (elemof(priv->mac) - 1), - &(const uint8_t [ETHER_ADDR_LEN]) - { "\xff\xff\xff\xff\xff\xff" })); + /* Register MAC address. */ + priv->mac[0] = mac; #ifndef NDEBUG { char ifname[IF_NAMESIZE]; - if (priv_get_ifname(priv, &ifname) == 0) + if (mlx4_get_ifname(priv, &ifname) == 0) DEBUG("port %u ifname is \"%s\"", priv->port, ifname); else @@ -5794,70 +598,53 @@ mlx4_pci_devinit(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) } #endif /* Get actual MTU if possible. */ - priv_get_mtu(priv, &priv->mtu); + mlx4_mtu_get(priv, &priv->mtu); DEBUG("port %u MTU is %u", priv->port, priv->mtu); - /* from rte_ethdev.c */ { char name[RTE_ETH_NAME_MAX_LEN]; snprintf(name, sizeof(name), "%s port %u", ibv_get_device_name(ibv_dev), port); - eth_dev = rte_eth_dev_allocate(name, RTE_ETH_DEV_PCI); + eth_dev = rte_eth_dev_allocate(name); } if (eth_dev == NULL) { ERROR("can not allocate rte ethdev"); - err = ENOMEM; + rte_errno = ENOMEM; goto port_error; } - - /* Secondary processes have to use local storage for their - * private data as well as a copy of eth_dev->data, but this - * pointer must not be modified before burst functions are - * actually called. */ - if (mlx4_is_secondary()) { - struct mlx4_secondary_data *sd = - &mlx4_secondary_data[eth_dev->data->port_id]; - - sd->primary_priv = eth_dev->data->dev_private; - if (sd->primary_priv == NULL) { - ERROR("no private data for port %u", - eth_dev->data->port_id); - err = EINVAL; - goto port_error; - } - sd->shared_dev_data = eth_dev->data; - rte_spinlock_init(&sd->lock); - memcpy(sd->data.name, sd->shared_dev_data->name, - sizeof(sd->data.name)); - sd->data.dev_private = priv; - sd->data.rx_mbuf_alloc_failed = 0; - sd->data.mtu = ETHER_MTU; - sd->data.port_id = sd->shared_dev_data->port_id; - sd->data.mac_addrs = priv->mac; - eth_dev->tx_pkt_burst = mlx4_tx_burst_secondary_setup; - eth_dev->rx_pkt_burst = mlx4_rx_burst_secondary_setup; - } else { - eth_dev->data->dev_private = priv; - eth_dev->data->rx_mbuf_alloc_failed = 0; - eth_dev->data->mtu = ETHER_MTU; - eth_dev->data->mac_addrs = priv->mac; - } - eth_dev->pci_dev = pci_dev; - + eth_dev->data->dev_private = priv; + eth_dev->data->mac_addrs = priv->mac; + eth_dev->device = &pci_dev->device; rte_eth_copy_pci_info(eth_dev, pci_dev); - - eth_dev->driver = &mlx4_driver; - + eth_dev->device->driver = &mlx4_driver.driver; + /* Initialize local interrupt handle for current port. */ + priv->intr_handle = (struct rte_intr_handle){ + .fd = -1, + .type = RTE_INTR_HANDLE_EXT, + }; + /* + * Override ethdev interrupt handle pointer with private + * handle instead of that of the parent PCI device used by + * default. This prevents it from being shared between all + * ports of the same PCI device since each of them is + * associated its own Verbs context. + * + * Rx interrupts in particular require this as the PMD has + * no control over the registration of queue interrupts + * besides setting up eth_dev->intr_handle, the rest is + * handled by rte_intr_rx_ctl(). + */ + eth_dev->intr_handle = &priv->intr_handle; priv->dev = eth_dev; eth_dev->dev_ops = &mlx4_dev_ops; - TAILQ_INIT(ð_dev->link_intr_cbs); - /* Bring Ethernet device up. */ DEBUG("forcing Ethernet interface up"); - priv_set_flags(priv, ~IFF_UP, IFF_UP); + mlx4_dev_set_link_up(priv->dev); + /* Update link status once if waiting for LSC. */ + if (eth_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) + mlx4_link_update(eth_dev, 0); continue; - port_error: rte_free(priv); if (pd) @@ -5868,27 +655,21 @@ port_error: rte_eth_dev_release_port(eth_dev); break; } - + if (i == device_attr.phys_port_cnt) + return 0; /* * XXX if something went wrong in the loop above, there is a resource * leak (ctx, pd, priv, dpdk ethdev) but we can do nothing about it as * long as the dpdk does not provide a way to deallocate a ethdev and a * way to enumerate the registered ethdevs to free the previous ones. */ - - /* no port found, complain */ - if (!mlx4_dev[idx].ports) { - err = ENODEV; - goto error; - } - error: if (attr_ctx) claim_zero(ibv_close_device(attr_ctx)); if (list) ibv_free_device_list(list); - assert(err >= 0); - return -err; + assert(rte_errno >= 0); + return -rte_errno; } static const struct rte_pci_id mlx4_pci_id_map[] = { @@ -5909,26 +690,29 @@ static const struct rte_pci_id mlx4_pci_id_map[] = { } }; -static struct eth_driver mlx4_driver = { - .pci_drv = { - .name = MLX4_DRIVER_NAME, - .id_table = mlx4_pci_id_map, - .devinit = mlx4_pci_devinit, - .drv_flags = RTE_PCI_DRV_INTR_LSC, +static struct rte_pci_driver mlx4_driver = { + .driver = { + .name = MLX4_DRIVER_NAME }, - .dev_private_size = sizeof(struct priv) + .id_table = mlx4_pci_id_map, + .probe = mlx4_pci_probe, + .drv_flags = RTE_PCI_DRV_INTR_LSC | + RTE_PCI_DRV_INTR_RMV, }; /** * Driver initialization routine. */ -static int -rte_mlx4_pmd_init(const char *name, const char *args) +RTE_INIT(rte_mlx4_pmd_init); +static void +rte_mlx4_pmd_init(void) { - (void)name; - (void)args; - - RTE_BUILD_BUG_ON(sizeof(wr_id_t) != sizeof(uint64_t)); + /* + * MLX4_DEVICE_FATAL_CLEANUP tells ibv_destroy functions we + * want to get success errno value in case of calling them + * when the device was removed. + */ + setenv("MLX4_DEVICE_FATAL_CLEANUP", "1", 1); /* * RDMAV_HUGEPAGES_SAFE tells ibv_fork_init() we intend to use * huge pages. Calling ibv_fork_init() during init allows @@ -5937,14 +721,10 @@ rte_mlx4_pmd_init(const char *name, const char *args) */ setenv("RDMAV_HUGEPAGES_SAFE", "1", 1); ibv_fork_init(); - rte_eal_pci_register(&mlx4_driver.pci_drv); - return 0; + rte_pci_register(&mlx4_driver); } -static struct rte_driver rte_mlx4_driver = { - .type = PMD_PDEV, - .init = rte_mlx4_pmd_init, -}; - -PMD_REGISTER_DRIVER(rte_mlx4_driver, mlx4); -DRIVER_REGISTER_PCI_TABLE(mlx4, mlx4_pci_id_map); +RTE_PMD_EXPORT_NAME(net_mlx4, __COUNTER__); +RTE_PMD_REGISTER_PCI_TABLE(net_mlx4, mlx4_pci_id_map); +RTE_PMD_REGISTER_KMOD_DEP(net_mlx4, + "* ib_uverbs & mlx4_en & mlx4_core & mlx4_ib"); diff --git a/dpdk/drivers/net/mlx4/mlx4.h b/dpdk/drivers/net/mlx4/mlx4.h index d0c7bc29..3aeef87e 100644 --- a/dpdk/drivers/net/mlx4/mlx4.h +++ b/dpdk/drivers/net/mlx4/mlx4.h @@ -1,8 +1,8 @@ /*- * BSD LICENSE * - * Copyright 2012-2015 6WIND S.A. - * Copyright 2012 Mellanox. + * Copyright 2012 6WIND S.A. + * Copyright 2012 Mellanox * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -34,37 +34,38 @@ #ifndef RTE_PMD_MLX4_H_ #define RTE_PMD_MLX4_H_ -#include +#include #include -#include +#include -/* - * Maximum number of simultaneous MAC addresses supported. - * - * According to ConnectX's Programmer Reference Manual: - * The L2 Address Match is implemented by comparing a MAC/VLAN combination - * of 128 MAC addresses and 127 VLAN values, comprising 128x127 possible - * L2 addresses. - */ +/* Verbs headers do not support -pedantic. */ +#ifdef PEDANTIC +#pragma GCC diagnostic ignored "-Wpedantic" +#endif +#include +#ifdef PEDANTIC +#pragma GCC diagnostic error "-Wpedantic" +#endif + +#include +#include +#include +#include +#include + +/** Maximum number of simultaneous MAC addresses. This value is arbitrary. */ #define MLX4_MAX_MAC_ADDRESSES 128 -/* Maximum number of simultaneous VLAN filters supported. See above. */ -#define MLX4_MAX_VLAN_IDS 127 - -/* Request send completion once in every 64 sends, might be less. */ +/** Request send completion once in every 64 sends, might be less. */ #define MLX4_PMD_TX_PER_COMP_REQ 64 -/* Maximum number of Scatter/Gather Elements per Work Request. */ -#ifndef MLX4_PMD_SGE_WR_N -#define MLX4_PMD_SGE_WR_N 4 -#endif - -/* Maximum size for inline data. */ -#ifndef MLX4_PMD_MAX_INLINE +/** Maximum size for inline data. */ #define MLX4_PMD_MAX_INLINE 0 -#endif -/* +/** Fixed RSS hash key size in bytes. Cannot be modified. */ +#define MLX4_RSS_HASH_KEY_SIZE 40 + +/** * Maximum number of cached Memory Pools (MPs) per TX queue. Each RTE MP * from which buffers are to be transmitted will have to be mapped by this * driver to their own Memory Region (MR). This is a slow operation. @@ -75,16 +76,11 @@ #define MLX4_PMD_TX_MP_CACHE 8 #endif -/* - * If defined, only use software counters. The PMD will never ask the hardware - * for these, and many of them won't be available. - */ -#ifndef MLX4_PMD_SOFT_COUNTERS -#define MLX4_PMD_SOFT_COUNTERS 1 -#endif +/** Interrupt alarm timeout value in microseconds. */ +#define MLX4_INTR_ALARM_TIMEOUT 100000 -/* Alarm timeout. */ -#define MLX4_ALARM_TIMEOUT_US 100000 +/** Port parameter. */ +#define MLX4_PMD_PORT_KVARG "port" enum { PCI_VENDOR_ID_MELLANOX = 0x15b3, @@ -96,68 +92,92 @@ enum { PCI_DEVICE_ID_MELLANOX_CONNECTX3PRO = 0x1007, }; -#define MLX4_DRIVER_NAME "librte_pmd_mlx4" +/** Driver name reported to lower layers and used in log output. */ +#define MLX4_DRIVER_NAME "net_mlx4" -/* Bit-field manipulation. */ -#define BITFIELD_DECLARE(bf, type, size) \ - type bf[(((size_t)(size) / (sizeof(type) * CHAR_BIT)) + \ - !!((size_t)(size) % (sizeof(type) * CHAR_BIT)))] -#define BITFIELD_DEFINE(bf, type, size) \ - BITFIELD_DECLARE((bf), type, (size)) = { 0 } -#define BITFIELD_SET(bf, b) \ - (assert((size_t)(b) < (sizeof(bf) * CHAR_BIT)), \ - (void)((bf)[((b) / (sizeof((bf)[0]) * CHAR_BIT))] |= \ - ((size_t)1 << ((b) % (sizeof((bf)[0]) * CHAR_BIT))))) -#define BITFIELD_RESET(bf, b) \ - (assert((size_t)(b) < (sizeof(bf) * CHAR_BIT)), \ - (void)((bf)[((b) / (sizeof((bf)[0]) * CHAR_BIT))] &= \ - ~((size_t)1 << ((b) % (sizeof((bf)[0]) * CHAR_BIT))))) -#define BITFIELD_ISSET(bf, b) \ - (assert((size_t)(b) < (sizeof(bf) * CHAR_BIT)), \ - !!(((bf)[((b) / (sizeof((bf)[0]) * CHAR_BIT))] & \ - ((size_t)1 << ((b) % (sizeof((bf)[0]) * CHAR_BIT)))))) +struct mlx4_drop; +struct mlx4_rss; +struct rxq; +struct txq; +struct rte_flow; -/* Number of elements in array. */ -#define elemof(a) (sizeof(a) / sizeof((a)[0])) +/** Memory region descriptor. */ +struct mlx4_mr { + LIST_ENTRY(mlx4_mr) next; /**< Next entry in list. */ + uintptr_t start; /**< Base address for memory region. */ + uintptr_t end; /**< End address for memory region. */ + uint32_t lkey; /**< L_Key extracted from @p mr. */ + uint32_t refcnt; /**< Reference count for this object. */ + struct priv *priv; /**< Back pointer to private data. */ + struct ibv_mr *mr; /**< Memory region associated with @p mp. */ + struct rte_mempool *mp; /**< Target memory pool (mempool). */ +}; -/* Cast pointer p to structure member m to its parent structure of type t. */ -#define containerof(p, t, m) ((t *)((uint8_t *)(p) - offsetof(t, m))) +/** Private data structure. */ +struct priv { + struct rte_eth_dev *dev; /**< Ethernet device. */ + struct ibv_context *ctx; /**< Verbs context. */ + struct ibv_device_attr device_attr; /**< Device properties. */ + struct ibv_pd *pd; /**< Protection Domain. */ + /* Device properties. */ + uint16_t mtu; /**< Configured MTU. */ + uint8_t port; /**< Physical port number. */ + uint32_t started:1; /**< Device started, flows enabled. */ + uint32_t vf:1; /**< This is a VF device. */ + uint32_t intr_alarm:1; /**< An interrupt alarm is scheduled. */ + uint32_t isolated:1; /**< Toggle isolated mode. */ + uint32_t hw_csum:1; /* Checksum offload is supported. */ + uint32_t hw_csum_l2tun:1; /* Checksum support for L2 tunnels. */ + struct rte_intr_handle intr_handle; /**< Port interrupt handle. */ + struct mlx4_drop *drop; /**< Shared resources for drop flow rules. */ + LIST_HEAD(, mlx4_rss) rss; /**< Shared targets for Rx flow rules. */ + LIST_HEAD(, rte_flow) flows; /**< Configured flow rule handles. */ + LIST_HEAD(, mlx4_mr) mr; /**< Registered memory regions. */ + rte_spinlock_t mr_lock; /**< Lock for @p mr access. */ + struct ether_addr mac[MLX4_MAX_MAC_ADDRESSES]; + /**< Configured MAC addresses. Unused entries are zeroed. */ +}; -/* Branch prediction helpers. */ -#ifndef likely -#define likely(c) __builtin_expect(!!(c), 1) -#endif -#ifndef unlikely -#define unlikely(c) __builtin_expect(!!(c), 0) -#endif +/* mlx4_ethdev.c */ -/* Debugging */ -#ifndef NDEBUG -#include -#define DEBUG__(m, ...) \ - (fprintf(stderr, "%s:%d: %s(): " m "%c", \ - __FILE__, __LINE__, __func__, __VA_ARGS__), \ - fflush(stderr), \ - (void)0) -/* - * Save/restore errno around DEBUG__(). - * XXX somewhat undefined behavior, but works. - */ -#define DEBUG_(...) \ - (errno = ((int []){ \ - *(volatile int *)&errno, \ - (DEBUG__(__VA_ARGS__), 0) \ - })[0]) -#define DEBUG(...) DEBUG_(__VA_ARGS__, '\n') -#define claim_zero(...) assert((__VA_ARGS__) == 0) -#define claim_nonzero(...) assert((__VA_ARGS__) != 0) -#define claim_positive(...) assert((__VA_ARGS__) >= 0) -#else /* NDEBUG */ -/* No-ops. */ -#define DEBUG(...) (void)0 -#define claim_zero(...) (__VA_ARGS__) -#define claim_nonzero(...) (__VA_ARGS__) -#define claim_positive(...) (__VA_ARGS__) -#endif /* NDEBUG */ +int mlx4_get_ifname(const struct priv *priv, char (*ifname)[IF_NAMESIZE]); +int mlx4_get_mac(struct priv *priv, uint8_t (*mac)[ETHER_ADDR_LEN]); +int mlx4_mtu_get(struct priv *priv, uint16_t *mtu); +int mlx4_mtu_set(struct rte_eth_dev *dev, uint16_t mtu); +int mlx4_dev_set_link_down(struct rte_eth_dev *dev); +int mlx4_dev_set_link_up(struct rte_eth_dev *dev); +void mlx4_promiscuous_enable(struct rte_eth_dev *dev); +void mlx4_promiscuous_disable(struct rte_eth_dev *dev); +void mlx4_allmulticast_enable(struct rte_eth_dev *dev); +void mlx4_allmulticast_disable(struct rte_eth_dev *dev); +void mlx4_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index); +int mlx4_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac_addr, + uint32_t index, uint32_t vmdq); +void mlx4_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr); +int mlx4_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on); +int mlx4_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats); +void mlx4_stats_reset(struct rte_eth_dev *dev); +void mlx4_dev_infos_get(struct rte_eth_dev *dev, + struct rte_eth_dev_info *info); +int mlx4_link_update(struct rte_eth_dev *dev, int wait_to_complete); +int mlx4_flow_ctrl_get(struct rte_eth_dev *dev, + struct rte_eth_fc_conf *fc_conf); +int mlx4_flow_ctrl_set(struct rte_eth_dev *dev, + struct rte_eth_fc_conf *fc_conf); +const uint32_t *mlx4_dev_supported_ptypes_get(struct rte_eth_dev *dev); + +/* mlx4_intr.c */ + +int mlx4_intr_uninstall(struct priv *priv); +int mlx4_intr_install(struct priv *priv); +int mlx4_rx_intr_disable(struct rte_eth_dev *dev, uint16_t idx); +int mlx4_rx_intr_enable(struct rte_eth_dev *dev, uint16_t idx); + +/* mlx4_mr.c */ + +struct mlx4_mr *mlx4_mr_get(struct priv *priv, struct rte_mempool *mp); +void mlx4_mr_put(struct mlx4_mr *mr); +uint32_t mlx4_txq_add_mr(struct txq *txq, struct rte_mempool *mp, + uint32_t i); #endif /* RTE_PMD_MLX4_H_ */ diff --git a/dpdk/drivers/net/mlx4/mlx4_ethdev.c b/dpdk/drivers/net/mlx4/mlx4_ethdev.c new file mode 100644 index 00000000..2f69e7d4 --- /dev/null +++ b/dpdk/drivers/net/mlx4/mlx4_ethdev.c @@ -0,0 +1,1062 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of 6WIND S.A. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** + * @file + * Miscellaneous control operations for mlx4 driver. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Verbs headers do not support -pedantic. */ +#ifdef PEDANTIC +#pragma GCC diagnostic ignored "-Wpedantic" +#endif +#include +#ifdef PEDANTIC +#pragma GCC diagnostic error "-Wpedantic" +#endif + +#include +#include +#include +#include +#include +#include + +#include "mlx4.h" +#include "mlx4_flow.h" +#include "mlx4_rxtx.h" +#include "mlx4_utils.h" + +/** + * Get interface name from private structure. + * + * @param[in] priv + * Pointer to private structure. + * @param[out] ifname + * Interface name output buffer. + * + * @return + * 0 on success, negative errno value otherwise and rte_errno is set. + */ +int +mlx4_get_ifname(const struct priv *priv, char (*ifname)[IF_NAMESIZE]) +{ + DIR *dir; + struct dirent *dent; + unsigned int dev_type = 0; + unsigned int dev_port_prev = ~0u; + char match[IF_NAMESIZE] = ""; + + { + MKSTR(path, "%s/device/net", priv->ctx->device->ibdev_path); + + dir = opendir(path); + if (dir == NULL) { + rte_errno = errno; + return -rte_errno; + } + } + while ((dent = readdir(dir)) != NULL) { + char *name = dent->d_name; + FILE *file; + unsigned int dev_port; + int r; + + if ((name[0] == '.') && + ((name[1] == '\0') || + ((name[1] == '.') && (name[2] == '\0')))) + continue; + + MKSTR(path, "%s/device/net/%s/%s", + priv->ctx->device->ibdev_path, name, + (dev_type ? "dev_id" : "dev_port")); + + file = fopen(path, "rb"); + if (file == NULL) { + if (errno != ENOENT) + continue; + /* + * Switch to dev_id when dev_port does not exist as + * is the case with Linux kernel versions < 3.15. + */ +try_dev_id: + match[0] = '\0'; + if (dev_type) + break; + dev_type = 1; + dev_port_prev = ~0u; + rewinddir(dir); + continue; + } + r = fscanf(file, (dev_type ? "%x" : "%u"), &dev_port); + fclose(file); + if (r != 1) + continue; + /* + * Switch to dev_id when dev_port returns the same value for + * all ports. May happen when using a MOFED release older than + * 3.0 with a Linux kernel >= 3.15. + */ + if (dev_port == dev_port_prev) + goto try_dev_id; + dev_port_prev = dev_port; + if (dev_port == (priv->port - 1u)) + snprintf(match, sizeof(match), "%s", name); + } + closedir(dir); + if (match[0] == '\0') { + rte_errno = ENODEV; + return -rte_errno; + } + strncpy(*ifname, match, sizeof(*ifname)); + return 0; +} + +/** + * Read from sysfs entry. + * + * @param[in] priv + * Pointer to private structure. + * @param[in] entry + * Entry name relative to sysfs path. + * @param[out] buf + * Data output buffer. + * @param size + * Buffer size. + * + * @return + * Number of bytes read on success, negative errno value otherwise and + * rte_errno is set. + */ +static int +mlx4_sysfs_read(const struct priv *priv, const char *entry, + char *buf, size_t size) +{ + char ifname[IF_NAMESIZE]; + FILE *file; + int ret; + + ret = mlx4_get_ifname(priv, &ifname); + if (ret) + return ret; + + MKSTR(path, "%s/device/net/%s/%s", priv->ctx->device->ibdev_path, + ifname, entry); + + file = fopen(path, "rb"); + if (file == NULL) { + rte_errno = errno; + return -rte_errno; + } + ret = fread(buf, 1, size, file); + if ((size_t)ret < size && ferror(file)) { + rte_errno = EIO; + ret = -rte_errno; + } else { + ret = size; + } + fclose(file); + return ret; +} + +/** + * Write to sysfs entry. + * + * @param[in] priv + * Pointer to private structure. + * @param[in] entry + * Entry name relative to sysfs path. + * @param[in] buf + * Data buffer. + * @param size + * Buffer size. + * + * @return + * Number of bytes written on success, negative errno value otherwise and + * rte_errno is set. + */ +static int +mlx4_sysfs_write(const struct priv *priv, const char *entry, + char *buf, size_t size) +{ + char ifname[IF_NAMESIZE]; + FILE *file; + int ret; + + ret = mlx4_get_ifname(priv, &ifname); + if (ret) + return ret; + + MKSTR(path, "%s/device/net/%s/%s", priv->ctx->device->ibdev_path, + ifname, entry); + + file = fopen(path, "wb"); + if (file == NULL) { + rte_errno = errno; + return -rte_errno; + } + ret = fwrite(buf, 1, size, file); + if ((size_t)ret < size || ferror(file)) { + rte_errno = EIO; + ret = -rte_errno; + } else { + ret = size; + } + fclose(file); + return ret; +} + +/** + * Get unsigned long sysfs property. + * + * @param priv + * Pointer to private structure. + * @param[in] name + * Entry name relative to sysfs path. + * @param[out] value + * Value output buffer. + * + * @return + * 0 on success, negative errno value otherwise and rte_errno is set. + */ +static int +mlx4_get_sysfs_ulong(struct priv *priv, const char *name, unsigned long *value) +{ + int ret; + unsigned long value_ret; + char value_str[32]; + + ret = mlx4_sysfs_read(priv, name, value_str, (sizeof(value_str) - 1)); + if (ret < 0) { + DEBUG("cannot read %s value from sysfs: %s", + name, strerror(rte_errno)); + return ret; + } + value_str[ret] = '\0'; + errno = 0; + value_ret = strtoul(value_str, NULL, 0); + if (errno) { + rte_errno = errno; + DEBUG("invalid %s value `%s': %s", name, value_str, + strerror(rte_errno)); + return -rte_errno; + } + *value = value_ret; + return 0; +} + +/** + * Set unsigned long sysfs property. + * + * @param priv + * Pointer to private structure. + * @param[in] name + * Entry name relative to sysfs path. + * @param value + * Value to set. + * + * @return + * 0 on success, negative errno value otherwise and rte_errno is set. + */ +static int +mlx4_set_sysfs_ulong(struct priv *priv, const char *name, unsigned long value) +{ + int ret; + MKSTR(value_str, "%lu", value); + + ret = mlx4_sysfs_write(priv, name, value_str, (sizeof(value_str) - 1)); + if (ret < 0) { + DEBUG("cannot write %s `%s' (%lu) to sysfs: %s", + name, value_str, value, strerror(rte_errno)); + return ret; + } + return 0; +} + +/** + * Perform ifreq ioctl() on associated Ethernet device. + * + * @param[in] priv + * Pointer to private structure. + * @param req + * Request number to pass to ioctl(). + * @param[out] ifr + * Interface request structure output buffer. + * + * @return + * 0 on success, negative errno value otherwise and rte_errno is set. + */ +static int +mlx4_ifreq(const struct priv *priv, int req, struct ifreq *ifr) +{ + int sock = socket(PF_INET, SOCK_DGRAM, IPPROTO_IP); + int ret; + + if (sock == -1) { + rte_errno = errno; + return -rte_errno; + } + ret = mlx4_get_ifname(priv, &ifr->ifr_name); + if (!ret && ioctl(sock, req, ifr) == -1) { + rte_errno = errno; + ret = -rte_errno; + } + close(sock); + return ret; +} + +/** + * Get MAC address by querying netdevice. + * + * @param[in] priv + * Pointer to private structure. + * @param[out] mac + * MAC address output buffer. + * + * @return + * 0 on success, negative errno value otherwise and rte_errno is set. + */ +int +mlx4_get_mac(struct priv *priv, uint8_t (*mac)[ETHER_ADDR_LEN]) +{ + struct ifreq request; + int ret = mlx4_ifreq(priv, SIOCGIFHWADDR, &request); + + if (ret) + return ret; + memcpy(mac, request.ifr_hwaddr.sa_data, ETHER_ADDR_LEN); + return 0; +} + +/** + * Get device MTU. + * + * @param priv + * Pointer to private structure. + * @param[out] mtu + * MTU value output buffer. + * + * @return + * 0 on success, negative errno value otherwise and rte_errno is set. + */ +int +mlx4_mtu_get(struct priv *priv, uint16_t *mtu) +{ + unsigned long ulong_mtu = 0; + int ret = mlx4_get_sysfs_ulong(priv, "mtu", &ulong_mtu); + + if (ret) + return ret; + *mtu = ulong_mtu; + return 0; +} + +/** + * DPDK callback to change the MTU. + * + * @param priv + * Pointer to Ethernet device structure. + * @param mtu + * MTU value to set. + * + * @return + * 0 on success, negative errno value otherwise and rte_errno is set. + */ +int +mlx4_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) +{ + struct priv *priv = dev->data->dev_private; + uint16_t new_mtu; + int ret = mlx4_set_sysfs_ulong(priv, "mtu", mtu); + + if (ret) + return ret; + ret = mlx4_mtu_get(priv, &new_mtu); + if (ret) + return ret; + if (new_mtu == mtu) { + priv->mtu = mtu; + return 0; + } + rte_errno = EINVAL; + return -rte_errno; +} + +/** + * Set device flags. + * + * @param priv + * Pointer to private structure. + * @param keep + * Bitmask for flags that must remain untouched. + * @param flags + * Bitmask for flags to modify. + * + * @return + * 0 on success, negative errno value otherwise and rte_errno is set. + */ +static int +mlx4_set_flags(struct priv *priv, unsigned int keep, unsigned int flags) +{ + unsigned long tmp = 0; + int ret = mlx4_get_sysfs_ulong(priv, "flags", &tmp); + + if (ret) + return ret; + tmp &= keep; + tmp |= (flags & (~keep)); + return mlx4_set_sysfs_ulong(priv, "flags", tmp); +} + +/** + * Change the link state (UP / DOWN). + * + * @param priv + * Pointer to Ethernet device private data. + * @param up + * Nonzero for link up, otherwise link down. + * + * @return + * 0 on success, negative errno value otherwise and rte_errno is set. + */ +static int +mlx4_dev_set_link(struct priv *priv, int up) +{ + int err; + + if (up) { + err = mlx4_set_flags(priv, ~IFF_UP, IFF_UP); + if (err) + return err; + } else { + err = mlx4_set_flags(priv, ~IFF_UP, ~IFF_UP); + if (err) + return err; + } + return 0; +} + +/** + * DPDK callback to bring the link DOWN. + * + * @param dev + * Pointer to Ethernet device structure. + * + * @return + * 0 on success, negative errno value otherwise and rte_errno is set. + */ +int +mlx4_dev_set_link_down(struct rte_eth_dev *dev) +{ + struct priv *priv = dev->data->dev_private; + + return mlx4_dev_set_link(priv, 0); +} + +/** + * DPDK callback to bring the link UP. + * + * @param dev + * Pointer to Ethernet device structure. + * + * @return + * 0 on success, negative errno value otherwise and rte_errno is set. + */ +int +mlx4_dev_set_link_up(struct rte_eth_dev *dev) +{ + struct priv *priv = dev->data->dev_private; + + return mlx4_dev_set_link(priv, 1); +} + +/** + * Supported Rx mode toggles. + * + * Even and odd values respectively stand for off and on. + */ +enum rxmode_toggle { + RXMODE_TOGGLE_PROMISC_OFF, + RXMODE_TOGGLE_PROMISC_ON, + RXMODE_TOGGLE_ALLMULTI_OFF, + RXMODE_TOGGLE_ALLMULTI_ON, +}; + +/** + * Helper function to toggle promiscuous and all multicast modes. + * + * @param dev + * Pointer to Ethernet device structure. + * @param toggle + * Toggle to set. + */ +static void +mlx4_rxmode_toggle(struct rte_eth_dev *dev, enum rxmode_toggle toggle) +{ + struct priv *priv = dev->data->dev_private; + const char *mode; + struct rte_flow_error error; + + switch (toggle) { + case RXMODE_TOGGLE_PROMISC_OFF: + case RXMODE_TOGGLE_PROMISC_ON: + mode = "promiscuous"; + dev->data->promiscuous = toggle & 1; + break; + case RXMODE_TOGGLE_ALLMULTI_OFF: + case RXMODE_TOGGLE_ALLMULTI_ON: + mode = "all multicast"; + dev->data->all_multicast = toggle & 1; + break; + } + if (!mlx4_flow_sync(priv, &error)) + return; + ERROR("cannot toggle %s mode (code %d, \"%s\")," + " flow error type %d, cause %p, message: %s", + mode, rte_errno, strerror(rte_errno), error.type, error.cause, + error.message ? error.message : "(unspecified)"); +} + +/** + * DPDK callback to enable promiscuous mode. + * + * @param dev + * Pointer to Ethernet device structure. + */ +void +mlx4_promiscuous_enable(struct rte_eth_dev *dev) +{ + mlx4_rxmode_toggle(dev, RXMODE_TOGGLE_PROMISC_ON); +} + +/** + * DPDK callback to disable promiscuous mode. + * + * @param dev + * Pointer to Ethernet device structure. + */ +void +mlx4_promiscuous_disable(struct rte_eth_dev *dev) +{ + mlx4_rxmode_toggle(dev, RXMODE_TOGGLE_PROMISC_OFF); +} + +/** + * DPDK callback to enable all multicast mode. + * + * @param dev + * Pointer to Ethernet device structure. + */ +void +mlx4_allmulticast_enable(struct rte_eth_dev *dev) +{ + mlx4_rxmode_toggle(dev, RXMODE_TOGGLE_ALLMULTI_ON); +} + +/** + * DPDK callback to disable all multicast mode. + * + * @param dev + * Pointer to Ethernet device structure. + */ +void +mlx4_allmulticast_disable(struct rte_eth_dev *dev) +{ + mlx4_rxmode_toggle(dev, RXMODE_TOGGLE_ALLMULTI_OFF); +} + +/** + * DPDK callback to remove a MAC address. + * + * @param dev + * Pointer to Ethernet device structure. + * @param index + * MAC address index. + */ +void +mlx4_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index) +{ + struct priv *priv = dev->data->dev_private; + struct rte_flow_error error; + + if (index >= RTE_DIM(priv->mac)) { + rte_errno = EINVAL; + return; + } + memset(&priv->mac[index], 0, sizeof(priv->mac[index])); + if (!mlx4_flow_sync(priv, &error)) + return; + ERROR("failed to synchronize flow rules after removing MAC address" + " at index %d (code %d, \"%s\")," + " flow error type %d, cause %p, message: %s", + index, rte_errno, strerror(rte_errno), error.type, error.cause, + error.message ? error.message : "(unspecified)"); +} + +/** + * DPDK callback to add a MAC address. + * + * @param dev + * Pointer to Ethernet device structure. + * @param mac_addr + * MAC address to register. + * @param index + * MAC address index. + * @param vmdq + * VMDq pool index to associate address with (ignored). + * + * @return + * 0 on success, negative errno value otherwise and rte_errno is set. + */ +int +mlx4_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac_addr, + uint32_t index, uint32_t vmdq) +{ + struct priv *priv = dev->data->dev_private; + struct rte_flow_error error; + int ret; + + (void)vmdq; + if (index >= RTE_DIM(priv->mac)) { + rte_errno = EINVAL; + return -rte_errno; + } + memcpy(&priv->mac[index], mac_addr, sizeof(priv->mac[index])); + ret = mlx4_flow_sync(priv, &error); + if (!ret) + return 0; + ERROR("failed to synchronize flow rules after adding MAC address" + " at index %d (code %d, \"%s\")," + " flow error type %d, cause %p, message: %s", + index, rte_errno, strerror(rte_errno), error.type, error.cause, + error.message ? error.message : "(unspecified)"); + return ret; +} + +/** + * DPDK callback to configure a VLAN filter. + * + * @param dev + * Pointer to Ethernet device structure. + * @param vlan_id + * VLAN ID to filter. + * @param on + * Toggle filter. + * + * @return + * 0 on success, negative errno value otherwise and rte_errno is set. + */ +int +mlx4_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) +{ + struct priv *priv = dev->data->dev_private; + struct rte_flow_error error; + unsigned int vidx = vlan_id / 64; + unsigned int vbit = vlan_id % 64; + uint64_t *v; + int ret; + + if (vidx >= RTE_DIM(dev->data->vlan_filter_conf.ids)) { + rte_errno = EINVAL; + return -rte_errno; + } + v = &dev->data->vlan_filter_conf.ids[vidx]; + *v &= ~(UINT64_C(1) << vbit); + *v |= (uint64_t)!!on << vbit; + ret = mlx4_flow_sync(priv, &error); + if (!ret) + return 0; + ERROR("failed to synchronize flow rules after %s VLAN filter on ID %u" + " (code %d, \"%s\"), " + " flow error type %d, cause %p, message: %s", + on ? "enabling" : "disabling", vlan_id, + rte_errno, strerror(rte_errno), error.type, error.cause, + error.message ? error.message : "(unspecified)"); + return ret; +} + +/** + * DPDK callback to set the primary MAC address. + * + * @param dev + * Pointer to Ethernet device structure. + * @param mac_addr + * MAC address to register. + */ +void +mlx4_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr) +{ + mlx4_mac_addr_add(dev, mac_addr, 0, 0); +} + +/** + * DPDK callback to get information about the device. + * + * @param dev + * Pointer to Ethernet device structure. + * @param[out] info + * Info structure output buffer. + */ +void +mlx4_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info) +{ + struct priv *priv = dev->data->dev_private; + unsigned int max; + char ifname[IF_NAMESIZE]; + + info->pci_dev = RTE_ETH_DEV_TO_PCI(dev); + /* FIXME: we should ask the device for these values. */ + info->min_rx_bufsize = 32; + info->max_rx_pktlen = 65536; + /* + * Since we need one CQ per QP, the limit is the minimum number + * between the two values. + */ + max = ((priv->device_attr.max_cq > priv->device_attr.max_qp) ? + priv->device_attr.max_qp : priv->device_attr.max_cq); + /* If max >= 65535 then max = 0, max_rx_queues is uint16_t. */ + if (max >= 65535) + max = 65535; + info->max_rx_queues = max; + info->max_tx_queues = max; + info->max_mac_addrs = RTE_DIM(priv->mac); + info->rx_offload_capa = 0; + info->tx_offload_capa = 0; + if (priv->hw_csum) { + info->tx_offload_capa |= (DEV_TX_OFFLOAD_IPV4_CKSUM | + DEV_TX_OFFLOAD_UDP_CKSUM | + DEV_TX_OFFLOAD_TCP_CKSUM); + info->rx_offload_capa |= (DEV_RX_OFFLOAD_IPV4_CKSUM | + DEV_RX_OFFLOAD_UDP_CKSUM | + DEV_RX_OFFLOAD_TCP_CKSUM); + } + if (priv->hw_csum_l2tun) + info->tx_offload_capa |= DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM; + if (mlx4_get_ifname(priv, &ifname) == 0) + info->if_index = if_nametoindex(ifname); + info->hash_key_size = MLX4_RSS_HASH_KEY_SIZE; + info->speed_capa = + ETH_LINK_SPEED_1G | + ETH_LINK_SPEED_10G | + ETH_LINK_SPEED_20G | + ETH_LINK_SPEED_40G | + ETH_LINK_SPEED_56G; +} + +/** + * DPDK callback to get device statistics. + * + * @param dev + * Pointer to Ethernet device structure. + * @param[out] stats + * Stats structure output buffer. + */ +int +mlx4_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) +{ + struct rte_eth_stats tmp; + unsigned int i; + unsigned int idx; + + memset(&tmp, 0, sizeof(tmp)); + /* Add software counters. */ + for (i = 0; i != dev->data->nb_rx_queues; ++i) { + struct rxq *rxq = dev->data->rx_queues[i]; + + if (rxq == NULL) + continue; + idx = rxq->stats.idx; + if (idx < RTE_ETHDEV_QUEUE_STAT_CNTRS) { + tmp.q_ipackets[idx] += rxq->stats.ipackets; + tmp.q_ibytes[idx] += rxq->stats.ibytes; + tmp.q_errors[idx] += (rxq->stats.idropped + + rxq->stats.rx_nombuf); + } + tmp.ipackets += rxq->stats.ipackets; + tmp.ibytes += rxq->stats.ibytes; + tmp.ierrors += rxq->stats.idropped; + tmp.rx_nombuf += rxq->stats.rx_nombuf; + } + for (i = 0; i != dev->data->nb_tx_queues; ++i) { + struct txq *txq = dev->data->tx_queues[i]; + + if (txq == NULL) + continue; + idx = txq->stats.idx; + if (idx < RTE_ETHDEV_QUEUE_STAT_CNTRS) { + tmp.q_opackets[idx] += txq->stats.opackets; + tmp.q_obytes[idx] += txq->stats.obytes; + tmp.q_errors[idx] += txq->stats.odropped; + } + tmp.opackets += txq->stats.opackets; + tmp.obytes += txq->stats.obytes; + tmp.oerrors += txq->stats.odropped; + } + *stats = tmp; + return 0; +} + +/** + * DPDK callback to clear device statistics. + * + * @param dev + * Pointer to Ethernet device structure. + */ +void +mlx4_stats_reset(struct rte_eth_dev *dev) +{ + unsigned int i; + + for (i = 0; i != dev->data->nb_rx_queues; ++i) { + struct rxq *rxq = dev->data->rx_queues[i]; + + if (rxq) + rxq->stats = (struct mlx4_rxq_stats){ + .idx = rxq->stats.idx, + }; + } + for (i = 0; i != dev->data->nb_tx_queues; ++i) { + struct txq *txq = dev->data->tx_queues[i]; + + if (txq) + txq->stats = (struct mlx4_txq_stats){ + .idx = txq->stats.idx, + }; + } +} + +/** + * DPDK callback to retrieve physical link information. + * + * @param dev + * Pointer to Ethernet device structure. + * @param wait_to_complete + * Wait for request completion (ignored). + * + * @return + * 0 on success, negative errno value otherwise and rte_errno is set. + */ +int +mlx4_link_update(struct rte_eth_dev *dev, int wait_to_complete) +{ + const struct priv *priv = dev->data->dev_private; + struct ethtool_cmd edata = { + .cmd = ETHTOOL_GSET, + }; + struct ifreq ifr; + struct rte_eth_link dev_link; + int link_speed = 0; + + if (priv == NULL) { + rte_errno = EINVAL; + return -rte_errno; + } + (void)wait_to_complete; + if (mlx4_ifreq(priv, SIOCGIFFLAGS, &ifr)) { + WARN("ioctl(SIOCGIFFLAGS) failed: %s", strerror(rte_errno)); + return -rte_errno; + } + memset(&dev_link, 0, sizeof(dev_link)); + dev_link.link_status = ((ifr.ifr_flags & IFF_UP) && + (ifr.ifr_flags & IFF_RUNNING)); + ifr.ifr_data = (void *)&edata; + if (mlx4_ifreq(priv, SIOCETHTOOL, &ifr)) { + WARN("ioctl(SIOCETHTOOL, ETHTOOL_GSET) failed: %s", + strerror(rte_errno)); + return -rte_errno; + } + link_speed = ethtool_cmd_speed(&edata); + if (link_speed == -1) + dev_link.link_speed = 0; + else + dev_link.link_speed = link_speed; + dev_link.link_duplex = ((edata.duplex == DUPLEX_HALF) ? + ETH_LINK_HALF_DUPLEX : ETH_LINK_FULL_DUPLEX); + dev_link.link_autoneg = !(dev->data->dev_conf.link_speeds & + ETH_LINK_SPEED_FIXED); + dev->data->dev_link = dev_link; + return 0; +} + +/** + * DPDK callback to get flow control status. + * + * @param dev + * Pointer to Ethernet device structure. + * @param[out] fc_conf + * Flow control output buffer. + * + * @return + * 0 on success, negative errno value otherwise and rte_errno is set. + */ +int +mlx4_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) +{ + struct priv *priv = dev->data->dev_private; + struct ifreq ifr; + struct ethtool_pauseparam ethpause = { + .cmd = ETHTOOL_GPAUSEPARAM, + }; + int ret; + + ifr.ifr_data = (void *)ðpause; + if (mlx4_ifreq(priv, SIOCETHTOOL, &ifr)) { + ret = rte_errno; + WARN("ioctl(SIOCETHTOOL, ETHTOOL_GPAUSEPARAM)" + " failed: %s", + strerror(rte_errno)); + goto out; + } + fc_conf->autoneg = ethpause.autoneg; + if (ethpause.rx_pause && ethpause.tx_pause) + fc_conf->mode = RTE_FC_FULL; + else if (ethpause.rx_pause) + fc_conf->mode = RTE_FC_RX_PAUSE; + else if (ethpause.tx_pause) + fc_conf->mode = RTE_FC_TX_PAUSE; + else + fc_conf->mode = RTE_FC_NONE; + ret = 0; +out: + assert(ret >= 0); + return -ret; +} + +/** + * DPDK callback to modify flow control parameters. + * + * @param dev + * Pointer to Ethernet device structure. + * @param[in] fc_conf + * Flow control parameters. + * + * @return + * 0 on success, negative errno value otherwise and rte_errno is set. + */ +int +mlx4_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) +{ + struct priv *priv = dev->data->dev_private; + struct ifreq ifr; + struct ethtool_pauseparam ethpause = { + .cmd = ETHTOOL_SPAUSEPARAM, + }; + int ret; + + ifr.ifr_data = (void *)ðpause; + ethpause.autoneg = fc_conf->autoneg; + if (((fc_conf->mode & RTE_FC_FULL) == RTE_FC_FULL) || + (fc_conf->mode & RTE_FC_RX_PAUSE)) + ethpause.rx_pause = 1; + else + ethpause.rx_pause = 0; + if (((fc_conf->mode & RTE_FC_FULL) == RTE_FC_FULL) || + (fc_conf->mode & RTE_FC_TX_PAUSE)) + ethpause.tx_pause = 1; + else + ethpause.tx_pause = 0; + if (mlx4_ifreq(priv, SIOCETHTOOL, &ifr)) { + ret = rte_errno; + WARN("ioctl(SIOCETHTOOL, ETHTOOL_SPAUSEPARAM)" + " failed: %s", + strerror(rte_errno)); + goto out; + } + ret = 0; +out: + assert(ret >= 0); + return -ret; +} + +/** + * DPDK callback to retrieve the received packet types that are recognized + * by the device. + * + * @param dev + * Pointer to Ethernet device structure. + * + * @return + * Pointer to an array of recognized packet types if in Rx burst mode, + * NULL otherwise. + */ +const uint32_t * +mlx4_dev_supported_ptypes_get(struct rte_eth_dev *dev) +{ + static const uint32_t ptypes[] = { + /* refers to rxq_cq_to_pkt_type() */ + RTE_PTYPE_L2_ETHER, + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN, + RTE_PTYPE_L3_IPV6_EXT_UNKNOWN, + RTE_PTYPE_L4_FRAG, + RTE_PTYPE_L4_TCP, + RTE_PTYPE_L4_UDP, + RTE_PTYPE_UNKNOWN + }; + static const uint32_t ptypes_l2tun[] = { + /* refers to rxq_cq_to_pkt_type() */ + RTE_PTYPE_L2_ETHER, + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN, + RTE_PTYPE_L3_IPV6_EXT_UNKNOWN, + RTE_PTYPE_L4_FRAG, + RTE_PTYPE_L4_TCP, + RTE_PTYPE_L4_UDP, + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN, + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN, + RTE_PTYPE_UNKNOWN + }; + struct priv *priv = dev->data->dev_private; + + if (dev->rx_pkt_burst == mlx4_rx_burst) { + if (priv->hw_csum_l2tun) + return ptypes_l2tun; + else + return ptypes; + } + return NULL; +} diff --git a/dpdk/drivers/net/mlx4/mlx4_flow.c b/dpdk/drivers/net/mlx4/mlx4_flow.c new file mode 100644 index 00000000..e81e24db --- /dev/null +++ b/dpdk/drivers/net/mlx4/mlx4_flow.c @@ -0,0 +1,1566 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of 6WIND S.A. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** + * @file + * Flow API operations for mlx4 driver. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +/* Verbs headers do not support -pedantic. */ +#ifdef PEDANTIC +#pragma GCC diagnostic ignored "-Wpedantic" +#endif +#include +#ifdef PEDANTIC +#pragma GCC diagnostic error "-Wpedantic" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include + +/* PMD headers. */ +#include "mlx4.h" +#include "mlx4_flow.h" +#include "mlx4_rxtx.h" +#include "mlx4_utils.h" + +/** Static initializer for a list of subsequent item types. */ +#define NEXT_ITEM(...) \ + (const enum rte_flow_item_type []){ \ + __VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \ + } + +/** Processor structure associated with a flow item. */ +struct mlx4_flow_proc_item { + /** Bit-mask for fields supported by this PMD. */ + const void *mask_support; + /** Bit-mask to use when @p item->mask is not provided. */ + const void *mask_default; + /** Size in bytes for @p mask_support and @p mask_default. */ + const unsigned int mask_sz; + /** Merge a pattern item into a flow rule handle. */ + int (*merge)(struct rte_flow *flow, + const struct rte_flow_item *item, + const struct mlx4_flow_proc_item *proc, + struct rte_flow_error *error); + /** Size in bytes of the destination structure. */ + const unsigned int dst_sz; + /** List of possible subsequent items. */ + const enum rte_flow_item_type *const next_item; +}; + +/** Shared resources for drop flow rules. */ +struct mlx4_drop { + struct ibv_qp *qp; /**< QP target. */ + struct ibv_cq *cq; /**< CQ associated with above QP. */ + struct priv *priv; /**< Back pointer to private data. */ + uint32_t refcnt; /**< Reference count. */ +}; + +/** + * Convert DPDK RSS hash fields to their Verbs equivalent. + * + * @param rss_hf + * Hash fields in DPDK format (see struct rte_eth_rss_conf). + * + * @return + * A valid Verbs RSS hash fields mask for mlx4 on success, (uint64_t)-1 + * otherwise and rte_errno is set. + */ +static uint64_t +mlx4_conv_rss_hf(uint64_t rss_hf) +{ + enum { IPV4, IPV6, TCP, UDP, }; + const uint64_t in[] = { + [IPV4] = (ETH_RSS_IPV4 | + ETH_RSS_FRAG_IPV4 | + ETH_RSS_NONFRAG_IPV4_TCP | + ETH_RSS_NONFRAG_IPV4_UDP | + ETH_RSS_NONFRAG_IPV4_OTHER), + [IPV6] = (ETH_RSS_IPV6 | + ETH_RSS_FRAG_IPV6 | + ETH_RSS_NONFRAG_IPV6_TCP | + ETH_RSS_NONFRAG_IPV6_UDP | + ETH_RSS_NONFRAG_IPV6_OTHER | + ETH_RSS_IPV6_EX | + ETH_RSS_IPV6_TCP_EX | + ETH_RSS_IPV6_UDP_EX), + [TCP] = (ETH_RSS_NONFRAG_IPV4_TCP | + ETH_RSS_NONFRAG_IPV6_TCP | + ETH_RSS_IPV6_TCP_EX), + /* + * UDP support is temporarily disabled due to an + * implementation issue in the kernel. + */ + [UDP] = 0, + }; + const uint64_t out[RTE_DIM(in)] = { + [IPV4] = IBV_RX_HASH_SRC_IPV4 | IBV_RX_HASH_DST_IPV4, + [IPV6] = IBV_RX_HASH_SRC_IPV6 | IBV_RX_HASH_DST_IPV6, + [TCP] = IBV_RX_HASH_SRC_PORT_TCP | IBV_RX_HASH_DST_PORT_TCP, + [UDP] = IBV_RX_HASH_SRC_PORT_UDP | IBV_RX_HASH_DST_PORT_UDP, + }; + uint64_t seen = 0; + uint64_t conv = 0; + unsigned int i; + + for (i = 0; i != RTE_DIM(in); ++i) + if (rss_hf & in[i]) { + seen |= rss_hf & in[i]; + conv |= out[i]; + } + if (!(rss_hf & ~seen)) + return conv; + rte_errno = ENOTSUP; + return (uint64_t)-1; +} + +/** + * Merge Ethernet pattern item into flow rule handle. + * + * Additional mlx4-specific constraints on supported fields: + * + * - No support for partial masks, except in the specific case of matching + * all multicast traffic (@p spec->dst and @p mask->dst equal to + * 01:00:00:00:00:00). + * - Not providing @p item->spec or providing an empty @p mask->dst is + * *only* supported if the rule doesn't specify additional matching + * criteria (i.e. rule is promiscuous-like). + * + * @param[in, out] flow + * Flow rule handle to update. + * @param[in] item + * Pattern item to merge. + * @param[in] proc + * Associated item-processing object. + * @param[out] error + * Perform verbose error reporting if not NULL. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +mlx4_flow_merge_eth(struct rte_flow *flow, + const struct rte_flow_item *item, + const struct mlx4_flow_proc_item *proc, + struct rte_flow_error *error) +{ + const struct rte_flow_item_eth *spec = item->spec; + const struct rte_flow_item_eth *mask = + spec ? (item->mask ? item->mask : proc->mask_default) : NULL; + struct ibv_flow_spec_eth *eth; + const char *msg; + unsigned int i; + + if (!mask) { + flow->promisc = 1; + } else { + uint32_t sum_dst = 0; + uint32_t sum_src = 0; + + for (i = 0; i != sizeof(mask->dst.addr_bytes); ++i) { + sum_dst += mask->dst.addr_bytes[i]; + sum_src += mask->src.addr_bytes[i]; + } + if (sum_src) { + msg = "mlx4 does not support source MAC matching"; + goto error; + } else if (!sum_dst) { + flow->promisc = 1; + } else if (sum_dst == 1 && mask->dst.addr_bytes[0] == 1) { + if (!(spec->dst.addr_bytes[0] & 1)) { + msg = "mlx4 does not support the explicit" + " exclusion of all multicast traffic"; + goto error; + } + flow->allmulti = 1; + } else if (sum_dst != (UINT8_C(0xff) * ETHER_ADDR_LEN)) { + msg = "mlx4 does not support matching partial" + " Ethernet fields"; + goto error; + } + } + if (!flow->ibv_attr) + return 0; + if (flow->promisc) { + flow->ibv_attr->type = IBV_FLOW_ATTR_ALL_DEFAULT; + return 0; + } + if (flow->allmulti) { + flow->ibv_attr->type = IBV_FLOW_ATTR_MC_DEFAULT; + return 0; + } + ++flow->ibv_attr->num_of_specs; + eth = (void *)((uintptr_t)flow->ibv_attr + flow->ibv_attr_size); + *eth = (struct ibv_flow_spec_eth) { + .type = IBV_FLOW_SPEC_ETH, + .size = sizeof(*eth), + }; + memcpy(eth->val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN); + memcpy(eth->mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN); + /* Remove unwanted bits from values. */ + for (i = 0; i < ETHER_ADDR_LEN; ++i) { + eth->val.dst_mac[i] &= eth->mask.dst_mac[i]; + } + return 0; +error: + return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, + item, msg); +} + +/** + * Merge VLAN pattern item into flow rule handle. + * + * Additional mlx4-specific constraints on supported fields: + * + * - Matching *all* VLAN traffic by omitting @p item->spec or providing an + * empty @p item->mask would also include non-VLAN traffic. Doing so is + * therefore unsupported. + * - No support for partial masks. + * + * @param[in, out] flow + * Flow rule handle to update. + * @param[in] item + * Pattern item to merge. + * @param[in] proc + * Associated item-processing object. + * @param[out] error + * Perform verbose error reporting if not NULL. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +mlx4_flow_merge_vlan(struct rte_flow *flow, + const struct rte_flow_item *item, + const struct mlx4_flow_proc_item *proc, + struct rte_flow_error *error) +{ + const struct rte_flow_item_vlan *spec = item->spec; + const struct rte_flow_item_vlan *mask = + spec ? (item->mask ? item->mask : proc->mask_default) : NULL; + struct ibv_flow_spec_eth *eth; + const char *msg; + + if (!mask || !mask->tci) { + msg = "mlx4 cannot match all VLAN traffic while excluding" + " non-VLAN traffic, TCI VID must be specified"; + goto error; + } + if (mask->tci != RTE_BE16(0x0fff)) { + msg = "mlx4 does not support partial TCI VID matching"; + goto error; + } + if (!flow->ibv_attr) + return 0; + eth = (void *)((uintptr_t)flow->ibv_attr + flow->ibv_attr_size - + sizeof(*eth)); + eth->val.vlan_tag = spec->tci; + eth->mask.vlan_tag = mask->tci; + eth->val.vlan_tag &= eth->mask.vlan_tag; + return 0; +error: + return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, + item, msg); +} + +/** + * Merge IPv4 pattern item into flow rule handle. + * + * Additional mlx4-specific constraints on supported fields: + * + * - No support for partial masks. + * + * @param[in, out] flow + * Flow rule handle to update. + * @param[in] item + * Pattern item to merge. + * @param[in] proc + * Associated item-processing object. + * @param[out] error + * Perform verbose error reporting if not NULL. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +mlx4_flow_merge_ipv4(struct rte_flow *flow, + const struct rte_flow_item *item, + const struct mlx4_flow_proc_item *proc, + struct rte_flow_error *error) +{ + const struct rte_flow_item_ipv4 *spec = item->spec; + const struct rte_flow_item_ipv4 *mask = + spec ? (item->mask ? item->mask : proc->mask_default) : NULL; + struct ibv_flow_spec_ipv4 *ipv4; + const char *msg; + + if (mask && + ((uint32_t)(mask->hdr.src_addr + 1) > UINT32_C(1) || + (uint32_t)(mask->hdr.dst_addr + 1) > UINT32_C(1))) { + msg = "mlx4 does not support matching partial IPv4 fields"; + goto error; + } + if (!flow->ibv_attr) + return 0; + ++flow->ibv_attr->num_of_specs; + ipv4 = (void *)((uintptr_t)flow->ibv_attr + flow->ibv_attr_size); + *ipv4 = (struct ibv_flow_spec_ipv4) { + .type = IBV_FLOW_SPEC_IPV4, + .size = sizeof(*ipv4), + }; + if (!spec) + return 0; + ipv4->val = (struct ibv_flow_ipv4_filter) { + .src_ip = spec->hdr.src_addr, + .dst_ip = spec->hdr.dst_addr, + }; + ipv4->mask = (struct ibv_flow_ipv4_filter) { + .src_ip = mask->hdr.src_addr, + .dst_ip = mask->hdr.dst_addr, + }; + /* Remove unwanted bits from values. */ + ipv4->val.src_ip &= ipv4->mask.src_ip; + ipv4->val.dst_ip &= ipv4->mask.dst_ip; + return 0; +error: + return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, + item, msg); +} + +/** + * Merge UDP pattern item into flow rule handle. + * + * Additional mlx4-specific constraints on supported fields: + * + * - No support for partial masks. + * + * @param[in, out] flow + * Flow rule handle to update. + * @param[in] item + * Pattern item to merge. + * @param[in] proc + * Associated item-processing object. + * @param[out] error + * Perform verbose error reporting if not NULL. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +mlx4_flow_merge_udp(struct rte_flow *flow, + const struct rte_flow_item *item, + const struct mlx4_flow_proc_item *proc, + struct rte_flow_error *error) +{ + const struct rte_flow_item_udp *spec = item->spec; + const struct rte_flow_item_udp *mask = + spec ? (item->mask ? item->mask : proc->mask_default) : NULL; + struct ibv_flow_spec_tcp_udp *udp; + const char *msg; + + if (mask && + ((uint16_t)(mask->hdr.src_port + 1) > UINT16_C(1) || + (uint16_t)(mask->hdr.dst_port + 1) > UINT16_C(1))) { + msg = "mlx4 does not support matching partial UDP fields"; + goto error; + } + if (!flow->ibv_attr) + return 0; + ++flow->ibv_attr->num_of_specs; + udp = (void *)((uintptr_t)flow->ibv_attr + flow->ibv_attr_size); + *udp = (struct ibv_flow_spec_tcp_udp) { + .type = IBV_FLOW_SPEC_UDP, + .size = sizeof(*udp), + }; + if (!spec) + return 0; + udp->val.dst_port = spec->hdr.dst_port; + udp->val.src_port = spec->hdr.src_port; + udp->mask.dst_port = mask->hdr.dst_port; + udp->mask.src_port = mask->hdr.src_port; + /* Remove unwanted bits from values. */ + udp->val.src_port &= udp->mask.src_port; + udp->val.dst_port &= udp->mask.dst_port; + return 0; +error: + return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, + item, msg); +} + +/** + * Merge TCP pattern item into flow rule handle. + * + * Additional mlx4-specific constraints on supported fields: + * + * - No support for partial masks. + * + * @param[in, out] flow + * Flow rule handle to update. + * @param[in] item + * Pattern item to merge. + * @param[in] proc + * Associated item-processing object. + * @param[out] error + * Perform verbose error reporting if not NULL. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +mlx4_flow_merge_tcp(struct rte_flow *flow, + const struct rte_flow_item *item, + const struct mlx4_flow_proc_item *proc, + struct rte_flow_error *error) +{ + const struct rte_flow_item_tcp *spec = item->spec; + const struct rte_flow_item_tcp *mask = + spec ? (item->mask ? item->mask : proc->mask_default) : NULL; + struct ibv_flow_spec_tcp_udp *tcp; + const char *msg; + + if (mask && + ((uint16_t)(mask->hdr.src_port + 1) > UINT16_C(1) || + (uint16_t)(mask->hdr.dst_port + 1) > UINT16_C(1))) { + msg = "mlx4 does not support matching partial TCP fields"; + goto error; + } + if (!flow->ibv_attr) + return 0; + ++flow->ibv_attr->num_of_specs; + tcp = (void *)((uintptr_t)flow->ibv_attr + flow->ibv_attr_size); + *tcp = (struct ibv_flow_spec_tcp_udp) { + .type = IBV_FLOW_SPEC_TCP, + .size = sizeof(*tcp), + }; + if (!spec) + return 0; + tcp->val.dst_port = spec->hdr.dst_port; + tcp->val.src_port = spec->hdr.src_port; + tcp->mask.dst_port = mask->hdr.dst_port; + tcp->mask.src_port = mask->hdr.src_port; + /* Remove unwanted bits from values. */ + tcp->val.src_port &= tcp->mask.src_port; + tcp->val.dst_port &= tcp->mask.dst_port; + return 0; +error: + return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, + item, msg); +} + +/** + * Perform basic sanity checks on a pattern item. + * + * @param[in] item + * Item specification. + * @param[in] proc + * Associated item-processing object. + * @param[out] error + * Perform verbose error reporting if not NULL. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +mlx4_flow_item_check(const struct rte_flow_item *item, + const struct mlx4_flow_proc_item *proc, + struct rte_flow_error *error) +{ + const uint8_t *mask; + unsigned int i; + + /* item->last and item->mask cannot exist without item->spec. */ + if (!item->spec && (item->mask || item->last)) + return rte_flow_error_set + (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item, + "\"mask\" or \"last\" field provided without a" + " corresponding \"spec\""); + /* No spec, no mask, no problem. */ + if (!item->spec) + return 0; + mask = item->mask ? + (const uint8_t *)item->mask : + (const uint8_t *)proc->mask_default; + assert(mask); + /* + * Single-pass check to make sure that: + * - Mask is supported, no bits are set outside proc->mask_support. + * - Both item->spec and item->last are included in mask. + */ + for (i = 0; i != proc->mask_sz; ++i) { + if (!mask[i]) + continue; + if ((mask[i] | ((const uint8_t *)proc->mask_support)[i]) != + ((const uint8_t *)proc->mask_support)[i]) + return rte_flow_error_set + (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, + item, "unsupported field found in \"mask\""); + if (item->last && + (((const uint8_t *)item->spec)[i] & mask[i]) != + (((const uint8_t *)item->last)[i] & mask[i])) + return rte_flow_error_set + (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, + item, + "range between \"spec\" and \"last\"" + " is larger than \"mask\""); + } + return 0; +} + +/** Graph of supported items and associated actions. */ +static const struct mlx4_flow_proc_item mlx4_flow_proc_item_list[] = { + [RTE_FLOW_ITEM_TYPE_END] = { + .next_item = NEXT_ITEM(RTE_FLOW_ITEM_TYPE_ETH), + }, + [RTE_FLOW_ITEM_TYPE_ETH] = { + .next_item = NEXT_ITEM(RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4), + .mask_support = &(const struct rte_flow_item_eth){ + /* Only destination MAC can be matched. */ + .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", + }, + .mask_default = &rte_flow_item_eth_mask, + .mask_sz = sizeof(struct rte_flow_item_eth), + .merge = mlx4_flow_merge_eth, + .dst_sz = sizeof(struct ibv_flow_spec_eth), + }, + [RTE_FLOW_ITEM_TYPE_VLAN] = { + .next_item = NEXT_ITEM(RTE_FLOW_ITEM_TYPE_IPV4), + .mask_support = &(const struct rte_flow_item_vlan){ + /* Only TCI VID matching is supported. */ + .tci = RTE_BE16(0x0fff), + }, + .mask_default = &rte_flow_item_vlan_mask, + .mask_sz = sizeof(struct rte_flow_item_vlan), + .merge = mlx4_flow_merge_vlan, + .dst_sz = 0, + }, + [RTE_FLOW_ITEM_TYPE_IPV4] = { + .next_item = NEXT_ITEM(RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_TCP), + .mask_support = &(const struct rte_flow_item_ipv4){ + .hdr = { + .src_addr = RTE_BE32(0xffffffff), + .dst_addr = RTE_BE32(0xffffffff), + }, + }, + .mask_default = &rte_flow_item_ipv4_mask, + .mask_sz = sizeof(struct rte_flow_item_ipv4), + .merge = mlx4_flow_merge_ipv4, + .dst_sz = sizeof(struct ibv_flow_spec_ipv4), + }, + [RTE_FLOW_ITEM_TYPE_UDP] = { + .mask_support = &(const struct rte_flow_item_udp){ + .hdr = { + .src_port = RTE_BE16(0xffff), + .dst_port = RTE_BE16(0xffff), + }, + }, + .mask_default = &rte_flow_item_udp_mask, + .mask_sz = sizeof(struct rte_flow_item_udp), + .merge = mlx4_flow_merge_udp, + .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp), + }, + [RTE_FLOW_ITEM_TYPE_TCP] = { + .mask_support = &(const struct rte_flow_item_tcp){ + .hdr = { + .src_port = RTE_BE16(0xffff), + .dst_port = RTE_BE16(0xffff), + }, + }, + .mask_default = &rte_flow_item_tcp_mask, + .mask_sz = sizeof(struct rte_flow_item_tcp), + .merge = mlx4_flow_merge_tcp, + .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp), + }, +}; + +/** + * Make sure a flow rule is supported and initialize associated structure. + * + * @param priv + * Pointer to private structure. + * @param[in] attr + * Flow rule attributes. + * @param[in] pattern + * Pattern specification (list terminated by the END pattern item). + * @param[in] actions + * Associated actions (list terminated by the END action). + * @param[out] error + * Perform verbose error reporting if not NULL. + * @param[in, out] addr + * Buffer where the resulting flow rule handle pointer must be stored. + * If NULL, stop processing after validation stage. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +mlx4_flow_prepare(struct priv *priv, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error, + struct rte_flow **addr) +{ + const struct rte_flow_item *item; + const struct rte_flow_action *action; + const struct mlx4_flow_proc_item *proc; + struct rte_flow temp = { .ibv_attr_size = sizeof(*temp.ibv_attr) }; + struct rte_flow *flow = &temp; + const char *msg = NULL; + + if (attr->group) + return rte_flow_error_set + (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR_GROUP, + NULL, "groups are not supported"); + if (attr->priority > MLX4_FLOW_PRIORITY_LAST) + return rte_flow_error_set + (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, + NULL, "maximum priority level is " + MLX4_STR_EXPAND(MLX4_FLOW_PRIORITY_LAST)); + if (attr->egress) + return rte_flow_error_set + (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, + NULL, "egress is not supported"); + if (!attr->ingress) + return rte_flow_error_set + (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, + NULL, "only ingress is supported"); +fill: + proc = mlx4_flow_proc_item_list; + /* Go over pattern. */ + for (item = pattern; item->type; ++item) { + const struct mlx4_flow_proc_item *next = NULL; + unsigned int i; + int err; + + if (item->type == RTE_FLOW_ITEM_TYPE_VOID) + continue; + if (item->type == MLX4_FLOW_ITEM_TYPE_INTERNAL) { + flow->internal = 1; + continue; + } + if (flow->promisc || flow->allmulti) { + msg = "mlx4 does not support additional matching" + " criteria combined with indiscriminate" + " matching on Ethernet headers"; + goto exit_item_not_supported; + } + for (i = 0; proc->next_item && proc->next_item[i]; ++i) { + if (proc->next_item[i] == item->type) { + next = &mlx4_flow_proc_item_list[item->type]; + break; + } + } + if (!next) + goto exit_item_not_supported; + proc = next; + /* + * Perform basic sanity checks only once, while handle is + * not allocated. + */ + if (flow == &temp) { + err = mlx4_flow_item_check(item, proc, error); + if (err) + return err; + } + if (proc->merge) { + err = proc->merge(flow, item, proc, error); + if (err) + return err; + } + flow->ibv_attr_size += proc->dst_sz; + } + /* Go over actions list. */ + for (action = actions; action->type; ++action) { + switch (action->type) { + const struct rte_flow_action_queue *queue; + const struct rte_flow_action_rss *rss; + const struct rte_eth_rss_conf *rss_conf; + unsigned int i; + + case RTE_FLOW_ACTION_TYPE_VOID: + continue; + case RTE_FLOW_ACTION_TYPE_DROP: + flow->drop = 1; + break; + case RTE_FLOW_ACTION_TYPE_QUEUE: + if (flow->rss) + break; + queue = action->conf; + if (queue->index >= priv->dev->data->nb_rx_queues) { + msg = "queue target index beyond number of" + " configured Rx queues"; + goto exit_action_not_supported; + } + flow->rss = mlx4_rss_get + (priv, 0, mlx4_rss_hash_key_default, 1, + &queue->index); + if (!flow->rss) { + msg = "not enough resources for additional" + " single-queue RSS context"; + goto exit_action_not_supported; + } + break; + case RTE_FLOW_ACTION_TYPE_RSS: + if (flow->rss) + break; + rss = action->conf; + /* Default RSS configuration if none is provided. */ + rss_conf = + rss->rss_conf ? + rss->rss_conf : + &(struct rte_eth_rss_conf){ + .rss_key = mlx4_rss_hash_key_default, + .rss_key_len = MLX4_RSS_HASH_KEY_SIZE, + .rss_hf = (ETH_RSS_IPV4 | + ETH_RSS_NONFRAG_IPV4_TCP | + ETH_RSS_IPV6 | + ETH_RSS_NONFRAG_IPV6_TCP), + }; + /* Sanity checks. */ + for (i = 0; i < rss->num; ++i) + if (rss->queue[i] >= + priv->dev->data->nb_rx_queues) + break; + if (i != rss->num) { + msg = "queue index target beyond number of" + " configured Rx queues"; + goto exit_action_not_supported; + } + if (!rte_is_power_of_2(rss->num)) { + msg = "for RSS, mlx4 requires the number of" + " queues to be a power of two"; + goto exit_action_not_supported; + } + if (rss_conf->rss_key_len != + sizeof(flow->rss->key)) { + msg = "mlx4 supports exactly one RSS hash key" + " length: " + MLX4_STR_EXPAND(MLX4_RSS_HASH_KEY_SIZE); + goto exit_action_not_supported; + } + for (i = 1; i < rss->num; ++i) + if (rss->queue[i] - rss->queue[i - 1] != 1) + break; + if (i != rss->num) { + msg = "mlx4 requires RSS contexts to use" + " consecutive queue indices only"; + goto exit_action_not_supported; + } + if (rss->queue[0] % rss->num) { + msg = "mlx4 requires the first queue of a RSS" + " context to be aligned on a multiple" + " of the context size"; + goto exit_action_not_supported; + } + flow->rss = mlx4_rss_get + (priv, mlx4_conv_rss_hf(rss_conf->rss_hf), + rss_conf->rss_key, rss->num, rss->queue); + if (!flow->rss) { + msg = "either invalid parameters or not enough" + " resources for additional multi-queue" + " RSS context"; + goto exit_action_not_supported; + } + break; + default: + goto exit_action_not_supported; + } + } + if (!flow->rss && !flow->drop) + return rte_flow_error_set + (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "no valid action"); + /* Validation ends here. */ + if (!addr) { + if (flow->rss) + mlx4_rss_put(flow->rss); + return 0; + } + if (flow == &temp) { + /* Allocate proper handle based on collected data. */ + const struct mlx4_malloc_vec vec[] = { + { + .align = alignof(struct rte_flow), + .size = sizeof(*flow), + .addr = (void **)&flow, + }, + { + .align = alignof(struct ibv_flow_attr), + .size = temp.ibv_attr_size, + .addr = (void **)&temp.ibv_attr, + }, + }; + + if (!mlx4_zmallocv(__func__, vec, RTE_DIM(vec))) + return rte_flow_error_set + (error, -rte_errno, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "flow rule handle allocation failure"); + /* Most fields will be updated by second pass. */ + *flow = (struct rte_flow){ + .ibv_attr = temp.ibv_attr, + .ibv_attr_size = sizeof(*flow->ibv_attr), + .rss = temp.rss, + }; + *flow->ibv_attr = (struct ibv_flow_attr){ + .type = IBV_FLOW_ATTR_NORMAL, + .size = sizeof(*flow->ibv_attr), + .priority = attr->priority, + .port = priv->port, + }; + goto fill; + } + *addr = flow; + return 0; +exit_item_not_supported: + return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, + item, msg ? msg : "item not supported"); +exit_action_not_supported: + return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, + action, msg ? msg : "action not supported"); +} + +/** + * Validate a flow supported by the NIC. + * + * @see rte_flow_validate() + * @see rte_flow_ops + */ +static int +mlx4_flow_validate(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error) +{ + struct priv *priv = dev->data->dev_private; + + return mlx4_flow_prepare(priv, attr, pattern, actions, error, NULL); +} + +/** + * Get a drop flow rule resources instance. + * + * @param priv + * Pointer to private structure. + * + * @return + * Pointer to drop flow resources on success, NULL otherwise and rte_errno + * is set. + */ +static struct mlx4_drop * +mlx4_drop_get(struct priv *priv) +{ + struct mlx4_drop *drop = priv->drop; + + if (drop) { + assert(drop->refcnt); + assert(drop->priv == priv); + ++drop->refcnt; + return drop; + } + drop = rte_malloc(__func__, sizeof(*drop), 0); + if (!drop) + goto error; + *drop = (struct mlx4_drop){ + .priv = priv, + .refcnt = 1, + }; + drop->cq = ibv_create_cq(priv->ctx, 1, NULL, NULL, 0); + if (!drop->cq) + goto error; + drop->qp = ibv_create_qp(priv->pd, + &(struct ibv_qp_init_attr){ + .send_cq = drop->cq, + .recv_cq = drop->cq, + .qp_type = IBV_QPT_RAW_PACKET, + }); + if (!drop->qp) + goto error; + priv->drop = drop; + return drop; +error: + if (drop->qp) + claim_zero(ibv_destroy_qp(drop->qp)); + if (drop->cq) + claim_zero(ibv_destroy_cq(drop->cq)); + if (drop) + rte_free(drop); + rte_errno = ENOMEM; + return NULL; +} + +/** + * Give back a drop flow rule resources instance. + * + * @param drop + * Pointer to drop flow rule resources. + */ +static void +mlx4_drop_put(struct mlx4_drop *drop) +{ + assert(drop->refcnt); + if (--drop->refcnt) + return; + drop->priv->drop = NULL; + claim_zero(ibv_destroy_qp(drop->qp)); + claim_zero(ibv_destroy_cq(drop->cq)); + rte_free(drop); +} + +/** + * Toggle a configured flow rule. + * + * @param priv + * Pointer to private structure. + * @param flow + * Flow rule handle to toggle. + * @param enable + * Whether associated Verbs flow must be created or removed. + * @param[out] error + * Perform verbose error reporting if not NULL. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +mlx4_flow_toggle(struct priv *priv, + struct rte_flow *flow, + int enable, + struct rte_flow_error *error) +{ + struct ibv_qp *qp = NULL; + const char *msg; + int err; + + if (!enable) { + if (!flow->ibv_flow) + return 0; + claim_zero(ibv_destroy_flow(flow->ibv_flow)); + flow->ibv_flow = NULL; + if (flow->drop) + mlx4_drop_put(priv->drop); + else if (flow->rss) + mlx4_rss_detach(flow->rss); + return 0; + } + assert(flow->ibv_attr); + if (!flow->internal && + !priv->isolated && + flow->ibv_attr->priority == MLX4_FLOW_PRIORITY_LAST) { + if (flow->ibv_flow) { + claim_zero(ibv_destroy_flow(flow->ibv_flow)); + flow->ibv_flow = NULL; + if (flow->drop) + mlx4_drop_put(priv->drop); + else if (flow->rss) + mlx4_rss_detach(flow->rss); + } + err = EACCES; + msg = ("priority level " + MLX4_STR_EXPAND(MLX4_FLOW_PRIORITY_LAST) + " is reserved when not in isolated mode"); + goto error; + } + if (flow->rss) { + struct mlx4_rss *rss = flow->rss; + int missing = 0; + unsigned int i; + + /* Stop at the first nonexistent target queue. */ + for (i = 0; i != rss->queues; ++i) + if (rss->queue_id[i] >= + priv->dev->data->nb_rx_queues || + !priv->dev->data->rx_queues[rss->queue_id[i]]) { + missing = 1; + break; + } + if (flow->ibv_flow) { + if (missing ^ !flow->drop) + return 0; + /* Verbs flow needs updating. */ + claim_zero(ibv_destroy_flow(flow->ibv_flow)); + flow->ibv_flow = NULL; + if (flow->drop) + mlx4_drop_put(priv->drop); + else + mlx4_rss_detach(rss); + } + if (!missing) { + err = mlx4_rss_attach(rss); + if (err) { + err = -err; + msg = "cannot create indirection table or hash" + " QP to associate flow rule with"; + goto error; + } + qp = rss->qp; + } + /* A missing target queue drops traffic implicitly. */ + flow->drop = missing; + } + if (flow->drop) { + if (flow->ibv_flow) + return 0; + mlx4_drop_get(priv); + if (!priv->drop) { + err = rte_errno; + msg = "resources for drop flow rule cannot be created"; + goto error; + } + qp = priv->drop->qp; + } + assert(qp); + if (flow->ibv_flow) + return 0; + flow->ibv_flow = ibv_create_flow(qp, flow->ibv_attr); + if (flow->ibv_flow) + return 0; + if (flow->drop) + mlx4_drop_put(priv->drop); + else if (flow->rss) + mlx4_rss_detach(flow->rss); + err = errno; + msg = "flow rule rejected by device"; +error: + return rte_flow_error_set + (error, err, RTE_FLOW_ERROR_TYPE_HANDLE, flow, msg); +} + +/** + * Create a flow. + * + * @see rte_flow_create() + * @see rte_flow_ops + */ +static struct rte_flow * +mlx4_flow_create(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error) +{ + struct priv *priv = dev->data->dev_private; + struct rte_flow *flow; + int err; + + err = mlx4_flow_prepare(priv, attr, pattern, actions, error, &flow); + if (err) + return NULL; + err = mlx4_flow_toggle(priv, flow, priv->started, error); + if (!err) { + struct rte_flow *curr = LIST_FIRST(&priv->flows); + + /* New rules are inserted after internal ones. */ + if (!curr || !curr->internal) { + LIST_INSERT_HEAD(&priv->flows, flow, next); + } else { + while (LIST_NEXT(curr, next) && + LIST_NEXT(curr, next)->internal) + curr = LIST_NEXT(curr, next); + LIST_INSERT_AFTER(curr, flow, next); + } + return flow; + } + if (flow->rss) + mlx4_rss_put(flow->rss); + rte_flow_error_set(error, -err, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + error->message); + rte_free(flow); + return NULL; +} + +/** + * Configure isolated mode. + * + * @see rte_flow_isolate() + * @see rte_flow_ops + */ +static int +mlx4_flow_isolate(struct rte_eth_dev *dev, + int enable, + struct rte_flow_error *error) +{ + struct priv *priv = dev->data->dev_private; + + if (!!enable == !!priv->isolated) + return 0; + priv->isolated = !!enable; + if (mlx4_flow_sync(priv, error)) { + priv->isolated = !enable; + return -rte_errno; + } + return 0; +} + +/** + * Destroy a flow rule. + * + * @see rte_flow_destroy() + * @see rte_flow_ops + */ +static int +mlx4_flow_destroy(struct rte_eth_dev *dev, + struct rte_flow *flow, + struct rte_flow_error *error) +{ + struct priv *priv = dev->data->dev_private; + int err = mlx4_flow_toggle(priv, flow, 0, error); + + if (err) + return err; + LIST_REMOVE(flow, next); + if (flow->rss) + mlx4_rss_put(flow->rss); + rte_free(flow); + return 0; +} + +/** + * Destroy user-configured flow rules. + * + * This function skips internal flows rules. + * + * @see rte_flow_flush() + * @see rte_flow_ops + */ +static int +mlx4_flow_flush(struct rte_eth_dev *dev, + struct rte_flow_error *error) +{ + struct priv *priv = dev->data->dev_private; + struct rte_flow *flow = LIST_FIRST(&priv->flows); + + while (flow) { + struct rte_flow *next = LIST_NEXT(flow, next); + + if (!flow->internal) + mlx4_flow_destroy(dev, flow, error); + flow = next; + } + return 0; +} + +/** + * Helper function to determine the next configured VLAN filter. + * + * @param priv + * Pointer to private structure. + * @param vlan + * VLAN ID to use as a starting point. + * + * @return + * Next configured VLAN ID or a high value (>= 4096) if there is none. + */ +static uint16_t +mlx4_flow_internal_next_vlan(struct priv *priv, uint16_t vlan) +{ + while (vlan < 4096) { + if (priv->dev->data->vlan_filter_conf.ids[vlan / 64] & + (UINT64_C(1) << (vlan % 64))) + return vlan; + ++vlan; + } + return vlan; +} + +/** + * Generate internal flow rules. + * + * Various flow rules are created depending on the mode the device is in: + * + * 1. Promiscuous: port MAC + catch-all (VLAN filtering is ignored). + * 2. All multicast: port MAC/VLAN + catch-all multicast. + * 3. Otherwise: port MAC/VLAN + broadcast MAC/VLAN. + * + * About MAC flow rules: + * + * - MAC flow rules are generated from @p dev->data->mac_addrs + * (@p priv->mac array). + * - An additional flow rule for Ethernet broadcasts is also generated. + * - All these are per-VLAN if @p dev->data->dev_conf.rxmode.hw_vlan_filter + * is enabled and VLAN filters are configured. + * + * @param priv + * Pointer to private structure. + * @param[out] error + * Perform verbose error reporting if not NULL. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +mlx4_flow_internal(struct priv *priv, struct rte_flow_error *error) +{ + struct rte_flow_attr attr = { + .priority = MLX4_FLOW_PRIORITY_LAST, + .ingress = 1, + }; + struct rte_flow_item_eth eth_spec; + const struct rte_flow_item_eth eth_mask = { + .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", + }; + const struct rte_flow_item_eth eth_allmulti = { + .dst.addr_bytes = "\x01\x00\x00\x00\x00\x00", + }; + struct rte_flow_item_vlan vlan_spec; + const struct rte_flow_item_vlan vlan_mask = { + .tci = RTE_BE16(0x0fff), + }; + struct rte_flow_item pattern[] = { + { + .type = MLX4_FLOW_ITEM_TYPE_INTERNAL, + }, + { + .type = RTE_FLOW_ITEM_TYPE_ETH, + .spec = ð_spec, + .mask = ð_mask, + }, + { + /* Replaced with VLAN if filtering is enabled. */ + .type = RTE_FLOW_ITEM_TYPE_END, + }, + { + .type = RTE_FLOW_ITEM_TYPE_END, + }, + }; + /* + * Round number of queues down to their previous power of 2 to + * comply with RSS context limitations. Extra queues silently do not + * get RSS by default. + */ + uint32_t queues = + rte_align32pow2(priv->dev->data->nb_rx_queues + 1) >> 1; + alignas(struct rte_flow_action_rss) uint8_t rss_conf_data + [offsetof(struct rte_flow_action_rss, queue) + + sizeof(((struct rte_flow_action_rss *)0)->queue[0]) * queues]; + struct rte_flow_action_rss *rss_conf = (void *)rss_conf_data; + struct rte_flow_action actions[] = { + { + .type = RTE_FLOW_ACTION_TYPE_RSS, + .conf = rss_conf, + }, + { + .type = RTE_FLOW_ACTION_TYPE_END, + }, + }; + struct ether_addr *rule_mac = ð_spec.dst; + rte_be16_t *rule_vlan = + priv->dev->data->dev_conf.rxmode.hw_vlan_filter && + !priv->dev->data->promiscuous ? + &vlan_spec.tci : + NULL; + int broadcast = + !priv->dev->data->promiscuous && + !priv->dev->data->all_multicast; + uint16_t vlan = 0; + struct rte_flow *flow; + unsigned int i; + int err = 0; + + /* Nothing to be done if there are no Rx queues. */ + if (!queues) + goto error; + /* Prepare default RSS configuration. */ + *rss_conf = (struct rte_flow_action_rss){ + .rss_conf = NULL, /* Rely on default fallback settings. */ + .num = queues, + }; + for (i = 0; i != queues; ++i) + rss_conf->queue[i] = i; + /* + * Set up VLAN item if filtering is enabled and at least one VLAN + * filter is configured. + */ + if (rule_vlan) { + vlan = mlx4_flow_internal_next_vlan(priv, 0); + if (vlan < 4096) { + pattern[2] = (struct rte_flow_item){ + .type = RTE_FLOW_ITEM_TYPE_VLAN, + .spec = &vlan_spec, + .mask = &vlan_mask, + }; +next_vlan: + *rule_vlan = rte_cpu_to_be_16(vlan); + } else { + rule_vlan = NULL; + } + } + for (i = 0; i != RTE_DIM(priv->mac) + broadcast; ++i) { + const struct ether_addr *mac; + + /* Broadcasts are handled by an extra iteration. */ + if (i < RTE_DIM(priv->mac)) + mac = &priv->mac[i]; + else + mac = ð_mask.dst; + if (is_zero_ether_addr(mac)) + continue; + /* Check if MAC flow rule is already present. */ + for (flow = LIST_FIRST(&priv->flows); + flow && flow->internal; + flow = LIST_NEXT(flow, next)) { + const struct ibv_flow_spec_eth *eth = + (const void *)((uintptr_t)flow->ibv_attr + + sizeof(*flow->ibv_attr)); + unsigned int j; + + if (!flow->mac) + continue; + assert(flow->ibv_attr->type == IBV_FLOW_ATTR_NORMAL); + assert(flow->ibv_attr->num_of_specs == 1); + assert(eth->type == IBV_FLOW_SPEC_ETH); + assert(flow->rss); + if (rule_vlan && + (eth->val.vlan_tag != *rule_vlan || + eth->mask.vlan_tag != RTE_BE16(0x0fff))) + continue; + if (!rule_vlan && eth->mask.vlan_tag) + continue; + for (j = 0; j != sizeof(mac->addr_bytes); ++j) + if (eth->val.dst_mac[j] != mac->addr_bytes[j] || + eth->mask.dst_mac[j] != UINT8_C(0xff) || + eth->val.src_mac[j] != UINT8_C(0x00) || + eth->mask.src_mac[j] != UINT8_C(0x00)) + break; + if (j != sizeof(mac->addr_bytes)) + continue; + if (flow->rss->queues != queues || + memcmp(flow->rss->queue_id, rss_conf->queue, + queues * sizeof(flow->rss->queue_id[0]))) + continue; + break; + } + if (!flow || !flow->internal) { + /* Not found, create a new flow rule. */ + memcpy(rule_mac, mac, sizeof(*mac)); + flow = mlx4_flow_create(priv->dev, &attr, pattern, + actions, error); + if (!flow) { + err = -rte_errno; + goto error; + } + } + flow->select = 1; + flow->mac = 1; + } + if (rule_vlan) { + vlan = mlx4_flow_internal_next_vlan(priv, vlan + 1); + if (vlan < 4096) + goto next_vlan; + } + /* Take care of promiscuous and all multicast flow rules. */ + if (!broadcast) { + for (flow = LIST_FIRST(&priv->flows); + flow && flow->internal; + flow = LIST_NEXT(flow, next)) { + if (priv->dev->data->promiscuous) { + if (flow->promisc) + break; + } else { + assert(priv->dev->data->all_multicast); + if (flow->allmulti) + break; + } + } + if (flow && flow->internal) { + assert(flow->rss); + if (flow->rss->queues != queues || + memcmp(flow->rss->queue_id, rss_conf->queue, + queues * sizeof(flow->rss->queue_id[0]))) + flow = NULL; + } + if (!flow || !flow->internal) { + /* Not found, create a new flow rule. */ + if (priv->dev->data->promiscuous) { + pattern[1].spec = NULL; + pattern[1].mask = NULL; + } else { + assert(priv->dev->data->all_multicast); + pattern[1].spec = ð_allmulti; + pattern[1].mask = ð_allmulti; + } + pattern[2] = pattern[3]; + flow = mlx4_flow_create(priv->dev, &attr, pattern, + actions, error); + if (!flow) { + err = -rte_errno; + goto error; + } + } + assert(flow->promisc || flow->allmulti); + flow->select = 1; + } +error: + /* Clear selection and clean up stale internal flow rules. */ + flow = LIST_FIRST(&priv->flows); + while (flow && flow->internal) { + struct rte_flow *next = LIST_NEXT(flow, next); + + if (!flow->select) + claim_zero(mlx4_flow_destroy(priv->dev, flow, error)); + else + flow->select = 0; + flow = next; + } + return err; +} + +/** + * Synchronize flow rules. + * + * This function synchronizes flow rules with the state of the device by + * taking into account isolated mode and whether target queues are + * configured. + * + * @param priv + * Pointer to private structure. + * @param[out] error + * Perform verbose error reporting if not NULL. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +int +mlx4_flow_sync(struct priv *priv, struct rte_flow_error *error) +{ + struct rte_flow *flow; + int ret; + + /* Internal flow rules are guaranteed to come first in the list. */ + if (priv->isolated) { + /* + * Get rid of them in isolated mode, stop at the first + * non-internal rule found. + */ + for (flow = LIST_FIRST(&priv->flows); + flow && flow->internal; + flow = LIST_FIRST(&priv->flows)) + claim_zero(mlx4_flow_destroy(priv->dev, flow, error)); + } else { + /* Refresh internal rules. */ + ret = mlx4_flow_internal(priv, error); + if (ret) + return ret; + } + /* Toggle the remaining flow rules . */ + LIST_FOREACH(flow, &priv->flows, next) { + ret = mlx4_flow_toggle(priv, flow, priv->started, error); + if (ret) + return ret; + } + if (!priv->started) + assert(!priv->drop); + return 0; +} + +/** + * Clean up all flow rules. + * + * Unlike mlx4_flow_flush(), this function takes care of all remaining flow + * rules regardless of whether they are internal or user-configured. + * + * @param priv + * Pointer to private structure. + */ +void +mlx4_flow_clean(struct priv *priv) +{ + struct rte_flow *flow; + + while ((flow = LIST_FIRST(&priv->flows))) + mlx4_flow_destroy(priv->dev, flow, NULL); + assert(LIST_EMPTY(&priv->rss)); +} + +static const struct rte_flow_ops mlx4_flow_ops = { + .validate = mlx4_flow_validate, + .create = mlx4_flow_create, + .destroy = mlx4_flow_destroy, + .flush = mlx4_flow_flush, + .isolate = mlx4_flow_isolate, +}; + +/** + * Manage filter operations. + * + * @param dev + * Pointer to Ethernet device structure. + * @param filter_type + * Filter type. + * @param filter_op + * Operation to perform. + * @param arg + * Pointer to operation-specific structure. + * + * @return + * 0 on success, negative errno value otherwise and rte_errno is set. + */ +int +mlx4_filter_ctrl(struct rte_eth_dev *dev, + enum rte_filter_type filter_type, + enum rte_filter_op filter_op, + void *arg) +{ + switch (filter_type) { + case RTE_ETH_FILTER_GENERIC: + if (filter_op != RTE_ETH_FILTER_GET) + break; + *(const void **)arg = &mlx4_flow_ops; + return 0; + default: + ERROR("%p: filter type (%d) not supported", + (void *)dev, filter_type); + break; + } + rte_errno = ENOTSUP; + return -rte_errno; +} diff --git a/dpdk/drivers/net/mlx4/mlx4_flow.h b/dpdk/drivers/net/mlx4/mlx4_flow.h new file mode 100644 index 00000000..651fd37b --- /dev/null +++ b/dpdk/drivers/net/mlx4/mlx4_flow.h @@ -0,0 +1,85 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of 6WIND S.A. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef RTE_PMD_MLX4_FLOW_H_ +#define RTE_PMD_MLX4_FLOW_H_ + +#include +#include + +/* Verbs headers do not support -pedantic. */ +#ifdef PEDANTIC +#pragma GCC diagnostic ignored "-Wpedantic" +#endif +#include +#ifdef PEDANTIC +#pragma GCC diagnostic error "-Wpedantic" +#endif + +#include +#include +#include +#include +#include + +/** Last and lowest priority level for a flow rule. */ +#define MLX4_FLOW_PRIORITY_LAST UINT32_C(0xfff) + +/** Meta pattern item used to distinguish internal rules. */ +#define MLX4_FLOW_ITEM_TYPE_INTERNAL ((enum rte_flow_item_type)-1) + +/** PMD-specific (mlx4) definition of a flow rule handle. */ +struct rte_flow { + LIST_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */ + struct ibv_flow *ibv_flow; /**< Verbs flow. */ + struct ibv_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */ + uint32_t ibv_attr_size; /**< Size of Verbs attributes. */ + uint32_t select:1; /**< Used by operations on the linked list. */ + uint32_t internal:1; /**< Internal flow rule outside isolated mode. */ + uint32_t mac:1; /**< Rule associated with a configured MAC address. */ + uint32_t promisc:1; /**< This rule matches everything. */ + uint32_t allmulti:1; /**< This rule matches all multicast traffic. */ + uint32_t drop:1; /**< This rule drops packets. */ + struct mlx4_rss *rss; /**< Rx target. */ +}; + +/* mlx4_flow.c */ + +int mlx4_flow_sync(struct priv *priv, struct rte_flow_error *error); +void mlx4_flow_clean(struct priv *priv); +int mlx4_filter_ctrl(struct rte_eth_dev *dev, + enum rte_filter_type filter_type, + enum rte_filter_op filter_op, + void *arg); + +#endif /* RTE_PMD_MLX4_FLOW_H_ */ diff --git a/dpdk/drivers/net/mlx4/mlx4_intr.c b/dpdk/drivers/net/mlx4/mlx4_intr.c new file mode 100644 index 00000000..50d19769 --- /dev/null +++ b/dpdk/drivers/net/mlx4/mlx4_intr.c @@ -0,0 +1,397 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of 6WIND S.A. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** + * @file + * Interrupts handling for mlx4 driver. + */ + +#include +#include +#include +#include + +/* Verbs headers do not support -pedantic. */ +#ifdef PEDANTIC +#pragma GCC diagnostic ignored "-Wpedantic" +#endif +#include +#ifdef PEDANTIC +#pragma GCC diagnostic error "-Wpedantic" +#endif + +#include +#include +#include +#include +#include + +#include "mlx4.h" +#include "mlx4_rxtx.h" +#include "mlx4_utils.h" + +static int mlx4_link_status_check(struct priv *priv); + +/** + * Clean up Rx interrupts handler. + * + * @param priv + * Pointer to private structure. + */ +static void +mlx4_rx_intr_vec_disable(struct priv *priv) +{ + struct rte_intr_handle *intr_handle = &priv->intr_handle; + + rte_intr_free_epoll_fd(intr_handle); + free(intr_handle->intr_vec); + intr_handle->nb_efd = 0; + intr_handle->intr_vec = NULL; +} + +/** + * Allocate queue vector and fill epoll fd list for Rx interrupts. + * + * @param priv + * Pointer to private structure. + * + * @return + * 0 on success, negative errno value otherwise and rte_errno is set. + */ +static int +mlx4_rx_intr_vec_enable(struct priv *priv) +{ + unsigned int i; + unsigned int rxqs_n = priv->dev->data->nb_rx_queues; + unsigned int n = RTE_MIN(rxqs_n, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID); + unsigned int count = 0; + struct rte_intr_handle *intr_handle = &priv->intr_handle; + + mlx4_rx_intr_vec_disable(priv); + intr_handle->intr_vec = malloc(n * sizeof(intr_handle->intr_vec[0])); + if (intr_handle->intr_vec == NULL) { + rte_errno = ENOMEM; + ERROR("failed to allocate memory for interrupt vector," + " Rx interrupts will not be supported"); + return -rte_errno; + } + for (i = 0; i != n; ++i) { + struct rxq *rxq = priv->dev->data->rx_queues[i]; + + /* Skip queues that cannot request interrupts. */ + if (!rxq || !rxq->channel) { + /* Use invalid intr_vec[] index to disable entry. */ + intr_handle->intr_vec[i] = + RTE_INTR_VEC_RXTX_OFFSET + + RTE_MAX_RXTX_INTR_VEC_ID; + continue; + } + if (count >= RTE_MAX_RXTX_INTR_VEC_ID) { + rte_errno = E2BIG; + ERROR("too many Rx queues for interrupt vector size" + " (%d), Rx interrupts cannot be enabled", + RTE_MAX_RXTX_INTR_VEC_ID); + mlx4_rx_intr_vec_disable(priv); + return -rte_errno; + } + intr_handle->intr_vec[i] = RTE_INTR_VEC_RXTX_OFFSET + count; + intr_handle->efds[count] = rxq->channel->fd; + count++; + } + if (!count) + mlx4_rx_intr_vec_disable(priv); + else + intr_handle->nb_efd = count; + return 0; +} + +/** + * Process scheduled link status check. + * + * If LSC interrupts are requested, process related callback. + * + * @param priv + * Pointer to private structure. + */ +static void +mlx4_link_status_alarm(struct priv *priv) +{ + const struct rte_intr_conf *const intr_conf = + &priv->dev->data->dev_conf.intr_conf; + + assert(priv->intr_alarm == 1); + priv->intr_alarm = 0; + if (intr_conf->lsc && !mlx4_link_status_check(priv)) + _rte_eth_dev_callback_process(priv->dev, + RTE_ETH_EVENT_INTR_LSC, + NULL, NULL); +} + +/** + * Check link status. + * + * In case of inconsistency, another check is scheduled. + * + * @param priv + * Pointer to private structure. + * + * @return + * 0 on success (link status is consistent), negative errno value + * otherwise and rte_errno is set. + */ +static int +mlx4_link_status_check(struct priv *priv) +{ + struct rte_eth_link *link = &priv->dev->data->dev_link; + int ret = mlx4_link_update(priv->dev, 0); + + if (ret) + return ret; + if ((!link->link_speed && link->link_status) || + (link->link_speed && !link->link_status)) { + if (!priv->intr_alarm) { + /* Inconsistent status, check again later. */ + ret = rte_eal_alarm_set(MLX4_INTR_ALARM_TIMEOUT, + (void (*)(void *)) + mlx4_link_status_alarm, + priv); + if (ret) + return ret; + priv->intr_alarm = 1; + } + rte_errno = EINPROGRESS; + return -rte_errno; + } + return 0; +} + +/** + * Handle interrupts from the NIC. + * + * @param priv + * Pointer to private structure. + */ +static void +mlx4_interrupt_handler(struct priv *priv) +{ + enum { LSC, RMV, }; + static const enum rte_eth_event_type type[] = { + [LSC] = RTE_ETH_EVENT_INTR_LSC, + [RMV] = RTE_ETH_EVENT_INTR_RMV, + }; + uint32_t caught[RTE_DIM(type)] = { 0 }; + struct ibv_async_event event; + const struct rte_intr_conf *const intr_conf = + &priv->dev->data->dev_conf.intr_conf; + unsigned int i; + + /* Read all message and acknowledge them. */ + while (!ibv_get_async_event(priv->ctx, &event)) { + switch (event.event_type) { + case IBV_EVENT_PORT_ACTIVE: + case IBV_EVENT_PORT_ERR: + if (intr_conf->lsc && !mlx4_link_status_check(priv)) + ++caught[LSC]; + break; + case IBV_EVENT_DEVICE_FATAL: + if (intr_conf->rmv) + ++caught[RMV]; + break; + default: + DEBUG("event type %d on physical port %d not handled", + event.event_type, event.element.port_num); + } + ibv_ack_async_event(&event); + } + for (i = 0; i != RTE_DIM(caught); ++i) + if (caught[i]) + _rte_eth_dev_callback_process(priv->dev, type[i], + NULL, NULL); +} + +/** + * MLX4 CQ notification . + * + * @param rxq + * Pointer to receive queue structure. + * @param solicited + * Is request solicited or not. + */ +static void +mlx4_arm_cq(struct rxq *rxq, int solicited) +{ + struct mlx4_cq *cq = &rxq->mcq; + uint64_t doorbell; + uint32_t sn = cq->arm_sn & MLX4_CQ_DB_GEQ_N_MASK; + uint32_t ci = cq->cons_index & MLX4_CQ_DB_CI_MASK; + uint32_t cmd = solicited ? MLX4_CQ_DB_REQ_NOT_SOL : MLX4_CQ_DB_REQ_NOT; + + *cq->arm_db = rte_cpu_to_be_32(sn << 28 | cmd | ci); + /* + * Make sure that the doorbell record in host memory is + * written before ringing the doorbell via PCI MMIO. + */ + rte_wmb(); + doorbell = sn << 28 | cmd | cq->cqn; + doorbell <<= 32; + doorbell |= ci; + rte_write64(rte_cpu_to_be_64(doorbell), cq->cq_db_reg); +} + +/** + * Uninstall interrupt handler. + * + * @param priv + * Pointer to private structure. + * + * @return + * 0 on success, negative errno value otherwise and rte_errno is set. + */ +int +mlx4_intr_uninstall(struct priv *priv) +{ + int err = rte_errno; /* Make sure rte_errno remains unchanged. */ + + if (priv->intr_handle.fd != -1) { + rte_intr_callback_unregister(&priv->intr_handle, + (void (*)(void *)) + mlx4_interrupt_handler, + priv); + priv->intr_handle.fd = -1; + } + rte_eal_alarm_cancel((void (*)(void *))mlx4_link_status_alarm, priv); + priv->intr_alarm = 0; + mlx4_rx_intr_vec_disable(priv); + rte_errno = err; + return 0; +} + +/** + * Install interrupt handler. + * + * @param priv + * Pointer to private structure. + * + * @return + * 0 on success, negative errno value otherwise and rte_errno is set. + */ +int +mlx4_intr_install(struct priv *priv) +{ + const struct rte_intr_conf *const intr_conf = + &priv->dev->data->dev_conf.intr_conf; + int rc; + + mlx4_intr_uninstall(priv); + if (intr_conf->rxq && mlx4_rx_intr_vec_enable(priv) < 0) + goto error; + if (intr_conf->lsc | intr_conf->rmv) { + priv->intr_handle.fd = priv->ctx->async_fd; + rc = rte_intr_callback_register(&priv->intr_handle, + (void (*)(void *)) + mlx4_interrupt_handler, + priv); + if (rc < 0) { + rte_errno = -rc; + goto error; + } + } + return 0; +error: + mlx4_intr_uninstall(priv); + return -rte_errno; +} + +/** + * DPDK callback for Rx queue interrupt disable. + * + * @param dev + * Pointer to Ethernet device structure. + * @param idx + * Rx queue index. + * + * @return + * 0 on success, negative errno value otherwise and rte_errno is set. + */ +int +mlx4_rx_intr_disable(struct rte_eth_dev *dev, uint16_t idx) +{ + struct rxq *rxq = dev->data->rx_queues[idx]; + struct ibv_cq *ev_cq; + void *ev_ctx; + int ret; + + if (!rxq || !rxq->channel) { + ret = EINVAL; + } else { + ret = ibv_get_cq_event(rxq->cq->channel, &ev_cq, &ev_ctx); + if (ret || ev_cq != rxq->cq) + ret = EINVAL; + } + if (ret) { + rte_errno = ret; + WARN("unable to disable interrupt on rx queue %d", + idx); + } else { + rxq->mcq.arm_sn++; + ibv_ack_cq_events(rxq->cq, 1); + } + return -ret; +} + +/** + * DPDK callback for Rx queue interrupt enable. + * + * @param dev + * Pointer to Ethernet device structure. + * @param idx + * Rx queue index. + * + * @return + * 0 on success, negative errno value otherwise and rte_errno is set. + */ +int +mlx4_rx_intr_enable(struct rte_eth_dev *dev, uint16_t idx) +{ + struct rxq *rxq = dev->data->rx_queues[idx]; + int ret = 0; + + if (!rxq || !rxq->channel) { + ret = EINVAL; + rte_errno = ret; + WARN("unable to arm interrupt on rx queue %d", idx); + } else { + mlx4_arm_cq(rxq, 0); + } + return -ret; +} diff --git a/dpdk/drivers/net/mlx4/mlx4_mr.c b/dpdk/drivers/net/mlx4/mlx4_mr.c new file mode 100644 index 00000000..2a3e2695 --- /dev/null +++ b/dpdk/drivers/net/mlx4/mlx4_mr.c @@ -0,0 +1,293 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of 6WIND S.A. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** + * @file + * Memory management functions for mlx4 driver. + */ + +#include +#include +#include +#include +#include +#include + +/* Verbs headers do not support -pedantic. */ +#ifdef PEDANTIC +#pragma GCC diagnostic ignored "-Wpedantic" +#endif +#include +#ifdef PEDANTIC +#pragma GCC diagnostic error "-Wpedantic" +#endif + +#include +#include +#include +#include +#include +#include +#include + +#include "mlx4_rxtx.h" +#include "mlx4_utils.h" + +struct mlx4_check_mempool_data { + int ret; + char *start; + char *end; +}; + +/** + * Called by mlx4_check_mempool() when iterating the memory chunks. + * + * @param[in] mp + * Pointer to memory pool (unused). + * @param[in, out] data + * Pointer to shared buffer with mlx4_check_mempool(). + * @param[in] memhdr + * Pointer to mempool chunk header. + * @param mem_idx + * Mempool element index (unused). + */ +static void +mlx4_check_mempool_cb(struct rte_mempool *mp, void *opaque, + struct rte_mempool_memhdr *memhdr, + unsigned int mem_idx) +{ + struct mlx4_check_mempool_data *data = opaque; + + (void)mp; + (void)mem_idx; + /* It already failed, skip the next chunks. */ + if (data->ret != 0) + return; + /* It is the first chunk. */ + if (data->start == NULL && data->end == NULL) { + data->start = memhdr->addr; + data->end = data->start + memhdr->len; + return; + } + if (data->end == memhdr->addr) { + data->end += memhdr->len; + return; + } + if (data->start == (char *)memhdr->addr + memhdr->len) { + data->start -= memhdr->len; + return; + } + /* Error, mempool is not virtually contiguous. */ + data->ret = -1; +} + +/** + * Check if a mempool can be used: it must be virtually contiguous. + * + * @param[in] mp + * Pointer to memory pool. + * @param[out] start + * Pointer to the start address of the mempool virtual memory area. + * @param[out] end + * Pointer to the end address of the mempool virtual memory area. + * + * @return + * 0 on success (mempool is virtually contiguous), -1 on error. + */ +static int +mlx4_check_mempool(struct rte_mempool *mp, uintptr_t *start, uintptr_t *end) +{ + struct mlx4_check_mempool_data data; + + memset(&data, 0, sizeof(data)); + rte_mempool_mem_iter(mp, mlx4_check_mempool_cb, &data); + *start = (uintptr_t)data.start; + *end = (uintptr_t)data.end; + return data.ret; +} + +/** + * Obtain a memory region from a memory pool. + * + * If a matching memory region already exists, it is returned with its + * reference count incremented, otherwise a new one is registered. + * + * @param priv + * Pointer to private structure. + * @param mp + * Pointer to memory pool. + * + * @return + * Memory region pointer, NULL in case of error and rte_errno is set. + */ +struct mlx4_mr * +mlx4_mr_get(struct priv *priv, struct rte_mempool *mp) +{ + const struct rte_memseg *ms = rte_eal_get_physmem_layout(); + uintptr_t start; + uintptr_t end; + unsigned int i; + struct mlx4_mr *mr; + + if (mlx4_check_mempool(mp, &start, &end) != 0) { + rte_errno = EINVAL; + ERROR("mempool %p: not virtually contiguous", + (void *)mp); + return NULL; + } + DEBUG("mempool %p area start=%p end=%p size=%zu", + (void *)mp, (void *)start, (void *)end, + (size_t)(end - start)); + /* Round start and end to page boundary if found in memory segments. */ + for (i = 0; (i < RTE_MAX_MEMSEG) && (ms[i].addr != NULL); ++i) { + uintptr_t addr = (uintptr_t)ms[i].addr; + size_t len = ms[i].len; + unsigned int align = ms[i].hugepage_sz; + + if ((start > addr) && (start < addr + len)) + start = RTE_ALIGN_FLOOR(start, align); + if ((end > addr) && (end < addr + len)) + end = RTE_ALIGN_CEIL(end, align); + } + DEBUG("mempool %p using start=%p end=%p size=%zu for MR", + (void *)mp, (void *)start, (void *)end, + (size_t)(end - start)); + rte_spinlock_lock(&priv->mr_lock); + LIST_FOREACH(mr, &priv->mr, next) + if (mp == mr->mp && start >= mr->start && end <= mr->end) + break; + if (mr) { + ++mr->refcnt; + goto release; + } + mr = rte_malloc(__func__, sizeof(*mr), 0); + if (!mr) { + rte_errno = ENOMEM; + goto release; + } + *mr = (struct mlx4_mr){ + .start = start, + .end = end, + .refcnt = 1, + .priv = priv, + .mr = ibv_reg_mr(priv->pd, (void *)start, end - start, + IBV_ACCESS_LOCAL_WRITE), + .mp = mp, + }; + if (mr->mr) { + mr->lkey = mr->mr->lkey; + LIST_INSERT_HEAD(&priv->mr, mr, next); + } else { + rte_free(mr); + mr = NULL; + rte_errno = errno ? errno : EINVAL; + } +release: + rte_spinlock_unlock(&priv->mr_lock); + return mr; +} + +/** + * Release a memory region. + * + * This function decrements its reference count and destroys it after + * reaching 0. + * + * Note to avoid race conditions given this function may be used from the + * data plane, it's extremely important that each user holds its own + * reference. + * + * @param mr + * Memory region to release. + */ +void +mlx4_mr_put(struct mlx4_mr *mr) +{ + struct priv *priv = mr->priv; + + rte_spinlock_lock(&priv->mr_lock); + assert(mr->refcnt); + if (--mr->refcnt) + goto release; + LIST_REMOVE(mr, next); + claim_zero(ibv_dereg_mr(mr->mr)); + rte_free(mr); +release: + rte_spinlock_unlock(&priv->mr_lock); +} + +/** + * Add memory region (MR) <-> memory pool (MP) association to txq->mp2mr[]. + * If mp2mr[] is full, remove an entry first. + * + * @param txq + * Pointer to Tx queue structure. + * @param[in] mp + * Memory pool for which a memory region lkey must be added. + * @param[in] i + * Index in memory pool (MP) where to add memory region (MR). + * + * @return + * Added mr->lkey on success, (uint32_t)-1 on failure. + */ +uint32_t +mlx4_txq_add_mr(struct txq *txq, struct rte_mempool *mp, uint32_t i) +{ + struct mlx4_mr *mr; + + /* Add a new entry, register MR first. */ + DEBUG("%p: discovered new memory pool \"%s\" (%p)", + (void *)txq, mp->name, (void *)mp); + mr = mlx4_mr_get(txq->priv, mp); + if (unlikely(mr == NULL)) { + DEBUG("%p: unable to configure MR, mlx4_mr_get() failed", + (void *)txq); + return (uint32_t)-1; + } + if (unlikely(i == RTE_DIM(txq->mp2mr))) { + /* Table is full, remove oldest entry. */ + DEBUG("%p: MR <-> MP table full, dropping oldest entry.", + (void *)txq); + --i; + mlx4_mr_put(txq->mp2mr[0].mr); + memmove(&txq->mp2mr[0], &txq->mp2mr[1], + (sizeof(txq->mp2mr) - sizeof(txq->mp2mr[0]))); + } + /* Store the new entry. */ + txq->mp2mr[i].mp = mp; + txq->mp2mr[i].mr = mr; + txq->mp2mr[i].lkey = mr->lkey; + DEBUG("%p: new MR lkey for MP \"%s\" (%p): 0x%08" PRIu32, + (void *)txq, mp->name, (void *)mp, txq->mp2mr[i].lkey); + return txq->mp2mr[i].lkey; +} diff --git a/dpdk/drivers/net/mlx4/mlx4_prm.h b/dpdk/drivers/net/mlx4/mlx4_prm.h new file mode 100644 index 00000000..fcc7c129 --- /dev/null +++ b/dpdk/drivers/net/mlx4/mlx4_prm.h @@ -0,0 +1,177 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of 6WIND S.A. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef MLX4_PRM_H_ +#define MLX4_PRM_H_ + +#include +#include +#include + +/* Verbs headers do not support -pedantic. */ +#ifdef PEDANTIC +#pragma GCC diagnostic ignored "-Wpedantic" +#endif +#include +#include +#ifdef PEDANTIC +#pragma GCC diagnostic error "-Wpedantic" +#endif + +/* ConnectX-3 Tx queue basic block. */ +#define MLX4_TXBB_SHIFT 6 +#define MLX4_TXBB_SIZE (1 << MLX4_TXBB_SHIFT) + +/* Typical TSO descriptor with 16 gather entries is 352 bytes. */ +#define MLX4_MAX_WQE_SIZE 512 +#define MLX4_MAX_WQE_TXBBS (MLX4_MAX_WQE_SIZE / MLX4_TXBB_SIZE) + +/* Send queue stamping/invalidating information. */ +#define MLX4_SQ_STAMP_STRIDE 64 +#define MLX4_SQ_STAMP_DWORDS (MLX4_SQ_STAMP_STRIDE / 4) +#define MLX4_SQ_STAMP_SHIFT 31 +#define MLX4_SQ_STAMP_VAL 0x7fffffff + +/* Work queue element (WQE) flags. */ +#define MLX4_BIT_WQE_OWN 0x80000000 +#define MLX4_WQE_CTRL_IIP_HDR_CSUM (1 << 28) +#define MLX4_WQE_CTRL_IL4_HDR_CSUM (1 << 27) + +#define MLX4_SIZE_TO_TXBBS(size) \ + (RTE_ALIGN((size), (MLX4_TXBB_SIZE)) >> (MLX4_TXBB_SHIFT)) + +/* CQE checksum flags. */ +enum { + MLX4_CQE_L2_TUNNEL_IPV4 = (int)(1u << 25), + MLX4_CQE_L2_TUNNEL_L4_CSUM = (int)(1u << 26), + MLX4_CQE_L2_TUNNEL = (int)(1u << 27), + MLX4_CQE_L2_VLAN_MASK = (int)(3u << 29), + MLX4_CQE_L2_TUNNEL_IPOK = (int)(1u << 31), +}; + +/* CQE status flags. */ +#define MLX4_CQE_STATUS_IPV4 (1 << 22) +#define MLX4_CQE_STATUS_IPV4F (1 << 23) +#define MLX4_CQE_STATUS_IPV6 (1 << 24) +#define MLX4_CQE_STATUS_IPV4OPT (1 << 25) +#define MLX4_CQE_STATUS_TCP (1 << 26) +#define MLX4_CQE_STATUS_UDP (1 << 27) +#define MLX4_CQE_STATUS_PTYPE_MASK \ + (MLX4_CQE_STATUS_IPV4 | \ + MLX4_CQE_STATUS_IPV4F | \ + MLX4_CQE_STATUS_IPV6 | \ + MLX4_CQE_STATUS_IPV4OPT | \ + MLX4_CQE_STATUS_TCP | \ + MLX4_CQE_STATUS_UDP) + +/* Send queue information. */ +struct mlx4_sq { + volatile uint8_t *buf; /**< SQ buffer. */ + volatile uint8_t *eob; /**< End of SQ buffer */ + uint32_t head; /**< SQ head counter in units of TXBBS. */ + uint32_t tail; /**< SQ tail counter in units of TXBBS. */ + uint32_t txbb_cnt; /**< Num of WQEBB in the Q (should be ^2). */ + uint32_t txbb_cnt_mask; /**< txbbs_cnt mask (txbb_cnt is ^2). */ + uint32_t headroom_txbbs; /**< Num of txbbs that should be kept free. */ + volatile uint32_t *db; /**< Pointer to the doorbell. */ + uint32_t doorbell_qpn; /**< qp number to write to the doorbell. */ +}; + +#define mlx4_get_send_wqe(sq, n) ((sq)->buf + ((n) * (MLX4_TXBB_SIZE))) + +/* Completion queue events, numbers and masks. */ +#define MLX4_CQ_DB_GEQ_N_MASK 0x3 +#define MLX4_CQ_DOORBELL 0x20 +#define MLX4_CQ_DB_CI_MASK 0xffffff + +/* Completion queue information. */ +struct mlx4_cq { + volatile void *cq_uar; /**< CQ user access region. */ + volatile void *cq_db_reg; /**< CQ doorbell register. */ + volatile uint32_t *set_ci_db; /**< Pointer to the CQ doorbell. */ + volatile uint32_t *arm_db; /**< Arming Rx events doorbell. */ + volatile uint8_t *buf; /**< Pointer to the completion queue buffer. */ + uint32_t cqe_cnt; /**< Number of entries in the queue. */ + uint32_t cqe_64:1; /**< CQ entry size is 64 bytes. */ + uint32_t cons_index; /**< Last queue entry that was handled. */ + uint32_t cqn; /**< CQ number. */ + int arm_sn; /**< Rx event counter. */ +}; + +/** + * Retrieve a CQE entry from a CQ. + * + * cqe = cq->buf + cons_index * cqe_size + cqe_offset + * + * Where cqe_size is 32 or 64 bytes and cqe_offset is 0 or 32 (depending on + * cqe_size). + * + * @param cq + * CQ to retrieve entry from. + * @param index + * Entry index. + * + * @return + * Pointer to CQE entry. + */ +static inline volatile struct mlx4_cqe * +mlx4_get_cqe(struct mlx4_cq *cq, uint32_t index) +{ + return (volatile struct mlx4_cqe *)(cq->buf + + ((index & (cq->cqe_cnt - 1)) << + (5 + cq->cqe_64)) + + (cq->cqe_64 << 5)); +} + +/** + * Transpose a flag in a value. + * + * @param val + * Input value. + * @param from + * Flag to retrieve from input value. + * @param to + * Flag to set in output value. + * + * @return + * Output value with transposed flag enabled if present on input. + */ +static inline uint64_t +mlx4_transpose(uint64_t val, uint64_t from, uint64_t to) +{ + return (from >= to ? + (val & from) / (from / to) : + (val & from) * (to / from)); +} + +#endif /* MLX4_PRM_H_ */ diff --git a/dpdk/drivers/net/mlx4/mlx4_rxq.c b/dpdk/drivers/net/mlx4/mlx4_rxq.c new file mode 100644 index 00000000..53313c56 --- /dev/null +++ b/dpdk/drivers/net/mlx4/mlx4_rxq.c @@ -0,0 +1,874 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of 6WIND S.A. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** + * @file + * Rx queues configuration for mlx4 driver. + */ + +#include +#include +#include +#include +#include + +/* Verbs headers do not support -pedantic. */ +#ifdef PEDANTIC +#pragma GCC diagnostic ignored "-Wpedantic" +#endif +#include +#include +#ifdef PEDANTIC +#pragma GCC diagnostic error "-Wpedantic" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "mlx4.h" +#include "mlx4_flow.h" +#include "mlx4_rxtx.h" +#include "mlx4_utils.h" + +/** + * Historical RSS hash key. + * + * This used to be the default for mlx4 in Linux before v3.19 switched to + * generating random hash keys through netdev_rss_key_fill(). + * + * It is used in this PMD for consistency with past DPDK releases but can + * now be overridden through user configuration. + * + * Note: this is not const to work around API quirks. + */ +uint8_t +mlx4_rss_hash_key_default[MLX4_RSS_HASH_KEY_SIZE] = { + 0x2c, 0xc6, 0x81, 0xd1, + 0x5b, 0xdb, 0xf4, 0xf7, + 0xfc, 0xa2, 0x83, 0x19, + 0xdb, 0x1a, 0x3e, 0x94, + 0x6b, 0x9e, 0x38, 0xd9, + 0x2c, 0x9c, 0x03, 0xd1, + 0xad, 0x99, 0x44, 0xa7, + 0xd9, 0x56, 0x3d, 0x59, + 0x06, 0x3c, 0x25, 0xf3, + 0xfc, 0x1f, 0xdc, 0x2a, +}; + +/** + * Obtain a RSS context with specified properties. + * + * Used when creating a flow rule targeting one or several Rx queues. + * + * If a matching RSS context already exists, it is returned with its + * reference count incremented. + * + * @param priv + * Pointer to private structure. + * @param fields + * Fields for RSS processing (Verbs format). + * @param[in] key + * Hash key to use (whose size is exactly MLX4_RSS_HASH_KEY_SIZE). + * @param queues + * Number of target queues. + * @param[in] queue_id + * Target queues. + * + * @return + * Pointer to RSS context on success, NULL otherwise and rte_errno is set. + */ +struct mlx4_rss * +mlx4_rss_get(struct priv *priv, uint64_t fields, + uint8_t key[MLX4_RSS_HASH_KEY_SIZE], + uint16_t queues, const uint16_t queue_id[]) +{ + struct mlx4_rss *rss; + size_t queue_id_size = sizeof(queue_id[0]) * queues; + + LIST_FOREACH(rss, &priv->rss, next) + if (fields == rss->fields && + queues == rss->queues && + !memcmp(key, rss->key, MLX4_RSS_HASH_KEY_SIZE) && + !memcmp(queue_id, rss->queue_id, queue_id_size)) { + ++rss->refcnt; + return rss; + } + rss = rte_malloc(__func__, offsetof(struct mlx4_rss, queue_id) + + queue_id_size, 0); + if (!rss) + goto error; + *rss = (struct mlx4_rss){ + .priv = priv, + .refcnt = 1, + .usecnt = 0, + .qp = NULL, + .ind = NULL, + .fields = fields, + .queues = queues, + }; + memcpy(rss->key, key, MLX4_RSS_HASH_KEY_SIZE); + memcpy(rss->queue_id, queue_id, queue_id_size); + LIST_INSERT_HEAD(&priv->rss, rss, next); + return rss; +error: + rte_errno = ENOMEM; + return NULL; +} + +/** + * Release a RSS context instance. + * + * Used when destroying a flow rule targeting one or several Rx queues. + * + * This function decrements the reference count of the context and destroys + * it after reaching 0. The context must have no users at this point; all + * prior calls to mlx4_rss_attach() must have been followed by matching + * calls to mlx4_rss_detach(). + * + * @param rss + * RSS context to release. + */ +void +mlx4_rss_put(struct mlx4_rss *rss) +{ + assert(rss->refcnt); + if (--rss->refcnt) + return; + assert(!rss->usecnt); + assert(!rss->qp); + assert(!rss->ind); + LIST_REMOVE(rss, next); + rte_free(rss); +} + +/** + * Attach a user to a RSS context instance. + * + * Used when the RSS QP and indirection table objects must be instantiated, + * that is, when a flow rule must be enabled. + * + * This function increments the usage count of the context. + * + * @param rss + * RSS context to attach to. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +int +mlx4_rss_attach(struct mlx4_rss *rss) +{ + assert(rss->refcnt); + if (rss->usecnt++) { + assert(rss->qp); + assert(rss->ind); + return 0; + } + + struct ibv_wq *ind_tbl[rss->queues]; + struct priv *priv = rss->priv; + const char *msg; + unsigned int i = 0; + int ret; + + if (!rte_is_power_of_2(RTE_DIM(ind_tbl))) { + ret = EINVAL; + msg = "number of RSS queues must be a power of two"; + goto error; + } + for (i = 0; i != RTE_DIM(ind_tbl); ++i) { + uint16_t id = rss->queue_id[i]; + struct rxq *rxq = NULL; + + if (id < priv->dev->data->nb_rx_queues) + rxq = priv->dev->data->rx_queues[id]; + if (!rxq) { + ret = EINVAL; + msg = "RSS target queue is not configured"; + goto error; + } + ret = mlx4_rxq_attach(rxq); + if (ret) { + ret = -ret; + msg = "unable to attach RSS target queue"; + goto error; + } + ind_tbl[i] = rxq->wq; + } + rss->ind = ibv_create_rwq_ind_table + (priv->ctx, + &(struct ibv_rwq_ind_table_init_attr){ + .log_ind_tbl_size = rte_log2_u32(RTE_DIM(ind_tbl)), + .ind_tbl = ind_tbl, + .comp_mask = 0, + }); + if (!rss->ind) { + ret = errno ? errno : EINVAL; + msg = "RSS indirection table creation failure"; + goto error; + } + rss->qp = ibv_create_qp_ex + (priv->ctx, + &(struct ibv_qp_init_attr_ex){ + .comp_mask = (IBV_QP_INIT_ATTR_PD | + IBV_QP_INIT_ATTR_RX_HASH | + IBV_QP_INIT_ATTR_IND_TABLE), + .qp_type = IBV_QPT_RAW_PACKET, + .pd = priv->pd, + .rwq_ind_tbl = rss->ind, + .rx_hash_conf = { + .rx_hash_function = IBV_RX_HASH_FUNC_TOEPLITZ, + .rx_hash_key_len = MLX4_RSS_HASH_KEY_SIZE, + .rx_hash_key = rss->key, + .rx_hash_fields_mask = rss->fields, + }, + }); + if (!rss->qp) { + ret = errno ? errno : EINVAL; + msg = "RSS hash QP creation failure"; + goto error; + } + ret = ibv_modify_qp + (rss->qp, + &(struct ibv_qp_attr){ + .qp_state = IBV_QPS_INIT, + .port_num = priv->port, + }, + IBV_QP_STATE | IBV_QP_PORT); + if (ret) { + msg = "failed to switch RSS hash QP to INIT state"; + goto error; + } + ret = ibv_modify_qp + (rss->qp, + &(struct ibv_qp_attr){ + .qp_state = IBV_QPS_RTR, + }, + IBV_QP_STATE); + if (ret) { + msg = "failed to switch RSS hash QP to RTR state"; + goto error; + } + return 0; +error: + if (rss->qp) { + claim_zero(ibv_destroy_qp(rss->qp)); + rss->qp = NULL; + } + if (rss->ind) { + claim_zero(ibv_destroy_rwq_ind_table(rss->ind)); + rss->ind = NULL; + } + while (i--) + mlx4_rxq_detach(priv->dev->data->rx_queues[rss->queue_id[i]]); + ERROR("mlx4: %s", msg); + --rss->usecnt; + rte_errno = ret; + return -ret; +} + +/** + * Detach a user from a RSS context instance. + * + * Used when disabling (not destroying) a flow rule. + * + * This function decrements the usage count of the context and destroys + * usage resources after reaching 0. + * + * @param rss + * RSS context to detach from. + */ +void +mlx4_rss_detach(struct mlx4_rss *rss) +{ + struct priv *priv = rss->priv; + unsigned int i; + + assert(rss->refcnt); + assert(rss->qp); + assert(rss->ind); + if (--rss->usecnt) + return; + claim_zero(ibv_destroy_qp(rss->qp)); + rss->qp = NULL; + claim_zero(ibv_destroy_rwq_ind_table(rss->ind)); + rss->ind = NULL; + for (i = 0; i != rss->queues; ++i) + mlx4_rxq_detach(priv->dev->data->rx_queues[rss->queue_id[i]]); +} + +/** + * Initialize common RSS context resources. + * + * Because ConnectX-3 hardware limitations require a fixed order in the + * indirection table, WQs must be allocated sequentially to be part of a + * common RSS context. + * + * Since a newly created WQ cannot be moved to a different context, this + * function allocates them all at once, one for each configured Rx queue, + * as well as all related resources (CQs and mbufs). + * + * This must therefore be done before creating any Rx flow rules relying on + * indirection tables. + * + * @param priv + * Pointer to private structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +int +mlx4_rss_init(struct priv *priv) +{ + struct rte_eth_dev *dev = priv->dev; + uint8_t log2_range = rte_log2_u32(dev->data->nb_rx_queues); + uint32_t wq_num_prev = 0; + const char *msg; + unsigned int i; + int ret; + + /* Prepare range for RSS contexts before creating the first WQ. */ + ret = mlx4dv_set_context_attr(priv->ctx, + MLX4DV_SET_CTX_ATTR_LOG_WQS_RANGE_SZ, + &log2_range); + if (ret) { + ERROR("cannot set up range size for RSS context to %u" + " (for %u Rx queues), error: %s", + 1 << log2_range, dev->data->nb_rx_queues, strerror(ret)); + rte_errno = ret; + return -ret; + } + for (i = 0; i != priv->dev->data->nb_rx_queues; ++i) { + struct rxq *rxq = priv->dev->data->rx_queues[i]; + struct ibv_cq *cq; + struct ibv_wq *wq; + uint32_t wq_num; + + /* Attach the configured Rx queues. */ + if (rxq) { + assert(!rxq->usecnt); + ret = mlx4_rxq_attach(rxq); + if (!ret) { + wq_num = rxq->wq->wq_num; + goto wq_num_check; + } + ret = -ret; + msg = "unable to create Rx queue resources"; + goto error; + } + /* + * WQs are temporarily allocated for unconfigured Rx queues + * to maintain proper index alignment in indirection table + * by skipping unused WQ numbers. + * + * The reason this works at all even though these WQs are + * immediately destroyed is that WQNs are allocated + * sequentially and are guaranteed to never be reused in the + * same context by the underlying implementation. + */ + cq = ibv_create_cq(priv->ctx, 1, NULL, NULL, 0); + if (!cq) { + ret = ENOMEM; + msg = "placeholder CQ creation failure"; + goto error; + } + wq = ibv_create_wq + (priv->ctx, + &(struct ibv_wq_init_attr){ + .wq_type = IBV_WQT_RQ, + .max_wr = 1, + .max_sge = 1, + .pd = priv->pd, + .cq = cq, + }); + if (wq) { + wq_num = wq->wq_num; + claim_zero(ibv_destroy_wq(wq)); + } else { + wq_num = 0; /* Shut up GCC 4.8 warnings. */ + } + claim_zero(ibv_destroy_cq(cq)); + if (!wq) { + ret = ENOMEM; + msg = "placeholder WQ creation failure"; + goto error; + } +wq_num_check: + /* + * While guaranteed by the implementation, make sure WQ + * numbers are really sequential (as the saying goes, + * trust, but verify). + */ + if (i && wq_num - wq_num_prev != 1) { + if (rxq) + mlx4_rxq_detach(rxq); + ret = ERANGE; + msg = "WQ numbers are not sequential"; + goto error; + } + wq_num_prev = wq_num; + } + return 0; +error: + ERROR("cannot initialize common RSS resources (queue %u): %s: %s", + i, msg, strerror(ret)); + while (i--) { + struct rxq *rxq = priv->dev->data->rx_queues[i]; + + if (rxq) + mlx4_rxq_detach(rxq); + } + rte_errno = ret; + return -ret; +} + +/** + * Release common RSS context resources. + * + * As the reverse of mlx4_rss_init(), this must be done after removing all + * flow rules relying on indirection tables. + * + * @param priv + * Pointer to private structure. + */ +void +mlx4_rss_deinit(struct priv *priv) +{ + unsigned int i; + + for (i = 0; i != priv->dev->data->nb_rx_queues; ++i) { + struct rxq *rxq = priv->dev->data->rx_queues[i]; + + if (rxq) { + assert(rxq->usecnt == 1); + mlx4_rxq_detach(rxq); + } + } +} + +/** + * Attach a user to a Rx queue. + * + * Used when the resources of an Rx queue must be instantiated for it to + * become in a usable state. + * + * This function increments the usage count of the Rx queue. + * + * @param rxq + * Pointer to Rx queue structure. + * + * @return + * 0 on success, negative errno value otherwise and rte_errno is set. + */ +int +mlx4_rxq_attach(struct rxq *rxq) +{ + if (rxq->usecnt++) { + assert(rxq->cq); + assert(rxq->wq); + assert(rxq->wqes); + assert(rxq->rq_db); + return 0; + } + + struct priv *priv = rxq->priv; + const uint32_t elts_n = 1 << rxq->elts_n; + const uint32_t sges_n = 1 << rxq->sges_n; + struct rte_mbuf *(*elts)[elts_n] = rxq->elts; + struct mlx4dv_obj mlxdv; + struct mlx4dv_rwq dv_rwq; + struct mlx4dv_cq dv_cq = { .comp_mask = MLX4DV_CQ_MASK_UAR, }; + const char *msg; + struct ibv_cq *cq = NULL; + struct ibv_wq *wq = NULL; + volatile struct mlx4_wqe_data_seg (*wqes)[]; + unsigned int i; + int ret; + + assert(rte_is_power_of_2(elts_n)); + cq = ibv_create_cq(priv->ctx, elts_n / sges_n, NULL, rxq->channel, 0); + if (!cq) { + ret = ENOMEM; + msg = "CQ creation failure"; + goto error; + } + wq = ibv_create_wq + (priv->ctx, + &(struct ibv_wq_init_attr){ + .wq_type = IBV_WQT_RQ, + .max_wr = elts_n / sges_n, + .max_sge = sges_n, + .pd = priv->pd, + .cq = cq, + }); + if (!wq) { + ret = errno ? errno : EINVAL; + msg = "WQ creation failure"; + goto error; + } + ret = ibv_modify_wq + (wq, + &(struct ibv_wq_attr){ + .attr_mask = IBV_WQ_ATTR_STATE, + .wq_state = IBV_WQS_RDY, + }); + if (ret) { + msg = "WQ state change to IBV_WQS_RDY failed"; + goto error; + } + /* Retrieve device queue information. */ + mlxdv.cq.in = cq; + mlxdv.cq.out = &dv_cq; + mlxdv.rwq.in = wq; + mlxdv.rwq.out = &dv_rwq; + ret = mlx4dv_init_obj(&mlxdv, MLX4DV_OBJ_RWQ | MLX4DV_OBJ_CQ); + if (ret) { + msg = "failed to obtain device information from WQ/CQ objects"; + goto error; + } + wqes = (volatile struct mlx4_wqe_data_seg (*)[]) + ((uintptr_t)dv_rwq.buf.buf + dv_rwq.rq.offset); + for (i = 0; i != RTE_DIM(*elts); ++i) { + volatile struct mlx4_wqe_data_seg *scat = &(*wqes)[i]; + struct rte_mbuf *buf = rte_pktmbuf_alloc(rxq->mp); + + if (buf == NULL) { + while (i--) { + rte_pktmbuf_free_seg((*elts)[i]); + (*elts)[i] = NULL; + } + ret = ENOMEM; + msg = "cannot allocate mbuf"; + goto error; + } + /* Headroom is reserved by rte_pktmbuf_alloc(). */ + assert(buf->data_off == RTE_PKTMBUF_HEADROOM); + /* Buffer is supposed to be empty. */ + assert(rte_pktmbuf_data_len(buf) == 0); + assert(rte_pktmbuf_pkt_len(buf) == 0); + /* Only the first segment keeps headroom. */ + if (i % sges_n) + buf->data_off = 0; + buf->port = rxq->port_id; + buf->data_len = rte_pktmbuf_tailroom(buf); + buf->pkt_len = rte_pktmbuf_tailroom(buf); + buf->nb_segs = 1; + *scat = (struct mlx4_wqe_data_seg){ + .addr = rte_cpu_to_be_64(rte_pktmbuf_mtod(buf, + uintptr_t)), + .byte_count = rte_cpu_to_be_32(buf->data_len), + .lkey = rte_cpu_to_be_32(rxq->mr->lkey), + }; + (*elts)[i] = buf; + } + DEBUG("%p: allocated and configured %u segments (max %u packets)", + (void *)rxq, elts_n, elts_n / sges_n); + rxq->cq = cq; + rxq->wq = wq; + rxq->wqes = wqes; + rxq->rq_db = dv_rwq.rdb; + rxq->mcq.buf = dv_cq.buf.buf; + rxq->mcq.cqe_cnt = dv_cq.cqe_cnt; + rxq->mcq.set_ci_db = dv_cq.set_ci_db; + rxq->mcq.cqe_64 = (dv_cq.cqe_size & 64) ? 1 : 0; + rxq->mcq.arm_db = dv_cq.arm_db; + rxq->mcq.arm_sn = dv_cq.arm_sn; + rxq->mcq.cqn = dv_cq.cqn; + rxq->mcq.cq_uar = dv_cq.cq_uar; + rxq->mcq.cq_db_reg = (uint8_t *)dv_cq.cq_uar + MLX4_CQ_DOORBELL; + /* Update doorbell counter. */ + rxq->rq_ci = elts_n / sges_n; + rte_wmb(); + *rxq->rq_db = rte_cpu_to_be_32(rxq->rq_ci); + return 0; +error: + if (wq) + claim_zero(ibv_destroy_wq(wq)); + if (cq) + claim_zero(ibv_destroy_cq(cq)); + rte_errno = ret; + ERROR("error while attaching Rx queue %p: %s: %s", + (void *)rxq, msg, strerror(ret)); + return -ret; +} + +/** + * Detach a user from a Rx queue. + * + * This function decrements the usage count of the Rx queue and destroys + * usage resources after reaching 0. + * + * @param rxq + * Pointer to Rx queue structure. + */ +void +mlx4_rxq_detach(struct rxq *rxq) +{ + unsigned int i; + struct rte_mbuf *(*elts)[1 << rxq->elts_n] = rxq->elts; + + if (--rxq->usecnt) + return; + rxq->rq_ci = 0; + memset(&rxq->mcq, 0, sizeof(rxq->mcq)); + rxq->rq_db = NULL; + rxq->wqes = NULL; + claim_zero(ibv_destroy_wq(rxq->wq)); + rxq->wq = NULL; + claim_zero(ibv_destroy_cq(rxq->cq)); + rxq->cq = NULL; + DEBUG("%p: freeing Rx queue elements", (void *)rxq); + for (i = 0; (i != RTE_DIM(*elts)); ++i) { + if (!(*elts)[i]) + continue; + rte_pktmbuf_free_seg((*elts)[i]); + (*elts)[i] = NULL; + } +} + +/** + * DPDK callback to configure a Rx queue. + * + * @param dev + * Pointer to Ethernet device structure. + * @param idx + * Rx queue index. + * @param desc + * Number of descriptors to configure in queue. + * @param socket + * NUMA socket on which memory must be allocated. + * @param[in] conf + * Thresholds parameters. + * @param mp + * Memory pool for buffer allocations. + * + * @return + * 0 on success, negative errno value otherwise and rte_errno is set. + */ +int +mlx4_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, + unsigned int socket, const struct rte_eth_rxconf *conf, + struct rte_mempool *mp) +{ + struct priv *priv = dev->data->dev_private; + uint32_t mb_len = rte_pktmbuf_data_room_size(mp); + struct rte_mbuf *(*elts)[rte_align32pow2(desc)]; + struct rxq *rxq; + struct mlx4_malloc_vec vec[] = { + { + .align = RTE_CACHE_LINE_SIZE, + .size = sizeof(*rxq), + .addr = (void **)&rxq, + }, + { + .align = RTE_CACHE_LINE_SIZE, + .size = sizeof(*elts), + .addr = (void **)&elts, + }, + }; + int ret; + + (void)conf; /* Thresholds configuration (ignored). */ + DEBUG("%p: configuring queue %u for %u descriptors", + (void *)dev, idx, desc); + if (idx >= dev->data->nb_rx_queues) { + rte_errno = EOVERFLOW; + ERROR("%p: queue index out of range (%u >= %u)", + (void *)dev, idx, dev->data->nb_rx_queues); + return -rte_errno; + } + rxq = dev->data->rx_queues[idx]; + if (rxq) { + rte_errno = EEXIST; + ERROR("%p: Rx queue %u already configured, release it first", + (void *)dev, idx); + return -rte_errno; + } + if (!desc) { + rte_errno = EINVAL; + ERROR("%p: invalid number of Rx descriptors", (void *)dev); + return -rte_errno; + } + if (desc != RTE_DIM(*elts)) { + desc = RTE_DIM(*elts); + WARN("%p: increased number of descriptors in Rx queue %u" + " to the next power of two (%u)", + (void *)dev, idx, desc); + } + /* Allocate and initialize Rx queue. */ + mlx4_zmallocv_socket("RXQ", vec, RTE_DIM(vec), socket); + if (!rxq) { + ERROR("%p: unable to allocate queue index %u", + (void *)dev, idx); + return -rte_errno; + } + *rxq = (struct rxq){ + .priv = priv, + .mp = mp, + .port_id = dev->data->port_id, + .sges_n = 0, + .elts_n = rte_log2_u32(desc), + .elts = elts, + /* Toggle Rx checksum offload if hardware supports it. */ + .csum = (priv->hw_csum && + dev->data->dev_conf.rxmode.hw_ip_checksum), + .csum_l2tun = (priv->hw_csum_l2tun && + dev->data->dev_conf.rxmode.hw_ip_checksum), + .l2tun_offload = priv->hw_csum_l2tun, + .stats = { + .idx = idx, + }, + .socket = socket, + }; + /* Enable scattered packets support for this queue if necessary. */ + assert(mb_len >= RTE_PKTMBUF_HEADROOM); + if (dev->data->dev_conf.rxmode.max_rx_pkt_len <= + (mb_len - RTE_PKTMBUF_HEADROOM)) { + ; + } else if (dev->data->dev_conf.rxmode.enable_scatter) { + uint32_t size = + RTE_PKTMBUF_HEADROOM + + dev->data->dev_conf.rxmode.max_rx_pkt_len; + uint32_t sges_n; + + /* + * Determine the number of SGEs needed for a full packet + * and round it to the next power of two. + */ + sges_n = rte_log2_u32((size / mb_len) + !!(size % mb_len)); + rxq->sges_n = sges_n; + /* Make sure sges_n did not overflow. */ + size = mb_len * (1 << rxq->sges_n); + size -= RTE_PKTMBUF_HEADROOM; + if (size < dev->data->dev_conf.rxmode.max_rx_pkt_len) { + rte_errno = EOVERFLOW; + ERROR("%p: too many SGEs (%u) needed to handle" + " requested maximum packet size %u", + (void *)dev, + 1 << sges_n, + dev->data->dev_conf.rxmode.max_rx_pkt_len); + goto error; + } + } else { + WARN("%p: the requested maximum Rx packet size (%u) is" + " larger than a single mbuf (%u) and scattered" + " mode has not been requested", + (void *)dev, + dev->data->dev_conf.rxmode.max_rx_pkt_len, + mb_len - RTE_PKTMBUF_HEADROOM); + } + DEBUG("%p: maximum number of segments per packet: %u", + (void *)dev, 1 << rxq->sges_n); + if (desc % (1 << rxq->sges_n)) { + rte_errno = EINVAL; + ERROR("%p: number of Rx queue descriptors (%u) is not a" + " multiple of maximum segments per packet (%u)", + (void *)dev, + desc, + 1 << rxq->sges_n); + goto error; + } + /* Use the entire Rx mempool as the memory region. */ + rxq->mr = mlx4_mr_get(priv, mp); + if (!rxq->mr) { + ERROR("%p: MR creation failure: %s", + (void *)dev, strerror(rte_errno)); + goto error; + } + if (dev->data->dev_conf.intr_conf.rxq) { + rxq->channel = ibv_create_comp_channel(priv->ctx); + if (rxq->channel == NULL) { + rte_errno = ENOMEM; + ERROR("%p: Rx interrupt completion channel creation" + " failure: %s", + (void *)dev, strerror(rte_errno)); + goto error; + } + if (mlx4_fd_set_non_blocking(rxq->channel->fd) < 0) { + ERROR("%p: unable to make Rx interrupt completion" + " channel non-blocking: %s", + (void *)dev, strerror(rte_errno)); + goto error; + } + } + DEBUG("%p: adding Rx queue %p to list", (void *)dev, (void *)rxq); + dev->data->rx_queues[idx] = rxq; + return 0; +error: + dev->data->rx_queues[idx] = NULL; + ret = rte_errno; + mlx4_rx_queue_release(rxq); + rte_errno = ret; + assert(rte_errno > 0); + return -rte_errno; +} + +/** + * DPDK callback to release a Rx queue. + * + * @param dpdk_rxq + * Generic Rx queue pointer. + */ +void +mlx4_rx_queue_release(void *dpdk_rxq) +{ + struct rxq *rxq = (struct rxq *)dpdk_rxq; + struct priv *priv; + unsigned int i; + + if (rxq == NULL) + return; + priv = rxq->priv; + for (i = 0; i != priv->dev->data->nb_rx_queues; ++i) + if (priv->dev->data->rx_queues[i] == rxq) { + DEBUG("%p: removing Rx queue %p from list", + (void *)priv->dev, (void *)rxq); + priv->dev->data->rx_queues[i] = NULL; + break; + } + assert(!rxq->cq); + assert(!rxq->wq); + assert(!rxq->wqes); + assert(!rxq->rq_db); + if (rxq->channel) + claim_zero(ibv_destroy_comp_channel(rxq->channel)); + if (rxq->mr) + mlx4_mr_put(rxq->mr); + rte_free(rxq); +} diff --git a/dpdk/drivers/net/mlx4/mlx4_rxtx.c b/dpdk/drivers/net/mlx4/mlx4_rxtx.c new file mode 100644 index 00000000..92b62577 --- /dev/null +++ b/dpdk/drivers/net/mlx4/mlx4_rxtx.c @@ -0,0 +1,1071 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of 6WIND S.A. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** + * @file + * Data plane functions for mlx4 driver. + */ + +#include +#include +#include + +/* Verbs headers do not support -pedantic. */ +#ifdef PEDANTIC +#pragma GCC diagnostic ignored "-Wpedantic" +#endif +#include +#ifdef PEDANTIC +#pragma GCC diagnostic error "-Wpedantic" +#endif + +#include +#include +#include +#include +#include +#include + +#include "mlx4.h" +#include "mlx4_prm.h" +#include "mlx4_rxtx.h" +#include "mlx4_utils.h" + +#define WQE_ONE_DATA_SEG_SIZE \ + (sizeof(struct mlx4_wqe_ctrl_seg) + sizeof(struct mlx4_wqe_data_seg)) + +/** + * Pointer-value pair structure used in tx_post_send for saving the first + * DWORD (32 byte) of a TXBB. + */ +struct pv { + volatile struct mlx4_wqe_data_seg *dseg; + uint32_t val; +}; + +/** A table to translate Rx completion flags to packet type. */ +uint32_t mlx4_ptype_table[0x100] __rte_cache_aligned = { + /* + * The index to the array should have: + * bit[7] - MLX4_CQE_L2_TUNNEL + * bit[6] - MLX4_CQE_L2_TUNNEL_IPV4 + * bit[5] - MLX4_CQE_STATUS_UDP + * bit[4] - MLX4_CQE_STATUS_TCP + * bit[3] - MLX4_CQE_STATUS_IPV4OPT + * bit[2] - MLX4_CQE_STATUS_IPV6 + * bit[1] - MLX4_CQE_STATUS_IPV4F + * bit[0] - MLX4_CQE_STATUS_IPV4 + * giving a total of up to 256 entries. + */ + [0x00] = RTE_PTYPE_L2_ETHER, + [0x01] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_NONFRAG, + [0x02] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_FRAG, + [0x03] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_FRAG, + [0x04] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN, + [0x09] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT, + [0x0a] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT | + RTE_PTYPE_L4_FRAG, + [0x11] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_TCP, + [0x12] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_TCP, + [0x14] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_L4_TCP, + [0x18] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT | + RTE_PTYPE_L4_TCP, + [0x19] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT | + RTE_PTYPE_L4_TCP, + [0x1a] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT | + RTE_PTYPE_L4_TCP, + [0x21] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_UDP, + [0x22] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_UDP, + [0x24] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_L4_UDP, + [0x28] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT | + RTE_PTYPE_L4_UDP, + [0x29] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT | + RTE_PTYPE_L4_UDP, + [0x2a] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT | + RTE_PTYPE_L4_UDP, + /* Tunneled - L3 IPV6 */ + [0x80] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN, + [0x81] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN, + [0x82] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_FRAG, + [0x83] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_FRAG, + [0x84] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN, + [0x88] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV4_EXT, + [0x89] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV4_EXT, + [0x8a] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_FRAG, + /* Tunneled - L3 IPV6, TCP */ + [0x91] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_TCP, + [0x92] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_FRAG | + RTE_PTYPE_INNER_L4_TCP, + [0x93] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_FRAG | + RTE_PTYPE_INNER_L4_TCP, + [0x94] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_TCP, + [0x98] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV4_EXT | + RTE_PTYPE_INNER_L4_TCP, + [0x99] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV4_EXT | + RTE_PTYPE_INNER_L4_TCP, + [0x9a] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_FRAG | + RTE_PTYPE_INNER_L4_TCP, + /* Tunneled - L3 IPV6, UDP */ + [0xa1] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_UDP, + [0xa2] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_FRAG | + RTE_PTYPE_INNER_L4_UDP, + [0xa3] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_FRAG | + RTE_PTYPE_INNER_L4_UDP, + [0xa4] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_UDP, + [0xa8] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV4_EXT | + RTE_PTYPE_INNER_L4_UDP, + [0xa9] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV4_EXT | + RTE_PTYPE_INNER_L4_UDP, + [0xaa] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_FRAG | + RTE_PTYPE_INNER_L4_UDP, + /* Tunneled - L3 IPV4 */ + [0xc0] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN, + [0xc1] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN, + [0xc2] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_FRAG, + [0xc3] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_FRAG, + [0xc4] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN, + [0xc8] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV4_EXT, + [0xc9] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV4_EXT, + [0xca] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV4_EXT | + RTE_PTYPE_INNER_L4_FRAG, + /* Tunneled - L3 IPV4, TCP */ + [0xd0] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_TCP, + [0xd1] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_TCP, + [0xd2] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_FRAG | + RTE_PTYPE_INNER_L4_TCP, + [0xd3] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_FRAG | + RTE_PTYPE_INNER_L4_TCP, + [0xd4] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_TCP, + [0xd8] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV4_EXT | + RTE_PTYPE_INNER_L4_TCP, + [0xd9] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV4_EXT | + RTE_PTYPE_INNER_L4_TCP, + [0xda] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_FRAG | + RTE_PTYPE_INNER_L4_TCP, + /* Tunneled - L3 IPV4, UDP */ + [0xe0] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_UDP, + [0xe1] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_UDP, + [0xe2] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_FRAG | + RTE_PTYPE_INNER_L4_UDP, + [0xe3] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_FRAG | + RTE_PTYPE_INNER_L4_UDP, + [0xe4] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_UDP, + [0xe8] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_UDP, + [0xe9] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_UDP, + [0xea] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_FRAG | + RTE_PTYPE_INNER_L4_UDP, +}; + +/** + * Stamp a WQE so it won't be reused by the HW. + * + * Routine is used when freeing WQE used by the chip or when failing + * building an WQ entry has failed leaving partial information on the queue. + * + * @param sq + * Pointer to the SQ structure. + * @param index + * Index of the freed WQE. + * @param num_txbbs + * Number of blocks to stamp. + * If < 0 the routine will use the size written in the WQ entry. + * @param owner + * The value of the WQE owner bit to use in the stamp. + * + * @return + * The number of Tx basic blocs (TXBB) the WQE contained. + */ +static int +mlx4_txq_stamp_freed_wqe(struct mlx4_sq *sq, uint16_t index, uint8_t owner) +{ + uint32_t stamp = rte_cpu_to_be_32(MLX4_SQ_STAMP_VAL | + (!!owner << MLX4_SQ_STAMP_SHIFT)); + volatile uint8_t *wqe = mlx4_get_send_wqe(sq, + (index & sq->txbb_cnt_mask)); + volatile uint32_t *ptr = (volatile uint32_t *)wqe; + int i; + int txbbs_size; + int num_txbbs; + + /* Extract the size from the control segment of the WQE. */ + num_txbbs = MLX4_SIZE_TO_TXBBS((((volatile struct mlx4_wqe_ctrl_seg *) + wqe)->fence_size & 0x3f) << 4); + txbbs_size = num_txbbs * MLX4_TXBB_SIZE; + /* Optimize the common case when there is no wrap-around. */ + if (wqe + txbbs_size <= sq->eob) { + /* Stamp the freed descriptor. */ + for (i = 0; i < txbbs_size; i += MLX4_SQ_STAMP_STRIDE) { + *ptr = stamp; + ptr += MLX4_SQ_STAMP_DWORDS; + } + } else { + /* Stamp the freed descriptor. */ + for (i = 0; i < txbbs_size; i += MLX4_SQ_STAMP_STRIDE) { + *ptr = stamp; + ptr += MLX4_SQ_STAMP_DWORDS; + if ((volatile uint8_t *)ptr >= sq->eob) { + ptr = (volatile uint32_t *)sq->buf; + stamp ^= RTE_BE32(0x80000000); + } + } + } + return num_txbbs; +} + +/** + * Manage Tx completions. + * + * When sending a burst, mlx4_tx_burst() posts several WRs. + * To improve performance, a completion event is only required once every + * MLX4_PMD_TX_PER_COMP_REQ sends. Doing so discards completion information + * for other WRs, but this information would not be used anyway. + * + * @param txq + * Pointer to Tx queue structure. + * + * @return + * 0 on success, -1 on failure. + */ +static int +mlx4_txq_complete(struct txq *txq, const unsigned int elts_n, + struct mlx4_sq *sq) +{ + unsigned int elts_comp = txq->elts_comp; + unsigned int elts_tail = txq->elts_tail; + unsigned int sq_tail = sq->tail; + struct mlx4_cq *cq = &txq->mcq; + volatile struct mlx4_cqe *cqe; + uint32_t cons_index = cq->cons_index; + uint16_t new_index; + uint16_t nr_txbbs = 0; + int pkts = 0; + + /* + * Traverse over all CQ entries reported and handle each WQ entry + * reported by them. + */ + do { + cqe = (volatile struct mlx4_cqe *)mlx4_get_cqe(cq, cons_index); + if (unlikely(!!(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK) ^ + !!(cons_index & cq->cqe_cnt))) + break; + /* + * Make sure we read the CQE after we read the ownership bit. + */ + rte_io_rmb(); +#ifndef NDEBUG + if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) == + MLX4_CQE_OPCODE_ERROR)) { + volatile struct mlx4_err_cqe *cqe_err = + (volatile struct mlx4_err_cqe *)cqe; + ERROR("%p CQE error - vendor syndrome: 0x%x" + " syndrome: 0x%x\n", + (void *)txq, cqe_err->vendor_err, + cqe_err->syndrome); + } +#endif /* NDEBUG */ + /* Get WQE index reported in the CQE. */ + new_index = + rte_be_to_cpu_16(cqe->wqe_index) & sq->txbb_cnt_mask; + do { + /* Free next descriptor. */ + sq_tail += nr_txbbs; + nr_txbbs = + mlx4_txq_stamp_freed_wqe(sq, + sq_tail & sq->txbb_cnt_mask, + !!(sq_tail & sq->txbb_cnt)); + pkts++; + } while ((sq_tail & sq->txbb_cnt_mask) != new_index); + cons_index++; + } while (1); + if (unlikely(pkts == 0)) + return 0; + /* Update CQ. */ + cq->cons_index = cons_index; + *cq->set_ci_db = rte_cpu_to_be_32(cq->cons_index & MLX4_CQ_DB_CI_MASK); + sq->tail = sq_tail + nr_txbbs; + /* Update the list of packets posted for transmission. */ + elts_comp -= pkts; + assert(elts_comp <= txq->elts_comp); + /* + * Assume completion status is successful as nothing can be done about + * it anyway. + */ + elts_tail += pkts; + if (elts_tail >= elts_n) + elts_tail -= elts_n; + txq->elts_tail = elts_tail; + txq->elts_comp = elts_comp; + return 0; +} + +/** + * Get memory pool (MP) from mbuf. If mbuf is indirect, the pool from which + * the cloned mbuf is allocated is returned instead. + * + * @param buf + * Pointer to mbuf. + * + * @return + * Memory pool where data is located for given mbuf. + */ +static struct rte_mempool * +mlx4_txq_mb2mp(struct rte_mbuf *buf) +{ + if (unlikely(RTE_MBUF_INDIRECT(buf))) + return rte_mbuf_from_indirect(buf)->pool; + return buf->pool; +} + +static int +mlx4_tx_burst_segs(struct rte_mbuf *buf, struct txq *txq, + volatile struct mlx4_wqe_ctrl_seg **pctrl) +{ + int wqe_real_size; + int nr_txbbs; + struct pv *pv = (struct pv *)txq->bounce_buf; + struct mlx4_sq *sq = &txq->msq; + uint32_t head_idx = sq->head & sq->txbb_cnt_mask; + volatile struct mlx4_wqe_ctrl_seg *ctrl; + volatile struct mlx4_wqe_data_seg *dseg; + struct rte_mbuf *sbuf; + uint32_t lkey; + uintptr_t addr; + uint32_t byte_count; + int pv_counter = 0; + + /* Calculate the needed work queue entry size for this packet. */ + wqe_real_size = sizeof(volatile struct mlx4_wqe_ctrl_seg) + + buf->nb_segs * sizeof(volatile struct mlx4_wqe_data_seg); + nr_txbbs = MLX4_SIZE_TO_TXBBS(wqe_real_size); + /* + * Check that there is room for this WQE in the send queue and that + * the WQE size is legal. + */ + if (((sq->head - sq->tail) + nr_txbbs + + sq->headroom_txbbs) >= sq->txbb_cnt || + nr_txbbs > MLX4_MAX_WQE_TXBBS) { + return -1; + } + /* Get the control and data entries of the WQE. */ + ctrl = (volatile struct mlx4_wqe_ctrl_seg *) + mlx4_get_send_wqe(sq, head_idx); + dseg = (volatile struct mlx4_wqe_data_seg *) + ((uintptr_t)ctrl + sizeof(struct mlx4_wqe_ctrl_seg)); + *pctrl = ctrl; + /* Fill the data segments with buffer information. */ + for (sbuf = buf; sbuf != NULL; sbuf = sbuf->next, dseg++) { + addr = rte_pktmbuf_mtod(sbuf, uintptr_t); + rte_prefetch0((volatile void *)addr); + /* Handle WQE wraparound. */ + if (dseg >= (volatile struct mlx4_wqe_data_seg *)sq->eob) + dseg = (volatile struct mlx4_wqe_data_seg *)sq->buf; + dseg->addr = rte_cpu_to_be_64(addr); + /* Memory region key (big endian) for this memory pool. */ + lkey = mlx4_txq_mp2mr(txq, mlx4_txq_mb2mp(sbuf)); + dseg->lkey = rte_cpu_to_be_32(lkey); + /* Calculate the needed work queue entry size for this packet */ + if (unlikely(dseg->lkey == rte_cpu_to_be_32((uint32_t)-1))) { + /* MR does not exist. */ + DEBUG("%p: unable to get MP <-> MR association", + (void *)txq); + /* + * Restamp entry in case of failure. + * Make sure that size is written correctly + * Note that we give ownership to the SW, not the HW. + */ + wqe_real_size = sizeof(struct mlx4_wqe_ctrl_seg) + + buf->nb_segs * sizeof(struct mlx4_wqe_data_seg); + ctrl->fence_size = (wqe_real_size >> 4) & 0x3f; + mlx4_txq_stamp_freed_wqe(sq, head_idx, + (sq->head & sq->txbb_cnt) ? 0 : 1); + return -1; + } + if (likely(sbuf->data_len)) { + byte_count = rte_cpu_to_be_32(sbuf->data_len); + } else { + /* + * Zero length segment is treated as inline segment + * with zero data. + */ + byte_count = RTE_BE32(0x80000000); + } + /* + * If the data segment is not at the beginning of a + * Tx basic block (TXBB) then write the byte count, + * else postpone the writing to just before updating the + * control segment. + */ + if ((uintptr_t)dseg & (uintptr_t)(MLX4_TXBB_SIZE - 1)) { +#if RTE_CACHE_LINE_SIZE < 64 + /* + * Need a barrier here before writing the byte_count + * fields to make sure that all the data is visible + * before the byte_count field is set. + * Otherwise, if the segment begins a new cacheline, + * the HCA prefetcher could grab the 64-byte chunk and + * get a valid (!= 0xffffffff) byte count but stale + * data, and end up sending the wrong data. + */ + rte_io_wmb(); +#endif /* RTE_CACHE_LINE_SIZE */ + dseg->byte_count = byte_count; + } else { + /* + * This data segment starts at the beginning of a new + * TXBB, so we need to postpone its byte_count writing + * for later. + */ + pv[pv_counter].dseg = dseg; + pv[pv_counter++].val = byte_count; + } + } + /* Write the first DWORD of each TXBB save earlier. */ + if (pv_counter) { + /* Need a barrier here before writing the byte_count. */ + rte_io_wmb(); + for (--pv_counter; pv_counter >= 0; pv_counter--) + pv[pv_counter].dseg->byte_count = pv[pv_counter].val; + } + /* Fill the control parameters for this packet. */ + ctrl->fence_size = (wqe_real_size >> 4) & 0x3f; + return nr_txbbs; +} + +/** + * DPDK callback for Tx. + * + * @param dpdk_txq + * Generic pointer to Tx queue structure. + * @param[in] pkts + * Packets to transmit. + * @param pkts_n + * Number of packets in array. + * + * @return + * Number of packets successfully transmitted (<= pkts_n). + */ +uint16_t +mlx4_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) +{ + struct txq *txq = (struct txq *)dpdk_txq; + unsigned int elts_head = txq->elts_head; + const unsigned int elts_n = txq->elts_n; + unsigned int bytes_sent = 0; + unsigned int i; + unsigned int max; + struct mlx4_sq *sq = &txq->msq; + int nr_txbbs; + + assert(txq->elts_comp_cd != 0); + if (likely(txq->elts_comp != 0)) + mlx4_txq_complete(txq, elts_n, sq); + max = (elts_n - (elts_head - txq->elts_tail)); + if (max > elts_n) + max -= elts_n; + assert(max >= 1); + assert(max <= elts_n); + /* Always leave one free entry in the ring. */ + --max; + if (max > pkts_n) + max = pkts_n; + for (i = 0; (i != max); ++i) { + struct rte_mbuf *buf = pkts[i]; + unsigned int elts_head_next = + (((elts_head + 1) == elts_n) ? 0 : elts_head + 1); + struct txq_elt *elt_next = &(*txq->elts)[elts_head_next]; + struct txq_elt *elt = &(*txq->elts)[elts_head]; + uint32_t owner_opcode = MLX4_OPCODE_SEND; + volatile struct mlx4_wqe_ctrl_seg *ctrl; + volatile struct mlx4_wqe_data_seg *dseg; + union { + uint32_t flags; + uint16_t flags16[2]; + } srcrb; + uint32_t head_idx = sq->head & sq->txbb_cnt_mask; + uint32_t lkey; + uintptr_t addr; + + /* Clean up old buffer. */ + if (likely(elt->buf != NULL)) { + struct rte_mbuf *tmp = elt->buf; + +#ifndef NDEBUG + /* Poisoning. */ + memset(elt, 0x66, sizeof(*elt)); +#endif + /* Faster than rte_pktmbuf_free(). */ + do { + struct rte_mbuf *next = tmp->next; + + rte_pktmbuf_free_seg(tmp); + tmp = next; + } while (tmp != NULL); + } + RTE_MBUF_PREFETCH_TO_FREE(elt_next->buf); + if (buf->nb_segs == 1) { + /* + * Check that there is room for this WQE in the send + * queue and that the WQE size is legal + */ + if (((sq->head - sq->tail) + 1 + sq->headroom_txbbs) >= + sq->txbb_cnt || 1 > MLX4_MAX_WQE_TXBBS) { + elt->buf = NULL; + break; + } + /* Get the control and data entries of the WQE. */ + ctrl = (volatile struct mlx4_wqe_ctrl_seg *) + mlx4_get_send_wqe(sq, head_idx); + dseg = (volatile struct mlx4_wqe_data_seg *) + ((uintptr_t)ctrl + + sizeof(struct mlx4_wqe_ctrl_seg)); + addr = rte_pktmbuf_mtod(buf, uintptr_t); + rte_prefetch0((volatile void *)addr); + /* Handle WQE wraparound. */ + if (dseg >= + (volatile struct mlx4_wqe_data_seg *)sq->eob) + dseg = (volatile struct mlx4_wqe_data_seg *) + sq->buf; + dseg->addr = rte_cpu_to_be_64(addr); + /* Memory region key (big endian). */ + lkey = mlx4_txq_mp2mr(txq, mlx4_txq_mb2mp(buf)); + dseg->lkey = rte_cpu_to_be_32(lkey); + if (unlikely(dseg->lkey == + rte_cpu_to_be_32((uint32_t)-1))) { + /* MR does not exist. */ + DEBUG("%p: unable to get MP <-> MR association", + (void *)txq); + /* + * Restamp entry in case of failure. + * Make sure that size is written correctly + * Note that we give ownership to the SW, + * not the HW. + */ + ctrl->fence_size = + (WQE_ONE_DATA_SEG_SIZE >> 4) & 0x3f; + mlx4_txq_stamp_freed_wqe(sq, head_idx, + (sq->head & sq->txbb_cnt) ? 0 : 1); + elt->buf = NULL; + break; + } + /* Never be TXBB aligned, no need compiler barrier. */ + dseg->byte_count = rte_cpu_to_be_32(buf->data_len); + /* Fill the control parameters for this packet. */ + ctrl->fence_size = (WQE_ONE_DATA_SEG_SIZE >> 4) & 0x3f; + nr_txbbs = 1; + } else { + nr_txbbs = mlx4_tx_burst_segs(buf, txq, &ctrl); + if (nr_txbbs < 0) { + elt->buf = NULL; + break; + } + } + /* + * For raw Ethernet, the SOLICIT flag is used to indicate + * that no ICRC should be calculated. + */ + txq->elts_comp_cd -= nr_txbbs; + if (unlikely(txq->elts_comp_cd <= 0)) { + txq->elts_comp_cd = txq->elts_comp_cd_init; + srcrb.flags = RTE_BE32(MLX4_WQE_CTRL_SOLICIT | + MLX4_WQE_CTRL_CQ_UPDATE); + } else { + srcrb.flags = RTE_BE32(MLX4_WQE_CTRL_SOLICIT); + } + /* Enable HW checksum offload if requested */ + if (txq->csum && + (buf->ol_flags & + (PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM))) { + const uint64_t is_tunneled = (buf->ol_flags & + (PKT_TX_TUNNEL_GRE | + PKT_TX_TUNNEL_VXLAN)); + + if (is_tunneled && txq->csum_l2tun) { + owner_opcode |= MLX4_WQE_CTRL_IIP_HDR_CSUM | + MLX4_WQE_CTRL_IL4_HDR_CSUM; + if (buf->ol_flags & PKT_TX_OUTER_IP_CKSUM) + srcrb.flags |= + RTE_BE32(MLX4_WQE_CTRL_IP_HDR_CSUM); + } else { + srcrb.flags |= + RTE_BE32(MLX4_WQE_CTRL_IP_HDR_CSUM | + MLX4_WQE_CTRL_TCP_UDP_CSUM); + } + } + if (txq->lb) { + /* + * Copy destination MAC address to the WQE, this allows + * loopback in eSwitch, so that VFs and PF can + * communicate with each other. + */ + srcrb.flags16[0] = *(rte_pktmbuf_mtod(buf, uint16_t *)); + ctrl->imm = *(rte_pktmbuf_mtod_offset(buf, uint32_t *, + sizeof(uint16_t))); + } else { + ctrl->imm = 0; + } + ctrl->srcrb_flags = srcrb.flags; + /* + * Make sure descriptor is fully written before + * setting ownership bit (because HW can start + * executing as soon as we do). + */ + rte_io_wmb(); + ctrl->owner_opcode = rte_cpu_to_be_32(owner_opcode | + ((sq->head & sq->txbb_cnt) ? + MLX4_BIT_WQE_OWN : 0)); + sq->head += nr_txbbs; + elt->buf = buf; + bytes_sent += buf->pkt_len; + elts_head = elts_head_next; + } + /* Take a shortcut if nothing must be sent. */ + if (unlikely(i == 0)) + return 0; + /* Increment send statistics counters. */ + txq->stats.opackets += i; + txq->stats.obytes += bytes_sent; + /* Make sure that descriptors are written before doorbell record. */ + rte_wmb(); + /* Ring QP doorbell. */ + rte_write32(txq->msq.doorbell_qpn, txq->msq.db); + txq->elts_head = elts_head; + txq->elts_comp += i; + return i; +} + +/** + * Translate Rx completion flags to packet type. + * + * @param[in] cqe + * Pointer to CQE. + * + * @return + * Packet type for struct rte_mbuf. + */ +static inline uint32_t +rxq_cq_to_pkt_type(volatile struct mlx4_cqe *cqe, + uint32_t l2tun_offload) +{ + uint8_t idx = 0; + uint32_t pinfo = rte_be_to_cpu_32(cqe->vlan_my_qpn); + uint32_t status = rte_be_to_cpu_32(cqe->status); + + /* + * The index to the array should have: + * bit[7] - MLX4_CQE_L2_TUNNEL + * bit[6] - MLX4_CQE_L2_TUNNEL_IPV4 + */ + if (l2tun_offload && (pinfo & MLX4_CQE_L2_TUNNEL)) + idx |= ((pinfo & MLX4_CQE_L2_TUNNEL) >> 20) | + ((pinfo & MLX4_CQE_L2_TUNNEL_IPV4) >> 19); + /* + * The index to the array should have: + * bit[5] - MLX4_CQE_STATUS_UDP + * bit[4] - MLX4_CQE_STATUS_TCP + * bit[3] - MLX4_CQE_STATUS_IPV4OPT + * bit[2] - MLX4_CQE_STATUS_IPV6 + * bit[1] - MLX4_CQE_STATUS_IPV4F + * bit[0] - MLX4_CQE_STATUS_IPV4 + * giving a total of up to 256 entries. + */ + idx |= ((status & MLX4_CQE_STATUS_PTYPE_MASK) >> 22); + return mlx4_ptype_table[idx]; +} + +/** + * Translate Rx completion flags to offload flags. + * + * @param flags + * Rx completion flags returned by mlx4_cqe_flags(). + * @param csum + * Whether Rx checksums are enabled. + * @param csum_l2tun + * Whether Rx L2 tunnel checksums are enabled. + * + * @return + * Offload flags (ol_flags) in mbuf format. + */ +static inline uint32_t +rxq_cq_to_ol_flags(uint32_t flags, int csum, int csum_l2tun) +{ + uint32_t ol_flags = 0; + + if (csum) + ol_flags |= + mlx4_transpose(flags, + MLX4_CQE_STATUS_IP_HDR_CSUM_OK, + PKT_RX_IP_CKSUM_GOOD) | + mlx4_transpose(flags, + MLX4_CQE_STATUS_TCP_UDP_CSUM_OK, + PKT_RX_L4_CKSUM_GOOD); + if ((flags & MLX4_CQE_L2_TUNNEL) && csum_l2tun) + ol_flags |= + mlx4_transpose(flags, + MLX4_CQE_L2_TUNNEL_IPOK, + PKT_RX_IP_CKSUM_GOOD) | + mlx4_transpose(flags, + MLX4_CQE_L2_TUNNEL_L4_CSUM, + PKT_RX_L4_CKSUM_GOOD); + return ol_flags; +} + +/** + * Extract checksum information from CQE flags. + * + * @param cqe + * Pointer to CQE structure. + * @param csum + * Whether Rx checksums are enabled. + * @param csum_l2tun + * Whether Rx L2 tunnel checksums are enabled. + * + * @return + * CQE checksum information. + */ +static inline uint32_t +mlx4_cqe_flags(volatile struct mlx4_cqe *cqe, int csum, int csum_l2tun) +{ + uint32_t flags = 0; + + /* + * The relevant bits are in different locations on their + * CQE fields therefore we can join them in one 32bit + * variable. + */ + if (csum) + flags = (rte_be_to_cpu_32(cqe->status) & + MLX4_CQE_STATUS_IPV4_CSUM_OK); + if (csum_l2tun) + flags |= (rte_be_to_cpu_32(cqe->vlan_my_qpn) & + (MLX4_CQE_L2_TUNNEL | + MLX4_CQE_L2_TUNNEL_IPOK | + MLX4_CQE_L2_TUNNEL_L4_CSUM | + MLX4_CQE_L2_TUNNEL_IPV4)); + return flags; +} + +/** + * Poll one CQE from CQ. + * + * @param rxq + * Pointer to the receive queue structure. + * @param[out] out + * Just polled CQE. + * + * @return + * Number of bytes of the CQE, 0 in case there is no completion. + */ +static unsigned int +mlx4_cq_poll_one(struct rxq *rxq, volatile struct mlx4_cqe **out) +{ + int ret = 0; + volatile struct mlx4_cqe *cqe = NULL; + struct mlx4_cq *cq = &rxq->mcq; + + cqe = (volatile struct mlx4_cqe *)mlx4_get_cqe(cq, cq->cons_index); + if (!!(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK) ^ + !!(cq->cons_index & cq->cqe_cnt)) + goto out; + /* + * Make sure we read CQ entry contents after we've checked the + * ownership bit. + */ + rte_rmb(); + assert(!(cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK)); + assert((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) != + MLX4_CQE_OPCODE_ERROR); + ret = rte_be_to_cpu_32(cqe->byte_cnt); + ++cq->cons_index; +out: + *out = cqe; + return ret; +} + +/** + * DPDK callback for Rx with scattered packets support. + * + * @param dpdk_rxq + * Generic pointer to Rx queue structure. + * @param[out] pkts + * Array to store received packets. + * @param pkts_n + * Maximum number of packets in array. + * + * @return + * Number of packets successfully received (<= pkts_n). + */ +uint16_t +mlx4_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) +{ + struct rxq *rxq = dpdk_rxq; + const uint32_t wr_cnt = (1 << rxq->elts_n) - 1; + const uint16_t sges_n = rxq->sges_n; + struct rte_mbuf *pkt = NULL; + struct rte_mbuf *seg = NULL; + unsigned int i = 0; + uint32_t rq_ci = rxq->rq_ci << sges_n; + int len = 0; + + while (pkts_n) { + volatile struct mlx4_cqe *cqe; + uint32_t idx = rq_ci & wr_cnt; + struct rte_mbuf *rep = (*rxq->elts)[idx]; + volatile struct mlx4_wqe_data_seg *scat = &(*rxq->wqes)[idx]; + + /* Update the 'next' pointer of the previous segment. */ + if (pkt) + seg->next = rep; + seg = rep; + rte_prefetch0(seg); + rte_prefetch0(scat); + rep = rte_mbuf_raw_alloc(rxq->mp); + if (unlikely(rep == NULL)) { + ++rxq->stats.rx_nombuf; + if (!pkt) { + /* + * No buffers before we even started, + * bail out silently. + */ + break; + } + while (pkt != seg) { + assert(pkt != (*rxq->elts)[idx]); + rep = pkt->next; + pkt->next = NULL; + pkt->nb_segs = 1; + rte_mbuf_raw_free(pkt); + pkt = rep; + } + break; + } + if (!pkt) { + /* Looking for the new packet. */ + len = mlx4_cq_poll_one(rxq, &cqe); + if (!len) { + rte_mbuf_raw_free(rep); + break; + } + if (unlikely(len < 0)) { + /* Rx error, packet is likely too large. */ + rte_mbuf_raw_free(rep); + ++rxq->stats.idropped; + goto skip; + } + pkt = seg; + /* Update packet information. */ + pkt->packet_type = + rxq_cq_to_pkt_type(cqe, rxq->l2tun_offload); + pkt->ol_flags = 0; + pkt->pkt_len = len; + if (rxq->csum | rxq->csum_l2tun) { + uint32_t flags = + mlx4_cqe_flags(cqe, + rxq->csum, + rxq->csum_l2tun); + + pkt->ol_flags = + rxq_cq_to_ol_flags(flags, + rxq->csum, + rxq->csum_l2tun); + } + } + rep->nb_segs = 1; + rep->port = rxq->port_id; + rep->data_len = seg->data_len; + rep->data_off = seg->data_off; + (*rxq->elts)[idx] = rep; + /* + * Fill NIC descriptor with the new buffer. The lkey and size + * of the buffers are already known, only the buffer address + * changes. + */ + scat->addr = rte_cpu_to_be_64(rte_pktmbuf_mtod(rep, uintptr_t)); + if (len > seg->data_len) { + len -= seg->data_len; + ++pkt->nb_segs; + ++rq_ci; + continue; + } + /* The last segment. */ + seg->data_len = len; + /* Increment bytes counter. */ + rxq->stats.ibytes += pkt->pkt_len; + /* Return packet. */ + *(pkts++) = pkt; + pkt = NULL; + --pkts_n; + ++i; +skip: + /* Align consumer index to the next stride. */ + rq_ci >>= sges_n; + ++rq_ci; + rq_ci <<= sges_n; + } + if (unlikely(i == 0 && (rq_ci >> sges_n) == rxq->rq_ci)) + return 0; + /* Update the consumer index. */ + rxq->rq_ci = rq_ci >> sges_n; + rte_wmb(); + *rxq->rq_db = rte_cpu_to_be_32(rxq->rq_ci); + *rxq->mcq.set_ci_db = + rte_cpu_to_be_32(rxq->mcq.cons_index & MLX4_CQ_DB_CI_MASK); + /* Increment packets counter. */ + rxq->stats.ipackets += i; + return i; +} + +/** + * Dummy DPDK callback for Tx. + * + * This function is used to temporarily replace the real callback during + * unsafe control operations on the queue, or in case of error. + * + * @param dpdk_txq + * Generic pointer to Tx queue structure. + * @param[in] pkts + * Packets to transmit. + * @param pkts_n + * Number of packets in array. + * + * @return + * Number of packets successfully transmitted (<= pkts_n). + */ +uint16_t +mlx4_tx_burst_removed(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) +{ + (void)dpdk_txq; + (void)pkts; + (void)pkts_n; + return 0; +} + +/** + * Dummy DPDK callback for Rx. + * + * This function is used to temporarily replace the real callback during + * unsafe control operations on the queue, or in case of error. + * + * @param dpdk_rxq + * Generic pointer to Rx queue structure. + * @param[out] pkts + * Array to store received packets. + * @param pkts_n + * Maximum number of packets in array. + * + * @return + * Number of packets successfully received (<= pkts_n). + */ +uint16_t +mlx4_rx_burst_removed(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) +{ + (void)dpdk_rxq; + (void)pkts; + (void)pkts_n; + return 0; +} diff --git a/dpdk/drivers/net/mlx4/mlx4_rxtx.h b/dpdk/drivers/net/mlx4/mlx4_rxtx.h new file mode 100644 index 00000000..463df2b0 --- /dev/null +++ b/dpdk/drivers/net/mlx4/mlx4_rxtx.h @@ -0,0 +1,215 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of 6WIND S.A. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef MLX4_RXTX_H_ +#define MLX4_RXTX_H_ + +#include +#include + +/* Verbs headers do not support -pedantic. */ +#ifdef PEDANTIC +#pragma GCC diagnostic ignored "-Wpedantic" +#endif +#include +#include +#ifdef PEDANTIC +#pragma GCC diagnostic error "-Wpedantic" +#endif + +#include +#include +#include + +#include "mlx4.h" +#include "mlx4_prm.h" + +/** Rx queue counters. */ +struct mlx4_rxq_stats { + unsigned int idx; /**< Mapping index. */ + uint64_t ipackets; /**< Total of successfully received packets. */ + uint64_t ibytes; /**< Total of successfully received bytes. */ + uint64_t idropped; /**< Total of packets dropped when Rx ring full. */ + uint64_t rx_nombuf; /**< Total of Rx mbuf allocation failures. */ +}; + +/** Rx queue descriptor. */ +struct rxq { + struct priv *priv; /**< Back pointer to private data. */ + struct rte_mempool *mp; /**< Memory pool for allocations. */ + struct mlx4_mr *mr; /**< Memory region. */ + struct ibv_cq *cq; /**< Completion queue. */ + struct ibv_wq *wq; /**< Work queue. */ + struct ibv_comp_channel *channel; /**< Rx completion channel. */ + uint16_t rq_ci; /**< Saved RQ consumer index. */ + uint16_t port_id; /**< Port ID for incoming packets. */ + uint16_t sges_n; /**< Number of segments per packet (log2 value). */ + uint16_t elts_n; /**< Mbuf queue size (log2 value). */ + struct rte_mbuf *(*elts)[]; /**< Rx elements. */ + volatile struct mlx4_wqe_data_seg (*wqes)[]; /**< HW queue entries. */ + volatile uint32_t *rq_db; /**< RQ doorbell record. */ + uint32_t csum:1; /**< Enable checksum offloading. */ + uint32_t csum_l2tun:1; /**< Same for L2 tunnels. */ + uint32_t l2tun_offload:1; /**< L2 tunnel offload is enabled. */ + struct mlx4_cq mcq; /**< Info for directly manipulating the CQ. */ + struct mlx4_rxq_stats stats; /**< Rx queue counters. */ + unsigned int socket; /**< CPU socket ID for allocations. */ + uint32_t usecnt; /**< Number of users relying on queue resources. */ + uint8_t data[]; /**< Remaining queue resources. */ +}; + +/** Shared flow target for Rx queues. */ +struct mlx4_rss { + LIST_ENTRY(mlx4_rss) next; /**< Next entry in list. */ + struct priv *priv; /**< Back pointer to private data. */ + uint32_t refcnt; /**< Reference count for this object. */ + uint32_t usecnt; /**< Number of users relying on @p qp and @p ind. */ + struct ibv_qp *qp; /**< Queue pair. */ + struct ibv_rwq_ind_table *ind; /**< Indirection table. */ + uint64_t fields; /**< Fields for RSS processing (Verbs format). */ + uint8_t key[MLX4_RSS_HASH_KEY_SIZE]; /**< Hash key to use. */ + uint16_t queues; /**< Number of target queues. */ + uint16_t queue_id[]; /**< Target queues. */ +}; + +/** Tx element. */ +struct txq_elt { + struct rte_mbuf *buf; /**< Buffer. */ +}; + +/** Rx queue counters. */ +struct mlx4_txq_stats { + unsigned int idx; /**< Mapping index. */ + uint64_t opackets; /**< Total of successfully sent packets. */ + uint64_t obytes; /**< Total of successfully sent bytes. */ + uint64_t odropped; /**< Total of packets not sent when Tx ring full. */ +}; + +/** Tx queue descriptor. */ +struct txq { + struct mlx4_sq msq; /**< Info for directly manipulating the SQ. */ + struct mlx4_cq mcq; /**< Info for directly manipulating the CQ. */ + unsigned int elts_head; /**< Current index in (*elts)[]. */ + unsigned int elts_tail; /**< First element awaiting completion. */ + unsigned int elts_comp; /**< Number of packets awaiting completion. */ + int elts_comp_cd; /**< Countdown for next completion. */ + unsigned int elts_comp_cd_init; /**< Initial value for countdown. */ + unsigned int elts_n; /**< (*elts)[] length. */ + struct txq_elt (*elts)[]; /**< Tx elements. */ + struct mlx4_txq_stats stats; /**< Tx queue counters. */ + uint32_t max_inline; /**< Max inline send size. */ + uint32_t csum:1; /**< Enable checksum offloading. */ + uint32_t csum_l2tun:1; /**< Same for L2 tunnels. */ + uint32_t lb:1; /**< Whether packets should be looped back by eSwitch. */ + uint8_t *bounce_buf; + /**< Memory used for storing the first DWORD of data TXBBs. */ + struct { + const struct rte_mempool *mp; /**< Cached memory pool. */ + struct mlx4_mr *mr; /**< Memory region (for mp). */ + uint32_t lkey; /**< mr->lkey copy. */ + } mp2mr[MLX4_PMD_TX_MP_CACHE]; /**< MP to MR translation table. */ + struct priv *priv; /**< Back pointer to private data. */ + unsigned int socket; /**< CPU socket ID for allocations. */ + struct ibv_cq *cq; /**< Completion queue. */ + struct ibv_qp *qp; /**< Queue pair. */ + uint8_t data[]; /**< Remaining queue resources. */ +}; + +/* mlx4_rxq.c */ + +uint8_t mlx4_rss_hash_key_default[MLX4_RSS_HASH_KEY_SIZE]; +int mlx4_rss_init(struct priv *priv); +void mlx4_rss_deinit(struct priv *priv); +struct mlx4_rss *mlx4_rss_get(struct priv *priv, uint64_t fields, + uint8_t key[MLX4_RSS_HASH_KEY_SIZE], + uint16_t queues, const uint16_t queue_id[]); +void mlx4_rss_put(struct mlx4_rss *rss); +int mlx4_rss_attach(struct mlx4_rss *rss); +void mlx4_rss_detach(struct mlx4_rss *rss); +int mlx4_rxq_attach(struct rxq *rxq); +void mlx4_rxq_detach(struct rxq *rxq); +int mlx4_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, + uint16_t desc, unsigned int socket, + const struct rte_eth_rxconf *conf, + struct rte_mempool *mp); +void mlx4_rx_queue_release(void *dpdk_rxq); + +/* mlx4_rxtx.c */ + +uint16_t mlx4_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, + uint16_t pkts_n); +uint16_t mlx4_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, + uint16_t pkts_n); +uint16_t mlx4_tx_burst_removed(void *dpdk_txq, struct rte_mbuf **pkts, + uint16_t pkts_n); +uint16_t mlx4_rx_burst_removed(void *dpdk_rxq, struct rte_mbuf **pkts, + uint16_t pkts_n); + +/* mlx4_txq.c */ + +int mlx4_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, + uint16_t desc, unsigned int socket, + const struct rte_eth_txconf *conf); +void mlx4_tx_queue_release(void *dpdk_txq); + +/** + * Get memory region (MR) <-> memory pool (MP) association from txq->mp2mr[]. + * Call mlx4_txq_add_mr() if MP is not registered yet. + * + * @param txq + * Pointer to Tx queue structure. + * @param[in] mp + * Memory pool for which a memory region lkey must be returned. + * + * @return + * mr->lkey on success, (uint32_t)-1 on failure. + */ +static inline uint32_t +mlx4_txq_mp2mr(struct txq *txq, struct rte_mempool *mp) +{ + unsigned int i; + + for (i = 0; (i != RTE_DIM(txq->mp2mr)); ++i) { + if (unlikely(txq->mp2mr[i].mp == NULL)) { + /* Unknown MP, add a new MR for it. */ + break; + } + if (txq->mp2mr[i].mp == mp) { + /* MP found MP. */ + return txq->mp2mr[i].lkey; + } + } + return mlx4_txq_add_mr(txq, mp, i); +} + +#endif /* MLX4_RXTX_H_ */ diff --git a/dpdk/drivers/net/mlx4/mlx4_txq.c b/dpdk/drivers/net/mlx4/mlx4_txq.c new file mode 100644 index 00000000..17f3b002 --- /dev/null +++ b/dpdk/drivers/net/mlx4/mlx4_txq.c @@ -0,0 +1,413 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of 6WIND S.A. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** + * @file + * Tx queues configuration for mlx4 driver. + */ + +#include +#include +#include +#include +#include + +/* Verbs headers do not support -pedantic. */ +#ifdef PEDANTIC +#pragma GCC diagnostic ignored "-Wpedantic" +#endif +#include +#ifdef PEDANTIC +#pragma GCC diagnostic error "-Wpedantic" +#endif + +#include +#include +#include +#include +#include +#include + +#include "mlx4.h" +#include "mlx4_prm.h" +#include "mlx4_rxtx.h" +#include "mlx4_utils.h" + +/** + * Free Tx queue elements. + * + * @param txq + * Pointer to Tx queue structure. + */ +static void +mlx4_txq_free_elts(struct txq *txq) +{ + unsigned int elts_head = txq->elts_head; + unsigned int elts_tail = txq->elts_tail; + struct txq_elt (*elts)[txq->elts_n] = txq->elts; + + DEBUG("%p: freeing WRs", (void *)txq); + while (elts_tail != elts_head) { + struct txq_elt *elt = &(*elts)[elts_tail]; + + assert(elt->buf != NULL); + rte_pktmbuf_free(elt->buf); + elt->buf = NULL; + if (++elts_tail == RTE_DIM(*elts)) + elts_tail = 0; + } + txq->elts_tail = txq->elts_head; +} + +struct txq_mp2mr_mbuf_check_data { + int ret; +}; + +/** + * Callback function for rte_mempool_obj_iter() to check whether a given + * mempool object looks like a mbuf. + * + * @param[in] mp + * The mempool pointer + * @param[in] arg + * Context data (struct mlx4_txq_mp2mr_mbuf_check_data). Contains the + * return value. + * @param[in] obj + * Object address. + * @param index + * Object index, unused. + */ +static void +mlx4_txq_mp2mr_mbuf_check(struct rte_mempool *mp, void *arg, void *obj, + uint32_t index) +{ + struct txq_mp2mr_mbuf_check_data *data = arg; + struct rte_mbuf *buf = obj; + + (void)index; + /* + * Check whether mbuf structure fits element size and whether mempool + * pointer is valid. + */ + if (sizeof(*buf) > mp->elt_size || buf->pool != mp) + data->ret = -1; +} + +/** + * Iterator function for rte_mempool_walk() to register existing mempools and + * fill the MP to MR cache of a Tx queue. + * + * @param[in] mp + * Memory Pool to register. + * @param *arg + * Pointer to Tx queue structure. + */ +static void +mlx4_txq_mp2mr_iter(struct rte_mempool *mp, void *arg) +{ + struct txq *txq = arg; + struct txq_mp2mr_mbuf_check_data data = { + .ret = 0, + }; + + /* Register mempool only if the first element looks like a mbuf. */ + if (rte_mempool_obj_iter(mp, mlx4_txq_mp2mr_mbuf_check, &data) == 0 || + data.ret == -1) + return; + mlx4_txq_mp2mr(txq, mp); +} + +/** + * Retrieves information needed in order to directly access the Tx queue. + * + * @param txq + * Pointer to Tx queue structure. + * @param mlxdv + * Pointer to device information for this Tx queue. + */ +static void +mlx4_txq_fill_dv_obj_info(struct txq *txq, struct mlx4dv_obj *mlxdv) +{ + struct mlx4_sq *sq = &txq->msq; + struct mlx4_cq *cq = &txq->mcq; + struct mlx4dv_qp *dqp = mlxdv->qp.out; + struct mlx4dv_cq *dcq = mlxdv->cq.out; + uint32_t sq_size = (uint32_t)dqp->rq.offset - (uint32_t)dqp->sq.offset; + + sq->buf = (uint8_t *)dqp->buf.buf + dqp->sq.offset; + /* Total length, including headroom and spare WQEs. */ + sq->eob = sq->buf + sq_size; + sq->head = 0; + sq->tail = 0; + sq->txbb_cnt = + (dqp->sq.wqe_cnt << dqp->sq.wqe_shift) >> MLX4_TXBB_SHIFT; + sq->txbb_cnt_mask = sq->txbb_cnt - 1; + sq->db = dqp->sdb; + sq->doorbell_qpn = dqp->doorbell_qpn; + sq->headroom_txbbs = + (2048 + (1 << dqp->sq.wqe_shift)) >> MLX4_TXBB_SHIFT; + cq->buf = dcq->buf.buf; + cq->cqe_cnt = dcq->cqe_cnt; + cq->set_ci_db = dcq->set_ci_db; + cq->cqe_64 = (dcq->cqe_size & 64) ? 1 : 0; +} + +/** + * DPDK callback to configure a Tx queue. + * + * @param dev + * Pointer to Ethernet device structure. + * @param idx + * Tx queue index. + * @param desc + * Number of descriptors to configure in queue. + * @param socket + * NUMA socket on which memory must be allocated. + * @param[in] conf + * Thresholds parameters. + * + * @return + * 0 on success, negative errno value otherwise and rte_errno is set. + */ +int +mlx4_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, + unsigned int socket, const struct rte_eth_txconf *conf) +{ + struct priv *priv = dev->data->dev_private; + struct mlx4dv_obj mlxdv; + struct mlx4dv_qp dv_qp; + struct mlx4dv_cq dv_cq; + struct txq_elt (*elts)[desc]; + struct ibv_qp_init_attr qp_init_attr; + struct txq *txq; + uint8_t *bounce_buf; + struct mlx4_malloc_vec vec[] = { + { + .align = RTE_CACHE_LINE_SIZE, + .size = sizeof(*txq), + .addr = (void **)&txq, + }, + { + .align = RTE_CACHE_LINE_SIZE, + .size = sizeof(*elts), + .addr = (void **)&elts, + }, + { + .align = RTE_CACHE_LINE_SIZE, + .size = MLX4_MAX_WQE_SIZE, + .addr = (void **)&bounce_buf, + }, + }; + int ret; + + (void)conf; /* Thresholds configuration (ignored). */ + DEBUG("%p: configuring queue %u for %u descriptors", + (void *)dev, idx, desc); + if (idx >= dev->data->nb_tx_queues) { + rte_errno = EOVERFLOW; + ERROR("%p: queue index out of range (%u >= %u)", + (void *)dev, idx, dev->data->nb_tx_queues); + return -rte_errno; + } + txq = dev->data->tx_queues[idx]; + if (txq) { + rte_errno = EEXIST; + DEBUG("%p: Tx queue %u already configured, release it first", + (void *)dev, idx); + return -rte_errno; + } + if (!desc) { + rte_errno = EINVAL; + ERROR("%p: invalid number of Tx descriptors", (void *)dev); + return -rte_errno; + } + /* Allocate and initialize Tx queue. */ + mlx4_zmallocv_socket("TXQ", vec, RTE_DIM(vec), socket); + if (!txq) { + ERROR("%p: unable to allocate queue index %u", + (void *)dev, idx); + return -rte_errno; + } + *txq = (struct txq){ + .priv = priv, + .stats = { + .idx = idx, + }, + .socket = socket, + .elts_n = desc, + .elts = elts, + .elts_head = 0, + .elts_tail = 0, + .elts_comp = 0, + /* + * Request send completion every MLX4_PMD_TX_PER_COMP_REQ + * packets or at least 4 times per ring. + */ + .elts_comp_cd = + RTE_MIN(MLX4_PMD_TX_PER_COMP_REQ, desc / 4), + .elts_comp_cd_init = + RTE_MIN(MLX4_PMD_TX_PER_COMP_REQ, desc / 4), + .csum = priv->hw_csum, + .csum_l2tun = priv->hw_csum_l2tun, + /* Enable Tx loopback for VF devices. */ + .lb = !!priv->vf, + .bounce_buf = bounce_buf, + }; + txq->cq = ibv_create_cq(priv->ctx, desc, NULL, NULL, 0); + if (!txq->cq) { + rte_errno = ENOMEM; + ERROR("%p: CQ creation failure: %s", + (void *)dev, strerror(rte_errno)); + goto error; + } + qp_init_attr = (struct ibv_qp_init_attr){ + .send_cq = txq->cq, + .recv_cq = txq->cq, + .cap = { + .max_send_wr = + RTE_MIN(priv->device_attr.max_qp_wr, desc), + .max_send_sge = 1, + .max_inline_data = MLX4_PMD_MAX_INLINE, + }, + .qp_type = IBV_QPT_RAW_PACKET, + /* No completion events must occur by default. */ + .sq_sig_all = 0, + }; + txq->qp = ibv_create_qp(priv->pd, &qp_init_attr); + if (!txq->qp) { + rte_errno = errno ? errno : EINVAL; + ERROR("%p: QP creation failure: %s", + (void *)dev, strerror(rte_errno)); + goto error; + } + txq->max_inline = qp_init_attr.cap.max_inline_data; + ret = ibv_modify_qp + (txq->qp, + &(struct ibv_qp_attr){ + .qp_state = IBV_QPS_INIT, + .port_num = priv->port, + }, + IBV_QP_STATE | IBV_QP_PORT); + if (ret) { + rte_errno = ret; + ERROR("%p: QP state to IBV_QPS_INIT failed: %s", + (void *)dev, strerror(rte_errno)); + goto error; + } + ret = ibv_modify_qp + (txq->qp, + &(struct ibv_qp_attr){ + .qp_state = IBV_QPS_RTR, + }, + IBV_QP_STATE); + if (ret) { + rte_errno = ret; + ERROR("%p: QP state to IBV_QPS_RTR failed: %s", + (void *)dev, strerror(rte_errno)); + goto error; + } + ret = ibv_modify_qp + (txq->qp, + &(struct ibv_qp_attr){ + .qp_state = IBV_QPS_RTS, + }, + IBV_QP_STATE); + if (ret) { + rte_errno = ret; + ERROR("%p: QP state to IBV_QPS_RTS failed: %s", + (void *)dev, strerror(rte_errno)); + goto error; + } + /* Retrieve device queue information. */ + mlxdv.cq.in = txq->cq; + mlxdv.cq.out = &dv_cq; + mlxdv.qp.in = txq->qp; + mlxdv.qp.out = &dv_qp; + ret = mlx4dv_init_obj(&mlxdv, MLX4DV_OBJ_QP | MLX4DV_OBJ_CQ); + if (ret) { + rte_errno = EINVAL; + ERROR("%p: failed to obtain information needed for" + " accessing the device queues", (void *)dev); + goto error; + } + mlx4_txq_fill_dv_obj_info(txq, &mlxdv); + /* Pre-register known mempools. */ + rte_mempool_walk(mlx4_txq_mp2mr_iter, txq); + DEBUG("%p: adding Tx queue %p to list", (void *)dev, (void *)txq); + dev->data->tx_queues[idx] = txq; + return 0; +error: + dev->data->tx_queues[idx] = NULL; + ret = rte_errno; + mlx4_tx_queue_release(txq); + rte_errno = ret; + assert(rte_errno > 0); + return -rte_errno; +} + +/** + * DPDK callback to release a Tx queue. + * + * @param dpdk_txq + * Generic Tx queue pointer. + */ +void +mlx4_tx_queue_release(void *dpdk_txq) +{ + struct txq *txq = (struct txq *)dpdk_txq; + struct priv *priv; + unsigned int i; + + if (txq == NULL) + return; + priv = txq->priv; + for (i = 0; i != priv->dev->data->nb_tx_queues; ++i) + if (priv->dev->data->tx_queues[i] == txq) { + DEBUG("%p: removing Tx queue %p from list", + (void *)priv->dev, (void *)txq); + priv->dev->data->tx_queues[i] = NULL; + break; + } + mlx4_txq_free_elts(txq); + if (txq->qp) + claim_zero(ibv_destroy_qp(txq->qp)); + if (txq->cq) + claim_zero(ibv_destroy_cq(txq->cq)); + for (i = 0; i != RTE_DIM(txq->mp2mr); ++i) { + if (!txq->mp2mr[i].mp) + break; + assert(txq->mp2mr[i].mr); + mlx4_mr_put(txq->mp2mr[i].mr); + } + rte_free(txq); +} diff --git a/dpdk/drivers/net/mlx4/mlx4_utils.c b/dpdk/drivers/net/mlx4/mlx4_utils.c new file mode 100644 index 00000000..f18c7145 --- /dev/null +++ b/dpdk/drivers/net/mlx4/mlx4_utils.c @@ -0,0 +1,217 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of 6WIND S.A. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** + * @file + * Utility functions used by the mlx4 driver. + */ + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "mlx4_utils.h" + +/** + * Make a file descriptor non-blocking. + * + * @param fd + * File descriptor to alter. + * + * @return + * 0 on success, negative errno value otherwise and rte_errno is set. + */ +int +mlx4_fd_set_non_blocking(int fd) +{ + int ret = fcntl(fd, F_GETFL); + + if (ret != -1 && !fcntl(fd, F_SETFL, ret | O_NONBLOCK)) + return 0; + assert(errno); + rte_errno = errno; + return -rte_errno; +} + +/** + * Internal helper to allocate memory once for several disparate objects. + * + * The most restrictive alignment constraint for standard objects is assumed + * to be sizeof(double) and is used as a default value. + * + * C11 code would include stdalign.h and use alignof(max_align_t) however + * we'll stick with C99 for the time being. + */ +static inline size_t +mlx4_mallocv_inline(const char *type, const struct mlx4_malloc_vec *vec, + unsigned int cnt, int zero, int socket) +{ + unsigned int i; + size_t size; + size_t least; + uint8_t *data = NULL; + int fill = !vec[0].addr; + +fill: + size = 0; + least = 0; + for (i = 0; i < cnt; ++i) { + size_t align = (uintptr_t)vec[i].align; + + if (!align) { + align = sizeof(double); + } else if (!rte_is_power_of_2(align)) { + rte_errno = EINVAL; + goto error; + } + if (least < align) + least = align; + align = RTE_ALIGN_CEIL(size, align); + size = align + vec[i].size; + if (fill && vec[i].addr) + *vec[i].addr = data + align; + } + if (fill) + return size; + if (!zero) + data = rte_malloc_socket(type, size, least, socket); + else + data = rte_zmalloc_socket(type, size, least, socket); + if (data) { + fill = 1; + goto fill; + } + rte_errno = ENOMEM; +error: + for (i = 0; i != cnt; ++i) + if (vec[i].addr) + *vec[i].addr = NULL; + return 0; +} + +/** + * Allocate memory once for several disparate objects. + * + * This function adds iovec-like semantics (e.g. readv()) to rte_malloc(). + * Memory is allocated once for several contiguous objects of nonuniform + * sizes and alignment constraints. + * + * Each entry of @p vec describes the size, alignment constraint and + * provides a buffer address where the resulting object pointer must be + * stored. + * + * The buffer of the first entry is guaranteed to point to the beginning of + * the allocated region and is safe to use with rte_free(). + * + * NULL buffers are silently ignored. + * + * Providing a NULL buffer in the first entry prevents this function from + * allocating any memory but has otherwise no effect on its behavior. In + * this case, the contents of remaining non-NULL buffers are updated with + * addresses relative to zero (i.e. offsets that would have been used during + * the allocation). + * + * @param[in] type + * A string identifying the type of allocated objects (useful for debug + * purposes, such as identifying the cause of a memory leak). Can be NULL. + * @param[in, out] vec + * Description of objects to allocate memory for. + * @param cnt + * Number of entries in @p vec. + * + * @return + * Size in bytes of the allocated region including any padding. In case of + * error, rte_errno is set, 0 is returned and NULL is stored in the + * non-NULL buffers pointed by @p vec. + * + * @see struct mlx4_malloc_vec + * @see rte_malloc() + */ +size_t +mlx4_mallocv(const char *type, const struct mlx4_malloc_vec *vec, + unsigned int cnt) +{ + return mlx4_mallocv_inline(type, vec, cnt, 0, SOCKET_ID_ANY); +} + +/** + * Combines the semantics of mlx4_mallocv() with those of rte_zmalloc(). + * + * @see mlx4_mallocv() + * @see rte_zmalloc() + */ +size_t +mlx4_zmallocv(const char *type, const struct mlx4_malloc_vec *vec, + unsigned int cnt) +{ + return mlx4_mallocv_inline(type, vec, cnt, 1, SOCKET_ID_ANY); +} + +/** + * Socket-aware version of mlx4_mallocv(). + * + * This function takes one additional parameter. + * + * @param socket + * NUMA socket to allocate memory on. If SOCKET_ID_ANY is used, this + * function will behave the same as mlx4_mallocv(). + * + * @see mlx4_mallocv() + * @see rte_malloc_socket() + */ +size_t +mlx4_mallocv_socket(const char *type, const struct mlx4_malloc_vec *vec, + unsigned int cnt, int socket) +{ + return mlx4_mallocv_inline(type, vec, cnt, 0, socket); +} + +/** + * Combines the semantics of mlx4_mallocv_socket() with those of + * mlx4_zmalloc_socket(). + * + * @see mlx4_mallocv_socket() + * @see rte_zmalloc_socket() + */ +size_t +mlx4_zmallocv_socket(const char *type, const struct mlx4_malloc_vec *vec, + unsigned int cnt, int socket) +{ + return mlx4_mallocv_inline(type, vec, cnt, 1, socket); +} diff --git a/dpdk/drivers/net/mlx4/mlx4_utils.h b/dpdk/drivers/net/mlx4/mlx4_utils.h new file mode 100644 index 00000000..4f11405a --- /dev/null +++ b/dpdk/drivers/net/mlx4/mlx4_utils.h @@ -0,0 +1,127 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of 6WIND S.A. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef MLX4_UTILS_H_ +#define MLX4_UTILS_H_ + +#include +#include +#include + +#include +#include + +#include "mlx4.h" + +#ifndef NDEBUG + +/* + * When debugging is enabled (NDEBUG not defined), file, line and function + * information replace the driver name (MLX4_DRIVER_NAME) in log messages. + */ + +/** Return the file name part of a path. */ +static inline const char * +pmd_drv_log_basename(const char *s) +{ + const char *n = s; + + while (*n) + if (*(n++) == '/') + s = n; + return s; +} + +#define PMD_DRV_LOG(level, ...) \ + RTE_LOG(level, PMD, \ + RTE_FMT("%s:%u: %s(): " RTE_FMT_HEAD(__VA_ARGS__,) "\n", \ + pmd_drv_log_basename(__FILE__), \ + __LINE__, \ + __func__, \ + RTE_FMT_TAIL(__VA_ARGS__,))) +#define DEBUG(...) PMD_DRV_LOG(DEBUG, __VA_ARGS__) +#define claim_zero(...) assert((__VA_ARGS__) == 0) + +#else /* NDEBUG */ + +/* + * Like assert(), DEBUG() becomes a no-op and claim_zero() does not perform + * any check when debugging is disabled. + */ + +#define PMD_DRV_LOG(level, ...) \ + RTE_LOG(level, PMD, \ + RTE_FMT(MLX4_DRIVER_NAME ": " \ + RTE_FMT_HEAD(__VA_ARGS__,) "\n", \ + RTE_FMT_TAIL(__VA_ARGS__,))) +#define DEBUG(...) (void)0 +#define claim_zero(...) (__VA_ARGS__) + +#endif /* NDEBUG */ + +#define INFO(...) PMD_DRV_LOG(INFO, __VA_ARGS__) +#define WARN(...) PMD_DRV_LOG(WARNING, __VA_ARGS__) +#define ERROR(...) PMD_DRV_LOG(ERR, __VA_ARGS__) + +/** Allocate a buffer on the stack and fill it with a printf format string. */ +#define MKSTR(name, ...) \ + char name[snprintf(NULL, 0, __VA_ARGS__) + 1]; \ + \ + snprintf(name, sizeof(name), __VA_ARGS__) + +/** Generate a string out of the provided arguments. */ +#define MLX4_STR(...) # __VA_ARGS__ + +/** Similar to MLX4_STR() with enclosed macros expanded first. */ +#define MLX4_STR_EXPAND(...) MLX4_STR(__VA_ARGS__) + +/** Object description used with mlx4_mallocv() and similar functions. */ +struct mlx4_malloc_vec { + size_t align; /**< Alignment constraint (power of 2), 0 if unknown. */ + size_t size; /**< Object size. */ + void **addr; /**< Storage for allocation address. */ +}; + +/* mlx4_utils.c */ + +int mlx4_fd_set_non_blocking(int fd); +size_t mlx4_mallocv(const char *type, const struct mlx4_malloc_vec *vec, + unsigned int cnt); +size_t mlx4_zmallocv(const char *type, const struct mlx4_malloc_vec *vec, + unsigned int cnt); +size_t mlx4_mallocv_socket(const char *type, const struct mlx4_malloc_vec *vec, + unsigned int cnt, int socket); +size_t mlx4_zmallocv_socket(const char *type, const struct mlx4_malloc_vec *vec, + unsigned int cnt, int socket); + +#endif /* MLX4_UTILS_H_ */ diff --git a/dpdk/drivers/net/mlx5/Makefile b/dpdk/drivers/net/mlx5/Makefile index cf87f0b1..a3984eb9 100644 --- a/dpdk/drivers/net/mlx5/Makefile +++ b/dpdk/drivers/net/mlx5/Makefile @@ -39,6 +39,10 @@ SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5.c SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_rxq.c SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_txq.c SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_rxtx.c +ifneq ($(filter y,$(CONFIG_RTE_ARCH_X86_64) \ + $(CONFIG_RTE_ARCH_ARM64)),) +SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_rxtx_vec.c +endif SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_trigger.c SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_ethdev.c SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_mac.c @@ -46,19 +50,13 @@ SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_rxmode.c SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_vlan.c SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_stats.c SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_rss.c -SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_fdir.c SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_mr.c - -# Dependencies. -DEPDIRS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += lib/librte_ether -DEPDIRS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += lib/librte_mbuf -DEPDIRS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += lib/librte_eal -DEPDIRS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += lib/librte_mempool -DEPDIRS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += lib/librte_kvargs +SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_flow.c +SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_socket.c # Basic CFLAGS. CFLAGS += -O3 -CFLAGS += -std=gnu99 -Wall -Wextra +CFLAGS += -std=c11 -Wall -Wextra CFLAGS += -g CFLAGS += -I. CFLAGS += -D_BSD_SOURCE @@ -66,7 +64,10 @@ CFLAGS += -D_DEFAULT_SOURCE CFLAGS += -D_XOPEN_SOURCE=600 CFLAGS += $(WERROR_FLAGS) CFLAGS += -Wno-strict-prototypes -LDLIBS += -libverbs +LDLIBS += -libverbs -lmlx5 +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs +LDLIBS += -lrte_bus_pci # A few warnings cannot be avoided in external headers. CFLAGS += -Wno-error=cast-qual @@ -104,22 +105,27 @@ endif mlx5_autoconf.h.new: FORCE -mlx5_autoconf.h.new: $(RTE_SDK)/scripts/auto-config-h.sh +mlx5_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh $Q $(RM) -f -- '$@' $Q sh -- '$<' '$@' \ - HAVE_VERBS_IBV_EXP_CQ_COMPRESSED_CQE \ - infiniband/verbs_exp.h \ - enum IBV_EXP_CQ_COMPRESSED_CQE \ + HAVE_IBV_DEVICE_VXLAN_SUPPORT \ + infiniband/verbs.h \ + enum IBV_DEVICE_VXLAN_SUPPORT \ $(AUTOCONF_OUTPUT) $Q sh -- '$<' '$@' \ - HAVE_VERBS_MLX5_ETH_VLAN_INLINE_HEADER_SIZE \ - infiniband/mlx5_hw.h \ - enum MLX5_ETH_VLAN_INLINE_HEADER_SIZE \ + HAVE_IBV_WQ_FLAG_RX_END_PADDING \ + infiniband/verbs.h \ + enum IBV_WQ_FLAG_RX_END_PADDING \ $(AUTOCONF_OUTPUT) $Q sh -- '$<' '$@' \ - HAVE_VERBS_MLX5_OPCODE_TSO \ - infiniband/mlx5_hw.h \ - enum MLX5_OPCODE_TSO \ + HAVE_IBV_MLX5_MOD_MPW \ + infiniband/mlx5dv.h \ + enum MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED \ + $(AUTOCONF_OUTPUT) + $Q sh -- '$<' '$@' \ + HAVE_IBV_MLX5_MOD_CQE_128B_COMP \ + infiniband/mlx5dv.h \ + enum MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP \ $(AUTOCONF_OUTPUT) $Q sh -- '$<' '$@' \ HAVE_ETHTOOL_LINK_MODE_25G \ @@ -136,6 +142,11 @@ mlx5_autoconf.h.new: $(RTE_SDK)/scripts/auto-config-h.sh /usr/include/linux/ethtool.h \ enum ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT \ $(AUTOCONF_OUTPUT) + $Q sh -- '$<' '$@' \ + HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT \ + infiniband/verbs.h \ + enum IBV_FLOW_SPEC_ACTION_COUNT \ + $(AUTOCONF_OUTPUT) # Create mlx5_autoconf.h or update it in case it differs from the new one. diff --git a/dpdk/drivers/net/mlx5/mlx5.c b/dpdk/drivers/net/mlx5/mlx5.c index 9448374e..45e0e8db 100644 --- a/dpdk/drivers/net/mlx5/mlx5.c +++ b/dpdk/drivers/net/mlx5/mlx5.c @@ -50,18 +50,13 @@ #pragma GCC diagnostic error "-Wpedantic" #endif -/* DPDK headers don't like -pedantic. */ -#ifdef PEDANTIC -#pragma GCC diagnostic ignored "-Wpedantic" -#endif #include #include +#include #include +#include #include #include -#ifdef PEDANTIC -#pragma GCC diagnostic error "-Wpedantic" -#endif #include "mlx5.h" #include "mlx5_utils.h" @@ -84,6 +79,44 @@ /* Device parameter to enable multi-packet send WQEs. */ #define MLX5_TXQ_MPW_EN "txq_mpw_en" +/* Device parameter to include 2 dsegs in the title WQEBB. */ +#define MLX5_TXQ_MPW_HDR_DSEG_EN "txq_mpw_hdr_dseg_en" + +/* Device parameter to limit the size of inlining packet. */ +#define MLX5_TXQ_MAX_INLINE_LEN "txq_max_inline_len" + +/* Device parameter to enable hardware TSO offload. */ +#define MLX5_TSO "tso" + +/* Device parameter to enable hardware Tx vector. */ +#define MLX5_TX_VEC_EN "tx_vec_en" + +/* Device parameter to enable hardware Rx vector. */ +#define MLX5_RX_VEC_EN "rx_vec_en" + +/* Default PMD specific parameter value. */ +#define MLX5_ARG_UNSET (-1) + +#ifndef HAVE_IBV_MLX5_MOD_MPW +#define MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED (1 << 2) +#define MLX5DV_CONTEXT_FLAGS_ENHANCED_MPW (1 << 3) +#endif + +#ifndef HAVE_IBV_MLX5_MOD_CQE_128B_COMP +#define MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP (1 << 4) +#endif + +struct mlx5_args { + int cqe_comp; + int txq_inline; + int txqs_inline; + int mps; + int mpw_hdr_dseg; + int inline_max_packet_sz; + int tso; + int tx_vec_en; + int rx_vec_en; +}; /** * Retrieve integer value from environment variable. * @@ -103,6 +136,50 @@ mlx5_getenv_int(const char *name) return atoi(val); } +/** + * Verbs callback to allocate a memory. This function should allocate the space + * according to the size provided residing inside a huge page. + * Please note that all allocation must respect the alignment from libmlx5 + * (i.e. currently sysconf(_SC_PAGESIZE)). + * + * @param[in] size + * The size in bytes of the memory to allocate. + * @param[in] data + * A pointer to the callback data. + * + * @return + * a pointer to the allocate space. + */ +static void * +mlx5_alloc_verbs_buf(size_t size, void *data) +{ + struct priv *priv = data; + void *ret; + size_t alignment = sysconf(_SC_PAGESIZE); + + assert(data != NULL); + ret = rte_malloc_socket(__func__, size, alignment, + priv->dev->device->numa_node); + DEBUG("Extern alloc size: %lu, align: %lu: %p", size, alignment, ret); + return ret; +} + +/** + * Verbs callback to free a memory. + * + * @param[in] ptr + * A pointer to the memory to free. + * @param[in] data + * A pointer to the callback data. + */ +static void +mlx5_free_verbs_buf(void *ptr, void *data __rte_unused) +{ + assert(data != NULL); + DEBUG("Extern free request: %p", ptr); + rte_free(ptr); +} + /** * DPDK callback to close the device. * @@ -116,6 +193,7 @@ mlx5_dev_close(struct rte_eth_dev *dev) { struct priv *priv = mlx5_get_priv(dev); unsigned int i; + int ret; priv_lock(priv); DEBUG("%p: closing device \"%s\"", @@ -123,48 +201,23 @@ mlx5_dev_close(struct rte_eth_dev *dev) ((priv->ctx != NULL) ? priv->ctx->device->name : "")); /* In case mlx5_dev_stop() has not been called. */ priv_dev_interrupt_handler_uninstall(priv, dev); - priv_special_flow_disable_all(priv); - priv_mac_addrs_disable(priv); - priv_destroy_hash_rxqs(priv); - - /* Remove flow director elements. */ - priv_fdir_disable(priv); - priv_fdir_delete_filters_list(priv); - + priv_dev_traffic_disable(priv, dev); /* Prevent crashes when queues are still in use. */ dev->rx_pkt_burst = removed_rx_burst; dev->tx_pkt_burst = removed_tx_burst; if (priv->rxqs != NULL) { /* XXX race condition if mlx5_rx_burst() is still running. */ usleep(1000); - for (i = 0; (i != priv->rxqs_n); ++i) { - struct rxq *rxq = (*priv->rxqs)[i]; - struct rxq_ctrl *rxq_ctrl; - - if (rxq == NULL) - continue; - rxq_ctrl = container_of(rxq, struct rxq_ctrl, rxq); - (*priv->rxqs)[i] = NULL; - rxq_cleanup(rxq_ctrl); - rte_free(rxq_ctrl); - } + for (i = 0; (i != priv->rxqs_n); ++i) + mlx5_priv_rxq_release(priv, i); priv->rxqs_n = 0; priv->rxqs = NULL; } if (priv->txqs != NULL) { /* XXX race condition if mlx5_tx_burst() is still running. */ usleep(1000); - for (i = 0; (i != priv->txqs_n); ++i) { - struct txq *txq = (*priv->txqs)[i]; - struct txq_ctrl *txq_ctrl; - - if (txq == NULL) - continue; - txq_ctrl = container_of(txq, struct txq_ctrl, txq); - (*priv->txqs)[i] = NULL; - txq_cleanup(txq_ctrl); - rte_free(txq_ctrl); - } + for (i = 0; (i != priv->txqs_n); ++i) + mlx5_priv_txq_release(priv, i); priv->txqs_n = 0; priv->txqs = NULL; } @@ -174,18 +227,40 @@ mlx5_dev_close(struct rte_eth_dev *dev) claim_zero(ibv_close_device(priv->ctx)); } else assert(priv->ctx == NULL); - if (priv->rss_conf != NULL) { - for (i = 0; (i != hash_rxq_init_n); ++i) - rte_free((*priv->rss_conf)[i]); - rte_free(priv->rss_conf); - } + if (priv->rss_conf.rss_key != NULL) + rte_free(priv->rss_conf.rss_key); if (priv->reta_idx != NULL) rte_free(priv->reta_idx); + priv_socket_uninit(priv); + ret = mlx5_priv_hrxq_ibv_verify(priv); + if (ret) + WARN("%p: some Hash Rx queue still remain", (void *)priv); + ret = mlx5_priv_ind_table_ibv_verify(priv); + if (ret) + WARN("%p: some Indirection table still remain", (void *)priv); + ret = mlx5_priv_rxq_ibv_verify(priv); + if (ret) + WARN("%p: some Verbs Rx queue still remain", (void *)priv); + ret = mlx5_priv_rxq_verify(priv); + if (ret) + WARN("%p: some Rx Queues still remain", (void *)priv); + ret = mlx5_priv_txq_ibv_verify(priv); + if (ret) + WARN("%p: some Verbs Tx queue still remain", (void *)priv); + ret = mlx5_priv_txq_verify(priv); + if (ret) + WARN("%p: some Tx Queues still remain", (void *)priv); + ret = priv_flow_verify(priv); + if (ret) + WARN("%p: some flows still remain", (void *)priv); + ret = priv_mr_verify(priv); + if (ret) + WARN("%p: some Memory Region still remain", (void *)priv); priv_unlock(priv); memset(priv, 0, sizeof(*priv)); } -static const struct eth_dev_ops mlx5_dev_ops = { +const struct eth_dev_ops mlx5_dev_ops = { .dev_configure = mlx5_dev_configure, .dev_start = mlx5_dev_start, .dev_stop = mlx5_dev_stop, @@ -199,6 +274,9 @@ static const struct eth_dev_ops mlx5_dev_ops = { .link_update = mlx5_link_update, .stats_get = mlx5_stats_get, .stats_reset = mlx5_stats_reset, + .xstats_get = mlx5_xstats_get, + .xstats_reset = mlx5_xstats_reset, + .xstats_get_names = mlx5_xstats_get_names, .dev_infos_get = mlx5_dev_infos_get, .dev_supported_ptypes_get = mlx5_dev_supported_ptypes_get, .vlan_filter_set = mlx5_vlan_filter_set, @@ -219,6 +297,57 @@ static const struct eth_dev_ops mlx5_dev_ops = { .rss_hash_update = mlx5_rss_hash_update, .rss_hash_conf_get = mlx5_rss_hash_conf_get, .filter_ctrl = mlx5_dev_filter_ctrl, + .rx_descriptor_status = mlx5_rx_descriptor_status, + .tx_descriptor_status = mlx5_tx_descriptor_status, + .rx_queue_intr_enable = mlx5_rx_intr_enable, + .rx_queue_intr_disable = mlx5_rx_intr_disable, +}; + +static const struct eth_dev_ops mlx5_dev_sec_ops = { + .stats_get = mlx5_stats_get, + .stats_reset = mlx5_stats_reset, + .xstats_get = mlx5_xstats_get, + .xstats_reset = mlx5_xstats_reset, + .xstats_get_names = mlx5_xstats_get_names, + .dev_infos_get = mlx5_dev_infos_get, + .rx_descriptor_status = mlx5_rx_descriptor_status, + .tx_descriptor_status = mlx5_tx_descriptor_status, +}; + +/* Available operators in flow isolated mode. */ +const struct eth_dev_ops mlx5_dev_ops_isolate = { + .dev_configure = mlx5_dev_configure, + .dev_start = mlx5_dev_start, + .dev_stop = mlx5_dev_stop, + .dev_set_link_down = mlx5_set_link_down, + .dev_set_link_up = mlx5_set_link_up, + .dev_close = mlx5_dev_close, + .link_update = mlx5_link_update, + .stats_get = mlx5_stats_get, + .stats_reset = mlx5_stats_reset, + .xstats_get = mlx5_xstats_get, + .xstats_reset = mlx5_xstats_reset, + .xstats_get_names = mlx5_xstats_get_names, + .dev_infos_get = mlx5_dev_infos_get, + .dev_supported_ptypes_get = mlx5_dev_supported_ptypes_get, + .vlan_filter_set = mlx5_vlan_filter_set, + .rx_queue_setup = mlx5_rx_queue_setup, + .tx_queue_setup = mlx5_tx_queue_setup, + .rx_queue_release = mlx5_rx_queue_release, + .tx_queue_release = mlx5_tx_queue_release, + .flow_ctrl_get = mlx5_dev_get_flow_ctrl, + .flow_ctrl_set = mlx5_dev_set_flow_ctrl, + .mac_addr_remove = mlx5_mac_addr_remove, + .mac_addr_add = mlx5_mac_addr_add, + .mac_addr_set = mlx5_mac_addr_set, + .mtu_set = mlx5_dev_set_mtu, + .vlan_strip_queue_set = mlx5_vlan_strip_queue_set, + .vlan_offload_set = mlx5_vlan_offload_set, + .filter_ctrl = mlx5_dev_filter_ctrl, + .rx_descriptor_status = mlx5_rx_descriptor_status, + .tx_descriptor_status = mlx5_tx_descriptor_status, + .rx_queue_intr_enable = mlx5_rx_intr_enable, + .rx_queue_intr_disable = mlx5_rx_intr_disable, }; static struct { @@ -270,7 +399,7 @@ mlx5_dev_idx(struct rte_pci_addr *pci_addr) static int mlx5_args_check(const char *key, const char *val, void *opaque) { - struct priv *priv = opaque; + struct mlx5_args *args = opaque; unsigned long tmp; errno = 0; @@ -280,13 +409,23 @@ mlx5_args_check(const char *key, const char *val, void *opaque) return errno; } if (strcmp(MLX5_RXQ_CQE_COMP_EN, key) == 0) { - priv->cqe_comp = !!tmp; + args->cqe_comp = !!tmp; } else if (strcmp(MLX5_TXQ_INLINE, key) == 0) { - priv->txq_inline = tmp; + args->txq_inline = tmp; } else if (strcmp(MLX5_TXQS_MIN_INLINE, key) == 0) { - priv->txqs_inline = tmp; + args->txqs_inline = tmp; } else if (strcmp(MLX5_TXQ_MPW_EN, key) == 0) { - priv->mps = !!tmp; + args->mps = !!tmp; + } else if (strcmp(MLX5_TXQ_MPW_HDR_DSEG_EN, key) == 0) { + args->mpw_hdr_dseg = !!tmp; + } else if (strcmp(MLX5_TXQ_MAX_INLINE_LEN, key) == 0) { + args->inline_max_packet_sz = tmp; + } else if (strcmp(MLX5_TSO, key) == 0) { + args->tso = !!tmp; + } else if (strcmp(MLX5_TX_VEC_EN, key) == 0) { + args->tx_vec_en = !!tmp; + } else if (strcmp(MLX5_RX_VEC_EN, key) == 0) { + args->rx_vec_en = !!tmp; } else { WARN("%s: unknown parameter", key); return -EINVAL; @@ -306,13 +445,18 @@ mlx5_args_check(const char *key, const char *val, void *opaque) * 0 on success, errno value on failure. */ static int -mlx5_args(struct priv *priv, struct rte_devargs *devargs) +mlx5_args(struct mlx5_args *args, struct rte_devargs *devargs) { const char **params = (const char *[]){ MLX5_RXQ_CQE_COMP_EN, MLX5_TXQ_INLINE, MLX5_TXQS_MIN_INLINE, MLX5_TXQ_MPW_EN, + MLX5_TXQ_MPW_HDR_DSEG_EN, + MLX5_TXQ_MAX_INLINE_LEN, + MLX5_TSO, + MLX5_TX_VEC_EN, + MLX5_RX_VEC_EN, NULL, }; struct rte_kvargs *kvlist; @@ -329,16 +473,50 @@ mlx5_args(struct priv *priv, struct rte_devargs *devargs) for (i = 0; (params[i] != NULL); ++i) { if (rte_kvargs_count(kvlist, params[i])) { ret = rte_kvargs_process(kvlist, params[i], - mlx5_args_check, priv); - if (ret != 0) + mlx5_args_check, args); + if (ret != 0) { + rte_kvargs_free(kvlist); return ret; + } } } rte_kvargs_free(kvlist); return 0; } -static struct eth_driver mlx5_driver; +static struct rte_pci_driver mlx5_driver; + +/** + * Assign parameters from args into priv, only non default + * values are considered. + * + * @param[out] priv + * Pointer to private structure. + * @param[in] args + * Pointer to args values. + */ +static void +mlx5_args_assign(struct priv *priv, struct mlx5_args *args) +{ + if (args->cqe_comp != MLX5_ARG_UNSET) + priv->cqe_comp = args->cqe_comp; + if (args->txq_inline != MLX5_ARG_UNSET) + priv->txq_inline = args->txq_inline; + if (args->txqs_inline != MLX5_ARG_UNSET) + priv->txqs_inline = args->txqs_inline; + if (args->mps != MLX5_ARG_UNSET) + priv->mps = args->mps ? priv->mps : 0; + if (args->mpw_hdr_dseg != MLX5_ARG_UNSET) + priv->mpw_hdr_dseg = args->mpw_hdr_dseg; + if (args->inline_max_packet_sz != MLX5_ARG_UNSET) + priv->inline_max_packet_sz = args->inline_max_packet_sz; + if (args->tso != MLX5_ARG_UNSET) + priv->tso = args->tso; + if (args->tx_vec_en != MLX5_ARG_UNSET) + priv->tx_vec_en = args->tx_vec_en; + if (args->rx_vec_en != MLX5_ARG_UNSET) + priv->rx_vec_en = args->rx_vec_en; +} /** * DPDK callback to register a PCI device. @@ -355,20 +533,26 @@ static struct eth_driver mlx5_driver; * 0 on success, negative errno value on failure. */ static int -mlx5_pci_devinit(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) +mlx5_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) { struct ibv_device **list; struct ibv_device *ibv_dev; int err = 0; struct ibv_context *attr_ctx = NULL; - struct ibv_device_attr device_attr; + struct ibv_device_attr_ex device_attr; unsigned int sriov; unsigned int mps; + unsigned int cqe_comp; + unsigned int tunnel_en = 0; int idx; int i; + struct mlx5dv_context attrs_out; +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT + struct ibv_counter_set_description cs_desc; +#endif (void)pci_drv; - assert(pci_drv == &mlx5_driver.pci_drv); + assert(pci_drv == &mlx5_driver); /* Get mlx5_dev[] index. */ idx = mlx5_dev_idx(&pci_dev->addr); if (idx == -1) { @@ -382,10 +566,8 @@ mlx5_pci_devinit(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) list = ibv_get_device_list(&i); if (list == NULL) { assert(errno); - if (errno == ENOSYS) { - WARN("cannot list devices, is ib_uverbs loaded?"); - return 0; - } + if (errno == ENOSYS) + ERROR("cannot list devices, is ib_uverbs loaded?"); return -errno; } assert(i >= 0); @@ -408,15 +590,29 @@ mlx5_pci_devinit(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) sriov = ((pci_dev->id.device_id == PCI_DEVICE_ID_MELLANOX_CONNECTX4VF) || (pci_dev->id.device_id == - PCI_DEVICE_ID_MELLANOX_CONNECTX4LXVF)); - /* Multi-packet send is only supported by ConnectX-4 Lx PF. */ - mps = (pci_dev->id.device_id == - PCI_DEVICE_ID_MELLANOX_CONNECTX4LX); + PCI_DEVICE_ID_MELLANOX_CONNECTX4LXVF) || + (pci_dev->id.device_id == + PCI_DEVICE_ID_MELLANOX_CONNECTX5VF) || + (pci_dev->id.device_id == + PCI_DEVICE_ID_MELLANOX_CONNECTX5EXVF)); + switch (pci_dev->id.device_id) { + case PCI_DEVICE_ID_MELLANOX_CONNECTX4: + tunnel_en = 1; + break; + case PCI_DEVICE_ID_MELLANOX_CONNECTX4LX: + case PCI_DEVICE_ID_MELLANOX_CONNECTX5: + case PCI_DEVICE_ID_MELLANOX_CONNECTX5VF: + case PCI_DEVICE_ID_MELLANOX_CONNECTX5EX: + case PCI_DEVICE_ID_MELLANOX_CONNECTX5EXVF: + tunnel_en = 1; + break; + default: + break; + } INFO("PCI information matches, using device \"%s\"" - " (SR-IOV: %s, MPS: %s)", + " (SR-IOV: %s)", list[i]->name, - sriov ? "true" : "false", - mps ? "true" : "false"); + sriov ? "true" : "false"); attr_ctx = ibv_open_device(list[i]); err = errno; break; @@ -425,11 +621,11 @@ mlx5_pci_devinit(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) ibv_free_device_list(list); switch (err) { case 0: - WARN("cannot access device, is mlx5_ib loaded?"); - return 0; + ERROR("cannot access device, is mlx5_ib loaded?"); + return -ENODEV; case EINVAL: - WARN("cannot use device, are drivers up to date?"); - return 0; + ERROR("cannot use device, are drivers up to date?"); + return -EINVAL; } assert(err > 0); return -err; @@ -437,11 +633,34 @@ mlx5_pci_devinit(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) ibv_dev = list[i]; DEBUG("device opened"); - if (ibv_query_device(attr_ctx, &device_attr)) + /* + * Multi-packet send is supported by ConnectX-4 Lx PF as well + * as all ConnectX-5 devices. + */ + mlx5dv_query_device(attr_ctx, &attrs_out); + if (attrs_out.flags & MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED) { + if (attrs_out.flags & MLX5DV_CONTEXT_FLAGS_ENHANCED_MPW) { + DEBUG("Enhanced MPW is supported"); + mps = MLX5_MPW_ENHANCED; + } else { + DEBUG("MPW is supported"); + mps = MLX5_MPW; + } + } else { + DEBUG("MPW isn't supported"); + mps = MLX5_MPW_DISABLED; + } + if (RTE_CACHE_LINE_SIZE == 128 && + !(attrs_out.flags & MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP)) + cqe_comp = 0; + else + cqe_comp = 1; + if (ibv_query_device_ex(attr_ctx, NULL, &device_attr)) goto error; - INFO("%u port(s) detected", device_attr.phys_port_cnt); + INFO("%u port(s) detected", device_attr.orig_attr.phys_port_cnt); - for (i = 0; i < device_attr.phys_port_cnt; i++) { + for (i = 0; i < device_attr.orig_attr.phys_port_cnt; i++) { + char name[RTE_ETH_NAME_MAX_LEN]; uint32_t port = i + 1; /* ports are indexed from one */ uint32_t test = (1 << i); struct ibv_context *ctx = NULL; @@ -449,23 +668,64 @@ mlx5_pci_devinit(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) struct ibv_pd *pd = NULL; struct priv *priv = NULL; struct rte_eth_dev *eth_dev; - struct ibv_exp_device_attr exp_device_attr; + struct ibv_device_attr_ex device_attr_ex; struct ether_addr mac; uint16_t num_vfs = 0; + struct ibv_device_attr_ex device_attr; + struct mlx5_args args = { + .cqe_comp = MLX5_ARG_UNSET, + .txq_inline = MLX5_ARG_UNSET, + .txqs_inline = MLX5_ARG_UNSET, + .mps = MLX5_ARG_UNSET, + .mpw_hdr_dseg = MLX5_ARG_UNSET, + .inline_max_packet_sz = MLX5_ARG_UNSET, + .tso = MLX5_ARG_UNSET, + .tx_vec_en = MLX5_ARG_UNSET, + .rx_vec_en = MLX5_ARG_UNSET, + }; - exp_device_attr.comp_mask = - IBV_EXP_DEVICE_ATTR_EXP_CAP_FLAGS | - IBV_EXP_DEVICE_ATTR_RX_HASH | - IBV_EXP_DEVICE_ATTR_VLAN_OFFLOADS | - IBV_EXP_DEVICE_ATTR_RX_PAD_END_ALIGN | - 0; + snprintf(name, sizeof(name), PCI_PRI_FMT, + pci_dev->addr.domain, pci_dev->addr.bus, + pci_dev->addr.devid, pci_dev->addr.function); + + mlx5_dev[idx].ports |= test; + + if (rte_eal_process_type() == RTE_PROC_SECONDARY) { + eth_dev = rte_eth_dev_attach_secondary(name); + if (eth_dev == NULL) { + ERROR("can not attach rte ethdev"); + err = ENOMEM; + goto error; + } + eth_dev->device = &pci_dev->device; + eth_dev->dev_ops = &mlx5_dev_sec_ops; + priv = eth_dev->data->dev_private; + /* Receive command fd from primary process */ + err = priv_socket_connect(priv); + if (err < 0) { + err = -err; + goto error; + } + /* Remap UAR for Tx queues. */ + err = priv_tx_uar_remap(priv, err); + if (err < 0) { + err = -err; + goto error; + } + priv_dev_select_rx_function(priv, eth_dev); + priv_dev_select_tx_function(priv, eth_dev); + continue; + } DEBUG("using port %u (%08" PRIx32 ")", port, test); ctx = ibv_open_device(ibv_dev); - if (ctx == NULL) + if (ctx == NULL) { + err = ENODEV; goto port_error; + } + ibv_query_device_ex(ctx, NULL, &device_attr); /* Check port status. */ err = ibv_query_port(ctx, port, &port_attr); if (err) { @@ -476,6 +736,7 @@ mlx5_pci_devinit(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) if (port_attr.link_layer != IBV_LINK_LAYER_ETHERNET) { ERROR("port %d is not configured in Ethernet mode", port); + err = EINVAL; goto port_error; } @@ -505,82 +766,118 @@ mlx5_pci_devinit(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) } priv->ctx = ctx; + strncpy(priv->ibdev_path, priv->ctx->device->ibdev_path, + sizeof(priv->ibdev_path)); priv->device_attr = device_attr; priv->port = port; priv->pd = pd; priv->mtu = ETHER_MTU; priv->mps = mps; /* Enable MPW by default if supported. */ - priv->cqe_comp = 1; /* Enable compression by default. */ - err = mlx5_args(priv, pci_dev->devargs); + priv->cqe_comp = cqe_comp; + priv->tunnel_en = tunnel_en; + /* Enable vector by default if supported. */ + priv->tx_vec_en = 1; + priv->rx_vec_en = 1; + err = mlx5_args(&args, pci_dev->device.devargs); if (err) { ERROR("failed to process device arguments: %s", strerror(err)); goto port_error; } - if (ibv_exp_query_device(ctx, &exp_device_attr)) { - ERROR("ibv_exp_query_device() failed"); + mlx5_args_assign(priv, &args); + if (ibv_query_device_ex(ctx, NULL, &device_attr_ex)) { + ERROR("ibv_query_device_ex() failed"); goto port_error; } priv->hw_csum = - ((exp_device_attr.exp_device_cap_flags & - IBV_EXP_DEVICE_RX_CSUM_TCP_UDP_PKT) && - (exp_device_attr.exp_device_cap_flags & - IBV_EXP_DEVICE_RX_CSUM_IP_PKT)); + !!(device_attr_ex.device_cap_flags_ex & + IBV_DEVICE_RAW_IP_CSUM); DEBUG("checksum offloading is %ssupported", (priv->hw_csum ? "" : "not ")); +#ifdef HAVE_IBV_DEVICE_VXLAN_SUPPORT priv->hw_csum_l2tun = !!(exp_device_attr.exp_device_cap_flags & - IBV_EXP_DEVICE_VXLAN_SUPPORT); - DEBUG("L2 tunnel checksum offloads are %ssupported", + IBV_DEVICE_VXLAN_SUPPORT); +#endif + DEBUG("Rx L2 tunnel checksum offloads are %ssupported", (priv->hw_csum_l2tun ? "" : "not ")); - priv->ind_table_max_size = exp_device_attr.rx_hash_caps.max_rwq_indirection_table_size; +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT + priv->counter_set_supported = !!(device_attr.max_counter_sets); + ibv_describe_counter_set(ctx, 0, &cs_desc); + DEBUG("counter type = %d, num of cs = %ld, attributes = %d", + cs_desc.counter_type, cs_desc.num_of_cs, + cs_desc.attributes); +#endif + priv->ind_table_max_size = + device_attr_ex.rss_caps.max_rwq_indirection_table_size; /* Remove this check once DPDK supports larger/variable * indirection tables. */ - if (priv->ind_table_max_size > (unsigned int)RSS_INDIRECTION_TABLE_SIZE) - priv->ind_table_max_size = RSS_INDIRECTION_TABLE_SIZE; + if (priv->ind_table_max_size > + (unsigned int)ETH_RSS_RETA_SIZE_512) + priv->ind_table_max_size = ETH_RSS_RETA_SIZE_512; DEBUG("maximum RX indirection table size is %u", priv->ind_table_max_size); - priv->hw_vlan_strip = !!(exp_device_attr.wq_vlan_offloads_cap & - IBV_EXP_RECEIVE_WQ_CVLAN_STRIP); + priv->hw_vlan_strip = !!(device_attr_ex.raw_packet_caps & + IBV_RAW_PACKET_CAP_CVLAN_STRIPPING); DEBUG("VLAN stripping is %ssupported", (priv->hw_vlan_strip ? "" : "not ")); - priv->hw_fcs_strip = !!(exp_device_attr.exp_device_cap_flags & - IBV_EXP_DEVICE_SCATTER_FCS); + priv->hw_fcs_strip = + !!(device_attr_ex.orig_attr.device_cap_flags & + IBV_WQ_FLAGS_SCATTER_FCS); DEBUG("FCS stripping configuration is %ssupported", (priv->hw_fcs_strip ? "" : "not ")); - priv->hw_padding = !!exp_device_attr.rx_pad_end_addr_align; +#ifdef HAVE_IBV_WQ_FLAG_RX_END_PADDING + priv->hw_padding = !!device_attr_ex.rx_pad_end_addr_align; +#endif DEBUG("hardware RX end alignment padding is %ssupported", (priv->hw_padding ? "" : "not ")); priv_get_num_vfs(priv, &num_vfs); priv->sriov = (num_vfs || sriov); + priv->tso = ((priv->tso) && + (device_attr_ex.tso_caps.max_tso > 0) && + (device_attr_ex.tso_caps.supported_qpts & + (1 << IBV_QPT_RAW_PACKET))); + if (priv->tso) + priv->max_tso_payload_sz = + device_attr_ex.tso_caps.max_tso; if (priv->mps && !mps) { ERROR("multi-packet send not supported on this device" " (" MLX5_TXQ_MPW_EN ")"); err = ENOTSUP; goto port_error; + } else if (priv->mps && priv->tso) { + WARN("multi-packet send not supported in conjunction " + "with TSO. MPS disabled"); + priv->mps = 0; } - /* Allocate and register default RSS hash keys. */ - priv->rss_conf = rte_calloc(__func__, hash_rxq_init_n, - sizeof((*priv->rss_conf)[0]), 0); - if (priv->rss_conf == NULL) { - err = ENOMEM; - goto port_error; + INFO("%sMPS is %s", + priv->mps == MLX5_MPW_ENHANCED ? "Enhanced " : "", + priv->mps != MLX5_MPW_DISABLED ? "enabled" : "disabled"); + /* Set default values for Enhanced MPW, a.k.a MPWv2. */ + if (priv->mps == MLX5_MPW_ENHANCED) { + if (args.txqs_inline == MLX5_ARG_UNSET) + priv->txqs_inline = MLX5_EMPW_MIN_TXQS; + if (args.inline_max_packet_sz == MLX5_ARG_UNSET) + priv->inline_max_packet_sz = + MLX5_EMPW_MAX_INLINE_LEN; + if (args.txq_inline == MLX5_ARG_UNSET) + priv->txq_inline = MLX5_WQE_SIZE_MAX - + MLX5_WQE_SIZE; + } + if (priv->cqe_comp && !cqe_comp) { + WARN("Rx CQE compression isn't supported"); + priv->cqe_comp = 0; } - err = rss_hash_rss_conf_new_key(priv, - rss_hash_default_key, - rss_hash_default_key_len, - ETH_RSS_PROTO_MASK); - if (err) - goto port_error; /* Configure the first MAC address by default. */ if (priv_get_mac(priv, &mac.addr_bytes)) { ERROR("cannot get MAC address, is mlx5_en loaded?" " (errno: %s)", strerror(errno)); + err = ENODEV; goto port_error; } INFO("port %u MAC address is %02x:%02x:%02x:%02x:%02x:%02x", @@ -588,14 +885,6 @@ mlx5_pci_devinit(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) mac.addr_bytes[0], mac.addr_bytes[1], mac.addr_bytes[2], mac.addr_bytes[3], mac.addr_bytes[4], mac.addr_bytes[5]); - /* Register MAC address. */ - claim_zero(priv_mac_addr_add(priv, 0, - (const uint8_t (*)[ETHER_ADDR_LEN]) - mac.addr_bytes)); - /* Initialize FD filters list. */ - err = fdir_init_filters_list(priv); - if (err) - goto port_error; #ifndef NDEBUG { char ifname[IF_NAMESIZE]; @@ -611,71 +900,46 @@ mlx5_pci_devinit(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) priv_get_mtu(priv, &priv->mtu); DEBUG("port %u MTU is %u", priv->port, priv->mtu); - /* from rte_ethdev.c */ - { - char name[RTE_ETH_NAME_MAX_LEN]; - - snprintf(name, sizeof(name), "%s port %u", - ibv_get_device_name(ibv_dev), port); - eth_dev = rte_eth_dev_allocate(name, RTE_ETH_DEV_PCI); - } + eth_dev = rte_eth_dev_allocate(name); if (eth_dev == NULL) { ERROR("can not allocate rte ethdev"); err = ENOMEM; goto port_error; } - - /* Secondary processes have to use local storage for their - * private data as well as a copy of eth_dev->data, but this - * pointer must not be modified before burst functions are - * actually called. */ - if (mlx5_is_secondary()) { - struct mlx5_secondary_data *sd = - &mlx5_secondary_data[eth_dev->data->port_id]; - sd->primary_priv = eth_dev->data->dev_private; - if (sd->primary_priv == NULL) { - ERROR("no private data for port %u", - eth_dev->data->port_id); - err = EINVAL; - goto port_error; - } - sd->shared_dev_data = eth_dev->data; - rte_spinlock_init(&sd->lock); - memcpy(sd->data.name, sd->shared_dev_data->name, - sizeof(sd->data.name)); - sd->data.dev_private = priv; - sd->data.rx_mbuf_alloc_failed = 0; - sd->data.mtu = ETHER_MTU; - sd->data.port_id = sd->shared_dev_data->port_id; - sd->data.mac_addrs = priv->mac; - eth_dev->tx_pkt_burst = mlx5_tx_burst_secondary_setup; - eth_dev->rx_pkt_burst = mlx5_rx_burst_secondary_setup; - } else { - eth_dev->data->dev_private = priv; - eth_dev->data->rx_mbuf_alloc_failed = 0; - eth_dev->data->mtu = ETHER_MTU; - eth_dev->data->mac_addrs = priv->mac; - } - - eth_dev->pci_dev = pci_dev; + eth_dev->data->dev_private = priv; + eth_dev->data->mac_addrs = priv->mac; + eth_dev->device = &pci_dev->device; rte_eth_copy_pci_info(eth_dev, pci_dev); - eth_dev->driver = &mlx5_driver; + eth_dev->device->driver = &mlx5_driver.driver; + /* + * Initialize burst functions to prevent crashes before link-up. + */ + eth_dev->rx_pkt_burst = removed_rx_burst; + eth_dev->tx_pkt_burst = removed_tx_burst; priv->dev = eth_dev; eth_dev->dev_ops = &mlx5_dev_ops; + /* Register MAC address. */ + claim_zero(mlx5_mac_addr_add(eth_dev, &mac, 0, 0)); + TAILQ_INIT(&priv->flows); + TAILQ_INIT(&priv->ctrl_flows); - TAILQ_INIT(ð_dev->link_intr_cbs); + /* Hint libmlx5 to use PMD allocator for data plane resources */ + struct mlx5dv_ctx_allocators alctr = { + .alloc = &mlx5_alloc_verbs_buf, + .free = &mlx5_free_verbs_buf, + .data = priv, + }; + mlx5dv_set_context_attr(ctx, MLX5DV_CTX_ATTR_BUF_ALLOCATORS, + (void *)((uintptr_t)&alctr)); /* Bring Ethernet device up. */ DEBUG("forcing Ethernet interface up"); priv_set_flags(priv, ~IFF_UP, IFF_UP); - mlx5_link_update_unlocked(priv->dev, 1); continue; port_error: - if (priv) { - rte_free(priv->rss_conf); + if (priv) rte_free(priv); - } if (pd) claim_zero(ibv_dealloc_pd(pd)); if (ctx) @@ -722,29 +986,45 @@ static const struct rte_pci_id mlx5_pci_id_map[] = { RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, PCI_DEVICE_ID_MELLANOX_CONNECTX4LXVF) }, + { + RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, + PCI_DEVICE_ID_MELLANOX_CONNECTX5) + }, + { + RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, + PCI_DEVICE_ID_MELLANOX_CONNECTX5VF) + }, + { + RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, + PCI_DEVICE_ID_MELLANOX_CONNECTX5EX) + }, + { + RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, + PCI_DEVICE_ID_MELLANOX_CONNECTX5EXVF) + }, { .vendor_id = 0 } }; -static struct eth_driver mlx5_driver = { - .pci_drv = { - .name = MLX5_DRIVER_NAME, - .id_table = mlx5_pci_id_map, - .devinit = mlx5_pci_devinit, - .drv_flags = RTE_PCI_DRV_INTR_LSC, +static struct rte_pci_driver mlx5_driver = { + .driver = { + .name = MLX5_DRIVER_NAME }, - .dev_private_size = sizeof(struct priv) + .id_table = mlx5_pci_id_map, + .probe = mlx5_pci_probe, + .drv_flags = RTE_PCI_DRV_INTR_LSC | RTE_PCI_DRV_INTR_RMV, }; /** * Driver initialization routine. */ -static int -rte_mlx5_pmd_init(const char *name, const char *args) +RTE_INIT(rte_mlx5_pmd_init); +static void +rte_mlx5_pmd_init(void) { - (void)name; - (void)args; + /* Build the static table for ptype conversion. */ + mlx5_set_ptype_table(); /* * RDMAV_HUGEPAGES_SAFE tells ibv_fork_init() we intend to use * huge pages. Calling ibv_fork_init() during init allows @@ -752,15 +1032,13 @@ rte_mlx5_pmd_init(const char *name, const char *args) * using this PMD, which is not supported in forked processes. */ setenv("RDMAV_HUGEPAGES_SAFE", "1", 1); + /* Match the size of Rx completion entry to the size of a cacheline. */ + if (RTE_CACHE_LINE_SIZE == 128) + setenv("MLX5_CQE_SIZE", "128", 0); ibv_fork_init(); - rte_eal_pci_register(&mlx5_driver.pci_drv); - return 0; + rte_pci_register(&mlx5_driver); } -static struct rte_driver rte_mlx5_driver = { - .type = PMD_PDEV, - .init = rte_mlx5_pmd_init, -}; - -PMD_REGISTER_DRIVER(rte_mlx5_driver, mlx5); -DRIVER_REGISTER_PCI_TABLE(mlx5, mlx5_pci_id_map); +RTE_PMD_EXPORT_NAME(net_mlx5, __COUNTER__); +RTE_PMD_REGISTER_PCI_TABLE(net_mlx5, mlx5_pci_id_map); +RTE_PMD_REGISTER_KMOD_DEP(net_mlx5, "* ib_uverbs & mlx5_core & mlx5_ib"); diff --git a/dpdk/drivers/net/mlx5/mlx5.h b/dpdk/drivers/net/mlx5/mlx5.h index 79b7a600..d49595bc 100644 --- a/dpdk/drivers/net/mlx5/mlx5.h +++ b/dpdk/drivers/net/mlx5/mlx5.h @@ -39,6 +39,7 @@ #include #include #include +#include /* Verbs header. */ /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ @@ -50,29 +51,19 @@ #pragma GCC diagnostic error "-Wpedantic" #endif -/* DPDK headers don't like -pedantic. */ -#ifdef PEDANTIC -#pragma GCC diagnostic ignored "-Wpedantic" -#endif +#include #include #include #include #include #include -#ifdef PEDANTIC -#pragma GCC diagnostic error "-Wpedantic" -#endif +#include #include "mlx5_utils.h" #include "mlx5_rxtx.h" #include "mlx5_autoconf.h" #include "mlx5_defs.h" -#if !defined(HAVE_VERBS_IBV_EXP_CQ_COMPRESSED_CQE) || \ - !defined(HAVE_VERBS_MLX5_ETH_VLAN_INLINE_HEADER_SIZE) -#error Mellanox OFED >= 3.3 is required, please refer to the documentation. -#endif - enum { PCI_VENDOR_ID_MELLANOX = 0x15b3, }; @@ -82,71 +73,84 @@ enum { PCI_DEVICE_ID_MELLANOX_CONNECTX4VF = 0x1014, PCI_DEVICE_ID_MELLANOX_CONNECTX4LX = 0x1015, PCI_DEVICE_ID_MELLANOX_CONNECTX4LXVF = 0x1016, + PCI_DEVICE_ID_MELLANOX_CONNECTX5 = 0x1017, + PCI_DEVICE_ID_MELLANOX_CONNECTX5VF = 0x1018, + PCI_DEVICE_ID_MELLANOX_CONNECTX5EX = 0x1019, + PCI_DEVICE_ID_MELLANOX_CONNECTX5EXVF = 0x101a, }; +struct mlx5_xstats_ctrl { + /* Number of device stats. */ + uint16_t stats_n; + /* Index in the device counters table. */ + uint16_t dev_table_idx[MLX5_MAX_XSTATS]; + uint64_t base[MLX5_MAX_XSTATS]; +}; + +/* Flow list . */ +TAILQ_HEAD(mlx5_flows, rte_flow); + struct priv { - struct rte_eth_dev *dev; /* Ethernet device. */ + struct rte_eth_dev *dev; /* Ethernet device of master process. */ struct ibv_context *ctx; /* Verbs context. */ - struct ibv_device_attr device_attr; /* Device properties. */ + struct ibv_device_attr_ex device_attr; /* Device properties. */ struct ibv_pd *pd; /* Protection Domain. */ - /* - * MAC addresses array and configuration bit-field. - * An extra entry that cannot be modified by the DPDK is reserved - * for broadcast frames (destination MAC address ff:ff:ff:ff:ff:ff). - */ - struct ether_addr mac[MLX5_MAX_MAC_ADDRESSES]; - BITFIELD_DECLARE(mac_configured, uint32_t, MLX5_MAX_MAC_ADDRESSES); + char ibdev_path[IBV_SYSFS_PATH_MAX]; /* IB device path for secondary */ + struct ether_addr mac[MLX5_MAX_MAC_ADDRESSES]; /* MAC addresses. */ uint16_t vlan_filter[MLX5_MAX_VLAN_IDS]; /* VLAN filters table. */ unsigned int vlan_filter_n; /* Number of configured VLAN filters. */ /* Device properties. */ uint16_t mtu; /* Configured MTU. */ uint8_t port; /* Physical port number. */ - unsigned int started:1; /* Device started, flows enabled. */ - unsigned int promisc_req:1; /* Promiscuous mode requested. */ - unsigned int allmulti_req:1; /* All multicast mode requested. */ unsigned int hw_csum:1; /* Checksum offload is supported. */ unsigned int hw_csum_l2tun:1; /* Same for L2 tunnels. */ unsigned int hw_vlan_strip:1; /* VLAN stripping is supported. */ unsigned int hw_fcs_strip:1; /* FCS stripping is supported. */ unsigned int hw_padding:1; /* End alignment padding is supported. */ unsigned int sriov:1; /* This is a VF or PF with VF devices. */ - unsigned int mps:1; /* Whether multi-packet send is supported. */ + unsigned int mps:2; /* Multi-packet send mode (0: disabled). */ + unsigned int mpw_hdr_dseg:1; /* Enable DSEGs in the title WQEBB. */ unsigned int cqe_comp:1; /* Whether CQE compression is enabled. */ unsigned int pending_alarm:1; /* An alarm is pending. */ + unsigned int tso:1; /* Whether TSO is supported. */ + unsigned int tunnel_en:1; + unsigned int isolated:1; /* Whether isolated mode is enabled. */ + unsigned int tx_vec_en:1; /* Whether Tx vector is enabled. */ + unsigned int rx_vec_en:1; /* Whether Rx vector is enabled. */ + unsigned int counter_set_supported:1; /* Counter set is supported. */ + /* Whether Tx offloads for tunneled packets are supported. */ + unsigned int max_tso_payload_sz; /* Maximum TCP payload for TSO. */ unsigned int txq_inline; /* Maximum packet size for inlining. */ unsigned int txqs_inline; /* Queue number threshold for inlining. */ + unsigned int inline_max_packet_sz; /* Max packet size for inlining. */ /* RX/TX queues. */ unsigned int rxqs_n; /* RX queues array size. */ unsigned int txqs_n; /* TX queues array size. */ - struct rxq *(*rxqs)[]; /* RX queues. */ - struct txq *(*txqs)[]; /* TX queues. */ - /* Indirection tables referencing all RX WQs. */ - struct ibv_exp_rwq_ind_table *(*ind_tables)[]; - unsigned int ind_tables_n; /* Number of indirection tables. */ + struct mlx5_rxq_data *(*rxqs)[]; /* RX queues. */ + struct mlx5_txq_data *(*txqs)[]; /* TX queues. */ unsigned int ind_table_max_size; /* Maximum indirection table size. */ - /* Hash RX QPs feeding the indirection table. */ - struct hash_rxq (*hash_rxqs)[]; - unsigned int hash_rxqs_n; /* Hash RX QPs array size. */ - /* RSS configuration array indexed by hash RX queue type. */ - struct rte_eth_rss_conf *(*rss_conf)[]; - uint64_t rss_hf; /* RSS DPDK bit field of active RSS. */ + struct rte_eth_rss_conf rss_conf; /* RSS configuration. */ struct rte_intr_handle intr_handle; /* Interrupt handler. */ unsigned int (*reta_idx)[]; /* RETA index table. */ unsigned int reta_idx_n; /* RETA index size. */ - struct fdir_filter_list *fdir_filter_list; /* Flow director rules. */ - struct fdir_queue *fdir_drop_queue; /* Flow director drop queue. */ + struct mlx5_hrxq_drop *flow_drop_queue; /* Flow drop queue. */ + struct mlx5_flows flows; /* RTE Flow rules. */ + struct mlx5_flows ctrl_flows; /* Control flow rules. */ + LIST_HEAD(mr, mlx5_mr) mr; /* Memory region. */ + LIST_HEAD(rxq, mlx5_rxq_ctrl) rxqsctrl; /* DPDK Rx queues. */ + LIST_HEAD(rxqibv, mlx5_rxq_ibv) rxqsibv; /* Verbs Rx queues. */ + LIST_HEAD(hrxq, mlx5_hrxq) hrxqs; /* Verbs Hash Rx queues. */ + LIST_HEAD(txq, mlx5_txq_ctrl) txqsctrl; /* DPDK Tx queues. */ + LIST_HEAD(txqibv, mlx5_txq_ibv) txqsibv; /* Verbs Tx queues. */ + /* Verbs Indirection tables. */ + LIST_HEAD(ind_tables, mlx5_ind_table_ibv) ind_tbls; uint32_t link_speed_capa; /* Link speed capabilities. */ + struct mlx5_xstats_ctrl xstats_ctrl; /* Extended stats control. */ rte_spinlock_t lock; /* Lock for control functions. */ + int primary_socket; /* Unix socket for primary process. */ + struct rte_intr_handle intr_handle_socket; /* Interrupt handler. */ }; -/* Local storage for secondary process data. */ -struct mlx5_secondary_data { - struct rte_eth_dev_data data; /* Local device data. */ - struct priv *primary_priv; /* Private structure from primary. */ - struct rte_eth_dev_data *shared_dev_data; /* Shared device data. */ - rte_spinlock_t lock; /* Port configuration lock. */ -} mlx5_secondary_data[RTE_MAX_ETHPORTS]; - /** * Lock private structure to protect it from concurrent access in the * control path. @@ -160,6 +164,22 @@ priv_lock(struct priv *priv) rte_spinlock_lock(&priv->lock); } +/** + * Try to lock private structure to protect it from concurrent access in the + * control path. + * + * @param priv + * Pointer to private structure. + * + * @return + * 1 if the lock is successfully taken; 0 otherwise. + */ +static inline int +priv_trylock(struct priv *priv) +{ + return rte_spinlock_trylock(&priv->lock); +} + /** * Unlock private structure. * @@ -182,13 +202,16 @@ struct priv *mlx5_get_priv(struct rte_eth_dev *dev); int mlx5_is_secondary(void); int priv_get_ifname(const struct priv *, char (*)[IF_NAMESIZE]); int priv_ifreq(const struct priv *, int req, struct ifreq *); +int priv_is_ib_cntr(const char *); +int priv_get_cntr_sysfs(struct priv *, const char *, uint64_t *); int priv_get_num_vfs(struct priv *, uint16_t *); int priv_get_mtu(struct priv *, uint16_t *); int priv_set_flags(struct priv *, unsigned int, unsigned int); int mlx5_dev_configure(struct rte_eth_dev *); void mlx5_dev_infos_get(struct rte_eth_dev *, struct rte_eth_dev_info *); const uint32_t *mlx5_dev_supported_ptypes_get(struct rte_eth_dev *dev); -int mlx5_link_update_unlocked(struct rte_eth_dev *, int); +int priv_link_update(struct priv *, int); +int priv_force_link_status_change(struct priv *, int); int mlx5_link_update(struct rte_eth_dev *, int); int mlx5_dev_set_mtu(struct rte_eth_dev *, uint16_t); int mlx5_dev_get_flow_ctrl(struct rte_eth_dev *, struct rte_eth_fc_conf *); @@ -196,33 +219,24 @@ int mlx5_dev_set_flow_ctrl(struct rte_eth_dev *, struct rte_eth_fc_conf *); int mlx5_ibv_device_to_pci_addr(const struct ibv_device *, struct rte_pci_addr *); void mlx5_dev_link_status_handler(void *); -void mlx5_dev_interrupt_handler(struct rte_intr_handle *, void *); +void mlx5_dev_interrupt_handler(void *); void priv_dev_interrupt_handler_uninstall(struct priv *, struct rte_eth_dev *); void priv_dev_interrupt_handler_install(struct priv *, struct rte_eth_dev *); int mlx5_set_link_down(struct rte_eth_dev *dev); int mlx5_set_link_up(struct rte_eth_dev *dev); -struct priv *mlx5_secondary_data_setup(struct priv *priv); -void priv_select_tx_function(struct priv *); -void priv_select_rx_function(struct priv *); +void priv_dev_select_tx_function(struct priv *priv, struct rte_eth_dev *dev); +void priv_dev_select_rx_function(struct priv *priv, struct rte_eth_dev *dev); /* mlx5_mac.c */ int priv_get_mac(struct priv *, uint8_t (*)[ETHER_ADDR_LEN]); -void hash_rxq_mac_addrs_del(struct hash_rxq *); -void priv_mac_addrs_disable(struct priv *); void mlx5_mac_addr_remove(struct rte_eth_dev *, uint32_t); -int hash_rxq_mac_addrs_add(struct hash_rxq *); -int priv_mac_addr_add(struct priv *, unsigned int, - const uint8_t (*)[ETHER_ADDR_LEN]); -int priv_mac_addrs_enable(struct priv *); -void mlx5_mac_addr_add(struct rte_eth_dev *, struct ether_addr *, uint32_t, - uint32_t); +int mlx5_mac_addr_add(struct rte_eth_dev *, struct ether_addr *, uint32_t, + uint32_t); void mlx5_mac_addr_set(struct rte_eth_dev *, struct ether_addr *); /* mlx5_rss.c */ -int rss_hash_rss_conf_new_key(struct priv *, const uint8_t *, unsigned int, - uint64_t); int mlx5_rss_hash_update(struct rte_eth_dev *, struct rte_eth_rss_conf *); int mlx5_rss_hash_conf_get(struct rte_eth_dev *, struct rte_eth_rss_conf *); int priv_rss_reta_index_resize(struct priv *, unsigned int); @@ -233,10 +247,6 @@ int mlx5_dev_rss_reta_update(struct rte_eth_dev *, /* mlx5_rxmode.c */ -int priv_special_flow_enable(struct priv *, enum hash_rxq_flow_type); -void priv_special_flow_disable(struct priv *, enum hash_rxq_flow_type); -int priv_special_flow_enable_all(struct priv *); -void priv_special_flow_disable_all(struct priv *); void mlx5_promiscuous_enable(struct rte_eth_dev *); void mlx5_promiscuous_disable(struct rte_eth_dev *); void mlx5_allmulticast_enable(struct rte_eth_dev *); @@ -244,28 +254,74 @@ void mlx5_allmulticast_disable(struct rte_eth_dev *); /* mlx5_stats.c */ -void mlx5_stats_get(struct rte_eth_dev *, struct rte_eth_stats *); +void priv_xstats_init(struct priv *); +int mlx5_stats_get(struct rte_eth_dev *, struct rte_eth_stats *); void mlx5_stats_reset(struct rte_eth_dev *); +int mlx5_xstats_get(struct rte_eth_dev *, + struct rte_eth_xstat *, unsigned int); +void mlx5_xstats_reset(struct rte_eth_dev *); +int mlx5_xstats_get_names(struct rte_eth_dev *, + struct rte_eth_xstat_name *, unsigned int); /* mlx5_vlan.c */ int mlx5_vlan_filter_set(struct rte_eth_dev *, uint16_t, int); -void mlx5_vlan_offload_set(struct rte_eth_dev *, int); +int mlx5_vlan_offload_set(struct rte_eth_dev *, int); void mlx5_vlan_strip_queue_set(struct rte_eth_dev *, uint16_t, int); /* mlx5_trigger.c */ int mlx5_dev_start(struct rte_eth_dev *); void mlx5_dev_stop(struct rte_eth_dev *); +int priv_dev_traffic_enable(struct priv *, struct rte_eth_dev *); +int priv_dev_traffic_disable(struct priv *, struct rte_eth_dev *); +int priv_dev_traffic_restart(struct priv *, struct rte_eth_dev *); +int mlx5_traffic_restart(struct rte_eth_dev *); -/* mlx5_fdir.c */ +/* mlx5_flow.c */ -void priv_fdir_queue_destroy(struct priv *, struct fdir_queue *); -int fdir_init_filters_list(struct priv *); -void priv_fdir_delete_filters_list(struct priv *); -void priv_fdir_disable(struct priv *); -void priv_fdir_enable(struct priv *); int mlx5_dev_filter_ctrl(struct rte_eth_dev *, enum rte_filter_type, enum rte_filter_op, void *); +int mlx5_flow_validate(struct rte_eth_dev *, const struct rte_flow_attr *, + const struct rte_flow_item [], + const struct rte_flow_action [], + struct rte_flow_error *); +struct rte_flow *mlx5_flow_create(struct rte_eth_dev *, + const struct rte_flow_attr *, + const struct rte_flow_item [], + const struct rte_flow_action [], + struct rte_flow_error *); +int mlx5_flow_destroy(struct rte_eth_dev *, struct rte_flow *, + struct rte_flow_error *); +void priv_flow_flush(struct priv *, struct mlx5_flows *); +int mlx5_flow_flush(struct rte_eth_dev *, struct rte_flow_error *); +int mlx5_flow_query(struct rte_eth_dev *, struct rte_flow *, + enum rte_flow_action_type, void *, + struct rte_flow_error *); +int mlx5_flow_isolate(struct rte_eth_dev *, int, struct rte_flow_error *); +int priv_flow_start(struct priv *, struct mlx5_flows *); +void priv_flow_stop(struct priv *, struct mlx5_flows *); +int priv_flow_verify(struct priv *); +int mlx5_ctrl_flow_vlan(struct rte_eth_dev *, struct rte_flow_item_eth *, + struct rte_flow_item_eth *, struct rte_flow_item_vlan *, + struct rte_flow_item_vlan *); +int mlx5_ctrl_flow(struct rte_eth_dev *, struct rte_flow_item_eth *, + struct rte_flow_item_eth *); +int priv_flow_create_drop_queue(struct priv *); +void priv_flow_delete_drop_queue(struct priv *); + +/* mlx5_socket.c */ + +int priv_socket_init(struct priv *priv); +int priv_socket_uninit(struct priv *priv); +void priv_socket_handle(struct priv *priv); +int priv_socket_connect(struct priv *priv); + +/* mlx5_mr.c */ + +struct mlx5_mr *priv_mr_new(struct priv *, struct rte_mempool *); +struct mlx5_mr *priv_mr_get(struct priv *, struct rte_mempool *); +int priv_mr_release(struct priv *, struct mlx5_mr *); +int priv_mr_verify(struct priv *); #endif /* RTE_PMD_MLX5_H_ */ diff --git a/dpdk/drivers/net/mlx5/mlx5_defs.h b/dpdk/drivers/net/mlx5/mlx5_defs.h index cc2a6f3e..24caf7e7 100644 --- a/dpdk/drivers/net/mlx5/mlx5_defs.h +++ b/dpdk/drivers/net/mlx5/mlx5_defs.h @@ -34,10 +34,12 @@ #ifndef RTE_PMD_MLX5_DEFS_H_ #define RTE_PMD_MLX5_DEFS_H_ +#include + #include "mlx5_autoconf.h" /* Reported driver name. */ -#define MLX5_DRIVER_NAME "librte_pmd_mlx5" +#define MLX5_DRIVER_NAME "net_mlx5" /* Maximum number of simultaneous MAC addresses. */ #define MLX5_MAX_MAC_ADDRESSES 128 @@ -45,17 +47,18 @@ /* Maximum number of simultaneous VLAN filters. */ #define MLX5_MAX_VLAN_IDS 128 -/* Maximum number of special flows. */ -#define MLX5_MAX_SPECIAL_FLOWS 4 - /* * Request TX completion every time descriptors reach this threshold since * the previous request. Must be a power of two for performance reasons. */ #define MLX5_TX_COMP_THRESH 32 -/* RSS Indirection table size. */ -#define RSS_INDIRECTION_TABLE_SIZE 256 +/* + * Request TX completion every time the total number of WQEBBs used for inlining + * packets exceeds the size of WQ divided by this divisor. Better to be power of + * two for performance. + */ +#define MLX5_TX_COMP_THRESH_INLINE_DIV (1 << 3) /* * Maximum number of cached Memory Pools (MPs) per TX queue. Each RTE MP @@ -79,4 +82,35 @@ /* Alarm timeout. */ #define MLX5_ALARM_TIMEOUT_US 100000 +/* Maximum number of extended statistics counters. */ +#define MLX5_MAX_XSTATS 32 + +/* Maximum Packet headers size (L2+L3+L4) for TSO. */ +#define MLX5_MAX_TSO_HEADER 128 + +/* Default minimum number of Tx queues for vectorized Tx. */ +#define MLX5_VPMD_MIN_TXQS 4 + +/* Threshold of buffer replenishment for vectorized Rx. */ +#define MLX5_VPMD_RXQ_RPLNSH_THRESH 64U + +/* Maximum size of burst for vectorized Rx. */ +#define MLX5_VPMD_RX_MAX_BURST MLX5_VPMD_RXQ_RPLNSH_THRESH + +/* + * Maximum size of burst for vectorized Tx. This is related to the maximum size + * of Enhanced MPW (eMPW) WQE as vectorized Tx is supported with eMPW. + * Careful when changing, large value can cause WQE DS to overlap. + */ +#define MLX5_VPMD_TX_MAX_BURST 32U + +/* Number of packets vectorized Rx can simultaneously process in a loop. */ +#define MLX5_VPMD_DESCS_PER_LOOP 4 + +/* Supported RSS */ +#define MLX5_RSS_HF_MASK (~(ETH_RSS_IP | ETH_RSS_UDP | ETH_RSS_TCP)) + +/* Maximum number of attempts to query link status before giving up. */ +#define MLX5_MAX_LINK_QUERY_ATTEMPTS 5 + #endif /* RTE_PMD_MLX5_DEFS_H_ */ diff --git a/dpdk/drivers/net/mlx5/mlx5_ethdev.c b/dpdk/drivers/net/mlx5/mlx5_ethdev.c index ba1ec2a6..ffe1cdd6 100644 --- a/dpdk/drivers/net/mlx5/mlx5_ethdev.c +++ b/dpdk/drivers/net/mlx5/mlx5_ethdev.c @@ -31,6 +31,8 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +#define _GNU_SOURCE + #include #include #include @@ -43,30 +45,79 @@ #include #include #include +#include #include #include #include +#include #include +#include +#include -/* DPDK headers don't like -pedantic. */ -#ifdef PEDANTIC -#pragma GCC diagnostic ignored "-Wpedantic" -#endif #include #include +#include #include #include #include #include #include -#ifdef PEDANTIC -#pragma GCC diagnostic error "-Wpedantic" -#endif #include "mlx5.h" #include "mlx5_rxtx.h" #include "mlx5_utils.h" +/* Add defines in case the running kernel is not the same as user headers. */ +#ifndef ETHTOOL_GLINKSETTINGS +struct ethtool_link_settings { + uint32_t cmd; + uint32_t speed; + uint8_t duplex; + uint8_t port; + uint8_t phy_address; + uint8_t autoneg; + uint8_t mdio_support; + uint8_t eth_to_mdix; + uint8_t eth_tp_mdix_ctrl; + int8_t link_mode_masks_nwords; + uint32_t reserved[8]; + uint32_t link_mode_masks[]; +}; + +#define ETHTOOL_GLINKSETTINGS 0x0000004c +#define ETHTOOL_LINK_MODE_1000baseT_Full_BIT 5 +#define ETHTOOL_LINK_MODE_Autoneg_BIT 6 +#define ETHTOOL_LINK_MODE_1000baseKX_Full_BIT 17 +#define ETHTOOL_LINK_MODE_10000baseKX4_Full_BIT 18 +#define ETHTOOL_LINK_MODE_10000baseKR_Full_BIT 19 +#define ETHTOOL_LINK_MODE_10000baseR_FEC_BIT 20 +#define ETHTOOL_LINK_MODE_20000baseMLD2_Full_BIT 21 +#define ETHTOOL_LINK_MODE_20000baseKR2_Full_BIT 22 +#define ETHTOOL_LINK_MODE_40000baseKR4_Full_BIT 23 +#define ETHTOOL_LINK_MODE_40000baseCR4_Full_BIT 24 +#define ETHTOOL_LINK_MODE_40000baseSR4_Full_BIT 25 +#define ETHTOOL_LINK_MODE_40000baseLR4_Full_BIT 26 +#define ETHTOOL_LINK_MODE_56000baseKR4_Full_BIT 27 +#define ETHTOOL_LINK_MODE_56000baseCR4_Full_BIT 28 +#define ETHTOOL_LINK_MODE_56000baseSR4_Full_BIT 29 +#define ETHTOOL_LINK_MODE_56000baseLR4_Full_BIT 30 +#endif +#ifndef HAVE_ETHTOOL_LINK_MODE_25G +#define ETHTOOL_LINK_MODE_25000baseCR_Full_BIT 31 +#define ETHTOOL_LINK_MODE_25000baseKR_Full_BIT 32 +#define ETHTOOL_LINK_MODE_25000baseSR_Full_BIT 33 +#endif +#ifndef HAVE_ETHTOOL_LINK_MODE_50G +#define ETHTOOL_LINK_MODE_50000baseCR2_Full_BIT 34 +#define ETHTOOL_LINK_MODE_50000baseKR2_Full_BIT 35 +#endif +#ifndef HAVE_ETHTOOL_LINK_MODE_100G +#define ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT 36 +#define ETHTOOL_LINK_MODE_100000baseSR4_Full_BIT 37 +#define ETHTOOL_LINK_MODE_100000baseCR4_Full_BIT 38 +#define ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT 39 +#endif + /** * Return private structure associated with an Ethernet device. * @@ -79,24 +130,7 @@ struct priv * mlx5_get_priv(struct rte_eth_dev *dev) { - struct mlx5_secondary_data *sd; - - if (!mlx5_is_secondary()) - return dev->data->dev_private; - sd = &mlx5_secondary_data[dev->data->port_id]; - return sd->data.dev_private; -} - -/** - * Check if running as a secondary process. - * - * @return - * Nonzero if running as a secondary process. - */ -inline int -mlx5_is_secondary(void) -{ - return rte_eal_process_type() != RTE_PROC_PRIMARY; + return dev->data->dev_private; } /** @@ -120,7 +154,7 @@ priv_get_ifname(const struct priv *priv, char (*ifname)[IF_NAMESIZE]) char match[IF_NAMESIZE] = ""; { - MKSTR(path, "%s/device/net", priv->ctx->device->ibdev_path); + MKSTR(path, "%s/device/net", priv->ibdev_path); dir = opendir(path); if (dir == NULL) @@ -138,7 +172,7 @@ priv_get_ifname(const struct priv *priv, char (*ifname)[IF_NAMESIZE]) continue; MKSTR(path, "%s/device/net/%s/%s", - priv->ctx->device->ibdev_path, name, + priv->ibdev_path, name, (dev_type ? "dev_id" : "dev_port")); file = fopen(path, "rb"); @@ -180,6 +214,23 @@ try_dev_id: return 0; } +/** + * Check if the counter is located on ib counters file. + * + * @param[in] cntr + * Counter name. + * + * @return + * 1 if counter is located on ib counters file , 0 otherwise. + */ +int +priv_is_ib_cntr(const char *cntr) +{ + if (!strcmp(cntr, "out_of_buffer")) + return 1; + return 0; +} + /** * Read from sysfs entry. * @@ -207,10 +258,15 @@ priv_sysfs_read(const struct priv *priv, const char *entry, if (priv_get_ifname(priv, &ifname)) return -1; - MKSTR(path, "%s/device/net/%s/%s", priv->ctx->device->ibdev_path, - ifname, entry); - - file = fopen(path, "rb"); + if (priv_is_ib_cntr(entry)) { + MKSTR(path, "%s/ports/1/hw_counters/%s", + priv->ibdev_path, entry); + file = fopen(path, "rb"); + } else { + MKSTR(path, "%s/device/net/%s/%s", + priv->ibdev_path, ifname, entry); + file = fopen(path, "rb"); + } if (file == NULL) return -1; ret = fread(buf, 1, size, file); @@ -251,8 +307,7 @@ priv_sysfs_write(const struct priv *priv, const char *entry, if (priv_get_ifname(priv, &ifname)) return -1; - MKSTR(path, "%s/device/net/%s/%s", priv->ctx->device->ibdev_path, - ifname, entry); + MKSTR(path, "%s/device/net/%s/%s", priv->ibdev_path, ifname, entry); file = fopen(path, "wb"); if (file == NULL) @@ -415,6 +470,30 @@ priv_get_mtu(struct priv *priv, uint16_t *mtu) return 0; } +/** + * Read device counter from sysfs. + * + * @param priv + * Pointer to private structure. + * @param name + * Counter name. + * @param[out] cntr + * Counter output buffer. + * + * @return + * 0 on success, -1 on failure and errno is set. + */ +int +priv_get_cntr_sysfs(struct priv *priv, const char *name, uint64_t *cntr) +{ + unsigned long ulong_ctr; + + if (priv_get_sysfs_ulong(priv, name, &ulong_ctr) == -1) + return -1; + *cntr = ulong_ctr; + return 0; +} + /** * Set device MTU. * @@ -485,8 +564,29 @@ dev_configure(struct rte_eth_dev *dev) unsigned int i; unsigned int j; unsigned int reta_idx_n; + const uint8_t use_app_rss_key = + !!dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key; - priv->rss_hf = dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf; + if (use_app_rss_key && + (dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key_len != + rss_hash_default_key_len)) { + /* MLX5 RSS only support 40bytes key. */ + return EINVAL; + } + priv->rss_conf.rss_key = + rte_realloc(priv->rss_conf.rss_key, + rss_hash_default_key_len, 0); + if (!priv->rss_conf.rss_key) { + ERROR("cannot allocate RSS hash key memory (%u)", rxqs_n); + return ENOMEM; + } + memcpy(priv->rss_conf.rss_key, + use_app_rss_key ? + dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key : + rss_hash_default_key, + rss_hash_default_key_len); + priv->rss_conf.rss_key_len = rss_hash_default_key_len; + priv->rss_conf.rss_hf = dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf; priv->rxqs = (void *)dev->data->rx_queues; priv->txqs = (void *)dev->data->tx_queues; if (txqs_n != priv->txqs_n) { @@ -537,9 +637,6 @@ mlx5_dev_configure(struct rte_eth_dev *dev) struct priv *priv = dev->data->dev_private; int ret; - if (mlx5_is_secondary()) - return -E_RTE_SECONDARY; - priv_lock(priv); ret = dev_configure(dev); assert(ret >= 0); @@ -562,6 +659,8 @@ mlx5_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info) unsigned int max; char ifname[IF_NAMESIZE]; + info->pci_dev = RTE_ETH_DEV_TO_PCI(dev); + priv_lock(priv); /* FIXME: we should ask the device for these values. */ info->min_rx_bufsize = 32; @@ -570,8 +669,8 @@ mlx5_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info) * Since we need one CQ per QP, the limit is the minimum number * between the two values. */ - max = ((priv->device_attr.max_cq > priv->device_attr.max_qp) ? - priv->device_attr.max_qp : priv->device_attr.max_cq); + max = RTE_MIN(priv->device_attr.orig_attr.max_cq, + priv->device_attr.orig_attr.max_qp); /* If max >= 65535 then max = 0, max_rx_queues is uint16_t. */ if (max >= 65535) max = 65535; @@ -584,7 +683,9 @@ mlx5_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info) DEV_RX_OFFLOAD_UDP_CKSUM | DEV_RX_OFFLOAD_TCP_CKSUM) : 0) | - (priv->hw_vlan_strip ? DEV_RX_OFFLOAD_VLAN_STRIP : 0); + (priv->hw_vlan_strip ? DEV_RX_OFFLOAD_VLAN_STRIP : 0) | + DEV_RX_OFFLOAD_TIMESTAMP; + if (!priv->mps) info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT; if (priv->hw_csum) @@ -592,18 +693,19 @@ mlx5_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info) (DEV_TX_OFFLOAD_IPV4_CKSUM | DEV_TX_OFFLOAD_UDP_CKSUM | DEV_TX_OFFLOAD_TCP_CKSUM); + if (priv->tso) + info->tx_offload_capa |= DEV_TX_OFFLOAD_TCP_TSO; + if (priv->tunnel_en) + info->tx_offload_capa |= (DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM | + DEV_TX_OFFLOAD_VXLAN_TNL_TSO | + DEV_TX_OFFLOAD_GRE_TNL_TSO); if (priv_get_ifname(priv, &ifname) == 0) info->if_index = if_nametoindex(ifname); - /* FIXME: RETA update/query API expects the callee to know the size of - * the indirection table, for this PMD the size varies depending on - * the number of RX queues, it becomes impossible to find the correct - * size if it is not fixed. - * The API should be updated to solve this problem. */ - info->reta_size = priv->ind_table_max_size; - info->hash_key_size = ((*priv->rss_conf) ? - (*priv->rss_conf)[0]->rss_key_len : - 0); + info->reta_size = priv->reta_idx_n ? + priv->reta_idx_n : priv->ind_table_max_size; + info->hash_key_size = priv->rss_conf.rss_key_len; info->speed_capa = priv->link_speed_capa; + info->flow_type_rss_offloads = ~MLX5_RSS_HF_MASK; priv_unlock(priv); } @@ -612,21 +714,30 @@ mlx5_dev_supported_ptypes_get(struct rte_eth_dev *dev) { static const uint32_t ptypes[] = { /* refers to rxq_cq_to_pkt_type() */ - RTE_PTYPE_L3_IPV4, - RTE_PTYPE_L3_IPV6, - RTE_PTYPE_INNER_L3_IPV4, - RTE_PTYPE_INNER_L3_IPV6, + RTE_PTYPE_L2_ETHER, + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN, + RTE_PTYPE_L3_IPV6_EXT_UNKNOWN, + RTE_PTYPE_L4_NONFRAG, + RTE_PTYPE_L4_FRAG, + RTE_PTYPE_L4_TCP, + RTE_PTYPE_L4_UDP, + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN, + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN, + RTE_PTYPE_INNER_L4_NONFRAG, + RTE_PTYPE_INNER_L4_FRAG, + RTE_PTYPE_INNER_L4_TCP, + RTE_PTYPE_INNER_L4_UDP, RTE_PTYPE_UNKNOWN - }; - if (dev->rx_pkt_burst == mlx5_rx_burst) + if (dev->rx_pkt_burst == mlx5_rx_burst || + dev->rx_pkt_burst == mlx5_rx_burst_vec) return ptypes; return NULL; } /** - * Retrieve physical link information (unlocked version using legacy ioctl). + * DPDK callback to retrieve physical link information. * * @param dev * Pointer to Ethernet device structure. @@ -644,6 +755,8 @@ mlx5_link_update_unlocked_gset(struct rte_eth_dev *dev, int wait_to_complete) struct rte_eth_link dev_link; int link_speed = 0; + /* priv_lock() is not taken to allow concurrent calls. */ + (void)wait_to_complete; if (priv_ifreq(priv, SIOCGIFFLAGS, &ifr)) { WARN("ioctl(SIOCGIFFLAGS) failed: %s", strerror(errno)); @@ -690,8 +803,7 @@ mlx5_link_update_unlocked_gset(struct rte_eth_dev *dev, int wait_to_complete) } /** - * Retrieve physical link information (unlocked version using new ioctl from - * Linux 4.5). + * Retrieve physical link information (unlocked version using new ioctl). * * @param dev * Pointer to Ethernet device structure. @@ -701,11 +813,8 @@ mlx5_link_update_unlocked_gset(struct rte_eth_dev *dev, int wait_to_complete) static int mlx5_link_update_unlocked_gs(struct rte_eth_dev *dev, int wait_to_complete) { -#ifdef ETHTOOL_GLINKSETTINGS struct priv *priv = mlx5_get_priv(dev); - struct ethtool_link_settings edata = { - .cmd = ETHTOOL_GLINKSETTINGS, - }; + struct ethtool_link_settings gcmd = { .cmd = ETHTOOL_GLINKSETTINGS }; struct ifreq ifr; struct rte_eth_link dev_link; uint64_t sc; @@ -718,59 +827,65 @@ mlx5_link_update_unlocked_gs(struct rte_eth_dev *dev, int wait_to_complete) memset(&dev_link, 0, sizeof(dev_link)); dev_link.link_status = ((ifr.ifr_flags & IFF_UP) && (ifr.ifr_flags & IFF_RUNNING)); - ifr.ifr_data = (void *)&edata; + ifr.ifr_data = (void *)&gcmd; if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { DEBUG("ioctl(SIOCETHTOOL, ETHTOOL_GLINKSETTINGS) failed: %s", strerror(errno)); return -1; } - dev_link.link_speed = edata.speed; - sc = edata.link_mode_masks[0] | - ((uint64_t)edata.link_mode_masks[1] << 32); + gcmd.link_mode_masks_nwords = -gcmd.link_mode_masks_nwords; + + alignas(struct ethtool_link_settings) + uint8_t data[offsetof(struct ethtool_link_settings, link_mode_masks) + + sizeof(uint32_t) * gcmd.link_mode_masks_nwords * 3]; + struct ethtool_link_settings *ecmd = (void *)data; + + *ecmd = gcmd; + ifr.ifr_data = (void *)ecmd; + if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { + DEBUG("ioctl(SIOCETHTOOL, ETHTOOL_GLINKSETTINGS) failed: %s", + strerror(errno)); + return -1; + } + dev_link.link_speed = ecmd->speed; + sc = ecmd->link_mode_masks[0] | + ((uint64_t)ecmd->link_mode_masks[1] << 32); priv->link_speed_capa = 0; - /* Link speeds available in kernel v4.5. */ - if (sc & ETHTOOL_LINK_MODE_Autoneg_BIT) + if (sc & MLX5_BITSHIFT(ETHTOOL_LINK_MODE_Autoneg_BIT)) priv->link_speed_capa |= ETH_LINK_SPEED_AUTONEG; - if (sc & (ETHTOOL_LINK_MODE_1000baseT_Full_BIT | - ETHTOOL_LINK_MODE_1000baseKX_Full_BIT)) + if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_1000baseT_Full_BIT) | + MLX5_BITSHIFT(ETHTOOL_LINK_MODE_1000baseKX_Full_BIT))) priv->link_speed_capa |= ETH_LINK_SPEED_1G; - if (sc & (ETHTOOL_LINK_MODE_10000baseKX4_Full_BIT | - ETHTOOL_LINK_MODE_10000baseKR_Full_BIT | - ETHTOOL_LINK_MODE_10000baseR_FEC_BIT)) + if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_10000baseKX4_Full_BIT) | + MLX5_BITSHIFT(ETHTOOL_LINK_MODE_10000baseKR_Full_BIT) | + MLX5_BITSHIFT(ETHTOOL_LINK_MODE_10000baseR_FEC_BIT))) priv->link_speed_capa |= ETH_LINK_SPEED_10G; - if (sc & (ETHTOOL_LINK_MODE_20000baseMLD2_Full_BIT | - ETHTOOL_LINK_MODE_20000baseKR2_Full_BIT)) + if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_20000baseMLD2_Full_BIT) | + MLX5_BITSHIFT(ETHTOOL_LINK_MODE_20000baseKR2_Full_BIT))) priv->link_speed_capa |= ETH_LINK_SPEED_20G; - if (sc & (ETHTOOL_LINK_MODE_40000baseKR4_Full_BIT | - ETHTOOL_LINK_MODE_40000baseCR4_Full_BIT | - ETHTOOL_LINK_MODE_40000baseSR4_Full_BIT | - ETHTOOL_LINK_MODE_40000baseLR4_Full_BIT)) + if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_40000baseKR4_Full_BIT) | + MLX5_BITSHIFT(ETHTOOL_LINK_MODE_40000baseCR4_Full_BIT) | + MLX5_BITSHIFT(ETHTOOL_LINK_MODE_40000baseSR4_Full_BIT) | + MLX5_BITSHIFT(ETHTOOL_LINK_MODE_40000baseLR4_Full_BIT))) priv->link_speed_capa |= ETH_LINK_SPEED_40G; - if (sc & (ETHTOOL_LINK_MODE_56000baseKR4_Full_BIT | - ETHTOOL_LINK_MODE_56000baseCR4_Full_BIT | - ETHTOOL_LINK_MODE_56000baseSR4_Full_BIT | - ETHTOOL_LINK_MODE_56000baseLR4_Full_BIT)) + if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_56000baseKR4_Full_BIT) | + MLX5_BITSHIFT(ETHTOOL_LINK_MODE_56000baseCR4_Full_BIT) | + MLX5_BITSHIFT(ETHTOOL_LINK_MODE_56000baseSR4_Full_BIT) | + MLX5_BITSHIFT(ETHTOOL_LINK_MODE_56000baseLR4_Full_BIT))) priv->link_speed_capa |= ETH_LINK_SPEED_56G; - /* Link speeds available in kernel v4.6. */ -#ifdef HAVE_ETHTOOL_LINK_MODE_25G - if (sc & (ETHTOOL_LINK_MODE_25000baseCR_Full_BIT | - ETHTOOL_LINK_MODE_25000baseKR_Full_BIT | - ETHTOOL_LINK_MODE_25000baseSR_Full_BIT)) + if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_25000baseCR_Full_BIT) | + MLX5_BITSHIFT(ETHTOOL_LINK_MODE_25000baseKR_Full_BIT) | + MLX5_BITSHIFT(ETHTOOL_LINK_MODE_25000baseSR_Full_BIT))) priv->link_speed_capa |= ETH_LINK_SPEED_25G; -#endif -#ifdef HAVE_ETHTOOL_LINK_MODE_50G - if (sc & (ETHTOOL_LINK_MODE_50000baseCR2_Full_BIT | - ETHTOOL_LINK_MODE_50000baseKR2_Full_BIT)) + if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_50000baseCR2_Full_BIT) | + MLX5_BITSHIFT(ETHTOOL_LINK_MODE_50000baseKR2_Full_BIT))) priv->link_speed_capa |= ETH_LINK_SPEED_50G; -#endif -#ifdef HAVE_ETHTOOL_LINK_MODE_100G - if (sc & (ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT | - ETHTOOL_LINK_MODE_100000baseSR4_Full_BIT | - ETHTOOL_LINK_MODE_100000baseCR4_Full_BIT | - ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT)) + if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT) | + MLX5_BITSHIFT(ETHTOOL_LINK_MODE_100000baseSR4_Full_BIT) | + MLX5_BITSHIFT(ETHTOOL_LINK_MODE_100000baseCR4_Full_BIT) | + MLX5_BITSHIFT(ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT))) priv->link_speed_capa |= ETH_LINK_SPEED_100G; -#endif - dev_link.link_duplex = ((edata.duplex == DUPLEX_HALF) ? + dev_link.link_duplex = ((ecmd->duplex == DUPLEX_HALF) ? ETH_LINK_HALF_DUPLEX : ETH_LINK_FULL_DUPLEX); dev_link.link_autoneg = !(dev->data->dev_conf.link_speeds & ETH_LINK_SPEED_FIXED); @@ -779,33 +894,118 @@ mlx5_link_update_unlocked_gs(struct rte_eth_dev *dev, int wait_to_complete) dev->data->dev_link = dev_link; return 0; } -#else - (void)dev; - (void)wait_to_complete; -#endif /* Link status is still the same. */ return -1; } /** - * DPDK callback to retrieve physical link information (unlocked version). + * Enable receiving and transmitting traffic. * - * @param dev - * Pointer to Ethernet device structure. + * @param priv + * Pointer to private structure. + */ +static void +priv_link_start(struct priv *priv) +{ + struct rte_eth_dev *dev = priv->dev; + int err; + + priv_dev_select_tx_function(priv, dev); + priv_dev_select_rx_function(priv, dev); + err = priv_dev_traffic_enable(priv, dev); + if (err) + ERROR("%p: error occurred while configuring control flows: %s", + (void *)priv, strerror(err)); + err = priv_flow_start(priv, &priv->flows); + if (err) + ERROR("%p: error occurred while configuring flows: %s", + (void *)priv, strerror(err)); +} + +/** + * Disable receiving and transmitting traffic. + * + * @param priv + * Pointer to private structure. + */ +static void +priv_link_stop(struct priv *priv) +{ + struct rte_eth_dev *dev = priv->dev; + + priv_flow_stop(priv, &priv->flows); + priv_dev_traffic_disable(priv, dev); + dev->rx_pkt_burst = removed_rx_burst; + dev->tx_pkt_burst = removed_tx_burst; +} + +/** + * Retrieve physical link information and update rx/tx_pkt_burst callbacks + * accordingly. + * + * @param priv + * Pointer to private structure. * @param wait_to_complete * Wait for request completion (ignored). */ int -mlx5_link_update_unlocked(struct rte_eth_dev *dev, int wait_to_complete) +priv_link_update(struct priv *priv, int wait_to_complete) { + struct rte_eth_dev *dev = priv->dev; + struct utsname utsname; + int ver[3]; int ret; + struct rte_eth_link dev_link = dev->data->dev_link; - ret = mlx5_link_update_unlocked_gs(dev, wait_to_complete); - if (ret < 0) + if (uname(&utsname) == -1 || + sscanf(utsname.release, "%d.%d.%d", + &ver[0], &ver[1], &ver[2]) != 3 || + KERNEL_VERSION(ver[0], ver[1], ver[2]) < KERNEL_VERSION(4, 9, 0)) ret = mlx5_link_update_unlocked_gset(dev, wait_to_complete); + else + ret = mlx5_link_update_unlocked_gs(dev, wait_to_complete); + /* If lsc interrupt is disabled, should always be ready for traffic. */ + if (!dev->data->dev_conf.intr_conf.lsc) { + priv_link_start(priv); + return ret; + } + /* Re-select burst callbacks only if link status has been changed. */ + if (!ret && dev_link.link_status != dev->data->dev_link.link_status) { + if (dev->data->dev_link.link_status == ETH_LINK_UP) + priv_link_start(priv); + else + priv_link_stop(priv); + } return ret; } +/** + * Querying the link status till it changes to the desired state. + * Number of query attempts is bounded by MLX5_MAX_LINK_QUERY_ATTEMPTS. + * + * @param priv + * Pointer to private structure. + * @param status + * Link desired status. + * + * @return + * 0 on success, negative errno value on failure. + */ +int +priv_force_link_status_change(struct priv *priv, int status) +{ + int try = 0; + + while (try < MLX5_MAX_LINK_QUERY_ATTEMPTS) { + priv_link_update(priv, 0); + if (priv->dev->data->dev_link.link_status == status) + return 0; + try++; + sleep(1); + } + return -EAGAIN; +} + /** * DPDK callback to retrieve physical link information. * @@ -817,11 +1017,11 @@ mlx5_link_update_unlocked(struct rte_eth_dev *dev, int wait_to_complete) int mlx5_link_update(struct rte_eth_dev *dev, int wait_to_complete) { - struct priv *priv = mlx5_get_priv(dev); + struct priv *priv = dev->data->dev_private; int ret; priv_lock(priv); - ret = mlx5_link_update_unlocked(dev, wait_to_complete); + ret = priv_link_update(priv, wait_to_complete); priv_unlock(priv); return ret; } @@ -829,12 +1029,6 @@ mlx5_link_update(struct rte_eth_dev *dev, int wait_to_complete) /** * DPDK callback to change the MTU. * - * Setting the MTU affects hardware MRU (packets larger than the MTU cannot be - * received). Use this as a hint to enable/disable scattered packets support - * and improve performance when not needed. - * Since failure is not an option, reconfiguring queues on the fly is not - * recommended. - * * @param dev * Pointer to Ethernet device structure. * @param in_mtu @@ -847,125 +1041,30 @@ int mlx5_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu) { struct priv *priv = dev->data->dev_private; + uint16_t kern_mtu; int ret = 0; - unsigned int i; - uint16_t (*rx_func)(void *, struct rte_mbuf **, uint16_t) = - mlx5_rx_burst; - unsigned int max_frame_len; - int rehash; - int restart = priv->started; - - if (mlx5_is_secondary()) - return -E_RTE_SECONDARY; priv_lock(priv); - /* Set kernel interface MTU first. */ - if (priv_set_mtu(priv, mtu)) { - ret = errno; - WARN("cannot set port %u MTU to %u: %s", priv->port, mtu, - strerror(ret)); + ret = priv_get_mtu(priv, &kern_mtu); + if (ret) goto out; - } else + /* Set kernel interface MTU first. */ + ret = priv_set_mtu(priv, mtu); + if (ret) + goto out; + ret = priv_get_mtu(priv, &kern_mtu); + if (ret) + goto out; + if (kern_mtu == mtu) { + priv->mtu = mtu; DEBUG("adapter port %u MTU set to %u", priv->port, mtu); - /* Temporarily replace RX handler with a fake one, assuming it has not - * been copied elsewhere. */ - dev->rx_pkt_burst = removed_rx_burst; - /* Make sure everyone has left mlx5_rx_burst() and uses - * removed_rx_burst() instead. */ - rte_wmb(); - usleep(1000); - /* MTU does not include header and CRC. */ - max_frame_len = ETHER_HDR_LEN + mtu + ETHER_CRC_LEN; - /* Check if at least one queue is going to need a SGE update. */ - for (i = 0; i != priv->rxqs_n; ++i) { - struct rxq *rxq = (*priv->rxqs)[i]; - unsigned int mb_len; - unsigned int size = RTE_PKTMBUF_HEADROOM + max_frame_len; - unsigned int sges_n; - - if (rxq == NULL) - continue; - mb_len = rte_pktmbuf_data_room_size(rxq->mp); - assert(mb_len >= RTE_PKTMBUF_HEADROOM); - /* - * Determine the number of SGEs needed for a full packet - * and round it to the next power of two. - */ - sges_n = log2above((size / mb_len) + !!(size % mb_len)); - if (sges_n != rxq->sges_n) - break; } - /* - * If all queues have the right number of SGEs, a simple rehash - * of their buffers is enough, otherwise SGE information can only - * be updated in a queue by recreating it. All resources that depend - * on queues (flows, indirection tables) must be recreated as well in - * that case. - */ - rehash = (i == priv->rxqs_n); - if (!rehash) { - /* Clean up everything as with mlx5_dev_stop(). */ - priv_special_flow_disable_all(priv); - priv_mac_addrs_disable(priv); - priv_destroy_hash_rxqs(priv); - priv_fdir_disable(priv); - priv_dev_interrupt_handler_uninstall(priv, dev); - } -recover: - /* Reconfigure each RX queue. */ - for (i = 0; (i != priv->rxqs_n); ++i) { - struct rxq *rxq = (*priv->rxqs)[i]; - struct rxq_ctrl *rxq_ctrl = - container_of(rxq, struct rxq_ctrl, rxq); - int sp; - unsigned int mb_len; - unsigned int tmp; - - if (rxq == NULL) - continue; - mb_len = rte_pktmbuf_data_room_size(rxq->mp); - assert(mb_len >= RTE_PKTMBUF_HEADROOM); - /* Toggle scattered support (sp) if necessary. */ - sp = (max_frame_len > (mb_len - RTE_PKTMBUF_HEADROOM)); - /* Provide new values to rxq_setup(). */ - dev->data->dev_conf.rxmode.jumbo_frame = sp; - dev->data->dev_conf.rxmode.max_rx_pkt_len = max_frame_len; - if (rehash) - ret = rxq_rehash(dev, rxq_ctrl); - else - ret = rxq_ctrl_setup(dev, rxq_ctrl, rxq->elts_n, - rxq_ctrl->socket, NULL, rxq->mp); - if (!ret) - continue; - /* Attempt to roll back in case of error. */ - tmp = (mb_len << rxq->sges_n) - RTE_PKTMBUF_HEADROOM; - if (max_frame_len != tmp) { - max_frame_len = tmp; - goto recover; - } - /* Double fault, disable RX. */ - break; - } - /* - * Use a safe RX burst function in case of error, otherwise mimic - * mlx5_dev_start(). - */ - if (ret) { - ERROR("unable to reconfigure RX queues, RX disabled"); - rx_func = removed_rx_burst; - } else if (restart && - !rehash && - !priv_create_hash_rxqs(priv) && - !priv_rehash_flows(priv)) { - if (dev->data->dev_conf.fdir_conf.mode == RTE_FDIR_MODE_NONE) - priv_fdir_enable(priv); - priv_dev_interrupt_handler_install(priv, dev); - } - priv->mtu = mtu; - /* Burst functions can now be called again. */ - rte_wmb(); - dev->rx_pkt_burst = rx_func; + priv_unlock(priv); + return 0; out: + ret = errno; + WARN("cannot set port %u MTU to %u: %s", priv->port, mtu, + strerror(ret)); priv_unlock(priv); assert(ret >= 0); return -ret; @@ -992,9 +1091,6 @@ mlx5_dev_get_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) }; int ret; - if (mlx5_is_secondary()) - return -E_RTE_SECONDARY; - ifr.ifr_data = (void *)ðpause; priv_lock(priv); if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { @@ -1043,9 +1139,6 @@ mlx5_dev_set_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) }; int ret; - if (mlx5_is_secondary()) - return -E_RTE_SECONDARY; - ifr.ifr_data = (void *)ðpause; ethpause.autoneg = fc_conf->autoneg; if (((fc_conf->mode & RTE_FC_FULL) == RTE_FC_FULL) || @@ -1113,7 +1206,7 @@ mlx5_ibv_device_to_pci_addr(const struct ibv_device *device, /* Extract information. */ if (sscanf(line, "PCI_SLOT_NAME=" - "%" SCNx16 ":%" SCNx8 ":%" SCNx8 ".%" SCNx8 "\n", + "%" SCNx32 ":%" SCNx8 ":%" SCNx8 ".%" SCNx8 "\n", &pci_addr->domain, &pci_addr->bus, &pci_addr->devid, @@ -1127,52 +1220,77 @@ mlx5_ibv_device_to_pci_addr(const struct ibv_device *device, } /** - * Link status handler. + * Update the link status. * * @param priv * Pointer to private structure. - * @param dev - * Pointer to the rte_eth_dev structure. * * @return - * Nonzero if the callback process can be called immediately. + * Zero if the callback process can be called immediately. */ static int -priv_dev_link_status_handler(struct priv *priv, struct rte_eth_dev *dev) +priv_link_status_update(struct priv *priv) +{ + struct rte_eth_link *link = &priv->dev->data->dev_link; + + priv_link_update(priv, 0); + if (((link->link_speed == 0) && link->link_status) || + ((link->link_speed != 0) && !link->link_status)) { + /* + * Inconsistent status. Event likely occurred before the + * kernel netdevice exposes the new status. + */ + if (!priv->pending_alarm) { + priv->pending_alarm = 1; + rte_eal_alarm_set(MLX5_ALARM_TIMEOUT_US, + mlx5_dev_link_status_handler, + priv->dev); + } + return 1; + } else if (unlikely(priv->pending_alarm)) { + /* Link interrupt occurred while alarm is already scheduled. */ + priv->pending_alarm = 0; + rte_eal_alarm_cancel(mlx5_dev_link_status_handler, priv->dev); + } + return 0; +} + +/** + * Device status handler. + * + * @param priv + * Pointer to private structure. + * @param events + * Pointer to event flags holder. + * + * @return + * Events bitmap of callback process which can be called immediately. + */ +static uint32_t +priv_dev_status_handler(struct priv *priv) { struct ibv_async_event event; - int port_change = 0; - int ret = 0; + uint32_t ret = 0; /* Read all message and acknowledge them. */ for (;;) { if (ibv_get_async_event(priv->ctx, &event)) break; - - if (event.event_type == IBV_EVENT_PORT_ACTIVE || - event.event_type == IBV_EVENT_PORT_ERR) - port_change = 1; + if ((event.event_type == IBV_EVENT_PORT_ACTIVE || + event.event_type == IBV_EVENT_PORT_ERR) && + (priv->dev->data->dev_conf.intr_conf.lsc == 1)) + ret |= (1 << RTE_ETH_EVENT_INTR_LSC); + else if (event.event_type == IBV_EVENT_DEVICE_FATAL && + priv->dev->data->dev_conf.intr_conf.rmv == 1) + ret |= (1 << RTE_ETH_EVENT_INTR_RMV); else DEBUG("event type %d on port %d not handled", event.event_type, event.element.port_num); ibv_ack_async_event(&event); } - - if (port_change ^ priv->pending_alarm) { - struct rte_eth_link *link = &dev->data->dev_link; - - priv->pending_alarm = 0; - mlx5_link_update_unlocked(dev, 0); - if (((link->link_speed == 0) && link->link_status) || - ((link->link_speed != 0) && !link->link_status)) { - /* Inconsistent status, check again later. */ - priv->pending_alarm = 1; - rte_eal_alarm_set(MLX5_ALARM_TIMEOUT_US, - mlx5_dev_link_status_handler, - dev); - } else - ret = 1; - } + if (ret & (1 << RTE_ETH_EVENT_INTR_LSC)) + if (priv_link_status_update(priv)) + ret &= ~(1 << RTE_ETH_EVENT_INTR_LSC); return ret; } @@ -1189,12 +1307,18 @@ mlx5_dev_link_status_handler(void *arg) struct priv *priv = dev->data->dev_private; int ret; - priv_lock(priv); - assert(priv->pending_alarm == 1); - ret = priv_dev_link_status_handler(priv, dev); + while (!priv_trylock(priv)) { + /* Alarm is being canceled. */ + if (priv->pending_alarm == 0) + return; + rte_pause(); + } + priv->pending_alarm = 0; + ret = priv_link_status_update(priv); priv_unlock(priv); - if (ret) - _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC); + if (!ret) + _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL, + NULL); } /** @@ -1206,18 +1330,38 @@ mlx5_dev_link_status_handler(void *arg) * Callback argument. */ void -mlx5_dev_interrupt_handler(struct rte_intr_handle *intr_handle, void *cb_arg) +mlx5_dev_interrupt_handler(void *cb_arg) { struct rte_eth_dev *dev = cb_arg; struct priv *priv = dev->data->dev_private; - int ret; + uint32_t events; - (void)intr_handle; priv_lock(priv); - ret = priv_dev_link_status_handler(priv, dev); + events = priv_dev_status_handler(priv); + priv_unlock(priv); + if (events & (1 << RTE_ETH_EVENT_INTR_LSC)) + _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL, + NULL); + if (events & (1 << RTE_ETH_EVENT_INTR_RMV)) + _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RMV, NULL, + NULL); +} + +/** + * Handle interrupts from the socket. + * + * @param cb_arg + * Callback argument. + */ +static void +mlx5_dev_handler_socket(void *cb_arg) +{ + struct rte_eth_dev *dev = cb_arg; + struct priv *priv = dev->data->dev_private; + + priv_lock(priv); + priv_socket_handle(priv); priv_unlock(priv); - if (ret) - _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC); } /** @@ -1231,16 +1375,21 @@ mlx5_dev_interrupt_handler(struct rte_intr_handle *intr_handle, void *cb_arg) void priv_dev_interrupt_handler_uninstall(struct priv *priv, struct rte_eth_dev *dev) { - if (!dev->data->dev_conf.intr_conf.lsc) - return; - rte_intr_callback_unregister(&priv->intr_handle, - mlx5_dev_interrupt_handler, - dev); - if (priv->pending_alarm) + if (dev->data->dev_conf.intr_conf.lsc || + dev->data->dev_conf.intr_conf.rmv) + rte_intr_callback_unregister(&priv->intr_handle, + mlx5_dev_interrupt_handler, dev); + if (priv->primary_socket) + rte_intr_callback_unregister(&priv->intr_handle_socket, + mlx5_dev_handler_socket, dev); + if (priv->pending_alarm) { + priv->pending_alarm = 0; rte_eal_alarm_cancel(mlx5_dev_link_status_handler, dev); - priv->pending_alarm = 0; + } priv->intr_handle.fd = 0; priv->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN; + priv->intr_handle_socket.fd = 0; + priv->intr_handle_socket.type = RTE_INTR_HANDLE_UNKNOWN; } /** @@ -1256,20 +1405,28 @@ priv_dev_interrupt_handler_install(struct priv *priv, struct rte_eth_dev *dev) { int rc, flags; - if (!dev->data->dev_conf.intr_conf.lsc) - return; assert(priv->ctx->async_fd > 0); flags = fcntl(priv->ctx->async_fd, F_GETFL); rc = fcntl(priv->ctx->async_fd, F_SETFL, flags | O_NONBLOCK); if (rc < 0) { INFO("failed to change file descriptor async event queue"); dev->data->dev_conf.intr_conf.lsc = 0; - } else { + dev->data->dev_conf.intr_conf.rmv = 0; + } + if (dev->data->dev_conf.intr_conf.lsc || + dev->data->dev_conf.intr_conf.rmv) { priv->intr_handle.fd = priv->ctx->async_fd; priv->intr_handle.type = RTE_INTR_HANDLE_EXT; rte_intr_callback_register(&priv->intr_handle, - mlx5_dev_interrupt_handler, - dev); + mlx5_dev_interrupt_handler, dev); + } + + rc = priv_socket_init(priv); + if (!rc && priv->primary_socket) { + priv->intr_handle_socket.fd = priv->primary_socket; + priv->intr_handle_socket.type = RTE_INTR_HANDLE_EXT; + rte_intr_callback_register(&priv->intr_handle_socket, + mlx5_dev_handler_socket, dev); } } @@ -1277,7 +1434,7 @@ priv_dev_interrupt_handler_install(struct priv *priv, struct rte_eth_dev *dev) * Change the link state (UP / DOWN). * * @param priv - * Pointer to Ethernet device structure. + * Pointer to private data structure. * @param up * Nonzero for link up, otherwise link down. * @@ -1285,25 +1442,9 @@ priv_dev_interrupt_handler_install(struct priv *priv, struct rte_eth_dev *dev) * 0 on success, errno value on failure. */ static int -priv_set_link(struct priv *priv, int up) +priv_dev_set_link(struct priv *priv, int up) { - struct rte_eth_dev *dev = priv->dev; - int err; - - if (up) { - err = priv_set_flags(priv, ~IFF_UP, IFF_UP); - if (err) - return err; - priv_select_tx_function(priv); - priv_select_rx_function(priv); - } else { - err = priv_set_flags(priv, ~IFF_UP, ~IFF_UP); - if (err) - return err; - dev->rx_pkt_burst = removed_rx_burst; - dev->tx_pkt_burst = removed_tx_burst; - } - return 0; + return priv_set_flags(priv, ~IFF_UP, up ? IFF_UP : ~IFF_UP); } /** @@ -1322,7 +1463,7 @@ mlx5_set_link_down(struct rte_eth_dev *dev) int err; priv_lock(priv); - err = priv_set_link(priv, 0); + err = priv_dev_set_link(priv, 0); priv_unlock(priv); return err; } @@ -1343,187 +1484,42 @@ mlx5_set_link_up(struct rte_eth_dev *dev) int err; priv_lock(priv); - err = priv_set_link(priv, 1); + err = priv_dev_set_link(priv, 1); priv_unlock(priv); return err; } -/** - * Configure secondary process queues from a private data pointer (primary - * or secondary) and update burst callbacks. Can take place only once. - * - * All queues must have been previously created by the primary process to - * avoid undefined behavior. - * - * @param priv - * Private data pointer from either primary or secondary process. - * - * @return - * Private data pointer from secondary process, NULL in case of error. - */ -struct priv * -mlx5_secondary_data_setup(struct priv *priv) -{ - unsigned int port_id = 0; - struct mlx5_secondary_data *sd; - void **tx_queues; - void **rx_queues; - unsigned int nb_tx_queues; - unsigned int nb_rx_queues; - unsigned int i; - - /* priv must be valid at this point. */ - assert(priv != NULL); - /* priv->dev must also be valid but may point to local memory from - * another process, possibly with the same address and must not - * be dereferenced yet. */ - assert(priv->dev != NULL); - /* Determine port ID by finding out where priv comes from. */ - while (1) { - sd = &mlx5_secondary_data[port_id]; - rte_spinlock_lock(&sd->lock); - /* Primary process? */ - if (sd->primary_priv == priv) - break; - /* Secondary process? */ - if (sd->data.dev_private == priv) - break; - rte_spinlock_unlock(&sd->lock); - if (++port_id == RTE_DIM(mlx5_secondary_data)) - port_id = 0; - } - /* Switch to secondary private structure. If private data has already - * been updated by another thread, there is nothing else to do. */ - priv = sd->data.dev_private; - if (priv->dev->data == &sd->data) - goto end; - /* Sanity checks. Secondary private structure is supposed to point - * to local eth_dev, itself still pointing to the shared device data - * structure allocated by the primary process. */ - assert(sd->shared_dev_data != &sd->data); - assert(sd->data.nb_tx_queues == 0); - assert(sd->data.tx_queues == NULL); - assert(sd->data.nb_rx_queues == 0); - assert(sd->data.rx_queues == NULL); - assert(priv != sd->primary_priv); - assert(priv->dev->data == sd->shared_dev_data); - assert(priv->txqs_n == 0); - assert(priv->txqs == NULL); - assert(priv->rxqs_n == 0); - assert(priv->rxqs == NULL); - nb_tx_queues = sd->shared_dev_data->nb_tx_queues; - nb_rx_queues = sd->shared_dev_data->nb_rx_queues; - /* Allocate local storage for queues. */ - tx_queues = rte_zmalloc("secondary ethdev->tx_queues", - sizeof(sd->data.tx_queues[0]) * nb_tx_queues, - RTE_CACHE_LINE_SIZE); - rx_queues = rte_zmalloc("secondary ethdev->rx_queues", - sizeof(sd->data.rx_queues[0]) * nb_rx_queues, - RTE_CACHE_LINE_SIZE); - if (tx_queues == NULL || rx_queues == NULL) - goto error; - /* Lock to prevent control operations during setup. */ - priv_lock(priv); - /* TX queues. */ - for (i = 0; i != nb_tx_queues; ++i) { - struct txq *primary_txq = (*sd->primary_priv->txqs)[i]; - struct txq_ctrl *primary_txq_ctrl; - struct txq_ctrl *txq_ctrl; - - if (primary_txq == NULL) - continue; - primary_txq_ctrl = container_of(primary_txq, - struct txq_ctrl, txq); - txq_ctrl = rte_calloc_socket("TXQ", 1, sizeof(*txq_ctrl) + - (1 << primary_txq->elts_n) * - sizeof(struct rte_mbuf *), 0, - primary_txq_ctrl->socket); - if (txq_ctrl != NULL) { - if (txq_ctrl_setup(priv->dev, - txq_ctrl, - primary_txq->elts_n, - primary_txq_ctrl->socket, - NULL) == 0) { - txq_ctrl->txq.stats.idx = - primary_txq->stats.idx; - tx_queues[i] = &txq_ctrl->txq; - continue; - } - rte_free(txq_ctrl); - } - while (i) { - txq_ctrl = tx_queues[--i]; - txq_cleanup(txq_ctrl); - rte_free(txq_ctrl); - } - goto error; - } - /* RX queues. */ - for (i = 0; i != nb_rx_queues; ++i) { - struct rxq_ctrl *primary_rxq = - container_of((*sd->primary_priv->rxqs)[i], - struct rxq_ctrl, rxq); - - if (primary_rxq == NULL) - continue; - /* Not supported yet. */ - rx_queues[i] = NULL; - } - /* Update everything. */ - priv->txqs = (void *)tx_queues; - priv->txqs_n = nb_tx_queues; - priv->rxqs = (void *)rx_queues; - priv->rxqs_n = nb_rx_queues; - sd->data.rx_queues = rx_queues; - sd->data.tx_queues = tx_queues; - sd->data.nb_rx_queues = nb_rx_queues; - sd->data.nb_tx_queues = nb_tx_queues; - sd->data.dev_link = sd->shared_dev_data->dev_link; - sd->data.mtu = sd->shared_dev_data->mtu; - memcpy(sd->data.rx_queue_state, sd->shared_dev_data->rx_queue_state, - sizeof(sd->data.rx_queue_state)); - memcpy(sd->data.tx_queue_state, sd->shared_dev_data->tx_queue_state, - sizeof(sd->data.tx_queue_state)); - sd->data.dev_flags = sd->shared_dev_data->dev_flags; - /* Use local data from now on. */ - rte_mb(); - priv->dev->data = &sd->data; - rte_mb(); - priv_select_tx_function(priv); - priv_select_rx_function(priv); - priv_unlock(priv); -end: - /* More sanity checks. */ - assert(priv->dev->data == &sd->data); - rte_spinlock_unlock(&sd->lock); - return priv; -error: - priv_unlock(priv); - rte_free(tx_queues); - rte_free(rx_queues); - rte_spinlock_unlock(&sd->lock); - return NULL; -} - /** * Configure the TX function to use. * * @param priv - * Pointer to private structure. + * Pointer to private data structure. + * @param dev + * Pointer to rte_eth_dev structure. */ void -priv_select_tx_function(struct priv *priv) +priv_dev_select_tx_function(struct priv *priv, struct rte_eth_dev *dev) { - priv->dev->tx_pkt_burst = mlx5_tx_burst; - /* Display warning for unsupported configurations. */ - if (priv->sriov && priv->mps) - WARN("multi-packet send WQE cannot be used on a SR-IOV setup"); + assert(priv != NULL); + assert(dev != NULL); + dev->tx_pkt_burst = mlx5_tx_burst; /* Select appropriate TX function. */ - if ((priv->sriov == 0) && priv->mps && priv->txq_inline) { - priv->dev->tx_pkt_burst = mlx5_tx_burst_mpw_inline; + if (priv->mps == MLX5_MPW_ENHANCED) { + if (priv_check_vec_tx_support(priv) > 0) { + if (priv_check_raw_vec_tx_support(priv) > 0) + dev->tx_pkt_burst = mlx5_tx_burst_raw_vec; + else + dev->tx_pkt_burst = mlx5_tx_burst_vec; + DEBUG("selected Enhanced MPW TX vectorized function"); + } else { + dev->tx_pkt_burst = mlx5_tx_burst_empw; + DEBUG("selected Enhanced MPW TX function"); + } + } else if (priv->mps && priv->txq_inline) { + dev->tx_pkt_burst = mlx5_tx_burst_mpw_inline; DEBUG("selected MPW inline TX function"); - } else if ((priv->sriov == 0) && priv->mps) { - priv->dev->tx_pkt_burst = mlx5_tx_burst_mpw; + } else if (priv->mps) { + dev->tx_pkt_burst = mlx5_tx_burst_mpw; DEBUG("selected MPW TX function"); } } @@ -1532,10 +1528,19 @@ priv_select_tx_function(struct priv *priv) * Configure the RX function to use. * * @param priv - * Pointer to private structure. + * Pointer to private data structure. + * @param dev + * Pointer to rte_eth_dev structure. */ void -priv_select_rx_function(struct priv *priv) +priv_dev_select_rx_function(struct priv *priv, struct rte_eth_dev *dev) { - priv->dev->rx_pkt_burst = mlx5_rx_burst; + assert(priv != NULL); + assert(dev != NULL); + if (priv_check_vec_rx_support(priv) > 0) { + dev->rx_pkt_burst = mlx5_rx_burst_vec; + DEBUG("selected RX vectorized function"); + } else { + dev->rx_pkt_burst = mlx5_rx_burst; + } } diff --git a/dpdk/drivers/net/mlx5/mlx5_fdir.c b/dpdk/drivers/net/mlx5/mlx5_fdir.c deleted file mode 100644 index 1acf6826..00000000 --- a/dpdk/drivers/net/mlx5/mlx5_fdir.c +++ /dev/null @@ -1,1082 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright 2015 6WIND S.A. - * Copyright 2015 Mellanox. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of 6WIND S.A. nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include -#include -#include -#include -#include - -/* Verbs header. */ -/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ -#ifdef PEDANTIC -#pragma GCC diagnostic ignored "-Wpedantic" -#endif -#include -#ifdef PEDANTIC -#pragma GCC diagnostic error "-Wpedantic" -#endif - -/* DPDK headers don't like -pedantic. */ -#ifdef PEDANTIC -#pragma GCC diagnostic ignored "-Wpedantic" -#endif -#include -#include -#include -#include -#ifdef PEDANTIC -#pragma GCC diagnostic error "-Wpedantic" -#endif - -#include "mlx5.h" -#include "mlx5_rxtx.h" - -struct fdir_flow_desc { - uint16_t dst_port; - uint16_t src_port; - uint32_t src_ip[4]; - uint32_t dst_ip[4]; - uint8_t mac[6]; - uint16_t vlan_tag; - enum hash_rxq_type type; -}; - -struct mlx5_fdir_filter { - LIST_ENTRY(mlx5_fdir_filter) next; - uint16_t queue; /* Queue assigned to if FDIR match. */ - enum rte_eth_fdir_behavior behavior; - struct fdir_flow_desc desc; - struct ibv_exp_flow *flow; -}; - -LIST_HEAD(fdir_filter_list, mlx5_fdir_filter); - -/** - * Convert struct rte_eth_fdir_filter to mlx5 filter descriptor. - * - * @param[in] fdir_filter - * DPDK filter structure to convert. - * @param[out] desc - * Resulting mlx5 filter descriptor. - * @param mode - * Flow director mode. - */ -static void -fdir_filter_to_flow_desc(const struct rte_eth_fdir_filter *fdir_filter, - struct fdir_flow_desc *desc, enum rte_fdir_mode mode) -{ - /* Initialize descriptor. */ - memset(desc, 0, sizeof(*desc)); - - /* Set VLAN ID. */ - desc->vlan_tag = fdir_filter->input.flow_ext.vlan_tci; - - /* Set MAC address. */ - if (mode == RTE_FDIR_MODE_PERFECT_MAC_VLAN) { - rte_memcpy(desc->mac, - fdir_filter->input.flow.mac_vlan_flow.mac_addr. - addr_bytes, - sizeof(desc->mac)); - desc->type = HASH_RXQ_ETH; - return; - } - - /* Set mode */ - switch (fdir_filter->input.flow_type) { - case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: - desc->type = HASH_RXQ_UDPV4; - break; - case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: - desc->type = HASH_RXQ_TCPV4; - break; - case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: - desc->type = HASH_RXQ_IPV4; - break; - case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: - desc->type = HASH_RXQ_UDPV6; - break; - case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: - desc->type = HASH_RXQ_TCPV6; - break; - case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: - desc->type = HASH_RXQ_IPV6; - break; - default: - break; - } - - /* Set flow values */ - switch (fdir_filter->input.flow_type) { - case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: - case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: - desc->src_port = fdir_filter->input.flow.udp4_flow.src_port; - desc->dst_port = fdir_filter->input.flow.udp4_flow.dst_port; - case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: - desc->src_ip[0] = fdir_filter->input.flow.ip4_flow.src_ip; - desc->dst_ip[0] = fdir_filter->input.flow.ip4_flow.dst_ip; - break; - case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: - case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: - desc->src_port = fdir_filter->input.flow.udp6_flow.src_port; - desc->dst_port = fdir_filter->input.flow.udp6_flow.dst_port; - /* Fall through. */ - case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: - rte_memcpy(desc->src_ip, - fdir_filter->input.flow.ipv6_flow.src_ip, - sizeof(desc->src_ip)); - rte_memcpy(desc->dst_ip, - fdir_filter->input.flow.ipv6_flow.dst_ip, - sizeof(desc->dst_ip)); - break; - default: - break; - } -} - -/** - * Check if two flow descriptors overlap according to configured mask. - * - * @param priv - * Private structure that provides flow director mask. - * @param desc1 - * First flow descriptor to compare. - * @param desc2 - * Second flow descriptor to compare. - * - * @return - * Nonzero if descriptors overlap. - */ -static int -priv_fdir_overlap(const struct priv *priv, - const struct fdir_flow_desc *desc1, - const struct fdir_flow_desc *desc2) -{ - const struct rte_eth_fdir_masks *mask = - &priv->dev->data->dev_conf.fdir_conf.mask; - unsigned int i; - - if (desc1->type != desc2->type) - return 0; - /* Ignore non masked bits. */ - for (i = 0; i != RTE_DIM(desc1->mac); ++i) - if ((desc1->mac[i] & mask->mac_addr_byte_mask) != - (desc2->mac[i] & mask->mac_addr_byte_mask)) - return 0; - if (((desc1->src_port & mask->src_port_mask) != - (desc2->src_port & mask->src_port_mask)) || - ((desc1->dst_port & mask->dst_port_mask) != - (desc2->dst_port & mask->dst_port_mask))) - return 0; - switch (desc1->type) { - case HASH_RXQ_IPV4: - case HASH_RXQ_UDPV4: - case HASH_RXQ_TCPV4: - if (((desc1->src_ip[0] & mask->ipv4_mask.src_ip) != - (desc2->src_ip[0] & mask->ipv4_mask.src_ip)) || - ((desc1->dst_ip[0] & mask->ipv4_mask.dst_ip) != - (desc2->dst_ip[0] & mask->ipv4_mask.dst_ip))) - return 0; - break; - case HASH_RXQ_IPV6: - case HASH_RXQ_UDPV6: - case HASH_RXQ_TCPV6: - for (i = 0; i != RTE_DIM(desc1->src_ip); ++i) - if (((desc1->src_ip[i] & mask->ipv6_mask.src_ip[i]) != - (desc2->src_ip[i] & mask->ipv6_mask.src_ip[i])) || - ((desc1->dst_ip[i] & mask->ipv6_mask.dst_ip[i]) != - (desc2->dst_ip[i] & mask->ipv6_mask.dst_ip[i]))) - return 0; - break; - default: - break; - } - return 1; -} - -/** - * Create flow director steering rule for a specific filter. - * - * @param priv - * Private structure. - * @param mlx5_fdir_filter - * Filter to create a steering rule for. - * @param fdir_queue - * Flow director queue for matching packets. - * - * @return - * 0 on success, errno value on failure. - */ -static int -priv_fdir_flow_add(struct priv *priv, - struct mlx5_fdir_filter *mlx5_fdir_filter, - struct fdir_queue *fdir_queue) -{ - struct ibv_exp_flow *flow; - struct fdir_flow_desc *desc = &mlx5_fdir_filter->desc; - enum rte_fdir_mode fdir_mode = - priv->dev->data->dev_conf.fdir_conf.mode; - struct rte_eth_fdir_masks *mask = - &priv->dev->data->dev_conf.fdir_conf.mask; - FLOW_ATTR_SPEC_ETH(data, priv_flow_attr(priv, NULL, 0, desc->type)); - struct ibv_exp_flow_attr *attr = &data->attr; - uintptr_t spec_offset = (uintptr_t)&data->spec; - struct ibv_exp_flow_spec_eth *spec_eth; - struct ibv_exp_flow_spec_ipv4 *spec_ipv4; - struct ibv_exp_flow_spec_ipv6 *spec_ipv6; - struct ibv_exp_flow_spec_tcp_udp *spec_tcp_udp; - struct mlx5_fdir_filter *iter_fdir_filter; - unsigned int i; - - /* Abort if an existing flow overlaps this one to avoid packet - * duplication, even if it targets another queue. */ - LIST_FOREACH(iter_fdir_filter, priv->fdir_filter_list, next) - if ((iter_fdir_filter != mlx5_fdir_filter) && - (iter_fdir_filter->flow != NULL) && - (priv_fdir_overlap(priv, - &mlx5_fdir_filter->desc, - &iter_fdir_filter->desc))) - return EEXIST; - - /* - * No padding must be inserted by the compiler between attr and spec. - * This layout is expected by libibverbs. - */ - assert(((uint8_t *)attr + sizeof(*attr)) == (uint8_t *)spec_offset); - priv_flow_attr(priv, attr, sizeof(data), desc->type); - - /* Set Ethernet spec */ - spec_eth = (struct ibv_exp_flow_spec_eth *)spec_offset; - - /* The first specification must be Ethernet. */ - assert(spec_eth->type == IBV_EXP_FLOW_SPEC_ETH); - assert(spec_eth->size == sizeof(*spec_eth)); - - /* VLAN ID */ - spec_eth->val.vlan_tag = desc->vlan_tag & mask->vlan_tci_mask; - spec_eth->mask.vlan_tag = mask->vlan_tci_mask; - - /* Update priority */ - attr->priority = 2; - - if (fdir_mode == RTE_FDIR_MODE_PERFECT_MAC_VLAN) { - /* MAC Address */ - for (i = 0; i != RTE_DIM(spec_eth->mask.dst_mac); ++i) { - spec_eth->val.dst_mac[i] = - desc->mac[i] & mask->mac_addr_byte_mask; - spec_eth->mask.dst_mac[i] = mask->mac_addr_byte_mask; - } - goto create_flow; - } - - switch (desc->type) { - case HASH_RXQ_IPV4: - case HASH_RXQ_UDPV4: - case HASH_RXQ_TCPV4: - spec_offset += spec_eth->size; - - /* Set IP spec */ - spec_ipv4 = (struct ibv_exp_flow_spec_ipv4 *)spec_offset; - - /* The second specification must be IP. */ - assert(spec_ipv4->type == IBV_EXP_FLOW_SPEC_IPV4); - assert(spec_ipv4->size == sizeof(*spec_ipv4)); - - spec_ipv4->val.src_ip = - desc->src_ip[0] & mask->ipv4_mask.src_ip; - spec_ipv4->val.dst_ip = - desc->dst_ip[0] & mask->ipv4_mask.dst_ip; - spec_ipv4->mask.src_ip = mask->ipv4_mask.src_ip; - spec_ipv4->mask.dst_ip = mask->ipv4_mask.dst_ip; - - /* Update priority */ - attr->priority = 1; - - if (desc->type == HASH_RXQ_IPV4) - goto create_flow; - - spec_offset += spec_ipv4->size; - break; - case HASH_RXQ_IPV6: - case HASH_RXQ_UDPV6: - case HASH_RXQ_TCPV6: - spec_offset += spec_eth->size; - - /* Set IP spec */ - spec_ipv6 = (struct ibv_exp_flow_spec_ipv6 *)spec_offset; - - /* The second specification must be IP. */ - assert(spec_ipv6->type == IBV_EXP_FLOW_SPEC_IPV6); - assert(spec_ipv6->size == sizeof(*spec_ipv6)); - - for (i = 0; i != RTE_DIM(desc->src_ip); ++i) { - ((uint32_t *)spec_ipv6->val.src_ip)[i] = - desc->src_ip[i] & mask->ipv6_mask.src_ip[i]; - ((uint32_t *)spec_ipv6->val.dst_ip)[i] = - desc->dst_ip[i] & mask->ipv6_mask.dst_ip[i]; - } - rte_memcpy(spec_ipv6->mask.src_ip, - mask->ipv6_mask.src_ip, - sizeof(spec_ipv6->mask.src_ip)); - rte_memcpy(spec_ipv6->mask.dst_ip, - mask->ipv6_mask.dst_ip, - sizeof(spec_ipv6->mask.dst_ip)); - - /* Update priority */ - attr->priority = 1; - - if (desc->type == HASH_RXQ_IPV6) - goto create_flow; - - spec_offset += spec_ipv6->size; - break; - default: - ERROR("invalid flow attribute type"); - return EINVAL; - } - - /* Set TCP/UDP flow specification. */ - spec_tcp_udp = (struct ibv_exp_flow_spec_tcp_udp *)spec_offset; - - /* The third specification must be TCP/UDP. */ - assert(spec_tcp_udp->type == IBV_EXP_FLOW_SPEC_TCP || - spec_tcp_udp->type == IBV_EXP_FLOW_SPEC_UDP); - assert(spec_tcp_udp->size == sizeof(*spec_tcp_udp)); - - spec_tcp_udp->val.src_port = desc->src_port & mask->src_port_mask; - spec_tcp_udp->val.dst_port = desc->dst_port & mask->dst_port_mask; - spec_tcp_udp->mask.src_port = mask->src_port_mask; - spec_tcp_udp->mask.dst_port = mask->dst_port_mask; - - /* Update priority */ - attr->priority = 0; - -create_flow: - - errno = 0; - flow = ibv_exp_create_flow(fdir_queue->qp, attr); - if (flow == NULL) { - /* It's not clear whether errno is always set in this case. */ - ERROR("%p: flow director configuration failed, errno=%d: %s", - (void *)priv, errno, - (errno ? strerror(errno) : "Unknown error")); - if (errno) - return errno; - return EINVAL; - } - - DEBUG("%p: added flow director rule (%p)", (void *)priv, (void *)flow); - mlx5_fdir_filter->flow = flow; - return 0; -} - -/** - * Destroy a flow director queue. - * - * @param fdir_queue - * Flow director queue to be destroyed. - */ -void -priv_fdir_queue_destroy(struct priv *priv, struct fdir_queue *fdir_queue) -{ - struct mlx5_fdir_filter *fdir_filter; - - /* Disable filter flows still applying to this queue. */ - LIST_FOREACH(fdir_filter, priv->fdir_filter_list, next) { - unsigned int idx = fdir_filter->queue; - struct rxq_ctrl *rxq_ctrl = - container_of((*priv->rxqs)[idx], struct rxq_ctrl, rxq); - - assert(idx < priv->rxqs_n); - if (fdir_queue == rxq_ctrl->fdir_queue && - fdir_filter->flow != NULL) { - claim_zero(ibv_exp_destroy_flow(fdir_filter->flow)); - fdir_filter->flow = NULL; - } - } - assert(fdir_queue->qp); - claim_zero(ibv_destroy_qp(fdir_queue->qp)); - assert(fdir_queue->ind_table); - claim_zero(ibv_exp_destroy_rwq_ind_table(fdir_queue->ind_table)); - if (fdir_queue->wq) - claim_zero(ibv_exp_destroy_wq(fdir_queue->wq)); - if (fdir_queue->cq) - claim_zero(ibv_destroy_cq(fdir_queue->cq)); -#ifndef NDEBUG - memset(fdir_queue, 0x2a, sizeof(*fdir_queue)); -#endif - rte_free(fdir_queue); -} - -/** - * Create a flow director queue. - * - * @param priv - * Private structure. - * @param wq - * Work queue to route matched packets to, NULL if one needs to - * be created. - * - * @return - * Related flow director queue on success, NULL otherwise. - */ -static struct fdir_queue * -priv_fdir_queue_create(struct priv *priv, struct ibv_exp_wq *wq, - unsigned int socket) -{ - struct fdir_queue *fdir_queue; - - fdir_queue = rte_calloc_socket(__func__, 1, sizeof(*fdir_queue), - 0, socket); - if (!fdir_queue) { - ERROR("cannot allocate flow director queue"); - return NULL; - } - assert(priv->pd); - assert(priv->ctx); - if (!wq) { - fdir_queue->cq = ibv_exp_create_cq( - priv->ctx, 1, NULL, NULL, 0, - &(struct ibv_exp_cq_init_attr){ - .comp_mask = 0, - }); - if (!fdir_queue->cq) { - ERROR("cannot create flow director CQ"); - goto error; - } - fdir_queue->wq = ibv_exp_create_wq( - priv->ctx, - &(struct ibv_exp_wq_init_attr){ - .wq_type = IBV_EXP_WQT_RQ, - .max_recv_wr = 1, - .max_recv_sge = 1, - .pd = priv->pd, - .cq = fdir_queue->cq, - }); - if (!fdir_queue->wq) { - ERROR("cannot create flow director WQ"); - goto error; - } - wq = fdir_queue->wq; - } - fdir_queue->ind_table = ibv_exp_create_rwq_ind_table( - priv->ctx, - &(struct ibv_exp_rwq_ind_table_init_attr){ - .pd = priv->pd, - .log_ind_tbl_size = 0, - .ind_tbl = &wq, - .comp_mask = 0, - }); - if (!fdir_queue->ind_table) { - ERROR("cannot create flow director indirection table"); - goto error; - } - fdir_queue->qp = ibv_exp_create_qp( - priv->ctx, - &(struct ibv_exp_qp_init_attr){ - .qp_type = IBV_QPT_RAW_PACKET, - .comp_mask = - IBV_EXP_QP_INIT_ATTR_PD | - IBV_EXP_QP_INIT_ATTR_PORT | - IBV_EXP_QP_INIT_ATTR_RX_HASH, - .pd = priv->pd, - .rx_hash_conf = &(struct ibv_exp_rx_hash_conf){ - .rx_hash_function = - IBV_EXP_RX_HASH_FUNC_TOEPLITZ, - .rx_hash_key_len = rss_hash_default_key_len, - .rx_hash_key = rss_hash_default_key, - .rx_hash_fields_mask = 0, - .rwq_ind_tbl = fdir_queue->ind_table, - }, - .port_num = priv->port, - }); - if (!fdir_queue->qp) { - ERROR("cannot create flow director hash RX QP"); - goto error; - } - return fdir_queue; -error: - assert(fdir_queue); - assert(!fdir_queue->qp); - if (fdir_queue->ind_table) - claim_zero(ibv_exp_destroy_rwq_ind_table - (fdir_queue->ind_table)); - if (fdir_queue->wq) - claim_zero(ibv_exp_destroy_wq(fdir_queue->wq)); - if (fdir_queue->cq) - claim_zero(ibv_destroy_cq(fdir_queue->cq)); - rte_free(fdir_queue); - return NULL; -} - -/** - * Get flow director queue for a specific RX queue, create it in case - * it does not exist. - * - * @param priv - * Private structure. - * @param idx - * RX queue index. - * - * @return - * Related flow director queue on success, NULL otherwise. - */ -static struct fdir_queue * -priv_get_fdir_queue(struct priv *priv, uint16_t idx) -{ - struct rxq_ctrl *rxq_ctrl = - container_of((*priv->rxqs)[idx], struct rxq_ctrl, rxq); - struct fdir_queue *fdir_queue = rxq_ctrl->fdir_queue; - - assert(rxq_ctrl->wq); - if (fdir_queue == NULL) { - fdir_queue = priv_fdir_queue_create(priv, rxq_ctrl->wq, - rxq_ctrl->socket); - rxq_ctrl->fdir_queue = fdir_queue; - } - return fdir_queue; -} - -/** - * Get or flow director drop queue. Create it if it does not exist. - * - * @param priv - * Private structure. - * - * @return - * Flow director drop queue on success, NULL otherwise. - */ -static struct fdir_queue * -priv_get_fdir_drop_queue(struct priv *priv) -{ - struct fdir_queue *fdir_queue = priv->fdir_drop_queue; - - if (fdir_queue == NULL) { - unsigned int socket = SOCKET_ID_ANY; - - /* Select a known NUMA socket if possible. */ - if (priv->rxqs_n && (*priv->rxqs)[0]) - socket = container_of((*priv->rxqs)[0], - struct rxq_ctrl, rxq)->socket; - fdir_queue = priv_fdir_queue_create(priv, NULL, socket); - priv->fdir_drop_queue = fdir_queue; - } - return fdir_queue; -} - -/** - * Enable flow director filter and create steering rules. - * - * @param priv - * Private structure. - * @param mlx5_fdir_filter - * Filter to create steering rule for. - * - * @return - * 0 on success, errno value on failure. - */ -static int -priv_fdir_filter_enable(struct priv *priv, - struct mlx5_fdir_filter *mlx5_fdir_filter) -{ - struct fdir_queue *fdir_queue; - - /* Check if flow already exists. */ - if (mlx5_fdir_filter->flow != NULL) - return 0; - - /* Get fdir_queue for specific queue. */ - if (mlx5_fdir_filter->behavior == RTE_ETH_FDIR_REJECT) - fdir_queue = priv_get_fdir_drop_queue(priv); - else - fdir_queue = priv_get_fdir_queue(priv, - mlx5_fdir_filter->queue); - - if (fdir_queue == NULL) { - ERROR("failed to create flow director rxq for queue %d", - mlx5_fdir_filter->queue); - return EINVAL; - } - - /* Create flow */ - return priv_fdir_flow_add(priv, mlx5_fdir_filter, fdir_queue); -} - -/** - * Initialize flow director filters list. - * - * @param priv - * Private structure. - * - * @return - * 0 on success, errno value on failure. - */ -int -fdir_init_filters_list(struct priv *priv) -{ - /* Filter list initialization should be done only once. */ - if (priv->fdir_filter_list) - return 0; - - /* Create filters list. */ - priv->fdir_filter_list = - rte_calloc(__func__, 1, sizeof(*priv->fdir_filter_list), 0); - - if (priv->fdir_filter_list == NULL) { - int err = ENOMEM; - - ERROR("cannot allocate flow director filter list: %s", - strerror(err)); - return err; - } - - LIST_INIT(priv->fdir_filter_list); - - return 0; -} - -/** - * Flush all filters. - * - * @param priv - * Private structure. - */ -static void -priv_fdir_filter_flush(struct priv *priv) -{ - struct mlx5_fdir_filter *mlx5_fdir_filter; - - while ((mlx5_fdir_filter = LIST_FIRST(priv->fdir_filter_list))) { - struct ibv_exp_flow *flow = mlx5_fdir_filter->flow; - - DEBUG("%p: flushing flow director filter %p", - (void *)priv, (void *)mlx5_fdir_filter); - LIST_REMOVE(mlx5_fdir_filter, next); - if (flow != NULL) - claim_zero(ibv_exp_destroy_flow(flow)); - rte_free(mlx5_fdir_filter); - } -} - -/** - * Remove all flow director filters and delete list. - * - * @param priv - * Private structure. - */ -void -priv_fdir_delete_filters_list(struct priv *priv) -{ - priv_fdir_filter_flush(priv); - rte_free(priv->fdir_filter_list); - priv->fdir_filter_list = NULL; -} - -/** - * Disable flow director, remove all steering rules. - * - * @param priv - * Private structure. - */ -void -priv_fdir_disable(struct priv *priv) -{ - unsigned int i; - struct mlx5_fdir_filter *mlx5_fdir_filter; - - /* Run on every flow director filter and destroy flow handle. */ - LIST_FOREACH(mlx5_fdir_filter, priv->fdir_filter_list, next) { - struct ibv_exp_flow *flow; - - /* Only valid elements should be in the list */ - assert(mlx5_fdir_filter != NULL); - flow = mlx5_fdir_filter->flow; - - /* Destroy flow handle */ - if (flow != NULL) { - claim_zero(ibv_exp_destroy_flow(flow)); - mlx5_fdir_filter->flow = NULL; - } - } - - /* Destroy flow director context in each RX queue. */ - for (i = 0; (i != priv->rxqs_n); i++) { - struct rxq_ctrl *rxq_ctrl = - container_of((*priv->rxqs)[i], struct rxq_ctrl, rxq); - - if (!rxq_ctrl->fdir_queue) - continue; - priv_fdir_queue_destroy(priv, rxq_ctrl->fdir_queue); - rxq_ctrl->fdir_queue = NULL; - } - if (priv->fdir_drop_queue) { - priv_fdir_queue_destroy(priv, priv->fdir_drop_queue); - priv->fdir_drop_queue = NULL; - } -} - -/** - * Enable flow director, create steering rules. - * - * @param priv - * Private structure. - */ -void -priv_fdir_enable(struct priv *priv) -{ - struct mlx5_fdir_filter *mlx5_fdir_filter; - - /* Run on every fdir filter and create flow handle */ - LIST_FOREACH(mlx5_fdir_filter, priv->fdir_filter_list, next) { - /* Only valid elements should be in the list */ - assert(mlx5_fdir_filter != NULL); - - priv_fdir_filter_enable(priv, mlx5_fdir_filter); - } -} - -/** - * Find specific filter in list. - * - * @param priv - * Private structure. - * @param fdir_filter - * Flow director filter to find. - * - * @return - * Filter element if found, otherwise NULL. - */ -static struct mlx5_fdir_filter * -priv_find_filter_in_list(struct priv *priv, - const struct rte_eth_fdir_filter *fdir_filter) -{ - struct fdir_flow_desc desc; - struct mlx5_fdir_filter *mlx5_fdir_filter; - enum rte_fdir_mode fdir_mode = priv->dev->data->dev_conf.fdir_conf.mode; - - /* Get flow director filter to look for. */ - fdir_filter_to_flow_desc(fdir_filter, &desc, fdir_mode); - - /* Look for the requested element. */ - LIST_FOREACH(mlx5_fdir_filter, priv->fdir_filter_list, next) { - /* Only valid elements should be in the list. */ - assert(mlx5_fdir_filter != NULL); - - /* Return matching filter. */ - if (!memcmp(&desc, &mlx5_fdir_filter->desc, sizeof(desc))) - return mlx5_fdir_filter; - } - - /* Filter not found */ - return NULL; -} - -/** - * Add new flow director filter and store it in list. - * - * @param priv - * Private structure. - * @param fdir_filter - * Flow director filter to add. - * - * @return - * 0 on success, errno value on failure. - */ -static int -priv_fdir_filter_add(struct priv *priv, - const struct rte_eth_fdir_filter *fdir_filter) -{ - struct mlx5_fdir_filter *mlx5_fdir_filter; - enum rte_fdir_mode fdir_mode = priv->dev->data->dev_conf.fdir_conf.mode; - int err = 0; - - /* Validate queue number. */ - if (fdir_filter->action.rx_queue >= priv->rxqs_n) { - ERROR("invalid queue number %d", fdir_filter->action.rx_queue); - return EINVAL; - } - - /* Duplicate filters are currently unsupported. */ - mlx5_fdir_filter = priv_find_filter_in_list(priv, fdir_filter); - if (mlx5_fdir_filter != NULL) { - ERROR("filter already exists"); - return EINVAL; - } - - /* Create new flow director filter. */ - mlx5_fdir_filter = - rte_calloc(__func__, 1, sizeof(*mlx5_fdir_filter), 0); - if (mlx5_fdir_filter == NULL) { - err = ENOMEM; - ERROR("cannot allocate flow director filter: %s", - strerror(err)); - return err; - } - - /* Set action parameters. */ - mlx5_fdir_filter->queue = fdir_filter->action.rx_queue; - mlx5_fdir_filter->behavior = fdir_filter->action.behavior; - - /* Convert to mlx5 filter descriptor. */ - fdir_filter_to_flow_desc(fdir_filter, - &mlx5_fdir_filter->desc, fdir_mode); - - /* Insert new filter into list. */ - LIST_INSERT_HEAD(priv->fdir_filter_list, mlx5_fdir_filter, next); - - DEBUG("%p: flow director filter %p added", - (void *)priv, (void *)mlx5_fdir_filter); - - /* Enable filter immediately if device is started. */ - if (priv->started) - err = priv_fdir_filter_enable(priv, mlx5_fdir_filter); - - return err; -} - -/** - * Update queue for specific filter. - * - * @param priv - * Private structure. - * @param fdir_filter - * Filter to be updated. - * - * @return - * 0 on success, errno value on failure. - */ -static int -priv_fdir_filter_update(struct priv *priv, - const struct rte_eth_fdir_filter *fdir_filter) -{ - struct mlx5_fdir_filter *mlx5_fdir_filter; - - /* Validate queue number. */ - if (fdir_filter->action.rx_queue >= priv->rxqs_n) { - ERROR("invalid queue number %d", fdir_filter->action.rx_queue); - return EINVAL; - } - - mlx5_fdir_filter = priv_find_filter_in_list(priv, fdir_filter); - if (mlx5_fdir_filter != NULL) { - struct ibv_exp_flow *flow = mlx5_fdir_filter->flow; - int err = 0; - - /* Update queue number. */ - mlx5_fdir_filter->queue = fdir_filter->action.rx_queue; - - /* Destroy flow handle. */ - if (flow != NULL) { - claim_zero(ibv_exp_destroy_flow(flow)); - mlx5_fdir_filter->flow = NULL; - } - DEBUG("%p: flow director filter %p updated", - (void *)priv, (void *)mlx5_fdir_filter); - - /* Enable filter if device is started. */ - if (priv->started) - err = priv_fdir_filter_enable(priv, mlx5_fdir_filter); - - return err; - } - - /* Filter not found, create it. */ - DEBUG("%p: filter not found for update, creating new filter", - (void *)priv); - return priv_fdir_filter_add(priv, fdir_filter); -} - -/** - * Delete specific filter. - * - * @param priv - * Private structure. - * @param fdir_filter - * Filter to be deleted. - * - * @return - * 0 on success, errno value on failure. - */ -static int -priv_fdir_filter_delete(struct priv *priv, - const struct rte_eth_fdir_filter *fdir_filter) -{ - struct mlx5_fdir_filter *mlx5_fdir_filter; - - mlx5_fdir_filter = priv_find_filter_in_list(priv, fdir_filter); - if (mlx5_fdir_filter != NULL) { - struct ibv_exp_flow *flow = mlx5_fdir_filter->flow; - - /* Remove element from list. */ - LIST_REMOVE(mlx5_fdir_filter, next); - - /* Destroy flow handle. */ - if (flow != NULL) { - claim_zero(ibv_exp_destroy_flow(flow)); - mlx5_fdir_filter->flow = NULL; - } - - DEBUG("%p: flow director filter %p deleted", - (void *)priv, (void *)mlx5_fdir_filter); - - /* Delete filter. */ - rte_free(mlx5_fdir_filter); - - return 0; - } - - ERROR("%p: flow director delete failed, cannot find filter", - (void *)priv); - return EINVAL; -} - -/** - * Get flow director information. - * - * @param priv - * Private structure. - * @param[out] fdir_info - * Resulting flow director information. - */ -static void -priv_fdir_info_get(struct priv *priv, struct rte_eth_fdir_info *fdir_info) -{ - struct rte_eth_fdir_masks *mask = - &priv->dev->data->dev_conf.fdir_conf.mask; - - fdir_info->mode = priv->dev->data->dev_conf.fdir_conf.mode; - fdir_info->guarant_spc = 0; - - rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask)); - - fdir_info->max_flexpayload = 0; - fdir_info->flow_types_mask[0] = 0; - - fdir_info->flex_payload_unit = 0; - fdir_info->max_flex_payload_segment_num = 0; - fdir_info->flex_payload_limit = 0; - memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf)); -} - -/** - * Deal with flow director operations. - * - * @param priv - * Pointer to private structure. - * @param filter_op - * Operation to perform. - * @param arg - * Pointer to operation-specific structure. - * - * @return - * 0 on success, errno value on failure. - */ -static int -priv_fdir_ctrl_func(struct priv *priv, enum rte_filter_op filter_op, void *arg) -{ - enum rte_fdir_mode fdir_mode = - priv->dev->data->dev_conf.fdir_conf.mode; - int ret = 0; - - if (filter_op == RTE_ETH_FILTER_NOP) - return 0; - - if (fdir_mode != RTE_FDIR_MODE_PERFECT && - fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) { - ERROR("%p: flow director mode %d not supported", - (void *)priv, fdir_mode); - return EINVAL; - } - - switch (filter_op) { - case RTE_ETH_FILTER_ADD: - ret = priv_fdir_filter_add(priv, arg); - break; - case RTE_ETH_FILTER_UPDATE: - ret = priv_fdir_filter_update(priv, arg); - break; - case RTE_ETH_FILTER_DELETE: - ret = priv_fdir_filter_delete(priv, arg); - break; - case RTE_ETH_FILTER_FLUSH: - priv_fdir_filter_flush(priv); - break; - case RTE_ETH_FILTER_INFO: - priv_fdir_info_get(priv, arg); - break; - default: - DEBUG("%p: unknown operation %u", (void *)priv, filter_op); - ret = EINVAL; - break; - } - return ret; -} - -/** - * Manage filter operations. - * - * @param dev - * Pointer to Ethernet device structure. - * @param filter_type - * Filter type. - * @param filter_op - * Operation to perform. - * @param arg - * Pointer to operation-specific structure. - * - * @return - * 0 on success, negative errno value on failure. - */ -int -mlx5_dev_filter_ctrl(struct rte_eth_dev *dev, - enum rte_filter_type filter_type, - enum rte_filter_op filter_op, - void *arg) -{ - int ret = EINVAL; - struct priv *priv = dev->data->dev_private; - - switch (filter_type) { - case RTE_ETH_FILTER_FDIR: - priv_lock(priv); - ret = priv_fdir_ctrl_func(priv, filter_op, arg); - priv_unlock(priv); - break; - default: - ERROR("%p: filter type (%d) not supported", - (void *)dev, filter_type); - break; - } - - return -ret; -} diff --git a/dpdk/drivers/net/mlx5/mlx5_flow.c b/dpdk/drivers/net/mlx5/mlx5_flow.c new file mode 100644 index 00000000..092644ff --- /dev/null +++ b/dpdk/drivers/net/mlx5/mlx5_flow.c @@ -0,0 +1,3050 @@ +/*- + * BSD LICENSE + * + * Copyright 2016 6WIND S.A. + * Copyright 2016 Mellanox. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of 6WIND S.A. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include + +/* Verbs header. */ +/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ +#ifdef PEDANTIC +#pragma GCC diagnostic ignored "-Wpedantic" +#endif +#include +#ifdef PEDANTIC +#pragma GCC diagnostic error "-Wpedantic" +#endif + +#include +#include +#include +#include + +#include "mlx5.h" +#include "mlx5_defs.h" +#include "mlx5_prm.h" + +/* Define minimal priority for control plane flows. */ +#define MLX5_CTRL_FLOW_PRIORITY 4 + +/* Internet Protocol versions. */ +#define MLX5_IPV4 4 +#define MLX5_IPV6 6 + +#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT +struct ibv_counter_set_init_attr { + int dummy; +}; +struct ibv_flow_spec_counter_action { + int dummy; +}; +struct ibv_counter_set { + int dummy; +}; + +static inline int +ibv_destroy_counter_set(struct ibv_counter_set *cs) +{ + (void)cs; + return -ENOTSUP; +} +#endif + +/* Dev ops structure defined in mlx5.c */ +extern const struct eth_dev_ops mlx5_dev_ops; +extern const struct eth_dev_ops mlx5_dev_ops_isolate; + +static int +mlx5_flow_create_eth(const struct rte_flow_item *item, + const void *default_mask, + void *data); + +static int +mlx5_flow_create_vlan(const struct rte_flow_item *item, + const void *default_mask, + void *data); + +static int +mlx5_flow_create_ipv4(const struct rte_flow_item *item, + const void *default_mask, + void *data); + +static int +mlx5_flow_create_ipv6(const struct rte_flow_item *item, + const void *default_mask, + void *data); + +static int +mlx5_flow_create_udp(const struct rte_flow_item *item, + const void *default_mask, + void *data); + +static int +mlx5_flow_create_tcp(const struct rte_flow_item *item, + const void *default_mask, + void *data); + +static int +mlx5_flow_create_vxlan(const struct rte_flow_item *item, + const void *default_mask, + void *data); + +struct mlx5_flow_parse; + +static void +mlx5_flow_create_copy(struct mlx5_flow_parse *parser, void *src, + unsigned int size); + +static int +mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id); + +static int +mlx5_flow_create_count(struct priv *priv, struct mlx5_flow_parse *parser); + +/* Hash RX queue types. */ +enum hash_rxq_type { + HASH_RXQ_TCPV4, + HASH_RXQ_UDPV4, + HASH_RXQ_IPV4, + HASH_RXQ_TCPV6, + HASH_RXQ_UDPV6, + HASH_RXQ_IPV6, + HASH_RXQ_ETH, +}; + +/* Initialization data for hash RX queue. */ +struct hash_rxq_init { + uint64_t hash_fields; /* Fields that participate in the hash. */ + uint64_t dpdk_rss_hf; /* Matching DPDK RSS hash fields. */ + unsigned int flow_priority; /* Flow priority to use. */ + unsigned int ip_version; /* Internet protocol. */ +}; + +/* Initialization data for hash RX queues. */ +const struct hash_rxq_init hash_rxq_init[] = { + [HASH_RXQ_TCPV4] = { + .hash_fields = (IBV_RX_HASH_SRC_IPV4 | + IBV_RX_HASH_DST_IPV4 | + IBV_RX_HASH_SRC_PORT_TCP | + IBV_RX_HASH_DST_PORT_TCP), + .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_TCP, + .flow_priority = 0, + .ip_version = MLX5_IPV4, + }, + [HASH_RXQ_UDPV4] = { + .hash_fields = (IBV_RX_HASH_SRC_IPV4 | + IBV_RX_HASH_DST_IPV4 | + IBV_RX_HASH_SRC_PORT_UDP | + IBV_RX_HASH_DST_PORT_UDP), + .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_UDP, + .flow_priority = 0, + .ip_version = MLX5_IPV4, + }, + [HASH_RXQ_IPV4] = { + .hash_fields = (IBV_RX_HASH_SRC_IPV4 | + IBV_RX_HASH_DST_IPV4), + .dpdk_rss_hf = (ETH_RSS_IPV4 | + ETH_RSS_FRAG_IPV4), + .flow_priority = 1, + .ip_version = MLX5_IPV4, + }, + [HASH_RXQ_TCPV6] = { + .hash_fields = (IBV_RX_HASH_SRC_IPV6 | + IBV_RX_HASH_DST_IPV6 | + IBV_RX_HASH_SRC_PORT_TCP | + IBV_RX_HASH_DST_PORT_TCP), + .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_TCP, + .flow_priority = 0, + .ip_version = MLX5_IPV6, + }, + [HASH_RXQ_UDPV6] = { + .hash_fields = (IBV_RX_HASH_SRC_IPV6 | + IBV_RX_HASH_DST_IPV6 | + IBV_RX_HASH_SRC_PORT_UDP | + IBV_RX_HASH_DST_PORT_UDP), + .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_UDP, + .flow_priority = 0, + .ip_version = MLX5_IPV6, + }, + [HASH_RXQ_IPV6] = { + .hash_fields = (IBV_RX_HASH_SRC_IPV6 | + IBV_RX_HASH_DST_IPV6), + .dpdk_rss_hf = (ETH_RSS_IPV6 | + ETH_RSS_FRAG_IPV6), + .flow_priority = 1, + .ip_version = MLX5_IPV6, + }, + [HASH_RXQ_ETH] = { + .hash_fields = 0, + .dpdk_rss_hf = 0, + .flow_priority = 2, + }, +}; + +/* Number of entries in hash_rxq_init[]. */ +const unsigned int hash_rxq_init_n = RTE_DIM(hash_rxq_init); + +/** Structure for holding counter stats. */ +struct mlx5_flow_counter_stats { + uint64_t hits; /**< Number of packets matched by the rule. */ + uint64_t bytes; /**< Number of bytes matched by the rule. */ +}; + +/** Structure for Drop queue. */ +struct mlx5_hrxq_drop { + struct ibv_rwq_ind_table *ind_table; /**< Indirection table. */ + struct ibv_qp *qp; /**< Verbs queue pair. */ + struct ibv_wq *wq; /**< Verbs work queue. */ + struct ibv_cq *cq; /**< Verbs completion queue. */ +}; + +/* Flows structures. */ +struct mlx5_flow { + uint64_t hash_fields; /**< Fields that participate in the hash. */ + struct ibv_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */ + struct ibv_flow *ibv_flow; /**< Verbs flow. */ + struct mlx5_hrxq *hrxq; /**< Hash Rx queues. */ +}; + +/* Drop flows structures. */ +struct mlx5_flow_drop { + struct ibv_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */ + struct ibv_flow *ibv_flow; /**< Verbs flow. */ +}; + +struct rte_flow { + TAILQ_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */ + uint32_t mark:1; /**< Set if the flow is marked. */ + uint32_t drop:1; /**< Drop queue. */ + uint16_t queues_n; /**< Number of entries in queue[]. */ + uint16_t (*queues)[]; /**< Queues indexes to use. */ + struct rte_eth_rss_conf rss_conf; /**< RSS configuration */ + uint8_t rss_key[40]; /**< copy of the RSS key. */ + struct ibv_counter_set *cs; /**< Holds the counters for the rule. */ + struct mlx5_flow_counter_stats counter_stats;/**mask is not provided. When + * \default_mask is also NULL, the full supported bit-mask (\mask) is + * used instead. + */ + const void *default_mask; + /** Bit-masks size in bytes. */ + const unsigned int mask_sz; + /** + * Conversion function from rte_flow to NIC specific flow. + * + * @param item + * rte_flow item to convert. + * @param default_mask + * Default bit-masks to use when item->mask is not provided. + * @param data + * Internal structure to store the conversion. + * + * @return + * 0 on success, negative value otherwise. + */ + int (*convert)(const struct rte_flow_item *item, + const void *default_mask, + void *data); + /** Size in bytes of the destination structure. */ + const unsigned int dst_sz; + /** List of possible following items. */ + const enum rte_flow_item_type *const items; +}; + +/** Valid action for this PMD. */ +static const enum rte_flow_action_type valid_actions[] = { + RTE_FLOW_ACTION_TYPE_DROP, + RTE_FLOW_ACTION_TYPE_QUEUE, + RTE_FLOW_ACTION_TYPE_MARK, + RTE_FLOW_ACTION_TYPE_FLAG, +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT + RTE_FLOW_ACTION_TYPE_COUNT, +#endif + RTE_FLOW_ACTION_TYPE_END, +}; + +/** Graph of supported items and associated actions. */ +static const struct mlx5_flow_items mlx5_flow_items[] = { + [RTE_FLOW_ITEM_TYPE_END] = { + .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_VXLAN), + }, + [RTE_FLOW_ITEM_TYPE_ETH] = { + .items = ITEMS(RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_IPV6), + .actions = valid_actions, + .mask = &(const struct rte_flow_item_eth){ + .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", + .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", + .type = -1, + }, + .default_mask = &rte_flow_item_eth_mask, + .mask_sz = sizeof(struct rte_flow_item_eth), + .convert = mlx5_flow_create_eth, + .dst_sz = sizeof(struct ibv_flow_spec_eth), + }, + [RTE_FLOW_ITEM_TYPE_VLAN] = { + .items = ITEMS(RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_IPV6), + .actions = valid_actions, + .mask = &(const struct rte_flow_item_vlan){ + .tci = -1, + }, + .default_mask = &rte_flow_item_vlan_mask, + .mask_sz = sizeof(struct rte_flow_item_vlan), + .convert = mlx5_flow_create_vlan, + .dst_sz = 0, + }, + [RTE_FLOW_ITEM_TYPE_IPV4] = { + .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_TCP), + .actions = valid_actions, + .mask = &(const struct rte_flow_item_ipv4){ + .hdr = { + .src_addr = -1, + .dst_addr = -1, + .type_of_service = -1, + .next_proto_id = -1, + }, + }, + .default_mask = &rte_flow_item_ipv4_mask, + .mask_sz = sizeof(struct rte_flow_item_ipv4), + .convert = mlx5_flow_create_ipv4, + .dst_sz = sizeof(struct ibv_flow_spec_ipv4_ext), + }, + [RTE_FLOW_ITEM_TYPE_IPV6] = { + .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_TCP), + .actions = valid_actions, + .mask = &(const struct rte_flow_item_ipv6){ + .hdr = { + .src_addr = { + 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, + }, + .dst_addr = { + 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, + }, + .vtc_flow = -1, + .proto = -1, + .hop_limits = -1, + }, + }, + .default_mask = &rte_flow_item_ipv6_mask, + .mask_sz = sizeof(struct rte_flow_item_ipv6), + .convert = mlx5_flow_create_ipv6, + .dst_sz = sizeof(struct ibv_flow_spec_ipv6), + }, + [RTE_FLOW_ITEM_TYPE_UDP] = { + .items = ITEMS(RTE_FLOW_ITEM_TYPE_VXLAN), + .actions = valid_actions, + .mask = &(const struct rte_flow_item_udp){ + .hdr = { + .src_port = -1, + .dst_port = -1, + }, + }, + .default_mask = &rte_flow_item_udp_mask, + .mask_sz = sizeof(struct rte_flow_item_udp), + .convert = mlx5_flow_create_udp, + .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp), + }, + [RTE_FLOW_ITEM_TYPE_TCP] = { + .actions = valid_actions, + .mask = &(const struct rte_flow_item_tcp){ + .hdr = { + .src_port = -1, + .dst_port = -1, + }, + }, + .default_mask = &rte_flow_item_tcp_mask, + .mask_sz = sizeof(struct rte_flow_item_tcp), + .convert = mlx5_flow_create_tcp, + .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp), + }, + [RTE_FLOW_ITEM_TYPE_VXLAN] = { + .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH), + .actions = valid_actions, + .mask = &(const struct rte_flow_item_vxlan){ + .vni = "\xff\xff\xff", + }, + .default_mask = &rte_flow_item_vxlan_mask, + .mask_sz = sizeof(struct rte_flow_item_vxlan), + .convert = mlx5_flow_create_vxlan, + .dst_sz = sizeof(struct ibv_flow_spec_tunnel), + }, +}; + +/** Structure to pass to the conversion function. */ +struct mlx5_flow_parse { + uint32_t inner; /**< Set once VXLAN is encountered. */ + uint32_t create:1; + /**< Whether resources should remain after a validate. */ + uint32_t drop:1; /**< Target is a drop queue. */ + uint32_t mark:1; /**< Mark is present in the flow. */ + uint32_t count:1; /**< Count is present in the flow. */ + uint32_t mark_id; /**< Mark identifier. */ + uint16_t queues[RTE_MAX_QUEUES_PER_PORT]; /**< Queues indexes to use. */ + uint16_t queues_n; /**< Number of entries in queue[]. */ + struct rte_eth_rss_conf rss_conf; /**< RSS configuration */ + uint8_t rss_key[40]; /**< copy of the RSS key. */ + enum hash_rxq_type layer; /**< Last pattern layer detected. */ + struct ibv_counter_set *cs; /**< Holds the counter set for the rule */ + struct { + struct ibv_flow_attr *ibv_attr; + /**< Pointer to Verbs attributes. */ + unsigned int offset; + /**< Current position or total size of the attribute. */ + } queue[RTE_DIM(hash_rxq_init)]; +}; + +static const struct rte_flow_ops mlx5_flow_ops = { + .validate = mlx5_flow_validate, + .create = mlx5_flow_create, + .destroy = mlx5_flow_destroy, + .flush = mlx5_flow_flush, +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT + .query = mlx5_flow_query, +#else + .query = NULL, +#endif + .isolate = mlx5_flow_isolate, +}; + +/* Convert FDIR request to Generic flow. */ +struct mlx5_fdir { + struct rte_flow_attr attr; + struct rte_flow_action actions[2]; + struct rte_flow_item items[4]; + struct rte_flow_item_eth l2; + struct rte_flow_item_eth l2_mask; + union { + struct rte_flow_item_ipv4 ipv4; + struct rte_flow_item_ipv6 ipv6; + } l3; + union { + struct rte_flow_item_udp udp; + struct rte_flow_item_tcp tcp; + } l4; + struct rte_flow_action_queue queue; +}; + +/* Verbs specification header. */ +struct ibv_spec_header { + enum ibv_flow_spec_type type; + uint16_t size; +}; + +/** + * Check support for a given item. + * + * @param item[in] + * Item specification. + * @param mask[in] + * Bit-masks covering supported fields to compare with spec, last and mask in + * \item. + * @param size + * Bit-Mask size in bytes. + * + * @return + * 0 on success. + */ +static int +mlx5_flow_item_validate(const struct rte_flow_item *item, + const uint8_t *mask, unsigned int size) +{ + int ret = 0; + + if (!item->spec && (item->mask || item->last)) + return -1; + if (item->spec && !item->mask) { + unsigned int i; + const uint8_t *spec = item->spec; + + for (i = 0; i < size; ++i) + if ((spec[i] | mask[i]) != mask[i]) + return -1; + } + if (item->last && !item->mask) { + unsigned int i; + const uint8_t *spec = item->last; + + for (i = 0; i < size; ++i) + if ((spec[i] | mask[i]) != mask[i]) + return -1; + } + if (item->mask) { + unsigned int i; + const uint8_t *spec = item->spec; + + for (i = 0; i < size; ++i) + if ((spec[i] | mask[i]) != mask[i]) + return -1; + } + if (item->spec && item->last) { + uint8_t spec[size]; + uint8_t last[size]; + const uint8_t *apply = mask; + unsigned int i; + + if (item->mask) + apply = item->mask; + for (i = 0; i < size; ++i) { + spec[i] = ((const uint8_t *)item->spec)[i] & apply[i]; + last[i] = ((const uint8_t *)item->last)[i] & apply[i]; + } + ret = memcmp(spec, last, size); + } + return ret; +} + +/** + * Copy the RSS configuration from the user ones, of the rss_conf is null, + * uses the driver one. + * + * @param priv + * Pointer to private structure. + * @param parser + * Internal parser structure. + * @param rss_conf + * User RSS configuration to save. + * + * @return + * 0 on success, errno value on failure. + */ +static int +priv_flow_convert_rss_conf(struct priv *priv, + struct mlx5_flow_parse *parser, + const struct rte_eth_rss_conf *rss_conf) +{ + /* + * This function is also called at the beginning of + * priv_flow_convert_actions() to initialize the parser with the + * device default RSS configuration. + */ + (void)priv; + if (rss_conf) { + if (rss_conf->rss_hf & MLX5_RSS_HF_MASK) + return EINVAL; + if (rss_conf->rss_key_len != 40) + return EINVAL; + if (rss_conf->rss_key_len && rss_conf->rss_key) { + parser->rss_conf.rss_key_len = rss_conf->rss_key_len; + memcpy(parser->rss_key, rss_conf->rss_key, + rss_conf->rss_key_len); + parser->rss_conf.rss_key = parser->rss_key; + } + parser->rss_conf.rss_hf = rss_conf->rss_hf; + } + return 0; +} + +/** + * Extract attribute to the parser. + * + * @param priv + * Pointer to private structure. + * @param[in] attr + * Flow rule attributes. + * @param[out] error + * Perform verbose error reporting if not NULL. + * @param[in, out] parser + * Internal parser structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +priv_flow_convert_attributes(struct priv *priv, + const struct rte_flow_attr *attr, + struct rte_flow_error *error, + struct mlx5_flow_parse *parser) +{ + (void)priv; + (void)parser; + if (attr->group) { + rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ATTR_GROUP, + NULL, + "groups are not supported"); + return -rte_errno; + } + if (attr->priority && attr->priority != MLX5_CTRL_FLOW_PRIORITY) { + rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, + NULL, + "priorities are not supported"); + return -rte_errno; + } + if (attr->egress) { + rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, + NULL, + "egress is not supported"); + return -rte_errno; + } + if (!attr->ingress) { + rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, + NULL, + "only ingress is supported"); + return -rte_errno; + } + return 0; +} + +/** + * Extract actions request to the parser. + * + * @param priv + * Pointer to private structure. + * @param[in] actions + * Associated actions (list terminated by the END action). + * @param[out] error + * Perform verbose error reporting if not NULL. + * @param[in, out] parser + * Internal parser structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +priv_flow_convert_actions(struct priv *priv, + const struct rte_flow_action actions[], + struct rte_flow_error *error, + struct mlx5_flow_parse *parser) +{ + /* + * Add default RSS configuration necessary for Verbs to create QP even + * if no RSS is necessary. + */ + priv_flow_convert_rss_conf(priv, parser, + (const struct rte_eth_rss_conf *) + &priv->rss_conf); + for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) { + if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) { + continue; + } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) { + parser->drop = 1; + } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) { + const struct rte_flow_action_queue *queue = + (const struct rte_flow_action_queue *) + actions->conf; + uint16_t n; + uint16_t found = 0; + + if (!queue || (queue->index > (priv->rxqs_n - 1))) + goto exit_action_not_supported; + for (n = 0; n < parser->queues_n; ++n) { + if (parser->queues[n] == queue->index) { + found = 1; + break; + } + } + if (parser->queues_n > 1 && !found) { + rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, + actions, + "queue action not in RSS queues"); + return -rte_errno; + } + if (!found) { + parser->queues_n = 1; + parser->queues[0] = queue->index; + } + } else if (actions->type == RTE_FLOW_ACTION_TYPE_RSS) { + const struct rte_flow_action_rss *rss = + (const struct rte_flow_action_rss *) + actions->conf; + uint16_t n; + + if (!rss || !rss->num) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + actions, + "no valid queues"); + return -rte_errno; + } + if (parser->queues_n == 1) { + uint16_t found = 0; + + assert(parser->queues_n); + for (n = 0; n < rss->num; ++n) { + if (parser->queues[0] == + rss->queue[n]) { + found = 1; + break; + } + } + if (!found) { + rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, + actions, + "queue action not in RSS" + " queues"); + return -rte_errno; + } + } + for (n = 0; n < rss->num; ++n) { + if (rss->queue[n] >= priv->rxqs_n) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + actions, + "queue id > number of" + " queues"); + return -rte_errno; + } + } + for (n = 0; n < rss->num; ++n) + parser->queues[n] = rss->queue[n]; + parser->queues_n = rss->num; + if (priv_flow_convert_rss_conf(priv, parser, + rss->rss_conf)) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + actions, + "wrong RSS configuration"); + return -rte_errno; + } + } else if (actions->type == RTE_FLOW_ACTION_TYPE_MARK) { + const struct rte_flow_action_mark *mark = + (const struct rte_flow_action_mark *) + actions->conf; + + if (!mark) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + actions, + "mark must be defined"); + return -rte_errno; + } else if (mark->id >= MLX5_FLOW_MARK_MAX) { + rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, + actions, + "mark must be between 0" + " and 16777199"); + return -rte_errno; + } + parser->mark = 1; + parser->mark_id = mark->id; + } else if (actions->type == RTE_FLOW_ACTION_TYPE_FLAG) { + parser->mark = 1; + } else if (actions->type == RTE_FLOW_ACTION_TYPE_COUNT && + priv->counter_set_supported) { + parser->count = 1; + } else { + goto exit_action_not_supported; + } + } + if (parser->drop && parser->mark) + parser->mark = 0; + if (!parser->queues_n && !parser->drop) { + rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE, + NULL, "no valid action"); + return -rte_errno; + } + return 0; +exit_action_not_supported: + rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, + actions, "action not supported"); + return -rte_errno; +} + +/** + * Validate items. + * + * @param priv + * Pointer to private structure. + * @param[in] items + * Pattern specification (list terminated by the END pattern item). + * @param[out] error + * Perform verbose error reporting if not NULL. + * @param[in, out] parser + * Internal parser structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +priv_flow_convert_items_validate(struct priv *priv, + const struct rte_flow_item items[], + struct rte_flow_error *error, + struct mlx5_flow_parse *parser) +{ + const struct mlx5_flow_items *cur_item = mlx5_flow_items; + unsigned int i; + + (void)priv; + /* Initialise the offsets to start after verbs attribute. */ + for (i = 0; i != hash_rxq_init_n; ++i) + parser->queue[i].offset = sizeof(struct ibv_flow_attr); + for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) { + const struct mlx5_flow_items *token = NULL; + unsigned int n; + int err; + + if (items->type == RTE_FLOW_ITEM_TYPE_VOID) + continue; + for (i = 0; + cur_item->items && + cur_item->items[i] != RTE_FLOW_ITEM_TYPE_END; + ++i) { + if (cur_item->items[i] == items->type) { + token = &mlx5_flow_items[items->type]; + break; + } + } + if (!token) + goto exit_item_not_supported; + cur_item = token; + err = mlx5_flow_item_validate(items, + (const uint8_t *)cur_item->mask, + cur_item->mask_sz); + if (err) + goto exit_item_not_supported; + if (items->type == RTE_FLOW_ITEM_TYPE_VXLAN) { + if (parser->inner) { + rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ITEM, + items, + "cannot recognize multiple" + " VXLAN encapsulations"); + return -rte_errno; + } + parser->inner = IBV_FLOW_SPEC_INNER; + } + if (parser->drop) { + parser->queue[HASH_RXQ_ETH].offset += cur_item->dst_sz; + } else { + for (n = 0; n != hash_rxq_init_n; ++n) + parser->queue[n].offset += cur_item->dst_sz; + } + } + if (parser->drop) { + parser->queue[HASH_RXQ_ETH].offset += + sizeof(struct ibv_flow_spec_action_drop); + } + if (parser->mark) { + for (i = 0; i != hash_rxq_init_n; ++i) + parser->queue[i].offset += + sizeof(struct ibv_flow_spec_action_tag); + } + if (parser->count) { + unsigned int size = sizeof(struct ibv_flow_spec_counter_action); + + for (i = 0; i != hash_rxq_init_n; ++i) + parser->queue[i].offset += size; + } + return 0; +exit_item_not_supported: + rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, + items, "item not supported"); + return -rte_errno; +} + +/** + * Allocate memory space to store verbs flow attributes. + * + * @param priv + * Pointer to private structure. + * @param[in] priority + * Flow priority. + * @param[in] size + * Amount of byte to allocate. + * @param[out] error + * Perform verbose error reporting if not NULL. + * + * @return + * A verbs flow attribute on success, NULL otherwise. + */ +static struct ibv_flow_attr* +priv_flow_convert_allocate(struct priv *priv, + unsigned int priority, + unsigned int size, + struct rte_flow_error *error) +{ + struct ibv_flow_attr *ibv_attr; + + (void)priv; + ibv_attr = rte_calloc(__func__, 1, size, 0); + if (!ibv_attr) { + rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, + "cannot allocate verbs spec attributes."); + return NULL; + } + ibv_attr->priority = priority; + return ibv_attr; +} + +/** + * Finalise verbs flow attributes. + * + * @param priv + * Pointer to private structure. + * @param[in, out] parser + * Internal parser structure. + */ +static void +priv_flow_convert_finalise(struct priv *priv, struct mlx5_flow_parse *parser) +{ + const unsigned int ipv4 = + hash_rxq_init[parser->layer].ip_version == MLX5_IPV4; + const enum hash_rxq_type hmin = ipv4 ? HASH_RXQ_TCPV4 : HASH_RXQ_TCPV6; + const enum hash_rxq_type hmax = ipv4 ? HASH_RXQ_IPV4 : HASH_RXQ_IPV6; + const enum hash_rxq_type ohmin = ipv4 ? HASH_RXQ_TCPV6 : HASH_RXQ_TCPV4; + const enum hash_rxq_type ohmax = ipv4 ? HASH_RXQ_IPV6 : HASH_RXQ_IPV4; + const enum hash_rxq_type ip = ipv4 ? HASH_RXQ_IPV4 : HASH_RXQ_IPV6; + unsigned int i; + + (void)priv; + if (parser->layer == HASH_RXQ_ETH) { + goto fill; + } else { + /* + * This layer becomes useless as the pattern define under + * layers. + */ + rte_free(parser->queue[HASH_RXQ_ETH].ibv_attr); + parser->queue[HASH_RXQ_ETH].ibv_attr = NULL; + } + /* Remove opposite kind of layer e.g. IPv6 if the pattern is IPv4. */ + for (i = ohmin; i != (ohmax + 1); ++i) { + if (!parser->queue[i].ibv_attr) + continue; + rte_free(parser->queue[i].ibv_attr); + parser->queue[i].ibv_attr = NULL; + } + /* Remove impossible flow according to the RSS configuration. */ + if (hash_rxq_init[parser->layer].dpdk_rss_hf & + parser->rss_conf.rss_hf) { + /* Remove any other flow. */ + for (i = hmin; i != (hmax + 1); ++i) { + if ((i == parser->layer) || + (!parser->queue[i].ibv_attr)) + continue; + rte_free(parser->queue[i].ibv_attr); + parser->queue[i].ibv_attr = NULL; + } + } else if (!parser->queue[ip].ibv_attr) { + /* no RSS possible with the current configuration. */ + parser->queues_n = 1; + return; + } +fill: + /* + * Fill missing layers in verbs specifications, or compute the correct + * offset to allocate the memory space for the attributes and + * specifications. + */ + for (i = 0; i != hash_rxq_init_n - 1; ++i) { + union { + struct ibv_flow_spec_ipv4_ext ipv4; + struct ibv_flow_spec_ipv6 ipv6; + struct ibv_flow_spec_tcp_udp udp_tcp; + } specs; + void *dst; + uint16_t size; + + if (i == parser->layer) + continue; + if (parser->layer == HASH_RXQ_ETH) { + if (hash_rxq_init[i].ip_version == MLX5_IPV4) { + size = sizeof(struct ibv_flow_spec_ipv4_ext); + specs.ipv4 = (struct ibv_flow_spec_ipv4_ext){ + .type = IBV_FLOW_SPEC_IPV4_EXT, + .size = size, + }; + } else { + size = sizeof(struct ibv_flow_spec_ipv6); + specs.ipv6 = (struct ibv_flow_spec_ipv6){ + .type = IBV_FLOW_SPEC_IPV6, + .size = size, + }; + } + if (parser->queue[i].ibv_attr) { + dst = (void *)((uintptr_t) + parser->queue[i].ibv_attr + + parser->queue[i].offset); + memcpy(dst, &specs, size); + ++parser->queue[i].ibv_attr->num_of_specs; + } + parser->queue[i].offset += size; + } + if ((i == HASH_RXQ_UDPV4) || (i == HASH_RXQ_TCPV4) || + (i == HASH_RXQ_UDPV6) || (i == HASH_RXQ_TCPV6)) { + size = sizeof(struct ibv_flow_spec_tcp_udp); + specs.udp_tcp = (struct ibv_flow_spec_tcp_udp) { + .type = ((i == HASH_RXQ_UDPV4 || + i == HASH_RXQ_UDPV6) ? + IBV_FLOW_SPEC_UDP : + IBV_FLOW_SPEC_TCP), + .size = size, + }; + if (parser->queue[i].ibv_attr) { + dst = (void *)((uintptr_t) + parser->queue[i].ibv_attr + + parser->queue[i].offset); + memcpy(dst, &specs, size); + ++parser->queue[i].ibv_attr->num_of_specs; + } + parser->queue[i].offset += size; + } + } +} + +/** + * Validate and convert a flow supported by the NIC. + * + * @param priv + * Pointer to private structure. + * @param[in] attr + * Flow rule attributes. + * @param[in] pattern + * Pattern specification (list terminated by the END pattern item). + * @param[in] actions + * Associated actions (list terminated by the END action). + * @param[out] error + * Perform verbose error reporting if not NULL. + * @param[in, out] parser + * Internal parser structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +priv_flow_convert(struct priv *priv, + const struct rte_flow_attr *attr, + const struct rte_flow_item items[], + const struct rte_flow_action actions[], + struct rte_flow_error *error, + struct mlx5_flow_parse *parser) +{ + const struct mlx5_flow_items *cur_item = mlx5_flow_items; + unsigned int i; + int ret; + + /* First step. Validate the attributes, items and actions. */ + *parser = (struct mlx5_flow_parse){ + .create = parser->create, + .layer = HASH_RXQ_ETH, + .mark_id = MLX5_FLOW_MARK_DEFAULT, + }; + ret = priv_flow_convert_attributes(priv, attr, error, parser); + if (ret) + return ret; + ret = priv_flow_convert_actions(priv, actions, error, parser); + if (ret) + return ret; + ret = priv_flow_convert_items_validate(priv, items, error, parser); + if (ret) + return ret; + priv_flow_convert_finalise(priv, parser); + /* + * Second step. + * Allocate the memory space to store verbs specifications. + */ + if (parser->drop) { + parser->queue[HASH_RXQ_ETH].ibv_attr = + priv_flow_convert_allocate + (priv, attr->priority, + parser->queue[HASH_RXQ_ETH].offset, + error); + if (!parser->queue[HASH_RXQ_ETH].ibv_attr) + return ENOMEM; + parser->queue[HASH_RXQ_ETH].offset = + sizeof(struct ibv_flow_attr); + } else { + for (i = 0; i != hash_rxq_init_n; ++i) { + unsigned int priority = + attr->priority + + hash_rxq_init[i].flow_priority; + unsigned int offset; + + if (!(parser->rss_conf.rss_hf & + hash_rxq_init[i].dpdk_rss_hf) && + (i != HASH_RXQ_ETH)) + continue; + offset = parser->queue[i].offset; + parser->queue[i].ibv_attr = + priv_flow_convert_allocate(priv, priority, + offset, error); + if (!parser->queue[i].ibv_attr) + goto exit_enomem; + parser->queue[i].offset = sizeof(struct ibv_flow_attr); + } + } + /* Third step. Conversion parse, fill the specifications. */ + parser->inner = 0; + for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) { + if (items->type == RTE_FLOW_ITEM_TYPE_VOID) + continue; + cur_item = &mlx5_flow_items[items->type]; + ret = cur_item->convert(items, + (cur_item->default_mask ? + cur_item->default_mask : + cur_item->mask), + parser); + if (ret) { + rte_flow_error_set(error, ret, + RTE_FLOW_ERROR_TYPE_ITEM, + items, "item not supported"); + goto exit_free; + } + } + if (parser->mark) + mlx5_flow_create_flag_mark(parser, parser->mark_id); + if (parser->count && parser->create) { + mlx5_flow_create_count(priv, parser); + if (!parser->cs) + goto exit_count_error; + } + /* + * Last step. Complete missing specification to reach the RSS + * configuration. + */ + if (!parser->drop) { + priv_flow_convert_finalise(priv, parser); + } else { + parser->queue[HASH_RXQ_ETH].ibv_attr->priority = + attr->priority + + hash_rxq_init[parser->layer].flow_priority; + } +exit_free: + /* Only verification is expected, all resources should be released. */ + if (!parser->create) { + for (i = 0; i != hash_rxq_init_n; ++i) { + if (parser->queue[i].ibv_attr) { + rte_free(parser->queue[i].ibv_attr); + parser->queue[i].ibv_attr = NULL; + } + } + } + return ret; +exit_enomem: + for (i = 0; i != hash_rxq_init_n; ++i) { + if (parser->queue[i].ibv_attr) { + rte_free(parser->queue[i].ibv_attr); + parser->queue[i].ibv_attr = NULL; + } + } + rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "cannot allocate verbs spec attributes."); + return ret; +exit_count_error: + rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "cannot create counter."); + return rte_errno; +} + +/** + * Copy the specification created into the flow. + * + * @param parser + * Internal parser structure. + * @param src + * Create specification. + * @param size + * Size in bytes of the specification to copy. + */ +static void +mlx5_flow_create_copy(struct mlx5_flow_parse *parser, void *src, + unsigned int size) +{ + unsigned int i; + void *dst; + + for (i = 0; i != hash_rxq_init_n; ++i) { + if (!parser->queue[i].ibv_attr) + continue; + /* Specification must be the same l3 type or none. */ + if (parser->layer == HASH_RXQ_ETH || + (hash_rxq_init[parser->layer].ip_version == + hash_rxq_init[i].ip_version) || + (hash_rxq_init[i].ip_version == 0)) { + dst = (void *)((uintptr_t)parser->queue[i].ibv_attr + + parser->queue[i].offset); + memcpy(dst, src, size); + ++parser->queue[i].ibv_attr->num_of_specs; + parser->queue[i].offset += size; + } + } +} + +/** + * Convert Ethernet item to Verbs specification. + * + * @param item[in] + * Item specification. + * @param default_mask[in] + * Default bit-masks to use when item->mask is not provided. + * @param data[in, out] + * User structure. + */ +static int +mlx5_flow_create_eth(const struct rte_flow_item *item, + const void *default_mask, + void *data) +{ + const struct rte_flow_item_eth *spec = item->spec; + const struct rte_flow_item_eth *mask = item->mask; + struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data; + const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth); + struct ibv_flow_spec_eth eth = { + .type = parser->inner | IBV_FLOW_SPEC_ETH, + .size = eth_size, + }; + + /* Don't update layer for the inner pattern. */ + if (!parser->inner) + parser->layer = HASH_RXQ_ETH; + if (spec) { + unsigned int i; + + if (!mask) + mask = default_mask; + memcpy(ð.val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN); + memcpy(ð.val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN); + eth.val.ether_type = spec->type; + memcpy(ð.mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN); + memcpy(ð.mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN); + eth.mask.ether_type = mask->type; + /* Remove unwanted bits from values. */ + for (i = 0; i < ETHER_ADDR_LEN; ++i) { + eth.val.dst_mac[i] &= eth.mask.dst_mac[i]; + eth.val.src_mac[i] &= eth.mask.src_mac[i]; + } + eth.val.ether_type &= eth.mask.ether_type; + } + mlx5_flow_create_copy(parser, ð, eth_size); + return 0; +} + +/** + * Convert VLAN item to Verbs specification. + * + * @param item[in] + * Item specification. + * @param default_mask[in] + * Default bit-masks to use when item->mask is not provided. + * @param data[in, out] + * User structure. + */ +static int +mlx5_flow_create_vlan(const struct rte_flow_item *item, + const void *default_mask, + void *data) +{ + const struct rte_flow_item_vlan *spec = item->spec; + const struct rte_flow_item_vlan *mask = item->mask; + struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data; + struct ibv_flow_spec_eth *eth; + const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth); + + if (spec) { + unsigned int i; + if (!mask) + mask = default_mask; + + for (i = 0; i != hash_rxq_init_n; ++i) { + if (!parser->queue[i].ibv_attr) + continue; + + eth = (void *)((uintptr_t)parser->queue[i].ibv_attr + + parser->queue[i].offset - eth_size); + eth->val.vlan_tag = spec->tci; + eth->mask.vlan_tag = mask->tci; + eth->val.vlan_tag &= eth->mask.vlan_tag; + } + } + return 0; +} + +/** + * Convert IPv4 item to Verbs specification. + * + * @param item[in] + * Item specification. + * @param default_mask[in] + * Default bit-masks to use when item->mask is not provided. + * @param data[in, out] + * User structure. + */ +static int +mlx5_flow_create_ipv4(const struct rte_flow_item *item, + const void *default_mask, + void *data) +{ + const struct rte_flow_item_ipv4 *spec = item->spec; + const struct rte_flow_item_ipv4 *mask = item->mask; + struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data; + unsigned int ipv4_size = sizeof(struct ibv_flow_spec_ipv4_ext); + struct ibv_flow_spec_ipv4_ext ipv4 = { + .type = parser->inner | IBV_FLOW_SPEC_IPV4_EXT, + .size = ipv4_size, + }; + + /* Don't update layer for the inner pattern. */ + if (!parser->inner) + parser->layer = HASH_RXQ_IPV4; + if (spec) { + if (!mask) + mask = default_mask; + ipv4.val = (struct ibv_flow_ipv4_ext_filter){ + .src_ip = spec->hdr.src_addr, + .dst_ip = spec->hdr.dst_addr, + .proto = spec->hdr.next_proto_id, + .tos = spec->hdr.type_of_service, + }; + ipv4.mask = (struct ibv_flow_ipv4_ext_filter){ + .src_ip = mask->hdr.src_addr, + .dst_ip = mask->hdr.dst_addr, + .proto = mask->hdr.next_proto_id, + .tos = mask->hdr.type_of_service, + }; + /* Remove unwanted bits from values. */ + ipv4.val.src_ip &= ipv4.mask.src_ip; + ipv4.val.dst_ip &= ipv4.mask.dst_ip; + ipv4.val.proto &= ipv4.mask.proto; + ipv4.val.tos &= ipv4.mask.tos; + } + mlx5_flow_create_copy(parser, &ipv4, ipv4_size); + return 0; +} + +/** + * Convert IPv6 item to Verbs specification. + * + * @param item[in] + * Item specification. + * @param default_mask[in] + * Default bit-masks to use when item->mask is not provided. + * @param data[in, out] + * User structure. + */ +static int +mlx5_flow_create_ipv6(const struct rte_flow_item *item, + const void *default_mask, + void *data) +{ + const struct rte_flow_item_ipv6 *spec = item->spec; + const struct rte_flow_item_ipv6 *mask = item->mask; + struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data; + unsigned int ipv6_size = sizeof(struct ibv_flow_spec_ipv6); + struct ibv_flow_spec_ipv6 ipv6 = { + .type = parser->inner | IBV_FLOW_SPEC_IPV6, + .size = ipv6_size, + }; + + /* Don't update layer for the inner pattern. */ + if (!parser->inner) + parser->layer = HASH_RXQ_IPV6; + if (spec) { + unsigned int i; + + if (!mask) + mask = default_mask; + memcpy(&ipv6.val.src_ip, spec->hdr.src_addr, + RTE_DIM(ipv6.val.src_ip)); + memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr, + RTE_DIM(ipv6.val.dst_ip)); + memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr, + RTE_DIM(ipv6.mask.src_ip)); + memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr, + RTE_DIM(ipv6.mask.dst_ip)); + ipv6.mask.flow_label = mask->hdr.vtc_flow; + ipv6.mask.next_hdr = mask->hdr.proto; + ipv6.mask.hop_limit = mask->hdr.hop_limits; + /* Remove unwanted bits from values. */ + for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) { + ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i]; + ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i]; + } + ipv6.val.flow_label &= ipv6.mask.flow_label; + ipv6.val.next_hdr &= ipv6.mask.next_hdr; + ipv6.val.hop_limit &= ipv6.mask.hop_limit; + } + mlx5_flow_create_copy(parser, &ipv6, ipv6_size); + return 0; +} + +/** + * Convert UDP item to Verbs specification. + * + * @param item[in] + * Item specification. + * @param default_mask[in] + * Default bit-masks to use when item->mask is not provided. + * @param data[in, out] + * User structure. + */ +static int +mlx5_flow_create_udp(const struct rte_flow_item *item, + const void *default_mask, + void *data) +{ + const struct rte_flow_item_udp *spec = item->spec; + const struct rte_flow_item_udp *mask = item->mask; + struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data; + unsigned int udp_size = sizeof(struct ibv_flow_spec_tcp_udp); + struct ibv_flow_spec_tcp_udp udp = { + .type = parser->inner | IBV_FLOW_SPEC_UDP, + .size = udp_size, + }; + + /* Don't update layer for the inner pattern. */ + if (!parser->inner) { + if (parser->layer == HASH_RXQ_IPV4) + parser->layer = HASH_RXQ_UDPV4; + else + parser->layer = HASH_RXQ_UDPV6; + } + if (spec) { + if (!mask) + mask = default_mask; + udp.val.dst_port = spec->hdr.dst_port; + udp.val.src_port = spec->hdr.src_port; + udp.mask.dst_port = mask->hdr.dst_port; + udp.mask.src_port = mask->hdr.src_port; + /* Remove unwanted bits from values. */ + udp.val.src_port &= udp.mask.src_port; + udp.val.dst_port &= udp.mask.dst_port; + } + mlx5_flow_create_copy(parser, &udp, udp_size); + return 0; +} + +/** + * Convert TCP item to Verbs specification. + * + * @param item[in] + * Item specification. + * @param default_mask[in] + * Default bit-masks to use when item->mask is not provided. + * @param data[in, out] + * User structure. + */ +static int +mlx5_flow_create_tcp(const struct rte_flow_item *item, + const void *default_mask, + void *data) +{ + const struct rte_flow_item_tcp *spec = item->spec; + const struct rte_flow_item_tcp *mask = item->mask; + struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data; + unsigned int tcp_size = sizeof(struct ibv_flow_spec_tcp_udp); + struct ibv_flow_spec_tcp_udp tcp = { + .type = parser->inner | IBV_FLOW_SPEC_TCP, + .size = tcp_size, + }; + + /* Don't update layer for the inner pattern. */ + if (!parser->inner) { + if (parser->layer == HASH_RXQ_IPV4) + parser->layer = HASH_RXQ_TCPV4; + else + parser->layer = HASH_RXQ_TCPV6; + } + if (spec) { + if (!mask) + mask = default_mask; + tcp.val.dst_port = spec->hdr.dst_port; + tcp.val.src_port = spec->hdr.src_port; + tcp.mask.dst_port = mask->hdr.dst_port; + tcp.mask.src_port = mask->hdr.src_port; + /* Remove unwanted bits from values. */ + tcp.val.src_port &= tcp.mask.src_port; + tcp.val.dst_port &= tcp.mask.dst_port; + } + mlx5_flow_create_copy(parser, &tcp, tcp_size); + return 0; +} + +/** + * Convert VXLAN item to Verbs specification. + * + * @param item[in] + * Item specification. + * @param default_mask[in] + * Default bit-masks to use when item->mask is not provided. + * @param data[in, out] + * User structure. + */ +static int +mlx5_flow_create_vxlan(const struct rte_flow_item *item, + const void *default_mask, + void *data) +{ + const struct rte_flow_item_vxlan *spec = item->spec; + const struct rte_flow_item_vxlan *mask = item->mask; + struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data; + unsigned int size = sizeof(struct ibv_flow_spec_tunnel); + struct ibv_flow_spec_tunnel vxlan = { + .type = parser->inner | IBV_FLOW_SPEC_VXLAN_TUNNEL, + .size = size, + }; + union vni { + uint32_t vlan_id; + uint8_t vni[4]; + } id; + + id.vni[0] = 0; + parser->inner = IBV_FLOW_SPEC_INNER; + if (spec) { + if (!mask) + mask = default_mask; + memcpy(&id.vni[1], spec->vni, 3); + vxlan.val.tunnel_id = id.vlan_id; + memcpy(&id.vni[1], mask->vni, 3); + vxlan.mask.tunnel_id = id.vlan_id; + /* Remove unwanted bits from values. */ + vxlan.val.tunnel_id &= vxlan.mask.tunnel_id; + } + /* + * Tunnel id 0 is equivalent as not adding a VXLAN layer, if only this + * layer is defined in the Verbs specification it is interpreted as + * wildcard and all packets will match this rule, if it follows a full + * stack layer (ex: eth / ipv4 / udp), all packets matching the layers + * before will also match this rule. + * To avoid such situation, VNI 0 is currently refused. + */ + if (!vxlan.val.tunnel_id) + return EINVAL; + mlx5_flow_create_copy(parser, &vxlan, size); + return 0; +} + +/** + * Convert mark/flag action to Verbs specification. + * + * @param parser + * Internal parser structure. + * @param mark_id + * Mark identifier. + */ +static int +mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id) +{ + unsigned int size = sizeof(struct ibv_flow_spec_action_tag); + struct ibv_flow_spec_action_tag tag = { + .type = IBV_FLOW_SPEC_ACTION_TAG, + .size = size, + .tag_id = mlx5_flow_mark_set(mark_id), + }; + + assert(parser->mark); + mlx5_flow_create_copy(parser, &tag, size); + return 0; +} + +/** + * Convert count action to Verbs specification. + * + * @param priv + * Pointer to private structure. + * @param parser + * Pointer to MLX5 flow parser structure. + * + * @return + * 0 on success, errno value on failure. + */ +static int +mlx5_flow_create_count(struct priv *priv __rte_unused, + struct mlx5_flow_parse *parser __rte_unused) +{ +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT + unsigned int size = sizeof(struct ibv_flow_spec_counter_action); + struct ibv_counter_set_init_attr init_attr = {0}; + struct ibv_flow_spec_counter_action counter = { + .type = IBV_FLOW_SPEC_ACTION_COUNT, + .size = size, + .counter_set_handle = 0, + }; + + init_attr.counter_set_id = 0; + parser->cs = ibv_create_counter_set(priv->ctx, &init_attr); + if (!parser->cs) + return EINVAL; + counter.counter_set_handle = parser->cs->handle; + mlx5_flow_create_copy(parser, &counter, size); +#endif + return 0; +} + +/** + * Complete flow rule creation with a drop queue. + * + * @param priv + * Pointer to private structure. + * @param parser + * Internal parser structure. + * @param flow + * Pointer to the rte_flow. + * @param[out] error + * Perform verbose error reporting if not NULL. + * + * @return + * 0 on success, errno value on failure. + */ +static int +priv_flow_create_action_queue_drop(struct priv *priv, + struct mlx5_flow_parse *parser, + struct rte_flow *flow, + struct rte_flow_error *error) +{ + struct ibv_flow_spec_action_drop *drop; + unsigned int size = sizeof(struct ibv_flow_spec_action_drop); + int err = 0; + + assert(priv->pd); + assert(priv->ctx); + flow->drop = 1; + drop = (void *)((uintptr_t)parser->queue[HASH_RXQ_ETH].ibv_attr + + parser->queue[HASH_RXQ_ETH].offset); + *drop = (struct ibv_flow_spec_action_drop){ + .type = IBV_FLOW_SPEC_ACTION_DROP, + .size = size, + }; + ++parser->queue[HASH_RXQ_ETH].ibv_attr->num_of_specs; + parser->queue[HASH_RXQ_ETH].offset += size; + flow->frxq[HASH_RXQ_ETH].ibv_attr = + parser->queue[HASH_RXQ_ETH].ibv_attr; + if (parser->count) + flow->cs = parser->cs; + if (!priv->dev->data->dev_started) + return 0; + parser->queue[HASH_RXQ_ETH].ibv_attr = NULL; + flow->frxq[HASH_RXQ_ETH].ibv_flow = + ibv_create_flow(priv->flow_drop_queue->qp, + flow->frxq[HASH_RXQ_ETH].ibv_attr); + if (!flow->frxq[HASH_RXQ_ETH].ibv_flow) { + rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, + NULL, "flow rule creation failure"); + err = ENOMEM; + goto error; + } + return 0; +error: + assert(flow); + if (flow->frxq[HASH_RXQ_ETH].ibv_flow) { + claim_zero(ibv_destroy_flow(flow->frxq[HASH_RXQ_ETH].ibv_flow)); + flow->frxq[HASH_RXQ_ETH].ibv_flow = NULL; + } + if (flow->frxq[HASH_RXQ_ETH].ibv_attr) { + rte_free(flow->frxq[HASH_RXQ_ETH].ibv_attr); + flow->frxq[HASH_RXQ_ETH].ibv_attr = NULL; + } + if (flow->cs) { + claim_zero(ibv_destroy_counter_set(flow->cs)); + flow->cs = NULL; + parser->cs = NULL; + } + return err; +} + +/** + * Create hash Rx queues when RSS is enabled. + * + * @param priv + * Pointer to private structure. + * @param parser + * Internal parser structure. + * @param flow + * Pointer to the rte_flow. + * @param[out] error + * Perform verbose error reporting if not NULL. + * + * @return + * 0 on success, a errno value otherwise and rte_errno is set. + */ +static int +priv_flow_create_action_queue_rss(struct priv *priv, + struct mlx5_flow_parse *parser, + struct rte_flow *flow, + struct rte_flow_error *error) +{ + unsigned int i; + + for (i = 0; i != hash_rxq_init_n; ++i) { + uint64_t hash_fields; + + if (!parser->queue[i].ibv_attr) + continue; + flow->frxq[i].ibv_attr = parser->queue[i].ibv_attr; + parser->queue[i].ibv_attr = NULL; + hash_fields = hash_rxq_init[i].hash_fields; + if (!priv->dev->data->dev_started) + continue; + flow->frxq[i].hrxq = + mlx5_priv_hrxq_get(priv, + parser->rss_conf.rss_key, + parser->rss_conf.rss_key_len, + hash_fields, + parser->queues, + parser->queues_n); + if (flow->frxq[i].hrxq) + continue; + flow->frxq[i].hrxq = + mlx5_priv_hrxq_new(priv, + parser->rss_conf.rss_key, + parser->rss_conf.rss_key_len, + hash_fields, + parser->queues, + parser->queues_n); + if (!flow->frxq[i].hrxq) { + rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_HANDLE, + NULL, "cannot create hash rxq"); + return ENOMEM; + } + } + return 0; +} + +/** + * Complete flow rule creation. + * + * @param priv + * Pointer to private structure. + * @param parser + * Internal parser structure. + * @param flow + * Pointer to the rte_flow. + * @param[out] error + * Perform verbose error reporting if not NULL. + * + * @return + * 0 on success, a errno value otherwise and rte_errno is set. + */ +static int +priv_flow_create_action_queue(struct priv *priv, + struct mlx5_flow_parse *parser, + struct rte_flow *flow, + struct rte_flow_error *error) +{ + int err = 0; + unsigned int i; + + assert(priv->pd); + assert(priv->ctx); + assert(!parser->drop); + err = priv_flow_create_action_queue_rss(priv, parser, flow, error); + if (err) + goto error; + if (parser->count) + flow->cs = parser->cs; + if (!priv->dev->data->dev_started) + return 0; + for (i = 0; i != hash_rxq_init_n; ++i) { + if (!flow->frxq[i].hrxq) + continue; + flow->frxq[i].ibv_flow = + ibv_create_flow(flow->frxq[i].hrxq->qp, + flow->frxq[i].ibv_attr); + if (!flow->frxq[i].ibv_flow) { + rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_HANDLE, + NULL, "flow rule creation failure"); + err = ENOMEM; + goto error; + } + DEBUG("%p type %d QP %p ibv_flow %p", + (void *)flow, i, + (void *)flow->frxq[i].hrxq, + (void *)flow->frxq[i].ibv_flow); + } + for (i = 0; i != parser->queues_n; ++i) { + struct mlx5_rxq_data *q = + (*priv->rxqs)[parser->queues[i]]; + + q->mark |= parser->mark; + } + return 0; +error: + assert(flow); + for (i = 0; i != hash_rxq_init_n; ++i) { + if (flow->frxq[i].ibv_flow) { + struct ibv_flow *ibv_flow = flow->frxq[i].ibv_flow; + + claim_zero(ibv_destroy_flow(ibv_flow)); + } + if (flow->frxq[i].hrxq) + mlx5_priv_hrxq_release(priv, flow->frxq[i].hrxq); + if (flow->frxq[i].ibv_attr) + rte_free(flow->frxq[i].ibv_attr); + } + if (flow->cs) { + claim_zero(ibv_destroy_counter_set(flow->cs)); + flow->cs = NULL; + parser->cs = NULL; + } + return err; +} + +/** + * Convert a flow. + * + * @param priv + * Pointer to private structure. + * @param list + * Pointer to a TAILQ flow list. + * @param[in] attr + * Flow rule attributes. + * @param[in] pattern + * Pattern specification (list terminated by the END pattern item). + * @param[in] actions + * Associated actions (list terminated by the END action). + * @param[out] error + * Perform verbose error reporting if not NULL. + * + * @return + * A flow on success, NULL otherwise. + */ +static struct rte_flow * +priv_flow_create(struct priv *priv, + struct mlx5_flows *list, + const struct rte_flow_attr *attr, + const struct rte_flow_item items[], + const struct rte_flow_action actions[], + struct rte_flow_error *error) +{ + struct mlx5_flow_parse parser = { .create = 1, }; + struct rte_flow *flow = NULL; + unsigned int i; + int err; + + err = priv_flow_convert(priv, attr, items, actions, error, &parser); + if (err) + goto exit; + flow = rte_calloc(__func__, 1, + sizeof(*flow) + parser.queues_n * sizeof(uint16_t), + 0); + if (!flow) { + rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, + "cannot allocate flow memory"); + return NULL; + } + /* Copy queues configuration. */ + flow->queues = (uint16_t (*)[])(flow + 1); + memcpy(flow->queues, parser.queues, parser.queues_n * sizeof(uint16_t)); + flow->queues_n = parser.queues_n; + flow->mark = parser.mark; + /* Copy RSS configuration. */ + flow->rss_conf = parser.rss_conf; + flow->rss_conf.rss_key = flow->rss_key; + memcpy(flow->rss_key, parser.rss_key, parser.rss_conf.rss_key_len); + /* finalise the flow. */ + if (parser.drop) + err = priv_flow_create_action_queue_drop(priv, &parser, flow, + error); + else + err = priv_flow_create_action_queue(priv, &parser, flow, error); + if (err) + goto exit; + TAILQ_INSERT_TAIL(list, flow, next); + DEBUG("Flow created %p", (void *)flow); + return flow; +exit: + for (i = 0; i != hash_rxq_init_n; ++i) { + if (parser.queue[i].ibv_attr) + rte_free(parser.queue[i].ibv_attr); + } + rte_free(flow); + return NULL; +} + +/** + * Validate a flow supported by the NIC. + * + * @see rte_flow_validate() + * @see rte_flow_ops + */ +int +mlx5_flow_validate(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item items[], + const struct rte_flow_action actions[], + struct rte_flow_error *error) +{ + struct priv *priv = dev->data->dev_private; + int ret; + struct mlx5_flow_parse parser = { .create = 0, }; + + priv_lock(priv); + ret = priv_flow_convert(priv, attr, items, actions, error, &parser); + priv_unlock(priv); + return ret; +} + +/** + * Create a flow. + * + * @see rte_flow_create() + * @see rte_flow_ops + */ +struct rte_flow * +mlx5_flow_create(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item items[], + const struct rte_flow_action actions[], + struct rte_flow_error *error) +{ + struct priv *priv = dev->data->dev_private; + struct rte_flow *flow; + + priv_lock(priv); + flow = priv_flow_create(priv, &priv->flows, attr, items, actions, + error); + priv_unlock(priv); + return flow; +} + +/** + * Destroy a flow. + * + * @param priv + * Pointer to private structure. + * @param list + * Pointer to a TAILQ flow list. + * @param[in] flow + * Flow to destroy. + */ +static void +priv_flow_destroy(struct priv *priv, + struct mlx5_flows *list, + struct rte_flow *flow) +{ + unsigned int i; + + if (flow->drop || !flow->mark) + goto free; + for (i = 0; i != flow->queues_n; ++i) { + struct rte_flow *tmp; + int mark = 0; + + /* + * To remove the mark from the queue, the queue must not be + * present in any other marked flow (RSS or not). + */ + TAILQ_FOREACH(tmp, list, next) { + unsigned int j; + uint16_t *tqs = NULL; + uint16_t tq_n = 0; + + if (!tmp->mark) + continue; + for (j = 0; j != hash_rxq_init_n; ++j) { + if (!tmp->frxq[j].hrxq) + continue; + tqs = tmp->frxq[j].hrxq->ind_table->queues; + tq_n = tmp->frxq[j].hrxq->ind_table->queues_n; + } + if (!tq_n) + continue; + for (j = 0; (j != tq_n) && !mark; j++) + if (tqs[j] == (*flow->queues)[i]) + mark = 1; + } + (*priv->rxqs)[(*flow->queues)[i]]->mark = mark; + } +free: + if (flow->drop) { + if (flow->frxq[HASH_RXQ_ETH].ibv_flow) + claim_zero(ibv_destroy_flow + (flow->frxq[HASH_RXQ_ETH].ibv_flow)); + rte_free(flow->frxq[HASH_RXQ_ETH].ibv_attr); + } else { + for (i = 0; i != hash_rxq_init_n; ++i) { + struct mlx5_flow *frxq = &flow->frxq[i]; + + if (frxq->ibv_flow) + claim_zero(ibv_destroy_flow(frxq->ibv_flow)); + if (frxq->hrxq) + mlx5_priv_hrxq_release(priv, frxq->hrxq); + if (frxq->ibv_attr) + rte_free(frxq->ibv_attr); + } + } + if (flow->cs) { + claim_zero(ibv_destroy_counter_set(flow->cs)); + flow->cs = NULL; + } + TAILQ_REMOVE(list, flow, next); + DEBUG("Flow destroyed %p", (void *)flow); + rte_free(flow); +} + +/** + * Destroy all flows. + * + * @param priv + * Pointer to private structure. + * @param list + * Pointer to a TAILQ flow list. + */ +void +priv_flow_flush(struct priv *priv, struct mlx5_flows *list) +{ + while (!TAILQ_EMPTY(list)) { + struct rte_flow *flow; + + flow = TAILQ_FIRST(list); + priv_flow_destroy(priv, list, flow); + } +} + +/** + * Create drop queue. + * + * @param priv + * Pointer to private structure. + * + * @return + * 0 on success. + */ +int +priv_flow_create_drop_queue(struct priv *priv) +{ + struct mlx5_hrxq_drop *fdq = NULL; + + assert(priv->pd); + assert(priv->ctx); + fdq = rte_calloc(__func__, 1, sizeof(*fdq), 0); + if (!fdq) { + WARN("cannot allocate memory for drop queue"); + goto error; + } + fdq->cq = ibv_create_cq(priv->ctx, 1, NULL, NULL, 0); + if (!fdq->cq) { + WARN("cannot allocate CQ for drop queue"); + goto error; + } + fdq->wq = ibv_create_wq(priv->ctx, + &(struct ibv_wq_init_attr){ + .wq_type = IBV_WQT_RQ, + .max_wr = 1, + .max_sge = 1, + .pd = priv->pd, + .cq = fdq->cq, + }); + if (!fdq->wq) { + WARN("cannot allocate WQ for drop queue"); + goto error; + } + fdq->ind_table = ibv_create_rwq_ind_table(priv->ctx, + &(struct ibv_rwq_ind_table_init_attr){ + .log_ind_tbl_size = 0, + .ind_tbl = &fdq->wq, + .comp_mask = 0, + }); + if (!fdq->ind_table) { + WARN("cannot allocate indirection table for drop queue"); + goto error; + } + fdq->qp = ibv_create_qp_ex(priv->ctx, + &(struct ibv_qp_init_attr_ex){ + .qp_type = IBV_QPT_RAW_PACKET, + .comp_mask = + IBV_QP_INIT_ATTR_PD | + IBV_QP_INIT_ATTR_IND_TABLE | + IBV_QP_INIT_ATTR_RX_HASH, + .rx_hash_conf = (struct ibv_rx_hash_conf){ + .rx_hash_function = + IBV_RX_HASH_FUNC_TOEPLITZ, + .rx_hash_key_len = rss_hash_default_key_len, + .rx_hash_key = rss_hash_default_key, + .rx_hash_fields_mask = 0, + }, + .rwq_ind_tbl = fdq->ind_table, + .pd = priv->pd + }); + if (!fdq->qp) { + WARN("cannot allocate QP for drop queue"); + goto error; + } + priv->flow_drop_queue = fdq; + return 0; +error: + if (fdq->qp) + claim_zero(ibv_destroy_qp(fdq->qp)); + if (fdq->ind_table) + claim_zero(ibv_destroy_rwq_ind_table(fdq->ind_table)); + if (fdq->wq) + claim_zero(ibv_destroy_wq(fdq->wq)); + if (fdq->cq) + claim_zero(ibv_destroy_cq(fdq->cq)); + if (fdq) + rte_free(fdq); + priv->flow_drop_queue = NULL; + return -1; +} + +/** + * Delete drop queue. + * + * @param priv + * Pointer to private structure. + */ +void +priv_flow_delete_drop_queue(struct priv *priv) +{ + struct mlx5_hrxq_drop *fdq = priv->flow_drop_queue; + + if (!fdq) + return; + if (fdq->qp) + claim_zero(ibv_destroy_qp(fdq->qp)); + if (fdq->ind_table) + claim_zero(ibv_destroy_rwq_ind_table(fdq->ind_table)); + if (fdq->wq) + claim_zero(ibv_destroy_wq(fdq->wq)); + if (fdq->cq) + claim_zero(ibv_destroy_cq(fdq->cq)); + rte_free(fdq); + priv->flow_drop_queue = NULL; +} + +/** + * Remove all flows. + * + * @param priv + * Pointer to private structure. + * @param list + * Pointer to a TAILQ flow list. + */ +void +priv_flow_stop(struct priv *priv, struct mlx5_flows *list) +{ + struct rte_flow *flow; + + TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next) { + unsigned int i; + struct mlx5_ind_table_ibv *ind_tbl = NULL; + + if (flow->drop) { + if (!flow->frxq[HASH_RXQ_ETH].ibv_flow) + continue; + claim_zero(ibv_destroy_flow + (flow->frxq[HASH_RXQ_ETH].ibv_flow)); + flow->frxq[HASH_RXQ_ETH].ibv_flow = NULL; + DEBUG("Flow %p removed", (void *)flow); + /* Next flow. */ + continue; + } + /* Verify the flow has not already been cleaned. */ + for (i = 0; i != hash_rxq_init_n; ++i) { + if (!flow->frxq[i].ibv_flow) + continue; + /* + * Indirection table may be necessary to remove the + * flags in the Rx queues. + * This helps to speed-up the process by avoiding + * another loop. + */ + ind_tbl = flow->frxq[i].hrxq->ind_table; + break; + } + if (i == hash_rxq_init_n) + return; + if (flow->mark) { + assert(ind_tbl); + for (i = 0; i != ind_tbl->queues_n; ++i) + (*priv->rxqs)[ind_tbl->queues[i]]->mark = 0; + } + for (i = 0; i != hash_rxq_init_n; ++i) { + if (!flow->frxq[i].ibv_flow) + continue; + claim_zero(ibv_destroy_flow(flow->frxq[i].ibv_flow)); + flow->frxq[i].ibv_flow = NULL; + mlx5_priv_hrxq_release(priv, flow->frxq[i].hrxq); + flow->frxq[i].hrxq = NULL; + } + DEBUG("Flow %p removed", (void *)flow); + } +} + +/** + * Add all flows. + * + * @param priv + * Pointer to private structure. + * @param list + * Pointer to a TAILQ flow list. + * + * @return + * 0 on success, a errno value otherwise and rte_errno is set. + */ +int +priv_flow_start(struct priv *priv, struct mlx5_flows *list) +{ + struct rte_flow *flow; + + TAILQ_FOREACH(flow, list, next) { + unsigned int i; + + if (flow->drop) { + flow->frxq[HASH_RXQ_ETH].ibv_flow = + ibv_create_flow + (priv->flow_drop_queue->qp, + flow->frxq[HASH_RXQ_ETH].ibv_attr); + if (!flow->frxq[HASH_RXQ_ETH].ibv_flow) { + DEBUG("Flow %p cannot be applied", + (void *)flow); + rte_errno = EINVAL; + return rte_errno; + } + DEBUG("Flow %p applied", (void *)flow); + /* Next flow. */ + continue; + } + for (i = 0; i != hash_rxq_init_n; ++i) { + if (!flow->frxq[i].ibv_attr) + continue; + flow->frxq[i].hrxq = + mlx5_priv_hrxq_get(priv, flow->rss_conf.rss_key, + flow->rss_conf.rss_key_len, + hash_rxq_init[i].hash_fields, + (*flow->queues), + flow->queues_n); + if (flow->frxq[i].hrxq) + goto flow_create; + flow->frxq[i].hrxq = + mlx5_priv_hrxq_new(priv, flow->rss_conf.rss_key, + flow->rss_conf.rss_key_len, + hash_rxq_init[i].hash_fields, + (*flow->queues), + flow->queues_n); + if (!flow->frxq[i].hrxq) { + DEBUG("Flow %p cannot be applied", + (void *)flow); + rte_errno = EINVAL; + return rte_errno; + } +flow_create: + flow->frxq[i].ibv_flow = + ibv_create_flow(flow->frxq[i].hrxq->qp, + flow->frxq[i].ibv_attr); + if (!flow->frxq[i].ibv_flow) { + DEBUG("Flow %p cannot be applied", + (void *)flow); + rte_errno = EINVAL; + return rte_errno; + } + DEBUG("Flow %p applied", (void *)flow); + } + if (!flow->mark) + continue; + for (i = 0; i != flow->queues_n; ++i) + (*priv->rxqs)[(*flow->queues)[i]]->mark = 1; + } + return 0; +} + +/** + * Verify the flow list is empty + * + * @param priv + * Pointer to private structure. + * + * @return the number of flows not released. + */ +int +priv_flow_verify(struct priv *priv) +{ + struct rte_flow *flow; + int ret = 0; + + TAILQ_FOREACH(flow, &priv->flows, next) { + DEBUG("%p: flow %p still referenced", (void *)priv, + (void *)flow); + ++ret; + } + return ret; +} + +/** + * Enable a control flow configured from the control plane. + * + * @param dev + * Pointer to Ethernet device. + * @param eth_spec + * An Ethernet flow spec to apply. + * @param eth_mask + * An Ethernet flow mask to apply. + * @param vlan_spec + * A VLAN flow spec to apply. + * @param vlan_mask + * A VLAN flow mask to apply. + * + * @return + * 0 on success. + */ +int +mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, + struct rte_flow_item_eth *eth_spec, + struct rte_flow_item_eth *eth_mask, + struct rte_flow_item_vlan *vlan_spec, + struct rte_flow_item_vlan *vlan_mask) +{ + struct priv *priv = dev->data->dev_private; + const struct rte_flow_attr attr = { + .ingress = 1, + .priority = MLX5_CTRL_FLOW_PRIORITY, + }; + struct rte_flow_item items[] = { + { + .type = RTE_FLOW_ITEM_TYPE_ETH, + .spec = eth_spec, + .last = NULL, + .mask = eth_mask, + }, + { + .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : + RTE_FLOW_ITEM_TYPE_END, + .spec = vlan_spec, + .last = NULL, + .mask = vlan_mask, + }, + { + .type = RTE_FLOW_ITEM_TYPE_END, + }, + }; + struct rte_flow_action actions[] = { + { + .type = RTE_FLOW_ACTION_TYPE_RSS, + }, + { + .type = RTE_FLOW_ACTION_TYPE_END, + }, + }; + struct rte_flow *flow; + struct rte_flow_error error; + unsigned int i; + union { + struct rte_flow_action_rss rss; + struct { + const struct rte_eth_rss_conf *rss_conf; + uint16_t num; + uint16_t queue[RTE_MAX_QUEUES_PER_PORT]; + } local; + } action_rss; + + if (!priv->reta_idx_n) + return EINVAL; + for (i = 0; i != priv->reta_idx_n; ++i) + action_rss.local.queue[i] = (*priv->reta_idx)[i]; + action_rss.local.rss_conf = &priv->rss_conf; + action_rss.local.num = priv->reta_idx_n; + actions[0].conf = (const void *)&action_rss.rss; + flow = priv_flow_create(priv, &priv->ctrl_flows, &attr, items, actions, + &error); + if (!flow) + return rte_errno; + return 0; +} + +/** + * Enable a flow control configured from the control plane. + * + * @param dev + * Pointer to Ethernet device. + * @param eth_spec + * An Ethernet flow spec to apply. + * @param eth_mask + * An Ethernet flow mask to apply. + * + * @return + * 0 on success. + */ +int +mlx5_ctrl_flow(struct rte_eth_dev *dev, + struct rte_flow_item_eth *eth_spec, + struct rte_flow_item_eth *eth_mask) +{ + return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); +} + +/** + * Destroy a flow. + * + * @see rte_flow_destroy() + * @see rte_flow_ops + */ +int +mlx5_flow_destroy(struct rte_eth_dev *dev, + struct rte_flow *flow, + struct rte_flow_error *error) +{ + struct priv *priv = dev->data->dev_private; + + (void)error; + priv_lock(priv); + priv_flow_destroy(priv, &priv->flows, flow); + priv_unlock(priv); + return 0; +} + +/** + * Destroy all flows. + * + * @see rte_flow_flush() + * @see rte_flow_ops + */ +int +mlx5_flow_flush(struct rte_eth_dev *dev, + struct rte_flow_error *error) +{ + struct priv *priv = dev->data->dev_private; + + (void)error; + priv_lock(priv); + priv_flow_flush(priv, &priv->flows); + priv_unlock(priv); + return 0; +} + +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT +/** + * Query flow counter. + * + * @param cs + * the counter set. + * @param counter_value + * returned data from the counter. + * + * @return + * 0 on success, a errno value otherwise and rte_errno is set. + */ +static int +priv_flow_query_count(struct ibv_counter_set *cs, + struct mlx5_flow_counter_stats *counter_stats, + struct rte_flow_query_count *query_count, + struct rte_flow_error *error) +{ + uint64_t counters[2]; + struct ibv_query_counter_set_attr query_cs_attr = { + .cs = cs, + .query_flags = IBV_COUNTER_SET_FORCE_UPDATE, + }; + struct ibv_counter_set_data query_out = { + .out = counters, + .outlen = 2 * sizeof(uint64_t), + }; + int res = ibv_query_counter_set(&query_cs_attr, &query_out); + + if (res) { + rte_flow_error_set(error, -res, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, + "cannot read counter"); + return -res; + } + query_count->hits_set = 1; + query_count->bytes_set = 1; + query_count->hits = counters[0] - counter_stats->hits; + query_count->bytes = counters[1] - counter_stats->bytes; + if (query_count->reset) { + counter_stats->hits = counters[0]; + counter_stats->bytes = counters[1]; + } + return 0; +} + +/** + * Query a flows. + * + * @see rte_flow_query() + * @see rte_flow_ops + */ +int +mlx5_flow_query(struct rte_eth_dev *dev, + struct rte_flow *flow, + enum rte_flow_action_type action __rte_unused, + void *data, + struct rte_flow_error *error) +{ + struct priv *priv = dev->data->dev_private; + int res = EINVAL; + + priv_lock(priv); + if (flow->cs) { + res = priv_flow_query_count(flow->cs, + &flow->counter_stats, + (struct rte_flow_query_count *)data, + error); + } else { + rte_flow_error_set(error, res, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, + "no counter found for flow"); + } + priv_unlock(priv); + return -res; +} +#endif + +/** + * Isolated mode. + * + * @see rte_flow_isolate() + * @see rte_flow_ops + */ +int +mlx5_flow_isolate(struct rte_eth_dev *dev, + int enable, + struct rte_flow_error *error) +{ + struct priv *priv = dev->data->dev_private; + + priv_lock(priv); + if (dev->data->dev_started) { + rte_flow_error_set(error, EBUSY, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, + "port must be stopped first"); + priv_unlock(priv); + return -rte_errno; + } + priv->isolated = !!enable; + if (enable) + priv->dev->dev_ops = &mlx5_dev_ops_isolate; + else + priv->dev->dev_ops = &mlx5_dev_ops; + priv_unlock(priv); + return 0; +} + +/** + * Convert a flow director filter to a generic flow. + * + * @param priv + * Private structure. + * @param fdir_filter + * Flow director filter to add. + * @param attributes + * Generic flow parameters structure. + * + * @return + * 0 on success, errno value on error. + */ +static int +priv_fdir_filter_convert(struct priv *priv, + const struct rte_eth_fdir_filter *fdir_filter, + struct mlx5_fdir *attributes) +{ + const struct rte_eth_fdir_input *input = &fdir_filter->input; + + /* Validate queue number. */ + if (fdir_filter->action.rx_queue >= priv->rxqs_n) { + ERROR("invalid queue number %d", fdir_filter->action.rx_queue); + return EINVAL; + } + attributes->attr.ingress = 1; + attributes->items[0] = (struct rte_flow_item) { + .type = RTE_FLOW_ITEM_TYPE_ETH, + .spec = &attributes->l2, + .mask = &attributes->l2_mask, + }; + switch (fdir_filter->action.behavior) { + case RTE_ETH_FDIR_ACCEPT: + attributes->actions[0] = (struct rte_flow_action){ + .type = RTE_FLOW_ACTION_TYPE_QUEUE, + .conf = &attributes->queue, + }; + break; + case RTE_ETH_FDIR_REJECT: + attributes->actions[0] = (struct rte_flow_action){ + .type = RTE_FLOW_ACTION_TYPE_DROP, + }; + break; + default: + ERROR("invalid behavior %d", fdir_filter->action.behavior); + return ENOTSUP; + } + attributes->queue.index = fdir_filter->action.rx_queue; + switch (fdir_filter->input.flow_type) { + case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: + attributes->l3.ipv4.hdr = (struct ipv4_hdr){ + .src_addr = input->flow.udp4_flow.ip.src_ip, + .dst_addr = input->flow.udp4_flow.ip.dst_ip, + .time_to_live = input->flow.udp4_flow.ip.ttl, + .type_of_service = input->flow.udp4_flow.ip.tos, + .next_proto_id = input->flow.udp4_flow.ip.proto, + }; + attributes->l4.udp.hdr = (struct udp_hdr){ + .src_port = input->flow.udp4_flow.src_port, + .dst_port = input->flow.udp4_flow.dst_port, + }; + attributes->items[1] = (struct rte_flow_item){ + .type = RTE_FLOW_ITEM_TYPE_IPV4, + .spec = &attributes->l3, + }; + attributes->items[2] = (struct rte_flow_item){ + .type = RTE_FLOW_ITEM_TYPE_UDP, + .spec = &attributes->l4, + }; + break; + case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: + attributes->l3.ipv4.hdr = (struct ipv4_hdr){ + .src_addr = input->flow.tcp4_flow.ip.src_ip, + .dst_addr = input->flow.tcp4_flow.ip.dst_ip, + .time_to_live = input->flow.tcp4_flow.ip.ttl, + .type_of_service = input->flow.tcp4_flow.ip.tos, + .next_proto_id = input->flow.tcp4_flow.ip.proto, + }; + attributes->l4.tcp.hdr = (struct tcp_hdr){ + .src_port = input->flow.tcp4_flow.src_port, + .dst_port = input->flow.tcp4_flow.dst_port, + }; + attributes->items[1] = (struct rte_flow_item){ + .type = RTE_FLOW_ITEM_TYPE_IPV4, + .spec = &attributes->l3, + }; + attributes->items[2] = (struct rte_flow_item){ + .type = RTE_FLOW_ITEM_TYPE_TCP, + .spec = &attributes->l4, + }; + break; + case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: + attributes->l3.ipv4.hdr = (struct ipv4_hdr){ + .src_addr = input->flow.ip4_flow.src_ip, + .dst_addr = input->flow.ip4_flow.dst_ip, + .time_to_live = input->flow.ip4_flow.ttl, + .type_of_service = input->flow.ip4_flow.tos, + .next_proto_id = input->flow.ip4_flow.proto, + }; + attributes->items[1] = (struct rte_flow_item){ + .type = RTE_FLOW_ITEM_TYPE_IPV4, + .spec = &attributes->l3, + }; + break; + case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: + attributes->l3.ipv6.hdr = (struct ipv6_hdr){ + .hop_limits = input->flow.udp6_flow.ip.hop_limits, + .proto = input->flow.udp6_flow.ip.proto, + }; + memcpy(attributes->l3.ipv6.hdr.src_addr, + input->flow.udp6_flow.ip.src_ip, + RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); + memcpy(attributes->l3.ipv6.hdr.dst_addr, + input->flow.udp6_flow.ip.dst_ip, + RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); + attributes->l4.udp.hdr = (struct udp_hdr){ + .src_port = input->flow.udp6_flow.src_port, + .dst_port = input->flow.udp6_flow.dst_port, + }; + attributes->items[1] = (struct rte_flow_item){ + .type = RTE_FLOW_ITEM_TYPE_IPV6, + .spec = &attributes->l3, + }; + attributes->items[2] = (struct rte_flow_item){ + .type = RTE_FLOW_ITEM_TYPE_UDP, + .spec = &attributes->l4, + }; + break; + case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: + attributes->l3.ipv6.hdr = (struct ipv6_hdr){ + .hop_limits = input->flow.tcp6_flow.ip.hop_limits, + .proto = input->flow.tcp6_flow.ip.proto, + }; + memcpy(attributes->l3.ipv6.hdr.src_addr, + input->flow.tcp6_flow.ip.src_ip, + RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); + memcpy(attributes->l3.ipv6.hdr.dst_addr, + input->flow.tcp6_flow.ip.dst_ip, + RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); + attributes->l4.tcp.hdr = (struct tcp_hdr){ + .src_port = input->flow.tcp6_flow.src_port, + .dst_port = input->flow.tcp6_flow.dst_port, + }; + attributes->items[1] = (struct rte_flow_item){ + .type = RTE_FLOW_ITEM_TYPE_IPV6, + .spec = &attributes->l3, + }; + attributes->items[2] = (struct rte_flow_item){ + .type = RTE_FLOW_ITEM_TYPE_TCP, + .spec = &attributes->l4, + }; + break; + case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: + attributes->l3.ipv6.hdr = (struct ipv6_hdr){ + .hop_limits = input->flow.ipv6_flow.hop_limits, + .proto = input->flow.ipv6_flow.proto, + }; + memcpy(attributes->l3.ipv6.hdr.src_addr, + input->flow.ipv6_flow.src_ip, + RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); + memcpy(attributes->l3.ipv6.hdr.dst_addr, + input->flow.ipv6_flow.dst_ip, + RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); + attributes->items[1] = (struct rte_flow_item){ + .type = RTE_FLOW_ITEM_TYPE_IPV6, + .spec = &attributes->l3, + }; + break; + default: + ERROR("invalid flow type%d", + fdir_filter->input.flow_type); + return ENOTSUP; + } + return 0; +} + +/** + * Add new flow director filter and store it in list. + * + * @param priv + * Private structure. + * @param fdir_filter + * Flow director filter to add. + * + * @return + * 0 on success, errno value on failure. + */ +static int +priv_fdir_filter_add(struct priv *priv, + const struct rte_eth_fdir_filter *fdir_filter) +{ + struct mlx5_fdir attributes = { + .attr.group = 0, + .l2_mask = { + .dst.addr_bytes = "\x00\x00\x00\x00\x00\x00", + .src.addr_bytes = "\x00\x00\x00\x00\x00\x00", + .type = 0, + }, + }; + struct mlx5_flow_parse parser = { + .layer = HASH_RXQ_ETH, + }; + struct rte_flow_error error; + struct rte_flow *flow; + int ret; + + ret = priv_fdir_filter_convert(priv, fdir_filter, &attributes); + if (ret) + return -ret; + ret = priv_flow_convert(priv, &attributes.attr, attributes.items, + attributes.actions, &error, &parser); + if (ret) + return -ret; + flow = priv_flow_create(priv, + &priv->flows, + &attributes.attr, + attributes.items, + attributes.actions, + &error); + if (flow) { + DEBUG("FDIR created %p", (void *)flow); + return 0; + } + return ENOTSUP; +} + +/** + * Delete specific filter. + * + * @param priv + * Private structure. + * @param fdir_filter + * Filter to be deleted. + * + * @return + * 0 on success, errno value on failure. + */ +static int +priv_fdir_filter_delete(struct priv *priv, + const struct rte_eth_fdir_filter *fdir_filter) +{ + struct mlx5_fdir attributes = { + .attr.group = 0, + }; + struct mlx5_flow_parse parser = { + .create = 1, + .layer = HASH_RXQ_ETH, + }; + struct rte_flow_error error; + struct rte_flow *flow; + unsigned int i; + int ret; + + ret = priv_fdir_filter_convert(priv, fdir_filter, &attributes); + if (ret) + return -ret; + ret = priv_flow_convert(priv, &attributes.attr, attributes.items, + attributes.actions, &error, &parser); + if (ret) + goto exit; + /* + * Special case for drop action which is only set in the + * specifications when the flow is created. In this situation the + * drop specification is missing. + */ + if (parser.drop) { + struct ibv_flow_spec_action_drop *drop; + + drop = (void *)((uintptr_t)parser.queue[HASH_RXQ_ETH].ibv_attr + + parser.queue[HASH_RXQ_ETH].offset); + *drop = (struct ibv_flow_spec_action_drop){ + .type = IBV_FLOW_SPEC_ACTION_DROP, + .size = sizeof(struct ibv_flow_spec_action_drop), + }; + parser.queue[HASH_RXQ_ETH].ibv_attr->num_of_specs++; + } + TAILQ_FOREACH(flow, &priv->flows, next) { + struct ibv_flow_attr *attr; + struct ibv_spec_header *attr_h; + void *spec; + struct ibv_flow_attr *flow_attr; + struct ibv_spec_header *flow_h; + void *flow_spec; + unsigned int specs_n; + + attr = parser.queue[HASH_RXQ_ETH].ibv_attr; + flow_attr = flow->frxq[HASH_RXQ_ETH].ibv_attr; + /* Compare first the attributes. */ + if (memcmp(attr, flow_attr, sizeof(struct ibv_flow_attr))) + continue; + if (attr->num_of_specs == 0) + continue; + spec = (void *)((uintptr_t)attr + + sizeof(struct ibv_flow_attr)); + flow_spec = (void *)((uintptr_t)flow_attr + + sizeof(struct ibv_flow_attr)); + specs_n = RTE_MIN(attr->num_of_specs, flow_attr->num_of_specs); + for (i = 0; i != specs_n; ++i) { + attr_h = spec; + flow_h = flow_spec; + if (memcmp(spec, flow_spec, + RTE_MIN(attr_h->size, flow_h->size))) + goto wrong_flow; + spec = (void *)((uintptr_t)spec + attr_h->size); + flow_spec = (void *)((uintptr_t)flow_spec + + flow_h->size); + } + /* At this point, the flow match. */ + break; +wrong_flow: + /* The flow does not match. */ + continue; + } + if (flow) + priv_flow_destroy(priv, &priv->flows, flow); +exit: + for (i = 0; i != hash_rxq_init_n; ++i) { + if (parser.queue[i].ibv_attr) + rte_free(parser.queue[i].ibv_attr); + } + return -ret; +} + +/** + * Update queue for specific filter. + * + * @param priv + * Private structure. + * @param fdir_filter + * Filter to be updated. + * + * @return + * 0 on success, errno value on failure. + */ +static int +priv_fdir_filter_update(struct priv *priv, + const struct rte_eth_fdir_filter *fdir_filter) +{ + int ret; + + ret = priv_fdir_filter_delete(priv, fdir_filter); + if (ret) + return ret; + ret = priv_fdir_filter_add(priv, fdir_filter); + return ret; +} + +/** + * Flush all filters. + * + * @param priv + * Private structure. + */ +static void +priv_fdir_filter_flush(struct priv *priv) +{ + priv_flow_flush(priv, &priv->flows); +} + +/** + * Get flow director information. + * + * @param priv + * Private structure. + * @param[out] fdir_info + * Resulting flow director information. + */ +static void +priv_fdir_info_get(struct priv *priv, struct rte_eth_fdir_info *fdir_info) +{ + struct rte_eth_fdir_masks *mask = + &priv->dev->data->dev_conf.fdir_conf.mask; + + fdir_info->mode = priv->dev->data->dev_conf.fdir_conf.mode; + fdir_info->guarant_spc = 0; + rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask)); + fdir_info->max_flexpayload = 0; + fdir_info->flow_types_mask[0] = 0; + fdir_info->flex_payload_unit = 0; + fdir_info->max_flex_payload_segment_num = 0; + fdir_info->flex_payload_limit = 0; + memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf)); +} + +/** + * Deal with flow director operations. + * + * @param priv + * Pointer to private structure. + * @param filter_op + * Operation to perform. + * @param arg + * Pointer to operation-specific structure. + * + * @return + * 0 on success, errno value on failure. + */ +static int +priv_fdir_ctrl_func(struct priv *priv, enum rte_filter_op filter_op, void *arg) +{ + enum rte_fdir_mode fdir_mode = + priv->dev->data->dev_conf.fdir_conf.mode; + int ret = 0; + + if (filter_op == RTE_ETH_FILTER_NOP) + return 0; + if (fdir_mode != RTE_FDIR_MODE_PERFECT && + fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) { + ERROR("%p: flow director mode %d not supported", + (void *)priv, fdir_mode); + return EINVAL; + } + switch (filter_op) { + case RTE_ETH_FILTER_ADD: + ret = priv_fdir_filter_add(priv, arg); + break; + case RTE_ETH_FILTER_UPDATE: + ret = priv_fdir_filter_update(priv, arg); + break; + case RTE_ETH_FILTER_DELETE: + ret = priv_fdir_filter_delete(priv, arg); + break; + case RTE_ETH_FILTER_FLUSH: + priv_fdir_filter_flush(priv); + break; + case RTE_ETH_FILTER_INFO: + priv_fdir_info_get(priv, arg); + break; + default: + DEBUG("%p: unknown operation %u", (void *)priv, + filter_op); + ret = EINVAL; + break; + } + return ret; +} + +/** + * Manage filter operations. + * + * @param dev + * Pointer to Ethernet device structure. + * @param filter_type + * Filter type. + * @param filter_op + * Operation to perform. + * @param arg + * Pointer to operation-specific structure. + * + * @return + * 0 on success, negative errno value on failure. + */ +int +mlx5_dev_filter_ctrl(struct rte_eth_dev *dev, + enum rte_filter_type filter_type, + enum rte_filter_op filter_op, + void *arg) +{ + int ret = EINVAL; + struct priv *priv = dev->data->dev_private; + + switch (filter_type) { + case RTE_ETH_FILTER_GENERIC: + if (filter_op != RTE_ETH_FILTER_GET) + return -EINVAL; + *(const void **)arg = &mlx5_flow_ops; + return 0; + case RTE_ETH_FILTER_FDIR: + priv_lock(priv); + ret = priv_fdir_ctrl_func(priv, filter_op, arg); + priv_unlock(priv); + break; + default: + ERROR("%p: filter type (%d) not supported", + (void *)dev, filter_type); + break; + } + return -ret; +} diff --git a/dpdk/drivers/net/mlx5/mlx5_mac.c b/dpdk/drivers/net/mlx5/mlx5_mac.c index 4fcfd3b8..9fb5ba5e 100644 --- a/dpdk/drivers/net/mlx5/mlx5_mac.c +++ b/dpdk/drivers/net/mlx5/mlx5_mac.c @@ -51,16 +51,9 @@ #pragma GCC diagnostic error "-Wpedantic" #endif -/* DPDK headers don't like -pedantic. */ -#ifdef PEDANTIC -#pragma GCC diagnostic ignored "-Wpedantic" -#endif #include #include #include -#ifdef PEDANTIC -#pragma GCC diagnostic error "-Wpedantic" -#endif #include "mlx5.h" #include "mlx5_utils.h" @@ -89,112 +82,6 @@ priv_get_mac(struct priv *priv, uint8_t (*mac)[ETHER_ADDR_LEN]) return 0; } -/** - * Delete MAC flow steering rule. - * - * @param hash_rxq - * Pointer to hash RX queue structure. - * @param mac_index - * MAC address index. - * @param vlan_index - * VLAN index to use. - */ -static void -hash_rxq_del_mac_flow(struct hash_rxq *hash_rxq, unsigned int mac_index, - unsigned int vlan_index) -{ -#ifndef NDEBUG - const uint8_t (*mac)[ETHER_ADDR_LEN] = - (const uint8_t (*)[ETHER_ADDR_LEN]) - hash_rxq->priv->mac[mac_index].addr_bytes; -#endif - - assert(mac_index < RTE_DIM(hash_rxq->mac_flow)); - assert(vlan_index < RTE_DIM(hash_rxq->mac_flow[mac_index])); - if (hash_rxq->mac_flow[mac_index][vlan_index] == NULL) - return; - DEBUG("%p: removing MAC address %02x:%02x:%02x:%02x:%02x:%02x index %u" - " VLAN index %u", - (void *)hash_rxq, - (*mac)[0], (*mac)[1], (*mac)[2], (*mac)[3], (*mac)[4], (*mac)[5], - mac_index, - vlan_index); - claim_zero(ibv_exp_destroy_flow(hash_rxq->mac_flow - [mac_index][vlan_index])); - hash_rxq->mac_flow[mac_index][vlan_index] = NULL; -} - -/** - * Unregister a MAC address from a hash RX queue. - * - * @param hash_rxq - * Pointer to hash RX queue structure. - * @param mac_index - * MAC address index. - */ -static void -hash_rxq_mac_addr_del(struct hash_rxq *hash_rxq, unsigned int mac_index) -{ - unsigned int i; - - assert(mac_index < RTE_DIM(hash_rxq->mac_flow)); - for (i = 0; (i != RTE_DIM(hash_rxq->mac_flow[mac_index])); ++i) - hash_rxq_del_mac_flow(hash_rxq, mac_index, i); -} - -/** - * Unregister all MAC addresses from a hash RX queue. - * - * @param hash_rxq - * Pointer to hash RX queue structure. - */ -void -hash_rxq_mac_addrs_del(struct hash_rxq *hash_rxq) -{ - unsigned int i; - - for (i = 0; (i != RTE_DIM(hash_rxq->mac_flow)); ++i) - hash_rxq_mac_addr_del(hash_rxq, i); -} - -/** - * Unregister a MAC address. - * - * This is done for each hash RX queue. - * - * @param priv - * Pointer to private structure. - * @param mac_index - * MAC address index. - */ -static void -priv_mac_addr_del(struct priv *priv, unsigned int mac_index) -{ - unsigned int i; - - assert(mac_index < RTE_DIM(priv->mac)); - if (!BITFIELD_ISSET(priv->mac_configured, mac_index)) - return; - for (i = 0; (i != priv->hash_rxqs_n); ++i) - hash_rxq_mac_addr_del(&(*priv->hash_rxqs)[i], mac_index); - BITFIELD_RESET(priv->mac_configured, mac_index); -} - -/** - * Unregister all MAC addresses from all hash RX queues. - * - * @param priv - * Pointer to private structure. - */ -void -priv_mac_addrs_disable(struct priv *priv) -{ - unsigned int i; - - for (i = 0; (i != priv->hash_rxqs_n); ++i) - hash_rxq_mac_addrs_del(&(*priv->hash_rxqs)[i]); -} - /** * DPDK callback to remove a MAC address. * @@ -206,256 +93,10 @@ priv_mac_addrs_disable(struct priv *priv) void mlx5_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index) { - struct priv *priv = dev->data->dev_private; - - if (mlx5_is_secondary()) - return; - - priv_lock(priv); - DEBUG("%p: removing MAC address from index %" PRIu32, - (void *)dev, index); - if (index >= RTE_DIM(priv->mac)) - goto end; - priv_mac_addr_del(priv, index); -end: - priv_unlock(priv); -} - -/** - * Add MAC flow steering rule. - * - * @param hash_rxq - * Pointer to hash RX queue structure. - * @param mac_index - * MAC address index to register. - * @param vlan_index - * VLAN index to use. - * - * @return - * 0 on success, errno value on failure. - */ -static int -hash_rxq_add_mac_flow(struct hash_rxq *hash_rxq, unsigned int mac_index, - unsigned int vlan_index) -{ - struct ibv_exp_flow *flow; - struct priv *priv = hash_rxq->priv; - const uint8_t (*mac)[ETHER_ADDR_LEN] = - (const uint8_t (*)[ETHER_ADDR_LEN]) - priv->mac[mac_index].addr_bytes; - FLOW_ATTR_SPEC_ETH(data, priv_flow_attr(priv, NULL, 0, hash_rxq->type)); - struct ibv_exp_flow_attr *attr = &data->attr; - struct ibv_exp_flow_spec_eth *spec = &data->spec; - unsigned int vlan_enabled = !!priv->vlan_filter_n; - unsigned int vlan_id = priv->vlan_filter[vlan_index]; - - assert(mac_index < RTE_DIM(hash_rxq->mac_flow)); - assert(vlan_index < RTE_DIM(hash_rxq->mac_flow[mac_index])); - if (hash_rxq->mac_flow[mac_index][vlan_index] != NULL) - return 0; - /* - * No padding must be inserted by the compiler between attr and spec. - * This layout is expected by libibverbs. - */ - assert(((uint8_t *)attr + sizeof(*attr)) == (uint8_t *)spec); - priv_flow_attr(priv, attr, sizeof(data), hash_rxq->type); - /* The first specification must be Ethernet. */ - assert(spec->type == IBV_EXP_FLOW_SPEC_ETH); - assert(spec->size == sizeof(*spec)); - *spec = (struct ibv_exp_flow_spec_eth){ - .type = IBV_EXP_FLOW_SPEC_ETH, - .size = sizeof(*spec), - .val = { - .dst_mac = { - (*mac)[0], (*mac)[1], (*mac)[2], - (*mac)[3], (*mac)[4], (*mac)[5] - }, - .vlan_tag = (vlan_enabled ? htons(vlan_id) : 0), - }, - .mask = { - .dst_mac = "\xff\xff\xff\xff\xff\xff", - .vlan_tag = (vlan_enabled ? htons(0xfff) : 0), - }, - }; - DEBUG("%p: adding MAC address %02x:%02x:%02x:%02x:%02x:%02x index %u" - " VLAN index %u filtering %s, ID %u", - (void *)hash_rxq, - (*mac)[0], (*mac)[1], (*mac)[2], (*mac)[3], (*mac)[4], (*mac)[5], - mac_index, - vlan_index, - (vlan_enabled ? "enabled" : "disabled"), - vlan_id); - /* Create related flow. */ - errno = 0; - flow = ibv_exp_create_flow(hash_rxq->qp, attr); - if (flow == NULL) { - /* It's not clear whether errno is always set in this case. */ - ERROR("%p: flow configuration failed, errno=%d: %s", - (void *)hash_rxq, errno, - (errno ? strerror(errno) : "Unknown error")); - if (errno) - return errno; - return EINVAL; - } - hash_rxq->mac_flow[mac_index][vlan_index] = flow; - return 0; -} - -/** - * Register a MAC address in a hash RX queue. - * - * @param hash_rxq - * Pointer to hash RX queue structure. - * @param mac_index - * MAC address index to register. - * - * @return - * 0 on success, errno value on failure. - */ -static int -hash_rxq_mac_addr_add(struct hash_rxq *hash_rxq, unsigned int mac_index) -{ - struct priv *priv = hash_rxq->priv; - unsigned int i = 0; - int ret; - - assert(mac_index < RTE_DIM(hash_rxq->mac_flow)); - assert(RTE_DIM(hash_rxq->mac_flow[mac_index]) == - RTE_DIM(priv->vlan_filter)); - /* Add a MAC address for each VLAN filter, or at least once. */ - do { - ret = hash_rxq_add_mac_flow(hash_rxq, mac_index, i); - if (ret) { - /* Failure, rollback. */ - while (i != 0) - hash_rxq_del_mac_flow(hash_rxq, mac_index, - --i); - return ret; - } - } while (++i < priv->vlan_filter_n); - return 0; -} - -/** - * Register all MAC addresses in a hash RX queue. - * - * @param hash_rxq - * Pointer to hash RX queue structure. - * - * @return - * 0 on success, errno value on failure. - */ -int -hash_rxq_mac_addrs_add(struct hash_rxq *hash_rxq) -{ - struct priv *priv = hash_rxq->priv; - unsigned int i; - int ret; - - assert(RTE_DIM(priv->mac) == RTE_DIM(hash_rxq->mac_flow)); - for (i = 0; (i != RTE_DIM(priv->mac)); ++i) { - if (!BITFIELD_ISSET(priv->mac_configured, i)) - continue; - ret = hash_rxq_mac_addr_add(hash_rxq, i); - if (!ret) - continue; - /* Failure, rollback. */ - while (i != 0) - hash_rxq_mac_addr_del(hash_rxq, --i); - assert(ret > 0); - return ret; - } - return 0; -} - -/** - * Register a MAC address. - * - * This is done for each hash RX queue. - * - * @param priv - * Pointer to private structure. - * @param mac_index - * MAC address index to use. - * @param mac - * MAC address to register. - * - * @return - * 0 on success, errno value on failure. - */ -int -priv_mac_addr_add(struct priv *priv, unsigned int mac_index, - const uint8_t (*mac)[ETHER_ADDR_LEN]) -{ - unsigned int i; - int ret; - - assert(mac_index < RTE_DIM(priv->mac)); - /* First, make sure this address isn't already configured. */ - for (i = 0; (i != RTE_DIM(priv->mac)); ++i) { - /* Skip this index, it's going to be reconfigured. */ - if (i == mac_index) - continue; - if (!BITFIELD_ISSET(priv->mac_configured, i)) - continue; - if (memcmp(priv->mac[i].addr_bytes, *mac, sizeof(*mac))) - continue; - /* Address already configured elsewhere, return with error. */ - return EADDRINUSE; - } - if (BITFIELD_ISSET(priv->mac_configured, mac_index)) - priv_mac_addr_del(priv, mac_index); - priv->mac[mac_index] = (struct ether_addr){ - { - (*mac)[0], (*mac)[1], (*mac)[2], - (*mac)[3], (*mac)[4], (*mac)[5] - } - }; - if (!priv_allow_flow_type(priv, HASH_RXQ_FLOW_TYPE_MAC)) - goto end; - for (i = 0; (i != priv->hash_rxqs_n); ++i) { - ret = hash_rxq_mac_addr_add(&(*priv->hash_rxqs)[i], mac_index); - if (!ret) - continue; - /* Failure, rollback. */ - while (i != 0) - hash_rxq_mac_addr_del(&(*priv->hash_rxqs)[--i], - mac_index); - return ret; - } -end: - BITFIELD_SET(priv->mac_configured, mac_index); - return 0; -} - -/** - * Register all MAC addresses in all hash RX queues. - * - * @param priv - * Pointer to private structure. - * - * @return - * 0 on success, errno value on failure. - */ -int -priv_mac_addrs_enable(struct priv *priv) -{ - unsigned int i; - int ret; - - if (!priv_allow_flow_type(priv, HASH_RXQ_FLOW_TYPE_MAC)) - return 0; - for (i = 0; (i != priv->hash_rxqs_n); ++i) { - ret = hash_rxq_mac_addrs_add(&(*priv->hash_rxqs)[i]); - if (!ret) - continue; - /* Failure, rollback. */ - while (i != 0) - hash_rxq_mac_addrs_del(&(*priv->hash_rxqs)[--i]); - assert(ret > 0); - return ret; - } - return 0; + assert(index < MLX5_MAX_MAC_ADDRESSES); + memset(&dev->data->mac_addrs[index], 0, sizeof(struct ether_addr)); + if (!dev->data->promiscuous && !dev->data->all_multicast) + mlx5_traffic_restart(dev); } /** @@ -469,27 +110,33 @@ priv_mac_addrs_enable(struct priv *priv) * MAC address index. * @param vmdq * VMDq pool index to associate address with (ignored). + * + * @return + * 0 on success. */ -void -mlx5_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac_addr, +int +mlx5_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac, uint32_t index, uint32_t vmdq) { - struct priv *priv = dev->data->dev_private; - - if (mlx5_is_secondary()) - return; + unsigned int i; + int ret = 0; (void)vmdq; - priv_lock(priv); - DEBUG("%p: adding MAC address at index %" PRIu32, - (void *)dev, index); - if (index >= RTE_DIM(priv->mac)) - goto end; - priv_mac_addr_add(priv, index, - (const uint8_t (*)[ETHER_ADDR_LEN]) - mac_addr->addr_bytes); -end: - priv_unlock(priv); + assert(index < MLX5_MAX_MAC_ADDRESSES); + /* First, make sure this address isn't already configured. */ + for (i = 0; (i != MLX5_MAX_MAC_ADDRESSES); ++i) { + /* Skip this index, it's going to be reconfigured. */ + if (i == index) + continue; + if (memcmp(&dev->data->mac_addrs[i], mac, sizeof(*mac))) + continue; + /* Address already configured elsewhere, return with error. */ + return EADDRINUSE; + } + dev->data->mac_addrs[index] = *mac; + if (!dev->data->promiscuous && !dev->data->all_multicast) + mlx5_traffic_restart(dev); + return ret; } /** @@ -504,6 +151,5 @@ void mlx5_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr) { DEBUG("%p: setting primary MAC address", (void *)dev); - mlx5_mac_addr_remove(dev, 0); mlx5_mac_addr_add(dev, mac_addr, 0, 0); } diff --git a/dpdk/drivers/net/mlx5/mlx5_mr.c b/dpdk/drivers/net/mlx5/mlx5_mr.c index 0a363846..2776dc70 100644 --- a/dpdk/drivers/net/mlx5/mlx5_mr.c +++ b/dpdk/drivers/net/mlx5/mlx5_mr.c @@ -41,14 +41,8 @@ #pragma GCC diagnostic error "-Wpedantic" #endif -/* DPDK headers don't like -pedantic. */ -#ifdef PEDANTIC -#pragma GCC diagnostic ignored "-Wpedantic" -#endif #include -#ifdef PEDANTIC -#pragma GCC diagnostic error "-Wpedantic" -#endif +#include #include "mlx5.h" #include "mlx5_rxtx.h" @@ -118,51 +112,57 @@ static int mlx5_check_mempool(struct rte_mempool *mp, uintptr_t *start, } /** - * Register mempool as a memory region. + * Register a Memory Region (MR) <-> Memory Pool (MP) association in + * txq->mp2mr[]. If mp2mr[] is full, remove an entry first. * - * @param pd - * Pointer to protection domain. - * @param mp - * Pointer to memory pool. + * This function should only be called by txq_mp2mr(). + * + * @param priv + * Pointer to private structure. + * @param txq + * Pointer to TX queue structure. + * @param[in] mp + * Memory Pool for which a Memory Region lkey must be returned. + * @param idx + * Index of the next available entry. * * @return - * Memory region pointer, NULL in case of error. + * mr on success, NULL on failure. */ -struct ibv_mr * -mlx5_mp2mr(struct ibv_pd *pd, struct rte_mempool *mp) +struct mlx5_mr* +priv_txq_mp2mr_reg(struct priv *priv, struct mlx5_txq_data *txq, + struct rte_mempool *mp, unsigned int idx) { - const struct rte_memseg *ms = rte_eal_get_physmem_layout(); - uintptr_t start; - uintptr_t end; - unsigned int i; + struct mlx5_txq_ctrl *txq_ctrl = + container_of(txq, struct mlx5_txq_ctrl, txq); + struct mlx5_mr *mr; - if (mlx5_check_mempool(mp, &start, &end) != 0) { - ERROR("mempool %p: not virtually contiguous", - (void *)mp); + /* Add a new entry, register MR first. */ + DEBUG("%p: discovered new memory pool \"%s\" (%p)", + (void *)txq_ctrl, mp->name, (void *)mp); + mr = priv_mr_get(priv, mp); + if (mr == NULL) + mr = priv_mr_new(priv, mp); + if (unlikely(mr == NULL)) { + DEBUG("%p: unable to configure MR, ibv_reg_mr() failed.", + (void *)txq_ctrl); return NULL; } - - DEBUG("mempool %p area start=%p end=%p size=%zu", - (void *)mp, (void *)start, (void *)end, - (size_t)(end - start)); - /* Round start and end to page boundary if found in memory segments. */ - for (i = 0; (i < RTE_MAX_MEMSEG) && (ms[i].addr != NULL); ++i) { - uintptr_t addr = (uintptr_t)ms[i].addr; - size_t len = ms[i].len; - unsigned int align = ms[i].hugepage_sz; - - if ((start > addr) && (start < addr + len)) - start = RTE_ALIGN_FLOOR(start, align); - if ((end > addr) && (end < addr + len)) - end = RTE_ALIGN_CEIL(end, align); + if (unlikely(idx == RTE_DIM(txq->mp2mr))) { + /* Table is full, remove oldest entry. */ + DEBUG("%p: MR <-> MP table full, dropping oldest entry.", + (void *)txq_ctrl); + --idx; + priv_mr_release(priv, txq->mp2mr[0]); + memmove(&txq->mp2mr[0], &txq->mp2mr[1], + (sizeof(txq->mp2mr) - sizeof(txq->mp2mr[0]))); } - DEBUG("mempool %p using start=%p end=%p size=%zu for MR", - (void *)mp, (void *)start, (void *)end, - (size_t)(end - start)); - return ibv_reg_mr(pd, - (void *)start, - end - start, - IBV_ACCESS_LOCAL_WRITE); + /* Store the new entry. */ + txq_ctrl->txq.mp2mr[idx] = mr; + DEBUG("%p: new MR lkey for MP \"%s\" (%p): 0x%08" PRIu32, + (void *)txq_ctrl, mp->name, (void *)mp, + txq_ctrl->txq.mp2mr[idx]->lkey); + return mr; } /** @@ -179,44 +179,23 @@ mlx5_mp2mr(struct ibv_pd *pd, struct rte_mempool *mp) * Index of the next available entry. * * @return - * mr->lkey on success, (uint32_t)-1 on failure. + * mr on success, NULL on failure. */ -uint32_t -txq_mp2mr_reg(struct txq *txq, struct rte_mempool *mp, unsigned int idx) +struct mlx5_mr* +mlx5_txq_mp2mr_reg(struct mlx5_txq_data *txq, struct rte_mempool *mp, + unsigned int idx) { - struct txq_ctrl *txq_ctrl = container_of(txq, struct txq_ctrl, txq); - struct ibv_mr *mr; + struct mlx5_txq_ctrl *txq_ctrl = + container_of(txq, struct mlx5_txq_ctrl, txq); + struct mlx5_mr *mr; - /* Add a new entry, register MR first. */ - DEBUG("%p: discovered new memory pool \"%s\" (%p)", - (void *)txq_ctrl, mp->name, (void *)mp); - mr = mlx5_mp2mr(txq_ctrl->priv->pd, mp); - if (unlikely(mr == NULL)) { - DEBUG("%p: unable to configure MR, ibv_reg_mr() failed.", - (void *)txq_ctrl); - return (uint32_t)-1; - } - if (unlikely(idx == RTE_DIM(txq_ctrl->txq.mp2mr))) { - /* Table is full, remove oldest entry. */ - DEBUG("%p: MR <-> MP table full, dropping oldest entry.", - (void *)txq_ctrl); - --idx; - claim_zero(ibv_dereg_mr(txq_ctrl->txq.mp2mr[0].mr)); - memmove(&txq_ctrl->txq.mp2mr[0], &txq_ctrl->txq.mp2mr[1], - (sizeof(txq_ctrl->txq.mp2mr) - - sizeof(txq_ctrl->txq.mp2mr[0]))); - } - /* Store the new entry. */ - txq_ctrl->txq.mp2mr[idx].mp = mp; - txq_ctrl->txq.mp2mr[idx].mr = mr; - txq_ctrl->txq.mp2mr[idx].lkey = htonl(mr->lkey); - DEBUG("%p: new MR lkey for MP \"%s\" (%p): 0x%08" PRIu32, - (void *)txq_ctrl, mp->name, (void *)mp, - txq_ctrl->txq.mp2mr[idx].lkey); - return txq_ctrl->txq.mp2mr[idx].lkey; + priv_lock(txq_ctrl->priv); + mr = priv_txq_mp2mr_reg(txq_ctrl->priv, txq, mp, idx); + priv_unlock(txq_ctrl->priv); + return mr; } -struct txq_mp2mr_mbuf_check_data { +struct mlx5_mp2mr_mbuf_check_data { int ret; }; @@ -238,7 +217,7 @@ static void txq_mp2mr_mbuf_check(struct rte_mempool *mp, void *arg, void *obj, uint32_t index __rte_unused) { - struct txq_mp2mr_mbuf_check_data *data = arg; + struct mlx5_mp2mr_mbuf_check_data *data = arg; struct rte_mbuf *buf = obj; /* @@ -259,25 +238,159 @@ txq_mp2mr_mbuf_check(struct rte_mempool *mp, void *arg, void *obj, * Pointer to TX queue structure. */ void -txq_mp2mr_iter(struct rte_mempool *mp, void *arg) +mlx5_mp2mr_iter(struct rte_mempool *mp, void *arg) { - struct txq_ctrl *txq_ctrl = arg; - struct txq_mp2mr_mbuf_check_data data = { + struct priv *priv = (struct priv *)arg; + struct mlx5_mp2mr_mbuf_check_data data = { .ret = 0, }; - unsigned int i; + struct mlx5_mr *mr; /* Register mempool only if the first element looks like a mbuf. */ if (rte_mempool_obj_iter(mp, txq_mp2mr_mbuf_check, &data) == 0 || data.ret == -1) return; - for (i = 0; (i != RTE_DIM(txq_ctrl->txq.mp2mr)); ++i) { - if (unlikely(txq_ctrl->txq.mp2mr[i].mp == NULL)) { - /* Unknown MP, add a new MR for it. */ - break; - } - if (txq_ctrl->txq.mp2mr[i].mp == mp) - return; + mr = priv_mr_get(priv, mp); + if (mr) { + priv_mr_release(priv, mr); + return; } - txq_mp2mr_reg(&txq_ctrl->txq, mp, i); + priv_mr_new(priv, mp); +} + +/** + * Register a new memory region from the mempool and store it in the memory + * region list. + * + * @param priv + * Pointer to private structure. + * @param mp + * Pointer to the memory pool to register. + * @return + * The memory region on success. + */ +struct mlx5_mr* +priv_mr_new(struct priv *priv, struct rte_mempool *mp) +{ + const struct rte_memseg *ms = rte_eal_get_physmem_layout(); + uintptr_t start; + uintptr_t end; + unsigned int i; + struct mlx5_mr *mr; + + mr = rte_zmalloc_socket(__func__, sizeof(*mr), 0, mp->socket_id); + if (!mr) { + DEBUG("unable to configure MR, ibv_reg_mr() failed."); + return NULL; + } + if (mlx5_check_mempool(mp, &start, &end) != 0) { + ERROR("mempool %p: not virtually contiguous", + (void *)mp); + return NULL; + } + DEBUG("mempool %p area start=%p end=%p size=%zu", + (void *)mp, (void *)start, (void *)end, + (size_t)(end - start)); + /* Save original addresses for exact MR lookup. */ + mr->start = start; + mr->end = end; + /* Round start and end to page boundary if found in memory segments. */ + for (i = 0; (i < RTE_MAX_MEMSEG) && (ms[i].addr != NULL); ++i) { + uintptr_t addr = (uintptr_t)ms[i].addr; + size_t len = ms[i].len; + unsigned int align = ms[i].hugepage_sz; + + if ((start > addr) && (start < addr + len)) + start = RTE_ALIGN_FLOOR(start, align); + if ((end > addr) && (end < addr + len)) + end = RTE_ALIGN_CEIL(end, align); + } + DEBUG("mempool %p using start=%p end=%p size=%zu for MR", + (void *)mp, (void *)start, (void *)end, + (size_t)(end - start)); + mr->mr = ibv_reg_mr(priv->pd, (void *)start, end - start, + IBV_ACCESS_LOCAL_WRITE); + mr->mp = mp; + mr->lkey = rte_cpu_to_be_32(mr->mr->lkey); + rte_atomic32_inc(&mr->refcnt); + DEBUG("%p: new Memory Region %p refcnt: %d", (void *)priv, + (void *)mr, rte_atomic32_read(&mr->refcnt)); + LIST_INSERT_HEAD(&priv->mr, mr, next); + return mr; +} + +/** + * Search the memory region object in the memory region list. + * + * @param priv + * Pointer to private structure. + * @param mp + * Pointer to the memory pool to register. + * @return + * The memory region on success. + */ +struct mlx5_mr* +priv_mr_get(struct priv *priv, struct rte_mempool *mp) +{ + struct mlx5_mr *mr; + + assert(mp); + if (LIST_EMPTY(&priv->mr)) + return NULL; + LIST_FOREACH(mr, &priv->mr, next) { + if (mr->mp == mp) { + rte_atomic32_inc(&mr->refcnt); + DEBUG("Memory Region %p refcnt: %d", + (void *)mr, rte_atomic32_read(&mr->refcnt)); + return mr; + } + } + return NULL; +} + +/** + * Release the memory region object. + * + * @param mr + * Pointer to memory region to release. + * + * @return + * 0 on success, errno on failure. + */ +int +priv_mr_release(struct priv *priv, struct mlx5_mr *mr) +{ + (void)priv; + assert(mr); + DEBUG("Memory Region %p refcnt: %d", + (void *)mr, rte_atomic32_read(&mr->refcnt)); + if (rte_atomic32_dec_and_test(&mr->refcnt)) { + claim_zero(ibv_dereg_mr(mr->mr)); + LIST_REMOVE(mr, next); + rte_free(mr); + return 0; + } + return EBUSY; +} + +/** + * Verify the flow list is empty + * + * @param priv + * Pointer to private structure. + * + * @return the number of object not released. + */ +int +priv_mr_verify(struct priv *priv) +{ + int ret = 0; + struct mlx5_mr *mr; + + LIST_FOREACH(mr, &priv->mr, next) { + DEBUG("%p: mr %p still referenced", (void *)priv, + (void *)mr); + ++ret; + } + return ret; } diff --git a/dpdk/drivers/net/mlx5/mlx5_prm.h b/dpdk/drivers/net/mlx5/mlx5_prm.h index 1c369cac..2de310bc 100644 --- a/dpdk/drivers/net/mlx5/mlx5_prm.h +++ b/dpdk/drivers/net/mlx5/mlx5_prm.h @@ -34,16 +34,19 @@ #ifndef RTE_PMD_MLX5_PRM_H_ #define RTE_PMD_MLX5_PRM_H_ +#include + /* Verbs header. */ /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ #ifdef PEDANTIC #pragma GCC diagnostic ignored "-Wpedantic" #endif -#include +#include #ifdef PEDANTIC #pragma GCC diagnostic error "-Wpedantic" #endif +#include #include "mlx5_autoconf.h" /* Get CQE owner bit. */ @@ -61,42 +64,95 @@ /* Invalidate a CQE. */ #define MLX5_CQE_INVALIDATE (MLX5_CQE_INVALID << 4) -/* CQE value to inform that VLAN is stripped. */ -#define MLX5_CQE_VLAN_STRIPPED 0x1 - /* Maximum number of packets a multi-packet WQE can handle. */ #define MLX5_MPW_DSEG_MAX 5 -/* Room for inline data in regular work queue element. */ -#define MLX5_WQE64_INL_DATA 12 +/* WQE DWORD size */ +#define MLX5_WQE_DWORD_SIZE 16 + +/* WQE size */ +#define MLX5_WQE_SIZE (4 * MLX5_WQE_DWORD_SIZE) + +/* Max size of a WQE session. */ +#define MLX5_WQE_SIZE_MAX 960U + +/* Compute the number of DS. */ +#define MLX5_WQE_DS(n) \ + (((n) + MLX5_WQE_DWORD_SIZE - 1) / MLX5_WQE_DWORD_SIZE) /* Room for inline data in multi-packet WQE. */ #define MLX5_MWQE64_INL_DATA 28 -#ifndef HAVE_VERBS_MLX5_OPCODE_TSO -#define MLX5_OPCODE_TSO MLX5_OPCODE_LSO_MPW /* Compat with OFED 3.3. */ -#endif +/* Default minimum number of Tx queues for inlining packets. */ +#define MLX5_EMPW_MIN_TXQS 8 -/* IPv4 packet. */ -#define MLX5_CQE_RX_IPV4_PACKET (1u << 2) +/* Default max packet length to be inlined. */ +#define MLX5_EMPW_MAX_INLINE_LEN (4U * MLX5_WQE_SIZE) + + +#define MLX5_OPC_MOD_ENHANCED_MPSW 0 +#define MLX5_OPCODE_ENHANCED_MPSW 0x29 + +/* CQE value to inform that VLAN is stripped. */ +#define MLX5_CQE_VLAN_STRIPPED (1u << 0) + +/* IPv4 options. */ +#define MLX5_CQE_RX_IP_EXT_OPTS_PACKET (1u << 1) /* IPv6 packet. */ -#define MLX5_CQE_RX_IPV6_PACKET (1u << 3) +#define MLX5_CQE_RX_IPV6_PACKET (1u << 2) -/* Outer IPv4 packet. */ -#define MLX5_CQE_RX_OUTER_IPV4_PACKET (1u << 7) +/* IPv4 packet. */ +#define MLX5_CQE_RX_IPV4_PACKET (1u << 3) -/* Outer IPv6 packet. */ -#define MLX5_CQE_RX_OUTER_IPV6_PACKET (1u << 8) +/* TCP packet. */ +#define MLX5_CQE_RX_TCP_PACKET (1u << 4) + +/* UDP packet. */ +#define MLX5_CQE_RX_UDP_PACKET (1u << 5) + +/* IP is fragmented. */ +#define MLX5_CQE_RX_IP_FRAG_PACKET (1u << 7) + +/* L2 header is valid. */ +#define MLX5_CQE_RX_L2_HDR_VALID (1u << 8) + +/* L3 header is valid. */ +#define MLX5_CQE_RX_L3_HDR_VALID (1u << 9) + +/* L4 header is valid. */ +#define MLX5_CQE_RX_L4_HDR_VALID (1u << 10) + +/* Outer packet, 0 IPv4, 1 IPv6. */ +#define MLX5_CQE_RX_OUTER_PACKET (1u << 1) /* Tunnel packet bit in the CQE. */ -#define MLX5_CQE_RX_TUNNEL_PACKET (1u << 4) +#define MLX5_CQE_RX_TUNNEL_PACKET (1u << 0) -/* Outer IP checksum OK. */ -#define MLX5_CQE_RX_OUTER_IP_CSUM_OK (1u << 5) +/* Inner L3 checksum offload (Tunneled packets only). */ +#define MLX5_ETH_WQE_L3_INNER_CSUM (1u << 4) -/* Outer UDP header and checksum OK. */ -#define MLX5_CQE_RX_OUTER_TCP_UDP_CSUM_OK (1u << 6) +/* Inner L4 checksum offload (Tunneled packets only). */ +#define MLX5_ETH_WQE_L4_INNER_CSUM (1u << 5) + +/* Is flow mark valid. */ +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN +#define MLX5_FLOW_MARK_IS_VALID(val) ((val) & 0xffffff00) +#else +#define MLX5_FLOW_MARK_IS_VALID(val) ((val) & 0xffffff) +#endif + +/* INVALID is used by packets matching no flow rules. */ +#define MLX5_FLOW_MARK_INVALID 0 + +/* Maximum allowed value to mark a packet. */ +#define MLX5_FLOW_MARK_MAX 0xfffff0 + +/* Default mark value used when none is provided. */ +#define MLX5_FLOW_MARK_DEFAULT 0xffffff + +/* Maximum number of DS in WQE. */ +#define MLX5_DSEG_MAX 63 /* Subset of struct mlx5_wqe_eth_seg. */ struct mlx5_wqe_eth_seg_small { @@ -106,63 +162,51 @@ struct mlx5_wqe_eth_seg_small { uint16_t mss; uint32_t rsvd2; uint16_t inline_hdr_sz; + uint8_t inline_hdr[2]; +} __rte_aligned(MLX5_WQE_DWORD_SIZE); + +struct mlx5_wqe_inl_small { + uint32_t byte_cnt; + uint8_t raw; +} __rte_aligned(MLX5_WQE_DWORD_SIZE); + +struct mlx5_wqe_ctrl { + uint32_t ctrl0; + uint32_t ctrl1; + uint32_t ctrl2; + uint32_t ctrl3; +} __rte_aligned(MLX5_WQE_DWORD_SIZE); + +/* Small common part of the WQE. */ +struct mlx5_wqe { + uint32_t ctrl[4]; + struct mlx5_wqe_eth_seg_small eseg; }; -/* Regular WQE. */ -struct mlx5_wqe_regular { - union { - struct mlx5_wqe_ctrl_seg ctrl; - uint32_t data[4]; - } ctrl; - struct mlx5_wqe_eth_seg eseg; - struct mlx5_wqe_data_seg dseg; -} __rte_aligned(64); +/* Vectorize WQE header. */ +struct mlx5_wqe_v { + rte_v128u32_t ctrl; + rte_v128u32_t eseg; +}; -/* Inline WQE. */ -struct mlx5_wqe_inl { - union { - struct mlx5_wqe_ctrl_seg ctrl; - uint32_t data[4]; - } ctrl; - struct mlx5_wqe_eth_seg eseg; - uint32_t byte_cnt; - uint8_t data[MLX5_WQE64_INL_DATA]; -} __rte_aligned(64); +/* WQE. */ +struct mlx5_wqe64 { + struct mlx5_wqe hdr; + uint8_t raw[32]; +} __rte_aligned(MLX5_WQE_SIZE); -/* Multi-packet WQE. */ -struct mlx5_wqe_mpw { - union { - struct mlx5_wqe_ctrl_seg ctrl; - uint32_t data[4]; - } ctrl; - struct mlx5_wqe_eth_seg_small eseg; - struct mlx5_wqe_data_seg dseg[2]; -} __rte_aligned(64); - -/* Multi-packet WQE with inline. */ -struct mlx5_wqe_mpw_inl { - union { - struct mlx5_wqe_ctrl_seg ctrl; - uint32_t data[4]; - } ctrl; - struct mlx5_wqe_eth_seg_small eseg; - uint32_t byte_cnt; - uint8_t data[MLX5_MWQE64_INL_DATA]; -} __rte_aligned(64); - -/* Union of all WQE types. */ -union mlx5_wqe { - struct mlx5_wqe_regular wqe; - struct mlx5_wqe_inl inl; - struct mlx5_wqe_mpw mpw; - struct mlx5_wqe_mpw_inl mpw_inl; - uint8_t data[64]; +/* MPW mode. */ +enum mlx5_mpw_mode { + MLX5_MPW_DISABLED, + MLX5_MPW, + MLX5_MPW_ENHANCED, /* Enhanced Multi-Packet Send WQE, a.k.a MPWv2. */ }; /* MPW session status. */ enum mlx5_mpw_state { MLX5_MPW_STATE_OPENED, MLX5_MPW_INL_STATE_OPENED, + MLX5_MPW_ENHANCED_STATE_OPENED, MLX5_MPW_STATE_CLOSED, }; @@ -172,7 +216,7 @@ struct mlx5_mpw { unsigned int pkts_n; unsigned int len; unsigned int total_len; - volatile union mlx5_wqe *wqe; + volatile struct mlx5_wqe *wqe; union { volatile struct mlx5_wqe_data_seg *dseg[MLX5_MPW_DSEG_MAX]; volatile uint8_t *raw; @@ -184,7 +228,118 @@ struct mlx5_cqe { #if (RTE_CACHE_LINE_SIZE == 128) uint8_t padding[64]; #endif - struct mlx5_cqe64 cqe64; + uint8_t pkt_info; + uint8_t rsvd0[11]; + uint32_t rx_hash_res; + uint8_t rx_hash_type; + uint8_t rsvd1[11]; + uint16_t hdr_type_etc; + uint16_t vlan_info; + uint8_t rsvd2[12]; + uint32_t byte_cnt; + uint64_t timestamp; + uint32_t sop_drop_qpn; + uint16_t wqe_counter; + uint8_t rsvd4; + uint8_t op_own; }; +/* Adding direct verbs to data-path. */ + +/* CQ sequence number mask. */ +#define MLX5_CQ_SQN_MASK 0x3 + +/* CQ sequence number index. */ +#define MLX5_CQ_SQN_OFFSET 28 + +/* CQ doorbell index mask. */ +#define MLX5_CI_MASK 0xffffff + +/* CQ doorbell offset. */ +#define MLX5_CQ_ARM_DB 1 + +/* CQ doorbell offset*/ +#define MLX5_CQ_DOORBELL 0x20 + +/* CQE format value. */ +#define MLX5_COMPRESSED 0x3 + +/* CQE format mask. */ +#define MLX5E_CQE_FORMAT_MASK 0xc + +/* MPW opcode. */ +#define MLX5_OPC_MOD_MPW 0x01 + +/* Compressed Rx CQE structure. */ +struct mlx5_mini_cqe8 { + union { + uint32_t rx_hash_result; + uint32_t checksum; + struct { + uint16_t wqe_counter; + uint8_t s_wqe_opcode; + uint8_t reserved; + } s_wqe_info; + }; + uint32_t byte_cnt; +}; + +/** + * Convert a user mark to flow mark. + * + * @param val + * Mark value to convert. + * + * @return + * Converted mark value. + */ +static inline uint32_t +mlx5_flow_mark_set(uint32_t val) +{ + uint32_t ret; + + /* + * Add one to the user value to differentiate un-marked flows from + * marked flows, if the ID is equal to MLX5_FLOW_MARK_DEFAULT it + * remains untouched. + */ + if (val != MLX5_FLOW_MARK_DEFAULT) + ++val; +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + /* + * Mark is 24 bits (minus reserved values) but is stored on a 32 bit + * word, byte-swapped by the kernel on little-endian systems. In this + * case, left-shifting the resulting big-endian value ensures the + * least significant 24 bits are retained when converting it back. + */ + ret = rte_cpu_to_be_32(val) >> 8; +#else + ret = val; +#endif + return ret; +} + +/** + * Convert a mark to user mark. + * + * @param val + * Mark value to convert. + * + * @return + * Converted mark value. + */ +static inline uint32_t +mlx5_flow_mark_get(uint32_t val) +{ + /* + * Subtract one from the retrieved value. It was added by + * mlx5_flow_mark_set() to distinguish unmarked flows. + */ +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + return (val >> 8) - 1; +#else + return val - 1; +#endif +} + #endif /* RTE_PMD_MLX5_PRM_H_ */ diff --git a/dpdk/drivers/net/mlx5/mlx5_rss.c b/dpdk/drivers/net/mlx5/mlx5_rss.c index 0bed74ee..f47bda66 100644 --- a/dpdk/drivers/net/mlx5/mlx5_rss.c +++ b/dpdk/drivers/net/mlx5/mlx5_rss.c @@ -47,87 +47,13 @@ #pragma GCC diagnostic error "-Wpedantic" #endif -/* DPDK headers don't like -pedantic. */ -#ifdef PEDANTIC -#pragma GCC diagnostic ignored "-Wpedantic" -#endif #include #include -#ifdef PEDANTIC -#pragma GCC diagnostic error "-Wpedantic" -#endif #include "mlx5.h" +#include "mlx5_defs.h" #include "mlx5_rxtx.h" -/** - * Get a RSS configuration hash key. - * - * @param priv - * Pointer to private structure. - * @param rss_hf - * RSS hash functions configuration must be retrieved for. - * - * @return - * Pointer to a RSS configuration structure or NULL if rss_hf cannot - * be matched. - */ -static struct rte_eth_rss_conf * -rss_hash_get(struct priv *priv, uint64_t rss_hf) -{ - unsigned int i; - - for (i = 0; (i != hash_rxq_init_n); ++i) { - uint64_t dpdk_rss_hf = hash_rxq_init[i].dpdk_rss_hf; - - if (!(dpdk_rss_hf & rss_hf)) - continue; - return (*priv->rss_conf)[i]; - } - return NULL; -} - -/** - * Register a RSS key. - * - * @param priv - * Pointer to private structure. - * @param key - * Hash key to register. - * @param key_len - * Hash key length in bytes. - * @param rss_hf - * RSS hash functions the provided key applies to. - * - * @return - * 0 on success, errno value on failure. - */ -int -rss_hash_rss_conf_new_key(struct priv *priv, const uint8_t *key, - unsigned int key_len, uint64_t rss_hf) -{ - unsigned int i; - - for (i = 0; (i != hash_rxq_init_n); ++i) { - struct rte_eth_rss_conf *rss_conf; - uint64_t dpdk_rss_hf = hash_rxq_init[i].dpdk_rss_hf; - - if (!(dpdk_rss_hf & rss_hf)) - continue; - rss_conf = rte_realloc((*priv->rss_conf)[i], - (sizeof(*rss_conf) + key_len), - 0); - if (!rss_conf) - return ENOMEM; - rss_conf->rss_key = (void *)(rss_conf + 1); - rss_conf->rss_key_len = key_len; - rss_conf->rss_hf = dpdk_rss_hf; - memcpy(rss_conf->rss_key, key, key_len); - (*priv->rss_conf)[i] = rss_conf; - } - return 0; -} - /** * DPDK callback to update the RSS hash configuration. * @@ -144,23 +70,28 @@ mlx5_rss_hash_update(struct rte_eth_dev *dev, struct rte_eth_rss_conf *rss_conf) { struct priv *priv = dev->data->dev_private; - int err = 0; + int ret = 0; priv_lock(priv); - - assert(priv->rss_conf != NULL); - - /* Apply configuration. */ - if (rss_conf->rss_key) - err = rss_hash_rss_conf_new_key(priv, - rss_conf->rss_key, - rss_conf->rss_key_len, - rss_conf->rss_hf); - /* Store protocols for which RSS is enabled. */ - priv->rss_hf = rss_conf->rss_hf; + if (rss_conf->rss_hf & MLX5_RSS_HF_MASK) { + ret = -EINVAL; + goto out; + } + if (rss_conf->rss_key && rss_conf->rss_key_len) { + priv->rss_conf.rss_key = rte_realloc(priv->rss_conf.rss_key, + rss_conf->rss_key_len, 0); + if (!priv->rss_conf.rss_key) { + ret = -ENOMEM; + goto out; + } + memcpy(priv->rss_conf.rss_key, rss_conf->rss_key, + rss_conf->rss_key_len); + priv->rss_conf.rss_key_len = rss_conf->rss_key_len; + } + priv->rss_conf.rss_hf = rss_conf->rss_hf; +out: priv_unlock(priv); - assert(err >= 0); - return -err; + return ret; } /** @@ -179,26 +110,17 @@ mlx5_rss_hash_conf_get(struct rte_eth_dev *dev, struct rte_eth_rss_conf *rss_conf) { struct priv *priv = dev->data->dev_private; - struct rte_eth_rss_conf *priv_rss_conf; - priv_lock(priv); - - assert(priv->rss_conf != NULL); - - priv_rss_conf = rss_hash_get(priv, rss_conf->rss_hf); - if (!priv_rss_conf) { - rss_conf->rss_hf = 0; - priv_unlock(priv); + if (!rss_conf) return -EINVAL; - } + priv_lock(priv); if (rss_conf->rss_key && - rss_conf->rss_key_len >= priv_rss_conf->rss_key_len) - memcpy(rss_conf->rss_key, - priv_rss_conf->rss_key, - priv_rss_conf->rss_key_len); - rss_conf->rss_key_len = priv_rss_conf->rss_key_len; - rss_conf->rss_hf = priv_rss_conf->rss_hf; - + (rss_conf->rss_key_len >= priv->rss_conf.rss_key_len)) { + memcpy(rss_conf->rss_key, priv->rss_conf.rss_key, + priv->rss_conf.rss_key_len); + } + rss_conf->rss_key_len = priv->rss_conf.rss_key_len; + rss_conf->rss_hf = priv->rss_conf.rss_hf; priv_unlock(priv); return 0; } @@ -257,13 +179,9 @@ priv_dev_rss_reta_query(struct priv *priv, { unsigned int idx; unsigned int i; - int ret; - - /* See RETA comment in mlx5_dev_infos_get(). */ - ret = priv_rss_reta_index_resize(priv, priv->ind_table_max_size); - if (ret) - return ret; + if (!reta_size || reta_size > priv->reta_idx_n) + return EINVAL; /* Fill each entry of the table even if its bit is not set. */ for (idx = 0, i = 0; (i != reta_size); ++i) { idx = i / RTE_RETA_GROUP_SIZE; @@ -296,8 +214,9 @@ priv_dev_rss_reta_update(struct priv *priv, unsigned int pos; int ret; - /* See RETA comment in mlx5_dev_infos_get(). */ - ret = priv_rss_reta_index_resize(priv, priv->ind_table_max_size); + if (!reta_size) + return EINVAL; + ret = priv_rss_reta_index_resize(priv, reta_size); if (ret) return ret; @@ -363,5 +282,9 @@ mlx5_dev_rss_reta_update(struct rte_eth_dev *dev, priv_lock(priv); ret = priv_dev_rss_reta_update(priv, reta_conf, reta_size); priv_unlock(priv); + if (dev->data->dev_started) { + mlx5_dev_stop(dev); + mlx5_dev_start(dev); + } return -ret; } diff --git a/dpdk/drivers/net/mlx5/mlx5_rxmode.c b/dpdk/drivers/net/mlx5/mlx5_rxmode.c index 173e6e84..6fb245ba 100644 --- a/dpdk/drivers/net/mlx5/mlx5_rxmode.c +++ b/dpdk/drivers/net/mlx5/mlx5_rxmode.c @@ -45,341 +45,12 @@ #pragma GCC diagnostic error "-Wpedantic" #endif -/* DPDK headers don't like -pedantic. */ -#ifdef PEDANTIC -#pragma GCC diagnostic ignored "-Wpedantic" -#endif #include -#ifdef PEDANTIC -#pragma GCC diagnostic error "-Wpedantic" -#endif #include "mlx5.h" #include "mlx5_rxtx.h" #include "mlx5_utils.h" -/* Initialization data for special flows. */ -static const struct special_flow_init special_flow_init[] = { - [HASH_RXQ_FLOW_TYPE_PROMISC] = { - .dst_mac_val = "\x00\x00\x00\x00\x00\x00", - .dst_mac_mask = "\x00\x00\x00\x00\x00\x00", - .hash_types = - 1 << HASH_RXQ_TCPV4 | - 1 << HASH_RXQ_UDPV4 | - 1 << HASH_RXQ_IPV4 | - 1 << HASH_RXQ_TCPV6 | - 1 << HASH_RXQ_UDPV6 | - 1 << HASH_RXQ_IPV6 | - 1 << HASH_RXQ_ETH | - 0, - .per_vlan = 0, - }, - [HASH_RXQ_FLOW_TYPE_ALLMULTI] = { - .dst_mac_val = "\x01\x00\x00\x00\x00\x00", - .dst_mac_mask = "\x01\x00\x00\x00\x00\x00", - .hash_types = - 1 << HASH_RXQ_UDPV4 | - 1 << HASH_RXQ_IPV4 | - 1 << HASH_RXQ_UDPV6 | - 1 << HASH_RXQ_IPV6 | - 1 << HASH_RXQ_ETH | - 0, - .per_vlan = 0, - }, - [HASH_RXQ_FLOW_TYPE_BROADCAST] = { - .dst_mac_val = "\xff\xff\xff\xff\xff\xff", - .dst_mac_mask = "\xff\xff\xff\xff\xff\xff", - .hash_types = - 1 << HASH_RXQ_UDPV4 | - 1 << HASH_RXQ_IPV4 | - 1 << HASH_RXQ_UDPV6 | - 1 << HASH_RXQ_IPV6 | - 1 << HASH_RXQ_ETH | - 0, - .per_vlan = 1, - }, - [HASH_RXQ_FLOW_TYPE_IPV6MULTI] = { - .dst_mac_val = "\x33\x33\x00\x00\x00\x00", - .dst_mac_mask = "\xff\xff\x00\x00\x00\x00", - .hash_types = - 1 << HASH_RXQ_UDPV6 | - 1 << HASH_RXQ_IPV6 | - 1 << HASH_RXQ_ETH | - 0, - .per_vlan = 1, - }, -}; - -/** - * Enable a special flow in a hash RX queue for a given VLAN index. - * - * @param hash_rxq - * Pointer to hash RX queue structure. - * @param flow_type - * Special flow type. - * @param vlan_index - * VLAN index to use. - * - * @return - * 0 on success, errno value on failure. - */ -static int -hash_rxq_special_flow_enable_vlan(struct hash_rxq *hash_rxq, - enum hash_rxq_flow_type flow_type, - unsigned int vlan_index) -{ - struct priv *priv = hash_rxq->priv; - struct ibv_exp_flow *flow; - FLOW_ATTR_SPEC_ETH(data, priv_flow_attr(priv, NULL, 0, hash_rxq->type)); - struct ibv_exp_flow_attr *attr = &data->attr; - struct ibv_exp_flow_spec_eth *spec = &data->spec; - const uint8_t *mac; - const uint8_t *mask; - unsigned int vlan_enabled = (priv->vlan_filter_n && - special_flow_init[flow_type].per_vlan); - unsigned int vlan_id = priv->vlan_filter[vlan_index]; - - /* Check if flow is relevant for this hash_rxq. */ - if (!(special_flow_init[flow_type].hash_types & (1 << hash_rxq->type))) - return 0; - /* Check if flow already exists. */ - if (hash_rxq->special_flow[flow_type][vlan_index] != NULL) - return 0; - - /* - * No padding must be inserted by the compiler between attr and spec. - * This layout is expected by libibverbs. - */ - assert(((uint8_t *)attr + sizeof(*attr)) == (uint8_t *)spec); - priv_flow_attr(priv, attr, sizeof(data), hash_rxq->type); - /* The first specification must be Ethernet. */ - assert(spec->type == IBV_EXP_FLOW_SPEC_ETH); - assert(spec->size == sizeof(*spec)); - - mac = special_flow_init[flow_type].dst_mac_val; - mask = special_flow_init[flow_type].dst_mac_mask; - *spec = (struct ibv_exp_flow_spec_eth){ - .type = IBV_EXP_FLOW_SPEC_ETH, - .size = sizeof(*spec), - .val = { - .dst_mac = { - mac[0], mac[1], mac[2], - mac[3], mac[4], mac[5], - }, - .vlan_tag = (vlan_enabled ? htons(vlan_id) : 0), - }, - .mask = { - .dst_mac = { - mask[0], mask[1], mask[2], - mask[3], mask[4], mask[5], - }, - .vlan_tag = (vlan_enabled ? htons(0xfff) : 0), - }, - }; - - errno = 0; - flow = ibv_exp_create_flow(hash_rxq->qp, attr); - if (flow == NULL) { - /* It's not clear whether errno is always set in this case. */ - ERROR("%p: flow configuration failed, errno=%d: %s", - (void *)hash_rxq, errno, - (errno ? strerror(errno) : "Unknown error")); - if (errno) - return errno; - return EINVAL; - } - hash_rxq->special_flow[flow_type][vlan_index] = flow; - DEBUG("%p: special flow %s (index %d) VLAN %u (index %u) enabled", - (void *)hash_rxq, hash_rxq_flow_type_str(flow_type), flow_type, - vlan_id, vlan_index); - return 0; -} - -/** - * Disable a special flow in a hash RX queue for a given VLAN index. - * - * @param hash_rxq - * Pointer to hash RX queue structure. - * @param flow_type - * Special flow type. - * @param vlan_index - * VLAN index to use. - */ -static void -hash_rxq_special_flow_disable_vlan(struct hash_rxq *hash_rxq, - enum hash_rxq_flow_type flow_type, - unsigned int vlan_index) -{ - struct ibv_exp_flow *flow = - hash_rxq->special_flow[flow_type][vlan_index]; - - if (flow == NULL) - return; - claim_zero(ibv_exp_destroy_flow(flow)); - hash_rxq->special_flow[flow_type][vlan_index] = NULL; - DEBUG("%p: special flow %s (index %d) VLAN %u (index %u) disabled", - (void *)hash_rxq, hash_rxq_flow_type_str(flow_type), flow_type, - hash_rxq->priv->vlan_filter[vlan_index], vlan_index); -} - -/** - * Enable a special flow in a hash RX queue. - * - * @param hash_rxq - * Pointer to hash RX queue structure. - * @param flow_type - * Special flow type. - * @param vlan_index - * VLAN index to use. - * - * @return - * 0 on success, errno value on failure. - */ -static int -hash_rxq_special_flow_enable(struct hash_rxq *hash_rxq, - enum hash_rxq_flow_type flow_type) -{ - struct priv *priv = hash_rxq->priv; - unsigned int i = 0; - int ret; - - assert((unsigned int)flow_type < RTE_DIM(hash_rxq->special_flow)); - assert(RTE_DIM(hash_rxq->special_flow[flow_type]) == - RTE_DIM(priv->vlan_filter)); - /* Add a special flow for each VLAN filter when relevant. */ - do { - ret = hash_rxq_special_flow_enable_vlan(hash_rxq, flow_type, i); - if (ret) { - /* Failure, rollback. */ - while (i != 0) - hash_rxq_special_flow_disable_vlan(hash_rxq, - flow_type, - --i); - return ret; - } - } while (special_flow_init[flow_type].per_vlan && - ++i < priv->vlan_filter_n); - return 0; -} - -/** - * Disable a special flow in a hash RX queue. - * - * @param hash_rxq - * Pointer to hash RX queue structure. - * @param flow_type - * Special flow type. - */ -static void -hash_rxq_special_flow_disable(struct hash_rxq *hash_rxq, - enum hash_rxq_flow_type flow_type) -{ - unsigned int i; - - assert((unsigned int)flow_type < RTE_DIM(hash_rxq->special_flow)); - for (i = 0; (i != RTE_DIM(hash_rxq->special_flow[flow_type])); ++i) - hash_rxq_special_flow_disable_vlan(hash_rxq, flow_type, i); -} - -/** - * Enable a special flow in all hash RX queues. - * - * @param priv - * Private structure. - * @param flow_type - * Special flow type. - * - * @return - * 0 on success, errno value on failure. - */ -int -priv_special_flow_enable(struct priv *priv, enum hash_rxq_flow_type flow_type) -{ - unsigned int i; - - if (!priv_allow_flow_type(priv, flow_type)) - return 0; - for (i = 0; (i != priv->hash_rxqs_n); ++i) { - struct hash_rxq *hash_rxq = &(*priv->hash_rxqs)[i]; - int ret; - - ret = hash_rxq_special_flow_enable(hash_rxq, flow_type); - if (!ret) - continue; - /* Failure, rollback. */ - while (i != 0) { - hash_rxq = &(*priv->hash_rxqs)[--i]; - hash_rxq_special_flow_disable(hash_rxq, flow_type); - } - return ret; - } - return 0; -} - -/** - * Disable a special flow in all hash RX queues. - * - * @param priv - * Private structure. - * @param flow_type - * Special flow type. - */ -void -priv_special_flow_disable(struct priv *priv, enum hash_rxq_flow_type flow_type) -{ - unsigned int i; - - for (i = 0; (i != priv->hash_rxqs_n); ++i) { - struct hash_rxq *hash_rxq = &(*priv->hash_rxqs)[i]; - - hash_rxq_special_flow_disable(hash_rxq, flow_type); - } -} - -/** - * Enable all special flows in all hash RX queues. - * - * @param priv - * Private structure. - */ -int -priv_special_flow_enable_all(struct priv *priv) -{ - enum hash_rxq_flow_type flow_type; - - for (flow_type = HASH_RXQ_FLOW_TYPE_PROMISC; - flow_type != HASH_RXQ_FLOW_TYPE_MAC; - ++flow_type) { - int ret; - - ret = priv_special_flow_enable(priv, flow_type); - if (!ret) - continue; - /* Failure, rollback. */ - while (flow_type) - priv_special_flow_disable(priv, --flow_type); - return ret; - } - return 0; -} - -/** - * Disable all special flows in all hash RX queues. - * - * @param priv - * Private structure. - */ -void -priv_special_flow_disable_all(struct priv *priv) -{ - enum hash_rxq_flow_type flow_type; - - for (flow_type = HASH_RXQ_FLOW_TYPE_PROMISC; - flow_type != HASH_RXQ_FLOW_TYPE_MAC; - ++flow_type) - priv_special_flow_disable(priv, flow_type); -} - /** * DPDK callback to enable promiscuous mode. * @@ -389,19 +60,8 @@ priv_special_flow_disable_all(struct priv *priv) void mlx5_promiscuous_enable(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; - int ret; - - if (mlx5_is_secondary()) - return; - - priv_lock(priv); - priv->promisc_req = 1; - ret = priv_rehash_flows(priv); - if (ret) - ERROR("error while enabling promiscuous mode: %s", - strerror(ret)); - priv_unlock(priv); + dev->data->promiscuous = 1; + mlx5_traffic_restart(dev); } /** @@ -413,19 +73,8 @@ mlx5_promiscuous_enable(struct rte_eth_dev *dev) void mlx5_promiscuous_disable(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; - int ret; - - if (mlx5_is_secondary()) - return; - - priv_lock(priv); - priv->promisc_req = 0; - ret = priv_rehash_flows(priv); - if (ret) - ERROR("error while disabling promiscuous mode: %s", - strerror(ret)); - priv_unlock(priv); + dev->data->promiscuous = 0; + mlx5_traffic_restart(dev); } /** @@ -437,19 +86,8 @@ mlx5_promiscuous_disable(struct rte_eth_dev *dev) void mlx5_allmulticast_enable(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; - int ret; - - if (mlx5_is_secondary()) - return; - - priv_lock(priv); - priv->allmulti_req = 1; - ret = priv_rehash_flows(priv); - if (ret) - ERROR("error while enabling allmulticast mode: %s", - strerror(ret)); - priv_unlock(priv); + dev->data->all_multicast = 1; + mlx5_traffic_restart(dev); } /** @@ -461,17 +99,6 @@ mlx5_allmulticast_enable(struct rte_eth_dev *dev) void mlx5_allmulticast_disable(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; - int ret; - - if (mlx5_is_secondary()) - return; - - priv_lock(priv); - priv->allmulti_req = 0; - ret = priv_rehash_flows(priv); - if (ret) - ERROR("error while disabling allmulticast mode: %s", - strerror(ret)); - priv_unlock(priv); + dev->data->all_multicast = 0; + mlx5_traffic_restart(dev); } diff --git a/dpdk/drivers/net/mlx5/mlx5_rxq.c b/dpdk/drivers/net/mlx5/mlx5_rxq.c index 7dbe8dd2..20f3ec6c 100644 --- a/dpdk/drivers/net/mlx5/mlx5_rxq.c +++ b/dpdk/drivers/net/mlx5/mlx5_rxq.c @@ -36,6 +36,8 @@ #include #include #include +#include +#include /* Verbs header. */ /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ @@ -43,23 +45,18 @@ #pragma GCC diagnostic ignored "-Wpedantic" #endif #include -#include -#include +#include #ifdef PEDANTIC #pragma GCC diagnostic error "-Wpedantic" #endif -/* DPDK headers don't like -pedantic. */ -#ifdef PEDANTIC -#pragma GCC diagnostic ignored "-Wpedantic" -#endif #include #include #include #include -#ifdef PEDANTIC -#pragma GCC diagnostic error "-Wpedantic" -#endif +#include +#include +#include #include "mlx5.h" #include "mlx5_rxtx.h" @@ -67,122 +64,6 @@ #include "mlx5_autoconf.h" #include "mlx5_defs.h" -/* Initialization data for hash RX queues. */ -const struct hash_rxq_init hash_rxq_init[] = { - [HASH_RXQ_TCPV4] = { - .hash_fields = (IBV_EXP_RX_HASH_SRC_IPV4 | - IBV_EXP_RX_HASH_DST_IPV4 | - IBV_EXP_RX_HASH_SRC_PORT_TCP | - IBV_EXP_RX_HASH_DST_PORT_TCP), - .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_TCP, - .flow_priority = 0, - .flow_spec.tcp_udp = { - .type = IBV_EXP_FLOW_SPEC_TCP, - .size = sizeof(hash_rxq_init[0].flow_spec.tcp_udp), - }, - .underlayer = &hash_rxq_init[HASH_RXQ_IPV4], - }, - [HASH_RXQ_UDPV4] = { - .hash_fields = (IBV_EXP_RX_HASH_SRC_IPV4 | - IBV_EXP_RX_HASH_DST_IPV4 | - IBV_EXP_RX_HASH_SRC_PORT_UDP | - IBV_EXP_RX_HASH_DST_PORT_UDP), - .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_UDP, - .flow_priority = 0, - .flow_spec.tcp_udp = { - .type = IBV_EXP_FLOW_SPEC_UDP, - .size = sizeof(hash_rxq_init[0].flow_spec.tcp_udp), - }, - .underlayer = &hash_rxq_init[HASH_RXQ_IPV4], - }, - [HASH_RXQ_IPV4] = { - .hash_fields = (IBV_EXP_RX_HASH_SRC_IPV4 | - IBV_EXP_RX_HASH_DST_IPV4), - .dpdk_rss_hf = (ETH_RSS_IPV4 | - ETH_RSS_FRAG_IPV4), - .flow_priority = 1, - .flow_spec.ipv4 = { - .type = IBV_EXP_FLOW_SPEC_IPV4, - .size = sizeof(hash_rxq_init[0].flow_spec.ipv4), - }, - .underlayer = &hash_rxq_init[HASH_RXQ_ETH], - }, - [HASH_RXQ_TCPV6] = { - .hash_fields = (IBV_EXP_RX_HASH_SRC_IPV6 | - IBV_EXP_RX_HASH_DST_IPV6 | - IBV_EXP_RX_HASH_SRC_PORT_TCP | - IBV_EXP_RX_HASH_DST_PORT_TCP), - .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_TCP, - .flow_priority = 0, - .flow_spec.tcp_udp = { - .type = IBV_EXP_FLOW_SPEC_TCP, - .size = sizeof(hash_rxq_init[0].flow_spec.tcp_udp), - }, - .underlayer = &hash_rxq_init[HASH_RXQ_IPV6], - }, - [HASH_RXQ_UDPV6] = { - .hash_fields = (IBV_EXP_RX_HASH_SRC_IPV6 | - IBV_EXP_RX_HASH_DST_IPV6 | - IBV_EXP_RX_HASH_SRC_PORT_UDP | - IBV_EXP_RX_HASH_DST_PORT_UDP), - .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_UDP, - .flow_priority = 0, - .flow_spec.tcp_udp = { - .type = IBV_EXP_FLOW_SPEC_UDP, - .size = sizeof(hash_rxq_init[0].flow_spec.tcp_udp), - }, - .underlayer = &hash_rxq_init[HASH_RXQ_IPV6], - }, - [HASH_RXQ_IPV6] = { - .hash_fields = (IBV_EXP_RX_HASH_SRC_IPV6 | - IBV_EXP_RX_HASH_DST_IPV6), - .dpdk_rss_hf = (ETH_RSS_IPV6 | - ETH_RSS_FRAG_IPV6), - .flow_priority = 1, - .flow_spec.ipv6 = { - .type = IBV_EXP_FLOW_SPEC_IPV6, - .size = sizeof(hash_rxq_init[0].flow_spec.ipv6), - }, - .underlayer = &hash_rxq_init[HASH_RXQ_ETH], - }, - [HASH_RXQ_ETH] = { - .hash_fields = 0, - .dpdk_rss_hf = 0, - .flow_priority = 2, - .flow_spec.eth = { - .type = IBV_EXP_FLOW_SPEC_ETH, - .size = sizeof(hash_rxq_init[0].flow_spec.eth), - }, - .underlayer = NULL, - }, -}; - -/* Number of entries in hash_rxq_init[]. */ -const unsigned int hash_rxq_init_n = RTE_DIM(hash_rxq_init); - -/* Initialization data for hash RX queue indirection tables. */ -static const struct ind_table_init ind_table_init[] = { - { - .max_size = -1u, /* Superseded by HW limitations. */ - .hash_types = - 1 << HASH_RXQ_TCPV4 | - 1 << HASH_RXQ_UDPV4 | - 1 << HASH_RXQ_IPV4 | - 1 << HASH_RXQ_TCPV6 | - 1 << HASH_RXQ_UDPV6 | - 1 << HASH_RXQ_IPV6 | - 0, - .hash_types_n = 6, - }, - { - .max_size = 1, - .hash_types = 1 << HASH_RXQ_ETH, - .hash_types_n = 1, - }, -}; - -#define IND_TABLE_INIT_N RTE_DIM(ind_table_init) - /* Default RSS hash key also used for ConnectX-3. */ uint8_t rss_hash_default_key[] = { 0x2c, 0xc6, 0x81, 0xd1, @@ -200,471 +81,29 @@ uint8_t rss_hash_default_key[] = { /* Length of the default RSS hash key. */ const size_t rss_hash_default_key_len = sizeof(rss_hash_default_key); -/** - * Populate flow steering rule for a given hash RX queue type using - * information from hash_rxq_init[]. Nothing is written to flow_attr when - * flow_attr_size is not large enough, but the required size is still returned. - * - * @param priv - * Pointer to private structure. - * @param[out] flow_attr - * Pointer to flow attribute structure to fill. Note that the allocated - * area must be larger and large enough to hold all flow specifications. - * @param flow_attr_size - * Entire size of flow_attr and trailing room for flow specifications. - * @param type - * Hash RX queue type to use for flow steering rule. - * - * @return - * Total size of the flow attribute buffer. No errors are defined. - */ -size_t -priv_flow_attr(struct priv *priv, struct ibv_exp_flow_attr *flow_attr, - size_t flow_attr_size, enum hash_rxq_type type) -{ - size_t offset = sizeof(*flow_attr); - const struct hash_rxq_init *init = &hash_rxq_init[type]; - - assert(priv != NULL); - assert((size_t)type < RTE_DIM(hash_rxq_init)); - do { - offset += init->flow_spec.hdr.size; - init = init->underlayer; - } while (init != NULL); - if (offset > flow_attr_size) - return offset; - flow_attr_size = offset; - init = &hash_rxq_init[type]; - *flow_attr = (struct ibv_exp_flow_attr){ - .type = IBV_EXP_FLOW_ATTR_NORMAL, - /* Priorities < 3 are reserved for flow director. */ - .priority = init->flow_priority + 3, - .num_of_specs = 0, - .port = priv->port, - .flags = 0, - }; - do { - offset -= init->flow_spec.hdr.size; - memcpy((void *)((uintptr_t)flow_attr + offset), - &init->flow_spec, - init->flow_spec.hdr.size); - ++flow_attr->num_of_specs; - init = init->underlayer; - } while (init != NULL); - return flow_attr_size; -} - -/** - * Convert hash type position in indirection table initializer to - * hash RX queue type. - * - * @param table - * Indirection table initializer. - * @param pos - * Hash type position. - * - * @return - * Hash RX queue type. - */ -static enum hash_rxq_type -hash_rxq_type_from_pos(const struct ind_table_init *table, unsigned int pos) -{ - enum hash_rxq_type type = HASH_RXQ_TCPV4; - - assert(pos < table->hash_types_n); - do { - if ((table->hash_types & (1 << type)) && (pos-- == 0)) - break; - ++type; - } while (1); - return type; -} - -/** - * Filter out disabled hash RX queue types from ind_table_init[]. - * - * @param priv - * Pointer to private structure. - * @param[out] table - * Output table. - * - * @return - * Number of table entries. - */ -static unsigned int -priv_make_ind_table_init(struct priv *priv, - struct ind_table_init (*table)[IND_TABLE_INIT_N]) -{ - uint64_t rss_hf; - unsigned int i; - unsigned int j; - unsigned int table_n = 0; - /* Mandatory to receive frames not handled by normal hash RX queues. */ - unsigned int hash_types_sup = 1 << HASH_RXQ_ETH; - - rss_hf = priv->rss_hf; - /* Process other protocols only if more than one queue. */ - if (priv->rxqs_n > 1) - for (i = 0; (i != hash_rxq_init_n); ++i) - if (rss_hf & hash_rxq_init[i].dpdk_rss_hf) - hash_types_sup |= (1 << i); - - /* Filter out entries whose protocols are not in the set. */ - for (i = 0, j = 0; (i != IND_TABLE_INIT_N); ++i) { - unsigned int nb; - unsigned int h; - - /* j is increased only if the table has valid protocols. */ - assert(j <= i); - (*table)[j] = ind_table_init[i]; - (*table)[j].hash_types &= hash_types_sup; - for (h = 0, nb = 0; (h != hash_rxq_init_n); ++h) - if (((*table)[j].hash_types >> h) & 0x1) - ++nb; - (*table)[i].hash_types_n = nb; - if (nb) { - ++table_n; - ++j; - } - } - return table_n; -} - -/** - * Initialize hash RX queues and indirection table. - * - * @param priv - * Pointer to private structure. - * - * @return - * 0 on success, errno value on failure. - */ -int -priv_create_hash_rxqs(struct priv *priv) -{ - struct ibv_exp_wq *wqs[priv->reta_idx_n]; - struct ind_table_init ind_table_init[IND_TABLE_INIT_N]; - unsigned int ind_tables_n = - priv_make_ind_table_init(priv, &ind_table_init); - unsigned int hash_rxqs_n = 0; - struct hash_rxq (*hash_rxqs)[] = NULL; - struct ibv_exp_rwq_ind_table *(*ind_tables)[] = NULL; - unsigned int i; - unsigned int j; - unsigned int k; - int err = 0; - - assert(priv->ind_tables == NULL); - assert(priv->ind_tables_n == 0); - assert(priv->hash_rxqs == NULL); - assert(priv->hash_rxqs_n == 0); - assert(priv->pd != NULL); - assert(priv->ctx != NULL); - if (priv->rxqs_n == 0) - return EINVAL; - assert(priv->rxqs != NULL); - if (ind_tables_n == 0) { - ERROR("all hash RX queue types have been filtered out," - " indirection table cannot be created"); - return EINVAL; - } - if (priv->rxqs_n & (priv->rxqs_n - 1)) { - INFO("%u RX queues are configured, consider rounding this" - " number to the next power of two for better balancing", - priv->rxqs_n); - DEBUG("indirection table extended to assume %u WQs", - priv->reta_idx_n); - } - for (i = 0; (i != priv->reta_idx_n); ++i) { - struct rxq_ctrl *rxq_ctrl; - - rxq_ctrl = container_of((*priv->rxqs)[(*priv->reta_idx)[i]], - struct rxq_ctrl, rxq); - wqs[i] = rxq_ctrl->wq; - } - /* Get number of hash RX queues to configure. */ - for (i = 0, hash_rxqs_n = 0; (i != ind_tables_n); ++i) - hash_rxqs_n += ind_table_init[i].hash_types_n; - DEBUG("allocating %u hash RX queues for %u WQs, %u indirection tables", - hash_rxqs_n, priv->rxqs_n, ind_tables_n); - /* Create indirection tables. */ - ind_tables = rte_calloc(__func__, ind_tables_n, - sizeof((*ind_tables)[0]), 0); - if (ind_tables == NULL) { - err = ENOMEM; - ERROR("cannot allocate indirection tables container: %s", - strerror(err)); - goto error; - } - for (i = 0; (i != ind_tables_n); ++i) { - struct ibv_exp_rwq_ind_table_init_attr ind_init_attr = { - .pd = priv->pd, - .log_ind_tbl_size = 0, /* Set below. */ - .ind_tbl = wqs, - .comp_mask = 0, - }; - unsigned int ind_tbl_size = ind_table_init[i].max_size; - struct ibv_exp_rwq_ind_table *ind_table; - - if (priv->reta_idx_n < ind_tbl_size) - ind_tbl_size = priv->reta_idx_n; - ind_init_attr.log_ind_tbl_size = log2above(ind_tbl_size); - errno = 0; - ind_table = ibv_exp_create_rwq_ind_table(priv->ctx, - &ind_init_attr); - if (ind_table != NULL) { - (*ind_tables)[i] = ind_table; - continue; - } - /* Not clear whether errno is set. */ - err = (errno ? errno : EINVAL); - ERROR("RX indirection table creation failed with error %d: %s", - err, strerror(err)); - goto error; - } - /* Allocate array that holds hash RX queues and related data. */ - hash_rxqs = rte_calloc(__func__, hash_rxqs_n, - sizeof((*hash_rxqs)[0]), 0); - if (hash_rxqs == NULL) { - err = ENOMEM; - ERROR("cannot allocate hash RX queues container: %s", - strerror(err)); - goto error; - } - for (i = 0, j = 0, k = 0; - ((i != hash_rxqs_n) && (j != ind_tables_n)); - ++i) { - struct hash_rxq *hash_rxq = &(*hash_rxqs)[i]; - enum hash_rxq_type type = - hash_rxq_type_from_pos(&ind_table_init[j], k); - struct rte_eth_rss_conf *priv_rss_conf = - (*priv->rss_conf)[type]; - struct ibv_exp_rx_hash_conf hash_conf = { - .rx_hash_function = IBV_EXP_RX_HASH_FUNC_TOEPLITZ, - .rx_hash_key_len = (priv_rss_conf ? - priv_rss_conf->rss_key_len : - rss_hash_default_key_len), - .rx_hash_key = (priv_rss_conf ? - priv_rss_conf->rss_key : - rss_hash_default_key), - .rx_hash_fields_mask = hash_rxq_init[type].hash_fields, - .rwq_ind_tbl = (*ind_tables)[j], - }; - struct ibv_exp_qp_init_attr qp_init_attr = { - .max_inl_recv = 0, /* Currently not supported. */ - .qp_type = IBV_QPT_RAW_PACKET, - .comp_mask = (IBV_EXP_QP_INIT_ATTR_PD | - IBV_EXP_QP_INIT_ATTR_RX_HASH), - .pd = priv->pd, - .rx_hash_conf = &hash_conf, - .port_num = priv->port, - }; - - DEBUG("using indirection table %u for hash RX queue %u type %d", - j, i, type); - *hash_rxq = (struct hash_rxq){ - .priv = priv, - .qp = ibv_exp_create_qp(priv->ctx, &qp_init_attr), - .type = type, - }; - if (hash_rxq->qp == NULL) { - err = (errno ? errno : EINVAL); - ERROR("Hash RX QP creation failure: %s", - strerror(err)); - goto error; - } - if (++k < ind_table_init[j].hash_types_n) - continue; - /* Switch to the next indirection table and reset hash RX - * queue type array index. */ - ++j; - k = 0; - } - priv->ind_tables = ind_tables; - priv->ind_tables_n = ind_tables_n; - priv->hash_rxqs = hash_rxqs; - priv->hash_rxqs_n = hash_rxqs_n; - assert(err == 0); - return 0; -error: - if (hash_rxqs != NULL) { - for (i = 0; (i != hash_rxqs_n); ++i) { - struct ibv_qp *qp = (*hash_rxqs)[i].qp; - - if (qp == NULL) - continue; - claim_zero(ibv_destroy_qp(qp)); - } - rte_free(hash_rxqs); - } - if (ind_tables != NULL) { - for (j = 0; (j != ind_tables_n); ++j) { - struct ibv_exp_rwq_ind_table *ind_table = - (*ind_tables)[j]; - - if (ind_table == NULL) - continue; - claim_zero(ibv_exp_destroy_rwq_ind_table(ind_table)); - } - rte_free(ind_tables); - } - return err; -} - -/** - * Clean up hash RX queues and indirection table. - * - * @param priv - * Pointer to private structure. - */ -void -priv_destroy_hash_rxqs(struct priv *priv) -{ - unsigned int i; - - DEBUG("destroying %u hash RX queues", priv->hash_rxqs_n); - if (priv->hash_rxqs_n == 0) { - assert(priv->hash_rxqs == NULL); - assert(priv->ind_tables == NULL); - return; - } - for (i = 0; (i != priv->hash_rxqs_n); ++i) { - struct hash_rxq *hash_rxq = &(*priv->hash_rxqs)[i]; - unsigned int j, k; - - assert(hash_rxq->priv == priv); - assert(hash_rxq->qp != NULL); - /* Also check that there are no remaining flows. */ - for (j = 0; (j != RTE_DIM(hash_rxq->special_flow)); ++j) - for (k = 0; - (k != RTE_DIM(hash_rxq->special_flow[j])); - ++k) - assert(hash_rxq->special_flow[j][k] == NULL); - for (j = 0; (j != RTE_DIM(hash_rxq->mac_flow)); ++j) - for (k = 0; (k != RTE_DIM(hash_rxq->mac_flow[j])); ++k) - assert(hash_rxq->mac_flow[j][k] == NULL); - claim_zero(ibv_destroy_qp(hash_rxq->qp)); - } - priv->hash_rxqs_n = 0; - rte_free(priv->hash_rxqs); - priv->hash_rxqs = NULL; - for (i = 0; (i != priv->ind_tables_n); ++i) { - struct ibv_exp_rwq_ind_table *ind_table = - (*priv->ind_tables)[i]; - - assert(ind_table != NULL); - claim_zero(ibv_exp_destroy_rwq_ind_table(ind_table)); - } - priv->ind_tables_n = 0; - rte_free(priv->ind_tables); - priv->ind_tables = NULL; -} - -/** - * Check whether a given flow type is allowed. - * - * @param priv - * Pointer to private structure. - * @param type - * Flow type to check. - * - * @return - * Nonzero if the given flow type is allowed. - */ -int -priv_allow_flow_type(struct priv *priv, enum hash_rxq_flow_type type) -{ - /* Only FLOW_TYPE_PROMISC is allowed when promiscuous mode - * has been requested. */ - if (priv->promisc_req) - return type == HASH_RXQ_FLOW_TYPE_PROMISC; - switch (type) { - case HASH_RXQ_FLOW_TYPE_PROMISC: - return !!priv->promisc_req; - case HASH_RXQ_FLOW_TYPE_ALLMULTI: - return !!priv->allmulti_req; - case HASH_RXQ_FLOW_TYPE_BROADCAST: - case HASH_RXQ_FLOW_TYPE_IPV6MULTI: - /* If allmulti is enabled, broadcast and ipv6multi - * are unnecessary. */ - return !priv->allmulti_req; - case HASH_RXQ_FLOW_TYPE_MAC: - return 1; - default: - /* Unsupported flow type is not allowed. */ - return 0; - } - return 0; -} - -/** - * Automatically enable/disable flows according to configuration. - * - * @param priv - * Private structure. - * - * @return - * 0 on success, errno value on failure. - */ -int -priv_rehash_flows(struct priv *priv) -{ - enum hash_rxq_flow_type i; - - for (i = HASH_RXQ_FLOW_TYPE_PROMISC; - i != RTE_DIM((*priv->hash_rxqs)[0].special_flow); - ++i) - if (!priv_allow_flow_type(priv, i)) { - priv_special_flow_disable(priv, i); - } else { - int ret = priv_special_flow_enable(priv, i); - - if (ret) - return ret; - } - if (priv_allow_flow_type(priv, HASH_RXQ_FLOW_TYPE_MAC)) - return priv_mac_addrs_enable(priv); - priv_mac_addrs_disable(priv); - return 0; -} - /** * Allocate RX queue elements. * * @param rxq_ctrl * Pointer to RX queue structure. - * @param elts_n - * Number of elements to allocate. - * @param[in] pool - * If not NULL, fetch buffers from this array instead of allocating them - * with rte_pktmbuf_alloc(). * * @return * 0 on success, errno value on failure. */ -static int -rxq_alloc_elts(struct rxq_ctrl *rxq_ctrl, unsigned int elts_n, - struct rte_mbuf *(*pool)[]) +int +rxq_alloc_elts(struct mlx5_rxq_ctrl *rxq_ctrl) { const unsigned int sges_n = 1 << rxq_ctrl->rxq.sges_n; + unsigned int elts_n = 1 << rxq_ctrl->rxq.elts_n; unsigned int i; int ret = 0; /* Iterate on segments. */ for (i = 0; (i != elts_n); ++i) { struct rte_mbuf *buf; - volatile struct mlx5_wqe_data_seg *scat = - &(*rxq_ctrl->rxq.wqes)[i]; - if (pool != NULL) { - buf = (*pool)[i]; - assert(buf != NULL); - rte_pktmbuf_reset(buf); - rte_pktmbuf_refcnt_update(buf, 1); - } else - buf = rte_pktmbuf_alloc(rxq_ctrl->rxq.mp); + buf = rte_pktmbuf_alloc(rxq_ctrl->rxq.mp); if (buf == NULL) { - assert(pool == NULL); ERROR("%p: empty mbuf pool", (void *)rxq_ctrl); ret = ENOMEM; goto error; @@ -682,21 +121,35 @@ rxq_alloc_elts(struct rxq_ctrl *rxq_ctrl, unsigned int elts_n, DATA_LEN(buf) = rte_pktmbuf_tailroom(buf); PKT_LEN(buf) = DATA_LEN(buf); NB_SEGS(buf) = 1; - /* scat->addr must be able to store a pointer. */ - assert(sizeof(scat->addr) >= sizeof(uintptr_t)); - *scat = (struct mlx5_wqe_data_seg){ - .addr = htonll(rte_pktmbuf_mtod(buf, uintptr_t)), - .byte_count = htonl(DATA_LEN(buf)), - .lkey = htonl(rxq_ctrl->mr->lkey), - }; (*rxq_ctrl->rxq.elts)[i] = buf; } + /* If Rx vector is activated. */ + if (rxq_check_vec_support(&rxq_ctrl->rxq) > 0) { + struct mlx5_rxq_data *rxq = &rxq_ctrl->rxq; + struct rte_mbuf *mbuf_init = &rxq->fake_mbuf; + int j; + + /* Initialize default rearm_data for vPMD. */ + mbuf_init->data_off = RTE_PKTMBUF_HEADROOM; + rte_mbuf_refcnt_set(mbuf_init, 1); + mbuf_init->nb_segs = 1; + mbuf_init->port = rxq->port_id; + /* + * prevent compiler reordering: + * rearm_data covers previous fields. + */ + rte_compiler_barrier(); + rxq->mbuf_initializer = + *(uint64_t *)&mbuf_init->rearm_data; + /* Padding with a fake mbuf for vectorized Rx. */ + for (j = 0; j < MLX5_VPMD_DESCS_PER_LOOP; ++j) + (*rxq->elts)[elts_n + j] = &rxq->fake_mbuf; + } DEBUG("%p: allocated and configured %u segments (max %u packets)", (void *)rxq_ctrl, elts_n, elts_n / (1 << rxq_ctrl->rxq.sges_n)); assert(ret == 0); return 0; error: - assert(pool == NULL); elts_n = i; for (i = 0; (i != elts_n); ++i) { if ((*rxq_ctrl->rxq.elts)[i] != NULL) @@ -715,18 +168,30 @@ error: * Pointer to RX queue structure. */ static void -rxq_free_elts(struct rxq_ctrl *rxq_ctrl) +rxq_free_elts(struct mlx5_rxq_ctrl *rxq_ctrl) { - unsigned int i; + struct mlx5_rxq_data *rxq = &rxq_ctrl->rxq; + const uint16_t q_n = (1 << rxq->elts_n); + const uint16_t q_mask = q_n - 1; + uint16_t used = q_n - (rxq->rq_ci - rxq->rq_pi); + uint16_t i; DEBUG("%p: freeing WRs", (void *)rxq_ctrl); - if (rxq_ctrl->rxq.elts == NULL) + if (rxq->elts == NULL) return; - - for (i = 0; (i != rxq_ctrl->rxq.elts_n); ++i) { - if ((*rxq_ctrl->rxq.elts)[i] != NULL) - rte_pktmbuf_free_seg((*rxq_ctrl->rxq.elts)[i]); - (*rxq_ctrl->rxq.elts)[i] = NULL; + /** + * Some mbuf in the Ring belongs to the application. They cannot be + * freed. + */ + if (rxq_check_vec_support(rxq) > 0) { + for (i = 0; i < used; ++i) + (*rxq->elts)[(rxq->rq_ci + i) & q_mask] = NULL; + rxq->rq_pi = rxq->rq_ci; + } + for (i = 0; (i != (1u << rxq->elts_n)); ++i) { + if ((*rxq->elts)[i] != NULL) + rte_pktmbuf_free_seg((*rxq->elts)[i]); + (*rxq->elts)[i] = NULL; } } @@ -739,459 +204,15 @@ rxq_free_elts(struct rxq_ctrl *rxq_ctrl) * Pointer to RX queue structure. */ void -rxq_cleanup(struct rxq_ctrl *rxq_ctrl) +mlx5_rxq_cleanup(struct mlx5_rxq_ctrl *rxq_ctrl) { - struct ibv_exp_release_intf_params params; - DEBUG("cleaning up %p", (void *)rxq_ctrl); - rxq_free_elts(rxq_ctrl); - if (rxq_ctrl->fdir_queue != NULL) - priv_fdir_queue_destroy(rxq_ctrl->priv, rxq_ctrl->fdir_queue); - if (rxq_ctrl->if_wq != NULL) { - assert(rxq_ctrl->priv != NULL); - assert(rxq_ctrl->priv->ctx != NULL); - assert(rxq_ctrl->wq != NULL); - params = (struct ibv_exp_release_intf_params){ - .comp_mask = 0, - }; - claim_zero(ibv_exp_release_intf(rxq_ctrl->priv->ctx, - rxq_ctrl->if_wq, - ¶ms)); - } - if (rxq_ctrl->if_cq != NULL) { - assert(rxq_ctrl->priv != NULL); - assert(rxq_ctrl->priv->ctx != NULL); - assert(rxq_ctrl->cq != NULL); - params = (struct ibv_exp_release_intf_params){ - .comp_mask = 0, - }; - claim_zero(ibv_exp_release_intf(rxq_ctrl->priv->ctx, - rxq_ctrl->if_cq, - ¶ms)); - } - if (rxq_ctrl->wq != NULL) - claim_zero(ibv_exp_destroy_wq(rxq_ctrl->wq)); - if (rxq_ctrl->cq != NULL) - claim_zero(ibv_destroy_cq(rxq_ctrl->cq)); - if (rxq_ctrl->rd != NULL) { - struct ibv_exp_destroy_res_domain_attr attr = { - .comp_mask = 0, - }; - - assert(rxq_ctrl->priv != NULL); - assert(rxq_ctrl->priv->ctx != NULL); - claim_zero(ibv_exp_destroy_res_domain(rxq_ctrl->priv->ctx, - rxq_ctrl->rd, - &attr)); - } - if (rxq_ctrl->mr != NULL) - claim_zero(ibv_dereg_mr(rxq_ctrl->mr)); + if (rxq_ctrl->ibv) + mlx5_priv_rxq_ibv_release(rxq_ctrl->priv, rxq_ctrl->ibv); memset(rxq_ctrl, 0, sizeof(*rxq_ctrl)); } /** - * Reconfigure RX queue buffers. - * - * rxq_rehash() does not allocate mbufs, which, if not done from the right - * thread (such as a control thread), may corrupt the pool. - * In case of failure, the queue is left untouched. - * - * @param dev - * Pointer to Ethernet device structure. - * @param rxq_ctrl - * RX queue pointer. - * - * @return - * 0 on success, errno value on failure. - */ -int -rxq_rehash(struct rte_eth_dev *dev, struct rxq_ctrl *rxq_ctrl) -{ - unsigned int elts_n = rxq_ctrl->rxq.elts_n; - unsigned int i; - struct ibv_exp_wq_attr mod; - int err; - - DEBUG("%p: rehashing queue %p with %u SGE(s) per packet", - (void *)dev, (void *)rxq_ctrl, 1 << rxq_ctrl->rxq.sges_n); - assert(!(elts_n % (1 << rxq_ctrl->rxq.sges_n))); - /* From now on, any failure will render the queue unusable. - * Reinitialize WQ. */ - mod = (struct ibv_exp_wq_attr){ - .attr_mask = IBV_EXP_WQ_ATTR_STATE, - .wq_state = IBV_EXP_WQS_RESET, - }; - err = ibv_exp_modify_wq(rxq_ctrl->wq, &mod); - if (err) { - ERROR("%p: cannot reset WQ: %s", (void *)dev, strerror(err)); - assert(err > 0); - return err; - } - /* Snatch mbufs from original queue. */ - claim_zero(rxq_alloc_elts(rxq_ctrl, elts_n, rxq_ctrl->rxq.elts)); - for (i = 0; i != elts_n; ++i) { - struct rte_mbuf *buf = (*rxq_ctrl->rxq.elts)[i]; - - assert(rte_mbuf_refcnt_read(buf) == 2); - rte_pktmbuf_free_seg(buf); - } - /* Change queue state to ready. */ - mod = (struct ibv_exp_wq_attr){ - .attr_mask = IBV_EXP_WQ_ATTR_STATE, - .wq_state = IBV_EXP_WQS_RDY, - }; - err = ibv_exp_modify_wq(rxq_ctrl->wq, &mod); - if (err) { - ERROR("%p: WQ state to IBV_EXP_WQS_RDY failed: %s", - (void *)dev, strerror(err)); - goto error; - } - /* Update doorbell counter. */ - rxq_ctrl->rxq.rq_ci = elts_n >> rxq_ctrl->rxq.sges_n; - rte_wmb(); - *rxq_ctrl->rxq.rq_db = htonl(rxq_ctrl->rxq.rq_ci); -error: - assert(err >= 0); - return err; -} - -/** - * Initialize RX queue. - * - * @param tmpl - * Pointer to RX queue control template. - * - * @return - * 0 on success, errno value on failure. - */ -static inline int -rxq_setup(struct rxq_ctrl *tmpl) -{ - struct ibv_cq *ibcq = tmpl->cq; - struct mlx5_cq *cq = to_mxxx(cq, cq); - struct mlx5_rwq *rwq = container_of(tmpl->wq, struct mlx5_rwq, wq); - struct rte_mbuf *(*elts)[tmpl->rxq.elts_n] = - rte_calloc_socket("RXQ", 1, sizeof(*elts), 0, tmpl->socket); - - if (cq->cqe_sz != RTE_CACHE_LINE_SIZE) { - ERROR("Wrong MLX5_CQE_SIZE environment variable value: " - "it should be set to %u", RTE_CACHE_LINE_SIZE); - return EINVAL; - } - if (elts == NULL) - return ENOMEM; - tmpl->rxq.rq_db = rwq->rq.db; - tmpl->rxq.cqe_n = ibcq->cqe + 1; - tmpl->rxq.cq_ci = 0; - tmpl->rxq.rq_ci = 0; - tmpl->rxq.cq_db = cq->dbrec; - tmpl->rxq.wqes = - (volatile struct mlx5_wqe_data_seg (*)[]) - (uintptr_t)rwq->rq.buff; - tmpl->rxq.cqes = - (volatile struct mlx5_cqe (*)[]) - (uintptr_t)cq->active_buf->buf; - tmpl->rxq.elts = elts; - return 0; -} - -/** - * Configure a RX queue. - * - * @param dev - * Pointer to Ethernet device structure. - * @param rxq_ctrl - * Pointer to RX queue structure. - * @param desc - * Number of descriptors to configure in queue. - * @param socket - * NUMA socket on which memory must be allocated. - * @param[in] conf - * Thresholds parameters. - * @param mp - * Memory pool for buffer allocations. - * - * @return - * 0 on success, errno value on failure. - */ -int -rxq_ctrl_setup(struct rte_eth_dev *dev, struct rxq_ctrl *rxq_ctrl, - uint16_t desc, unsigned int socket, - const struct rte_eth_rxconf *conf, struct rte_mempool *mp) -{ - struct priv *priv = dev->data->dev_private; - struct rxq_ctrl tmpl = { - .priv = priv, - .socket = socket, - .rxq = { - .elts_n = desc, - .mp = mp, - }, - }; - struct ibv_exp_wq_attr mod; - union { - struct ibv_exp_query_intf_params params; - struct ibv_exp_cq_init_attr cq; - struct ibv_exp_res_domain_init_attr rd; - struct ibv_exp_wq_init_attr wq; - struct ibv_exp_cq_attr cq_attr; - } attr; - enum ibv_exp_query_intf_status status; - unsigned int mb_len = rte_pktmbuf_data_room_size(mp); - unsigned int cqe_n = desc - 1; - struct rte_mbuf *(*elts)[desc] = NULL; - int ret = 0; - - (void)conf; /* Thresholds configuration (ignored). */ - /* Enable scattered packets support for this queue if necessary. */ - assert(mb_len >= RTE_PKTMBUF_HEADROOM); - /* If smaller than MRU, multi-segment support must be enabled. */ - if (mb_len < (priv->mtu > dev->data->dev_conf.rxmode.max_rx_pkt_len ? - dev->data->dev_conf.rxmode.max_rx_pkt_len : - priv->mtu)) - dev->data->dev_conf.rxmode.jumbo_frame = 1; - if ((dev->data->dev_conf.rxmode.jumbo_frame) && - (dev->data->dev_conf.rxmode.max_rx_pkt_len > - (mb_len - RTE_PKTMBUF_HEADROOM))) { - unsigned int size = - RTE_PKTMBUF_HEADROOM + - dev->data->dev_conf.rxmode.max_rx_pkt_len; - unsigned int sges_n; - - /* - * Determine the number of SGEs needed for a full packet - * and round it to the next power of two. - */ - sges_n = log2above((size / mb_len) + !!(size % mb_len)); - tmpl.rxq.sges_n = sges_n; - /* Make sure rxq.sges_n did not overflow. */ - size = mb_len * (1 << tmpl.rxq.sges_n); - size -= RTE_PKTMBUF_HEADROOM; - if (size < dev->data->dev_conf.rxmode.max_rx_pkt_len) { - ERROR("%p: too many SGEs (%u) needed to handle" - " requested maximum packet size %u", - (void *)dev, - 1 << sges_n, - dev->data->dev_conf.rxmode.max_rx_pkt_len); - return EOVERFLOW; - } - } - DEBUG("%p: maximum number of segments per packet: %u", - (void *)dev, 1 << tmpl.rxq.sges_n); - if (desc % (1 << tmpl.rxq.sges_n)) { - ERROR("%p: number of RX queue descriptors (%u) is not a" - " multiple of SGEs per packet (%u)", - (void *)dev, - desc, - 1 << tmpl.rxq.sges_n); - return EINVAL; - } - /* Toggle RX checksum offload if hardware supports it. */ - if (priv->hw_csum) - tmpl.rxq.csum = !!dev->data->dev_conf.rxmode.hw_ip_checksum; - if (priv->hw_csum_l2tun) - tmpl.rxq.csum_l2tun = - !!dev->data->dev_conf.rxmode.hw_ip_checksum; - /* Use the entire RX mempool as the memory region. */ - tmpl.mr = mlx5_mp2mr(priv->pd, mp); - if (tmpl.mr == NULL) { - ret = EINVAL; - ERROR("%p: MR creation failure: %s", - (void *)dev, strerror(ret)); - goto error; - } - attr.rd = (struct ibv_exp_res_domain_init_attr){ - .comp_mask = (IBV_EXP_RES_DOMAIN_THREAD_MODEL | - IBV_EXP_RES_DOMAIN_MSG_MODEL), - .thread_model = IBV_EXP_THREAD_SINGLE, - .msg_model = IBV_EXP_MSG_HIGH_BW, - }; - tmpl.rd = ibv_exp_create_res_domain(priv->ctx, &attr.rd); - if (tmpl.rd == NULL) { - ret = ENOMEM; - ERROR("%p: RD creation failure: %s", - (void *)dev, strerror(ret)); - goto error; - } - attr.cq = (struct ibv_exp_cq_init_attr){ - .comp_mask = IBV_EXP_CQ_INIT_ATTR_RES_DOMAIN, - .res_domain = tmpl.rd, - }; - if (priv->cqe_comp) { - attr.cq.comp_mask |= IBV_EXP_CQ_INIT_ATTR_FLAGS; - attr.cq.flags |= IBV_EXP_CQ_COMPRESSED_CQE; - cqe_n = (desc * 2) - 1; /* Double the number of CQEs. */ - } - tmpl.cq = ibv_exp_create_cq(priv->ctx, cqe_n, NULL, NULL, 0, - &attr.cq); - if (tmpl.cq == NULL) { - ret = ENOMEM; - ERROR("%p: CQ creation failure: %s", - (void *)dev, strerror(ret)); - goto error; - } - DEBUG("priv->device_attr.max_qp_wr is %d", - priv->device_attr.max_qp_wr); - DEBUG("priv->device_attr.max_sge is %d", - priv->device_attr.max_sge); - /* Configure VLAN stripping. */ - tmpl.rxq.vlan_strip = (priv->hw_vlan_strip && - !!dev->data->dev_conf.rxmode.hw_vlan_strip); - attr.wq = (struct ibv_exp_wq_init_attr){ - .wq_context = NULL, /* Could be useful in the future. */ - .wq_type = IBV_EXP_WQT_RQ, - /* Max number of outstanding WRs. */ - .max_recv_wr = desc >> tmpl.rxq.sges_n, - /* Max number of scatter/gather elements in a WR. */ - .max_recv_sge = 1 << tmpl.rxq.sges_n, - .pd = priv->pd, - .cq = tmpl.cq, - .comp_mask = - IBV_EXP_CREATE_WQ_RES_DOMAIN | - IBV_EXP_CREATE_WQ_VLAN_OFFLOADS | - 0, - .res_domain = tmpl.rd, - .vlan_offloads = (tmpl.rxq.vlan_strip ? - IBV_EXP_RECEIVE_WQ_CVLAN_STRIP : - 0), - }; - /* By default, FCS (CRC) is stripped by hardware. */ - if (dev->data->dev_conf.rxmode.hw_strip_crc) { - tmpl.rxq.crc_present = 0; - } else if (priv->hw_fcs_strip) { - /* Ask HW/Verbs to leave CRC in place when supported. */ - attr.wq.flags |= IBV_EXP_CREATE_WQ_FLAG_SCATTER_FCS; - attr.wq.comp_mask |= IBV_EXP_CREATE_WQ_FLAGS; - tmpl.rxq.crc_present = 1; - } else { - WARN("%p: CRC stripping has been disabled but will still" - " be performed by hardware, make sure MLNX_OFED and" - " firmware are up to date", - (void *)dev); - tmpl.rxq.crc_present = 0; - } - DEBUG("%p: CRC stripping is %s, %u bytes will be subtracted from" - " incoming frames to hide it", - (void *)dev, - tmpl.rxq.crc_present ? "disabled" : "enabled", - tmpl.rxq.crc_present << 2); - if (!mlx5_getenv_int("MLX5_PMD_ENABLE_PADDING")) - ; /* Nothing else to do. */ - else if (priv->hw_padding) { - INFO("%p: enabling packet padding on queue %p", - (void *)dev, (void *)rxq_ctrl); - attr.wq.flags |= IBV_EXP_CREATE_WQ_FLAG_RX_END_PADDING; - attr.wq.comp_mask |= IBV_EXP_CREATE_WQ_FLAGS; - } else - WARN("%p: packet padding has been requested but is not" - " supported, make sure MLNX_OFED and firmware are" - " up to date", - (void *)dev); - - tmpl.wq = ibv_exp_create_wq(priv->ctx, &attr.wq); - if (tmpl.wq == NULL) { - ret = (errno ? errno : EINVAL); - ERROR("%p: WQ creation failure: %s", - (void *)dev, strerror(ret)); - goto error; - } - /* - * Make sure number of WRs*SGEs match expectations since a queue - * cannot allocate more than "desc" buffers. - */ - if (((int)attr.wq.max_recv_wr != (desc >> tmpl.rxq.sges_n)) || - ((int)attr.wq.max_recv_sge != (1 << tmpl.rxq.sges_n))) { - ERROR("%p: requested %u*%u but got %u*%u WRs*SGEs", - (void *)dev, - (desc >> tmpl.rxq.sges_n), (1 << tmpl.rxq.sges_n), - attr.wq.max_recv_wr, attr.wq.max_recv_sge); - ret = EINVAL; - goto error; - } - /* Save port ID. */ - tmpl.rxq.port_id = dev->data->port_id; - DEBUG("%p: RTE port ID: %u", (void *)rxq_ctrl, tmpl.rxq.port_id); - attr.params = (struct ibv_exp_query_intf_params){ - .intf_scope = IBV_EXP_INTF_GLOBAL, - .intf_version = 1, - .intf = IBV_EXP_INTF_CQ, - .obj = tmpl.cq, - }; - tmpl.if_cq = ibv_exp_query_intf(priv->ctx, &attr.params, &status); - if (tmpl.if_cq == NULL) { - ERROR("%p: CQ interface family query failed with status %d", - (void *)dev, status); - goto error; - } - attr.params = (struct ibv_exp_query_intf_params){ - .intf_scope = IBV_EXP_INTF_GLOBAL, - .intf = IBV_EXP_INTF_WQ, - .obj = tmpl.wq, - }; - tmpl.if_wq = ibv_exp_query_intf(priv->ctx, &attr.params, &status); - if (tmpl.if_wq == NULL) { - ERROR("%p: WQ interface family query failed with status %d", - (void *)dev, status); - goto error; - } - /* Change queue state to ready. */ - mod = (struct ibv_exp_wq_attr){ - .attr_mask = IBV_EXP_WQ_ATTR_STATE, - .wq_state = IBV_EXP_WQS_RDY, - }; - ret = ibv_exp_modify_wq(tmpl.wq, &mod); - if (ret) { - ERROR("%p: WQ state to IBV_EXP_WQS_RDY failed: %s", - (void *)dev, strerror(ret)); - goto error; - } - ret = rxq_setup(&tmpl); - if (ret) { - ERROR("%p: cannot initialize RX queue structure: %s", - (void *)dev, strerror(ret)); - goto error; - } - /* Reuse buffers from original queue if possible. */ - if (rxq_ctrl->rxq.elts_n) { - assert(rxq_ctrl->rxq.elts_n == desc); - assert(rxq_ctrl->rxq.elts != tmpl.rxq.elts); - ret = rxq_alloc_elts(&tmpl, desc, rxq_ctrl->rxq.elts); - } else - ret = rxq_alloc_elts(&tmpl, desc, NULL); - if (ret) { - ERROR("%p: RXQ allocation failed: %s", - (void *)dev, strerror(ret)); - goto error; - } - /* Clean up rxq in case we're reinitializing it. */ - DEBUG("%p: cleaning-up old rxq just in case", (void *)rxq_ctrl); - rxq_cleanup(rxq_ctrl); - /* Move mbuf pointers to dedicated storage area in RX queue. */ - elts = (void *)(rxq_ctrl + 1); - rte_memcpy(elts, tmpl.rxq.elts, sizeof(*elts)); -#ifndef NDEBUG - memset(tmpl.rxq.elts, 0x55, sizeof(*elts)); -#endif - rte_free(tmpl.rxq.elts); - tmpl.rxq.elts = elts; - *rxq_ctrl = tmpl; - /* Update doorbell counter. */ - rxq_ctrl->rxq.rq_ci = desc >> rxq_ctrl->rxq.sges_n; - rte_wmb(); - *rxq_ctrl->rxq.rq_db = htonl(rxq_ctrl->rxq.rq_ci); - DEBUG("%p: rxq updated with %p", (void *)rxq_ctrl, (void *)&tmpl); - assert(ret == 0); - return 0; -error: - elts = tmpl.rxq.elts; - rxq_cleanup(&tmpl); - rte_free(elts); - assert(ret > 0); - return ret; -} - -/** - * DPDK callback to configure a RX queue. * * @param dev * Pointer to Ethernet device structure. @@ -1215,13 +236,12 @@ mlx5_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, struct rte_mempool *mp) { struct priv *priv = dev->data->dev_private; - struct rxq *rxq = (*priv->rxqs)[idx]; - struct rxq_ctrl *rxq_ctrl = container_of(rxq, struct rxq_ctrl, rxq); - int ret; - - if (mlx5_is_secondary()) - return -E_RTE_SECONDARY; + struct mlx5_rxq_data *rxq = (*priv->rxqs)[idx]; + struct mlx5_rxq_ctrl *rxq_ctrl = + container_of(rxq, struct mlx5_rxq_ctrl, rxq); + int ret = 0; + (void)conf; priv_lock(priv); if (!rte_is_power_of_2(desc)) { desc = 1 << log2above(desc); @@ -1237,37 +257,24 @@ mlx5_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, priv_unlock(priv); return -EOVERFLOW; } - if (rxq != NULL) { - DEBUG("%p: reusing already allocated queue index %u (%p)", - (void *)dev, idx, (void *)rxq); - if (priv->started) { - priv_unlock(priv); - return -EEXIST; - } - (*priv->rxqs)[idx] = NULL; - rxq_cleanup(rxq_ctrl); - } else { - rxq_ctrl = rte_calloc_socket("RXQ", 1, sizeof(*rxq_ctrl) + - desc * sizeof(struct rte_mbuf *), - 0, socket); - if (rxq_ctrl == NULL) { - ERROR("%p: unable to allocate queue index %u", - (void *)dev, idx); - priv_unlock(priv); - return -ENOMEM; - } + if (!mlx5_priv_rxq_releasable(priv, idx)) { + ret = EBUSY; + ERROR("%p: unable to release queue index %u", + (void *)dev, idx); + goto out; } - ret = rxq_ctrl_setup(dev, rxq_ctrl, desc, socket, conf, mp); - if (ret) - rte_free(rxq_ctrl); - else { - rxq_ctrl->rxq.stats.idx = idx; - DEBUG("%p: adding RX queue %p to list", - (void *)dev, (void *)rxq_ctrl); - (*priv->rxqs)[idx] = &rxq_ctrl->rxq; - /* Update receive callback. */ - priv_select_rx_function(priv); + mlx5_priv_rxq_release(priv, idx); + rxq_ctrl = mlx5_priv_rxq_new(priv, idx, desc, socket, mp); + if (!rxq_ctrl) { + ERROR("%p: unable to allocate queue index %u", + (void *)dev, idx); + ret = ENOMEM; + goto out; } + DEBUG("%p: adding RX queue %p to list", + (void *)dev, (void *)rxq_ctrl); + (*priv->rxqs)[idx] = &rxq_ctrl->rxq; +out: priv_unlock(priv); return -ret; } @@ -1281,68 +288,1131 @@ mlx5_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, void mlx5_rx_queue_release(void *dpdk_rxq) { - struct rxq *rxq = (struct rxq *)dpdk_rxq; - struct rxq_ctrl *rxq_ctrl; + struct mlx5_rxq_data *rxq = (struct mlx5_rxq_data *)dpdk_rxq; + struct mlx5_rxq_ctrl *rxq_ctrl; struct priv *priv; - unsigned int i; - - if (mlx5_is_secondary()) - return; if (rxq == NULL) return; - rxq_ctrl = container_of(rxq, struct rxq_ctrl, rxq); + rxq_ctrl = container_of(rxq, struct mlx5_rxq_ctrl, rxq); priv = rxq_ctrl->priv; priv_lock(priv); - for (i = 0; (i != priv->rxqs_n); ++i) - if ((*priv->rxqs)[i] == rxq) { - DEBUG("%p: removing RX queue %p from list", - (void *)priv->dev, (void *)rxq_ctrl); - (*priv->rxqs)[i] = NULL; - break; - } - rxq_cleanup(rxq_ctrl); - rte_free(rxq_ctrl); + if (!mlx5_priv_rxq_releasable(priv, rxq_ctrl->rxq.stats.idx)) + rte_panic("Rx queue %p is still used by a flow and cannot be" + " removed\n", (void *)rxq_ctrl); + mlx5_priv_rxq_release(priv, rxq_ctrl->rxq.stats.idx); priv_unlock(priv); } /** - * DPDK callback for RX in secondary processes. + * Allocate queue vector and fill epoll fd list for Rx interrupts. * - * This function configures all queues from primary process information - * if necessary before reverting to the normal RX burst callback. - * - * @param dpdk_rxq - * Generic pointer to RX queue structure. - * @param[out] pkts - * Array to store received packets. - * @param pkts_n - * Maximum number of packets in array. + * @param priv + * Pointer to private structure. * * @return - * Number of packets successfully received (<= pkts_n). + * 0 on success, negative on failure. */ -uint16_t -mlx5_rx_burst_secondary_setup(void *dpdk_rxq, struct rte_mbuf **pkts, - uint16_t pkts_n) +int +priv_rx_intr_vec_enable(struct priv *priv) { - struct rxq *rxq = dpdk_rxq; - struct rxq_ctrl *rxq_ctrl = container_of(rxq, struct rxq_ctrl, rxq); - struct priv *priv = mlx5_secondary_data_setup(rxq_ctrl->priv); - struct priv *primary_priv; - unsigned int index; + unsigned int i; + unsigned int rxqs_n = priv->rxqs_n; + unsigned int n = RTE_MIN(rxqs_n, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID); + unsigned int count = 0; + struct rte_intr_handle *intr_handle = priv->dev->intr_handle; - if (priv == NULL) + if (!priv->dev->data->dev_conf.intr_conf.rxq) return 0; - primary_priv = - mlx5_secondary_data[priv->dev->data->port_id].primary_priv; - /* Look for queue index in both private structures. */ - for (index = 0; index != priv->rxqs_n; ++index) - if (((*primary_priv->rxqs)[index] == rxq) || - ((*priv->rxqs)[index] == rxq)) - break; - if (index == priv->rxqs_n) - return 0; - rxq = (*priv->rxqs)[index]; - return priv->dev->rx_pkt_burst(rxq, pkts, pkts_n); + priv_rx_intr_vec_disable(priv); + intr_handle->intr_vec = malloc(n * sizeof(intr_handle->intr_vec[0])); + if (intr_handle->intr_vec == NULL) { + ERROR("failed to allocate memory for interrupt vector," + " Rx interrupts will not be supported"); + return -ENOMEM; + } + intr_handle->type = RTE_INTR_HANDLE_EXT; + for (i = 0; i != n; ++i) { + /* This rxq ibv must not be released in this function. */ + struct mlx5_rxq_ibv *rxq_ibv = mlx5_priv_rxq_ibv_get(priv, i); + int fd; + int flags; + int rc; + + /* Skip queues that cannot request interrupts. */ + if (!rxq_ibv || !rxq_ibv->channel) { + /* Use invalid intr_vec[] index to disable entry. */ + intr_handle->intr_vec[i] = + RTE_INTR_VEC_RXTX_OFFSET + + RTE_MAX_RXTX_INTR_VEC_ID; + continue; + } + if (count >= RTE_MAX_RXTX_INTR_VEC_ID) { + ERROR("too many Rx queues for interrupt vector size" + " (%d), Rx interrupts cannot be enabled", + RTE_MAX_RXTX_INTR_VEC_ID); + priv_rx_intr_vec_disable(priv); + return -1; + } + fd = rxq_ibv->channel->fd; + flags = fcntl(fd, F_GETFL); + rc = fcntl(fd, F_SETFL, flags | O_NONBLOCK); + if (rc < 0) { + ERROR("failed to make Rx interrupt file descriptor" + " %d non-blocking for queue index %d", fd, i); + priv_rx_intr_vec_disable(priv); + return -1; + } + intr_handle->intr_vec[i] = RTE_INTR_VEC_RXTX_OFFSET + count; + intr_handle->efds[count] = fd; + count++; + } + if (!count) + priv_rx_intr_vec_disable(priv); + else + intr_handle->nb_efd = count; + return 0; +} + +/** + * Clean up Rx interrupts handler. + * + * @param priv + * Pointer to private structure. + */ +void +priv_rx_intr_vec_disable(struct priv *priv) +{ + struct rte_intr_handle *intr_handle = priv->dev->intr_handle; + unsigned int i; + unsigned int rxqs_n = priv->rxqs_n; + unsigned int n = RTE_MIN(rxqs_n, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID); + + if (!priv->dev->data->dev_conf.intr_conf.rxq) + return; + if (!intr_handle->intr_vec) + goto free; + for (i = 0; i != n; ++i) { + struct mlx5_rxq_ctrl *rxq_ctrl; + struct mlx5_rxq_data *rxq_data; + + if (intr_handle->intr_vec[i] == RTE_INTR_VEC_RXTX_OFFSET + + RTE_MAX_RXTX_INTR_VEC_ID) + continue; + /** + * Need to access directly the queue to release the reference + * kept in priv_rx_intr_vec_enable(). + */ + rxq_data = (*priv->rxqs)[i]; + rxq_ctrl = container_of(rxq_data, struct mlx5_rxq_ctrl, rxq); + mlx5_priv_rxq_ibv_release(priv, rxq_ctrl->ibv); + } +free: + rte_intr_free_epoll_fd(intr_handle); + if (intr_handle->intr_vec) + free(intr_handle->intr_vec); + intr_handle->nb_efd = 0; + intr_handle->intr_vec = NULL; +} + +/** + * MLX5 CQ notification . + * + * @param rxq + * Pointer to receive queue structure. + * @param sq_n_rxq + * Sequence number per receive queue . + */ +static inline void +mlx5_arm_cq(struct mlx5_rxq_data *rxq, int sq_n_rxq) +{ + int sq_n = 0; + uint32_t doorbell_hi; + uint64_t doorbell; + void *cq_db_reg = (char *)rxq->cq_uar + MLX5_CQ_DOORBELL; + + sq_n = sq_n_rxq & MLX5_CQ_SQN_MASK; + doorbell_hi = sq_n << MLX5_CQ_SQN_OFFSET | (rxq->cq_ci & MLX5_CI_MASK); + doorbell = (uint64_t)doorbell_hi << 32; + doorbell |= rxq->cqn; + rxq->cq_db[MLX5_CQ_ARM_DB] = rte_cpu_to_be_32(doorbell_hi); + rte_wmb(); + rte_write64(rte_cpu_to_be_64(doorbell), cq_db_reg); +} + +/** + * DPDK callback for Rx queue interrupt enable. + * + * @param dev + * Pointer to Ethernet device structure. + * @param rx_queue_id + * Rx queue number. + * + * @return + * 0 on success, negative on failure. + */ +int +mlx5_rx_intr_enable(struct rte_eth_dev *dev, uint16_t rx_queue_id) +{ + struct priv *priv = mlx5_get_priv(dev); + struct mlx5_rxq_data *rxq_data; + struct mlx5_rxq_ctrl *rxq_ctrl; + int ret = 0; + + priv_lock(priv); + rxq_data = (*priv->rxqs)[rx_queue_id]; + if (!rxq_data) { + ret = EINVAL; + goto exit; + } + rxq_ctrl = container_of(rxq_data, struct mlx5_rxq_ctrl, rxq); + if (rxq_ctrl->irq) { + struct mlx5_rxq_ibv *rxq_ibv; + + rxq_ibv = mlx5_priv_rxq_ibv_get(priv, rx_queue_id); + if (!rxq_ibv) { + ret = EINVAL; + goto exit; + } + mlx5_arm_cq(rxq_data, rxq_data->cq_arm_sn); + mlx5_priv_rxq_ibv_release(priv, rxq_ibv); + } +exit: + priv_unlock(priv); + if (ret) + WARN("unable to arm interrupt on rx queue %d", rx_queue_id); + return -ret; +} + +/** + * DPDK callback for Rx queue interrupt disable. + * + * @param dev + * Pointer to Ethernet device structure. + * @param rx_queue_id + * Rx queue number. + * + * @return + * 0 on success, negative on failure. + */ +int +mlx5_rx_intr_disable(struct rte_eth_dev *dev, uint16_t rx_queue_id) +{ + struct priv *priv = mlx5_get_priv(dev); + struct mlx5_rxq_data *rxq_data; + struct mlx5_rxq_ctrl *rxq_ctrl; + struct mlx5_rxq_ibv *rxq_ibv = NULL; + struct ibv_cq *ev_cq; + void *ev_ctx; + int ret = 0; + + priv_lock(priv); + rxq_data = (*priv->rxqs)[rx_queue_id]; + if (!rxq_data) { + ret = EINVAL; + goto exit; + } + rxq_ctrl = container_of(rxq_data, struct mlx5_rxq_ctrl, rxq); + if (!rxq_ctrl->irq) + goto exit; + rxq_ibv = mlx5_priv_rxq_ibv_get(priv, rx_queue_id); + if (!rxq_ibv) { + ret = EINVAL; + goto exit; + } + ret = ibv_get_cq_event(rxq_ibv->channel, &ev_cq, &ev_ctx); + if (ret || ev_cq != rxq_ibv->cq) { + ret = EINVAL; + goto exit; + } + rxq_data->cq_arm_sn++; + ibv_ack_cq_events(rxq_ibv->cq, 1); +exit: + if (rxq_ibv) + mlx5_priv_rxq_ibv_release(priv, rxq_ibv); + priv_unlock(priv); + if (ret) + WARN("unable to disable interrupt on rx queue %d", + rx_queue_id); + return -ret; +} + +/** + * Create the Rx queue Verbs object. + * + * @param priv + * Pointer to private structure. + * @param idx + * Queue index in DPDK Rx queue array + * + * @return + * The Verbs object initialised if it can be created. + */ +struct mlx5_rxq_ibv* +mlx5_priv_rxq_ibv_new(struct priv *priv, uint16_t idx) +{ + struct mlx5_rxq_data *rxq_data = (*priv->rxqs)[idx]; + struct mlx5_rxq_ctrl *rxq_ctrl = + container_of(rxq_data, struct mlx5_rxq_ctrl, rxq); + struct ibv_wq_attr mod; + union { + struct { + struct ibv_cq_init_attr_ex ibv; + struct mlx5dv_cq_init_attr mlx5; + } cq; + struct ibv_wq_init_attr wq; + struct ibv_cq_ex cq_attr; + } attr; + unsigned int cqe_n = (1 << rxq_data->elts_n) - 1; + struct mlx5_rxq_ibv *tmpl; + struct mlx5dv_cq cq_info; + struct mlx5dv_rwq rwq; + unsigned int i; + int ret = 0; + struct mlx5dv_obj obj; + + assert(rxq_data); + assert(!rxq_ctrl->ibv); + tmpl = rte_calloc_socket(__func__, 1, sizeof(*tmpl), 0, + rxq_ctrl->socket); + if (!tmpl) { + ERROR("%p: cannot allocate verbs resources", + (void *)rxq_ctrl); + goto error; + } + tmpl->rxq_ctrl = rxq_ctrl; + /* Use the entire RX mempool as the memory region. */ + tmpl->mr = priv_mr_get(priv, rxq_data->mp); + if (!tmpl->mr) { + tmpl->mr = priv_mr_new(priv, rxq_data->mp); + if (!tmpl->mr) { + ERROR("%p: MR creation failure", (void *)rxq_ctrl); + goto error; + } + } + if (rxq_ctrl->irq) { + tmpl->channel = ibv_create_comp_channel(priv->ctx); + if (!tmpl->channel) { + ERROR("%p: Comp Channel creation failure", + (void *)rxq_ctrl); + goto error; + } + } + attr.cq.ibv = (struct ibv_cq_init_attr_ex){ + .cqe = cqe_n, + .channel = tmpl->channel, + .comp_mask = 0, + }; + attr.cq.mlx5 = (struct mlx5dv_cq_init_attr){ + .comp_mask = 0, + }; + if (priv->cqe_comp && !rxq_data->hw_timestamp) { + attr.cq.mlx5.comp_mask |= + MLX5DV_CQ_INIT_ATTR_MASK_COMPRESSED_CQE; + attr.cq.mlx5.cqe_comp_res_format = MLX5DV_CQE_RES_FORMAT_HASH; + /* + * For vectorized Rx, it must not be doubled in order to + * make cq_ci and rq_ci aligned. + */ + if (rxq_check_vec_support(rxq_data) < 0) + attr.cq.ibv.cqe *= 2; + } else if (priv->cqe_comp && rxq_data->hw_timestamp) { + DEBUG("Rx CQE compression is disabled for HW timestamp"); + } + tmpl->cq = ibv_cq_ex_to_cq(mlx5dv_create_cq(priv->ctx, &attr.cq.ibv, + &attr.cq.mlx5)); + if (tmpl->cq == NULL) { + ERROR("%p: CQ creation failure", (void *)rxq_ctrl); + goto error; + } + DEBUG("priv->device_attr.max_qp_wr is %d", + priv->device_attr.orig_attr.max_qp_wr); + DEBUG("priv->device_attr.max_sge is %d", + priv->device_attr.orig_attr.max_sge); + attr.wq = (struct ibv_wq_init_attr){ + .wq_context = NULL, /* Could be useful in the future. */ + .wq_type = IBV_WQT_RQ, + /* Max number of outstanding WRs. */ + .max_wr = (1 << rxq_data->elts_n) >> rxq_data->sges_n, + /* Max number of scatter/gather elements in a WR. */ + .max_sge = 1 << rxq_data->sges_n, + .pd = priv->pd, + .cq = tmpl->cq, + .comp_mask = + IBV_WQ_FLAGS_CVLAN_STRIPPING | + 0, + .create_flags = (rxq_data->vlan_strip ? + IBV_WQ_FLAGS_CVLAN_STRIPPING : + 0), + }; + /* By default, FCS (CRC) is stripped by hardware. */ + if (rxq_data->crc_present) { + attr.wq.create_flags |= IBV_WQ_FLAGS_SCATTER_FCS; + attr.wq.comp_mask |= IBV_WQ_INIT_ATTR_FLAGS; + } +#ifdef HAVE_IBV_WQ_FLAG_RX_END_PADDING + if (priv->hw_padding) { + attr.wq.create_flags |= IBV_WQ_FLAG_RX_END_PADDING; + attr.wq.comp_mask |= IBV_WQ_INIT_ATTR_FLAGS; + } +#endif + tmpl->wq = ibv_create_wq(priv->ctx, &attr.wq); + if (tmpl->wq == NULL) { + ERROR("%p: WQ creation failure", (void *)rxq_ctrl); + goto error; + } + /* + * Make sure number of WRs*SGEs match expectations since a queue + * cannot allocate more than "desc" buffers. + */ + if (((int)attr.wq.max_wr != + ((1 << rxq_data->elts_n) >> rxq_data->sges_n)) || + ((int)attr.wq.max_sge != (1 << rxq_data->sges_n))) { + ERROR("%p: requested %u*%u but got %u*%u WRs*SGEs", + (void *)rxq_ctrl, + ((1 << rxq_data->elts_n) >> rxq_data->sges_n), + (1 << rxq_data->sges_n), + attr.wq.max_wr, attr.wq.max_sge); + goto error; + } + /* Change queue state to ready. */ + mod = (struct ibv_wq_attr){ + .attr_mask = IBV_WQ_ATTR_STATE, + .wq_state = IBV_WQS_RDY, + }; + ret = ibv_modify_wq(tmpl->wq, &mod); + if (ret) { + ERROR("%p: WQ state to IBV_WQS_RDY failed", + (void *)rxq_ctrl); + goto error; + } + obj.cq.in = tmpl->cq; + obj.cq.out = &cq_info; + obj.rwq.in = tmpl->wq; + obj.rwq.out = &rwq; + ret = mlx5dv_init_obj(&obj, MLX5DV_OBJ_CQ | MLX5DV_OBJ_RWQ); + if (ret != 0) + goto error; + if (cq_info.cqe_size != RTE_CACHE_LINE_SIZE) { + ERROR("Wrong MLX5_CQE_SIZE environment variable value: " + "it should be set to %u", RTE_CACHE_LINE_SIZE); + goto error; + } + /* Fill the rings. */ + rxq_data->wqes = (volatile struct mlx5_wqe_data_seg (*)[]) + (uintptr_t)rwq.buf; + for (i = 0; (i != (unsigned int)(1 << rxq_data->elts_n)); ++i) { + struct rte_mbuf *buf = (*rxq_data->elts)[i]; + volatile struct mlx5_wqe_data_seg *scat = &(*rxq_data->wqes)[i]; + + /* scat->addr must be able to store a pointer. */ + assert(sizeof(scat->addr) >= sizeof(uintptr_t)); + *scat = (struct mlx5_wqe_data_seg){ + .addr = rte_cpu_to_be_64(rte_pktmbuf_mtod(buf, + uintptr_t)), + .byte_count = rte_cpu_to_be_32(DATA_LEN(buf)), + .lkey = tmpl->mr->lkey, + }; + } + rxq_data->rq_db = rwq.dbrec; + rxq_data->cqe_n = log2above(cq_info.cqe_cnt); + rxq_data->cq_ci = 0; + rxq_data->rq_ci = 0; + rxq_data->rq_pi = 0; + rxq_data->zip = (struct rxq_zip){ + .ai = 0, + }; + rxq_data->cq_db = cq_info.dbrec; + rxq_data->cqes = (volatile struct mlx5_cqe (*)[])(uintptr_t)cq_info.buf; + rxq_data->cq_uar = cq_info.cq_uar; + rxq_data->cqn = cq_info.cqn; + rxq_data->cq_arm_sn = 0; + /* Update doorbell counter. */ + rxq_data->rq_ci = (1 << rxq_data->elts_n) >> rxq_data->sges_n; + rte_wmb(); + *rxq_data->rq_db = rte_cpu_to_be_32(rxq_data->rq_ci); + DEBUG("%p: rxq updated with %p", (void *)rxq_ctrl, (void *)&tmpl); + rte_atomic32_inc(&tmpl->refcnt); + DEBUG("%p: Verbs Rx queue %p: refcnt %d", (void *)priv, + (void *)tmpl, rte_atomic32_read(&tmpl->refcnt)); + LIST_INSERT_HEAD(&priv->rxqsibv, tmpl, next); + return tmpl; +error: + if (tmpl->wq) + claim_zero(ibv_destroy_wq(tmpl->wq)); + if (tmpl->cq) + claim_zero(ibv_destroy_cq(tmpl->cq)); + if (tmpl->channel) + claim_zero(ibv_destroy_comp_channel(tmpl->channel)); + if (tmpl->mr) + priv_mr_release(priv, tmpl->mr); + return NULL; +} + +/** + * Get an Rx queue Verbs object. + * + * @param priv + * Pointer to private structure. + * @param idx + * Queue index in DPDK Rx queue array + * + * @return + * The Verbs object if it exists. + */ +struct mlx5_rxq_ibv* +mlx5_priv_rxq_ibv_get(struct priv *priv, uint16_t idx) +{ + struct mlx5_rxq_data *rxq_data = (*priv->rxqs)[idx]; + struct mlx5_rxq_ctrl *rxq_ctrl; + + if (idx >= priv->rxqs_n) + return NULL; + if (!rxq_data) + return NULL; + rxq_ctrl = container_of(rxq_data, struct mlx5_rxq_ctrl, rxq); + if (rxq_ctrl->ibv) { + priv_mr_get(priv, rxq_data->mp); + rte_atomic32_inc(&rxq_ctrl->ibv->refcnt); + DEBUG("%p: Verbs Rx queue %p: refcnt %d", (void *)priv, + (void *)rxq_ctrl->ibv, + rte_atomic32_read(&rxq_ctrl->ibv->refcnt)); + } + return rxq_ctrl->ibv; +} + +/** + * Release an Rx verbs queue object. + * + * @param priv + * Pointer to private structure. + * @param rxq_ibv + * Verbs Rx queue object. + * + * @return + * 0 on success, errno value on failure. + */ +int +mlx5_priv_rxq_ibv_release(struct priv *priv, struct mlx5_rxq_ibv *rxq_ibv) +{ + int ret; + + assert(rxq_ibv); + assert(rxq_ibv->wq); + assert(rxq_ibv->cq); + assert(rxq_ibv->mr); + ret = priv_mr_release(priv, rxq_ibv->mr); + if (!ret) + rxq_ibv->mr = NULL; + DEBUG("%p: Verbs Rx queue %p: refcnt %d", (void *)priv, + (void *)rxq_ibv, rte_atomic32_read(&rxq_ibv->refcnt)); + if (rte_atomic32_dec_and_test(&rxq_ibv->refcnt)) { + rxq_free_elts(rxq_ibv->rxq_ctrl); + claim_zero(ibv_destroy_wq(rxq_ibv->wq)); + claim_zero(ibv_destroy_cq(rxq_ibv->cq)); + if (rxq_ibv->channel) + claim_zero(ibv_destroy_comp_channel(rxq_ibv->channel)); + LIST_REMOVE(rxq_ibv, next); + rte_free(rxq_ibv); + return 0; + } + return EBUSY; +} + +/** + * Verify the Verbs Rx queue list is empty + * + * @param priv + * Pointer to private structure. + * + * @return the number of object not released. + */ +int +mlx5_priv_rxq_ibv_verify(struct priv *priv) +{ + int ret = 0; + struct mlx5_rxq_ibv *rxq_ibv; + + LIST_FOREACH(rxq_ibv, &priv->rxqsibv, next) { + DEBUG("%p: Verbs Rx queue %p still referenced", (void *)priv, + (void *)rxq_ibv); + ++ret; + } + return ret; +} + +/** + * Return true if a single reference exists on the object. + * + * @param priv + * Pointer to private structure. + * @param rxq_ibv + * Verbs Rx queue object. + */ +int +mlx5_priv_rxq_ibv_releasable(struct priv *priv, struct mlx5_rxq_ibv *rxq_ibv) +{ + (void)priv; + assert(rxq_ibv); + return (rte_atomic32_read(&rxq_ibv->refcnt) == 1); +} + +/** + * Create a DPDK Rx queue. + * + * @param priv + * Pointer to private structure. + * @param idx + * TX queue index. + * @param desc + * Number of descriptors to configure in queue. + * @param socket + * NUMA socket on which memory must be allocated. + * + * @return + * A DPDK queue object on success. + */ +struct mlx5_rxq_ctrl* +mlx5_priv_rxq_new(struct priv *priv, uint16_t idx, uint16_t desc, + unsigned int socket, struct rte_mempool *mp) +{ + struct rte_eth_dev *dev = priv->dev; + struct mlx5_rxq_ctrl *tmpl; + const uint16_t desc_n = + desc + priv->rx_vec_en * MLX5_VPMD_DESCS_PER_LOOP; + unsigned int mb_len = rte_pktmbuf_data_room_size(mp); + + tmpl = rte_calloc_socket("RXQ", 1, + sizeof(*tmpl) + + desc_n * sizeof(struct rte_mbuf *), + 0, socket); + if (!tmpl) + return NULL; + tmpl->socket = socket; + if (priv->dev->data->dev_conf.intr_conf.rxq) + tmpl->irq = 1; + /* Enable scattered packets support for this queue if necessary. */ + assert(mb_len >= RTE_PKTMBUF_HEADROOM); + if (dev->data->dev_conf.rxmode.max_rx_pkt_len <= + (mb_len - RTE_PKTMBUF_HEADROOM)) { + tmpl->rxq.sges_n = 0; + } else if (dev->data->dev_conf.rxmode.enable_scatter) { + unsigned int size = + RTE_PKTMBUF_HEADROOM + + dev->data->dev_conf.rxmode.max_rx_pkt_len; + unsigned int sges_n; + + /* + * Determine the number of SGEs needed for a full packet + * and round it to the next power of two. + */ + sges_n = log2above((size / mb_len) + !!(size % mb_len)); + tmpl->rxq.sges_n = sges_n; + /* Make sure rxq.sges_n did not overflow. */ + size = mb_len * (1 << tmpl->rxq.sges_n); + size -= RTE_PKTMBUF_HEADROOM; + if (size < dev->data->dev_conf.rxmode.max_rx_pkt_len) { + ERROR("%p: too many SGEs (%u) needed to handle" + " requested maximum packet size %u", + (void *)dev, + 1 << sges_n, + dev->data->dev_conf.rxmode.max_rx_pkt_len); + goto error; + } + } else { + WARN("%p: the requested maximum Rx packet size (%u) is" + " larger than a single mbuf (%u) and scattered" + " mode has not been requested", + (void *)dev, + dev->data->dev_conf.rxmode.max_rx_pkt_len, + mb_len - RTE_PKTMBUF_HEADROOM); + } + DEBUG("%p: maximum number of segments per packet: %u", + (void *)dev, 1 << tmpl->rxq.sges_n); + if (desc % (1 << tmpl->rxq.sges_n)) { + ERROR("%p: number of RX queue descriptors (%u) is not a" + " multiple of SGEs per packet (%u)", + (void *)dev, + desc, + 1 << tmpl->rxq.sges_n); + goto error; + } + /* Toggle RX checksum offload if hardware supports it. */ + if (priv->hw_csum) + tmpl->rxq.csum = !!dev->data->dev_conf.rxmode.hw_ip_checksum; + if (priv->hw_csum_l2tun) + tmpl->rxq.csum_l2tun = + !!dev->data->dev_conf.rxmode.hw_ip_checksum; + tmpl->rxq.hw_timestamp = + !!dev->data->dev_conf.rxmode.hw_timestamp; + /* Configure VLAN stripping. */ + tmpl->rxq.vlan_strip = (priv->hw_vlan_strip && + !!dev->data->dev_conf.rxmode.hw_vlan_strip); + /* By default, FCS (CRC) is stripped by hardware. */ + if (dev->data->dev_conf.rxmode.hw_strip_crc) { + tmpl->rxq.crc_present = 0; + } else if (priv->hw_fcs_strip) { + tmpl->rxq.crc_present = 1; + } else { + WARN("%p: CRC stripping has been disabled but will still" + " be performed by hardware, make sure MLNX_OFED and" + " firmware are up to date", + (void *)dev); + tmpl->rxq.crc_present = 0; + } + DEBUG("%p: CRC stripping is %s, %u bytes will be subtracted from" + " incoming frames to hide it", + (void *)dev, + tmpl->rxq.crc_present ? "disabled" : "enabled", + tmpl->rxq.crc_present << 2); + /* Save port ID. */ + tmpl->rxq.rss_hash = priv->rxqs_n > 1; + tmpl->rxq.port_id = dev->data->port_id; + tmpl->priv = priv; + tmpl->rxq.mp = mp; + tmpl->rxq.stats.idx = idx; + tmpl->rxq.elts_n = log2above(desc); + tmpl->rxq.elts = + (struct rte_mbuf *(*)[1 << tmpl->rxq.elts_n])(tmpl + 1); + rte_atomic32_inc(&tmpl->refcnt); + DEBUG("%p: Rx queue %p: refcnt %d", (void *)priv, + (void *)tmpl, rte_atomic32_read(&tmpl->refcnt)); + LIST_INSERT_HEAD(&priv->rxqsctrl, tmpl, next); + return tmpl; +error: + rte_free(tmpl); + return NULL; +} + +/** + * Get a Rx queue. + * + * @param priv + * Pointer to private structure. + * @param idx + * TX queue index. + * + * @return + * A pointer to the queue if it exists. + */ +struct mlx5_rxq_ctrl* +mlx5_priv_rxq_get(struct priv *priv, uint16_t idx) +{ + struct mlx5_rxq_ctrl *rxq_ctrl = NULL; + + if ((*priv->rxqs)[idx]) { + rxq_ctrl = container_of((*priv->rxqs)[idx], + struct mlx5_rxq_ctrl, + rxq); + + mlx5_priv_rxq_ibv_get(priv, idx); + rte_atomic32_inc(&rxq_ctrl->refcnt); + DEBUG("%p: Rx queue %p: refcnt %d", (void *)priv, + (void *)rxq_ctrl, rte_atomic32_read(&rxq_ctrl->refcnt)); + } + return rxq_ctrl; +} + +/** + * Release a Rx queue. + * + * @param priv + * Pointer to private structure. + * @param idx + * TX queue index. + * + * @return + * 0 on success, errno value on failure. + */ +int +mlx5_priv_rxq_release(struct priv *priv, uint16_t idx) +{ + struct mlx5_rxq_ctrl *rxq_ctrl; + + if (!(*priv->rxqs)[idx]) + return 0; + rxq_ctrl = container_of((*priv->rxqs)[idx], struct mlx5_rxq_ctrl, rxq); + assert(rxq_ctrl->priv); + if (rxq_ctrl->ibv) { + int ret; + + ret = mlx5_priv_rxq_ibv_release(rxq_ctrl->priv, rxq_ctrl->ibv); + if (!ret) + rxq_ctrl->ibv = NULL; + } + DEBUG("%p: Rx queue %p: refcnt %d", (void *)priv, + (void *)rxq_ctrl, rte_atomic32_read(&rxq_ctrl->refcnt)); + if (rte_atomic32_dec_and_test(&rxq_ctrl->refcnt)) { + LIST_REMOVE(rxq_ctrl, next); + rte_free(rxq_ctrl); + (*priv->rxqs)[idx] = NULL; + return 0; + } + return EBUSY; +} + +/** + * Verify if the queue can be released. + * + * @param priv + * Pointer to private structure. + * @param idx + * TX queue index. + * + * @return + * 1 if the queue can be released. + */ +int +mlx5_priv_rxq_releasable(struct priv *priv, uint16_t idx) +{ + struct mlx5_rxq_ctrl *rxq_ctrl; + + if (!(*priv->rxqs)[idx]) + return -1; + rxq_ctrl = container_of((*priv->rxqs)[idx], struct mlx5_rxq_ctrl, rxq); + return (rte_atomic32_read(&rxq_ctrl->refcnt) == 1); +} + +/** + * Verify the Rx Queue list is empty + * + * @param priv + * Pointer to private structure. + * + * @return the number of object not released. + */ +int +mlx5_priv_rxq_verify(struct priv *priv) +{ + struct mlx5_rxq_ctrl *rxq_ctrl; + int ret = 0; + + LIST_FOREACH(rxq_ctrl, &priv->rxqsctrl, next) { + DEBUG("%p: Rx Queue %p still referenced", (void *)priv, + (void *)rxq_ctrl); + ++ret; + } + return ret; +} + +/** + * Create an indirection table. + * + * @param priv + * Pointer to private structure. + * @param queues + * Queues entering in the indirection table. + * @param queues_n + * Number of queues in the array. + * + * @return + * A new indirection table. + */ +struct mlx5_ind_table_ibv* +mlx5_priv_ind_table_ibv_new(struct priv *priv, uint16_t queues[], + uint16_t queues_n) +{ + struct mlx5_ind_table_ibv *ind_tbl; + const unsigned int wq_n = rte_is_power_of_2(queues_n) ? + log2above(queues_n) : + log2above(priv->ind_table_max_size); + struct ibv_wq *wq[1 << wq_n]; + unsigned int i; + unsigned int j; + + ind_tbl = rte_calloc(__func__, 1, sizeof(*ind_tbl) + + queues_n * sizeof(uint16_t), 0); + if (!ind_tbl) + return NULL; + for (i = 0; i != queues_n; ++i) { + struct mlx5_rxq_ctrl *rxq = + mlx5_priv_rxq_get(priv, queues[i]); + + if (!rxq) + goto error; + wq[i] = rxq->ibv->wq; + ind_tbl->queues[i] = queues[i]; + } + ind_tbl->queues_n = queues_n; + /* Finalise indirection table. */ + for (j = 0; i != (unsigned int)(1 << wq_n); ++i, ++j) + wq[i] = wq[j]; + ind_tbl->ind_table = ibv_create_rwq_ind_table( + priv->ctx, + &(struct ibv_rwq_ind_table_init_attr){ + .log_ind_tbl_size = wq_n, + .ind_tbl = wq, + .comp_mask = 0, + }); + if (!ind_tbl->ind_table) + goto error; + rte_atomic32_inc(&ind_tbl->refcnt); + LIST_INSERT_HEAD(&priv->ind_tbls, ind_tbl, next); + DEBUG("%p: Indirection table %p: refcnt %d", (void *)priv, + (void *)ind_tbl, rte_atomic32_read(&ind_tbl->refcnt)); + return ind_tbl; +error: + rte_free(ind_tbl); + DEBUG("%p cannot create indirection table", (void *)priv); + return NULL; +} + +/** + * Get an indirection table. + * + * @param priv + * Pointer to private structure. + * @param queues + * Queues entering in the indirection table. + * @param queues_n + * Number of queues in the array. + * + * @return + * An indirection table if found. + */ +struct mlx5_ind_table_ibv* +mlx5_priv_ind_table_ibv_get(struct priv *priv, uint16_t queues[], + uint16_t queues_n) +{ + struct mlx5_ind_table_ibv *ind_tbl; + + LIST_FOREACH(ind_tbl, &priv->ind_tbls, next) { + if ((ind_tbl->queues_n == queues_n) && + (memcmp(ind_tbl->queues, queues, + ind_tbl->queues_n * sizeof(ind_tbl->queues[0])) + == 0)) + break; + } + if (ind_tbl) { + unsigned int i; + + rte_atomic32_inc(&ind_tbl->refcnt); + DEBUG("%p: Indirection table %p: refcnt %d", (void *)priv, + (void *)ind_tbl, rte_atomic32_read(&ind_tbl->refcnt)); + for (i = 0; i != ind_tbl->queues_n; ++i) + mlx5_priv_rxq_get(priv, ind_tbl->queues[i]); + } + return ind_tbl; +} + +/** + * Release an indirection table. + * + * @param priv + * Pointer to private structure. + * @param ind_table + * Indirection table to release. + * + * @return + * 0 on success, errno value on failure. + */ +int +mlx5_priv_ind_table_ibv_release(struct priv *priv, + struct mlx5_ind_table_ibv *ind_tbl) +{ + unsigned int i; + + DEBUG("%p: Indirection table %p: refcnt %d", (void *)priv, + (void *)ind_tbl, rte_atomic32_read(&ind_tbl->refcnt)); + if (rte_atomic32_dec_and_test(&ind_tbl->refcnt)) + claim_zero(ibv_destroy_rwq_ind_table(ind_tbl->ind_table)); + for (i = 0; i != ind_tbl->queues_n; ++i) + claim_nonzero(mlx5_priv_rxq_release(priv, ind_tbl->queues[i])); + if (!rte_atomic32_read(&ind_tbl->refcnt)) { + LIST_REMOVE(ind_tbl, next); + rte_free(ind_tbl); + return 0; + } + return EBUSY; +} + +/** + * Verify the Rx Queue list is empty + * + * @param priv + * Pointer to private structure. + * + * @return the number of object not released. + */ +int +mlx5_priv_ind_table_ibv_verify(struct priv *priv) +{ + struct mlx5_ind_table_ibv *ind_tbl; + int ret = 0; + + LIST_FOREACH(ind_tbl, &priv->ind_tbls, next) { + DEBUG("%p: Verbs indirection table %p still referenced", + (void *)priv, (void *)ind_tbl); + ++ret; + } + return ret; +} + +/** + * Create an Rx Hash queue. + * + * @param priv + * Pointer to private structure. + * @param rss_key + * RSS key for the Rx hash queue. + * @param rss_key_len + * RSS key length. + * @param hash_fields + * Verbs protocol hash field to make the RSS on. + * @param queues + * Queues entering in hash queue. In case of empty hash_fields only the + * first queue index will be taken for the indirection table. + * @param queues_n + * Number of queues. + * + * @return + * An hash Rx queue on success. + */ +struct mlx5_hrxq* +mlx5_priv_hrxq_new(struct priv *priv, uint8_t *rss_key, uint8_t rss_key_len, + uint64_t hash_fields, uint16_t queues[], uint16_t queues_n) +{ + struct mlx5_hrxq *hrxq; + struct mlx5_ind_table_ibv *ind_tbl; + struct ibv_qp *qp; + + queues_n = hash_fields ? queues_n : 1; + ind_tbl = mlx5_priv_ind_table_ibv_get(priv, queues, queues_n); + if (!ind_tbl) + ind_tbl = mlx5_priv_ind_table_ibv_new(priv, queues, queues_n); + if (!ind_tbl) + return NULL; + qp = ibv_create_qp_ex( + priv->ctx, + &(struct ibv_qp_init_attr_ex){ + .qp_type = IBV_QPT_RAW_PACKET, + .comp_mask = + IBV_QP_INIT_ATTR_PD | + IBV_QP_INIT_ATTR_IND_TABLE | + IBV_QP_INIT_ATTR_RX_HASH, + .rx_hash_conf = (struct ibv_rx_hash_conf){ + .rx_hash_function = IBV_RX_HASH_FUNC_TOEPLITZ, + .rx_hash_key_len = rss_key_len, + .rx_hash_key = rss_key, + .rx_hash_fields_mask = hash_fields, + }, + .rwq_ind_tbl = ind_tbl->ind_table, + .pd = priv->pd, + }); + if (!qp) + goto error; + hrxq = rte_calloc(__func__, 1, sizeof(*hrxq) + rss_key_len, 0); + if (!hrxq) + goto error; + hrxq->ind_table = ind_tbl; + hrxq->qp = qp; + hrxq->rss_key_len = rss_key_len; + hrxq->hash_fields = hash_fields; + memcpy(hrxq->rss_key, rss_key, rss_key_len); + rte_atomic32_inc(&hrxq->refcnt); + LIST_INSERT_HEAD(&priv->hrxqs, hrxq, next); + DEBUG("%p: Hash Rx queue %p: refcnt %d", (void *)priv, + (void *)hrxq, rte_atomic32_read(&hrxq->refcnt)); + return hrxq; +error: + mlx5_priv_ind_table_ibv_release(priv, ind_tbl); + if (qp) + claim_zero(ibv_destroy_qp(qp)); + return NULL; +} + +/** + * Get an Rx Hash queue. + * + * @param priv + * Pointer to private structure. + * @param rss_conf + * RSS configuration for the Rx hash queue. + * @param queues + * Queues entering in hash queue. In case of empty hash_fields only the + * first queue index will be taken for the indirection table. + * @param queues_n + * Number of queues. + * + * @return + * An hash Rx queue on success. + */ +struct mlx5_hrxq* +mlx5_priv_hrxq_get(struct priv *priv, uint8_t *rss_key, uint8_t rss_key_len, + uint64_t hash_fields, uint16_t queues[], uint16_t queues_n) +{ + struct mlx5_hrxq *hrxq; + + queues_n = hash_fields ? queues_n : 1; + LIST_FOREACH(hrxq, &priv->hrxqs, next) { + struct mlx5_ind_table_ibv *ind_tbl; + + if (hrxq->rss_key_len != rss_key_len) + continue; + if (memcmp(hrxq->rss_key, rss_key, rss_key_len)) + continue; + if (hrxq->hash_fields != hash_fields) + continue; + ind_tbl = mlx5_priv_ind_table_ibv_get(priv, queues, queues_n); + if (!ind_tbl) + continue; + if (ind_tbl != hrxq->ind_table) { + mlx5_priv_ind_table_ibv_release(priv, ind_tbl); + continue; + } + rte_atomic32_inc(&hrxq->refcnt); + DEBUG("%p: Hash Rx queue %p: refcnt %d", (void *)priv, + (void *)hrxq, rte_atomic32_read(&hrxq->refcnt)); + return hrxq; + } + return NULL; +} + +/** + * Release the hash Rx queue. + * + * @param priv + * Pointer to private structure. + * @param hrxq + * Pointer to Hash Rx queue to release. + * + * @return + * 0 on success, errno value on failure. + */ +int +mlx5_priv_hrxq_release(struct priv *priv, struct mlx5_hrxq *hrxq) +{ + DEBUG("%p: Hash Rx queue %p: refcnt %d", (void *)priv, + (void *)hrxq, rte_atomic32_read(&hrxq->refcnt)); + if (rte_atomic32_dec_and_test(&hrxq->refcnt)) { + claim_zero(ibv_destroy_qp(hrxq->qp)); + mlx5_priv_ind_table_ibv_release(priv, hrxq->ind_table); + LIST_REMOVE(hrxq, next); + rte_free(hrxq); + return 0; + } + claim_nonzero(mlx5_priv_ind_table_ibv_release(priv, hrxq->ind_table)); + return EBUSY; +} + +/** + * Verify the Rx Queue list is empty + * + * @param priv + * Pointer to private structure. + * + * @return the number of object not released. + */ +int +mlx5_priv_hrxq_ibv_verify(struct priv *priv) +{ + struct mlx5_hrxq *hrxq; + int ret = 0; + + LIST_FOREACH(hrxq, &priv->hrxqs, next) { + DEBUG("%p: Verbs Hash Rx queue %p still referenced", + (void *)priv, (void *)hrxq); + ++ret; + } + return ret; } diff --git a/dpdk/drivers/net/mlx5/mlx5_rxtx.c b/dpdk/drivers/net/mlx5/mlx5_rxtx.c index 79f7fa99..32bfa307 100644 --- a/dpdk/drivers/net/mlx5/mlx5_rxtx.c +++ b/dpdk/drivers/net/mlx5/mlx5_rxtx.c @@ -42,25 +42,17 @@ #pragma GCC diagnostic ignored "-Wpedantic" #endif #include -#include -#include +#include #ifdef PEDANTIC #pragma GCC diagnostic error "-Wpedantic" #endif -/* DPDK headers don't like -pedantic. */ -#ifdef PEDANTIC -#pragma GCC diagnostic ignored "-Wpedantic" -#endif #include #include #include #include #include #include -#ifdef PEDANTIC -#pragma GCC diagnostic error "-Wpedantic" -#endif #include "mlx5.h" #include "mlx5_utils.h" @@ -69,382 +61,263 @@ #include "mlx5_defs.h" #include "mlx5_prm.h" -#ifndef NDEBUG +static __rte_always_inline uint32_t +rxq_cq_to_pkt_type(volatile struct mlx5_cqe *cqe); + +static __rte_always_inline int +mlx5_rx_poll_len(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe, + uint16_t cqe_cnt, uint32_t *rss_hash); + +static __rte_always_inline uint32_t +rxq_cq_to_ol_flags(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe); + +uint32_t mlx5_ptype_table[] __rte_cache_aligned = { + [0xff] = RTE_PTYPE_ALL_MASK, /* Last entry for errored packet. */ +}; /** - * Verify or set magic value in CQE. + * Build a table to translate Rx completion flags to packet type. * - * @param cqe - * Pointer to CQE. + * @note: fix mlx5_dev_supported_ptypes_get() if any change here. + */ +void +mlx5_set_ptype_table(void) +{ + unsigned int i; + uint32_t (*p)[RTE_DIM(mlx5_ptype_table)] = &mlx5_ptype_table; + + /* Last entry must not be overwritten, reserved for errored packet. */ + for (i = 0; i < RTE_DIM(mlx5_ptype_table) - 1; ++i) + (*p)[i] = RTE_PTYPE_UNKNOWN; + /* + * The index to the array should have: + * bit[1:0] = l3_hdr_type + * bit[4:2] = l4_hdr_type + * bit[5] = ip_frag + * bit[6] = tunneled + * bit[7] = outer_l3_type + */ + /* L2 */ + (*p)[0x00] = RTE_PTYPE_L2_ETHER; + /* L3 */ + (*p)[0x01] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_L4_NONFRAG; + (*p)[0x02] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_NONFRAG; + /* Fragmented */ + (*p)[0x21] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_L4_FRAG; + (*p)[0x22] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_FRAG; + /* TCP */ + (*p)[0x05] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_L4_TCP; + (*p)[0x06] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_TCP; + /* UDP */ + (*p)[0x09] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_L4_UDP; + (*p)[0x0a] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_UDP; + /* Repeat with outer_l3_type being set. Just in case. */ + (*p)[0x81] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_L4_NONFRAG; + (*p)[0x82] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_NONFRAG; + (*p)[0xa1] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_L4_FRAG; + (*p)[0xa2] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_FRAG; + (*p)[0x85] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_L4_TCP; + (*p)[0x86] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_TCP; + (*p)[0x89] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_L4_UDP; + (*p)[0x8a] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_UDP; + /* Tunneled - L3 */ + (*p)[0x41] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_NONFRAG; + (*p)[0x42] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_NONFRAG; + (*p)[0xc1] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_NONFRAG; + (*p)[0xc2] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_NONFRAG; + /* Tunneled - Fragmented */ + (*p)[0x61] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_FRAG; + (*p)[0x62] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_FRAG; + (*p)[0xe1] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_FRAG; + (*p)[0xe2] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_FRAG; + /* Tunneled - TCP */ + (*p)[0x45] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_TCP; + (*p)[0x46] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_TCP; + (*p)[0xc5] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_TCP; + (*p)[0xc6] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_TCP; + /* Tunneled - UDP */ + (*p)[0x49] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_UDP; + (*p)[0x4a] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_UDP; + (*p)[0xc9] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_UDP; + (*p)[0xca] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_UDP; +} + +/** + * Return the size of tailroom of WQ. + * + * @param txq + * Pointer to TX queue structure. + * @param addr + * Pointer to tail of WQ. * * @return - * 0 the first time. + * Size of tailroom. */ -static inline int -check_cqe64_seen(volatile struct mlx5_cqe64 *cqe) +static inline size_t +tx_mlx5_wq_tailroom(struct mlx5_txq_data *txq, void *addr) { - static const uint8_t magic[] = "seen"; - volatile uint8_t (*buf)[sizeof(cqe->rsvd40)] = &cqe->rsvd40; - int ret = 1; - unsigned int i; + size_t tailroom; + tailroom = (uintptr_t)(txq->wqes) + + (1 << txq->wqe_n) * MLX5_WQE_SIZE - + (uintptr_t)addr; + return tailroom; +} - for (i = 0; i < sizeof(magic) && i < sizeof(*buf); ++i) - if (!ret || (*buf)[i] != magic[i]) { - ret = 0; - (*buf)[i] = magic[i]; - } +/** + * Copy data to tailroom of circular queue. + * + * @param dst + * Pointer to destination. + * @param src + * Pointer to source. + * @param n + * Number of bytes to copy. + * @param base + * Pointer to head of queue. + * @param tailroom + * Size of tailroom from dst. + * + * @return + * Pointer after copied data. + */ +static inline void * +mlx5_copy_to_wq(void *dst, const void *src, size_t n, + void *base, size_t tailroom) +{ + void *ret; + + if (n > tailroom) { + rte_memcpy(dst, src, tailroom); + rte_memcpy(base, (void *)((uintptr_t)src + tailroom), + n - tailroom); + ret = (uint8_t *)base + n - tailroom; + } else { + rte_memcpy(dst, src, n); + ret = (n == tailroom) ? base : (uint8_t *)dst + n; + } return ret; } -#endif /* NDEBUG */ - -static inline int -check_cqe64(volatile struct mlx5_cqe64 *cqe, - unsigned int cqes_n, const uint16_t ci) - __attribute__((always_inline)); - /** - * Check whether CQE is valid. + * DPDK callback to check the status of a tx descriptor. * - * @param cqe - * Pointer to CQE. - * @param cqes_n - * Size of completion queue. - * @param ci - * Consumer index. + * @param tx_queue + * The tx queue. + * @param[in] offset + * The index of the descriptor in the ring. * * @return - * 0 on success, 1 on failure. + * The status of the tx descriptor. */ -static inline int -check_cqe64(volatile struct mlx5_cqe64 *cqe, - unsigned int cqes_n, const uint16_t ci) +int +mlx5_tx_descriptor_status(void *tx_queue, uint16_t offset) { - uint16_t idx = ci & cqes_n; - uint8_t op_own = cqe->op_own; - uint8_t op_owner = MLX5_CQE_OWNER(op_own); - uint8_t op_code = MLX5_CQE_OPCODE(op_own); + struct mlx5_txq_data *txq = tx_queue; + uint16_t used; - if (unlikely((op_owner != (!!(idx))) || (op_code == MLX5_CQE_INVALID))) - return 1; /* No CQE. */ -#ifndef NDEBUG - if ((op_code == MLX5_CQE_RESP_ERR) || - (op_code == MLX5_CQE_REQ_ERR)) { - volatile struct mlx5_err_cqe *err_cqe = (volatile void *)cqe; - uint8_t syndrome = err_cqe->syndrome; - - if ((syndrome == MLX5_CQE_SYNDROME_LOCAL_LENGTH_ERR) || - (syndrome == MLX5_CQE_SYNDROME_REMOTE_ABORTED_ERR)) - return 0; - if (!check_cqe64_seen(cqe)) - ERROR("unexpected CQE error %u (0x%02x)" - " syndrome 0x%02x", - op_code, op_code, syndrome); - return 1; - } else if ((op_code != MLX5_CQE_RESP_SEND) && - (op_code != MLX5_CQE_REQ)) { - if (!check_cqe64_seen(cqe)) - ERROR("unexpected CQE opcode %u (0x%02x)", - op_code, op_code); - return 1; - } -#endif /* NDEBUG */ - return 0; + mlx5_tx_complete(txq); + used = txq->elts_head - txq->elts_tail; + if (offset < used) + return RTE_ETH_TX_DESC_FULL; + return RTE_ETH_TX_DESC_DONE; } /** - * Manage TX completions. + * DPDK callback to check the status of a rx descriptor. * - * When sending a burst, mlx5_tx_burst() posts several WRs. + * @param rx_queue + * The rx queue. + * @param[in] offset + * The index of the descriptor in the ring. * - * @param txq - * Pointer to TX queue structure. + * @return + * The status of the tx descriptor. */ -static void -txq_complete(struct txq *txq) +int +mlx5_rx_descriptor_status(void *rx_queue, uint16_t offset) { - const unsigned int elts_n = txq->elts_n; - const unsigned int cqe_n = txq->cqe_n; + struct mlx5_rxq_data *rxq = rx_queue; + struct rxq_zip *zip = &rxq->zip; + volatile struct mlx5_cqe *cqe; + const unsigned int cqe_n = (1 << rxq->cqe_n); const unsigned int cqe_cnt = cqe_n - 1; - uint16_t elts_free = txq->elts_tail; - uint16_t elts_tail; - uint16_t cq_ci = txq->cq_ci; - volatile struct mlx5_cqe64 *cqe = NULL; - volatile union mlx5_wqe *wqe; + unsigned int cq_ci; + unsigned int used; - do { - volatile struct mlx5_cqe64 *tmp; - - tmp = &(*txq->cqes)[cq_ci & cqe_cnt].cqe64; - if (check_cqe64(tmp, cqe_n, cq_ci)) - break; - cqe = tmp; -#ifndef NDEBUG - if (MLX5_CQE_FORMAT(cqe->op_own) == MLX5_COMPRESSED) { - if (!check_cqe64_seen(cqe)) - ERROR("unexpected compressed CQE, TX stopped"); - return; - } - if ((MLX5_CQE_OPCODE(cqe->op_own) == MLX5_CQE_RESP_ERR) || - (MLX5_CQE_OPCODE(cqe->op_own) == MLX5_CQE_REQ_ERR)) { - if (!check_cqe64_seen(cqe)) - ERROR("unexpected error CQE, TX stopped"); - return; - } -#endif /* NDEBUG */ - ++cq_ci; - } while (1); - if (unlikely(cqe == NULL)) - return; - wqe = &(*txq->wqes)[htons(cqe->wqe_counter) & (txq->wqe_n - 1)]; - elts_tail = wqe->wqe.ctrl.data[3]; - assert(elts_tail < txq->wqe_n); - /* Free buffers. */ - while (elts_free != elts_tail) { - struct rte_mbuf *elt = (*txq->elts)[elts_free]; - unsigned int elts_free_next = - (elts_free + 1) & (elts_n - 1); - struct rte_mbuf *elt_next = (*txq->elts)[elts_free_next]; - -#ifndef NDEBUG - /* Poisoning. */ - memset(&(*txq->elts)[elts_free], - 0x66, - sizeof((*txq->elts)[elts_free])); -#endif - RTE_MBUF_PREFETCH_TO_FREE(elt_next); - /* Only one segment needs to be freed. */ - rte_pktmbuf_free_seg(elt); - elts_free = elts_free_next; - } - txq->cq_ci = cq_ci; - txq->elts_tail = elts_tail; - /* Update the consumer index. */ - rte_wmb(); - *txq->cq_db = htonl(cq_ci); -} - -/** - * Get Memory Pool (MP) from mbuf. If mbuf is indirect, the pool from which - * the cloned mbuf is allocated is returned instead. - * - * @param buf - * Pointer to mbuf. - * - * @return - * Memory pool where data is located for given mbuf. - */ -static struct rte_mempool * -txq_mb2mp(struct rte_mbuf *buf) -{ - if (unlikely(RTE_MBUF_INDIRECT(buf))) - return rte_mbuf_from_indirect(buf)->pool; - return buf->pool; -} - -static inline uint32_t -txq_mp2mr(struct txq *txq, struct rte_mempool *mp) - __attribute__((always_inline)); - -/** - * Get Memory Region (MR) <-> Memory Pool (MP) association from txq->mp2mr[]. - * Add MP to txq->mp2mr[] if it's not registered yet. If mp2mr[] is full, - * remove an entry first. - * - * @param txq - * Pointer to TX queue structure. - * @param[in] mp - * Memory Pool for which a Memory Region lkey must be returned. - * - * @return - * mr->lkey on success, (uint32_t)-1 on failure. - */ -static inline uint32_t -txq_mp2mr(struct txq *txq, struct rte_mempool *mp) -{ - unsigned int i; - uint32_t lkey = (uint32_t)-1; - - for (i = 0; (i != RTE_DIM(txq->mp2mr)); ++i) { - if (unlikely(txq->mp2mr[i].mp == NULL)) { - /* Unknown MP, add a new MR for it. */ - break; - } - if (txq->mp2mr[i].mp == mp) { - assert(txq->mp2mr[i].lkey != (uint32_t)-1); - assert(htonl(txq->mp2mr[i].mr->lkey) == - txq->mp2mr[i].lkey); - lkey = txq->mp2mr[i].lkey; - break; - } - } - if (unlikely(lkey == (uint32_t)-1)) - lkey = txq_mp2mr_reg(txq, mp, i); - return lkey; -} - -/** - * Write a regular WQE. - * - * @param txq - * Pointer to TX queue structure. - * @param wqe - * Pointer to the WQE to fill. - * @param buf - * Buffer. - * @param length - * Packet length. - * - * @return ds - * Number of DS elements consumed. - */ -static inline unsigned int -mlx5_wqe_write(struct txq *txq, volatile union mlx5_wqe *wqe, - struct rte_mbuf *buf, uint32_t length) -{ - uintptr_t raw = (uintptr_t)&wqe->wqe.eseg.inline_hdr_start; - uint16_t ds; - uint16_t pkt_inline_sz = 16; - uintptr_t addr = rte_pktmbuf_mtod(buf, uintptr_t); - struct mlx5_wqe_data_seg *dseg = NULL; - - assert(length >= 16); - /* Start the know and common part of the WQE structure. */ - wqe->wqe.ctrl.data[0] = htonl((txq->wqe_ci << 8) | MLX5_OPCODE_SEND); - wqe->wqe.ctrl.data[2] = 0; - wqe->wqe.ctrl.data[3] = 0; - wqe->wqe.eseg.rsvd0 = 0; - wqe->wqe.eseg.rsvd1 = 0; - wqe->wqe.eseg.mss = 0; - wqe->wqe.eseg.rsvd2 = 0; - /* Start by copying the Ethernet Header. */ - rte_mov16((uint8_t *)raw, (uint8_t *)addr); - length -= 16; - addr += 16; - /* Replace the Ethernet type by the VLAN if necessary. */ - if (buf->ol_flags & PKT_TX_VLAN_PKT) { - uint32_t vlan = htonl(0x81000000 | buf->vlan_tci); - - memcpy((uint8_t *)(raw + 16 - sizeof(vlan)), - &vlan, sizeof(vlan)); - addr -= sizeof(vlan); - length += sizeof(vlan); - } - /* Inline if enough room. */ - if (txq->max_inline != 0) { - uintptr_t end = (uintptr_t)&(*txq->wqes)[txq->wqe_n]; - uint16_t max_inline = txq->max_inline * RTE_CACHE_LINE_SIZE; - uint16_t room; - - raw += 16; - room = end - (uintptr_t)raw; - if (room > max_inline) { - uintptr_t addr_end = (addr + max_inline) & - ~(RTE_CACHE_LINE_SIZE - 1); - uint16_t copy_b = ((addr_end - addr) > length) ? - length : - (addr_end - addr); - - rte_memcpy((void *)raw, (void *)addr, copy_b); - addr += copy_b; - length -= copy_b; - pkt_inline_sz += copy_b; - /* Sanity check. */ - assert(addr <= addr_end); - } - /* Store the inlined packet size in the WQE. */ - wqe->wqe.eseg.inline_hdr_sz = htons(pkt_inline_sz); - /* - * 2 DWORDs consumed by the WQE header + 1 DSEG + - * the size of the inline part of the packet. - */ - ds = 2 + ((pkt_inline_sz - 2 + 15) / 16); - if (length > 0) { - dseg = (struct mlx5_wqe_data_seg *) - ((uintptr_t)wqe + (ds * 16)); - if ((uintptr_t)dseg >= end) - dseg = (struct mlx5_wqe_data_seg *) - ((uintptr_t)&(*txq->wqes)[0]); - goto use_dseg; - } + /* if we are processing a compressed cqe */ + if (zip->ai) { + used = zip->cqe_cnt - zip->ca; + cq_ci = zip->cq_ci; } else { - /* Add the remaining packet as a simple ds. */ - ds = 3; - /* - * No inline has been done in the packet, only the Ethernet - * Header as been stored. - */ - wqe->wqe.eseg.inline_hdr_sz = htons(16); - dseg = (struct mlx5_wqe_data_seg *) - ((uintptr_t)wqe + (ds * 16)); -use_dseg: - *dseg = (struct mlx5_wqe_data_seg) { - .addr = htonll(addr), - .byte_count = htonl(length), - .lkey = txq_mp2mr(txq, txq_mb2mp(buf)), - }; - ++ds; + used = 0; + cq_ci = rxq->cq_ci; } - wqe->wqe.ctrl.data[1] = htonl(txq->qp_num_8s | ds); - return ds; -} + cqe = &(*rxq->cqes)[cq_ci & cqe_cnt]; + while (check_cqe(cqe, cqe_n, cq_ci) == 0) { + int8_t op_own; + unsigned int n; -/** - * Ring TX queue doorbell. - * - * @param txq - * Pointer to TX queue structure. - */ -static inline void -mlx5_tx_dbrec(struct txq *txq) -{ - uint8_t *dst = (uint8_t *)((uintptr_t)txq->bf_reg + txq->bf_offset); - uint32_t data[4] = { - htonl((txq->wqe_ci << 8) | MLX5_OPCODE_SEND), - htonl(txq->qp_num_8s), - 0, - 0, - }; - rte_wmb(); - *txq->qp_db = htonl(txq->wqe_ci); - /* Ensure ordering between DB record and BF copy. */ - rte_wmb(); - rte_mov16(dst, (uint8_t *)data); - txq->bf_offset ^= txq->bf_buf_size; -} - -/** - * Prefetch a CQE. - * - * @param txq - * Pointer to TX queue structure. - * @param cqe_ci - * CQE consumer index. - */ -static inline void -tx_prefetch_cqe(struct txq *txq, uint16_t ci) -{ - volatile struct mlx5_cqe64 *cqe; - - cqe = &(*txq->cqes)[ci & (txq->cqe_n - 1)].cqe64; - rte_prefetch0(cqe); -} - -/** - * Prefetch a WQE. - * - * @param txq - * Pointer to TX queue structure. - * @param wqe_ci - * WQE consumer index. - */ -static inline void -tx_prefetch_wqe(struct txq *txq, uint16_t ci) -{ - volatile union mlx5_wqe *wqe; - - wqe = &(*txq->wqes)[ci & (txq->wqe_n - 1)]; - rte_prefetch0(wqe); + op_own = cqe->op_own; + if (MLX5_CQE_FORMAT(op_own) == MLX5_COMPRESSED) + n = rte_be_to_cpu_32(cqe->byte_cnt); + else + n = 1; + cq_ci += n; + used += n; + cqe = &(*rxq->cqes)[cq_ci & cqe_cnt]; + } + used = RTE_MIN(used, (1U << rxq->elts_n) - 1); + if (offset < used) + return RTE_ETH_RX_DESC_DONE; + return RTE_ETH_RX_DESC_AVAIL; } /** @@ -463,120 +336,375 @@ tx_prefetch_wqe(struct txq *txq, uint16_t ci) uint16_t mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) { - struct txq *txq = (struct txq *)dpdk_txq; + struct mlx5_txq_data *txq = (struct mlx5_txq_data *)dpdk_txq; uint16_t elts_head = txq->elts_head; - const unsigned int elts_n = txq->elts_n; + const uint16_t elts_n = 1 << txq->elts_n; + const uint16_t elts_m = elts_n - 1; unsigned int i = 0; unsigned int j = 0; - unsigned int max; + unsigned int k = 0; + uint16_t max_elts; + unsigned int max_inline = txq->max_inline; + const unsigned int inline_en = !!max_inline && txq->inline_en; + uint16_t max_wqe; unsigned int comp; - volatile union mlx5_wqe *wqe = NULL; + volatile struct mlx5_wqe_v *wqe = NULL; + volatile struct mlx5_wqe_ctrl *last_wqe = NULL; + unsigned int segs_n = 0; + struct rte_mbuf *buf = NULL; + uint8_t *raw; if (unlikely(!pkts_n)) return 0; /* Prefetch first packet cacheline. */ - tx_prefetch_cqe(txq, txq->cq_ci); - tx_prefetch_cqe(txq, txq->cq_ci + 1); rte_prefetch0(*pkts); /* Start processing. */ - txq_complete(txq); - max = (elts_n - (elts_head - txq->elts_tail)); - if (max > elts_n) - max -= elts_n; + mlx5_tx_complete(txq); + max_elts = (elts_n - (elts_head - txq->elts_tail)); + max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi); + if (unlikely(!max_wqe)) + return 0; do { - struct rte_mbuf *buf = *(pkts++); - unsigned int elts_head_next; + volatile rte_v128u32_t *dseg = NULL; uint32_t length; - unsigned int segs_n = buf->nb_segs; - volatile struct mlx5_wqe_data_seg *dseg; - unsigned int ds = sizeof(*wqe) / 16; + unsigned int ds = 0; + unsigned int sg = 0; /* counter of additional segs attached. */ + uintptr_t addr; + uint64_t naddr; + uint16_t pkt_inline_sz = MLX5_WQE_DWORD_SIZE + 2; + uint16_t tso_header_sz = 0; + uint16_t ehdr; + uint8_t cs_flags; + uint64_t tso = 0; + uint16_t tso_segsz = 0; +#ifdef MLX5_PMD_SOFT_COUNTERS + uint32_t total_length = 0; +#endif + /* first_seg */ + buf = *pkts; + segs_n = buf->nb_segs; /* * Make sure there is enough room to store this packet and * that one ring entry remains unused. */ assert(segs_n); - if (max < segs_n + 1) + if (max_elts < segs_n) break; - max -= segs_n; - --pkts_n; - elts_head_next = (elts_head + 1) & (elts_n - 1); - wqe = &(*txq->wqes)[txq->wqe_ci & (txq->wqe_n - 1)]; - tx_prefetch_wqe(txq, txq->wqe_ci); - tx_prefetch_wqe(txq, txq->wqe_ci + 1); - if (pkts_n) - rte_prefetch0(*pkts); + max_elts -= segs_n; + --segs_n; + if (unlikely(--max_wqe == 0)) + break; + wqe = (volatile struct mlx5_wqe_v *) + tx_mlx5_wqe(txq, txq->wqe_ci); + rte_prefetch0(tx_mlx5_wqe(txq, txq->wqe_ci + 1)); + if (pkts_n - i > 1) + rte_prefetch0(*(pkts + 1)); + addr = rte_pktmbuf_mtod(buf, uintptr_t); length = DATA_LEN(buf); - /* Update element. */ - (*txq->elts)[elts_head] = buf; - /* Prefetch next buffer data. */ - if (pkts_n) - rte_prefetch0(rte_pktmbuf_mtod(*pkts, - volatile void *)); - /* Should we enable HW CKSUM offload */ - if (buf->ol_flags & - (PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM)) { - wqe->wqe.eseg.cs_flags = - MLX5_ETH_WQE_L3_CSUM | - MLX5_ETH_WQE_L4_CSUM; - } else { - wqe->wqe.eseg.cs_flags = 0; - } - ds = mlx5_wqe_write(txq, wqe, buf, length); - if (segs_n == 1) - goto skip_segs; - dseg = (volatile struct mlx5_wqe_data_seg *) - (((uintptr_t)wqe) + ds * 16); - while (--segs_n) { - /* - * Spill on next WQE when the current one does not have - * enough room left. Size of WQE must a be a multiple - * of data segment size. - */ - assert(!(sizeof(*wqe) % sizeof(*dseg))); - if (!(ds % (sizeof(*wqe) / 16))) - dseg = (volatile void *) - &(*txq->wqes)[txq->wqe_ci++ & - (txq->wqe_n - 1)]; - else - ++dseg; - ++ds; - buf = buf->next; - assert(buf); - /* Store segment information. */ - dseg->byte_count = htonl(DATA_LEN(buf)); - dseg->lkey = txq_mp2mr(txq, txq_mb2mp(buf)); - dseg->addr = htonll(rte_pktmbuf_mtod(buf, uintptr_t)); - (*txq->elts)[elts_head_next] = buf; - elts_head_next = (elts_head_next + 1) & (elts_n - 1); + ehdr = (((uint8_t *)addr)[1] << 8) | + ((uint8_t *)addr)[0]; #ifdef MLX5_PMD_SOFT_COUNTERS - length += DATA_LEN(buf); + total_length = length; #endif - ++j; + if (length < (MLX5_WQE_DWORD_SIZE + 2)) { + txq->stats.oerrors++; + break; } - /* Update DS field in WQE. */ - wqe->wqe.ctrl.data[1] &= htonl(0xffffffc0); - wqe->wqe.ctrl.data[1] |= htonl(ds & 0x3f); -skip_segs: + /* Update element. */ + (*txq->elts)[elts_head & elts_m] = buf; + /* Prefetch next buffer data. */ + if (pkts_n - i > 1) + rte_prefetch0( + rte_pktmbuf_mtod(*(pkts + 1), volatile void *)); + cs_flags = txq_ol_cksum_to_cs(txq, buf); + raw = ((uint8_t *)(uintptr_t)wqe) + 2 * MLX5_WQE_DWORD_SIZE; + /* Replace the Ethernet type by the VLAN if necessary. */ + if (buf->ol_flags & PKT_TX_VLAN_PKT) { + uint32_t vlan = rte_cpu_to_be_32(0x81000000 | + buf->vlan_tci); + unsigned int len = 2 * ETHER_ADDR_LEN - 2; + + addr += 2; + length -= 2; + /* Copy Destination and source mac address. */ + memcpy((uint8_t *)raw, ((uint8_t *)addr), len); + /* Copy VLAN. */ + memcpy((uint8_t *)raw + len, &vlan, sizeof(vlan)); + /* Copy missing two bytes to end the DSeg. */ + memcpy((uint8_t *)raw + len + sizeof(vlan), + ((uint8_t *)addr) + len, 2); + addr += len + 2; + length -= (len + 2); + } else { + memcpy((uint8_t *)raw, ((uint8_t *)addr) + 2, + MLX5_WQE_DWORD_SIZE); + length -= pkt_inline_sz; + addr += pkt_inline_sz; + } + raw += MLX5_WQE_DWORD_SIZE; + if (txq->tso_en) { + tso = buf->ol_flags & PKT_TX_TCP_SEG; + if (tso) { + uintptr_t end = (uintptr_t) + (((uintptr_t)txq->wqes) + + (1 << txq->wqe_n) * + MLX5_WQE_SIZE); + unsigned int copy_b; + uint8_t vlan_sz = (buf->ol_flags & + PKT_TX_VLAN_PKT) ? 4 : 0; + const uint64_t is_tunneled = + buf->ol_flags & + (PKT_TX_TUNNEL_GRE | + PKT_TX_TUNNEL_VXLAN); + + tso_header_sz = buf->l2_len + vlan_sz + + buf->l3_len + buf->l4_len; + tso_segsz = buf->tso_segsz; + if (unlikely(tso_segsz == 0)) { + txq->stats.oerrors++; + break; + } + if (is_tunneled && txq->tunnel_en) { + tso_header_sz += buf->outer_l2_len + + buf->outer_l3_len; + cs_flags |= MLX5_ETH_WQE_L4_INNER_CSUM; + } else { + cs_flags |= MLX5_ETH_WQE_L4_CSUM; + } + if (unlikely(tso_header_sz > + MLX5_MAX_TSO_HEADER)) { + txq->stats.oerrors++; + break; + } + copy_b = tso_header_sz - pkt_inline_sz; + /* First seg must contain all headers. */ + assert(copy_b <= length); + if (copy_b && + ((end - (uintptr_t)raw) > copy_b)) { + uint16_t n = (MLX5_WQE_DS(copy_b) - + 1 + 3) / 4; + + if (unlikely(max_wqe < n)) + break; + max_wqe -= n; + rte_memcpy((void *)raw, + (void *)addr, copy_b); + addr += copy_b; + length -= copy_b; + /* Include padding for TSO header. */ + copy_b = MLX5_WQE_DS(copy_b) * + MLX5_WQE_DWORD_SIZE; + pkt_inline_sz += copy_b; + raw += copy_b; + } else { + /* NOP WQE. */ + wqe->ctrl = (rte_v128u32_t){ + rte_cpu_to_be_32( + txq->wqe_ci << 8), + rte_cpu_to_be_32( + txq->qp_num_8s | 1), + 0, + 0, + }; + ds = 1; +#ifdef MLX5_PMD_SOFT_COUNTERS + total_length = 0; +#endif + k++; + goto next_wqe; + } + } + } + /* Inline if enough room. */ + if (inline_en || tso) { + uint32_t inl; + uintptr_t end = (uintptr_t) + (((uintptr_t)txq->wqes) + + (1 << txq->wqe_n) * MLX5_WQE_SIZE); + unsigned int inline_room = max_inline * + RTE_CACHE_LINE_SIZE - + (pkt_inline_sz - 2) - + !!tso * sizeof(inl); + uintptr_t addr_end = (addr + inline_room) & + ~(RTE_CACHE_LINE_SIZE - 1); + unsigned int copy_b = (addr_end > addr) ? + RTE_MIN((addr_end - addr), length) : + 0; + + if (copy_b && ((end - (uintptr_t)raw) > copy_b)) { + /* + * One Dseg remains in the current WQE. To + * keep the computation positive, it is + * removed after the bytes to Dseg conversion. + */ + uint16_t n = (MLX5_WQE_DS(copy_b) - 1 + 3) / 4; + + if (unlikely(max_wqe < n)) + break; + max_wqe -= n; + if (tso) { + inl = rte_cpu_to_be_32(copy_b | + MLX5_INLINE_SEG); + rte_memcpy((void *)raw, + (void *)&inl, sizeof(inl)); + raw += sizeof(inl); + pkt_inline_sz += sizeof(inl); + } + rte_memcpy((void *)raw, (void *)addr, copy_b); + addr += copy_b; + length -= copy_b; + pkt_inline_sz += copy_b; + } + /* + * 2 DWORDs consumed by the WQE header + ETH segment + + * the size of the inline part of the packet. + */ + ds = 2 + MLX5_WQE_DS(pkt_inline_sz - 2); + if (length > 0) { + if (ds % (MLX5_WQE_SIZE / + MLX5_WQE_DWORD_SIZE) == 0) { + if (unlikely(--max_wqe == 0)) + break; + dseg = (volatile rte_v128u32_t *) + tx_mlx5_wqe(txq, txq->wqe_ci + + ds / 4); + } else { + dseg = (volatile rte_v128u32_t *) + ((uintptr_t)wqe + + (ds * MLX5_WQE_DWORD_SIZE)); + } + goto use_dseg; + } else if (!segs_n) { + goto next_pkt; + } else { + /* dseg will be advance as part of next_seg */ + dseg = (volatile rte_v128u32_t *) + ((uintptr_t)wqe + + ((ds - 1) * MLX5_WQE_DWORD_SIZE)); + goto next_seg; + } + } else { + /* + * No inline has been done in the packet, only the + * Ethernet Header as been stored. + */ + dseg = (volatile rte_v128u32_t *) + ((uintptr_t)wqe + (3 * MLX5_WQE_DWORD_SIZE)); + ds = 3; +use_dseg: + /* Add the remaining packet as a simple ds. */ + naddr = rte_cpu_to_be_64(addr); + *dseg = (rte_v128u32_t){ + rte_cpu_to_be_32(length), + mlx5_tx_mb2mr(txq, buf), + naddr, + naddr >> 32, + }; + ++ds; + if (!segs_n) + goto next_pkt; + } +next_seg: + assert(buf); + assert(ds); + assert(wqe); + /* + * Spill on next WQE when the current one does not have + * enough room left. Size of WQE must a be a multiple + * of data segment size. + */ + assert(!(MLX5_WQE_SIZE % MLX5_WQE_DWORD_SIZE)); + if (!(ds % (MLX5_WQE_SIZE / MLX5_WQE_DWORD_SIZE))) { + if (unlikely(--max_wqe == 0)) + break; + dseg = (volatile rte_v128u32_t *) + tx_mlx5_wqe(txq, txq->wqe_ci + ds / 4); + rte_prefetch0(tx_mlx5_wqe(txq, + txq->wqe_ci + ds / 4 + 1)); + } else { + ++dseg; + } + ++ds; + buf = buf->next; + assert(buf); + length = DATA_LEN(buf); +#ifdef MLX5_PMD_SOFT_COUNTERS + total_length += length; +#endif + /* Store segment information. */ + naddr = rte_cpu_to_be_64(rte_pktmbuf_mtod(buf, uintptr_t)); + *dseg = (rte_v128u32_t){ + rte_cpu_to_be_32(length), + mlx5_tx_mb2mr(txq, buf), + naddr, + naddr >> 32, + }; + (*txq->elts)[++elts_head & elts_m] = buf; + ++sg; + /* Advance counter only if all segs are successfully posted. */ + if (sg < segs_n) + goto next_seg; + else + j += sg; +next_pkt: + if (ds > MLX5_DSEG_MAX) { + txq->stats.oerrors++; + break; + } + ++elts_head; + ++pkts; + ++i; + /* Initialize known and common part of the WQE structure. */ + if (tso) { + wqe->ctrl = (rte_v128u32_t){ + rte_cpu_to_be_32((txq->wqe_ci << 8) | + MLX5_OPCODE_TSO), + rte_cpu_to_be_32(txq->qp_num_8s | ds), + 0, + 0, + }; + wqe->eseg = (rte_v128u32_t){ + 0, + cs_flags | (rte_cpu_to_be_16(tso_segsz) << 16), + 0, + (ehdr << 16) | rte_cpu_to_be_16(tso_header_sz), + }; + } else { + wqe->ctrl = (rte_v128u32_t){ + rte_cpu_to_be_32((txq->wqe_ci << 8) | + MLX5_OPCODE_SEND), + rte_cpu_to_be_32(txq->qp_num_8s | ds), + 0, + 0, + }; + wqe->eseg = (rte_v128u32_t){ + 0, + cs_flags, + 0, + (ehdr << 16) | rte_cpu_to_be_16(pkt_inline_sz), + }; + } +next_wqe: + txq->wqe_ci += (ds + 3) / 4; + /* Save the last successful WQE for completion request */ + last_wqe = (volatile struct mlx5_wqe_ctrl *)wqe; #ifdef MLX5_PMD_SOFT_COUNTERS /* Increment sent bytes counter. */ - txq->stats.obytes += length; + txq->stats.obytes += total_length; #endif - /* Increment consumer index. */ - txq->wqe_ci += (ds + 3) / 4; - elts_head = elts_head_next; - ++i; - } while (pkts_n); + } while (i < pkts_n); /* Take a shortcut if nothing must be sent. */ - if (unlikely(i == 0)) + if (unlikely((i + k) == 0)) return 0; + txq->elts_head += (i + j); /* Check whether completion threshold has been reached. */ - comp = txq->elts_comp + i + j; + comp = txq->elts_comp + i + j + k; if (comp >= MLX5_TX_COMP_THRESH) { /* Request completion on last WQE. */ - wqe->wqe.ctrl.data[2] = htonl(8); + last_wqe->ctrl2 = rte_cpu_to_be_32(8); /* Save elts_head in unused "immediate" field of WQE. */ - wqe->wqe.ctrl.data[3] = elts_head; + last_wqe->ctrl3 = txq->elts_head; txq->elts_comp = 0; } else { txq->elts_comp = comp; @@ -586,8 +714,7 @@ skip_segs: txq->stats.opackets += i; #endif /* Ring QP doorbell. */ - mlx5_tx_dbrec(txq); - txq->elts_head = elts_head; + mlx5_tx_dbrec(txq, (volatile struct mlx5_wqe *)last_wqe); return i; } @@ -602,30 +729,32 @@ skip_segs: * Packet length. */ static inline void -mlx5_mpw_new(struct txq *txq, struct mlx5_mpw *mpw, uint32_t length) +mlx5_mpw_new(struct mlx5_txq_data *txq, struct mlx5_mpw *mpw, uint32_t length) { - uint16_t idx = txq->wqe_ci & (txq->wqe_n - 1); + uint16_t idx = txq->wqe_ci & ((1 << txq->wqe_n) - 1); volatile struct mlx5_wqe_data_seg (*dseg)[MLX5_MPW_DSEG_MAX] = (volatile struct mlx5_wqe_data_seg (*)[]) - (uintptr_t)&(*txq->wqes)[(idx + 1) & (txq->wqe_n - 1)]; + tx_mlx5_wqe(txq, idx + 1); mpw->state = MLX5_MPW_STATE_OPENED; mpw->pkts_n = 0; mpw->len = length; mpw->total_len = 0; - mpw->wqe = &(*txq->wqes)[idx]; - mpw->wqe->mpw.eseg.mss = htons(length); - mpw->wqe->mpw.eseg.inline_hdr_sz = 0; - mpw->wqe->mpw.eseg.rsvd0 = 0; - mpw->wqe->mpw.eseg.rsvd1 = 0; - mpw->wqe->mpw.eseg.rsvd2 = 0; - mpw->wqe->mpw.ctrl.data[0] = htonl((MLX5_OPC_MOD_MPW << 24) | - (txq->wqe_ci << 8) | - MLX5_OPCODE_TSO); - mpw->wqe->mpw.ctrl.data[2] = 0; - mpw->wqe->mpw.ctrl.data[3] = 0; - mpw->data.dseg[0] = &mpw->wqe->mpw.dseg[0]; - mpw->data.dseg[1] = &mpw->wqe->mpw.dseg[1]; + mpw->wqe = (volatile struct mlx5_wqe *)tx_mlx5_wqe(txq, idx); + mpw->wqe->eseg.mss = rte_cpu_to_be_16(length); + mpw->wqe->eseg.inline_hdr_sz = 0; + mpw->wqe->eseg.rsvd0 = 0; + mpw->wqe->eseg.rsvd1 = 0; + mpw->wqe->eseg.rsvd2 = 0; + mpw->wqe->ctrl[0] = rte_cpu_to_be_32((MLX5_OPC_MOD_MPW << 24) | + (txq->wqe_ci << 8) | + MLX5_OPCODE_TSO); + mpw->wqe->ctrl[2] = 0; + mpw->wqe->ctrl[3] = 0; + mpw->data.dseg[0] = (volatile struct mlx5_wqe_data_seg *) + (((uintptr_t)mpw->wqe) + (2 * MLX5_WQE_DWORD_SIZE)); + mpw->data.dseg[1] = (volatile struct mlx5_wqe_data_seg *) + (((uintptr_t)mpw->wqe) + (3 * MLX5_WQE_DWORD_SIZE)); mpw->data.dseg[2] = &(*dseg)[0]; mpw->data.dseg[3] = &(*dseg)[1]; mpw->data.dseg[4] = &(*dseg)[2]; @@ -640,7 +769,7 @@ mlx5_mpw_new(struct txq *txq, struct mlx5_mpw *mpw, uint32_t length) * Pointer to MPW session structure. */ static inline void -mlx5_mpw_close(struct txq *txq, struct mlx5_mpw *mpw) +mlx5_mpw_close(struct mlx5_txq_data *txq, struct mlx5_mpw *mpw) { unsigned int num = mpw->pkts_n; @@ -648,14 +777,14 @@ mlx5_mpw_close(struct txq *txq, struct mlx5_mpw *mpw) * Store size in multiple of 16 bytes. Control and Ethernet segments * count as 2. */ - mpw->wqe->mpw.ctrl.data[1] = htonl(txq->qp_num_8s | (2 + num)); + mpw->wqe->ctrl[1] = rte_cpu_to_be_32(txq->qp_num_8s | (2 + num)); mpw->state = MLX5_MPW_STATE_CLOSED; if (num < 3) ++txq->wqe_ci; else txq->wqe_ci += 2; - tx_prefetch_wqe(txq, txq->wqe_ci); - tx_prefetch_wqe(txq, txq->wqe_ci + 1); + rte_prefetch0(tx_mlx5_wqe(txq, txq->wqe_ci)); + rte_prefetch0(tx_mlx5_wqe(txq, txq->wqe_ci + 1)); } /** @@ -674,12 +803,14 @@ mlx5_mpw_close(struct txq *txq, struct mlx5_mpw *mpw) uint16_t mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) { - struct txq *txq = (struct txq *)dpdk_txq; + struct mlx5_txq_data *txq = (struct mlx5_txq_data *)dpdk_txq; uint16_t elts_head = txq->elts_head; - const unsigned int elts_n = txq->elts_n; + const uint16_t elts_n = 1 << txq->elts_n; + const uint16_t elts_m = elts_n - 1; unsigned int i = 0; unsigned int j = 0; - unsigned int max; + uint16_t max_elts; + uint16_t max_wqe; unsigned int comp; struct mlx5_mpw mpw = { .state = MLX5_MPW_STATE_CLOSED, @@ -688,37 +819,35 @@ mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) if (unlikely(!pkts_n)) return 0; /* Prefetch first packet cacheline. */ - tx_prefetch_cqe(txq, txq->cq_ci); - tx_prefetch_wqe(txq, txq->wqe_ci); - tx_prefetch_wqe(txq, txq->wqe_ci + 1); + rte_prefetch0(tx_mlx5_wqe(txq, txq->wqe_ci)); + rte_prefetch0(tx_mlx5_wqe(txq, txq->wqe_ci + 1)); /* Start processing. */ - txq_complete(txq); - max = (elts_n - (elts_head - txq->elts_tail)); - if (max > elts_n) - max -= elts_n; + mlx5_tx_complete(txq); + max_elts = (elts_n - (elts_head - txq->elts_tail)); + max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi); + if (unlikely(!max_wqe)) + return 0; do { struct rte_mbuf *buf = *(pkts++); - unsigned int elts_head_next; uint32_t length; unsigned int segs_n = buf->nb_segs; - uint32_t cs_flags = 0; + uint32_t cs_flags; /* * Make sure there is enough room to store this packet and * that one ring entry remains unused. */ assert(segs_n); - if (max < segs_n + 1) + if (max_elts < segs_n) break; /* Do not bother with large packets MPW cannot handle. */ - if (segs_n > MLX5_MPW_DSEG_MAX) + if (segs_n > MLX5_MPW_DSEG_MAX) { + txq->stats.oerrors++; break; - max -= segs_n; + } + max_elts -= segs_n; --pkts_n; - /* Should we enable HW CKSUM offload */ - if (buf->ol_flags & - (PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM)) - cs_flags = MLX5_ETH_WQE_L3_CSUM | MLX5_ETH_WQE_L4_CSUM; + cs_flags = txq_ol_cksum_to_cs(txq, buf); /* Retrieve packet information. */ length = PKT_LEN(buf); assert(length); @@ -726,11 +855,19 @@ mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) if ((mpw.state == MLX5_MPW_STATE_OPENED) && ((mpw.len != length) || (segs_n != 1) || - (mpw.wqe->mpw.eseg.cs_flags != cs_flags))) + (mpw.wqe->eseg.cs_flags != cs_flags))) mlx5_mpw_close(txq, &mpw); if (mpw.state == MLX5_MPW_STATE_CLOSED) { + /* + * Multi-Packet WQE consumes at most two WQE. + * mlx5_mpw_new() expects to be able to use such + * resources. + */ + if (unlikely(max_wqe < 2)) + break; + max_wqe -= 2; mlx5_mpw_new(txq, &mpw, length); - mpw.wqe->mpw.eseg.cs_flags = cs_flags; + mpw.wqe->eseg.cs_flags = cs_flags; } /* Multi-segment packets must be alone in their MPW. */ assert((segs_n == 1) || (mpw.pkts_n == 0)); @@ -741,17 +878,15 @@ mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) volatile struct mlx5_wqe_data_seg *dseg; uintptr_t addr; - elts_head_next = (elts_head + 1) & (elts_n - 1); assert(buf); - (*txq->elts)[elts_head] = buf; + (*txq->elts)[elts_head++ & elts_m] = buf; dseg = mpw.data.dseg[mpw.pkts_n]; addr = rte_pktmbuf_mtod(buf, uintptr_t); *dseg = (struct mlx5_wqe_data_seg){ - .byte_count = htonl(DATA_LEN(buf)), - .lkey = txq_mp2mr(txq, txq_mb2mp(buf)), - .addr = htonll(addr), + .byte_count = rte_cpu_to_be_32(DATA_LEN(buf)), + .lkey = mlx5_tx_mb2mr(txq, buf), + .addr = rte_cpu_to_be_64(addr), }; - elts_head = elts_head_next; #if defined(MLX5_PMD_SOFT_COUNTERS) || !defined(NDEBUG) length += DATA_LEN(buf); #endif @@ -762,7 +897,6 @@ mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) assert(length == mpw.len); if (mpw.pkts_n == MLX5_MPW_DSEG_MAX) mlx5_mpw_close(txq, &mpw); - elts_head = elts_head_next; #ifdef MLX5_PMD_SOFT_COUNTERS /* Increment sent bytes counter. */ txq->stats.obytes += length; @@ -776,12 +910,12 @@ mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) /* "j" includes both packets and segments. */ comp = txq->elts_comp + j; if (comp >= MLX5_TX_COMP_THRESH) { - volatile union mlx5_wqe *wqe = mpw.wqe; + volatile struct mlx5_wqe *wqe = mpw.wqe; /* Request completion on last WQE. */ - wqe->mpw.ctrl.data[2] = htonl(8); + wqe->ctrl[2] = rte_cpu_to_be_32(8); /* Save elts_head in unused "immediate" field of WQE. */ - wqe->mpw.ctrl.data[3] = elts_head; + wqe->ctrl[3] = elts_head; txq->elts_comp = 0; } else { txq->elts_comp = comp; @@ -793,7 +927,7 @@ mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) /* Ring QP doorbell. */ if (mpw.state == MLX5_MPW_STATE_OPENED) mlx5_mpw_close(txq, &mpw); - mlx5_tx_dbrec(txq); + mlx5_tx_dbrec(txq, mpw.wqe); txq->elts_head = elts_head; return i; } @@ -809,27 +943,31 @@ mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) * Packet length. */ static inline void -mlx5_mpw_inline_new(struct txq *txq, struct mlx5_mpw *mpw, uint32_t length) +mlx5_mpw_inline_new(struct mlx5_txq_data *txq, struct mlx5_mpw *mpw, + uint32_t length) { - uint16_t idx = txq->wqe_ci & (txq->wqe_n - 1); + uint16_t idx = txq->wqe_ci & ((1 << txq->wqe_n) - 1); + struct mlx5_wqe_inl_small *inl; mpw->state = MLX5_MPW_INL_STATE_OPENED; mpw->pkts_n = 0; mpw->len = length; mpw->total_len = 0; - mpw->wqe = &(*txq->wqes)[idx]; - mpw->wqe->mpw_inl.ctrl.data[0] = htonl((MLX5_OPC_MOD_MPW << 24) | - (txq->wqe_ci << 8) | - MLX5_OPCODE_TSO); - mpw->wqe->mpw_inl.ctrl.data[2] = 0; - mpw->wqe->mpw_inl.ctrl.data[3] = 0; - mpw->wqe->mpw_inl.eseg.mss = htons(length); - mpw->wqe->mpw_inl.eseg.inline_hdr_sz = 0; - mpw->wqe->mpw_inl.eseg.cs_flags = 0; - mpw->wqe->mpw_inl.eseg.rsvd0 = 0; - mpw->wqe->mpw_inl.eseg.rsvd1 = 0; - mpw->wqe->mpw_inl.eseg.rsvd2 = 0; - mpw->data.raw = &mpw->wqe->mpw_inl.data[0]; + mpw->wqe = (volatile struct mlx5_wqe *)tx_mlx5_wqe(txq, idx); + mpw->wqe->ctrl[0] = rte_cpu_to_be_32((MLX5_OPC_MOD_MPW << 24) | + (txq->wqe_ci << 8) | + MLX5_OPCODE_TSO); + mpw->wqe->ctrl[2] = 0; + mpw->wqe->ctrl[3] = 0; + mpw->wqe->eseg.mss = rte_cpu_to_be_16(length); + mpw->wqe->eseg.inline_hdr_sz = 0; + mpw->wqe->eseg.cs_flags = 0; + mpw->wqe->eseg.rsvd0 = 0; + mpw->wqe->eseg.rsvd1 = 0; + mpw->wqe->eseg.rsvd2 = 0; + inl = (struct mlx5_wqe_inl_small *) + (((uintptr_t)mpw->wqe) + 2 * MLX5_WQE_DWORD_SIZE); + mpw->data.raw = (uint8_t *)&inl->raw; } /** @@ -841,20 +979,22 @@ mlx5_mpw_inline_new(struct txq *txq, struct mlx5_mpw *mpw, uint32_t length) * Pointer to MPW session structure. */ static inline void -mlx5_mpw_inline_close(struct txq *txq, struct mlx5_mpw *mpw) +mlx5_mpw_inline_close(struct mlx5_txq_data *txq, struct mlx5_mpw *mpw) { unsigned int size; + struct mlx5_wqe_inl_small *inl = (struct mlx5_wqe_inl_small *) + (((uintptr_t)mpw->wqe) + (2 * MLX5_WQE_DWORD_SIZE)); - size = sizeof(*mpw->wqe) - MLX5_MWQE64_INL_DATA + mpw->total_len; + size = MLX5_WQE_SIZE - MLX5_MWQE64_INL_DATA + mpw->total_len; /* * Store size in multiple of 16 bytes. Control and Ethernet segments * count as 2. */ - mpw->wqe->mpw_inl.ctrl.data[1] = - htonl(txq->qp_num_8s | ((size + 15) / 16)); + mpw->wqe->ctrl[1] = rte_cpu_to_be_32(txq->qp_num_8s | + MLX5_WQE_DS(size)); mpw->state = MLX5_MPW_STATE_CLOSED; - mpw->wqe->mpw_inl.byte_cnt = htonl(mpw->total_len | MLX5_INLINE_SEG); - txq->wqe_ci += (size + (sizeof(*mpw->wqe) - 1)) / sizeof(*mpw->wqe); + inl->byte_cnt = rte_cpu_to_be_32(mpw->total_len | MLX5_INLINE_SEG); + txq->wqe_ci += (size + (MLX5_WQE_SIZE - 1)) / MLX5_WQE_SIZE; } /** @@ -874,66 +1014,80 @@ uint16_t mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) { - struct txq *txq = (struct txq *)dpdk_txq; + struct mlx5_txq_data *txq = (struct mlx5_txq_data *)dpdk_txq; uint16_t elts_head = txq->elts_head; - const unsigned int elts_n = txq->elts_n; + const uint16_t elts_n = 1 << txq->elts_n; + const uint16_t elts_m = elts_n - 1; unsigned int i = 0; unsigned int j = 0; - unsigned int max; + uint16_t max_elts; + uint16_t max_wqe; unsigned int comp; unsigned int inline_room = txq->max_inline * RTE_CACHE_LINE_SIZE; struct mlx5_mpw mpw = { .state = MLX5_MPW_STATE_CLOSED, }; + /* + * Compute the maximum number of WQE which can be consumed by inline + * code. + * - 2 DSEG for: + * - 1 control segment, + * - 1 Ethernet segment, + * - N Dseg from the inline request. + */ + const unsigned int wqe_inl_n = + ((2 * MLX5_WQE_DWORD_SIZE + + txq->max_inline * RTE_CACHE_LINE_SIZE) + + RTE_CACHE_LINE_SIZE - 1) / RTE_CACHE_LINE_SIZE; if (unlikely(!pkts_n)) return 0; /* Prefetch first packet cacheline. */ - tx_prefetch_cqe(txq, txq->cq_ci); - tx_prefetch_wqe(txq, txq->wqe_ci); - tx_prefetch_wqe(txq, txq->wqe_ci + 1); + rte_prefetch0(tx_mlx5_wqe(txq, txq->wqe_ci)); + rte_prefetch0(tx_mlx5_wqe(txq, txq->wqe_ci + 1)); /* Start processing. */ - txq_complete(txq); - max = (elts_n - (elts_head - txq->elts_tail)); - if (max > elts_n) - max -= elts_n; + mlx5_tx_complete(txq); + max_elts = (elts_n - (elts_head - txq->elts_tail)); do { struct rte_mbuf *buf = *(pkts++); - unsigned int elts_head_next; uintptr_t addr; uint32_t length; unsigned int segs_n = buf->nb_segs; - uint32_t cs_flags = 0; + uint8_t cs_flags; /* * Make sure there is enough room to store this packet and * that one ring entry remains unused. */ assert(segs_n); - if (max < segs_n + 1) + if (max_elts < segs_n) break; /* Do not bother with large packets MPW cannot handle. */ - if (segs_n > MLX5_MPW_DSEG_MAX) + if (segs_n > MLX5_MPW_DSEG_MAX) { + txq->stats.oerrors++; break; - max -= segs_n; + } + max_elts -= segs_n; --pkts_n; - /* Should we enable HW CKSUM offload */ - if (buf->ol_flags & - (PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM)) - cs_flags = MLX5_ETH_WQE_L3_CSUM | MLX5_ETH_WQE_L4_CSUM; + /* + * Compute max_wqe in case less WQE were consumed in previous + * iteration. + */ + max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi); + cs_flags = txq_ol_cksum_to_cs(txq, buf); /* Retrieve packet information. */ length = PKT_LEN(buf); /* Start new session if packet differs. */ if (mpw.state == MLX5_MPW_STATE_OPENED) { if ((mpw.len != length) || (segs_n != 1) || - (mpw.wqe->mpw.eseg.cs_flags != cs_flags)) + (mpw.wqe->eseg.cs_flags != cs_flags)) mlx5_mpw_close(txq, &mpw); } else if (mpw.state == MLX5_MPW_INL_STATE_OPENED) { if ((mpw.len != length) || (segs_n != 1) || (length > inline_room) || - (mpw.wqe->mpw_inl.eseg.cs_flags != cs_flags)) { + (mpw.wqe->eseg.cs_flags != cs_flags)) { mlx5_mpw_inline_close(txq, &mpw); inline_room = txq->max_inline * RTE_CACHE_LINE_SIZE; @@ -942,11 +1096,22 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts, if (mpw.state == MLX5_MPW_STATE_CLOSED) { if ((segs_n != 1) || (length > inline_room)) { + /* + * Multi-Packet WQE consumes at most two WQE. + * mlx5_mpw_new() expects to be able to use + * such resources. + */ + if (unlikely(max_wqe < 2)) + break; + max_wqe -= 2; mlx5_mpw_new(txq, &mpw, length); - mpw.wqe->mpw.eseg.cs_flags = cs_flags; + mpw.wqe->eseg.cs_flags = cs_flags; } else { + if (unlikely(max_wqe < wqe_inl_n)) + break; + max_wqe -= wqe_inl_n; mlx5_mpw_inline_new(txq, &mpw, length); - mpw.wqe->mpw_inl.eseg.cs_flags = cs_flags; + mpw.wqe->eseg.cs_flags = cs_flags; } } /* Multi-segment packets must be alone in their MPW. */ @@ -960,18 +1125,16 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts, do { volatile struct mlx5_wqe_data_seg *dseg; - elts_head_next = - (elts_head + 1) & (elts_n - 1); assert(buf); - (*txq->elts)[elts_head] = buf; + (*txq->elts)[elts_head++ & elts_m] = buf; dseg = mpw.data.dseg[mpw.pkts_n]; addr = rte_pktmbuf_mtod(buf, uintptr_t); *dseg = (struct mlx5_wqe_data_seg){ - .byte_count = htonl(DATA_LEN(buf)), - .lkey = txq_mp2mr(txq, txq_mb2mp(buf)), - .addr = htonll(addr), + .byte_count = + rte_cpu_to_be_32(DATA_LEN(buf)), + .lkey = mlx5_tx_mb2mr(txq, buf), + .addr = rte_cpu_to_be_64(addr), }; - elts_head = elts_head_next; #if defined(MLX5_PMD_SOFT_COUNTERS) || !defined(NDEBUG) length += DATA_LEN(buf); #endif @@ -988,18 +1151,18 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts, assert(mpw.state == MLX5_MPW_INL_STATE_OPENED); assert(length <= inline_room); assert(length == DATA_LEN(buf)); - elts_head_next = (elts_head + 1) & (elts_n - 1); addr = rte_pktmbuf_mtod(buf, uintptr_t); - (*txq->elts)[elts_head] = buf; + (*txq->elts)[elts_head++ & elts_m] = buf; /* Maximum number of bytes before wrapping. */ - max = ((uintptr_t)&(*txq->wqes)[txq->wqe_n] - + max = ((((uintptr_t)(txq->wqes)) + + (1 << txq->wqe_n) * + MLX5_WQE_SIZE) - (uintptr_t)mpw.data.raw); if (length > max) { rte_memcpy((void *)(uintptr_t)mpw.data.raw, (void *)addr, max); - mpw.data.raw = - (volatile void *)&(*txq->wqes)[0]; + mpw.data.raw = (volatile void *)txq->wqes; rte_memcpy((void *)(uintptr_t)mpw.data.raw, (void *)(addr + max), length - max); @@ -1008,13 +1171,15 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts, rte_memcpy((void *)(uintptr_t)mpw.data.raw, (void *)addr, length); - mpw.data.raw += length; + + if (length == max) + mpw.data.raw = + (volatile void *)txq->wqes; + else + mpw.data.raw += length; } - if ((uintptr_t)mpw.data.raw == - (uintptr_t)&(*txq->wqes)[txq->wqe_n]) - mpw.data.raw = - (volatile void *)&(*txq->wqes)[0]; ++mpw.pkts_n; + mpw.total_len += length; ++j; if (mpw.pkts_n == MLX5_MPW_DSEG_MAX) { mlx5_mpw_inline_close(txq, &mpw); @@ -1024,8 +1189,6 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts, inline_room -= length; } } - mpw.total_len += length; - elts_head = elts_head_next; #ifdef MLX5_PMD_SOFT_COUNTERS /* Increment sent bytes counter. */ txq->stats.obytes += length; @@ -1039,12 +1202,12 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts, /* "j" includes both packets and segments. */ comp = txq->elts_comp + j; if (comp >= MLX5_TX_COMP_THRESH) { - volatile union mlx5_wqe *wqe = mpw.wqe; + volatile struct mlx5_wqe *wqe = mpw.wqe; /* Request completion on last WQE. */ - wqe->mpw_inl.ctrl.data[2] = htonl(8); + wqe->ctrl[2] = rte_cpu_to_be_32(8); /* Save elts_head in unused "immediate" field of WQE. */ - wqe->mpw_inl.ctrl.data[3] = elts_head; + wqe->ctrl[3] = elts_head; txq->elts_comp = 0; } else { txq->elts_comp = comp; @@ -1058,7 +1221,355 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts, mlx5_mpw_inline_close(txq, &mpw); else if (mpw.state == MLX5_MPW_STATE_OPENED) mlx5_mpw_close(txq, &mpw); - mlx5_tx_dbrec(txq); + mlx5_tx_dbrec(txq, mpw.wqe); + txq->elts_head = elts_head; + return i; +} + +/** + * Open an Enhanced MPW session. + * + * @param txq + * Pointer to TX queue structure. + * @param mpw + * Pointer to MPW session structure. + * @param length + * Packet length. + */ +static inline void +mlx5_empw_new(struct mlx5_txq_data *txq, struct mlx5_mpw *mpw, int padding) +{ + uint16_t idx = txq->wqe_ci & ((1 << txq->wqe_n) - 1); + + mpw->state = MLX5_MPW_ENHANCED_STATE_OPENED; + mpw->pkts_n = 0; + mpw->total_len = sizeof(struct mlx5_wqe); + mpw->wqe = (volatile struct mlx5_wqe *)tx_mlx5_wqe(txq, idx); + mpw->wqe->ctrl[0] = + rte_cpu_to_be_32((MLX5_OPC_MOD_ENHANCED_MPSW << 24) | + (txq->wqe_ci << 8) | + MLX5_OPCODE_ENHANCED_MPSW); + mpw->wqe->ctrl[2] = 0; + mpw->wqe->ctrl[3] = 0; + memset((void *)(uintptr_t)&mpw->wqe->eseg, 0, MLX5_WQE_DWORD_SIZE); + if (unlikely(padding)) { + uintptr_t addr = (uintptr_t)(mpw->wqe + 1); + + /* Pad the first 2 DWORDs with zero-length inline header. */ + *(volatile uint32_t *)addr = rte_cpu_to_be_32(MLX5_INLINE_SEG); + *(volatile uint32_t *)(addr + MLX5_WQE_DWORD_SIZE) = + rte_cpu_to_be_32(MLX5_INLINE_SEG); + mpw->total_len += 2 * MLX5_WQE_DWORD_SIZE; + /* Start from the next WQEBB. */ + mpw->data.raw = (volatile void *)(tx_mlx5_wqe(txq, idx + 1)); + } else { + mpw->data.raw = (volatile void *)(mpw->wqe + 1); + } +} + +/** + * Close an Enhanced MPW session. + * + * @param txq + * Pointer to TX queue structure. + * @param mpw + * Pointer to MPW session structure. + * + * @return + * Number of consumed WQEs. + */ +static inline uint16_t +mlx5_empw_close(struct mlx5_txq_data *txq, struct mlx5_mpw *mpw) +{ + uint16_t ret; + + /* Store size in multiple of 16 bytes. Control and Ethernet segments + * count as 2. + */ + mpw->wqe->ctrl[1] = rte_cpu_to_be_32(txq->qp_num_8s | + MLX5_WQE_DS(mpw->total_len)); + mpw->state = MLX5_MPW_STATE_CLOSED; + ret = (mpw->total_len + (MLX5_WQE_SIZE - 1)) / MLX5_WQE_SIZE; + txq->wqe_ci += ret; + return ret; +} + +/** + * DPDK callback for TX with Enhanced MPW support. + * + * @param dpdk_txq + * Generic pointer to TX queue structure. + * @param[in] pkts + * Packets to transmit. + * @param pkts_n + * Number of packets in array. + * + * @return + * Number of packets successfully transmitted (<= pkts_n). + */ +uint16_t +mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) +{ + struct mlx5_txq_data *txq = (struct mlx5_txq_data *)dpdk_txq; + uint16_t elts_head = txq->elts_head; + const uint16_t elts_n = 1 << txq->elts_n; + const uint16_t elts_m = elts_n - 1; + unsigned int i = 0; + unsigned int j = 0; + uint16_t max_elts; + uint16_t max_wqe; + unsigned int max_inline = txq->max_inline * RTE_CACHE_LINE_SIZE; + unsigned int mpw_room = 0; + unsigned int inl_pad = 0; + uint32_t inl_hdr; + struct mlx5_mpw mpw = { + .state = MLX5_MPW_STATE_CLOSED, + }; + + if (unlikely(!pkts_n)) + return 0; + /* Start processing. */ + mlx5_tx_complete(txq); + max_elts = (elts_n - (elts_head - txq->elts_tail)); + /* A CQE slot must always be available. */ + assert((1u << txq->cqe_n) - (txq->cq_pi - txq->cq_ci)); + max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi); + if (unlikely(!max_wqe)) + return 0; + do { + struct rte_mbuf *buf = *(pkts++); + uintptr_t addr; + uint64_t naddr; + unsigned int n; + unsigned int do_inline = 0; /* Whether inline is possible. */ + uint32_t length; + unsigned int segs_n = buf->nb_segs; + uint8_t cs_flags; + + /* + * Make sure there is enough room to store this packet and + * that one ring entry remains unused. + */ + assert(segs_n); + if (max_elts - j < segs_n) + break; + /* Do not bother with large packets MPW cannot handle. */ + if (segs_n > MLX5_MPW_DSEG_MAX) { + txq->stats.oerrors++; + break; + } + cs_flags = txq_ol_cksum_to_cs(txq, buf); + /* Retrieve packet information. */ + length = PKT_LEN(buf); + /* Start new session if: + * - multi-segment packet + * - no space left even for a dseg + * - next packet can be inlined with a new WQE + * - cs_flag differs + * It can't be MLX5_MPW_STATE_OPENED as always have a single + * segmented packet. + */ + if (mpw.state == MLX5_MPW_ENHANCED_STATE_OPENED) { + if ((segs_n != 1) || + (inl_pad + sizeof(struct mlx5_wqe_data_seg) > + mpw_room) || + (length <= txq->inline_max_packet_sz && + inl_pad + sizeof(inl_hdr) + length > + mpw_room) || + (mpw.wqe->eseg.cs_flags != cs_flags)) + max_wqe -= mlx5_empw_close(txq, &mpw); + } + if (unlikely(mpw.state == MLX5_MPW_STATE_CLOSED)) { + if (unlikely(segs_n != 1)) { + /* Fall back to legacy MPW. + * A MPW session consumes 2 WQEs at most to + * include MLX5_MPW_DSEG_MAX pointers. + */ + if (unlikely(max_wqe < 2)) + break; + mlx5_mpw_new(txq, &mpw, length); + } else { + /* In Enhanced MPW, inline as much as the budget + * is allowed. The remaining space is to be + * filled with dsegs. If the title WQEBB isn't + * padded, it will have 2 dsegs there. + */ + mpw_room = RTE_MIN(MLX5_WQE_SIZE_MAX, + (max_inline ? max_inline : + pkts_n * MLX5_WQE_DWORD_SIZE) + + MLX5_WQE_SIZE); + if (unlikely(max_wqe * MLX5_WQE_SIZE < + mpw_room)) + break; + /* Don't pad the title WQEBB to not waste WQ. */ + mlx5_empw_new(txq, &mpw, 0); + mpw_room -= mpw.total_len; + inl_pad = 0; + do_inline = + length <= txq->inline_max_packet_sz && + sizeof(inl_hdr) + length <= mpw_room && + !txq->mpw_hdr_dseg; + } + mpw.wqe->eseg.cs_flags = cs_flags; + } else { + /* Evaluate whether the next packet can be inlined. + * Inlininig is possible when: + * - length is less than configured value + * - length fits for remaining space + * - not required to fill the title WQEBB with dsegs + */ + do_inline = + length <= txq->inline_max_packet_sz && + inl_pad + sizeof(inl_hdr) + length <= + mpw_room && + (!txq->mpw_hdr_dseg || + mpw.total_len >= MLX5_WQE_SIZE); + } + /* Multi-segment packets must be alone in their MPW. */ + assert((segs_n == 1) || (mpw.pkts_n == 0)); + if (unlikely(mpw.state == MLX5_MPW_STATE_OPENED)) { +#if defined(MLX5_PMD_SOFT_COUNTERS) || !defined(NDEBUG) + length = 0; +#endif + do { + volatile struct mlx5_wqe_data_seg *dseg; + + assert(buf); + (*txq->elts)[elts_head++ & elts_m] = buf; + dseg = mpw.data.dseg[mpw.pkts_n]; + addr = rte_pktmbuf_mtod(buf, uintptr_t); + *dseg = (struct mlx5_wqe_data_seg){ + .byte_count = rte_cpu_to_be_32( + DATA_LEN(buf)), + .lkey = mlx5_tx_mb2mr(txq, buf), + .addr = rte_cpu_to_be_64(addr), + }; +#if defined(MLX5_PMD_SOFT_COUNTERS) || !defined(NDEBUG) + length += DATA_LEN(buf); +#endif + buf = buf->next; + ++j; + ++mpw.pkts_n; + } while (--segs_n); + /* A multi-segmented packet takes one MPW session. + * TODO: Pack more multi-segmented packets if possible. + */ + mlx5_mpw_close(txq, &mpw); + if (mpw.pkts_n < 3) + max_wqe--; + else + max_wqe -= 2; + } else if (do_inline) { + /* Inline packet into WQE. */ + unsigned int max; + + assert(mpw.state == MLX5_MPW_ENHANCED_STATE_OPENED); + assert(length == DATA_LEN(buf)); + inl_hdr = rte_cpu_to_be_32(length | MLX5_INLINE_SEG); + addr = rte_pktmbuf_mtod(buf, uintptr_t); + mpw.data.raw = (volatile void *) + ((uintptr_t)mpw.data.raw + inl_pad); + max = tx_mlx5_wq_tailroom(txq, + (void *)(uintptr_t)mpw.data.raw); + /* Copy inline header. */ + mpw.data.raw = (volatile void *) + mlx5_copy_to_wq( + (void *)(uintptr_t)mpw.data.raw, + &inl_hdr, + sizeof(inl_hdr), + (void *)(uintptr_t)txq->wqes, + max); + max = tx_mlx5_wq_tailroom(txq, + (void *)(uintptr_t)mpw.data.raw); + /* Copy packet data. */ + mpw.data.raw = (volatile void *) + mlx5_copy_to_wq( + (void *)(uintptr_t)mpw.data.raw, + (void *)addr, + length, + (void *)(uintptr_t)txq->wqes, + max); + ++mpw.pkts_n; + mpw.total_len += (inl_pad + sizeof(inl_hdr) + length); + /* No need to get completion as the entire packet is + * copied to WQ. Free the buf right away. + */ + rte_pktmbuf_free_seg(buf); + mpw_room -= (inl_pad + sizeof(inl_hdr) + length); + /* Add pad in the next packet if any. */ + inl_pad = (((uintptr_t)mpw.data.raw + + (MLX5_WQE_DWORD_SIZE - 1)) & + ~(MLX5_WQE_DWORD_SIZE - 1)) - + (uintptr_t)mpw.data.raw; + } else { + /* No inline. Load a dseg of packet pointer. */ + volatile rte_v128u32_t *dseg; + + assert(mpw.state == MLX5_MPW_ENHANCED_STATE_OPENED); + assert((inl_pad + sizeof(*dseg)) <= mpw_room); + assert(length == DATA_LEN(buf)); + if (!tx_mlx5_wq_tailroom(txq, + (void *)((uintptr_t)mpw.data.raw + + inl_pad))) + dseg = (volatile void *)txq->wqes; + else + dseg = (volatile void *) + ((uintptr_t)mpw.data.raw + + inl_pad); + (*txq->elts)[elts_head++ & elts_m] = buf; + addr = rte_pktmbuf_mtod(buf, uintptr_t); + for (n = 0; n * RTE_CACHE_LINE_SIZE < length; n++) + rte_prefetch2((void *)(addr + + n * RTE_CACHE_LINE_SIZE)); + naddr = rte_cpu_to_be_64(addr); + *dseg = (rte_v128u32_t) { + rte_cpu_to_be_32(length), + mlx5_tx_mb2mr(txq, buf), + naddr, + naddr >> 32, + }; + mpw.data.raw = (volatile void *)(dseg + 1); + mpw.total_len += (inl_pad + sizeof(*dseg)); + ++j; + ++mpw.pkts_n; + mpw_room -= (inl_pad + sizeof(*dseg)); + inl_pad = 0; + } +#ifdef MLX5_PMD_SOFT_COUNTERS + /* Increment sent bytes counter. */ + txq->stats.obytes += length; +#endif + ++i; + } while (i < pkts_n); + /* Take a shortcut if nothing must be sent. */ + if (unlikely(i == 0)) + return 0; + /* Check whether completion threshold has been reached. */ + if (txq->elts_comp + j >= MLX5_TX_COMP_THRESH || + (uint16_t)(txq->wqe_ci - txq->mpw_comp) >= + (1 << txq->wqe_n) / MLX5_TX_COMP_THRESH_INLINE_DIV) { + volatile struct mlx5_wqe *wqe = mpw.wqe; + + /* Request completion on last WQE. */ + wqe->ctrl[2] = rte_cpu_to_be_32(8); + /* Save elts_head in unused "immediate" field of WQE. */ + wqe->ctrl[3] = elts_head; + txq->elts_comp = 0; + txq->mpw_comp = txq->wqe_ci; + txq->cq_pi++; + } else { + txq->elts_comp += j; + } +#ifdef MLX5_PMD_SOFT_COUNTERS + /* Increment sent packets counter. */ + txq->stats.opackets += i; +#endif + if (mpw.state == MLX5_MPW_ENHANCED_STATE_OPENED) + mlx5_empw_close(txq, &mpw); + else if (mpw.state == MLX5_MPW_STATE_OPENED) + mlx5_mpw_close(txq, &mpw); + /* Ring QP doorbell. */ + mlx5_tx_dbrec(txq, mpw.wqe); txq->elts_head = elts_head; return i; } @@ -1075,35 +1586,22 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts, * Packet type for struct rte_mbuf. */ static inline uint32_t -rxq_cq_to_pkt_type(volatile struct mlx5_cqe64 *cqe) +rxq_cq_to_pkt_type(volatile struct mlx5_cqe *cqe) { - uint32_t pkt_type; - uint8_t flags = cqe->l4_hdr_type_etc; - uint8_t info = cqe->rsvd0[0]; + uint8_t idx; + uint8_t pinfo = cqe->pkt_info; + uint16_t ptype = cqe->hdr_type_etc; - if (info & MLX5_CQE_RX_TUNNEL_PACKET) - pkt_type = - TRANSPOSE(flags, - MLX5_CQE_RX_OUTER_IPV4_PACKET, - RTE_PTYPE_L3_IPV4) | - TRANSPOSE(flags, - MLX5_CQE_RX_OUTER_IPV6_PACKET, - RTE_PTYPE_L3_IPV6) | - TRANSPOSE(flags, - MLX5_CQE_RX_IPV4_PACKET, - RTE_PTYPE_INNER_L3_IPV4) | - TRANSPOSE(flags, - MLX5_CQE_RX_IPV6_PACKET, - RTE_PTYPE_INNER_L3_IPV6); - else - pkt_type = - TRANSPOSE(flags, - MLX5_CQE_L3_HDR_TYPE_IPV6, - RTE_PTYPE_L3_IPV6) | - TRANSPOSE(flags, - MLX5_CQE_L3_HDR_TYPE_IPV4, - RTE_PTYPE_L3_IPV4); - return pkt_type; + /* + * The index to the array should have: + * bit[1:0] = l3_hdr_type + * bit[4:2] = l4_hdr_type + * bit[5] = ip_frag + * bit[6] = tunneled + * bit[7] = outer_l3_type + */ + idx = ((pinfo & 0x3) << 6) | ((ptype & 0xfc00) >> 10); + return mlx5_ptype_table[idx]; } /** @@ -1115,27 +1613,39 @@ rxq_cq_to_pkt_type(volatile struct mlx5_cqe64 *cqe) * Pointer to RX queue. * @param cqe * CQE to process. + * @param[out] rss_hash + * Packet RSS Hash result. * * @return * Packet size in bytes (0 if there is none), -1 in case of completion * with error. */ static inline int -mlx5_rx_poll_len(struct rxq *rxq, volatile struct mlx5_cqe64 *cqe, - uint16_t cqe_cnt) +mlx5_rx_poll_len(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe, + uint16_t cqe_cnt, uint32_t *rss_hash) { struct rxq_zip *zip = &rxq->zip; uint16_t cqe_n = cqe_cnt + 1; int len = 0; + uint16_t idx, end; /* Process compressed data in the CQE and mini arrays. */ if (zip->ai) { volatile struct mlx5_mini_cqe8 (*mc)[8] = (volatile struct mlx5_mini_cqe8 (*)[8]) - (uintptr_t)(&(*rxq->cqes)[zip->ca & cqe_cnt].cqe64); + (uintptr_t)(&(*rxq->cqes)[zip->ca & cqe_cnt].pkt_info); - len = ntohl((*mc)[zip->ai & 7].byte_cnt); + len = rte_be_to_cpu_32((*mc)[zip->ai & 7].byte_cnt); + *rss_hash = rte_be_to_cpu_32((*mc)[zip->ai & 7].rx_hash_result); if ((++zip->ai & 7) == 0) { + /* Invalidate consumed CQEs */ + idx = zip->ca; + end = zip->na; + while (idx != end) { + (*rxq->cqes)[idx & cqe_cnt].op_own = + MLX5_CQE_INVALIDATE; + ++idx; + } /* * Increment consumer index to skip the number of * CQEs consumed. Hardware leaves holes in the CQ @@ -1145,11 +1655,12 @@ mlx5_rx_poll_len(struct rxq *rxq, volatile struct mlx5_cqe64 *cqe, zip->na += 8; } if (unlikely(rxq->zip.ai == rxq->zip.cqe_cnt)) { - uint16_t idx = rxq->cq_ci; - uint16_t end = zip->cq_ci; + /* Invalidate the rest */ + idx = zip->ca; + end = zip->cq_ci; while (idx != end) { - (*rxq->cqes)[idx & cqe_cnt].cqe64.op_own = + (*rxq->cqes)[idx & cqe_cnt].op_own = MLX5_CQE_INVALIDATE; ++idx; } @@ -1161,7 +1672,7 @@ mlx5_rx_poll_len(struct rxq *rxq, volatile struct mlx5_cqe64 *cqe, int ret; int8_t op_own; - ret = check_cqe64(cqe, cqe_n, rxq->cq_ci); + ret = check_cqe(cqe, cqe_n, rxq->cq_ci); if (unlikely(ret == 1)) return 0; ++rxq->cq_ci; @@ -1170,10 +1681,10 @@ mlx5_rx_poll_len(struct rxq *rxq, volatile struct mlx5_cqe64 *cqe, volatile struct mlx5_mini_cqe8 (*mc)[8] = (volatile struct mlx5_mini_cqe8 (*)[8]) (uintptr_t)(&(*rxq->cqes)[rxq->cq_ci & - cqe_cnt].cqe64); + cqe_cnt].pkt_info); /* Fix endianness. */ - zip->cqe_cnt = ntohl(cqe->byte_cnt); + zip->cqe_cnt = rte_be_to_cpu_32(cqe->byte_cnt); /* * Current mini array position is the one returned by * check_cqe64(). @@ -1182,16 +1693,25 @@ mlx5_rx_poll_len(struct rxq *rxq, volatile struct mlx5_cqe64 *cqe, * special case the second one is located 7 CQEs after * the initial CQE instead of 8 for subsequent ones. */ - zip->ca = rxq->cq_ci & cqe_cnt; + zip->ca = rxq->cq_ci; zip->na = zip->ca + 7; /* Compute the next non compressed CQE. */ --rxq->cq_ci; zip->cq_ci = rxq->cq_ci + zip->cqe_cnt; /* Get packet size to return. */ - len = ntohl((*mc)[0].byte_cnt); + len = rte_be_to_cpu_32((*mc)[0].byte_cnt); + *rss_hash = rte_be_to_cpu_32((*mc)[0].rx_hash_result); zip->ai = 1; + /* Prefetch all the entries to be invalidated */ + idx = zip->ca; + end = zip->cq_ci; + while (idx != end) { + rte_prefetch0(&(*rxq->cqes)[(idx) & cqe_cnt]); + ++idx; + } } else { - len = ntohl(cqe->byte_cnt); + len = rte_be_to_cpu_32(cqe->byte_cnt); + *rss_hash = rte_be_to_cpu_32(cqe->rx_hash_res); } /* Error while receiving packet. */ if (unlikely(MLX5_CQE_OPCODE(op_own) == MLX5_CQE_RESP_ERR)) @@ -1212,38 +1732,26 @@ mlx5_rx_poll_len(struct rxq *rxq, volatile struct mlx5_cqe64 *cqe, * Offload flags (ol_flags) for struct rte_mbuf. */ static inline uint32_t -rxq_cq_to_ol_flags(struct rxq *rxq, volatile struct mlx5_cqe64 *cqe) +rxq_cq_to_ol_flags(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe) { uint32_t ol_flags = 0; - uint8_t l3_hdr = (cqe->l4_hdr_type_etc) & MLX5_CQE_L3_HDR_TYPE_MASK; - uint8_t l4_hdr = (cqe->l4_hdr_type_etc) & MLX5_CQE_L4_HDR_TYPE_MASK; - uint8_t info = cqe->rsvd0[0]; + uint16_t flags = rte_be_to_cpu_16(cqe->hdr_type_etc); - if ((l3_hdr == MLX5_CQE_L3_HDR_TYPE_IPV4) || - (l3_hdr == MLX5_CQE_L3_HDR_TYPE_IPV6)) + ol_flags = + TRANSPOSE(flags, + MLX5_CQE_RX_L3_HDR_VALID, + PKT_RX_IP_CKSUM_GOOD) | + TRANSPOSE(flags, + MLX5_CQE_RX_L4_HDR_VALID, + PKT_RX_L4_CKSUM_GOOD); + if ((cqe->pkt_info & MLX5_CQE_RX_TUNNEL_PACKET) && (rxq->csum_l2tun)) ol_flags |= - (!(cqe->hds_ip_ext & MLX5_CQE_L3_OK) * - PKT_RX_IP_CKSUM_BAD); - if ((l4_hdr == MLX5_CQE_L4_HDR_TYPE_TCP) || - (l4_hdr == MLX5_CQE_L4_HDR_TYPE_TCP_EMP_ACK) || - (l4_hdr == MLX5_CQE_L4_HDR_TYPE_TCP_ACK) || - (l4_hdr == MLX5_CQE_L4_HDR_TYPE_UDP)) - ol_flags |= - (!(cqe->hds_ip_ext & MLX5_CQE_L4_OK) * - PKT_RX_L4_CKSUM_BAD); - /* - * PKT_RX_IP_CKSUM_BAD and PKT_RX_L4_CKSUM_BAD are used in place - * of PKT_RX_EIP_CKSUM_BAD because the latter is not functional - * (its value is 0). - */ - if ((info & MLX5_CQE_RX_TUNNEL_PACKET) && (rxq->csum_l2tun)) - ol_flags |= - TRANSPOSE(~cqe->l4_hdr_type_etc, - MLX5_CQE_RX_OUTER_IP_CSUM_OK, - PKT_RX_IP_CKSUM_BAD) | - TRANSPOSE(~cqe->l4_hdr_type_etc, - MLX5_CQE_RX_OUTER_TCP_UDP_CSUM_OK, - PKT_RX_L4_CKSUM_BAD); + TRANSPOSE(flags, + MLX5_CQE_RX_L3_HDR_VALID, + PKT_RX_IP_CKSUM_GOOD) | + TRANSPOSE(flags, + MLX5_CQE_RX_L4_HDR_VALID, + PKT_RX_L4_CKSUM_GOOD); return ol_flags; } @@ -1263,22 +1771,23 @@ rxq_cq_to_ol_flags(struct rxq *rxq, volatile struct mlx5_cqe64 *cqe) uint16_t mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) { - struct rxq *rxq = dpdk_rxq; - const unsigned int wqe_cnt = rxq->elts_n - 1; - const unsigned int cqe_cnt = rxq->cqe_n - 1; + struct mlx5_rxq_data *rxq = dpdk_rxq; + const unsigned int wqe_cnt = (1 << rxq->elts_n) - 1; + const unsigned int cqe_cnt = (1 << rxq->cqe_n) - 1; const unsigned int sges_n = rxq->sges_n; struct rte_mbuf *pkt = NULL; struct rte_mbuf *seg = NULL; - volatile struct mlx5_cqe64 *cqe = - &(*rxq->cqes)[rxq->cq_ci & cqe_cnt].cqe64; + volatile struct mlx5_cqe *cqe = + &(*rxq->cqes)[rxq->cq_ci & cqe_cnt]; unsigned int i = 0; unsigned int rq_ci = rxq->rq_ci << sges_n; - int len; + int len = 0; /* keep its value across iterations. */ while (pkts_n) { unsigned int idx = rq_ci & wqe_cnt; volatile struct mlx5_wqe_data_seg *wqe = &(*rxq->wqes)[idx]; struct rte_mbuf *rep = (*rxq->elts)[idx]; + uint32_t rss_hash_res = 0; if (pkt) NEXT(seg) = rep; @@ -1298,65 +1807,79 @@ mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) } while (pkt != seg) { assert(pkt != (*rxq->elts)[idx]); - seg = NEXT(pkt); - rte_mbuf_refcnt_set(pkt, 0); - __rte_mbuf_raw_free(pkt); - pkt = seg; + rep = NEXT(pkt); + NEXT(pkt) = NULL; + NB_SEGS(pkt) = 1; + rte_mbuf_raw_free(pkt); + pkt = rep; } break; } if (!pkt) { - cqe = &(*rxq->cqes)[rxq->cq_ci & cqe_cnt].cqe64; - len = mlx5_rx_poll_len(rxq, cqe, cqe_cnt); - if (len == 0) { - rte_mbuf_refcnt_set(rep, 0); - __rte_mbuf_raw_free(rep); + cqe = &(*rxq->cqes)[rxq->cq_ci & cqe_cnt]; + len = mlx5_rx_poll_len(rxq, cqe, cqe_cnt, + &rss_hash_res); + if (!len) { + rte_mbuf_raw_free(rep); break; } if (unlikely(len == -1)) { /* RX error, packet is likely too large. */ - rte_mbuf_refcnt_set(rep, 0); - __rte_mbuf_raw_free(rep); + rte_mbuf_raw_free(rep); ++rxq->stats.idropped; goto skip; } pkt = seg; assert(len >= (rxq->crc_present << 2)); /* Update packet information. */ - pkt->packet_type = 0; + pkt->packet_type = rxq_cq_to_pkt_type(cqe); pkt->ol_flags = 0; - if (rxq->csum | rxq->csum_l2tun | rxq->vlan_strip | - rxq->crc_present) { - if (rxq->csum) { - pkt->packet_type = - rxq_cq_to_pkt_type(cqe); - pkt->ol_flags = - rxq_cq_to_ol_flags(rxq, cqe); - } - if (cqe->l4_hdr_type_etc & - MLX5_CQE_VLAN_STRIPPED) { - pkt->ol_flags |= PKT_RX_VLAN_PKT | - PKT_RX_VLAN_STRIPPED; - pkt->vlan_tci = ntohs(cqe->vlan_info); - } - if (rxq->crc_present) - len -= ETHER_CRC_LEN; + if (rss_hash_res && rxq->rss_hash) { + pkt->hash.rss = rss_hash_res; + pkt->ol_flags = PKT_RX_RSS_HASH; } + if (rxq->mark && + MLX5_FLOW_MARK_IS_VALID(cqe->sop_drop_qpn)) { + pkt->ol_flags |= PKT_RX_FDIR; + if (cqe->sop_drop_qpn != + rte_cpu_to_be_32(MLX5_FLOW_MARK_DEFAULT)) { + uint32_t mark = cqe->sop_drop_qpn; + + pkt->ol_flags |= PKT_RX_FDIR_ID; + pkt->hash.fdir.hi = + mlx5_flow_mark_get(mark); + } + } + if (rxq->csum | rxq->csum_l2tun) + pkt->ol_flags |= rxq_cq_to_ol_flags(rxq, cqe); + if (rxq->vlan_strip && + (cqe->hdr_type_etc & + rte_cpu_to_be_16(MLX5_CQE_VLAN_STRIPPED))) { + pkt->ol_flags |= PKT_RX_VLAN | + PKT_RX_VLAN_STRIPPED; + pkt->vlan_tci = + rte_be_to_cpu_16(cqe->vlan_info); + } + if (rxq->hw_timestamp) { + pkt->timestamp = + rte_be_to_cpu_64(cqe->timestamp); + pkt->ol_flags |= PKT_RX_TIMESTAMP; + } + if (rxq->crc_present) + len -= ETHER_CRC_LEN; PKT_LEN(pkt) = len; } DATA_LEN(rep) = DATA_LEN(seg); PKT_LEN(rep) = PKT_LEN(seg); SET_DATA_OFF(rep, DATA_OFF(seg)); - NB_SEGS(rep) = NB_SEGS(seg); PORT(rep) = PORT(seg); - NEXT(rep) = NULL; (*rxq->elts)[idx] = rep; /* * Fill NIC descriptor with the new buffer. The lkey and size * of the buffers are already known, only the buffer address * changes. */ - wqe->addr = htonll(rte_pktmbuf_mtod(rep, uintptr_t)); + wqe->addr = rte_cpu_to_be_64(rte_pktmbuf_mtod(rep, uintptr_t)); if (len > DATA_LEN(seg)) { len -= DATA_LEN(seg); ++NB_SEGS(pkt); @@ -1383,10 +1906,10 @@ skip: return 0; /* Update the consumer index. */ rxq->rq_ci = rq_ci >> sges_n; - rte_wmb(); - *rxq->cq_db = htonl(rxq->cq_ci); - rte_wmb(); - *rxq->rq_db = htonl(rxq->rq_ci); + rte_io_wmb(); + *rxq->cq_db = rte_cpu_to_be_32(rxq->cq_ci); + rte_io_wmb(); + *rxq->rq_db = rte_cpu_to_be_32(rxq->rq_ci); #ifdef MLX5_PMD_SOFT_COUNTERS /* Increment packets counter. */ rxq->stats.ipackets += i; @@ -1443,3 +1966,65 @@ removed_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) (void)pkts_n; return 0; } + +/* + * Vectorized Rx/Tx routines are not compiled in when required vector + * instructions are not supported on a target architecture. The following null + * stubs are needed for linkage when those are not included outside of this file + * (e.g. mlx5_rxtx_vec_sse.c for x86). + */ + +uint16_t __attribute__((weak)) +mlx5_tx_burst_raw_vec(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) +{ + (void)dpdk_txq; + (void)pkts; + (void)pkts_n; + return 0; +} + +uint16_t __attribute__((weak)) +mlx5_tx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) +{ + (void)dpdk_txq; + (void)pkts; + (void)pkts_n; + return 0; +} + +uint16_t __attribute__((weak)) +mlx5_rx_burst_vec(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) +{ + (void)dpdk_rxq; + (void)pkts; + (void)pkts_n; + return 0; +} + +int __attribute__((weak)) +priv_check_raw_vec_tx_support(struct priv *priv) +{ + (void)priv; + return -ENOTSUP; +} + +int __attribute__((weak)) +priv_check_vec_tx_support(struct priv *priv) +{ + (void)priv; + return -ENOTSUP; +} + +int __attribute__((weak)) +rxq_check_vec_support(struct mlx5_rxq_data *rxq) +{ + (void)rxq; + return -ENOTSUP; +} + +int __attribute__((weak)) +priv_check_vec_rx_support(struct priv *priv) +{ + (void)priv; + return -ENOTSUP; +} diff --git a/dpdk/drivers/net/mlx5/mlx5_rxtx.h b/dpdk/drivers/net/mlx5/mlx5_rxtx.h index 05779ef5..de5b769e 100644 --- a/dpdk/drivers/net/mlx5/mlx5_rxtx.h +++ b/dpdk/drivers/net/mlx5/mlx5_rxtx.h @@ -36,6 +36,7 @@ #include #include +#include /* Verbs header. */ /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ @@ -43,20 +44,16 @@ #pragma GCC diagnostic ignored "-Wpedantic" #endif #include -#include +#include #ifdef PEDANTIC #pragma GCC diagnostic error "-Wpedantic" #endif -/* DPDK headers don't like -pedantic. */ -#ifdef PEDANTIC -#pragma GCC diagnostic ignored "-Wpedantic" -#endif #include #include -#ifdef PEDANTIC -#pragma GCC diagnostic error "-Wpedantic" -#endif +#include +#include +#include #include "mlx5_utils.h" #include "mlx5.h" @@ -80,19 +77,22 @@ struct mlx5_txq_stats { uint64_t opackets; /**< Total of successfully sent packets. */ uint64_t obytes; /**< Total of successfully sent bytes. */ #endif - uint64_t odropped; /**< Total of packets not sent when TX ring full. */ -}; - -/* Flow director queue structure. */ -struct fdir_queue { - struct ibv_qp *qp; /* Associated RX QP. */ - struct ibv_exp_rwq_ind_table *ind_table; /* Indirection table. */ - struct ibv_exp_wq *wq; /* Work queue. */ - struct ibv_cq *cq; /* Completion queue. */ + uint64_t oerrors; /**< Total number of failed transmitted packets. */ }; struct priv; +/* Memory region queue object. */ +struct mlx5_mr { + LIST_ENTRY(mlx5_mr) next; /**< Pointer to the next element. */ + rte_atomic32_t refcnt; /*<< Reference counter. */ + uint32_t lkey; /*<< rte_cpu_to_be_32(mr->lkey) */ + uintptr_t start; /* Start address of MR */ + uintptr_t end; /* End address of MR */ + struct ibv_mr *mr; /*<< Memory Region. */ + struct rte_mempool *mp; /*<< Memory Pool. */ +}; + /* Compressed CQE context. */ struct rxq_zip { uint16_t ai; /* Array index. */ @@ -103,227 +103,553 @@ struct rxq_zip { }; /* RX queue descriptor. */ -struct rxq { +struct mlx5_rxq_data { unsigned int csum:1; /* Enable checksum offloading. */ unsigned int csum_l2tun:1; /* Same for L2 tunnels. */ + unsigned int hw_timestamp:1; /* Enable HW timestamp. */ unsigned int vlan_strip:1; /* Enable VLAN stripping. */ unsigned int crc_present:1; /* CRC must be subtracted. */ unsigned int sges_n:2; /* Log 2 of SGEs (max buffers per packet). */ - uint16_t rq_ci; - uint16_t cq_ci; - uint16_t elts_n; - uint16_t cqe_n; /* Number of CQ elements. */ + unsigned int cqe_n:4; /* Log 2 of CQ elements. */ + unsigned int elts_n:4; /* Log 2 of Mbufs. */ + unsigned int rss_hash:1; /* RSS hash result is enabled. */ + unsigned int mark:1; /* Marked flow available on the queue. */ + unsigned int :15; /* Remaining bits. */ + volatile uint32_t *rq_db; + volatile uint32_t *cq_db; uint16_t port_id; + uint16_t rq_ci; + uint16_t rq_pi; + uint16_t cq_ci; volatile struct mlx5_wqe_data_seg(*wqes)[]; volatile struct mlx5_cqe(*cqes)[]; struct rxq_zip zip; /* Compressed context. */ - volatile uint32_t *rq_db; - volatile uint32_t *cq_db; struct rte_mbuf *(*elts)[]; struct rte_mempool *mp; struct mlx5_rxq_stats stats; + uint64_t mbuf_initializer; /* Default rearm_data for vectorized Rx. */ + struct rte_mbuf fake_mbuf; /* elts padding for vectorized Rx. */ + void *cq_uar; /* CQ user access region. */ + uint32_t cqn; /* CQ number. */ + uint8_t cq_arm_sn; /* CQ arm seq number. */ } __rte_cache_aligned; -/* RX queue control descriptor. */ -struct rxq_ctrl { - struct priv *priv; /* Back pointer to private data. */ +/* Verbs Rx queue elements. */ +struct mlx5_rxq_ibv { + LIST_ENTRY(mlx5_rxq_ibv) next; /* Pointer to the next element. */ + rte_atomic32_t refcnt; /* Reference counter. */ + struct mlx5_rxq_ctrl *rxq_ctrl; /* Back pointer to parent. */ struct ibv_cq *cq; /* Completion Queue. */ - struct ibv_exp_wq *wq; /* Work Queue. */ - struct ibv_exp_res_domain *rd; /* Resource Domain. */ - struct fdir_queue *fdir_queue; /* Flow director queue. */ - struct ibv_mr *mr; /* Memory Region (for mp). */ - struct ibv_exp_wq_family *if_wq; /* WQ burst interface. */ - struct ibv_exp_cq_family_v1 *if_cq; /* CQ interface. */ - unsigned int socket; /* CPU socket ID for allocations. */ - struct rxq rxq; /* Data path structure. */ + struct ibv_wq *wq; /* Work Queue. */ + struct ibv_comp_channel *channel; + struct mlx5_mr *mr; /* Memory Region (for mp). */ }; -/* Hash RX queue types. */ -enum hash_rxq_type { - HASH_RXQ_TCPV4, - HASH_RXQ_UDPV4, - HASH_RXQ_IPV4, - HASH_RXQ_TCPV6, - HASH_RXQ_UDPV6, - HASH_RXQ_IPV6, - HASH_RXQ_ETH, -}; - -/* Flow structure with Ethernet specification. It is packed to prevent padding - * between attr and spec as this layout is expected by libibverbs. */ -struct flow_attr_spec_eth { - struct ibv_exp_flow_attr attr; - struct ibv_exp_flow_spec_eth spec; -} __attribute__((packed)); - -/* Define a struct flow_attr_spec_eth object as an array of at least - * "size" bytes. Room after the first index is normally used to store - * extra flow specifications. */ -#define FLOW_ATTR_SPEC_ETH(name, size) \ - struct flow_attr_spec_eth name \ - [((size) / sizeof(struct flow_attr_spec_eth)) + \ - !!((size) % sizeof(struct flow_attr_spec_eth))] - -/* Initialization data for hash RX queue. */ -struct hash_rxq_init { - uint64_t hash_fields; /* Fields that participate in the hash. */ - uint64_t dpdk_rss_hf; /* Matching DPDK RSS hash fields. */ - unsigned int flow_priority; /* Flow priority to use. */ - union { - struct { - enum ibv_exp_flow_spec_type type; - uint16_t size; - } hdr; - struct ibv_exp_flow_spec_tcp_udp tcp_udp; - struct ibv_exp_flow_spec_ipv4 ipv4; - struct ibv_exp_flow_spec_ipv6 ipv6; - struct ibv_exp_flow_spec_eth eth; - } flow_spec; /* Flow specification template. */ - const struct hash_rxq_init *underlayer; /* Pointer to underlayer. */ -}; - -/* Initialization data for indirection table. */ -struct ind_table_init { - unsigned int max_size; /* Maximum number of WQs. */ - /* Hash RX queues using this table. */ - unsigned int hash_types; - unsigned int hash_types_n; -}; - -/* Initialization data for special flows. */ -struct special_flow_init { - uint8_t dst_mac_val[6]; - uint8_t dst_mac_mask[6]; - unsigned int hash_types; - unsigned int per_vlan:1; -}; - -enum hash_rxq_flow_type { - HASH_RXQ_FLOW_TYPE_PROMISC, - HASH_RXQ_FLOW_TYPE_ALLMULTI, - HASH_RXQ_FLOW_TYPE_BROADCAST, - HASH_RXQ_FLOW_TYPE_IPV6MULTI, - HASH_RXQ_FLOW_TYPE_MAC, -}; - -#ifndef NDEBUG -static inline const char * -hash_rxq_flow_type_str(enum hash_rxq_flow_type flow_type) -{ - switch (flow_type) { - case HASH_RXQ_FLOW_TYPE_PROMISC: - return "promiscuous"; - case HASH_RXQ_FLOW_TYPE_ALLMULTI: - return "allmulticast"; - case HASH_RXQ_FLOW_TYPE_BROADCAST: - return "broadcast"; - case HASH_RXQ_FLOW_TYPE_IPV6MULTI: - return "IPv6 multicast"; - case HASH_RXQ_FLOW_TYPE_MAC: - return "MAC"; - } - return NULL; -} -#endif /* NDEBUG */ - -struct hash_rxq { +/* RX queue control descriptor. */ +struct mlx5_rxq_ctrl { + LIST_ENTRY(mlx5_rxq_ctrl) next; /* Pointer to the next element. */ + rte_atomic32_t refcnt; /* Reference counter. */ struct priv *priv; /* Back pointer to private data. */ - struct ibv_qp *qp; /* Hash RX QP. */ - enum hash_rxq_type type; /* Hash RX queue type. */ - /* MAC flow steering rules, one per VLAN ID. */ - struct ibv_exp_flow *mac_flow - [MLX5_MAX_MAC_ADDRESSES][MLX5_MAX_VLAN_IDS]; - struct ibv_exp_flow *special_flow - [MLX5_MAX_SPECIAL_FLOWS][MLX5_MAX_VLAN_IDS]; + struct mlx5_rxq_ibv *ibv; /* Verbs elements. */ + struct mlx5_rxq_data rxq; /* Data path structure. */ + unsigned int socket; /* CPU socket ID for allocations. */ + unsigned int irq:1; /* Whether IRQ is enabled. */ +}; + +/* Indirection table. */ +struct mlx5_ind_table_ibv { + LIST_ENTRY(mlx5_ind_table_ibv) next; /* Pointer to the next element. */ + rte_atomic32_t refcnt; /* Reference counter. */ + struct ibv_rwq_ind_table *ind_table; /**< Indirection table. */ + uint16_t queues_n; /**< Number of queues in the list. */ + uint16_t queues[]; /**< Queue list. */ +}; + +/* Hash Rx queue. */ +struct mlx5_hrxq { + LIST_ENTRY(mlx5_hrxq) next; /* Pointer to the next element. */ + rte_atomic32_t refcnt; /* Reference counter. */ + struct mlx5_ind_table_ibv *ind_table; /* Indirection table. */ + struct ibv_qp *qp; /* Verbs queue pair. */ + uint64_t hash_fields; /* Verbs Hash fields. */ + uint8_t rss_key_len; /* Hash key length in bytes. */ + uint8_t rss_key[]; /* Hash key. */ }; /* TX queue descriptor. */ -struct txq { - uint16_t elts_head; /* Current index in (*elts)[]. */ - uint16_t elts_tail; /* First element awaiting completion. */ +__extension__ +struct mlx5_txq_data { + uint16_t elts_head; /* Current counter in (*elts)[]. */ + uint16_t elts_tail; /* Counter of first element awaiting completion. */ uint16_t elts_comp; /* Counter since last completion request. */ - uint16_t elts_n; /* (*elts)[] length. */ + uint16_t mpw_comp; /* WQ index since last completion request. */ uint16_t cq_ci; /* Consumer index for completion queue. */ - uint16_t cqe_n; /* Number of CQ elements. */ + uint16_t cq_pi; /* Producer index for completion queue. */ uint16_t wqe_ci; /* Consumer index for work queue. */ - uint16_t wqe_n; /* Number of WQ elements. */ - uint16_t bf_offset; /* Blueflame offset. */ - uint16_t bf_buf_size; /* Blueflame size. */ + uint16_t wqe_pi; /* Producer index for work queue. */ + uint16_t elts_n:4; /* (*elts)[] length (in log2). */ + uint16_t cqe_n:4; /* Number of CQ elements (in log2). */ + uint16_t wqe_n:4; /* Number of of WQ elements (in log2). */ + uint16_t inline_en:1; /* When set inline is enabled. */ + uint16_t tso_en:1; /* When set hardware TSO is enabled. */ + uint16_t tunnel_en:1; + /* When set TX offload for tunneled packets are supported. */ + uint16_t mpw_hdr_dseg:1; /* Enable DSEGs in the title WQEBB. */ uint16_t max_inline; /* Multiple of RTE_CACHE_LINE_SIZE to inline. */ + uint16_t inline_max_packet_sz; /* Max packet size for inlining. */ + uint16_t mr_cache_idx; /* Index of last hit entry. */ uint32_t qp_num_8s; /* QP number shifted by 8. */ + uint32_t flags; /* Flags for Tx Queue. */ volatile struct mlx5_cqe (*cqes)[]; /* Completion queue. */ - volatile union mlx5_wqe (*wqes)[]; /* Work queue. */ + volatile void *wqes; /* Work queue (use volatile to write into). */ volatile uint32_t *qp_db; /* Work queue doorbell. */ volatile uint32_t *cq_db; /* Completion queue doorbell. */ volatile void *bf_reg; /* Blueflame register. */ - struct { - const struct rte_mempool *mp; /* Cached Memory Pool. */ - struct ibv_mr *mr; /* Memory Region (for mp). */ - uint32_t lkey; /* htonl(mr->lkey) */ - } mp2mr[MLX5_PMD_TX_MP_CACHE]; /* MP to MR translation table. */ + struct mlx5_mr *mp2mr[MLX5_PMD_TX_MP_CACHE]; /* MR translation table. */ struct rte_mbuf *(*elts)[]; /* TX elements. */ struct mlx5_txq_stats stats; /* TX queue counters. */ } __rte_cache_aligned; -/* TX queue control descriptor. */ -struct txq_ctrl { - struct priv *priv; /* Back pointer to private data. */ +/* Verbs Rx queue elements. */ +struct mlx5_txq_ibv { + LIST_ENTRY(mlx5_txq_ibv) next; /* Pointer to the next element. */ + rte_atomic32_t refcnt; /* Reference counter. */ struct ibv_cq *cq; /* Completion Queue. */ struct ibv_qp *qp; /* Queue Pair. */ - struct ibv_exp_qp_burst_family *if_qp; /* QP burst interface. */ - struct ibv_exp_cq_family *if_cq; /* CQ interface. */ - struct ibv_exp_res_domain *rd; /* Resource Domain. */ +}; + +/* TX queue control descriptor. */ +struct mlx5_txq_ctrl { + LIST_ENTRY(mlx5_txq_ctrl) next; /* Pointer to the next element. */ + rte_atomic32_t refcnt; /* Reference counter. */ + struct priv *priv; /* Back pointer to private data. */ unsigned int socket; /* CPU socket ID for allocations. */ - struct txq txq; /* Data path structure. */ + unsigned int max_inline_data; /* Max inline data. */ + unsigned int max_tso_header; /* Max TSO header size. */ + struct mlx5_txq_ibv *ibv; /* Verbs queue object. */ + struct mlx5_txq_data txq; /* Data path structure. */ + off_t uar_mmap_offset; /* UAR mmap offset for non-primary process. */ }; /* mlx5_rxq.c */ -extern const struct hash_rxq_init hash_rxq_init[]; -extern const unsigned int hash_rxq_init_n; - extern uint8_t rss_hash_default_key[]; extern const size_t rss_hash_default_key_len; -size_t priv_flow_attr(struct priv *, struct ibv_exp_flow_attr *, - size_t, enum hash_rxq_type); -int priv_create_hash_rxqs(struct priv *); -void priv_destroy_hash_rxqs(struct priv *); -int priv_allow_flow_type(struct priv *, enum hash_rxq_flow_type); -int priv_rehash_flows(struct priv *); -void rxq_cleanup(struct rxq_ctrl *); -int rxq_rehash(struct rte_eth_dev *, struct rxq_ctrl *); -int rxq_ctrl_setup(struct rte_eth_dev *, struct rxq_ctrl *, uint16_t, - unsigned int, const struct rte_eth_rxconf *, - struct rte_mempool *); +void mlx5_rxq_cleanup(struct mlx5_rxq_ctrl *); int mlx5_rx_queue_setup(struct rte_eth_dev *, uint16_t, uint16_t, unsigned int, const struct rte_eth_rxconf *, struct rte_mempool *); void mlx5_rx_queue_release(void *); -uint16_t mlx5_rx_burst_secondary_setup(void *, struct rte_mbuf **, uint16_t); +int priv_rx_intr_vec_enable(struct priv *priv); +void priv_rx_intr_vec_disable(struct priv *priv); +int mlx5_rx_intr_enable(struct rte_eth_dev *dev, uint16_t rx_queue_id); +int mlx5_rx_intr_disable(struct rte_eth_dev *dev, uint16_t rx_queue_id); +struct mlx5_rxq_ibv *mlx5_priv_rxq_ibv_new(struct priv *, uint16_t); +struct mlx5_rxq_ibv *mlx5_priv_rxq_ibv_get(struct priv *, uint16_t); +int mlx5_priv_rxq_ibv_release(struct priv *, struct mlx5_rxq_ibv *); +int mlx5_priv_rxq_ibv_releasable(struct priv *, struct mlx5_rxq_ibv *); +int mlx5_priv_rxq_ibv_verify(struct priv *); +struct mlx5_rxq_ctrl *mlx5_priv_rxq_new(struct priv *, uint16_t, + uint16_t, unsigned int, + struct rte_mempool *); +struct mlx5_rxq_ctrl *mlx5_priv_rxq_get(struct priv *, uint16_t); +int mlx5_priv_rxq_release(struct priv *, uint16_t); +int mlx5_priv_rxq_releasable(struct priv *, uint16_t); +int mlx5_priv_rxq_verify(struct priv *); +int rxq_alloc_elts(struct mlx5_rxq_ctrl *); +struct mlx5_ind_table_ibv *mlx5_priv_ind_table_ibv_new(struct priv *, + uint16_t [], + uint16_t); +struct mlx5_ind_table_ibv *mlx5_priv_ind_table_ibv_get(struct priv *, + uint16_t [], + uint16_t); +int mlx5_priv_ind_table_ibv_release(struct priv *, struct mlx5_ind_table_ibv *); +int mlx5_priv_ind_table_ibv_verify(struct priv *); +struct mlx5_hrxq *mlx5_priv_hrxq_new(struct priv *, uint8_t *, uint8_t, + uint64_t, uint16_t [], uint16_t); +struct mlx5_hrxq *mlx5_priv_hrxq_get(struct priv *, uint8_t *, uint8_t, + uint64_t, uint16_t [], uint16_t); +int mlx5_priv_hrxq_release(struct priv *, struct mlx5_hrxq *); +int mlx5_priv_hrxq_ibv_verify(struct priv *); /* mlx5_txq.c */ -void txq_cleanup(struct txq_ctrl *); -int txq_ctrl_setup(struct rte_eth_dev *, struct txq_ctrl *, uint16_t, - unsigned int, const struct rte_eth_txconf *); int mlx5_tx_queue_setup(struct rte_eth_dev *, uint16_t, uint16_t, unsigned int, const struct rte_eth_txconf *); void mlx5_tx_queue_release(void *); -uint16_t mlx5_tx_burst_secondary_setup(void *, struct rte_mbuf **, uint16_t); +int priv_tx_uar_remap(struct priv *priv, int fd); +struct mlx5_txq_ibv *mlx5_priv_txq_ibv_new(struct priv *, uint16_t); +struct mlx5_txq_ibv *mlx5_priv_txq_ibv_get(struct priv *, uint16_t); +int mlx5_priv_txq_ibv_release(struct priv *, struct mlx5_txq_ibv *); +int mlx5_priv_txq_ibv_releasable(struct priv *, struct mlx5_txq_ibv *); +int mlx5_priv_txq_ibv_verify(struct priv *); +struct mlx5_txq_ctrl *mlx5_priv_txq_new(struct priv *, uint16_t, + uint16_t, unsigned int, + const struct rte_eth_txconf *); +struct mlx5_txq_ctrl *mlx5_priv_txq_get(struct priv *, uint16_t); +int mlx5_priv_txq_release(struct priv *, uint16_t); +int mlx5_priv_txq_releasable(struct priv *, uint16_t); +int mlx5_priv_txq_verify(struct priv *); +void txq_alloc_elts(struct mlx5_txq_ctrl *); /* mlx5_rxtx.c */ +extern uint32_t mlx5_ptype_table[]; + +void mlx5_set_ptype_table(void); uint16_t mlx5_tx_burst(void *, struct rte_mbuf **, uint16_t); uint16_t mlx5_tx_burst_mpw(void *, struct rte_mbuf **, uint16_t); uint16_t mlx5_tx_burst_mpw_inline(void *, struct rte_mbuf **, uint16_t); +uint16_t mlx5_tx_burst_empw(void *, struct rte_mbuf **, uint16_t); uint16_t mlx5_rx_burst(void *, struct rte_mbuf **, uint16_t); uint16_t removed_tx_burst(void *, struct rte_mbuf **, uint16_t); uint16_t removed_rx_burst(void *, struct rte_mbuf **, uint16_t); +int mlx5_rx_descriptor_status(void *, uint16_t); +int mlx5_tx_descriptor_status(void *, uint16_t); + +/* Vectorized version of mlx5_rxtx.c */ +int priv_check_raw_vec_tx_support(struct priv *); +int priv_check_vec_tx_support(struct priv *); +int rxq_check_vec_support(struct mlx5_rxq_data *); +int priv_check_vec_rx_support(struct priv *); +uint16_t mlx5_tx_burst_raw_vec(void *, struct rte_mbuf **, uint16_t); +uint16_t mlx5_tx_burst_vec(void *, struct rte_mbuf **, uint16_t); +uint16_t mlx5_rx_burst_vec(void *, struct rte_mbuf **, uint16_t); /* mlx5_mr.c */ -struct ibv_mr *mlx5_mp2mr(struct ibv_pd *, struct rte_mempool *); -void txq_mp2mr_iter(struct rte_mempool *, void *); -uint32_t txq_mp2mr_reg(struct txq *, struct rte_mempool *, unsigned int); +void mlx5_mp2mr_iter(struct rte_mempool *, void *); +struct mlx5_mr *priv_txq_mp2mr_reg(struct priv *priv, struct mlx5_txq_data *, + struct rte_mempool *, unsigned int); +struct mlx5_mr *mlx5_txq_mp2mr_reg(struct mlx5_txq_data *, struct rte_mempool *, + unsigned int); + +#ifndef NDEBUG +/** + * Verify or set magic value in CQE. + * + * @param cqe + * Pointer to CQE. + * + * @return + * 0 the first time. + */ +static inline int +check_cqe_seen(volatile struct mlx5_cqe *cqe) +{ + static const uint8_t magic[] = "seen"; + volatile uint8_t (*buf)[sizeof(cqe->rsvd0)] = &cqe->rsvd0; + int ret = 1; + unsigned int i; + + for (i = 0; i < sizeof(magic) && i < sizeof(*buf); ++i) + if (!ret || (*buf)[i] != magic[i]) { + ret = 0; + (*buf)[i] = magic[i]; + } + return ret; +} +#endif /* NDEBUG */ + +/** + * Check whether CQE is valid. + * + * @param cqe + * Pointer to CQE. + * @param cqes_n + * Size of completion queue. + * @param ci + * Consumer index. + * + * @return + * 0 on success, 1 on failure. + */ +static __rte_always_inline int +check_cqe(volatile struct mlx5_cqe *cqe, + unsigned int cqes_n, const uint16_t ci) +{ + uint16_t idx = ci & cqes_n; + uint8_t op_own = cqe->op_own; + uint8_t op_owner = MLX5_CQE_OWNER(op_own); + uint8_t op_code = MLX5_CQE_OPCODE(op_own); + + if (unlikely((op_owner != (!!(idx))) || (op_code == MLX5_CQE_INVALID))) + return 1; /* No CQE. */ +#ifndef NDEBUG + if ((op_code == MLX5_CQE_RESP_ERR) || + (op_code == MLX5_CQE_REQ_ERR)) { + volatile struct mlx5_err_cqe *err_cqe = (volatile void *)cqe; + uint8_t syndrome = err_cqe->syndrome; + + if ((syndrome == MLX5_CQE_SYNDROME_LOCAL_LENGTH_ERR) || + (syndrome == MLX5_CQE_SYNDROME_REMOTE_ABORTED_ERR)) + return 0; + if (!check_cqe_seen(cqe)) { + ERROR("unexpected CQE error %u (0x%02x)" + " syndrome 0x%02x", + op_code, op_code, syndrome); + rte_hexdump(stderr, "MLX5 Error CQE:", + (const void *)((uintptr_t)err_cqe), + sizeof(*err_cqe)); + } + return 1; + } else if ((op_code != MLX5_CQE_RESP_SEND) && + (op_code != MLX5_CQE_REQ)) { + if (!check_cqe_seen(cqe)) { + ERROR("unexpected CQE opcode %u (0x%02x)", + op_code, op_code); + rte_hexdump(stderr, "MLX5 CQE:", + (const void *)((uintptr_t)cqe), + sizeof(*cqe)); + } + return 1; + } +#endif /* NDEBUG */ + return 0; +} + +/** + * Return the address of the WQE. + * + * @param txq + * Pointer to TX queue structure. + * @param wqe_ci + * WQE consumer index. + * + * @return + * WQE address. + */ +static inline uintptr_t * +tx_mlx5_wqe(struct mlx5_txq_data *txq, uint16_t ci) +{ + ci &= ((1 << txq->wqe_n) - 1); + return (uintptr_t *)((uintptr_t)txq->wqes + ci * MLX5_WQE_SIZE); +} + +/** + * Manage TX completions. + * + * When sending a burst, mlx5_tx_burst() posts several WRs. + * + * @param txq + * Pointer to TX queue structure. + */ +static __rte_always_inline void +mlx5_tx_complete(struct mlx5_txq_data *txq) +{ + const uint16_t elts_n = 1 << txq->elts_n; + const uint16_t elts_m = elts_n - 1; + const unsigned int cqe_n = 1 << txq->cqe_n; + const unsigned int cqe_cnt = cqe_n - 1; + uint16_t elts_free = txq->elts_tail; + uint16_t elts_tail; + uint16_t cq_ci = txq->cq_ci; + volatile struct mlx5_cqe *cqe = NULL; + volatile struct mlx5_wqe_ctrl *ctrl; + struct rte_mbuf *m, *free[elts_n]; + struct rte_mempool *pool = NULL; + unsigned int blk_n = 0; + + cqe = &(*txq->cqes)[cq_ci & cqe_cnt]; + if (unlikely(check_cqe(cqe, cqe_n, cq_ci))) + return; +#ifndef NDEBUG + if ((MLX5_CQE_OPCODE(cqe->op_own) == MLX5_CQE_RESP_ERR) || + (MLX5_CQE_OPCODE(cqe->op_own) == MLX5_CQE_REQ_ERR)) { + if (!check_cqe_seen(cqe)) { + ERROR("unexpected error CQE, TX stopped"); + rte_hexdump(stderr, "MLX5 TXQ:", + (const void *)((uintptr_t)txq->wqes), + ((1 << txq->wqe_n) * + MLX5_WQE_SIZE)); + } + return; + } +#endif /* NDEBUG */ + ++cq_ci; + txq->wqe_pi = rte_be_to_cpu_16(cqe->wqe_counter); + ctrl = (volatile struct mlx5_wqe_ctrl *) + tx_mlx5_wqe(txq, txq->wqe_pi); + elts_tail = ctrl->ctrl3; + assert((elts_tail & elts_m) < (1 << txq->wqe_n)); + /* Free buffers. */ + while (elts_free != elts_tail) { + m = rte_pktmbuf_prefree_seg((*txq->elts)[elts_free++ & elts_m]); + if (likely(m != NULL)) { + if (likely(m->pool == pool)) { + free[blk_n++] = m; + } else { + if (likely(pool != NULL)) + rte_mempool_put_bulk(pool, + (void *)free, + blk_n); + free[0] = m; + pool = m->pool; + blk_n = 1; + } + } + } + if (blk_n) + rte_mempool_put_bulk(pool, (void *)free, blk_n); +#ifndef NDEBUG + elts_free = txq->elts_tail; + /* Poisoning. */ + while (elts_free != elts_tail) { + memset(&(*txq->elts)[elts_free & elts_m], + 0x66, + sizeof((*txq->elts)[elts_free & elts_m])); + ++elts_free; + } +#endif + txq->cq_ci = cq_ci; + txq->elts_tail = elts_tail; + /* Update the consumer index. */ + rte_compiler_barrier(); + *txq->cq_db = rte_cpu_to_be_32(cq_ci); +} + +/** + * Get Memory Pool (MP) from mbuf. If mbuf is indirect, the pool from which + * the cloned mbuf is allocated is returned instead. + * + * @param buf + * Pointer to mbuf. + * + * @return + * Memory pool where data is located for given mbuf. + */ +static struct rte_mempool * +mlx5_tx_mb2mp(struct rte_mbuf *buf) +{ + if (unlikely(RTE_MBUF_INDIRECT(buf))) + return rte_mbuf_from_indirect(buf)->pool; + return buf->pool; +} + +/** + * Get Memory Region (MR) <-> rte_mbuf association from txq->mp2mr[]. + * Add MP to txq->mp2mr[] if it's not registered yet. If mp2mr[] is full, + * remove an entry first. + * + * @param txq + * Pointer to TX queue structure. + * @param[in] mp + * Memory Pool for which a Memory Region lkey must be returned. + * + * @return + * mr->lkey on success, (uint32_t)-1 on failure. + */ +static __rte_always_inline uint32_t +mlx5_tx_mb2mr(struct mlx5_txq_data *txq, struct rte_mbuf *mb) +{ + uint16_t i = txq->mr_cache_idx; + uintptr_t addr = rte_pktmbuf_mtod(mb, uintptr_t); + struct mlx5_mr *mr; + + assert(i < RTE_DIM(txq->mp2mr)); + if (likely(txq->mp2mr[i]->start <= addr && txq->mp2mr[i]->end > addr)) + return txq->mp2mr[i]->lkey; + for (i = 0; (i != RTE_DIM(txq->mp2mr)); ++i) { + if (unlikely(txq->mp2mr[i] == NULL || + txq->mp2mr[i]->mr == NULL)) { + /* Unknown MP, add a new MR for it. */ + break; + } + if (txq->mp2mr[i]->start <= addr && + txq->mp2mr[i]->end > addr) { + assert(txq->mp2mr[i]->lkey != (uint32_t)-1); + assert(rte_cpu_to_be_32(txq->mp2mr[i]->mr->lkey) == + txq->mp2mr[i]->lkey); + txq->mr_cache_idx = i; + return txq->mp2mr[i]->lkey; + } + } + mr = mlx5_txq_mp2mr_reg(txq, mlx5_tx_mb2mp(mb), i); + /* + * Request the reference to use in this queue, the original one is + * kept by the control plane. + */ + if (mr) { + rte_atomic32_inc(&mr->refcnt); + txq->mr_cache_idx = i >= RTE_DIM(txq->mp2mr) ? i - 1 : i; + return mr->lkey; + } + return (uint32_t)-1; +} + +/** + * Ring TX queue doorbell and flush the update if requested. + * + * @param txq + * Pointer to TX queue structure. + * @param wqe + * Pointer to the last WQE posted in the NIC. + * @param cond + * Request for write memory barrier after BlueFlame update. + */ +static __rte_always_inline void +mlx5_tx_dbrec_cond_wmb(struct mlx5_txq_data *txq, volatile struct mlx5_wqe *wqe, + int cond) +{ + uint64_t *dst = (uint64_t *)((uintptr_t)txq->bf_reg); + volatile uint64_t *src = ((volatile uint64_t *)wqe); + + rte_io_wmb(); + *txq->qp_db = rte_cpu_to_be_32(txq->wqe_ci); + /* Ensure ordering between DB record and BF copy. */ + rte_wmb(); + *dst = *src; + if (cond) + rte_wmb(); +} + +/** + * Ring TX queue doorbell and flush the update by write memory barrier. + * + * @param txq + * Pointer to TX queue structure. + * @param wqe + * Pointer to the last WQE posted in the NIC. + */ +static __rte_always_inline void +mlx5_tx_dbrec(struct mlx5_txq_data *txq, volatile struct mlx5_wqe *wqe) +{ + mlx5_tx_dbrec_cond_wmb(txq, wqe, 1); +} + +/** + * Convert the Checksum offloads to Verbs. + * + * @param txq_data + * Pointer to the Tx queue. + * @param buf + * Pointer to the mbuf. + * + * @return + * the converted cs_flags. + */ +static __rte_always_inline uint8_t +txq_ol_cksum_to_cs(struct mlx5_txq_data *txq_data, struct rte_mbuf *buf) +{ + uint8_t cs_flags = 0; + + /* Should we enable HW CKSUM offload */ + if (buf->ol_flags & + (PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM | + PKT_TX_OUTER_IP_CKSUM)) { + if (txq_data->tunnel_en && + (buf->ol_flags & + (PKT_TX_TUNNEL_GRE | PKT_TX_TUNNEL_VXLAN))) { + cs_flags = MLX5_ETH_WQE_L3_INNER_CSUM | + MLX5_ETH_WQE_L4_INNER_CSUM; + if (buf->ol_flags & PKT_TX_OUTER_IP_CKSUM) + cs_flags |= MLX5_ETH_WQE_L3_CSUM; + } else { + cs_flags = MLX5_ETH_WQE_L3_CSUM | + MLX5_ETH_WQE_L4_CSUM; + } + } + return cs_flags; +} #endif /* RTE_PMD_MLX5_RXTX_H_ */ diff --git a/dpdk/drivers/net/mlx5/mlx5_rxtx_vec.c b/dpdk/drivers/net/mlx5/mlx5_rxtx_vec.c new file mode 100644 index 00000000..101aa156 --- /dev/null +++ b/dpdk/drivers/net/mlx5/mlx5_rxtx_vec.c @@ -0,0 +1,371 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of 6WIND S.A. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include + +/* Verbs header. */ +/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ +#ifdef PEDANTIC +#pragma GCC diagnostic ignored "-Wpedantic" +#endif +#include +#include +#ifdef PEDANTIC +#pragma GCC diagnostic error "-Wpedantic" +#endif + +#include +#include +#include + +#include "mlx5.h" +#include "mlx5_utils.h" +#include "mlx5_rxtx.h" +#include "mlx5_rxtx_vec.h" +#include "mlx5_autoconf.h" +#include "mlx5_defs.h" +#include "mlx5_prm.h" + +#if defined RTE_ARCH_X86_64 +#include "mlx5_rxtx_vec_sse.h" +#elif defined RTE_ARCH_ARM64 +#include "mlx5_rxtx_vec_neon.h" +#else +#error "This should not be compiled if SIMD instructions are not supported." +#endif + +/** + * Count the number of continuous single segment packets. + * + * @param pkts + * Pointer to array of packets. + * @param pkts_n + * Number of packets. + * + * @return + * Number of continuous single segment packets. + */ +static inline unsigned int +txq_check_multiseg(struct rte_mbuf **pkts, uint16_t pkts_n) +{ + unsigned int pos; + + if (!pkts_n) + return 0; + /* Count the number of continuous single segment packets. */ + for (pos = 0; pos < pkts_n; ++pos) + if (NB_SEGS(pkts[pos]) > 1) + break; + return pos; +} + +/** + * Count the number of packets having same ol_flags and calculate cs_flags. + * + * @param txq + * Pointer to TX queue structure. + * @param pkts + * Pointer to array of packets. + * @param pkts_n + * Number of packets. + * @param cs_flags + * Pointer of flags to be returned. + * + * @return + * Number of packets having same ol_flags. + */ +static inline unsigned int +txq_calc_offload(struct mlx5_txq_data *txq, struct rte_mbuf **pkts, + uint16_t pkts_n, uint8_t *cs_flags) +{ + unsigned int pos; + const uint64_t ol_mask = + PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM | + PKT_TX_UDP_CKSUM | PKT_TX_TUNNEL_GRE | + PKT_TX_TUNNEL_VXLAN | PKT_TX_OUTER_IP_CKSUM; + + if (!pkts_n) + return 0; + /* Count the number of packets having same ol_flags. */ + for (pos = 1; pos < pkts_n; ++pos) + if ((pkts[pos]->ol_flags ^ pkts[0]->ol_flags) & ol_mask) + break; + *cs_flags = txq_ol_cksum_to_cs(txq, pkts[0]); + return pos; +} + +/** + * DPDK callback for vectorized TX. + * + * @param dpdk_txq + * Generic pointer to TX queue structure. + * @param[in] pkts + * Packets to transmit. + * @param pkts_n + * Number of packets in array. + * + * @return + * Number of packets successfully transmitted (<= pkts_n). + */ +uint16_t +mlx5_tx_burst_raw_vec(void *dpdk_txq, struct rte_mbuf **pkts, + uint16_t pkts_n) +{ + struct mlx5_txq_data *txq = (struct mlx5_txq_data *)dpdk_txq; + uint16_t nb_tx = 0; + + while (pkts_n > nb_tx) { + uint16_t n; + uint16_t ret; + + n = RTE_MIN((uint16_t)(pkts_n - nb_tx), MLX5_VPMD_TX_MAX_BURST); + ret = txq_burst_v(txq, &pkts[nb_tx], n, 0); + nb_tx += ret; + if (!ret) + break; + } + return nb_tx; +} + +/** + * DPDK callback for vectorized TX with multi-seg packets and offload. + * + * @param dpdk_txq + * Generic pointer to TX queue structure. + * @param[in] pkts + * Packets to transmit. + * @param pkts_n + * Number of packets in array. + * + * @return + * Number of packets successfully transmitted (<= pkts_n). + */ +uint16_t +mlx5_tx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) +{ + struct mlx5_txq_data *txq = (struct mlx5_txq_data *)dpdk_txq; + uint16_t nb_tx = 0; + + while (pkts_n > nb_tx) { + uint8_t cs_flags = 0; + uint16_t n; + uint16_t ret; + + /* Transmit multi-seg packets in the head of pkts list. */ + if (!(txq->flags & ETH_TXQ_FLAGS_NOMULTSEGS) && + NB_SEGS(pkts[nb_tx]) > 1) + nb_tx += txq_scatter_v(txq, + &pkts[nb_tx], + pkts_n - nb_tx); + n = RTE_MIN((uint16_t)(pkts_n - nb_tx), MLX5_VPMD_TX_MAX_BURST); + if (!(txq->flags & ETH_TXQ_FLAGS_NOMULTSEGS)) + n = txq_check_multiseg(&pkts[nb_tx], n); + if (!(txq->flags & ETH_TXQ_FLAGS_NOOFFLOADS)) + n = txq_calc_offload(txq, &pkts[nb_tx], n, &cs_flags); + ret = txq_burst_v(txq, &pkts[nb_tx], n, cs_flags); + nb_tx += ret; + if (!ret) + break; + } + return nb_tx; +} + +/** + * Skip error packets. + * + * @param rxq + * Pointer to RX queue structure. + * @param[out] pkts + * Array to store received packets. + * @param pkts_n + * Maximum number of packets in array. + * + * @return + * Number of packets successfully received (<= pkts_n). + */ +static uint16_t +rxq_handle_pending_error(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, + uint16_t pkts_n) +{ + uint16_t n = 0; + unsigned int i; +#ifdef MLX5_PMD_SOFT_COUNTERS + uint32_t err_bytes = 0; +#endif + + for (i = 0; i < pkts_n; ++i) { + struct rte_mbuf *pkt = pkts[i]; + + if (pkt->packet_type == RTE_PTYPE_ALL_MASK) { +#ifdef MLX5_PMD_SOFT_COUNTERS + err_bytes += PKT_LEN(pkt); +#endif + rte_pktmbuf_free_seg(pkt); + } else { + pkts[n++] = pkt; + } + } + rxq->stats.idropped += (pkts_n - n); +#ifdef MLX5_PMD_SOFT_COUNTERS + /* Correct counters of errored completions. */ + rxq->stats.ipackets -= (pkts_n - n); + rxq->stats.ibytes -= err_bytes; +#endif + return n; +} + +/** + * DPDK callback for vectorized RX. + * + * @param dpdk_rxq + * Generic pointer to RX queue structure. + * @param[out] pkts + * Array to store received packets. + * @param pkts_n + * Maximum number of packets in array. + * + * @return + * Number of packets successfully received (<= pkts_n). + */ +uint16_t +mlx5_rx_burst_vec(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) +{ + struct mlx5_rxq_data *rxq = dpdk_rxq; + uint16_t nb_rx; + uint64_t err = 0; + + nb_rx = rxq_burst_v(rxq, pkts, pkts_n, &err); + if (unlikely(err)) + nb_rx = rxq_handle_pending_error(rxq, pkts, nb_rx); + return nb_rx; +} + +/** + * Check Tx queue flags are set for raw vectorized Tx. + * + * @param priv + * Pointer to private structure. + * + * @return + * 1 if supported, negative errno value if not. + */ +int __attribute__((cold)) +priv_check_raw_vec_tx_support(struct priv *priv) +{ + uint16_t i; + + /* All the configured queues should support. */ + for (i = 0; i < priv->txqs_n; ++i) { + struct mlx5_txq_data *txq = (*priv->txqs)[i]; + + if (!(txq->flags & ETH_TXQ_FLAGS_NOMULTSEGS) || + !(txq->flags & ETH_TXQ_FLAGS_NOOFFLOADS)) + break; + } + if (i != priv->txqs_n) + return -ENOTSUP; + return 1; +} + +/** + * Check a device can support vectorized TX. + * + * @param priv + * Pointer to private structure. + * + * @return + * 1 if supported, negative errno value if not. + */ +int __attribute__((cold)) +priv_check_vec_tx_support(struct priv *priv) +{ + if (!priv->tx_vec_en || + priv->txqs_n > MLX5_VPMD_MIN_TXQS || + priv->mps != MLX5_MPW_ENHANCED || + priv->tso) + return -ENOTSUP; + return 1; +} + +/** + * Check a RX queue can support vectorized RX. + * + * @param rxq + * Pointer to RX queue. + * + * @return + * 1 if supported, negative errno value if not. + */ +int __attribute__((cold)) +rxq_check_vec_support(struct mlx5_rxq_data *rxq) +{ + struct mlx5_rxq_ctrl *ctrl = + container_of(rxq, struct mlx5_rxq_ctrl, rxq); + + if (!ctrl->priv->rx_vec_en || rxq->sges_n != 0) + return -ENOTSUP; + return 1; +} + +/** + * Check a device can support vectorized RX. + * + * @param priv + * Pointer to private structure. + * + * @return + * 1 if supported, negative errno value if not. + */ +int __attribute__((cold)) +priv_check_vec_rx_support(struct priv *priv) +{ + uint16_t i; + + if (!priv->rx_vec_en) + return -ENOTSUP; + /* All the configured queues should support. */ + for (i = 0; i < priv->rxqs_n; ++i) { + struct mlx5_rxq_data *rxq = (*priv->rxqs)[i]; + + if (!rxq) + continue; + if (rxq_check_vec_support(rxq) < 0) + break; + } + if (i != priv->rxqs_n) + return -ENOTSUP; + return 1; +} diff --git a/dpdk/drivers/net/mlx5/mlx5_rxtx_vec.h b/dpdk/drivers/net/mlx5/mlx5_rxtx_vec.h new file mode 100644 index 00000000..1f08ed0b --- /dev/null +++ b/dpdk/drivers/net/mlx5/mlx5_rxtx_vec.h @@ -0,0 +1,130 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of 6WIND S.A. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef RTE_PMD_MLX5_RXTX_VEC_H_ +#define RTE_PMD_MLX5_RXTX_VEC_H_ + +#include +#include + +#include "mlx5_autoconf.h" +#include "mlx5_prm.h" + +/* + * Compile time sanity check for vectorized functions. + */ + +#define S_ASSERT_RTE_MBUF(s) \ + static_assert(s, "A field of struct rte_mbuf is changed") +#define S_ASSERT_MLX5_CQE(s) \ + static_assert(s, "A field of struct mlx5_cqe is changed") + +/* rxq_cq_decompress_v() */ +S_ASSERT_RTE_MBUF(offsetof(struct rte_mbuf, pkt_len) == + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 4); +S_ASSERT_RTE_MBUF(offsetof(struct rte_mbuf, data_len) == + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 8); +S_ASSERT_RTE_MBUF(offsetof(struct rte_mbuf, hash) == + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 12); + +/* rxq_cq_to_ptype_oflags_v() */ +S_ASSERT_RTE_MBUF(offsetof(struct rte_mbuf, ol_flags) == + offsetof(struct rte_mbuf, rearm_data) + 8); +S_ASSERT_RTE_MBUF(offsetof(struct rte_mbuf, rearm_data) == + RTE_ALIGN(offsetof(struct rte_mbuf, rearm_data), 16)); + +/* rxq_burst_v() */ +S_ASSERT_RTE_MBUF(offsetof(struct rte_mbuf, pkt_len) == + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 4); +S_ASSERT_RTE_MBUF(offsetof(struct rte_mbuf, data_len) == + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 8); +#if (RTE_CACHE_LINE_SIZE == 128) +S_ASSERT_MLX5_CQE(offsetof(struct mlx5_cqe, pkt_info) == 64); +#else +S_ASSERT_MLX5_CQE(offsetof(struct mlx5_cqe, pkt_info) == 0); +#endif +S_ASSERT_MLX5_CQE(offsetof(struct mlx5_cqe, rx_hash_res) == + offsetof(struct mlx5_cqe, pkt_info) + 12); +S_ASSERT_MLX5_CQE(offsetof(struct mlx5_cqe, rsvd1) + + sizeof(((struct mlx5_cqe *)0)->rsvd1) == + offsetof(struct mlx5_cqe, hdr_type_etc)); +S_ASSERT_MLX5_CQE(offsetof(struct mlx5_cqe, vlan_info) == + offsetof(struct mlx5_cqe, hdr_type_etc) + 2); +S_ASSERT_MLX5_CQE(offsetof(struct mlx5_cqe, rsvd2) + + sizeof(((struct mlx5_cqe *)0)->rsvd2) == + offsetof(struct mlx5_cqe, byte_cnt)); +S_ASSERT_MLX5_CQE(offsetof(struct mlx5_cqe, sop_drop_qpn) == + RTE_ALIGN(offsetof(struct mlx5_cqe, sop_drop_qpn), 8)); +S_ASSERT_MLX5_CQE(offsetof(struct mlx5_cqe, op_own) == + offsetof(struct mlx5_cqe, sop_drop_qpn) + 7); + +/** + * Replenish buffers for RX in bulk. + * + * @param rxq + * Pointer to RX queue structure. + * @param n + * Number of buffers to be replenished. + */ +static inline void +mlx5_rx_replenish_bulk_mbuf(struct mlx5_rxq_data *rxq, uint16_t n) +{ + const uint16_t q_n = 1 << rxq->elts_n; + const uint16_t q_mask = q_n - 1; + uint16_t elts_idx = rxq->rq_ci & q_mask; + struct rte_mbuf **elts = &(*rxq->elts)[elts_idx]; + volatile struct mlx5_wqe_data_seg *wq = &(*rxq->wqes)[elts_idx]; + unsigned int i; + + assert(n >= MLX5_VPMD_RXQ_RPLNSH_THRESH); + assert(n <= (uint16_t)(q_n - (rxq->rq_ci - rxq->rq_pi))); + assert(MLX5_VPMD_RXQ_RPLNSH_THRESH > MLX5_VPMD_DESCS_PER_LOOP); + /* Not to cross queue end. */ + n = RTE_MIN(n - MLX5_VPMD_DESCS_PER_LOOP, q_n - elts_idx); + if (rte_mempool_get_bulk(rxq->mp, (void *)elts, n) < 0) { + rxq->stats.rx_nombuf += n; + return; + } + for (i = 0; i < n; ++i) + wq[i].addr = rte_cpu_to_be_64((uintptr_t)elts[i]->buf_addr + + RTE_PKTMBUF_HEADROOM); + rxq->rq_ci += n; + /* Prevent overflowing into consumed mbufs. */ + elts_idx = rxq->rq_ci & q_mask; + for (i = 0; i < MLX5_VPMD_DESCS_PER_LOOP; ++i) + (*rxq->elts)[elts_idx + i] = &rxq->fake_mbuf; + rte_io_wmb(); + *rxq->rq_db = rte_cpu_to_be_32(rxq->rq_ci); +} + +#endif /* RTE_PMD_MLX5_RXTX_VEC_H_ */ diff --git a/dpdk/drivers/net/mlx5/mlx5_rxtx_vec_neon.h b/dpdk/drivers/net/mlx5/mlx5_rxtx_vec_neon.h new file mode 100644 index 00000000..06f83ef1 --- /dev/null +++ b/dpdk/drivers/net/mlx5/mlx5_rxtx_vec_neon.h @@ -0,0 +1,1031 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of 6WIND S.A. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef RTE_PMD_MLX5_RXTX_VEC_NEON_H_ +#define RTE_PMD_MLX5_RXTX_VEC_NEON_H_ + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "mlx5.h" +#include "mlx5_utils.h" +#include "mlx5_rxtx.h" +#include "mlx5_rxtx_vec.h" +#include "mlx5_autoconf.h" +#include "mlx5_defs.h" +#include "mlx5_prm.h" + +#pragma GCC diagnostic ignored "-Wcast-qual" + +/** + * Fill in buffer descriptors in a multi-packet send descriptor. + * + * @param txq + * Pointer to TX queue structure. + * @param dseg + * Pointer to buffer descriptor to be written. + * @param pkts + * Pointer to array of packets to be sent. + * @param n + * Number of packets to be filled. + */ +static inline void +txq_wr_dseg_v(struct mlx5_txq_data *txq, uint8_t *dseg, + struct rte_mbuf **pkts, unsigned int n) +{ + unsigned int pos; + uintptr_t addr; + const uint8x16_t dseg_shuf_m = { + 3, 2, 1, 0, /* length, bswap32 */ + 4, 5, 6, 7, /* lkey */ + 15, 14, 13, 12, /* addr, bswap64 */ + 11, 10, 9, 8 + }; +#ifdef MLX5_PMD_SOFT_COUNTERS + uint32_t tx_byte = 0; +#endif + + for (pos = 0; pos < n; ++pos, dseg += MLX5_WQE_DWORD_SIZE) { + uint8x16_t desc; + struct rte_mbuf *pkt = pkts[pos]; + + addr = rte_pktmbuf_mtod(pkt, uintptr_t); + desc = vreinterpretq_u8_u32((uint32x4_t) { + DATA_LEN(pkt), + mlx5_tx_mb2mr(txq, pkt), + addr, + addr >> 32 }); + desc = vqtbl1q_u8(desc, dseg_shuf_m); + vst1q_u8(dseg, desc); +#ifdef MLX5_PMD_SOFT_COUNTERS + tx_byte += DATA_LEN(pkt); +#endif + } +#ifdef MLX5_PMD_SOFT_COUNTERS + txq->stats.obytes += tx_byte; +#endif +} + +/** + * Send multi-segmented packets until it encounters a single segment packet in + * the pkts list. + * + * @param txq + * Pointer to TX queue structure. + * @param pkts + * Pointer to array of packets to be sent. + * @param pkts_n + * Number of packets to be sent. + * + * @return + * Number of packets successfully transmitted (<= pkts_n). + */ +static uint16_t +txq_scatter_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts, + uint16_t pkts_n) +{ + uint16_t elts_head = txq->elts_head; + const uint16_t elts_n = 1 << txq->elts_n; + const uint16_t elts_m = elts_n - 1; + const uint16_t wq_n = 1 << txq->wqe_n; + const uint16_t wq_mask = wq_n - 1; + const unsigned int nb_dword_per_wqebb = + MLX5_WQE_SIZE / MLX5_WQE_DWORD_SIZE; + const unsigned int nb_dword_in_hdr = + sizeof(struct mlx5_wqe) / MLX5_WQE_DWORD_SIZE; + unsigned int n; + volatile struct mlx5_wqe *wqe = NULL; + + assert(elts_n > pkts_n); + mlx5_tx_complete(txq); + if (unlikely(!pkts_n)) + return 0; + for (n = 0; n < pkts_n; ++n) { + struct rte_mbuf *buf = pkts[n]; + unsigned int segs_n = buf->nb_segs; + unsigned int ds = nb_dword_in_hdr; + unsigned int len = PKT_LEN(buf); + uint16_t wqe_ci = txq->wqe_ci; + const uint8x16_t ctrl_shuf_m = { + 3, 2, 1, 0, /* bswap32 */ + 7, 6, 5, 4, /* bswap32 */ + 11, 10, 9, 8, /* bswap32 */ + 12, 13, 14, 15 + }; + uint8_t cs_flags; + uint16_t max_elts; + uint16_t max_wqe; + uint8x16_t *t_wqe; + uint8_t *dseg; + uint8x16_t ctrl; + + assert(segs_n); + max_elts = elts_n - (elts_head - txq->elts_tail); + max_wqe = wq_n - (txq->wqe_ci - txq->wqe_pi); + /* + * A MPW session consumes 2 WQEs at most to + * include MLX5_MPW_DSEG_MAX pointers. + */ + if (segs_n == 1 || + max_elts < segs_n || max_wqe < 2) + break; + wqe = &((volatile struct mlx5_wqe64 *) + txq->wqes)[wqe_ci & wq_mask].hdr; + cs_flags = txq_ol_cksum_to_cs(txq, buf); + /* Title WQEBB pointer. */ + t_wqe = (uint8x16_t *)wqe; + dseg = (uint8_t *)(wqe + 1); + do { + if (!(ds++ % nb_dword_per_wqebb)) { + dseg = (uint8_t *) + &((volatile struct mlx5_wqe64 *) + txq->wqes)[++wqe_ci & wq_mask]; + } + txq_wr_dseg_v(txq, dseg, &buf, 1); + dseg += MLX5_WQE_DWORD_SIZE; + (*txq->elts)[elts_head++ & elts_m] = buf; + buf = buf->next; + } while (--segs_n); + ++wqe_ci; + /* Fill CTRL in the header. */ + ctrl = vreinterpretq_u8_u32((uint32x4_t) { + MLX5_OPC_MOD_MPW << 24 | + txq->wqe_ci << 8 | MLX5_OPCODE_TSO, + txq->qp_num_8s | ds, 0, 0}); + ctrl = vqtbl1q_u8(ctrl, ctrl_shuf_m); + vst1q_u8((void *)t_wqe, ctrl); + /* Fill ESEG in the header. */ + vst1q_u16((void *)(t_wqe + 1), + (uint16x8_t) { 0, 0, cs_flags, rte_cpu_to_be_16(len), + 0, 0, 0, 0 }); + txq->wqe_ci = wqe_ci; + } + if (!n) + return 0; + txq->elts_comp += (uint16_t)(elts_head - txq->elts_head); + txq->elts_head = elts_head; + if (txq->elts_comp >= MLX5_TX_COMP_THRESH) { + wqe->ctrl[2] = rte_cpu_to_be_32(8); + wqe->ctrl[3] = txq->elts_head; + txq->elts_comp = 0; + ++txq->cq_pi; + } +#ifdef MLX5_PMD_SOFT_COUNTERS + txq->stats.opackets += n; +#endif + mlx5_tx_dbrec(txq, wqe); + return n; +} + +/** + * Send burst of packets with Enhanced MPW. If it encounters a multi-seg packet, + * it returns to make it processed by txq_scatter_v(). All the packets in + * the pkts list should be single segment packets having same offload flags. + * This must be checked by txq_check_multiseg() and txq_calc_offload(). + * + * @param txq + * Pointer to TX queue structure. + * @param pkts + * Pointer to array of packets to be sent. + * @param pkts_n + * Number of packets to be sent (<= MLX5_VPMD_TX_MAX_BURST). + * @param cs_flags + * Checksum offload flags to be written in the descriptor. + * + * @return + * Number of packets successfully transmitted (<= pkts_n). + */ +static inline uint16_t +txq_burst_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts, uint16_t pkts_n, + uint8_t cs_flags) +{ + struct rte_mbuf **elts; + uint16_t elts_head = txq->elts_head; + const uint16_t elts_n = 1 << txq->elts_n; + const uint16_t elts_m = elts_n - 1; + const unsigned int nb_dword_per_wqebb = + MLX5_WQE_SIZE / MLX5_WQE_DWORD_SIZE; + const unsigned int nb_dword_in_hdr = + sizeof(struct mlx5_wqe) / MLX5_WQE_DWORD_SIZE; + unsigned int n = 0; + unsigned int pos; + uint16_t max_elts; + uint16_t max_wqe; + uint32_t comp_req = 0; + const uint16_t wq_n = 1 << txq->wqe_n; + const uint16_t wq_mask = wq_n - 1; + uint16_t wq_idx = txq->wqe_ci & wq_mask; + volatile struct mlx5_wqe64 *wq = + &((volatile struct mlx5_wqe64 *)txq->wqes)[wq_idx]; + volatile struct mlx5_wqe *wqe = (volatile struct mlx5_wqe *)wq; + const uint8x16_t ctrl_shuf_m = { + 3, 2, 1, 0, /* bswap32 */ + 7, 6, 5, 4, /* bswap32 */ + 11, 10, 9, 8, /* bswap32 */ + 12, 13, 14, 15 + }; + uint8x16_t *t_wqe; + uint8_t *dseg; + uint8x16_t ctrl; + + /* Make sure all packets can fit into a single WQE. */ + assert(elts_n > pkts_n); + mlx5_tx_complete(txq); + max_elts = (elts_n - (elts_head - txq->elts_tail)); + max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi); + pkts_n = RTE_MIN((unsigned int)RTE_MIN(pkts_n, max_wqe), max_elts); + if (unlikely(!pkts_n)) + return 0; + elts = &(*txq->elts)[elts_head & elts_m]; + /* Loop for available tailroom first. */ + n = RTE_MIN(elts_n - (elts_head & elts_m), pkts_n); + for (pos = 0; pos < (n & -2); pos += 2) + vst1q_u64((void *)&elts[pos], vld1q_u64((void *)&pkts[pos])); + if (n & 1) + elts[pos] = pkts[pos]; + /* Check if it crosses the end of the queue. */ + if (unlikely(n < pkts_n)) { + elts = &(*txq->elts)[0]; + for (pos = 0; pos < pkts_n - n; ++pos) + elts[pos] = pkts[n + pos]; + } + txq->elts_head += pkts_n; + /* Save title WQEBB pointer. */ + t_wqe = (uint8x16_t *)wqe; + dseg = (uint8_t *)(wqe + 1); + /* Calculate the number of entries to the end. */ + n = RTE_MIN( + (wq_n - wq_idx) * nb_dword_per_wqebb - nb_dword_in_hdr, + pkts_n); + /* Fill DSEGs. */ + txq_wr_dseg_v(txq, dseg, pkts, n); + /* Check if it crosses the end of the queue. */ + if (n < pkts_n) { + dseg = (uint8_t *)txq->wqes; + txq_wr_dseg_v(txq, dseg, &pkts[n], pkts_n - n); + } + if (txq->elts_comp + pkts_n < MLX5_TX_COMP_THRESH) { + txq->elts_comp += pkts_n; + } else { + /* Request a completion. */ + txq->elts_comp = 0; + ++txq->cq_pi; + comp_req = 8; + } + /* Fill CTRL in the header. */ + ctrl = vreinterpretq_u8_u32((uint32x4_t) { + MLX5_OPC_MOD_ENHANCED_MPSW << 24 | + txq->wqe_ci << 8 | MLX5_OPCODE_ENHANCED_MPSW, + txq->qp_num_8s | (pkts_n + 2), + comp_req, + txq->elts_head }); + ctrl = vqtbl1q_u8(ctrl, ctrl_shuf_m); + vst1q_u8((void *)t_wqe, ctrl); + /* Fill ESEG in the header. */ + vst1q_u8((void *)(t_wqe + 1), + (uint8x16_t) { 0, 0, 0, 0, + cs_flags, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0 }); +#ifdef MLX5_PMD_SOFT_COUNTERS + txq->stats.opackets += pkts_n; +#endif + txq->wqe_ci += (nb_dword_in_hdr + pkts_n + (nb_dword_per_wqebb - 1)) / + nb_dword_per_wqebb; + /* Ring QP doorbell. */ + mlx5_tx_dbrec_cond_wmb(txq, wqe, pkts_n < MLX5_VPMD_TX_MAX_BURST); + return pkts_n; +} + +/** + * Store free buffers to RX SW ring. + * + * @param rxq + * Pointer to RX queue structure. + * @param pkts + * Pointer to array of packets to be stored. + * @param pkts_n + * Number of packets to be stored. + */ +static inline void +rxq_copy_mbuf_v(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, uint16_t n) +{ + const uint16_t q_mask = (1 << rxq->elts_n) - 1; + struct rte_mbuf **elts = &(*rxq->elts)[rxq->rq_pi & q_mask]; + unsigned int pos; + uint16_t p = n & -2; + + for (pos = 0; pos < p; pos += 2) { + uint64x2_t mbp; + + mbp = vld1q_u64((void *)&elts[pos]); + vst1q_u64((void *)&pkts[pos], mbp); + } + if (n & 1) + pkts[pos] = elts[pos]; +} + +/** + * Decompress a compressed completion and fill in mbufs in RX SW ring with data + * extracted from the title completion descriptor. + * + * @param rxq + * Pointer to RX queue structure. + * @param cq + * Pointer to completion array having a compressed completion at first. + * @param elts + * Pointer to SW ring to be filled. The first mbuf has to be pre-built from + * the title completion descriptor to be copied to the rest of mbufs. + */ +static inline void +rxq_cq_decompress_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq, + struct rte_mbuf **elts) +{ + volatile struct mlx5_mini_cqe8 *mcq = (void *)&(cq + 1)->pkt_info; + struct rte_mbuf *t_pkt = elts[0]; /* Title packet is pre-built. */ + unsigned int pos; + unsigned int i; + unsigned int inv = 0; + /* Mask to shuffle from extracted mini CQE to mbuf. */ + const uint8x16_t mcqe_shuf_m1 = { + -1, -1, -1, -1, /* skip packet_type */ + 7, 6, -1, -1, /* pkt_len, bswap16 */ + 7, 6, /* data_len, bswap16 */ + -1, -1, /* skip vlan_tci */ + 3, 2, 1, 0 /* hash.rss, bswap32 */ + }; + const uint8x16_t mcqe_shuf_m2 = { + -1, -1, -1, -1, /* skip packet_type */ + 15, 14, -1, -1, /* pkt_len, bswap16 */ + 15, 14, /* data_len, bswap16 */ + -1, -1, /* skip vlan_tci */ + 11, 10, 9, 8 /* hash.rss, bswap32 */ + }; + /* Restore the compressed count. Must be 16 bits. */ + const uint16_t mcqe_n = t_pkt->data_len + + (rxq->crc_present * ETHER_CRC_LEN); + const uint64x2_t rearm = + vld1q_u64((void *)&t_pkt->rearm_data); + const uint32x4_t rxdf_mask = { + 0xffffffff, /* packet_type */ + 0, /* skip pkt_len */ + 0xffff0000, /* vlan_tci, skip data_len */ + 0, /* skip hash.rss */ + }; + const uint8x16_t rxdf = + vandq_u8(vld1q_u8((void *)&t_pkt->rx_descriptor_fields1), + vreinterpretq_u8_u32(rxdf_mask)); + const uint16x8_t crc_adj = { + 0, 0, + rxq->crc_present * ETHER_CRC_LEN, 0, + rxq->crc_present * ETHER_CRC_LEN, 0, + 0, 0 + }; + const uint32_t flow_tag = t_pkt->hash.fdir.hi; +#ifdef MLX5_PMD_SOFT_COUNTERS + uint32_t rcvd_byte = 0; +#endif + /* Mask to shuffle byte_cnt to add up stats. Do bswap16 for all. */ + const uint8x8_t len_shuf_m = { + 7, 6, /* 1st mCQE */ + 15, 14, /* 2nd mCQE */ + 23, 22, /* 3rd mCQE */ + 31, 30 /* 4th mCQE */ + }; + + /* + * A. load mCQEs into a 128bit register. + * B. store rearm data to mbuf. + * C. combine data from mCQEs with rx_descriptor_fields1. + * D. store rx_descriptor_fields1. + * E. store flow tag (rte_flow mark). + */ + for (pos = 0; pos < mcqe_n; ) { + uint8_t *p = (void *)&mcq[pos % 8]; + uint8_t *e0 = (void *)&elts[pos]->rearm_data; + uint8_t *e1 = (void *)&elts[pos + 1]->rearm_data; + uint8_t *e2 = (void *)&elts[pos + 2]->rearm_data; + uint8_t *e3 = (void *)&elts[pos + 3]->rearm_data; + uint16x4_t byte_cnt; +#ifdef MLX5_PMD_SOFT_COUNTERS + uint16x4_t invalid_mask = + vcreate_u16(mcqe_n - pos < MLX5_VPMD_DESCS_PER_LOOP ? + -1UL << ((mcqe_n - pos) * + sizeof(uint16_t) * 8) : 0); +#endif + + if (!(pos & 0x7) && pos + 8 < mcqe_n) + rte_prefetch0((void *)(cq + pos + 8)); + __asm__ volatile ( + /* A.1 load mCQEs into a 128bit register. */ + "ld1 {v16.16b - v17.16b}, [%[mcq]] \n\t" + /* B.1 store rearm data to mbuf. */ + "st1 {%[rearm].2d}, [%[e0]] \n\t" + "add %[e0], %[e0], #16 \n\t" + "st1 {%[rearm].2d}, [%[e1]] \n\t" + "add %[e1], %[e1], #16 \n\t" + /* C.1 combine data from mCQEs with rx_descriptor_fields1. */ + "tbl v18.16b, {v16.16b}, %[mcqe_shuf_m1].16b \n\t" + "tbl v19.16b, {v16.16b}, %[mcqe_shuf_m2].16b \n\t" + "sub v18.8h, v18.8h, %[crc_adj].8h \n\t" + "sub v19.8h, v19.8h, %[crc_adj].8h \n\t" + "orr v18.16b, v18.16b, %[rxdf].16b \n\t" + "orr v19.16b, v19.16b, %[rxdf].16b \n\t" + /* D.1 store rx_descriptor_fields1. */ + "st1 {v18.2d}, [%[e0]] \n\t" + "st1 {v19.2d}, [%[e1]] \n\t" + /* B.1 store rearm data to mbuf. */ + "st1 {%[rearm].2d}, [%[e2]] \n\t" + "add %[e2], %[e2], #16 \n\t" + "st1 {%[rearm].2d}, [%[e3]] \n\t" + "add %[e3], %[e3], #16 \n\t" + /* C.1 combine data from mCQEs with rx_descriptor_fields1. */ + "tbl v18.16b, {v17.16b}, %[mcqe_shuf_m1].16b \n\t" + "tbl v19.16b, {v17.16b}, %[mcqe_shuf_m2].16b \n\t" + "sub v18.8h, v18.8h, %[crc_adj].8h \n\t" + "sub v19.8h, v19.8h, %[crc_adj].8h \n\t" + "orr v18.16b, v18.16b, %[rxdf].16b \n\t" + "orr v19.16b, v19.16b, %[rxdf].16b \n\t" + /* D.1 store rx_descriptor_fields1. */ + "st1 {v18.2d}, [%[e2]] \n\t" + "st1 {v19.2d}, [%[e3]] \n\t" +#ifdef MLX5_PMD_SOFT_COUNTERS + "tbl %[byte_cnt].8b, {v16.16b - v17.16b}, %[len_shuf_m].8b \n\t" +#endif + :[byte_cnt]"=&w"(byte_cnt) + :[mcq]"r"(p), + [rxdf]"w"(rxdf), + [rearm]"w"(rearm), + [e3]"r"(e3), [e2]"r"(e2), [e1]"r"(e1), [e0]"r"(e0), + [mcqe_shuf_m1]"w"(mcqe_shuf_m1), + [mcqe_shuf_m2]"w"(mcqe_shuf_m2), + [crc_adj]"w"(crc_adj), + [len_shuf_m]"w"(len_shuf_m) + :"memory", "v16", "v17", "v18", "v19"); +#ifdef MLX5_PMD_SOFT_COUNTERS + byte_cnt = vbic_u16(byte_cnt, invalid_mask); + rcvd_byte += vget_lane_u64(vpaddl_u32(vpaddl_u16(byte_cnt)), 0); +#endif + if (rxq->mark) { + /* E.1 store flow tag (rte_flow mark). */ + elts[pos]->hash.fdir.hi = flow_tag; + elts[pos + 1]->hash.fdir.hi = flow_tag; + elts[pos + 2]->hash.fdir.hi = flow_tag; + elts[pos + 3]->hash.fdir.hi = flow_tag; + } + pos += MLX5_VPMD_DESCS_PER_LOOP; + /* Move to next CQE and invalidate consumed CQEs. */ + if (!(pos & 0x7) && pos < mcqe_n) { + mcq = (void *)&(cq + pos)->pkt_info; + for (i = 0; i < 8; ++i) + cq[inv++].op_own = MLX5_CQE_INVALIDATE; + } + } + /* Invalidate the rest of CQEs. */ + for (; inv < mcqe_n; ++inv) + cq[inv].op_own = MLX5_CQE_INVALIDATE; +#ifdef MLX5_PMD_SOFT_COUNTERS + rxq->stats.ipackets += mcqe_n; + rxq->stats.ibytes += rcvd_byte; +#endif + rxq->cq_ci += mcqe_n; +} + +/** + * Calculate packet type and offload flag for mbuf and store it. + * + * @param rxq + * Pointer to RX queue structure. + * @param ptype_info + * Array of four 4bytes packet type info extracted from the original + * completion descriptor. + * @param flow_tag + * Array of four 4bytes flow ID extracted from the original completion + * descriptor. + * @param op_err + * Opcode vector having responder error status. Each field is 4B. + * @param pkts + * Pointer to array of packets to be filled. + */ +static inline void +rxq_cq_to_ptype_oflags_v(struct mlx5_rxq_data *rxq, + uint32x4_t ptype_info, uint32x4_t flow_tag, + uint16x4_t op_err, struct rte_mbuf **pkts) +{ + uint16x4_t ptype; + uint32x4_t pinfo, cv_flags; + uint32x4_t ol_flags = + vdupq_n_u32(rxq->rss_hash * PKT_RX_RSS_HASH | + rxq->hw_timestamp * PKT_RX_TIMESTAMP); + const uint32x4_t ptype_ol_mask = { 0x106, 0x106, 0x106, 0x106 }; + const uint8x16_t cv_flag_sel = { + 0, + (uint8_t)(PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED), + (uint8_t)(PKT_RX_IP_CKSUM_GOOD >> 1), + 0, + (uint8_t)(PKT_RX_L4_CKSUM_GOOD >> 1), + 0, + (uint8_t)((PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD) >> 1), + 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + const uint32x4_t cv_mask = + vdupq_n_u32(PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD | + PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED); + const uint64x1_t mbuf_init = vld1_u64(&rxq->mbuf_initializer); + const uint64x1_t r32_mask = vcreate_u64(0xffffffff); + uint64x2_t rearm0, rearm1, rearm2, rearm3; + + if (rxq->mark) { + const uint32x4_t ft_def = vdupq_n_u32(MLX5_FLOW_MARK_DEFAULT); + const uint32x4_t fdir_flags = vdupq_n_u32(PKT_RX_FDIR); + uint32x4_t fdir_id_flags = vdupq_n_u32(PKT_RX_FDIR_ID); + uint32x4_t invalid_mask; + + /* Check if flow tag is non-zero then set PKT_RX_FDIR. */ + invalid_mask = vceqzq_u32(flow_tag); + ol_flags = vorrq_u32(ol_flags, + vbicq_u32(fdir_flags, invalid_mask)); + /* Mask out invalid entries. */ + fdir_id_flags = vbicq_u32(fdir_id_flags, invalid_mask); + /* Check if flow tag MLX5_FLOW_MARK_DEFAULT. */ + ol_flags = vorrq_u32(ol_flags, + vbicq_u32(fdir_id_flags, + vceqq_u32(flow_tag, ft_def))); + } + /* + * ptype_info has the following: + * bit[1] = l3_ok + * bit[2] = l4_ok + * bit[8] = cv + * bit[11:10] = l3_hdr_type + * bit[14:12] = l4_hdr_type + * bit[15] = ip_frag + * bit[16] = tunneled + * bit[17] = outer_l3_type + */ + ptype = vshrn_n_u32(ptype_info, 10); + /* Errored packets will have RTE_PTYPE_ALL_MASK. */ + ptype = vorr_u16(ptype, op_err); + pkts[0]->packet_type = + mlx5_ptype_table[vget_lane_u8(vreinterpret_u8_u16(ptype), 6)]; + pkts[1]->packet_type = + mlx5_ptype_table[vget_lane_u8(vreinterpret_u8_u16(ptype), 4)]; + pkts[2]->packet_type = + mlx5_ptype_table[vget_lane_u8(vreinterpret_u8_u16(ptype), 2)]; + pkts[3]->packet_type = + mlx5_ptype_table[vget_lane_u8(vreinterpret_u8_u16(ptype), 0)]; + /* Fill flags for checksum and VLAN. */ + pinfo = vandq_u32(ptype_info, ptype_ol_mask); + pinfo = vreinterpretq_u32_u8( + vqtbl1q_u8(cv_flag_sel, vreinterpretq_u8_u32(pinfo))); + /* Locate checksum flags at byte[2:1] and merge with VLAN flags. */ + cv_flags = vshlq_n_u32(pinfo, 9); + cv_flags = vorrq_u32(pinfo, cv_flags); + /* Move back flags to start from byte[0]. */ + cv_flags = vshrq_n_u32(cv_flags, 8); + /* Mask out garbage bits. */ + cv_flags = vandq_u32(cv_flags, cv_mask); + /* Merge to ol_flags. */ + ol_flags = vorrq_u32(ol_flags, cv_flags); + /* Merge mbuf_init and ol_flags, and store. */ + rearm0 = vcombine_u64(mbuf_init, + vshr_n_u64(vget_high_u64(vreinterpretq_u64_u32( + ol_flags)), 32)); + rearm1 = vcombine_u64(mbuf_init, + vand_u64(vget_high_u64(vreinterpretq_u64_u32( + ol_flags)), r32_mask)); + rearm2 = vcombine_u64(mbuf_init, + vshr_n_u64(vget_low_u64(vreinterpretq_u64_u32( + ol_flags)), 32)); + rearm3 = vcombine_u64(mbuf_init, + vand_u64(vget_low_u64(vreinterpretq_u64_u32( + ol_flags)), r32_mask)); + vst1q_u64((void *)&pkts[0]->rearm_data, rearm0); + vst1q_u64((void *)&pkts[1]->rearm_data, rearm1); + vst1q_u64((void *)&pkts[2]->rearm_data, rearm2); + vst1q_u64((void *)&pkts[3]->rearm_data, rearm3); +} + +/** + * Receive burst of packets. An errored completion also consumes a mbuf, but the + * packet_type is set to be RTE_PTYPE_ALL_MASK. Marked mbufs should be freed + * before returning to application. + * + * @param rxq + * Pointer to RX queue structure. + * @param[out] pkts + * Array to store received packets. + * @param pkts_n + * Maximum number of packets in array. + * @param[out] err + * Pointer to a flag. Set non-zero value if pkts array has at least one error + * packet to handle. + * + * @return + * Number of packets received including errors (<= pkts_n). + */ +static inline uint16_t +rxq_burst_v(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, uint16_t pkts_n, + uint64_t *err) +{ + const uint16_t q_n = 1 << rxq->cqe_n; + const uint16_t q_mask = q_n - 1; + volatile struct mlx5_cqe *cq; + struct rte_mbuf **elts; + unsigned int pos; + uint64_t n; + uint16_t repl_n; + uint64_t comp_idx = MLX5_VPMD_DESCS_PER_LOOP; + uint16_t nocmp_n = 0; + uint16_t rcvd_pkt = 0; + unsigned int cq_idx = rxq->cq_ci & q_mask; + unsigned int elts_idx; + const uint16x4_t ownership = vdup_n_u16(!(rxq->cq_ci & (q_mask + 1))); + const uint16x4_t owner_check = vcreate_u16(0x0001000100010001); + const uint16x4_t opcode_check = vcreate_u16(0x00f000f000f000f0); + const uint16x4_t format_check = vcreate_u16(0x000c000c000c000c); + const uint16x4_t resp_err_check = vcreate_u16(0x00e000e000e000e0); +#ifdef MLX5_PMD_SOFT_COUNTERS + uint32_t rcvd_byte = 0; +#endif + /* Mask to generate 16B length vector. */ + const uint8x8_t len_shuf_m = { + 52, 53, /* 4th CQE */ + 36, 37, /* 3rd CQE */ + 20, 21, /* 2nd CQE */ + 4, 5 /* 1st CQE */ + }; + /* Mask to extract 16B data from a 64B CQE. */ + const uint8x16_t cqe_shuf_m = { + 28, 29, /* hdr_type_etc */ + 0, /* pkt_info */ + -1, /* null */ + 47, 46, /* byte_cnt, bswap16 */ + 31, 30, /* vlan_info, bswap16 */ + 15, 14, 13, 12, /* rx_hash_res, bswap32 */ + 57, 58, 59, /* flow_tag */ + 63 /* op_own */ + }; + /* Mask to generate 16B data for mbuf. */ + const uint8x16_t mb_shuf_m = { + 4, 5, -1, -1, /* pkt_len */ + 4, 5, /* data_len */ + 6, 7, /* vlan_tci */ + 8, 9, 10, 11, /* hash.rss */ + 12, 13, 14, -1 /* hash.fdir.hi */ + }; + /* Mask to generate 16B owner vector. */ + const uint8x8_t owner_shuf_m = { + 63, -1, /* 4th CQE */ + 47, -1, /* 3rd CQE */ + 31, -1, /* 2nd CQE */ + 15, -1 /* 1st CQE */ + }; + /* Mask to generate a vector having packet_type/ol_flags. */ + const uint8x16_t ptype_shuf_m = { + 48, 49, 50, -1, /* 4th CQE */ + 32, 33, 34, -1, /* 3rd CQE */ + 16, 17, 18, -1, /* 2nd CQE */ + 0, 1, 2, -1 /* 1st CQE */ + }; + /* Mask to generate a vector having flow tags. */ + const uint8x16_t ftag_shuf_m = { + 60, 61, 62, -1, /* 4th CQE */ + 44, 45, 46, -1, /* 3rd CQE */ + 28, 29, 30, -1, /* 2nd CQE */ + 12, 13, 14, -1 /* 1st CQE */ + }; + const uint16x8_t crc_adj = { + 0, 0, rxq->crc_present * ETHER_CRC_LEN, 0, 0, 0, 0, 0 + }; + const uint32x4_t flow_mark_adj = { 0, 0, 0, rxq->mark * (-1) }; + + assert(rxq->sges_n == 0); + assert(rxq->cqe_n == rxq->elts_n); + cq = &(*rxq->cqes)[cq_idx]; + rte_prefetch_non_temporal(cq); + rte_prefetch_non_temporal(cq + 1); + rte_prefetch_non_temporal(cq + 2); + rte_prefetch_non_temporal(cq + 3); + pkts_n = RTE_MIN(pkts_n, MLX5_VPMD_RX_MAX_BURST); + /* + * Order of indexes: + * rq_ci >= cq_ci >= rq_pi + * Definition of indexes: + * rq_ci - cq_ci := # of buffers owned by HW (posted). + * cq_ci - rq_pi := # of buffers not returned to app (decompressed). + * N - (rq_ci - rq_pi) := # of buffers consumed (to be replenished). + */ + repl_n = q_n - (rxq->rq_ci - rxq->rq_pi); + if (repl_n >= MLX5_VPMD_RXQ_RPLNSH_THRESH) + mlx5_rx_replenish_bulk_mbuf(rxq, repl_n); + /* See if there're unreturned mbufs from compressed CQE. */ + rcvd_pkt = rxq->cq_ci - rxq->rq_pi; + if (rcvd_pkt > 0) { + rcvd_pkt = RTE_MIN(rcvd_pkt, pkts_n); + rxq_copy_mbuf_v(rxq, pkts, rcvd_pkt); + rxq->rq_pi += rcvd_pkt; + pkts += rcvd_pkt; + } + elts_idx = rxq->rq_pi & q_mask; + elts = &(*rxq->elts)[elts_idx]; + /* Not to overflow pkts array. */ + pkts_n = RTE_ALIGN_FLOOR(pkts_n - rcvd_pkt, MLX5_VPMD_DESCS_PER_LOOP); + /* Not to cross queue end. */ + pkts_n = RTE_MIN(pkts_n, q_n - elts_idx); + if (!pkts_n) + return rcvd_pkt; + /* At this point, there shouldn't be any remained packets. */ + assert(rxq->rq_pi == rxq->cq_ci); + /* + * Note that vectors have reverse order - {v3, v2, v1, v0}, because + * there's no instruction to count trailing zeros. __builtin_clzl() is + * used instead. + * + * A. copy 4 mbuf pointers from elts ring to returing pkts. + * B. load 64B CQE and extract necessary fields + * Final 16bytes cqes[] extracted from original 64bytes CQE has the + * following structure: + * struct { + * uint16_t hdr_type_etc; + * uint8_t pkt_info; + * uint8_t rsvd; + * uint16_t byte_cnt; + * uint16_t vlan_info; + * uint32_t rx_has_res; + * uint8_t flow_tag[3]; + * uint8_t op_own; + * } c; + * C. fill in mbuf. + * D. get valid CQEs. + * E. find compressed CQE. + */ + for (pos = 0; + pos < pkts_n; + pos += MLX5_VPMD_DESCS_PER_LOOP) { + uint16x4_t op_own; + uint16x4_t opcode, owner_mask, invalid_mask; + uint16x4_t comp_mask; + uint16x4_t mask; + uint16x4_t byte_cnt; + uint32x4_t ptype_info, flow_tag; + uint8_t *p0, *p1, *p2, *p3; + uint8_t *e0 = (void *)&elts[pos]->pkt_len; + uint8_t *e1 = (void *)&elts[pos + 1]->pkt_len; + uint8_t *e2 = (void *)&elts[pos + 2]->pkt_len; + uint8_t *e3 = (void *)&elts[pos + 3]->pkt_len; + void *elts_p = (void *)&elts[pos]; + void *pkts_p = (void *)&pkts[pos]; + + /* A.0 do not cross the end of CQ. */ + mask = vcreate_u16(pkts_n - pos < MLX5_VPMD_DESCS_PER_LOOP ? + -1UL >> ((pkts_n - pos) * + sizeof(uint16_t) * 8) : 0); + p0 = (void *)&cq[pos].pkt_info; + p1 = p0 + (pkts_n - pos > 1) * sizeof(struct mlx5_cqe); + p2 = p1 + (pkts_n - pos > 2) * sizeof(struct mlx5_cqe); + p3 = p2 + (pkts_n - pos > 3) * sizeof(struct mlx5_cqe); + /* Prefetch next 4 CQEs. */ + if (pkts_n - pos >= 2 * MLX5_VPMD_DESCS_PER_LOOP) { + unsigned int next = pos + MLX5_VPMD_DESCS_PER_LOOP; + rte_prefetch_non_temporal(&cq[next]); + rte_prefetch_non_temporal(&cq[next + 1]); + rte_prefetch_non_temporal(&cq[next + 2]); + rte_prefetch_non_temporal(&cq[next + 3]); + } + __asm__ volatile ( + /* B.1 (CQE 3) load a block having op_own. */ + "ld1 {v19.16b}, [%[p3]] \n\t" + "sub %[p3], %[p3], #48 \n\t" + /* B.2 (CQE 3) load the rest blocks. */ + "ld1 {v16.16b - v18.16b}, [%[p3]] \n\t" + /* B.3 (CQE 3) extract 16B fields. */ + "tbl v23.16b, {v16.16b - v19.16b}, %[cqe_shuf_m].16b \n\t" + /* B.4 (CQE 3) adjust CRC length. */ + "sub v23.8h, v23.8h, %[crc_adj].8h \n\t" + /* B.1 (CQE 2) load a block having op_own. */ + "ld1 {v19.16b}, [%[p2]] \n\t" + "sub %[p2], %[p2], #48 \n\t" + /* C.1 (CQE 3) generate final structure for mbuf. */ + "tbl v15.16b, {v23.16b}, %[mb_shuf_m].16b \n\t" + /* B.2 (CQE 2) load the rest blocks. */ + "ld1 {v16.16b - v18.16b}, [%[p2]] \n\t" + /* B.3 (CQE 2) extract 16B fields. */ + "tbl v22.16b, {v16.16b - v19.16b}, %[cqe_shuf_m].16b \n\t" + /* B.4 (CQE 2) adjust CRC length. */ + "sub v22.8h, v22.8h, %[crc_adj].8h \n\t" + /* B.1 (CQE 1) load a block having op_own. */ + "ld1 {v19.16b}, [%[p1]] \n\t" + "sub %[p1], %[p1], #48 \n\t" + /* C.1 (CQE 2) generate final structure for mbuf. */ + "tbl v14.16b, {v22.16b}, %[mb_shuf_m].16b \n\t" + /* B.2 (CQE 1) load the rest blocks. */ + "ld1 {v16.16b - v18.16b}, [%[p1]] \n\t" + /* B.3 (CQE 1) extract 16B fields. */ + "tbl v21.16b, {v16.16b - v19.16b}, %[cqe_shuf_m].16b \n\t" + /* B.4 (CQE 1) adjust CRC length. */ + "sub v21.8h, v21.8h, %[crc_adj].8h \n\t" + /* B.1 (CQE 0) load a block having op_own. */ + "ld1 {v19.16b}, [%[p0]] \n\t" + "sub %[p0], %[p0], #48 \n\t" + /* C.1 (CQE 1) generate final structure for mbuf. */ + "tbl v13.16b, {v21.16b}, %[mb_shuf_m].16b \n\t" + /* B.2 (CQE 0) load the rest blocks. */ + "ld1 {v16.16b - v18.16b}, [%[p0]] \n\t" + /* B.3 (CQE 0) extract 16B fields. */ + "tbl v20.16b, {v16.16b - v19.16b}, %[cqe_shuf_m].16b \n\t" + /* B.4 (CQE 0) adjust CRC length. */ + "sub v20.8h, v20.8h, %[crc_adj].8h \n\t" + /* A.1 load mbuf pointers. */ + "ld1 {v24.2d - v25.2d}, [%[elts_p]] \n\t" + /* D.1 extract op_own byte. */ + "tbl %[op_own].8b, {v20.16b - v23.16b}, %[owner_shuf_m].8b \n\t" + /* C.2 (CQE 3) adjust flow mark. */ + "add v15.4s, v15.4s, %[flow_mark_adj].4s \n\t" + /* C.3 (CQE 3) fill in mbuf - rx_descriptor_fields1. */ + "st1 {v15.2d}, [%[e3]] \n\t" + /* C.2 (CQE 2) adjust flow mark. */ + "add v14.4s, v14.4s, %[flow_mark_adj].4s \n\t" + /* C.3 (CQE 2) fill in mbuf - rx_descriptor_fields1. */ + "st1 {v14.2d}, [%[e2]] \n\t" + /* C.1 (CQE 0) generate final structure for mbuf. */ + "tbl v12.16b, {v20.16b}, %[mb_shuf_m].16b \n\t" + /* C.2 (CQE 1) adjust flow mark. */ + "add v13.4s, v13.4s, %[flow_mark_adj].4s \n\t" + /* C.3 (CQE 1) fill in mbuf - rx_descriptor_fields1. */ + "st1 {v13.2d}, [%[e1]] \n\t" +#ifdef MLX5_PMD_SOFT_COUNTERS + /* Extract byte_cnt. */ + "tbl %[byte_cnt].8b, {v20.16b - v23.16b}, %[len_shuf_m].8b \n\t" +#endif + /* Extract ptype_info. */ + "tbl %[ptype_info].16b, {v20.16b - v23.16b}, %[ptype_shuf_m].16b \n\t" + /* Extract flow_tag. */ + "tbl %[flow_tag].16b, {v20.16b - v23.16b}, %[ftag_shuf_m].16b \n\t" + /* A.2 copy mbuf pointers. */ + "st1 {v24.2d - v25.2d}, [%[pkts_p]] \n\t" + /* C.2 (CQE 0) adjust flow mark. */ + "add v12.4s, v12.4s, %[flow_mark_adj].4s \n\t" + /* C.3 (CQE 1) fill in mbuf - rx_descriptor_fields1. */ + "st1 {v12.2d}, [%[e0]] \n\t" + :[op_own]"=&w"(op_own), + [byte_cnt]"=&w"(byte_cnt), + [ptype_info]"=&w"(ptype_info), + [flow_tag]"=&w"(flow_tag) + :[p3]"r"(p3 + 48), [p2]"r"(p2 + 48), + [p1]"r"(p1 + 48), [p0]"r"(p0 + 48), + [e3]"r"(e3), [e2]"r"(e2), [e1]"r"(e1), [e0]"r"(e0), + [elts_p]"r"(elts_p), + [pkts_p]"r"(pkts_p), + [cqe_shuf_m]"w"(cqe_shuf_m), + [mb_shuf_m]"w"(mb_shuf_m), + [owner_shuf_m]"w"(owner_shuf_m), + [len_shuf_m]"w"(len_shuf_m), + [ptype_shuf_m]"w"(ptype_shuf_m), + [ftag_shuf_m]"w"(ftag_shuf_m), + [crc_adj]"w"(crc_adj), + [flow_mark_adj]"w"(flow_mark_adj) + :"memory", + "v12", "v13", "v14", "v15", + "v16", "v17", "v18", "v19", + "v20", "v21", "v22", "v23", + "v24", "v25"); + /* D.2 flip owner bit to mark CQEs from last round. */ + owner_mask = vand_u16(op_own, owner_check); + owner_mask = vceq_u16(owner_mask, ownership); + /* D.3 get mask for invalidated CQEs. */ + opcode = vand_u16(op_own, opcode_check); + invalid_mask = vceq_u16(opcode_check, opcode); + /* E.1 find compressed CQE format. */ + comp_mask = vand_u16(op_own, format_check); + comp_mask = vceq_u16(comp_mask, format_check); + /* D.4 mask out beyond boundary. */ + invalid_mask = vorr_u16(invalid_mask, mask); + /* D.5 merge invalid_mask with invalid owner. */ + invalid_mask = vorr_u16(invalid_mask, owner_mask); + /* E.2 mask out invalid entries. */ + comp_mask = vbic_u16(comp_mask, invalid_mask); + /* E.3 get the first compressed CQE. */ + comp_idx = __builtin_clzl(vget_lane_u64(vreinterpret_u64_u16( + comp_mask), 0)) / + (sizeof(uint16_t) * 8); + /* D.6 mask out entries after the compressed CQE. */ + mask = vcreate_u16(comp_idx < MLX5_VPMD_DESCS_PER_LOOP ? + -1UL >> (comp_idx * sizeof(uint16_t) * 8) : + 0); + invalid_mask = vorr_u16(invalid_mask, mask); + /* D.7 count non-compressed valid CQEs. */ + n = __builtin_clzl(vget_lane_u64(vreinterpret_u64_u16( + invalid_mask), 0)) / (sizeof(uint16_t) * 8); + nocmp_n += n; + /* D.2 get the final invalid mask. */ + mask = vcreate_u16(n < MLX5_VPMD_DESCS_PER_LOOP ? + -1UL >> (n * sizeof(uint16_t) * 8) : 0); + invalid_mask = vorr_u16(invalid_mask, mask); + /* D.3 check error in opcode. */ + opcode = vceq_u16(resp_err_check, opcode); + opcode = vbic_u16(opcode, invalid_mask); + /* D.4 mark if any error is set */ + *err |= vget_lane_u64(vreinterpret_u64_u16(opcode), 0); + /* C.4 fill in mbuf - rearm_data and packet_type. */ + rxq_cq_to_ptype_oflags_v(rxq, ptype_info, flow_tag, + opcode, &elts[pos]); + if (rxq->hw_timestamp) { + elts[pos]->timestamp = + rte_be_to_cpu_64( + container_of(p0, struct mlx5_cqe, + pkt_info)->timestamp); + elts[pos + 1]->timestamp = + rte_be_to_cpu_64( + container_of(p1, struct mlx5_cqe, + pkt_info)->timestamp); + elts[pos + 2]->timestamp = + rte_be_to_cpu_64( + container_of(p2, struct mlx5_cqe, + pkt_info)->timestamp); + elts[pos + 3]->timestamp = + rte_be_to_cpu_64( + container_of(p3, struct mlx5_cqe, + pkt_info)->timestamp); + } +#ifdef MLX5_PMD_SOFT_COUNTERS + /* Add up received bytes count. */ + byte_cnt = vbic_u16(byte_cnt, invalid_mask); + rcvd_byte += vget_lane_u64(vpaddl_u32(vpaddl_u16(byte_cnt)), 0); +#endif + /* + * Break the loop unless more valid CQE is expected, or if + * there's a compressed CQE. + */ + if (n != MLX5_VPMD_DESCS_PER_LOOP) + break; + } + /* If no new CQE seen, return without updating cq_db. */ + if (unlikely(!nocmp_n && comp_idx == MLX5_VPMD_DESCS_PER_LOOP)) + return rcvd_pkt; + /* Update the consumer indexes for non-compressed CQEs. */ + assert(nocmp_n <= pkts_n); + rxq->cq_ci += nocmp_n; + rxq->rq_pi += nocmp_n; + rcvd_pkt += nocmp_n; +#ifdef MLX5_PMD_SOFT_COUNTERS + rxq->stats.ipackets += nocmp_n; + rxq->stats.ibytes += rcvd_byte; +#endif + /* Decompress the last CQE if compressed. */ + if (comp_idx < MLX5_VPMD_DESCS_PER_LOOP && comp_idx == n) { + assert(comp_idx == (nocmp_n % MLX5_VPMD_DESCS_PER_LOOP)); + rxq_cq_decompress_v(rxq, &cq[nocmp_n], &elts[nocmp_n]); + /* Return more packets if needed. */ + if (nocmp_n < pkts_n) { + uint16_t n = rxq->cq_ci - rxq->rq_pi; + + n = RTE_MIN(n, pkts_n - nocmp_n); + rxq_copy_mbuf_v(rxq, &pkts[nocmp_n], n); + rxq->rq_pi += n; + rcvd_pkt += n; + } + } + rte_compiler_barrier(); + *rxq->cq_db = rte_cpu_to_be_32(rxq->cq_ci); + return rcvd_pkt; +} + +#endif /* RTE_PMD_MLX5_RXTX_VEC_NEON_H_ */ diff --git a/dpdk/drivers/net/mlx5/mlx5_rxtx_vec_sse.h b/dpdk/drivers/net/mlx5/mlx5_rxtx_vec_sse.h new file mode 100644 index 00000000..7ef2c59e --- /dev/null +++ b/dpdk/drivers/net/mlx5/mlx5_rxtx_vec_sse.h @@ -0,0 +1,986 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of 6WIND S.A. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef RTE_PMD_MLX5_RXTX_VEC_SSE_H_ +#define RTE_PMD_MLX5_RXTX_VEC_SSE_H_ + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "mlx5.h" +#include "mlx5_utils.h" +#include "mlx5_rxtx.h" +#include "mlx5_rxtx_vec.h" +#include "mlx5_autoconf.h" +#include "mlx5_defs.h" +#include "mlx5_prm.h" + +#ifndef __INTEL_COMPILER +#pragma GCC diagnostic ignored "-Wcast-qual" +#endif + +/** + * Fill in buffer descriptors in a multi-packet send descriptor. + * + * @param txq + * Pointer to TX queue structure. + * @param dseg + * Pointer to buffer descriptor to be written. + * @param pkts + * Pointer to array of packets to be sent. + * @param n + * Number of packets to be filled. + */ +static inline void +txq_wr_dseg_v(struct mlx5_txq_data *txq, __m128i *dseg, + struct rte_mbuf **pkts, unsigned int n) +{ + unsigned int pos; + uintptr_t addr; + const __m128i shuf_mask_dseg = + _mm_set_epi8(8, 9, 10, 11, /* addr, bswap64 */ + 12, 13, 14, 15, + 7, 6, 5, 4, /* lkey */ + 0, 1, 2, 3 /* length, bswap32 */); +#ifdef MLX5_PMD_SOFT_COUNTERS + uint32_t tx_byte = 0; +#endif + + for (pos = 0; pos < n; ++pos, ++dseg) { + __m128i desc; + struct rte_mbuf *pkt = pkts[pos]; + + addr = rte_pktmbuf_mtod(pkt, uintptr_t); + desc = _mm_set_epi32(addr >> 32, + addr, + mlx5_tx_mb2mr(txq, pkt), + DATA_LEN(pkt)); + desc = _mm_shuffle_epi8(desc, shuf_mask_dseg); + _mm_store_si128(dseg, desc); +#ifdef MLX5_PMD_SOFT_COUNTERS + tx_byte += DATA_LEN(pkt); +#endif + } +#ifdef MLX5_PMD_SOFT_COUNTERS + txq->stats.obytes += tx_byte; +#endif +} + +/** + * Send multi-segmented packets until it encounters a single segment packet in + * the pkts list. + * + * @param txq + * Pointer to TX queue structure. + * @param pkts + * Pointer to array of packets to be sent. + * @param pkts_n + * Number of packets to be sent. + * + * @return + * Number of packets successfully transmitted (<= pkts_n). + */ +static uint16_t +txq_scatter_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts, + uint16_t pkts_n) +{ + uint16_t elts_head = txq->elts_head; + const uint16_t elts_n = 1 << txq->elts_n; + const uint16_t elts_m = elts_n - 1; + const uint16_t wq_n = 1 << txq->wqe_n; + const uint16_t wq_mask = wq_n - 1; + const unsigned int nb_dword_per_wqebb = + MLX5_WQE_SIZE / MLX5_WQE_DWORD_SIZE; + const unsigned int nb_dword_in_hdr = + sizeof(struct mlx5_wqe) / MLX5_WQE_DWORD_SIZE; + unsigned int n; + volatile struct mlx5_wqe *wqe = NULL; + + assert(elts_n > pkts_n); + mlx5_tx_complete(txq); + if (unlikely(!pkts_n)) + return 0; + for (n = 0; n < pkts_n; ++n) { + struct rte_mbuf *buf = pkts[n]; + unsigned int segs_n = buf->nb_segs; + unsigned int ds = nb_dword_in_hdr; + unsigned int len = PKT_LEN(buf); + uint16_t wqe_ci = txq->wqe_ci; + const __m128i shuf_mask_ctrl = + _mm_set_epi8(15, 14, 13, 12, + 8, 9, 10, 11, /* bswap32 */ + 4, 5, 6, 7, /* bswap32 */ + 0, 1, 2, 3 /* bswap32 */); + uint8_t cs_flags; + uint16_t max_elts; + uint16_t max_wqe; + __m128i *t_wqe, *dseg; + __m128i ctrl; + + assert(segs_n); + max_elts = elts_n - (elts_head - txq->elts_tail); + max_wqe = wq_n - (txq->wqe_ci - txq->wqe_pi); + /* + * A MPW session consumes 2 WQEs at most to + * include MLX5_MPW_DSEG_MAX pointers. + */ + if (segs_n == 1 || + max_elts < segs_n || max_wqe < 2) + break; + if (segs_n > MLX5_MPW_DSEG_MAX) { + txq->stats.oerrors++; + break; + } + wqe = &((volatile struct mlx5_wqe64 *) + txq->wqes)[wqe_ci & wq_mask].hdr; + cs_flags = txq_ol_cksum_to_cs(txq, buf); + /* Title WQEBB pointer. */ + t_wqe = (__m128i *)wqe; + dseg = (__m128i *)(wqe + 1); + do { + if (!(ds++ % nb_dword_per_wqebb)) { + dseg = (__m128i *) + &((volatile struct mlx5_wqe64 *) + txq->wqes)[++wqe_ci & wq_mask]; + } + txq_wr_dseg_v(txq, dseg++, &buf, 1); + (*txq->elts)[elts_head++ & elts_m] = buf; + buf = buf->next; + } while (--segs_n); + ++wqe_ci; + /* Fill CTRL in the header. */ + ctrl = _mm_set_epi32(0, 0, txq->qp_num_8s | ds, + MLX5_OPC_MOD_MPW << 24 | + txq->wqe_ci << 8 | MLX5_OPCODE_TSO); + ctrl = _mm_shuffle_epi8(ctrl, shuf_mask_ctrl); + _mm_store_si128(t_wqe, ctrl); + /* Fill ESEG in the header. */ + _mm_store_si128(t_wqe + 1, + _mm_set_epi16(0, 0, 0, 0, + rte_cpu_to_be_16(len), cs_flags, + 0, 0)); + txq->wqe_ci = wqe_ci; + } + if (!n) + return 0; + txq->elts_comp += (uint16_t)(elts_head - txq->elts_head); + txq->elts_head = elts_head; + if (txq->elts_comp >= MLX5_TX_COMP_THRESH) { + wqe->ctrl[2] = rte_cpu_to_be_32(8); + wqe->ctrl[3] = txq->elts_head; + txq->elts_comp = 0; + ++txq->cq_pi; + } +#ifdef MLX5_PMD_SOFT_COUNTERS + txq->stats.opackets += n; +#endif + mlx5_tx_dbrec(txq, wqe); + return n; +} + +/** + * Send burst of packets with Enhanced MPW. If it encounters a multi-seg packet, + * it returns to make it processed by txq_scatter_v(). All the packets in + * the pkts list should be single segment packets having same offload flags. + * This must be checked by txq_check_multiseg() and txq_calc_offload(). + * + * @param txq + * Pointer to TX queue structure. + * @param pkts + * Pointer to array of packets to be sent. + * @param pkts_n + * Number of packets to be sent (<= MLX5_VPMD_TX_MAX_BURST). + * @param cs_flags + * Checksum offload flags to be written in the descriptor. + * + * @return + * Number of packets successfully transmitted (<= pkts_n). + */ +static inline uint16_t +txq_burst_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts, uint16_t pkts_n, + uint8_t cs_flags) +{ + struct rte_mbuf **elts; + uint16_t elts_head = txq->elts_head; + const uint16_t elts_n = 1 << txq->elts_n; + const uint16_t elts_m = elts_n - 1; + const unsigned int nb_dword_per_wqebb = + MLX5_WQE_SIZE / MLX5_WQE_DWORD_SIZE; + const unsigned int nb_dword_in_hdr = + sizeof(struct mlx5_wqe) / MLX5_WQE_DWORD_SIZE; + unsigned int n = 0; + unsigned int pos; + uint16_t max_elts; + uint16_t max_wqe; + uint32_t comp_req = 0; + const uint16_t wq_n = 1 << txq->wqe_n; + const uint16_t wq_mask = wq_n - 1; + uint16_t wq_idx = txq->wqe_ci & wq_mask; + volatile struct mlx5_wqe64 *wq = + &((volatile struct mlx5_wqe64 *)txq->wqes)[wq_idx]; + volatile struct mlx5_wqe *wqe = (volatile struct mlx5_wqe *)wq; + const __m128i shuf_mask_ctrl = + _mm_set_epi8(15, 14, 13, 12, + 8, 9, 10, 11, /* bswap32 */ + 4, 5, 6, 7, /* bswap32 */ + 0, 1, 2, 3 /* bswap32 */); + __m128i *t_wqe, *dseg; + __m128i ctrl; + + /* Make sure all packets can fit into a single WQE. */ + assert(elts_n > pkts_n); + mlx5_tx_complete(txq); + max_elts = (elts_n - (elts_head - txq->elts_tail)); + max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi); + pkts_n = RTE_MIN((unsigned int)RTE_MIN(pkts_n, max_wqe), max_elts); + assert(pkts_n <= MLX5_DSEG_MAX - nb_dword_in_hdr); + if (unlikely(!pkts_n)) + return 0; + elts = &(*txq->elts)[elts_head & elts_m]; + /* Loop for available tailroom first. */ + n = RTE_MIN(elts_n - (elts_head & elts_m), pkts_n); + for (pos = 0; pos < (n & -2); pos += 2) + _mm_storeu_si128((__m128i *)&elts[pos], + _mm_loadu_si128((__m128i *)&pkts[pos])); + if (n & 1) + elts[pos] = pkts[pos]; + /* Check if it crosses the end of the queue. */ + if (unlikely(n < pkts_n)) { + elts = &(*txq->elts)[0]; + for (pos = 0; pos < pkts_n - n; ++pos) + elts[pos] = pkts[n + pos]; + } + txq->elts_head += pkts_n; + /* Save title WQEBB pointer. */ + t_wqe = (__m128i *)wqe; + dseg = (__m128i *)(wqe + 1); + /* Calculate the number of entries to the end. */ + n = RTE_MIN( + (wq_n - wq_idx) * nb_dword_per_wqebb - nb_dword_in_hdr, + pkts_n); + /* Fill DSEGs. */ + txq_wr_dseg_v(txq, dseg, pkts, n); + /* Check if it crosses the end of the queue. */ + if (n < pkts_n) { + dseg = (__m128i *)txq->wqes; + txq_wr_dseg_v(txq, dseg, &pkts[n], pkts_n - n); + } + if (txq->elts_comp + pkts_n < MLX5_TX_COMP_THRESH) { + txq->elts_comp += pkts_n; + } else { + /* Request a completion. */ + txq->elts_comp = 0; + ++txq->cq_pi; + comp_req = 8; + } + /* Fill CTRL in the header. */ + ctrl = _mm_set_epi32(txq->elts_head, comp_req, + txq->qp_num_8s | (pkts_n + 2), + MLX5_OPC_MOD_ENHANCED_MPSW << 24 | + txq->wqe_ci << 8 | MLX5_OPCODE_ENHANCED_MPSW); + ctrl = _mm_shuffle_epi8(ctrl, shuf_mask_ctrl); + _mm_store_si128(t_wqe, ctrl); + /* Fill ESEG in the header. */ + _mm_store_si128(t_wqe + 1, + _mm_set_epi8(0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, cs_flags, + 0, 0, 0, 0)); +#ifdef MLX5_PMD_SOFT_COUNTERS + txq->stats.opackets += pkts_n; +#endif + txq->wqe_ci += (nb_dword_in_hdr + pkts_n + (nb_dword_per_wqebb - 1)) / + nb_dword_per_wqebb; + /* Ring QP doorbell. */ + mlx5_tx_dbrec_cond_wmb(txq, wqe, pkts_n < MLX5_VPMD_TX_MAX_BURST); + return pkts_n; +} + +/** + * Store free buffers to RX SW ring. + * + * @param rxq + * Pointer to RX queue structure. + * @param pkts + * Pointer to array of packets to be stored. + * @param pkts_n + * Number of packets to be stored. + */ +static inline void +rxq_copy_mbuf_v(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, uint16_t n) +{ + const uint16_t q_mask = (1 << rxq->elts_n) - 1; + struct rte_mbuf **elts = &(*rxq->elts)[rxq->rq_pi & q_mask]; + unsigned int pos; + uint16_t p = n & -2; + + for (pos = 0; pos < p; pos += 2) { + __m128i mbp; + + mbp = _mm_loadu_si128((__m128i *)&elts[pos]); + _mm_storeu_si128((__m128i *)&pkts[pos], mbp); + } + if (n & 1) + pkts[pos] = elts[pos]; +} + +/** + * Decompress a compressed completion and fill in mbufs in RX SW ring with data + * extracted from the title completion descriptor. + * + * @param rxq + * Pointer to RX queue structure. + * @param cq + * Pointer to completion array having a compressed completion at first. + * @param elts + * Pointer to SW ring to be filled. The first mbuf has to be pre-built from + * the title completion descriptor to be copied to the rest of mbufs. + */ +static inline void +rxq_cq_decompress_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq, + struct rte_mbuf **elts) +{ + volatile struct mlx5_mini_cqe8 *mcq = (void *)(cq + 1); + struct rte_mbuf *t_pkt = elts[0]; /* Title packet is pre-built. */ + unsigned int pos; + unsigned int i; + unsigned int inv = 0; + /* Mask to shuffle from extracted mini CQE to mbuf. */ + const __m128i shuf_mask1 = + _mm_set_epi8(0, 1, 2, 3, /* rss, bswap32 */ + -1, -1, /* skip vlan_tci */ + 6, 7, /* data_len, bswap16 */ + -1, -1, 6, 7, /* pkt_len, bswap16 */ + -1, -1, -1, -1 /* skip packet_type */); + const __m128i shuf_mask2 = + _mm_set_epi8(8, 9, 10, 11, /* rss, bswap32 */ + -1, -1, /* skip vlan_tci */ + 14, 15, /* data_len, bswap16 */ + -1, -1, 14, 15, /* pkt_len, bswap16 */ + -1, -1, -1, -1 /* skip packet_type */); + /* Restore the compressed count. Must be 16 bits. */ + const uint16_t mcqe_n = t_pkt->data_len + + (rxq->crc_present * ETHER_CRC_LEN); + const __m128i rearm = + _mm_loadu_si128((__m128i *)&t_pkt->rearm_data); + const __m128i rxdf = + _mm_loadu_si128((__m128i *)&t_pkt->rx_descriptor_fields1); + const __m128i crc_adj = + _mm_set_epi16(0, 0, 0, + rxq->crc_present * ETHER_CRC_LEN, + 0, + rxq->crc_present * ETHER_CRC_LEN, + 0, 0); + const uint32_t flow_tag = t_pkt->hash.fdir.hi; +#ifdef MLX5_PMD_SOFT_COUNTERS + const __m128i zero = _mm_setzero_si128(); + const __m128i ones = _mm_cmpeq_epi32(zero, zero); + uint32_t rcvd_byte = 0; + /* Mask to shuffle byte_cnt to add up stats. Do bswap16 for all. */ + const __m128i len_shuf_mask = + _mm_set_epi8(-1, -1, -1, -1, + -1, -1, -1, -1, + 14, 15, 6, 7, + 10, 11, 2, 3); +#endif + + /* + * A. load mCQEs into a 128bit register. + * B. store rearm data to mbuf. + * C. combine data from mCQEs with rx_descriptor_fields1. + * D. store rx_descriptor_fields1. + * E. store flow tag (rte_flow mark). + */ + for (pos = 0; pos < mcqe_n; ) { + __m128i mcqe1, mcqe2; + __m128i rxdf1, rxdf2; +#ifdef MLX5_PMD_SOFT_COUNTERS + __m128i byte_cnt, invalid_mask; +#endif + + if (!(pos & 0x7) && pos + 8 < mcqe_n) + rte_prefetch0((void *)(cq + pos + 8)); + /* A.1 load mCQEs into a 128bit register. */ + mcqe1 = _mm_loadu_si128((__m128i *)&mcq[pos % 8]); + mcqe2 = _mm_loadu_si128((__m128i *)&mcq[pos % 8 + 2]); + /* B.1 store rearm data to mbuf. */ + _mm_storeu_si128((__m128i *)&elts[pos]->rearm_data, rearm); + _mm_storeu_si128((__m128i *)&elts[pos + 1]->rearm_data, rearm); + /* C.1 combine data from mCQEs with rx_descriptor_fields1. */ + rxdf1 = _mm_shuffle_epi8(mcqe1, shuf_mask1); + rxdf2 = _mm_shuffle_epi8(mcqe1, shuf_mask2); + rxdf1 = _mm_sub_epi16(rxdf1, crc_adj); + rxdf2 = _mm_sub_epi16(rxdf2, crc_adj); + rxdf1 = _mm_blend_epi16(rxdf1, rxdf, 0x23); + rxdf2 = _mm_blend_epi16(rxdf2, rxdf, 0x23); + /* D.1 store rx_descriptor_fields1. */ + _mm_storeu_si128((__m128i *) + &elts[pos]->rx_descriptor_fields1, + rxdf1); + _mm_storeu_si128((__m128i *) + &elts[pos + 1]->rx_descriptor_fields1, + rxdf2); + /* B.1 store rearm data to mbuf. */ + _mm_storeu_si128((__m128i *)&elts[pos + 2]->rearm_data, rearm); + _mm_storeu_si128((__m128i *)&elts[pos + 3]->rearm_data, rearm); + /* C.1 combine data from mCQEs with rx_descriptor_fields1. */ + rxdf1 = _mm_shuffle_epi8(mcqe2, shuf_mask1); + rxdf2 = _mm_shuffle_epi8(mcqe2, shuf_mask2); + rxdf1 = _mm_sub_epi16(rxdf1, crc_adj); + rxdf2 = _mm_sub_epi16(rxdf2, crc_adj); + rxdf1 = _mm_blend_epi16(rxdf1, rxdf, 0x23); + rxdf2 = _mm_blend_epi16(rxdf2, rxdf, 0x23); + /* D.1 store rx_descriptor_fields1. */ + _mm_storeu_si128((__m128i *) + &elts[pos + 2]->rx_descriptor_fields1, + rxdf1); + _mm_storeu_si128((__m128i *) + &elts[pos + 3]->rx_descriptor_fields1, + rxdf2); +#ifdef MLX5_PMD_SOFT_COUNTERS + invalid_mask = _mm_set_epi64x(0, + (mcqe_n - pos) * + sizeof(uint16_t) * 8); + invalid_mask = _mm_sll_epi64(ones, invalid_mask); + mcqe1 = _mm_srli_si128(mcqe1, 4); + byte_cnt = _mm_blend_epi16(mcqe1, mcqe2, 0xcc); + byte_cnt = _mm_shuffle_epi8(byte_cnt, len_shuf_mask); + byte_cnt = _mm_andnot_si128(invalid_mask, byte_cnt); + byte_cnt = _mm_hadd_epi16(byte_cnt, zero); + rcvd_byte += _mm_cvtsi128_si64(_mm_hadd_epi16(byte_cnt, zero)); +#endif + if (rxq->mark) { + /* E.1 store flow tag (rte_flow mark). */ + elts[pos]->hash.fdir.hi = flow_tag; + elts[pos + 1]->hash.fdir.hi = flow_tag; + elts[pos + 2]->hash.fdir.hi = flow_tag; + elts[pos + 3]->hash.fdir.hi = flow_tag; + } + pos += MLX5_VPMD_DESCS_PER_LOOP; + /* Move to next CQE and invalidate consumed CQEs. */ + if (!(pos & 0x7) && pos < mcqe_n) { + mcq = (void *)(cq + pos); + for (i = 0; i < 8; ++i) + cq[inv++].op_own = MLX5_CQE_INVALIDATE; + } + } + /* Invalidate the rest of CQEs. */ + for (; inv < mcqe_n; ++inv) + cq[inv].op_own = MLX5_CQE_INVALIDATE; +#ifdef MLX5_PMD_SOFT_COUNTERS + rxq->stats.ipackets += mcqe_n; + rxq->stats.ibytes += rcvd_byte; +#endif + rxq->cq_ci += mcqe_n; +} + +/** + * Calculate packet type and offload flag for mbuf and store it. + * + * @param rxq + * Pointer to RX queue structure. + * @param cqes[4] + * Array of four 16bytes completions extracted from the original completion + * descriptor. + * @param op_err + * Opcode vector having responder error status. Each field is 4B. + * @param pkts + * Pointer to array of packets to be filled. + */ +static inline void +rxq_cq_to_ptype_oflags_v(struct mlx5_rxq_data *rxq, __m128i cqes[4], + __m128i op_err, struct rte_mbuf **pkts) +{ + __m128i pinfo0, pinfo1; + __m128i pinfo, ptype; + __m128i ol_flags = _mm_set1_epi32(rxq->rss_hash * PKT_RX_RSS_HASH | + rxq->hw_timestamp * PKT_RX_TIMESTAMP); + __m128i cv_flags; + const __m128i zero = _mm_setzero_si128(); + const __m128i ptype_mask = + _mm_set_epi32(0xfd06, 0xfd06, 0xfd06, 0xfd06); + const __m128i ptype_ol_mask = + _mm_set_epi32(0x106, 0x106, 0x106, 0x106); + const __m128i pinfo_mask = + _mm_set_epi32(0x3, 0x3, 0x3, 0x3); + const __m128i cv_flag_sel = + _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, + (uint8_t)((PKT_RX_IP_CKSUM_GOOD | + PKT_RX_L4_CKSUM_GOOD) >> 1), + 0, + (uint8_t)(PKT_RX_L4_CKSUM_GOOD >> 1), + 0, + (uint8_t)(PKT_RX_IP_CKSUM_GOOD >> 1), + (uint8_t)(PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED), + 0); + const __m128i cv_mask = + _mm_set_epi32(PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD | + PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED, + PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD | + PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED, + PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD | + PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED, + PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD | + PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED); + const __m128i mbuf_init = + _mm_loadl_epi64((__m128i *)&rxq->mbuf_initializer); + __m128i rearm0, rearm1, rearm2, rearm3; + + /* Extract pkt_info field. */ + pinfo0 = _mm_unpacklo_epi32(cqes[0], cqes[1]); + pinfo1 = _mm_unpacklo_epi32(cqes[2], cqes[3]); + pinfo = _mm_unpacklo_epi64(pinfo0, pinfo1); + /* Extract hdr_type_etc field. */ + pinfo0 = _mm_unpackhi_epi32(cqes[0], cqes[1]); + pinfo1 = _mm_unpackhi_epi32(cqes[2], cqes[3]); + ptype = _mm_unpacklo_epi64(pinfo0, pinfo1); + if (rxq->mark) { + const __m128i pinfo_ft_mask = + _mm_set_epi32(0xffffff00, 0xffffff00, + 0xffffff00, 0xffffff00); + const __m128i fdir_flags = _mm_set1_epi32(PKT_RX_FDIR); + __m128i fdir_id_flags = _mm_set1_epi32(PKT_RX_FDIR_ID); + __m128i flow_tag, invalid_mask; + + flow_tag = _mm_and_si128(pinfo, pinfo_ft_mask); + /* Check if flow tag is non-zero then set PKT_RX_FDIR. */ + invalid_mask = _mm_cmpeq_epi32(flow_tag, zero); + ol_flags = _mm_or_si128(ol_flags, + _mm_andnot_si128(invalid_mask, + fdir_flags)); + /* Mask out invalid entries. */ + fdir_id_flags = _mm_andnot_si128(invalid_mask, fdir_id_flags); + /* Check if flow tag MLX5_FLOW_MARK_DEFAULT. */ + ol_flags = _mm_or_si128(ol_flags, + _mm_andnot_si128( + _mm_cmpeq_epi32(flow_tag, + pinfo_ft_mask), + fdir_id_flags)); + } + /* + * Merge the two fields to generate the following: + * bit[1] = l3_ok + * bit[2] = l4_ok + * bit[8] = cv + * bit[11:10] = l3_hdr_type + * bit[14:12] = l4_hdr_type + * bit[15] = ip_frag + * bit[16] = tunneled + * bit[17] = outer_l3_type + */ + ptype = _mm_and_si128(ptype, ptype_mask); + pinfo = _mm_and_si128(pinfo, pinfo_mask); + pinfo = _mm_slli_epi32(pinfo, 16); + /* Make pinfo has merged fields for ol_flags calculation. */ + pinfo = _mm_or_si128(ptype, pinfo); + ptype = _mm_srli_epi32(pinfo, 10); + ptype = _mm_packs_epi32(ptype, zero); + /* Errored packets will have RTE_PTYPE_ALL_MASK. */ + op_err = _mm_srli_epi16(op_err, 8); + ptype = _mm_or_si128(ptype, op_err); + pkts[0]->packet_type = mlx5_ptype_table[_mm_extract_epi8(ptype, 0)]; + pkts[1]->packet_type = mlx5_ptype_table[_mm_extract_epi8(ptype, 2)]; + pkts[2]->packet_type = mlx5_ptype_table[_mm_extract_epi8(ptype, 4)]; + pkts[3]->packet_type = mlx5_ptype_table[_mm_extract_epi8(ptype, 6)]; + /* Fill flags for checksum and VLAN. */ + pinfo = _mm_and_si128(pinfo, ptype_ol_mask); + pinfo = _mm_shuffle_epi8(cv_flag_sel, pinfo); + /* Locate checksum flags at byte[2:1] and merge with VLAN flags. */ + cv_flags = _mm_slli_epi32(pinfo, 9); + cv_flags = _mm_or_si128(pinfo, cv_flags); + /* Move back flags to start from byte[0]. */ + cv_flags = _mm_srli_epi32(cv_flags, 8); + /* Mask out garbage bits. */ + cv_flags = _mm_and_si128(cv_flags, cv_mask); + /* Merge to ol_flags. */ + ol_flags = _mm_or_si128(ol_flags, cv_flags); + /* Merge mbuf_init and ol_flags. */ + rearm0 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(ol_flags, 8), 0x30); + rearm1 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(ol_flags, 4), 0x30); + rearm2 = _mm_blend_epi16(mbuf_init, ol_flags, 0x30); + rearm3 = _mm_blend_epi16(mbuf_init, _mm_srli_si128(ol_flags, 4), 0x30); + /* Write 8B rearm_data and 8B ol_flags. */ + _mm_store_si128((__m128i *)&pkts[0]->rearm_data, rearm0); + _mm_store_si128((__m128i *)&pkts[1]->rearm_data, rearm1); + _mm_store_si128((__m128i *)&pkts[2]->rearm_data, rearm2); + _mm_store_si128((__m128i *)&pkts[3]->rearm_data, rearm3); +} + +/** + * Receive burst of packets. An errored completion also consumes a mbuf, but the + * packet_type is set to be RTE_PTYPE_ALL_MASK. Marked mbufs should be freed + * before returning to application. + * + * @param rxq + * Pointer to RX queue structure. + * @param[out] pkts + * Array to store received packets. + * @param pkts_n + * Maximum number of packets in array. + * @param[out] err + * Pointer to a flag. Set non-zero value if pkts array has at least one error + * packet to handle. + * + * @return + * Number of packets received including errors (<= pkts_n). + */ +static inline uint16_t +rxq_burst_v(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, uint16_t pkts_n, + uint64_t *err) +{ + const uint16_t q_n = 1 << rxq->cqe_n; + const uint16_t q_mask = q_n - 1; + volatile struct mlx5_cqe *cq; + struct rte_mbuf **elts; + unsigned int pos; + uint64_t n; + uint16_t repl_n; + uint64_t comp_idx = MLX5_VPMD_DESCS_PER_LOOP; + uint16_t nocmp_n = 0; + uint16_t rcvd_pkt = 0; + unsigned int cq_idx = rxq->cq_ci & q_mask; + unsigned int elts_idx; + unsigned int ownership = !!(rxq->cq_ci & (q_mask + 1)); + const __m128i owner_check = + _mm_set_epi64x(0x0100000001000000LL, 0x0100000001000000LL); + const __m128i opcode_check = + _mm_set_epi64x(0xf0000000f0000000LL, 0xf0000000f0000000LL); + const __m128i format_check = + _mm_set_epi64x(0x0c0000000c000000LL, 0x0c0000000c000000LL); + const __m128i resp_err_check = + _mm_set_epi64x(0xe0000000e0000000LL, 0xe0000000e0000000LL); +#ifdef MLX5_PMD_SOFT_COUNTERS + uint32_t rcvd_byte = 0; + /* Mask to shuffle byte_cnt to add up stats. Do bswap16 for all. */ + const __m128i len_shuf_mask = + _mm_set_epi8(-1, -1, -1, -1, + -1, -1, -1, -1, + 12, 13, 8, 9, + 4, 5, 0, 1); +#endif + /* Mask to shuffle from extracted CQE to mbuf. */ + const __m128i shuf_mask = + _mm_set_epi8(-1, 3, 2, 1, /* fdir.hi */ + 12, 13, 14, 15, /* rss, bswap32 */ + 10, 11, /* vlan_tci, bswap16 */ + 4, 5, /* data_len, bswap16 */ + -1, -1, /* zero out 2nd half of pkt_len */ + 4, 5 /* pkt_len, bswap16 */); + /* Mask to blend from the last Qword to the first DQword. */ + const __m128i blend_mask = + _mm_set_epi8(-1, -1, -1, -1, + -1, -1, -1, -1, + 0, 0, 0, 0, + 0, 0, 0, -1); + const __m128i zero = _mm_setzero_si128(); + const __m128i ones = _mm_cmpeq_epi32(zero, zero); + const __m128i crc_adj = + _mm_set_epi16(0, 0, 0, 0, 0, + rxq->crc_present * ETHER_CRC_LEN, + 0, + rxq->crc_present * ETHER_CRC_LEN); + const __m128i flow_mark_adj = _mm_set_epi32(rxq->mark * (-1), 0, 0, 0); + + assert(rxq->sges_n == 0); + assert(rxq->cqe_n == rxq->elts_n); + cq = &(*rxq->cqes)[cq_idx]; + rte_prefetch0(cq); + rte_prefetch0(cq + 1); + rte_prefetch0(cq + 2); + rte_prefetch0(cq + 3); + pkts_n = RTE_MIN(pkts_n, MLX5_VPMD_RX_MAX_BURST); + /* + * Order of indexes: + * rq_ci >= cq_ci >= rq_pi + * Definition of indexes: + * rq_ci - cq_ci := # of buffers owned by HW (posted). + * cq_ci - rq_pi := # of buffers not returned to app (decompressed). + * N - (rq_ci - rq_pi) := # of buffers consumed (to be replenished). + */ + repl_n = q_n - (rxq->rq_ci - rxq->rq_pi); + if (repl_n >= MLX5_VPMD_RXQ_RPLNSH_THRESH) + mlx5_rx_replenish_bulk_mbuf(rxq, repl_n); + /* See if there're unreturned mbufs from compressed CQE. */ + rcvd_pkt = rxq->cq_ci - rxq->rq_pi; + if (rcvd_pkt > 0) { + rcvd_pkt = RTE_MIN(rcvd_pkt, pkts_n); + rxq_copy_mbuf_v(rxq, pkts, rcvd_pkt); + rxq->rq_pi += rcvd_pkt; + pkts += rcvd_pkt; + } + elts_idx = rxq->rq_pi & q_mask; + elts = &(*rxq->elts)[elts_idx]; + /* Not to overflow pkts array. */ + pkts_n = RTE_ALIGN_FLOOR(pkts_n - rcvd_pkt, MLX5_VPMD_DESCS_PER_LOOP); + /* Not to cross queue end. */ + pkts_n = RTE_MIN(pkts_n, q_n - elts_idx); + if (!pkts_n) + return rcvd_pkt; + /* At this point, there shouldn't be any remained packets. */ + assert(rxq->rq_pi == rxq->cq_ci); + /* + * A. load first Qword (8bytes) in one loop. + * B. copy 4 mbuf pointers from elts ring to returing pkts. + * C. load remained CQE data and extract necessary fields. + * Final 16bytes cqes[] extracted from original 64bytes CQE has the + * following structure: + * struct { + * uint8_t pkt_info; + * uint8_t flow_tag[3]; + * uint16_t byte_cnt; + * uint8_t rsvd4; + * uint8_t op_own; + * uint16_t hdr_type_etc; + * uint16_t vlan_info; + * uint32_t rx_has_res; + * } c; + * D. fill in mbuf. + * E. get valid CQEs. + * F. find compressed CQE. + */ + for (pos = 0; + pos < pkts_n; + pos += MLX5_VPMD_DESCS_PER_LOOP) { + __m128i cqes[MLX5_VPMD_DESCS_PER_LOOP]; + __m128i cqe_tmp1, cqe_tmp2; + __m128i pkt_mb0, pkt_mb1, pkt_mb2, pkt_mb3; + __m128i op_own, op_own_tmp1, op_own_tmp2; + __m128i opcode, owner_mask, invalid_mask; + __m128i comp_mask; + __m128i mask; +#ifdef MLX5_PMD_SOFT_COUNTERS + __m128i byte_cnt; +#endif + __m128i mbp1, mbp2; + __m128i p = _mm_set_epi16(0, 0, 0, 0, 3, 2, 1, 0); + unsigned int p1, p2, p3; + + /* Prefetch next 4 CQEs. */ + if (pkts_n - pos >= 2 * MLX5_VPMD_DESCS_PER_LOOP) { + rte_prefetch0(&cq[pos + MLX5_VPMD_DESCS_PER_LOOP]); + rte_prefetch0(&cq[pos + MLX5_VPMD_DESCS_PER_LOOP + 1]); + rte_prefetch0(&cq[pos + MLX5_VPMD_DESCS_PER_LOOP + 2]); + rte_prefetch0(&cq[pos + MLX5_VPMD_DESCS_PER_LOOP + 3]); + } + /* A.0 do not cross the end of CQ. */ + mask = _mm_set_epi64x(0, (pkts_n - pos) * sizeof(uint16_t) * 8); + mask = _mm_sll_epi64(ones, mask); + p = _mm_andnot_si128(mask, p); + /* A.1 load cqes. */ + p3 = _mm_extract_epi16(p, 3); + cqes[3] = _mm_loadl_epi64((__m128i *) + &cq[pos + p3].sop_drop_qpn); + rte_compiler_barrier(); + p2 = _mm_extract_epi16(p, 2); + cqes[2] = _mm_loadl_epi64((__m128i *) + &cq[pos + p2].sop_drop_qpn); + rte_compiler_barrier(); + /* B.1 load mbuf pointers. */ + mbp1 = _mm_loadu_si128((__m128i *)&elts[pos]); + mbp2 = _mm_loadu_si128((__m128i *)&elts[pos + 2]); + /* A.1 load a block having op_own. */ + p1 = _mm_extract_epi16(p, 1); + cqes[1] = _mm_loadl_epi64((__m128i *) + &cq[pos + p1].sop_drop_qpn); + rte_compiler_barrier(); + cqes[0] = _mm_loadl_epi64((__m128i *) + &cq[pos].sop_drop_qpn); + /* B.2 copy mbuf pointers. */ + _mm_storeu_si128((__m128i *)&pkts[pos], mbp1); + _mm_storeu_si128((__m128i *)&pkts[pos + 2], mbp2); + rte_compiler_barrier(); + /* C.1 load remained CQE data and extract necessary fields. */ + cqe_tmp2 = _mm_load_si128((__m128i *)&cq[pos + p3]); + cqe_tmp1 = _mm_load_si128((__m128i *)&cq[pos + p2]); + cqes[3] = _mm_blendv_epi8(cqes[3], cqe_tmp2, blend_mask); + cqes[2] = _mm_blendv_epi8(cqes[2], cqe_tmp1, blend_mask); + cqe_tmp2 = _mm_loadu_si128((__m128i *)&cq[pos + p3].rsvd1[3]); + cqe_tmp1 = _mm_loadu_si128((__m128i *)&cq[pos + p2].rsvd1[3]); + cqes[3] = _mm_blend_epi16(cqes[3], cqe_tmp2, 0x30); + cqes[2] = _mm_blend_epi16(cqes[2], cqe_tmp1, 0x30); + cqe_tmp2 = _mm_loadl_epi64((__m128i *)&cq[pos + p3].rsvd2[10]); + cqe_tmp1 = _mm_loadl_epi64((__m128i *)&cq[pos + p2].rsvd2[10]); + cqes[3] = _mm_blend_epi16(cqes[3], cqe_tmp2, 0x04); + cqes[2] = _mm_blend_epi16(cqes[2], cqe_tmp1, 0x04); + /* C.2 generate final structure for mbuf with swapping bytes. */ + pkt_mb3 = _mm_shuffle_epi8(cqes[3], shuf_mask); + pkt_mb2 = _mm_shuffle_epi8(cqes[2], shuf_mask); + /* C.3 adjust CRC length. */ + pkt_mb3 = _mm_sub_epi16(pkt_mb3, crc_adj); + pkt_mb2 = _mm_sub_epi16(pkt_mb2, crc_adj); + /* C.4 adjust flow mark. */ + pkt_mb3 = _mm_add_epi32(pkt_mb3, flow_mark_adj); + pkt_mb2 = _mm_add_epi32(pkt_mb2, flow_mark_adj); + /* D.1 fill in mbuf - rx_descriptor_fields1. */ + _mm_storeu_si128((void *)&pkts[pos + 3]->pkt_len, pkt_mb3); + _mm_storeu_si128((void *)&pkts[pos + 2]->pkt_len, pkt_mb2); + /* E.1 extract op_own field. */ + op_own_tmp2 = _mm_unpacklo_epi32(cqes[2], cqes[3]); + /* C.1 load remained CQE data and extract necessary fields. */ + cqe_tmp2 = _mm_load_si128((__m128i *)&cq[pos + p1]); + cqe_tmp1 = _mm_load_si128((__m128i *)&cq[pos]); + cqes[1] = _mm_blendv_epi8(cqes[1], cqe_tmp2, blend_mask); + cqes[0] = _mm_blendv_epi8(cqes[0], cqe_tmp1, blend_mask); + cqe_tmp2 = _mm_loadu_si128((__m128i *)&cq[pos + p1].rsvd1[3]); + cqe_tmp1 = _mm_loadu_si128((__m128i *)&cq[pos].rsvd1[3]); + cqes[1] = _mm_blend_epi16(cqes[1], cqe_tmp2, 0x30); + cqes[0] = _mm_blend_epi16(cqes[0], cqe_tmp1, 0x30); + cqe_tmp2 = _mm_loadl_epi64((__m128i *)&cq[pos + p1].rsvd2[10]); + cqe_tmp1 = _mm_loadl_epi64((__m128i *)&cq[pos].rsvd2[10]); + cqes[1] = _mm_blend_epi16(cqes[1], cqe_tmp2, 0x04); + cqes[0] = _mm_blend_epi16(cqes[0], cqe_tmp1, 0x04); + /* C.2 generate final structure for mbuf with swapping bytes. */ + pkt_mb1 = _mm_shuffle_epi8(cqes[1], shuf_mask); + pkt_mb0 = _mm_shuffle_epi8(cqes[0], shuf_mask); + /* C.3 adjust CRC length. */ + pkt_mb1 = _mm_sub_epi16(pkt_mb1, crc_adj); + pkt_mb0 = _mm_sub_epi16(pkt_mb0, crc_adj); + /* C.4 adjust flow mark. */ + pkt_mb1 = _mm_add_epi32(pkt_mb1, flow_mark_adj); + pkt_mb0 = _mm_add_epi32(pkt_mb0, flow_mark_adj); + /* E.1 extract op_own byte. */ + op_own_tmp1 = _mm_unpacklo_epi32(cqes[0], cqes[1]); + op_own = _mm_unpackhi_epi64(op_own_tmp1, op_own_tmp2); + /* D.1 fill in mbuf - rx_descriptor_fields1. */ + _mm_storeu_si128((void *)&pkts[pos + 1]->pkt_len, pkt_mb1); + _mm_storeu_si128((void *)&pkts[pos]->pkt_len, pkt_mb0); + /* E.2 flip owner bit to mark CQEs from last round. */ + owner_mask = _mm_and_si128(op_own, owner_check); + if (ownership) + owner_mask = _mm_xor_si128(owner_mask, owner_check); + owner_mask = _mm_cmpeq_epi32(owner_mask, owner_check); + owner_mask = _mm_packs_epi32(owner_mask, zero); + /* E.3 get mask for invalidated CQEs. */ + opcode = _mm_and_si128(op_own, opcode_check); + invalid_mask = _mm_cmpeq_epi32(opcode_check, opcode); + invalid_mask = _mm_packs_epi32(invalid_mask, zero); + /* E.4 mask out beyond boundary. */ + invalid_mask = _mm_or_si128(invalid_mask, mask); + /* E.5 merge invalid_mask with invalid owner. */ + invalid_mask = _mm_or_si128(invalid_mask, owner_mask); + /* F.1 find compressed CQE format. */ + comp_mask = _mm_and_si128(op_own, format_check); + comp_mask = _mm_cmpeq_epi32(comp_mask, format_check); + comp_mask = _mm_packs_epi32(comp_mask, zero); + /* F.2 mask out invalid entries. */ + comp_mask = _mm_andnot_si128(invalid_mask, comp_mask); + comp_idx = _mm_cvtsi128_si64(comp_mask); + /* F.3 get the first compressed CQE. */ + comp_idx = comp_idx ? + __builtin_ctzll(comp_idx) / + (sizeof(uint16_t) * 8) : + MLX5_VPMD_DESCS_PER_LOOP; + /* E.6 mask out entries after the compressed CQE. */ + mask = _mm_set_epi64x(0, comp_idx * sizeof(uint16_t) * 8); + mask = _mm_sll_epi64(ones, mask); + invalid_mask = _mm_or_si128(invalid_mask, mask); + /* E.7 count non-compressed valid CQEs. */ + n = _mm_cvtsi128_si64(invalid_mask); + n = n ? __builtin_ctzll(n) / (sizeof(uint16_t) * 8) : + MLX5_VPMD_DESCS_PER_LOOP; + nocmp_n += n; + /* D.2 get the final invalid mask. */ + mask = _mm_set_epi64x(0, n * sizeof(uint16_t) * 8); + mask = _mm_sll_epi64(ones, mask); + invalid_mask = _mm_or_si128(invalid_mask, mask); + /* D.3 check error in opcode. */ + opcode = _mm_cmpeq_epi32(resp_err_check, opcode); + opcode = _mm_packs_epi32(opcode, zero); + opcode = _mm_andnot_si128(invalid_mask, opcode); + /* D.4 mark if any error is set */ + *err |= _mm_cvtsi128_si64(opcode); + /* D.5 fill in mbuf - rearm_data and packet_type. */ + rxq_cq_to_ptype_oflags_v(rxq, cqes, opcode, &pkts[pos]); + if (rxq->hw_timestamp) { + pkts[pos]->timestamp = + rte_be_to_cpu_64(cq[pos].timestamp); + pkts[pos + 1]->timestamp = + rte_be_to_cpu_64(cq[pos + p1].timestamp); + pkts[pos + 2]->timestamp = + rte_be_to_cpu_64(cq[pos + p2].timestamp); + pkts[pos + 3]->timestamp = + rte_be_to_cpu_64(cq[pos + p3].timestamp); + } +#ifdef MLX5_PMD_SOFT_COUNTERS + /* Add up received bytes count. */ + byte_cnt = _mm_shuffle_epi8(op_own, len_shuf_mask); + byte_cnt = _mm_andnot_si128(invalid_mask, byte_cnt); + byte_cnt = _mm_hadd_epi16(byte_cnt, zero); + rcvd_byte += _mm_cvtsi128_si64(_mm_hadd_epi16(byte_cnt, zero)); +#endif + /* + * Break the loop unless more valid CQE is expected, or if + * there's a compressed CQE. + */ + if (n != MLX5_VPMD_DESCS_PER_LOOP) + break; + } + /* If no new CQE seen, return without updating cq_db. */ + if (unlikely(!nocmp_n && comp_idx == MLX5_VPMD_DESCS_PER_LOOP)) + return rcvd_pkt; + /* Update the consumer indexes for non-compressed CQEs. */ + assert(nocmp_n <= pkts_n); + rxq->cq_ci += nocmp_n; + rxq->rq_pi += nocmp_n; + rcvd_pkt += nocmp_n; +#ifdef MLX5_PMD_SOFT_COUNTERS + rxq->stats.ipackets += nocmp_n; + rxq->stats.ibytes += rcvd_byte; +#endif + /* Decompress the last CQE if compressed. */ + if (comp_idx < MLX5_VPMD_DESCS_PER_LOOP && comp_idx == n) { + assert(comp_idx == (nocmp_n % MLX5_VPMD_DESCS_PER_LOOP)); + rxq_cq_decompress_v(rxq, &cq[nocmp_n], &elts[nocmp_n]); + /* Return more packets if needed. */ + if (nocmp_n < pkts_n) { + uint16_t n = rxq->cq_ci - rxq->rq_pi; + + n = RTE_MIN(n, pkts_n - nocmp_n); + rxq_copy_mbuf_v(rxq, &pkts[nocmp_n], n); + rxq->rq_pi += n; + rcvd_pkt += n; + } + } + rte_compiler_barrier(); + *rxq->cq_db = rte_cpu_to_be_32(rxq->cq_ci); + return rcvd_pkt; +} + +#endif /* RTE_PMD_MLX5_RXTX_VEC_SSE_H_ */ diff --git a/dpdk/drivers/net/mlx5/mlx5_socket.c b/dpdk/drivers/net/mlx5/mlx5_socket.c new file mode 100644 index 00000000..5cd1ab80 --- /dev/null +++ b/dpdk/drivers/net/mlx5/mlx5_socket.c @@ -0,0 +1,294 @@ +/*- + * BSD LICENSE + * + * Copyright 2016 6WIND S.A. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of 6WIND S.A. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include +#include + +#include "mlx5.h" +#include "mlx5_utils.h" + +/** + * Initialise the socket to communicate with the secondary process + * + * @param[in] priv + * Pointer to private structure. + * + * @return + * 0 on success, errno value on failure. + */ +int +priv_socket_init(struct priv *priv) +{ + struct sockaddr_un sun = { + .sun_family = AF_UNIX, + }; + int ret; + int flags; + struct stat file_stat; + + /* + * Initialise the socket to communicate with the secondary + * process. + */ + ret = socket(AF_UNIX, SOCK_STREAM, 0); + if (ret < 0) { + WARN("secondary process not supported: %s", strerror(errno)); + return ret; + } + priv->primary_socket = ret; + flags = fcntl(priv->primary_socket, F_GETFL, 0); + if (flags == -1) + goto out; + ret = fcntl(priv->primary_socket, F_SETFL, flags | O_NONBLOCK); + if (ret < 0) + goto out; + snprintf(sun.sun_path, sizeof(sun.sun_path), "/var/tmp/%s_%d", + MLX5_DRIVER_NAME, priv->primary_socket); + ret = stat(sun.sun_path, &file_stat); + if (!ret) + claim_zero(remove(sun.sun_path)); + ret = bind(priv->primary_socket, (const struct sockaddr *)&sun, + sizeof(sun)); + if (ret < 0) { + WARN("cannot bind socket, secondary process not supported: %s", + strerror(errno)); + goto close; + } + ret = listen(priv->primary_socket, 0); + if (ret < 0) { + WARN("Secondary process not supported: %s", strerror(errno)); + goto close; + } + return ret; +close: + remove(sun.sun_path); +out: + claim_zero(close(priv->primary_socket)); + priv->primary_socket = 0; + return -(ret); +} + +/** + * Un-Initialise the socket to communicate with the secondary process + * + * @param[in] priv + * Pointer to private structure. + * + * @return + * 0 on success, errno value on failure. + */ +int +priv_socket_uninit(struct priv *priv) +{ + MKSTR(path, "/var/tmp/%s_%d", MLX5_DRIVER_NAME, priv->primary_socket); + claim_zero(close(priv->primary_socket)); + priv->primary_socket = 0; + claim_zero(remove(path)); + return 0; +} + +/** + * Handle socket interrupts. + * + * @param priv + * Pointer to private structure. + */ +void +priv_socket_handle(struct priv *priv) +{ + int conn_sock; + int ret = 0; + struct cmsghdr *cmsg = NULL; + struct ucred *cred = NULL; + char buf[CMSG_SPACE(sizeof(struct ucred))] = { 0 }; + char vbuf[1024] = { 0 }; + struct iovec io = { + .iov_base = vbuf, + .iov_len = sizeof(*vbuf), + }; + struct msghdr msg = { + .msg_iov = &io, + .msg_iovlen = 1, + .msg_control = buf, + .msg_controllen = sizeof(buf), + }; + int *fd; + + /* Accept the connection from the client. */ + conn_sock = accept(priv->primary_socket, NULL, NULL); + if (conn_sock < 0) { + WARN("connection failed: %s", strerror(errno)); + return; + } + ret = setsockopt(conn_sock, SOL_SOCKET, SO_PASSCRED, &(int){1}, + sizeof(int)); + if (ret < 0) { + WARN("cannot change socket options"); + goto out; + } + ret = recvmsg(conn_sock, &msg, MSG_WAITALL); + if (ret < 0) { + WARN("received an empty message: %s", strerror(errno)); + goto out; + } + /* Expect to receive credentials only. */ + cmsg = CMSG_FIRSTHDR(&msg); + if (cmsg == NULL) { + WARN("no message"); + goto out; + } + if ((cmsg->cmsg_type == SCM_CREDENTIALS) && + (cmsg->cmsg_len >= sizeof(*cred))) { + cred = (struct ucred *)CMSG_DATA(cmsg); + assert(cred != NULL); + } + cmsg = CMSG_NXTHDR(&msg, cmsg); + if (cmsg != NULL) { + WARN("Message wrongly formatted"); + goto out; + } + /* Make sure all the ancillary data was received and valid. */ + if ((cred == NULL) || (cred->uid != getuid()) || + (cred->gid != getgid())) { + WARN("wrong credentials"); + goto out; + } + /* Set-up the ancillary data. */ + cmsg = CMSG_FIRSTHDR(&msg); + assert(cmsg != NULL); + cmsg->cmsg_level = SOL_SOCKET; + cmsg->cmsg_type = SCM_RIGHTS; + cmsg->cmsg_len = CMSG_LEN(sizeof(priv->ctx->cmd_fd)); + fd = (int *)CMSG_DATA(cmsg); + *fd = priv->ctx->cmd_fd; + ret = sendmsg(conn_sock, &msg, 0); + if (ret < 0) + WARN("cannot send response"); +out: + close(conn_sock); +} + +/** + * Connect to the primary process. + * + * @param[in] priv + * Pointer to private structure. + * + * @return + * fd on success, negative errno value on failure. + */ +int +priv_socket_connect(struct priv *priv) +{ + struct sockaddr_un sun = { + .sun_family = AF_UNIX, + }; + int socket_fd; + int *fd = NULL; + int ret; + struct ucred *cred; + char buf[CMSG_SPACE(sizeof(*cred))] = { 0 }; + char vbuf[1024] = { 0 }; + struct iovec io = { + .iov_base = vbuf, + .iov_len = sizeof(*vbuf), + }; + struct msghdr msg = { + .msg_control = buf, + .msg_controllen = sizeof(buf), + .msg_iov = &io, + .msg_iovlen = 1, + }; + struct cmsghdr *cmsg; + + ret = socket(AF_UNIX, SOCK_STREAM, 0); + if (ret < 0) { + WARN("cannot connect to primary"); + return ret; + } + socket_fd = ret; + snprintf(sun.sun_path, sizeof(sun.sun_path), "/var/tmp/%s_%d", + MLX5_DRIVER_NAME, priv->primary_socket); + ret = connect(socket_fd, (const struct sockaddr *)&sun, sizeof(sun)); + if (ret < 0) { + WARN("cannot connect to primary"); + goto out; + } + cmsg = CMSG_FIRSTHDR(&msg); + if (cmsg == NULL) { + DEBUG("cannot get first message"); + goto out; + } + cmsg->cmsg_level = SOL_SOCKET; + cmsg->cmsg_type = SCM_CREDENTIALS; + cmsg->cmsg_len = CMSG_LEN(sizeof(*cred)); + cred = (struct ucred *)CMSG_DATA(cmsg); + if (cred == NULL) { + DEBUG("no credentials received"); + goto out; + } + cred->pid = getpid(); + cred->uid = getuid(); + cred->gid = getgid(); + ret = sendmsg(socket_fd, &msg, MSG_DONTWAIT); + if (ret < 0) { + WARN("cannot send credentials to primary: %s", + strerror(errno)); + goto out; + } + ret = recvmsg(socket_fd, &msg, MSG_WAITALL); + if (ret <= 0) { + WARN("no message from primary: %s", strerror(errno)); + goto out; + } + cmsg = CMSG_FIRSTHDR(&msg); + if (cmsg == NULL) { + WARN("No file descriptor received"); + goto out; + } + fd = (int *)CMSG_DATA(cmsg); + if (*fd <= 0) { + WARN("no file descriptor received: %s", strerror(errno)); + ret = *fd; + goto out; + } + ret = *fd; +out: + close(socket_fd); + return ret; +} diff --git a/dpdk/drivers/net/mlx5/mlx5_stats.c b/dpdk/drivers/net/mlx5/mlx5_stats.c index f2b5781a..2427585f 100644 --- a/dpdk/drivers/net/mlx5/mlx5_stats.c +++ b/dpdk/drivers/net/mlx5/mlx5_stats.c @@ -31,19 +31,283 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -/* DPDK headers don't like -pedantic. */ -#ifdef PEDANTIC -#pragma GCC diagnostic ignored "-Wpedantic" -#endif +#include +#include + #include -#ifdef PEDANTIC -#pragma GCC diagnostic error "-Wpedantic" -#endif +#include +#include #include "mlx5.h" #include "mlx5_rxtx.h" #include "mlx5_defs.h" +struct mlx5_counter_ctrl { + /* Name of the counter. */ + char dpdk_name[RTE_ETH_XSTATS_NAME_SIZE]; + /* Name of the counter on the device table. */ + char ctr_name[RTE_ETH_XSTATS_NAME_SIZE]; +}; + +static const struct mlx5_counter_ctrl mlx5_counters_init[] = { + { + .dpdk_name = "rx_port_unicast_bytes", + .ctr_name = "rx_vport_unicast_bytes", + }, + { + .dpdk_name = "rx_port_multicast_bytes", + .ctr_name = "rx_vport_multicast_bytes", + }, + { + .dpdk_name = "rx_port_broadcast_bytes", + .ctr_name = "rx_vport_broadcast_bytes", + }, + { + .dpdk_name = "rx_port_unicast_packets", + .ctr_name = "rx_vport_unicast_packets", + }, + { + .dpdk_name = "rx_port_multicast_packets", + .ctr_name = "rx_vport_multicast_packets", + }, + { + .dpdk_name = "rx_port_broadcast_packets", + .ctr_name = "rx_vport_broadcast_packets", + }, + { + .dpdk_name = "tx_port_unicast_bytes", + .ctr_name = "tx_vport_unicast_bytes", + }, + { + .dpdk_name = "tx_port_multicast_bytes", + .ctr_name = "tx_vport_multicast_bytes", + }, + { + .dpdk_name = "tx_port_broadcast_bytes", + .ctr_name = "tx_vport_broadcast_bytes", + }, + { + .dpdk_name = "tx_port_unicast_packets", + .ctr_name = "tx_vport_unicast_packets", + }, + { + .dpdk_name = "tx_port_multicast_packets", + .ctr_name = "tx_vport_multicast_packets", + }, + { + .dpdk_name = "tx_port_broadcast_packets", + .ctr_name = "tx_vport_broadcast_packets", + }, + { + .dpdk_name = "rx_wqe_err", + .ctr_name = "rx_wqe_err", + }, + { + .dpdk_name = "rx_crc_errors_phy", + .ctr_name = "rx_crc_errors_phy", + }, + { + .dpdk_name = "rx_in_range_len_errors_phy", + .ctr_name = "rx_in_range_len_errors_phy", + }, + { + .dpdk_name = "rx_symbol_err_phy", + .ctr_name = "rx_symbol_err_phy", + }, + { + .dpdk_name = "tx_errors_phy", + .ctr_name = "tx_errors_phy", + }, + { + .dpdk_name = "rx_out_of_buffer", + .ctr_name = "out_of_buffer", + }, +}; + +static const unsigned int xstats_n = RTE_DIM(mlx5_counters_init); + +/** + * Read device counters table. + * + * @param priv + * Pointer to private structure. + * @param[out] stats + * Counters table output buffer. + * + * @return + * 0 on success and stats is filled, negative on error. + */ +static int +priv_read_dev_counters(struct priv *priv, uint64_t *stats) +{ + struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl; + unsigned int i; + struct ifreq ifr; + unsigned int stats_sz = xstats_ctrl->stats_n * sizeof(uint64_t); + unsigned char et_stat_buf[sizeof(struct ethtool_stats) + stats_sz]; + struct ethtool_stats *et_stats = (struct ethtool_stats *)et_stat_buf; + + et_stats->cmd = ETHTOOL_GSTATS; + et_stats->n_stats = xstats_ctrl->stats_n; + ifr.ifr_data = (caddr_t)et_stats; + if (priv_ifreq(priv, SIOCETHTOOL, &ifr) != 0) { + WARN("unable to read statistic values from device"); + return -1; + } + for (i = 0; i != xstats_n; ++i) { + if (priv_is_ib_cntr(mlx5_counters_init[i].ctr_name)) + priv_get_cntr_sysfs(priv, + mlx5_counters_init[i].ctr_name, + &stats[i]); + else + stats[i] = (uint64_t) + et_stats->data[xstats_ctrl->dev_table_idx[i]]; + } + return 0; +} + +/** + * Query the number of statistics provided by ETHTOOL. + * + * @param priv + * Pointer to private structure. + * + * @return + * Number of statistics on success, -1 on error. + */ +static int +priv_ethtool_get_stats_n(struct priv *priv) { + struct ethtool_drvinfo drvinfo; + struct ifreq ifr; + + drvinfo.cmd = ETHTOOL_GDRVINFO; + ifr.ifr_data = (caddr_t)&drvinfo; + if (priv_ifreq(priv, SIOCETHTOOL, &ifr) != 0) { + WARN("unable to query number of statistics"); + return -1; + } + return drvinfo.n_stats; +} + +/** + * Init the structures to read device counters. + * + * @param priv + * Pointer to private structure. + */ +void +priv_xstats_init(struct priv *priv) +{ + struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl; + unsigned int i; + unsigned int j; + struct ifreq ifr; + struct ethtool_gstrings *strings = NULL; + unsigned int dev_stats_n; + unsigned int str_sz; + + dev_stats_n = priv_ethtool_get_stats_n(priv); + if (dev_stats_n < 1) { + WARN("no extended statistics available"); + return; + } + xstats_ctrl->stats_n = dev_stats_n; + /* Allocate memory to grab stat names and values. */ + str_sz = dev_stats_n * ETH_GSTRING_LEN; + strings = (struct ethtool_gstrings *) + rte_malloc("xstats_strings", + str_sz + sizeof(struct ethtool_gstrings), 0); + if (!strings) { + WARN("unable to allocate memory for xstats"); + return; + } + strings->cmd = ETHTOOL_GSTRINGS; + strings->string_set = ETH_SS_STATS; + strings->len = dev_stats_n; + ifr.ifr_data = (caddr_t)strings; + if (priv_ifreq(priv, SIOCETHTOOL, &ifr) != 0) { + WARN("unable to get statistic names"); + goto free; + } + for (j = 0; j != xstats_n; ++j) + xstats_ctrl->dev_table_idx[j] = dev_stats_n; + for (i = 0; i != dev_stats_n; ++i) { + const char *curr_string = (const char *) + &strings->data[i * ETH_GSTRING_LEN]; + + for (j = 0; j != xstats_n; ++j) { + if (!strcmp(mlx5_counters_init[j].ctr_name, + curr_string)) { + xstats_ctrl->dev_table_idx[j] = i; + break; + } + } + } + for (j = 0; j != xstats_n; ++j) { + if (priv_is_ib_cntr(mlx5_counters_init[j].ctr_name)) + continue; + if (xstats_ctrl->dev_table_idx[j] >= dev_stats_n) { + WARN("counter \"%s\" is not recognized", + mlx5_counters_init[j].dpdk_name); + goto free; + } + } + /* Copy to base at first time. */ + assert(xstats_n <= MLX5_MAX_XSTATS); + priv_read_dev_counters(priv, xstats_ctrl->base); +free: + rte_free(strings); +} + +/** + * Get device extended statistics. + * + * @param priv + * Pointer to private structure. + * @param[out] stats + * Pointer to rte extended stats table. + * + * @return + * Number of extended stats on success and stats is filled, + * negative on error. + */ +static int +priv_xstats_get(struct priv *priv, struct rte_eth_xstat *stats) +{ + struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl; + unsigned int i; + unsigned int n = xstats_n; + uint64_t counters[n]; + + if (priv_read_dev_counters(priv, counters) < 0) + return -1; + for (i = 0; i != xstats_n; ++i) { + stats[i].id = i; + stats[i].value = (counters[i] - xstats_ctrl->base[i]); + } + return n; +} + +/** + * Reset device extended statistics. + * + * @param priv + * Pointer to private structure. + */ +static void +priv_xstats_reset(struct priv *priv) +{ + struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl; + unsigned int i; + unsigned int n = xstats_n; + uint64_t counters[n]; + + if (priv_read_dev_counters(priv, counters) < 0) + return; + for (i = 0; i != n; ++i) + xstats_ctrl->base[i] = counters[i]; +} + /** * DPDK callback to get device statistics. * @@ -52,7 +316,7 @@ * @param[out] stats * Stats structure output buffer. */ -void +int mlx5_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) { struct priv *priv = mlx5_get_priv(dev); @@ -63,7 +327,7 @@ mlx5_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) priv_lock(priv); /* Add software counters. */ for (i = 0; (i != priv->rxqs_n); ++i) { - struct rxq *rxq = (*priv->rxqs)[i]; + struct mlx5_rxq_data *rxq = (*priv->rxqs)[i]; if (rxq == NULL) continue; @@ -84,7 +348,7 @@ mlx5_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) tmp.rx_nombuf += rxq->stats.rx_nombuf; } for (i = 0; (i != priv->txqs_n); ++i) { - struct txq *txq = (*priv->txqs)[i]; + struct mlx5_txq_data *txq = (*priv->txqs)[i]; if (txq == NULL) continue; @@ -94,19 +358,20 @@ mlx5_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) tmp.q_opackets[idx] += txq->stats.opackets; tmp.q_obytes[idx] += txq->stats.obytes; #endif - tmp.q_errors[idx] += txq->stats.odropped; + tmp.q_errors[idx] += txq->stats.oerrors; } #ifdef MLX5_PMD_SOFT_COUNTERS tmp.opackets += txq->stats.opackets; tmp.obytes += txq->stats.obytes; #endif - tmp.oerrors += txq->stats.odropped; + tmp.oerrors += txq->stats.oerrors; } #ifndef MLX5_PMD_SOFT_COUNTERS /* FIXME: retrieve and add hardware counters. */ #endif *stats = tmp; priv_unlock(priv); + return 0; } /** @@ -142,3 +407,98 @@ mlx5_stats_reset(struct rte_eth_dev *dev) #endif priv_unlock(priv); } + +/** + * DPDK callback to get extended device statistics. + * + * @param dev + * Pointer to Ethernet device structure. + * @param[out] stats + * Stats table output buffer. + * @param n + * The size of the stats table. + * + * @return + * Number of xstats on success, negative on failure. + */ +int +mlx5_xstats_get(struct rte_eth_dev *dev, + struct rte_eth_xstat *stats, unsigned int n) +{ + struct priv *priv = mlx5_get_priv(dev); + int ret = xstats_n; + + if (n >= xstats_n && stats) { + struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl; + int stats_n; + + priv_lock(priv); + stats_n = priv_ethtool_get_stats_n(priv); + if (stats_n < 0) { + priv_unlock(priv); + return -1; + } + if (xstats_ctrl->stats_n != stats_n) + priv_xstats_init(priv); + ret = priv_xstats_get(priv, stats); + priv_unlock(priv); + } + return ret; +} + +/** + * DPDK callback to clear device extended statistics. + * + * @param dev + * Pointer to Ethernet device structure. + */ +void +mlx5_xstats_reset(struct rte_eth_dev *dev) +{ + struct priv *priv = mlx5_get_priv(dev); + struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl; + int stats_n; + + priv_lock(priv); + stats_n = priv_ethtool_get_stats_n(priv); + if (stats_n < 0) + goto unlock; + if (xstats_ctrl->stats_n != stats_n) + priv_xstats_init(priv); + priv_xstats_reset(priv); +unlock: + priv_unlock(priv); +} + +/** + * DPDK callback to retrieve names of extended device statistics + * + * @param dev + * Pointer to Ethernet device structure. + * @param[out] xstats_names + * Buffer to insert names into. + * @param n + * Number of names. + * + * @return + * Number of xstats names. + */ +int +mlx5_xstats_get_names(struct rte_eth_dev *dev, + struct rte_eth_xstat_name *xstats_names, unsigned int n) +{ + struct priv *priv = mlx5_get_priv(dev); + unsigned int i; + + if (n >= xstats_n && xstats_names) { + priv_lock(priv); + for (i = 0; i != xstats_n; ++i) { + strncpy(xstats_names[i].name, + mlx5_counters_init[i].dpdk_name, + RTE_ETH_XSTATS_NAME_SIZE); + xstats_names[i].name[RTE_ETH_XSTATS_NAME_SIZE - 1] = 0; + } + priv_unlock(priv); + } + return xstats_n; +} diff --git a/dpdk/drivers/net/mlx5/mlx5_trigger.c b/dpdk/drivers/net/mlx5/mlx5_trigger.c index d4dccd88..d682ea2c 100644 --- a/dpdk/drivers/net/mlx5/mlx5_trigger.c +++ b/dpdk/drivers/net/mlx5/mlx5_trigger.c @@ -30,23 +30,93 @@ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +#include -/* DPDK headers don't like -pedantic. */ -#ifdef PEDANTIC -#pragma GCC diagnostic ignored "-Wpedantic" -#endif #include #include #include #include -#ifdef PEDANTIC -#pragma GCC diagnostic error "-Wpedantic" -#endif #include "mlx5.h" #include "mlx5_rxtx.h" #include "mlx5_utils.h" +static void +priv_txq_stop(struct priv *priv) +{ + unsigned int i; + + for (i = 0; i != priv->txqs_n; ++i) + mlx5_priv_txq_release(priv, i); +} + +static int +priv_txq_start(struct priv *priv) +{ + unsigned int i; + int ret = 0; + + /* Add memory regions to Tx queues. */ + for (i = 0; i != priv->txqs_n; ++i) { + unsigned int idx = 0; + struct mlx5_mr *mr; + struct mlx5_txq_ctrl *txq_ctrl = mlx5_priv_txq_get(priv, i); + + if (!txq_ctrl) + continue; + LIST_FOREACH(mr, &priv->mr, next) { + priv_txq_mp2mr_reg(priv, &txq_ctrl->txq, mr->mp, idx++); + if (idx == MLX5_PMD_TX_MP_CACHE) + break; + } + txq_alloc_elts(txq_ctrl); + txq_ctrl->ibv = mlx5_priv_txq_ibv_new(priv, i); + if (!txq_ctrl->ibv) { + ret = ENOMEM; + goto error; + } + } + return -ret; +error: + priv_txq_stop(priv); + return -ret; +} + +static void +priv_rxq_stop(struct priv *priv) +{ + unsigned int i; + + for (i = 0; i != priv->rxqs_n; ++i) + mlx5_priv_rxq_release(priv, i); +} + +static int +priv_rxq_start(struct priv *priv) +{ + unsigned int i; + int ret = 0; + + for (i = 0; i != priv->rxqs_n; ++i) { + struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_priv_rxq_get(priv, i); + + if (!rxq_ctrl) + continue; + ret = rxq_alloc_elts(rxq_ctrl); + if (ret) + goto error; + rxq_ctrl->ibv = mlx5_priv_rxq_ibv_new(priv, i); + if (!rxq_ctrl->ibv) { + ret = ENOMEM; + goto error; + } + } + return -ret; +error: + priv_rxq_stop(priv); + return -ret; +} + /** * DPDK callback to start the device. * @@ -62,36 +132,62 @@ int mlx5_dev_start(struct rte_eth_dev *dev) { struct priv *priv = dev->data->dev_private; + struct mlx5_mr *mr = NULL; int err; - if (mlx5_is_secondary()) - return -E_RTE_SECONDARY; - + dev->data->dev_started = 1; priv_lock(priv); - if (priv->started) { - priv_unlock(priv); - return 0; + err = priv_flow_create_drop_queue(priv); + if (err) { + ERROR("%p: Drop queue allocation failed: %s", + (void *)dev, strerror(err)); + goto error; } DEBUG("%p: allocating and configuring hash RX queues", (void *)dev); - err = priv_create_hash_rxqs(priv); - if (!err) - err = priv_rehash_flows(priv); - if (!err) - priv->started = 1; - else { - ERROR("%p: an error occurred while configuring hash RX queues:" - " %s", - (void *)priv, strerror(err)); - /* Rollback. */ - priv_special_flow_disable_all(priv); - priv_mac_addrs_disable(priv); - priv_destroy_hash_rxqs(priv); + rte_mempool_walk(mlx5_mp2mr_iter, priv); + err = priv_txq_start(priv); + if (err) { + ERROR("%p: TXQ allocation failed: %s", + (void *)dev, strerror(err)); + goto error; + } + err = priv_rxq_start(priv); + if (err) { + ERROR("%p: RXQ allocation failed: %s", + (void *)dev, strerror(err)); + goto error; + } + err = priv_rx_intr_vec_enable(priv); + if (err) { + ERROR("%p: RX interrupt vector creation failed", + (void *)priv); + goto error; + } + priv_xstats_init(priv); + /* Update link status and Tx/Rx callbacks for the first time. */ + memset(&dev->data->dev_link, 0, sizeof(struct rte_eth_link)); + INFO("Forcing port %u link to be up", dev->data->port_id); + err = priv_force_link_status_change(priv, ETH_LINK_UP); + if (err) { + DEBUG("Failed to set port %u link to be up", + dev->data->port_id); + goto error; } - if (dev->data->dev_conf.fdir_conf.mode != RTE_FDIR_MODE_NONE) - priv_fdir_enable(priv); priv_dev_interrupt_handler_install(priv, dev); priv_unlock(priv); - return -err; + return 0; +error: + /* Rollback. */ + dev->data->dev_started = 0; + for (mr = LIST_FIRST(&priv->mr); mr; mr = LIST_FIRST(&priv->mr)) + priv_mr_release(priv, mr); + priv_flow_stop(priv, &priv->flows); + priv_dev_traffic_disable(priv, dev); + priv_txq_stop(priv); + priv_rxq_stop(priv); + priv_flow_delete_drop_queue(priv); + priv_unlock(priv); + return err; } /** @@ -106,21 +202,212 @@ void mlx5_dev_stop(struct rte_eth_dev *dev) { struct priv *priv = dev->data->dev_private; - - if (mlx5_is_secondary()) - return; + struct mlx5_mr *mr; priv_lock(priv); - if (!priv->started) { - priv_unlock(priv); - return; - } + dev->data->dev_started = 0; + /* Prevent crashes when queues are still in use. */ + dev->rx_pkt_burst = removed_rx_burst; + dev->tx_pkt_burst = removed_tx_burst; + rte_wmb(); + usleep(1000 * priv->rxqs_n); DEBUG("%p: cleaning up and destroying hash RX queues", (void *)dev); - priv_special_flow_disable_all(priv); - priv_mac_addrs_disable(priv); - priv_destroy_hash_rxqs(priv); - priv_fdir_disable(priv); + priv_flow_stop(priv, &priv->flows); + priv_dev_traffic_disable(priv, dev); + priv_rx_intr_vec_disable(priv); priv_dev_interrupt_handler_uninstall(priv, dev); - priv->started = 0; + priv_txq_stop(priv); + priv_rxq_stop(priv); + for (mr = LIST_FIRST(&priv->mr); mr; mr = LIST_FIRST(&priv->mr)) + priv_mr_release(priv, mr); + priv_flow_delete_drop_queue(priv); priv_unlock(priv); } + +/** + * Enable traffic flows configured by control plane + * + * @param priv + * Pointer to Ethernet device private data. + * @param dev + * Pointer to Ethernet device structure. + * + * @return + * 0 on success. + */ +int +priv_dev_traffic_enable(struct priv *priv, struct rte_eth_dev *dev) +{ + struct rte_flow_item_eth bcast = { + .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", + }; + struct rte_flow_item_eth ipv6_multi_spec = { + .dst.addr_bytes = "\x33\x33\x00\x00\x00\x00", + }; + struct rte_flow_item_eth ipv6_multi_mask = { + .dst.addr_bytes = "\xff\xff\x00\x00\x00\x00", + }; + struct rte_flow_item_eth unicast = { + .src.addr_bytes = "\x00\x00\x00\x00\x00\x00", + }; + struct rte_flow_item_eth unicast_mask = { + .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", + }; + const unsigned int vlan_filter_n = priv->vlan_filter_n; + const struct ether_addr cmp = { + .addr_bytes = "\x00\x00\x00\x00\x00\x00", + }; + unsigned int i; + unsigned int j; + int ret; + + if (priv->isolated) + return 0; + if (dev->data->promiscuous) { + struct rte_flow_item_eth promisc = { + .dst.addr_bytes = "\x00\x00\x00\x00\x00\x00", + .src.addr_bytes = "\x00\x00\x00\x00\x00\x00", + .type = 0, + }; + + claim_zero(mlx5_ctrl_flow(dev, &promisc, &promisc)); + return 0; + } + if (dev->data->all_multicast) { + struct rte_flow_item_eth multicast = { + .dst.addr_bytes = "\x01\x00\x00\x00\x00\x00", + .src.addr_bytes = "\x00\x00\x00\x00\x00\x00", + .type = 0, + }; + + claim_zero(mlx5_ctrl_flow(dev, &multicast, &multicast)); + } else { + /* Add broadcast/multicast flows. */ + for (i = 0; i != vlan_filter_n; ++i) { + uint16_t vlan = priv->vlan_filter[i]; + + struct rte_flow_item_vlan vlan_spec = { + .tci = rte_cpu_to_be_16(vlan), + }; + struct rte_flow_item_vlan vlan_mask = { + .tci = 0xffff, + }; + + ret = mlx5_ctrl_flow_vlan(dev, &bcast, &bcast, + &vlan_spec, &vlan_mask); + if (ret) + goto error; + ret = mlx5_ctrl_flow_vlan(dev, &ipv6_multi_spec, + &ipv6_multi_mask, + &vlan_spec, &vlan_mask); + if (ret) + goto error; + } + if (!vlan_filter_n) { + ret = mlx5_ctrl_flow(dev, &bcast, &bcast); + if (ret) + goto error; + ret = mlx5_ctrl_flow(dev, &ipv6_multi_spec, + &ipv6_multi_mask); + if (ret) + goto error; + } + } + /* Add MAC address flows. */ + for (i = 0; i != MLX5_MAX_MAC_ADDRESSES; ++i) { + struct ether_addr *mac = &dev->data->mac_addrs[i]; + + if (!memcmp(mac, &cmp, sizeof(*mac))) + continue; + memcpy(&unicast.dst.addr_bytes, + mac->addr_bytes, + ETHER_ADDR_LEN); + for (j = 0; j != vlan_filter_n; ++j) { + uint16_t vlan = priv->vlan_filter[j]; + + struct rte_flow_item_vlan vlan_spec = { + .tci = rte_cpu_to_be_16(vlan), + }; + struct rte_flow_item_vlan vlan_mask = { + .tci = 0xffff, + }; + + ret = mlx5_ctrl_flow_vlan(dev, &unicast, + &unicast_mask, + &vlan_spec, + &vlan_mask); + if (ret) + goto error; + } + if (!vlan_filter_n) { + ret = mlx5_ctrl_flow(dev, &unicast, + &unicast_mask); + if (ret) + goto error; + } + } + return 0; +error: + return rte_errno; +} + + +/** + * Disable traffic flows configured by control plane + * + * @param priv + * Pointer to Ethernet device private data. + * @param dev + * Pointer to Ethernet device structure. + * + * @return + * 0 on success. + */ +int +priv_dev_traffic_disable(struct priv *priv, struct rte_eth_dev *dev) +{ + (void)dev; + priv_flow_flush(priv, &priv->ctrl_flows); + return 0; +} + +/** + * Restart traffic flows configured by control plane + * + * @param priv + * Pointer to Ethernet device private data. + * @param dev + * Pointer to Ethernet device structure. + * + * @return + * 0 on success. + */ +int +priv_dev_traffic_restart(struct priv *priv, struct rte_eth_dev *dev) +{ + if (dev->data->dev_started) { + priv_dev_traffic_disable(priv, dev); + priv_dev_traffic_enable(priv, dev); + } + return 0; +} + +/** + * Restart traffic flows configured by control plane + * + * @param dev + * Pointer to Ethernet device structure. + * + * @return + * 0 on success. + */ +int +mlx5_traffic_restart(struct rte_eth_dev *dev) +{ + struct priv *priv = dev->data->dev_private; + + priv_lock(priv); + priv_dev_traffic_restart(priv, dev); + priv_unlock(priv); + return 0; +} diff --git a/dpdk/drivers/net/mlx5/mlx5_txq.c b/dpdk/drivers/net/mlx5/mlx5_txq.c index e4510efe..7ca99f5a 100644 --- a/dpdk/drivers/net/mlx5/mlx5_txq.c +++ b/dpdk/drivers/net/mlx5/mlx5_txq.c @@ -36,6 +36,8 @@ #include #include #include +#include +#include /* Verbs header. */ /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ @@ -47,45 +49,31 @@ #pragma GCC diagnostic error "-Wpedantic" #endif -/* DPDK headers don't like -pedantic. */ -#ifdef PEDANTIC -#pragma GCC diagnostic ignored "-Wpedantic" -#endif #include #include #include #include -#ifdef PEDANTIC -#pragma GCC diagnostic error "-Wpedantic" -#endif #include "mlx5_utils.h" #include "mlx5_defs.h" #include "mlx5.h" #include "mlx5_rxtx.h" #include "mlx5_autoconf.h" -#include "mlx5_defs.h" /** * Allocate TX queue elements. * * @param txq_ctrl * Pointer to TX queue structure. - * @param elts_n - * Number of elements to allocate. */ -static void -txq_alloc_elts(struct txq_ctrl *txq_ctrl, unsigned int elts_n) +void +txq_alloc_elts(struct mlx5_txq_ctrl *txq_ctrl) { + const unsigned int elts_n = 1 << txq_ctrl->txq.elts_n; unsigned int i; for (i = 0; (i != elts_n); ++i) (*txq_ctrl->txq.elts)[i] = NULL; - for (i = 0; (i != txq_ctrl->txq.wqe_n); ++i) { - volatile union mlx5_wqe *wqe = &(*txq_ctrl->txq.wqes)[i]; - - memset((void *)(uintptr_t)wqe, 0x0, sizeof(*wqe)); - } DEBUG("%p: allocated and configured %u WRs", (void *)txq_ctrl, elts_n); txq_ctrl->txq.elts_head = 0; txq_ctrl->txq.elts_tail = 0; @@ -99,11 +87,12 @@ txq_alloc_elts(struct txq_ctrl *txq_ctrl, unsigned int elts_n) * Pointer to TX queue structure. */ static void -txq_free_elts(struct txq_ctrl *txq_ctrl) +txq_free_elts(struct mlx5_txq_ctrl *txq_ctrl) { - unsigned int elts_n = txq_ctrl->txq.elts_n; - unsigned int elts_head = txq_ctrl->txq.elts_head; - unsigned int elts_tail = txq_ctrl->txq.elts_tail; + const uint16_t elts_n = 1 << txq_ctrl->txq.elts_n; + const uint16_t elts_m = elts_n - 1; + uint16_t elts_head = txq_ctrl->txq.elts_head; + uint16_t elts_tail = txq_ctrl->txq.elts_tail; struct rte_mbuf *(*elts)[elts_n] = txq_ctrl->txq.elts; DEBUG("%p: freeing WRs", (void *)txq_ctrl); @@ -112,332 +101,20 @@ txq_free_elts(struct txq_ctrl *txq_ctrl) txq_ctrl->txq.elts_comp = 0; while (elts_tail != elts_head) { - struct rte_mbuf *elt = (*elts)[elts_tail]; + struct rte_mbuf *elt = (*elts)[elts_tail & elts_m]; assert(elt != NULL); - rte_pktmbuf_free(elt); + rte_pktmbuf_free_seg(elt); #ifndef NDEBUG /* Poisoning. */ - memset(&(*elts)[elts_tail], + memset(&(*elts)[elts_tail & elts_m], 0x77, - sizeof((*elts)[elts_tail])); + sizeof((*elts)[elts_tail & elts_m])); #endif - if (++elts_tail == elts_n) - elts_tail = 0; + ++elts_tail; } } -/** - * Clean up a TX queue. - * - * Destroy objects, free allocated memory and reset the structure for reuse. - * - * @param txq_ctrl - * Pointer to TX queue structure. - */ -void -txq_cleanup(struct txq_ctrl *txq_ctrl) -{ - struct ibv_exp_release_intf_params params; - size_t i; - - DEBUG("cleaning up %p", (void *)txq_ctrl); - txq_free_elts(txq_ctrl); - if (txq_ctrl->if_qp != NULL) { - assert(txq_ctrl->priv != NULL); - assert(txq_ctrl->priv->ctx != NULL); - assert(txq_ctrl->qp != NULL); - params = (struct ibv_exp_release_intf_params){ - .comp_mask = 0, - }; - claim_zero(ibv_exp_release_intf(txq_ctrl->priv->ctx, - txq_ctrl->if_qp, - ¶ms)); - } - if (txq_ctrl->if_cq != NULL) { - assert(txq_ctrl->priv != NULL); - assert(txq_ctrl->priv->ctx != NULL); - assert(txq_ctrl->cq != NULL); - params = (struct ibv_exp_release_intf_params){ - .comp_mask = 0, - }; - claim_zero(ibv_exp_release_intf(txq_ctrl->priv->ctx, - txq_ctrl->if_cq, - ¶ms)); - } - if (txq_ctrl->qp != NULL) - claim_zero(ibv_destroy_qp(txq_ctrl->qp)); - if (txq_ctrl->cq != NULL) - claim_zero(ibv_destroy_cq(txq_ctrl->cq)); - if (txq_ctrl->rd != NULL) { - struct ibv_exp_destroy_res_domain_attr attr = { - .comp_mask = 0, - }; - - assert(txq_ctrl->priv != NULL); - assert(txq_ctrl->priv->ctx != NULL); - claim_zero(ibv_exp_destroy_res_domain(txq_ctrl->priv->ctx, - txq_ctrl->rd, - &attr)); - } - for (i = 0; (i != RTE_DIM(txq_ctrl->txq.mp2mr)); ++i) { - if (txq_ctrl->txq.mp2mr[i].mp == NULL) - break; - assert(txq_ctrl->txq.mp2mr[i].mr != NULL); - claim_zero(ibv_dereg_mr(txq_ctrl->txq.mp2mr[i].mr)); - } - memset(txq_ctrl, 0, sizeof(*txq_ctrl)); -} - -/** - * Initialize TX queue. - * - * @param tmpl - * Pointer to TX queue control template. - * @param txq_ctrl - * Pointer to TX queue control. - * - * @return - * 0 on success, errno value on failure. - */ -static inline int -txq_setup(struct txq_ctrl *tmpl, struct txq_ctrl *txq_ctrl) -{ - struct mlx5_qp *qp = to_mqp(tmpl->qp); - struct ibv_cq *ibcq = tmpl->cq; - struct mlx5_cq *cq = to_mxxx(cq, cq); - - if (cq->cqe_sz != RTE_CACHE_LINE_SIZE) { - ERROR("Wrong MLX5_CQE_SIZE environment variable value: " - "it should be set to %u", RTE_CACHE_LINE_SIZE); - return EINVAL; - } - tmpl->txq.cqe_n = ibcq->cqe + 1; - tmpl->txq.qp_num_8s = qp->ctrl_seg.qp_num << 8; - tmpl->txq.wqes = - (volatile union mlx5_wqe (*)[]) - (uintptr_t)qp->gen_data.sqstart; - tmpl->txq.wqe_n = qp->sq.wqe_cnt; - tmpl->txq.qp_db = &qp->gen_data.db[MLX5_SND_DBR]; - tmpl->txq.bf_reg = qp->gen_data.bf->reg; - tmpl->txq.bf_offset = qp->gen_data.bf->offset; - tmpl->txq.bf_buf_size = qp->gen_data.bf->buf_size; - tmpl->txq.cq_db = cq->dbrec; - tmpl->txq.cqes = - (volatile struct mlx5_cqe (*)[]) - (uintptr_t)cq->active_buf->buf; - tmpl->txq.elts = - (struct rte_mbuf *(*)[tmpl->txq.elts_n]) - ((uintptr_t)txq_ctrl + sizeof(*txq_ctrl)); - return 0; -} - -/** - * Configure a TX queue. - * - * @param dev - * Pointer to Ethernet device structure. - * @param txq_ctrl - * Pointer to TX queue structure. - * @param desc - * Number of descriptors to configure in queue. - * @param socket - * NUMA socket on which memory must be allocated. - * @param[in] conf - * Thresholds parameters. - * - * @return - * 0 on success, errno value on failure. - */ -int -txq_ctrl_setup(struct rte_eth_dev *dev, struct txq_ctrl *txq_ctrl, - uint16_t desc, unsigned int socket, - const struct rte_eth_txconf *conf) -{ - struct priv *priv = mlx5_get_priv(dev); - struct txq_ctrl tmpl = { - .priv = priv, - .socket = socket, - }; - union { - struct ibv_exp_query_intf_params params; - struct ibv_exp_qp_init_attr init; - struct ibv_exp_res_domain_init_attr rd; - struct ibv_exp_cq_init_attr cq; - struct ibv_exp_qp_attr mod; - struct ibv_exp_cq_attr cq_attr; - } attr; - enum ibv_exp_query_intf_status status; - int ret = 0; - - if (mlx5_getenv_int("MLX5_ENABLE_CQE_COMPRESSION")) { - ret = ENOTSUP; - ERROR("MLX5_ENABLE_CQE_COMPRESSION must never be set"); - goto error; - } - (void)conf; /* Thresholds configuration (ignored). */ - assert(desc > MLX5_TX_COMP_THRESH); - tmpl.txq.elts_n = desc; - /* MRs will be registered in mp2mr[] later. */ - attr.rd = (struct ibv_exp_res_domain_init_attr){ - .comp_mask = (IBV_EXP_RES_DOMAIN_THREAD_MODEL | - IBV_EXP_RES_DOMAIN_MSG_MODEL), - .thread_model = IBV_EXP_THREAD_SINGLE, - .msg_model = IBV_EXP_MSG_HIGH_BW, - }; - tmpl.rd = ibv_exp_create_res_domain(priv->ctx, &attr.rd); - if (tmpl.rd == NULL) { - ret = ENOMEM; - ERROR("%p: RD creation failure: %s", - (void *)dev, strerror(ret)); - goto error; - } - attr.cq = (struct ibv_exp_cq_init_attr){ - .comp_mask = IBV_EXP_CQ_INIT_ATTR_RES_DOMAIN, - .res_domain = tmpl.rd, - }; - tmpl.cq = ibv_exp_create_cq(priv->ctx, - (((desc / MLX5_TX_COMP_THRESH) - 1) ? - ((desc / MLX5_TX_COMP_THRESH) - 1) : 1), - NULL, NULL, 0, &attr.cq); - if (tmpl.cq == NULL) { - ret = ENOMEM; - ERROR("%p: CQ creation failure: %s", - (void *)dev, strerror(ret)); - goto error; - } - DEBUG("priv->device_attr.max_qp_wr is %d", - priv->device_attr.max_qp_wr); - DEBUG("priv->device_attr.max_sge is %d", - priv->device_attr.max_sge); - attr.init = (struct ibv_exp_qp_init_attr){ - /* CQ to be associated with the send queue. */ - .send_cq = tmpl.cq, - /* CQ to be associated with the receive queue. */ - .recv_cq = tmpl.cq, - .cap = { - /* Max number of outstanding WRs. */ - .max_send_wr = ((priv->device_attr.max_qp_wr < desc) ? - priv->device_attr.max_qp_wr : - desc), - /* - * Max number of scatter/gather elements in a WR, - * must be 1 to prevent libmlx5 from trying to affect - * too much memory. TX gather is not impacted by the - * priv->device_attr.max_sge limit and will still work - * properly. - */ - .max_send_sge = 1, - }, - .qp_type = IBV_QPT_RAW_PACKET, - /* Do *NOT* enable this, completions events are managed per - * TX burst. */ - .sq_sig_all = 0, - .pd = priv->pd, - .res_domain = tmpl.rd, - .comp_mask = (IBV_EXP_QP_INIT_ATTR_PD | - IBV_EXP_QP_INIT_ATTR_RES_DOMAIN), - }; - if (priv->txq_inline && (priv->txqs_n >= priv->txqs_inline)) { - tmpl.txq.max_inline = - ((priv->txq_inline + (RTE_CACHE_LINE_SIZE - 1)) / - RTE_CACHE_LINE_SIZE); - attr.init.cap.max_inline_data = - tmpl.txq.max_inline * RTE_CACHE_LINE_SIZE; - } - tmpl.qp = ibv_exp_create_qp(priv->ctx, &attr.init); - if (tmpl.qp == NULL) { - ret = (errno ? errno : EINVAL); - ERROR("%p: QP creation failure: %s", - (void *)dev, strerror(ret)); - goto error; - } - DEBUG("TX queue capabilities: max_send_wr=%u, max_send_sge=%u," - " max_inline_data=%u", - attr.init.cap.max_send_wr, - attr.init.cap.max_send_sge, - attr.init.cap.max_inline_data); - attr.mod = (struct ibv_exp_qp_attr){ - /* Move the QP to this state. */ - .qp_state = IBV_QPS_INIT, - /* Primary port number. */ - .port_num = priv->port - }; - ret = ibv_exp_modify_qp(tmpl.qp, &attr.mod, - (IBV_EXP_QP_STATE | IBV_EXP_QP_PORT)); - if (ret) { - ERROR("%p: QP state to IBV_QPS_INIT failed: %s", - (void *)dev, strerror(ret)); - goto error; - } - ret = txq_setup(&tmpl, txq_ctrl); - if (ret) { - ERROR("%p: cannot initialize TX queue structure: %s", - (void *)dev, strerror(ret)); - goto error; - } - txq_alloc_elts(&tmpl, desc); - attr.mod = (struct ibv_exp_qp_attr){ - .qp_state = IBV_QPS_RTR - }; - ret = ibv_exp_modify_qp(tmpl.qp, &attr.mod, IBV_EXP_QP_STATE); - if (ret) { - ERROR("%p: QP state to IBV_QPS_RTR failed: %s", - (void *)dev, strerror(ret)); - goto error; - } - attr.mod.qp_state = IBV_QPS_RTS; - ret = ibv_exp_modify_qp(tmpl.qp, &attr.mod, IBV_EXP_QP_STATE); - if (ret) { - ERROR("%p: QP state to IBV_QPS_RTS failed: %s", - (void *)dev, strerror(ret)); - goto error; - } - attr.params = (struct ibv_exp_query_intf_params){ - .intf_scope = IBV_EXP_INTF_GLOBAL, - .intf = IBV_EXP_INTF_CQ, - .obj = tmpl.cq, - }; - tmpl.if_cq = ibv_exp_query_intf(priv->ctx, &attr.params, &status); - if (tmpl.if_cq == NULL) { - ret = EINVAL; - ERROR("%p: CQ interface family query failed with status %d", - (void *)dev, status); - goto error; - } - attr.params = (struct ibv_exp_query_intf_params){ - .intf_scope = IBV_EXP_INTF_GLOBAL, - .intf = IBV_EXP_INTF_QP_BURST, - .intf_version = 1, - .obj = tmpl.qp, - /* Enable multi-packet send if supported. */ - .family_flags = - ((priv->mps && !priv->sriov) ? - IBV_EXP_QP_BURST_CREATE_ENABLE_MULTI_PACKET_SEND_WR : - 0), - }; - tmpl.if_qp = ibv_exp_query_intf(priv->ctx, &attr.params, &status); - if (tmpl.if_qp == NULL) { - ret = EINVAL; - ERROR("%p: QP interface family query failed with status %d", - (void *)dev, status); - goto error; - } - /* Clean up txq in case we're reinitializing it. */ - DEBUG("%p: cleaning-up old txq just in case", (void *)txq_ctrl); - txq_cleanup(txq_ctrl); - *txq_ctrl = tmpl; - DEBUG("%p: txq updated with %p", (void *)txq_ctrl, (void *)&tmpl); - /* Pre-register known mempools. */ - rte_mempool_walk(txq_mp2mr_iter, txq_ctrl); - assert(ret == 0); - return 0; -error: - txq_cleanup(&tmpl); - assert(ret > 0); - return ret; -} - /** * DPDK callback to configure a TX queue. * @@ -460,12 +137,10 @@ mlx5_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, unsigned int socket, const struct rte_eth_txconf *conf) { struct priv *priv = dev->data->dev_private; - struct txq *txq = (*priv->txqs)[idx]; - struct txq_ctrl *txq_ctrl = container_of(txq, struct txq_ctrl, txq); - int ret; - - if (mlx5_is_secondary()) - return -E_RTE_SECONDARY; + struct mlx5_txq_data *txq = (*priv->txqs)[idx]; + struct mlx5_txq_ctrl *txq_ctrl = + container_of(txq, struct mlx5_txq_ctrl, txq); + int ret = 0; priv_lock(priv); if (desc <= MLX5_TX_COMP_THRESH) { @@ -489,39 +164,24 @@ mlx5_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, priv_unlock(priv); return -EOVERFLOW; } - if (txq != NULL) { - DEBUG("%p: reusing already allocated queue index %u (%p)", - (void *)dev, idx, (void *)txq); - if (priv->started) { - priv_unlock(priv); - return -EEXIST; - } - (*priv->txqs)[idx] = NULL; - txq_cleanup(txq_ctrl); - } else { - txq_ctrl = - rte_calloc_socket("TXQ", 1, - sizeof(*txq_ctrl) + - desc * sizeof(struct rte_mbuf *), - 0, socket); - if (txq_ctrl == NULL) { - ERROR("%p: unable to allocate queue index %u", - (void *)dev, idx); - priv_unlock(priv); - return -ENOMEM; - } + if (!mlx5_priv_txq_releasable(priv, idx)) { + ret = EBUSY; + ERROR("%p: unable to release queue index %u", + (void *)dev, idx); + goto out; } - ret = txq_ctrl_setup(dev, txq_ctrl, desc, socket, conf); - if (ret) - rte_free(txq_ctrl); - else { - txq_ctrl->txq.stats.idx = idx; - DEBUG("%p: adding TX queue %p to list", - (void *)dev, (void *)txq_ctrl); - (*priv->txqs)[idx] = &txq_ctrl->txq; - /* Update send callback. */ - priv_select_tx_function(priv); + mlx5_priv_txq_release(priv, idx); + txq_ctrl = mlx5_priv_txq_new(priv, idx, desc, socket, conf); + if (!txq_ctrl) { + ERROR("%p: unable to allocate queue index %u", + (void *)dev, idx); + ret = ENOMEM; + goto out; } + DEBUG("%p: adding TX queue %p to list", + (void *)dev, (void *)txq_ctrl); + (*priv->txqs)[idx] = &txq_ctrl->txq; +out: priv_unlock(priv); return -ret; } @@ -535,68 +195,593 @@ mlx5_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, void mlx5_tx_queue_release(void *dpdk_txq) { - struct txq *txq = (struct txq *)dpdk_txq; - struct txq_ctrl *txq_ctrl; + struct mlx5_txq_data *txq = (struct mlx5_txq_data *)dpdk_txq; + struct mlx5_txq_ctrl *txq_ctrl; struct priv *priv; unsigned int i; - if (mlx5_is_secondary()) - return; - if (txq == NULL) return; - txq_ctrl = container_of(txq, struct txq_ctrl, txq); + txq_ctrl = container_of(txq, struct mlx5_txq_ctrl, txq); priv = txq_ctrl->priv; priv_lock(priv); for (i = 0; (i != priv->txqs_n); ++i) if ((*priv->txqs)[i] == txq) { DEBUG("%p: removing TX queue %p from list", (void *)priv->dev, (void *)txq_ctrl); - (*priv->txqs)[i] = NULL; + mlx5_priv_txq_release(priv, i); break; } - txq_cleanup(txq_ctrl); - rte_free(txq_ctrl); priv_unlock(priv); } + /** - * DPDK callback for TX in secondary processes. + * Map locally UAR used in Tx queues for BlueFlame doorbell. * - * This function configures all queues from primary process information - * if necessary before reverting to the normal TX burst callback. - * - * @param dpdk_txq - * Generic pointer to TX queue structure. - * @param[in] pkts - * Packets to transmit. - * @param pkts_n - * Number of packets in array. + * @param[in] priv + * Pointer to private structure. + * @param fd + * Verbs file descriptor to map UAR pages. * * @return - * Number of packets successfully transmitted (<= pkts_n). + * 0 on success, errno value on failure. */ -uint16_t -mlx5_tx_burst_secondary_setup(void *dpdk_txq, struct rte_mbuf **pkts, - uint16_t pkts_n) +int +priv_tx_uar_remap(struct priv *priv, int fd) { - struct txq *txq = dpdk_txq; - struct txq_ctrl *txq_ctrl = container_of(txq, struct txq_ctrl, txq); - struct priv *priv = mlx5_secondary_data_setup(txq_ctrl->priv); - struct priv *primary_priv; - unsigned int index; + unsigned int i, j; + uintptr_t pages[priv->txqs_n]; + unsigned int pages_n = 0; + uintptr_t uar_va; + void *addr; + struct mlx5_txq_data *txq; + struct mlx5_txq_ctrl *txq_ctrl; + int already_mapped; + size_t page_size = sysconf(_SC_PAGESIZE); - if (priv == NULL) - return 0; - primary_priv = - mlx5_secondary_data[priv->dev->data->port_id].primary_priv; - /* Look for queue index in both private structures. */ - for (index = 0; index != priv->txqs_n; ++index) - if (((*primary_priv->txqs)[index] == txq) || - ((*priv->txqs)[index] == txq)) - break; - if (index == priv->txqs_n) - return 0; - txq = (*priv->txqs)[index]; - return priv->dev->tx_pkt_burst(txq, pkts, pkts_n); + memset(pages, 0, priv->txqs_n * sizeof(uintptr_t)); + /* + * As rdma-core, UARs are mapped in size of OS page size. + * Use aligned address to avoid duplicate mmap. + * Ref to libmlx5 function: mlx5_init_context() + */ + for (i = 0; i != priv->txqs_n; ++i) { + if (!(*priv->txqs)[i]) + continue; + txq = (*priv->txqs)[i]; + txq_ctrl = container_of(txq, struct mlx5_txq_ctrl, txq); + uar_va = (uintptr_t)txq_ctrl->txq.bf_reg; + uar_va = RTE_ALIGN_FLOOR(uar_va, page_size); + already_mapped = 0; + for (j = 0; j != pages_n; ++j) { + if (pages[j] == uar_va) { + already_mapped = 1; + break; + } + } + if (already_mapped) + continue; + pages[pages_n++] = uar_va; + addr = mmap((void *)uar_va, page_size, + PROT_WRITE, MAP_FIXED | MAP_SHARED, fd, + txq_ctrl->uar_mmap_offset); + if (addr != (void *)uar_va) { + ERROR("call to mmap failed on UAR for txq %d\n", i); + return -1; + } + } + return 0; +} + +/** + * Create the Tx queue Verbs object. + * + * @param priv + * Pointer to private structure. + * @param idx + * Queue index in DPDK Rx queue array + * + * @return + * The Verbs object initialised if it can be created. + */ +struct mlx5_txq_ibv* +mlx5_priv_txq_ibv_new(struct priv *priv, uint16_t idx) +{ + struct mlx5_txq_data *txq_data = (*priv->txqs)[idx]; + struct mlx5_txq_ctrl *txq_ctrl = + container_of(txq_data, struct mlx5_txq_ctrl, txq); + struct mlx5_txq_ibv tmpl; + struct mlx5_txq_ibv *txq_ibv; + union { + struct ibv_qp_init_attr_ex init; + struct ibv_cq_init_attr_ex cq; + struct ibv_qp_attr mod; + struct ibv_cq_ex cq_attr; + } attr; + unsigned int cqe_n; + struct mlx5dv_qp qp = { .comp_mask = MLX5DV_QP_MASK_UAR_MMAP_OFFSET }; + struct mlx5dv_cq cq_info; + struct mlx5dv_obj obj; + const int desc = 1 << txq_data->elts_n; + int ret = 0; + + assert(txq_data); + if (mlx5_getenv_int("MLX5_ENABLE_CQE_COMPRESSION")) { + ERROR("MLX5_ENABLE_CQE_COMPRESSION must never be set"); + goto error; + } + memset(&tmpl, 0, sizeof(struct mlx5_txq_ibv)); + /* MRs will be registered in mp2mr[] later. */ + attr.cq = (struct ibv_cq_init_attr_ex){ + .comp_mask = 0, + }; + cqe_n = ((desc / MLX5_TX_COMP_THRESH) - 1) ? + ((desc / MLX5_TX_COMP_THRESH) - 1) : 1; + if (priv->mps == MLX5_MPW_ENHANCED) + cqe_n += MLX5_TX_COMP_THRESH_INLINE_DIV; + tmpl.cq = ibv_create_cq(priv->ctx, cqe_n, NULL, NULL, 0); + if (tmpl.cq == NULL) { + ERROR("%p: CQ creation failure", (void *)txq_ctrl); + goto error; + } + attr.init = (struct ibv_qp_init_attr_ex){ + /* CQ to be associated with the send queue. */ + .send_cq = tmpl.cq, + /* CQ to be associated with the receive queue. */ + .recv_cq = tmpl.cq, + .cap = { + /* Max number of outstanding WRs. */ + .max_send_wr = + ((priv->device_attr.orig_attr.max_qp_wr < + desc) ? + priv->device_attr.orig_attr.max_qp_wr : + desc), + /* + * Max number of scatter/gather elements in a WR, + * must be 1 to prevent libmlx5 from trying to affect + * too much memory. TX gather is not impacted by the + * priv->device_attr.max_sge limit and will still work + * properly. + */ + .max_send_sge = 1, + }, + .qp_type = IBV_QPT_RAW_PACKET, + /* + * Do *NOT* enable this, completions events are managed per + * Tx burst. + */ + .sq_sig_all = 0, + .pd = priv->pd, + .comp_mask = IBV_QP_INIT_ATTR_PD, + }; + if (txq_data->inline_en) + attr.init.cap.max_inline_data = txq_ctrl->max_inline_data; + if (txq_data->tso_en) { + attr.init.max_tso_header = txq_ctrl->max_tso_header; + attr.init.comp_mask |= IBV_QP_INIT_ATTR_MAX_TSO_HEADER; + } + tmpl.qp = ibv_create_qp_ex(priv->ctx, &attr.init); + if (tmpl.qp == NULL) { + ERROR("%p: QP creation failure", (void *)txq_ctrl); + goto error; + } + attr.mod = (struct ibv_qp_attr){ + /* Move the QP to this state. */ + .qp_state = IBV_QPS_INIT, + /* Primary port number. */ + .port_num = priv->port + }; + ret = ibv_modify_qp(tmpl.qp, &attr.mod, (IBV_QP_STATE | IBV_QP_PORT)); + if (ret) { + ERROR("%p: QP state to IBV_QPS_INIT failed", (void *)txq_ctrl); + goto error; + } + attr.mod = (struct ibv_qp_attr){ + .qp_state = IBV_QPS_RTR + }; + ret = ibv_modify_qp(tmpl.qp, &attr.mod, IBV_QP_STATE); + if (ret) { + ERROR("%p: QP state to IBV_QPS_RTR failed", (void *)txq_ctrl); + goto error; + } + attr.mod.qp_state = IBV_QPS_RTS; + ret = ibv_modify_qp(tmpl.qp, &attr.mod, IBV_QP_STATE); + if (ret) { + ERROR("%p: QP state to IBV_QPS_RTS failed", (void *)txq_ctrl); + goto error; + } + txq_ibv = rte_calloc_socket(__func__, 1, sizeof(struct mlx5_txq_ibv), 0, + txq_ctrl->socket); + if (!txq_ibv) { + ERROR("%p: cannot allocate memory", (void *)txq_ctrl); + goto error; + } + obj.cq.in = tmpl.cq; + obj.cq.out = &cq_info; + obj.qp.in = tmpl.qp; + obj.qp.out = &qp; + ret = mlx5dv_init_obj(&obj, MLX5DV_OBJ_CQ | MLX5DV_OBJ_QP); + if (ret != 0) + goto error; + if (cq_info.cqe_size != RTE_CACHE_LINE_SIZE) { + ERROR("Wrong MLX5_CQE_SIZE environment variable value: " + "it should be set to %u", RTE_CACHE_LINE_SIZE); + goto error; + } + txq_data->cqe_n = log2above(cq_info.cqe_cnt); + txq_data->qp_num_8s = tmpl.qp->qp_num << 8; + txq_data->wqes = qp.sq.buf; + txq_data->wqe_n = log2above(qp.sq.wqe_cnt); + txq_data->qp_db = &qp.dbrec[MLX5_SND_DBR]; + txq_data->bf_reg = qp.bf.reg; + txq_data->cq_db = cq_info.dbrec; + txq_data->cqes = + (volatile struct mlx5_cqe (*)[]) + (uintptr_t)cq_info.buf; + txq_data->cq_ci = 0; + txq_data->cq_pi = 0; + txq_data->wqe_ci = 0; + txq_data->wqe_pi = 0; + txq_ibv->qp = tmpl.qp; + txq_ibv->cq = tmpl.cq; + rte_atomic32_inc(&txq_ibv->refcnt); + if (qp.comp_mask & MLX5DV_QP_MASK_UAR_MMAP_OFFSET) { + txq_ctrl->uar_mmap_offset = qp.uar_mmap_offset; + } else { + ERROR("Failed to retrieve UAR info, invalid libmlx5.so version"); + goto error; + } + DEBUG("%p: Verbs Tx queue %p: refcnt %d", (void *)priv, + (void *)txq_ibv, rte_atomic32_read(&txq_ibv->refcnt)); + LIST_INSERT_HEAD(&priv->txqsibv, txq_ibv, next); + return txq_ibv; +error: + if (tmpl.cq) + claim_zero(ibv_destroy_cq(tmpl.cq)); + if (tmpl.qp) + claim_zero(ibv_destroy_qp(tmpl.qp)); + return NULL; +} + +/** + * Get an Tx queue Verbs object. + * + * @param priv + * Pointer to private structure. + * @param idx + * Queue index in DPDK Rx queue array + * + * @return + * The Verbs object if it exists. + */ +struct mlx5_txq_ibv* +mlx5_priv_txq_ibv_get(struct priv *priv, uint16_t idx) +{ + struct mlx5_txq_ctrl *txq_ctrl; + + if (idx >= priv->txqs_n) + return NULL; + if (!(*priv->txqs)[idx]) + return NULL; + txq_ctrl = container_of((*priv->txqs)[idx], struct mlx5_txq_ctrl, txq); + if (txq_ctrl->ibv) { + rte_atomic32_inc(&txq_ctrl->ibv->refcnt); + DEBUG("%p: Verbs Tx queue %p: refcnt %d", (void *)priv, + (void *)txq_ctrl->ibv, + rte_atomic32_read(&txq_ctrl->ibv->refcnt)); + } + return txq_ctrl->ibv; +} + +/** + * Release an Tx verbs queue object. + * + * @param priv + * Pointer to private structure. + * @param txq_ibv + * Verbs Tx queue object. + * + * @return + * 0 on success, errno on failure. + */ +int +mlx5_priv_txq_ibv_release(struct priv *priv, struct mlx5_txq_ibv *txq_ibv) +{ + (void)priv; + assert(txq_ibv); + DEBUG("%p: Verbs Tx queue %p: refcnt %d", (void *)priv, + (void *)txq_ibv, rte_atomic32_read(&txq_ibv->refcnt)); + if (rte_atomic32_dec_and_test(&txq_ibv->refcnt)) { + claim_zero(ibv_destroy_qp(txq_ibv->qp)); + claim_zero(ibv_destroy_cq(txq_ibv->cq)); + LIST_REMOVE(txq_ibv, next); + rte_free(txq_ibv); + return 0; + } + return EBUSY; +} + +/** + * Return true if a single reference exists on the object. + * + * @param priv + * Pointer to private structure. + * @param txq_ibv + * Verbs Tx queue object. + */ +int +mlx5_priv_txq_ibv_releasable(struct priv *priv, struct mlx5_txq_ibv *txq_ibv) +{ + (void)priv; + assert(txq_ibv); + return (rte_atomic32_read(&txq_ibv->refcnt) == 1); +} + +/** + * Verify the Verbs Tx queue list is empty + * + * @param priv + * Pointer to private structure. + * + * @return the number of object not released. + */ +int +mlx5_priv_txq_ibv_verify(struct priv *priv) +{ + int ret = 0; + struct mlx5_txq_ibv *txq_ibv; + + LIST_FOREACH(txq_ibv, &priv->txqsibv, next) { + DEBUG("%p: Verbs Tx queue %p still referenced", (void *)priv, + (void *)txq_ibv); + ++ret; + } + return ret; +} + +/** + * Create a DPDK Tx queue. + * + * @param priv + * Pointer to private structure. + * @param idx + * TX queue index. + * @param desc + * Number of descriptors to configure in queue. + * @param socket + * NUMA socket on which memory must be allocated. + * @param[in] conf + * Thresholds parameters. + * + * @return + * A DPDK queue object on success. + */ +struct mlx5_txq_ctrl* +mlx5_priv_txq_new(struct priv *priv, uint16_t idx, uint16_t desc, + unsigned int socket, + const struct rte_eth_txconf *conf) +{ + const unsigned int max_tso_inline = + ((MLX5_MAX_TSO_HEADER + (RTE_CACHE_LINE_SIZE - 1)) / + RTE_CACHE_LINE_SIZE); + struct mlx5_txq_ctrl *tmpl; + + tmpl = rte_calloc_socket("TXQ", 1, + sizeof(*tmpl) + + desc * sizeof(struct rte_mbuf *), + 0, socket); + if (!tmpl) + return NULL; + assert(desc > MLX5_TX_COMP_THRESH); + tmpl->txq.flags = conf->txq_flags; + tmpl->priv = priv; + tmpl->socket = socket; + tmpl->txq.elts_n = log2above(desc); + if (priv->mps == MLX5_MPW_ENHANCED) + tmpl->txq.mpw_hdr_dseg = priv->mpw_hdr_dseg; + /* MRs will be registered in mp2mr[] later. */ + DEBUG("priv->device_attr.max_qp_wr is %d", + priv->device_attr.orig_attr.max_qp_wr); + DEBUG("priv->device_attr.max_sge is %d", + priv->device_attr.orig_attr.max_sge); + if (priv->txq_inline && (priv->txqs_n >= priv->txqs_inline)) { + unsigned int ds_cnt; + + tmpl->txq.max_inline = + ((priv->txq_inline + (RTE_CACHE_LINE_SIZE - 1)) / + RTE_CACHE_LINE_SIZE); + tmpl->txq.inline_en = 1; + /* TSO and MPS can't be enabled concurrently. */ + assert(!priv->tso || !priv->mps); + if (priv->mps == MLX5_MPW_ENHANCED) { + tmpl->txq.inline_max_packet_sz = + priv->inline_max_packet_sz; + /* To minimize the size of data set, avoid requesting + * too large WQ. + */ + tmpl->max_inline_data = + ((RTE_MIN(priv->txq_inline, + priv->inline_max_packet_sz) + + (RTE_CACHE_LINE_SIZE - 1)) / + RTE_CACHE_LINE_SIZE) * RTE_CACHE_LINE_SIZE; + } else if (priv->tso) { + int inline_diff = tmpl->txq.max_inline - max_tso_inline; + + /* + * Adjust inline value as Verbs aggregates + * tso_inline and txq_inline fields. + */ + tmpl->max_inline_data = inline_diff > 0 ? + inline_diff * + RTE_CACHE_LINE_SIZE : + 0; + } else { + tmpl->max_inline_data = + tmpl->txq.max_inline * RTE_CACHE_LINE_SIZE; + } + /* + * Check if the inline size is too large in a way which + * can make the WQE DS to overflow. + * Considering in calculation: + * WQE CTRL (1 DS) + * WQE ETH (1 DS) + * Inline part (N DS) + */ + ds_cnt = 2 + (tmpl->txq.max_inline / MLX5_WQE_DWORD_SIZE); + if (ds_cnt > MLX5_DSEG_MAX) { + unsigned int max_inline = (MLX5_DSEG_MAX - 2) * + MLX5_WQE_DWORD_SIZE; + + max_inline = max_inline - (max_inline % + RTE_CACHE_LINE_SIZE); + WARN("txq inline is too large (%d) setting it to " + "the maximum possible: %d\n", + priv->txq_inline, max_inline); + tmpl->txq.max_inline = max_inline / RTE_CACHE_LINE_SIZE; + } + } + if (priv->tso) { + tmpl->max_tso_header = max_tso_inline * RTE_CACHE_LINE_SIZE; + tmpl->txq.max_inline = RTE_MAX(tmpl->txq.max_inline, + max_tso_inline); + tmpl->txq.tso_en = 1; + } + if (priv->tunnel_en) + tmpl->txq.tunnel_en = 1; + tmpl->txq.elts = + (struct rte_mbuf *(*)[1 << tmpl->txq.elts_n])(tmpl + 1); + tmpl->txq.stats.idx = idx; + rte_atomic32_inc(&tmpl->refcnt); + DEBUG("%p: Tx queue %p: refcnt %d", (void *)priv, + (void *)tmpl, rte_atomic32_read(&tmpl->refcnt)); + LIST_INSERT_HEAD(&priv->txqsctrl, tmpl, next); + return tmpl; +} + +/** + * Get a Tx queue. + * + * @param priv + * Pointer to private structure. + * @param idx + * TX queue index. + * + * @return + * A pointer to the queue if it exists. + */ +struct mlx5_txq_ctrl* +mlx5_priv_txq_get(struct priv *priv, uint16_t idx) +{ + struct mlx5_txq_ctrl *ctrl = NULL; + + if ((*priv->txqs)[idx]) { + ctrl = container_of((*priv->txqs)[idx], struct mlx5_txq_ctrl, + txq); + unsigned int i; + + mlx5_priv_txq_ibv_get(priv, idx); + for (i = 0; i != MLX5_PMD_TX_MP_CACHE; ++i) { + struct mlx5_mr *mr = NULL; + + (void)mr; + if (ctrl->txq.mp2mr[i]) { + mr = priv_mr_get(priv, ctrl->txq.mp2mr[i]->mp); + assert(mr); + } + } + rte_atomic32_inc(&ctrl->refcnt); + DEBUG("%p: Tx queue %p: refcnt %d", (void *)priv, + (void *)ctrl, rte_atomic32_read(&ctrl->refcnt)); + } + return ctrl; +} + +/** + * Release a Tx queue. + * + * @param priv + * Pointer to private structure. + * @param idx + * TX queue index. + * + * @return + * 0 on success, errno on failure. + */ +int +mlx5_priv_txq_release(struct priv *priv, uint16_t idx) +{ + unsigned int i; + struct mlx5_txq_ctrl *txq; + + if (!(*priv->txqs)[idx]) + return 0; + txq = container_of((*priv->txqs)[idx], struct mlx5_txq_ctrl, txq); + DEBUG("%p: Tx queue %p: refcnt %d", (void *)priv, + (void *)txq, rte_atomic32_read(&txq->refcnt)); + if (txq->ibv) { + int ret; + + ret = mlx5_priv_txq_ibv_release(priv, txq->ibv); + if (!ret) + txq->ibv = NULL; + } + for (i = 0; i != MLX5_PMD_TX_MP_CACHE; ++i) { + if (txq->txq.mp2mr[i]) { + priv_mr_release(priv, txq->txq.mp2mr[i]); + txq->txq.mp2mr[i] = NULL; + } + } + if (rte_atomic32_dec_and_test(&txq->refcnt)) { + txq_free_elts(txq); + LIST_REMOVE(txq, next); + rte_free(txq); + (*priv->txqs)[idx] = NULL; + return 0; + } + return EBUSY; +} + +/** + * Verify if the queue can be released. + * + * @param priv + * Pointer to private structure. + * @param idx + * TX queue index. + * + * @return + * 1 if the queue can be released. + */ +int +mlx5_priv_txq_releasable(struct priv *priv, uint16_t idx) +{ + struct mlx5_txq_ctrl *txq; + + if (!(*priv->txqs)[idx]) + return -1; + txq = container_of((*priv->txqs)[idx], struct mlx5_txq_ctrl, txq); + return (rte_atomic32_read(&txq->refcnt) == 1); +} + +/** + * Verify the Tx Queue list is empty + * + * @param priv + * Pointer to private structure. + * + * @return the number of object not released. + */ +int +mlx5_priv_txq_verify(struct priv *priv) +{ + struct mlx5_txq_ctrl *txq; + int ret = 0; + + LIST_FOREACH(txq, &priv->txqsctrl, next) { + DEBUG("%p: Tx Queue %p still referenced", (void *)priv, + (void *)txq); + ++ret; + } + return ret; } diff --git a/dpdk/drivers/net/mlx5/mlx5_utils.h b/dpdk/drivers/net/mlx5/mlx5_utils.h index a824787f..2fbd10b1 100644 --- a/dpdk/drivers/net/mlx5/mlx5_utils.h +++ b/dpdk/drivers/net/mlx5/mlx5_utils.h @@ -35,6 +35,7 @@ #define RTE_PMD_MLX5_UTILS_H_ #include +#include #include #include #include @@ -61,6 +62,9 @@ !!(((bf)[((b) / (sizeof((bf)[0]) * CHAR_BIT))] & \ ((size_t)1 << ((b) % (sizeof((bf)[0]) * CHAR_BIT)))))) +/* Convert a bit number to the corresponding 64-bit mask */ +#define MLX5_BITSHIFT(v) (UINT64_C(1) << (v)) + /* Save and restore errno around argument evaluation. */ #define ERRNO_SAFE(x) ((errno = (int []){ errno, ((x), 0) }[0])) @@ -128,11 +132,13 @@ pmd_drv_log_basename(const char *s) #define DEBUG(...) PMD_DRV_LOG(DEBUG, __VA_ARGS__) #define claim_zero(...) assert((__VA_ARGS__) == 0) +#define claim_nonzero(...) assert((__VA_ARGS__) != 0) #else /* NDEBUG */ #define DEBUG(...) (void)0 #define claim_zero(...) (__VA_ARGS__) +#define claim_nonzero(...) (__VA_ARGS__) #endif /* NDEBUG */ diff --git a/dpdk/drivers/net/mlx5/mlx5_vlan.c b/dpdk/drivers/net/mlx5/mlx5_vlan.c index 1b0fa40a..198a69e3 100644 --- a/dpdk/drivers/net/mlx5/mlx5_vlan.c +++ b/dpdk/drivers/net/mlx5/mlx5_vlan.c @@ -36,75 +36,13 @@ #include #include -/* DPDK headers don't like -pedantic. */ -#ifdef PEDANTIC -#pragma GCC diagnostic ignored "-Wpedantic" -#endif #include #include -#ifdef PEDANTIC -#pragma GCC diagnostic error "-Wpedantic" -#endif #include "mlx5_utils.h" #include "mlx5.h" #include "mlx5_autoconf.h" -/** - * Configure a VLAN filter. - * - * @param dev - * Pointer to Ethernet device structure. - * @param vlan_id - * VLAN ID to filter. - * @param on - * Toggle filter. - * - * @return - * 0 on success, errno value on failure. - */ -static int -vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) -{ - struct priv *priv = dev->data->dev_private; - unsigned int i; - - DEBUG("%p: %s VLAN filter ID %" PRIu16, - (void *)dev, (on ? "enable" : "disable"), vlan_id); - assert(priv->vlan_filter_n <= RTE_DIM(priv->vlan_filter)); - for (i = 0; (i != priv->vlan_filter_n); ++i) - if (priv->vlan_filter[i] == vlan_id) - break; - /* Check if there's room for another VLAN filter. */ - if (i == RTE_DIM(priv->vlan_filter)) - return ENOMEM; - if (i < priv->vlan_filter_n) { - assert(priv->vlan_filter_n != 0); - /* Enabling an existing VLAN filter has no effect. */ - if (on) - return 0; - /* Remove VLAN filter from list. */ - --priv->vlan_filter_n; - memmove(&priv->vlan_filter[i], - &priv->vlan_filter[i + 1], - sizeof(priv->vlan_filter[i]) * - (priv->vlan_filter_n - i)); - priv->vlan_filter[priv->vlan_filter_n] = 0; - } else { - assert(i == priv->vlan_filter_n); - /* Disabling an unknown VLAN filter has no effect. */ - if (!on) - return 0; - /* Add new VLAN filter. */ - priv->vlan_filter[priv->vlan_filter_n] = vlan_id; - ++priv->vlan_filter_n; - } - /* Rehash flows in all hash RX queues. */ - priv_mac_addrs_disable(priv); - priv_special_flow_disable_all(priv); - return priv_rehash_flows(priv); -} - /** * DPDK callback to configure a VLAN filter. * @@ -122,13 +60,47 @@ int mlx5_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) { struct priv *priv = dev->data->dev_private; - int ret; + unsigned int i; + int ret = 0; priv_lock(priv); - ret = vlan_filter_set(dev, vlan_id, on); + DEBUG("%p: %s VLAN filter ID %" PRIu16, + (void *)dev, (on ? "enable" : "disable"), vlan_id); + assert(priv->vlan_filter_n <= RTE_DIM(priv->vlan_filter)); + for (i = 0; (i != priv->vlan_filter_n); ++i) + if (priv->vlan_filter[i] == vlan_id) + break; + /* Check if there's room for another VLAN filter. */ + if (i == RTE_DIM(priv->vlan_filter)) { + ret = -ENOMEM; + goto out; + } + if (i < priv->vlan_filter_n) { + assert(priv->vlan_filter_n != 0); + /* Enabling an existing VLAN filter has no effect. */ + if (on) + goto out; + /* Remove VLAN filter from list. */ + --priv->vlan_filter_n; + memmove(&priv->vlan_filter[i], + &priv->vlan_filter[i + 1], + sizeof(priv->vlan_filter[i]) * + (priv->vlan_filter_n - i)); + priv->vlan_filter[priv->vlan_filter_n] = 0; + } else { + assert(i == priv->vlan_filter_n); + /* Disabling an unknown VLAN filter has no effect. */ + if (!on) + goto out; + /* Add new VLAN filter. */ + priv->vlan_filter[priv->vlan_filter_n] = vlan_id; + ++priv->vlan_filter_n; + } + if (dev->data->dev_started) + priv_dev_traffic_restart(priv, dev); +out: priv_unlock(priv); - assert(ret >= 0); - return -ret; + return ret; } /** @@ -144,22 +116,29 @@ mlx5_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) static void priv_vlan_strip_queue_set(struct priv *priv, uint16_t idx, int on) { - struct rxq *rxq = (*priv->rxqs)[idx]; - struct rxq_ctrl *rxq_ctrl = container_of(rxq, struct rxq_ctrl, rxq); - struct ibv_exp_wq_attr mod; + struct mlx5_rxq_data *rxq = (*priv->rxqs)[idx]; + struct mlx5_rxq_ctrl *rxq_ctrl = + container_of(rxq, struct mlx5_rxq_ctrl, rxq); + struct ibv_wq_attr mod; uint16_t vlan_offloads = - (on ? IBV_EXP_RECEIVE_WQ_CVLAN_STRIP : 0) | + (on ? IBV_WQ_FLAGS_CVLAN_STRIPPING : 0) | 0; int err; DEBUG("set VLAN offloads 0x%x for port %d queue %d", vlan_offloads, rxq->port_id, idx); - mod = (struct ibv_exp_wq_attr){ - .attr_mask = IBV_EXP_WQ_ATTR_VLAN_OFFLOADS, - .vlan_offloads = vlan_offloads, + if (!rxq_ctrl->ibv) { + /* Update related bits in RX queue. */ + rxq->vlan_strip = !!on; + return; + } + mod = (struct ibv_wq_attr){ + .attr_mask = IBV_WQ_ATTR_FLAGS, + .flags_mask = IBV_WQ_FLAGS_CVLAN_STRIPPING, + .flags = vlan_offloads, }; - err = ibv_exp_modify_wq(rxq_ctrl->wq, &mod); + err = ibv_modify_wq(rxq_ctrl->ibv->wq, &mod); if (err) { ERROR("%p: failed to modified stripping mode: %s", (void *)priv, strerror(err)); @@ -210,7 +189,7 @@ mlx5_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on) * @param mask * VLAN offload bit mask. */ -void +int mlx5_vlan_offload_set(struct rte_eth_dev *dev, int mask) { struct priv *priv = dev->data->dev_private; @@ -221,7 +200,7 @@ mlx5_vlan_offload_set(struct rte_eth_dev *dev, int mask) if (!priv->hw_vlan_strip) { ERROR("VLAN stripping is not supported"); - return; + return 0; } /* Run on every RX queue and set/reset VLAN stripping. */ @@ -230,4 +209,6 @@ mlx5_vlan_offload_set(struct rte_eth_dev *dev, int mask) priv_vlan_strip_queue_set(priv, i, hw_vlan_strip); priv_unlock(priv); } + + return 0; } diff --git a/dpdk/drivers/net/mpipe/Makefile b/dpdk/drivers/net/mpipe/Makefile deleted file mode 100644 index 846e2e07..00000000 --- a/dpdk/drivers/net/mpipe/Makefile +++ /dev/null @@ -1,47 +0,0 @@ -# -# Copyright 2015 EZchip Semiconductor Ltd. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in -# the documentation and/or other materials provided with the -# distribution. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS -# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE -# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, -# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. - -include $(RTE_SDK)/mk/rte.vars.mk - -# library name -LIB = librte_pmd_mpipe.a - -CFLAGS += $(WERROR_FLAGS) -O3 -LDLIBS += -lgxio - -EXPORT_MAP := rte_pmd_mpipe_version.map - -LIBABIVER := 1 - -SRCS-$(CONFIG_RTE_LIBRTE_MPIPE_PMD) += mpipe_tilegx.c - -DEPDIRS-$(CONFIG_RTE_LIBRTE_MPIPE_PMD) += lib/librte_eal lib/librte_ether -DEPDIRS-$(CONFIG_RTE_LIBRTE_MPIPE_PMD) += lib/librte_mempool lib/librte_mbuf -DEPDIRS-$(CONFIG_RTE_LIBRTE_MPIPE_PMD) += lib/librte_net - -include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/net/mpipe/mpipe_tilegx.c b/dpdk/drivers/net/mpipe/mpipe_tilegx.c deleted file mode 100644 index 93f87308..00000000 --- a/dpdk/drivers/net/mpipe/mpipe_tilegx.c +++ /dev/null @@ -1,1655 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2015 EZchip Semiconductor Ltd. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of EZchip Semiconductor nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include - -#include -#include -#include -#include -#include -#include - -#include -#include - -#include - -#ifdef RTE_LIBRTE_MPIPE_PMD_DEBUG -#define PMD_DEBUG_RX(...) RTE_LOG(DEBUG, PMD, __VA_ARGS__) -#define PMD_DEBUG_TX(...) RTE_LOG(DEBUG, PMD, __VA_ARGS__) -#else -#define PMD_DEBUG_RX(...) -#define PMD_DEBUG_TX(...) -#endif - -#define MPIPE_MAX_CHANNELS 128 -#define MPIPE_TX_MAX_QUEUES 128 -#define MPIPE_RX_MAX_QUEUES 16 -#define MPIPE_TX_DESCS 512 -#define MPIPE_RX_BUCKETS 256 -#define MPIPE_RX_STACK_SIZE 65536 -#define MPIPE_RX_IP_ALIGN 2 -#define MPIPE_BSM_ALIGN 128 - -#define MPIPE_LINK_UPDATE_TIMEOUT 10 /* s */ -#define MPIPE_LINK_UPDATE_INTERVAL 100000 /* us */ - -struct mpipe_channel_config { - int enable; - int first_bucket; - int num_buckets; - int head_room; - gxio_mpipe_rules_stacks_t stacks; -}; - -struct mpipe_context { - rte_spinlock_t lock; - gxio_mpipe_context_t context; - struct mpipe_channel_config channels[MPIPE_MAX_CHANNELS]; -}; - -/* Per-core local data. */ -struct mpipe_local { - int mbuf_push_debt[RTE_MAX_ETHPORTS]; /* Buffer push debt. */ -} __rte_cache_aligned; - -#define MPIPE_BUF_DEBT_THRESHOLD 32 -static __thread struct mpipe_local mpipe_local; -static struct mpipe_context mpipe_contexts[GXIO_MPIPE_INSTANCE_MAX]; -static int mpipe_instances; -static const char *drivername = "MPIPE PMD"; - -/* Per queue statistics. */ -struct mpipe_queue_stats { - uint64_t packets, bytes, errors, nomem; -}; - -/* Common tx/rx queue fields. */ -struct mpipe_queue { - struct mpipe_dev_priv *priv; /* "priv" data of its device. */ - uint16_t nb_desc; /* Number of tx descriptors. */ - uint16_t port_id; /* Device index. */ - uint16_t stat_idx; /* Queue stats index. */ - uint8_t queue_idx; /* Queue index. */ - uint8_t link_status; /* 0 = link down. */ - struct mpipe_queue_stats stats; /* Stat data for the queue. */ -}; - -/* Transmit queue description. */ -struct mpipe_tx_queue { - struct mpipe_queue q; /* Common stuff. */ -}; - -/* Receive queue description. */ -struct mpipe_rx_queue { - struct mpipe_queue q; /* Common stuff. */ - gxio_mpipe_iqueue_t iqueue; /* mPIPE iqueue. */ - gxio_mpipe_idesc_t *next_desc; /* Next idesc to process. */ - int avail_descs; /* Number of available descs. */ - void *rx_ring_mem; /* DMA ring memory. */ -}; - -struct mpipe_dev_priv { - gxio_mpipe_context_t *context; /* mPIPE context. */ - gxio_mpipe_link_t link; /* mPIPE link for the device. */ - gxio_mpipe_equeue_t equeue; /* mPIPE equeue. */ - unsigned equeue_size; /* mPIPE equeue desc count. */ - int instance; /* mPIPE instance. */ - int ering; /* mPIPE eDMA ring. */ - int stack; /* mPIPE buffer stack. */ - int channel; /* Device channel. */ - int port_id; /* DPDK port index. */ - struct rte_eth_dev *eth_dev; /* DPDK device. */ - struct rte_mbuf **tx_comps; /* TX completion array. */ - struct rte_mempool *rx_mpool; /* mpool used by the rx queues. */ - unsigned rx_offset; /* Receive head room. */ - unsigned rx_size_code; /* mPIPE rx buffer size code. */ - int is_xaui:1, /* Is this an xgbe or gbe? */ - initialized:1, /* Initialized port? */ - running:1; /* Running port? */ - struct ether_addr mac_addr; /* MAC address. */ - unsigned nb_rx_queues; /* Configured tx queues. */ - unsigned nb_tx_queues; /* Configured rx queues. */ - int first_bucket; /* mPIPE bucket start index. */ - int first_ring; /* mPIPE notif ring start index. */ - int notif_group; /* mPIPE notif group. */ - rte_atomic32_t dp_count __rte_cache_aligned; /* DP Entry count. */ - int tx_stat_mapping[RTE_ETHDEV_QUEUE_STAT_CNTRS]; - int rx_stat_mapping[RTE_ETHDEV_QUEUE_STAT_CNTRS]; -}; - -#define mpipe_priv(dev) \ - ((struct mpipe_dev_priv*)(dev)->data->dev_private) - -#define mpipe_name(priv) \ - ((priv)->eth_dev->data->name) - -#define mpipe_rx_queue(priv, n) \ - ((struct mpipe_rx_queue *)(priv)->eth_dev->data->rx_queues[n]) - -#define mpipe_tx_queue(priv, n) \ - ((struct mpipe_tx_queue *)(priv)->eth_dev->data->tx_queues[n]) - -static void -mpipe_xmit_flush(struct mpipe_dev_priv *priv); - -static void -mpipe_recv_flush(struct mpipe_dev_priv *priv); - -static int mpipe_equeue_sizes[] = { - [GXIO_MPIPE_EQUEUE_ENTRY_512] = 512, - [GXIO_MPIPE_EQUEUE_ENTRY_2K] = 2048, - [GXIO_MPIPE_EQUEUE_ENTRY_8K] = 8192, - [GXIO_MPIPE_EQUEUE_ENTRY_64K] = 65536, -}; - -static int mpipe_iqueue_sizes[] = { - [GXIO_MPIPE_IQUEUE_ENTRY_128] = 128, - [GXIO_MPIPE_IQUEUE_ENTRY_512] = 512, - [GXIO_MPIPE_IQUEUE_ENTRY_2K] = 2048, - [GXIO_MPIPE_IQUEUE_ENTRY_64K] = 65536, -}; - -static int mpipe_buffer_sizes[] = { - [GXIO_MPIPE_BUFFER_SIZE_128] = 128, - [GXIO_MPIPE_BUFFER_SIZE_256] = 256, - [GXIO_MPIPE_BUFFER_SIZE_512] = 512, - [GXIO_MPIPE_BUFFER_SIZE_1024] = 1024, - [GXIO_MPIPE_BUFFER_SIZE_1664] = 1664, - [GXIO_MPIPE_BUFFER_SIZE_4096] = 4096, - [GXIO_MPIPE_BUFFER_SIZE_10368] = 10368, - [GXIO_MPIPE_BUFFER_SIZE_16384] = 16384, -}; - -static gxio_mpipe_context_t * -mpipe_context(int instance) -{ - if (instance < 0 || instance >= mpipe_instances) - return NULL; - return &mpipe_contexts[instance].context; -} - -static int mpipe_channel_config(int instance, int channel, - struct mpipe_channel_config *config) -{ - struct mpipe_channel_config *data; - struct mpipe_context *context; - gxio_mpipe_rules_t rules; - int idx, rc = 0; - - if (instance < 0 || instance >= mpipe_instances || - channel < 0 || channel >= MPIPE_MAX_CHANNELS) - return -EINVAL; - - context = &mpipe_contexts[instance]; - - rte_spinlock_lock(&context->lock); - - gxio_mpipe_rules_init(&rules, &context->context); - - for (idx = 0; idx < MPIPE_MAX_CHANNELS; idx++) { - data = (channel == idx) ? config : &context->channels[idx]; - - if (!data->enable) - continue; - - rc = gxio_mpipe_rules_begin(&rules, data->first_bucket, - data->num_buckets, &data->stacks); - if (rc < 0) { - goto done; - } - - rc = gxio_mpipe_rules_add_channel(&rules, idx); - if (rc < 0) { - goto done; - } - - rc = gxio_mpipe_rules_set_headroom(&rules, data->head_room); - if (rc < 0) { - goto done; - } - } - - rc = gxio_mpipe_rules_commit(&rules); - if (rc == 0) { - memcpy(&context->channels[channel], config, sizeof(*config)); - } - -done: - rte_spinlock_unlock(&context->lock); - - return rc; -} - -static int -mpipe_get_size_index(int *array, int count, int size, - bool roundup) -{ - int i, last = -1; - - for (i = 0; i < count && array[i] < size; i++) { - if (array[i]) - last = i; - } - - if (roundup) - return i < count ? (int)i : -ENOENT; - else - return last >= 0 ? last : -ENOENT; -} - -static int -mpipe_calc_size(int *array, int count, int size) -{ - int index = mpipe_get_size_index(array, count, size, 1); - return index < 0 ? index : array[index]; -} - -static int mpipe_equeue_size(int size) -{ - int result; - result = mpipe_calc_size(mpipe_equeue_sizes, - RTE_DIM(mpipe_equeue_sizes), size); - return result; -} - -static int mpipe_iqueue_size(int size) -{ - int result; - result = mpipe_calc_size(mpipe_iqueue_sizes, - RTE_DIM(mpipe_iqueue_sizes), size); - return result; -} - -static int mpipe_buffer_size_index(int size) -{ - int result; - result = mpipe_get_size_index(mpipe_buffer_sizes, - RTE_DIM(mpipe_buffer_sizes), size, 0); - return result; -} - -static inline int -mpipe_dev_atomic_read_link_status(struct rte_eth_dev *dev, - struct rte_eth_link *link) -{ - struct rte_eth_link *dst = link; - struct rte_eth_link *src = &(dev->data->dev_link); - - if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst, - *(uint64_t *)src) == 0) - return -1; - - return 0; -} - -static inline int -mpipe_dev_atomic_write_link_status(struct rte_eth_dev *dev, - struct rte_eth_link *link) -{ - struct rte_eth_link *dst = &(dev->data->dev_link); - struct rte_eth_link *src = link; - - if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst, - *(uint64_t *)src) == 0) - return -1; - - return 0; -} - -static void -mpipe_infos_get(struct rte_eth_dev *dev __rte_unused, - struct rte_eth_dev_info *dev_info) -{ - dev_info->min_rx_bufsize = 128; - dev_info->max_rx_pktlen = 1518; - dev_info->max_tx_queues = MPIPE_TX_MAX_QUEUES; - dev_info->max_rx_queues = MPIPE_RX_MAX_QUEUES; - dev_info->max_mac_addrs = 1; - dev_info->rx_offload_capa = 0; - dev_info->tx_offload_capa = 0; -} - -static int -mpipe_configure(struct rte_eth_dev *dev) -{ - struct mpipe_dev_priv *priv = mpipe_priv(dev); - - if (dev->data->nb_tx_queues > MPIPE_TX_MAX_QUEUES) { - RTE_LOG(ERR, PMD, "%s: Too many tx queues: %d > %d\n", - mpipe_name(priv), dev->data->nb_tx_queues, - MPIPE_TX_MAX_QUEUES); - return -EINVAL; - } - priv->nb_tx_queues = dev->data->nb_tx_queues; - - if (dev->data->nb_rx_queues > MPIPE_RX_MAX_QUEUES) { - RTE_LOG(ERR, PMD, "%s: Too many rx queues: %d > %d\n", - mpipe_name(priv), dev->data->nb_rx_queues, - MPIPE_RX_MAX_QUEUES); - } - priv->nb_rx_queues = dev->data->nb_rx_queues; - - return 0; -} - -static inline int -mpipe_link_compare(struct rte_eth_link *link1, - struct rte_eth_link *link2) -{ - return (*(uint64_t *)link1 == *(uint64_t *)link2) - ? -1 : 0; -} - -static int -mpipe_link_update(struct rte_eth_dev *dev, int wait_to_complete) -{ - struct mpipe_dev_priv *priv = mpipe_priv(dev); - struct rte_eth_link old, new; - int64_t state, speed; - int count, rc; - - memset(&old, 0, sizeof(old)); - memset(&new, 0, sizeof(new)); - mpipe_dev_atomic_read_link_status(dev, &old); - - for (count = 0, rc = 0; count < MPIPE_LINK_UPDATE_TIMEOUT; count++) { - if (!priv->initialized) - break; - - state = gxio_mpipe_link_get_attr(&priv->link, - GXIO_MPIPE_LINK_CURRENT_STATE); - if (state < 0) - break; - - speed = state & GXIO_MPIPE_LINK_SPEED_MASK; - - new.link_autoneg = (dev->data->dev_conf.link_speeds & - ETH_LINK_SPEED_AUTONEG); - if (speed == GXIO_MPIPE_LINK_1G) { - new.link_speed = ETH_SPEED_NUM_1G; - new.link_duplex = ETH_LINK_FULL_DUPLEX; - new.link_status = ETH_LINK_UP; - } else if (speed == GXIO_MPIPE_LINK_10G) { - new.link_speed = ETH_SPEED_NUM_10G; - new.link_duplex = ETH_LINK_FULL_DUPLEX; - new.link_status = ETH_LINK_UP; - } - - rc = mpipe_link_compare(&old, &new); - if (rc == 0 || !wait_to_complete) - break; - - rte_delay_us(MPIPE_LINK_UPDATE_INTERVAL); - } - - mpipe_dev_atomic_write_link_status(dev, &new); - return rc; -} - -static int -mpipe_set_link(struct rte_eth_dev *dev, int up) -{ - struct mpipe_dev_priv *priv = mpipe_priv(dev); - int rc; - - rc = gxio_mpipe_link_set_attr(&priv->link, - GXIO_MPIPE_LINK_DESIRED_STATE, - up ? GXIO_MPIPE_LINK_ANYSPEED : 0); - if (rc < 0) { - RTE_LOG(ERR, PMD, "%s: Failed to set link %s.\n", - mpipe_name(priv), up ? "up" : "down"); - } else { - mpipe_link_update(dev, 0); - } - - return rc; -} - -static int -mpipe_set_link_up(struct rte_eth_dev *dev) -{ - return mpipe_set_link(dev, 1); -} - -static int -mpipe_set_link_down(struct rte_eth_dev *dev) -{ - return mpipe_set_link(dev, 0); -} - -static inline void -mpipe_dp_enter(struct mpipe_dev_priv *priv) -{ - __insn_mtspr(SPR_DSTREAM_PF, 0); - rte_atomic32_inc(&priv->dp_count); -} - -static inline void -mpipe_dp_exit(struct mpipe_dev_priv *priv) -{ - rte_atomic32_dec(&priv->dp_count); -} - -static inline void -mpipe_dp_wait(struct mpipe_dev_priv *priv) -{ - while (rte_atomic32_read(&priv->dp_count) != 0) { - rte_pause(); - } -} - -static inline int -mpipe_mbuf_stack_index(struct mpipe_dev_priv *priv, struct rte_mbuf *mbuf) -{ - return (mbuf->port < RTE_MAX_ETHPORTS) ? - mpipe_priv(&rte_eth_devices[mbuf->port])->stack : - priv->stack; -} - -static inline struct rte_mbuf * -mpipe_recv_mbuf(struct mpipe_dev_priv *priv, gxio_mpipe_idesc_t *idesc, - int in_port) -{ - void *va = gxio_mpipe_idesc_get_va(idesc); - uint16_t size = gxio_mpipe_idesc_get_xfer_size(idesc); - struct rte_mbuf *mbuf = RTE_PTR_SUB(va, priv->rx_offset); - - rte_pktmbuf_reset(mbuf); - mbuf->data_off = (uintptr_t)va - (uintptr_t)mbuf->buf_addr; - mbuf->port = in_port; - mbuf->data_len = size; - mbuf->pkt_len = size; - mbuf->hash.rss = gxio_mpipe_idesc_get_flow_hash(idesc); - - PMD_DEBUG_RX("%s: RX mbuf %p, buffer %p, buf_addr %p, size %d\n", - mpipe_name(priv), mbuf, va, mbuf->buf_addr, size); - - return mbuf; -} - -static inline void -mpipe_recv_push(struct mpipe_dev_priv *priv, struct rte_mbuf *mbuf) -{ - const int offset = RTE_PKTMBUF_HEADROOM + MPIPE_RX_IP_ALIGN; - void *buf_addr = RTE_PTR_ADD(mbuf->buf_addr, offset); - - gxio_mpipe_push_buffer(priv->context, priv->stack, buf_addr); - PMD_DEBUG_RX("%s: Pushed mbuf %p, buffer %p into stack %d\n", - mpipe_name(priv), mbuf, buf_addr, priv->stack); -} - -static inline void -mpipe_recv_fill_stack(struct mpipe_dev_priv *priv, int count) -{ - struct rte_mbuf *mbuf; - int i; - - for (i = 0; i < count; i++) { - mbuf = rte_mbuf_raw_alloc(priv->rx_mpool); - if (!mbuf) - break; - mpipe_recv_push(priv, mbuf); - } - - PMD_DEBUG_RX("%s: Filled %d/%d buffers\n", mpipe_name(priv), i, count); -} - -static inline void -mpipe_recv_flush_stack(struct mpipe_dev_priv *priv) -{ - const int offset = priv->rx_offset & ~RTE_MEMPOOL_ALIGN_MASK; - uint8_t in_port = priv->port_id; - struct rte_mbuf *mbuf; - void *va; - - while (1) { - va = gxio_mpipe_pop_buffer(priv->context, priv->stack); - if (!va) - break; - mbuf = RTE_PTR_SUB(va, offset); - - PMD_DEBUG_RX("%s: Flushing mbuf %p, va %p\n", - mpipe_name(priv), mbuf, va); - - mbuf->data_off = (uintptr_t)va - (uintptr_t)mbuf->buf_addr; - mbuf->refcnt = 1; - mbuf->nb_segs = 1; - mbuf->port = in_port; - mbuf->packet_type = 0; - mbuf->data_len = 0; - mbuf->pkt_len = 0; - - __rte_mbuf_raw_free(mbuf); - } -} - -static void -mpipe_register_segment(struct mpipe_dev_priv *priv, const struct rte_memseg *ms) -{ - size_t size = ms->hugepage_sz; - uint8_t *addr, *end; - int rc; - - for (addr = ms->addr, end = addr + ms->len; addr < end; addr += size) { - rc = gxio_mpipe_register_page(priv->context, priv->stack, addr, - size, 0); - if (rc < 0) - break; - } - - if (rc < 0) { - RTE_LOG(ERR, PMD, "%s: Could not register memseg @%p, %d.\n", - mpipe_name(priv), ms->addr, rc); - } else { - RTE_LOG(DEBUG, PMD, "%s: Registered segment %p - %p\n", - mpipe_name(priv), ms->addr, - RTE_PTR_ADD(ms->addr, ms->len - 1)); - } -} - -static int -mpipe_recv_init(struct mpipe_dev_priv *priv) -{ - const struct rte_memseg *seg = rte_eal_get_physmem_layout(); - size_t stack_size; - void *stack_mem; - int rc; - - if (!priv->rx_mpool) { - RTE_LOG(ERR, PMD, "%s: No buffer pool.\n", - mpipe_name(priv)); - return -ENODEV; - } - - /* Allocate one NotifRing for each queue. */ - rc = gxio_mpipe_alloc_notif_rings(priv->context, MPIPE_RX_MAX_QUEUES, - 0, 0); - if (rc < 0) { - RTE_LOG(ERR, PMD, "%s: Failed to allocate notif rings.\n", - mpipe_name(priv)); - return rc; - } - priv->first_ring = rc; - - /* Allocate a NotifGroup. */ - rc = gxio_mpipe_alloc_notif_groups(priv->context, 1, 0, 0); - if (rc < 0) { - RTE_LOG(ERR, PMD, "%s: Failed to allocate rx group.\n", - mpipe_name(priv)); - return rc; - } - priv->notif_group = rc; - - /* Allocate required buckets. */ - rc = gxio_mpipe_alloc_buckets(priv->context, MPIPE_RX_BUCKETS, 0, 0); - if (rc < 0) { - RTE_LOG(ERR, PMD, "%s: Failed to allocate buckets.\n", - mpipe_name(priv)); - return rc; - } - priv->first_bucket = rc; - - rc = gxio_mpipe_alloc_buffer_stacks(priv->context, 1, 0, 0); - if (rc < 0) { - RTE_LOG(ERR, PMD, "%s: Failed to allocate buffer stack.\n", - mpipe_name(priv)); - return rc; - } - priv->stack = rc; - - while (seg && seg->addr) - mpipe_register_segment(priv, seg++); - - stack_size = gxio_mpipe_calc_buffer_stack_bytes(MPIPE_RX_STACK_SIZE); - stack_mem = rte_zmalloc(NULL, stack_size, 65536); - if (!stack_mem) { - RTE_LOG(ERR, PMD, "%s: Failed to allocate buffer memory.\n", - mpipe_name(priv)); - return -ENOMEM; - } else { - RTE_LOG(DEBUG, PMD, "%s: Buffer stack memory %p - %p.\n", - mpipe_name(priv), stack_mem, - RTE_PTR_ADD(stack_mem, stack_size - 1)); - } - - rc = gxio_mpipe_init_buffer_stack(priv->context, priv->stack, - priv->rx_size_code, stack_mem, - stack_size, 0); - if (rc < 0) { - RTE_LOG(ERR, PMD, "%s: Failed to initialize buffer stack.\n", - mpipe_name(priv)); - return rc; - } - - return 0; -} - -static int -mpipe_xmit_init(struct mpipe_dev_priv *priv) -{ - size_t ring_size; - void *ring_mem; - int rc; - - /* Allocate eDMA ring. */ - rc = gxio_mpipe_alloc_edma_rings(priv->context, 1, 0, 0); - if (rc < 0) { - RTE_LOG(ERR, PMD, "%s: Failed to alloc tx ring.\n", - mpipe_name(priv)); - return rc; - } - priv->ering = rc; - - rc = mpipe_equeue_size(MPIPE_TX_DESCS); - if (rc < 0) { - RTE_LOG(ERR, PMD, "%s: Cannot allocate %d equeue descs.\n", - mpipe_name(priv), (int)MPIPE_TX_DESCS); - return -ENOMEM; - } - priv->equeue_size = rc; - - /* Initialize completion array. */ - ring_size = sizeof(priv->tx_comps[0]) * priv->equeue_size; - priv->tx_comps = rte_zmalloc(NULL, ring_size, RTE_CACHE_LINE_SIZE); - if (!priv->tx_comps) { - RTE_LOG(ERR, PMD, "%s: Failed to allocate egress comps.\n", - mpipe_name(priv)); - return -ENOMEM; - } - - /* Allocate eDMA ring memory. */ - ring_size = sizeof(gxio_mpipe_edesc_t) * priv->equeue_size; - ring_mem = rte_zmalloc(NULL, ring_size, ring_size); - if (!ring_mem) { - RTE_LOG(ERR, PMD, "%s: Failed to allocate egress descs.\n", - mpipe_name(priv)); - return -ENOMEM; - } else { - RTE_LOG(DEBUG, PMD, "%s: eDMA ring memory %p - %p.\n", - mpipe_name(priv), ring_mem, - RTE_PTR_ADD(ring_mem, ring_size - 1)); - } - - /* Initialize eDMA ring. */ - rc = gxio_mpipe_equeue_init(&priv->equeue, priv->context, priv->ering, - priv->channel, ring_mem, ring_size, 0); - if (rc < 0) { - RTE_LOG(ERR, PMD, "%s: Failed to init equeue\n", - mpipe_name(priv)); - return rc; - } - - return 0; -} - -static int -mpipe_link_init(struct mpipe_dev_priv *priv) -{ - int rc; - - /* Open the link. */ - rc = gxio_mpipe_link_open(&priv->link, priv->context, - mpipe_name(priv), GXIO_MPIPE_LINK_AUTO_NONE); - if (rc < 0) { - RTE_LOG(ERR, PMD, "%s: Failed to open link.\n", - mpipe_name(priv)); - return rc; - } - - /* Get the channel index. */ - rc = gxio_mpipe_link_channel(&priv->link); - if (rc < 0) { - RTE_LOG(ERR, PMD, "%s: Bad channel\n", - mpipe_name(priv)); - return rc; - } - priv->channel = rc; - - return 0; -} - -static int -mpipe_init(struct mpipe_dev_priv *priv) -{ - int rc; - - if (priv->initialized) - return 0; - - rc = mpipe_recv_init(priv); - if (rc < 0) { - RTE_LOG(ERR, PMD, "%s: Failed to init rx.\n", - mpipe_name(priv)); - return rc; - } - - rc = mpipe_xmit_init(priv); - if (rc < 0) { - RTE_LOG(ERR, PMD, "%s: Failed to init tx.\n", - mpipe_name(priv)); - rte_free(priv); - return rc; - } - - priv->initialized = 1; - - return 0; -} - -static int -mpipe_start(struct rte_eth_dev *dev) -{ - struct mpipe_dev_priv *priv = mpipe_priv(dev); - struct mpipe_channel_config config; - struct mpipe_rx_queue *rx_queue; - struct rte_eth_link eth_link; - unsigned queue, buffers = 0; - size_t ring_size; - void *ring_mem; - int rc; - - memset(ð_link, 0, sizeof(eth_link)); - mpipe_dev_atomic_write_link_status(dev, ð_link); - - rc = mpipe_init(priv); - if (rc < 0) - return rc; - - /* Initialize NotifRings. */ - for (queue = 0; queue < priv->nb_rx_queues; queue++) { - rx_queue = mpipe_rx_queue(priv, queue); - ring_size = rx_queue->q.nb_desc * sizeof(gxio_mpipe_idesc_t); - - ring_mem = rte_malloc(NULL, ring_size, ring_size); - if (!ring_mem) { - RTE_LOG(ERR, PMD, "%s: Failed to alloc rx descs.\n", - mpipe_name(priv)); - return -ENOMEM; - } else { - RTE_LOG(DEBUG, PMD, "%s: iDMA ring %d memory %p - %p.\n", - mpipe_name(priv), queue, ring_mem, - RTE_PTR_ADD(ring_mem, ring_size - 1)); - } - - rc = gxio_mpipe_iqueue_init(&rx_queue->iqueue, priv->context, - priv->first_ring + queue, ring_mem, - ring_size, 0); - if (rc < 0) { - RTE_LOG(ERR, PMD, "%s: Failed to init rx queue.\n", - mpipe_name(priv)); - return rc; - } - - rx_queue->rx_ring_mem = ring_mem; - buffers += rx_queue->q.nb_desc; - } - - /* Initialize ingress NotifGroup and buckets. */ - rc = gxio_mpipe_init_notif_group_and_buckets(priv->context, - priv->notif_group, priv->first_ring, priv->nb_rx_queues, - priv->first_bucket, MPIPE_RX_BUCKETS, - GXIO_MPIPE_BUCKET_STATIC_FLOW_AFFINITY); - if (rc < 0) { - RTE_LOG(ERR, PMD, "%s: Failed to init group and buckets.\n", - mpipe_name(priv)); - return rc; - } - - /* Configure the classifier to deliver packets from this port. */ - config.enable = 1; - config.first_bucket = priv->first_bucket; - config.num_buckets = MPIPE_RX_BUCKETS; - memset(&config.stacks, 0xff, sizeof(config.stacks)); - config.stacks.stacks[priv->rx_size_code] = priv->stack; - config.head_room = priv->rx_offset & RTE_MEMPOOL_ALIGN_MASK; - - rc = mpipe_channel_config(priv->instance, priv->channel, - &config); - if (rc < 0) { - RTE_LOG(ERR, PMD, "%s: Failed to setup classifier.\n", - mpipe_name(priv)); - return rc; - } - - /* Fill empty buffers into the buffer stack. */ - mpipe_recv_fill_stack(priv, buffers); - - /* Bring up the link. */ - mpipe_set_link_up(dev); - - /* Start xmit/recv on queues. */ - for (queue = 0; queue < priv->nb_tx_queues; queue++) - mpipe_tx_queue(priv, queue)->q.link_status = ETH_LINK_UP; - for (queue = 0; queue < priv->nb_rx_queues; queue++) - mpipe_rx_queue(priv, queue)->q.link_status = ETH_LINK_UP; - priv->running = 1; - - return 0; -} - -static void -mpipe_stop(struct rte_eth_dev *dev) -{ - struct mpipe_dev_priv *priv = mpipe_priv(dev); - struct mpipe_channel_config config; - unsigned queue; - int rc; - - for (queue = 0; queue < priv->nb_tx_queues; queue++) - mpipe_tx_queue(priv, queue)->q.link_status = ETH_LINK_DOWN; - for (queue = 0; queue < priv->nb_rx_queues; queue++) - mpipe_rx_queue(priv, queue)->q.link_status = ETH_LINK_DOWN; - - /* Make sure the link_status writes land. */ - rte_wmb(); - - /* - * Wait for link_status change to register with straggling datapath - * threads. - */ - mpipe_dp_wait(priv); - - /* Bring down the link. */ - mpipe_set_link_down(dev); - - /* Remove classifier rules. */ - memset(&config, 0, sizeof(config)); - rc = mpipe_channel_config(priv->instance, priv->channel, - &config); - if (rc < 0) { - RTE_LOG(ERR, PMD, "%s: Failed to stop classifier.\n", - mpipe_name(priv)); - } - - /* Flush completed xmit packets. */ - mpipe_xmit_flush(priv); - - /* Flush buffer stacks. */ - mpipe_recv_flush(priv); - - priv->running = 0; -} - -static void -mpipe_close(struct rte_eth_dev *dev) -{ - struct mpipe_dev_priv *priv = mpipe_priv(dev); - if (priv->running) - mpipe_stop(dev); -} - -static void -mpipe_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) -{ - struct mpipe_dev_priv *priv = mpipe_priv(dev); - struct mpipe_tx_queue *tx_queue; - struct mpipe_rx_queue *rx_queue; - unsigned i; - uint16_t idx; - - memset(stats, 0, sizeof(*stats)); - - for (i = 0; i < priv->nb_tx_queues; i++) { - tx_queue = mpipe_tx_queue(priv, i); - - stats->opackets += tx_queue->q.stats.packets; - stats->obytes += tx_queue->q.stats.bytes; - stats->oerrors += tx_queue->q.stats.errors; - - idx = tx_queue->q.stat_idx; - if (idx != (uint16_t)-1) { - stats->q_opackets[idx] += tx_queue->q.stats.packets; - stats->q_obytes[idx] += tx_queue->q.stats.bytes; - stats->q_errors[idx] += tx_queue->q.stats.errors; - } - } - - for (i = 0; i < priv->nb_rx_queues; i++) { - rx_queue = mpipe_rx_queue(priv, i); - - stats->ipackets += rx_queue->q.stats.packets; - stats->ibytes += rx_queue->q.stats.bytes; - stats->ierrors += rx_queue->q.stats.errors; - stats->rx_nombuf += rx_queue->q.stats.nomem; - - idx = rx_queue->q.stat_idx; - if (idx != (uint16_t)-1) { - stats->q_ipackets[idx] += rx_queue->q.stats.packets; - stats->q_ibytes[idx] += rx_queue->q.stats.bytes; - stats->q_errors[idx] += rx_queue->q.stats.errors; - } - } -} - -static void -mpipe_stats_reset(struct rte_eth_dev *dev) -{ - struct mpipe_dev_priv *priv = mpipe_priv(dev); - struct mpipe_tx_queue *tx_queue; - struct mpipe_rx_queue *rx_queue; - unsigned i; - - for (i = 0; i < priv->nb_tx_queues; i++) { - tx_queue = mpipe_tx_queue(priv, i); - memset(&tx_queue->q.stats, 0, sizeof(tx_queue->q.stats)); - } - - for (i = 0; i < priv->nb_rx_queues; i++) { - rx_queue = mpipe_rx_queue(priv, i); - memset(&rx_queue->q.stats, 0, sizeof(rx_queue->q.stats)); - } -} - -static int -mpipe_queue_stats_mapping_set(struct rte_eth_dev *dev, uint16_t queue_id, - uint8_t stat_idx, uint8_t is_rx) -{ - struct mpipe_dev_priv *priv = mpipe_priv(dev); - - if (is_rx) { - priv->rx_stat_mapping[stat_idx] = queue_id; - } else { - priv->tx_stat_mapping[stat_idx] = queue_id; - } - - return 0; -} - -static int -mpipe_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, - uint16_t nb_desc, unsigned int socket_id __rte_unused, - const struct rte_eth_txconf *tx_conf __rte_unused) -{ - struct mpipe_tx_queue *tx_queue = dev->data->tx_queues[queue_idx]; - struct mpipe_dev_priv *priv = mpipe_priv(dev); - uint16_t idx; - - tx_queue = rte_realloc(tx_queue, sizeof(*tx_queue), - RTE_CACHE_LINE_SIZE); - if (!tx_queue) { - RTE_LOG(ERR, PMD, "%s: Failed to allocate TX queue.\n", - mpipe_name(priv)); - return -ENOMEM; - } - - memset(&tx_queue->q, 0, sizeof(tx_queue->q)); - tx_queue->q.priv = priv; - tx_queue->q.queue_idx = queue_idx; - tx_queue->q.port_id = dev->data->port_id; - tx_queue->q.nb_desc = nb_desc; - - tx_queue->q.stat_idx = -1; - for (idx = 0; idx < RTE_ETHDEV_QUEUE_STAT_CNTRS; idx++) { - if (priv->tx_stat_mapping[idx] == queue_idx) - tx_queue->q.stat_idx = idx; - } - - dev->data->tx_queues[queue_idx] = tx_queue; - - return 0; -} - -static void -mpipe_tx_queue_release(void *_txq) -{ - rte_free(_txq); -} - -static int -mpipe_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, - uint16_t nb_desc, unsigned int socket_id __rte_unused, - const struct rte_eth_rxconf *rx_conf __rte_unused, - struct rte_mempool *mp) -{ - struct mpipe_rx_queue *rx_queue = dev->data->rx_queues[queue_idx]; - struct mpipe_dev_priv *priv = mpipe_priv(dev); - uint16_t idx; - int size, rc; - - rc = mpipe_iqueue_size(nb_desc); - if (rc < 0) { - RTE_LOG(ERR, PMD, "%s: Cannot allocate %d iqueue descs.\n", - mpipe_name(priv), (int)nb_desc); - return -ENOMEM; - } - - if (rc != nb_desc) { - RTE_LOG(WARNING, PMD, "%s: Extending RX descs from %d to %d.\n", - mpipe_name(priv), (int)nb_desc, rc); - nb_desc = rc; - } - - size = sizeof(*rx_queue); - rx_queue = rte_realloc(rx_queue, size, RTE_CACHE_LINE_SIZE); - if (!rx_queue) { - RTE_LOG(ERR, PMD, "%s: Failed to allocate RX queue.\n", - mpipe_name(priv)); - return -ENOMEM; - } - - memset(&rx_queue->q, 0, sizeof(rx_queue->q)); - rx_queue->q.priv = priv; - rx_queue->q.nb_desc = nb_desc; - rx_queue->q.port_id = dev->data->port_id; - rx_queue->q.queue_idx = queue_idx; - - if (!priv->rx_mpool) { - int size = (rte_pktmbuf_data_room_size(mp) - - RTE_PKTMBUF_HEADROOM - - MPIPE_RX_IP_ALIGN); - - priv->rx_offset = (sizeof(struct rte_mbuf) + - rte_pktmbuf_priv_size(mp) + - RTE_PKTMBUF_HEADROOM + - MPIPE_RX_IP_ALIGN); - if (size < 0) { - RTE_LOG(ERR, PMD, "%s: Bad buffer size %d.\n", - mpipe_name(priv), - rte_pktmbuf_data_room_size(mp)); - return -ENOMEM; - } - - priv->rx_size_code = mpipe_buffer_size_index(size); - priv->rx_mpool = mp; - } - - if (priv->rx_mpool != mp) { - RTE_LOG(WARNING, PMD, "%s: Ignoring multiple buffer pools.\n", - mpipe_name(priv)); - } - - rx_queue->q.stat_idx = -1; - for (idx = 0; idx < RTE_ETHDEV_QUEUE_STAT_CNTRS; idx++) { - if (priv->rx_stat_mapping[idx] == queue_idx) - rx_queue->q.stat_idx = idx; - } - - dev->data->rx_queues[queue_idx] = rx_queue; - - return 0; -} - -static void -mpipe_rx_queue_release(void *_rxq) -{ - rte_free(_rxq); -} - -#define MPIPE_XGBE_ENA_HASH_MULTI \ - (1UL << MPIPE_XAUI_RECEIVE_CONFIGURATION__ENA_HASH_MULTI_SHIFT) -#define MPIPE_XGBE_ENA_HASH_UNI \ - (1UL << MPIPE_XAUI_RECEIVE_CONFIGURATION__ENA_HASH_UNI_SHIFT) -#define MPIPE_XGBE_COPY_ALL \ - (1UL << MPIPE_XAUI_RECEIVE_CONFIGURATION__COPY_ALL_SHIFT) -#define MPIPE_GBE_ENA_MULTI_HASH \ - (1UL << MPIPE_GBE_NETWORK_CONFIGURATION__MULTI_HASH_ENA_SHIFT) -#define MPIPE_GBE_ENA_UNI_HASH \ - (1UL << MPIPE_GBE_NETWORK_CONFIGURATION__UNI_HASH_ENA_SHIFT) -#define MPIPE_GBE_COPY_ALL \ - (1UL << MPIPE_GBE_NETWORK_CONFIGURATION__COPY_ALL_SHIFT) - -static void -mpipe_promiscuous_enable(struct rte_eth_dev *dev) -{ - struct mpipe_dev_priv *priv = mpipe_priv(dev); - int64_t reg; - int addr; - - if (priv->is_xaui) { - addr = MPIPE_XAUI_RECEIVE_CONFIGURATION; - reg = gxio_mpipe_link_mac_rd(&priv->link, addr); - reg &= ~MPIPE_XGBE_ENA_HASH_MULTI; - reg &= ~MPIPE_XGBE_ENA_HASH_UNI; - reg |= MPIPE_XGBE_COPY_ALL; - gxio_mpipe_link_mac_wr(&priv->link, addr, reg); - } else { - addr = MPIPE_GBE_NETWORK_CONFIGURATION; - reg = gxio_mpipe_link_mac_rd(&priv->link, addr); - reg &= ~MPIPE_GBE_ENA_MULTI_HASH; - reg &= ~MPIPE_GBE_ENA_UNI_HASH; - reg |= MPIPE_GBE_COPY_ALL; - gxio_mpipe_link_mac_wr(&priv->link, addr, reg); - } -} - -static void -mpipe_promiscuous_disable(struct rte_eth_dev *dev) -{ - struct mpipe_dev_priv *priv = mpipe_priv(dev); - int64_t reg; - int addr; - - if (priv->is_xaui) { - addr = MPIPE_XAUI_RECEIVE_CONFIGURATION; - reg = gxio_mpipe_link_mac_rd(&priv->link, addr); - reg |= MPIPE_XGBE_ENA_HASH_MULTI; - reg |= MPIPE_XGBE_ENA_HASH_UNI; - reg &= ~MPIPE_XGBE_COPY_ALL; - gxio_mpipe_link_mac_wr(&priv->link, addr, reg); - } else { - addr = MPIPE_GBE_NETWORK_CONFIGURATION; - reg = gxio_mpipe_link_mac_rd(&priv->link, addr); - reg |= MPIPE_GBE_ENA_MULTI_HASH; - reg |= MPIPE_GBE_ENA_UNI_HASH; - reg &= ~MPIPE_GBE_COPY_ALL; - gxio_mpipe_link_mac_wr(&priv->link, addr, reg); - } -} - -static const struct eth_dev_ops mpipe_dev_ops = { - .dev_infos_get = mpipe_infos_get, - .dev_configure = mpipe_configure, - .dev_start = mpipe_start, - .dev_stop = mpipe_stop, - .dev_close = mpipe_close, - .stats_get = mpipe_stats_get, - .stats_reset = mpipe_stats_reset, - .queue_stats_mapping_set = mpipe_queue_stats_mapping_set, - .tx_queue_setup = mpipe_tx_queue_setup, - .rx_queue_setup = mpipe_rx_queue_setup, - .tx_queue_release = mpipe_tx_queue_release, - .rx_queue_release = mpipe_rx_queue_release, - .link_update = mpipe_link_update, - .dev_set_link_up = mpipe_set_link_up, - .dev_set_link_down = mpipe_set_link_down, - .promiscuous_enable = mpipe_promiscuous_enable, - .promiscuous_disable = mpipe_promiscuous_disable, -}; - -static inline void -mpipe_xmit_null(struct mpipe_dev_priv *priv, int64_t start, int64_t end) -{ - gxio_mpipe_edesc_t null_desc = { { .bound = 1, .ns = 1 } }; - gxio_mpipe_equeue_t *equeue = &priv->equeue; - int64_t slot; - - for (slot = start; slot < end; slot++) { - gxio_mpipe_equeue_put_at(equeue, null_desc, slot); - } -} - -static void -mpipe_xmit_flush(struct mpipe_dev_priv *priv) -{ - gxio_mpipe_equeue_t *equeue = &priv->equeue; - int64_t slot; - - /* Post a dummy descriptor and wait for its return. */ - slot = gxio_mpipe_equeue_reserve(equeue, 1); - if (slot < 0) { - RTE_LOG(ERR, PMD, "%s: Failed to reserve stop slot.\n", - mpipe_name(priv)); - return; - } - - mpipe_xmit_null(priv, slot, slot + 1); - - while (!gxio_mpipe_equeue_is_complete(equeue, slot, 1)) { - rte_pause(); - } - - for (slot = 0; slot < priv->equeue_size; slot++) { - if (priv->tx_comps[slot]) - rte_pktmbuf_free_seg(priv->tx_comps[slot]); - } -} - -static void -mpipe_recv_flush(struct mpipe_dev_priv *priv) -{ - uint8_t in_port = priv->port_id; - struct mpipe_rx_queue *rx_queue; - gxio_mpipe_iqueue_t *iqueue; - gxio_mpipe_idesc_t idesc; - struct rte_mbuf *mbuf; - unsigned queue; - - /* Release packets on the buffer stack. */ - mpipe_recv_flush_stack(priv); - - /* Flush packets sitting in recv queues. */ - for (queue = 0; queue < priv->nb_rx_queues; queue++) { - rx_queue = mpipe_rx_queue(priv, queue); - iqueue = &rx_queue->iqueue; - while (gxio_mpipe_iqueue_try_get(iqueue, &idesc) >= 0) { - /* Skip idesc with the 'buffer error' bit set. */ - if (idesc.be) - continue; - mbuf = mpipe_recv_mbuf(priv, &idesc, in_port); - rte_pktmbuf_free(mbuf); - } - rte_free(rx_queue->rx_ring_mem); - } -} - -static inline uint16_t -mpipe_do_xmit(struct mpipe_tx_queue *tx_queue, struct rte_mbuf **tx_pkts, - uint16_t nb_pkts) -{ - struct mpipe_dev_priv *priv = tx_queue->q.priv; - gxio_mpipe_equeue_t *equeue = &priv->equeue; - unsigned nb_bytes = 0; - unsigned nb_sent = 0; - int nb_slots, i; - uint8_t port_id; - - PMD_DEBUG_TX("Trying to transmit %d packets on %s:%d.\n", - nb_pkts, mpipe_name(tx_queue->q.priv), - tx_queue->q.queue_idx); - - /* Optimistic assumption that we need exactly one slot per packet. */ - nb_slots = RTE_MIN(nb_pkts, MPIPE_TX_DESCS / 2); - - do { - struct rte_mbuf *mbuf = NULL, *pkt = NULL; - int64_t slot; - - /* Reserve eDMA ring slots. */ - slot = gxio_mpipe_equeue_try_reserve_fast(equeue, nb_slots); - if (unlikely(slot < 0)) { - break; - } - - for (i = 0; i < nb_slots; i++) { - unsigned idx = (slot + i) & (priv->equeue_size - 1); - rte_prefetch0(priv->tx_comps[idx]); - } - - /* Fill up slots with descriptor and completion info. */ - for (i = 0; i < nb_slots; i++) { - unsigned idx = (slot + i) & (priv->equeue_size - 1); - gxio_mpipe_edesc_t desc; - struct rte_mbuf *next; - - /* Starting on a new packet? */ - if (likely(!mbuf)) { - int room = nb_slots - i; - - pkt = mbuf = tx_pkts[nb_sent]; - - /* Bail out if we run out of descs. */ - if (unlikely(pkt->nb_segs > room)) - break; - - nb_sent++; - } - - /* We have a segment to send. */ - next = mbuf->next; - - if (priv->tx_comps[idx]) - rte_pktmbuf_free_seg(priv->tx_comps[idx]); - - port_id = (mbuf->port < RTE_MAX_ETHPORTS) ? - mbuf->port : priv->port_id; - desc = (gxio_mpipe_edesc_t) { { - .va = rte_pktmbuf_mtod(mbuf, uintptr_t), - .xfer_size = rte_pktmbuf_data_len(mbuf), - .bound = next ? 0 : 1, - .stack_idx = mpipe_mbuf_stack_index(priv, mbuf), - .size = priv->rx_size_code, - } }; - if (mpipe_local.mbuf_push_debt[port_id] > 0) { - mpipe_local.mbuf_push_debt[port_id]--; - desc.hwb = 1; - priv->tx_comps[idx] = NULL; - } else - priv->tx_comps[idx] = mbuf; - - nb_bytes += mbuf->data_len; - gxio_mpipe_equeue_put_at(equeue, desc, slot + i); - - PMD_DEBUG_TX("%s:%d: Sending packet %p, len %d\n", - mpipe_name(priv), - tx_queue->q.queue_idx, - rte_pktmbuf_mtod(mbuf, void *), - rte_pktmbuf_data_len(mbuf)); - - mbuf = next; - } - - if (unlikely(nb_sent < nb_pkts)) { - - /* Fill remaining slots with null descriptors. */ - mpipe_xmit_null(priv, slot + i, slot + nb_slots); - - /* - * Calculate exact number of descriptors needed for - * the next go around. - */ - nb_slots = 0; - for (i = nb_sent; i < nb_pkts; i++) { - nb_slots += tx_pkts[i]->nb_segs; - } - - nb_slots = RTE_MIN(nb_slots, MPIPE_TX_DESCS / 2); - } - } while (nb_sent < nb_pkts); - - tx_queue->q.stats.packets += nb_sent; - tx_queue->q.stats.bytes += nb_bytes; - - return nb_sent; -} - -static inline uint16_t -mpipe_do_recv(struct mpipe_rx_queue *rx_queue, struct rte_mbuf **rx_pkts, - uint16_t nb_pkts) -{ - struct mpipe_dev_priv *priv = rx_queue->q.priv; - gxio_mpipe_iqueue_t *iqueue = &rx_queue->iqueue; - gxio_mpipe_idesc_t *first_idesc, *idesc, *last_idesc; - uint8_t in_port = rx_queue->q.port_id; - const unsigned look_ahead = 8; - int room = nb_pkts, rc = 0; - unsigned nb_packets = 0; - unsigned nb_dropped = 0; - unsigned nb_nomem = 0; - unsigned nb_bytes = 0; - unsigned nb_descs, i; - - while (room && !rc) { - if (rx_queue->avail_descs < room) { - rc = gxio_mpipe_iqueue_try_peek(iqueue, - &rx_queue->next_desc); - rx_queue->avail_descs = rc < 0 ? 0 : rc; - } - - if (unlikely(!rx_queue->avail_descs)) { - break; - } - - nb_descs = RTE_MIN(room, rx_queue->avail_descs); - - first_idesc = rx_queue->next_desc; - last_idesc = first_idesc + nb_descs; - - rx_queue->next_desc += nb_descs; - rx_queue->avail_descs -= nb_descs; - - for (i = 1; i < look_ahead; i++) { - rte_prefetch0(first_idesc + i); - } - - PMD_DEBUG_RX("%s:%d: Trying to receive %d packets\n", - mpipe_name(rx_queue->q.priv), - rx_queue->q.queue_idx, - nb_descs); - - for (idesc = first_idesc; idesc < last_idesc; idesc++) { - struct rte_mbuf *mbuf; - - PMD_DEBUG_RX("%s:%d: processing idesc %d/%d\n", - mpipe_name(priv), - rx_queue->q.queue_idx, - nb_packets, nb_descs); - - rte_prefetch0(idesc + look_ahead); - - PMD_DEBUG_RX("%s:%d: idesc %p, %s%s%s%s%s%s%s%s%s%s" - "size: %d, bkt: %d, chan: %d, ring: %d, sqn: %lu, va: %lu\n", - mpipe_name(priv), - rx_queue->q.queue_idx, - idesc, - idesc->me ? "me, " : "", - idesc->tr ? "tr, " : "", - idesc->ce ? "ce, " : "", - idesc->ct ? "ct, " : "", - idesc->cs ? "cs, " : "", - idesc->nr ? "nr, " : "", - idesc->sq ? "sq, " : "", - idesc->ts ? "ts, " : "", - idesc->ps ? "ps, " : "", - idesc->be ? "be, " : "", - idesc->l2_size, - idesc->bucket_id, - idesc->channel, - idesc->notif_ring, - (unsigned long)idesc->packet_sqn, - (unsigned long)idesc->va); - - if (unlikely(gxio_mpipe_idesc_has_error(idesc))) { - nb_dropped++; - gxio_mpipe_iqueue_drop(iqueue, idesc); - PMD_DEBUG_RX("%s:%d: Descriptor error\n", - mpipe_name(rx_queue->q.priv), - rx_queue->q.queue_idx); - continue; - } - - if (mpipe_local.mbuf_push_debt[in_port] < - MPIPE_BUF_DEBT_THRESHOLD) - mpipe_local.mbuf_push_debt[in_port]++; - else { - mbuf = rte_mbuf_raw_alloc(priv->rx_mpool); - if (unlikely(!mbuf)) { - nb_nomem++; - gxio_mpipe_iqueue_drop(iqueue, idesc); - PMD_DEBUG_RX("%s:%d: alloc failure\n", - mpipe_name(rx_queue->q.priv), - rx_queue->q.queue_idx); - continue; - } - - mpipe_recv_push(priv, mbuf); - } - - /* Get and setup the mbuf for the received packet. */ - mbuf = mpipe_recv_mbuf(priv, idesc, in_port); - - /* Update results and statistics counters. */ - rx_pkts[nb_packets] = mbuf; - nb_bytes += mbuf->pkt_len; - nb_packets++; - } - - /* - * We release the ring in bursts, but do not track and release - * buckets. This therefore breaks dynamic flow affinity, but - * we always operate in static affinity mode, and so we're OK - * with this optimization. - */ - gxio_mpipe_iqueue_advance(iqueue, nb_descs); - gxio_mpipe_credit(iqueue->context, iqueue->ring, -1, nb_descs); - - /* - * Go around once more if we haven't yet peeked the queue, and - * if we have more room to receive. - */ - room = nb_pkts - nb_packets; - } - - rx_queue->q.stats.packets += nb_packets; - rx_queue->q.stats.bytes += nb_bytes; - rx_queue->q.stats.errors += nb_dropped; - rx_queue->q.stats.nomem += nb_nomem; - - PMD_DEBUG_RX("%s:%d: RX: %d/%d pkts/bytes, %d/%d drops/nomem\n", - mpipe_name(rx_queue->q.priv), rx_queue->q.queue_idx, - nb_packets, nb_bytes, nb_dropped, nb_nomem); - - return nb_packets; -} - -static uint16_t -mpipe_recv_pkts(void *_rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) -{ - struct mpipe_rx_queue *rx_queue = _rxq; - uint16_t result = 0; - - if (rx_queue) { - mpipe_dp_enter(rx_queue->q.priv); - if (likely(rx_queue->q.link_status)) - result = mpipe_do_recv(rx_queue, rx_pkts, nb_pkts); - mpipe_dp_exit(rx_queue->q.priv); - } - - return result; -} - -static uint16_t -mpipe_xmit_pkts(void *_txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) -{ - struct mpipe_tx_queue *tx_queue = _txq; - uint16_t result = 0; - - if (tx_queue) { - mpipe_dp_enter(tx_queue->q.priv); - if (likely(tx_queue->q.link_status)) - result = mpipe_do_xmit(tx_queue, tx_pkts, nb_pkts); - mpipe_dp_exit(tx_queue->q.priv); - } - - return result; -} - -static int -mpipe_link_mac(const char *ifname, uint8_t *mac) -{ - int rc, idx; - char name[GXIO_MPIPE_LINK_NAME_LEN]; - - for (idx = 0, rc = 0; !rc; idx++) { - rc = gxio_mpipe_link_enumerate_mac(idx, name, mac); - if (!rc && !strncmp(name, ifname, GXIO_MPIPE_LINK_NAME_LEN)) - return 0; - } - return -ENODEV; -} - -static int -rte_pmd_mpipe_devinit(const char *ifname, - const char *params __rte_unused) -{ - gxio_mpipe_context_t *context; - struct rte_eth_dev *eth_dev; - struct mpipe_dev_priv *priv; - int instance, rc; - uint8_t *mac; - - /* Get the mPIPE instance that the device belongs to. */ - instance = gxio_mpipe_link_instance(ifname); - context = mpipe_context(instance); - if (!context) { - RTE_LOG(ERR, PMD, "%s: No device for link.\n", ifname); - return -ENODEV; - } - - priv = rte_zmalloc(NULL, sizeof(*priv), 0); - if (!priv) { - RTE_LOG(ERR, PMD, "%s: Failed to allocate priv.\n", ifname); - return -ENOMEM; - } - - memset(&priv->tx_stat_mapping, 0xff, sizeof(priv->tx_stat_mapping)); - memset(&priv->rx_stat_mapping, 0xff, sizeof(priv->rx_stat_mapping)); - priv->context = context; - priv->instance = instance; - priv->is_xaui = (strncmp(ifname, "xgbe", 4) == 0); - priv->channel = -1; - - mac = priv->mac_addr.addr_bytes; - rc = mpipe_link_mac(ifname, mac); - if (rc < 0) { - RTE_LOG(ERR, PMD, "%s: Failed to enumerate link.\n", ifname); - rte_free(priv); - return -ENODEV; - } - - eth_dev = rte_eth_dev_allocate(ifname, RTE_ETH_DEV_VIRTUAL); - if (!eth_dev) { - RTE_LOG(ERR, PMD, "%s: Failed to allocate device.\n", ifname); - rte_free(priv); - return -ENOMEM; - } - - RTE_LOG(INFO, PMD, "%s: Initialized mpipe device" - "(mac %02x:%02x:%02x:%02x:%02x:%02x).\n", - ifname, mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]); - - priv->eth_dev = eth_dev; - priv->port_id = eth_dev->data->port_id; - eth_dev->data->dev_private = priv; - eth_dev->data->mac_addrs = &priv->mac_addr; - - eth_dev->data->dev_flags = 0; - eth_dev->data->kdrv = RTE_KDRV_NONE; - eth_dev->driver = NULL; - eth_dev->data->drv_name = drivername; - eth_dev->data->numa_node = instance; - - eth_dev->dev_ops = &mpipe_dev_ops; - eth_dev->rx_pkt_burst = &mpipe_recv_pkts; - eth_dev->tx_pkt_burst = &mpipe_xmit_pkts; - - rc = mpipe_link_init(priv); - if (rc < 0) { - RTE_LOG(ERR, PMD, "%s: Failed to init link.\n", - mpipe_name(priv)); - return rc; - } - - return 0; -} - -static struct rte_driver pmd_mpipe_xgbe_drv = { - .type = PMD_VDEV, - .init = rte_pmd_mpipe_devinit, -}; - -static struct rte_driver pmd_mpipe_gbe_drv = { - .type = PMD_VDEV, - .init = rte_pmd_mpipe_devinit, -}; - -PMD_REGISTER_DRIVER(pmd_mpipe_xgbe_drv, xgbe); -PMD_REGISTER_DRIVER(pmd_mpipe_gbe_drv, gbe); - -static void __attribute__((constructor, used)) -mpipe_init_contexts(void) -{ - struct mpipe_context *context; - int rc, instance; - - for (instance = 0; instance < GXIO_MPIPE_INSTANCE_MAX; instance++) { - context = &mpipe_contexts[instance]; - - rte_spinlock_init(&context->lock); - rc = gxio_mpipe_init(&context->context, instance); - if (rc < 0) - break; - } - - mpipe_instances = instance; -} diff --git a/dpdk/drivers/net/mpipe/rte_pmd_mpipe_version.map b/dpdk/drivers/net/mpipe/rte_pmd_mpipe_version.map deleted file mode 100644 index ad607bbe..00000000 --- a/dpdk/drivers/net/mpipe/rte_pmd_mpipe_version.map +++ /dev/null @@ -1,3 +0,0 @@ -DPDK_2.2 { - local: *; -}; diff --git a/dpdk/drivers/net/mrvl/Makefile b/dpdk/drivers/net/mrvl/Makefile new file mode 100644 index 00000000..815c3bae --- /dev/null +++ b/dpdk/drivers/net/mrvl/Makefile @@ -0,0 +1,68 @@ +# BSD LICENSE +# +# Copyright(c) 2017 Marvell International Ltd. +# Copyright(c) 2017 Semihalf. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +include $(RTE_SDK)/mk/rte.vars.mk + +ifneq ($(MAKECMDGOALS),clean) +ifneq ($(MAKECMDGOALS),config) +ifeq ($(LIBMUSDK_PATH),) +$(error "Please define LIBMUSDK_PATH environment variable") +endif +endif +endif + +# library name +LIB = librte_pmd_mrvl.a + +# library version +LIBABIVER := 1 + +# versioning export map +EXPORT_MAP := rte_pmd_mrvl_version.map + +# external library dependencies +CFLAGS += -I$(LIBMUSDK_PATH)/include +CFLAGS += -DMVCONF_ARCH_DMA_ADDR_T_64BIT +CFLAGS += -DCONF_PP2_BPOOL_COOKIE_SIZE=32 +CFLAGS += $(WERROR_FLAGS) +CFLAGS += -O3 +LDLIBS += -L$(LIBMUSDK_PATH)/lib +LDLIBS += -lmusdk +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs -lrte_cfgfile +LDLIBS += -lrte_bus_vdev + +# library source files +SRCS-$(CONFIG_RTE_LIBRTE_MRVL_PMD) += mrvl_ethdev.c +SRCS-$(CONFIG_RTE_LIBRTE_MRVL_PMD) += mrvl_qos.c + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/net/mrvl/mrvl_ethdev.c b/dpdk/drivers/net/mrvl/mrvl_ethdev.c new file mode 100644 index 00000000..9a358194 --- /dev/null +++ b/dpdk/drivers/net/mrvl/mrvl_ethdev.c @@ -0,0 +1,2386 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Marvell International Ltd. + * Copyright(c) 2017 Semihalf. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Semihalf nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include + +/* Unluckily, container_of is defined by both DPDK and MUSDK, + * we'll declare only one version. + * + * Note that it is not used in this PMD anyway. + */ +#ifdef container_of +#undef container_of +#endif + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "mrvl_ethdev.h" +#include "mrvl_qos.h" + +/* bitmask with reserved hifs */ +#define MRVL_MUSDK_HIFS_RESERVED 0x0F +/* bitmask with reserved bpools */ +#define MRVL_MUSDK_BPOOLS_RESERVED 0x07 +/* bitmask with reserved kernel RSS tables */ +#define MRVL_MUSDK_RSS_RESERVED 0x01 +/* maximum number of available hifs */ +#define MRVL_MUSDK_HIFS_MAX 9 + +/* prefetch shift */ +#define MRVL_MUSDK_PREFETCH_SHIFT 2 + +/* TCAM has 25 entries reserved for uc/mc filter entries */ +#define MRVL_MAC_ADDRS_MAX 25 +#define MRVL_MATCH_LEN 16 +#define MRVL_PKT_EFFEC_OFFS (MRVL_PKT_OFFS + MV_MH_SIZE) +/* Maximum allowable packet size */ +#define MRVL_PKT_SIZE_MAX (10240 - MV_MH_SIZE) + +#define MRVL_IFACE_NAME_ARG "iface" +#define MRVL_CFG_ARG "cfg" + +#define MRVL_BURST_SIZE 64 + +#define MRVL_ARP_LENGTH 28 + +#define MRVL_COOKIE_ADDR_INVALID ~0ULL + +#define MRVL_COOKIE_HIGH_ADDR_SHIFT (sizeof(pp2_cookie_t) * 8) +#define MRVL_COOKIE_HIGH_ADDR_MASK (~0ULL << MRVL_COOKIE_HIGH_ADDR_SHIFT) + +/* Memory size (in bytes) for MUSDK dma buffers */ +#define MRVL_MUSDK_DMA_MEMSIZE 41943040 + +static const char * const valid_args[] = { + MRVL_IFACE_NAME_ARG, + MRVL_CFG_ARG, + NULL +}; + +static int used_hifs = MRVL_MUSDK_HIFS_RESERVED; +static struct pp2_hif *hifs[RTE_MAX_LCORE]; +static int used_bpools[PP2_NUM_PKT_PROC] = { + MRVL_MUSDK_BPOOLS_RESERVED, + MRVL_MUSDK_BPOOLS_RESERVED +}; + +struct pp2_bpool *mrvl_port_to_bpool_lookup[RTE_MAX_ETHPORTS]; +int mrvl_port_bpool_size[PP2_NUM_PKT_PROC][PP2_BPOOL_NUM_POOLS][RTE_MAX_LCORE]; +uint64_t cookie_addr_high = MRVL_COOKIE_ADDR_INVALID; + +struct mrvl_ifnames { + const char *names[PP2_NUM_ETH_PPIO * PP2_NUM_PKT_PROC]; + int idx; +}; + +/* + * To use buffer harvesting based on loopback port shadow queue structure + * was introduced for buffers information bookkeeping. + * + * Before sending the packet, related buffer information (pp2_buff_inf) is + * stored in shadow queue. After packet is transmitted no longer used + * packet buffer is released back to it's original hardware pool, + * on condition it originated from interface. + * In case it was generated by application itself i.e: mbuf->port field is + * 0xff then its released to software mempool. + */ +struct mrvl_shadow_txq { + int head; /* write index - used when sending buffers */ + int tail; /* read index - used when releasing buffers */ + u16 size; /* queue occupied size */ + u16 num_to_release; /* number of buffers sent, that can be released */ + struct buff_release_entry ent[MRVL_PP2_TX_SHADOWQ_SIZE]; /* q entries */ +}; + +struct mrvl_rxq { + struct mrvl_priv *priv; + struct rte_mempool *mp; + int queue_id; + int port_id; + int cksum_enabled; + uint64_t bytes_recv; + uint64_t drop_mac; +}; + +struct mrvl_txq { + struct mrvl_priv *priv; + int queue_id; + int port_id; + uint64_t bytes_sent; + struct mrvl_shadow_txq shadow_txqs[RTE_MAX_LCORE]; +}; + +static int mrvl_lcore_first; +static int mrvl_lcore_last; +static int mrvl_dev_num; + +static int mrvl_fill_bpool(struct mrvl_rxq *rxq, int num); +static inline void mrvl_free_sent_buffers(struct pp2_ppio *ppio, + struct pp2_hif *hif, unsigned int core_id, + struct mrvl_shadow_txq *sq, int qid, int force); + +static inline int +mrvl_get_bpool_size(int pp2_id, int pool_id) +{ + int i; + int size = 0; + + for (i = mrvl_lcore_first; i <= mrvl_lcore_last; i++) + size += mrvl_port_bpool_size[pp2_id][pool_id][i]; + + return size; +} + +static inline int +mrvl_reserve_bit(int *bitmap, int max) +{ + int n = sizeof(*bitmap) * 8 - __builtin_clz(*bitmap); + + if (n >= max) + return -1; + + *bitmap |= 1 << n; + + return n; +} + +static int +mrvl_init_hif(int core_id) +{ + struct pp2_hif_params params; + char match[MRVL_MATCH_LEN]; + int ret; + + ret = mrvl_reserve_bit(&used_hifs, MRVL_MUSDK_HIFS_MAX); + if (ret < 0) { + RTE_LOG(ERR, PMD, "Failed to allocate hif %d\n", core_id); + return ret; + } + + snprintf(match, sizeof(match), "hif-%d", ret); + memset(¶ms, 0, sizeof(params)); + params.match = match; + params.out_size = MRVL_PP2_AGGR_TXQD_MAX; + ret = pp2_hif_init(¶ms, &hifs[core_id]); + if (ret) { + RTE_LOG(ERR, PMD, "Failed to initialize hif %d\n", core_id); + return ret; + } + + return 0; +} + +static inline struct pp2_hif* +mrvl_get_hif(struct mrvl_priv *priv, int core_id) +{ + int ret; + + if (likely(hifs[core_id] != NULL)) + return hifs[core_id]; + + rte_spinlock_lock(&priv->lock); + + ret = mrvl_init_hif(core_id); + if (ret < 0) { + RTE_LOG(ERR, PMD, "Failed to allocate hif %d\n", core_id); + goto out; + } + + if (core_id < mrvl_lcore_first) + mrvl_lcore_first = core_id; + + if (core_id > mrvl_lcore_last) + mrvl_lcore_last = core_id; +out: + rte_spinlock_unlock(&priv->lock); + + return hifs[core_id]; +} + +/** + * Configure rss based on dpdk rss configuration. + * + * @param priv + * Pointer to private structure. + * @param rss_conf + * Pointer to RSS configuration. + * + * @return + * 0 on success, negative error value otherwise. + */ +static int +mrvl_configure_rss(struct mrvl_priv *priv, struct rte_eth_rss_conf *rss_conf) +{ + if (rss_conf->rss_key) + RTE_LOG(WARNING, PMD, "Changing hash key is not supported\n"); + + if (rss_conf->rss_hf == 0) { + priv->ppio_params.inqs_params.hash_type = PP2_PPIO_HASH_T_NONE; + } else if (rss_conf->rss_hf & ETH_RSS_IPV4) { + priv->ppio_params.inqs_params.hash_type = + PP2_PPIO_HASH_T_2_TUPLE; + } else if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) { + priv->ppio_params.inqs_params.hash_type = + PP2_PPIO_HASH_T_5_TUPLE; + priv->rss_hf_tcp = 1; + } else if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) { + priv->ppio_params.inqs_params.hash_type = + PP2_PPIO_HASH_T_5_TUPLE; + priv->rss_hf_tcp = 0; + } else { + return -EINVAL; + } + + return 0; +} + +/** + * Ethernet device configuration. + * + * Prepare the driver for a given number of TX and RX queues and + * configure RSS. + * + * @param dev + * Pointer to Ethernet device structure. + * + * @return + * 0 on success, negative error value otherwise. + */ +static int +mrvl_dev_configure(struct rte_eth_dev *dev) +{ + struct mrvl_priv *priv = dev->data->dev_private; + int ret; + + if (dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_NONE && + dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_RSS) { + RTE_LOG(INFO, PMD, "Unsupported rx multi queue mode %d\n", + dev->data->dev_conf.rxmode.mq_mode); + return -EINVAL; + } + + if (!dev->data->dev_conf.rxmode.hw_strip_crc) { + RTE_LOG(INFO, PMD, + "L2 CRC stripping is always enabled in hw\n"); + dev->data->dev_conf.rxmode.hw_strip_crc = 1; + } + + if (dev->data->dev_conf.rxmode.hw_vlan_strip) { + RTE_LOG(INFO, PMD, "VLAN stripping not supported\n"); + return -EINVAL; + } + + if (dev->data->dev_conf.rxmode.split_hdr_size) { + RTE_LOG(INFO, PMD, "Split headers not supported\n"); + return -EINVAL; + } + + if (dev->data->dev_conf.rxmode.enable_scatter) { + RTE_LOG(INFO, PMD, "RX Scatter/Gather not supported\n"); + return -EINVAL; + } + + if (dev->data->dev_conf.rxmode.enable_lro) { + RTE_LOG(INFO, PMD, "LRO not supported\n"); + return -EINVAL; + } + + if (dev->data->dev_conf.rxmode.jumbo_frame) + dev->data->mtu = dev->data->dev_conf.rxmode.max_rx_pkt_len - + ETHER_HDR_LEN - ETHER_CRC_LEN; + + ret = mrvl_configure_rxqs(priv, dev->data->port_id, + dev->data->nb_rx_queues); + if (ret < 0) + return ret; + + priv->ppio_params.outqs_params.num_outqs = dev->data->nb_tx_queues; + priv->ppio_params.maintain_stats = 1; + priv->nb_rx_queues = dev->data->nb_rx_queues; + + if (dev->data->nb_rx_queues == 1 && + dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_RSS) { + RTE_LOG(WARNING, PMD, "Disabling hash for 1 rx queue\n"); + priv->ppio_params.inqs_params.hash_type = PP2_PPIO_HASH_T_NONE; + + return 0; + } + + return mrvl_configure_rss(priv, + &dev->data->dev_conf.rx_adv_conf.rss_conf); +} + +/** + * DPDK callback to change the MTU. + * + * Setting the MTU affects hardware MRU (packets larger than the MRU + * will be dropped). + * + * @param dev + * Pointer to Ethernet device structure. + * @param mtu + * New MTU. + * + * @return + * 0 on success, negative error value otherwise. + */ +static int +mrvl_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) +{ + struct mrvl_priv *priv = dev->data->dev_private; + /* extra MV_MH_SIZE bytes are required for Marvell tag */ + uint16_t mru = mtu + MV_MH_SIZE + ETHER_HDR_LEN + ETHER_CRC_LEN; + int ret; + + if (mtu < ETHER_MIN_MTU || mru > MRVL_PKT_SIZE_MAX) + return -EINVAL; + + ret = pp2_ppio_set_mru(priv->ppio, mru); + if (ret) + return ret; + + return pp2_ppio_set_mtu(priv->ppio, mtu); +} + +/** + * DPDK callback to bring the link up. + * + * @param dev + * Pointer to Ethernet device structure. + * + * @return + * 0 on success, negative error value otherwise. + */ +static int +mrvl_dev_set_link_up(struct rte_eth_dev *dev) +{ + struct mrvl_priv *priv = dev->data->dev_private; + int ret; + + ret = pp2_ppio_enable(priv->ppio); + if (ret) + return ret; + + /* + * mtu/mru can be updated if pp2_ppio_enable() was called at least once + * as pp2_ppio_enable() changes port->t_mode from default 0 to + * PP2_TRAFFIC_INGRESS_EGRESS. + * + * Set mtu to default DPDK value here. + */ + ret = mrvl_mtu_set(dev, dev->data->mtu); + if (ret) + pp2_ppio_disable(priv->ppio); + + dev->data->dev_link.link_status = ETH_LINK_UP; + + return ret; +} + +/** + * DPDK callback to bring the link down. + * + * @param dev + * Pointer to Ethernet device structure. + * + * @return + * 0 on success, negative error value otherwise. + */ +static int +mrvl_dev_set_link_down(struct rte_eth_dev *dev) +{ + struct mrvl_priv *priv = dev->data->dev_private; + int ret; + + ret = pp2_ppio_disable(priv->ppio); + if (ret) + return ret; + + dev->data->dev_link.link_status = ETH_LINK_DOWN; + + return ret; +} + +/** + * DPDK callback to start the device. + * + * @param dev + * Pointer to Ethernet device structure. + * + * @return + * 0 on success, negative errno value on failure. + */ +static int +mrvl_dev_start(struct rte_eth_dev *dev) +{ + struct mrvl_priv *priv = dev->data->dev_private; + char match[MRVL_MATCH_LEN]; + int ret = 0, def_init_size; + + snprintf(match, sizeof(match), "ppio-%d:%d", + priv->pp_id, priv->ppio_id); + priv->ppio_params.match = match; + + /* + * Calculate the minimum bpool size for refill feature as follows: + * 2 default burst sizes multiply by number of rx queues. + * If the bpool size will be below this value, new buffers will + * be added to the pool. + */ + priv->bpool_min_size = priv->nb_rx_queues * MRVL_BURST_SIZE * 2; + + /* In case initial bpool size configured in queues setup is + * smaller than minimum size add more buffers + */ + def_init_size = priv->bpool_min_size + MRVL_BURST_SIZE * 2; + if (priv->bpool_init_size < def_init_size) { + int buffs_to_add = def_init_size - priv->bpool_init_size; + + priv->bpool_init_size += buffs_to_add; + ret = mrvl_fill_bpool(dev->data->rx_queues[0], buffs_to_add); + if (ret) + RTE_LOG(ERR, PMD, "Failed to add buffers to bpool\n"); + } + + /* + * Calculate the maximum bpool size for refill feature as follows: + * maximum number of descriptors in rx queue multiply by number + * of rx queues plus minimum bpool size. + * In case the bpool size will exceed this value, superfluous buffers + * will be removed + */ + priv->bpool_max_size = (priv->nb_rx_queues * MRVL_PP2_RXD_MAX) + + priv->bpool_min_size; + + ret = pp2_ppio_init(&priv->ppio_params, &priv->ppio); + if (ret) + return ret; + + /* + * In case there are some some stale uc/mc mac addresses flush them + * here. It cannot be done during mrvl_dev_close() as port information + * is already gone at that point (due to pp2_ppio_deinit() in + * mrvl_dev_stop()). + */ + if (!priv->uc_mc_flushed) { + ret = pp2_ppio_flush_mac_addrs(priv->ppio, 1, 1); + if (ret) { + RTE_LOG(ERR, PMD, + "Failed to flush uc/mc filter list\n"); + goto out; + } + priv->uc_mc_flushed = 1; + } + + if (!priv->vlan_flushed) { + ret = pp2_ppio_flush_vlan(priv->ppio); + if (ret) { + RTE_LOG(ERR, PMD, "Failed to flush vlan list\n"); + /* + * TODO + * once pp2_ppio_flush_vlan() is supported jump to out + * goto out; + */ + } + priv->vlan_flushed = 1; + } + + /* For default QoS config, don't start classifier. */ + if (mrvl_qos_cfg) { + ret = mrvl_start_qos_mapping(priv); + if (ret) { + pp2_ppio_deinit(priv->ppio); + return ret; + } + } + + ret = mrvl_dev_set_link_up(dev); + if (ret) + goto out; + + return 0; +out: + pp2_ppio_deinit(priv->ppio); + return ret; +} + +/** + * Flush receive queues. + * + * @param dev + * Pointer to Ethernet device structure. + */ +static void +mrvl_flush_rx_queues(struct rte_eth_dev *dev) +{ + int i; + + RTE_LOG(INFO, PMD, "Flushing rx queues\n"); + for (i = 0; i < dev->data->nb_rx_queues; i++) { + int ret, num; + + do { + struct mrvl_rxq *q = dev->data->rx_queues[i]; + struct pp2_ppio_desc descs[MRVL_PP2_RXD_MAX]; + + num = MRVL_PP2_RXD_MAX; + ret = pp2_ppio_recv(q->priv->ppio, + q->priv->rxq_map[q->queue_id].tc, + q->priv->rxq_map[q->queue_id].inq, + descs, (uint16_t *)&num); + } while (ret == 0 && num); + } +} + +/** + * Flush transmit shadow queues. + * + * @param dev + * Pointer to Ethernet device structure. + */ +static void +mrvl_flush_tx_shadow_queues(struct rte_eth_dev *dev) +{ + int i, j; + struct mrvl_txq *txq; + + RTE_LOG(INFO, PMD, "Flushing tx shadow queues\n"); + for (i = 0; i < dev->data->nb_tx_queues; i++) { + txq = (struct mrvl_txq *)dev->data->tx_queues[i]; + + for (j = 0; j < RTE_MAX_LCORE; j++) { + struct mrvl_shadow_txq *sq; + + if (!hifs[j]) + continue; + + sq = &txq->shadow_txqs[j]; + mrvl_free_sent_buffers(txq->priv->ppio, + hifs[j], j, sq, txq->queue_id, 1); + while (sq->tail != sq->head) { + uint64_t addr = cookie_addr_high | + sq->ent[sq->tail].buff.cookie; + rte_pktmbuf_free( + (struct rte_mbuf *)addr); + sq->tail = (sq->tail + 1) & + MRVL_PP2_TX_SHADOWQ_MASK; + } + memset(sq, 0, sizeof(*sq)); + } + } +} + +/** + * Flush hardware bpool (buffer-pool). + * + * @param dev + * Pointer to Ethernet device structure. + */ +static void +mrvl_flush_bpool(struct rte_eth_dev *dev) +{ + struct mrvl_priv *priv = dev->data->dev_private; + struct pp2_hif *hif; + uint32_t num; + int ret; + unsigned int core_id = rte_lcore_id(); + + if (core_id == LCORE_ID_ANY) + core_id = 0; + + hif = mrvl_get_hif(priv, core_id); + + ret = pp2_bpool_get_num_buffs(priv->bpool, &num); + if (ret) { + RTE_LOG(ERR, PMD, "Failed to get bpool buffers number\n"); + return; + } + + while (num--) { + struct pp2_buff_inf inf; + uint64_t addr; + + ret = pp2_bpool_get_buff(hif, priv->bpool, &inf); + if (ret) + break; + + addr = cookie_addr_high | inf.cookie; + rte_pktmbuf_free((struct rte_mbuf *)addr); + } +} + +/** + * DPDK callback to stop the device. + * + * @param dev + * Pointer to Ethernet device structure. + */ +static void +mrvl_dev_stop(struct rte_eth_dev *dev) +{ + struct mrvl_priv *priv = dev->data->dev_private; + + mrvl_dev_set_link_down(dev); + mrvl_flush_rx_queues(dev); + mrvl_flush_tx_shadow_queues(dev); + if (priv->qos_tbl) { + pp2_cls_qos_tbl_deinit(priv->qos_tbl); + priv->qos_tbl = NULL; + } + pp2_ppio_deinit(priv->ppio); + priv->ppio = NULL; +} + +/** + * DPDK callback to close the device. + * + * @param dev + * Pointer to Ethernet device structure. + */ +static void +mrvl_dev_close(struct rte_eth_dev *dev) +{ + struct mrvl_priv *priv = dev->data->dev_private; + size_t i; + + for (i = 0; i < priv->ppio_params.inqs_params.num_tcs; ++i) { + struct pp2_ppio_tc_params *tc_params = + &priv->ppio_params.inqs_params.tcs_params[i]; + + if (tc_params->inqs_params) { + rte_free(tc_params->inqs_params); + tc_params->inqs_params = NULL; + } + } + + mrvl_flush_bpool(dev); +} + +/** + * DPDK callback to retrieve physical link information. + * + * @param dev + * Pointer to Ethernet device structure. + * @param wait_to_complete + * Wait for request completion (ignored). + * + * @return + * 0 on success, negative error value otherwise. + */ +static int +mrvl_link_update(struct rte_eth_dev *dev, int wait_to_complete __rte_unused) +{ + /* + * TODO + * once MUSDK provides necessary API use it here + */ + struct ethtool_cmd edata; + struct ifreq req; + int ret, fd; + + edata.cmd = ETHTOOL_GSET; + + strcpy(req.ifr_name, dev->data->name); + req.ifr_data = (void *)&edata; + + fd = socket(AF_INET, SOCK_DGRAM, 0); + if (fd == -1) + return -EFAULT; + + ret = ioctl(fd, SIOCETHTOOL, &req); + if (ret == -1) { + close(fd); + return -EFAULT; + } + + close(fd); + + switch (ethtool_cmd_speed(&edata)) { + case SPEED_10: + dev->data->dev_link.link_speed = ETH_SPEED_NUM_10M; + break; + case SPEED_100: + dev->data->dev_link.link_speed = ETH_SPEED_NUM_100M; + break; + case SPEED_1000: + dev->data->dev_link.link_speed = ETH_SPEED_NUM_1G; + break; + case SPEED_10000: + dev->data->dev_link.link_speed = ETH_SPEED_NUM_10G; + break; + default: + dev->data->dev_link.link_speed = ETH_SPEED_NUM_NONE; + } + + dev->data->dev_link.link_duplex = edata.duplex ? ETH_LINK_FULL_DUPLEX : + ETH_LINK_HALF_DUPLEX; + dev->data->dev_link.link_autoneg = edata.autoneg ? ETH_LINK_AUTONEG : + ETH_LINK_FIXED; + + return 0; +} + +/** + * DPDK callback to enable promiscuous mode. + * + * @param dev + * Pointer to Ethernet device structure. + */ +static void +mrvl_promiscuous_enable(struct rte_eth_dev *dev) +{ + struct mrvl_priv *priv = dev->data->dev_private; + int ret; + + ret = pp2_ppio_set_uc_promisc(priv->ppio, 1); + if (ret) + RTE_LOG(ERR, PMD, "Failed to enable promiscuous mode\n"); +} + +/** + * DPDK callback to enable allmulti mode. + * + * @param dev + * Pointer to Ethernet device structure. + */ +static void +mrvl_allmulticast_enable(struct rte_eth_dev *dev) +{ + struct mrvl_priv *priv = dev->data->dev_private; + int ret; + + ret = pp2_ppio_set_mc_promisc(priv->ppio, 1); + if (ret) + RTE_LOG(ERR, PMD, "Failed enable all-multicast mode\n"); +} + +/** + * DPDK callback to disable promiscuous mode. + * + * @param dev + * Pointer to Ethernet device structure. + */ +static void +mrvl_promiscuous_disable(struct rte_eth_dev *dev) +{ + struct mrvl_priv *priv = dev->data->dev_private; + int ret; + + ret = pp2_ppio_set_uc_promisc(priv->ppio, 0); + if (ret) + RTE_LOG(ERR, PMD, "Failed to disable promiscuous mode\n"); +} + +/** + * DPDK callback to disable allmulticast mode. + * + * @param dev + * Pointer to Ethernet device structure. + */ +static void +mrvl_allmulticast_disable(struct rte_eth_dev *dev) +{ + struct mrvl_priv *priv = dev->data->dev_private; + int ret; + + ret = pp2_ppio_set_mc_promisc(priv->ppio, 0); + if (ret) + RTE_LOG(ERR, PMD, "Failed to disable all-multicast mode\n"); +} + +/** + * DPDK callback to remove a MAC address. + * + * @param dev + * Pointer to Ethernet device structure. + * @param index + * MAC address index. + */ +static void +mrvl_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index) +{ + struct mrvl_priv *priv = dev->data->dev_private; + char buf[ETHER_ADDR_FMT_SIZE]; + int ret; + + ret = pp2_ppio_remove_mac_addr(priv->ppio, + dev->data->mac_addrs[index].addr_bytes); + if (ret) { + ether_format_addr(buf, sizeof(buf), + &dev->data->mac_addrs[index]); + RTE_LOG(ERR, PMD, "Failed to remove mac %s\n", buf); + } +} + +/** + * DPDK callback to add a MAC address. + * + * @param dev + * Pointer to Ethernet device structure. + * @param mac_addr + * MAC address to register. + * @param index + * MAC address index. + * @param vmdq + * VMDq pool index to associate address with (unused). + * + * @return + * 0 on success, negative error value otherwise. + */ +static int +mrvl_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac_addr, + uint32_t index, uint32_t vmdq __rte_unused) +{ + struct mrvl_priv *priv = dev->data->dev_private; + char buf[ETHER_ADDR_FMT_SIZE]; + int ret; + + if (index == 0) + /* For setting index 0, mrvl_mac_addr_set() should be used.*/ + return -1; + + /* + * Maximum number of uc addresses can be tuned via kernel module mvpp2x + * parameter uc_filter_max. Maximum number of mc addresses is then + * MRVL_MAC_ADDRS_MAX - uc_filter_max. Currently it defaults to 4 and + * 21 respectively. + * + * If more than uc_filter_max uc addresses were added to filter list + * then NIC will switch to promiscuous mode automatically. + * + * If more than MRVL_MAC_ADDRS_MAX - uc_filter_max number mc addresses + * were added to filter list then NIC will switch to all-multicast mode + * automatically. + */ + ret = pp2_ppio_add_mac_addr(priv->ppio, mac_addr->addr_bytes); + if (ret) { + ether_format_addr(buf, sizeof(buf), mac_addr); + RTE_LOG(ERR, PMD, "Failed to add mac %s\n", buf); + return -1; + } + + return 0; +} + +/** + * DPDK callback to set the primary MAC address. + * + * @param dev + * Pointer to Ethernet device structure. + * @param mac_addr + * MAC address to register. + */ +static void +mrvl_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr) +{ + struct mrvl_priv *priv = dev->data->dev_private; + + pp2_ppio_set_mac_addr(priv->ppio, mac_addr->addr_bytes); + /* + * TODO + * Port stops sending packets if pp2_ppio_set_mac_addr() + * was called after pp2_ppio_enable(). As a quick fix issue + * enable port once again. + */ + pp2_ppio_enable(priv->ppio); +} + +/** + * DPDK callback to get device statistics. + * + * @param dev + * Pointer to Ethernet device structure. + * @param stats + * Stats structure output buffer. + * + * @return + * 0 on success, negative error value otherwise. + */ +static int +mrvl_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) +{ + struct mrvl_priv *priv = dev->data->dev_private; + struct pp2_ppio_statistics ppio_stats; + uint64_t drop_mac = 0; + unsigned int i, idx, ret; + + for (i = 0; i < dev->data->nb_rx_queues; i++) { + struct mrvl_rxq *rxq = dev->data->rx_queues[i]; + struct pp2_ppio_inq_statistics rx_stats; + + if (!rxq) + continue; + + idx = rxq->queue_id; + if (unlikely(idx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)) { + RTE_LOG(ERR, PMD, + "rx queue %d stats out of range (0 - %d)\n", + idx, RTE_ETHDEV_QUEUE_STAT_CNTRS - 1); + continue; + } + + ret = pp2_ppio_inq_get_statistics(priv->ppio, + priv->rxq_map[idx].tc, + priv->rxq_map[idx].inq, + &rx_stats, 0); + if (unlikely(ret)) { + RTE_LOG(ERR, PMD, + "Failed to update rx queue %d stats\n", idx); + break; + } + + stats->q_ibytes[idx] = rxq->bytes_recv; + stats->q_ipackets[idx] = rx_stats.enq_desc - rxq->drop_mac; + stats->q_errors[idx] = rx_stats.drop_early + + rx_stats.drop_fullq + + rx_stats.drop_bm + + rxq->drop_mac; + stats->ibytes += rxq->bytes_recv; + drop_mac += rxq->drop_mac; + } + + for (i = 0; i < dev->data->nb_tx_queues; i++) { + struct mrvl_txq *txq = dev->data->tx_queues[i]; + struct pp2_ppio_outq_statistics tx_stats; + + if (!txq) + continue; + + idx = txq->queue_id; + if (unlikely(idx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)) { + RTE_LOG(ERR, PMD, + "tx queue %d stats out of range (0 - %d)\n", + idx, RTE_ETHDEV_QUEUE_STAT_CNTRS - 1); + } + + ret = pp2_ppio_outq_get_statistics(priv->ppio, idx, + &tx_stats, 0); + if (unlikely(ret)) { + RTE_LOG(ERR, PMD, + "Failed to update tx queue %d stats\n", idx); + break; + } + + stats->q_opackets[idx] = tx_stats.deq_desc; + stats->q_obytes[idx] = txq->bytes_sent; + stats->obytes += txq->bytes_sent; + } + + ret = pp2_ppio_get_statistics(priv->ppio, &ppio_stats, 0); + if (unlikely(ret)) { + RTE_LOG(ERR, PMD, "Failed to update port statistics\n"); + return ret; + } + + stats->ipackets += ppio_stats.rx_packets - drop_mac; + stats->opackets += ppio_stats.tx_packets; + stats->imissed += ppio_stats.rx_fullq_dropped + + ppio_stats.rx_bm_dropped + + ppio_stats.rx_early_dropped + + ppio_stats.rx_fifo_dropped + + ppio_stats.rx_cls_dropped; + stats->ierrors = drop_mac; + + return 0; +} + +/** + * DPDK callback to clear device statistics. + * + * @param dev + * Pointer to Ethernet device structure. + */ +static void +mrvl_stats_reset(struct rte_eth_dev *dev) +{ + struct mrvl_priv *priv = dev->data->dev_private; + int i; + + for (i = 0; i < dev->data->nb_rx_queues; i++) { + struct mrvl_rxq *rxq = dev->data->rx_queues[i]; + + pp2_ppio_inq_get_statistics(priv->ppio, priv->rxq_map[i].tc, + priv->rxq_map[i].inq, NULL, 1); + rxq->bytes_recv = 0; + rxq->drop_mac = 0; + } + + for (i = 0; i < dev->data->nb_tx_queues; i++) { + struct mrvl_txq *txq = dev->data->tx_queues[i]; + + pp2_ppio_outq_get_statistics(priv->ppio, i, NULL, 1); + txq->bytes_sent = 0; + } + + pp2_ppio_get_statistics(priv->ppio, NULL, 1); +} + +/** + * DPDK callback to get information about the device. + * + * @param dev + * Pointer to Ethernet device structure (unused). + * @param info + * Info structure output buffer. + */ +static void +mrvl_dev_infos_get(struct rte_eth_dev *dev __rte_unused, + struct rte_eth_dev_info *info) +{ + info->speed_capa = ETH_LINK_SPEED_10M | + ETH_LINK_SPEED_100M | + ETH_LINK_SPEED_1G | + ETH_LINK_SPEED_10G; + + info->max_rx_queues = MRVL_PP2_RXQ_MAX; + info->max_tx_queues = MRVL_PP2_TXQ_MAX; + info->max_mac_addrs = MRVL_MAC_ADDRS_MAX; + + info->rx_desc_lim.nb_max = MRVL_PP2_RXD_MAX; + info->rx_desc_lim.nb_min = MRVL_PP2_RXD_MIN; + info->rx_desc_lim.nb_align = MRVL_PP2_RXD_ALIGN; + + info->tx_desc_lim.nb_max = MRVL_PP2_TXD_MAX; + info->tx_desc_lim.nb_min = MRVL_PP2_TXD_MIN; + info->tx_desc_lim.nb_align = MRVL_PP2_TXD_ALIGN; + + info->rx_offload_capa = DEV_RX_OFFLOAD_JUMBO_FRAME | + DEV_RX_OFFLOAD_VLAN_FILTER | + DEV_RX_OFFLOAD_IPV4_CKSUM | + DEV_RX_OFFLOAD_UDP_CKSUM | + DEV_RX_OFFLOAD_TCP_CKSUM; + + info->tx_offload_capa = DEV_TX_OFFLOAD_IPV4_CKSUM | + DEV_TX_OFFLOAD_UDP_CKSUM | + DEV_TX_OFFLOAD_TCP_CKSUM; + + info->flow_type_rss_offloads = ETH_RSS_IPV4 | + ETH_RSS_NONFRAG_IPV4_TCP | + ETH_RSS_NONFRAG_IPV4_UDP; + + /* By default packets are dropped if no descriptors are available */ + info->default_rxconf.rx_drop_en = 1; + + info->max_rx_pktlen = MRVL_PKT_SIZE_MAX; +} + +/** + * Return supported packet types. + * + * @param dev + * Pointer to Ethernet device structure (unused). + * + * @return + * Const pointer to the table with supported packet types. + */ +static const uint32_t * +mrvl_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused) +{ + static const uint32_t ptypes[] = { + RTE_PTYPE_L2_ETHER, + RTE_PTYPE_L3_IPV4, + RTE_PTYPE_L3_IPV4_EXT, + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN, + RTE_PTYPE_L3_IPV6, + RTE_PTYPE_L3_IPV6_EXT, + RTE_PTYPE_L2_ETHER_ARP, + RTE_PTYPE_L4_TCP, + RTE_PTYPE_L4_UDP + }; + + return ptypes; +} + +/** + * DPDK callback to get information about specific receive queue. + * + * @param dev + * Pointer to Ethernet device structure. + * @param rx_queue_id + * Receive queue index. + * @param qinfo + * Receive queue information structure. + */ +static void mrvl_rxq_info_get(struct rte_eth_dev *dev, uint16_t rx_queue_id, + struct rte_eth_rxq_info *qinfo) +{ + struct mrvl_rxq *q = dev->data->rx_queues[rx_queue_id]; + struct mrvl_priv *priv = dev->data->dev_private; + int inq = priv->rxq_map[rx_queue_id].inq; + int tc = priv->rxq_map[rx_queue_id].tc; + struct pp2_ppio_tc_params *tc_params = + &priv->ppio_params.inqs_params.tcs_params[tc]; + + qinfo->mp = q->mp; + qinfo->nb_desc = tc_params->inqs_params[inq].size; +} + +/** + * DPDK callback to get information about specific transmit queue. + * + * @param dev + * Pointer to Ethernet device structure. + * @param tx_queue_id + * Transmit queue index. + * @param qinfo + * Transmit queue information structure. + */ +static void mrvl_txq_info_get(struct rte_eth_dev *dev, uint16_t tx_queue_id, + struct rte_eth_txq_info *qinfo) +{ + struct mrvl_priv *priv = dev->data->dev_private; + + qinfo->nb_desc = + priv->ppio_params.outqs_params.outqs_params[tx_queue_id].size; +} + +/** + * DPDK callback to Configure a VLAN filter. + * + * @param dev + * Pointer to Ethernet device structure. + * @param vlan_id + * VLAN ID to filter. + * @param on + * Toggle filter. + * + * @return + * 0 on success, negative error value otherwise. + */ +static int +mrvl_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) +{ + struct mrvl_priv *priv = dev->data->dev_private; + + return on ? pp2_ppio_add_vlan(priv->ppio, vlan_id) : + pp2_ppio_remove_vlan(priv->ppio, vlan_id); +} + +/** + * Release buffers to hardware bpool (buffer-pool) + * + * @param rxq + * Receive queue pointer. + * @param num + * Number of buffers to release to bpool. + * + * @return + * 0 on success, negative error value otherwise. + */ +static int +mrvl_fill_bpool(struct mrvl_rxq *rxq, int num) +{ + struct buff_release_entry entries[MRVL_PP2_TXD_MAX]; + struct rte_mbuf *mbufs[MRVL_PP2_TXD_MAX]; + int i, ret; + unsigned int core_id; + struct pp2_hif *hif; + struct pp2_bpool *bpool; + + core_id = rte_lcore_id(); + if (core_id == LCORE_ID_ANY) + core_id = 0; + + hif = mrvl_get_hif(rxq->priv, core_id); + if (!hif) + return -1; + + bpool = rxq->priv->bpool; + + ret = rte_pktmbuf_alloc_bulk(rxq->mp, mbufs, num); + if (ret) + return ret; + + if (cookie_addr_high == MRVL_COOKIE_ADDR_INVALID) + cookie_addr_high = + (uint64_t)mbufs[0] & MRVL_COOKIE_HIGH_ADDR_MASK; + + for (i = 0; i < num; i++) { + if (((uint64_t)mbufs[i] & MRVL_COOKIE_HIGH_ADDR_MASK) + != cookie_addr_high) { + RTE_LOG(ERR, PMD, + "mbuf virtual addr high 0x%lx out of range\n", + (uint64_t)mbufs[i] >> 32); + goto out; + } + + entries[i].buff.addr = + rte_mbuf_data_iova_default(mbufs[i]); + entries[i].buff.cookie = (pp2_cookie_t)(uint64_t)mbufs[i]; + entries[i].bpool = bpool; + } + + pp2_bpool_put_buffs(hif, entries, (uint16_t *)&i); + mrvl_port_bpool_size[bpool->pp2_id][bpool->id][core_id] += i; + + if (i != num) + goto out; + + return 0; +out: + for (; i < num; i++) + rte_pktmbuf_free(mbufs[i]); + + return -1; +} + +/** + * DPDK callback to configure the receive queue. + * + * @param dev + * Pointer to Ethernet device structure. + * @param idx + * RX queue index. + * @param desc + * Number of descriptors to configure in queue. + * @param socket + * NUMA socket on which memory must be allocated. + * @param conf + * Thresholds parameters (unused_). + * @param mp + * Memory pool for buffer allocations. + * + * @return + * 0 on success, negative error value otherwise. + */ +static int +mrvl_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, + unsigned int socket, + const struct rte_eth_rxconf *conf __rte_unused, + struct rte_mempool *mp) +{ + struct mrvl_priv *priv = dev->data->dev_private; + struct mrvl_rxq *rxq; + uint32_t min_size, + max_rx_pkt_len = dev->data->dev_conf.rxmode.max_rx_pkt_len; + int ret, tc, inq; + + if (priv->rxq_map[idx].tc == MRVL_UNKNOWN_TC) { + /* + * Unknown TC mapping, mapping will not have a correct queue. + */ + RTE_LOG(ERR, PMD, "Unknown TC mapping for queue %hu eth%hhu\n", + idx, priv->ppio_id); + return -EFAULT; + } + + min_size = rte_pktmbuf_data_room_size(mp) - RTE_PKTMBUF_HEADROOM - + MRVL_PKT_EFFEC_OFFS; + if (min_size < max_rx_pkt_len) { + RTE_LOG(ERR, PMD, + "Mbuf size must be increased to %u bytes to hold up to %u bytes of data.\n", + max_rx_pkt_len + RTE_PKTMBUF_HEADROOM + + MRVL_PKT_EFFEC_OFFS, + max_rx_pkt_len); + return -EINVAL; + } + + if (dev->data->rx_queues[idx]) { + rte_free(dev->data->rx_queues[idx]); + dev->data->rx_queues[idx] = NULL; + } + + rxq = rte_zmalloc_socket("rxq", sizeof(*rxq), 0, socket); + if (!rxq) + return -ENOMEM; + + rxq->priv = priv; + rxq->mp = mp; + rxq->cksum_enabled = dev->data->dev_conf.rxmode.hw_ip_checksum; + rxq->queue_id = idx; + rxq->port_id = dev->data->port_id; + mrvl_port_to_bpool_lookup[rxq->port_id] = priv->bpool; + + tc = priv->rxq_map[rxq->queue_id].tc, + inq = priv->rxq_map[rxq->queue_id].inq; + priv->ppio_params.inqs_params.tcs_params[tc].inqs_params[inq].size = + desc; + + ret = mrvl_fill_bpool(rxq, desc); + if (ret) { + rte_free(rxq); + return ret; + } + + priv->bpool_init_size += desc; + + dev->data->rx_queues[idx] = rxq; + + return 0; +} + +/** + * DPDK callback to release the receive queue. + * + * @param rxq + * Generic receive queue pointer. + */ +static void +mrvl_rx_queue_release(void *rxq) +{ + struct mrvl_rxq *q = rxq; + struct pp2_ppio_tc_params *tc_params; + int i, num, tc, inq; + struct pp2_hif *hif; + unsigned int core_id = rte_lcore_id(); + + if (core_id == LCORE_ID_ANY) + core_id = 0; + + hif = mrvl_get_hif(q->priv, core_id); + + if (!q || !hif) + return; + + tc = q->priv->rxq_map[q->queue_id].tc; + inq = q->priv->rxq_map[q->queue_id].inq; + tc_params = &q->priv->ppio_params.inqs_params.tcs_params[tc]; + num = tc_params->inqs_params[inq].size; + for (i = 0; i < num; i++) { + struct pp2_buff_inf inf; + uint64_t addr; + + pp2_bpool_get_buff(hif, q->priv->bpool, &inf); + addr = cookie_addr_high | inf.cookie; + rte_pktmbuf_free((struct rte_mbuf *)addr); + } + + rte_free(q); +} + +/** + * DPDK callback to configure the transmit queue. + * + * @param dev + * Pointer to Ethernet device structure. + * @param idx + * Transmit queue index. + * @param desc + * Number of descriptors to configure in the queue. + * @param socket + * NUMA socket on which memory must be allocated. + * @param conf + * Thresholds parameters (unused). + * + * @return + * 0 on success, negative error value otherwise. + */ +static int +mrvl_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, + unsigned int socket, + const struct rte_eth_txconf *conf __rte_unused) +{ + struct mrvl_priv *priv = dev->data->dev_private; + struct mrvl_txq *txq; + + if (dev->data->tx_queues[idx]) { + rte_free(dev->data->tx_queues[idx]); + dev->data->tx_queues[idx] = NULL; + } + + txq = rte_zmalloc_socket("txq", sizeof(*txq), 0, socket); + if (!txq) + return -ENOMEM; + + txq->priv = priv; + txq->queue_id = idx; + txq->port_id = dev->data->port_id; + dev->data->tx_queues[idx] = txq; + + priv->ppio_params.outqs_params.outqs_params[idx].size = desc; + priv->ppio_params.outqs_params.outqs_params[idx].weight = 1; + + return 0; +} + +/** + * DPDK callback to release the transmit queue. + * + * @param txq + * Generic transmit queue pointer. + */ +static void +mrvl_tx_queue_release(void *txq) +{ + struct mrvl_txq *q = txq; + + if (!q) + return; + + rte_free(q); +} + +/** + * Update RSS hash configuration + * + * @param dev + * Pointer to Ethernet device structure. + * @param rss_conf + * Pointer to RSS configuration. + * + * @return + * 0 on success, negative error value otherwise. + */ +static int +mrvl_rss_hash_update(struct rte_eth_dev *dev, + struct rte_eth_rss_conf *rss_conf) +{ + struct mrvl_priv *priv = dev->data->dev_private; + + return mrvl_configure_rss(priv, rss_conf); +} + +/** + * DPDK callback to get RSS hash configuration. + * + * @param dev + * Pointer to Ethernet device structure. + * @rss_conf + * Pointer to RSS configuration. + * + * @return + * Always 0. + */ +static int +mrvl_rss_hash_conf_get(struct rte_eth_dev *dev, + struct rte_eth_rss_conf *rss_conf) +{ + struct mrvl_priv *priv = dev->data->dev_private; + enum pp2_ppio_hash_type hash_type = + priv->ppio_params.inqs_params.hash_type; + + rss_conf->rss_key = NULL; + + if (hash_type == PP2_PPIO_HASH_T_NONE) + rss_conf->rss_hf = 0; + else if (hash_type == PP2_PPIO_HASH_T_2_TUPLE) + rss_conf->rss_hf = ETH_RSS_IPV4; + else if (hash_type == PP2_PPIO_HASH_T_5_TUPLE && priv->rss_hf_tcp) + rss_conf->rss_hf = ETH_RSS_NONFRAG_IPV4_TCP; + else if (hash_type == PP2_PPIO_HASH_T_5_TUPLE && !priv->rss_hf_tcp) + rss_conf->rss_hf = ETH_RSS_NONFRAG_IPV4_UDP; + + return 0; +} + +static const struct eth_dev_ops mrvl_ops = { + .dev_configure = mrvl_dev_configure, + .dev_start = mrvl_dev_start, + .dev_stop = mrvl_dev_stop, + .dev_set_link_up = mrvl_dev_set_link_up, + .dev_set_link_down = mrvl_dev_set_link_down, + .dev_close = mrvl_dev_close, + .link_update = mrvl_link_update, + .promiscuous_enable = mrvl_promiscuous_enable, + .allmulticast_enable = mrvl_allmulticast_enable, + .promiscuous_disable = mrvl_promiscuous_disable, + .allmulticast_disable = mrvl_allmulticast_disable, + .mac_addr_remove = mrvl_mac_addr_remove, + .mac_addr_add = mrvl_mac_addr_add, + .mac_addr_set = mrvl_mac_addr_set, + .mtu_set = mrvl_mtu_set, + .stats_get = mrvl_stats_get, + .stats_reset = mrvl_stats_reset, + .dev_infos_get = mrvl_dev_infos_get, + .dev_supported_ptypes_get = mrvl_dev_supported_ptypes_get, + .rxq_info_get = mrvl_rxq_info_get, + .txq_info_get = mrvl_txq_info_get, + .vlan_filter_set = mrvl_vlan_filter_set, + .rx_queue_setup = mrvl_rx_queue_setup, + .rx_queue_release = mrvl_rx_queue_release, + .tx_queue_setup = mrvl_tx_queue_setup, + .tx_queue_release = mrvl_tx_queue_release, + .rss_hash_update = mrvl_rss_hash_update, + .rss_hash_conf_get = mrvl_rss_hash_conf_get, +}; + +/** + * Return packet type information and l3/l4 offsets. + * + * @param desc + * Pointer to the received packet descriptor. + * @param l3_offset + * l3 packet offset. + * @param l4_offset + * l4 packet offset. + * + * @return + * Packet type information. + */ +static inline uint64_t +mrvl_desc_to_packet_type_and_offset(struct pp2_ppio_desc *desc, + uint8_t *l3_offset, uint8_t *l4_offset) +{ + enum pp2_inq_l3_type l3_type; + enum pp2_inq_l4_type l4_type; + uint64_t packet_type; + + pp2_ppio_inq_desc_get_l3_info(desc, &l3_type, l3_offset); + pp2_ppio_inq_desc_get_l4_info(desc, &l4_type, l4_offset); + + packet_type = RTE_PTYPE_L2_ETHER; + + switch (l3_type) { + case PP2_INQ_L3_TYPE_IPV4_NO_OPTS: + packet_type |= RTE_PTYPE_L3_IPV4; + break; + case PP2_INQ_L3_TYPE_IPV4_OK: + packet_type |= RTE_PTYPE_L3_IPV4_EXT; + break; + case PP2_INQ_L3_TYPE_IPV4_TTL_ZERO: + packet_type |= RTE_PTYPE_L3_IPV4_EXT_UNKNOWN; + break; + case PP2_INQ_L3_TYPE_IPV6_NO_EXT: + packet_type |= RTE_PTYPE_L3_IPV6; + break; + case PP2_INQ_L3_TYPE_IPV6_EXT: + packet_type |= RTE_PTYPE_L3_IPV6_EXT; + break; + case PP2_INQ_L3_TYPE_ARP: + packet_type |= RTE_PTYPE_L2_ETHER_ARP; + /* + * In case of ARP l4_offset is set to wrong value. + * Set it to proper one so that later on mbuf->l3_len can be + * calculated subtracting l4_offset and l3_offset. + */ + *l4_offset = *l3_offset + MRVL_ARP_LENGTH; + break; + default: + RTE_LOG(DEBUG, PMD, "Failed to recognise l3 packet type\n"); + break; + } + + switch (l4_type) { + case PP2_INQ_L4_TYPE_TCP: + packet_type |= RTE_PTYPE_L4_TCP; + break; + case PP2_INQ_L4_TYPE_UDP: + packet_type |= RTE_PTYPE_L4_UDP; + break; + default: + RTE_LOG(DEBUG, PMD, "Failed to recognise l4 packet type\n"); + break; + } + + return packet_type; +} + +/** + * Get offload information from the received packet descriptor. + * + * @param desc + * Pointer to the received packet descriptor. + * + * @return + * Mbuf offload flags. + */ +static inline uint64_t +mrvl_desc_to_ol_flags(struct pp2_ppio_desc *desc) +{ + uint64_t flags; + enum pp2_inq_desc_status status; + + status = pp2_ppio_inq_desc_get_l3_pkt_error(desc); + if (unlikely(status != PP2_DESC_ERR_OK)) + flags = PKT_RX_IP_CKSUM_BAD; + else + flags = PKT_RX_IP_CKSUM_GOOD; + + status = pp2_ppio_inq_desc_get_l4_pkt_error(desc); + if (unlikely(status != PP2_DESC_ERR_OK)) + flags |= PKT_RX_L4_CKSUM_BAD; + else + flags |= PKT_RX_L4_CKSUM_GOOD; + + return flags; +} + +/** + * DPDK callback for receive. + * + * @param rxq + * Generic pointer to the receive queue. + * @param rx_pkts + * Array to store received packets. + * @param nb_pkts + * Maximum number of packets in array. + * + * @return + * Number of packets successfully received. + */ +static uint16_t +mrvl_rx_pkt_burst(void *rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) +{ + struct mrvl_rxq *q = rxq; + struct pp2_ppio_desc descs[nb_pkts]; + struct pp2_bpool *bpool; + int i, ret, rx_done = 0; + int num; + struct pp2_hif *hif; + unsigned int core_id = rte_lcore_id(); + + hif = mrvl_get_hif(q->priv, core_id); + + if (unlikely(!q->priv->ppio || !hif)) + return 0; + + bpool = q->priv->bpool; + + ret = pp2_ppio_recv(q->priv->ppio, q->priv->rxq_map[q->queue_id].tc, + q->priv->rxq_map[q->queue_id].inq, descs, &nb_pkts); + if (unlikely(ret < 0)) { + RTE_LOG(ERR, PMD, "Failed to receive packets\n"); + return 0; + } + mrvl_port_bpool_size[bpool->pp2_id][bpool->id][core_id] -= nb_pkts; + + for (i = 0; i < nb_pkts; i++) { + struct rte_mbuf *mbuf; + uint8_t l3_offset, l4_offset; + enum pp2_inq_desc_status status; + uint64_t addr; + + if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) { + struct pp2_ppio_desc *pref_desc; + u64 pref_addr; + + pref_desc = &descs[i + MRVL_MUSDK_PREFETCH_SHIFT]; + pref_addr = cookie_addr_high | + pp2_ppio_inq_desc_get_cookie(pref_desc); + rte_mbuf_prefetch_part1((struct rte_mbuf *)(pref_addr)); + rte_mbuf_prefetch_part2((struct rte_mbuf *)(pref_addr)); + } + + addr = cookie_addr_high | + pp2_ppio_inq_desc_get_cookie(&descs[i]); + mbuf = (struct rte_mbuf *)addr; + rte_pktmbuf_reset(mbuf); + + /* drop packet in case of mac, overrun or resource error */ + status = pp2_ppio_inq_desc_get_l2_pkt_error(&descs[i]); + if (unlikely(status != PP2_DESC_ERR_OK)) { + struct pp2_buff_inf binf = { + .addr = rte_mbuf_data_iova_default(mbuf), + .cookie = (pp2_cookie_t)(uint64_t)mbuf, + }; + + pp2_bpool_put_buff(hif, bpool, &binf); + mrvl_port_bpool_size + [bpool->pp2_id][bpool->id][core_id]++; + q->drop_mac++; + continue; + } + + mbuf->data_off += MRVL_PKT_EFFEC_OFFS; + mbuf->pkt_len = pp2_ppio_inq_desc_get_pkt_len(&descs[i]); + mbuf->data_len = mbuf->pkt_len; + mbuf->port = q->port_id; + mbuf->packet_type = + mrvl_desc_to_packet_type_and_offset(&descs[i], + &l3_offset, + &l4_offset); + mbuf->l2_len = l3_offset; + mbuf->l3_len = l4_offset - l3_offset; + + if (likely(q->cksum_enabled)) + mbuf->ol_flags = mrvl_desc_to_ol_flags(&descs[i]); + + rx_pkts[rx_done++] = mbuf; + q->bytes_recv += mbuf->pkt_len; + } + + if (rte_spinlock_trylock(&q->priv->lock) == 1) { + num = mrvl_get_bpool_size(bpool->pp2_id, bpool->id); + + if (unlikely(num <= q->priv->bpool_min_size || + (!rx_done && num < q->priv->bpool_init_size))) { + ret = mrvl_fill_bpool(q, MRVL_BURST_SIZE); + if (ret) + RTE_LOG(ERR, PMD, "Failed to fill bpool\n"); + } else if (unlikely(num > q->priv->bpool_max_size)) { + int i; + int pkt_to_remove = num - q->priv->bpool_init_size; + struct rte_mbuf *mbuf; + struct pp2_buff_inf buff; + + RTE_LOG(DEBUG, PMD, + "\nport-%d:%d: bpool %d oversize - remove %d buffers (pool size: %d -> %d)\n", + bpool->pp2_id, q->priv->ppio->port_id, + bpool->id, pkt_to_remove, num, + q->priv->bpool_init_size); + + for (i = 0; i < pkt_to_remove; i++) { + ret = pp2_bpool_get_buff(hif, bpool, &buff); + if (ret) + break; + mbuf = (struct rte_mbuf *) + (cookie_addr_high | buff.cookie); + rte_pktmbuf_free(mbuf); + } + mrvl_port_bpool_size + [bpool->pp2_id][bpool->id][core_id] -= i; + } + rte_spinlock_unlock(&q->priv->lock); + } + + return rx_done; +} + +/** + * Prepare offload information. + * + * @param ol_flags + * Offload flags. + * @param packet_type + * Packet type bitfield. + * @param l3_type + * Pointer to the pp2_ouq_l3_type structure. + * @param l4_type + * Pointer to the pp2_outq_l4_type structure. + * @param gen_l3_cksum + * Will be set to 1 in case l3 checksum is computed. + * @param l4_cksum + * Will be set to 1 in case l4 checksum is computed. + * + * @return + * 0 on success, negative error value otherwise. + */ +static inline int +mrvl_prepare_proto_info(uint64_t ol_flags, uint32_t packet_type, + enum pp2_outq_l3_type *l3_type, + enum pp2_outq_l4_type *l4_type, + int *gen_l3_cksum, + int *gen_l4_cksum) +{ + /* + * Based on ol_flags prepare information + * for pp2_ppio_outq_desc_set_proto_info() which setups descriptor + * for offloading. + */ + if (ol_flags & PKT_TX_IPV4) { + *l3_type = PP2_OUTQ_L3_TYPE_IPV4; + *gen_l3_cksum = ol_flags & PKT_TX_IP_CKSUM ? 1 : 0; + } else if (ol_flags & PKT_TX_IPV6) { + *l3_type = PP2_OUTQ_L3_TYPE_IPV6; + /* no checksum for ipv6 header */ + *gen_l3_cksum = 0; + } else { + /* if something different then stop processing */ + return -1; + } + + ol_flags &= PKT_TX_L4_MASK; + if ((packet_type & RTE_PTYPE_L4_TCP) && + ol_flags == PKT_TX_TCP_CKSUM) { + *l4_type = PP2_OUTQ_L4_TYPE_TCP; + *gen_l4_cksum = 1; + } else if ((packet_type & RTE_PTYPE_L4_UDP) && + ol_flags == PKT_TX_UDP_CKSUM) { + *l4_type = PP2_OUTQ_L4_TYPE_UDP; + *gen_l4_cksum = 1; + } else { + *l4_type = PP2_OUTQ_L4_TYPE_OTHER; + /* no checksum for other type */ + *gen_l4_cksum = 0; + } + + return 0; +} + +/** + * Release already sent buffers to bpool (buffer-pool). + * + * @param ppio + * Pointer to the port structure. + * @param hif + * Pointer to the MUSDK hardware interface. + * @param sq + * Pointer to the shadow queue. + * @param qid + * Queue id number. + * @param force + * Force releasing packets. + */ +static inline void +mrvl_free_sent_buffers(struct pp2_ppio *ppio, struct pp2_hif *hif, + unsigned int core_id, struct mrvl_shadow_txq *sq, + int qid, int force) +{ + struct buff_release_entry *entry; + uint16_t nb_done = 0, num = 0, skip_bufs = 0; + int i; + + pp2_ppio_get_num_outq_done(ppio, hif, qid, &nb_done); + + sq->num_to_release += nb_done; + + if (likely(!force && + sq->num_to_release < MRVL_PP2_BUF_RELEASE_BURST_SIZE)) + return; + + nb_done = sq->num_to_release; + sq->num_to_release = 0; + + for (i = 0; i < nb_done; i++) { + entry = &sq->ent[sq->tail + num]; + if (unlikely(!entry->buff.addr)) { + RTE_LOG(ERR, PMD, + "Shadow memory @%d: cookie(%lx), pa(%lx)!\n", + sq->tail, (u64)entry->buff.cookie, + (u64)entry->buff.addr); + skip_bufs = 1; + goto skip; + } + + if (unlikely(!entry->bpool)) { + struct rte_mbuf *mbuf; + + mbuf = (struct rte_mbuf *) + (cookie_addr_high | entry->buff.cookie); + rte_pktmbuf_free(mbuf); + skip_bufs = 1; + goto skip; + } + + mrvl_port_bpool_size + [entry->bpool->pp2_id][entry->bpool->id][core_id]++; + num++; + if (unlikely(sq->tail + num == MRVL_PP2_TX_SHADOWQ_SIZE)) + goto skip; + continue; +skip: + if (likely(num)) + pp2_bpool_put_buffs(hif, &sq->ent[sq->tail], &num); + num += skip_bufs; + sq->tail = (sq->tail + num) & MRVL_PP2_TX_SHADOWQ_MASK; + sq->size -= num; + num = 0; + skip_bufs = 0; + } + + if (likely(num)) { + pp2_bpool_put_buffs(hif, &sq->ent[sq->tail], &num); + sq->tail = (sq->tail + num) & MRVL_PP2_TX_SHADOWQ_MASK; + sq->size -= num; + } +} + +/** + * DPDK callback for transmit. + * + * @param txq + * Generic pointer transmit queue. + * @param tx_pkts + * Packets to transmit. + * @param nb_pkts + * Number of packets in array. + * + * @return + * Number of packets successfully transmitted. + */ +static uint16_t +mrvl_tx_pkt_burst(void *txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) +{ + struct mrvl_txq *q = txq; + struct mrvl_shadow_txq *sq; + struct pp2_hif *hif; + struct pp2_ppio_desc descs[nb_pkts]; + unsigned int core_id = rte_lcore_id(); + int i, ret, bytes_sent = 0; + uint16_t num, sq_free_size; + uint64_t addr; + + hif = mrvl_get_hif(q->priv, core_id); + sq = &q->shadow_txqs[core_id]; + + if (unlikely(!q->priv->ppio || !hif)) + return 0; + + if (sq->size) + mrvl_free_sent_buffers(q->priv->ppio, hif, core_id, + sq, q->queue_id, 0); + + sq_free_size = MRVL_PP2_TX_SHADOWQ_SIZE - sq->size - 1; + if (unlikely(nb_pkts > sq_free_size)) { + RTE_LOG(DEBUG, PMD, + "No room in shadow queue for %d packets! %d packets will be sent.\n", + nb_pkts, sq_free_size); + nb_pkts = sq_free_size; + } + + for (i = 0; i < nb_pkts; i++) { + struct rte_mbuf *mbuf = tx_pkts[i]; + int gen_l3_cksum, gen_l4_cksum; + enum pp2_outq_l3_type l3_type; + enum pp2_outq_l4_type l4_type; + + if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) { + struct rte_mbuf *pref_pkt_hdr; + + pref_pkt_hdr = tx_pkts[i + MRVL_MUSDK_PREFETCH_SHIFT]; + rte_mbuf_prefetch_part1(pref_pkt_hdr); + rte_mbuf_prefetch_part2(pref_pkt_hdr); + } + + sq->ent[sq->head].buff.cookie = (pp2_cookie_t)(uint64_t)mbuf; + sq->ent[sq->head].buff.addr = + rte_mbuf_data_iova_default(mbuf); + sq->ent[sq->head].bpool = + (unlikely(mbuf->port == 0xff || mbuf->refcnt > 1)) ? + NULL : mrvl_port_to_bpool_lookup[mbuf->port]; + sq->head = (sq->head + 1) & MRVL_PP2_TX_SHADOWQ_MASK; + sq->size++; + + pp2_ppio_outq_desc_reset(&descs[i]); + pp2_ppio_outq_desc_set_phys_addr(&descs[i], + rte_pktmbuf_iova(mbuf)); + pp2_ppio_outq_desc_set_pkt_offset(&descs[i], 0); + pp2_ppio_outq_desc_set_pkt_len(&descs[i], + rte_pktmbuf_pkt_len(mbuf)); + + bytes_sent += rte_pktmbuf_pkt_len(mbuf); + /* + * in case unsupported ol_flags were passed + * do not update descriptor offload information + */ + ret = mrvl_prepare_proto_info(mbuf->ol_flags, mbuf->packet_type, + &l3_type, &l4_type, &gen_l3_cksum, + &gen_l4_cksum); + if (unlikely(ret)) + continue; + + pp2_ppio_outq_desc_set_proto_info(&descs[i], l3_type, l4_type, + mbuf->l2_len, + mbuf->l2_len + mbuf->l3_len, + gen_l3_cksum, gen_l4_cksum); + } + + num = nb_pkts; + pp2_ppio_send(q->priv->ppio, hif, q->queue_id, descs, &nb_pkts); + /* number of packets that were not sent */ + if (unlikely(num > nb_pkts)) { + for (i = nb_pkts; i < num; i++) { + sq->head = (MRVL_PP2_TX_SHADOWQ_SIZE + sq->head - 1) & + MRVL_PP2_TX_SHADOWQ_MASK; + addr = cookie_addr_high | sq->ent[sq->head].buff.cookie; + bytes_sent -= + rte_pktmbuf_pkt_len((struct rte_mbuf *)addr); + } + sq->size -= num - nb_pkts; + } + + q->bytes_sent += bytes_sent; + + return nb_pkts; +} + +/** + * Initialize packet processor. + * + * @return + * 0 on success, negative error value otherwise. + */ +static int +mrvl_init_pp2(void) +{ + struct pp2_init_params init_params; + + memset(&init_params, 0, sizeof(init_params)); + init_params.hif_reserved_map = MRVL_MUSDK_HIFS_RESERVED; + init_params.bm_pool_reserved_map = MRVL_MUSDK_BPOOLS_RESERVED; + init_params.rss_tbl_reserved_map = MRVL_MUSDK_RSS_RESERVED; + + return pp2_init(&init_params); +} + +/** + * Deinitialize packet processor. + * + * @return + * 0 on success, negative error value otherwise. + */ +static void +mrvl_deinit_pp2(void) +{ + pp2_deinit(); +} + +/** + * Create private device structure. + * + * @param dev_name + * Pointer to the port name passed in the initialization parameters. + * + * @return + * Pointer to the newly allocated private device structure. + */ +static struct mrvl_priv * +mrvl_priv_create(const char *dev_name) +{ + struct pp2_bpool_params bpool_params; + char match[MRVL_MATCH_LEN]; + struct mrvl_priv *priv; + int ret, bpool_bit; + + priv = rte_zmalloc_socket(dev_name, sizeof(*priv), 0, rte_socket_id()); + if (!priv) + return NULL; + + ret = pp2_netdev_get_ppio_info((char *)(uintptr_t)dev_name, + &priv->pp_id, &priv->ppio_id); + if (ret) + goto out_free_priv; + + bpool_bit = mrvl_reserve_bit(&used_bpools[priv->pp_id], + PP2_BPOOL_NUM_POOLS); + if (bpool_bit < 0) + goto out_free_priv; + priv->bpool_bit = bpool_bit; + + snprintf(match, sizeof(match), "pool-%d:%d", priv->pp_id, + priv->bpool_bit); + memset(&bpool_params, 0, sizeof(bpool_params)); + bpool_params.match = match; + bpool_params.buff_len = MRVL_PKT_SIZE_MAX + MRVL_PKT_EFFEC_OFFS; + ret = pp2_bpool_init(&bpool_params, &priv->bpool); + if (ret) + goto out_clear_bpool_bit; + + priv->ppio_params.type = PP2_PPIO_T_NIC; + rte_spinlock_init(&priv->lock); + + return priv; +out_clear_bpool_bit: + used_bpools[priv->pp_id] &= ~(1 << priv->bpool_bit); +out_free_priv: + rte_free(priv); + return NULL; +} + +/** + * Create device representing Ethernet port. + * + * @param name + * Pointer to the port's name. + * + * @return + * 0 on success, negative error value otherwise. + */ +static int +mrvl_eth_dev_create(struct rte_vdev_device *vdev, const char *name) +{ + int ret, fd = socket(AF_INET, SOCK_DGRAM, 0); + struct rte_eth_dev *eth_dev; + struct mrvl_priv *priv; + struct ifreq req; + + eth_dev = rte_eth_dev_allocate(name); + if (!eth_dev) + return -ENOMEM; + + priv = mrvl_priv_create(name); + if (!priv) { + ret = -ENOMEM; + goto out_free_dev; + } + + eth_dev->data->mac_addrs = + rte_zmalloc("mac_addrs", + ETHER_ADDR_LEN * MRVL_MAC_ADDRS_MAX, 0); + if (!eth_dev->data->mac_addrs) { + RTE_LOG(ERR, PMD, "Failed to allocate space for eth addrs\n"); + ret = -ENOMEM; + goto out_free_priv; + } + + memset(&req, 0, sizeof(req)); + strcpy(req.ifr_name, name); + ret = ioctl(fd, SIOCGIFHWADDR, &req); + if (ret) + goto out_free_mac; + + memcpy(eth_dev->data->mac_addrs[0].addr_bytes, + req.ifr_addr.sa_data, ETHER_ADDR_LEN); + + eth_dev->rx_pkt_burst = mrvl_rx_pkt_burst; + eth_dev->tx_pkt_burst = mrvl_tx_pkt_burst; + eth_dev->data->kdrv = RTE_KDRV_NONE; + eth_dev->data->dev_private = priv; + eth_dev->device = &vdev->device; + eth_dev->dev_ops = &mrvl_ops; + + return 0; +out_free_mac: + rte_free(eth_dev->data->mac_addrs); +out_free_dev: + rte_eth_dev_release_port(eth_dev); +out_free_priv: + rte_free(priv); + + return ret; +} + +/** + * Cleanup previously created device representing Ethernet port. + * + * @param name + * Pointer to the port name. + */ +static void +mrvl_eth_dev_destroy(const char *name) +{ + struct rte_eth_dev *eth_dev; + struct mrvl_priv *priv; + + eth_dev = rte_eth_dev_allocated(name); + if (!eth_dev) + return; + + priv = eth_dev->data->dev_private; + pp2_bpool_deinit(priv->bpool); + used_bpools[priv->pp_id] &= ~(1 << priv->bpool_bit); + rte_free(priv); + rte_free(eth_dev->data->mac_addrs); + rte_eth_dev_release_port(eth_dev); +} + +/** + * Callback used by rte_kvargs_process() during argument parsing. + * + * @param key + * Pointer to the parsed key (unused). + * @param value + * Pointer to the parsed value. + * @param extra_args + * Pointer to the extra arguments which contains address of the + * table of pointers to parsed interface names. + * + * @return + * Always 0. + */ +static int +mrvl_get_ifnames(const char *key __rte_unused, const char *value, + void *extra_args) +{ + struct mrvl_ifnames *ifnames = extra_args; + + ifnames->names[ifnames->idx++] = value; + + return 0; +} + +/** + * Deinitialize per-lcore MUSDK hardware interfaces (hifs). + */ +static void +mrvl_deinit_hifs(void) +{ + int i; + + for (i = mrvl_lcore_first; i <= mrvl_lcore_last; i++) { + if (hifs[i]) + pp2_hif_deinit(hifs[i]); + } + used_hifs = MRVL_MUSDK_HIFS_RESERVED; + memset(hifs, 0, sizeof(hifs)); +} + +/** + * DPDK callback to register the virtual device. + * + * @param vdev + * Pointer to the virtual device. + * + * @return + * 0 on success, negative error value otherwise. + */ +static int +rte_pmd_mrvl_probe(struct rte_vdev_device *vdev) +{ + struct rte_kvargs *kvlist; + struct mrvl_ifnames ifnames; + int ret = -EINVAL; + uint32_t i, ifnum, cfgnum; + const char *params; + + params = rte_vdev_device_args(vdev); + if (!params) + return -EINVAL; + + kvlist = rte_kvargs_parse(params, valid_args); + if (!kvlist) + return -EINVAL; + + ifnum = rte_kvargs_count(kvlist, MRVL_IFACE_NAME_ARG); + if (ifnum > RTE_DIM(ifnames.names)) + goto out_free_kvlist; + + ifnames.idx = 0; + rte_kvargs_process(kvlist, MRVL_IFACE_NAME_ARG, + mrvl_get_ifnames, &ifnames); + + + /* + * The below system initialization should be done only once, + * on the first provided configuration file + */ + if (!mrvl_qos_cfg) { + cfgnum = rte_kvargs_count(kvlist, MRVL_CFG_ARG); + RTE_LOG(INFO, PMD, "Parsing config file!\n"); + if (cfgnum > 1) { + RTE_LOG(ERR, PMD, "Cannot handle more than one config file!\n"); + goto out_free_kvlist; + } else if (cfgnum == 1) { + rte_kvargs_process(kvlist, MRVL_CFG_ARG, + mrvl_get_qoscfg, &mrvl_qos_cfg); + } + } + + if (mrvl_dev_num) + goto init_devices; + + RTE_LOG(INFO, PMD, "Perform MUSDK initializations\n"); + /* + * ret == -EEXIST is correct, it means DMA + * has been already initialized (by another PMD). + */ + ret = mv_sys_dma_mem_init(MRVL_MUSDK_DMA_MEMSIZE); + if (ret < 0) { + if (ret != -EEXIST) + goto out_free_kvlist; + else + RTE_LOG(INFO, PMD, + "DMA memory has been already initialized by a different driver.\n"); + } + + ret = mrvl_init_pp2(); + if (ret) { + RTE_LOG(ERR, PMD, "Failed to init PP!\n"); + goto out_deinit_dma; + } + + memset(mrvl_port_bpool_size, 0, sizeof(mrvl_port_bpool_size)); + + mrvl_lcore_first = RTE_MAX_LCORE; + mrvl_lcore_last = 0; + +init_devices: + for (i = 0; i < ifnum; i++) { + RTE_LOG(INFO, PMD, "Creating %s\n", ifnames.names[i]); + ret = mrvl_eth_dev_create(vdev, ifnames.names[i]); + if (ret) + goto out_cleanup; + } + mrvl_dev_num += ifnum; + + rte_kvargs_free(kvlist); + + return 0; +out_cleanup: + for (; i > 0; i--) + mrvl_eth_dev_destroy(ifnames.names[i]); + + if (mrvl_dev_num == 0) + mrvl_deinit_pp2(); +out_deinit_dma: + if (mrvl_dev_num == 0) + mv_sys_dma_mem_destroy(); +out_free_kvlist: + rte_kvargs_free(kvlist); + + return ret; +} + +/** + * DPDK callback to remove virtual device. + * + * @param vdev + * Pointer to the removed virtual device. + * + * @return + * 0 on success, negative error value otherwise. + */ +static int +rte_pmd_mrvl_remove(struct rte_vdev_device *vdev) +{ + int i; + const char *name; + + name = rte_vdev_device_name(vdev); + if (!name) + return -EINVAL; + + RTE_LOG(INFO, PMD, "Removing %s\n", name); + + for (i = 0; i < rte_eth_dev_count(); i++) { + char ifname[RTE_ETH_NAME_MAX_LEN]; + + rte_eth_dev_get_name_by_port(i, ifname); + mrvl_eth_dev_destroy(ifname); + mrvl_dev_num--; + } + + if (mrvl_dev_num == 0) { + RTE_LOG(INFO, PMD, "Perform MUSDK deinit\n"); + mrvl_deinit_hifs(); + mrvl_deinit_pp2(); + mv_sys_dma_mem_destroy(); + } + + return 0; +} + +static struct rte_vdev_driver pmd_mrvl_drv = { + .probe = rte_pmd_mrvl_probe, + .remove = rte_pmd_mrvl_remove, +}; + +RTE_PMD_REGISTER_VDEV(net_mrvl, pmd_mrvl_drv); +RTE_PMD_REGISTER_ALIAS(net_mrvl, eth_mrvl); diff --git a/dpdk/drivers/net/mrvl/mrvl_ethdev.h b/dpdk/drivers/net/mrvl/mrvl_ethdev.h new file mode 100644 index 00000000..7764da14 --- /dev/null +++ b/dpdk/drivers/net/mrvl/mrvl_ethdev.h @@ -0,0 +1,113 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Marvell International Ltd. + * Copyright(c) 2017 Semihalf. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _MRVL_ETHDEV_H_ +#define _MRVL_ETHDEV_H_ + +#include +#include +#include + +/** Maximum number of rx queues per port */ +#define MRVL_PP2_RXQ_MAX 32 + +/** Maximum number of tx queues per port */ +#define MRVL_PP2_TXQ_MAX 8 + +/** Minimum number of descriptors in tx queue */ +#define MRVL_PP2_TXD_MIN 16 + +/** Maximum number of descriptors in tx queue */ +#define MRVL_PP2_TXD_MAX 2048 + +/** Tx queue descriptors alignment */ +#define MRVL_PP2_TXD_ALIGN 16 + +/** Minimum number of descriptors in rx queue */ +#define MRVL_PP2_RXD_MIN 16 + +/** Maximum number of descriptors in rx queue */ +#define MRVL_PP2_RXD_MAX 2048 + +/** Rx queue descriptors alignment */ +#define MRVL_PP2_RXD_ALIGN 16 + +/** Maximum number of descriptors in tx aggregated queue */ +#define MRVL_PP2_AGGR_TXQD_MAX 2048 + +/** Maximum number of Traffic Classes. */ +#define MRVL_PP2_TC_MAX 8 + +/** Packet offset inside RX buffer. */ +#define MRVL_PKT_OFFS 64 + +/** Maximum number of descriptors in shadow queue. Must be power of 2 */ +#define MRVL_PP2_TX_SHADOWQ_SIZE MRVL_PP2_TXD_MAX + +/** Shadow queue size mask (since shadow queue size is power of 2) */ +#define MRVL_PP2_TX_SHADOWQ_MASK (MRVL_PP2_TX_SHADOWQ_SIZE - 1) + +/** Minimum number of sent buffers to release from shadow queue to BM */ +#define MRVL_PP2_BUF_RELEASE_BURST_SIZE 64 + +struct mrvl_priv { + /* Hot fields, used in fast path. */ + struct pp2_bpool *bpool; /**< BPool pointer */ + struct pp2_ppio *ppio; /**< Port handler pointer */ + rte_spinlock_t lock; /**< Spinlock for checking bpool status */ + uint16_t bpool_max_size; /**< BPool maximum size */ + uint16_t bpool_min_size; /**< BPool minimum size */ + uint16_t bpool_init_size; /**< Configured BPool size */ + + /** Mapping for DPDK rx queue->(TC, MRVL relative inq) */ + struct { + uint8_t tc; /**< Traffic Class */ + uint8_t inq; /**< Relative in-queue number */ + } rxq_map[MRVL_PP2_RXQ_MAX] __rte_cache_aligned; + + /* Configuration data, used sporadically. */ + uint8_t pp_id; + uint8_t ppio_id; + uint8_t bpool_bit; + uint8_t rss_hf_tcp; + uint8_t uc_mc_flushed; + uint8_t vlan_flushed; + + struct pp2_ppio_params ppio_params; + struct pp2_cls_qos_tbl_params qos_tbl_params; + struct pp2_cls_tbl *qos_tbl; + uint16_t nb_rx_queues; +}; + +#endif /* _MRVL_ETHDEV_H_ */ diff --git a/dpdk/drivers/net/mrvl/mrvl_qos.c b/dpdk/drivers/net/mrvl/mrvl_qos.c new file mode 100644 index 00000000..fbb36813 --- /dev/null +++ b/dpdk/drivers/net/mrvl/mrvl_qos.c @@ -0,0 +1,636 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Marvell International Ltd. + * Copyright(c) 2017 Semihalf. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +/* Unluckily, container_of is defined by both DPDK and MUSDK, + * we'll declare only one version. + * + * Note that it is not used in this PMD anyway. + */ +#ifdef container_of +#undef container_of +#endif + +#include "mrvl_qos.h" + +/* Parsing tokens. Defined conveniently, so that any correction is easy. */ +#define MRVL_TOK_DEFAULT "default" +#define MRVL_TOK_DEFAULT_TC "default_tc" +#define MRVL_TOK_DSCP "dscp" +#define MRVL_TOK_MAPPING_PRIORITY "mapping_priority" +#define MRVL_TOK_IP "ip" +#define MRVL_TOK_IP_VLAN "ip/vlan" +#define MRVL_TOK_PCP "pcp" +#define MRVL_TOK_PORT "port" +#define MRVL_TOK_RXQ "rxq" +#define MRVL_TOK_SP "SP" +#define MRVL_TOK_TC "tc" +#define MRVL_TOK_TXQ "txq" +#define MRVL_TOK_VLAN "vlan" +#define MRVL_TOK_VLAN_IP "vlan/ip" +#define MRVL_TOK_WEIGHT "weight" + +/** Number of tokens in range a-b = 2. */ +#define MAX_RNG_TOKENS 2 + +/** Maximum possible value of PCP. */ +#define MAX_PCP 7 + +/** Maximum possible value of DSCP. */ +#define MAX_DSCP 63 + +/** Global QoS configuration. */ +struct mrvl_qos_cfg *mrvl_qos_cfg; + +/** + * Convert string to uint32_t with extra checks for result correctness. + * + * @param string String to convert. + * @param val Conversion result. + * @returns 0 in case of success, negative value otherwise. + */ +static int +get_val_securely(const char *string, uint32_t *val) +{ + char *endptr; + size_t len = strlen(string); + + if (len == 0) + return -1; + + errno = 0; + *val = strtoul(string, &endptr, 0); + if (errno != 0 || RTE_PTR_DIFF(endptr, string) != len) + return -2; + + return 0; +} + +/** + * Read out-queue configuration from file. + * + * @param file Path to the configuration file. + * @param port Port number. + * @param outq Out queue number. + * @param cfg Pointer to the Marvell QoS configuration structure. + * @returns 0 in case of success, negative value otherwise. + */ +static int +get_outq_cfg(struct rte_cfgfile *file, int port, int outq, + struct mrvl_qos_cfg *cfg) +{ + char sec_name[32]; + const char *entry; + uint32_t val; + + snprintf(sec_name, sizeof(sec_name), "%s %d %s %d", + MRVL_TOK_PORT, port, MRVL_TOK_TXQ, outq); + + /* Skip non-existing */ + if (rte_cfgfile_num_sections(file, sec_name, strlen(sec_name)) <= 0) + return 0; + + entry = rte_cfgfile_get_entry(file, sec_name, + MRVL_TOK_WEIGHT); + if (entry) { + if (get_val_securely(entry, &val) < 0) + return -1; + cfg->port[port].outq[outq].weight = (uint8_t)val; + } + + return 0; +} + +/** + * Gets multiple-entry values and places them in table. + * + * Entry can be anything, e.g. "1 2-3 5 6 7-9". This needs to be converted to + * table entries, respectively: {1, 2, 3, 5, 6, 7, 8, 9}. + * As all result table's elements are always 1-byte long, we + * won't overcomplicate the function, but we'll keep API generic, + * check if someone hasn't changed element size and make it simple + * to extend to other sizes. + * + * This function is purely utilitary, it does not print any error, only returns + * different error numbers. + * + * @param entry[in] Values string to parse. + * @param tab[out] Results table. + * @param elem_sz[in] Element size (in bytes). + * @param max_elems[in] Number of results table elements available. + * @param max val[in] Maximum value allowed. + * @returns Number of correctly parsed elements in case of success. + * @retval -1 Wrong element size. + * @retval -2 More tokens than result table allows. + * @retval -3 Wrong range syntax. + * @retval -4 Wrong range values. + * @retval -5 Maximum value exceeded. + */ +static int +get_entry_values(const char *entry, uint8_t *tab, + size_t elem_sz, uint8_t max_elems, uint8_t max_val) +{ + /* There should not be more tokens than max elements. + * Add 1 for error trap. + */ + char *tokens[max_elems + 1]; + + /* Begin, End + error trap = 3. */ + char *rng_tokens[MAX_RNG_TOKENS + 1]; + long beg, end; + uint32_t token_val; + int nb_tokens, nb_rng_tokens; + int i; + int values = 0; + char val; + char entry_cpy[CFG_VALUE_LEN]; + + if (elem_sz != 1) + return -1; + + /* Copy the entry to safely use rte_strsplit(). */ + snprintf(entry_cpy, RTE_DIM(entry_cpy), "%s", entry); + + /* + * If there are more tokens than array size, rte_strsplit will + * not return error, just array size. + */ + nb_tokens = rte_strsplit(entry_cpy, strlen(entry_cpy), + tokens, max_elems + 1, ' '); + + /* Quick check, will be refined later. */ + if (nb_tokens > max_elems) + return -2; + + for (i = 0; i < nb_tokens; ++i) { + if (strchr(tokens[i], '-') != NULL) { + /* + * Split to begin and end tokens. + * We want to catch error cases too, thus we leave + * option for number of tokens to be more than 2. + */ + nb_rng_tokens = rte_strsplit(tokens[i], + strlen(tokens[i]), rng_tokens, + RTE_DIM(rng_tokens), '-'); + if (nb_rng_tokens != 2) + return -3; + + /* Range and sanity checks. */ + if (get_val_securely(rng_tokens[0], &token_val) < 0) + return -4; + beg = (char)token_val; + if (get_val_securely(rng_tokens[1], &token_val) < 0) + return -4; + end = (char)token_val; + if (beg < 0 || beg > UCHAR_MAX || + end < 0 || end > UCHAR_MAX || end < beg) + return -4; + + for (val = beg; val <= end; ++val) { + if (val > max_val) + return -5; + + *tab = val; + tab = RTE_PTR_ADD(tab, elem_sz); + ++values; + if (values >= max_elems) + return -2; + } + } else { + /* Single values. */ + if (get_val_securely(tokens[i], &token_val) < 0) + return -5; + val = (char)token_val; + if (val > max_val) + return -5; + + *tab = val; + tab = RTE_PTR_ADD(tab, elem_sz); + ++values; + if (values >= max_elems) + return -2; + } + } + + return values; +} + +/** + * Parse Traffic Class'es mapping configuration. + * + * @param file Config file handle. + * @param port Which port to look for. + * @param tc Which Traffic Class to look for. + * @param cfg[out] Parsing results. + * @returns 0 in case of success, negative value otherwise. + */ +static int +parse_tc_cfg(struct rte_cfgfile *file, int port, int tc, + struct mrvl_qos_cfg *cfg) +{ + char sec_name[32]; + const char *entry; + int n; + + snprintf(sec_name, sizeof(sec_name), "%s %d %s %d", + MRVL_TOK_PORT, port, MRVL_TOK_TC, tc); + + /* Skip non-existing */ + if (rte_cfgfile_num_sections(file, sec_name, strlen(sec_name)) <= 0) + return 0; + + entry = rte_cfgfile_get_entry(file, sec_name, MRVL_TOK_RXQ); + if (entry) { + n = get_entry_values(entry, + cfg->port[port].tc[tc].inq, + sizeof(cfg->port[port].tc[tc].inq[0]), + RTE_DIM(cfg->port[port].tc[tc].inq), + MRVL_PP2_RXQ_MAX); + if (n < 0) { + RTE_LOG(ERR, PMD, "Error %d while parsing: %s\n", + n, entry); + return n; + } + cfg->port[port].tc[tc].inqs = n; + } + + entry = rte_cfgfile_get_entry(file, sec_name, MRVL_TOK_PCP); + if (entry) { + n = get_entry_values(entry, + cfg->port[port].tc[tc].pcp, + sizeof(cfg->port[port].tc[tc].pcp[0]), + RTE_DIM(cfg->port[port].tc[tc].pcp), + MAX_PCP); + if (n < 0) { + RTE_LOG(ERR, PMD, "Error %d while parsing: %s\n", + n, entry); + return n; + } + cfg->port[port].tc[tc].pcps = n; + } + + entry = rte_cfgfile_get_entry(file, sec_name, MRVL_TOK_DSCP); + if (entry) { + n = get_entry_values(entry, + cfg->port[port].tc[tc].dscp, + sizeof(cfg->port[port].tc[tc].dscp[0]), + RTE_DIM(cfg->port[port].tc[tc].dscp), + MAX_DSCP); + if (n < 0) { + RTE_LOG(ERR, PMD, "Error %d while parsing: %s\n", + n, entry); + return n; + } + cfg->port[port].tc[tc].dscps = n; + } + return 0; +} + +/** + * Parse QoS configuration - rte_kvargs_process handler. + * + * Opens configuration file and parses its content. + * + * @param key Unused. + * @param path Path to config file. + * @param extra_args Pointer to configuration structure. + * @returns 0 in case of success, exits otherwise. + */ +int +mrvl_get_qoscfg(const char *key __rte_unused, const char *path, + void *extra_args) +{ + struct mrvl_qos_cfg **cfg = extra_args; + struct rte_cfgfile *file = rte_cfgfile_load(path, 0); + uint32_t val; + int n, i, ret; + const char *entry; + char sec_name[32]; + + if (file == NULL) + rte_exit(EXIT_FAILURE, "Cannot load configuration %s\n", path); + + /* Create configuration. This is never accessed on the fast path, + * so we can ignore socket. + */ + *cfg = rte_zmalloc("mrvl_qos_cfg", sizeof(struct mrvl_qos_cfg), 0); + if (*cfg == NULL) + rte_exit(EXIT_FAILURE, "Cannot allocate configuration %s\n", + path); + + n = rte_cfgfile_num_sections(file, MRVL_TOK_PORT, + sizeof(MRVL_TOK_PORT) - 1); + + if (n == 0) { + /* This is weird, but not bad. */ + RTE_LOG(WARNING, PMD, "Empty configuration file?\n"); + return 0; + } + + /* Use the number of ports given as vdev parameters. */ + for (n = 0; n < (PP2_NUM_ETH_PPIO * PP2_NUM_PKT_PROC); ++n) { + snprintf(sec_name, sizeof(sec_name), "%s %d %s", + MRVL_TOK_PORT, n, MRVL_TOK_DEFAULT); + + /* Skip ports non-existing in configuration. */ + if (rte_cfgfile_num_sections(file, sec_name, + strlen(sec_name)) <= 0) { + (*cfg)->port[n].use_global_defaults = 1; + (*cfg)->port[n].mapping_priority = + PP2_CLS_QOS_TBL_VLAN_IP_PRI; + continue; + } + + entry = rte_cfgfile_get_entry(file, sec_name, + MRVL_TOK_DEFAULT_TC); + if (entry) { + if (get_val_securely(entry, &val) < 0 || + val > USHRT_MAX) + return -1; + (*cfg)->port[n].default_tc = (uint8_t)val; + } else { + RTE_LOG(ERR, PMD, + "Default Traffic Class required in custom configuration!\n"); + return -1; + } + + entry = rte_cfgfile_get_entry(file, sec_name, + MRVL_TOK_MAPPING_PRIORITY); + if (entry) { + if (!strncmp(entry, MRVL_TOK_VLAN_IP, + sizeof(MRVL_TOK_VLAN_IP))) + (*cfg)->port[n].mapping_priority = + PP2_CLS_QOS_TBL_VLAN_IP_PRI; + else if (!strncmp(entry, MRVL_TOK_IP_VLAN, + sizeof(MRVL_TOK_IP_VLAN))) + (*cfg)->port[n].mapping_priority = + PP2_CLS_QOS_TBL_IP_VLAN_PRI; + else if (!strncmp(entry, MRVL_TOK_IP, + sizeof(MRVL_TOK_IP))) + (*cfg)->port[n].mapping_priority = + PP2_CLS_QOS_TBL_IP_PRI; + else if (!strncmp(entry, MRVL_TOK_VLAN, + sizeof(MRVL_TOK_VLAN))) + (*cfg)->port[n].mapping_priority = + PP2_CLS_QOS_TBL_VLAN_PRI; + else + rte_exit(EXIT_FAILURE, + "Error in parsing %s value (%s)!\n", + MRVL_TOK_MAPPING_PRIORITY, entry); + } else { + (*cfg)->port[n].mapping_priority = + PP2_CLS_QOS_TBL_VLAN_IP_PRI; + } + + for (i = 0; i < MRVL_PP2_RXQ_MAX; ++i) { + ret = get_outq_cfg(file, n, i, *cfg); + if (ret < 0) + rte_exit(EXIT_FAILURE, + "Error %d parsing port %d outq %d!\n", + ret, n, i); + } + + for (i = 0; i < MRVL_PP2_TC_MAX; ++i) { + ret = parse_tc_cfg(file, n, i, *cfg); + if (ret < 0) + rte_exit(EXIT_FAILURE, + "Error %d parsing port %d tc %d!\n", + ret, n, i); + } + } + + return 0; +} + +/** + * Setup Traffic Class. + * + * Fill in TC parameters in single MUSDK TC config entry. + * @param param TC parameters entry. + * @param inqs Number of MUSDK in-queues in this TC. + * @param bpool Bpool for this TC. + * @returns 0 in case of success, exits otherwise. + */ +static int +setup_tc(struct pp2_ppio_tc_params *param, uint8_t inqs, + struct pp2_bpool *bpool) +{ + struct pp2_ppio_inq_params *inq_params; + + param->pkt_offset = MRVL_PKT_OFFS; + param->pools[0] = bpool; + + inq_params = rte_zmalloc_socket("inq_params", + inqs * sizeof(*inq_params), + 0, rte_socket_id()); + if (!inq_params) + return -ENOMEM; + + param->num_in_qs = inqs; + + /* Release old config if necessary. */ + if (param->inqs_params) + rte_free(param->inqs_params); + + param->inqs_params = inq_params; + + return 0; +} + +/** + * Configure RX Queues in a given port. + * + * Sets up RX queues, their Traffic Classes and DPDK rxq->(TC,inq) mapping. + * + * @param priv Port's private data + * @param portid DPDK port ID + * @param max_queues Maximum number of queues to configure. + * @returns 0 in case of success, negative value otherwise. + */ +int +mrvl_configure_rxqs(struct mrvl_priv *priv, uint16_t portid, + uint16_t max_queues) +{ + size_t i, tc; + + if (mrvl_qos_cfg == NULL || + mrvl_qos_cfg->port[portid].use_global_defaults) { + /* No port configuration, use default: 1 TC, no QoS. */ + priv->ppio_params.inqs_params.num_tcs = 1; + setup_tc(&priv->ppio_params.inqs_params.tcs_params[0], + max_queues, priv->bpool); + + /* Direct mapping of queues i.e. 0->0, 1->1 etc. */ + for (i = 0; i < max_queues; ++i) { + priv->rxq_map[i].tc = 0; + priv->rxq_map[i].inq = i; + } + return 0; + } + + /* We need only a subset of configuration. */ + struct port_cfg *port_cfg = &mrvl_qos_cfg->port[portid]; + + priv->qos_tbl_params.type = port_cfg->mapping_priority; + + /* + * We need to reverse mapping, from tc->pcp (better from usability + * point of view) to pcp->tc (configurable in MUSDK). + * First, set all map elements to "default". + */ + for (i = 0; i < RTE_DIM(priv->qos_tbl_params.pcp_cos_map); ++i) + priv->qos_tbl_params.pcp_cos_map[i].tc = port_cfg->default_tc; + + /* Then, fill in all known values. */ + for (tc = 0; tc < RTE_DIM(port_cfg->tc); ++tc) { + if (port_cfg->tc[tc].pcps > RTE_DIM(port_cfg->tc[0].pcp)) { + /* Better safe than sorry. */ + RTE_LOG(ERR, PMD, + "Too many PCPs configured in TC %zu!\n", tc); + return -1; + } + for (i = 0; i < port_cfg->tc[tc].pcps; ++i) { + priv->qos_tbl_params.pcp_cos_map[ + port_cfg->tc[tc].pcp[i]].tc = tc; + } + } + + /* + * The same logic goes with DSCP. + * First, set all map elements to "default". + */ + for (i = 0; i < RTE_DIM(priv->qos_tbl_params.dscp_cos_map); ++i) + priv->qos_tbl_params.dscp_cos_map[i].tc = + port_cfg->default_tc; + + /* Fill in all known values. */ + for (tc = 0; tc < RTE_DIM(port_cfg->tc); ++tc) { + if (port_cfg->tc[tc].dscps > RTE_DIM(port_cfg->tc[0].dscp)) { + /* Better safe than sorry. */ + RTE_LOG(ERR, PMD, + "Too many DSCPs configured in TC %zu!\n", tc); + return -1; + } + for (i = 0; i < port_cfg->tc[tc].dscps; ++i) { + priv->qos_tbl_params.dscp_cos_map[ + port_cfg->tc[tc].dscp[i]].tc = tc; + } + } + + /* + * Surprisingly, similar logic goes with queue mapping. + * We need only to store qid->tc mapping, + * to know TC when queue is read. + */ + for (i = 0; i < RTE_DIM(priv->rxq_map); ++i) + priv->rxq_map[i].tc = MRVL_UNKNOWN_TC; + + /* Set up DPDKq->(TC,inq) mapping. */ + for (tc = 0; tc < RTE_DIM(port_cfg->tc); ++tc) { + if (port_cfg->tc[tc].inqs > RTE_DIM(port_cfg->tc[0].inq)) { + /* Overflow. */ + RTE_LOG(ERR, PMD, + "Too many RX queues configured per TC %zu!\n", + tc); + return -1; + } + for (i = 0; i < port_cfg->tc[tc].inqs; ++i) { + uint8_t idx = port_cfg->tc[tc].inq[i]; + + if (idx > RTE_DIM(priv->rxq_map)) { + RTE_LOG(ERR, PMD, "Bad queue index %d!\n", idx); + return -1; + } + + priv->rxq_map[idx].tc = tc; + priv->rxq_map[idx].inq = i; + } + } + + /* + * Set up TC configuration. TCs need to be sequenced: 0, 1, 2 + * with no gaps. Empty TC means end of processing. + */ + for (i = 0; i < MRVL_PP2_TC_MAX; ++i) { + if (port_cfg->tc[i].inqs == 0) + break; + setup_tc(&priv->ppio_params.inqs_params.tcs_params[i], + port_cfg->tc[i].inqs, + priv->bpool); + } + + priv->ppio_params.inqs_params.num_tcs = i; + + return 0; +} + +/** + * Start QoS mapping. + * + * Finalize QoS table configuration and initialize it in SDK. It can be done + * only after port is started, so we have a valid ppio reference. + * + * @param priv Port's private (configuration) data. + * @returns 0 in case of success, exits otherwise. + */ +int +mrvl_start_qos_mapping(struct mrvl_priv *priv) +{ + size_t i; + + if (priv->ppio == NULL) { + RTE_LOG(ERR, PMD, "ppio must not be NULL here!\n"); + return -1; + } + + for (i = 0; i < RTE_DIM(priv->qos_tbl_params.pcp_cos_map); ++i) + priv->qos_tbl_params.pcp_cos_map[i].ppio = priv->ppio; + + for (i = 0; i < RTE_DIM(priv->qos_tbl_params.dscp_cos_map); ++i) + priv->qos_tbl_params.dscp_cos_map[i].ppio = priv->ppio; + + /* Initialize Classifier QoS table. */ + + return pp2_cls_qos_tbl_init(&priv->qos_tbl_params, &priv->qos_tbl); +} diff --git a/dpdk/drivers/net/mrvl/mrvl_qos.h b/dpdk/drivers/net/mrvl/mrvl_qos.h new file mode 100644 index 00000000..ae7508c9 --- /dev/null +++ b/dpdk/drivers/net/mrvl/mrvl_qos.h @@ -0,0 +1,113 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Marvell International Ltd. + * Copyright(c) 2017 Semihalf. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _MRVL_QOS_H_ +#define _MRVL_QOS_H_ + +#include + +#include "mrvl_ethdev.h" + +/** Code Points per Traffic Class. Equals max(DSCP, PCP). */ +#define MRVL_CP_PER_TC (64) + +/** Value used as "unknown". */ +#define MRVL_UNKNOWN_TC (0xFF) + +/* QoS config. */ +struct mrvl_qos_cfg { + struct port_cfg { + struct { + uint8_t inq[MRVL_PP2_RXQ_MAX]; + uint8_t dscp[MRVL_CP_PER_TC]; + uint8_t pcp[MRVL_CP_PER_TC]; + uint8_t inqs; + uint8_t dscps; + uint8_t pcps; + } tc[MRVL_PP2_TC_MAX]; + struct { + uint8_t weight; + } outq[MRVL_PP2_RXQ_MAX]; + enum pp2_cls_qos_tbl_type mapping_priority; + uint16_t inqs; + uint16_t outqs; + uint8_t default_tc; + uint8_t use_global_defaults; + } port[RTE_MAX_ETHPORTS]; +}; + +/** Global QoS configuration. */ +extern struct mrvl_qos_cfg *mrvl_qos_cfg; + +/** + * Parse QoS configuration - rte_kvargs_process handler. + * + * Opens configuration file and parses its content. + * + * @param key Unused. + * @param path Path to config file. + * @param extra_args Pointer to configuration structure. + * @returns 0 in case of success, exits otherwise. + */ +int +mrvl_get_qoscfg(const char *key __rte_unused, const char *path, + void *extra_args); + +/** + * Configure RX Queues in a given port. + * + * Sets up RX queues, their Traffic Classes and DPDK rxq->(TC,inq) mapping. + * + * @param priv Port's private data + * @param portid DPDK port ID + * @param max_queues Maximum number of queues to configure. + * @returns 0 in case of success, negative value otherwise. + */ +int +mrvl_configure_rxqs(struct mrvl_priv *priv, uint16_t portid, + uint16_t max_queues); + +/** + * Start QoS mapping. + * + * Finalize QoS table configuration and initialize it in SDK. It can be done + * only after port is started, so we have a valid ppio reference. + * + * @param priv Port's private (configuration) data. + * @returns 0 in case of success, exits otherwise. + */ +int +mrvl_start_qos_mapping(struct mrvl_priv *priv); + +#endif /* _MRVL_QOS_H_ */ diff --git a/dpdk/drivers/net/mrvl/rte_pmd_mrvl_version.map b/dpdk/drivers/net/mrvl/rte_pmd_mrvl_version.map new file mode 100644 index 00000000..a7530317 --- /dev/null +++ b/dpdk/drivers/net/mrvl/rte_pmd_mrvl_version.map @@ -0,0 +1,3 @@ +DPDK_17.11 { + local: *; +}; diff --git a/dpdk/drivers/net/nfp/Makefile b/dpdk/drivers/net/nfp/Makefile index 4cadd131..4ba066ac 100644 --- a/dpdk/drivers/net/nfp/Makefile +++ b/dpdk/drivers/net/nfp/Makefile @@ -40,6 +40,9 @@ CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) LDLIBS += -lm +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs +LDLIBS += -lrte_bus_pci EXPORT_MAP := rte_pmd_nfp_version.map @@ -49,10 +52,7 @@ LIBABIVER := 1 # all source are stored in SRCS-y # SRCS-$(CONFIG_RTE_LIBRTE_NFP_PMD) += nfp_net.c - -# this lib depends upon: -DEPDIRS-$(CONFIG_RTE_LIBRTE_NFP_PMD) += lib/librte_eal lib/librte_ether -DEPDIRS-$(CONFIG_RTE_LIBRTE_NFP_PMD) += lib/librte_mempool lib/librte_mbuf -DEPDIRS-$(CONFIG_RTE_LIBRTE_NFP_PMD) += lib/librte_net +SRCS-$(CONFIG_RTE_LIBRTE_NFP_PMD) += nfp_nfpu.c +SRCS-$(CONFIG_RTE_LIBRTE_NFP_PMD) += nfp_nspu.c include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/net/nfp/nfp_net.c b/dpdk/drivers/net/nfp/nfp_net.c index 815296cb..fa8ff3cb 100644 --- a/dpdk/drivers/net/nfp/nfp_net.c +++ b/dpdk/drivers/net/nfp/nfp_net.c @@ -39,13 +39,12 @@ * Netronome vNIC DPDK Poll-Mode Driver: Main entry point */ -#include - #include #include #include #include #include +#include #include #include #include @@ -56,6 +55,7 @@ #include #include +#include "nfp_nfpu.h" #include "nfp_net_pmd.h" #include "nfp_net_logs.h" #include "nfp_net_ctrl.h" @@ -63,8 +63,7 @@ /* Prototypes */ static void nfp_net_close(struct rte_eth_dev *dev); static int nfp_net_configure(struct rte_eth_dev *dev); -static void nfp_net_dev_interrupt_handler(struct rte_intr_handle *handle, - void *param); +static void nfp_net_dev_interrupt_handler(void *param); static void nfp_net_dev_interrupt_delayed_handler(void *param); static int nfp_net_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu); static void nfp_net_infos_get(struct rte_eth_dev *dev, @@ -89,7 +88,7 @@ static int nfp_net_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, uint16_t nb_desc, unsigned int socket_id, const struct rte_eth_txconf *tx_conf); static int nfp_net_start(struct rte_eth_dev *dev); -static void nfp_net_stats_get(struct rte_eth_dev *dev, +static int nfp_net_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats); static void nfp_net_stats_reset(struct rte_eth_dev *dev); static void nfp_net_stop(struct rte_eth_dev *dev); @@ -101,13 +100,13 @@ static uint16_t nfp_net_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, * happen to be at the same offset on the NFP6000 and the NFP3200 so * we use a single macro here. */ -#define NFP_PCIE_QUEUE(_q) (0x80000 + (0x800 * ((_q) & 0xff))) +#define NFP_PCIE_QUEUE(_q) (0x800 * ((_q) & 0xff)) /* Maximum value which can be added to a queue with one transaction */ #define NFP_QCP_MAX_ADD 0x7f #define RTE_MBUF_DMA_ADDR_DEFAULT(mb) \ - (uint64_t)((mb)->buf_physaddr + RTE_PKTMBUF_HEADROOM) + (uint64_t)((mb)->buf_iova + RTE_PKTMBUF_HEADROOM) /* nfp_qcp_ptr - Read or Write Pointer of a queue */ enum nfp_qcp_ptr { @@ -205,26 +204,6 @@ nn_cfg_writeq(struct nfp_net_hw *hw, int off, uint64_t val) nn_writeq(rte_cpu_to_le_64(val), hw->ctrl_bar + off); } -/* Creating memzone for hardware rings. */ -static const struct rte_memzone * -ring_dma_zone_reserve(struct rte_eth_dev *dev, const char *ring_name, - uint16_t queue_id, uint32_t ring_size, int socket_id) -{ - char z_name[RTE_MEMZONE_NAMESIZE]; - const struct rte_memzone *mz; - - snprintf(z_name, sizeof(z_name), "%s_%s_%d_%d", - dev->driver->pci_drv.name, - ring_name, dev->data->port_id, queue_id); - - mz = rte_memzone_lookup(z_name); - if (mz) - return mz; - - return rte_memzone_reserve_aligned(z_name, ring_size, socket_id, 0, - NFP_MEMZONE_ALIGN); -} - /* * Atomically reads link status information from global structure rte_eth_dev. * @@ -308,7 +287,6 @@ static void nfp_net_reset_rx_queue(struct nfp_net_rxq *rxq) { nfp_net_rx_queue_release_mbufs(rxq); - rxq->wr_p = 0; rxq->rd_p = 0; rxq->nb_rx_hold = 0; } @@ -347,8 +325,6 @@ nfp_net_reset_tx_queue(struct nfp_net_txq *txq) nfp_net_tx_queue_release_mbufs(txq); txq->wr_p = 0; txq->rd_p = 0; - txq->tail = 0; - txq->qcp_rd_p = 0; } static int @@ -377,12 +353,12 @@ __nfp_net_reconfig(struct nfp_net_hw *hw, uint32_t update) if (new == 0) break; if (new & NFP_NET_CFG_UPDATE_ERR) { - PMD_INIT_LOG(ERR, "Reconfig error: 0x%08x\n", new); + PMD_INIT_LOG(ERR, "Reconfig error: 0x%08x", new); return -1; } if (cnt >= NFP_NET_POLL_TIMEOUT) { PMD_INIT_LOG(ERR, "Reconfig timeout for 0x%08x after" - " %dms\n", update, cnt); + " %dms", update, cnt); rte_panic("Exiting\n"); } nanosleep(&wait, 0); /* waiting for a 1ms */ @@ -426,7 +402,7 @@ nfp_net_reconfig(struct nfp_net_hw *hw, uint32_t ctrl, uint32_t update) * Reconfig errors imply situations where they can be handled. * Otherwise, rte_panic is called inside __nfp_net_reconfig */ - PMD_INIT_LOG(ERR, "Error nfp_net reconfig for ctrl: %x update: %x\n", + PMD_INIT_LOG(ERR, "Error nfp_net reconfig for ctrl: %x update: %x", ctrl, update); return -EIO; } @@ -456,7 +432,7 @@ nfp_net_configure(struct rte_eth_dev *dev) * called after that internal process */ - PMD_INIT_LOG(DEBUG, "Configure\n"); + PMD_INIT_LOG(DEBUG, "Configure"); dev_conf = &dev->data->dev_conf; rxmode = &dev_conf->rxmode; @@ -464,7 +440,7 @@ nfp_net_configure(struct rte_eth_dev *dev) /* Checking TX mode */ if (txmode->mq_mode) { - PMD_INIT_LOG(INFO, "TX mq_mode DCB and VMDq not supported\n"); + PMD_INIT_LOG(INFO, "TX mq_mode DCB and VMDq not supported"); return -EINVAL; } @@ -474,13 +450,13 @@ nfp_net_configure(struct rte_eth_dev *dev) update = NFP_NET_CFG_UPDATE_RSS; new_ctrl = NFP_NET_CFG_CTRL_RSS; } else { - PMD_INIT_LOG(INFO, "RSS not supported\n"); + PMD_INIT_LOG(INFO, "RSS not supported"); return -EINVAL; } } if (rxmode->split_hdr_size) { - PMD_INIT_LOG(INFO, "rxmode does not support split header\n"); + PMD_INIT_LOG(INFO, "rxmode does not support split header"); return -EINVAL; } @@ -488,13 +464,13 @@ nfp_net_configure(struct rte_eth_dev *dev) if (hw->cap & NFP_NET_CFG_CTRL_RXCSUM) { new_ctrl |= NFP_NET_CFG_CTRL_RXCSUM; } else { - PMD_INIT_LOG(INFO, "RXCSUM not supported\n"); + PMD_INIT_LOG(INFO, "RXCSUM not supported"); return -EINVAL; } } if (rxmode->hw_vlan_filter) { - PMD_INIT_LOG(INFO, "VLAN filter not supported\n"); + PMD_INIT_LOG(INFO, "VLAN filter not supported"); return -EINVAL; } @@ -502,32 +478,52 @@ nfp_net_configure(struct rte_eth_dev *dev) if (hw->cap & NFP_NET_CFG_CTRL_RXVLAN) { new_ctrl |= NFP_NET_CFG_CTRL_RXVLAN; } else { - PMD_INIT_LOG(INFO, "hw vlan strip not supported\n"); + PMD_INIT_LOG(INFO, "hw vlan strip not supported"); return -EINVAL; } } if (rxmode->hw_vlan_extend) { - PMD_INIT_LOG(INFO, "VLAN extended not supported\n"); + PMD_INIT_LOG(INFO, "VLAN extended not supported"); return -EINVAL; } - /* Supporting VLAN insertion by default */ + if (rxmode->jumbo_frame) + hw->mtu = rxmode->max_rx_pkt_len; + + if (!rxmode->hw_strip_crc) + PMD_INIT_LOG(INFO, "HW does strip CRC and it is not configurable"); + + if (rxmode->enable_scatter) { + PMD_INIT_LOG(INFO, "Scatter not supported"); + return -EINVAL; + } + + /* If next capabilities are supported, configure them by default */ + + /* VLAN insertion */ if (hw->cap & NFP_NET_CFG_CTRL_TXVLAN) new_ctrl |= NFP_NET_CFG_CTRL_TXVLAN; - if (rxmode->jumbo_frame) - /* this is handled in rte_eth_dev_configure */ + /* L2 broadcast */ + if (hw->cap & NFP_NET_CFG_CTRL_L2BC) + new_ctrl |= NFP_NET_CFG_CTRL_L2BC; - if (rxmode->hw_strip_crc) { - PMD_INIT_LOG(INFO, "strip CRC not supported\n"); - return -EINVAL; - } + /* L2 multicast */ + if (hw->cap & NFP_NET_CFG_CTRL_L2MC) + new_ctrl |= NFP_NET_CFG_CTRL_L2MC; - if (rxmode->enable_scatter) { - PMD_INIT_LOG(INFO, "Scatter not supported\n"); - return -EINVAL; - } + /* TX checksum offload */ + if (hw->cap & NFP_NET_CFG_CTRL_TXCSUM) + new_ctrl |= NFP_NET_CFG_CTRL_TXCSUM; + + /* LSO offload */ + if (hw->cap & NFP_NET_CFG_CTRL_LSO) + new_ctrl |= NFP_NET_CFG_CTRL_LSO; + + /* RX gather */ + if (hw->cap & NFP_NET_CFG_CTRL_GATHER) + new_ctrl |= NFP_NET_CFG_CTRL_GATHER; if (!new_ctrl) return 0; @@ -607,18 +603,8 @@ nfp_net_rx_freelist_setup(struct rte_eth_dev *dev) static void nfp_net_params_setup(struct nfp_net_hw *hw) { - uint32_t *mac_address; - nn_cfg_writel(hw, NFP_NET_CFG_MTU, hw->mtu); nn_cfg_writel(hw, NFP_NET_CFG_FLBUFSZ, hw->flbufsz); - - /* A MAC address is 8 bytes long */ - mac_address = (uint32_t *)(hw->mac_addr); - - nn_cfg_writel(hw, NFP_NET_CFG_MACADDR, - rte_cpu_to_be_32(*mac_address)); - nn_cfg_writel(hw, NFP_NET_CFG_MACADDR + 4, - rte_cpu_to_be_32(*(mac_address + 4))); } static void @@ -627,16 +613,136 @@ nfp_net_cfg_queue_setup(struct nfp_net_hw *hw) hw->qcp_cfg = hw->tx_bar + NFP_QCP_QUEUE_ADDR_SZ; } +#define ETH_ADDR_LEN 6 + +static void +nfp_eth_copy_mac_reverse(uint8_t *dst, const uint8_t *src) +{ + int i; + + for (i = 0; i < ETH_ADDR_LEN; i++) + dst[ETH_ADDR_LEN - i - 1] = src[i]; +} + +static int +nfp_net_pf_read_mac(struct nfp_net_hw *hw, int port) +{ + union eth_table_entry *entry; + int idx, i; + + idx = port; + entry = hw->eth_table; + + /* Reading NFP ethernet table obtained before */ + for (i = 0; i < NSP_ETH_MAX_COUNT; i++) { + if (!(entry->port & NSP_ETH_PORT_LANES_MASK)) { + /* port not in use */ + entry++; + continue; + } + if (idx == 0) + break; + idx--; + entry++; + } + + if (i == NSP_ETH_MAX_COUNT) + return -EINVAL; + + /* + * hw points to port0 private data. We need hw now pointing to + * right port. + */ + hw += port; + nfp_eth_copy_mac_reverse((uint8_t *)&hw->mac_addr, + (uint8_t *)&entry->mac_addr); + + return 0; +} + +static void +nfp_net_vf_read_mac(struct nfp_net_hw *hw) +{ + uint32_t tmp; + + tmp = rte_be_to_cpu_32(nn_cfg_readl(hw, NFP_NET_CFG_MACADDR)); + memcpy(&hw->mac_addr[0], &tmp, sizeof(struct ether_addr)); + + tmp = rte_be_to_cpu_32(nn_cfg_readl(hw, NFP_NET_CFG_MACADDR + 4)); + memcpy(&hw->mac_addr[4], &tmp, 2); +} + +static void +nfp_net_write_mac(struct nfp_net_hw *hw, uint8_t *mac) +{ + uint32_t mac0 = *(uint32_t *)mac; + uint16_t mac1; + + nn_writel(rte_cpu_to_be_32(mac0), hw->ctrl_bar + NFP_NET_CFG_MACADDR); + + mac += 4; + mac1 = *(uint16_t *)mac; + nn_writew(rte_cpu_to_be_16(mac1), + hw->ctrl_bar + NFP_NET_CFG_MACADDR + 6); +} + +static int +nfp_configure_rx_interrupt(struct rte_eth_dev *dev, + struct rte_intr_handle *intr_handle) +{ + struct nfp_net_hw *hw; + int i; + + if (!intr_handle->intr_vec) { + intr_handle->intr_vec = + rte_zmalloc("intr_vec", + dev->data->nb_rx_queues * sizeof(int), 0); + if (!intr_handle->intr_vec) { + PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues" + " intr_vec", dev->data->nb_rx_queues); + return -ENOMEM; + } + } + + hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); + + if (intr_handle->type == RTE_INTR_HANDLE_UIO) { + PMD_INIT_LOG(INFO, "VF: enabling RX interrupt with UIO"); + /* UIO just supports one queue and no LSC*/ + nn_cfg_writeb(hw, NFP_NET_CFG_RXR_VEC(0), 0); + intr_handle->intr_vec[0] = 0; + } else { + PMD_INIT_LOG(INFO, "VF: enabling RX interrupt with VFIO"); + for (i = 0; i < dev->data->nb_rx_queues; i++) { + /* + * The first msix vector is reserved for non + * efd interrupts + */ + nn_cfg_writeb(hw, NFP_NET_CFG_RXR_VEC(i), i + 1); + intr_handle->intr_vec[i] = i + 1; + PMD_INIT_LOG(DEBUG, "intr_vec[%d]= %d\n", i, + intr_handle->intr_vec[i]); + } + } + + /* Avoiding TX interrupts */ + hw->ctrl |= NFP_NET_CFG_CTRL_MSIX_TX_OFF; + return 0; +} + static int nfp_net_start(struct rte_eth_dev *dev) { + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; uint32_t new_ctrl, update = 0; struct nfp_net_hw *hw; + uint32_t intr_vector; int ret; hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); - PMD_INIT_LOG(DEBUG, "Start\n"); + PMD_INIT_LOG(DEBUG, "Start"); /* Disabling queues just in case... */ nfp_net_disable_queues(dev); @@ -647,9 +753,41 @@ nfp_net_start(struct rte_eth_dev *dev) /* Enabling the required queues in the device */ nfp_net_enable_queues(dev); + /* check and configure queue intr-vector mapping */ + if (dev->data->dev_conf.intr_conf.rxq != 0) { + if (hw->pf_multiport_enabled) { + PMD_INIT_LOG(ERR, "PMD rx interrupt is not supported " + "with NFP multiport PF"); + return -EINVAL; + } + if (intr_handle->type == RTE_INTR_HANDLE_UIO) { + /* + * Better not to share LSC with RX interrupts. + * Unregistering LSC interrupt handler + */ + rte_intr_callback_unregister(&pci_dev->intr_handle, + nfp_net_dev_interrupt_handler, (void *)dev); + + if (dev->data->nb_rx_queues > 1) { + PMD_INIT_LOG(ERR, "PMD rx interrupt only " + "supports 1 queue with UIO"); + return -EIO; + } + } + intr_vector = dev->data->nb_rx_queues; + if (rte_intr_efd_enable(intr_handle, intr_vector)) + return -1; + + nfp_configure_rx_interrupt(dev, intr_handle); + update = NFP_NET_CFG_UPDATE_MSIX; + } + + rte_intr_enable(intr_handle); + /* Enable device */ - new_ctrl = hw->ctrl | NFP_NET_CFG_CTRL_ENABLE | NFP_NET_CFG_UPDATE_MSIX; - update = NFP_NET_CFG_UPDATE_GEN | NFP_NET_CFG_UPDATE_RING; + new_ctrl = hw->ctrl | NFP_NET_CFG_CTRL_ENABLE; + + update |= NFP_NET_CFG_UPDATE_GEN | NFP_NET_CFG_UPDATE_RING; if (hw->cap & NFP_NET_CFG_CTRL_RINGCFG) new_ctrl |= NFP_NET_CFG_CTRL_RINGCFG; @@ -667,6 +805,10 @@ nfp_net_start(struct rte_eth_dev *dev) goto error; } + if (hw->is_pf) + /* Configure the physical port up */ + nfp_nsp_eth_config(hw->nspu_desc, hw->pf_port_idx, 1); + hw->ctrl = new_ctrl; return 0; @@ -695,8 +837,11 @@ static void nfp_net_stop(struct rte_eth_dev *dev) { int i; + struct nfp_net_hw *hw; - PMD_INIT_LOG(DEBUG, "Stop\n"); + PMD_INIT_LOG(DEBUG, "Stop"); + + hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); nfp_net_disable_queues(dev); @@ -710,6 +855,10 @@ nfp_net_stop(struct rte_eth_dev *dev) nfp_net_reset_rx_queue( (struct nfp_net_rxq *)dev->data->rx_queues[i]); } + + if (hw->is_pf) + /* Configure the physical port down */ + nfp_nsp_eth_config(hw->nspu_desc, hw->pf_port_idx, 0); } /* Reset and stop device. The device can not be restarted. */ @@ -717,21 +866,40 @@ static void nfp_net_close(struct rte_eth_dev *dev) { struct nfp_net_hw *hw; + struct rte_pci_device *pci_dev; + int i; - PMD_INIT_LOG(DEBUG, "Close\n"); + PMD_INIT_LOG(DEBUG, "Close"); hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); + pci_dev = RTE_ETH_DEV_TO_PCI(dev); /* * We assume that the DPDK application is stopping all the * threads/queues before calling the device close function. */ - nfp_net_stop(dev); + nfp_net_disable_queues(dev); - rte_intr_disable(&dev->pci_dev->intr_handle); + /* Clear queues */ + for (i = 0; i < dev->data->nb_tx_queues; i++) { + nfp_net_reset_tx_queue( + (struct nfp_net_txq *)dev->data->tx_queues[i]); + } + + for (i = 0; i < dev->data->nb_rx_queues; i++) { + nfp_net_reset_rx_queue( + (struct nfp_net_rxq *)dev->data->rx_queues[i]); + } + + rte_intr_disable(&pci_dev->intr_handle); nn_cfg_writeb(hw, NFP_NET_CFG_LSC, 0xff); + /* unregister callback func from eal lib */ + rte_intr_callback_unregister(&pci_dev->intr_handle, + nfp_net_dev_interrupt_handler, + (void *)dev); + /* * The ixgbe PMD driver disables the pcie master on the * device. The i40e does not... @@ -749,7 +917,7 @@ nfp_net_promisc_enable(struct rte_eth_dev *dev) hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); if (!(hw->cap & NFP_NET_CFG_CTRL_PROMISC)) { - PMD_INIT_LOG(INFO, "Promiscuous mode not supported\n"); + PMD_INIT_LOG(INFO, "Promiscuous mode not supported"); return; } @@ -810,6 +978,17 @@ nfp_net_link_update(struct rte_eth_dev *dev, __rte_unused int wait_to_complete) struct rte_eth_link link, old; uint32_t nn_link_status; + static const uint32_t ls_to_ethtool[] = { + [NFP_NET_CFG_STS_LINK_RATE_UNSUPPORTED] = ETH_SPEED_NUM_NONE, + [NFP_NET_CFG_STS_LINK_RATE_UNKNOWN] = ETH_SPEED_NUM_NONE, + [NFP_NET_CFG_STS_LINK_RATE_1G] = ETH_SPEED_NUM_1G, + [NFP_NET_CFG_STS_LINK_RATE_10G] = ETH_SPEED_NUM_10G, + [NFP_NET_CFG_STS_LINK_RATE_25G] = ETH_SPEED_NUM_25G, + [NFP_NET_CFG_STS_LINK_RATE_40G] = ETH_SPEED_NUM_40G, + [NFP_NET_CFG_STS_LINK_RATE_50G] = ETH_SPEED_NUM_50G, + [NFP_NET_CFG_STS_LINK_RATE_100G] = ETH_SPEED_NUM_100G, + }; + PMD_DRV_LOG(DEBUG, "Link update\n"); hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); @@ -825,8 +1004,14 @@ nfp_net_link_update(struct rte_eth_dev *dev, __rte_unused int wait_to_complete) link.link_status = ETH_LINK_UP; link.link_duplex = ETH_LINK_FULL_DUPLEX; - /* Other cards can limit the tx and rx rate per VF */ - link.link_speed = ETH_SPEED_NUM_40G; + + nn_link_status = (nn_link_status >> NFP_NET_CFG_STS_LINK_RATE_SHIFT) & + NFP_NET_CFG_STS_LINK_RATE_MASK; + + if (nn_link_status >= RTE_DIM(ls_to_ethtool)) + link.link_speed = ETH_SPEED_NUM_NONE; + else + link.link_speed = ls_to_ethtool[nn_link_status]; if (old.link_status != link.link_status) { nfp_net_dev_atomic_write_link_status(dev, &link); @@ -840,7 +1025,7 @@ nfp_net_link_update(struct rte_eth_dev *dev, __rte_unused int wait_to_complete) return -1; } -static void +static int nfp_net_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) { int i; @@ -851,6 +1036,8 @@ nfp_net_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) /* RTE_ETHDEV_QUEUE_STAT_CNTRS default value is 16 */ + memset(&nfp_dev_stats, 0, sizeof(nfp_dev_stats)); + /* reading per RX ring stats */ for (i = 0; i < dev->data->nb_rx_queues; i++) { if (i == RTE_ETHDEV_QUEUE_STAT_CNTRS) @@ -926,8 +1113,11 @@ nfp_net_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) nfp_dev_stats.imissed -= hw->eth_stats_base.imissed; - if (stats) + if (stats) { memcpy(stats, &nfp_dev_stats, sizeof(*stats)); + return 0; + } + return -EINVAL; } static void @@ -1000,11 +1190,11 @@ nfp_net_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); - dev_info->driver_name = dev->driver->pci_drv.name; + dev_info->pci_dev = RTE_ETH_DEV_TO_PCI(dev); dev_info->max_rx_queues = (uint16_t)hw->max_rx_queues; dev_info->max_tx_queues = (uint16_t)hw->max_tx_queues; dev_info->min_rx_bufsize = ETHER_MIN_MTU; - dev_info->max_rx_pktlen = hw->mtu; + dev_info->max_rx_pktlen = hw->max_mtu; /* Next should change when PF support is implemented */ dev_info->max_mac_addrs = 1; @@ -1021,8 +1211,8 @@ nfp_net_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) if (hw->cap & NFP_NET_CFG_CTRL_TXCSUM) dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_IPV4_CKSUM | - DEV_RX_OFFLOAD_UDP_CKSUM | - DEV_RX_OFFLOAD_TCP_CKSUM; + DEV_TX_OFFLOAD_UDP_CKSUM | + DEV_TX_OFFLOAD_TCP_CKSUM; dev_info->default_rxconf = (struct rte_eth_rxconf) { .rx_thresh = { @@ -1046,10 +1236,20 @@ nfp_net_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) ETH_TXQ_FLAGS_NOOFFLOADS, }; + dev_info->flow_type_rss_offloads = ETH_RSS_NONFRAG_IPV4_TCP | + ETH_RSS_NONFRAG_IPV4_UDP | + ETH_RSS_NONFRAG_IPV6_TCP | + ETH_RSS_NONFRAG_IPV6_UDP; + dev_info->reta_size = NFP_NET_CFG_RSS_ITBL_SZ; dev_info->hash_key_size = NFP_NET_CFG_RSS_KEY_SZ; - dev_info->speed_capa = ETH_LINK_SPEED_40G | ETH_LINK_SPEED_100G; + dev_info->speed_capa = ETH_SPEED_NUM_1G | ETH_LINK_SPEED_10G | + ETH_SPEED_NUM_25G | ETH_SPEED_NUM_40G | + ETH_SPEED_NUM_50G | ETH_LINK_SPEED_100G; + + if (hw->cap & NFP_NET_CFG_CTRL_LSO) + dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_TCP_TSO; } static const uint32_t * @@ -1079,13 +1279,7 @@ nfp_net_rx_queue_count(struct rte_eth_dev *dev, uint16_t queue_idx) rxq = (struct nfp_net_rxq *)dev->data->rx_queues[queue_idx]; - if (rxq == NULL) { - PMD_INIT_LOG(ERR, "Bad queue: %u\n", queue_idx); - return 0; - } - - idx = rxq->rd_p % rxq->rx_count; - rxds = &rxq->rxds[idx]; + idx = rxq->rd_p; count = 0; @@ -1093,7 +1287,7 @@ nfp_net_rx_queue_count(struct rte_eth_dev *dev, uint16_t queue_idx) * Other PMDs are just checking the DD bit in intervals of 4 * descriptors and counting all four if the first has the DD * bit on. Of course, this is not accurate but can be good for - * perfomance. But ideally that should be done in descriptors + * performance. But ideally that should be done in descriptors * chunks belonging to the same cache line */ @@ -1113,25 +1307,65 @@ nfp_net_rx_queue_count(struct rte_eth_dev *dev, uint16_t queue_idx) return count; } +static int +nfp_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) +{ + struct rte_pci_device *pci_dev; + struct nfp_net_hw *hw; + int base = 0; + + hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); + pci_dev = RTE_ETH_DEV_TO_PCI(dev); + + if (pci_dev->intr_handle.type != RTE_INTR_HANDLE_UIO) + base = 1; + + /* Make sure all updates are written before un-masking */ + rte_wmb(); + nn_cfg_writeb(hw, NFP_NET_CFG_ICR(base + queue_id), + NFP_NET_CFG_ICR_UNMASKED); + return 0; +} + +static int +nfp_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id) +{ + struct rte_pci_device *pci_dev; + struct nfp_net_hw *hw; + int base = 0; + + hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); + pci_dev = RTE_ETH_DEV_TO_PCI(dev); + + if (pci_dev->intr_handle.type != RTE_INTR_HANDLE_UIO) + base = 1; + + /* Make sure all updates are written before un-masking */ + rte_wmb(); + nn_cfg_writeb(hw, NFP_NET_CFG_ICR(base + queue_id), 0x1); + return 0; +} + static void nfp_net_dev_link_status_print(struct rte_eth_dev *dev) { + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); struct rte_eth_link link; memset(&link, 0, sizeof(link)); nfp_net_dev_atomic_read_link_status(dev, &link); if (link.link_status) RTE_LOG(INFO, PMD, "Port %d: Link Up - speed %u Mbps - %s\n", - (int)(dev->data->port_id), (unsigned)link.link_speed, + dev->data->port_id, link.link_speed, link.link_duplex == ETH_LINK_FULL_DUPLEX ? "full-duplex" : "half-duplex"); else RTE_LOG(INFO, PMD, " Port %d: Link Down\n", - (int)(dev->data->port_id)); + dev->data->port_id); RTE_LOG(INFO, PMD, "PCI Address: %04d:%02d:%02d:%d\n", - dev->pci_dev->addr.domain, dev->pci_dev->addr.bus, - dev->pci_dev->addr.devid, dev->pci_dev->addr.function); + pci_dev->addr.domain, pci_dev->addr.bus, + pci_dev->addr.devid, pci_dev->addr.function); } /* Interrupt configuration and handling */ @@ -1146,13 +1380,15 @@ static void nfp_net_irq_unmask(struct rte_eth_dev *dev) { struct nfp_net_hw *hw; + struct rte_pci_device *pci_dev; hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); + pci_dev = RTE_ETH_DEV_TO_PCI(dev); if (hw->ctrl & NFP_NET_CFG_CTRL_MSIXAUTO) { /* If MSI-X auto-masking is used, clear the entry */ rte_wmb(); - rte_intr_enable(&dev->pci_dev->intr_handle); + rte_intr_enable(&pci_dev->intr_handle); } else { /* Make sure all updates are written before un-masking */ rte_wmb(); @@ -1162,8 +1398,7 @@ nfp_net_irq_unmask(struct rte_eth_dev *dev) } static void -nfp_net_dev_interrupt_handler(__rte_unused struct rte_intr_handle *handle, - void *param) +nfp_net_dev_interrupt_handler(void *param) { int64_t timeout; struct rte_eth_link link; @@ -1213,7 +1448,7 @@ nfp_net_dev_interrupt_delayed_handler(void *param) struct rte_eth_dev *dev = (struct rte_eth_dev *)param; nfp_net_link_update(dev, 0); - _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC); + _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL, NULL); nfp_net_dev_link_status_print(dev); @@ -1232,6 +1467,13 @@ nfp_net_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) if ((mtu < ETHER_MIN_MTU) || ((uint32_t)mtu > hw->max_mtu)) return -EINVAL; + /* mtu setting is forbidden if port is started */ + if (dev->data->dev_started) { + PMD_DRV_LOG(ERR, "port %d must be stopped before configuration", + dev->data->port_id); + return -EBUSY; + } + /* switch to jumbo mode if needed */ if ((uint32_t)mtu > ETHER_MAX_LEN) dev->data->dev_conf.rxmode.jumbo_frame = 1; @@ -1315,9 +1557,10 @@ nfp_net_rx_queue_setup(struct rte_eth_dev *dev, * handle the maximum ring size is allocated in order to allow for * resizing in later calls to the queue setup function. */ - tz = ring_dma_zone_reserve(dev, "rx_ring", queue_idx, + tz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_idx, sizeof(struct nfp_net_rx_desc) * - NFP_NET_MAX_RX_DESC, socket_id); + NFP_NET_MAX_RX_DESC, NFP_MEMZONE_ALIGN, + socket_id); if (tz == NULL) { RTE_LOG(ERR, PMD, "Error allocatig rx dma\n"); @@ -1326,7 +1569,7 @@ nfp_net_rx_queue_setup(struct rte_eth_dev *dev, } /* Saving physical and virtual addresses for the RX ring */ - rxq->dma = (uint64_t)tz->phys_addr; + rxq->dma = (uint64_t)tz->iova; rxq->rxds = (struct nfp_net_rx_desc *)tz->addr; /* mbuf pointers array for referencing mbufs linked to RX descriptors */ @@ -1351,7 +1594,7 @@ nfp_net_rx_queue_setup(struct rte_eth_dev *dev, * of descriptors in log2 format */ nn_cfg_writeq(hw, NFP_NET_CFG_RXR_ADDR(queue_idx), rxq->dma); - nn_cfg_writeb(hw, NFP_NET_CFG_RXR_SZ(queue_idx), log2(nb_desc)); + nn_cfg_writeb(hw, NFP_NET_CFG_RXR_SZ(queue_idx), rte_log2_u32(nb_desc)); return 0; } @@ -1384,8 +1627,6 @@ nfp_net_rx_fill_freelist(struct nfp_net_rxq *rxq) rxd->fld.dma_addr_lo = dma_addr & 0xffffffff; rxe[i].mbuf = mbuf; PMD_RX_LOG(DEBUG, "[%d]: %" PRIx64 "\n", i, dma_addr); - - rxq->wr_p++; } /* Make sure all writes are flushed before telling the hardware */ @@ -1431,7 +1672,7 @@ nfp_net_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, "tx_free_thresh must be less than the number of TX " "descriptors. (tx_free_thresh=%u port=%d " "queue=%d)\n", (unsigned int)tx_free_thresh, - (int)dev->data->port_id, (int)queue_idx); + dev->data->port_id, (int)queue_idx); return -(EINVAL); } @@ -1459,9 +1700,10 @@ nfp_net_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, * handle the maximum ring size is allocated in order to allow for * resizing in later calls to the queue setup function. */ - tz = ring_dma_zone_reserve(dev, "tx_ring", queue_idx, + tz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_idx, sizeof(struct nfp_net_tx_desc) * - NFP_NET_MAX_TX_DESC, socket_id); + NFP_NET_MAX_TX_DESC, NFP_MEMZONE_ALIGN, + socket_id); if (tz == NULL) { RTE_LOG(ERR, PMD, "Error allocating tx dma\n"); nfp_net_tx_queue_release(txq); @@ -1469,7 +1711,6 @@ nfp_net_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, } txq->tx_count = nb_desc; - txq->tail = 0; txq->tx_free_thresh = tx_free_thresh; txq->tx_pthresh = tx_conf->tx_thresh.pthresh; txq->tx_hthresh = tx_conf->tx_thresh.hthresh; @@ -1484,7 +1725,7 @@ nfp_net_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, txq->txq_flags = tx_conf->txq_flags; /* Saving physical and virtual addresses for the TX ring */ - txq->dma = (uint64_t)tz->phys_addr; + txq->dma = (uint64_t)tz->iova; txq->txds = (struct nfp_net_tx_desc *)tz->addr; /* mbuf pointers array for referencing mbufs linked to TX descriptors */ @@ -1508,11 +1749,38 @@ nfp_net_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, * of descriptors in log2 format */ nn_cfg_writeq(hw, NFP_NET_CFG_TXR_ADDR(queue_idx), txq->dma); - nn_cfg_writeb(hw, NFP_NET_CFG_TXR_SZ(queue_idx), log2(nb_desc)); + nn_cfg_writeb(hw, NFP_NET_CFG_TXR_SZ(queue_idx), rte_log2_u32(nb_desc)); return 0; } +/* nfp_net_tx_tso - Set TX descriptor for TSO */ +static inline void +nfp_net_tx_tso(struct nfp_net_txq *txq, struct nfp_net_tx_desc *txd, + struct rte_mbuf *mb) +{ + uint64_t ol_flags; + struct nfp_net_hw *hw = txq->hw; + + if (!(hw->cap & NFP_NET_CFG_CTRL_LSO)) + goto clean_txd; + + ol_flags = mb->ol_flags; + + if (!(ol_flags & PKT_TX_TCP_SEG)) + goto clean_txd; + + txd->l4_offset = mb->l2_len + mb->l3_len + mb->l4_len; + txd->lso = rte_cpu_to_le_16(mb->tso_segsz); + txd->flags = PCIE_DESC_TX_LSO; + return; + +clean_txd: + txd->flags = 0; + txd->l4_offset = 0; + txd->lso = 0; +} + /* nfp_net_tx_cksum - Set TX CSUM offload flags in TX descriptor */ static inline void nfp_net_tx_cksum(struct nfp_net_txq *txq, struct nfp_net_tx_desc *txd, @@ -1575,6 +1843,8 @@ nfp_net_rx_cksum(struct nfp_net_rxq *rxq, struct nfp_net_rx_desc *rxd, #define NFP_HASH_OFFSET ((uint8_t *)mbuf->buf_addr + mbuf->data_off - 4) #define NFP_HASH_TYPE_OFFSET ((uint8_t *)mbuf->buf_addr + mbuf->data_off - 8) +#define NFP_DESC_META_LEN(d) (d->rxd.meta_len_dd & PCIE_DESC_RX_META_LEN_MASK) + /* * nfp_net_set_hash - Set mbuf hash data * @@ -1585,25 +1855,58 @@ static inline void nfp_net_set_hash(struct nfp_net_rxq *rxq, struct nfp_net_rx_desc *rxd, struct rte_mbuf *mbuf) { - uint32_t hash; - uint32_t hash_type; struct nfp_net_hw *hw = rxq->hw; + uint8_t *meta_offset; + uint32_t meta_info; + uint32_t hash = 0; + uint32_t hash_type = 0; if (!(hw->ctrl & NFP_NET_CFG_CTRL_RSS)) return; - if (!(rxd->rxd.flags & PCIE_DESC_RX_RSS)) + if (NFD_CFG_MAJOR_VERSION_of(hw->ver) <= 3) { + if (!(rxd->rxd.flags & PCIE_DESC_RX_RSS)) + return; + + hash = rte_be_to_cpu_32(*(uint32_t *)NFP_HASH_OFFSET); + hash_type = rte_be_to_cpu_32(*(uint32_t *)NFP_HASH_TYPE_OFFSET); + + } else if (NFP_DESC_META_LEN(rxd)) { + /* + * new metadata api: + * <---- 32 bit -----> + * m field type word + * e data field #2 + * t data field #1 + * a data field #0 + * ==================== + * packet data + * + * Field type word contains up to 8 4bit field types + * A 4bit field type refers to a data field word + * A data field word can have several 4bit field types + */ + meta_offset = rte_pktmbuf_mtod(mbuf, uint8_t *); + meta_offset -= NFP_DESC_META_LEN(rxd); + meta_info = rte_be_to_cpu_32(*(uint32_t *)meta_offset); + meta_offset += 4; + /* NFP PMD just supports metadata for hashing */ + switch (meta_info & NFP_NET_META_FIELD_MASK) { + case NFP_NET_META_HASH: + /* next field type is about the hash type */ + meta_info >>= NFP_NET_META_FIELD_SIZE; + /* hash value is in the data field */ + hash = rte_be_to_cpu_32(*(uint32_t *)meta_offset); + hash_type = meta_info & NFP_NET_META_FIELD_MASK; + break; + default: + /* Unsupported metadata can be a performance issue */ + return; + } + } else { return; + } - hash = rte_be_to_cpu_32(*(uint32_t *)NFP_HASH_OFFSET); - hash_type = rte_be_to_cpu_32(*(uint32_t *)NFP_HASH_TYPE_OFFSET); - - /* - * hash type is sharing the same word with input port info - * 31-8: input port - * 7:0: hash type - */ - hash_type &= 0xff; mbuf->hash.rss = hash; mbuf->ol_flags |= PKT_RX_RSS_HASH; @@ -1622,29 +1925,6 @@ nfp_net_set_hash(struct nfp_net_rxq *rxq, struct nfp_net_rx_desc *rxd, } } -/* nfp_net_check_port - Set mbuf in_port field */ -static void -nfp_net_check_port(struct nfp_net_rx_desc *rxd, struct rte_mbuf *mbuf) -{ - uint32_t port; - - if (!(rxd->rxd.flags & PCIE_DESC_RX_INGRESS_PORT)) { - mbuf->port = 0; - return; - } - - port = rte_be_to_cpu_32(*(uint32_t *)((uint8_t *)mbuf->buf_addr + - mbuf->data_off - 8)); - - /* - * hash type is sharing the same word with input port info - * 31-8: input port - * 7:0: hash type - */ - port = (uint8_t)(port >> 8); - mbuf->port = port; -} - static inline void nfp_net_mbuf_alloc_failed(struct nfp_net_rxq *rxq) { @@ -1690,7 +1970,6 @@ nfp_net_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) struct nfp_net_hw *hw; struct rte_mbuf *mb; struct rte_mbuf *new_mb; - int idx; uint16_t nb_hold; uint64_t dma_addr; int avail; @@ -1701,7 +1980,7 @@ nfp_net_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) * DPDK just checks the queue is lower than max queues * enabled. But the queue needs to be configured */ - RTE_LOG(ERR, PMD, "RX Bad queue\n"); + RTE_LOG_DP(ERR, PMD, "RX Bad queue\n"); return -EINVAL; } @@ -1710,11 +1989,9 @@ nfp_net_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) nb_hold = 0; while (avail < nb_pkts) { - idx = rxq->rd_p % rxq->rx_count; - - rxb = &rxq->rxbufs[idx]; + rxb = &rxq->rxbufs[rxq->rd_p]; if (unlikely(rxb == NULL)) { - RTE_LOG(ERR, PMD, "rxb does not exist!\n"); + RTE_LOG_DP(ERR, PMD, "rxb does not exist!\n"); break; } @@ -1724,7 +2001,7 @@ nfp_net_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) */ rte_rmb(); - rxds = &rxq->rxds[idx]; + rxds = &rxq->rxds[rxq->rd_p]; if ((rxds->rxd.meta_len_dd & PCIE_DESC_RX_DD) == 0) break; @@ -1734,9 +2011,9 @@ nfp_net_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) */ new_mb = rte_pktmbuf_alloc(rxq->mem_pool); if (unlikely(new_mb == NULL)) { - RTE_LOG(DEBUG, PMD, "RX mbuf alloc failed port_id=%u " - "queue_id=%u\n", (unsigned)rxq->port_id, - (unsigned)rxq->qidx); + RTE_LOG_DP(DEBUG, PMD, + "RX mbuf alloc failed port_id=%u queue_id=%u\n", + rxq->port_id, (unsigned int)rxq->qidx); nfp_net_mbuf_alloc_failed(rxq); break; } @@ -1765,7 +2042,7 @@ nfp_net_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) * responsibility of avoiding it. But we have * to give some info about the error */ - RTE_LOG(ERR, PMD, + RTE_LOG_DP(ERR, PMD, "mbuf overflow likely due to the RX offset.\n" "\t\tYour mbuf size should have extra space for" " RX offset=%u bytes.\n" @@ -1794,13 +2071,10 @@ nfp_net_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) /* Checking the checksum flag */ nfp_net_rx_cksum(rxq, rxds, mb); - /* Checking the port flag */ - nfp_net_check_port(rxds, mb); - if ((rxds->rxd.flags & PCIE_DESC_RX_VLAN) && (hw->ctrl & NFP_NET_CFG_CTRL_RXVLAN)) { mb->vlan_tci = rte_cpu_to_le_32(rxds->rxd.vlan); - mb->ol_flags |= PKT_RX_VLAN_PKT | PKT_RX_VLAN_STRIPPED; + mb->ol_flags |= PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED; } /* Adding the mbuff to the mbuff array passed by the app */ @@ -1815,13 +2089,15 @@ nfp_net_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) rxds->fld.dma_addr_lo = dma_addr & 0xffffffff; rxq->rd_p++; + if (unlikely(rxq->rd_p == rxq->rx_count)) /* wrapping?*/ + rxq->rd_p = 0; } if (nb_hold == 0) return nb_hold; PMD_RX_LOG(DEBUG, "RX port_id=%u queue_id=%u, %d packets received\n", - (unsigned)rxq->port_id, (unsigned)rxq->qidx, nb_hold); + rxq->port_id, (unsigned int)rxq->qidx, nb_hold); nb_hold += rxq->nb_rx_hold; @@ -1832,7 +2108,7 @@ nfp_net_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) rte_wmb(); if (nb_hold > rxq->rx_free_thresh) { PMD_RX_LOG(DEBUG, "port=%u queue=%u nb_hold=%u avail=%u\n", - (unsigned)rxq->port_id, (unsigned)rxq->qidx, + rxq->port_id, (unsigned int)rxq->qidx, (unsigned)nb_hold, (unsigned)avail); nfp_qcp_ptr_add(rxq->qcp_fl, NFP_QCP_WRITE_PTR, nb_hold); nb_hold = 0; @@ -1860,33 +2136,40 @@ nfp_net_tx_free_bufs(struct nfp_net_txq *txq) /* Work out how many packets have been sent */ qcp_rd_p = nfp_qcp_read(txq->qcp_q, NFP_QCP_READ_PTR); - if (qcp_rd_p == txq->qcp_rd_p) { + if (qcp_rd_p == txq->rd_p) { PMD_TX_LOG(DEBUG, "queue %u: It seems harrier is not sending " "packets (%u, %u)\n", txq->qidx, - qcp_rd_p, txq->qcp_rd_p); + qcp_rd_p, txq->rd_p); return 0; } - if (qcp_rd_p > txq->qcp_rd_p) - todo = qcp_rd_p - txq->qcp_rd_p; + if (qcp_rd_p > txq->rd_p) + todo = qcp_rd_p - txq->rd_p; else - todo = qcp_rd_p + txq->tx_count - txq->qcp_rd_p; + todo = qcp_rd_p + txq->tx_count - txq->rd_p; - PMD_TX_LOG(DEBUG, "qcp_rd_p %u, txq->qcp_rd_p: %u, qcp->rd_p: %u\n", - qcp_rd_p, txq->qcp_rd_p, txq->rd_p); + PMD_TX_LOG(DEBUG, "qcp_rd_p %u, txq->rd_p: %u, qcp->rd_p: %u\n", + qcp_rd_p, txq->rd_p, txq->rd_p); if (todo == 0) return todo; - txq->qcp_rd_p += todo; - txq->qcp_rd_p %= txq->tx_count; txq->rd_p += todo; + if (unlikely(txq->rd_p >= txq->tx_count)) + txq->rd_p -= txq->tx_count; return todo; } /* Leaving always free descriptors for avoiding wrapping confusion */ -#define NFP_FREE_TX_DESC(t) (t->tx_count - (t->wr_p - t->rd_p) - 8) +static inline +uint32_t nfp_free_tx_desc(struct nfp_net_txq *txq) +{ + if (txq->wr_p >= txq->rd_p) + return txq->tx_count - (txq->wr_p - txq->rd_p) - 8; + else + return txq->rd_p - txq->wr_p - 8; +} /* * nfp_net_txq_full - Check if the TX queue free descriptors @@ -1897,9 +2180,9 @@ nfp_net_tx_free_bufs(struct nfp_net_txq *txq) * This function uses the host copy* of read/write pointers */ static inline -int nfp_net_txq_full(struct nfp_net_txq *txq) +uint32_t nfp_net_txq_full(struct nfp_net_txq *txq) { - return NFP_FREE_TX_DESC(txq) < txq->tx_free_thresh; + return (nfp_free_tx_desc(txq) < txq->tx_free_thresh); } static uint16_t @@ -1907,7 +2190,7 @@ nfp_net_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { struct nfp_net_txq *txq; struct nfp_net_hw *hw; - struct nfp_net_tx_desc *txds; + struct nfp_net_tx_desc *txds, txd; struct rte_mbuf *pkt; uint64_t dma_addr; int pkt_size, dma_size; @@ -1917,15 +2200,15 @@ nfp_net_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) txq = tx_queue; hw = txq->hw; - txds = &txq->txds[txq->tail]; + txds = &txq->txds[txq->wr_p]; PMD_TX_LOG(DEBUG, "working for queue %u at pos %d and %u packets\n", - txq->qidx, txq->tail, nb_pkts); + txq->qidx, txq->wr_p, nb_pkts); - if ((NFP_FREE_TX_DESC(txq) < nb_pkts) || (nfp_net_txq_full(txq))) + if ((nfp_free_tx_desc(txq) < nb_pkts) || (nfp_net_txq_full(txq))) nfp_net_tx_free_bufs(txq); - free_descs = (uint16_t)NFP_FREE_TX_DESC(txq); + free_descs = (uint16_t)nfp_free_tx_desc(txq); if (unlikely(free_descs == 0)) return 0; @@ -1938,7 +2221,7 @@ nfp_net_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) /* Sending packets */ while ((i < nb_pkts) && free_descs) { /* Grabbing the mbuf linked to the current descriptor */ - lmbuf = &txq->txbufs[txq->tail].mbuf; + lmbuf = &txq->txbufs[txq->wr_p].mbuf; /* Warming the cache for releasing the mbuf later on */ RTE_MBUF_PREFETCH_TO_FREE(*lmbuf); @@ -1946,7 +2229,7 @@ nfp_net_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) if (unlikely((pkt->nb_segs > 1) && !(hw->cap & NFP_NET_CFG_CTRL_GATHER))) { - PMD_INIT_LOG(INFO, "NFP_NET_CFG_CTRL_GATHER not set\n"); + PMD_INIT_LOG(INFO, "NFP_NET_CFG_CTRL_GATHER not set"); rte_panic("Multisegment packet unsupported\n"); } @@ -1956,19 +2239,18 @@ nfp_net_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) /* * Checksum and VLAN flags just in the first descriptor for a - * multisegment packet + * multisegment packet, but TSO info needs to be in all of them. */ - nfp_net_tx_cksum(txq, txds, pkt); + txd.data_len = pkt->pkt_len; + nfp_net_tx_tso(txq, &txd, pkt); + nfp_net_tx_cksum(txq, &txd, pkt); if ((pkt->ol_flags & PKT_TX_VLAN_PKT) && (hw->cap & NFP_NET_CFG_CTRL_TXVLAN)) { - txds->flags |= PCIE_DESC_TX_VLAN; - txds->vlan = pkt->vlan_tci; + txd.flags |= PCIE_DESC_TX_VLAN; + txd.vlan = pkt->vlan_tci; } - if (pkt->ol_flags & PKT_TX_TCP_SEG) - rte_panic("TSO is not supported\n"); - /* * mbuf data_len is the data in one segment and pkt_len data * in the whole packet. When the packet is just one segment, @@ -1976,44 +2258,48 @@ nfp_net_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) */ pkt_size = pkt->pkt_len; - /* Releasing mbuf which was prefetched above */ - if (*lmbuf) - rte_pktmbuf_free(*lmbuf); - /* - * Linking mbuf with descriptor for being released - * next time descriptor is used - */ - *lmbuf = pkt; + while (pkt) { + /* Copying TSO, VLAN and cksum info */ + *txds = txd; + + /* Releasing mbuf used by this descriptor previously*/ + if (*lmbuf) + rte_pktmbuf_free_seg(*lmbuf); + + /* + * Linking mbuf with descriptor for being released + * next time descriptor is used + */ + *lmbuf = pkt; - while (pkt_size) { dma_size = pkt->data_len; - dma_addr = rte_mbuf_data_dma_addr(pkt); + dma_addr = rte_mbuf_data_iova(pkt); PMD_TX_LOG(DEBUG, "Working with mbuf at dma address:" "%" PRIx64 "\n", dma_addr); /* Filling descriptors fields */ txds->dma_len = dma_size; - txds->data_len = pkt->pkt_len; + txds->data_len = txd.data_len; txds->dma_addr_hi = (dma_addr >> 32) & 0xff; txds->dma_addr_lo = (dma_addr & 0xffffffff); ASSERT(free_descs > 0); free_descs--; txq->wr_p++; - txq->tail++; - if (unlikely(txq->tail == txq->tx_count)) /* wrapping?*/ - txq->tail = 0; + if (unlikely(txq->wr_p == txq->tx_count)) /* wrapping?*/ + txq->wr_p = 0; pkt_size -= dma_size; - if (!pkt_size) { + if (!pkt_size) /* End of packet */ txds->offset_eop |= PCIE_DESC_TX_EOP; - } else { + else txds->offset_eop &= PCIE_DESC_TX_OFFSET_MASK; - pkt = pkt->next; - } + + pkt = pkt->next; /* Referencing next free TX descriptor */ - txds = &txq->txds[txq->tail]; + txds = &txq->txds[txq->wr_p]; + lmbuf = &txq->txbufs[txq->wr_p].mbuf; issued_descs++; } i++; @@ -2027,19 +2313,20 @@ xmit_end: return i; } -static void +static int nfp_net_vlan_offload_set(struct rte_eth_dev *dev, int mask) { uint32_t new_ctrl, update; struct nfp_net_hw *hw; + int ret; hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); new_ctrl = 0; if ((mask & ETH_VLAN_FILTER_OFFLOAD) || - (mask & ETH_VLAN_FILTER_OFFLOAD)) - RTE_LOG(INFO, PMD, "Not support for ETH_VLAN_FILTER_OFFLOAD or" - " ETH_VLAN_FILTER_EXTEND"); + (mask & ETH_VLAN_EXTEND_OFFLOAD)) + RTE_LOG(INFO, PMD, "No support for ETH_VLAN_FILTER_OFFLOAD or" + " ETH_VLAN_EXTEND_OFFLOAD"); /* Enable vlan strip if it is not configured yet */ if ((mask & ETH_VLAN_STRIP_OFFLOAD) && @@ -2052,14 +2339,15 @@ nfp_net_vlan_offload_set(struct rte_eth_dev *dev, int mask) new_ctrl = hw->ctrl & ~NFP_NET_CFG_CTRL_RXVLAN; if (new_ctrl == 0) - return; + return 0; update = NFP_NET_CFG_UPDATE_GEN; - if (nfp_net_reconfig(hw, new_ctrl, update) < 0) - return; + ret = nfp_net_reconfig(hw, new_ctrl, update); + if (!ret) + hw->ctrl = new_ctrl; - hw->ctrl = new_ctrl; + return ret; } /* Update Redirection Table(RETA) of Receive Side Scaling of Ethernet device */ @@ -2111,7 +2399,8 @@ nfp_net_reta_update(struct rte_eth_dev *dev, reta &= ~(0xFF << (8 * j)); reta |= reta_conf[idx].reta[shift + j] << (8 * j); } - nn_cfg_writel(hw, NFP_NET_CFG_RSS_ITBL + shift, reta); + nn_cfg_writel(hw, NFP_NET_CFG_RSS_ITBL + (idx * 64) + shift, + reta); } update = NFP_NET_CFG_UPDATE_RSS; @@ -2158,7 +2447,8 @@ nfp_net_reta_query(struct rte_eth_dev *dev, if (!mask) continue; - reta = nn_cfg_readl(hw, NFP_NET_CFG_RSS_ITBL + shift); + reta = nn_cfg_readl(hw, NFP_NET_CFG_RSS_ITBL + (idx * 64) + + shift); for (j = 0; j < 4; j++) { if (!(mask & (0x1 << j))) continue; @@ -2208,6 +2498,9 @@ nfp_net_rss_hash_update(struct rte_eth_dev *dev, NFP_NET_CFG_RSS_IPV6_TCP | NFP_NET_CFG_RSS_IPV6_UDP; + cfg_rss_ctrl |= NFP_NET_CFG_RSS_MASK; + cfg_rss_ctrl |= NFP_NET_CFG_RSS_TOEPLITZ; + /* configuring where to apply the RSS hash */ nn_cfg_writel(hw, NFP_NET_CFG_RSS_CTRL, cfg_rss_ctrl); @@ -2300,21 +2593,80 @@ static const struct eth_dev_ops nfp_net_eth_dev_ops = { .rx_queue_count = nfp_net_rx_queue_count, .tx_queue_setup = nfp_net_tx_queue_setup, .tx_queue_release = nfp_net_tx_queue_release, + .rx_queue_intr_enable = nfp_rx_queue_intr_enable, + .rx_queue_intr_disable = nfp_rx_queue_intr_disable, }; +/* + * All eth_dev created got its private data, but before nfp_net_init, that + * private data is referencing private data for all the PF ports. This is due + * to how the vNIC bars are mapped based on first port, so all ports need info + * about port 0 private data. Inside nfp_net_init the private data pointer is + * changed to the right address for each port once the bars have been mapped. + * + * This functions helps to find out which port and therefore which offset + * inside the private data array to use. + */ +static int +get_pf_port_number(char *name) +{ + char *pf_str = name; + int size = 0; + + while ((*pf_str != '_') && (*pf_str != '\0') && (size++ < 30)) + pf_str++; + + if (size == 30) + /* + * This should not happen at all and it would mean major + * implementation fault. + */ + rte_panic("nfp_net: problem with pf device name\n"); + + /* Expecting _portX with X within [0,7] */ + pf_str += 5; + + return (int)strtol(pf_str, NULL, 10); +} + static int nfp_net_init(struct rte_eth_dev *eth_dev) { struct rte_pci_device *pci_dev; - struct nfp_net_hw *hw; + struct nfp_net_hw *hw, *hwport0; - uint32_t tx_bar_off, rx_bar_off; + uint64_t tx_bar_off = 0, rx_bar_off = 0; uint32_t start_q; int stride = 4; + nspu_desc_t *nspu_desc = NULL; + uint64_t bar_offset; + int port = 0; + PMD_INIT_FUNC_TRACE(); - hw = NFP_NET_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); + pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); + + if ((pci_dev->id.device_id == PCI_DEVICE_ID_NFP4000_PF_NIC) || + (pci_dev->id.device_id == PCI_DEVICE_ID_NFP6000_PF_NIC)) { + port = get_pf_port_number(eth_dev->data->name); + if (port < 0 || port > 7) { + RTE_LOG(ERR, PMD, "Port value is wrong\n"); + return -ENODEV; + } + + PMD_INIT_LOG(DEBUG, "Working with PF port value %d\n", port); + + /* This points to port 0 private data */ + hwport0 = NFP_NET_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); + + /* This points to the specific port private data */ + hw = &hwport0[port]; + hw->pf_port_idx = port; + } else { + hw = NFP_NET_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); + hwport0 = 0; + } eth_dev->dev_ops = &nfp_net_eth_dev_ops; eth_dev->rx_pkt_burst = &nfp_net_recv_pkts; @@ -2324,7 +2676,6 @@ nfp_net_init(struct rte_eth_dev *eth_dev) if (rte_eal_process_type() != RTE_PROC_PRIMARY) return 0; - pci_dev = eth_dev->pci_dev; rte_eth_copy_pci_info(eth_dev, pci_dev); hw->device_id = pci_dev->id.device_id; @@ -2332,7 +2683,7 @@ nfp_net_init(struct rte_eth_dev *eth_dev) hw->subsystem_device_id = pci_dev->id.subsystem_device_id; hw->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id; - PMD_INIT_LOG(DEBUG, "nfp_net: device (%u:%u) %u:%u:%u:%u\n", + PMD_INIT_LOG(DEBUG, "nfp_net: device (%u:%u) %u:%u:%u:%u", pci_dev->id.vendor_id, pci_dev->id.device_id, pci_dev->addr.domain, pci_dev->addr.bus, pci_dev->addr.devid, pci_dev->addr.function); @@ -2343,11 +2694,42 @@ nfp_net_init(struct rte_eth_dev *eth_dev) "hw->ctrl_bar is NULL. BAR0 not configured\n"); return -ENODEV; } + + if (hw->is_pf && port == 0) { + nspu_desc = hw->nspu_desc; + + if (nfp_nsp_map_ctrl_bar(nspu_desc, &bar_offset) != 0) { + /* + * A firmware should be there after PF probe so this + * should not happen. + */ + RTE_LOG(ERR, PMD, "PF BAR symbol resolution failed\n"); + return -ENODEV; + } + + /* vNIC PF control BAR is a subset of PF PCI device BAR */ + hw->ctrl_bar += bar_offset; + PMD_INIT_LOG(DEBUG, "ctrl bar: %p\n", hw->ctrl_bar); + } + + if (port > 0) { + if (!hwport0->ctrl_bar) + return -ENODEV; + + /* address based on port0 offset */ + hw->ctrl_bar = hwport0->ctrl_bar + + (port * NFP_PF_CSR_SLICE_SIZE); + } + + PMD_INIT_LOG(DEBUG, "ctrl bar: %p\n", hw->ctrl_bar); + hw->max_rx_queues = nn_cfg_readl(hw, NFP_NET_CFG_MAX_RXRINGS); hw->max_tx_queues = nn_cfg_readl(hw, NFP_NET_CFG_MAX_TXRINGS); /* Work out where in the BAR the queues start. */ switch (pci_dev->id.device_id) { + case PCI_DEVICE_ID_NFP4000_PF_NIC: + case PCI_DEVICE_ID_NFP6000_PF_NIC: case PCI_DEVICE_ID_NFP6000_VF_NIC: start_q = nn_cfg_readl(hw, NFP_NET_CFG_START_TXQ); tx_bar_off = NFP_PCIE_QUEUE(start_q); @@ -2359,13 +2741,36 @@ nfp_net_init(struct rte_eth_dev *eth_dev) return -ENODEV; } - PMD_INIT_LOG(DEBUG, "tx_bar_off: 0x%08x\n", tx_bar_off); - PMD_INIT_LOG(DEBUG, "rx_bar_off: 0x%08x\n", rx_bar_off); + PMD_INIT_LOG(DEBUG, "tx_bar_off: 0x%" PRIx64 "\n", tx_bar_off); + PMD_INIT_LOG(DEBUG, "rx_bar_off: 0x%" PRIx64 "\n", rx_bar_off); - hw->tx_bar = (uint8_t *)pci_dev->mem_resource[2].addr + tx_bar_off; - hw->rx_bar = (uint8_t *)pci_dev->mem_resource[2].addr + rx_bar_off; + if (hw->is_pf && port == 0) { + /* configure access to tx/rx vNIC BARs */ + nfp_nsp_map_queues_bar(nspu_desc, &bar_offset); + PMD_INIT_LOG(DEBUG, "tx/rx bar_offset: %" PRIx64 "\n", + bar_offset); + hwport0->hw_queues = (uint8_t *)pci_dev->mem_resource[0].addr; - PMD_INIT_LOG(DEBUG, "ctrl_bar: %p, tx_bar: %p, rx_bar: %p\n", + /* vNIC PF tx/rx BARs are a subset of PF PCI device */ + hwport0->hw_queues += bar_offset; + + /* Lets seize the chance to read eth table from hw */ + if (nfp_nsp_eth_read_table(nspu_desc, &hw->eth_table)) + return -ENODEV; + } + + if (hw->is_pf) { + hw->tx_bar = hwport0->hw_queues + tx_bar_off; + hw->rx_bar = hwport0->hw_queues + rx_bar_off; + eth_dev->data->dev_private = hw; + } else { + hw->tx_bar = (uint8_t *)pci_dev->mem_resource[2].addr + + tx_bar_off; + hw->rx_bar = (uint8_t *)pci_dev->mem_resource[2].addr + + rx_bar_off; + } + + PMD_INIT_LOG(DEBUG, "ctrl_bar: %p, tx_bar: %p, rx_bar: %p", hw->ctrl_bar, hw->tx_bar, hw->rx_bar); nfp_net_cfg_queue_setup(hw); @@ -2374,17 +2779,19 @@ nfp_net_init(struct rte_eth_dev *eth_dev) hw->ver = nn_cfg_readl(hw, NFP_NET_CFG_VERSION); hw->cap = nn_cfg_readl(hw, NFP_NET_CFG_CAP); hw->max_mtu = nn_cfg_readl(hw, NFP_NET_CFG_MAX_MTU); - hw->mtu = hw->max_mtu; + hw->mtu = ETHER_MTU; if (NFD_CFG_MAJOR_VERSION_of(hw->ver) < 2) hw->rx_offset = NFP_NET_RX_OFFSET; else hw->rx_offset = nn_cfg_readl(hw, NFP_NET_CFG_RX_OFFSET_ADDR); - PMD_INIT_LOG(INFO, "VER: %#x, Maximum supported MTU: %d\n", + PMD_INIT_LOG(INFO, "VER: %#x, Maximum supported MTU: %d", hw->ver, hw->max_mtu); - PMD_INIT_LOG(INFO, "CAP: %#x, %s%s%s%s%s%s%s%s%s\n", hw->cap, + PMD_INIT_LOG(INFO, "CAP: %#x, %s%s%s%s%s%s%s%s%s%s%s", hw->cap, hw->cap & NFP_NET_CFG_CTRL_PROMISC ? "PROMISC " : "", + hw->cap & NFP_NET_CFG_CTRL_L2BC ? "L2BCFILT " : "", + hw->cap & NFP_NET_CFG_CTRL_L2MC ? "L2MCFILT " : "", hw->cap & NFP_NET_CFG_CTRL_RXCSUM ? "RXCSUM " : "", hw->cap & NFP_NET_CFG_CTRL_TXCSUM ? "TXCSUM " : "", hw->cap & NFP_NET_CFG_CTRL_RXVLAN ? "RXVLAN " : "", @@ -2394,13 +2801,12 @@ nfp_net_init(struct rte_eth_dev *eth_dev) hw->cap & NFP_NET_CFG_CTRL_LSO ? "TSO " : "", hw->cap & NFP_NET_CFG_CTRL_RSS ? "RSS " : ""); - pci_dev = eth_dev->pci_dev; hw->ctrl = 0; hw->stride_rx = stride; hw->stride_tx = stride; - PMD_INIT_LOG(INFO, "max_rx_queues: %u, max_tx_queues: %u\n", + PMD_INIT_LOG(INFO, "max_rx_queues: %u, max_tx_queues: %u", hw->max_rx_queues, hw->max_tx_queues); /* Initializing spinlock for reconfigs */ @@ -2413,8 +2819,18 @@ nfp_net_init(struct rte_eth_dev *eth_dev) return -ENOMEM; } - /* Using random mac addresses for VFs */ - eth_random_addr(&hw->mac_addr[0]); + if (hw->is_pf) { + nfp_net_pf_read_mac(hwport0, port); + nfp_net_write_mac(hw, (uint8_t *)&hw->mac_addr); + } else { + nfp_net_vf_read_mac(hw); + } + + if (!is_valid_assigned_ether_addr((struct ether_addr *)&hw->mac_addr)) { + /* Using random mac addresses for VFs */ + eth_random_addr(&hw->mac_addr[0]); + nfp_net_write_mac(hw, (uint8_t *)&hw->mac_addr); + } /* Copying mac address to DPDK eth_dev struct */ ether_addr_copy((struct ether_addr *)hw->mac_addr, @@ -2432,9 +2848,6 @@ nfp_net_init(struct rte_eth_dev *eth_dev) nfp_net_dev_interrupt_handler, (void *)eth_dev); - /* enable uio intr after callback register */ - rte_intr_enable(&pci_dev->intr_handle); - /* Telling the firmware about the LSC interrupt entry */ nn_cfg_writeb(hw, NFP_NET_CFG_LSC, NFP_NET_IRQ_LSC_IDX); @@ -2444,11 +2857,153 @@ nfp_net_init(struct rte_eth_dev *eth_dev) return 0; } -static struct rte_pci_id pci_id_nfp_net_map[] = { +static int +nfp_pf_create_dev(struct rte_pci_device *dev, int port, int ports, + nfpu_desc_t *nfpu_desc, void **priv) +{ + struct rte_eth_dev *eth_dev; + struct nfp_net_hw *hw; + char *port_name; + int ret; + + port_name = rte_zmalloc("nfp_pf_port_name", 100, 0); + if (!port_name) + return -ENOMEM; + + if (ports > 1) + sprintf(port_name, "%s_port%d", dev->device.name, port); + else + sprintf(port_name, "%s", dev->device.name); + + eth_dev = rte_eth_dev_allocate(port_name); + if (!eth_dev) + return -ENOMEM; + + if (port == 0) { + *priv = rte_zmalloc(port_name, + sizeof(struct nfp_net_adapter) * ports, + RTE_CACHE_LINE_SIZE); + if (!*priv) { + rte_eth_dev_release_port(eth_dev); + return -ENOMEM; + } + } + + eth_dev->data->dev_private = *priv; + + /* + * dev_private pointing to port0 dev_private because we need + * to configure vNIC bars based on port0 at nfp_net_init. + * Then dev_private is adjusted per port. + */ + hw = (struct nfp_net_hw *)(eth_dev->data->dev_private) + port; + hw->nspu_desc = nfpu_desc->nspu; + hw->nfpu_desc = nfpu_desc; + hw->is_pf = 1; + if (ports > 1) + hw->pf_multiport_enabled = 1; + + eth_dev->device = &dev->device; + rte_eth_copy_pci_info(eth_dev, dev); + + ret = nfp_net_init(eth_dev); + + if (ret) + rte_eth_dev_release_port(eth_dev); + + rte_free(port_name); + + return ret; +} + +static int nfp_pf_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, + struct rte_pci_device *dev) +{ + nfpu_desc_t *nfpu_desc; + nspu_desc_t *nspu_desc; + uint64_t offset_symbol; + uint8_t *bar_offset; + int major, minor; + int total_ports; + void *priv = 0; + int ret = -ENODEV; + int i; + + if (!dev) + return ret; + + nfpu_desc = rte_malloc("nfp nfpu", sizeof(nfpu_desc_t), 0); + if (!nfpu_desc) + return -ENOMEM; + + if (nfpu_open(dev, nfpu_desc, 0) < 0) { + RTE_LOG(ERR, PMD, + "nfpu_open failed\n"); + goto nfpu_error; + } + + nspu_desc = nfpu_desc->nspu; + + + /* Check NSP ABI version */ + if (nfp_nsp_get_abi_version(nspu_desc, &major, &minor) < 0) { + RTE_LOG(INFO, PMD, "NFP NSP not present\n"); + goto error; + } + PMD_INIT_LOG(INFO, "nspu ABI version: %d.%d\n", major, minor); + + if ((major == 0) && (minor < 20)) { + RTE_LOG(INFO, PMD, "NFP NSP ABI version too old. Required 0.20 or higher\n"); + goto error; + } + + ret = nfp_nsp_fw_setup(nspu_desc, "nfd_cfg_pf0_num_ports", + &offset_symbol); + if (ret) + goto error; + + bar_offset = (uint8_t *)dev->mem_resource[0].addr; + bar_offset += offset_symbol; + total_ports = (uint32_t)*bar_offset; + PMD_INIT_LOG(INFO, "Total pf ports: %d\n", total_ports); + + if (total_ports <= 0 || total_ports > 8) { + RTE_LOG(ERR, PMD, "nfd_cfg_pf0_num_ports symbol with wrong value"); + ret = -ENODEV; + goto error; + } + + for (i = 0; i < total_ports; i++) { + ret = nfp_pf_create_dev(dev, i, total_ports, nfpu_desc, &priv); + if (ret) + goto error; + } + + return 0; + +error: + nfpu_close(nfpu_desc); +nfpu_error: + rte_free(nfpu_desc); + + return ret; +} + +static const struct rte_pci_id pci_id_nfp_pf_net_map[] = { + { + RTE_PCI_DEVICE(PCI_VENDOR_ID_NETRONOME, + PCI_DEVICE_ID_NFP4000_PF_NIC) + }, { RTE_PCI_DEVICE(PCI_VENDOR_ID_NETRONOME, PCI_DEVICE_ID_NFP6000_PF_NIC) }, + { + .vendor_id = 0, + }, +}; + +static const struct rte_pci_id pci_id_nfp_vf_net_map[] = { { RTE_PCI_DEVICE(PCI_VENDOR_ID_NETRONOME, PCI_DEVICE_ID_NFP6000_VF_NIC) @@ -2458,36 +3013,54 @@ static struct rte_pci_id pci_id_nfp_net_map[] = { }, }; -static struct eth_driver rte_nfp_net_pmd = { - { - .name = "rte_nfp_net_pmd", - .id_table = pci_id_nfp_net_map, - .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC | - RTE_PCI_DRV_DETACHABLE, - }, - .eth_dev_init = nfp_net_init, - .dev_private_size = sizeof(struct nfp_net_adapter), -}; - -static int -nfp_net_pmd_init(const char *name __rte_unused, - const char *params __rte_unused) +static int eth_nfp_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, + struct rte_pci_device *pci_dev) { - PMD_INIT_FUNC_TRACE(); - PMD_INIT_LOG(INFO, "librte_pmd_nfp_net version %s\n", - NFP_NET_PMD_VERSION); - - rte_eth_driver_register(&rte_nfp_net_pmd); - return 0; + return rte_eth_dev_pci_generic_probe(pci_dev, + sizeof(struct nfp_net_adapter), nfp_net_init); } -static struct rte_driver rte_nfp_net_driver = { - .type = PMD_PDEV, - .init = nfp_net_pmd_init, +static int eth_nfp_pci_remove(struct rte_pci_device *pci_dev) +{ + struct rte_eth_dev *eth_dev; + struct nfp_net_hw *hw, *hwport0; + int port = 0; + + eth_dev = rte_eth_dev_allocated(pci_dev->device.name); + if ((pci_dev->id.device_id == PCI_DEVICE_ID_NFP4000_PF_NIC) || + (pci_dev->id.device_id == PCI_DEVICE_ID_NFP6000_PF_NIC)) { + port = get_pf_port_number(eth_dev->data->name); + hwport0 = NFP_NET_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); + hw = &hwport0[port]; + } else { + hw = NFP_NET_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); + } + /* hotplug is not possible with multiport PF */ + if (hw->pf_multiport_enabled) + return -ENOTSUP; + return rte_eth_dev_pci_generic_remove(pci_dev, NULL); +} + +static struct rte_pci_driver rte_nfp_net_pf_pmd = { + .id_table = pci_id_nfp_pf_net_map, + .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC, + .probe = nfp_pf_pci_probe, + .remove = eth_nfp_pci_remove, }; -PMD_REGISTER_DRIVER(rte_nfp_net_driver, nfp); -DRIVER_REGISTER_PCI_TABLE(nfp, pci_id_nfp_net_map); +static struct rte_pci_driver rte_nfp_net_vf_pmd = { + .id_table = pci_id_nfp_vf_net_map, + .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC, + .probe = eth_nfp_pci_probe, + .remove = eth_nfp_pci_remove, +}; + +RTE_PMD_REGISTER_PCI(net_nfp_pf, rte_nfp_net_pf_pmd); +RTE_PMD_REGISTER_PCI(net_nfp_vf, rte_nfp_net_vf_pmd); +RTE_PMD_REGISTER_PCI_TABLE(net_nfp_pf, pci_id_nfp_pf_net_map); +RTE_PMD_REGISTER_PCI_TABLE(net_nfp_vf, pci_id_nfp_vf_net_map); +RTE_PMD_REGISTER_KMOD_DEP(net_nfp_pf, "* igb_uio | uio_pci_generic | vfio"); +RTE_PMD_REGISTER_KMOD_DEP(net_nfp_vf, "* igb_uio | uio_pci_generic | vfio"); /* * Local variables: diff --git a/dpdk/drivers/net/nfp/nfp_net_ctrl.h b/dpdk/drivers/net/nfp/nfp_net_ctrl.h index fce82515..1ebd99ca 100644 --- a/dpdk/drivers/net/nfp/nfp_net_ctrl.h +++ b/dpdk/drivers/net/nfp/nfp_net_ctrl.h @@ -52,6 +52,13 @@ /* Offset in Freelist buffer where packet starts on RX */ #define NFP_NET_RX_OFFSET 32 +/* working with metadata api (NFD version > 3.0) */ +#define NFP_NET_META_FIELD_SIZE 4 +#define NFP_NET_META_FIELD_MASK ((1 << NFP_NET_META_FIELD_SIZE) - 1) + +/* Prepend field types */ +#define NFP_NET_META_HASH 1 /* next field carries hash type */ + /* Hash type pre-pended when a RSS hash was computed */ #define NFP_NET_RSS_NONE 0 #define NFP_NET_RSS_IPV4 1 @@ -112,6 +119,7 @@ #define NFP_NET_CFG_CTRL_L2SWITCH_LOCAL (0x1 << 23) /* Switch to local */ #define NFP_NET_CFG_CTRL_VXLAN (0x1 << 24) /* Enable VXLAN */ #define NFP_NET_CFG_CTRL_NVGRE (0x1 << 25) /* Enable NVGRE */ +#define NFP_NET_CFG_CTRL_MSIX_TX_OFF (0x1 << 26) /* Disable MSIX for TX */ #define NFP_NET_CFG_UPDATE 0x0004 #define NFP_NET_CFG_UPDATE_GEN (0x1 << 0) /* General update */ #define NFP_NET_CFG_UPDATE_RING (0x1 << 1) /* Ring config change */ @@ -157,6 +165,17 @@ #define NFP_NET_CFG_VERSION_MINOR(x) (((x) & 0xff) << 0) #define NFP_NET_CFG_STS 0x0034 #define NFP_NET_CFG_STS_LINK (0x1 << 0) /* Link up or down */ +/* Link rate */ +#define NFP_NET_CFG_STS_LINK_RATE_SHIFT 1 +#define NFP_NET_CFG_STS_LINK_RATE_MASK 0xF +#define NFP_NET_CFG_STS_LINK_RATE_UNSUPPORTED 0 +#define NFP_NET_CFG_STS_LINK_RATE_UNKNOWN 1 +#define NFP_NET_CFG_STS_LINK_RATE_1G 2 +#define NFP_NET_CFG_STS_LINK_RATE_10G 3 +#define NFP_NET_CFG_STS_LINK_RATE_25G 4 +#define NFP_NET_CFG_STS_LINK_RATE_40G 5 +#define NFP_NET_CFG_STS_LINK_RATE_50G 6 +#define NFP_NET_CFG_STS_LINK_RATE_100G 7 #define NFP_NET_CFG_CAP 0x0038 #define NFP_NET_CFG_MAX_TXRINGS 0x003c #define NFP_NET_CFG_MAX_RXRINGS 0x0040 @@ -315,6 +334,9 @@ #define NFP_NET_CFG_RXR_STATS(_x) (NFP_NET_CFG_RXR_STATS_BASE + \ ((_x) * 0x10)) +/* PF multiport offset */ +#define NFP_PF_CSR_SLICE_SIZE (32 * 1024) + #endif /* _NFP_NET_CTRL_H_ */ /* * Local variables: diff --git a/dpdk/drivers/net/nfp/nfp_net_eth.h b/dpdk/drivers/net/nfp/nfp_net_eth.h new file mode 100644 index 00000000..af57f03c --- /dev/null +++ b/dpdk/drivers/net/nfp/nfp_net_eth.h @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2017 Netronome Systems, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * vim:shiftwidth=8:noexpandtab + * + * @file dpdk/pmd/nfp_net_eth.h + * + * Netronome NFP_NET PDM driver + */ + +union eth_table_entry { + struct { + uint64_t port; + uint64_t state; + uint8_t mac_addr[6]; + uint8_t resv[2]; + uint64_t control; + }; + uint64_t raw[4]; +}; + +#ifndef BIT_ULL +#define BIT_ULL(a) (1ULL << (a)) +#endif + +#define NSP_ETH_NBI_PORT_COUNT 24 +#define NSP_ETH_MAX_COUNT (2 * NSP_ETH_NBI_PORT_COUNT) +#define NSP_ETH_TABLE_SIZE (NSP_ETH_MAX_COUNT * sizeof(union eth_table_entry)) + +#define NSP_ETH_PORT_LANES 0xf +#define NSP_ETH_PORT_INDEX 0xff00 +#define NSP_ETH_PORT_LABEL 0x3f000000000000 +#define NSP_ETH_PORT_PHYLABEL 0xfc0000000000000 + +#define NSP_ETH_PORT_LANES_MASK rte_cpu_to_le_64(NSP_ETH_PORT_LANES) + +#define NSP_ETH_STATE_CONFIGURED BIT_ULL(0) +#define NSP_ETH_STATE_ENABLED BIT_ULL(1) +#define NSP_ETH_STATE_TX_ENABLED BIT_ULL(2) +#define NSP_ETH_STATE_RX_ENABLED BIT_ULL(3) +#define NSP_ETH_STATE_RATE 0xf00 +#define NSP_ETH_STATE_INTERFACE 0xff000 +#define NSP_ETH_STATE_MEDIA 0x300000 +#define NSP_ETH_STATE_OVRD_CHNG BIT_ULL(22) +#define NSP_ETH_STATE_ANEG 0x3800000 + +#define NSP_ETH_CTRL_CONFIGURED BIT_ULL(0) +#define NSP_ETH_CTRL_ENABLED BIT_ULL(1) +#define NSP_ETH_CTRL_TX_ENABLED BIT_ULL(2) +#define NSP_ETH_CTRL_RX_ENABLED BIT_ULL(3) +#define NSP_ETH_CTRL_SET_RATE BIT_ULL(4) +#define NSP_ETH_CTRL_SET_LANES BIT_ULL(5) +#define NSP_ETH_CTRL_SET_ANEG BIT_ULL(6) diff --git a/dpdk/drivers/net/nfp/nfp_net_pmd.h b/dpdk/drivers/net/nfp/nfp_net_pmd.h index c1809720..1ae0ea62 100644 --- a/dpdk/drivers/net/nfp/nfp_net_pmd.h +++ b/dpdk/drivers/net/nfp/nfp_net_pmd.h @@ -42,6 +42,7 @@ #define NFP_NET_PMD_VERSION "0.1" #define PCI_VENDOR_ID_NETRONOME 0x19ee +#define PCI_DEVICE_ID_NFP4000_PF_NIC 0x4000 #define PCI_DEVICE_ID_NFP6000_PF_NIC 0x6000 #define PCI_DEVICE_ID_NFP6000_VF_NIC 0x6003 @@ -121,25 +122,31 @@ struct nfp_net_adapter; #define NFD_CFG_MINOR_VERSION_of(x) (((x) >> 0) & 0xff) #include +#include static inline uint8_t nn_readb(volatile const void *addr) { - return *((volatile const uint8_t *)(addr)); + return rte_read8(addr); } static inline void nn_writeb(uint8_t val, volatile void *addr) { - *((volatile uint8_t *)(addr)) = val; + rte_write8(val, addr); } static inline uint32_t nn_readl(volatile const void *addr) { - return *((volatile const uint32_t *)(addr)); + return rte_read32(addr); } static inline void nn_writel(uint32_t val, volatile void *addr) { - *((volatile uint32_t *)(addr)) = val; + rte_write32(val, addr); +} + +static inline void nn_writew(uint16_t val, volatile void *addr) +{ + rte_write16(val, addr); } static inline uint64_t nn_readq(volatile void *addr) @@ -216,12 +223,10 @@ struct nfp_net_txq { uint32_t wr_p; uint32_t rd_p; - uint32_t qcp_rd_p; uint32_t tx_count; uint32_t tx_free_thresh; - uint32_t tail; /* * For each descriptor keep a reference to the mbuff and @@ -240,7 +245,7 @@ struct nfp_net_txq { struct nfp_net_tx_desc *txds; /* - * At this point 56 bytes have been used for all the fields in the + * At this point 48 bytes have been used for all the fields in the * TX critical path. We have room for 8 bytes and still all placed * in a cache line. We are not using the threshold values below nor * the txq_flags but if we need to, we can add the most used in the @@ -251,7 +256,7 @@ struct nfp_net_txq { uint32_t tx_hthresh; /* not used by now. Future? */ uint32_t tx_wthresh; /* not used by now. Future? */ uint32_t txq_flags; /* not used by now. Future? */ - uint8_t port_id; + uint16_t port_id; int qidx; int tx_qcidx; __le64 dma; @@ -269,7 +274,7 @@ struct nfp_net_txq { #define PCIE_DESC_RX_I_TCP_CSUM_OK (1 << 11) #define PCIE_DESC_RX_I_UDP_CSUM (1 << 10) #define PCIE_DESC_RX_I_UDP_CSUM_OK (1 << 9) -#define PCIE_DESC_RX_INGRESS_PORT (1 << 8) +#define PCIE_DESC_RX_SPARE (1 << 8) #define PCIE_DESC_RX_EOP (1 << 7) #define PCIE_DESC_RX_IP4_CSUM (1 << 6) #define PCIE_DESC_RX_IP4_CSUM_OK (1 << 5) @@ -326,7 +331,6 @@ struct nfp_net_rxq { * freelist descriptors and @rd_p is where the driver start * reading descriptors for newly arrive packets from. */ - uint32_t wr_p; uint32_t rd_p; /* @@ -433,6 +437,13 @@ struct nfp_net_hw { struct nfp_cpp_area *rx_area; struct nfp_cpp_area *msix_area; #endif + uint8_t *hw_queues; + uint8_t is_pf; + uint8_t pf_port_idx; + uint8_t pf_multiport_enabled; + union eth_table_entry *eth_table; + nspu_desc_t *nspu_desc; + nfpu_desc_t *nfpu_desc; }; struct nfp_net_adapter { diff --git a/dpdk/drivers/net/nfp/nfp_nfpu.c b/dpdk/drivers/net/nfp/nfp_nfpu.c new file mode 100644 index 00000000..f11afef3 --- /dev/null +++ b/dpdk/drivers/net/nfp/nfp_nfpu.c @@ -0,0 +1,108 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "nfp_nfpu.h" + +/* PF BAR and expansion BAR for the NSP interface */ +#define NFP_CFG_PCIE_BAR 0 +#define NFP_CFG_EXP_BAR 7 + +#define NFP_CFG_EXP_BAR_CFG_BASE 0x30000 + +/* There could be other NFP userspace tools using the NSP interface. + * Make sure there is no other process using it and locking the access for + * avoiding problems. + */ +static int +nspv_aquire_process_lock(nfpu_desc_t *desc) +{ + int rc; + struct flock lock; + char lockname[30]; + + memset(&lock, 0, sizeof(lock)); + + snprintf(lockname, sizeof(lockname), "/var/lock/nfp%d", desc->nfp); + + /* Using S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH */ + desc->lock = open(lockname, O_RDWR | O_CREAT, 0666); + + if (desc->lock < 0) + return desc->lock; + + lock.l_type = F_WRLCK; + lock.l_whence = SEEK_SET; + rc = -1; + while (rc != 0) { + rc = fcntl(desc->lock, F_SETLK, &lock); + if (rc < 0) { + if ((errno != EAGAIN) && (errno != EACCES)) { + close(desc->lock); + return rc; + } + } + } + + return 0; +} + +int +nfpu_open(struct rte_pci_device *pci_dev, nfpu_desc_t *desc, int nfp) +{ + void *cfg_base, *mem_base; + size_t barsz; + int ret = 0; + int i = 0; + + desc->nfp = nfp; + + ret = nspv_aquire_process_lock(desc); + if (ret) + return -1; + + barsz = pci_dev->mem_resource[0].len; + + /* barsz in log2 */ + while (barsz >>= 1) + i++; + + barsz = i; + + /* Sanity check: we can assume any bar size less than 1MB an error */ + if (barsz < 20) + return -1; + + /* Getting address for NFP expansion BAR registers */ + cfg_base = pci_dev->mem_resource[0].addr; + cfg_base = (uint8_t *)cfg_base + NFP_CFG_EXP_BAR_CFG_BASE; + + /* Getting address for NFP NSP interface registers */ + mem_base = pci_dev->mem_resource[0].addr; + mem_base = (uint8_t *)mem_base + (NFP_CFG_EXP_BAR << (barsz - 3)); + + + desc->nspu = rte_malloc("nfp nspu", sizeof(nspu_desc_t), 0); + nfp_nspu_init(desc->nspu, desc->nfp, NFP_CFG_PCIE_BAR, barsz, + NFP_CFG_EXP_BAR, cfg_base, mem_base); + + return ret; +} + +int +nfpu_close(nfpu_desc_t *desc) +{ + rte_free(desc->nspu); + close(desc->lock); + unlink("/var/lock/nfp0"); + return 0; +} diff --git a/dpdk/drivers/net/nfp/nfp_nfpu.h b/dpdk/drivers/net/nfp/nfp_nfpu.h new file mode 100644 index 00000000..e56fa099 --- /dev/null +++ b/dpdk/drivers/net/nfp/nfp_nfpu.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2017 Netronome Systems, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * vim:shiftwidth=8:noexpandtab + * + * @file dpdk/pmd/nfp_nfpu.h + * + * Netronome NFP_NET PDM driver + */ + +/* + * NFP User interface creates a window for talking with NFP NSP processor + */ + + +#include +#include "nfp_nspu.h" + +typedef struct { + int nfp; + int lock; + nspu_desc_t *nspu; +} nfpu_desc_t; + +int nfpu_open(struct rte_pci_device *pci_dev, nfpu_desc_t *desc, int nfp); +int nfpu_close(nfpu_desc_t *desc); diff --git a/dpdk/drivers/net/nfp/nfp_nspu.c b/dpdk/drivers/net/nfp/nfp_nspu.c new file mode 100644 index 00000000..f9089832 --- /dev/null +++ b/dpdk/drivers/net/nfp/nfp_nspu.c @@ -0,0 +1,642 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "nfp_nfpu.h" + +#define CFG_EXP_BAR_ADDR_SZ 1 +#define CFG_EXP_BAR_MAP_TYPE 1 + +#define EXP_BAR_TARGET_SHIFT 23 +#define EXP_BAR_LENGTH_SHIFT 27 /* 0=32, 1=64 bit increment */ +#define EXP_BAR_MAP_TYPE_SHIFT 29 /* Bulk BAR map */ + +/* NFP target for NSP access */ +#define NFP_NSP_TARGET 7 + +/* Expansion BARs for mapping PF vnic BARs */ +#define NFP_NET_PF_CFG_EXP_BAR 6 +#define NFP_NET_PF_HW_QUEUES_EXP_BAR 5 + +/* + * This is an NFP internal address used for configuring properly an NFP + * expansion BAR. + */ +#define MEM_CMD_BASE_ADDR 0x8100000000 + +/* NSP interface registers */ +#define NSP_BASE (MEM_CMD_BASE_ADDR + 0x22100) +#define NSP_STATUS 0x00 +#define NSP_COMMAND 0x08 +#define NSP_BUFFER 0x10 +#define NSP_DEFAULT_BUF 0x18 +#define NSP_DEFAULT_BUF_CFG 0x20 + +#define NSP_MAGIC 0xab10 +#define NSP_STATUS_MAGIC(x) (((x) >> 48) & 0xffff) +#define NSP_STATUS_MAJOR(x) (int)(((x) >> 44) & 0xf) +#define NSP_STATUS_MINOR(x) (int)(((x) >> 32) & 0xfff) + +/* NSP commands */ +#define NSP_CMD_RESET 1 +#define NSP_CMD_FW_LOAD 6 +#define NSP_CMD_READ_ETH_TABLE 7 +#define NSP_CMD_WRITE_ETH_TABLE 8 +#define NSP_CMD_GET_SYMBOL 14 + +#define NSP_BUFFER_CFG_SIZE_MASK (0xff) + +#define NSP_REG_ADDR(d, off, reg) ((uint8_t *)(d)->mem_base + (off) + (reg)) +#define NSP_REG_VAL(p) (*(uint64_t *)(p)) + +/* + * An NFP expansion BAR is configured for allowing access to a specific NFP + * target: + * + * IN: + * desc: struct with basic NSP addresses to work with + * expbar: NFP PF expansion BAR index to configure + * tgt: NFP target to configure access + * addr: NFP target address + * + * OUT: + * pcie_offset: NFP PCI BAR offset to work with + */ +static void +nfp_nspu_mem_bar_cfg(nspu_desc_t *desc, int expbar, int tgt, + uint64_t addr, uint64_t *pcie_offset) +{ + uint64_t x, y, barsz; + uint32_t *expbar_ptr; + + barsz = desc->barsz; + + /* + * NFP CPP address to configure. This comes from NFP 6000 + * datasheet document based on Bulk mapping. + */ + x = (addr >> (barsz - 3)) << (21 - (40 - (barsz - 3))); + x |= CFG_EXP_BAR_MAP_TYPE << EXP_BAR_MAP_TYPE_SHIFT; + x |= CFG_EXP_BAR_ADDR_SZ << EXP_BAR_LENGTH_SHIFT; + x |= tgt << EXP_BAR_TARGET_SHIFT; + + /* Getting expansion bar configuration register address */ + expbar_ptr = (uint32_t *)desc->cfg_base; + /* Each physical PCI BAR has 8 NFP expansion BARs */ + expbar_ptr += (desc->pcie_bar * 8) + expbar; + + /* Writing to the expansion BAR register */ + *expbar_ptr = (uint32_t)x; + + /* Getting the pcie offset to work with from userspace */ + y = addr & ((uint64_t)(1 << (barsz - 3)) - 1); + *pcie_offset = y; +} + +/* + * Configuring an expansion bar for accessing NSP userspace interface. This + * function configures always the same expansion bar, which implies access to + * previously configured NFP target is lost. + */ +static void +nspu_xlate(nspu_desc_t *desc, uint64_t addr, uint64_t *pcie_offset) +{ + nfp_nspu_mem_bar_cfg(desc, desc->exp_bar, NFP_NSP_TARGET, addr, + pcie_offset); +} + +int +nfp_nsp_get_abi_version(nspu_desc_t *desc, int *major, int *minor) +{ + uint64_t pcie_offset; + uint64_t nsp_reg; + + nspu_xlate(desc, NSP_BASE, &pcie_offset); + nsp_reg = NSP_REG_VAL(NSP_REG_ADDR(desc, pcie_offset, NSP_STATUS)); + + if (NSP_STATUS_MAGIC(nsp_reg) != NSP_MAGIC) + return -1; + + *major = NSP_STATUS_MAJOR(nsp_reg); + *minor = NSP_STATUS_MINOR(nsp_reg); + + return 0; +} + +int +nfp_nspu_init(nspu_desc_t *desc, int nfp, int pcie_bar, size_t pcie_barsz, + int exp_bar, void *exp_bar_cfg_base, void *exp_bar_mmap) +{ + uint64_t offset, buffaddr; + uint64_t nsp_reg; + + desc->nfp = nfp; + desc->pcie_bar = pcie_bar; + desc->exp_bar = exp_bar; + desc->barsz = pcie_barsz; + desc->windowsz = 1 << (desc->barsz - 3); + desc->cfg_base = exp_bar_cfg_base; + desc->mem_base = exp_bar_mmap; + + nspu_xlate(desc, NSP_BASE, &offset); + + /* + * Other NSPU clients can use other buffers. Let's tell NSPU we use the + * default buffer. + */ + buffaddr = NSP_REG_VAL(NSP_REG_ADDR(desc, offset, NSP_DEFAULT_BUF)); + NSP_REG_VAL(NSP_REG_ADDR(desc, offset, NSP_BUFFER)) = buffaddr; + + /* NFP internal addresses are 40 bits. Clean all other bits here */ + buffaddr = buffaddr & (((uint64_t)1 << 40) - 1); + desc->bufaddr = buffaddr; + + /* Lets get information about the buffer */ + nsp_reg = NSP_REG_VAL(NSP_REG_ADDR(desc, offset, NSP_DEFAULT_BUF_CFG)); + + /* Buffer size comes in MBs. Coversion to bytes */ + desc->buf_size = ((size_t)nsp_reg & NSP_BUFFER_CFG_SIZE_MASK) << 20; + + return 0; +} + +#define NSPU_NFP_BUF(addr, base, off) \ + (*(uint64_t *)((uint8_t *)(addr)->mem_base + ((base) | (off)))) + +#define NSPU_HOST_BUF(base, off) (*(uint64_t *)((uint8_t *)(base) + (off))) + +static int +nspu_buff_write(nspu_desc_t *desc, void *buffer, size_t size) +{ + uint64_t pcie_offset, pcie_window_base, pcie_window_offset; + uint64_t windowsz = desc->windowsz; + uint64_t buffaddr, j, i = 0; + int ret = 0; + + if (size > desc->buf_size) + return -1; + + buffaddr = desc->bufaddr; + windowsz = desc->windowsz; + + while (i < size) { + /* Expansion bar reconfiguration per window size */ + nspu_xlate(desc, buffaddr + i, &pcie_offset); + pcie_window_base = pcie_offset & (~(windowsz - 1)); + pcie_window_offset = pcie_offset & (windowsz - 1); + for (j = pcie_window_offset; ((j < windowsz) && (i < size)); + j += 8) { + NSPU_NFP_BUF(desc, pcie_window_base, j) = + NSPU_HOST_BUF(buffer, i); + i += 8; + } + } + + return ret; +} + +static int +nspu_buff_read(nspu_desc_t *desc, void *buffer, size_t size) +{ + uint64_t pcie_offset, pcie_window_base, pcie_window_offset; + uint64_t windowsz, i = 0, j; + uint64_t buffaddr; + int ret = 0; + + if (size > desc->buf_size) + return -1; + + buffaddr = desc->bufaddr; + windowsz = desc->windowsz; + + while (i < size) { + /* Expansion bar reconfiguration per window size */ + nspu_xlate(desc, buffaddr + i, &pcie_offset); + pcie_window_base = pcie_offset & (~(windowsz - 1)); + pcie_window_offset = pcie_offset & (windowsz - 1); + for (j = pcie_window_offset; ((j < windowsz) && (i < size)); + j += 8) { + NSPU_HOST_BUF(buffer, i) = + NSPU_NFP_BUF(desc, pcie_window_base, j); + i += 8; + } + } + + return ret; +} + +static int +nspu_command(nspu_desc_t *desc, uint16_t cmd, int read, int write, + void *buffer, size_t rsize, size_t wsize) +{ + uint64_t status, cmd_reg; + uint64_t offset; + int retry = 0; + int retries = 120; + int ret = 0; + + /* Same expansion BAR is used for different things */ + nspu_xlate(desc, NSP_BASE, &offset); + + status = NSP_REG_VAL(NSP_REG_ADDR(desc, offset, NSP_STATUS)); + + while ((status & 0x1) && (retry < retries)) { + status = NSP_REG_VAL(NSP_REG_ADDR(desc, offset, NSP_STATUS)); + retry++; + sleep(1); + } + + if (retry == retries) + return -1; + + if (write) { + ret = nspu_buff_write(desc, buffer, wsize); + if (ret) + return ret; + + /* Expansion BAR changes when writing the buffer */ + nspu_xlate(desc, NSP_BASE, &offset); + } + + NSP_REG_VAL(NSP_REG_ADDR(desc, offset, NSP_COMMAND)) = + (uint64_t)wsize << 32 | (uint64_t)cmd << 16 | 1; + + retry = 0; + + cmd_reg = NSP_REG_VAL(NSP_REG_ADDR(desc, offset, NSP_COMMAND)); + while ((cmd_reg & 0x1) && (retry < retries)) { + cmd_reg = NSP_REG_VAL(NSP_REG_ADDR(desc, offset, NSP_COMMAND)); + retry++; + sleep(1); + } + if (retry == retries) + return -1; + + retry = 0; + status = NSP_REG_VAL(NSP_REG_ADDR(desc, offset, NSP_STATUS)); + while ((status & 0x1) && (retry < retries)) { + status = NSP_REG_VAL(NSP_REG_ADDR(desc, offset, NSP_STATUS)); + retry++; + sleep(1); + } + + if (retry == retries) + return -1; + + ret = status & (0xff << 8); + if (ret) + return ret; + + if (read) { + ret = nspu_buff_read(desc, buffer, rsize); + if (ret) + return ret; + } + + return ret; +} + +static int +nfp_fw_reset(nspu_desc_t *nspu_desc) +{ + int res; + + res = nspu_command(nspu_desc, NSP_CMD_RESET, 0, 0, 0, 0, 0); + + if (res < 0) + RTE_LOG(INFO, PMD, "fw reset failed: error %d", res); + + return res; +} + +#define DEFAULT_FW_PATH "/lib/firmware/netronome" +#define DEFAULT_FW_FILENAME "nic_dpdk_default.nffw" + +static int +nfp_fw_upload(nspu_desc_t *nspu_desc) +{ + int fw_f; + char *fw_buf; + char filename[100]; + struct stat file_stat; + off_t fsize, bytes; + ssize_t size; + int ret; + + size = nspu_desc->buf_size; + + sprintf(filename, "%s/%s", DEFAULT_FW_PATH, DEFAULT_FW_FILENAME); + fw_f = open(filename, O_RDONLY); + if (fw_f < 0) { + RTE_LOG(INFO, PMD, "Firmware file %s/%s not found.", + DEFAULT_FW_PATH, DEFAULT_FW_FILENAME); + return -ENOENT; + } + + if (fstat(fw_f, &file_stat) < 0) { + RTE_LOG(INFO, PMD, "Firmware file %s/%s size is unknown", + DEFAULT_FW_PATH, DEFAULT_FW_FILENAME); + close(fw_f); + return -ENOENT; + } + + fsize = file_stat.st_size; + RTE_LOG(DEBUG, PMD, "Firmware file with size: %" PRIu64 "\n", + (uint64_t)fsize); + + if (fsize > (off_t)size) { + RTE_LOG(INFO, PMD, "fw file too big: %" PRIu64 + " bytes (%" PRIu64 " max)", + (uint64_t)fsize, (uint64_t)size); + close(fw_f); + return -EINVAL; + } + + fw_buf = malloc((size_t)size); + if (!fw_buf) { + RTE_LOG(INFO, PMD, "malloc failed for fw buffer"); + close(fw_f); + return -ENOMEM; + } + memset(fw_buf, 0, size); + + bytes = read(fw_f, fw_buf, fsize); + if (bytes != fsize) { + RTE_LOG(INFO, PMD, "Reading fw to buffer failed.\n" + "Just %" PRIu64 " of %" PRIu64 " bytes read.", + (uint64_t)bytes, (uint64_t)fsize); + free(fw_buf); + close(fw_f); + return -EIO; + } + + ret = nspu_command(nspu_desc, NSP_CMD_FW_LOAD, 0, 1, fw_buf, 0, bytes); + + free(fw_buf); + close(fw_f); + + return ret; +} + +/* Firmware symbol descriptor size */ +#define NFP_SYM_DESC_LEN 40 + +#define SYMBOL_DATA(b, off) (*(int64_t *)((b) + (off))) +#define SYMBOL_UDATA(b, off) (*(uint64_t *)((b) + (off))) + +/* Firmware symbols contain information about how to access what they + * represent. It can be as simple as an numeric variable declared at a + * specific NFP memory, but it can also be more complex structures and + * related to specific hardware functionalities or components. Target, + * domain and address allow to create the BAR window for accessing such + * hw object and size defines the length to map. + * + * A vNIC is a network interface implemented inside the NFP and using a + * subset of device PCI BARs. Specific firmware symbols allow to map those + * vNIC bars by host drivers like the NFP PMD. + * + * Accessing what the symbol represents implies to map the access through + * a PCI BAR window. NFP expansion BARs are used in this regard through + * the NSPU interface. + */ +static int +nfp_nspu_set_bar_from_symbl(nspu_desc_t *desc, const char *symbl, + uint32_t expbar, uint64_t *pcie_offset, + ssize_t *size) +{ + int64_t type; + int64_t target; + int64_t domain; + uint64_t addr; + char *sym_buf; + int ret = 0; + + sym_buf = malloc(desc->buf_size); + if (!sym_buf) + return -ENOMEM; + + strncpy(sym_buf, symbl, strlen(symbl)); + ret = nspu_command(desc, NSP_CMD_GET_SYMBOL, 1, 1, sym_buf, + NFP_SYM_DESC_LEN, strlen(symbl)); + if (ret) { + RTE_LOG(DEBUG, PMD, "symbol resolution (%s) failed\n", symbl); + goto clean; + } + + /* Reading symbol information */ + type = SYMBOL_DATA(sym_buf, 0); + target = SYMBOL_DATA(sym_buf, 8); + domain = SYMBOL_DATA(sym_buf, 16); + addr = SYMBOL_UDATA(sym_buf, 24); + *size = (ssize_t)SYMBOL_UDATA(sym_buf, 32); + + if (type != 1) { + RTE_LOG(INFO, PMD, "wrong symbol type\n"); + ret = -EINVAL; + goto clean; + } + if (!(target == 7 || target == -7)) { + RTE_LOG(INFO, PMD, "wrong symbol target\n"); + ret = -EINVAL; + goto clean; + } + if (domain == 8 || domain == 9) { + RTE_LOG(INFO, PMD, "wrong symbol domain\n"); + ret = -EINVAL; + goto clean; + } + + /* Adjusting address based on symbol location */ + if ((domain >= 24) && (domain < 28) && (target == 7)) { + addr = 1ULL << 37 | addr | ((uint64_t)domain & 0x3) << 35; + } else { + addr = 1ULL << 39 | addr | ((uint64_t)domain & 0x3f) << 32; + if (target == -7) + target = 7; + } + + /* Configuring NFP expansion bar for mapping specific PCI BAR window */ + nfp_nspu_mem_bar_cfg(desc, expbar, target, addr, pcie_offset); + + /* This is the PCI BAR offset to use by the host */ + *pcie_offset |= ((expbar & 0x7) << (desc->barsz - 3)); + +clean: + free(sym_buf); + return ret; +} + +int +nfp_nsp_fw_setup(nspu_desc_t *desc, const char *sym, uint64_t *pcie_offset) +{ + ssize_t bar0_sym_size; + + /* If the symbol resolution works, it implies a firmware app + * is already there. + */ + if (!nfp_nspu_set_bar_from_symbl(desc, sym, NFP_NET_PF_CFG_EXP_BAR, + pcie_offset, &bar0_sym_size)) + return 0; + + /* No firmware app detected or not the right one */ + RTE_LOG(INFO, PMD, "No firmware detected. Resetting NFP...\n"); + if (nfp_fw_reset(desc) < 0) { + RTE_LOG(ERR, PMD, "nfp fw reset failed\n"); + return -ENODEV; + } + + RTE_LOG(INFO, PMD, "Reset done.\n"); + RTE_LOG(INFO, PMD, "Uploading firmware...\n"); + + if (nfp_fw_upload(desc) < 0) { + RTE_LOG(ERR, PMD, "nfp fw upload failed\n"); + return -ENODEV; + } + + RTE_LOG(INFO, PMD, "Done.\n"); + + /* Now the symbol should be there */ + if (nfp_nspu_set_bar_from_symbl(desc, sym, NFP_NET_PF_CFG_EXP_BAR, + pcie_offset, &bar0_sym_size)) { + RTE_LOG(ERR, PMD, "nfp PF BAR symbol resolution failed\n"); + return -ENODEV; + } + + return 0; +} + +int +nfp_nsp_map_ctrl_bar(nspu_desc_t *desc, uint64_t *pcie_offset) +{ + ssize_t bar0_sym_size; + + if (nfp_nspu_set_bar_from_symbl(desc, "_pf0_net_bar0", + NFP_NET_PF_CFG_EXP_BAR, + pcie_offset, &bar0_sym_size)) + return -ENODEV; + + return 0; +} + +/* + * This is a hardcoded fixed NFP internal CPP bus address for the hw queues unit + * inside the PCIE island. + */ +#define NFP_CPP_PCIE_QUEUES ((uint64_t)(1ULL << 39) | 0x80000 | \ + ((uint64_t)0x4 & 0x3f) << 32) + +/* Configure a specific NFP expansion bar for accessing the vNIC rx/tx BARs */ +void +nfp_nsp_map_queues_bar(nspu_desc_t *desc, uint64_t *pcie_offset) +{ + nfp_nspu_mem_bar_cfg(desc, NFP_NET_PF_HW_QUEUES_EXP_BAR, 0, + NFP_CPP_PCIE_QUEUES, pcie_offset); + + /* This is the pcie offset to use by the host */ + *pcie_offset |= ((NFP_NET_PF_HW_QUEUES_EXP_BAR & 0x7) << (27 - 3)); +} + +int +nfp_nsp_eth_config(nspu_desc_t *desc, int port, int up) +{ + union eth_table_entry *entries, *entry; + int modified; + int ret, idx; + int i; + + idx = port; + + RTE_LOG(INFO, PMD, "Hw ethernet port %d configure...\n", port); + rte_spinlock_lock(&desc->nsp_lock); + entries = malloc(NSP_ETH_TABLE_SIZE); + if (!entries) { + rte_spinlock_unlock(&desc->nsp_lock); + return -ENOMEM; + } + + ret = nspu_command(desc, NSP_CMD_READ_ETH_TABLE, 1, 0, entries, + NSP_ETH_TABLE_SIZE, 0); + if (ret) { + rte_spinlock_unlock(&desc->nsp_lock); + free(entries); + return ret; + } + + entry = entries; + + for (i = 0; i < NSP_ETH_MAX_COUNT; i++) { + /* ports in use do not appear sequentially in the table */ + if (!(entry->port & NSP_ETH_PORT_LANES_MASK)) { + /* entry not in use */ + entry++; + continue; + } + if (idx == 0) + break; + idx--; + entry++; + } + + if (i == NSP_ETH_MAX_COUNT) { + rte_spinlock_unlock(&desc->nsp_lock); + free(entries); + return -EINVAL; + } + + if (up && !(entry->state & NSP_ETH_STATE_CONFIGURED)) { + entry->control |= NSP_ETH_STATE_CONFIGURED; + modified = 1; + } + + if (!up && (entry->state & NSP_ETH_STATE_CONFIGURED)) { + entry->control &= ~NSP_ETH_STATE_CONFIGURED; + modified = 1; + } + + if (modified) { + ret = nspu_command(desc, NSP_CMD_WRITE_ETH_TABLE, 0, 1, entries, + 0, NSP_ETH_TABLE_SIZE); + if (!ret) + RTE_LOG(INFO, PMD, + "Hw ethernet port %d configure done\n", port); + else + RTE_LOG(INFO, PMD, + "Hw ethernet port %d configure failed\n", port); + } + rte_spinlock_unlock(&desc->nsp_lock); + free(entries); + return ret; +} + +int +nfp_nsp_eth_read_table(nspu_desc_t *desc, union eth_table_entry **table) +{ + int ret; + + if (!table) + return -EINVAL; + + RTE_LOG(INFO, PMD, "Reading hw ethernet table...\n"); + + /* port 0 allocates the eth table and read it using NSPU */ + *table = malloc(NSP_ETH_TABLE_SIZE); + if (!*table) + return -ENOMEM; + + ret = nspu_command(desc, NSP_CMD_READ_ETH_TABLE, 1, 0, *table, + NSP_ETH_TABLE_SIZE, 0); + if (ret) + return ret; + + RTE_LOG(INFO, PMD, "Done\n"); + + return 0; +} diff --git a/dpdk/drivers/net/nfp/nfp_nspu.h b/dpdk/drivers/net/nfp/nfp_nspu.h new file mode 100644 index 00000000..8c33835e --- /dev/null +++ b/dpdk/drivers/net/nfp/nfp_nspu.h @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2017 Netronome Systems, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * vim:shiftwidth=8:noexpandtab + * + * @file dpdk/pmd/nfp_nspu.h + * + * Netronome NFP_NET PDM driver + */ + +/* + * NSP is the NFP Service Processor. NSPU is NSP Userspace interface. + * + * NFP NSP helps with firmware/hardware configuration. NSP is another component + * in NFP programmable processor and accessing it from host requires to firstly + * configure a specific NFP PCI expansion BAR. + * + * Once access is ready, configuration can be done reading and writing + * from/to a specific PF PCI BAR window. This same interface will allow to + * create other PCI BAR windows for accessing other NFP components. + * + * This file includes low-level functions, using the NSPU interface, and high + * level functions, invoked by the PMD for using NSP services. This allows + * firmware upload, vNIC PCI BARs mapping and other low-level configurations + * like link setup. + * + * NSP access is done during initialization and it is not involved at all with + * the fast path. + */ + +#include +#include "nfp_net_eth.h" + +typedef struct { + int nfp; /* NFP device */ + int pcie_bar; /* PF PCI BAR to work with */ + int exp_bar; /* Expansion BAR number used by NSPU */ + int barsz; /* PCIE BAR log2 size */ + uint64_t bufaddr; /* commands buffer address */ + size_t buf_size; /* commands buffer size */ + uint64_t windowsz; /* NSPU BAR window size */ + void *cfg_base; /* Expansion BARs address */ + void *mem_base; /* NSP interface */ + rte_spinlock_t nsp_lock; +} nspu_desc_t; + +int nfp_nspu_init(nspu_desc_t *desc, int nfp, int pcie_bar, size_t pcie_barsz, + int exp_bar, void *exp_bar_cfg_base, void *exp_bar_mmap); +int nfp_nsp_get_abi_version(nspu_desc_t *desc, int *major, int *minor); +int nfp_nsp_fw_setup(nspu_desc_t *desc, const char *sym, uint64_t *pcie_offset); +int nfp_nsp_map_ctrl_bar(nspu_desc_t *desc, uint64_t *pcie_offset); +void nfp_nsp_map_queues_bar(nspu_desc_t *desc, uint64_t *pcie_offset); +int nfp_nsp_eth_config(nspu_desc_t *desc, int port, int up); +int nfp_nsp_eth_read_table(nspu_desc_t *desc, union eth_table_entry **table); diff --git a/dpdk/drivers/net/null/Makefile b/dpdk/drivers/net/null/Makefile index 0c909c6f..9331ccac 100644 --- a/dpdk/drivers/net/null/Makefile +++ b/dpdk/drivers/net/null/Makefile @@ -38,26 +38,17 @@ LIB = librte_pmd_null.a CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs +LDLIBS += -lrte_bus_vdev EXPORT_MAP := rte_pmd_null_version.map -LIBABIVER := 1 +LIBABIVER := 2 # # all source are stored in SRCS-y # SRCS-$(CONFIG_RTE_LIBRTE_PMD_NULL) += rte_eth_null.c -# -# Export include files -# -SYMLINK-y-include += rte_eth_null.h - -# this lib depends upon: -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_NULL) += lib/librte_eal -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_NULL) += lib/librte_mbuf -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_NULL) += lib/librte_mempool -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_NULL) += lib/librte_ether -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_NULL) += lib/librte_kvargs - include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/net/null/rte_eth_null.c b/dpdk/drivers/net/null/rte_eth_null.c index 7a248842..726a5c5e 100644 --- a/dpdk/drivers/net/null/rte_eth_null.c +++ b/dpdk/drivers/net/null/rte_eth_null.c @@ -33,14 +33,13 @@ #include #include +#include #include #include -#include +#include #include #include -#include "rte_eth_null.h" - #define ETH_NULL_PACKET_SIZE_ARG "size" #define ETH_NULL_PACKET_COPY_ARG "copy" @@ -69,7 +68,7 @@ struct null_queue { struct pmd_internals { unsigned packet_size; unsigned packet_copy; - uint8_t port_id; + uint16_t port_id; struct null_queue rx_null_queues[RTE_MAX_QUEUES_PER_PORT]; struct null_queue tx_null_queues[RTE_MAX_QUEUES_PER_PORT]; @@ -88,12 +87,11 @@ struct pmd_internals { static struct ether_addr eth_addr = { .addr_bytes = {0} }; -static const char *drivername = "Null PMD"; static struct rte_eth_link pmd_link = { .link_speed = ETH_SPEED_NUM_10G, .link_duplex = ETH_LINK_FULL_DUPLEX, .link_status = ETH_LINK_DOWN, - .link_autoneg = ETH_LINK_SPEED_AUTONEG, + .link_autoneg = ETH_LINK_AUTONEG, }; static uint16_t @@ -113,8 +111,6 @@ eth_null_rx(void *q, struct rte_mbuf **bufs, uint16_t nb_bufs) break; bufs[i]->data_len = (uint16_t)packet_size; bufs[i]->pkt_len = packet_size; - bufs[i]->nb_segs = 1; - bufs[i]->next = NULL; bufs[i]->port = h->internals->port_id; } @@ -142,8 +138,6 @@ eth_null_copy_rx(void *q, struct rte_mbuf **bufs, uint16_t nb_bufs) packet_size); bufs[i]->data_len = (uint16_t)packet_size; bufs[i]->pkt_len = packet_size; - bufs[i]->nb_segs = 1; - bufs[i]->next = NULL; bufs[i]->port = h->internals->port_id; } @@ -295,18 +289,16 @@ eth_dev_info(struct rte_eth_dev *dev, return; internals = dev->data->dev_private; - dev_info->driver_name = drivername; dev_info->max_mac_addrs = 1; dev_info->max_rx_pktlen = (uint32_t)-1; dev_info->max_rx_queues = RTE_DIM(internals->rx_null_queues); dev_info->max_tx_queues = RTE_DIM(internals->tx_null_queues); dev_info->min_rx_bufsize = 0; - dev_info->pci_dev = NULL; dev_info->reta_size = internals->reta_size; dev_info->flow_type_rss_offloads = internals->flow_type_rss_offloads; } -static void +static int eth_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *igb_stats) { unsigned i, num_stats; @@ -314,7 +306,7 @@ eth_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *igb_stats) const struct pmd_internals *internal; if ((dev == NULL) || (igb_stats == NULL)) - return; + return -EINVAL; internal = dev->data->dev_private; num_stats = RTE_MIN((unsigned)RTE_ETHDEV_QUEUE_STAT_CNTRS, @@ -341,6 +333,8 @@ eth_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *igb_stats) igb_stats->ipackets = rx_total; igb_stats->opackets = tx_total; igb_stats->oerrors = tx_err_total; + + return 0; } static void @@ -480,9 +474,10 @@ static const struct eth_dev_ops ops = { .rss_hash_conf_get = eth_rss_hash_conf_get }; -int -eth_dev_null_create(const char *name, - const unsigned numa_node, +static struct rte_vdev_driver pmd_null_drv; + +static int +eth_dev_null_create(struct rte_vdev_device *dev, unsigned packet_size, unsigned packet_copy) { @@ -499,27 +494,25 @@ eth_dev_null_create(const char *name, 0xBE, 0xAC, 0x01, 0xFA }; - if (name == NULL) - return -EINVAL; + if (dev->device.numa_node == SOCKET_ID_ANY) + dev->device.numa_node = rte_socket_id(); RTE_LOG(INFO, PMD, "Creating null ethdev on numa socket %u\n", - numa_node); + dev->device.numa_node); /* now do all data allocation - for eth_dev structure, dummy pci driver * and internal (private) data */ - data = rte_zmalloc_socket(name, sizeof(*data), 0, numa_node); - if (data == NULL) - goto error; + data = rte_zmalloc_socket(rte_vdev_device_name(dev), sizeof(*data), 0, + dev->device.numa_node); + if (!data) + return -ENOMEM; - internals = rte_zmalloc_socket(name, sizeof(*internals), 0, numa_node); - if (internals == NULL) - goto error; - - /* reserve an ethdev entry */ - eth_dev = rte_eth_dev_allocate(name, RTE_ETH_DEV_VIRTUAL); - if (eth_dev == NULL) - goto error; + eth_dev = rte_eth_vdev_allocate(dev, sizeof(*internals)); + if (!eth_dev) { + rte_free(data); + return -ENOMEM; + } /* now put it all together * - store queue data in internals, @@ -530,6 +523,7 @@ eth_dev_null_create(const char *name, /* NOTE: we'll replace the data element, of originally allocated eth_dev * so the nulls are local per-process */ + internals = eth_dev->data->dev_private; internals->packet_size = packet_size; internals->packet_copy = packet_copy; internals->port_id = eth_dev->data->port_id; @@ -539,25 +533,15 @@ eth_dev_null_create(const char *name, rte_memcpy(internals->rss_key, default_rss_key, 40); - data->dev_private = internals; - data->port_id = eth_dev->data->port_id; + rte_memcpy(data, eth_dev->data, sizeof(*data)); data->nb_rx_queues = (uint16_t)nb_rx_queues; data->nb_tx_queues = (uint16_t)nb_tx_queues; data->dev_link = pmd_link; data->mac_addrs = ð_addr; - strncpy(data->name, eth_dev->data->name, strlen(eth_dev->data->name)); eth_dev->data = data; eth_dev->dev_ops = &ops; - TAILQ_INIT(ð_dev->link_intr_cbs); - - eth_dev->driver = NULL; - data->dev_flags = RTE_ETH_DEV_DETACHABLE; - data->kdrv = RTE_KDRV_NONE; - data->drv_name = drivername; - data->numa_node = numa_node; - /* finally assign rx and tx ops */ if (packet_copy) { eth_dev->rx_pkt_burst = eth_null_copy_rx; @@ -568,12 +552,6 @@ eth_dev_null_create(const char *name, } return 0; - -error: - rte_free(data); - rte_free(internals); - - return -1; } static inline int @@ -611,21 +589,21 @@ get_packet_copy_arg(const char *key __rte_unused, } static int -rte_pmd_null_devinit(const char *name, const char *params) +rte_pmd_null_probe(struct rte_vdev_device *dev) { - unsigned numa_node; + const char *name, *params; unsigned packet_size = default_packet_size; unsigned packet_copy = default_packet_copy; struct rte_kvargs *kvlist = NULL; int ret; - if (name == NULL) + if (!dev) return -EINVAL; + name = rte_vdev_device_name(dev); + params = rte_vdev_device_args(dev); RTE_LOG(INFO, PMD, "Initializing pmd_null for %s\n", name); - numa_node = rte_socket_id(); - if (params != NULL) { kvlist = rte_kvargs_parse(params, valid_arguments); if (kvlist == NULL) @@ -654,7 +632,7 @@ rte_pmd_null_devinit(const char *name, const char *params) "packet copy is %s\n", packet_size, packet_copy ? "enabled" : "disabled"); - ret = eth_dev_null_create(name, numa_node, packet_size, packet_copy); + ret = eth_dev_null_create(dev, packet_size, packet_copy); free_kvlist: if (kvlist) @@ -663,18 +641,18 @@ free_kvlist: } static int -rte_pmd_null_devuninit(const char *name) +rte_pmd_null_remove(struct rte_vdev_device *dev) { struct rte_eth_dev *eth_dev = NULL; - if (name == NULL) + if (!dev) return -EINVAL; RTE_LOG(INFO, PMD, "Closing null ethdev on numa socket %u\n", rte_socket_id()); /* find the ethdev entry */ - eth_dev = rte_eth_dev_allocated(name); + eth_dev = rte_eth_dev_allocated(rte_vdev_device_name(dev)); if (eth_dev == NULL) return -1; @@ -686,13 +664,13 @@ rte_pmd_null_devuninit(const char *name) return 0; } -static struct rte_driver pmd_null_drv = { - .type = PMD_VDEV, - .init = rte_pmd_null_devinit, - .uninit = rte_pmd_null_devuninit, +static struct rte_vdev_driver pmd_null_drv = { + .probe = rte_pmd_null_probe, + .remove = rte_pmd_null_remove, }; -PMD_REGISTER_DRIVER(pmd_null_drv, eth_null); -DRIVER_REGISTER_PARAM_STRING(eth_null, +RTE_PMD_REGISTER_VDEV(net_null, pmd_null_drv); +RTE_PMD_REGISTER_ALIAS(net_null, eth_null); +RTE_PMD_REGISTER_PARAM_STRING(net_null, "size= " "copy="); diff --git a/dpdk/drivers/net/null/rte_eth_null.h b/dpdk/drivers/net/null/rte_eth_null.h deleted file mode 100644 index abada8c2..00000000 --- a/dpdk/drivers/net/null/rte_eth_null.h +++ /dev/null @@ -1,40 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2015 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef RTE_ETH_NULL_H_ -#define RTE_ETH_NULL_H_ - -int eth_dev_null_create(const char *name, const unsigned numa_node, - unsigned packet_size, unsigned packet_copy); - -#endif /* RTE_ETH_NULL_H_ */ diff --git a/dpdk/drivers/net/null/rte_pmd_null_version.map b/dpdk/drivers/net/null/rte_pmd_null_version.map index 84b1d0fe..ef353984 100644 --- a/dpdk/drivers/net/null/rte_pmd_null_version.map +++ b/dpdk/drivers/net/null/rte_pmd_null_version.map @@ -2,10 +2,3 @@ DPDK_2.0 { local: *; }; - -DPDK_2.2 { - global: - - eth_dev_null_create; - -} DPDK_2.0; diff --git a/dpdk/drivers/net/octeontx/Makefile b/dpdk/drivers/net/octeontx/Makefile new file mode 100644 index 00000000..9c27fdfe --- /dev/null +++ b/dpdk/drivers/net/octeontx/Makefile @@ -0,0 +1,79 @@ +# BSD LICENSE +# +# Copyright(c) 2017 Cavium Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Cavium Networks nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +include $(RTE_SDK)/mk/rte.vars.mk + +# +# library name +# +LIB = librte_pmd_octeontx.a + +CFLAGS += $(WERROR_FLAGS) +CFLAGS += -I$(RTE_SDK)/drivers/mempool/octeontx/ + +EXPORT_MAP := rte_pmd_octeontx_version.map + +LIBABIVER := 1 + +OBJS_BASE_DRIVER=$(patsubst %.c,%.o,$(notdir $(wildcard $(SRCDIR)/base/*.c))) +$(foreach obj, $(OBJS_BASE_DRIVER), $(eval CFLAGS_$(obj)+=$(CFLAGS_BASE_DRIVER))) + +VPATH += $(SRCDIR)/base + +# +# all source are stored in SRCS-y +# +SRCS-$(CONFIG_RTE_LIBRTE_OCTEONTX_PMD) += octeontx_rxtx.c +SRCS-$(CONFIG_RTE_LIBRTE_OCTEONTX_PMD) += octeontx_pkovf.c +SRCS-$(CONFIG_RTE_LIBRTE_OCTEONTX_PMD) += octeontx_pkivf.c +SRCS-$(CONFIG_RTE_LIBRTE_OCTEONTX_PMD) += octeontx_bgx.c +SRCS-$(CONFIG_RTE_LIBRTE_OCTEONTX_PMD) += octeontx_ethdev.c + +ifeq ($(CONFIG_RTE_TOOLCHAIN_GCC),y) +CFLAGS_octeontx_rxtx.o += -fno-prefetch-loop-arrays + +ifeq ($(shell test $(GCC_VERSION) -ge 46 && echo 1), 1) +CFLAGS_octeontx_rxtx.o += -O3 -Ofast +else +CFLAGS_octeontx_rxtx.o += -O3 -ffast-math +endif + +else +CFLAGS_octeontx_rxtx.o += -O3 -Ofast +endif + +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs +LDLIBS += -lrte_mempool_octeontx +LDLIBS += -lrte_eventdev +LDLIBS += -lrte_bus_pci +LDLIBS += -lrte_bus_vdev + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/net/octeontx/base/octeontx_bgx.c b/dpdk/drivers/net/octeontx/base/octeontx_bgx.c new file mode 100644 index 00000000..c2d0d433 --- /dev/null +++ b/dpdk/drivers/net/octeontx/base/octeontx_bgx.c @@ -0,0 +1,273 @@ +/* + * BSD LICENSE + * + * Copyright (C) Cavium Inc. 2017. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium networks nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +#include "octeontx_bgx.h" + +int +octeontx_bgx_port_open(int port, octeontx_mbox_bgx_port_conf_t *conf) +{ + struct octeontx_mbox_hdr hdr; + octeontx_mbox_bgx_port_conf_t bgx_conf; + int len = sizeof(octeontx_mbox_bgx_port_conf_t); + int res; + + memset(&bgx_conf, 0, sizeof(octeontx_mbox_bgx_port_conf_t)); + hdr.coproc = OCTEONTX_BGX_COPROC; + hdr.msg = MBOX_BGX_PORT_OPEN; + hdr.vfid = port; + + res = octeontx_ssovf_mbox_send(&hdr, NULL, 0, &bgx_conf, len); + if (res < 0) + return -EACCES; + + conf->enable = bgx_conf.enable; + conf->promisc = bgx_conf.promisc; + conf->bpen = bgx_conf.bpen; + conf->node = bgx_conf.node; + conf->base_chan = bgx_conf.base_chan; + conf->num_chans = bgx_conf.num_chans; + conf->mtu = bgx_conf.mtu; + conf->bgx = bgx_conf.bgx; + conf->lmac = bgx_conf.lmac; + conf->mode = bgx_conf.mode; + conf->pkind = bgx_conf.pkind; + memcpy(conf->macaddr, bgx_conf.macaddr, 6); + + return res; +} + +int +octeontx_bgx_port_close(int port) +{ + struct octeontx_mbox_hdr hdr; + int res; + + hdr.coproc = OCTEONTX_BGX_COPROC; + hdr.msg = MBOX_BGX_PORT_CLOSE; + hdr.vfid = port; + + res = octeontx_ssovf_mbox_send(&hdr, NULL, 0, NULL, 0); + if (res < 0) + return -EACCES; + + return res; +} + +int +octeontx_bgx_port_start(int port) +{ + struct octeontx_mbox_hdr hdr; + int res; + + hdr.coproc = OCTEONTX_BGX_COPROC; + hdr.msg = MBOX_BGX_PORT_START; + hdr.vfid = port; + + res = octeontx_ssovf_mbox_send(&hdr, NULL, 0, NULL, 0); + if (res < 0) + return -EACCES; + + return res; +} + +int +octeontx_bgx_port_stop(int port) +{ + struct octeontx_mbox_hdr hdr; + int res; + + hdr.coproc = OCTEONTX_BGX_COPROC; + hdr.msg = MBOX_BGX_PORT_STOP; + hdr.vfid = port; + + res = octeontx_ssovf_mbox_send(&hdr, NULL, 0, NULL, 0); + if (res < 0) + return -EACCES; + + return res; +} + +int +octeontx_bgx_port_get_config(int port, octeontx_mbox_bgx_port_conf_t *conf) +{ + struct octeontx_mbox_hdr hdr; + octeontx_mbox_bgx_port_conf_t bgx_conf; + int len = sizeof(octeontx_mbox_bgx_port_conf_t); + int res; + + hdr.coproc = OCTEONTX_BGX_COPROC; + hdr.msg = MBOX_BGX_PORT_GET_CONFIG; + hdr.vfid = port; + + memset(&bgx_conf, 0, sizeof(octeontx_mbox_bgx_port_conf_t)); + res = octeontx_ssovf_mbox_send(&hdr, NULL, 0, &bgx_conf, len); + if (res < 0) + return -EACCES; + + conf->enable = bgx_conf.enable; + conf->promisc = bgx_conf.promisc; + conf->bpen = bgx_conf.bpen; + conf->node = bgx_conf.node; + conf->base_chan = bgx_conf.base_chan; + conf->num_chans = bgx_conf.num_chans; + conf->mtu = bgx_conf.mtu; + conf->bgx = bgx_conf.bgx; + conf->lmac = bgx_conf.lmac; + conf->mode = bgx_conf.mode; + conf->pkind = bgx_conf.pkind; + memcpy(conf->macaddr, bgx_conf.macaddr, 6); + + return res; +} + +int +octeontx_bgx_port_status(int port, octeontx_mbox_bgx_port_status_t *stat) +{ + struct octeontx_mbox_hdr hdr; + octeontx_mbox_bgx_port_status_t bgx_stat; + int len = sizeof(octeontx_mbox_bgx_port_status_t); + int res; + + hdr.coproc = OCTEONTX_BGX_COPROC; + hdr.msg = MBOX_BGX_PORT_GET_STATUS; + hdr.vfid = port; + + res = octeontx_ssovf_mbox_send(&hdr, NULL, 0, &bgx_stat, len); + if (res < 0) + return -EACCES; + + stat->link_up = bgx_stat.link_up; + + return res; +} + +int +octeontx_bgx_port_stats(int port, octeontx_mbox_bgx_port_stats_t *stats) +{ + struct octeontx_mbox_hdr hdr; + octeontx_mbox_bgx_port_stats_t bgx_stats; + int len = sizeof(octeontx_mbox_bgx_port_stats_t); + int res; + + hdr.coproc = OCTEONTX_BGX_COPROC; + hdr.msg = MBOX_BGX_PORT_GET_STATS; + hdr.vfid = port; + + res = octeontx_ssovf_mbox_send(&hdr, NULL, 0, &bgx_stats, len); + if (res < 0) + return -EACCES; + + stats->rx_packets = bgx_stats.rx_packets; + stats->rx_bytes = bgx_stats.rx_bytes; + stats->rx_dropped = bgx_stats.rx_dropped; + stats->rx_errors = bgx_stats.rx_errors; + stats->tx_packets = bgx_stats.tx_packets; + stats->tx_bytes = bgx_stats.tx_bytes; + stats->tx_dropped = bgx_stats.tx_dropped; + stats->tx_errors = bgx_stats.tx_errors; + return res; +} + +int +octeontx_bgx_port_stats_clr(int port) +{ + struct octeontx_mbox_hdr hdr; + int res; + + hdr.coproc = OCTEONTX_BGX_COPROC; + hdr.msg = MBOX_BGX_PORT_CLR_STATS; + hdr.vfid = port; + + res = octeontx_ssovf_mbox_send(&hdr, NULL, 0, NULL, 0); + if (res < 0) + return -EACCES; + + return res; +} + +int +octeontx_bgx_port_link_status(int port) +{ + struct octeontx_mbox_hdr hdr; + uint8_t link; + int len = sizeof(uint8_t); + int res; + + hdr.coproc = OCTEONTX_BGX_COPROC; + hdr.msg = MBOX_BGX_PORT_GET_LINK_STATUS; + hdr.vfid = port; + + res = octeontx_ssovf_mbox_send(&hdr, NULL, 0, &link, len); + if (res < 0) + return -EACCES; + + return link; +} + +int +octeontx_bgx_port_promisc_set(int port, int en) +{ + struct octeontx_mbox_hdr hdr; + uint8_t prom; + int res; + + hdr.coproc = OCTEONTX_BGX_COPROC; + hdr.msg = MBOX_BGX_PORT_SET_PROMISC; + hdr.vfid = port; + prom = en ? 1 : 0; + + res = octeontx_ssovf_mbox_send(&hdr, &prom, sizeof(prom), NULL, 0); + if (res < 0) + return -EACCES; + + return res; +} + +int +octeontx_bgx_port_mac_set(int port, uint8_t *mac_addr) +{ + struct octeontx_mbox_hdr hdr; + int len = 6; + int res = 0; + + hdr.coproc = OCTEONTX_BGX_COPROC; + hdr.msg = MBOX_BGX_PORT_SET_MACADDR; + hdr.vfid = port; + + res = octeontx_ssovf_mbox_send(&hdr, mac_addr, len, NULL, 0); + if (res < 0) + return -EACCES; + + return res; +} diff --git a/dpdk/drivers/net/octeontx/base/octeontx_bgx.h b/dpdk/drivers/net/octeontx/base/octeontx_bgx.h new file mode 100644 index 00000000..f740a1d9 --- /dev/null +++ b/dpdk/drivers/net/octeontx/base/octeontx_bgx.h @@ -0,0 +1,150 @@ +/* + * BSD LICENSE + * + * Copyright (C) Cavium Inc. 2017. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium networks nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __OCTEONTX_BGX_H__ +#define __OCTEONTX_BGX_H__ + +#include +#include + +#include + +#define OCTEONTX_BGX_COPROC 6 + +/* BGX messages */ +#define MBOX_BGX_PORT_OPEN 0 +#define MBOX_BGX_PORT_CLOSE 1 +#define MBOX_BGX_PORT_START 2 +#define MBOX_BGX_PORT_STOP 3 +#define MBOX_BGX_PORT_GET_CONFIG 4 +#define MBOX_BGX_PORT_GET_STATUS 5 +#define MBOX_BGX_PORT_GET_STATS 6 +#define MBOX_BGX_PORT_CLR_STATS 7 +#define MBOX_BGX_PORT_GET_LINK_STATUS 8 +#define MBOX_BGX_PORT_SET_PROMISC 9 +#define MBOX_BGX_PORT_SET_MACADDR 10 +#define MBOX_BGX_PORT_SET_BP 11 +#define MBOX_BGX_PORT_SET_BCAST 12 +#define MBOX_BGX_PORT_SET_MCAST 13 + +/* BGX port configuration parameters: */ +typedef struct octeontx_mbox_bgx_port_conf { + uint8_t enable; + uint8_t promisc; + uint8_t bpen; + uint8_t macaddr[6]; /* MAC address.*/ + uint8_t fcs_strip; + uint8_t bcast_mode; + uint8_t mcast_mode; + uint8_t node; /* CPU node */ + uint16_t base_chan; + uint16_t num_chans; + uint16_t mtu; + uint8_t bgx; + uint8_t lmac; + uint8_t mode; + uint8_t pkind; +} octeontx_mbox_bgx_port_conf_t; + +/* BGX port status: */ +typedef struct octeontx_mbox_bgx_port_status { + uint8_t link_up; + uint8_t bp; +} octeontx_mbox_bgx_port_status_t; + +/* BGX port statistics: */ +typedef struct octeontx_mbox_bgx_port_stats { + uint64_t rx_packets; + uint64_t tx_packets; + uint64_t rx_bytes; + uint64_t tx_bytes; + uint64_t rx_errors; + uint64_t tx_errors; + uint64_t rx_dropped; + uint64_t tx_dropped; + uint64_t multicast; + uint64_t collisions; + + uint64_t rx_length_errors; + uint64_t rx_over_errors; + uint64_t rx_crc_errors; + uint64_t rx_frame_errors; + uint64_t rx_fifo_errors; + uint64_t rx_missed_errors; + + /* Detailed transmit errors. */ + uint64_t tx_aborted_errors; + uint64_t tx_carrier_errors; + uint64_t tx_fifo_errors; + uint64_t tx_heartbeat_errors; + uint64_t tx_window_errors; + + /* Extended statistics based on RFC2819. */ + uint64_t rx_1_to_64_packets; + uint64_t rx_65_to_127_packets; + uint64_t rx_128_to_255_packets; + uint64_t rx_256_to_511_packets; + uint64_t rx_512_to_1023_packets; + uint64_t rx_1024_to_1522_packets; + uint64_t rx_1523_to_max_packets; + + uint64_t tx_1_to_64_packets; + uint64_t tx_65_to_127_packets; + uint64_t tx_128_to_255_packets; + uint64_t tx_256_to_511_packets; + uint64_t tx_512_to_1023_packets; + uint64_t tx_1024_to_1522_packets; + uint64_t tx_1523_to_max_packets; + + uint64_t tx_multicast_packets; + uint64_t rx_broadcast_packets; + uint64_t tx_broadcast_packets; + uint64_t rx_undersized_errors; + uint64_t rx_oversize_errors; + uint64_t rx_fragmented_errors; + uint64_t rx_jabber_errors; +} octeontx_mbox_bgx_port_stats_t; + +int octeontx_bgx_port_open(int port, octeontx_mbox_bgx_port_conf_t *conf); +int octeontx_bgx_port_close(int port); +int octeontx_bgx_port_start(int port); +int octeontx_bgx_port_stop(int port); +int octeontx_bgx_port_get_config(int port, octeontx_mbox_bgx_port_conf_t *conf); +int octeontx_bgx_port_status(int port, octeontx_mbox_bgx_port_status_t *stat); +int octeontx_bgx_port_stats(int port, octeontx_mbox_bgx_port_stats_t *stats); +int octeontx_bgx_port_stats_clr(int port); +int octeontx_bgx_port_link_status(int port); +int octeontx_bgx_port_promisc_set(int port, int en); +int octeontx_bgx_port_mac_set(int port, uint8_t *mac_addr); + +#endif /* __OCTEONTX_BGX_H__ */ + diff --git a/dpdk/drivers/net/octeontx/base/octeontx_io.h b/dpdk/drivers/net/octeontx/base/octeontx_io.h new file mode 100644 index 00000000..ec4ce1dc --- /dev/null +++ b/dpdk/drivers/net/octeontx/base/octeontx_io.h @@ -0,0 +1,156 @@ +/* + * BSD LICENSE + * + * Copyright (C) Cavium Inc. 2017. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium networks nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __OCTEONTX_IO_H__ +#define __OCTEONTX_IO_H__ + +#include +#include + +#include + +/* In Cavium OcteonTX SoC, all accesses to the device registers are + * implicitly strongly ordered. So, The relaxed version of IO operation is + * safe to use with out any IO memory barriers. + */ +#define octeontx_read64 rte_read64_relaxed +#define octeontx_write64 rte_write64_relaxed + +/* ARM64 specific functions */ +#if defined(RTE_ARCH_ARM64) +#define octeontx_prefetch_store_keep(_ptr) ({\ + asm volatile("prfm pstl1keep, %a0\n" : : "p" (_ptr)); }) + +#define octeontx_load_pair(val0, val1, addr) ({ \ + asm volatile( \ + "ldp %x[x0], %x[x1], [%x[p1]]" \ + :[x0]"=r"(val0), [x1]"=r"(val1) \ + :[p1]"r"(addr) \ + ); }) + +#define octeontx_store_pair(val0, val1, addr) ({ \ + asm volatile( \ + "stp %x[x0], %x[x1], [%x[p1]]" \ + ::[x0]"r"(val0), [x1]"r"(val1), [p1]"r"(addr) \ + ); }) +#else /* Un optimized functions for building on non arm64 arch */ + +#define octeontx_prefetch_store_keep(_ptr) do {} while (0) + +#define octeontx_load_pair(val0, val1, addr) \ +do { \ + val0 = rte_read64(addr); \ + val1 = rte_read64(((uint8_t *)addr) + 8); \ +} while (0) + +#define octeontx_store_pair(val0, val1, addr) \ +do { \ + rte_write64(val0, addr); \ + rte_write64(val1, (((uint8_t *)addr) + 8)); \ +} while (0) +#endif + +#if defined(RTE_ARCH_ARM64) +/** + * Perform an atomic fetch-and-add operation. + */ +static inline uint64_t +octeontx_reg_ldadd_u64(void *addr, int64_t off) +{ + uint64_t old_val; + + __asm__ volatile( + " .cpu generic+lse\n" + " ldadd %1, %0, [%2]\n" + : "=r" (old_val) : "r" (off), "r" (addr) : "memory"); + + return old_val; +} + +/** + * Perform a LMTST operation - an atomic write of up to 128 byte to + * an I/O block that supports this operation type. + * + * @param lmtline_va is the address where LMTLINE is mapped + * @param ioreg_va is the virtual address of the device register + * @param cmdbuf is the array of peripheral commands to execute + * @param cmdsize is the number of 64-bit words in 'cmdbuf' + * + * @return N/A + */ +static inline void +octeontx_reg_lmtst(void *lmtline_va, void *ioreg_va, const uint64_t cmdbuf[], + uint64_t cmdsize) +{ + uint64_t result; + uint64_t word_count; + uint64_t *lmtline = lmtline_va; + + word_count = cmdsize; + + do { + /* Copy commands to LMTLINE */ + for (result = 0; result < word_count; result += 2) { + lmtline[result + 0] = cmdbuf[result + 0]; + lmtline[result + 1] = cmdbuf[result + 1]; + } + + /* LDEOR initiates atomic transfer to I/O device */ + __asm__ volatile( + " .cpu generic+lse\n" + " ldeor xzr, %0, [%1]\n" + : "=r" (result) : "r" (ioreg_va) : "memory"); + } while (!result); +} + +#else + +static inline uint64_t +octeontx_reg_ldadd_u64(void *addr, int64_t off) +{ + RTE_SET_USED(addr); + RTE_SET_USED(off); + return 0; +} + +static inline void +octeontx_reg_lmtst(void *lmtline_va, void *ioreg_va, const uint64_t cmdbuf[], + uint64_t cmdsize) +{ + RTE_SET_USED(lmtline_va); + RTE_SET_USED(ioreg_va); + RTE_SET_USED(cmdbuf); + RTE_SET_USED(cmdsize); +} + +#endif +#endif /* __OCTEONTX_IO_H__ */ diff --git a/dpdk/drivers/net/octeontx/base/octeontx_pki_var.h b/dpdk/drivers/net/octeontx/base/octeontx_pki_var.h new file mode 100644 index 00000000..def6cbb9 --- /dev/null +++ b/dpdk/drivers/net/octeontx/base/octeontx_pki_var.h @@ -0,0 +1,237 @@ +/* + * BSD LICENSE + * + * Copyright (C) Cavium Inc. 2017. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium networks nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __OCTEONTX_PKI_VAR_H__ +#define __OCTEONTX_PKI_VAR_H__ + +#include + +#define OCTTX_PACKET_WQE_SKIP 128 +#define OCTTX_PACKET_FIRST_SKIP 240 +#define OCTTX_PACKET_LATER_SKIP 128 + +/* WQE descriptor */ +typedef union octtx_wqe_s { + uint64_t w[6]; + + struct { +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + struct { + uint64_t pknd : 6; + uint64_t rsvd0 : 10; + uint64_t style : 8; + uint64_t bufs : 8; + uint64_t chan : 12; + uint64_t apad : 3; + uint64_t rsvd1 : 1; + uint64_t aura : 12; + uint64_t rsvd2 : 4; + } w0; + + struct { + uint64_t tag : 32; + uint64_t tt : 2; + uint64_t grp : 10; + uint64_t rsvd0 : 2; + uint64_t rsvd1 : 2; + uint64_t len : 16; + } w1; + + struct { + uint64_t op_code : 8; + uint64_t err_lev : 3; + uint64_t raw : 1; + uint64_t l2m : 1; + uint64_t l2b : 1; + uint64_t l3m : 1; + uint64_t l3b : 1; + uint64_t l3fr : 1; + uint64_t pf1 : 1; + uint64_t pf2 : 1; + uint64_t pf3 : 1; + uint64_t pf4 : 1; + uint64_t sh : 1; + uint64_t vs : 1; + uint64_t vv : 1; + uint64_t rsvd0 : 8; + uint64_t lae : 1; + uint64_t lbty : 5; + uint64_t lcty : 5; + uint64_t ldty : 5; + uint64_t lety : 5; + uint64_t lfty : 5; + uint64_t lgty : 5; + uint64_t sw : 1; + } w2; + + struct { + uint64_t addr; /* Byte addr of start-of-pkt */ + } w3; + + struct { + uint64_t laptr : 8; + uint64_t lbptr : 8; + uint64_t lcptr : 8; + uint64_t ldprt : 8; + uint64_t leptr : 8; + uint64_t lfptr : 8; + uint64_t lgptr : 8; + uint64_t vlptr : 8; + } w4; + + struct { + uint64_t rsvd0 : 47; + uint64_t dwd : 1; + uint64_t size : 16; + } w5; +#else + struct { + uint64_t rsvd2 : 4; + uint64_t aura : 12; + uint64_t rsvd1 : 1; + uint64_t apad : 3; + uint64_t chan : 12; + uint64_t bufs : 8; + uint64_t style : 8; + uint64_t rsvd0 : 10; + uint64_t pknd : 6; + } w0; + + struct { + uint64_t len : 16; + uint64_t rsvd1 : 2; + uint64_t rsvd0 : 2; + uint64_t grp : 10; + uint64_t tt : 2; + uint64_t tag : 32; + } w1; + + struct { + uint64_t sw : 1; + uint64_t lgty : 5; + uint64_t lfty : 5; + uint64_t lety : 5; + uint64_t ldty : 5; + uint64_t lcty : 5; + uint64_t lbty : 5; + uint64_t lae : 1; + uint64_t rsvd0 : 8; + uint64_t vv : 1; + uint64_t vs : 1; + uint64_t sh : 1; + uint64_t pf4 : 1; + uint64_t pf3 : 1; + uint64_t pf2 : 1; + uint64_t pf1 : 1; + uint64_t l3fr : 1; + uint64_t l3b : 1; + uint64_t l3m : 1; + uint64_t l2b : 1; + uint64_t l2m : 1; + uint64_t raw : 1; + uint64_t err_lev : 3; + uint64_t op_code : 8; + } w2; + + struct { + uint64_t addr; /* Byte addr of start-of-pkt */ + } w3; + + struct { + uint64_t vlptr : 8; + uint64_t lgptr : 8; + uint64_t lfptr : 8; + uint64_t leptr : 8; + uint64_t ldprt : 8; + uint64_t lcptr : 8; + uint64_t lbptr : 8; + uint64_t laptr : 8; + } w4; +#endif + } s; + +} __rte_packed octtx_wqe_t; + +enum occtx_pki_ltype_e { + OCCTX_PKI_LTYPE_NONE = 0, + OCCTX_PKI_LTYPE_ENET = 1, + OCCTX_PKI_LTYPE_VLAN = 2, + OCCTX_PKI_LTYPE_SNAP_PAYLD = 5, + OCCTX_PKI_LTYPE_ARP = 6, + OCCTX_PKI_LTYPE_RARP = 7, + OCCTX_PKI_LTYPE_IP4 = 8, + OCCTX_PKI_LTYPE_IP4_OPT = 9, + OCCTX_PKI_LTYPE_IP6 = 0xa, + OCCTX_PKI_LTYPE_IP6_OPT = 0xb, + OCCTX_PKI_LTYPE_IPSEC_ESP = 0xc, + OCCTX_PKI_LTYPE_IPFRAG = 0xd, + OCCTX_PKI_LTYPE_IPCOMP = 0xe, + OCCTX_PKI_LTYPE_TCP = 0x10, + OCCTX_PKI_LTYPE_UDP = 0x11, + OCCTX_PKI_LTYPE_SCTP = 0x12, + OCCTX_PKI_LTYPE_UDP_VXLAN = 0x13, + OCCTX_PKI_LTYPE_GRE = 0x14, + OCCTX_PKI_LTYPE_NVGRE = 0x15, + OCCTX_PKI_LTYPE_GTP = 0x16, + OCCTX_PKI_LTYPE_UDP_GENEVE = 0x17, + OCCTX_PKI_LTYPE_SW28 = 0x1c, + OCCTX_PKI_LTYPE_SW29 = 0x1d, + OCCTX_PKI_LTYPE_SW30 = 0x1e, + OCCTX_PKI_LTYPE_SW31 = 0x1f, + OCCTX_PKI_LTYPE_LAST +}; + +enum lc_type_e { + LC_NONE = OCCTX_PKI_LTYPE_NONE, + LC_IPV4 = OCCTX_PKI_LTYPE_IP4, + LC_IPV4_OPT = OCCTX_PKI_LTYPE_IP4_OPT, + LC_IPV6 = OCCTX_PKI_LTYPE_IP6, + LC_IPV6_OPT = OCCTX_PKI_LTYPE_IP6_OPT, +}; + +enum le_type_e { + LE_NONE = OCCTX_PKI_LTYPE_NONE, +}; + +enum lf_type_e { + LF_NONE = OCCTX_PKI_LTYPE_NONE, + LF_IPSEC_ESP = OCCTX_PKI_LTYPE_IPSEC_ESP, + LF_IPFRAG = OCCTX_PKI_LTYPE_IPFRAG, + LF_IPCOMP = OCCTX_PKI_LTYPE_IPCOMP, + LF_TCP = OCCTX_PKI_LTYPE_TCP, + LF_UDP = OCCTX_PKI_LTYPE_UDP, + LF_GRE = OCCTX_PKI_LTYPE_GRE, + LF_UDP_GENEVE = OCCTX_PKI_LTYPE_UDP_GENEVE, + LF_UDP_VXLAN = OCCTX_PKI_LTYPE_UDP_VXLAN, + LF_NVGRE = OCCTX_PKI_LTYPE_NVGRE, +}; +#endif /* __OCTEONTX_PKI_VAR_H__ */ diff --git a/dpdk/drivers/net/octeontx/base/octeontx_pkivf.c b/dpdk/drivers/net/octeontx/base/octeontx_pkivf.c new file mode 100644 index 00000000..b97f05cd --- /dev/null +++ b/dpdk/drivers/net/octeontx/base/octeontx_pkivf.c @@ -0,0 +1,169 @@ +/* + * BSD LICENSE + * + * Copyright (C) Cavium Inc. 2017. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium networks nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include + +#include +#include + +#include "octeontx_pkivf.h" + +int +octeontx_pki_port_open(int port) +{ + struct octeontx_mbox_hdr hdr; + int res; + + hdr.coproc = OCTEONTX_PKI_COPROC; + hdr.msg = MBOX_PKI_PORT_OPEN; + hdr.vfid = port; + + res = octeontx_ssovf_mbox_send(&hdr, NULL, 0, NULL, 0); + if (res < 0) + return -EACCES; + return res; +} + +int +octeontx_pki_port_hash_config(int port, pki_hash_cfg_t *hash_cfg) +{ + struct octeontx_mbox_hdr hdr; + int res; + + mbox_pki_hash_cfg_t h_cfg = *(mbox_pki_hash_cfg_t *)hash_cfg; + int len = sizeof(mbox_pki_hash_cfg_t); + + hdr.coproc = OCTEONTX_PKI_COPROC; + hdr.msg = MBOX_PKI_PORT_HASH_CONFIG; + hdr.vfid = port; + + res = octeontx_ssovf_mbox_send(&hdr, &h_cfg, len, NULL, 0); + if (res < 0) + return -EACCES; + + return res; +} + +int +octeontx_pki_port_pktbuf_config(int port, pki_pktbuf_cfg_t *buf_cfg) +{ + struct octeontx_mbox_hdr hdr; + int res; + + mbox_pki_pktbuf_cfg_t b_cfg = *(mbox_pki_pktbuf_cfg_t *)buf_cfg; + int len = sizeof(mbox_pki_pktbuf_cfg_t); + + hdr.coproc = OCTEONTX_PKI_COPROC; + hdr.msg = MBOX_PKI_PORT_PKTBUF_CONFIG; + hdr.vfid = port; + + res = octeontx_ssovf_mbox_send(&hdr, &b_cfg, len, NULL, 0); + if (res < 0) + return -EACCES; + return res; +} + +int +octeontx_pki_port_create_qos(int port, pki_qos_cfg_t *qos_cfg) +{ + struct octeontx_mbox_hdr hdr; + int res; + + mbox_pki_qos_cfg_t q_cfg = *(mbox_pki_qos_cfg_t *)qos_cfg; + int len = sizeof(mbox_pki_qos_cfg_t); + + hdr.coproc = OCTEONTX_PKI_COPROC; + hdr.msg = MBOX_PKI_PORT_CREATE_QOS; + hdr.vfid = port; + + res = octeontx_ssovf_mbox_send(&hdr, &q_cfg, len, NULL, 0); + if (res < 0) + return -EACCES; + + return res; +} + + +int +octeontx_pki_port_errchk_config(int port, pki_errchk_cfg_t *cfg) +{ + struct octeontx_mbox_hdr hdr; + int res; + + mbox_pki_errcheck_cfg_t e_cfg; + e_cfg = *((mbox_pki_errcheck_cfg_t *)(cfg)); + int len = sizeof(mbox_pki_errcheck_cfg_t); + + hdr.coproc = OCTEONTX_PKI_COPROC; + hdr.msg = MBOX_PKI_PORT_ERRCHK_CONFIG; + hdr.vfid = port; + + res = octeontx_ssovf_mbox_send(&hdr, &e_cfg, len, NULL, 0); + if (res < 0) + return -EACCES; + + return res; +} + +#define PCI_VENDOR_ID_CAVIUM 0x177D +#define PCI_DEVICE_ID_OCTEONTX_PKI_VF 0xA0DD + +/* PKIVF pcie device */ +static int +pkivf_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) +{ + RTE_SET_USED(pci_drv); + RTE_SET_USED(pci_dev); + + /* For secondary processes, the primary has done all the work */ + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return 0; + + return 0; +} + +static const struct rte_pci_id pci_pkivf_map[] = { + { + RTE_PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, + PCI_DEVICE_ID_OCTEONTX_PKI_VF) + }, + { + .vendor_id = 0, + }, +}; + +static struct rte_pci_driver pci_pkivf = { + .id_table = pci_pkivf_map, + .drv_flags = RTE_PCI_DRV_NEED_MAPPING, + .probe = pkivf_probe, +}; + +RTE_PMD_REGISTER_PCI(octeontx_pkivf, pci_pkivf); diff --git a/dpdk/drivers/net/octeontx/base/octeontx_pkivf.h b/dpdk/drivers/net/octeontx/base/octeontx_pkivf.h new file mode 100644 index 00000000..004dedcc --- /dev/null +++ b/dpdk/drivers/net/octeontx/base/octeontx_pkivf.h @@ -0,0 +1,553 @@ +/* + * BSD LICENSE + * + * Copyright (C) Cavium Inc. 2017. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium networks nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __OCTEONTX_PKI_H__ +#define __OCTEONTX_PKI_H__ + +#include + +#include + +#define OCTEONTX_PKI_COPROC 5 + +/* PKI messages */ + +#define MBOX_PKI_PORT_OPEN 1 +#define MBOX_PKI_PORT_START 2 +#define MBOX_PKI_PORT_STOP 3 +#define MBOX_PKI_PORT_CLOSE 4 +#define MBOX_PKI_PORT_CONFIG 5 +#define MBOX_PKI_PORT_OPT_PARSER_CONFIG 6 +#define MBOX_PKI_PORT_CUSTOM_PARSER_CONFIG 7 +#define MBOX_PKI_PORT_PKTBUF_CONFIG 8 +#define MBOX_PKI_PORT_HASH_CONFIG 9 +#define MBOX_PKI_PORT_ERRCHK_CONFIG 10 +#define MBOX_PKI_PORT_CREATE_QOS 11 +#define MBOX_PKI_PORT_MODIFY_QOS 12 +#define MBOX_PKI_PORT_DELETE_QOS 13 +#define MBOX_PKI_PORT_PKTDROP_CONFIG 14 +#define MBOX_PKI_PORT_WQE_GEN_CONFIG 15 +#define MBOX_PKI_BACKPRESSURE_CONFIG 16 +#define MBOX_PKI_PORT_GET_STATS 17 +#define MBOX_PKI_PORT_RESET_STATS 18 +#define MBOX_PKI_GET_PORT_CONFIG 19 +#define MBOX_PKI_GET_PORT_QOS_CONFIG 20 + +#define MBOX_PKI_MAX_QOS_ENTRY 64 + +/* pki pkind parse mode */ +enum { + MBOX_PKI_PARSE_LA_TO_LG = 0, + MBOX_PKI_PARSE_LB_TO_LG = 1, + MBOX_PKI_PARSE_LC_TO_LG = 3, + MBOX_PKI_PARSE_LG = 0x3f, + MBOX_PKI_PARSE_NOTHING = 0x7f +}; + +/* Interface types: */ +enum { + OCTTX_PORT_TYPE_NET, /* Network interface ports */ + OCTTX_PORT_TYPE_INT, /* CPU internal interface ports */ + OCTTX_PORT_TYPE_PCI, /* DPI/PCIe interface ports */ + OCTTX_PORT_TYPE_MAX +}; + +/* pki port config */ +typedef struct mbox_pki_port_type { + uint8_t port_type; +} mbox_pki_port_t; + +/* pki port config */ +typedef struct mbox_pki_port_cfg { + uint8_t port_type; + struct { + uint8_t fcs_pres:1; + uint8_t fcs_skip:1; + uint8_t parse_mode:1; + uint8_t mpls_parse:1; + uint8_t inst_hdr_parse:1; + uint8_t fulc_parse:1; + uint8_t dsa_parse:1; + uint8_t hg2_parse:1; + uint8_t hg_parse:1; + } mmask; + uint8_t fcs_pres; + uint8_t fcs_skip; + uint8_t parse_mode; + uint8_t mpls_parse; + uint8_t inst_hdr_parse; + uint8_t fulc_parse; + uint8_t dsa_parse; + uint8_t hg2_parse; + uint8_t hg_parse; +} mbox_pki_prt_cfg_t; + +/* pki Flow/style packet buffer config */ +typedef struct mbox_pki_port_pktbuf_cfg { + uint8_t port_type; + struct { + uint16_t f_mbuff_size:1; + uint16_t f_wqe_skip:1; + uint16_t f_first_skip:1; + uint16_t f_later_skip:1; + uint16_t f_pkt_outside_wqe:1; + uint16_t f_wqe_endian:1; + uint16_t f_cache_mode:1; + } mmask; + uint16_t mbuff_size; + uint16_t wqe_skip; + uint16_t first_skip; + uint16_t later_skip; + uint8_t pkt_outside_wqe; + uint8_t wqe_endian; + uint8_t cache_mode; +} mbox_pki_pktbuf_cfg_t; + +/* pki flow/style tag config */ +typedef struct mbox_pki_port_hash_cfg { + uint8_t port_type; + uint32_t tag_slf:1; + uint32_t tag_sle:1; + uint32_t tag_sld:1; + uint32_t tag_slc:1; + uint32_t tag_dlf:1; + uint32_t tag_dle:1; + uint32_t tag_dld:1; + uint32_t tag_dlc:1; + uint32_t tag_prt:1; + uint32_t tag_vlan0:1; + uint32_t tag_vlan1:1; + uint32_t tag_ip_pctl:1; + uint32_t tag_sync:1; + uint32_t tag_spi:1; + uint32_t tag_gtp:1; + uint32_t tag_vni:1; +} mbox_pki_hash_cfg_t; + +/* pki flow/style errcheck config */ +typedef struct mbox_pki_port_errcheck_cfg { + uint8_t port_type; + struct { + uint32_t f_ip6_udp_opt:1; + uint32_t f_lenerr_en:1; + uint32_t f_maxerr_en:1; + uint32_t f_minerr_en:1; + uint32_t f_fcs_chk:1; + uint32_t f_fcs_strip:1; + uint32_t f_len_lf:1; + uint32_t f_len_le:1; + uint32_t f_len_ld:1; + uint32_t f_len_lc:1; + uint32_t f_csum_lf:1; + uint32_t f_csum_le:1; + uint32_t f_csum_ld:1; + uint32_t f_csum_lc:1; + uint32_t f_min_frame_len; + uint32_t f_max_frame_len; + } mmask; + uint64_t ip6_udp_opt:1; + uint64_t lenerr_en:1; + uint64_t maxerr_en:1; + uint64_t minerr_en:1; + uint64_t fcs_chk:1; + uint64_t fcs_strip:1; + uint64_t len_lf:1; + uint64_t len_le:1; + uint64_t len_ld:1; + uint64_t len_lc:1; + uint64_t csum_lf:1; + uint64_t csum_le:1; + uint64_t csum_ld:1; + uint64_t csum_lc:1; + uint64_t min_frame_len; + uint64_t max_frame_len; +} mbox_pki_errcheck_cfg_t; + +/* CACHE MODE*/ +enum { + MBOX_PKI_OPC_MODE_STT = 0LL, + MBOX_PKI_OPC_MODE_STF = 1LL, + MBOX_PKI_OPC_MODE_STF1_STT = 2LL, + MBOX_PKI_OPC_MODE_STF2_STT = 3LL +}; + +/* PKI QPG QOS*/ +enum { + MBOX_PKI_QPG_QOS_NONE = 0, + MBOX_PKI_QPG_QOS_VLAN, + MBOX_PKI_QPG_QOS_MPLS, + MBOX_PKI_QPG_QOS_DSA_SRC, + MBOX_PKI_QPG_QOS_DIFFSERV, + MBOX_PKI_QPG_QOS_HIGIG, +}; + +struct mbox_pki_qos_entry { + uint16_t port_add; + uint16_t ggrp_ok; + uint16_t ggrp_bad; + uint16_t gaura; + uint8_t grptag_ok; + uint8_t grptag_bad; +}; + +/* pki flow/style enable qos */ +typedef struct mbox_pki_port_create_qos { + uint8_t port_type; + uint8_t qpg_qos; + uint8_t num_entry; + uint8_t tag_type; + uint8_t drop_policy; + struct mbox_pki_qos_entry qos_entry[MBOX_PKI_MAX_QOS_ENTRY]; +} mbox_pki_qos_cfg_t; + +/* pki flow/style enable qos */ +typedef struct mbox_pki_port_modify_qos_entry { + uint8_t port_type; + uint16_t index; + struct { + uint8_t f_port_add:1; + uint8_t f_grp_ok:1; + uint8_t f_grp_bad:1; + uint8_t f_gaura:1; + uint8_t f_grptag_ok:1; + uint8_t f_grptag_bad:1; + uint8_t f_tag_type:1; + } mmask; + uint8_t tag_type; + struct mbox_pki_qos_entry qos_entry; +} mbox_pki_mod_qos_t; + +/* pki flow/style enable qos */ +typedef struct mbox_pki_port_delete_qos_entry { + uint8_t port_type; + uint16_t index; +} mbox_pki_del_qos_t; + +/* PKI maximum constants */ +#define PKI_VF_MAX (1) +#define PKI_MAX_PKTLEN (32768) + +/* pki pkind parse mode */ +enum { + PKI_PARSE_LA_TO_LG = 0, + PKI_PARSE_LB_TO_LG = 1, + PKI_PARSE_LC_TO_LG = 3, + PKI_PARSE_LG = 0x3f, + PKI_PARSE_NOTHING = 0x7f +}; + +/* pki port config */ +typedef struct pki_port_cfg { + uint8_t port_type; + struct { + uint8_t fcs_pres:1; + uint8_t fcs_skip:1; + uint8_t parse_mode:1; + uint8_t mpls_parse:1; + uint8_t inst_hdr_parse:1; + uint8_t fulc_parse:1; + uint8_t dsa_parse:1; + uint8_t hg2_parse:1; + uint8_t hg_parse:1; + } mmask; + uint8_t fcs_pres; + uint8_t fcs_skip; + uint8_t parse_mode; + uint8_t mpls_parse; + uint8_t inst_hdr_parse; + uint8_t fulc_parse; + uint8_t dsa_parse; + uint8_t hg2_parse; + uint8_t hg_parse; +} pki_prt_cfg_t; + + +/* pki Flow/style packet buffer config */ +typedef struct pki_port_pktbuf_cfg { + uint8_t port_type; + struct { + uint16_t f_mbuff_size:1; + uint16_t f_wqe_skip:1; + uint16_t f_first_skip:1; + uint16_t f_later_skip:1; + uint16_t f_pkt_outside_wqe:1; + uint16_t f_wqe_endian:1; + uint16_t f_cache_mode:1; + } mmask; + uint16_t mbuff_size; + uint16_t wqe_skip; + uint16_t first_skip; + uint16_t later_skip; + uint8_t pkt_outside_wqe; + uint8_t wqe_endian; + uint8_t cache_mode; +} pki_pktbuf_cfg_t; + +/* pki flow/style tag config */ +typedef struct pki_port_hash_cfg { + uint8_t port_type; + uint32_t tag_slf:1; + uint32_t tag_sle:1; + uint32_t tag_sld:1; + uint32_t tag_slc:1; + uint32_t tag_dlf:1; + uint32_t tag_dle:1; + uint32_t tag_dld:1; + uint32_t tag_dlc:1; + uint32_t tag_prt:1; + uint32_t tag_vlan0:1; + uint32_t tag_vlan1:1; + uint32_t tag_ip_pctl:1; + uint32_t tag_sync:1; + uint32_t tag_spi:1; + uint32_t tag_gtp:1; + uint32_t tag_vni:1; +} pki_hash_cfg_t; + +/* pki flow/style errcheck config */ +typedef struct pki_port_errcheck_cfg { + uint8_t port_type; + struct { + uint32_t f_ip6_udp_opt:1; + uint32_t f_lenerr_en:1; + uint32_t f_maxerr_en:1; + uint32_t f_minerr_en:1; + uint32_t f_fcs_chk:1; + uint32_t f_fcs_strip:1; + uint32_t f_len_lf:1; + uint32_t f_len_le:1; + uint32_t f_len_ld:1; + uint32_t f_len_lc:1; + uint32_t f_csum_lf:1; + uint32_t f_csum_le:1; + uint32_t f_csum_ld:1; + uint32_t f_csum_lc:1; + uint32_t f_min_frame_len; + uint32_t f_max_frame_len; + } mmask; + uint64_t ip6_udp_opt:1; + uint64_t lenerr_en:1; + uint64_t maxerr_en:1; + uint64_t minerr_en:1; + uint64_t fcs_chk:1; + uint64_t fcs_strip:1; + uint64_t len_lf:1; + uint64_t len_le:1; + uint64_t len_ld:1; + uint64_t len_lc:1; + uint64_t csum_lf:1; + uint64_t csum_le:1; + uint64_t csum_ld:1; + uint64_t csum_lc:1; + uint64_t min_frame_len; + uint64_t max_frame_len; +} pki_errchk_cfg_t; + + +/* CACHE MODE*/ +enum { + PKI_OPC_MODE_STT = 0LL, + PKI_OPC_MODE_STF = 1LL, + PKI_OPC_MODE_STF1_STT = 2LL, + PKI_OPC_MODE_STF2_STT = 3LL +}; + +/* PKI QPG QOS*/ +enum { + PKI_QPG_QOS_NONE = 0, + PKI_QPG_QOS_VLAN, + PKI_QPG_QOS_MPLS, + PKI_QPG_QOS_DSA_SRC, + PKI_QPG_QOS_DIFFSERV, + PKI_QPG_QOS_HIGIG, +}; + +struct pki_qos_entry { + uint16_t port_add; + uint16_t ggrp_ok; + uint16_t ggrp_bad; + uint16_t gaura; + uint8_t grptag_ok; + uint8_t grptag_bad; + uint8_t ena_red; + uint8_t ena_drop; +}; + +#define PKO_MAX_QOS_ENTRY 64 + +/* pki flow/style enable qos */ +typedef struct pki_port_create_qos { + uint8_t port_type; + uint8_t qpg_qos; + uint8_t num_entry; + uint8_t tag_type; + uint8_t drop_policy; + struct pki_qos_entry qos_entry[PKO_MAX_QOS_ENTRY]; +} pki_qos_cfg_t; + +/* pki flow/style enable qos */ +typedef struct pki_port_delete_qos_entry { + uint8_t port_type; + uint16_t index; +} pki_del_qos_t; + +/* pki flow/style enable qos */ +typedef struct pki_port_modify_qos_entry { + uint8_t port_type; + uint16_t index; + struct { + uint8_t f_port_add:1; + uint8_t f_grp_ok:1; + uint8_t f_grp_bad:1; + uint8_t f_gaura:1; + uint8_t f_grptag_ok:1; + uint8_t f_grptag_bad:1; + uint8_t f_tag_type:1; + } mmask; + uint8_t tag_type; + struct pki_qos_entry qos_entry; +} pki_mod_qos_t; + +static inline int +octeontx_pki_port_modify_qos(int port, pki_mod_qos_t *qos_cfg) +{ + struct octeontx_mbox_hdr hdr; + int res; + + mbox_pki_mod_qos_t q_cfg = *(mbox_pki_mod_qos_t *)qos_cfg; + int len = sizeof(mbox_pki_mod_qos_t); + + hdr.coproc = OCTEONTX_PKI_COPROC; + hdr.msg = MBOX_PKI_PORT_MODIFY_QOS; + hdr.vfid = port; + + res = octeontx_ssovf_mbox_send(&hdr, &q_cfg, len, NULL, 0); + if (res < 0) + return -EACCES; + + return res; +} + +static inline int +octeontx_pki_port_delete_qos(int port, pki_del_qos_t *qos_cfg) +{ + struct octeontx_mbox_hdr hdr; + int res; + + mbox_pki_del_qos_t q_cfg = *(mbox_pki_del_qos_t *)qos_cfg; + int len = sizeof(mbox_pki_del_qos_t); + + hdr.coproc = OCTEONTX_PKI_COPROC; + hdr.msg = MBOX_PKI_PORT_DELETE_QOS; + hdr.vfid = port; + + res = octeontx_ssovf_mbox_send(&hdr, &q_cfg, len, NULL, 0); + if (res < 0) + return -EACCES; + + return res; +} + +static inline int +octeontx_pki_port_close(int port) +{ + struct octeontx_mbox_hdr hdr; + int res; + + mbox_pki_port_t ptype; + int len = sizeof(mbox_pki_port_t); + memset(&ptype, 0, len); + ptype.port_type = OCTTX_PORT_TYPE_NET; + + hdr.coproc = OCTEONTX_PKI_COPROC; + hdr.msg = MBOX_PKI_PORT_CLOSE; + hdr.vfid = port; + + res = octeontx_ssovf_mbox_send(&hdr, &ptype, len, NULL, 0); + if (res < 0) + return -EACCES; + + return res; +} + +static inline int +octeontx_pki_port_start(int port) +{ + struct octeontx_mbox_hdr hdr; + int res; + + mbox_pki_port_t ptype; + int len = sizeof(mbox_pki_port_t); + memset(&ptype, 0, len); + ptype.port_type = OCTTX_PORT_TYPE_NET; + + hdr.coproc = OCTEONTX_PKI_COPROC; + hdr.msg = MBOX_PKI_PORT_START; + hdr.vfid = port; + + res = octeontx_ssovf_mbox_send(&hdr, &ptype, len, NULL, 0); + if (res < 0) + return -EACCES; + + return res; +} + +static inline int +octeontx_pki_port_stop(int port) +{ + struct octeontx_mbox_hdr hdr; + int res; + + mbox_pki_port_t ptype; + int len = sizeof(mbox_pki_port_t); + memset(&ptype, 0, len); + ptype.port_type = OCTTX_PORT_TYPE_NET; + + hdr.coproc = OCTEONTX_PKI_COPROC; + hdr.msg = MBOX_PKI_PORT_STOP; + hdr.vfid = port; + + res = octeontx_ssovf_mbox_send(&hdr, &ptype, len, NULL, 0); + if (res < 0) + return -EACCES; + + return res; +} + +int octeontx_pki_port_open(int port); +int octeontx_pki_port_hash_config(int port, pki_hash_cfg_t *hash_cfg); +int octeontx_pki_port_pktbuf_config(int port, pki_pktbuf_cfg_t *buf_cfg); +int octeontx_pki_port_create_qos(int port, pki_qos_cfg_t *qos_cfg); +int octeontx_pki_port_close(int port); +int octeontx_pki_port_errchk_config(int port, pki_errchk_cfg_t *cfg); + +#endif /* __OCTEONTX_PKI_H__ */ diff --git a/dpdk/drivers/net/octeontx/base/octeontx_pkovf.c b/dpdk/drivers/net/octeontx/base/octeontx_pkovf.c new file mode 100644 index 00000000..f01d948e --- /dev/null +++ b/dpdk/drivers/net/octeontx/base/octeontx_pkovf.c @@ -0,0 +1,617 @@ +/* + * BSD LICENSE + * + * Copyright (C) Cavium Inc. 2017. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium networks nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "../octeontx_logs.h" +#include "octeontx_io.h" +#include "octeontx_pkovf.h" + +struct octeontx_pko_iomem { + uint8_t *va; + rte_iova_t iova; + size_t size; +}; + +#define PKO_IOMEM_NULL (struct octeontx_pko_iomem){0, 0, 0} + +struct octeontx_pko_fc_ctl_s { + int64_t buf_cnt; + int64_t padding[(PKO_DQ_FC_STRIDE / 8) - 1]; +}; + +struct octeontx_pkovf { + uint8_t *bar0; + uint8_t *bar2; + uint16_t domain; + uint16_t vfid; +}; + +struct octeontx_pko_vf_ctl_s { + rte_spinlock_t lock; + + struct octeontx_pko_iomem fc_iomem; + struct octeontx_pko_fc_ctl_s *fc_ctl; + struct octeontx_pkovf pko[PKO_VF_MAX]; + struct { + uint64_t chanid; + } dq_map[PKO_VF_MAX * PKO_VF_NUM_DQ]; +}; + +static struct octeontx_pko_vf_ctl_s pko_vf_ctl; + +static void * +octeontx_pko_dq_vf_bar0(uint16_t txq) +{ + int vf_ix; + + vf_ix = txq / PKO_VF_NUM_DQ; + return pko_vf_ctl.pko[vf_ix].bar0; +} + +static int +octeontx_pko_dq_gdq(uint16_t txq) +{ + return txq % PKO_VF_NUM_DQ; +} + +/** + * Open a PKO DQ. + */ +static inline +int octeontx_pko_dq_open(uint16_t txq) +{ + unsigned int reg_off; + uint8_t *vf_bar0; + uint64_t rtn; + int gdq; + + vf_bar0 = octeontx_pko_dq_vf_bar0(txq); + gdq = octeontx_pko_dq_gdq(txq); + + if (unlikely(gdq < 0 || vf_bar0 == NULL)) + return -EINVAL; + *(volatile int64_t*)(pko_vf_ctl.fc_ctl + txq) = + PKO_DQ_FC_DEPTH_PAGES - PKO_DQ_FC_SKID; + + rte_wmb(); + + octeontx_write64(PKO_DQ_FC_DEPTH_PAGES, + vf_bar0 + PKO_VF_DQ_FC_STATUS(gdq)); + + /* Set the register to return descriptor (packet) count as DEPTH */ + /* KIND=1, NCB_QUERY_RSP=0 */ + octeontx_write64(1ull << PKO_DQ_KIND_BIT, + vf_bar0 + PKO_VF_DQ_WM_CTL(gdq)); + reg_off = PKO_VF_DQ_OP_OPEN(gdq); + + rtn = octeontx_reg_ldadd_u64(vf_bar0 + reg_off, 0); + + /* PKO_DQOP_E::OPEN */ + if (((rtn >> PKO_DQ_OP_BIT) & 0x3) != 0x1) + return -EIO; + + switch (rtn >> PKO_DQ_STATUS_BIT) { + case 0xC: /* DQALREADYCREATED */ + case 0x0: /* PASS */ + break; + default: + return -EIO; + } + + /* DRAIN=0, DRAIN_NULL_LINK=0, SW_XOFF=0 */ + octeontx_write64(0, vf_bar0 + PKO_VF_DQ_SW_XOFF(gdq)); + + return rtn & ((1ull << PKO_DQ_OP_BIT) - 1); +} + +/** + * Close a PKO DQ + * Flush all packets pending. + */ +static inline +int octeontx_pko_dq_close(uint16_t txq) +{ + unsigned int reg_off; + uint8_t *vf_bar0; + uint64_t rtn; + int res; + + vf_bar0 = octeontx_pko_dq_vf_bar0(txq); + res = octeontx_pko_dq_gdq(txq); + + if (unlikely(res < 0 || vf_bar0 == NULL)) + return -EINVAL; + + reg_off = PKO_VF_DQ_OP_CLOSE(res); + + rtn = octeontx_reg_ldadd_u64(vf_bar0 + reg_off, 0); + + /* PKO_DQOP_E::CLOSE */ + if (((rtn >> PKO_DQ_OP_BIT) & 0x3) != 0x2) + return -EIO; + + switch (rtn >> PKO_DQ_STATUS_BIT) { + case 0xD: /* DQNOTCREATED */ + case 0x0: /* PASS */ + break; + default: + return -EIO; + } + + res = rtn & ((1ull << PKO_DQ_OP_BIT) - 1); /* DEPTH */ + return res; +} + +/* Flush all packets pending on a DQ */ +static inline +int octeontx_pko_dq_drain(uint16_t txq) +{ + unsigned int gdq; + uint8_t *vf_bar0; + uint64_t reg; + int res, timo = PKO_DQ_DRAIN_TO; + + vf_bar0 = octeontx_pko_dq_vf_bar0(txq); + res = octeontx_pko_dq_gdq(txq); + gdq = res; + + /* DRAIN=1, DRAIN_NULL_LINK=0, SW_XOFF=1 */ + octeontx_write64(0x3, vf_bar0 + PKO_VF_DQ_SW_XOFF(gdq)); + /* Wait until buffers leave DQs */ + reg = octeontx_read64(vf_bar0 + PKO_VF_DQ_WM_CNT(gdq)); + while (reg && timo > 0) { + rte_delay_us(100); + timo--; + reg = octeontx_read64(vf_bar0 + PKO_VF_DQ_WM_CNT(gdq)); + } + /* DRAIN=0, DRAIN_NULL_LINK=0, SW_XOFF=0 */ + octeontx_write64(0, vf_bar0 + PKO_VF_DQ_SW_XOFF(gdq)); + + return reg; +} + +static inline int +octeontx_pko_dq_range_lookup(struct octeontx_pko_vf_ctl_s *ctl, uint64_t chanid, + unsigned int dq_num, unsigned int dq_from) +{ + unsigned int dq, dq_cnt; + unsigned int dq_base; + + dq_cnt = 0; + dq = dq_from; + while (dq < RTE_DIM(ctl->dq_map)) { + dq_base = dq; + dq_cnt = 0; + while (ctl->dq_map[dq].chanid == ~chanid && + dq < RTE_DIM(ctl->dq_map)) { + dq_cnt++; + if (dq_cnt == dq_num) + return dq_base; + dq++; + } + dq++; + } + return -1; +} + +static inline void +octeontx_pko_dq_range_assign(struct octeontx_pko_vf_ctl_s *ctl, uint64_t chanid, + unsigned int dq_base, unsigned int dq_num) +{ + unsigned int dq, dq_cnt; + + dq_cnt = 0; + while (dq_cnt < dq_num) { + dq = dq_base + dq_cnt; + + octeontx_log_dbg("DQ# %u assigned to CHAN# %" PRIx64 "", dq, + chanid); + + ctl->dq_map[dq].chanid = ~chanid; + dq_cnt++; + } +} + +static inline int +octeontx_pko_dq_claim(struct octeontx_pko_vf_ctl_s *ctl, unsigned int dq_base, + unsigned int dq_num, uint64_t chanid) +{ + const uint64_t null_chanid = ~0ull; + int dq; + + rte_spinlock_lock(&ctl->lock); + + dq = octeontx_pko_dq_range_lookup(ctl, null_chanid, dq_num, dq_base); + if (dq < 0 || (unsigned int)dq != dq_base) { + rte_spinlock_unlock(&ctl->lock); + return -1; + } + octeontx_pko_dq_range_assign(ctl, chanid, dq_base, dq_num); + + rte_spinlock_unlock(&ctl->lock); + + return 0; +} + +static inline int +octeontx_pko_dq_free(struct octeontx_pko_vf_ctl_s *ctl, uint64_t chanid) +{ + const uint64_t null_chanid = ~0ull; + unsigned int dq = 0, dq_cnt = 0; + + rte_spinlock_lock(&ctl->lock); + while (dq < RTE_DIM(ctl->dq_map)) { + if (ctl->dq_map[dq].chanid == ~chanid) { + ctl->dq_map[dq].chanid = ~null_chanid; + dq_cnt++; + } + dq++; + } + rte_spinlock_unlock(&ctl->lock); + + return dq_cnt > 0 ? 0 : -EINVAL; +} + +int +octeontx_pko_channel_open(int dq_base, int dq_num, int chanid) +{ + struct octeontx_pko_vf_ctl_s *ctl = &pko_vf_ctl; + int res; + + res = octeontx_pko_dq_claim(ctl, dq_base, dq_num, chanid); + if (res < 0) + return -1; + + return 0; +} + +int +octeontx_pko_channel_close(int chanid) +{ + struct octeontx_pko_vf_ctl_s *ctl = &pko_vf_ctl; + int res; + + res = octeontx_pko_dq_free(ctl, chanid); + if (res < 0) + return -1; + + return 0; +} + +static inline int +octeontx_pko_chan_start(struct octeontx_pko_vf_ctl_s *ctl, uint64_t chanid) +{ + unsigned int dq_vf; + unsigned int dq, dq_cnt; + + dq_cnt = 0; + dq = 0; + while (dq < RTE_DIM(ctl->dq_map)) { + dq_vf = dq / PKO_VF_NUM_DQ; + + if (!ctl->pko[dq_vf].bar0) { + dq += PKO_VF_NUM_DQ; + continue; + } + + if (ctl->dq_map[dq].chanid != ~chanid) { + dq++; + continue; + } + + if (octeontx_pko_dq_open(dq) < 0) + break; + + dq_cnt++; + dq++; + } + + return dq_cnt; +} + +int +octeontx_pko_channel_start(int chanid) +{ + struct octeontx_pko_vf_ctl_s *ctl = &pko_vf_ctl; + int dq_cnt; + + dq_cnt = octeontx_pko_chan_start(ctl, chanid); + if (dq_cnt < 0) + return -1; + + return dq_cnt; +} + +static inline int +octeontx_pko_chan_stop(struct octeontx_pko_vf_ctl_s *ctl, uint64_t chanid) +{ + unsigned int dq, dq_cnt, dq_vf; + int res; + + dq_cnt = 0; + dq = 0; + while (dq < RTE_DIM(ctl->dq_map)) { + dq_vf = dq / PKO_VF_NUM_DQ; + + if (!ctl->pko[dq_vf].bar0) { + dq += PKO_VF_NUM_DQ; + continue; + } + + if (ctl->dq_map[dq].chanid != ~chanid) { + dq++; + continue; + } + + res = octeontx_pko_dq_drain(dq); + if (res > 0) + octeontx_log_err("draining DQ%d, buffers left: %x", + dq, res); + + res = octeontx_pko_dq_close(dq); + if (res < 0) + octeontx_log_err("closing DQ%d failed\n", dq); + + dq_cnt++; + dq++; + } + return dq_cnt; +} + +int +octeontx_pko_channel_stop(int chanid) +{ + struct octeontx_pko_vf_ctl_s *ctl = &pko_vf_ctl; + + octeontx_pko_chan_stop(ctl, chanid); + return 0; +} + +static inline int +octeontx_pko_channel_query(struct octeontx_pko_vf_ctl_s *ctl, uint64_t chanid, + void *out, size_t out_elem_size, + size_t dq_num, octeontx_pko_dq_getter_t getter) +{ + octeontx_dq_t curr; + unsigned int dq_vf; + unsigned int dq; + + RTE_SET_USED(out_elem_size); + memset(&curr, 0, sizeof(octeontx_dq_t)); + + dq_vf = dq_num / PKO_VF_NUM_DQ; + dq = dq_num % PKO_VF_NUM_DQ; + + if (!ctl->pko[dq_vf].bar0) + return -EINVAL; + + if (ctl->dq_map[dq_num].chanid != ~chanid) + return -EINVAL; + + uint8_t *iter = (uint8_t *)out; + curr.lmtline_va = ctl->pko[dq_vf].bar2; + curr.ioreg_va = (void *)((uintptr_t)ctl->pko[dq_vf].bar0 + + PKO_VF_DQ_OP_SEND((dq), 0)); + curr.fc_status_va = ctl->fc_ctl + dq; + + octeontx_log_dbg("lmtline=%p ioreg_va=%p fc_status_va=%p", + curr.lmtline_va, curr.ioreg_va, + curr.fc_status_va); + + getter(&curr, (void *)iter); + return 0; +} + +int +octeontx_pko_channel_query_dqs(int chanid, void *out, size_t out_elem_size, + size_t dq_num, octeontx_pko_dq_getter_t getter) +{ + struct octeontx_pko_vf_ctl_s *ctl = &pko_vf_ctl; + int dq_cnt; + + dq_cnt = octeontx_pko_channel_query(ctl, chanid, out, out_elem_size, + dq_num, getter); + if (dq_cnt < 0) + return -1; + + return dq_cnt; +} + +int +octeontx_pko_vf_count(void) +{ + int vf_cnt; + + vf_cnt = 0; + while (pko_vf_ctl.pko[vf_cnt].bar0) + vf_cnt++; + + return vf_cnt; +} + +int +octeontx_pko_init_fc(const size_t pko_vf_count) +{ + int dq_ix; + uint64_t reg; + uint8_t *vf_bar0; + size_t vf_idx; + size_t fc_mem_size; + + fc_mem_size = sizeof(struct octeontx_pko_fc_ctl_s) * + pko_vf_count * PKO_VF_NUM_DQ; + + pko_vf_ctl.fc_iomem.va = rte_malloc(NULL, fc_mem_size, 128); + if (unlikely(!pko_vf_ctl.fc_iomem.va)) { + octeontx_log_err("fc_iomem: not enough memory"); + return -ENOMEM; + } + + pko_vf_ctl.fc_iomem.iova = rte_malloc_virt2iova((void *) + pko_vf_ctl.fc_iomem.va); + pko_vf_ctl.fc_iomem.size = fc_mem_size; + + pko_vf_ctl.fc_ctl = + (struct octeontx_pko_fc_ctl_s *)pko_vf_ctl.fc_iomem.va; + + /* Configure Flow-Control feature for all DQs of open VFs */ + for (vf_idx = 0; vf_idx < pko_vf_count; vf_idx++) { + dq_ix = vf_idx * PKO_VF_NUM_DQ; + + vf_bar0 = pko_vf_ctl.pko[vf_idx].bar0; + + reg = (pko_vf_ctl.fc_iomem.iova + + (sizeof(struct octeontx_pko_fc_ctl_s) * dq_ix)) & ~0x7F; + reg |= /* BASE */ + (0x2 << 3) | /* HYST_BITS */ + (((PKO_DQ_FC_STRIDE == PKO_DQ_FC_STRIDE_16) ? 1 : 0) << 2) | + (0x1 << 0); /* ENABLE */ + + octeontx_write64(reg, vf_bar0 + PKO_VF_DQ_FC_CONFIG); + + octeontx_log_dbg("PKO: bar0 %p VF_idx %d DQ_FC_CFG=%" PRIx64 "", + vf_bar0, (int)vf_idx, reg); + } + return 0; +} + +void +octeontx_pko_fc_free(void) +{ + rte_free(pko_vf_ctl.fc_iomem.va); +} + +static void +octeontx_pkovf_setup(void) +{ + static bool init_once; + + if (!init_once) { + unsigned int i; + + rte_spinlock_init(&pko_vf_ctl.lock); + + pko_vf_ctl.fc_iomem = PKO_IOMEM_NULL; + pko_vf_ctl.fc_ctl = NULL; + + for (i = 0; i < PKO_VF_MAX; i++) { + pko_vf_ctl.pko[i].bar0 = NULL; + pko_vf_ctl.pko[i].bar2 = NULL; + pko_vf_ctl.pko[i].domain = ~(uint16_t)0; + pko_vf_ctl.pko[i].vfid = ~(uint16_t)0; + } + + for (i = 0; i < (PKO_VF_MAX * PKO_VF_NUM_DQ); i++) + pko_vf_ctl.dq_map[i].chanid = 0; + + init_once = true; + } +} + +/* PKOVF pcie device*/ +static int +pkovf_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) +{ + uint64_t val; + uint16_t vfid; + uint16_t domain; + uint8_t *bar0; + uint8_t *bar2; + struct octeontx_pkovf *res; + + RTE_SET_USED(pci_drv); + + /* For secondary processes, the primary has done all the work */ + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return 0; + + if (pci_dev->mem_resource[0].addr == NULL || + pci_dev->mem_resource[2].addr == NULL) { + octeontx_log_err("Empty bars %p %p", + pci_dev->mem_resource[0].addr, + pci_dev->mem_resource[2].addr); + return -ENODEV; + } + bar0 = pci_dev->mem_resource[0].addr; + bar2 = pci_dev->mem_resource[2].addr; + + octeontx_pkovf_setup(); + + /* get vfid and domain */ + val = octeontx_read64(bar0 + PKO_VF_DQ_FC_CONFIG); + domain = (val >> 7) & 0xffff; + vfid = (val >> 23) & 0xffff; + + if (unlikely(vfid >= PKO_VF_MAX)) { + octeontx_log_err("pko: Invalid vfid %d", vfid); + return -EINVAL; + } + + res = &pko_vf_ctl.pko[vfid]; + res->vfid = vfid; + res->domain = domain; + res->bar0 = bar0; + res->bar2 = bar2; + + octeontx_log_dbg("Domain=%d group=%d", res->domain, res->vfid); + return 0; +} + +#define PCI_VENDOR_ID_CAVIUM 0x177D +#define PCI_DEVICE_ID_OCTEONTX_PKO_VF 0xA049 + +static const struct rte_pci_id pci_pkovf_map[] = { + { + RTE_PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, + PCI_DEVICE_ID_OCTEONTX_PKO_VF) + }, + { + .vendor_id = 0, + }, +}; + +static struct rte_pci_driver pci_pkovf = { + .id_table = pci_pkovf_map, + .drv_flags = RTE_PCI_DRV_NEED_MAPPING, + .probe = pkovf_probe, +}; + +RTE_PMD_REGISTER_PCI(octeontx_pkovf, pci_pkovf); diff --git a/dpdk/drivers/net/octeontx/base/octeontx_pkovf.h b/dpdk/drivers/net/octeontx/base/octeontx_pkovf.h new file mode 100644 index 00000000..cfc3715d --- /dev/null +++ b/dpdk/drivers/net/octeontx/base/octeontx_pkovf.h @@ -0,0 +1,97 @@ +/* + * BSD LICENSE + * + * Copyright (C) Cavium Inc. 2017. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium networks nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __OCTEONTX_PKO_H__ +#define __OCTEONTX_PKO_H__ + +/* PKO maximum constants */ +#define PKO_VF_MAX (32) +#define PKO_VF_NUM_DQ (8) +#define PKO_MAX_NUM_DQ (8) +#define PKO_DQ_DRAIN_TO (1000) + +#define PKO_DQ_FC_SKID (4) +#define PKO_DQ_FC_DEPTH_PAGES (2048) +#define PKO_DQ_FC_STRIDE_16 (16) +#define PKO_DQ_FC_STRIDE_128 (128) +#define PKO_DQ_FC_STRIDE PKO_DQ_FC_STRIDE_16 + +#define PKO_DQ_KIND_BIT 49 +#define PKO_DQ_STATUS_BIT 60 +#define PKO_DQ_OP_BIT 48 + +/* PKO VF register offsets from VF_BAR0 */ +#define PKO_VF_DQ_SW_XOFF(gdq) (0x000100 | (gdq) << 17) +#define PKO_VF_DQ_WM_CTL(gdq) (0x000130 | (gdq) << 17) +#define PKO_VF_DQ_WM_CNT(gdq) (0x000150 | (gdq) << 17) +#define PKO_VF_DQ_FC_CONFIG (0x000160) +#define PKO_VF_DQ_FC_STATUS(gdq) (0x000168 | (gdq) << 17) +#define PKO_VF_DQ_OP_SEND(gdq, op) (0x001000 | (gdq) << 17 | (op) << 3) +#define PKO_VF_DQ_OP_OPEN(gdq) (0x001100 | (gdq) << 17) +#define PKO_VF_DQ_OP_CLOSE(gdq) (0x001200 | (gdq) << 17) +#define PKO_VF_DQ_OP_QUERY(gdq) (0x001300 | (gdq) << 17) + +/* pko_send_hdr_s + pko_send_link */ +#define PKO_CMD_SZ (2 << 1) +#define PKO_SEND_GATHER_SUBDC (0x0ull << 60) +#define PKO_SEND_GATHER_LDTYPE(x) ((x) << 58) +#define PKO_SEND_GATHER_GAUAR(x) ((x) << 24) + +typedef struct octeontx_dq_s { + void *lmtline_va; + void *ioreg_va; + void *fc_status_va; +} octeontx_dq_t; + +/** + * Function for extracting information out of a given DQ. + * + * It is intended to be used in slow path (configuration) in + * octeontx_pko_channel_query(). + * + * @param dq The DQ to extract information from. + * @param out Pointer to the user's structure he wants to fill. + */ +typedef void (*octeontx_pko_dq_getter_t)(octeontx_dq_t *dq, void *out); + +int +octeontx_pko_channel_query_dqs(int chanid, void *out, size_t out_elem_size, + size_t dq_num, octeontx_pko_dq_getter_t getter); +int octeontx_pko_channel_open(int dq_base, int dq_num, int chanid); +int octeontx_pko_channel_close(int chanid); +int octeontx_pko_channel_start(int chanid); +int octeontx_pko_channel_stop(int chanid); +int octeontx_pko_vf_count(void); +int octeontx_pko_init_fc(const size_t pko_vf_count); +void octeontx_pko_fc_free(void); + +#endif /* __OCTEONTX_PKO_H__ */ diff --git a/dpdk/drivers/net/octeontx/octeontx_ethdev.c b/dpdk/drivers/net/octeontx/octeontx_ethdev.c new file mode 100644 index 00000000..eca3a39f --- /dev/null +++ b/dpdk/drivers/net/octeontx/octeontx_ethdev.c @@ -0,0 +1,1339 @@ +/* + * BSD LICENSE + * + * Copyright (C) Cavium Inc. 2017. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium networks nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "octeontx_ethdev.h" +#include "octeontx_rxtx.h" +#include "octeontx_logs.h" + +struct octeontx_vdev_init_params { + uint8_t nr_port; +}; + +uint16_t +rte_octeontx_pchan_map[OCTEONTX_MAX_BGX_PORTS][OCTEONTX_MAX_LMAC_PER_BGX]; + +enum octeontx_link_speed { + OCTEONTX_LINK_SPEED_SGMII, + OCTEONTX_LINK_SPEED_XAUI, + OCTEONTX_LINK_SPEED_RXAUI, + OCTEONTX_LINK_SPEED_10G_R, + OCTEONTX_LINK_SPEED_40G_R, + OCTEONTX_LINK_SPEED_RESERVE1, + OCTEONTX_LINK_SPEED_QSGMII, + OCTEONTX_LINK_SPEED_RESERVE2 +}; + +/* Parse integer from integer argument */ +static int +parse_integer_arg(const char *key __rte_unused, + const char *value, void *extra_args) +{ + int *i = (int *)extra_args; + + *i = atoi(value); + if (*i < 0) { + octeontx_log_err("argument has to be positive."); + return -1; + } + + return 0; +} + +static int +octeontx_parse_vdev_init_params(struct octeontx_vdev_init_params *params, + struct rte_vdev_device *dev) +{ + struct rte_kvargs *kvlist = NULL; + int ret = 0; + + static const char * const octeontx_vdev_valid_params[] = { + OCTEONTX_VDEV_NR_PORT_ARG, + NULL + }; + + const char *input_args = rte_vdev_device_args(dev); + if (params == NULL) + return -EINVAL; + + + if (input_args) { + kvlist = rte_kvargs_parse(input_args, + octeontx_vdev_valid_params); + if (kvlist == NULL) + return -1; + + ret = rte_kvargs_process(kvlist, + OCTEONTX_VDEV_NR_PORT_ARG, + &parse_integer_arg, + ¶ms->nr_port); + if (ret < 0) + goto free_kvlist; + } + +free_kvlist: + rte_kvargs_free(kvlist); + return ret; +} + +static int +octeontx_port_open(struct octeontx_nic *nic) +{ + octeontx_mbox_bgx_port_conf_t bgx_port_conf; + int res; + + res = 0; + + PMD_INIT_FUNC_TRACE(); + + res = octeontx_bgx_port_open(nic->port_id, &bgx_port_conf); + if (res < 0) { + octeontx_log_err("failed to open port %d", res); + return res; + } + + nic->node = bgx_port_conf.node; + nic->port_ena = bgx_port_conf.enable; + nic->base_ichan = bgx_port_conf.base_chan; + nic->base_ochan = bgx_port_conf.base_chan; + nic->num_ichans = bgx_port_conf.num_chans; + nic->num_ochans = bgx_port_conf.num_chans; + nic->mtu = bgx_port_conf.mtu; + nic->bpen = bgx_port_conf.bpen; + nic->fcs_strip = bgx_port_conf.fcs_strip; + nic->bcast_mode = bgx_port_conf.bcast_mode; + nic->mcast_mode = bgx_port_conf.mcast_mode; + nic->speed = bgx_port_conf.mode; + + memcpy(&nic->mac_addr[0], &bgx_port_conf.macaddr[0], ETHER_ADDR_LEN); + + octeontx_log_dbg("port opened %d", nic->port_id); + return res; +} + +static void +octeontx_port_close(struct octeontx_nic *nic) +{ + PMD_INIT_FUNC_TRACE(); + + octeontx_bgx_port_close(nic->port_id); + octeontx_log_dbg("port closed %d", nic->port_id); +} + +static int +octeontx_port_start(struct octeontx_nic *nic) +{ + PMD_INIT_FUNC_TRACE(); + + return octeontx_bgx_port_start(nic->port_id); +} + +static int +octeontx_port_stop(struct octeontx_nic *nic) +{ + PMD_INIT_FUNC_TRACE(); + + return octeontx_bgx_port_stop(nic->port_id); +} + +static void +octeontx_port_promisc_set(struct octeontx_nic *nic, int en) +{ + struct rte_eth_dev *dev; + int res; + + res = 0; + PMD_INIT_FUNC_TRACE(); + dev = nic->dev; + + res = octeontx_bgx_port_promisc_set(nic->port_id, en); + if (res < 0) + octeontx_log_err("failed to set promiscuous mode %d", + nic->port_id); + + /* Set proper flag for the mode */ + dev->data->promiscuous = (en != 0) ? 1 : 0; + + octeontx_log_dbg("port %d : promiscuous mode %s", + nic->port_id, en ? "set" : "unset"); +} + +static int +octeontx_port_stats(struct octeontx_nic *nic, struct rte_eth_stats *stats) +{ + octeontx_mbox_bgx_port_stats_t bgx_stats; + int res; + + PMD_INIT_FUNC_TRACE(); + + res = octeontx_bgx_port_stats(nic->port_id, &bgx_stats); + if (res < 0) { + octeontx_log_err("failed to get port stats %d", nic->port_id); + return res; + } + + stats->ipackets = bgx_stats.rx_packets; + stats->ibytes = bgx_stats.rx_bytes; + stats->imissed = bgx_stats.rx_dropped; + stats->ierrors = bgx_stats.rx_errors; + stats->opackets = bgx_stats.tx_packets; + stats->obytes = bgx_stats.tx_bytes; + stats->oerrors = bgx_stats.tx_errors; + + octeontx_log_dbg("port%d stats inpkts=%" PRIx64 " outpkts=%" PRIx64 "", + nic->port_id, stats->ipackets, stats->opackets); + + return 0; +} + +static void +octeontx_port_stats_clr(struct octeontx_nic *nic) +{ + PMD_INIT_FUNC_TRACE(); + + octeontx_bgx_port_stats_clr(nic->port_id); +} + +static inline void +devconf_set_default_sane_values(struct rte_event_dev_config *dev_conf, + struct rte_event_dev_info *info) +{ + memset(dev_conf, 0, sizeof(struct rte_event_dev_config)); + dev_conf->dequeue_timeout_ns = info->min_dequeue_timeout_ns; + + dev_conf->nb_event_ports = info->max_event_ports; + dev_conf->nb_event_queues = info->max_event_queues; + + dev_conf->nb_event_queue_flows = info->max_event_queue_flows; + dev_conf->nb_event_port_dequeue_depth = + info->max_event_port_dequeue_depth; + dev_conf->nb_event_port_enqueue_depth = + info->max_event_port_enqueue_depth; + dev_conf->nb_event_port_enqueue_depth = + info->max_event_port_enqueue_depth; + dev_conf->nb_events_limit = + info->max_num_events; +} + +static int +octeontx_dev_configure(struct rte_eth_dev *dev) +{ + struct rte_eth_dev_data *data = dev->data; + struct rte_eth_conf *conf = &data->dev_conf; + struct rte_eth_rxmode *rxmode = &conf->rxmode; + struct rte_eth_txmode *txmode = &conf->txmode; + struct octeontx_nic *nic = octeontx_pmd_priv(dev); + int ret; + + PMD_INIT_FUNC_TRACE(); + RTE_SET_USED(conf); + + if (!rte_eal_has_hugepages()) { + octeontx_log_err("huge page is not configured"); + return -EINVAL; + } + + if (txmode->mq_mode) { + octeontx_log_err("tx mq_mode DCB or VMDq not supported"); + return -EINVAL; + } + + if (rxmode->mq_mode != ETH_MQ_RX_NONE && + rxmode->mq_mode != ETH_MQ_RX_RSS) { + octeontx_log_err("unsupported rx qmode %d", rxmode->mq_mode); + return -EINVAL; + } + + if (!rxmode->hw_strip_crc) { + PMD_INIT_LOG(NOTICE, "can't disable hw crc strip"); + rxmode->hw_strip_crc = 1; + } + + if (rxmode->hw_ip_checksum) { + PMD_INIT_LOG(NOTICE, "rxcksum not supported"); + rxmode->hw_ip_checksum = 0; + } + + if (rxmode->split_hdr_size) { + octeontx_log_err("rxmode does not support split header"); + return -EINVAL; + } + + if (rxmode->hw_vlan_filter) { + octeontx_log_err("VLAN filter not supported"); + return -EINVAL; + } + + if (rxmode->hw_vlan_extend) { + octeontx_log_err("VLAN extended not supported"); + return -EINVAL; + } + + if (rxmode->enable_lro) { + octeontx_log_err("LRO not supported"); + return -EINVAL; + } + + if (conf->link_speeds & ETH_LINK_SPEED_FIXED) { + octeontx_log_err("setting link speed/duplex not supported"); + return -EINVAL; + } + + if (conf->dcb_capability_en) { + octeontx_log_err("DCB enable not supported"); + return -EINVAL; + } + + if (conf->fdir_conf.mode != RTE_FDIR_MODE_NONE) { + octeontx_log_err("flow director not supported"); + return -EINVAL; + } + + nic->num_tx_queues = dev->data->nb_tx_queues; + + ret = octeontx_pko_channel_open(nic->port_id * PKO_VF_NUM_DQ, + nic->num_tx_queues, + nic->base_ochan); + if (ret) { + octeontx_log_err("failed to open channel %d no-of-txq %d", + nic->base_ochan, nic->num_tx_queues); + return -EFAULT; + } + + nic->pki.classifier_enable = false; + nic->pki.hash_enable = true; + nic->pki.initialized = false; + + return 0; +} + +static void +octeontx_dev_close(struct rte_eth_dev *dev) +{ + struct octeontx_txq *txq = NULL; + struct octeontx_nic *nic = octeontx_pmd_priv(dev); + unsigned int i; + int ret; + + PMD_INIT_FUNC_TRACE(); + + rte_event_dev_close(nic->evdev); + + ret = octeontx_pko_channel_close(nic->base_ochan); + if (ret < 0) { + octeontx_log_err("failed to close channel %d VF%d %d %d", + nic->base_ochan, nic->port_id, nic->num_tx_queues, + ret); + } + /* Free txq resources for this port */ + for (i = 0; i < nic->num_tx_queues; i++) { + txq = dev->data->tx_queues[i]; + if (!txq) + continue; + + rte_free(txq); + } +} + +static int +octeontx_dev_start(struct rte_eth_dev *dev) +{ + struct octeontx_nic *nic = octeontx_pmd_priv(dev); + int ret; + + ret = 0; + + PMD_INIT_FUNC_TRACE(); + /* + * Tx start + */ + dev->tx_pkt_burst = octeontx_xmit_pkts; + ret = octeontx_pko_channel_start(nic->base_ochan); + if (ret < 0) { + octeontx_log_err("fail to conf VF%d no. txq %d chan %d ret %d", + nic->port_id, nic->num_tx_queues, nic->base_ochan, + ret); + goto error; + } + + /* + * Rx start + */ + dev->rx_pkt_burst = octeontx_recv_pkts; + ret = octeontx_pki_port_start(nic->port_id); + if (ret < 0) { + octeontx_log_err("fail to start Rx on port %d", nic->port_id); + goto channel_stop_error; + } + + /* + * Start port + */ + ret = octeontx_port_start(nic); + if (ret < 0) { + octeontx_log_err("failed start port %d", ret); + goto pki_port_stop_error; + } + + PMD_TX_LOG(DEBUG, "pko: start channel %d no.of txq %d port %d", + nic->base_ochan, nic->num_tx_queues, nic->port_id); + + ret = rte_event_dev_start(nic->evdev); + if (ret < 0) { + octeontx_log_err("failed to start evdev: ret (%d)", ret); + goto pki_port_stop_error; + } + + /* Success */ + return ret; + +pki_port_stop_error: + octeontx_pki_port_stop(nic->port_id); +channel_stop_error: + octeontx_pko_channel_stop(nic->base_ochan); +error: + return ret; +} + +static void +octeontx_dev_stop(struct rte_eth_dev *dev) +{ + struct octeontx_nic *nic = octeontx_pmd_priv(dev); + int ret; + + PMD_INIT_FUNC_TRACE(); + + rte_event_dev_stop(nic->evdev); + + ret = octeontx_port_stop(nic); + if (ret < 0) { + octeontx_log_err("failed to req stop port %d res=%d", + nic->port_id, ret); + return; + } + + ret = octeontx_pki_port_stop(nic->port_id); + if (ret < 0) { + octeontx_log_err("failed to stop pki port %d res=%d", + nic->port_id, ret); + return; + } + + ret = octeontx_pko_channel_stop(nic->base_ochan); + if (ret < 0) { + octeontx_log_err("failed to stop channel %d VF%d %d %d", + nic->base_ochan, nic->port_id, nic->num_tx_queues, + ret); + return; + } + + dev->tx_pkt_burst = NULL; + dev->rx_pkt_burst = NULL; +} + +static void +octeontx_dev_promisc_enable(struct rte_eth_dev *dev) +{ + struct octeontx_nic *nic = octeontx_pmd_priv(dev); + + PMD_INIT_FUNC_TRACE(); + octeontx_port_promisc_set(nic, 1); +} + +static void +octeontx_dev_promisc_disable(struct rte_eth_dev *dev) +{ + struct octeontx_nic *nic = octeontx_pmd_priv(dev); + + PMD_INIT_FUNC_TRACE(); + octeontx_port_promisc_set(nic, 0); +} + +static inline int +octeontx_atomic_write_link_status(struct rte_eth_dev *dev, + struct rte_eth_link *link) +{ + struct rte_eth_link *dst = &dev->data->dev_link; + struct rte_eth_link *src = link; + + if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst, + *(uint64_t *)src) == 0) + return -1; + + return 0; +} + +static int +octeontx_port_link_status(struct octeontx_nic *nic) +{ + int res; + + PMD_INIT_FUNC_TRACE(); + res = octeontx_bgx_port_link_status(nic->port_id); + if (res < 0) { + octeontx_log_err("failed to get port %d link status", + nic->port_id); + return res; + } + + nic->link_up = (uint8_t)res; + octeontx_log_dbg("port %d link status %d", nic->port_id, nic->link_up); + + return res; +} + +/* + * Return 0 means link status changed, -1 means not changed + */ +static int +octeontx_dev_link_update(struct rte_eth_dev *dev, + int wait_to_complete __rte_unused) +{ + struct octeontx_nic *nic = octeontx_pmd_priv(dev); + struct rte_eth_link link; + int res; + + res = 0; + PMD_INIT_FUNC_TRACE(); + + res = octeontx_port_link_status(nic); + if (res < 0) { + octeontx_log_err("failed to request link status %d", res); + return res; + } + + link.link_status = nic->link_up; + + switch (nic->speed) { + case OCTEONTX_LINK_SPEED_SGMII: + link.link_speed = ETH_SPEED_NUM_1G; + break; + + case OCTEONTX_LINK_SPEED_XAUI: + link.link_speed = ETH_SPEED_NUM_10G; + break; + + case OCTEONTX_LINK_SPEED_RXAUI: + case OCTEONTX_LINK_SPEED_10G_R: + link.link_speed = ETH_SPEED_NUM_10G; + break; + case OCTEONTX_LINK_SPEED_QSGMII: + link.link_speed = ETH_SPEED_NUM_5G; + break; + case OCTEONTX_LINK_SPEED_40G_R: + link.link_speed = ETH_SPEED_NUM_40G; + break; + + case OCTEONTX_LINK_SPEED_RESERVE1: + case OCTEONTX_LINK_SPEED_RESERVE2: + default: + octeontx_log_err("incorrect link speed %d", nic->speed); + break; + } + + link.link_duplex = ETH_LINK_FULL_DUPLEX; + link.link_autoneg = ETH_LINK_AUTONEG; + + return octeontx_atomic_write_link_status(dev, &link); +} + +static int +octeontx_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) +{ + struct octeontx_nic *nic = octeontx_pmd_priv(dev); + + PMD_INIT_FUNC_TRACE(); + return octeontx_port_stats(nic, stats); +} + +static void +octeontx_dev_stats_reset(struct rte_eth_dev *dev) +{ + struct octeontx_nic *nic = octeontx_pmd_priv(dev); + + PMD_INIT_FUNC_TRACE(); + octeontx_port_stats_clr(nic); +} + +static void +octeontx_dev_default_mac_addr_set(struct rte_eth_dev *dev, + struct ether_addr *addr) +{ + struct octeontx_nic *nic = octeontx_pmd_priv(dev); + int ret; + + ret = octeontx_bgx_port_mac_set(nic->port_id, addr->addr_bytes); + if (ret != 0) + octeontx_log_err("failed to set MAC address on port %d", + nic->port_id); +} + +static void +octeontx_dev_info(struct rte_eth_dev *dev, + struct rte_eth_dev_info *dev_info) +{ + RTE_SET_USED(dev); + + /* Autonegotiation may be disabled */ + dev_info->speed_capa = ETH_LINK_SPEED_FIXED; + dev_info->speed_capa |= ETH_LINK_SPEED_10M | ETH_LINK_SPEED_100M | + ETH_LINK_SPEED_1G | ETH_LINK_SPEED_10G | + ETH_LINK_SPEED_40G; + + dev_info->driver_name = RTE_STR(rte_octeontx_pmd); + dev_info->max_mac_addrs = 1; + dev_info->max_rx_pktlen = PKI_MAX_PKTLEN; + dev_info->max_rx_queues = 1; + dev_info->max_tx_queues = PKO_MAX_NUM_DQ; + dev_info->min_rx_bufsize = 0; + dev_info->pci_dev = NULL; + + dev_info->default_rxconf = (struct rte_eth_rxconf) { + .rx_free_thresh = 0, + .rx_drop_en = 0, + }; + + dev_info->default_txconf = (struct rte_eth_txconf) { + .tx_free_thresh = 0, + .txq_flags = + ETH_TXQ_FLAGS_NOMULTSEGS | + ETH_TXQ_FLAGS_NOOFFLOADS | + ETH_TXQ_FLAGS_NOXSUMS, + }; + + dev_info->tx_offload_capa = DEV_TX_OFFLOAD_MT_LOCKFREE; +} + +static void +octeontx_dq_info_getter(octeontx_dq_t *dq, void *out) +{ + ((octeontx_dq_t *)out)->lmtline_va = dq->lmtline_va; + ((octeontx_dq_t *)out)->ioreg_va = dq->ioreg_va; + ((octeontx_dq_t *)out)->fc_status_va = dq->fc_status_va; +} + +static int +octeontx_vf_start_tx_queue(struct rte_eth_dev *dev, struct octeontx_nic *nic, + uint16_t qidx) +{ + struct octeontx_txq *txq; + int res; + + PMD_INIT_FUNC_TRACE(); + + if (dev->data->tx_queue_state[qidx] == RTE_ETH_QUEUE_STATE_STARTED) + return 0; + + txq = dev->data->tx_queues[qidx]; + + res = octeontx_pko_channel_query_dqs(nic->base_ochan, + &txq->dq, + sizeof(octeontx_dq_t), + txq->queue_id, + octeontx_dq_info_getter); + if (res < 0) { + res = -EFAULT; + goto close_port; + } + + dev->data->tx_queue_state[qidx] = RTE_ETH_QUEUE_STATE_STARTED; + return res; + +close_port: + (void)octeontx_port_stop(nic); + octeontx_pko_channel_stop(nic->base_ochan); + octeontx_pko_channel_close(nic->base_ochan); + dev->data->tx_queue_state[qidx] = RTE_ETH_QUEUE_STATE_STOPPED; + return res; +} + +static int +octeontx_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t qidx) +{ + struct octeontx_nic *nic = octeontx_pmd_priv(dev); + + PMD_INIT_FUNC_TRACE(); + qidx = qidx % PKO_VF_NUM_DQ; + return octeontx_vf_start_tx_queue(dev, nic, qidx); +} + +static inline int +octeontx_vf_stop_tx_queue(struct rte_eth_dev *dev, struct octeontx_nic *nic, + uint16_t qidx) +{ + int ret = 0; + + RTE_SET_USED(nic); + PMD_INIT_FUNC_TRACE(); + + if (dev->data->tx_queue_state[qidx] == RTE_ETH_QUEUE_STATE_STOPPED) + return 0; + + dev->data->tx_queue_state[qidx] = RTE_ETH_QUEUE_STATE_STOPPED; + return ret; +} + +static int +octeontx_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t qidx) +{ + struct octeontx_nic *nic = octeontx_pmd_priv(dev); + + PMD_INIT_FUNC_TRACE(); + qidx = qidx % PKO_VF_NUM_DQ; + + return octeontx_vf_stop_tx_queue(dev, nic, qidx); +} + +static void +octeontx_dev_tx_queue_release(void *tx_queue) +{ + struct octeontx_txq *txq = tx_queue; + int res; + + PMD_INIT_FUNC_TRACE(); + + if (txq) { + res = octeontx_dev_tx_queue_stop(txq->eth_dev, txq->queue_id); + if (res < 0) + octeontx_log_err("failed stop tx_queue(%d)\n", + txq->queue_id); + + rte_free(txq); + } +} + +static int +octeontx_dev_tx_queue_setup(struct rte_eth_dev *dev, uint16_t qidx, + uint16_t nb_desc, unsigned int socket_id, + const struct rte_eth_txconf *tx_conf) +{ + struct octeontx_nic *nic = octeontx_pmd_priv(dev); + struct octeontx_txq *txq = NULL; + uint16_t dq_num; + int res = 0; + + RTE_SET_USED(nb_desc); + RTE_SET_USED(socket_id); + RTE_SET_USED(tx_conf); + + dq_num = (nic->port_id * PKO_VF_NUM_DQ) + qidx; + + /* Socket id check */ + if (socket_id != (unsigned int)SOCKET_ID_ANY && + socket_id != (unsigned int)nic->node) + PMD_TX_LOG(INFO, "socket_id expected %d, configured %d", + socket_id, nic->node); + + /* Free memory prior to re-allocation if needed. */ + if (dev->data->tx_queues[qidx] != NULL) { + PMD_TX_LOG(DEBUG, "freeing memory prior to re-allocation %d", + qidx); + octeontx_dev_tx_queue_release(dev->data->tx_queues[qidx]); + dev->data->tx_queues[qidx] = NULL; + } + + /* Allocating tx queue data structure */ + txq = rte_zmalloc_socket("ethdev TX queue", sizeof(struct octeontx_txq), + RTE_CACHE_LINE_SIZE, nic->node); + if (txq == NULL) { + octeontx_log_err("failed to allocate txq=%d", qidx); + res = -ENOMEM; + goto err; + } + + txq->eth_dev = dev; + txq->queue_id = dq_num; + dev->data->tx_queues[qidx] = txq; + dev->data->tx_queue_state[qidx] = RTE_ETH_QUEUE_STATE_STOPPED; + + res = octeontx_pko_channel_query_dqs(nic->base_ochan, + &txq->dq, + sizeof(octeontx_dq_t), + txq->queue_id, + octeontx_dq_info_getter); + if (res < 0) { + res = -EFAULT; + goto err; + } + + PMD_TX_LOG(DEBUG, "[%d]:[%d] txq=%p nb_desc=%d lmtline=%p ioreg_va=%p fc_status_va=%p", + qidx, txq->queue_id, txq, nb_desc, txq->dq.lmtline_va, + txq->dq.ioreg_va, + txq->dq.fc_status_va); + + return res; + +err: + if (txq) + rte_free(txq); + + return res; +} + +static int +octeontx_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t qidx, + uint16_t nb_desc, unsigned int socket_id, + const struct rte_eth_rxconf *rx_conf, + struct rte_mempool *mb_pool) +{ + struct octeontx_nic *nic = octeontx_pmd_priv(dev); + struct rte_mempool_ops *mp_ops = NULL; + struct octeontx_rxq *rxq = NULL; + pki_pktbuf_cfg_t pktbuf_conf; + pki_hash_cfg_t pki_hash; + pki_qos_cfg_t pki_qos; + uintptr_t pool; + int ret, port; + uint8_t gaura; + unsigned int ev_queues = (nic->ev_queues * nic->port_id) + qidx; + unsigned int ev_ports = (nic->ev_ports * nic->port_id) + qidx; + + RTE_SET_USED(nb_desc); + + memset(&pktbuf_conf, 0, sizeof(pktbuf_conf)); + memset(&pki_hash, 0, sizeof(pki_hash)); + memset(&pki_qos, 0, sizeof(pki_qos)); + + mp_ops = rte_mempool_get_ops(mb_pool->ops_index); + if (strcmp(mp_ops->name, "octeontx_fpavf")) { + octeontx_log_err("failed to find octeontx_fpavf mempool"); + return -ENOTSUP; + } + + /* Handle forbidden configurations */ + if (nic->pki.classifier_enable) { + octeontx_log_err("cannot setup queue %d. " + "Classifier option unsupported", qidx); + return -EINVAL; + } + + port = nic->port_id; + + /* Rx deferred start is not supported */ + if (rx_conf->rx_deferred_start) { + octeontx_log_err("rx deferred start not supported"); + return -EINVAL; + } + + /* Verify queue index */ + if (qidx >= dev->data->nb_rx_queues) { + octeontx_log_err("QID %d not supporteded (0 - %d available)\n", + qidx, (dev->data->nb_rx_queues - 1)); + return -ENOTSUP; + } + + /* Socket id check */ + if (socket_id != (unsigned int)SOCKET_ID_ANY && + socket_id != (unsigned int)nic->node) + PMD_RX_LOG(INFO, "socket_id expected %d, configured %d", + socket_id, nic->node); + + /* Allocating rx queue data structure */ + rxq = rte_zmalloc_socket("ethdev RX queue", sizeof(struct octeontx_rxq), + RTE_CACHE_LINE_SIZE, nic->node); + if (rxq == NULL) { + octeontx_log_err("failed to allocate rxq=%d", qidx); + return -ENOMEM; + } + + if (!nic->pki.initialized) { + pktbuf_conf.port_type = 0; + pki_hash.port_type = 0; + pki_qos.port_type = 0; + + pktbuf_conf.mmask.f_wqe_skip = 1; + pktbuf_conf.mmask.f_first_skip = 1; + pktbuf_conf.mmask.f_later_skip = 1; + pktbuf_conf.mmask.f_mbuff_size = 1; + pktbuf_conf.mmask.f_cache_mode = 1; + + pktbuf_conf.wqe_skip = OCTTX_PACKET_WQE_SKIP; + pktbuf_conf.first_skip = OCTTX_PACKET_FIRST_SKIP; + pktbuf_conf.later_skip = OCTTX_PACKET_LATER_SKIP; + pktbuf_conf.mbuff_size = (mb_pool->elt_size - + RTE_PKTMBUF_HEADROOM - + sizeof(struct rte_mbuf)); + + pktbuf_conf.cache_mode = PKI_OPC_MODE_STF2_STT; + + ret = octeontx_pki_port_pktbuf_config(port, &pktbuf_conf); + if (ret != 0) { + octeontx_log_err("fail to configure pktbuf for port %d", + port); + rte_free(rxq); + return ret; + } + PMD_RX_LOG(DEBUG, "Port %d Rx pktbuf configured:\n" + "\tmbuf_size:\t0x%0x\n" + "\twqe_skip:\t0x%0x\n" + "\tfirst_skip:\t0x%0x\n" + "\tlater_skip:\t0x%0x\n" + "\tcache_mode:\t%s\n", + port, + pktbuf_conf.mbuff_size, + pktbuf_conf.wqe_skip, + pktbuf_conf.first_skip, + pktbuf_conf.later_skip, + (pktbuf_conf.cache_mode == + PKI_OPC_MODE_STT) ? + "STT" : + (pktbuf_conf.cache_mode == + PKI_OPC_MODE_STF) ? + "STF" : + (pktbuf_conf.cache_mode == + PKI_OPC_MODE_STF1_STT) ? + "STF1_STT" : "STF2_STT"); + + if (nic->pki.hash_enable) { + pki_hash.tag_dlc = 1; + pki_hash.tag_slc = 1; + pki_hash.tag_dlf = 1; + pki_hash.tag_slf = 1; + pki_hash.tag_prt = 1; + octeontx_pki_port_hash_config(port, &pki_hash); + } + + pool = (uintptr_t)mb_pool->pool_id; + + /* Get the gpool Id */ + gaura = octeontx_fpa_bufpool_gpool(pool); + + pki_qos.qpg_qos = PKI_QPG_QOS_NONE; + pki_qos.num_entry = 1; + pki_qos.drop_policy = 0; + pki_qos.tag_type = 0L; + pki_qos.qos_entry[0].port_add = 0; + pki_qos.qos_entry[0].gaura = gaura; + pki_qos.qos_entry[0].ggrp_ok = ev_queues; + pki_qos.qos_entry[0].ggrp_bad = ev_queues; + pki_qos.qos_entry[0].grptag_bad = 0; + pki_qos.qos_entry[0].grptag_ok = 0; + + ret = octeontx_pki_port_create_qos(port, &pki_qos); + if (ret < 0) { + octeontx_log_err("failed to create QOS port=%d, q=%d", + port, qidx); + rte_free(rxq); + return ret; + } + nic->pki.initialized = true; + } + + rxq->port_id = nic->port_id; + rxq->eth_dev = dev; + rxq->queue_id = qidx; + rxq->evdev = nic->evdev; + rxq->ev_queues = ev_queues; + rxq->ev_ports = ev_ports; + + dev->data->rx_queues[qidx] = rxq; + dev->data->rx_queue_state[qidx] = RTE_ETH_QUEUE_STATE_STOPPED; + return 0; +} + +static void +octeontx_dev_rx_queue_release(void *rxq) +{ + rte_free(rxq); +} + +static const uint32_t * +octeontx_dev_supported_ptypes_get(struct rte_eth_dev *dev) +{ + static const uint32_t ptypes[] = { + RTE_PTYPE_L3_IPV4, + RTE_PTYPE_L3_IPV4_EXT, + RTE_PTYPE_L3_IPV6, + RTE_PTYPE_L3_IPV6_EXT, + RTE_PTYPE_L4_TCP, + RTE_PTYPE_L4_UDP, + RTE_PTYPE_L4_FRAG, + RTE_PTYPE_UNKNOWN + }; + + if (dev->rx_pkt_burst == octeontx_recv_pkts) + return ptypes; + + return NULL; +} + +/* Initialize and register driver with DPDK Application */ +static const struct eth_dev_ops octeontx_dev_ops = { + .dev_configure = octeontx_dev_configure, + .dev_infos_get = octeontx_dev_info, + .dev_close = octeontx_dev_close, + .dev_start = octeontx_dev_start, + .dev_stop = octeontx_dev_stop, + .promiscuous_enable = octeontx_dev_promisc_enable, + .promiscuous_disable = octeontx_dev_promisc_disable, + .link_update = octeontx_dev_link_update, + .stats_get = octeontx_dev_stats_get, + .stats_reset = octeontx_dev_stats_reset, + .mac_addr_set = octeontx_dev_default_mac_addr_set, + .tx_queue_start = octeontx_dev_tx_queue_start, + .tx_queue_stop = octeontx_dev_tx_queue_stop, + .tx_queue_setup = octeontx_dev_tx_queue_setup, + .tx_queue_release = octeontx_dev_tx_queue_release, + .rx_queue_setup = octeontx_dev_rx_queue_setup, + .rx_queue_release = octeontx_dev_rx_queue_release, + .dev_supported_ptypes_get = octeontx_dev_supported_ptypes_get, +}; + +/* Create Ethdev interface per BGX LMAC ports */ +static int +octeontx_create(struct rte_vdev_device *dev, int port, uint8_t evdev, + int socket_id) +{ + int res; + char octtx_name[OCTEONTX_MAX_NAME_LEN]; + struct octeontx_nic *nic = NULL; + struct rte_eth_dev *eth_dev = NULL; + struct rte_eth_dev_data *data = NULL; + const char *name = rte_vdev_device_name(dev); + + PMD_INIT_FUNC_TRACE(); + + sprintf(octtx_name, "%s_%d", name, port); + if (rte_eal_process_type() != RTE_PROC_PRIMARY) { + eth_dev = rte_eth_dev_attach_secondary(octtx_name); + if (eth_dev == NULL) + return -ENODEV; + + eth_dev->tx_pkt_burst = octeontx_xmit_pkts; + eth_dev->rx_pkt_burst = octeontx_recv_pkts; + return 0; + } + + data = rte_zmalloc_socket(octtx_name, sizeof(*data), 0, socket_id); + if (data == NULL) { + octeontx_log_err("failed to allocate devdata"); + res = -ENOMEM; + goto err; + } + + nic = rte_zmalloc_socket(octtx_name, sizeof(*nic), 0, socket_id); + if (nic == NULL) { + octeontx_log_err("failed to allocate nic structure"); + res = -ENOMEM; + goto err; + } + + nic->port_id = port; + nic->evdev = evdev; + + res = octeontx_port_open(nic); + if (res < 0) + goto err; + + /* Rx side port configuration */ + res = octeontx_pki_port_open(port); + if (res != 0) { + octeontx_log_err("failed to open PKI port %d", port); + res = -ENODEV; + goto err; + } + + /* Reserve an ethdev entry */ + eth_dev = rte_eth_dev_allocate(octtx_name); + if (eth_dev == NULL) { + octeontx_log_err("failed to allocate rte_eth_dev"); + res = -ENOMEM; + goto err; + } + + eth_dev->device = &dev->device; + eth_dev->intr_handle = NULL; + eth_dev->data->kdrv = RTE_KDRV_NONE; + eth_dev->data->numa_node = dev->device.numa_node; + + rte_memcpy(data, (eth_dev)->data, sizeof(*data)); + data->dev_private = nic; + + data->port_id = eth_dev->data->port_id; + snprintf(data->name, sizeof(data->name), "%s", eth_dev->data->name); + + nic->ev_queues = 1; + nic->ev_ports = 1; + + data->dev_link.link_status = ETH_LINK_DOWN; + data->dev_started = 0; + data->promiscuous = 0; + data->all_multicast = 0; + data->scattered_rx = 0; + + data->mac_addrs = rte_zmalloc_socket(octtx_name, ETHER_ADDR_LEN, 0, + socket_id); + if (data->mac_addrs == NULL) { + octeontx_log_err("failed to allocate memory for mac_addrs"); + res = -ENOMEM; + goto err; + } + + eth_dev->data = data; + eth_dev->dev_ops = &octeontx_dev_ops; + + /* Finally save ethdev pointer to the NIC structure */ + nic->dev = eth_dev; + + if (nic->port_id != data->port_id) { + octeontx_log_err("eth_dev->port_id (%d) is diff to orig (%d)", + data->port_id, nic->port_id); + res = -EINVAL; + goto err; + } + + /* Update port_id mac to eth_dev */ + memcpy(data->mac_addrs, nic->mac_addr, ETHER_ADDR_LEN); + + PMD_INIT_LOG(DEBUG, "ethdev info: "); + PMD_INIT_LOG(DEBUG, "port %d, port_ena %d ochan %d num_ochan %d tx_q %d", + nic->port_id, nic->port_ena, + nic->base_ochan, nic->num_ochans, + nic->num_tx_queues); + PMD_INIT_LOG(DEBUG, "speed %d mtu %d", nic->speed, nic->mtu); + + rte_octeontx_pchan_map[(nic->base_ochan >> 8) & 0x7] + [(nic->base_ochan >> 4) & 0xF] = data->port_id; + + return data->port_id; + +err: + if (port) + octeontx_port_close(nic); + + if (eth_dev != NULL) { + rte_free(eth_dev->data->mac_addrs); + rte_free(data); + rte_free(nic); + rte_eth_dev_release_port(eth_dev); + } + + return res; +} + +/* Un initialize octeontx device */ +static int +octeontx_remove(struct rte_vdev_device *dev) +{ + char octtx_name[OCTEONTX_MAX_NAME_LEN]; + struct rte_eth_dev *eth_dev = NULL; + struct octeontx_nic *nic = NULL; + int i; + + if (dev == NULL) + return -EINVAL; + + for (i = 0; i < OCTEONTX_VDEV_DEFAULT_MAX_NR_PORT; i++) { + sprintf(octtx_name, "eth_octeontx_%d", i); + + /* reserve an ethdev entry */ + eth_dev = rte_eth_dev_allocated(octtx_name); + if (eth_dev == NULL) + return -ENODEV; + + nic = octeontx_pmd_priv(eth_dev); + rte_event_dev_stop(nic->evdev); + PMD_INIT_LOG(INFO, "Closing octeontx device %s", octtx_name); + + rte_free(eth_dev->data->mac_addrs); + rte_free(eth_dev->data->dev_private); + rte_free(eth_dev->data); + rte_eth_dev_release_port(eth_dev); + rte_event_dev_close(nic->evdev); + } + + /* Free FC resource */ + octeontx_pko_fc_free(); + + return 0; +} + +/* Initialize octeontx device */ +static int +octeontx_probe(struct rte_vdev_device *dev) +{ + const char *dev_name; + static int probe_once; + uint8_t socket_id, qlist; + int tx_vfcnt, port_id, evdev, qnum, pnum, res, i; + struct rte_event_dev_config dev_conf; + const char *eventdev_name = "event_octeontx"; + struct rte_event_dev_info info; + + struct octeontx_vdev_init_params init_params = { + OCTEONTX_VDEV_DEFAULT_MAX_NR_PORT + }; + + dev_name = rte_vdev_device_name(dev); + res = octeontx_parse_vdev_init_params(&init_params, dev); + if (res < 0) + return -EINVAL; + + if (init_params.nr_port > OCTEONTX_VDEV_DEFAULT_MAX_NR_PORT) { + octeontx_log_err("nr_port (%d) > max (%d)", init_params.nr_port, + OCTEONTX_VDEV_DEFAULT_MAX_NR_PORT); + return -ENOTSUP; + } + + PMD_INIT_LOG(DEBUG, "initializing %s pmd", dev_name); + + socket_id = rte_socket_id(); + + tx_vfcnt = octeontx_pko_vf_count(); + + if (tx_vfcnt < init_params.nr_port) { + octeontx_log_err("not enough PKO (%d) for port number (%d)", + tx_vfcnt, init_params.nr_port); + return -EINVAL; + } + evdev = rte_event_dev_get_dev_id(eventdev_name); + if (evdev < 0) { + octeontx_log_err("eventdev %s not found", eventdev_name); + return -ENODEV; + } + + res = rte_event_dev_info_get(evdev, &info); + if (res < 0) { + octeontx_log_err("failed to eventdev info %d", res); + return -EINVAL; + } + + PMD_INIT_LOG(DEBUG, "max_queue %d max_port %d", + info.max_event_queues, info.max_event_ports); + + if (octeontx_pko_init_fc(tx_vfcnt)) + return -ENOMEM; + + devconf_set_default_sane_values(&dev_conf, &info); + res = rte_event_dev_configure(evdev, &dev_conf); + if (res < 0) + goto parse_error; + + rte_event_dev_attr_get(evdev, RTE_EVENT_DEV_ATTR_PORT_COUNT, + (uint32_t *)&pnum); + rte_event_dev_attr_get(evdev, RTE_EVENT_DEV_ATTR_QUEUE_COUNT, + (uint32_t *)&qnum); + if (pnum < qnum) { + octeontx_log_err("too few event ports (%d) for event_q(%d)", + pnum, qnum); + res = -EINVAL; + goto parse_error; + } + if (pnum > qnum) { + /* + * We don't poll on event ports + * that do not have any queues assigned. + */ + pnum = qnum; + PMD_INIT_LOG(INFO, + "reducing number of active event ports to %d", pnum); + } + for (i = 0; i < qnum; i++) { + res = rte_event_queue_setup(evdev, i, NULL); + if (res < 0) { + octeontx_log_err("failed to setup event_q(%d): res %d", + i, res); + goto parse_error; + } + } + + for (i = 0; i < pnum; i++) { + res = rte_event_port_setup(evdev, i, NULL); + if (res < 0) { + res = -ENODEV; + octeontx_log_err("failed to setup ev port(%d) res=%d", + i, res); + goto parse_error; + } + /* Link one queue to one event port */ + qlist = i; + res = rte_event_port_link(evdev, i, &qlist, NULL, 1); + if (res < 0) { + res = -ENODEV; + octeontx_log_err("failed to link port (%d): res=%d", + i, res); + goto parse_error; + } + } + + /* Create ethdev interface */ + for (i = 0; i < init_params.nr_port; i++) { + port_id = octeontx_create(dev, i, evdev, socket_id); + if (port_id < 0) { + octeontx_log_err("failed to create device %s", + dev_name); + res = -ENODEV; + goto parse_error; + } + + PMD_INIT_LOG(INFO, "created ethdev %s for port %d", dev_name, + port_id); + } + + if (probe_once) { + octeontx_log_err("interface %s not supported", dev_name); + octeontx_remove(dev); + res = -ENOTSUP; + goto parse_error; + } + probe_once = 1; + + return 0; + +parse_error: + octeontx_pko_fc_free(); + return res; +} + +static struct rte_vdev_driver octeontx_pmd_drv = { + .probe = octeontx_probe, + .remove = octeontx_remove, +}; + +RTE_PMD_REGISTER_VDEV(OCTEONTX_PMD, octeontx_pmd_drv); +RTE_PMD_REGISTER_ALIAS(OCTEONTX_PMD, eth_octeontx); +RTE_PMD_REGISTER_PARAM_STRING(OCTEONTX_PMD, "nr_port= "); diff --git a/dpdk/drivers/net/octeontx/octeontx_ethdev.h b/dpdk/drivers/net/octeontx/octeontx_ethdev.h new file mode 100644 index 00000000..d37bb8a2 --- /dev/null +++ b/dpdk/drivers/net/octeontx/octeontx_ethdev.h @@ -0,0 +1,115 @@ +/* + * BSD LICENSE + * + * Copyright (C) Cavium Inc. 2017. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium networks nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef __OCTEONTX_ETHDEV_H__ +#define __OCTEONTX_ETHDEV_H__ + +#include + +#include +#include +#include +#include +#include + +#include + +#include "base/octeontx_bgx.h" +#include "base/octeontx_pki_var.h" +#include "base/octeontx_pkivf.h" +#include "base/octeontx_pkovf.h" +#include "base/octeontx_io.h" + +#define OCTEONTX_VDEV_DEFAULT_MAX_NR_PORT 12 +#define OCTEONTX_VDEV_NR_PORT_ARG ("nr_port") +#define OCTEONTX_MAX_NAME_LEN 32 + +#define OCTEONTX_MAX_BGX_PORTS 4 +#define OCTEONTX_MAX_LMAC_PER_BGX 4 + +static inline struct octeontx_nic * +octeontx_pmd_priv(struct rte_eth_dev *dev) +{ + return dev->data->dev_private; +} + +extern uint16_t +rte_octeontx_pchan_map[OCTEONTX_MAX_BGX_PORTS][OCTEONTX_MAX_LMAC_PER_BGX]; + +/* Octeontx ethdev nic */ +struct octeontx_nic { + struct rte_eth_dev *dev; + int node; + int port_id; + int port_ena; + int base_ichan; + int num_ichans; + int base_ochan; + int num_ochans; + uint8_t evdev; + uint8_t bpen; + uint8_t fcs_strip; + uint8_t bcast_mode; + uint8_t mcast_mode; + uint16_t num_tx_queues; + uint64_t hwcap; + uint8_t link_up; + uint8_t duplex; + uint8_t speed; + uint16_t mtu; + uint8_t mac_addr[ETHER_ADDR_LEN]; + /* Rx port parameters */ + struct { + bool classifier_enable; + bool hash_enable; + bool initialized; + } pki; + + uint16_t ev_queues; + uint16_t ev_ports; +} __rte_cache_aligned; + +struct octeontx_txq { + uint16_t queue_id; + octeontx_dq_t dq; + struct rte_eth_dev *eth_dev; +} __rte_cache_aligned; + +struct octeontx_rxq { + uint16_t queue_id; + uint16_t port_id; + uint8_t evdev; + struct rte_eth_dev *eth_dev; + uint16_t ev_queues; + uint16_t ev_ports; +} __rte_cache_aligned; + +#endif /* __OCTEONTX_ETHDEV_H__ */ diff --git a/dpdk/drivers/net/octeontx/octeontx_logs.h b/dpdk/drivers/net/octeontx/octeontx_logs.h new file mode 100644 index 00000000..d5da7331 --- /dev/null +++ b/dpdk/drivers/net/octeontx/octeontx_logs.h @@ -0,0 +1,76 @@ +/* + * BSD LICENSE + * + * Copyright (C) Cavium Inc. 2017. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium networks nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef __OCTEONTX_LOGS_H__ +#define __OCTEONTX_LOGS_H__ + +#define PMD_INIT_LOG(level, fmt, args...) \ + RTE_LOG(level, PMD, "%s(): " fmt "\n", __func__, ## args) + +#ifdef RTE_LIBRTE_OCTEONTX_DEBUG_INIT +#define PMD_INIT_FUNC_TRACE() PMD_INIT_LOG(DEBUG, ">>") +#else +#define PMD_INIT_FUNC_TRACE() do { } while (0) +#endif + +#ifdef RTE_LIBRTE_OCTEONTX_DEBUG_RX +#define PMD_RX_LOG(level, fmt, args...) \ + RTE_LOG(level, PMD, "%s(): " fmt "\n", __func__, ## args) +#else +#define PMD_RX_LOG(level, fmt, args...) do { } while (0) +#endif + +#ifdef RTE_LIBRTE_OCTEONTX_DEBUG_TX +#define PMD_TX_LOG(level, fmt, args...) \ + RTE_LOG(level, PMD, "%s(): " fmt "\n", __func__, ## args) +#else +#define PMD_TX_LOG(level, fmt, args...) do { } while (0) +#endif + +#ifdef RTE_LIBRTE_OCTEONTX_DEBUG_DRIVER +#define PMD_DRV_LOG(level, fmt, args...) \ + RTE_LOG(level, PMD, "%s(): " fmt "\n", __func__, ## args) +#else +#define PMD_DRV_LOG(level, fmt, args...) do { } while (0) +#endif + +#ifdef RTE_LIBRTE_OCTEONTX_DEBUG_MBOX +#define PMD_MBOX_LOG(level, fmt, args...) \ + RTE_LOG(level, PMD, "%s(): " fmt "\n", __func__, ## args) +#else +#define PMD_MBOX_LOG(level, fmt, args...) do { } while (0) +#endif + +#define octeontx_log_err(s, ...) PMD_INIT_LOG(ERR, s, ##__VA_ARGS__) +#define octeontx_log_dbg(s, ...) PMD_DRV_LOG(DEBUG, s, ##__VA_ARGS__) +#define octeontx_mbox_log(s, ...) PMD_MBOX_LOG(DEBUG, s, ##__VA_ARGS__) + +#endif /* __OCTEONTX_LOGS_H__*/ diff --git a/dpdk/drivers/net/octeontx/octeontx_rxtx.c b/dpdk/drivers/net/octeontx/octeontx_rxtx.c new file mode 100644 index 00000000..c97d5b35 --- /dev/null +++ b/dpdk/drivers/net/octeontx/octeontx_rxtx.c @@ -0,0 +1,127 @@ +/* + * BSD LICENSE + * + * Copyright (C) Cavium, Inc. 2017. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "octeontx_ethdev.h" +#include "octeontx_rxtx.h" +#include "octeontx_logs.h" + + +static __rte_always_inline uint16_t __hot +__octeontx_xmit_pkts(void *lmtline_va, void *ioreg_va, int64_t *fc_status_va, + struct rte_mbuf *tx_pkt) +{ + uint64_t cmd_buf[4]; + uint16_t gaura_id; + + if (unlikely(*((volatile int64_t *)fc_status_va) < 0)) + return -ENOSPC; + + /* Get the gaura Id */ + gaura_id = octeontx_fpa_bufpool_gpool((uintptr_t)tx_pkt->pool->pool_id); + + /* Setup PKO_SEND_HDR_S */ + cmd_buf[0] = tx_pkt->data_len & 0xffff; + cmd_buf[1] = 0x0; + + /* Set don't free bit if reference count > 1 */ + if (rte_mbuf_refcnt_read(tx_pkt) > 1) + cmd_buf[0] |= (1ULL << 58); /* SET DF */ + + /* Setup PKO_SEND_GATHER_S */ + cmd_buf[(1 << 1) | 1] = rte_mbuf_data_iova(tx_pkt); + cmd_buf[(1 << 1) | 0] = PKO_SEND_GATHER_SUBDC | + PKO_SEND_GATHER_LDTYPE(0x1ull) | + PKO_SEND_GATHER_GAUAR((long)gaura_id) | + tx_pkt->data_len; + + octeontx_reg_lmtst(lmtline_va, ioreg_va, cmd_buf, PKO_CMD_SZ); + + return 0; +} + +uint16_t __hot +octeontx_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) +{ + int count; + struct octeontx_txq *txq = tx_queue; + octeontx_dq_t *dq = &txq->dq; + int res; + + count = 0; + + while (count < nb_pkts) { + res = __octeontx_xmit_pkts(dq->lmtline_va, dq->ioreg_va, + dq->fc_status_va, + tx_pkts[count]); + if (res < 0) + break; + + count++; + } + + return count; /* return number of pkts transmitted */ +} + +uint16_t __hot +octeontx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) +{ + struct octeontx_rxq *rxq; + struct rte_event ev; + size_t count; + uint16_t valid_event; + + rxq = rx_queue; + count = 0; + while (count < nb_pkts) { + valid_event = rte_event_dequeue_burst(rxq->evdev, + rxq->ev_ports, &ev, + 1, 0); + if (!valid_event) + break; + rx_pkts[count++] = ev.mbuf; + } + + return count; /* return number of pkts received */ +} diff --git a/dpdk/drivers/net/octeontx/octeontx_rxtx.h b/dpdk/drivers/net/octeontx/octeontx_rxtx.h new file mode 100644 index 00000000..382ff2b2 --- /dev/null +++ b/dpdk/drivers/net/octeontx/octeontx_rxtx.h @@ -0,0 +1,137 @@ +/* + * BSD LICENSE + * + * Copyright (C) Cavium Inc. 2017. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium networks nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __OCTEONTX_RXTX_H__ +#define __OCTEONTX_RXTX_H__ + +#include + +#ifndef __hot +#define __hot __attribute__((hot)) +#endif + +/* Packet type table */ +#define PTYPE_SIZE OCCTX_PKI_LTYPE_LAST + +static const uint32_t __rte_cache_aligned +ptype_table[PTYPE_SIZE][PTYPE_SIZE][PTYPE_SIZE] = { + [LC_NONE][LE_NONE][LF_NONE] = RTE_PTYPE_UNKNOWN, + [LC_NONE][LE_NONE][LF_IPSEC_ESP] = RTE_PTYPE_UNKNOWN, + [LC_NONE][LE_NONE][LF_IPFRAG] = RTE_PTYPE_L4_FRAG, + [LC_NONE][LE_NONE][LF_IPCOMP] = RTE_PTYPE_UNKNOWN, + [LC_NONE][LE_NONE][LF_TCP] = RTE_PTYPE_L4_TCP, + [LC_NONE][LE_NONE][LF_UDP] = RTE_PTYPE_L4_UDP, + [LC_NONE][LE_NONE][LF_GRE] = RTE_PTYPE_TUNNEL_GRE, + [LC_NONE][LE_NONE][LF_UDP_GENEVE] = RTE_PTYPE_TUNNEL_GENEVE, + [LC_NONE][LE_NONE][LF_UDP_VXLAN] = RTE_PTYPE_TUNNEL_VXLAN, + [LC_NONE][LE_NONE][LF_NVGRE] = RTE_PTYPE_TUNNEL_NVGRE, + + [LC_IPV4][LE_NONE][LF_NONE] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_UNKNOWN, + [LC_IPV4][LE_NONE][LF_IPSEC_ESP] = + RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L3_IPV4, + [LC_IPV4][LE_NONE][LF_IPFRAG] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_FRAG, + [LC_IPV4][LE_NONE][LF_IPCOMP] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_UNKNOWN, + [LC_IPV4][LE_NONE][LF_TCP] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_TCP, + [LC_IPV4][LE_NONE][LF_UDP] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_UDP, + [LC_IPV4][LE_NONE][LF_GRE] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_GRE, + [LC_IPV4][LE_NONE][LF_UDP_GENEVE] = + RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_GENEVE, + [LC_IPV4][LE_NONE][LF_UDP_VXLAN] = + RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_VXLAN, + [LC_IPV4][LE_NONE][LF_NVGRE] = + RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_NVGRE, + + [LC_IPV4_OPT][LE_NONE][LF_NONE] = + RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_UNKNOWN, + [LC_IPV4_OPT][LE_NONE][LF_IPSEC_ESP] = + RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L3_IPV4, + [LC_IPV4_OPT][LE_NONE][LF_IPFRAG] = + RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_FRAG, + [LC_IPV4_OPT][LE_NONE][LF_IPCOMP] = + RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_UNKNOWN, + [LC_IPV4_OPT][LE_NONE][LF_TCP] = + RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_TCP, + [LC_IPV4_OPT][LE_NONE][LF_UDP] = + RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_UDP, + [LC_IPV4_OPT][LE_NONE][LF_GRE] = + RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_GRE, + [LC_IPV4_OPT][LE_NONE][LF_UDP_GENEVE] = + RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_GENEVE, + [LC_IPV4_OPT][LE_NONE][LF_UDP_VXLAN] = + RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_VXLAN, + [LC_IPV4_OPT][LE_NONE][LF_NVGRE] = + RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_NVGRE, + + [LC_IPV6][LE_NONE][LF_NONE] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_UNKNOWN, + [LC_IPV6][LE_NONE][LF_IPSEC_ESP] = + RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L3_IPV4, + [LC_IPV6][LE_NONE][LF_IPFRAG] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_FRAG, + [LC_IPV6][LE_NONE][LF_IPCOMP] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_UNKNOWN, + [LC_IPV6][LE_NONE][LF_TCP] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_TCP, + [LC_IPV6][LE_NONE][LF_UDP] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_UDP, + [LC_IPV6][LE_NONE][LF_GRE] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_TUNNEL_GRE, + [LC_IPV6][LE_NONE][LF_UDP_GENEVE] = + RTE_PTYPE_L3_IPV6 | RTE_PTYPE_TUNNEL_GENEVE, + [LC_IPV6][LE_NONE][LF_UDP_VXLAN] = + RTE_PTYPE_L3_IPV6 | RTE_PTYPE_TUNNEL_VXLAN, + [LC_IPV6][LE_NONE][LF_NVGRE] = + RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_NVGRE, + [LC_IPV6_OPT][LE_NONE][LF_NONE] = + RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_UNKNOWN, + [LC_IPV6_OPT][LE_NONE][LF_IPSEC_ESP] = + RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L3_IPV4, + [LC_IPV6_OPT][LE_NONE][LF_IPFRAG] = + RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_FRAG, + [LC_IPV6_OPT][LE_NONE][LF_IPCOMP] = + RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_UNKNOWN, + [LC_IPV6_OPT][LE_NONE][LF_TCP] = + RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_TCP, + [LC_IPV6_OPT][LE_NONE][LF_UDP] = + RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_UDP, + [LC_IPV6_OPT][LE_NONE][LF_GRE] = + RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_TUNNEL_GRE, + [LC_IPV6_OPT][LE_NONE][LF_UDP_GENEVE] = + RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_TUNNEL_GENEVE, + [LC_IPV6_OPT][LE_NONE][LF_UDP_VXLAN] = + RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_TUNNEL_VXLAN, + [LC_IPV6_OPT][LE_NONE][LF_NVGRE] = + RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_TUNNEL_NVGRE, + +}; + +uint16_t +octeontx_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts); + +uint16_t +octeontx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts); + +#endif /* __OCTEONTX_RXTX_H__ */ diff --git a/dpdk/drivers/net/octeontx/rte_pmd_octeontx_version.map b/dpdk/drivers/net/octeontx/rte_pmd_octeontx_version.map new file mode 100644 index 00000000..40d68c79 --- /dev/null +++ b/dpdk/drivers/net/octeontx/rte_pmd_octeontx_version.map @@ -0,0 +1,7 @@ +DPDK_17.11 { + global: + + rte_octeontx_pchan_map; + + local: *; +}; diff --git a/dpdk/drivers/net/pcap/Makefile b/dpdk/drivers/net/pcap/Makefile index 89ac4024..b6487d42 100644 --- a/dpdk/drivers/net/pcap/Makefile +++ b/dpdk/drivers/net/pcap/Makefile @@ -40,6 +40,9 @@ LIB = librte_pmd_pcap.a CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) LDLIBS += -lpcap +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs +LDLIBS += -lrte_bus_vdev EXPORT_MAP := rte_pmd_pcap_version.map @@ -55,11 +58,4 @@ SRCS-$(CONFIG_RTE_LIBRTE_PMD_PCAP) += rte_eth_pcap.c # SYMLINK-y-include += -# this lib depends upon: -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_PCAP) += lib/librte_eal -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_PCAP) += lib/librte_mbuf -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_PCAP) += lib/librte_mempool -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_PCAP) += lib/librte_ether -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_PCAP) += lib/librte_kvargs - include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/net/pcap/rte_eth_pcap.c b/dpdk/drivers/net/pcap/rte_eth_pcap.c index 7b7126bf..3385d043 100644 --- a/dpdk/drivers/net/pcap/rte_eth_pcap.c +++ b/dpdk/drivers/net/pcap/rte_eth_pcap.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. * Copyright(c) 2014 6WIND S.A. * All rights reserved. * @@ -33,23 +33,24 @@ */ #include -#include -#include -#include -#include -#include -#include -#include -#include #include #include +#include +#include +#include +#include +#include +#include +#include + #define RTE_ETH_PCAP_SNAPSHOT_LEN 65535 #define RTE_ETH_PCAP_SNAPLEN ETHER_MAX_JUMBO_FRAME_LEN #define RTE_ETH_PCAP_PROMISC 1 #define RTE_ETH_PCAP_TIMEOUT -1 + #define ETH_PCAP_RX_PCAP_ARG "rx_pcap" #define ETH_PCAP_TX_PCAP_ARG "tx_pcap" #define ETH_PCAP_RX_IFACE_ARG "rx_iface" @@ -58,19 +59,25 @@ #define ETH_PCAP_ARG_MAXLEN 64 +#define RTE_PMD_PCAP_MAX_QUEUES 16 + static char errbuf[PCAP_ERRBUF_SIZE]; static unsigned char tx_pcap_data[RTE_ETH_PCAP_SNAPLEN]; static struct timeval start_time; static uint64_t start_cycles; static uint64_t hz; +struct queue_stat { + volatile unsigned long pkts; + volatile unsigned long bytes; + volatile unsigned long err_pkts; +}; + struct pcap_rx_queue { pcap_t *pcap; - uint8_t in_port; + uint16_t in_port; struct rte_mempool *mb_pool; - volatile unsigned long rx_pkts; - volatile unsigned long rx_bytes; - volatile unsigned long err_pkts; + struct queue_stat rx_stat; char name[PATH_MAX]; char type[ETH_PCAP_ARG_MAXLEN]; }; @@ -78,36 +85,29 @@ struct pcap_rx_queue { struct pcap_tx_queue { pcap_dumper_t *dumper; pcap_t *pcap; - volatile unsigned long tx_pkts; - volatile unsigned long tx_bytes; - volatile unsigned long err_pkts; + struct queue_stat tx_stat; char name[PATH_MAX]; char type[ETH_PCAP_ARG_MAXLEN]; }; -struct rx_pcaps { - unsigned num_of_rx; - pcap_t *pcaps[RTE_PMD_RING_MAX_RX_RINGS]; - const char *names[RTE_PMD_RING_MAX_RX_RINGS]; - const char *types[RTE_PMD_RING_MAX_RX_RINGS]; -}; - -struct tx_pcaps { - unsigned num_of_tx; - pcap_dumper_t *dumpers[RTE_PMD_RING_MAX_TX_RINGS]; - pcap_t *pcaps[RTE_PMD_RING_MAX_RX_RINGS]; - const char *names[RTE_PMD_RING_MAX_RX_RINGS]; - const char *types[RTE_PMD_RING_MAX_RX_RINGS]; -}; - struct pmd_internals { - struct pcap_rx_queue rx_queue[RTE_PMD_RING_MAX_RX_RINGS]; - struct pcap_tx_queue tx_queue[RTE_PMD_RING_MAX_TX_RINGS]; + struct pcap_rx_queue rx_queue[RTE_PMD_PCAP_MAX_QUEUES]; + struct pcap_tx_queue tx_queue[RTE_PMD_PCAP_MAX_QUEUES]; int if_index; int single_iface; }; -const char *valid_arguments[] = { +struct pmd_devargs { + unsigned int num_of_queue; + struct devargs_queue { + pcap_dumper_t *dumper; + pcap_t *pcap; + const char *name; + const char *type; + } queue[RTE_PMD_PCAP_MAX_QUEUES]; +}; + +static const char *valid_arguments[] = { ETH_PCAP_RX_PCAP_ARG, ETH_PCAP_TX_PCAP_ARG, ETH_PCAP_RX_IFACE_ARG, @@ -116,29 +116,24 @@ const char *valid_arguments[] = { NULL }; -static int open_single_tx_pcap(const char *pcap_filename, pcap_dumper_t **dumper); -static int open_single_rx_pcap(const char *pcap_filename, pcap_t **pcap); -static int open_single_iface(const char *iface, pcap_t **pcap); +static struct ether_addr eth_addr = { + .addr_bytes = { 0, 0, 0, 0x1, 0x2, 0x3 } +}; -static struct ether_addr eth_addr = { .addr_bytes = { 0, 0, 0, 0x1, 0x2, 0x3 } }; -static const char *drivername = "Pcap PMD"; static struct rte_eth_link pmd_link = { .link_speed = ETH_SPEED_NUM_10G, .link_duplex = ETH_LINK_FULL_DUPLEX, .link_status = ETH_LINK_DOWN, - .link_autoneg = ETH_LINK_SPEED_FIXED, + .link_autoneg = ETH_LINK_AUTONEG, }; static int -eth_pcap_rx_jumbo(struct rte_mempool *mb_pool, - struct rte_mbuf *mbuf, - const u_char *data, - uint16_t data_len) +eth_pcap_rx_jumbo(struct rte_mempool *mb_pool, struct rte_mbuf *mbuf, + const u_char *data, uint16_t data_len) { - struct rte_mbuf *m = mbuf; - /* Copy the first segment. */ uint16_t len = rte_pktmbuf_tailroom(mbuf); + struct rte_mbuf *m = mbuf; rte_memcpy(rte_pktmbuf_append(mbuf, len), data, len); data_len -= len; @@ -178,7 +173,7 @@ eth_pcap_gather_data(unsigned char *data, struct rte_mbuf *mbuf) while (mbuf) { rte_memcpy(data + data_len, rte_pktmbuf_mtod(mbuf, void *), - mbuf->data_len); + mbuf->data_len); data_len += mbuf->data_len; mbuf = mbuf->next; @@ -186,11 +181,9 @@ eth_pcap_gather_data(unsigned char *data, struct rte_mbuf *mbuf) } static uint16_t -eth_pcap_rx(void *queue, - struct rte_mbuf **bufs, - uint16_t nb_pkts) +eth_pcap_rx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) { - unsigned i; + unsigned int i; struct pcap_pkthdr header; const u_char *packet; struct rte_mbuf *mbuf; @@ -210,17 +203,17 @@ eth_pcap_rx(void *queue, packet = pcap_next(pcap_q->pcap, &header); if (unlikely(packet == NULL)) break; - else - mbuf = rte_pktmbuf_alloc(pcap_q->mb_pool); + + mbuf = rte_pktmbuf_alloc(pcap_q->mb_pool); if (unlikely(mbuf == NULL)) break; /* Now get the space available for data in the mbuf */ - buf_size = (uint16_t)(rte_pktmbuf_data_room_size(pcap_q->mb_pool) - - RTE_PKTMBUF_HEADROOM); + buf_size = rte_pktmbuf_data_room_size(pcap_q->mb_pool) - + RTE_PKTMBUF_HEADROOM; if (header.caplen <= buf_size) { - /* pcap packet will fit in the mbuf, go ahead and copy */ + /* pcap packet will fit in the mbuf, can copy it */ rte_memcpy(rte_pktmbuf_mtod(mbuf, void *), packet, header.caplen); mbuf->data_len = (uint16_t)header.caplen; @@ -241,8 +234,9 @@ eth_pcap_rx(void *queue, num_rx++; rx_bytes += header.caplen; } - pcap_q->rx_pkts += num_rx; - pcap_q->rx_bytes += rx_bytes; + pcap_q->rx_stat.pkts += num_rx; + pcap_q->rx_stat.bytes += rx_bytes; + return num_rx; } @@ -253,7 +247,7 @@ calculate_timestamp(struct timeval *ts) { cycles = rte_get_timer_cycles() - start_cycles; cur_time.tv_sec = cycles / hz; - cur_time.tv_usec = (cycles % hz) * 10e6 / hz; + cur_time.tv_usec = (cycles % hz) * 1e6 / hz; timeradd(&start_time, &cur_time, ts); } @@ -261,11 +255,9 @@ calculate_timestamp(struct timeval *ts) { * Callback to handle writing packets to a pcap file. */ static uint16_t -eth_pcap_tx_dumper(void *queue, - struct rte_mbuf **bufs, - uint16_t nb_pkts) +eth_pcap_tx_dumper(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) { - unsigned i; + unsigned int i; struct rte_mbuf *mbuf; struct pcap_tx_queue *dumper_q = queue; uint16_t num_tx = 0; @@ -275,7 +267,8 @@ eth_pcap_tx_dumper(void *queue, if (dumper_q->dumper == NULL || nb_pkts == 0) return 0; - /* writes the nb_pkts packets to the previously opened pcap file dumper */ + /* writes the nb_pkts packets to the previously opened pcap file + * dumper */ for (i = 0; i < nb_pkts; i++) { mbuf = bufs[i]; calculate_timestamp(&header.ts); @@ -292,8 +285,7 @@ eth_pcap_tx_dumper(void *queue, tx_pcap_data); } else { RTE_LOG(ERR, PMD, - "Dropping PCAP packet. " - "Size (%d) > max jumbo size (%d).\n", + "Dropping PCAP packet. Size (%d) > max jumbo size (%d).\n", mbuf->pkt_len, ETHER_MAX_JUMBO_FRAME_LEN); @@ -302,9 +294,9 @@ eth_pcap_tx_dumper(void *queue, } } - rte_pktmbuf_free(mbuf); num_tx++; tx_bytes += mbuf->pkt_len; + rte_pktmbuf_free(mbuf); } /* @@ -313,9 +305,10 @@ eth_pcap_tx_dumper(void *queue, * we flush the pcap dumper within each burst. */ pcap_dump_flush(dumper_q->dumper); - dumper_q->tx_pkts += num_tx; - dumper_q->tx_bytes += tx_bytes; - dumper_q->err_pkts += nb_pkts - num_tx; + dumper_q->tx_stat.pkts += num_tx; + dumper_q->tx_stat.bytes += tx_bytes; + dumper_q->tx_stat.err_pkts += nb_pkts - num_tx; + return num_tx; } @@ -323,11 +316,9 @@ eth_pcap_tx_dumper(void *queue, * Callback to handle sending packets through a real NIC. */ static uint16_t -eth_pcap_tx(void *queue, - struct rte_mbuf **bufs, - uint16_t nb_pkts) +eth_pcap_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) { - unsigned i; + unsigned int i; int ret; struct rte_mbuf *mbuf; struct pcap_tx_queue *tx_queue = queue; @@ -342,18 +333,16 @@ eth_pcap_tx(void *queue, if (likely(mbuf->nb_segs == 1)) { ret = pcap_sendpacket(tx_queue->pcap, - rte_pktmbuf_mtod(mbuf, u_char *), - mbuf->pkt_len); + rte_pktmbuf_mtod(mbuf, u_char *), + mbuf->pkt_len); } else { if (mbuf->pkt_len <= ETHER_MAX_JUMBO_FRAME_LEN) { eth_pcap_gather_data(tx_pcap_data, mbuf); ret = pcap_sendpacket(tx_queue->pcap, - tx_pcap_data, - mbuf->pkt_len); + tx_pcap_data, mbuf->pkt_len); } else { RTE_LOG(ERR, PMD, - "Dropping PCAP packet. " - "Size (%d) > max jumbo size (%d).\n", + "Dropping PCAP packet. Size (%d) > max jumbo size (%d).\n", mbuf->pkt_len, ETHER_MAX_JUMBO_FRAME_LEN); @@ -369,16 +358,86 @@ eth_pcap_tx(void *queue, rte_pktmbuf_free(mbuf); } - tx_queue->tx_pkts += num_tx; - tx_queue->tx_bytes += tx_bytes; - tx_queue->err_pkts += nb_pkts - num_tx; + tx_queue->tx_stat.pkts += num_tx; + tx_queue->tx_stat.bytes += tx_bytes; + tx_queue->tx_stat.err_pkts += nb_pkts - num_tx; + return num_tx; } +/* + * pcap_open_live wrapper function + */ +static inline int +open_iface_live(const char *iface, pcap_t **pcap) { + *pcap = pcap_open_live(iface, RTE_ETH_PCAP_SNAPLEN, + RTE_ETH_PCAP_PROMISC, RTE_ETH_PCAP_TIMEOUT, errbuf); + + if (*pcap == NULL) { + RTE_LOG(ERR, PMD, "Couldn't open %s: %s\n", iface, errbuf); + return -1; + } + + return 0; +} + +static int +open_single_iface(const char *iface, pcap_t **pcap) +{ + if (open_iface_live(iface, pcap) < 0) { + RTE_LOG(ERR, PMD, "Couldn't open interface %s\n", iface); + return -1; + } + + return 0; +} + +static int +open_single_tx_pcap(const char *pcap_filename, pcap_dumper_t **dumper) +{ + pcap_t *tx_pcap; + + /* + * We need to create a dummy empty pcap_t to use it + * with pcap_dump_open(). We create big enough an Ethernet + * pcap holder. + */ + tx_pcap = pcap_open_dead(DLT_EN10MB, RTE_ETH_PCAP_SNAPSHOT_LEN); + if (tx_pcap == NULL) { + RTE_LOG(ERR, PMD, "Couldn't create dead pcap\n"); + return -1; + } + + /* The dumper is created using the previous pcap_t reference */ + *dumper = pcap_dump_open(tx_pcap, pcap_filename); + if (*dumper == NULL) { + pcap_close(tx_pcap); + RTE_LOG(ERR, PMD, "Couldn't open %s for writing.\n", + pcap_filename); + return -1; + } + + pcap_close(tx_pcap); + return 0; +} + +static int +open_single_rx_pcap(const char *pcap_filename, pcap_t **pcap) +{ + *pcap = pcap_open_offline(pcap_filename, errbuf); + if (*pcap == NULL) { + RTE_LOG(ERR, PMD, "Couldn't open %s: %s\n", pcap_filename, + errbuf); + return -1; + } + + return 0; +} + static int eth_dev_start(struct rte_eth_dev *dev) { - unsigned i; + unsigned int i; struct pmd_internals *internals = dev->data->dev_private; struct pcap_tx_queue *tx; struct pcap_rx_queue *rx; @@ -400,12 +459,12 @@ eth_dev_start(struct rte_eth_dev *dev) for (i = 0; i < dev->data->nb_tx_queues; i++) { tx = &internals->tx_queue[i]; - if (!tx->dumper && strcmp(tx->type, ETH_PCAP_TX_PCAP_ARG) == 0) { + if (!tx->dumper && + strcmp(tx->type, ETH_PCAP_TX_PCAP_ARG) == 0) { if (open_single_tx_pcap(tx->name, &tx->dumper) < 0) return -1; - } - - else if (!tx->pcap && strcmp(tx->type, ETH_PCAP_TX_IFACE_ARG) == 0) { + } else if (!tx->pcap && + strcmp(tx->type, ETH_PCAP_TX_IFACE_ARG) == 0) { if (open_single_iface(tx->name, &tx->pcap) < 0) return -1; } @@ -421,17 +480,15 @@ eth_dev_start(struct rte_eth_dev *dev) if (strcmp(rx->type, ETH_PCAP_RX_PCAP_ARG) == 0) { if (open_single_rx_pcap(rx->name, &rx->pcap) < 0) return -1; - } - - else if (strcmp(rx->type, ETH_PCAP_RX_IFACE_ARG) == 0) { + } else if (strcmp(rx->type, ETH_PCAP_RX_IFACE_ARG) == 0) { if (open_single_iface(rx->name, &rx->pcap) < 0) return -1; } } status_up: - dev->data->dev_link.link_status = ETH_LINK_UP; + return 0; } @@ -443,7 +500,7 @@ status_up: static void eth_dev_stop(struct rte_eth_dev *dev) { - unsigned i; + unsigned int i; struct pmd_internals *internals = dev->data->dev_private; struct pcap_tx_queue *tx; struct pcap_rx_queue *rx; @@ -496,21 +553,19 @@ eth_dev_info(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) { struct pmd_internals *internals = dev->data->dev_private; - dev_info->driver_name = drivername; + dev_info->if_index = internals->if_index; dev_info->max_mac_addrs = 1; dev_info->max_rx_pktlen = (uint32_t) -1; dev_info->max_rx_queues = dev->data->nb_rx_queues; dev_info->max_tx_queues = dev->data->nb_tx_queues; dev_info->min_rx_bufsize = 0; - dev_info->pci_dev = NULL; } -static void -eth_stats_get(struct rte_eth_dev *dev, - struct rte_eth_stats *igb_stats) +static int +eth_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) { - unsigned i; + unsigned int i; unsigned long rx_packets_total = 0, rx_bytes_total = 0; unsigned long tx_packets_total = 0, tx_bytes_total = 0; unsigned long tx_packets_err_total = 0; @@ -518,42 +573,46 @@ eth_stats_get(struct rte_eth_dev *dev, for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS && i < dev->data->nb_rx_queues; i++) { - igb_stats->q_ipackets[i] = internal->rx_queue[i].rx_pkts; - igb_stats->q_ibytes[i] = internal->rx_queue[i].rx_bytes; - rx_packets_total += igb_stats->q_ipackets[i]; - rx_bytes_total += igb_stats->q_ibytes[i]; + stats->q_ipackets[i] = internal->rx_queue[i].rx_stat.pkts; + stats->q_ibytes[i] = internal->rx_queue[i].rx_stat.bytes; + rx_packets_total += stats->q_ipackets[i]; + rx_bytes_total += stats->q_ibytes[i]; } for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS && i < dev->data->nb_tx_queues; i++) { - igb_stats->q_opackets[i] = internal->tx_queue[i].tx_pkts; - igb_stats->q_obytes[i] = internal->tx_queue[i].tx_bytes; - igb_stats->q_errors[i] = internal->tx_queue[i].err_pkts; - tx_packets_total += igb_stats->q_opackets[i]; - tx_bytes_total += igb_stats->q_obytes[i]; - tx_packets_err_total += igb_stats->q_errors[i]; + stats->q_opackets[i] = internal->tx_queue[i].tx_stat.pkts; + stats->q_obytes[i] = internal->tx_queue[i].tx_stat.bytes; + stats->q_errors[i] = internal->tx_queue[i].tx_stat.err_pkts; + tx_packets_total += stats->q_opackets[i]; + tx_bytes_total += stats->q_obytes[i]; + tx_packets_err_total += stats->q_errors[i]; } - igb_stats->ipackets = rx_packets_total; - igb_stats->ibytes = rx_bytes_total; - igb_stats->opackets = tx_packets_total; - igb_stats->obytes = tx_bytes_total; - igb_stats->oerrors = tx_packets_err_total; + stats->ipackets = rx_packets_total; + stats->ibytes = rx_bytes_total; + stats->opackets = tx_packets_total; + stats->obytes = tx_bytes_total; + stats->oerrors = tx_packets_err_total; + + return 0; } static void eth_stats_reset(struct rte_eth_dev *dev) { - unsigned i; + unsigned int i; struct pmd_internals *internal = dev->data->dev_private; + for (i = 0; i < dev->data->nb_rx_queues; i++) { - internal->rx_queue[i].rx_pkts = 0; - internal->rx_queue[i].rx_bytes = 0; + internal->rx_queue[i].rx_stat.pkts = 0; + internal->rx_queue[i].rx_stat.bytes = 0; } + for (i = 0; i < dev->data->nb_tx_queues; i++) { - internal->tx_queue[i].tx_pkts = 0; - internal->tx_queue[i].tx_bytes = 0; - internal->tx_queue[i].err_pkts = 0; + internal->tx_queue[i].tx_stat.pkts = 0; + internal->tx_queue[i].tx_stat.bytes = 0; + internal->tx_queue[i].tx_stat.err_pkts = 0; } } @@ -584,9 +643,11 @@ eth_rx_queue_setup(struct rte_eth_dev *dev, { struct pmd_internals *internals = dev->data->dev_private; struct pcap_rx_queue *pcap_q = &internals->rx_queue[rx_queue_id]; + pcap_q->mb_pool = mb_pool; dev->data->rx_queues[rx_queue_id] = pcap_q; pcap_q->in_port = dev->data->port_id; + return 0; } @@ -597,15 +658,16 @@ eth_tx_queue_setup(struct rte_eth_dev *dev, unsigned int socket_id __rte_unused, const struct rte_eth_txconf *tx_conf __rte_unused) { - struct pmd_internals *internals = dev->data->dev_private; + dev->data->tx_queues[tx_queue_id] = &internals->tx_queue[tx_queue_id]; + return 0; } static const struct eth_dev_ops ops = { .dev_start = eth_dev_start, - .dev_stop = eth_dev_stop, + .dev_stop = eth_dev_stop, .dev_close = eth_dev_close, .dev_configure = eth_dev_configure, .dev_infos_get = eth_dev_info, @@ -625,33 +687,23 @@ static const struct eth_dev_ops ops = { static int open_rx_pcap(const char *key, const char *value, void *extra_args) { - unsigned i; + unsigned int i; const char *pcap_filename = value; - struct rx_pcaps *pcaps = extra_args; + struct pmd_devargs *rx = extra_args; pcap_t *pcap = NULL; - for (i = 0; i < pcaps->num_of_rx; i++) { + for (i = 0; i < rx->num_of_queue; i++) { if (open_single_rx_pcap(pcap_filename, &pcap) < 0) return -1; - pcaps->pcaps[i] = pcap; - pcaps->names[i] = pcap_filename; - pcaps->types[i] = key; + rx->queue[i].pcap = pcap; + rx->queue[i].name = pcap_filename; + rx->queue[i].type = key; } return 0; } -static int -open_single_rx_pcap(const char *pcap_filename, pcap_t **pcap) -{ - if ((*pcap = pcap_open_offline(pcap_filename, errbuf)) == NULL) { - RTE_LOG(ERR, PMD, "Couldn't open %s: %s\n", pcap_filename, errbuf); - return -1; - } - return 0; -} - /* * Opens a pcap file for writing and stores a reference to it * for use it later on. @@ -659,63 +711,23 @@ open_single_rx_pcap(const char *pcap_filename, pcap_t **pcap) static int open_tx_pcap(const char *key, const char *value, void *extra_args) { - unsigned i; + unsigned int i; const char *pcap_filename = value; - struct tx_pcaps *dumpers = extra_args; + struct pmd_devargs *dumpers = extra_args; pcap_dumper_t *dumper; - for (i = 0; i < dumpers->num_of_tx; i++) { + for (i = 0; i < dumpers->num_of_queue; i++) { if (open_single_tx_pcap(pcap_filename, &dumper) < 0) return -1; - dumpers->dumpers[i] = dumper; - dumpers->names[i] = pcap_filename; - dumpers->types[i] = key; + dumpers->queue[i].dumper = dumper; + dumpers->queue[i].name = pcap_filename; + dumpers->queue[i].type = key; } return 0; } -static int -open_single_tx_pcap(const char *pcap_filename, pcap_dumper_t **dumper) -{ - pcap_t *tx_pcap; - /* - * We need to create a dummy empty pcap_t to use it - * with pcap_dump_open(). We create big enough an Ethernet - * pcap holder. - */ - - if ((tx_pcap = pcap_open_dead(DLT_EN10MB, RTE_ETH_PCAP_SNAPSHOT_LEN)) - == NULL) { - RTE_LOG(ERR, PMD, "Couldn't create dead pcap\n"); - return -1; - } - - /* The dumper is created using the previous pcap_t reference */ - if ((*dumper = pcap_dump_open(tx_pcap, pcap_filename)) == NULL) { - RTE_LOG(ERR, PMD, "Couldn't open %s for writing.\n", pcap_filename); - return -1; - } - - return 0; -} - -/* - * pcap_open_live wrapper function - */ -static inline int -open_iface_live(const char *iface, pcap_t **pcap) { - *pcap = pcap_open_live(iface, RTE_ETH_PCAP_SNAPLEN, - RTE_ETH_PCAP_PROMISC, RTE_ETH_PCAP_TIMEOUT, errbuf); - - if (*pcap == NULL) { - RTE_LOG(ERR, PMD, "Couldn't open %s: %s\n", iface, errbuf); - return -1; - } - return 0; -} - /* * Opens an interface for reading and writing */ @@ -723,15 +735,15 @@ static inline int open_rx_tx_iface(const char *key, const char *value, void *extra_args) { const char *iface = value; - struct rx_pcaps *pcaps = extra_args; + struct pmd_devargs *tx = extra_args; pcap_t *pcap = NULL; if (open_single_iface(iface, &pcap) < 0) return -1; - pcaps->pcaps[0] = pcap; - pcaps->names[0] = iface; - pcaps->types[0] = key; + tx->queue[0].pcap = pcap; + tx->queue[0].name = iface; + tx->queue[0].type = key; return 0; } @@ -742,17 +754,17 @@ open_rx_tx_iface(const char *key, const char *value, void *extra_args) static inline int open_rx_iface(const char *key, const char *value, void *extra_args) { - unsigned i; + unsigned int i; const char *iface = value; - struct rx_pcaps *pcaps = extra_args; + struct pmd_devargs *rx = extra_args; pcap_t *pcap = NULL; - for (i = 0; i < pcaps->num_of_rx; i++) { + for (i = 0; i < rx->num_of_queue; i++) { if (open_single_iface(iface, &pcap) < 0) return -1; - pcaps->pcaps[i] = pcap; - pcaps->names[i] = iface; - pcaps->types[i] = key; + rx->queue[i].pcap = pcap; + rx->queue[i].name = iface; + rx->queue[i].type = key; } return 0; @@ -764,73 +776,52 @@ open_rx_iface(const char *key, const char *value, void *extra_args) static int open_tx_iface(const char *key, const char *value, void *extra_args) { - unsigned i; + unsigned int i; const char *iface = value; - struct tx_pcaps *pcaps = extra_args; + struct pmd_devargs *tx = extra_args; pcap_t *pcap; - for (i = 0; i < pcaps->num_of_tx; i++) { + for (i = 0; i < tx->num_of_queue; i++) { if (open_single_iface(iface, &pcap) < 0) return -1; - pcaps->pcaps[i] = pcap; - pcaps->names[i] = iface; - pcaps->types[i] = key; + tx->queue[i].pcap = pcap; + tx->queue[i].name = iface; + tx->queue[i].type = key; } return 0; } -static int -open_single_iface(const char *iface, pcap_t **pcap) -{ - if (open_iface_live(iface, pcap) < 0) { - RTE_LOG(ERR, PMD, "Couldn't open interface %s\n", iface); - return -1; - } - - return 0; -} +static struct rte_vdev_driver pmd_pcap_drv; static int -rte_pmd_init_internals(const char *name, const unsigned nb_rx_queues, - const unsigned nb_tx_queues, - const unsigned numa_node, +pmd_init_internals(struct rte_vdev_device *vdev, + const unsigned int nb_rx_queues, + const unsigned int nb_tx_queues, struct pmd_internals **internals, - struct rte_eth_dev **eth_dev, - struct rte_kvargs *kvlist) + struct rte_eth_dev **eth_dev) { struct rte_eth_dev_data *data = NULL; - unsigned k_idx; - struct rte_kvargs_pair *pair = NULL; + unsigned int numa_node = vdev->device.numa_node; + const char *name; - for (k_idx = 0; k_idx < kvlist->count; k_idx++) { - pair = &kvlist->pairs[k_idx]; - if (strstr(pair->key, ETH_PCAP_IFACE_ARG) != NULL) - break; - } - - RTE_LOG(INFO, PMD, - "Creating pcap-backed ethdev on numa socket %u\n", numa_node); + name = rte_vdev_device_name(vdev); + RTE_LOG(INFO, PMD, "Creating pcap-backed ethdev on numa socket %d\n", + numa_node); /* now do all data allocation - for eth_dev structure * and internal (private) data */ data = rte_zmalloc_socket(name, sizeof(*data), 0, numa_node); if (data == NULL) - goto error; - - *internals = rte_zmalloc_socket(name, sizeof(**internals), 0, numa_node); - if (*internals == NULL) - goto error; + return -1; /* reserve an ethdev entry */ - *eth_dev = rte_eth_dev_allocate(name, RTE_ETH_DEV_VIRTUAL); - if (*eth_dev == NULL) - goto error; - - /* check length of device name */ - if ((strlen((*eth_dev)->data->name) + 1) > sizeof(data->name)) - goto error; + *eth_dev = rte_eth_vdev_allocate(vdev, sizeof(**internals)); + if (*eth_dev == NULL) { + rte_free(data); + return -1; + } /* now put it all together * - store queue data in internals, @@ -838,49 +829,33 @@ rte_pmd_init_internals(const char *name, const unsigned nb_rx_queues, * - point eth_dev_data to internals * - and point eth_dev structure to new eth_dev_data structure */ - /* NOTE: we'll replace the data element, of originally allocated eth_dev - * so the rings are local per-process */ - - if (pair == NULL) - (*internals)->if_index = 0; - else - (*internals)->if_index = if_nametoindex(pair->value); - - data->dev_private = *internals; - data->port_id = (*eth_dev)->data->port_id; - snprintf(data->name, sizeof(data->name), "%s", (*eth_dev)->data->name); + *internals = (*eth_dev)->data->dev_private; + rte_memcpy(data, (*eth_dev)->data, sizeof(*data)); data->nb_rx_queues = (uint16_t)nb_rx_queues; data->nb_tx_queues = (uint16_t)nb_tx_queues; data->dev_link = pmd_link; data->mac_addrs = ð_addr; - strncpy(data->name, - (*eth_dev)->data->name, strlen((*eth_dev)->data->name)); + /* + * NOTE: we'll replace the data element, of originally allocated + * eth_dev so the rings are local per-process + */ (*eth_dev)->data = data; (*eth_dev)->dev_ops = &ops; - (*eth_dev)->driver = NULL; - data->dev_flags = RTE_ETH_DEV_DETACHABLE; - data->kdrv = RTE_KDRV_NONE; - data->drv_name = drivername; - data->numa_node = numa_node; return 0; - -error: - rte_free(data); - rte_free(*internals); - - return -1; } static int -rte_eth_from_pcaps_common(const char *name, struct rx_pcaps *rx_queues, - const unsigned nb_rx_queues, struct tx_pcaps *tx_queues, - const unsigned nb_tx_queues, const unsigned numa_node, +eth_from_pcaps_common(struct rte_vdev_device *vdev, + struct pmd_devargs *rx_queues, const unsigned int nb_rx_queues, + struct pmd_devargs *tx_queues, const unsigned int nb_tx_queues, struct rte_kvargs *kvlist, struct pmd_internals **internals, struct rte_eth_dev **eth_dev) { - unsigned i; + struct rte_kvargs_pair *pair = NULL; + unsigned int k_idx; + unsigned int i; /* do some parameter checking */ if (rx_queues == NULL && nb_rx_queues > 0) @@ -888,110 +863,92 @@ rte_eth_from_pcaps_common(const char *name, struct rx_pcaps *rx_queues, if (tx_queues == NULL && nb_tx_queues > 0) return -1; - if (rte_pmd_init_internals(name, nb_rx_queues, nb_tx_queues, numa_node, - internals, eth_dev, kvlist) < 0) + if (pmd_init_internals(vdev, nb_rx_queues, nb_tx_queues, internals, + eth_dev) < 0) return -1; for (i = 0; i < nb_rx_queues; i++) { - (*internals)->rx_queue[i].pcap = rx_queues->pcaps[i]; - snprintf((*internals)->rx_queue[i].name, - sizeof((*internals)->rx_queue[i].name), "%s", - rx_queues->names[i]); - snprintf((*internals)->rx_queue[i].type, - sizeof((*internals)->rx_queue[i].type), "%s", - rx_queues->types[i]); + struct pcap_rx_queue *rx = &(*internals)->rx_queue[i]; + struct devargs_queue *queue = &rx_queues->queue[i]; + + rx->pcap = queue->pcap; + snprintf(rx->name, sizeof(rx->name), "%s", queue->name); + snprintf(rx->type, sizeof(rx->type), "%s", queue->type); } + for (i = 0; i < nb_tx_queues; i++) { - (*internals)->tx_queue[i].dumper = tx_queues->dumpers[i]; - snprintf((*internals)->tx_queue[i].name, - sizeof((*internals)->tx_queue[i].name), "%s", - tx_queues->names[i]); - snprintf((*internals)->tx_queue[i].type, - sizeof((*internals)->tx_queue[i].type), "%s", - tx_queues->types[i]); + struct pcap_tx_queue *tx = &(*internals)->tx_queue[i]; + struct devargs_queue *queue = &tx_queues->queue[i]; + + tx->dumper = queue->dumper; + tx->pcap = queue->pcap; + snprintf(tx->name, sizeof(tx->name), "%s", queue->name); + snprintf(tx->type, sizeof(tx->type), "%s", queue->type); } - return 0; -} + for (k_idx = 0; k_idx < kvlist->count; k_idx++) { + pair = &kvlist->pairs[k_idx]; + if (strstr(pair->key, ETH_PCAP_IFACE_ARG) != NULL) + break; + } -static int -rte_eth_from_pcaps_n_dumpers(const char *name, - struct rx_pcaps *rx_queues, - const unsigned nb_rx_queues, - struct tx_pcaps *tx_queues, - const unsigned nb_tx_queues, - const unsigned numa_node, - struct rte_kvargs *kvlist) -{ - struct pmd_internals *internals = NULL; - struct rte_eth_dev *eth_dev = NULL; - int ret; - - ret = rte_eth_from_pcaps_common(name, rx_queues, nb_rx_queues, - tx_queues, nb_tx_queues, numa_node, kvlist, - &internals, ð_dev); - - if (ret < 0) - return ret; - - /* using multiple pcaps/interfaces */ - internals->single_iface = 0; - - eth_dev->rx_pkt_burst = eth_pcap_rx; - eth_dev->tx_pkt_burst = eth_pcap_tx_dumper; + if (pair == NULL) + (*internals)->if_index = 0; + else + (*internals)->if_index = if_nametoindex(pair->value); return 0; } static int -rte_eth_from_pcaps(const char *name, - struct rx_pcaps *rx_queues, - const unsigned nb_rx_queues, - struct tx_pcaps *tx_queues, - const unsigned nb_tx_queues, - const unsigned numa_node, - struct rte_kvargs *kvlist, - int single_iface) +eth_from_pcaps(struct rte_vdev_device *vdev, + struct pmd_devargs *rx_queues, const unsigned int nb_rx_queues, + struct pmd_devargs *tx_queues, const unsigned int nb_tx_queues, + struct rte_kvargs *kvlist, int single_iface, + unsigned int using_dumpers) { struct pmd_internals *internals = NULL; struct rte_eth_dev *eth_dev = NULL; int ret; - ret = rte_eth_from_pcaps_common(name, rx_queues, nb_rx_queues, - tx_queues, nb_tx_queues, numa_node, kvlist, - &internals, ð_dev); + ret = eth_from_pcaps_common(vdev, rx_queues, nb_rx_queues, + tx_queues, nb_tx_queues, kvlist, &internals, ð_dev); if (ret < 0) return ret; - /* store wether we are using a single interface for rx/tx or not */ + /* store weather we are using a single interface for rx/tx or not */ internals->single_iface = single_iface; eth_dev->rx_pkt_burst = eth_pcap_rx; - eth_dev->tx_pkt_burst = eth_pcap_tx; + + if (using_dumpers) + eth_dev->tx_pkt_burst = eth_pcap_tx_dumper; + else + eth_dev->tx_pkt_burst = eth_pcap_tx; return 0; } - static int -rte_pmd_pcap_devinit(const char *name, const char *params) +pmd_pcap_probe(struct rte_vdev_device *dev) { - unsigned numa_node, using_dumpers = 0; - int ret; + const char *name; + unsigned int is_rx_pcap = 0, is_tx_pcap = 0; struct rte_kvargs *kvlist; - struct rx_pcaps pcaps = {0}; - struct tx_pcaps dumpers = {0}; + struct pmd_devargs pcaps = {0}; + struct pmd_devargs dumpers = {0}; + int single_iface = 0; + int ret; + name = rte_vdev_device_name(dev); RTE_LOG(INFO, PMD, "Initializing pmd_pcap for %s\n", name); - numa_node = rte_socket_id(); - gettimeofday(&start_time, NULL); start_cycles = rte_get_timer_cycles(); hz = rte_get_timer_hz(); - kvlist = rte_kvargs_parse(params, valid_arguments); + kvlist = rte_kvargs_parse(rte_vdev_device_args(dev), valid_arguments); if (kvlist == NULL) return -1; @@ -1003,29 +960,39 @@ rte_pmd_pcap_devinit(const char *name, const char *params) ret = rte_kvargs_process(kvlist, ETH_PCAP_IFACE_ARG, &open_rx_tx_iface, &pcaps); + if (ret < 0) goto free_kvlist; - dumpers.pcaps[0] = pcaps.pcaps[0]; - dumpers.names[0] = pcaps.names[0]; - dumpers.types[0] = pcaps.types[0]; - ret = rte_eth_from_pcaps(name, &pcaps, 1, &dumpers, 1, - numa_node, kvlist, 1); - goto free_kvlist; + + dumpers.queue[0] = pcaps.queue[0]; + + single_iface = 1; + pcaps.num_of_queue = 1; + dumpers.num_of_queue = 1; + + goto create_eth; } /* * We check whether we want to open a RX stream from a real NIC or a * pcap file */ - if ((pcaps.num_of_rx = rte_kvargs_count(kvlist, ETH_PCAP_RX_PCAP_ARG))) { + pcaps.num_of_queue = rte_kvargs_count(kvlist, ETH_PCAP_RX_PCAP_ARG); + if (pcaps.num_of_queue) + is_rx_pcap = 1; + else + pcaps.num_of_queue = rte_kvargs_count(kvlist, + ETH_PCAP_RX_IFACE_ARG); + + if (pcaps.num_of_queue > RTE_PMD_PCAP_MAX_QUEUES) + pcaps.num_of_queue = RTE_PMD_PCAP_MAX_QUEUES; + + if (is_rx_pcap) ret = rte_kvargs_process(kvlist, ETH_PCAP_RX_PCAP_ARG, &open_rx_pcap, &pcaps); - } else { - pcaps.num_of_rx = rte_kvargs_count(kvlist, - ETH_PCAP_RX_IFACE_ARG); + else ret = rte_kvargs_process(kvlist, ETH_PCAP_RX_IFACE_ARG, &open_rx_iface, &pcaps); - } if (ret < 0) goto free_kvlist; @@ -1034,46 +1001,49 @@ rte_pmd_pcap_devinit(const char *name, const char *params) * We check whether we want to open a TX stream to a real NIC or a * pcap file */ - if ((dumpers.num_of_tx = rte_kvargs_count(kvlist, - ETH_PCAP_TX_PCAP_ARG))) { + dumpers.num_of_queue = rte_kvargs_count(kvlist, ETH_PCAP_TX_PCAP_ARG); + if (dumpers.num_of_queue) + is_tx_pcap = 1; + else + dumpers.num_of_queue = rte_kvargs_count(kvlist, + ETH_PCAP_TX_IFACE_ARG); + + if (dumpers.num_of_queue > RTE_PMD_PCAP_MAX_QUEUES) + dumpers.num_of_queue = RTE_PMD_PCAP_MAX_QUEUES; + + if (is_tx_pcap) ret = rte_kvargs_process(kvlist, ETH_PCAP_TX_PCAP_ARG, &open_tx_pcap, &dumpers); - using_dumpers = 1; - } else { - dumpers.num_of_tx = rte_kvargs_count(kvlist, - ETH_PCAP_TX_IFACE_ARG); + else ret = rte_kvargs_process(kvlist, ETH_PCAP_TX_IFACE_ARG, &open_tx_iface, &dumpers); - } if (ret < 0) goto free_kvlist; - if (using_dumpers) - ret = rte_eth_from_pcaps_n_dumpers(name, &pcaps, pcaps.num_of_rx, - &dumpers, dumpers.num_of_tx, numa_node, kvlist); - else - ret = rte_eth_from_pcaps(name, &pcaps, pcaps.num_of_rx, &dumpers, - dumpers.num_of_tx, numa_node, kvlist, 0); +create_eth: + ret = eth_from_pcaps(dev, &pcaps, pcaps.num_of_queue, &dumpers, + dumpers.num_of_queue, kvlist, single_iface, is_tx_pcap); free_kvlist: rte_kvargs_free(kvlist); + return ret; } static int -rte_pmd_pcap_devuninit(const char *name) +pmd_pcap_remove(struct rte_vdev_device *dev) { struct rte_eth_dev *eth_dev = NULL; - RTE_LOG(INFO, PMD, "Closing pcap ethdev on numa socket %u\n", + RTE_LOG(INFO, PMD, "Closing pcap ethdev on numa socket %d\n", rte_socket_id()); - if (name == NULL) + if (!dev) return -1; /* reserve an ethdev entry */ - eth_dev = rte_eth_dev_allocated(name); + eth_dev = rte_eth_dev_allocated(rte_vdev_device_name(dev)); if (eth_dev == NULL) return -1; @@ -1085,16 +1055,16 @@ rte_pmd_pcap_devuninit(const char *name) return 0; } -static struct rte_driver pmd_pcap_drv = { - .type = PMD_VDEV, - .init = rte_pmd_pcap_devinit, - .uninit = rte_pmd_pcap_devuninit, +static struct rte_vdev_driver pmd_pcap_drv = { + .probe = pmd_pcap_probe, + .remove = pmd_pcap_remove, }; -PMD_REGISTER_DRIVER(pmd_pcap_drv, eth_pcap); -DRIVER_REGISTER_PARAM_STRING(eth_pcap, - "rx_pcap= " - "tx_pcap= " - "rx_iface= " - "tx_iface= " - "iface="); +RTE_PMD_REGISTER_VDEV(net_pcap, pmd_pcap_drv); +RTE_PMD_REGISTER_ALIAS(net_pcap, eth_pcap); +RTE_PMD_REGISTER_PARAM_STRING(net_pcap, + ETH_PCAP_RX_PCAP_ARG "= " + ETH_PCAP_TX_PCAP_ARG "= " + ETH_PCAP_RX_IFACE_ARG "= " + ETH_PCAP_TX_IFACE_ARG "= " + ETH_PCAP_IFACE_ARG "="); diff --git a/dpdk/drivers/net/qede/Makefile b/dpdk/drivers/net/qede/Makefile index 7965a831..ccbffa45 100644 --- a/dpdk/drivers/net/qede/Makefile +++ b/dpdk/drivers/net/qede/Makefile @@ -13,8 +13,9 @@ LIB = librte_pmd_qede.a CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) - -LDLIBS += -lz +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs +LDLIBS += -lrte_bus_pci EXPORT_MAP := rte_pmd_qede_version.map @@ -48,16 +49,19 @@ endif endif ifeq ($(CONFIG_RTE_TOOLCHAIN_GCC),y) -ifeq ($(shell gcc -Wno-unused-but-set-variable -Werror -E - < /dev/null > /dev/null 2>&1; echo $$?),0) +ifeq ($(shell test $(GCC_VERSION) -ge 44 && echo 1), 1) CFLAGS_BASE_DRIVER += -Wno-unused-but-set-variable endif CFLAGS_BASE_DRIVER += -Wno-missing-declarations -ifeq ($(shell gcc -Wno-maybe-uninitialized -Werror -E - < /dev/null > /dev/null 2>&1; echo $$?),0) +ifeq ($(shell test $(GCC_VERSION) -ge 46 && echo 1), 1) CFLAGS_BASE_DRIVER += -Wno-maybe-uninitialized endif CFLAGS_BASE_DRIVER += -Wno-strict-prototypes ifeq ($(shell test $(GCC_VERSION) -ge 60 && echo 1), 1) CFLAGS_BASE_DRIVER += -Wno-shift-negative-value +ifeq ($(shell test $(GCC_VERSION) -ge 70 && echo 1), 1) +CFLAGS_BASE_DRIVER += -Wno-implicit-fallthrough +endif endif else ifeq ($(CONFIG_RTE_TOOLCHAIN_CLANG),y) CFLAGS_BASE_DRIVER += -Wno-format-extra-args @@ -68,8 +72,9 @@ CFLAGS_BASE_DRIVER += -Wno-sometimes-uninitialized ifeq ($(shell clang -Wno-pointer-bool-conversion -Werror -E - < /dev/null > /dev/null 2>&1; echo $$?),0) CFLAGS_BASE_DRIVER += -Wno-pointer-bool-conversion endif -else -CFLAGS_BASE_DRIVER += -wd188 #188: enumerated type mixed with another type +else #ICC +CFLAGS_BASE_DRIVER += -wd188 #188: enumerated type mixed with another type +CFLAGS_qede_ethdev.o += -wd279 #279: controlling expression is constant endif # @@ -77,34 +82,31 @@ endif # to disable warnings in them # # -BASE_DRIVER_OBJS=$(patsubst %.c,%.o,$(notdir $(wildcard $(SRCDIR)/base/*.c))) -$(foreach obj, $(BASE_DRIVER_OBJS), $(eval CFLAGS+=$(CFLAGS_BASE_DRIVER))) +BASE_DRIVER_OBJS=$(sort $(patsubst %.c,%.o,$(notdir $(wildcard $(SRCDIR)/base/*.c)))) +$(foreach obj, $(BASE_DRIVER_OBJS), $(eval CFLAGS_$(obj)+=$(CFLAGS_BASE_DRIVER))) + +VPATH += $(SRCDIR)/base # # all source are stored in SRCS-y # -SRCS-$(CONFIG_RTE_LIBRTE_QEDE_PMD) += base/ecore_dev.c -SRCS-$(CONFIG_RTE_LIBRTE_QEDE_PMD) += base/ecore_hw.c -SRCS-$(CONFIG_RTE_LIBRTE_QEDE_PMD) += base/ecore_cxt.c -SRCS-$(CONFIG_RTE_LIBRTE_QEDE_PMD) += base/ecore_l2.c -SRCS-$(CONFIG_RTE_LIBRTE_QEDE_PMD) += base/ecore_sp_commands.c -SRCS-$(CONFIG_RTE_LIBRTE_QEDE_PMD) += base/ecore_init_fw_funcs.c -SRCS-$(CONFIG_RTE_LIBRTE_QEDE_PMD) += base/ecore_spq.c -SRCS-$(CONFIG_RTE_LIBRTE_QEDE_PMD) += base/ecore_init_ops.c -SRCS-$(CONFIG_RTE_LIBRTE_QEDE_PMD) += base/ecore_mcp.c -SRCS-$(CONFIG_RTE_LIBRTE_QEDE_PMD) += base/ecore_int.c -SRCS-$(CONFIG_RTE_LIBRTE_QEDE_PMD) += base/ecore_dcbx.c -SRCS-$(CONFIG_RTE_LIBRTE_QEDE_PMD) += base/bcm_osal.c -SRCS-$(CONFIG_RTE_LIBRTE_QEDE_PMD) += base/ecore_sriov.c -SRCS-$(CONFIG_RTE_LIBRTE_QEDE_PMD) += base/ecore_vf.c +SRCS-$(CONFIG_RTE_LIBRTE_QEDE_PMD) += ecore_dev.c +SRCS-$(CONFIG_RTE_LIBRTE_QEDE_PMD) += ecore_hw.c +SRCS-$(CONFIG_RTE_LIBRTE_QEDE_PMD) += ecore_cxt.c +SRCS-$(CONFIG_RTE_LIBRTE_QEDE_PMD) += ecore_l2.c +SRCS-$(CONFIG_RTE_LIBRTE_QEDE_PMD) += ecore_sp_commands.c +SRCS-$(CONFIG_RTE_LIBRTE_QEDE_PMD) += ecore_init_fw_funcs.c +SRCS-$(CONFIG_RTE_LIBRTE_QEDE_PMD) += ecore_spq.c +SRCS-$(CONFIG_RTE_LIBRTE_QEDE_PMD) += ecore_init_ops.c +SRCS-$(CONFIG_RTE_LIBRTE_QEDE_PMD) += ecore_mcp.c +SRCS-$(CONFIG_RTE_LIBRTE_QEDE_PMD) += ecore_int.c +SRCS-$(CONFIG_RTE_LIBRTE_QEDE_PMD) += ecore_dcbx.c +SRCS-$(CONFIG_RTE_LIBRTE_QEDE_PMD) += bcm_osal.c +SRCS-$(CONFIG_RTE_LIBRTE_QEDE_PMD) += ecore_sriov.c +SRCS-$(CONFIG_RTE_LIBRTE_QEDE_PMD) += ecore_vf.c SRCS-$(CONFIG_RTE_LIBRTE_QEDE_PMD) += qede_ethdev.c -SRCS-$(CONFIG_RTE_LIBRTE_QEDE_PMD) += qede_eth_if.c SRCS-$(CONFIG_RTE_LIBRTE_QEDE_PMD) += qede_main.c SRCS-$(CONFIG_RTE_LIBRTE_QEDE_PMD) += qede_rxtx.c - -# dependent libs: -DEPDIRS-$(CONFIG_RTE_LIBRTE_QEDE_PMD) += lib/librte_eal lib/librte_ether -DEPDIRS-$(CONFIG_RTE_LIBRTE_QEDE_PMD) += lib/librte_mempool lib/librte_mbuf -DEPDIRS-$(CONFIG_RTE_LIBRTE_QEDE_PMD) += lib/librte_net +SRCS-$(CONFIG_RTE_LIBRTE_QEDE_PMD) += qede_fdir.c include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/net/qede/base/bcm_osal.c b/dpdk/drivers/net/qede/base/bcm_osal.c index 16029b58..fe42f325 100644 --- a/dpdk/drivers/net/qede/base/bcm_osal.c +++ b/dpdk/drivers/net/qede/base/bcm_osal.c @@ -6,8 +6,6 @@ * See LICENSE.qede_pmd for copyright and licensing details. */ -#include - #include #include @@ -18,6 +16,10 @@ #include "ecore_mcp_api.h" #include "ecore_l2_api.h" +/* Array of memzone pointers */ +static const struct rte_memzone *ecore_mz_mapping[RTE_MAX_MEMZONE]; +/* Counter to track current memzone allocated */ +uint16_t ecore_mz_count; unsigned long qede_log2_align(unsigned long n) { @@ -65,6 +67,27 @@ inline bool qede_test_bit(u32 nr, unsigned long *addr) return res; } +static inline u32 qede_ffb(unsigned long word) +{ + unsigned long first_bit; + + first_bit = __builtin_ffsl(word); + return first_bit ? (first_bit - 1) : OSAL_BITS_PER_UL; +} + +inline u32 qede_find_first_bit(unsigned long *addr, u32 limit) +{ + u32 i; + u32 nwords = 0; + OSAL_BUILD_BUG_ON(!limit); + nwords = (limit - 1) / OSAL_BITS_PER_UL + 1; + for (i = 0; i < nwords; i++) + if (addr[i] != 0) + break; + + return (i == nwords) ? limit : i * OSAL_BITS_PER_UL + qede_ffb(addr[i]); +} + static inline u32 qede_ffz(unsigned long word) { unsigned long first_zero; @@ -79,9 +102,7 @@ inline u32 qede_find_first_zero_bit(unsigned long *addr, u32 limit) u32 nwords = 0; OSAL_BUILD_BUG_ON(!limit); nwords = (limit - 1) / OSAL_BITS_PER_UL + 1; - for (i = 0; i < nwords; i++) - if (~(addr[i] != 0)) - break; + for (i = 0; i < nwords && ~(addr[i]) == 0; i++); return (i == nwords) ? limit : i * OSAL_BITS_PER_UL + qede_ffz(addr[i]); } @@ -101,6 +122,13 @@ void *osal_dma_alloc_coherent(struct ecore_dev *p_dev, uint32_t core_id = rte_lcore_id(); unsigned int socket_id; + if (ecore_mz_count >= RTE_MAX_MEMZONE) { + DP_ERR(p_dev, "Memzone allocation count exceeds %u\n", + RTE_MAX_MEMZONE); + *phys = 0; + return OSAL_NULL; + } + OSAL_MEM_ZERO(mz_name, sizeof(*mz_name)); snprintf(mz_name, sizeof(mz_name) - 1, "%lx", (unsigned long)rte_get_timer_cycles()); @@ -116,10 +144,12 @@ void *osal_dma_alloc_coherent(struct ecore_dev *p_dev, *phys = 0; return OSAL_NULL; } - *phys = mz->phys_addr; - DP_VERBOSE(p_dev, ECORE_MSG_PROBE, - "size=%zu phys=0x%" PRIx64 " virt=%p on socket=%u\n", - mz->len, mz->phys_addr, mz->addr, socket_id); + *phys = mz->iova; + ecore_mz_mapping[ecore_mz_count++] = mz; + DP_VERBOSE(p_dev, ECORE_MSG_SP, + "Allocated dma memory size=%zu phys=0x%lx" + " virt=%p core=%d\n", + mz->len, (unsigned long)mz->iova, mz->addr, core_id); return mz->addr; } @@ -131,6 +161,13 @@ void *osal_dma_alloc_coherent_aligned(struct ecore_dev *p_dev, uint32_t core_id = rte_lcore_id(); unsigned int socket_id; + if (ecore_mz_count >= RTE_MAX_MEMZONE) { + DP_ERR(p_dev, "Memzone allocation count exceeds %u\n", + RTE_MAX_MEMZONE); + *phys = 0; + return OSAL_NULL; + } + OSAL_MEM_ZERO(mz_name, sizeof(*mz_name)); snprintf(mz_name, sizeof(mz_name) - 1, "%lx", (unsigned long)rte_get_timer_cycles()); @@ -145,13 +182,32 @@ void *osal_dma_alloc_coherent_aligned(struct ecore_dev *p_dev, *phys = 0; return OSAL_NULL; } - *phys = mz->phys_addr; - DP_VERBOSE(p_dev, ECORE_MSG_PROBE, - "aligned memory size=%zu phys=0x%" PRIx64 " virt=%p core=%d\n", - mz->len, mz->phys_addr, mz->addr, core_id); + *phys = mz->iova; + ecore_mz_mapping[ecore_mz_count++] = mz; + DP_VERBOSE(p_dev, ECORE_MSG_SP, + "Allocated aligned dma memory size=%zu phys=0x%lx" + " virt=%p core=%d\n", + mz->len, (unsigned long)mz->iova, mz->addr, core_id); return mz->addr; } +void osal_dma_free_mem(struct ecore_dev *p_dev, dma_addr_t phys) +{ + uint16_t j; + + for (j = 0 ; j < ecore_mz_count; j++) { + if (phys == ecore_mz_mapping[j]->iova) { + DP_VERBOSE(p_dev, ECORE_MSG_SP, + "Free memzone %s\n", ecore_mz_mapping[j]->name); + rte_memzone_free(ecore_mz_mapping[j]); + return; + } + } + + DP_ERR(p_dev, "Unexpected memory free request\n"); +} + +#ifdef CONFIG_ECORE_ZIPPED_FW u32 qede_unzip_data(struct ecore_hwfn *p_hwfn, u32 input_len, u8 *input_buf, u32 max_size, u8 *unzip_buf) { @@ -182,6 +238,7 @@ u32 qede_unzip_data(struct ecore_hwfn *p_hwfn, u32 input_len, return p_hwfn->stream->total_out / 4; } +#endif void qede_get_mcp_proto_stats(struct ecore_dev *edev, @@ -192,11 +249,58 @@ qede_get_mcp_proto_stats(struct ecore_dev *edev, if (type == ECORE_MCP_LAN_STATS) { ecore_get_vport_stats(edev, &lan_stats); - stats->lan_stats.ucast_rx_pkts = lan_stats.rx_ucast_pkts; - stats->lan_stats.ucast_tx_pkts = lan_stats.tx_ucast_pkts; + + /* @DPDK */ + stats->lan_stats.ucast_rx_pkts = lan_stats.common.rx_ucast_pkts; + stats->lan_stats.ucast_tx_pkts = lan_stats.common.tx_ucast_pkts; + stats->lan_stats.fcs_err = -1; } else { DP_INFO(edev, "Statistics request type %d not supported\n", type); } } + +void +qede_hw_err_notify(struct ecore_hwfn *p_hwfn, enum ecore_hw_err_type err_type) +{ + char err_str[64]; + + switch (err_type) { + case ECORE_HW_ERR_FAN_FAIL: + strcpy(err_str, "Fan Failure"); + break; + case ECORE_HW_ERR_MFW_RESP_FAIL: + strcpy(err_str, "MFW Response Failure"); + break; + case ECORE_HW_ERR_HW_ATTN: + strcpy(err_str, "HW Attention"); + break; + case ECORE_HW_ERR_DMAE_FAIL: + strcpy(err_str, "DMAE Failure"); + break; + case ECORE_HW_ERR_RAMROD_FAIL: + strcpy(err_str, "Ramrod Failure"); + break; + case ECORE_HW_ERR_FW_ASSERT: + strcpy(err_str, "FW Assertion"); + break; + default: + strcpy(err_str, "Unknown"); + } + + DP_ERR(p_hwfn, "HW error occurred [%s]\n", err_str); + ecore_int_attn_clr_enable(p_hwfn->p_dev, true); +} + +u32 qede_crc32(u32 crc, u8 *ptr, u32 length) +{ + int i; + + while (length--) { + crc ^= *ptr++; + for (i = 0; i < 8; i++) + crc = (crc >> 1) ^ ((crc & 1) ? 0xedb88320 : 0); + } + return crc; +} diff --git a/dpdk/drivers/net/qede/base/bcm_osal.h b/dpdk/drivers/net/qede/base/bcm_osal.h index 3e2aeb03..52c2f0ec 100644 --- a/dpdk/drivers/net/qede/base/bcm_osal.h +++ b/dpdk/drivers/net/qede/base/bcm_osal.h @@ -18,14 +18,17 @@ #include #include #include +#include /* Forward declaration */ struct ecore_dev; struct ecore_hwfn; +struct ecore_ptt; struct ecore_vf_acquire_sw_info; struct vf_pf_resc_request; enum ecore_mcp_protocol_type; union ecore_mcp_protocol_stats; +enum ecore_hw_err_type; void qed_link_update(struct ecore_hwfn *hwfn); @@ -41,6 +44,10 @@ void qed_link_update(struct ecore_hwfn *hwfn); #endif #endif +#define OSAL_WARN(arg1, arg2, arg3, ...) (0) + +#define UNUSED(x) (void)(x) + /* Memory Types */ typedef uint8_t u8; typedef uint16_t u16; @@ -56,7 +63,7 @@ typedef u32 OSAL_BE32; #define osal_uintptr_t uintptr_t -typedef phys_addr_t dma_addr_t; +typedef rte_iova_t dma_addr_t; typedef rte_spinlock_t osal_spinlock_t; @@ -86,8 +93,12 @@ typedef int bool; #define OSAL_ALLOC(dev, GFP, size) rte_malloc("qede", size, 0) #define OSAL_ZALLOC(dev, GFP, size) rte_zmalloc("qede", size, 0) #define OSAL_CALLOC(dev, GFP, num, size) rte_calloc("qede", num, size, 0) -#define OSAL_VALLOC(dev, size) rte_malloc("qede", size, 0) -#define OSAL_FREE(dev, memory) rte_free((void *)memory) +#define OSAL_VZALLOC(dev, size) rte_zmalloc("qede", size, 0) +#define OSAL_FREE(dev, memory) \ + do { \ + rte_free((void *)memory); \ + memory = OSAL_NULL; \ + } while (0) #define OSAL_VFREE(dev, memory) OSAL_FREE(dev, memory) #define OSAL_MEM_ZERO(mem, size) bzero(mem, size) #define OSAL_MEMCPY(dst, src, size) rte_memcpy(dst, src, size) @@ -100,29 +111,31 @@ void *osal_dma_alloc_coherent(struct ecore_dev *, dma_addr_t *, size_t); void *osal_dma_alloc_coherent_aligned(struct ecore_dev *, dma_addr_t *, size_t, int); +void osal_dma_free_mem(struct ecore_dev *edev, dma_addr_t phys); + #define OSAL_DMA_ALLOC_COHERENT(dev, phys, size) \ osal_dma_alloc_coherent(dev, phys, size) #define OSAL_DMA_ALLOC_COHERENT_ALIGNED(dev, phys, size, align) \ osal_dma_alloc_coherent_aligned(dev, phys, size, align) -/* TODO: */ -#define OSAL_DMA_FREE_COHERENT(dev, virt, phys, size) nothing +#define OSAL_DMA_FREE_COHERENT(dev, virt, phys, size) \ + osal_dma_free_mem(dev, phys) /* HW reads/writes */ -#define DIRECT_REG_RD(_dev, _reg_addr) \ - (*((volatile u32 *) (_reg_addr))) +#define DIRECT_REG_RD(_dev, _reg_addr) rte_read32(_reg_addr) #define REG_RD(_p_hwfn, _reg_offset) \ DIRECT_REG_RD(_p_hwfn, \ ((u8 *)(uintptr_t)(_p_hwfn->regview) + (_reg_offset))) -#define DIRECT_REG_WR16(_reg_addr, _val) \ - (*((volatile u16 *)(_reg_addr)) = _val) +#define DIRECT_REG_WR16(_reg_addr, _val) rte_write16((_val), (_reg_addr)) -#define DIRECT_REG_WR(_dev, _reg_addr, _val) \ - (*((volatile u32 *)(_reg_addr)) = _val) +#define DIRECT_REG_WR(_dev, _reg_addr, _val) rte_write32((_val), (_reg_addr)) + +#define DIRECT_REG_WR_RELAXED(_dev, _reg_addr, _val) \ + rte_write32_relaxed((_val), (_reg_addr)) #define REG_WR(_p_hwfn, _reg_offset, _val) \ DIRECT_REG_WR(NULL, \ @@ -132,9 +145,13 @@ void *osal_dma_alloc_coherent_aligned(struct ecore_dev *, dma_addr_t *, DIRECT_REG_WR16(((u8 *)(uintptr_t)(_p_hwfn->regview) + \ (_reg_offset)), (u16)_val) -#define DOORBELL(_p_hwfn, _db_addr, _val) \ - DIRECT_REG_WR(_p_hwfn, \ - ((u8 *)(uintptr_t)(_p_hwfn->doorbells) + (_db_addr)), (u32)_val) +#define DOORBELL(_p_hwfn, _db_addr, _val) \ + DIRECT_REG_WR_RELAXED((_p_hwfn), \ + ((u8 *)(uintptr_t)(_p_hwfn->doorbells) + \ + (_db_addr)), (u32)_val) + +#define DIRECT_REG_WR64(hwfn, addr, value) nothing +#define DIRECT_REG_RD64(hwfn, addr) 0 /* Mutexes */ @@ -150,7 +167,12 @@ typedef pthread_mutex_t osal_mutex_t; #define OSAL_SPIN_LOCK_INIT(lock) rte_spinlock_init(lock) #define OSAL_SPIN_LOCK(lock) rte_spinlock_lock(lock) #define OSAL_SPIN_UNLOCK(lock) rte_spinlock_unlock(lock) -#define OSAL_SPIN_LOCK_IRQSAVE(lock, flags) nothing +#define OSAL_SPIN_LOCK_IRQSAVE(lock, flags) \ + do { \ + UNUSED(lock); \ + flags = 0; \ + UNUSED(flags); \ + } while (0) #define OSAL_SPIN_UNLOCK_IRQSAVE(lock, flags) nothing #define OSAL_SPIN_LOCK_ALLOC(hwfn, lock) nothing #define OSAL_SPIN_LOCK_DEALLOC(lock) nothing @@ -160,6 +182,7 @@ typedef pthread_mutex_t osal_mutex_t; #define OSAL_DPC_ALLOC(hwfn) OSAL_ALLOC(hwfn, GFP, sizeof(osal_dpc_t)) #define OSAL_DPC_INIT(dpc, hwfn) nothing #define OSAL_POLL_MODE_DPC(hwfn) nothing +#define OSAL_DPC_SYNC(hwfn) nothing /* Lists */ @@ -284,7 +307,8 @@ typedef struct osal_list_t { #define OSAL_WMB(dev) rte_wmb() #define OSAL_DMA_SYNC(dev, addr, length, is_post) nothing -#define OSAL_BITS_PER_BYTE (8) +#define OSAL_BIT(nr) (1UL << (nr)) +#define OSAL_BITS_PER_BYTE (8) #define OSAL_BITS_PER_UL (sizeof(unsigned long) * OSAL_BITS_PER_BYTE) #define OSAL_BITS_PER_UL_MASK (OSAL_BITS_PER_UL - 1) @@ -301,6 +325,10 @@ bool qede_test_bit(u32, unsigned long *); #define OSAL_TEST_BIT(bit, bitmap) \ qede_test_bit(bit, bitmap) +u32 qede_find_first_bit(unsigned long *, u32); +#define OSAL_FIND_FIRST_BIT(bitmap, length) \ + qede_find_first_bit(bitmap, length) + u32 qede_find_first_zero_bit(unsigned long *, u32); #define OSAL_FIND_FIRST_ZERO_BIT(bitmap, length) \ qede_find_first_zero_bit(bitmap, length) @@ -308,7 +336,10 @@ u32 qede_find_first_zero_bit(unsigned long *, u32); #define OSAL_BUILD_BUG_ON(cond) nothing #define ETH_ALEN ETHER_ADDR_LEN +#define OSAL_BITMAP_WEIGHT(bitmap, count) 0 + #define OSAL_LINK_UPDATE(hwfn) qed_link_update(hwfn) +#define OSAL_TRANSCEIVER_UPDATE(hwfn) nothing #define OSAL_DCBX_AEN(hwfn, mib_type) nothing /* SR-IOV channel */ @@ -317,6 +348,7 @@ u32 qede_find_first_zero_bit(unsigned long *, u32); #define OSAL_VF_SEND_MSG2PF(dev, done, msg, reply_addr, msg_size, reply_size) 0 #define OSAL_VF_CQE_COMPLETION(_dev_p, _cqe, _protocol) (0) #define OSAL_PF_VF_MSG(hwfn, vfid) 0 +#define OSAL_PF_VF_MALICIOUS(hwfn, vfid) nothing #define OSAL_IOV_CHK_UCAST(hwfn, vfid, params) 0 #define OSAL_IOV_POST_START_VPORT(hwfn, vf, vport_id, opaque_fid) nothing #define OSAL_IOV_VF_ACQUIRE(hwfn, vfid) 0 @@ -324,11 +356,16 @@ u32 qede_find_first_zero_bit(unsigned long *, u32); #define OSAL_IOV_VF_VPORT_UPDATE(hwfn, vfid, p_params, p_mask) 0 #define OSAL_VF_UPDATE_ACQUIRE_RESC_RESP(_dev_p, _resc_resp) 0 #define OSAL_IOV_GET_OS_TYPE() 0 +#define OSAL_IOV_VF_MSG_TYPE(hwfn, vfid, vf_msg_type) nothing +#define OSAL_IOV_PF_RESP_TYPE(hwfn, vfid, pf_resp_type) nothing +#define OSAL_IOV_VF_VPORT_STOP(hwfn, vf) nothing u32 qede_unzip_data(struct ecore_hwfn *p_hwfn, u32 input_len, u8 *input_buf, u32 max_size, u8 *unzip_buf); void qede_vf_fill_driver_data(struct ecore_hwfn *, struct vf_pf_resc_request *, struct ecore_vf_acquire_sw_info *); +void qede_hw_err_notify(struct ecore_hwfn *p_hwfn, + enum ecore_hw_err_type err_type); #define OSAL_VF_FILL_ACQUIRE_RESC_REQ(_dev_p, _resc_req, _os_info) \ qede_vf_fill_driver_data(_dev_p, _resc_req, _os_info) @@ -337,10 +374,11 @@ void qede_vf_fill_driver_data(struct ecore_hwfn *, struct vf_pf_resc_request *, /* TODO: */ #define OSAL_SCHEDULE_RECOVERY_HANDLER(hwfn) nothing -#define OSAL_HW_ERROR_OCCURRED(hwfn, err_type) nothing +#define OSAL_HW_ERROR_OCCURRED(hwfn, err_type) \ + qede_hw_err_notify(hwfn, err_type) #define OSAL_NVM_IS_ACCESS_ENABLED(hwfn) (1) -#define OSAL_NUM_ACTIVE_CPU() 0 +#define OSAL_NUM_CPUS() 0 /* Utility functions */ @@ -377,16 +415,21 @@ u32 qede_osal_log2(u32); #define OSAL_ARRAY_SIZE(arr) RTE_DIM(arr) #define OSAL_SPRINTF(name, pattern, ...) \ sprintf(name, pattern, ##__VA_ARGS__) +#define OSAL_SNPRINTF(buf, size, format, ...) \ + snprintf(buf, size, format, ##__VA_ARGS__) #define OSAL_STRLEN(string) strlen(string) #define OSAL_STRCPY(dst, string) strcpy(dst, string) #define OSAL_STRNCPY(dst, string, len) strncpy(dst, string, len) #define OSAL_STRCMP(str1, str2) strcmp(str1, str2) +#define OSAL_STRTOUL(str, base, res) 0 #define OSAL_INLINE inline #define OSAL_REG_ADDR(_p_hwfn, _offset) \ (void *)((u8 *)(uintptr_t)(_p_hwfn->regview) + (_offset)) #define OSAL_PAGE_SIZE 4096 +#define OSAL_CACHE_LINE_SIZE RTE_CACHE_LINE_SIZE #define OSAL_IOMEM volatile +#define OSAL_UNUSED __attribute__((unused)) #define OSAL_UNLIKELY(x) __builtin_expect(!!(x), 0) #define OSAL_MIN_T(type, __min1, __min2) \ ((type)(__min1) < (type)(__min2) ? (type)(__min1) : (type)(__min2)) @@ -400,4 +443,16 @@ void qede_get_mcp_proto_stats(struct ecore_dev *, enum ecore_mcp_protocol_type, #define OSAL_SLOWPATH_IRQ_REQ(p_hwfn) (0) +u32 qede_crc32(u32 crc, u8 *ptr, u32 length); +#define OSAL_CRC32(crc, buf, length) qede_crc32(crc, buf, length) +#define OSAL_CRC8_POPULATE(table, polynomial) nothing +#define OSAL_CRC8(table, pdata, nbytes, crc) 0 +#define OSAL_MFW_TLV_REQ(p_hwfn) nothing +#define OSAL_MFW_FILL_TLV_DATA(type, buf, data) (0) +#define OSAL_MFW_CMD_PREEMPT(p_hwfn) nothing +#define OSAL_PF_VALIDATE_MODIFY_TUNN_CONFIG(p_hwfn, mask, b_update, tunn) 0 + +#define OSAL_DIV_S64(a, b) ((a) / (b)) +#define OSAL_LLDP_RX_TLVS(p_hwfn, tlv_buf, tlv_size) nothing + #endif /* __BCM_OSAL_H */ diff --git a/dpdk/drivers/net/qede/base/common_hsi.h b/dpdk/drivers/net/qede/base/common_hsi.h index 295a41f9..9a6059ac 100644 --- a/dpdk/drivers/net/qede/base/common_hsi.h +++ b/dpdk/drivers/net/qede/base/common_hsi.h @@ -8,12 +8,97 @@ #ifndef __COMMON_HSI__ #define __COMMON_HSI__ +/********************************/ +/* PROTOCOL COMMON FW CONSTANTS */ +/********************************/ + +/* Temporarily here should be added to HSI automatically by resource allocation + * tool. + */ +#define T_TEST_AGG_INT_TEMP 6 +#define M_TEST_AGG_INT_TEMP 8 +#define U_TEST_AGG_INT_TEMP 6 +#define X_TEST_AGG_INT_TEMP 14 +#define Y_TEST_AGG_INT_TEMP 4 +#define P_TEST_AGG_INT_TEMP 4 + +#define X_FINAL_CLEANUP_AGG_INT 1 + +#define EVENT_RING_PAGE_SIZE_BYTES 4096 + +#define NUM_OF_GLOBAL_QUEUES 128 +#define COMMON_QUEUE_ENTRY_MAX_BYTE_SIZE 64 + +#define ISCSI_CDU_TASK_SEG_TYPE 0 +#define FCOE_CDU_TASK_SEG_TYPE 0 +#define RDMA_CDU_TASK_SEG_TYPE 1 + +#define FW_ASSERT_GENERAL_ATTN_IDX 32 + +#define MAX_PINNED_CCFC 32 + +#define EAGLE_ENG1_WORKAROUND_NIG_FLOWCTRL_MODE 3 + +/* Queue Zone sizes in bytes */ +#define TSTORM_QZONE_SIZE 8 /*tstorm_scsi_queue_zone*/ +#define MSTORM_QZONE_SIZE 16 /*mstorm_eth_queue_zone. Used only for RX + *producer of VFs in backward compatibility + *mode. + */ +#define USTORM_QZONE_SIZE 8 /*ustorm_eth_queue_zone*/ +#define XSTORM_QZONE_SIZE 8 /*xstorm_eth_queue_zone*/ +#define YSTORM_QZONE_SIZE 0 +#define PSTORM_QZONE_SIZE 0 + +/*Log of mstorm default VF zone size.*/ +#define MSTORM_VF_ZONE_DEFAULT_SIZE_LOG 7 +/*Maximum number of RX queues that can be allocated to VF by default*/ +#define ETH_MAX_NUM_RX_QUEUES_PER_VF_DEFAULT 16 +/*Maximum number of RX queues that can be allocated to VF with doubled VF zone + * size. Up to 96 VF supported in this mode + */ +#define ETH_MAX_NUM_RX_QUEUES_PER_VF_DOUBLE 48 +/*Maximum number of RX queues that can be allocated to VF with 4 VF zone size. + * Up to 48 VF supported in this mode + */ +#define ETH_MAX_NUM_RX_QUEUES_PER_VF_QUAD 112 + + +/********************************/ +/* CORE (LIGHT L2) FW CONSTANTS */ +/********************************/ + +#define CORE_LL2_MAX_RAMROD_PER_CON 8 +#define CORE_LL2_TX_BD_PAGE_SIZE_BYTES 4096 +#define CORE_LL2_RX_BD_PAGE_SIZE_BYTES 4096 +#define CORE_LL2_RX_CQE_PAGE_SIZE_BYTES 4096 +#define CORE_LL2_RX_NUM_NEXT_PAGE_BDS 1 + +#define CORE_LL2_TX_MAX_BDS_PER_PACKET 12 #define CORE_SPQE_PAGE_SIZE_BYTES 4096 -#define FW_MAJOR_VERSION 8 -#define FW_MINOR_VERSION 7 -#define FW_REVISION_VERSION 7 +/* + * Usually LL2 queues are opened in pairs TX-RX. + * There is a hard restriction on number of RX queues (limited by Tstorm RAM) + * and TX counters (Pstorm RAM). + * Number of TX queues is almost unlimited. + * The constants are different so as to allow asymmetric LL2 connections + */ + +#define MAX_NUM_LL2_RX_QUEUES 48 +#define MAX_NUM_LL2_TX_STATS_COUNTERS 48 + + +/****************************************************************************/ +/* Include firmware version number only- do not add constants here to avoid */ +/* redundunt compilations */ +/****************************************************************************/ + + +#define FW_MAJOR_VERSION 8 +#define FW_MINOR_VERSION 30 +#define FW_REVISION_VERSION 12 #define FW_ENGINEERING_VERSION 0 /***********************/ @@ -21,55 +106,73 @@ /***********************/ /* PCI functions */ -#define MAX_NUM_PORTS_K2 (4) -#define MAX_NUM_PORTS_BB (2) -#define MAX_NUM_PORTS (MAX_NUM_PORTS_K2) +#define MAX_NUM_PORTS_BB (2) +#define MAX_NUM_PORTS_K2 (4) +#define MAX_NUM_PORTS_E5 (4) +#define MAX_NUM_PORTS (MAX_NUM_PORTS_E5) -#define MAX_NUM_PFS_K2 (16) -#define MAX_NUM_PFS_BB (8) -#define MAX_NUM_PFS (MAX_NUM_PFS_K2) -#define MAX_NUM_OF_PFS_IN_CHIP (16) /* On both engines */ +#define MAX_NUM_PFS_BB (8) +#define MAX_NUM_PFS_K2 (16) +#define MAX_NUM_PFS_E5 (16) +#define MAX_NUM_PFS (MAX_NUM_PFS_E5) +#define MAX_NUM_OF_PFS_IN_CHIP (16) /* On both engines */ -#define MAX_NUM_VFS_K2 (192) -#define MAX_NUM_VFS_BB (120) -#define MAX_NUM_VFS (MAX_NUM_VFS_K2) +#define MAX_NUM_VFS_BB (120) +#define MAX_NUM_VFS_K2 (192) +#define MAX_NUM_VFS_E4 (MAX_NUM_VFS_K2) +#define MAX_NUM_VFS_E5 (240) +#define COMMON_MAX_NUM_VFS (MAX_NUM_VFS_E5) -#define MAX_NUM_FUNCTIONS_BB (MAX_NUM_PFS_BB + MAX_NUM_VFS_BB) -#define MAX_NUM_FUNCTIONS (MAX_NUM_PFS + MAX_NUM_VFS) +#define MAX_NUM_FUNCTIONS_BB (MAX_NUM_PFS_BB + MAX_NUM_VFS_BB) +#define MAX_NUM_FUNCTIONS_K2 (MAX_NUM_PFS_K2 + MAX_NUM_VFS_K2) +#define MAX_NUM_FUNCTIONS (MAX_NUM_PFS + MAX_NUM_VFS_E4) -#define MAX_FUNCTION_NUMBER_BB (MAX_NUM_PFS + MAX_NUM_VFS_BB) -#define MAX_FUNCTION_NUMBER (MAX_NUM_PFS + MAX_NUM_VFS) +/* in both BB and K2, the VF number starts from 16. so for arrays containing all + * possible PFs and VFs - we need a constant for this size + */ +#define MAX_FUNCTION_NUMBER_BB (MAX_NUM_PFS + MAX_NUM_VFS_BB) +#define MAX_FUNCTION_NUMBER_K2 (MAX_NUM_PFS + MAX_NUM_VFS_K2) +#define MAX_FUNCTION_NUMBER_E4 (MAX_NUM_PFS + MAX_NUM_VFS_E4) +#define MAX_FUNCTION_NUMBER_E5 (MAX_NUM_PFS + MAX_NUM_VFS_E5) +#define COMMON_MAX_FUNCTION_NUMBER (MAX_NUM_PFS + MAX_NUM_VFS_E5) -#define MAX_NUM_VPORTS_K2 (208) -#define MAX_NUM_VPORTS_BB (160) -#define MAX_NUM_VPORTS (MAX_NUM_VPORTS_K2) +#define MAX_NUM_VPORTS_K2 (208) +#define MAX_NUM_VPORTS_BB (160) +#define MAX_NUM_VPORTS_E4 (MAX_NUM_VPORTS_K2) +#define MAX_NUM_VPORTS_E5 (256) +#define COMMON_MAX_NUM_VPORTS (MAX_NUM_VPORTS_E5) -#define MAX_NUM_L2_QUEUES_K2 (320) #define MAX_NUM_L2_QUEUES_BB (256) -#define MAX_NUM_L2_QUEUES (MAX_NUM_L2_QUEUES_K2) +#define MAX_NUM_L2_QUEUES_K2 (320) +#define MAX_NUM_L2_QUEUES_E5 (320) /* TODO_E5_VITALY - fix to 512 */ +#define MAX_NUM_L2_QUEUES (MAX_NUM_L2_QUEUES_E5) /* Traffic classes in network-facing blocks (PBF, BTB, NIG, BRB, PRS and QM) */ -#define NUM_PHYS_TCS_4PORT_K2 (4) -#define NUM_OF_PHYS_TCS (8) - -#define NUM_TCS_4PORT_K2 (NUM_PHYS_TCS_4PORT_K2 + 1) -#define NUM_OF_TCS (NUM_OF_PHYS_TCS + 1) - -#define LB_TC (NUM_OF_PHYS_TCS) - -/* Num of possible traffic priority values */ -#define NUM_OF_PRIO (8) - -#define MAX_NUM_VOQS_K2 (NUM_TCS_4PORT_K2 * MAX_NUM_PORTS_K2) -#define MAX_NUM_VOQS_BB (NUM_OF_TCS * MAX_NUM_PORTS_BB) -#define MAX_NUM_VOQS (MAX_NUM_VOQS_K2) -#define MAX_PHYS_VOQS (NUM_OF_PHYS_TCS * MAX_NUM_PORTS_BB) +#define NUM_PHYS_TCS_4PORT_K2 4 +#define NUM_PHYS_TCS_4PORT_TX_E5 6 +#define NUM_PHYS_TCS_4PORT_RX_E5 4 +#define NUM_OF_PHYS_TCS 8 +#define PURE_LB_TC NUM_OF_PHYS_TCS +#define NUM_TCS_4PORT_K2 (NUM_PHYS_TCS_4PORT_K2 + 1) +#define NUM_TCS_4PORT_TX_E5 (NUM_PHYS_TCS_4PORT_TX_E5 + 1) +#define NUM_TCS_4PORT_RX_E5 (NUM_PHYS_TCS_4PORT_RX_E5 + 1) +#define NUM_OF_TCS (NUM_OF_PHYS_TCS + 1) /* CIDs */ -#define NUM_OF_CONNECTION_TYPES (8) -#define NUM_OF_LCIDS (320) -#define NUM_OF_LTIDS (320) +#define NUM_OF_CONNECTION_TYPES_E4 (8) +#define NUM_OF_CONNECTION_TYPES_E5 (16) +#define NUM_OF_TASK_TYPES (8) +#define NUM_OF_LCIDS (320) +#define NUM_OF_LTIDS (320) +/* Global PXP windows (GTT) */ +#define NUM_OF_GTT 19 +#define GTT_DWORD_SIZE_BITS 10 +#define GTT_BYTE_SIZE_BITS (GTT_DWORD_SIZE_BITS + 2) +#define GTT_DWORD_SIZE (1 << GTT_DWORD_SIZE_BITS) + +/* Tools Version */ +#define TOOLS_VERSION 10 /*****************/ /* CDU CONSTANTS */ /*****************/ @@ -77,14 +180,28 @@ #define CDU_SEG_TYPE_OFFSET_REG_TYPE_SHIFT (17) #define CDU_SEG_TYPE_OFFSET_REG_OFFSET_MASK (0x1ffff) +#define CDU_VF_FL_SEG_TYPE_OFFSET_REG_TYPE_SHIFT (12) +#define CDU_VF_FL_SEG_TYPE_OFFSET_REG_OFFSET_MASK (0xfff) + +#define CDU_CONTEXT_VALIDATION_CFG_ENABLE_SHIFT (0) +#define CDU_CONTEXT_VALIDATION_CFG_VALIDATION_TYPE_SHIFT (1) +#define CDU_CONTEXT_VALIDATION_CFG_USE_TYPE (2) +#define CDU_CONTEXT_VALIDATION_CFG_USE_REGION (3) +#define CDU_CONTEXT_VALIDATION_CFG_USE_CID (4) +#define CDU_CONTEXT_VALIDATION_CFG_USE_ACTIVE (5) + + /*****************/ /* DQ CONSTANTS */ /*****************/ /* DEMS */ #define DQ_DEMS_LEGACY 0 +#define DQ_DEMS_TOE_MORE_TO_SEND 3 +#define DQ_DEMS_TOE_LOCAL_ADV_WND 4 +#define DQ_DEMS_ROCE_CQ_CONS 7 -/* XCM agg val selection */ +/* XCM agg val selection (HW) */ #define DQ_XCM_AGG_VAL_SEL_WORD2 0 #define DQ_XCM_AGG_VAL_SEL_WORD3 1 #define DQ_XCM_AGG_VAL_SEL_WORD4 2 @@ -94,7 +211,7 @@ #define DQ_XCM_AGG_VAL_SEL_REG5 6 #define DQ_XCM_AGG_VAL_SEL_REG6 7 -/* XCM agg val selection */ +/* XCM agg val selection (FW) */ #define DQ_XCM_ETH_EDPM_NUM_BDS_CMD \ DQ_XCM_AGG_VAL_SEL_WORD2 #define DQ_XCM_ETH_TX_BD_CONS_CMD \ @@ -108,8 +225,49 @@ #define DQ_XCM_CORE_SPQ_PROD_CMD \ DQ_XCM_AGG_VAL_SEL_WORD4 #define DQ_XCM_ETH_GO_TO_BD_CONS_CMD DQ_XCM_AGG_VAL_SEL_WORD5 +#define DQ_XCM_FCOE_SQ_CONS_CMD DQ_XCM_AGG_VAL_SEL_WORD3 +#define DQ_XCM_FCOE_SQ_PROD_CMD DQ_XCM_AGG_VAL_SEL_WORD4 +#define DQ_XCM_FCOE_X_FERQ_PROD_CMD DQ_XCM_AGG_VAL_SEL_WORD5 +#define DQ_XCM_ISCSI_SQ_CONS_CMD DQ_XCM_AGG_VAL_SEL_WORD3 +#define DQ_XCM_ISCSI_SQ_PROD_CMD DQ_XCM_AGG_VAL_SEL_WORD4 +#define DQ_XCM_ISCSI_MORE_TO_SEND_SEQ_CMD DQ_XCM_AGG_VAL_SEL_REG3 +#define DQ_XCM_ISCSI_EXP_STAT_SN_CMD DQ_XCM_AGG_VAL_SEL_REG6 +#define DQ_XCM_ROCE_SQ_PROD_CMD DQ_XCM_AGG_VAL_SEL_WORD4 +#define DQ_XCM_TOE_TX_BD_PROD_CMD DQ_XCM_AGG_VAL_SEL_WORD4 +#define DQ_XCM_TOE_MORE_TO_SEND_SEQ_CMD DQ_XCM_AGG_VAL_SEL_REG3 +#define DQ_XCM_TOE_LOCAL_ADV_WND_SEQ_CMD DQ_XCM_AGG_VAL_SEL_REG4 -/* XCM agg counter flag selection */ +/* UCM agg val selection (HW) */ +#define DQ_UCM_AGG_VAL_SEL_WORD0 0 +#define DQ_UCM_AGG_VAL_SEL_WORD1 1 +#define DQ_UCM_AGG_VAL_SEL_WORD2 2 +#define DQ_UCM_AGG_VAL_SEL_WORD3 3 +#define DQ_UCM_AGG_VAL_SEL_REG0 4 +#define DQ_UCM_AGG_VAL_SEL_REG1 5 +#define DQ_UCM_AGG_VAL_SEL_REG2 6 +#define DQ_UCM_AGG_VAL_SEL_REG3 7 + +/* UCM agg val selection (FW) */ +#define DQ_UCM_ETH_PMD_TX_CONS_CMD DQ_UCM_AGG_VAL_SEL_WORD2 +#define DQ_UCM_ETH_PMD_RX_CONS_CMD DQ_UCM_AGG_VAL_SEL_WORD3 +#define DQ_UCM_ROCE_CQ_CONS_CMD DQ_UCM_AGG_VAL_SEL_REG0 +#define DQ_UCM_ROCE_CQ_PROD_CMD DQ_UCM_AGG_VAL_SEL_REG2 + +/* TCM agg val selection (HW) */ +#define DQ_TCM_AGG_VAL_SEL_WORD0 0 +#define DQ_TCM_AGG_VAL_SEL_WORD1 1 +#define DQ_TCM_AGG_VAL_SEL_WORD2 2 +#define DQ_TCM_AGG_VAL_SEL_WORD3 3 +#define DQ_TCM_AGG_VAL_SEL_REG1 4 +#define DQ_TCM_AGG_VAL_SEL_REG2 5 +#define DQ_TCM_AGG_VAL_SEL_REG6 6 +#define DQ_TCM_AGG_VAL_SEL_REG9 7 + +/* TCM agg val selection (FW) */ +#define DQ_TCM_L2B_BD_PROD_CMD DQ_TCM_AGG_VAL_SEL_WORD1 +#define DQ_TCM_ROCE_RQ_PROD_CMD DQ_TCM_AGG_VAL_SEL_WORD0 + +/* XCM agg counter flag selection (HW) */ #define DQ_XCM_AGG_FLG_SHIFT_BIT14 0 #define DQ_XCM_AGG_FLG_SHIFT_BIT15 1 #define DQ_XCM_AGG_FLG_SHIFT_CF12 2 @@ -119,7 +277,7 @@ #define DQ_XCM_AGG_FLG_SHIFT_CF22 6 #define DQ_XCM_AGG_FLG_SHIFT_CF23 7 -/* XCM agg counter flag selection */ +/* XCM agg counter flag selection (FW) */ #define DQ_XCM_ETH_DQ_CF_CMD (1 << \ DQ_XCM_AGG_FLG_SHIFT_CF18) #define DQ_XCM_CORE_DQ_CF_CMD (1 << \ @@ -134,6 +292,81 @@ DQ_XCM_AGG_FLG_SHIFT_CF22) #define DQ_XCM_ETH_TPH_EN_CMD (1 << \ DQ_XCM_AGG_FLG_SHIFT_CF23) +#define DQ_XCM_FCOE_SLOW_PATH_CMD (1 << DQ_XCM_AGG_FLG_SHIFT_CF22) +#define DQ_XCM_ISCSI_DQ_FLUSH_CMD (1 << DQ_XCM_AGG_FLG_SHIFT_CF19) +#define DQ_XCM_ISCSI_SLOW_PATH_CMD (1 << DQ_XCM_AGG_FLG_SHIFT_CF22) +#define DQ_XCM_ISCSI_PROC_ONLY_CLEANUP_CMD (1 << DQ_XCM_AGG_FLG_SHIFT_CF23) +#define DQ_XCM_TOE_DQ_FLUSH_CMD (1 << DQ_XCM_AGG_FLG_SHIFT_CF19) +#define DQ_XCM_TOE_SLOW_PATH_CMD (1 << DQ_XCM_AGG_FLG_SHIFT_CF22) + +/* UCM agg counter flag selection (HW) */ +#define DQ_UCM_AGG_FLG_SHIFT_CF0 0 +#define DQ_UCM_AGG_FLG_SHIFT_CF1 1 +#define DQ_UCM_AGG_FLG_SHIFT_CF3 2 +#define DQ_UCM_AGG_FLG_SHIFT_CF4 3 +#define DQ_UCM_AGG_FLG_SHIFT_CF5 4 +#define DQ_UCM_AGG_FLG_SHIFT_CF6 5 +#define DQ_UCM_AGG_FLG_SHIFT_RULE0EN 6 +#define DQ_UCM_AGG_FLG_SHIFT_RULE1EN 7 + +/* UCM agg counter flag selection (FW) */ +#define DQ_UCM_ETH_PMD_TX_ARM_CMD (1 << DQ_UCM_AGG_FLG_SHIFT_CF4) +#define DQ_UCM_ETH_PMD_RX_ARM_CMD (1 << DQ_UCM_AGG_FLG_SHIFT_CF5) +#define DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD (1 << DQ_UCM_AGG_FLG_SHIFT_CF4) +#define DQ_UCM_ROCE_CQ_ARM_CF_CMD (1 << DQ_UCM_AGG_FLG_SHIFT_CF5) +#define DQ_UCM_TOE_TIMER_STOP_ALL_CMD (1 << DQ_UCM_AGG_FLG_SHIFT_CF3) +#define DQ_UCM_TOE_SLOW_PATH_CF_CMD (1 << DQ_UCM_AGG_FLG_SHIFT_CF4) +#define DQ_UCM_TOE_DQ_CF_CMD (1 << DQ_UCM_AGG_FLG_SHIFT_CF5) + +/* TCM agg counter flag selection (HW) */ +#define DQ_TCM_AGG_FLG_SHIFT_CF0 0 +#define DQ_TCM_AGG_FLG_SHIFT_CF1 1 +#define DQ_TCM_AGG_FLG_SHIFT_CF2 2 +#define DQ_TCM_AGG_FLG_SHIFT_CF3 3 +#define DQ_TCM_AGG_FLG_SHIFT_CF4 4 +#define DQ_TCM_AGG_FLG_SHIFT_CF5 5 +#define DQ_TCM_AGG_FLG_SHIFT_CF6 6 +#define DQ_TCM_AGG_FLG_SHIFT_CF7 7 + +/* TCM agg counter flag selection (FW) */ +#define DQ_TCM_FCOE_FLUSH_Q0_CMD (1 << DQ_TCM_AGG_FLG_SHIFT_CF1) +#define DQ_TCM_FCOE_DUMMY_TIMER_CMD (1 << DQ_TCM_AGG_FLG_SHIFT_CF2) +#define DQ_TCM_FCOE_TIMER_STOP_ALL_CMD (1 << DQ_TCM_AGG_FLG_SHIFT_CF3) +#define DQ_TCM_ISCSI_FLUSH_Q0_CMD (1 << DQ_TCM_AGG_FLG_SHIFT_CF1) +#define DQ_TCM_ISCSI_TIMER_STOP_ALL_CMD (1 << DQ_TCM_AGG_FLG_SHIFT_CF3) +#define DQ_TCM_TOE_FLUSH_Q0_CMD (1 << DQ_TCM_AGG_FLG_SHIFT_CF1) +#define DQ_TCM_TOE_TIMER_STOP_ALL_CMD (1 << DQ_TCM_AGG_FLG_SHIFT_CF3) +#define DQ_TCM_IWARP_POST_RQ_CF_CMD (1 << DQ_TCM_AGG_FLG_SHIFT_CF1) + +/* PWM address mapping */ +#define DQ_PWM_OFFSET_DPM_BASE 0x0 +#define DQ_PWM_OFFSET_DPM_END 0x27 +#define DQ_PWM_OFFSET_XCM16_BASE 0x40 +#define DQ_PWM_OFFSET_XCM32_BASE 0x44 +#define DQ_PWM_OFFSET_UCM16_BASE 0x48 +#define DQ_PWM_OFFSET_UCM32_BASE 0x4C +#define DQ_PWM_OFFSET_UCM16_4 0x50 +#define DQ_PWM_OFFSET_TCM16_BASE 0x58 +#define DQ_PWM_OFFSET_TCM32_BASE 0x5C +#define DQ_PWM_OFFSET_XCM_FLAGS 0x68 +#define DQ_PWM_OFFSET_UCM_FLAGS 0x69 +#define DQ_PWM_OFFSET_TCM_FLAGS 0x6B + +#define DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD (DQ_PWM_OFFSET_XCM16_BASE + 2) +#define DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT (DQ_PWM_OFFSET_UCM32_BASE) +#define DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_16BIT (DQ_PWM_OFFSET_UCM16_4) +#define DQ_PWM_OFFSET_UCM_RDMA_INT_TIMEOUT (DQ_PWM_OFFSET_UCM16_BASE + 2) +#define DQ_PWM_OFFSET_UCM_RDMA_ARM_FLAGS (DQ_PWM_OFFSET_UCM_FLAGS) +#define DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD (DQ_PWM_OFFSET_TCM16_BASE + 1) +#define DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD (DQ_PWM_OFFSET_TCM16_BASE + 3) + +#define DQ_REGION_SHIFT (12) + +/* DPM */ +#define DQ_DPM_WQE_BUFF_SIZE (320) + +/* Conn type ranges */ +#define DQ_CONN_TYPE_RANGE_SHIFT (4) /*****************/ /* QM CONSTANTS */ @@ -152,11 +385,17 @@ /* number of queues in a PF queue group */ #define QM_PF_QUEUE_GROUP_SIZE 8 +/* the size of a single queue element in bytes */ +#define QM_PQ_ELEMENT_SIZE 4 + /* base number of Tx PQs in the CM PQ representation. * should be used when storing PQ IDs in CM PQ registers and context */ #define CM_TX_PQ_BASE 0x200 +/* number of global Vport/QCN rate limiters */ +#define MAX_QM_GLOBAL_RLS 256 + /* QM registers data */ #define QM_LINE_CRD_REG_WIDTH 16 #define QM_LINE_CRD_REG_SIGN_BIT (1 << (QM_LINE_CRD_REG_WIDTH - 1)) @@ -175,45 +414,51 @@ #define CAU_FSM_ETH_TX 1 /* Number of Protocol Indices per Status Block */ -#define PIS_PER_SB 12 +#define PIS_PER_SB_E4 12 +#define PIS_PER_SB_E5 8 +#define MAX_PIS_PER_SB_E4 OSAL_MAX_T(PIS_PER_SB_E4, PIS_PER_SB_E5) + +/* fsm is stopped or not valid for this sb */ +#define CAU_HC_STOPPED_STATE 3 +/* fsm is working without interrupt coalescing for this sb*/ +#define CAU_HC_DISABLE_STATE 4 +/* fsm is working with interrupt coalescing for this sb*/ +#define CAU_HC_ENABLE_STATE 0 -#define CAU_HC_STOPPED_STATE 3 -#define CAU_HC_DISABLE_STATE 4 -#define CAU_HC_ENABLE_STATE 0 /*****************/ /* IGU CONSTANTS */ /*****************/ -#define MAX_SB_PER_PATH_K2 (368) -#define MAX_SB_PER_PATH_BB (288) -#define MAX_TOT_SB_PER_PATH \ - MAX_SB_PER_PATH_K2 +#define MAX_SB_PER_PATH_K2 (368) +#define MAX_SB_PER_PATH_BB (288) +#define MAX_SB_PER_PATH_E5 (512) +#define MAX_TOT_SB_PER_PATH MAX_SB_PER_PATH_E5 -#define MAX_SB_PER_PF_MIMD 129 -#define MAX_SB_PER_PF_SIMD 64 -#define MAX_SB_PER_VF 64 +#define MAX_SB_PER_PF_MIMD 129 +#define MAX_SB_PER_PF_SIMD 64 +#define MAX_SB_PER_VF 64 /* Memory addresses on the BAR for the IGU Sub Block */ -#define IGU_MEM_BASE 0x0000 +#define IGU_MEM_BASE 0x0000 -#define IGU_MEM_MSIX_BASE 0x0000 -#define IGU_MEM_MSIX_UPPER 0x0101 -#define IGU_MEM_MSIX_RESERVED_UPPER 0x01ff +#define IGU_MEM_MSIX_BASE 0x0000 +#define IGU_MEM_MSIX_UPPER 0x0101 +#define IGU_MEM_MSIX_RESERVED_UPPER 0x01ff -#define IGU_MEM_PBA_MSIX_BASE 0x0200 -#define IGU_MEM_PBA_MSIX_UPPER 0x0202 -#define IGU_MEM_PBA_MSIX_RESERVED_UPPER 0x03ff +#define IGU_MEM_PBA_MSIX_BASE 0x0200 +#define IGU_MEM_PBA_MSIX_UPPER 0x0202 +#define IGU_MEM_PBA_MSIX_RESERVED_UPPER 0x03ff -#define IGU_CMD_INT_ACK_BASE 0x0400 -#define IGU_CMD_INT_ACK_UPPER (IGU_CMD_INT_ACK_BASE + \ - MAX_TOT_SB_PER_PATH - \ - 1) -#define IGU_CMD_INT_ACK_RESERVED_UPPER 0x05ff +#define IGU_CMD_INT_ACK_BASE 0x0400 +#define IGU_CMD_INT_ACK_UPPER (IGU_CMD_INT_ACK_BASE + \ + MAX_TOT_SB_PER_PATH - \ + 1) +#define IGU_CMD_INT_ACK_RESERVED_UPPER 0x05ff -#define IGU_CMD_ATTN_BIT_UPD_UPPER 0x05f0 -#define IGU_CMD_ATTN_BIT_SET_UPPER 0x05f1 -#define IGU_CMD_ATTN_BIT_CLR_UPPER 0x05f2 +#define IGU_CMD_ATTN_BIT_UPD_UPPER 0x05f0 +#define IGU_CMD_ATTN_BIT_SET_UPPER 0x05f1 +#define IGU_CMD_ATTN_BIT_CLR_UPPER 0x05f2 #define IGU_REG_SISR_MDPC_WMASK_UPPER 0x05f3 #define IGU_REG_SISR_MDPC_WMASK_LSB_UPPER 0x05f4 @@ -221,8 +466,8 @@ #define IGU_REG_SISR_MDPC_WOMASK_UPPER 0x05f6 #define IGU_CMD_PROD_UPD_BASE 0x0600 -#define IGU_CMD_PROD_UPD_UPPER (IGU_CMD_PROD_UPD_BASE +\ - MAX_TOT_SB_PER_PATH - \ +#define IGU_CMD_PROD_UPD_UPPER (IGU_CMD_PROD_UPD_BASE + \ + MAX_TOT_SB_PER_PATH - \ 1) #define IGU_CMD_PROD_UPD_RESERVED_UPPER 0x07ff @@ -230,21 +475,31 @@ /* PXP CONSTANTS */ /*****************/ +/* Bars for Blocks */ +#define PXP_BAR_GRC 0 +#define PXP_BAR_TSDM 0 +#define PXP_BAR_USDM 0 +#define PXP_BAR_XSDM 0 +#define PXP_BAR_MSDM 0 +#define PXP_BAR_YSDM 0 +#define PXP_BAR_PSDM 0 +#define PXP_BAR_IGU 0 +#define PXP_BAR_DQ 1 + /* PTT and GTT */ -#define PXP_NUM_PF_WINDOWS 12 #define PXP_PER_PF_ENTRY_SIZE 8 #define PXP_NUM_GLOBAL_WINDOWS 243 #define PXP_GLOBAL_ENTRY_SIZE 4 -#define PXP_ADMIN_WINDOW_ALLOWED_LENGTH 4 +#define PXP_ADMIN_WINDOW_ALLOWED_LENGTH 4 #define PXP_PF_WINDOW_ADMIN_START 0 #define PXP_PF_WINDOW_ADMIN_LENGTH 0x1000 #define PXP_PF_WINDOW_ADMIN_END (PXP_PF_WINDOW_ADMIN_START + \ - PXP_PF_WINDOW_ADMIN_LENGTH - 1) + PXP_PF_WINDOW_ADMIN_LENGTH - 1) #define PXP_PF_WINDOW_ADMIN_PER_PF_START 0 #define PXP_PF_WINDOW_ADMIN_PER_PF_LENGTH (PXP_NUM_PF_WINDOWS * \ PXP_PER_PF_ENTRY_SIZE) -#define PXP_PF_WINDOW_ADMIN_PER_PF_END (PXP_PF_WINDOW_ADMIN_PER_PF_START + \ - PXP_PF_WINDOW_ADMIN_PER_PF_LENGTH - 1) +#define PXP_PF_WINDOW_ADMIN_PER_PF_END (PXP_PF_WINDOW_ADMIN_PER_PF_START + \ + PXP_PF_WINDOW_ADMIN_PER_PF_LENGTH - 1) #define PXP_PF_WINDOW_ADMIN_GLOBAL_START 0x200 #define PXP_PF_WINDOW_ADMIN_GLOBAL_LENGTH (PXP_NUM_GLOBAL_WINDOWS * \ PXP_GLOBAL_ENTRY_SIZE) @@ -256,6 +511,8 @@ #define PXP_PF_ME_OPAQUE_ADDR 0x1f8 #define PXP_PF_ME_CONCRETE_ADDR 0x1fc +#define PXP_NUM_PF_WINDOWS 12 + #define PXP_EXTERNAL_BAR_PF_WINDOW_START 0x1000 #define PXP_EXTERNAL_BAR_PF_WINDOW_NUM PXP_NUM_PF_WINDOWS #define PXP_EXTERNAL_BAR_PF_WINDOW_SINGLE_SIZE 0x1000 @@ -277,20 +534,168 @@ (PXP_EXTERNAL_BAR_GLOBAL_WINDOW_START + \ PXP_EXTERNAL_BAR_GLOBAL_WINDOW_LENGTH - 1) -#define PXP_ILT_PAGE_SIZE_NUM_BITS_MIN 12 -#define PXP_ILT_BLOCK_FACTOR_MULTIPLIER 1024 +/* PF BAR */ +#define PXP_BAR0_START_GRC 0x0000 +#define PXP_BAR0_GRC_LENGTH 0x1C00000 +#define PXP_BAR0_END_GRC \ + (PXP_BAR0_START_GRC + PXP_BAR0_GRC_LENGTH - 1) -/* ILT Records */ +#define PXP_BAR0_START_IGU 0x1C00000 +#define PXP_BAR0_IGU_LENGTH 0x10000 +#define PXP_BAR0_END_IGU \ + (PXP_BAR0_START_IGU + PXP_BAR0_IGU_LENGTH - 1) + +#define PXP_BAR0_START_TSDM 0x1C80000 +#define PXP_BAR0_SDM_LENGTH 0x40000 +#define PXP_BAR0_SDM_RESERVED_LENGTH 0x40000 +#define PXP_BAR0_END_TSDM \ + (PXP_BAR0_START_TSDM + PXP_BAR0_SDM_LENGTH - 1) + +#define PXP_BAR0_START_MSDM 0x1D00000 +#define PXP_BAR0_END_MSDM \ + (PXP_BAR0_START_MSDM + PXP_BAR0_SDM_LENGTH - 1) + +#define PXP_BAR0_START_USDM 0x1D80000 +#define PXP_BAR0_END_USDM \ + (PXP_BAR0_START_USDM + PXP_BAR0_SDM_LENGTH - 1) + +#define PXP_BAR0_START_XSDM 0x1E00000 +#define PXP_BAR0_END_XSDM \ + (PXP_BAR0_START_XSDM + PXP_BAR0_SDM_LENGTH - 1) + +#define PXP_BAR0_START_YSDM 0x1E80000 +#define PXP_BAR0_END_YSDM \ + (PXP_BAR0_START_YSDM + PXP_BAR0_SDM_LENGTH - 1) + +#define PXP_BAR0_START_PSDM 0x1F00000 +#define PXP_BAR0_END_PSDM \ + (PXP_BAR0_START_PSDM + PXP_BAR0_SDM_LENGTH - 1) + +#define PXP_BAR0_FIRST_INVALID_ADDRESS \ + (PXP_BAR0_END_PSDM + 1) + +/* VF BAR */ +#define PXP_VF_BAR0 0 + +#define PXP_VF_BAR0_START_IGU 0 +#define PXP_VF_BAR0_IGU_LENGTH 0x3000 +#define PXP_VF_BAR0_END_IGU \ + (PXP_VF_BAR0_START_IGU + PXP_VF_BAR0_IGU_LENGTH - 1) + +#define PXP_VF_BAR0_START_DQ 0x3000 +#define PXP_VF_BAR0_DQ_LENGTH 0x200 +#define PXP_VF_BAR0_DQ_OPAQUE_OFFSET 0 +#define PXP_VF_BAR0_ME_OPAQUE_ADDRESS \ + (PXP_VF_BAR0_START_DQ + PXP_VF_BAR0_DQ_OPAQUE_OFFSET) +#define PXP_VF_BAR0_ME_CONCRETE_ADDRESS \ + (PXP_VF_BAR0_ME_OPAQUE_ADDRESS + 4) +#define PXP_VF_BAR0_END_DQ \ + (PXP_VF_BAR0_START_DQ + PXP_VF_BAR0_DQ_LENGTH - 1) + +#define PXP_VF_BAR0_START_TSDM_ZONE_B 0x3200 +#define PXP_VF_BAR0_SDM_LENGTH_ZONE_B 0x200 +#define PXP_VF_BAR0_END_TSDM_ZONE_B \ + (PXP_VF_BAR0_START_TSDM_ZONE_B + PXP_VF_BAR0_SDM_LENGTH_ZONE_B - 1) + +#define PXP_VF_BAR0_START_MSDM_ZONE_B 0x3400 +#define PXP_VF_BAR0_END_MSDM_ZONE_B \ + (PXP_VF_BAR0_START_MSDM_ZONE_B + PXP_VF_BAR0_SDM_LENGTH_ZONE_B - 1) + +#define PXP_VF_BAR0_START_USDM_ZONE_B 0x3600 +#define PXP_VF_BAR0_END_USDM_ZONE_B \ + (PXP_VF_BAR0_START_USDM_ZONE_B + PXP_VF_BAR0_SDM_LENGTH_ZONE_B - 1) + +#define PXP_VF_BAR0_START_XSDM_ZONE_B 0x3800 +#define PXP_VF_BAR0_END_XSDM_ZONE_B \ + (PXP_VF_BAR0_START_XSDM_ZONE_B + PXP_VF_BAR0_SDM_LENGTH_ZONE_B - 1) + +#define PXP_VF_BAR0_START_YSDM_ZONE_B 0x3a00 +#define PXP_VF_BAR0_END_YSDM_ZONE_B \ + (PXP_VF_BAR0_START_YSDM_ZONE_B + PXP_VF_BAR0_SDM_LENGTH_ZONE_B - 1) + +#define PXP_VF_BAR0_START_PSDM_ZONE_B 0x3c00 +#define PXP_VF_BAR0_END_PSDM_ZONE_B \ + (PXP_VF_BAR0_START_PSDM_ZONE_B + PXP_VF_BAR0_SDM_LENGTH_ZONE_B - 1) + +#define PXP_VF_BAR0_START_GRC 0x3E00 +#define PXP_VF_BAR0_GRC_LENGTH 0x200 +#define PXP_VF_BAR0_END_GRC \ + (PXP_VF_BAR0_START_GRC + PXP_VF_BAR0_GRC_LENGTH - 1) + +#define PXP_VF_BAR0_START_SDM_ZONE_A 0x4000 +#define PXP_VF_BAR0_END_SDM_ZONE_A 0x10000 + +#define PXP_VF_BAR0_START_IGU2 0x10000 +#define PXP_VF_BAR0_IGU2_LENGTH 0xD000 +#define PXP_VF_BAR0_END_IGU2 \ + (PXP_VF_BAR0_START_IGU2 + PXP_VF_BAR0_IGU2_LENGTH - 1) + +#define PXP_VF_BAR0_GRC_WINDOW_LENGTH 32 + +#define PXP_ILT_PAGE_SIZE_NUM_BITS_MIN 12 +#define PXP_ILT_BLOCK_FACTOR_MULTIPLIER 1024 + +// ILT Records #define PXP_NUM_ILT_RECORDS_BB 7600 #define PXP_NUM_ILT_RECORDS_K2 11000 -#define MAX_NUM_ILT_RECORDS MAX(PXP_NUM_ILT_RECORDS_BB, PXP_NUM_ILT_RECORDS_K2) +#define MAX_NUM_ILT_RECORDS \ + OSAL_MAX_T(PXP_NUM_ILT_RECORDS_BB, PXP_NUM_ILT_RECORDS_K2) + +#define PXP_NUM_ILT_RECORDS_E5 13664 + + +// Host Interface +#define PXP_QUEUES_ZONE_MAX_NUM_E4 320 +#define PXP_QUEUES_ZONE_MAX_NUM_E5 512 + + +/*****************/ +/* PRM CONSTANTS */ +/*****************/ +#define PRM_DMA_PAD_BYTES_NUM 2 +/*****************/ +/* SDMs CONSTANTS */ +/*****************/ + + +#define SDM_OP_GEN_TRIG_NONE 0 +#define SDM_OP_GEN_TRIG_WAKE_THREAD 1 +#define SDM_OP_GEN_TRIG_AGG_INT 2 +#define SDM_OP_GEN_TRIG_LOADER 4 +#define SDM_OP_GEN_TRIG_INDICATE_ERROR 6 +#define SDM_OP_GEN_TRIG_INC_ORDER_CNT 9 + +/***********************************************************/ +/* Completion types */ +/***********************************************************/ + +#define SDM_COMP_TYPE_NONE 0 +#define SDM_COMP_TYPE_WAKE_THREAD 1 +#define SDM_COMP_TYPE_AGG_INT 2 +/* Send direct message to local CM and/or remote CMs. Destinations are defined + * by vector in CompParams. + */ +#define SDM_COMP_TYPE_CM 3 +#define SDM_COMP_TYPE_LOADER 4 +/* Send direct message to PXP (like "internal write" command) to write to remote + * Storm RAM via remote SDM + */ +#define SDM_COMP_TYPE_PXP 5 +/* Indicate error per thread */ +#define SDM_COMP_TYPE_INDICATE_ERROR 6 +#define SDM_COMP_TYPE_RELEASE_THREAD 7 +/* Write to local RAM as a completion */ +#define SDM_COMP_TYPE_RAM 8 +#define SDM_COMP_TYPE_INC_ORDER_CNT 9 /* Applicable only for E4 */ + /******************/ /* PBF CONSTANTS */ /******************/ /* Number of PBF command queue lines. Each line is 32B. */ -#define PBF_MAX_CMD_LINES 3328 +#define PBF_MAX_CMD_LINES_E4 3328 +#define PBF_MAX_CMD_LINES_E5 5280 /* Number of BTB blocks. Each block is 256B. */ #define BTB_MAX_BLOCKS 1440 @@ -299,79 +704,80 @@ /* PRS CONSTANTS */ /*****************/ -/* Async data KCQ CQE */ -struct async_data { - __le32 cid; - __le16 itid; - u8 error_code; - u8 fw_debug_param; +#define PRS_GFT_CAM_LINES_NO_MATCH 31 + +/* + * Interrupt coalescing TimeSet + */ +struct coalescing_timeset { + u8 value; +/* Interrupt coalescing TimeSet (timeout_ticks = TimeSet shl (TimerRes+1)) */ +#define COALESCING_TIMESET_TIMESET_MASK 0x7F +#define COALESCING_TIMESET_TIMESET_SHIFT 0 +/* Only if this flag is set, timeset will take effect */ +#define COALESCING_TIMESET_VALID_MASK 0x1 +#define COALESCING_TIMESET_VALID_SHIFT 7 }; -struct regpair { - __le32 lo /* low word for reg-pair */; - __le32 hi /* high word for reg-pair */; +struct common_queue_zone { + __le16 ring_drv_data_consumer; + __le16 reserved; }; -struct vf_pf_channel_eqe_data { - struct regpair msg_addr /* VF-PF message address */; +/* + * ETH Rx producers data + */ +struct eth_rx_prod_data { + __le16 bd_prod /* BD producer. */; + __le16 cqe_prod /* CQE producer. */; }; + +struct tcp_ulp_connect_done_params { + __le16 mss; + u8 snd_wnd_scale; + u8 flags; +#define TCP_ULP_CONNECT_DONE_PARAMS_TS_EN_MASK 0x1 +#define TCP_ULP_CONNECT_DONE_PARAMS_TS_EN_SHIFT 0 +#define TCP_ULP_CONNECT_DONE_PARAMS_RESERVED_MASK 0x7F +#define TCP_ULP_CONNECT_DONE_PARAMS_RESERVED_SHIFT 1 +}; + +struct iscsi_connect_done_results { + __le16 icid /* Context ID of the connection */; + __le16 conn_id /* Driver connection ID */; +/* decided tcp params after connect done */ + struct tcp_ulp_connect_done_params params; +}; + + struct iscsi_eqe_data { - __le32 cid /* Context ID of the connection */; - __le16 conn_id - /* Task Id of the task (for error that happened on a a task) */; + __le16 icid /* Context ID of the connection */; + __le16 conn_id /* Driver connection ID */; + __le16 reserved; +/* error code - relevant only if the opcode indicates its an error */ u8 error_code; - u8 reserved0; + u8 error_pdu_opcode_reserved; +/* The processed PDUs opcode on which happened the error - updated for specific + * error codes, by default=0xFF + */ +#define ISCSI_EQE_DATA_ERROR_PDU_OPCODE_MASK 0x3F +#define ISCSI_EQE_DATA_ERROR_PDU_OPCODE_SHIFT 0 +/* Indication for driver is the error_pdu_opcode field has valid value */ +#define ISCSI_EQE_DATA_ERROR_PDU_OPCODE_VALID_MASK 0x1 +#define ISCSI_EQE_DATA_ERROR_PDU_OPCODE_VALID_SHIFT 6 +#define ISCSI_EQE_DATA_RESERVED0_MASK 0x1 +#define ISCSI_EQE_DATA_RESERVED0_SHIFT 7 }; + /* - * Event Ring malicious VF data + * Multi function mode */ -struct malicious_vf_eqe_data { - u8 vf_id /* Malicious VF ID */; /* WARNING:CAMELCASE */ - u8 err_id /* Malicious VF error */; - __le16 reserved[3]; -}; - -/* - * Event Ring initial cleanup data - */ -struct initial_cleanup_eqe_data { - u8 vf_id /* VF ID */; /* WARNING:CAMELCASE */ - u8 reserved[7]; -}; - - -union event_ring_data { - u8 bytes[8] /* Byte Array */; - struct vf_pf_channel_eqe_data vf_pf_channel /* VF-PF Channel data */; - struct iscsi_eqe_data iscsi_info /* Dedicated fields to iscsi data */; - struct regpair roce_handle /* WARNING:CAMELCASE */ - /* Dedicated field for RoCE affiliated asynchronous error */; - struct malicious_vf_eqe_data malicious_vf /* Malicious VF data */; - struct initial_cleanup_eqe_data vf_init_cleanup - /* VF Initial Cleanup data */; -}; -/* Event Ring Entry */ -struct event_ring_entry { - u8 protocol_id; - u8 opcode; - __le16 reserved0; - __le16 echo; - u8 fw_return_code; - u8 flags; -#define EVENT_RING_ENTRY_ASYNC_MASK 0x1 -#define EVENT_RING_ENTRY_ASYNC_SHIFT 0 -#define EVENT_RING_ENTRY_RESERVED1_MASK 0x7F -#define EVENT_RING_ENTRY_RESERVED1_SHIFT 1 - union event_ring_data data; -}; - -/* Multi function mode */ enum mf_mode { - SF, - MF_OVLAN, - MF_NPAR, + ERROR_MODE /* Unsupported mode */, + MF_OVLAN /* Multi function based on outer VLAN */, + MF_NPAR /* Multi function based on MAC address (NIC partitioning) */, MAX_MF_MODE }; @@ -390,35 +796,67 @@ enum protocol_type { MAX_PROTOCOL_TYPE }; + +struct regpair { + __le32 lo /* low word for reg-pair */; + __le32 hi /* high word for reg-pair */; +}; + + + +/* + * Ustorm Queue Zone + */ +struct ustorm_eth_queue_zone { +/* Rx interrupt coalescing TimeSet */ + struct coalescing_timeset int_coalescing_timeset; + u8 reserved[3]; +}; + + +struct ustorm_queue_zone { + struct ustorm_eth_queue_zone eth; + struct common_queue_zone common; +}; + /* status block structure */ struct cau_pi_entry { - u32 prod; + __le32 prod; +/* A per protocol indexPROD value. */ #define CAU_PI_ENTRY_PROD_VAL_MASK 0xFFFF #define CAU_PI_ENTRY_PROD_VAL_SHIFT 0 +/* This value determines the TimeSet that the PI is associated with */ #define CAU_PI_ENTRY_PI_TIMESET_MASK 0x7F #define CAU_PI_ENTRY_PI_TIMESET_SHIFT 16 +/* Select the FSM within the SB */ #define CAU_PI_ENTRY_FSM_SEL_MASK 0x1 #define CAU_PI_ENTRY_FSM_SEL_SHIFT 23 +/* Select the FSM within the SB */ #define CAU_PI_ENTRY_RESERVED_MASK 0xFF #define CAU_PI_ENTRY_RESERVED_SHIFT 24 }; /* status block structure */ struct cau_sb_entry { - u32 data; + __le32 data; +/* The SB PROD index which is sent to the IGU. */ #define CAU_SB_ENTRY_SB_PROD_MASK 0xFFFFFF #define CAU_SB_ENTRY_SB_PROD_SHIFT 0 -#define CAU_SB_ENTRY_STATE0_MASK 0xF +#define CAU_SB_ENTRY_STATE0_MASK 0xF /* RX state */ #define CAU_SB_ENTRY_STATE0_SHIFT 24 -#define CAU_SB_ENTRY_STATE1_MASK 0xF +#define CAU_SB_ENTRY_STATE1_MASK 0xF /* TX state */ #define CAU_SB_ENTRY_STATE1_SHIFT 28 - u32 params; + __le32 params; +/* Indicates the RX TimeSet that this SB is associated with. */ #define CAU_SB_ENTRY_SB_TIMESET0_MASK 0x7F #define CAU_SB_ENTRY_SB_TIMESET0_SHIFT 0 +/* Indicates the TX TimeSet that this SB is associated with. */ #define CAU_SB_ENTRY_SB_TIMESET1_MASK 0x7F #define CAU_SB_ENTRY_SB_TIMESET1_SHIFT 7 +/* This value will determine the RX FSM timer resolution in ticks */ #define CAU_SB_ENTRY_TIMER_RES0_MASK 0x3 #define CAU_SB_ENTRY_TIMER_RES0_SHIFT 14 +/* This value will determine the TX FSM timer resolution in ticks */ #define CAU_SB_ENTRY_TIMER_RES1_MASK 0x3 #define CAU_SB_ENTRY_TIMER_RES1_SHIFT 16 #define CAU_SB_ENTRY_VF_NUMBER_MASK 0xFF @@ -427,56 +865,199 @@ struct cau_sb_entry { #define CAU_SB_ENTRY_VF_VALID_SHIFT 26 #define CAU_SB_ENTRY_PF_NUMBER_MASK 0xF #define CAU_SB_ENTRY_PF_NUMBER_SHIFT 27 +/* If set then indicates that the TPH STAG is equal to the SB number. Otherwise + * the STAG will be equal to all ones. + */ #define CAU_SB_ENTRY_TPH_MASK 0x1 #define CAU_SB_ENTRY_TPH_SHIFT 31 }; + +/* + * Igu cleanup bit values to distinguish between clean or producer consumer + * update. + */ +enum command_type_bit { + IGU_COMMAND_TYPE_NOP = 0, + IGU_COMMAND_TYPE_SET = 1, + MAX_COMMAND_TYPE_BIT +}; + + /* core doorbell data */ struct core_db_data { u8 params; +/* destination of doorbell (use enum db_dest) */ #define CORE_DB_DATA_DEST_MASK 0x3 #define CORE_DB_DATA_DEST_SHIFT 0 +/* aggregative command to CM (use enum db_agg_cmd_sel) */ #define CORE_DB_DATA_AGG_CMD_MASK 0x3 #define CORE_DB_DATA_AGG_CMD_SHIFT 2 -#define CORE_DB_DATA_BYPASS_EN_MASK 0x1 +#define CORE_DB_DATA_BYPASS_EN_MASK 0x1 /* enable QM bypass */ #define CORE_DB_DATA_BYPASS_EN_SHIFT 4 #define CORE_DB_DATA_RESERVED_MASK 0x1 #define CORE_DB_DATA_RESERVED_SHIFT 5 +/* aggregative value selection */ #define CORE_DB_DATA_AGG_VAL_SEL_MASK 0x3 #define CORE_DB_DATA_AGG_VAL_SEL_SHIFT 6 +/* bit for every DQ counter flags in CM context that DQ can increment */ u8 agg_flags; __le16 spq_prod; }; /* Enum of doorbell aggregative command selection */ enum db_agg_cmd_sel { - DB_AGG_CMD_NOP, - DB_AGG_CMD_SET, - DB_AGG_CMD_ADD, - DB_AGG_CMD_MAX, + DB_AGG_CMD_NOP /* No operation */, + DB_AGG_CMD_SET /* Set the value */, + DB_AGG_CMD_ADD /* Add the value */, + DB_AGG_CMD_MAX /* Set max of current and new value */, MAX_DB_AGG_CMD_SEL }; /* Enum of doorbell destination */ enum db_dest { - DB_DEST_XCM, - DB_DEST_UCM, - DB_DEST_TCM, + DB_DEST_XCM /* TX doorbell to XCM */, + DB_DEST_UCM /* RX doorbell to UCM */, + DB_DEST_TCM /* RX doorbell to TCM */, DB_NUM_DESTINATIONS, MAX_DB_DEST }; + +/* + * Enum of doorbell DPM types + */ +enum db_dpm_type { + DPM_LEGACY /* Legacy DPM- to Xstorm RAM */, + DPM_RDMA /* RDMA DPM (only RoCE in E4) - to NIG */, +/* L2 DPM inline- to PBF, with packet data on doorbell */ + DPM_L2_INLINE, + DPM_L2_BD /* L2 DPM with BD- to PBF, with TX BD data on doorbell */, + MAX_DB_DPM_TYPE +}; + +/* + * Structure for doorbell data, in L2 DPM mode, for the first doorbell in a DPM + * burst + */ +struct db_l2_dpm_data { + __le16 icid /* internal CID */; + __le16 bd_prod /* bd producer value to update */; + __le32 params; +/* Size in QWORD-s of the DPM burst */ +#define DB_L2_DPM_DATA_SIZE_MASK 0x3F +#define DB_L2_DPM_DATA_SIZE_SHIFT 0 +/* Type of DPM transaction (DPM_L2_INLINE or DPM_L2_BD) (use enum db_dpm_type) + */ +#define DB_L2_DPM_DATA_DPM_TYPE_MASK 0x3 +#define DB_L2_DPM_DATA_DPM_TYPE_SHIFT 6 +#define DB_L2_DPM_DATA_NUM_BDS_MASK 0xFF /* number of BD-s */ +#define DB_L2_DPM_DATA_NUM_BDS_SHIFT 8 +/* size of the packet to be transmitted in bytes */ +#define DB_L2_DPM_DATA_PKT_SIZE_MASK 0x7FF +#define DB_L2_DPM_DATA_PKT_SIZE_SHIFT 16 +#define DB_L2_DPM_DATA_RESERVED0_MASK 0x1 +#define DB_L2_DPM_DATA_RESERVED0_SHIFT 27 +/* In DPM_L2_BD mode: the number of SGE-s */ +#define DB_L2_DPM_DATA_SGE_NUM_MASK 0x7 +#define DB_L2_DPM_DATA_SGE_NUM_SHIFT 28 +/* Flag indicating whether to enable GFS search */ +#define DB_L2_DPM_DATA_RESERVED1_MASK 0x1 +#define DB_L2_DPM_DATA_RESERVED1_SHIFT 31 +}; + +/* + * Structure for SGE in a DPM doorbell of type DPM_L2_BD + */ +struct db_l2_dpm_sge { + struct regpair addr /* Single continuous buffer */; + __le16 nbytes /* Number of bytes in this BD. */; + __le16 bitfields; +/* The TPH STAG index value */ +#define DB_L2_DPM_SGE_TPH_ST_INDEX_MASK 0x1FF +#define DB_L2_DPM_SGE_TPH_ST_INDEX_SHIFT 0 +#define DB_L2_DPM_SGE_RESERVED0_MASK 0x3 +#define DB_L2_DPM_SGE_RESERVED0_SHIFT 9 +/* Indicate if ST hint is requested or not */ +#define DB_L2_DPM_SGE_ST_VALID_MASK 0x1 +#define DB_L2_DPM_SGE_ST_VALID_SHIFT 11 +#define DB_L2_DPM_SGE_RESERVED1_MASK 0xF +#define DB_L2_DPM_SGE_RESERVED1_SHIFT 12 + __le32 reserved2; +}; + /* Structure for doorbell address, in legacy mode */ struct db_legacy_addr { __le32 addr; #define DB_LEGACY_ADDR_RESERVED0_MASK 0x3 #define DB_LEGACY_ADDR_RESERVED0_SHIFT 0 +/* doorbell extraction mode specifier- 0 if not used */ #define DB_LEGACY_ADDR_DEMS_MASK 0x7 #define DB_LEGACY_ADDR_DEMS_SHIFT 2 -#define DB_LEGACY_ADDR_ICID_MASK 0x7FFFFFF +#define DB_LEGACY_ADDR_ICID_MASK 0x7FFFFFF /* internal CID */ #define DB_LEGACY_ADDR_ICID_SHIFT 5 }; +/* + * Structure for doorbell address, in PWM mode + */ +struct db_pwm_addr { + __le32 addr; +#define DB_PWM_ADDR_RESERVED0_MASK 0x7 +#define DB_PWM_ADDR_RESERVED0_SHIFT 0 +/* Offset in PWM address space */ +#define DB_PWM_ADDR_OFFSET_MASK 0x7F +#define DB_PWM_ADDR_OFFSET_SHIFT 3 +#define DB_PWM_ADDR_WID_MASK 0x3 /* Window ID */ +#define DB_PWM_ADDR_WID_SHIFT 10 +#define DB_PWM_ADDR_DPI_MASK 0xFFFF /* Doorbell page ID */ +#define DB_PWM_ADDR_DPI_SHIFT 12 +#define DB_PWM_ADDR_RESERVED1_MASK 0xF +#define DB_PWM_ADDR_RESERVED1_SHIFT 28 +}; + +/* + * Parameters to RDMA firmware, passed in EDPM doorbell + */ +struct db_rdma_dpm_params { + __le32 params; +/* Size in QWORD-s of the DPM burst */ +#define DB_RDMA_DPM_PARAMS_SIZE_MASK 0x3F +#define DB_RDMA_DPM_PARAMS_SIZE_SHIFT 0 +/* Type of DPM transacation (DPM_RDMA) (use enum db_dpm_type) */ +#define DB_RDMA_DPM_PARAMS_DPM_TYPE_MASK 0x3 +#define DB_RDMA_DPM_PARAMS_DPM_TYPE_SHIFT 6 +/* opcode for RDMA operation */ +#define DB_RDMA_DPM_PARAMS_OPCODE_MASK 0xFF +#define DB_RDMA_DPM_PARAMS_OPCODE_SHIFT 8 +/* the size of the WQE payload in bytes */ +#define DB_RDMA_DPM_PARAMS_WQE_SIZE_MASK 0x7FF +#define DB_RDMA_DPM_PARAMS_WQE_SIZE_SHIFT 16 +#define DB_RDMA_DPM_PARAMS_RESERVED0_MASK 0x1 +#define DB_RDMA_DPM_PARAMS_RESERVED0_SHIFT 27 +/* RoCE completion flag */ +#define DB_RDMA_DPM_PARAMS_COMPLETION_FLG_MASK 0x1 +#define DB_RDMA_DPM_PARAMS_COMPLETION_FLG_SHIFT 28 +#define DB_RDMA_DPM_PARAMS_S_FLG_MASK 0x1 /* RoCE S flag */ +#define DB_RDMA_DPM_PARAMS_S_FLG_SHIFT 29 +#define DB_RDMA_DPM_PARAMS_RESERVED1_MASK 0x1 +#define DB_RDMA_DPM_PARAMS_RESERVED1_SHIFT 30 +/* Connection type is iWARP */ +#define DB_RDMA_DPM_PARAMS_CONN_TYPE_IS_IWARP_MASK 0x1 +#define DB_RDMA_DPM_PARAMS_CONN_TYPE_IS_IWARP_SHIFT 31 +}; + +/* + * Structure for doorbell data, in RDMA DPM mode, for the first doorbell in a + * DPM burst + */ +struct db_rdma_dpm_data { + __le16 icid /* internal CID */; + __le16 prod_val /* aggregated value to update */; +/* parameters passed to RDMA firmware */ + struct db_rdma_dpm_params params; +}; + /* Igu interrupt command */ enum igu_int_cmd { IGU_INT_ENABLE = 0, @@ -488,22 +1069,25 @@ enum igu_int_cmd { /* IGU producer or consumer update command */ struct igu_prod_cons_update { - u32 sb_id_and_flags; + __le32 sb_id_and_flags; #define IGU_PROD_CONS_UPDATE_SB_INDEX_MASK 0xFFFFFF #define IGU_PROD_CONS_UPDATE_SB_INDEX_SHIFT 0 #define IGU_PROD_CONS_UPDATE_UPDATE_FLAG_MASK 0x1 #define IGU_PROD_CONS_UPDATE_UPDATE_FLAG_SHIFT 24 +/* interrupt enable/disable/nop (use enum igu_int_cmd) */ #define IGU_PROD_CONS_UPDATE_ENABLE_INT_MASK 0x3 #define IGU_PROD_CONS_UPDATE_ENABLE_INT_SHIFT 25 +/* (use enum igu_seg_access) */ #define IGU_PROD_CONS_UPDATE_SEGMENT_ACCESS_MASK 0x1 #define IGU_PROD_CONS_UPDATE_SEGMENT_ACCESS_SHIFT 27 #define IGU_PROD_CONS_UPDATE_TIMER_MASK_MASK 0x1 #define IGU_PROD_CONS_UPDATE_TIMER_MASK_SHIFT 28 #define IGU_PROD_CONS_UPDATE_RESERVED0_MASK 0x3 #define IGU_PROD_CONS_UPDATE_RESERVED0_SHIFT 29 +/* must always be set cleared (use enum command_type_bit) */ #define IGU_PROD_CONS_UPDATE_COMMAND_TYPE_MASK 0x1 #define IGU_PROD_CONS_UPDATE_COMMAND_TYPE_SHIFT 31 - u32 reserved1; + __le32 reserved1; }; /* Igu segments access for default status block only */ @@ -513,46 +1097,173 @@ enum igu_seg_access { MAX_IGU_SEG_ACCESS }; + +/* + * Enumeration for L3 type field of parsing_and_err_flags_union. L3Type: + * 0 - unknown (not ip) ,1 - Ipv4, 2 - Ipv6 (this field can be filled according + * to the last-ethertype) + */ +enum l3_type { + e_l3_type_unknown, + e_l3_type_ipv4, + e_l3_type_ipv6, + MAX_L3_TYPE +}; + + +/* + * Enumeration for l4Protocol field of parsing_and_err_flags_union. L4-protocol + * 0 - none, 1 - TCP, 2- UDP. if the packet is IPv4 fragment, and its not the + * first fragment, the protocol-type should be set to none. + */ +enum l4_protocol { + e_l4_protocol_none, + e_l4_protocol_tcp, + e_l4_protocol_udp, + MAX_L4_PROTOCOL +}; + + +/* + * Parsing and error flags field. + */ struct parsing_and_err_flags { __le16 flags; +/* L3Type: 0 - unknown (not ip) ,1 - Ipv4, 2 - Ipv6 (this field can be filled + * according to the last-ethertype) (use enum l3_type) + */ #define PARSING_AND_ERR_FLAGS_L3TYPE_MASK 0x3 #define PARSING_AND_ERR_FLAGS_L3TYPE_SHIFT 0 +/* L4-protocol 0 - none, 1 - TCP, 2- UDP. if the packet is IPv4 fragment, and + * its not the first fragment, the protocol-type should be set to none. + * (use enum l4_protocol) + */ #define PARSING_AND_ERR_FLAGS_L4PROTOCOL_MASK 0x3 #define PARSING_AND_ERR_FLAGS_L4PROTOCOL_SHIFT 2 +/* Set if the packet is IPv4 fragment. */ #define PARSING_AND_ERR_FLAGS_IPV4FRAG_MASK 0x1 #define PARSING_AND_ERR_FLAGS_IPV4FRAG_SHIFT 4 +/* Set if VLAN tag exists. Invalid if tunnel type are IP GRE or IP GENEVE. */ #define PARSING_AND_ERR_FLAGS_TAG8021QEXIST_MASK 0x1 #define PARSING_AND_ERR_FLAGS_TAG8021QEXIST_SHIFT 5 +/* Set if L4 checksum was calculated. */ #define PARSING_AND_ERR_FLAGS_L4CHKSMWASCALCULATED_MASK 0x1 #define PARSING_AND_ERR_FLAGS_L4CHKSMWASCALCULATED_SHIFT 6 +/* Set for PTP packet. */ #define PARSING_AND_ERR_FLAGS_TIMESYNCPKT_MASK 0x1 #define PARSING_AND_ERR_FLAGS_TIMESYNCPKT_SHIFT 7 +/* Set if PTP timestamp recorded. */ #define PARSING_AND_ERR_FLAGS_TIMESTAMPRECORDED_MASK 0x1 #define PARSING_AND_ERR_FLAGS_TIMESTAMPRECORDED_SHIFT 8 +/* Set if either version-mismatch or hdr-len-error or ipv4-cksm is set or ipv6 + * ver mismatch + */ #define PARSING_AND_ERR_FLAGS_IPHDRERROR_MASK 0x1 #define PARSING_AND_ERR_FLAGS_IPHDRERROR_SHIFT 9 +/* Set if L4 checksum validation failed. Valid only if L4 checksum was + * calculated. + */ #define PARSING_AND_ERR_FLAGS_L4CHKSMERROR_MASK 0x1 #define PARSING_AND_ERR_FLAGS_L4CHKSMERROR_SHIFT 10 +/* Set if GRE/VXLAN/GENEVE tunnel detected. */ #define PARSING_AND_ERR_FLAGS_TUNNELEXIST_MASK 0x1 #define PARSING_AND_ERR_FLAGS_TUNNELEXIST_SHIFT 11 +/* Set if VLAN tag exists in tunnel header. */ #define PARSING_AND_ERR_FLAGS_TUNNEL8021QTAGEXIST_MASK 0x1 #define PARSING_AND_ERR_FLAGS_TUNNEL8021QTAGEXIST_SHIFT 12 +/* Set if either tunnel-ipv4-version-mismatch or tunnel-ipv4-hdr-len-error or + * tunnel-ipv4-cksm is set or tunneling ipv6 ver mismatch + */ #define PARSING_AND_ERR_FLAGS_TUNNELIPHDRERROR_MASK 0x1 #define PARSING_AND_ERR_FLAGS_TUNNELIPHDRERROR_SHIFT 13 +/* Set if GRE or VXLAN/GENEVE UDP checksum was calculated. */ #define PARSING_AND_ERR_FLAGS_TUNNELL4CHKSMWASCALCULATED_MASK 0x1 #define PARSING_AND_ERR_FLAGS_TUNNELL4CHKSMWASCALCULATED_SHIFT 14 +/* Set if tunnel L4 checksum validation failed. Valid only if tunnel L4 checksum + * was calculated. + */ #define PARSING_AND_ERR_FLAGS_TUNNELL4CHKSMERROR_MASK 0x1 #define PARSING_AND_ERR_FLAGS_TUNNELL4CHKSMERROR_SHIFT 15 }; + +/* + * Parsing error flags bitmap. + */ +struct parsing_err_flags { + __le16 flags; +/* MAC error indication */ +#define PARSING_ERR_FLAGS_MAC_ERROR_MASK 0x1 +#define PARSING_ERR_FLAGS_MAC_ERROR_SHIFT 0 +/* truncation error indication */ +#define PARSING_ERR_FLAGS_TRUNC_ERROR_MASK 0x1 +#define PARSING_ERR_FLAGS_TRUNC_ERROR_SHIFT 1 +/* packet too small indication */ +#define PARSING_ERR_FLAGS_PKT_TOO_SMALL_MASK 0x1 +#define PARSING_ERR_FLAGS_PKT_TOO_SMALL_SHIFT 2 +/* Header Missing Tag */ +#define PARSING_ERR_FLAGS_ANY_HDR_MISSING_TAG_MASK 0x1 +#define PARSING_ERR_FLAGS_ANY_HDR_MISSING_TAG_SHIFT 3 +/* from frame cracker output */ +#define PARSING_ERR_FLAGS_ANY_HDR_IP_VER_MISMTCH_MASK 0x1 +#define PARSING_ERR_FLAGS_ANY_HDR_IP_VER_MISMTCH_SHIFT 4 +/* from frame cracker output */ +#define PARSING_ERR_FLAGS_ANY_HDR_IP_V4_HDR_LEN_TOO_SMALL_MASK 0x1 +#define PARSING_ERR_FLAGS_ANY_HDR_IP_V4_HDR_LEN_TOO_SMALL_SHIFT 5 +/* set this error if: 1. total-len is smaller than hdr-len 2. total-ip-len + * indicates number that is bigger than real packet length 3. tunneling: + * total-ip-length of the outer header points to offset that is smaller than + * the one pointed to by the total-ip-len of the inner hdr. + */ +#define PARSING_ERR_FLAGS_ANY_HDR_IP_BAD_TOTAL_LEN_MASK 0x1 +#define PARSING_ERR_FLAGS_ANY_HDR_IP_BAD_TOTAL_LEN_SHIFT 6 +/* from frame cracker output */ +#define PARSING_ERR_FLAGS_IP_V4_CHKSM_ERROR_MASK 0x1 +#define PARSING_ERR_FLAGS_IP_V4_CHKSM_ERROR_SHIFT 7 +/* from frame cracker output. for either TCP or UDP */ +#define PARSING_ERR_FLAGS_ANY_HDR_L4_IP_LEN_MISMTCH_MASK 0x1 +#define PARSING_ERR_FLAGS_ANY_HDR_L4_IP_LEN_MISMTCH_SHIFT 8 +/* from frame cracker output */ +#define PARSING_ERR_FLAGS_ZERO_UDP_IP_V6_CHKSM_MASK 0x1 +#define PARSING_ERR_FLAGS_ZERO_UDP_IP_V6_CHKSM_SHIFT 9 +/* cksm calculated and value isn't 0xffff or L4-cksm-wasnt-calculated for any + * reason, like: udp/ipv4 checksum is 0 etc. + */ +#define PARSING_ERR_FLAGS_INNER_L4_CHKSM_ERROR_MASK 0x1 +#define PARSING_ERR_FLAGS_INNER_L4_CHKSM_ERROR_SHIFT 10 +/* from frame cracker output */ +#define PARSING_ERR_FLAGS_ANY_HDR_ZERO_TTL_OR_HOP_LIM_MASK 0x1 +#define PARSING_ERR_FLAGS_ANY_HDR_ZERO_TTL_OR_HOP_LIM_SHIFT 11 +/* from frame cracker output */ +#define PARSING_ERR_FLAGS_NON_8021Q_TAG_EXISTS_IN_BOTH_HDRS_MASK 0x1 +#define PARSING_ERR_FLAGS_NON_8021Q_TAG_EXISTS_IN_BOTH_HDRS_SHIFT 12 +/* set if geneve option size was over 32 byte */ +#define PARSING_ERR_FLAGS_GENEVE_OPTION_OVERSIZED_MASK 0x1 +#define PARSING_ERR_FLAGS_GENEVE_OPTION_OVERSIZED_SHIFT 13 +/* from frame cracker output */ +#define PARSING_ERR_FLAGS_TUNNEL_IP_V4_CHKSM_ERROR_MASK 0x1 +#define PARSING_ERR_FLAGS_TUNNEL_IP_V4_CHKSM_ERROR_SHIFT 14 +/* from frame cracker output */ +#define PARSING_ERR_FLAGS_TUNNEL_L4_CHKSM_ERROR_MASK 0x1 +#define PARSING_ERR_FLAGS_TUNNEL_L4_CHKSM_ERROR_SHIFT 15 +}; + + +/* + * Pb context + */ +struct pb_context { + __le32 crc[4]; +}; + /* Concrete Function ID. */ struct pxp_concrete_fid { __le16 fid; -#define PXP_CONCRETE_FID_PFID_MASK 0xF +#define PXP_CONCRETE_FID_PFID_MASK 0xF /* Parent PFID */ #define PXP_CONCRETE_FID_PFID_SHIFT 0 -#define PXP_CONCRETE_FID_PORT_MASK 0x3 +#define PXP_CONCRETE_FID_PORT_MASK 0x3 /* port number */ #define PXP_CONCRETE_FID_PORT_SHIFT 4 -#define PXP_CONCRETE_FID_PATH_MASK 0x1 +#define PXP_CONCRETE_FID_PATH_MASK 0x1 /* path number */ #define PXP_CONCRETE_FID_PATH_SHIFT 6 #define PXP_CONCRETE_FID_VFVALID_MASK 0x1 #define PXP_CONCRETE_FID_VFVALID_SHIFT 7 @@ -611,104 +1322,323 @@ struct pxp_ptt_entry { struct pxp_pretend_cmd pretend; }; -/* RSS hash type */ + +/* + * VF Zone A Permission Register. + */ +struct pxp_vf_zone_a_permission { + __le32 control; +#define PXP_VF_ZONE_A_PERMISSION_VFID_MASK 0xFF +#define PXP_VF_ZONE_A_PERMISSION_VFID_SHIFT 0 +#define PXP_VF_ZONE_A_PERMISSION_VALID_MASK 0x1 +#define PXP_VF_ZONE_A_PERMISSION_VALID_SHIFT 8 +#define PXP_VF_ZONE_A_PERMISSION_RESERVED0_MASK 0x7F +#define PXP_VF_ZONE_A_PERMISSION_RESERVED0_SHIFT 9 +#define PXP_VF_ZONE_A_PERMISSION_RESERVED1_MASK 0xFFFF +#define PXP_VF_ZONE_A_PERMISSION_RESERVED1_SHIFT 16 +}; + + +/* + * Rdif context + */ +struct rdif_task_context { + __le32 initial_ref_tag; + __le16 app_tag_value; + __le16 app_tag_mask; + u8 flags0; +#define RDIF_TASK_CONTEXT_IGNORE_APP_TAG_MASK 0x1 +#define RDIF_TASK_CONTEXT_IGNORE_APP_TAG_SHIFT 0 +#define RDIF_TASK_CONTEXT_INITIAL_REF_TAG_VALID_MASK 0x1 +#define RDIF_TASK_CONTEXT_INITIAL_REF_TAG_VALID_SHIFT 1 +/* 0 = IP checksum, 1 = CRC */ +#define RDIF_TASK_CONTEXT_HOST_GUARD_TYPE_MASK 0x1 +#define RDIF_TASK_CONTEXT_HOST_GUARD_TYPE_SHIFT 2 +#define RDIF_TASK_CONTEXT_SET_ERROR_WITH_EOP_MASK 0x1 +#define RDIF_TASK_CONTEXT_SET_ERROR_WITH_EOP_SHIFT 3 +/* 1/2/3 - Protection Type */ +#define RDIF_TASK_CONTEXT_PROTECTION_TYPE_MASK 0x3 +#define RDIF_TASK_CONTEXT_PROTECTION_TYPE_SHIFT 4 +/* 0=0x0000, 1=0xffff */ +#define RDIF_TASK_CONTEXT_CRC_SEED_MASK 0x1 +#define RDIF_TASK_CONTEXT_CRC_SEED_SHIFT 6 +/* Keep reference tag constant */ +#define RDIF_TASK_CONTEXT_KEEP_REF_TAG_CONST_MASK 0x1 +#define RDIF_TASK_CONTEXT_KEEP_REF_TAG_CONST_SHIFT 7 + u8 partial_dif_data[7]; + __le16 partial_crc_value; + __le16 partial_checksum_value; + __le32 offset_in_io; + __le16 flags1; +#define RDIF_TASK_CONTEXT_VALIDATE_GUARD_MASK 0x1 +#define RDIF_TASK_CONTEXT_VALIDATE_GUARD_SHIFT 0 +#define RDIF_TASK_CONTEXT_VALIDATE_APP_TAG_MASK 0x1 +#define RDIF_TASK_CONTEXT_VALIDATE_APP_TAG_SHIFT 1 +#define RDIF_TASK_CONTEXT_VALIDATE_REF_TAG_MASK 0x1 +#define RDIF_TASK_CONTEXT_VALIDATE_REF_TAG_SHIFT 2 +#define RDIF_TASK_CONTEXT_FORWARD_GUARD_MASK 0x1 +#define RDIF_TASK_CONTEXT_FORWARD_GUARD_SHIFT 3 +#define RDIF_TASK_CONTEXT_FORWARD_APP_TAG_MASK 0x1 +#define RDIF_TASK_CONTEXT_FORWARD_APP_TAG_SHIFT 4 +#define RDIF_TASK_CONTEXT_FORWARD_REF_TAG_MASK 0x1 +#define RDIF_TASK_CONTEXT_FORWARD_REF_TAG_SHIFT 5 +/* 0=512B, 1=1KB, 2=2KB, 3=4KB, 4=8KB */ +#define RDIF_TASK_CONTEXT_INTERVAL_SIZE_MASK 0x7 +#define RDIF_TASK_CONTEXT_INTERVAL_SIZE_SHIFT 6 +/* 0=None, 1=DIF, 2=DIX */ +#define RDIF_TASK_CONTEXT_HOST_INTERFACE_MASK 0x3 +#define RDIF_TASK_CONTEXT_HOST_INTERFACE_SHIFT 9 +/* DIF tag right at the beginning of DIF interval */ +#define RDIF_TASK_CONTEXT_DIF_BEFORE_DATA_MASK 0x1 +#define RDIF_TASK_CONTEXT_DIF_BEFORE_DATA_SHIFT 11 +#define RDIF_TASK_CONTEXT_RESERVED0_MASK 0x1 +#define RDIF_TASK_CONTEXT_RESERVED0_SHIFT 12 +/* 0=None, 1=DIF */ +#define RDIF_TASK_CONTEXT_NETWORK_INTERFACE_MASK 0x1 +#define RDIF_TASK_CONTEXT_NETWORK_INTERFACE_SHIFT 13 +/* Forward application tag with mask */ +#define RDIF_TASK_CONTEXT_FORWARD_APP_TAG_WITH_MASK_MASK 0x1 +#define RDIF_TASK_CONTEXT_FORWARD_APP_TAG_WITH_MASK_SHIFT 14 +/* Forward reference tag with mask */ +#define RDIF_TASK_CONTEXT_FORWARD_REF_TAG_WITH_MASK_MASK 0x1 +#define RDIF_TASK_CONTEXT_FORWARD_REF_TAG_WITH_MASK_SHIFT 15 + __le16 state; +#define RDIF_TASK_CONTEXT_RECEIVED_DIF_BYTES_LEFT_MASK 0xF +#define RDIF_TASK_CONTEXT_RECEIVED_DIF_BYTES_LEFT_SHIFT 0 +#define RDIF_TASK_CONTEXT_TRANSMITED_DIF_BYTES_LEFT_MASK 0xF +#define RDIF_TASK_CONTEXT_TRANSMITED_DIF_BYTES_LEFT_SHIFT 4 +#define RDIF_TASK_CONTEXT_ERROR_IN_IO_MASK 0x1 +#define RDIF_TASK_CONTEXT_ERROR_IN_IO_SHIFT 8 +#define RDIF_TASK_CONTEXT_CHECKSUM_OVERFLOW_MASK 0x1 +#define RDIF_TASK_CONTEXT_CHECKSUM_OVERFLOW_SHIFT 9 +/* mask for refernce tag handling */ +#define RDIF_TASK_CONTEXT_REF_TAG_MASK_MASK 0xF +#define RDIF_TASK_CONTEXT_REF_TAG_MASK_SHIFT 10 +#define RDIF_TASK_CONTEXT_RESERVED1_MASK 0x3 +#define RDIF_TASK_CONTEXT_RESERVED1_SHIFT 14 + __le32 reserved2; +}; + +/* + * RSS hash type + */ enum rss_hash_type { - RSS_HASH_TYPE_DEFAULT = 0, - RSS_HASH_TYPE_IPV4 = 1, - RSS_HASH_TYPE_TCP_IPV4 = 2, - RSS_HASH_TYPE_IPV6 = 3, - RSS_HASH_TYPE_TCP_IPV6 = 4, - RSS_HASH_TYPE_UDP_IPV4 = 5, - RSS_HASH_TYPE_UDP_IPV6 = 6, + RSS_HASH_TYPE_DEFAULT = 0, + RSS_HASH_TYPE_IPV4 = 1, + RSS_HASH_TYPE_TCP_IPV4 = 2, + RSS_HASH_TYPE_IPV6 = 3, + RSS_HASH_TYPE_TCP_IPV6 = 4, + RSS_HASH_TYPE_UDP_IPV4 = 5, + RSS_HASH_TYPE_UDP_IPV6 = 6, MAX_RSS_HASH_TYPE }; -/* status block structure */ -struct status_block { - __le16 pi_array[PIS_PER_SB]; - __le32 sb_num; -#define STATUS_BLOCK_SB_NUM_MASK 0x1FF -#define STATUS_BLOCK_SB_NUM_SHIFT 0 -#define STATUS_BLOCK_ZERO_PAD_MASK 0x7F -#define STATUS_BLOCK_ZERO_PAD_SHIFT 9 -#define STATUS_BLOCK_ZERO_PAD2_MASK 0xFFFF -#define STATUS_BLOCK_ZERO_PAD2_SHIFT 16 +/* + * status block structure + */ +struct status_block_e4 { + __le16 pi_array[PIS_PER_SB_E4]; + __le32 sb_num; +#define STATUS_BLOCK_E4_SB_NUM_MASK 0x1FF +#define STATUS_BLOCK_E4_SB_NUM_SHIFT 0 +#define STATUS_BLOCK_E4_ZERO_PAD_MASK 0x7F +#define STATUS_BLOCK_E4_ZERO_PAD_SHIFT 9 +#define STATUS_BLOCK_E4_ZERO_PAD2_MASK 0xFFFF +#define STATUS_BLOCK_E4_ZERO_PAD2_SHIFT 16 __le32 prod_index; -#define STATUS_BLOCK_PROD_INDEX_MASK 0xFFFFFF -#define STATUS_BLOCK_PROD_INDEX_SHIFT 0 -#define STATUS_BLOCK_ZERO_PAD3_MASK 0xFF -#define STATUS_BLOCK_ZERO_PAD3_SHIFT 24 +#define STATUS_BLOCK_E4_PROD_INDEX_MASK 0xFFFFFF +#define STATUS_BLOCK_E4_PROD_INDEX_SHIFT 0 +#define STATUS_BLOCK_E4_ZERO_PAD3_MASK 0xFF +#define STATUS_BLOCK_E4_ZERO_PAD3_SHIFT 24 }; -/* @DPDK */ -#define X_FINAL_CLEANUP_AGG_INT 1 -#define SDM_COMP_TYPE_AGG_INT 2 -#define MAX_NUM_LL2_RX_QUEUES 32 -#define QM_PQ_ELEMENT_SIZE 4 -#define PXP_VF_BAR0_START_IGU 0 -#define EAGLE_ENG1_WORKAROUND_NIG_FLOWCTRL_MODE 3 -#define TSTORM_QZONE_SIZE 8 -#define MSTORM_QZONE_SIZE 16 -#define USTORM_QZONE_SIZE 8 -#define XSTORM_QZONE_SIZE 0 -#define YSTORM_QZONE_SIZE 8 -#define PSTORM_QZONE_SIZE 0 +/* + * status block structure + */ +struct status_block_e5 { + __le16 pi_array[PIS_PER_SB_E5]; + __le32 sb_num; +#define STATUS_BLOCK_E5_SB_NUM_MASK 0x1FF +#define STATUS_BLOCK_E5_SB_NUM_SHIFT 0 +#define STATUS_BLOCK_E5_ZERO_PAD_MASK 0x7F +#define STATUS_BLOCK_E5_ZERO_PAD_SHIFT 9 +#define STATUS_BLOCK_E5_ZERO_PAD2_MASK 0xFFFF +#define STATUS_BLOCK_E5_ZERO_PAD2_SHIFT 16 + __le32 prod_index; +#define STATUS_BLOCK_E5_PROD_INDEX_MASK 0xFFFFFF +#define STATUS_BLOCK_E5_PROD_INDEX_SHIFT 0 +#define STATUS_BLOCK_E5_ZERO_PAD3_MASK 0xFF +#define STATUS_BLOCK_E5_ZERO_PAD3_SHIFT 24 +}; -/* VF BAR */ -#define PXP_VF_BAR0 0 -#define PXP_VF_BAR0_START_GRC 0x3E00 -#define PXP_VF_BAR0_GRC_LENGTH 0x200 -#define PXP_VF_BAR0_END_GRC \ -(PXP_VF_BAR0_START_GRC + PXP_VF_BAR0_GRC_LENGTH - 1) +/* + * Tdif context + */ +struct tdif_task_context { + __le32 initial_ref_tag; + __le16 app_tag_value; + __le16 app_tag_mask; + __le16 partial_crc_value_b; + __le16 partial_checksum_value_b; + __le16 stateB; +#define TDIF_TASK_CONTEXT_RECEIVED_DIF_BYTES_LEFT_B_MASK 0xF +#define TDIF_TASK_CONTEXT_RECEIVED_DIF_BYTES_LEFT_B_SHIFT 0 +#define TDIF_TASK_CONTEXT_TRANSMITED_DIF_BYTES_LEFT_B_MASK 0xF +#define TDIF_TASK_CONTEXT_TRANSMITED_DIF_BYTES_LEFT_B_SHIFT 4 +#define TDIF_TASK_CONTEXT_ERROR_IN_IO_B_MASK 0x1 +#define TDIF_TASK_CONTEXT_ERROR_IN_IO_B_SHIFT 8 +#define TDIF_TASK_CONTEXT_CHECKSUM_VERFLOW_MASK 0x1 +#define TDIF_TASK_CONTEXT_CHECKSUM_VERFLOW_SHIFT 9 +#define TDIF_TASK_CONTEXT_RESERVED0_MASK 0x3F +#define TDIF_TASK_CONTEXT_RESERVED0_SHIFT 10 + u8 reserved1; + u8 flags0; +#define TDIF_TASK_CONTEXT_IGNORE_APP_TAG_MASK 0x1 +#define TDIF_TASK_CONTEXT_IGNORE_APP_TAG_SHIFT 0 +#define TDIF_TASK_CONTEXT_INITIAL_REF_TAG_VALID_MASK 0x1 +#define TDIF_TASK_CONTEXT_INITIAL_REF_TAG_VALID_SHIFT 1 +/* 0 = IP checksum, 1 = CRC */ +#define TDIF_TASK_CONTEXT_HOST_GUARD_TYPE_MASK 0x1 +#define TDIF_TASK_CONTEXT_HOST_GUARD_TYPE_SHIFT 2 +#define TDIF_TASK_CONTEXT_SET_ERROR_WITH_EOP_MASK 0x1 +#define TDIF_TASK_CONTEXT_SET_ERROR_WITH_EOP_SHIFT 3 +/* 1/2/3 - Protection Type */ +#define TDIF_TASK_CONTEXT_PROTECTION_TYPE_MASK 0x3 +#define TDIF_TASK_CONTEXT_PROTECTION_TYPE_SHIFT 4 +/* 0=0x0000, 1=0xffff */ +#define TDIF_TASK_CONTEXT_CRC_SEED_MASK 0x1 +#define TDIF_TASK_CONTEXT_CRC_SEED_SHIFT 6 +#define TDIF_TASK_CONTEXT_RESERVED2_MASK 0x1 +#define TDIF_TASK_CONTEXT_RESERVED2_SHIFT 7 + __le32 flags1; +#define TDIF_TASK_CONTEXT_VALIDATE_GUARD_MASK 0x1 +#define TDIF_TASK_CONTEXT_VALIDATE_GUARD_SHIFT 0 +#define TDIF_TASK_CONTEXT_VALIDATE_APP_TAG_MASK 0x1 +#define TDIF_TASK_CONTEXT_VALIDATE_APP_TAG_SHIFT 1 +#define TDIF_TASK_CONTEXT_VALIDATE_REF_TAG_MASK 0x1 +#define TDIF_TASK_CONTEXT_VALIDATE_REF_TAG_SHIFT 2 +#define TDIF_TASK_CONTEXT_FORWARD_GUARD_MASK 0x1 +#define TDIF_TASK_CONTEXT_FORWARD_GUARD_SHIFT 3 +#define TDIF_TASK_CONTEXT_FORWARD_APP_TAG_MASK 0x1 +#define TDIF_TASK_CONTEXT_FORWARD_APP_TAG_SHIFT 4 +#define TDIF_TASK_CONTEXT_FORWARD_REF_TAG_MASK 0x1 +#define TDIF_TASK_CONTEXT_FORWARD_REF_TAG_SHIFT 5 +/* 0=512B, 1=1KB, 2=2KB, 3=4KB, 4=8KB */ +#define TDIF_TASK_CONTEXT_INTERVAL_SIZE_MASK 0x7 +#define TDIF_TASK_CONTEXT_INTERVAL_SIZE_SHIFT 6 +/* 0=None, 1=DIF, 2=DIX */ +#define TDIF_TASK_CONTEXT_HOST_INTERFACE_MASK 0x3 +#define TDIF_TASK_CONTEXT_HOST_INTERFACE_SHIFT 9 +/* DIF tag right at the beginning of DIF interval */ +#define TDIF_TASK_CONTEXT_DIF_BEFORE_DATA_MASK 0x1 +#define TDIF_TASK_CONTEXT_DIF_BEFORE_DATA_SHIFT 11 +#define TDIF_TASK_CONTEXT_RESERVED3_MASK 0x1 /* reserved */ +#define TDIF_TASK_CONTEXT_RESERVED3_SHIFT 12 +/* 0=None, 1=DIF */ +#define TDIF_TASK_CONTEXT_NETWORK_INTERFACE_MASK 0x1 +#define TDIF_TASK_CONTEXT_NETWORK_INTERFACE_SHIFT 13 +#define TDIF_TASK_CONTEXT_RECEIVED_DIF_BYTES_LEFT_A_MASK 0xF +#define TDIF_TASK_CONTEXT_RECEIVED_DIF_BYTES_LEFT_A_SHIFT 14 +#define TDIF_TASK_CONTEXT_TRANSMITED_DIF_BYTES_LEFT_A_MASK 0xF +#define TDIF_TASK_CONTEXT_TRANSMITED_DIF_BYTES_LEFT_A_SHIFT 18 +#define TDIF_TASK_CONTEXT_ERROR_IN_IO_A_MASK 0x1 +#define TDIF_TASK_CONTEXT_ERROR_IN_IO_A_SHIFT 22 +#define TDIF_TASK_CONTEXT_CHECKSUM_OVERFLOW_A_MASK 0x1 +#define TDIF_TASK_CONTEXT_CHECKSUM_OVERFLOW_A_SHIFT 23 +/* mask for refernce tag handling */ +#define TDIF_TASK_CONTEXT_REF_TAG_MASK_MASK 0xF +#define TDIF_TASK_CONTEXT_REF_TAG_MASK_SHIFT 24 +/* Forward application tag with mask */ +#define TDIF_TASK_CONTEXT_FORWARD_APP_TAG_WITH_MASK_MASK 0x1 +#define TDIF_TASK_CONTEXT_FORWARD_APP_TAG_WITH_MASK_SHIFT 28 +/* Forward reference tag with mask */ +#define TDIF_TASK_CONTEXT_FORWARD_REF_TAG_WITH_MASK_MASK 0x1 +#define TDIF_TASK_CONTEXT_FORWARD_REF_TAG_WITH_MASK_SHIFT 29 +/* Keep reference tag constant */ +#define TDIF_TASK_CONTEXT_KEEP_REF_TAG_CONST_MASK 0x1 +#define TDIF_TASK_CONTEXT_KEEP_REF_TAG_CONST_SHIFT 30 +#define TDIF_TASK_CONTEXT_RESERVED4_MASK 0x1 +#define TDIF_TASK_CONTEXT_RESERVED4_SHIFT 31 + __le32 offset_in_io_b; + __le16 partial_crc_value_a; + __le16 partial_checksum_value_a; + __le32 offset_in_io_a; + u8 partial_dif_data_a[8]; + u8 partial_dif_data_b[8]; +}; -#define PXP_VF_BAR0_START_IGU 0 -#define PXP_VF_BAR0_IGU_LENGTH 0x3000 -#define PXP_VF_BAR0_END_IGU \ -(PXP_VF_BAR0_START_IGU + PXP_VF_BAR0_IGU_LENGTH - 1) -#define PXP_VF_BAR0_START_DQ 0x3000 -#define PXP_VF_BAR0_DQ_LENGTH 0x200 -#define PXP_VF_BAR0_DQ_OPAQUE_OFFSET 0 -#define PXP_VF_BAR0_ME_OPAQUE_ADDRESS \ -(PXP_VF_BAR0_START_DQ + PXP_VF_BAR0_DQ_OPAQUE_OFFSET) -#define PXP_VF_BAR0_ME_CONCRETE_ADDRESS \ -(PXP_VF_BAR0_ME_OPAQUE_ADDRESS + 4) -#define PXP_VF_BAR0_END_DQ \ -(PXP_VF_BAR0_START_DQ + PXP_VF_BAR0_DQ_LENGTH - 1) +/* + * Timers context + */ +struct timers_context { + __le32 logical_client_0; +/* Expiration time of logical client 0 */ +#define TIMERS_CONTEXT_EXPIRATIONTIMELC0_MASK 0x7FFFFFF +#define TIMERS_CONTEXT_EXPIRATIONTIMELC0_SHIFT 0 +#define TIMERS_CONTEXT_RESERVED0_MASK 0x1 +#define TIMERS_CONTEXT_RESERVED0_SHIFT 27 +/* Valid bit of logical client 0 */ +#define TIMERS_CONTEXT_VALIDLC0_MASK 0x1 +#define TIMERS_CONTEXT_VALIDLC0_SHIFT 28 +/* Active bit of logical client 0 */ +#define TIMERS_CONTEXT_ACTIVELC0_MASK 0x1 +#define TIMERS_CONTEXT_ACTIVELC0_SHIFT 29 +#define TIMERS_CONTEXT_RESERVED1_MASK 0x3 +#define TIMERS_CONTEXT_RESERVED1_SHIFT 30 + __le32 logical_client_1; +/* Expiration time of logical client 1 */ +#define TIMERS_CONTEXT_EXPIRATIONTIMELC1_MASK 0x7FFFFFF +#define TIMERS_CONTEXT_EXPIRATIONTIMELC1_SHIFT 0 +#define TIMERS_CONTEXT_RESERVED2_MASK 0x1 +#define TIMERS_CONTEXT_RESERVED2_SHIFT 27 +/* Valid bit of logical client 1 */ +#define TIMERS_CONTEXT_VALIDLC1_MASK 0x1 +#define TIMERS_CONTEXT_VALIDLC1_SHIFT 28 +/* Active bit of logical client 1 */ +#define TIMERS_CONTEXT_ACTIVELC1_MASK 0x1 +#define TIMERS_CONTEXT_ACTIVELC1_SHIFT 29 +#define TIMERS_CONTEXT_RESERVED3_MASK 0x3 +#define TIMERS_CONTEXT_RESERVED3_SHIFT 30 + __le32 logical_client_2; +/* Expiration time of logical client 2 */ +#define TIMERS_CONTEXT_EXPIRATIONTIMELC2_MASK 0x7FFFFFF +#define TIMERS_CONTEXT_EXPIRATIONTIMELC2_SHIFT 0 +#define TIMERS_CONTEXT_RESERVED4_MASK 0x1 +#define TIMERS_CONTEXT_RESERVED4_SHIFT 27 +/* Valid bit of logical client 2 */ +#define TIMERS_CONTEXT_VALIDLC2_MASK 0x1 +#define TIMERS_CONTEXT_VALIDLC2_SHIFT 28 +/* Active bit of logical client 2 */ +#define TIMERS_CONTEXT_ACTIVELC2_MASK 0x1 +#define TIMERS_CONTEXT_ACTIVELC2_SHIFT 29 +#define TIMERS_CONTEXT_RESERVED5_MASK 0x3 +#define TIMERS_CONTEXT_RESERVED5_SHIFT 30 + __le32 host_expiration_fields; +/* Expiration time on host (closest one) */ +#define TIMERS_CONTEXT_HOSTEXPRIRATIONVALUE_MASK 0x7FFFFFF +#define TIMERS_CONTEXT_HOSTEXPRIRATIONVALUE_SHIFT 0 +#define TIMERS_CONTEXT_RESERVED6_MASK 0x1 +#define TIMERS_CONTEXT_RESERVED6_SHIFT 27 +/* Valid bit of host expiration */ +#define TIMERS_CONTEXT_HOSTEXPRIRATIONVALID_MASK 0x1 +#define TIMERS_CONTEXT_HOSTEXPRIRATIONVALID_SHIFT 28 +#define TIMERS_CONTEXT_RESERVED7_MASK 0x7 +#define TIMERS_CONTEXT_RESERVED7_SHIFT 29 +}; -#define PXP_VF_BAR0_START_TSDM_ZONE_B 0x3200 -#define PXP_VF_BAR0_SDM_LENGTH_ZONE_B 0x200 -#define PXP_VF_BAR0_END_TSDM_ZONE_B \ -(PXP_VF_BAR0_START_TSDM_ZONE_B + PXP_VF_BAR0_SDM_LENGTH_ZONE_B - 1) -#define PXP_VF_BAR0_START_MSDM_ZONE_B 0x3400 -#define PXP_VF_BAR0_END_MSDM_ZONE_B \ -(PXP_VF_BAR0_START_MSDM_ZONE_B + PXP_VF_BAR0_SDM_LENGTH_ZONE_B - 1) - -#define PXP_VF_BAR0_START_USDM_ZONE_B 0x3600 -#define PXP_VF_BAR0_END_USDM_ZONE_B \ -(PXP_VF_BAR0_START_USDM_ZONE_B + PXP_VF_BAR0_SDM_LENGTH_ZONE_B - 1) - -#define PXP_VF_BAR0_START_XSDM_ZONE_B 0x3800 -#define PXP_VF_BAR0_END_XSDM_ZONE_B \ -(PXP_VF_BAR0_START_XSDM_ZONE_B + PXP_VF_BAR0_SDM_LENGTH_ZONE_B - 1) - -#define PXP_VF_BAR0_START_YSDM_ZONE_B 0x3a00 -#define PXP_VF_BAR0_END_YSDM_ZONE_B \ -(PXP_VF_BAR0_START_YSDM_ZONE_B + PXP_VF_BAR0_SDM_LENGTH_ZONE_B - 1) - -#define PXP_VF_BAR0_START_PSDM_ZONE_B 0x3c00 -#define PXP_VF_BAR0_END_PSDM_ZONE_B \ -(PXP_VF_BAR0_START_PSDM_ZONE_B + PXP_VF_BAR0_SDM_LENGTH_ZONE_B - 1) - -#define PXP_VF_BAR0_START_SDM_ZONE_A 0x4000 -#define PXP_VF_BAR0_END_SDM_ZONE_A 0x10000 - -#define PXP_VF_BAR0_GRC_WINDOW_LENGTH 32 - -#define PXP_ILT_PAGE_SIZE_NUM_BITS_MIN 12 -#define PXP_ILT_BLOCK_FACTOR_MULTIPLIER 1024 +/* + * Enum for next_protocol field of tunnel_parsing_flags + */ +enum tunnel_next_protocol { + e_unknown = 0, + e_l2 = 1, + e_ipv4 = 2, + e_ipv6 = 3, + MAX_TUNNEL_NEXT_PROTOCOL +}; #endif /* __COMMON_HSI__ */ diff --git a/dpdk/drivers/net/qede/base/ecore.h b/dpdk/drivers/net/qede/base/ecore.h index d682a785..ce5f3a90 100644 --- a/dpdk/drivers/net/qede/base/ecore.h +++ b/dpdk/drivers/net/qede/base/ecore.h @@ -9,20 +9,48 @@ #ifndef __ECORE_H #define __ECORE_H +/* @DPDK */ +#include +#include +#include + +#define CONFIG_ECORE_BINARY_FW +#undef CONFIG_ECORE_ZIPPED_FW + +#ifdef CONFIG_ECORE_ZIPPED_FW +#include +#endif + #include "ecore_hsi_common.h" -#include "ecore_hsi_tools.h" +#include "ecore_hsi_debug_tools.h" +#include "ecore_hsi_init_func.h" +#include "ecore_hsi_init_tool.h" #include "ecore_proto_if.h" #include "mcp_public.h" -#define MAX_HWFNS_PER_DEVICE (4) -#define NAME_SIZE 64 /* @DPDK */ -#define VER_SIZE 16 -/* @DPDK ARRAY_DECL */ -#define ECORE_WFQ_UNIT 100 -#include "../qede_logs.h" /* @DPDK */ +#define ECORE_MAJOR_VERSION 8 +#define ECORE_MINOR_VERSION 30 +#define ECORE_REVISION_VERSION 8 +#define ECORE_ENGINEERING_VERSION 0 +#define ECORE_VERSION \ + ((ECORE_MAJOR_VERSION << 24) | (ECORE_MINOR_VERSION << 16) | \ + (ECORE_REVISION_VERSION << 8) | ECORE_ENGINEERING_VERSION) + +#define STORM_FW_VERSION \ + ((FW_MAJOR_VERSION << 24) | (FW_MINOR_VERSION << 16) | \ + (FW_REVISION_VERSION << 8) | FW_ENGINEERING_VERSION) + +#define MAX_HWFNS_PER_DEVICE 2 +#define NAME_SIZE 128 /* @DPDK */ +#define ECORE_WFQ_UNIT 100 +#include "../qede_logs.h" /* @DPDK */ + +#define ISCSI_BDQ_ID(_port_id) (_port_id) +#define FCOE_BDQ_ID(_port_id) (_port_id + 2) /* Constants */ #define ECORE_WID_SIZE (1024) +#define ECORE_MIN_WIDS (4) /* Configurable */ #define ECORE_PF_DEMS_SIZE (4) @@ -39,6 +67,7 @@ enum ecore_nvm_cmd { ECORE_NVM_READ_NVRAM = DRV_MSG_CODE_NVM_READ_NVRAM, ECORE_NVM_WRITE_NVRAM = DRV_MSG_CODE_NVM_WRITE_NVRAM, ECORE_NVM_DEL_FILE = DRV_MSG_CODE_NVM_DEL_FILE, + ECORE_EXT_PHY_FW_UPGRADE = DRV_MSG_CODE_EXT_PHY_FW_UPGRADE, ECORE_NVM_SET_SECURE_MODE = DRV_MSG_CODE_SET_SECURE_MODE, ECORE_PHY_RAW_READ = DRV_MSG_CODE_PHY_RAW_READ, ECORE_PHY_RAW_WRITE = DRV_MSG_CODE_PHY_RAW_WRITE, @@ -65,22 +94,30 @@ enum ecore_nvm_cmd { #define SET_FIELD(value, name, flag) \ do { \ (value) &= ~(name##_MASK << name##_SHIFT); \ - (value) |= (((u64)flag) << (name##_SHIFT)); \ + (value) |= ((((u64)flag) & (u64)name##_MASK) << (name##_SHIFT));\ } while (0) #define GET_FIELD(value, name) \ (((value) >> (name##_SHIFT)) & name##_MASK) + +#define GET_MFW_FIELD(name, field) \ + (((name) & (field ## _MASK)) >> (field ## _OFFSET)) + +#define SET_MFW_FIELD(name, field, value) \ +do { \ + (name) &= ~((field ## _MASK)); \ + (name) |= (((value) << (field ## _OFFSET)) & (field ## _MASK)); \ +} while (0) #endif static OSAL_INLINE u32 DB_ADDR(u32 cid, u32 DEMS) { u32 db_addr = FIELD_VALUE(DB_LEGACY_ADDR_DEMS, DEMS) | - (cid * ECORE_PF_DEMS_SIZE); + (cid * ECORE_PF_DEMS_SIZE); return db_addr; } -/* @DPDK: This is a backport from latest ecore for TSS fix */ static OSAL_INLINE u32 DB_ADDR_VF(u32 cid, u32 DEMS) { u32 db_addr = FIELD_VALUE(DB_LEGACY_ADDR_DEMS, DEMS) | @@ -93,6 +130,7 @@ static OSAL_INLINE u32 DB_ADDR_VF(u32 cid, u32 DEMS) ((sizeof(type_name) + (u32)(1 << (p_hwfn->p_dev->cache_shift)) - 1) & \ ~((1 << (p_hwfn->p_dev->cache_shift)) - 1)) +#ifndef LINUX_REMOVE #ifndef U64_HI #define U64_HI(val) ((u32)(((u64)(val)) >> 32)) #endif @@ -100,13 +138,14 @@ static OSAL_INLINE u32 DB_ADDR_VF(u32 cid, u32 DEMS) #ifndef U64_LO #define U64_LO(val) ((u32)(((u64)(val)) & 0xffffffff)) #endif +#endif #ifndef __EXTRACT__LINUX__ enum DP_LEVEL { - ECORE_LEVEL_VERBOSE = 0x0, - ECORE_LEVEL_INFO = 0x1, - ECORE_LEVEL_NOTICE = 0x2, - ECORE_LEVEL_ERR = 0x3, + ECORE_LEVEL_VERBOSE = 0x0, + ECORE_LEVEL_INFO = 0x1, + ECORE_LEVEL_NOTICE = 0x2, + ECORE_LEVEL_ERR = 0x3, }; #define ECORE_LOG_LEVEL_SHIFT (30) @@ -116,31 +155,34 @@ enum DP_LEVEL { enum DP_MODULE { #ifndef LINUX_REMOVE - ECORE_MSG_DRV = 0x0001, - ECORE_MSG_PROBE = 0x0002, - ECORE_MSG_LINK = 0x0004, - ECORE_MSG_TIMER = 0x0008, - ECORE_MSG_IFDOWN = 0x0010, - ECORE_MSG_IFUP = 0x0020, - ECORE_MSG_RX_ERR = 0x0040, - ECORE_MSG_TX_ERR = 0x0080, - ECORE_MSG_TX_QUEUED = 0x0100, - ECORE_MSG_INTR = 0x0200, - ECORE_MSG_TX_DONE = 0x0400, - ECORE_MSG_RX_STATUS = 0x0800, - ECORE_MSG_PKTDATA = 0x1000, - ECORE_MSG_HW = 0x2000, - ECORE_MSG_WOL = 0x4000, + ECORE_MSG_DRV = 0x0001, + ECORE_MSG_PROBE = 0x0002, + ECORE_MSG_LINK = 0x0004, + ECORE_MSG_TIMER = 0x0008, + ECORE_MSG_IFDOWN = 0x0010, + ECORE_MSG_IFUP = 0x0020, + ECORE_MSG_RX_ERR = 0x0040, + ECORE_MSG_TX_ERR = 0x0080, + ECORE_MSG_TX_QUEUED = 0x0100, + ECORE_MSG_INTR = 0x0200, + ECORE_MSG_TX_DONE = 0x0400, + ECORE_MSG_RX_STATUS = 0x0800, + ECORE_MSG_PKTDATA = 0x1000, + ECORE_MSG_HW = 0x2000, + ECORE_MSG_WOL = 0x4000, #endif - ECORE_MSG_SPQ = 0x10000, - ECORE_MSG_STATS = 0x20000, - ECORE_MSG_DCB = 0x40000, - ECORE_MSG_IOV = 0x80000, - ECORE_MSG_SP = 0x100000, - ECORE_MSG_STORAGE = 0x200000, - ECORE_MSG_CXT = 0x800000, - ECORE_MSG_ILT = 0x2000000, - ECORE_MSG_DEBUG = 0x8000000, + ECORE_MSG_SPQ = 0x10000, + ECORE_MSG_STATS = 0x20000, + ECORE_MSG_DCB = 0x40000, + ECORE_MSG_IOV = 0x80000, + ECORE_MSG_SP = 0x100000, + ECORE_MSG_STORAGE = 0x200000, + ECORE_MSG_OOO = 0x200000, + ECORE_MSG_CXT = 0x800000, + ECORE_MSG_LL2 = 0x1000000, + ECORE_MSG_ILT = 0x2000000, + ECORE_MSG_RDMA = 0x4000000, + ECORE_MSG_DEBUG = 0x8000000, /* to be added...up to 0x8000000 */ }; #endif @@ -159,13 +201,15 @@ struct ecore_sb_attn_info; struct ecore_cxt_mngr; struct ecore_dma_mem; struct ecore_sb_sp_info; +struct ecore_ll2_info; +struct ecore_l2_info; struct ecore_igu_info; struct ecore_mcp_info; struct ecore_dcbx_info; struct ecore_rt_data { - u32 *init_val; - bool *b_valid; + u32 *init_val; + bool *b_valid; }; enum ecore_tunn_mode { @@ -181,68 +225,47 @@ enum ecore_tunn_clss { ECORE_TUNN_CLSS_MAC_VNI, ECORE_TUNN_CLSS_INNER_MAC_VLAN, ECORE_TUNN_CLSS_INNER_MAC_VNI, + ECORE_TUNN_CLSS_MAC_VLAN_DUAL_STAGE, MAX_ECORE_TUNN_CLSS, }; -struct ecore_tunn_start_params { - unsigned long tunn_mode; - u16 vxlan_udp_port; - u16 geneve_udp_port; - u8 update_vxlan_udp_port; - u8 update_geneve_udp_port; - u8 tunn_clss_vxlan; - u8 tunn_clss_l2geneve; - u8 tunn_clss_ipgeneve; - u8 tunn_clss_l2gre; - u8 tunn_clss_ipgre; +struct ecore_tunn_update_type { + bool b_update_mode; + bool b_mode_enabled; + enum ecore_tunn_clss tun_cls; }; -struct ecore_tunn_update_params { - unsigned long tunn_mode_update_mask; - unsigned long tunn_mode; - u16 vxlan_udp_port; - u16 geneve_udp_port; - u8 update_rx_pf_clss; - u8 update_tx_pf_clss; - u8 update_vxlan_udp_port; - u8 update_geneve_udp_port; - u8 tunn_clss_vxlan; - u8 tunn_clss_l2geneve; - u8 tunn_clss_ipgeneve; - u8 tunn_clss_l2gre; - u8 tunn_clss_ipgre; +struct ecore_tunn_update_udp_port { + bool b_update_port; + u16 port; }; -struct ecore_hw_sriov_info { - /* standard SRIOV capability fields, mostly for debugging */ - int pos; /* capability position */ - int nres; /* number of resources */ - u32 cap; /* SR-IOV Capabilities */ - u16 ctrl; /* SR-IOV Control */ - u16 total_vfs; /* total VFs associated with the PF */ - u16 num_vfs; /* number of vfs that have been started */ - u64 active_vfs[3]; /* bitfield of active vfs */ -#define ECORE_IS_VF_ACTIVE(_p_dev, _rel_vf_id) \ - (!!(_p_dev->sriov_info.active_vfs[_rel_vf_id / 64] & \ - (1ULL << (_rel_vf_id % 64)))) - u16 initial_vfs; /* initial VFs associated with the PF */ - u16 nr_virtfn; /* number of VFs available */ - u16 offset; /* first VF Routing ID offset */ - u16 stride; /* following VF stride */ - u16 vf_device_id; /* VF device id */ - u32 pgsz; /* page size for BAR alignment */ - u8 link; /* Function Dependency Link */ +struct ecore_tunnel_info { + struct ecore_tunn_update_type vxlan; + struct ecore_tunn_update_type l2_geneve; + struct ecore_tunn_update_type ip_geneve; + struct ecore_tunn_update_type l2_gre; + struct ecore_tunn_update_type ip_gre; - bool b_hw_channel; /* Whether PF uses the HW-channel */ + struct ecore_tunn_update_udp_port vxlan_port; + struct ecore_tunn_update_udp_port geneve_port; + + bool b_update_rx_cls; + bool b_update_tx_cls; }; /* The PCI personality is not quite synonymous to protocol ID: * 1. All personalities need CORE connections - * 2. The Ethernet personality may support also the RoCE protocol + * 2. The Ethernet personality may support also the RoCE/iWARP protocol */ enum ecore_pci_personality { ECORE_PCI_ETH, - ECORE_PCI_DEFAULT /* default in shmem */ + ECORE_PCI_FCOE, + ECORE_PCI_ISCSI, + ECORE_PCI_ETH_ROCE, + ECORE_PCI_ETH_IWARP, + ECORE_PCI_ETH_RDMA, + ECORE_PCI_DEFAULT /* default in shmem */ }; /* All VFs are symmetric, all counters are PF + all VFs */ @@ -254,12 +277,10 @@ struct ecore_qm_iids { #define MAX_PF_PER_PORT 8 -/*@@@TBD MK RESC: need to remove and use MCP interface instead */ /* HW / FW resources, output of features supported below, most information * is received from MFW. */ -enum ECORE_RESOURCES { - ECORE_SB, +enum ecore_resources { ECORE_L2_QUEUE, ECORE_VPORT, ECORE_RSS_ENG, @@ -267,24 +288,38 @@ enum ECORE_RESOURCES { ECORE_RL, ECORE_MAC, ECORE_VLAN, + ECORE_RDMA_CNQ_RAM, ECORE_ILT, + ECORE_LL2_QUEUE, ECORE_CMDQS_CQS, + ECORE_RDMA_STATS_QUEUE, + ECORE_BDQ, + + /* This is needed only internally for matching against the IGU. + * In case of legacy MFW, would be set to `0'. + */ + ECORE_SB, + ECORE_MAX_RESC, }; /* Features that require resources, given as input to the resource management * algorithm, the output are the resources above */ -enum ECORE_FEATURE { +enum ecore_feature { ECORE_PF_L2_QUE, ECORE_PF_TC, ECORE_VF, ECORE_EXTRA_VF_QUE, ECORE_VMQ, + ECORE_RDMA_CNQ, + ECORE_ISCSI_CQ, + ECORE_FCOE_CQ, + ECORE_VF_L2_QUE, ECORE_MAX_FEATURES, }; -enum ECORE_PORT_MODE { +enum ecore_port_mode { ECORE_PORT_MODE_DE_2X40G, ECORE_PORT_MODE_DE_2X50G, ECORE_PORT_MODE_DE_1X100G, @@ -293,11 +328,17 @@ enum ECORE_PORT_MODE { ECORE_PORT_MODE_DE_4X20G, ECORE_PORT_MODE_DE_1X40G, ECORE_PORT_MODE_DE_2X25G, - ECORE_PORT_MODE_DE_1X25G + ECORE_PORT_MODE_DE_1X25G, + ECORE_PORT_MODE_DE_4X25G, + ECORE_PORT_MODE_DE_2X10G, }; enum ecore_dev_cap { ECORE_DEV_CAP_ETH, + ECORE_DEV_CAP_FCOE, + ECORE_DEV_CAP_ISCSI, + ECORE_DEV_CAP_ROCE, + ECORE_DEV_CAP_IWARP }; #ifndef __EXTRACT__LINUX__ @@ -311,25 +352,55 @@ enum ecore_hw_err_type { }; #endif +enum ecore_db_rec_exec { + DB_REC_DRY_RUN, + DB_REC_REAL_DEAL, + DB_REC_ONCE, +}; + struct ecore_hw_info { /* PCI personality */ enum ecore_pci_personality personality; +#define ECORE_IS_RDMA_PERSONALITY(dev) \ + ((dev)->hw_info.personality == ECORE_PCI_ETH_ROCE || \ + (dev)->hw_info.personality == ECORE_PCI_ETH_IWARP || \ + (dev)->hw_info.personality == ECORE_PCI_ETH_RDMA) +#define ECORE_IS_ROCE_PERSONALITY(dev) \ + ((dev)->hw_info.personality == ECORE_PCI_ETH_ROCE || \ + (dev)->hw_info.personality == ECORE_PCI_ETH_RDMA) +#define ECORE_IS_IWARP_PERSONALITY(dev) \ + ((dev)->hw_info.personality == ECORE_PCI_ETH_IWARP || \ + (dev)->hw_info.personality == ECORE_PCI_ETH_RDMA) +#define ECORE_IS_L2_PERSONALITY(dev) \ + ((dev)->hw_info.personality == ECORE_PCI_ETH || \ + ECORE_IS_RDMA_PERSONALITY(dev)) +#define ECORE_IS_FCOE_PERSONALITY(dev) \ + ((dev)->hw_info.personality == ECORE_PCI_FCOE) +#define ECORE_IS_ISCSI_PERSONALITY(dev) \ + ((dev)->hw_info.personality == ECORE_PCI_ISCSI) /* Resource Allocation scheme results */ u32 resc_start[ECORE_MAX_RESC]; u32 resc_num[ECORE_MAX_RESC]; u32 feat_num[ECORE_MAX_FEATURES]; -#define RESC_START(_p_hwfn, resc) ((_p_hwfn)->hw_info.resc_start[resc]) -#define RESC_NUM(_p_hwfn, resc) ((_p_hwfn)->hw_info.resc_num[resc]) -#define RESC_END(_p_hwfn, resc) (RESC_START(_p_hwfn, resc) + \ + #define RESC_START(_p_hwfn, resc) ((_p_hwfn)->hw_info.resc_start[resc]) + #define RESC_NUM(_p_hwfn, resc) ((_p_hwfn)->hw_info.resc_num[resc]) + #define RESC_END(_p_hwfn, resc) (RESC_START(_p_hwfn, resc) + \ RESC_NUM(_p_hwfn, resc)) -#define FEAT_NUM(_p_hwfn, resc) ((_p_hwfn)->hw_info.feat_num[resc]) + #define FEAT_NUM(_p_hwfn, resc) ((_p_hwfn)->hw_info.feat_num[resc]) - u8 num_tc; - u8 ooo_tc; + /* Amount of traffic classes HW supports */ + u8 num_hw_tc; + +/* Amount of TCs which should be active according to DCBx or upper layer driver + * configuration + */ + + u8 num_active_tc; + + /* The traffic class used by PF for it's offloaded protocol */ u8 offload_tc; - u8 non_offload_tc; u32 concrete_fid; u16 opaque_fid; @@ -337,25 +408,24 @@ struct ecore_hw_info { u32 part_num[4]; unsigned char hw_mac_addr[ETH_ALEN]; + u64 node_wwn; /* For FCoE only */ + u64 port_wwn; /* For FCoE only */ + + u16 num_iscsi_conns; + u16 num_fcoe_conns; struct ecore_igu_info *p_igu_info; /* Sriov */ - u32 first_vf_in_pf; u8 max_chains_per_vf; u32 port_mode; - u32 hw_mode; + u32 hw_mode; unsigned long device_capabilities; -}; -struct ecore_hw_cid_data { - u32 cid; - bool b_cid_allocated; - u8 vfid; /* 1-based; 0 signals this is for a PF */ + /* Default DCBX mode */ + u8 dcbx_mode; - /* Additional identifiers */ - u16 opaque_fid; - u8 vport_id; + u16 mtu; }; /* maximun size of read/write commands (HW limit) */ @@ -363,7 +433,7 @@ struct ecore_hw_cid_data { struct ecore_dmae_info { /* Mutex for synchronizing access to functions */ - osal_mutex_t mutex; + osal_mutex_t mutex; u8 channel; @@ -387,33 +457,43 @@ struct ecore_dmae_info { }; struct ecore_wfq_data { - u32 default_min_speed; /* When wfq feature is not configured */ - u32 min_speed; /* when feature is configured for any 1 vport */ + u32 default_min_speed; /* When wfq feature is not configured */ + u32 min_speed; /* when feature is configured for any 1 vport */ bool configured; }; struct ecore_qm_info { - struct init_qm_pq_params *qm_pq_params; + struct init_qm_pq_params *qm_pq_params; struct init_qm_vport_params *qm_vport_params; - struct init_qm_port_params *qm_port_params; - u16 start_pq; - u8 start_vport; - u8 pure_lb_pq; - u8 offload_pq; - u8 pure_ack_pq; - u8 ooo_pq; - u8 vf_queues_offset; - u16 num_pqs; - u16 num_vf_pqs; - u8 num_vports; - u8 max_phys_tcs_per_port; - bool pf_rl_en; - bool pf_wfq_en; - bool vport_rl_en; - bool vport_wfq_en; - u8 pf_wfq; - u32 pf_rl; - struct ecore_wfq_data *wfq_data; + struct init_qm_port_params *qm_port_params; + u16 start_pq; + u8 start_vport; + u16 pure_lb_pq; + u16 offload_pq; + u16 pure_ack_pq; + u16 ooo_pq; + u16 first_vf_pq; + u16 first_mcos_pq; + u16 first_rl_pq; + u16 num_pqs; + u16 num_vf_pqs; + u8 num_vports; + u8 max_phys_tcs_per_port; + u8 ooo_tc; + bool pf_rl_en; + bool pf_wfq_en; + bool vport_rl_en; + bool vport_wfq_en; + u8 pf_wfq; + u32 pf_rl; + struct ecore_wfq_data *wfq_data; + u8 num_pf_rls; +}; + +struct ecore_db_recovery_info { + osal_list_t list; + osal_spinlock_t lock; + u32 db_recovery_counter; }; struct storm_stats { @@ -421,9 +501,6 @@ struct storm_stats { u32 len; }; -#define CONFIG_ECORE_BINARY_FW -#define CONFIG_ECORE_ZIPPED_FW - struct ecore_fw_data { #ifdef CONFIG_ECORE_BINARY_FW struct fw_ver_info *fw_ver_info; @@ -434,247 +511,357 @@ struct ecore_fw_data { u32 init_ops_size; }; +enum ecore_mf_mode_bit { + /* Supports PF-classification based on tag */ + ECORE_MF_OVLAN_CLSS, + + /* Supports PF-classification based on MAC */ + ECORE_MF_LLH_MAC_CLSS, + + /* Supports PF-classification based on protocol type */ + ECORE_MF_LLH_PROTO_CLSS, + + /* Requires a default PF to be set */ + ECORE_MF_NEED_DEF_PF, + + /* Allow LL2 to multicast/broadcast */ + ECORE_MF_LL2_NON_UNICAST, + + /* Allow Cross-PF [& child VFs] Tx-switching */ + ECORE_MF_INTER_PF_SWITCH, + + /* TODO - if we ever re-utilize any of this logic, we can rename */ + ECORE_MF_UFP_SPECIFIC, + + ECORE_MF_DISABLE_ARFS, +}; + +enum ecore_ufp_mode { + ECORE_UFP_MODE_ETS, + ECORE_UFP_MODE_VNIC_BW, +}; + +enum ecore_ufp_pri_type { + ECORE_UFP_PRI_OS, + ECORE_UFP_PRI_VNIC +}; + +struct ecore_ufp_info { + enum ecore_ufp_pri_type pri_type; + enum ecore_ufp_mode mode; + u8 tc; +}; + +enum BAR_ID { + BAR_ID_0, /* used for GRC */ + BAR_ID_1 /* Used for doorbells */ +}; + struct ecore_hwfn { - struct ecore_dev *p_dev; - u8 my_id; /* ID inside the PF */ + struct ecore_dev *p_dev; + u8 my_id; /* ID inside the PF */ #define IS_LEAD_HWFN(edev) (!((edev)->my_id)) - u8 rel_pf_id; /* Relative to engine */ - u8 abs_pf_id; + u8 rel_pf_id; /* Relative to engine*/ + u8 abs_pf_id; #define ECORE_PATH_ID(_p_hwfn) \ - (ECORE_IS_K2((_p_hwfn)->p_dev) ? 0 : ((_p_hwfn)->abs_pf_id & 1)) - u8 port_id; - bool b_active; + (ECORE_IS_BB((_p_hwfn)->p_dev) ? ((_p_hwfn)->abs_pf_id & 1) : 0) + u8 port_id; + bool b_active; - u32 dp_module; - u8 dp_level; - char name[NAME_SIZE]; - void *dp_ctx; + u32 dp_module; + u8 dp_level; + char name[NAME_SIZE]; + void *dp_ctx; - bool first_on_engine; - bool hw_init_done; + bool first_on_engine; + bool hw_init_done; - u8 num_funcs_on_engine; + u8 num_funcs_on_engine; + u8 enabled_func_idx; /* BAR access */ - void OSAL_IOMEM *regview; - void OSAL_IOMEM *doorbells; - u64 db_phys_addr; - unsigned long db_size; + void OSAL_IOMEM *regview; + void OSAL_IOMEM *doorbells; + u64 db_phys_addr; + unsigned long db_size; /* PTT pool */ - struct ecore_ptt_pool *p_ptt_pool; + struct ecore_ptt_pool *p_ptt_pool; /* HW info */ - struct ecore_hw_info hw_info; + struct ecore_hw_info hw_info; /* rt_array (for init-tool) */ - struct ecore_rt_data rt_data; + struct ecore_rt_data rt_data; /* SPQ */ - struct ecore_spq *p_spq; + struct ecore_spq *p_spq; /* EQ */ - struct ecore_eq *p_eq; + struct ecore_eq *p_eq; - /* Consolidate Q */ - struct ecore_consq *p_consq; + /* Consolidate Q*/ + struct ecore_consq *p_consq; /* Slow-Path definitions */ - osal_dpc_t sp_dpc; - bool b_sp_dpc_enabled; + osal_dpc_t sp_dpc; + bool b_sp_dpc_enabled; - struct ecore_ptt *p_main_ptt; - struct ecore_ptt *p_dpc_ptt; + struct ecore_ptt *p_main_ptt; + struct ecore_ptt *p_dpc_ptt; - struct ecore_sb_sp_info *p_sp_sb; - struct ecore_sb_attn_info *p_sb_attn; + struct ecore_sb_sp_info *p_sp_sb; + struct ecore_sb_attn_info *p_sb_attn; /* Protocol related */ - struct ecore_ooo_info *p_ooo_info; - struct ecore_pf_params pf_params; + bool using_ll2; + struct ecore_ll2_info *p_ll2_info; + struct ecore_ooo_info *p_ooo_info; + struct ecore_iscsi_info *p_iscsi_info; + struct ecore_fcoe_info *p_fcoe_info; + struct ecore_rdma_info *p_rdma_info; + struct ecore_pf_params pf_params; - /* Array of sb_info of all status blocks */ - struct ecore_sb_info *sbs_info[MAX_SB_PER_PF_MIMD]; - u16 num_sbs; + bool b_rdma_enabled_in_prs; + u32 rdma_prs_search_reg; - struct ecore_cxt_mngr *p_cxt_mngr; + struct ecore_cxt_mngr *p_cxt_mngr; - /* Flag indicating whether interrupts are enabled or not */ - bool b_int_enabled; - bool b_int_requested; + /* Flag indicating whether interrupts are enabled or not*/ + bool b_int_enabled; + bool b_int_requested; /* True if the driver requests for the link */ - bool b_drv_link_init; + bool b_drv_link_init; - struct ecore_vf_iov *vf_iov_info; - struct ecore_pf_iov *pf_iov_info; - struct ecore_mcp_info *mcp_info; - struct ecore_dcbx_info *p_dcbx_info; + struct ecore_vf_iov *vf_iov_info; + struct ecore_pf_iov *pf_iov_info; + struct ecore_mcp_info *mcp_info; + struct ecore_dcbx_info *p_dcbx_info; + struct ecore_ufp_info ufp_info; - struct ecore_hw_cid_data *p_tx_cids; - struct ecore_hw_cid_data *p_rx_cids; - - struct ecore_dmae_info dmae_info; + struct ecore_dmae_info dmae_info; /* QM init */ - struct ecore_qm_info qm_info; + struct ecore_qm_info qm_info; - /* Buffer for unzipping firmware data */ #ifdef CONFIG_ECORE_ZIPPED_FW + /* Buffer for unzipping firmware data */ void *unzip_buf; #endif - struct dbg_tools_data dbg_info; + struct dbg_tools_data dbg_info; - struct z_stream_s *stream; + struct z_stream_s *stream; /* PWM region specific data */ - u32 dpi_size; - u32 dpi_count; - u32 dpi_start_offset; /* this is used to - * calculate th - * doorbell address - */ + u32 dpi_size; + u32 dpi_count; + u32 dpi_start_offset; /* this is used to + * calculate th + * doorbell address + */ + + /* If one of the following is set then EDPM shouldn't be used */ + u8 dcbx_no_edpm; + u8 db_bar_no_edpm; + + /* L2-related */ + struct ecore_l2_info *p_l2_info; + + /* Mechanism for recovering from doorbell drop */ + struct ecore_db_recovery_info db_recovery_info; + + /* @DPDK */ + struct ecore_ptt *p_arfs_ptt; }; -#ifndef __EXTRACT__LINUX__ enum ecore_mf_mode { ECORE_MF_DEFAULT, ECORE_MF_OVLAN, ECORE_MF_NPAR, + ECORE_MF_UFP, +}; + +/* @DPDK */ +struct ecore_dbg_feature { + u8 *dump_buf; + u32 buf_size; + u32 dumped_dwords; +}; + +enum qed_dbg_features { + DBG_FEATURE_BUS, + DBG_FEATURE_GRC, + DBG_FEATURE_IDLE_CHK, + DBG_FEATURE_MCP_TRACE, + DBG_FEATURE_REG_FIFO, + DBG_FEATURE_PROTECTION_OVERRIDE, + DBG_FEATURE_NUM +}; + +enum ecore_dev_type { + ECORE_DEV_TYPE_BB, + ECORE_DEV_TYPE_AH, }; -#endif struct ecore_dev { - u32 dp_module; - u8 dp_level; - char name[NAME_SIZE]; - void *dp_ctx; + u32 dp_module; + u8 dp_level; + char name[NAME_SIZE]; + void *dp_ctx; - u8 type; -#define ECORE_DEV_TYPE_BB (0 << 0) -#define ECORE_DEV_TYPE_AH (1 << 0) + enum ecore_dev_type type; /* Translate type/revision combo into the proper conditions */ #define ECORE_IS_BB(dev) ((dev)->type == ECORE_DEV_TYPE_BB) -#define ECORE_IS_BB_A0(dev) (ECORE_IS_BB(dev) && \ - CHIP_REV_IS_A0(dev)) -#define ECORE_IS_BB_B0(dev) (ECORE_IS_BB(dev) && \ - CHIP_REV_IS_B0(dev)) +#define ECORE_IS_BB_A0(dev) (ECORE_IS_BB(dev) && CHIP_REV_IS_A0(dev)) +#ifndef ASIC_ONLY +#define ECORE_IS_BB_B0(dev) ((ECORE_IS_BB(dev) && CHIP_REV_IS_B0(dev)) || \ + (CHIP_REV_IS_TEDIBEAR(dev))) +#else +#define ECORE_IS_BB_B0(dev) (ECORE_IS_BB(dev) && CHIP_REV_IS_B0(dev)) +#endif #define ECORE_IS_AH(dev) ((dev)->type == ECORE_DEV_TYPE_AH) #define ECORE_IS_K2(dev) ECORE_IS_AH(dev) -#define ECORE_GET_TYPE(dev) (ECORE_IS_BB_A0(dev) ? CHIP_BB_A0 : \ - ECORE_IS_BB_B0(dev) ? CHIP_BB_B0 : CHIP_K2) u16 vendor_id; u16 device_id; +#define ECORE_DEV_ID_MASK 0xff00 +#define ECORE_DEV_ID_MASK_BB 0x1600 +#define ECORE_DEV_ID_MASK_AH 0x8000 - u16 chip_num; + u16 chip_num; #define CHIP_NUM_MASK 0xffff -#define CHIP_NUM_SHIFT 16 +#define CHIP_NUM_SHIFT 0 - u16 chip_rev; + u8 chip_rev; #define CHIP_REV_MASK 0xf -#define CHIP_REV_SHIFT 12 +#define CHIP_REV_SHIFT 0 #ifndef ASIC_ONLY -#define CHIP_REV_IS_TEDIBEAR(_p_dev) ((_p_dev)->chip_rev == 0x5) -#define CHIP_REV_IS_EMUL_A0(_p_dev) ((_p_dev)->chip_rev == 0xe) -#define CHIP_REV_IS_EMUL_B0(_p_dev) ((_p_dev)->chip_rev == 0xc) -#define CHIP_REV_IS_EMUL(_p_dev) (CHIP_REV_IS_EMUL_A0(_p_dev) || \ - CHIP_REV_IS_EMUL_B0(_p_dev)) -#define CHIP_REV_IS_FPGA_A0(_p_dev) ((_p_dev)->chip_rev == 0xf) -#define CHIP_REV_IS_FPGA_B0(_p_dev) ((_p_dev)->chip_rev == 0xd) -#define CHIP_REV_IS_FPGA(_p_dev) (CHIP_REV_IS_FPGA_A0(_p_dev) || \ - CHIP_REV_IS_FPGA_B0(_p_dev)) +#define CHIP_REV_IS_TEDIBEAR(_p_dev) ((_p_dev)->chip_rev == 0x5) +#define CHIP_REV_IS_EMUL_A0(_p_dev) ((_p_dev)->chip_rev == 0xe) +#define CHIP_REV_IS_EMUL_B0(_p_dev) ((_p_dev)->chip_rev == 0xc) +#define CHIP_REV_IS_EMUL(_p_dev) \ + (CHIP_REV_IS_EMUL_A0(_p_dev) || CHIP_REV_IS_EMUL_B0(_p_dev)) +#define CHIP_REV_IS_FPGA_A0(_p_dev) ((_p_dev)->chip_rev == 0xf) +#define CHIP_REV_IS_FPGA_B0(_p_dev) ((_p_dev)->chip_rev == 0xd) +#define CHIP_REV_IS_FPGA(_p_dev) \ + (CHIP_REV_IS_FPGA_A0(_p_dev) || CHIP_REV_IS_FPGA_B0(_p_dev)) #define CHIP_REV_IS_SLOW(_p_dev) \ - (CHIP_REV_IS_EMUL(_p_dev) || CHIP_REV_IS_FPGA(_p_dev)) + (CHIP_REV_IS_EMUL(_p_dev) || CHIP_REV_IS_FPGA(_p_dev)) #define CHIP_REV_IS_A0(_p_dev) \ - (CHIP_REV_IS_EMUL_A0(_p_dev) || \ - CHIP_REV_IS_FPGA_A0(_p_dev) || \ - !(_p_dev)->chip_rev) + (CHIP_REV_IS_EMUL_A0(_p_dev) || CHIP_REV_IS_FPGA_A0(_p_dev) || \ + (!(_p_dev)->chip_rev && !(_p_dev)->chip_metal)) #define CHIP_REV_IS_B0(_p_dev) \ - (CHIP_REV_IS_EMUL_B0(_p_dev) || \ - CHIP_REV_IS_FPGA_B0(_p_dev) || \ - (_p_dev)->chip_rev == 1) -#define CHIP_REV_IS_ASIC(_p_dev) (!CHIP_REV_IS_SLOW(_p_dev)) + (CHIP_REV_IS_EMUL_B0(_p_dev) || CHIP_REV_IS_FPGA_B0(_p_dev) || \ + ((_p_dev)->chip_rev == 1 && !(_p_dev)->chip_metal)) +#define CHIP_REV_IS_ASIC(_p_dev) !CHIP_REV_IS_SLOW(_p_dev) #else -#define CHIP_REV_IS_A0(_p_dev) (!(_p_dev)->chip_rev) -#define CHIP_REV_IS_B0(_p_dev) ((_p_dev)->chip_rev == 1) +#define CHIP_REV_IS_A0(_p_dev) \ + (!(_p_dev)->chip_rev && !(_p_dev)->chip_metal) +#define CHIP_REV_IS_B0(_p_dev) \ + ((_p_dev)->chip_rev == 1 && !(_p_dev)->chip_metal) #endif - u16 chip_metal; + u8 chip_metal; #define CHIP_METAL_MASK 0xff -#define CHIP_METAL_SHIFT 4 +#define CHIP_METAL_SHIFT 0 - u16 chip_bond_id; -#define CHIP_BOND_ID_MASK 0xf + u8 chip_bond_id; +#define CHIP_BOND_ID_MASK 0xff #define CHIP_BOND_ID_SHIFT 0 - u8 num_engines; - u8 num_ports_in_engines; - u8 num_funcs_in_port; + u8 num_engines; + u8 num_ports; + u8 num_ports_in_engine; + u8 num_funcs_in_port; - u8 path_id; - enum ecore_mf_mode mf_mode; -#define IS_MF_DEFAULT(_p_hwfn) \ - (((_p_hwfn)->p_dev)->mf_mode == ECORE_MF_DEFAULT) -#define IS_MF_SI(_p_hwfn) (((_p_hwfn)->p_dev)->mf_mode == ECORE_MF_NPAR) -#define IS_MF_SD(_p_hwfn) (((_p_hwfn)->p_dev)->mf_mode == ECORE_MF_OVLAN) + u8 path_id; + + unsigned long mf_bits; + enum ecore_mf_mode mf_mode; +#define IS_MF_DEFAULT(_p_hwfn) \ + (((_p_hwfn)->p_dev)->mf_mode == ECORE_MF_DEFAULT) +#define IS_MF_SI(_p_hwfn) \ + (((_p_hwfn)->p_dev)->mf_mode == ECORE_MF_NPAR) +#define IS_MF_SD(_p_hwfn) \ + (((_p_hwfn)->p_dev)->mf_mode == ECORE_MF_OVLAN) + + int pcie_width; + int pcie_speed; - int pcie_width; - int pcie_speed; - u8 ver_str[VER_SIZE]; /* Add MF related configuration */ - u8 mcp_rev; - u8 boot_mode; + u8 mcp_rev; + u8 boot_mode; - u8 wol; + u8 wol; - u32 int_mode; - enum ecore_coalescing_mode int_coalescing_mode; - u8 rx_coalesce_usecs; - u8 tx_coalesce_usecs; + u32 int_mode; + enum ecore_coalescing_mode int_coalescing_mode; + u16 rx_coalesce_usecs; + u16 tx_coalesce_usecs; /* Start Bar offset of first hwfn */ - void OSAL_IOMEM *regview; - void OSAL_IOMEM *doorbells; - u64 db_phys_addr; - unsigned long db_size; + void OSAL_IOMEM *regview; + void OSAL_IOMEM *doorbells; + u64 db_phys_addr; + unsigned long db_size; /* PCI */ - u8 cache_shift; + u8 cache_shift; /* Init */ - const struct iro *iro_arr; -#define IRO (p_hwfn->p_dev->iro_arr) + const struct iro *iro_arr; + #define IRO (p_hwfn->p_dev->iro_arr) /* HW functions */ - u8 num_hwfns; - struct ecore_hwfn hwfns[MAX_HWFNS_PER_DEVICE]; + u8 num_hwfns; + struct ecore_hwfn hwfns[MAX_HWFNS_PER_DEVICE]; +#define ECORE_IS_CMT(dev) ((dev)->num_hwfns > 1) /* SRIOV */ - struct ecore_hw_sriov_info sriov_info; - unsigned long tunn_mode; -#define IS_ECORE_SRIOV(edev) (!!((edev)->sriov_info.total_vfs)) - bool b_is_vf; + struct ecore_hw_sriov_info *p_iov_info; +#define IS_ECORE_SRIOV(p_dev) (!!(p_dev)->p_iov_info) + struct ecore_tunnel_info tunnel; + bool b_is_vf; + bool b_dont_override_vf_msix; - u32 drv_type; + u32 drv_type; - struct ecore_eth_stats *reset_stats; - struct ecore_fw_data *fw_data; + u32 rdma_max_sge; + u32 rdma_max_inline; + u32 rdma_max_srq_sge; - u32 mcp_nvm_resp; + struct ecore_eth_stats *reset_stats; + struct ecore_fw_data *fw_data; + + u32 mcp_nvm_resp; /* Recovery */ - bool recov_in_prog; + bool recov_in_prog; + +/* Indicates whether should prevent attentions from being reasserted */ + + bool attn_clr_en; + + /* Indicates whether allowing the MFW to collect a crash dump */ + bool allow_mdump; + + /* Indicates if the reg_fifo is checked after any register access */ + bool chk_reg_fifo; #ifndef ASIC_ONLY - bool b_is_emul_full; + bool b_is_emul_full; #endif - void *firmware; - - u64 fw_len; +#ifdef CONFIG_ECORE_BINARY_FW /* @DPDK */ + void *firmware; + u64 fw_len; +#endif + /* @DPDK */ + struct ecore_dbg_feature dbg_features[DBG_FEATURE_NUM]; + u8 engine_for_debug; }; #define NUM_OF_VFS(dev) (ECORE_IS_BB(dev) ? MAX_NUM_VFS_BB \ @@ -688,12 +875,7 @@ struct ecore_dev { #define NUM_OF_ENG_PFS(dev) (ECORE_IS_BB(dev) ? MAX_NUM_PFS_BB \ : MAX_NUM_PFS_K2) -#define ENABLE_EAGLE_ENG1_WORKAROUND(p_hwfn) ( \ - (ECORE_IS_BB_A0(p_hwfn->p_dev)) && \ - (ECORE_PATH_ID(p_hwfn) == 1) && \ - ((p_hwfn->hw_info.port_mode == ECORE_PORT_MODE_DE_2X40G) || \ - (p_hwfn->hw_info.port_mode == ECORE_PORT_MODE_DE_2X50G) || \ - (p_hwfn->hw_info.port_mode == ECORE_PORT_MODE_DE_2X25G))) +#define CRC8_TABLE_SIZE 256 /** * @brief ecore_concrete_to_sw_fid - get the sw function id from @@ -703,11 +885,10 @@ struct ecore_dev { * * @return OSAL_INLINE u8 */ -static OSAL_INLINE u8 ecore_concrete_to_sw_fid(struct ecore_dev *p_dev, - u32 concrete_fid) +static OSAL_INLINE u8 ecore_concrete_to_sw_fid(u32 concrete_fid) { - u8 vfid = GET_FIELD(concrete_fid, PXP_CONCRETE_FID_VFID); - u8 pfid = GET_FIELD(concrete_fid, PXP_CONCRETE_FID_PFID); + u8 vfid = GET_FIELD(concrete_fid, PXP_CONCRETE_FID_VFID); + u8 pfid = GET_FIELD(concrete_fid, PXP_CONCRETE_FID_PFID); u8 vf_valid = GET_FIELD(concrete_fid, PXP_CONCRETE_FID_VFVALID); u8 sw_fid; @@ -719,23 +900,12 @@ static OSAL_INLINE u8 ecore_concrete_to_sw_fid(struct ecore_dev *p_dev, return sw_fid; } -#define PURE_LB_TC 8 -#define OOO_LB_TC 9 - -static OSAL_INLINE u16 ecore_sriov_get_next_vf(struct ecore_hwfn *p_hwfn, - u16 rel_vf_id) -{ - u16 i; - - for (i = rel_vf_id; i < p_hwfn->p_dev->sriov_info.total_vfs; i++) - if (ECORE_IS_VF_ACTIVE(p_hwfn->p_dev, i)) - return i; - - return p_hwfn->p_dev->sriov_info.total_vfs; -} +#define PKT_LB_TC 9 +#define MAX_NUM_VOQS_E4 20 int ecore_configure_vport_wfq(struct ecore_dev *p_dev, u16 vp_id, u32 rate); void ecore_configure_vp_wfq_on_link_change(struct ecore_dev *p_dev, + struct ecore_ptt *p_ptt, u32 min_pf_rate); int ecore_configure_pf_max_bandwidth(struct ecore_dev *p_dev, u8 max_bw); @@ -743,11 +913,37 @@ int ecore_configure_pf_min_bandwidth(struct ecore_dev *p_dev, u8 min_bw); void ecore_clean_wfq_db(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt); int ecore_device_num_engines(struct ecore_dev *p_dev); int ecore_device_num_ports(struct ecore_dev *p_dev); +void ecore_set_fw_mac_addr(__le16 *fw_msb, __le16 *fw_mid, __le16 *fw_lsb, + u8 *mac); -#define ecore_for_each_vf(_p_hwfn, _i) \ - for (_i = ecore_sriov_get_next_vf(_p_hwfn, 0); \ - _i < _p_hwfn->p_dev->sriov_info.total_vfs; \ - _i = ecore_sriov_get_next_vf(_p_hwfn, _i + 1)) +/* Flags for indication of required queues */ +#define PQ_FLAGS_RLS (1 << 0) +#define PQ_FLAGS_MCOS (1 << 1) +#define PQ_FLAGS_LB (1 << 2) +#define PQ_FLAGS_OOO (1 << 3) +#define PQ_FLAGS_ACK (1 << 4) +#define PQ_FLAGS_OFLD (1 << 5) +#define PQ_FLAGS_VFS (1 << 6) + +/* physical queue index for cm context intialization */ +u16 ecore_get_cm_pq_idx(struct ecore_hwfn *p_hwfn, u32 pq_flags); +u16 ecore_get_cm_pq_idx_mcos(struct ecore_hwfn *p_hwfn, u8 tc); +u16 ecore_get_cm_pq_idx_vf(struct ecore_hwfn *p_hwfn, u16 vf); +u16 ecore_get_cm_pq_idx_rl(struct ecore_hwfn *p_hwfn, u8 qpid); + +const char *ecore_hw_get_resc_name(enum ecore_resources res_id); + +/* doorbell recovery mechanism */ +void ecore_db_recovery_dp(struct ecore_hwfn *p_hwfn); +void ecore_db_recovery_execute(struct ecore_hwfn *p_hwfn, + enum ecore_db_rec_exec); + +/* amount of resources used in qm init */ +u8 ecore_init_qm_get_num_tcs(struct ecore_hwfn *p_hwfn); +u16 ecore_init_qm_get_num_vfs(struct ecore_hwfn *p_hwfn); +u16 ecore_init_qm_get_num_pf_rls(struct ecore_hwfn *p_hwfn); +u16 ecore_init_qm_get_num_vports(struct ecore_hwfn *p_hwfn); +u16 ecore_init_qm_get_num_pqs(struct ecore_hwfn *p_hwfn); #define ECORE_LEADING_HWFN(dev) (&dev->hwfns[0]) diff --git a/dpdk/drivers/net/qede/base/ecore_chain.h b/dpdk/drivers/net/qede/base/ecore_chain.h index c5734490..ba272a91 100644 --- a/dpdk/drivers/net/qede/base/ecore_chain.h +++ b/dpdk/drivers/net/qede/base/ecore_chain.h @@ -54,21 +54,9 @@ struct ecore_chain_pbl_u32 { u32 cons_page_idx; }; -struct ecore_chain_pbl { - /* Base address of a pre-allocated buffer for pbl */ - dma_addr_t p_phys_table; - void *p_virt_table; - - /* Table for keeping the virtual addresses of the chain pages, - * respectively to the physical addresses in the pbl table. - */ - void **pp_virt_addr_tbl; - - /* Index to current used page by producer/consumer */ - union { - struct ecore_chain_pbl_u16 pbl16; - struct ecore_chain_pbl_u32 pbl32; - } u; +struct ecore_chain_ext_pbl { + dma_addr_t p_pbl_phys; + void *p_pbl_virt; }; struct ecore_chain_u16 { @@ -84,40 +72,77 @@ struct ecore_chain_u32 { }; struct ecore_chain { - /* Address of first page of the chain */ - void *p_virt_addr; - dma_addr_t p_phys_addr; - + /* fastpath portion of the chain - required for commands such + * as produce / consume. + */ /* Point to next element to produce/consume */ void *p_prod_elem; void *p_cons_elem; - enum ecore_chain_mode mode; - enum ecore_chain_use_mode intended_use; + /* Fastpath portions of the PBL [if exists] */ + + struct { + /* Table for keeping the virtual addresses of the chain pages, + * respectively to the physical addresses in the pbl table. + */ + void **pp_virt_addr_tbl; + + union { + struct ecore_chain_pbl_u16 u16; + struct ecore_chain_pbl_u32 u32; + } c; + } pbl; - enum ecore_chain_cnt_type cnt_type; union { struct ecore_chain_u16 chain16; struct ecore_chain_u32 chain32; } u; - u32 page_cnt; + /* Capacity counts only usable elements */ + u32 capacity; + u32 page_cnt; - /* Number of elements - capacity is for usable elements only, - * while size will contain total number of elements [for entire chain]. + /* A u8 would suffice for mode, but it would save as a lot of headaches + * on castings & defaults. */ - u32 capacity; - u32 size; + enum ecore_chain_mode mode; /* Elements information for fast calculations */ u16 elem_per_page; u16 elem_per_page_mask; - u16 elem_unusable; - u16 usable_per_page; u16 elem_size; u16 next_page_mask; + u16 usable_per_page; + u8 elem_unusable; - struct ecore_chain_pbl pbl; + u8 cnt_type; + + /* Slowpath of the chain - required for initialization and destruction, + * but isn't involved in regular functionality. + */ + + /* Base address of a pre-allocated buffer for pbl */ + struct { + dma_addr_t p_phys_table; + void *p_virt_table; + } pbl_sp; + + /* Address of first page of the chain - the address is required + * for fastpath operation [consume/produce] but only for the the SINGLE + * flavour which isn't considered fastpath [== SPQ]. + */ + void *p_virt_addr; + dma_addr_t p_phys_addr; + + /* Total number of elements [for entire chain] */ + u32 size; + + u8 intended_use; + + /* TBD - do we really need this? Couldn't find usage for it */ + bool b_external_pbl; + + void *dp_ctx; }; #define ECORE_CHAIN_PBL_ENTRY_SIZE (8) @@ -126,10 +151,10 @@ struct ecore_chain { #define UNUSABLE_ELEMS_PER_PAGE(elem_size, mode) \ ((mode == ECORE_CHAIN_MODE_NEXT_PTR) ? \ - (1 + ((sizeof(struct ecore_chain_next) - 1) / \ - (elem_size))) : 0) + (u8)(1 + ((sizeof(struct ecore_chain_next) - 1) / \ + (elem_size))) : 0) -#define USABLE_ELEMS_PER_PAGE(elem_size, mode) \ +#define USABLE_ELEMS_PER_PAGE(elem_size, mode) \ ((u32)(ELEMS_PER_PAGE(elem_size) - \ UNUSABLE_ELEMS_PER_PAGE(elem_size, mode))) @@ -183,7 +208,7 @@ static OSAL_INLINE u16 ecore_chain_get_elem_left(struct ecore_chain *p_chain) (u32)p_chain->u.chain16.cons_idx); if (p_chain->mode == ECORE_CHAIN_MODE_NEXT_PTR) used -= p_chain->u.chain16.prod_idx / p_chain->elem_per_page - - p_chain->u.chain16.cons_idx / p_chain->elem_per_page; + p_chain->u.chain16.cons_idx / p_chain->elem_per_page; return (u16)(p_chain->capacity - used); } @@ -196,11 +221,11 @@ ecore_chain_get_elem_left_u32(struct ecore_chain *p_chain) OSAL_ASSERT(is_chain_u32(p_chain)); used = (u32)(((u64)ECORE_U32_MAX + 1 + - (u64)(p_chain->u.chain32.prod_idx)) - - (u64)p_chain->u.chain32.cons_idx); + (u64)(p_chain->u.chain32.prod_idx)) - + (u64)p_chain->u.chain32.cons_idx); if (p_chain->mode == ECORE_CHAIN_MODE_NEXT_PTR) used -= p_chain->u.chain32.prod_idx / p_chain->elem_per_page - - p_chain->u.chain32.cons_idx / p_chain->elem_per_page; + p_chain->u.chain32.cons_idx / p_chain->elem_per_page; return p_chain->capacity - used; } @@ -236,7 +261,7 @@ u16 ecore_chain_get_usable_per_page(struct ecore_chain *p_chain) } static OSAL_INLINE -u16 ecore_chain_get_unusable_per_page(struct ecore_chain *p_chain) +u8 ecore_chain_get_unusable_per_page(struct ecore_chain *p_chain) { return p_chain->elem_unusable; } @@ -254,7 +279,7 @@ static OSAL_INLINE u32 ecore_chain_get_page_cnt(struct ecore_chain *p_chain) static OSAL_INLINE dma_addr_t ecore_chain_get_pbl_phys(struct ecore_chain *p_chain) { - return p_chain->pbl.p_phys_table; + return p_chain->pbl_sp.p_phys_table; } /** @@ -279,9 +304,9 @@ ecore_chain_advance_page(struct ecore_chain *p_chain, void **p_next_elem, p_next = (struct ecore_chain_next *)(*p_next_elem); *p_next_elem = p_next->next_virt; if (is_chain_u16(p_chain)) - *(u16 *)idx_to_inc += p_chain->elem_unusable; + *(u16 *)idx_to_inc += (u16)p_chain->elem_unusable; else - *(u32 *)idx_to_inc += p_chain->elem_unusable; + *(u32 *)idx_to_inc += (u16)p_chain->elem_unusable; break; case ECORE_CHAIN_MODE_SINGLE: *p_next_elem = p_chain->p_virt_addr; @@ -307,21 +332,23 @@ ecore_chain_advance_page(struct ecore_chain *p_chain, void **p_next_elem, (((p)->u.chain32.idx & (p)->elem_per_page_mask) == (p)->usable_per_page) #define is_unusable_next_idx(p, idx) \ - ((((p)->u.chain16.idx + 1) & (p)->elem_per_page_mask) == \ - (p)->usable_per_page) + ((((p)->u.chain16.idx + 1) & \ + (p)->elem_per_page_mask) == (p)->usable_per_page) #define is_unusable_next_idx_u32(p, idx) \ - ((((p)->u.chain32.idx + 1) & (p)->elem_per_page_mask) \ - == (p)->usable_per_page) + ((((p)->u.chain32.idx + 1) & \ + (p)->elem_per_page_mask) == (p)->usable_per_page) #define test_and_skip(p, idx) \ do { \ if (is_chain_u16(p)) { \ if (is_unusable_idx(p, idx)) \ - (p)->u.chain16.idx += (p)->elem_unusable; \ + (p)->u.chain16.idx += \ + (p)->elem_unusable; \ } else { \ if (is_unusable_idx_u32(p, idx)) \ - (p)->u.chain32.idx += (p)->elem_unusable; \ + (p)->u.chain32.idx += \ + (p)->elem_unusable; \ } \ } while (0) @@ -380,7 +407,7 @@ static OSAL_INLINE void *ecore_chain_produce(struct ecore_chain *p_chain) if ((p_chain->u.chain16.prod_idx & p_chain->elem_per_page_mask) == p_chain->next_page_mask) { p_prod_idx = &p_chain->u.chain16.prod_idx; - p_prod_page_idx = &p_chain->pbl.u.pbl16.prod_page_idx; + p_prod_page_idx = &p_chain->pbl.c.u16.prod_page_idx; ecore_chain_advance_page(p_chain, &p_chain->p_prod_elem, p_prod_idx, p_prod_page_idx); } @@ -389,7 +416,7 @@ static OSAL_INLINE void *ecore_chain_produce(struct ecore_chain *p_chain) if ((p_chain->u.chain32.prod_idx & p_chain->elem_per_page_mask) == p_chain->next_page_mask) { p_prod_idx = &p_chain->u.chain32.prod_idx; - p_prod_page_idx = &p_chain->pbl.u.pbl32.prod_page_idx; + p_prod_page_idx = &p_chain->pbl.c.u32.prod_page_idx; ecore_chain_advance_page(p_chain, &p_chain->p_prod_elem, p_prod_idx, p_prod_page_idx); } @@ -454,7 +481,7 @@ static OSAL_INLINE void *ecore_chain_consume(struct ecore_chain *p_chain) if ((p_chain->u.chain16.cons_idx & p_chain->elem_per_page_mask) == p_chain->next_page_mask) { p_cons_idx = &p_chain->u.chain16.cons_idx; - p_cons_page_idx = &p_chain->pbl.u.pbl16.cons_page_idx; + p_cons_page_idx = &p_chain->pbl.c.u16.cons_page_idx; ecore_chain_advance_page(p_chain, &p_chain->p_cons_elem, p_cons_idx, p_cons_page_idx); } @@ -463,7 +490,7 @@ static OSAL_INLINE void *ecore_chain_consume(struct ecore_chain *p_chain) if ((p_chain->u.chain32.cons_idx & p_chain->elem_per_page_mask) == p_chain->next_page_mask) { p_cons_idx = &p_chain->u.chain32.cons_idx; - p_cons_page_idx = &p_chain->pbl.u.pbl32.cons_page_idx; + p_cons_page_idx = &p_chain->pbl.c.u32.cons_page_idx; ecore_chain_advance_page(p_chain, &p_chain->p_cons_elem, p_cons_idx, p_cons_page_idx); } @@ -507,25 +534,26 @@ static OSAL_INLINE void ecore_chain_reset(struct ecore_chain *p_chain) u32 reset_val = p_chain->page_cnt - 1; if (is_chain_u16(p_chain)) { - p_chain->pbl.u.pbl16.prod_page_idx = (u16)reset_val; - p_chain->pbl.u.pbl16.cons_page_idx = (u16)reset_val; + p_chain->pbl.c.u16.prod_page_idx = (u16)reset_val; + p_chain->pbl.c.u16.cons_page_idx = (u16)reset_val; } else { - p_chain->pbl.u.pbl32.prod_page_idx = reset_val; - p_chain->pbl.u.pbl32.cons_page_idx = reset_val; + p_chain->pbl.c.u32.prod_page_idx = reset_val; + p_chain->pbl.c.u32.cons_page_idx = reset_val; } } switch (p_chain->intended_use) { - case ECORE_CHAIN_USE_TO_CONSUME_PRODUCE: - case ECORE_CHAIN_USE_TO_PRODUCE: - /* Do nothing */ - break; - case ECORE_CHAIN_USE_TO_CONSUME: /* produce empty elements */ for (i = 0; i < p_chain->capacity; i++) ecore_chain_recycle_consumed(p_chain); break; + + case ECORE_CHAIN_USE_TO_CONSUME_PRODUCE: + case ECORE_CHAIN_USE_TO_PRODUCE: + default: + /* Do nothing */ + break; } } @@ -540,20 +568,21 @@ static OSAL_INLINE void ecore_chain_reset(struct ecore_chain *p_chain) * @param intended_use * @param mode * @param cnt_type + * @param dp_ctx */ static OSAL_INLINE void ecore_chain_init_params(struct ecore_chain *p_chain, u32 page_cnt, u8 elem_size, enum ecore_chain_use_mode intended_use, enum ecore_chain_mode mode, - enum ecore_chain_cnt_type cnt_type) + enum ecore_chain_cnt_type cnt_type, void *dp_ctx) { /* chain fixed parameters */ p_chain->p_virt_addr = OSAL_NULL; p_chain->p_phys_addr = 0; p_chain->elem_size = elem_size; - p_chain->intended_use = intended_use; + p_chain->intended_use = (u8)intended_use; p_chain->mode = mode; - p_chain->cnt_type = cnt_type; + p_chain->cnt_type = (u8)cnt_type; p_chain->elem_per_page = ELEMS_PER_PAGE(elem_size); p_chain->usable_per_page = USABLE_ELEMS_PER_PAGE(elem_size, mode); @@ -565,10 +594,12 @@ ecore_chain_init_params(struct ecore_chain *p_chain, u32 page_cnt, u8 elem_size, p_chain->page_cnt = page_cnt; p_chain->capacity = p_chain->usable_per_page * page_cnt; p_chain->size = p_chain->elem_per_page * page_cnt; - - p_chain->pbl.p_phys_table = 0; - p_chain->pbl.p_virt_table = OSAL_NULL; + p_chain->b_external_pbl = false; + p_chain->pbl_sp.p_phys_table = 0; + p_chain->pbl_sp.p_virt_table = OSAL_NULL; p_chain->pbl.pp_virt_addr_tbl = OSAL_NULL; + + p_chain->dp_ctx = dp_ctx; } /** @@ -609,8 +640,8 @@ static OSAL_INLINE void ecore_chain_init_pbl_mem(struct ecore_chain *p_chain, dma_addr_t p_phys_pbl, void **pp_virt_addr_tbl) { - p_chain->pbl.p_phys_table = p_phys_pbl; - p_chain->pbl.p_virt_table = p_virt_pbl; + p_chain->pbl_sp.p_phys_table = p_phys_pbl; + p_chain->pbl_sp.p_virt_table = p_virt_pbl; p_chain->pbl.pp_virt_addr_tbl = pp_virt_addr_tbl; } @@ -721,4 +752,14 @@ static OSAL_INLINE void ecore_chain_pbl_zero_mem(struct ecore_chain *p_chain) ECORE_CHAIN_PAGE_SIZE); } +int ecore_chain_print(struct ecore_chain *p_chain, char *buffer, + u32 buffer_size, u32 *element_indx, u32 stop_indx, + bool print_metadata, + int (*func_ptr_print_element)(struct ecore_chain *p_chain, + void *p_element, + char *buffer), + int (*func_ptr_print_metadata)(struct ecore_chain + *p_chain, + char *buffer)); + #endif /* __ECORE_CHAIN_H__ */ diff --git a/dpdk/drivers/net/qede/base/ecore_cxt.c b/dpdk/drivers/net/qede/base/ecore_cxt.c index 1201c1a9..50bd66da 100644 --- a/dpdk/drivers/net/qede/base/ecore_cxt.c +++ b/dpdk/drivers/net/qede/base/ecore_cxt.c @@ -8,6 +8,7 @@ #include "bcm_osal.h" #include "reg_addr.h" +#include "common_hsi.h" #include "ecore_hsi_common.h" #include "ecore_hsi_eth.h" #include "ecore_rt_defs.h" @@ -18,6 +19,8 @@ #include "ecore_cxt.h" #include "ecore_hw.h" #include "ecore_dev_api.h" +#include "ecore_sriov.h" +#include "ecore_mcp.h" /* Max number of connection types in HW (DQ/CDU etc.) */ #define MAX_CONN_TYPES PROTOCOLID_COMMON @@ -38,10 +41,7 @@ #define TM_ELEM_SIZE 4 /* ILT constants */ -/* If for some reason, HW P size is modified to be less than 32K, - * special handling needs to be made for CDU initialization - */ -#define ILT_DEFAULT_HW_P_SIZE 3 +#define ILT_DEFAULT_HW_P_SIZE 4 #define ILT_PAGE_IN_BYTES(hw_p_size) (1U << ((hw_p_size) + 12)) #define ILT_CFG_REG(cli, reg) PSWRQ2_REG_##cli##_##reg##_RT_OFFSET @@ -56,8 +56,17 @@ /* connection context union */ union conn_context { - struct core_conn_context core_ctx; - struct eth_conn_context eth_ctx; + struct e4_core_conn_context core_ctx; + struct e4_eth_conn_context eth_ctx; +}; + +/* TYPE-0 task context - iSCSI, FCOE */ +union type0_task_context { +}; + +/* TYPE-1 task context - ROCE */ +union type1_task_context { + struct regpair reserved; /* @DPDK */ }; struct src_ent { @@ -71,6 +80,14 @@ struct src_ent { #define CONN_CXT_SIZE(p_hwfn) \ ALIGNED_TYPE_SIZE(union conn_context, p_hwfn) +#define SRQ_CXT_SIZE (sizeof(struct regpair) * 8) /* @DPDK */ + +#define TYPE0_TASK_CXT_SIZE(p_hwfn) \ + ALIGNED_TYPE_SIZE(union type0_task_context, p_hwfn) + +/* Alignment is inherent to the type1_task_context structure */ +#define TYPE1_TASK_CXT_SIZE(p_hwfn) sizeof(union type1_task_context) + /* PF per protocl configuration object */ #define TASK_SEGMENTS (NUM_TASK_PF_SEGMENTS + NUM_TASK_VF_SEGMENTS) #define TASK_SEGMENT_VF (NUM_TASK_PF_SEGMENTS) @@ -83,7 +100,6 @@ struct ecore_tid_seg { struct ecore_conn_type_cfg { u32 cid_count; - u32 cid_start; u32 cids_per_vf; struct ecore_tid_seg tid_seg[TASK_SEGMENTS]; }; @@ -96,6 +112,7 @@ struct ecore_conn_type_cfg { #define ILT_CLI_PF_BLOCKS (1 + NUM_TASK_PF_SEGMENTS * 2) #define ILT_CLI_VF_BLOCKS (1 + NUM_TASK_VF_SEGMENTS * 2) #define CDUC_BLK (0) +#define SRQ_BLK (0) #define CDUT_SEG_BLK(n) (1 + (u8)(n)) #define CDUT_FL_SEG_BLK(n, X) (1 + (n) + NUM_TASK_##X##_SEGMENTS) @@ -105,6 +122,7 @@ enum ilt_clients { ILT_CLI_QM, ILT_CLI_TM, ILT_CLI_SRC, + ILT_CLI_TSDM, ILT_CLI_MAX }; @@ -174,29 +192,52 @@ struct ecore_cxt_mngr { /* Acquired CIDs */ struct ecore_cid_acquired_map acquired[MAX_CONN_TYPES]; + /* TBD - do we want this allocated to reserve space? */ + struct ecore_cid_acquired_map + acquired_vf[MAX_CONN_TYPES][COMMON_MAX_NUM_VFS]; /* ILT shadow table */ struct ecore_dma_mem *ilt_shadow; u32 pf_start_line; + /* Mutex for a dynamic ILT allocation */ + osal_mutex_t mutex; + /* SRC T2 */ struct ecore_dma_mem *t2; u32 t2_num_pages; u64 first_free; u64 last_free; -}; -/* check if resources/configuration is required according to protocol type */ -static OSAL_INLINE bool src_proto(enum protocol_type type) -{ - return type == PROTOCOLID_TOE; -} + /* The infrastructure originally was very generic and context/task + * oriented - per connection-type we would set how many of those + * are needed, and later when determining how much memory we're + * needing for a given block we'd iterate over all the relevant + * connection-types. + * But since then we've had some additional resources, some of which + * require memory which is indepent of the general context/task + * scheme. We add those here explicitly per-feature. + */ + + /* total number of SRQ's for this hwfn */ + u32 srq_count; + + /* Maximal number of L2 steering filters */ + u32 arfs_count; + + /* TODO - VF arfs filters ? */ +}; static OSAL_INLINE bool tm_cid_proto(enum protocol_type type) { return type == PROTOCOLID_TOE; } +static bool tm_tid_proto(enum protocol_type type) +{ + return type == PROTOCOLID_FCOE; +} + /* counts the iids for the CDU/CDUC ILT client configuration */ struct ecore_cdu_iids { u32 pf_cids; @@ -220,18 +261,18 @@ struct ecore_src_iids { u32 per_vf_cids; }; -static OSAL_INLINE void ecore_cxt_src_iids(struct ecore_cxt_mngr *p_mngr, - struct ecore_src_iids *iids) +static void ecore_cxt_src_iids(struct ecore_cxt_mngr *p_mngr, + struct ecore_src_iids *iids) { u32 i; for (i = 0; i < MAX_CONN_TYPES; i++) { - if (!src_proto(i)) - continue; - iids->pf_cids += p_mngr->conn_cfg[i].cid_count; iids->per_vf_cids += p_mngr->conn_cfg[i].cids_per_vf; } + + /* Add L2 filtering filters in addition */ + iids->pf_cids += p_mngr->arfs_count; } /* counts the iids for the Timers block configuration */ @@ -243,17 +284,43 @@ struct ecore_tm_iids { u32 per_vf_tids; }; -static OSAL_INLINE void ecore_cxt_tm_iids(struct ecore_cxt_mngr *p_mngr, - struct ecore_tm_iids *iids) +static void ecore_cxt_tm_iids(struct ecore_cxt_mngr *p_mngr, + struct ecore_tm_iids *iids) { + bool tm_vf_required = false; + bool tm_required = false; u32 i, j; for (i = 0; i < MAX_CONN_TYPES; i++) { struct ecore_conn_type_cfg *p_cfg = &p_mngr->conn_cfg[i]; - if (tm_cid_proto(i)) { + if (tm_cid_proto(i) || tm_required) { + if (p_cfg->cid_count) + tm_required = true; + iids->pf_cids += p_cfg->cid_count; - iids->per_vf_cids += p_cfg->cids_per_vf; + } + + if (tm_cid_proto(i) || tm_vf_required) { + if (p_cfg->cids_per_vf) + tm_vf_required = true; + + } + + if (tm_tid_proto(i)) { + struct ecore_tid_seg *segs = p_cfg->tid_seg; + + /* for each segment there is at most one + * protocol for which count is not 0. + */ + for (j = 0; j < NUM_TASK_PF_SEGMENTS; j++) + iids->pf_tids[j] += segs[j].count; + + /* The last array elelment is for the VFs. As for PF + * segments there can be only one protocol for + * which this value is not 0. + */ + iids->per_vf_tids += segs[NUM_TASK_PF_SEGMENTS].count; } } @@ -267,7 +334,8 @@ static OSAL_INLINE void ecore_cxt_tm_iids(struct ecore_cxt_mngr *p_mngr, } } -void ecore_cxt_qm_iids(struct ecore_hwfn *p_hwfn, struct ecore_qm_iids *iids) +static void ecore_cxt_qm_iids(struct ecore_hwfn *p_hwfn, + struct ecore_qm_iids *iids) { struct ecore_cxt_mngr *p_mngr = p_hwfn->p_cxt_mngr; struct ecore_tid_seg *segs; @@ -316,8 +384,22 @@ static struct ecore_tid_seg *ecore_cxt_tid_seg_info(struct ecore_hwfn *p_hwfn, return OSAL_NULL; } +static void ecore_cxt_set_srq_count(struct ecore_hwfn *p_hwfn, u32 num_srqs) +{ + struct ecore_cxt_mngr *p_mgr = p_hwfn->p_cxt_mngr; + + p_mgr->srq_count = num_srqs; +} + +u32 ecore_cxt_get_srq_count(struct ecore_hwfn *p_hwfn) +{ + struct ecore_cxt_mngr *p_mgr = p_hwfn->p_cxt_mngr; + + return p_mgr->srq_count; +} + /* set the iids (cid/tid) count per protocol */ -void ecore_cxt_set_proto_cid_count(struct ecore_hwfn *p_hwfn, +static void ecore_cxt_set_proto_cid_count(struct ecore_hwfn *p_hwfn, enum protocol_type type, u32 cid_count, u32 vf_cid_cnt) { @@ -343,7 +425,7 @@ u32 ecore_cxt_get_proto_cid_start(struct ecore_hwfn *p_hwfn, return p_hwfn->p_cxt_mngr->acquired[type].start_cid; } -static u32 ecore_cxt_get_proto_tid_count(struct ecore_hwfn *p_hwfn, +u32 ecore_cxt_get_proto_tid_count(struct ecore_hwfn *p_hwfn, enum protocol_type type) { u32 cnt = 0; @@ -378,7 +460,7 @@ static void ecore_ilt_cli_blk_fill(struct ecore_ilt_client_cfg *p_cli, { u32 ilt_size = ILT_PAGE_IN_BYTES(p_cli->p_size.val); - /* verfiy called once for each block */ + /* verify that it's called once for each block */ if (p_blk->total_size) return; @@ -405,7 +487,8 @@ static void ecore_ilt_cli_adv_line(struct ecore_hwfn *p_hwfn, p_cli->last.val = *p_line - 1; DP_VERBOSE(p_hwfn, ECORE_MSG_ILT, - "ILT[Client %d] - Lines: [%08x - %08x]. Block - Size %08x [Real %08x] Start line %d\n", + "ILT[Client %d] - Lines: [%08x - %08x]. Block - Size %08x" + " [Real %08x] Start line %d\n", client_id, p_cli->first.val, p_cli->last.val, p_blk->total_size, p_blk->real_size_in_page, p_blk->start_line); @@ -453,7 +536,7 @@ enum _ecore_status_t ecore_cxt_cfg_ilt_compute(struct ecore_hwfn *p_hwfn) p_mngr->pf_start_line = RESC_START(p_hwfn, ECORE_ILT); DP_VERBOSE(p_hwfn, ECORE_MSG_ILT, - "hwfn [%d] - Set context manager starting line to be 0x%08x\n", + "hwfn [%d] - Set context mngr starting line to be 0x%08x\n", p_hwfn->my_id, p_hwfn->p_cxt_mngr->pf_start_line); /* CDUC */ @@ -605,7 +688,7 @@ enum _ecore_status_t ecore_cxt_cfg_ilt_compute(struct ecore_hwfn *p_hwfn) p_blk = &p_cli->pf_blks[0]; ecore_cxt_qm_iids(p_hwfn, &qm_iids); - total = ecore_qm_pf_mem_size(p_hwfn->rel_pf_id, qm_iids.cids, + total = ecore_qm_pf_mem_size(qm_iids.cids, qm_iids.vf_cids, qm_iids.tids, p_hwfn->qm_info.num_pqs, p_hwfn->qm_info.num_vf_pqs); @@ -670,14 +753,28 @@ enum _ecore_status_t ecore_cxt_cfg_ilt_compute(struct ecore_hwfn *p_hwfn) ecore_ilt_cli_adv_line(p_hwfn, p_cli, p_blk, &curr_line, ILT_CLI_TM); - p_cli->pf_total_lines = curr_line - p_blk->start_line; + p_cli->vf_total_lines = curr_line - p_blk->start_line; for (i = 1; i < p_mngr->vf_count; i++) { ecore_ilt_cli_adv_line(p_hwfn, p_cli, p_blk, &curr_line, ILT_CLI_TM); } } + /* TSDM (SRQ CONTEXT) */ + total = ecore_cxt_get_srq_count(p_hwfn); + + if (total) { + p_cli = &p_mngr->clients[ILT_CLI_TSDM]; + p_blk = &p_cli->pf_blks[SRQ_BLK]; + ecore_ilt_cli_blk_fill(p_cli, p_blk, curr_line, + total * SRQ_CXT_SIZE, SRQ_CXT_SIZE); + + ecore_ilt_cli_adv_line(p_hwfn, p_cli, p_blk, &curr_line, + ILT_CLI_TSDM); + p_cli->pf_total_lines = curr_line - p_blk->start_line; + } + if (curr_line - p_hwfn->p_cxt_mngr->pf_start_line > RESC_NUM(p_hwfn, ECORE_ILT)) { DP_ERR(p_hwfn, "too many ilt lines...#lines=%d\n", @@ -704,7 +801,6 @@ static void ecore_cxt_src_t2_free(struct ecore_hwfn *p_hwfn) p_mngr->t2[i].size); OSAL_FREE(p_hwfn->p_dev, p_mngr->t2); - p_mngr->t2 = OSAL_NULL; } static enum _ecore_status_t ecore_cxt_src_t2_alloc(struct ecore_hwfn *p_hwfn) @@ -787,7 +883,7 @@ static enum _ecore_status_t ecore_cxt_src_t2_alloc(struct ecore_hwfn *p_hwfn) val = 0; entries[j].next = OSAL_CPU_TO_BE64(val); - conn_num -= ent_per_page; + conn_num -= ent_num; } return ECORE_SUCCESS; @@ -797,18 +893,22 @@ t2_fail: return rc; } +#define for_each_ilt_valid_client(pos, clients) \ + for (pos = 0; pos < ILT_CLI_MAX; pos++) \ + if (!clients[pos].active) { \ + continue; \ + } else \ + + /* Total number of ILT lines used by this PF */ static u32 ecore_cxt_ilt_shadow_size(struct ecore_ilt_client_cfg *ilt_clients) { u32 size = 0; u32 i; - for (i = 0; i < ILT_CLI_MAX; i++) - if (!ilt_clients[i].active) - continue; - else - size += (ilt_clients[i].last.val - - ilt_clients[i].first.val + 1); + for_each_ilt_valid_client(i, ilt_clients) + size += (ilt_clients[i].last.val - + ilt_clients[i].first.val + 1); return size; } @@ -842,7 +942,7 @@ ecore_ilt_blk_alloc(struct ecore_hwfn *p_hwfn, u32 lines, line, sz_left, lines_to_skip = 0; /* Special handling for RoCE that supports dynamic allocation */ - if (ilt_client == ILT_CLI_CDUT) + if (ilt_client == ILT_CLI_CDUT || ilt_client == ILT_CLI_TSDM) return ECORE_SUCCESS; lines_to_skip = p_blk->dynamic_line_cnt; @@ -876,9 +976,9 @@ ecore_ilt_blk_alloc(struct ecore_hwfn *p_hwfn, ilt_shadow[line].size = size; DP_VERBOSE(p_hwfn, ECORE_MSG_ILT, - "ILT shadow: Line [%d] Physical 0x%" PRIx64 + "ILT shadow: Line [%d] Physical 0x%lx" " Virtual %p Size %d\n", - line, (u64)p_phys, p_virt, size); + line, (unsigned long)p_phys, p_virt, size); sz_left -= size; line++; @@ -892,15 +992,16 @@ static enum _ecore_status_t ecore_ilt_shadow_alloc(struct ecore_hwfn *p_hwfn) struct ecore_cxt_mngr *p_mngr = p_hwfn->p_cxt_mngr; struct ecore_ilt_client_cfg *clients = p_mngr->clients; struct ecore_ilt_cli_blk *p_blk; - enum _ecore_status_t rc; u32 size, i, j, k; + enum _ecore_status_t rc; size = ecore_cxt_ilt_shadow_size(clients); p_mngr->ilt_shadow = OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL, size * sizeof(struct ecore_dma_mem)); if (!p_mngr->ilt_shadow) { - DP_NOTICE(p_hwfn, true, "Failed to allocate ilt shadow table"); + DP_NOTICE(p_hwfn, true, + "Failed to allocate ilt shadow table\n"); rc = ECORE_NOMEM; goto ilt_shadow_fail; } @@ -909,10 +1010,7 @@ static enum _ecore_status_t ecore_ilt_shadow_alloc(struct ecore_hwfn *p_hwfn) "Allocated 0x%x bytes for ilt shadow\n", (u32)(size * sizeof(struct ecore_dma_mem))); - for (i = 0; i < ILT_CLI_MAX; i++) - if (!clients[i].active) { - continue; - } else { + for_each_ilt_valid_client(i, clients) { for (j = 0; j < ILT_CLI_PF_BLOCKS; j++) { p_blk = &clients[i].pf_blks[j]; rc = ecore_ilt_blk_alloc(p_hwfn, p_blk, i, 0); @@ -942,44 +1040,75 @@ ilt_shadow_fail: static void ecore_cid_map_free(struct ecore_hwfn *p_hwfn) { struct ecore_cxt_mngr *p_mngr = p_hwfn->p_cxt_mngr; - u32 type; + u32 type, vf; for (type = 0; type < MAX_CONN_TYPES; type++) { OSAL_FREE(p_hwfn->p_dev, p_mngr->acquired[type].cid_map); p_mngr->acquired[type].max_count = 0; p_mngr->acquired[type].start_cid = 0; + + for (vf = 0; vf < COMMON_MAX_NUM_VFS; vf++) { + OSAL_FREE(p_hwfn->p_dev, + p_mngr->acquired_vf[type][vf].cid_map); + p_mngr->acquired_vf[type][vf].max_count = 0; + p_mngr->acquired_vf[type][vf].start_cid = 0; + } } } +static enum _ecore_status_t +ecore_cid_map_alloc_single(struct ecore_hwfn *p_hwfn, u32 type, + u32 cid_start, u32 cid_count, + struct ecore_cid_acquired_map *p_map) +{ + u32 size; + + if (!cid_count) + return ECORE_SUCCESS; + + size = MAP_WORD_SIZE * DIV_ROUND_UP(cid_count, BITS_PER_MAP_WORD); + p_map->cid_map = OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL, size); + if (p_map->cid_map == OSAL_NULL) + return ECORE_NOMEM; + + p_map->max_count = cid_count; + p_map->start_cid = cid_start; + + DP_VERBOSE(p_hwfn, ECORE_MSG_CXT, + "Type %08x start: %08x count %08x\n", + type, p_map->start_cid, p_map->max_count); + + return ECORE_SUCCESS; +} + static enum _ecore_status_t ecore_cid_map_alloc(struct ecore_hwfn *p_hwfn) { struct ecore_cxt_mngr *p_mngr = p_hwfn->p_cxt_mngr; - u32 start_cid = 0; - u32 type; + u32 start_cid = 0, vf_start_cid = 0; + u32 type, vf; for (type = 0; type < MAX_CONN_TYPES; type++) { - u32 cid_cnt = p_hwfn->p_cxt_mngr->conn_cfg[type].cid_count; - u32 size; + struct ecore_conn_type_cfg *p_cfg = &p_mngr->conn_cfg[type]; + struct ecore_cid_acquired_map *p_map; - if (cid_cnt == 0) - continue; - - size = MAP_WORD_SIZE * DIV_ROUND_UP(cid_cnt, BITS_PER_MAP_WORD); - p_mngr->acquired[type].cid_map = OSAL_ZALLOC(p_hwfn->p_dev, - GFP_KERNEL, size); - if (!p_mngr->acquired[type].cid_map) + /* Handle PF maps */ + p_map = &p_mngr->acquired[type]; + if (ecore_cid_map_alloc_single(p_hwfn, type, start_cid, + p_cfg->cid_count, p_map)) goto cid_map_fail; - p_mngr->acquired[type].max_count = cid_cnt; - p_mngr->acquired[type].start_cid = start_cid; + /* Handle VF maps */ + for (vf = 0; vf < COMMON_MAX_NUM_VFS; vf++) { + p_map = &p_mngr->acquired_vf[type][vf]; + if (ecore_cid_map_alloc_single(p_hwfn, type, + vf_start_cid, + p_cfg->cids_per_vf, + p_map)) + goto cid_map_fail; + } - p_hwfn->p_cxt_mngr->conn_cfg[type].cid_start = start_cid; - - DP_VERBOSE(p_hwfn, ECORE_MSG_CXT, - "Type %08x start: %08x count %08x\n", - type, p_mngr->acquired[type].start_cid, - p_mngr->acquired[type].max_count); - start_cid += cid_cnt; + start_cid += p_cfg->cid_count; + vf_start_cid += p_cfg->cids_per_vf; } return ECORE_SUCCESS; @@ -991,6 +1120,7 @@ cid_map_fail: enum _ecore_status_t ecore_cxt_mngr_alloc(struct ecore_hwfn *p_hwfn) { + struct ecore_ilt_client_cfg *clients; struct ecore_cxt_mngr *p_mngr; u32 i; @@ -1002,35 +1132,50 @@ enum _ecore_status_t ecore_cxt_mngr_alloc(struct ecore_hwfn *p_hwfn) } /* Initialize ILT client registers */ - p_mngr->clients[ILT_CLI_CDUC].first.reg = ILT_CFG_REG(CDUC, FIRST_ILT); - p_mngr->clients[ILT_CLI_CDUC].last.reg = ILT_CFG_REG(CDUC, LAST_ILT); - p_mngr->clients[ILT_CLI_CDUC].p_size.reg = ILT_CFG_REG(CDUC, P_SIZE); + clients = p_mngr->clients; + clients[ILT_CLI_CDUC].first.reg = ILT_CFG_REG(CDUC, FIRST_ILT); + clients[ILT_CLI_CDUC].last.reg = ILT_CFG_REG(CDUC, LAST_ILT); + clients[ILT_CLI_CDUC].p_size.reg = ILT_CFG_REG(CDUC, P_SIZE); - p_mngr->clients[ILT_CLI_QM].first.reg = ILT_CFG_REG(QM, FIRST_ILT); - p_mngr->clients[ILT_CLI_QM].last.reg = ILT_CFG_REG(QM, LAST_ILT); - p_mngr->clients[ILT_CLI_QM].p_size.reg = ILT_CFG_REG(QM, P_SIZE); + clients[ILT_CLI_QM].first.reg = ILT_CFG_REG(QM, FIRST_ILT); + clients[ILT_CLI_QM].last.reg = ILT_CFG_REG(QM, LAST_ILT); + clients[ILT_CLI_QM].p_size.reg = ILT_CFG_REG(QM, P_SIZE); - p_mngr->clients[ILT_CLI_TM].first.reg = ILT_CFG_REG(TM, FIRST_ILT); - p_mngr->clients[ILT_CLI_TM].last.reg = ILT_CFG_REG(TM, LAST_ILT); - p_mngr->clients[ILT_CLI_TM].p_size.reg = ILT_CFG_REG(TM, P_SIZE); + clients[ILT_CLI_TM].first.reg = ILT_CFG_REG(TM, FIRST_ILT); + clients[ILT_CLI_TM].last.reg = ILT_CFG_REG(TM, LAST_ILT); + clients[ILT_CLI_TM].p_size.reg = ILT_CFG_REG(TM, P_SIZE); - p_mngr->clients[ILT_CLI_SRC].first.reg = ILT_CFG_REG(SRC, FIRST_ILT); - p_mngr->clients[ILT_CLI_SRC].last.reg = ILT_CFG_REG(SRC, LAST_ILT); - p_mngr->clients[ILT_CLI_SRC].p_size.reg = ILT_CFG_REG(SRC, P_SIZE); + clients[ILT_CLI_SRC].first.reg = ILT_CFG_REG(SRC, FIRST_ILT); + clients[ILT_CLI_SRC].last.reg = ILT_CFG_REG(SRC, LAST_ILT); + clients[ILT_CLI_SRC].p_size.reg = ILT_CFG_REG(SRC, P_SIZE); - p_mngr->clients[ILT_CLI_CDUT].first.reg = ILT_CFG_REG(CDUT, FIRST_ILT); - p_mngr->clients[ILT_CLI_CDUT].last.reg = ILT_CFG_REG(CDUT, LAST_ILT); - p_mngr->clients[ILT_CLI_CDUT].p_size.reg = ILT_CFG_REG(CDUT, P_SIZE); + clients[ILT_CLI_CDUT].first.reg = ILT_CFG_REG(CDUT, FIRST_ILT); + clients[ILT_CLI_CDUT].last.reg = ILT_CFG_REG(CDUT, LAST_ILT); + clients[ILT_CLI_CDUT].p_size.reg = ILT_CFG_REG(CDUT, P_SIZE); - /* default ILT page size for all clients is 32K */ + clients[ILT_CLI_TSDM].first.reg = ILT_CFG_REG(TSDM, FIRST_ILT); + clients[ILT_CLI_TSDM].last.reg = ILT_CFG_REG(TSDM, LAST_ILT); + clients[ILT_CLI_TSDM].p_size.reg = ILT_CFG_REG(TSDM, P_SIZE); + + /* default ILT page size for all clients is 64K */ for (i = 0; i < ILT_CLI_MAX; i++) p_mngr->clients[i].p_size.val = ILT_DEFAULT_HW_P_SIZE; - /* Initialize task sizes */ - p_mngr->task_type_size[0] = 512; /* @DPDK */ - p_mngr->task_type_size[1] = 128; /* @DPDK */ + /* due to removal of ISCSI/FCoE files union type0_task_context + * task_type_size will be 0. So hardcoded for now. + */ + p_mngr->task_type_size[0] = 512; /* @DPDK */ + p_mngr->task_type_size[1] = 128; /* @DPDK */ + + if (p_hwfn->p_dev->p_iov_info) + p_mngr->vf_count = p_hwfn->p_dev->p_iov_info->total_vfs; + + /* Initialize the dynamic ILT allocation mutex */ +#ifdef CONFIG_ECORE_LOCK_ALLOC + OSAL_MUTEX_ALLOC(p_hwfn, &p_mngr->mutex); +#endif + OSAL_MUTEX_INIT(&p_mngr->mutex); - p_mngr->vf_count = p_hwfn->p_dev->sriov_info.total_vfs; /* Set the cxt mangr pointer priori to further allocations */ p_hwfn->p_cxt_mngr = p_mngr; @@ -1077,26 +1222,43 @@ void ecore_cxt_mngr_free(struct ecore_hwfn *p_hwfn) ecore_cid_map_free(p_hwfn); ecore_cxt_src_t2_free(p_hwfn); ecore_ilt_shadow_free(p_hwfn); +#ifdef CONFIG_ECORE_LOCK_ALLOC + OSAL_MUTEX_DEALLOC(&p_hwfn->p_cxt_mngr->mutex); +#endif OSAL_FREE(p_hwfn->p_dev, p_hwfn->p_cxt_mngr); - - p_hwfn->p_cxt_mngr = OSAL_NULL; } void ecore_cxt_mngr_setup(struct ecore_hwfn *p_hwfn) { struct ecore_cxt_mngr *p_mngr = p_hwfn->p_cxt_mngr; + struct ecore_cid_acquired_map *p_map; + struct ecore_conn_type_cfg *p_cfg; int type; + u32 len; /* Reset acquired cids */ for (type = 0; type < MAX_CONN_TYPES; type++) { - u32 cid_cnt = p_hwfn->p_cxt_mngr->conn_cfg[type].cid_count; - u32 i; + u32 vf; - if (cid_cnt == 0) + p_cfg = &p_mngr->conn_cfg[type]; + if (p_cfg->cid_count) { + p_map = &p_mngr->acquired[type]; + len = DIV_ROUND_UP(p_map->max_count, + BITS_PER_MAP_WORD) * + MAP_WORD_SIZE; + OSAL_MEM_ZERO(p_map->cid_map, len); + } + + if (!p_cfg->cids_per_vf) continue; - for (i = 0; i < DIV_ROUND_UP(cid_cnt, BITS_PER_MAP_WORD); i++) - p_mngr->acquired[type].cid_map[i] = 0; + for (vf = 0; vf < COMMON_MAX_NUM_VFS; vf++) { + p_map = &p_mngr->acquired_vf[type][vf]; + len = DIV_ROUND_UP(p_map->max_count, + BITS_PER_MAP_WORD) * + MAP_WORD_SIZE; + OSAL_MEM_ZERO(p_map->cid_map, len); + } } } @@ -1265,40 +1427,35 @@ static void ecore_cdu_init_pf(struct ecore_hwfn *p_hwfn) } } -void ecore_qm_init_pf(struct ecore_hwfn *p_hwfn) +void ecore_qm_init_pf(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt) { struct ecore_qm_info *qm_info = &p_hwfn->qm_info; + struct ecore_mcp_link_state *p_link; struct ecore_qm_iids iids; OSAL_MEM_ZERO(&iids, sizeof(iids)); ecore_cxt_qm_iids(p_hwfn, &iids); - ecore_qm_pf_rt_init(p_hwfn, p_hwfn->p_main_ptt, p_hwfn->port_id, + p_link = &ECORE_LEADING_HWFN(p_hwfn->p_dev)->mcp_info->link_output; + + ecore_qm_pf_rt_init(p_hwfn, p_ptt, p_hwfn->port_id, p_hwfn->rel_pf_id, qm_info->max_phys_tcs_per_port, - p_hwfn->first_on_engine, iids.cids, iids.vf_cids, iids.tids, qm_info->start_pq, qm_info->num_pqs - qm_info->num_vf_pqs, qm_info->num_vf_pqs, qm_info->start_vport, qm_info->num_vports, qm_info->pf_wfq, - qm_info->pf_rl, p_hwfn->qm_info.qm_pq_params, + qm_info->pf_rl, p_link->speed, + p_hwfn->qm_info.qm_pq_params, p_hwfn->qm_info.qm_vport_params); } /* CM PF */ -static enum _ecore_status_t ecore_cm_init_pf(struct ecore_hwfn *p_hwfn) +static void ecore_cm_init_pf(struct ecore_hwfn *p_hwfn) { - union ecore_qm_pq_params pq_params; - u16 pq; - - /* XCM pure-LB queue */ - OSAL_MEMSET(&pq_params, 0, sizeof(pq_params)); - pq_params.core.tc = LB_TC; - pq = ecore_get_qm_pq(p_hwfn, PROTOCOLID_CORE, &pq_params); - STORE_RT_REG(p_hwfn, XCM_REG_CON_PHY_Q3_RT_OFFSET, pq); - - return ECORE_SUCCESS; + STORE_RT_REG(p_hwfn, XCM_REG_CON_PHY_Q3_RT_OFFSET, + ecore_get_cm_pq_idx(p_hwfn, PQ_FLAGS_LB)); } /* DQ PF */ @@ -1362,10 +1519,7 @@ static void ecore_ilt_bounds_init(struct ecore_hwfn *p_hwfn) int i; ilt_clients = p_hwfn->p_cxt_mngr->clients; - for (i = 0; i < ILT_CLI_MAX; i++) - if (!ilt_clients[i].active) { - continue; - } else { + for_each_ilt_valid_client(i, ilt_clients) { STORE_RT_REG(p_hwfn, ilt_clients[i].first.reg, ilt_clients[i].first.val); @@ -1383,13 +1537,16 @@ static void ecore_ilt_vf_bounds_init(struct ecore_hwfn *p_hwfn) u32 blk_factor; /* For simplicty we set the 'block' to be an ILT page */ - STORE_RT_REG(p_hwfn, - PSWRQ2_REG_VF_BASE_RT_OFFSET, - p_hwfn->hw_info.first_vf_in_pf); - STORE_RT_REG(p_hwfn, - PSWRQ2_REG_VF_LAST_ILT_RT_OFFSET, - p_hwfn->hw_info.first_vf_in_pf + - p_hwfn->p_dev->sriov_info.total_vfs); + if (p_hwfn->p_dev->p_iov_info) { + struct ecore_hw_sriov_info *p_iov = p_hwfn->p_dev->p_iov_info; + + STORE_RT_REG(p_hwfn, + PSWRQ2_REG_VF_BASE_RT_OFFSET, + p_iov->first_vf_in_pf); + STORE_RT_REG(p_hwfn, + PSWRQ2_REG_VF_LAST_ILT_RT_OFFSET, + p_iov->first_vf_in_pf + p_iov->total_vfs); + } p_cli = &p_hwfn->p_cxt_mngr->clients[ILT_CLI_CDUC]; blk_factor = OSAL_LOG2(ILT_PAGE_IN_BYTES(p_cli->p_size.val) >> 10); @@ -1448,10 +1605,7 @@ static void ecore_ilt_init_pf(struct ecore_hwfn *p_hwfn) p_shdw = p_mngr->ilt_shadow; clients = p_hwfn->p_cxt_mngr->clients; - for (i = 0; i < ILT_CLI_MAX; i++) - if (!clients[i].active) { - continue; - } else { + for_each_ilt_valid_client(i, clients) { /* Client's 1st val and RT array are absolute, ILT shadows' * lines are relative. */ @@ -1474,9 +1628,10 @@ static void ecore_ilt_init_pf(struct ecore_hwfn *p_hwfn) DP_VERBOSE(p_hwfn, ECORE_MSG_ILT, "Setting RT[0x%08x] from" " ILT[0x%08x] [Client is %d] to" - " Physical addr: 0x%" PRIx64 "\n", + " Physical addr: 0x%lx\n", rt_offst, line, i, - (u64)(p_shdw[line].p_phys >> 12)); + (unsigned long)(p_shdw[line]. + p_phys >> 12)); } STORE_RT_REG_AGG(p_hwfn, rt_offst, ilt_hw_entry); @@ -1508,6 +1663,9 @@ static void ecore_src_init_pf(struct ecore_hwfn *p_hwfn) p_hwfn->p_cxt_mngr->first_free); STORE_RT_REG_AGG(p_hwfn, SRC_REG_LASTFREE_RT_OFFSET, p_hwfn->p_cxt_mngr->last_free); + DP_VERBOSE(p_hwfn, ECORE_MSG_ILT, + "Configured SEARCHER for 0x%08x connections\n", + conn_num); } /* Timers PF */ @@ -1545,11 +1703,11 @@ static void ecore_tm_init_pf(struct ecore_hwfn *p_hwfn) SET_FIELD(cfg_word, TM_CFG_NUM_IDS, tm_iids.per_vf_cids); SET_FIELD(cfg_word, TM_CFG_PRE_SCAN_OFFSET, 0); SET_FIELD(cfg_word, TM_CFG_PARENT_PF, p_hwfn->rel_pf_id); - SET_FIELD(cfg_word, TM_CFG_CID_PRE_SCAN_ROWS, 0); + SET_FIELD(cfg_word, TM_CFG_CID_PRE_SCAN_ROWS, 0); /* scan all */ rt_reg = TM_REG_CONFIG_CONN_MEM_RT_OFFSET + (sizeof(cfg_word) / sizeof(u32)) * - (p_hwfn->hw_info.first_vf_in_pf + i); + (p_hwfn->p_dev->p_iov_info->first_vf_in_pf + i); STORE_RT_REG_AGG(p_hwfn, rt_reg, cfg_word); } @@ -1557,7 +1715,7 @@ static void ecore_tm_init_pf(struct ecore_hwfn *p_hwfn) SET_FIELD(cfg_word, TM_CFG_NUM_IDS, tm_iids.pf_cids); SET_FIELD(cfg_word, TM_CFG_PRE_SCAN_OFFSET, 0); SET_FIELD(cfg_word, TM_CFG_PARENT_PF, 0); /* n/a for PF */ - SET_FIELD(cfg_word, TM_CFG_CID_PRE_SCAN_ROWS, 0); + SET_FIELD(cfg_word, TM_CFG_CID_PRE_SCAN_ROWS, 0); /* scan all */ rt_reg = TM_REG_CONFIG_CONN_MEM_RT_OFFSET + (sizeof(cfg_word) / sizeof(u32)) * @@ -1583,7 +1741,7 @@ static void ecore_tm_init_pf(struct ecore_hwfn *p_hwfn) rt_reg = TM_REG_CONFIG_TASK_MEM_RT_OFFSET + (sizeof(cfg_word) / sizeof(u32)) * - (p_hwfn->hw_info.first_vf_in_pf + i); + (p_hwfn->p_dev->p_iov_info->first_vf_in_pf + i); STORE_RT_REG_AGG(p_hwfn, rt_reg, cfg_word); } @@ -1613,115 +1771,186 @@ static void ecore_tm_init_pf(struct ecore_hwfn *p_hwfn) /* @@@TBD how to enable the scan for the VFs */ } -static void ecore_prs_init_common(struct ecore_hwfn *p_hwfn) +static void ecore_prs_init_pf(struct ecore_hwfn *p_hwfn) { + struct ecore_cxt_mngr *p_mngr = p_hwfn->p_cxt_mngr; + struct ecore_conn_type_cfg *p_fcoe; + struct ecore_tid_seg *p_tid; + + p_fcoe = &p_mngr->conn_cfg[PROTOCOLID_FCOE]; + + /* If FCoE is active set the MAX OX_ID (tid) in the Parser */ + if (!p_fcoe->cid_count) + return; + + p_tid = &p_fcoe->tid_seg[ECORE_CXT_FCOE_TID_SEG]; + STORE_RT_REG_AGG(p_hwfn, + PRS_REG_TASK_ID_MAX_INITIATOR_PF_RT_OFFSET, + p_tid->count); } void ecore_cxt_hw_init_common(struct ecore_hwfn *p_hwfn) { /* CDU configuration */ ecore_cdu_init_common(p_hwfn); - ecore_prs_init_common(p_hwfn); } -void ecore_cxt_hw_init_pf(struct ecore_hwfn *p_hwfn) +void ecore_cxt_hw_init_pf(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt) { - ecore_qm_init_pf(p_hwfn); + ecore_qm_init_pf(p_hwfn, p_ptt); ecore_cm_init_pf(p_hwfn); ecore_dq_init_pf(p_hwfn); ecore_cdu_init_pf(p_hwfn); ecore_ilt_init_pf(p_hwfn); ecore_src_init_pf(p_hwfn); ecore_tm_init_pf(p_hwfn); + ecore_prs_init_pf(p_hwfn); } -enum _ecore_status_t ecore_cxt_acquire_cid(struct ecore_hwfn *p_hwfn, - enum protocol_type type, u32 *p_cid) +enum _ecore_status_t _ecore_cxt_acquire_cid(struct ecore_hwfn *p_hwfn, + enum protocol_type type, + u32 *p_cid, u8 vfid) { struct ecore_cxt_mngr *p_mngr = p_hwfn->p_cxt_mngr; + struct ecore_cid_acquired_map *p_map; u32 rel_cid; - if (type >= MAX_CONN_TYPES || !p_mngr->acquired[type].cid_map) { + if (type >= MAX_CONN_TYPES) { DP_NOTICE(p_hwfn, true, "Invalid protocol type %d", type); return ECORE_INVAL; } - rel_cid = OSAL_FIND_FIRST_ZERO_BIT(p_mngr->acquired[type].cid_map, - p_mngr->acquired[type].max_count); + if (vfid >= COMMON_MAX_NUM_VFS && vfid != ECORE_CXT_PF_CID) { + DP_NOTICE(p_hwfn, true, "VF [%02x] is out of range\n", vfid); + return ECORE_INVAL; + } - if (rel_cid >= p_mngr->acquired[type].max_count) { - DP_NOTICE(p_hwfn, false, "no CID available for protocol %d", + /* Determine the right map to take this CID from */ + if (vfid == ECORE_CXT_PF_CID) + p_map = &p_mngr->acquired[type]; + else + p_map = &p_mngr->acquired_vf[type][vfid]; + + if (p_map->cid_map == OSAL_NULL) { + DP_NOTICE(p_hwfn, true, "Invalid protocol type %d", type); + return ECORE_INVAL; + } + + rel_cid = OSAL_FIND_FIRST_ZERO_BIT(p_map->cid_map, + p_map->max_count); + + if (rel_cid >= p_map->max_count) { + DP_NOTICE(p_hwfn, false, "no CID available for protocol %d\n", type); return ECORE_NORESOURCES; } - OSAL_SET_BIT(rel_cid, p_mngr->acquired[type].cid_map); + OSAL_SET_BIT(rel_cid, p_map->cid_map); - *p_cid = rel_cid + p_mngr->acquired[type].start_cid; + *p_cid = rel_cid + p_map->start_cid; + + DP_VERBOSE(p_hwfn, ECORE_MSG_CXT, + "Acquired cid 0x%08x [rel. %08x] vfid %02x type %d\n", + *p_cid, rel_cid, vfid, type); return ECORE_SUCCESS; } +enum _ecore_status_t ecore_cxt_acquire_cid(struct ecore_hwfn *p_hwfn, + enum protocol_type type, + u32 *p_cid) +{ + return _ecore_cxt_acquire_cid(p_hwfn, type, p_cid, ECORE_CXT_PF_CID); +} + static bool ecore_cxt_test_cid_acquired(struct ecore_hwfn *p_hwfn, - u32 cid, enum protocol_type *p_type) + u32 cid, u8 vfid, + enum protocol_type *p_type, + struct ecore_cid_acquired_map **pp_map) { struct ecore_cxt_mngr *p_mngr = p_hwfn->p_cxt_mngr; - struct ecore_cid_acquired_map *p_map; - enum protocol_type p; u32 rel_cid; /* Iterate over protocols and find matching cid range */ - for (p = 0; p < MAX_CONN_TYPES; p++) { - p_map = &p_mngr->acquired[p]; + for (*p_type = 0; *p_type < MAX_CONN_TYPES; (*p_type)++) { + if (vfid == ECORE_CXT_PF_CID) + *pp_map = &p_mngr->acquired[*p_type]; + else + *pp_map = &p_mngr->acquired_vf[*p_type][vfid]; - if (!p_map->cid_map) + if (!((*pp_map)->cid_map)) continue; - if (cid >= p_map->start_cid && - cid < p_map->start_cid + p_map->max_count) { + if (cid >= (*pp_map)->start_cid && + cid < (*pp_map)->start_cid + (*pp_map)->max_count) { break; } } - *p_type = p; + if (*p_type == MAX_CONN_TYPES) { + DP_NOTICE(p_hwfn, true, "Invalid CID %d vfid %02x", cid, vfid); + goto fail; + } - if (p == MAX_CONN_TYPES) { - DP_NOTICE(p_hwfn, true, "Invalid CID %d", cid); - return false; - } - rel_cid = cid - p_map->start_cid; - if (!OSAL_TEST_BIT(rel_cid, p_map->cid_map)) { - DP_NOTICE(p_hwfn, true, "CID %d not acquired", cid); - return false; + rel_cid = cid - (*pp_map)->start_cid; + if (!OSAL_TEST_BIT(rel_cid, (*pp_map)->cid_map)) { + DP_NOTICE(p_hwfn, true, + "CID %d [vifd %02x] not acquired", cid, vfid); + goto fail; } + return true; +fail: + *p_type = MAX_CONN_TYPES; + *pp_map = OSAL_NULL; + return false; } -void ecore_cxt_release_cid(struct ecore_hwfn *p_hwfn, u32 cid) +void _ecore_cxt_release_cid(struct ecore_hwfn *p_hwfn, u32 cid, u8 vfid) { - struct ecore_cxt_mngr *p_mngr = p_hwfn->p_cxt_mngr; + struct ecore_cid_acquired_map *p_map = OSAL_NULL; enum protocol_type type; bool b_acquired; u32 rel_cid; + if (vfid != ECORE_CXT_PF_CID && vfid > COMMON_MAX_NUM_VFS) { + DP_NOTICE(p_hwfn, true, + "Trying to return incorrect CID belonging to VF %02x\n", + vfid); + return; + } + /* Test acquired and find matching per-protocol map */ - b_acquired = ecore_cxt_test_cid_acquired(p_hwfn, cid, &type); + b_acquired = ecore_cxt_test_cid_acquired(p_hwfn, cid, vfid, + &type, &p_map); if (!b_acquired) return; - rel_cid = cid - p_mngr->acquired[type].start_cid; - OSAL_CLEAR_BIT(rel_cid, p_mngr->acquired[type].cid_map); + rel_cid = cid - p_map->start_cid; + OSAL_CLEAR_BIT(rel_cid, p_map->cid_map); + + DP_VERBOSE(p_hwfn, ECORE_MSG_CXT, + "Released CID 0x%08x [rel. %08x] vfid %02x type %d\n", + cid, rel_cid, vfid, type); +} + +void ecore_cxt_release_cid(struct ecore_hwfn *p_hwfn, u32 cid) +{ + _ecore_cxt_release_cid(p_hwfn, cid, ECORE_CXT_PF_CID); } enum _ecore_status_t ecore_cxt_get_cid_info(struct ecore_hwfn *p_hwfn, struct ecore_cxt_info *p_info) { struct ecore_cxt_mngr *p_mngr = p_hwfn->p_cxt_mngr; + struct ecore_cid_acquired_map *p_map = OSAL_NULL; u32 conn_cxt_size, hw_p_size, cxts_per_p, line; enum protocol_type type; bool b_acquired; /* Test acquired and find matching per-protocol map */ - b_acquired = ecore_cxt_test_cid_acquired(p_hwfn, p_info->iid, &type); + b_acquired = ecore_cxt_test_cid_acquired(p_hwfn, p_info->iid, + ECORE_CXT_PF_CID, + &type, &p_map); if (!b_acquired) return ECORE_INVAL; @@ -1760,14 +1989,24 @@ enum _ecore_status_t ecore_cxt_set_pf_params(struct ecore_hwfn *p_hwfn) switch (p_hwfn->hw_info.personality) { case ECORE_PCI_ETH: { - struct ecore_eth_pf_params *p_params = + u32 count = 0; + + struct ecore_eth_pf_params *p_params = &p_hwfn->pf_params.eth_pf_params; - ecore_cxt_set_proto_cid_count(p_hwfn, - PROTOCOLID_ETH, - p_params->num_cons, 1); /* FIXME VF count... */ + if (!p_params->num_vf_cons) + p_params->num_vf_cons = ETH_PF_PARAMS_VF_CONS_DEFAULT; + ecore_cxt_set_proto_cid_count(p_hwfn, PROTOCOLID_ETH, + p_params->num_cons, + p_params->num_vf_cons); - break; + count = p_params->num_arfs_filters; + + if (!OSAL_TEST_BIT(ECORE_MF_DISABLE_ARFS, + &p_hwfn->p_dev->mf_bits)) + p_hwfn->p_cxt_mngr->arfs_count = count; + + break; } default: return ECORE_INVAL; @@ -1776,45 +2015,121 @@ enum _ecore_status_t ecore_cxt_set_pf_params(struct ecore_hwfn *p_hwfn) return ECORE_SUCCESS; } -enum _ecore_status_t ecore_cxt_get_tid_mem_info(struct ecore_hwfn *p_hwfn, - struct ecore_tid_mem *p_info) +/* This function is very RoCE oriented, if another protocol in the future + * will want this feature we'll need to modify the function to be more generic + */ +enum _ecore_status_t +ecore_cxt_dynamic_ilt_alloc(struct ecore_hwfn *p_hwfn, + enum ecore_cxt_elem_type elem_type, + u32 iid) { - struct ecore_cxt_mngr *p_mngr = p_hwfn->p_cxt_mngr; - u32 proto, seg, total_lines, i, shadow_line; + u32 reg_offset, shadow_line, elem_size, hw_p_size, elems_per_p, line; struct ecore_ilt_client_cfg *p_cli; - struct ecore_ilt_cli_blk *p_fl_seg; - struct ecore_tid_seg *p_seg_info; + struct ecore_ilt_cli_blk *p_blk; + struct ecore_ptt *p_ptt; + dma_addr_t p_phys; + u64 ilt_hw_entry; + void *p_virt; + enum _ecore_status_t rc = ECORE_SUCCESS; - /* Verify the personality */ - switch (p_hwfn->hw_info.personality) { + switch (elem_type) { + case ECORE_ELEM_CXT: + p_cli = &p_hwfn->p_cxt_mngr->clients[ILT_CLI_CDUC]; + elem_size = CONN_CXT_SIZE(p_hwfn); + p_blk = &p_cli->pf_blks[CDUC_BLK]; + break; + case ECORE_ELEM_SRQ: + p_cli = &p_hwfn->p_cxt_mngr->clients[ILT_CLI_TSDM]; + elem_size = SRQ_CXT_SIZE; + p_blk = &p_cli->pf_blks[SRQ_BLK]; + break; + case ECORE_ELEM_TASK: + p_cli = &p_hwfn->p_cxt_mngr->clients[ILT_CLI_CDUT]; + elem_size = TYPE1_TASK_CXT_SIZE(p_hwfn); + p_blk = &p_cli->pf_blks[CDUT_SEG_BLK(ECORE_CXT_ROCE_TID_SEG)]; + break; default: + DP_NOTICE(p_hwfn, false, + "ECORE_INVALID elem type = %d", elem_type); return ECORE_INVAL; } - p_cli = &p_mngr->clients[ILT_CLI_CDUT]; - if (!p_cli->active) - return ECORE_INVAL; + /* Calculate line in ilt */ + hw_p_size = p_cli->p_size.val; + elems_per_p = ILT_PAGE_IN_BYTES(hw_p_size) / elem_size; + line = p_blk->start_line + (iid / elems_per_p); + shadow_line = line - p_hwfn->p_cxt_mngr->pf_start_line; - p_seg_info = &p_mngr->conn_cfg[proto].tid_seg[seg]; - if (!p_seg_info->has_fl_mem) - return ECORE_INVAL; + /* If line is already allocated, do nothing, otherwise allocate it and + * write it to the PSWRQ2 registers. + * This section can be run in parallel from different contexts and thus + * a mutex protection is needed. + */ - p_fl_seg = &p_cli->pf_blks[CDUT_FL_SEG_BLK(seg, PF)]; - total_lines = DIV_ROUND_UP(p_fl_seg->total_size, - p_fl_seg->real_size_in_page); + OSAL_MUTEX_ACQUIRE(&p_hwfn->p_cxt_mngr->mutex); - for (i = 0; i < total_lines; i++) { - shadow_line = i + p_fl_seg->start_line - - p_hwfn->p_cxt_mngr->pf_start_line; - p_info->blocks[i] = p_mngr->ilt_shadow[shadow_line].p_virt; + if (p_hwfn->p_cxt_mngr->ilt_shadow[shadow_line].p_virt) + goto out0; + + p_ptt = ecore_ptt_acquire(p_hwfn); + if (!p_ptt) { + DP_NOTICE(p_hwfn, false, + "ECORE_TIME_OUT on ptt acquire - dynamic allocation"); + rc = ECORE_TIMEOUT; + goto out0; } - p_info->waste = ILT_PAGE_IN_BYTES(p_cli->p_size.val) - - p_fl_seg->real_size_in_page; - p_info->tid_size = p_mngr->task_type_size[p_seg_info->type]; - p_info->num_tids_per_block = p_fl_seg->real_size_in_page / - p_info->tid_size; - return ECORE_SUCCESS; + p_virt = OSAL_DMA_ALLOC_COHERENT(p_hwfn->p_dev, + &p_phys, + p_blk->real_size_in_page); + if (!p_virt) { + rc = ECORE_NOMEM; + goto out1; + } + OSAL_MEM_ZERO(p_virt, p_blk->real_size_in_page); + + p_hwfn->p_cxt_mngr->ilt_shadow[shadow_line].p_virt = p_virt; + p_hwfn->p_cxt_mngr->ilt_shadow[shadow_line].p_phys = p_phys; + p_hwfn->p_cxt_mngr->ilt_shadow[shadow_line].size = + p_blk->real_size_in_page; + + /* compute absolute offset */ + reg_offset = PSWRQ2_REG_ILT_MEMORY + + (line * ILT_REG_SIZE_IN_BYTES * ILT_ENTRY_IN_REGS); + + ilt_hw_entry = 0; + SET_FIELD(ilt_hw_entry, ILT_ENTRY_VALID, 1ULL); + SET_FIELD(ilt_hw_entry, + ILT_ENTRY_PHY_ADDR, + (p_hwfn->p_cxt_mngr->ilt_shadow[shadow_line].p_phys >> 12)); + +/* Write via DMAE since the PSWRQ2_REG_ILT_MEMORY line is a wide-bus */ + + ecore_dmae_host2grc(p_hwfn, p_ptt, (u64)(osal_uintptr_t)&ilt_hw_entry, + reg_offset, sizeof(ilt_hw_entry) / sizeof(u32), + 0 /* no flags */); + + if (elem_type == ECORE_ELEM_CXT) { + u32 last_cid_allocated = (1 + (iid / elems_per_p)) * + elems_per_p; + + /* Update the relevant register in the parser */ + ecore_wr(p_hwfn, p_ptt, PRS_REG_ROCE_DEST_QP_MAX_PF, + last_cid_allocated - 1); + + if (!p_hwfn->b_rdma_enabled_in_prs) { + /* Enable RoCE search */ + ecore_wr(p_hwfn, p_ptt, p_hwfn->rdma_prs_search_reg, 1); + p_hwfn->b_rdma_enabled_in_prs = true; + } + } + +out1: + ecore_ptt_release(p_hwfn, p_ptt); +out0: + OSAL_MUTEX_RELEASE(&p_hwfn->p_cxt_mngr->mutex); + + return rc; } /* This function is very RoCE oriented, if another protocol in the future @@ -1825,8 +2140,8 @@ ecore_cxt_free_ilt_range(struct ecore_hwfn *p_hwfn, enum ecore_cxt_elem_type elem_type, u32 start_iid, u32 count) { - u32 reg_offset, elem_size, hw_p_size, elems_per_p; u32 start_line, end_line, shadow_start_line, shadow_end_line; + u32 reg_offset, elem_size, hw_p_size, elems_per_p; struct ecore_ilt_client_cfg *p_cli; struct ecore_ilt_cli_blk *p_blk; u32 end_iid = start_iid + count; @@ -1834,10 +2149,26 @@ ecore_cxt_free_ilt_range(struct ecore_hwfn *p_hwfn, u64 ilt_hw_entry = 0; u32 i; - if (elem_type == ECORE_ELEM_CXT) { + switch (elem_type) { + case ECORE_ELEM_CXT: p_cli = &p_hwfn->p_cxt_mngr->clients[ILT_CLI_CDUC]; elem_size = CONN_CXT_SIZE(p_hwfn); p_blk = &p_cli->pf_blks[CDUC_BLK]; + break; + case ECORE_ELEM_SRQ: + p_cli = &p_hwfn->p_cxt_mngr->clients[ILT_CLI_TSDM]; + elem_size = SRQ_CXT_SIZE; + p_blk = &p_cli->pf_blks[SRQ_BLK]; + break; + case ECORE_ELEM_TASK: + p_cli = &p_hwfn->p_cxt_mngr->clients[ILT_CLI_CDUT]; + elem_size = TYPE1_TASK_CXT_SIZE(p_hwfn); + p_blk = &p_cli->pf_blks[CDUT_SEG_BLK(ECORE_CXT_ROCE_TID_SEG)]; + break; + default: + DP_NOTICE(p_hwfn, false, + "ECORE_INVALID elem type = %d", elem_type); + return ECORE_INVAL; } /* Calculate line in ilt */ @@ -1876,86 +2207,17 @@ ecore_cxt_free_ilt_range(struct ecore_hwfn *p_hwfn, ((start_line++) * ILT_REG_SIZE_IN_BYTES * ILT_ENTRY_IN_REGS); - ecore_wr(p_hwfn, p_ptt, reg_offset, U64_LO(ilt_hw_entry)); - ecore_wr(p_hwfn, p_ptt, reg_offset + ILT_REG_SIZE_IN_BYTES, - U64_HI(ilt_hw_entry)); + /* Write via DMAE since the PSWRQ2_REG_ILT_MEMORY line is a + * wide-bus. + */ + ecore_dmae_host2grc(p_hwfn, p_ptt, + (u64)(osal_uintptr_t)&ilt_hw_entry, + reg_offset, + sizeof(ilt_hw_entry) / sizeof(u32), + 0 /* no flags */); } ecore_ptt_release(p_hwfn, p_ptt); return ECORE_SUCCESS; } - -enum _ecore_status_t ecore_cxt_free_proto_ilt(struct ecore_hwfn *p_hwfn, - enum protocol_type proto) -{ - enum _ecore_status_t rc; - u32 cid; - - /* Free Connection CXT */ - rc = ecore_cxt_free_ilt_range(p_hwfn, ECORE_ELEM_CXT, - ecore_cxt_get_proto_cid_start(p_hwfn, - proto), - ecore_cxt_get_proto_cid_count(p_hwfn, - proto, - &cid)); - - if (rc) - return rc; - - /* Free Task CXT */ - rc = ecore_cxt_free_ilt_range(p_hwfn, ECORE_ELEM_TASK, 0, - ecore_cxt_get_proto_tid_count(p_hwfn, - proto)); - - return rc; -} - -enum _ecore_status_t ecore_cxt_get_task_ctx(struct ecore_hwfn *p_hwfn, - u32 tid, - u8 ctx_type, void **pp_task_ctx) -{ - struct ecore_cxt_mngr *p_mngr = p_hwfn->p_cxt_mngr; - struct ecore_ilt_client_cfg *p_cli; - struct ecore_ilt_cli_blk *p_seg; - struct ecore_tid_seg *p_seg_info; - u32 proto, seg; - u32 total_lines; - u32 tid_size, ilt_idx; - u32 num_tids_per_block; - - /* Verify the personality */ - switch (p_hwfn->hw_info.personality) { - default: - return ECORE_INVAL; - } - - p_cli = &p_mngr->clients[ILT_CLI_CDUT]; - if (!p_cli->active) - return ECORE_INVAL; - - p_seg_info = &p_mngr->conn_cfg[proto].tid_seg[seg]; - - if (ctx_type == ECORE_CTX_WORKING_MEM) { - p_seg = &p_cli->pf_blks[CDUT_SEG_BLK(seg)]; - } else if (ctx_type == ECORE_CTX_FL_MEM) { - if (!p_seg_info->has_fl_mem) - return ECORE_INVAL; - p_seg = &p_cli->pf_blks[CDUT_FL_SEG_BLK(seg, PF)]; - } else { - return ECORE_INVAL; - } - total_lines = DIV_ROUND_UP(p_seg->total_size, p_seg->real_size_in_page); - tid_size = p_mngr->task_type_size[p_seg_info->type]; - num_tids_per_block = p_seg->real_size_in_page / tid_size; - - if (total_lines < tid / num_tids_per_block) - return ECORE_INVAL; - - ilt_idx = tid / num_tids_per_block + p_seg->start_line - - p_mngr->pf_start_line; - *pp_task_ctx = (u8 *)p_mngr->ilt_shadow[ilt_idx].p_virt + - (tid % num_tids_per_block) * tid_size; - - return ECORE_SUCCESS; -} diff --git a/dpdk/drivers/net/qede/base/ecore_cxt.h b/dpdk/drivers/net/qede/base/ecore_cxt.h index 1ac95f98..54761e4e 100644 --- a/dpdk/drivers/net/qede/base/ecore_cxt.h +++ b/dpdk/drivers/net/qede/base/ecore_cxt.h @@ -13,24 +13,27 @@ #include "ecore_proto_if.h" #include "ecore_cxt_api.h" +/* Tasks segments definitions */ +#define ECORE_CXT_ISCSI_TID_SEG PROTOCOLID_ISCSI /* 0 */ +#define ECORE_CXT_FCOE_TID_SEG PROTOCOLID_FCOE /* 1 */ +#define ECORE_CXT_ROCE_TID_SEG PROTOCOLID_ROCE /* 2 */ + enum ecore_cxt_elem_type { ECORE_ELEM_CXT, + ECORE_ELEM_SRQ, ECORE_ELEM_TASK }; u32 ecore_cxt_get_proto_cid_count(struct ecore_hwfn *p_hwfn, - enum protocol_type type, u32 *vf_cid); + enum protocol_type type, + u32 *vf_cid); + +u32 ecore_cxt_get_proto_tid_count(struct ecore_hwfn *p_hwfn, + enum protocol_type type); u32 ecore_cxt_get_proto_cid_start(struct ecore_hwfn *p_hwfn, enum protocol_type type); - -/** - * @brief ecore_cxt_qm_iids - fills the cid/tid counts for the QM configuration - * - * @param p_hwfn - * @param iids [out], a structure holding all the counters - */ -void ecore_cxt_qm_iids(struct ecore_hwfn *p_hwfn, struct ecore_qm_iids *iids); +u32 ecore_cxt_get_srq_count(struct ecore_hwfn *p_hwfn); /** * @brief ecore_cxt_set_pf_params - Set the PF params for cxt init @@ -41,18 +44,6 @@ void ecore_cxt_qm_iids(struct ecore_hwfn *p_hwfn, struct ecore_qm_iids *iids); */ enum _ecore_status_t ecore_cxt_set_pf_params(struct ecore_hwfn *p_hwfn); -/** - * @brief ecore_cxt_set_proto_cid_count - Set the max cids per protocol for cxt - * init - * - * @param p_hwfn - * @param type - * @param cid_cnt - number of pf cids - * @param vf_cid_cnt - number of vf cids - */ -void ecore_cxt_set_proto_cid_count(struct ecore_hwfn *p_hwfn, - enum protocol_type type, - u32 cid_cnt, u32 vf_cid_cnt); /** * @brief ecore_cxt_cfg_ilt_compute - compute ILT init parameters * @@ -107,15 +98,17 @@ void ecore_cxt_hw_init_common(struct ecore_hwfn *p_hwfn); * @brief ecore_cxt_hw_init_pf - Initailze ILT and DQ, PF phase, per path. * * @param p_hwfn + * @param p_ptt */ -void ecore_cxt_hw_init_pf(struct ecore_hwfn *p_hwfn); +void ecore_cxt_hw_init_pf(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt); /** * @brief ecore_qm_init_pf - Initailze the QM PF phase, per path * * @param p_hwfn + * @param p_ptt */ -void ecore_qm_init_pf(struct ecore_hwfn *p_hwfn); +void ecore_qm_init_pf(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt); /** * @brief Reconfigures QM pf on the fly @@ -128,14 +121,70 @@ void ecore_qm_init_pf(struct ecore_hwfn *p_hwfn); enum _ecore_status_t ecore_qm_reconf(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt); +#define ECORE_CXT_PF_CID (0xff) + /** -* @brief ecore_cxt_release - Release a cid -* -* @param p_hwfn -* @param cid -*/ + * @brief ecore_cxt_release - Release a cid + * + * @param p_hwfn + * @param cid + */ void ecore_cxt_release_cid(struct ecore_hwfn *p_hwfn, u32 cid); +/** + * @brief ecore_cxt_release - Release a cid belonging to a vf-queue + * + * @param p_hwfn + * @param cid + * @param vfid - engine relative index. ECORE_CXT_PF_CID if belongs to PF + */ +void _ecore_cxt_release_cid(struct ecore_hwfn *p_hwfn, + u32 cid, u8 vfid); + +/** + * @brief ecore_cxt_acquire - Acquire a new cid of a specific protocol type + * + * @param p_hwfn + * @param type + * @param p_cid + * + * @return enum _ecore_status_t + */ +enum _ecore_status_t ecore_cxt_acquire_cid(struct ecore_hwfn *p_hwfn, + enum protocol_type type, + u32 *p_cid); + +/** + * @brief _ecore_cxt_acquire - Acquire a new cid of a specific protocol type + * for a vf-queue + * + * @param p_hwfn + * @param type + * @param p_cid + * @param vfid - engine relative index. ECORE_CXT_PF_CID if belongs to PF + * + * @return enum _ecore_status_t + */ +enum _ecore_status_t _ecore_cxt_acquire_cid(struct ecore_hwfn *p_hwfn, + enum protocol_type type, + u32 *p_cid, u8 vfid); + +/** + * @brief ecore_cxt_get_tid_mem_info - function checks if the + * page containing the iid in the ilt is already + * allocated, if it is not it allocates the page. + * + * @param p_hwfn + * @param elem_type + * @param iid + * + * @return enum _ecore_status_t + */ +enum _ecore_status_t +ecore_cxt_dynamic_ilt_alloc(struct ecore_hwfn *p_hwfn, + enum ecore_cxt_elem_type elem_type, + u32 iid); + /** * @brief ecore_cxt_free_proto_ilt - function frees ilt pages * associated with the protocol passed. @@ -150,8 +199,5 @@ enum _ecore_status_t ecore_cxt_free_proto_ilt(struct ecore_hwfn *p_hwfn, #define ECORE_CTX_WORKING_MEM 0 #define ECORE_CTX_FL_MEM 1 -enum _ecore_status_t ecore_cxt_get_task_ctx(struct ecore_hwfn *p_hwfn, - u32 tid, - u8 ctx_type, void **task_ctx); #endif /* _ECORE_CID_ */ diff --git a/dpdk/drivers/net/qede/base/ecore_cxt_api.h b/dpdk/drivers/net/qede/base/ecore_cxt_api.h index d98dddb2..6d87620d 100644 --- a/dpdk/drivers/net/qede/base/ecore_cxt_api.h +++ b/dpdk/drivers/net/qede/base/ecore_cxt_api.h @@ -12,9 +12,9 @@ struct ecore_hwfn; struct ecore_cxt_info { - void *p_cxt; - u32 iid; - enum protocol_type type; + void *p_cxt; + u32 iid; + enum protocol_type type; }; #define MAX_TID_BLOCKS 512 @@ -22,37 +22,9 @@ struct ecore_tid_mem { u32 tid_size; u32 num_tids_per_block; u32 waste; - u8 *blocks[MAX_TID_BLOCKS]; /* 4K */ + u8 *blocks[MAX_TID_BLOCKS]; /* 4K */ }; -static OSAL_INLINE void *get_task_mem(struct ecore_tid_mem *info, u32 tid) -{ - /* note: waste is superfluous */ - return (void *)(info->blocks[tid / info->num_tids_per_block] + - (tid % info->num_tids_per_block) * info->tid_size); - - /* more elaborate alternative with no modulo - * u32 mask = info->tid_size * info->num_tids_per_block + - * info->waste - 1; - * u32 index = tid / info->num_tids_per_block; - * u32 offset = tid * info->tid_size + index * info->waste; - * return (void *)(blocks[index] + (offset & mask)); - */ -} - -/** -* @brief ecore_cxt_acquire - Acquire a new cid of a specific protocol type -* -* @param p_hwfn -* @param type -* @param p_cid -* -* @return enum _ecore_status_t -*/ -enum _ecore_status_t ecore_cxt_acquire_cid(struct ecore_hwfn *p_hwfn, - enum protocol_type type, - u32 *p_cid); - /** * @brief ecoreo_cid_get_cxt_info - Returns the context info for a specific cid * @@ -65,15 +37,4 @@ enum _ecore_status_t ecore_cxt_acquire_cid(struct ecore_hwfn *p_hwfn, enum _ecore_status_t ecore_cxt_get_cid_info(struct ecore_hwfn *p_hwfn, struct ecore_cxt_info *p_info); -/** -* @brief ecore_cxt_get_tid_mem_info -* -* @param p_hwfn -* @param p_info -* -* @return enum _ecore_status_t -*/ -enum _ecore_status_t ecore_cxt_get_tid_mem_info(struct ecore_hwfn *p_hwfn, - struct ecore_tid_mem *p_info); - #endif diff --git a/dpdk/drivers/net/qede/base/ecore_dcbx.c b/dpdk/drivers/net/qede/base/ecore_dcbx.c index 6a966cb9..21ddda92 100644 --- a/dpdk/drivers/net/qede/base/ecore_dcbx.c +++ b/dpdk/drivers/net/qede/base/ecore_dcbx.c @@ -13,9 +13,9 @@ #include "ecore_cxt.h" #include "ecore_gtt_reg_addr.h" #include "ecore_iro.h" +#include "ecore_iov_api.h" #define ECORE_DCBX_MAX_MIB_READ_TRY (100) -#define ECORE_MAX_PFC_PRIORITIES 8 #define ECORE_ETH_TYPE_DEFAULT (0) #define ECORE_DCBX_INVALID_PRIORITY 0xFF @@ -24,144 +24,156 @@ * the traffic class corresponding to the priority. */ #define ECORE_DCBX_PRIO2TC(prio_tc_tbl, prio) \ - ((u32)(pri_tc_tbl >> ((7 - prio) * 4)) & 0x7) + ((u32)(prio_tc_tbl >> ((7 - prio) * 4)) & 0x7) static bool ecore_dcbx_app_ethtype(u32 app_info_bitmap) { - return (ECORE_MFW_GET_FIELD(app_info_bitmap, DCBX_APP_SF) == - DCBX_APP_SF_ETHTYPE) ? true : false; + return !!(GET_MFW_FIELD(app_info_bitmap, DCBX_APP_SF) == + DCBX_APP_SF_ETHTYPE); +} + +static bool ecore_dcbx_ieee_app_ethtype(u32 app_info_bitmap) +{ + u8 mfw_val = GET_MFW_FIELD(app_info_bitmap, DCBX_APP_SF_IEEE); + + /* Old MFW */ + if (mfw_val == DCBX_APP_SF_IEEE_RESERVED) + return ecore_dcbx_app_ethtype(app_info_bitmap); + + return !!(mfw_val == DCBX_APP_SF_IEEE_ETHTYPE); } static bool ecore_dcbx_app_port(u32 app_info_bitmap) { - return (ECORE_MFW_GET_FIELD(app_info_bitmap, DCBX_APP_SF) == - DCBX_APP_SF_PORT) ? true : false; + return !!(GET_MFW_FIELD(app_info_bitmap, DCBX_APP_SF) == + DCBX_APP_SF_PORT); } -static bool ecore_dcbx_default_tlv(u32 app_info_bitmap, u16 proto_id) +static bool ecore_dcbx_ieee_app_port(u32 app_info_bitmap, u8 type) { - return (ecore_dcbx_app_ethtype(app_info_bitmap) && - proto_id == ECORE_ETH_TYPE_DEFAULT) ? true : false; + u8 mfw_val = GET_MFW_FIELD(app_info_bitmap, DCBX_APP_SF_IEEE); + + /* Old MFW */ + if (mfw_val == DCBX_APP_SF_IEEE_RESERVED) + return ecore_dcbx_app_port(app_info_bitmap); + + return !!(mfw_val == type || mfw_val == DCBX_APP_SF_IEEE_TCP_UDP_PORT); } -static bool ecore_dcbx_enabled(u32 dcbx_cfg_bitmap) +static bool ecore_dcbx_default_tlv(u32 app_info_bitmap, u16 proto_id, bool ieee) { - return (ECORE_MFW_GET_FIELD(dcbx_cfg_bitmap, DCBX_CONFIG_VERSION) == - DCBX_CONFIG_VERSION_DISABLED) ? false : true; + bool ethtype; + + if (ieee) + ethtype = ecore_dcbx_ieee_app_ethtype(app_info_bitmap); + else + ethtype = ecore_dcbx_app_ethtype(app_info_bitmap); + + return !!(ethtype && (proto_id == ECORE_ETH_TYPE_DEFAULT)); } -static bool ecore_dcbx_cee(u32 dcbx_cfg_bitmap) +static bool ecore_dcbx_iwarp_tlv(struct ecore_hwfn *p_hwfn, u32 app_info_bitmap, + u16 proto_id, bool ieee) { - return (ECORE_MFW_GET_FIELD(dcbx_cfg_bitmap, DCBX_CONFIG_VERSION) == - DCBX_CONFIG_VERSION_CEE) ? true : false; -} + bool port; -static bool ecore_dcbx_ieee(u32 dcbx_cfg_bitmap) -{ - return (ECORE_MFW_GET_FIELD(dcbx_cfg_bitmap, DCBX_CONFIG_VERSION) == - DCBX_CONFIG_VERSION_IEEE) ? true : false; -} + if (!p_hwfn->p_dcbx_info->iwarp_port) + return false; -/* @@@TBD A0 Eagle workaround */ -void ecore_dcbx_eagle_workaround(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, bool set_to_pfc) -{ - if (!ENABLE_EAGLE_ENG1_WORKAROUND(p_hwfn)) - return; + if (ieee) + port = ecore_dcbx_ieee_app_port(app_info_bitmap, + DCBX_APP_SF_IEEE_TCP_PORT); + else + port = ecore_dcbx_app_port(app_info_bitmap); - ecore_wr(p_hwfn, p_ptt, - YSEM_REG_FAST_MEMORY + 0x20000 /* RAM in FASTMEM */ + - YSTORM_FLOW_CONTROL_MODE_OFFSET, - set_to_pfc ? flow_ctrl_pfc : flow_ctrl_pause); - ecore_wr(p_hwfn, p_ptt, NIG_REG_FLOWCTRL_MODE, - EAGLE_ENG1_WORKAROUND_NIG_FLOWCTRL_MODE); + return !!(port && (proto_id == p_hwfn->p_dcbx_info->iwarp_port)); } static void ecore_dcbx_dp_protocol(struct ecore_hwfn *p_hwfn, struct ecore_dcbx_results *p_data) { - struct ecore_hw_info *p_info = &p_hwfn->hw_info; enum dcbx_protocol_type id; - bool enable, update; - u8 prio, tc, size; - const char *name; /* @DPDK */ int i; - size = OSAL_ARRAY_SIZE(ecore_dcbx_app_update); + DP_VERBOSE(p_hwfn, ECORE_MSG_DCB, "DCBX negotiated: %d\n", + p_data->dcbx_enabled); - DP_INFO(p_hwfn, "DCBX negotiated: %d\n", p_data->dcbx_enabled); - - for (i = 0; i < size; i++) { + for (i = 0; i < OSAL_ARRAY_SIZE(ecore_dcbx_app_update); i++) { id = ecore_dcbx_app_update[i].id; - name = ecore_dcbx_app_update[i].name; - enable = p_data->arr[id].enable; - update = p_data->arr[id].update; - tc = p_data->arr[id].tc; - prio = p_data->arr[id].priority; - - DP_INFO(p_hwfn, - "%s info: update %d, enable %d, prio %d, tc %d, num_tc %d\n", - name, update, enable, prio, tc, p_info->num_tc); + DP_VERBOSE(p_hwfn, ECORE_MSG_DCB, + "%s info: update %d, enable %d, prio %d, tc %d," + " num_active_tc %d dscp_enable = %d dscp_val = %d\n", + ecore_dcbx_app_update[i].name, + p_data->arr[id].update, + p_data->arr[id].enable, p_data->arr[id].priority, + p_data->arr[id].tc, p_hwfn->hw_info.num_active_tc, + p_data->arr[id].dscp_enable, + p_data->arr[id].dscp_val); } } +u8 ecore_dcbx_get_dscp_value(struct ecore_hwfn *p_hwfn, u8 pri) +{ + struct ecore_dcbx_dscp_params *dscp = &p_hwfn->p_dcbx_info->get.dscp; + u8 i; + + if (!dscp->enabled) + return ECORE_DCBX_DSCP_DISABLED; + + for (i = 0; i < ECORE_DCBX_DSCP_SIZE; i++) + if (pri == dscp->dscp_pri_map[i]) + return i; + + return ECORE_DCBX_DSCP_DISABLED; +} + static void -ecore_dcbx_set_pf_tcs(struct ecore_hw_info *p_info, - u8 tc, enum ecore_pci_personality personality) -{ - /* QM reconf data */ - if (p_info->personality == personality) { - if (personality == ECORE_PCI_ETH) - p_info->non_offload_tc = tc; - else - p_info->offload_tc = tc; - } -} - -void ecore_dcbx_set_params(struct ecore_dcbx_results *p_data, - struct ecore_hw_info *p_info, - bool enable, bool update, u8 prio, u8 tc, + struct ecore_hwfn *p_hwfn, + bool enable, u8 prio, u8 tc, enum dcbx_protocol_type type, enum ecore_pci_personality personality) { /* PF update ramrod data */ - p_data->arr[type].update = update; p_data->arr[type].enable = enable; p_data->arr[type].priority = prio; p_data->arr[type].tc = tc; + p_data->arr[type].dscp_val = ecore_dcbx_get_dscp_value(p_hwfn, prio); + if (p_data->arr[type].dscp_val == ECORE_DCBX_DSCP_DISABLED) { + p_data->arr[type].dscp_enable = false; + p_data->arr[type].dscp_val = 0; + } else { + p_data->arr[type].dscp_enable = true; + } + p_data->arr[type].update = UPDATE_DCB_DSCP; - ecore_dcbx_set_pf_tcs(p_info, tc, personality); + /* QM reconf data */ + if (p_hwfn->hw_info.personality == personality) + p_hwfn->hw_info.offload_tc = tc; } /* Update app protocol data and hw_info fields with the TLV info */ static void ecore_dcbx_update_app_info(struct ecore_dcbx_results *p_data, struct ecore_hwfn *p_hwfn, - bool enable, bool update, u8 prio, u8 tc, + bool enable, u8 prio, u8 tc, enum dcbx_protocol_type type) { - struct ecore_hw_info *p_info = &p_hwfn->hw_info; enum ecore_pci_personality personality; enum dcbx_protocol_type id; - const char *name; /* @DPDK */ - u8 size; int i; - size = OSAL_ARRAY_SIZE(ecore_dcbx_app_update); - - for (i = 0; i < size; i++) { + for (i = 0; i < OSAL_ARRAY_SIZE(ecore_dcbx_app_update); i++) { id = ecore_dcbx_app_update[i].id; if (type != id) continue; personality = ecore_dcbx_app_update[i].personality; - name = ecore_dcbx_app_update[i].name; - ecore_dcbx_set_params(p_data, p_info, enable, update, + ecore_dcbx_set_params(p_data, p_hwfn, enable, prio, tc, type, personality); } } @@ -197,64 +209,85 @@ ecore_dcbx_get_app_priority(u8 pri_bitmap, u8 *priority) static bool ecore_dcbx_get_app_protocol_type(struct ecore_hwfn *p_hwfn, - u32 app_prio_bitmap, u16 id, int *type) + u32 app_prio_bitmap, u16 id, + enum dcbx_protocol_type *type, bool ieee) { - bool status = false; - - if (ecore_dcbx_default_tlv(app_prio_bitmap, id)) { + if (ecore_dcbx_default_tlv(app_prio_bitmap, id, ieee)) { *type = DCBX_PROTOCOL_ETH; - status = true; } else { - DP_ERR(p_hwfn, "Unsupported protocol %d\n", id); + *type = DCBX_MAX_PROTOCOL_TYPE; + DP_VERBOSE(p_hwfn, ECORE_MSG_DCB, + "No action required, App TLV entry = 0x%x\n", + app_prio_bitmap); + return false; } - return status; + return true; } -/* Parse app TLV's to update TC information in hw_info structure for +/* Parse app TLV's to update TC information in hw_info structure for * reconfiguring QM. Get protocol specific data for PF update ramrod command. */ static enum _ecore_status_t ecore_dcbx_process_tlv(struct ecore_hwfn *p_hwfn, struct ecore_dcbx_results *p_data, struct dcbx_app_priority_entry *p_tbl, u32 pri_tc_tbl, - int count, bool dcbx_enabled) + int count, u8 dcbx_version) { - enum _ecore_status_t rc = ECORE_SUCCESS; - u8 tc, priority, priority_map; - int i, type = -1; + enum dcbx_protocol_type type; + bool enable, ieee, eth_tlv; + u8 tc, priority_map; u16 protocol_id; - bool enable; + u8 priority; + enum _ecore_status_t rc = ECORE_SUCCESS; + int i; - DP_VERBOSE(p_hwfn, ECORE_MSG_DCB, "Num APP entries = %d\n", count); + DP_VERBOSE(p_hwfn, ECORE_MSG_DCB, + "Num APP entries = %d pri_tc_tbl = 0x%x dcbx_version = %u\n", + count, pri_tc_tbl, dcbx_version); + ieee = (dcbx_version == DCBX_CONFIG_VERSION_IEEE); + eth_tlv = false; /* Parse APP TLV */ for (i = 0; i < count; i++) { - protocol_id = ECORE_MFW_GET_FIELD(p_tbl[i].entry, - DCBX_APP_PROTOCOL_ID); - priority_map = ECORE_MFW_GET_FIELD(p_tbl[i].entry, - DCBX_APP_PRI_MAP); + protocol_id = GET_MFW_FIELD(p_tbl[i].entry, + DCBX_APP_PROTOCOL_ID); + priority_map = GET_MFW_FIELD(p_tbl[i].entry, DCBX_APP_PRI_MAP); + DP_VERBOSE(p_hwfn, ECORE_MSG_DCB, "Id = 0x%x pri_map = %u\n", + protocol_id, priority_map); rc = ecore_dcbx_get_app_priority(priority_map, &priority); if (rc == ECORE_INVAL) { DP_ERR(p_hwfn, "Invalid priority\n"); - return rc; + return ECORE_INVAL; } tc = ECORE_DCBX_PRIO2TC(pri_tc_tbl, priority); if (ecore_dcbx_get_app_protocol_type(p_hwfn, p_tbl[i].entry, - protocol_id, &type)) { + protocol_id, &type, + ieee)) { /* ETH always have the enable bit reset, as it gets * vlan information per packet. For other protocols, * should be set according to the dcbx_enabled * indication, but we only got here if there was an * app tlv for the protocol, so dcbx must be enabled. */ - enable = (type == DCBX_PROTOCOL_ETH ? false : true); + if (type == DCBX_PROTOCOL_ETH) { + enable = false; + eth_tlv = true; + } else { + enable = true; + } - ecore_dcbx_update_app_info(p_data, p_hwfn, enable, true, + ecore_dcbx_update_app_info(p_data, p_hwfn, enable, priority, tc, type); } } + + /* If Eth TLV is not detected, use UFP TC as default TC */ + if (OSAL_TEST_BIT(ECORE_MF_UFP_SPECIFIC, + &p_hwfn->p_dev->mf_bits) && !eth_tlv) + p_data->arr[DCBX_PROTOCOL_ETH].tc = p_hwfn->ufp_info.tc; + /* Update ramrod protocol data and hw_info fields * with default info when corresponding APP TLV's are not detected. * The enabled field has a different logic for ethernet as only for @@ -267,8 +300,8 @@ ecore_dcbx_process_tlv(struct ecore_hwfn *p_hwfn, if (p_data->arr[type].update) continue; - enable = (type == DCBX_PROTOCOL_ETH) ? false : dcbx_enabled; - ecore_dcbx_update_app_info(p_data, p_hwfn, enable, true, + enable = (type == DCBX_PROTOCOL_ETH) ? false : !!dcbx_version; + ecore_dcbx_update_app_info(p_data, p_hwfn, enable, priority, tc, type); } @@ -282,20 +315,17 @@ static enum _ecore_status_t ecore_dcbx_process_mib_info(struct ecore_hwfn *p_hwfn) { struct dcbx_app_priority_feature *p_app; - enum _ecore_status_t rc = ECORE_SUCCESS; - struct ecore_dcbx_results data = { 0 }; struct dcbx_app_priority_entry *p_tbl; + struct ecore_dcbx_results data = { 0 }; struct dcbx_ets_feature *p_ets; struct ecore_hw_info *p_info; u32 pri_tc_tbl, flags; - bool dcbx_enabled; + u8 dcbx_version; int num_entries; + enum _ecore_status_t rc = ECORE_SUCCESS; - /* If DCBx version is non zero, then negotiation was - * successfuly performed - */ flags = p_hwfn->p_dcbx_info->operational.flags; - dcbx_enabled = ECORE_MFW_GET_FIELD(flags, DCBX_CONFIG_VERSION) != 0; + dcbx_version = GET_MFW_FIELD(flags, DCBX_CONFIG_VERSION); p_app = &p_hwfn->p_dcbx_info->operational.features.app; p_tbl = p_app->app_pri_tbl; @@ -304,16 +334,17 @@ ecore_dcbx_process_mib_info(struct ecore_hwfn *p_hwfn) pri_tc_tbl = p_ets->pri_tc_tbl[0]; p_info = &p_hwfn->hw_info; - num_entries = ECORE_MFW_GET_FIELD(p_app->flags, DCBX_APP_NUM_ENTRIES); + num_entries = GET_MFW_FIELD(p_app->flags, DCBX_APP_NUM_ENTRIES); rc = ecore_dcbx_process_tlv(p_hwfn, &data, p_tbl, pri_tc_tbl, - num_entries, dcbx_enabled); + num_entries, dcbx_version); if (rc != ECORE_SUCCESS) return rc; - p_info->num_tc = ECORE_MFW_GET_FIELD(p_ets->flags, DCBX_ETS_MAX_TCS); + p_info->num_active_tc = GET_MFW_FIELD(p_ets->flags, DCBX_ETS_MAX_TCS); + p_hwfn->qm_info.ooo_tc = GET_MFW_FIELD(p_ets->flags, DCBX_OOO_TC); data.pf_id = p_hwfn->rel_pf_id; - data.dcbx_enabled = dcbx_enabled; + data.dcbx_enabled = !!dcbx_version; ecore_dcbx_dp_protocol(p_hwfn, &data); @@ -329,16 +360,25 @@ ecore_dcbx_copy_mib(struct ecore_hwfn *p_hwfn, struct ecore_dcbx_mib_meta_data *p_data, enum ecore_mib_read_type type) { - enum _ecore_status_t rc = ECORE_SUCCESS; u32 prefix_seq_num, suffix_seq_num; int read_count = 0; + enum _ecore_status_t rc = ECORE_SUCCESS; + /* The data is considered to be valid only if both sequence numbers are + * the same. + */ do { if (type == ECORE_DCBX_REMOTE_LLDP_MIB) { ecore_memcpy_from(p_hwfn, p_ptt, p_data->lldp_remote, p_data->addr, p_data->size); prefix_seq_num = p_data->lldp_remote->prefix_seq_num; suffix_seq_num = p_data->lldp_remote->suffix_seq_num; + } else if (type == ECORE_DCBX_LLDP_TLVS) { + ecore_memcpy_from(p_hwfn, p_ptt, p_data->lldp_tlvs, + p_data->addr, p_data->size); + prefix_seq_num = p_data->lldp_tlvs->prefix_seq_num; + suffix_seq_num = p_data->lldp_tlvs->suffix_seq_num; + } else { ecore_memcpy_from(p_hwfn, p_ptt, p_data->mib, p_data->addr, p_data->size); @@ -348,14 +388,16 @@ ecore_dcbx_copy_mib(struct ecore_hwfn *p_hwfn, read_count++; DP_VERBOSE(p_hwfn, ECORE_MSG_DCB, - "mib type = %d, try count = %d prefix seq num = %d suffix seq num = %d\n", + "mib type = %d, try count = %d prefix seq num =" + " %d suffix seq num = %d\n", type, read_count, prefix_seq_num, suffix_seq_num); } while ((prefix_seq_num != suffix_seq_num) && (read_count < ECORE_DCBX_MAX_MIB_READ_TRY)); if (read_count >= ECORE_DCBX_MAX_MIB_READ_TRY) { DP_ERR(p_hwfn, - "MIB read err, mib type = %d, try count = %d prefix seq num = %d suffix seq num = %d\n", + "MIB read err, mib type = %d, try count =" + " %d prefix seq num = %d suffix seq num = %d\n", type, read_count, prefix_seq_num, suffix_seq_num); rc = ECORE_IO; } @@ -363,57 +405,111 @@ ecore_dcbx_copy_mib(struct ecore_hwfn *p_hwfn, return rc; } -static enum _ecore_status_t +static void ecore_dcbx_get_priority_info(struct ecore_hwfn *p_hwfn, struct ecore_dcbx_app_prio *p_prio, struct ecore_dcbx_results *p_results) { - enum _ecore_status_t rc = ECORE_SUCCESS; + u8 val; if (p_results->arr[DCBX_PROTOCOL_ETH].update && - p_results->arr[DCBX_PROTOCOL_ETH].enable) { + p_results->arr[DCBX_PROTOCOL_ETH].enable) p_prio->eth = p_results->arr[DCBX_PROTOCOL_ETH].priority; - DP_VERBOSE(p_hwfn, ECORE_MSG_DCB, - "Priority: eth %d\n", p_prio->eth); - } - return rc; + DP_VERBOSE(p_hwfn, ECORE_MSG_DCB, + "Priorities: eth %d\n", + p_prio->eth); } static void ecore_dcbx_get_app_data(struct ecore_hwfn *p_hwfn, struct dcbx_app_priority_feature *p_app, struct dcbx_app_priority_entry *p_tbl, - struct ecore_dcbx_params *p_params) + struct ecore_dcbx_params *p_params, bool ieee) { + struct ecore_app_entry *entry; + u8 pri_map; int i; - p_params->app_willing = ECORE_MFW_GET_FIELD(p_app->flags, - DCBX_APP_WILLING); - p_params->app_valid = ECORE_MFW_GET_FIELD(p_app->flags, - DCBX_APP_ENABLED); - p_params->num_app_entries = ECORE_MFW_GET_FIELD(p_app->flags, - DCBX_APP_ENABLED); - for (i = 0; i < DCBX_MAX_APP_PROTOCOL; i++) - p_params->app_bitmap[i] = p_tbl[i].entry; + p_params->app_willing = GET_MFW_FIELD(p_app->flags, DCBX_APP_WILLING); + p_params->app_valid = GET_MFW_FIELD(p_app->flags, DCBX_APP_ENABLED); + p_params->app_error = GET_MFW_FIELD(p_app->flags, DCBX_APP_ERROR); + p_params->num_app_entries = GET_MFW_FIELD(p_app->flags, + DCBX_APP_NUM_ENTRIES); + for (i = 0; i < p_params->num_app_entries; i++) { + entry = &p_params->app_entry[i]; + if (ieee) { + u8 sf_ieee; + u32 val; + + sf_ieee = GET_MFW_FIELD(p_tbl[i].entry, + DCBX_APP_SF_IEEE); + switch (sf_ieee) { + case DCBX_APP_SF_IEEE_RESERVED: + /* Old MFW */ + val = GET_MFW_FIELD(p_tbl[i].entry, + DCBX_APP_SF); + entry->sf_ieee = val ? + ECORE_DCBX_SF_IEEE_TCP_UDP_PORT : + ECORE_DCBX_SF_IEEE_ETHTYPE; + break; + case DCBX_APP_SF_IEEE_ETHTYPE: + entry->sf_ieee = ECORE_DCBX_SF_IEEE_ETHTYPE; + break; + case DCBX_APP_SF_IEEE_TCP_PORT: + entry->sf_ieee = ECORE_DCBX_SF_IEEE_TCP_PORT; + break; + case DCBX_APP_SF_IEEE_UDP_PORT: + entry->sf_ieee = ECORE_DCBX_SF_IEEE_UDP_PORT; + break; + case DCBX_APP_SF_IEEE_TCP_UDP_PORT: + entry->sf_ieee = + ECORE_DCBX_SF_IEEE_TCP_UDP_PORT; + break; + } + } else { + entry->ethtype = !(GET_MFW_FIELD(p_tbl[i].entry, + DCBX_APP_SF)); + } + + pri_map = GET_MFW_FIELD(p_tbl[i].entry, DCBX_APP_PRI_MAP); + ecore_dcbx_get_app_priority(pri_map, &entry->prio); + entry->proto_id = GET_MFW_FIELD(p_tbl[i].entry, + DCBX_APP_PROTOCOL_ID); + ecore_dcbx_get_app_protocol_type(p_hwfn, p_tbl[i].entry, + entry->proto_id, + &entry->proto_type, ieee); + } DP_VERBOSE(p_hwfn, ECORE_MSG_DCB, - "APP params: willing %d, valid %d\n", - p_params->app_willing, p_params->app_valid); + "APP params: willing %d, valid %d error = %d\n", + p_params->app_willing, p_params->app_valid, + p_params->app_error); } static void ecore_dcbx_get_pfc_data(struct ecore_hwfn *p_hwfn, u32 pfc, struct ecore_dcbx_params *p_params) { - p_params->pfc_willing = ECORE_MFW_GET_FIELD(pfc, DCBX_PFC_WILLING); - p_params->max_pfc_tc = ECORE_MFW_GET_FIELD(pfc, DCBX_PFC_CAPS); - p_params->pfc_enabled = ECORE_MFW_GET_FIELD(pfc, DCBX_PFC_ENABLED); - p_params->pfc_bitmap = pfc; + u8 pfc_map; + + p_params->pfc.willing = GET_MFW_FIELD(pfc, DCBX_PFC_WILLING); + p_params->pfc.max_tc = GET_MFW_FIELD(pfc, DCBX_PFC_CAPS); + p_params->pfc.enabled = GET_MFW_FIELD(pfc, DCBX_PFC_ENABLED); + pfc_map = GET_MFW_FIELD(pfc, DCBX_PFC_PRI_EN_BITMAP); + p_params->pfc.prio[0] = !!(pfc_map & DCBX_PFC_PRI_EN_BITMAP_PRI_0); + p_params->pfc.prio[1] = !!(pfc_map & DCBX_PFC_PRI_EN_BITMAP_PRI_1); + p_params->pfc.prio[2] = !!(pfc_map & DCBX_PFC_PRI_EN_BITMAP_PRI_2); + p_params->pfc.prio[3] = !!(pfc_map & DCBX_PFC_PRI_EN_BITMAP_PRI_3); + p_params->pfc.prio[4] = !!(pfc_map & DCBX_PFC_PRI_EN_BITMAP_PRI_4); + p_params->pfc.prio[5] = !!(pfc_map & DCBX_PFC_PRI_EN_BITMAP_PRI_5); + p_params->pfc.prio[6] = !!(pfc_map & DCBX_PFC_PRI_EN_BITMAP_PRI_6); + p_params->pfc.prio[7] = !!(pfc_map & DCBX_PFC_PRI_EN_BITMAP_PRI_7); DP_VERBOSE(p_hwfn, ECORE_MSG_DCB, - "PFC params: willing %d, pfc_bitmap %d\n", - p_params->pfc_willing, p_params->pfc_bitmap); + "PFC params: willing %d, pfc_bitmap %u max_tc = %u enabled = %d\n", + p_params->pfc.willing, pfc_map, p_params->pfc.max_tc, + p_params->pfc.enabled); } static void @@ -421,28 +517,31 @@ ecore_dcbx_get_ets_data(struct ecore_hwfn *p_hwfn, struct dcbx_ets_feature *p_ets, struct ecore_dcbx_params *p_params) { + u32 bw_map[2], tsa_map[2], pri_map; int i; - p_params->ets_willing = ECORE_MFW_GET_FIELD(p_ets->flags, - DCBX_ETS_WILLING); - p_params->ets_enabled = ECORE_MFW_GET_FIELD(p_ets->flags, - DCBX_ETS_ENABLED); - p_params->max_ets_tc = ECORE_MFW_GET_FIELD(p_ets->flags, - DCBX_ETS_MAX_TCS); - p_params->ets_pri_tc_tbl[0] = p_ets->pri_tc_tbl[0]; - + p_params->ets_willing = GET_MFW_FIELD(p_ets->flags, DCBX_ETS_WILLING); + p_params->ets_enabled = GET_MFW_FIELD(p_ets->flags, DCBX_ETS_ENABLED); + p_params->ets_cbs = GET_MFW_FIELD(p_ets->flags, DCBX_ETS_CBS); + p_params->max_ets_tc = GET_MFW_FIELD(p_ets->flags, DCBX_ETS_MAX_TCS); DP_VERBOSE(p_hwfn, ECORE_MSG_DCB, - "ETS params: willing %d, pri_tc_tbl_0 %x max_ets_tc %d\n", - p_params->ets_willing, p_params->ets_pri_tc_tbl[0], + "ETS params: willing %d, enabled = %d ets_cbs %d pri_tc_tbl_0 %x max_ets_tc %d\n", + p_params->ets_willing, p_params->ets_enabled, + p_params->ets_cbs, p_ets->pri_tc_tbl[0], p_params->max_ets_tc); /* 8 bit tsa and bw data corresponding to each of the 8 TC's are * encoded in a type u32 array of size 2. */ - for (i = 0; i < 2; i++) { - p_params->ets_tc_tsa_tbl[i] = p_ets->tc_tsa_tbl[i]; - p_params->ets_tc_bw_tbl[i] = p_ets->tc_bw_tbl[i]; - + bw_map[0] = OSAL_BE32_TO_CPU(p_ets->tc_bw_tbl[0]); + bw_map[1] = OSAL_BE32_TO_CPU(p_ets->tc_bw_tbl[1]); + tsa_map[0] = OSAL_BE32_TO_CPU(p_ets->tc_tsa_tbl[0]); + tsa_map[1] = OSAL_BE32_TO_CPU(p_ets->tc_tsa_tbl[1]); + pri_map = p_ets->pri_tc_tbl[0]; + for (i = 0; i < ECORE_MAX_PFC_PRIORITIES; i++) { + p_params->ets_tc_bw_tbl[i] = ((u8 *)bw_map)[i]; + p_params->ets_tc_tsa_tbl[i] = ((u8 *)tsa_map)[i]; + p_params->ets_pri_tc_tbl[i] = ECORE_DCBX_PRIO2TC(pri_map, i); DP_VERBOSE(p_hwfn, ECORE_MSG_DCB, "elem %d bw_tbl %x tsa_tbl %x\n", i, p_params->ets_tc_bw_tbl[i], @@ -450,84 +549,81 @@ ecore_dcbx_get_ets_data(struct ecore_hwfn *p_hwfn, } } -static enum _ecore_status_t +static void ecore_dcbx_get_common_params(struct ecore_hwfn *p_hwfn, struct dcbx_app_priority_feature *p_app, struct dcbx_app_priority_entry *p_tbl, struct dcbx_ets_feature *p_ets, - u32 pfc, struct ecore_dcbx_params *p_params) + u32 pfc, struct ecore_dcbx_params *p_params, + bool ieee) { - ecore_dcbx_get_app_data(p_hwfn, p_app, p_tbl, p_params); + ecore_dcbx_get_app_data(p_hwfn, p_app, p_tbl, p_params, ieee); ecore_dcbx_get_ets_data(p_hwfn, p_ets, p_params); ecore_dcbx_get_pfc_data(p_hwfn, pfc, p_params); - - return ECORE_SUCCESS; } -static enum _ecore_status_t +static void ecore_dcbx_get_local_params(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, struct ecore_dcbx_get *params) { - struct ecore_dcbx_admin_params *p_local; - struct dcbx_app_priority_feature *p_app; - struct dcbx_app_priority_entry *p_tbl; - struct ecore_dcbx_params *p_data; - struct dcbx_ets_feature *p_ets; - u32 pfc; + struct dcbx_features *p_feat; - p_local = ¶ms->local; - p_data = &p_local->params; - p_app = &p_hwfn->p_dcbx_info->local_admin.features.app; - p_tbl = p_app->app_pri_tbl; - p_ets = &p_hwfn->p_dcbx_info->local_admin.features.ets; - pfc = p_hwfn->p_dcbx_info->local_admin.features.pfc; - - ecore_dcbx_get_common_params(p_hwfn, p_app, p_tbl, p_ets, pfc, p_data); - p_local->valid = true; - - return ECORE_SUCCESS; + p_feat = &p_hwfn->p_dcbx_info->local_admin.features; + ecore_dcbx_get_common_params(p_hwfn, &p_feat->app, + p_feat->app.app_pri_tbl, &p_feat->ets, + p_feat->pfc, ¶ms->local.params, false); + params->local.valid = true; } -static enum _ecore_status_t +static void ecore_dcbx_get_remote_params(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, struct ecore_dcbx_get *params) { - struct ecore_dcbx_remote_params *p_remote; - struct dcbx_app_priority_feature *p_app; - struct dcbx_app_priority_entry *p_tbl; - struct ecore_dcbx_params *p_data; - struct dcbx_ets_feature *p_ets; - u32 pfc; + struct dcbx_features *p_feat; - p_remote = ¶ms->remote; - p_data = &p_remote->params; - p_app = &p_hwfn->p_dcbx_info->remote.features.app; - p_tbl = p_app->app_pri_tbl; - p_ets = &p_hwfn->p_dcbx_info->remote.features.ets; - pfc = p_hwfn->p_dcbx_info->remote.features.pfc; - - ecore_dcbx_get_common_params(p_hwfn, p_app, p_tbl, p_ets, pfc, p_data); - p_remote->valid = true; - - return ECORE_SUCCESS; + p_feat = &p_hwfn->p_dcbx_info->remote.features; + ecore_dcbx_get_common_params(p_hwfn, &p_feat->app, + p_feat->app.app_pri_tbl, &p_feat->ets, + p_feat->pfc, ¶ms->remote.params, + false); + params->remote.valid = true; } -static enum _ecore_status_t +static void ecore_dcbx_get_dscp_params(struct ecore_hwfn *p_hwfn, + struct ecore_dcbx_get *params) +{ + struct ecore_dcbx_dscp_params *p_dscp; + struct dcb_dscp_map *p_dscp_map; + int i, j, entry; + u32 pri_map; + + p_dscp = ¶ms->dscp; + p_dscp_map = &p_hwfn->p_dcbx_info->dscp_map; + p_dscp->enabled = GET_MFW_FIELD(p_dscp_map->flags, DCB_DSCP_ENABLE); + + /* MFW encodes 64 dscp entries into 8 element array of u32 entries, + * where each entry holds the 4bit priority map for 8 dscp entries. + */ + for (i = 0, entry = 0; i < ECORE_DCBX_DSCP_SIZE / 8; i++) { + pri_map = OSAL_BE32_TO_CPU(p_dscp_map->dscp_pri_map[i]); + DP_VERBOSE(p_hwfn, ECORE_MSG_DCB, "elem %d pri_map 0x%x\n", + entry, pri_map); + for (j = 0; j < ECORE_DCBX_DSCP_SIZE / 8; j++, entry++) + p_dscp->dscp_pri_map[entry] = (u32)(pri_map >> + (j * 4)) & 0xf; + } +} + +static void ecore_dcbx_get_operational_params(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, struct ecore_dcbx_get *params) { struct ecore_dcbx_operational_params *p_operational; - enum _ecore_status_t rc = ECORE_SUCCESS; - struct dcbx_app_priority_feature *p_app; - struct dcbx_app_priority_entry *p_tbl; struct ecore_dcbx_results *p_results; - struct ecore_dcbx_params *p_data; - struct dcbx_ets_feature *p_ets; + struct dcbx_features *p_feat; bool enabled, err; - u32 pfc, flags; + u32 flags; + bool val; flags = p_hwfn->p_dcbx_info->operational.flags; @@ -535,122 +631,112 @@ ecore_dcbx_get_operational_params(struct ecore_hwfn *p_hwfn, * was successfuly performed */ p_operational = ¶ms->operational; - enabled = ecore_dcbx_enabled(flags); + enabled = !!(GET_MFW_FIELD(flags, DCBX_CONFIG_VERSION) != + DCBX_CONFIG_VERSION_DISABLED); if (!enabled) { p_operational->enabled = enabled; p_operational->valid = false; - return ECORE_INVAL; + DP_VERBOSE(p_hwfn, ECORE_MSG_DCB, "Dcbx is disabled\n"); + return; } - p_data = &p_operational->params; + p_feat = &p_hwfn->p_dcbx_info->operational.features; p_results = &p_hwfn->p_dcbx_info->results; - p_app = &p_hwfn->p_dcbx_info->operational.features.app; - p_tbl = p_app->app_pri_tbl; - p_ets = &p_hwfn->p_dcbx_info->operational.features.ets; - pfc = p_hwfn->p_dcbx_info->operational.features.pfc; - p_operational->ieee = ecore_dcbx_ieee(flags); - p_operational->cee = ecore_dcbx_cee(flags); + val = !!(GET_MFW_FIELD(flags, DCBX_CONFIG_VERSION) == + DCBX_CONFIG_VERSION_IEEE); + p_operational->ieee = val; + + val = !!(GET_MFW_FIELD(flags, DCBX_CONFIG_VERSION) == + DCBX_CONFIG_VERSION_CEE); + p_operational->cee = val; + + val = !!(GET_MFW_FIELD(flags, DCBX_CONFIG_VERSION) == + DCBX_CONFIG_VERSION_STATIC); + p_operational->local = val; DP_VERBOSE(p_hwfn, ECORE_MSG_DCB, - "Version support: ieee %d, cee %d\n", - p_operational->ieee, p_operational->cee); + "Version support: ieee %d, cee %d, static %d\n", + p_operational->ieee, p_operational->cee, + p_operational->local); - ecore_dcbx_get_common_params(p_hwfn, p_app, p_tbl, p_ets, pfc, p_data); + ecore_dcbx_get_common_params(p_hwfn, &p_feat->app, + p_feat->app.app_pri_tbl, &p_feat->ets, + p_feat->pfc, ¶ms->operational.params, + p_operational->ieee); ecore_dcbx_get_priority_info(p_hwfn, &p_operational->app_prio, p_results); - err = ECORE_MFW_GET_FIELD(p_app->flags, DCBX_APP_ERROR); + err = GET_MFW_FIELD(p_feat->app.flags, DCBX_APP_ERROR); p_operational->err = err; p_operational->enabled = enabled; p_operational->valid = true; - - return rc; } -static enum _ecore_status_t -ecore_dcbx_get_local_lldp_params(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, - struct ecore_dcbx_get *params) +static void ecore_dcbx_get_local_lldp_params(struct ecore_hwfn *p_hwfn, + struct ecore_dcbx_get *params) { - struct ecore_dcbx_lldp_local *p_local; - osal_size_t size; - u32 *dest; + struct lldp_config_params_s *p_local; - p_local = ¶ms->lldp_local; + p_local = &p_hwfn->p_dcbx_info->lldp_local[LLDP_NEAREST_BRIDGE]; - size = OSAL_ARRAY_SIZE(p_local->local_chassis_id); - dest = p_hwfn->p_dcbx_info->get.lldp_local.local_chassis_id; - OSAL_MEMCPY(dest, p_local->local_chassis_id, size); - - size = OSAL_ARRAY_SIZE(p_local->local_port_id); - dest = p_hwfn->p_dcbx_info->get.lldp_local.local_port_id; - OSAL_MEMCPY(dest, p_local->local_port_id, size); - - return ECORE_SUCCESS; + OSAL_MEMCPY(params->lldp_local.local_chassis_id, + p_local->local_chassis_id, + sizeof(params->lldp_local.local_chassis_id)); + OSAL_MEMCPY(params->lldp_local.local_port_id, p_local->local_port_id, + sizeof(params->lldp_local.local_port_id)); } -static enum _ecore_status_t -ecore_dcbx_get_remote_lldp_params(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, - struct ecore_dcbx_get *params) +static void ecore_dcbx_get_remote_lldp_params(struct ecore_hwfn *p_hwfn, + struct ecore_dcbx_get *params) { - struct ecore_dcbx_lldp_remote *p_remote; - osal_size_t size; - u32 *dest; + struct lldp_status_params_s *p_remote; - p_remote = ¶ms->lldp_remote; + p_remote = &p_hwfn->p_dcbx_info->lldp_remote[LLDP_NEAREST_BRIDGE]; - size = OSAL_ARRAY_SIZE(p_remote->peer_chassis_id); - dest = p_hwfn->p_dcbx_info->get.lldp_remote.peer_chassis_id; - OSAL_MEMCPY(dest, p_remote->peer_chassis_id, size); - - size = OSAL_ARRAY_SIZE(p_remote->peer_port_id); - dest = p_hwfn->p_dcbx_info->get.lldp_remote.peer_port_id; - OSAL_MEMCPY(dest, p_remote->peer_port_id, size); - - return ECORE_SUCCESS; + OSAL_MEMCPY(params->lldp_remote.peer_chassis_id, + p_remote->peer_chassis_id, + sizeof(params->lldp_remote.peer_chassis_id)); + OSAL_MEMCPY(params->lldp_remote.peer_port_id, p_remote->peer_port_id, + sizeof(params->lldp_remote.peer_port_id)); } static enum _ecore_status_t ecore_dcbx_get_params(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, enum ecore_mib_read_type type) + struct ecore_dcbx_get *p_params, + enum ecore_mib_read_type type) { - enum _ecore_status_t rc = ECORE_SUCCESS; - struct ecore_dcbx_get *p_params; - - p_params = &p_hwfn->p_dcbx_info->get; - switch (type) { case ECORE_DCBX_REMOTE_MIB: - ecore_dcbx_get_remote_params(p_hwfn, p_ptt, p_params); + ecore_dcbx_get_remote_params(p_hwfn, p_params); break; case ECORE_DCBX_LOCAL_MIB: - ecore_dcbx_get_local_params(p_hwfn, p_ptt, p_params); + ecore_dcbx_get_local_params(p_hwfn, p_params); break; case ECORE_DCBX_OPERATIONAL_MIB: - ecore_dcbx_get_operational_params(p_hwfn, p_ptt, p_params); + ecore_dcbx_get_operational_params(p_hwfn, p_params); break; case ECORE_DCBX_REMOTE_LLDP_MIB: - rc = ecore_dcbx_get_remote_lldp_params(p_hwfn, p_ptt, p_params); + ecore_dcbx_get_remote_lldp_params(p_hwfn, p_params); break; case ECORE_DCBX_LOCAL_LLDP_MIB: - rc = ecore_dcbx_get_local_lldp_params(p_hwfn, p_ptt, p_params); + ecore_dcbx_get_local_lldp_params(p_hwfn, p_params); break; default: DP_ERR(p_hwfn, "MIB read err, unknown mib type %d\n", type); return ECORE_INVAL; } - return rc; + return ECORE_SUCCESS; } static enum _ecore_status_t ecore_dcbx_read_local_lldp_mib(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt) { - enum _ecore_status_t rc = ECORE_SUCCESS; struct ecore_dcbx_mib_meta_data data; + enum _ecore_status_t rc = ECORE_SUCCESS; + OSAL_MEM_ZERO(&data, sizeof(data)); data.addr = p_hwfn->mcp_info->port_addr + offsetof(struct public_port, lldp_config_params); data.lldp_local = p_hwfn->p_dcbx_info->lldp_local; @@ -665,9 +751,10 @@ ecore_dcbx_read_remote_lldp_mib(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, enum ecore_mib_read_type type) { - enum _ecore_status_t rc = ECORE_SUCCESS; struct ecore_dcbx_mib_meta_data data; + enum _ecore_status_t rc = ECORE_SUCCESS; + OSAL_MEM_ZERO(&data, sizeof(data)); data.addr = p_hwfn->mcp_info->port_addr + offsetof(struct public_port, lldp_status_params); data.lldp_remote = p_hwfn->p_dcbx_info->lldp_remote; @@ -685,6 +772,7 @@ ecore_dcbx_read_operational_mib(struct ecore_hwfn *p_hwfn, struct ecore_dcbx_mib_meta_data data; enum _ecore_status_t rc = ECORE_SUCCESS; + OSAL_MEM_ZERO(&data, sizeof(data)); data.addr = p_hwfn->mcp_info->port_addr + offsetof(struct public_port, operational_dcbx_mib); data.mib = &p_hwfn->p_dcbx_info->operational; @@ -702,6 +790,7 @@ ecore_dcbx_read_remote_mib(struct ecore_hwfn *p_hwfn, struct ecore_dcbx_mib_meta_data data; enum _ecore_status_t rc = ECORE_SUCCESS; + OSAL_MEM_ZERO(&data, sizeof(data)); data.addr = p_hwfn->mcp_info->port_addr + offsetof(struct public_port, remote_dcbx_mib); data.mib = &p_hwfn->p_dcbx_info->remote; @@ -717,6 +806,7 @@ ecore_dcbx_read_local_mib(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt) struct ecore_dcbx_mib_meta_data data; enum _ecore_status_t rc = ECORE_SUCCESS; + OSAL_MEM_ZERO(&data, sizeof(data)); data.addr = p_hwfn->mcp_info->port_addr + offsetof(struct public_port, local_admin_dcbx_mib); data.local_admin = &p_hwfn->p_dcbx_info->local_admin; @@ -727,14 +817,27 @@ ecore_dcbx_read_local_mib(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt) return rc; } +static void +ecore_dcbx_read_dscp_mib(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt) +{ + struct ecore_dcbx_mib_meta_data data; + + data.addr = p_hwfn->mcp_info->port_addr + + offsetof(struct public_port, dcb_dscp_map); + data.dscp_map = &p_hwfn->p_dcbx_info->dscp_map; + data.size = sizeof(struct dcb_dscp_map); + ecore_memcpy_from(p_hwfn, p_ptt, data.dscp_map, data.addr, data.size); +} + static enum _ecore_status_t ecore_dcbx_read_mib(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, enum ecore_mib_read_type type) { - enum _ecore_status_t rc = ECORE_SUCCESS; + enum _ecore_status_t rc = ECORE_INVAL; switch (type) { case ECORE_DCBX_OPERATIONAL_MIB: + ecore_dcbx_read_dscp_mib(p_hwfn, p_ptt); rc = ecore_dcbx_read_operational_mib(p_hwfn, p_ptt, type); break; case ECORE_DCBX_REMOTE_MIB: @@ -751,10 +854,9 @@ static enum _ecore_status_t ecore_dcbx_read_mib(struct ecore_hwfn *p_hwfn, break; default: DP_ERR(p_hwfn, "MIB read err, unknown mib type %d\n", type); - return ECORE_INVAL; } - return rc; + return ECORE_SUCCESS; } /* @@ -773,26 +875,31 @@ ecore_dcbx_mib_update_event(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, return rc; if (type == ECORE_DCBX_OPERATIONAL_MIB) { + ecore_dcbx_get_dscp_params(p_hwfn, &p_hwfn->p_dcbx_info->get); + rc = ecore_dcbx_process_mib_info(p_hwfn); if (!rc) { - bool enabled; - /* reconfigure tcs of QM queues according * to negotiation results */ ecore_qm_reconf(p_hwfn, p_ptt); /* update storm FW with negotiation results */ - ecore_sp_pf_update(p_hwfn); - - /* set eagle enigne 1 flow control workaround - * according to negotiation results - */ - enabled = p_hwfn->p_dcbx_info->results.dcbx_enabled; - ecore_dcbx_eagle_workaround(p_hwfn, p_ptt, enabled); + ecore_sp_pf_update_dcbx(p_hwfn); } } - ecore_dcbx_get_params(p_hwfn, p_ptt, type); + + ecore_dcbx_get_params(p_hwfn, &p_hwfn->p_dcbx_info->get, type); + + /* Update the DSCP to TC mapping bit if required */ + if ((type == ECORE_DCBX_OPERATIONAL_MIB) && + p_hwfn->p_dcbx_info->dscp_nig_update) { + u8 val = !!p_hwfn->p_dcbx_info->get.dscp.enabled; + + ecore_wr(p_hwfn, p_ptt, NIG_REG_DSCP_TO_TC_MAP_ENABLE, val); + p_hwfn->p_dcbx_info->dscp_nig_update = false; + } + OSAL_DCBX_AEN(p_hwfn, type); return rc; @@ -800,21 +907,21 @@ ecore_dcbx_mib_update_event(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, enum _ecore_status_t ecore_dcbx_info_alloc(struct ecore_hwfn *p_hwfn) { - enum _ecore_status_t rc = ECORE_SUCCESS; - p_hwfn->p_dcbx_info = OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL, - sizeof(struct ecore_dcbx_info)); + sizeof(*p_hwfn->p_dcbx_info)); if (!p_hwfn->p_dcbx_info) { DP_NOTICE(p_hwfn, true, "Failed to allocate `struct ecore_dcbx_info'"); - rc = ECORE_NOMEM; + return ECORE_NOMEM; } - return rc; + p_hwfn->p_dcbx_info->iwarp_port = + p_hwfn->pf_params.rdma_pf_params.iwarp_port; + + return ECORE_SUCCESS; } -void ecore_dcbx_info_free(struct ecore_hwfn *p_hwfn, - struct ecore_dcbx_info *p_dcbx_info) +void ecore_dcbx_info_free(struct ecore_hwfn *p_hwfn) { OSAL_FREE(p_hwfn->p_dev, p_hwfn->p_dcbx_info); } @@ -826,6 +933,8 @@ static void ecore_dcbx_update_protocol_data(struct protocol_dcb_data *p_data, p_data->dcb_enable_flag = p_src->arr[type].enable; p_data->dcb_priority = p_src->arr[type].priority; p_data->dcb_tc = p_src->arr[type].tc; + p_data->dscp_enable_flag = p_src->arr[type].dscp_enable; + p_data->dscp_val = p_src->arr[type].dscp_val; } /* Set pf update ramrod command params */ @@ -833,55 +942,591 @@ void ecore_dcbx_set_pf_update_params(struct ecore_dcbx_results *p_src, struct pf_update_ramrod_data *p_dest) { struct protocol_dcb_data *p_dcb_data; - bool update_flag; - - p_dest->pf_id = p_src->pf_id; + u8 update_flag; update_flag = p_src->arr[DCBX_PROTOCOL_ETH].update; - p_dest->update_eth_dcb_data_flag = update_flag; + p_dest->update_eth_dcb_data_mode = update_flag; + update_flag = p_src->arr[DCBX_PROTOCOL_IWARP].update; + p_dest->update_iwarp_dcb_data_mode = update_flag; p_dcb_data = &p_dest->eth_dcb_data; ecore_dcbx_update_protocol_data(p_dcb_data, p_src, DCBX_PROTOCOL_ETH); -} - -static -enum _ecore_status_t ecore_dcbx_query(struct ecore_hwfn *p_hwfn, - enum ecore_mib_read_type type) -{ - struct ecore_ptt *p_ptt; - enum _ecore_status_t rc; - - p_ptt = ecore_ptt_acquire(p_hwfn); - if (!p_ptt) { - rc = ECORE_TIMEOUT; - DP_ERR(p_hwfn, "rc = %d\n", rc); - return rc; - } - - rc = ecore_dcbx_read_mib(p_hwfn, p_ptt, type); - if (rc != ECORE_SUCCESS) - goto out; - - rc = ecore_dcbx_get_params(p_hwfn, p_ptt, type); - -out: - ecore_ptt_release(p_hwfn, p_ptt); - return rc; + p_dcb_data = &p_dest->iwarp_dcb_data; + ecore_dcbx_update_protocol_data(p_dcb_data, p_src, DCBX_PROTOCOL_IWARP); } enum _ecore_status_t ecore_dcbx_query_params(struct ecore_hwfn *p_hwfn, struct ecore_dcbx_get *p_get, enum ecore_mib_read_type type) { + struct ecore_ptt *p_ptt; enum _ecore_status_t rc; - rc = ecore_dcbx_query(p_hwfn, type); - if (rc) - return rc; + if (IS_VF(p_hwfn->p_dev)) + return ECORE_INVAL; - if (p_get != OSAL_NULL) - OSAL_MEMCPY(p_get, &p_hwfn->p_dcbx_info->get, - sizeof(struct ecore_dcbx_get)); + p_ptt = ecore_ptt_acquire(p_hwfn); + if (!p_ptt) + return ECORE_TIMEOUT; + + rc = ecore_dcbx_read_mib(p_hwfn, p_ptt, type); + if (rc != ECORE_SUCCESS) + goto out; + + ecore_dcbx_get_dscp_params(p_hwfn, p_get); + + rc = ecore_dcbx_get_params(p_hwfn, p_get, type); + +out: + ecore_ptt_release(p_hwfn, p_ptt); + return rc; +} + +static void +ecore_dcbx_set_pfc_data(struct ecore_hwfn *p_hwfn, + u32 *pfc, struct ecore_dcbx_params *p_params) +{ + u8 pfc_map = 0; + int i; + + if (p_params->pfc.willing) + *pfc |= DCBX_PFC_WILLING_MASK; + else + *pfc &= ~DCBX_PFC_WILLING_MASK; + + if (p_params->pfc.enabled) + *pfc |= DCBX_PFC_ENABLED_MASK; + else + *pfc &= ~DCBX_PFC_ENABLED_MASK; + + *pfc &= ~DCBX_PFC_CAPS_MASK; + *pfc |= (u32)p_params->pfc.max_tc << DCBX_PFC_CAPS_OFFSET; + + for (i = 0; i < ECORE_MAX_PFC_PRIORITIES; i++) + if (p_params->pfc.prio[i]) + pfc_map |= (1 << i); + *pfc &= ~DCBX_PFC_PRI_EN_BITMAP_MASK; + *pfc |= (pfc_map << DCBX_PFC_PRI_EN_BITMAP_OFFSET); + + DP_VERBOSE(p_hwfn, ECORE_MSG_DCB, "pfc = 0x%x\n", *pfc); +} + +static void +ecore_dcbx_set_ets_data(struct ecore_hwfn *p_hwfn, + struct dcbx_ets_feature *p_ets, + struct ecore_dcbx_params *p_params) +{ + u8 *bw_map, *tsa_map; + u32 val; + int i; + + if (p_params->ets_willing) + p_ets->flags |= DCBX_ETS_WILLING_MASK; + else + p_ets->flags &= ~DCBX_ETS_WILLING_MASK; + + if (p_params->ets_cbs) + p_ets->flags |= DCBX_ETS_CBS_MASK; + else + p_ets->flags &= ~DCBX_ETS_CBS_MASK; + + if (p_params->ets_enabled) + p_ets->flags |= DCBX_ETS_ENABLED_MASK; + else + p_ets->flags &= ~DCBX_ETS_ENABLED_MASK; + + p_ets->flags &= ~DCBX_ETS_MAX_TCS_MASK; + p_ets->flags |= (u32)p_params->max_ets_tc << DCBX_ETS_MAX_TCS_OFFSET; + + bw_map = (u8 *)&p_ets->tc_bw_tbl[0]; + tsa_map = (u8 *)&p_ets->tc_tsa_tbl[0]; + p_ets->pri_tc_tbl[0] = 0; + for (i = 0; i < ECORE_MAX_PFC_PRIORITIES; i++) { + bw_map[i] = p_params->ets_tc_bw_tbl[i]; + tsa_map[i] = p_params->ets_tc_tsa_tbl[i]; + /* Copy the priority value to the corresponding 4 bits in the + * traffic class table. + */ + val = (((u32)p_params->ets_pri_tc_tbl[i]) << ((7 - i) * 4)); + p_ets->pri_tc_tbl[0] |= val; + } + for (i = 0; i < 2; i++) { + p_ets->tc_bw_tbl[i] = OSAL_CPU_TO_BE32(p_ets->tc_bw_tbl[i]); + p_ets->tc_tsa_tbl[i] = OSAL_CPU_TO_BE32(p_ets->tc_tsa_tbl[i]); + } + + DP_VERBOSE(p_hwfn, ECORE_MSG_DCB, + "flags = 0x%x pri_tc = 0x%x tc_bwl[] = {0x%x, 0x%x} tc_tsa = {0x%x, 0x%x}\n", + p_ets->flags, p_ets->pri_tc_tbl[0], p_ets->tc_bw_tbl[0], + p_ets->tc_bw_tbl[1], p_ets->tc_tsa_tbl[0], + p_ets->tc_tsa_tbl[1]); +} + +static void +ecore_dcbx_set_app_data(struct ecore_hwfn *p_hwfn, + struct dcbx_app_priority_feature *p_app, + struct ecore_dcbx_params *p_params, bool ieee) +{ + u32 *entry; + int i; + + if (p_params->app_willing) + p_app->flags |= DCBX_APP_WILLING_MASK; + else + p_app->flags &= ~DCBX_APP_WILLING_MASK; + + if (p_params->app_valid) + p_app->flags |= DCBX_APP_ENABLED_MASK; + else + p_app->flags &= ~DCBX_APP_ENABLED_MASK; + + p_app->flags &= ~DCBX_APP_NUM_ENTRIES_MASK; + p_app->flags |= (u32)p_params->num_app_entries << + DCBX_APP_NUM_ENTRIES_OFFSET; + + for (i = 0; i < p_params->num_app_entries; i++) { + entry = &p_app->app_pri_tbl[i].entry; + *entry = 0; + if (ieee) { + *entry &= ~(DCBX_APP_SF_IEEE_MASK | DCBX_APP_SF_MASK); + switch (p_params->app_entry[i].sf_ieee) { + case ECORE_DCBX_SF_IEEE_ETHTYPE: + *entry |= ((u32)DCBX_APP_SF_IEEE_ETHTYPE << + DCBX_APP_SF_IEEE_OFFSET); + *entry |= ((u32)DCBX_APP_SF_ETHTYPE << + DCBX_APP_SF_OFFSET); + break; + case ECORE_DCBX_SF_IEEE_TCP_PORT: + *entry |= ((u32)DCBX_APP_SF_IEEE_TCP_PORT << + DCBX_APP_SF_IEEE_OFFSET); + *entry |= ((u32)DCBX_APP_SF_PORT << + DCBX_APP_SF_OFFSET); + break; + case ECORE_DCBX_SF_IEEE_UDP_PORT: + *entry |= ((u32)DCBX_APP_SF_IEEE_UDP_PORT << + DCBX_APP_SF_IEEE_OFFSET); + *entry |= ((u32)DCBX_APP_SF_PORT << + DCBX_APP_SF_OFFSET); + break; + case ECORE_DCBX_SF_IEEE_TCP_UDP_PORT: + *entry |= (u32)DCBX_APP_SF_IEEE_TCP_UDP_PORT << + DCBX_APP_SF_IEEE_OFFSET; + *entry |= ((u32)DCBX_APP_SF_PORT << + DCBX_APP_SF_OFFSET); + break; + } + } else { + *entry &= ~DCBX_APP_SF_MASK; + if (p_params->app_entry[i].ethtype) + *entry |= ((u32)DCBX_APP_SF_ETHTYPE << + DCBX_APP_SF_OFFSET); + else + *entry |= ((u32)DCBX_APP_SF_PORT << + DCBX_APP_SF_OFFSET); + } + *entry &= ~DCBX_APP_PROTOCOL_ID_MASK; + *entry |= ((u32)p_params->app_entry[i].proto_id << + DCBX_APP_PROTOCOL_ID_OFFSET); + *entry &= ~DCBX_APP_PRI_MAP_MASK; + *entry |= ((u32)(p_params->app_entry[i].prio) << + DCBX_APP_PRI_MAP_OFFSET); + } + + DP_VERBOSE(p_hwfn, ECORE_MSG_DCB, "flags = 0x%x\n", p_app->flags); +} + +static void +ecore_dcbx_set_local_params(struct ecore_hwfn *p_hwfn, + struct dcbx_local_params *local_admin, + struct ecore_dcbx_set *params) +{ + bool ieee = false; + + local_admin->flags = 0; + OSAL_MEMCPY(&local_admin->features, + &p_hwfn->p_dcbx_info->operational.features, + sizeof(local_admin->features)); + + if (params->enabled) { + local_admin->config = params->ver_num; + ieee = !!(params->ver_num & DCBX_CONFIG_VERSION_IEEE); + } else { + local_admin->config = DCBX_CONFIG_VERSION_DISABLED; + } + + DP_VERBOSE(p_hwfn, ECORE_MSG_DCB, "Dcbx version = %d\n", + local_admin->config); + + if (params->override_flags & ECORE_DCBX_OVERRIDE_PFC_CFG) + ecore_dcbx_set_pfc_data(p_hwfn, &local_admin->features.pfc, + ¶ms->config.params); + + if (params->override_flags & ECORE_DCBX_OVERRIDE_ETS_CFG) + ecore_dcbx_set_ets_data(p_hwfn, &local_admin->features.ets, + ¶ms->config.params); + + if (params->override_flags & ECORE_DCBX_OVERRIDE_APP_CFG) + ecore_dcbx_set_app_data(p_hwfn, &local_admin->features.app, + ¶ms->config.params, ieee); +} + +static enum _ecore_status_t +ecore_dcbx_set_dscp_params(struct ecore_hwfn *p_hwfn, + struct dcb_dscp_map *p_dscp_map, + struct ecore_dcbx_set *p_params) +{ + int entry, i, j; + u32 val; + + OSAL_MEMCPY(p_dscp_map, &p_hwfn->p_dcbx_info->dscp_map, + sizeof(*p_dscp_map)); + + p_dscp_map->flags &= ~DCB_DSCP_ENABLE_MASK; + if (p_params->dscp.enabled) + p_dscp_map->flags |= DCB_DSCP_ENABLE_MASK; + + for (i = 0, entry = 0; i < 8; i++) { + val = 0; + for (j = 0; j < 8; j++, entry++) + val |= (((u32)p_params->dscp.dscp_pri_map[entry]) << + (j * 4)); + + p_dscp_map->dscp_pri_map[i] = OSAL_CPU_TO_BE32(val); + } + + p_hwfn->p_dcbx_info->dscp_nig_update = true; + + DP_VERBOSE(p_hwfn, ECORE_MSG_DCB, "flags = 0x%x\n", p_dscp_map->flags); + DP_VERBOSE(p_hwfn, ECORE_MSG_DCB, + "pri_map[] = 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x\n", + p_dscp_map->dscp_pri_map[0], p_dscp_map->dscp_pri_map[1], + p_dscp_map->dscp_pri_map[2], p_dscp_map->dscp_pri_map[3], + p_dscp_map->dscp_pri_map[4], p_dscp_map->dscp_pri_map[5], + p_dscp_map->dscp_pri_map[6], p_dscp_map->dscp_pri_map[7]); + + return ECORE_SUCCESS; +} + +enum _ecore_status_t ecore_dcbx_config_params(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + struct ecore_dcbx_set *params, + bool hw_commit) +{ + struct dcbx_local_params local_admin; + struct ecore_dcbx_mib_meta_data data; + struct dcb_dscp_map dscp_map; + u32 resp = 0, param = 0; + enum _ecore_status_t rc = ECORE_SUCCESS; + + OSAL_MEMCPY(&p_hwfn->p_dcbx_info->set, params, + sizeof(p_hwfn->p_dcbx_info->set)); + if (!hw_commit) + return ECORE_SUCCESS; + + OSAL_MEMSET(&local_admin, 0, sizeof(local_admin)); + ecore_dcbx_set_local_params(p_hwfn, &local_admin, params); + + data.addr = p_hwfn->mcp_info->port_addr + + offsetof(struct public_port, local_admin_dcbx_mib); + data.local_admin = &local_admin; + data.size = sizeof(struct dcbx_local_params); + ecore_memcpy_to(p_hwfn, p_ptt, data.addr, data.local_admin, data.size); + + if (params->override_flags & ECORE_DCBX_OVERRIDE_DSCP_CFG) { + OSAL_MEMSET(&dscp_map, 0, sizeof(dscp_map)); + ecore_dcbx_set_dscp_params(p_hwfn, &dscp_map, params); + + data.addr = p_hwfn->mcp_info->port_addr + + offsetof(struct public_port, dcb_dscp_map); + data.dscp_map = &dscp_map; + data.size = sizeof(struct dcb_dscp_map); + ecore_memcpy_to(p_hwfn, p_ptt, data.addr, data.dscp_map, + data.size); + } + + rc = ecore_mcp_cmd(p_hwfn, p_ptt, DRV_MSG_CODE_SET_DCBX, + 1 << DRV_MB_PARAM_LLDP_SEND_OFFSET, &resp, ¶m); + if (rc != ECORE_SUCCESS) + DP_NOTICE(p_hwfn, false, + "Failed to send DCBX update request\n"); + + return rc; +} + +enum _ecore_status_t ecore_dcbx_get_config_params(struct ecore_hwfn *p_hwfn, + struct ecore_dcbx_set *params) +{ + struct ecore_dcbx_get *dcbx_info; + int rc; + + if (p_hwfn->p_dcbx_info->set.config.valid) { + OSAL_MEMCPY(params, &p_hwfn->p_dcbx_info->set, + sizeof(struct ecore_dcbx_set)); + return ECORE_SUCCESS; + } + + dcbx_info = OSAL_ALLOC(p_hwfn->p_dev, GFP_KERNEL, + sizeof(*dcbx_info)); + if (!dcbx_info) + return ECORE_NOMEM; + + OSAL_MEMSET(dcbx_info, 0, sizeof(*dcbx_info)); + rc = ecore_dcbx_query_params(p_hwfn, dcbx_info, + ECORE_DCBX_OPERATIONAL_MIB); + if (rc) { + OSAL_FREE(p_hwfn->p_dev, dcbx_info); + return rc; + } + p_hwfn->p_dcbx_info->set.override_flags = 0; + + p_hwfn->p_dcbx_info->set.ver_num = DCBX_CONFIG_VERSION_DISABLED; + if (dcbx_info->operational.cee) + p_hwfn->p_dcbx_info->set.ver_num |= DCBX_CONFIG_VERSION_CEE; + if (dcbx_info->operational.ieee) + p_hwfn->p_dcbx_info->set.ver_num |= DCBX_CONFIG_VERSION_IEEE; + if (dcbx_info->operational.local) + p_hwfn->p_dcbx_info->set.ver_num |= DCBX_CONFIG_VERSION_STATIC; + + p_hwfn->p_dcbx_info->set.enabled = dcbx_info->operational.enabled; + OSAL_MEMCPY(&p_hwfn->p_dcbx_info->set.dscp, + &p_hwfn->p_dcbx_info->get.dscp, + sizeof(struct ecore_dcbx_dscp_params)); + OSAL_MEMCPY(&p_hwfn->p_dcbx_info->set.config.params, + &dcbx_info->operational.params, + sizeof(p_hwfn->p_dcbx_info->set.config.params)); + p_hwfn->p_dcbx_info->set.config.valid = true; + + OSAL_MEMCPY(params, &p_hwfn->p_dcbx_info->set, + sizeof(struct ecore_dcbx_set)); + + OSAL_FREE(p_hwfn->p_dev, dcbx_info); + + return ECORE_SUCCESS; +} + +enum _ecore_status_t ecore_lldp_register_tlv(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + enum ecore_lldp_agent agent, + u8 tlv_type) +{ + u32 mb_param = 0, mcp_resp = 0, mcp_param = 0, val = 0; + enum _ecore_status_t rc = ECORE_SUCCESS; + + switch (agent) { + case ECORE_LLDP_NEAREST_BRIDGE: + val = LLDP_NEAREST_BRIDGE; + break; + case ECORE_LLDP_NEAREST_NON_TPMR_BRIDGE: + val = LLDP_NEAREST_NON_TPMR_BRIDGE; + break; + case ECORE_LLDP_NEAREST_CUSTOMER_BRIDGE: + val = LLDP_NEAREST_CUSTOMER_BRIDGE; + break; + default: + DP_ERR(p_hwfn, "Invalid agent type %d\n", agent); + return ECORE_INVAL; + } + + SET_MFW_FIELD(mb_param, DRV_MB_PARAM_LLDP_AGENT, val); + SET_MFW_FIELD(mb_param, DRV_MB_PARAM_LLDP_TLV_RX_TYPE, tlv_type); + + rc = ecore_mcp_cmd(p_hwfn, p_ptt, DRV_MSG_CODE_REGISTER_LLDP_TLVS_RX, + mb_param, &mcp_resp, &mcp_param); + if (rc != ECORE_SUCCESS) + DP_NOTICE(p_hwfn, false, "Failed to register TLV\n"); + + return rc; +} + +enum _ecore_status_t +ecore_lldp_mib_update_event(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt) +{ + struct ecore_dcbx_mib_meta_data data; + enum _ecore_status_t rc = ECORE_SUCCESS; + struct lldp_received_tlvs_s tlvs; + int i; + + for (i = 0; i < LLDP_MAX_LLDP_AGENTS; i++) { + OSAL_MEM_ZERO(&data, sizeof(data)); + data.addr = p_hwfn->mcp_info->port_addr + + offsetof(struct public_port, lldp_received_tlvs[i]); + data.lldp_tlvs = &tlvs; + data.size = sizeof(tlvs); + rc = ecore_dcbx_copy_mib(p_hwfn, p_ptt, &data, + ECORE_DCBX_LLDP_TLVS); + if (rc != ECORE_SUCCESS) { + DP_NOTICE(p_hwfn, false, "Failed to read lldp TLVs\n"); + return rc; + } + + if (!tlvs.length) + continue; + + for (i = 0; i < MAX_TLV_BUFFER; i++) + tlvs.tlvs_buffer[i] = + OSAL_CPU_TO_BE32(tlvs.tlvs_buffer[i]); + + OSAL_LLDP_RX_TLVS(p_hwfn, tlvs.tlvs_buffer, tlvs.length); + } + + return rc; +} + +enum _ecore_status_t +ecore_lldp_get_params(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, + struct ecore_lldp_config_params *p_params) +{ + struct lldp_config_params_s lldp_params; + u32 addr, val; + int i; + + switch (p_params->agent) { + case ECORE_LLDP_NEAREST_BRIDGE: + val = LLDP_NEAREST_BRIDGE; + break; + case ECORE_LLDP_NEAREST_NON_TPMR_BRIDGE: + val = LLDP_NEAREST_NON_TPMR_BRIDGE; + break; + case ECORE_LLDP_NEAREST_CUSTOMER_BRIDGE: + val = LLDP_NEAREST_CUSTOMER_BRIDGE; + break; + default: + DP_ERR(p_hwfn, "Invalid agent type %d\n", p_params->agent); + return ECORE_INVAL; + } + + addr = p_hwfn->mcp_info->port_addr + + offsetof(struct public_port, lldp_config_params[val]); + + ecore_memcpy_from(p_hwfn, p_ptt, &lldp_params, addr, + sizeof(lldp_params)); + + p_params->tx_interval = GET_MFW_FIELD(lldp_params.config, + LLDP_CONFIG_TX_INTERVAL); + p_params->tx_hold = GET_MFW_FIELD(lldp_params.config, LLDP_CONFIG_HOLD); + p_params->tx_credit = GET_MFW_FIELD(lldp_params.config, + LLDP_CONFIG_MAX_CREDIT); + p_params->rx_enable = GET_MFW_FIELD(lldp_params.config, + LLDP_CONFIG_ENABLE_RX); + p_params->tx_enable = GET_MFW_FIELD(lldp_params.config, + LLDP_CONFIG_ENABLE_TX); + + OSAL_MEMCPY(p_params->chassis_id_tlv, lldp_params.local_chassis_id, + sizeof(p_params->chassis_id_tlv)); + for (i = 0; i < ECORE_LLDP_CHASSIS_ID_STAT_LEN; i++) + p_params->chassis_id_tlv[i] = + OSAL_BE32_TO_CPU(p_params->chassis_id_tlv[i]); + + OSAL_MEMCPY(p_params->port_id_tlv, lldp_params.local_port_id, + sizeof(p_params->port_id_tlv)); + for (i = 0; i < ECORE_LLDP_PORT_ID_STAT_LEN; i++) + p_params->port_id_tlv[i] = + OSAL_BE32_TO_CPU(p_params->port_id_tlv[i]); + + return ECORE_SUCCESS; +} + +enum _ecore_status_t +ecore_lldp_set_params(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, + struct ecore_lldp_config_params *p_params) +{ + u32 mb_param = 0, mcp_resp = 0, mcp_param = 0; + struct lldp_config_params_s lldp_params; + enum _ecore_status_t rc = ECORE_SUCCESS; + u32 addr, val; + int i; + + switch (p_params->agent) { + case ECORE_LLDP_NEAREST_BRIDGE: + val = LLDP_NEAREST_BRIDGE; + break; + case ECORE_LLDP_NEAREST_NON_TPMR_BRIDGE: + val = LLDP_NEAREST_NON_TPMR_BRIDGE; + break; + case ECORE_LLDP_NEAREST_CUSTOMER_BRIDGE: + val = LLDP_NEAREST_CUSTOMER_BRIDGE; + break; + default: + DP_ERR(p_hwfn, "Invalid agent type %d\n", p_params->agent); + return ECORE_INVAL; + } + + SET_MFW_FIELD(mb_param, DRV_MB_PARAM_LLDP_AGENT, val); + addr = p_hwfn->mcp_info->port_addr + + offsetof(struct public_port, lldp_config_params[val]); + + OSAL_MEMSET(&lldp_params, 0, sizeof(lldp_params)); + SET_MFW_FIELD(lldp_params.config, LLDP_CONFIG_TX_INTERVAL, + p_params->tx_interval); + SET_MFW_FIELD(lldp_params.config, LLDP_CONFIG_HOLD, p_params->tx_hold); + SET_MFW_FIELD(lldp_params.config, LLDP_CONFIG_MAX_CREDIT, + p_params->tx_credit); + SET_MFW_FIELD(lldp_params.config, LLDP_CONFIG_ENABLE_RX, + !!p_params->rx_enable); + SET_MFW_FIELD(lldp_params.config, LLDP_CONFIG_ENABLE_TX, + !!p_params->tx_enable); + + for (i = 0; i < ECORE_LLDP_CHASSIS_ID_STAT_LEN; i++) + p_params->chassis_id_tlv[i] = + OSAL_CPU_TO_BE32(p_params->chassis_id_tlv[i]); + OSAL_MEMCPY(lldp_params.local_chassis_id, p_params->chassis_id_tlv, + sizeof(lldp_params.local_chassis_id)); + + for (i = 0; i < ECORE_LLDP_PORT_ID_STAT_LEN; i++) + p_params->port_id_tlv[i] = + OSAL_CPU_TO_BE32(p_params->port_id_tlv[i]); + OSAL_MEMCPY(lldp_params.local_port_id, p_params->port_id_tlv, + sizeof(lldp_params.local_port_id)); + + ecore_memcpy_to(p_hwfn, p_ptt, addr, &lldp_params, sizeof(lldp_params)); + + rc = ecore_mcp_cmd(p_hwfn, p_ptt, DRV_MSG_CODE_SET_LLDP, + mb_param, &mcp_resp, &mcp_param); + if (rc != ECORE_SUCCESS) + DP_NOTICE(p_hwfn, false, "SET_LLDP failed, error = %d\n", rc); + + return rc; +} + +enum _ecore_status_t +ecore_lldp_set_system_tlvs(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, + struct ecore_lldp_sys_tlvs *p_params) +{ + u32 mb_param = 0, mcp_resp = 0, mcp_param = 0; + enum _ecore_status_t rc = ECORE_SUCCESS; + struct lldp_system_tlvs_buffer_s lld_tlv_buf; + u32 addr, *p_val; + u8 len; + int i; + + p_val = (u32 *)p_params->buf; + for (i = 0; i < ECORE_LLDP_SYS_TLV_SIZE / 4; i++) + p_val[i] = OSAL_CPU_TO_BE32(p_val[i]); + + OSAL_MEMSET(&lld_tlv_buf, 0, sizeof(lld_tlv_buf)); + SET_MFW_FIELD(lld_tlv_buf.flags, LLDP_SYSTEM_TLV_VALID, 1); + SET_MFW_FIELD(lld_tlv_buf.flags, LLDP_SYSTEM_TLV_MANDATORY, + !!p_params->discard_mandatory_tlv); + SET_MFW_FIELD(lld_tlv_buf.flags, LLDP_SYSTEM_TLV_LENGTH, + p_params->buf_size); + len = ECORE_LLDP_SYS_TLV_SIZE / 2; + OSAL_MEMCPY(lld_tlv_buf.data, p_params->buf, len); + + addr = p_hwfn->mcp_info->port_addr + + offsetof(struct public_port, system_lldp_tlvs_buf); + ecore_memcpy_to(p_hwfn, p_ptt, addr, &lld_tlv_buf, sizeof(lld_tlv_buf)); + + if (p_params->buf_size > len) { + addr = p_hwfn->mcp_info->port_addr + + offsetof(struct public_port, system_lldp_tlvs_buf2); + ecore_memcpy_to(p_hwfn, p_ptt, addr, &p_params->buf[len], + ECORE_LLDP_SYS_TLV_SIZE / 2); + } + + rc = ecore_mcp_cmd(p_hwfn, p_ptt, DRV_MSG_CODE_SET_LLDP, + mb_param, &mcp_resp, &mcp_param); + if (rc != ECORE_SUCCESS) + DP_NOTICE(p_hwfn, false, "SET_LLDP failed, error = %d\n", rc); return rc; } diff --git a/dpdk/drivers/net/qede/base/ecore_dcbx.h b/dpdk/drivers/net/qede/base/ecore_dcbx.h index d577f4e1..469e42dd 100644 --- a/dpdk/drivers/net/qede/base/ecore_dcbx.h +++ b/dpdk/drivers/net/qede/base/ecore_dcbx.h @@ -17,39 +17,48 @@ #include "ecore_hsi_common.h" #include "ecore_dcbx_api.h" -#define ECORE_MFW_GET_FIELD(name, field) \ - (((name) & (field ## _MASK)) >> (field ## _SHIFT)) +#define ECORE_DCBX_DSCP_DISABLED 0XFF struct ecore_dcbx_info { struct lldp_status_params_s lldp_remote[LLDP_MAX_LLDP_AGENTS]; struct lldp_config_params_s lldp_local[LLDP_MAX_LLDP_AGENTS]; struct dcbx_local_params local_admin; struct ecore_dcbx_results results; + struct dcb_dscp_map dscp_map; + bool dscp_nig_update; struct dcbx_mib operational; struct dcbx_mib remote; struct ecore_dcbx_set set; struct ecore_dcbx_get get; u8 dcbx_cap; + u16 iwarp_port; }; -/* Upper layer driver interface routines */ -enum _ecore_status_t ecore_dcbx_config_params(struct ecore_hwfn *, - struct ecore_ptt *, - struct ecore_dcbx_set *); +struct ecore_dcbx_mib_meta_data { + struct lldp_config_params_s *lldp_local; + struct lldp_status_params_s *lldp_remote; + struct lldp_received_tlvs_s *lldp_tlvs; + struct dcbx_local_params *local_admin; + struct dcb_dscp_map *dscp_map; + struct dcbx_mib *mib; + osal_size_t size; + u32 addr; +}; /* ECORE local interface routines */ enum _ecore_status_t ecore_dcbx_mib_update_event(struct ecore_hwfn *, struct ecore_ptt *, enum ecore_mib_read_type); -enum _ecore_status_t ecore_dcbx_read_lldp_params(struct ecore_hwfn *, - struct ecore_ptt *); enum _ecore_status_t ecore_dcbx_info_alloc(struct ecore_hwfn *p_hwfn); -void ecore_dcbx_info_free(struct ecore_hwfn *, struct ecore_dcbx_info *); +void ecore_dcbx_info_free(struct ecore_hwfn *p_hwfn); void ecore_dcbx_set_pf_update_params(struct ecore_dcbx_results *p_src, struct pf_update_ramrod_data *p_dest); -/* @@@TBD eagle phy workaround */ -void ecore_dcbx_eagle_workaround(struct ecore_hwfn *, struct ecore_ptt *, - bool set_to_pfc); + +/* Returns TOS value for a given priority */ +u8 ecore_dcbx_get_dscp_value(struct ecore_hwfn *p_hwfn, u8 pri); + +enum _ecore_status_t +ecore_lldp_mib_update_event(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt); #endif /* __ECORE_DCBX_H__ */ diff --git a/dpdk/drivers/net/qede/base/ecore_dcbx_api.h b/dpdk/drivers/net/qede/base/ecore_dcbx_api.h index 7767d48e..9ff4df4c 100644 --- a/dpdk/drivers/net/qede/base/ecore_dcbx_api.h +++ b/dpdk/drivers/net/qede/base/ecore_dcbx_api.h @@ -9,7 +9,7 @@ #ifndef __ECORE_DCBX_API_H__ #define __ECORE_DCBX_API_H__ -#include "ecore.h" +#include "ecore_status.h" #define DCBX_CONFIG_MAX_APP_PROTOCOL 4 @@ -18,72 +18,95 @@ enum ecore_mib_read_type { ECORE_DCBX_REMOTE_MIB, ECORE_DCBX_LOCAL_MIB, ECORE_DCBX_REMOTE_LLDP_MIB, - ECORE_DCBX_LOCAL_LLDP_MIB + ECORE_DCBX_LOCAL_LLDP_MIB, + ECORE_DCBX_LLDP_TLVS }; struct ecore_dcbx_app_data { bool enable; /* DCB enabled */ - bool update; /* Update indication */ + u8 update; /* Update indication */ u8 priority; /* Priority */ u8 tc; /* Traffic Class */ + bool dscp_enable; /* DSCP enabled */ + u8 dscp_val; /* DSCP value */ }; #ifndef __EXTRACT__LINUX__ enum dcbx_protocol_type { + DCBX_PROTOCOL_ISCSI, + DCBX_PROTOCOL_FCOE, + DCBX_PROTOCOL_ROCE, + DCBX_PROTOCOL_ROCE_V2, DCBX_PROTOCOL_ETH, + DCBX_PROTOCOL_IWARP, DCBX_MAX_PROTOCOL_TYPE }; -#ifdef LINUX_REMOVE -/* We can't assume THE HSI values are available to clients, so we need - * to redefine those here. - */ -#ifndef LLDP_CHASSIS_ID_STAT_LEN -#define LLDP_CHASSIS_ID_STAT_LEN 4 -#endif -#ifndef LLDP_PORT_ID_STAT_LEN -#define LLDP_PORT_ID_STAT_LEN 4 -#endif -#ifndef DCBX_MAX_APP_PROTOCOL -#define DCBX_MAX_APP_PROTOCOL 32 -#endif - -#endif +#define ECORE_LLDP_CHASSIS_ID_STAT_LEN 4 +#define ECORE_LLDP_PORT_ID_STAT_LEN 4 +#define ECORE_DCBX_MAX_APP_PROTOCOL 32 +#define ECORE_MAX_PFC_PRIORITIES 8 +#define ECORE_DCBX_DSCP_SIZE 64 struct ecore_dcbx_lldp_remote { - u32 peer_chassis_id[LLDP_CHASSIS_ID_STAT_LEN]; - u32 peer_port_id[LLDP_PORT_ID_STAT_LEN]; - bool enable_rx; - bool enable_tx; - u32 tx_interval; - u32 max_credit; + u32 peer_chassis_id[ECORE_LLDP_CHASSIS_ID_STAT_LEN]; + u32 peer_port_id[ECORE_LLDP_PORT_ID_STAT_LEN]; + bool enable_rx; + bool enable_tx; + u32 tx_interval; + u32 max_credit; }; struct ecore_dcbx_lldp_local { - u32 local_chassis_id[LLDP_CHASSIS_ID_STAT_LEN]; - u32 local_port_id[LLDP_PORT_ID_STAT_LEN]; + u32 local_chassis_id[ECORE_LLDP_CHASSIS_ID_STAT_LEN]; + u32 local_port_id[ECORE_LLDP_PORT_ID_STAT_LEN]; }; struct ecore_dcbx_app_prio { - u8 eth; + u8 roce; + u8 roce_v2; + u8 fcoe; + u8 iscsi; + u8 eth; +}; + +struct ecore_dbcx_pfc_params { + bool willing; + bool enabled; + u8 prio[ECORE_MAX_PFC_PRIORITIES]; + u8 max_tc; +}; + +enum ecore_dcbx_sf_ieee_type { + ECORE_DCBX_SF_IEEE_ETHTYPE, + ECORE_DCBX_SF_IEEE_TCP_PORT, + ECORE_DCBX_SF_IEEE_UDP_PORT, + ECORE_DCBX_SF_IEEE_TCP_UDP_PORT +}; + +struct ecore_app_entry { + bool ethtype; + enum ecore_dcbx_sf_ieee_type sf_ieee; + bool enabled; + u8 prio; + u16 proto_id; + enum dcbx_protocol_type proto_type; }; struct ecore_dcbx_params { - u32 app_bitmap[DCBX_MAX_APP_PROTOCOL]; - u16 num_app_entries; - bool app_willing; - bool app_valid; - bool ets_willing; - bool ets_enabled; - bool valid; /* Indicate validity of params */ - u32 ets_pri_tc_tbl[1]; - u32 ets_tc_bw_tbl[2]; - u32 ets_tc_tsa_tbl[2]; - bool pfc_willing; - bool pfc_enabled; - u32 pfc_bitmap; - u8 max_pfc_tc; - u8 max_ets_tc; + struct ecore_app_entry app_entry[ECORE_DCBX_MAX_APP_PROTOCOL]; + u16 num_app_entries; + bool app_willing; + bool app_valid; + bool app_error; + bool ets_willing; + bool ets_enabled; + bool ets_cbs; + u8 ets_pri_tc_tbl[ECORE_MAX_PFC_PRIORITIES]; + u8 ets_tc_bw_tbl[ECORE_MAX_PFC_PRIORITIES]; + u8 ets_tc_tsa_tbl[ECORE_MAX_PFC_PRIORITIES]; + struct ecore_dbcx_pfc_params pfc; + u8 max_ets_tc; }; struct ecore_dcbx_admin_params { @@ -103,22 +126,41 @@ struct ecore_dcbx_operational_params { bool enabled; bool ieee; bool cee; + bool local; u32 err; }; +struct ecore_dcbx_dscp_params { + bool enabled; + u8 dscp_pri_map[ECORE_DCBX_DSCP_SIZE]; +}; + struct ecore_dcbx_get { struct ecore_dcbx_operational_params operational; struct ecore_dcbx_lldp_remote lldp_remote; struct ecore_dcbx_lldp_local lldp_local; struct ecore_dcbx_remote_params remote; struct ecore_dcbx_admin_params local; + struct ecore_dcbx_dscp_params dscp; }; #endif +#define ECORE_DCBX_VERSION_DISABLED 0 +#define ECORE_DCBX_VERSION_IEEE 1 +#define ECORE_DCBX_VERSION_CEE 2 +#define ECORE_DCBX_VERSION_DYNAMIC 3 + struct ecore_dcbx_set { - struct ecore_dcbx_admin_params config; +#define ECORE_DCBX_OVERRIDE_STATE (1 << 0) +#define ECORE_DCBX_OVERRIDE_PFC_CFG (1 << 1) +#define ECORE_DCBX_OVERRIDE_ETS_CFG (1 << 2) +#define ECORE_DCBX_OVERRIDE_APP_CFG (1 << 3) +#define ECORE_DCBX_OVERRIDE_DSCP_CFG (1 << 4) + u32 override_flags; bool enabled; + struct ecore_dcbx_admin_params config; u32 ver_num; + struct ecore_dcbx_dscp_params dscp; }; struct ecore_dcbx_results { @@ -129,32 +171,71 @@ struct ecore_dcbx_results { struct ecore_dcbx_app_metadata { enum dcbx_protocol_type id; - const char *name; /* @DPDK */ + const char *name; /* @DPDK */ enum ecore_pci_personality personality; }; -struct ecore_dcbx_mib_meta_data { - struct lldp_config_params_s *lldp_local; - struct lldp_status_params_s *lldp_remote; - struct dcbx_local_params *local_admin; - struct dcbx_mib *mib; - osal_size_t size; - u32 addr; +enum ecore_lldp_agent { + ECORE_LLDP_NEAREST_BRIDGE = 0, + ECORE_LLDP_NEAREST_NON_TPMR_BRIDGE, + ECORE_LLDP_NEAREST_CUSTOMER_BRIDGE, + ECORE_LLDP_MAX_AGENTS }; -void -ecore_dcbx_set_params(struct ecore_dcbx_results *p_data, - struct ecore_hw_info *p_info, - bool enable, bool update, u8 prio, u8 tc, - enum dcbx_protocol_type type, - enum ecore_pci_personality personality); +struct ecore_lldp_config_params { + enum ecore_lldp_agent agent; + u8 tx_interval; + u8 tx_hold; + u8 tx_credit; + bool rx_enable; + bool tx_enable; + u32 chassis_id_tlv[ECORE_LLDP_CHASSIS_ID_STAT_LEN]; + u32 port_id_tlv[ECORE_LLDP_PORT_ID_STAT_LEN]; +}; + +#define ECORE_LLDP_SYS_TLV_SIZE 256 +struct ecore_lldp_sys_tlvs { + bool discard_mandatory_tlv; + u8 buf[ECORE_LLDP_SYS_TLV_SIZE]; + u16 buf_size; +}; enum _ecore_status_t ecore_dcbx_query_params(struct ecore_hwfn *, struct ecore_dcbx_get *, enum ecore_mib_read_type); +enum _ecore_status_t ecore_dcbx_get_config_params(struct ecore_hwfn *, + struct ecore_dcbx_set *); + +enum _ecore_status_t ecore_dcbx_config_params(struct ecore_hwfn *, + struct ecore_ptt *, + struct ecore_dcbx_set *, + bool); + +enum _ecore_status_t ecore_lldp_register_tlv(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + enum ecore_lldp_agent agent, + u8 tlv_type); + +enum _ecore_status_t +ecore_lldp_get_params(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, + struct ecore_lldp_config_params *p_params); + +enum _ecore_status_t +ecore_lldp_set_params(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, + struct ecore_lldp_config_params *p_params); + +enum _ecore_status_t +ecore_lldp_set_system_tlvs(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, + struct ecore_lldp_sys_tlvs *p_params); + static const struct ecore_dcbx_app_metadata ecore_dcbx_app_update[] = { - {DCBX_PROTOCOL_ETH, "ETH", ECORE_PCI_ETH} + {DCBX_PROTOCOL_ISCSI, "ISCSI", ECORE_PCI_ISCSI}, + {DCBX_PROTOCOL_FCOE, "FCOE", ECORE_PCI_FCOE}, + {DCBX_PROTOCOL_ROCE, "ROCE", ECORE_PCI_ETH_ROCE}, + {DCBX_PROTOCOL_ROCE_V2, "ROCE_V2", ECORE_PCI_ETH_ROCE}, + {DCBX_PROTOCOL_ETH, "ETH", ECORE_PCI_ETH}, + {DCBX_PROTOCOL_IWARP, "IWARP", ECORE_PCI_ETH_IWARP} }; #endif /* __ECORE_DCBX_API_H__ */ diff --git a/dpdk/drivers/net/qede/base/ecore_dev.c b/dpdk/drivers/net/qede/base/ecore_dev.c index 0a689694..da1830ce 100644 --- a/dpdk/drivers/net/qede/base/ecore_dev.c +++ b/dpdk/drivers/net/qede/base/ecore_dev.c @@ -28,49 +28,370 @@ #include "mcp_public.h" #include "ecore_iro.h" #include "nvm_cfg.h" -#include "ecore_dev_api.h" -#include "ecore_attn_values.h" #include "ecore_dcbx.h" +#include "ecore_l2.h" + +/* TODO - there's a bug in DCBx re-configuration flows in MF, as the QM + * registers involved are not split and thus configuration is a race where + * some of the PFs configuration might be lost. + * Eventually, this needs to move into a MFW-covered HW-lock as arbitration + * mechanism as this doesn't cover some cases [E.g., PDA or scenarios where + * there's more than a single compiled ecore component in system]. + */ +static osal_spinlock_t qm_lock; +static bool qm_lock_init; + +/******************** Doorbell Recovery *******************/ +/* The doorbell recovery mechanism consists of a list of entries which represent + * doorbelling entities (l2 queues, roce sq/rq/cqs, the slowpath spq, etc). Each + * entity needs to register with the mechanism and provide the parameters + * describing it's doorbell, including a location where last used doorbell data + * can be found. The doorbell execute function will traverse the list and + * doorbell all of the registered entries. + */ +struct ecore_db_recovery_entry { + osal_list_entry_t list_entry; + void OSAL_IOMEM *db_addr; + void *db_data; + enum ecore_db_rec_width db_width; + enum ecore_db_rec_space db_space; + u8 hwfn_idx; +}; + +/* display a single doorbell recovery entry */ +void ecore_db_recovery_dp_entry(struct ecore_hwfn *p_hwfn, + struct ecore_db_recovery_entry *db_entry, + const char *action) +{ + DP_VERBOSE(p_hwfn, ECORE_MSG_SPQ, "(%s: db_entry %p, addr %p, data %p, width %s, %s space, hwfn %d)\n", + action, db_entry, db_entry->db_addr, db_entry->db_data, + db_entry->db_width == DB_REC_WIDTH_32B ? "32b" : "64b", + db_entry->db_space == DB_REC_USER ? "user" : "kernel", + db_entry->hwfn_idx); +} + +/* doorbell address sanity (address within doorbell bar range) */ +bool ecore_db_rec_sanity(struct ecore_dev *p_dev, void OSAL_IOMEM *db_addr, + void *db_data) +{ + /* make sure doorbell address is within the doorbell bar */ + if (db_addr < p_dev->doorbells || (u8 *)db_addr > + (u8 *)p_dev->doorbells + p_dev->db_size) { + OSAL_WARN(true, + "Illegal doorbell address: %p. Legal range for doorbell addresses is [%p..%p]\n", + db_addr, p_dev->doorbells, + (u8 *)p_dev->doorbells + p_dev->db_size); + return false; + } + + /* make sure doorbell data pointer is not null */ + if (!db_data) { + OSAL_WARN(true, "Illegal doorbell data pointer: %p", db_data); + return false; + } + + return true; +} + +/* find hwfn according to the doorbell address */ +struct ecore_hwfn *ecore_db_rec_find_hwfn(struct ecore_dev *p_dev, + void OSAL_IOMEM *db_addr) +{ + struct ecore_hwfn *p_hwfn; + + /* In CMT doorbell bar is split down the middle between engine 0 and + * enigne 1 + */ + if (ECORE_IS_CMT(p_dev)) + p_hwfn = db_addr < p_dev->hwfns[1].doorbells ? + &p_dev->hwfns[0] : &p_dev->hwfns[1]; + else + p_hwfn = ECORE_LEADING_HWFN(p_dev); + + return p_hwfn; +} + +/* add a new entry to the doorbell recovery mechanism */ +enum _ecore_status_t ecore_db_recovery_add(struct ecore_dev *p_dev, + void OSAL_IOMEM *db_addr, + void *db_data, + enum ecore_db_rec_width db_width, + enum ecore_db_rec_space db_space) +{ + struct ecore_db_recovery_entry *db_entry; + struct ecore_hwfn *p_hwfn; + + /* shortcircuit VFs, for now */ + if (IS_VF(p_dev)) { + DP_VERBOSE(p_dev, ECORE_MSG_IOV, "db recovery - skipping VF doorbell\n"); + return ECORE_SUCCESS; + } + + /* sanitize doorbell address */ + if (!ecore_db_rec_sanity(p_dev, db_addr, db_data)) + return ECORE_INVAL; + + /* obtain hwfn from doorbell address */ + p_hwfn = ecore_db_rec_find_hwfn(p_dev, db_addr); + + /* create entry */ + db_entry = OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL, sizeof(*db_entry)); + if (!db_entry) { + DP_NOTICE(p_dev, false, "Failed to allocate a db recovery entry\n"); + return ECORE_NOMEM; + } + + /* populate entry */ + db_entry->db_addr = db_addr; + db_entry->db_data = db_data; + db_entry->db_width = db_width; + db_entry->db_space = db_space; + db_entry->hwfn_idx = p_hwfn->my_id; + + /* display */ + ecore_db_recovery_dp_entry(p_hwfn, db_entry, "Adding"); + + /* protect the list */ + OSAL_SPIN_LOCK(&p_hwfn->db_recovery_info.lock); + OSAL_LIST_PUSH_TAIL(&db_entry->list_entry, + &p_hwfn->db_recovery_info.list); + OSAL_SPIN_UNLOCK(&p_hwfn->db_recovery_info.lock); + + return ECORE_SUCCESS; +} + +/* remove an entry from the doorbell recovery mechanism */ +enum _ecore_status_t ecore_db_recovery_del(struct ecore_dev *p_dev, + void OSAL_IOMEM *db_addr, + void *db_data) +{ + struct ecore_db_recovery_entry *db_entry = OSAL_NULL; + enum _ecore_status_t rc = ECORE_INVAL; + struct ecore_hwfn *p_hwfn; + + /* shortcircuit VFs, for now */ + if (IS_VF(p_dev)) { + DP_VERBOSE(p_dev, ECORE_MSG_IOV, "db recovery - skipping VF doorbell\n"); + return ECORE_SUCCESS; + } + + /* sanitize doorbell address */ + if (!ecore_db_rec_sanity(p_dev, db_addr, db_data)) + return ECORE_INVAL; + + /* obtain hwfn from doorbell address */ + p_hwfn = ecore_db_rec_find_hwfn(p_dev, db_addr); + + /* protect the list */ + OSAL_SPIN_LOCK(&p_hwfn->db_recovery_info.lock); + OSAL_LIST_FOR_EACH_ENTRY(db_entry, + &p_hwfn->db_recovery_info.list, + list_entry, + struct ecore_db_recovery_entry) { + /* search according to db_data addr since db_addr is not unique + * (roce) + */ + if (db_entry->db_data == db_data) { + ecore_db_recovery_dp_entry(p_hwfn, db_entry, + "Deleting"); + OSAL_LIST_REMOVE_ENTRY(&db_entry->list_entry, + &p_hwfn->db_recovery_info.list); + rc = ECORE_SUCCESS; + break; + } + } + + OSAL_SPIN_UNLOCK(&p_hwfn->db_recovery_info.lock); + + if (rc == ECORE_INVAL) + /*OSAL_WARN(true,*/ + DP_NOTICE(p_hwfn, false, + "Failed to find element in list. Key (db_data addr) was %p. db_addr was %p\n", + db_data, db_addr); + else + OSAL_FREE(p_dev, db_entry); + + return rc; +} + +/* initialize the doorbell recovery mechanism */ +enum _ecore_status_t ecore_db_recovery_setup(struct ecore_hwfn *p_hwfn) +{ + DP_VERBOSE(p_hwfn, ECORE_MSG_SPQ, "Setting up db recovery\n"); + + /* make sure db_size was set in p_dev */ + if (!p_hwfn->p_dev->db_size) { + DP_ERR(p_hwfn->p_dev, "db_size not set\n"); + return ECORE_INVAL; + } + + OSAL_LIST_INIT(&p_hwfn->db_recovery_info.list); +#ifdef CONFIG_ECORE_LOCK_ALLOC + OSAL_SPIN_LOCK_ALLOC(p_hwfn, &p_hwfn->db_recovery_info.lock); +#endif + OSAL_SPIN_LOCK_INIT(&p_hwfn->db_recovery_info.lock); + p_hwfn->db_recovery_info.db_recovery_counter = 0; + + return ECORE_SUCCESS; +} + +/* destroy the doorbell recovery mechanism */ +void ecore_db_recovery_teardown(struct ecore_hwfn *p_hwfn) +{ + struct ecore_db_recovery_entry *db_entry = OSAL_NULL; + + DP_VERBOSE(p_hwfn, ECORE_MSG_SPQ, "Tearing down db recovery\n"); + if (!OSAL_LIST_IS_EMPTY(&p_hwfn->db_recovery_info.list)) { + DP_VERBOSE(p_hwfn, false, "Doorbell Recovery teardown found the doorbell recovery list was not empty (Expected in disorderly driver unload (e.g. recovery) otherwise this probably means some flow forgot to db_recovery_del). Prepare to purge doorbell recovery list...\n"); + while (!OSAL_LIST_IS_EMPTY(&p_hwfn->db_recovery_info.list)) { + db_entry = OSAL_LIST_FIRST_ENTRY( + &p_hwfn->db_recovery_info.list, + struct ecore_db_recovery_entry, + list_entry); + ecore_db_recovery_dp_entry(p_hwfn, db_entry, "Purging"); + OSAL_LIST_REMOVE_ENTRY(&db_entry->list_entry, + &p_hwfn->db_recovery_info.list); + OSAL_FREE(p_hwfn->p_dev, db_entry); + } + } +#ifdef CONFIG_ECORE_LOCK_ALLOC + OSAL_SPIN_LOCK_DEALLOC(&p_hwfn->db_recovery_info.lock); +#endif + p_hwfn->db_recovery_info.db_recovery_counter = 0; +} + +/* print the content of the doorbell recovery mechanism */ +void ecore_db_recovery_dp(struct ecore_hwfn *p_hwfn) +{ + struct ecore_db_recovery_entry *db_entry = OSAL_NULL; + + DP_NOTICE(p_hwfn, false, + "Dispalying doorbell recovery database. Counter was %d\n", + p_hwfn->db_recovery_info.db_recovery_counter); + + /* protect the list */ + OSAL_SPIN_LOCK(&p_hwfn->db_recovery_info.lock); + OSAL_LIST_FOR_EACH_ENTRY(db_entry, + &p_hwfn->db_recovery_info.list, + list_entry, + struct ecore_db_recovery_entry) { + ecore_db_recovery_dp_entry(p_hwfn, db_entry, "Printing"); + } + + OSAL_SPIN_UNLOCK(&p_hwfn->db_recovery_info.lock); +} + +/* ring the doorbell of a single doorbell recovery entry */ +void ecore_db_recovery_ring(struct ecore_hwfn *p_hwfn, + struct ecore_db_recovery_entry *db_entry, + enum ecore_db_rec_exec db_exec) +{ + /* Print according to width */ + if (db_entry->db_width == DB_REC_WIDTH_32B) + DP_VERBOSE(p_hwfn, ECORE_MSG_SPQ, "%s doorbell address %p data %x\n", + db_exec == DB_REC_DRY_RUN ? "would have rung" : "ringing", + db_entry->db_addr, *(u32 *)db_entry->db_data); + else + DP_VERBOSE(p_hwfn, ECORE_MSG_SPQ, "%s doorbell address %p data %lx\n", + db_exec == DB_REC_DRY_RUN ? "would have rung" : "ringing", + db_entry->db_addr, + *(unsigned long *)(db_entry->db_data)); + + /* Sanity */ + if (!ecore_db_rec_sanity(p_hwfn->p_dev, db_entry->db_addr, + db_entry->db_data)) + return; + + /* Flush the write combined buffer. Since there are multiple doorbelling + * entities using the same address, if we don't flush, a transaction + * could be lost. + */ + OSAL_WMB(p_hwfn->p_dev); + + /* Ring the doorbell */ + if (db_exec == DB_REC_REAL_DEAL || db_exec == DB_REC_ONCE) { + if (db_entry->db_width == DB_REC_WIDTH_32B) + DIRECT_REG_WR(p_hwfn, db_entry->db_addr, + *(u32 *)(db_entry->db_data)); + else + DIRECT_REG_WR64(p_hwfn, db_entry->db_addr, + *(u64 *)(db_entry->db_data)); + } + + /* Flush the write combined buffer. Next doorbell may come from a + * different entity to the same address... + */ + OSAL_WMB(p_hwfn->p_dev); +} + +/* traverse the doorbell recovery entry list and ring all the doorbells */ +void ecore_db_recovery_execute(struct ecore_hwfn *p_hwfn, + enum ecore_db_rec_exec db_exec) +{ + struct ecore_db_recovery_entry *db_entry = OSAL_NULL; + + if (db_exec != DB_REC_ONCE) { + DP_NOTICE(p_hwfn, false, "Executing doorbell recovery. Counter was %d\n", + p_hwfn->db_recovery_info.db_recovery_counter); + + /* track amount of times recovery was executed */ + p_hwfn->db_recovery_info.db_recovery_counter++; + } + + /* protect the list */ + OSAL_SPIN_LOCK(&p_hwfn->db_recovery_info.lock); + OSAL_LIST_FOR_EACH_ENTRY(db_entry, + &p_hwfn->db_recovery_info.list, + list_entry, + struct ecore_db_recovery_entry) { + ecore_db_recovery_ring(p_hwfn, db_entry, db_exec); + if (db_exec == DB_REC_ONCE) + break; + } + + OSAL_SPIN_UNLOCK(&p_hwfn->db_recovery_info.lock); +} +/******************** Doorbell Recovery end ****************/ /* Configurable */ -#define ECORE_MIN_DPIS (4) /* The minimal number of DPIs required +#define ECORE_MIN_DPIS (4) /* The minimal num of DPIs required to * load the driver. The number was * arbitrarily set. */ /* Derived */ -#define ECORE_MIN_PWM_REGION ((ECORE_WID_SIZE) * (ECORE_MIN_DPIS)) +#define ECORE_MIN_PWM_REGION (ECORE_WID_SIZE * ECORE_MIN_DPIS) -enum BAR_ID { - BAR_ID_0, /* used for GRC */ - BAR_ID_1 /* Used for doorbells */ -}; - -static u32 ecore_hw_bar_size(struct ecore_hwfn *p_hwfn, enum BAR_ID bar_id) +static u32 ecore_hw_bar_size(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + enum BAR_ID bar_id) { u32 bar_reg = (bar_id == BAR_ID_0 ? PGLUE_B_REG_PF_BAR0_SIZE : PGLUE_B_REG_PF_BAR1_SIZE); - u32 val = ecore_rd(p_hwfn, p_hwfn->p_main_ptt, bar_reg); + u32 val; + + if (IS_VF(p_hwfn->p_dev)) + return ecore_vf_hw_bar_size(p_hwfn, bar_id); + + val = ecore_rd(p_hwfn, p_ptt, bar_reg); + if (val) + return 1 << (val + 15); /* The above registers were updated in the past only in CMT mode. Since * they were found to be useful MFW started updating them from 8.7.7.0. * In older MFW versions they are set to 0 which means disabled. */ - if (!val) { - if (p_hwfn->p_dev->num_hwfns > 1) { - DP_NOTICE(p_hwfn, false, - "BAR size not configured. Assuming BAR" - " size of 256kB for GRC and 512kB for DB\n"); - return BAR_ID_0 ? 256 * 1024 : 512 * 1024; - } - - DP_NOTICE(p_hwfn, false, - "BAR size not configured. Assuming BAR" - " size of 512kB for GRC and 512kB for DB\n"); - return 512 * 1024; + if (ECORE_IS_CMT(p_hwfn->p_dev)) { + DP_INFO(p_hwfn, + "BAR size not configured. Assuming BAR size of 256kB for GRC and 512kB for DB\n"); + val = BAR_ID_0 ? 256 * 1024 : 512 * 1024; + } else { + DP_INFO(p_hwfn, + "BAR size not configured. Assuming BAR size of 512kB for GRC and 512kB for DB\n"); + val = 512 * 1024; } - return 1 << (val + 15); + return val; } void ecore_init_dp(struct ecore_dev *p_dev, @@ -101,7 +422,9 @@ void ecore_init_struct(struct ecore_dev *p_dev) p_hwfn->my_id = i; p_hwfn->b_active = false; +#ifdef CONFIG_ECORE_LOCK_ALLOC OSAL_MUTEX_ALLOC(p_hwfn, &p_hwfn->dmae_info.mutex); +#endif OSAL_MUTEX_INIT(&p_hwfn->dmae_info.mutex); } @@ -117,211 +440,623 @@ static void ecore_qm_info_free(struct ecore_hwfn *p_hwfn) struct ecore_qm_info *qm_info = &p_hwfn->qm_info; OSAL_FREE(p_hwfn->p_dev, qm_info->qm_pq_params); - qm_info->qm_pq_params = OSAL_NULL; OSAL_FREE(p_hwfn->p_dev, qm_info->qm_vport_params); - qm_info->qm_vport_params = OSAL_NULL; OSAL_FREE(p_hwfn->p_dev, qm_info->qm_port_params); - qm_info->qm_port_params = OSAL_NULL; OSAL_FREE(p_hwfn->p_dev, qm_info->wfq_data); - qm_info->wfq_data = OSAL_NULL; } void ecore_resc_free(struct ecore_dev *p_dev) { int i; - if (IS_VF(p_dev)) + if (IS_VF(p_dev)) { + for_each_hwfn(p_dev, i) + ecore_l2_free(&p_dev->hwfns[i]); return; + } OSAL_FREE(p_dev, p_dev->fw_data); - p_dev->fw_data = OSAL_NULL; OSAL_FREE(p_dev, p_dev->reset_stats); - for_each_hwfn(p_dev, i) { - struct ecore_hwfn *p_hwfn = &p_dev->hwfns[i]; - - OSAL_FREE(p_dev, p_hwfn->p_tx_cids); - p_hwfn->p_tx_cids = OSAL_NULL; - OSAL_FREE(p_dev, p_hwfn->p_rx_cids); - p_hwfn->p_rx_cids = OSAL_NULL; - } - for_each_hwfn(p_dev, i) { struct ecore_hwfn *p_hwfn = &p_dev->hwfns[i]; ecore_cxt_mngr_free(p_hwfn); ecore_qm_info_free(p_hwfn); ecore_spq_free(p_hwfn); - ecore_eq_free(p_hwfn, p_hwfn->p_eq); - ecore_consq_free(p_hwfn, p_hwfn->p_consq); + ecore_eq_free(p_hwfn); + ecore_consq_free(p_hwfn); ecore_int_free(p_hwfn); ecore_iov_free(p_hwfn); + ecore_l2_free(p_hwfn); ecore_dmae_info_free(p_hwfn); - ecore_dcbx_info_free(p_hwfn, p_hwfn->p_dcbx_info); + ecore_dcbx_info_free(p_hwfn); /* @@@TBD Flush work-queue ? */ + + /* destroy doorbell recovery mechanism */ + ecore_db_recovery_teardown(p_hwfn); } } -static enum _ecore_status_t ecore_init_qm_info(struct ecore_hwfn *p_hwfn, - bool b_sleepable) +/******************** QM initialization *******************/ + +/* bitmaps for indicating active traffic classes. + * Special case for Arrowhead 4 port + */ +/* 0..3 actualy used, 4 serves OOO, 7 serves high priority stuff (e.g. DCQCN) */ +#define ACTIVE_TCS_BMAP 0x9f +/* 0..3 actually used, OOO and high priority stuff all use 3 */ +#define ACTIVE_TCS_BMAP_4PORT_K2 0xf + +/* determines the physical queue flags for a given PF. */ +static u32 ecore_get_pq_flags(struct ecore_hwfn *p_hwfn) { - u8 num_vports, vf_offset = 0, i, vport_id, num_ports; - struct ecore_qm_info *qm_info = &p_hwfn->qm_info; - struct init_qm_port_params *p_qm_port; - u16 num_pqs, multi_cos_tcs = 1; -#ifdef CONFIG_ECORE_SRIOV - u16 num_vfs = p_hwfn->p_dev->sriov_info.total_vfs; -#else - u16 num_vfs = 0; -#endif + u32 flags; - OSAL_MEM_ZERO(qm_info, sizeof(*qm_info)); + /* common flags */ + flags = PQ_FLAGS_LB; -#ifndef ASIC_ONLY - /* @TMP - Don't allocate QM queues for VFs on emulation */ - if (CHIP_REV_IS_EMUL(p_hwfn->p_dev)) { - DP_NOTICE(p_hwfn, false, - "Emulation - skip configuring QM queues for VFs\n"); - num_vfs = 0; + /* feature flags */ + if (IS_ECORE_SRIOV(p_hwfn->p_dev)) + flags |= PQ_FLAGS_VFS; + + /* protocol flags */ + switch (p_hwfn->hw_info.personality) { + case ECORE_PCI_ETH: + flags |= PQ_FLAGS_MCOS; + break; + case ECORE_PCI_FCOE: + flags |= PQ_FLAGS_OFLD; + break; + case ECORE_PCI_ISCSI: + flags |= PQ_FLAGS_ACK | PQ_FLAGS_OOO | PQ_FLAGS_OFLD; + break; + case ECORE_PCI_ETH_ROCE: + flags |= PQ_FLAGS_MCOS | PQ_FLAGS_OFLD; + break; + case ECORE_PCI_ETH_IWARP: + flags |= PQ_FLAGS_MCOS | PQ_FLAGS_ACK | PQ_FLAGS_OOO | + PQ_FLAGS_OFLD; + break; + default: + DP_ERR(p_hwfn, "unknown personality %d\n", + p_hwfn->hw_info.personality); + return 0; } -#endif + return flags; +} - num_pqs = multi_cos_tcs + num_vfs + 1; /* The '1' is for pure-LB */ - num_vports = (u8)RESC_NUM(p_hwfn, ECORE_VPORT); +/* Getters for resource amounts necessary for qm initialization */ +u8 ecore_init_qm_get_num_tcs(struct ecore_hwfn *p_hwfn) +{ + return p_hwfn->hw_info.num_hw_tc; +} - /* Sanity checking that setup requires legal number of resources */ - if (num_pqs > RESC_NUM(p_hwfn, ECORE_PQ)) { - DP_ERR(p_hwfn, - "Need too many Physical queues - 0x%04x when" - " only %04x are available\n", - num_pqs, RESC_NUM(p_hwfn, ECORE_PQ)); - return ECORE_INVAL; - } +u16 ecore_init_qm_get_num_vfs(struct ecore_hwfn *p_hwfn) +{ + return IS_ECORE_SRIOV(p_hwfn->p_dev) ? + p_hwfn->p_dev->p_iov_info->total_vfs : 0; +} - /* PQs will be arranged as follows: First per-TC PQ, then pure-LB queue, - * then special queues, then per-VF PQ. +#define NUM_DEFAULT_RLS 1 + +u16 ecore_init_qm_get_num_pf_rls(struct ecore_hwfn *p_hwfn) +{ + u16 num_pf_rls, num_vfs = ecore_init_qm_get_num_vfs(p_hwfn); + + /* @DPDK */ + /* num RLs can't exceed resource amount of rls or vports or the + * dcqcn qps */ - qm_info->qm_pq_params = OSAL_ZALLOC(p_hwfn->p_dev, - b_sleepable ? GFP_KERNEL : - GFP_ATOMIC, - sizeof(struct init_qm_pq_params) * - num_pqs); - if (!qm_info->qm_pq_params) - goto alloc_err; + num_pf_rls = (u16)OSAL_MIN_T(u32, RESC_NUM(p_hwfn, ECORE_RL), + (u16)RESC_NUM(p_hwfn, ECORE_VPORT)); - qm_info->qm_vport_params = OSAL_ZALLOC(p_hwfn->p_dev, - b_sleepable ? GFP_KERNEL : - GFP_ATOMIC, - sizeof(struct - init_qm_vport_params) * - num_vports); - if (!qm_info->qm_vport_params) - goto alloc_err; - - qm_info->qm_port_params = OSAL_ZALLOC(p_hwfn->p_dev, - b_sleepable ? GFP_KERNEL : - GFP_ATOMIC, - sizeof(struct init_qm_port_params) - * MAX_NUM_PORTS); - if (!qm_info->qm_port_params) - goto alloc_err; - - qm_info->wfq_data = OSAL_ZALLOC(p_hwfn->p_dev, - b_sleepable ? GFP_KERNEL : - GFP_ATOMIC, - sizeof(struct ecore_wfq_data) * - num_vports); - - if (!qm_info->wfq_data) - goto alloc_err; - - vport_id = (u8)RESC_START(p_hwfn, ECORE_VPORT); - - /* First init per-TC PQs */ - for (i = 0; i < multi_cos_tcs; i++) { - struct init_qm_pq_params *params = &qm_info->qm_pq_params[i]; - - if (p_hwfn->hw_info.personality == ECORE_PCI_ETH) { - params->vport_id = vport_id; - params->tc_id = p_hwfn->hw_info.non_offload_tc; - params->wrr_group = 1; /* @@@TBD ECORE_WRR_MEDIUM */ - } else { - params->vport_id = vport_id; - params->tc_id = p_hwfn->hw_info.offload_tc; - params->wrr_group = 1; /* @@@TBD ECORE_WRR_MEDIUM */ - } + /* make sure after we reserve the default and VF rls we'll have + * something left + */ + if (num_pf_rls < num_vfs + NUM_DEFAULT_RLS) { + DP_NOTICE(p_hwfn, false, + "no rate limiters left for PF rate limiting" + " [num_pf_rls %d num_vfs %d]\n", num_pf_rls, num_vfs); + return 0; } - /* Then init pure-LB PQ */ - qm_info->pure_lb_pq = i; - qm_info->qm_pq_params[i].vport_id = - (u8)RESC_START(p_hwfn, ECORE_VPORT); - qm_info->qm_pq_params[i].tc_id = PURE_LB_TC; - qm_info->qm_pq_params[i].wrr_group = 1; - i++; + /* subtract rls necessary for VFs and one default one for the PF */ + num_pf_rls -= num_vfs + NUM_DEFAULT_RLS; - /* Then init per-VF PQs */ - vf_offset = i; - for (i = 0; i < num_vfs; i++) { - /* First vport is used by the PF */ - qm_info->qm_pq_params[vf_offset + i].vport_id = vport_id + - i + 1; - qm_info->qm_pq_params[vf_offset + i].tc_id = - p_hwfn->hw_info.non_offload_tc; - qm_info->qm_pq_params[vf_offset + i].wrr_group = 1; - }; + return num_pf_rls; +} - qm_info->vf_queues_offset = vf_offset; - qm_info->num_pqs = num_pqs; - qm_info->num_vports = num_vports; +u16 ecore_init_qm_get_num_vports(struct ecore_hwfn *p_hwfn) +{ + u32 pq_flags = ecore_get_pq_flags(p_hwfn); - /* Initialize qm port parameters */ - num_ports = p_hwfn->p_dev->num_ports_in_engines; - for (i = 0; i < num_ports; i++) { - p_qm_port = &qm_info->qm_port_params[i]; - p_qm_port->active = 1; - /* @@@TMP - was NUM_OF_PHYS_TCS; Changed until dcbx will - * be in place - */ - if (num_ports == 4) - p_qm_port->num_active_phys_tcs = 2; - else - p_qm_port->num_active_phys_tcs = 5; - p_qm_port->num_pbf_cmd_lines = PBF_MAX_CMD_LINES / num_ports; - p_qm_port->num_btb_blocks = BTB_MAX_BLOCKS / num_ports; - } + /* all pqs share the same vport (hence the 1 below), except for vfs + * and pf_rl pqs + */ + return (!!(PQ_FLAGS_RLS & pq_flags)) * + ecore_init_qm_get_num_pf_rls(p_hwfn) + + (!!(PQ_FLAGS_VFS & pq_flags)) * + ecore_init_qm_get_num_vfs(p_hwfn) + 1; +} - if (ECORE_IS_AH(p_hwfn->p_dev) && (num_ports == 4)) - qm_info->max_phys_tcs_per_port = NUM_PHYS_TCS_4PORT_K2; - else - qm_info->max_phys_tcs_per_port = NUM_OF_PHYS_TCS; +/* calc amount of PQs according to the requested flags */ +u16 ecore_init_qm_get_num_pqs(struct ecore_hwfn *p_hwfn) +{ + u32 pq_flags = ecore_get_pq_flags(p_hwfn); + return (!!(PQ_FLAGS_RLS & pq_flags)) * + ecore_init_qm_get_num_pf_rls(p_hwfn) + + (!!(PQ_FLAGS_MCOS & pq_flags)) * + ecore_init_qm_get_num_tcs(p_hwfn) + + (!!(PQ_FLAGS_LB & pq_flags)) + + (!!(PQ_FLAGS_OOO & pq_flags)) + + (!!(PQ_FLAGS_ACK & pq_flags)) + + (!!(PQ_FLAGS_OFLD & pq_flags)) + + (!!(PQ_FLAGS_VFS & pq_flags)) * + ecore_init_qm_get_num_vfs(p_hwfn); +} + +/* initialize the top level QM params */ +static void ecore_init_qm_params(struct ecore_hwfn *p_hwfn) +{ + struct ecore_qm_info *qm_info = &p_hwfn->qm_info; + bool four_port; + + /* pq and vport bases for this PF */ qm_info->start_pq = (u16)RESC_START(p_hwfn, ECORE_PQ); - - qm_info->num_vf_pqs = num_vfs; qm_info->start_vport = (u8)RESC_START(p_hwfn, ECORE_VPORT); - for (i = 0; i < qm_info->num_vports; i++) - qm_info->qm_vport_params[i].vport_wfq = 1; - - qm_info->pf_wfq = 0; - qm_info->pf_rl = 0; + /* rate limiting and weighted fair queueing are always enabled */ qm_info->vport_rl_en = 1; qm_info->vport_wfq_en = 1; - return ECORE_SUCCESS; + /* TC config is different for AH 4 port */ + four_port = p_hwfn->p_dev->num_ports_in_engine == MAX_NUM_PORTS_K2; -alloc_err: - DP_NOTICE(p_hwfn, false, "Failed to allocate memory for QM params\n"); - ecore_qm_info_free(p_hwfn); - return ECORE_NOMEM; + /* in AH 4 port we have fewer TCs per port */ + qm_info->max_phys_tcs_per_port = four_port ? NUM_PHYS_TCS_4PORT_K2 : + NUM_OF_PHYS_TCS; + + /* unless MFW indicated otherwise, ooo_tc should be 3 for AH 4 port and + * 4 otherwise + */ + if (!qm_info->ooo_tc) + qm_info->ooo_tc = four_port ? DCBX_TCP_OOO_K2_4PORT_TC : + DCBX_TCP_OOO_TC; +} + +/* initialize qm vport params */ +static void ecore_init_qm_vport_params(struct ecore_hwfn *p_hwfn) +{ + struct ecore_qm_info *qm_info = &p_hwfn->qm_info; + u8 i; + + /* all vports participate in weighted fair queueing */ + for (i = 0; i < ecore_init_qm_get_num_vports(p_hwfn); i++) + qm_info->qm_vport_params[i].vport_wfq = 1; +} + +/* initialize qm port params */ +static void ecore_init_qm_port_params(struct ecore_hwfn *p_hwfn) +{ + /* Initialize qm port parameters */ + u8 i, active_phys_tcs, num_ports = p_hwfn->p_dev->num_ports_in_engine; + + /* indicate how ooo and high pri traffic is dealt with */ + active_phys_tcs = num_ports == MAX_NUM_PORTS_K2 ? + ACTIVE_TCS_BMAP_4PORT_K2 : ACTIVE_TCS_BMAP; + + for (i = 0; i < num_ports; i++) { + struct init_qm_port_params *p_qm_port = + &p_hwfn->qm_info.qm_port_params[i]; + + p_qm_port->active = 1; + p_qm_port->active_phys_tcs = active_phys_tcs; + p_qm_port->num_pbf_cmd_lines = PBF_MAX_CMD_LINES_E4 / num_ports; + p_qm_port->num_btb_blocks = BTB_MAX_BLOCKS / num_ports; + } +} + +/* Reset the params which must be reset for qm init. QM init may be called as + * a result of flows other than driver load (e.g. dcbx renegotiation). Other + * params may be affected by the init but would simply recalculate to the same + * values. The allocations made for QM init, ports, vports, pqs and vfqs are not + * affected as these amounts stay the same. + */ +static void ecore_init_qm_reset_params(struct ecore_hwfn *p_hwfn) +{ + struct ecore_qm_info *qm_info = &p_hwfn->qm_info; + + qm_info->num_pqs = 0; + qm_info->num_vports = 0; + qm_info->num_pf_rls = 0; + qm_info->num_vf_pqs = 0; + qm_info->first_vf_pq = 0; + qm_info->first_mcos_pq = 0; + qm_info->first_rl_pq = 0; +} + +static void ecore_init_qm_advance_vport(struct ecore_hwfn *p_hwfn) +{ + struct ecore_qm_info *qm_info = &p_hwfn->qm_info; + + qm_info->num_vports++; + + if (qm_info->num_vports > ecore_init_qm_get_num_vports(p_hwfn)) + DP_ERR(p_hwfn, + "vport overflow! qm_info->num_vports %d," + " qm_init_get_num_vports() %d\n", + qm_info->num_vports, + ecore_init_qm_get_num_vports(p_hwfn)); +} + +/* initialize a single pq and manage qm_info resources accounting. + * The pq_init_flags param determines whether the PQ is rate limited + * (for VF or PF) + * and whether a new vport is allocated to the pq or not (i.e. vport will be + * shared) + */ + +/* flags for pq init */ +#define PQ_INIT_SHARE_VPORT (1 << 0) +#define PQ_INIT_PF_RL (1 << 1) +#define PQ_INIT_VF_RL (1 << 2) + +/* defines for pq init */ +#define PQ_INIT_DEFAULT_WRR_GROUP 1 +#define PQ_INIT_DEFAULT_TC 0 +#define PQ_INIT_OFLD_TC (p_hwfn->hw_info.offload_tc) + +static void ecore_init_qm_pq(struct ecore_hwfn *p_hwfn, + struct ecore_qm_info *qm_info, + u8 tc, u32 pq_init_flags) +{ + u16 pq_idx = qm_info->num_pqs, max_pq = + ecore_init_qm_get_num_pqs(p_hwfn); + + if (pq_idx > max_pq) + DP_ERR(p_hwfn, + "pq overflow! pq %d, max pq %d\n", pq_idx, max_pq); + + /* init pq params */ + qm_info->qm_pq_params[pq_idx].vport_id = qm_info->start_vport + + qm_info->num_vports; + qm_info->qm_pq_params[pq_idx].tc_id = tc; + qm_info->qm_pq_params[pq_idx].wrr_group = PQ_INIT_DEFAULT_WRR_GROUP; + qm_info->qm_pq_params[pq_idx].rl_valid = + (pq_init_flags & PQ_INIT_PF_RL || + pq_init_flags & PQ_INIT_VF_RL); + + /* qm params accounting */ + qm_info->num_pqs++; + if (!(pq_init_flags & PQ_INIT_SHARE_VPORT)) + qm_info->num_vports++; + + if (pq_init_flags & PQ_INIT_PF_RL) + qm_info->num_pf_rls++; + + if (qm_info->num_vports > ecore_init_qm_get_num_vports(p_hwfn)) + DP_ERR(p_hwfn, + "vport overflow! qm_info->num_vports %d," + " qm_init_get_num_vports() %d\n", + qm_info->num_vports, + ecore_init_qm_get_num_vports(p_hwfn)); + + if (qm_info->num_pf_rls > ecore_init_qm_get_num_pf_rls(p_hwfn)) + DP_ERR(p_hwfn, "rl overflow! qm_info->num_pf_rls %d," + " qm_init_get_num_pf_rls() %d\n", + qm_info->num_pf_rls, + ecore_init_qm_get_num_pf_rls(p_hwfn)); +} + +/* get pq index according to PQ_FLAGS */ +static u16 *ecore_init_qm_get_idx_from_flags(struct ecore_hwfn *p_hwfn, + u32 pq_flags) +{ + struct ecore_qm_info *qm_info = &p_hwfn->qm_info; + + /* Can't have multiple flags set here */ + if (OSAL_BITMAP_WEIGHT((unsigned long *)&pq_flags, + sizeof(pq_flags)) > 1) + goto err; + + switch (pq_flags) { + case PQ_FLAGS_RLS: + return &qm_info->first_rl_pq; + case PQ_FLAGS_MCOS: + return &qm_info->first_mcos_pq; + case PQ_FLAGS_LB: + return &qm_info->pure_lb_pq; + case PQ_FLAGS_OOO: + return &qm_info->ooo_pq; + case PQ_FLAGS_ACK: + return &qm_info->pure_ack_pq; + case PQ_FLAGS_OFLD: + return &qm_info->offload_pq; + case PQ_FLAGS_VFS: + return &qm_info->first_vf_pq; + default: + goto err; + } + +err: + DP_ERR(p_hwfn, "BAD pq flags %d\n", pq_flags); + return OSAL_NULL; +} + +/* save pq index in qm info */ +static void ecore_init_qm_set_idx(struct ecore_hwfn *p_hwfn, + u32 pq_flags, u16 pq_val) +{ + u16 *base_pq_idx = ecore_init_qm_get_idx_from_flags(p_hwfn, pq_flags); + + *base_pq_idx = p_hwfn->qm_info.start_pq + pq_val; +} + +/* get tx pq index, with the PQ TX base already set (ready for context init) */ +u16 ecore_get_cm_pq_idx(struct ecore_hwfn *p_hwfn, u32 pq_flags) +{ + u16 *base_pq_idx = ecore_init_qm_get_idx_from_flags(p_hwfn, pq_flags); + + return *base_pq_idx + CM_TX_PQ_BASE; +} + +u16 ecore_get_cm_pq_idx_mcos(struct ecore_hwfn *p_hwfn, u8 tc) +{ + u8 max_tc = ecore_init_qm_get_num_tcs(p_hwfn); + + if (tc > max_tc) + DP_ERR(p_hwfn, "tc %d must be smaller than %d\n", tc, max_tc); + + return ecore_get_cm_pq_idx(p_hwfn, PQ_FLAGS_MCOS) + tc; +} + +u16 ecore_get_cm_pq_idx_vf(struct ecore_hwfn *p_hwfn, u16 vf) +{ + u16 max_vf = ecore_init_qm_get_num_vfs(p_hwfn); + + if (vf > max_vf) + DP_ERR(p_hwfn, "vf %d must be smaller than %d\n", vf, max_vf); + + return ecore_get_cm_pq_idx(p_hwfn, PQ_FLAGS_VFS) + vf; +} + +u16 ecore_get_cm_pq_idx_rl(struct ecore_hwfn *p_hwfn, u8 rl) +{ + u16 max_rl = ecore_init_qm_get_num_pf_rls(p_hwfn); + + if (rl > max_rl) + DP_ERR(p_hwfn, "rl %d must be smaller than %d\n", rl, max_rl); + + return ecore_get_cm_pq_idx(p_hwfn, PQ_FLAGS_RLS) + rl; +} + +/* Functions for creating specific types of pqs */ +static void ecore_init_qm_lb_pq(struct ecore_hwfn *p_hwfn) +{ + struct ecore_qm_info *qm_info = &p_hwfn->qm_info; + + if (!(ecore_get_pq_flags(p_hwfn) & PQ_FLAGS_LB)) + return; + + ecore_init_qm_set_idx(p_hwfn, PQ_FLAGS_LB, qm_info->num_pqs); + ecore_init_qm_pq(p_hwfn, qm_info, PURE_LB_TC, PQ_INIT_SHARE_VPORT); +} + +static void ecore_init_qm_ooo_pq(struct ecore_hwfn *p_hwfn) +{ + struct ecore_qm_info *qm_info = &p_hwfn->qm_info; + + if (!(ecore_get_pq_flags(p_hwfn) & PQ_FLAGS_OOO)) + return; + + ecore_init_qm_set_idx(p_hwfn, PQ_FLAGS_OOO, qm_info->num_pqs); + ecore_init_qm_pq(p_hwfn, qm_info, qm_info->ooo_tc, PQ_INIT_SHARE_VPORT); +} + +static void ecore_init_qm_pure_ack_pq(struct ecore_hwfn *p_hwfn) +{ + struct ecore_qm_info *qm_info = &p_hwfn->qm_info; + + if (!(ecore_get_pq_flags(p_hwfn) & PQ_FLAGS_ACK)) + return; + + ecore_init_qm_set_idx(p_hwfn, PQ_FLAGS_ACK, qm_info->num_pqs); + ecore_init_qm_pq(p_hwfn, qm_info, PQ_INIT_OFLD_TC, PQ_INIT_SHARE_VPORT); +} + +static void ecore_init_qm_offload_pq(struct ecore_hwfn *p_hwfn) +{ + struct ecore_qm_info *qm_info = &p_hwfn->qm_info; + + if (!(ecore_get_pq_flags(p_hwfn) & PQ_FLAGS_OFLD)) + return; + + ecore_init_qm_set_idx(p_hwfn, PQ_FLAGS_OFLD, qm_info->num_pqs); + ecore_init_qm_pq(p_hwfn, qm_info, PQ_INIT_OFLD_TC, PQ_INIT_SHARE_VPORT); +} + +static void ecore_init_qm_mcos_pqs(struct ecore_hwfn *p_hwfn) +{ + struct ecore_qm_info *qm_info = &p_hwfn->qm_info; + u8 tc_idx; + + if (!(ecore_get_pq_flags(p_hwfn) & PQ_FLAGS_MCOS)) + return; + + ecore_init_qm_set_idx(p_hwfn, PQ_FLAGS_MCOS, qm_info->num_pqs); + for (tc_idx = 0; tc_idx < ecore_init_qm_get_num_tcs(p_hwfn); tc_idx++) + ecore_init_qm_pq(p_hwfn, qm_info, tc_idx, PQ_INIT_SHARE_VPORT); +} + +static void ecore_init_qm_vf_pqs(struct ecore_hwfn *p_hwfn) +{ + struct ecore_qm_info *qm_info = &p_hwfn->qm_info; + u16 vf_idx, num_vfs = ecore_init_qm_get_num_vfs(p_hwfn); + + if (!(ecore_get_pq_flags(p_hwfn) & PQ_FLAGS_VFS)) + return; + + ecore_init_qm_set_idx(p_hwfn, PQ_FLAGS_VFS, qm_info->num_pqs); + + qm_info->num_vf_pqs = num_vfs; + for (vf_idx = 0; vf_idx < num_vfs; vf_idx++) + ecore_init_qm_pq(p_hwfn, qm_info, PQ_INIT_DEFAULT_TC, + PQ_INIT_VF_RL); +} + +static void ecore_init_qm_rl_pqs(struct ecore_hwfn *p_hwfn) +{ + u16 pf_rls_idx, num_pf_rls = ecore_init_qm_get_num_pf_rls(p_hwfn); + struct ecore_qm_info *qm_info = &p_hwfn->qm_info; + + if (!(ecore_get_pq_flags(p_hwfn) & PQ_FLAGS_RLS)) + return; + + ecore_init_qm_set_idx(p_hwfn, PQ_FLAGS_RLS, qm_info->num_pqs); + for (pf_rls_idx = 0; pf_rls_idx < num_pf_rls; pf_rls_idx++) + ecore_init_qm_pq(p_hwfn, qm_info, PQ_INIT_OFLD_TC, + PQ_INIT_PF_RL); +} + +static void ecore_init_qm_pq_params(struct ecore_hwfn *p_hwfn) +{ + /* rate limited pqs, must come first (FW assumption) */ + ecore_init_qm_rl_pqs(p_hwfn); + + /* pqs for multi cos */ + ecore_init_qm_mcos_pqs(p_hwfn); + + /* pure loopback pq */ + ecore_init_qm_lb_pq(p_hwfn); + + /* out of order pq */ + ecore_init_qm_ooo_pq(p_hwfn); + + /* pure ack pq */ + ecore_init_qm_pure_ack_pq(p_hwfn); + + /* pq for offloaded protocol */ + ecore_init_qm_offload_pq(p_hwfn); + + /* done sharing vports */ + ecore_init_qm_advance_vport(p_hwfn); + + /* pqs for vfs */ + ecore_init_qm_vf_pqs(p_hwfn); +} + +/* compare values of getters against resources amounts */ +static enum _ecore_status_t ecore_init_qm_sanity(struct ecore_hwfn *p_hwfn) +{ + if (ecore_init_qm_get_num_vports(p_hwfn) > + RESC_NUM(p_hwfn, ECORE_VPORT)) { + DP_ERR(p_hwfn, "requested amount of vports exceeds resource\n"); + return ECORE_INVAL; + } + + if (ecore_init_qm_get_num_pqs(p_hwfn) > RESC_NUM(p_hwfn, ECORE_PQ)) { + DP_ERR(p_hwfn, "requested amount of pqs exceeds resource\n"); + return ECORE_INVAL; + } + + return ECORE_SUCCESS; +} + +/* + * Function for verbose printing of the qm initialization results + */ +static void ecore_dp_init_qm_params(struct ecore_hwfn *p_hwfn) +{ + struct ecore_qm_info *qm_info = &p_hwfn->qm_info; + struct init_qm_vport_params *vport; + struct init_qm_port_params *port; + struct init_qm_pq_params *pq; + int i, tc; + + /* top level params */ + DP_VERBOSE(p_hwfn, ECORE_MSG_HW, + "qm init top level params: start_pq %d, start_vport %d," + " pure_lb_pq %d, offload_pq %d, pure_ack_pq %d\n", + qm_info->start_pq, qm_info->start_vport, qm_info->pure_lb_pq, + qm_info->offload_pq, qm_info->pure_ack_pq); + DP_VERBOSE(p_hwfn, ECORE_MSG_HW, + "ooo_pq %d, first_vf_pq %d, num_pqs %d, num_vf_pqs %d," + " num_vports %d, max_phys_tcs_per_port %d\n", + qm_info->ooo_pq, qm_info->first_vf_pq, qm_info->num_pqs, + qm_info->num_vf_pqs, qm_info->num_vports, + qm_info->max_phys_tcs_per_port); + DP_VERBOSE(p_hwfn, ECORE_MSG_HW, + "pf_rl_en %d, pf_wfq_en %d, vport_rl_en %d, vport_wfq_en %d," + " pf_wfq %d, pf_rl %d, num_pf_rls %d, pq_flags %x\n", + qm_info->pf_rl_en, qm_info->pf_wfq_en, qm_info->vport_rl_en, + qm_info->vport_wfq_en, qm_info->pf_wfq, qm_info->pf_rl, + qm_info->num_pf_rls, ecore_get_pq_flags(p_hwfn)); + + /* port table */ + for (i = 0; i < p_hwfn->p_dev->num_ports_in_engine; i++) { + port = &qm_info->qm_port_params[i]; + DP_VERBOSE(p_hwfn, ECORE_MSG_HW, + "port idx %d, active %d, active_phys_tcs %d," + " num_pbf_cmd_lines %d, num_btb_blocks %d," + " reserved %d\n", + i, port->active, port->active_phys_tcs, + port->num_pbf_cmd_lines, port->num_btb_blocks, + port->reserved); + } + + /* vport table */ + for (i = 0; i < qm_info->num_vports; i++) { + vport = &qm_info->qm_vport_params[i]; + DP_VERBOSE(p_hwfn, ECORE_MSG_HW, + "vport idx %d, vport_rl %d, wfq %d," + " first_tx_pq_id [ ", + qm_info->start_vport + i, vport->vport_rl, + vport->vport_wfq); + for (tc = 0; tc < NUM_OF_TCS; tc++) + DP_VERBOSE(p_hwfn, ECORE_MSG_HW, "%d ", + vport->first_tx_pq_id[tc]); + DP_VERBOSE(p_hwfn, ECORE_MSG_HW, "]\n"); + } + + /* pq table */ + for (i = 0; i < qm_info->num_pqs; i++) { + pq = &qm_info->qm_pq_params[i]; + DP_VERBOSE(p_hwfn, ECORE_MSG_HW, + "pq idx %d, vport_id %d, tc %d, wrr_grp %d," + " rl_valid %d\n", + qm_info->start_pq + i, pq->vport_id, pq->tc_id, + pq->wrr_group, pq->rl_valid); + } +} + +static void ecore_init_qm_info(struct ecore_hwfn *p_hwfn) +{ + /* reset params required for init run */ + ecore_init_qm_reset_params(p_hwfn); + + /* init QM top level params */ + ecore_init_qm_params(p_hwfn); + + /* init QM port params */ + ecore_init_qm_port_params(p_hwfn); + + /* init QM vport params */ + ecore_init_qm_vport_params(p_hwfn); + + /* init QM physical queue params */ + ecore_init_qm_pq_params(p_hwfn); + + /* display all that init */ + ecore_dp_init_qm_params(p_hwfn); } /* This function reconfigures the QM pf on the fly. * For this purpose we: * 1. reconfigure the QM database - * 2. set new values to runtime arrat + * 2. set new values to runtime array * 3. send an sdm_qm_cmd through the rbc interface to stop the QM * 4. activate init tool in QM_PF stage * 5. send an sdm_qm_cmd through rbc interface to release the QM @@ -330,24 +1065,17 @@ enum _ecore_status_t ecore_qm_reconf(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt) { struct ecore_qm_info *qm_info = &p_hwfn->qm_info; - enum _ecore_status_t rc; bool b_rc; - - /* qm_info is allocated in ecore_init_qm_info() which is already called - * from ecore_resc_alloc() or previous call of ecore_qm_reconf(). - * The allocated size may change each init, so we free it before next - * allocation. - */ - ecore_qm_info_free(p_hwfn); + enum _ecore_status_t rc; /* initialize ecore's qm data structure */ - rc = ecore_init_qm_info(p_hwfn, false); - if (rc != ECORE_SUCCESS) - return rc; + ecore_init_qm_info(p_hwfn); /* stop PF's qm queues */ + OSAL_SPIN_LOCK(&qm_lock); b_rc = ecore_send_qm_stop_cmd(p_hwfn, p_ptt, false, true, qm_info->start_pq, qm_info->num_pqs); + OSAL_SPIN_UNLOCK(&qm_lock); if (!b_rc) return ECORE_INVAL; @@ -355,7 +1083,7 @@ enum _ecore_status_t ecore_qm_reconf(struct ecore_hwfn *p_hwfn, ecore_init_clear_rt_data(p_hwfn); /* prepare QM portion of runtime array */ - ecore_qm_init_pf(p_hwfn); + ecore_qm_init_pf(p_hwfn, p_ptt); /* activate init tool on runtime array */ rc = ecore_init_run(p_hwfn, p_ptt, PHASE_QM_PF, p_hwfn->rel_pf_id, @@ -364,76 +1092,105 @@ enum _ecore_status_t ecore_qm_reconf(struct ecore_hwfn *p_hwfn, return rc; /* start PF's qm queues */ + OSAL_SPIN_LOCK(&qm_lock); b_rc = ecore_send_qm_stop_cmd(p_hwfn, p_ptt, true, true, qm_info->start_pq, qm_info->num_pqs); - if (!rc) + OSAL_SPIN_UNLOCK(&qm_lock); + if (!b_rc) return ECORE_INVAL; return ECORE_SUCCESS; } +static enum _ecore_status_t ecore_alloc_qm_data(struct ecore_hwfn *p_hwfn) +{ + struct ecore_qm_info *qm_info = &p_hwfn->qm_info; + enum _ecore_status_t rc; + + rc = ecore_init_qm_sanity(p_hwfn); + if (rc != ECORE_SUCCESS) + goto alloc_err; + + qm_info->qm_pq_params = OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL, + sizeof(struct init_qm_pq_params) * + ecore_init_qm_get_num_pqs(p_hwfn)); + if (!qm_info->qm_pq_params) + goto alloc_err; + + qm_info->qm_vport_params = OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL, + sizeof(struct init_qm_vport_params) * + ecore_init_qm_get_num_vports(p_hwfn)); + if (!qm_info->qm_vport_params) + goto alloc_err; + + qm_info->qm_port_params = OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL, + sizeof(struct init_qm_port_params) * + p_hwfn->p_dev->num_ports_in_engine); + if (!qm_info->qm_port_params) + goto alloc_err; + + qm_info->wfq_data = OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL, + sizeof(struct ecore_wfq_data) * + ecore_init_qm_get_num_vports(p_hwfn)); + if (!qm_info->wfq_data) + goto alloc_err; + + return ECORE_SUCCESS; + +alloc_err: + DP_NOTICE(p_hwfn, false, "Failed to allocate memory for QM params\n"); + ecore_qm_info_free(p_hwfn); + return ECORE_NOMEM; +} +/******************** End QM initialization ***************/ + enum _ecore_status_t ecore_resc_alloc(struct ecore_dev *p_dev) { enum _ecore_status_t rc = ECORE_SUCCESS; - struct ecore_consq *p_consq; - struct ecore_eq *p_eq; int i; - if (IS_VF(p_dev)) + if (IS_VF(p_dev)) { + for_each_hwfn(p_dev, i) { + rc = ecore_l2_alloc(&p_dev->hwfns[i]); + if (rc != ECORE_SUCCESS) + return rc; + } return rc; + } p_dev->fw_data = OSAL_ZALLOC(p_dev, GFP_KERNEL, - sizeof(struct ecore_fw_data)); + sizeof(*p_dev->fw_data)); if (!p_dev->fw_data) return ECORE_NOMEM; - /* Allocate Memory for the Queue->CID mapping */ for_each_hwfn(p_dev, i) { struct ecore_hwfn *p_hwfn = &p_dev->hwfns[i]; + u32 n_eqes, num_cons; - /* @@@TMP - resc management, change to actual required size */ - int tx_size = sizeof(struct ecore_hw_cid_data) * - RESC_NUM(p_hwfn, ECORE_L2_QUEUE); - int rx_size = sizeof(struct ecore_hw_cid_data) * - RESC_NUM(p_hwfn, ECORE_L2_QUEUE); - - p_hwfn->p_tx_cids = OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL, - tx_size); - if (!p_hwfn->p_tx_cids) { - DP_NOTICE(p_hwfn, true, - "Failed to allocate memory for Tx Cids\n"); - goto alloc_no_mem; - } - - p_hwfn->p_rx_cids = OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL, - rx_size); - if (!p_hwfn->p_rx_cids) { - DP_NOTICE(p_hwfn, true, - "Failed to allocate memory for Rx Cids\n"); - goto alloc_no_mem; - } - } - - for_each_hwfn(p_dev, i) { - struct ecore_hwfn *p_hwfn = &p_dev->hwfns[i]; + /* initialize the doorbell recovery mechanism */ + rc = ecore_db_recovery_setup(p_hwfn); + if (rc) + goto alloc_err; /* First allocate the context manager structure */ rc = ecore_cxt_mngr_alloc(p_hwfn); if (rc) goto alloc_err; - /* Set the HW cid/tid numbers (in the contest manager) + /* Set the HW cid/tid numbers (in the context manager) * Must be done prior to any further computations. */ rc = ecore_cxt_set_pf_params(p_hwfn); if (rc) goto alloc_err; - /* Prepare and process QM requirements */ - rc = ecore_init_qm_info(p_hwfn, true); + rc = ecore_alloc_qm_data(p_hwfn); if (rc) goto alloc_err; + /* init qm info */ + ecore_init_qm_info(p_hwfn); + /* Compute the ILT client partition */ rc = ecore_cxt_cfg_ilt_compute(p_hwfn); if (rc) @@ -464,22 +1221,77 @@ enum _ecore_status_t ecore_resc_alloc(struct ecore_dev *p_dev) goto alloc_err; /* EQ */ - p_eq = ecore_eq_alloc(p_hwfn, 256); - if (!p_eq) - goto alloc_no_mem; - p_hwfn->p_eq = p_eq; + n_eqes = ecore_chain_get_capacity(&p_hwfn->p_spq->chain); + if (ECORE_IS_RDMA_PERSONALITY(p_hwfn)) { + /* Calculate the EQ size + * --------------------- + * Each ICID may generate up to one event at a time i.e. + * the event must be handled/cleared before a new one + * can be generated. We calculate the sum of events per + * protocol and create an EQ deep enough to handle the + * worst case: + * - Core - according to SPQ. + * - RoCE - per QP there are a couple of ICIDs, one + * responder and one requester, each can + * generate an EQE => n_eqes_qp = 2 * n_qp. + * Each CQ can generate an EQE. There are 2 CQs + * per QP => n_eqes_cq = 2 * n_qp. + * Hence the RoCE total is 4 * n_qp or + * 2 * num_cons. + * - ENet - There can be up to two events per VF. One + * for VF-PF channel and another for VF FLR + * initial cleanup. The number of VFs is + * bounded by MAX_NUM_VFS_BB, and is much + * smaller than RoCE's so we avoid exact + * calculation. + */ + if (ECORE_IS_ROCE_PERSONALITY(p_hwfn)) { + num_cons = + ecore_cxt_get_proto_cid_count( + p_hwfn, + PROTOCOLID_ROCE, + OSAL_NULL); + num_cons *= 2; + } else { + num_cons = ecore_cxt_get_proto_cid_count( + p_hwfn, + PROTOCOLID_IWARP, + OSAL_NULL); + } + n_eqes += num_cons + 2 * MAX_NUM_VFS_BB; + } else if (p_hwfn->hw_info.personality == ECORE_PCI_ISCSI) { + num_cons = + ecore_cxt_get_proto_cid_count(p_hwfn, + PROTOCOLID_ISCSI, + OSAL_NULL); + n_eqes += 2 * num_cons; + } - p_consq = ecore_consq_alloc(p_hwfn); - if (!p_consq) + if (n_eqes > 0xFFFF) { + DP_ERR(p_hwfn, "Cannot allocate 0x%x EQ elements." + "The maximum of a u16 chain is 0x%x\n", + n_eqes, 0xFFFF); goto alloc_no_mem; - p_hwfn->p_consq = p_consq; + } + + rc = ecore_eq_alloc(p_hwfn, (u16)n_eqes); + if (rc) + goto alloc_err; + + rc = ecore_consq_alloc(p_hwfn); + if (rc) + goto alloc_err; + + rc = ecore_l2_alloc(p_hwfn); + if (rc != ECORE_SUCCESS) + goto alloc_err; /* DMA info initialization */ rc = ecore_dmae_info_alloc(p_hwfn); if (rc) { DP_NOTICE(p_hwfn, true, - "Failed to allocate memory for" - " dmae_info structure\n"); + "Failed to allocate memory for dmae_info" + " structure\n"); goto alloc_err; } @@ -487,13 +1299,13 @@ enum _ecore_status_t ecore_resc_alloc(struct ecore_dev *p_dev) rc = ecore_dcbx_info_alloc(p_hwfn); if (rc) { DP_NOTICE(p_hwfn, true, - "Failed to allocate memory for dcbxstruct\n"); + "Failed to allocate memory for dcbx structure\n"); goto alloc_err; } } p_dev->reset_stats = OSAL_ZALLOC(p_dev, GFP_KERNEL, - sizeof(struct ecore_eth_stats)); + sizeof(*p_dev->reset_stats)); if (!p_dev->reset_stats) { DP_NOTICE(p_dev, true, "Failed to allocate reset statistics\n"); goto alloc_no_mem; @@ -512,16 +1324,19 @@ void ecore_resc_setup(struct ecore_dev *p_dev) { int i; - if (IS_VF(p_dev)) + if (IS_VF(p_dev)) { + for_each_hwfn(p_dev, i) + ecore_l2_setup(&p_dev->hwfns[i]); return; + } for_each_hwfn(p_dev, i) { struct ecore_hwfn *p_hwfn = &p_dev->hwfns[i]; ecore_cxt_mngr_setup(p_hwfn); ecore_spq_setup(p_hwfn); - ecore_eq_setup(p_hwfn, p_hwfn->p_eq); - ecore_consq_setup(p_hwfn, p_hwfn->p_consq); + ecore_eq_setup(p_hwfn); + ecore_consq_setup(p_hwfn); /* Read shadow of current MFW mailbox */ ecore_mcp_read_mb(p_hwfn, p_hwfn->p_main_ptt); @@ -531,7 +1346,8 @@ void ecore_resc_setup(struct ecore_dev *p_dev) ecore_int_setup(p_hwfn, p_hwfn->p_main_ptt); - ecore_iov_setup(p_hwfn, p_hwfn->p_main_ptt); + ecore_l2_setup(p_hwfn); + ecore_iov_setup(p_hwfn); } } @@ -564,20 +1380,21 @@ enum _ecore_status_t ecore_final_cleanup(struct ecore_hwfn *p_hwfn, command |= id << SDM_AGG_INT_COMP_PARAMS_AGG_VECTOR_BIT_SHIFT; command |= SDM_COMP_TYPE_AGG_INT << SDM_OP_GEN_COMP_TYPE_SHIFT; - /* Make sure notification is not set before initiating final cleanup */ +/* Make sure notification is not set before initiating final cleanup */ + if (REG_RD(p_hwfn, addr)) { DP_NOTICE(p_hwfn, false, - "Unexpected; Found final cleanup notification " - "before initiating final cleanup\n"); + "Unexpected; Found final cleanup notification"); + DP_NOTICE(p_hwfn, false, + " before initiating final cleanup\n"); REG_WR(p_hwfn, addr, 0); } DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, - "Sending final cleanup for PFVF[%d] [Command %08x\n]", - id, OSAL_CPU_TO_LE32(command)); + "Sending final cleanup for PFVF[%d] [Command %08x]\n", + id, command); - ecore_wr(p_hwfn, p_ptt, XSDM_REG_OPERATION_GEN, - OSAL_CPU_TO_LE32(command)); + ecore_wr(p_hwfn, p_ptt, XSDM_REG_OPERATION_GEN, command); /* Poll until completion */ while (!REG_RD(p_hwfn, addr) && count--) @@ -595,28 +1412,22 @@ enum _ecore_status_t ecore_final_cleanup(struct ecore_hwfn *p_hwfn, return rc; } -static void ecore_calc_hw_mode(struct ecore_hwfn *p_hwfn) +static enum _ecore_status_t ecore_calc_hw_mode(struct ecore_hwfn *p_hwfn) { int hw_mode = 0; - switch (ECORE_GET_TYPE(p_hwfn->p_dev)) { - case CHIP_BB_A0: - hw_mode |= 1 << MODE_BB_A0; - break; - case CHIP_BB_B0: - hw_mode |= 1 << MODE_BB_B0; - break; - case CHIP_K2: + if (ECORE_IS_BB_B0(p_hwfn->p_dev)) { + hw_mode |= 1 << MODE_BB; + } else if (ECORE_IS_AH(p_hwfn->p_dev)) { hw_mode |= 1 << MODE_K2; - break; - default: - DP_NOTICE(p_hwfn, true, "Can't initialize chip ID %d\n", - ECORE_GET_TYPE(p_hwfn->p_dev)); - return; + } else { + DP_NOTICE(p_hwfn, true, "Unknown chip type %#x\n", + p_hwfn->p_dev->type); + return ECORE_INVAL; } /* Ports per engine is based on the values in CNIG_REG_NW_PORT_MODE */ - switch (p_hwfn->p_dev->num_ports_in_engines) { + switch (p_hwfn->p_dev->num_ports_in_engine) { case 1: hw_mode |= 1 << MODE_PORTS_PER_ENG_1; break; @@ -629,23 +1440,15 @@ static void ecore_calc_hw_mode(struct ecore_hwfn *p_hwfn) default: DP_NOTICE(p_hwfn, true, "num_ports_in_engine = %d not supported\n", - p_hwfn->p_dev->num_ports_in_engines); - return; + p_hwfn->p_dev->num_ports_in_engine); + return ECORE_INVAL; } - switch (p_hwfn->p_dev->mf_mode) { - case ECORE_MF_DEFAULT: - case ECORE_MF_NPAR: - hw_mode |= 1 << MODE_MF_SI; - break; - case ECORE_MF_OVLAN: + if (OSAL_TEST_BIT(ECORE_MF_OVLAN_CLSS, + &p_hwfn->p_dev->mf_bits)) hw_mode |= 1 << MODE_MF_SD; - break; - default: - DP_NOTICE(p_hwfn, true, - "Unsupported MF mode, init as DEFAULT\n"); + else hw_mode |= 1 << MODE_MF_SI; - } #ifndef ASIC_ONLY if (CHIP_REV_IS_SLOW(p_hwfn->p_dev)) { @@ -661,10 +1464,7 @@ static void ecore_calc_hw_mode(struct ecore_hwfn *p_hwfn) #endif hw_mode |= 1 << MODE_ASIC; - if (ENABLE_EAGLE_ENG1_WORKAROUND(p_hwfn)) - hw_mode |= 1 << MODE_EAGLE_ENG1_WORKAROUND; - - if (p_hwfn->p_dev->num_hwfns > 1) + if (ECORE_IS_CMT(p_hwfn->p_dev)) hw_mode |= 1 << MODE_100G; p_hwfn->hw_info.hw_mode = hw_mode; @@ -672,67 +1472,45 @@ static void ecore_calc_hw_mode(struct ecore_hwfn *p_hwfn) DP_VERBOSE(p_hwfn, (ECORE_MSG_PROBE | ECORE_MSG_IFUP), "Configuring function for hw_mode: 0x%08x\n", p_hwfn->hw_info.hw_mode); + + return ECORE_SUCCESS; } #ifndef ASIC_ONLY /* MFW-replacement initializations for non-ASIC */ -static void ecore_hw_init_chip(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt) +static enum _ecore_status_t ecore_hw_init_chip(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt) { + struct ecore_dev *p_dev = p_hwfn->p_dev; u32 pl_hv = 1; int i; - if (CHIP_REV_IS_EMUL(p_hwfn->p_dev) && ECORE_IS_AH(p_hwfn->p_dev)) - pl_hv |= 0x600; + if (CHIP_REV_IS_EMUL(p_dev)) { + if (ECORE_IS_AH(p_dev)) + pl_hv |= 0x600; + } ecore_wr(p_hwfn, p_ptt, MISCS_REG_RESET_PL_HV + 4, pl_hv); - if (CHIP_REV_IS_EMUL(p_hwfn->p_dev) && ECORE_IS_AH(p_hwfn->p_dev)) - ecore_wr(p_hwfn, p_ptt, MISCS_REG_RESET_PL_HV_2, 0x3ffffff); - - /* initialize interrupt masks */ - for (i = 0; - i < - attn_blocks[BLOCK_MISCS].chip_regs[ECORE_GET_TYPE(p_hwfn->p_dev)]. - num_of_int_regs; i++) - ecore_wr(p_hwfn, p_ptt, - attn_blocks[BLOCK_MISCS]. - chip_regs[ECORE_GET_TYPE(p_hwfn->p_dev)].int_regs[i]-> - mask_addr, 0); - - if (!CHIP_REV_IS_EMUL(p_hwfn->p_dev) || !ECORE_IS_AH(p_hwfn->p_dev)) - ecore_wr(p_hwfn, p_ptt, - attn_blocks[BLOCK_CNIG]. - chip_regs[ECORE_GET_TYPE(p_hwfn->p_dev)].int_regs[0]-> - mask_addr, 0); - ecore_wr(p_hwfn, p_ptt, - attn_blocks[BLOCK_PGLCS]. - chip_regs[ECORE_GET_TYPE(p_hwfn->p_dev)].int_regs[0]-> - mask_addr, 0); - ecore_wr(p_hwfn, p_ptt, - attn_blocks[BLOCK_CPMU]. - chip_regs[ECORE_GET_TYPE(p_hwfn->p_dev)].int_regs[0]-> - mask_addr, 0); - /* Currently A0 and B0 interrupt bits are the same in pglue_b; - * If this changes, need to set this according to chip type. <14/09/23> - */ - ecore_wr(p_hwfn, p_ptt, - attn_blocks[BLOCK_PGLUE_B]. - chip_regs[ECORE_GET_TYPE(p_hwfn->p_dev)].int_regs[0]-> - mask_addr, 0x80000); + if (CHIP_REV_IS_EMUL(p_dev) && + (ECORE_IS_AH(p_dev))) + ecore_wr(p_hwfn, p_ptt, MISCS_REG_RESET_PL_HV_2_K2_E5, + 0x3ffffff); /* initialize port mode to 4x10G_E (10G with 4x10 SERDES) */ /* CNIG_REG_NW_PORT_MODE is same for A0 and B0 */ - if (!CHIP_REV_IS_EMUL(p_hwfn->p_dev) || !ECORE_IS_AH(p_hwfn->p_dev)) - ecore_wr(p_hwfn, p_ptt, CNIG_REG_NW_PORT_MODE_BB_B0, 4); + if (!CHIP_REV_IS_EMUL(p_dev) || ECORE_IS_BB(p_dev)) + ecore_wr(p_hwfn, p_ptt, CNIG_REG_NW_PORT_MODE_BB, 4); - if (CHIP_REV_IS_EMUL(p_hwfn->p_dev) && ECORE_IS_AH(p_hwfn->p_dev)) { - /* 2 for 4-port, 1 for 2-port, 0 for 1-port */ - ecore_wr(p_hwfn, p_ptt, MISC_REG_PORT_MODE, - (p_hwfn->p_dev->num_ports_in_engines >> 1)); + if (CHIP_REV_IS_EMUL(p_dev)) { + if (ECORE_IS_AH(p_dev)) { + /* 2 for 4-port, 1 for 2-port, 0 for 1-port */ + ecore_wr(p_hwfn, p_ptt, MISC_REG_PORT_MODE, + (p_dev->num_ports_in_engine >> 1)); - ecore_wr(p_hwfn, p_ptt, MISC_REG_BLOCK_256B_EN, - p_hwfn->p_dev->num_ports_in_engines == 4 ? 0 : 3); + ecore_wr(p_hwfn, p_ptt, MISC_REG_BLOCK_256B_EN, + p_dev->num_ports_in_engine == 4 ? 0 : 3); + } } /* Poll on RBC */ @@ -745,6 +1523,8 @@ static void ecore_hw_init_chip(struct ecore_hwfn *p_hwfn, if (i == 100) DP_NOTICE(p_hwfn, true, "RBC done failed to complete in PSWRQ2\n"); + + return ECORE_SUCCESS; } #endif @@ -755,7 +1535,7 @@ static void ecore_hw_init_chip(struct ecore_hwfn *p_hwfn, static void ecore_init_cau_rt_data(struct ecore_dev *p_dev) { u32 offset = CAU_REG_SB_VAR_MEMORY_RT_OFFSET; - int i, sb_id; + int i, igu_sb_id; for_each_hwfn(p_dev, i) { struct ecore_hwfn *p_hwfn = &p_dev->hwfns[i]; @@ -765,39 +1545,99 @@ static void ecore_init_cau_rt_data(struct ecore_dev *p_dev) p_igu_info = p_hwfn->hw_info.p_igu_info; - for (sb_id = 0; sb_id < ECORE_MAPPING_MEMORY_SIZE(p_dev); - sb_id++) { - p_block = &p_igu_info->igu_map.igu_blocks[sb_id]; + for (igu_sb_id = 0; + igu_sb_id < ECORE_MAPPING_MEMORY_SIZE(p_dev); + igu_sb_id++) { + p_block = &p_igu_info->entry[igu_sb_id]; if (!p_block->is_pf) continue; ecore_init_cau_sb_entry(p_hwfn, &sb_entry, p_block->function_id, 0, 0); - STORE_RT_REG_AGG(p_hwfn, offset + sb_id * 2, sb_entry); + STORE_RT_REG_AGG(p_hwfn, offset + igu_sb_id * 2, + sb_entry); } } } +static void ecore_init_cache_line_size(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt) +{ + u32 val, wr_mbs, cache_line_size; + + val = ecore_rd(p_hwfn, p_ptt, PSWRQ2_REG_WR_MBS0); + switch (val) { + case 0: + wr_mbs = 128; + break; + case 1: + wr_mbs = 256; + break; + case 2: + wr_mbs = 512; + break; + default: + DP_INFO(p_hwfn, + "Unexpected value of PSWRQ2_REG_WR_MBS0 [0x%x]. Avoid configuring PGLUE_B_REG_CACHE_LINE_SIZE.\n", + val); + return; + } + + cache_line_size = OSAL_MIN_T(u32, OSAL_CACHE_LINE_SIZE, wr_mbs); + switch (cache_line_size) { + case 32: + val = 0; + break; + case 64: + val = 1; + break; + case 128: + val = 2; + break; + case 256: + val = 3; + break; + default: + DP_INFO(p_hwfn, + "Unexpected value of cache line size [0x%x]. Avoid configuring PGLUE_B_REG_CACHE_LINE_SIZE.\n", + cache_line_size); + } + + if (wr_mbs < OSAL_CACHE_LINE_SIZE) + DP_INFO(p_hwfn, + "The cache line size for padding is suboptimal for performance [OS cache line size 0x%x, wr mbs 0x%x]\n", + OSAL_CACHE_LINE_SIZE, wr_mbs); + + STORE_RT_REG(p_hwfn, PGLUE_REG_B_CACHE_LINE_SIZE_RT_OFFSET, val); + if (val > 0) { + STORE_RT_REG(p_hwfn, PSWRQ2_REG_DRAM_ALIGN_WR_RT_OFFSET, val); + STORE_RT_REG(p_hwfn, PSWRQ2_REG_DRAM_ALIGN_RD_RT_OFFSET, val); + } +} + static enum _ecore_status_t ecore_hw_init_common(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, int hw_mode) { struct ecore_qm_info *qm_info = &p_hwfn->qm_info; - enum _ecore_status_t rc = ECORE_SUCCESS; struct ecore_dev *p_dev = p_hwfn->p_dev; u8 vf_id, max_num_vfs; u16 num_pfs, pf_id; u32 concrete_fid; + enum _ecore_status_t rc = ECORE_SUCCESS; ecore_init_cau_rt_data(p_dev); /* Program GTT windows */ - ecore_gtt_init(p_hwfn); + ecore_gtt_init(p_hwfn, p_ptt); #ifndef ASIC_ONLY - if (CHIP_REV_IS_EMUL(p_hwfn->p_dev)) - ecore_hw_init_chip(p_hwfn, p_hwfn->p_main_ptt); + if (CHIP_REV_IS_EMUL(p_dev)) { + rc = ecore_hw_init_chip(p_hwfn, p_ptt); + if (rc != ECORE_SUCCESS) + return rc; + } #endif if (p_hwfn->mcp_info) { @@ -808,7 +1648,7 @@ static enum _ecore_status_t ecore_hw_init_common(struct ecore_hwfn *p_hwfn, } ecore_qm_common_rt_init(p_hwfn, - p_hwfn->p_dev->num_ports_in_engines, + p_dev->num_ports_in_engine, qm_info->max_phys_tcs_per_port, qm_info->pf_rl_en, qm_info->pf_wfq_en, qm_info->vport_rl_en, qm_info->vport_wfq_en, @@ -816,17 +1656,7 @@ static enum _ecore_status_t ecore_hw_init_common(struct ecore_hwfn *p_hwfn, ecore_cxt_hw_init_common(p_hwfn); - /* Close gate from NIG to BRB/Storm; By default they are open, but - * we close them to prevent NIG from passing data to reset blocks. - * Should have been done in the ENGINE phase, but init-tool lacks - * proper port-pretend capabilities. - */ - ecore_wr(p_hwfn, p_ptt, NIG_REG_RX_BRB_OUT_EN, 0); - ecore_wr(p_hwfn, p_ptt, NIG_REG_STORM_OUT_EN, 0); - ecore_port_pretend(p_hwfn, p_ptt, p_hwfn->port_id ^ 1); - ecore_wr(p_hwfn, p_ptt, NIG_REG_RX_BRB_OUT_EN, 0); - ecore_wr(p_hwfn, p_ptt, NIG_REG_STORM_OUT_EN, 0); - ecore_port_unpretend(p_hwfn, p_ptt); + ecore_init_cache_line_size(p_hwfn, p_ptt); rc = ecore_init_run(p_hwfn, p_ptt, PHASE_ENGINE, ANY_PHASE_ID, hw_mode); if (rc != ECORE_SUCCESS) @@ -838,10 +1668,16 @@ static enum _ecore_status_t ecore_hw_init_common(struct ecore_hwfn *p_hwfn, ecore_wr(p_hwfn, p_ptt, PSWRQ2_REG_L2P_VALIDATE_VFID, 0); ecore_wr(p_hwfn, p_ptt, PGLUE_B_REG_USE_CLIENTID_IN_TAG, 1); - if (ECORE_IS_BB(p_hwfn->p_dev)) { - num_pfs = NUM_OF_ENG_PFS(p_hwfn->p_dev); - if (num_pfs == 1) - return rc; + if (ECORE_IS_BB(p_dev)) { + /* Workaround clears ROCE search for all functions to prevent + * involving non initialized function in processing ROCE packet. + */ + num_pfs = NUM_OF_ENG_PFS(p_dev); + for (pf_id = 0; pf_id < num_pfs; pf_id++) { + ecore_fid_pretend(p_hwfn, p_ptt, pf_id); + ecore_wr(p_hwfn, p_ptt, PRS_REG_SEARCH_ROCE, 0x0); + ecore_wr(p_hwfn, p_ptt, PRS_REG_SEARCH_TCP, 0x0); + } /* pretend to original PF */ ecore_fid_pretend(p_hwfn, p_ptt, p_hwfn->rel_pf_id); } @@ -852,12 +1688,14 @@ static enum _ecore_status_t ecore_hw_init_common(struct ecore_hwfn *p_hwfn, * This is not done inside the init tool since it currently can't * perform a pretending to VFs. */ - max_num_vfs = ECORE_IS_AH(p_hwfn->p_dev) ? MAX_NUM_VFS_K2 - : MAX_NUM_VFS_BB; + max_num_vfs = ECORE_IS_AH(p_dev) ? MAX_NUM_VFS_K2 : MAX_NUM_VFS_BB; for (vf_id = 0; vf_id < max_num_vfs; vf_id++) { concrete_fid = ecore_vfid_to_concrete(p_hwfn, vf_id); ecore_fid_pretend(p_hwfn, p_ptt, (u16)concrete_fid); ecore_wr(p_hwfn, p_ptt, CCFC_REG_STRONG_ENABLE_VF, 0x1); + ecore_wr(p_hwfn, p_ptt, CCFC_REG_WEAK_ENABLE_VF, 0x0); + ecore_wr(p_hwfn, p_ptt, TCFC_REG_STRONG_ENABLE_VF, 0x1); + ecore_wr(p_hwfn, p_ptt, TCFC_REG_WEAK_ENABLE_VF, 0x0); } /* pretend to original PF */ ecore_fid_pretend(p_hwfn, p_ptt, p_hwfn->rel_pf_id); @@ -877,20 +1715,19 @@ static void ecore_wr_nw_port(struct ecore_hwfn *p_hwfn, { DP_VERBOSE(p_hwfn, ECORE_MSG_LINK, "CMD: %08x, ADDR: 0x%08x, DATA: %08x:%08x\n", - ecore_rd(p_hwfn, p_ptt, CNIG_REG_PMEG_IF_CMD_BB_B0) | + ecore_rd(p_hwfn, p_ptt, CNIG_REG_PMEG_IF_CMD_BB) | (8 << PMEG_IF_BYTE_COUNT), (reg_type << 25) | (addr << 8) | port, (u32)((data >> 32) & 0xffffffff), (u32)(data & 0xffffffff)); - ecore_wr(p_hwfn, p_ptt, CNIG_REG_PMEG_IF_CMD_BB_B0, - (ecore_rd(p_hwfn, p_ptt, CNIG_REG_PMEG_IF_CMD_BB_B0) & + ecore_wr(p_hwfn, p_ptt, CNIG_REG_PMEG_IF_CMD_BB, + (ecore_rd(p_hwfn, p_ptt, CNIG_REG_PMEG_IF_CMD_BB) & 0xffff00fe) | (8 << PMEG_IF_BYTE_COUNT)); - ecore_wr(p_hwfn, p_ptt, CNIG_REG_PMEG_IF_ADDR_BB_B0, + ecore_wr(p_hwfn, p_ptt, CNIG_REG_PMEG_IF_ADDR_BB, (reg_type << 25) | (addr << 8) | port); - ecore_wr(p_hwfn, p_ptt, CNIG_REG_PMEG_IF_WRDATA_BB_B0, - data & 0xffffffff); - ecore_wr(p_hwfn, p_ptt, CNIG_REG_PMEG_IF_WRDATA_BB_B0, + ecore_wr(p_hwfn, p_ptt, CNIG_REG_PMEG_IF_WRDATA_BB, data & 0xffffffff); + ecore_wr(p_hwfn, p_ptt, CNIG_REG_PMEG_IF_WRDATA_BB, (data >> 32) & 0xffffffff); } @@ -906,53 +1743,22 @@ static void ecore_wr_nw_port(struct ecore_hwfn *p_hwfn, #define XLMAC_PAUSE_CTRL (0x60d) #define XLMAC_PFC_CTRL (0x60e) -static void ecore_emul_link_init_ah(struct ecore_hwfn *p_hwfn, +static void ecore_emul_link_init_bb(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt) -{ - u8 port = p_hwfn->port_id; - u32 mac_base = NWM_REG_MAC0 + (port << 2) * NWM_REG_MAC0_SIZE; - - ecore_wr(p_hwfn, p_ptt, CNIG_REG_NIG_PORT0_CONF_K2 + (port << 2), - (1 << CNIG_REG_NIG_PORT0_CONF_NIG_PORT_ENABLE_0_SHIFT) | - (port << CNIG_REG_NIG_PORT0_CONF_NIG_PORT_NWM_PORT_MAP_0_SHIFT) - | (0 << CNIG_REG_NIG_PORT0_CONF_NIG_PORT_RATE_0_SHIFT)); - - ecore_wr(p_hwfn, p_ptt, mac_base + ETH_MAC_REG_XIF_MODE, - 1 << ETH_MAC_REG_XIF_MODE_XGMII_SHIFT); - - ecore_wr(p_hwfn, p_ptt, mac_base + ETH_MAC_REG_FRM_LENGTH, - 9018 << ETH_MAC_REG_FRM_LENGTH_FRM_LENGTH_SHIFT); - - ecore_wr(p_hwfn, p_ptt, mac_base + ETH_MAC_REG_TX_IPG_LENGTH, - 0xc << ETH_MAC_REG_TX_IPG_LENGTH_TXIPG_SHIFT); - - ecore_wr(p_hwfn, p_ptt, mac_base + ETH_MAC_REG_RX_FIFO_SECTIONS, - 8 << ETH_MAC_REG_RX_FIFO_SECTIONS_RX_SECTION_FULL_SHIFT); - - ecore_wr(p_hwfn, p_ptt, mac_base + ETH_MAC_REG_TX_FIFO_SECTIONS, - (0xA << ETH_MAC_REG_TX_FIFO_SECTIONS_TX_SECTION_EMPTY_SHIFT) | - (8 << ETH_MAC_REG_TX_FIFO_SECTIONS_TX_SECTION_FULL_SHIFT)); - - ecore_wr(p_hwfn, p_ptt, mac_base + ETH_MAC_REG_COMMAND_CONFIG, 0xa853); -} - -static void ecore_emul_link_init(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt) { u8 loopback = 0, port = p_hwfn->port_id * 2; DP_INFO(p_hwfn->p_dev, "Configurating Emulation Link %02x\n", port); - if (ECORE_IS_AH(p_hwfn->p_dev)) { - ecore_emul_link_init_ah(p_hwfn, p_ptt); - return; - } - + /* XLPORT MAC MODE *//* 0 Quad, 4 Single... */ ecore_wr_nw_port(p_hwfn, p_ptt, XLPORT_MODE_REG, (0x4 << 4) | 0x4, 1, - port); + port); ecore_wr_nw_port(p_hwfn, p_ptt, XLPORT_MAC_CONTROL, 0, 1, port); + /* XLMAC: SOFT RESET */ ecore_wr_nw_port(p_hwfn, p_ptt, XLMAC_CTRL, 0x40, 0, port); + /* XLMAC: Port Speed >= 10Gbps */ ecore_wr_nw_port(p_hwfn, p_ptt, XLMAC_MODE, 0x40, 0, port); + /* XLMAC: Max Size */ ecore_wr_nw_port(p_hwfn, p_ptt, XLMAC_RX_MAX_SIZE, 0x3fff, 0, port); ecore_wr_nw_port(p_hwfn, p_ptt, XLMAC_TX_CTRL, 0x01000000800ULL | (0xa << 12) | ((u64)1 << 38), @@ -961,15 +1767,64 @@ static void ecore_emul_link_init(struct ecore_hwfn *p_hwfn, ecore_wr_nw_port(p_hwfn, p_ptt, XLMAC_PFC_CTRL, 0x30ffffc000ULL, 0, port); ecore_wr_nw_port(p_hwfn, p_ptt, XLMAC_CTRL, 0x3 | (loopback << 2), 0, - port); - ecore_wr_nw_port(p_hwfn, p_ptt, XLMAC_CTRL, 0x1003 | (loopback << 2), - 0, port); + port); /* XLMAC: TX_EN, RX_EN */ + /* XLMAC: TX_EN, RX_EN, SW_LINK_STATUS */ + ecore_wr_nw_port(p_hwfn, p_ptt, XLMAC_CTRL, + 0x1003 | (loopback << 2), 0, port); + /* Enabled Parallel PFC interface */ ecore_wr_nw_port(p_hwfn, p_ptt, XLPORT_FLOW_CONTROL_CONFIG, 1, 0, port); + + /* XLPORT port enable */ ecore_wr_nw_port(p_hwfn, p_ptt, XLPORT_ENABLE_REG, 0xf, 1, port); } -static void ecore_link_init(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, u8 port) +static void ecore_emul_link_init_ah_e5(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt) +{ + u8 port = p_hwfn->port_id; + u32 mac_base = NWM_REG_MAC0_K2_E5 + (port << 2) * NWM_REG_MAC0_SIZE; + + DP_INFO(p_hwfn->p_dev, "Configurating Emulation Link %02x\n", port); + + ecore_wr(p_hwfn, p_ptt, CNIG_REG_NIG_PORT0_CONF_K2_E5 + (port << 2), + (1 << CNIG_REG_NIG_PORT0_CONF_NIG_PORT_ENABLE_0_K2_E5_SHIFT) | + (port << + CNIG_REG_NIG_PORT0_CONF_NIG_PORT_NWM_PORT_MAP_0_K2_E5_SHIFT) | + (0 << CNIG_REG_NIG_PORT0_CONF_NIG_PORT_RATE_0_K2_E5_SHIFT)); + + ecore_wr(p_hwfn, p_ptt, mac_base + ETH_MAC_REG_XIF_MODE_K2_E5, + 1 << ETH_MAC_REG_XIF_MODE_XGMII_K2_E5_SHIFT); + + ecore_wr(p_hwfn, p_ptt, mac_base + ETH_MAC_REG_FRM_LENGTH_K2_E5, + 9018 << ETH_MAC_REG_FRM_LENGTH_FRM_LENGTH_K2_E5_SHIFT); + + ecore_wr(p_hwfn, p_ptt, mac_base + ETH_MAC_REG_TX_IPG_LENGTH_K2_E5, + 0xc << ETH_MAC_REG_TX_IPG_LENGTH_TXIPG_K2_E5_SHIFT); + + ecore_wr(p_hwfn, p_ptt, mac_base + ETH_MAC_REG_RX_FIFO_SECTIONS_K2_E5, + 8 << ETH_MAC_REG_RX_FIFO_SECTIONS_RX_SECTION_FULL_K2_E5_SHIFT); + + ecore_wr(p_hwfn, p_ptt, mac_base + ETH_MAC_REG_TX_FIFO_SECTIONS_K2_E5, + (0xA << + ETH_MAC_REG_TX_FIFO_SECTIONS_TX_SECTION_EMPTY_K2_E5_SHIFT) | + (8 << + ETH_MAC_REG_TX_FIFO_SECTIONS_TX_SECTION_FULL_K2_E5_SHIFT)); + + ecore_wr(p_hwfn, p_ptt, mac_base + ETH_MAC_REG_COMMAND_CONFIG_K2_E5, + 0xa853); +} + +static void ecore_emul_link_init(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt) +{ + if (ECORE_IS_AH(p_hwfn->p_dev)) + ecore_emul_link_init_ah_e5(p_hwfn, p_ptt); + else /* BB */ + ecore_emul_link_init_bb(p_hwfn, p_ptt); +} + +static void ecore_link_init_bb(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, u8 port) { int port_offset = port ? 0x800 : 0; u32 xmac_rxctrl = 0; @@ -977,15 +1832,15 @@ static void ecore_link_init(struct ecore_hwfn *p_hwfn, /* Reset of XMAC */ /* FIXME: move to common start */ ecore_wr(p_hwfn, p_ptt, MISC_REG_RESET_PL_PDA_VAUX + 2 * sizeof(u32), - MISC_REG_RESET_REG_2_XMAC_BIT); /* Clear */ + MISC_REG_RESET_REG_2_XMAC_BIT); /* Clear */ OSAL_MSLEEP(1); ecore_wr(p_hwfn, p_ptt, MISC_REG_RESET_PL_PDA_VAUX + sizeof(u32), - MISC_REG_RESET_REG_2_XMAC_BIT); /* Set */ + MISC_REG_RESET_REG_2_XMAC_BIT); /* Set */ - ecore_wr(p_hwfn, p_ptt, MISC_REG_XMAC_CORE_PORT_MODE, 1); + ecore_wr(p_hwfn, p_ptt, MISC_REG_XMAC_CORE_PORT_MODE_BB, 1); /* Set the number of ports on the Warp Core to 10G */ - ecore_wr(p_hwfn, p_ptt, MISC_REG_XMAC_PHY_PORT_MODE, 3); + ecore_wr(p_hwfn, p_ptt, MISC_REG_XMAC_PHY_PORT_MODE_BB, 3); /* Soft reset of XMAC */ ecore_wr(p_hwfn, p_ptt, MISC_REG_RESET_PL_PDA_VAUX + 2 * sizeof(u32), @@ -996,35 +1851,218 @@ static void ecore_link_init(struct ecore_hwfn *p_hwfn, /* FIXME: move to common end */ if (CHIP_REV_IS_FPGA(p_hwfn->p_dev)) - ecore_wr(p_hwfn, p_ptt, XMAC_REG_MODE + port_offset, 0x20); + ecore_wr(p_hwfn, p_ptt, XMAC_REG_MODE_BB + port_offset, 0x20); /* Set Max packet size: initialize XMAC block register for port 0 */ - ecore_wr(p_hwfn, p_ptt, XMAC_REG_RX_MAX_SIZE + port_offset, 0x2710); + ecore_wr(p_hwfn, p_ptt, XMAC_REG_RX_MAX_SIZE_BB + port_offset, 0x2710); /* CRC append for Tx packets: init XMAC block register for port 1 */ - ecore_wr(p_hwfn, p_ptt, XMAC_REG_TX_CTRL_LO + port_offset, 0xC800); + ecore_wr(p_hwfn, p_ptt, XMAC_REG_TX_CTRL_LO_BB + port_offset, 0xC800); /* Enable TX and RX: initialize XMAC block register for port 1 */ - ecore_wr(p_hwfn, p_ptt, XMAC_REG_CTRL + port_offset, - XMAC_REG_CTRL_TX_EN | XMAC_REG_CTRL_RX_EN); - xmac_rxctrl = ecore_rd(p_hwfn, p_ptt, XMAC_REG_RX_CTRL + port_offset); - xmac_rxctrl |= XMAC_REG_RX_CTRL_PROCESS_VARIABLE_PREAMBLE; - ecore_wr(p_hwfn, p_ptt, XMAC_REG_RX_CTRL + port_offset, xmac_rxctrl); + ecore_wr(p_hwfn, p_ptt, XMAC_REG_CTRL_BB + port_offset, + XMAC_REG_CTRL_TX_EN_BB | XMAC_REG_CTRL_RX_EN_BB); + xmac_rxctrl = ecore_rd(p_hwfn, p_ptt, + XMAC_REG_RX_CTRL_BB + port_offset); + xmac_rxctrl |= XMAC_REG_RX_CTRL_PROCESS_VARIABLE_PREAMBLE_BB; + ecore_wr(p_hwfn, p_ptt, XMAC_REG_RX_CTRL_BB + port_offset, xmac_rxctrl); } #endif +static enum _ecore_status_t +ecore_hw_init_dpi_size(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, u32 pwm_region_size, u32 n_cpus) +{ + u32 dpi_bit_shift, dpi_count, dpi_page_size; + u32 min_dpis; + u32 n_wids; + + /* Calculate DPI size + * ------------------ + * The PWM region contains Doorbell Pages. The first is reserverd for + * the kernel for, e.g, L2. The others are free to be used by non- + * trusted applications, typically from user space. Each page, called a + * doorbell page is sectioned into windows that allow doorbells to be + * issued in parallel by the kernel/application. The size of such a + * window (a.k.a. WID) is 1kB. + * Summary: + * 1kB WID x N WIDS = DPI page size + * DPI page size x N DPIs = PWM region size + * Notes: + * The size of the DPI page size must be in multiples of OSAL_PAGE_SIZE + * in order to ensure that two applications won't share the same page. + * It also must contain at least one WID per CPU to allow parallelism. + * It also must be a power of 2, since it is stored as a bit shift. + * + * The DPI page size is stored in a register as 'dpi_bit_shift' so that + * 0 is 4kB, 1 is 8kB and etc. Hence the minimum size is 4,096 + * containing 4 WIDs. + */ + n_wids = OSAL_MAX_T(u32, ECORE_MIN_WIDS, n_cpus); + dpi_page_size = ECORE_WID_SIZE * OSAL_ROUNDUP_POW_OF_TWO(n_wids); + dpi_page_size = (dpi_page_size + OSAL_PAGE_SIZE - 1) & + ~(OSAL_PAGE_SIZE - 1); + dpi_bit_shift = OSAL_LOG2(dpi_page_size / 4096); + dpi_count = pwm_region_size / dpi_page_size; + + min_dpis = p_hwfn->pf_params.rdma_pf_params.min_dpis; + min_dpis = OSAL_MAX_T(u32, ECORE_MIN_DPIS, min_dpis); + + /* Update hwfn */ + p_hwfn->dpi_size = dpi_page_size; + p_hwfn->dpi_count = dpi_count; + + /* Update registers */ + ecore_wr(p_hwfn, p_ptt, DORQ_REG_PF_DPI_BIT_SHIFT, dpi_bit_shift); + + if (dpi_count < min_dpis) + return ECORE_NORESOURCES; + + return ECORE_SUCCESS; +} + +enum ECORE_ROCE_EDPM_MODE { + ECORE_ROCE_EDPM_MODE_ENABLE = 0, + ECORE_ROCE_EDPM_MODE_FORCE_ON = 1, + ECORE_ROCE_EDPM_MODE_DISABLE = 2, +}; + +static enum _ecore_status_t +ecore_hw_init_pf_doorbell_bar(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt) +{ + u32 pwm_regsize, norm_regsize; + u32 non_pwm_conn, min_addr_reg1; + u32 db_bar_size, n_cpus; + u32 roce_edpm_mode; + u32 pf_dems_shift; + enum _ecore_status_t rc = ECORE_SUCCESS; + u8 cond; + + db_bar_size = ecore_hw_bar_size(p_hwfn, p_ptt, BAR_ID_1); + if (ECORE_IS_CMT(p_hwfn->p_dev)) + db_bar_size /= 2; + + /* Calculate doorbell regions + * ----------------------------------- + * The doorbell BAR is made of two regions. The first is called normal + * region and the second is called PWM region. In the normal region + * each ICID has its own set of addresses so that writing to that + * specific address identifies the ICID. In the Process Window Mode + * region the ICID is given in the data written to the doorbell. The + * above per PF register denotes the offset in the doorbell BAR in which + * the PWM region begins. + * The normal region has ECORE_PF_DEMS_SIZE bytes per ICID, that is per + * non-PWM connection. The calculation below computes the total non-PWM + * connections. The DORQ_REG_PF_MIN_ADDR_REG1 register is + * in units of 4,096 bytes. + */ + non_pwm_conn = ecore_cxt_get_proto_cid_start(p_hwfn, PROTOCOLID_CORE) + + ecore_cxt_get_proto_cid_count(p_hwfn, PROTOCOLID_CORE, + OSAL_NULL) + + ecore_cxt_get_proto_cid_count(p_hwfn, PROTOCOLID_ETH, OSAL_NULL); + norm_regsize = ROUNDUP(ECORE_PF_DEMS_SIZE * non_pwm_conn, + OSAL_PAGE_SIZE); + min_addr_reg1 = norm_regsize / 4096; + pwm_regsize = db_bar_size - norm_regsize; + + /* Check that the normal and PWM sizes are valid */ + if (db_bar_size < norm_regsize) { + DP_ERR(p_hwfn->p_dev, + "Doorbell BAR size 0x%x is too small (normal region is 0x%0x )\n", + db_bar_size, norm_regsize); + return ECORE_NORESOURCES; + } + if (pwm_regsize < ECORE_MIN_PWM_REGION) { + DP_ERR(p_hwfn->p_dev, + "PWM region size 0x%0x is too small. Should be at least 0x%0x (Doorbell BAR size is 0x%x and normal region size is 0x%0x)\n", + pwm_regsize, ECORE_MIN_PWM_REGION, db_bar_size, + norm_regsize); + return ECORE_NORESOURCES; + } + + /* Calculate number of DPIs */ + roce_edpm_mode = p_hwfn->pf_params.rdma_pf_params.roce_edpm_mode; + if ((roce_edpm_mode == ECORE_ROCE_EDPM_MODE_ENABLE) || + ((roce_edpm_mode == ECORE_ROCE_EDPM_MODE_FORCE_ON))) { + /* Either EDPM is mandatory, or we are attempting to allocate a + * WID per CPU. + */ + n_cpus = OSAL_NUM_CPUS(); + rc = ecore_hw_init_dpi_size(p_hwfn, p_ptt, pwm_regsize, n_cpus); + } + + cond = ((rc != ECORE_SUCCESS) && + (roce_edpm_mode == ECORE_ROCE_EDPM_MODE_ENABLE)) || + (roce_edpm_mode == ECORE_ROCE_EDPM_MODE_DISABLE); + if (cond || p_hwfn->dcbx_no_edpm) { + /* Either EDPM is disabled from user configuration, or it is + * disabled via DCBx, or it is not mandatory and we failed to + * allocated a WID per CPU. + */ + n_cpus = 1; + rc = ecore_hw_init_dpi_size(p_hwfn, p_ptt, pwm_regsize, n_cpus); + + /* If we entered this flow due to DCBX then the DPM register is + * already configured. + */ + } + + DP_INFO(p_hwfn, + "doorbell bar: normal_region_size=%d, pwm_region_size=%d", + norm_regsize, pwm_regsize); + DP_INFO(p_hwfn, + " dpi_size=%d, dpi_count=%d, roce_edpm=%s\n", + p_hwfn->dpi_size, p_hwfn->dpi_count, + ((p_hwfn->dcbx_no_edpm) || (p_hwfn->db_bar_no_edpm)) ? + "disabled" : "enabled"); + + /* Check return codes from above calls */ + if (rc != ECORE_SUCCESS) { + DP_ERR(p_hwfn, + "Failed to allocate enough DPIs\n"); + return ECORE_NORESOURCES; + } + + /* Update hwfn */ + p_hwfn->dpi_start_offset = norm_regsize; + + /* Update registers */ + /* DEMS size is configured log2 of DWORDs, hence the division by 4 */ + pf_dems_shift = OSAL_LOG2(ECORE_PF_DEMS_SIZE / 4); + ecore_wr(p_hwfn, p_ptt, DORQ_REG_PF_ICID_BIT_SHIFT_NORM, pf_dems_shift); + ecore_wr(p_hwfn, p_ptt, DORQ_REG_PF_MIN_ADDR_REG1, min_addr_reg1); + + return ECORE_SUCCESS; +} + static enum _ecore_status_t ecore_hw_init_port(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, int hw_mode) { - enum _ecore_status_t rc = ECORE_SUCCESS; + u32 ppf_to_eng_sel[NIG_REG_PPF_TO_ENGINE_SEL_RT_SIZE]; + u32 val; + enum _ecore_status_t rc = ECORE_SUCCESS; + u8 i; + + /* In CMT for non-RoCE packets - use connection based classification */ + val = ECORE_IS_CMT(p_hwfn->p_dev) ? 0x8 : 0x0; + for (i = 0; i < NIG_REG_PPF_TO_ENGINE_SEL_RT_SIZE; i++) + ppf_to_eng_sel[i] = val; + STORE_RT_REG_AGG(p_hwfn, NIG_REG_PPF_TO_ENGINE_SEL_RT_OFFSET, + ppf_to_eng_sel); + + /* In CMT the gate should be cleared by the 2nd hwfn */ + if (!ECORE_IS_CMT(p_hwfn->p_dev) || !IS_LEAD_HWFN(p_hwfn)) + STORE_RT_REG(p_hwfn, NIG_REG_BRB_GATE_DNTFWD_PORT_RT_OFFSET, 0); - /* Init sequence */ rc = ecore_init_run(p_hwfn, p_ptt, PHASE_PORT, p_hwfn->port_id, hw_mode); if (rc != ECORE_SUCCESS) return rc; + ecore_wr(p_hwfn, p_ptt, PGLUE_B_REG_MASTER_WRITE_PAD_ENABLE, 0); + #ifndef ASIC_ONLY if (CHIP_REV_IS_ASIC(p_hwfn->p_dev)) return ECORE_SUCCESS; @@ -1032,9 +2070,10 @@ static enum _ecore_status_t ecore_hw_init_port(struct ecore_hwfn *p_hwfn, if (CHIP_REV_IS_FPGA(p_hwfn->p_dev)) { if (ECORE_IS_AH(p_hwfn->p_dev)) return ECORE_SUCCESS; - ecore_link_init(p_hwfn, p_ptt, p_hwfn->port_id); + else if (ECORE_IS_BB(p_hwfn->p_dev)) + ecore_link_init_bb(p_hwfn, p_ptt, p_hwfn->port_id); } else if (CHIP_REV_IS_EMUL(p_hwfn->p_dev)) { - if (p_hwfn->p_dev->num_hwfns > 1) { + if (ECORE_IS_CMT(p_hwfn->p_dev)) { /* Activate OPTE in CMT */ u32 val; @@ -1062,96 +2101,20 @@ static enum _ecore_status_t ecore_hw_init_port(struct ecore_hwfn *p_hwfn, return rc; } -static enum _ecore_status_t -ecore_hw_init_pf_doorbell_bar(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt) -{ - u32 pwm_regsize, norm_regsize; - u32 non_pwm_conn, min_addr_reg1; - u32 db_bar_size, n_cpus; - u32 pf_dems_shift; - int rc = ECORE_SUCCESS; - - db_bar_size = ecore_hw_bar_size(p_hwfn, BAR_ID_1); - if (p_hwfn->p_dev->num_hwfns > 1) - db_bar_size /= 2; - - /* Calculate doorbell regions - * ----------------------------------- - * The doorbell BAR is made of two regions. The first is called normal - * region and the second is called PWM region. In the normal region - * each ICID has its own set of addresses so that writing to that - * specific address identifies the ICID. In the Process Window Mode - * region the ICID is given in the data written to the doorbell. The - * above per PF register denotes the offset in the doorbell BAR in which - * the PWM region begins. - * The normal region has ECORE_PF_DEMS_SIZE bytes per ICID, that is per - * non-PWM connection. The calculation below computes the total non-PWM - * connections. The DORQ_REG_PF_MIN_ADDR_REG1 register is - * in units of 4,096 bytes. - */ - non_pwm_conn = ecore_cxt_get_proto_cid_start(p_hwfn, PROTOCOLID_CORE) + - ecore_cxt_get_proto_cid_count(p_hwfn, PROTOCOLID_CORE, - OSAL_NULL) + - ecore_cxt_get_proto_cid_count(p_hwfn, PROTOCOLID_ETH, OSAL_NULL); - norm_regsize = ROUNDUP(ECORE_PF_DEMS_SIZE * non_pwm_conn, 4096); - min_addr_reg1 = norm_regsize / 4096; - pwm_regsize = db_bar_size - norm_regsize; - - /* Check that the normal and PWM sizes are valid */ - if (db_bar_size < norm_regsize) { - DP_ERR(p_hwfn->p_dev, - "Doorbell BAR size 0x%x is too" - " small (normal region is 0x%0x )\n", - db_bar_size, norm_regsize); - return ECORE_NORESOURCES; - } - if (pwm_regsize < ECORE_MIN_PWM_REGION) { - DP_ERR(p_hwfn->p_dev, - "PWM region size 0x%0x is too small." - " Should be at least 0x%0x (Doorbell BAR size" - " is 0x%x and normal region size is 0x%0x)\n", - pwm_regsize, ECORE_MIN_PWM_REGION, db_bar_size, - norm_regsize); - return ECORE_NORESOURCES; - } - - /* Update hwfn */ - p_hwfn->dpi_start_offset = norm_regsize; /* this is later used to - * calculate the doorbell - * address - */ - - /* Update registers */ - /* DEMS size is configured log2 of DWORDs, hence the division by 4 */ - pf_dems_shift = OSAL_LOG2(ECORE_PF_DEMS_SIZE / 4); - ecore_wr(p_hwfn, p_ptt, DORQ_REG_PF_ICID_BIT_SHIFT_NORM, pf_dems_shift); - ecore_wr(p_hwfn, p_ptt, DORQ_REG_PF_MIN_ADDR_REG1, min_addr_reg1); - - DP_INFO(p_hwfn, - "Doorbell size 0x%x, Normal region 0x%x, PWM region 0x%x\n", - db_bar_size, norm_regsize, pwm_regsize); - DP_INFO(p_hwfn, "DPI size 0x%x, DPI count 0x%x\n", p_hwfn->dpi_size, - p_hwfn->dpi_count); - - return ECORE_SUCCESS; -} - static enum _ecore_status_t ecore_hw_init_pf(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, - struct ecore_tunn_start_params *p_tunn, + struct ecore_tunnel_info *p_tunn, int hw_mode, bool b_hw_start, enum ecore_int_mode int_mode, bool allow_npar_tx_switch) { - enum _ecore_status_t rc = ECORE_SUCCESS; u8 rel_pf_id = p_hwfn->rel_pf_id; u32 prs_reg; + enum _ecore_status_t rc = ECORE_SUCCESS; u16 ctrl; int pos; - /* ILT/DQ/CM/QM */ if (p_hwfn->mcp_info) { struct ecore_mcp_function_info *p_info; @@ -1160,11 +2123,11 @@ ecore_hw_init_pf(struct ecore_hwfn *p_hwfn, p_hwfn->qm_info.pf_wfq = p_info->bandwidth_min; /* Update rate limit once we'll actually have a link */ - p_hwfn->qm_info.pf_rl = 100; + p_hwfn->qm_info.pf_rl = 100000; } - ecore_cxt_hw_init_pf(p_hwfn); + ecore_cxt_hw_init_pf(p_hwfn, p_ptt); - ecore_int_igu_init_rt(p_hwfn); /* @@@TBD TODO MichalS multi hwfn ?? */ + ecore_int_igu_init_rt(p_hwfn); /* Set VLAN in NIG if needed */ if (hw_mode & (1 << MODE_MF_SD)) { @@ -1172,6 +2135,11 @@ ecore_hw_init_pf(struct ecore_hwfn *p_hwfn, STORE_RT_REG(p_hwfn, NIG_REG_LLH_FUNC_TAG_EN_RT_OFFSET, 1); STORE_RT_REG(p_hwfn, NIG_REG_LLH_FUNC_TAG_VALUE_RT_OFFSET, p_hwfn->hw_info.ovlan); + + DP_VERBOSE(p_hwfn, ECORE_MSG_HW, + "Configuring LLH_FUNC_FILTER_HDR_SEL\n"); + STORE_RT_REG(p_hwfn, NIG_REG_LLH_FUNC_FILTER_HDR_SEL_RT_OFFSET, + 1); } /* Enable classification by MAC if needed */ @@ -1183,18 +2151,15 @@ ecore_hw_init_pf(struct ecore_hwfn *p_hwfn, } /* Protocl Configuration - @@@TBD - should we set 0 otherwise? */ - STORE_RT_REG(p_hwfn, PRS_REG_SEARCH_TCP_RT_OFFSET, 0); + STORE_RT_REG(p_hwfn, PRS_REG_SEARCH_TCP_RT_OFFSET, + (p_hwfn->hw_info.personality == ECORE_PCI_ISCSI) ? 1 : 0); + STORE_RT_REG(p_hwfn, PRS_REG_SEARCH_FCOE_RT_OFFSET, + (p_hwfn->hw_info.personality == ECORE_PCI_FCOE) ? 1 : 0); + STORE_RT_REG(p_hwfn, PRS_REG_SEARCH_ROCE_RT_OFFSET, 0); /* perform debug configuration when chip is out of reset */ OSAL_BEFORE_PF_START((void *)p_hwfn->p_dev, p_hwfn->my_id); - /* Cleanup chip from previous driver if such remains exist */ - rc = ecore_final_cleanup(p_hwfn, p_ptt, rel_pf_id, false); - if (rc != ECORE_SUCCESS) { - ecore_hw_err_notify(p_hwfn, ECORE_HW_ERR_RAMROD_FAIL); - return rc; - } - /* PF Init sequence */ rc = ecore_init_run(p_hwfn, p_ptt, PHASE_PF, rel_pf_id, hw_mode); if (rc) @@ -1213,49 +2178,28 @@ ecore_hw_init_pf(struct ecore_hwfn *p_hwfn, * PCI config space. */ /* Not in use @DPDK - * pos = OSAL_PCI_FIND_CAPABILITY(p_hwfn->p_dev, PCI_CAP_ID_EXP); - * if (!pos) { - * DP_NOTICE(p_hwfn, true, - * "Failed to find the PCI Express" - * " Capability structure in the PCI config space\n"); - * return ECORE_IO; - * } - * OSAL_PCI_READ_CONFIG_WORD(p_hwfn->p_dev, pos + PCI_EXP_DEVCTL, - * &ctrl); - * ctrl &= ~PCI_EXP_DEVCTL_RELAX_EN; - * OSAL_PCI_WRITE_CONFIG_WORD(p_hwfn->p_dev, pos + PCI_EXP_DEVCTL, - * &ctrl); - */ - -#ifndef ASIC_ONLY - /*@@TMP - On B0 build 1, need to mask the datapath_registers parity */ - if (CHIP_REV_IS_EMUL_B0(p_hwfn->p_dev) && - (p_hwfn->p_dev->chip_metal == 1)) { - u32 reg_addr, tmp; - - reg_addr = - attn_blocks[BLOCK_PGLUE_B]. - chip_regs[ECORE_GET_TYPE(p_hwfn->p_dev)].prty_regs[0]-> - mask_addr; - DP_NOTICE(p_hwfn, false, - "Masking datapath registers parity on" - " B0 emulation [build 1]\n"); - tmp = ecore_rd(p_hwfn, p_ptt, reg_addr); - tmp |= (1 << 0); /* Was PRTY_MASK_DATAPATH_REGISTERS */ - ecore_wr(p_hwfn, p_ptt, reg_addr, tmp); - } -#endif + * pos = OSAL_PCI_FIND_CAPABILITY(p_hwfn->p_dev, PCI_CAP_ID_EXP); + * if (!pos) { + * DP_NOTICE(p_hwfn, true, + * "Failed to find the PCIe Cap\n"); + * return ECORE_IO; + * } + * OSAL_PCI_READ_CONFIG_WORD(p_hwfn->p_dev, pos + PCI_EXP_DEVCTL, &ctrl); + * ctrl &= ~PCI_EXP_DEVCTL_RELAX_EN; + * OSAL_PCI_WRITE_CONFIG_WORD(p_hwfn->p_dev, pos + PCI_EXP_DEVCTL, ctrl); + */ rc = ecore_hw_init_pf_doorbell_bar(p_hwfn, p_ptt); if (rc) return rc; - if (b_hw_start) { /* enable interrupts */ - ecore_int_igu_enable(p_hwfn, p_ptt, int_mode); + rc = ecore_int_igu_enable(p_hwfn, p_ptt, int_mode); + if (rc != ECORE_SUCCESS) + return rc; /* send function start command */ - rc = ecore_sp_pf_start(p_hwfn, p_tunn, p_hwfn->p_dev->mf_mode, + rc = ecore_sp_pf_start(p_hwfn, p_ptt, p_tunn, allow_npar_tx_switch); if (rc) { DP_NOTICE(p_hwfn, true, @@ -1265,14 +2209,27 @@ ecore_hw_init_pf(struct ecore_hwfn *p_hwfn, DP_VERBOSE(p_hwfn, ECORE_MSG_STORAGE, "PRS_REG_SEARCH_TAG1: %x\n", prs_reg); + if (p_hwfn->hw_info.personality == ECORE_PCI_FCOE) { + ecore_wr(p_hwfn, p_ptt, PRS_REG_SEARCH_TAG1, + (1 << 2)); + ecore_wr(p_hwfn, p_ptt, + PRS_REG_PKT_LEN_STAT_TAGS_NOT_COUNTED_FIRST, + 0x100); + } DP_VERBOSE(p_hwfn, ECORE_MSG_STORAGE, - "PRS_REG_SEARCH register after start PFn\n"); + "PRS_REG_SEARCH registers after start PFn\n"); prs_reg = ecore_rd(p_hwfn, p_ptt, PRS_REG_SEARCH_TCP); DP_VERBOSE(p_hwfn, ECORE_MSG_STORAGE, "PRS_REG_SEARCH_TCP: %x\n", prs_reg); prs_reg = ecore_rd(p_hwfn, p_ptt, PRS_REG_SEARCH_UDP); DP_VERBOSE(p_hwfn, ECORE_MSG_STORAGE, "PRS_REG_SEARCH_UDP: %x\n", prs_reg); + prs_reg = ecore_rd(p_hwfn, p_ptt, PRS_REG_SEARCH_FCOE); + DP_VERBOSE(p_hwfn, ECORE_MSG_STORAGE, + "PRS_REG_SEARCH_FCOE: %x\n", prs_reg); + prs_reg = ecore_rd(p_hwfn, p_ptt, PRS_REG_SEARCH_ROCE); + DP_VERBOSE(p_hwfn, ECORE_MSG_STORAGE, + "PRS_REG_SEARCH_ROCE: %x\n", prs_reg); prs_reg = ecore_rd(p_hwfn, p_ptt, PRS_REG_SEARCH_TCP_FIRST_FRAG); DP_VERBOSE(p_hwfn, ECORE_MSG_STORAGE, @@ -1286,17 +2243,17 @@ ecore_hw_init_pf(struct ecore_hwfn *p_hwfn, return rc; } -static enum _ecore_status_t -ecore_change_pci_hwfn(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, u8 enable) +enum _ecore_status_t ecore_pglueb_set_pfid_enable(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + bool b_enable) { - u32 delay_idx = 0, val, set_val = enable ? 1 : 0; + u32 delay_idx = 0, val, set_val = b_enable ? 1 : 0; - /* Change PF in PXP */ + /* Configure the PF's internal FID_enable for master transactions */ ecore_wr(p_hwfn, p_ptt, PGLUE_B_REG_INTERNAL_PFID_ENABLE_MASTER, set_val); - /* wait until value is set - try for 1 second every 50us */ + /* Wait until value is set - try for 1 second every 50us */ for (delay_idx = 0; delay_idx < 20000; delay_idx++) { val = ecore_rd(p_hwfn, p_ptt, PGLUE_B_REG_INTERNAL_PFID_ENABLE_MASTER); @@ -1325,64 +2282,119 @@ static void ecore_reset_mb_shadow(struct ecore_hwfn *p_hwfn, p_hwfn->mcp_info->mfw_mb_length); } -enum _ecore_status_t ecore_hw_init(struct ecore_dev *p_dev, - struct ecore_tunn_start_params *p_tunn, - bool b_hw_start, - enum ecore_int_mode int_mode, - bool allow_npar_tx_switch, - const u8 *bin_fw_data) +static void ecore_pglueb_clear_err(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt) { - enum _ecore_status_t rc, mfw_rc; - u32 load_code, param; - int i, j; + ecore_wr(p_hwfn, p_ptt, PGLUE_B_REG_WAS_ERROR_PF_31_0_CLR, + 1 << p_hwfn->abs_pf_id); +} + +static void +ecore_fill_load_req_params(struct ecore_load_req_params *p_load_req, + struct ecore_drv_load_params *p_drv_load) +{ + /* Make sure that if ecore-client didn't provide inputs, all the + * expected defaults are indeed zero. + */ + OSAL_BUILD_BUG_ON(ECORE_DRV_ROLE_OS != 0); + OSAL_BUILD_BUG_ON(ECORE_LOAD_REQ_LOCK_TO_DEFAULT != 0); + OSAL_BUILD_BUG_ON(ECORE_OVERRIDE_FORCE_LOAD_NONE != 0); + + OSAL_MEM_ZERO(p_load_req, sizeof(*p_load_req)); + + if (p_drv_load != OSAL_NULL) { + p_load_req->drv_role = p_drv_load->is_crash_kernel ? + ECORE_DRV_ROLE_KDUMP : + ECORE_DRV_ROLE_OS; + p_load_req->timeout_val = p_drv_load->mfw_timeout_val; + p_load_req->avoid_eng_reset = p_drv_load->avoid_eng_reset; + p_load_req->override_force_load = + p_drv_load->override_force_load; + } +} + +enum _ecore_status_t ecore_vf_start(struct ecore_hwfn *p_hwfn, + struct ecore_hw_init_params *p_params) +{ + if (p_params->p_tunn) { + ecore_vf_set_vf_start_tunn_update_param(p_params->p_tunn); + ecore_vf_pf_tunnel_param_update(p_hwfn, p_params->p_tunn); + } + + p_hwfn->b_int_enabled = 1; + + return ECORE_SUCCESS; +} + +enum _ecore_status_t ecore_hw_init(struct ecore_dev *p_dev, + struct ecore_hw_init_params *p_params) +{ + struct ecore_load_req_params load_req_params; + u32 load_code, resp, param, drv_mb_param; + bool b_default_mtu = true; + struct ecore_hwfn *p_hwfn; + enum _ecore_status_t rc = ECORE_SUCCESS; + int i; + + if ((p_params->int_mode == ECORE_INT_MODE_MSI) && ECORE_IS_CMT(p_dev)) { + DP_NOTICE(p_dev, false, + "MSI mode is not supported for CMT devices\n"); + return ECORE_INVAL; + } if (IS_PF(p_dev)) { - rc = ecore_init_fw_data(p_dev, bin_fw_data); + rc = ecore_init_fw_data(p_dev, p_params->bin_fw_data); if (rc != ECORE_SUCCESS) return rc; } for_each_hwfn(p_dev, i) { - struct ecore_hwfn *p_hwfn = &p_dev->hwfns[i]; + p_hwfn = &p_dev->hwfns[i]; + + /* If management didn't provide a default, set one of our own */ + if (!p_hwfn->hw_info.mtu) { + p_hwfn->hw_info.mtu = 1500; + b_default_mtu = false; + } if (IS_VF(p_dev)) { - rc = ecore_vf_pf_init(p_hwfn); - if (rc) - return rc; + ecore_vf_start(p_hwfn, p_params); continue; } - /* Enable DMAE in PXP */ - rc = ecore_change_pci_hwfn(p_hwfn, p_hwfn->p_main_ptt, true); + rc = ecore_calc_hw_mode(p_hwfn); + if (rc != ECORE_SUCCESS) + return rc; - ecore_calc_hw_mode(p_hwfn); - /* @@@TBD need to add here: - * Check for fan failure - * Prev_unload - */ - rc = ecore_mcp_load_req(p_hwfn, p_hwfn->p_main_ptt, &load_code); - if (rc) { + ecore_fill_load_req_params(&load_req_params, + p_params->p_drv_load_params); + rc = ecore_mcp_load_req(p_hwfn, p_hwfn->p_main_ptt, + &load_req_params); + if (rc != ECORE_SUCCESS) { DP_NOTICE(p_hwfn, true, - "Failed sending LOAD_REQ command\n"); + "Failed sending a LOAD_REQ command\n"); return rc; } + load_code = load_req_params.load_code; + DP_VERBOSE(p_hwfn, ECORE_MSG_SP, + "Load request was sent. Load code: 0x%x\n", + load_code); + + ecore_mcp_set_capabilities(p_hwfn, p_hwfn->p_main_ptt); + /* CQ75580: * When coming back from hiberbate state, the registers from * which shadow is read initially are not initialized. It turns * out that these registers get initialized during the call to * ecore_mcp_load_req request. So we need to reread them here * to get the proper shadow register value. - * Note: This is a workaround for the missinginig MFW + * Note: This is a workaround for the missing MFW * initialization. It may be removed once the implementation * is done. */ ecore_reset_mb_shadow(p_hwfn, p_hwfn->p_main_ptt); - DP_VERBOSE(p_hwfn, ECORE_MSG_SP, - "Load request was sent.Resp:0x%x, Load code: 0x%x\n", - rc, load_code); - /* Only relevant for recovery: * Clear the indication after the LOAD_REQ command is responded * by the MFW. @@ -1392,96 +2404,135 @@ enum _ecore_status_t ecore_hw_init(struct ecore_dev *p_dev, p_hwfn->first_on_engine = (load_code == FW_MSG_CODE_DRV_LOAD_ENGINE); + if (!qm_lock_init) { + OSAL_SPIN_LOCK_INIT(&qm_lock); + qm_lock_init = true; + } + + /* Clean up chip from previous driver if such remains exist. + * This is not needed when the PF is the first one on the + * engine, since afterwards we are going to init the FW. + */ + if (load_code != FW_MSG_CODE_DRV_LOAD_ENGINE) { + rc = ecore_final_cleanup(p_hwfn, p_hwfn->p_main_ptt, + p_hwfn->rel_pf_id, false); + if (rc != ECORE_SUCCESS) { + ecore_hw_err_notify(p_hwfn, + ECORE_HW_ERR_RAMROD_FAIL); + goto load_err; + } + } + + /* Log and clean previous pglue_b errors if such exist */ + ecore_pglueb_rbc_attn_handler(p_hwfn, p_hwfn->p_main_ptt); + ecore_pglueb_clear_err(p_hwfn, p_hwfn->p_main_ptt); + + /* Enable the PF's internal FID_enable in the PXP */ + rc = ecore_pglueb_set_pfid_enable(p_hwfn, p_hwfn->p_main_ptt, + true); + if (rc != ECORE_SUCCESS) + goto load_err; + switch (load_code) { case FW_MSG_CODE_DRV_LOAD_ENGINE: rc = ecore_hw_init_common(p_hwfn, p_hwfn->p_main_ptt, p_hwfn->hw_info.hw_mode); - if (rc) + if (rc != ECORE_SUCCESS) break; /* Fall into */ case FW_MSG_CODE_DRV_LOAD_PORT: rc = ecore_hw_init_port(p_hwfn, p_hwfn->p_main_ptt, p_hwfn->hw_info.hw_mode); - if (rc) + if (rc != ECORE_SUCCESS) break; - - if (ENABLE_EAGLE_ENG1_WORKAROUND(p_hwfn)) { - struct init_nig_pri_tc_map_req tc_map; - - OSAL_MEM_ZERO(&tc_map, sizeof(tc_map)); - - /* remove this once flow control is - * implemented - */ - for (j = 0; j < NUM_OF_VLAN_PRIORITIES; j++) { - tc_map.pri[j].tc_id = 0; - tc_map.pri[j].valid = 1; - } - ecore_init_nig_pri_tc_map(p_hwfn, - p_hwfn->p_main_ptt, - &tc_map); - } - /* fallthrough */ + /* Fall into */ case FW_MSG_CODE_DRV_LOAD_FUNCTION: rc = ecore_hw_init_pf(p_hwfn, p_hwfn->p_main_ptt, - p_tunn, p_hwfn->hw_info.hw_mode, - b_hw_start, int_mode, - allow_npar_tx_switch); + p_params->p_tunn, + p_hwfn->hw_info.hw_mode, + p_params->b_hw_start, + p_params->int_mode, + p_params->allow_npar_tx_switch); break; default: + DP_NOTICE(p_hwfn, false, + "Unexpected load code [0x%08x]", load_code); rc = ECORE_NOTIMPL; break; } - if (rc != ECORE_SUCCESS) + if (rc != ECORE_SUCCESS) { DP_NOTICE(p_hwfn, true, - "init phase failed loadcode 0x%x (rc %d)\n", + "init phase failed for loadcode 0x%x (rc %d)\n", load_code, rc); + goto load_err; + } - /* ACK mfw regardless of success or failure of initialization */ - mfw_rc = ecore_mcp_cmd(p_hwfn, p_hwfn->p_main_ptt, - DRV_MSG_CODE_LOAD_DONE, - 0, &load_code, ¶m); + rc = ecore_mcp_load_done(p_hwfn, p_hwfn->p_main_ptt); if (rc != ECORE_SUCCESS) return rc; - if (mfw_rc != ECORE_SUCCESS) { - DP_NOTICE(p_hwfn, true, - "Failed sending LOAD_DONE command\n"); - return mfw_rc; - } /* send DCBX attention request command */ DP_VERBOSE(p_hwfn, ECORE_MSG_DCB, - "sending phony dcbx set command to trigger DCBx" - " attention handling\n"); - mfw_rc = ecore_mcp_cmd(p_hwfn, p_hwfn->p_main_ptt, - DRV_MSG_CODE_SET_DCBX, - 1 << DRV_MB_PARAM_DCBX_NOTIFY_SHIFT, - &load_code, ¶m); - if (mfw_rc != ECORE_SUCCESS) { + "sending phony dcbx set command to trigger DCBx attention handling\n"); + rc = ecore_mcp_cmd(p_hwfn, p_hwfn->p_main_ptt, + DRV_MSG_CODE_SET_DCBX, + 1 << DRV_MB_PARAM_DCBX_NOTIFY_OFFSET, &resp, + ¶m); + if (rc != ECORE_SUCCESS) { DP_NOTICE(p_hwfn, true, "Failed to send DCBX attention request\n"); - return mfw_rc; + return rc; } p_hwfn->hw_init_done = true; } - return ECORE_SUCCESS; + if (IS_PF(p_dev)) { + p_hwfn = ECORE_LEADING_HWFN(p_dev); + drv_mb_param = STORM_FW_VERSION; + rc = ecore_mcp_cmd(p_hwfn, p_hwfn->p_main_ptt, + DRV_MSG_CODE_OV_UPDATE_STORM_FW_VER, + drv_mb_param, &resp, ¶m); + if (rc != ECORE_SUCCESS) + DP_INFO(p_hwfn, "Failed to update firmware version\n"); + + if (!b_default_mtu) + rc = ecore_mcp_ov_update_mtu(p_hwfn, p_hwfn->p_main_ptt, + p_hwfn->hw_info.mtu); + if (rc != ECORE_SUCCESS) + DP_INFO(p_hwfn, "Failed to update default mtu\n"); + + rc = ecore_mcp_ov_update_driver_state(p_hwfn, + p_hwfn->p_main_ptt, + ECORE_OV_DRIVER_STATE_DISABLED); + if (rc != ECORE_SUCCESS) + DP_INFO(p_hwfn, "Failed to update driver state\n"); + } + + return rc; + +load_err: + /* The MFW load lock should be released regardless of success or failure + * of initialization. + * TODO: replace this with an attempt to send cancel_load. + */ + ecore_mcp_load_done(p_hwfn, p_hwfn->p_main_ptt); + return rc; } #define ECORE_HW_STOP_RETRY_LIMIT (10) -static OSAL_INLINE void ecore_hw_timers_stop(struct ecore_dev *p_dev, - struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt) +static void ecore_hw_timers_stop(struct ecore_dev *p_dev, + struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt) { int i; /* close timers */ ecore_wr(p_hwfn, p_ptt, TM_REG_PF_ENABLE_CONN, 0x0); ecore_wr(p_hwfn, p_ptt, TM_REG_PF_ENABLE_TASK, 0x0); - for (i = 0; i < ECORE_HW_STOP_RETRY_LIMIT && - !p_dev->recov_in_prog; i++) { + for (i = 0; i < ECORE_HW_STOP_RETRY_LIMIT && !p_dev->recov_in_prog; + i++) { if ((!ecore_rd(p_hwfn, p_ptt, TM_REG_PF_SCAN_ACTIVE_CONN)) && (!ecore_rd(p_hwfn, p_ptt, TM_REG_PF_SCAN_ACTIVE_TASK))) @@ -1492,14 +2543,14 @@ static OSAL_INLINE void ecore_hw_timers_stop(struct ecore_dev *p_dev, */ OSAL_MSLEEP(1); } - if (i == ECORE_HW_STOP_RETRY_LIMIT) - DP_NOTICE(p_hwfn, true, - "Timers linear scans are not over" - " [Connection %02x Tasks %02x]\n", - (u8)ecore_rd(p_hwfn, p_ptt, - TM_REG_PF_SCAN_ACTIVE_CONN), - (u8)ecore_rd(p_hwfn, p_ptt, - TM_REG_PF_SCAN_ACTIVE_TASK)); + + if (i < ECORE_HW_STOP_RETRY_LIMIT) + return; + + DP_NOTICE(p_hwfn, true, "Timers linear scans are not over" + " [Connection %02x Tasks %02x]\n", + (u8)ecore_rd(p_hwfn, p_ptt, TM_REG_PF_SCAN_ACTIVE_CONN), + (u8)ecore_rd(p_hwfn, p_ptt, TM_REG_PF_SCAN_ACTIVE_TASK)); } void ecore_hw_timers_stop_all(struct ecore_dev *p_dev) @@ -1514,34 +2565,77 @@ void ecore_hw_timers_stop_all(struct ecore_dev *p_dev) } } +static enum _ecore_status_t ecore_verify_reg_val(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + u32 addr, u32 expected_val) +{ + u32 val = ecore_rd(p_hwfn, p_ptt, addr); + + if (val != expected_val) { + DP_NOTICE(p_hwfn, true, + "Value at address 0x%08x is 0x%08x while the expected value is 0x%08x\n", + addr, val, expected_val); + return ECORE_UNKNOWN_ERROR; + } + + return ECORE_SUCCESS; +} + enum _ecore_status_t ecore_hw_stop(struct ecore_dev *p_dev) { - enum _ecore_status_t rc = ECORE_SUCCESS, t_rc; + struct ecore_hwfn *p_hwfn; + struct ecore_ptt *p_ptt; + enum _ecore_status_t rc, rc2 = ECORE_SUCCESS; int j; for_each_hwfn(p_dev, j) { - struct ecore_hwfn *p_hwfn = &p_dev->hwfns[j]; - struct ecore_ptt *p_ptt = p_hwfn->p_main_ptt; + p_hwfn = &p_dev->hwfns[j]; + p_ptt = p_hwfn->p_main_ptt; DP_VERBOSE(p_hwfn, ECORE_MSG_IFDOWN, "Stopping hw/fw\n"); if (IS_VF(p_dev)) { ecore_vf_pf_int_cleanup(p_hwfn); + rc = ecore_vf_pf_reset(p_hwfn); + if (rc != ECORE_SUCCESS) { + DP_NOTICE(p_hwfn, true, + "ecore_vf_pf_reset failed. rc = %d.\n", + rc); + rc2 = ECORE_UNKNOWN_ERROR; + } continue; } /* mark the hw as uninitialized... */ p_hwfn->hw_init_done = false; + /* Send unload command to MCP */ + if (!p_dev->recov_in_prog) { + rc = ecore_mcp_unload_req(p_hwfn, p_ptt); + if (rc != ECORE_SUCCESS) { + DP_NOTICE(p_hwfn, true, + "Failed sending a UNLOAD_REQ command. rc = %d.\n", + rc); + rc2 = ECORE_UNKNOWN_ERROR; + } + } + + OSAL_DPC_SYNC(p_hwfn); + + /* After this point no MFW attentions are expected, e.g. prevent + * race between pf stop and dcbx pf update. + */ + rc = ecore_sp_pf_stop(p_hwfn); - if (rc) + if (rc != ECORE_SUCCESS) { DP_NOTICE(p_hwfn, true, - "Failed to close PF against FW. Continue to" - " stop HW to prevent illegal host access" - " by the device\n"); + "Failed to close PF against FW [rc = %d]. Continue to stop HW to prevent illegal host access by the device.\n", + rc); + rc2 = ECORE_UNKNOWN_ERROR; + } /* perform debug action after PF stop was sent */ - OSAL_AFTER_PF_STOP((void *)p_hwfn->p_dev, p_hwfn->my_id); + OSAL_AFTER_PF_STOP((void *)p_dev, p_hwfn->my_id); /* close NIG to BRB gate */ ecore_wr(p_hwfn, p_ptt, @@ -1550,6 +2644,8 @@ enum _ecore_status_t ecore_hw_stop(struct ecore_dev *p_dev) /* close parser */ ecore_wr(p_hwfn, p_ptt, PRS_REG_SEARCH_TCP, 0x0); ecore_wr(p_hwfn, p_ptt, PRS_REG_SEARCH_UDP, 0x0); + ecore_wr(p_hwfn, p_ptt, PRS_REG_SEARCH_FCOE, 0x0); + ecore_wr(p_hwfn, p_ptt, PRS_REG_SEARCH_ROCE, 0x0); ecore_wr(p_hwfn, p_ptt, PRS_REG_SEARCH_OPENFLOW, 0x0); /* @@@TBD - clean transmission queues (5.b) */ @@ -1564,36 +2660,76 @@ enum _ecore_status_t ecore_hw_stop(struct ecore_dev *p_dev) ecore_wr(p_hwfn, p_ptt, IGU_REG_LEADING_EDGE_LATCH, 0); ecore_wr(p_hwfn, p_ptt, IGU_REG_TRAILING_EDGE_LATCH, 0); ecore_int_igu_init_pure_rt(p_hwfn, p_ptt, false, true); + rc = ecore_int_igu_reset_cam_default(p_hwfn, p_ptt); + if (rc != ECORE_SUCCESS) { + DP_NOTICE(p_hwfn, true, + "Failed to return IGU CAM to default\n"); + rc2 = ECORE_UNKNOWN_ERROR; + } + /* Need to wait 1ms to guarantee SBs are cleared */ OSAL_MSLEEP(1); + + if (!p_dev->recov_in_prog) { + ecore_verify_reg_val(p_hwfn, p_ptt, + QM_REG_USG_CNT_PF_TX, 0); + ecore_verify_reg_val(p_hwfn, p_ptt, + QM_REG_USG_CNT_PF_OTHER, 0); + /* @@@TBD - assert on incorrect xCFC values (10.b) */ + } + + /* Disable PF in HW blocks */ + ecore_wr(p_hwfn, p_ptt, DORQ_REG_PF_DB_ENABLE, 0); + ecore_wr(p_hwfn, p_ptt, QM_REG_PF_EN, 0); + + if (!p_dev->recov_in_prog) { + ecore_mcp_unload_done(p_hwfn, p_ptt); + if (rc != ECORE_SUCCESS) { + DP_NOTICE(p_hwfn, true, + "Failed sending a UNLOAD_DONE command. rc = %d.\n", + rc); + rc2 = ECORE_UNKNOWN_ERROR; + } + } + } /* hwfn loop */ + + if (IS_PF(p_dev) && !p_dev->recov_in_prog) { + p_hwfn = ECORE_LEADING_HWFN(p_dev); + p_ptt = ECORE_LEADING_HWFN(p_dev)->p_main_ptt; + + /* Clear the PF's internal FID_enable in the PXP. + * In CMT this should only be done for first hw-function, and + * only after all transactions have stopped for all active + * hw-functions. + */ + rc = ecore_pglueb_set_pfid_enable(p_hwfn, p_hwfn->p_main_ptt, + false); + if (rc != ECORE_SUCCESS) { + DP_NOTICE(p_hwfn, true, + "ecore_pglueb_set_pfid_enable() failed. rc = %d.\n", + rc); + rc2 = ECORE_UNKNOWN_ERROR; + } } - if (IS_PF(p_dev)) { - /* Disable DMAE in PXP - in CMT, this should only be done for - * first hw-function, and only after all transactions have - * stopped for all active hw-functions. - */ - t_rc = ecore_change_pci_hwfn(&p_dev->hwfns[0], - p_dev->hwfns[0].p_main_ptt, false); - if (t_rc != ECORE_SUCCESS) - rc = t_rc; - } - - return rc; + return rc2; } -void ecore_hw_stop_fastpath(struct ecore_dev *p_dev) +enum _ecore_status_t ecore_hw_stop_fastpath(struct ecore_dev *p_dev) { int j; for_each_hwfn(p_dev, j) { struct ecore_hwfn *p_hwfn = &p_dev->hwfns[j]; - struct ecore_ptt *p_ptt = p_hwfn->p_main_ptt; + struct ecore_ptt *p_ptt; if (IS_VF(p_dev)) { ecore_vf_pf_int_cleanup(p_hwfn); continue; } + p_ptt = ecore_ptt_acquire(p_hwfn); + if (!p_ptt) + return ECORE_AGAIN; DP_VERBOSE(p_hwfn, ECORE_MSG_IFDOWN, "Shutting down the fastpath\n"); @@ -1603,6 +2739,8 @@ void ecore_hw_stop_fastpath(struct ecore_dev *p_dev) ecore_wr(p_hwfn, p_ptt, PRS_REG_SEARCH_TCP, 0x0); ecore_wr(p_hwfn, p_ptt, PRS_REG_SEARCH_UDP, 0x0); + ecore_wr(p_hwfn, p_ptt, PRS_REG_SEARCH_FCOE, 0x0); + ecore_wr(p_hwfn, p_ptt, PRS_REG_SEARCH_ROCE, 0x0); ecore_wr(p_hwfn, p_ptt, PRS_REG_SEARCH_OPENFLOW, 0x0); /* @@@TBD - clean transmission queues (5.b) */ @@ -1613,102 +2751,39 @@ void ecore_hw_stop_fastpath(struct ecore_dev *p_dev) ecore_int_igu_init_pure_rt(p_hwfn, p_ptt, false, false); /* Need to wait 1ms to guarantee SBs are cleared */ OSAL_MSLEEP(1); + ecore_ptt_release(p_hwfn, p_ptt); } + + return ECORE_SUCCESS; } -void ecore_hw_start_fastpath(struct ecore_hwfn *p_hwfn) +enum _ecore_status_t ecore_hw_start_fastpath(struct ecore_hwfn *p_hwfn) { - struct ecore_ptt *p_ptt = p_hwfn->p_main_ptt; + struct ecore_ptt *p_ptt; if (IS_VF(p_hwfn->p_dev)) - return; + return ECORE_SUCCESS; + + p_ptt = ecore_ptt_acquire(p_hwfn); + if (!p_ptt) + return ECORE_AGAIN; + + /* If roce info is allocated it means roce is initialized and should + * be enabled in searcher. + */ + if (p_hwfn->p_rdma_info) { + if (p_hwfn->b_rdma_enabled_in_prs) + ecore_wr(p_hwfn, p_ptt, + p_hwfn->rdma_prs_search_reg, 0x1); + ecore_wr(p_hwfn, p_ptt, TM_REG_PF_ENABLE_CONN, 0x1); + } /* Re-open incoming traffic */ - ecore_wr(p_hwfn, p_hwfn->p_main_ptt, + ecore_wr(p_hwfn, p_ptt, NIG_REG_RX_LLH_BRB_GATE_DNTFWD_PERPF, 0x0); -} + ecore_ptt_release(p_hwfn, p_ptt); -static enum _ecore_status_t ecore_reg_assert(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, u32 reg, - bool expected) -{ - u32 assert_val = ecore_rd(p_hwfn, p_ptt, reg); - - if (assert_val != expected) { - DP_NOTICE(p_hwfn, true, "Value at address 0x%08x != 0x%08x\n", - reg, expected); - return ECORE_UNKNOWN_ERROR; - } - - return 0; -} - -enum _ecore_status_t ecore_hw_reset(struct ecore_dev *p_dev) -{ - enum _ecore_status_t rc = ECORE_SUCCESS; - u32 unload_resp, unload_param; - int i; - - for_each_hwfn(p_dev, i) { - struct ecore_hwfn *p_hwfn = &p_dev->hwfns[i]; - - if (IS_VF(p_dev)) { - rc = ecore_vf_pf_reset(p_hwfn); - if (rc) - return rc; - continue; - } - - DP_VERBOSE(p_hwfn, ECORE_MSG_IFDOWN, "Resetting hw/fw\n"); - - /* Check for incorrect states */ - if (!p_dev->recov_in_prog) { - ecore_reg_assert(p_hwfn, p_hwfn->p_main_ptt, - QM_REG_USG_CNT_PF_TX, 0); - ecore_reg_assert(p_hwfn, p_hwfn->p_main_ptt, - QM_REG_USG_CNT_PF_OTHER, 0); - /* @@@TBD - assert on incorrect xCFC values (10.b) */ - } - - /* Disable PF in HW blocks */ - ecore_wr(p_hwfn, p_hwfn->p_main_ptt, DORQ_REG_PF_DB_ENABLE, 0); - ecore_wr(p_hwfn, p_hwfn->p_main_ptt, QM_REG_PF_EN, 0); - ecore_wr(p_hwfn, p_hwfn->p_main_ptt, - TCFC_REG_STRONG_ENABLE_PF, 0); - ecore_wr(p_hwfn, p_hwfn->p_main_ptt, - CCFC_REG_STRONG_ENABLE_PF, 0); - - if (p_dev->recov_in_prog) { - DP_VERBOSE(p_hwfn, ECORE_MSG_IFDOWN, - "Recovery is in progress -> skip " - "sending unload_req/done\n"); - break; - } - - /* Send unload command to MCP */ - rc = ecore_mcp_cmd(p_hwfn, p_hwfn->p_main_ptt, - DRV_MSG_CODE_UNLOAD_REQ, - DRV_MB_PARAM_UNLOAD_WOL_MCP, - &unload_resp, &unload_param); - if (rc != ECORE_SUCCESS) { - DP_NOTICE(p_hwfn, true, - "ecore_hw_reset: UNLOAD_REQ failed\n"); - /* @@TBD - what to do? for now, assume ENG. */ - unload_resp = FW_MSG_CODE_DRV_UNLOAD_ENGINE; - } - - rc = ecore_mcp_cmd(p_hwfn, p_hwfn->p_main_ptt, - DRV_MSG_CODE_UNLOAD_DONE, - 0, &unload_resp, &unload_param); - if (rc != ECORE_SUCCESS) { - DP_NOTICE(p_hwfn, - true, "ecore_hw_reset: UNLOAD_DONE failed\n"); - /* @@@TBD - Should it really ASSERT here ? */ - return rc; - } - } - - return rc; + return ECORE_SUCCESS; } /* Free hwfn memory and resources acquired in hw_hwfn_prepare */ @@ -1722,14 +2797,28 @@ static void ecore_hw_hwfn_free(struct ecore_hwfn *p_hwfn) static void ecore_hw_hwfn_prepare(struct ecore_hwfn *p_hwfn) { /* clear indirect access */ - ecore_wr(p_hwfn, p_hwfn->p_main_ptt, PGLUE_B_REG_PGL_ADDR_88_F0, 0); - ecore_wr(p_hwfn, p_hwfn->p_main_ptt, PGLUE_B_REG_PGL_ADDR_8C_F0, 0); - ecore_wr(p_hwfn, p_hwfn->p_main_ptt, PGLUE_B_REG_PGL_ADDR_90_F0, 0); - ecore_wr(p_hwfn, p_hwfn->p_main_ptt, PGLUE_B_REG_PGL_ADDR_94_F0, 0); + if (ECORE_IS_AH(p_hwfn->p_dev)) { + ecore_wr(p_hwfn, p_hwfn->p_main_ptt, + PGLUE_B_REG_PGL_ADDR_E8_F0_K2_E5, 0); + ecore_wr(p_hwfn, p_hwfn->p_main_ptt, + PGLUE_B_REG_PGL_ADDR_EC_F0_K2_E5, 0); + ecore_wr(p_hwfn, p_hwfn->p_main_ptt, + PGLUE_B_REG_PGL_ADDR_F0_F0_K2_E5, 0); + ecore_wr(p_hwfn, p_hwfn->p_main_ptt, + PGLUE_B_REG_PGL_ADDR_F4_F0_K2_E5, 0); + } else { + ecore_wr(p_hwfn, p_hwfn->p_main_ptt, + PGLUE_B_REG_PGL_ADDR_88_F0_BB, 0); + ecore_wr(p_hwfn, p_hwfn->p_main_ptt, + PGLUE_B_REG_PGL_ADDR_8C_F0_BB, 0); + ecore_wr(p_hwfn, p_hwfn->p_main_ptt, + PGLUE_B_REG_PGL_ADDR_90_F0_BB, 0); + ecore_wr(p_hwfn, p_hwfn->p_main_ptt, + PGLUE_B_REG_PGL_ADDR_94_F0_BB, 0); + } - /* Clean Previous errors if such exist */ - ecore_wr(p_hwfn, p_hwfn->p_main_ptt, - PGLUE_B_REG_WAS_ERROR_PF_31_0_CLR, 1 << p_hwfn->abs_pf_id); + /* Clean previous pglue_b errors if such exist */ + ecore_pglueb_clear_err(p_hwfn, p_hwfn->p_main_ptt); /* enable internal target-read */ ecore_wr(p_hwfn, p_hwfn->p_main_ptt, @@ -1740,12 +2829,11 @@ static void get_function_id(struct ecore_hwfn *p_hwfn) { /* ME Register */ p_hwfn->hw_info.opaque_fid = (u16)REG_RD(p_hwfn, - PXP_PF_ME_OPAQUE_ADDR); + PXP_PF_ME_OPAQUE_ADDR); p_hwfn->hw_info.concrete_fid = REG_RD(p_hwfn, PXP_PF_ME_CONCRETE_ADDR); /* Bits 16-19 from the ME registers are the pf_num */ - /* @@ @TBD - check, may be wrong after B0 implementation for CMT */ p_hwfn->abs_pf_id = (p_hwfn->hw_info.concrete_fid >> 16) & 0xf; p_hwfn->rel_pf_id = GET_FIELD(p_hwfn->hw_info.concrete_fid, PXP_CONCRETE_FID_PFID); @@ -1760,68 +2848,397 @@ static void get_function_id(struct ecore_hwfn *p_hwfn) static void ecore_hw_set_feat(struct ecore_hwfn *p_hwfn) { u32 *feat_num = p_hwfn->hw_info.feat_num; - int num_features = 1; + struct ecore_sb_cnt_info sb_cnt; + u32 non_l2_sbs = 0; + + OSAL_MEM_ZERO(&sb_cnt, sizeof(sb_cnt)); + ecore_int_get_num_sbs(p_hwfn, &sb_cnt); /* L2 Queues require each: 1 status block. 1 L2 queue */ - feat_num[ECORE_PF_L2_QUE] = - OSAL_MIN_T(u32, - RESC_NUM(p_hwfn, ECORE_SB) / num_features, - RESC_NUM(p_hwfn, ECORE_L2_QUEUE)); + if (ECORE_IS_L2_PERSONALITY(p_hwfn)) { + /* Start by allocating VF queues, then PF's */ + feat_num[ECORE_VF_L2_QUE] = + OSAL_MIN_T(u32, + RESC_NUM(p_hwfn, ECORE_L2_QUEUE), + sb_cnt.iov_cnt); + feat_num[ECORE_PF_L2_QUE] = + OSAL_MIN_T(u32, + sb_cnt.cnt - non_l2_sbs, + RESC_NUM(p_hwfn, ECORE_L2_QUEUE) - + FEAT_NUM(p_hwfn, ECORE_VF_L2_QUE)); + } + + if (ECORE_IS_FCOE_PERSONALITY(p_hwfn)) + feat_num[ECORE_FCOE_CQ] = + OSAL_MIN_T(u32, sb_cnt.cnt, RESC_NUM(p_hwfn, + ECORE_CMDQS_CQS)); + + if (ECORE_IS_ISCSI_PERSONALITY(p_hwfn)) + feat_num[ECORE_ISCSI_CQ] = + OSAL_MIN_T(u32, sb_cnt.cnt, RESC_NUM(p_hwfn, + ECORE_CMDQS_CQS)); DP_VERBOSE(p_hwfn, ECORE_MSG_PROBE, - "#PF_L2_QUEUES=%d #SBS=%d num_features=%d\n", - feat_num[ECORE_PF_L2_QUE], - RESC_NUM(p_hwfn, ECORE_SB), num_features); + "#PF_L2_QUEUE=%d VF_L2_QUEUES=%d #ROCE_CNQ=%d #FCOE_CQ=%d #ISCSI_CQ=%d #SB=%d\n", + (int)FEAT_NUM(p_hwfn, ECORE_PF_L2_QUE), + (int)FEAT_NUM(p_hwfn, ECORE_VF_L2_QUE), + (int)FEAT_NUM(p_hwfn, ECORE_RDMA_CNQ), + (int)FEAT_NUM(p_hwfn, ECORE_FCOE_CQ), + (int)FEAT_NUM(p_hwfn, ECORE_ISCSI_CQ), + (int)sb_cnt.cnt); } -/* @@@TBD MK RESC: This info is currently hard code and set as if we were MF - * need to read it from shmem... - */ -static enum _ecore_status_t ecore_hw_get_resc(struct ecore_hwfn *p_hwfn) +const char *ecore_hw_get_resc_name(enum ecore_resources res_id) +{ + switch (res_id) { + case ECORE_L2_QUEUE: + return "L2_QUEUE"; + case ECORE_VPORT: + return "VPORT"; + case ECORE_RSS_ENG: + return "RSS_ENG"; + case ECORE_PQ: + return "PQ"; + case ECORE_RL: + return "RL"; + case ECORE_MAC: + return "MAC"; + case ECORE_VLAN: + return "VLAN"; + case ECORE_RDMA_CNQ_RAM: + return "RDMA_CNQ_RAM"; + case ECORE_ILT: + return "ILT"; + case ECORE_LL2_QUEUE: + return "LL2_QUEUE"; + case ECORE_CMDQS_CQS: + return "CMDQS_CQS"; + case ECORE_RDMA_STATS_QUEUE: + return "RDMA_STATS_QUEUE"; + case ECORE_BDQ: + return "BDQ"; + case ECORE_SB: + return "SB"; + default: + return "UNKNOWN_RESOURCE"; + } +} + +static enum _ecore_status_t +__ecore_hw_set_soft_resc_size(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + enum ecore_resources res_id, + u32 resc_max_val, + u32 *p_mcp_resp) +{ + enum _ecore_status_t rc; + + rc = ecore_mcp_set_resc_max_val(p_hwfn, p_ptt, res_id, + resc_max_val, p_mcp_resp); + if (rc != ECORE_SUCCESS) { + DP_NOTICE(p_hwfn, true, + "MFW response failure for a max value setting of resource %d [%s]\n", + res_id, ecore_hw_get_resc_name(res_id)); + return rc; + } + + if (*p_mcp_resp != FW_MSG_CODE_RESOURCE_ALLOC_OK) + DP_INFO(p_hwfn, + "Failed to set the max value of resource %d [%s]. mcp_resp = 0x%08x.\n", + res_id, ecore_hw_get_resc_name(res_id), *p_mcp_resp); + + return ECORE_SUCCESS; +} + +static enum _ecore_status_t +ecore_hw_set_soft_resc_size(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt) +{ + bool b_ah = ECORE_IS_AH(p_hwfn->p_dev); + u32 resc_max_val, mcp_resp; + u8 res_id; + enum _ecore_status_t rc; + + for (res_id = 0; res_id < ECORE_MAX_RESC; res_id++) { + /* @DPDK */ + switch (res_id) { + case ECORE_LL2_QUEUE: + case ECORE_RDMA_CNQ_RAM: + case ECORE_RDMA_STATS_QUEUE: + case ECORE_BDQ: + resc_max_val = 0; + break; + default: + continue; + } + + rc = __ecore_hw_set_soft_resc_size(p_hwfn, p_ptt, res_id, + resc_max_val, &mcp_resp); + if (rc != ECORE_SUCCESS) + return rc; + + /* There's no point to continue to the next resource if the + * command is not supported by the MFW. + * We do continue if the command is supported but the resource + * is unknown to the MFW. Such a resource will be later + * configured with the default allocation values. + */ + if (mcp_resp == FW_MSG_CODE_UNSUPPORTED) + return ECORE_NOTIMPL; + } + + return ECORE_SUCCESS; +} + +static +enum _ecore_status_t ecore_hw_get_dflt_resc(struct ecore_hwfn *p_hwfn, + enum ecore_resources res_id, + u32 *p_resc_num, u32 *p_resc_start) { - u32 *resc_start = p_hwfn->hw_info.resc_start; u8 num_funcs = p_hwfn->num_funcs_on_engine; - u32 *resc_num = p_hwfn->hw_info.resc_num; - int i, max_vf_vlan_filters; - struct ecore_sb_cnt_info sb_cnt_info; bool b_ah = ECORE_IS_AH(p_hwfn->p_dev); - OSAL_MEM_ZERO(&sb_cnt_info, sizeof(sb_cnt_info)); + switch (res_id) { + case ECORE_L2_QUEUE: + *p_resc_num = (b_ah ? MAX_NUM_L2_QUEUES_K2 : + MAX_NUM_L2_QUEUES_BB) / num_funcs; + break; + case ECORE_VPORT: + *p_resc_num = (b_ah ? MAX_NUM_VPORTS_K2 : + MAX_NUM_VPORTS_BB) / num_funcs; + break; + case ECORE_RSS_ENG: + *p_resc_num = (b_ah ? ETH_RSS_ENGINE_NUM_K2 : + ETH_RSS_ENGINE_NUM_BB) / num_funcs; + break; + case ECORE_PQ: + *p_resc_num = (b_ah ? MAX_QM_TX_QUEUES_K2 : + MAX_QM_TX_QUEUES_BB) / num_funcs; + break; + case ECORE_RL: + *p_resc_num = MAX_QM_GLOBAL_RLS / num_funcs; + break; + case ECORE_MAC: + case ECORE_VLAN: + /* Each VFC resource can accommodate both a MAC and a VLAN */ + *p_resc_num = ETH_NUM_MAC_FILTERS / num_funcs; + break; + case ECORE_ILT: + *p_resc_num = (b_ah ? PXP_NUM_ILT_RECORDS_K2 : + PXP_NUM_ILT_RECORDS_BB) / num_funcs; + break; + case ECORE_LL2_QUEUE: + *p_resc_num = MAX_NUM_LL2_RX_QUEUES / num_funcs; + break; + case ECORE_RDMA_CNQ_RAM: + case ECORE_CMDQS_CQS: + /* CNQ/CMDQS are the same resource */ + /* @DPDK */ + *p_resc_num = (NUM_OF_GLOBAL_QUEUES / 2) / num_funcs; + break; + case ECORE_RDMA_STATS_QUEUE: + /* @DPDK */ + *p_resc_num = (b_ah ? MAX_NUM_VPORTS_K2 : + MAX_NUM_VPORTS_BB) / num_funcs; + break; + case ECORE_BDQ: + /* @DPDK */ + *p_resc_num = 0; + break; + default: + break; + } -#ifdef CONFIG_ECORE_SRIOV - max_vf_vlan_filters = ECORE_ETH_MAX_VF_NUM_VLAN_FILTERS; -#else - max_vf_vlan_filters = 0; + + switch (res_id) { + case ECORE_BDQ: + if (!*p_resc_num) + *p_resc_start = 0; + break; + case ECORE_SB: + /* Since we want its value to reflect whether MFW supports + * the new scheme, have a default of 0. + */ + *p_resc_num = 0; + break; + default: + *p_resc_start = *p_resc_num * p_hwfn->enabled_func_idx; + break; + } + + return ECORE_SUCCESS; +} + +static enum _ecore_status_t +__ecore_hw_set_resc_info(struct ecore_hwfn *p_hwfn, enum ecore_resources res_id, + bool drv_resc_alloc) +{ + u32 dflt_resc_num = 0, dflt_resc_start = 0; + u32 mcp_resp, *p_resc_num, *p_resc_start; + enum _ecore_status_t rc; + + p_resc_num = &RESC_NUM(p_hwfn, res_id); + p_resc_start = &RESC_START(p_hwfn, res_id); + + rc = ecore_hw_get_dflt_resc(p_hwfn, res_id, &dflt_resc_num, + &dflt_resc_start); + if (rc != ECORE_SUCCESS) { + DP_ERR(p_hwfn, + "Failed to get default amount for resource %d [%s]\n", + res_id, ecore_hw_get_resc_name(res_id)); + return rc; + } + +#ifndef ASIC_ONLY + if (CHIP_REV_IS_SLOW(p_hwfn->p_dev)) { + *p_resc_num = dflt_resc_num; + *p_resc_start = dflt_resc_start; + goto out; + } #endif - ecore_int_get_num_sbs(p_hwfn, &sb_cnt_info); - resc_num[ECORE_SB] = OSAL_MIN_T(u32, - (MAX_SB_PER_PATH_BB / num_funcs), - sb_cnt_info.sb_cnt); + rc = ecore_mcp_get_resc_info(p_hwfn, p_hwfn->p_main_ptt, res_id, + &mcp_resp, p_resc_num, p_resc_start); + if (rc != ECORE_SUCCESS) { + DP_NOTICE(p_hwfn, true, + "MFW response failure for an allocation request for" + " resource %d [%s]\n", + res_id, ecore_hw_get_resc_name(res_id)); + return rc; + } - resc_num[ECORE_L2_QUEUE] = (b_ah ? MAX_NUM_L2_QUEUES_K2 : - MAX_NUM_L2_QUEUES_BB) / num_funcs; - resc_num[ECORE_VPORT] = (b_ah ? MAX_NUM_VPORTS_K2 : - MAX_NUM_VPORTS_BB) / num_funcs; - resc_num[ECORE_RSS_ENG] = (b_ah ? ETH_RSS_ENGINE_NUM_K2 : - ETH_RSS_ENGINE_NUM_BB) / num_funcs; - resc_num[ECORE_PQ] = (b_ah ? MAX_QM_TX_QUEUES_K2 : - MAX_QM_TX_QUEUES_BB) / num_funcs; - resc_num[ECORE_RL] = 8; - resc_num[ECORE_MAC] = ETH_NUM_MAC_FILTERS / num_funcs; - resc_num[ECORE_VLAN] = (ETH_NUM_VLAN_FILTERS - - max_vf_vlan_filters + - 1 /*For vlan0 */) / num_funcs; + /* Default driver values are applied in the following cases: + * - The resource allocation MB command is not supported by the MFW + * - There is an internal error in the MFW while processing the request + * - The resource ID is unknown to the MFW + */ + if (mcp_resp != FW_MSG_CODE_RESOURCE_ALLOC_OK) { + DP_INFO(p_hwfn, + "Failed to receive allocation info for resource %d [%s]." + " mcp_resp = 0x%x. Applying default values" + " [%d,%d].\n", + res_id, ecore_hw_get_resc_name(res_id), mcp_resp, + dflt_resc_num, dflt_resc_start); - /* TODO - there will be a problem in AH - there are only 11k lines */ - resc_num[ECORE_ILT] = (b_ah ? PXP_NUM_ILT_RECORDS_K2 : - PXP_NUM_ILT_RECORDS_BB) / num_funcs; + *p_resc_num = dflt_resc_num; + *p_resc_start = dflt_resc_start; + goto out; + } + + if ((*p_resc_num != dflt_resc_num || + *p_resc_start != dflt_resc_start) && + res_id != ECORE_SB) { + DP_INFO(p_hwfn, + "MFW allocation for resource %d [%s] differs from default values [%d,%d vs. %d,%d]%s\n", + res_id, ecore_hw_get_resc_name(res_id), *p_resc_num, + *p_resc_start, dflt_resc_num, dflt_resc_start, + drv_resc_alloc ? " - Applying default values" : ""); + if (drv_resc_alloc) { + *p_resc_num = dflt_resc_num; + *p_resc_start = dflt_resc_start; + } + } +out: + return ECORE_SUCCESS; +} + +static enum _ecore_status_t ecore_hw_set_resc_info(struct ecore_hwfn *p_hwfn, + bool drv_resc_alloc) +{ + enum _ecore_status_t rc; + u8 res_id; + + for (res_id = 0; res_id < ECORE_MAX_RESC; res_id++) { + rc = __ecore_hw_set_resc_info(p_hwfn, res_id, drv_resc_alloc); + if (rc != ECORE_SUCCESS) + return rc; + } + + return ECORE_SUCCESS; +} + +static enum _ecore_status_t ecore_hw_get_resc(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + bool drv_resc_alloc) +{ + struct ecore_resc_unlock_params resc_unlock_params; + struct ecore_resc_lock_params resc_lock_params; + bool b_ah = ECORE_IS_AH(p_hwfn->p_dev); + u8 res_id; + enum _ecore_status_t rc; +#ifndef ASIC_ONLY + u32 *resc_start = p_hwfn->hw_info.resc_start; + u32 *resc_num = p_hwfn->hw_info.resc_num; + /* For AH, an equal share of the ILT lines between the maximal number of + * PFs is not enough for RoCE. This would be solved by the future + * resource allocation scheme, but isn't currently present for + * FPGA/emulation. For now we keep a number that is sufficient for RoCE + * to work - the BB number of ILT lines divided by its max PFs number. + */ + u32 roce_min_ilt_lines = PXP_NUM_ILT_RECORDS_BB / MAX_NUM_PFS_BB; +#endif + + /* Setting the max values of the soft resources and the following + * resources allocation queries should be atomic. Since several PFs can + * run in parallel - a resource lock is needed. + * If either the resource lock or resource set value commands are not + * supported - skip the the max values setting, release the lock if + * needed, and proceed to the queries. Other failures, including a + * failure to acquire the lock, will cause this function to fail. + * Old drivers that don't acquire the lock can run in parallel, and + * their allocation values won't be affected by the updated max values. + */ + ecore_mcp_resc_lock_default_init(&resc_lock_params, &resc_unlock_params, + ECORE_RESC_LOCK_RESC_ALLOC, false); + + rc = ecore_mcp_resc_lock(p_hwfn, p_ptt, &resc_lock_params); + if (rc != ECORE_SUCCESS && rc != ECORE_NOTIMPL) { + return rc; + } else if (rc == ECORE_NOTIMPL) { + DP_INFO(p_hwfn, + "Skip the max values setting of the soft resources since the resource lock is not supported by the MFW\n"); + } else if (rc == ECORE_SUCCESS && !resc_lock_params.b_granted) { + DP_NOTICE(p_hwfn, false, + "Failed to acquire the resource lock for the resource allocation commands\n"); + rc = ECORE_BUSY; + goto unlock_and_exit; + } else { + rc = ecore_hw_set_soft_resc_size(p_hwfn, p_ptt); + if (rc != ECORE_SUCCESS && rc != ECORE_NOTIMPL) { + DP_NOTICE(p_hwfn, false, + "Failed to set the max values of the soft resources\n"); + goto unlock_and_exit; + } else if (rc == ECORE_NOTIMPL) { + DP_INFO(p_hwfn, + "Skip the max values setting of the soft resources since it is not supported by the MFW\n"); + rc = ecore_mcp_resc_unlock(p_hwfn, p_ptt, + &resc_unlock_params); + if (rc != ECORE_SUCCESS) + DP_INFO(p_hwfn, + "Failed to release the resource lock for the resource allocation commands\n"); + } + } + + rc = ecore_hw_set_resc_info(p_hwfn, drv_resc_alloc); + if (rc != ECORE_SUCCESS) + goto unlock_and_exit; + + if (resc_lock_params.b_granted && !resc_unlock_params.b_released) { + rc = ecore_mcp_resc_unlock(p_hwfn, p_ptt, + &resc_unlock_params); + if (rc != ECORE_SUCCESS) + DP_INFO(p_hwfn, + "Failed to release the resource lock for the resource allocation commands\n"); + } #ifndef ASIC_ONLY if (CHIP_REV_IS_SLOW(p_hwfn->p_dev)) { /* Reduced build contains less PQs */ - if (!(p_hwfn->p_dev->b_is_emul_full)) + if (!(p_hwfn->p_dev->b_is_emul_full)) { resc_num[ECORE_PQ] = 32; + resc_start[ECORE_PQ] = resc_num[ECORE_PQ] * + p_hwfn->enabled_func_idx; + } /* For AH emulation, since we have a possible maximal number of * 16 enabled PFs, in case there are not enough ILT lines - @@ -1829,19 +3246,17 @@ static enum _ecore_status_t ecore_hw_get_resc(struct ecore_hwfn *p_hwfn) * only with less ILT lines. */ if (!p_hwfn->rel_pf_id && p_hwfn->p_dev->b_is_emul_full) - resc_num[ECORE_ILT] = resc_num[ECORE_ILT]; + resc_num[ECORE_ILT] = OSAL_MAX_T(u32, + resc_num[ECORE_ILT], + roce_min_ilt_lines); } -#endif - for (i = 0; i < ECORE_MAX_RESC; i++) - resc_start[i] = resc_num[i] * p_hwfn->rel_pf_id; - -#ifndef ASIC_ONLY /* Correct the common ILT calculation if PF0 has more */ if (CHIP_REV_IS_SLOW(p_hwfn->p_dev) && p_hwfn->p_dev->b_is_emul_full && - p_hwfn->rel_pf_id && resc_num[ECORE_ILT]) - resc_start[ECORE_ILT] += resc_num[ECORE_ILT]; + p_hwfn->rel_pf_id && resc_num[ECORE_ILT] < roce_min_ilt_lines) + resc_start[ECORE_ILT] += roce_min_ilt_lines - + resc_num[ECORE_ILT]; #endif /* Sanity for ILT */ @@ -1855,98 +3270,127 @@ static enum _ecore_status_t ecore_hw_get_resc(struct ecore_hwfn *p_hwfn) return ECORE_INVAL; } + /* This will also learn the number of SBs from MFW */ + if (ecore_int_igu_reset_cam(p_hwfn, p_ptt)) + return ECORE_INVAL; + ecore_hw_set_feat(p_hwfn); DP_VERBOSE(p_hwfn, ECORE_MSG_PROBE, - "The numbers for each resource are:\n" - "SB = %d start = %d\n" - "L2_QUEUE = %d start = %d\n" - "VPORT = %d start = %d\n" - "PQ = %d start = %d\n" - "RL = %d start = %d\n" - "MAC = %d start = %d\n" - "VLAN = %d start = %d\n" - "ILT = %d start = %d\n" - "CMDQS_CQS = %d start = %d\n", - RESC_NUM(p_hwfn, ECORE_SB), RESC_START(p_hwfn, ECORE_SB), - RESC_NUM(p_hwfn, ECORE_L2_QUEUE), - RESC_START(p_hwfn, ECORE_L2_QUEUE), - RESC_NUM(p_hwfn, ECORE_VPORT), - RESC_START(p_hwfn, ECORE_VPORT), - RESC_NUM(p_hwfn, ECORE_PQ), RESC_START(p_hwfn, ECORE_PQ), - RESC_NUM(p_hwfn, ECORE_RL), RESC_START(p_hwfn, ECORE_RL), - RESC_NUM(p_hwfn, ECORE_MAC), RESC_START(p_hwfn, ECORE_MAC), - RESC_NUM(p_hwfn, ECORE_VLAN), - RESC_START(p_hwfn, ECORE_VLAN), - RESC_NUM(p_hwfn, ECORE_ILT), RESC_START(p_hwfn, ECORE_ILT), - RESC_NUM(p_hwfn, ECORE_CMDQS_CQS), - RESC_START(p_hwfn, ECORE_CMDQS_CQS)); + "The numbers for each resource are:\n"); + for (res_id = 0; res_id < ECORE_MAX_RESC; res_id++) + DP_VERBOSE(p_hwfn, ECORE_MSG_PROBE, "%s = %d start = %d\n", + ecore_hw_get_resc_name(res_id), + RESC_NUM(p_hwfn, res_id), + RESC_START(p_hwfn, res_id)); return ECORE_SUCCESS; + +unlock_and_exit: + if (resc_lock_params.b_granted && !resc_unlock_params.b_released) + ecore_mcp_resc_unlock(p_hwfn, p_ptt, + &resc_unlock_params); + return rc; } -static enum _ecore_status_t ecore_hw_get_nvm_info(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt) +static enum _ecore_status_t +ecore_hw_get_nvm_info(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + struct ecore_hw_prepare_params *p_params) { - u32 nvm_cfg1_offset, mf_mode, addr, generic_cont0, core_cfg; - u32 port_cfg_addr, link_temp, device_capabilities; + u32 nvm_cfg1_offset, mf_mode, addr, generic_cont0, core_cfg, dcbx_mode; + u32 port_cfg_addr, link_temp, nvm_cfg_addr, device_capabilities; + struct ecore_mcp_link_capabilities *p_caps; struct ecore_mcp_link_params *link; + enum _ecore_status_t rc; /* Read global nvm_cfg address */ - u32 nvm_cfg_addr = ecore_rd(p_hwfn, p_ptt, MISC_REG_GEN_PURP_CR0); + nvm_cfg_addr = ecore_rd(p_hwfn, p_ptt, MISC_REG_GEN_PURP_CR0); /* Verify MCP has initialized it */ - if (nvm_cfg_addr == 0) { + if (!nvm_cfg_addr) { DP_NOTICE(p_hwfn, false, "Shared memory not initialized\n"); + if (p_params->b_relaxed_probe) + p_params->p_relaxed_res = ECORE_HW_PREPARE_FAILED_NVM; return ECORE_INVAL; } - /* Read nvm_cfg1 (Notice this is just offset, and not offsize (TBD) */ +/* Read nvm_cfg1 (Notice this is just offset, and not offsize (TBD) */ + nvm_cfg1_offset = ecore_rd(p_hwfn, p_ptt, nvm_cfg_addr + 4); addr = MCP_REG_SCRATCH + nvm_cfg1_offset + - OFFSETOF(struct nvm_cfg1, glob) + OFFSETOF(struct nvm_cfg1_glob, - core_cfg); + OFFSETOF(struct nvm_cfg1, glob) + + OFFSETOF(struct nvm_cfg1_glob, core_cfg); core_cfg = ecore_rd(p_hwfn, p_ptt, addr); switch ((core_cfg & NVM_CFG1_GLOB_NETWORK_PORT_MODE_MASK) >> NVM_CFG1_GLOB_NETWORK_PORT_MODE_OFFSET) { - case NVM_CFG1_GLOB_NETWORK_PORT_MODE_DE_2X40G: + case NVM_CFG1_GLOB_NETWORK_PORT_MODE_BB_2X40G: p_hwfn->hw_info.port_mode = ECORE_PORT_MODE_DE_2X40G; break; - case NVM_CFG1_GLOB_NETWORK_PORT_MODE_DE_2X50G: + case NVM_CFG1_GLOB_NETWORK_PORT_MODE_2X50G: p_hwfn->hw_info.port_mode = ECORE_PORT_MODE_DE_2X50G; break; - case NVM_CFG1_GLOB_NETWORK_PORT_MODE_DE_1X100G: + case NVM_CFG1_GLOB_NETWORK_PORT_MODE_BB_1X100G: p_hwfn->hw_info.port_mode = ECORE_PORT_MODE_DE_1X100G; break; - case NVM_CFG1_GLOB_NETWORK_PORT_MODE_DE_4X10G_F: + case NVM_CFG1_GLOB_NETWORK_PORT_MODE_4X10G_F: p_hwfn->hw_info.port_mode = ECORE_PORT_MODE_DE_4X10G_F; break; - case NVM_CFG1_GLOB_NETWORK_PORT_MODE_DE_4X10G_E: + case NVM_CFG1_GLOB_NETWORK_PORT_MODE_BB_4X10G_E: p_hwfn->hw_info.port_mode = ECORE_PORT_MODE_DE_4X10G_E; break; - case NVM_CFG1_GLOB_NETWORK_PORT_MODE_DE_4X20G: + case NVM_CFG1_GLOB_NETWORK_PORT_MODE_BB_4X20G: p_hwfn->hw_info.port_mode = ECORE_PORT_MODE_DE_4X20G; break; - case NVM_CFG1_GLOB_NETWORK_PORT_MODE_DE_1X40G: + case NVM_CFG1_GLOB_NETWORK_PORT_MODE_1X40G: p_hwfn->hw_info.port_mode = ECORE_PORT_MODE_DE_1X40G; break; - case NVM_CFG1_GLOB_NETWORK_PORT_MODE_DE_2X25G: + case NVM_CFG1_GLOB_NETWORK_PORT_MODE_2X25G: p_hwfn->hw_info.port_mode = ECORE_PORT_MODE_DE_2X25G; break; - case NVM_CFG1_GLOB_NETWORK_PORT_MODE_DE_1X25G: + case NVM_CFG1_GLOB_NETWORK_PORT_MODE_2X10G: + p_hwfn->hw_info.port_mode = ECORE_PORT_MODE_DE_2X10G; + break; + case NVM_CFG1_GLOB_NETWORK_PORT_MODE_1X25G: p_hwfn->hw_info.port_mode = ECORE_PORT_MODE_DE_1X25G; break; + case NVM_CFG1_GLOB_NETWORK_PORT_MODE_4X25G: + p_hwfn->hw_info.port_mode = ECORE_PORT_MODE_DE_4X25G; + break; default: DP_NOTICE(p_hwfn, true, "Unknown port mode in 0x%08x\n", core_cfg); break; } + /* Read DCBX configuration */ + port_cfg_addr = MCP_REG_SCRATCH + nvm_cfg1_offset + + OFFSETOF(struct nvm_cfg1, port[MFW_PORT(p_hwfn)]); + dcbx_mode = ecore_rd(p_hwfn, p_ptt, + port_cfg_addr + + OFFSETOF(struct nvm_cfg1_port, generic_cont0)); + dcbx_mode = (dcbx_mode & NVM_CFG1_PORT_DCBX_MODE_MASK) + >> NVM_CFG1_PORT_DCBX_MODE_OFFSET; + switch (dcbx_mode) { + case NVM_CFG1_PORT_DCBX_MODE_DYNAMIC: + p_hwfn->hw_info.dcbx_mode = ECORE_DCBX_VERSION_DYNAMIC; + break; + case NVM_CFG1_PORT_DCBX_MODE_CEE: + p_hwfn->hw_info.dcbx_mode = ECORE_DCBX_VERSION_CEE; + break; + case NVM_CFG1_PORT_DCBX_MODE_IEEE: + p_hwfn->hw_info.dcbx_mode = ECORE_DCBX_VERSION_IEEE; + break; + default: + p_hwfn->hw_info.dcbx_mode = ECORE_DCBX_VERSION_DISABLED; + } + /* Read default link configuration */ link = &p_hwfn->mcp_info->link_input; + p_caps = &p_hwfn->mcp_info->link_capabilities; port_cfg_addr = MCP_REG_SCRATCH + nvm_cfg1_offset + OFFSETOF(struct nvm_cfg1, port[MFW_PORT(p_hwfn)]); link_temp = ecore_rd(p_hwfn, p_ptt, @@ -1954,13 +3398,11 @@ static enum _ecore_status_t ecore_hw_get_nvm_info(struct ecore_hwfn *p_hwfn, OFFSETOF(struct nvm_cfg1_port, speed_cap_mask)); link_temp &= NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_MASK; link->speed.advertised_speeds = link_temp; - - link_temp = link->speed.advertised_speeds; - p_hwfn->mcp_info->link_capabilities.speed_capabilities = link_temp; + p_caps->speed_capabilities = link->speed.advertised_speeds; link_temp = ecore_rd(p_hwfn, p_ptt, - port_cfg_addr + - OFFSETOF(struct nvm_cfg1_port, link_settings)); + port_cfg_addr + + OFFSETOF(struct nvm_cfg1_port, link_settings)); switch ((link_temp & NVM_CFG1_PORT_DRV_LINK_SPEED_MASK) >> NVM_CFG1_PORT_DRV_LINK_SPEED_OFFSET) { case NVM_CFG1_PORT_DRV_LINK_SPEED_AUTONEG: @@ -1981,13 +3423,16 @@ static enum _ecore_status_t ecore_hw_get_nvm_info(struct ecore_hwfn *p_hwfn, case NVM_CFG1_PORT_DRV_LINK_SPEED_50G: link->speed.forced_speed = 50000; break; - case NVM_CFG1_PORT_DRV_LINK_SPEED_100G: + case NVM_CFG1_PORT_DRV_LINK_SPEED_BB_100G: link->speed.forced_speed = 100000; break; default: DP_NOTICE(p_hwfn, true, "Unknown Speed in 0x%08x\n", link_temp); } + p_caps->default_speed = link->speed.forced_speed; + p_caps->default_speed_autoneg = link->speed.autoneg; + link_temp &= NVM_CFG1_PORT_DRV_FLOW_CONTROL_MASK; link_temp >>= NVM_CFG1_PORT_DRV_FLOW_CONTROL_OFFSET; link->pause.autoneg = !!(link_temp & @@ -1998,16 +3443,47 @@ static enum _ecore_status_t ecore_hw_get_nvm_info(struct ecore_hwfn *p_hwfn, NVM_CFG1_PORT_DRV_FLOW_CONTROL_TX); link->loopback_mode = 0; + if (p_hwfn->mcp_info->capabilities & FW_MB_PARAM_FEATURE_SUPPORT_EEE) { + link_temp = ecore_rd(p_hwfn, p_ptt, port_cfg_addr + + OFFSETOF(struct nvm_cfg1_port, ext_phy)); + link_temp &= NVM_CFG1_PORT_EEE_POWER_SAVING_MODE_MASK; + link_temp >>= NVM_CFG1_PORT_EEE_POWER_SAVING_MODE_OFFSET; + p_caps->default_eee = ECORE_MCP_EEE_ENABLED; + link->eee.enable = true; + switch (link_temp) { + case NVM_CFG1_PORT_EEE_POWER_SAVING_MODE_DISABLED: + p_caps->default_eee = ECORE_MCP_EEE_DISABLED; + link->eee.enable = false; + break; + case NVM_CFG1_PORT_EEE_POWER_SAVING_MODE_BALANCED: + p_caps->eee_lpi_timer = EEE_TX_TIMER_USEC_BALANCED_TIME; + break; + case NVM_CFG1_PORT_EEE_POWER_SAVING_MODE_AGGRESSIVE: + p_caps->eee_lpi_timer = + EEE_TX_TIMER_USEC_AGGRESSIVE_TIME; + break; + case NVM_CFG1_PORT_EEE_POWER_SAVING_MODE_LOW_LATENCY: + p_caps->eee_lpi_timer = EEE_TX_TIMER_USEC_LATENCY_TIME; + break; + } + + link->eee.tx_lpi_timer = p_caps->eee_lpi_timer; + link->eee.tx_lpi_enable = link->eee.enable; + link->eee.adv_caps = ECORE_EEE_1G_ADV | ECORE_EEE_10G_ADV; + } else { + p_caps->default_eee = ECORE_MCP_EEE_UNSUPPORTED; + } + DP_VERBOSE(p_hwfn, ECORE_MSG_LINK, - "Read default link: Speed 0x%08x, Adv. Speed 0x%08x," - " AN: 0x%02x, PAUSE AN: 0x%02x\n", + "Read default link: Speed 0x%08x, Adv. Speed 0x%08x, AN: 0x%02x, PAUSE AN: 0x%02x\n EEE: %02x [%08x usec]", link->speed.forced_speed, link->speed.advertised_speeds, - link->speed.autoneg, link->pause.autoneg); + link->speed.autoneg, link->pause.autoneg, + p_caps->default_eee, p_caps->eee_lpi_timer); /* Read Multi-function information from shmem */ addr = MCP_REG_SCRATCH + nvm_cfg1_offset + - OFFSETOF(struct nvm_cfg1, glob) + - OFFSETOF(struct nvm_cfg1_glob, generic_cont0); + OFFSETOF(struct nvm_cfg1, glob) + + OFFSETOF(struct nvm_cfg1_glob, generic_cont0); generic_cont0 = ecore_rd(p_hwfn, p_ptt, addr); @@ -2015,6 +3491,41 @@ static enum _ecore_status_t ecore_hw_get_nvm_info(struct ecore_hwfn *p_hwfn, NVM_CFG1_GLOB_MF_MODE_OFFSET; switch (mf_mode) { + case NVM_CFG1_GLOB_MF_MODE_MF_ALLOWED: + p_hwfn->p_dev->mf_bits = 1 << ECORE_MF_OVLAN_CLSS; + break; + case NVM_CFG1_GLOB_MF_MODE_UFP: + p_hwfn->p_dev->mf_bits = 1 << ECORE_MF_OVLAN_CLSS | + 1 << ECORE_MF_UFP_SPECIFIC; + break; + + case NVM_CFG1_GLOB_MF_MODE_NPAR1_0: + p_hwfn->p_dev->mf_bits = 1 << ECORE_MF_LLH_MAC_CLSS | + 1 << ECORE_MF_LLH_PROTO_CLSS | + 1 << ECORE_MF_LL2_NON_UNICAST | + 1 << ECORE_MF_INTER_PF_SWITCH | + 1 << ECORE_MF_DISABLE_ARFS; + break; + case NVM_CFG1_GLOB_MF_MODE_DEFAULT: + p_hwfn->p_dev->mf_bits = 1 << ECORE_MF_LLH_MAC_CLSS | + 1 << ECORE_MF_LLH_PROTO_CLSS | + 1 << ECORE_MF_LL2_NON_UNICAST; + if (ECORE_IS_BB(p_hwfn->p_dev)) + p_hwfn->p_dev->mf_bits |= 1 << ECORE_MF_NEED_DEF_PF; + break; + } + DP_INFO(p_hwfn, "Multi function mode is 0x%lx\n", + p_hwfn->p_dev->mf_bits); + + if (ECORE_IS_CMT(p_hwfn->p_dev)) + p_hwfn->p_dev->mf_bits |= (1 << ECORE_MF_DISABLE_ARFS); + + /* It's funny since we have another switch, but it's easier + * to throw this away in linux this way. Long term, it might be + * better to have have getters for needed ECORE_MF_* fields, + * convert client code and eliminate this. + */ + switch (mf_mode) { case NVM_CFG1_GLOB_MF_MODE_MF_ALLOWED: p_hwfn->p_dev->mf_mode = ECORE_MF_OVLAN; break; @@ -2024,158 +3535,285 @@ static enum _ecore_status_t ecore_hw_get_nvm_info(struct ecore_hwfn *p_hwfn, case NVM_CFG1_GLOB_MF_MODE_DEFAULT: p_hwfn->p_dev->mf_mode = ECORE_MF_DEFAULT; break; + case NVM_CFG1_GLOB_MF_MODE_UFP: + p_hwfn->p_dev->mf_mode = ECORE_MF_UFP; + break; } - DP_INFO(p_hwfn, "Multi function mode is %08x\n", - p_hwfn->p_dev->mf_mode); /* Read Multi-function information from shmem */ addr = MCP_REG_SCRATCH + nvm_cfg1_offset + - OFFSETOF(struct nvm_cfg1, glob) + - OFFSETOF(struct nvm_cfg1_glob, device_capabilities); + OFFSETOF(struct nvm_cfg1, glob) + + OFFSETOF(struct nvm_cfg1_glob, device_capabilities); device_capabilities = ecore_rd(p_hwfn, p_ptt, addr); if (device_capabilities & NVM_CFG1_GLOB_DEVICE_CAPABILITIES_ETHERNET) OSAL_SET_BIT(ECORE_DEV_CAP_ETH, - &p_hwfn->hw_info.device_capabilities); + &p_hwfn->hw_info.device_capabilities); + if (device_capabilities & NVM_CFG1_GLOB_DEVICE_CAPABILITIES_FCOE) + OSAL_SET_BIT(ECORE_DEV_CAP_FCOE, + &p_hwfn->hw_info.device_capabilities); + if (device_capabilities & NVM_CFG1_GLOB_DEVICE_CAPABILITIES_ISCSI) + OSAL_SET_BIT(ECORE_DEV_CAP_ISCSI, + &p_hwfn->hw_info.device_capabilities); + if (device_capabilities & NVM_CFG1_GLOB_DEVICE_CAPABILITIES_ROCE) + OSAL_SET_BIT(ECORE_DEV_CAP_ROCE, + &p_hwfn->hw_info.device_capabilities); + if (device_capabilities & NVM_CFG1_GLOB_DEVICE_CAPABILITIES_IWARP) + OSAL_SET_BIT(ECORE_DEV_CAP_IWARP, + &p_hwfn->hw_info.device_capabilities); - return ecore_mcp_fill_shmem_func_info(p_hwfn, p_ptt); + rc = ecore_mcp_fill_shmem_func_info(p_hwfn, p_ptt); + if (rc != ECORE_SUCCESS && p_params->b_relaxed_probe) { + rc = ECORE_SUCCESS; + p_params->p_relaxed_res = ECORE_HW_PREPARE_BAD_MCP; + } + + return rc; } static void ecore_get_num_funcs(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt) { - u8 num_funcs; - u32 tmp, mask; + u8 num_funcs, enabled_func_idx = p_hwfn->rel_pf_id; + u32 reg_function_hide, tmp, eng_mask, low_pfs_mask; + struct ecore_dev *p_dev = p_hwfn->p_dev; - num_funcs = ECORE_IS_AH(p_hwfn->p_dev) ? MAX_NUM_PFS_K2 - : MAX_NUM_PFS_BB; + num_funcs = ECORE_IS_AH(p_dev) ? MAX_NUM_PFS_K2 : MAX_NUM_PFS_BB; /* Bit 0 of MISCS_REG_FUNCTION_HIDE indicates whether the bypass values * in the other bits are selected. * Bits 1-15 are for functions 1-15, respectively, and their value is * '0' only for enabled functions (function 0 always exists and * enabled). - * In case of CMT, only the "even" functions are enabled, and thus the - * number of functions for both hwfns is learnt from the same bits. + * In case of CMT in BB, only the "even" functions are enabled, and thus + * the number of functions for both hwfns is learnt from the same bits. */ + if (ECORE_IS_BB(p_dev) || ECORE_IS_AH(p_dev)) { + reg_function_hide = ecore_rd(p_hwfn, p_ptt, + MISCS_REG_FUNCTION_HIDE_BB_K2); + } else { /* E5 */ + reg_function_hide = 0; + } - tmp = ecore_rd(p_hwfn, p_ptt, MISCS_REG_FUNCTION_HIDE); - if (tmp & 0x1) { - if (ECORE_PATH_ID(p_hwfn) && p_hwfn->p_dev->num_hwfns == 1) { - num_funcs = 0; - mask = 0xaaaa; + if (reg_function_hide & 0x1) { + if (ECORE_IS_BB(p_dev)) { + if (ECORE_PATH_ID(p_hwfn) && !ECORE_IS_CMT(p_dev)) { + num_funcs = 0; + eng_mask = 0xaaaa; + } else { + num_funcs = 1; + eng_mask = 0x5554; + } } else { num_funcs = 1; - mask = 0x5554; + eng_mask = 0xfffe; } - tmp = (tmp ^ 0xffffffff) & mask; + /* Get the number of the enabled functions on the engine */ + tmp = (reg_function_hide ^ 0xffffffff) & eng_mask; while (tmp) { if (tmp & 0x1) num_funcs++; tmp >>= 0x1; } + + /* Get the PF index within the enabled functions */ + low_pfs_mask = (0x1 << p_hwfn->abs_pf_id) - 1; + tmp = reg_function_hide & eng_mask & low_pfs_mask; + while (tmp) { + if (tmp & 0x1) + enabled_func_idx--; + tmp >>= 0x1; + } } p_hwfn->num_funcs_on_engine = num_funcs; + p_hwfn->enabled_func_idx = enabled_func_idx; #ifndef ASIC_ONLY - if (CHIP_REV_IS_FPGA(p_hwfn->p_dev)) { + if (CHIP_REV_IS_FPGA(p_dev)) { DP_NOTICE(p_hwfn, false, - "FPGA: Limit number of PFs to 4 [would affect" - " resource allocation, needed for IOV]\n"); + "FPGA: Limit number of PFs to 4 [would affect resource allocation, needed for IOV]\n"); p_hwfn->num_funcs_on_engine = 4; } #endif - DP_VERBOSE(p_hwfn, ECORE_MSG_PROBE, "num_funcs_on_engine = %d\n", - p_hwfn->num_funcs_on_engine); + DP_VERBOSE(p_hwfn, ECORE_MSG_PROBE, + "PF [rel_id %d, abs_id %d] occupies index %d within the %d enabled functions on the engine\n", + p_hwfn->rel_pf_id, p_hwfn->abs_pf_id, + p_hwfn->enabled_func_idx, p_hwfn->num_funcs_on_engine); } static void ecore_hw_info_port_num_bb(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt) { + struct ecore_dev *p_dev = p_hwfn->p_dev; u32 port_mode; #ifndef ASIC_ONLY /* Read the port mode */ - if (CHIP_REV_IS_FPGA(p_hwfn->p_dev)) + if (CHIP_REV_IS_FPGA(p_dev)) port_mode = 4; - else if (CHIP_REV_IS_EMUL(p_hwfn->p_dev) && - (p_hwfn->p_dev->num_hwfns > 1)) + else if (CHIP_REV_IS_EMUL(p_dev) && ECORE_IS_CMT(p_dev)) /* In CMT on emulation, assume 1 port */ port_mode = 1; else #endif - port_mode = ecore_rd(p_hwfn, p_ptt, - CNIG_REG_NW_PORT_MODE_BB_B0); + port_mode = ecore_rd(p_hwfn, p_ptt, CNIG_REG_NW_PORT_MODE_BB); if (port_mode < 3) { - p_hwfn->p_dev->num_ports_in_engines = 1; + p_dev->num_ports_in_engine = 1; } else if (port_mode <= 5) { - p_hwfn->p_dev->num_ports_in_engines = 2; + p_dev->num_ports_in_engine = 2; } else { DP_NOTICE(p_hwfn, true, "PORT MODE: %d not supported\n", - p_hwfn->p_dev->num_ports_in_engines); + p_dev->num_ports_in_engine); - /* Default num_ports_in_engines to something */ - p_hwfn->p_dev->num_ports_in_engines = 1; + /* Default num_ports_in_engine to something */ + p_dev->num_ports_in_engine = 1; } } -static void ecore_hw_info_port_num_ah(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt) +static void ecore_hw_info_port_num_ah_e5(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt) { + struct ecore_dev *p_dev = p_hwfn->p_dev; u32 port; int i; - p_hwfn->p_dev->num_ports_in_engines = 0; + p_dev->num_ports_in_engine = 0; - for (i = 0; i < MAX_NUM_PORTS_K2; i++) { - port = ecore_rd(p_hwfn, p_ptt, - CNIG_REG_NIG_PORT0_CONF_K2 + (i * 4)); - if (port & 1) - p_hwfn->p_dev->num_ports_in_engines++; +#ifndef ASIC_ONLY + if (CHIP_REV_IS_EMUL(p_dev)) { + port = ecore_rd(p_hwfn, p_ptt, MISCS_REG_ECO_RESERVED); + switch ((port & 0xf000) >> 12) { + case 1: + p_dev->num_ports_in_engine = 1; + break; + case 3: + p_dev->num_ports_in_engine = 2; + break; + case 0xf: + p_dev->num_ports_in_engine = 4; + break; + default: + DP_NOTICE(p_hwfn, false, + "Unknown port mode in ECO_RESERVED %08x\n", + port); + } + } else +#endif + for (i = 0; i < MAX_NUM_PORTS_K2; i++) { + port = ecore_rd(p_hwfn, p_ptt, + CNIG_REG_NIG_PORT0_CONF_K2_E5 + + (i * 4)); + if (port & 1) + p_dev->num_ports_in_engine++; + } + + if (!p_dev->num_ports_in_engine) { + DP_NOTICE(p_hwfn, true, "All NIG ports are inactive\n"); + + /* Default num_ports_in_engine to something */ + p_dev->num_ports_in_engine = 1; } } static void ecore_hw_info_port_num(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt) { - if (ECORE_IS_BB(p_hwfn->p_dev)) + struct ecore_dev *p_dev = p_hwfn->p_dev; + + /* Determine the number of ports per engine */ + if (ECORE_IS_BB(p_dev)) ecore_hw_info_port_num_bb(p_hwfn, p_ptt); else - ecore_hw_info_port_num_ah(p_hwfn, p_ptt); + ecore_hw_info_port_num_ah_e5(p_hwfn, p_ptt); + + /* Get the total number of ports of the device */ + if (ECORE_IS_CMT(p_dev)) { + /* In CMT there is always only one port */ + p_dev->num_ports = 1; +#ifndef ASIC_ONLY + } else if (CHIP_REV_IS_EMUL(p_dev) || CHIP_REV_IS_TEDIBEAR(p_dev)) { + p_dev->num_ports = p_dev->num_ports_in_engine * + ecore_device_num_engines(p_dev); +#endif + } else { + u32 addr, global_offsize, global_addr; + + addr = SECTION_OFFSIZE_ADDR(p_hwfn->mcp_info->public_base, + PUBLIC_GLOBAL); + global_offsize = ecore_rd(p_hwfn, p_ptt, addr); + global_addr = SECTION_ADDR(global_offsize, 0); + addr = global_addr + OFFSETOF(struct public_global, max_ports); + p_dev->num_ports = (u8)ecore_rd(p_hwfn, p_ptt, addr); + } +} + +static void ecore_mcp_get_eee_caps(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt) +{ + struct ecore_mcp_link_capabilities *p_caps; + u32 eee_status; + + p_caps = &p_hwfn->mcp_info->link_capabilities; + if (p_caps->default_eee == ECORE_MCP_EEE_UNSUPPORTED) + return; + + p_caps->eee_speed_caps = 0; + eee_status = ecore_rd(p_hwfn, p_ptt, p_hwfn->mcp_info->port_addr + + OFFSETOF(struct public_port, eee_status)); + eee_status = (eee_status & EEE_SUPPORTED_SPEED_MASK) >> + EEE_SUPPORTED_SPEED_OFFSET; + if (eee_status & EEE_1G_SUPPORTED) + p_caps->eee_speed_caps |= ECORE_EEE_1G_ADV; + if (eee_status & EEE_10G_ADV) + p_caps->eee_speed_caps |= ECORE_EEE_10G_ADV; } static enum _ecore_status_t -ecore_get_hw_info(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, - enum ecore_pci_personality personality) +ecore_get_hw_info(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, + enum ecore_pci_personality personality, + struct ecore_hw_prepare_params *p_params) { + bool drv_resc_alloc = p_params->drv_resc_alloc; enum _ecore_status_t rc; - rc = ecore_iov_hw_info(p_hwfn, p_hwfn->p_main_ptt); - if (rc) - return rc; + /* Since all information is common, only first hwfns should do this */ + if (IS_LEAD_HWFN(p_hwfn)) { + rc = ecore_iov_hw_info(p_hwfn); + if (rc != ECORE_SUCCESS) { + if (p_params->b_relaxed_probe) + p_params->p_relaxed_res = + ECORE_HW_PREPARE_BAD_IOV; + else + return rc; + } + } - /* TODO In get_hw_info, amoungst others: - * Get MCP FW revision and determine according to it the supported - * featrues (e.g. DCB) - * Get boot mode - * ecore_get_pcie_width_speed, WOL capability. - * Number of global CQ-s (for storage - */ - ecore_hw_info_port_num(p_hwfn, p_ptt); + if (IS_LEAD_HWFN(p_hwfn)) + ecore_hw_info_port_num(p_hwfn, p_ptt); + + ecore_mcp_get_capabilities(p_hwfn, p_ptt); #ifndef ASIC_ONLY - if (CHIP_REV_IS_ASIC(p_hwfn->p_dev)) + if (CHIP_REV_IS_ASIC(p_hwfn->p_dev)) { +#endif + rc = ecore_hw_get_nvm_info(p_hwfn, p_ptt, p_params); + if (rc != ECORE_SUCCESS) + return rc; +#ifndef ASIC_ONLY + } #endif - ecore_hw_get_nvm_info(p_hwfn, p_ptt); rc = ecore_int_igu_read_cam(p_hwfn, p_ptt); - if (rc) - return rc; + if (rc != ECORE_SUCCESS) { + if (p_params->b_relaxed_probe) + p_params->p_relaxed_res = ECORE_HW_PREPARE_BAD_IGU; + else + return rc; + } #ifndef ASIC_ONLY if (CHIP_REV_IS_ASIC(p_hwfn->p_dev) && ecore_mcp_is_init(p_hwfn)) { @@ -2197,37 +3835,69 @@ ecore_get_hw_info(struct ecore_hwfn *p_hwfn, p_hwfn->mcp_info->func_info.ovlan; ecore_mcp_cmd_port_init(p_hwfn, p_ptt); + + ecore_mcp_get_eee_caps(p_hwfn, p_ptt); + + ecore_mcp_read_ufp_config(p_hwfn, p_ptt); } - if (personality != ECORE_PCI_DEFAULT) + if (personality != ECORE_PCI_DEFAULT) { p_hwfn->hw_info.personality = personality; - else if (ecore_mcp_is_init(p_hwfn)) - p_hwfn->hw_info.personality = - p_hwfn->mcp_info->func_info.protocol; + } else if (ecore_mcp_is_init(p_hwfn)) { + enum ecore_pci_personality protocol; + + protocol = p_hwfn->mcp_info->func_info.protocol; + p_hwfn->hw_info.personality = protocol; + } #ifndef ASIC_ONLY /* To overcome ILT lack for emulation, until at least until we'll have * a definite answer from system about it, allow only PF0 to be RoCE. */ - if (CHIP_REV_IS_EMUL(p_hwfn->p_dev) && ECORE_IS_AH(p_hwfn->p_dev)) - p_hwfn->hw_info.personality = ECORE_PCI_ETH; + if (CHIP_REV_IS_EMUL(p_hwfn->p_dev) && ECORE_IS_AH(p_hwfn->p_dev)) { + if (!p_hwfn->rel_pf_id) + p_hwfn->hw_info.personality = ECORE_PCI_ETH_ROCE; + else + p_hwfn->hw_info.personality = ECORE_PCI_ETH; + } #endif + /* although in BB some constellations may support more than 4 tcs, + * that can result in performance penalty in some cases. 4 + * represents a good tradeoff between performance and flexibility. + */ + p_hwfn->hw_info.num_hw_tc = NUM_PHYS_TCS_4PORT_K2; + + /* start out with a single active tc. This can be increased either + * by dcbx negotiation or by upper layer driver + */ + p_hwfn->hw_info.num_active_tc = 1; + ecore_get_num_funcs(p_hwfn, p_ptt); - /* Feat num is dependent on personality and on the number of functions - * on the engine. Therefore it should be come after personality - * initialization and after getting the number of functions. + if (ecore_mcp_is_init(p_hwfn)) + p_hwfn->hw_info.mtu = p_hwfn->mcp_info->func_info.mtu; + + /* In case of forcing the driver's default resource allocation, calling + * ecore_hw_get_resc() should come after initializing the personality + * and after getting the number of functions, since the calculation of + * the resources/features depends on them. + * This order is not harmful if not forcing. */ - return ecore_hw_get_resc(p_hwfn); + rc = ecore_hw_get_resc(p_hwfn, p_ptt, drv_resc_alloc); + if (rc != ECORE_SUCCESS && p_params->b_relaxed_probe) { + rc = ECORE_SUCCESS; + p_params->p_relaxed_res = ECORE_HW_PREPARE_BAD_MCP; + } + + return rc; } -/* @TMP - this should move to a proper .h */ -#define CHIP_NUM_AH 0x8070 - -static enum _ecore_status_t ecore_get_dev_info(struct ecore_dev *p_dev) +static enum _ecore_status_t ecore_get_dev_info(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt) { - struct ecore_hwfn *p_hwfn = ECORE_LEADING_HWFN(p_dev); + struct ecore_dev *p_dev = p_hwfn->p_dev; + u16 device_id_mask; u32 tmp; /* Read Vendor Id / Device Id */ @@ -2236,22 +3906,28 @@ static enum _ecore_status_t ecore_get_dev_info(struct ecore_dev *p_dev) OSAL_PCI_READ_CONFIG_WORD(p_dev, PCICFG_DEVICE_ID_OFFSET, &p_dev->device_id); - p_dev->chip_num = (u16)ecore_rd(p_hwfn, p_hwfn->p_main_ptt, - MISCS_REG_CHIP_NUM); - p_dev->chip_rev = (u16)ecore_rd(p_hwfn, p_hwfn->p_main_ptt, - MISCS_REG_CHIP_REV); - - MASK_FIELD(CHIP_REV, p_dev->chip_rev); - /* Determine type */ - if (p_dev->device_id == CHIP_NUM_AH) - p_dev->type = ECORE_DEV_TYPE_AH; - else + device_id_mask = p_dev->device_id & ECORE_DEV_ID_MASK; + switch (device_id_mask) { + case ECORE_DEV_ID_MASK_BB: p_dev->type = ECORE_DEV_TYPE_BB; + break; + case ECORE_DEV_ID_MASK_AH: + p_dev->type = ECORE_DEV_TYPE_AH; + break; + default: + DP_NOTICE(p_hwfn, true, "Unknown device id 0x%x\n", + p_dev->device_id); + return ECORE_ABORTED; + } + + tmp = ecore_rd(p_hwfn, p_ptt, MISCS_REG_CHIP_NUM); + p_dev->chip_num = (u16)GET_FIELD(tmp, CHIP_NUM); + tmp = ecore_rd(p_hwfn, p_ptt, MISCS_REG_CHIP_REV); + p_dev->chip_rev = (u8)GET_FIELD(tmp, CHIP_REV); /* Learn number of HW-functions */ - tmp = ecore_rd(p_hwfn, p_hwfn->p_main_ptt, - MISCS_REG_CMT_ENABLED_FOR_PAIR); + tmp = ecore_rd(p_hwfn, p_ptt, MISCS_REG_CMT_ENABLED_FOR_PAIR); if (tmp & (1 << p_hwfn->rel_pf_id)) { DP_NOTICE(p_dev->hwfns, false, "device in CMT mode\n"); @@ -2271,33 +3947,29 @@ static enum _ecore_status_t ecore_get_dev_info(struct ecore_dev *p_dev) } #endif - p_dev->chip_bond_id = ecore_rd(p_hwfn, p_hwfn->p_main_ptt, - MISCS_REG_CHIP_TEST_REG) >> 4; - MASK_FIELD(CHIP_BOND_ID, p_dev->chip_bond_id); - p_dev->chip_metal = (u16)ecore_rd(p_hwfn, p_hwfn->p_main_ptt, - MISCS_REG_CHIP_METAL); - MASK_FIELD(CHIP_METAL, p_dev->chip_metal); + tmp = ecore_rd(p_hwfn, p_ptt, MISCS_REG_CHIP_TEST_REG); + p_dev->chip_bond_id = (u8)GET_FIELD(tmp, CHIP_BOND_ID); + tmp = ecore_rd(p_hwfn, p_ptt, MISCS_REG_CHIP_METAL); + p_dev->chip_metal = (u8)GET_FIELD(tmp, CHIP_METAL); + DP_INFO(p_dev->hwfns, - "Chip details - %s%d, Num: %04x Rev: %04x Bond id: %04x" - " Metal: %04x\n", + "Chip details - %s %c%d, Num: %04x Rev: %02x Bond id: %02x Metal: %02x\n", ECORE_IS_BB(p_dev) ? "BB" : "AH", - CHIP_REV_IS_A0(p_dev) ? 0 : 1, + 'A' + p_dev->chip_rev, (int)p_dev->chip_metal, p_dev->chip_num, p_dev->chip_rev, p_dev->chip_bond_id, p_dev->chip_metal); - if (ECORE_IS_BB(p_dev) && CHIP_REV_IS_A0(p_dev)) { + if (ECORE_IS_BB_A0(p_dev)) { DP_NOTICE(p_dev->hwfns, false, "The chip type/rev (BB A0) is not supported!\n"); return ECORE_ABORTED; } #ifndef ASIC_ONLY if (CHIP_REV_IS_EMUL(p_dev) && ECORE_IS_AH(p_dev)) - ecore_wr(p_hwfn, p_hwfn->p_main_ptt, - MISCS_REG_PLL_MAIN_CTRL_4, 0x1); + ecore_wr(p_hwfn, p_ptt, MISCS_REG_PLL_MAIN_CTRL_4, 0x1); if (CHIP_REV_IS_EMUL(p_dev)) { - tmp = ecore_rd(p_hwfn, p_hwfn->p_main_ptt, - MISCS_REG_ECO_RESERVED); + tmp = ecore_rd(p_hwfn, p_ptt, MISCS_REG_ECO_RESERVED); if (tmp & (1 << 29)) { DP_NOTICE(p_hwfn, false, "Emulation: Running on a FULL build\n"); @@ -2312,6 +3984,7 @@ static enum _ecore_status_t ecore_get_dev_info(struct ecore_dev *p_dev) return ECORE_SUCCESS; } +#ifndef LINUX_REMOVE void ecore_prepare_hibernate(struct ecore_dev *p_dev) { int j; @@ -2326,27 +3999,36 @@ void ecore_prepare_hibernate(struct ecore_dev *p_dev) "Mark hw/fw uninitialized\n"); p_hwfn->hw_init_done = false; - p_hwfn->first_on_engine = false; + + ecore_ptt_invalidate(p_hwfn); } } +#endif static enum _ecore_status_t ecore_hw_prepare_single(struct ecore_hwfn *p_hwfn, - void OSAL_IOMEM *p_regview, - void OSAL_IOMEM *p_doorbells, - enum ecore_pci_personality personality) + void OSAL_IOMEM * p_regview, + void OSAL_IOMEM * p_doorbells, + struct ecore_hw_prepare_params *p_params) { + struct ecore_mdump_retain_data mdump_retain; + struct ecore_dev *p_dev = p_hwfn->p_dev; + struct ecore_mdump_info mdump_info; enum _ecore_status_t rc = ECORE_SUCCESS; /* Split PCI bars evenly between hwfns */ p_hwfn->regview = p_regview; p_hwfn->doorbells = p_doorbells; + if (IS_VF(p_dev)) + return ecore_vf_hw_prepare(p_hwfn); + /* Validate that chip access is feasible */ if (REG_RD(p_hwfn, PXP_PF_ME_OPAQUE_ADDR) == 0xffffffff) { DP_ERR(p_hwfn, - "Reading the ME register returns all Fs;" - " Preventing further chip access\n"); + "Reading the ME register returns all Fs; Preventing further chip access\n"); + if (p_params->b_relaxed_probe) + p_params->p_relaxed_res = ECORE_HW_PREPARE_FAILED_ME; return ECORE_INVAL; } @@ -2356,6 +4038,8 @@ ecore_hw_prepare_single(struct ecore_hwfn *p_hwfn, rc = ecore_ptt_pool_alloc(p_hwfn); if (rc) { DP_NOTICE(p_hwfn, true, "Failed to prepare hwfn's hw\n"); + if (p_params->b_relaxed_probe) + p_params->p_relaxed_res = ECORE_HW_PREPARE_FAILED_MEM; goto err0; } @@ -2364,9 +4048,13 @@ ecore_hw_prepare_single(struct ecore_hwfn *p_hwfn, /* First hwfn learns basic information, e.g., number of hwfns */ if (!p_hwfn->my_id) { - rc = ecore_get_dev_info(p_hwfn->p_dev); - if (rc != ECORE_SUCCESS) + rc = ecore_get_dev_info(p_hwfn, p_hwfn->p_main_ptt); + if (rc != ECORE_SUCCESS) { + if (p_params->b_relaxed_probe) + p_params->p_relaxed_res = + ECORE_HW_PREPARE_FAILED_DEV; goto err1; + } } ecore_hw_hwfn_prepare(p_hwfn); @@ -2375,37 +4063,81 @@ ecore_hw_prepare_single(struct ecore_hwfn *p_hwfn, rc = ecore_mcp_cmd_init(p_hwfn, p_hwfn->p_main_ptt); if (rc) { DP_NOTICE(p_hwfn, true, "Failed initializing mcp command\n"); + if (p_params->b_relaxed_probe) + p_params->p_relaxed_res = ECORE_HW_PREPARE_FAILED_MEM; goto err1; } /* Read the device configuration information from the HW and SHMEM */ - rc = ecore_get_hw_info(p_hwfn, p_hwfn->p_main_ptt, personality); + rc = ecore_get_hw_info(p_hwfn, p_hwfn->p_main_ptt, + p_params->personality, p_params); if (rc) { DP_NOTICE(p_hwfn, true, "Failed to get HW information\n"); goto err2; } + /* Sending a mailbox to the MFW should be after ecore_get_hw_info() is + * called, since among others it sets the ports number in an engine. + */ + if (p_params->initiate_pf_flr && IS_LEAD_HWFN(p_hwfn) && + !p_dev->recov_in_prog) { + rc = ecore_mcp_initiate_pf_flr(p_hwfn, p_hwfn->p_main_ptt); + if (rc != ECORE_SUCCESS) + DP_NOTICE(p_hwfn, false, "Failed to initiate PF FLR\n"); + } + + /* Check if mdump logs/data are present and update the epoch value */ + if (IS_LEAD_HWFN(p_hwfn)) { +#ifndef ASIC_ONLY + if (!CHIP_REV_IS_EMUL(p_dev)) { +#endif + rc = ecore_mcp_mdump_get_info(p_hwfn, p_hwfn->p_main_ptt, + &mdump_info); + if (rc == ECORE_SUCCESS && mdump_info.num_of_logs) + DP_NOTICE(p_hwfn, false, + "* * * IMPORTANT - HW ERROR register dump captured by device * * *\n"); + + rc = ecore_mcp_mdump_get_retain(p_hwfn, p_hwfn->p_main_ptt, + &mdump_retain); + if (rc == ECORE_SUCCESS && mdump_retain.valid) + DP_NOTICE(p_hwfn, false, + "mdump retained data: epoch 0x%08x, pf 0x%x, status 0x%08x\n", + mdump_retain.epoch, mdump_retain.pf, + mdump_retain.status); + + ecore_mcp_mdump_set_values(p_hwfn, p_hwfn->p_main_ptt, + p_params->epoch); +#ifndef ASIC_ONLY + } +#endif + } + /* Allocate the init RT array and initialize the init-ops engine */ rc = ecore_init_alloc(p_hwfn); if (rc) { DP_NOTICE(p_hwfn, true, "Failed to allocate the init array\n"); + if (p_params->b_relaxed_probe) + p_params->p_relaxed_res = ECORE_HW_PREPARE_FAILED_MEM; goto err2; } #ifndef ASIC_ONLY - if (CHIP_REV_IS_FPGA(p_hwfn->p_dev)) { + if (CHIP_REV_IS_FPGA(p_dev)) { DP_NOTICE(p_hwfn, false, "FPGA: workaround; Prevent DMAE parities\n"); - ecore_wr(p_hwfn, p_hwfn->p_main_ptt, PCIE_REG_PRTY_MASK, 7); + ecore_wr(p_hwfn, p_hwfn->p_main_ptt, PCIE_REG_PRTY_MASK_K2_E5, + 7); DP_NOTICE(p_hwfn, false, "FPGA: workaround: Set VF bar0 size\n"); ecore_wr(p_hwfn, p_hwfn->p_main_ptt, - PGLUE_B_REG_VF_BAR0_SIZE, 4); + PGLUE_B_REG_VF_BAR0_SIZE_K2_E5, 4); } #endif return rc; err2: + if (IS_LEAD_HWFN(p_hwfn)) + ecore_iov_free_hw_info(p_dev); ecore_mcp_free(p_hwfn); err1: ecore_hw_hwfn_free(p_hwfn); @@ -2413,62 +4145,85 @@ err0: return rc; } -enum _ecore_status_t ecore_hw_prepare(struct ecore_dev *p_dev, int personality) +enum _ecore_status_t ecore_hw_prepare(struct ecore_dev *p_dev, + struct ecore_hw_prepare_params *p_params) { struct ecore_hwfn *p_hwfn = ECORE_LEADING_HWFN(p_dev); enum _ecore_status_t rc; - if (IS_VF(p_dev)) - return ecore_vf_hw_prepare(p_dev); + p_dev->chk_reg_fifo = p_params->chk_reg_fifo; + p_dev->allow_mdump = p_params->allow_mdump; + + if (p_params->b_relaxed_probe) + p_params->p_relaxed_res = ECORE_HW_PREPARE_SUCCESS; /* Store the precompiled init data ptrs */ - ecore_init_iro_array(p_dev); + if (IS_PF(p_dev)) + ecore_init_iro_array(p_dev); /* Initialize the first hwfn - will learn number of hwfns */ rc = ecore_hw_prepare_single(p_hwfn, p_dev->regview, - p_dev->doorbells, personality); + p_dev->doorbells, p_params); if (rc != ECORE_SUCCESS) return rc; - personality = p_hwfn->hw_info.personality; + p_params->personality = p_hwfn->hw_info.personality; - /* initialalize 2nd hwfn if necessary */ - if (p_dev->num_hwfns > 1) { + /* initilalize 2nd hwfn if necessary */ + if (ECORE_IS_CMT(p_dev)) { void OSAL_IOMEM *p_regview, *p_doorbell; u8 OSAL_IOMEM *addr; /* adjust bar offset for second engine */ addr = (u8 OSAL_IOMEM *)p_dev->regview + - ecore_hw_bar_size(p_hwfn, BAR_ID_0) / 2; + ecore_hw_bar_size(p_hwfn, + p_hwfn->p_main_ptt, + BAR_ID_0) / 2; p_regview = (void OSAL_IOMEM *)addr; addr = (u8 OSAL_IOMEM *)p_dev->doorbells + - ecore_hw_bar_size(p_hwfn, BAR_ID_1) / 2; + ecore_hw_bar_size(p_hwfn, + p_hwfn->p_main_ptt, + BAR_ID_1) / 2; p_doorbell = (void OSAL_IOMEM *)addr; /* prepare second hw function */ rc = ecore_hw_prepare_single(&p_dev->hwfns[1], p_regview, - p_doorbell, personality); + p_doorbell, p_params); /* in case of error, need to free the previously - * initialiazed hwfn 0 + * initiliazed hwfn 0. */ if (rc != ECORE_SUCCESS) { - ecore_init_free(p_hwfn); - ecore_mcp_free(p_hwfn); - ecore_hw_hwfn_free(p_hwfn); + if (p_params->b_relaxed_probe) + p_params->p_relaxed_res = + ECORE_HW_PREPARE_FAILED_ENG2; + + if (IS_PF(p_dev)) { + ecore_init_free(p_hwfn); + ecore_mcp_free(p_hwfn); + ecore_hw_hwfn_free(p_hwfn); + } else { + DP_NOTICE(p_dev, true, + "What do we need to free when VF hwfn1 init fails\n"); + } return rc; } } - return ECORE_SUCCESS; + return rc; } void ecore_hw_remove(struct ecore_dev *p_dev) { + struct ecore_hwfn *p_hwfn = ECORE_LEADING_HWFN(p_dev); int i; + if (IS_PF(p_dev)) + ecore_mcp_ov_update_driver_state(p_hwfn, p_hwfn->p_main_ptt, + ECORE_OV_DRIVER_STATE_NOT_LOADED); + for_each_hwfn(p_dev, i) { struct ecore_hwfn *p_hwfn = &p_dev->hwfns[i]; @@ -2481,8 +4236,12 @@ void ecore_hw_remove(struct ecore_dev *p_dev) ecore_hw_hwfn_free(p_hwfn); ecore_mcp_free(p_hwfn); +#ifdef CONFIG_ECORE_LOCK_ALLOC OSAL_MUTEX_DEALLOC(&p_hwfn->dmae_info.mutex); +#endif } + + ecore_iov_free_hw_info(p_dev); } static void ecore_chain_free_next_ptr(struct ecore_dev *p_dev, @@ -2528,13 +4287,13 @@ static void ecore_chain_free_pbl(struct ecore_dev *p_dev, struct ecore_chain *p_chain) { void **pp_virt_addr_tbl = p_chain->pbl.pp_virt_addr_tbl; - u8 *p_pbl_virt = (u8 *)p_chain->pbl.p_virt_table; + u8 *p_pbl_virt = (u8 *)p_chain->pbl_sp.p_virt_table; u32 page_cnt = p_chain->page_cnt, i, pbl_size; if (!pp_virt_addr_tbl) return; - if (!p_chain->pbl.p_virt_table) + if (!p_pbl_virt) goto out; for (i = 0; i < page_cnt; i++) { @@ -2549,8 +4308,10 @@ static void ecore_chain_free_pbl(struct ecore_dev *p_dev, } pbl_size = page_cnt * ECORE_CHAIN_PBL_ENTRY_SIZE; - OSAL_DMA_FREE_COHERENT(p_dev, p_chain->pbl.p_virt_table, - p_chain->pbl.p_phys_table, pbl_size); + + if (!p_chain->b_external_pbl) + OSAL_DMA_FREE_COHERENT(p_dev, p_chain->pbl_sp.p_virt_table, + p_chain->pbl_sp.p_phys_table, pbl_size); out: OSAL_VFREE(p_dev, p_chain->pbl.pp_virt_addr_tbl); } @@ -2588,8 +4349,7 @@ ecore_chain_alloc_sanity_check(struct ecore_dev *p_dev, (cnt_type == ECORE_CHAIN_CNT_TYPE_U32 && chain_size > ECORE_U32_MAX)) { DP_NOTICE(p_dev, true, - "The actual chain size (0x%lx) is larger than" - " the maximal possible value\n", + "The actual chain size (0x%lx) is larger than the maximal possible value\n", (unsigned long)chain_size); return ECORE_INVAL; } @@ -2636,8 +4396,8 @@ ecore_chain_alloc_next_ptr(struct ecore_dev *p_dev, struct ecore_chain *p_chain) static enum _ecore_status_t ecore_chain_alloc_single(struct ecore_dev *p_dev, struct ecore_chain *p_chain) { - void *p_virt = OSAL_NULL; dma_addr_t p_phys = 0; + void *p_virt = OSAL_NULL; p_virt = OSAL_DMA_ALLOC_COHERENT(p_dev, &p_phys, ECORE_CHAIN_PAGE_SIZE); if (!p_virt) { @@ -2651,37 +4411,47 @@ ecore_chain_alloc_single(struct ecore_dev *p_dev, struct ecore_chain *p_chain) return ECORE_SUCCESS; } -static enum _ecore_status_t ecore_chain_alloc_pbl(struct ecore_dev *p_dev, - struct ecore_chain *p_chain) +static enum _ecore_status_t +ecore_chain_alloc_pbl(struct ecore_dev *p_dev, + struct ecore_chain *p_chain, + struct ecore_chain_ext_pbl *ext_pbl) { - void *p_virt = OSAL_NULL; - u8 *p_pbl_virt = OSAL_NULL; - void **pp_virt_addr_tbl = OSAL_NULL; - dma_addr_t p_phys = 0, p_pbl_phys = 0; u32 page_cnt = p_chain->page_cnt, size, i; + dma_addr_t p_phys = 0, p_pbl_phys = 0; + void **pp_virt_addr_tbl = OSAL_NULL; + u8 *p_pbl_virt = OSAL_NULL; + void *p_virt = OSAL_NULL; size = page_cnt * sizeof(*pp_virt_addr_tbl); - pp_virt_addr_tbl = (void **)OSAL_VALLOC(p_dev, size); + pp_virt_addr_tbl = (void **)OSAL_VZALLOC(p_dev, size); if (!pp_virt_addr_tbl) { DP_NOTICE(p_dev, true, - "Failed to allocate memory for the chain" - " virtual addresses table\n"); + "Failed to allocate memory for the chain virtual addresses table\n"); return ECORE_NOMEM; } - OSAL_MEM_ZERO(pp_virt_addr_tbl, size); /* The allocation of the PBL table is done with its full size, since it * is expected to be successive. + * ecore_chain_init_pbl_mem() is called even in a case of an allocation + * failure, since pp_virt_addr_tbl was previously allocated, and it + * should be saved to allow its freeing during the error flow. */ size = page_cnt * ECORE_CHAIN_PBL_ENTRY_SIZE; - p_pbl_virt = OSAL_DMA_ALLOC_COHERENT(p_dev, &p_pbl_phys, size); - if (!p_pbl_virt) { - DP_NOTICE(p_dev, true, "Failed to allocate chain pbl memory\n"); - return ECORE_NOMEM; + + if (ext_pbl == OSAL_NULL) { + p_pbl_virt = OSAL_DMA_ALLOC_COHERENT(p_dev, &p_pbl_phys, size); + } else { + p_pbl_virt = ext_pbl->p_pbl_virt; + p_pbl_phys = ext_pbl->p_pbl_phys; + p_chain->b_external_pbl = true; } ecore_chain_init_pbl_mem(p_chain, p_pbl_virt, p_pbl_phys, pp_virt_addr_tbl); + if (!p_pbl_virt) { + DP_NOTICE(p_dev, true, "Failed to allocate chain pbl memory\n"); + return ECORE_NOMEM; + } for (i = 0; i < page_cnt; i++) { p_virt = OSAL_DMA_ALLOC_COHERENT(p_dev, &p_phys, @@ -2713,7 +4483,8 @@ enum _ecore_status_t ecore_chain_alloc(struct ecore_dev *p_dev, enum ecore_chain_mode mode, enum ecore_chain_cnt_type cnt_type, u32 num_elems, osal_size_t elem_size, - struct ecore_chain *p_chain) + struct ecore_chain *p_chain, + struct ecore_chain_ext_pbl *ext_pbl) { u32 page_cnt; enum _ecore_status_t rc = ECORE_SUCCESS; @@ -2728,14 +4499,13 @@ enum _ecore_status_t ecore_chain_alloc(struct ecore_dev *p_dev, if (rc) { DP_NOTICE(p_dev, true, "Cannot allocate a chain with the given arguments:\n" - " [use_mode %d, mode %d, cnt_type %d, num_elems %d," - " elem_size %zu]\n", + "[use_mode %d, mode %d, cnt_type %d, num_elems %d, elem_size %zu]\n", intended_use, mode, cnt_type, num_elems, elem_size); return rc; } ecore_chain_init_params(p_chain, page_cnt, (u8)elem_size, intended_use, - mode, cnt_type); + mode, cnt_type, p_dev->dp_ctx); switch (mode) { case ECORE_CHAIN_MODE_NEXT_PTR: @@ -2745,7 +4515,7 @@ enum _ecore_status_t ecore_chain_alloc(struct ecore_dev *p_dev, rc = ecore_chain_alloc_single(p_dev, p_chain); break; case ECORE_CHAIN_MODE_PBL: - rc = ecore_chain_alloc_pbl(p_dev, p_chain); + rc = ecore_chain_alloc_pbl(p_dev, p_chain, ext_pbl); break; } if (rc) @@ -2767,8 +4537,7 @@ enum _ecore_status_t ecore_fw_l2_queue(struct ecore_hwfn *p_hwfn, min = (u16)RESC_START(p_hwfn, ECORE_L2_QUEUE); max = min + RESC_NUM(p_hwfn, ECORE_L2_QUEUE); DP_NOTICE(p_hwfn, true, - "l2_queue id [%d] is not valid, available" - " indices [%d - %d]\n", + "l2_queue id [%d] is not valid, available indices [%d - %d]\n", src_id, min, max); return ECORE_INVAL; @@ -2788,8 +4557,7 @@ enum _ecore_status_t ecore_fw_vport(struct ecore_hwfn *p_hwfn, min = (u8)RESC_START(p_hwfn, ECORE_VPORT); max = min + RESC_NUM(p_hwfn, ECORE_VPORT); DP_NOTICE(p_hwfn, true, - "vport id [%d] is not valid, available" - " indices [%d - %d]\n", + "vport id [%d] is not valid, available indices [%d - %d]\n", src_id, min, max); return ECORE_INVAL; @@ -2809,7 +4577,7 @@ enum _ecore_status_t ecore_fw_rss_eng(struct ecore_hwfn *p_hwfn, min = (u8)RESC_START(p_hwfn, ECORE_RSS_ENG); max = min + RESC_NUM(p_hwfn, ECORE_RSS_ENG); DP_NOTICE(p_hwfn, true, - "rss_eng id [%d] is not valid,avail idx [%d - %d]\n", + "rss_eng id [%d] is not valid, available indices [%d - %d]\n", src_id, min, max); return ECORE_INVAL; @@ -2820,180 +4588,387 @@ enum _ecore_status_t ecore_fw_rss_eng(struct ecore_hwfn *p_hwfn, return ECORE_SUCCESS; } -enum _ecore_status_t ecore_llh_add_mac_filter(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, - u8 *p_filter) +static enum _ecore_status_t +ecore_llh_add_mac_filter_bb_ah(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, u32 high, u32 low, + u32 *p_entry_num) { - u32 high, low, en; + u32 en; int i; - if (!(IS_MF_SI(p_hwfn) || IS_MF_DEFAULT(p_hwfn))) - return ECORE_SUCCESS; - - high = p_filter[1] | (p_filter[0] << 8); - low = p_filter[5] | (p_filter[4] << 8) | - (p_filter[3] << 16) | (p_filter[2] << 24); - /* Find a free entry and utilize it */ for (i = 0; i < NIG_REG_LLH_FUNC_FILTER_EN_SIZE; i++) { en = ecore_rd(p_hwfn, p_ptt, - NIG_REG_LLH_FUNC_FILTER_EN + i * sizeof(u32)); + NIG_REG_LLH_FUNC_FILTER_EN_BB_K2 + + i * sizeof(u32)); if (en) continue; ecore_wr(p_hwfn, p_ptt, - NIG_REG_LLH_FUNC_FILTER_VALUE + + NIG_REG_LLH_FUNC_FILTER_VALUE_BB_K2 + 2 * i * sizeof(u32), low); ecore_wr(p_hwfn, p_ptt, - NIG_REG_LLH_FUNC_FILTER_VALUE + + NIG_REG_LLH_FUNC_FILTER_VALUE_BB_K2 + (2 * i + 1) * sizeof(u32), high); ecore_wr(p_hwfn, p_ptt, - NIG_REG_LLH_FUNC_FILTER_MODE + i * sizeof(u32), 0); - ecore_wr(p_hwfn, p_ptt, - NIG_REG_LLH_FUNC_FILTER_PROTOCOL_TYPE + + NIG_REG_LLH_FUNC_FILTER_MODE_BB_K2 + i * sizeof(u32), 0); ecore_wr(p_hwfn, p_ptt, - NIG_REG_LLH_FUNC_FILTER_EN + i * sizeof(u32), 1); + NIG_REG_LLH_FUNC_FILTER_PROTOCOL_TYPE_BB_K2 + + i * sizeof(u32), 0); + ecore_wr(p_hwfn, p_ptt, + NIG_REG_LLH_FUNC_FILTER_EN_BB_K2 + + i * sizeof(u32), 1); break; } - if (i >= NIG_REG_LLH_FUNC_FILTER_EN_SIZE) { + + if (i >= NIG_REG_LLH_FUNC_FILTER_EN_SIZE) + return ECORE_NORESOURCES; + + *p_entry_num = i; + + return ECORE_SUCCESS; +} + +enum _ecore_status_t ecore_llh_add_mac_filter(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, u8 *p_filter) +{ + u32 high, low, entry_num; + enum _ecore_status_t rc = ECORE_SUCCESS; + + if (!OSAL_TEST_BIT(ECORE_MF_LLH_MAC_CLSS, + &p_hwfn->p_dev->mf_bits)) + return ECORE_SUCCESS; + + high = p_filter[1] | (p_filter[0] << 8); + low = p_filter[5] | (p_filter[4] << 8) | + (p_filter[3] << 16) | (p_filter[2] << 24); + + if (ECORE_IS_BB(p_hwfn->p_dev) || ECORE_IS_AH(p_hwfn->p_dev)) + rc = ecore_llh_add_mac_filter_bb_ah(p_hwfn, p_ptt, high, low, + &entry_num); + if (rc != ECORE_SUCCESS) { DP_NOTICE(p_hwfn, false, "Failed to find an empty LLH filter to utilize\n"); - return ECORE_INVAL; + return rc; } DP_VERBOSE(p_hwfn, ECORE_MSG_HW, - "MAC: %x:%x:%x:%x:%x:%x is added at %d\n", - p_filter[0], p_filter[1], p_filter[2], - p_filter[3], p_filter[4], p_filter[5], i); + "MAC: %02hhx:%02hhx:%02hhx:%02hhx:%02hhx:%02hhx is added at %d\n", + p_filter[0], p_filter[1], p_filter[2], p_filter[3], + p_filter[4], p_filter[5], entry_num); + + return rc; +} + +static enum _ecore_status_t +ecore_llh_remove_mac_filter_bb_ah(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, u32 high, u32 low, + u32 *p_entry_num) +{ + int i; + + /* Find the entry and clean it */ + for (i = 0; i < NIG_REG_LLH_FUNC_FILTER_EN_SIZE; i++) { + if (ecore_rd(p_hwfn, p_ptt, + NIG_REG_LLH_FUNC_FILTER_VALUE_BB_K2 + + 2 * i * sizeof(u32)) != low) + continue; + if (ecore_rd(p_hwfn, p_ptt, + NIG_REG_LLH_FUNC_FILTER_VALUE_BB_K2 + + (2 * i + 1) * sizeof(u32)) != high) + continue; + + ecore_wr(p_hwfn, p_ptt, + NIG_REG_LLH_FUNC_FILTER_EN_BB_K2 + i * sizeof(u32), 0); + ecore_wr(p_hwfn, p_ptt, + NIG_REG_LLH_FUNC_FILTER_VALUE_BB_K2 + + 2 * i * sizeof(u32), 0); + ecore_wr(p_hwfn, p_ptt, + NIG_REG_LLH_FUNC_FILTER_VALUE_BB_K2 + + (2 * i + 1) * sizeof(u32), 0); + break; + } + + if (i >= NIG_REG_LLH_FUNC_FILTER_EN_SIZE) + return ECORE_INVAL; + + *p_entry_num = i; return ECORE_SUCCESS; } void ecore_llh_remove_mac_filter(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, u8 *p_filter) + struct ecore_ptt *p_ptt, u8 *p_filter) { - u32 high, low; - int i; + u32 high, low, entry_num; + enum _ecore_status_t rc = ECORE_SUCCESS; - if (!(IS_MF_SI(p_hwfn) || IS_MF_DEFAULT(p_hwfn))) + if (!OSAL_TEST_BIT(ECORE_MF_LLH_MAC_CLSS, + &p_hwfn->p_dev->mf_bits)) return; high = p_filter[1] | (p_filter[0] << 8); low = p_filter[5] | (p_filter[4] << 8) | - (p_filter[3] << 16) | (p_filter[2] << 24); + (p_filter[3] << 16) | (p_filter[2] << 24); - /* Find the entry and clean it */ - for (i = 0; i < NIG_REG_LLH_FUNC_FILTER_EN_SIZE; i++) { - if (ecore_rd(p_hwfn, p_ptt, - NIG_REG_LLH_FUNC_FILTER_VALUE + - 2 * i * sizeof(u32)) != low) - continue; - if (ecore_rd(p_hwfn, p_ptt, - NIG_REG_LLH_FUNC_FILTER_VALUE + - (2 * i + 1) * sizeof(u32)) != high) - continue; - - ecore_wr(p_hwfn, p_ptt, - NIG_REG_LLH_FUNC_FILTER_EN + i * sizeof(u32), 0); - ecore_wr(p_hwfn, p_ptt, - NIG_REG_LLH_FUNC_FILTER_VALUE + - 2 * i * sizeof(u32), 0); - ecore_wr(p_hwfn, p_ptt, - NIG_REG_LLH_FUNC_FILTER_VALUE + - (2 * i + 1) * sizeof(u32), 0); - break; - } - if (i >= NIG_REG_LLH_FUNC_FILTER_EN_SIZE) + if (ECORE_IS_BB(p_hwfn->p_dev) || ECORE_IS_AH(p_hwfn->p_dev)) + rc = ecore_llh_remove_mac_filter_bb_ah(p_hwfn, p_ptt, high, + low, &entry_num); + if (rc != ECORE_SUCCESS) { DP_NOTICE(p_hwfn, false, "Tried to remove a non-configured filter\n"); + return; + } + + + DP_VERBOSE(p_hwfn, ECORE_MSG_HW, + "MAC: %02hhx:%02hhx:%02hhx:%02hhx:%02hhx:%02hhx was removed from %d\n", + p_filter[0], p_filter[1], p_filter[2], p_filter[3], + p_filter[4], p_filter[5], entry_num); } -enum _ecore_status_t ecore_llh_add_ethertype_filter(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, - u16 filter) +static enum _ecore_status_t +ecore_llh_add_protocol_filter_bb_ah(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + enum ecore_llh_port_filter_type_t type, + u32 high, u32 low, u32 *p_entry_num) { - u32 high, low, en; + u32 en; int i; - if (!(IS_MF_SI(p_hwfn) || IS_MF_DEFAULT(p_hwfn))) - return ECORE_SUCCESS; - - high = filter; - low = 0; - /* Find a free entry and utilize it */ for (i = 0; i < NIG_REG_LLH_FUNC_FILTER_EN_SIZE; i++) { en = ecore_rd(p_hwfn, p_ptt, - NIG_REG_LLH_FUNC_FILTER_EN + i * sizeof(u32)); + NIG_REG_LLH_FUNC_FILTER_EN_BB_K2 + + i * sizeof(u32)); if (en) continue; ecore_wr(p_hwfn, p_ptt, - NIG_REG_LLH_FUNC_FILTER_VALUE + + NIG_REG_LLH_FUNC_FILTER_VALUE_BB_K2 + 2 * i * sizeof(u32), low); ecore_wr(p_hwfn, p_ptt, - NIG_REG_LLH_FUNC_FILTER_VALUE + + NIG_REG_LLH_FUNC_FILTER_VALUE_BB_K2 + (2 * i + 1) * sizeof(u32), high); ecore_wr(p_hwfn, p_ptt, - NIG_REG_LLH_FUNC_FILTER_MODE + i * sizeof(u32), 1); - ecore_wr(p_hwfn, p_ptt, - NIG_REG_LLH_FUNC_FILTER_PROTOCOL_TYPE + + NIG_REG_LLH_FUNC_FILTER_MODE_BB_K2 + i * sizeof(u32), 1); ecore_wr(p_hwfn, p_ptt, - NIG_REG_LLH_FUNC_FILTER_EN + i * sizeof(u32), 1); + NIG_REG_LLH_FUNC_FILTER_PROTOCOL_TYPE_BB_K2 + + i * sizeof(u32), 1 << type); + ecore_wr(p_hwfn, p_ptt, + NIG_REG_LLH_FUNC_FILTER_EN_BB_K2 + i * sizeof(u32), 1); break; } - if (i >= NIG_REG_LLH_FUNC_FILTER_EN_SIZE) { - DP_NOTICE(p_hwfn, false, - "Failed to find an empty LLH filter to utilize\n"); - return ECORE_INVAL; - } - DP_VERBOSE(p_hwfn, ECORE_MSG_HW, - "ETH type: %x is added at %d\n", filter, i); + if (i >= NIG_REG_LLH_FUNC_FILTER_EN_SIZE) + return ECORE_NORESOURCES; + + *p_entry_num = i; return ECORE_SUCCESS; } -void ecore_llh_remove_ethertype_filter(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, u16 filter) +enum _ecore_status_t +ecore_llh_add_protocol_filter(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + u16 source_port_or_eth_type, + u16 dest_port, + enum ecore_llh_port_filter_type_t type) { - u32 high, low; - int i; + u32 high, low, entry_num; + enum _ecore_status_t rc = ECORE_SUCCESS; - if (!(IS_MF_SI(p_hwfn) || IS_MF_DEFAULT(p_hwfn))) - return; + if (!OSAL_TEST_BIT(ECORE_MF_LLH_PROTO_CLSS, + &p_hwfn->p_dev->mf_bits)) + return rc; - high = filter; + high = 0; low = 0; + switch (type) { + case ECORE_LLH_FILTER_ETHERTYPE: + high = source_port_or_eth_type; + break; + case ECORE_LLH_FILTER_TCP_SRC_PORT: + case ECORE_LLH_FILTER_UDP_SRC_PORT: + low = source_port_or_eth_type << 16; + break; + case ECORE_LLH_FILTER_TCP_DEST_PORT: + case ECORE_LLH_FILTER_UDP_DEST_PORT: + low = dest_port; + break; + case ECORE_LLH_FILTER_TCP_SRC_AND_DEST_PORT: + case ECORE_LLH_FILTER_UDP_SRC_AND_DEST_PORT: + low = (source_port_or_eth_type << 16) | dest_port; + break; + default: + DP_NOTICE(p_hwfn, true, + "Non valid LLH protocol filter type %d\n", type); + return ECORE_INVAL; + } + + if (ECORE_IS_BB(p_hwfn->p_dev) || ECORE_IS_AH(p_hwfn->p_dev)) + rc = ecore_llh_add_protocol_filter_bb_ah(p_hwfn, p_ptt, type, + high, low, &entry_num); + if (rc != ECORE_SUCCESS) { + DP_NOTICE(p_hwfn, false, + "Failed to find an empty LLH filter to utilize\n"); + return rc; + } + switch (type) { + case ECORE_LLH_FILTER_ETHERTYPE: + DP_VERBOSE(p_hwfn, ECORE_MSG_HW, + "ETH type %x is added at %d\n", + source_port_or_eth_type, entry_num); + break; + case ECORE_LLH_FILTER_TCP_SRC_PORT: + DP_VERBOSE(p_hwfn, ECORE_MSG_HW, + "TCP src port %x is added at %d\n", + source_port_or_eth_type, entry_num); + break; + case ECORE_LLH_FILTER_UDP_SRC_PORT: + DP_VERBOSE(p_hwfn, ECORE_MSG_HW, + "UDP src port %x is added at %d\n", + source_port_or_eth_type, entry_num); + break; + case ECORE_LLH_FILTER_TCP_DEST_PORT: + DP_VERBOSE(p_hwfn, ECORE_MSG_HW, + "TCP dst port %x is added at %d\n", dest_port, + entry_num); + break; + case ECORE_LLH_FILTER_UDP_DEST_PORT: + DP_VERBOSE(p_hwfn, ECORE_MSG_HW, + "UDP dst port %x is added at %d\n", dest_port, + entry_num); + break; + case ECORE_LLH_FILTER_TCP_SRC_AND_DEST_PORT: + DP_VERBOSE(p_hwfn, ECORE_MSG_HW, + "TCP src/dst ports %x/%x are added at %d\n", + source_port_or_eth_type, dest_port, entry_num); + break; + case ECORE_LLH_FILTER_UDP_SRC_AND_DEST_PORT: + DP_VERBOSE(p_hwfn, ECORE_MSG_HW, + "UDP src/dst ports %x/%x are added at %d\n", + source_port_or_eth_type, dest_port, entry_num); + break; + } + + return rc; +} + +static enum _ecore_status_t +ecore_llh_remove_protocol_filter_bb_ah(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + enum ecore_llh_port_filter_type_t type, + u32 high, u32 low, u32 *p_entry_num) +{ + int i; + /* Find the entry and clean it */ for (i = 0; i < NIG_REG_LLH_FUNC_FILTER_EN_SIZE; i++) { + if (!ecore_rd(p_hwfn, p_ptt, + NIG_REG_LLH_FUNC_FILTER_EN_BB_K2 + + i * sizeof(u32))) + continue; + if (!ecore_rd(p_hwfn, p_ptt, + NIG_REG_LLH_FUNC_FILTER_MODE_BB_K2 + + i * sizeof(u32))) + continue; + if (!(ecore_rd(p_hwfn, p_ptt, + NIG_REG_LLH_FUNC_FILTER_PROTOCOL_TYPE_BB_K2 + + i * sizeof(u32)) & (1 << type))) + continue; if (ecore_rd(p_hwfn, p_ptt, - NIG_REG_LLH_FUNC_FILTER_VALUE + + NIG_REG_LLH_FUNC_FILTER_VALUE_BB_K2 + 2 * i * sizeof(u32)) != low) continue; if (ecore_rd(p_hwfn, p_ptt, - NIG_REG_LLH_FUNC_FILTER_VALUE + + NIG_REG_LLH_FUNC_FILTER_VALUE_BB_K2 + (2 * i + 1) * sizeof(u32)) != high) continue; ecore_wr(p_hwfn, p_ptt, - NIG_REG_LLH_FUNC_FILTER_EN + i * sizeof(u32), 0); + NIG_REG_LLH_FUNC_FILTER_EN_BB_K2 + i * sizeof(u32), 0); ecore_wr(p_hwfn, p_ptt, - NIG_REG_LLH_FUNC_FILTER_VALUE + + NIG_REG_LLH_FUNC_FILTER_MODE_BB_K2 + + i * sizeof(u32), 0); + ecore_wr(p_hwfn, p_ptt, + NIG_REG_LLH_FUNC_FILTER_PROTOCOL_TYPE_BB_K2 + + i * sizeof(u32), 0); + ecore_wr(p_hwfn, p_ptt, + NIG_REG_LLH_FUNC_FILTER_VALUE_BB_K2 + 2 * i * sizeof(u32), 0); ecore_wr(p_hwfn, p_ptt, - NIG_REG_LLH_FUNC_FILTER_VALUE + + NIG_REG_LLH_FUNC_FILTER_VALUE_BB_K2 + (2 * i + 1) * sizeof(u32), 0); break; } + if (i >= NIG_REG_LLH_FUNC_FILTER_EN_SIZE) - DP_NOTICE(p_hwfn, false, - "Tried to remove a non-configured filter\n"); + return ECORE_INVAL; + + *p_entry_num = i; + + return ECORE_SUCCESS; } -void ecore_llh_clear_all_filters(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt) +void +ecore_llh_remove_protocol_filter(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + u16 source_port_or_eth_type, + u16 dest_port, + enum ecore_llh_port_filter_type_t type) +{ + u32 high, low, entry_num; + enum _ecore_status_t rc = ECORE_SUCCESS; + + if (!OSAL_TEST_BIT(ECORE_MF_LLH_PROTO_CLSS, + &p_hwfn->p_dev->mf_bits)) + return; + + high = 0; + low = 0; + + switch (type) { + case ECORE_LLH_FILTER_ETHERTYPE: + high = source_port_or_eth_type; + break; + case ECORE_LLH_FILTER_TCP_SRC_PORT: + case ECORE_LLH_FILTER_UDP_SRC_PORT: + low = source_port_or_eth_type << 16; + break; + case ECORE_LLH_FILTER_TCP_DEST_PORT: + case ECORE_LLH_FILTER_UDP_DEST_PORT: + low = dest_port; + break; + case ECORE_LLH_FILTER_TCP_SRC_AND_DEST_PORT: + case ECORE_LLH_FILTER_UDP_SRC_AND_DEST_PORT: + low = (source_port_or_eth_type << 16) | dest_port; + break; + default: + DP_NOTICE(p_hwfn, true, + "Non valid LLH protocol filter type %d\n", type); + return; + } + + if (ECORE_IS_BB(p_hwfn->p_dev) || ECORE_IS_AH(p_hwfn->p_dev)) + rc = ecore_llh_remove_protocol_filter_bb_ah(p_hwfn, p_ptt, type, + high, low, + &entry_num); + if (rc != ECORE_SUCCESS) { + DP_NOTICE(p_hwfn, false, + "Tried to remove a non-configured filter [type %d, source_port_or_eth_type 0x%x, dest_port 0x%x]\n", + type, source_port_or_eth_type, dest_port); + return; + } + + DP_VERBOSE(p_hwfn, ECORE_MSG_HW, + "Protocol filter [type %d, source_port_or_eth_type 0x%x, dest_port 0x%x] was removed from %d\n", + type, source_port_or_eth_type, dest_port, entry_num); +} + +static void ecore_llh_clear_all_filters_bb_ah(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt) { int i; @@ -3002,112 +4977,54 @@ void ecore_llh_clear_all_filters(struct ecore_hwfn *p_hwfn, for (i = 0; i < NIG_REG_LLH_FUNC_FILTER_EN_SIZE; i++) { ecore_wr(p_hwfn, p_ptt, - NIG_REG_LLH_FUNC_FILTER_EN + i * sizeof(u32), 0); + NIG_REG_LLH_FUNC_FILTER_EN_BB_K2 + + i * sizeof(u32), 0); ecore_wr(p_hwfn, p_ptt, - NIG_REG_LLH_FUNC_FILTER_VALUE + + NIG_REG_LLH_FUNC_FILTER_VALUE_BB_K2 + 2 * i * sizeof(u32), 0); ecore_wr(p_hwfn, p_ptt, - NIG_REG_LLH_FUNC_FILTER_VALUE + + NIG_REG_LLH_FUNC_FILTER_VALUE_BB_K2 + (2 * i + 1) * sizeof(u32), 0); } } -enum _ecore_status_t ecore_test_registers(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt) +void ecore_llh_clear_all_filters(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt) { - u32 reg_tbl[] = { - BRB_REG_HEADER_SIZE, - BTB_REG_HEADER_SIZE, - CAU_REG_LONG_TIMEOUT_THRESHOLD, - CCFC_REG_ACTIVITY_COUNTER, - CDU_REG_CID_ADDR_PARAMS, - DBG_REG_CLIENT_ENABLE, - DMAE_REG_INIT, - DORQ_REG_IFEN, - GRC_REG_TIMEOUT_EN, - IGU_REG_BLOCK_CONFIGURATION, - MCM_REG_INIT, - MCP2_REG_DBG_DWORD_ENABLE, - MISC_REG_PORT_MODE, - MISCS_REG_CLK_100G_MODE, - MSDM_REG_ENABLE_IN1, - MSEM_REG_ENABLE_IN, - NIG_REG_CM_HDR, - NCSI_REG_CONFIG, - PBF_REG_INIT, - PTU_REG_ATC_INIT_ARRAY, - PCM_REG_INIT, - PGLUE_B_REG_ADMIN_PER_PF_REGION, - PRM_REG_DISABLE_PRM, - PRS_REG_SOFT_RST, - PSDM_REG_ENABLE_IN1, - PSEM_REG_ENABLE_IN, - PSWRQ_REG_DBG_SELECT, - PSWRQ2_REG_CDUT_P_SIZE, - PSWHST_REG_DISCARD_INTERNAL_WRITES, - PSWHST2_REG_DBGSYN_ALMOST_FULL_THR, - PSWRD_REG_DBG_SELECT, - PSWRD2_REG_CONF11, - PSWWR_REG_USDM_FULL_TH, - PSWWR2_REG_CDU_FULL_TH2, - QM_REG_MAXPQSIZE_0, - RSS_REG_RSS_INIT_EN, - RDIF_REG_STOP_ON_ERROR, - SRC_REG_SOFT_RST, - TCFC_REG_ACTIVITY_COUNTER, - TCM_REG_INIT, - TM_REG_PXP_READ_DATA_FIFO_INIT, - TSDM_REG_ENABLE_IN1, - TSEM_REG_ENABLE_IN, - TDIF_REG_STOP_ON_ERROR, - UCM_REG_INIT, - UMAC_REG_IPG_HD_BKP_CNTL_BB_B0, - USDM_REG_ENABLE_IN1, - USEM_REG_ENABLE_IN, - XCM_REG_INIT, - XSDM_REG_ENABLE_IN1, - XSEM_REG_ENABLE_IN, - YCM_REG_INIT, - YSDM_REG_ENABLE_IN1, - YSEM_REG_ENABLE_IN, - XYLD_REG_SCBD_STRICT_PRIO, - TMLD_REG_SCBD_STRICT_PRIO, - MULD_REG_SCBD_STRICT_PRIO, - YULD_REG_SCBD_STRICT_PRIO, - }; - u32 test_val[] = { 0x0, 0x1 }; - u32 val, save_val, i, j; + if (!OSAL_TEST_BIT(ECORE_MF_LLH_PROTO_CLSS, + &p_hwfn->p_dev->mf_bits) && + !OSAL_TEST_BIT(ECORE_MF_LLH_MAC_CLSS, + &p_hwfn->p_dev->mf_bits)) + return; - for (i = 0; i < OSAL_ARRAY_SIZE(test_val); i++) { - for (j = 0; j < OSAL_ARRAY_SIZE(reg_tbl); j++) { - save_val = ecore_rd(p_hwfn, p_ptt, reg_tbl[j]); - ecore_wr(p_hwfn, p_ptt, reg_tbl[j], test_val[i]); - val = ecore_rd(p_hwfn, p_ptt, reg_tbl[j]); - /* Restore the original register's value */ - ecore_wr(p_hwfn, p_ptt, reg_tbl[j], save_val); - if (val != test_val[i]) { - DP_INFO(p_hwfn->p_dev, - "offset 0x%x: val 0x%x != 0x%x\n", - reg_tbl[j], val, test_val[i]); - return ECORE_AGAIN; - } - } + if (ECORE_IS_BB(p_hwfn->p_dev) || ECORE_IS_AH(p_hwfn->p_dev)) + ecore_llh_clear_all_filters_bb_ah(p_hwfn, p_ptt); +} + +enum _ecore_status_t +ecore_llh_set_function_as_default(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt) +{ + if (OSAL_TEST_BIT(ECORE_MF_NEED_DEF_PF, &p_hwfn->p_dev->mf_bits)) { + ecore_wr(p_hwfn, p_ptt, + NIG_REG_LLH_TAGMAC_DEF_PF_VECTOR, + 1 << p_hwfn->abs_pf_id / 2); + ecore_wr(p_hwfn, p_ptt, PRS_REG_MSG_INFO, 0); + return ECORE_SUCCESS; } - return ECORE_SUCCESS; + + DP_NOTICE(p_hwfn, false, + "This function can't be set as default\n"); + return ECORE_INVAL; } static enum _ecore_status_t ecore_set_coalesce(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, - u32 hw_addr, void *p_qzone, - osal_size_t qzone_size, + u32 hw_addr, void *p_eth_qzone, + osal_size_t eth_qzone_size, u8 timeset) { - struct coalescing_timeset *p_coalesce_timeset; - - if (IS_VF(p_hwfn->p_dev)) { - DP_NOTICE(p_hwfn, true, "VF coalescing config not supported\n"); - return ECORE_INVAL; - } + struct coalescing_timeset *p_coal_timeset; if (p_hwfn->p_dev->int_coalescing_mode != ECORE_COAL_MODE_ENABLE) { DP_NOTICE(p_hwfn, true, @@ -3115,71 +5032,129 @@ static enum _ecore_status_t ecore_set_coalesce(struct ecore_hwfn *p_hwfn, return ECORE_INVAL; } - OSAL_MEMSET(p_qzone, 0, qzone_size); - p_coalesce_timeset = p_qzone; - p_coalesce_timeset->timeset = timeset; - p_coalesce_timeset->valid = 1; - ecore_memcpy_to(p_hwfn, p_ptt, hw_addr, p_qzone, qzone_size); + p_coal_timeset = p_eth_qzone; + OSAL_MEMSET(p_eth_qzone, 0, eth_qzone_size); + SET_FIELD(p_coal_timeset->value, COALESCING_TIMESET_TIMESET, timeset); + SET_FIELD(p_coal_timeset->value, COALESCING_TIMESET_VALID, 1); + ecore_memcpy_to(p_hwfn, p_ptt, hw_addr, p_eth_qzone, eth_qzone_size); return ECORE_SUCCESS; } +enum _ecore_status_t ecore_set_queue_coalesce(struct ecore_hwfn *p_hwfn, + u16 rx_coal, u16 tx_coal, + void *p_handle) +{ + struct ecore_queue_cid *p_cid = (struct ecore_queue_cid *)p_handle; + enum _ecore_status_t rc = ECORE_SUCCESS; + struct ecore_ptt *p_ptt; + + /* TODO - Configuring a single queue's coalescing but + * claiming all queues are abiding same configuration + * for PF and VF both. + */ + + if (IS_VF(p_hwfn->p_dev)) + return ecore_vf_pf_set_coalesce(p_hwfn, rx_coal, + tx_coal, p_cid); + + p_ptt = ecore_ptt_acquire(p_hwfn); + if (!p_ptt) + return ECORE_AGAIN; + + if (rx_coal) { + rc = ecore_set_rxq_coalesce(p_hwfn, p_ptt, rx_coal, p_cid); + if (rc) + goto out; + p_hwfn->p_dev->rx_coalesce_usecs = rx_coal; + } + + if (tx_coal) { + rc = ecore_set_txq_coalesce(p_hwfn, p_ptt, tx_coal, p_cid); + if (rc) + goto out; + p_hwfn->p_dev->tx_coalesce_usecs = tx_coal; + } +out: + ecore_ptt_release(p_hwfn, p_ptt); + + return rc; +} + enum _ecore_status_t ecore_set_rxq_coalesce(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, - u8 coalesce, u8 qid) + u16 coalesce, + struct ecore_queue_cid *p_cid) { - struct ustorm_eth_queue_zone qzone; - u16 fw_qid = 0; + struct ustorm_eth_queue_zone eth_qzone; + u8 timeset, timer_res; u32 address; - u8 timeset; enum _ecore_status_t rc; - rc = ecore_fw_l2_queue(p_hwfn, (u16)qid, &fw_qid); + /* Coalesce = (timeset << timer-resolution), timeset is 7bit wide */ + if (coalesce <= 0x7F) { + timer_res = 0; + } else if (coalesce <= 0xFF) { + timer_res = 1; + } else if (coalesce <= 0x1FF) { + timer_res = 2; + } else { + DP_ERR(p_hwfn, "Invalid coalesce value - %d\n", coalesce); + return ECORE_INVAL; + } + timeset = (u8)(coalesce >> timer_res); + + rc = ecore_int_set_timer_res(p_hwfn, p_ptt, timer_res, + p_cid->sb_igu_id, false); if (rc != ECORE_SUCCESS) - return rc; + goto out; - address = BAR0_MAP_REG_USDM_RAM + USTORM_ETH_QUEUE_ZONE_OFFSET(fw_qid); - /* Translate the coalescing time into a timeset, according to: - * Timeout[Rx] = TimeSet[Rx] << (TimerRes[Rx] + 1) - */ - timeset = coalesce >> (ECORE_CAU_DEF_RX_TIMER_RES + 1); + address = BAR0_MAP_REG_USDM_RAM + + USTORM_ETH_QUEUE_ZONE_OFFSET(p_cid->abs.queue_id); - rc = ecore_set_coalesce(p_hwfn, p_ptt, address, &qzone, + rc = ecore_set_coalesce(p_hwfn, p_ptt, address, ð_qzone, sizeof(struct ustorm_eth_queue_zone), timeset); if (rc != ECORE_SUCCESS) goto out; - p_hwfn->p_dev->rx_coalesce_usecs = coalesce; out: return rc; } enum _ecore_status_t ecore_set_txq_coalesce(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, - u8 coalesce, u8 qid) + u16 coalesce, + struct ecore_queue_cid *p_cid) { - struct ystorm_eth_queue_zone qzone; - u16 fw_qid = 0; + struct xstorm_eth_queue_zone eth_qzone; + u8 timeset, timer_res; u32 address; - u8 timeset; enum _ecore_status_t rc; - rc = ecore_fw_l2_queue(p_hwfn, (u16)qid, &fw_qid); - if (rc != ECORE_SUCCESS) - return rc; + /* Coalesce = (timeset << timer-resolution), timeset is 7bit wide */ + if (coalesce <= 0x7F) { + timer_res = 0; + } else if (coalesce <= 0xFF) { + timer_res = 1; + } else if (coalesce <= 0x1FF) { + timer_res = 2; + } else { + DP_ERR(p_hwfn, "Invalid coalesce value - %d\n", coalesce); + return ECORE_INVAL; + } - address = BAR0_MAP_REG_YSDM_RAM + YSTORM_ETH_QUEUE_ZONE_OFFSET(fw_qid); - /* Translate the coalescing time into a timeset, according to: - * Timeout[Tx] = TimeSet[Tx] << (TimerRes[Tx] + 1) - */ - timeset = coalesce >> (ECORE_CAU_DEF_TX_TIMER_RES + 1); + timeset = (u8)(coalesce >> timer_res); - rc = ecore_set_coalesce(p_hwfn, p_ptt, address, &qzone, - sizeof(struct ystorm_eth_queue_zone), timeset); + rc = ecore_int_set_timer_res(p_hwfn, p_ptt, timer_res, + p_cid->sb_igu_id, true); if (rc != ECORE_SUCCESS) goto out; - p_hwfn->p_dev->tx_coalesce_usecs = coalesce; + address = BAR0_MAP_REG_XSDM_RAM + + XSTORM_ETH_QUEUE_ZONE_OFFSET(p_cid->abs.queue_id); + + rc = ecore_set_coalesce(p_hwfn, p_ptt, address, ð_qzone, + sizeof(struct xstorm_eth_queue_zone), timeset); out: return rc; } @@ -3193,56 +5168,52 @@ static void ecore_configure_wfq_for_all_vports(struct ecore_hwfn *p_hwfn, u32 min_pf_rate) { struct init_qm_vport_params *vport_params; - int i, num_vports; + int i; vport_params = p_hwfn->qm_info.qm_vport_params; - num_vports = p_hwfn->qm_info.num_vports; - for (i = 0; i < num_vports; i++) { + for (i = 0; i < p_hwfn->qm_info.num_vports; i++) { u32 wfq_speed = p_hwfn->qm_info.wfq_data[i].min_speed; - vport_params[i].vport_wfq = - (wfq_speed * ECORE_WFQ_UNIT) / min_pf_rate; + vport_params[i].vport_wfq = (wfq_speed * ECORE_WFQ_UNIT) / + min_pf_rate; ecore_init_vport_wfq(p_hwfn, p_ptt, vport_params[i].first_tx_pq_id, vport_params[i].vport_wfq); } } -static void -ecore_init_wfq_default_param(struct ecore_hwfn *p_hwfn, u32 min_pf_rate) +static void ecore_init_wfq_default_param(struct ecore_hwfn *p_hwfn) { - int i, num_vports; - u32 min_speed; + int i; - num_vports = p_hwfn->qm_info.num_vports; - min_speed = min_pf_rate / num_vports; - - for (i = 0; i < num_vports; i++) { + for (i = 0; i < p_hwfn->qm_info.num_vports; i++) p_hwfn->qm_info.qm_vport_params[i].vport_wfq = 1; - p_hwfn->qm_info.wfq_data[i].default_min_speed = min_speed; - } } static void ecore_disable_wfq_for_all_vports(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, - u32 min_pf_rate) + struct ecore_ptt *p_ptt) { struct init_qm_vport_params *vport_params; - int i, num_vports; + int i; vport_params = p_hwfn->qm_info.qm_vport_params; - num_vports = p_hwfn->qm_info.num_vports; - for (i = 0; i < num_vports; i++) { - ecore_init_wfq_default_param(p_hwfn, min_pf_rate); + for (i = 0; i < p_hwfn->qm_info.num_vports; i++) { + ecore_init_wfq_default_param(p_hwfn); ecore_init_vport_wfq(p_hwfn, p_ptt, vport_params[i].first_tx_pq_id, vport_params[i].vport_wfq); } } -/* validate wfq for a given vport and required min rate */ +/* This function performs several validations for WFQ + * configuration and required min rate for a given vport + * 1. req_rate must be greater than one percent of min_pf_rate. + * 2. req_rate should not cause other vports [not configured for WFQ explicitly] + * rates to get less than one percent of min_pf_rate. + * 3. total_req_min_rate [all vports min rate sum] shouldn't exceed min_pf_rate. + */ static enum _ecore_status_t ecore_init_wfq_param(struct ecore_hwfn *p_hwfn, u16 vport_id, u32 req_rate, u32 min_pf_rate) @@ -3252,7 +5223,8 @@ static enum _ecore_status_t ecore_init_wfq_param(struct ecore_hwfn *p_hwfn, num_vports = p_hwfn->qm_info.num_vports; - /* Check pre-set data for some of the vports */ +/* Accounting for the vports which are configured for WFQ explicitly */ + for (i = 0; i < num_vports; i++) { u32 tmp_speed; @@ -3266,36 +5238,27 @@ static enum _ecore_status_t ecore_init_wfq_param(struct ecore_hwfn *p_hwfn, /* Include current vport data as well */ req_count++; total_req_min_rate += req_rate; - non_requested_count = p_hwfn->qm_info.num_vports - req_count; + non_requested_count = num_vports - req_count; /* validate possible error cases */ - if (req_rate > min_pf_rate) { + if (req_rate < min_pf_rate / ECORE_WFQ_UNIT) { DP_VERBOSE(p_hwfn, ECORE_MSG_LINK, - "Vport [%d] - Requested rate[%d Mbps] is greater" - " than configured PF min rate[%d Mbps]\n", - vport_id, req_rate, min_pf_rate); - return ECORE_INVAL; - } - - if (req_rate * ECORE_WFQ_UNIT / min_pf_rate < 1) { - DP_VERBOSE(p_hwfn, ECORE_MSG_LINK, - "Vport [%d] - Requested rate[%d Mbps] is less than" - " one percent of configured PF min rate[%d Mbps]\n", + "Vport [%d] - Requested rate[%d Mbps] is less than one percent of configured PF min rate[%d Mbps]\n", vport_id, req_rate, min_pf_rate); return ECORE_INVAL; } /* TBD - for number of vports greater than 100 */ - if (ECORE_WFQ_UNIT / p_hwfn->qm_info.num_vports < 1) { + if (num_vports > ECORE_WFQ_UNIT) { DP_VERBOSE(p_hwfn, ECORE_MSG_LINK, - "Number of vports are greater than 100\n"); + "Number of vports is greater than %d\n", + ECORE_WFQ_UNIT); return ECORE_INVAL; } if (total_req_min_rate > min_pf_rate) { DP_VERBOSE(p_hwfn, ECORE_MSG_LINK, - "Total requested min rate for all vports[%d Mbps]" - "is greater than configured PF min rate[%d Mbps]\n", + "Total requested min rate for all vports[%d Mbps] is greater than configured PF min rate[%d Mbps]\n", total_req_min_rate, min_pf_rate); return ECORE_INVAL; } @@ -3305,8 +5268,12 @@ static enum _ecore_status_t ecore_init_wfq_param(struct ecore_hwfn *p_hwfn, left_rate_per_vp = total_left_rate / non_requested_count; /* validate if non requested get < 1% of min bw */ - if (left_rate_per_vp * ECORE_WFQ_UNIT / min_pf_rate < 1) + if (left_rate_per_vp < min_pf_rate / ECORE_WFQ_UNIT) { + DP_VERBOSE(p_hwfn, ECORE_MSG_LINK, + "Non WFQ configured vports rate [%d Mbps] is less than one percent of configured PF min rate[%d Mbps]\n", + left_rate_per_vp, min_pf_rate); return ECORE_INVAL; + } /* now req_rate for given vport passes all scenarios. * assign final wfq rates to all vports. @@ -3355,32 +5322,32 @@ static int __ecore_configure_vp_wfq_on_link_change(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, u32 min_pf_rate) { - int rc = ECORE_SUCCESS; bool use_wfq = false; - u16 i, num_vports; - - num_vports = p_hwfn->qm_info.num_vports; + int rc = ECORE_SUCCESS; + u16 i; /* Validate all pre configured vports for wfq */ - for (i = 0; i < num_vports; i++) { - if (p_hwfn->qm_info.wfq_data[i].configured) { - u32 rate = p_hwfn->qm_info.wfq_data[i].min_speed; + for (i = 0; i < p_hwfn->qm_info.num_vports; i++) { + u32 rate; - use_wfq = true; - rc = ecore_init_wfq_param(p_hwfn, i, rate, min_pf_rate); - if (rc == ECORE_INVAL) { - DP_NOTICE(p_hwfn, false, - "Validation failed while" - " configuring min rate\n"); - break; - } + if (!p_hwfn->qm_info.wfq_data[i].configured) + continue; + + rate = p_hwfn->qm_info.wfq_data[i].min_speed; + use_wfq = true; + + rc = ecore_init_wfq_param(p_hwfn, i, rate, min_pf_rate); + if (rc != ECORE_SUCCESS) { + DP_NOTICE(p_hwfn, false, + "WFQ validation failed while configuring min rate\n"); + break; } } if (rc == ECORE_SUCCESS && use_wfq) ecore_configure_wfq_for_all_vports(p_hwfn, p_ptt, min_pf_rate); else - ecore_disable_wfq_for_all_vports(p_hwfn, p_ptt, min_pf_rate); + ecore_disable_wfq_for_all_vports(p_hwfn, p_ptt); return rc; } @@ -3394,9 +5361,9 @@ int ecore_configure_vport_wfq(struct ecore_dev *p_dev, u16 vp_id, u32 rate) int i, rc = ECORE_INVAL; /* TBD - for multiple hardware functions - that is 100 gig */ - if (p_dev->num_hwfns > 1) { + if (ECORE_IS_CMT(p_dev)) { DP_NOTICE(p_dev, false, - "WFQ configuration is not supported for this dev\n"); + "WFQ configuration is not supported for this device\n"); return rc; } @@ -3423,22 +5390,22 @@ int ecore_configure_vport_wfq(struct ecore_dev *p_dev, u16 vp_id, u32 rate) /* API to configure WFQ from mcp link change */ void ecore_configure_vp_wfq_on_link_change(struct ecore_dev *p_dev, + struct ecore_ptt *p_ptt, u32 min_pf_rate) { int i; /* TBD - for multiple hardware functions - that is 100 gig */ - if (p_dev->num_hwfns > 1) { + if (ECORE_IS_CMT(p_dev)) { DP_VERBOSE(p_dev, ECORE_MSG_LINK, - "WFQ configuration is not supported for this dev\n"); + "WFQ configuration is not supported for this device\n"); return; } for_each_hwfn(p_dev, i) { struct ecore_hwfn *p_hwfn = &p_dev->hwfns[i]; - __ecore_configure_vp_wfq_on_link_change(p_hwfn, - p_hwfn->p_dpc_ptt, + __ecore_configure_vp_wfq_on_link_change(p_hwfn, p_ptt, min_pf_rate); } } @@ -3452,12 +5419,21 @@ int __ecore_configure_pf_max_bandwidth(struct ecore_hwfn *p_hwfn, p_hwfn->mcp_info->func_info.bandwidth_max = max_bw; - if (!p_link->line_speed) + if (!p_link->line_speed && (max_bw != 100)) return rc; p_link->speed = (p_link->line_speed * max_bw) / 100; + p_hwfn->qm_info.pf_rl = p_link->speed; - rc = ecore_init_pf_rl(p_hwfn, p_ptt, p_hwfn->rel_pf_id, p_link->speed); + /* Since the limiter also affects Tx-switched traffic, we don't want it + * to limit such traffic in case there's no actual limit. + * In that case, set limit to imaginary high boundary. + */ + if (max_bw == 100) + p_hwfn->qm_info.pf_rl = 100000; + + rc = ecore_init_pf_rl(p_hwfn, p_ptt, p_hwfn->rel_pf_id, + p_hwfn->qm_info.pf_rl); DP_VERBOSE(p_hwfn, ECORE_MSG_LINK, "Configured MAX bandwidth to be %08x Mb/sec\n", @@ -3490,12 +5466,11 @@ int ecore_configure_pf_max_bandwidth(struct ecore_dev *p_dev, u8 max_bw) rc = __ecore_configure_pf_max_bandwidth(p_hwfn, p_ptt, p_link, max_bw); - if (rc != ECORE_SUCCESS) { - ecore_ptt_release(p_hwfn, p_ptt); - return rc; - } ecore_ptt_release(p_hwfn, p_ptt); + + if (rc != ECORE_SUCCESS) + break; } return rc; @@ -3509,6 +5484,7 @@ int __ecore_configure_pf_min_bandwidth(struct ecore_hwfn *p_hwfn, int rc = ECORE_SUCCESS; p_hwfn->mcp_info->func_info.bandwidth_min = min_bw; + p_hwfn->qm_info.pf_wfq = min_bw; if (!p_link->line_speed) return rc; @@ -3574,8 +5550,7 @@ void ecore_clean_wfq_db(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt) p_link = &p_hwfn->mcp_info->link_output; if (p_link->min_pf_rate) - ecore_disable_wfq_for_all_vports(p_hwfn, p_ptt, - p_link->min_pf_rate); + ecore_disable_wfq_for_all_vports(p_hwfn, p_ptt); OSAL_MEMSET(p_hwfn->qm_info.wfq_data, 0, sizeof(*p_hwfn->qm_info.wfq_data) * @@ -3589,9 +5564,18 @@ int ecore_device_num_engines(struct ecore_dev *p_dev) int ecore_device_num_ports(struct ecore_dev *p_dev) { - /* in CMT always only one port */ - if (p_dev->num_hwfns > 1) - return 1; - - return p_dev->num_ports_in_engines * ecore_device_num_engines(p_dev); + return p_dev->num_ports; +} + +void ecore_set_fw_mac_addr(__le16 *fw_msb, + __le16 *fw_mid, + __le16 *fw_lsb, + u8 *mac) +{ + ((u8 *)fw_msb)[0] = mac[1]; + ((u8 *)fw_msb)[1] = mac[0]; + ((u8 *)fw_mid)[0] = mac[3]; + ((u8 *)fw_mid)[1] = mac[2]; + ((u8 *)fw_lsb)[0] = mac[5]; + ((u8 *)fw_lsb)[1] = mac[4]; } diff --git a/dpdk/drivers/net/qede/base/ecore_dev_api.h b/dpdk/drivers/net/qede/base/ecore_dev_api.h index 535b82b5..98bcabe8 100644 --- a/dpdk/drivers/net/qede/base/ecore_dev_api.h +++ b/dpdk/drivers/net/qede/base/ecore_dev_api.h @@ -13,8 +13,6 @@ #include "ecore_chain.h" #include "ecore_int_api.h" -struct ecore_tunn_start_params; - /** * @brief ecore_init_dp - initialize the debug level * @@ -24,7 +22,9 @@ struct ecore_tunn_start_params; * @param dp_ctx */ void ecore_init_dp(struct ecore_dev *p_dev, - u32 dp_module, u8 dp_level, void *dp_ctx); + u32 dp_module, + u8 dp_level, + void *dp_ctx); /** * @brief ecore_init_struct - initialize the device structure to @@ -57,26 +57,65 @@ enum _ecore_status_t ecore_resc_alloc(struct ecore_dev *p_dev); */ void ecore_resc_setup(struct ecore_dev *p_dev); +enum ecore_override_force_load { + ECORE_OVERRIDE_FORCE_LOAD_NONE, + ECORE_OVERRIDE_FORCE_LOAD_ALWAYS, + ECORE_OVERRIDE_FORCE_LOAD_NEVER, +}; + +struct ecore_drv_load_params { + /* Indicates whether the driver is running over a crash kernel. + * As part of the load request, this will be used for providing the + * driver role to the MFW. + * In case of a crash kernel over PDA - this should be set to false. + */ + bool is_crash_kernel; + + /* The timeout value that the MFW should use when locking the engine for + * the driver load process. + * A value of '0' means the default value, and '255' means no timeout. + */ + u8 mfw_timeout_val; +#define ECORE_LOAD_REQ_LOCK_TO_DEFAULT 0 +#define ECORE_LOAD_REQ_LOCK_TO_NONE 255 + + /* Avoid engine reset when first PF loads on it */ + bool avoid_eng_reset; + + /* Allow overriding the default force load behavior */ + enum ecore_override_force_load override_force_load; +}; + +struct ecore_hw_init_params { + /* Tunneling parameters */ + struct ecore_tunnel_info *p_tunn; + + bool b_hw_start; + + /* Interrupt mode [msix, inta, etc.] to use */ + enum ecore_int_mode int_mode; + + /* NPAR tx switching to be used for vports configured for tx-switching + */ + bool allow_npar_tx_switch; + + /* Binary fw data pointer in binary fw file */ + const u8 *bin_fw_data; + + /* Driver load parameters */ + struct ecore_drv_load_params *p_drv_load_params; +}; + /** * @brief ecore_hw_init - * * @param p_dev - * @param p_tunn - tunneling parameters - * @param b_hw_start - * @param int_mode - interrupt mode [msix, inta, etc.] to use. - * @param allow_npar_tx_switch - npar tx switching to be used - * for vports configured for tx-switching. - * @param bin_fw_data - binary fw data pointer in binary fw file. - * Pass NULL if not using binary fw file. + * @param p_params * * @return enum _ecore_status_t */ enum _ecore_status_t ecore_hw_init(struct ecore_dev *p_dev, - struct ecore_tunn_start_params *p_tunn, - bool b_hw_start, - enum ecore_int_mode int_mode, - bool allow_npar_tx_switch, - const u8 *bin_fw_data); + struct ecore_hw_init_params *p_params); /** * @brief ecore_hw_timers_stop_all - @@ -98,14 +137,16 @@ enum _ecore_status_t ecore_hw_stop(struct ecore_dev *p_dev); /** * @brief ecore_hw_stop_fastpath -should be called incase - * slowpath is still required for the device, but - * fastpath is not. + * slowpath is still required for the device, + * but fastpath is not. * * @param p_dev * + * @return enum _ecore_status_t */ -void ecore_hw_stop_fastpath(struct ecore_dev *p_dev); +enum _ecore_status_t ecore_hw_stop_fastpath(struct ecore_dev *p_dev); +#ifndef LINUX_REMOVE /** * @brief ecore_prepare_hibernate -should be called when * the system is going into the hibernate state @@ -115,33 +156,118 @@ void ecore_hw_stop_fastpath(struct ecore_dev *p_dev); */ void ecore_prepare_hibernate(struct ecore_dev *p_dev); +enum ecore_db_rec_width { + DB_REC_WIDTH_32B, + DB_REC_WIDTH_64B, +}; + +enum ecore_db_rec_space { + DB_REC_KERNEL, + DB_REC_USER, +}; + +/** + * @brief db_recovery_add - add doorbell information to the doorbell + * recovery mechanism. + * + * @param p_dev + * @param db_addr - doorbell address + * @param db_data - address of where db_data is stored + * @param db_width - doorbell is 32b pr 64b + * @param db_space - doorbell recovery addresses are user or kernel space + */ +enum _ecore_status_t ecore_db_recovery_add(struct ecore_dev *p_dev, + void OSAL_IOMEM *db_addr, + void *db_data, + enum ecore_db_rec_width db_width, + enum ecore_db_rec_space db_space); + +/** + * @brief db_recovery_del - remove doorbell information from the doorbell + * recovery mechanism. db_data serves as key (db_addr is not unique). + * + * @param cdev + * @param db_addr - doorbell address + * @param db_data - address where db_data is stored. Serves as key for the + * entry to delete. + */ +enum _ecore_status_t ecore_db_recovery_del(struct ecore_dev *p_dev, + void OSAL_IOMEM *db_addr, + void *db_data); + +static OSAL_INLINE bool ecore_is_mf_ufp(struct ecore_hwfn *p_hwfn) +{ + return !!OSAL_TEST_BIT(ECORE_MF_UFP_SPECIFIC, &p_hwfn->p_dev->mf_bits); +} + +#endif + /** * @brief ecore_hw_start_fastpath -restart fastpath traffic, * only if hw_stop_fastpath was called - * @param p_dev - * - */ -void ecore_hw_start_fastpath(struct ecore_hwfn *p_hwfn); - -/** - * @brief ecore_hw_reset - - * - * @param p_dev + * @param p_hwfn * * @return enum _ecore_status_t */ -enum _ecore_status_t ecore_hw_reset(struct ecore_dev *p_dev); +enum _ecore_status_t ecore_hw_start_fastpath(struct ecore_hwfn *p_hwfn); + +enum ecore_hw_prepare_result { + ECORE_HW_PREPARE_SUCCESS, + + /* FAILED results indicate probe has failed & cleaned up */ + ECORE_HW_PREPARE_FAILED_ENG2, + ECORE_HW_PREPARE_FAILED_ME, + ECORE_HW_PREPARE_FAILED_MEM, + ECORE_HW_PREPARE_FAILED_DEV, + ECORE_HW_PREPARE_FAILED_NVM, + + /* BAD results indicate probe is passed even though some wrongness + * has occurred; Trying to actually use [I.e., hw_init()] might have + * dire reprecautions. + */ + ECORE_HW_PREPARE_BAD_IOV, + ECORE_HW_PREPARE_BAD_MCP, + ECORE_HW_PREPARE_BAD_IGU, +}; + +struct ecore_hw_prepare_params { + /* Personality to initialize */ + int personality; + + /* Force the driver's default resource allocation */ + bool drv_resc_alloc; + + /* Check the reg_fifo after any register access */ + bool chk_reg_fifo; + + /* Request the MFW to initiate PF FLR */ + bool initiate_pf_flr; + + /* The OS Epoch time in seconds */ + u32 epoch; + + /* Allow the MFW to collect a crash dump */ + bool allow_mdump; + + /* Allow prepare to pass even if some initializations are failing. + * If set, the `p_prepare_res' field would be set with the return, + * and might allow probe to pass even if there are certain issues. + */ + bool b_relaxed_probe; + enum ecore_hw_prepare_result p_relaxed_res; +}; /** * @brief ecore_hw_prepare - * * @param p_dev - * @param personality - personality to initialize + * @param p_params * * @return enum _ecore_status_t */ -enum _ecore_status_t ecore_hw_prepare(struct ecore_dev *p_dev, int personality); +enum _ecore_status_t ecore_hw_prepare(struct ecore_dev *p_dev, + struct ecore_hw_prepare_params *p_params); /** * @brief ecore_hw_remove - @@ -172,10 +298,10 @@ struct ecore_ptt *ecore_ptt_acquire(struct ecore_hwfn *p_hwfn); * @param p_hwfn * @param p_ptt */ -void ecore_ptt_release(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt); +void ecore_ptt_release(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt); -#ifndef __EXTRACT__LINUX__ -struct ecore_eth_stats { +struct ecore_eth_stats_common { u64 no_buff_discards; u64 packet_too_big_discard; u64 ttl0_discard; @@ -207,11 +333,6 @@ struct ecore_eth_stats { u64 rx_256_to_511_byte_packets; u64 rx_512_to_1023_byte_packets; u64 rx_1024_to_1518_byte_packets; - u64 rx_1519_to_1522_byte_packets; - u64 rx_1519_to_2047_byte_packets; - u64 rx_2048_to_4095_byte_packets; - u64 rx_4096_to_9216_byte_packets; - u64 rx_9217_to_16383_byte_packets; u64 rx_crc_errors; u64 rx_mac_crtl_frames; u64 rx_pause_frames; @@ -228,14 +349,8 @@ struct ecore_eth_stats { u64 tx_256_to_511_byte_packets; u64 tx_512_to_1023_byte_packets; u64 tx_1024_to_1518_byte_packets; - u64 tx_1519_to_2047_byte_packets; - u64 tx_2048_to_4095_byte_packets; - u64 tx_4096_to_9216_byte_packets; - u64 tx_9217_to_16383_byte_packets; u64 tx_pause_frames; u64 tx_pfc_frames; - u64 tx_lpi_entry_count; - u64 tx_total_collisions; u64 brb_truncates; u64 brb_discards; u64 rx_mac_bytes; @@ -249,7 +364,33 @@ struct ecore_eth_stats { u64 tx_mac_bc_packets; u64 tx_mac_ctrl_frames; }; -#endif + +struct ecore_eth_stats_bb { + u64 rx_1519_to_1522_byte_packets; + u64 rx_1519_to_2047_byte_packets; + u64 rx_2048_to_4095_byte_packets; + u64 rx_4096_to_9216_byte_packets; + u64 rx_9217_to_16383_byte_packets; + u64 tx_1519_to_2047_byte_packets; + u64 tx_2048_to_4095_byte_packets; + u64 tx_4096_to_9216_byte_packets; + u64 tx_9217_to_16383_byte_packets; + u64 tx_lpi_entry_count; + u64 tx_total_collisions; +}; + +struct ecore_eth_stats_ah { + u64 rx_1519_to_max_byte_packets; + u64 tx_1519_to_max_byte_packets; +}; + +struct ecore_eth_stats { + struct ecore_eth_stats_common common; + union { + struct ecore_eth_stats_bb bb; + struct ecore_eth_stats_ah ah; + }; +}; enum ecore_dmae_address_type_t { ECORE_DMAE_ADDRESS_HOST_VIRT, @@ -270,61 +411,66 @@ enum ecore_dmae_address_type_t { #define ECORE_DMAE_FLAG_COMPLETION_DST 0x00000008 struct ecore_dmae_params { - u32 flags; /* consists of ECORE_DMAE_FLAG_* values */ + u32 flags; /* consists of ECORE_DMAE_FLAG_* values */ u8 src_vfid; u8 dst_vfid; }; /** -* @brief ecore_dmae_host2grc - copy data from source addr to -* dmae registers using the given ptt -* -* @param p_hwfn -* @param p_ptt -* @param source_addr -* @param grc_addr (dmae_data_offset) -* @param size_in_dwords -* @param flags (one of the flags defined above) -*/ + * @brief ecore_dmae_host2grc - copy data from source addr to + * dmae registers using the given ptt + * + * @param p_hwfn + * @param p_ptt + * @param source_addr + * @param grc_addr (dmae_data_offset) + * @param size_in_dwords + * @param flags (one of the flags defined above) + */ enum _ecore_status_t ecore_dmae_host2grc(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, u64 source_addr, - u32 grc_addr, u32 size_in_dwords, u32 flags); + u32 grc_addr, + u32 size_in_dwords, + u32 flags); /** -* @brief ecore_dmae_grc2host - Read data from dmae data offset -* to source address using the given ptt -* -* @param p_ptt -* @param grc_addr (dmae_data_offset) -* @param dest_addr -* @param size_in_dwords -* @param flags - one of the flags defined above -*/ + * @brief ecore_dmae_grc2host - Read data from dmae data offset + * to source address using the given ptt + * + * @param p_ptt + * @param grc_addr (dmae_data_offset) + * @param dest_addr + * @param size_in_dwords + * @param flags - one of the flags defined above + */ enum _ecore_status_t ecore_dmae_grc2host(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, u32 grc_addr, - dma_addr_t dest_addr, u32 size_in_dwords, u32 flags); + dma_addr_t dest_addr, + u32 size_in_dwords, + u32 flags); /** -* @brief ecore_dmae_host2host - copy data from to source address -* to a destination address (for SRIOV) using the given ptt -* -* @param p_hwfn -* @param p_ptt -* @param source_addr -* @param dest_addr -* @param size_in_dwords -* @param params -*/ + * @brief ecore_dmae_host2host - copy data from to source address + * to a destination address (for SRIOV) using the given ptt + * + * @param p_hwfn + * @param p_ptt + * @param source_addr + * @param dest_addr + * @param size_in_dwords + * @param params + */ enum _ecore_status_t ecore_dmae_host2host(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, dma_addr_t source_addr, dma_addr_t dest_addr, - u32 size_in_dwords, struct ecore_dmae_params *p_params); + u32 size_in_dwords, + struct ecore_dmae_params *p_params); /** * @brief ecore_chain_alloc - Allocate and initialize a chain @@ -344,7 +490,9 @@ ecore_chain_alloc(struct ecore_dev *p_dev, enum ecore_chain_mode mode, enum ecore_chain_cnt_type cnt_type, u32 num_elems, - osal_size_t elem_size, struct ecore_chain *p_chain); + osal_size_t elem_size, + struct ecore_chain *p_chain, + struct ecore_chain_ext_pbl *ext_pbl); /** * @brief ecore_chain_free - Free chain DMA memory @@ -352,7 +500,8 @@ ecore_chain_alloc(struct ecore_dev *p_dev, * @param p_hwfn * @param p_chain */ -void ecore_chain_free(struct ecore_dev *p_dev, struct ecore_chain *p_chain); +void ecore_chain_free(struct ecore_dev *p_dev, + struct ecore_chain *p_chain); /** * @@brief ecore_fw_l2_queue - Get absolute L2 queue ID @@ -364,7 +513,8 @@ void ecore_chain_free(struct ecore_dev *p_dev, struct ecore_chain *p_chain); * @return enum _ecore_status_t */ enum _ecore_status_t ecore_fw_l2_queue(struct ecore_hwfn *p_hwfn, - u16 src_id, u16 *dst_id); + u16 src_id, + u16 *dst_id); /** * @@brief ecore_fw_vport - Get absolute vport ID @@ -376,7 +526,8 @@ enum _ecore_status_t ecore_fw_l2_queue(struct ecore_hwfn *p_hwfn, * @return enum _ecore_status_t */ enum _ecore_status_t ecore_fw_vport(struct ecore_hwfn *p_hwfn, - u8 src_id, u8 *dst_id); + u8 src_id, + u8 *dst_id); /** * @@brief ecore_fw_rss_eng - Get absolute RSS engine ID @@ -388,7 +539,8 @@ enum _ecore_status_t ecore_fw_vport(struct ecore_hwfn *p_hwfn, * @return enum _ecore_status_t */ enum _ecore_status_t ecore_fw_rss_eng(struct ecore_hwfn *p_hwfn, - u8 src_id, u8 *dst_id); + u8 src_id, + u8 *dst_id); /** * @brief ecore_llh_add_mac_filter - configures a MAC filter in llh @@ -398,8 +550,8 @@ enum _ecore_status_t ecore_fw_rss_eng(struct ecore_hwfn *p_hwfn, * @param p_filter - MAC to add */ enum _ecore_status_t ecore_llh_add_mac_filter(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, - u8 *p_filter); + struct ecore_ptt *p_ptt, + u8 *p_filter); /** * @brief ecore_llh_remove_mac_filter - removes a MAC filtre from llh @@ -409,28 +561,50 @@ enum _ecore_status_t ecore_llh_add_mac_filter(struct ecore_hwfn *p_hwfn, * @param p_filter - MAC to remove */ void ecore_llh_remove_mac_filter(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, u8 *p_filter); + struct ecore_ptt *p_ptt, + u8 *p_filter); + +enum ecore_llh_port_filter_type_t { + ECORE_LLH_FILTER_ETHERTYPE, + ECORE_LLH_FILTER_TCP_SRC_PORT, + ECORE_LLH_FILTER_TCP_DEST_PORT, + ECORE_LLH_FILTER_TCP_SRC_AND_DEST_PORT, + ECORE_LLH_FILTER_UDP_SRC_PORT, + ECORE_LLH_FILTER_UDP_DEST_PORT, + ECORE_LLH_FILTER_UDP_SRC_AND_DEST_PORT +}; /** - * @brief ecore_llh_add_ethertype_filter - configures a ethertype filter in llh + * @brief ecore_llh_add_protocol_filter - configures a protocol filter in llh * * @param p_hwfn * @param p_ptt - * @param filter - ethertype to add + * @param source_port_or_eth_type - source port or ethertype to add + * @param dest_port - destination port to add + * @param type - type of filters and comparing */ -enum _ecore_status_t ecore_llh_add_ethertype_filter(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, - u16 filter); +enum _ecore_status_t +ecore_llh_add_protocol_filter(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + u16 source_port_or_eth_type, + u16 dest_port, + enum ecore_llh_port_filter_type_t type); /** - * @brief ecore_llh_remove_ethertype_filter - removes a ethertype llh filter + * @brief ecore_llh_remove_protocol_filter - remove a protocol filter in llh * * @param p_hwfn * @param p_ptt - * @param filter - ethertype to remove + * @param source_port_or_eth_type - source port or ethertype to add + * @param dest_port - destination port to add + * @param type - type of filters and comparing */ -void ecore_llh_remove_ethertype_filter(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, u16 filter); +void +ecore_llh_remove_protocol_filter(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + u16 source_port_or_eth_type, + u16 dest_port, + enum ecore_llh_port_filter_type_t type); /** * @brief ecore_llh_clear_all_filters - removes all MAC filters from llh @@ -439,10 +613,20 @@ void ecore_llh_remove_ethertype_filter(struct ecore_hwfn *p_hwfn, * @param p_ptt */ void ecore_llh_clear_all_filters(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt); + struct ecore_ptt *p_ptt); - /** -*@brief Cleanup of previous driver remains prior to load +/** + * @brief ecore_llh_set_function_as_default - set function as default per port + * + * @param p_hwfn + * @param p_ptt + */ +enum _ecore_status_t +ecore_llh_set_function_as_default(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt); + +/** + *@brief Cleanup of previous driver remains prior to load * * @param p_hwfn * @param p_ptt @@ -451,47 +635,54 @@ void ecore_llh_clear_all_filters(struct ecore_hwfn *p_hwfn, * * @return enum _ecore_status_t */ -enum _ecore_status_t ecore_final_cleanup(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, - u16 id, bool is_vf); +enum _ecore_status_t ecore_final_cleanup(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + u16 id, + bool is_vf); /** - * @brief ecore_test_registers - Perform register tests + * @brief ecore_get_queue_coalesce - Retrieve coalesce value for a given queue. + * + * @param p_hwfn + * @param p_coal - store coalesce value read from the hardware. + * @param p_handle + * + * @return enum _ecore_status_t + **/ +enum _ecore_status_t +ecore_get_queue_coalesce(struct ecore_hwfn *p_hwfn, u16 *coal, + void *handle); + +/** + * @brief ecore_set_queue_coalesce - Configure coalesce parameters for Rx and + * Tx queue. The fact that we can configure coalescing to up to 511, but on + * varying accuracy [the bigger the value the less accurate] up to a mistake + * of 3usec for the highest values. + * While the API allows setting coalescing per-qid, all queues sharing a SB + * should be in same range [i.e., either 0-0x7f, 0x80-0xff or 0x100-0x1ff] + * otherwise configuration would break. + * + * @param p_hwfn + * @param rx_coal - Rx Coalesce value in micro seconds. + * @param tx_coal - TX Coalesce value in micro seconds. + * @param p_handle + * + * @return enum _ecore_status_t + **/ +enum _ecore_status_t +ecore_set_queue_coalesce(struct ecore_hwfn *p_hwfn, u16 rx_coal, + u16 tx_coal, void *p_handle); + +/** + * @brief ecore_pglueb_set_pfid_enable - Enable or disable PCI BUS MASTER * * @param p_hwfn * @param p_ptt - * - * @return enum _ecore_status_t - */ -enum _ecore_status_t ecore_test_registers(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt); - -/** - * @brief ecore_set_rxq_coalesce - Configure coalesce parameters for an Rx queue - * - * @param p_hwfn - * @param p_ptt - * @param coalesce - Coalesce value in micro seconds. - * @param qid - Queue index. + * @param b_enable - true/false * * @return enum _ecore_status_t */ -enum _ecore_status_t ecore_set_rxq_coalesce(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, - u8 coalesce, u8 qid); - -/** - * @brief ecore_set_txq_coalesce - Configure coalesce parameters for a Tx queue - * - * @param p_hwfn - * @param p_ptt - * @param coalesce - Coalesce value in micro seconds. - * @param qid - Queue index. - * - * @return enum _ecore_status_t - */ -enum _ecore_status_t ecore_set_txq_coalesce(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, - u8 coalesce, u8 qid); - +enum _ecore_status_t ecore_pglueb_set_pfid_enable(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + bool b_enable); #endif diff --git a/dpdk/drivers/net/qede/base/ecore_gtt_reg_addr.h b/dpdk/drivers/net/qede/base/ecore_gtt_reg_addr.h index cc49fc7b..2acd864d 100644 --- a/dpdk/drivers/net/qede/base/ecore_gtt_reg_addr.h +++ b/dpdk/drivers/net/qede/base/ecore_gtt_reg_addr.h @@ -10,33 +10,43 @@ #define GTT_REG_ADDR_H /* Win 2 */ -#define GTT_BAR0_MAP_REG_IGU_CMD 0x00f000UL +/* Access:RW DataWidth:0x20 */ +#define GTT_BAR0_MAP_REG_IGU_CMD 0x00f000UL /* Win 3 */ -#define GTT_BAR0_MAP_REG_TSDM_RAM 0x010000UL +/* Access:RW DataWidth:0x20 */ +#define GTT_BAR0_MAP_REG_TSDM_RAM 0x010000UL /* Win 4 */ -#define GTT_BAR0_MAP_REG_MSDM_RAM 0x011000UL +/* Access:RW DataWidth:0x20 */ +#define GTT_BAR0_MAP_REG_MSDM_RAM 0x011000UL /* Win 5 */ -#define GTT_BAR0_MAP_REG_MSDM_RAM_1024 0x012000UL +/* Access:RW DataWidth:0x20 */ +#define GTT_BAR0_MAP_REG_MSDM_RAM_1024 0x012000UL /* Win 6 */ -#define GTT_BAR0_MAP_REG_USDM_RAM 0x013000UL +/* Access:RW DataWidth:0x20 */ +#define GTT_BAR0_MAP_REG_USDM_RAM 0x013000UL /* Win 7 */ -#define GTT_BAR0_MAP_REG_USDM_RAM_1024 0x014000UL +/* Access:RW DataWidth:0x20 */ +#define GTT_BAR0_MAP_REG_USDM_RAM_1024 0x014000UL /* Win 8 */ -#define GTT_BAR0_MAP_REG_USDM_RAM_2048 0x015000UL +/* Access:RW DataWidth:0x20 */ +#define GTT_BAR0_MAP_REG_USDM_RAM_2048 0x015000UL /* Win 9 */ -#define GTT_BAR0_MAP_REG_XSDM_RAM 0x016000UL +/* Access:RW DataWidth:0x20 */ +#define GTT_BAR0_MAP_REG_XSDM_RAM 0x016000UL /* Win 10 */ -#define GTT_BAR0_MAP_REG_YSDM_RAM 0x017000UL +/* Access:RW DataWidth:0x20 */ +#define GTT_BAR0_MAP_REG_YSDM_RAM 0x017000UL /* Win 11 */ -#define GTT_BAR0_MAP_REG_PSDM_RAM 0x018000UL +/* Access:RW DataWidth:0x20 */ +#define GTT_BAR0_MAP_REG_PSDM_RAM 0x018000UL #endif diff --git a/dpdk/drivers/net/qede/base/ecore_gtt_values.h b/dpdk/drivers/net/qede/base/ecore_gtt_values.h index f2efe24e..2ddc5f19 100644 --- a/dpdk/drivers/net/qede/base/ecore_gtt_values.h +++ b/dpdk/drivers/net/qede/base/ecore_gtt_values.h @@ -11,16 +11,16 @@ static u32 pxp_global_win[] = { 0, 0, - 0x1c02, /* win 2: addr=0x1c02000, size=4096 bytes */ - 0x1c80, /* win 3: addr=0x1c80000, size=4096 bytes */ - 0x1d00, /* win 4: addr=0x1d00000, size=4096 bytes */ - 0x1d01, /* win 5: addr=0x1d01000, size=4096 bytes */ - 0x1d80, /* win 6: addr=0x1d80000, size=4096 bytes */ - 0x1d81, /* win 7: addr=0x1d81000, size=4096 bytes */ - 0x1d82, /* win 8: addr=0x1d82000, size=4096 bytes */ - 0x1e00, /* win 9: addr=0x1e00000, size=4096 bytes */ - 0x1e80, /* win 10: addr=0x1e80000, size=4096 bytes */ - 0x1f00, /* win 11: addr=0x1f00000, size=4096 bytes */ + 0x1c02, /* win 2: addr=0x1c02000, size=4096 bytes */ + 0x1c80, /* win 3: addr=0x1c80000, size=4096 bytes */ + 0x1d00, /* win 4: addr=0x1d00000, size=4096 bytes */ + 0x1d01, /* win 5: addr=0x1d01000, size=4096 bytes */ + 0x1d80, /* win 6: addr=0x1d80000, size=4096 bytes */ + 0x1d81, /* win 7: addr=0x1d81000, size=4096 bytes */ + 0x1d82, /* win 8: addr=0x1d82000, size=4096 bytes */ + 0x1e00, /* win 9: addr=0x1e00000, size=4096 bytes */ + 0x1e80, /* win 10: addr=0x1e80000, size=4096 bytes */ + 0x1f00, /* win 11: addr=0x1f00000, size=4096 bytes */ 0, 0, 0, diff --git a/dpdk/drivers/net/qede/base/ecore_hsi_common.h b/dpdk/drivers/net/qede/base/ecore_hsi_common.h index e341b95c..d8abd604 100644 --- a/dpdk/drivers/net/qede/base/ecore_hsi_common.h +++ b/dpdk/drivers/net/qede/base/ecore_hsi_common.h @@ -13,6 +13,7 @@ /********************************/ #include "common_hsi.h" + /* * opcodes for the event ring */ @@ -25,10 +26,12 @@ enum common_event_opcode { COMMON_EVENT_VF_FLR, COMMON_EVENT_PF_UPDATE, COMMON_EVENT_MALICIOUS_VF, + COMMON_EVENT_RL_UPDATE, COMMON_EVENT_EMPTY, MAX_COMMON_EVENT_OPCODE }; + /* * Common Ramrod Command IDs */ @@ -39,10 +42,12 @@ enum common_ramrod_cmd_id { COMMON_RAMROD_VF_START /* VF Function Start */, COMMON_RAMROD_VF_STOP /* VF Function Stop Ramrod */, COMMON_RAMROD_PF_UPDATE /* PF update Ramrod */, + COMMON_RAMROD_RL_UPDATE /* QCN/DCQCN RL update Ramrod */, COMMON_RAMROD_EMPTY /* Empty Ramrod */, MAX_COMMON_RAMROD_CMD_ID }; + /* * The core storm context for the Ystorm */ @@ -63,223 +68,313 @@ struct pstorm_core_conn_st_ctx { struct xstorm_core_conn_st_ctx { __le32 spq_base_lo /* SPQ Ring Base Address low dword */; __le32 spq_base_hi /* SPQ Ring Base Address high dword */; - struct regpair consolid_base_addr /* Consolidation Ring Base Address */ - ; +/* Consolidation Ring Base Address */ + struct regpair consolid_base_addr; __le16 spq_cons /* SPQ Ring Consumer */; __le16 consolid_cons /* Consolidation Ring Consumer */; __le32 reserved0[55] /* Pad to 15 cycles */; }; -struct xstorm_core_conn_ag_ctx { +struct e4_xstorm_core_conn_ag_ctx { u8 reserved0 /* cdu_validation */; u8 core_state /* state */; u8 flags0; -#define XSTORM_CORE_CONN_AG_CTX_EXIST_IN_QM0_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_EXIST_IN_QM0_SHIFT 0 -#define XSTORM_CORE_CONN_AG_CTX_RESERVED1_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_RESERVED1_SHIFT 1 -#define XSTORM_CORE_CONN_AG_CTX_RESERVED2_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_RESERVED2_SHIFT 2 -#define XSTORM_CORE_CONN_AG_CTX_EXIST_IN_QM3_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_EXIST_IN_QM3_SHIFT 3 -#define XSTORM_CORE_CONN_AG_CTX_RESERVED3_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_RESERVED3_SHIFT 4 -#define XSTORM_CORE_CONN_AG_CTX_RESERVED4_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_RESERVED4_SHIFT 5 -#define XSTORM_CORE_CONN_AG_CTX_RESERVED5_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_RESERVED5_SHIFT 6 -#define XSTORM_CORE_CONN_AG_CTX_RESERVED6_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_RESERVED6_SHIFT 7 +/* exist_in_qm0 */ +#define E4_XSTORM_CORE_CONN_AG_CTX_EXIST_IN_QM0_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_EXIST_IN_QM0_SHIFT 0 +/* exist_in_qm1 */ +#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED1_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED1_SHIFT 1 +/* exist_in_qm2 */ +#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED2_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED2_SHIFT 2 +/* exist_in_qm3 */ +#define E4_XSTORM_CORE_CONN_AG_CTX_EXIST_IN_QM3_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_EXIST_IN_QM3_SHIFT 3 +/* bit4 */ +#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED3_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED3_SHIFT 4 +/* cf_array_active */ +#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED4_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED4_SHIFT 5 +/* bit6 */ +#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED5_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED5_SHIFT 6 +/* bit7 */ +#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED6_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED6_SHIFT 7 u8 flags1; -#define XSTORM_CORE_CONN_AG_CTX_RESERVED7_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_RESERVED7_SHIFT 0 -#define XSTORM_CORE_CONN_AG_CTX_RESERVED8_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_RESERVED8_SHIFT 1 -#define XSTORM_CORE_CONN_AG_CTX_RESERVED9_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_RESERVED9_SHIFT 2 -#define XSTORM_CORE_CONN_AG_CTX_BIT11_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_BIT11_SHIFT 3 -#define XSTORM_CORE_CONN_AG_CTX_BIT12_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_BIT12_SHIFT 4 -#define XSTORM_CORE_CONN_AG_CTX_BIT13_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_BIT13_SHIFT 5 -#define XSTORM_CORE_CONN_AG_CTX_TX_RULE_ACTIVE_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_TX_RULE_ACTIVE_SHIFT 6 -#define XSTORM_CORE_CONN_AG_CTX_DQ_CF_ACTIVE_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_DQ_CF_ACTIVE_SHIFT 7 +/* bit8 */ +#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED7_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED7_SHIFT 0 +/* bit9 */ +#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED8_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED8_SHIFT 1 +/* bit10 */ +#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED9_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED9_SHIFT 2 +/* bit11 */ +#define E4_XSTORM_CORE_CONN_AG_CTX_BIT11_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_BIT11_SHIFT 3 +/* bit12 */ +#define E4_XSTORM_CORE_CONN_AG_CTX_BIT12_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_BIT12_SHIFT 4 +/* bit13 */ +#define E4_XSTORM_CORE_CONN_AG_CTX_BIT13_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_BIT13_SHIFT 5 +/* bit14 */ +#define E4_XSTORM_CORE_CONN_AG_CTX_TX_RULE_ACTIVE_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_TX_RULE_ACTIVE_SHIFT 6 +/* bit15 */ +#define E4_XSTORM_CORE_CONN_AG_CTX_DQ_CF_ACTIVE_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_DQ_CF_ACTIVE_SHIFT 7 u8 flags2; -#define XSTORM_CORE_CONN_AG_CTX_CF0_MASK 0x3 -#define XSTORM_CORE_CONN_AG_CTX_CF0_SHIFT 0 -#define XSTORM_CORE_CONN_AG_CTX_CF1_MASK 0x3 -#define XSTORM_CORE_CONN_AG_CTX_CF1_SHIFT 2 -#define XSTORM_CORE_CONN_AG_CTX_CF2_MASK 0x3 -#define XSTORM_CORE_CONN_AG_CTX_CF2_SHIFT 4 -#define XSTORM_CORE_CONN_AG_CTX_CF3_MASK 0x3 -#define XSTORM_CORE_CONN_AG_CTX_CF3_SHIFT 6 +/* timer0cf */ +#define E4_XSTORM_CORE_CONN_AG_CTX_CF0_MASK 0x3 +#define E4_XSTORM_CORE_CONN_AG_CTX_CF0_SHIFT 0 +/* timer1cf */ +#define E4_XSTORM_CORE_CONN_AG_CTX_CF1_MASK 0x3 +#define E4_XSTORM_CORE_CONN_AG_CTX_CF1_SHIFT 2 +/* timer2cf */ +#define E4_XSTORM_CORE_CONN_AG_CTX_CF2_MASK 0x3 +#define E4_XSTORM_CORE_CONN_AG_CTX_CF2_SHIFT 4 +/* timer_stop_all */ +#define E4_XSTORM_CORE_CONN_AG_CTX_CF3_MASK 0x3 +#define E4_XSTORM_CORE_CONN_AG_CTX_CF3_SHIFT 6 u8 flags3; -#define XSTORM_CORE_CONN_AG_CTX_CF4_MASK 0x3 -#define XSTORM_CORE_CONN_AG_CTX_CF4_SHIFT 0 -#define XSTORM_CORE_CONN_AG_CTX_CF5_MASK 0x3 -#define XSTORM_CORE_CONN_AG_CTX_CF5_SHIFT 2 -#define XSTORM_CORE_CONN_AG_CTX_CF6_MASK 0x3 -#define XSTORM_CORE_CONN_AG_CTX_CF6_SHIFT 4 -#define XSTORM_CORE_CONN_AG_CTX_CF7_MASK 0x3 -#define XSTORM_CORE_CONN_AG_CTX_CF7_SHIFT 6 +#define E4_XSTORM_CORE_CONN_AG_CTX_CF4_MASK 0x3 /* cf4 */ +#define E4_XSTORM_CORE_CONN_AG_CTX_CF4_SHIFT 0 +#define E4_XSTORM_CORE_CONN_AG_CTX_CF5_MASK 0x3 /* cf5 */ +#define E4_XSTORM_CORE_CONN_AG_CTX_CF5_SHIFT 2 +#define E4_XSTORM_CORE_CONN_AG_CTX_CF6_MASK 0x3 /* cf6 */ +#define E4_XSTORM_CORE_CONN_AG_CTX_CF6_SHIFT 4 +#define E4_XSTORM_CORE_CONN_AG_CTX_CF7_MASK 0x3 /* cf7 */ +#define E4_XSTORM_CORE_CONN_AG_CTX_CF7_SHIFT 6 u8 flags4; -#define XSTORM_CORE_CONN_AG_CTX_CF8_MASK 0x3 -#define XSTORM_CORE_CONN_AG_CTX_CF8_SHIFT 0 -#define XSTORM_CORE_CONN_AG_CTX_CF9_MASK 0x3 -#define XSTORM_CORE_CONN_AG_CTX_CF9_SHIFT 2 -#define XSTORM_CORE_CONN_AG_CTX_CF10_MASK 0x3 -#define XSTORM_CORE_CONN_AG_CTX_CF10_SHIFT 4 -#define XSTORM_CORE_CONN_AG_CTX_CF11_MASK 0x3 -#define XSTORM_CORE_CONN_AG_CTX_CF11_SHIFT 6 +#define E4_XSTORM_CORE_CONN_AG_CTX_CF8_MASK 0x3 /* cf8 */ +#define E4_XSTORM_CORE_CONN_AG_CTX_CF8_SHIFT 0 +#define E4_XSTORM_CORE_CONN_AG_CTX_CF9_MASK 0x3 /* cf9 */ +#define E4_XSTORM_CORE_CONN_AG_CTX_CF9_SHIFT 2 +/* cf10 */ +#define E4_XSTORM_CORE_CONN_AG_CTX_CF10_MASK 0x3 +#define E4_XSTORM_CORE_CONN_AG_CTX_CF10_SHIFT 4 +/* cf11 */ +#define E4_XSTORM_CORE_CONN_AG_CTX_CF11_MASK 0x3 +#define E4_XSTORM_CORE_CONN_AG_CTX_CF11_SHIFT 6 u8 flags5; -#define XSTORM_CORE_CONN_AG_CTX_CF12_MASK 0x3 -#define XSTORM_CORE_CONN_AG_CTX_CF12_SHIFT 0 -#define XSTORM_CORE_CONN_AG_CTX_CF13_MASK 0x3 -#define XSTORM_CORE_CONN_AG_CTX_CF13_SHIFT 2 -#define XSTORM_CORE_CONN_AG_CTX_CF14_MASK 0x3 -#define XSTORM_CORE_CONN_AG_CTX_CF14_SHIFT 4 -#define XSTORM_CORE_CONN_AG_CTX_CF15_MASK 0x3 -#define XSTORM_CORE_CONN_AG_CTX_CF15_SHIFT 6 +/* cf12 */ +#define E4_XSTORM_CORE_CONN_AG_CTX_CF12_MASK 0x3 +#define E4_XSTORM_CORE_CONN_AG_CTX_CF12_SHIFT 0 +/* cf13 */ +#define E4_XSTORM_CORE_CONN_AG_CTX_CF13_MASK 0x3 +#define E4_XSTORM_CORE_CONN_AG_CTX_CF13_SHIFT 2 +/* cf14 */ +#define E4_XSTORM_CORE_CONN_AG_CTX_CF14_MASK 0x3 +#define E4_XSTORM_CORE_CONN_AG_CTX_CF14_SHIFT 4 +/* cf15 */ +#define E4_XSTORM_CORE_CONN_AG_CTX_CF15_MASK 0x3 +#define E4_XSTORM_CORE_CONN_AG_CTX_CF15_SHIFT 6 u8 flags6; -#define XSTORM_CORE_CONN_AG_CTX_CONSOLID_PROD_CF_MASK 0x3 -#define XSTORM_CORE_CONN_AG_CTX_CONSOLID_PROD_CF_SHIFT 0 -#define XSTORM_CORE_CONN_AG_CTX_CF17_MASK 0x3 -#define XSTORM_CORE_CONN_AG_CTX_CF17_SHIFT 2 -#define XSTORM_CORE_CONN_AG_CTX_DQ_CF_MASK 0x3 -#define XSTORM_CORE_CONN_AG_CTX_DQ_CF_SHIFT 4 -#define XSTORM_CORE_CONN_AG_CTX_TERMINATE_CF_MASK 0x3 -#define XSTORM_CORE_CONN_AG_CTX_TERMINATE_CF_SHIFT 6 +/* cf16 */ +#define E4_XSTORM_CORE_CONN_AG_CTX_CONSOLID_PROD_CF_MASK 0x3 +#define E4_XSTORM_CORE_CONN_AG_CTX_CONSOLID_PROD_CF_SHIFT 0 +/* cf_array_cf */ +#define E4_XSTORM_CORE_CONN_AG_CTX_CF17_MASK 0x3 +#define E4_XSTORM_CORE_CONN_AG_CTX_CF17_SHIFT 2 +/* cf18 */ +#define E4_XSTORM_CORE_CONN_AG_CTX_DQ_CF_MASK 0x3 +#define E4_XSTORM_CORE_CONN_AG_CTX_DQ_CF_SHIFT 4 +/* cf19 */ +#define E4_XSTORM_CORE_CONN_AG_CTX_TERMINATE_CF_MASK 0x3 +#define E4_XSTORM_CORE_CONN_AG_CTX_TERMINATE_CF_SHIFT 6 u8 flags7; -#define XSTORM_CORE_CONN_AG_CTX_FLUSH_Q0_MASK 0x3 -#define XSTORM_CORE_CONN_AG_CTX_FLUSH_Q0_SHIFT 0 -#define XSTORM_CORE_CONN_AG_CTX_RESERVED10_MASK 0x3 -#define XSTORM_CORE_CONN_AG_CTX_RESERVED10_SHIFT 2 -#define XSTORM_CORE_CONN_AG_CTX_SLOW_PATH_MASK 0x3 -#define XSTORM_CORE_CONN_AG_CTX_SLOW_PATH_SHIFT 4 -#define XSTORM_CORE_CONN_AG_CTX_CF0EN_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_CF0EN_SHIFT 6 -#define XSTORM_CORE_CONN_AG_CTX_CF1EN_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_CF1EN_SHIFT 7 +/* cf20 */ +#define E4_XSTORM_CORE_CONN_AG_CTX_FLUSH_Q0_MASK 0x3 +#define E4_XSTORM_CORE_CONN_AG_CTX_FLUSH_Q0_SHIFT 0 +/* cf21 */ +#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED10_MASK 0x3 +#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED10_SHIFT 2 +/* cf22 */ +#define E4_XSTORM_CORE_CONN_AG_CTX_SLOW_PATH_MASK 0x3 +#define E4_XSTORM_CORE_CONN_AG_CTX_SLOW_PATH_SHIFT 4 +/* cf0en */ +#define E4_XSTORM_CORE_CONN_AG_CTX_CF0EN_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_CF0EN_SHIFT 6 +/* cf1en */ +#define E4_XSTORM_CORE_CONN_AG_CTX_CF1EN_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_CF1EN_SHIFT 7 u8 flags8; -#define XSTORM_CORE_CONN_AG_CTX_CF2EN_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_CF2EN_SHIFT 0 -#define XSTORM_CORE_CONN_AG_CTX_CF3EN_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_CF3EN_SHIFT 1 -#define XSTORM_CORE_CONN_AG_CTX_CF4EN_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_CF4EN_SHIFT 2 -#define XSTORM_CORE_CONN_AG_CTX_CF5EN_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_CF5EN_SHIFT 3 -#define XSTORM_CORE_CONN_AG_CTX_CF6EN_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_CF6EN_SHIFT 4 -#define XSTORM_CORE_CONN_AG_CTX_CF7EN_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_CF7EN_SHIFT 5 -#define XSTORM_CORE_CONN_AG_CTX_CF8EN_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_CF8EN_SHIFT 6 -#define XSTORM_CORE_CONN_AG_CTX_CF9EN_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_CF9EN_SHIFT 7 +/* cf2en */ +#define E4_XSTORM_CORE_CONN_AG_CTX_CF2EN_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_CF2EN_SHIFT 0 +/* cf3en */ +#define E4_XSTORM_CORE_CONN_AG_CTX_CF3EN_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_CF3EN_SHIFT 1 +/* cf4en */ +#define E4_XSTORM_CORE_CONN_AG_CTX_CF4EN_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_CF4EN_SHIFT 2 +/* cf5en */ +#define E4_XSTORM_CORE_CONN_AG_CTX_CF5EN_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_CF5EN_SHIFT 3 +/* cf6en */ +#define E4_XSTORM_CORE_CONN_AG_CTX_CF6EN_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_CF6EN_SHIFT 4 +/* cf7en */ +#define E4_XSTORM_CORE_CONN_AG_CTX_CF7EN_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_CF7EN_SHIFT 5 +/* cf8en */ +#define E4_XSTORM_CORE_CONN_AG_CTX_CF8EN_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_CF8EN_SHIFT 6 +/* cf9en */ +#define E4_XSTORM_CORE_CONN_AG_CTX_CF9EN_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_CF9EN_SHIFT 7 u8 flags9; -#define XSTORM_CORE_CONN_AG_CTX_CF10EN_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_CF10EN_SHIFT 0 -#define XSTORM_CORE_CONN_AG_CTX_CF11EN_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_CF11EN_SHIFT 1 -#define XSTORM_CORE_CONN_AG_CTX_CF12EN_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_CF12EN_SHIFT 2 -#define XSTORM_CORE_CONN_AG_CTX_CF13EN_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_CF13EN_SHIFT 3 -#define XSTORM_CORE_CONN_AG_CTX_CF14EN_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_CF14EN_SHIFT 4 -#define XSTORM_CORE_CONN_AG_CTX_CF15EN_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_CF15EN_SHIFT 5 -#define XSTORM_CORE_CONN_AG_CTX_CONSOLID_PROD_CF_EN_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_CONSOLID_PROD_CF_EN_SHIFT 6 -#define XSTORM_CORE_CONN_AG_CTX_CF17EN_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_CF17EN_SHIFT 7 +/* cf10en */ +#define E4_XSTORM_CORE_CONN_AG_CTX_CF10EN_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_CF10EN_SHIFT 0 +/* cf11en */ +#define E4_XSTORM_CORE_CONN_AG_CTX_CF11EN_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_CF11EN_SHIFT 1 +/* cf12en */ +#define E4_XSTORM_CORE_CONN_AG_CTX_CF12EN_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_CF12EN_SHIFT 2 +/* cf13en */ +#define E4_XSTORM_CORE_CONN_AG_CTX_CF13EN_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_CF13EN_SHIFT 3 +/* cf14en */ +#define E4_XSTORM_CORE_CONN_AG_CTX_CF14EN_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_CF14EN_SHIFT 4 +/* cf15en */ +#define E4_XSTORM_CORE_CONN_AG_CTX_CF15EN_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_CF15EN_SHIFT 5 +/* cf16en */ +#define E4_XSTORM_CORE_CONN_AG_CTX_CONSOLID_PROD_CF_EN_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_CONSOLID_PROD_CF_EN_SHIFT 6 +/* cf_array_cf_en */ +#define E4_XSTORM_CORE_CONN_AG_CTX_CF17EN_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_CF17EN_SHIFT 7 u8 flags10; -#define XSTORM_CORE_CONN_AG_CTX_DQ_CF_EN_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_DQ_CF_EN_SHIFT 0 -#define XSTORM_CORE_CONN_AG_CTX_TERMINATE_CF_EN_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_TERMINATE_CF_EN_SHIFT 1 -#define XSTORM_CORE_CONN_AG_CTX_FLUSH_Q0_EN_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_FLUSH_Q0_EN_SHIFT 2 -#define XSTORM_CORE_CONN_AG_CTX_RESERVED11_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_RESERVED11_SHIFT 3 -#define XSTORM_CORE_CONN_AG_CTX_SLOW_PATH_EN_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_SLOW_PATH_EN_SHIFT 4 -#define XSTORM_CORE_CONN_AG_CTX_CF23EN_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_CF23EN_SHIFT 5 -#define XSTORM_CORE_CONN_AG_CTX_RESERVED12_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_RESERVED12_SHIFT 6 -#define XSTORM_CORE_CONN_AG_CTX_RESERVED13_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_RESERVED13_SHIFT 7 +/* cf18en */ +#define E4_XSTORM_CORE_CONN_AG_CTX_DQ_CF_EN_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_DQ_CF_EN_SHIFT 0 +/* cf19en */ +#define E4_XSTORM_CORE_CONN_AG_CTX_TERMINATE_CF_EN_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_TERMINATE_CF_EN_SHIFT 1 +/* cf20en */ +#define E4_XSTORM_CORE_CONN_AG_CTX_FLUSH_Q0_EN_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_FLUSH_Q0_EN_SHIFT 2 +/* cf21en */ +#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED11_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED11_SHIFT 3 +/* cf22en */ +#define E4_XSTORM_CORE_CONN_AG_CTX_SLOW_PATH_EN_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_SLOW_PATH_EN_SHIFT 4 +/* cf23en */ +#define E4_XSTORM_CORE_CONN_AG_CTX_CF23EN_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_CF23EN_SHIFT 5 +/* rule0en */ +#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED12_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED12_SHIFT 6 +/* rule1en */ +#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED13_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED13_SHIFT 7 u8 flags11; -#define XSTORM_CORE_CONN_AG_CTX_RESERVED14_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_RESERVED14_SHIFT 0 -#define XSTORM_CORE_CONN_AG_CTX_RESERVED15_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_RESERVED15_SHIFT 1 -#define XSTORM_CORE_CONN_AG_CTX_TX_DEC_RULE_EN_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_TX_DEC_RULE_EN_SHIFT 2 -#define XSTORM_CORE_CONN_AG_CTX_RULE5EN_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_RULE5EN_SHIFT 3 -#define XSTORM_CORE_CONN_AG_CTX_RULE6EN_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_RULE6EN_SHIFT 4 -#define XSTORM_CORE_CONN_AG_CTX_RULE7EN_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_RULE7EN_SHIFT 5 -#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED1_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED1_SHIFT 6 -#define XSTORM_CORE_CONN_AG_CTX_RULE9EN_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_RULE9EN_SHIFT 7 +/* rule2en */ +#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED14_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED14_SHIFT 0 +/* rule3en */ +#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED15_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED15_SHIFT 1 +/* rule4en */ +#define E4_XSTORM_CORE_CONN_AG_CTX_TX_DEC_RULE_EN_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_TX_DEC_RULE_EN_SHIFT 2 +/* rule5en */ +#define E4_XSTORM_CORE_CONN_AG_CTX_RULE5EN_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_RULE5EN_SHIFT 3 +/* rule6en */ +#define E4_XSTORM_CORE_CONN_AG_CTX_RULE6EN_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_RULE6EN_SHIFT 4 +/* rule7en */ +#define E4_XSTORM_CORE_CONN_AG_CTX_RULE7EN_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_RULE7EN_SHIFT 5 +/* rule8en */ +#define E4_XSTORM_CORE_CONN_AG_CTX_A0_RESERVED1_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_A0_RESERVED1_SHIFT 6 +/* rule9en */ +#define E4_XSTORM_CORE_CONN_AG_CTX_RULE9EN_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_RULE9EN_SHIFT 7 u8 flags12; -#define XSTORM_CORE_CONN_AG_CTX_RULE10EN_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_RULE10EN_SHIFT 0 -#define XSTORM_CORE_CONN_AG_CTX_RULE11EN_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_RULE11EN_SHIFT 1 -#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED2_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED2_SHIFT 2 -#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED3_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED3_SHIFT 3 -#define XSTORM_CORE_CONN_AG_CTX_RULE14EN_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_RULE14EN_SHIFT 4 -#define XSTORM_CORE_CONN_AG_CTX_RULE15EN_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_RULE15EN_SHIFT 5 -#define XSTORM_CORE_CONN_AG_CTX_RULE16EN_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_RULE16EN_SHIFT 6 -#define XSTORM_CORE_CONN_AG_CTX_RULE17EN_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_RULE17EN_SHIFT 7 +/* rule10en */ +#define E4_XSTORM_CORE_CONN_AG_CTX_RULE10EN_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_RULE10EN_SHIFT 0 +/* rule11en */ +#define E4_XSTORM_CORE_CONN_AG_CTX_RULE11EN_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_RULE11EN_SHIFT 1 +/* rule12en */ +#define E4_XSTORM_CORE_CONN_AG_CTX_A0_RESERVED2_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_A0_RESERVED2_SHIFT 2 +/* rule13en */ +#define E4_XSTORM_CORE_CONN_AG_CTX_A0_RESERVED3_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_A0_RESERVED3_SHIFT 3 +/* rule14en */ +#define E4_XSTORM_CORE_CONN_AG_CTX_RULE14EN_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_RULE14EN_SHIFT 4 +/* rule15en */ +#define E4_XSTORM_CORE_CONN_AG_CTX_RULE15EN_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_RULE15EN_SHIFT 5 +/* rule16en */ +#define E4_XSTORM_CORE_CONN_AG_CTX_RULE16EN_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_RULE16EN_SHIFT 6 +/* rule17en */ +#define E4_XSTORM_CORE_CONN_AG_CTX_RULE17EN_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_RULE17EN_SHIFT 7 u8 flags13; -#define XSTORM_CORE_CONN_AG_CTX_RULE18EN_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_RULE18EN_SHIFT 0 -#define XSTORM_CORE_CONN_AG_CTX_RULE19EN_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_RULE19EN_SHIFT 1 -#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED4_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED4_SHIFT 2 -#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED5_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED5_SHIFT 3 -#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED6_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED6_SHIFT 4 -#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED7_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED7_SHIFT 5 -#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED8_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED8_SHIFT 6 -#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED9_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED9_SHIFT 7 +/* rule18en */ +#define E4_XSTORM_CORE_CONN_AG_CTX_RULE18EN_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_RULE18EN_SHIFT 0 +/* rule19en */ +#define E4_XSTORM_CORE_CONN_AG_CTX_RULE19EN_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_RULE19EN_SHIFT 1 +/* rule20en */ +#define E4_XSTORM_CORE_CONN_AG_CTX_A0_RESERVED4_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_A0_RESERVED4_SHIFT 2 +/* rule21en */ +#define E4_XSTORM_CORE_CONN_AG_CTX_A0_RESERVED5_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_A0_RESERVED5_SHIFT 3 +/* rule22en */ +#define E4_XSTORM_CORE_CONN_AG_CTX_A0_RESERVED6_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_A0_RESERVED6_SHIFT 4 +/* rule23en */ +#define E4_XSTORM_CORE_CONN_AG_CTX_A0_RESERVED7_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_A0_RESERVED7_SHIFT 5 +/* rule24en */ +#define E4_XSTORM_CORE_CONN_AG_CTX_A0_RESERVED8_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_A0_RESERVED8_SHIFT 6 +/* rule25en */ +#define E4_XSTORM_CORE_CONN_AG_CTX_A0_RESERVED9_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_A0_RESERVED9_SHIFT 7 u8 flags14; -#define XSTORM_CORE_CONN_AG_CTX_BIT16_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_BIT16_SHIFT 0 -#define XSTORM_CORE_CONN_AG_CTX_BIT17_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_BIT17_SHIFT 1 -#define XSTORM_CORE_CONN_AG_CTX_BIT18_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_BIT18_SHIFT 2 -#define XSTORM_CORE_CONN_AG_CTX_BIT19_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_BIT19_SHIFT 3 -#define XSTORM_CORE_CONN_AG_CTX_BIT20_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_BIT20_SHIFT 4 -#define XSTORM_CORE_CONN_AG_CTX_BIT21_MASK 0x1 -#define XSTORM_CORE_CONN_AG_CTX_BIT21_SHIFT 5 -#define XSTORM_CORE_CONN_AG_CTX_CF23_MASK 0x3 -#define XSTORM_CORE_CONN_AG_CTX_CF23_SHIFT 6 +/* bit16 */ +#define E4_XSTORM_CORE_CONN_AG_CTX_BIT16_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_BIT16_SHIFT 0 +/* bit17 */ +#define E4_XSTORM_CORE_CONN_AG_CTX_BIT17_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_BIT17_SHIFT 1 +/* bit18 */ +#define E4_XSTORM_CORE_CONN_AG_CTX_BIT18_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_BIT18_SHIFT 2 +/* bit19 */ +#define E4_XSTORM_CORE_CONN_AG_CTX_BIT19_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_BIT19_SHIFT 3 +/* bit20 */ +#define E4_XSTORM_CORE_CONN_AG_CTX_BIT20_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_BIT20_SHIFT 4 +/* bit21 */ +#define E4_XSTORM_CORE_CONN_AG_CTX_BIT21_MASK 0x1 +#define E4_XSTORM_CORE_CONN_AG_CTX_BIT21_SHIFT 5 +/* cf23 */ +#define E4_XSTORM_CORE_CONN_AG_CTX_CF23_MASK 0x3 +#define E4_XSTORM_CORE_CONN_AG_CTX_CF23_SHIFT 6 u8 byte2 /* byte2 */; __le16 physical_q0 /* physical_q0 */; __le16 consolid_prod /* physical_q1 */; @@ -315,7 +410,7 @@ struct xstorm_core_conn_ag_ctx { u8 byte13 /* byte13 */; u8 byte14 /* byte14 */; u8 byte15 /* byte15 */; - u8 byte16 /* byte16 */; + u8 e5_reserved /* e5_reserved */; __le16 word11 /* word11 */; __le32 reg10 /* reg10 */; __le32 reg11 /* reg11 */; @@ -333,89 +428,89 @@ struct xstorm_core_conn_ag_ctx { __le16 word15 /* word15 */; }; -struct tstorm_core_conn_ag_ctx { +struct e4_tstorm_core_conn_ag_ctx { u8 byte0 /* cdu_validation */; u8 byte1 /* state */; u8 flags0; -#define TSTORM_CORE_CONN_AG_CTX_BIT0_MASK 0x1 -#define TSTORM_CORE_CONN_AG_CTX_BIT0_SHIFT 0 -#define TSTORM_CORE_CONN_AG_CTX_BIT1_MASK 0x1 -#define TSTORM_CORE_CONN_AG_CTX_BIT1_SHIFT 1 -#define TSTORM_CORE_CONN_AG_CTX_BIT2_MASK 0x1 -#define TSTORM_CORE_CONN_AG_CTX_BIT2_SHIFT 2 -#define TSTORM_CORE_CONN_AG_CTX_BIT3_MASK 0x1 -#define TSTORM_CORE_CONN_AG_CTX_BIT3_SHIFT 3 -#define TSTORM_CORE_CONN_AG_CTX_BIT4_MASK 0x1 -#define TSTORM_CORE_CONN_AG_CTX_BIT4_SHIFT 4 -#define TSTORM_CORE_CONN_AG_CTX_BIT5_MASK 0x1 -#define TSTORM_CORE_CONN_AG_CTX_BIT5_SHIFT 5 -#define TSTORM_CORE_CONN_AG_CTX_CF0_MASK 0x3 -#define TSTORM_CORE_CONN_AG_CTX_CF0_SHIFT 6 +#define E4_TSTORM_CORE_CONN_AG_CTX_BIT0_MASK 0x1 /* exist_in_qm0 */ +#define E4_TSTORM_CORE_CONN_AG_CTX_BIT0_SHIFT 0 +#define E4_TSTORM_CORE_CONN_AG_CTX_BIT1_MASK 0x1 /* exist_in_qm1 */ +#define E4_TSTORM_CORE_CONN_AG_CTX_BIT1_SHIFT 1 +#define E4_TSTORM_CORE_CONN_AG_CTX_BIT2_MASK 0x1 /* bit2 */ +#define E4_TSTORM_CORE_CONN_AG_CTX_BIT2_SHIFT 2 +#define E4_TSTORM_CORE_CONN_AG_CTX_BIT3_MASK 0x1 /* bit3 */ +#define E4_TSTORM_CORE_CONN_AG_CTX_BIT3_SHIFT 3 +#define E4_TSTORM_CORE_CONN_AG_CTX_BIT4_MASK 0x1 /* bit4 */ +#define E4_TSTORM_CORE_CONN_AG_CTX_BIT4_SHIFT 4 +#define E4_TSTORM_CORE_CONN_AG_CTX_BIT5_MASK 0x1 /* bit5 */ +#define E4_TSTORM_CORE_CONN_AG_CTX_BIT5_SHIFT 5 +#define E4_TSTORM_CORE_CONN_AG_CTX_CF0_MASK 0x3 /* timer0cf */ +#define E4_TSTORM_CORE_CONN_AG_CTX_CF0_SHIFT 6 u8 flags1; -#define TSTORM_CORE_CONN_AG_CTX_CF1_MASK 0x3 -#define TSTORM_CORE_CONN_AG_CTX_CF1_SHIFT 0 -#define TSTORM_CORE_CONN_AG_CTX_CF2_MASK 0x3 -#define TSTORM_CORE_CONN_AG_CTX_CF2_SHIFT 2 -#define TSTORM_CORE_CONN_AG_CTX_CF3_MASK 0x3 -#define TSTORM_CORE_CONN_AG_CTX_CF3_SHIFT 4 -#define TSTORM_CORE_CONN_AG_CTX_CF4_MASK 0x3 -#define TSTORM_CORE_CONN_AG_CTX_CF4_SHIFT 6 +#define E4_TSTORM_CORE_CONN_AG_CTX_CF1_MASK 0x3 /* timer1cf */ +#define E4_TSTORM_CORE_CONN_AG_CTX_CF1_SHIFT 0 +#define E4_TSTORM_CORE_CONN_AG_CTX_CF2_MASK 0x3 /* timer2cf */ +#define E4_TSTORM_CORE_CONN_AG_CTX_CF2_SHIFT 2 +#define E4_TSTORM_CORE_CONN_AG_CTX_CF3_MASK 0x3 /* timer_stop_all */ +#define E4_TSTORM_CORE_CONN_AG_CTX_CF3_SHIFT 4 +#define E4_TSTORM_CORE_CONN_AG_CTX_CF4_MASK 0x3 /* cf4 */ +#define E4_TSTORM_CORE_CONN_AG_CTX_CF4_SHIFT 6 u8 flags2; -#define TSTORM_CORE_CONN_AG_CTX_CF5_MASK 0x3 -#define TSTORM_CORE_CONN_AG_CTX_CF5_SHIFT 0 -#define TSTORM_CORE_CONN_AG_CTX_CF6_MASK 0x3 -#define TSTORM_CORE_CONN_AG_CTX_CF6_SHIFT 2 -#define TSTORM_CORE_CONN_AG_CTX_CF7_MASK 0x3 -#define TSTORM_CORE_CONN_AG_CTX_CF7_SHIFT 4 -#define TSTORM_CORE_CONN_AG_CTX_CF8_MASK 0x3 -#define TSTORM_CORE_CONN_AG_CTX_CF8_SHIFT 6 +#define E4_TSTORM_CORE_CONN_AG_CTX_CF5_MASK 0x3 /* cf5 */ +#define E4_TSTORM_CORE_CONN_AG_CTX_CF5_SHIFT 0 +#define E4_TSTORM_CORE_CONN_AG_CTX_CF6_MASK 0x3 /* cf6 */ +#define E4_TSTORM_CORE_CONN_AG_CTX_CF6_SHIFT 2 +#define E4_TSTORM_CORE_CONN_AG_CTX_CF7_MASK 0x3 /* cf7 */ +#define E4_TSTORM_CORE_CONN_AG_CTX_CF7_SHIFT 4 +#define E4_TSTORM_CORE_CONN_AG_CTX_CF8_MASK 0x3 /* cf8 */ +#define E4_TSTORM_CORE_CONN_AG_CTX_CF8_SHIFT 6 u8 flags3; -#define TSTORM_CORE_CONN_AG_CTX_CF9_MASK 0x3 -#define TSTORM_CORE_CONN_AG_CTX_CF9_SHIFT 0 -#define TSTORM_CORE_CONN_AG_CTX_CF10_MASK 0x3 -#define TSTORM_CORE_CONN_AG_CTX_CF10_SHIFT 2 -#define TSTORM_CORE_CONN_AG_CTX_CF0EN_MASK 0x1 -#define TSTORM_CORE_CONN_AG_CTX_CF0EN_SHIFT 4 -#define TSTORM_CORE_CONN_AG_CTX_CF1EN_MASK 0x1 -#define TSTORM_CORE_CONN_AG_CTX_CF1EN_SHIFT 5 -#define TSTORM_CORE_CONN_AG_CTX_CF2EN_MASK 0x1 -#define TSTORM_CORE_CONN_AG_CTX_CF2EN_SHIFT 6 -#define TSTORM_CORE_CONN_AG_CTX_CF3EN_MASK 0x1 -#define TSTORM_CORE_CONN_AG_CTX_CF3EN_SHIFT 7 +#define E4_TSTORM_CORE_CONN_AG_CTX_CF9_MASK 0x3 /* cf9 */ +#define E4_TSTORM_CORE_CONN_AG_CTX_CF9_SHIFT 0 +#define E4_TSTORM_CORE_CONN_AG_CTX_CF10_MASK 0x3 /* cf10 */ +#define E4_TSTORM_CORE_CONN_AG_CTX_CF10_SHIFT 2 +#define E4_TSTORM_CORE_CONN_AG_CTX_CF0EN_MASK 0x1 /* cf0en */ +#define E4_TSTORM_CORE_CONN_AG_CTX_CF0EN_SHIFT 4 +#define E4_TSTORM_CORE_CONN_AG_CTX_CF1EN_MASK 0x1 /* cf1en */ +#define E4_TSTORM_CORE_CONN_AG_CTX_CF1EN_SHIFT 5 +#define E4_TSTORM_CORE_CONN_AG_CTX_CF2EN_MASK 0x1 /* cf2en */ +#define E4_TSTORM_CORE_CONN_AG_CTX_CF2EN_SHIFT 6 +#define E4_TSTORM_CORE_CONN_AG_CTX_CF3EN_MASK 0x1 /* cf3en */ +#define E4_TSTORM_CORE_CONN_AG_CTX_CF3EN_SHIFT 7 u8 flags4; -#define TSTORM_CORE_CONN_AG_CTX_CF4EN_MASK 0x1 -#define TSTORM_CORE_CONN_AG_CTX_CF4EN_SHIFT 0 -#define TSTORM_CORE_CONN_AG_CTX_CF5EN_MASK 0x1 -#define TSTORM_CORE_CONN_AG_CTX_CF5EN_SHIFT 1 -#define TSTORM_CORE_CONN_AG_CTX_CF6EN_MASK 0x1 -#define TSTORM_CORE_CONN_AG_CTX_CF6EN_SHIFT 2 -#define TSTORM_CORE_CONN_AG_CTX_CF7EN_MASK 0x1 -#define TSTORM_CORE_CONN_AG_CTX_CF7EN_SHIFT 3 -#define TSTORM_CORE_CONN_AG_CTX_CF8EN_MASK 0x1 -#define TSTORM_CORE_CONN_AG_CTX_CF8EN_SHIFT 4 -#define TSTORM_CORE_CONN_AG_CTX_CF9EN_MASK 0x1 -#define TSTORM_CORE_CONN_AG_CTX_CF9EN_SHIFT 5 -#define TSTORM_CORE_CONN_AG_CTX_CF10EN_MASK 0x1 -#define TSTORM_CORE_CONN_AG_CTX_CF10EN_SHIFT 6 -#define TSTORM_CORE_CONN_AG_CTX_RULE0EN_MASK 0x1 -#define TSTORM_CORE_CONN_AG_CTX_RULE0EN_SHIFT 7 +#define E4_TSTORM_CORE_CONN_AG_CTX_CF4EN_MASK 0x1 /* cf4en */ +#define E4_TSTORM_CORE_CONN_AG_CTX_CF4EN_SHIFT 0 +#define E4_TSTORM_CORE_CONN_AG_CTX_CF5EN_MASK 0x1 /* cf5en */ +#define E4_TSTORM_CORE_CONN_AG_CTX_CF5EN_SHIFT 1 +#define E4_TSTORM_CORE_CONN_AG_CTX_CF6EN_MASK 0x1 /* cf6en */ +#define E4_TSTORM_CORE_CONN_AG_CTX_CF6EN_SHIFT 2 +#define E4_TSTORM_CORE_CONN_AG_CTX_CF7EN_MASK 0x1 /* cf7en */ +#define E4_TSTORM_CORE_CONN_AG_CTX_CF7EN_SHIFT 3 +#define E4_TSTORM_CORE_CONN_AG_CTX_CF8EN_MASK 0x1 /* cf8en */ +#define E4_TSTORM_CORE_CONN_AG_CTX_CF8EN_SHIFT 4 +#define E4_TSTORM_CORE_CONN_AG_CTX_CF9EN_MASK 0x1 /* cf9en */ +#define E4_TSTORM_CORE_CONN_AG_CTX_CF9EN_SHIFT 5 +#define E4_TSTORM_CORE_CONN_AG_CTX_CF10EN_MASK 0x1 /* cf10en */ +#define E4_TSTORM_CORE_CONN_AG_CTX_CF10EN_SHIFT 6 +#define E4_TSTORM_CORE_CONN_AG_CTX_RULE0EN_MASK 0x1 /* rule0en */ +#define E4_TSTORM_CORE_CONN_AG_CTX_RULE0EN_SHIFT 7 u8 flags5; -#define TSTORM_CORE_CONN_AG_CTX_RULE1EN_MASK 0x1 -#define TSTORM_CORE_CONN_AG_CTX_RULE1EN_SHIFT 0 -#define TSTORM_CORE_CONN_AG_CTX_RULE2EN_MASK 0x1 -#define TSTORM_CORE_CONN_AG_CTX_RULE2EN_SHIFT 1 -#define TSTORM_CORE_CONN_AG_CTX_RULE3EN_MASK 0x1 -#define TSTORM_CORE_CONN_AG_CTX_RULE3EN_SHIFT 2 -#define TSTORM_CORE_CONN_AG_CTX_RULE4EN_MASK 0x1 -#define TSTORM_CORE_CONN_AG_CTX_RULE4EN_SHIFT 3 -#define TSTORM_CORE_CONN_AG_CTX_RULE5EN_MASK 0x1 -#define TSTORM_CORE_CONN_AG_CTX_RULE5EN_SHIFT 4 -#define TSTORM_CORE_CONN_AG_CTX_RULE6EN_MASK 0x1 -#define TSTORM_CORE_CONN_AG_CTX_RULE6EN_SHIFT 5 -#define TSTORM_CORE_CONN_AG_CTX_RULE7EN_MASK 0x1 -#define TSTORM_CORE_CONN_AG_CTX_RULE7EN_SHIFT 6 -#define TSTORM_CORE_CONN_AG_CTX_RULE8EN_MASK 0x1 -#define TSTORM_CORE_CONN_AG_CTX_RULE8EN_SHIFT 7 +#define E4_TSTORM_CORE_CONN_AG_CTX_RULE1EN_MASK 0x1 /* rule1en */ +#define E4_TSTORM_CORE_CONN_AG_CTX_RULE1EN_SHIFT 0 +#define E4_TSTORM_CORE_CONN_AG_CTX_RULE2EN_MASK 0x1 /* rule2en */ +#define E4_TSTORM_CORE_CONN_AG_CTX_RULE2EN_SHIFT 1 +#define E4_TSTORM_CORE_CONN_AG_CTX_RULE3EN_MASK 0x1 /* rule3en */ +#define E4_TSTORM_CORE_CONN_AG_CTX_RULE3EN_SHIFT 2 +#define E4_TSTORM_CORE_CONN_AG_CTX_RULE4EN_MASK 0x1 /* rule4en */ +#define E4_TSTORM_CORE_CONN_AG_CTX_RULE4EN_SHIFT 3 +#define E4_TSTORM_CORE_CONN_AG_CTX_RULE5EN_MASK 0x1 /* rule5en */ +#define E4_TSTORM_CORE_CONN_AG_CTX_RULE5EN_SHIFT 4 +#define E4_TSTORM_CORE_CONN_AG_CTX_RULE6EN_MASK 0x1 /* rule6en */ +#define E4_TSTORM_CORE_CONN_AG_CTX_RULE6EN_SHIFT 5 +#define E4_TSTORM_CORE_CONN_AG_CTX_RULE7EN_MASK 0x1 /* rule7en */ +#define E4_TSTORM_CORE_CONN_AG_CTX_RULE7EN_SHIFT 6 +#define E4_TSTORM_CORE_CONN_AG_CTX_RULE8EN_MASK 0x1 /* rule8en */ +#define E4_TSTORM_CORE_CONN_AG_CTX_RULE8EN_SHIFT 7 __le32 reg0 /* reg0 */; __le32 reg1 /* reg1 */; __le32 reg2 /* reg2 */; @@ -437,63 +532,63 @@ struct tstorm_core_conn_ag_ctx { __le32 reg10 /* reg10 */; }; -struct ustorm_core_conn_ag_ctx { +struct e4_ustorm_core_conn_ag_ctx { u8 reserved /* cdu_validation */; u8 byte1 /* state */; u8 flags0; -#define USTORM_CORE_CONN_AG_CTX_BIT0_MASK 0x1 -#define USTORM_CORE_CONN_AG_CTX_BIT0_SHIFT 0 -#define USTORM_CORE_CONN_AG_CTX_BIT1_MASK 0x1 -#define USTORM_CORE_CONN_AG_CTX_BIT1_SHIFT 1 -#define USTORM_CORE_CONN_AG_CTX_CF0_MASK 0x3 -#define USTORM_CORE_CONN_AG_CTX_CF0_SHIFT 2 -#define USTORM_CORE_CONN_AG_CTX_CF1_MASK 0x3 -#define USTORM_CORE_CONN_AG_CTX_CF1_SHIFT 4 -#define USTORM_CORE_CONN_AG_CTX_CF2_MASK 0x3 -#define USTORM_CORE_CONN_AG_CTX_CF2_SHIFT 6 +#define E4_USTORM_CORE_CONN_AG_CTX_BIT0_MASK 0x1 /* exist_in_qm0 */ +#define E4_USTORM_CORE_CONN_AG_CTX_BIT0_SHIFT 0 +#define E4_USTORM_CORE_CONN_AG_CTX_BIT1_MASK 0x1 /* exist_in_qm1 */ +#define E4_USTORM_CORE_CONN_AG_CTX_BIT1_SHIFT 1 +#define E4_USTORM_CORE_CONN_AG_CTX_CF0_MASK 0x3 /* timer0cf */ +#define E4_USTORM_CORE_CONN_AG_CTX_CF0_SHIFT 2 +#define E4_USTORM_CORE_CONN_AG_CTX_CF1_MASK 0x3 /* timer1cf */ +#define E4_USTORM_CORE_CONN_AG_CTX_CF1_SHIFT 4 +#define E4_USTORM_CORE_CONN_AG_CTX_CF2_MASK 0x3 /* timer2cf */ +#define E4_USTORM_CORE_CONN_AG_CTX_CF2_SHIFT 6 u8 flags1; -#define USTORM_CORE_CONN_AG_CTX_CF3_MASK 0x3 -#define USTORM_CORE_CONN_AG_CTX_CF3_SHIFT 0 -#define USTORM_CORE_CONN_AG_CTX_CF4_MASK 0x3 -#define USTORM_CORE_CONN_AG_CTX_CF4_SHIFT 2 -#define USTORM_CORE_CONN_AG_CTX_CF5_MASK 0x3 -#define USTORM_CORE_CONN_AG_CTX_CF5_SHIFT 4 -#define USTORM_CORE_CONN_AG_CTX_CF6_MASK 0x3 -#define USTORM_CORE_CONN_AG_CTX_CF6_SHIFT 6 +#define E4_USTORM_CORE_CONN_AG_CTX_CF3_MASK 0x3 /* timer_stop_all */ +#define E4_USTORM_CORE_CONN_AG_CTX_CF3_SHIFT 0 +#define E4_USTORM_CORE_CONN_AG_CTX_CF4_MASK 0x3 /* cf4 */ +#define E4_USTORM_CORE_CONN_AG_CTX_CF4_SHIFT 2 +#define E4_USTORM_CORE_CONN_AG_CTX_CF5_MASK 0x3 /* cf5 */ +#define E4_USTORM_CORE_CONN_AG_CTX_CF5_SHIFT 4 +#define E4_USTORM_CORE_CONN_AG_CTX_CF6_MASK 0x3 /* cf6 */ +#define E4_USTORM_CORE_CONN_AG_CTX_CF6_SHIFT 6 u8 flags2; -#define USTORM_CORE_CONN_AG_CTX_CF0EN_MASK 0x1 -#define USTORM_CORE_CONN_AG_CTX_CF0EN_SHIFT 0 -#define USTORM_CORE_CONN_AG_CTX_CF1EN_MASK 0x1 -#define USTORM_CORE_CONN_AG_CTX_CF1EN_SHIFT 1 -#define USTORM_CORE_CONN_AG_CTX_CF2EN_MASK 0x1 -#define USTORM_CORE_CONN_AG_CTX_CF2EN_SHIFT 2 -#define USTORM_CORE_CONN_AG_CTX_CF3EN_MASK 0x1 -#define USTORM_CORE_CONN_AG_CTX_CF3EN_SHIFT 3 -#define USTORM_CORE_CONN_AG_CTX_CF4EN_MASK 0x1 -#define USTORM_CORE_CONN_AG_CTX_CF4EN_SHIFT 4 -#define USTORM_CORE_CONN_AG_CTX_CF5EN_MASK 0x1 -#define USTORM_CORE_CONN_AG_CTX_CF5EN_SHIFT 5 -#define USTORM_CORE_CONN_AG_CTX_CF6EN_MASK 0x1 -#define USTORM_CORE_CONN_AG_CTX_CF6EN_SHIFT 6 -#define USTORM_CORE_CONN_AG_CTX_RULE0EN_MASK 0x1 -#define USTORM_CORE_CONN_AG_CTX_RULE0EN_SHIFT 7 +#define E4_USTORM_CORE_CONN_AG_CTX_CF0EN_MASK 0x1 /* cf0en */ +#define E4_USTORM_CORE_CONN_AG_CTX_CF0EN_SHIFT 0 +#define E4_USTORM_CORE_CONN_AG_CTX_CF1EN_MASK 0x1 /* cf1en */ +#define E4_USTORM_CORE_CONN_AG_CTX_CF1EN_SHIFT 1 +#define E4_USTORM_CORE_CONN_AG_CTX_CF2EN_MASK 0x1 /* cf2en */ +#define E4_USTORM_CORE_CONN_AG_CTX_CF2EN_SHIFT 2 +#define E4_USTORM_CORE_CONN_AG_CTX_CF3EN_MASK 0x1 /* cf3en */ +#define E4_USTORM_CORE_CONN_AG_CTX_CF3EN_SHIFT 3 +#define E4_USTORM_CORE_CONN_AG_CTX_CF4EN_MASK 0x1 /* cf4en */ +#define E4_USTORM_CORE_CONN_AG_CTX_CF4EN_SHIFT 4 +#define E4_USTORM_CORE_CONN_AG_CTX_CF5EN_MASK 0x1 /* cf5en */ +#define E4_USTORM_CORE_CONN_AG_CTX_CF5EN_SHIFT 5 +#define E4_USTORM_CORE_CONN_AG_CTX_CF6EN_MASK 0x1 /* cf6en */ +#define E4_USTORM_CORE_CONN_AG_CTX_CF6EN_SHIFT 6 +#define E4_USTORM_CORE_CONN_AG_CTX_RULE0EN_MASK 0x1 /* rule0en */ +#define E4_USTORM_CORE_CONN_AG_CTX_RULE0EN_SHIFT 7 u8 flags3; -#define USTORM_CORE_CONN_AG_CTX_RULE1EN_MASK 0x1 -#define USTORM_CORE_CONN_AG_CTX_RULE1EN_SHIFT 0 -#define USTORM_CORE_CONN_AG_CTX_RULE2EN_MASK 0x1 -#define USTORM_CORE_CONN_AG_CTX_RULE2EN_SHIFT 1 -#define USTORM_CORE_CONN_AG_CTX_RULE3EN_MASK 0x1 -#define USTORM_CORE_CONN_AG_CTX_RULE3EN_SHIFT 2 -#define USTORM_CORE_CONN_AG_CTX_RULE4EN_MASK 0x1 -#define USTORM_CORE_CONN_AG_CTX_RULE4EN_SHIFT 3 -#define USTORM_CORE_CONN_AG_CTX_RULE5EN_MASK 0x1 -#define USTORM_CORE_CONN_AG_CTX_RULE5EN_SHIFT 4 -#define USTORM_CORE_CONN_AG_CTX_RULE6EN_MASK 0x1 -#define USTORM_CORE_CONN_AG_CTX_RULE6EN_SHIFT 5 -#define USTORM_CORE_CONN_AG_CTX_RULE7EN_MASK 0x1 -#define USTORM_CORE_CONN_AG_CTX_RULE7EN_SHIFT 6 -#define USTORM_CORE_CONN_AG_CTX_RULE8EN_MASK 0x1 -#define USTORM_CORE_CONN_AG_CTX_RULE8EN_SHIFT 7 +#define E4_USTORM_CORE_CONN_AG_CTX_RULE1EN_MASK 0x1 /* rule1en */ +#define E4_USTORM_CORE_CONN_AG_CTX_RULE1EN_SHIFT 0 +#define E4_USTORM_CORE_CONN_AG_CTX_RULE2EN_MASK 0x1 /* rule2en */ +#define E4_USTORM_CORE_CONN_AG_CTX_RULE2EN_SHIFT 1 +#define E4_USTORM_CORE_CONN_AG_CTX_RULE3EN_MASK 0x1 /* rule3en */ +#define E4_USTORM_CORE_CONN_AG_CTX_RULE3EN_SHIFT 2 +#define E4_USTORM_CORE_CONN_AG_CTX_RULE4EN_MASK 0x1 /* rule4en */ +#define E4_USTORM_CORE_CONN_AG_CTX_RULE4EN_SHIFT 3 +#define E4_USTORM_CORE_CONN_AG_CTX_RULE5EN_MASK 0x1 /* rule5en */ +#define E4_USTORM_CORE_CONN_AG_CTX_RULE5EN_SHIFT 4 +#define E4_USTORM_CORE_CONN_AG_CTX_RULE6EN_MASK 0x1 /* rule6en */ +#define E4_USTORM_CORE_CONN_AG_CTX_RULE6EN_SHIFT 5 +#define E4_USTORM_CORE_CONN_AG_CTX_RULE7EN_MASK 0x1 /* rule7en */ +#define E4_USTORM_CORE_CONN_AG_CTX_RULE7EN_SHIFT 6 +#define E4_USTORM_CORE_CONN_AG_CTX_RULE8EN_MASK 0x1 /* rule8en */ +#define E4_USTORM_CORE_CONN_AG_CTX_RULE8EN_SHIFT 7 u8 byte2 /* byte2 */; u8 byte3 /* byte3 */; __le16 word0 /* conn_dpi */; @@ -523,28 +618,29 @@ struct ustorm_core_conn_st_ctx { /* * core connection context */ -struct core_conn_context { - struct ystorm_core_conn_st_ctx ystorm_st_context - /* ystorm storm context */; +struct e4_core_conn_context { +/* ystorm storm context */ + struct ystorm_core_conn_st_ctx ystorm_st_context; struct regpair ystorm_st_padding[2] /* padding */; - struct pstorm_core_conn_st_ctx pstorm_st_context - /* pstorm storm context */; +/* pstorm storm context */ + struct pstorm_core_conn_st_ctx pstorm_st_context; struct regpair pstorm_st_padding[2] /* padding */; - struct xstorm_core_conn_st_ctx xstorm_st_context - /* xstorm storm context */; - struct xstorm_core_conn_ag_ctx xstorm_ag_context - /* xstorm aggregative context */; - struct tstorm_core_conn_ag_ctx tstorm_ag_context - /* tstorm aggregative context */; - struct ustorm_core_conn_ag_ctx ustorm_ag_context - /* ustorm aggregative context */; - struct mstorm_core_conn_st_ctx mstorm_st_context - /* mstorm storm context */; - struct ustorm_core_conn_st_ctx ustorm_st_context - /* ustorm storm context */; +/* xstorm storm context */ + struct xstorm_core_conn_st_ctx xstorm_st_context; +/* xstorm aggregative context */ + struct e4_xstorm_core_conn_ag_ctx xstorm_ag_context; +/* tstorm aggregative context */ + struct e4_tstorm_core_conn_ag_ctx tstorm_ag_context; +/* ustorm aggregative context */ + struct e4_ustorm_core_conn_ag_ctx ustorm_ag_context; +/* mstorm storm context */ + struct mstorm_core_conn_st_ctx mstorm_st_context; +/* ustorm storm context */ + struct ustorm_core_conn_st_ctx ustorm_st_context; struct regpair ustorm_st_padding[2] /* padding */; }; + /* * How ll2 should deal with packet upon errors */ @@ -555,6 +651,7 @@ enum core_error_handle { MAX_CORE_ERROR_HANDLE }; + /* * opcodes for the event ring */ @@ -563,20 +660,24 @@ enum core_event_opcode { CORE_EVENT_TX_QUEUE_STOP, CORE_EVENT_RX_QUEUE_START, CORE_EVENT_RX_QUEUE_STOP, + CORE_EVENT_RX_QUEUE_FLUSH, + CORE_EVENT_TX_QUEUE_UPDATE, MAX_CORE_EVENT_OPCODE }; + /* * The L4 pseudo checksum mode for Core */ enum core_l4_pseudo_checksum_mode { - CORE_L4_PSEUDO_CSUM_CORRECT_LENGTH - , - CORE_L4_PSEUDO_CSUM_ZERO_LENGTH - /* Pseudo Checksum on packet is calculated with zero length. */, +/* Pseudo Checksum on packet is calculated with the correct packet length. */ + CORE_L4_PSEUDO_CSUM_CORRECT_LENGTH, +/* Pseudo Checksum on packet is calculated with zero length. */ + CORE_L4_PSEUDO_CSUM_ZERO_LENGTH, MAX_CORE_L4_PSEUDO_CHECKSUM_MODE }; + /* * Light-L2 RX Producers in Tstorm RAM */ @@ -587,24 +688,26 @@ struct core_ll2_port_stats { struct regpair gsi_crcchksm_error; }; + /* * Ethernet TX Per Queue Stats */ struct core_ll2_pstorm_per_queue_stat { - struct regpair sent_ucast_bytes - /* number of total bytes sent without errors */; - struct regpair sent_mcast_bytes - /* number of total bytes sent without errors */; - struct regpair sent_bcast_bytes - /* number of total bytes sent without errors */; - struct regpair sent_ucast_pkts - /* number of total packets sent without errors */; - struct regpair sent_mcast_pkts - /* number of total packets sent without errors */; - struct regpair sent_bcast_pkts - /* number of total packets sent without errors */; +/* number of total bytes sent without errors */ + struct regpair sent_ucast_bytes; +/* number of total bytes sent without errors */ + struct regpair sent_mcast_bytes; +/* number of total bytes sent without errors */ + struct regpair sent_bcast_bytes; +/* number of total packets sent without errors */ + struct regpair sent_ucast_pkts; +/* number of total packets sent without errors */ + struct regpair sent_mcast_pkts; +/* number of total packets sent without errors */ + struct regpair sent_bcast_pkts; }; + /* * Light-L2 RX Producers in Tstorm RAM */ @@ -614,13 +717,15 @@ struct core_ll2_rx_prod { __le32 reserved; }; + struct core_ll2_tstorm_per_queue_stat { - struct regpair packet_too_big_discard - /* Number of packets discarded because they are bigger than MTU */; - struct regpair no_buff_discard - /* Number of packets discarded due to lack of host buffers */; +/* Number of packets discarded because they are bigger than MTU */ + struct regpair packet_too_big_discard; +/* Number of packets discarded due to lack of host buffers */ + struct regpair no_buff_discard; }; + struct core_ll2_ustorm_per_queue_stat { struct regpair rcv_ucast_bytes; struct regpair rcv_mcast_bytes; @@ -630,6 +735,7 @@ struct core_ll2_ustorm_per_queue_stat { struct regpair rcv_bcast_pkts; }; + /* * Core Ramrod Command IDs (light L2) */ @@ -639,22 +745,38 @@ enum core_ramrod_cmd_id { CORE_RAMROD_TX_QUEUE_START /* TX Queue Start Ramrod */, CORE_RAMROD_RX_QUEUE_STOP /* RX Queue Stop Ramrod */, CORE_RAMROD_TX_QUEUE_STOP /* TX Queue Stop Ramrod */, + CORE_RAMROD_RX_QUEUE_FLUSH /* RX Flush queue Ramrod */, + CORE_RAMROD_TX_QUEUE_UPDATE /* TX Queue Update Ramrod */, MAX_CORE_RAMROD_CMD_ID }; + +/* + * Core RX CQE Type for Light L2 + */ +enum core_roce_flavor_type { + CORE_ROCE, + CORE_RROCE, + MAX_CORE_ROCE_FLAVOR_TYPE +}; + + /* * Specifies how ll2 should deal with packets errors: packet_too_big and no_buff */ struct core_rx_action_on_error { u8 error_type; +/* ll2 how to handle error packet_too_big (use enum core_error_handle) */ #define CORE_RX_ACTION_ON_ERROR_PACKET_TOO_BIG_MASK 0x3 #define CORE_RX_ACTION_ON_ERROR_PACKET_TOO_BIG_SHIFT 0 +/* ll2 how to handle error with no_buff (use enum core_error_handle) */ #define CORE_RX_ACTION_ON_ERROR_NO_BUFF_MASK 0x3 #define CORE_RX_ACTION_ON_ERROR_NO_BUFF_SHIFT 2 #define CORE_RX_ACTION_ON_ERROR_RESERVED_MASK 0xF #define CORE_RX_ACTION_ON_ERROR_RESERVED_SHIFT 4 }; + /* * Core RX BD for Light L2 */ @@ -663,6 +785,7 @@ struct core_rx_bd { __le16 reserved[4]; }; + /* * Core RX CM offload BD for Light L2 */ @@ -677,10 +800,12 @@ struct core_rx_bd_with_buff_len { */ union core_rx_bd_union { struct core_rx_bd rx_bd /* Core Rx Bd static buffer size */; - struct core_rx_bd_with_buff_len rx_bd_with_len - /* Core Rx Bd with dynamic buffer length */; +/* Core Rx Bd with dynamic buffer length */ + struct core_rx_bd_with_buff_len rx_bd_with_len; }; + + /* * Opaque Data for Light L2 RX CQE . */ @@ -688,6 +813,7 @@ struct core_rx_cqe_opaque_data { __le32 data[2] /* Opaque CQE Data */; }; + /* * Core RX CQE Type for Light L2 */ @@ -699,19 +825,25 @@ enum core_rx_cqe_type { MAX_CORE_RX_CQE_TYPE }; + /* * Core RX CQE for Light L2 . */ struct core_rx_fast_path_cqe { u8 type /* CQE type */; - u8 placement_offset - /* Offset (in bytes) of the packet from start of the buffer */; - struct parsing_and_err_flags parse_flags - /* Parsing and error flags from the parser */; +/* Offset (in bytes) of the packet from start of the buffer */ + u8 placement_offset; +/* Parsing and error flags from the parser */ + struct parsing_and_err_flags parse_flags; __le16 packet_length /* Total packet length (from the parser) */; __le16 vlan /* 802.1q VLAN tag */; struct core_rx_cqe_opaque_data opaque_data /* Opaque Data */; - __le32 reserved[4]; +/* bit- map: each bit represents a specific error. errors indications are + * provided by the cracker. see spec for detailed description + */ + struct parsing_err_flags err_flags; + __le16 reserved0; + __le32 reserved1[3]; }; /* @@ -720,14 +852,16 @@ struct core_rx_fast_path_cqe { struct core_rx_gsi_offload_cqe { u8 type /* CQE type */; u8 data_length_error /* set if gsi data is bigger than buff */; - struct parsing_and_err_flags parse_flags - /* Parsing and error flags from the parser */; +/* Parsing and error flags from the parser */ + struct parsing_and_err_flags parse_flags; __le16 data_length /* Total packet length (from the parser) */; __le16 vlan /* 802.1q VLAN tag */; __le32 src_mac_addrhi /* hi 4 bytes source mac address */; __le16 src_mac_addrlo /* lo 2 bytes of source mac address */; - u8 reserved1[2]; - __le32 gid_dst[4] /* Gid destination address */; +/* These are the lower 16 bit of QP id in RoCE BTH header */ + __le16 qp_id; + __le32 src_qp /* Source QP from DETH header */; + __le32 reserved[3]; }; /* @@ -737,7 +871,8 @@ struct core_rx_slow_path_cqe { u8 type /* CQE type */; u8 ramrod_cmd_id; __le16 echo; - __le32 reserved1[7]; + struct core_rx_cqe_opaque_data opaque_data /* Opaque Data */; + __le32 reserved1[5]; }; /* @@ -749,6 +884,10 @@ union core_rx_cqe_union { struct core_rx_slow_path_cqe rx_cqe_sp /* Slow path CQE */; }; + + + + /* * Ramrod data for rx queue start ramrod */ @@ -762,18 +901,31 @@ struct core_rx_start_ramrod_data { u8 complete_event_flg /* post completion to the event ring if set */; u8 drop_ttl0_flg /* drop packet with ttl0 if set */; __le16 num_of_pbl_pages /* Num of pages in CQE PBL */; - u8 inner_vlan_removal_en - /* if set, 802.1q tags will be removed and copied to CQE */; +/* if set, 802.1q tags will be removed and copied to CQE */ +/* if set, 802.1q tags will be removed and copied to CQE */ + u8 inner_vlan_stripping_en; +/* if set, outer tag wont be stripped, valid only in MF OVLAN. */ + u8 outer_vlan_stripping_dis; u8 queue_id /* Light L2 RX Queue ID */; u8 main_func_queue /* Is this the main queue for the PF */; +/* Duplicate broadcast packets to LL2 main queue in mf_si mode. Valid if + * main_func_queue is set. + */ u8 mf_si_bcast_accept_all; +/* Duplicate multicast packets to LL2 main queue in mf_si mode. Valid if + * main_func_queue is set. + */ u8 mf_si_mcast_accept_all; +/* Specifies how ll2 should deal with packets errors: packet_too_big and + * no_buff + */ struct core_rx_action_on_error action_on_error; - u8 gsi_offload_flag - /* set when in GSI offload mode on ROCE connection */; - u8 reserved[7]; +/* set when in GSI offload mode on ROCE connection */ + u8 gsi_offload_flag; + u8 reserved[6]; }; + /* * Ramrod data for rx queue stop ramrod */ @@ -785,27 +937,58 @@ struct core_rx_stop_ramrod_data { __le16 reserved2[2]; }; + /* * Flags for Core TX BD */ -struct core_tx_bd_flags { - u8 as_bitfield; -#define CORE_TX_BD_FLAGS_FORCE_VLAN_MODE_MASK 0x1 -#define CORE_TX_BD_FLAGS_FORCE_VLAN_MODE_SHIFT 0 -#define CORE_TX_BD_FLAGS_VLAN_INSERTION_MASK 0x1 -#define CORE_TX_BD_FLAGS_VLAN_INSERTION_SHIFT 1 -#define CORE_TX_BD_FLAGS_START_BD_MASK 0x1 -#define CORE_TX_BD_FLAGS_START_BD_SHIFT 2 -#define CORE_TX_BD_FLAGS_IP_CSUM_MASK 0x1 -#define CORE_TX_BD_FLAGS_IP_CSUM_SHIFT 3 -#define CORE_TX_BD_FLAGS_L4_CSUM_MASK 0x1 -#define CORE_TX_BD_FLAGS_L4_CSUM_SHIFT 4 -#define CORE_TX_BD_FLAGS_IPV6_EXT_MASK 0x1 -#define CORE_TX_BD_FLAGS_IPV6_EXT_SHIFT 5 -#define CORE_TX_BD_FLAGS_L4_PROTOCOL_MASK 0x1 -#define CORE_TX_BD_FLAGS_L4_PROTOCOL_SHIFT 6 -#define CORE_TX_BD_FLAGS_L4_PSEUDO_CSUM_MODE_MASK 0x1 -#define CORE_TX_BD_FLAGS_L4_PSEUDO_CSUM_MODE_SHIFT 7 +struct core_tx_bd_data { + __le16 as_bitfield; +/* Do not allow additional VLAN manipulations on this packet (DCB) */ +#define CORE_TX_BD_DATA_FORCE_VLAN_MODE_MASK 0x1 +#define CORE_TX_BD_DATA_FORCE_VLAN_MODE_SHIFT 0 +/* Insert VLAN into packet */ +#define CORE_TX_BD_DATA_VLAN_INSERTION_MASK 0x1 +#define CORE_TX_BD_DATA_VLAN_INSERTION_SHIFT 1 +/* This is the first BD of the packet (for debug) */ +#define CORE_TX_BD_DATA_START_BD_MASK 0x1 +#define CORE_TX_BD_DATA_START_BD_SHIFT 2 +/* Calculate the IP checksum for the packet */ +#define CORE_TX_BD_DATA_IP_CSUM_MASK 0x1 +#define CORE_TX_BD_DATA_IP_CSUM_SHIFT 3 +/* Calculate the L4 checksum for the packet */ +#define CORE_TX_BD_DATA_L4_CSUM_MASK 0x1 +#define CORE_TX_BD_DATA_L4_CSUM_SHIFT 4 +/* Packet is IPv6 with extensions */ +#define CORE_TX_BD_DATA_IPV6_EXT_MASK 0x1 +#define CORE_TX_BD_DATA_IPV6_EXT_SHIFT 5 +/* If IPv6+ext, and if l4_csum is 1, than this field indicates L4 protocol: + * 0-TCP, 1-UDP + */ +#define CORE_TX_BD_DATA_L4_PROTOCOL_MASK 0x1 +#define CORE_TX_BD_DATA_L4_PROTOCOL_SHIFT 6 +/* The pseudo checksum mode to place in the L4 checksum field. Required only + * when IPv6+ext and l4_csum is set. (use enum core_l4_pseudo_checksum_mode) + */ +#define CORE_TX_BD_DATA_L4_PSEUDO_CSUM_MODE_MASK 0x1 +#define CORE_TX_BD_DATA_L4_PSEUDO_CSUM_MODE_SHIFT 7 +/* Number of BDs that make up one packet - width wide enough to present + * CORE_LL2_TX_MAX_BDS_PER_PACKET + */ +#define CORE_TX_BD_DATA_NBDS_MASK 0xF +#define CORE_TX_BD_DATA_NBDS_SHIFT 8 +/* Use roce_flavor enum - Differentiate between Roce flavors is valid when + * connType is ROCE (use enum core_roce_flavor_type) + */ +#define CORE_TX_BD_DATA_ROCE_FLAV_MASK 0x1 +#define CORE_TX_BD_DATA_ROCE_FLAV_SHIFT 12 +/* Calculate ip length */ +#define CORE_TX_BD_DATA_IP_LEN_MASK 0x1 +#define CORE_TX_BD_DATA_IP_LEN_SHIFT 13 +/* disables the STAG insertion, relevant only in MF OVLAN mode. */ +#define CORE_TX_BD_DATA_DISABLE_STAG_INSERTION_MASK 0x1 +#define CORE_TX_BD_DATA_DISABLE_STAG_INSERTION_SHIFT 14 +#define CORE_TX_BD_DATA_RESERVED0_MASK 0x1 +#define CORE_TX_BD_DATA_RESERVED0_SHIFT 15 }; /* @@ -814,40 +997,55 @@ struct core_tx_bd_flags { struct core_tx_bd { struct regpair addr /* Buffer Address */; __le16 nbytes /* Number of Bytes in Buffer */; - __le16 vlan /* VLAN to insert to packet (if insertion flag set) */; - u8 nbds /* Number of BDs that make up one packet */; - struct core_tx_bd_flags bd_flags /* BD Flags */; - __le16 l4_hdr_offset_w; +/* Network packets: VLAN to insert to packet (if insertion flag set) LoopBack + * packets: echo data to pass to Rx + */ + __le16 nw_vlan_or_lb_echo; + struct core_tx_bd_data bd_data /* BD Flags */; + __le16 bitfield1; +/* L4 Header Offset from start of packet (in Words). This is needed if both + * l4_csum and ipv6_ext are set + */ +#define CORE_TX_BD_L4_HDR_OFFSET_W_MASK 0x3FFF +#define CORE_TX_BD_L4_HDR_OFFSET_W_SHIFT 0 +/* Packet destination - Network, Loopback or Drop (use enum core_tx_dest) */ +#define CORE_TX_BD_TX_DST_MASK 0x3 +#define CORE_TX_BD_TX_DST_SHIFT 14 }; + + /* * Light L2 TX Destination */ enum core_tx_dest { - CORE_TX_DEST_NW /* Light L2 TX Destination to the Network */, - CORE_TX_DEST_LB /* Light L2 TX Destination to the Loopback */, + CORE_TX_DEST_NW /* TX Destination to the Network */, + CORE_TX_DEST_LB /* TX Destination to the Loopback */, + CORE_TX_DEST_RESERVED, + CORE_TX_DEST_DROP /* TX Drop */, MAX_CORE_TX_DEST }; + /* - * Ramrod data for rx queue start ramrod + * Ramrod data for tx queue start ramrod */ struct core_tx_start_ramrod_data { struct regpair pbl_base_addr /* Address of the pbl page */; __le16 mtu /* Maximum transmission unit */; __le16 sb_id /* Status block ID */; u8 sb_index /* Status block protocol index */; - u8 tx_dest /* TX Destination (either Network or LB) */; u8 stats_en /* Statistics Enable */; u8 stats_id /* Statistics Counter ID */; + u8 conn_type /* connection type that loaded ll2 */; __le16 pbl_size /* Number of BD pages pointed by PBL */; __le16 qm_pq_id /* QM PQ ID */; - u8 conn_type /* connection type that loaded ll2 */; - u8 gsi_offload_flag - /* set when in GSI offload mode on ROCE connection */; - u8 resrved[2]; +/* set when in GSI offload mode on ROCE connection */ + u8 gsi_offload_flag; + u8 resrved[3]; }; + /* * Ramrod data for tx queue stop ramrod */ @@ -855,49 +1053,144 @@ struct core_tx_stop_ramrod_data { __le32 reserved0[2]; }; + +/* + * Ramrod data for tx queue update ramrod + */ +struct core_tx_update_ramrod_data { + u8 update_qm_pq_id_flg /* Flag to Update QM PQ ID */; + u8 reserved0; + __le16 qm_pq_id /* Updated QM PQ ID */; + __le32 reserved1[1]; +}; + + +/* + * Enum flag for what type of dcb data to update + */ +enum dcb_dscp_update_mode { +/* use when no change should be done to dcb data */ + DONT_UPDATE_DCB_DSCP, + UPDATE_DCB /* use to update only l2 (vlan) priority */, + UPDATE_DSCP /* use to update only l3 dscp */, + UPDATE_DCB_DSCP /* update vlan pri and dscp */, + MAX_DCB_DSCP_UPDATE_FLAG +}; + + +struct eth_mstorm_per_pf_stat { + struct regpair gre_discard_pkts /* Dropped GRE RX packets */; + struct regpair vxlan_discard_pkts /* Dropped VXLAN RX packets */; + struct regpair geneve_discard_pkts /* Dropped GENEVE RX packets */; + struct regpair lb_discard_pkts /* Dropped Tx switched packets */; +}; + + struct eth_mstorm_per_queue_stat { +/* Number of packets discarded because TTL=0 (in IPv4) or hopLimit=0 (IPv6) */ struct regpair ttl0_discard; +/* Number of packets discarded because they are bigger than MTU */ struct regpair packet_too_big_discard; +/* Number of packets discarded due to lack of host buffers (BDs/SGEs/CQEs) */ struct regpair no_buff_discard; +/* Number of packets discarded because of no active Rx connection */ struct regpair not_active_discard; +/* number of coalesced packets in all TPA aggregations */ struct regpair tpa_coalesced_pkts; +/* total number of TPA aggregations */ struct regpair tpa_coalesced_events; +/* number of aggregations, which abnormally ended */ struct regpair tpa_aborts_num; +/* total TCP payload length in all TPA aggregations */ struct regpair tpa_coalesced_bytes; }; + +/* + * Ethernet TX Per PF + */ +struct eth_pstorm_per_pf_stat { +/* number of total ucast bytes sent on loopback port without errors */ + struct regpair sent_lb_ucast_bytes; +/* number of total mcast bytes sent on loopback port without errors */ + struct regpair sent_lb_mcast_bytes; +/* number of total bcast bytes sent on loopback port without errors */ + struct regpair sent_lb_bcast_bytes; +/* number of total ucast packets sent on loopback port without errors */ + struct regpair sent_lb_ucast_pkts; +/* number of total mcast packets sent on loopback port without errors */ + struct regpair sent_lb_mcast_pkts; +/* number of total bcast packets sent on loopback port without errors */ + struct regpair sent_lb_bcast_pkts; + struct regpair sent_gre_bytes /* Sent GRE bytes */; + struct regpair sent_vxlan_bytes /* Sent VXLAN bytes */; + struct regpair sent_geneve_bytes /* Sent GENEVE bytes */; + struct regpair sent_gre_pkts /* Sent GRE packets */; + struct regpair sent_vxlan_pkts /* Sent VXLAN packets */; + struct regpair sent_geneve_pkts /* Sent GENEVE packets */; + struct regpair gre_drop_pkts /* Dropped GRE TX packets */; + struct regpair vxlan_drop_pkts /* Dropped VXLAN TX packets */; + struct regpair geneve_drop_pkts /* Dropped GENEVE TX packets */; +}; + + /* * Ethernet TX Per Queue Stats */ struct eth_pstorm_per_queue_stat { - struct regpair sent_ucast_bytes - /* number of total bytes sent without errors */; - struct regpair sent_mcast_bytes - /* number of total bytes sent without errors */; - struct regpair sent_bcast_bytes - /* number of total bytes sent without errors */; - struct regpair sent_ucast_pkts - /* number of total packets sent without errors */; - struct regpair sent_mcast_pkts - /* number of total packets sent without errors */; - struct regpair sent_bcast_pkts - /* number of total packets sent without errors */; - struct regpair error_drop_pkts - /* number of total packets dropped due to errors */; +/* number of total bytes sent without errors */ + struct regpair sent_ucast_bytes; +/* number of total bytes sent without errors */ + struct regpair sent_mcast_bytes; +/* number of total bytes sent without errors */ + struct regpair sent_bcast_bytes; +/* number of total packets sent without errors */ + struct regpair sent_ucast_pkts; +/* number of total packets sent without errors */ + struct regpair sent_mcast_pkts; +/* number of total packets sent without errors */ + struct regpair sent_bcast_pkts; +/* number of total packets dropped due to errors */ + struct regpair error_drop_pkts; }; + /* * ETH Rx producers data */ struct eth_rx_rate_limit { +/* Rate Limit Multiplier - (Storm Clock (MHz) * 8 / Desired Bandwidth (MB/s)) */ __le16 mult; - __le16 cnst - /* Constant term to add (or subtract from number of cycles) */; +/* Constant term to add (or subtract from number of cycles) */ + __le16 cnst; u8 add_sub_cnst /* Add (1) or subtract (0) constant term */; u8 reserved0; __le16 reserved1; }; + +struct eth_ustorm_per_pf_stat { +/* number of total ucast bytes received on loopback port without errors */ + struct regpair rcv_lb_ucast_bytes; +/* number of total mcast bytes received on loopback port without errors */ + struct regpair rcv_lb_mcast_bytes; +/* number of total bcast bytes received on loopback port without errors */ + struct regpair rcv_lb_bcast_bytes; +/* number of total ucast packets received on loopback port without errors */ + struct regpair rcv_lb_ucast_pkts; +/* number of total mcast packets received on loopback port without errors */ + struct regpair rcv_lb_mcast_pkts; +/* number of total bcast packets received on loopback port without errors */ + struct regpair rcv_lb_bcast_pkts; + struct regpair rcv_gre_bytes /* Received GRE bytes */; + struct regpair rcv_vxlan_bytes /* Received VXLAN bytes */; + struct regpair rcv_geneve_bytes /* Received GENEVE bytes */; + struct regpair rcv_gre_pkts /* Received GRE packets */; + struct regpair rcv_vxlan_pkts /* Received VXLAN packets */; + struct regpair rcv_geneve_pkts /* Received GENEVE packets */; +}; + + struct eth_ustorm_per_queue_stat { struct regpair rcv_ucast_bytes; struct regpair rcv_mcast_bytes; @@ -907,6 +1200,64 @@ struct eth_ustorm_per_queue_stat { struct regpair rcv_bcast_pkts; }; + +/* + * Event Ring VF-PF Channel data + */ +struct vf_pf_channel_eqe_data { + struct regpair msg_addr /* VF-PF message address */; +}; + +/* + * Event Ring malicious VF data + */ +struct malicious_vf_eqe_data { + u8 vf_id /* Malicious VF ID */; + u8 err_id /* Malicious VF error (use enum malicious_vf_error_id) */; + __le16 reserved[3]; +}; + +/* + * Event Ring initial cleanup data + */ +struct initial_cleanup_eqe_data { + u8 vf_id /* VF ID */; + u8 reserved[7]; +}; + +/* + * Event Data Union + */ +union event_ring_data { + u8 bytes[8] /* Byte Array */; + struct vf_pf_channel_eqe_data vf_pf_channel /* VF-PF Channel data */; + struct iscsi_eqe_data iscsi_info /* Dedicated fields to iscsi data */; +/* Dedicated fields to iscsi connect done results */ + struct iscsi_connect_done_results iscsi_conn_done_info; + struct malicious_vf_eqe_data malicious_vf /* Malicious VF data */; +/* VF Initial Cleanup data */ + struct initial_cleanup_eqe_data vf_init_cleanup; +}; + + +/* + * Event Ring Entry + */ +struct event_ring_entry { + u8 protocol_id /* Event Protocol ID (use enum protocol_type) */; + u8 opcode /* Event Opcode */; + __le16 reserved0 /* Reserved */; + __le16 echo /* Echo value from ramrod data on the host */; + u8 fw_return_code /* FW return code for SP ramrods */; + u8 flags; +/* 0: synchronous EQE - a completion of SP message. 1: asynchronous EQE */ +#define EVENT_RING_ENTRY_ASYNC_MASK 0x1 +#define EVENT_RING_ENTRY_ASYNC_SHIFT 0 +#define EVENT_RING_ENTRY_RESERVED1_MASK 0x7F +#define EVENT_RING_ENTRY_RESERVED1_SHIFT 1 + union event_ring_data data; +}; + /* * Event Ring Next Page Address */ @@ -920,10 +1271,12 @@ struct event_ring_next_addr { */ union event_ring_element { struct event_ring_entry entry /* Event Ring Entry */; - struct event_ring_next_addr next_addr /* Event Ring Next Page Address */ - ; +/* Event Ring Next Page Address */ + struct event_ring_next_addr next_addr; }; + + /* * Ports mode */ @@ -933,6 +1286,28 @@ enum fw_flow_ctrl_mode { MAX_FW_FLOW_CTRL_MODE }; + +/* + * GFT profile type. + */ +enum gft_profile_type { + GFT_PROFILE_TYPE_4_TUPLE /* 4 tuple, IP type and L4 type match. */, +/* L4 destination port, IP type and L4 type match. */ + GFT_PROFILE_TYPE_L4_DST_PORT, + GFT_PROFILE_TYPE_IP_DST_PORT /* IP destination port and IP type. */, + MAX_GFT_PROFILE_TYPE +}; + + +/* + * Major and Minor hsi Versions + */ +struct hsi_fp_ver_struct { + u8 minor_ver_arr[2] /* Minor Version of hsi loading pf */; + u8 major_ver_arr[2] /* Major Version of driver loading pf */; +}; + + /* * Integration Phase */ @@ -943,57 +1318,115 @@ enum integ_phase { MAX_INTEG_PHASE }; + +/* + * Ports mode + */ +enum iwarp_ll2_tx_queues { +/* LL2 queue for OOO packets sent in-order by the driver */ + IWARP_LL2_IN_ORDER_TX_QUEUE = 1, +/* LL2 queue for unaligned packets sent aligned by the driver */ + IWARP_LL2_ALIGNED_TX_QUEUE, +/* LL2 queue for unaligned packets sent aligned and was right-trimmed by the + * driver + */ + IWARP_LL2_ALIGNED_RIGHT_TRIMMED_TX_QUEUE, + IWARP_LL2_ERROR /* Error indication */, + MAX_IWARP_LL2_TX_QUEUES +}; + + /* * Malicious VF error ID */ enum malicious_vf_error_id { MALICIOUS_VF_NO_ERROR /* Zero placeholder value */, - VF_PF_CHANNEL_NOT_READY - /* Writing to VF/PF channel when it is not ready */, +/* Writing to VF/PF channel when it is not ready */ + VF_PF_CHANNEL_NOT_READY, VF_ZONE_MSG_NOT_VALID /* VF channel message is not valid */, VF_ZONE_FUNC_NOT_ENABLED /* Parent PF of VF channel is not active */, - ETH_PACKET_TOO_SMALL - /* TX packet is shorter then reported on BDs or from minimal size */ - , - ETH_ILLEGAL_VLAN_MODE - /* Tx packet with marked as insert VLAN when its illegal */, +/* TX packet is shorter then reported on BDs or from minimal size */ + ETH_PACKET_TOO_SMALL, +/* Tx packet with marked as insert VLAN when its illegal */ + ETH_ILLEGAL_VLAN_MODE, ETH_MTU_VIOLATION /* TX packet is greater then MTU */, - ETH_ILLEGAL_INBAND_TAGS /* TX packet has illegal inband tags marked */, - ETH_VLAN_INSERT_AND_INBAND_VLAN /* Vlan cant be added to inband tag */, - ETH_ILLEGAL_NBDS /* indicated number of BDs for the packet is illegal */ - , +/* TX packet has illegal inband tags marked */ + ETH_ILLEGAL_INBAND_TAGS, +/* Vlan cant be added to inband tag */ + ETH_VLAN_INSERT_AND_INBAND_VLAN, +/* indicated number of BDs for the packet is illegal */ + ETH_ILLEGAL_NBDS, ETH_FIRST_BD_WO_SOP /* 1st BD must have start_bd flag set */, - ETH_INSUFFICIENT_BDS - /* There are not enough BDs for transmission of even one packet */, +/* There are not enough BDs for transmission of even one packet */ + ETH_INSUFFICIENT_BDS, ETH_ILLEGAL_LSO_HDR_NBDS /* Header NBDs value is illegal */, ETH_ILLEGAL_LSO_MSS /* LSO MSS value is more than allowed */, - ETH_ZERO_SIZE_BD - /* empty BD (which not contains control flags) is illegal */, +/* empty BD (which not contains control flags) is illegal */ + ETH_ZERO_SIZE_BD, ETH_ILLEGAL_LSO_HDR_LEN /* LSO header size is above the limit */, - ETH_INSUFFICIENT_PAYLOAD - , +/* In LSO its expected that on the local BD ring there will be at least MSS + * bytes of data + */ + ETH_INSUFFICIENT_PAYLOAD, ETH_EDPM_OUT_OF_SYNC /* Valid BDs on local ring after EDPM L2 sync */, - ETH_TUNN_IPV6_EXT_NBD_ERR - /* Tunneled packet with IPv6+Ext without a proper number of BDs */, +/* Tunneled packet with IPv6+Ext without a proper number of BDs */ + ETH_TUNN_IPV6_EXT_NBD_ERR, + ETH_CONTROL_PACKET_VIOLATION /* VF sent control frame such as PFC */, + ETH_ANTI_SPOOFING_ERR /* Anti-Spoofing verification failure */, MAX_MALICIOUS_VF_ERROR_ID }; + + /* * Mstorm non-triggering VF zone */ struct mstorm_non_trigger_vf_zone { - struct eth_mstorm_per_queue_stat eth_queue_stat - /* VF statistic bucket */; +/* VF statistic bucket */ + struct eth_mstorm_per_queue_stat eth_queue_stat; +/* VF RX queues producers */ + struct eth_rx_prod_data + eth_rx_queue_producers[ETH_MAX_NUM_RX_QUEUES_PER_VF_QUAD]; }; + /* * Mstorm VF zone */ struct mstorm_vf_zone { - struct mstorm_non_trigger_vf_zone non_trigger - /* non-interrupt-triggering zone */; +/* non-interrupt-triggering zone */ + struct mstorm_non_trigger_vf_zone non_trigger; }; + +/* + * vlan header including TPID and TCI fields + */ +struct vlan_header { + __le16 tpid /* Tag Protocol Identifier */; + __le16 tci /* Tag Control Information */; +}; + +/* + * outer tag configurations + */ +struct outer_tag_config_struct { +/* Enables the STAG Priority Change , Should be 1 for Bette Davis and UFP with + * Host Control mode. Else - 0 + */ + u8 enable_stag_pri_change; +/* If inner_to_outer_pri_map is initialize then set pri_map_valid */ + u8 pri_map_valid; + u8 reserved[2]; +/* In case mf_mode is MF_OVLAN, this field specifies the outer tag protocol + * identifier and outer tag control information + */ + struct vlan_header outer_tag; +/* Map from inner to outer priority. Set pri_map_valid when init map */ + u8 inner_to_outer_pri_map[8]; +}; + + /* * personality per PF */ @@ -1009,29 +1442,31 @@ enum personality_type { MAX_PERSONALITY_TYPE }; + /* * tunnel configuration */ struct pf_start_tunnel_config { - u8 set_vxlan_udp_port_flg /* Set VXLAN tunnel UDP destination port. */; - u8 set_geneve_udp_port_flg /* Set GENEVE tunnel UDP destination port. */ - ; - u8 tx_enable_vxlan /* If set, enable VXLAN tunnel in TX path. */; - u8 tx_enable_l2geneve /* If set, enable l2 GENEVE tunnel in TX path. */ - ; - u8 tx_enable_ipgeneve /* If set, enable IP GENEVE tunnel in TX path. */ - ; - u8 tx_enable_l2gre /* If set, enable l2 GRE tunnel in TX path. */; - u8 tx_enable_ipgre /* If set, enable IP GRE tunnel in TX path. */; - u8 tunnel_clss_vxlan /* Classification scheme for VXLAN tunnel. */; - u8 tunnel_clss_l2geneve - /* Classification scheme for l2 GENEVE tunnel. */; - u8 tunnel_clss_ipgeneve - /* Classification scheme for ip GENEVE tunnel. */; - u8 tunnel_clss_l2gre /* Classification scheme for l2 GRE tunnel. */; - u8 tunnel_clss_ipgre /* Classification scheme for ip GRE tunnel. */; - __le16 vxlan_udp_port /* VXLAN tunnel UDP destination port. */; - __le16 geneve_udp_port /* GENEVE tunnel UDP destination port. */; +/* Set VXLAN tunnel UDP destination port to vxlan_udp_port. If not set - + * FW will use a default port + */ + u8 set_vxlan_udp_port_flg; +/* Set GENEVE tunnel UDP destination port to geneve_udp_port. If not set - + * FW will use a default port + */ + u8 set_geneve_udp_port_flg; + u8 tunnel_clss_vxlan /* Rx classification scheme for VXLAN tunnel. */; +/* Rx classification scheme for l2 GENEVE tunnel. */ + u8 tunnel_clss_l2geneve; +/* Rx classification scheme for ip GENEVE tunnel. */ + u8 tunnel_clss_ipgeneve; + u8 tunnel_clss_l2gre /* Rx classification scheme for l2 GRE tunnel. */; + u8 tunnel_clss_ipgre /* Rx classification scheme for ip GRE tunnel. */; + u8 reserved; +/* VXLAN tunnel UDP destination port. Valid if set_vxlan_udp_port_flg=1 */ + __le16 vxlan_udp_port; +/* GENEVE tunnel UDP destination port. Valid if set_geneve_udp_port_flg=1 */ + __le16 geneve_udp_port; }; /* @@ -1039,97 +1474,118 @@ struct pf_start_tunnel_config { */ struct pf_start_ramrod_data { struct regpair event_ring_pbl_addr /* Address of event ring PBL */; - struct regpair consolid_q_pbl_addr - /* PBL address of consolidation queue */; - struct pf_start_tunnel_config tunnel_config /* tunnel configuration. */ - ; +/* PBL address of consolidation queue */ + struct regpair consolid_q_pbl_addr; +/* tunnel configuration. */ + struct pf_start_tunnel_config tunnel_config; __le16 event_ring_sb_id /* Status block ID */; +/* All VfIds owned by Pf will be from baseVfId till baseVfId+numVfs */ u8 base_vf_id; - ; u8 num_vfs /* Amount of vfs owned by PF */; u8 event_ring_num_pages /* Number of PBL pages in event ring */; u8 event_ring_sb_index /* Status block index */; u8 path_id /* HW path ID (engine ID) */; u8 warning_as_error /* In FW asserts, treat warning as error */; - u8 dont_log_ramrods - /* If not set - throw a warning for each ramrod (for debug) */; +/* If not set - throw a warning for each ramrod (for debug) */ + u8 dont_log_ramrods; u8 personality /* define what type of personality is new PF */; +/* Log type mask. Each bit set enables a corresponding event type logging. + * Event types are defined as ASSERT_LOG_TYPE_xxx + */ __le16 log_type_mask; u8 mf_mode /* Multi function mode */; u8 integ_phase /* Integration phase */; +/* If set, inter-pf tx switching is allowed in Switch Independent func mode */ u8 allow_npar_tx_switching; - u8 inner_to_outer_pri_map[8]; - u8 pri_map_valid - /* If inner_to_outer_pri_map is initialize then set pri_map_valid */ - ; - __le32 outer_tag; - u8 reserved0[4]; + u8 reserved0; +/* FP HSI version to be used by FW */ + struct hsi_fp_ver_struct hsi_fp_ver; +/* Outer tag configurations */ + struct outer_tag_config_struct outer_tag_config; }; + + /* * Data for port update ramrod */ struct protocol_dcb_data { u8 dcb_enable_flag /* dcbEnable flag value */; + u8 dscp_enable_flag /* If set use dscp value */; u8 dcb_priority /* dcbPri flag value */; u8 dcb_tc /* dcb TC value */; - u8 reserved; + u8 dscp_val /* dscp value to write if dscp_enable_flag is set */; + u8 reserved0; }; /* - * tunnel configuration + * Update tunnel configuration */ struct pf_update_tunnel_config { +/* Update RX per PF tunnel classification scheme. */ u8 update_rx_pf_clss; - u8 update_tx_pf_clss; - u8 set_vxlan_udp_port_flg - /* Update VXLAN tunnel UDP destination port. */; - u8 set_geneve_udp_port_flg - /* Update GENEVE tunnel UDP destination port. */; - u8 tx_enable_vxlan /* If set, enable VXLAN tunnel in TX path. */; - u8 tx_enable_l2geneve /* If set, enable l2 GENEVE tunnel in TX path. */ - ; - u8 tx_enable_ipgeneve /* If set, enable IP GENEVE tunnel in TX path. */ - ; - u8 tx_enable_l2gre /* If set, enable l2 GRE tunnel in TX path. */; - u8 tx_enable_ipgre /* If set, enable IP GRE tunnel in TX path. */; +/* Update per PORT default tunnel RX classification scheme for traffic with + * unknown unicast outer MAC in NPAR mode. + */ + u8 update_rx_def_ucast_clss; +/* Update per PORT default tunnel RX classification scheme for traffic with non + * unicast outer MAC in NPAR mode. + */ + u8 update_rx_def_non_ucast_clss; +/* Update VXLAN tunnel UDP destination port. */ + u8 set_vxlan_udp_port_flg; +/* Update GENEVE tunnel UDP destination port. */ + u8 set_geneve_udp_port_flg; u8 tunnel_clss_vxlan /* Classification scheme for VXLAN tunnel. */; - u8 tunnel_clss_l2geneve - /* Classification scheme for l2 GENEVE tunnel. */; - u8 tunnel_clss_ipgeneve - /* Classification scheme for ip GENEVE tunnel. */; +/* Classification scheme for l2 GENEVE tunnel. */ + u8 tunnel_clss_l2geneve; +/* Classification scheme for ip GENEVE tunnel. */ + u8 tunnel_clss_ipgeneve; u8 tunnel_clss_l2gre /* Classification scheme for l2 GRE tunnel. */; u8 tunnel_clss_ipgre /* Classification scheme for ip GRE tunnel. */; __le16 vxlan_udp_port /* VXLAN tunnel UDP destination port. */; __le16 geneve_udp_port /* GENEVE tunnel UDP destination port. */; - __le16 reserved[3]; + __le16 reserved; }; /* * Data for port update ramrod */ struct pf_update_ramrod_data { - u8 pf_id; - u8 update_eth_dcb_data_flag /* Update Eth DCB data indication */; - u8 update_fcoe_dcb_data_flag /* Update FCOE DCB data indication */; - u8 update_iscsi_dcb_data_flag /* Update iSCSI DCB data indication */; - u8 update_roce_dcb_data_flag /* Update ROCE DCB data indication */; - u8 update_iwarp_dcb_data_flag /* Update IWARP DCB data indication */; +/* Update Eth DCB data indication (use enum dcb_dscp_update_mode) */ + u8 update_eth_dcb_data_mode; +/* Update FCOE DCB data indication (use enum dcb_dscp_update_mode) */ + u8 update_fcoe_dcb_data_mode; +/* Update iSCSI DCB data indication (use enum dcb_dscp_update_mode) */ + u8 update_iscsi_dcb_data_mode; + u8 update_roce_dcb_data_mode /* Update ROCE DCB data indication */; +/* Update RROCE (RoceV2) DCB data indication */ + u8 update_rroce_dcb_data_mode; + u8 update_iwarp_dcb_data_mode /* Update IWARP DCB data indication */; u8 update_mf_vlan_flag /* Update MF outer vlan Id */; - u8 reserved; +/* Update Enable STAG Priority Change indication */ + u8 update_enable_stag_pri_change; struct protocol_dcb_data eth_dcb_data /* core eth related fields */; struct protocol_dcb_data fcoe_dcb_data /* core fcoe related fields */; - struct protocol_dcb_data iscsi_dcb_data /* core iscsi related fields */ - ; +/* core iscsi related fields */ + struct protocol_dcb_data iscsi_dcb_data; struct protocol_dcb_data roce_dcb_data /* core roce related fields */; - struct protocol_dcb_data iwarp_dcb_data /* core iwarp related fields */ - ; +/* core roce related fields */ + struct protocol_dcb_data rroce_dcb_data; +/* core iwarp related fields */ + struct protocol_dcb_data iwarp_dcb_data; __le16 mf_vlan /* new outer vlan id value */; - __le16 reserved2; - struct pf_update_tunnel_config tunnel_config /* tunnel configuration. */ - ; +/* enables the inner to outer TAG priority mapping. Should be 1 for Bette Davis + * and UFP with Host Control mode, else - 0. + */ + u8 enable_stag_pri_change; + u8 reserved; +/* tunnel configuration. */ + struct pf_update_tunnel_config tunnel_config; }; + + /* * Ports mode */ @@ -1142,6 +1598,19 @@ enum ports_mode { MAX_PORTS_MODE }; + + +/* + * use to index in hsi_fp_[major|minor]_ver_arr per protocol + */ +enum protocol_version_array_key { + ETH_VER_KEY = 0, + ROCE_VER_KEY, + MAX_PROTOCOL_VERSION_ARRAY_KEY +}; + + + /* * RDMA TX Stats */ @@ -1154,20 +1623,22 @@ struct rdma_sent_stats { * Pstorm non-triggering VF zone */ struct pstorm_non_trigger_vf_zone { - struct eth_pstorm_per_queue_stat eth_queue_stat - /* VF statistic bucket */; +/* VF statistic bucket */ + struct eth_pstorm_per_queue_stat eth_queue_stat; struct rdma_sent_stats rdma_stats /* RoCE sent statistics */; }; + /* * Pstorm VF zone */ struct pstorm_vf_zone { - struct pstorm_non_trigger_vf_zone non_trigger - /* non-interrupt-triggering zone */; +/* non-interrupt-triggering zone */ + struct pstorm_non_trigger_vf_zone non_trigger; struct regpair reserved[7] /* vf_zone size mus be power of 2 */; }; + /* * Ramrod Header of SPQE */ @@ -1178,6 +1649,7 @@ struct ramrod_header { __le16 echo /* Ramrod echo */; }; + /* * RDMA RX Stats */ @@ -1186,6 +1658,34 @@ struct rdma_rcv_stats { struct regpair rcv_pkts /* number of total RDMA packets received */; }; + + +/* + * Data for update QCN/DCQCN RL ramrod + */ +struct rl_update_ramrod_data { + u8 qcn_update_param_flg /* Update QCN global params: timeout. */; +/* Update DCQCN global params: timeout, g, k. */ + u8 dcqcn_update_param_flg; + u8 rl_init_flg /* Init RL parameters, when RL disabled. */; + u8 rl_start_flg /* Start RL in IDLE state. Set rate to maximum. */; + u8 rl_stop_flg /* Stop RL. */; + u8 rl_id_first /* ID of first or single RL, that will be updated. */; +/* ID of last RL, that will be updated. If clear, single RL will updated. */ + u8 rl_id_last; + u8 rl_dc_qcn_flg /* If set, RL will used for DCQCN. */; + __le32 rl_bc_rate /* Byte Counter Limit. */; + __le16 rl_max_rate /* Maximum rate in 1.6 Mbps resolution. */; + __le16 rl_r_ai /* Active increase rate. */; + __le16 rl_r_hai /* Hyper active increase rate. */; + __le16 dcqcn_g /* DCQCN Alpha update gain in 1/64K resolution . */; + __le32 dcqcn_k_us /* DCQCN Alpha update interval. */; + __le32 dcqcn_timeuot_us /* DCQCN timeout. */; + __le32 qcn_timeuot_us /* QCN timeout. */; + __le32 reserved[2]; +}; + + /* * Slowpath Element (SPQE) */ @@ -1194,6 +1694,7 @@ struct slow_path_element { struct regpair data_ptr /* Pointer to the Ramrod Data on the Host */; }; + /* * Tstorm non-triggering VF zone */ @@ -1201,65 +1702,90 @@ struct tstorm_non_trigger_vf_zone { struct rdma_rcv_stats rdma_stats /* RoCE received statistics */; }; + struct tstorm_per_port_stat { - struct regpair trunc_error_discard - /* packet is dropped because it was truncated in NIG */; - struct regpair mac_error_discard - /* packet is dropped because of Ethernet FCS error */; - struct regpair mftag_filter_discard - /* packet is dropped because classification was unsuccessful */; +/* packet is dropped because it was truncated in NIG */ + struct regpair trunc_error_discard; +/* packet is dropped because of Ethernet FCS error */ + struct regpair mac_error_discard; +/* packet is dropped because classification was unsuccessful */ + struct regpair mftag_filter_discard; +/* packet was passed to Ethernet and dropped because of no mac filter match */ struct regpair eth_mac_filter_discard; +/* packet passed to Light L2 and dropped because Light L2 is not configured for + * this PF + */ struct regpair ll2_mac_filter_discard; +/* packet passed to Light L2 and dropped because Light L2 is not configured for + * this PF + */ struct regpair ll2_conn_disabled_discard; - struct regpair iscsi_irregular_pkt - /* packet is an ISCSI irregular packet */; - struct regpair fcoe_irregular_pkt - /* packet is an FCOE irregular packet */; - struct regpair roce_irregular_pkt - /* packet is an ROCE irregular packet */; - struct regpair eth_irregular_pkt /* packet is an ETH irregular packet */ - ; - struct regpair toe_irregular_pkt /* packet is an TOE irregular packet */ - ; - struct regpair preroce_irregular_pkt - /* packet is an PREROCE irregular packet */; +/* packet is an ISCSI irregular packet */ + struct regpair iscsi_irregular_pkt; +/* packet is an FCOE irregular packet */ + struct regpair fcoe_irregular_pkt; +/* packet is an ROCE irregular packet */ + struct regpair roce_irregular_pkt; +/* packet is an IWARP irregular packet */ + struct regpair iwarp_irregular_pkt; +/* packet is an ETH irregular packet */ + struct regpair eth_irregular_pkt; +/* packet is an TOE irregular packet */ + struct regpair toe_irregular_pkt; +/* packet is an PREROCE irregular packet */ + struct regpair preroce_irregular_pkt; + struct regpair eth_gre_tunn_filter_discard /* GRE dropped packets */; +/* VXLAN dropped packets */ + struct regpair eth_vxlan_tunn_filter_discard; +/* GENEVE dropped packets */ + struct regpair eth_geneve_tunn_filter_discard; }; + /* * Tstorm VF zone */ struct tstorm_vf_zone { - struct tstorm_non_trigger_vf_zone non_trigger - /* non-interrupt-triggering zone */; +/* non-interrupt-triggering zone */ + struct tstorm_non_trigger_vf_zone non_trigger; }; + /* * Tunnel classification scheme */ enum tunnel_clss { - TUNNEL_CLSS_MAC_VLAN = - 0 - /* Use MAC & VLAN from first L2 header for vport classification. */ - , - TUNNEL_CLSS_MAC_VNI - , - TUNNEL_CLSS_INNER_MAC_VLAN - /* Use MAC and VLAN from last L2 header for vport classification */ - , - TUNNEL_CLSS_INNER_MAC_VNI - , +/* Use MAC and VLAN from first L2 header for vport classification. */ + TUNNEL_CLSS_MAC_VLAN = 0, +/* Use MAC from first L2 header and VNI from tunnel header for vport + * classification + */ + TUNNEL_CLSS_MAC_VNI, +/* Use MAC and VLAN from last L2 header for vport classification */ + TUNNEL_CLSS_INNER_MAC_VLAN, +/* Use MAC from last L2 header and VNI from tunnel header for vport + * classification + */ + TUNNEL_CLSS_INNER_MAC_VNI, +/* Use MAC and VLAN from last L2 header for vport classification. If no exact + * match, use MAC and VLAN from first L2 header for classification. + */ + TUNNEL_CLSS_MAC_VLAN_DUAL_STAGE, MAX_TUNNEL_CLSS }; + + /* * Ustorm non-triggering VF zone */ struct ustorm_non_trigger_vf_zone { - struct eth_ustorm_per_queue_stat eth_queue_stat - /* VF statistic bucket */; +/* VF statistic bucket */ + struct eth_ustorm_per_queue_stat eth_queue_stat; struct regpair vf_pf_msg_addr /* VF-PF message address */; }; + /* * Ustorm triggering VF zone */ @@ -1268,36 +1794,49 @@ struct ustorm_trigger_vf_zone { u8 reserved[7]; }; + /* * Ustorm VF zone */ struct ustorm_vf_zone { - struct ustorm_non_trigger_vf_zone non_trigger - /* non-interrupt-triggering zone */; +/* non-interrupt-triggering zone */ + struct ustorm_non_trigger_vf_zone non_trigger; struct ustorm_trigger_vf_zone trigger /* interrupt triggering zone */; }; + /* * VF-PF channel data */ struct vf_pf_channel_data { +/* 0: VF-PF Channel NOT ready. Waiting for ack from PF driver. 1: VF-PF Channel + * is ready for a new transaction. + */ __le32 ready; +/* 0: VF-PF Channel is invalid because of malicious VF. 1: VF-PF Channel is + * valid. + */ u8 valid; u8 reserved0; __le16 reserved1; }; + /* * Ramrod data for VF start ramrod */ struct vf_start_ramrod_data { u8 vf_id /* VF ID */; +/* If set, initial cleanup ack will be sent to parent PF SP event queue */ u8 enable_flr_ack; __le16 opaque_fid /* VF opaque FID */; u8 personality /* define what type of personality is new VF */; - u8 reserved[3]; + u8 reserved[7]; +/* FP HSI version to be used by FW */ + struct hsi_fp_ver_struct hsi_fp_ver; }; + /* * Ramrod data for VF start ramrod */ @@ -1308,6 +1847,24 @@ struct vf_stop_ramrod_data { __le32 reserved2; }; + +/* + * VF zone size mode. + */ +enum vf_zone_size_mode { +/* Default VF zone size. Up to 192 VF supported. */ + VF_ZONE_SIZE_MODE_DEFAULT, +/* Doubled VF zone size. Up to 96 VF supported. */ + VF_ZONE_SIZE_MODE_DOUBLE, +/* Quad VF zone size. Up to 48 VF supported. */ + VF_ZONE_SIZE_MODE_QUAD, + MAX_VF_ZONE_SIZE_MODE +}; + + + + + /* * Attentions status block */ @@ -1319,236 +1876,103 @@ struct atten_status_block { __le32 reserved1; }; -enum block_addr { - GRCBASE_GRC = 0x50000, - GRCBASE_MISCS = 0x9000, - GRCBASE_MISC = 0x8000, - GRCBASE_DBU = 0xa000, - GRCBASE_PGLUE_B = 0x2a8000, - GRCBASE_CNIG = 0x218000, - GRCBASE_CPMU = 0x30000, - GRCBASE_NCSI = 0x40000, - GRCBASE_OPTE = 0x53000, - GRCBASE_BMB = 0x540000, - GRCBASE_PCIE = 0x54000, - GRCBASE_MCP = 0xe00000, - GRCBASE_MCP2 = 0x52000, - GRCBASE_PSWHST = 0x2a0000, - GRCBASE_PSWHST2 = 0x29e000, - GRCBASE_PSWRD = 0x29c000, - GRCBASE_PSWRD2 = 0x29d000, - GRCBASE_PSWWR = 0x29a000, - GRCBASE_PSWWR2 = 0x29b000, - GRCBASE_PSWRQ = 0x280000, - GRCBASE_PSWRQ2 = 0x240000, - GRCBASE_PGLCS = 0x0, - GRCBASE_DMAE = 0xc000, - GRCBASE_PTU = 0x560000, - GRCBASE_TCM = 0x1180000, - GRCBASE_MCM = 0x1200000, - GRCBASE_UCM = 0x1280000, - GRCBASE_XCM = 0x1000000, - GRCBASE_YCM = 0x1080000, - GRCBASE_PCM = 0x1100000, - GRCBASE_QM = 0x2f0000, - GRCBASE_TM = 0x2c0000, - GRCBASE_DORQ = 0x100000, - GRCBASE_BRB = 0x340000, - GRCBASE_SRC = 0x238000, - GRCBASE_PRS = 0x1f0000, - GRCBASE_TSDM = 0xfb0000, - GRCBASE_MSDM = 0xfc0000, - GRCBASE_USDM = 0xfd0000, - GRCBASE_XSDM = 0xf80000, - GRCBASE_YSDM = 0xf90000, - GRCBASE_PSDM = 0xfa0000, - GRCBASE_TSEM = 0x1700000, - GRCBASE_MSEM = 0x1800000, - GRCBASE_USEM = 0x1900000, - GRCBASE_XSEM = 0x1400000, - GRCBASE_YSEM = 0x1500000, - GRCBASE_PSEM = 0x1600000, - GRCBASE_RSS = 0x238800, - GRCBASE_TMLD = 0x4d0000, - GRCBASE_MULD = 0x4e0000, - GRCBASE_YULD = 0x4c8000, - GRCBASE_XYLD = 0x4c0000, - GRCBASE_PRM = 0x230000, - GRCBASE_PBF_PB1 = 0xda0000, - GRCBASE_PBF_PB2 = 0xda4000, - GRCBASE_RPB = 0x23c000, - GRCBASE_BTB = 0xdb0000, - GRCBASE_PBF = 0xd80000, - GRCBASE_RDIF = 0x300000, - GRCBASE_TDIF = 0x310000, - GRCBASE_CDU = 0x580000, - GRCBASE_CCFC = 0x2e0000, - GRCBASE_TCFC = 0x2d0000, - GRCBASE_IGU = 0x180000, - GRCBASE_CAU = 0x1c0000, - GRCBASE_UMAC = 0x51000, - GRCBASE_XMAC = 0x210000, - GRCBASE_DBG = 0x10000, - GRCBASE_NIG = 0x500000, - GRCBASE_WOL = 0x600000, - GRCBASE_BMBN = 0x610000, - GRCBASE_IPC = 0x20000, - GRCBASE_NWM = 0x800000, - GRCBASE_NWS = 0x700000, - GRCBASE_MS = 0x6a0000, - GRCBASE_PHY_PCIE = 0x620000, - GRCBASE_MISC_AEU = 0x8000, - GRCBASE_BAR0_MAP = 0x1c00000, - MAX_BLOCK_ADDR -}; - -enum block_id { - BLOCK_GRC, - BLOCK_MISCS, - BLOCK_MISC, - BLOCK_DBU, - BLOCK_PGLUE_B, - BLOCK_CNIG, - BLOCK_CPMU, - BLOCK_NCSI, - BLOCK_OPTE, - BLOCK_BMB, - BLOCK_PCIE, - BLOCK_MCP, - BLOCK_MCP2, - BLOCK_PSWHST, - BLOCK_PSWHST2, - BLOCK_PSWRD, - BLOCK_PSWRD2, - BLOCK_PSWWR, - BLOCK_PSWWR2, - BLOCK_PSWRQ, - BLOCK_PSWRQ2, - BLOCK_PGLCS, - BLOCK_DMAE, - BLOCK_PTU, - BLOCK_TCM, - BLOCK_MCM, - BLOCK_UCM, - BLOCK_XCM, - BLOCK_YCM, - BLOCK_PCM, - BLOCK_QM, - BLOCK_TM, - BLOCK_DORQ, - BLOCK_BRB, - BLOCK_SRC, - BLOCK_PRS, - BLOCK_TSDM, - BLOCK_MSDM, - BLOCK_USDM, - BLOCK_XSDM, - BLOCK_YSDM, - BLOCK_PSDM, - BLOCK_TSEM, - BLOCK_MSEM, - BLOCK_USEM, - BLOCK_XSEM, - BLOCK_YSEM, - BLOCK_PSEM, - BLOCK_RSS, - BLOCK_TMLD, - BLOCK_MULD, - BLOCK_YULD, - BLOCK_XYLD, - BLOCK_PRM, - BLOCK_PBF_PB1, - BLOCK_PBF_PB2, - BLOCK_RPB, - BLOCK_BTB, - BLOCK_PBF, - BLOCK_RDIF, - BLOCK_TDIF, - BLOCK_CDU, - BLOCK_CCFC, - BLOCK_TCFC, - BLOCK_IGU, - BLOCK_CAU, - BLOCK_UMAC, - BLOCK_XMAC, - BLOCK_DBG, - BLOCK_NIG, - BLOCK_WOL, - BLOCK_BMBN, - BLOCK_IPC, - BLOCK_NWM, - BLOCK_NWS, - BLOCK_MS, - BLOCK_PHY_PCIE, - BLOCK_MISC_AEU, - BLOCK_BAR0_MAP, - MAX_BLOCK_ID -}; - -/* - * Igu cleanup bit values to distinguish between clean or producer consumer - */ -enum command_type_bit { - IGU_COMMAND_TYPE_NOP = 0, - IGU_COMMAND_TYPE_SET = 1, - MAX_COMMAND_TYPE_BIT -}; /* * DMAE command */ struct dmae_cmd { __le32 opcode; +/* DMA Source. 0 - PCIe, 1 - GRC (use enum dmae_cmd_src_enum) */ #define DMAE_CMD_SRC_MASK 0x1 #define DMAE_CMD_SRC_SHIFT 0 +/* DMA destination. 0 - None, 1 - PCIe, 2 - GRC, 3 - None + * (use enum dmae_cmd_dst_enum) + */ #define DMAE_CMD_DST_MASK 0x3 #define DMAE_CMD_DST_SHIFT 1 +/* Completion destination. 0 - PCie, 1 - GRC (use enum dmae_cmd_c_dst_enum) */ #define DMAE_CMD_C_DST_MASK 0x1 #define DMAE_CMD_C_DST_SHIFT 3 +/* Reset the CRC result (do not use the previous result as the seed) */ #define DMAE_CMD_CRC_RESET_MASK 0x1 #define DMAE_CMD_CRC_RESET_SHIFT 4 +/* Reset the source address in the next go to the same source address of the + * previous go + */ #define DMAE_CMD_SRC_ADDR_RESET_MASK 0x1 #define DMAE_CMD_SRC_ADDR_RESET_SHIFT 5 +/* Reset the destination address in the next go to the same destination address + * of the previous go + */ #define DMAE_CMD_DST_ADDR_RESET_MASK 0x1 #define DMAE_CMD_DST_ADDR_RESET_SHIFT 6 +/* 0 completion function is the same as src function, 1 - 0 completion + * function is the same as dst function (use enum dmae_cmd_comp_func_enum) + */ #define DMAE_CMD_COMP_FUNC_MASK 0x1 #define DMAE_CMD_COMP_FUNC_SHIFT 7 +/* 0 - Do not write a completion word, 1 - Write a completion word + * (use enum dmae_cmd_comp_word_en_enum) + */ #define DMAE_CMD_COMP_WORD_EN_MASK 0x1 #define DMAE_CMD_COMP_WORD_EN_SHIFT 8 +/* 0 - Do not write a CRC word, 1 - Write a CRC word + * (use enum dmae_cmd_comp_crc_en_enum) + */ #define DMAE_CMD_COMP_CRC_EN_MASK 0x1 #define DMAE_CMD_COMP_CRC_EN_SHIFT 9 +/* The CRC word should be taken from the DMAE address space from address 9+X, + * where X is the value in these bits. + */ #define DMAE_CMD_COMP_CRC_OFFSET_MASK 0x7 #define DMAE_CMD_COMP_CRC_OFFSET_SHIFT 10 #define DMAE_CMD_RESERVED1_MASK 0x1 #define DMAE_CMD_RESERVED1_SHIFT 13 #define DMAE_CMD_ENDIANITY_MODE_MASK 0x3 #define DMAE_CMD_ENDIANITY_MODE_SHIFT 14 +/* The field specifies how the completion word is affected by PCIe read error. 0 + * Send a regular completion, 1 - Send a completion with an error indication, + * 2 do not send a completion (use enum dmae_cmd_error_handling_enum) + */ #define DMAE_CMD_ERR_HANDLING_MASK 0x3 #define DMAE_CMD_ERR_HANDLING_SHIFT 16 +/* The port ID to be placed on the RF FID field of the GRC bus. this field is + * used both when GRC is the destination and when it is the source of the DMAE + * transaction. + */ #define DMAE_CMD_PORT_ID_MASK 0x3 #define DMAE_CMD_PORT_ID_SHIFT 18 +/* Source PCI function number [3:0] */ #define DMAE_CMD_SRC_PF_ID_MASK 0xF #define DMAE_CMD_SRC_PF_ID_SHIFT 20 +/* Destination PCI function number [3:0] */ #define DMAE_CMD_DST_PF_ID_MASK 0xF #define DMAE_CMD_DST_PF_ID_SHIFT 24 -#define DMAE_CMD_SRC_VF_ID_VALID_MASK 0x1 +#define DMAE_CMD_SRC_VF_ID_VALID_MASK 0x1 /* Source VFID valid */ #define DMAE_CMD_SRC_VF_ID_VALID_SHIFT 28 -#define DMAE_CMD_DST_VF_ID_VALID_MASK 0x1 +#define DMAE_CMD_DST_VF_ID_VALID_MASK 0x1 /* Destination VFID valid */ #define DMAE_CMD_DST_VF_ID_VALID_SHIFT 29 #define DMAE_CMD_RESERVED2_MASK 0x3 #define DMAE_CMD_RESERVED2_SHIFT 30 - __le32 src_addr_lo - /* PCIe source address low in bytes or GRC source address in DW */; +/* PCIe source address low in bytes or GRC source address in DW */ + __le32 src_addr_lo; +/* PCIe source address high in bytes or reserved (if source is GRC) */ __le32 src_addr_hi; +/* PCIe destination address low in bytes or GRC destination address in DW */ __le32 dst_addr_lo; +/* PCIe destination address high in bytes or reserved (if destination is GRC) */ __le32 dst_addr_hi; - __le16 length /* Length in DW */; + __le16 length_dw /* Length in DW */; __le16 opcode_b; -#define DMAE_CMD_SRC_VF_ID_MASK 0xFF +#define DMAE_CMD_SRC_VF_ID_MASK 0xFF /* Source VF id */ #define DMAE_CMD_SRC_VF_ID_SHIFT 0 -#define DMAE_CMD_DST_VF_ID_MASK 0xFF +#define DMAE_CMD_DST_VF_ID_MASK 0xFF /* Destination VF id */ #define DMAE_CMD_DST_VF_ID_SHIFT 8 - __le32 comp_addr_lo /* PCIe completion address low or grc address */; +/* PCIe completion address low in bytes or GRC completion address in DW */ + __le32 comp_addr_lo; +/* PCIe completion address high in bytes or reserved (if completion address is + * GRC) + */ __le32 comp_addr_hi; __le32 comp_val /* Value to write to completion address */; __le32 crc32 /* crc16 result */; @@ -1561,43 +1985,151 @@ struct dmae_cmd { __le16 xsum8 /* checksum8 result */; }; -struct fw_ver_num { - u8 major /* Firmware major version number */; - u8 minor /* Firmware minor version number */; - u8 rev /* Firmware revision version number */; - u8 eng /* Firmware engineering version number (for bootleg versions) */ - ; + +enum dmae_cmd_comp_crc_en_enum { + dmae_cmd_comp_crc_disabled /* Do not write a CRC word */, + dmae_cmd_comp_crc_enabled /* Write a CRC word */, + MAX_DMAE_CMD_COMP_CRC_EN_ENUM }; -struct fw_ver_info { - __le16 tools_ver /* Tools version number */; - u8 image_id /* FW image ID (e.g. main, l2b, kuku) */; - u8 reserved1; - struct fw_ver_num num /* FW version number */; - __le32 timestamp /* FW Timestamp in unix time (sec. since 1970) */; - __le32 reserved2; + +enum dmae_cmd_comp_func_enum { +/* completion word and/or CRC will be sent to SRC-PCI function/SRC VFID */ + dmae_cmd_comp_func_to_src, +/* completion word and/or CRC will be sent to DST-PCI function/DST VFID */ + dmae_cmd_comp_func_to_dst, + MAX_DMAE_CMD_COMP_FUNC_ENUM }; -struct storm_ram_section { - __le16 offset - /* The offset of the section in the RAM (in 64 bit units) */; - __le16 size /* The size of the section (in 64 bit units) */; + +enum dmae_cmd_comp_word_en_enum { + dmae_cmd_comp_word_disabled /* Do not write a completion word */, + dmae_cmd_comp_word_enabled /* Write the completion word */, + MAX_DMAE_CMD_COMP_WORD_EN_ENUM }; -struct fw_info { - struct fw_ver_info ver /* FW version information */; - struct storm_ram_section fw_asserts_section - /* The FW Asserts offset/size in Storm RAM */; - __le32 reserved; + +enum dmae_cmd_c_dst_enum { + dmae_cmd_c_dst_pcie, + dmae_cmd_c_dst_grc, + MAX_DMAE_CMD_C_DST_ENUM }; -struct fw_info_location { - __le32 grc_addr /* GRC address where the fw_info struct is located. */; - __le32 size - /* Size of the fw_info structure (thats located at the grc_addr). */ - ; + +enum dmae_cmd_dst_enum { + dmae_cmd_dst_none_0, + dmae_cmd_dst_pcie, + dmae_cmd_dst_grc, + dmae_cmd_dst_none_3, + MAX_DMAE_CMD_DST_ENUM }; + +enum dmae_cmd_error_handling_enum { +/* Send a regular completion (with no error indication) */ + dmae_cmd_error_handling_send_regular_comp, +/* Send a completion with an error indication (i.e. set bit 31 of the completion + * word) + */ + dmae_cmd_error_handling_send_comp_with_err, + dmae_cmd_error_handling_dont_send_comp /* Do not send a completion */, + MAX_DMAE_CMD_ERROR_HANDLING_ENUM +}; + + +enum dmae_cmd_src_enum { + dmae_cmd_src_pcie /* The source is the PCIe */, + dmae_cmd_src_grc /* The source is the GRC */, + MAX_DMAE_CMD_SRC_ENUM +}; + + +struct e4_mstorm_core_conn_ag_ctx { + u8 byte0 /* cdu_validation */; + u8 byte1 /* state */; + u8 flags0; +#define E4_MSTORM_CORE_CONN_AG_CTX_BIT0_MASK 0x1 /* exist_in_qm0 */ +#define E4_MSTORM_CORE_CONN_AG_CTX_BIT0_SHIFT 0 +#define E4_MSTORM_CORE_CONN_AG_CTX_BIT1_MASK 0x1 /* exist_in_qm1 */ +#define E4_MSTORM_CORE_CONN_AG_CTX_BIT1_SHIFT 1 +#define E4_MSTORM_CORE_CONN_AG_CTX_CF0_MASK 0x3 /* cf0 */ +#define E4_MSTORM_CORE_CONN_AG_CTX_CF0_SHIFT 2 +#define E4_MSTORM_CORE_CONN_AG_CTX_CF1_MASK 0x3 /* cf1 */ +#define E4_MSTORM_CORE_CONN_AG_CTX_CF1_SHIFT 4 +#define E4_MSTORM_CORE_CONN_AG_CTX_CF2_MASK 0x3 /* cf2 */ +#define E4_MSTORM_CORE_CONN_AG_CTX_CF2_SHIFT 6 + u8 flags1; +#define E4_MSTORM_CORE_CONN_AG_CTX_CF0EN_MASK 0x1 /* cf0en */ +#define E4_MSTORM_CORE_CONN_AG_CTX_CF0EN_SHIFT 0 +#define E4_MSTORM_CORE_CONN_AG_CTX_CF1EN_MASK 0x1 /* cf1en */ +#define E4_MSTORM_CORE_CONN_AG_CTX_CF1EN_SHIFT 1 +#define E4_MSTORM_CORE_CONN_AG_CTX_CF2EN_MASK 0x1 /* cf2en */ +#define E4_MSTORM_CORE_CONN_AG_CTX_CF2EN_SHIFT 2 +#define E4_MSTORM_CORE_CONN_AG_CTX_RULE0EN_MASK 0x1 /* rule0en */ +#define E4_MSTORM_CORE_CONN_AG_CTX_RULE0EN_SHIFT 3 +#define E4_MSTORM_CORE_CONN_AG_CTX_RULE1EN_MASK 0x1 /* rule1en */ +#define E4_MSTORM_CORE_CONN_AG_CTX_RULE1EN_SHIFT 4 +#define E4_MSTORM_CORE_CONN_AG_CTX_RULE2EN_MASK 0x1 /* rule2en */ +#define E4_MSTORM_CORE_CONN_AG_CTX_RULE2EN_SHIFT 5 +#define E4_MSTORM_CORE_CONN_AG_CTX_RULE3EN_MASK 0x1 /* rule3en */ +#define E4_MSTORM_CORE_CONN_AG_CTX_RULE3EN_SHIFT 6 +#define E4_MSTORM_CORE_CONN_AG_CTX_RULE4EN_MASK 0x1 /* rule4en */ +#define E4_MSTORM_CORE_CONN_AG_CTX_RULE4EN_SHIFT 7 + __le16 word0 /* word0 */; + __le16 word1 /* word1 */; + __le32 reg0 /* reg0 */; + __le32 reg1 /* reg1 */; +}; + + + + + +struct e4_ystorm_core_conn_ag_ctx { + u8 byte0 /* cdu_validation */; + u8 byte1 /* state */; + u8 flags0; +#define E4_YSTORM_CORE_CONN_AG_CTX_BIT0_MASK 0x1 /* exist_in_qm0 */ +#define E4_YSTORM_CORE_CONN_AG_CTX_BIT0_SHIFT 0 +#define E4_YSTORM_CORE_CONN_AG_CTX_BIT1_MASK 0x1 /* exist_in_qm1 */ +#define E4_YSTORM_CORE_CONN_AG_CTX_BIT1_SHIFT 1 +#define E4_YSTORM_CORE_CONN_AG_CTX_CF0_MASK 0x3 /* cf0 */ +#define E4_YSTORM_CORE_CONN_AG_CTX_CF0_SHIFT 2 +#define E4_YSTORM_CORE_CONN_AG_CTX_CF1_MASK 0x3 /* cf1 */ +#define E4_YSTORM_CORE_CONN_AG_CTX_CF1_SHIFT 4 +#define E4_YSTORM_CORE_CONN_AG_CTX_CF2_MASK 0x3 /* cf2 */ +#define E4_YSTORM_CORE_CONN_AG_CTX_CF2_SHIFT 6 + u8 flags1; +#define E4_YSTORM_CORE_CONN_AG_CTX_CF0EN_MASK 0x1 /* cf0en */ +#define E4_YSTORM_CORE_CONN_AG_CTX_CF0EN_SHIFT 0 +#define E4_YSTORM_CORE_CONN_AG_CTX_CF1EN_MASK 0x1 /* cf1en */ +#define E4_YSTORM_CORE_CONN_AG_CTX_CF1EN_SHIFT 1 +#define E4_YSTORM_CORE_CONN_AG_CTX_CF2EN_MASK 0x1 /* cf2en */ +#define E4_YSTORM_CORE_CONN_AG_CTX_CF2EN_SHIFT 2 +#define E4_YSTORM_CORE_CONN_AG_CTX_RULE0EN_MASK 0x1 /* rule0en */ +#define E4_YSTORM_CORE_CONN_AG_CTX_RULE0EN_SHIFT 3 +#define E4_YSTORM_CORE_CONN_AG_CTX_RULE1EN_MASK 0x1 /* rule1en */ +#define E4_YSTORM_CORE_CONN_AG_CTX_RULE1EN_SHIFT 4 +#define E4_YSTORM_CORE_CONN_AG_CTX_RULE2EN_MASK 0x1 /* rule2en */ +#define E4_YSTORM_CORE_CONN_AG_CTX_RULE2EN_SHIFT 5 +#define E4_YSTORM_CORE_CONN_AG_CTX_RULE3EN_MASK 0x1 /* rule3en */ +#define E4_YSTORM_CORE_CONN_AG_CTX_RULE3EN_SHIFT 6 +#define E4_YSTORM_CORE_CONN_AG_CTX_RULE4EN_MASK 0x1 /* rule4en */ +#define E4_YSTORM_CORE_CONN_AG_CTX_RULE4EN_SHIFT 7 + u8 byte2 /* byte2 */; + u8 byte3 /* byte3 */; + __le16 word0 /* word0 */; + __le32 reg0 /* reg0 */; + __le32 reg1 /* reg1 */; + __le16 word1 /* word1 */; + __le16 word2 /* word2 */; + __le16 word3 /* word3 */; + __le16 word4 /* word4 */; + __le32 reg2 /* reg2 */; + __le32 reg3 /* reg3 */; +}; + + /* * IGU cleanup command */ @@ -1605,15 +2137,18 @@ struct igu_cleanup { __le32 sb_id_and_flags; #define IGU_CLEANUP_RESERVED0_MASK 0x7FFFFFF #define IGU_CLEANUP_RESERVED0_SHIFT 0 +/* cleanup clear - 0, set - 1 */ #define IGU_CLEANUP_CLEANUP_SET_MASK 0x1 #define IGU_CLEANUP_CLEANUP_SET_SHIFT 27 #define IGU_CLEANUP_CLEANUP_TYPE_MASK 0x7 #define IGU_CLEANUP_CLEANUP_TYPE_SHIFT 28 +/* must always be set (use enum command_type_bit) */ #define IGU_CLEANUP_COMMAND_TYPE_MASK 0x1 #define IGU_CLEANUP_COMMAND_TYPE_SHIFT 31 __le32 reserved1; }; + /* * IGU firmware driver command */ @@ -1622,6 +2157,7 @@ union igu_command { struct igu_cleanup cleanup; }; + /* * IGU firmware driver command */ @@ -1632,10 +2168,12 @@ struct igu_command_reg_ctrl { #define IGU_COMMAND_REG_CTRL_PXP_BAR_ADDR_SHIFT 0 #define IGU_COMMAND_REG_CTRL_RESERVED_MASK 0x7 #define IGU_COMMAND_REG_CTRL_RESERVED_SHIFT 12 +/* command typ: 0 - read, 1 - write */ #define IGU_COMMAND_REG_CTRL_COMMAND_TYPE_MASK 0x1 #define IGU_COMMAND_REG_CTRL_COMMAND_TYPE_SHIFT 15 }; + /* * IGU mapping line structure */ @@ -1645,9 +2183,10 @@ struct igu_mapping_line { #define IGU_MAPPING_LINE_VALID_SHIFT 0 #define IGU_MAPPING_LINE_VECTOR_NUMBER_MASK 0xFF #define IGU_MAPPING_LINE_VECTOR_NUMBER_SHIFT 1 +/* In BB: VF-0-120, PF-0-7; In K2: VF-0-191, PF-0-15 */ #define IGU_MAPPING_LINE_FUNCTION_NUMBER_MASK 0xFF #define IGU_MAPPING_LINE_FUNCTION_NUMBER_SHIFT 9 -#define IGU_MAPPING_LINE_PF_VALID_MASK 0x1 +#define IGU_MAPPING_LINE_PF_VALID_MASK 0x1 /* PF-1, VF-0 */ #define IGU_MAPPING_LINE_PF_VALID_SHIFT 17 #define IGU_MAPPING_LINE_IPS_GROUP_MASK 0x3F #define IGU_MAPPING_LINE_IPS_GROUP_SHIFT 18 @@ -1655,6 +2194,7 @@ struct igu_mapping_line { #define IGU_MAPPING_LINE_RESERVED_SHIFT 24 }; + /* * IGU MSIX line structure */ @@ -1672,102 +2212,48 @@ struct igu_msix_vector { #define IGU_MSIX_VECTOR_RESERVED1_SHIFT 24 }; -enum init_modes { - MODE_BB_A0, - MODE_BB_B0, - MODE_K2, - MODE_ASIC, - MODE_EMUL_REDUCED, - MODE_EMUL_FULL, - MODE_FPGA, - MODE_CHIPSIM, - MODE_SF, - MODE_MF_SD, - MODE_MF_SI, - MODE_PORTS_PER_ENG_1, - MODE_PORTS_PER_ENG_2, - MODE_PORTS_PER_ENG_4, - MODE_100G, - MODE_EAGLE_ENG1_WORKAROUND, - MAX_INIT_MODES -}; - -enum init_phases { - PHASE_ENGINE, - PHASE_PORT, - PHASE_PF, - PHASE_VF, - PHASE_QM_PF, - MAX_INIT_PHASES -}; - -struct mstorm_core_conn_ag_ctx { - u8 byte0 /* cdu_validation */; - u8 byte1 /* state */; - u8 flags0; -#define MSTORM_CORE_CONN_AG_CTX_BIT0_MASK 0x1 -#define MSTORM_CORE_CONN_AG_CTX_BIT0_SHIFT 0 -#define MSTORM_CORE_CONN_AG_CTX_BIT1_MASK 0x1 -#define MSTORM_CORE_CONN_AG_CTX_BIT1_SHIFT 1 -#define MSTORM_CORE_CONN_AG_CTX_CF0_MASK 0x3 -#define MSTORM_CORE_CONN_AG_CTX_CF0_SHIFT 2 -#define MSTORM_CORE_CONN_AG_CTX_CF1_MASK 0x3 -#define MSTORM_CORE_CONN_AG_CTX_CF1_SHIFT 4 -#define MSTORM_CORE_CONN_AG_CTX_CF2_MASK 0x3 -#define MSTORM_CORE_CONN_AG_CTX_CF2_SHIFT 6 - u8 flags1; -#define MSTORM_CORE_CONN_AG_CTX_CF0EN_MASK 0x1 -#define MSTORM_CORE_CONN_AG_CTX_CF0EN_SHIFT 0 -#define MSTORM_CORE_CONN_AG_CTX_CF1EN_MASK 0x1 -#define MSTORM_CORE_CONN_AG_CTX_CF1EN_SHIFT 1 -#define MSTORM_CORE_CONN_AG_CTX_CF2EN_MASK 0x1 -#define MSTORM_CORE_CONN_AG_CTX_CF2EN_SHIFT 2 -#define MSTORM_CORE_CONN_AG_CTX_RULE0EN_MASK 0x1 -#define MSTORM_CORE_CONN_AG_CTX_RULE0EN_SHIFT 3 -#define MSTORM_CORE_CONN_AG_CTX_RULE1EN_MASK 0x1 -#define MSTORM_CORE_CONN_AG_CTX_RULE1EN_SHIFT 4 -#define MSTORM_CORE_CONN_AG_CTX_RULE2EN_MASK 0x1 -#define MSTORM_CORE_CONN_AG_CTX_RULE2EN_SHIFT 5 -#define MSTORM_CORE_CONN_AG_CTX_RULE3EN_MASK 0x1 -#define MSTORM_CORE_CONN_AG_CTX_RULE3EN_SHIFT 6 -#define MSTORM_CORE_CONN_AG_CTX_RULE4EN_MASK 0x1 -#define MSTORM_CORE_CONN_AG_CTX_RULE4EN_SHIFT 7 - __le16 word0 /* word0 */; - __le16 word1 /* word1 */; - __le32 reg0 /* reg0 */; - __le32 reg1 /* reg1 */; -}; /* * per encapsulation type enabling flags */ struct prs_reg_encapsulation_type_en { u8 flags; +/* Enable bit for Ethernet-over-GRE (L2 GRE) encapsulation. */ #define PRS_REG_ENCAPSULATION_TYPE_EN_ETH_OVER_GRE_ENABLE_MASK 0x1 #define PRS_REG_ENCAPSULATION_TYPE_EN_ETH_OVER_GRE_ENABLE_SHIFT 0 +/* Enable bit for IP-over-GRE (IP GRE) encapsulation. */ #define PRS_REG_ENCAPSULATION_TYPE_EN_IP_OVER_GRE_ENABLE_MASK 0x1 #define PRS_REG_ENCAPSULATION_TYPE_EN_IP_OVER_GRE_ENABLE_SHIFT 1 +/* Enable bit for VXLAN encapsulation. */ #define PRS_REG_ENCAPSULATION_TYPE_EN_VXLAN_ENABLE_MASK 0x1 #define PRS_REG_ENCAPSULATION_TYPE_EN_VXLAN_ENABLE_SHIFT 2 +/* Enable bit for T-Tag encapsulation. */ #define PRS_REG_ENCAPSULATION_TYPE_EN_T_TAG_ENABLE_MASK 0x1 #define PRS_REG_ENCAPSULATION_TYPE_EN_T_TAG_ENABLE_SHIFT 3 +/* Enable bit for Ethernet-over-GENEVE (L2 GENEVE) encapsulation. */ #define PRS_REG_ENCAPSULATION_TYPE_EN_ETH_OVER_GENEVE_ENABLE_MASK 0x1 #define PRS_REG_ENCAPSULATION_TYPE_EN_ETH_OVER_GENEVE_ENABLE_SHIFT 4 +/* Enable bit for IP-over-GENEVE (IP GENEVE) encapsulation. */ #define PRS_REG_ENCAPSULATION_TYPE_EN_IP_OVER_GENEVE_ENABLE_MASK 0x1 #define PRS_REG_ENCAPSULATION_TYPE_EN_IP_OVER_GENEVE_ENABLE_SHIFT 5 #define PRS_REG_ENCAPSULATION_TYPE_EN_RESERVED_MASK 0x3 #define PRS_REG_ENCAPSULATION_TYPE_EN_RESERVED_SHIFT 6 }; + enum pxp_tph_st_hint { TPH_ST_HINT_BIDIR /* Read/Write access by Host and Device */, TPH_ST_HINT_REQUESTER /* Read/Write access by Device */, - TPH_ST_HINT_TARGET - /* Device Write and Host Read, or Host Write and Device Read */, +/* Device Write and Host Read, or Host Write and Device Read */ + TPH_ST_HINT_TARGET, +/* Device Write and Host Read, or Host Write and Device Read - with temporal + * reuse + */ TPH_ST_HINT_TARGET_PRIO, MAX_PXP_TPH_ST_HINT }; + /* * QM hardware structure of enable bypass credit mask */ @@ -1791,6 +2277,7 @@ struct qm_rf_bypass_mask { #define QM_RF_BYPASS_MASK_RESERVED1_SHIFT 7 }; + /* * QM hardware structure of opportunistic credit mask */ @@ -1818,50 +2305,58 @@ struct qm_rf_opportunistic_mask { #define QM_RF_OPPORTUNISTIC_MASK_RESERVED1_SHIFT 9 }; + /* * QM hardware structure of QM map memory */ -struct qm_rf_pq_map { +struct qm_rf_pq_map_e4 { __le32 reg; -#define QM_RF_PQ_MAP_PQ_VALID_MASK 0x1 -#define QM_RF_PQ_MAP_PQ_VALID_SHIFT 0 -#define QM_RF_PQ_MAP_RL_ID_MASK 0xFF -#define QM_RF_PQ_MAP_RL_ID_SHIFT 1 -#define QM_RF_PQ_MAP_VP_PQ_ID_MASK 0x1FF -#define QM_RF_PQ_MAP_VP_PQ_ID_SHIFT 9 -#define QM_RF_PQ_MAP_VOQ_MASK 0x1F -#define QM_RF_PQ_MAP_VOQ_SHIFT 18 -#define QM_RF_PQ_MAP_WRR_WEIGHT_GROUP_MASK 0x3 -#define QM_RF_PQ_MAP_WRR_WEIGHT_GROUP_SHIFT 23 -#define QM_RF_PQ_MAP_RL_VALID_MASK 0x1 -#define QM_RF_PQ_MAP_RL_VALID_SHIFT 25 -#define QM_RF_PQ_MAP_RESERVED_MASK 0x3F -#define QM_RF_PQ_MAP_RESERVED_SHIFT 26 +#define QM_RF_PQ_MAP_E4_PQ_VALID_MASK 0x1 /* PQ active */ +#define QM_RF_PQ_MAP_E4_PQ_VALID_SHIFT 0 +#define QM_RF_PQ_MAP_E4_RL_ID_MASK 0xFF /* RL ID */ +#define QM_RF_PQ_MAP_E4_RL_ID_SHIFT 1 +/* the first PQ associated with the VPORT and VOQ of this PQ */ +#define QM_RF_PQ_MAP_E4_VP_PQ_ID_MASK 0x1FF +#define QM_RF_PQ_MAP_E4_VP_PQ_ID_SHIFT 9 +#define QM_RF_PQ_MAP_E4_VOQ_MASK 0x1F /* VOQ */ +#define QM_RF_PQ_MAP_E4_VOQ_SHIFT 18 +#define QM_RF_PQ_MAP_E4_WRR_WEIGHT_GROUP_MASK 0x3 /* WRR weight */ +#define QM_RF_PQ_MAP_E4_WRR_WEIGHT_GROUP_SHIFT 23 +#define QM_RF_PQ_MAP_E4_RL_VALID_MASK 0x1 /* RL active */ +#define QM_RF_PQ_MAP_E4_RL_VALID_SHIFT 25 +#define QM_RF_PQ_MAP_E4_RESERVED_MASK 0x3F +#define QM_RF_PQ_MAP_E4_RESERVED_SHIFT 26 }; + /* * Completion params for aggregated interrupt completion */ struct sdm_agg_int_comp_params { __le16 params; +/* the number of aggregated interrupt, 0-31 */ #define SDM_AGG_INT_COMP_PARAMS_AGG_INT_INDEX_MASK 0x3F #define SDM_AGG_INT_COMP_PARAMS_AGG_INT_INDEX_SHIFT 0 +/* 1 - set a bit in aggregated vector, 0 - dont set */ #define SDM_AGG_INT_COMP_PARAMS_AGG_VECTOR_ENABLE_MASK 0x1 #define SDM_AGG_INT_COMP_PARAMS_AGG_VECTOR_ENABLE_SHIFT 6 +/* Number of bit in the aggregated vector, 0-279 (TBD) */ #define SDM_AGG_INT_COMP_PARAMS_AGG_VECTOR_BIT_MASK 0x1FF #define SDM_AGG_INT_COMP_PARAMS_AGG_VECTOR_BIT_SHIFT 7 }; + /* * SDM operation gen command (generate aggregative interrupt) */ struct sdm_op_gen { __le32 command; +/* completion parameters 0-15 */ #define SDM_OP_GEN_COMP_PARAM_MASK 0xFFFF #define SDM_OP_GEN_COMP_PARAM_SHIFT 0 -#define SDM_OP_GEN_COMP_TYPE_MASK 0xF +#define SDM_OP_GEN_COMP_TYPE_MASK 0xF /* completion type 16-19 */ #define SDM_OP_GEN_COMP_TYPE_SHIFT 16 -#define SDM_OP_GEN_RESERVED_MASK 0xFFF +#define SDM_OP_GEN_RESERVED_MASK 0xFFF /* reserved 20-31 */ #define SDM_OP_GEN_RESERVED_SHIFT 20 }; @@ -1869,32 +2364,32 @@ struct ystorm_core_conn_ag_ctx { u8 byte0 /* cdu_validation */; u8 byte1 /* state */; u8 flags0; -#define YSTORM_CORE_CONN_AG_CTX_BIT0_MASK 0x1 +#define YSTORM_CORE_CONN_AG_CTX_BIT0_MASK 0x1 /* exist_in_qm0 */ #define YSTORM_CORE_CONN_AG_CTX_BIT0_SHIFT 0 -#define YSTORM_CORE_CONN_AG_CTX_BIT1_MASK 0x1 +#define YSTORM_CORE_CONN_AG_CTX_BIT1_MASK 0x1 /* exist_in_qm1 */ #define YSTORM_CORE_CONN_AG_CTX_BIT1_SHIFT 1 -#define YSTORM_CORE_CONN_AG_CTX_CF0_MASK 0x3 +#define YSTORM_CORE_CONN_AG_CTX_CF0_MASK 0x3 /* cf0 */ #define YSTORM_CORE_CONN_AG_CTX_CF0_SHIFT 2 -#define YSTORM_CORE_CONN_AG_CTX_CF1_MASK 0x3 +#define YSTORM_CORE_CONN_AG_CTX_CF1_MASK 0x3 /* cf1 */ #define YSTORM_CORE_CONN_AG_CTX_CF1_SHIFT 4 -#define YSTORM_CORE_CONN_AG_CTX_CF2_MASK 0x3 +#define YSTORM_CORE_CONN_AG_CTX_CF2_MASK 0x3 /* cf2 */ #define YSTORM_CORE_CONN_AG_CTX_CF2_SHIFT 6 u8 flags1; -#define YSTORM_CORE_CONN_AG_CTX_CF0EN_MASK 0x1 +#define YSTORM_CORE_CONN_AG_CTX_CF0EN_MASK 0x1 /* cf0en */ #define YSTORM_CORE_CONN_AG_CTX_CF0EN_SHIFT 0 -#define YSTORM_CORE_CONN_AG_CTX_CF1EN_MASK 0x1 +#define YSTORM_CORE_CONN_AG_CTX_CF1EN_MASK 0x1 /* cf1en */ #define YSTORM_CORE_CONN_AG_CTX_CF1EN_SHIFT 1 -#define YSTORM_CORE_CONN_AG_CTX_CF2EN_MASK 0x1 +#define YSTORM_CORE_CONN_AG_CTX_CF2EN_MASK 0x1 /* cf2en */ #define YSTORM_CORE_CONN_AG_CTX_CF2EN_SHIFT 2 -#define YSTORM_CORE_CONN_AG_CTX_RULE0EN_MASK 0x1 +#define YSTORM_CORE_CONN_AG_CTX_RULE0EN_MASK 0x1 /* rule0en */ #define YSTORM_CORE_CONN_AG_CTX_RULE0EN_SHIFT 3 -#define YSTORM_CORE_CONN_AG_CTX_RULE1EN_MASK 0x1 +#define YSTORM_CORE_CONN_AG_CTX_RULE1EN_MASK 0x1 /* rule1en */ #define YSTORM_CORE_CONN_AG_CTX_RULE1EN_SHIFT 4 -#define YSTORM_CORE_CONN_AG_CTX_RULE2EN_MASK 0x1 +#define YSTORM_CORE_CONN_AG_CTX_RULE2EN_MASK 0x1 /* rule2en */ #define YSTORM_CORE_CONN_AG_CTX_RULE2EN_SHIFT 5 -#define YSTORM_CORE_CONN_AG_CTX_RULE3EN_MASK 0x1 +#define YSTORM_CORE_CONN_AG_CTX_RULE3EN_MASK 0x1 /* rule3en */ #define YSTORM_CORE_CONN_AG_CTX_RULE3EN_SHIFT 6 -#define YSTORM_CORE_CONN_AG_CTX_RULE4EN_MASK 0x1 +#define YSTORM_CORE_CONN_AG_CTX_RULE4EN_MASK 0x1 /* rule4en */ #define YSTORM_CORE_CONN_AG_CTX_RULE4EN_SHIFT 7 u8 byte2 /* byte2 */; u8 byte3 /* byte3 */; diff --git a/dpdk/drivers/net/qede/base/ecore_hsi_debug_tools.h b/dpdk/drivers/net/qede/base/ecore_hsi_debug_tools.h new file mode 100644 index 00000000..ebb66482 --- /dev/null +++ b/dpdk/drivers/net/qede/base/ecore_hsi_debug_tools.h @@ -0,0 +1,1116 @@ +/* + * Copyright (c) 2016 QLogic Corporation. + * All rights reserved. + * www.qlogic.com + * + * See LICENSE.qede_pmd for copyright and licensing details. + */ + +#ifndef __ECORE_HSI_DEBUG_TOOLS__ +#define __ECORE_HSI_DEBUG_TOOLS__ +/****************************************/ +/* Debug Tools HSI constants and macros */ +/****************************************/ + + +enum block_addr { + GRCBASE_GRC = 0x50000, + GRCBASE_MISCS = 0x9000, + GRCBASE_MISC = 0x8000, + GRCBASE_DBU = 0xa000, + GRCBASE_PGLUE_B = 0x2a8000, + GRCBASE_CNIG = 0x218000, + GRCBASE_CPMU = 0x30000, + GRCBASE_NCSI = 0x40000, + GRCBASE_OPTE = 0x53000, + GRCBASE_BMB = 0x540000, + GRCBASE_PCIE = 0x54000, + GRCBASE_MCP = 0xe00000, + GRCBASE_MCP2 = 0x52000, + GRCBASE_PSWHST = 0x2a0000, + GRCBASE_PSWHST2 = 0x29e000, + GRCBASE_PSWRD = 0x29c000, + GRCBASE_PSWRD2 = 0x29d000, + GRCBASE_PSWWR = 0x29a000, + GRCBASE_PSWWR2 = 0x29b000, + GRCBASE_PSWRQ = 0x280000, + GRCBASE_PSWRQ2 = 0x240000, + GRCBASE_PGLCS = 0x0, + GRCBASE_DMAE = 0xc000, + GRCBASE_PTU = 0x560000, + GRCBASE_TCM = 0x1180000, + GRCBASE_MCM = 0x1200000, + GRCBASE_UCM = 0x1280000, + GRCBASE_XCM = 0x1000000, + GRCBASE_YCM = 0x1080000, + GRCBASE_PCM = 0x1100000, + GRCBASE_QM = 0x2f0000, + GRCBASE_TM = 0x2c0000, + GRCBASE_DORQ = 0x100000, + GRCBASE_BRB = 0x340000, + GRCBASE_SRC = 0x238000, + GRCBASE_PRS = 0x1f0000, + GRCBASE_TSDM = 0xfb0000, + GRCBASE_MSDM = 0xfc0000, + GRCBASE_USDM = 0xfd0000, + GRCBASE_XSDM = 0xf80000, + GRCBASE_YSDM = 0xf90000, + GRCBASE_PSDM = 0xfa0000, + GRCBASE_TSEM = 0x1700000, + GRCBASE_MSEM = 0x1800000, + GRCBASE_USEM = 0x1900000, + GRCBASE_XSEM = 0x1400000, + GRCBASE_YSEM = 0x1500000, + GRCBASE_PSEM = 0x1600000, + GRCBASE_RSS = 0x238800, + GRCBASE_TMLD = 0x4d0000, + GRCBASE_MULD = 0x4e0000, + GRCBASE_YULD = 0x4c8000, + GRCBASE_XYLD = 0x4c0000, + GRCBASE_PTLD = 0x590000, + GRCBASE_YPLD = 0x5b0000, + GRCBASE_PRM = 0x230000, + GRCBASE_PBF_PB1 = 0xda0000, + GRCBASE_PBF_PB2 = 0xda4000, + GRCBASE_RPB = 0x23c000, + GRCBASE_BTB = 0xdb0000, + GRCBASE_PBF = 0xd80000, + GRCBASE_RDIF = 0x300000, + GRCBASE_TDIF = 0x310000, + GRCBASE_CDU = 0x580000, + GRCBASE_CCFC = 0x2e0000, + GRCBASE_TCFC = 0x2d0000, + GRCBASE_IGU = 0x180000, + GRCBASE_CAU = 0x1c0000, + GRCBASE_RGFS = 0xf00000, + GRCBASE_RGSRC = 0x320000, + GRCBASE_TGFS = 0xd00000, + GRCBASE_TGSRC = 0x322000, + GRCBASE_UMAC = 0x51000, + GRCBASE_XMAC = 0x210000, + GRCBASE_DBG = 0x10000, + GRCBASE_NIG = 0x500000, + GRCBASE_WOL = 0x600000, + GRCBASE_BMBN = 0x610000, + GRCBASE_IPC = 0x20000, + GRCBASE_NWM = 0x800000, + GRCBASE_NWS = 0x700000, + GRCBASE_MS = 0x6a0000, + GRCBASE_PHY_PCIE = 0x620000, + GRCBASE_LED = 0x6b8000, + GRCBASE_AVS_WRAP = 0x6b0000, + GRCBASE_MISC_AEU = 0x8000, + GRCBASE_BAR0_MAP = 0x1c00000, + MAX_BLOCK_ADDR +}; + + +enum block_id { + BLOCK_GRC, + BLOCK_MISCS, + BLOCK_MISC, + BLOCK_DBU, + BLOCK_PGLUE_B, + BLOCK_CNIG, + BLOCK_CPMU, + BLOCK_NCSI, + BLOCK_OPTE, + BLOCK_BMB, + BLOCK_PCIE, + BLOCK_MCP, + BLOCK_MCP2, + BLOCK_PSWHST, + BLOCK_PSWHST2, + BLOCK_PSWRD, + BLOCK_PSWRD2, + BLOCK_PSWWR, + BLOCK_PSWWR2, + BLOCK_PSWRQ, + BLOCK_PSWRQ2, + BLOCK_PGLCS, + BLOCK_DMAE, + BLOCK_PTU, + BLOCK_TCM, + BLOCK_MCM, + BLOCK_UCM, + BLOCK_XCM, + BLOCK_YCM, + BLOCK_PCM, + BLOCK_QM, + BLOCK_TM, + BLOCK_DORQ, + BLOCK_BRB, + BLOCK_SRC, + BLOCK_PRS, + BLOCK_TSDM, + BLOCK_MSDM, + BLOCK_USDM, + BLOCK_XSDM, + BLOCK_YSDM, + BLOCK_PSDM, + BLOCK_TSEM, + BLOCK_MSEM, + BLOCK_USEM, + BLOCK_XSEM, + BLOCK_YSEM, + BLOCK_PSEM, + BLOCK_RSS, + BLOCK_TMLD, + BLOCK_MULD, + BLOCK_YULD, + BLOCK_XYLD, + BLOCK_PTLD, + BLOCK_YPLD, + BLOCK_PRM, + BLOCK_PBF_PB1, + BLOCK_PBF_PB2, + BLOCK_RPB, + BLOCK_BTB, + BLOCK_PBF, + BLOCK_RDIF, + BLOCK_TDIF, + BLOCK_CDU, + BLOCK_CCFC, + BLOCK_TCFC, + BLOCK_IGU, + BLOCK_CAU, + BLOCK_RGFS, + BLOCK_RGSRC, + BLOCK_TGFS, + BLOCK_TGSRC, + BLOCK_UMAC, + BLOCK_XMAC, + BLOCK_DBG, + BLOCK_NIG, + BLOCK_WOL, + BLOCK_BMBN, + BLOCK_IPC, + BLOCK_NWM, + BLOCK_NWS, + BLOCK_MS, + BLOCK_PHY_PCIE, + BLOCK_LED, + BLOCK_AVS_WRAP, + BLOCK_MISC_AEU, + BLOCK_BAR0_MAP, + MAX_BLOCK_ID +}; + + +/* + * binary debug buffer types + */ +enum bin_dbg_buffer_type { + BIN_BUF_DBG_MODE_TREE /* init modes tree */, + BIN_BUF_DBG_DUMP_REG /* GRC Dump registers */, + BIN_BUF_DBG_DUMP_MEM /* GRC Dump memories */, + BIN_BUF_DBG_IDLE_CHK_REGS /* Idle Check registers */, + BIN_BUF_DBG_IDLE_CHK_IMMS /* Idle Check immediates */, + BIN_BUF_DBG_IDLE_CHK_RULES /* Idle Check rules */, + BIN_BUF_DBG_IDLE_CHK_PARSING_DATA /* Idle Check parsing data */, + BIN_BUF_DBG_ATTN_BLOCKS /* Attention blocks */, + BIN_BUF_DBG_ATTN_REGS /* Attention registers */, + BIN_BUF_DBG_ATTN_INDEXES /* Attention indexes */, + BIN_BUF_DBG_ATTN_NAME_OFFSETS /* Attention name offsets */, + BIN_BUF_DBG_BUS_BLOCKS /* Debug Bus blocks */, + BIN_BUF_DBG_BUS_LINES /* Debug Bus lines */, + BIN_BUF_DBG_BUS_BLOCKS_USER_DATA /* Debug Bus blocks user data */, + BIN_BUF_DBG_BUS_LINE_NAME_OFFSETS /* Debug Bus line name offsets */, + BIN_BUF_DBG_PARSING_STRINGS /* Debug Tools parsing strings */, + MAX_BIN_DBG_BUFFER_TYPE +}; + + +/* + * Attention bit mapping + */ +struct dbg_attn_bit_mapping { + __le16 data; +/* The index of an attention in the blocks attentions list + * (if is_unused_bit_cnt=0), or a number of consecutive unused attention bits + * (if is_unused_bit_cnt=1) + */ +#define DBG_ATTN_BIT_MAPPING_VAL_MASK 0x7FFF +#define DBG_ATTN_BIT_MAPPING_VAL_SHIFT 0 +/* if set, the val field indicates the number of consecutive unused attention + * bits + */ +#define DBG_ATTN_BIT_MAPPING_IS_UNUSED_BIT_CNT_MASK 0x1 +#define DBG_ATTN_BIT_MAPPING_IS_UNUSED_BIT_CNT_SHIFT 15 +}; + + +/* + * Attention block per-type data + */ +struct dbg_attn_block_type_data { +/* Offset of this block attention names in the debug attention name offsets + * array + */ + __le16 names_offset; + __le16 reserved1; + u8 num_regs /* Number of attention registers in this block */; + u8 reserved2; +/* Offset of this blocks attention registers in the attention registers array + * (in dbg_attn_reg units) + */ + __le16 regs_offset; +}; + +/* + * Block attentions + */ +struct dbg_attn_block { +/* attention block per-type data. Count must match the number of elements in + * dbg_attn_type. + */ + struct dbg_attn_block_type_data per_type_data[2]; +}; + + +/* + * Attention register result + */ +struct dbg_attn_reg_result { + __le32 data; +/* STS attention register GRC address (in dwords) */ +#define DBG_ATTN_REG_RESULT_STS_ADDRESS_MASK 0xFFFFFF +#define DBG_ATTN_REG_RESULT_STS_ADDRESS_SHIFT 0 +/* Number of attention indexes in this register */ +#define DBG_ATTN_REG_RESULT_NUM_REG_ATTN_MASK 0xFF +#define DBG_ATTN_REG_RESULT_NUM_REG_ATTN_SHIFT 24 +/* The offset of this registers attentions within the blocks attentions + * list (a value in the range 0..number of block attentions-1) + */ + __le16 attn_idx_offset; + __le16 reserved; + __le32 sts_val /* Value read from the STS attention register */; + __le32 mask_val /* Value read from the MASK attention register */; +}; + +/* + * Attention block result + */ +struct dbg_attn_block_result { + u8 block_id /* Registers block ID */; + u8 data; +/* Value from dbg_attn_type enum */ +#define DBG_ATTN_BLOCK_RESULT_ATTN_TYPE_MASK 0x3 +#define DBG_ATTN_BLOCK_RESULT_ATTN_TYPE_SHIFT 0 +/* Number of registers in block in which at least one attention bit is set */ +#define DBG_ATTN_BLOCK_RESULT_NUM_REGS_MASK 0x3F +#define DBG_ATTN_BLOCK_RESULT_NUM_REGS_SHIFT 2 +/* Offset of this registers block attention names in the attention name offsets + * array + */ + __le16 names_offset; +/* result data for each register in the block in which at least one attention + * bit is set + */ + struct dbg_attn_reg_result reg_results[15]; +}; + + + +/* + * mode header + */ +struct dbg_mode_hdr { + __le16 data; +/* indicates if a mode expression should be evaluated (0/1) */ +#define DBG_MODE_HDR_EVAL_MODE_MASK 0x1 +#define DBG_MODE_HDR_EVAL_MODE_SHIFT 0 +/* offset (in bytes) in modes expression buffer. valid only if eval_mode is + * set. + */ +#define DBG_MODE_HDR_MODES_BUF_OFFSET_MASK 0x7FFF +#define DBG_MODE_HDR_MODES_BUF_OFFSET_SHIFT 1 +}; + +/* + * Attention register + */ +struct dbg_attn_reg { + struct dbg_mode_hdr mode /* Mode header */; +/* The offset of this registers attentions within the blocks attentions + * list (a value in the range 0..number of block attentions-1) + */ + __le16 attn_idx_offset; + __le32 data; +/* STS attention register GRC address (in dwords) */ +#define DBG_ATTN_REG_STS_ADDRESS_MASK 0xFFFFFF +#define DBG_ATTN_REG_STS_ADDRESS_SHIFT 0 +/* Number of attention in this register */ +#define DBG_ATTN_REG_NUM_REG_ATTN_MASK 0xFF +#define DBG_ATTN_REG_NUM_REG_ATTN_SHIFT 24 +/* STS_CLR attention register GRC address (in dwords) */ + __le32 sts_clr_address; +/* MASK attention register GRC address (in dwords) */ + __le32 mask_address; +}; + + + +/* + * attention types + */ +enum dbg_attn_type { + ATTN_TYPE_INTERRUPT, + ATTN_TYPE_PARITY, + MAX_DBG_ATTN_TYPE +}; + + +/* + * Debug Bus block data + */ +struct dbg_bus_block { +/* Number of debug lines in this block (excluding signature & latency events) */ + u8 num_of_lines; +/* Indicates if this block has a latency events debug line (0/1). */ + u8 has_latency_events; +/* Offset of this blocks lines in the Debug Bus lines array. */ + __le16 lines_offset; +}; + + +/* + * Debug Bus block user data + */ +struct dbg_bus_block_user_data { +/* Number of debug lines in this block (excluding signature & latency events) */ + u8 num_of_lines; +/* Indicates if this block has a latency events debug line (0/1). */ + u8 has_latency_events; +/* Offset of this blocks lines in the debug bus line name offsets array. */ + __le16 names_offset; +}; + + +/* + * Block Debug line data + */ +struct dbg_bus_line { + u8 data; +/* Number of groups in the line (0-3) */ +#define DBG_BUS_LINE_NUM_OF_GROUPS_MASK 0xF +#define DBG_BUS_LINE_NUM_OF_GROUPS_SHIFT 0 +/* Indicates if this is a 128b line (0) or a 256b line (1). */ +#define DBG_BUS_LINE_IS_256B_MASK 0x1 +#define DBG_BUS_LINE_IS_256B_SHIFT 4 +#define DBG_BUS_LINE_RESERVED_MASK 0x7 +#define DBG_BUS_LINE_RESERVED_SHIFT 5 +/* Four 2-bit values, indicating the size of each group minus 1 (i.e. + * value=0 means size=1, value=1 means size=2, etc), starting from lsb. + * The sizes are in dwords (if is_256b=0) or in qwords (if is_256b=1). + */ + u8 group_sizes; +}; + + +/* + * condition header for registers dump + */ +struct dbg_dump_cond_hdr { + struct dbg_mode_hdr mode /* Mode header */; + u8 block_id /* block ID */; + u8 data_size /* size in dwords of the data following this header */; +}; + + +/* + * memory data for registers dump + */ +struct dbg_dump_mem { + __le32 dword0; +/* register address (in dwords) */ +#define DBG_DUMP_MEM_ADDRESS_MASK 0xFFFFFF +#define DBG_DUMP_MEM_ADDRESS_SHIFT 0 +#define DBG_DUMP_MEM_MEM_GROUP_ID_MASK 0xFF /* memory group ID */ +#define DBG_DUMP_MEM_MEM_GROUP_ID_SHIFT 24 + __le32 dword1; +/* register size (in dwords) */ +#define DBG_DUMP_MEM_LENGTH_MASK 0xFFFFFF +#define DBG_DUMP_MEM_LENGTH_SHIFT 0 +/* indicates if the register is wide-bus */ +#define DBG_DUMP_MEM_WIDE_BUS_MASK 0x1 +#define DBG_DUMP_MEM_WIDE_BUS_SHIFT 24 +#define DBG_DUMP_MEM_RESERVED_MASK 0x7F +#define DBG_DUMP_MEM_RESERVED_SHIFT 25 +}; + + +/* + * register data for registers dump + */ +struct dbg_dump_reg { + __le32 data; +/* register address (in dwords) */ +#define DBG_DUMP_REG_ADDRESS_MASK 0x7FFFFF /* register address (in dwords) */ +#define DBG_DUMP_REG_ADDRESS_SHIFT 0 +/* indicates if the register is wide-bus */ +#define DBG_DUMP_REG_WIDE_BUS_MASK 0x1 +#define DBG_DUMP_REG_WIDE_BUS_SHIFT 23 +#define DBG_DUMP_REG_LENGTH_MASK 0xFF /* register size (in dwords) */ +#define DBG_DUMP_REG_LENGTH_SHIFT 24 +}; + + +/* + * split header for registers dump + */ +struct dbg_dump_split_hdr { + __le32 hdr; +/* size in dwords of the data following this header */ +#define DBG_DUMP_SPLIT_HDR_DATA_SIZE_MASK 0xFFFFFF +#define DBG_DUMP_SPLIT_HDR_DATA_SIZE_SHIFT 0 +#define DBG_DUMP_SPLIT_HDR_SPLIT_TYPE_ID_MASK 0xFF /* split type ID */ +#define DBG_DUMP_SPLIT_HDR_SPLIT_TYPE_ID_SHIFT 24 +}; + + +/* + * condition header for idle check + */ +struct dbg_idle_chk_cond_hdr { + struct dbg_mode_hdr mode /* Mode header */; +/* size in dwords of the data following this header */ + __le16 data_size; +}; + + +/* + * Idle Check condition register + */ +struct dbg_idle_chk_cond_reg { + __le32 data; +/* Register GRC address (in dwords) */ +#define DBG_IDLE_CHK_COND_REG_ADDRESS_MASK 0x7FFFFF +#define DBG_IDLE_CHK_COND_REG_ADDRESS_SHIFT 0 +/* indicates if the register is wide-bus */ +#define DBG_IDLE_CHK_COND_REG_WIDE_BUS_MASK 0x1 +#define DBG_IDLE_CHK_COND_REG_WIDE_BUS_SHIFT 23 +/* value from block_id enum */ +#define DBG_IDLE_CHK_COND_REG_BLOCK_ID_MASK 0xFF +#define DBG_IDLE_CHK_COND_REG_BLOCK_ID_SHIFT 24 + __le16 num_entries /* number of registers entries to check */; + u8 entry_size /* size of registers entry (in dwords) */; + u8 start_entry /* index of the first entry to check */; +}; + + +/* + * Idle Check info register + */ +struct dbg_idle_chk_info_reg { + __le32 data; +/* Register GRC address (in dwords) */ +#define DBG_IDLE_CHK_INFO_REG_ADDRESS_MASK 0x7FFFFF +#define DBG_IDLE_CHK_INFO_REG_ADDRESS_SHIFT 0 +/* indicates if the register is wide-bus */ +#define DBG_IDLE_CHK_INFO_REG_WIDE_BUS_MASK 0x1 +#define DBG_IDLE_CHK_INFO_REG_WIDE_BUS_SHIFT 23 +/* value from block_id enum */ +#define DBG_IDLE_CHK_INFO_REG_BLOCK_ID_MASK 0xFF +#define DBG_IDLE_CHK_INFO_REG_BLOCK_ID_SHIFT 24 + __le16 size /* register size in dwords */; + struct dbg_mode_hdr mode /* Mode header */; +}; + + +/* + * Idle Check register + */ +union dbg_idle_chk_reg { + struct dbg_idle_chk_cond_reg cond_reg /* condition register */; + struct dbg_idle_chk_info_reg info_reg /* info register */; +}; + + +/* + * Idle Check result header + */ +struct dbg_idle_chk_result_hdr { + __le16 rule_id /* Failing rule index */; + __le16 mem_entry_id /* Failing memory entry index */; + u8 num_dumped_cond_regs /* number of dumped condition registers */; + u8 num_dumped_info_regs /* number of dumped condition registers */; + u8 severity /* from dbg_idle_chk_severity_types enum */; + u8 reserved; +}; + + +/* + * Idle Check result register header + */ +struct dbg_idle_chk_result_reg_hdr { + u8 data; +/* indicates if this register is a memory */ +#define DBG_IDLE_CHK_RESULT_REG_HDR_IS_MEM_MASK 0x1 +#define DBG_IDLE_CHK_RESULT_REG_HDR_IS_MEM_SHIFT 0 +/* register index within the failing rule */ +#define DBG_IDLE_CHK_RESULT_REG_HDR_REG_ID_MASK 0x7F +#define DBG_IDLE_CHK_RESULT_REG_HDR_REG_ID_SHIFT 1 + u8 start_entry /* index of the first checked entry */; + __le16 size /* register size in dwords */; +}; + + +/* + * Idle Check rule + */ +struct dbg_idle_chk_rule { + __le16 rule_id /* Idle Check rule ID */; + u8 severity /* value from dbg_idle_chk_severity_types enum */; + u8 cond_id /* Condition ID */; + u8 num_cond_regs /* number of condition registers */; + u8 num_info_regs /* number of info registers */; + u8 num_imms /* number of immediates in the condition */; + u8 reserved1; +/* offset of this rules registers in the idle check register array + * (in dbg_idle_chk_reg units) + */ + __le16 reg_offset; +/* offset of this rules immediate values in the immediate values array + * (in dwords) + */ + __le16 imm_offset; +}; + + +/* + * Idle Check rule parsing data + */ +struct dbg_idle_chk_rule_parsing_data { + __le32 data; +/* indicates if this register has a FW message */ +#define DBG_IDLE_CHK_RULE_PARSING_DATA_HAS_FW_MSG_MASK 0x1 +#define DBG_IDLE_CHK_RULE_PARSING_DATA_HAS_FW_MSG_SHIFT 0 +/* Offset of this rules strings in the debug strings array (in bytes) */ +#define DBG_IDLE_CHK_RULE_PARSING_DATA_STR_OFFSET_MASK 0x7FFFFFFF +#define DBG_IDLE_CHK_RULE_PARSING_DATA_STR_OFFSET_SHIFT 1 +}; + + +/* + * idle check severity types + */ +enum dbg_idle_chk_severity_types { +/* idle check failure should cause an error */ + IDLE_CHK_SEVERITY_ERROR, +/* idle check failure should cause an error only if theres no traffic */ + IDLE_CHK_SEVERITY_ERROR_NO_TRAFFIC, +/* idle check failure should cause a warning */ + IDLE_CHK_SEVERITY_WARNING, + MAX_DBG_IDLE_CHK_SEVERITY_TYPES +}; + + + +/* + * Debug Bus block data + */ +struct dbg_bus_block_data { + __le16 data; +/* 4-bit value: bit i set -> dword/qword i is enabled. */ +#define DBG_BUS_BLOCK_DATA_ENABLE_MASK_MASK 0xF +#define DBG_BUS_BLOCK_DATA_ENABLE_MASK_SHIFT 0 +/* Number of dwords/qwords to shift right the debug data (0-3) */ +#define DBG_BUS_BLOCK_DATA_RIGHT_SHIFT_MASK 0xF +#define DBG_BUS_BLOCK_DATA_RIGHT_SHIFT_SHIFT 4 +/* 4-bit value: bit i set -> dword/qword i is forced valid. */ +#define DBG_BUS_BLOCK_DATA_FORCE_VALID_MASK_MASK 0xF +#define DBG_BUS_BLOCK_DATA_FORCE_VALID_MASK_SHIFT 8 +/* 4-bit value: bit i set -> dword/qword i frame bit is forced. */ +#define DBG_BUS_BLOCK_DATA_FORCE_FRAME_MASK_MASK 0xF +#define DBG_BUS_BLOCK_DATA_FORCE_FRAME_MASK_SHIFT 12 + u8 line_num /* Debug line number to select */; + u8 hw_id /* HW ID associated with the block */; +}; + + +/* + * Debug Bus Clients + */ +enum dbg_bus_clients { + DBG_BUS_CLIENT_RBCN, + DBG_BUS_CLIENT_RBCP, + DBG_BUS_CLIENT_RBCR, + DBG_BUS_CLIENT_RBCT, + DBG_BUS_CLIENT_RBCU, + DBG_BUS_CLIENT_RBCF, + DBG_BUS_CLIENT_RBCX, + DBG_BUS_CLIENT_RBCS, + DBG_BUS_CLIENT_RBCH, + DBG_BUS_CLIENT_RBCZ, + DBG_BUS_CLIENT_OTHER_ENGINE, + DBG_BUS_CLIENT_TIMESTAMP, + DBG_BUS_CLIENT_CPU, + DBG_BUS_CLIENT_RBCY, + DBG_BUS_CLIENT_RBCQ, + DBG_BUS_CLIENT_RBCM, + DBG_BUS_CLIENT_RBCB, + DBG_BUS_CLIENT_RBCW, + DBG_BUS_CLIENT_RBCV, + MAX_DBG_BUS_CLIENTS +}; + + +/* + * Debug Bus constraint operation types + */ +enum dbg_bus_constraint_ops { + DBG_BUS_CONSTRAINT_OP_EQ /* equal */, + DBG_BUS_CONSTRAINT_OP_NE /* not equal */, + DBG_BUS_CONSTRAINT_OP_LT /* less than */, + DBG_BUS_CONSTRAINT_OP_LTC /* less than (cyclic) */, + DBG_BUS_CONSTRAINT_OP_LE /* less than or equal */, + DBG_BUS_CONSTRAINT_OP_LEC /* less than or equal (cyclic) */, + DBG_BUS_CONSTRAINT_OP_GT /* greater than */, + DBG_BUS_CONSTRAINT_OP_GTC /* greater than (cyclic) */, + DBG_BUS_CONSTRAINT_OP_GE /* greater than or equal */, + DBG_BUS_CONSTRAINT_OP_GEC /* greater than or equal (cyclic) */, + MAX_DBG_BUS_CONSTRAINT_OPS +}; + + +/* + * Debug Bus trigger state data + */ +struct dbg_bus_trigger_state_data { + u8 data; +/* 4-bit value: bit i set -> dword i of the trigger state block + * (after right shift) is enabled. + */ +#define DBG_BUS_TRIGGER_STATE_DATA_BLOCK_SHIFTED_ENABLE_MASK_MASK 0xF +#define DBG_BUS_TRIGGER_STATE_DATA_BLOCK_SHIFTED_ENABLE_MASK_SHIFT 0 +/* 4-bit value: bit i set -> dword i is compared by a constraint */ +#define DBG_BUS_TRIGGER_STATE_DATA_CONSTRAINT_DWORD_MASK_MASK 0xF +#define DBG_BUS_TRIGGER_STATE_DATA_CONSTRAINT_DWORD_MASK_SHIFT 4 +}; + +/* + * Debug Bus memory address + */ +struct dbg_bus_mem_addr { + __le32 lo; + __le32 hi; +}; + +/* + * Debug Bus PCI buffer data + */ +struct dbg_bus_pci_buf_data { + struct dbg_bus_mem_addr phys_addr /* PCI buffer physical address */; + struct dbg_bus_mem_addr virt_addr /* PCI buffer virtual address */; + __le32 size /* PCI buffer size in bytes */; +}; + +/* + * Debug Bus Storm EID range filter params + */ +struct dbg_bus_storm_eid_range_params { + u8 min /* Minimal event ID to filter on */; + u8 max /* Maximal event ID to filter on */; +}; + +/* + * Debug Bus Storm EID mask filter params + */ +struct dbg_bus_storm_eid_mask_params { + u8 val /* Event ID value */; + u8 mask /* Event ID mask. 1s in the mask = dont care bits. */; +}; + +/* + * Debug Bus Storm EID filter params + */ +union dbg_bus_storm_eid_params { +/* EID range filter params */ + struct dbg_bus_storm_eid_range_params range; +/* EID mask filter params */ + struct dbg_bus_storm_eid_mask_params mask; +}; + +/* + * Debug Bus Storm data + */ +struct dbg_bus_storm_data { + u8 enabled /* indicates if the Storm is enabled for recording */; + u8 mode /* Storm debug mode, valid only if the Storm is enabled */; + u8 hw_id /* HW ID associated with the Storm */; + u8 eid_filter_en /* Indicates if EID filtering is performed (0/1) */; +/* 1 = EID range filter, 0 = EID mask filter. Valid only if eid_filter_en is + * set, + */ + u8 eid_range_not_mask; + u8 cid_filter_en /* Indicates if CID filtering is performed (0/1) */; +/* EID filter params to filter on. Valid only if eid_filter_en is set. */ + union dbg_bus_storm_eid_params eid_filter_params; +/* CID to filter on. Valid only if cid_filter_en is set. */ + __le32 cid; +}; + +/* + * Debug Bus data + */ +struct dbg_bus_data { + __le32 app_version /* The tools version number of the application */; + u8 state /* The current debug bus state */; + u8 hw_dwords /* HW dwords per cycle */; +/* The HW IDs of the recorded HW blocks, where bits i*3..i*3+2 contain the + * HW ID of dword/qword i + */ + __le16 hw_id_mask; + u8 num_enabled_blocks /* Number of blocks enabled for recording */; + u8 num_enabled_storms /* Number of Storms enabled for recording */; + u8 target /* Output target */; + u8 one_shot_en /* Indicates if one-shot mode is enabled (0/1) */; + u8 grc_input_en /* Indicates if GRC recording is enabled (0/1) */; +/* Indicates if timestamp recording is enabled (0/1) */ + u8 timestamp_input_en; + u8 filter_en /* Indicates if the recording filter is enabled (0/1) */; +/* If true, the next added constraint belong to the filter. Otherwise, + * it belongs to the last added trigger state. Valid only if either filter or + * triggers are enabled. + */ + u8 adding_filter; +/* Indicates if the recording filter should be applied before the trigger. + * Valid only if both filter and trigger are enabled (0/1) + */ + u8 filter_pre_trigger; +/* Indicates if the recording filter should be applied after the trigger. + * Valid only if both filter and trigger are enabled (0/1) + */ + u8 filter_post_trigger; + __le16 reserved; +/* Indicates if the recording trigger is enabled (0/1) */ + u8 trigger_en; +/* trigger states data */ + struct dbg_bus_trigger_state_data trigger_states[3]; + u8 next_trigger_state /* ID of next trigger state to be added */; +/* ID of next filter/trigger constraint to be added */ + u8 next_constraint_id; +/* If true, all inputs are associated with HW ID 0. Otherwise, each input is + * assigned a different HW ID (0/1) + */ + u8 unify_inputs; +/* Indicates if the other engine sends it NW recording to this engine (0/1) */ + u8 rcv_from_other_engine; +/* Debug Bus PCI buffer data. Valid only when the target is + * DBG_BUS_TARGET_ID_PCI. + */ + struct dbg_bus_pci_buf_data pci_buf; +/* Debug Bus data for each block */ + struct dbg_bus_block_data blocks[88]; +/* Debug Bus data for each block */ + struct dbg_bus_storm_data storms[6]; +}; + + +/* + * Debug bus filter types + */ +enum dbg_bus_filter_types { + DBG_BUS_FILTER_TYPE_OFF /* filter always off */, + DBG_BUS_FILTER_TYPE_PRE /* filter before trigger only */, + DBG_BUS_FILTER_TYPE_POST /* filter after trigger only */, + DBG_BUS_FILTER_TYPE_ON /* filter always on */, + MAX_DBG_BUS_FILTER_TYPES +}; + + +/* + * Debug bus frame modes + */ +enum dbg_bus_frame_modes { + DBG_BUS_FRAME_MODE_0HW_4ST = 0 /* 0 HW dwords, 4 Storm dwords */, + DBG_BUS_FRAME_MODE_4HW_0ST = 3 /* 4 HW dwords, 0 Storm dwords */, + DBG_BUS_FRAME_MODE_8HW_0ST = 4 /* 8 HW dwords, 0 Storm dwords */, + MAX_DBG_BUS_FRAME_MODES +}; + + +/* + * Debug bus other engine mode + */ +enum dbg_bus_other_engine_modes { + DBG_BUS_OTHER_ENGINE_MODE_NONE, + DBG_BUS_OTHER_ENGINE_MODE_DOUBLE_BW_TX, + DBG_BUS_OTHER_ENGINE_MODE_DOUBLE_BW_RX, + DBG_BUS_OTHER_ENGINE_MODE_CROSS_ENGINE_TX, + DBG_BUS_OTHER_ENGINE_MODE_CROSS_ENGINE_RX, + MAX_DBG_BUS_OTHER_ENGINE_MODES +}; + + + +/* + * Debug bus post-trigger recording types + */ +enum dbg_bus_post_trigger_types { + DBG_BUS_POST_TRIGGER_RECORD /* start recording after trigger */, + DBG_BUS_POST_TRIGGER_DROP /* drop data after trigger */, + MAX_DBG_BUS_POST_TRIGGER_TYPES +}; + + +/* + * Debug bus pre-trigger recording types + */ +enum dbg_bus_pre_trigger_types { + DBG_BUS_PRE_TRIGGER_START_FROM_ZERO /* start recording from time 0 */, +/* start recording some chunks before trigger */ + DBG_BUS_PRE_TRIGGER_NUM_CHUNKS, + DBG_BUS_PRE_TRIGGER_DROP /* drop data before trigger */, + MAX_DBG_BUS_PRE_TRIGGER_TYPES +}; + + +/* + * Debug bus SEMI frame modes + */ +enum dbg_bus_semi_frame_modes { +/* 0 slow dwords, 4 fast dwords */ + DBG_BUS_SEMI_FRAME_MODE_0SLOW_4FAST = 0, +/* 4 slow dwords, 0 fast dwords */ + DBG_BUS_SEMI_FRAME_MODE_4SLOW_0FAST = 3, + MAX_DBG_BUS_SEMI_FRAME_MODES +}; + + +/* + * Debug bus states + */ +enum dbg_bus_states { + DBG_BUS_STATE_IDLE /* debug bus idle state (not recording) */, +/* debug bus is ready for configuration and recording */ + DBG_BUS_STATE_READY, + DBG_BUS_STATE_RECORDING /* debug bus is currently recording */, + DBG_BUS_STATE_STOPPED /* debug bus recording has stopped */, + MAX_DBG_BUS_STATES +}; + + + + + + +/* + * Debug Bus Storm modes + */ +enum dbg_bus_storm_modes { + DBG_BUS_STORM_MODE_PRINTF /* store data (fast debug) */, + DBG_BUS_STORM_MODE_PRAM_ADDR /* pram address (fast debug) */, + DBG_BUS_STORM_MODE_DRA_RW /* DRA read/write data (fast debug) */, + DBG_BUS_STORM_MODE_DRA_W /* DRA write data (fast debug) */, + DBG_BUS_STORM_MODE_LD_ST_ADDR /* load/store address (fast debug) */, + DBG_BUS_STORM_MODE_DRA_FSM /* DRA state machines (fast debug) */, + DBG_BUS_STORM_MODE_RH /* recording handlers (fast debug) */, + DBG_BUS_STORM_MODE_FOC /* FOC: FIN + DRA Rd (slow debug) */, + DBG_BUS_STORM_MODE_EXT_STORE /* FOC: External Store (slow) */, + MAX_DBG_BUS_STORM_MODES +}; + + +/* + * Debug bus target IDs + */ +enum dbg_bus_targets { +/* records debug bus to DBG block internal buffer */ + DBG_BUS_TARGET_ID_INT_BUF, + DBG_BUS_TARGET_ID_NIG /* records debug bus to the NW */, + DBG_BUS_TARGET_ID_PCI /* records debug bus to a PCI buffer */, + MAX_DBG_BUS_TARGETS +}; + + + +/* + * GRC Dump data + */ +struct dbg_grc_data { +/* Indicates if the GRC parameters were initialized */ + u8 params_initialized; + u8 reserved1; + __le16 reserved2; +/* Value of each GRC parameter. Array size must match enum dbg_grc_params. */ + __le32 param_val[48]; +}; + + +/* + * Debug GRC params + */ +enum dbg_grc_params { + DBG_GRC_PARAM_DUMP_TSTORM /* dump Tstorm memories (0/1) */, + DBG_GRC_PARAM_DUMP_MSTORM /* dump Mstorm memories (0/1) */, + DBG_GRC_PARAM_DUMP_USTORM /* dump Ustorm memories (0/1) */, + DBG_GRC_PARAM_DUMP_XSTORM /* dump Xstorm memories (0/1) */, + DBG_GRC_PARAM_DUMP_YSTORM /* dump Ystorm memories (0/1) */, + DBG_GRC_PARAM_DUMP_PSTORM /* dump Pstorm memories (0/1) */, + DBG_GRC_PARAM_DUMP_REGS /* dump non-memory registers (0/1) */, + DBG_GRC_PARAM_DUMP_RAM /* dump Storm internal RAMs (0/1) */, + DBG_GRC_PARAM_DUMP_PBUF /* dump Storm passive buffer (0/1) */, + DBG_GRC_PARAM_DUMP_IOR /* dump Storm IORs (0/1) */, + DBG_GRC_PARAM_DUMP_VFC /* dump VFC memories (0/1) */, + DBG_GRC_PARAM_DUMP_CM_CTX /* dump CM contexts (0/1) */, + DBG_GRC_PARAM_DUMP_PXP /* dump PXP memories (0/1) */, + DBG_GRC_PARAM_DUMP_RSS /* dump RSS memories (0/1) */, + DBG_GRC_PARAM_DUMP_CAU /* dump CAU memories (0/1) */, + DBG_GRC_PARAM_DUMP_QM /* dump QM memories (0/1) */, + DBG_GRC_PARAM_DUMP_MCP /* dump MCP memories (0/1) */, + DBG_GRC_PARAM_RESERVED /* reserved */, + DBG_GRC_PARAM_DUMP_CFC /* dump CFC memories (0/1) */, + DBG_GRC_PARAM_DUMP_IGU /* dump IGU memories (0/1) */, + DBG_GRC_PARAM_DUMP_BRB /* dump BRB memories (0/1) */, + DBG_GRC_PARAM_DUMP_BTB /* dump BTB memories (0/1) */, + DBG_GRC_PARAM_DUMP_BMB /* dump BMB memories (0/1) */, + DBG_GRC_PARAM_DUMP_NIG /* dump NIG memories (0/1) */, + DBG_GRC_PARAM_DUMP_MULD /* dump MULD memories (0/1) */, + DBG_GRC_PARAM_DUMP_PRS /* dump PRS memories (0/1) */, + DBG_GRC_PARAM_DUMP_DMAE /* dump PRS memories (0/1) */, + DBG_GRC_PARAM_DUMP_TM /* dump TM (timers) memories (0/1) */, + DBG_GRC_PARAM_DUMP_SDM /* dump SDM memories (0/1) */, + DBG_GRC_PARAM_DUMP_DIF /* dump DIF memories (0/1) */, + DBG_GRC_PARAM_DUMP_STATIC /* dump static debug data (0/1) */, + DBG_GRC_PARAM_UNSTALL /* un-stall Storms after dump (0/1) */, + DBG_GRC_PARAM_NUM_LCIDS /* number of LCIDs (0..320) */, + DBG_GRC_PARAM_NUM_LTIDS /* number of LTIDs (0..320) */, +/* preset: exclude all memories from dump (1 only) */ + DBG_GRC_PARAM_EXCLUDE_ALL, +/* preset: include memories for crash dump (1 only) */ + DBG_GRC_PARAM_CRASH, +/* perform dump only if MFW is responding (0/1) */ + DBG_GRC_PARAM_PARITY_SAFE, + DBG_GRC_PARAM_DUMP_CM /* dump CM memories (0/1) */, + DBG_GRC_PARAM_DUMP_PHY /* dump PHY memories (0/1) */, + DBG_GRC_PARAM_NO_MCP /* dont perform MCP commands (0/1) */, + DBG_GRC_PARAM_NO_FW_VER /* dont read FW/MFW version (0/1) */, + MAX_DBG_GRC_PARAMS +}; + + +/* + * Debug reset registers + */ +enum dbg_reset_regs { + DBG_RESET_REG_MISCS_PL_UA, + DBG_RESET_REG_MISCS_PL_HV, + DBG_RESET_REG_MISCS_PL_HV_2, + DBG_RESET_REG_MISC_PL_UA, + DBG_RESET_REG_MISC_PL_HV, + DBG_RESET_REG_MISC_PL_PDA_VMAIN_1, + DBG_RESET_REG_MISC_PL_PDA_VMAIN_2, + DBG_RESET_REG_MISC_PL_PDA_VAUX, + MAX_DBG_RESET_REGS +}; + + +/* + * Debug status codes + */ +enum dbg_status { + DBG_STATUS_OK, + DBG_STATUS_APP_VERSION_NOT_SET, + DBG_STATUS_UNSUPPORTED_APP_VERSION, + DBG_STATUS_DBG_BLOCK_NOT_RESET, + DBG_STATUS_INVALID_ARGS, + DBG_STATUS_OUTPUT_ALREADY_SET, + DBG_STATUS_INVALID_PCI_BUF_SIZE, + DBG_STATUS_PCI_BUF_ALLOC_FAILED, + DBG_STATUS_PCI_BUF_NOT_ALLOCATED, + DBG_STATUS_TOO_MANY_INPUTS, + DBG_STATUS_INPUT_OVERLAP, + DBG_STATUS_HW_ONLY_RECORDING, + DBG_STATUS_STORM_ALREADY_ENABLED, + DBG_STATUS_STORM_NOT_ENABLED, + DBG_STATUS_BLOCK_ALREADY_ENABLED, + DBG_STATUS_BLOCK_NOT_ENABLED, + DBG_STATUS_NO_INPUT_ENABLED, + DBG_STATUS_NO_FILTER_TRIGGER_64B, + DBG_STATUS_FILTER_ALREADY_ENABLED, + DBG_STATUS_TRIGGER_ALREADY_ENABLED, + DBG_STATUS_TRIGGER_NOT_ENABLED, + DBG_STATUS_CANT_ADD_CONSTRAINT, + DBG_STATUS_TOO_MANY_TRIGGER_STATES, + DBG_STATUS_TOO_MANY_CONSTRAINTS, + DBG_STATUS_RECORDING_NOT_STARTED, + DBG_STATUS_DATA_DIDNT_TRIGGER, + DBG_STATUS_NO_DATA_RECORDED, + DBG_STATUS_DUMP_BUF_TOO_SMALL, + DBG_STATUS_DUMP_NOT_CHUNK_ALIGNED, + DBG_STATUS_UNKNOWN_CHIP, + DBG_STATUS_VIRT_MEM_ALLOC_FAILED, + DBG_STATUS_BLOCK_IN_RESET, + DBG_STATUS_INVALID_TRACE_SIGNATURE, + DBG_STATUS_INVALID_NVRAM_BUNDLE, + DBG_STATUS_NVRAM_GET_IMAGE_FAILED, + DBG_STATUS_NON_ALIGNED_NVRAM_IMAGE, + DBG_STATUS_NVRAM_READ_FAILED, + DBG_STATUS_IDLE_CHK_PARSE_FAILED, + DBG_STATUS_MCP_TRACE_BAD_DATA, + DBG_STATUS_MCP_TRACE_NO_META, + DBG_STATUS_MCP_COULD_NOT_HALT, + DBG_STATUS_MCP_COULD_NOT_RESUME, + DBG_STATUS_RESERVED2, + DBG_STATUS_SEMI_FIFO_NOT_EMPTY, + DBG_STATUS_IGU_FIFO_BAD_DATA, + DBG_STATUS_MCP_COULD_NOT_MASK_PRTY, + DBG_STATUS_FW_ASSERTS_PARSE_FAILED, + DBG_STATUS_REG_FIFO_BAD_DATA, + DBG_STATUS_PROTECTION_OVERRIDE_BAD_DATA, + DBG_STATUS_DBG_ARRAY_NOT_SET, + DBG_STATUS_FILTER_BUG, + DBG_STATUS_NON_MATCHING_LINES, + DBG_STATUS_INVALID_TRIGGER_DWORD_OFFSET, + DBG_STATUS_DBG_BUS_IN_USE, + MAX_DBG_STATUS +}; + + +/* + * Debug Storms IDs + */ +enum dbg_storms { + DBG_TSTORM_ID, + DBG_MSTORM_ID, + DBG_USTORM_ID, + DBG_XSTORM_ID, + DBG_YSTORM_ID, + DBG_PSTORM_ID, + MAX_DBG_STORMS +}; + + +/* + * Idle Check data + */ +struct idle_chk_data { + __le32 buf_size /* Idle check buffer size in dwords */; +/* Indicates if the idle check buffer size was set (0/1) */ + u8 buf_size_set; + u8 reserved1; + __le16 reserved2; +}; + +/* + * Debug Tools data (per HW function) + */ +struct dbg_tools_data { + struct dbg_grc_data grc /* GRC Dump data */; + struct dbg_bus_data bus /* Debug Bus data */; + struct idle_chk_data idle_chk /* Idle Check data */; + u8 mode_enable[40] /* Indicates if a mode is enabled (0/1) */; +/* Indicates if a block is in reset state (0/1) */ + u8 block_in_reset[88]; + u8 chip_id /* Chip ID (from enum chip_ids) */; + u8 platform_id /* Platform ID */; + u8 initialized /* Indicates if the data was initialized */; + u8 use_dmae /* Indicates if DMAE should be used */; +/* Numbers of registers that were read since last log */ + __le32 num_regs_read; +}; + + +#endif /* __ECORE_HSI_DEBUG_TOOLS__ */ diff --git a/dpdk/drivers/net/qede/base/ecore_hsi_eth.h b/dpdk/drivers/net/qede/base/ecore_hsi_eth.h index 80f4165f..ffbf5c71 100644 --- a/dpdk/drivers/net/qede/base/ecore_hsi_eth.h +++ b/dpdk/drivers/net/qede/base/ecore_hsi_eth.h @@ -34,223 +34,319 @@ struct xstorm_eth_conn_st_ctx { __le32 reserved[60]; }; -struct xstorm_eth_conn_ag_ctx { +struct e4_xstorm_eth_conn_ag_ctx { u8 reserved0 /* cdu_validation */; u8 eth_state /* state */; u8 flags0; -#define XSTORM_ETH_CONN_AG_CTX_EXIST_IN_QM0_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_EXIST_IN_QM0_SHIFT 0 -#define XSTORM_ETH_CONN_AG_CTX_RESERVED1_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_RESERVED1_SHIFT 1 -#define XSTORM_ETH_CONN_AG_CTX_RESERVED2_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_RESERVED2_SHIFT 2 -#define XSTORM_ETH_CONN_AG_CTX_EXIST_IN_QM3_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_EXIST_IN_QM3_SHIFT 3 -#define XSTORM_ETH_CONN_AG_CTX_RESERVED3_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_RESERVED3_SHIFT 4 -#define XSTORM_ETH_CONN_AG_CTX_RESERVED4_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_RESERVED4_SHIFT 5 -#define XSTORM_ETH_CONN_AG_CTX_RESERVED5_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_RESERVED5_SHIFT 6 -#define XSTORM_ETH_CONN_AG_CTX_RESERVED6_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_RESERVED6_SHIFT 7 +/* exist_in_qm0 */ +#define E4_XSTORM_ETH_CONN_AG_CTX_EXIST_IN_QM0_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_EXIST_IN_QM0_SHIFT 0 +/* exist_in_qm1 */ +#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED1_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED1_SHIFT 1 +/* exist_in_qm2 */ +#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED2_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED2_SHIFT 2 +/* exist_in_qm3 */ +#define E4_XSTORM_ETH_CONN_AG_CTX_EXIST_IN_QM3_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_EXIST_IN_QM3_SHIFT 3 +/* bit4 */ +#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED3_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED3_SHIFT 4 +/* cf_array_active */ +#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED4_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED4_SHIFT 5 +/* bit6 */ +#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED5_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED5_SHIFT 6 +/* bit7 */ +#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED6_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED6_SHIFT 7 u8 flags1; -#define XSTORM_ETH_CONN_AG_CTX_RESERVED7_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_RESERVED7_SHIFT 0 -#define XSTORM_ETH_CONN_AG_CTX_RESERVED8_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_RESERVED8_SHIFT 1 -#define XSTORM_ETH_CONN_AG_CTX_RESERVED9_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_RESERVED9_SHIFT 2 -#define XSTORM_ETH_CONN_AG_CTX_BIT11_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_BIT11_SHIFT 3 -#define XSTORM_ETH_CONN_AG_CTX_BIT12_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_BIT12_SHIFT 4 -#define XSTORM_ETH_CONN_AG_CTX_BIT13_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_BIT13_SHIFT 5 -#define XSTORM_ETH_CONN_AG_CTX_TX_RULE_ACTIVE_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_TX_RULE_ACTIVE_SHIFT 6 -#define XSTORM_ETH_CONN_AG_CTX_DQ_CF_ACTIVE_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_DQ_CF_ACTIVE_SHIFT 7 +/* bit8 */ +#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED7_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED7_SHIFT 0 +/* bit9 */ +#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED8_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED8_SHIFT 1 +/* bit10 */ +#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED9_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED9_SHIFT 2 +/* bit11 */ +#define E4_XSTORM_ETH_CONN_AG_CTX_BIT11_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_BIT11_SHIFT 3 +/* bit12 */ +#define E4_XSTORM_ETH_CONN_AG_CTX_BIT12_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_BIT12_SHIFT 4 +/* bit13 */ +#define E4_XSTORM_ETH_CONN_AG_CTX_BIT13_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_BIT13_SHIFT 5 +/* bit14 */ +#define E4_XSTORM_ETH_CONN_AG_CTX_TX_RULE_ACTIVE_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_TX_RULE_ACTIVE_SHIFT 6 +/* bit15 */ +#define E4_XSTORM_ETH_CONN_AG_CTX_DQ_CF_ACTIVE_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_DQ_CF_ACTIVE_SHIFT 7 u8 flags2; -#define XSTORM_ETH_CONN_AG_CTX_CF0_MASK 0x3 -#define XSTORM_ETH_CONN_AG_CTX_CF0_SHIFT 0 -#define XSTORM_ETH_CONN_AG_CTX_CF1_MASK 0x3 -#define XSTORM_ETH_CONN_AG_CTX_CF1_SHIFT 2 -#define XSTORM_ETH_CONN_AG_CTX_CF2_MASK 0x3 -#define XSTORM_ETH_CONN_AG_CTX_CF2_SHIFT 4 -#define XSTORM_ETH_CONN_AG_CTX_CF3_MASK 0x3 -#define XSTORM_ETH_CONN_AG_CTX_CF3_SHIFT 6 +/* timer0cf */ +#define E4_XSTORM_ETH_CONN_AG_CTX_CF0_MASK 0x3 +#define E4_XSTORM_ETH_CONN_AG_CTX_CF0_SHIFT 0 +/* timer1cf */ +#define E4_XSTORM_ETH_CONN_AG_CTX_CF1_MASK 0x3 +#define E4_XSTORM_ETH_CONN_AG_CTX_CF1_SHIFT 2 +/* timer2cf */ +#define E4_XSTORM_ETH_CONN_AG_CTX_CF2_MASK 0x3 +#define E4_XSTORM_ETH_CONN_AG_CTX_CF2_SHIFT 4 +/* timer_stop_all */ +#define E4_XSTORM_ETH_CONN_AG_CTX_CF3_MASK 0x3 +#define E4_XSTORM_ETH_CONN_AG_CTX_CF3_SHIFT 6 u8 flags3; -#define XSTORM_ETH_CONN_AG_CTX_CF4_MASK 0x3 -#define XSTORM_ETH_CONN_AG_CTX_CF4_SHIFT 0 -#define XSTORM_ETH_CONN_AG_CTX_CF5_MASK 0x3 -#define XSTORM_ETH_CONN_AG_CTX_CF5_SHIFT 2 -#define XSTORM_ETH_CONN_AG_CTX_CF6_MASK 0x3 -#define XSTORM_ETH_CONN_AG_CTX_CF6_SHIFT 4 -#define XSTORM_ETH_CONN_AG_CTX_CF7_MASK 0x3 -#define XSTORM_ETH_CONN_AG_CTX_CF7_SHIFT 6 +/* cf4 */ +#define E4_XSTORM_ETH_CONN_AG_CTX_CF4_MASK 0x3 +#define E4_XSTORM_ETH_CONN_AG_CTX_CF4_SHIFT 0 +/* cf5 */ +#define E4_XSTORM_ETH_CONN_AG_CTX_CF5_MASK 0x3 +#define E4_XSTORM_ETH_CONN_AG_CTX_CF5_SHIFT 2 +/* cf6 */ +#define E4_XSTORM_ETH_CONN_AG_CTX_CF6_MASK 0x3 +#define E4_XSTORM_ETH_CONN_AG_CTX_CF6_SHIFT 4 +/* cf7 */ +#define E4_XSTORM_ETH_CONN_AG_CTX_CF7_MASK 0x3 +#define E4_XSTORM_ETH_CONN_AG_CTX_CF7_SHIFT 6 u8 flags4; -#define XSTORM_ETH_CONN_AG_CTX_CF8_MASK 0x3 -#define XSTORM_ETH_CONN_AG_CTX_CF8_SHIFT 0 -#define XSTORM_ETH_CONN_AG_CTX_CF9_MASK 0x3 -#define XSTORM_ETH_CONN_AG_CTX_CF9_SHIFT 2 -#define XSTORM_ETH_CONN_AG_CTX_CF10_MASK 0x3 -#define XSTORM_ETH_CONN_AG_CTX_CF10_SHIFT 4 -#define XSTORM_ETH_CONN_AG_CTX_CF11_MASK 0x3 -#define XSTORM_ETH_CONN_AG_CTX_CF11_SHIFT 6 +/* cf8 */ +#define E4_XSTORM_ETH_CONN_AG_CTX_CF8_MASK 0x3 +#define E4_XSTORM_ETH_CONN_AG_CTX_CF8_SHIFT 0 +/* cf9 */ +#define E4_XSTORM_ETH_CONN_AG_CTX_CF9_MASK 0x3 +#define E4_XSTORM_ETH_CONN_AG_CTX_CF9_SHIFT 2 +/* cf10 */ +#define E4_XSTORM_ETH_CONN_AG_CTX_CF10_MASK 0x3 +#define E4_XSTORM_ETH_CONN_AG_CTX_CF10_SHIFT 4 +/* cf11 */ +#define E4_XSTORM_ETH_CONN_AG_CTX_CF11_MASK 0x3 +#define E4_XSTORM_ETH_CONN_AG_CTX_CF11_SHIFT 6 u8 flags5; -#define XSTORM_ETH_CONN_AG_CTX_CF12_MASK 0x3 -#define XSTORM_ETH_CONN_AG_CTX_CF12_SHIFT 0 -#define XSTORM_ETH_CONN_AG_CTX_CF13_MASK 0x3 -#define XSTORM_ETH_CONN_AG_CTX_CF13_SHIFT 2 -#define XSTORM_ETH_CONN_AG_CTX_CF14_MASK 0x3 -#define XSTORM_ETH_CONN_AG_CTX_CF14_SHIFT 4 -#define XSTORM_ETH_CONN_AG_CTX_CF15_MASK 0x3 -#define XSTORM_ETH_CONN_AG_CTX_CF15_SHIFT 6 +/* cf12 */ +#define E4_XSTORM_ETH_CONN_AG_CTX_CF12_MASK 0x3 +#define E4_XSTORM_ETH_CONN_AG_CTX_CF12_SHIFT 0 +/* cf13 */ +#define E4_XSTORM_ETH_CONN_AG_CTX_CF13_MASK 0x3 +#define E4_XSTORM_ETH_CONN_AG_CTX_CF13_SHIFT 2 +/* cf14 */ +#define E4_XSTORM_ETH_CONN_AG_CTX_CF14_MASK 0x3 +#define E4_XSTORM_ETH_CONN_AG_CTX_CF14_SHIFT 4 +/* cf15 */ +#define E4_XSTORM_ETH_CONN_AG_CTX_CF15_MASK 0x3 +#define E4_XSTORM_ETH_CONN_AG_CTX_CF15_SHIFT 6 u8 flags6; -#define XSTORM_ETH_CONN_AG_CTX_GO_TO_BD_CONS_CF_MASK 0x3 -#define XSTORM_ETH_CONN_AG_CTX_GO_TO_BD_CONS_CF_SHIFT 0 -#define XSTORM_ETH_CONN_AG_CTX_MULTI_UNICAST_CF_MASK 0x3 -#define XSTORM_ETH_CONN_AG_CTX_MULTI_UNICAST_CF_SHIFT 2 -#define XSTORM_ETH_CONN_AG_CTX_DQ_CF_MASK 0x3 -#define XSTORM_ETH_CONN_AG_CTX_DQ_CF_SHIFT 4 -#define XSTORM_ETH_CONN_AG_CTX_TERMINATE_CF_MASK 0x3 -#define XSTORM_ETH_CONN_AG_CTX_TERMINATE_CF_SHIFT 6 +/* cf16 */ +#define E4_XSTORM_ETH_CONN_AG_CTX_GO_TO_BD_CONS_CF_MASK 0x3 +#define E4_XSTORM_ETH_CONN_AG_CTX_GO_TO_BD_CONS_CF_SHIFT 0 +/* cf_array_cf */ +#define E4_XSTORM_ETH_CONN_AG_CTX_MULTI_UNICAST_CF_MASK 0x3 +#define E4_XSTORM_ETH_CONN_AG_CTX_MULTI_UNICAST_CF_SHIFT 2 +/* cf18 */ +#define E4_XSTORM_ETH_CONN_AG_CTX_DQ_CF_MASK 0x3 +#define E4_XSTORM_ETH_CONN_AG_CTX_DQ_CF_SHIFT 4 +/* cf19 */ +#define E4_XSTORM_ETH_CONN_AG_CTX_TERMINATE_CF_MASK 0x3 +#define E4_XSTORM_ETH_CONN_AG_CTX_TERMINATE_CF_SHIFT 6 u8 flags7; -#define XSTORM_ETH_CONN_AG_CTX_FLUSH_Q0_MASK 0x3 -#define XSTORM_ETH_CONN_AG_CTX_FLUSH_Q0_SHIFT 0 -#define XSTORM_ETH_CONN_AG_CTX_RESERVED10_MASK 0x3 -#define XSTORM_ETH_CONN_AG_CTX_RESERVED10_SHIFT 2 -#define XSTORM_ETH_CONN_AG_CTX_SLOW_PATH_MASK 0x3 -#define XSTORM_ETH_CONN_AG_CTX_SLOW_PATH_SHIFT 4 -#define XSTORM_ETH_CONN_AG_CTX_CF0EN_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_CF0EN_SHIFT 6 -#define XSTORM_ETH_CONN_AG_CTX_CF1EN_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_CF1EN_SHIFT 7 +/* cf20 */ +#define E4_XSTORM_ETH_CONN_AG_CTX_FLUSH_Q0_MASK 0x3 +#define E4_XSTORM_ETH_CONN_AG_CTX_FLUSH_Q0_SHIFT 0 +/* cf21 */ +#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED10_MASK 0x3 +#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED10_SHIFT 2 +/* cf22 */ +#define E4_XSTORM_ETH_CONN_AG_CTX_SLOW_PATH_MASK 0x3 +#define E4_XSTORM_ETH_CONN_AG_CTX_SLOW_PATH_SHIFT 4 +/* cf0en */ +#define E4_XSTORM_ETH_CONN_AG_CTX_CF0EN_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_CF0EN_SHIFT 6 +/* cf1en */ +#define E4_XSTORM_ETH_CONN_AG_CTX_CF1EN_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_CF1EN_SHIFT 7 u8 flags8; -#define XSTORM_ETH_CONN_AG_CTX_CF2EN_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_CF2EN_SHIFT 0 -#define XSTORM_ETH_CONN_AG_CTX_CF3EN_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_CF3EN_SHIFT 1 -#define XSTORM_ETH_CONN_AG_CTX_CF4EN_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_CF4EN_SHIFT 2 -#define XSTORM_ETH_CONN_AG_CTX_CF5EN_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_CF5EN_SHIFT 3 -#define XSTORM_ETH_CONN_AG_CTX_CF6EN_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_CF6EN_SHIFT 4 -#define XSTORM_ETH_CONN_AG_CTX_CF7EN_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_CF7EN_SHIFT 5 -#define XSTORM_ETH_CONN_AG_CTX_CF8EN_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_CF8EN_SHIFT 6 -#define XSTORM_ETH_CONN_AG_CTX_CF9EN_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_CF9EN_SHIFT 7 +/* cf2en */ +#define E4_XSTORM_ETH_CONN_AG_CTX_CF2EN_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_CF2EN_SHIFT 0 +/* cf3en */ +#define E4_XSTORM_ETH_CONN_AG_CTX_CF3EN_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_CF3EN_SHIFT 1 +/* cf4en */ +#define E4_XSTORM_ETH_CONN_AG_CTX_CF4EN_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_CF4EN_SHIFT 2 +/* cf5en */ +#define E4_XSTORM_ETH_CONN_AG_CTX_CF5EN_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_CF5EN_SHIFT 3 +/* cf6en */ +#define E4_XSTORM_ETH_CONN_AG_CTX_CF6EN_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_CF6EN_SHIFT 4 +/* cf7en */ +#define E4_XSTORM_ETH_CONN_AG_CTX_CF7EN_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_CF7EN_SHIFT 5 +/* cf8en */ +#define E4_XSTORM_ETH_CONN_AG_CTX_CF8EN_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_CF8EN_SHIFT 6 +/* cf9en */ +#define E4_XSTORM_ETH_CONN_AG_CTX_CF9EN_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_CF9EN_SHIFT 7 u8 flags9; -#define XSTORM_ETH_CONN_AG_CTX_CF10EN_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_CF10EN_SHIFT 0 -#define XSTORM_ETH_CONN_AG_CTX_CF11EN_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_CF11EN_SHIFT 1 -#define XSTORM_ETH_CONN_AG_CTX_CF12EN_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_CF12EN_SHIFT 2 -#define XSTORM_ETH_CONN_AG_CTX_CF13EN_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_CF13EN_SHIFT 3 -#define XSTORM_ETH_CONN_AG_CTX_CF14EN_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_CF14EN_SHIFT 4 -#define XSTORM_ETH_CONN_AG_CTX_CF15EN_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_CF15EN_SHIFT 5 -#define XSTORM_ETH_CONN_AG_CTX_GO_TO_BD_CONS_CF_EN_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_GO_TO_BD_CONS_CF_EN_SHIFT 6 -#define XSTORM_ETH_CONN_AG_CTX_MULTI_UNICAST_CF_EN_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_MULTI_UNICAST_CF_EN_SHIFT 7 +/* cf10en */ +#define E4_XSTORM_ETH_CONN_AG_CTX_CF10EN_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_CF10EN_SHIFT 0 +/* cf11en */ +#define E4_XSTORM_ETH_CONN_AG_CTX_CF11EN_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_CF11EN_SHIFT 1 +/* cf12en */ +#define E4_XSTORM_ETH_CONN_AG_CTX_CF12EN_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_CF12EN_SHIFT 2 +/* cf13en */ +#define E4_XSTORM_ETH_CONN_AG_CTX_CF13EN_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_CF13EN_SHIFT 3 +/* cf14en */ +#define E4_XSTORM_ETH_CONN_AG_CTX_CF14EN_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_CF14EN_SHIFT 4 +/* cf15en */ +#define E4_XSTORM_ETH_CONN_AG_CTX_CF15EN_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_CF15EN_SHIFT 5 +/* cf16en */ +#define E4_XSTORM_ETH_CONN_AG_CTX_GO_TO_BD_CONS_CF_EN_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_GO_TO_BD_CONS_CF_EN_SHIFT 6 +/* cf_array_cf_en */ +#define E4_XSTORM_ETH_CONN_AG_CTX_MULTI_UNICAST_CF_EN_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_MULTI_UNICAST_CF_EN_SHIFT 7 u8 flags10; -#define XSTORM_ETH_CONN_AG_CTX_DQ_CF_EN_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_DQ_CF_EN_SHIFT 0 -#define XSTORM_ETH_CONN_AG_CTX_TERMINATE_CF_EN_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_TERMINATE_CF_EN_SHIFT 1 -#define XSTORM_ETH_CONN_AG_CTX_FLUSH_Q0_EN_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_FLUSH_Q0_EN_SHIFT 2 -#define XSTORM_ETH_CONN_AG_CTX_RESERVED11_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_RESERVED11_SHIFT 3 -#define XSTORM_ETH_CONN_AG_CTX_SLOW_PATH_EN_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_SLOW_PATH_EN_SHIFT 4 -#define XSTORM_ETH_CONN_AG_CTX_TPH_ENABLE_EN_RESERVED_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_TPH_ENABLE_EN_RESERVED_SHIFT 5 -#define XSTORM_ETH_CONN_AG_CTX_RESERVED12_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_RESERVED12_SHIFT 6 -#define XSTORM_ETH_CONN_AG_CTX_RESERVED13_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_RESERVED13_SHIFT 7 +/* cf18en */ +#define E4_XSTORM_ETH_CONN_AG_CTX_DQ_CF_EN_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_DQ_CF_EN_SHIFT 0 +/* cf19en */ +#define E4_XSTORM_ETH_CONN_AG_CTX_TERMINATE_CF_EN_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_TERMINATE_CF_EN_SHIFT 1 +/* cf20en */ +#define E4_XSTORM_ETH_CONN_AG_CTX_FLUSH_Q0_EN_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_FLUSH_Q0_EN_SHIFT 2 +/* cf21en */ +#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED11_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED11_SHIFT 3 +/* cf22en */ +#define E4_XSTORM_ETH_CONN_AG_CTX_SLOW_PATH_EN_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_SLOW_PATH_EN_SHIFT 4 +/* cf23en */ +#define E4_XSTORM_ETH_CONN_AG_CTX_TPH_ENABLE_EN_RESERVED_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_TPH_ENABLE_EN_RESERVED_SHIFT 5 +/* rule0en */ +#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED12_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED12_SHIFT 6 +/* rule1en */ +#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED13_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED13_SHIFT 7 u8 flags11; -#define XSTORM_ETH_CONN_AG_CTX_RESERVED14_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_RESERVED14_SHIFT 0 -#define XSTORM_ETH_CONN_AG_CTX_RESERVED15_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_RESERVED15_SHIFT 1 -#define XSTORM_ETH_CONN_AG_CTX_TX_DEC_RULE_EN_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_TX_DEC_RULE_EN_SHIFT 2 -#define XSTORM_ETH_CONN_AG_CTX_RULE5EN_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_RULE5EN_SHIFT 3 -#define XSTORM_ETH_CONN_AG_CTX_RULE6EN_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_RULE6EN_SHIFT 4 -#define XSTORM_ETH_CONN_AG_CTX_RULE7EN_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_RULE7EN_SHIFT 5 -#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED1_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED1_SHIFT 6 -#define XSTORM_ETH_CONN_AG_CTX_RULE9EN_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_RULE9EN_SHIFT 7 +/* rule2en */ +#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED14_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED14_SHIFT 0 +/* rule3en */ +#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED15_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED15_SHIFT 1 +/* rule4en */ +#define E4_XSTORM_ETH_CONN_AG_CTX_TX_DEC_RULE_EN_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_TX_DEC_RULE_EN_SHIFT 2 +/* rule5en */ +#define E4_XSTORM_ETH_CONN_AG_CTX_RULE5EN_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_RULE5EN_SHIFT 3 +/* rule6en */ +#define E4_XSTORM_ETH_CONN_AG_CTX_RULE6EN_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_RULE6EN_SHIFT 4 +/* rule7en */ +#define E4_XSTORM_ETH_CONN_AG_CTX_RULE7EN_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_RULE7EN_SHIFT 5 +/* rule8en */ +#define E4_XSTORM_ETH_CONN_AG_CTX_A0_RESERVED1_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_A0_RESERVED1_SHIFT 6 +/* rule9en */ +#define E4_XSTORM_ETH_CONN_AG_CTX_RULE9EN_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_RULE9EN_SHIFT 7 u8 flags12; -#define XSTORM_ETH_CONN_AG_CTX_RULE10EN_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_RULE10EN_SHIFT 0 -#define XSTORM_ETH_CONN_AG_CTX_RULE11EN_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_RULE11EN_SHIFT 1 -#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED2_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED2_SHIFT 2 -#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED3_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED3_SHIFT 3 -#define XSTORM_ETH_CONN_AG_CTX_RULE14EN_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_RULE14EN_SHIFT 4 -#define XSTORM_ETH_CONN_AG_CTX_RULE15EN_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_RULE15EN_SHIFT 5 -#define XSTORM_ETH_CONN_AG_CTX_RULE16EN_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_RULE16EN_SHIFT 6 -#define XSTORM_ETH_CONN_AG_CTX_RULE17EN_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_RULE17EN_SHIFT 7 +/* rule10en */ +#define E4_XSTORM_ETH_CONN_AG_CTX_RULE10EN_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_RULE10EN_SHIFT 0 +/* rule11en */ +#define E4_XSTORM_ETH_CONN_AG_CTX_RULE11EN_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_RULE11EN_SHIFT 1 +/* rule12en */ +#define E4_XSTORM_ETH_CONN_AG_CTX_A0_RESERVED2_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_A0_RESERVED2_SHIFT 2 +/* rule13en */ +#define E4_XSTORM_ETH_CONN_AG_CTX_A0_RESERVED3_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_A0_RESERVED3_SHIFT 3 +/* rule14en */ +#define E4_XSTORM_ETH_CONN_AG_CTX_RULE14EN_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_RULE14EN_SHIFT 4 +/* rule15en */ +#define E4_XSTORM_ETH_CONN_AG_CTX_RULE15EN_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_RULE15EN_SHIFT 5 +/* rule16en */ +#define E4_XSTORM_ETH_CONN_AG_CTX_RULE16EN_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_RULE16EN_SHIFT 6 +/* rule17en */ +#define E4_XSTORM_ETH_CONN_AG_CTX_RULE17EN_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_RULE17EN_SHIFT 7 u8 flags13; -#define XSTORM_ETH_CONN_AG_CTX_RULE18EN_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_RULE18EN_SHIFT 0 -#define XSTORM_ETH_CONN_AG_CTX_RULE19EN_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_RULE19EN_SHIFT 1 -#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED4_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED4_SHIFT 2 -#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED5_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED5_SHIFT 3 -#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED6_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED6_SHIFT 4 -#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED7_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED7_SHIFT 5 -#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED8_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED8_SHIFT 6 -#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED9_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED9_SHIFT 7 +/* rule18en */ +#define E4_XSTORM_ETH_CONN_AG_CTX_RULE18EN_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_RULE18EN_SHIFT 0 +/* rule19en */ +#define E4_XSTORM_ETH_CONN_AG_CTX_RULE19EN_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_RULE19EN_SHIFT 1 +/* rule20en */ +#define E4_XSTORM_ETH_CONN_AG_CTX_A0_RESERVED4_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_A0_RESERVED4_SHIFT 2 +/* rule21en */ +#define E4_XSTORM_ETH_CONN_AG_CTX_A0_RESERVED5_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_A0_RESERVED5_SHIFT 3 +/* rule22en */ +#define E4_XSTORM_ETH_CONN_AG_CTX_A0_RESERVED6_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_A0_RESERVED6_SHIFT 4 +/* rule23en */ +#define E4_XSTORM_ETH_CONN_AG_CTX_A0_RESERVED7_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_A0_RESERVED7_SHIFT 5 +/* rule24en */ +#define E4_XSTORM_ETH_CONN_AG_CTX_A0_RESERVED8_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_A0_RESERVED8_SHIFT 6 +/* rule25en */ +#define E4_XSTORM_ETH_CONN_AG_CTX_A0_RESERVED9_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_A0_RESERVED9_SHIFT 7 u8 flags14; -#define XSTORM_ETH_CONN_AG_CTX_EDPM_USE_EXT_HDR_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_EDPM_USE_EXT_HDR_SHIFT 0 -#define XSTORM_ETH_CONN_AG_CTX_EDPM_SEND_RAW_L3L4_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_EDPM_SEND_RAW_L3L4_SHIFT 1 -#define XSTORM_ETH_CONN_AG_CTX_EDPM_INBAND_PROP_HDR_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_EDPM_INBAND_PROP_HDR_SHIFT 2 -#define XSTORM_ETH_CONN_AG_CTX_EDPM_SEND_EXT_TUNNEL_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_EDPM_SEND_EXT_TUNNEL_SHIFT 3 -#define XSTORM_ETH_CONN_AG_CTX_L2_EDPM_ENABLE_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_L2_EDPM_ENABLE_SHIFT 4 -#define XSTORM_ETH_CONN_AG_CTX_ROCE_EDPM_ENABLE_MASK 0x1 -#define XSTORM_ETH_CONN_AG_CTX_ROCE_EDPM_ENABLE_SHIFT 5 -#define XSTORM_ETH_CONN_AG_CTX_TPH_ENABLE_MASK 0x3 -#define XSTORM_ETH_CONN_AG_CTX_TPH_ENABLE_SHIFT 6 +/* bit16 */ +#define E4_XSTORM_ETH_CONN_AG_CTX_EDPM_USE_EXT_HDR_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_EDPM_USE_EXT_HDR_SHIFT 0 +/* bit17 */ +#define E4_XSTORM_ETH_CONN_AG_CTX_EDPM_SEND_RAW_L3L4_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_EDPM_SEND_RAW_L3L4_SHIFT 1 +/* bit18 */ +#define E4_XSTORM_ETH_CONN_AG_CTX_EDPM_INBAND_PROP_HDR_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_EDPM_INBAND_PROP_HDR_SHIFT 2 +/* bit19 */ +#define E4_XSTORM_ETH_CONN_AG_CTX_EDPM_SEND_EXT_TUNNEL_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_EDPM_SEND_EXT_TUNNEL_SHIFT 3 +/* bit20 */ +#define E4_XSTORM_ETH_CONN_AG_CTX_L2_EDPM_ENABLE_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_L2_EDPM_ENABLE_SHIFT 4 +/* bit21 */ +#define E4_XSTORM_ETH_CONN_AG_CTX_ROCE_EDPM_ENABLE_MASK 0x1 +#define E4_XSTORM_ETH_CONN_AG_CTX_ROCE_EDPM_ENABLE_SHIFT 5 +/* cf23 */ +#define E4_XSTORM_ETH_CONN_AG_CTX_TPH_ENABLE_MASK 0x3 +#define E4_XSTORM_ETH_CONN_AG_CTX_TPH_ENABLE_SHIFT 6 u8 edpm_event_id /* byte2 */; __le16 physical_q0 /* physical_q0 */; - __le16 word1 /* physical_q1 */; + __le16 e5_reserved1 /* physical_q1 */; __le16 edpm_num_bds /* physical_q2 */; __le16 tx_bd_cons /* word3 */; __le16 tx_bd_prod /* word4 */; - __le16 go_to_bd_cons /* word5 */; + __le16 tx_class /* word5 */; __le16 conn_dpi /* conn_dpi */; u8 byte3 /* byte3 */; u8 byte4 /* byte4 */; @@ -279,7 +375,7 @@ struct xstorm_eth_conn_ag_ctx { u8 byte13 /* byte13 */; u8 byte14 /* byte14 */; u8 byte15 /* byte15 */; - u8 byte16 /* byte16 */; + u8 e5_reserved /* e5_reserved */; __le16 word11 /* word11 */; __le32 reg10 /* reg10 */; __le32 reg11 /* reg11 */; @@ -304,38 +400,48 @@ struct ystorm_eth_conn_st_ctx { __le32 reserved[8]; }; -struct ystorm_eth_conn_ag_ctx { +struct e4_ystorm_eth_conn_ag_ctx { u8 byte0 /* cdu_validation */; - u8 byte1 /* state */; + u8 state /* state */; u8 flags0; -#define YSTORM_ETH_CONN_AG_CTX_BIT0_MASK 0x1 -#define YSTORM_ETH_CONN_AG_CTX_BIT0_SHIFT 0 -#define YSTORM_ETH_CONN_AG_CTX_BIT1_MASK 0x1 -#define YSTORM_ETH_CONN_AG_CTX_BIT1_SHIFT 1 -#define YSTORM_ETH_CONN_AG_CTX_TX_BD_CONS_UPD_CF_MASK 0x3 -#define YSTORM_ETH_CONN_AG_CTX_TX_BD_CONS_UPD_CF_SHIFT 2 -#define YSTORM_ETH_CONN_AG_CTX_PMD_TERMINATE_CF_MASK 0x3 -#define YSTORM_ETH_CONN_AG_CTX_PMD_TERMINATE_CF_SHIFT 4 -#define YSTORM_ETH_CONN_AG_CTX_CF2_MASK 0x3 -#define YSTORM_ETH_CONN_AG_CTX_CF2_SHIFT 6 +/* exist_in_qm0 */ +#define E4_YSTORM_ETH_CONN_AG_CTX_BIT0_MASK 0x1 +#define E4_YSTORM_ETH_CONN_AG_CTX_BIT0_SHIFT 0 +/* exist_in_qm1 */ +#define E4_YSTORM_ETH_CONN_AG_CTX_BIT1_MASK 0x1 +#define E4_YSTORM_ETH_CONN_AG_CTX_BIT1_SHIFT 1 +#define E4_YSTORM_ETH_CONN_AG_CTX_TX_BD_CONS_UPD_CF_MASK 0x3 /* cf0 */ +#define E4_YSTORM_ETH_CONN_AG_CTX_TX_BD_CONS_UPD_CF_SHIFT 2 +#define E4_YSTORM_ETH_CONN_AG_CTX_PMD_TERMINATE_CF_MASK 0x3 /* cf1 */ +#define E4_YSTORM_ETH_CONN_AG_CTX_PMD_TERMINATE_CF_SHIFT 4 +#define E4_YSTORM_ETH_CONN_AG_CTX_CF2_MASK 0x3 /* cf2 */ +#define E4_YSTORM_ETH_CONN_AG_CTX_CF2_SHIFT 6 u8 flags1; -#define YSTORM_ETH_CONN_AG_CTX_TX_BD_CONS_UPD_CF_EN_MASK 0x1 -#define YSTORM_ETH_CONN_AG_CTX_TX_BD_CONS_UPD_CF_EN_SHIFT 0 -#define YSTORM_ETH_CONN_AG_CTX_PMD_TERMINATE_CF_EN_MASK 0x1 -#define YSTORM_ETH_CONN_AG_CTX_PMD_TERMINATE_CF_EN_SHIFT 1 -#define YSTORM_ETH_CONN_AG_CTX_CF2EN_MASK 0x1 -#define YSTORM_ETH_CONN_AG_CTX_CF2EN_SHIFT 2 -#define YSTORM_ETH_CONN_AG_CTX_RULE0EN_MASK 0x1 -#define YSTORM_ETH_CONN_AG_CTX_RULE0EN_SHIFT 3 -#define YSTORM_ETH_CONN_AG_CTX_RULE1EN_MASK 0x1 -#define YSTORM_ETH_CONN_AG_CTX_RULE1EN_SHIFT 4 -#define YSTORM_ETH_CONN_AG_CTX_RULE2EN_MASK 0x1 -#define YSTORM_ETH_CONN_AG_CTX_RULE2EN_SHIFT 5 -#define YSTORM_ETH_CONN_AG_CTX_RULE3EN_MASK 0x1 -#define YSTORM_ETH_CONN_AG_CTX_RULE3EN_SHIFT 6 -#define YSTORM_ETH_CONN_AG_CTX_RULE4EN_MASK 0x1 -#define YSTORM_ETH_CONN_AG_CTX_RULE4EN_SHIFT 7 - u8 byte2 /* byte2 */; +/* cf0en */ +#define E4_YSTORM_ETH_CONN_AG_CTX_TX_BD_CONS_UPD_CF_EN_MASK 0x1 +#define E4_YSTORM_ETH_CONN_AG_CTX_TX_BD_CONS_UPD_CF_EN_SHIFT 0 +/* cf1en */ +#define E4_YSTORM_ETH_CONN_AG_CTX_PMD_TERMINATE_CF_EN_MASK 0x1 +#define E4_YSTORM_ETH_CONN_AG_CTX_PMD_TERMINATE_CF_EN_SHIFT 1 +/* cf2en */ +#define E4_YSTORM_ETH_CONN_AG_CTX_CF2EN_MASK 0x1 +#define E4_YSTORM_ETH_CONN_AG_CTX_CF2EN_SHIFT 2 +/* rule0en */ +#define E4_YSTORM_ETH_CONN_AG_CTX_RULE0EN_MASK 0x1 +#define E4_YSTORM_ETH_CONN_AG_CTX_RULE0EN_SHIFT 3 +/* rule1en */ +#define E4_YSTORM_ETH_CONN_AG_CTX_RULE1EN_MASK 0x1 +#define E4_YSTORM_ETH_CONN_AG_CTX_RULE1EN_SHIFT 4 +/* rule2en */ +#define E4_YSTORM_ETH_CONN_AG_CTX_RULE2EN_MASK 0x1 +#define E4_YSTORM_ETH_CONN_AG_CTX_RULE2EN_SHIFT 5 +/* rule3en */ +#define E4_YSTORM_ETH_CONN_AG_CTX_RULE3EN_MASK 0x1 +#define E4_YSTORM_ETH_CONN_AG_CTX_RULE3EN_SHIFT 6 +/* rule4en */ +#define E4_YSTORM_ETH_CONN_AG_CTX_RULE4EN_MASK 0x1 +#define E4_YSTORM_ETH_CONN_AG_CTX_RULE4EN_SHIFT 7 + u8 tx_q0_int_coallecing_timeset /* byte2 */; u8 byte3 /* byte3 */; __le16 word0 /* word0 */; __le32 terminate_spqe /* reg0 */; @@ -348,89 +454,89 @@ struct ystorm_eth_conn_ag_ctx { __le32 reg3 /* reg3 */; }; -struct tstorm_eth_conn_ag_ctx { +struct e4_tstorm_eth_conn_ag_ctx { u8 byte0 /* cdu_validation */; u8 byte1 /* state */; u8 flags0; -#define TSTORM_ETH_CONN_AG_CTX_BIT0_MASK 0x1 -#define TSTORM_ETH_CONN_AG_CTX_BIT0_SHIFT 0 -#define TSTORM_ETH_CONN_AG_CTX_BIT1_MASK 0x1 -#define TSTORM_ETH_CONN_AG_CTX_BIT1_SHIFT 1 -#define TSTORM_ETH_CONN_AG_CTX_BIT2_MASK 0x1 -#define TSTORM_ETH_CONN_AG_CTX_BIT2_SHIFT 2 -#define TSTORM_ETH_CONN_AG_CTX_BIT3_MASK 0x1 -#define TSTORM_ETH_CONN_AG_CTX_BIT3_SHIFT 3 -#define TSTORM_ETH_CONN_AG_CTX_BIT4_MASK 0x1 -#define TSTORM_ETH_CONN_AG_CTX_BIT4_SHIFT 4 -#define TSTORM_ETH_CONN_AG_CTX_BIT5_MASK 0x1 -#define TSTORM_ETH_CONN_AG_CTX_BIT5_SHIFT 5 -#define TSTORM_ETH_CONN_AG_CTX_CF0_MASK 0x3 -#define TSTORM_ETH_CONN_AG_CTX_CF0_SHIFT 6 +#define E4_TSTORM_ETH_CONN_AG_CTX_BIT0_MASK 0x1 /* exist_in_qm0 */ +#define E4_TSTORM_ETH_CONN_AG_CTX_BIT0_SHIFT 0 +#define E4_TSTORM_ETH_CONN_AG_CTX_BIT1_MASK 0x1 /* exist_in_qm1 */ +#define E4_TSTORM_ETH_CONN_AG_CTX_BIT1_SHIFT 1 +#define E4_TSTORM_ETH_CONN_AG_CTX_BIT2_MASK 0x1 /* bit2 */ +#define E4_TSTORM_ETH_CONN_AG_CTX_BIT2_SHIFT 2 +#define E4_TSTORM_ETH_CONN_AG_CTX_BIT3_MASK 0x1 /* bit3 */ +#define E4_TSTORM_ETH_CONN_AG_CTX_BIT3_SHIFT 3 +#define E4_TSTORM_ETH_CONN_AG_CTX_BIT4_MASK 0x1 /* bit4 */ +#define E4_TSTORM_ETH_CONN_AG_CTX_BIT4_SHIFT 4 +#define E4_TSTORM_ETH_CONN_AG_CTX_BIT5_MASK 0x1 /* bit5 */ +#define E4_TSTORM_ETH_CONN_AG_CTX_BIT5_SHIFT 5 +#define E4_TSTORM_ETH_CONN_AG_CTX_CF0_MASK 0x3 /* timer0cf */ +#define E4_TSTORM_ETH_CONN_AG_CTX_CF0_SHIFT 6 u8 flags1; -#define TSTORM_ETH_CONN_AG_CTX_CF1_MASK 0x3 -#define TSTORM_ETH_CONN_AG_CTX_CF1_SHIFT 0 -#define TSTORM_ETH_CONN_AG_CTX_CF2_MASK 0x3 -#define TSTORM_ETH_CONN_AG_CTX_CF2_SHIFT 2 -#define TSTORM_ETH_CONN_AG_CTX_CF3_MASK 0x3 -#define TSTORM_ETH_CONN_AG_CTX_CF3_SHIFT 4 -#define TSTORM_ETH_CONN_AG_CTX_CF4_MASK 0x3 -#define TSTORM_ETH_CONN_AG_CTX_CF4_SHIFT 6 +#define E4_TSTORM_ETH_CONN_AG_CTX_CF1_MASK 0x3 /* timer1cf */ +#define E4_TSTORM_ETH_CONN_AG_CTX_CF1_SHIFT 0 +#define E4_TSTORM_ETH_CONN_AG_CTX_CF2_MASK 0x3 /* timer2cf */ +#define E4_TSTORM_ETH_CONN_AG_CTX_CF2_SHIFT 2 +#define E4_TSTORM_ETH_CONN_AG_CTX_CF3_MASK 0x3 /* timer_stop_all */ +#define E4_TSTORM_ETH_CONN_AG_CTX_CF3_SHIFT 4 +#define E4_TSTORM_ETH_CONN_AG_CTX_CF4_MASK 0x3 /* cf4 */ +#define E4_TSTORM_ETH_CONN_AG_CTX_CF4_SHIFT 6 u8 flags2; -#define TSTORM_ETH_CONN_AG_CTX_CF5_MASK 0x3 -#define TSTORM_ETH_CONN_AG_CTX_CF5_SHIFT 0 -#define TSTORM_ETH_CONN_AG_CTX_CF6_MASK 0x3 -#define TSTORM_ETH_CONN_AG_CTX_CF6_SHIFT 2 -#define TSTORM_ETH_CONN_AG_CTX_CF7_MASK 0x3 -#define TSTORM_ETH_CONN_AG_CTX_CF7_SHIFT 4 -#define TSTORM_ETH_CONN_AG_CTX_CF8_MASK 0x3 -#define TSTORM_ETH_CONN_AG_CTX_CF8_SHIFT 6 +#define E4_TSTORM_ETH_CONN_AG_CTX_CF5_MASK 0x3 /* cf5 */ +#define E4_TSTORM_ETH_CONN_AG_CTX_CF5_SHIFT 0 +#define E4_TSTORM_ETH_CONN_AG_CTX_CF6_MASK 0x3 /* cf6 */ +#define E4_TSTORM_ETH_CONN_AG_CTX_CF6_SHIFT 2 +#define E4_TSTORM_ETH_CONN_AG_CTX_CF7_MASK 0x3 /* cf7 */ +#define E4_TSTORM_ETH_CONN_AG_CTX_CF7_SHIFT 4 +#define E4_TSTORM_ETH_CONN_AG_CTX_CF8_MASK 0x3 /* cf8 */ +#define E4_TSTORM_ETH_CONN_AG_CTX_CF8_SHIFT 6 u8 flags3; -#define TSTORM_ETH_CONN_AG_CTX_CF9_MASK 0x3 -#define TSTORM_ETH_CONN_AG_CTX_CF9_SHIFT 0 -#define TSTORM_ETH_CONN_AG_CTX_CF10_MASK 0x3 -#define TSTORM_ETH_CONN_AG_CTX_CF10_SHIFT 2 -#define TSTORM_ETH_CONN_AG_CTX_CF0EN_MASK 0x1 -#define TSTORM_ETH_CONN_AG_CTX_CF0EN_SHIFT 4 -#define TSTORM_ETH_CONN_AG_CTX_CF1EN_MASK 0x1 -#define TSTORM_ETH_CONN_AG_CTX_CF1EN_SHIFT 5 -#define TSTORM_ETH_CONN_AG_CTX_CF2EN_MASK 0x1 -#define TSTORM_ETH_CONN_AG_CTX_CF2EN_SHIFT 6 -#define TSTORM_ETH_CONN_AG_CTX_CF3EN_MASK 0x1 -#define TSTORM_ETH_CONN_AG_CTX_CF3EN_SHIFT 7 +#define E4_TSTORM_ETH_CONN_AG_CTX_CF9_MASK 0x3 /* cf9 */ +#define E4_TSTORM_ETH_CONN_AG_CTX_CF9_SHIFT 0 +#define E4_TSTORM_ETH_CONN_AG_CTX_CF10_MASK 0x3 /* cf10 */ +#define E4_TSTORM_ETH_CONN_AG_CTX_CF10_SHIFT 2 +#define E4_TSTORM_ETH_CONN_AG_CTX_CF0EN_MASK 0x1 /* cf0en */ +#define E4_TSTORM_ETH_CONN_AG_CTX_CF0EN_SHIFT 4 +#define E4_TSTORM_ETH_CONN_AG_CTX_CF1EN_MASK 0x1 /* cf1en */ +#define E4_TSTORM_ETH_CONN_AG_CTX_CF1EN_SHIFT 5 +#define E4_TSTORM_ETH_CONN_AG_CTX_CF2EN_MASK 0x1 /* cf2en */ +#define E4_TSTORM_ETH_CONN_AG_CTX_CF2EN_SHIFT 6 +#define E4_TSTORM_ETH_CONN_AG_CTX_CF3EN_MASK 0x1 /* cf3en */ +#define E4_TSTORM_ETH_CONN_AG_CTX_CF3EN_SHIFT 7 u8 flags4; -#define TSTORM_ETH_CONN_AG_CTX_CF4EN_MASK 0x1 -#define TSTORM_ETH_CONN_AG_CTX_CF4EN_SHIFT 0 -#define TSTORM_ETH_CONN_AG_CTX_CF5EN_MASK 0x1 -#define TSTORM_ETH_CONN_AG_CTX_CF5EN_SHIFT 1 -#define TSTORM_ETH_CONN_AG_CTX_CF6EN_MASK 0x1 -#define TSTORM_ETH_CONN_AG_CTX_CF6EN_SHIFT 2 -#define TSTORM_ETH_CONN_AG_CTX_CF7EN_MASK 0x1 -#define TSTORM_ETH_CONN_AG_CTX_CF7EN_SHIFT 3 -#define TSTORM_ETH_CONN_AG_CTX_CF8EN_MASK 0x1 -#define TSTORM_ETH_CONN_AG_CTX_CF8EN_SHIFT 4 -#define TSTORM_ETH_CONN_AG_CTX_CF9EN_MASK 0x1 -#define TSTORM_ETH_CONN_AG_CTX_CF9EN_SHIFT 5 -#define TSTORM_ETH_CONN_AG_CTX_CF10EN_MASK 0x1 -#define TSTORM_ETH_CONN_AG_CTX_CF10EN_SHIFT 6 -#define TSTORM_ETH_CONN_AG_CTX_RULE0EN_MASK 0x1 -#define TSTORM_ETH_CONN_AG_CTX_RULE0EN_SHIFT 7 +#define E4_TSTORM_ETH_CONN_AG_CTX_CF4EN_MASK 0x1 /* cf4en */ +#define E4_TSTORM_ETH_CONN_AG_CTX_CF4EN_SHIFT 0 +#define E4_TSTORM_ETH_CONN_AG_CTX_CF5EN_MASK 0x1 /* cf5en */ +#define E4_TSTORM_ETH_CONN_AG_CTX_CF5EN_SHIFT 1 +#define E4_TSTORM_ETH_CONN_AG_CTX_CF6EN_MASK 0x1 /* cf6en */ +#define E4_TSTORM_ETH_CONN_AG_CTX_CF6EN_SHIFT 2 +#define E4_TSTORM_ETH_CONN_AG_CTX_CF7EN_MASK 0x1 /* cf7en */ +#define E4_TSTORM_ETH_CONN_AG_CTX_CF7EN_SHIFT 3 +#define E4_TSTORM_ETH_CONN_AG_CTX_CF8EN_MASK 0x1 /* cf8en */ +#define E4_TSTORM_ETH_CONN_AG_CTX_CF8EN_SHIFT 4 +#define E4_TSTORM_ETH_CONN_AG_CTX_CF9EN_MASK 0x1 /* cf9en */ +#define E4_TSTORM_ETH_CONN_AG_CTX_CF9EN_SHIFT 5 +#define E4_TSTORM_ETH_CONN_AG_CTX_CF10EN_MASK 0x1 /* cf10en */ +#define E4_TSTORM_ETH_CONN_AG_CTX_CF10EN_SHIFT 6 +#define E4_TSTORM_ETH_CONN_AG_CTX_RULE0EN_MASK 0x1 /* rule0en */ +#define E4_TSTORM_ETH_CONN_AG_CTX_RULE0EN_SHIFT 7 u8 flags5; -#define TSTORM_ETH_CONN_AG_CTX_RULE1EN_MASK 0x1 -#define TSTORM_ETH_CONN_AG_CTX_RULE1EN_SHIFT 0 -#define TSTORM_ETH_CONN_AG_CTX_RULE2EN_MASK 0x1 -#define TSTORM_ETH_CONN_AG_CTX_RULE2EN_SHIFT 1 -#define TSTORM_ETH_CONN_AG_CTX_RULE3EN_MASK 0x1 -#define TSTORM_ETH_CONN_AG_CTX_RULE3EN_SHIFT 2 -#define TSTORM_ETH_CONN_AG_CTX_RULE4EN_MASK 0x1 -#define TSTORM_ETH_CONN_AG_CTX_RULE4EN_SHIFT 3 -#define TSTORM_ETH_CONN_AG_CTX_RULE5EN_MASK 0x1 -#define TSTORM_ETH_CONN_AG_CTX_RULE5EN_SHIFT 4 -#define TSTORM_ETH_CONN_AG_CTX_RX_BD_EN_MASK 0x1 -#define TSTORM_ETH_CONN_AG_CTX_RX_BD_EN_SHIFT 5 -#define TSTORM_ETH_CONN_AG_CTX_RULE7EN_MASK 0x1 -#define TSTORM_ETH_CONN_AG_CTX_RULE7EN_SHIFT 6 -#define TSTORM_ETH_CONN_AG_CTX_RULE8EN_MASK 0x1 -#define TSTORM_ETH_CONN_AG_CTX_RULE8EN_SHIFT 7 +#define E4_TSTORM_ETH_CONN_AG_CTX_RULE1EN_MASK 0x1 /* rule1en */ +#define E4_TSTORM_ETH_CONN_AG_CTX_RULE1EN_SHIFT 0 +#define E4_TSTORM_ETH_CONN_AG_CTX_RULE2EN_MASK 0x1 /* rule2en */ +#define E4_TSTORM_ETH_CONN_AG_CTX_RULE2EN_SHIFT 1 +#define E4_TSTORM_ETH_CONN_AG_CTX_RULE3EN_MASK 0x1 /* rule3en */ +#define E4_TSTORM_ETH_CONN_AG_CTX_RULE3EN_SHIFT 2 +#define E4_TSTORM_ETH_CONN_AG_CTX_RULE4EN_MASK 0x1 /* rule4en */ +#define E4_TSTORM_ETH_CONN_AG_CTX_RULE4EN_SHIFT 3 +#define E4_TSTORM_ETH_CONN_AG_CTX_RULE5EN_MASK 0x1 /* rule5en */ +#define E4_TSTORM_ETH_CONN_AG_CTX_RULE5EN_SHIFT 4 +#define E4_TSTORM_ETH_CONN_AG_CTX_RX_BD_EN_MASK 0x1 /* rule6en */ +#define E4_TSTORM_ETH_CONN_AG_CTX_RX_BD_EN_SHIFT 5 +#define E4_TSTORM_ETH_CONN_AG_CTX_RULE7EN_MASK 0x1 /* rule7en */ +#define E4_TSTORM_ETH_CONN_AG_CTX_RULE7EN_SHIFT 6 +#define E4_TSTORM_ETH_CONN_AG_CTX_RULE8EN_MASK 0x1 /* rule8en */ +#define E4_TSTORM_ETH_CONN_AG_CTX_RULE8EN_SHIFT 7 __le32 reg0 /* reg0 */; __le32 reg1 /* reg1 */; __le32 reg2 /* reg2 */; @@ -452,63 +558,88 @@ struct tstorm_eth_conn_ag_ctx { __le32 reg10 /* reg10 */; }; -struct ustorm_eth_conn_ag_ctx { +struct e4_ustorm_eth_conn_ag_ctx { u8 byte0 /* cdu_validation */; u8 byte1 /* state */; u8 flags0; -#define USTORM_ETH_CONN_AG_CTX_BIT0_MASK 0x1 -#define USTORM_ETH_CONN_AG_CTX_BIT0_SHIFT 0 -#define USTORM_ETH_CONN_AG_CTX_BIT1_MASK 0x1 -#define USTORM_ETH_CONN_AG_CTX_BIT1_SHIFT 1 -#define USTORM_ETH_CONN_AG_CTX_TX_PMD_TERMINATE_CF_MASK 0x3 -#define USTORM_ETH_CONN_AG_CTX_TX_PMD_TERMINATE_CF_SHIFT 2 -#define USTORM_ETH_CONN_AG_CTX_RX_PMD_TERMINATE_CF_MASK 0x3 -#define USTORM_ETH_CONN_AG_CTX_RX_PMD_TERMINATE_CF_SHIFT 4 -#define USTORM_ETH_CONN_AG_CTX_CF2_MASK 0x3 -#define USTORM_ETH_CONN_AG_CTX_CF2_SHIFT 6 +/* exist_in_qm0 */ +#define E4_USTORM_ETH_CONN_AG_CTX_BIT0_MASK 0x1 +#define E4_USTORM_ETH_CONN_AG_CTX_BIT0_SHIFT 0 +/* exist_in_qm1 */ +#define E4_USTORM_ETH_CONN_AG_CTX_BIT1_MASK 0x1 +#define E4_USTORM_ETH_CONN_AG_CTX_BIT1_SHIFT 1 +/* timer0cf */ +#define E4_USTORM_ETH_CONN_AG_CTX_TX_PMD_TERMINATE_CF_MASK 0x3 +#define E4_USTORM_ETH_CONN_AG_CTX_TX_PMD_TERMINATE_CF_SHIFT 2 +/* timer1cf */ +#define E4_USTORM_ETH_CONN_AG_CTX_RX_PMD_TERMINATE_CF_MASK 0x3 +#define E4_USTORM_ETH_CONN_AG_CTX_RX_PMD_TERMINATE_CF_SHIFT 4 +/* timer2cf */ +#define E4_USTORM_ETH_CONN_AG_CTX_CF2_MASK 0x3 +#define E4_USTORM_ETH_CONN_AG_CTX_CF2_SHIFT 6 u8 flags1; -#define USTORM_ETH_CONN_AG_CTX_CF3_MASK 0x3 -#define USTORM_ETH_CONN_AG_CTX_CF3_SHIFT 0 -#define USTORM_ETH_CONN_AG_CTX_TX_ARM_CF_MASK 0x3 -#define USTORM_ETH_CONN_AG_CTX_TX_ARM_CF_SHIFT 2 -#define USTORM_ETH_CONN_AG_CTX_RX_ARM_CF_MASK 0x3 -#define USTORM_ETH_CONN_AG_CTX_RX_ARM_CF_SHIFT 4 -#define USTORM_ETH_CONN_AG_CTX_TX_BD_CONS_UPD_CF_MASK 0x3 -#define USTORM_ETH_CONN_AG_CTX_TX_BD_CONS_UPD_CF_SHIFT 6 +/* timer_stop_all */ +#define E4_USTORM_ETH_CONN_AG_CTX_CF3_MASK 0x3 +#define E4_USTORM_ETH_CONN_AG_CTX_CF3_SHIFT 0 +/* cf4 */ +#define E4_USTORM_ETH_CONN_AG_CTX_TX_ARM_CF_MASK 0x3 +#define E4_USTORM_ETH_CONN_AG_CTX_TX_ARM_CF_SHIFT 2 +/* cf5 */ +#define E4_USTORM_ETH_CONN_AG_CTX_RX_ARM_CF_MASK 0x3 +#define E4_USTORM_ETH_CONN_AG_CTX_RX_ARM_CF_SHIFT 4 +/* cf6 */ +#define E4_USTORM_ETH_CONN_AG_CTX_TX_BD_CONS_UPD_CF_MASK 0x3 +#define E4_USTORM_ETH_CONN_AG_CTX_TX_BD_CONS_UPD_CF_SHIFT 6 u8 flags2; -#define USTORM_ETH_CONN_AG_CTX_TX_PMD_TERMINATE_CF_EN_MASK 0x1 -#define USTORM_ETH_CONN_AG_CTX_TX_PMD_TERMINATE_CF_EN_SHIFT 0 -#define USTORM_ETH_CONN_AG_CTX_RX_PMD_TERMINATE_CF_EN_MASK 0x1 -#define USTORM_ETH_CONN_AG_CTX_RX_PMD_TERMINATE_CF_EN_SHIFT 1 -#define USTORM_ETH_CONN_AG_CTX_CF2EN_MASK 0x1 -#define USTORM_ETH_CONN_AG_CTX_CF2EN_SHIFT 2 -#define USTORM_ETH_CONN_AG_CTX_CF3EN_MASK 0x1 -#define USTORM_ETH_CONN_AG_CTX_CF3EN_SHIFT 3 -#define USTORM_ETH_CONN_AG_CTX_TX_ARM_CF_EN_MASK 0x1 -#define USTORM_ETH_CONN_AG_CTX_TX_ARM_CF_EN_SHIFT 4 -#define USTORM_ETH_CONN_AG_CTX_RX_ARM_CF_EN_MASK 0x1 -#define USTORM_ETH_CONN_AG_CTX_RX_ARM_CF_EN_SHIFT 5 -#define USTORM_ETH_CONN_AG_CTX_TX_BD_CONS_UPD_CF_EN_MASK 0x1 -#define USTORM_ETH_CONN_AG_CTX_TX_BD_CONS_UPD_CF_EN_SHIFT 6 -#define USTORM_ETH_CONN_AG_CTX_RULE0EN_MASK 0x1 -#define USTORM_ETH_CONN_AG_CTX_RULE0EN_SHIFT 7 +/* cf0en */ +#define E4_USTORM_ETH_CONN_AG_CTX_TX_PMD_TERMINATE_CF_EN_MASK 0x1 +#define E4_USTORM_ETH_CONN_AG_CTX_TX_PMD_TERMINATE_CF_EN_SHIFT 0 +/* cf1en */ +#define E4_USTORM_ETH_CONN_AG_CTX_RX_PMD_TERMINATE_CF_EN_MASK 0x1 +#define E4_USTORM_ETH_CONN_AG_CTX_RX_PMD_TERMINATE_CF_EN_SHIFT 1 +/* cf2en */ +#define E4_USTORM_ETH_CONN_AG_CTX_CF2EN_MASK 0x1 +#define E4_USTORM_ETH_CONN_AG_CTX_CF2EN_SHIFT 2 +/* cf3en */ +#define E4_USTORM_ETH_CONN_AG_CTX_CF3EN_MASK 0x1 +#define E4_USTORM_ETH_CONN_AG_CTX_CF3EN_SHIFT 3 +/* cf4en */ +#define E4_USTORM_ETH_CONN_AG_CTX_TX_ARM_CF_EN_MASK 0x1 +#define E4_USTORM_ETH_CONN_AG_CTX_TX_ARM_CF_EN_SHIFT 4 +/* cf5en */ +#define E4_USTORM_ETH_CONN_AG_CTX_RX_ARM_CF_EN_MASK 0x1 +#define E4_USTORM_ETH_CONN_AG_CTX_RX_ARM_CF_EN_SHIFT 5 +/* cf6en */ +#define E4_USTORM_ETH_CONN_AG_CTX_TX_BD_CONS_UPD_CF_EN_MASK 0x1 +#define E4_USTORM_ETH_CONN_AG_CTX_TX_BD_CONS_UPD_CF_EN_SHIFT 6 +/* rule0en */ +#define E4_USTORM_ETH_CONN_AG_CTX_RULE0EN_MASK 0x1 +#define E4_USTORM_ETH_CONN_AG_CTX_RULE0EN_SHIFT 7 u8 flags3; -#define USTORM_ETH_CONN_AG_CTX_RULE1EN_MASK 0x1 -#define USTORM_ETH_CONN_AG_CTX_RULE1EN_SHIFT 0 -#define USTORM_ETH_CONN_AG_CTX_RULE2EN_MASK 0x1 -#define USTORM_ETH_CONN_AG_CTX_RULE2EN_SHIFT 1 -#define USTORM_ETH_CONN_AG_CTX_RULE3EN_MASK 0x1 -#define USTORM_ETH_CONN_AG_CTX_RULE3EN_SHIFT 2 -#define USTORM_ETH_CONN_AG_CTX_RULE4EN_MASK 0x1 -#define USTORM_ETH_CONN_AG_CTX_RULE4EN_SHIFT 3 -#define USTORM_ETH_CONN_AG_CTX_RULE5EN_MASK 0x1 -#define USTORM_ETH_CONN_AG_CTX_RULE5EN_SHIFT 4 -#define USTORM_ETH_CONN_AG_CTX_RULE6EN_MASK 0x1 -#define USTORM_ETH_CONN_AG_CTX_RULE6EN_SHIFT 5 -#define USTORM_ETH_CONN_AG_CTX_RULE7EN_MASK 0x1 -#define USTORM_ETH_CONN_AG_CTX_RULE7EN_SHIFT 6 -#define USTORM_ETH_CONN_AG_CTX_RULE8EN_MASK 0x1 -#define USTORM_ETH_CONN_AG_CTX_RULE8EN_SHIFT 7 +/* rule1en */ +#define E4_USTORM_ETH_CONN_AG_CTX_RULE1EN_MASK 0x1 +#define E4_USTORM_ETH_CONN_AG_CTX_RULE1EN_SHIFT 0 +/* rule2en */ +#define E4_USTORM_ETH_CONN_AG_CTX_RULE2EN_MASK 0x1 +#define E4_USTORM_ETH_CONN_AG_CTX_RULE2EN_SHIFT 1 +/* rule3en */ +#define E4_USTORM_ETH_CONN_AG_CTX_RULE3EN_MASK 0x1 +#define E4_USTORM_ETH_CONN_AG_CTX_RULE3EN_SHIFT 2 +/* rule4en */ +#define E4_USTORM_ETH_CONN_AG_CTX_RULE4EN_MASK 0x1 +#define E4_USTORM_ETH_CONN_AG_CTX_RULE4EN_SHIFT 3 +/* rule5en */ +#define E4_USTORM_ETH_CONN_AG_CTX_RULE5EN_MASK 0x1 +#define E4_USTORM_ETH_CONN_AG_CTX_RULE5EN_SHIFT 4 +/* rule6en */ +#define E4_USTORM_ETH_CONN_AG_CTX_RULE6EN_MASK 0x1 +#define E4_USTORM_ETH_CONN_AG_CTX_RULE6EN_SHIFT 5 +/* rule7en */ +#define E4_USTORM_ETH_CONN_AG_CTX_RULE7EN_MASK 0x1 +#define E4_USTORM_ETH_CONN_AG_CTX_RULE7EN_SHIFT 6 +/* rule8en */ +#define E4_USTORM_ETH_CONN_AG_CTX_RULE8EN_MASK 0x1 +#define E4_USTORM_ETH_CONN_AG_CTX_RULE8EN_SHIFT 7 u8 byte2 /* byte2 */; u8 byte3 /* byte3 */; __le16 word0 /* conn_dpi */; @@ -538,84 +669,81 @@ struct mstorm_eth_conn_st_ctx { /* * eth connection context */ -struct eth_conn_context { - struct tstorm_eth_conn_st_ctx tstorm_st_context - /* tstorm storm context */; +struct e4_eth_conn_context { +/* tstorm storm context */ + struct tstorm_eth_conn_st_ctx tstorm_st_context; struct regpair tstorm_st_padding[2] /* padding */; - struct pstorm_eth_conn_st_ctx pstorm_st_context - /* pstorm storm context */; - struct xstorm_eth_conn_st_ctx xstorm_st_context - /* xstorm storm context */; - struct xstorm_eth_conn_ag_ctx xstorm_ag_context - /* xstorm aggregative context */; - struct ystorm_eth_conn_st_ctx ystorm_st_context - /* ystorm storm context */; - struct ystorm_eth_conn_ag_ctx ystorm_ag_context - /* ystorm aggregative context */; - struct tstorm_eth_conn_ag_ctx tstorm_ag_context - /* tstorm aggregative context */; - struct ustorm_eth_conn_ag_ctx ustorm_ag_context - /* ustorm aggregative context */; - struct ustorm_eth_conn_st_ctx ustorm_st_context - /* ustorm storm context */; - struct mstorm_eth_conn_st_ctx mstorm_st_context - /* mstorm storm context */; +/* pstorm storm context */ + struct pstorm_eth_conn_st_ctx pstorm_st_context; +/* xstorm storm context */ + struct xstorm_eth_conn_st_ctx xstorm_st_context; +/* xstorm aggregative context */ + struct e4_xstorm_eth_conn_ag_ctx xstorm_ag_context; +/* ystorm storm context */ + struct ystorm_eth_conn_st_ctx ystorm_st_context; +/* ystorm aggregative context */ + struct e4_ystorm_eth_conn_ag_ctx ystorm_ag_context; +/* tstorm aggregative context */ + struct e4_tstorm_eth_conn_ag_ctx tstorm_ag_context; +/* ustorm aggregative context */ + struct e4_ustorm_eth_conn_ag_ctx ustorm_ag_context; +/* ustorm storm context */ + struct ustorm_eth_conn_st_ctx ustorm_st_context; +/* mstorm storm context */ + struct mstorm_eth_conn_st_ctx mstorm_st_context; }; + /* * Ethernet filter types: mac/vlan/pair */ enum eth_error_code { ETH_OK = 0x00 /* command succeeded */, - ETH_FILTERS_MAC_ADD_FAIL_FULL - /* mac add filters command failed due to cam full state */, - ETH_FILTERS_MAC_ADD_FAIL_FULL_MTT2 - /* mac add filters command failed due to mtt2 full state */, - ETH_FILTERS_MAC_ADD_FAIL_DUP_MTT2 - /* mac add filters command failed due to duplicate mac address */, - ETH_FILTERS_MAC_ADD_FAIL_DUP_STT2 - /* mac add filters command failed due to duplicate mac address */, - ETH_FILTERS_MAC_DEL_FAIL_NOF - /* mac delete filters command failed due to not found state */, - ETH_FILTERS_MAC_DEL_FAIL_NOF_MTT2 - /* mac delete filters command failed due to not found state */, - ETH_FILTERS_MAC_DEL_FAIL_NOF_STT2 - /* mac delete filters command failed due to not found state */, - ETH_FILTERS_MAC_ADD_FAIL_ZERO_MAC - /* mac add filters command failed due to MAC Address of - * 00:00:00:00:00:00 - */ - , - ETH_FILTERS_VLAN_ADD_FAIL_FULL - /* vlan add filters command failed due to cam full state */, - ETH_FILTERS_VLAN_ADD_FAIL_DUP - /* vlan add filters command failed due to duplicate VLAN filter */, - ETH_FILTERS_VLAN_DEL_FAIL_NOF - /* vlan delete filters command failed due to not found state */, - ETH_FILTERS_VLAN_DEL_FAIL_NOF_TT1 - /* vlan delete filters command failed due to not found state */, - ETH_FILTERS_PAIR_ADD_FAIL_DUP - /* pair add filters command failed due to duplicate request */, - ETH_FILTERS_PAIR_ADD_FAIL_FULL - /* pair add filters command failed due to full state */, - ETH_FILTERS_PAIR_ADD_FAIL_FULL_MAC - /* pair add filters command failed due to full state */, - ETH_FILTERS_PAIR_DEL_FAIL_NOF - /* pair add filters command failed due not found state */, - ETH_FILTERS_PAIR_DEL_FAIL_NOF_TT1 - /* pair add filters command failed due not found state */, - ETH_FILTERS_PAIR_ADD_FAIL_ZERO_MAC - /* pair add filters command failed due to MAC Address of - * 00:00:00:00:00:00 - */ - , - ETH_FILTERS_VNI_ADD_FAIL_FULL - /* vni add filters command failed due to cam full state */, - ETH_FILTERS_VNI_ADD_FAIL_DUP - /* vni add filters command failed due to duplicate VNI filter */, +/* mac add filters command failed due to cam full state */ + ETH_FILTERS_MAC_ADD_FAIL_FULL, +/* mac add filters command failed due to mtt2 full state */ + ETH_FILTERS_MAC_ADD_FAIL_FULL_MTT2, +/* mac add filters command failed due to duplicate mac address */ + ETH_FILTERS_MAC_ADD_FAIL_DUP_MTT2, +/* mac add filters command failed due to duplicate mac address */ + ETH_FILTERS_MAC_ADD_FAIL_DUP_STT2, +/* mac delete filters command failed due to not found state */ + ETH_FILTERS_MAC_DEL_FAIL_NOF, +/* mac delete filters command failed due to not found state */ + ETH_FILTERS_MAC_DEL_FAIL_NOF_MTT2, +/* mac delete filters command failed due to not found state */ + ETH_FILTERS_MAC_DEL_FAIL_NOF_STT2, +/* mac add filters command failed due to MAC Address of 00:00:00:00:00:00 */ + ETH_FILTERS_MAC_ADD_FAIL_ZERO_MAC, +/* vlan add filters command failed due to cam full state */ + ETH_FILTERS_VLAN_ADD_FAIL_FULL, +/* vlan add filters command failed due to duplicate VLAN filter */ + ETH_FILTERS_VLAN_ADD_FAIL_DUP, +/* vlan delete filters command failed due to not found state */ + ETH_FILTERS_VLAN_DEL_FAIL_NOF, +/* vlan delete filters command failed due to not found state */ + ETH_FILTERS_VLAN_DEL_FAIL_NOF_TT1, +/* pair add filters command failed due to duplicate request */ + ETH_FILTERS_PAIR_ADD_FAIL_DUP, +/* pair add filters command failed due to full state */ + ETH_FILTERS_PAIR_ADD_FAIL_FULL, +/* pair add filters command failed due to full state */ + ETH_FILTERS_PAIR_ADD_FAIL_FULL_MAC, +/* pair add filters command failed due not found state */ + ETH_FILTERS_PAIR_DEL_FAIL_NOF, +/* pair add filters command failed due not found state */ + ETH_FILTERS_PAIR_DEL_FAIL_NOF_TT1, +/* pair add filters command failed due to MAC Address of 00:00:00:00:00:00 */ + ETH_FILTERS_PAIR_ADD_FAIL_ZERO_MAC, +/* vni add filters command failed due to cam full state */ + ETH_FILTERS_VNI_ADD_FAIL_FULL, +/* vni add filters command failed due to duplicate VNI filter */ + ETH_FILTERS_VNI_ADD_FAIL_DUP, + ETH_FILTERS_GFT_UPDATE_FAIL /* Fail update GFT filter. */, MAX_ETH_ERROR_CODE }; + /* * opcodes for the event ring */ @@ -635,12 +763,13 @@ enum eth_event_opcode { ETH_EVENT_RX_CREATE_OPENFLOW_ACTION, ETH_EVENT_RX_ADD_UDP_FILTER, ETH_EVENT_RX_DELETE_UDP_FILTER, - ETH_EVENT_RX_ADD_GFT_FILTER, - ETH_EVENT_RX_DELETE_GFT_FILTER, ETH_EVENT_RX_CREATE_GFT_ACTION, + ETH_EVENT_RX_GFT_UPDATE_FILTER, + ETH_EVENT_TX_QUEUE_UPDATE, MAX_ETH_EVENT_OPCODE }; + /* * Classify rule types in E2/E3 */ @@ -648,11 +777,12 @@ enum eth_filter_action { ETH_FILTER_ACTION_UNUSED, ETH_FILTER_ACTION_REMOVE, ETH_FILTER_ACTION_ADD, - ETH_FILTER_ACTION_REMOVE_ALL - /* Remove all filters of given type and vport ID. */, +/* Remove all filters of given type and vport ID. */ + ETH_FILTER_ACTION_REMOVE_ALL, MAX_ETH_FILTER_ACTION }; + /* * Command for adding/removing a classification rule $$KEEP_ENDIANNESS$$ */ @@ -668,6 +798,7 @@ struct eth_filter_cmd { __le16 vlan_id; }; + /* * $$KEEP_ENDIANNESS$$ */ @@ -675,10 +806,14 @@ struct eth_filter_cmd_header { u8 rx /* If set, apply these commands to the RX path */; u8 tx /* If set, apply these commands to the TX path */; u8 cmd_cnt /* Number of filter commands */; +/* 0 - dont assert in case of filter configuration error. Just return an error + * code. 1 - assert in case of filter configuration error. + */ u8 assert_on_error; u8 reserved1[4]; }; + /* * Ethernet filter types: mac/vlan/pair */ @@ -690,25 +825,27 @@ enum eth_filter_type { ETH_FILTER_TYPE_INNER_MAC /* Add/remove a inner MAC address */, ETH_FILTER_TYPE_INNER_VLAN /* Add/remove a inner VLAN */, ETH_FILTER_TYPE_INNER_PAIR /* Add/remove a inner MAC-VLAN pair */, - ETH_FILTER_TYPE_INNER_MAC_VNI_PAIR /* Add/remove a inner MAC-VNI pair */ - , +/* Add/remove a inner MAC-VNI pair */ + ETH_FILTER_TYPE_INNER_MAC_VNI_PAIR, ETH_FILTER_TYPE_MAC_VNI_PAIR /* Add/remove a MAC-VNI pair */, ETH_FILTER_TYPE_VNI /* Add/remove a VNI */, MAX_ETH_FILTER_TYPE }; + /* * eth IPv4 Fragment Type */ enum eth_ipv4_frag_type { ETH_IPV4_NOT_FRAG /* IPV4 Packet Not Fragmented */, - ETH_IPV4_FIRST_FRAG - /* First Fragment of IPv4 Packet (contains headers) */, - ETH_IPV4_NON_FIRST_FRAG - /* Non-First Fragment of IPv4 Packet (does not contain headers) */, +/* First Fragment of IPv4 Packet (contains headers) */ + ETH_IPV4_FIRST_FRAG, +/* Non-First Fragment of IPv4 Packet (does not contain headers) */ + ETH_IPV4_NON_FIRST_FRAG, MAX_ETH_IPV4_FRAG_TYPE }; + /* * eth IPv4 Fragment Type */ @@ -718,6 +855,7 @@ enum eth_ip_type { MAX_ETH_IP_TYPE }; + /* * Ethernet Ramrod Command IDs */ @@ -732,105 +870,135 @@ enum eth_ramrod_cmd_id { ETH_RAMROD_TX_QUEUE_STOP /* TX Queue Stop Ramrod */, ETH_RAMROD_FILTERS_UPDATE /* Add or Remove Mac/Vlan/Pair filters */, ETH_RAMROD_RX_QUEUE_UPDATE /* RX Queue Update Ramrod */, - ETH_RAMROD_RX_CREATE_OPENFLOW_ACTION - /* RX - Create an Openflow Action */, - ETH_RAMROD_RX_ADD_OPENFLOW_FILTER - /* RX - Add an Openflow Filter to the Searcher */, - ETH_RAMROD_RX_DELETE_OPENFLOW_FILTER - /* RX - Delete an Openflow Filter to the Searcher */, - ETH_RAMROD_RX_ADD_UDP_FILTER /* RX - Add a UDP Filter to the Searcher */ - , - ETH_RAMROD_RX_DELETE_UDP_FILTER - /* RX - Delete a UDP Filter to the Searcher */, - ETH_RAMROD_RX_CREATE_GFT_ACTION /* RX - Create an Gft Action */, - ETH_RAMROD_RX_DELETE_GFT_FILTER - /* RX - Delete an GFT Filter to the Searcher */, - ETH_RAMROD_RX_ADD_GFT_FILTER - /* RX - Add an GFT Filter to the Searcher */, +/* RX - Create an Openflow Action */ + ETH_RAMROD_RX_CREATE_OPENFLOW_ACTION, +/* RX - Add an Openflow Filter to the Searcher */ + ETH_RAMROD_RX_ADD_OPENFLOW_FILTER, +/* RX - Delete an Openflow Filter to the Searcher */ + ETH_RAMROD_RX_DELETE_OPENFLOW_FILTER, +/* RX - Add a UDP Filter to the Searcher */ + ETH_RAMROD_RX_ADD_UDP_FILTER, +/* RX - Delete a UDP Filter to the Searcher */ + ETH_RAMROD_RX_DELETE_UDP_FILTER, + ETH_RAMROD_RX_CREATE_GFT_ACTION /* RX - Create a Gft Action */, +/* RX - Add/Delete a GFT Filter to the Searcher */ + ETH_RAMROD_GFT_UPDATE_FILTER, + ETH_RAMROD_TX_QUEUE_UPDATE /* TX Queue Update Ramrod */, MAX_ETH_RAMROD_CMD_ID }; + /* * return code from eth sp ramrods */ struct eth_return_code { u8 value; +/* error code (use enum eth_error_code) */ #define ETH_RETURN_CODE_ERR_CODE_MASK 0x1F #define ETH_RETURN_CODE_ERR_CODE_SHIFT 0 #define ETH_RETURN_CODE_RESERVED_MASK 0x3 #define ETH_RETURN_CODE_RESERVED_SHIFT 5 +/* rx path - 0, tx path - 1 */ #define ETH_RETURN_CODE_RX_TX_MASK 0x1 #define ETH_RETURN_CODE_RX_TX_SHIFT 7 }; + /* * What to do in case an error occurs */ enum eth_tx_err { ETH_TX_ERR_DROP /* Drop erroneous packet. */, - ETH_TX_ERR_ASSERT_MALICIOUS - /* Assert an interrupt for PF, declare as malicious for VF */, +/* Assert an interrupt for PF, declare as malicious for VF */ + ETH_TX_ERR_ASSERT_MALICIOUS, MAX_ETH_TX_ERR }; + /* * Array of the different error type behaviors */ struct eth_tx_err_vals { __le16 values; +/* Wrong VLAN insertion mode (use enum eth_tx_err) */ #define ETH_TX_ERR_VALS_ILLEGAL_VLAN_MODE_MASK 0x1 #define ETH_TX_ERR_VALS_ILLEGAL_VLAN_MODE_SHIFT 0 +/* Packet is below minimal size (use enum eth_tx_err) */ #define ETH_TX_ERR_VALS_PACKET_TOO_SMALL_MASK 0x1 #define ETH_TX_ERR_VALS_PACKET_TOO_SMALL_SHIFT 1 +/* Vport has sent spoofed packet (use enum eth_tx_err) */ #define ETH_TX_ERR_VALS_ANTI_SPOOFING_ERR_MASK 0x1 #define ETH_TX_ERR_VALS_ANTI_SPOOFING_ERR_SHIFT 2 +/* Packet with illegal type of inband tag (use enum eth_tx_err) */ #define ETH_TX_ERR_VALS_ILLEGAL_INBAND_TAGS_MASK 0x1 #define ETH_TX_ERR_VALS_ILLEGAL_INBAND_TAGS_SHIFT 3 +/* Packet marked for VLAN insertion when inband tag is present + * (use enum eth_tx_err) + */ #define ETH_TX_ERR_VALS_VLAN_INSERTION_W_INBAND_TAG_MASK 0x1 #define ETH_TX_ERR_VALS_VLAN_INSERTION_W_INBAND_TAG_SHIFT 4 +/* Non LSO packet larger than MTU (use enum eth_tx_err) */ #define ETH_TX_ERR_VALS_MTU_VIOLATION_MASK 0x1 #define ETH_TX_ERR_VALS_MTU_VIOLATION_SHIFT 5 +/* VF/PF has sent LLDP/PFC or any other type of control packet which is not + * allowed to (use enum eth_tx_err) + */ #define ETH_TX_ERR_VALS_ILLEGAL_CONTROL_FRAME_MASK 0x1 #define ETH_TX_ERR_VALS_ILLEGAL_CONTROL_FRAME_SHIFT 6 #define ETH_TX_ERR_VALS_RESERVED_MASK 0x1FF #define ETH_TX_ERR_VALS_RESERVED_SHIFT 7 }; + /* * vport rss configuration data */ struct eth_vport_rss_config { __le16 capabilities; +/* configuration of the IpV4 2-tuple capability */ #define ETH_VPORT_RSS_CONFIG_IPV4_CAPABILITY_MASK 0x1 #define ETH_VPORT_RSS_CONFIG_IPV4_CAPABILITY_SHIFT 0 +/* configuration of the IpV6 2-tuple capability */ #define ETH_VPORT_RSS_CONFIG_IPV6_CAPABILITY_MASK 0x1 #define ETH_VPORT_RSS_CONFIG_IPV6_CAPABILITY_SHIFT 1 +/* configuration of the IpV4 4-tuple capability for TCP */ #define ETH_VPORT_RSS_CONFIG_IPV4_TCP_CAPABILITY_MASK 0x1 #define ETH_VPORT_RSS_CONFIG_IPV4_TCP_CAPABILITY_SHIFT 2 +/* configuration of the IpV6 4-tuple capability for TCP */ #define ETH_VPORT_RSS_CONFIG_IPV6_TCP_CAPABILITY_MASK 0x1 #define ETH_VPORT_RSS_CONFIG_IPV6_TCP_CAPABILITY_SHIFT 3 +/* configuration of the IpV4 4-tuple capability for UDP */ #define ETH_VPORT_RSS_CONFIG_IPV4_UDP_CAPABILITY_MASK 0x1 #define ETH_VPORT_RSS_CONFIG_IPV4_UDP_CAPABILITY_SHIFT 4 +/* configuration of the IpV6 4-tuple capability for UDP */ #define ETH_VPORT_RSS_CONFIG_IPV6_UDP_CAPABILITY_MASK 0x1 #define ETH_VPORT_RSS_CONFIG_IPV6_UDP_CAPABILITY_SHIFT 5 +/* configuration of the 5-tuple capability */ #define ETH_VPORT_RSS_CONFIG_EN_5_TUPLE_CAPABILITY_MASK 0x1 #define ETH_VPORT_RSS_CONFIG_EN_5_TUPLE_CAPABILITY_SHIFT 6 +/* if set update the rss keys */ #define ETH_VPORT_RSS_CONFIG_RESERVED0_MASK 0x1FF #define ETH_VPORT_RSS_CONFIG_RESERVED0_SHIFT 7 +/* The RSS engine ID. Must be allocated to each vport with RSS enabled. + * Total number of RSS engines is ETH_RSS_ENGINE_NUM_ , according to chip type. + */ u8 rss_id; u8 rss_mode /* The RSS mode for this function */; u8 update_rss_key /* if set update the rss key */; - u8 update_rss_ind_table /* if set update the indirection table */; - u8 update_rss_capabilities /* if set update the capabilities */; +/* if set update the indirection table values */ + u8 update_rss_ind_table; +/* if set update the capabilities and indirection table size. */ + u8 update_rss_capabilities; u8 tbl_size /* rss mask (Tbl size) */; __le32 reserved2[2]; - __le16 indirection_table[ETH_RSS_IND_TABLE_ENTRIES_NUM] - /* RSS indirection table */; - __le32 rss_key[ETH_RSS_KEY_SIZE_REGS] /* RSS key supplied to us by OS */ - ; +/* RSS indirection table */ + __le16 indirection_table[ETH_RSS_IND_TABLE_ENTRIES_NUM]; +/* RSS key supplied to us by OS */ + __le32 rss_key[ETH_RSS_KEY_SIZE_REGS]; __le32 reserved3[2]; }; + /* * eth vport RSS mode */ @@ -840,21 +1008,28 @@ enum eth_vport_rss_mode { MAX_ETH_VPORT_RSS_MODE }; + /* * Command for setting classification flags for a vport $$KEEP_ENDIANNESS$$ */ struct eth_vport_rx_mode { __le16 state; +/* drop all unicast packets */ #define ETH_VPORT_RX_MODE_UCAST_DROP_ALL_MASK 0x1 #define ETH_VPORT_RX_MODE_UCAST_DROP_ALL_SHIFT 0 +/* accept all unicast packets (subject to vlan) */ #define ETH_VPORT_RX_MODE_UCAST_ACCEPT_ALL_MASK 0x1 #define ETH_VPORT_RX_MODE_UCAST_ACCEPT_ALL_SHIFT 1 +/* accept all unmatched unicast packets */ #define ETH_VPORT_RX_MODE_UCAST_ACCEPT_UNMATCHED_MASK 0x1 #define ETH_VPORT_RX_MODE_UCAST_ACCEPT_UNMATCHED_SHIFT 2 +/* drop all multicast packets */ #define ETH_VPORT_RX_MODE_MCAST_DROP_ALL_MASK 0x1 #define ETH_VPORT_RX_MODE_MCAST_DROP_ALL_SHIFT 3 +/* accept all multicast packets (subject to vlan) */ #define ETH_VPORT_RX_MODE_MCAST_ACCEPT_ALL_MASK 0x1 #define ETH_VPORT_RX_MODE_MCAST_ACCEPT_ALL_SHIFT 4 +/* accept all broadcast packets (subject to vlan) */ #define ETH_VPORT_RX_MODE_BCAST_ACCEPT_ALL_MASK 0x1 #define ETH_VPORT_RX_MODE_BCAST_ACCEPT_ALL_SHIFT 5 #define ETH_VPORT_RX_MODE_RESERVED1_MASK 0x3FF @@ -862,6 +1037,7 @@ struct eth_vport_rx_mode { __le16 reserved2[3]; }; + /* * Command for setting tpa parameters */ @@ -870,39 +1046,45 @@ struct eth_vport_tpa_param { u8 tpa_ipv6_en_flg /* Enable TPA for IPv6 packets */; u8 tpa_ipv4_tunn_en_flg /* Enable TPA for IPv4 over tunnel */; u8 tpa_ipv6_tunn_en_flg /* Enable TPA for IPv6 over tunnel */; +/* If set, start each tpa segment on new SGE (GRO mode). One SGE per segment + * allowed + */ u8 tpa_pkt_split_flg; - u8 tpa_hdr_data_split_flg - /* If set, put header of first TPA segment on bd and data on SGE */ - ; - u8 tpa_gro_consistent_flg - /* If set, GRO data consistent will checked for TPA continue */; - u8 tpa_max_aggs_num - /* maximum number of opened aggregations per v-port */; +/* If set, put header of first TPA segment on bd and data on SGE */ + u8 tpa_hdr_data_split_flg; +/* If set, GRO data consistent will checked for TPA continue */ + u8 tpa_gro_consistent_flg; +/* maximum number of opened aggregations per v-port */ + u8 tpa_max_aggs_num; __le16 tpa_max_size /* maximal size for the aggregated TPA packets */; - __le16 tpa_min_size_to_start - /* minimum TCP payload size for a packet to start aggregation */; - __le16 tpa_min_size_to_cont - /* minimum TCP payload size for a packet to continue aggregation */ - ; - u8 max_buff_num - /* maximal number of buffers that can be used for one aggregation */ - ; +/* minimum TCP payload size for a packet to start aggregation */ + __le16 tpa_min_size_to_start; +/* minimum TCP payload size for a packet to continue aggregation */ + __le16 tpa_min_size_to_cont; +/* maximal number of buffers that can be used for one aggregation */ + u8 max_buff_num; u8 reserved; }; + /* * Command for setting classification flags for a vport $$KEEP_ENDIANNESS$$ */ struct eth_vport_tx_mode { __le16 state; +/* drop all unicast packets */ #define ETH_VPORT_TX_MODE_UCAST_DROP_ALL_MASK 0x1 #define ETH_VPORT_TX_MODE_UCAST_DROP_ALL_SHIFT 0 +/* accept all unicast packets (subject to vlan) */ #define ETH_VPORT_TX_MODE_UCAST_ACCEPT_ALL_MASK 0x1 #define ETH_VPORT_TX_MODE_UCAST_ACCEPT_ALL_SHIFT 1 +/* drop all multicast packets */ #define ETH_VPORT_TX_MODE_MCAST_DROP_ALL_MASK 0x1 #define ETH_VPORT_TX_MODE_MCAST_DROP_ALL_SHIFT 2 +/* accept all multicast packets (subject to vlan) */ #define ETH_VPORT_TX_MODE_MCAST_ACCEPT_ALL_MASK 0x1 #define ETH_VPORT_TX_MODE_MCAST_ACCEPT_ALL_SHIFT 3 +/* accept all broadcast packets (subject to vlan) */ #define ETH_VPORT_TX_MODE_BCAST_ACCEPT_ALL_MASK 0x1 #define ETH_VPORT_TX_MODE_BCAST_ACCEPT_ALL_SHIFT 4 #define ETH_VPORT_TX_MODE_RESERVED1_MASK 0x7FF @@ -910,17 +1092,19 @@ struct eth_vport_tx_mode { __le16 reserved2[3]; }; + /* - * Ramrod data for rx add gft filter data + * GFT filter update action type. */ -struct rx_add_gft_filter_data { - struct regpair pkt_hdr_addr /* Packet Header That Defines GFT Filter */ - ; - __le16 action_icid /* ICID of Action to run for this filter */; - __le16 pkt_hdr_length /* Packet Header Length */; - u8 reserved[4]; +enum gft_filter_update_action { + GFT_ADD_FILTER, + GFT_DELETE_FILTER, + MAX_GFT_FILTER_UPDATE_ACTION }; + + + /* * Ramrod data for rx add openflow filter */ @@ -929,10 +1113,12 @@ struct rx_add_openflow_filter_data { u8 priority /* Searcher String - Packet priority */; u8 reserved0; __le32 tenant_id /* Searcher String - Tenant ID */; - __le16 dst_mac_hi /* Searcher String - Destination Mac Bytes 0 to 1 */; - __le16 dst_mac_mid /* Searcher String - Destination Mac Bytes 2 to 3 */ - ; - __le16 dst_mac_lo /* Searcher String - Destination Mac Bytes 4 to 5 */; +/* Searcher String - Destination Mac Bytes 0 to 1 */ + __le16 dst_mac_hi; +/* Searcher String - Destination Mac Bytes 2 to 3 */ + __le16 dst_mac_mid; +/* Searcher String - Destination Mac Bytes 4 to 5 */ + __le16 dst_mac_lo; __le16 src_mac_hi /* Searcher String - Source Mac 0 to 1 */; __le16 src_mac_mid /* Searcher String - Source Mac 2 to 3 */; __le16 src_mac_lo /* Searcher String - Source Mac 4 to 5 */; @@ -948,6 +1134,7 @@ struct rx_add_openflow_filter_data { __le16 l4_src_port /* Searcher String - TCP/UDP Source Port */; }; + /* * Ramrod data for rx create gft action */ @@ -956,6 +1143,7 @@ struct rx_create_gft_action_data { u8 reserved[7]; }; + /* * Ramrod data for rx create openflow action */ @@ -964,12 +1152,13 @@ struct rx_create_openflow_action_data { u8 reserved[7]; }; + /* * Ramrod data for rx queue start ramrod */ struct rx_queue_start_ramrod_data { __le16 rx_queue_id /* ID of RX queue */; - __le16 num_of_pbl_pages /* Num of pages in CQE PBL */; + __le16 num_of_pbl_pages /* Number of pages in CQE PBL */; __le16 bd_max_bytes /* maximal bytes that can be places on the bd */; __le16 sb_id /* Status block ID */; u8 sb_index /* index of the protocol index */; @@ -980,25 +1169,35 @@ struct rx_queue_start_ramrod_data { u8 stats_counter_id /* Statistics counter ID */; u8 pin_context /* Pin context in CCFC to improve performance */; u8 pxp_tph_valid_bd /* PXP command TPH Valid - for BD/SGE fetch */; - u8 pxp_tph_valid_pkt /* PXP command TPH Valid - for packet placement */ - ; - u8 pxp_st_hint - /* PXP command Steering tag hint. Use enum pxp_tph_st_hint */; +/* PXP command TPH Valid - for packet placement */ + u8 pxp_tph_valid_pkt; +/* PXP command Steering tag hint. Use enum pxp_tph_st_hint */ + u8 pxp_st_hint; __le16 pxp_st_index /* PXP command Steering tag index */; - u8 pmd_mode - /* Indicates that current queue belongs to poll-mode driver */; +/* Indicates that current queue belongs to poll-mode driver */ + u8 pmd_mode; +/* Indicates that the current queue is using the TX notification queue + * mechanism - should be set only for PMD queue + */ u8 notify_en; - u8 toggle_val - /* Initial value for the toggle valid bit - used in PMD mode */; - u8 reserved[7]; +/* Initial value for the toggle valid bit - used in PMD mode */ + u8 toggle_val; +/* Index of RX producers in VF zone. Used for VF only. */ + u8 vf_rx_prod_index; +/* Backward compatibility mode. If set, unprotected mStorm queue zone will used + * for VF RX producers instead of VF zone. + */ + u8 vf_rx_prod_use_zone_a; + u8 reserved[5]; __le16 reserved1 /* FW reserved. */; struct regpair cqe_pbl_addr /* Base address on host of CQE PBL */; struct regpair bd_base /* bd address of the first bd page */; struct regpair reserved2 /* FW reserved. */; }; + /* - * Ramrod data for rx queue start ramrod + * Ramrod data for rx queue stop ramrod */ struct rx_queue_stop_ramrod_data { __le16 rx_queue_id /* ID of RX queue */; @@ -1008,6 +1207,7 @@ struct rx_queue_stop_ramrod_data { u8 reserved[3]; }; + /* * Ramrod data for rx queue update ramrod */ @@ -1025,6 +1225,7 @@ struct rx_queue_update_ramrod_data { struct regpair reserved6 /* FW reserved. */; }; + /* * Ramrod data for rx Add UDP Filter */ @@ -1034,38 +1235,80 @@ struct rx_udp_filter_data { u8 ip_type /* Searcher String - IP Type */; u8 tenant_id_exists /* Searcher String - Tenant ID Exists */; __le16 reserved1; +/* Searcher String - IP Destination Address, for IPv4 use ip_dst_addr[0] only */ __le32 ip_dst_addr[4]; - /* Searcher String-IP Dest Addr for IPv4 use ip_dst_addr[0] only */ - ; - __le32 ip_src_addr[4] - /* Searcher String-IP Src Addr, for IPv4 use ip_dst_addr[0] only */ - ; +/* Searcher String - IP Source Address, for IPv4 use ip_dst_addr[0] only */ + __le32 ip_src_addr[4]; __le16 udp_dst_port /* Searcher String - UDP Destination Port */; __le16 udp_src_port /* Searcher String - UDP Source Port */; __le32 tenant_id /* Searcher String - Tenant ID */; }; + /* - * Ramrod data for rx queue start ramrod + * add or delete GFT filter - filter is packet header of type of packet wished + * to pass certain FW flow + */ +struct rx_update_gft_filter_data { +/* Pointer to Packet Header That Defines GFT Filter */ + struct regpair pkt_hdr_addr; + __le16 pkt_hdr_length /* Packet Header Length */; +/* Action icid. Valid if action_icid_valid flag set. */ + __le16 action_icid; + __le16 rx_qid /* RX queue ID. Valid if rx_qid_valid set. */; + __le16 flow_id /* RX flow ID. Valid if flow_id_valid set. */; + u8 vport_id /* RX vport Id. */; +/* If set, action_icid will used for GFT filter update. */ + u8 action_icid_valid; +/* If set, rx_qid will used for traffic steering, in additional to vport_id. + * flow_id_valid must be cleared. If cleared, queue ID will selected by RSS. + */ + u8 rx_qid_valid; +/* If set, flow_id will reported by CQE, rx_qid_valid must be cleared. If + * cleared, flow_id 0 will reported by CQE. + */ + u8 flow_id_valid; + u8 filter_action /* Use to set type of action on filter */; +/* 0 - dont assert in case of error. Just return an error code. 1 - assert in + * case of error. + */ + u8 assert_on_error; + u8 reserved[2]; +}; + + + +/* + * Ramrod data for tx queue start ramrod */ struct tx_queue_start_ramrod_data { __le16 sb_id /* Status block ID */; u8 sb_index /* Status block protocol index */; u8 vport_id /* VPort ID */; - u8 reserved0 /* FW reserved. */; + u8 reserved0 /* FW reserved. (qcn_rl_en) */; u8 stats_counter_id /* Statistics counter ID to use */; __le16 qm_pq_id /* QM PQ ID */; u8 flags; +/* 0: Enable QM opportunistic flow. 1: Disable QM opportunistic flow */ #define TX_QUEUE_START_RAMROD_DATA_DISABLE_OPPORTUNISTIC_MASK 0x1 #define TX_QUEUE_START_RAMROD_DATA_DISABLE_OPPORTUNISTIC_SHIFT 0 +/* If set, Test Mode - packets will be duplicated by Xstorm handler */ #define TX_QUEUE_START_RAMROD_DATA_TEST_MODE_PKT_DUP_MASK 0x1 #define TX_QUEUE_START_RAMROD_DATA_TEST_MODE_PKT_DUP_SHIFT 1 +/* If set, Test Mode - packets destination will be determined by dest_port_mode + * field from Tx BD + */ #define TX_QUEUE_START_RAMROD_DATA_TEST_MODE_TX_DEST_MASK 0x1 #define TX_QUEUE_START_RAMROD_DATA_TEST_MODE_TX_DEST_SHIFT 2 +/* Indicates that current queue belongs to poll-mode driver */ #define TX_QUEUE_START_RAMROD_DATA_PMD_MODE_MASK 0x1 #define TX_QUEUE_START_RAMROD_DATA_PMD_MODE_SHIFT 3 +/* Indicates that the current queue is using the TX notification queue + * mechanism - should be set only for PMD queue + */ #define TX_QUEUE_START_RAMROD_DATA_NOTIFY_EN_MASK 0x1 #define TX_QUEUE_START_RAMROD_DATA_NOTIFY_EN_SHIFT 4 +/* Pin context in CCFC to improve performance */ #define TX_QUEUE_START_RAMROD_DATA_PIN_CONTEXT_MASK 0x1 #define TX_QUEUE_START_RAMROD_DATA_PIN_CONTEXT_SHIFT 5 #define TX_QUEUE_START_RAMROD_DATA_RESERVED1_MASK 0x3 @@ -1074,17 +1317,25 @@ struct tx_queue_start_ramrod_data { u8 pxp_tph_valid_bd /* PXP command TPH Valid - for BD fetch */; u8 pxp_tph_valid_pkt /* PXP command TPH Valid - for packet fetch */; __le16 pxp_st_index /* PXP command Steering tag index */; - __le16 comp_agg_size /* TX completion min agg size - for PMD queues */; +/* TX completion min agg size - for PMD queues */ + __le16 comp_agg_size; __le16 queue_zone_id /* queue zone ID to use */; - __le16 test_dup_count /* In Test Mode, number of duplications */; + __le16 reserved2 /* FW reserved. (test_dup_count) */; __le16 pbl_size /* Number of BD pages pointed by PBL */; - __le16 tx_queue_id - /* unique Queue ID - currently used only by PMD flow */; +/* unique Queue ID - currently used only by PMD flow */ + __le16 tx_queue_id; +/* Unique Same-As-Last Resource ID - improves performance for same-as-last + * packets per connection (range 0..ETH_TX_NUM_SAME_AS_LAST_ENTRIES-1 IDs + * available) + */ + __le16 same_as_last_id; + __le16 reserved[3]; struct regpair pbl_base_addr /* address of the pbl page */; - struct regpair bd_cons_address - /* BD consumer address in host - for PMD queues */; +/* BD consumer address in host - for PMD queues */ + struct regpair bd_cons_address; }; + /* * Ramrod data for tx queue stop ramrod */ @@ -1092,16 +1343,30 @@ struct tx_queue_stop_ramrod_data { __le16 reserved[4]; }; + +/* + * Ramrod data for tx queue update ramrod + */ +struct tx_queue_update_ramrod_data { + __le16 update_qm_pq_id_flg /* Flag to Update QM PQ ID */; + __le16 qm_pq_id /* Updated QM PQ ID */; + __le32 reserved0; + struct regpair reserved1[5]; +}; + + + /* * Ramrod data for vport update ramrod */ struct vport_filter_update_ramrod_data { - struct eth_filter_cmd_header filter_cmd_hdr - /* Header for Filter Commands (RX/TX, Add/Remove/Replace, etc) */; - struct eth_filter_cmd filter_cmds[ETH_FILTER_RULES_COUNT] - /* Filter Commands */; +/* Header for Filter Commands (RX/TX, Add/Remove/Replace, etc) */ + struct eth_filter_cmd_header filter_cmd_hdr; +/* Filter Commands */ + struct eth_filter_cmd filter_cmds[ETH_FILTER_RULES_COUNT]; }; + /* * Ramrod data for vport start ramrod */ @@ -1113,25 +1378,35 @@ struct vport_start_ramrod_data { u8 inner_vlan_removal_en; struct eth_vport_rx_mode rx_mode /* Rx filter data */; struct eth_vport_tx_mode tx_mode /* Tx filter data */; - struct eth_vport_tpa_param tpa_param /* TPA configuration parameters */ - ; +/* TPA configuration parameters */ + struct eth_vport_tpa_param tpa_param; __le16 default_vlan /* Default Vlan value to be forced by FW */; u8 tx_switching_en /* Tx switching is enabled for current Vport */; - u8 anti_spoofing_en - /* Anti-spoofing verification is set for current Vport */; - u8 default_vlan_en - /* If set, the default Vlan value is forced by the FW */; - u8 handle_ptp_pkts /* If set, the vport handles PTP Timesync Packets */ - ; +/* Anti-spoofing verification is set for current Vport */ + u8 anti_spoofing_en; +/* If set, the default Vlan value is forced by the FW */ + u8 default_vlan_en; +/* If set, the vport handles PTP Timesync Packets */ + u8 handle_ptp_pkts; +/* If enable then innerVlan will be striped and not written to cqe */ u8 silent_vlan_removal_en; - /* If enable then innerVlan will be striped and not written to cqe */ +/* If set untagged filter (vlan0) is added to current Vport, otherwise port is + * marked as any-vlan + */ u8 untagged; - struct eth_tx_err_vals tx_err_behav - /* Desired behavior per TX error type */; +/* Desired behavior per TX error type */ + struct eth_tx_err_vals tx_err_behav; +/* If set, ETH header padding will not inserted. placement_offset will be zero. + */ u8 zero_placement_offset; - u8 reserved[7]; +/* If set, control frames will be filtered according to MAC check. */ + u8 ctl_frame_mac_check_en; +/* If set, control frames will be filtered according to ethtype check. */ + u8 ctl_frame_ethtype_check_en; + u8 reserved[5]; }; + /* * Ramrod data for vport stop ramrod */ @@ -1140,6 +1415,7 @@ struct vport_stop_ramrod_data { u8 reserved[7]; }; + /* * Ramrod data for vport update ramrod */ @@ -1151,39 +1427,51 @@ struct vport_update_ramrod_data_cmn { u8 tx_active_flg /* tx active flag value */; u8 update_rx_mode_flg /* set if rx state data should be handled */; u8 update_tx_mode_flg /* set if tx state data should be handled */; - u8 update_approx_mcast_flg - /* set if approx. mcast data should be handled */; +/* set if approx. mcast data should be handled */ + u8 update_approx_mcast_flg; u8 update_rss_flg /* set if rss data should be handled */; - u8 update_inner_vlan_removal_en_flg - /* set if inner_vlan_removal_en should be handled */; +/* set if inner_vlan_removal_en should be handled */ + u8 update_inner_vlan_removal_en_flg; u8 inner_vlan_removal_en; +/* set if tpa parameters should be handled, TPA must be disable before */ u8 update_tpa_param_flg; u8 update_tpa_en_flg /* set if tpa enable changes */; - u8 update_tx_switching_en_flg - /* set if tx switching en flag should be handled */; +/* set if tx switching en flag should be handled */ + u8 update_tx_switching_en_flg; u8 tx_switching_en /* tx switching en value */; - u8 update_anti_spoofing_en_flg - /* set if anti spoofing flag should be handled */; +/* set if anti spoofing flag should be handled */ + u8 update_anti_spoofing_en_flg; u8 anti_spoofing_en /* Anti-spoofing verification en value */; - u8 update_handle_ptp_pkts - /* set if handle_ptp_pkts should be handled. */; - u8 handle_ptp_pkts /* If set, the vport handles PTP Timesync Packets */ - ; - u8 update_default_vlan_en_flg - /* If set, the default Vlan enable flag is updated */; - u8 default_vlan_en - /* If set, the default Vlan value is forced by the FW */; - u8 update_default_vlan_flg - /* If set, the default Vlan value is updated */; +/* set if handle_ptp_pkts should be handled. */ + u8 update_handle_ptp_pkts; +/* If set, the vport handles PTP Timesync Packets */ + u8 handle_ptp_pkts; +/* If set, the default Vlan enable flag is updated */ + u8 update_default_vlan_en_flg; +/* If set, the default Vlan value is forced by the FW */ + u8 default_vlan_en; +/* If set, the default Vlan value is updated */ + u8 update_default_vlan_flg; __le16 default_vlan /* Default Vlan value to be forced by FW */; - u8 update_accept_any_vlan_flg - /* set if accept_any_vlan should be handled */; +/* set if accept_any_vlan should be handled */ + u8 update_accept_any_vlan_flg; u8 accept_any_vlan /* accept_any_vlan updated value */; +/* Set to remove vlan silently, update_inner_vlan_removal_en_flg must be enabled + * as well. If Rx is in noSgl mode send rx_queue_update_ramrod_data + */ u8 silent_vlan_removal_en; - u8 update_mtu_flg - /* If set, MTU will be updated. Vport must be not active. */; +/* If set, MTU will be updated. Vport must be not active. */ + u8 update_mtu_flg; __le16 mtu /* New MTU value. Used if update_mtu_flg are set */; - u8 reserved[2]; +/* If set, ctl_frame_mac_check_en and ctl_frame_ethtype_check_en will be + * updated + */ + u8 update_ctl_frame_checks_en_flg; +/* If set, control frames will be filtered according to MAC check. */ + u8 ctl_frame_mac_check_en; +/* If set, control frames will be filtered according to ethtype check. */ + u8 ctl_frame_ethtype_check_en; + u8 reserved[15]; }; struct vport_update_ramrod_mcast { @@ -1194,54 +1482,738 @@ struct vport_update_ramrod_mcast { * Ramrod data for vport update ramrod */ struct vport_update_ramrod_data { - struct vport_update_ramrod_data_cmn common - /* Common data for all vport update ramrods */; +/* Common data for all vport update ramrods */ + struct vport_update_ramrod_data_cmn common; struct eth_vport_rx_mode rx_mode /* vport rx mode bitmap */; struct eth_vport_tx_mode tx_mode /* vport tx mode bitmap */; - struct eth_vport_tpa_param tpa_param /* TPA configuration parameters */ - ; +/* TPA configuration parameters */ + struct eth_vport_tpa_param tpa_param; struct vport_update_ramrod_mcast approx_mcast; struct eth_vport_rss_config rss_config /* rss config data */; }; + + + + + +struct E4XstormEthConnAgCtxDqExtLdPart { + u8 reserved0 /* cdu_validation */; + u8 eth_state /* state */; + u8 flags0; +/* exist_in_qm0 */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_EXIST_IN_QM0_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_EXIST_IN_QM0_SHIFT 0 +/* exist_in_qm1 */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED1_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED1_SHIFT 1 +/* exist_in_qm2 */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED2_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED2_SHIFT 2 +/* exist_in_qm3 */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_EXIST_IN_QM3_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_EXIST_IN_QM3_SHIFT 3 +/* bit4 */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED3_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED3_SHIFT 4 +/* cf_array_active */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED4_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED4_SHIFT 5 +/* bit6 */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED5_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED5_SHIFT 6 +/* bit7 */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED6_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED6_SHIFT 7 + u8 flags1; +/* bit8 */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED7_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED7_SHIFT 0 +/* bit9 */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED8_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED8_SHIFT 1 +/* bit10 */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED9_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED9_SHIFT 2 +/* bit11 */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_BIT11_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_BIT11_SHIFT 3 +/* bit12 */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_BIT12_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_BIT12_SHIFT 4 +/* bit13 */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_BIT13_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_BIT13_SHIFT 5 +/* bit14 */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_TX_RULE_ACTIVE_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_TX_RULE_ACTIVE_SHIFT 6 +/* bit15 */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_DQ_CF_ACTIVE_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_DQ_CF_ACTIVE_SHIFT 7 + u8 flags2; +/* timer0cf */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF0_MASK 0x3 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF0_SHIFT 0 +/* timer1cf */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF1_MASK 0x3 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF1_SHIFT 2 +/* timer2cf */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF2_MASK 0x3 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF2_SHIFT 4 +/* timer_stop_all */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF3_MASK 0x3 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF3_SHIFT 6 + u8 flags3; +/* cf4 */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF4_MASK 0x3 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF4_SHIFT 0 +/* cf5 */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF5_MASK 0x3 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF5_SHIFT 2 +/* cf6 */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF6_MASK 0x3 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF6_SHIFT 4 +/* cf7 */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF7_MASK 0x3 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF7_SHIFT 6 + u8 flags4; +/* cf8 */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF8_MASK 0x3 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF8_SHIFT 0 +/* cf9 */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF9_MASK 0x3 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF9_SHIFT 2 +/* cf10 */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF10_MASK 0x3 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF10_SHIFT 4 +/* cf11 */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF11_MASK 0x3 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF11_SHIFT 6 + u8 flags5; +/* cf12 */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF12_MASK 0x3 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF12_SHIFT 0 +/* cf13 */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF13_MASK 0x3 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF13_SHIFT 2 +/* cf14 */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF14_MASK 0x3 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF14_SHIFT 4 +/* cf15 */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF15_MASK 0x3 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF15_SHIFT 6 + u8 flags6; +/* cf16 */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_GO_TO_BD_CONS_CF_MASK 0x3 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_GO_TO_BD_CONS_CF_SHIFT 0 +/* cf_array_cf */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_MULTI_UNICAST_CF_MASK 0x3 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_MULTI_UNICAST_CF_SHIFT 2 +/* cf18 */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_DQ_CF_MASK 0x3 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_DQ_CF_SHIFT 4 +/* cf19 */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_TERMINATE_CF_MASK 0x3 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_TERMINATE_CF_SHIFT 6 + u8 flags7; +/* cf20 */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_FLUSH_Q0_MASK 0x3 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_FLUSH_Q0_SHIFT 0 +/* cf21 */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED10_MASK 0x3 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED10_SHIFT 2 +/* cf22 */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_SLOW_PATH_MASK 0x3 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_SLOW_PATH_SHIFT 4 +/* cf0en */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF0EN_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF0EN_SHIFT 6 +/* cf1en */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF1EN_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF1EN_SHIFT 7 + u8 flags8; +/* cf2en */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF2EN_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF2EN_SHIFT 0 +/* cf3en */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF3EN_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF3EN_SHIFT 1 +/* cf4en */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF4EN_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF4EN_SHIFT 2 +/* cf5en */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF5EN_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF5EN_SHIFT 3 +/* cf6en */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF6EN_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF6EN_SHIFT 4 +/* cf7en */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF7EN_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF7EN_SHIFT 5 +/* cf8en */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF8EN_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF8EN_SHIFT 6 +/* cf9en */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF9EN_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF9EN_SHIFT 7 + u8 flags9; +/* cf10en */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF10EN_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF10EN_SHIFT 0 +/* cf11en */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF11EN_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF11EN_SHIFT 1 +/* cf12en */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF12EN_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF12EN_SHIFT 2 +/* cf13en */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF13EN_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF13EN_SHIFT 3 +/* cf14en */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF14EN_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF14EN_SHIFT 4 +/* cf15en */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF15EN_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF15EN_SHIFT 5 +/* cf16en */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_GO_TO_BD_CONS_CF_EN_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_GO_TO_BD_CONS_CF_EN_SHIFT 6 +/* cf_array_cf_en */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_MULTI_UNICAST_CF_EN_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_MULTI_UNICAST_CF_EN_SHIFT 7 + u8 flags10; +/* cf18en */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_DQ_CF_EN_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_DQ_CF_EN_SHIFT 0 +/* cf19en */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_TERMINATE_CF_EN_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_TERMINATE_CF_EN_SHIFT 1 +/* cf20en */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_FLUSH_Q0_EN_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_FLUSH_Q0_EN_SHIFT 2 +/* cf21en */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED11_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED11_SHIFT 3 +/* cf22en */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_SLOW_PATH_EN_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_SLOW_PATH_EN_SHIFT 4 +/* cf23en */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_TPH_ENABLE_EN_RESERVED_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_TPH_ENABLE_EN_RESERVED_SHIFT 5 +/* rule0en */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED12_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED12_SHIFT 6 +/* rule1en */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED13_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED13_SHIFT 7 + u8 flags11; +/* rule2en */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED14_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED14_SHIFT 0 +/* rule3en */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED15_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED15_SHIFT 1 +/* rule4en */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_TX_DEC_RULE_EN_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_TX_DEC_RULE_EN_SHIFT 2 +/* rule5en */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RULE5EN_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RULE5EN_SHIFT 3 +/* rule6en */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RULE6EN_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RULE6EN_SHIFT 4 +/* rule7en */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RULE7EN_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RULE7EN_SHIFT 5 +/* rule8en */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED1_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED1_SHIFT 6 +/* rule9en */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RULE9EN_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RULE9EN_SHIFT 7 + u8 flags12; +/* rule10en */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RULE10EN_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RULE10EN_SHIFT 0 +/* rule11en */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RULE11EN_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RULE11EN_SHIFT 1 +/* rule12en */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED2_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED2_SHIFT 2 +/* rule13en */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED3_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED3_SHIFT 3 +/* rule14en */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RULE14EN_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RULE14EN_SHIFT 4 +/* rule15en */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RULE15EN_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RULE15EN_SHIFT 5 +/* rule16en */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RULE16EN_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RULE16EN_SHIFT 6 +/* rule17en */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RULE17EN_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RULE17EN_SHIFT 7 + u8 flags13; +/* rule18en */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RULE18EN_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RULE18EN_SHIFT 0 +/* rule19en */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RULE19EN_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RULE19EN_SHIFT 1 +/* rule20en */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED4_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED4_SHIFT 2 +/* rule21en */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED5_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED5_SHIFT 3 +/* rule22en */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED6_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED6_SHIFT 4 +/* rule23en */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED7_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED7_SHIFT 5 +/* rule24en */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED8_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED8_SHIFT 6 +/* rule25en */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED9_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED9_SHIFT 7 + u8 flags14; +/* bit16 */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_EDPM_USE_EXT_HDR_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_EDPM_USE_EXT_HDR_SHIFT 0 +/* bit17 */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_EDPM_SEND_RAW_L3L4_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_EDPM_SEND_RAW_L3L4_SHIFT 1 +/* bit18 */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_EDPM_INBAND_PROP_HDR_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_EDPM_INBAND_PROP_HDR_SHIFT 2 +/* bit19 */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_EDPM_SEND_EXT_TUNNEL_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_EDPM_SEND_EXT_TUNNEL_SHIFT 3 +/* bit20 */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_L2_EDPM_ENABLE_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_L2_EDPM_ENABLE_SHIFT 4 +/* bit21 */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_ROCE_EDPM_ENABLE_MASK 0x1 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_ROCE_EDPM_ENABLE_SHIFT 5 +/* cf23 */ +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_TPH_ENABLE_MASK 0x3 +#define E4XSTORMETHCONNAGCTXDQEXTLDPART_TPH_ENABLE_SHIFT 6 + u8 edpm_event_id /* byte2 */; + __le16 physical_q0 /* physical_q0 */; + __le16 e5_reserved1 /* physical_q1 */; + __le16 edpm_num_bds /* physical_q2 */; + __le16 tx_bd_cons /* word3 */; + __le16 tx_bd_prod /* word4 */; + __le16 tx_class /* word5 */; + __le16 conn_dpi /* conn_dpi */; + u8 byte3 /* byte3 */; + u8 byte4 /* byte4 */; + u8 byte5 /* byte5 */; + u8 byte6 /* byte6 */; + __le32 reg0 /* reg0 */; + __le32 reg1 /* reg1 */; + __le32 reg2 /* reg2 */; + __le32 reg3 /* reg3 */; + __le32 reg4 /* reg4 */; +}; + + +struct e4_mstorm_eth_conn_ag_ctx { + u8 byte0 /* cdu_validation */; + u8 byte1 /* state */; + u8 flags0; +#define E4_MSTORM_ETH_CONN_AG_CTX_EXIST_IN_QM0_MASK 0x1 /* exist_in_qm0 */ +#define E4_MSTORM_ETH_CONN_AG_CTX_EXIST_IN_QM0_SHIFT 0 +#define E4_MSTORM_ETH_CONN_AG_CTX_BIT1_MASK 0x1 /* exist_in_qm1 */ +#define E4_MSTORM_ETH_CONN_AG_CTX_BIT1_SHIFT 1 +#define E4_MSTORM_ETH_CONN_AG_CTX_CF0_MASK 0x3 /* cf0 */ +#define E4_MSTORM_ETH_CONN_AG_CTX_CF0_SHIFT 2 +#define E4_MSTORM_ETH_CONN_AG_CTX_CF1_MASK 0x3 /* cf1 */ +#define E4_MSTORM_ETH_CONN_AG_CTX_CF1_SHIFT 4 +#define E4_MSTORM_ETH_CONN_AG_CTX_CF2_MASK 0x3 /* cf2 */ +#define E4_MSTORM_ETH_CONN_AG_CTX_CF2_SHIFT 6 + u8 flags1; +#define E4_MSTORM_ETH_CONN_AG_CTX_CF0EN_MASK 0x1 /* cf0en */ +#define E4_MSTORM_ETH_CONN_AG_CTX_CF0EN_SHIFT 0 +#define E4_MSTORM_ETH_CONN_AG_CTX_CF1EN_MASK 0x1 /* cf1en */ +#define E4_MSTORM_ETH_CONN_AG_CTX_CF1EN_SHIFT 1 +#define E4_MSTORM_ETH_CONN_AG_CTX_CF2EN_MASK 0x1 /* cf2en */ +#define E4_MSTORM_ETH_CONN_AG_CTX_CF2EN_SHIFT 2 +#define E4_MSTORM_ETH_CONN_AG_CTX_RULE0EN_MASK 0x1 /* rule0en */ +#define E4_MSTORM_ETH_CONN_AG_CTX_RULE0EN_SHIFT 3 +#define E4_MSTORM_ETH_CONN_AG_CTX_RULE1EN_MASK 0x1 /* rule1en */ +#define E4_MSTORM_ETH_CONN_AG_CTX_RULE1EN_SHIFT 4 +#define E4_MSTORM_ETH_CONN_AG_CTX_RULE2EN_MASK 0x1 /* rule2en */ +#define E4_MSTORM_ETH_CONN_AG_CTX_RULE2EN_SHIFT 5 +#define E4_MSTORM_ETH_CONN_AG_CTX_RULE3EN_MASK 0x1 /* rule3en */ +#define E4_MSTORM_ETH_CONN_AG_CTX_RULE3EN_SHIFT 6 +#define E4_MSTORM_ETH_CONN_AG_CTX_RULE4EN_MASK 0x1 /* rule4en */ +#define E4_MSTORM_ETH_CONN_AG_CTX_RULE4EN_SHIFT 7 + __le16 word0 /* word0 */; + __le16 word1 /* word1 */; + __le32 reg0 /* reg0 */; + __le32 reg1 /* reg1 */; +}; + + + + + +struct e4_xstorm_eth_hw_conn_ag_ctx { + u8 reserved0 /* cdu_validation */; + u8 eth_state /* state */; + u8 flags0; +/* exist_in_qm0 */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_EXIST_IN_QM0_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_EXIST_IN_QM0_SHIFT 0 +/* exist_in_qm1 */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED1_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED1_SHIFT 1 +/* exist_in_qm2 */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED2_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED2_SHIFT 2 +/* exist_in_qm3 */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_EXIST_IN_QM3_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_EXIST_IN_QM3_SHIFT 3 +/* bit4 */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED3_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED3_SHIFT 4 +/* cf_array_active */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED4_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED4_SHIFT 5 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED5_MASK 0x1 /* bit6 */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED5_SHIFT 6 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED6_MASK 0x1 /* bit7 */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED6_SHIFT 7 + u8 flags1; +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED7_MASK 0x1 /* bit8 */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED7_SHIFT 0 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED8_MASK 0x1 /* bit9 */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED8_SHIFT 1 +/* bit10 */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED9_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED9_SHIFT 2 +/* bit11 */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_BIT11_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_BIT11_SHIFT 3 +/* bit12 */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_BIT12_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_BIT12_SHIFT 4 +/* bit13 */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_BIT13_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_BIT13_SHIFT 5 +/* bit14 */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_TX_RULE_ACTIVE_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_TX_RULE_ACTIVE_SHIFT 6 +/* bit15 */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_DQ_CF_ACTIVE_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_DQ_CF_ACTIVE_SHIFT 7 + u8 flags2; +/* timer0cf */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF0_MASK 0x3 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF0_SHIFT 0 +/* timer1cf */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF1_MASK 0x3 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF1_SHIFT 2 +/* timer2cf */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF2_MASK 0x3 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF2_SHIFT 4 +/* timer_stop_all */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF3_MASK 0x3 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF3_SHIFT 6 + u8 flags3; +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF4_MASK 0x3 /* cf4 */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF4_SHIFT 0 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF5_MASK 0x3 /* cf5 */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF5_SHIFT 2 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF6_MASK 0x3 /* cf6 */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF6_SHIFT 4 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF7_MASK 0x3 /* cf7 */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF7_SHIFT 6 + u8 flags4; +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF8_MASK 0x3 /* cf8 */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF8_SHIFT 0 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF9_MASK 0x3 /* cf9 */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF9_SHIFT 2 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF10_MASK 0x3 /* cf10 */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF10_SHIFT 4 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF11_MASK 0x3 /* cf11 */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF11_SHIFT 6 + u8 flags5; +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF12_MASK 0x3 /* cf12 */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF12_SHIFT 0 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF13_MASK 0x3 /* cf13 */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF13_SHIFT 2 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF14_MASK 0x3 /* cf14 */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF14_SHIFT 4 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF15_MASK 0x3 /* cf15 */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF15_SHIFT 6 + u8 flags6; +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_GO_TO_BD_CONS_CF_MASK 0x3 /* cf16 */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_GO_TO_BD_CONS_CF_SHIFT 0 +/* cf_array_cf */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_MULTI_UNICAST_CF_MASK 0x3 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_MULTI_UNICAST_CF_SHIFT 2 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_DQ_CF_MASK 0x3 /* cf18 */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_DQ_CF_SHIFT 4 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_TERMINATE_CF_MASK 0x3 /* cf19 */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_TERMINATE_CF_SHIFT 6 + u8 flags7; +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_FLUSH_Q0_MASK 0x3 /* cf20 */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_FLUSH_Q0_SHIFT 0 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED10_MASK 0x3 /* cf21 */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED10_SHIFT 2 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_SLOW_PATH_MASK 0x3 /* cf22 */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_SLOW_PATH_SHIFT 4 +/* cf0en */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF0EN_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF0EN_SHIFT 6 +/* cf1en */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF1EN_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF1EN_SHIFT 7 + u8 flags8; +/* cf2en */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF2EN_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF2EN_SHIFT 0 +/* cf3en */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF3EN_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF3EN_SHIFT 1 +/* cf4en */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF4EN_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF4EN_SHIFT 2 +/* cf5en */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF5EN_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF5EN_SHIFT 3 +/* cf6en */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF6EN_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF6EN_SHIFT 4 +/* cf7en */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF7EN_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF7EN_SHIFT 5 +/* cf8en */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF8EN_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF8EN_SHIFT 6 +/* cf9en */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF9EN_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF9EN_SHIFT 7 + u8 flags9; +/* cf10en */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF10EN_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF10EN_SHIFT 0 +/* cf11en */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF11EN_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF11EN_SHIFT 1 +/* cf12en */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF12EN_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF12EN_SHIFT 2 +/* cf13en */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF13EN_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF13EN_SHIFT 3 +/* cf14en */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF14EN_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF14EN_SHIFT 4 +/* cf15en */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF15EN_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF15EN_SHIFT 5 +/* cf16en */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_GO_TO_BD_CONS_CF_EN_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_GO_TO_BD_CONS_CF_EN_SHIFT 6 +/* cf_array_cf_en */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_MULTI_UNICAST_CF_EN_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_MULTI_UNICAST_CF_EN_SHIFT 7 + u8 flags10; +/* cf18en */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_DQ_CF_EN_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_DQ_CF_EN_SHIFT 0 +/* cf19en */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_TERMINATE_CF_EN_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_TERMINATE_CF_EN_SHIFT 1 +/* cf20en */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_FLUSH_Q0_EN_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_FLUSH_Q0_EN_SHIFT 2 +/* cf21en */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED11_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED11_SHIFT 3 +/* cf22en */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_SLOW_PATH_EN_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_SLOW_PATH_EN_SHIFT 4 +/* cf23en */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_TPH_ENABLE_EN_RESERVED_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_TPH_ENABLE_EN_RESERVED_SHIFT 5 +/* rule0en */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED12_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED12_SHIFT 6 +/* rule1en */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED13_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED13_SHIFT 7 + u8 flags11; +/* rule2en */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED14_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED14_SHIFT 0 +/* rule3en */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED15_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED15_SHIFT 1 +/* rule4en */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_TX_DEC_RULE_EN_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_TX_DEC_RULE_EN_SHIFT 2 +/* rule5en */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RULE5EN_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RULE5EN_SHIFT 3 +/* rule6en */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RULE6EN_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RULE6EN_SHIFT 4 +/* rule7en */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RULE7EN_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RULE7EN_SHIFT 5 +/* rule8en */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED1_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED1_SHIFT 6 +/* rule9en */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RULE9EN_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RULE9EN_SHIFT 7 + u8 flags12; +/* rule10en */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RULE10EN_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RULE10EN_SHIFT 0 +/* rule11en */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RULE11EN_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RULE11EN_SHIFT 1 +/* rule12en */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED2_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED2_SHIFT 2 +/* rule13en */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED3_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED3_SHIFT 3 +/* rule14en */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RULE14EN_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RULE14EN_SHIFT 4 +/* rule15en */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RULE15EN_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RULE15EN_SHIFT 5 +/* rule16en */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RULE16EN_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RULE16EN_SHIFT 6 +/* rule17en */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RULE17EN_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RULE17EN_SHIFT 7 + u8 flags13; +/* rule18en */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RULE18EN_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RULE18EN_SHIFT 0 +/* rule19en */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RULE19EN_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RULE19EN_SHIFT 1 +/* rule20en */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED4_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED4_SHIFT 2 +/* rule21en */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED5_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED5_SHIFT 3 +/* rule22en */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED6_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED6_SHIFT 4 +/* rule23en */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED7_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED7_SHIFT 5 +/* rule24en */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED8_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED8_SHIFT 6 +/* rule25en */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED9_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED9_SHIFT 7 + u8 flags14; +/* bit16 */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_EDPM_USE_EXT_HDR_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_EDPM_USE_EXT_HDR_SHIFT 0 +/* bit17 */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_EDPM_SEND_RAW_L3L4_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_EDPM_SEND_RAW_L3L4_SHIFT 1 +/* bit18 */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_EDPM_INBAND_PROP_HDR_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_EDPM_INBAND_PROP_HDR_SHIFT 2 +/* bit19 */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_EDPM_SEND_EXT_TUNNEL_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_EDPM_SEND_EXT_TUNNEL_SHIFT 3 +/* bit20 */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_L2_EDPM_ENABLE_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_L2_EDPM_ENABLE_SHIFT 4 +/* bit21 */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_ROCE_EDPM_ENABLE_MASK 0x1 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_ROCE_EDPM_ENABLE_SHIFT 5 +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_TPH_ENABLE_MASK 0x3 /* cf23 */ +#define E4_XSTORM_ETH_HW_CONN_AG_CTX_TPH_ENABLE_SHIFT 6 + u8 edpm_event_id /* byte2 */; + __le16 physical_q0 /* physical_q0 */; + __le16 e5_reserved1 /* physical_q1 */; + __le16 edpm_num_bds /* physical_q2 */; + __le16 tx_bd_cons /* word3 */; + __le16 tx_bd_prod /* word4 */; + __le16 tx_class /* word5 */; + __le16 conn_dpi /* conn_dpi */; +}; + + + /* * GFT CAM line struct */ struct gft_cam_line { __le32 camline; +/* Indication if the line is valid. */ #define GFT_CAM_LINE_VALID_MASK 0x1 #define GFT_CAM_LINE_VALID_SHIFT 0 +/* Data bits, the word that compared with the profile key */ #define GFT_CAM_LINE_DATA_MASK 0x3FFF #define GFT_CAM_LINE_DATA_SHIFT 1 +/* Mask bits, indicate the bits in the data that are Dont-Care */ #define GFT_CAM_LINE_MASK_BITS_MASK 0x3FFF #define GFT_CAM_LINE_MASK_BITS_SHIFT 15 #define GFT_CAM_LINE_RESERVED1_MASK 0x7 #define GFT_CAM_LINE_RESERVED1_SHIFT 29 }; + /* * GFT CAM line struct (for driversim use) */ struct gft_cam_line_mapped { __le32 camline; +/* Indication if the line is valid. */ #define GFT_CAM_LINE_MAPPED_VALID_MASK 0x1 #define GFT_CAM_LINE_MAPPED_VALID_SHIFT 0 +/* use enum gft_profile_ip_version (use enum gft_profile_ip_version) */ #define GFT_CAM_LINE_MAPPED_IP_VERSION_MASK 0x1 #define GFT_CAM_LINE_MAPPED_IP_VERSION_SHIFT 1 +/* use enum gft_profile_ip_version (use enum gft_profile_ip_version) */ #define GFT_CAM_LINE_MAPPED_TUNNEL_IP_VERSION_MASK 0x1 #define GFT_CAM_LINE_MAPPED_TUNNEL_IP_VERSION_SHIFT 2 +/* use enum gft_profile_upper_protocol_type + * (use enum gft_profile_upper_protocol_type) + */ #define GFT_CAM_LINE_MAPPED_UPPER_PROTOCOL_TYPE_MASK 0xF #define GFT_CAM_LINE_MAPPED_UPPER_PROTOCOL_TYPE_SHIFT 3 +/* use enum gft_profile_tunnel_type (use enum gft_profile_tunnel_type) */ #define GFT_CAM_LINE_MAPPED_TUNNEL_TYPE_MASK 0xF #define GFT_CAM_LINE_MAPPED_TUNNEL_TYPE_SHIFT 7 #define GFT_CAM_LINE_MAPPED_PF_ID_MASK 0xF #define GFT_CAM_LINE_MAPPED_PF_ID_SHIFT 11 +/* use enum gft_profile_ip_version (use enum gft_profile_ip_version) */ #define GFT_CAM_LINE_MAPPED_IP_VERSION_MASK_MASK 0x1 #define GFT_CAM_LINE_MAPPED_IP_VERSION_MASK_SHIFT 15 +/* use enum gft_profile_ip_version (use enum gft_profile_ip_version) */ #define GFT_CAM_LINE_MAPPED_TUNNEL_IP_VERSION_MASK_MASK 0x1 #define GFT_CAM_LINE_MAPPED_TUNNEL_IP_VERSION_MASK_SHIFT 16 +/* use enum gft_profile_upper_protocol_type + * (use enum gft_profile_upper_protocol_type) + */ #define GFT_CAM_LINE_MAPPED_UPPER_PROTOCOL_TYPE_MASK_MASK 0xF #define GFT_CAM_LINE_MAPPED_UPPER_PROTOCOL_TYPE_MASK_SHIFT 17 +/* use enum gft_profile_tunnel_type (use enum gft_profile_tunnel_type) */ #define GFT_CAM_LINE_MAPPED_TUNNEL_TYPE_MASK_MASK 0xF #define GFT_CAM_LINE_MAPPED_TUNNEL_TYPE_MASK_SHIFT 21 #define GFT_CAM_LINE_MAPPED_PF_ID_MASK_MASK 0xF @@ -1250,11 +2222,13 @@ struct gft_cam_line_mapped { #define GFT_CAM_LINE_MAPPED_RESERVED1_SHIFT 29 }; + union gft_cam_line_union { struct gft_cam_line cam_line; struct gft_cam_line_mapped cam_line_mapped; }; + /* * Used in gft_profile_key: Indication for ip version */ @@ -1264,17 +2238,24 @@ enum gft_profile_ip_version { MAX_GFT_PROFILE_IP_VERSION }; + /* * Profile key stucr fot GFT logic in Prs */ struct gft_profile_key { __le16 profile_key; +/* use enum gft_profile_ip_version (use enum gft_profile_ip_version) */ #define GFT_PROFILE_KEY_IP_VERSION_MASK 0x1 #define GFT_PROFILE_KEY_IP_VERSION_SHIFT 0 +/* use enum gft_profile_ip_version (use enum gft_profile_ip_version) */ #define GFT_PROFILE_KEY_TUNNEL_IP_VERSION_MASK 0x1 #define GFT_PROFILE_KEY_TUNNEL_IP_VERSION_SHIFT 1 +/* use enum gft_profile_upper_protocol_type + * (use enum gft_profile_upper_protocol_type) + */ #define GFT_PROFILE_KEY_UPPER_PROTOCOL_TYPE_MASK 0xF #define GFT_PROFILE_KEY_UPPER_PROTOCOL_TYPE_SHIFT 2 +/* use enum gft_profile_tunnel_type (use enum gft_profile_tunnel_type) */ #define GFT_PROFILE_KEY_TUNNEL_TYPE_MASK 0xF #define GFT_PROFILE_KEY_TUNNEL_TYPE_SHIFT 6 #define GFT_PROFILE_KEY_PF_ID_MASK 0xF @@ -1283,6 +2264,7 @@ struct gft_profile_key { #define GFT_PROFILE_KEY_RESERVED0_SHIFT 14 }; + /* * Used in gft_profile_key: Indication for tunnel type */ @@ -1296,6 +2278,7 @@ enum gft_profile_tunnel_type { MAX_GFT_PROFILE_TUNNEL_TYPE }; + /* * Used in gft_profile_key: Indication for protocol type */ @@ -1319,11 +2302,12 @@ enum gft_profile_upper_protocol_type { MAX_GFT_PROFILE_UPPER_PROTOCOL_TYPE }; + /* * GFT RAM line struct */ struct gft_ram_line { - __le32 low32bits; + __le32 lo; #define GFT_RAM_LINE_VLAN_SELECT_MASK 0x3 #define GFT_RAM_LINE_VLAN_SELECT_SHIFT 0 #define GFT_RAM_LINE_TUNNEL_ENTROPHY_MASK 0x1 @@ -1386,7 +2370,7 @@ struct gft_ram_line { #define GFT_RAM_LINE_DST_PORT_SHIFT 30 #define GFT_RAM_LINE_SRC_PORT_MASK 0x1 #define GFT_RAM_LINE_SRC_PORT_SHIFT 31 - __le32 high32bits; + __le32 hi; #define GFT_RAM_LINE_DSCP_MASK 0x1 #define GFT_RAM_LINE_DSCP_SHIFT 0 #define GFT_RAM_LINE_OVER_IP_PROTOCOL_MASK 0x1 @@ -1411,6 +2395,7 @@ struct gft_ram_line { #define GFT_RAM_LINE_RESERVED1_SHIFT 10 }; + /* * Used in the first 2 bits for gft_ram_line: Indication for vlan mask */ @@ -1422,491 +2407,5 @@ enum gft_vlan_select { MAX_GFT_VLAN_SELECT }; -struct mstorm_eth_conn_ag_ctx { - u8 byte0 /* cdu_validation */; - u8 byte1 /* state */; - u8 flags0; -#define MSTORM_ETH_CONN_AG_CTX_EXIST_IN_QM0_MASK 0x1 -#define MSTORM_ETH_CONN_AG_CTX_EXIST_IN_QM0_SHIFT 0 -#define MSTORM_ETH_CONN_AG_CTX_BIT1_MASK 0x1 -#define MSTORM_ETH_CONN_AG_CTX_BIT1_SHIFT 1 -#define MSTORM_ETH_CONN_AG_CTX_CF0_MASK 0x3 -#define MSTORM_ETH_CONN_AG_CTX_CF0_SHIFT 2 -#define MSTORM_ETH_CONN_AG_CTX_CF1_MASK 0x3 -#define MSTORM_ETH_CONN_AG_CTX_CF1_SHIFT 4 -#define MSTORM_ETH_CONN_AG_CTX_CF2_MASK 0x3 -#define MSTORM_ETH_CONN_AG_CTX_CF2_SHIFT 6 - u8 flags1; -#define MSTORM_ETH_CONN_AG_CTX_CF0EN_MASK 0x1 -#define MSTORM_ETH_CONN_AG_CTX_CF0EN_SHIFT 0 -#define MSTORM_ETH_CONN_AG_CTX_CF1EN_MASK 0x1 -#define MSTORM_ETH_CONN_AG_CTX_CF1EN_SHIFT 1 -#define MSTORM_ETH_CONN_AG_CTX_CF2EN_MASK 0x1 -#define MSTORM_ETH_CONN_AG_CTX_CF2EN_SHIFT 2 -#define MSTORM_ETH_CONN_AG_CTX_RULE0EN_MASK 0x1 -#define MSTORM_ETH_CONN_AG_CTX_RULE0EN_SHIFT 3 -#define MSTORM_ETH_CONN_AG_CTX_RULE1EN_MASK 0x1 -#define MSTORM_ETH_CONN_AG_CTX_RULE1EN_SHIFT 4 -#define MSTORM_ETH_CONN_AG_CTX_RULE2EN_MASK 0x1 -#define MSTORM_ETH_CONN_AG_CTX_RULE2EN_SHIFT 5 -#define MSTORM_ETH_CONN_AG_CTX_RULE3EN_MASK 0x1 -#define MSTORM_ETH_CONN_AG_CTX_RULE3EN_SHIFT 6 -#define MSTORM_ETH_CONN_AG_CTX_RULE4EN_MASK 0x1 -#define MSTORM_ETH_CONN_AG_CTX_RULE4EN_SHIFT 7 - __le16 word0 /* word0 */; - __le16 word1 /* word1 */; - __le32 reg0 /* reg0 */; - __le32 reg1 /* reg1 */; -}; - -/* @DPDK: xstormEthConnAgCtxDqExtLdPart */ -struct xstorm_eth_conn_ag_ctx_dq_ext_ld_part { - u8 reserved0 /* cdu_validation */; - u8 eth_state /* state */; - u8 flags0; -#define XSTORMETHCONNAGCTXDQEXTLDPART_EXIST_IN_QM0_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_EXIST_IN_QM0_SHIFT 0 -#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED1_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED1_SHIFT 1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED2_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED2_SHIFT 2 -#define XSTORMETHCONNAGCTXDQEXTLDPART_EXIST_IN_QM3_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_EXIST_IN_QM3_SHIFT 3 -#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED3_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED3_SHIFT 4 -#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED4_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED4_SHIFT 5 -#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED5_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED5_SHIFT 6 -#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED6_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED6_SHIFT 7 - u8 flags1; -#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED7_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED7_SHIFT 0 -#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED8_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED8_SHIFT 1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED9_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED9_SHIFT 2 -#define XSTORMETHCONNAGCTXDQEXTLDPART_BIT11_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_BIT11_SHIFT 3 -#define XSTORMETHCONNAGCTXDQEXTLDPART_BIT12_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_BIT12_SHIFT 4 -#define XSTORMETHCONNAGCTXDQEXTLDPART_BIT13_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_BIT13_SHIFT 5 -#define XSTORMETHCONNAGCTXDQEXTLDPART_TX_RULE_ACTIVE_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_TX_RULE_ACTIVE_SHIFT 6 -#define XSTORMETHCONNAGCTXDQEXTLDPART_DQ_CF_ACTIVE_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_DQ_CF_ACTIVE_SHIFT 7 - u8 flags2; -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF0_MASK 0x3 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF0_SHIFT 0 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF1_MASK 0x3 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF1_SHIFT 2 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF2_MASK 0x3 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF2_SHIFT 4 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF3_MASK 0x3 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF3_SHIFT 6 - u8 flags3; -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF4_MASK 0x3 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF4_SHIFT 0 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF5_MASK 0x3 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF5_SHIFT 2 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF6_MASK 0x3 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF6_SHIFT 4 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF7_MASK 0x3 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF7_SHIFT 6 - u8 flags4; -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF8_MASK 0x3 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF8_SHIFT 0 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF9_MASK 0x3 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF9_SHIFT 2 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF10_MASK 0x3 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF10_SHIFT 4 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF11_MASK 0x3 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF11_SHIFT 6 - u8 flags5; -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF12_MASK 0x3 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF12_SHIFT 0 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF13_MASK 0x3 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF13_SHIFT 2 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF14_MASK 0x3 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF14_SHIFT 4 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF15_MASK 0x3 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF15_SHIFT 6 - u8 flags6; -#define XSTORMETHCONNAGCTXDQEXTLDPART_GO_TO_BD_CONS_CF_MASK 0x3 -#define XSTORMETHCONNAGCTXDQEXTLDPART_GO_TO_BD_CONS_CF_SHIFT 0 -#define XSTORMETHCONNAGCTXDQEXTLDPART_MULTI_UNICAST_CF_MASK 0x3 -#define XSTORMETHCONNAGCTXDQEXTLDPART_MULTI_UNICAST_CF_SHIFT 2 -#define XSTORMETHCONNAGCTXDQEXTLDPART_DQ_CF_MASK 0x3 -#define XSTORMETHCONNAGCTXDQEXTLDPART_DQ_CF_SHIFT 4 -#define XSTORMETHCONNAGCTXDQEXTLDPART_TERMINATE_CF_MASK 0x3 -#define XSTORMETHCONNAGCTXDQEXTLDPART_TERMINATE_CF_SHIFT 6 - u8 flags7; -#define XSTORMETHCONNAGCTXDQEXTLDPART_FLUSH_Q0_MASK 0x3 -#define XSTORMETHCONNAGCTXDQEXTLDPART_FLUSH_Q0_SHIFT 0 -#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED10_MASK 0x3 -#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED10_SHIFT 2 -#define XSTORMETHCONNAGCTXDQEXTLDPART_SLOW_PATH_MASK 0x3 -#define XSTORMETHCONNAGCTXDQEXTLDPART_SLOW_PATH_SHIFT 4 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF0EN_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF0EN_SHIFT 6 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF1EN_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF1EN_SHIFT 7 - u8 flags8; -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF2EN_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF2EN_SHIFT 0 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF3EN_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF3EN_SHIFT 1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF4EN_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF4EN_SHIFT 2 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF5EN_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF5EN_SHIFT 3 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF6EN_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF6EN_SHIFT 4 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF7EN_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF7EN_SHIFT 5 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF8EN_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF8EN_SHIFT 6 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF9EN_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF9EN_SHIFT 7 - u8 flags9; -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF10EN_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF10EN_SHIFT 0 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF11EN_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF11EN_SHIFT 1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF12EN_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF12EN_SHIFT 2 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF13EN_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF13EN_SHIFT 3 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF14EN_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF14EN_SHIFT 4 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF15EN_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_CF15EN_SHIFT 5 -#define XSTORMETHCONNAGCTXDQEXTLDPART_GO_TO_BD_CONS_CF_EN_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_GO_TO_BD_CONS_CF_EN_SHIFT 6 -#define XSTORMETHCONNAGCTXDQEXTLDPART_MULTI_UNICAST_CF_EN_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_MULTI_UNICAST_CF_EN_SHIFT 7 - u8 flags10; -#define XSTORMETHCONNAGCTXDQEXTLDPART_DQ_CF_EN_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_DQ_CF_EN_SHIFT 0 -#define XSTORMETHCONNAGCTXDQEXTLDPART_TERMINATE_CF_EN_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_TERMINATE_CF_EN_SHIFT 1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_FLUSH_Q0_EN_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_FLUSH_Q0_EN_SHIFT 2 -#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED11_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED11_SHIFT 3 -#define XSTORMETHCONNAGCTXDQEXTLDPART_SLOW_PATH_EN_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_SLOW_PATH_EN_SHIFT 4 -#define XSTORMETHCONNAGCTXDQEXTLDPART_TPH_ENABLE_EN_RESERVED_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_TPH_ENABLE_EN_RESERVED_SHIFT 5 -#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED12_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED12_SHIFT 6 -#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED13_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED13_SHIFT 7 - u8 flags11; -#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED14_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED14_SHIFT 0 -#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED15_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED15_SHIFT 1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_TX_DEC_RULE_EN_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_TX_DEC_RULE_EN_SHIFT 2 -#define XSTORMETHCONNAGCTXDQEXTLDPART_RULE5EN_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_RULE5EN_SHIFT 3 -#define XSTORMETHCONNAGCTXDQEXTLDPART_RULE6EN_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_RULE6EN_SHIFT 4 -#define XSTORMETHCONNAGCTXDQEXTLDPART_RULE7EN_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_RULE7EN_SHIFT 5 -#define XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED1_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED1_SHIFT 6 -#define XSTORMETHCONNAGCTXDQEXTLDPART_RULE9EN_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_RULE9EN_SHIFT 7 - u8 flags12; -#define XSTORMETHCONNAGCTXDQEXTLDPART_RULE10EN_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_RULE10EN_SHIFT 0 -#define XSTORMETHCONNAGCTXDQEXTLDPART_RULE11EN_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_RULE11EN_SHIFT 1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED2_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED2_SHIFT 2 -#define XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED3_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED3_SHIFT 3 -#define XSTORMETHCONNAGCTXDQEXTLDPART_RULE14EN_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_RULE14EN_SHIFT 4 -#define XSTORMETHCONNAGCTXDQEXTLDPART_RULE15EN_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_RULE15EN_SHIFT 5 -#define XSTORMETHCONNAGCTXDQEXTLDPART_RULE16EN_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_RULE16EN_SHIFT 6 -#define XSTORMETHCONNAGCTXDQEXTLDPART_RULE17EN_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_RULE17EN_SHIFT 7 - u8 flags13; -#define XSTORMETHCONNAGCTXDQEXTLDPART_RULE18EN_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_RULE18EN_SHIFT 0 -#define XSTORMETHCONNAGCTXDQEXTLDPART_RULE19EN_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_RULE19EN_SHIFT 1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED4_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED4_SHIFT 2 -#define XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED5_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED5_SHIFT 3 -#define XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED6_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED6_SHIFT 4 -#define XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED7_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED7_SHIFT 5 -#define XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED8_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED8_SHIFT 6 -#define XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED9_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED9_SHIFT 7 - u8 flags14; -#define XSTORMETHCONNAGCTXDQEXTLDPART_EDPM_USE_EXT_HDR_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_EDPM_USE_EXT_HDR_SHIFT 0 -#define XSTORMETHCONNAGCTXDQEXTLDPART_EDPM_SEND_RAW_L3L4_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_EDPM_SEND_RAW_L3L4_SHIFT 1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_EDPM_INBAND_PROP_HDR_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_EDPM_INBAND_PROP_HDR_SHIFT 2 -#define XSTORMETHCONNAGCTXDQEXTLDPART_EDPM_SEND_EXT_TUNNEL_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_EDPM_SEND_EXT_TUNNEL_SHIFT 3 -#define XSTORMETHCONNAGCTXDQEXTLDPART_L2_EDPM_ENABLE_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_L2_EDPM_ENABLE_SHIFT 4 -#define XSTORMETHCONNAGCTXDQEXTLDPART_ROCE_EDPM_ENABLE_MASK 0x1 -#define XSTORMETHCONNAGCTXDQEXTLDPART_ROCE_EDPM_ENABLE_SHIFT 5 -#define XSTORMETHCONNAGCTXDQEXTLDPART_TPH_ENABLE_MASK 0x3 -#define XSTORMETHCONNAGCTXDQEXTLDPART_TPH_ENABLE_SHIFT 6 - u8 edpm_event_id /* byte2 */; - __le16 physical_q0 /* physical_q0 */; - __le16 word1 /* physical_q1 */; - __le16 edpm_num_bds /* physical_q2 */; - __le16 tx_bd_cons /* word3 */; - __le16 tx_bd_prod /* word4 */; - __le16 go_to_bd_cons /* word5 */; - __le16 conn_dpi /* conn_dpi */; - u8 byte3 /* byte3 */; - u8 byte4 /* byte4 */; - u8 byte5 /* byte5 */; - u8 byte6 /* byte6 */; - __le32 reg0 /* reg0 */; - __le32 reg1 /* reg1 */; - __le32 reg2 /* reg2 */; - __le32 reg3 /* reg3 */; - __le32 reg4 /* reg4 */; -}; - -struct xstorm_eth_hw_conn_ag_ctx { - u8 reserved0 /* cdu_validation */; - u8 eth_state /* state */; - u8 flags0; -#define XSTORM_ETH_HW_CONN_AG_CTX_EXIST_IN_QM0_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_EXIST_IN_QM0_SHIFT 0 -#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED1_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED1_SHIFT 1 -#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED2_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED2_SHIFT 2 -#define XSTORM_ETH_HW_CONN_AG_CTX_EXIST_IN_QM3_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_EXIST_IN_QM3_SHIFT 3 -#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED3_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED3_SHIFT 4 -#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED4_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED4_SHIFT 5 -#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED5_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED5_SHIFT 6 -#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED6_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED6_SHIFT 7 - u8 flags1; -#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED7_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED7_SHIFT 0 -#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED8_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED8_SHIFT 1 -#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED9_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED9_SHIFT 2 -#define XSTORM_ETH_HW_CONN_AG_CTX_BIT11_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_BIT11_SHIFT 3 -#define XSTORM_ETH_HW_CONN_AG_CTX_BIT12_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_BIT12_SHIFT 4 -#define XSTORM_ETH_HW_CONN_AG_CTX_BIT13_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_BIT13_SHIFT 5 -#define XSTORM_ETH_HW_CONN_AG_CTX_TX_RULE_ACTIVE_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_TX_RULE_ACTIVE_SHIFT 6 -#define XSTORM_ETH_HW_CONN_AG_CTX_DQ_CF_ACTIVE_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_DQ_CF_ACTIVE_SHIFT 7 - u8 flags2; -#define XSTORM_ETH_HW_CONN_AG_CTX_CF0_MASK 0x3 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF0_SHIFT 0 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF1_MASK 0x3 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF1_SHIFT 2 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF2_MASK 0x3 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF2_SHIFT 4 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF3_MASK 0x3 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF3_SHIFT 6 - u8 flags3; -#define XSTORM_ETH_HW_CONN_AG_CTX_CF4_MASK 0x3 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF4_SHIFT 0 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF5_MASK 0x3 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF5_SHIFT 2 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF6_MASK 0x3 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF6_SHIFT 4 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF7_MASK 0x3 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF7_SHIFT 6 - u8 flags4; -#define XSTORM_ETH_HW_CONN_AG_CTX_CF8_MASK 0x3 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF8_SHIFT 0 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF9_MASK 0x3 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF9_SHIFT 2 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF10_MASK 0x3 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF10_SHIFT 4 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF11_MASK 0x3 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF11_SHIFT 6 - u8 flags5; -#define XSTORM_ETH_HW_CONN_AG_CTX_CF12_MASK 0x3 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF12_SHIFT 0 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF13_MASK 0x3 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF13_SHIFT 2 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF14_MASK 0x3 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF14_SHIFT 4 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF15_MASK 0x3 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF15_SHIFT 6 - u8 flags6; -#define XSTORM_ETH_HW_CONN_AG_CTX_GO_TO_BD_CONS_CF_MASK 0x3 -#define XSTORM_ETH_HW_CONN_AG_CTX_GO_TO_BD_CONS_CF_SHIFT 0 -#define XSTORM_ETH_HW_CONN_AG_CTX_MULTI_UNICAST_CF_MASK 0x3 -#define XSTORM_ETH_HW_CONN_AG_CTX_MULTI_UNICAST_CF_SHIFT 2 -#define XSTORM_ETH_HW_CONN_AG_CTX_DQ_CF_MASK 0x3 -#define XSTORM_ETH_HW_CONN_AG_CTX_DQ_CF_SHIFT 4 -#define XSTORM_ETH_HW_CONN_AG_CTX_TERMINATE_CF_MASK 0x3 -#define XSTORM_ETH_HW_CONN_AG_CTX_TERMINATE_CF_SHIFT 6 - u8 flags7; -#define XSTORM_ETH_HW_CONN_AG_CTX_FLUSH_Q0_MASK 0x3 -#define XSTORM_ETH_HW_CONN_AG_CTX_FLUSH_Q0_SHIFT 0 -#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED10_MASK 0x3 -#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED10_SHIFT 2 -#define XSTORM_ETH_HW_CONN_AG_CTX_SLOW_PATH_MASK 0x3 -#define XSTORM_ETH_HW_CONN_AG_CTX_SLOW_PATH_SHIFT 4 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF0EN_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF0EN_SHIFT 6 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF1EN_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF1EN_SHIFT 7 - u8 flags8; -#define XSTORM_ETH_HW_CONN_AG_CTX_CF2EN_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF2EN_SHIFT 0 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF3EN_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF3EN_SHIFT 1 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF4EN_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF4EN_SHIFT 2 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF5EN_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF5EN_SHIFT 3 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF6EN_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF6EN_SHIFT 4 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF7EN_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF7EN_SHIFT 5 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF8EN_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF8EN_SHIFT 6 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF9EN_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF9EN_SHIFT 7 - u8 flags9; -#define XSTORM_ETH_HW_CONN_AG_CTX_CF10EN_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF10EN_SHIFT 0 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF11EN_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF11EN_SHIFT 1 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF12EN_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF12EN_SHIFT 2 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF13EN_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF13EN_SHIFT 3 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF14EN_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF14EN_SHIFT 4 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF15EN_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_CF15EN_SHIFT 5 -#define XSTORM_ETH_HW_CONN_AG_CTX_GO_TO_BD_CONS_CF_EN_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_GO_TO_BD_CONS_CF_EN_SHIFT 6 -#define XSTORM_ETH_HW_CONN_AG_CTX_MULTI_UNICAST_CF_EN_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_MULTI_UNICAST_CF_EN_SHIFT 7 - u8 flags10; -#define XSTORM_ETH_HW_CONN_AG_CTX_DQ_CF_EN_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_DQ_CF_EN_SHIFT 0 -#define XSTORM_ETH_HW_CONN_AG_CTX_TERMINATE_CF_EN_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_TERMINATE_CF_EN_SHIFT 1 -#define XSTORM_ETH_HW_CONN_AG_CTX_FLUSH_Q0_EN_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_FLUSH_Q0_EN_SHIFT 2 -#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED11_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED11_SHIFT 3 -#define XSTORM_ETH_HW_CONN_AG_CTX_SLOW_PATH_EN_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_SLOW_PATH_EN_SHIFT 4 -#define XSTORM_ETH_HW_CONN_AG_CTX_TPH_ENABLE_EN_RESERVED_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_TPH_ENABLE_EN_RESERVED_SHIFT 5 -#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED12_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED12_SHIFT 6 -#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED13_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED13_SHIFT 7 - u8 flags11; -#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED14_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED14_SHIFT 0 -#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED15_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED15_SHIFT 1 -#define XSTORM_ETH_HW_CONN_AG_CTX_TX_DEC_RULE_EN_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_TX_DEC_RULE_EN_SHIFT 2 -#define XSTORM_ETH_HW_CONN_AG_CTX_RULE5EN_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_RULE5EN_SHIFT 3 -#define XSTORM_ETH_HW_CONN_AG_CTX_RULE6EN_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_RULE6EN_SHIFT 4 -#define XSTORM_ETH_HW_CONN_AG_CTX_RULE7EN_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_RULE7EN_SHIFT 5 -#define XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED1_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED1_SHIFT 6 -#define XSTORM_ETH_HW_CONN_AG_CTX_RULE9EN_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_RULE9EN_SHIFT 7 - u8 flags12; -#define XSTORM_ETH_HW_CONN_AG_CTX_RULE10EN_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_RULE10EN_SHIFT 0 -#define XSTORM_ETH_HW_CONN_AG_CTX_RULE11EN_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_RULE11EN_SHIFT 1 -#define XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED2_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED2_SHIFT 2 -#define XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED3_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED3_SHIFT 3 -#define XSTORM_ETH_HW_CONN_AG_CTX_RULE14EN_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_RULE14EN_SHIFT 4 -#define XSTORM_ETH_HW_CONN_AG_CTX_RULE15EN_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_RULE15EN_SHIFT 5 -#define XSTORM_ETH_HW_CONN_AG_CTX_RULE16EN_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_RULE16EN_SHIFT 6 -#define XSTORM_ETH_HW_CONN_AG_CTX_RULE17EN_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_RULE17EN_SHIFT 7 - u8 flags13; -#define XSTORM_ETH_HW_CONN_AG_CTX_RULE18EN_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_RULE18EN_SHIFT 0 -#define XSTORM_ETH_HW_CONN_AG_CTX_RULE19EN_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_RULE19EN_SHIFT 1 -#define XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED4_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED4_SHIFT 2 -#define XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED5_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED5_SHIFT 3 -#define XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED6_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED6_SHIFT 4 -#define XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED7_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED7_SHIFT 5 -#define XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED8_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED8_SHIFT 6 -#define XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED9_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED9_SHIFT 7 - u8 flags14; -#define XSTORM_ETH_HW_CONN_AG_CTX_EDPM_USE_EXT_HDR_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_EDPM_USE_EXT_HDR_SHIFT 0 -#define XSTORM_ETH_HW_CONN_AG_CTX_EDPM_SEND_RAW_L3L4_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_EDPM_SEND_RAW_L3L4_SHIFT 1 -#define XSTORM_ETH_HW_CONN_AG_CTX_EDPM_INBAND_PROP_HDR_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_EDPM_INBAND_PROP_HDR_SHIFT 2 -#define XSTORM_ETH_HW_CONN_AG_CTX_EDPM_SEND_EXT_TUNNEL_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_EDPM_SEND_EXT_TUNNEL_SHIFT 3 -#define XSTORM_ETH_HW_CONN_AG_CTX_L2_EDPM_ENABLE_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_L2_EDPM_ENABLE_SHIFT 4 -#define XSTORM_ETH_HW_CONN_AG_CTX_ROCE_EDPM_ENABLE_MASK 0x1 -#define XSTORM_ETH_HW_CONN_AG_CTX_ROCE_EDPM_ENABLE_SHIFT 5 -#define XSTORM_ETH_HW_CONN_AG_CTX_TPH_ENABLE_MASK 0x3 -#define XSTORM_ETH_HW_CONN_AG_CTX_TPH_ENABLE_SHIFT 6 - u8 edpm_event_id /* byte2 */; - __le16 physical_q0 /* physical_q0 */; - __le16 word1 /* physical_q1 */; - __le16 edpm_num_bds /* physical_q2 */; - __le16 tx_bd_cons /* word3 */; - __le16 tx_bd_prod /* word4 */; - __le16 go_to_bd_cons /* word5 */; - __le16 conn_dpi /* conn_dpi */; -}; #endif /* __ECORE_HSI_ETH__ */ diff --git a/dpdk/drivers/net/qede/base/ecore_hsi_init_func.h b/dpdk/drivers/net/qede/base/ecore_hsi_init_func.h new file mode 100644 index 00000000..48b0048f --- /dev/null +++ b/dpdk/drivers/net/qede/base/ecore_hsi_init_func.h @@ -0,0 +1,136 @@ +/* + * Copyright (c) 2016 QLogic Corporation. + * All rights reserved. + * www.qlogic.com + * + * See LICENSE.qede_pmd for copyright and licensing details. + */ + +#ifndef __ECORE_HSI_INIT_FUNC__ +#define __ECORE_HSI_INIT_FUNC__ +/********************************/ +/* HSI Init Functions constants */ +/********************************/ + +/* Number of VLAN priorities */ +#define NUM_OF_VLAN_PRIORITIES 8 + +/* Size of CRC8 lookup table */ +#ifndef LINUX_REMOVE +#define CRC8_TABLE_SIZE 256 +#endif + +/* + * BRB RAM init requirements + */ +struct init_brb_ram_req { + __le32 guranteed_per_tc /* guaranteed size per TC, in bytes */; + __le32 headroom_per_tc /* headroom size per TC, in bytes */; + __le32 min_pkt_size /* min packet size, in bytes */; + __le32 max_ports_per_engine /* min packet size, in bytes */; + u8 num_active_tcs[MAX_NUM_PORTS] /* number of active TCs per port */; +}; + + +/* + * ETS per-TC init requirements + */ +struct init_ets_tc_req { +/* if set, this TC participates in the arbitration with a strict priority + * (the priority is equal to the TC ID) + */ + u8 use_sp; +/* if set, this TC participates in the arbitration with a WFQ weight + * (indicated by the weight field) + */ + u8 use_wfq; +/* An arbitration weight. Valid only if use_wfq is set. */ + __le16 weight; +}; + +/* + * ETS init requirements + */ +struct init_ets_req { + __le32 mtu /* Max packet size (in bytes) */; +/* ETS initialization requirements per TC. */ + struct init_ets_tc_req tc_req[NUM_OF_TCS]; +}; + + + +/* + * NIG LB RL init requirements + */ +struct init_nig_lb_rl_req { +/* Global MAC+LB RL rate (in Mbps). If set to 0, the RL will be disabled. */ + __le16 lb_mac_rate; +/* Global LB RL rate (in Mbps). If set to 0, the RL will be disabled. */ + __le16 lb_rate; + __le32 mtu /* Max packet size (in bytes) */; +/* RL rate per physical TC (in Mbps). If set to 0, the RL will be disabled. */ + __le16 tc_rate[NUM_OF_PHYS_TCS]; +}; + + +/* + * NIG TC mapping for each priority + */ +struct init_nig_pri_tc_map_entry { + u8 tc_id /* the mapped TC ID */; + u8 valid /* indicates if the mapping entry is valid */; +}; + + +/* + * NIG priority to TC map init requirements + */ +struct init_nig_pri_tc_map_req { + struct init_nig_pri_tc_map_entry pri[NUM_OF_VLAN_PRIORITIES]; +}; + + +/* + * QM per-port init parameters + */ +struct init_qm_port_params { + u8 active /* Indicates if this port is active */; +/* Vector of valid bits for active TCs used by this port */ + u8 active_phys_tcs; +/* number of PBF command lines that can be used by this port */ + __le16 num_pbf_cmd_lines; +/* number of BTB blocks that can be used by this port */ + __le16 num_btb_blocks; + __le16 reserved; +}; + + +/* + * QM per-PQ init parameters + */ +struct init_qm_pq_params { + u8 vport_id /* VPORT ID */; + u8 tc_id /* TC ID */; + u8 wrr_group /* WRR group */; +/* Indicates if a rate limiter should be allocated for the PQ (0/1) */ + u8 rl_valid; +}; + + +/* + * QM per-vport init parameters + */ +struct init_qm_vport_params { +/* rate limit in Mb/sec units. a value of 0 means dont configure. ignored if + * VPORT RL is globally disabled. + */ + __le32 vport_rl; +/* WFQ weight. A value of 0 means dont configure. ignored if VPORT WFQ is + * globally disabled. + */ + __le16 vport_wfq; +/* the first Tx PQ ID associated with this VPORT for each TC. */ + __le16 first_tx_pq_id[NUM_OF_TCS]; +}; + +#endif /* __ECORE_HSI_INIT_FUNC__ */ diff --git a/dpdk/drivers/net/qede/base/ecore_hsi_init_tool.h b/dpdk/drivers/net/qede/base/ecore_hsi_init_tool.h new file mode 100644 index 00000000..1f57e9b2 --- /dev/null +++ b/dpdk/drivers/net/qede/base/ecore_hsi_init_tool.h @@ -0,0 +1,454 @@ +/* + * Copyright (c) 2016 QLogic Corporation. + * All rights reserved. + * www.qlogic.com + * + * See LICENSE.qede_pmd for copyright and licensing details. + */ + +#ifndef __ECORE_HSI_INIT_TOOL__ +#define __ECORE_HSI_INIT_TOOL__ +/**************************************/ +/* Init Tool HSI constants and macros */ +/**************************************/ + +/* Width of GRC address in bits (addresses are specified in dwords) */ +#define GRC_ADDR_BITS 23 +#define MAX_GRC_ADDR ((1 << GRC_ADDR_BITS) - 1) + +/* indicates an init that should be applied to any phase ID */ +#define ANY_PHASE_ID 0xffff + +/* Max size in dwords of a zipped array */ +#define MAX_ZIPPED_SIZE 8192 + +enum chip_ids { + CHIP_BB, + CHIP_K2, + CHIP_E5, + MAX_CHIP_IDS +}; + + +struct fw_asserts_ram_section { +/* The offset of the section in the RAM in RAM lines (64-bit units) */ + __le16 section_ram_line_offset; +/* The size of the section in RAM lines (64-bit units) */ + __le16 section_ram_line_size; +/* The offset of the asserts list within the section in dwords */ + u8 list_dword_offset; +/* The size of an assert list element in dwords */ + u8 list_element_dword_size; + u8 list_num_elements /* The number of elements in the asserts list */; +/* The offset of the next list index field within the section in dwords */ + u8 list_next_index_dword_offset; +}; + + +struct fw_ver_num { + u8 major /* Firmware major version number */; + u8 minor /* Firmware minor version number */; + u8 rev /* Firmware revision version number */; +/* Firmware engineering version number (for bootleg versions) */ + u8 eng; +}; + +struct fw_ver_info { + __le16 tools_ver /* Tools version number */; + u8 image_id /* FW image ID (e.g. main, l2b, kuku) */; + u8 reserved1; + struct fw_ver_num num /* FW version number */; + __le32 timestamp /* FW Timestamp in unix time (sec. since 1970) */; + __le32 reserved2; +}; + +struct fw_info { + struct fw_ver_info ver /* FW version information */; +/* Info regarding the FW asserts section in the Storm RAM */ + struct fw_asserts_ram_section fw_asserts_section; +}; + + +struct fw_info_location { +/* GRC address where the fw_info struct is located. */ + __le32 grc_addr; +/* Size of the fw_info structure (thats located at the grc_addr). */ + __le32 size; +}; + +/* + * Binary buffer header + */ +struct bin_buffer_hdr { +/* buffer offset in bytes from the beginning of the binary file */ + __le32 offset; + __le32 length /* buffer length in bytes */; +}; + + +/* + * binary init buffer types + */ +enum bin_init_buffer_type { + BIN_BUF_INIT_FW_VER_INFO /* fw_ver_info struct */, + BIN_BUF_INIT_CMD /* init commands */, + BIN_BUF_INIT_VAL /* init data */, + BIN_BUF_INIT_MODE_TREE /* init modes tree */, + BIN_BUF_INIT_IRO /* internal RAM offsets */, + MAX_BIN_INIT_BUFFER_TYPE +}; + + +/* + * init array header: raw + */ +struct init_array_raw_hdr { + __le32 data; +/* Init array type, from init_array_types enum */ +#define INIT_ARRAY_RAW_HDR_TYPE_MASK 0xF +#define INIT_ARRAY_RAW_HDR_TYPE_SHIFT 0 +/* init array params */ +#define INIT_ARRAY_RAW_HDR_PARAMS_MASK 0xFFFFFFF +#define INIT_ARRAY_RAW_HDR_PARAMS_SHIFT 4 +}; + +/* + * init array header: standard + */ +struct init_array_standard_hdr { + __le32 data; +/* Init array type, from init_array_types enum */ +#define INIT_ARRAY_STANDARD_HDR_TYPE_MASK 0xF +#define INIT_ARRAY_STANDARD_HDR_TYPE_SHIFT 0 +/* Init array size (in dwords) */ +#define INIT_ARRAY_STANDARD_HDR_SIZE_MASK 0xFFFFFFF +#define INIT_ARRAY_STANDARD_HDR_SIZE_SHIFT 4 +}; + +/* + * init array header: zipped + */ +struct init_array_zipped_hdr { + __le32 data; +/* Init array type, from init_array_types enum */ +#define INIT_ARRAY_ZIPPED_HDR_TYPE_MASK 0xF +#define INIT_ARRAY_ZIPPED_HDR_TYPE_SHIFT 0 +/* Init array zipped size (in bytes) */ +#define INIT_ARRAY_ZIPPED_HDR_ZIPPED_SIZE_MASK 0xFFFFFFF +#define INIT_ARRAY_ZIPPED_HDR_ZIPPED_SIZE_SHIFT 4 +}; + +/* + * init array header: pattern + */ +struct init_array_pattern_hdr { + __le32 data; +/* Init array type, from init_array_types enum */ +#define INIT_ARRAY_PATTERN_HDR_TYPE_MASK 0xF +#define INIT_ARRAY_PATTERN_HDR_TYPE_SHIFT 0 +/* pattern size in dword */ +#define INIT_ARRAY_PATTERN_HDR_PATTERN_SIZE_MASK 0xF +#define INIT_ARRAY_PATTERN_HDR_PATTERN_SIZE_SHIFT 4 +/* pattern repetitions */ +#define INIT_ARRAY_PATTERN_HDR_REPETITIONS_MASK 0xFFFFFF +#define INIT_ARRAY_PATTERN_HDR_REPETITIONS_SHIFT 8 +}; + +/* + * init array header union + */ +union init_array_hdr { + struct init_array_raw_hdr raw /* raw init array header */; +/* standard init array header */ + struct init_array_standard_hdr standard; + struct init_array_zipped_hdr zipped /* zipped init array header */; + struct init_array_pattern_hdr pattern /* pattern init array header */; +}; + + +enum init_modes { + MODE_BB_A0_DEPRECATED, + MODE_BB, + MODE_K2, + MODE_ASIC, + MODE_EMUL_REDUCED, + MODE_EMUL_FULL, + MODE_FPGA, + MODE_CHIPSIM, + MODE_SF, + MODE_MF_SD, + MODE_MF_SI, + MODE_PORTS_PER_ENG_1, + MODE_PORTS_PER_ENG_2, + MODE_PORTS_PER_ENG_4, + MODE_100G, + MODE_E5, + MAX_INIT_MODES +}; + + +enum init_phases { + PHASE_ENGINE, + PHASE_PORT, + PHASE_PF, + PHASE_VF, + PHASE_QM_PF, + MAX_INIT_PHASES +}; + + +enum init_split_types { + SPLIT_TYPE_NONE, + SPLIT_TYPE_PORT, + SPLIT_TYPE_PF, + SPLIT_TYPE_PORT_PF, + SPLIT_TYPE_VF, + MAX_INIT_SPLIT_TYPES +}; + + +/* + * init array types + */ +enum init_array_types { + INIT_ARR_STANDARD /* standard init array */, + INIT_ARR_ZIPPED /* zipped init array */, + INIT_ARR_PATTERN /* a repeated pattern */, + MAX_INIT_ARRAY_TYPES +}; + + + +/* + * init operation: callback + */ +struct init_callback_op { + __le32 op_data; +/* Init operation, from init_op_types enum */ +#define INIT_CALLBACK_OP_OP_MASK 0xF +#define INIT_CALLBACK_OP_OP_SHIFT 0 +#define INIT_CALLBACK_OP_RESERVED_MASK 0xFFFFFFF +#define INIT_CALLBACK_OP_RESERVED_SHIFT 4 + __le16 callback_id /* Callback ID */; + __le16 block_id /* Blocks ID */; +}; + + +/* + * init operation: delay + */ +struct init_delay_op { + __le32 op_data; +/* Init operation, from init_op_types enum */ +#define INIT_DELAY_OP_OP_MASK 0xF +#define INIT_DELAY_OP_OP_SHIFT 0 +#define INIT_DELAY_OP_RESERVED_MASK 0xFFFFFFF +#define INIT_DELAY_OP_RESERVED_SHIFT 4 + __le32 delay /* delay in us */; +}; + + +/* + * init operation: if_mode + */ +struct init_if_mode_op { + __le32 op_data; +/* Init operation, from init_op_types enum */ +#define INIT_IF_MODE_OP_OP_MASK 0xF +#define INIT_IF_MODE_OP_OP_SHIFT 0 +#define INIT_IF_MODE_OP_RESERVED1_MASK 0xFFF +#define INIT_IF_MODE_OP_RESERVED1_SHIFT 4 +/* Commands to skip if the modes dont match */ +#define INIT_IF_MODE_OP_CMD_OFFSET_MASK 0xFFFF +#define INIT_IF_MODE_OP_CMD_OFFSET_SHIFT 16 + __le16 reserved2; +/* offset (in bytes) in modes expression buffer */ + __le16 modes_buf_offset; +}; + + +/* + * init operation: if_phase + */ +struct init_if_phase_op { + __le32 op_data; +/* Init operation, from init_op_types enum */ +#define INIT_IF_PHASE_OP_OP_MASK 0xF +#define INIT_IF_PHASE_OP_OP_SHIFT 0 +/* Indicates if DMAE is enabled in this phase */ +#define INIT_IF_PHASE_OP_DMAE_ENABLE_MASK 0x1 +#define INIT_IF_PHASE_OP_DMAE_ENABLE_SHIFT 4 +#define INIT_IF_PHASE_OP_RESERVED1_MASK 0x7FF +#define INIT_IF_PHASE_OP_RESERVED1_SHIFT 5 +/* Commands to skip if the phases dont match */ +#define INIT_IF_PHASE_OP_CMD_OFFSET_MASK 0xFFFF +#define INIT_IF_PHASE_OP_CMD_OFFSET_SHIFT 16 + __le32 phase_data; +#define INIT_IF_PHASE_OP_PHASE_MASK 0xFF /* Init phase */ +#define INIT_IF_PHASE_OP_PHASE_SHIFT 0 +#define INIT_IF_PHASE_OP_RESERVED2_MASK 0xFF +#define INIT_IF_PHASE_OP_RESERVED2_SHIFT 8 +#define INIT_IF_PHASE_OP_PHASE_ID_MASK 0xFFFF /* Init phase ID */ +#define INIT_IF_PHASE_OP_PHASE_ID_SHIFT 16 +}; + + +/* + * init mode operators + */ +enum init_mode_ops { + INIT_MODE_OP_NOT /* init mode not operator */, + INIT_MODE_OP_OR /* init mode or operator */, + INIT_MODE_OP_AND /* init mode and operator */, + MAX_INIT_MODE_OPS +}; + + +/* + * init operation: raw + */ +struct init_raw_op { + __le32 op_data; +/* Init operation, from init_op_types enum */ +#define INIT_RAW_OP_OP_MASK 0xF +#define INIT_RAW_OP_OP_SHIFT 0 +#define INIT_RAW_OP_PARAM1_MASK 0xFFFFFFF /* init param 1 */ +#define INIT_RAW_OP_PARAM1_SHIFT 4 + __le32 param2 /* Init param 2 */; +}; + +/* + * init array params + */ +struct init_op_array_params { + __le16 size /* array size in dwords */; + __le16 offset /* array start offset in dwords */; +}; + +/* + * Write init operation arguments + */ +union init_write_args { +/* value to write, used when init source is INIT_SRC_INLINE */ + __le32 inline_val; +/* number of zeros to write, used when init source is INIT_SRC_ZEROS */ + __le32 zeros_count; +/* array offset to write, used when init source is INIT_SRC_ARRAY */ + __le32 array_offset; +/* runtime array params to write, used when init source is INIT_SRC_RUNTIME */ + struct init_op_array_params runtime; +}; + +/* + * init operation: write + */ +struct init_write_op { + __le32 data; +/* init operation, from init_op_types enum */ +#define INIT_WRITE_OP_OP_MASK 0xF +#define INIT_WRITE_OP_OP_SHIFT 0 +/* init source type, taken from init_source_types enum */ +#define INIT_WRITE_OP_SOURCE_MASK 0x7 +#define INIT_WRITE_OP_SOURCE_SHIFT 4 +#define INIT_WRITE_OP_RESERVED_MASK 0x1 +#define INIT_WRITE_OP_RESERVED_SHIFT 7 +/* indicates if the register is wide-bus */ +#define INIT_WRITE_OP_WIDE_BUS_MASK 0x1 +#define INIT_WRITE_OP_WIDE_BUS_SHIFT 8 +/* internal (absolute) GRC address, in dwords */ +#define INIT_WRITE_OP_ADDRESS_MASK 0x7FFFFF +#define INIT_WRITE_OP_ADDRESS_SHIFT 9 + union init_write_args args /* Write init operation arguments */; +}; + +/* + * init operation: read + */ +struct init_read_op { + __le32 op_data; +/* init operation, from init_op_types enum */ +#define INIT_READ_OP_OP_MASK 0xF +#define INIT_READ_OP_OP_SHIFT 0 +/* polling type, from init_poll_types enum */ +#define INIT_READ_OP_POLL_TYPE_MASK 0xF +#define INIT_READ_OP_POLL_TYPE_SHIFT 4 +#define INIT_READ_OP_RESERVED_MASK 0x1 +#define INIT_READ_OP_RESERVED_SHIFT 8 +/* internal (absolute) GRC address, in dwords */ +#define INIT_READ_OP_ADDRESS_MASK 0x7FFFFF +#define INIT_READ_OP_ADDRESS_SHIFT 9 +/* expected polling value, used only when polling is done */ + __le32 expected_val; +}; + +/* + * Init operations union + */ +union init_op { + struct init_raw_op raw /* raw init operation */; + struct init_write_op write /* write init operation */; + struct init_read_op read /* read init operation */; + struct init_if_mode_op if_mode /* if_mode init operation */; + struct init_if_phase_op if_phase /* if_phase init operation */; + struct init_callback_op callback /* callback init operation */; + struct init_delay_op delay /* delay init operation */; +}; + + + +/* + * Init command operation types + */ +enum init_op_types { + INIT_OP_READ /* GRC read init command */, + INIT_OP_WRITE /* GRC write init command */, +/* Skip init commands if the init modes expression doesn't match */ + INIT_OP_IF_MODE, +/* Skip init commands if the init phase doesn't match */ + INIT_OP_IF_PHASE, + INIT_OP_DELAY /* delay init command */, + INIT_OP_CALLBACK /* callback init command */, + MAX_INIT_OP_TYPES +}; + + +/* + * init polling types + */ +enum init_poll_types { + INIT_POLL_NONE /* No polling */, + INIT_POLL_EQ /* init value is included in the init command */, + INIT_POLL_OR /* init value is all zeros */, + INIT_POLL_AND /* init value is an array of values */, + MAX_INIT_POLL_TYPES +}; + + + + +/* + * init source types + */ +enum init_source_types { + INIT_SRC_INLINE /* init value is included in the init command */, + INIT_SRC_ZEROS /* init value is all zeros */, + INIT_SRC_ARRAY /* init value is an array of values */, + INIT_SRC_RUNTIME /* init value is provided during runtime */, + MAX_INIT_SOURCE_TYPES +}; + + + + +/* + * Internal RAM Offsets macro data + */ +struct iro { + __le32 base /* RAM field offset */; + __le16 m1 /* multiplier 1 */; + __le16 m2 /* multiplier 2 */; + __le16 m3 /* multiplier 3 */; + __le16 size /* RAM field size */; +}; + +#endif /* __ECORE_HSI_INIT_TOOL__ */ diff --git a/dpdk/drivers/net/qede/base/ecore_hsi_tools.h b/dpdk/drivers/net/qede/base/ecore_hsi_tools.h deleted file mode 100644 index 18eea762..00000000 --- a/dpdk/drivers/net/qede/base/ecore_hsi_tools.h +++ /dev/null @@ -1,1081 +0,0 @@ -/* - * Copyright (c) 2016 QLogic Corporation. - * All rights reserved. - * www.qlogic.com - * - * See LICENSE.qede_pmd for copyright and licensing details. - */ - -#ifndef __ECORE_HSI_TOOLS__ -#define __ECORE_HSI_TOOLS__ -/**********************************/ -/* Tools HSI constants and macros */ -/**********************************/ - -/*********************************** Init ************************************/ - -/* Width of GRC address in bits (addresses are specified in dwords) */ -#define GRC_ADDR_BITS 23 -#define MAX_GRC_ADDR ((1 << GRC_ADDR_BITS) - 1) - -/* indicates an init that should be applied to any phase ID */ -#define ANY_PHASE_ID 0xffff - -/* init pattern size in bytes */ -#define INIT_PATTERN_SIZE_BITS 4 -#define MAX_INIT_PATTERN_SIZE (1 << INIT_PATTERN_SIZE_BITS) - -/* Max size in dwords of a zipped array */ -#define MAX_ZIPPED_SIZE 8192 - -/* Global PXP window */ -#define NUM_OF_PXP_WIN 19 -#define PXP_WIN_DWORD_SIZE_BITS 10 -#define PXP_WIN_DWORD_SIZE (1 << PXP_WIN_DWORD_SIZE_BITS) -#define PXP_WIN_BYTE_SIZE_BITS (PXP_WIN_DWORD_SIZE_BITS + 2) -#define PXP_WIN_BYTE_SIZE (PXP_WIN_DWORD_SIZE * 4) - -/********************************* GRC Dump **********************************/ - -/* width of GRC dump register sequence length in bits */ -#define DUMP_SEQ_LEN_BITS 8 -#define DUMP_SEQ_LEN_MAX_VAL ((1 << DUMP_SEQ_LEN_BITS) - 1) - -/* width of GRC dump memory length in bits */ -#define DUMP_MEM_LEN_BITS 18 -#define DUMP_MEM_LEN_MAX_VAL ((1 << DUMP_MEM_LEN_BITS) - 1) - -/* width of register type ID in bits */ -#define REG_TYPE_ID_BITS 6 -#define REG_TYPE_ID_MAX_VAL ((1 << REG_TYPE_ID_BITS) - 1) - -/* width of block ID in bits */ -#define BLOCK_ID_BITS 8 -#define BLOCK_ID_MAX_VAL ((1 << BLOCK_ID_BITS) - 1) - -/******************************** Idle Check *********************************/ - -/* max number of idle check predicate immediates */ -#define MAX_IDLE_CHK_PRED_IMM 3 - -/* max number of idle check argument registers */ -#define MAX_IDLE_CHK_READ_REGS 3 - -/* max number of idle check loops */ -#define MAX_IDLE_CHK_LOOPS 0x10000 - -/* max idle check address increment */ -#define MAX_IDLE_CHK_INCREMENT 0x10000 - -/* inicates an undefined idle check line index */ -#define IDLE_CHK_UNDEFINED_LINE_IDX 0xffffff - -/* max number of register values following the idle check header for LSI */ -#define IDLE_CHK_MAX_LSI_DUMP_REGS 2 - -/* arguments for IDLE_CHK_MACRO_TYPE_QM_RD_WR */ -#define IDLE_CHK_QM_RD_WR_PTR 0 -#define IDLE_CHK_QM_RD_WR_BANK 1 - -/**************************************/ -/* HSI Functions constants and macros */ -/**************************************/ - -/* Number of VLAN priorities */ -#define NUM_OF_VLAN_PRIORITIES 8 - -/* the MCP Trace meta data signautre is duplicated in the - * perl script that generats the NVRAM images - */ -#define MCP_TRACE_META_IMAGE_SIGNATURE 0x669955aa - -/* Maximal number of RAM lines occupied by FW Asserts data */ -#define MAX_FW_ASSERTS_RAM_LINES 800 - -/* - * Binary buffer header - */ -struct bin_buffer_hdr { - __le32 offset - /* buffer offset in bytes from the beginning of the binary file */; - __le32 length /* buffer length in bytes */; -}; - -/* - * binary buffer types - */ -enum bin_buffer_type { - BIN_BUF_FW_VER_INFO /* fw_ver_info struct */, - BIN_BUF_INIT_CMD /* init commands */, - BIN_BUF_INIT_VAL /* init data */, - BIN_BUF_INIT_MODE_TREE /* init modes tree */, - BIN_BUF_IRO /* internal RAM offsets array */, - MAX_BIN_BUFFER_TYPE -}; - -/* - * Chip IDs - */ -enum chip_ids { - CHIP_BB_A0 /* BB A0 chip ID */, - CHIP_BB_B0 /* BB B0 chip ID */, - CHIP_K2 /* AH chip ID */, - MAX_CHIP_IDS -}; - -/* - * memory dump descriptor - */ -struct dbg_dump_mem_desc { - __le32 dword0; -#define DBG_DUMP_MEM_DESC_ADDRESS_MASK 0xFFFFFF -#define DBG_DUMP_MEM_DESC_ADDRESS_SHIFT 0 -#define DBG_DUMP_MEM_DESC_ASIC_CHIP_MASK_MASK 0xF -#define DBG_DUMP_MEM_DESC_ASIC_CHIP_MASK_SHIFT 24 -#define DBG_DUMP_MEM_DESC_SIM_CHIP_MASK_MASK 0xF -#define DBG_DUMP_MEM_DESC_SIM_CHIP_MASK_SHIFT 28 - __le32 dword1; -#define DBG_DUMP_MEM_DESC_LENGTH_MASK 0x3FFFF -#define DBG_DUMP_MEM_DESC_LENGTH_SHIFT 0 -#define DBG_DUMP_MEM_DESC_REG_TYPE_ID_MASK 0x3F -#define DBG_DUMP_MEM_DESC_REG_TYPE_ID_SHIFT 18 -#define DBG_DUMP_MEM_DESC_BLOCK_ID_MASK 0xFF -#define DBG_DUMP_MEM_DESC_BLOCK_ID_SHIFT 24 -}; - -/* - * registers dump descriptor: chip - */ -struct dbg_dump_regs_chip_desc { - __le32 data; -#define DBG_DUMP_REGS_CHIP_DESC_IS_CHIP_MASK_MASK 0x1 -#define DBG_DUMP_REGS_CHIP_DESC_IS_CHIP_MASK_SHIFT 0 -#define DBG_DUMP_REGS_CHIP_DESC_ASIC_CHIP_MASK_MASK 0x7FFFFF -#define DBG_DUMP_REGS_CHIP_DESC_ASIC_CHIP_MASK_SHIFT 1 -#define DBG_DUMP_REGS_CHIP_DESC_SIM_CHIP_MASK_MASK 0xFF -#define DBG_DUMP_REGS_CHIP_DESC_SIM_CHIP_MASK_SHIFT 24 -}; - -/* - * registers dump descriptor: raw - */ -struct dbg_dump_regs_raw_desc { - __le32 data; -#define DBG_DUMP_REGS_RAW_DESC_IS_CHIP_MASK_MASK 0x1 -#define DBG_DUMP_REGS_RAW_DESC_IS_CHIP_MASK_SHIFT 0 -#define DBG_DUMP_REGS_RAW_DESC_PARAM1_MASK 0x7FFFFF -#define DBG_DUMP_REGS_RAW_DESC_PARAM1_SHIFT 1 -#define DBG_DUMP_REGS_RAW_DESC_PARAM2_MASK 0xFF -#define DBG_DUMP_REGS_RAW_DESC_PARAM2_SHIFT 24 -}; - -/* - * registers dump descriptor: sequence - */ -struct dbg_dump_regs_seq_desc { - __le32 data; -#define DBG_DUMP_REGS_SEQ_DESC_IS_CHIP_MASK_MASK 0x1 -#define DBG_DUMP_REGS_SEQ_DESC_IS_CHIP_MASK_SHIFT 0 -#define DBG_DUMP_REGS_SEQ_DESC_ADDRESS_MASK 0x7FFFFF -#define DBG_DUMP_REGS_SEQ_DESC_ADDRESS_SHIFT 1 -#define DBG_DUMP_REGS_SEQ_DESC_LENGTH_MASK 0xFF -#define DBG_DUMP_REGS_SEQ_DESC_LENGTH_SHIFT 24 -}; - -/* - * registers dump descriptor - */ -union dbg_dump_regs_desc { - struct dbg_dump_regs_raw_desc raw /* dumped registers raw descriptor */ - ; - struct dbg_dump_regs_seq_desc seq /* dumped registers seq descriptor */ - ; - struct dbg_dump_regs_chip_desc chip - /* dumped registers chip descriptor */; -}; - -/* - * idle check macro types - */ -enum idle_chk_macro_types { - IDLE_CHK_MACRO_TYPE_COMPARE /* parametric register comparison */, - IDLE_CHK_MACRO_TYPE_QM_RD_WR /* compare QM r/w pointers and banks */, - MAX_IDLE_CHK_MACRO_TYPES -}; - -/* - * Idle Check result header - */ -struct idle_chk_result_hdr { - __le16 rule_idx /* Idle check rule index in CSV file */; - __le16 loop_idx /* the loop index in which the failure occurred */; - __le16 num_fw_values; - __le16 data; -#define IDLE_CHK_RESULT_HDR_NUM_LSI_VALUES_MASK 0xF -#define IDLE_CHK_RESULT_HDR_NUM_LSI_VALUES_SHIFT 0 -#define IDLE_CHK_RESULT_HDR_LOOP_VALID_MASK 0x1 -#define IDLE_CHK_RESULT_HDR_LOOP_VALID_SHIFT 4 -#define IDLE_CHK_RESULT_HDR_SEVERITY_MASK 0x7 -#define IDLE_CHK_RESULT_HDR_SEVERITY_SHIFT 5 -#define IDLE_CHK_RESULT_HDR_MACRO_TYPE_MASK 0xF -#define IDLE_CHK_RESULT_HDR_MACRO_TYPE_SHIFT 8 -#define IDLE_CHK_RESULT_HDR_MACRO_TYPE_ARG_MASK 0xF -#define IDLE_CHK_RESULT_HDR_MACRO_TYPE_ARG_SHIFT 12 -}; - -/* - * Idle Check rule - */ -struct idle_chk_rule { - __le32 data; -#define IDLE_CHK_RULE_ASIC_CHIP_MASK_MASK 0xF -#define IDLE_CHK_RULE_ASIC_CHIP_MASK_SHIFT 0 -#define IDLE_CHK_RULE_SIM_CHIP_MASK_MASK 0xF -#define IDLE_CHK_RULE_SIM_CHIP_MASK_SHIFT 4 -#define IDLE_CHK_RULE_BLOCK_ID_MASK 0xFF -#define IDLE_CHK_RULE_BLOCK_ID_SHIFT 8 -#define IDLE_CHK_RULE_MACRO_TYPE_MASK 0xF -#define IDLE_CHK_RULE_MACRO_TYPE_SHIFT 16 -#define IDLE_CHK_RULE_SEVERITY_MASK 0x7 -#define IDLE_CHK_RULE_SEVERITY_SHIFT 20 -#define IDLE_CHK_RULE_RESERVED_MASK 0x1 -#define IDLE_CHK_RULE_RESERVED_SHIFT 23 -#define IDLE_CHK_RULE_PRED_ID_MASK 0xFF -#define IDLE_CHK_RULE_PRED_ID_SHIFT 24 - __le16 loop; - __le16 increment - /* address increment of first argument register on each iteration */ - ; - __le32 reg_addr[3]; - __le32 pred_imm[3] - /* immediate values passed as arguments to the idle check rule */; -}; - -/* - * idle check severity types - */ -enum idle_chk_severity_types { - IDLE_CHK_SEVERITY_ERROR /* idle check failure should cause an error */, - IDLE_CHK_SEVERITY_ERROR_NO_TRAFFIC - , - IDLE_CHK_SEVERITY_WARNING - /* idle check failure should cause a warning */, - MAX_IDLE_CHK_SEVERITY_TYPES -}; - -/* - * init array header: raw - */ -struct init_array_raw_hdr { - __le32 data; -#define INIT_ARRAY_RAW_HDR_TYPE_MASK 0xF -#define INIT_ARRAY_RAW_HDR_TYPE_SHIFT 0 -#define INIT_ARRAY_RAW_HDR_PARAMS_MASK 0xFFFFFFF -#define INIT_ARRAY_RAW_HDR_PARAMS_SHIFT 4 -}; - -/* - * init array header: standard - */ -struct init_array_standard_hdr { - __le32 data; -#define INIT_ARRAY_STANDARD_HDR_TYPE_MASK 0xF -#define INIT_ARRAY_STANDARD_HDR_TYPE_SHIFT 0 -#define INIT_ARRAY_STANDARD_HDR_SIZE_MASK 0xFFFFFFF -#define INIT_ARRAY_STANDARD_HDR_SIZE_SHIFT 4 -}; - -/* - * init array header: zipped - */ -struct init_array_zipped_hdr { - __le32 data; -#define INIT_ARRAY_ZIPPED_HDR_TYPE_MASK 0xF -#define INIT_ARRAY_ZIPPED_HDR_TYPE_SHIFT 0 -#define INIT_ARRAY_ZIPPED_HDR_ZIPPED_SIZE_MASK 0xFFFFFFF -#define INIT_ARRAY_ZIPPED_HDR_ZIPPED_SIZE_SHIFT 4 -}; - -/* - * init array header: pattern - */ -struct init_array_pattern_hdr { - __le32 data; -#define INIT_ARRAY_PATTERN_HDR_TYPE_MASK 0xF -#define INIT_ARRAY_PATTERN_HDR_TYPE_SHIFT 0 -#define INIT_ARRAY_PATTERN_HDR_PATTERN_SIZE_MASK 0xF -#define INIT_ARRAY_PATTERN_HDR_PATTERN_SIZE_SHIFT 4 -#define INIT_ARRAY_PATTERN_HDR_REPETITIONS_MASK 0xFFFFFF -#define INIT_ARRAY_PATTERN_HDR_REPETITIONS_SHIFT 8 -}; - -/* - * init array header union - */ -union init_array_hdr { - struct init_array_raw_hdr raw /* raw init array header */; - struct init_array_standard_hdr standard /* standard init array header */ - ; - struct init_array_zipped_hdr zipped /* zipped init array header */; - struct init_array_pattern_hdr pattern /* pattern init array header */; -}; - -/* - * init array types - */ -enum init_array_types { - INIT_ARR_STANDARD /* standard init array */, - INIT_ARR_ZIPPED /* zipped init array */, - INIT_ARR_PATTERN /* a repeated pattern */, - MAX_INIT_ARRAY_TYPES -}; - -/* - * init operation: callback - */ -struct init_callback_op { - __le32 op_data; -#define INIT_CALLBACK_OP_OP_MASK 0xF -#define INIT_CALLBACK_OP_OP_SHIFT 0 -#define INIT_CALLBACK_OP_RESERVED_MASK 0xFFFFFFF -#define INIT_CALLBACK_OP_RESERVED_SHIFT 4 - __le16 callback_id /* Callback ID */; - __le16 block_id /* Blocks ID */; -}; - -/* - * init operation: delay - */ -struct init_delay_op { - __le32 op_data; -#define INIT_DELAY_OP_OP_MASK 0xF -#define INIT_DELAY_OP_OP_SHIFT 0 -#define INIT_DELAY_OP_RESERVED_MASK 0xFFFFFFF -#define INIT_DELAY_OP_RESERVED_SHIFT 4 - __le32 delay /* delay in us */; -}; - -/* - * init operation: if_mode - */ -struct init_if_mode_op { - __le32 op_data; -#define INIT_IF_MODE_OP_OP_MASK 0xF -#define INIT_IF_MODE_OP_OP_SHIFT 0 -#define INIT_IF_MODE_OP_RESERVED1_MASK 0xFFF -#define INIT_IF_MODE_OP_RESERVED1_SHIFT 4 -#define INIT_IF_MODE_OP_CMD_OFFSET_MASK 0xFFFF -#define INIT_IF_MODE_OP_CMD_OFFSET_SHIFT 16 - __le16 reserved2; - __le16 modes_buf_offset - /* offset (in bytes) in modes expression buffer */; -}; - -/* - * init operation: if_phase - */ -struct init_if_phase_op { - __le32 op_data; -#define INIT_IF_PHASE_OP_OP_MASK 0xF -#define INIT_IF_PHASE_OP_OP_SHIFT 0 -#define INIT_IF_PHASE_OP_DMAE_ENABLE_MASK 0x1 -#define INIT_IF_PHASE_OP_DMAE_ENABLE_SHIFT 4 -#define INIT_IF_PHASE_OP_RESERVED1_MASK 0x7FF -#define INIT_IF_PHASE_OP_RESERVED1_SHIFT 5 -#define INIT_IF_PHASE_OP_CMD_OFFSET_MASK 0xFFFF -#define INIT_IF_PHASE_OP_CMD_OFFSET_SHIFT 16 - __le32 phase_data; -#define INIT_IF_PHASE_OP_PHASE_MASK 0xFF -#define INIT_IF_PHASE_OP_PHASE_SHIFT 0 -#define INIT_IF_PHASE_OP_RESERVED2_MASK 0xFF -#define INIT_IF_PHASE_OP_RESERVED2_SHIFT 8 -#define INIT_IF_PHASE_OP_PHASE_ID_MASK 0xFFFF -#define INIT_IF_PHASE_OP_PHASE_ID_SHIFT 16 -}; - -/* - * init mode operators - */ -enum init_mode_ops { - INIT_MODE_OP_NOT /* init mode not operator */, - INIT_MODE_OP_OR /* init mode or operator */, - INIT_MODE_OP_AND /* init mode and operator */, - MAX_INIT_MODE_OPS -}; - -/* - * init operation: raw - */ -struct init_raw_op { - __le32 op_data; -#define INIT_RAW_OP_OP_MASK 0xF -#define INIT_RAW_OP_OP_SHIFT 0 -#define INIT_RAW_OP_PARAM1_MASK 0xFFFFFFF -#define INIT_RAW_OP_PARAM1_SHIFT 4 - __le32 param2 /* Init param 2 */; -}; - -/* - * init array params - */ -struct init_op_array_params { - __le16 size /* array size in dwords */; - __le16 offset /* array start offset in dwords */; -}; - -/* - * Write init operation arguments - */ -union init_write_args { - __le32 inline_val - /* value to write, used when init source is INIT_SRC_INLINE */; - __le32 zeros_count; - __le32 array_offset - /* array offset to write, used when init source is INIT_SRC_ARRAY */ - ; - struct init_op_array_params runtime; -}; - -/* - * init operation: write - */ -struct init_write_op { - __le32 data; -#define INIT_WRITE_OP_OP_MASK 0xF -#define INIT_WRITE_OP_OP_SHIFT 0 -#define INIT_WRITE_OP_SOURCE_MASK 0x7 -#define INIT_WRITE_OP_SOURCE_SHIFT 4 -#define INIT_WRITE_OP_RESERVED_MASK 0x1 -#define INIT_WRITE_OP_RESERVED_SHIFT 7 -#define INIT_WRITE_OP_WIDE_BUS_MASK 0x1 -#define INIT_WRITE_OP_WIDE_BUS_SHIFT 8 -#define INIT_WRITE_OP_ADDRESS_MASK 0x7FFFFF -#define INIT_WRITE_OP_ADDRESS_SHIFT 9 - union init_write_args args /* Write init operation arguments */; -}; - -/* - * init operation: read - */ -struct init_read_op { - __le32 op_data; -#define INIT_READ_OP_OP_MASK 0xF -#define INIT_READ_OP_OP_SHIFT 0 -#define INIT_READ_OP_POLL_TYPE_MASK 0xF -#define INIT_READ_OP_POLL_TYPE_SHIFT 4 -#define INIT_READ_OP_RESERVED_MASK 0x1 -#define INIT_READ_OP_RESERVED_SHIFT 8 -#define INIT_READ_OP_ADDRESS_MASK 0x7FFFFF -#define INIT_READ_OP_ADDRESS_SHIFT 9 - __le32 expected_val - /* expected polling value, used only when polling is done */; -}; - -/* - * Init operations union - */ -union init_op { - struct init_raw_op raw /* raw init operation */; - struct init_write_op write /* write init operation */; - struct init_read_op read /* read init operation */; - struct init_if_mode_op if_mode /* if_mode init operation */; - struct init_if_phase_op if_phase /* if_phase init operation */; - struct init_callback_op callback /* callback init operation */; - struct init_delay_op delay /* delay init operation */; -}; - -/* - * Init command operation types - */ -enum init_op_types { - INIT_OP_READ /* GRC read init command */, - INIT_OP_WRITE /* GRC write init command */, - INIT_OP_IF_MODE - /* Skip init commands if the init modes expression doesn't match */, - INIT_OP_IF_PHASE - /* Skip init commands if the init phase doesn't match */, - INIT_OP_DELAY /* delay init command */, - INIT_OP_CALLBACK /* callback init command */, - MAX_INIT_OP_TYPES -}; - -/* - * init polling types - */ -enum init_poll_types { - INIT_POLL_NONE /* No polling */, - INIT_POLL_EQ /* init value is included in the init command */, - INIT_POLL_OR /* init value is all zeros */, - INIT_POLL_AND /* init value is an array of values */, - MAX_INIT_POLL_TYPES -}; - -/* - * init source types - */ -enum init_source_types { - INIT_SRC_INLINE /* init value is included in the init command */, - INIT_SRC_ZEROS /* init value is all zeros */, - INIT_SRC_ARRAY /* init value is an array of values */, - INIT_SRC_RUNTIME /* init value is provided during runtime */, - MAX_INIT_SOURCE_TYPES -}; - -/* - * Internal RAM Offsets macro data - */ -struct iro { - __le32 base /* RAM field offset */; - __le16 m1 /* multiplier 1 */; - __le16 m2 /* multiplier 2 */; - __le16 m3 /* multiplier 3 */; - __le16 size /* RAM field size */; -}; - -/* - * register descriptor - */ -struct reg_desc { - __le32 data; -#define REG_DESC_ADDRESS_MASK 0xFFFFFF -#define REG_DESC_ADDRESS_SHIFT 0 -#define REG_DESC_SIZE_MASK 0xFF -#define REG_DESC_SIZE_SHIFT 24 -}; - -/* - * Debug Bus block data - */ -struct dbg_bus_block_data { - u8 enabled /* Indicates if the block is enabled for recording (0/1) */; - u8 hw_id /* HW ID associated with the block */; - u8 line_num /* Debug line number to select */; - u8 right_shift /* Number of units to right the debug data (0-3) */; - u8 cycle_en /* 4-bit value: bit i set -> unit i is enabled. */; - u8 force_valid /* 4-bit value: bit i set -> unit i is forced valid. */; - u8 force_frame - /* 4-bit value: bit i set -> unit i frame bit is forced. */; - u8 reserved; -}; - -/* - * Debug Bus Clients - */ -enum dbg_bus_clients { - DBG_BUS_CLIENT_RBCN, - DBG_BUS_CLIENT_RBCP, - DBG_BUS_CLIENT_RBCR, - DBG_BUS_CLIENT_RBCT, - DBG_BUS_CLIENT_RBCU, - DBG_BUS_CLIENT_RBCF, - DBG_BUS_CLIENT_RBCX, - DBG_BUS_CLIENT_RBCS, - DBG_BUS_CLIENT_RBCH, - DBG_BUS_CLIENT_RBCZ, - DBG_BUS_CLIENT_OTHER_ENGINE, - DBG_BUS_CLIENT_TIMESTAMP, - DBG_BUS_CLIENT_CPU, - DBG_BUS_CLIENT_RBCY, - DBG_BUS_CLIENT_RBCQ, - DBG_BUS_CLIENT_RBCM, - DBG_BUS_CLIENT_RBCB, - DBG_BUS_CLIENT_RBCW, - DBG_BUS_CLIENT_RBCV, - MAX_DBG_BUS_CLIENTS -}; - -/* - * Debug Bus constraint operation types - */ -enum dbg_bus_constraint_ops { - DBG_BUS_CONSTRAINT_OP_EQ /* equal */, - DBG_BUS_CONSTRAINT_OP_NE /* not equal */, - DBG_BUS_CONSTRAINT_OP_LT /* less than */, - DBG_BUS_CONSTRAINT_OP_LTC /* less than (cyclic) */, - DBG_BUS_CONSTRAINT_OP_LE /* less than or equal */, - DBG_BUS_CONSTRAINT_OP_LEC /* less than or equal (cyclic) */, - DBG_BUS_CONSTRAINT_OP_GT /* greater than */, - DBG_BUS_CONSTRAINT_OP_GTC /* greater than (cyclic) */, - DBG_BUS_CONSTRAINT_OP_GE /* greater than or equal */, - DBG_BUS_CONSTRAINT_OP_GEC /* greater than or equal (cyclic) */, - MAX_DBG_BUS_CONSTRAINT_OPS -}; - -/* - * Debug Bus memory address - */ -struct dbg_bus_mem_addr { - __le32 lo; - __le32 hi; -}; - -/* - * Debug Bus PCI buffer data - */ -struct dbg_bus_pci_buf_data { - struct dbg_bus_mem_addr phys_addr /* PCI buffer physical address */; - struct dbg_bus_mem_addr virt_addr /* PCI buffer virtual address */; - __le32 size /* PCI buffer size in bytes */; -}; - -/* - * Debug Bus Storm EID range filter params - */ -struct dbg_bus_storm_eid_range_params { - u8 min /* Minimal event ID to filter on */; - u8 max /* Maximal event ID to filter on */; -}; - -/* - * Debug Bus Storm EID mask filter params - */ -struct dbg_bus_storm_eid_mask_params { - u8 val /* Event ID value */; - u8 mask /* Event ID mask. 1s in the mask = dont care bits. */; -}; - -/* - * Debug Bus Storm EID filter params - */ -union dbg_bus_storm_eid_params { - struct dbg_bus_storm_eid_range_params range - /* EID range filter params */; - struct dbg_bus_storm_eid_mask_params mask /* EID mask filter params */; -}; - -/* - * Debug Bus Storm data - */ -struct dbg_bus_storm_data { - u8 fast_enabled; - u8 fast_mode - /* Fast debug Storm mode, valid only if fast_enabled is set */; - u8 slow_enabled; - u8 slow_mode - /* Slow debug Storm mode, valid only if slow_enabled is set */; - u8 hw_id /* HW ID associated with the Storm */; - u8 eid_filter_en /* Indicates if EID filtering is performed (0/1) */; - u8 eid_range_not_mask; - u8 cid_filter_en /* Indicates if CID filtering is performed (0/1) */; - union dbg_bus_storm_eid_params eid_filter_params; - __le16 reserved; - __le32 cid /* CID to filter on. Valid only if cid_filter_en is set. */; -}; - -/* - * Debug Bus data - */ -struct dbg_bus_data { - __le32 app_version /* The tools version number of the application */; - u8 state /* The current debug bus state */; - u8 hw_dwords /* HW dwords per cycle */; - u8 next_hw_id /* Next HW ID to be associated with an input */; - u8 num_enabled_blocks /* Number of blocks enabled for recording */; - u8 num_enabled_storms /* Number of Storms enabled for recording */; - u8 target /* Output target */; - u8 next_trigger_state /* ID of next trigger state to be added */; - u8 next_constraint_id - /* ID of next filter/trigger constraint to be added */; - u8 one_shot_en /* Indicates if one-shot mode is enabled (0/1) */; - u8 grc_input_en /* Indicates if GRC recording is enabled (0/1) */; - u8 timestamp_input_en - /* Indicates if timestamp recording is enabled (0/1) */; - u8 filter_en /* Indicates if the recording filter is enabled (0/1) */; - u8 trigger_en /* Indicates if the recording trigger is enabled (0/1) */ - ; - u8 adding_filter; - u8 filter_pre_trigger; - u8 filter_post_trigger; - u8 unify_inputs; - u8 rcv_from_other_engine; - struct dbg_bus_pci_buf_data pci_buf; - __le16 reserved; - struct dbg_bus_block_data blocks[80] /* Debug Bus data for each block */ - ; - struct dbg_bus_storm_data storms[6] /* Debug Bus data for each block */ - ; -}; - -/* - * Debug bus filter types - */ -enum dbg_bus_filter_types { - DBG_BUS_FILTER_TYPE_OFF /* filter always off */, - DBG_BUS_FILTER_TYPE_PRE /* filter before trigger only */, - DBG_BUS_FILTER_TYPE_POST /* filter after trigger only */, - DBG_BUS_FILTER_TYPE_ON /* filter always on */, - MAX_DBG_BUS_FILTER_TYPES -}; - -/* - * Debug bus frame modes - */ -enum dbg_bus_frame_modes { - DBG_BUS_FRAME_MODE_0HW_4ST = 0 /* 0 HW dwords, 4 Storm dwords */, - DBG_BUS_FRAME_MODE_4HW_0ST = 3 /* 4 HW dwords, 0 Storm dwords */, - DBG_BUS_FRAME_MODE_8HW_0ST = 4 /* 8 HW dwords, 0 Storm dwords */, - MAX_DBG_BUS_FRAME_MODES -}; - -/* - * Debug bus input types - */ -enum dbg_bus_input_types { - DBG_BUS_INPUT_TYPE_STORM, - DBG_BUS_INPUT_TYPE_BLOCK, - MAX_DBG_BUS_INPUT_TYPES -}; - -/* - * Debug bus other engine mode - */ -enum dbg_bus_other_engine_modes { - DBG_BUS_OTHER_ENGINE_MODE_NONE, - DBG_BUS_OTHER_ENGINE_MODE_DOUBLE_BW_TX, - DBG_BUS_OTHER_ENGINE_MODE_DOUBLE_BW_RX, - DBG_BUS_OTHER_ENGINE_MODE_CROSS_ENGINE_TX, - DBG_BUS_OTHER_ENGINE_MODE_CROSS_ENGINE_RX, - MAX_DBG_BUS_OTHER_ENGINE_MODES -}; - -/* - * Debug bus post-trigger recording types - */ -enum dbg_bus_post_trigger_types { - DBG_BUS_POST_TRIGGER_RECORD /* start recording after trigger */, - DBG_BUS_POST_TRIGGER_DROP /* drop data after trigger */, - MAX_DBG_BUS_POST_TRIGGER_TYPES -}; - -/* - * Debug bus pre-trigger recording types - */ -enum dbg_bus_pre_trigger_types { - DBG_BUS_PRE_TRIGGER_START_FROM_ZERO /* start recording from time 0 */, - DBG_BUS_PRE_TRIGGER_NUM_CHUNKS - /* start recording some chunks before trigger */, - DBG_BUS_PRE_TRIGGER_DROP /* drop data before trigger */, - MAX_DBG_BUS_PRE_TRIGGER_TYPES -}; - -/* - * Debug bus SEMI frame modes - */ -enum dbg_bus_semi_frame_modes { - DBG_BUS_SEMI_FRAME_MODE_0SLOW_4FAST = - 0 /* 0 slow dwords, 4 fast dwords */, - DBG_BUS_SEMI_FRAME_MODE_4SLOW_0FAST = - 3 /* 4 slow dwords, 0 fast dwords */, - MAX_DBG_BUS_SEMI_FRAME_MODES -}; - -/* - * Debug bus states - */ -enum dbg_bus_states { - DBG_BUS_STATE_BEFORE_RECORD /* before debug bus the recording starts */ - , - DBG_BUS_STATE_DURING_RECORD /* during debug bus recording */, - DBG_BUS_STATE_AFTER_RECORD /* after debug bus recording */, - MAX_DBG_BUS_STATES -}; - -/* - * Debug Bus Storm modes - */ -enum dbg_bus_storm_modes { - DBG_BUS_STORM_MODE_PRINTF /* store data (fast debug) */, - DBG_BUS_STORM_MODE_PRAM_ADDR /* pram address (fast debug) */, - DBG_BUS_STORM_MODE_DRA_RW /* DRA read/write data (fast debug) */, - DBG_BUS_STORM_MODE_DRA_W /* DRA write data (fast debug) */, - DBG_BUS_STORM_MODE_LD_ST_ADDR /* load/store address (fast debug) */, - DBG_BUS_STORM_MODE_DRA_FSM /* DRA state machines (fast debug) */, - DBG_BUS_STORM_MODE_RH /* recording handlers (fast debug) */, - DBG_BUS_STORM_MODE_FOC /* FOC: FIN + DRA Rd (slow debug) */, - DBG_BUS_STORM_MODE_EXT_STORE /* FOC: External Store (slow) */, - MAX_DBG_BUS_STORM_MODES -}; - -/* - * Debug bus target IDs - */ -enum dbg_bus_targets { - DBG_BUS_TARGET_ID_INT_BUF - /* records debug bus to DBG block internal buffer */, - DBG_BUS_TARGET_ID_NIG /* records debug bus to the NW */, - DBG_BUS_TARGET_ID_PCI /* records debug bus to a PCI buffer */, - MAX_DBG_BUS_TARGETS -}; - -/* - * GRC Dump data - */ -struct dbg_grc_data { - u8 is_updated /* Indicates if the GRC Dump data is updated (0/1) */; - u8 chip_id /* Chip ID */; - u8 chip_mask /* Chip mask */; - u8 reserved; - __le32 max_dump_dwords /* Max GRC Dump size in dwords */; - __le32 param_val[40]; - u8 param_set_by_user[40]; -}; - -/* - * Debug GRC params - */ -enum dbg_grc_params { - DBG_GRC_PARAM_DUMP_TSTORM /* dump Tstorm memories (0/1) */, - DBG_GRC_PARAM_DUMP_MSTORM /* dump Mstorm memories (0/1) */, - DBG_GRC_PARAM_DUMP_USTORM /* dump Ustorm memories (0/1) */, - DBG_GRC_PARAM_DUMP_XSTORM /* dump Xstorm memories (0/1) */, - DBG_GRC_PARAM_DUMP_YSTORM /* dump Ystorm memories (0/1) */, - DBG_GRC_PARAM_DUMP_PSTORM /* dump Pstorm memories (0/1) */, - DBG_GRC_PARAM_DUMP_REGS /* dump non-memory registers (0/1) */, - DBG_GRC_PARAM_DUMP_RAM /* dump Storm internal RAMs (0/1) */, - DBG_GRC_PARAM_DUMP_PBUF /* dump Storm passive buffer (0/1) */, - DBG_GRC_PARAM_DUMP_IOR /* dump Storm IORs (0/1) */, - DBG_GRC_PARAM_DUMP_VFC /* dump VFC memories (0/1) */, - DBG_GRC_PARAM_DUMP_CM_CTX /* dump CM contexts (0/1) */, - DBG_GRC_PARAM_DUMP_PXP /* dump PXP memories (0/1) */, - DBG_GRC_PARAM_DUMP_RSS /* dump RSS memories (0/1) */, - DBG_GRC_PARAM_DUMP_CAU /* dump CAU memories (0/1) */, - DBG_GRC_PARAM_DUMP_QM /* dump QM memories (0/1) */, - DBG_GRC_PARAM_DUMP_MCP /* dump MCP memories (0/1) */, - DBG_GRC_PARAM_RESERVED /* reserved */, - DBG_GRC_PARAM_DUMP_CFC /* dump CFC memories (0/1) */, - DBG_GRC_PARAM_DUMP_IGU /* dump IGU memories (0/1) */, - DBG_GRC_PARAM_DUMP_BRB /* dump BRB memories (0/1) */, - DBG_GRC_PARAM_DUMP_BTB /* dump BTB memories (0/1) */, - DBG_GRC_PARAM_DUMP_BMB /* dump BMB memories (0/1) */, - DBG_GRC_PARAM_DUMP_NIG /* dump NIG memories (0/1) */, - DBG_GRC_PARAM_DUMP_MULD /* dump MULD memories (0/1) */, - DBG_GRC_PARAM_DUMP_PRS /* dump PRS memories (0/1) */, - DBG_GRC_PARAM_DUMP_DMAE /* dump PRS memories (0/1) */, - DBG_GRC_PARAM_DUMP_TM /* dump TM (timers) memories (0/1) */, - DBG_GRC_PARAM_DUMP_SDM /* dump SDM memories (0/1) */, - DBG_GRC_PARAM_DUMP_STATIC /* dump static debug data (0/1) */, - DBG_GRC_PARAM_UNSTALL /* un-stall Storms after dump (0/1) */, - DBG_GRC_PARAM_NUM_LCIDS /* number of LCIDs (0..320) */, - DBG_GRC_PARAM_NUM_LTIDS /* number of LTIDs (0..320) */, - DBG_GRC_PARAM_EXCLUDE_ALL - /* preset: exclude all memories from dump (1 only) */, - DBG_GRC_PARAM_CRASH - /* preset: include memories for crash dump (1 only) */, - DBG_GRC_PARAM_PARITY_SAFE - /* perform dump only if MFW is responding (0/1) */, - DBG_GRC_PARAM_DUMP_CM /* dump CM memories (0/1) */, - MAX_DBG_GRC_PARAMS -}; - -/* - * Debug reset registers - */ -enum dbg_reset_regs { - DBG_RESET_REG_MISCS_PL_UA, - DBG_RESET_REG_MISCS_PL_HV, - DBG_RESET_REG_MISC_PL_UA, - DBG_RESET_REG_MISC_PL_HV, - DBG_RESET_REG_MISC_PL_PDA_VMAIN_1, - DBG_RESET_REG_MISC_PL_PDA_VMAIN_2, - DBG_RESET_REG_MISC_PL_PDA_VAUX, - MAX_DBG_RESET_REGS -}; - -/* - * @DPDK Debug status codes - */ -enum dbg_status { - DBG_STATUS_OK, - DBG_STATUS_APP_VERSION_NOT_SET, - DBG_STATUS_UNSUPPORTED_APP_VERSION, - DBG_STATUS_DBG_BLOCK_NOT_RESET, - DBG_STATUS_INVALID_ARGS, - DBG_STATUS_OUTPUT_ALREADY_SET, - DBG_STATUS_INVALID_PCI_BUF_SIZE, - DBG_STATUS_PCI_BUF_ALLOC_FAILED, - DBG_STATUS_PCI_BUF_NOT_ALLOCATED, - DBG_STATUS_TOO_MANY_INPUTS, - DBG_STATUS_INPUT_OVERLAP, - DBG_STATUS_HW_ONLY_RECORDING, - DBG_STATUS_STORM_ALREADY_ENABLED, - DBG_STATUS_STORM_NOT_ENABLED, - DBG_STATUS_BLOCK_ALREADY_ENABLED, - DBG_STATUS_BLOCK_NOT_ENABLED, - DBG_STATUS_NO_INPUT_ENABLED, - DBG_STATUS_NO_FILTER_TRIGGER_64B, - DBG_STATUS_FILTER_ALREADY_ENABLED, - DBG_STATUS_TRIGGER_ALREADY_ENABLED, - DBG_STATUS_TRIGGER_NOT_ENABLED, - DBG_STATUS_CANT_ADD_CONSTRAINT, - DBG_STATUS_TOO_MANY_TRIGGER_STATES, - DBG_STATUS_TOO_MANY_CONSTRAINTS, - DBG_STATUS_RECORDING_NOT_STARTED, - DBG_STATUS_NO_DATA_TRIGGERED, - DBG_STATUS_NO_DATA_RECORDED, - DBG_STATUS_DUMP_BUF_TOO_SMALL, - DBG_STATUS_DUMP_NOT_CHUNK_ALIGNED, - DBG_STATUS_UNKNOWN_CHIP, - DBG_STATUS_VIRT_MEM_ALLOC_FAILED, - DBG_STATUS_BLOCK_IN_RESET, - DBG_STATUS_INVALID_TRACE_SIGNATURE, - DBG_STATUS_INVALID_NVRAM_BUNDLE, - DBG_STATUS_NVRAM_GET_IMAGE_FAILED, - DBG_STATUS_NON_ALIGNED_NVRAM_IMAGE, - DBG_STATUS_NVRAM_READ_FAILED, - DBG_STATUS_IDLE_CHK_PARSE_FAILED, - DBG_STATUS_MCP_TRACE_BAD_DATA, - DBG_STATUS_MCP_TRACE_NO_META, - DBG_STATUS_MCP_COULD_NOT_HALT, - DBG_STATUS_MCP_COULD_NOT_RESUME, - DBG_STATUS_DMAE_FAILED, - DBG_STATUS_SEMI_FIFO_NOT_EMPTY, - DBG_STATUS_IGU_FIFO_BAD_DATA, - DBG_STATUS_MCP_COULD_NOT_MASK_PRTY, - DBG_STATUS_FW_ASSERTS_PARSE_FAILED, - DBG_STATUS_REG_FIFO_BAD_DATA, - DBG_STATUS_PROTECTION_OVERRIDE_BAD_DATA, - MAX_DBG_STATUS -}; - -/* - * Debug Storms IDs - */ -enum dbg_storms { - DBG_TSTORM_ID, - DBG_MSTORM_ID, - DBG_USTORM_ID, - DBG_XSTORM_ID, - DBG_YSTORM_ID, - DBG_PSTORM_ID, - MAX_DBG_STORMS -}; - -/* - * Idle Check data - */ -struct idle_chk_data { - __le32 buf_size /* Idle check buffer size in dwords */; - u8 buf_size_set - /* Indicates if the idle check buffer size was set (0/1) */; - u8 reserved1; - __le16 reserved2; -}; - -/* - * Idle Check data - */ -struct mcp_trace_data { - __le32 buf_size /* MCP Trace buffer size in dwords */; - u8 buf_size_set - /* Indicates if the MCP Trace buffer size was set (0/1) */; - u8 reserved1; - __le16 reserved2; -}; - -/* - * Debug Tools data (per HW function) - */ -struct dbg_tools_data { - struct dbg_grc_data grc /* GRC Dump data */; - struct dbg_bus_data bus /* Debug Bus data */; - struct idle_chk_data idle_chk /* Idle Check data */; - struct mcp_trace_data mcp_trace /* MCP Trace data */; - u8 block_in_reset[80] /* Indicates if a block is in reset state (0/1) */ - ; - u8 chip_id /* Chip ID (from enum chip_ids) */; - u8 chip_mask - /* Chip mask = bit index chip_id is set, the rest are cleared */; - u8 initialized /* Indicates if the data was initialized */; - u8 reset_state_updated - /* Indicates if blocks reset state is updated (0/1) */; -}; - -/* - * BRB RAM init requirements - */ -struct init_brb_ram_req { - __le32 guranteed_per_tc /* guaranteed size per TC, in bytes */; - __le32 headroom_per_tc /* headroom size per TC, in bytes */; - __le32 min_pkt_size /* min packet size, in bytes */; - __le32 max_ports_per_engine /* min packet size, in bytes */; - u8 num_active_tcs[MAX_NUM_PORTS] /* number of active TCs per port */; -}; - -/* - * ETS per-TC init requirements - */ -struct init_ets_tc_req { - u8 use_sp; - u8 use_wfq; - __le16 weight /* An arbitration weight. Valid only if use_wfq is set. */ - ; -}; - -/* - * ETS init requirements - */ -struct init_ets_req { - __le32 mtu /* Max packet size (in bytes) */; - struct init_ets_tc_req tc_req[NUM_OF_TCS] - /* ETS initialization requirements per TC. */; -}; - -/* - * NIG LB RL init requirements - */ -struct init_nig_lb_rl_req { - __le16 lb_mac_rate; - __le16 lb_rate; - __le32 mtu /* Max packet size (in bytes) */; - __le16 tc_rate[NUM_OF_PHYS_TCS]; -}; - -/* - * NIG TC mapping for each priority - */ -struct init_nig_pri_tc_map_entry { - u8 tc_id /* the mapped TC ID */; - u8 valid /* indicates if the mapping entry is valid */; -}; - -/* - * NIG priority to TC map init requirements - */ -struct init_nig_pri_tc_map_req { - struct init_nig_pri_tc_map_entry pri[NUM_OF_VLAN_PRIORITIES]; -}; - -/* - * QM per-port init parameters - */ -struct init_qm_port_params { - u8 active /* Indicates if this port is active */; - u8 num_active_phys_tcs /* number of physical TCs used by this port */; - __le16 num_pbf_cmd_lines - /* number of PBF command lines that can be used by this port */; - __le16 num_btb_blocks - /* number of BTB blocks that can be used by this port */; - __le16 reserved; -}; - -/* - * QM per-PQ init parameters - */ -struct init_qm_pq_params { - u8 vport_id /* VPORT ID */; - u8 tc_id /* TC ID */; - u8 wrr_group /* WRR group */; - u8 reserved; -}; - -/* - * QM per-vport init parameters - */ -struct init_qm_vport_params { - __le32 vport_rl; - __le16 vport_wfq; - __le16 first_tx_pq_id[NUM_OF_TCS] - /* the first Tx PQ ID associated with this VPORT for each TC. */; -}; - -#endif /* __ECORE_HSI_TOOLS__ */ diff --git a/dpdk/drivers/net/qede/base/ecore_hw.c b/dpdk/drivers/net/qede/base/ecore_hw.c index 5403b94b..84f273b0 100644 --- a/dpdk/drivers/net/qede/base/ecore_hw.c +++ b/dpdk/drivers/net/qede/base/ecore_hw.c @@ -23,27 +23,28 @@ #define ECORE_BAR_ACQUIRE_TIMEOUT 1000 /* Invalid values */ -#define ECORE_BAR_INVALID_OFFSET -1 +#define ECORE_BAR_INVALID_OFFSET (OSAL_CPU_TO_LE32(-1)) struct ecore_ptt { osal_list_entry_t list_entry; unsigned int idx; struct pxp_ptt_entry pxp; + u8 hwfn_id; }; struct ecore_ptt_pool { osal_list_t free_list; - osal_spinlock_t lock; + osal_spinlock_t lock; /* ptt synchronized access */ struct ecore_ptt ptts[PXP_EXTERNAL_BAR_PF_WINDOW_NUM]; }; enum _ecore_status_t ecore_ptt_pool_alloc(struct ecore_hwfn *p_hwfn) { - struct ecore_ptt_pool *p_pool; + struct ecore_ptt_pool *p_pool = OSAL_ALLOC(p_hwfn->p_dev, + GFP_KERNEL, + sizeof(*p_pool)); int i; - p_pool = OSAL_ALLOC(p_hwfn->p_dev, GFP_KERNEL, - sizeof(struct ecore_ptt_pool)); if (!p_pool) return ECORE_NOMEM; @@ -52,6 +53,7 @@ enum _ecore_status_t ecore_ptt_pool_alloc(struct ecore_hwfn *p_hwfn) p_pool->ptts[i].idx = i; p_pool->ptts[i].pxp.offset = ECORE_BAR_INVALID_OFFSET; p_pool->ptts[i].pxp.pretend.control = 0; + p_pool->ptts[i].hwfn_id = p_hwfn->my_id; /* There are special PTT entries that are taken only by design. * The rest are added ot the list for general usage. @@ -62,7 +64,9 @@ enum _ecore_status_t ecore_ptt_pool_alloc(struct ecore_hwfn *p_hwfn) } p_hwfn->p_ptt_pool = p_pool; +#ifdef CONFIG_ECORE_LOCK_ALLOC OSAL_SPIN_LOCK_ALLOC(p_hwfn, &p_pool->lock); +#endif OSAL_SPIN_LOCK_INIT(&p_pool->lock); return ECORE_SUCCESS; @@ -81,10 +85,11 @@ void ecore_ptt_invalidate(struct ecore_hwfn *p_hwfn) void ecore_ptt_pool_free(struct ecore_hwfn *p_hwfn) { +#ifdef CONFIG_ECORE_LOCK_ALLOC if (p_hwfn->p_ptt_pool) OSAL_SPIN_LOCK_DEALLOC(&p_hwfn->p_ptt_pool->lock); +#endif OSAL_FREE(p_hwfn->p_dev, p_hwfn->p_ptt_pool); - p_hwfn->p_ptt_pool = OSAL_NULL; } struct ecore_ptt *ecore_ptt_acquire(struct ecore_hwfn *p_hwfn) @@ -95,42 +100,46 @@ struct ecore_ptt *ecore_ptt_acquire(struct ecore_hwfn *p_hwfn) /* Take the free PTT from the list */ for (i = 0; i < ECORE_BAR_ACQUIRE_TIMEOUT; i++) { OSAL_SPIN_LOCK(&p_hwfn->p_ptt_pool->lock); - if (!OSAL_LIST_IS_EMPTY(&p_hwfn->p_ptt_pool->free_list)) - break; + if (!OSAL_LIST_IS_EMPTY(&p_hwfn->p_ptt_pool->free_list)) { + p_ptt = OSAL_LIST_FIRST_ENTRY( + &p_hwfn->p_ptt_pool->free_list, + struct ecore_ptt, list_entry); + OSAL_LIST_REMOVE_ENTRY(&p_ptt->list_entry, + &p_hwfn->p_ptt_pool->free_list); + + OSAL_SPIN_UNLOCK(&p_hwfn->p_ptt_pool->lock); + + DP_VERBOSE(p_hwfn, ECORE_MSG_HW, + "allocated ptt %d\n", p_ptt->idx); + + return p_ptt; + } + OSAL_SPIN_UNLOCK(&p_hwfn->p_ptt_pool->lock); OSAL_MSLEEP(1); } - /* We should not time-out, but it can happen... --> Lock isn't held */ - if (i == ECORE_BAR_ACQUIRE_TIMEOUT) { - DP_NOTICE(p_hwfn, true, "Failed to allocate PTT\n"); - return OSAL_NULL; - } - - p_ptt = OSAL_LIST_FIRST_ENTRY(&p_hwfn->p_ptt_pool->free_list, - struct ecore_ptt, list_entry); - OSAL_LIST_REMOVE_ENTRY(&p_ptt->list_entry, - &p_hwfn->p_ptt_pool->free_list); - OSAL_SPIN_UNLOCK(&p_hwfn->p_ptt_pool->lock); - - DP_VERBOSE(p_hwfn, ECORE_MSG_HW, "allocated ptt %d\n", p_ptt->idx); - - return p_ptt; + DP_NOTICE(p_hwfn, true, + "PTT acquire timeout - failed to allocate PTT\n"); + return OSAL_NULL; } void ecore_ptt_release(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt) { /* This PTT should not be set to pretend if it is being released */ + /* TODO - add some pretend sanity checks, to make sure pretend + * isn't set on this ptt + */ OSAL_SPIN_LOCK(&p_hwfn->p_ptt_pool->lock); OSAL_LIST_PUSH_HEAD(&p_ptt->list_entry, &p_hwfn->p_ptt_pool->free_list); OSAL_SPIN_UNLOCK(&p_hwfn->p_ptt_pool->lock); } -u32 ecore_ptt_get_hw_addr(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt) +static u32 ecore_ptt_get_hw_addr(struct ecore_ptt *p_ptt) { /* The HW is using DWORDS and we need to translate it to Bytes */ - return p_ptt->pxp.offset << 2; + return OSAL_LE32_TO_CPU(p_ptt->pxp.offset) << 2; } static u32 ecore_ptt_config_addr(struct ecore_ptt *p_ptt) @@ -150,7 +159,7 @@ void ecore_ptt_set_win(struct ecore_hwfn *p_hwfn, { u32 prev_hw_addr; - prev_hw_addr = ecore_ptt_get_hw_addr(p_hwfn, p_ptt); + prev_hw_addr = ecore_ptt_get_hw_addr(p_ptt); if (new_hw_addr == prev_hw_addr) return; @@ -161,21 +170,27 @@ void ecore_ptt_set_win(struct ecore_hwfn *p_hwfn, p_ptt->idx, new_hw_addr); /* The HW is using DWORDS and the address is in Bytes */ - p_ptt->pxp.offset = new_hw_addr >> 2; + p_ptt->pxp.offset = OSAL_CPU_TO_LE32(new_hw_addr >> 2); REG_WR(p_hwfn, ecore_ptt_config_addr(p_ptt) + - OFFSETOF(struct pxp_ptt_entry, offset), p_ptt->pxp.offset); + OFFSETOF(struct pxp_ptt_entry, offset), + OSAL_LE32_TO_CPU(p_ptt->pxp.offset)); } static u32 ecore_set_ptt(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, u32 hw_addr) { - u32 win_hw_addr = ecore_ptt_get_hw_addr(p_hwfn, p_ptt); + u32 win_hw_addr = ecore_ptt_get_hw_addr(p_ptt); u32 offset; offset = hw_addr - win_hw_addr; + if (p_ptt->hwfn_id != p_hwfn->my_id) + DP_NOTICE(p_hwfn, true, + "ptt[%d] of hwfn[%02x] is used by hwfn[%02x]!\n", + p_ptt->idx, p_ptt->hwfn_id, p_hwfn->my_id); + /* Verify the address is within the window */ if (hw_addr < win_hw_addr || offset >= PXP_EXTERNAL_BAR_PF_WINDOW_SINGLE_SIZE) { @@ -198,11 +213,37 @@ struct ecore_ptt *ecore_get_reserved_ptt(struct ecore_hwfn *p_hwfn, return &p_hwfn->p_ptt_pool->ptts[ptt_idx]; } +static bool ecore_is_reg_fifo_empty(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt) +{ + bool is_empty = true; + u32 bar_addr; + + if (!p_hwfn->p_dev->chk_reg_fifo) + goto out; + + /* ecore_rd() cannot be used here since it calls this function */ + bar_addr = ecore_set_ptt(p_hwfn, p_ptt, GRC_REG_TRACE_FIFO_VALID_DATA); + is_empty = REG_RD(p_hwfn, bar_addr) == 0; + +#ifndef ASIC_ONLY + if (CHIP_REV_IS_SLOW(p_hwfn->p_dev)) + OSAL_UDELAY(100); +#endif + +out: + return is_empty; +} + void ecore_wr(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, u32 hw_addr, u32 val) { - u32 bar_addr = ecore_set_ptt(p_hwfn, p_ptt, hw_addr); + bool prev_fifo_err; + u32 bar_addr; + prev_fifo_err = !ecore_is_reg_fifo_empty(p_hwfn, p_ptt); + + bar_addr = ecore_set_ptt(p_hwfn, p_ptt, hw_addr); REG_WR(p_hwfn, bar_addr, val); DP_VERBOSE(p_hwfn, ECORE_MSG_HW, "bar_addr 0x%x, hw_addr 0x%x, val 0x%x\n", @@ -212,12 +253,21 @@ void ecore_wr(struct ecore_hwfn *p_hwfn, if (CHIP_REV_IS_SLOW(p_hwfn->p_dev)) OSAL_UDELAY(100); #endif + + OSAL_WARN(!prev_fifo_err && !ecore_is_reg_fifo_empty(p_hwfn, p_ptt), + "reg_fifo err was caused by a call to ecore_wr(0x%x, 0x%x)\n", + hw_addr, val); } u32 ecore_rd(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, u32 hw_addr) { - u32 bar_addr = ecore_set_ptt(p_hwfn, p_ptt, hw_addr); - u32 val = REG_RD(p_hwfn, bar_addr); + bool prev_fifo_err; + u32 bar_addr, val; + + prev_fifo_err = !ecore_is_reg_fifo_empty(p_hwfn, p_ptt); + + bar_addr = ecore_set_ptt(p_hwfn, p_ptt, hw_addr); + val = REG_RD(p_hwfn, bar_addr); DP_VERBOSE(p_hwfn, ECORE_MSG_HW, "bar_addr 0x%x, hw_addr 0x%x, val 0x%x\n", @@ -228,6 +278,10 @@ u32 ecore_rd(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, u32 hw_addr) OSAL_UDELAY(100); #endif + OSAL_WARN(!prev_fifo_err && !ecore_is_reg_fifo_empty(p_hwfn, p_ptt), + "reg_fifo error was caused by a call to ecore_rd(0x%x)\n", + hw_addr); + return val; } @@ -292,60 +346,59 @@ void ecore_memcpy_to(struct ecore_hwfn *p_hwfn, void ecore_fid_pretend(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, u16 fid) { - void *p_pretend; u16 control = 0; SET_FIELD(control, PXP_PRETEND_CMD_IS_CONCRETE, 1); SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_FUNCTION, 1); - /* Every pretend undos prev pretends, including previous port pretend */ +/* Every pretend undos prev pretends, including previous port pretend */ + SET_FIELD(control, PXP_PRETEND_CMD_PORT, 0); SET_FIELD(control, PXP_PRETEND_CMD_USE_PORT, 0); SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_PORT, 1); - p_ptt->pxp.pretend.control = OSAL_CPU_TO_LE16(control); if (!GET_FIELD(fid, PXP_CONCRETE_FID_VFVALID)) fid = GET_FIELD(fid, PXP_CONCRETE_FID_PFID); + p_ptt->pxp.pretend.control = OSAL_CPU_TO_LE16(control); p_ptt->pxp.pretend.fid.concrete_fid.fid = OSAL_CPU_TO_LE16(fid); - p_pretend = &p_ptt->pxp.pretend; REG_WR(p_hwfn, ecore_ptt_config_addr(p_ptt) + - OFFSETOF(struct pxp_ptt_entry, pretend), *(u32 *)p_pretend); + OFFSETOF(struct pxp_ptt_entry, pretend), + *(u32 *)&p_ptt->pxp.pretend); } void ecore_port_pretend(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, u8 port_id) { - void *p_pretend; u16 control = 0; SET_FIELD(control, PXP_PRETEND_CMD_PORT, port_id); SET_FIELD(control, PXP_PRETEND_CMD_USE_PORT, 1); SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_PORT, 1); - p_ptt->pxp.pretend.control = control; + p_ptt->pxp.pretend.control = OSAL_CPU_TO_LE16(control); - p_pretend = &p_ptt->pxp.pretend; REG_WR(p_hwfn, ecore_ptt_config_addr(p_ptt) + - OFFSETOF(struct pxp_ptt_entry, pretend), *(u32 *)p_pretend); + OFFSETOF(struct pxp_ptt_entry, pretend), + *(u32 *)&p_ptt->pxp.pretend); } void ecore_port_unpretend(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt) { - void *p_pretend; u16 control = 0; SET_FIELD(control, PXP_PRETEND_CMD_PORT, 0); SET_FIELD(control, PXP_PRETEND_CMD_USE_PORT, 0); SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_PORT, 1); - p_ptt->pxp.pretend.control = control; - p_pretend = &p_ptt->pxp.pretend; + p_ptt->pxp.pretend.control = OSAL_CPU_TO_LE16(control); + REG_WR(p_hwfn, ecore_ptt_config_addr(p_ptt) + - OFFSETOF(struct pxp_ptt_entry, pretend), *(u32 *)p_pretend); + OFFSETOF(struct pxp_ptt_entry, pretend), + *(u32 *)&p_ptt->pxp.pretend); } u32 ecore_vfid_to_concrete(struct ecore_hwfn *p_hwfn, u8 vfid) @@ -442,15 +495,16 @@ static u32 ecore_dmae_idx_to_go_cmd(u8 idx) { OSAL_BUILD_BUG_ON((DMAE_REG_GO_C31 - DMAE_REG_GO_C0) != 31 * 4); - return DMAE_REG_GO_C0 + idx * 4; + /* All the DMAE 'go' registers form an array in internal memory */ + return DMAE_REG_GO_C0 + (idx << 2); } -static enum _ecore_status_t -ecore_dmae_post_command(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt) +static enum _ecore_status_t ecore_dmae_post_command(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt) { struct dmae_cmd *p_command = p_hwfn->dmae_info.p_dmae_cmd; - enum _ecore_status_t ecore_status = ECORE_SUCCESS; u8 idx_cmd = p_hwfn->dmae_info.channel, i; + enum _ecore_status_t ecore_status = ECORE_SUCCESS; /* verify address is not OSAL_NULL */ if ((((!p_command->dst_addr_lo) && (!p_command->dst_addr_hi)) || @@ -459,13 +513,14 @@ ecore_dmae_post_command(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt) "source or destination address 0 idx_cmd=%d\n" "opcode = [0x%08x,0x%04x] len=0x%x" " src=0x%x:%x dst=0x%x:%x\n", - idx_cmd, (u32)p_command->opcode, - (u16)p_command->opcode_b, - (int)p_command->length, - (int)p_command->src_addr_hi, - (int)p_command->src_addr_lo, - (int)p_command->dst_addr_hi, - (int)p_command->dst_addr_lo); + idx_cmd, + OSAL_LE32_TO_CPU(p_command->opcode), + OSAL_LE16_TO_CPU(p_command->opcode_b), + OSAL_LE16_TO_CPU(p_command->length_dw), + OSAL_LE32_TO_CPU(p_command->src_addr_hi), + OSAL_LE32_TO_CPU(p_command->src_addr_lo), + OSAL_LE32_TO_CPU(p_command->dst_addr_hi), + OSAL_LE32_TO_CPU(p_command->dst_addr_lo)); return ECORE_INVAL; } @@ -473,12 +528,14 @@ ecore_dmae_post_command(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt) DP_VERBOSE(p_hwfn, ECORE_MSG_HW, "Posting DMAE command [idx %d]: opcode = [0x%08x,0x%04x]" "len=0x%x src=0x%x:%x dst=0x%x:%x\n", - idx_cmd, (u32)p_command->opcode, - (u16)p_command->opcode_b, - (int)p_command->length, - (int)p_command->src_addr_hi, - (int)p_command->src_addr_lo, - (int)p_command->dst_addr_hi, (int)p_command->dst_addr_lo); + idx_cmd, + OSAL_LE32_TO_CPU(p_command->opcode), + OSAL_LE16_TO_CPU(p_command->opcode_b), + OSAL_LE16_TO_CPU(p_command->length_dw), + OSAL_LE32_TO_CPU(p_command->src_addr_hi), + OSAL_LE32_TO_CPU(p_command->src_addr_lo), + OSAL_LE32_TO_CPU(p_command->dst_addr_hi), + OSAL_LE32_TO_CPU(p_command->dst_addr_lo)); /* Copy the command to DMAE - need to do it before every call * for source/dest address no reset. @@ -514,8 +571,7 @@ enum _ecore_status_t ecore_dmae_info_alloc(struct ecore_hwfn *p_hwfn) if (*p_comp == OSAL_NULL) { DP_NOTICE(p_hwfn, true, "Failed to allocate `p_completion_word'\n"); - ecore_dmae_info_free(p_hwfn); - return ECORE_NOMEM; + goto err; } p_addr = &p_hwfn->dmae_info.dmae_cmd_phys_addr; @@ -524,8 +580,7 @@ enum _ecore_status_t ecore_dmae_info_alloc(struct ecore_hwfn *p_hwfn) if (*p_cmd == OSAL_NULL) { DP_NOTICE(p_hwfn, true, "Failed to allocate `struct dmae_cmd'\n"); - ecore_dmae_info_free(p_hwfn); - return ECORE_NOMEM; + goto err; } p_addr = &p_hwfn->dmae_info.intermediate_buffer_phys_addr; @@ -534,14 +589,15 @@ enum _ecore_status_t ecore_dmae_info_alloc(struct ecore_hwfn *p_hwfn) if (*p_buff == OSAL_NULL) { DP_NOTICE(p_hwfn, true, "Failed to allocate `intermediate_buffer'\n"); - ecore_dmae_info_free(p_hwfn); - return ECORE_NOMEM; + goto err; } - /* DMAE_E4_TODO : Need to change this to reflect proper channel */ p_hwfn->dmae_info.channel = p_hwfn->rel_pf_id; return ECORE_SUCCESS; +err: + ecore_dmae_info_free(p_hwfn); + return ECORE_NOMEM; } void ecore_dmae_info_free(struct ecore_hwfn *p_hwfn) @@ -580,8 +636,8 @@ void ecore_dmae_info_free(struct ecore_hwfn *p_hwfn) static enum _ecore_status_t ecore_dmae_operation_wait(struct ecore_hwfn *p_hwfn) { - enum _ecore_status_t ecore_status = ECORE_SUCCESS; u32 wait_cnt_limit = 10000, wait_cnt = 0; + enum _ecore_status_t ecore_status = ECORE_SUCCESS; #ifndef ASIC_ONLY u32 factor = (CHIP_REV_IS_EMUL(p_hwfn->p_dev) ? @@ -598,9 +654,6 @@ static enum _ecore_status_t ecore_dmae_operation_wait(struct ecore_hwfn *p_hwfn) */ OSAL_BARRIER(p_hwfn->p_dev); while (*p_hwfn->dmae_info.p_completion_word != DMAE_COMPLETION_VAL) { - /* DMAE_E4_TODO : using OSAL_MSLEEP instead of mm_wait since mm - * functions are getting depriciated. Need to review for future. - */ OSAL_UDELAY(DMAE_MIN_WAIT_TIME); if (++wait_cnt > wait_cnt_limit) { DP_NOTICE(p_hwfn->p_dev, ECORE_MSG_HW, @@ -629,7 +682,7 @@ ecore_dmae_execute_sub_operation(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, u64 src_addr, u64 dst_addr, - u8 src_type, u8 dst_type, u32 length) + u8 src_type, u8 dst_type, u32 length_dw) { dma_addr_t phys = p_hwfn->dmae_info.intermediate_buffer_phys_addr; struct dmae_cmd *cmd = p_hwfn->dmae_info.p_dmae_cmd; @@ -638,16 +691,16 @@ ecore_dmae_execute_sub_operation(struct ecore_hwfn *p_hwfn, switch (src_type) { case ECORE_DMAE_ADDRESS_GRC: case ECORE_DMAE_ADDRESS_HOST_PHYS: - cmd->src_addr_hi = DMA_HI(src_addr); - cmd->src_addr_lo = DMA_LO(src_addr); + cmd->src_addr_hi = OSAL_CPU_TO_LE32(DMA_HI(src_addr)); + cmd->src_addr_lo = OSAL_CPU_TO_LE32(DMA_LO(src_addr)); break; /* for virt source addresses we use the intermediate buffer. */ case ECORE_DMAE_ADDRESS_HOST_VIRT: - cmd->src_addr_hi = DMA_HI(phys); - cmd->src_addr_lo = DMA_LO(phys); + cmd->src_addr_hi = OSAL_CPU_TO_LE32(DMA_HI(phys)); + cmd->src_addr_lo = OSAL_CPU_TO_LE32(DMA_LO(phys)); OSAL_MEMCPY(&p_hwfn->dmae_info.p_intermediate_buffer[0], (void *)(osal_uintptr_t)src_addr, - length * sizeof(u32)); + length_dw * sizeof(u32)); break; default: return ECORE_INVAL; @@ -656,26 +709,26 @@ ecore_dmae_execute_sub_operation(struct ecore_hwfn *p_hwfn, switch (dst_type) { case ECORE_DMAE_ADDRESS_GRC: case ECORE_DMAE_ADDRESS_HOST_PHYS: - cmd->dst_addr_hi = DMA_HI(dst_addr); - cmd->dst_addr_lo = DMA_LO(dst_addr); + cmd->dst_addr_hi = OSAL_CPU_TO_LE32(DMA_HI(dst_addr)); + cmd->dst_addr_lo = OSAL_CPU_TO_LE32(DMA_LO(dst_addr)); break; /* for virt destination address we use the intermediate buff. */ case ECORE_DMAE_ADDRESS_HOST_VIRT: - cmd->dst_addr_hi = DMA_HI(phys); - cmd->dst_addr_lo = DMA_LO(phys); + cmd->dst_addr_hi = OSAL_CPU_TO_LE32(DMA_HI(phys)); + cmd->dst_addr_lo = OSAL_CPU_TO_LE32(DMA_LO(phys)); break; default: return ECORE_INVAL; } - cmd->length = (u16)length; + cmd->length_dw = OSAL_CPU_TO_LE16((u16)length_dw); if (src_type == ECORE_DMAE_ADDRESS_HOST_VIRT || src_type == ECORE_DMAE_ADDRESS_HOST_PHYS) OSAL_DMA_SYNC(p_hwfn->p_dev, (void *)HILO_U64(cmd->src_addr_hi, cmd->src_addr_lo), - length * sizeof(u32), false); + length_dw * sizeof(u32), false); ecore_dmae_post_command(p_hwfn, p_ptt); @@ -687,21 +740,21 @@ ecore_dmae_execute_sub_operation(struct ecore_hwfn *p_hwfn, OSAL_DMA_SYNC(p_hwfn->p_dev, (void *)HILO_U64(cmd->src_addr_hi, cmd->src_addr_lo), - length * sizeof(u32), true); + length_dw * sizeof(u32), true); if (ecore_status != ECORE_SUCCESS) { DP_NOTICE(p_hwfn, ECORE_MSG_HW, - "ecore_dmae_host2grc: Wait Failed. source_addr" - " 0x%lx, grc_addr 0x%lx, size_in_dwords 0x%x\n", + "Wait Failed. source_addr 0x%lx, grc_addr 0x%lx, size_in_dwords 0x%x, intermediate buffer 0x%lx.\n", (unsigned long)src_addr, (unsigned long)dst_addr, - length); + length_dw, + (unsigned long)p_hwfn->dmae_info.intermediate_buffer_phys_addr); return ecore_status; } if (dst_type == ECORE_DMAE_ADDRESS_HOST_VIRT) OSAL_MEMCPY((void *)(osal_uintptr_t)(dst_addr), &p_hwfn->dmae_info.p_intermediate_buffer[0], - length * sizeof(u32)); + length_dw * sizeof(u32)); return ECORE_SUCCESS; } @@ -719,18 +772,39 @@ ecore_dmae_execute_command(struct ecore_hwfn *p_hwfn, dma_addr_t phys = p_hwfn->dmae_info.completion_word_phys_addr; u16 length_cur = 0, i = 0, cnt_split = 0, length_mod = 0; struct dmae_cmd *cmd = p_hwfn->dmae_info.p_dmae_cmd; - enum _ecore_status_t ecore_status = ECORE_SUCCESS; u64 src_addr_split = 0, dst_addr_split = 0; u16 length_limit = DMAE_MAX_RW_SIZE; + enum _ecore_status_t ecore_status = ECORE_SUCCESS; u32 offset = 0; + if (p_hwfn->p_dev->recov_in_prog) { + DP_VERBOSE(p_hwfn, ECORE_MSG_HW, + "Recovery is in progress. Avoid DMAE transaction [{src: addr 0x%lx, type %d}, {dst: addr 0x%lx, type %d}, size %d].\n", + (unsigned long)src_addr, src_type, + (unsigned long)dst_addr, dst_type, + size_in_dwords); + /* Return success to let the flow to be completed successfully + * w/o any error handling. + */ + return ECORE_SUCCESS; + } + + if (!cmd) { + DP_NOTICE(p_hwfn, true, + "ecore_dmae_execute_sub_operation failed. Invalid state. source_addr 0x%lx, destination addr 0x%lx, size_in_dwords 0x%x\n", + (unsigned long)src_addr, + (unsigned long)dst_addr, + length_cur); + return ECORE_INVAL; + } + ecore_dmae_opcode(p_hwfn, (src_type == ECORE_DMAE_ADDRESS_GRC), (dst_type == ECORE_DMAE_ADDRESS_GRC), p_params); - cmd->comp_addr_lo = DMA_LO(phys); - cmd->comp_addr_hi = DMA_HI(phys); - cmd->comp_val = DMAE_COMPLETION_VAL; + cmd->comp_addr_lo = OSAL_CPU_TO_LE32(DMA_LO(phys)); + cmd->comp_addr_hi = OSAL_CPU_TO_LE32(DMA_HI(phys)); + cmd->comp_val = OSAL_CPU_TO_LE32(DMAE_COMPLETION_VAL); /* Check if the grc_addr is valid like < MAX_GRC_OFFSET */ cnt_split = size_in_dwords / length_limit; @@ -856,44 +930,6 @@ ecore_dmae_host2host(struct ecore_hwfn *p_hwfn, return rc; } -u16 ecore_get_qm_pq(struct ecore_hwfn *p_hwfn, - enum protocol_type proto, - union ecore_qm_pq_params *p_params) -{ - u16 pq_id = 0; - - if ((proto == PROTOCOLID_CORE || - proto == PROTOCOLID_ETH) && !p_params) { - DP_NOTICE(p_hwfn, true, - "Protocol %d received NULL PQ params\n", proto); - return 0; - } - - switch (proto) { - case PROTOCOLID_CORE: - if (p_params->core.tc == LB_TC) - pq_id = p_hwfn->qm_info.pure_lb_pq; - else if (p_params->core.tc == OOO_LB_TC) - pq_id = p_hwfn->qm_info.ooo_pq; - else - pq_id = p_hwfn->qm_info.offload_pq; - break; - case PROTOCOLID_ETH: - pq_id = p_params->eth.tc; - /* TODO - multi-CoS for VFs? */ - if (p_params->eth.is_vf) - pq_id += p_hwfn->qm_info.vf_queues_offset + - p_params->eth.vf_id; - break; - default: - pq_id = 0; - } - - pq_id = CM_TX_PQ_BASE + pq_id + RESC_START(p_hwfn, ECORE_PQ); - - return pq_id; -} - void ecore_hw_err_notify(struct ecore_hwfn *p_hwfn, enum ecore_hw_err_type err_type) { diff --git a/dpdk/drivers/net/qede/base/ecore_hw.h b/dpdk/drivers/net/qede/base/ecore_hw.h index 89499447..0b9814f5 100644 --- a/dpdk/drivers/net/qede/base/ecore_hw.h +++ b/dpdk/drivers/net/qede/base/ecore_hw.h @@ -71,8 +71,10 @@ enum _dmae_cmd_crc_mask { * @brief ecore_gtt_init - Initialize GTT windows * * @param p_hwfn +* @param p_ptt */ -void ecore_gtt_init(struct ecore_hwfn *p_hwfn); +void ecore_gtt_init(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt); /** * @brief ecore_ptt_invalidate - Forces all ptt entries to be re-configured @@ -97,16 +99,6 @@ enum _ecore_status_t ecore_ptt_pool_alloc(struct ecore_hwfn *p_hwfn); */ void ecore_ptt_pool_free(struct ecore_hwfn *p_hwfn); -/** - * @brief ecore_ptt_get_hw_addr - Get PTT's GRC/HW address - * - * @param p_hwfn - * @param p_ptt - * - * @return u32 - */ -u32 ecore_ptt_get_hw_addr(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt); - /** * @brief ecore_ptt_get_bar_addr - Get PPT's external BAR address * @@ -115,7 +107,7 @@ u32 ecore_ptt_get_hw_addr(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt); * * @return u32 */ -u32 ecore_ptt_get_bar_addr(struct ecore_ptt *p_ptt); +u32 ecore_ptt_get_bar_addr(struct ecore_ptt *p_ptt); /** * @brief ecore_ptt_set_win - Set PTT Window's GRC BAR address @@ -124,8 +116,9 @@ u32 ecore_ptt_get_bar_addr(struct ecore_ptt *p_ptt); * @param new_hw_addr * @param p_ptt */ -void ecore_ptt_set_win(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, u32 new_hw_addr); +void ecore_ptt_set_win(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + u32 new_hw_addr); /** * @brief ecore_get_reserved_ptt - Get a specific reserved PTT @@ -135,8 +128,8 @@ void ecore_ptt_set_win(struct ecore_hwfn *p_hwfn, * * @return struct ecore_ptt * */ -struct ecore_ptt *ecore_get_reserved_ptt(struct ecore_hwfn *p_hwfn, - enum reserved_ptts ptt_idx); +struct ecore_ptt *ecore_get_reserved_ptt(struct ecore_hwfn *p_hwfn, + enum reserved_ptts ptt_idx); /** * @brief ecore_wr - Write value to BAR using the given ptt @@ -146,8 +139,10 @@ struct ecore_ptt *ecore_get_reserved_ptt(struct ecore_hwfn *p_hwfn, * @param val * @param hw_addr */ -void ecore_wr(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, u32 hw_addr, u32 val); +void ecore_wr(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + u32 hw_addr, + u32 val); /** * @brief ecore_rd - Read value from BAR using the given ptt @@ -157,7 +152,9 @@ void ecore_wr(struct ecore_hwfn *p_hwfn, * @param val * @param hw_addr */ -u32 ecore_rd(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, u32 hw_addr); +u32 ecore_rd(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + u32 hw_addr); /** * @brief ecore_memcpy_from - copy n bytes from BAR using the given @@ -169,9 +166,11 @@ u32 ecore_rd(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, u32 hw_addr); * @param hw_addr * @param n */ -void ecore_memcpy_from(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, - void *dest, u32 hw_addr, osal_size_t n); +void ecore_memcpy_from(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + void *dest, + u32 hw_addr, + osal_size_t n); /** * @brief ecore_memcpy_to - copy n bytes to BAR using the given @@ -183,9 +182,11 @@ void ecore_memcpy_from(struct ecore_hwfn *p_hwfn, * @param src * @param n */ -void ecore_memcpy_to(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, - u32 hw_addr, void *src, osal_size_t n); +void ecore_memcpy_to(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + u32 hw_addr, + void *src, + osal_size_t n); /** * @brief ecore_fid_pretend - pretend to another function when * accessing the ptt window. There is no way to unpretend @@ -197,8 +198,9 @@ void ecore_memcpy_to(struct ecore_hwfn *p_hwfn, * @param fid - fid field of pxp_pretend structure. Can contain * either pf / vf, port/path fields are don't care. */ -void ecore_fid_pretend(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, u16 fid); +void ecore_fid_pretend(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + u16 fid); /** * @brief ecore_port_pretend - pretend to another port when @@ -208,8 +210,9 @@ void ecore_fid_pretend(struct ecore_hwfn *p_hwfn, * @param p_ptt * @param port_id - the port to pretend to */ -void ecore_port_pretend(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, u8 port_id); +void ecore_port_pretend(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + u8 port_id); /** * @brief ecore_port_unpretend - cancel any previously set port @@ -218,7 +221,8 @@ void ecore_port_pretend(struct ecore_hwfn *p_hwfn, * @param p_hwfn * @param p_ptt */ -void ecore_port_unpretend(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt); +void ecore_port_unpretend(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt); /** * @brief ecore_vfid_to_concrete - build a concrete FID for a @@ -235,7 +239,7 @@ u32 ecore_vfid_to_concrete(struct ecore_hwfn *p_hwfn, u8 vfid); * which is part of p_hwfn. * @param p_hwfn */ -enum _ecore_status_t ecore_dmae_info_alloc(struct ecore_hwfn *p_hwfn); +enum _ecore_status_t ecore_dmae_info_alloc(struct ecore_hwfn *p_hwfn); /** * @brief ecore_dmae_info_free - Free the dmae_info structure @@ -243,22 +247,7 @@ enum _ecore_status_t ecore_dmae_info_alloc(struct ecore_hwfn *p_hwfn); * * @param p_hwfn */ -void ecore_dmae_info_free(struct ecore_hwfn *p_hwfn); - -union ecore_qm_pq_params { - struct { - u8 tc; - } core; - - struct { - u8 is_vf; - u8 vf_id; - u8 tc; - } eth; -}; - -u16 ecore_get_qm_pq(struct ecore_hwfn *p_hwfn, - enum protocol_type proto, union ecore_qm_pq_params *params); +void ecore_dmae_info_free(struct ecore_hwfn *p_hwfn); enum _ecore_status_t ecore_init_fw_data(struct ecore_dev *p_dev, const u8 *fw_data); diff --git a/dpdk/drivers/net/qede/base/ecore_hw_defs.h b/dpdk/drivers/net/qede/base/ecore_hw_defs.h index fa518cec..4456af43 100644 --- a/dpdk/drivers/net/qede/base/ecore_hw_defs.h +++ b/dpdk/drivers/net/qede/base/ecore_hw_defs.h @@ -10,19 +10,30 @@ #define _ECORE_IGU_DEF_H_ /* Fields of IGU PF CONFIGRATION REGISTER */ -#define IGU_PF_CONF_FUNC_EN (0x1 << 0) /* function enable */ -#define IGU_PF_CONF_MSI_MSIX_EN (0x1 << 1) /* MSI/MSIX enable */ -#define IGU_PF_CONF_INT_LINE_EN (0x1 << 2) /* INT enable */ -#define IGU_PF_CONF_ATTN_BIT_EN (0x1 << 3) /* attention enable */ -#define IGU_PF_CONF_SINGLE_ISR_EN (0x1 << 4) /* single ISR mode enable */ -#define IGU_PF_CONF_SIMD_MODE (0x1 << 5) /* simd all ones mode */ +/* function enable */ +#define IGU_PF_CONF_FUNC_EN (0x1 << 0) +/* MSI/MSIX enable */ +#define IGU_PF_CONF_MSI_MSIX_EN (0x1 << 1) +/* INT enable */ +#define IGU_PF_CONF_INT_LINE_EN (0x1 << 2) +/* attention enable */ +#define IGU_PF_CONF_ATTN_BIT_EN (0x1 << 3) +/* single ISR mode enable */ +#define IGU_PF_CONF_SINGLE_ISR_EN (0x1 << 4) +/* simd all ones mode */ +#define IGU_PF_CONF_SIMD_MODE (0x1 << 5) /* Fields of IGU VF CONFIGRATION REGISTER */ -#define IGU_VF_CONF_FUNC_EN (0x1 << 0) /* function enable */ -#define IGU_VF_CONF_MSI_MSIX_EN (0x1 << 1) /* MSI/MSIX enable */ -#define IGU_VF_CONF_SINGLE_ISR_EN (0x1 << 4) /* single ISR mode enable */ -#define IGU_VF_CONF_PARENT_MASK (0xF) /* Parent PF */ -#define IGU_VF_CONF_PARENT_SHIFT 5 /* Parent PF */ +/* function enable */ +#define IGU_VF_CONF_FUNC_EN (0x1 << 0) +/* MSI/MSIX enable */ +#define IGU_VF_CONF_MSI_MSIX_EN (0x1 << 1) +/* single ISR mode enable */ +#define IGU_VF_CONF_SINGLE_ISR_EN (0x1 << 4) +/* Parent PF */ +#define IGU_VF_CONF_PARENT_MASK (0xF) +/* Parent PF */ +#define IGU_VF_CONF_PARENT_SHIFT 5 /* Igu control commands */ @@ -36,13 +47,13 @@ enum igu_ctrl_cmd { */ struct igu_ctrl_reg { u32 ctrl_data; -#define IGU_CTRL_REG_FID_MASK 0xFFFF /* Opaque_FID */ +#define IGU_CTRL_REG_FID_MASK 0xFFFF /* Opaque_FID */ #define IGU_CTRL_REG_FID_SHIFT 0 -#define IGU_CTRL_REG_PXP_ADDR_MASK 0xFFF /* Command address */ +#define IGU_CTRL_REG_PXP_ADDR_MASK 0xFFF /* Command address */ #define IGU_CTRL_REG_PXP_ADDR_SHIFT 16 #define IGU_CTRL_REG_RESERVED_MASK 0x1 #define IGU_CTRL_REG_RESERVED_SHIFT 28 -#define IGU_CTRL_REG_TYPE_MASK 0x1 /* use enum igu_ctrl_cmd */ +#define IGU_CTRL_REG_TYPE_MASK 0x1 /* use enum igu_ctrl_cmd */ #define IGU_CTRL_REG_TYPE_SHIFT 31 }; diff --git a/dpdk/drivers/net/qede/base/ecore_init_fw_funcs.c b/dpdk/drivers/net/qede/base/ecore_init_fw_funcs.c index 5324e052..1da80a65 100644 --- a/dpdk/drivers/net/qede/base/ecore_init_fw_funcs.c +++ b/dpdk/drivers/net/qede/base/ecore_init_fw_funcs.c @@ -12,113 +12,219 @@ #include "reg_addr.h" #include "ecore_rt_defs.h" #include "ecore_hsi_common.h" -#include "ecore_hsi_tools.h" +#include "ecore_hsi_init_func.h" +#include "ecore_hsi_eth.h" +#include "ecore_hsi_init_tool.h" +#include "ecore_iro.h" #include "ecore_init_fw_funcs.h" -/* @DPDK CmInterfaceEnum */ -enum cm_interface_enum { - MCM_SEC, - MCM_PRI, - UCM_SEC, - UCM_PRI, - TCM_SEC, - TCM_PRI, - YCM_SEC, - YCM_PRI, - XCM_SEC, - XCM_PRI, - NUM_OF_CM_INTERFACES +#define CDU_VALIDATION_DEFAULT_CFG 61 + +static u16 con_region_offsets[3][NUM_OF_CONNECTION_TYPES_E4] = { + { 400, 336, 352, 304, 304, 384, 416, 352}, /* region 3 offsets */ + { 528, 496, 416, 448, 448, 512, 544, 480}, /* region 4 offsets */ + { 608, 544, 496, 512, 576, 592, 624, 560} /* region 5 offsets */ }; -/* general constants */ -#define QM_PQ_MEM_4KB(pq_size) \ -(pq_size ? DIV_ROUND_UP((pq_size + 1) * QM_PQ_ELEMENT_SIZE, 0x1000) : 0) -#define QM_PQ_SIZE_256B(pq_size) \ -(pq_size ? DIV_ROUND_UP(pq_size, 0x100) - 1 : 0) -#define QM_INVALID_PQ_ID 0xffff -/* feature enable */ -#define QM_BYPASS_EN 1 -#define QM_BYTE_CRD_EN 1 -/* other PQ constants */ -#define QM_OTHER_PQS_PER_PF 4 -/* WFQ constants */ -#define QM_WFQ_UPPER_BOUND 62500000 +static u16 task_region_offsets[1][NUM_OF_CONNECTION_TYPES_E4] = { + { 240, 240, 112, 0, 0, 0, 0, 96} /* region 1 offsets */ +}; + +/* General constants */ +#define QM_PQ_MEM_4KB(pq_size) (pq_size ? DIV_ROUND_UP((pq_size + 1) * \ + QM_PQ_ELEMENT_SIZE, 0x1000) : 0) +#define QM_PQ_SIZE_256B(pq_size) (pq_size ? DIV_ROUND_UP(pq_size, 0x100) - 1 : \ + 0) +#define QM_INVALID_PQ_ID 0xffff + +/* Feature enable */ +#define QM_BYPASS_EN 1 +#define QM_BYTE_CRD_EN 1 + +/* Other PQ constants */ +#define QM_OTHER_PQS_PER_PF 4 + +/* VOQ constants */ +#define QM_E5_NUM_EXT_VOQ (MAX_NUM_PORTS_E5 * NUM_OF_TCS) + +/* WFQ constants: */ + +/* Upper bound in MB, 10 * burst size of 1ms in 50Gbps */ +#define QM_WFQ_UPPER_BOUND 62500000 + +/* Bit of VOQ in WFQ VP PQ map */ #define QM_WFQ_VP_PQ_VOQ_SHIFT 0 -#define QM_WFQ_VP_PQ_PF_SHIFT 5 + +/* Bit of PF in WFQ VP PQ map */ +#define QM_WFQ_VP_PQ_PF_E4_SHIFT 5 +#define QM_WFQ_VP_PQ_PF_E5_SHIFT 6 + +/* 0x9000 = 4*9*1024 */ #define QM_WFQ_INC_VAL(weight) ((weight) * 0x9000) -#define QM_WFQ_MAX_INC_VAL 43750000 -/* RL constants */ -#define QM_RL_UPPER_BOUND 62500000 -#define QM_RL_PERIOD 5 + +/* Max WFQ increment value is 0.7 * upper bound */ +#define QM_WFQ_MAX_INC_VAL ((QM_WFQ_UPPER_BOUND * 7) / 10) + +/* Number of VOQs in E5 QmWfqCrd register */ +#define QM_WFQ_CRD_E5_NUM_VOQS 16 + +/* RL constants: */ + +/* Period in us */ +#define QM_RL_PERIOD 5 + +/* Period in 25MHz cycles */ #define QM_RL_PERIOD_CLK_25M (25 * QM_RL_PERIOD) + +/* RL increment value - rate is specified in mbps. the factor of 1.01 was +* added after seeing only 99% factor reached in a 25Gbps port with DPDK RFC +* 2544 test. In this scenario the PF RL was reducing the line rate to 99% +* although the credit increment value was the correct one and FW calculated +* correct packet sizes. The reason for the inaccuracy of the RL is unknown at +* this point. +*/ #define QM_RL_INC_VAL(rate) \ -OSAL_MAX_T(u32, (((rate ? rate : 1000000) * QM_RL_PERIOD * 1.01) / 8), 1) -#define QM_RL_MAX_INC_VAL 43750000 + OSAL_MAX_T(u32, (u32)(((rate ? rate : 100000) * QM_RL_PERIOD * 101) / \ + (8 * 100)), 1) + +/* PF RL Upper bound is set to 10 * burst size of 1ms in 50Gbps */ +#define QM_PF_RL_UPPER_BOUND 62500000 + +/* Max PF RL increment value is 0.7 * upper bound */ +#define QM_PF_RL_MAX_INC_VAL ((QM_PF_RL_UPPER_BOUND * 7) / 10) + +/* Vport RL Upper bound, link speed is in Mpbs */ +#define QM_VP_RL_UPPER_BOUND(speed) \ + ((u32)OSAL_MAX_T(u32, QM_RL_INC_VAL(speed), 9700 + 1000)) + +/* Max Vport RL increment value is the Vport RL upper bound */ +#define QM_VP_RL_MAX_INC_VAL(speed) QM_VP_RL_UPPER_BOUND(speed) + +/* Vport RL credit threshold in case of QM bypass */ +#define QM_VP_RL_BYPASS_THRESH_SPEED (QM_VP_RL_UPPER_BOUND(10000) - 1) + /* AFullOprtnstcCrdMask constants */ #define QM_OPPOR_LINE_VOQ_DEF 1 #define QM_OPPOR_FW_STOP_DEF 0 #define QM_OPPOR_PQ_EMPTY_DEF 1 -#define EAGLE_WORKAROUND_TC 7 -/* Command Queue constants */ -#define PBF_CMDQ_PURE_LB_LINES 150 -#define PBF_CMDQ_EAGLE_WORKAROUND_LINES 8 /* eagle workaround CmdQ */ -#define PBF_CMDQ_LINES_RT_OFFSET(voq) \ -(PBF_REG_YCMD_QS_NUM_LINES_VOQ0_RT_OFFSET + \ -voq * (PBF_REG_YCMD_QS_NUM_LINES_VOQ1_RT_OFFSET \ -- PBF_REG_YCMD_QS_NUM_LINES_VOQ0_RT_OFFSET)) -#define PBF_BTB_GUARANTEED_RT_OFFSET(voq) \ -(PBF_REG_BTB_GUARANTEED_VOQ0_RT_OFFSET + voq * \ -(PBF_REG_BTB_GUARANTEED_VOQ1_RT_OFFSET - PBF_REG_BTB_GUARANTEED_VOQ0_RT_OFFSET)) + +/* Command Queue constants: */ + +/* Pure LB CmdQ lines (+spare) */ +#define PBF_CMDQ_PURE_LB_LINES 150 + +#define PBF_CMDQ_LINES_E5_RSVD_RATIO 8 + +#define PBF_CMDQ_LINES_RT_OFFSET(ext_voq) \ + (PBF_REG_YCMD_QS_NUM_LINES_VOQ0_RT_OFFSET + \ + ext_voq * \ + (PBF_REG_YCMD_QS_NUM_LINES_VOQ1_RT_OFFSET - \ + PBF_REG_YCMD_QS_NUM_LINES_VOQ0_RT_OFFSET)) + +#define PBF_BTB_GUARANTEED_RT_OFFSET(ext_voq) \ + (PBF_REG_BTB_GUARANTEED_VOQ0_RT_OFFSET + \ + ext_voq * \ + (PBF_REG_BTB_GUARANTEED_VOQ1_RT_OFFSET - \ + PBF_REG_BTB_GUARANTEED_VOQ0_RT_OFFSET)) + #define QM_VOQ_LINE_CRD(pbf_cmd_lines) \ ((((pbf_cmd_lines) - 4) * 2) | QM_LINE_CRD_REG_SIGN_BIT) + /* BTB: blocks constants (block size = 256B) */ -#define BTB_JUMBO_PKT_BLOCKS 38 /* 256B blocks in 9700B packet */ -#define BTB_HEADROOM_BLOCKS BTB_JUMBO_PKT_BLOCKS /* headroom per-port */ -#define BTB_EAGLE_WORKAROUND_BLOCKS 4 /* eagle workaround blocks */ + +/* 256B blocks in 9700B packet */ +#define BTB_JUMBO_PKT_BLOCKS 38 + +/* Headroom per-port */ +#define BTB_HEADROOM_BLOCKS BTB_JUMBO_PKT_BLOCKS #define BTB_PURE_LB_FACTOR 10 -#define BTB_PURE_LB_RATIO 7 /* factored (hence really 0.7) */ + +/* Factored (hence really 0.7) */ +#define BTB_PURE_LB_RATIO 7 + /* QM stop command constants */ -#define QM_STOP_PQ_MASK_WIDTH 32 -#define QM_STOP_CMD_ADDR 0x2 -#define QM_STOP_CMD_STRUCT_SIZE 2 +#define QM_STOP_PQ_MASK_WIDTH 32 +#define QM_STOP_CMD_ADDR 2 +#define QM_STOP_CMD_STRUCT_SIZE 2 #define QM_STOP_CMD_PAUSE_MASK_OFFSET 0 #define QM_STOP_CMD_PAUSE_MASK_SHIFT 0 -#define QM_STOP_CMD_PAUSE_MASK_MASK -1 -#define QM_STOP_CMD_GROUP_ID_OFFSET 1 -#define QM_STOP_CMD_GROUP_ID_SHIFT 16 -#define QM_STOP_CMD_GROUP_ID_MASK 15 -#define QM_STOP_CMD_PQ_TYPE_OFFSET 1 -#define QM_STOP_CMD_PQ_TYPE_SHIFT 24 -#define QM_STOP_CMD_PQ_TYPE_MASK 1 -#define QM_STOP_CMD_MAX_POLL_COUNT 100 -#define QM_STOP_CMD_POLL_PERIOD_US 500 +#define QM_STOP_CMD_PAUSE_MASK_MASK 0xffffffff /* @DPDK */ +#define QM_STOP_CMD_GROUP_ID_OFFSET 1 +#define QM_STOP_CMD_GROUP_ID_SHIFT 16 +#define QM_STOP_CMD_GROUP_ID_MASK 15 +#define QM_STOP_CMD_PQ_TYPE_OFFSET 1 +#define QM_STOP_CMD_PQ_TYPE_SHIFT 24 +#define QM_STOP_CMD_PQ_TYPE_MASK 1 +#define QM_STOP_CMD_MAX_POLL_COUNT 100 +#define QM_STOP_CMD_POLL_PERIOD_US 500 + /* QM command macros */ -#define QM_CMD_STRUCT_SIZE(cmd) cmd##_STRUCT_SIZE +#define QM_CMD_STRUCT_SIZE(cmd) cmd##_STRUCT_SIZE #define QM_CMD_SET_FIELD(var, cmd, field, value) \ -SET_FIELD(var[cmd##_##field##_OFFSET], cmd##_##field, value) -/* QM: VOQ macros */ -#define PHYS_VOQ(port, tc, max_phys_tcs_per_port) \ -((port) * (max_phys_tcs_per_port) + (tc)) -#define LB_VOQ(port) (MAX_PHYS_VOQS + (port)) -#define VOQ(port, tc, max_phys_tcs_per_port) \ -((tc) < LB_TC ? PHYS_VOQ(port, tc, max_phys_tcs_per_port) : LB_VOQ(port)) + SET_FIELD(var[cmd##_##field##_OFFSET], cmd##_##field, value) + +#define QM_INIT_TX_PQ_MAP(p_hwfn, map, chip, pq_id, rl_valid, \ + vp_pq_id, rl_id, ext_voq, wrr) \ + do { \ + OSAL_MEMSET(&map, 0, sizeof(map)); \ + SET_FIELD(map.reg, QM_RF_PQ_MAP_##chip##_PQ_VALID, 1); \ + SET_FIELD(map.reg, QM_RF_PQ_MAP_##chip##_RL_VALID, rl_valid); \ + SET_FIELD(map.reg, QM_RF_PQ_MAP_##chip##_VP_PQ_ID, vp_pq_id); \ + SET_FIELD(map.reg, QM_RF_PQ_MAP_##chip##_RL_ID, rl_id); \ + SET_FIELD(map.reg, QM_RF_PQ_MAP_##chip##_VOQ, ext_voq); \ + SET_FIELD(map.reg, \ + QM_RF_PQ_MAP_##chip##_WRR_WEIGHT_GROUP, wrr); \ + STORE_RT_REG(p_hwfn, QM_REG_TXPQMAP_RT_OFFSET + pq_id, \ + *((u32 *)&map)); \ + } while (0) + +#define WRITE_PQ_INFO_TO_RAM 1 +#define PQ_INFO_ELEMENT(vp, pf, tc, port, rl_valid, rl) \ + (((vp) << 0) | ((pf) << 12) | ((tc) << 16) | \ + ((port) << 20) | ((rl_valid) << 22) | ((rl) << 24)) +#define PQ_INFO_RAM_GRC_ADDRESS(pq_id) \ + (XSEM_REG_FAST_MEMORY + SEM_FAST_REG_INT_RAM + 21768 + (pq_id) * 4) + /******************** INTERNAL IMPLEMENTATION *********************/ + +/* Returns the external VOQ number */ +static u8 ecore_get_ext_voq(struct ecore_hwfn *p_hwfn, + u8 port_id, + u8 tc, + u8 max_phys_tcs_per_port) +{ + if (tc == PURE_LB_TC) + return NUM_OF_PHYS_TCS * (MAX_NUM_PORTS_BB) + port_id; + else + return port_id * (max_phys_tcs_per_port) + tc; +} + /* Prepare PF RL enable/disable runtime init values */ static void ecore_enable_pf_rl(struct ecore_hwfn *p_hwfn, bool pf_rl_en) { STORE_RT_REG(p_hwfn, QM_REG_RLPFENABLE_RT_OFFSET, pf_rl_en ? 1 : 0); if (pf_rl_en) { - /* enable RLs for all VOQs */ + u8 num_ext_voqs = MAX_NUM_VOQS_E4; + u64 voq_bit_mask = ((u64)1 << num_ext_voqs) - 1; + + /* Enable RLs for all VOQs */ STORE_RT_REG(p_hwfn, QM_REG_RLPFVOQENABLE_RT_OFFSET, - (1 << MAX_NUM_VOQS) - 1); - /* write RL period */ + (u32)voq_bit_mask); +#ifdef QM_REG_RLPFVOQENABLE_MSB_RT_OFFSET + if (num_ext_voqs >= 32) + STORE_RT_REG(p_hwfn, QM_REG_RLPFVOQENABLE_MSB_RT_OFFSET, + (u32)(voq_bit_mask >> 32)); +#endif + + /* Write RL period */ STORE_RT_REG(p_hwfn, QM_REG_RLPFPERIOD_RT_OFFSET, QM_RL_PERIOD_CLK_25M); STORE_RT_REG(p_hwfn, QM_REG_RLPFPERIODTIMER_RT_OFFSET, QM_RL_PERIOD_CLK_25M); - /* set credit threshold for QM bypass flow */ + + /* Set credit threshold for QM bypass flow */ if (QM_BYPASS_EN) STORE_RT_REG(p_hwfn, QM_REG_AFULLQMBYPTHRPFRL_RT_OFFSET, - QM_RL_UPPER_BOUND); + QM_PF_RL_UPPER_BOUND); } } @@ -126,7 +232,8 @@ static void ecore_enable_pf_rl(struct ecore_hwfn *p_hwfn, bool pf_rl_en) static void ecore_enable_pf_wfq(struct ecore_hwfn *p_hwfn, bool pf_wfq_en) { STORE_RT_REG(p_hwfn, QM_REG_WFQPFENABLE_RT_OFFSET, pf_wfq_en ? 1 : 0); - /* set credit threshold for QM bypass flow */ + + /* Set credit threshold for QM bypass flow */ if (pf_wfq_en && QM_BYPASS_EN) STORE_RT_REG(p_hwfn, QM_REG_AFULLQMBYPTHRPFWFQ_RT_OFFSET, QM_WFQ_UPPER_BOUND); @@ -138,16 +245,17 @@ static void ecore_enable_vport_rl(struct ecore_hwfn *p_hwfn, bool vport_rl_en) STORE_RT_REG(p_hwfn, QM_REG_RLGLBLENABLE_RT_OFFSET, vport_rl_en ? 1 : 0); if (vport_rl_en) { - /* write RL period (use timer 0 only) */ + /* Write RL period (use timer 0 only) */ STORE_RT_REG(p_hwfn, QM_REG_RLGLBLPERIOD_0_RT_OFFSET, QM_RL_PERIOD_CLK_25M); STORE_RT_REG(p_hwfn, QM_REG_RLGLBLPERIODTIMER_0_RT_OFFSET, QM_RL_PERIOD_CLK_25M); - /* set credit threshold for QM bypass flow */ + + /* Set credit threshold for QM bypass flow */ if (QM_BYPASS_EN) STORE_RT_REG(p_hwfn, QM_REG_AFULLQMBYPTHRGLBLRL_RT_OFFSET, - QM_RL_UPPER_BOUND); + QM_VP_RL_BYPASS_THRESH_SPEED); } } @@ -156,7 +264,8 @@ static void ecore_enable_vport_wfq(struct ecore_hwfn *p_hwfn, bool vport_wfq_en) { STORE_RT_REG(p_hwfn, QM_REG_WFQVPENABLE_RT_OFFSET, vport_wfq_en ? 1 : 0); - /* set credit threshold for QM bypass flow */ + + /* Set credit threshold for QM bypass flow */ if (vport_wfq_en && QM_BYPASS_EN) STORE_RT_REG(p_hwfn, QM_REG_AFULLQMBYPTHRVPWFQ_RT_OFFSET, QM_WFQ_UPPER_BOUND); @@ -166,18 +275,19 @@ static void ecore_enable_vport_wfq(struct ecore_hwfn *p_hwfn, bool vport_wfq_en) * the specified VOQ */ static void ecore_cmdq_lines_voq_rt_init(struct ecore_hwfn *p_hwfn, - u8 voq, u16 cmdq_lines) + u8 ext_voq, + u16 cmdq_lines) { u32 qm_line_crd; - bool is_bb_a0 = ECORE_IS_BB_A0(p_hwfn->p_dev); - if (is_bb_a0) - cmdq_lines = OSAL_MIN_T(u32, cmdq_lines, 1022); + qm_line_crd = QM_VOQ_LINE_CRD(cmdq_lines); - OVERWRITE_RT_REG(p_hwfn, PBF_CMDQ_LINES_RT_OFFSET(voq), + + OVERWRITE_RT_REG(p_hwfn, PBF_CMDQ_LINES_RT_OFFSET(ext_voq), (u32)cmdq_lines); - STORE_RT_REG(p_hwfn, QM_REG_VOQCRDLINE_RT_OFFSET + voq, qm_line_crd); - STORE_RT_REG(p_hwfn, QM_REG_VOQINITCRDLINE_RT_OFFSET + voq, - qm_line_crd); + STORE_RT_REG(p_hwfn, QM_REG_VOQCRDLINE_RT_OFFSET + ext_voq, + qm_line_crd); + STORE_RT_REG(p_hwfn, QM_REG_VOQINITCRDLINE_RT_OFFSET + ext_voq, + qm_line_crd); } /* Prepare runtime init values to allocate PBF command queue lines. */ @@ -187,45 +297,49 @@ static void ecore_cmdq_lines_rt_init(struct ecore_hwfn *p_hwfn, struct init_qm_port_params port_params[MAX_NUM_PORTS]) { - u8 tc, voq, port_id; - bool eagle_workaround = ENABLE_EAGLE_ENG1_WORKAROUND(p_hwfn); - /* clear PBF lines for all VOQs */ - for (voq = 0; voq < MAX_NUM_VOQS; voq++) - STORE_RT_REG(p_hwfn, PBF_CMDQ_LINES_RT_OFFSET(voq), 0); + u8 tc, ext_voq, port_id, num_tcs_in_port; + u8 num_ext_voqs = MAX_NUM_VOQS_E4; + + /* Clear PBF lines of all VOQs */ + for (ext_voq = 0; ext_voq < num_ext_voqs; ext_voq++) + STORE_RT_REG(p_hwfn, PBF_CMDQ_LINES_RT_OFFSET(ext_voq), 0); + for (port_id = 0; port_id < max_ports_per_engine; port_id++) { - if (port_params[port_id].active) { - u16 phys_lines, phys_lines_per_tc; - phys_lines = - port_params[port_id].num_pbf_cmd_lines - - PBF_CMDQ_PURE_LB_LINES; - if (eagle_workaround) - phys_lines -= PBF_CMDQ_EAGLE_WORKAROUND_LINES; - /* find #lines per active physical TC */ - phys_lines_per_tc = - phys_lines / - port_params[port_id].num_active_phys_tcs; - /* init registers per active TC */ - for (tc = 0; - tc < port_params[port_id].num_active_phys_tcs; - tc++) { - voq = - PHYS_VOQ(port_id, tc, - max_phys_tcs_per_port); - ecore_cmdq_lines_voq_rt_init(p_hwfn, voq, + u16 phys_lines, phys_lines_per_tc; + + if (!port_params[port_id].active) + continue; + + /* Find number of command queue lines to divide between the + * active physical TCs. In E5, 1/8 of the lines are reserved. + * the lines for pure LB TC are subtracted. + */ + phys_lines = port_params[port_id].num_pbf_cmd_lines; + phys_lines -= PBF_CMDQ_PURE_LB_LINES; + + /* Find #lines per active physical TC */ + num_tcs_in_port = 0; + for (tc = 0; tc < max_phys_tcs_per_port; tc++) + if (((port_params[port_id].active_phys_tcs >> tc) & + 0x1) == 1) + num_tcs_in_port++; + phys_lines_per_tc = phys_lines / num_tcs_in_port; + + /* Init registers per active TC */ + for (tc = 0; tc < max_phys_tcs_per_port; tc++) { + ext_voq = ecore_get_ext_voq(p_hwfn, port_id, tc, + max_phys_tcs_per_port); + if (((port_params[port_id].active_phys_tcs >> tc) & + 0x1) == 1) + ecore_cmdq_lines_voq_rt_init(p_hwfn, ext_voq, phys_lines_per_tc); - } - /* init registers for pure LB TC */ - ecore_cmdq_lines_voq_rt_init(p_hwfn, LB_VOQ(port_id), - PBF_CMDQ_PURE_LB_LINES); - /* init registers for eagle workaround */ - if (eagle_workaround) { - voq = - PHYS_VOQ(port_id, EAGLE_WORKAROUND_TC, - max_phys_tcs_per_port); - ecore_cmdq_lines_voq_rt_init(p_hwfn, voq, - PBF_CMDQ_EAGLE_WORKAROUND_LINES); - } } + + /* Init registers for pure LB TC */ + ext_voq = ecore_get_ext_voq(p_hwfn, port_id, PURE_LB_TC, + max_phys_tcs_per_port); + ecore_cmdq_lines_voq_rt_init(p_hwfn, ext_voq, + PBF_CMDQ_PURE_LB_LINES); } } @@ -255,54 +369,52 @@ static void ecore_btb_blocks_rt_init(struct ecore_hwfn *p_hwfn, struct init_qm_port_params port_params[MAX_NUM_PORTS]) { - u8 tc, voq, port_id; u32 usable_blocks, pure_lb_blocks, phys_blocks; - bool eagle_workaround = ENABLE_EAGLE_ENG1_WORKAROUND(p_hwfn); + u8 tc, ext_voq, port_id, num_tcs_in_port; + for (port_id = 0; port_id < max_ports_per_engine; port_id++) { - if (port_params[port_id].active) { - /* subtract headroom blocks */ - usable_blocks = - port_params[port_id].num_btb_blocks - - BTB_HEADROOM_BLOCKS; - if (eagle_workaround) - usable_blocks -= BTB_EAGLE_WORKAROUND_BLOCKS; - pure_lb_blocks = - (usable_blocks * BTB_PURE_LB_FACTOR) / - (port_params[port_id].num_active_phys_tcs * - BTB_PURE_LB_FACTOR + BTB_PURE_LB_RATIO); - pure_lb_blocks = - OSAL_MAX_T(u32, BTB_JUMBO_PKT_BLOCKS, - pure_lb_blocks / BTB_PURE_LB_FACTOR); - phys_blocks = - (usable_blocks - - pure_lb_blocks) / - port_params[port_id].num_active_phys_tcs; - /* init physical TCs */ - for (tc = 0; - tc < port_params[port_id].num_active_phys_tcs; - tc++) { - voq = - PHYS_VOQ(port_id, tc, - max_phys_tcs_per_port); + if (!port_params[port_id].active) + continue; + + /* Subtract headroom blocks */ + usable_blocks = port_params[port_id].num_btb_blocks - + BTB_HEADROOM_BLOCKS; + + /* Find blocks per physical TC. use factor to avoid floating + * arithmethic. + */ + num_tcs_in_port = 0; + for (tc = 0; tc < NUM_OF_PHYS_TCS; tc++) + if (((port_params[port_id].active_phys_tcs >> tc) & + 0x1) == 1) + num_tcs_in_port++; + + pure_lb_blocks = (usable_blocks * BTB_PURE_LB_FACTOR) / + (num_tcs_in_port * BTB_PURE_LB_FACTOR + + BTB_PURE_LB_RATIO); + pure_lb_blocks = OSAL_MAX_T(u32, BTB_JUMBO_PKT_BLOCKS, + pure_lb_blocks / + BTB_PURE_LB_FACTOR); + phys_blocks = (usable_blocks - pure_lb_blocks) / + num_tcs_in_port; + + /* Init physical TCs */ + for (tc = 0; tc < NUM_OF_PHYS_TCS; tc++) { + if (((port_params[port_id].active_phys_tcs >> tc) & + 0x1) == 1) { + ext_voq = ecore_get_ext_voq(p_hwfn, port_id, tc, + max_phys_tcs_per_port); STORE_RT_REG(p_hwfn, - PBF_BTB_GUARANTEED_RT_OFFSET(voq), - phys_blocks); - } - /* init pure LB TC */ - STORE_RT_REG(p_hwfn, - PBF_BTB_GUARANTEED_RT_OFFSET(LB_VOQ - (port_id)), - pure_lb_blocks); - /* init eagle workaround */ - if (eagle_workaround) { - voq = - PHYS_VOQ(port_id, EAGLE_WORKAROUND_TC, - max_phys_tcs_per_port); - STORE_RT_REG(p_hwfn, - PBF_BTB_GUARANTEED_RT_OFFSET(voq), - BTB_EAGLE_WORKAROUND_BLOCKS); + PBF_BTB_GUARANTEED_RT_OFFSET(ext_voq), + phys_blocks); } } + + /* Init pure LB TC */ + ext_voq = ecore_get_ext_voq(p_hwfn, port_id, PURE_LB_TC, + max_phys_tcs_per_port); + STORE_RT_REG(p_hwfn, PBF_BTB_GUARANTEED_RT_OFFSET(ext_voq), + pure_lb_blocks); } } @@ -312,7 +424,6 @@ static void ecore_tx_pq_map_rt_init(struct ecore_hwfn *p_hwfn, u8 port_id, u8 pf_id, u8 max_phys_tcs_per_port, - bool is_first_pf, u32 num_pf_cids, u32 num_vf_cids, u16 start_pq, @@ -323,113 +434,136 @@ static void ecore_tx_pq_map_rt_init(struct ecore_hwfn *p_hwfn, struct init_qm_pq_params *pq_params, struct init_qm_vport_params *vport_params) { - u16 i, pq_id, pq_group; - u16 num_pqs = num_pf_pqs + num_vf_pqs; - u16 first_pq_group = start_pq / QM_PF_QUEUE_GROUP_SIZE; - u16 last_pq_group = (start_pq + num_pqs - 1) / QM_PF_QUEUE_GROUP_SIZE; - bool is_bb_a0 = ECORE_IS_BB_A0(p_hwfn->p_dev); - /* a bit per Tx PQ indicating if the PQ is associated with a VF */ + /* A bit per Tx PQ indicating if the PQ is associated with a VF */ u32 tx_pq_vf_mask[MAX_QM_TX_QUEUES / QM_PF_QUEUE_GROUP_SIZE] = { 0 }; - u32 tx_pq_vf_mask_width = is_bb_a0 ? 32 : QM_PF_QUEUE_GROUP_SIZE; - u32 num_tx_pq_vf_masks = MAX_QM_TX_QUEUES / tx_pq_vf_mask_width; - u32 pq_mem_4kb = QM_PQ_MEM_4KB(num_pf_cids); - u32 vport_pq_mem_4kb = QM_PQ_MEM_4KB(num_vf_cids); - u32 mem_addr_4kb = base_mem_addr_4kb; - /* set mapping from PQ group to PF */ + u32 num_tx_pq_vf_masks = MAX_QM_TX_QUEUES / QM_PF_QUEUE_GROUP_SIZE; + u16 num_pqs, first_pq_group, last_pq_group, i, pq_id, pq_group; + u32 pq_mem_4kb, vport_pq_mem_4kb, mem_addr_4kb; + + num_pqs = num_pf_pqs + num_vf_pqs; + + first_pq_group = start_pq / QM_PF_QUEUE_GROUP_SIZE; + last_pq_group = (start_pq + num_pqs - 1) / QM_PF_QUEUE_GROUP_SIZE; + + pq_mem_4kb = QM_PQ_MEM_4KB(num_pf_cids); + vport_pq_mem_4kb = QM_PQ_MEM_4KB(num_vf_cids); + mem_addr_4kb = base_mem_addr_4kb; + + /* Set mapping from PQ group to PF */ for (pq_group = first_pq_group; pq_group <= last_pq_group; pq_group++) STORE_RT_REG(p_hwfn, QM_REG_PQTX2PF_0_RT_OFFSET + pq_group, (u32)(pf_id)); - /* set PQ sizes */ + + /* Set PQ sizes */ STORE_RT_REG(p_hwfn, QM_REG_MAXPQSIZE_0_RT_OFFSET, QM_PQ_SIZE_256B(num_pf_cids)); STORE_RT_REG(p_hwfn, QM_REG_MAXPQSIZE_1_RT_OFFSET, QM_PQ_SIZE_256B(num_vf_cids)); - /* go over all Tx PQs */ + + /* Go over all Tx PQs */ for (i = 0, pq_id = start_pq; i < num_pqs; i++, pq_id++) { - struct qm_rf_pq_map tx_pq_map; - u8 voq = - VOQ(port_id, pq_params[i].tc_id, max_phys_tcs_per_port); - bool is_vf_pq = (i >= num_pf_pqs); - /* update first Tx PQ of VPORT/TC */ - u8 vport_id_in_pf = pq_params[i].vport_id - start_vport; - u16 first_tx_pq_id = - vport_params[vport_id_in_pf].first_tx_pq_id[pq_params[i]. - tc_id]; + u32 max_qm_global_rls = MAX_QM_GLOBAL_RLS; + u8 ext_voq, vport_id_in_pf; + bool is_vf_pq, rl_valid; + u16 first_tx_pq_id; + + ext_voq = ecore_get_ext_voq(p_hwfn, port_id, pq_params[i].tc_id, + max_phys_tcs_per_port); + is_vf_pq = (i >= num_pf_pqs); + rl_valid = pq_params[i].rl_valid && pq_params[i].vport_id < + max_qm_global_rls; + + /* Update first Tx PQ of VPORT/TC */ + vport_id_in_pf = pq_params[i].vport_id - start_vport; + first_tx_pq_id = + vport_params[vport_id_in_pf].first_tx_pq_id[pq_params[i].tc_id]; if (first_tx_pq_id == QM_INVALID_PQ_ID) { - /* create new VP PQ */ + u32 map_val = (ext_voq << QM_WFQ_VP_PQ_VOQ_SHIFT) | + (pf_id << (QM_WFQ_VP_PQ_PF_E4_SHIFT)); + + /* Create new VP PQ */ vport_params[vport_id_in_pf]. first_tx_pq_id[pq_params[i].tc_id] = pq_id; first_tx_pq_id = pq_id; - /* map VP PQ to VOQ and PF */ - STORE_RT_REG(p_hwfn, - QM_REG_WFQVPMAP_RT_OFFSET + first_tx_pq_id, - (voq << QM_WFQ_VP_PQ_VOQ_SHIFT) | (pf_id << - QM_WFQ_VP_PQ_PF_SHIFT)); + + /* Map VP PQ to VOQ and PF */ + STORE_RT_REG(p_hwfn, QM_REG_WFQVPMAP_RT_OFFSET + + first_tx_pq_id, map_val); } - /* fill PQ map entry */ - OSAL_MEMSET(&tx_pq_map, 0, sizeof(tx_pq_map)); - SET_FIELD(tx_pq_map.reg, QM_RF_PQ_MAP_PQ_VALID, 1); - SET_FIELD(tx_pq_map.reg, QM_RF_PQ_MAP_RL_VALID, - is_vf_pq ? 1 : 0); - SET_FIELD(tx_pq_map.reg, QM_RF_PQ_MAP_VP_PQ_ID, first_tx_pq_id); - SET_FIELD(tx_pq_map.reg, QM_RF_PQ_MAP_RL_ID, - is_vf_pq ? pq_params[i].vport_id : 0); - SET_FIELD(tx_pq_map.reg, QM_RF_PQ_MAP_VOQ, voq); - SET_FIELD(tx_pq_map.reg, QM_RF_PQ_MAP_WRR_WEIGHT_GROUP, - pq_params[i].wrr_group); - /* write PQ map entry to CAM */ - STORE_RT_REG(p_hwfn, QM_REG_TXPQMAP_RT_OFFSET + pq_id, - *((u32 *)&tx_pq_map)); - /* set base address */ + + /* Check RL ID */ + if (pq_params[i].rl_valid && pq_params[i].vport_id >= + max_qm_global_rls) + DP_NOTICE(p_hwfn, true, + "Invalid VPORT ID for rate limiter config\n"); + + /* Prepare PQ map entry */ + struct qm_rf_pq_map_e4 tx_pq_map; + QM_INIT_TX_PQ_MAP(p_hwfn, tx_pq_map, E4, pq_id, rl_valid ? + 1 : 0, + first_tx_pq_id, rl_valid ? + pq_params[i].vport_id : 0, + ext_voq, pq_params[i].wrr_group); + + /* Set base address */ STORE_RT_REG(p_hwfn, QM_REG_BASEADDRTXPQ_RT_OFFSET + pq_id, mem_addr_4kb); - /* check if VF PQ */ + + /* Write PQ info to RAM */ + if (WRITE_PQ_INFO_TO_RAM != 0) { + u32 pq_info = 0; + pq_info = PQ_INFO_ELEMENT(first_tx_pq_id, pf_id, + pq_params[i].tc_id, port_id, + rl_valid ? 1 : 0, rl_valid ? + pq_params[i].vport_id : 0); + ecore_wr(p_hwfn, p_ptt, PQ_INFO_RAM_GRC_ADDRESS(pq_id), + pq_info); + } + + /* If VF PQ, add indication to PQ VF mask */ if (is_vf_pq) { - tx_pq_vf_mask[pq_id / tx_pq_vf_mask_width] |= - (1 << (pq_id % tx_pq_vf_mask_width)); + tx_pq_vf_mask[pq_id / QM_PF_QUEUE_GROUP_SIZE] |= + (1 << (pq_id % QM_PF_QUEUE_GROUP_SIZE)); mem_addr_4kb += vport_pq_mem_4kb; } else { mem_addr_4kb += pq_mem_4kb; } } - /* store Tx PQ VF mask to size select register */ - for (i = 0; i < num_tx_pq_vf_masks; i++) { - if (tx_pq_vf_mask[i]) { - if (is_bb_a0) { - u32 curr_mask = - is_first_pf ? 0 : ecore_rd(p_hwfn, p_ptt, - QM_REG_MAXPQSIZETXSEL_0 - + i * 4); - STORE_RT_REG(p_hwfn, - QM_REG_MAXPQSIZETXSEL_0_RT_OFFSET + - i, curr_mask | tx_pq_vf_mask[i]); - } else - STORE_RT_REG(p_hwfn, - QM_REG_MAXPQSIZETXSEL_0_RT_OFFSET + - i, tx_pq_vf_mask[i]); - } - } + + /* Store Tx PQ VF mask to size select register */ + for (i = 0; i < num_tx_pq_vf_masks; i++) + if (tx_pq_vf_mask[i]) + STORE_RT_REG(p_hwfn, QM_REG_MAXPQSIZETXSEL_0_RT_OFFSET + + i, tx_pq_vf_mask[i]); } /* Prepare Other PQ mapping runtime init values for the specified PF */ static void ecore_other_pq_map_rt_init(struct ecore_hwfn *p_hwfn, - u8 port_id, u8 pf_id, u32 num_pf_cids, - u32 num_tids, u32 base_mem_addr_4kb) + u32 num_tids, + u32 base_mem_addr_4kb) { - u16 i, pq_id; - u16 pq_group = pf_id; - u32 pq_size = num_pf_cids + num_tids; - u32 pq_mem_4kb = QM_PQ_MEM_4KB(pq_size); - u32 mem_addr_4kb = base_mem_addr_4kb; - /* map PQ group to PF */ + u32 pq_size, pq_mem_4kb, mem_addr_4kb; + u16 i, pq_id, pq_group; + + /* A single other PQ group is used in each PF, where PQ group i is used + * in PF i. + */ + pq_group = pf_id; + pq_size = num_pf_cids + num_tids; + pq_mem_4kb = QM_PQ_MEM_4KB(pq_size); + mem_addr_4kb = base_mem_addr_4kb; + + /* Map PQ group to PF */ STORE_RT_REG(p_hwfn, QM_REG_PQOTHER2PF_0_RT_OFFSET + pq_group, (u32)(pf_id)); - /* set PQ sizes */ + + /* Set PQ sizes */ STORE_RT_REG(p_hwfn, QM_REG_MAXPQSIZE_2_RT_OFFSET, QM_PQ_SIZE_256B(pq_size)); - /* set base address */ + + /* Set base address */ for (i = 0, pq_id = pf_id * QM_PF_QUEUE_GROUP_SIZE; i < QM_OTHER_PQS_PER_PF; i++, pq_id++) { STORE_RT_REG(p_hwfn, QM_REG_BASEADDROTHERPQ_RT_OFFSET + pq_id, @@ -438,6 +572,9 @@ static void ecore_other_pq_map_rt_init(struct ecore_hwfn *p_hwfn, } } +/* Prepare PF WFQ runtime init values for the specified PF. + * Return -1 on error. + */ static int ecore_pf_wfq_rt_init(struct ecore_hwfn *p_hwfn, u8 port_id, u8 pf_id, @@ -446,101 +583,136 @@ static int ecore_pf_wfq_rt_init(struct ecore_hwfn *p_hwfn, u16 num_tx_pqs, struct init_qm_pq_params *pq_params) { + u32 inc_val, crd_reg_offset; + u8 ext_voq; u16 i; - u32 inc_val; - u32 crd_reg_offset = - (pf_id < - MAX_NUM_PFS_BB ? QM_REG_WFQPFCRD_RT_OFFSET : - QM_REG_WFQPFCRD_MSB_RT_OFFSET) + (pf_id % MAX_NUM_PFS_BB); + inc_val = QM_WFQ_INC_VAL(pf_wfq); - if (inc_val == 0 || inc_val > QM_WFQ_MAX_INC_VAL) { - DP_NOTICE(p_hwfn, true, "Invalid PF WFQ weight configuration"); + if (!inc_val || inc_val > QM_WFQ_MAX_INC_VAL) { + DP_NOTICE(p_hwfn, true, + "Invalid PF WFQ weight configuration\n"); return -1; } + for (i = 0; i < num_tx_pqs; i++) { - u8 voq = - VOQ(port_id, pq_params[i].tc_id, max_phys_tcs_per_port); - OVERWRITE_RT_REG(p_hwfn, crd_reg_offset + voq * MAX_NUM_PFS_BB, - QM_WFQ_CRD_REG_SIGN_BIT); + ext_voq = ecore_get_ext_voq(p_hwfn, port_id, pq_params[i].tc_id, + max_phys_tcs_per_port); + crd_reg_offset = (pf_id < MAX_NUM_PFS_BB ? + QM_REG_WFQPFCRD_RT_OFFSET : + QM_REG_WFQPFCRD_MSB_RT_OFFSET) + + ext_voq * MAX_NUM_PFS_BB + + (pf_id % MAX_NUM_PFS_BB); + OVERWRITE_RT_REG(p_hwfn, crd_reg_offset, + (u32)QM_WFQ_CRD_REG_SIGN_BIT); + STORE_RT_REG(p_hwfn, QM_REG_WFQPFUPPERBOUND_RT_OFFSET + pf_id, + QM_WFQ_UPPER_BOUND | (u32)QM_WFQ_CRD_REG_SIGN_BIT); + STORE_RT_REG(p_hwfn, QM_REG_WFQPFWEIGHT_RT_OFFSET + pf_id, + inc_val); } - STORE_RT_REG(p_hwfn, QM_REG_WFQPFUPPERBOUND_RT_OFFSET + pf_id, - QM_WFQ_UPPER_BOUND | QM_WFQ_CRD_REG_SIGN_BIT); - STORE_RT_REG(p_hwfn, QM_REG_WFQPFWEIGHT_RT_OFFSET + pf_id, inc_val); + return 0; } -/* Prepare PF RL runtime init values for the specified PF. Return -1 on err */ +/* Prepare PF RL runtime init values for the specified PF. + * Return -1 on error. + */ static int ecore_pf_rl_rt_init(struct ecore_hwfn *p_hwfn, u8 pf_id, u32 pf_rl) { - u32 inc_val = QM_RL_INC_VAL(pf_rl); - if (inc_val > QM_RL_MAX_INC_VAL) { - DP_NOTICE(p_hwfn, true, "Invalid PF rate limit configuration"); + u32 inc_val; + + inc_val = QM_RL_INC_VAL(pf_rl); + if (inc_val > QM_PF_RL_MAX_INC_VAL) { + DP_NOTICE(p_hwfn, true, + "Invalid PF rate limit configuration\n"); return -1; } + STORE_RT_REG(p_hwfn, QM_REG_RLPFCRD_RT_OFFSET + pf_id, - QM_RL_CRD_REG_SIGN_BIT); + (u32)QM_RL_CRD_REG_SIGN_BIT); STORE_RT_REG(p_hwfn, QM_REG_RLPFUPPERBOUND_RT_OFFSET + pf_id, - QM_RL_UPPER_BOUND | QM_RL_CRD_REG_SIGN_BIT); + QM_PF_RL_UPPER_BOUND | (u32)QM_RL_CRD_REG_SIGN_BIT); STORE_RT_REG(p_hwfn, QM_REG_RLPFINCVAL_RT_OFFSET + pf_id, inc_val); + return 0; } +/* Prepare VPORT WFQ runtime init values for the specified VPORTs. + * Return -1 on error. + */ static int ecore_vp_wfq_rt_init(struct ecore_hwfn *p_hwfn, u8 num_vports, struct init_qm_vport_params *vport_params) { - u8 tc, i; + u16 vport_pq_id; u32 inc_val; - /* go over all PF VPORTs */ + u8 tc, i; + + /* Go over all PF VPORTs */ for (i = 0; i < num_vports; i++) { - if (vport_params[i].vport_wfq) { - inc_val = QM_WFQ_INC_VAL(vport_params[i].vport_wfq); - if (inc_val > QM_WFQ_MAX_INC_VAL) { - DP_NOTICE(p_hwfn, true, - "Invalid VPORT WFQ weight config"); - return -1; - } - for (tc = 0; tc < NUM_OF_TCS; tc++) { - u16 vport_pq_id = - vport_params[i].first_tx_pq_id[tc]; - if (vport_pq_id != QM_INVALID_PQ_ID) { - STORE_RT_REG(p_hwfn, - QM_REG_WFQVPCRD_RT_OFFSET + - vport_pq_id, - QM_WFQ_CRD_REG_SIGN_BIT); - STORE_RT_REG(p_hwfn, - QM_REG_WFQVPWEIGHT_RT_OFFSET - + vport_pq_id, inc_val); - } + if (!vport_params[i].vport_wfq) + continue; + + inc_val = QM_WFQ_INC_VAL(vport_params[i].vport_wfq); + if (inc_val > QM_WFQ_MAX_INC_VAL) { + DP_NOTICE(p_hwfn, true, + "Invalid VPORT WFQ weight configuration\n"); + return -1; + } + + /* Each VPORT can have several VPORT PQ IDs for various TCs */ + for (tc = 0; tc < NUM_OF_TCS; tc++) { + vport_pq_id = vport_params[i].first_tx_pq_id[tc]; + if (vport_pq_id != QM_INVALID_PQ_ID) { + STORE_RT_REG(p_hwfn, QM_REG_WFQVPCRD_RT_OFFSET + + vport_pq_id, + (u32)QM_WFQ_CRD_REG_SIGN_BIT); + STORE_RT_REG(p_hwfn, + QM_REG_WFQVPWEIGHT_RT_OFFSET + + vport_pq_id, inc_val); } } } return 0; } -/* Prepare VPORT RL runtime init values for specified VPORT. Ret -1 on error. */ +/* Prepare VPORT RL runtime init values for the specified VPORTs. + * Return -1 on error. + */ static int ecore_vport_rl_rt_init(struct ecore_hwfn *p_hwfn, u8 start_vport, u8 num_vports, + u32 link_speed, struct init_qm_vport_params *vport_params) { u8 i, vport_id; - /* go over all PF VPORTs */ + u32 inc_val; + + if (start_vport + num_vports >= MAX_QM_GLOBAL_RLS) { + DP_NOTICE(p_hwfn, true, + "Invalid VPORT ID for rate limiter configuration\n"); + return -1; + } + + /* Go over all PF VPORTs */ for (i = 0, vport_id = start_vport; i < num_vports; i++, vport_id++) { - u32 inc_val = QM_RL_INC_VAL(vport_params[i].vport_rl); - if (inc_val > QM_RL_MAX_INC_VAL) { + inc_val = QM_RL_INC_VAL(vport_params[i].vport_rl ? + vport_params[i].vport_rl : link_speed); + if (inc_val > QM_VP_RL_MAX_INC_VAL(link_speed)) { DP_NOTICE(p_hwfn, true, - "Invalid VPORT rate-limit configuration"); + "Invalid VPORT rate-limit configuration\n"); return -1; } + STORE_RT_REG(p_hwfn, QM_REG_RLGLBLCRD_RT_OFFSET + vport_id, - QM_RL_CRD_REG_SIGN_BIT); + (u32)QM_RL_CRD_REG_SIGN_BIT); STORE_RT_REG(p_hwfn, QM_REG_RLGLBLUPPERBOUND_RT_OFFSET + vport_id, - QM_RL_UPPER_BOUND | QM_RL_CRD_REG_SIGN_BIT); + QM_VP_RL_UPPER_BOUND(link_speed) | + (u32)QM_RL_CRD_REG_SIGN_BIT); STORE_RT_REG(p_hwfn, QM_REG_RLGLBLINCVAL_RT_OFFSET + vport_id, inc_val); } + return 0; } @@ -548,39 +720,49 @@ static bool ecore_poll_on_qm_cmd_ready(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt) { u32 reg_val, i; - for (i = 0, reg_val = 0; i < QM_STOP_CMD_MAX_POLL_COUNT && reg_val == 0; + + for (i = 0, reg_val = 0; i < QM_STOP_CMD_MAX_POLL_COUNT && !reg_val; i++) { OSAL_UDELAY(QM_STOP_CMD_POLL_PERIOD_US); reg_val = ecore_rd(p_hwfn, p_ptt, QM_REG_SDMCMDREADY); } - /* check if timeout while waiting for SDM command ready */ + + /* Check if timeout while waiting for SDM command ready */ if (i == QM_STOP_CMD_MAX_POLL_COUNT) { - DP_VERBOSE(p_hwfn, ECORE_MSG_HW, + DP_VERBOSE(p_hwfn, ECORE_MSG_DEBUG, "Timeout waiting for QM SDM cmd ready signal\n"); return false; } + return true; } static bool ecore_send_qm_cmd(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, - u32 cmd_addr, u32 cmd_data_lsb, u32 cmd_data_msb) + u32 cmd_addr, + u32 cmd_data_lsb, + u32 cmd_data_msb) { if (!ecore_poll_on_qm_cmd_ready(p_hwfn, p_ptt)) return false; + ecore_wr(p_hwfn, p_ptt, QM_REG_SDMCMDADDR, cmd_addr); ecore_wr(p_hwfn, p_ptt, QM_REG_SDMCMDDATALSB, cmd_data_lsb); ecore_wr(p_hwfn, p_ptt, QM_REG_SDMCMDDATAMSB, cmd_data_msb); ecore_wr(p_hwfn, p_ptt, QM_REG_SDMCMDGO, 1); ecore_wr(p_hwfn, p_ptt, QM_REG_SDMCMDGO, 0); + return ecore_poll_on_qm_cmd_ready(p_hwfn, p_ptt); } + /******************** INTERFACE IMPLEMENTATION *********************/ -u32 ecore_qm_pf_mem_size(u8 pf_id, - u32 num_pf_cids, - u32 num_vf_cids, - u32 num_tids, u16 num_pf_pqs, u16 num_vf_pqs) + +u32 ecore_qm_pf_mem_size(u32 num_pf_cids, + u32 num_vf_cids, + u32 num_tids, + u16 num_pf_pqs, + u16 num_vf_pqs) { return QM_PQ_MEM_4KB(num_pf_cids) * num_pf_pqs + QM_PQ_MEM_4KB(num_vf_cids) * num_vf_pqs + @@ -597,45 +779,42 @@ int ecore_qm_common_rt_init(struct ecore_hwfn *p_hwfn, struct init_qm_port_params port_params[MAX_NUM_PORTS]) { - u8 port_id; - /* init AFullOprtnstcCrdMask */ - u32 mask = - (QM_OPPOR_LINE_VOQ_DEF << QM_RF_OPPORTUNISTIC_MASK_LINEVOQ_SHIFT) | - (QM_BYTE_CRD_EN << QM_RF_OPPORTUNISTIC_MASK_BYTEVOQ_SHIFT) | - (pf_wfq_en << QM_RF_OPPORTUNISTIC_MASK_PFWFQ_SHIFT) | - (vport_wfq_en << QM_RF_OPPORTUNISTIC_MASK_VPWFQ_SHIFT) | - (pf_rl_en << QM_RF_OPPORTUNISTIC_MASK_PFRL_SHIFT) | - (vport_rl_en << QM_RF_OPPORTUNISTIC_MASK_VPQCNRL_SHIFT) | - (QM_OPPOR_FW_STOP_DEF << QM_RF_OPPORTUNISTIC_MASK_FWPAUSE_SHIFT) | - (QM_OPPOR_PQ_EMPTY_DEF << - QM_RF_OPPORTUNISTIC_MASK_QUEUEEMPTY_SHIFT); + u32 mask; + + /* Init AFullOprtnstcCrdMask */ + mask = (QM_OPPOR_LINE_VOQ_DEF << + QM_RF_OPPORTUNISTIC_MASK_LINEVOQ_SHIFT) | + (QM_BYTE_CRD_EN << QM_RF_OPPORTUNISTIC_MASK_BYTEVOQ_SHIFT) | + (pf_wfq_en << QM_RF_OPPORTUNISTIC_MASK_PFWFQ_SHIFT) | + (vport_wfq_en << QM_RF_OPPORTUNISTIC_MASK_VPWFQ_SHIFT) | + (pf_rl_en << QM_RF_OPPORTUNISTIC_MASK_PFRL_SHIFT) | + (vport_rl_en << QM_RF_OPPORTUNISTIC_MASK_VPQCNRL_SHIFT) | + (QM_OPPOR_FW_STOP_DEF << + QM_RF_OPPORTUNISTIC_MASK_FWPAUSE_SHIFT) | + (QM_OPPOR_PQ_EMPTY_DEF << + QM_RF_OPPORTUNISTIC_MASK_QUEUEEMPTY_SHIFT); STORE_RT_REG(p_hwfn, QM_REG_AFULLOPRTNSTCCRDMASK_RT_OFFSET, mask); - /* check eagle workaround */ - for (port_id = 0; port_id < max_ports_per_engine; port_id++) { - if (port_params[port_id].active && - port_params[port_id].num_active_phys_tcs > - EAGLE_WORKAROUND_TC && - ENABLE_EAGLE_ENG1_WORKAROUND(p_hwfn)) { - DP_NOTICE(p_hwfn, true, - "Can't config 8 TCs with Eagle" - " eng1 workaround"); - return -1; - } - } - /* enable/disable PF RL */ + + /* Enable/disable PF RL */ ecore_enable_pf_rl(p_hwfn, pf_rl_en); - /* enable/disable PF WFQ */ + + /* Enable/disable PF WFQ */ ecore_enable_pf_wfq(p_hwfn, pf_wfq_en); - /* enable/disable VPORT RL */ + + /* Enable/disable VPORT RL */ ecore_enable_vport_rl(p_hwfn, vport_rl_en); - /* enable/disable VPORT WFQ */ + + /* Enable/disable VPORT WFQ */ ecore_enable_vport_wfq(p_hwfn, vport_wfq_en); - /* init PBF CMDQ line credit */ + + /* Init PBF CMDQ line credit */ ecore_cmdq_lines_rt_init(p_hwfn, max_ports_per_engine, max_phys_tcs_per_port, port_params); - /* init BTB blocks in PBF */ + + /* Init BTB blocks in PBF */ ecore_btb_blocks_rt_init(p_hwfn, max_ports_per_engine, max_phys_tcs_per_port, port_params); + return 0; } @@ -644,7 +823,6 @@ int ecore_qm_pf_rt_init(struct ecore_hwfn *p_hwfn, u8 port_id, u8 pf_id, u8 max_phys_tcs_per_port, - bool is_first_pf, u32 num_pf_cids, u32 num_vf_cids, u32 num_tids, @@ -655,69 +833,88 @@ int ecore_qm_pf_rt_init(struct ecore_hwfn *p_hwfn, u8 num_vports, u16 pf_wfq, u32 pf_rl, + u32 link_speed, struct init_qm_pq_params *pq_params, struct init_qm_vport_params *vport_params) { + u32 other_mem_size_4kb; u8 tc, i; - u32 other_mem_size_4kb = - QM_PQ_MEM_4KB(num_pf_cids + num_tids) * QM_OTHER_PQS_PER_PF; - /* clear first Tx PQ ID array for each VPORT */ + + other_mem_size_4kb = QM_PQ_MEM_4KB(num_pf_cids + num_tids) * + QM_OTHER_PQS_PER_PF; + + /* Clear first Tx PQ ID array for each VPORT */ for (i = 0; i < num_vports; i++) for (tc = 0; tc < NUM_OF_TCS; tc++) vport_params[i].first_tx_pq_id[tc] = QM_INVALID_PQ_ID; - /* map Other PQs (if any) */ + + /* Map Other PQs (if any) */ #if QM_OTHER_PQS_PER_PF > 0 - ecore_other_pq_map_rt_init(p_hwfn, port_id, pf_id, num_pf_cids, - num_tids, 0); + ecore_other_pq_map_rt_init(p_hwfn, pf_id, num_pf_cids, num_tids, 0); #endif - /* map Tx PQs */ + + /* Map Tx PQs */ ecore_tx_pq_map_rt_init(p_hwfn, p_ptt, port_id, pf_id, - max_phys_tcs_per_port, is_first_pf, num_pf_cids, - num_vf_cids, start_pq, num_pf_pqs, num_vf_pqs, - start_vport, other_mem_size_4kb, pq_params, - vport_params); - /* init PF WFQ */ + max_phys_tcs_per_port, num_pf_cids, num_vf_cids, + start_pq, num_pf_pqs, num_vf_pqs, start_vport, + other_mem_size_4kb, pq_params, vport_params); + + /* Init PF WFQ */ if (pf_wfq) if (ecore_pf_wfq_rt_init (p_hwfn, port_id, pf_id, pf_wfq, max_phys_tcs_per_port, - num_pf_pqs + num_vf_pqs, pq_params) != 0) + num_pf_pqs + num_vf_pqs, pq_params)) return -1; - /* init PF RL */ - if (ecore_pf_rl_rt_init(p_hwfn, pf_id, pf_rl) != 0) + + /* Init PF RL */ + if (ecore_pf_rl_rt_init(p_hwfn, pf_id, pf_rl)) return -1; - /* set VPORT WFQ */ - if (ecore_vp_wfq_rt_init(p_hwfn, num_vports, vport_params) != 0) + + /* Set VPORT WFQ */ + if (ecore_vp_wfq_rt_init(p_hwfn, num_vports, vport_params)) return -1; - /* set VPORT RL */ + + /* Set VPORT RL */ if (ecore_vport_rl_rt_init - (p_hwfn, start_vport, num_vports, vport_params) != 0) + (p_hwfn, start_vport, num_vports, link_speed, vport_params)) return -1; + return 0; } int ecore_init_pf_wfq(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, u8 pf_id, u16 pf_wfq) { - u32 inc_val = QM_WFQ_INC_VAL(pf_wfq); - if (inc_val == 0 || inc_val > QM_WFQ_MAX_INC_VAL) { - DP_NOTICE(p_hwfn, true, "Invalid PF WFQ weight configuration"); + u32 inc_val; + + inc_val = QM_WFQ_INC_VAL(pf_wfq); + if (!inc_val || inc_val > QM_WFQ_MAX_INC_VAL) { + DP_NOTICE(p_hwfn, true, + "Invalid PF WFQ weight configuration\n"); return -1; } + ecore_wr(p_hwfn, p_ptt, QM_REG_WFQPFWEIGHT + pf_id * 4, inc_val); + return 0; } int ecore_init_pf_rl(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, u8 pf_id, u32 pf_rl) { - u32 inc_val = QM_RL_INC_VAL(pf_rl); - if (inc_val > QM_RL_MAX_INC_VAL) { - DP_NOTICE(p_hwfn, true, "Invalid PF rate limit configuration"); + u32 inc_val; + + inc_val = QM_RL_INC_VAL(pf_rl); + if (inc_val > QM_PF_RL_MAX_INC_VAL) { + DP_NOTICE(p_hwfn, true, + "Invalid PF rate limit configuration\n"); return -1; } + ecore_wr(p_hwfn, p_ptt, QM_REG_RLPFCRD + pf_id * 4, - QM_RL_CRD_REG_SIGN_BIT); + (u32)QM_RL_CRD_REG_SIGN_BIT); ecore_wr(p_hwfn, p_ptt, QM_REG_RLPFINCVAL + pf_id * 4, inc_val); + return 0; } @@ -725,35 +922,52 @@ int ecore_init_vport_wfq(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, u16 first_tx_pq_id[NUM_OF_TCS], u16 vport_wfq) { + u16 vport_pq_id; + u32 inc_val; u8 tc; - u32 inc_val = QM_WFQ_INC_VAL(vport_wfq); - if (inc_val == 0 || inc_val > QM_WFQ_MAX_INC_VAL) { + + inc_val = QM_WFQ_INC_VAL(vport_wfq); + if (!inc_val || inc_val > QM_WFQ_MAX_INC_VAL) { DP_NOTICE(p_hwfn, true, - "Invalid VPORT WFQ weight configuration"); + "Invalid VPORT WFQ weight configuration\n"); return -1; } + for (tc = 0; tc < NUM_OF_TCS; tc++) { - u16 vport_pq_id = first_tx_pq_id[tc]; + vport_pq_id = first_tx_pq_id[tc]; if (vport_pq_id != QM_INVALID_PQ_ID) { ecore_wr(p_hwfn, p_ptt, QM_REG_WFQVPWEIGHT + vport_pq_id * 4, inc_val); } } + return 0; } int ecore_init_vport_rl(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, u8 vport_id, u32 vport_rl) + struct ecore_ptt *p_ptt, u8 vport_id, + u32 vport_rl, + u32 link_speed) { - u32 inc_val = QM_RL_INC_VAL(vport_rl); - if (inc_val > QM_RL_MAX_INC_VAL) { + u32 inc_val, max_qm_global_rls = MAX_QM_GLOBAL_RLS; + + if (vport_id >= max_qm_global_rls) { DP_NOTICE(p_hwfn, true, - "Invalid VPORT rate-limit configuration"); + "Invalid VPORT ID for rate limiter configuration\n"); return -1; } + + inc_val = QM_RL_INC_VAL(vport_rl ? vport_rl : link_speed); + if (inc_val > QM_VP_RL_MAX_INC_VAL(link_speed)) { + DP_NOTICE(p_hwfn, true, + "Invalid VPORT rate-limit configuration\n"); + return -1; + } + ecore_wr(p_hwfn, p_ptt, QM_REG_RLGLBLCRD + vport_id * 4, - QM_RL_CRD_REG_SIGN_BIT); + (u32)QM_RL_CRD_REG_SIGN_BIT); ecore_wr(p_hwfn, p_ptt, QM_REG_RLGLBLINCVAL + vport_id * 4, inc_val); + return 0; } @@ -763,15 +977,20 @@ bool ecore_send_qm_stop_cmd(struct ecore_hwfn *p_hwfn, bool is_tx_pq, u16 start_pq, u16 num_pqs) { u32 cmd_arr[QM_CMD_STRUCT_SIZE(QM_STOP_CMD)] = { 0 }; - u32 pq_mask = 0, last_pq = start_pq + num_pqs - 1, pq_id; - /* set command's PQ type */ + u32 pq_mask = 0, last_pq, pq_id; + + last_pq = start_pq + num_pqs - 1; + + /* Set command's PQ type */ QM_CMD_SET_FIELD(cmd_arr, QM_STOP_CMD, PQ_TYPE, is_tx_pq ? 0 : 1); - /* go over requested PQs */ + + /* Go over requested PQs */ for (pq_id = start_pq; pq_id <= last_pq; pq_id++) { - /* set PQ bit in mask (stop command only) */ + /* Set PQ bit in mask (stop command only) */ if (!is_release_cmd) pq_mask |= (1 << (pq_id % QM_STOP_PQ_MASK_WIDTH)); - /* if last PQ or end of PQ mask, write command */ + + /* If last PQ or end of PQ mask, write command */ if ((pq_id == last_pq) || (pq_id % QM_STOP_PQ_MASK_WIDTH == (QM_STOP_PQ_MASK_WIDTH - 1))) { @@ -786,68 +1005,92 @@ bool ecore_send_qm_stop_cmd(struct ecore_hwfn *p_hwfn, pq_mask = 0; } } + return true; } + /* NIG: ETS configuration constants */ #define NIG_TX_ETS_CLIENT_OFFSET 4 #define NIG_LB_ETS_CLIENT_OFFSET 1 #define NIG_ETS_MIN_WFQ_BYTES 1600 + /* NIG: ETS constants */ #define NIG_ETS_UP_BOUND(weight, mtu) \ -(2 * ((weight) > (mtu) ? (weight) : (mtu))) + (2 * ((weight) > (mtu) ? (weight) : (mtu))) + /* NIG: RL constants */ -#define NIG_RL_BASE_TYPE 1 /* byte base type */ -#define NIG_RL_PERIOD 1 /* in us */ + +/* Byte base type value */ +#define NIG_RL_BASE_TYPE 1 + +/* Period in us */ +#define NIG_RL_PERIOD 1 + +/* Period in 25MHz cycles */ #define NIG_RL_PERIOD_CLK_25M (25 * NIG_RL_PERIOD) + +/* Rate in mbps */ #define NIG_RL_INC_VAL(rate) (((rate) * NIG_RL_PERIOD) / 8) + #define NIG_RL_MAX_VAL(inc_val, mtu) \ -(2 * ((inc_val) > (mtu) ? (inc_val) : (mtu))) + (2 * ((inc_val) > (mtu) ? (inc_val) : (mtu))) + /* NIG: packet prioritry configuration constants */ -#define NIG_PRIORITY_MAP_TC_BITS 4 +#define NIG_PRIORITY_MAP_TC_BITS 4 + + void ecore_init_nig_ets(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, struct init_ets_req *req, bool is_lb) { - u8 tc, sp_tc_map = 0, wfq_tc_map = 0; - u8 num_tc = is_lb ? NUM_OF_TCS : NUM_OF_PHYS_TCS; - u8 tc_client_offset = - is_lb ? NIG_LB_ETS_CLIENT_OFFSET : NIG_TX_ETS_CLIENT_OFFSET; - u32 min_weight = 0xffffffff; - u32 tc_weight_base_addr = - is_lb ? NIG_REG_LB_ARB_CREDIT_WEIGHT_0 : - NIG_REG_TX_ARB_CREDIT_WEIGHT_0; - u32 tc_weight_addr_diff = - is_lb ? NIG_REG_LB_ARB_CREDIT_WEIGHT_1 - - NIG_REG_LB_ARB_CREDIT_WEIGHT_0 : NIG_REG_TX_ARB_CREDIT_WEIGHT_1 - - NIG_REG_TX_ARB_CREDIT_WEIGHT_0; - u32 tc_bound_base_addr = - is_lb ? NIG_REG_LB_ARB_CREDIT_UPPER_BOUND_0 : - NIG_REG_TX_ARB_CREDIT_UPPER_BOUND_0; - u32 tc_bound_addr_diff = - is_lb ? NIG_REG_LB_ARB_CREDIT_UPPER_BOUND_1 - - NIG_REG_LB_ARB_CREDIT_UPPER_BOUND_0 : - NIG_REG_TX_ARB_CREDIT_UPPER_BOUND_1 - - NIG_REG_TX_ARB_CREDIT_UPPER_BOUND_0; + u32 min_weight, tc_weight_base_addr, tc_weight_addr_diff; + u32 tc_bound_base_addr, tc_bound_addr_diff; + u8 sp_tc_map = 0, wfq_tc_map = 0; + u8 tc, num_tc, tc_client_offset; + + num_tc = is_lb ? NUM_OF_TCS : NUM_OF_PHYS_TCS; + tc_client_offset = is_lb ? NIG_LB_ETS_CLIENT_OFFSET : + NIG_TX_ETS_CLIENT_OFFSET; + min_weight = 0xffffffff; + tc_weight_base_addr = is_lb ? NIG_REG_LB_ARB_CREDIT_WEIGHT_0 : + NIG_REG_TX_ARB_CREDIT_WEIGHT_0; + tc_weight_addr_diff = is_lb ? NIG_REG_LB_ARB_CREDIT_WEIGHT_1 - + NIG_REG_LB_ARB_CREDIT_WEIGHT_0 : + NIG_REG_TX_ARB_CREDIT_WEIGHT_1 - + NIG_REG_TX_ARB_CREDIT_WEIGHT_0; + tc_bound_base_addr = is_lb ? NIG_REG_LB_ARB_CREDIT_UPPER_BOUND_0 : + NIG_REG_TX_ARB_CREDIT_UPPER_BOUND_0; + tc_bound_addr_diff = is_lb ? NIG_REG_LB_ARB_CREDIT_UPPER_BOUND_1 - + NIG_REG_LB_ARB_CREDIT_UPPER_BOUND_0 : + NIG_REG_TX_ARB_CREDIT_UPPER_BOUND_1 - + NIG_REG_TX_ARB_CREDIT_UPPER_BOUND_0; + for (tc = 0; tc < num_tc; tc++) { struct init_ets_tc_req *tc_req = &req->tc_req[tc]; - /* update SP map */ + + /* Update SP map */ if (tc_req->use_sp) sp_tc_map |= (1 << tc); - if (tc_req->use_wfq) { - /* update WFQ map */ - wfq_tc_map |= (1 << tc); - /* find minimal weight */ - if (tc_req->weight < min_weight) - min_weight = tc_req->weight; - } + + if (!tc_req->use_wfq) + continue; + + /* Update WFQ map */ + wfq_tc_map |= (1 << tc); + + /* Find minimal weight */ + if (tc_req->weight < min_weight) + min_weight = tc_req->weight; } - /* write SP map */ + + /* Write SP map */ ecore_wr(p_hwfn, p_ptt, is_lb ? NIG_REG_LB_ARB_CLIENT_IS_STRICT : NIG_REG_TX_ARB_CLIENT_IS_STRICT, (sp_tc_map << tc_client_offset)); - /* write WFQ map */ + + /* Write WFQ map */ ecore_wr(p_hwfn, p_ptt, is_lb ? NIG_REG_LB_ARB_CLIENT_IS_SUBJECT2WFQ : NIG_REG_TX_ARB_CLIENT_IS_SUBJECT2WFQ, @@ -855,22 +1098,23 @@ void ecore_init_nig_ets(struct ecore_hwfn *p_hwfn, /* write WFQ weights */ for (tc = 0; tc < num_tc; tc++, tc_client_offset++) { struct init_ets_tc_req *tc_req = &req->tc_req[tc]; - if (tc_req->use_wfq) { - /* translate weight to bytes */ - u32 byte_weight = - (NIG_ETS_MIN_WFQ_BYTES * tc_req->weight) / - min_weight; - /* write WFQ weight */ - ecore_wr(p_hwfn, p_ptt, - tc_weight_base_addr + - tc_weight_addr_diff * tc_client_offset, - byte_weight); - /* write WFQ upper bound */ - ecore_wr(p_hwfn, p_ptt, - tc_bound_base_addr + - tc_bound_addr_diff * tc_client_offset, - NIG_ETS_UP_BOUND(byte_weight, req->mtu)); - } + u32 byte_weight; + + if (!tc_req->use_wfq) + continue; + + /* Translate weight to bytes */ + byte_weight = (NIG_ETS_MIN_WFQ_BYTES * tc_req->weight) / + min_weight; + + /* Write WFQ weight */ + ecore_wr(p_hwfn, p_ptt, tc_weight_base_addr + + tc_weight_addr_diff * tc_client_offset, byte_weight); + + /* Write WFQ upper bound */ + ecore_wr(p_hwfn, p_ptt, tc_bound_base_addr + + tc_bound_addr_diff * tc_client_offset, + NIG_ETS_UP_BOUND(byte_weight, req->mtu)); } } @@ -878,16 +1122,18 @@ void ecore_init_nig_lb_rl(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, struct init_nig_lb_rl_req *req) { - u8 tc; u32 ctrl, inc_val, reg_offset; - /* disable global MAC+LB RL */ + u8 tc; + + /* Disable global MAC+LB RL */ ctrl = NIG_RL_BASE_TYPE << NIG_REG_TX_LB_GLBRATELIMIT_CTRL_TX_LB_GLBRATELIMIT_BASE_TYPE_SHIFT; ecore_wr(p_hwfn, p_ptt, NIG_REG_TX_LB_GLBRATELIMIT_CTRL, ctrl); - /* configure and enable global MAC+LB RL */ + + /* Configure and enable global MAC+LB RL */ if (req->lb_mac_rate) { - /* configure */ + /* Configure */ ecore_wr(p_hwfn, p_ptt, NIG_REG_TX_LB_GLBRATELIMIT_INC_PERIOD, NIG_RL_PERIOD_CLK_25M); inc_val = NIG_RL_INC_VAL(req->lb_mac_rate); @@ -895,20 +1141,23 @@ void ecore_init_nig_lb_rl(struct ecore_hwfn *p_hwfn, inc_val); ecore_wr(p_hwfn, p_ptt, NIG_REG_TX_LB_GLBRATELIMIT_MAX_VALUE, NIG_RL_MAX_VAL(inc_val, req->mtu)); - /* enable */ + + /* Enable */ ctrl |= 1 << NIG_REG_TX_LB_GLBRATELIMIT_CTRL_TX_LB_GLBRATELIMIT_EN_SHIFT; ecore_wr(p_hwfn, p_ptt, NIG_REG_TX_LB_GLBRATELIMIT_CTRL, ctrl); } - /* disable global LB-only RL */ + + /* Disable global LB-only RL */ ctrl = NIG_RL_BASE_TYPE << NIG_REG_LB_BRBRATELIMIT_CTRL_LB_BRBRATELIMIT_BASE_TYPE_SHIFT; ecore_wr(p_hwfn, p_ptt, NIG_REG_LB_BRBRATELIMIT_CTRL, ctrl); - /* configure and enable global LB-only RL */ + + /* Configure and enable global LB-only RL */ if (req->lb_rate) { - /* configure */ + /* Configure */ ecore_wr(p_hwfn, p_ptt, NIG_REG_LB_BRBRATELIMIT_INC_PERIOD, NIG_RL_PERIOD_CLK_25M); inc_val = NIG_RL_INC_VAL(req->lb_rate); @@ -916,41 +1165,41 @@ void ecore_init_nig_lb_rl(struct ecore_hwfn *p_hwfn, inc_val); ecore_wr(p_hwfn, p_ptt, NIG_REG_LB_BRBRATELIMIT_MAX_VALUE, NIG_RL_MAX_VAL(inc_val, req->mtu)); - /* enable */ + + /* Enable */ ctrl |= 1 << NIG_REG_LB_BRBRATELIMIT_CTRL_LB_BRBRATELIMIT_EN_SHIFT; ecore_wr(p_hwfn, p_ptt, NIG_REG_LB_BRBRATELIMIT_CTRL, ctrl); } - /* per-TC RLs */ + + /* Per-TC RLs */ for (tc = 0, reg_offset = 0; tc < NUM_OF_PHYS_TCS; tc++, reg_offset += 4) { - /* disable TC RL */ + /* Disable TC RL */ ctrl = NIG_RL_BASE_TYPE << NIG_REG_LB_TCRATELIMIT_CTRL_0_LB_TCRATELIMIT_BASE_TYPE_0_SHIFT; ecore_wr(p_hwfn, p_ptt, NIG_REG_LB_TCRATELIMIT_CTRL_0 + reg_offset, ctrl); - /* configure and enable TC RL */ - if (req->tc_rate[tc]) { - /* configure */ - ecore_wr(p_hwfn, p_ptt, - NIG_REG_LB_TCRATELIMIT_INC_PERIOD_0 + - reg_offset, NIG_RL_PERIOD_CLK_25M); - inc_val = NIG_RL_INC_VAL(req->tc_rate[tc]); - ecore_wr(p_hwfn, p_ptt, - NIG_REG_LB_TCRATELIMIT_INC_VALUE_0 + - reg_offset, inc_val); - ecore_wr(p_hwfn, p_ptt, - NIG_REG_LB_TCRATELIMIT_MAX_VALUE_0 + - reg_offset, NIG_RL_MAX_VAL(inc_val, req->mtu)); - /* enable */ - ctrl |= - 1 << - NIG_REG_LB_TCRATELIMIT_CTRL_0_LB_TCRATELIMIT_EN_0_SHIFT; - ecore_wr(p_hwfn, p_ptt, - NIG_REG_LB_TCRATELIMIT_CTRL_0 + reg_offset, - ctrl); - } + + /* Configure and enable TC RL */ + if (!req->tc_rate[tc]) + continue; + + /* Configure */ + ecore_wr(p_hwfn, p_ptt, NIG_REG_LB_TCRATELIMIT_INC_PERIOD_0 + + reg_offset, NIG_RL_PERIOD_CLK_25M); + inc_val = NIG_RL_INC_VAL(req->tc_rate[tc]); + ecore_wr(p_hwfn, p_ptt, NIG_REG_LB_TCRATELIMIT_INC_VALUE_0 + + reg_offset, inc_val); + ecore_wr(p_hwfn, p_ptt, NIG_REG_LB_TCRATELIMIT_MAX_VALUE_0 + + reg_offset, NIG_RL_MAX_VAL(inc_val, req->mtu)); + + /* Enable */ + ctrl |= 1 << + NIG_REG_LB_TCRATELIMIT_CTRL_0_LB_TCRATELIMIT_EN_0_SHIFT; + ecore_wr(p_hwfn, p_ptt, NIG_REG_LB_TCRATELIMIT_CTRL_0 + + reg_offset, ctrl); } } @@ -958,20 +1207,23 @@ void ecore_init_nig_pri_tc_map(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, struct init_nig_pri_tc_map_req *req) { - u8 pri, tc; - u32 pri_tc_mask = 0; u8 tc_pri_mask[NUM_OF_PHYS_TCS] = { 0 }; + u32 pri_tc_mask = 0; + u8 pri, tc; + for (pri = 0; pri < NUM_OF_VLAN_PRIORITIES; pri++) { - if (req->pri[pri].valid) { - pri_tc_mask |= - (req->pri[pri]. - tc_id << (pri * NIG_PRIORITY_MAP_TC_BITS)); - tc_pri_mask[req->pri[pri].tc_id] |= (1 << pri); - } + if (!req->pri[pri].valid) + continue; + + pri_tc_mask |= (req->pri[pri].tc_id << + (pri * NIG_PRIORITY_MAP_TC_BITS)); + tc_pri_mask[req->pri[pri].tc_id] |= (1 << pri); } - /* write priority -> TC mask */ + + /* Write priority -> TC mask */ ecore_wr(p_hwfn, p_ptt, NIG_REG_PKT_PRIORITY_TO_TC, pri_tc_mask); - /* write TC -> priority mask */ + + /* Write TC -> priority mask */ for (tc = 0; tc < NUM_OF_PHYS_TCS; tc++) { ecore_wr(p_hwfn, p_ptt, NIG_REG_PRIORITY_FOR_TC_0 + tc * 4, tc_pri_mask[tc]); @@ -980,110 +1232,133 @@ void ecore_init_nig_pri_tc_map(struct ecore_hwfn *p_hwfn, } } + /* PRS: ETS configuration constants */ -#define PRS_ETS_MIN_WFQ_BYTES 1600 +#define PRS_ETS_MIN_WFQ_BYTES 1600 #define PRS_ETS_UP_BOUND(weight, mtu) \ -(2 * ((weight) > (mtu) ? (weight) : (mtu))) + (2 * ((weight) > (mtu) ? (weight) : (mtu))) + + void ecore_init_prs_ets(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, struct init_ets_req *req) { + u32 tc_weight_addr_diff, tc_bound_addr_diff, min_weight = 0xffffffff; u8 tc, sp_tc_map = 0, wfq_tc_map = 0; - u32 min_weight = 0xffffffff; - u32 tc_weight_addr_diff = - PRS_REG_ETS_ARB_CREDIT_WEIGHT_1 - PRS_REG_ETS_ARB_CREDIT_WEIGHT_0; - u32 tc_bound_addr_diff = - PRS_REG_ETS_ARB_CREDIT_UPPER_BOUND_1 - - PRS_REG_ETS_ARB_CREDIT_UPPER_BOUND_0; + + tc_weight_addr_diff = PRS_REG_ETS_ARB_CREDIT_WEIGHT_1 - + PRS_REG_ETS_ARB_CREDIT_WEIGHT_0; + tc_bound_addr_diff = PRS_REG_ETS_ARB_CREDIT_UPPER_BOUND_1 - + PRS_REG_ETS_ARB_CREDIT_UPPER_BOUND_0; + for (tc = 0; tc < NUM_OF_TCS; tc++) { struct init_ets_tc_req *tc_req = &req->tc_req[tc]; - /* update SP map */ + + /* Update SP map */ if (tc_req->use_sp) sp_tc_map |= (1 << tc); - if (tc_req->use_wfq) { - /* update WFQ map */ - wfq_tc_map |= (1 << tc); - /* find minimal weight */ - if (tc_req->weight < min_weight) - min_weight = tc_req->weight; - } + + if (!tc_req->use_wfq) + continue; + + /* Update WFQ map */ + wfq_tc_map |= (1 << tc); + + /* Find minimal weight */ + if (tc_req->weight < min_weight) + min_weight = tc_req->weight; } + /* write SP map */ ecore_wr(p_hwfn, p_ptt, PRS_REG_ETS_ARB_CLIENT_IS_STRICT, sp_tc_map); + /* write WFQ map */ ecore_wr(p_hwfn, p_ptt, PRS_REG_ETS_ARB_CLIENT_IS_SUBJECT2WFQ, wfq_tc_map); + /* write WFQ weights */ for (tc = 0; tc < NUM_OF_TCS; tc++) { struct init_ets_tc_req *tc_req = &req->tc_req[tc]; - if (tc_req->use_wfq) { - /* translate weight to bytes */ - u32 byte_weight = - (PRS_ETS_MIN_WFQ_BYTES * tc_req->weight) / - min_weight; - /* write WFQ weight */ - ecore_wr(p_hwfn, p_ptt, - PRS_REG_ETS_ARB_CREDIT_WEIGHT_0 + - tc * tc_weight_addr_diff, byte_weight); - /* write WFQ upper bound */ - ecore_wr(p_hwfn, p_ptt, - PRS_REG_ETS_ARB_CREDIT_UPPER_BOUND_0 + - tc * tc_bound_addr_diff, - PRS_ETS_UP_BOUND(byte_weight, req->mtu)); - } + u32 byte_weight; + + if (!tc_req->use_wfq) + continue; + + /* Translate weight to bytes */ + byte_weight = (PRS_ETS_MIN_WFQ_BYTES * tc_req->weight) / + min_weight; + + /* Write WFQ weight */ + ecore_wr(p_hwfn, p_ptt, PRS_REG_ETS_ARB_CREDIT_WEIGHT_0 + tc * + tc_weight_addr_diff, byte_weight); + + /* Write WFQ upper bound */ + ecore_wr(p_hwfn, p_ptt, PRS_REG_ETS_ARB_CREDIT_UPPER_BOUND_0 + + tc * tc_bound_addr_diff, PRS_ETS_UP_BOUND(byte_weight, + req->mtu)); } } + /* BRB: RAM configuration constants */ #define BRB_TOTAL_RAM_BLOCKS_BB 4800 #define BRB_TOTAL_RAM_BLOCKS_K2 5632 -#define BRB_BLOCK_SIZE 128 /* in bytes */ +#define BRB_BLOCK_SIZE 128 #define BRB_MIN_BLOCKS_PER_TC 9 -#define BRB_HYST_BYTES 10240 -#define BRB_HYST_BLOCKS (BRB_HYST_BYTES / BRB_BLOCK_SIZE) -/* - * temporary big RAM allocation - should be updated - */ +#define BRB_HYST_BYTES 10240 +#define BRB_HYST_BLOCKS (BRB_HYST_BYTES / BRB_BLOCK_SIZE) + +/* Temporary big RAM allocation - should be updated */ void ecore_init_brb_ram(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, struct init_brb_ram_req *req) { - u8 port, active_ports = 0; + u32 tc_headroom_blocks, min_pkt_size_blocks, total_blocks; u32 active_port_blocks, reg_offset = 0; - u32 tc_headroom_blocks = - (u32)DIV_ROUND_UP(req->headroom_per_tc, BRB_BLOCK_SIZE); - u32 min_pkt_size_blocks = - (u32)DIV_ROUND_UP(req->min_pkt_size, BRB_BLOCK_SIZE); - u32 total_blocks = - ECORE_IS_K2(p_hwfn-> - p_dev) ? BRB_TOTAL_RAM_BLOCKS_K2 : - BRB_TOTAL_RAM_BLOCKS_BB; - /* find number of active ports */ + u8 port, active_ports = 0; + + tc_headroom_blocks = (u32)DIV_ROUND_UP(req->headroom_per_tc, + BRB_BLOCK_SIZE); + min_pkt_size_blocks = (u32)DIV_ROUND_UP(req->min_pkt_size, + BRB_BLOCK_SIZE); + total_blocks = ECORE_IS_K2(p_hwfn->p_dev) ? BRB_TOTAL_RAM_BLOCKS_K2 : + BRB_TOTAL_RAM_BLOCKS_BB; + + /* Find number of active ports */ for (port = 0; port < MAX_NUM_PORTS; port++) if (req->num_active_tcs[port]) active_ports++; + active_port_blocks = (u32)(total_blocks / active_ports); + for (port = 0; port < req->max_ports_per_engine; port++) { - /* calculate per-port sizes */ - u32 tc_guaranteed_blocks = - (u32)DIV_ROUND_UP(req->guranteed_per_tc, BRB_BLOCK_SIZE); - u32 port_blocks = - req->num_active_tcs[port] ? active_port_blocks : 0; - u32 port_guaranteed_blocks = - req->num_active_tcs[port] * tc_guaranteed_blocks; - u32 port_shared_blocks = port_blocks - port_guaranteed_blocks; - u32 full_xoff_th = - req->num_active_tcs[port] * BRB_MIN_BLOCKS_PER_TC; - u32 full_xon_th = full_xoff_th + min_pkt_size_blocks; - u32 pause_xoff_th = tc_headroom_blocks; - u32 pause_xon_th = pause_xoff_th + min_pkt_size_blocks; + u32 port_blocks, port_shared_blocks, port_guaranteed_blocks; + u32 full_xoff_th, full_xon_th, pause_xoff_th, pause_xon_th; + u32 tc_guaranteed_blocks; u8 tc; - /* init total size per port */ + + /* Calculate per-port sizes */ + tc_guaranteed_blocks = (u32)DIV_ROUND_UP(req->guranteed_per_tc, + BRB_BLOCK_SIZE); + port_blocks = req->num_active_tcs[port] ? active_port_blocks : + 0; + port_guaranteed_blocks = req->num_active_tcs[port] * + tc_guaranteed_blocks; + port_shared_blocks = port_blocks - port_guaranteed_blocks; + full_xoff_th = req->num_active_tcs[port] * + BRB_MIN_BLOCKS_PER_TC; + full_xon_th = full_xoff_th + min_pkt_size_blocks; + pause_xoff_th = tc_headroom_blocks; + pause_xon_th = pause_xoff_th + min_pkt_size_blocks; + + /* Init total size per port */ ecore_wr(p_hwfn, p_ptt, BRB_REG_TOTAL_MAC_SIZE + port * 4, port_blocks); - /* init shared size per port */ + + /* Init shared size per port */ ecore_wr(p_hwfn, p_ptt, BRB_REG_SHARED_HR_AREA + port * 4, port_shared_blocks); + for (tc = 0; tc < NUM_OF_TCS; tc++, reg_offset += 4) { - /* clear init values for non-active TCs */ + /* Clear init values for non-active TCs */ if (tc == req->num_active_tcs[port]) { tc_guaranteed_blocks = 0; full_xoff_th = 0; @@ -1091,13 +1366,18 @@ void ecore_init_brb_ram(struct ecore_hwfn *p_hwfn, pause_xoff_th = 0; pause_xon_th = 0; } - /* init guaranteed size per TC */ + + /* Init guaranteed size per TC */ ecore_wr(p_hwfn, p_ptt, BRB_REG_TC_GUARANTIED_0 + reg_offset, tc_guaranteed_blocks); ecore_wr(p_hwfn, p_ptt, BRB_REG_MAIN_TC_GUARANTIED_HYST_0 + reg_offset, BRB_HYST_BLOCKS); + + /* Init pause/full thresholds per physical TC - for + * loopback traffic. + */ ecore_wr(p_hwfn, p_ptt, BRB_REG_LB_TC_FULL_XOFF_THRESHOLD_0 + reg_offset, full_xoff_th); @@ -1110,6 +1390,10 @@ void ecore_init_brb_ram(struct ecore_hwfn *p_hwfn, ecore_wr(p_hwfn, p_ptt, BRB_REG_LB_TC_PAUSE_XON_THRESHOLD_0 + reg_offset, pause_xon_th); + + /* Init pause/full thresholds per physical TC - for + * main traffic. + */ ecore_wr(p_hwfn, p_ptt, BRB_REG_MAIN_TC_FULL_XOFF_THRESHOLD_0 + reg_offset, full_xoff_th); @@ -1126,24 +1410,11 @@ void ecore_init_brb_ram(struct ecore_hwfn *p_hwfn, } } -/*In MF should be called once per engine to set EtherType of OuterTag*/ -void ecore_set_engine_mf_ovlan_eth_type(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, u32 eth_type) +/* In MF should be called once per port to set EtherType of OuterTag */ +void ecore_set_port_mf_ovlan_eth_type(struct ecore_hwfn *p_hwfn, u32 ethType) { - /* update PRS register */ - STORE_RT_REG(p_hwfn, PRS_REG_TAG_ETHERTYPE_0_RT_OFFSET, eth_type); - /* update NIG register */ - STORE_RT_REG(p_hwfn, NIG_REG_TAG_ETHERTYPE_0_RT_OFFSET, eth_type); - /* update PBF register */ - STORE_RT_REG(p_hwfn, PBF_REG_TAG_ETHERTYPE_0_RT_OFFSET, eth_type); -} - -/*In MF should be called once per port to set EtherType of OuterTag*/ -void ecore_set_port_mf_ovlan_eth_type(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, u32 eth_type) -{ - /* update DORQ register */ - STORE_RT_REG(p_hwfn, DORQ_REG_TAG1_ETHERTYPE_RT_OFFSET, eth_type); + /* Update DORQ register */ + STORE_RT_REG(p_hwfn, DORQ_REG_TAG1_ETHERTYPE_RT_OFFSET, ethType); } #define SET_TUNNEL_TYPE_ENABLE_BIT(var, offset, enable) \ @@ -1152,11 +1423,13 @@ void ecore_set_port_mf_ovlan_eth_type(struct ecore_hwfn *p_hwfn, void ecore_set_vxlan_dest_port(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, u16 dest_port) { - /* update PRS register */ + /* Update PRS register */ ecore_wr(p_hwfn, p_ptt, PRS_REG_VXLAN_PORT, dest_port); - /* update NIG register */ - ecore_wr(p_hwfn, p_ptt, NIG_REG_VXLAN_PORT, dest_port); - /* update PBF register */ + + /* Update NIG register */ + ecore_wr(p_hwfn, p_ptt, NIG_REG_VXLAN_CTRL, dest_port); + + /* Update PBF register */ ecore_wr(p_hwfn, p_ptt, PBF_REG_VXLAN_PORT, dest_port); } @@ -1164,23 +1437,26 @@ void ecore_set_vxlan_enable(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, bool vxlan_enable) { u32 reg_val; - /* update PRS register */ + + /* Update PRS register */ reg_val = ecore_rd(p_hwfn, p_ptt, PRS_REG_ENCAPSULATION_TYPE_EN); SET_TUNNEL_TYPE_ENABLE_BIT(reg_val, PRS_REG_ENCAPSULATION_TYPE_EN_VXLAN_ENABLE_SHIFT, vxlan_enable); ecore_wr(p_hwfn, p_ptt, PRS_REG_ENCAPSULATION_TYPE_EN, reg_val); if (reg_val) { - ecore_wr(p_hwfn, p_ptt, PRS_REG_OUTPUT_FORMAT_4_0, - PRS_ETH_TUNN_FIC_FORMAT); + ecore_wr(p_hwfn, p_ptt, PRS_REG_OUTPUT_FORMAT_4_0_BB_K2, + (u32)PRS_ETH_TUNN_FIC_FORMAT); } - /* update NIG register */ + + /* Update NIG register */ reg_val = ecore_rd(p_hwfn, p_ptt, NIG_REG_ENC_TYPE_ENABLE); SET_TUNNEL_TYPE_ENABLE_BIT(reg_val, NIG_REG_ENC_TYPE_ENABLE_VXLAN_ENABLE_SHIFT, vxlan_enable); ecore_wr(p_hwfn, p_ptt, NIG_REG_ENC_TYPE_ENABLE, reg_val); - /* update DORQ register */ + + /* Update DORQ register */ ecore_wr(p_hwfn, p_ptt, DORQ_REG_L2_EDPM_TUNNEL_VXLAN_EN, vxlan_enable ? 1 : 0); } @@ -1190,7 +1466,8 @@ void ecore_set_gre_enable(struct ecore_hwfn *p_hwfn, bool eth_gre_enable, bool ip_gre_enable) { u32 reg_val; - /* update PRS register */ + + /* Update PRS register */ reg_val = ecore_rd(p_hwfn, p_ptt, PRS_REG_ENCAPSULATION_TYPE_EN); SET_TUNNEL_TYPE_ENABLE_BIT(reg_val, PRS_REG_ENCAPSULATION_TYPE_EN_ETH_OVER_GRE_ENABLE_SHIFT, @@ -1200,10 +1477,11 @@ void ecore_set_gre_enable(struct ecore_hwfn *p_hwfn, ip_gre_enable); ecore_wr(p_hwfn, p_ptt, PRS_REG_ENCAPSULATION_TYPE_EN, reg_val); if (reg_val) { - ecore_wr(p_hwfn, p_ptt, PRS_REG_OUTPUT_FORMAT_4_0, - PRS_ETH_TUNN_FIC_FORMAT); + ecore_wr(p_hwfn, p_ptt, PRS_REG_OUTPUT_FORMAT_4_0_BB_K2, + (u32)PRS_ETH_TUNN_FIC_FORMAT); } - /* update NIG register */ + + /* Update NIG register */ reg_val = ecore_rd(p_hwfn, p_ptt, NIG_REG_ENC_TYPE_ENABLE); SET_TUNNEL_TYPE_ENABLE_BIT(reg_val, NIG_REG_ENC_TYPE_ENABLE_ETH_OVER_GRE_ENABLE_SHIFT, @@ -1212,7 +1490,8 @@ void ecore_set_gre_enable(struct ecore_hwfn *p_hwfn, NIG_REG_ENC_TYPE_ENABLE_IP_OVER_GRE_ENABLE_SHIFT, ip_gre_enable); ecore_wr(p_hwfn, p_ptt, NIG_REG_ENC_TYPE_ENABLE, reg_val); - /* update DORQ registers */ + + /* Update DORQ registers */ ecore_wr(p_hwfn, p_ptt, DORQ_REG_L2_EDPM_TUNNEL_GRE_ETH_EN, eth_gre_enable ? 1 : 0); ecore_wr(p_hwfn, p_ptt, DORQ_REG_L2_EDPM_TUNNEL_GRE_IP_EN, @@ -1222,14 +1501,13 @@ void ecore_set_gre_enable(struct ecore_hwfn *p_hwfn, void ecore_set_geneve_dest_port(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, u16 dest_port) { - /* geneve tunnel not supported in BB_A0 */ - if (ECORE_IS_BB_A0(p_hwfn->p_dev)) - return; - /* update PRS register */ + /* Update PRS register */ ecore_wr(p_hwfn, p_ptt, PRS_REG_NGE_PORT, dest_port); - /* update NIG register */ + + /* Update NIG register */ ecore_wr(p_hwfn, p_ptt, NIG_REG_NGE_PORT, dest_port); - /* update PBF register */ + + /* Update PBF register */ ecore_wr(p_hwfn, p_ptt, PBF_REG_NGE_PORT, dest_port); } @@ -1238,10 +1516,8 @@ void ecore_set_geneve_enable(struct ecore_hwfn *p_hwfn, bool eth_geneve_enable, bool ip_geneve_enable) { u32 reg_val; - /* geneve tunnel not supported in BB_A0 */ - if (ECORE_IS_BB_A0(p_hwfn->p_dev)) - return; - /* update PRS register */ + + /* Update PRS register */ reg_val = ecore_rd(p_hwfn, p_ptt, PRS_REG_ENCAPSULATION_TYPE_EN); SET_TUNNEL_TYPE_ENABLE_BIT(reg_val, PRS_REG_ENCAPSULATION_TYPE_EN_ETH_OVER_GENEVE_ENABLE_SHIFT, @@ -1251,25 +1527,397 @@ void ecore_set_geneve_enable(struct ecore_hwfn *p_hwfn, ip_geneve_enable); ecore_wr(p_hwfn, p_ptt, PRS_REG_ENCAPSULATION_TYPE_EN, reg_val); if (reg_val) { - ecore_wr(p_hwfn, p_ptt, PRS_REG_OUTPUT_FORMAT_4_0, - PRS_ETH_TUNN_FIC_FORMAT); + ecore_wr(p_hwfn, p_ptt, PRS_REG_OUTPUT_FORMAT_4_0_BB_K2, + (u32)PRS_ETH_TUNN_FIC_FORMAT); } - /* update NIG register */ + + /* Update NIG register */ ecore_wr(p_hwfn, p_ptt, NIG_REG_NGE_ETH_ENABLE, eth_geneve_enable ? 1 : 0); ecore_wr(p_hwfn, p_ptt, NIG_REG_NGE_IP_ENABLE, ip_geneve_enable ? 1 : 0); - /* comp ver */ - reg_val = (ip_geneve_enable || eth_geneve_enable) ? 1 : 0; - ecore_wr(p_hwfn, p_ptt, NIG_REG_NGE_COMP_VER, reg_val); - ecore_wr(p_hwfn, p_ptt, PBF_REG_NGE_COMP_VER, reg_val); - ecore_wr(p_hwfn, p_ptt, PRS_REG_NGE_COMP_VER, reg_val); - /* EDPM with geneve tunnel not supported in BB_B0 */ + + /* EDPM with geneve tunnel not supported in BB */ if (ECORE_IS_BB_B0(p_hwfn->p_dev)) return; - /* update DORQ registers */ - ecore_wr(p_hwfn, p_ptt, DORQ_REG_L2_EDPM_TUNNEL_NGE_ETH_EN, + + /* Update DORQ registers */ + ecore_wr(p_hwfn, p_ptt, DORQ_REG_L2_EDPM_TUNNEL_NGE_ETH_EN_K2_E5, eth_geneve_enable ? 1 : 0); - ecore_wr(p_hwfn, p_ptt, DORQ_REG_L2_EDPM_TUNNEL_NGE_IP_EN, + ecore_wr(p_hwfn, p_ptt, DORQ_REG_L2_EDPM_TUNNEL_NGE_IP_EN_K2_E5, ip_geneve_enable ? 1 : 0); } + + +#define T_ETH_PACKET_ACTION_GFT_EVENTID 23 +#define PARSER_ETH_CONN_GFT_ACTION_CM_HDR 272 +#define T_ETH_PACKET_MATCH_RFS_EVENTID 25 +#define PARSER_ETH_CONN_CM_HDR 0 +#define CAM_LINE_SIZE sizeof(u32) +#define RAM_LINE_SIZE sizeof(u64) +#define REG_SIZE sizeof(u32) + +void ecore_gft_disable(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + u16 pf_id) +{ + /* disable gft search for PF */ + ecore_wr(p_hwfn, p_ptt, PRS_REG_SEARCH_GFT, 0); + + /* Clean ram & cam for next gft session*/ + + /* Zero camline */ + ecore_wr(p_hwfn, p_ptt, PRS_REG_GFT_CAM + CAM_LINE_SIZE * pf_id, 0); + + /* Zero ramline */ + ecore_wr(p_hwfn, p_ptt, PRS_REG_GFT_PROFILE_MASK_RAM + + RAM_LINE_SIZE * pf_id, 0); + ecore_wr(p_hwfn, p_ptt, PRS_REG_GFT_PROFILE_MASK_RAM + + RAM_LINE_SIZE * pf_id + REG_SIZE, 0); +} + + +void ecore_set_gft_event_id_cm_hdr(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt) +{ + u32 rfs_cm_hdr_event_id; + + /* Set RFS event ID to be awakened i Tstorm By Prs */ + rfs_cm_hdr_event_id = ecore_rd(p_hwfn, p_ptt, PRS_REG_CM_HDR_GFT); + rfs_cm_hdr_event_id |= T_ETH_PACKET_ACTION_GFT_EVENTID << + PRS_REG_CM_HDR_GFT_EVENT_ID_SHIFT; + rfs_cm_hdr_event_id |= PARSER_ETH_CONN_GFT_ACTION_CM_HDR << + PRS_REG_CM_HDR_GFT_CM_HDR_SHIFT; + ecore_wr(p_hwfn, p_ptt, PRS_REG_CM_HDR_GFT, rfs_cm_hdr_event_id); +} + +void ecore_gft_config(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + u16 pf_id, + bool tcp, + bool udp, + bool ipv4, + bool ipv6, + enum gft_profile_type profile_type) +{ + u32 reg_val, cam_line, ram_line_lo, ram_line_hi; + + if (!ipv6 && !ipv4) + DP_NOTICE(p_hwfn, true, "gft_config: must accept at least on of - ipv4 or ipv6'\n"); + if (!tcp && !udp) + DP_NOTICE(p_hwfn, true, "gft_config: must accept at least on of - udp or tcp\n"); + if (profile_type >= MAX_GFT_PROFILE_TYPE) + DP_NOTICE(p_hwfn, true, "gft_config: unsupported gft_profile_type\n"); + + /* Set RFS event ID to be awakened i Tstorm By Prs */ + reg_val = T_ETH_PACKET_MATCH_RFS_EVENTID << + PRS_REG_CM_HDR_GFT_EVENT_ID_SHIFT; + reg_val |= PARSER_ETH_CONN_CM_HDR << PRS_REG_CM_HDR_GFT_CM_HDR_SHIFT; + ecore_wr(p_hwfn, p_ptt, PRS_REG_CM_HDR_GFT, reg_val); + + /* Do not load context only cid in PRS on match. */ + ecore_wr(p_hwfn, p_ptt, PRS_REG_LOAD_L2_FILTER, 0); + + /* Do not use tenant ID exist bit for gft search*/ + ecore_wr(p_hwfn, p_ptt, PRS_REG_SEARCH_TENANT_ID, 0); + + /* Set Cam */ + cam_line = 0; + SET_FIELD(cam_line, GFT_CAM_LINE_MAPPED_VALID, 1); + + /* Filters are per PF!! */ + SET_FIELD(cam_line, GFT_CAM_LINE_MAPPED_PF_ID_MASK, + GFT_CAM_LINE_MAPPED_PF_ID_MASK_MASK); + SET_FIELD(cam_line, GFT_CAM_LINE_MAPPED_PF_ID, pf_id); + + if (!(tcp && udp)) { + SET_FIELD(cam_line, + GFT_CAM_LINE_MAPPED_UPPER_PROTOCOL_TYPE_MASK, + GFT_CAM_LINE_MAPPED_UPPER_PROTOCOL_TYPE_MASK_MASK); + if (tcp) + SET_FIELD(cam_line, + GFT_CAM_LINE_MAPPED_UPPER_PROTOCOL_TYPE, + GFT_PROFILE_TCP_PROTOCOL); + else + SET_FIELD(cam_line, + GFT_CAM_LINE_MAPPED_UPPER_PROTOCOL_TYPE, + GFT_PROFILE_UDP_PROTOCOL); + } + + if (!(ipv4 && ipv6)) { + SET_FIELD(cam_line, GFT_CAM_LINE_MAPPED_IP_VERSION_MASK, 1); + if (ipv4) + SET_FIELD(cam_line, GFT_CAM_LINE_MAPPED_IP_VERSION, + GFT_PROFILE_IPV4); + else + SET_FIELD(cam_line, GFT_CAM_LINE_MAPPED_IP_VERSION, + GFT_PROFILE_IPV6); + } + + /* Write characteristics to cam */ + ecore_wr(p_hwfn, p_ptt, PRS_REG_GFT_CAM + CAM_LINE_SIZE * pf_id, + cam_line); + cam_line = ecore_rd(p_hwfn, p_ptt, + PRS_REG_GFT_CAM + CAM_LINE_SIZE * pf_id); + + /* Write line to RAM - compare to filter 4 tuple */ + ram_line_lo = 0; + ram_line_hi = 0; + + if (profile_type == GFT_PROFILE_TYPE_4_TUPLE) { + SET_FIELD(ram_line_hi, GFT_RAM_LINE_DST_IP, 1); + SET_FIELD(ram_line_hi, GFT_RAM_LINE_SRC_IP, 1); + SET_FIELD(ram_line_hi, GFT_RAM_LINE_OVER_IP_PROTOCOL, 1); + SET_FIELD(ram_line_lo, GFT_RAM_LINE_ETHERTYPE, 1); + SET_FIELD(ram_line_lo, GFT_RAM_LINE_SRC_PORT, 1); + SET_FIELD(ram_line_lo, GFT_RAM_LINE_DST_PORT, 1); + } else if (profile_type == GFT_PROFILE_TYPE_L4_DST_PORT) { + SET_FIELD(ram_line_hi, GFT_RAM_LINE_OVER_IP_PROTOCOL, 1); + SET_FIELD(ram_line_lo, GFT_RAM_LINE_ETHERTYPE, 1); + SET_FIELD(ram_line_lo, GFT_RAM_LINE_DST_PORT, 1); + } else if (profile_type == GFT_PROFILE_TYPE_IP_DST_PORT) { + SET_FIELD(ram_line_hi, GFT_RAM_LINE_DST_IP, 1); + SET_FIELD(ram_line_lo, GFT_RAM_LINE_ETHERTYPE, 1); + } + + ecore_wr(p_hwfn, p_ptt, + PRS_REG_GFT_PROFILE_MASK_RAM + RAM_LINE_SIZE * pf_id, + ram_line_lo); + ecore_wr(p_hwfn, p_ptt, + PRS_REG_GFT_PROFILE_MASK_RAM + RAM_LINE_SIZE * pf_id + + REG_SIZE, ram_line_hi); + + /* Set default profile so that no filter match will happen */ + ecore_wr(p_hwfn, p_ptt, PRS_REG_GFT_PROFILE_MASK_RAM + RAM_LINE_SIZE * + PRS_GFT_CAM_LINES_NO_MATCH, 0xffffffff); + ecore_wr(p_hwfn, p_ptt, PRS_REG_GFT_PROFILE_MASK_RAM + RAM_LINE_SIZE * + PRS_GFT_CAM_LINES_NO_MATCH + REG_SIZE, 0x3ff); + + /* Enable gft search */ + ecore_wr(p_hwfn, p_ptt, PRS_REG_SEARCH_GFT, 1); +} + +/* Configure VF zone size mode */ +void ecore_config_vf_zone_size_mode(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, u16 mode, + bool runtime_init) +{ + u32 msdm_vf_size_log = MSTORM_VF_ZONE_DEFAULT_SIZE_LOG; + u32 msdm_vf_offset_mask; + + if (mode == VF_ZONE_SIZE_MODE_DOUBLE) + msdm_vf_size_log += 1; + else if (mode == VF_ZONE_SIZE_MODE_QUAD) + msdm_vf_size_log += 2; + + msdm_vf_offset_mask = (1 << msdm_vf_size_log) - 1; + + if (runtime_init) { + STORE_RT_REG(p_hwfn, + PGLUE_REG_B_MSDM_VF_SHIFT_B_RT_OFFSET, + msdm_vf_size_log); + STORE_RT_REG(p_hwfn, + PGLUE_REG_B_MSDM_OFFSET_MASK_B_RT_OFFSET, + msdm_vf_offset_mask); + } else { + ecore_wr(p_hwfn, p_ptt, + PGLUE_B_REG_MSDM_VF_SHIFT_B, msdm_vf_size_log); + ecore_wr(p_hwfn, p_ptt, + PGLUE_B_REG_MSDM_OFFSET_MASK_B, msdm_vf_offset_mask); + } +} + +/* Get mstorm statistics for offset by VF zone size mode */ +u32 ecore_get_mstorm_queue_stat_offset(struct ecore_hwfn *p_hwfn, + u16 stat_cnt_id, + u16 vf_zone_size_mode) +{ + u32 offset = MSTORM_QUEUE_STAT_OFFSET(stat_cnt_id); + + if ((vf_zone_size_mode != VF_ZONE_SIZE_MODE_DEFAULT) && + (stat_cnt_id > MAX_NUM_PFS)) { + if (vf_zone_size_mode == VF_ZONE_SIZE_MODE_DOUBLE) + offset += (1 << MSTORM_VF_ZONE_DEFAULT_SIZE_LOG) * + (stat_cnt_id - MAX_NUM_PFS); + else if (vf_zone_size_mode == VF_ZONE_SIZE_MODE_QUAD) + offset += 3 * (1 << MSTORM_VF_ZONE_DEFAULT_SIZE_LOG) * + (stat_cnt_id - MAX_NUM_PFS); + } + + return offset; +} + +/* Get mstorm VF producer offset by VF zone size mode */ +u32 ecore_get_mstorm_eth_vf_prods_offset(struct ecore_hwfn *p_hwfn, + u8 vf_id, + u8 vf_queue_id, + u16 vf_zone_size_mode) +{ + u32 offset = MSTORM_ETH_VF_PRODS_OFFSET(vf_id, vf_queue_id); + + if (vf_zone_size_mode != VF_ZONE_SIZE_MODE_DEFAULT) { + if (vf_zone_size_mode == VF_ZONE_SIZE_MODE_DOUBLE) + offset += (1 << MSTORM_VF_ZONE_DEFAULT_SIZE_LOG) * + vf_id; + else if (vf_zone_size_mode == VF_ZONE_SIZE_MODE_QUAD) + offset += 3 * (1 << MSTORM_VF_ZONE_DEFAULT_SIZE_LOG) * + vf_id; + } + + return offset; +} + +#ifndef LINUX_REMOVE +#define CRC8_INIT_VALUE 0xFF +#endif +static u8 cdu_crc8_table[CRC8_TABLE_SIZE]; + +/* Calculate and return CDU validation byte per connection type / region / + * cid + */ +static u8 ecore_calc_cdu_validation_byte(u8 conn_type, u8 region, u32 cid) +{ + const u8 validation_cfg = CDU_VALIDATION_DEFAULT_CFG; + + static u8 crc8_table_valid; /*automatically initialized to 0*/ + u8 crc, validation_byte = 0; + u32 validation_string = 0; + u32 data_to_crc; + + if (crc8_table_valid == 0) { + OSAL_CRC8_POPULATE(cdu_crc8_table, 0x07); + crc8_table_valid = 1; + } + + /* + * The CRC is calculated on the String-to-compress: + * [31:8] = {CID[31:20],CID[11:0]} + * [7:4] = Region + * [3:0] = Type + */ + if ((validation_cfg >> CDU_CONTEXT_VALIDATION_CFG_USE_CID) & 1) + validation_string |= (cid & 0xFFF00000) | ((cid & 0xFFF) << 8); + + if ((validation_cfg >> CDU_CONTEXT_VALIDATION_CFG_USE_REGION) & 1) + validation_string |= ((region & 0xF) << 4); + + if ((validation_cfg >> CDU_CONTEXT_VALIDATION_CFG_USE_TYPE) & 1) + validation_string |= (conn_type & 0xF); + + /* Convert to big-endian and calculate CRC8*/ + data_to_crc = OSAL_BE32_TO_CPU(validation_string); + + crc = OSAL_CRC8(cdu_crc8_table, (u8 *)&data_to_crc, sizeof(data_to_crc), + CRC8_INIT_VALUE); + + /* The validation byte [7:0] is composed: + * for type A validation + * [7] = active configuration bit + * [6:0] = crc[6:0] + * + * for type B validation + * [7] = active configuration bit + * [6:3] = connection_type[3:0] + * [2:0] = crc[2:0] + */ + + validation_byte |= ((validation_cfg >> + CDU_CONTEXT_VALIDATION_CFG_USE_ACTIVE) & 1) << 7; + + if ((validation_cfg >> + CDU_CONTEXT_VALIDATION_CFG_VALIDATION_TYPE_SHIFT) & 1) + validation_byte |= ((conn_type & 0xF) << 3) | (crc & 0x7); + else + validation_byte |= crc & 0x7F; + + return validation_byte; +} + +/* Calcualte and set validation bytes for session context */ +void ecore_calc_session_ctx_validation(void *p_ctx_mem, u16 ctx_size, + u8 ctx_type, u32 cid) +{ + u8 *x_val_ptr, *t_val_ptr, *u_val_ptr, *p_ctx; + + p_ctx = (u8 *)p_ctx_mem; + x_val_ptr = &p_ctx[con_region_offsets[0][ctx_type]]; + t_val_ptr = &p_ctx[con_region_offsets[1][ctx_type]]; + u_val_ptr = &p_ctx[con_region_offsets[2][ctx_type]]; + + OSAL_MEMSET(p_ctx, 0, ctx_size); + + *x_val_ptr = ecore_calc_cdu_validation_byte(ctx_type, 3, cid); + *t_val_ptr = ecore_calc_cdu_validation_byte(ctx_type, 4, cid); + *u_val_ptr = ecore_calc_cdu_validation_byte(ctx_type, 5, cid); +} + +/* Calcualte and set validation bytes for task context */ +void ecore_calc_task_ctx_validation(void *p_ctx_mem, u16 ctx_size, u8 ctx_type, + u32 tid) +{ + u8 *p_ctx, *region1_val_ptr; + + p_ctx = (u8 *)p_ctx_mem; + region1_val_ptr = &p_ctx[task_region_offsets[0][ctx_type]]; + + OSAL_MEMSET(p_ctx, 0, ctx_size); + + *region1_val_ptr = ecore_calc_cdu_validation_byte(ctx_type, 1, tid); +} + +/* Memset session context to 0 while preserving validation bytes */ +void ecore_memset_session_ctx(void *p_ctx_mem, u32 ctx_size, u8 ctx_type) +{ + u8 *x_val_ptr, *t_val_ptr, *u_val_ptr, *p_ctx; + u8 x_val, t_val, u_val; + + p_ctx = (u8 *)p_ctx_mem; + x_val_ptr = &p_ctx[con_region_offsets[0][ctx_type]]; + t_val_ptr = &p_ctx[con_region_offsets[1][ctx_type]]; + u_val_ptr = &p_ctx[con_region_offsets[2][ctx_type]]; + + x_val = *x_val_ptr; + t_val = *t_val_ptr; + u_val = *u_val_ptr; + + OSAL_MEMSET(p_ctx, 0, ctx_size); + + *x_val_ptr = x_val; + *t_val_ptr = t_val; + *u_val_ptr = u_val; +} + +/* Memset task context to 0 while preserving validation bytes */ +void ecore_memset_task_ctx(void *p_ctx_mem, u32 ctx_size, u8 ctx_type) +{ + u8 *p_ctx, *region1_val_ptr; + u8 region1_val; + + p_ctx = (u8 *)p_ctx_mem; + region1_val_ptr = &p_ctx[task_region_offsets[0][ctx_type]]; + + region1_val = *region1_val_ptr; + + OSAL_MEMSET(p_ctx, 0, ctx_size); + + *region1_val_ptr = region1_val; +} + +/* Enable and configure context validation */ +void ecore_enable_context_validation(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt) +{ + u32 ctx_validation; + + /* Enable validation for connection region 3 - bits [31:24] */ + ctx_validation = CDU_VALIDATION_DEFAULT_CFG << 24; + ecore_wr(p_hwfn, p_ptt, CDU_REG_CCFC_CTX_VALID0, ctx_validation); + + /* Enable validation for connection region 5 - bits [15: 8] */ + ctx_validation = CDU_VALIDATION_DEFAULT_CFG << 8; + ecore_wr(p_hwfn, p_ptt, CDU_REG_CCFC_CTX_VALID1, ctx_validation); + + /* Enable validation for connection region 1 - bits [15: 8] */ + ctx_validation = CDU_VALIDATION_DEFAULT_CFG << 8; + ecore_wr(p_hwfn, p_ptt, CDU_REG_TCFC_CTX_VALID0, ctx_validation); +} diff --git a/dpdk/drivers/net/qede/base/ecore_init_fw_funcs.h b/dpdk/drivers/net/qede/base/ecore_init_fw_funcs.h index 5280cd7a..ab560e59 100644 --- a/dpdk/drivers/net/qede/base/ecore_init_fw_funcs.h +++ b/dpdk/drivers/net/qede/base/ecore_init_fw_funcs.h @@ -8,58 +8,88 @@ #ifndef _INIT_FW_FUNCS_H #define _INIT_FW_FUNCS_H -/* forward declarations */ +/* Forward declarations */ + struct init_qm_pq_params; + /** - * @brief ecore_qm_pf_mem_size - prepare QM ILT sizes + * @brief ecore_qm_pf_mem_size - Prepare QM ILT sizes * * Returns the required host memory size in 4KB units. * Must be called before all QM init HSI functions. * - * @param pf_id - physical function ID - * @param num_pf_cids - number of connections used by this PF - * @param num_vf_cids - number of connections used by VFs of this PF - * @param num_tids - number of tasks used by this PF - * @param num_pf_pqs - number of PQs used by this PF - * @param num_vf_pqs - number of PQs used by VFs of this PF + * @param num_pf_cids - number of connections used by this PF + * @param num_vf_cids - number of connections used by VFs of this PF + * @param num_tids - number of tasks used by this PF + * @param num_pf_pqs - number of PQs used by this PF + * @param num_vf_pqs - number of PQs used by VFs of this PF * * @return The required host memory size in 4KB units. */ -u32 ecore_qm_pf_mem_size(u8 pf_id, - u32 num_pf_cids, - u32 num_vf_cids, - u32 num_tids, u16 num_pf_pqs, u16 num_vf_pqs); +u32 ecore_qm_pf_mem_size(u32 num_pf_cids, + u32 num_vf_cids, + u32 num_tids, + u16 num_pf_pqs, + u16 num_vf_pqs); + /** - * @brief ecore_qm_common_rt_init - - * Prepare QM runtime init values for the engine phase + * @brief ecore_qm_common_rt_init - Prepare QM runtime init values for engine + * phase * * @param p_hwfn * @param max_ports_per_engine - max number of ports per engine in HW * @param max_phys_tcs_per_port - max number of physical TCs per port in HW - * @param pf_rl_en - enable per-PF rate limiters - * @param pf_wfq_en - enable per-PF WFQ - * @param vport_rl_en - enable per-VPORT rate limiters - * @param vport_wfq_en - enable per-VPORT WFQ - * @param port_params- array of size MAX_NUM_PORTS with parameters for each port + * @param pf_rl_en - enable per-PF rate limiters + * @param pf_wfq_en - enable per-PF WFQ + * @param vport_rl_en - enable per-VPORT rate limiters + * @param vport_wfq_en - enable per-VPORT WFQ + * @param port_params - array of size MAX_NUM_PORTS with params for each port * * @return 0 on success, -1 on error. */ int ecore_qm_common_rt_init(struct ecore_hwfn *p_hwfn, - u8 max_ports_per_engine, - u8 max_phys_tcs_per_port, - bool pf_rl_en, - bool pf_wfq_en, - bool vport_rl_en, - bool vport_wfq_en, - struct init_qm_port_params - port_params[MAX_NUM_PORTS]); + u8 max_ports_per_engine, + u8 max_phys_tcs_per_port, + bool pf_rl_en, + bool pf_wfq_en, + bool vport_rl_en, + bool vport_wfq_en, + struct init_qm_port_params port_params[MAX_NUM_PORTS]); +/** + * @brief ecore_qm_pf_rt_init Prepare QM runtime init values for the PF phase + * + * @param p_hwfn + * @param p_ptt - ptt window used for writing the registers + * @param port_id - port ID + * @param pf_id - PF ID + * @param max_phys_tcs_per_port - max number of physical TCs per port in HW + * @param num_pf_cids - number of connections used by this PF + * @param num_vf_cids - number of connections used by VFs of this PF + * @param num_tids - number of tasks used by this PF + * @param start_pq - first Tx PQ ID associated with this PF + * @param num_pf_pqs - number of Tx PQs associated with this PF + * (non-VF) + * @param num_vf_pqs - number of Tx PQs associated with a VF + * @param start_vport - first VPORT ID associated with this PF + * @param num_vports - number of VPORTs associated with this PF + * @param pf_wfq - WFQ weight. if PF WFQ is globally disabled, the weight must + * be 0. otherwise, the weight must be non-zero. + * @param pf_rl - rate limit in Mb/sec units. a value of 0 means don't + * configure. ignored if PF RL is globally disabled. + * @param link_speed - link speed in Mbps. + * @param pq_params - array of size (num_pf_pqs+num_vf_pqs) with parameters for + * each Tx PQ associated with the specified PF. + * @param vport_params - array of size num_vports with parameters for each + * associated VPORT. + * + * @return 0 on success, -1 on error. + */ int ecore_qm_pf_rt_init(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, u8 port_id, u8 pf_id, u8 max_phys_tcs_per_port, - bool is_first_pf, u32 num_pf_cids, u32 num_vf_cids, u32 num_tids, @@ -70,8 +100,10 @@ int ecore_qm_pf_rt_init(struct ecore_hwfn *p_hwfn, u8 num_vports, u16 pf_wfq, u32 pf_rl, + u32 link_speed, struct init_qm_pq_params *pq_params, struct init_qm_vport_params *vport_params); + /** * @brief ecore_init_pf_wfq Initializes the WFQ weight of the specified PF * @@ -83,21 +115,27 @@ int ecore_qm_pf_rt_init(struct ecore_hwfn *p_hwfn, * @return 0 on success, -1 on error. */ int ecore_init_pf_wfq(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, u8 pf_id, u16 pf_wfq); + struct ecore_ptt *p_ptt, + u8 pf_id, + u16 pf_wfq); + /** - * @brief ecore_init_pf_rl Initializes the rate limit of the specified PF + * @brief ecore_init_pf_rl - Initializes the rate limit of the specified PF * * @param p_hwfn - * @param p_ptt - ptt window used for writing the registers + * @param p_ptt - ptt window used for writing the registers * @param pf_id - PF ID * @param pf_rl - rate limit in Mb/sec units * * @return 0 on success, -1 on error. */ int ecore_init_pf_rl(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, u8 pf_id, u32 pf_rl); + struct ecore_ptt *p_ptt, + u8 pf_id, + u32 pf_rl); + /** - * @brief ecore_init_vport_wfq Initializes the WFQ weight of the specified VPORT + * @brief ecore_init_vport_wfq Initializes the WFQ weight of specified VPORT * * @param p_hwfn * @param p_ptt - ptt window used for writing the registers @@ -109,20 +147,28 @@ int ecore_init_pf_rl(struct ecore_hwfn *p_hwfn, * @return 0 on success, -1 on error. */ int ecore_init_vport_wfq(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, - u16 first_tx_pq_id[NUM_OF_TCS], u16 vport_wfq); + struct ecore_ptt *p_ptt, + u16 first_tx_pq_id[NUM_OF_TCS], + u16 vport_wfq); + /** - * @brief ecore_init_vport_rl Initializes the rate limit of the specified VPORT + * @brief ecore_init_vport_rl - Initializes the rate limit of the specified + * VPORT. * - * @param p_hwfn - * @param p_ptt - ptt window used for writing the registers - * @param vport_id - VPORT ID - * @param vport_rl - rate limit in Mb/sec units + * @param p_hwfn - HW device data + * @param p_ptt - ptt window used for writing the registers + * @param vport_id - VPORT ID + * @param vport_rl - rate limit in Mb/sec units + * @param link_speed - link speed in Mbps. * * @return 0 on success, -1 on error. */ int ecore_init_vport_rl(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, u8 vport_id, u32 vport_rl); + struct ecore_ptt *p_ptt, + u8 vport_id, + u32 vport_rl, + u32 link_speed); + /** * @brief ecore_send_qm_stop_cmd Sends a stop command to the QM * @@ -133,13 +179,17 @@ int ecore_init_vport_rl(struct ecore_hwfn *p_hwfn, * @param start_pq - first PQ ID to stop * @param num_pqs - Number of PQs to stop, starting from start_pq. * - * @return bool, true if successful, false if timeout occurred while - * waiting for QM command done. + * @return bool, true if successful, false if timeout occurred while waiting + * for QM command done. */ bool ecore_send_qm_stop_cmd(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, - bool is_release_cmd, - bool is_tx_pq, u16 start_pq, u16 num_pqs); + struct ecore_ptt *p_ptt, + bool is_release_cmd, + bool is_tx_pq, + u16 start_pq, + u16 num_pqs); +#ifndef UNUSED_HSI_FUNC + /** * @brief ecore_init_nig_ets - initializes the NIG ETS arbiter * @@ -152,8 +202,10 @@ bool ecore_send_qm_stop_cmd(struct ecore_hwfn *p_hwfn, * requirements are ignored when is_lb is cleared. */ void ecore_init_nig_ets(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, - struct init_ets_req *req, bool is_lb); + struct ecore_ptt *p_ptt, + struct init_ets_req *req, + bool is_lb); + /** * @brief ecore_init_nig_lb_rl - initializes the NIG LB RLs * @@ -163,8 +215,10 @@ void ecore_init_nig_ets(struct ecore_hwfn *p_hwfn, * @param req - the NIG LB RLs initialization requirements. */ void ecore_init_nig_lb_rl(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, - struct init_nig_lb_rl_req *req); + struct ecore_ptt *p_ptt, + struct init_nig_lb_rl_req *req); +#endif /* UNUSED_HSI_FUNC */ + /** * @brief ecore_init_nig_pri_tc_map - initializes the NIG priority to TC map. * @@ -174,8 +228,10 @@ void ecore_init_nig_lb_rl(struct ecore_hwfn *p_hwfn, * @param req - required mapping from prioirties to TCs. */ void ecore_init_nig_pri_tc_map(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, - struct init_nig_pri_tc_map_req *req); + struct ecore_ptt *p_ptt, + struct init_nig_pri_tc_map_req *req); + +#ifndef UNUSED_HSI_FUNC /** * @brief ecore_init_prs_ets - initializes the PRS Rx ETS arbiter * @@ -185,7 +241,11 @@ void ecore_init_nig_pri_tc_map(struct ecore_hwfn *p_hwfn, * @param req - the PRS ETS initialization requirements. */ void ecore_init_prs_ets(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, struct init_ets_req *req); + struct ecore_ptt *p_ptt, + struct init_ets_req *req); +#endif /* UNUSED_HSI_FUNC */ + +#ifndef UNUSED_HSI_FUNC /** * @brief ecore_init_brb_ram - initializes BRB RAM sizes per TC * @@ -195,43 +255,44 @@ void ecore_init_prs_ets(struct ecore_hwfn *p_hwfn, * @param req - the BRB RAM initialization requirements. */ void ecore_init_brb_ram(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, struct init_brb_ram_req *req); + struct ecore_ptt *p_ptt, + struct init_brb_ram_req *req); +#endif /* UNUSED_HSI_FUNC */ + +#ifndef UNUSED_HSI_FUNC /** - * @brief ecore_set_engine_mf_ovlan_eth_type - initializes Nig,Prs,Pbf - * and llh ethType Regs to input ethType - * should Be called once per engine if engine is in BD mode. + * @brief ecore_set_port_mf_ovlan_eth_type - initializes DORQ ethType Regs to + * input ethType should Be called + * once per port. * - * @param p_ptt - ptt window used for writing the registers. - * @param ethType - etherType to configure - */ -void ecore_set_engine_mf_ovlan_eth_type(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, u32 eth_type); -/** - * @brief ecore_set_port_mf_ovlan_eth_type - initializes DORQ ethType Regs - * to input ethType - * should Be called once per port. - * - * @param p_ptt - ptt window used for writing the registers. + * @param p_hwfn - HW device data * @param ethType - etherType to configure */ void ecore_set_port_mf_ovlan_eth_type(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, u32 eth_type); + u32 ethType); +#endif /* UNUSED_HSI_FUNC */ + /** - * @brief ecore_set_vxlan_dest_port - init vxlan tunnel destination udp port + * @brief ecore_set_vxlan_dest_port - initializes vxlan tunnel destination udp + * port * * @param p_ptt - ptt window used for writing the registers. * @param dest_port - vxlan destination udp port. */ void ecore_set_vxlan_dest_port(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, u16 dest_port); + struct ecore_ptt *p_ptt, + u16 dest_port); + /** * @brief ecore_set_vxlan_enable - enable or disable VXLAN tunnel in HW * - * @param p_ptt - ptt window used for writing the registers. - * @param vxlan_enable - vxlan enable flag. + * @param p_ptt - ptt window used for writing the registers. + * @param vxlan_enable - vxlan enable flag. */ void ecore_set_vxlan_enable(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, bool vxlan_enable); + struct ecore_ptt *p_ptt, + bool vxlan_enable); + /** * @brief ecore_set_gre_enable - enable or disable GRE tunnel in HW * @@ -241,15 +302,20 @@ void ecore_set_vxlan_enable(struct ecore_hwfn *p_hwfn, */ void ecore_set_gre_enable(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, - bool eth_gre_enable, bool ip_gre_enable); + bool eth_gre_enable, + bool ip_gre_enable); + /** - * @brief ecore_set_geneve_dest_port - init geneve tunnel destination udp port + * @brief ecore_set_geneve_dest_port - initializes geneve tunnel destination + * udp port * * @param p_ptt - ptt window used for writing the registers. * @param dest_port - geneve destination udp port. */ void ecore_set_geneve_dest_port(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, u16 dest_port); + struct ecore_ptt *p_ptt, + u16 dest_port); + /** * @brief ecore_set_gre_enable - enable or disable GRE tunnel in HW * @@ -259,5 +325,141 @@ void ecore_set_geneve_dest_port(struct ecore_hwfn *p_hwfn, */ void ecore_set_geneve_enable(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, - bool eth_geneve_enable, bool ip_geneve_enable); + bool eth_geneve_enable, + bool ip_geneve_enable); +#ifndef UNUSED_HSI_FUNC + +/** +* @brief ecore_set_gft_event_id_cm_hdr - configure GFT event id and cm header +* +* @param p_ptt - ptt window used for writing the registers. +*/ +void ecore_set_gft_event_id_cm_hdr(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt); + +/** + * @brief ecore_gft_disable - Disable and GFT + * + * @param p_hwfn - HW device data + * @param p_ptt - ptt window used for writing the registers. + * @param pf_id - pf on which to disable GFT. + */ +void ecore_gft_disable(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + u16 pf_id); + +/** + * @brief ecore_gft_config - Enable and configure HW for GFT +* +* @param p_ptt - ptt window used for writing the registers. + * @param pf_id - pf on which to enable GFT. +* @param tcp - set profile tcp packets. +* @param udp - set profile udp packet. +* @param ipv4 - set profile ipv4 packet. +* @param ipv6 - set profile ipv6 packet. + * @param profile_type - define packet same fields. Use enum gft_profile_type. +*/ +void ecore_gft_config(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + u16 pf_id, + bool tcp, + bool udp, + bool ipv4, + bool ipv6, + enum gft_profile_type profile_type); +#endif /* UNUSED_HSI_FUNC */ + +/** +* @brief ecore_config_vf_zone_size_mode - Configure VF zone size mode. Must be +* used before first ETH queue started. +* +* +* @param p_ptt - ptt window used for writing the registers. Don't care +* if runtime_init used +* @param mode - VF zone size mode. Use enum vf_zone_size_mode. +* @param runtime_init - Set 1 to init runtime registers in engine phase. Set 0 +* if VF zone size mode configured after engine phase. +*/ +void ecore_config_vf_zone_size_mode(struct ecore_hwfn *p_hwfn, struct ecore_ptt + *p_ptt, u16 mode, bool runtime_init); + +/** + * @brief ecore_get_mstorm_queue_stat_offset - Get mstorm statistics offset by + * VF zone size mode. +* +* @param stat_cnt_id - statistic counter id +* @param vf_zone_size_mode - VF zone size mode. Use enum vf_zone_size_mode. +*/ +u32 ecore_get_mstorm_queue_stat_offset(struct ecore_hwfn *p_hwfn, + u16 stat_cnt_id, u16 vf_zone_size_mode); + +/** + * @brief ecore_get_mstorm_eth_vf_prods_offset - VF producer offset by VF zone + * size mode. +* +* @param vf_id - vf id. +* @param vf_queue_id - per VF rx queue id. +* @param vf_zone_size_mode - vf zone size mode. Use enum vf_zone_size_mode. +*/ +u32 ecore_get_mstorm_eth_vf_prods_offset(struct ecore_hwfn *p_hwfn, u8 vf_id, u8 + vf_queue_id, u16 vf_zone_size_mode); +/** + * @brief ecore_enable_context_validation - Enable and configure context + * validation. + * + * @param p_ptt - ptt window used for writing the registers. + */ +void ecore_enable_context_validation(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt); +/** + * @brief ecore_calc_session_ctx_validation - Calcualte validation byte for + * session context. + * + * @param p_ctx_mem - pointer to context memory. + * @param ctx_size - context size. + * @param ctx_type - context type. + * @param cid - context cid. + */ +void ecore_calc_session_ctx_validation(void *p_ctx_mem, + u16 ctx_size, + u8 ctx_type, + u32 cid); + +/** + * @brief ecore_calc_task_ctx_validation - Calcualte validation byte for task + * context. + * + * @param p_ctx_mem - pointer to context memory. + * @param ctx_size - context size. + * @param ctx_type - context type. + * @param tid - context tid. + */ +void ecore_calc_task_ctx_validation(void *p_ctx_mem, + u16 ctx_size, + u8 ctx_type, + u32 tid); + +/** + * @brief ecore_memset_session_ctx - Memset session context to 0 while + * preserving validation bytes. + * + * @param p_hwfn - HW device data + * @param p_ctx_mem - pointer to context memory. + * @param ctx_size - size to initialzie. + * @param ctx_type - context type. + */ +void ecore_memset_session_ctx(void *p_ctx_mem, + u32 ctx_size, + u8 ctx_type); +/** + * @brief ecore_memset_task_ctx - Memset task context to 0 while preserving + * validation bytes. + * + * @param p_ctx_mem - pointer to context memory. + * @param ctx_size - size to initialzie. + * @param ctx_type - context type. + */ +void ecore_memset_task_ctx(void *p_ctx_mem, + u32 ctx_size, + u8 ctx_type); #endif diff --git a/dpdk/drivers/net/qede/base/ecore_init_ops.c b/dpdk/drivers/net/qede/base/ecore_init_ops.c index 326eb926..91633c11 100644 --- a/dpdk/drivers/net/qede/base/ecore_init_ops.c +++ b/dpdk/drivers/net/qede/base/ecore_init_ops.c @@ -40,6 +40,13 @@ void ecore_init_clear_rt_data(struct ecore_hwfn *p_hwfn) void ecore_init_store_rt_reg(struct ecore_hwfn *p_hwfn, u32 rt_offset, u32 val) { + if (rt_offset >= RUNTIME_ARRAY_SIZE) { + DP_ERR(p_hwfn, + "Avoid storing %u in rt_data at index %u since RUNTIME_ARRAY_SIZE is %u!\n", + val, rt_offset, RUNTIME_ARRAY_SIZE); + return; + } + p_hwfn->rt_data.init_val[rt_offset] = val; p_hwfn->rt_data.b_valid[rt_offset] = true; } @@ -49,6 +56,14 @@ void ecore_init_store_rt_agg(struct ecore_hwfn *p_hwfn, { osal_size_t i; + if ((rt_offset + size - 1) >= RUNTIME_ARRAY_SIZE) { + DP_ERR(p_hwfn, + "Avoid storing values in rt_data at indices %u-%u since RUNTIME_ARRAY_SIZE is %u!\n", + rt_offset, (u32)(rt_offset + size - 1), + RUNTIME_ARRAY_SIZE); + return; + } + for (i = 0; i < size / sizeof(u32); i++) { p_hwfn->rt_data.init_val[rt_offset + i] = p_val[i]; p_hwfn->rt_data.b_valid[rt_offset + i] = true; @@ -63,8 +78,8 @@ static enum _ecore_status_t ecore_init_rt(struct ecore_hwfn *p_hwfn, { u32 *p_init_val = &p_hwfn->rt_data.init_val[rt_offset]; bool *p_valid = &p_hwfn->rt_data.b_valid[rt_offset]; - enum _ecore_status_t rc = ECORE_SUCCESS; u16 i, segment; + enum _ecore_status_t rc = ECORE_SUCCESS; /* Since not all RT entries are initialized, go over the RT and * for each segment of initialized values use DMA. @@ -161,8 +176,7 @@ static enum _ecore_status_t ecore_init_array_dmae(struct ecore_hwfn *p_hwfn, static enum _ecore_status_t ecore_init_fill_dmae(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, - u32 addr, u32 fill, - u32 fill_count) + u32 addr, u32 fill_count) { static u32 zero_buffer[DMAE_MAX_RW_SIZE]; @@ -190,19 +204,19 @@ static enum _ecore_status_t ecore_init_cmd_array(struct ecore_hwfn *p_hwfn, bool b_must_dmae, bool b_can_dmae) { + u32 dmae_array_offset = OSAL_LE32_TO_CPU(cmd->args.array_offset); + u32 data = OSAL_LE32_TO_CPU(cmd->data); + u32 addr = GET_FIELD(data, INIT_WRITE_OP_ADDRESS) << 2; #ifdef CONFIG_ECORE_ZIPPED_FW u32 offset, output_len, input_len, max_size; #endif - u32 dmae_array_offset = OSAL_LE32_TO_CPU(cmd->args.array_offset); struct ecore_dev *p_dev = p_hwfn->p_dev; - enum _ecore_status_t rc = ECORE_SUCCESS; union init_array_hdr *hdr; const u32 *array_data; - u32 size, addr, data; + enum _ecore_status_t rc = ECORE_SUCCESS; + u32 size; array_data = p_dev->fw_data->arr_data; - data = OSAL_LE32_TO_CPU(cmd->data); - addr = GET_FIELD(data, INIT_WRITE_OP_ADDRESS) << 2; hdr = (union init_array_hdr *) (uintptr_t)(array_data + dmae_array_offset); @@ -251,9 +265,9 @@ static enum _ecore_status_t ecore_init_cmd_array(struct ecore_hwfn *p_hwfn, b_can_dmae); if (rc) break; - } - break; } + break; + } case INIT_ARR_STANDARD: size = GET_FIELD(data, INIT_ARRAY_STANDARD_HDR_SIZE); rc = ecore_init_array_dmae(p_hwfn, p_ptt, addr, @@ -272,13 +286,10 @@ static enum _ecore_status_t ecore_init_cmd_wr(struct ecore_hwfn *p_hwfn, struct init_write_op *p_cmd, bool b_can_dmae) { + u32 data = OSAL_LE32_TO_CPU(p_cmd->data); + bool b_must_dmae = GET_FIELD(data, INIT_WRITE_OP_WIDE_BUS); + u32 addr = GET_FIELD(data, INIT_WRITE_OP_ADDRESS) << 2; enum _ecore_status_t rc = ECORE_SUCCESS; - bool b_must_dmae; - u32 addr, data; - - data = OSAL_LE32_TO_CPU(p_cmd->data); - b_must_dmae = GET_FIELD(data, INIT_WRITE_OP_WIDE_BUS); - addr = GET_FIELD(data, INIT_WRITE_OP_ADDRESS) << 2; /* Sanitize */ if (b_must_dmae && !b_can_dmae) { @@ -297,7 +308,7 @@ static enum _ecore_status_t ecore_init_cmd_wr(struct ecore_hwfn *p_hwfn, case INIT_SRC_ZEROS: data = OSAL_LE32_TO_CPU(p_cmd->args.zeros_count); if (b_must_dmae || (b_can_dmae && (data >= 64))) - rc = ecore_init_fill_dmae(p_hwfn, p_ptt, addr, 0, data); + rc = ecore_init_fill_dmae(p_hwfn, p_ptt, addr, data); else ecore_init_fill(p_hwfn, p_ptt, addr, 0, data); break; @@ -306,10 +317,10 @@ static enum _ecore_status_t ecore_init_cmd_wr(struct ecore_hwfn *p_hwfn, b_must_dmae, b_can_dmae); break; case INIT_SRC_RUNTIME: - ecore_init_rt(p_hwfn, p_ptt, addr, - OSAL_LE16_TO_CPU(p_cmd->args.runtime.offset), - OSAL_LE16_TO_CPU(p_cmd->args.runtime.size), - b_must_dmae); + rc = ecore_init_rt(p_hwfn, p_ptt, addr, + OSAL_LE16_TO_CPU(p_cmd->args.runtime.offset), + OSAL_LE16_TO_CPU(p_cmd->args.runtime.size), + b_must_dmae); break; } @@ -385,10 +396,13 @@ static void ecore_init_cmd_rd(struct ecore_hwfn *p_hwfn, OSAL_LE32_TO_CPU(cmd->op_data)); } -/* init_ops callbacks entry point */ +/* init_ops callbacks entry point. + * OSAL_UNUSED is temporary used to avoid unused-parameter compilation warnings. + * Should be removed when the function is actually used. + */ static void ecore_init_cmd_cb(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, - struct init_callback_op *p_cmd) + struct ecore_ptt OSAL_UNUSED * p_ptt, + struct init_callback_op OSAL_UNUSED * p_cmd) { DP_NOTICE(p_hwfn, true, "Currently init values have no need of callbacks\n"); @@ -432,17 +446,16 @@ static u32 ecore_init_cmd_mode(struct ecore_hwfn *p_hwfn, INIT_IF_MODE_OP_CMD_OFFSET); } -static u32 ecore_init_cmd_phase(struct ecore_hwfn *p_hwfn, - struct init_if_phase_op *p_cmd, +static u32 ecore_init_cmd_phase(struct init_if_phase_op *p_cmd, u32 phase, u32 phase_id) { u32 data = OSAL_LE32_TO_CPU(p_cmd->phase_data); + u32 op_data = OSAL_LE32_TO_CPU(p_cmd->op_data); if (!(GET_FIELD(data, INIT_IF_PHASE_OP_PHASE) == phase && (GET_FIELD(data, INIT_IF_PHASE_OP_PHASE_ID) == ANY_PHASE_ID || GET_FIELD(data, INIT_IF_PHASE_OP_PHASE_ID) == phase_id))) - return GET_FIELD(OSAL_LE32_TO_CPU(p_cmd->op_data), - INIT_IF_PHASE_OP_CMD_OFFSET); + return GET_FIELD(op_data, INIT_IF_PHASE_OP_CMD_OFFSET); else return 0; } @@ -452,10 +465,10 @@ enum _ecore_status_t ecore_init_run(struct ecore_hwfn *p_hwfn, int phase, int phase_id, int modes) { struct ecore_dev *p_dev = p_hwfn->p_dev; - enum _ecore_status_t rc = ECORE_SUCCESS; u32 cmd_num, num_init_ops; union init_op *init_ops; bool b_dmae = false; + enum _ecore_status_t rc = ECORE_SUCCESS; num_init_ops = p_dev->fw_data->init_ops_size; init_ops = p_dev->fw_data->init_ops; @@ -488,8 +501,8 @@ enum _ecore_status_t ecore_init_run(struct ecore_hwfn *p_hwfn, modes); break; case INIT_OP_IF_PHASE: - cmd_num += ecore_init_cmd_phase(p_hwfn, &cmd->if_phase, - phase, phase_id); + cmd_num += ecore_init_cmd_phase(&cmd->if_phase, phase, + phase_id); b_dmae = GET_FIELD(data, INIT_IF_PHASE_OP_DMAE_ENABLE); break; case INIT_OP_DELAY: @@ -513,7 +526,8 @@ enum _ecore_status_t ecore_init_run(struct ecore_hwfn *p_hwfn, return rc; } -void ecore_gtt_init(struct ecore_hwfn *p_hwfn) +void ecore_gtt_init(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt) { u32 gtt_base; u32 i; @@ -525,13 +539,13 @@ void ecore_gtt_init(struct ecore_hwfn *p_hwfn) * not too bright, but it should work on the simple FPGA/EMUL * scenarios. */ - bool initialized = false; /* @DPDK */ + static bool initialized; int poll_cnt = 500; u32 val; /* initialize PTT/GTT (poll for completion) */ if (!initialized) { - ecore_wr(p_hwfn, p_hwfn->p_main_ptt, + ecore_wr(p_hwfn, p_ptt, PGLUE_B_REG_START_INIT_PTT_GTT, 1); initialized = true; } @@ -540,7 +554,7 @@ void ecore_gtt_init(struct ecore_hwfn *p_hwfn) /* ptt might be overrided by HW until this is done */ OSAL_UDELAY(10); ecore_ptt_invalidate(p_hwfn); - val = ecore_rd(p_hwfn, p_hwfn->p_main_ptt, + val = ecore_rd(p_hwfn, p_ptt, PGLUE_B_REG_INIT_DONE_PTT_GTT); } while ((val != 1) && --poll_cnt); @@ -560,7 +574,11 @@ void ecore_gtt_init(struct ecore_hwfn *p_hwfn) } enum _ecore_status_t ecore_init_fw_data(struct ecore_dev *p_dev, - const u8 *data) +#ifdef CONFIG_ECORE_BINARY_FW + const u8 *fw_data) +#else + const u8 OSAL_UNUSED * fw_data) +#endif { struct ecore_fw_data *fw = p_dev->fw_data; @@ -568,24 +586,24 @@ enum _ecore_status_t ecore_init_fw_data(struct ecore_dev *p_dev, struct bin_buffer_hdr *buf_hdr; u32 offset, len; - if (!data) { + if (!fw_data) { DP_NOTICE(p_dev, true, "Invalid fw data\n"); return ECORE_INVAL; } - buf_hdr = (struct bin_buffer_hdr *)(uintptr_t)data; + buf_hdr = (struct bin_buffer_hdr *)(uintptr_t)fw_data; - offset = buf_hdr[BIN_BUF_FW_VER_INFO].offset; - fw->fw_ver_info = (struct fw_ver_info *)((uintptr_t)(data + offset)); + offset = buf_hdr[BIN_BUF_INIT_FW_VER_INFO].offset; + fw->fw_ver_info = (struct fw_ver_info *)((uintptr_t)(fw_data + offset)); offset = buf_hdr[BIN_BUF_INIT_CMD].offset; - fw->init_ops = (union init_op *)((uintptr_t)(data + offset)); + fw->init_ops = (union init_op *)((uintptr_t)(fw_data + offset)); offset = buf_hdr[BIN_BUF_INIT_VAL].offset; - fw->arr_data = (u32 *)((uintptr_t)(data + offset)); + fw->arr_data = (u32 *)((uintptr_t)(fw_data + offset)); offset = buf_hdr[BIN_BUF_INIT_MODE_TREE].offset; - fw->modes_tree_buf = (u8 *)((uintptr_t)(data + offset)); + fw->modes_tree_buf = (u8 *)((uintptr_t)(fw_data + offset)); len = buf_hdr[BIN_BUF_INIT_CMD].length; fw->init_ops_size = len / sizeof(struct init_raw_op); #else diff --git a/dpdk/drivers/net/qede/base/ecore_init_ops.h b/dpdk/drivers/net/qede/base/ecore_init_ops.h index 8a6fce41..e293a4a3 100644 --- a/dpdk/drivers/net/qede/base/ecore_init_ops.h +++ b/dpdk/drivers/net/qede/base/ecore_init_ops.h @@ -32,7 +32,9 @@ void ecore_init_iro_array(struct ecore_dev *p_dev); */ enum _ecore_status_t ecore_init_run(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, - int phase, int phase_id, int modes); + int phase, + int phase_id, + int modes); /** * @brief ecore_init_hwfn_allocate - Allocate RT array, Store 'values' ptrs. @@ -52,6 +54,7 @@ enum _ecore_status_t ecore_init_alloc(struct ecore_hwfn *p_hwfn); */ void ecore_init_free(struct ecore_hwfn *p_hwfn); + /** * @brief ecore_init_clear_rt_data - Clears the runtime init array. * @@ -68,7 +71,9 @@ void ecore_init_clear_rt_data(struct ecore_hwfn *p_hwfn); * @param rt_offset * @param val */ -void ecore_init_store_rt_reg(struct ecore_hwfn *p_hwfn, u32 rt_offset, u32 val); +void ecore_init_store_rt_reg(struct ecore_hwfn *p_hwfn, + u32 rt_offset, + u32 val); #define STORE_RT_REG(hwfn, offset, val) \ ecore_init_store_rt_reg(hwfn, offset, val) @@ -87,11 +92,14 @@ void ecore_init_store_rt_reg(struct ecore_hwfn *p_hwfn, u32 rt_offset, u32 val); */ void ecore_init_store_rt_agg(struct ecore_hwfn *p_hwfn, - u32 rt_offset, u32 *val, osal_size_t size); + u32 rt_offset, + u32 *val, + osal_size_t size); #define STORE_RT_REG_AGG(hwfn, offset, val) \ ecore_init_store_rt_agg(hwfn, offset, (u32 *)&val, sizeof(val)) + /** * @brief * Initialize GTT global windows and set admin window @@ -99,5 +107,6 @@ void ecore_init_store_rt_agg(struct ecore_hwfn *p_hwfn, * * @param p_hwfn */ -void ecore_gtt_init(struct ecore_hwfn *p_hwfn); +void ecore_gtt_init(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt); #endif /* __ECORE_INIT_OPS__ */ diff --git a/dpdk/drivers/net/qede/base/ecore_int.c b/dpdk/drivers/net/qede/base/ecore_int.c index ea0fd7aa..e6cef85b 100644 --- a/dpdk/drivers/net/qede/base/ecore_int.c +++ b/dpdk/drivers/net/qede/base/ecore_int.c @@ -9,7 +9,6 @@ #include "bcm_osal.h" #include "ecore.h" #include "ecore_spq.h" -#include "reg_addr.h" #include "ecore_gtt_reg_addr.h" #include "ecore_init_ops.h" #include "ecore_rt_defs.h" @@ -21,7 +20,6 @@ #include "ecore_hw_defs.h" #include "ecore_hsi_common.h" #include "ecore_mcp.h" -#include "ecore_attn_values.h" struct ecore_pi_info { ecore_int_comp_cb_t comp_cb; @@ -31,7 +29,7 @@ struct ecore_pi_info { struct ecore_sb_sp_info { struct ecore_sb_info sb_info; /* per protocol index data */ - struct ecore_pi_info pi_info_arr[PIS_PER_SB]; + struct ecore_pi_info pi_info_arr[PIS_PER_SB_E4]; }; enum ecore_attention_type { @@ -60,9 +58,12 @@ struct aeu_invert_reg_bit { #define ATTENTION_OFFSET_MASK (0x000ff000) #define ATTENTION_OFFSET_SHIFT (12) +#define ATTENTION_BB_MASK (0x00700000) +#define ATTENTION_BB_SHIFT (20) +#define ATTENTION_BB(value) ((value) << ATTENTION_BB_SHIFT) +#define ATTENTION_BB_DIFFERENT (1 << 23) + #define ATTENTION_CLEAR_ENABLE (1 << 28) -#define ATTENTION_FW_DUMP (1 << 29) -#define ATTENTION_PANIC_DUMP (1 << 30) unsigned int flags; /* Callback to call if attention will be triggered */ @@ -100,7 +101,7 @@ static enum _ecore_status_t ecore_mcp_attn_cb(struct ecore_hwfn *p_hwfn) #define ECORE_PSWHST_ATTNETION_DISABLED_WRITE_SHIFT (0) #define ECORE_PSWHST_ATTENTION_VF_DISABLED (0x1) #define ECORE_PSWHST_ATTENTION_INCORRECT_ACCESS (0x1) -#define ECORE_PSWHST_ATTENTION_INCORRECT_ACCESS_WR_MASK (0x1) +#define ECORE_PSWHST_ATTENTION_INCORRECT_ACCESS_WR_MASK (0x1) #define ECORE_PSWHST_ATTENTION_INCORRECT_ACCESS_WR_SHIFT (0) #define ECORE_PSWHST_ATTENTION_INCORRECT_ACCESS_CLIENT_MASK (0x1e) #define ECORE_PSWHST_ATTENTION_INCORRECT_ACCESS_CLIENT_SHIFT (1) @@ -246,21 +247,21 @@ static enum _ecore_status_t ecore_grc_attn_cb(struct ecore_hwfn *p_hwfn) tmp2 = ecore_rd(p_hwfn, p_hwfn->p_dpc_ptt, GRC_REG_TIMEOUT_ATTN_ACCESS_DATA_1); - DP_INFO(p_hwfn->p_dev, - "GRC timeout [%08x:%08x] - %s Address [%08x] [Master %s]" - " [PF: %02x %s %02x]\n", - tmp2, tmp, - (tmp & ECORE_GRC_ATTENTION_RDWR_BIT) ? "Write to" : "Read from", - (tmp & ECORE_GRC_ATTENTION_ADDRESS_MASK) << 2, - grc_timeout_attn_master_to_str((tmp & - ECORE_GRC_ATTENTION_MASTER_MASK) >> - ECORE_GRC_ATTENTION_MASTER_SHIFT), - (tmp2 & ECORE_GRC_ATTENTION_PF_MASK), - (((tmp2 & ECORE_GRC_ATTENTION_PRIV_MASK) >> + DP_NOTICE(p_hwfn->p_dev, false, + "GRC timeout [%08x:%08x] - %s Address [%08x] [Master %s] [PF: %02x %s %02x]\n", + tmp2, tmp, + (tmp & ECORE_GRC_ATTENTION_RDWR_BIT) ? "Write to" + : "Read from", + (tmp & ECORE_GRC_ATTENTION_ADDRESS_MASK) << 2, + grc_timeout_attn_master_to_str( + (tmp & ECORE_GRC_ATTENTION_MASTER_MASK) >> + ECORE_GRC_ATTENTION_MASTER_SHIFT), + (tmp2 & ECORE_GRC_ATTENTION_PF_MASK), + (((tmp2 & ECORE_GRC_ATTENTION_PRIV_MASK) >> ECORE_GRC_ATTENTION_PRIV_SHIFT) == - ECORE_GRC_ATTENTION_PRIV_VF) ? "VF" : "(Irrelevant:)", - (tmp2 & ECORE_GRC_ATTENTION_VF_MASK) >> - ECORE_GRC_ATTENTION_VF_SHIFT); + ECORE_GRC_ATTENTION_PRIV_VF) ? "VF" : "(Irrelevant:)", + (tmp2 & ECORE_GRC_ATTENTION_VF_MASK) >> + ECORE_GRC_ATTENTION_VF_SHIFT); out: /* Regardles of anything else, clean the validity bit */ @@ -282,155 +283,117 @@ out: #define ECORE_PGLUE_ATTENTION_ICPL_VALID (1 << 23) #define ECORE_PGLUE_ATTENTION_ZLR_VALID (1 << 25) #define ECORE_PGLUE_ATTENTION_ILT_VALID (1 << 23) -static enum _ecore_status_t ecore_pglub_rbc_attn_cb(struct ecore_hwfn *p_hwfn) + +enum _ecore_status_t ecore_pglueb_rbc_attn_handler(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt) { - u32 tmp, reg_addr; + u32 tmp; - reg_addr = - attn_blocks[BLOCK_PGLUE_B].chip_regs[ECORE_GET_TYPE(p_hwfn->p_dev)]. - int_regs[0]->mask_addr; - - /* Mask unnecessary attentions -@TBD move to MFW */ - tmp = ecore_rd(p_hwfn, p_hwfn->p_dpc_ptt, reg_addr); - tmp |= (1 << 19); /* Was PGL_PCIE_ATTN */ - ecore_wr(p_hwfn, p_hwfn->p_dpc_ptt, reg_addr, tmp); - - tmp = ecore_rd(p_hwfn, p_hwfn->p_dpc_ptt, - PGLUE_B_REG_TX_ERR_WR_DETAILS2); + tmp = ecore_rd(p_hwfn, p_ptt, PGLUE_B_REG_TX_ERR_WR_DETAILS2); if (tmp & ECORE_PGLUE_ATTENTION_VALID) { u32 addr_lo, addr_hi, details; - addr_lo = ecore_rd(p_hwfn, p_hwfn->p_dpc_ptt, + addr_lo = ecore_rd(p_hwfn, p_ptt, PGLUE_B_REG_TX_ERR_WR_ADD_31_0); - addr_hi = ecore_rd(p_hwfn, p_hwfn->p_dpc_ptt, + addr_hi = ecore_rd(p_hwfn, p_ptt, PGLUE_B_REG_TX_ERR_WR_ADD_63_32); - details = ecore_rd(p_hwfn, p_hwfn->p_dpc_ptt, + details = ecore_rd(p_hwfn, p_ptt, PGLUE_B_REG_TX_ERR_WR_DETAILS); - DP_INFO(p_hwfn, - "Illegal write by chip to [%08x:%08x] blocked." - "Details: %08x [PFID %02x, VFID %02x, VF_VALID %02x]" - " Details2 %08x [Was_error %02x BME deassert %02x" - " FID_enable deassert %02x]\n", - addr_hi, addr_lo, details, - (u8)((details & - ECORE_PGLUE_ATTENTION_DETAILS_PFID_MASK) >> - ECORE_PGLUE_ATTENTION_DETAILS_PFID_SHIFT), - (u8)((details & - ECORE_PGLUE_ATTENTION_DETAILS_VFID_MASK) >> - ECORE_PGLUE_ATTENTION_DETAILS_VFID_SHIFT), - (u8)((details & ECORE_PGLUE_ATTENTION_DETAILS_VF_VALID) - ? 1 : 0), tmp, - (u8)((tmp & ECORE_PGLUE_ATTENTION_DETAILS2_WAS_ERR) ? 1 - : 0), - (u8)((tmp & ECORE_PGLUE_ATTENTION_DETAILS2_BME) ? 1 : - 0), - (u8)((tmp & ECORE_PGLUE_ATTENTION_DETAILS2_FID_EN) ? 1 - : 0)); + DP_NOTICE(p_hwfn, false, + "Illegal write by chip to [%08x:%08x] blocked. Details: %08x [PFID %02x, VFID %02x, VF_VALID %02x] Details2 %08x [Was_error %02x BME deassert %02x FID_enable deassert %02x]\n", + addr_hi, addr_lo, details, + (u8)((details & + ECORE_PGLUE_ATTENTION_DETAILS_PFID_MASK) >> + ECORE_PGLUE_ATTENTION_DETAILS_PFID_SHIFT), + (u8)((details & + ECORE_PGLUE_ATTENTION_DETAILS_VFID_MASK) >> + ECORE_PGLUE_ATTENTION_DETAILS_VFID_SHIFT), + (u8)((details & + ECORE_PGLUE_ATTENTION_DETAILS_VF_VALID) ? 1 : 0), + tmp, + (u8)((tmp & ECORE_PGLUE_ATTENTION_DETAILS2_WAS_ERR) ? + 1 : 0), + (u8)((tmp & ECORE_PGLUE_ATTENTION_DETAILS2_BME) ? + 1 : 0), + (u8)((tmp & ECORE_PGLUE_ATTENTION_DETAILS2_FID_EN) ? + 1 : 0)); } - tmp = ecore_rd(p_hwfn, p_hwfn->p_dpc_ptt, - PGLUE_B_REG_TX_ERR_RD_DETAILS2); + tmp = ecore_rd(p_hwfn, p_ptt, PGLUE_B_REG_TX_ERR_RD_DETAILS2); if (tmp & ECORE_PGLUE_ATTENTION_RD_VALID) { u32 addr_lo, addr_hi, details; - addr_lo = ecore_rd(p_hwfn, p_hwfn->p_dpc_ptt, + addr_lo = ecore_rd(p_hwfn, p_ptt, PGLUE_B_REG_TX_ERR_RD_ADD_31_0); - addr_hi = ecore_rd(p_hwfn, p_hwfn->p_dpc_ptt, + addr_hi = ecore_rd(p_hwfn, p_ptt, PGLUE_B_REG_TX_ERR_RD_ADD_63_32); - details = ecore_rd(p_hwfn, p_hwfn->p_dpc_ptt, + details = ecore_rd(p_hwfn, p_ptt, PGLUE_B_REG_TX_ERR_RD_DETAILS); - DP_INFO(p_hwfn, - "Illegal read by chip from [%08x:%08x] blocked." - " Details: %08x [PFID %02x, VFID %02x, VF_VALID %02x]" - " Details2 %08x [Was_error %02x BME deassert %02x" - " FID_enable deassert %02x]\n", - addr_hi, addr_lo, details, - (u8)((details & - ECORE_PGLUE_ATTENTION_DETAILS_PFID_MASK) >> - ECORE_PGLUE_ATTENTION_DETAILS_PFID_SHIFT), - (u8)((details & - ECORE_PGLUE_ATTENTION_DETAILS_VFID_MASK) >> - ECORE_PGLUE_ATTENTION_DETAILS_VFID_SHIFT), - (u8)((details & ECORE_PGLUE_ATTENTION_DETAILS_VF_VALID) - ? 1 : 0), tmp, - (u8)((tmp & ECORE_PGLUE_ATTENTION_DETAILS2_WAS_ERR) ? 1 - : 0), - (u8)((tmp & ECORE_PGLUE_ATTENTION_DETAILS2_BME) ? 1 : - 0), - (u8)((tmp & ECORE_PGLUE_ATTENTION_DETAILS2_FID_EN) ? 1 - : 0)); + DP_NOTICE(p_hwfn, false, + "Illegal read by chip from [%08x:%08x] blocked. Details: %08x [PFID %02x, VFID %02x, VF_VALID %02x] Details2 %08x [Was_error %02x BME deassert %02x FID_enable deassert %02x]\n", + addr_hi, addr_lo, details, + (u8)((details & + ECORE_PGLUE_ATTENTION_DETAILS_PFID_MASK) >> + ECORE_PGLUE_ATTENTION_DETAILS_PFID_SHIFT), + (u8)((details & + ECORE_PGLUE_ATTENTION_DETAILS_VFID_MASK) >> + ECORE_PGLUE_ATTENTION_DETAILS_VFID_SHIFT), + (u8)((details & + ECORE_PGLUE_ATTENTION_DETAILS_VF_VALID) ? 1 : 0), + tmp, + (u8)((tmp & ECORE_PGLUE_ATTENTION_DETAILS2_WAS_ERR) ? + 1 : 0), + (u8)((tmp & ECORE_PGLUE_ATTENTION_DETAILS2_BME) ? + 1 : 0), + (u8)((tmp & ECORE_PGLUE_ATTENTION_DETAILS2_FID_EN) ? + 1 : 0)); } - tmp = ecore_rd(p_hwfn, p_hwfn->p_dpc_ptt, - PGLUE_B_REG_TX_ERR_WR_DETAILS_ICPL); + tmp = ecore_rd(p_hwfn, p_ptt, PGLUE_B_REG_TX_ERR_WR_DETAILS_ICPL); if (tmp & ECORE_PGLUE_ATTENTION_ICPL_VALID) - DP_INFO(p_hwfn, "ICPL error - %08x\n", tmp); + DP_NOTICE(p_hwfn, false, "ICPL erorr - %08x\n", tmp); - tmp = ecore_rd(p_hwfn, p_hwfn->p_dpc_ptt, - PGLUE_B_REG_MASTER_ZLR_ERR_DETAILS); + tmp = ecore_rd(p_hwfn, p_ptt, PGLUE_B_REG_MASTER_ZLR_ERR_DETAILS); if (tmp & ECORE_PGLUE_ATTENTION_ZLR_VALID) { u32 addr_hi, addr_lo; - addr_lo = ecore_rd(p_hwfn, p_hwfn->p_dpc_ptt, + addr_lo = ecore_rd(p_hwfn, p_ptt, PGLUE_B_REG_MASTER_ZLR_ERR_ADD_31_0); - addr_hi = ecore_rd(p_hwfn, p_hwfn->p_dpc_ptt, + addr_hi = ecore_rd(p_hwfn, p_ptt, PGLUE_B_REG_MASTER_ZLR_ERR_ADD_63_32); - DP_INFO(p_hwfn, "ICPL error - %08x [Address %08x:%08x]\n", - tmp, addr_hi, addr_lo); + DP_NOTICE(p_hwfn, false, + "ICPL erorr - %08x [Address %08x:%08x]\n", + tmp, addr_hi, addr_lo); } - tmp = ecore_rd(p_hwfn, p_hwfn->p_dpc_ptt, - PGLUE_B_REG_VF_ILT_ERR_DETAILS2); + tmp = ecore_rd(p_hwfn, p_ptt, PGLUE_B_REG_VF_ILT_ERR_DETAILS2); if (tmp & ECORE_PGLUE_ATTENTION_ILT_VALID) { u32 addr_hi, addr_lo, details; - addr_lo = ecore_rd(p_hwfn, p_hwfn->p_dpc_ptt, + addr_lo = ecore_rd(p_hwfn, p_ptt, PGLUE_B_REG_VF_ILT_ERR_ADD_31_0); - addr_hi = ecore_rd(p_hwfn, p_hwfn->p_dpc_ptt, + addr_hi = ecore_rd(p_hwfn, p_ptt, PGLUE_B_REG_VF_ILT_ERR_ADD_63_32); - details = ecore_rd(p_hwfn, p_hwfn->p_dpc_ptt, + details = ecore_rd(p_hwfn, p_ptt, PGLUE_B_REG_VF_ILT_ERR_DETAILS); - DP_INFO(p_hwfn, - "ILT error - Details %08x Details2 %08x" - " [Address %08x:%08x]\n", - details, tmp, addr_hi, addr_lo); + DP_NOTICE(p_hwfn, false, + "ILT error - Details %08x Details2 %08x [Address %08x:%08x]\n", + details, tmp, addr_hi, addr_lo); } /* Clear the indications */ - ecore_wr(p_hwfn, p_hwfn->p_dpc_ptt, - PGLUE_B_REG_LATCHED_ERRORS_CLR, (1 << 2)); + ecore_wr(p_hwfn, p_ptt, PGLUE_B_REG_LATCHED_ERRORS_CLR, (1 << 2)); return ECORE_SUCCESS; } -static enum _ecore_status_t ecore_nig_attn_cb(struct ecore_hwfn *p_hwfn) +static enum _ecore_status_t ecore_pglueb_rbc_attn_cb(struct ecore_hwfn *p_hwfn) { - u32 tmp, reg_addr; - - /* Mask unnecessary attentions -@TBD move to MFW */ - reg_addr = - attn_blocks[BLOCK_NIG].chip_regs[ECORE_GET_TYPE(p_hwfn->p_dev)]. - int_regs[3]->mask_addr; - tmp = ecore_rd(p_hwfn, p_hwfn->p_dpc_ptt, reg_addr); - tmp |= (1 << 0); /* Was 3_P0_TX_PAUSE_TOO_LONG_INT */ - tmp |= NIG_REG_INT_MASK_3_P0_LB_TC1_PAUSE_TOO_LONG_INT; - ecore_wr(p_hwfn, p_hwfn->p_dpc_ptt, reg_addr, tmp); - - reg_addr = - attn_blocks[BLOCK_NIG].chip_regs[ECORE_GET_TYPE(p_hwfn->p_dev)]. - int_regs[5]->mask_addr; - tmp = ecore_rd(p_hwfn, p_hwfn->p_dpc_ptt, reg_addr); - tmp |= (1 << 0); /* Was 5_P1_TX_PAUSE_TOO_LONG_INT */ - ecore_wr(p_hwfn, p_hwfn->p_dpc_ptt, reg_addr, tmp); - - /* TODO - a bit risky to return success here; But alternative is to - * actually read the multitdue of interrupt register of the block. - */ - return ECORE_SUCCESS; + return ecore_pglueb_rbc_attn_handler(p_hwfn, p_hwfn->p_dpc_ptt); } static enum _ecore_status_t ecore_fw_assertion(struct ecore_hwfn *p_hwfn) @@ -450,31 +413,136 @@ ecore_general_attention_35(struct ecore_hwfn *p_hwfn) return ECORE_SUCCESS; } -#define ECORE_DORQ_ATTENTION_REASON_MASK (0xfffff) -#define ECORE_DORQ_ATTENTION_OPAQUE_MASK (0xffff) -#define ECORE_DORQ_ATTENTION_SIZE_MASK (0x7f) -#define ECORE_DORQ_ATTENTION_SIZE_SHIFT (16) +#define ECORE_DORQ_ATTENTION_REASON_MASK (0xfffff) +#define ECORE_DORQ_ATTENTION_OPAQUE_MASK (0xffff) +#define ECORE_DORQ_ATTENTION_OPAQUE_SHIFT (0x0) +#define ECORE_DORQ_ATTENTION_SIZE_MASK (0x7f) +#define ECORE_DORQ_ATTENTION_SIZE_SHIFT (16) + +#define ECORE_DB_REC_COUNT 10 +#define ECORE_DB_REC_INTERVAL 100 + +/* assumes sticky overflow indication was set for this PF */ +static enum _ecore_status_t ecore_db_rec_attn(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt) +{ + u8 count = ECORE_DB_REC_COUNT; + u32 usage = 1; + + /* wait for usage to zero or count to run out. This is necessary since + * EDPM doorbell transactions can take multiple 64b cycles, and as such + * can "split" over the pci. Possibly, the doorbell drop can happen with + * half an EDPM in the queue and other half dropped. Another EDPM + * doorbell to the same address (from doorbell recovery mechanism or + * from the doorbelling entity) could have first half dropped and second + * half interperted as continuation of the first. To prevent such + * malformed doorbells from reaching the device, flush the queue before + * releaseing the overflow sticky indication. + */ + while (count-- && usage) { + usage = ecore_rd(p_hwfn, p_ptt, DORQ_REG_PF_USAGE_CNT); + OSAL_UDELAY(ECORE_DB_REC_INTERVAL); + } + + /* should have been depleted by now */ + if (usage) { + DP_NOTICE(p_hwfn->p_dev, false, + "DB recovery: doorbell usage failed to zero after %d usec. usage was %x\n", + ECORE_DB_REC_INTERVAL * ECORE_DB_REC_COUNT, usage); + return ECORE_TIMEOUT; + } + + /* flush any pedning (e)dpm as they may never arrive */ + ecore_wr(p_hwfn, p_ptt, DORQ_REG_DPM_FORCE_ABORT, 0x1); + + /* release overflow sticky indication (stop silently dropping + * everything) + */ + ecore_wr(p_hwfn, p_ptt, DORQ_REG_PF_OVFL_STICKY, 0x0); + + /* repeat all last doorbells (doorbell drop recovery) */ + ecore_db_recovery_execute(p_hwfn, DB_REC_REAL_DEAL); + + return ECORE_SUCCESS; +} static enum _ecore_status_t ecore_dorq_attn_cb(struct ecore_hwfn *p_hwfn) { - u32 reason; + u32 int_sts, first_drop_reason, details, address, overflow, + all_drops_reason; + struct ecore_ptt *p_ptt = p_hwfn->p_dpc_ptt; + enum _ecore_status_t rc; - reason = ecore_rd(p_hwfn, p_hwfn->p_dpc_ptt, DORQ_REG_DB_DROP_REASON) & - ECORE_DORQ_ATTENTION_REASON_MASK; - if (reason) { - u32 details = ecore_rd(p_hwfn, p_hwfn->p_dpc_ptt, - DORQ_REG_DB_DROP_DETAILS); + int_sts = ecore_rd(p_hwfn, p_ptt, DORQ_REG_INT_STS); + DP_NOTICE(p_hwfn->p_dev, false, "DORQ attention. int_sts was %x\n", + int_sts); - DP_INFO(p_hwfn->p_dev, - "DORQ db_drop: address 0x%08x Opaque FID 0x%04x" - " Size [bytes] 0x%08x Reason: 0x%08x\n", - ecore_rd(p_hwfn, p_hwfn->p_dpc_ptt, - DORQ_REG_DB_DROP_DETAILS_ADDRESS), - (u16)(details & ECORE_DORQ_ATTENTION_OPAQUE_MASK), - ((details & ECORE_DORQ_ATTENTION_SIZE_MASK) >> - ECORE_DORQ_ATTENTION_SIZE_SHIFT) * 4, reason); + /* int_sts may be zero since all PFs were interrupted for doorbell + * overflow but another one already handled it. Can abort here. If + * This PF also requires overflow recovery we will be interrupted again + */ + if (!int_sts) + return ECORE_SUCCESS; + + /* check if db_drop or overflow happened */ + if (int_sts & (DORQ_REG_INT_STS_DB_DROP | + DORQ_REG_INT_STS_DORQ_FIFO_OVFL_ERR)) { + /* obtain data about db drop/overflow */ + first_drop_reason = ecore_rd(p_hwfn, p_ptt, + DORQ_REG_DB_DROP_REASON) & + ECORE_DORQ_ATTENTION_REASON_MASK; + details = ecore_rd(p_hwfn, p_ptt, + DORQ_REG_DB_DROP_DETAILS); + address = ecore_rd(p_hwfn, p_ptt, + DORQ_REG_DB_DROP_DETAILS_ADDRESS); + overflow = ecore_rd(p_hwfn, p_ptt, + DORQ_REG_PF_OVFL_STICKY); + all_drops_reason = ecore_rd(p_hwfn, p_ptt, + DORQ_REG_DB_DROP_DETAILS_REASON); + + /* log info */ + DP_NOTICE(p_hwfn->p_dev, false, + "Doorbell drop occurred\n" + "Address\t\t0x%08x\t(second BAR address)\n" + "FID\t\t0x%04x\t\t(Opaque FID)\n" + "Size\t\t0x%04x\t\t(in bytes)\n" + "1st drop reason\t0x%08x\t(details on first drop since last handling)\n" + "Sticky reasons\t0x%08x\t(all drop reasons since last handling)\n" + "Overflow\t0x%x\t\t(a per PF indication)\n", + address, + GET_FIELD(details, ECORE_DORQ_ATTENTION_OPAQUE), + GET_FIELD(details, ECORE_DORQ_ATTENTION_SIZE) * 4, + first_drop_reason, all_drops_reason, overflow); + + /* if this PF caused overflow, initiate recovery */ + if (overflow) { + rc = ecore_db_rec_attn(p_hwfn, p_ptt); + if (rc != ECORE_SUCCESS) + return rc; + } + + /* clear the doorbell drop details and prepare for next drop */ + ecore_wr(p_hwfn, p_ptt, DORQ_REG_DB_DROP_DETAILS_REL, 0); + + /* mark interrupt as handeld (note: even if drop was due to a + * different reason than overflow we mark as handled) + */ + ecore_wr(p_hwfn, p_ptt, DORQ_REG_INT_STS_WR, + DORQ_REG_INT_STS_DB_DROP | + DORQ_REG_INT_STS_DORQ_FIFO_OVFL_ERR); + + /* if there are no indications otherthan drop indications, + * success + */ + if ((int_sts & ~(DORQ_REG_INT_STS_DB_DROP | + DORQ_REG_INT_STS_DORQ_FIFO_OVFL_ERR | + DORQ_REG_INT_STS_DORQ_FIFO_AFULL)) == 0) + return ECORE_SUCCESS; } + /* some other indication was present - non recoverable */ + DP_INFO(p_hwfn, "DORQ fatal attention\n"); + return ECORE_INVAL; } @@ -506,7 +574,26 @@ static enum _ecore_status_t ecore_tm_attn_cb(struct ecore_hwfn *p_hwfn) return ECORE_INVAL; } -/* Notice aeu_invert_reg must be defined in the same order of bits as HW; */ +/* Instead of major changes to the data-structure, we have a some 'special' + * identifiers for sources that changed meaning between adapters. + */ +enum aeu_invert_reg_special_type { + AEU_INVERT_REG_SPECIAL_CNIG_0, + AEU_INVERT_REG_SPECIAL_CNIG_1, + AEU_INVERT_REG_SPECIAL_CNIG_2, + AEU_INVERT_REG_SPECIAL_CNIG_3, + AEU_INVERT_REG_SPECIAL_MAX, +}; + +static struct aeu_invert_reg_bit +aeu_descs_special[AEU_INVERT_REG_SPECIAL_MAX] = { + {"CNIG port 0", ATTENTION_SINGLE, OSAL_NULL, BLOCK_CNIG}, + {"CNIG port 1", ATTENTION_SINGLE, OSAL_NULL, BLOCK_CNIG}, + {"CNIG port 2", ATTENTION_SINGLE, OSAL_NULL, BLOCK_CNIG}, + {"CNIG port 3", ATTENTION_SINGLE, OSAL_NULL, BLOCK_CNIG}, +}; + +/* Notice aeu_invert_reg must be defined in the same order of bits as HW; */ static struct aeu_invert_reg aeu_descs[NUM_ATTN_REGS] = { { { /* After Invert 1 */ @@ -519,7 +606,7 @@ static struct aeu_invert_reg aeu_descs[NUM_ATTN_REGS] = { { /* After Invert 2 */ {"PGLUE config_space", ATTENTION_SINGLE, OSAL_NULL, MAX_BLOCK_ID}, {"PGLUE misc_flr", ATTENTION_SINGLE, OSAL_NULL, MAX_BLOCK_ID}, - {"PGLUE B RBC", ATTENTION_PAR_INT, ecore_pglub_rbc_attn_cb, + {"PGLUE B RBC", ATTENTION_PAR_INT, ecore_pglueb_rbc_attn_cb, BLOCK_PGLUE_B}, {"PGLUE misc_mctp", ATTENTION_SINGLE, OSAL_NULL, MAX_BLOCK_ID}, {"Flash event", ATTENTION_SINGLE, OSAL_NULL, MAX_BLOCK_ID}, @@ -549,8 +636,18 @@ static struct aeu_invert_reg aeu_descs[NUM_ATTN_REGS] = { OSAL_NULL, MAX_BLOCK_ID}, {"General Attention 35", ATTENTION_SINGLE | ATTENTION_CLEAR_ENABLE, ecore_general_attention_35, MAX_BLOCK_ID}, - {"CNIG port %d", (4 << ATTENTION_LENGTH_SHIFT), OSAL_NULL, - BLOCK_CNIG}, + {"NWS Parity", ATTENTION_PAR | ATTENTION_BB_DIFFERENT | + ATTENTION_BB(AEU_INVERT_REG_SPECIAL_CNIG_0), + OSAL_NULL, BLOCK_NWS}, + {"NWS Interrupt", ATTENTION_SINGLE | ATTENTION_BB_DIFFERENT | + ATTENTION_BB(AEU_INVERT_REG_SPECIAL_CNIG_1), + OSAL_NULL, BLOCK_NWS}, + {"NWM Parity", ATTENTION_PAR | ATTENTION_BB_DIFFERENT | + ATTENTION_BB(AEU_INVERT_REG_SPECIAL_CNIG_2), + OSAL_NULL, BLOCK_NWM}, + {"NWM Interrupt", ATTENTION_SINGLE | ATTENTION_BB_DIFFERENT | + ATTENTION_BB(AEU_INVERT_REG_SPECIAL_CNIG_3), + OSAL_NULL, BLOCK_NWM}, {"MCP CPU", ATTENTION_SINGLE, ecore_mcp_attn_cb, MAX_BLOCK_ID}, {"MCP Watchdog timer", ATTENTION_SINGLE, OSAL_NULL, MAX_BLOCK_ID}, {"MCP M2P", ATTENTION_SINGLE, OSAL_NULL, MAX_BLOCK_ID}, @@ -559,7 +656,7 @@ static struct aeu_invert_reg aeu_descs[NUM_ATTN_REGS] = { {"MSTAT per-path", ATTENTION_PAR_INT, OSAL_NULL, MAX_BLOCK_ID}, {"Reserved %d", (6 << ATTENTION_LENGTH_SHIFT), OSAL_NULL, MAX_BLOCK_ID}, - {"NIG", ATTENTION_PAR_INT, ecore_nig_attn_cb, BLOCK_NIG}, + {"NIG", ATTENTION_PAR_INT, OSAL_NULL, BLOCK_NIG}, {"BMB/OPTE/MCP", ATTENTION_PAR_INT, OSAL_NULL, BLOCK_BMB}, {"BTB", ATTENTION_PAR_INT, OSAL_NULL, BLOCK_BTB}, {"BRB", ATTENTION_PAR_INT, OSAL_NULL, BLOCK_BRB}, @@ -672,6 +769,27 @@ static struct aeu_invert_reg aeu_descs[NUM_ATTN_REGS] = { }; +static struct aeu_invert_reg_bit * +ecore_int_aeu_translate(struct ecore_hwfn *p_hwfn, + struct aeu_invert_reg_bit *p_bit) +{ + if (!ECORE_IS_BB(p_hwfn->p_dev)) + return p_bit; + + if (!(p_bit->flags & ATTENTION_BB_DIFFERENT)) + return p_bit; + + return &aeu_descs_special[(p_bit->flags & ATTENTION_BB_MASK) >> + ATTENTION_BB_SHIFT]; +} + +static bool ecore_int_is_parity_flag(struct ecore_hwfn *p_hwfn, + struct aeu_invert_reg_bit *p_bit) +{ + return !!(ecore_int_aeu_translate(p_hwfn, p_bit)->flags & + ATTENTION_PARITY); +} + #define ATTN_STATE_BITS (0xfff) #define ATTN_BITS_MASKABLE (0x3ff) struct ecore_sb_attn_info { @@ -761,46 +879,12 @@ static enum _ecore_status_t ecore_int_assertion(struct ecore_hwfn *p_hwfn, return ECORE_SUCCESS; } -static void ecore_int_deassertion_print_bit(struct ecore_hwfn *p_hwfn, - struct attn_hw_reg *p_reg_desc, - struct attn_hw_block *p_block, - enum ecore_attention_type type, - u32 val, u32 mask) +static void ecore_int_attn_print(struct ecore_hwfn *p_hwfn, + enum block_id id, enum dbg_attn_type type, + bool b_clear) { - int j; -#ifdef ATTN_DESC - const char **description; - - if (type == ECORE_ATTN_TYPE_ATTN) - description = p_block->int_desc; - else - description = p_block->prty_desc; -#endif - - for (j = 0; j < p_reg_desc->num_of_bits; j++) { - if (val & (1 << j)) { -#ifdef ATTN_DESC - DP_NOTICE(p_hwfn, false, - "%s (%s): %s [reg %d [0x%08x], bit %d]%s\n", - p_block->name, - type == ECORE_ATTN_TYPE_ATTN ? "Interrupt" : - "Parity", - description[p_reg_desc->bit_attn_idx[j]], - p_reg_desc->reg_idx, - p_reg_desc->sts_addr, j, - (mask & (1 << j)) ? " [MASKED]" : ""); -#else - DP_NOTICE(p_hwfn->p_dev, false, - "%s (%s): [reg %d [0x%08x], bit %d]%s\n", - p_block->name, - type == ECORE_ATTN_TYPE_ATTN ? "Interrupt" : - "Parity", - p_reg_desc->reg_idx, - p_reg_desc->sts_addr, j, - (mask & (1 << j)) ? " [MASKED]" : ""); -#endif - } - } + /* @DPDK */ + DP_NOTICE(p_hwfn->p_dev, false, "[block_id %d type %d]\n", id, type); } /** @@ -818,151 +902,91 @@ static void ecore_int_deassertion_print_bit(struct ecore_hwfn *p_hwfn, static enum _ecore_status_t ecore_int_deassertion_aeu_bit(struct ecore_hwfn *p_hwfn, struct aeu_invert_reg_bit *p_aeu, - u32 aeu_en_reg, u32 bitmask) + u32 aeu_en_reg, + const char *p_bit_name, + u32 bitmask) { enum _ecore_status_t rc = ECORE_INVAL; - u32 val, mask; - -#ifndef REMOVE_DBG - u32 interrupts[20]; /* TODO- change into HSI define once supplied */ - - OSAL_MEMSET(interrupts, 0, sizeof(u32) * 20); /* FIXME real size) */ -#endif + bool b_fatal = false; DP_INFO(p_hwfn, "Deasserted attention `%s'[%08x]\n", - p_aeu->bit_name, bitmask); + p_bit_name, bitmask); /* Call callback before clearing the interrupt status */ if (p_aeu->cb) { DP_INFO(p_hwfn, "`%s (attention)': Calling Callback function\n", - p_aeu->bit_name); + p_bit_name); rc = p_aeu->cb(p_hwfn); } - /* Handle HW block interrupt registers */ + if (rc != ECORE_SUCCESS) + b_fatal = true; + + /* Print HW block interrupt registers */ if (p_aeu->block_index != MAX_BLOCK_ID) { - u16 chip_type = ECORE_GET_TYPE(p_hwfn->p_dev); - struct attn_hw_block *p_block; - int i; - - p_block = &attn_blocks[p_aeu->block_index]; - - /* Handle each interrupt register */ - for (i = 0; - i < p_block->chip_regs[chip_type].num_of_int_regs; i++) { - struct attn_hw_reg *p_reg_desc; - u32 sts_addr; - - p_reg_desc = p_block->chip_regs[chip_type].int_regs[i]; - - /* In case of fatal attention, don't clear the status - * so it would appear in idle check. - */ - if (rc == ECORE_SUCCESS) - sts_addr = p_reg_desc->sts_clr_addr; - else - sts_addr = p_reg_desc->sts_addr; - - val = ecore_rd(p_hwfn, p_hwfn->p_dpc_ptt, sts_addr); - mask = ecore_rd(p_hwfn, p_hwfn->p_dpc_ptt, - p_reg_desc->mask_addr); - ecore_int_deassertion_print_bit(p_hwfn, p_reg_desc, - p_block, - ECORE_ATTN_TYPE_ATTN, - val, mask); - -#ifndef REMOVE_DBG - interrupts[i] = val; -#endif - } - } + ecore_int_attn_print(p_hwfn, p_aeu->block_index, + ATTN_TYPE_INTERRUPT, !b_fatal); +} + /* @DPDK */ /* Reach assertion if attention is fatal */ - if (rc != ECORE_SUCCESS) { + if (b_fatal || (strcmp(p_bit_name, "PGLUE B RBC") == 0)) { DP_NOTICE(p_hwfn, true, "`%s': Fatal attention\n", - p_aeu->bit_name); + p_bit_name); ecore_hw_err_notify(p_hwfn, ECORE_HW_ERR_HW_ATTN); } /* Prevent this Attention from being asserted in the future */ - if (p_aeu->flags & ATTENTION_CLEAR_ENABLE) { + if (p_aeu->flags & ATTENTION_CLEAR_ENABLE || + p_hwfn->p_dev->attn_clr_en) { u32 val; u32 mask = ~bitmask; val = ecore_rd(p_hwfn, p_hwfn->p_dpc_ptt, aeu_en_reg); ecore_wr(p_hwfn, p_hwfn->p_dpc_ptt, aeu_en_reg, (val & mask)); - DP_INFO(p_hwfn, "`%s' - Disabled future attentions\n", - p_aeu->bit_name); - } - - if (p_aeu->flags & (ATTENTION_FW_DUMP | ATTENTION_PANIC_DUMP)) { - /* @@@TODO - what to dump? */ - DP_ERR(p_hwfn->p_dev, "`%s' - Dumps aren't implemented yet\n", - p_aeu->bit_name); - return ECORE_NOTIMPL; + DP_ERR(p_hwfn, "`%s' - Disabled future attentions\n", + p_bit_name); } return rc; } -static void ecore_int_parity_print(struct ecore_hwfn *p_hwfn, - struct aeu_invert_reg_bit *p_aeu, - struct attn_hw_block *p_block, u8 bit_index) -{ - u16 chip_type = ECORE_GET_TYPE(p_hwfn->p_dev); - int i; - - for (i = 0; i < p_block->chip_regs[chip_type].num_of_prty_regs; i++) { - struct attn_hw_reg *p_reg_desc; - u32 val, mask; - - p_reg_desc = p_block->chip_regs[chip_type].prty_regs[i]; - - val = ecore_rd(p_hwfn, p_hwfn->p_dpc_ptt, - p_reg_desc->sts_clr_addr); - mask = ecore_rd(p_hwfn, p_hwfn->p_dpc_ptt, - p_reg_desc->mask_addr); - DP_VERBOSE(p_hwfn, ECORE_MSG_INTR, - "%s[%d] - parity register[%d] is %08x [mask is %08x]\n", - p_aeu->bit_name, bit_index, i, val, mask); - ecore_int_deassertion_print_bit(p_hwfn, p_reg_desc, - p_block, - ECORE_ATTN_TYPE_PARITY, - val, mask); - } -} - /** * @brief ecore_int_deassertion_parity - handle a single parity AEU source * * @param p_hwfn - * @param p_aeu - descriptor of an AEU bit which caused the - * parity + * @param p_aeu - descriptor of an AEU bit which caused the parity + * @param aeu_en_reg - address of the AEU enable register * @param bit_index */ static void ecore_int_deassertion_parity(struct ecore_hwfn *p_hwfn, struct aeu_invert_reg_bit *p_aeu, - u8 bit_index) + u32 aeu_en_reg, u8 bit_index) { - u32 block_id = p_aeu->block_index; + u32 block_id = p_aeu->block_index, mask, val; - DP_INFO(p_hwfn->p_dev, "%s[%d] parity attention is set\n", - p_aeu->bit_name, bit_index); + DP_NOTICE(p_hwfn->p_dev, false, + "%s parity attention is set [address 0x%08x, bit %d]\n", + p_aeu->bit_name, aeu_en_reg, bit_index); if (block_id != MAX_BLOCK_ID) { - ecore_int_parity_print(p_hwfn, p_aeu, &attn_blocks[block_id], - bit_index); + ecore_int_attn_print(p_hwfn, block_id, ATTN_TYPE_PARITY, false); /* In A0, there's a single parity bit for several blocks */ if (block_id == BLOCK_BTB) { - ecore_int_parity_print(p_hwfn, p_aeu, - &attn_blocks[BLOCK_OPTE], - bit_index); - ecore_int_parity_print(p_hwfn, p_aeu, - &attn_blocks[BLOCK_MCP], - bit_index); + ecore_int_attn_print(p_hwfn, BLOCK_OPTE, + ATTN_TYPE_PARITY, false); + ecore_int_attn_print(p_hwfn, BLOCK_MCP, + ATTN_TYPE_PARITY, false); } } + + /* Prevent this parity error from being re-asserted */ + mask = ~(0x1 << bit_index); + val = ecore_rd(p_hwfn, p_hwfn->p_dpc_ptt, aeu_en_reg); + ecore_wr(p_hwfn, p_hwfn->p_dpc_ptt, aeu_en_reg, val & mask); + DP_INFO(p_hwfn, "`%s' - Disabled future parity errors\n", + p_aeu->bit_name); } /** @@ -977,8 +1001,7 @@ static enum _ecore_status_t ecore_int_deassertion(struct ecore_hwfn *p_hwfn, u16 deasserted_bits) { struct ecore_sb_attn_info *sb_attn_sw = p_hwfn->p_sb_attn; - u32 aeu_inv_arr[NUM_ATTN_REGS], aeu_mask; - bool b_parity = false; + u32 aeu_inv_arr[NUM_ATTN_REGS], aeu_mask, aeu_en, en; u8 i, j, k, bit_idx; enum _ecore_status_t rc = ECORE_SUCCESS; @@ -994,11 +1017,11 @@ static enum _ecore_status_t ecore_int_deassertion(struct ecore_hwfn *p_hwfn, /* Handle parity attentions first */ for (i = 0; i < NUM_ATTN_REGS; i++) { struct aeu_invert_reg *p_aeu = &sb_attn_sw->p_aeu_desc[i]; - u32 en = ecore_rd(p_hwfn, p_hwfn->p_dpc_ptt, - MISC_REG_AEU_ENABLE1_IGU_OUT_0 + - i * sizeof(u32)); + u32 parities; - u32 parities = sb_attn_sw->parity_mask[i] & aeu_inv_arr[i] & en; + aeu_en = MISC_REG_AEU_ENABLE1_IGU_OUT_0 + i * sizeof(u32); + en = ecore_rd(p_hwfn, p_hwfn->p_dpc_ptt, aeu_en); + parities = sb_attn_sw->parity_mask[i] & aeu_inv_arr[i] & en; /* Skip register in which no parity bit is currently set */ if (!parities) @@ -1007,12 +1030,10 @@ static enum _ecore_status_t ecore_int_deassertion(struct ecore_hwfn *p_hwfn, for (j = 0, bit_idx = 0; bit_idx < 32; j++) { struct aeu_invert_reg_bit *p_bit = &p_aeu->bits[j]; - if ((p_bit->flags & ATTENTION_PARITY) && - !!(parities & (1 << bit_idx))) { + if (ecore_int_is_parity_flag(p_hwfn, p_bit) && + !!(parities & (1 << bit_idx))) ecore_int_deassertion_parity(p_hwfn, p_bit, - bit_idx); - b_parity = true; - } + aeu_en, bit_idx); bit_idx += ATTENTION_LENGTH(p_bit->flags); } @@ -1027,10 +1048,13 @@ static enum _ecore_status_t ecore_int_deassertion(struct ecore_hwfn *p_hwfn, continue; for (i = 0; i < NUM_ATTN_REGS; i++) { - u32 aeu_en = MISC_REG_AEU_ENABLE1_IGU_OUT_0 + - i * sizeof(u32) + k * sizeof(u32) * NUM_ATTN_REGS; - u32 en = ecore_rd(p_hwfn, p_hwfn->p_dpc_ptt, aeu_en); - u32 bits = aeu_inv_arr[i] & en; + u32 bits; + + aeu_en = MISC_REG_AEU_ENABLE1_IGU_OUT_0 + + i * sizeof(u32) + + k * sizeof(u32) * NUM_ATTN_REGS; + en = ecore_rd(p_hwfn, p_hwfn->p_dpc_ptt, aeu_en); + bits = aeu_inv_arr[i] & en; /* Skip if no bit from this group is currently set */ if (!bits) @@ -1041,29 +1065,59 @@ static enum _ecore_status_t ecore_int_deassertion(struct ecore_hwfn *p_hwfn, * previous assertion. */ for (j = 0, bit_idx = 0; bit_idx < 32; j++) { + unsigned long int bitmask; u8 bit, bit_len; - u32 bitmask; + /* Need to account bits with changed meaning */ p_aeu = &sb_attn_sw->p_aeu_desc[i].bits[j]; - /* No need to handle attention-only bits */ - if (p_aeu->flags == ATTENTION_PAR) - continue; - bit = bit_idx; bit_len = ATTENTION_LENGTH(p_aeu->flags); - if (p_aeu->flags & ATTENTION_PAR_INT) { + if (ecore_int_is_parity_flag(p_hwfn, p_aeu)) { /* Skip Parity */ bit++; bit_len--; } + /* Find the bits relating to HW-block, then + * shift so they'll become LSB. + */ bitmask = bits & (((1 << bit_len) - 1) << bit); + bitmask >>= bit; + if (bitmask) { + u32 flags = p_aeu->flags; + char bit_name[30]; + u8 num; + + num = (u8)OSAL_FIND_FIRST_BIT(&bitmask, + bit_len); + + /* Some bits represent more than a + * a single interrupt. Correctly print + * their name. + */ + if (ATTENTION_LENGTH(flags) > 2 || + ((flags & ATTENTION_PAR_INT) && + ATTENTION_LENGTH(flags) > 1)) + OSAL_SNPRINTF(bit_name, 30, + p_aeu->bit_name, + num); + else + OSAL_STRNCPY(bit_name, + p_aeu->bit_name, + 30); + + /* We now need to pass bitmask in its + * correct position. + */ + bitmask <<= bit; + /* Handle source of the attention */ ecore_int_deassertion_aeu_bit(p_hwfn, p_aeu, aeu_en, + bit_name, bitmask); } @@ -1097,8 +1151,8 @@ static enum _ecore_status_t ecore_int_attentions(struct ecore_hwfn *p_hwfn) struct ecore_sb_attn_info *p_sb_attn_sw = p_hwfn->p_sb_attn; struct atten_status_block *p_sb_attn = p_sb_attn_sw->sb_attn; u16 index = 0, asserted_bits, deasserted_bits; - enum _ecore_status_t rc = ECORE_SUCCESS; u32 attn_bits = 0, attn_acks = 0; + enum _ecore_status_t rc = ECORE_SUCCESS; /* Read current attention bits/acks - safeguard against attentions * by guaranting work on a synchronized timeframe @@ -1170,13 +1224,11 @@ void ecore_int_sp_dpc(osal_int_ptr_t hwfn_cookie) struct ecore_pi_info *pi_info = OSAL_NULL; struct ecore_sb_attn_info *sb_attn; struct ecore_sb_info *sb_info; - static int arr_size; + int arr_size; u16 rc = 0; - if (!p_hwfn) { - DP_ERR(p_hwfn->p_dev, "DPC called - no hwfn!\n"); + if (!p_hwfn) return; - } if (!p_hwfn->p_sp_sb) { DP_ERR(p_hwfn->p_dev, "DPC called - no p_sp_sb\n"); @@ -1237,7 +1289,8 @@ void ecore_int_sp_dpc(osal_int_ptr_t hwfn_cookie) return; } - /* Check the validity of the DPC ptt. If not ack interrupts and fail */ +/* Check the validity of the DPC ptt. If not ack interrupts and fail */ + if (!p_hwfn->p_dpc_ptt) { DP_NOTICE(p_hwfn->p_dev, true, "Failed to allocate PTT\n"); ecore_sb_ack(sb_info, IGU_INT_ENABLE, 1); @@ -1322,12 +1375,13 @@ static void ecore_int_sb_attn_init(struct ecore_hwfn *p_hwfn, for (i = 0; i < NUM_ATTN_REGS; i++) { /* j is array index, k is bit index */ for (j = 0, k = 0; k < 32; j++) { - unsigned int flags = aeu_descs[i].bits[j].flags; + struct aeu_invert_reg_bit *p_aeu; - if (flags & ATTENTION_PARITY) + p_aeu = &aeu_descs[i].bits[j]; + if (ecore_int_is_parity_flag(p_hwfn, p_aeu)) sb_info->parity_mask[i] |= 1 << k; - k += ATTENTION_LENGTH(flags); + k += ATTENTION_LENGTH(p_aeu->flags); } DP_VERBOSE(p_hwfn, ECORE_MSG_INTR, "Attn Mask [Reg %d]: 0x%08x\n", @@ -1350,10 +1404,10 @@ static enum _ecore_status_t ecore_int_sb_attn_alloc(struct ecore_hwfn *p_hwfn, void *p_virt; /* SB struct */ - p_sb = OSAL_ALLOC(p_dev, GFP_KERNEL, sizeof(struct ecore_sb_attn_info)); + p_sb = OSAL_ALLOC(p_dev, GFP_KERNEL, sizeof(*p_sb)); if (!p_sb) { DP_NOTICE(p_dev, true, - "Failed to allocate `struct ecore_sb_attn_info'"); + "Failed to allocate `struct ecore_sb_attn_info'\n"); return ECORE_NOMEM; } @@ -1362,7 +1416,7 @@ static enum _ecore_status_t ecore_int_sb_attn_alloc(struct ecore_hwfn *p_hwfn, SB_ATTN_ALIGNED_SIZE(p_hwfn)); if (!p_virt) { DP_NOTICE(p_dev, true, - "Failed to allocate status block (attentions)"); + "Failed to allocate status block (attentions)\n"); OSAL_FREE(p_dev, p_sb); return ECORE_NOMEM; } @@ -1375,17 +1429,8 @@ static enum _ecore_status_t ecore_int_sb_attn_alloc(struct ecore_hwfn *p_hwfn, } /* coalescing timeout = timeset << (timer_res + 1) */ -#ifdef RTE_LIBRTE_QEDE_RX_COAL_US -#define ECORE_CAU_DEF_RX_USECS RTE_LIBRTE_QEDE_RX_COAL_US -#else #define ECORE_CAU_DEF_RX_USECS 24 -#endif - -#ifdef RTE_LIBRTE_QEDE_TX_COAL_US -#define ECORE_CAU_DEF_TX_USECS RTE_LIBRTE_QEDE_TX_COAL_US -#else #define ECORE_CAU_DEF_TX_USECS 48 -#endif void ecore_init_cau_sb_entry(struct ecore_hwfn *p_hwfn, struct cau_sb_entry *p_sb_entry, @@ -1393,6 +1438,7 @@ void ecore_init_cau_sb_entry(struct ecore_hwfn *p_hwfn, { struct ecore_dev *p_dev = p_hwfn->p_dev; u32 cau_state; + u8 timer_res; OSAL_MEMSET(p_sb_entry, 0, sizeof(*p_sb_entry)); @@ -1402,32 +1448,80 @@ void ecore_init_cau_sb_entry(struct ecore_hwfn *p_hwfn, SET_FIELD(p_sb_entry->params, CAU_SB_ENTRY_SB_TIMESET0, 0x7F); SET_FIELD(p_sb_entry->params, CAU_SB_ENTRY_SB_TIMESET1, 0x7F); - /* setting the time resultion to a fixed value ( = 1) */ - SET_FIELD(p_sb_entry->params, CAU_SB_ENTRY_TIMER_RES0, - ECORE_CAU_DEF_RX_TIMER_RES); - SET_FIELD(p_sb_entry->params, CAU_SB_ENTRY_TIMER_RES1, - ECORE_CAU_DEF_TX_TIMER_RES); - cau_state = CAU_HC_DISABLE_STATE; if (p_dev->int_coalescing_mode == ECORE_COAL_MODE_ENABLE) { cau_state = CAU_HC_ENABLE_STATE; - if (!p_dev->rx_coalesce_usecs) { + if (!p_dev->rx_coalesce_usecs) p_dev->rx_coalesce_usecs = ECORE_CAU_DEF_RX_USECS; - DP_INFO(p_dev, "Coalesce params rx-usecs=%u\n", - p_dev->rx_coalesce_usecs); - } - if (!p_dev->tx_coalesce_usecs) { + if (!p_dev->tx_coalesce_usecs) p_dev->tx_coalesce_usecs = ECORE_CAU_DEF_TX_USECS; - DP_INFO(p_dev, "Coalesce params tx-usecs=%u\n", - p_dev->tx_coalesce_usecs); - } } + /* Coalesce = (timeset << timer-res), timeset is 7bit wide */ + if (p_dev->rx_coalesce_usecs <= 0x7F) + timer_res = 0; + else if (p_dev->rx_coalesce_usecs <= 0xFF) + timer_res = 1; + else + timer_res = 2; + SET_FIELD(p_sb_entry->params, CAU_SB_ENTRY_TIMER_RES0, timer_res); + + if (p_dev->tx_coalesce_usecs <= 0x7F) + timer_res = 0; + else if (p_dev->tx_coalesce_usecs <= 0xFF) + timer_res = 1; + else + timer_res = 2; + SET_FIELD(p_sb_entry->params, CAU_SB_ENTRY_TIMER_RES1, timer_res); + SET_FIELD(p_sb_entry->data, CAU_SB_ENTRY_STATE0, cau_state); SET_FIELD(p_sb_entry->data, CAU_SB_ENTRY_STATE1, cau_state); } +static void _ecore_int_cau_conf_pi(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + u16 igu_sb_id, u32 pi_index, + enum ecore_coalescing_fsm coalescing_fsm, + u8 timeset) +{ + struct cau_pi_entry pi_entry; + u32 sb_offset, pi_offset; + + if (IS_VF(p_hwfn->p_dev)) + return;/* @@@TBD MichalK- VF CAU... */ + + sb_offset = igu_sb_id * PIS_PER_SB_E4; + OSAL_MEMSET(&pi_entry, 0, sizeof(struct cau_pi_entry)); + + SET_FIELD(pi_entry.prod, CAU_PI_ENTRY_PI_TIMESET, timeset); + if (coalescing_fsm == ECORE_COAL_RX_STATE_MACHINE) + SET_FIELD(pi_entry.prod, CAU_PI_ENTRY_FSM_SEL, 0); + else + SET_FIELD(pi_entry.prod, CAU_PI_ENTRY_FSM_SEL, 1); + + pi_offset = sb_offset + pi_index; + if (p_hwfn->hw_init_done) { + ecore_wr(p_hwfn, p_ptt, + CAU_REG_PI_MEMORY + pi_offset * sizeof(u32), + *((u32 *)&(pi_entry))); + } else { + STORE_RT_REG(p_hwfn, + CAU_REG_PI_MEMORY_RT_OFFSET + pi_offset, + *((u32 *)&(pi_entry))); + } +} + +void ecore_int_cau_conf_pi(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + struct ecore_sb_info *p_sb, u32 pi_index, + enum ecore_coalescing_fsm coalescing_fsm, + u8 timeset) +{ + _ecore_int_cau_conf_pi(p_hwfn, p_ptt, p_sb->igu_sb_id, + pi_index, coalescing_fsm, timeset); +} + void ecore_int_cau_conf_sb(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, dma_addr_t sb_phys, u16 igu_sb_id, @@ -1463,58 +1557,42 @@ void ecore_int_cau_conf_sb(struct ecore_hwfn *p_hwfn, /* Configure pi coalescing if set */ if (p_hwfn->p_dev->int_coalescing_mode == ECORE_COAL_MODE_ENABLE) { - u8 num_tc = 1; /* @@@TBD aelior ECORE_MULTI_COS */ - u8 timeset = p_hwfn->p_dev->rx_coalesce_usecs >> - (ECORE_CAU_DEF_RX_TIMER_RES + 1); + /* eth will open queues for all tcs, so configure all of them + * properly, rather than just the active ones + */ + u8 num_tc = p_hwfn->hw_info.num_hw_tc; + + u8 timeset, timer_res; u8 i; - ecore_int_cau_conf_pi(p_hwfn, p_ptt, igu_sb_id, RX_PI, - ECORE_COAL_RX_STATE_MACHINE, timeset); - - timeset = p_hwfn->p_dev->tx_coalesce_usecs >> - (ECORE_CAU_DEF_TX_TIMER_RES + 1); + /* timeset = (coalesce >> timer-res), timeset is 7bit wide */ + if (p_hwfn->p_dev->rx_coalesce_usecs <= 0x7F) + timer_res = 0; + else if (p_hwfn->p_dev->rx_coalesce_usecs <= 0xFF) + timer_res = 1; + else + timer_res = 2; + timeset = (u8)(p_hwfn->p_dev->rx_coalesce_usecs >> timer_res); + _ecore_int_cau_conf_pi(p_hwfn, p_ptt, igu_sb_id, RX_PI, + ECORE_COAL_RX_STATE_MACHINE, + timeset); + if (p_hwfn->p_dev->tx_coalesce_usecs <= 0x7F) + timer_res = 0; + else if (p_hwfn->p_dev->tx_coalesce_usecs <= 0xFF) + timer_res = 1; + else + timer_res = 2; + timeset = (u8)(p_hwfn->p_dev->tx_coalesce_usecs >> timer_res); for (i = 0; i < num_tc; i++) { - ecore_int_cau_conf_pi(p_hwfn, p_ptt, - igu_sb_id, TX_PI(i), - ECORE_COAL_TX_STATE_MACHINE, - timeset); + _ecore_int_cau_conf_pi(p_hwfn, p_ptt, + igu_sb_id, TX_PI(i), + ECORE_COAL_TX_STATE_MACHINE, + timeset); } } } -void ecore_int_cau_conf_pi(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, - u16 igu_sb_id, u32 pi_index, - enum ecore_coalescing_fsm coalescing_fsm, u8 timeset) -{ - struct cau_pi_entry pi_entry; - u32 sb_offset, pi_offset; - - if (IS_VF(p_hwfn->p_dev)) - return; /* @@@TBD MichalK- VF CAU... */ - - sb_offset = igu_sb_id * PIS_PER_SB; - OSAL_MEMSET(&pi_entry, 0, sizeof(struct cau_pi_entry)); - - SET_FIELD(pi_entry.prod, CAU_PI_ENTRY_PI_TIMESET, timeset); - if (coalescing_fsm == ECORE_COAL_RX_STATE_MACHINE) - SET_FIELD(pi_entry.prod, CAU_PI_ENTRY_FSM_SEL, 0); - else - SET_FIELD(pi_entry.prod, CAU_PI_ENTRY_FSM_SEL, 1); - - pi_offset = sb_offset + pi_index; - if (p_hwfn->hw_init_done) { - ecore_wr(p_hwfn, p_ptt, - CAU_REG_PI_MEMORY + pi_offset * sizeof(u32), - *((u32 *)&(pi_entry))); - } else { - STORE_RT_REG(p_hwfn, - CAU_REG_PI_MEMORY_RT_OFFSET + pi_offset, - *((u32 *)&(pi_entry))); - } -} - void ecore_int_sb_setup(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, struct ecore_sb_info *sb_info) { @@ -1527,16 +1605,50 @@ void ecore_int_sb_setup(struct ecore_hwfn *p_hwfn, sb_info->igu_sb_id, 0, 0); } -/** - * @brief ecore_get_igu_sb_id - given a sw sb_id return the - * igu_sb_id - * - * @param p_hwfn - * @param sb_id - * - * @return u16 - */ -static u16 ecore_get_igu_sb_id(struct ecore_hwfn *p_hwfn, u16 sb_id) +struct ecore_igu_block * +ecore_get_igu_free_sb(struct ecore_hwfn *p_hwfn, bool b_is_pf) +{ + struct ecore_igu_block *p_block; + u16 igu_id; + + for (igu_id = 0; igu_id < ECORE_MAPPING_MEMORY_SIZE(p_hwfn->p_dev); + igu_id++) { + p_block = &p_hwfn->hw_info.p_igu_info->entry[igu_id]; + + if (!(p_block->status & ECORE_IGU_STATUS_VALID) || + !(p_block->status & ECORE_IGU_STATUS_FREE)) + continue; + + if (!!(p_block->status & ECORE_IGU_STATUS_PF) == + b_is_pf) + return p_block; + } + + return OSAL_NULL; +} + +static u16 ecore_get_pf_igu_sb_id(struct ecore_hwfn *p_hwfn, + u16 vector_id) +{ + struct ecore_igu_block *p_block; + u16 igu_id; + + for (igu_id = 0; igu_id < ECORE_MAPPING_MEMORY_SIZE(p_hwfn->p_dev); + igu_id++) { + p_block = &p_hwfn->hw_info.p_igu_info->entry[igu_id]; + + if (!(p_block->status & ECORE_IGU_STATUS_VALID) || + !p_block->is_pf || + p_block->vector_number != vector_id) + continue; + + return igu_id; + } + + return ECORE_SB_INVALID_IDX; +} + +u16 ecore_get_igu_sb_id(struct ecore_hwfn *p_hwfn, u16 sb_id) { u16 igu_sb_id; @@ -1544,11 +1656,15 @@ static u16 ecore_get_igu_sb_id(struct ecore_hwfn *p_hwfn, u16 sb_id) if (sb_id == ECORE_SP_SB_ID) igu_sb_id = p_hwfn->hw_info.p_igu_info->igu_dsb_id; else if (IS_PF(p_hwfn->p_dev)) - igu_sb_id = sb_id + p_hwfn->hw_info.p_igu_info->igu_base_sb; + igu_sb_id = ecore_get_pf_igu_sb_id(p_hwfn, sb_id + 1); else igu_sb_id = ecore_vf_get_igu_sb_id(p_hwfn, sb_id); - if (sb_id == ECORE_SP_SB_ID) + if (igu_sb_id == ECORE_SB_INVALID_IDX) + DP_NOTICE(p_hwfn, true, + "Slowpath SB vector %04x doesn't exist\n", + sb_id); + else if (sb_id == ECORE_SP_SB_ID) DP_VERBOSE(p_hwfn, ECORE_MSG_INTR, "Slowpath SB index in IGU is 0x%04x\n", igu_sb_id); else @@ -1569,9 +1685,24 @@ enum _ecore_status_t ecore_int_sb_init(struct ecore_hwfn *p_hwfn, sb_info->igu_sb_id = ecore_get_igu_sb_id(p_hwfn, sb_id); + if (sb_info->igu_sb_id == ECORE_SB_INVALID_IDX) + return ECORE_INVAL; + + /* Let the igu info reference the client's SB info */ if (sb_id != ECORE_SP_SB_ID) { - p_hwfn->sbs_info[sb_id] = sb_info; - p_hwfn->num_sbs++; + if (IS_PF(p_hwfn->p_dev)) { + struct ecore_igu_info *p_info; + struct ecore_igu_block *p_block; + + p_info = p_hwfn->hw_info.p_igu_info; + p_block = &p_info->entry[sb_info->igu_sb_id]; + + p_block->sb_info = sb_info; + p_block->status &= ~ECORE_IGU_STATUS_FREE; + p_info->usage.free_cnt--; + } else { + ecore_vf_set_sb_info(p_hwfn, sb_id, sb_info); + } } #ifdef ECORE_CONFIG_DIRECT_HWFN sb_info->p_hwfn = p_hwfn; @@ -1603,20 +1734,35 @@ enum _ecore_status_t ecore_int_sb_release(struct ecore_hwfn *p_hwfn, struct ecore_sb_info *sb_info, u16 sb_id) { - if (sb_id == ECORE_SP_SB_ID) { - DP_ERR(p_hwfn, "Do Not free sp sb using this function"); - return ECORE_INVAL; - } + struct ecore_igu_info *p_info; + struct ecore_igu_block *p_block; + + if (sb_info == OSAL_NULL) + return ECORE_SUCCESS; /* zero status block and ack counter */ sb_info->sb_ack = 0; OSAL_MEMSET(sb_info->sb_virt, 0, sizeof(*sb_info->sb_virt)); - if (p_hwfn->sbs_info[sb_id] != OSAL_NULL) { - p_hwfn->sbs_info[sb_id] = OSAL_NULL; - p_hwfn->num_sbs--; + if (IS_VF(p_hwfn->p_dev)) { + ecore_vf_set_sb_info(p_hwfn, sb_id, OSAL_NULL); + return ECORE_SUCCESS; } + p_info = p_hwfn->hw_info.p_igu_info; + p_block = &p_info->entry[sb_info->igu_sb_id]; + + /* Vector 0 is reserved to Default SB */ + if (p_block->vector_number == 0) { + DP_ERR(p_hwfn, "Do Not free sp sb using this function"); + return ECORE_INVAL; + } + + /* Lose reference to client's SB info, and fix counters */ + p_block->sb_info = OSAL_NULL; + p_block->status |= ECORE_IGU_STATUS_FREE; + p_info->usage.free_cnt++; + return ECORE_SUCCESS; } @@ -1647,10 +1793,10 @@ static enum _ecore_status_t ecore_int_sp_sb_alloc(struct ecore_hwfn *p_hwfn, /* SB struct */ p_sb = OSAL_ALLOC(p_hwfn->p_dev, GFP_KERNEL, - sizeof(struct ecore_sb_sp_info)); + sizeof(*p_sb)); if (!p_sb) { DP_NOTICE(p_hwfn, true, - "Failed to allocate `struct ecore_sb_info'"); + "Failed to allocate `struct ecore_sb_info'\n"); return ECORE_NOMEM; } @@ -1658,7 +1804,7 @@ static enum _ecore_status_t ecore_int_sp_sb_alloc(struct ecore_hwfn *p_hwfn, p_virt = OSAL_DMA_ALLOC_COHERENT(p_hwfn->p_dev, &p_phys, SB_ALIGNED_SIZE(p_hwfn)); if (!p_virt) { - DP_NOTICE(p_hwfn, true, "Failed to allocate status block"); + DP_NOTICE(p_hwfn, true, "Failed to allocate status block\n"); OSAL_FREE(p_hwfn->p_dev, p_sb); return ECORE_NOMEM; } @@ -1719,14 +1865,14 @@ void ecore_int_igu_enable_int(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, enum ecore_int_mode int_mode) { - u32 igu_pf_conf = IGU_PF_CONF_FUNC_EN; + u32 igu_pf_conf = IGU_PF_CONF_FUNC_EN | IGU_PF_CONF_ATTN_BIT_EN; #ifndef ASIC_ONLY - if (CHIP_REV_IS_FPGA(p_hwfn->p_dev)) + if (CHIP_REV_IS_FPGA(p_hwfn->p_dev)) { DP_INFO(p_hwfn, "FPGA - don't enable ATTN generation in IGU\n"); - else + igu_pf_conf &= ~IGU_PF_CONF_ATTN_BIT_EN; + } #endif - igu_pf_conf |= IGU_PF_CONF_ATTN_BIT_EN; p_hwfn->p_dev->int_mode = int_mode; switch (p_hwfn->p_dev->int_mode) { @@ -1767,6 +1913,7 @@ static void ecore_int_igu_enable_attn(struct ecore_hwfn *p_hwfn, ecore_wr(p_hwfn, p_ptt, IGU_REG_TRAILING_EDGE_LATCH, 0xfff); ecore_wr(p_hwfn, p_ptt, IGU_REG_ATTENTION_ENABLE, 0xfff); + /* Flush the writes to IGU */ OSAL_MMIOWB(p_hwfn->p_dev); /* Unmask AEU signals toward IGU */ @@ -1775,34 +1922,9 @@ static void ecore_int_igu_enable_attn(struct ecore_hwfn *p_hwfn, enum _ecore_status_t ecore_int_igu_enable(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, - enum ecore_int_mode int_mode) + enum ecore_int_mode int_mode) { enum _ecore_status_t rc = ECORE_SUCCESS; - u32 tmp, reg_addr; - - /* @@@tmp - Mask General HW attentions 0-31, Enable 32-36 */ - tmp = ecore_rd(p_hwfn, p_ptt, MISC_REG_AEU_ENABLE4_IGU_OUT_0); - tmp |= 0xf; - ecore_wr(p_hwfn, p_ptt, MISC_REG_AEU_ENABLE3_IGU_OUT_0, 0); - ecore_wr(p_hwfn, p_ptt, MISC_REG_AEU_ENABLE4_IGU_OUT_0, tmp); - - /* @@@tmp - Starting with MFW 8.2.1.0 we've started hitting AVS stop - * attentions. Since we're waiting for BRCM answer regarding this - * attention, in the meanwhile we simply mask it. - */ - tmp = ecore_rd(p_hwfn, p_ptt, MISC_REG_AEU_ENABLE4_IGU_OUT_0); - tmp &= ~0x800; - ecore_wr(p_hwfn, p_ptt, MISC_REG_AEU_ENABLE4_IGU_OUT_0, tmp); - - /* @@@tmp - Mask interrupt sources - should move to init tool; - * Also, correct for A0 [might still change in B0. - */ - reg_addr = - attn_blocks[BLOCK_BRB].chip_regs[ECORE_GET_TYPE(p_hwfn->p_dev)]. - int_regs[0]->mask_addr; - tmp = ecore_rd(p_hwfn, p_ptt, reg_addr); - tmp |= (1 << 21); /* Was PKT4_LEN_ERROR */ - ecore_wr(p_hwfn, p_ptt, reg_addr, tmp); ecore_int_igu_enable_attn(p_hwfn, p_ptt); @@ -1836,12 +1958,14 @@ void ecore_int_igu_disable_int(struct ecore_hwfn *p_hwfn, } #define IGU_CLEANUP_SLEEP_LENGTH (1000) -void ecore_int_igu_cleanup_sb(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, - u32 sb_id, bool cleanup_set, u16 opaque_fid) +static void ecore_int_igu_cleanup_sb(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + u32 igu_sb_id, + bool cleanup_set, + u16 opaque_fid) { u32 cmd_ctrl = 0, val = 0, sb_bit = 0, sb_bit_addr = 0, data = 0; - u32 pxp_addr = IGU_CMD_INT_ACK_BASE + sb_id; + u32 pxp_addr = IGU_CMD_INT_ACK_BASE + igu_sb_id; u32 sleep_cnt = IGU_CLEANUP_SLEEP_LENGTH; u8 type = 0; /* FIXME MichalS type??? */ @@ -1868,11 +1992,12 @@ void ecore_int_igu_cleanup_sb(struct ecore_hwfn *p_hwfn, ecore_wr(p_hwfn, p_ptt, IGU_REG_COMMAND_REG_CTRL, cmd_ctrl); + /* Flush the write to IGU */ OSAL_MMIOWB(p_hwfn->p_dev); /* calculate where to read the status bit from */ - sb_bit = 1 << (sb_id % 32); - sb_bit_addr = sb_id / 32 * sizeof(u32); + sb_bit = 1 << (igu_sb_id % 32); + sb_bit_addr = igu_sb_id / 32 * sizeof(u32); sb_bit_addr += IGU_REG_CLEANUP_STATUS_0 + (0x80 * type); @@ -1887,35 +2012,60 @@ void ecore_int_igu_cleanup_sb(struct ecore_hwfn *p_hwfn, if (!sleep_cnt) DP_NOTICE(p_hwfn, true, "Timeout waiting for clear status 0x%08x [for sb %d]\n", - val, sb_id); + val, igu_sb_id); } void ecore_int_igu_init_pure_rt_single(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, - u32 sb_id, u16 opaque, bool b_set) + u16 igu_sb_id, u16 opaque, bool b_set) { - int pi; + struct ecore_igu_block *p_block; + int pi, i; + + p_block = &p_hwfn->hw_info.p_igu_info->entry[igu_sb_id]; + DP_VERBOSE(p_hwfn, ECORE_MSG_INTR, + "Cleaning SB [%04x]: func_id= %d is_pf = %d vector_num = 0x%0x\n", + igu_sb_id, p_block->function_id, p_block->is_pf, + p_block->vector_number); /* Set */ if (b_set) - ecore_int_igu_cleanup_sb(p_hwfn, p_ptt, sb_id, 1, opaque); + ecore_int_igu_cleanup_sb(p_hwfn, p_ptt, igu_sb_id, 1, opaque); /* Clear */ - ecore_int_igu_cleanup_sb(p_hwfn, p_ptt, sb_id, 0, opaque); + ecore_int_igu_cleanup_sb(p_hwfn, p_ptt, igu_sb_id, 0, opaque); + + /* Wait for the IGU SB to cleanup */ + for (i = 0; i < IGU_CLEANUP_SLEEP_LENGTH; i++) { + u32 val; + + val = ecore_rd(p_hwfn, p_ptt, + IGU_REG_WRITE_DONE_PENDING + + ((igu_sb_id / 32) * 4)); + if (val & (1 << (igu_sb_id % 32))) + OSAL_UDELAY(10); + else + break; + } + if (i == IGU_CLEANUP_SLEEP_LENGTH) + DP_NOTICE(p_hwfn, true, + "Failed SB[0x%08x] still appearing in WRITE_DONE_PENDING\n", + igu_sb_id); /* Clear the CAU for the SB */ for (pi = 0; pi < 12; pi++) ecore_wr(p_hwfn, p_ptt, - CAU_REG_PI_MEMORY + (sb_id * 12 + pi) * 4, 0); + CAU_REG_PI_MEMORY + (igu_sb_id * 12 + pi) * 4, 0); } void ecore_int_igu_init_pure_rt(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, bool b_set, bool b_slowpath) { - u32 igu_base_sb = p_hwfn->hw_info.p_igu_info->igu_base_sb; - u32 igu_sb_cnt = p_hwfn->hw_info.p_igu_info->igu_sb_cnt; - u32 sb_id = 0, val = 0; + struct ecore_igu_info *p_info = p_hwfn->hw_info.p_igu_info; + struct ecore_igu_block *p_block; + u16 igu_sb_id = 0; + u32 val = 0; /* @@@TBD MichalK temporary... should be moved to init-tool... */ val = ecore_rd(p_hwfn, p_ptt, IGU_REG_BLOCK_CONFIGURATION); @@ -1924,53 +2074,204 @@ void ecore_int_igu_init_pure_rt(struct ecore_hwfn *p_hwfn, ecore_wr(p_hwfn, p_ptt, IGU_REG_BLOCK_CONFIGURATION, val); /* end temporary */ - DP_VERBOSE(p_hwfn, ECORE_MSG_INTR, - "IGU cleaning SBs [%d,...,%d]\n", - igu_base_sb, igu_base_sb + igu_sb_cnt - 1); + for (igu_sb_id = 0; + igu_sb_id < ECORE_MAPPING_MEMORY_SIZE(p_hwfn->p_dev); + igu_sb_id++) { + p_block = &p_info->entry[igu_sb_id]; - for (sb_id = igu_base_sb; sb_id < igu_base_sb + igu_sb_cnt; sb_id++) - ecore_int_igu_init_pure_rt_single(p_hwfn, p_ptt, sb_id, + if (!(p_block->status & ECORE_IGU_STATUS_VALID) || + !p_block->is_pf || + (p_block->status & ECORE_IGU_STATUS_DSB)) + continue; + + ecore_int_igu_init_pure_rt_single(p_hwfn, p_ptt, igu_sb_id, p_hwfn->hw_info.opaque_fid, b_set); + } - if (!b_slowpath) - return; - - sb_id = p_hwfn->hw_info.p_igu_info->igu_dsb_id; - DP_VERBOSE(p_hwfn, ECORE_MSG_INTR, - "IGU cleaning slowpath SB [%d]\n", sb_id); - ecore_int_igu_init_pure_rt_single(p_hwfn, p_ptt, sb_id, - p_hwfn->hw_info.opaque_fid, b_set); + if (b_slowpath) + ecore_int_igu_init_pure_rt_single(p_hwfn, p_ptt, + p_info->igu_dsb_id, + p_hwfn->hw_info.opaque_fid, + b_set); } -static u32 ecore_int_igu_read_cam_block(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, u16 sb_id) +int ecore_int_igu_reset_cam(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt) +{ + struct ecore_igu_info *p_info = p_hwfn->hw_info.p_igu_info; + struct ecore_igu_block *p_block; + int pf_sbs, vf_sbs; + u16 igu_sb_id; + u32 val, rval; + + if (!RESC_NUM(p_hwfn, ECORE_SB)) { + /* We're using an old MFW - have to prevent any switching + * of SBs between PF and VFs as later driver wouldn't be + * able to tell which belongs to which. + */ + p_info->b_allow_pf_vf_change = false; + } else { + /* Use the numbers the MFW have provided - + * don't forget MFW accounts for the default SB as well. + */ + p_info->b_allow_pf_vf_change = true; + + if (p_info->usage.cnt != RESC_NUM(p_hwfn, ECORE_SB) - 1) { + DP_INFO(p_hwfn, + "MFW notifies of 0x%04x PF SBs; IGU indicates of only 0x%04x\n", + RESC_NUM(p_hwfn, ECORE_SB) - 1, + p_info->usage.cnt); + p_info->usage.cnt = RESC_NUM(p_hwfn, ECORE_SB) - 1; + } + + /* TODO - how do we learn about VF SBs from MFW? */ + if (IS_PF_SRIOV(p_hwfn)) { + u16 vfs = p_hwfn->p_dev->p_iov_info->total_vfs; + + if (vfs != p_info->usage.iov_cnt) + DP_VERBOSE(p_hwfn, ECORE_MSG_INTR, + "0x%04x VF SBs in IGU CAM != PCI configuration 0x%04x\n", + p_info->usage.iov_cnt, vfs); + + /* At this point we know how many SBs we have totally + * in IGU + number of PF SBs. So we can validate that + * we'd have sufficient for VF. + */ + if (vfs > p_info->usage.free_cnt + + p_info->usage.free_cnt_iov - + p_info->usage.cnt) { + DP_NOTICE(p_hwfn, true, + "Not enough SBs for VFs - 0x%04x SBs, from which %04x PFs and %04x are required\n", + p_info->usage.free_cnt + + p_info->usage.free_cnt_iov, + p_info->usage.cnt, vfs); + return ECORE_INVAL; + } + } + } + + /* Cap the number of VFs SBs by the number of VFs */ + if (IS_PF_SRIOV(p_hwfn)) + p_info->usage.iov_cnt = p_hwfn->p_dev->p_iov_info->total_vfs; + + /* Mark all SBs as free, now in the right PF/VFs division */ + p_info->usage.free_cnt = p_info->usage.cnt; + p_info->usage.free_cnt_iov = p_info->usage.iov_cnt; + p_info->usage.orig = p_info->usage.cnt; + p_info->usage.iov_orig = p_info->usage.iov_cnt; + + /* We now proceed to re-configure the IGU cam to reflect the initial + * configuration. We can start with the Default SB. + */ + pf_sbs = p_info->usage.cnt; + vf_sbs = p_info->usage.iov_cnt; + + for (igu_sb_id = p_info->igu_dsb_id; + igu_sb_id < ECORE_MAPPING_MEMORY_SIZE(p_hwfn->p_dev); + igu_sb_id++) { + p_block = &p_info->entry[igu_sb_id]; + val = 0; + + if (!(p_block->status & ECORE_IGU_STATUS_VALID)) + continue; + + if (p_block->status & ECORE_IGU_STATUS_DSB) { + p_block->function_id = p_hwfn->rel_pf_id; + p_block->is_pf = 1; + p_block->vector_number = 0; + p_block->status = ECORE_IGU_STATUS_VALID | + ECORE_IGU_STATUS_PF | + ECORE_IGU_STATUS_DSB; + } else if (pf_sbs) { + pf_sbs--; + p_block->function_id = p_hwfn->rel_pf_id; + p_block->is_pf = 1; + p_block->vector_number = p_info->usage.cnt - pf_sbs; + p_block->status = ECORE_IGU_STATUS_VALID | + ECORE_IGU_STATUS_PF | + ECORE_IGU_STATUS_FREE; + } else if (vf_sbs) { + p_block->function_id = + p_hwfn->p_dev->p_iov_info->first_vf_in_pf + + p_info->usage.iov_cnt - vf_sbs; + p_block->is_pf = 0; + p_block->vector_number = 0; + p_block->status = ECORE_IGU_STATUS_VALID | + ECORE_IGU_STATUS_FREE; + vf_sbs--; + } else { + p_block->function_id = 0; + p_block->is_pf = 0; + p_block->vector_number = 0; + } + + SET_FIELD(val, IGU_MAPPING_LINE_FUNCTION_NUMBER, + p_block->function_id); + SET_FIELD(val, IGU_MAPPING_LINE_PF_VALID, p_block->is_pf); + SET_FIELD(val, IGU_MAPPING_LINE_VECTOR_NUMBER, + p_block->vector_number); + + /* VF entries would be enabled when VF is initializaed */ + SET_FIELD(val, IGU_MAPPING_LINE_VALID, p_block->is_pf); + + rval = ecore_rd(p_hwfn, p_ptt, + IGU_REG_MAPPING_MEMORY + + sizeof(u32) * igu_sb_id); + + if (rval != val) { + ecore_wr(p_hwfn, p_ptt, + IGU_REG_MAPPING_MEMORY + + sizeof(u32) * igu_sb_id, + val); + + DP_VERBOSE(p_hwfn, ECORE_MSG_INTR, + "IGU reset: [SB 0x%04x] func_id = %d is_pf = %d vector_num = 0x%x [%08x -> %08x]\n", + igu_sb_id, p_block->function_id, + p_block->is_pf, p_block->vector_number, + rval, val); + } + } + + return 0; +} + +int ecore_int_igu_reset_cam_default(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt) +{ + struct ecore_sb_cnt_info *p_cnt = &p_hwfn->hw_info.p_igu_info->usage; + + /* Return all the usage indications to default prior to the reset; + * The reset expects the !orig to reflect the initial status of the + * SBs, and would re-calculate the originals based on those. + */ + p_cnt->cnt = p_cnt->orig; + p_cnt->free_cnt = p_cnt->orig; + p_cnt->iov_cnt = p_cnt->iov_orig; + p_cnt->free_cnt_iov = p_cnt->iov_orig; + p_cnt->orig = 0; + p_cnt->iov_orig = 0; + + /* TODO - we probably need to re-configure the CAU as well... */ + return ecore_int_igu_reset_cam(p_hwfn, p_ptt); +} + +static void ecore_int_igu_read_cam_block(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + u16 igu_sb_id) { u32 val = ecore_rd(p_hwfn, p_ptt, - IGU_REG_MAPPING_MEMORY + sizeof(u32) * sb_id); + IGU_REG_MAPPING_MEMORY + sizeof(u32) * igu_sb_id); struct ecore_igu_block *p_block; - p_block = &p_hwfn->hw_info.p_igu_info->igu_map.igu_blocks[sb_id]; - - /* stop scanning when hit first invalid PF entry */ - if (!GET_FIELD(val, IGU_MAPPING_LINE_VALID) && - GET_FIELD(val, IGU_MAPPING_LINE_PF_VALID)) - goto out; + p_block = &p_hwfn->hw_info.p_igu_info->entry[igu_sb_id]; /* Fill the block information */ - p_block->status = ECORE_IGU_STATUS_VALID; p_block->function_id = GET_FIELD(val, IGU_MAPPING_LINE_FUNCTION_NUMBER); p_block->is_pf = GET_FIELD(val, IGU_MAPPING_LINE_PF_VALID); p_block->vector_number = GET_FIELD(val, IGU_MAPPING_LINE_VECTOR_NUMBER); - DP_VERBOSE(p_hwfn, ECORE_MSG_INTR, - "IGU_BLOCK: [SB 0x%04x, Value in CAM 0x%08x] func_id = %d" - " is_pf = %d vector_num = 0x%x\n", - sb_id, val, p_block->function_id, p_block->is_pf, - p_block->vector_number); - -out: - return val; + p_block->igu_sb_id = igu_sb_id; } enum _ecore_status_t ecore_int_igu_read_cam(struct ecore_hwfn *p_hwfn, @@ -1978,117 +2279,217 @@ enum _ecore_status_t ecore_int_igu_read_cam(struct ecore_hwfn *p_hwfn, { struct ecore_igu_info *p_igu_info; struct ecore_igu_block *p_block; - u16 sb_id, last_iov_sb_id = 0; - u32 min_vf, max_vf, val; - u16 prev_sb_id = 0xFF; + u32 min_vf = 0, max_vf = 0; + u16 igu_sb_id; - p_hwfn->hw_info.p_igu_info = OSAL_ALLOC(p_hwfn->p_dev, - GFP_KERNEL, - sizeof(*p_igu_info)); + p_hwfn->hw_info.p_igu_info = OSAL_ZALLOC(p_hwfn->p_dev, + GFP_KERNEL, + sizeof(*p_igu_info)); if (!p_hwfn->hw_info.p_igu_info) return ECORE_NOMEM; - - OSAL_MEMSET(p_hwfn->hw_info.p_igu_info, 0, sizeof(*p_igu_info)); - p_igu_info = p_hwfn->hw_info.p_igu_info; - /* Initialize base sb / sb cnt for PFs and VFs */ - p_igu_info->igu_base_sb = 0xffff; - p_igu_info->igu_sb_cnt = 0; - p_igu_info->igu_dsb_id = 0xffff; - p_igu_info->igu_base_sb_iov = 0xffff; + /* Distinguish between existent and onn-existent default SB */ + p_igu_info->igu_dsb_id = ECORE_SB_INVALID_IDX; -#ifdef CONFIG_ECORE_SRIOV - min_vf = p_hwfn->hw_info.first_vf_in_pf; - max_vf = p_hwfn->hw_info.first_vf_in_pf + - p_hwfn->p_dev->sriov_info.total_vfs; -#else - min_vf = 0; - max_vf = 0; -#endif + /* Find the range of VF ids whose SB belong to this PF */ + if (p_hwfn->p_dev->p_iov_info) { + struct ecore_hw_sriov_info *p_iov = p_hwfn->p_dev->p_iov_info; - for (sb_id = 0; sb_id < ECORE_MAPPING_MEMORY_SIZE(p_hwfn->p_dev); - sb_id++) { - p_block = &p_igu_info->igu_map.igu_blocks[sb_id]; - val = ecore_int_igu_read_cam_block(p_hwfn, p_ptt, sb_id); - if (!GET_FIELD(val, IGU_MAPPING_LINE_VALID) && - GET_FIELD(val, IGU_MAPPING_LINE_PF_VALID)) - break; - - if (p_block->is_pf) { - if (p_block->function_id == p_hwfn->rel_pf_id) { - p_block->status |= ECORE_IGU_STATUS_PF; - - if (p_block->vector_number == 0) { - if (p_igu_info->igu_dsb_id == 0xffff) - p_igu_info->igu_dsb_id = sb_id; - } else { - if (p_igu_info->igu_base_sb == 0xffff) { - p_igu_info->igu_base_sb = sb_id; - } else if (prev_sb_id != sb_id - 1) { - DP_NOTICE(p_hwfn->p_dev, false, - "consecutive igu" - " vectors for HWFN" - " %x broken", - p_hwfn->rel_pf_id); - break; - } - prev_sb_id = sb_id; - /* we don't count the default */ - (p_igu_info->igu_sb_cnt)++; - } - } - } else { - if ((p_block->function_id >= min_vf) && - (p_block->function_id < max_vf)) { - /* Available for VFs of this PF */ - if (p_igu_info->igu_base_sb_iov == 0xffff) { - p_igu_info->igu_base_sb_iov = sb_id; - } else if (last_iov_sb_id != sb_id - 1) { - if (!val) - DP_VERBOSE(p_hwfn->p_dev, - ECORE_MSG_INTR, - "First uninited IGU" - " CAM entry at" - " index 0x%04x\n", - sb_id); - else - DP_NOTICE(p_hwfn->p_dev, false, - "Consecutive igu" - " vectors for HWFN" - " %x vfs is broken" - " [jumps from %04x" - " to %04x]\n", - p_hwfn->rel_pf_id, - last_iov_sb_id, - sb_id); - break; - } - p_block->status |= ECORE_IGU_STATUS_FREE; - p_hwfn->hw_info.p_igu_info->free_blks++; - last_iov_sb_id = sb_id; - } - } + min_vf = p_iov->first_vf_in_pf; + max_vf = p_iov->first_vf_in_pf + p_iov->total_vfs; } - p_igu_info->igu_sb_cnt_iov = p_igu_info->free_blks; - DP_VERBOSE(p_hwfn, ECORE_MSG_INTR, - "IGU igu_base_sb=0x%x [IOV 0x%x] igu_sb_cnt=%d [IOV 0x%x] " - "igu_dsb_id=0x%x\n", - p_igu_info->igu_base_sb, p_igu_info->igu_base_sb_iov, - p_igu_info->igu_sb_cnt, p_igu_info->igu_sb_cnt_iov, - p_igu_info->igu_dsb_id); + for (igu_sb_id = 0; + igu_sb_id < ECORE_MAPPING_MEMORY_SIZE(p_hwfn->p_dev); + igu_sb_id++) { + /* Read current entry; Notice it might not belong to this PF */ + ecore_int_igu_read_cam_block(p_hwfn, p_ptt, igu_sb_id); + p_block = &p_igu_info->entry[igu_sb_id]; - if (p_igu_info->igu_base_sb == 0xffff || - p_igu_info->igu_dsb_id == 0xffff || p_igu_info->igu_sb_cnt == 0) { + if ((p_block->is_pf) && + (p_block->function_id == p_hwfn->rel_pf_id)) { + p_block->status = ECORE_IGU_STATUS_PF | + ECORE_IGU_STATUS_VALID | + ECORE_IGU_STATUS_FREE; + + if (p_igu_info->igu_dsb_id != ECORE_SB_INVALID_IDX) + p_igu_info->usage.cnt++; + } else if (!(p_block->is_pf) && + (p_block->function_id >= min_vf) && + (p_block->function_id < max_vf)) { + /* Available for VFs of this PF */ + p_block->status = ECORE_IGU_STATUS_VALID | + ECORE_IGU_STATUS_FREE; + + if (p_igu_info->igu_dsb_id != ECORE_SB_INVALID_IDX) + p_igu_info->usage.iov_cnt++; + } + + /* Mark the First entry belonging to the PF or its VFs + * as the default SB [we'll reset IGU prior to first usage]. + */ + if ((p_block->status & ECORE_IGU_STATUS_VALID) && + (p_igu_info->igu_dsb_id == ECORE_SB_INVALID_IDX)) { + p_igu_info->igu_dsb_id = igu_sb_id; + p_block->status |= ECORE_IGU_STATUS_DSB; + } + + /* While this isn't suitable for all clients, limit number + * of prints by having each PF print only its entries with the + * exception of PF0 which would print everything. + */ + if ((p_block->status & ECORE_IGU_STATUS_VALID) || + (p_hwfn->abs_pf_id == 0)) + DP_VERBOSE(p_hwfn, ECORE_MSG_INTR, + "IGU_BLOCK: [SB 0x%04x] func_id = %d is_pf = %d vector_num = 0x%x\n", + igu_sb_id, p_block->function_id, + p_block->is_pf, p_block->vector_number); + } + + if (p_igu_info->igu_dsb_id == ECORE_SB_INVALID_IDX) { DP_NOTICE(p_hwfn, true, - "IGU CAM returned invalid values igu_base_sb=0x%x " - "igu_sb_cnt=%d igu_dsb_id=0x%x\n", - p_igu_info->igu_base_sb, p_igu_info->igu_sb_cnt, + "IGU CAM returned invalid values igu_dsb_id=0x%x\n", p_igu_info->igu_dsb_id); return ECORE_INVAL; } + /* All non default SB are considered free at this point */ + p_igu_info->usage.free_cnt = p_igu_info->usage.cnt; + p_igu_info->usage.free_cnt_iov = p_igu_info->usage.iov_cnt; + + DP_VERBOSE(p_hwfn, ECORE_MSG_INTR, + "igu_dsb_id=0x%x, num Free SBs - PF: %04x VF: %04x [might change after resource allocation]\n", + p_igu_info->igu_dsb_id, p_igu_info->usage.cnt, + p_igu_info->usage.iov_cnt); + + return ECORE_SUCCESS; +} + +enum _ecore_status_t +ecore_int_igu_relocate_sb(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, + u16 sb_id, bool b_to_vf) +{ + struct ecore_igu_info *p_info = p_hwfn->hw_info.p_igu_info; + struct ecore_igu_block *p_block = OSAL_NULL; + u16 igu_sb_id = 0, vf_num = 0; + u32 val = 0; + + if (IS_VF(p_hwfn->p_dev) || !IS_PF_SRIOV(p_hwfn)) + return ECORE_INVAL; + + if (sb_id == ECORE_SP_SB_ID) + return ECORE_INVAL; + + if (!p_info->b_allow_pf_vf_change) { + DP_INFO(p_hwfn, "Can't relocate SBs as MFW is too old.\n"); + return ECORE_INVAL; + } + + /* If we're moving a SB from PF to VF, the client had to specify + * which vector it wants to move. + */ + if (b_to_vf) { + igu_sb_id = ecore_get_pf_igu_sb_id(p_hwfn, sb_id + 1); + if (igu_sb_id == ECORE_SB_INVALID_IDX) + return ECORE_INVAL; + } + + /* If we're moving a SB from VF to PF, need to validate there isn't + * already a line configured for that vector. + */ + if (!b_to_vf) { + if (ecore_get_pf_igu_sb_id(p_hwfn, sb_id + 1) != + ECORE_SB_INVALID_IDX) + return ECORE_INVAL; + } + + /* We need to validate that the SB can actually be relocated. + * This would also handle the previous case where we've explicitly + * stated which IGU SB needs to move. + */ + for (; igu_sb_id < ECORE_MAPPING_MEMORY_SIZE(p_hwfn->p_dev); + igu_sb_id++) { + p_block = &p_info->entry[igu_sb_id]; + + if (!(p_block->status & ECORE_IGU_STATUS_VALID) || + !(p_block->status & ECORE_IGU_STATUS_FREE) || + (!!(p_block->status & ECORE_IGU_STATUS_PF) != b_to_vf)) { + if (b_to_vf) + return ECORE_INVAL; + else + continue; + } + + break; + } + + if (igu_sb_id == ECORE_MAPPING_MEMORY_SIZE(p_hwfn->p_dev)) { + DP_VERBOSE(p_hwfn, (ECORE_MSG_INTR | ECORE_MSG_IOV), + "Failed to find a free SB to move\n"); + return ECORE_INVAL; + } + + /* At this point, p_block points to the SB we want to relocate */ + if (b_to_vf) { + p_block->status &= ~ECORE_IGU_STATUS_PF; + + /* It doesn't matter which VF number we choose, since we're + * going to disable the line; But let's keep it in range. + */ + vf_num = (u16)p_hwfn->p_dev->p_iov_info->first_vf_in_pf; + + p_block->function_id = (u8)vf_num; + p_block->is_pf = 0; + p_block->vector_number = 0; + + p_info->usage.cnt--; + p_info->usage.free_cnt--; + p_info->usage.iov_cnt++; + p_info->usage.free_cnt_iov++; + + /* TODO - if SBs aren't really the limiting factor, + * then it might not be accurate [in the since that + * we might not need decrement the feature]. + */ + p_hwfn->hw_info.feat_num[ECORE_PF_L2_QUE]--; + p_hwfn->hw_info.feat_num[ECORE_VF_L2_QUE]++; + } else { + p_block->status |= ECORE_IGU_STATUS_PF; + p_block->function_id = p_hwfn->rel_pf_id; + p_block->is_pf = 1; + p_block->vector_number = sb_id + 1; + + p_info->usage.cnt++; + p_info->usage.free_cnt++; + p_info->usage.iov_cnt--; + p_info->usage.free_cnt_iov--; + + p_hwfn->hw_info.feat_num[ECORE_PF_L2_QUE]++; + p_hwfn->hw_info.feat_num[ECORE_VF_L2_QUE]--; + } + + /* Update the IGU and CAU with the new configuration */ + SET_FIELD(val, IGU_MAPPING_LINE_FUNCTION_NUMBER, + p_block->function_id); + SET_FIELD(val, IGU_MAPPING_LINE_PF_VALID, p_block->is_pf); + SET_FIELD(val, IGU_MAPPING_LINE_VALID, p_block->is_pf); + SET_FIELD(val, IGU_MAPPING_LINE_VECTOR_NUMBER, + p_block->vector_number); + + ecore_wr(p_hwfn, p_ptt, + IGU_REG_MAPPING_MEMORY + sizeof(u32) * igu_sb_id, + val); + + ecore_int_cau_conf_sb(p_hwfn, p_ptt, 0, + igu_sb_id, vf_num, + p_block->is_pf ? 0 : 1); + + DP_VERBOSE(p_hwfn, ECORE_MSG_INTR, + "Relocation: [SB 0x%04x] func_id = %d is_pf = %d vector_num = 0x%x\n", + igu_sb_id, p_block->function_id, + p_block->is_pf, p_block->vector_number); + return ECORE_SUCCESS; } @@ -2188,32 +2589,13 @@ void ecore_int_setup(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt) void ecore_int_get_num_sbs(struct ecore_hwfn *p_hwfn, struct ecore_sb_cnt_info *p_sb_cnt_info) { - struct ecore_igu_info *info = p_hwfn->hw_info.p_igu_info; + struct ecore_igu_info *p_igu_info = p_hwfn->hw_info.p_igu_info; - if (!info || !p_sb_cnt_info) + if (!p_igu_info || !p_sb_cnt_info) return; - p_sb_cnt_info->sb_cnt = info->igu_sb_cnt; - p_sb_cnt_info->sb_iov_cnt = info->igu_sb_cnt_iov; - p_sb_cnt_info->sb_free_blk = info->free_blks; -} - -u16 ecore_int_queue_id_from_sb_id(struct ecore_hwfn *p_hwfn, u16 sb_id) -{ - struct ecore_igu_info *p_info = p_hwfn->hw_info.p_igu_info; - - /* Determine origin of SB id */ - if ((sb_id >= p_info->igu_base_sb) && - (sb_id < p_info->igu_base_sb + p_info->igu_sb_cnt)) { - return sb_id - p_info->igu_base_sb; - } else if ((sb_id >= p_info->igu_base_sb_iov) && - (sb_id < p_info->igu_base_sb_iov + p_info->igu_sb_cnt_iov)) { - return sb_id - p_info->igu_base_sb_iov + p_info->igu_sb_cnt; - } - - DP_NOTICE(p_hwfn, true, "SB %d not in range for function\n", - sb_id); - return 0; + OSAL_MEMCPY(p_sb_cnt_info, &p_igu_info->usage, + sizeof(*p_sb_cnt_info)); } void ecore_int_disable_post_isr_release(struct ecore_dev *p_dev) @@ -2223,3 +2605,73 @@ void ecore_int_disable_post_isr_release(struct ecore_dev *p_dev) for_each_hwfn(p_dev, i) p_dev->hwfns[i].b_int_requested = false; } + +void ecore_int_attn_clr_enable(struct ecore_dev *p_dev, bool clr_enable) +{ + p_dev->attn_clr_en = clr_enable; +} + +enum _ecore_status_t ecore_int_set_timer_res(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + u8 timer_res, u16 sb_id, bool tx) +{ + struct cau_sb_entry sb_entry; + enum _ecore_status_t rc; + + if (!p_hwfn->hw_init_done) { + DP_ERR(p_hwfn, "hardware not initialized yet\n"); + return ECORE_INVAL; + } + + rc = ecore_dmae_grc2host(p_hwfn, p_ptt, CAU_REG_SB_VAR_MEMORY + + sb_id * sizeof(u64), + (u64)(osal_uintptr_t)&sb_entry, 2, 0); + if (rc != ECORE_SUCCESS) { + DP_ERR(p_hwfn, "dmae_grc2host failed %d\n", rc); + return rc; + } + + if (tx) + SET_FIELD(sb_entry.params, CAU_SB_ENTRY_TIMER_RES1, timer_res); + else + SET_FIELD(sb_entry.params, CAU_SB_ENTRY_TIMER_RES0, timer_res); + + rc = ecore_dmae_host2grc(p_hwfn, p_ptt, + (u64)(osal_uintptr_t)&sb_entry, + CAU_REG_SB_VAR_MEMORY + + sb_id * sizeof(u64), 2, 0); + if (rc != ECORE_SUCCESS) { + DP_ERR(p_hwfn, "dmae_host2grc failed %d\n", rc); + return rc; + } + + return rc; +} + +enum _ecore_status_t ecore_int_get_sb_dbg(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + struct ecore_sb_info *p_sb, + struct ecore_sb_info_dbg *p_info) +{ + u16 sbid = p_sb->igu_sb_id; + int i; + + if (IS_VF(p_hwfn->p_dev)) + return ECORE_INVAL; + + if (sbid > NUM_OF_SBS(p_hwfn->p_dev)) + return ECORE_INVAL; + + p_info->igu_prod = ecore_rd(p_hwfn, p_ptt, + IGU_REG_PRODUCER_MEMORY + sbid * 4); + p_info->igu_cons = ecore_rd(p_hwfn, p_ptt, + IGU_REG_CONSUMER_MEM + sbid * 4); + + for (i = 0; i < PIS_PER_SB_E4; i++) + p_info->pi[i] = (u16)ecore_rd(p_hwfn, p_ptt, + CAU_REG_PI_MEMORY + + sbid * 4 * PIS_PER_SB_E4 + + i * 4); + + return ECORE_SUCCESS; +} diff --git a/dpdk/drivers/net/qede/base/ecore_int.h b/dpdk/drivers/net/qede/base/ecore_int.h index 17c95213..563051c3 100644 --- a/dpdk/drivers/net/qede/base/ecore_int.h +++ b/dpdk/drivers/net/qede/base/ecore_int.h @@ -19,33 +19,78 @@ #define ECORE_SB_EVENT_MASK 0x0003 #define SB_ALIGNED_SIZE(p_hwfn) \ - ALIGNED_TYPE_SIZE(struct status_block, p_hwfn) + ALIGNED_TYPE_SIZE(struct status_block_e4, p_hwfn) + +#define ECORE_SB_INVALID_IDX 0xffff struct ecore_igu_block { u8 status; #define ECORE_IGU_STATUS_FREE 0x01 #define ECORE_IGU_STATUS_VALID 0x02 #define ECORE_IGU_STATUS_PF 0x04 +#define ECORE_IGU_STATUS_DSB 0x08 u8 vector_number; u8 function_id; u8 is_pf; -}; -struct ecore_igu_map { - struct ecore_igu_block igu_blocks[MAX_TOT_SB_PER_PATH]; + /* Index inside IGU [meant for back reference] */ + u16 igu_sb_id; + + struct ecore_sb_info *sb_info; }; struct ecore_igu_info { - struct ecore_igu_map igu_map; + struct ecore_igu_block entry[MAX_TOT_SB_PER_PATH]; u16 igu_dsb_id; - u16 igu_base_sb; - u16 igu_base_sb_iov; - u16 igu_sb_cnt; - u16 igu_sb_cnt_iov; - u16 free_blks; + + /* The numbers can shift when using APIs to switch SBs between PF and + * VF. + */ + struct ecore_sb_cnt_info usage; + + /* Determine whether we can shift SBs between VFs and PFs */ + bool b_allow_pf_vf_change; }; +/** + * @brief - Make sure the IGU CAM reflects the resources provided by MFW + * + * @param p_hwfn + * @param p_ptt + */ +int ecore_int_igu_reset_cam(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt); + +/** + * @brief - Make sure IGU CAM reflects the default resources once again, + * starting with a 'dirty' SW database. + * @param p_hwfn + * @param p_ptt + */ +int ecore_int_igu_reset_cam_default(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt); + +/** + * @brief Translate the weakly-defined client sb-id into an IGU sb-id + * + * @param p_hwfn + * @param sb_id - user provided sb_id + * + * @return an index inside IGU CAM where the SB resides + */ +u16 ecore_get_igu_sb_id(struct ecore_hwfn *p_hwfn, u16 sb_id); + +/** + * @brief return a pointer to an unused valid SB + * + * @param p_hwfn + * @param b_is_pf - true iff we want a SB belonging to a PF + * + * @return point to an igu_block, OSAL_NULL if none is available + */ +struct ecore_igu_block * +ecore_get_igu_free_sb(struct ecore_hwfn *p_hwfn, bool b_is_pf); /* TODO Names of function may change... */ void ecore_int_igu_init_pure_rt(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, @@ -115,22 +160,6 @@ enum _ecore_status_t ecore_int_unregister_cb(struct ecore_hwfn *p_hwfn, u8 pi); */ u16 ecore_int_get_sp_sb_id(struct ecore_hwfn *p_hwfn); -/** - * @brief Status block cleanup. Should be called for each status - * block that will be used -> both PF / VF - * - * @param p_hwfn - * @param p_ptt - * @param sb_id - igu status block id - * @param cleanup_set - set(1) / clear(0) - * @param opaque_fid - the function for which to perform - * cleanup, for example a PF on behalf of - * its VFs. - */ -void ecore_int_igu_cleanup_sb(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, - u32 sb_id, bool cleanup_set, u16 opaque_fid); - /** * @brief Status block cleanup. Should be called for each status * block that will be used -> both PF / VF @@ -141,9 +170,11 @@ void ecore_int_igu_cleanup_sb(struct ecore_hwfn *p_hwfn, * @param opaque - opaque fid of the sb owner. * @param cleanup_set - set(1) / clear(0) */ -void ecore_int_igu_init_pure_rt_single(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, - u32 sb_id, u16 opaque, bool b_set); +void ecore_int_igu_init_pure_rt_single(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + u16 sb_id, + u16 opaque, + bool b_set); /** * @brief ecore_int_cau_conf - configure cau for a given status @@ -169,8 +200,8 @@ void ecore_int_cau_conf_sb(struct ecore_hwfn *p_hwfn, * * @return enum _ecore_status_t */ -enum _ecore_status_t ecore_int_alloc(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt); +enum _ecore_status_t ecore_int_alloc(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt); /** * @brief ecore_int_free @@ -187,16 +218,6 @@ void ecore_int_free(struct ecore_hwfn *p_hwfn); */ void ecore_int_setup(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt); -/** - * @brief - Returns an Rx queue index appropriate for usage with given SB. - * - * @param p_hwfn - * @param sb_id - absolute index of SB - * - * @return index of Rx queue - */ -u16 ecore_int_queue_id_from_sb_id(struct ecore_hwfn *p_hwfn, u16 sb_id); - /** * @brief - Enable Interrupt & Attention for hw function * @@ -223,6 +244,9 @@ void ecore_init_cau_sb_entry(struct ecore_hwfn *p_hwfn, struct cau_sb_entry *p_sb_entry, u8 pf_id, u16 vf_number, u8 vf_valid); +enum _ecore_status_t ecore_int_set_timer_res(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + u8 timer_res, u16 sb_id, bool tx); #ifndef ASIC_ONLY #define ECORE_MAPPING_MEMORY_SIZE(dev) \ ((CHIP_REV_IS_SLOW(dev) && (!(dev)->b_is_emul_full)) ? \ @@ -231,4 +255,7 @@ void ecore_init_cau_sb_entry(struct ecore_hwfn *p_hwfn, #define ECORE_MAPPING_MEMORY_SIZE(dev) NUM_OF_SBS(dev) #endif +enum _ecore_status_t ecore_pglueb_rbc_attn_handler(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt); + #endif /* __ECORE_INT_H__ */ diff --git a/dpdk/drivers/net/qede/base/ecore_int_api.h b/dpdk/drivers/net/qede/base/ecore_int_api.h index f6db807e..24cdf5ed 100644 --- a/dpdk/drivers/net/qede/base/ecore_int_api.h +++ b/dpdk/drivers/net/qede/base/ecore_int_api.h @@ -26,7 +26,7 @@ enum ecore_int_mode { #endif struct ecore_sb_info { - struct status_block *sb_virt; + struct status_block_e4 *sb_virt; dma_addr_t sb_phys; u32 sb_ack; /* Last given ack */ u16 igu_sb_id; @@ -41,10 +41,22 @@ struct ecore_sb_info { struct ecore_dev *p_dev; }; +struct ecore_sb_info_dbg { + u32 igu_prod; + u32 igu_cons; + u16 pi[PIS_PER_SB_E4]; +}; + struct ecore_sb_cnt_info { - int sb_cnt; - int sb_iov_cnt; - int sb_free_blk; + /* Original, current, and free SBs for PF */ + int orig; + int cnt; + int free_cnt; + + /* Original, current and free SBS for child VFs */ + int iov_orig; + int iov_cnt; + int free_cnt_iov; }; static OSAL_INLINE u16 ecore_sb_update_sb_idx(struct ecore_sb_info *sb_info) @@ -55,7 +67,7 @@ static OSAL_INLINE u16 ecore_sb_update_sb_idx(struct ecore_sb_info *sb_info) /* barrier(); status block is written to by the chip */ /* FIXME: need some sort of barrier. */ prod = OSAL_LE32_TO_CPU(sb_info->sb_virt->prod_index) & - STATUS_BLOCK_PROD_INDEX_MASK; + STATUS_BLOCK_E4_PROD_INDEX_MASK; if (sb_info->sb_ack != prod) { sb_info->sb_ack = prod; rc |= ECORE_SB_IDX; @@ -108,7 +120,7 @@ static OSAL_INLINE void __internal_ram_wr(struct ecore_hwfn *p_hwfn, void OSAL_IOMEM *addr, int size, u32 *data) #else -static OSAL_INLINE void __internal_ram_wr(void *p_hwfn, +static OSAL_INLINE void __internal_ram_wr(__rte_unused void *p_hwfn, void OSAL_IOMEM *addr, int size, u32 *data) #endif @@ -120,19 +132,37 @@ static OSAL_INLINE void __internal_ram_wr(void *p_hwfn, } #ifdef ECORE_CONFIG_DIRECT_HWFN -static OSAL_INLINE void internal_ram_wr(struct ecore_hwfn *p_hwfn, - void OSAL_IOMEM *addr, - int size, u32 *data) +static OSAL_INLINE void __internal_ram_wr_relaxed(struct ecore_hwfn *p_hwfn, + void OSAL_IOMEM * addr, + int size, u32 *data) +#else +static OSAL_INLINE void __internal_ram_wr_relaxed(__rte_unused void *p_hwfn, + void OSAL_IOMEM * addr, + int size, u32 *data) +#endif { - __internal_ram_wr(p_hwfn, addr, size, data); + unsigned int i; + + for (i = 0; i < size / sizeof(*data); i++) + DIRECT_REG_WR_RELAXED(p_hwfn, &((u32 OSAL_IOMEM *)addr)[i], + data[i]); +} + +#ifdef ECORE_CONFIG_DIRECT_HWFN +static OSAL_INLINE void internal_ram_wr(struct ecore_hwfn *p_hwfn, + void OSAL_IOMEM * addr, + int size, u32 *data) +{ + __internal_ram_wr_relaxed(p_hwfn, addr, size, data); } #else static OSAL_INLINE void internal_ram_wr(void OSAL_IOMEM *addr, - int size, u32 *data) + int size, u32 *data) { - __internal_ram_wr(OSAL_NULL, addr, size, data); + __internal_ram_wr_relaxed(OSAL_NULL, addr, size, data); } #endif + #endif struct ecore_hwfn; @@ -149,17 +179,17 @@ enum ecore_coalescing_fsm { * * @param p_hwfn * @param p_ptt - * @param igu_sb_id + * @param p_sb * @param pi_index * @param state * @param timeset */ -void ecore_int_cau_conf_pi(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, - u16 igu_sb_id, - u32 pi_index, - enum ecore_coalescing_fsm coalescing_fsm, - u8 timeset); +void ecore_int_cau_conf_pi(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + struct ecore_sb_info *p_sb, + u32 pi_index, + enum ecore_coalescing_fsm coalescing_fsm, + u8 timeset); /** * @@ -195,6 +225,7 @@ void ecore_int_igu_disable_int(struct ecore_hwfn *p_hwfn, u64 ecore_int_igu_read_sisr_reg(struct ecore_hwfn *p_hwfn); #define ECORE_SP_SB_ID 0xffff + /** * @brief ecore_int_sb_init - Initializes the sb_info structure. * @@ -274,4 +305,44 @@ void ecore_int_get_num_sbs(struct ecore_hwfn *p_hwfn, */ void ecore_int_disable_post_isr_release(struct ecore_dev *p_dev); +/** + * @brief ecore_int_attn_clr_enable - sets whether the general behavior is + * preventing attentions from being reasserted, or following the + * attributes of the specific attention. + * + * @param p_dev + * @param clr_enable + * + */ +void ecore_int_attn_clr_enable(struct ecore_dev *p_dev, bool clr_enable); + +/** + * @brief Read debug information regarding a given SB. + * + * @param p_hwfn + * @param p_ptt + * @param p_sb - point to Status block for which we want to get info. + * @param p_info - pointer to struct to fill with information regarding SB. + * + * @return ECORE_SUCCESS if pointer is filled; failure otherwise. + */ +enum _ecore_status_t ecore_int_get_sb_dbg(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + struct ecore_sb_info *p_sb, + struct ecore_sb_info_dbg *p_info); + +/** + * @brief - Move a free Status block between PF and child VF + * + * @param p_hwfn + * @param p_ptt + * @param sb_id - The PF fastpath vector to be moved [re-assigned if claiming + * from VF, given-up if moving to VF] + * @param b_to_vf - PF->VF == true, VF->PF == false + * + * @return ECORE_SUCCESS if SB successfully moved. + */ +enum _ecore_status_t +ecore_int_igu_relocate_sb(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, + u16 sb_id, bool b_to_vf); #endif diff --git a/dpdk/drivers/net/qede/base/ecore_iov_api.h b/dpdk/drivers/net/qede/base/ecore_iov_api.h index b34a9c6b..218ef50b 100644 --- a/dpdk/drivers/net/qede/base/ecore_iov_api.h +++ b/dpdk/drivers/net/qede/base/ecore_iov_api.h @@ -9,34 +9,50 @@ #ifndef __ECORE_SRIOV_API_H__ #define __ECORE_SRIOV_API_H__ +#include "common_hsi.h" #include "ecore_status.h" +#define ECORE_ETH_VF_NUM_MAC_FILTERS 1 +#define ECORE_ETH_VF_NUM_VLAN_FILTERS 2 #define ECORE_VF_ARRAY_LENGTH (3) #define IS_VF(p_dev) ((p_dev)->b_is_vf) #define IS_PF(p_dev) (!((p_dev)->b_is_vf)) #ifdef CONFIG_ECORE_SRIOV -#define IS_PF_SRIOV(p_hwfn) (!!((p_hwfn)->p_dev->sriov_info.total_vfs)) +#define IS_PF_SRIOV(p_hwfn) (!!((p_hwfn)->p_dev->p_iov_info)) #else #define IS_PF_SRIOV(p_hwfn) (0) #endif #define IS_PF_SRIOV_ALLOC(p_hwfn) (!!((p_hwfn)->pf_iov_info)) -#define IS_PF_PDA(p_hwfn) 0 /* @@TBD Michalk */ +#define IS_PF_PDA(p_hwfn) 0 /* @@TBD Michalk */ /* @@@ TBD MichalK - what should this number be*/ #define ECORE_MAX_VF_CHAINS_PER_PF 16 /* vport update extended feature tlvs flags */ enum ecore_iov_vport_update_flag { - ECORE_IOV_VP_UPDATE_ACTIVATE = 0, - ECORE_IOV_VP_UPDATE_VLAN_STRIP = 1, - ECORE_IOV_VP_UPDATE_TX_SWITCH = 2, - ECORE_IOV_VP_UPDATE_MCAST = 3, - ECORE_IOV_VP_UPDATE_ACCEPT_PARAM = 4, - ECORE_IOV_VP_UPDATE_RSS = 5, - ECORE_IOV_VP_UPDATE_ACCEPT_ANY_VLAN = 6, - ECORE_IOV_VP_UPDATE_SGE_TPA = 7, - ECORE_IOV_VP_UPDATE_MAX = 8, + ECORE_IOV_VP_UPDATE_ACTIVATE = 0, + ECORE_IOV_VP_UPDATE_VLAN_STRIP = 1, + ECORE_IOV_VP_UPDATE_TX_SWITCH = 2, + ECORE_IOV_VP_UPDATE_MCAST = 3, + ECORE_IOV_VP_UPDATE_ACCEPT_PARAM = 4, + ECORE_IOV_VP_UPDATE_RSS = 5, + ECORE_IOV_VP_UPDATE_ACCEPT_ANY_VLAN = 6, + ECORE_IOV_VP_UPDATE_SGE_TPA = 7, + ECORE_IOV_VP_UPDATE_MAX = 8, +}; + +/* PF to VF STATUS is part of vfpf-channel API + * and must be forward compatible +*/ +enum ecore_iov_pf_to_vf_status { + PFVF_STATUS_WAITING = 0, + PFVF_STATUS_SUCCESS, + PFVF_STATUS_FAILURE, + PFVF_STATUS_NOT_SUPPORTED, + PFVF_STATUS_NO_RESOURCE, + PFVF_STATUS_FORCED, + PFVF_STATUS_MALICIOUS, }; struct ecore_mcp_link_params; @@ -53,6 +69,14 @@ struct ecore_mcp_link_capabilities; struct ecore_vf_acquire_sw_info { u32 driver_version; u8 os_type; + + /* We have several close releases that all use ~same FW with different + * versions [making it incompatible as the versioning scheme is still + * tied directly to FW version], allow to override the checking. Only + * those versions would actually support this feature [so it would not + * break forward compatibility with newer HV drivers that are no longer + * suited]. + */ bool override_fw_version; }; @@ -64,24 +88,46 @@ struct ecore_public_vf_info { u16 forced_vlan; }; +struct ecore_iov_vf_init_params { + u16 rel_vf_id; + + /* Number of requested Queues; Currently, don't support different + * number of Rx/Tx queues. + */ + /* TODO - remove this limitation */ + u16 num_queues; + + /* Allow the client to choose which qzones to use for Rx/Tx, + * and which queue_base to use for Tx queues on a per-queue basis. + * Notice values should be relative to the PF resources. + */ + u16 req_rx_queue[ECORE_MAX_VF_CHAINS_PER_PF]; + u16 req_tx_queue[ECORE_MAX_VF_CHAINS_PER_PF]; + + u8 vport_id; + + /* Should be set in case RSS is going to be used for VF */ + u8 rss_eng_id; +}; + #ifdef CONFIG_ECORE_SW_CHANNEL /* This is SW channel related only... */ enum mbx_state { - VF_PF_UNKNOWN_STATE = 0, - VF_PF_WAIT_FOR_START_REQUEST = 1, - VF_PF_WAIT_FOR_NEXT_CHUNK_OF_REQUEST = 2, - VF_PF_REQUEST_IN_PROCESSING = 3, - VF_PF_RESPONSE_READY = 4, + VF_PF_UNKNOWN_STATE = 0, + VF_PF_WAIT_FOR_START_REQUEST = 1, + VF_PF_WAIT_FOR_NEXT_CHUNK_OF_REQUEST = 2, + VF_PF_REQUEST_IN_PROCESSING = 3, + VF_PF_RESPONSE_READY = 4, }; struct ecore_iov_sw_mbx { - enum mbx_state mbx_state; + enum mbx_state mbx_state; - u32 request_size; - u32 request_offset; + u32 request_size; + u32 request_offset; - u32 response_size; - u32 response_offset; + u32 response_size; + u32 response_offset; }; /** @@ -92,32 +138,58 @@ struct ecore_iov_sw_mbx { * * @return struct ecore_iov_sw_mbx* */ -struct ecore_iov_sw_mbx *ecore_iov_get_vf_sw_mbx(struct ecore_hwfn *p_hwfn, - u16 rel_vf_id); +struct ecore_iov_sw_mbx* +ecore_iov_get_vf_sw_mbx(struct ecore_hwfn *p_hwfn, + u16 rel_vf_id); #endif +/* This struct is part of ecore_dev and contains data relevant to all hwfns; + * Initialized only if SR-IOV cpabability is exposed in PCIe config space. + */ +struct ecore_hw_sriov_info { + /* standard SRIOV capability fields, mostly for debugging */ + int pos; /* capability position */ + int nres; /* number of resources */ + u32 cap; /* SR-IOV Capabilities */ + u16 ctrl; /* SR-IOV Control */ + u16 total_vfs; /* total VFs associated with the PF */ + u16 num_vfs; /* number of vfs that have been started */ + u16 initial_vfs; /* initial VFs associated with the PF */ + u16 nr_virtfn; /* number of VFs available */ + u16 offset; /* first VF Routing ID offset */ + u16 stride; /* following VF stride */ + u16 vf_device_id; /* VF device id */ + u32 pgsz; /* page size for BAR alignment */ + u8 link; /* Function Dependency Link */ + + u32 first_vf_in_pf; +}; + #ifdef CONFIG_ECORE_SRIOV +#ifndef LINUX_REMOVE /** * @brief mark/clear all VFs before/after an incoming PCIe sriov * disable. * - * @param p_hwfn + * @param p_dev * @param to_disable */ -void ecore_iov_set_vfs_to_disable(struct ecore_hwfn *p_hwfn, u8 to_disable); +void ecore_iov_set_vfs_to_disable(struct ecore_dev *p_dev, + u8 to_disable); /** - * @brief mark/clear chosen VFs before/after an incoming PCIe + * @brief mark/clear chosen VF before/after an incoming PCIe * sriov disable. * - * @param p_hwfn + * @param p_dev + * @param rel_vf_id * @param to_disable */ -void ecore_iov_set_vf_to_disable(struct ecore_hwfn *p_hwfn, - u16 rel_vf_id, u8 to_disable); +void ecore_iov_set_vf_to_disable(struct ecore_dev *p_dev, + u16 rel_vf_id, + u8 to_disable); /** - * * @brief ecore_iov_init_hw_for_vf - initialize the HW for * enabling access of a VF. Also includes preparing the * IGU for VF access. This needs to be called AFTER hw is @@ -125,14 +197,14 @@ void ecore_iov_set_vf_to_disable(struct ecore_hwfn *p_hwfn, * * @param p_hwfn * @param p_ptt - * @param rel_vf_id - * @param num_rx_queues + * @param p_params * * @return enum _ecore_status_t */ enum _ecore_status_t ecore_iov_init_hw_for_vf(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, - u16 rel_vf_id, u16 num_rx_queues); + struct ecore_iov_vf_init_params + *p_params); /** * @brief ecore_iov_process_mbx_req - process a request received @@ -143,7 +215,8 @@ enum _ecore_status_t ecore_iov_init_hw_for_vf(struct ecore_hwfn *p_hwfn, * @param vfid */ void ecore_iov_process_mbx_req(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, int vfid); + struct ecore_ptt *p_ptt, + int vfid); /** * @brief ecore_iov_release_hw_for_vf - called once upper layer @@ -161,7 +234,6 @@ enum _ecore_status_t ecore_iov_release_hw_for_vf(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, u16 rel_vf_id); -#ifndef LINUX_REMOVE /** * @brief ecore_iov_set_vf_ctx - set a context for a given VF * @@ -172,8 +244,8 @@ enum _ecore_status_t ecore_iov_release_hw_for_vf(struct ecore_hwfn *p_hwfn, * @return enum _ecore_status_t */ enum _ecore_status_t ecore_iov_set_vf_ctx(struct ecore_hwfn *p_hwfn, - u16 vf_id, void *ctx); -#endif + u16 vf_id, + void *ctx); /** * @brief FLR cleanup for all VFs @@ -197,7 +269,8 @@ enum _ecore_status_t ecore_iov_vf_flr_cleanup(struct ecore_hwfn *p_hwfn, */ enum _ecore_status_t ecore_iov_single_vf_flr_cleanup(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, u16 rel_vf_id); + struct ecore_ptt *p_ptt, + u16 rel_vf_id); /** * @brief Update the bulletin with link information. Notice this does NOT @@ -238,7 +311,8 @@ void ecore_iov_get_link(struct ecore_hwfn *p_hwfn, * * @return bool */ -bool ecore_iov_is_vf_pending_flr(struct ecore_hwfn *p_hwfn, u16 rel_vf_id); +bool ecore_iov_is_vf_pending_flr(struct ecore_hwfn *p_hwfn, + u16 rel_vf_id); /** * @brief Check if given VF ID @vfid is valid @@ -249,11 +323,13 @@ bool ecore_iov_is_vf_pending_flr(struct ecore_hwfn *p_hwfn, u16 rel_vf_id); * @param p_hwfn * @param rel_vf_id - Relative VF ID * @param b_enabled_only - consider only enabled VF + * @param b_non_malicious - true iff we want to validate vf isn't malicious. * * @return bool - true for valid VF ID */ bool ecore_iov_is_valid_vfid(struct ecore_hwfn *p_hwfn, - int rel_vf_id, bool b_enabled_only); + int rel_vf_id, + bool b_enabled_only, bool b_non_malicious); /** * @brief Get VF's public info structure @@ -264,26 +340,18 @@ bool ecore_iov_is_valid_vfid(struct ecore_hwfn *p_hwfn, * * @return struct ecore_public_vf_info * */ -struct ecore_public_vf_info *ecore_iov_get_public_vf_info(struct ecore_hwfn - *p_hwfn, u16 vfid, - bool b_enabled_only); +struct ecore_public_vf_info* +ecore_iov_get_public_vf_info(struct ecore_hwfn *p_hwfn, + u16 vfid, bool b_enabled_only); /** - * @brief Set pending events bitmap for given @vfid - * - * @param p_hwfn - * @param vfid - */ -void ecore_iov_pf_add_pending_events(struct ecore_hwfn *p_hwfn, u8 vfid); - -/** - * @brief Copy pending events bitmap in @events and clear - * original copy of events + * @brief fills a bitmask of all VFs which have pending unhandled + * messages. * * @param p_hwfn */ -void ecore_iov_pf_get_and_clear_pending_events(struct ecore_hwfn *p_hwfn, - u64 *events); +void ecore_iov_pf_get_pending_events(struct ecore_hwfn *p_hwfn, + u64 *events); /** * @brief Copy VF's message to PF's buffer @@ -295,7 +363,8 @@ void ecore_iov_pf_get_and_clear_pending_events(struct ecore_hwfn *p_hwfn, * @return enum _ecore_status_t */ enum _ecore_status_t ecore_iov_copy_vf_msg(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *ptt, int vfid); + struct ecore_ptt *ptt, + int vfid); /** * @brief Set forced MAC address in PFs copy of bulletin board * and configures FW/HW to support the configuration. @@ -318,17 +387,6 @@ void ecore_iov_bulletin_set_forced_mac(struct ecore_hwfn *p_hwfn, enum _ecore_status_t ecore_iov_bulletin_set_mac(struct ecore_hwfn *p_hwfn, u8 *mac, int vfid); -/** - * @brief Set forced VLAN [pvid] in PFs copy of bulletin board - * and configures FW/HW to support the configuration. - * Setting of pvid 0 would clear the feature. - * @param p_hwfn - * @param pvid - * @param vfid - */ -void ecore_iov_bulletin_set_forced_vlan(struct ecore_hwfn *p_hwfn, - u16 pvid, int vfid); - /** * @brief Set default behaviour of VF in case no vlans are configured for it * whether to accept only untagged traffic or all. @@ -342,7 +400,9 @@ void ecore_iov_bulletin_set_forced_vlan(struct ecore_hwfn *p_hwfn, */ enum _ecore_status_t ecore_iov_bulletin_set_forced_untagged_default(struct ecore_hwfn *p_hwfn, - bool b_untagged_only, int vfid); + bool b_untagged_only, + int vfid); + /** * @brief Get VFs opaque fid. * @@ -354,14 +414,15 @@ void ecore_iov_get_vfs_opaque_fid(struct ecore_hwfn *p_hwfn, int vfid, u16 *opaque_fid); /** - * @brief Get VFs VPORT id. - * + * @brief Set forced VLAN [pvid] in PFs copy of bulletin board + * and configures FW/HW to support the configuration. + * Setting of pvid 0 would clear the feature. * @param p_hwfn + * @param pvid * @param vfid - * @param vport id */ -void ecore_iov_get_vfs_vport_id(struct ecore_hwfn *p_hwfn, int vfid, - u8 *p_vport_id); +void ecore_iov_bulletin_set_forced_vlan(struct ecore_hwfn *p_hwfn, + u16 pvid, int vfid); /** * @brief Check if VF has VPORT instance. This can be used @@ -457,9 +518,9 @@ void ecore_iov_get_vf_req_virt_mbx_params(struct ecore_hwfn *p_hwfn, * @param p_reply_virt_size */ void ecore_iov_get_vf_reply_virt_mbx_params(struct ecore_hwfn *p_hwfn, - u16 rel_vf_id, + u16 rel_vf_id, void **pp_reply_virt_addr, - u16 *p_reply_virt_size); + u16 *p_reply_virt_size); /** * @brief Validate if the given length is a valid vfpf message @@ -486,7 +547,8 @@ u32 ecore_iov_pfvf_msg_length(void); * * @return OSAL_NULL if mac isn't forced; Otherwise, returns MAC. */ -u8 *ecore_iov_bulletin_get_forced_mac(struct ecore_hwfn *p_hwfn, u16 rel_vf_id); +u8 *ecore_iov_bulletin_get_forced_mac(struct ecore_hwfn *p_hwfn, + u16 rel_vf_id); /** * @brief Returns pvid if one is configured @@ -535,7 +597,8 @@ enum _ecore_status_t ecore_iov_get_vf_stats(struct ecore_hwfn *p_hwfn, * * @return num of rxqs chains. */ -u8 ecore_iov_get_vf_num_rxqs(struct ecore_hwfn *p_hwfn, u16 rel_vf_id); +u8 ecore_iov_get_vf_num_rxqs(struct ecore_hwfn *p_hwfn, + u16 rel_vf_id); /** * @brief - Retrieves num of active rxqs chains @@ -545,7 +608,8 @@ u8 ecore_iov_get_vf_num_rxqs(struct ecore_hwfn *p_hwfn, u16 rel_vf_id); * * @return */ -u8 ecore_iov_get_vf_num_active_rxqs(struct ecore_hwfn *p_hwfn, u16 rel_vf_id); +u8 ecore_iov_get_vf_num_active_rxqs(struct ecore_hwfn *p_hwfn, + u16 rel_vf_id); /** * @brief - Retrieves ctx pointer @@ -555,7 +619,8 @@ u8 ecore_iov_get_vf_num_active_rxqs(struct ecore_hwfn *p_hwfn, u16 rel_vf_id); * * @return */ -void *ecore_iov_get_vf_ctx(struct ecore_hwfn *p_hwfn, u16 rel_vf_id); +void *ecore_iov_get_vf_ctx(struct ecore_hwfn *p_hwfn, + u16 rel_vf_id); /** * @brief - Retrieves VF`s num sbs @@ -565,7 +630,8 @@ void *ecore_iov_get_vf_ctx(struct ecore_hwfn *p_hwfn, u16 rel_vf_id); * * @return */ -u8 ecore_iov_get_vf_num_sbs(struct ecore_hwfn *p_hwfn, u16 rel_vf_id); +u8 ecore_iov_get_vf_num_sbs(struct ecore_hwfn *p_hwfn, + u16 rel_vf_id); /** * @brief - Returm true if VF is waiting for acquire @@ -575,7 +641,8 @@ u8 ecore_iov_get_vf_num_sbs(struct ecore_hwfn *p_hwfn, u16 rel_vf_id); * * @return */ -bool ecore_iov_is_vf_wait_for_acquire(struct ecore_hwfn *p_hwfn, u16 rel_vf_id); +bool ecore_iov_is_vf_wait_for_acquire(struct ecore_hwfn *p_hwfn, + u16 rel_vf_id); /** * @brief - Returm true if VF is acquired but not initialized @@ -596,7 +663,19 @@ bool ecore_iov_is_vf_acquired_not_initialized(struct ecore_hwfn *p_hwfn, * * @return */ -bool ecore_iov_is_vf_initialized(struct ecore_hwfn *p_hwfn, u16 rel_vf_id); +bool ecore_iov_is_vf_initialized(struct ecore_hwfn *p_hwfn, + u16 rel_vf_id); + +/** + * @brief - Returm true if VF has started in FW + * + * @param p_hwfn + * @param rel_vf_id + * + * @return + */ +bool ecore_iov_is_vf_started(struct ecore_hwfn *p_hwfn, + u16 rel_vf_id); /** * @brief - Get VF's vport min rate configured. @@ -607,327 +686,59 @@ bool ecore_iov_is_vf_initialized(struct ecore_hwfn *p_hwfn, u16 rel_vf_id); */ int ecore_iov_get_vf_min_rate(struct ecore_hwfn *p_hwfn, int vfid); +#endif + /** - * @brief - Configure min rate for VF's vport. - * @param p_dev - * @param vfid - * @param - rate in Mbps + * @brief ecore_pf_configure_vf_queue_coalesce - PF configure coalesce + * parameters of VFs for Rx and Tx queue. + * While the API allows setting coalescing per-qid, all queues sharing a SB + * should be in same range [i.e., either 0-0x7f, 0x80-0xff or 0x100-0x1ff] + * otherwise configuration would break. * - * @return + * @param p_hwfn + * @param rx_coal - Rx Coalesce value in micro seconds. + * @param tx_coal - TX Coalesce value in micro seconds. + * @param vf_id + * @param qid + * + * @return int + **/ +enum _ecore_status_t +ecore_iov_pf_configure_vf_queue_coalesce(struct ecore_hwfn *p_hwfn, + u16 rx_coal, u16 tx_coal, + u16 vf_id, u16 qid); + +/** + * @brief - Given a VF index, return index of next [including that] active VF. + * + * @param p_hwfn + * @param rel_vf_id + * + * @return MAX_NUM_VFS_E4 in case no further active VFs, otherwise index. */ -enum _ecore_status_t ecore_iov_configure_min_tx_rate(struct ecore_dev *p_dev, - int vfid, u32 rate); -#else -static OSAL_INLINE void ecore_iov_set_vfs_to_disable(struct ecore_hwfn *p_hwfn, - u8 to_disable) -{ -} +u16 ecore_iov_get_next_active_vf(struct ecore_hwfn *p_hwfn, u16 rel_vf_id); -static OSAL_INLINE void ecore_iov_set_vf_to_disable(struct ecore_hwfn *p_hwfn, - u16 rel_vf_id, - u8 to_disable) -{ -} +void ecore_iov_bulletin_set_udp_ports(struct ecore_hwfn *p_hwfn, int vfid, + u16 vxlan_port, u16 geneve_port); -static OSAL_INLINE enum _ecore_status_t ecore_iov_init_hw_for_vf(struct - ecore_hwfn - * p_hwfn, - struct - ecore_ptt - * p_ptt, - u16 rel_vf_id, - u16 - num_rx_queues) -{ - return ECORE_INVAL; -} - -static OSAL_INLINE void ecore_iov_process_mbx_req(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, - int vfid) -{ -} - -static OSAL_INLINE enum _ecore_status_t ecore_iov_release_hw_for_vf(struct - ecore_hwfn - * p_hwfn, - struct - ecore_ptt - * p_ptt, - u16 - rel_vf_id) -{ - return ECORE_SUCCESS; -} - -#ifndef LINUX_REMOVE -static OSAL_INLINE enum _ecore_status_t ecore_iov_set_vf_ctx(struct ecore_hwfn - *p_hwfn, u16 vf_id, - void *ctx) -{ - return ECORE_INVAL; -} +#ifdef CONFIG_ECORE_SW_CHANNEL +/** + * @brief Set whether PF should communicate with VF using SW/HW channel + * Needs to be called for an enabled VF before acquire is over + * [latest good point for doing that is OSAL_IOV_VF_ACQUIRE()] + * + * @param p_hwfn + * @param vfid - relative vf index + * @param b_is_hw - true iff PF is to use HW channel for communication + */ +void ecore_iov_set_vf_hw_channel(struct ecore_hwfn *p_hwfn, int vfid, + bool b_is_hw); #endif -static OSAL_INLINE enum _ecore_status_t ecore_iov_vf_flr_cleanup(struct - ecore_hwfn - * p_hwfn, - struct - ecore_ptt - * p_ptt) -{ - return ECORE_INVAL; -} +#endif /* CONFIG_ECORE_SRIOV */ -static OSAL_INLINE enum _ecore_status_t ecore_iov_single_vf_flr_cleanup( - struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, u16 rel_vf_id) -{ - return ECORE_INVAL; -} +#define ecore_for_each_vf(_p_hwfn, _i) \ + for (_i = ecore_iov_get_next_active_vf(_p_hwfn, 0); \ + _i < MAX_NUM_VFS_E4; \ + _i = ecore_iov_get_next_active_vf(_p_hwfn, _i + 1)) -static OSAL_INLINE void ecore_iov_set_link(struct ecore_hwfn *p_hwfn, u16 vfid, - struct ecore_mcp_link_params *params, - struct ecore_mcp_link_state *link, - struct ecore_mcp_link_capabilities - *p_caps) -{ -} - -static OSAL_INLINE void ecore_iov_get_link(struct ecore_hwfn *p_hwfn, u16 vfid, - struct ecore_mcp_link_params *params, - struct ecore_mcp_link_state *link, - struct ecore_mcp_link_capabilities - *p_caps) -{ -} - -static OSAL_INLINE bool ecore_iov_is_vf_pending_flr(struct ecore_hwfn *p_hwfn, - u16 rel_vf_id) -{ - return false; -} - -static OSAL_INLINE bool ecore_iov_is_valid_vfid(struct ecore_hwfn *p_hwfn, - int rel_vf_id, - bool b_enabled_only) -{ - return false; -} - -static OSAL_INLINE struct ecore_public_vf_info * - ecore_iov_get_public_vf_info(struct ecore_hwfn *p_hwfn, u16 vfid, - bool b_enabled_only) -{ - return OSAL_NULL; -} - -static OSAL_INLINE void ecore_iov_pf_add_pending_events(struct ecore_hwfn - *p_hwfn, u8 vfid) -{ -} - -static OSAL_INLINE void ecore_iov_pf_get_and_clear_pending_events(struct - ecore_hwfn - * p_hwfn, - u64 *events) -{ -} - -static OSAL_INLINE enum _ecore_status_t ecore_iov_copy_vf_msg(struct ecore_hwfn - *p_hwfn, - struct ecore_ptt - *ptt, int vfid) -{ - return ECORE_INVAL; -} - -static OSAL_INLINE void ecore_iov_bulletin_set_forced_mac(struct ecore_hwfn - *p_hwfn, u8 *mac, - int vfid) -{ -} - -static OSAL_INLINE enum _ecore_status_t ecore_iov_bulletin_set_mac(struct - ecore_hwfn - * p_hwfn, - u8 *mac, - int vfid) -{ - return ECORE_INVAL; -} - -static OSAL_INLINE void ecore_iov_bulletin_set_forced_vlan(struct ecore_hwfn - p_hwfn, u16 pvid, - int vfid) -{ -} - -static OSAL_INLINE void ecore_iov_get_vfs_opaque_fid(struct ecore_hwfn *p_hwfn, - int vfid, u16 *opaque_fid) -{ -} - -static OSAL_INLINE void ecore_iov_get_vfs_vport_id(struct ecore_hwfn *p_hwfn, - int vfid, u8 *p_vport_id) -{ -} - -static OSAL_INLINE bool ecore_iov_vf_has_vport_instance(struct ecore_hwfn - *p_hwfn, int vfid) -{ - return false; -} - -static OSAL_INLINE enum _ecore_status_t ecore_iov_post_vf_bulletin(struct - ecore_hwfn - * p_hwfn, - int vfid, - struct - ecore_ptt - * p_ptt) -{ - return ECORE_INVAL; -} - -static OSAL_INLINE bool ecore_iov_is_vf_stopped(struct ecore_hwfn *p_hwfn, - int vfid) -{ - return false; -} - -static OSAL_INLINE enum _ecore_status_t ecore_iov_spoofchk_set(struct ecore_hwfn - *p_hwfn, - int vfid, - bool val) -{ - return ECORE_INVAL; -} - -static OSAL_INLINE bool ecore_iov_spoofchk_get(struct ecore_hwfn *p_hwfn, - int vfid) -{ - return false; -} - -static OSAL_INLINE bool ecore_iov_pf_sanity_check(struct ecore_hwfn *p_hwfn, - int vfid) -{ - return false; -} - -static OSAL_INLINE u8 ecore_iov_vf_chains_per_pf(struct ecore_hwfn *p_hwfn) -{ - return 0; -} - -static OSAL_INLINE void ecore_iov_get_vf_req_virt_mbx_params(struct ecore_hwfn - *p_hwfn, - u16 rel_vf_id, - void - **pp_req_virt_addr, - u16 * - p_req_virt_size) -{ -} - -static OSAL_INLINE void ecore_iov_get_vf_reply_virt_mbx_params(struct ecore_hwfn - *p_hwfn, - u16 rel_vf_id, - void - **pp_reply_virt_addr, - u16 * - p_reply_virt_size) -{ -} - -static OSAL_INLINE bool ecore_iov_is_valid_vfpf_msg_length(u32 length) -{ - return false; -} - -static OSAL_INLINE u32 ecore_iov_pfvf_msg_length(void) -{ - return 0; -} - -static OSAL_INLINE u8 *ecore_iov_bulletin_get_forced_mac(struct ecore_hwfn - *p_hwfn, u16 rel_vf_id) -{ - return OSAL_NULL; -} - -static OSAL_INLINE u16 ecore_iov_bulletin_get_forced_vlan(struct ecore_hwfn - *p_hwfn, - u16 rel_vf_id) -{ - return 0; -} - -static OSAL_INLINE enum _ecore_status_t ecore_iov_configure_tx_rate(struct - ecore_hwfn - * p_hwfn, - struct - ecore_ptt - * p_ptt, - int vfid, - int val) -{ - return ECORE_INVAL; -} - -static OSAL_INLINE u8 ecore_iov_get_vf_num_rxqs(struct ecore_hwfn *p_hwfn, - u16 rel_vf_id) -{ - return 0; -} - -static OSAL_INLINE u8 ecore_iov_get_vf_num_active_rxqs(struct ecore_hwfn - *p_hwfn, u16 rel_vf_id) -{ - return 0; -} - -static OSAL_INLINE void *ecore_iov_get_vf_ctx(struct ecore_hwfn *p_hwfn, - u16 rel_vf_id) -{ - return OSAL_NULL; -} - -static OSAL_INLINE u8 ecore_iov_get_vf_num_sbs(struct ecore_hwfn *p_hwfn, - u16 rel_vf_id) -{ - return 0; -} - -static OSAL_INLINE bool ecore_iov_is_vf_wait_for_acquire(struct ecore_hwfn - *p_hwfn, u16 rel_vf_id) -{ - return false; -} - -static OSAL_INLINE bool ecore_iov_is_vf_acquired_not_initialized(struct - ecore_hwfn - * p_hwfn, - u16 rel_vf_id) -{ - return false; -} - -static OSAL_INLINE bool ecore_iov_is_vf_initialized(struct ecore_hwfn *p_hwfn, - u16 rel_vf_id) -{ - return false; -} - -static OSAL_INLINE int ecore_iov_get_vf_min_rate(struct ecore_hwfn *p_hwfn, - int vfid) -{ - return 0; -} - -static OSAL_INLINE enum _ecore_status_t ecore_iov_configure_min_tx_rate( - struct ecore_dev *p_dev, int vfid, u32 rate) -{ - return ECORE_INVAL; -} -#endif #endif diff --git a/dpdk/drivers/net/qede/base/ecore_iro.h b/dpdk/drivers/net/qede/base/ecore_iro.h index dd53ea9c..360d7f88 100644 --- a/dpdk/drivers/net/qede/base/ecore_iro.h +++ b/dpdk/drivers/net/qede/base/ecore_iro.h @@ -10,106 +10,196 @@ #define __IRO_H__ /* Ystorm flow control mode. Use enum fw_flow_ctrl_mode */ -#define YSTORM_FLOW_CONTROL_MODE_OFFSET (IRO[0].base) -#define YSTORM_FLOW_CONTROL_MODE_SIZE (IRO[0].size) +#define YSTORM_FLOW_CONTROL_MODE_OFFSET (IRO[0].base) +#define YSTORM_FLOW_CONTROL_MODE_SIZE (IRO[0].size) /* Tstorm port statistics */ -#define TSTORM_PORT_STAT_OFFSET(port_id) \ -(IRO[1].base + ((port_id) * IRO[1].m1)) -#define TSTORM_PORT_STAT_SIZE (IRO[1].size) +#define TSTORM_PORT_STAT_OFFSET(port_id) (IRO[1].base + ((port_id) * IRO[1].m1)) +#define TSTORM_PORT_STAT_SIZE (IRO[1].size) +/* Tstorm ll2 port statistics */ +#define TSTORM_LL2_PORT_STAT_OFFSET(port_id) (IRO[2].base + \ + ((port_id) * IRO[2].m1)) +#define TSTORM_LL2_PORT_STAT_SIZE (IRO[2].size) /* Ustorm VF-PF Channel ready flag */ -#define USTORM_VF_PF_CHANNEL_READY_OFFSET(vf_id) \ -(IRO[3].base + ((vf_id) * IRO[3].m1)) -#define USTORM_VF_PF_CHANNEL_READY_SIZE (IRO[3].size) +#define USTORM_VF_PF_CHANNEL_READY_OFFSET(vf_id) (IRO[3].base + \ + ((vf_id) * IRO[3].m1)) +#define USTORM_VF_PF_CHANNEL_READY_SIZE (IRO[3].size) /* Ustorm Final flr cleanup ack */ -#define USTORM_FLR_FINAL_ACK_OFFSET(pf_id) \ -(IRO[4].base + ((pf_id) * IRO[4].m1)) -#define USTORM_FLR_FINAL_ACK_SIZE (IRO[4].size) +#define USTORM_FLR_FINAL_ACK_OFFSET(pf_id) (IRO[4].base + ((pf_id) * IRO[4].m1)) +#define USTORM_FLR_FINAL_ACK_SIZE (IRO[4].size) /* Ustorm Event ring consumer */ -#define USTORM_EQE_CONS_OFFSET(pf_id) \ -(IRO[5].base + ((pf_id) * IRO[5].m1)) -#define USTORM_EQE_CONS_SIZE (IRO[5].size) +#define USTORM_EQE_CONS_OFFSET(pf_id) (IRO[5].base + ((pf_id) * IRO[5].m1)) +#define USTORM_EQE_CONS_SIZE (IRO[5].size) +/* Ustorm eth queue zone */ +#define USTORM_ETH_QUEUE_ZONE_OFFSET(queue_zone_id) (IRO[6].base + \ + ((queue_zone_id) * IRO[6].m1)) +#define USTORM_ETH_QUEUE_ZONE_SIZE (IRO[6].size) /* Ustorm Common Queue ring consumer */ -#define USTORM_COMMON_QUEUE_CONS_OFFSET(global_queue_id) \ -(IRO[6].base + ((global_queue_id) * IRO[6].m1)) -#define USTORM_COMMON_QUEUE_CONS_SIZE (IRO[6].size) +#define USTORM_COMMON_QUEUE_CONS_OFFSET(queue_zone_id) (IRO[7].base + \ + ((queue_zone_id) * IRO[7].m1)) +#define USTORM_COMMON_QUEUE_CONS_SIZE (IRO[7].size) /* Xstorm Integration Test Data */ -#define XSTORM_INTEG_TEST_DATA_OFFSET (IRO[7].base) -#define XSTORM_INTEG_TEST_DATA_SIZE (IRO[7].size) +#define XSTORM_INTEG_TEST_DATA_OFFSET (IRO[8].base) +#define XSTORM_INTEG_TEST_DATA_SIZE (IRO[8].size) /* Ystorm Integration Test Data */ -#define YSTORM_INTEG_TEST_DATA_OFFSET (IRO[8].base) -#define YSTORM_INTEG_TEST_DATA_SIZE (IRO[8].size) +#define YSTORM_INTEG_TEST_DATA_OFFSET (IRO[9].base) +#define YSTORM_INTEG_TEST_DATA_SIZE (IRO[9].size) /* Pstorm Integration Test Data */ -#define PSTORM_INTEG_TEST_DATA_OFFSET (IRO[9].base) -#define PSTORM_INTEG_TEST_DATA_SIZE (IRO[9].size) +#define PSTORM_INTEG_TEST_DATA_OFFSET (IRO[10].base) +#define PSTORM_INTEG_TEST_DATA_SIZE (IRO[10].size) /* Tstorm Integration Test Data */ -#define TSTORM_INTEG_TEST_DATA_OFFSET (IRO[10].base) -#define TSTORM_INTEG_TEST_DATA_SIZE (IRO[10].size) +#define TSTORM_INTEG_TEST_DATA_OFFSET (IRO[11].base) +#define TSTORM_INTEG_TEST_DATA_SIZE (IRO[11].size) /* Mstorm Integration Test Data */ -#define MSTORM_INTEG_TEST_DATA_OFFSET (IRO[11].base) -#define MSTORM_INTEG_TEST_DATA_SIZE (IRO[11].size) +#define MSTORM_INTEG_TEST_DATA_OFFSET (IRO[12].base) +#define MSTORM_INTEG_TEST_DATA_SIZE (IRO[12].size) /* Ustorm Integration Test Data */ -#define USTORM_INTEG_TEST_DATA_OFFSET (IRO[12].base) -#define USTORM_INTEG_TEST_DATA_SIZE (IRO[12].size) +#define USTORM_INTEG_TEST_DATA_OFFSET (IRO[13].base) +#define USTORM_INTEG_TEST_DATA_SIZE (IRO[13].size) +/* Tstorm producers */ +#define TSTORM_LL2_RX_PRODS_OFFSET(core_rx_queue_id) (IRO[14].base + \ + ((core_rx_queue_id) * IRO[14].m1)) +#define TSTORM_LL2_RX_PRODS_SIZE (IRO[14].size) +/* Tstorm LightL2 queue statistics */ +#define CORE_LL2_TSTORM_PER_QUEUE_STAT_OFFSET(core_rx_queue_id) \ + (IRO[15].base + ((core_rx_queue_id) * IRO[15].m1)) +#define CORE_LL2_TSTORM_PER_QUEUE_STAT_SIZE (IRO[15].size) +/* Ustorm LiteL2 queue statistics */ +#define CORE_LL2_USTORM_PER_QUEUE_STAT_OFFSET(core_rx_queue_id) \ + (IRO[16].base + ((core_rx_queue_id) * IRO[16].m1)) +#define CORE_LL2_USTORM_PER_QUEUE_STAT_SIZE (IRO[16].size) +/* Pstorm LiteL2 queue statistics */ +#define CORE_LL2_PSTORM_PER_QUEUE_STAT_OFFSET(core_tx_stats_id) \ + (IRO[17].base + ((core_tx_stats_id) * IRO[17].m1)) +#define CORE_LL2_PSTORM_PER_QUEUE_STAT_SIZE (IRO[17].size) /* Mstorm queue statistics */ -#define MSTORM_QUEUE_STAT_OFFSET(stat_counter_id) \ -(IRO[17].base + ((stat_counter_id) * IRO[17].m1)) -#define MSTORM_QUEUE_STAT_SIZE (IRO[17].size) -/* Mstorm producers */ -#define MSTORM_PRODS_OFFSET(queue_id) \ -(IRO[18].base + ((queue_id) * IRO[18].m1)) -#define MSTORM_PRODS_SIZE (IRO[18].size) +#define MSTORM_QUEUE_STAT_OFFSET(stat_counter_id) (IRO[18].base + \ + ((stat_counter_id) * IRO[18].m1)) +#define MSTORM_QUEUE_STAT_SIZE (IRO[18].size) +/* Mstorm ETH PF queues producers */ +#define MSTORM_ETH_PF_PRODS_OFFSET(queue_id) (IRO[19].base + \ + ((queue_id) * IRO[19].m1)) +#define MSTORM_ETH_PF_PRODS_SIZE (IRO[19].size) +/* Mstorm ETH VF queues producers offset in RAM. Used in default VF zone size + * mode. + */ +#define MSTORM_ETH_VF_PRODS_OFFSET(vf_id, vf_queue_id) (IRO[20].base + \ + ((vf_id) * IRO[20].m1) + ((vf_queue_id) * IRO[20].m2)) +#define MSTORM_ETH_VF_PRODS_SIZE (IRO[20].size) /* TPA agregation timeout in us resolution (on ASIC) */ -#define MSTORM_TPA_TIMEOUT_US_OFFSET (IRO[19].base) -#define MSTORM_TPA_TIMEOUT_US_SIZE (IRO[19].size) +#define MSTORM_TPA_TIMEOUT_US_OFFSET (IRO[21].base) +#define MSTORM_TPA_TIMEOUT_US_SIZE (IRO[21].size) +/* Mstorm pf statistics */ +#define MSTORM_ETH_PF_STAT_OFFSET(pf_id) (IRO[22].base + ((pf_id) * IRO[22].m1)) +#define MSTORM_ETH_PF_STAT_SIZE (IRO[22].size) /* Ustorm queue statistics */ -#define USTORM_QUEUE_STAT_OFFSET(stat_counter_id) \ -(IRO[20].base + ((stat_counter_id) * IRO[20].m1)) -#define USTORM_QUEUE_STAT_SIZE (IRO[20].size) -/* Ustorm queue zone */ -#define USTORM_ETH_QUEUE_ZONE_OFFSET(queue_id) \ -(IRO[21].base + ((queue_id) * IRO[21].m1)) -#define USTORM_ETH_QUEUE_ZONE_SIZE (IRO[21].size) +#define USTORM_QUEUE_STAT_OFFSET(stat_counter_id) (IRO[23].base + \ + ((stat_counter_id) * IRO[23].m1)) +#define USTORM_QUEUE_STAT_SIZE (IRO[23].size) +/* Ustorm pf statistics */ +#define USTORM_ETH_PF_STAT_OFFSET(pf_id) (IRO[24].base + ((pf_id) * IRO[24].m1)) +#define USTORM_ETH_PF_STAT_SIZE (IRO[24].size) /* Pstorm queue statistics */ -#define PSTORM_QUEUE_STAT_OFFSET(stat_counter_id) \ -(IRO[22].base + ((stat_counter_id) * IRO[22].m1)) -#define PSTORM_QUEUE_STAT_SIZE (IRO[22].size) +#define PSTORM_QUEUE_STAT_OFFSET(stat_counter_id) (IRO[25].base + \ + ((stat_counter_id) * IRO[25].m1)) +#define PSTORM_QUEUE_STAT_SIZE (IRO[25].size) +/* Pstorm pf statistics */ +#define PSTORM_ETH_PF_STAT_OFFSET(pf_id) (IRO[26].base + ((pf_id) * IRO[26].m1)) +#define PSTORM_ETH_PF_STAT_SIZE (IRO[26].size) +/* Control frame's EthType configuration for TX control frame security */ +#define PSTORM_CTL_FRAME_ETHTYPE_OFFSET(ethType_id) (IRO[27].base + \ + ((ethType_id) * IRO[27].m1)) +#define PSTORM_CTL_FRAME_ETHTYPE_SIZE (IRO[27].size) /* Tstorm last parser message */ -#define TSTORM_ETH_PRS_INPUT_OFFSET (IRO[23].base) -#define TSTORM_ETH_PRS_INPUT_SIZE (IRO[23].size) +#define TSTORM_ETH_PRS_INPUT_OFFSET (IRO[28].base) +#define TSTORM_ETH_PRS_INPUT_SIZE (IRO[28].size) /* Tstorm Eth limit Rx rate */ -#define ETH_RX_RATE_LIMIT_OFFSET(pf_id) \ -(IRO[24].base + ((pf_id) * IRO[24].m1)) -#define ETH_RX_RATE_LIMIT_SIZE (IRO[24].size) -/* Ystorm queue zone */ -#define YSTORM_ETH_QUEUE_ZONE_OFFSET(queue_id) \ -(IRO[25].base + ((queue_id) * IRO[25].m1)) -#define YSTORM_ETH_QUEUE_ZONE_SIZE (IRO[25].size) +#define ETH_RX_RATE_LIMIT_OFFSET(pf_id) (IRO[29].base + ((pf_id) * IRO[29].m1)) +#define ETH_RX_RATE_LIMIT_SIZE (IRO[29].size) +/* Xstorm queue zone */ +#define XSTORM_ETH_QUEUE_ZONE_OFFSET(queue_id) (IRO[30].base + \ + ((queue_id) * IRO[30].m1)) +#define XSTORM_ETH_QUEUE_ZONE_SIZE (IRO[30].size) /* Ystorm cqe producer */ -#define YSTORM_TOE_CQ_PROD_OFFSET(rss_id) \ -(IRO[26].base + ((rss_id) * IRO[26].m1)) -#define YSTORM_TOE_CQ_PROD_SIZE (IRO[26].size) +#define YSTORM_TOE_CQ_PROD_OFFSET(rss_id) (IRO[31].base + \ + ((rss_id) * IRO[31].m1)) +#define YSTORM_TOE_CQ_PROD_SIZE (IRO[31].size) /* Ustorm cqe producer */ -#define USTORM_TOE_CQ_PROD_OFFSET(rss_id) \ -(IRO[27].base + ((rss_id) * IRO[27].m1)) -#define USTORM_TOE_CQ_PROD_SIZE (IRO[27].size) +#define USTORM_TOE_CQ_PROD_OFFSET(rss_id) (IRO[32].base + \ + ((rss_id) * IRO[32].m1)) +#define USTORM_TOE_CQ_PROD_SIZE (IRO[32].size) /* Ustorm grq producer */ -#define USTORM_TOE_GRQ_PROD_OFFSET(pf_id) \ -(IRO[28].base + ((pf_id) * IRO[28].m1)) -#define USTORM_TOE_GRQ_PROD_SIZE (IRO[28].size) +#define USTORM_TOE_GRQ_PROD_OFFSET(pf_id) (IRO[33].base + \ + ((pf_id) * IRO[33].m1)) +#define USTORM_TOE_GRQ_PROD_SIZE (IRO[33].size) /* Tstorm cmdq-cons of given command queue-id */ -#define TSTORM_SCSI_CMDQ_CONS_OFFSET(cmdq_queue_id) \ -(IRO[29].base + ((cmdq_queue_id) * IRO[29].m1)) -#define TSTORM_SCSI_CMDQ_CONS_SIZE (IRO[29].size) -#define TSTORM_SCSI_BDQ_EXT_PROD_OFFSET(func_id, bdq_id) \ -(IRO[30].base + ((func_id) * IRO[30].m1) + ((bdq_id) * IRO[30].m2)) -#define TSTORM_SCSI_BDQ_EXT_PROD_SIZE (IRO[30].size) -/* Mstorm rq-cons of given queue-id */ -#define MSTORM_SCSI_RQ_CONS_OFFSET(rq_queue_id) \ -(IRO[31].base + ((rq_queue_id) * IRO[31].m1)) -#define MSTORM_SCSI_RQ_CONS_SIZE (IRO[31].size) -/* Mstorm bdq-external-producer of given BDQ function ID, BDqueue-id */ -#define MSTORM_SCSI_BDQ_EXT_PROD_OFFSET(func_id, bdq_id) \ -(IRO[32].base + ((func_id) * IRO[32].m1) + ((bdq_id) * IRO[32].m2)) -#define MSTORM_SCSI_BDQ_EXT_PROD_SIZE (IRO[32].size) +#define TSTORM_SCSI_CMDQ_CONS_OFFSET(cmdq_queue_id) (IRO[34].base + \ + ((cmdq_queue_id) * IRO[34].m1)) +#define TSTORM_SCSI_CMDQ_CONS_SIZE (IRO[34].size) +/* Tstorm (reflects M-Storm) bdq-external-producer of given function ID, + * BDqueue-id + */ +#define TSTORM_SCSI_BDQ_EXT_PROD_OFFSET(func_id, bdq_id) (IRO[35].base + \ + ((func_id) * IRO[35].m1) + ((bdq_id) * IRO[35].m2)) +#define TSTORM_SCSI_BDQ_EXT_PROD_SIZE (IRO[35].size) +/* Mstorm bdq-external-producer of given BDQ resource ID, BDqueue-id */ +#define MSTORM_SCSI_BDQ_EXT_PROD_OFFSET(func_id, bdq_id) (IRO[36].base + \ + ((func_id) * IRO[36].m1) + ((bdq_id) * IRO[36].m2)) +#define MSTORM_SCSI_BDQ_EXT_PROD_SIZE (IRO[36].size) +/* Tstorm iSCSI RX stats */ +#define TSTORM_ISCSI_RX_STATS_OFFSET(pf_id) (IRO[37].base + \ + ((pf_id) * IRO[37].m1)) +#define TSTORM_ISCSI_RX_STATS_SIZE (IRO[37].size) +/* Mstorm iSCSI RX stats */ +#define MSTORM_ISCSI_RX_STATS_OFFSET(pf_id) (IRO[38].base + \ + ((pf_id) * IRO[38].m1)) +#define MSTORM_ISCSI_RX_STATS_SIZE (IRO[38].size) +/* Ustorm iSCSI RX stats */ +#define USTORM_ISCSI_RX_STATS_OFFSET(pf_id) (IRO[39].base + \ + ((pf_id) * IRO[39].m1)) +#define USTORM_ISCSI_RX_STATS_SIZE (IRO[39].size) +/* Xstorm iSCSI TX stats */ +#define XSTORM_ISCSI_TX_STATS_OFFSET(pf_id) (IRO[40].base + \ + ((pf_id) * IRO[40].m1)) +#define XSTORM_ISCSI_TX_STATS_SIZE (IRO[40].size) +/* Ystorm iSCSI TX stats */ +#define YSTORM_ISCSI_TX_STATS_OFFSET(pf_id) (IRO[41].base + \ + ((pf_id) * IRO[41].m1)) +#define YSTORM_ISCSI_TX_STATS_SIZE (IRO[41].size) +/* Pstorm iSCSI TX stats */ +#define PSTORM_ISCSI_TX_STATS_OFFSET(pf_id) (IRO[42].base + \ + ((pf_id) * IRO[42].m1)) +#define PSTORM_ISCSI_TX_STATS_SIZE (IRO[42].size) +/* Tstorm FCoE RX stats */ +#define TSTORM_FCOE_RX_STATS_OFFSET(pf_id) (IRO[43].base + \ + ((pf_id) * IRO[43].m1)) +#define TSTORM_FCOE_RX_STATS_SIZE (IRO[43].size) +/* Pstorm FCoE TX stats */ +#define PSTORM_FCOE_TX_STATS_OFFSET(pf_id) (IRO[44].base + \ + ((pf_id) * IRO[44].m1)) +#define PSTORM_FCOE_TX_STATS_SIZE (IRO[44].size) +/* Pstorm RDMA queue statistics */ +#define PSTORM_RDMA_QUEUE_STAT_OFFSET(rdma_stat_counter_id) \ + (IRO[45].base + ((rdma_stat_counter_id) * IRO[45].m1)) +#define PSTORM_RDMA_QUEUE_STAT_SIZE (IRO[45].size) +/* Tstorm RDMA queue statistics */ +#define TSTORM_RDMA_QUEUE_STAT_OFFSET(rdma_stat_counter_id) (IRO[46].base + \ + ((rdma_stat_counter_id) * IRO[46].m1)) +#define TSTORM_RDMA_QUEUE_STAT_SIZE (IRO[46].size) +/* Xstorm iWARP rxmit stats */ +#define XSTORM_IWARP_RXMIT_STATS_OFFSET(pf_id) (IRO[47].base + \ + ((pf_id) * IRO[47].m1)) +#define XSTORM_IWARP_RXMIT_STATS_SIZE (IRO[47].size) +/* Tstorm RoCE Event Statistics */ +#define TSTORM_ROCE_EVENTS_STAT_OFFSET(roce_pf_id) (IRO[48].base + \ + ((roce_pf_id) * IRO[48].m1)) +#define TSTORM_ROCE_EVENTS_STAT_SIZE (IRO[48].size) +/* DCQCN Received Statistics */ +#define YSTORM_ROCE_DCQCN_RECEIVED_STATS_OFFSET(roce_pf_id) (IRO[49].base + \ + ((roce_pf_id) * IRO[49].m1)) +#define YSTORM_ROCE_DCQCN_RECEIVED_STATS_SIZE (IRO[49].size) +/* DCQCN Sent Statistics */ +#define PSTORM_ROCE_DCQCN_SENT_STATS_OFFSET(roce_pf_id) (IRO[50].base + \ + ((roce_pf_id) * IRO[50].m1)) +#define PSTORM_ROCE_DCQCN_SENT_STATS_SIZE (IRO[50].size) #endif /* __IRO_H__ */ diff --git a/dpdk/drivers/net/qede/base/ecore_iro_values.h b/dpdk/drivers/net/qede/base/ecore_iro_values.h index c818b580..41532eeb 100644 --- a/dpdk/drivers/net/qede/base/ecore_iro_values.h +++ b/dpdk/drivers/net/qede/base/ecore_iro_values.h @@ -9,51 +9,109 @@ #ifndef __IRO_VALUES_H__ #define __IRO_VALUES_H__ -static const struct iro iro_arr[44] = { - {0x0, 0x0, 0x0, 0x0, 0x8}, - {0x4db0, 0x60, 0x0, 0x0, 0x60}, - {0x6418, 0x20, 0x0, 0x0, 0x20}, - {0x500, 0x8, 0x0, 0x0, 0x4}, - {0x480, 0x8, 0x0, 0x0, 0x4}, - {0x0, 0x8, 0x0, 0x0, 0x2}, - {0x80, 0x8, 0x0, 0x0, 0x2}, - {0x4938, 0x0, 0x0, 0x0, 0x78}, - {0x3df0, 0x0, 0x0, 0x0, 0x78}, - {0x29b0, 0x0, 0x0, 0x0, 0x78}, - {0x4d38, 0x0, 0x0, 0x0, 0x78}, - {0x56c8, 0x0, 0x0, 0x0, 0x78}, - {0x7e48, 0x0, 0x0, 0x0, 0x78}, - {0xa28, 0x8, 0x0, 0x0, 0x8}, - {0x61f8, 0x10, 0x0, 0x0, 0x10}, - {0xb500, 0x30, 0x0, 0x0, 0x30}, - {0x95b8, 0x30, 0x0, 0x0, 0x30}, - {0x5898, 0x40, 0x0, 0x0, 0x40}, - {0x1f8, 0x10, 0x0, 0x0, 0x8}, - {0xa228, 0x0, 0x0, 0x0, 0x4}, - {0x8050, 0x40, 0x0, 0x0, 0x30}, - {0xcf8, 0x8, 0x0, 0x0, 0x8}, - {0x2b48, 0x80, 0x0, 0x0, 0x38}, - {0xadf0, 0x0, 0x0, 0x0, 0xf0}, - {0xaee0, 0x8, 0x0, 0x0, 0x8}, - {0x80, 0x8, 0x0, 0x0, 0x8}, - {0xac0, 0x8, 0x0, 0x0, 0x8}, - {0x2578, 0x8, 0x0, 0x0, 0x8}, - {0x24f8, 0x8, 0x0, 0x0, 0x8}, - {0x0, 0x8, 0x0, 0x0, 0x8}, - {0x200, 0x10, 0x8, 0x0, 0x8}, - {0x17f8, 0x8, 0x0, 0x0, 0x2}, - {0x19f8, 0x10, 0x8, 0x0, 0x2}, - {0xd988, 0x38, 0x0, 0x0, 0x24}, - {0x11040, 0x10, 0x0, 0x0, 0x8}, - {0x11670, 0x38, 0x0, 0x0, 0x18}, - {0xaeb8, 0x30, 0x0, 0x0, 0x10}, - {0x86f8, 0x28, 0x0, 0x0, 0x18}, - {0xebf8, 0x10, 0x0, 0x0, 0x10}, - {0xde08, 0x40, 0x0, 0x0, 0x30}, - {0x121a0, 0x38, 0x0, 0x0, 0x8}, - {0xf060, 0x20, 0x0, 0x0, 0x20}, - {0x2b80, 0x80, 0x0, 0x0, 0x10}, - {0x50a0, 0x10, 0x0, 0x0, 0x10}, +static const struct iro iro_arr[51] = { +/* YSTORM_FLOW_CONTROL_MODE_OFFSET */ + { 0x0, 0x0, 0x0, 0x0, 0x8}, +/* TSTORM_PORT_STAT_OFFSET(port_id) */ + { 0x4cb0, 0x80, 0x0, 0x0, 0x80}, +/* TSTORM_LL2_PORT_STAT_OFFSET(port_id) */ + { 0x6508, 0x20, 0x0, 0x0, 0x20}, +/* USTORM_VF_PF_CHANNEL_READY_OFFSET(vf_id) */ + { 0xb00, 0x8, 0x0, 0x0, 0x4}, +/* USTORM_FLR_FINAL_ACK_OFFSET(pf_id) */ + { 0xa80, 0x8, 0x0, 0x0, 0x4}, +/* USTORM_EQE_CONS_OFFSET(pf_id) */ + { 0x0, 0x8, 0x0, 0x0, 0x2}, +/* USTORM_ETH_QUEUE_ZONE_OFFSET(queue_zone_id) */ + { 0x80, 0x8, 0x0, 0x0, 0x4}, +/* USTORM_COMMON_QUEUE_CONS_OFFSET(queue_zone_id) */ + { 0x84, 0x8, 0x0, 0x0, 0x2}, +/* XSTORM_INTEG_TEST_DATA_OFFSET */ + { 0x4c40, 0x0, 0x0, 0x0, 0x78}, +/* YSTORM_INTEG_TEST_DATA_OFFSET */ + { 0x3e10, 0x0, 0x0, 0x0, 0x78}, +/* PSTORM_INTEG_TEST_DATA_OFFSET */ + { 0x2b50, 0x0, 0x0, 0x0, 0x78}, +/* TSTORM_INTEG_TEST_DATA_OFFSET */ + { 0x4c38, 0x0, 0x0, 0x0, 0x78}, +/* MSTORM_INTEG_TEST_DATA_OFFSET */ + { 0x4990, 0x0, 0x0, 0x0, 0x78}, +/* USTORM_INTEG_TEST_DATA_OFFSET */ + { 0x7f48, 0x0, 0x0, 0x0, 0x78}, +/* TSTORM_LL2_RX_PRODS_OFFSET(core_rx_queue_id) */ + { 0xa28, 0x8, 0x0, 0x0, 0x8}, +/* CORE_LL2_TSTORM_PER_QUEUE_STAT_OFFSET(core_rx_queue_id) */ + { 0x61e8, 0x10, 0x0, 0x0, 0x10}, +/* CORE_LL2_USTORM_PER_QUEUE_STAT_OFFSET(core_rx_queue_id) */ + { 0xb820, 0x30, 0x0, 0x0, 0x30}, +/* CORE_LL2_PSTORM_PER_QUEUE_STAT_OFFSET(core_tx_stats_id) */ + { 0x96b8, 0x30, 0x0, 0x0, 0x30}, +/* MSTORM_QUEUE_STAT_OFFSET(stat_counter_id) */ + { 0x4b60, 0x80, 0x0, 0x0, 0x40}, +/* MSTORM_ETH_PF_PRODS_OFFSET(queue_id) */ + { 0x1f8, 0x4, 0x0, 0x0, 0x4}, +/* MSTORM_ETH_VF_PRODS_OFFSET(vf_id,vf_queue_id) */ + { 0x53a0, 0x80, 0x4, 0x0, 0x4}, +/* MSTORM_TPA_TIMEOUT_US_OFFSET */ + { 0xc7c8, 0x0, 0x0, 0x0, 0x4}, +/* MSTORM_ETH_PF_STAT_OFFSET(pf_id) */ + { 0x4ba0, 0x80, 0x0, 0x0, 0x20}, +/* USTORM_QUEUE_STAT_OFFSET(stat_counter_id) */ + { 0x8150, 0x40, 0x0, 0x0, 0x30}, +/* USTORM_ETH_PF_STAT_OFFSET(pf_id) */ + { 0xe770, 0x60, 0x0, 0x0, 0x60}, +/* PSTORM_QUEUE_STAT_OFFSET(stat_counter_id) */ + { 0x2ce8, 0x80, 0x0, 0x0, 0x38}, +/* PSTORM_ETH_PF_STAT_OFFSET(pf_id) */ + { 0xf2b0, 0x78, 0x0, 0x0, 0x78}, +/* PSTORM_CTL_FRAME_ETHTYPE_OFFSET(ethType_id) */ + { 0x1f8, 0x4, 0x0, 0x0, 0x4}, +/* TSTORM_ETH_PRS_INPUT_OFFSET */ + { 0xaef8, 0x0, 0x0, 0x0, 0xf0}, +/* ETH_RX_RATE_LIMIT_OFFSET(pf_id) */ + { 0xafe8, 0x8, 0x0, 0x0, 0x8}, +/* XSTORM_ETH_QUEUE_ZONE_OFFSET(queue_id) */ + { 0x1f8, 0x8, 0x0, 0x0, 0x8}, +/* YSTORM_TOE_CQ_PROD_OFFSET(rss_id) */ + { 0xac0, 0x8, 0x0, 0x0, 0x8}, +/* USTORM_TOE_CQ_PROD_OFFSET(rss_id) */ + { 0x2578, 0x8, 0x0, 0x0, 0x8}, +/* USTORM_TOE_GRQ_PROD_OFFSET(pf_id) */ + { 0x24f8, 0x8, 0x0, 0x0, 0x8}, +/* TSTORM_SCSI_CMDQ_CONS_OFFSET(cmdq_queue_id) */ + { 0x0, 0x8, 0x0, 0x0, 0x8}, +/* TSTORM_SCSI_BDQ_EXT_PROD_OFFSET(func_id,bdq_id) */ + { 0x200, 0x18, 0x8, 0x0, 0x8}, +/* MSTORM_SCSI_BDQ_EXT_PROD_OFFSET(func_id,bdq_id) */ + { 0xb78, 0x18, 0x8, 0x0, 0x2}, +/* TSTORM_ISCSI_RX_STATS_OFFSET(pf_id) */ + { 0xd878, 0x50, 0x0, 0x0, 0x3c}, +/* MSTORM_ISCSI_RX_STATS_OFFSET(pf_id) */ + { 0x12908, 0x18, 0x0, 0x0, 0x10}, +/* USTORM_ISCSI_RX_STATS_OFFSET(pf_id) */ + { 0x11aa8, 0x40, 0x0, 0x0, 0x18}, +/* XSTORM_ISCSI_TX_STATS_OFFSET(pf_id) */ + { 0xa580, 0x50, 0x0, 0x0, 0x20}, +/* YSTORM_ISCSI_TX_STATS_OFFSET(pf_id) */ + { 0x86f8, 0x40, 0x0, 0x0, 0x28}, +/* PSTORM_ISCSI_TX_STATS_OFFSET(pf_id) */ + { 0x102f8, 0x18, 0x0, 0x0, 0x10}, +/* TSTORM_FCOE_RX_STATS_OFFSET(pf_id) */ + { 0xde28, 0x48, 0x0, 0x0, 0x38}, +/* PSTORM_FCOE_TX_STATS_OFFSET(pf_id) */ + { 0x10760, 0x20, 0x0, 0x0, 0x20}, +/* PSTORM_RDMA_QUEUE_STAT_OFFSET(rdma_stat_counter_id) */ + { 0x2d20, 0x80, 0x0, 0x0, 0x10}, +/* TSTORM_RDMA_QUEUE_STAT_OFFSET(rdma_stat_counter_id) */ + { 0x5020, 0x10, 0x0, 0x0, 0x10}, +/* XSTORM_IWARP_RXMIT_STATS_OFFSET(pf_id) */ + { 0xc9b0, 0x30, 0x0, 0x0, 0x10}, +/* TSTORM_ROCE_EVENTS_STAT_OFFSET(roce_pf_id) */ + { 0xeec0, 0x10, 0x0, 0x0, 0x10}, +/* YSTORM_ROCE_DCQCN_RECEIVED_STATS_OFFSET(roce_pf_id) */ + { 0xa398, 0x10, 0x0, 0x0, 0x10}, +/* PSTORM_ROCE_DCQCN_SENT_STATS_OFFSET(roce_pf_id) */ + { 0x13100, 0x8, 0x0, 0x0, 0x8}, }; #endif /* __IRO_VALUES_H__ */ diff --git a/dpdk/drivers/net/qede/base/ecore_l2.c b/dpdk/drivers/net/qede/base/ecore_l2.c index 9e6ef5a4..e3afc8a3 100644 --- a/dpdk/drivers/net/qede/base/ecore_l2.c +++ b/dpdk/drivers/net/qede/base/ecore_l2.c @@ -29,16 +29,328 @@ #define ECORE_MAX_SGES_NUM 16 #define CRC32_POLY 0x1edc6f41 +struct ecore_l2_info { + u32 queues; + unsigned long **pp_qid_usage; + + /* The lock is meant to synchronize access to the qid usage */ + osal_mutex_t lock; +}; + +enum _ecore_status_t ecore_l2_alloc(struct ecore_hwfn *p_hwfn) +{ + struct ecore_l2_info *p_l2_info; + unsigned long **pp_qids; + u32 i; + + if (!ECORE_IS_L2_PERSONALITY(p_hwfn)) + return ECORE_SUCCESS; + + p_l2_info = OSAL_VZALLOC(p_hwfn->p_dev, sizeof(*p_l2_info)); + if (!p_l2_info) + return ECORE_NOMEM; + p_hwfn->p_l2_info = p_l2_info; + + if (IS_PF(p_hwfn->p_dev)) { + p_l2_info->queues = RESC_NUM(p_hwfn, ECORE_L2_QUEUE); + } else { + u8 rx = 0, tx = 0; + + ecore_vf_get_num_rxqs(p_hwfn, &rx); + ecore_vf_get_num_txqs(p_hwfn, &tx); + + p_l2_info->queues = (u32)OSAL_MAX_T(u8, rx, tx); + } + + pp_qids = OSAL_VZALLOC(p_hwfn->p_dev, + sizeof(unsigned long *) * + p_l2_info->queues); + if (pp_qids == OSAL_NULL) + return ECORE_NOMEM; + p_l2_info->pp_qid_usage = pp_qids; + + for (i = 0; i < p_l2_info->queues; i++) { + pp_qids[i] = OSAL_VZALLOC(p_hwfn->p_dev, + MAX_QUEUES_PER_QZONE / 8); + if (pp_qids[i] == OSAL_NULL) + return ECORE_NOMEM; + } + +#ifdef CONFIG_ECORE_LOCK_ALLOC + OSAL_MUTEX_ALLOC(p_hwfn, &p_l2_info->lock); +#endif + + return ECORE_SUCCESS; +} + +void ecore_l2_setup(struct ecore_hwfn *p_hwfn) +{ + if (!ECORE_IS_L2_PERSONALITY(p_hwfn)) + return; + + OSAL_MUTEX_INIT(&p_hwfn->p_l2_info->lock); +} + +void ecore_l2_free(struct ecore_hwfn *p_hwfn) +{ + u32 i; + + if (!ECORE_IS_L2_PERSONALITY(p_hwfn)) + return; + + if (p_hwfn->p_l2_info == OSAL_NULL) + return; + + if (p_hwfn->p_l2_info->pp_qid_usage == OSAL_NULL) + goto out_l2_info; + + /* Free until hit first uninitialized entry */ + for (i = 0; i < p_hwfn->p_l2_info->queues; i++) { + if (p_hwfn->p_l2_info->pp_qid_usage[i] == OSAL_NULL) + break; + OSAL_VFREE(p_hwfn->p_dev, + p_hwfn->p_l2_info->pp_qid_usage[i]); + } + +#ifdef CONFIG_ECORE_LOCK_ALLOC + /* Lock is last to initialize, if everything else was */ + if (i == p_hwfn->p_l2_info->queues) + OSAL_MUTEX_DEALLOC(&p_hwfn->p_l2_info->lock); +#endif + + OSAL_VFREE(p_hwfn->p_dev, p_hwfn->p_l2_info->pp_qid_usage); + +out_l2_info: + OSAL_VFREE(p_hwfn->p_dev, p_hwfn->p_l2_info); + p_hwfn->p_l2_info = OSAL_NULL; +} + +/* TODO - we'll need locking around these... */ +static bool ecore_eth_queue_qid_usage_add(struct ecore_hwfn *p_hwfn, + struct ecore_queue_cid *p_cid) +{ + struct ecore_l2_info *p_l2_info = p_hwfn->p_l2_info; + u16 queue_id = p_cid->rel.queue_id; + bool b_rc = true; + u8 first; + + OSAL_MUTEX_ACQUIRE(&p_l2_info->lock); + + if (queue_id > p_l2_info->queues) { + DP_NOTICE(p_hwfn, true, + "Requested to increase usage for qzone %04x out of %08x\n", + queue_id, p_l2_info->queues); + b_rc = false; + goto out; + } + + first = (u8)OSAL_FIND_FIRST_ZERO_BIT(p_l2_info->pp_qid_usage[queue_id], + MAX_QUEUES_PER_QZONE); + if (first >= MAX_QUEUES_PER_QZONE) { + b_rc = false; + goto out; + } + + OSAL_SET_BIT(first, p_l2_info->pp_qid_usage[queue_id]); + p_cid->qid_usage_idx = first; + +out: + OSAL_MUTEX_RELEASE(&p_l2_info->lock); + return b_rc; +} + +static void ecore_eth_queue_qid_usage_del(struct ecore_hwfn *p_hwfn, + struct ecore_queue_cid *p_cid) +{ + OSAL_MUTEX_ACQUIRE(&p_hwfn->p_l2_info->lock); + + OSAL_CLEAR_BIT(p_cid->qid_usage_idx, + p_hwfn->p_l2_info->pp_qid_usage[p_cid->rel.queue_id]); + + OSAL_MUTEX_RELEASE(&p_hwfn->p_l2_info->lock); +} + +void ecore_eth_queue_cid_release(struct ecore_hwfn *p_hwfn, + struct ecore_queue_cid *p_cid) +{ + bool b_legacy_vf = !!(p_cid->vf_legacy & + ECORE_QCID_LEGACY_VF_CID); + + /* VFs' CIDs are 0-based in PF-view, and uninitialized on VF. + * For legacy vf-queues, the CID doesn't go through here. + */ + if (IS_PF(p_hwfn->p_dev) && !b_legacy_vf) + _ecore_cxt_release_cid(p_hwfn, p_cid->cid, p_cid->vfid); + + /* VFs maintain the index inside queue-zone on their own */ + if (p_cid->vfid == ECORE_QUEUE_CID_PF) + ecore_eth_queue_qid_usage_del(p_hwfn, p_cid); + + OSAL_VFREE(p_hwfn->p_dev, p_cid); +} + +/* The internal is only meant to be directly called by PFs initializeing CIDs + * for their VFs. + */ +static struct ecore_queue_cid * +_ecore_eth_queue_to_cid(struct ecore_hwfn *p_hwfn, + u16 opaque_fid, u32 cid, + struct ecore_queue_start_common_params *p_params, + bool b_is_rx, + struct ecore_queue_cid_vf_params *p_vf_params) +{ + struct ecore_queue_cid *p_cid; + enum _ecore_status_t rc; + + p_cid = OSAL_VZALLOC(p_hwfn->p_dev, sizeof(*p_cid)); + if (p_cid == OSAL_NULL) + return OSAL_NULL; + + p_cid->opaque_fid = opaque_fid; + p_cid->cid = cid; + p_cid->p_owner = p_hwfn; + + /* Fill in parameters */ + p_cid->rel.vport_id = p_params->vport_id; + p_cid->rel.queue_id = p_params->queue_id; + p_cid->rel.stats_id = p_params->stats_id; + p_cid->sb_igu_id = p_params->p_sb->igu_sb_id; + p_cid->b_is_rx = b_is_rx; + p_cid->sb_idx = p_params->sb_idx; + + /* Fill-in bits related to VFs' queues if information was provided */ + if (p_vf_params != OSAL_NULL) { + p_cid->vfid = p_vf_params->vfid; + p_cid->vf_qid = p_vf_params->vf_qid; + p_cid->vf_legacy = p_vf_params->vf_legacy; + } else { + p_cid->vfid = ECORE_QUEUE_CID_PF; + } + + /* Don't try calculating the absolute indices for VFs */ + if (IS_VF(p_hwfn->p_dev)) { + p_cid->abs = p_cid->rel; + + goto out; + } + + /* Calculate the engine-absolute indices of the resources. + * This would guarantee they're valid later on. + * In some cases [SBs] we already have the right values. + */ + rc = ecore_fw_vport(p_hwfn, p_cid->rel.vport_id, &p_cid->abs.vport_id); + if (rc != ECORE_SUCCESS) + goto fail; + + rc = ecore_fw_l2_queue(p_hwfn, p_cid->rel.queue_id, + &p_cid->abs.queue_id); + if (rc != ECORE_SUCCESS) + goto fail; + + /* In case of a PF configuring its VF's queues, the stats-id is already + * absolute [since there's a single index that's suitable per-VF]. + */ + if (p_cid->vfid == ECORE_QUEUE_CID_PF) { + rc = ecore_fw_vport(p_hwfn, p_cid->rel.stats_id, + &p_cid->abs.stats_id); + if (rc != ECORE_SUCCESS) + goto fail; + } else { + p_cid->abs.stats_id = p_cid->rel.stats_id; + } + +out: + /* VF-images have provided the qid_usage_idx on their own. + * Otherwise, we need to allocate a unique one. + */ + if (!p_vf_params) { + if (!ecore_eth_queue_qid_usage_add(p_hwfn, p_cid)) + goto fail; + } else { + p_cid->qid_usage_idx = p_vf_params->qid_usage_idx; + } + + DP_VERBOSE(p_hwfn, ECORE_MSG_SP, + "opaque_fid: %04x CID %08x vport %02x [%02x] qzone %04x.%02x [%04x] stats %02x [%02x] SB %04x PI %02x\n", + p_cid->opaque_fid, p_cid->cid, + p_cid->rel.vport_id, p_cid->abs.vport_id, + p_cid->rel.queue_id, p_cid->qid_usage_idx, + p_cid->abs.queue_id, + p_cid->rel.stats_id, p_cid->abs.stats_id, + p_cid->sb_igu_id, p_cid->sb_idx); + + return p_cid; + +fail: + OSAL_VFREE(p_hwfn->p_dev, p_cid); + return OSAL_NULL; +} + +struct ecore_queue_cid * +ecore_eth_queue_to_cid(struct ecore_hwfn *p_hwfn, u16 opaque_fid, + struct ecore_queue_start_common_params *p_params, + bool b_is_rx, + struct ecore_queue_cid_vf_params *p_vf_params) +{ + struct ecore_queue_cid *p_cid; + u8 vfid = ECORE_CXT_PF_CID; + bool b_legacy_vf = false; + u32 cid = 0; + + /* In case of legacy VFs, The CID can be derived from the additional + * VF parameters - the VF assumes queue X uses CID X, so we can simply + * use the vf_qid for this purpose as well. + */ + if (p_vf_params) { + vfid = p_vf_params->vfid; + + if (p_vf_params->vf_legacy & + ECORE_QCID_LEGACY_VF_CID) { + b_legacy_vf = true; + cid = p_vf_params->vf_qid; + } + } + + /* Get a unique firmware CID for this queue, in case it's a PF. + * VF's don't need a CID as the queue configuration will be done + * by PF. + */ + if (IS_PF(p_hwfn->p_dev) && !b_legacy_vf) { + if (_ecore_cxt_acquire_cid(p_hwfn, PROTOCOLID_ETH, + &cid, vfid) != ECORE_SUCCESS) { + DP_NOTICE(p_hwfn, true, "Failed to acquire cid\n"); + return OSAL_NULL; + } + } + + p_cid = _ecore_eth_queue_to_cid(p_hwfn, opaque_fid, cid, + p_params, b_is_rx, p_vf_params); + if ((p_cid == OSAL_NULL) && IS_PF(p_hwfn->p_dev) && !b_legacy_vf) + _ecore_cxt_release_cid(p_hwfn, cid, vfid); + + return p_cid; +} + +static struct ecore_queue_cid * +ecore_eth_queue_to_cid_pf(struct ecore_hwfn *p_hwfn, u16 opaque_fid, + bool b_is_rx, + struct ecore_queue_start_common_params *p_params) +{ + return ecore_eth_queue_to_cid(p_hwfn, opaque_fid, p_params, b_is_rx, + OSAL_NULL); +} + enum _ecore_status_t ecore_sp_eth_vport_start(struct ecore_hwfn *p_hwfn, struct ecore_sp_vport_start_params *p_params) { struct vport_start_ramrod_data *p_ramrod = OSAL_NULL; struct ecore_spq_entry *p_ent = OSAL_NULL; - enum _ecore_status_t rc = ECORE_NOTIMPL; struct ecore_sp_init_data init_data; + struct eth_vport_tpa_param *p_tpa; + u16 rx_mode = 0, tx_err = 0; u8 abs_vport_id = 0; - u16 rx_mode = 0; + enum _ecore_status_t rc = ECORE_NOTIMPL; rc = ecore_fw_vport(p_hwfn, p_params->vport_id, &abs_vport_id); if (rc != ECORE_SUCCESS) @@ -60,8 +372,8 @@ ecore_sp_eth_vport_start(struct ecore_hwfn *p_hwfn, p_ramrod->vport_id = abs_vport_id; p_ramrod->mtu = OSAL_CPU_TO_LE16(p_params->mtu); - p_ramrod->inner_vlan_removal_en = p_params->remove_inner_vlan; p_ramrod->handle_ptp_pkts = p_params->handle_ptp_pkts; + p_ramrod->inner_vlan_removal_en = p_params->remove_inner_vlan; p_ramrod->drop_ttl0_en = p_params->drop_ttl0; p_ramrod->untagged = p_params->only_untagged; p_ramrod->zero_placement_offset = p_params->zero_placement_offset; @@ -71,21 +383,47 @@ ecore_sp_eth_vport_start(struct ecore_hwfn *p_hwfn, p_ramrod->rx_mode.state = OSAL_CPU_TO_LE16(rx_mode); + /* Handle requests for strict behavior on transmission errors */ + SET_FIELD(tx_err, ETH_TX_ERR_VALS_ILLEGAL_VLAN_MODE, + p_params->b_err_illegal_vlan_mode ? + ETH_TX_ERR_ASSERT_MALICIOUS : 0); + SET_FIELD(tx_err, ETH_TX_ERR_VALS_PACKET_TOO_SMALL, + p_params->b_err_small_pkt ? + ETH_TX_ERR_ASSERT_MALICIOUS : 0); + SET_FIELD(tx_err, ETH_TX_ERR_VALS_ANTI_SPOOFING_ERR, + p_params->b_err_anti_spoof ? + ETH_TX_ERR_ASSERT_MALICIOUS : 0); + SET_FIELD(tx_err, ETH_TX_ERR_VALS_ILLEGAL_INBAND_TAGS, + p_params->b_err_illegal_inband_mode ? + ETH_TX_ERR_ASSERT_MALICIOUS : 0); + SET_FIELD(tx_err, ETH_TX_ERR_VALS_VLAN_INSERTION_W_INBAND_TAG, + p_params->b_err_vlan_insert_with_inband ? + ETH_TX_ERR_ASSERT_MALICIOUS : 0); + SET_FIELD(tx_err, ETH_TX_ERR_VALS_MTU_VIOLATION, + p_params->b_err_big_pkt ? + ETH_TX_ERR_ASSERT_MALICIOUS : 0); + SET_FIELD(tx_err, ETH_TX_ERR_VALS_ILLEGAL_CONTROL_FRAME, + p_params->b_err_ctrl_frame ? + ETH_TX_ERR_ASSERT_MALICIOUS : 0); + p_ramrod->tx_err_behav.values = OSAL_CPU_TO_LE16(tx_err); + /* TPA related fields */ - OSAL_MEMSET(&p_ramrod->tpa_param, 0, - sizeof(struct eth_vport_tpa_param)); - p_ramrod->tpa_param.max_buff_num = p_params->max_buffers_per_cqe; + p_tpa = &p_ramrod->tpa_param; + OSAL_MEMSET(p_tpa, 0, sizeof(struct eth_vport_tpa_param)); + p_tpa->max_buff_num = p_params->max_buffers_per_cqe; switch (p_params->tpa_mode) { case ECORE_TPA_MODE_GRO: - p_ramrod->tpa_param.tpa_max_aggs_num = ETH_TPA_MAX_AGGS_NUM; - p_ramrod->tpa_param.tpa_max_size = (u16)-1; - p_ramrod->tpa_param.tpa_min_size_to_cont = p_params->mtu / 2; - p_ramrod->tpa_param.tpa_min_size_to_start = p_params->mtu / 2; - p_ramrod->tpa_param.tpa_ipv4_en_flg = 1; - p_ramrod->tpa_param.tpa_ipv6_en_flg = 1; - p_ramrod->tpa_param.tpa_pkt_split_flg = 1; - p_ramrod->tpa_param.tpa_gro_consistent_flg = 1; + p_tpa->tpa_max_aggs_num = ETH_TPA_MAX_AGGS_NUM; + p_tpa->tpa_max_size = (u16)-1; + p_tpa->tpa_min_size_to_cont = p_params->mtu / 2; + p_tpa->tpa_min_size_to_start = p_params->mtu / 2; + p_tpa->tpa_ipv4_en_flg = 1; + p_tpa->tpa_ipv6_en_flg = 1; + p_tpa->tpa_ipv4_tunn_en_flg = 1; + p_tpa->tpa_ipv6_tunn_en_flg = 1; + p_tpa->tpa_pkt_split_flg = 1; + p_tpa->tpa_gro_consistent_flg = 1; break; default: break; @@ -97,9 +435,11 @@ ecore_sp_eth_vport_start(struct ecore_hwfn *p_hwfn, p_ramrod->tx_switching_en = 0; #endif + p_ramrod->ctl_frame_mac_check_en = !!p_params->check_mac; + p_ramrod->ctl_frame_ethtype_check_en = !!p_params->check_ethtype; + /* Software Function ID in hwfn (PFs are 0 - 15, VFs are 16 - 135) */ - p_ramrod->sw_fid = ecore_concrete_to_sw_fid(p_hwfn->p_dev, - p_params->concrete_fid); + p_ramrod->sw_fid = ecore_concrete_to_sw_fid(p_params->concrete_fid); return ecore_spq_post(p_hwfn, p_ent, OSAL_NULL); } @@ -124,10 +464,10 @@ ecore_sp_vport_update_rss(struct ecore_hwfn *p_hwfn, struct vport_update_ramrod_data *p_ramrod, struct ecore_rss_params *p_rss) { - enum _ecore_status_t rc = ECORE_SUCCESS; struct eth_vport_rss_config *p_config; - u16 abs_l2_queue = 0; - int i; + u16 capabilities = 0; + int i, table_size; + enum _ecore_status_t rc = ECORE_SUCCESS; if (!p_rss) { p_ramrod->common.update_rss_flg = 0; @@ -152,26 +492,26 @@ ecore_sp_vport_update_rss(struct ecore_hwfn *p_hwfn, p_config->capabilities = 0; - SET_FIELD(p_config->capabilities, + SET_FIELD(capabilities, ETH_VPORT_RSS_CONFIG_IPV4_CAPABILITY, !!(p_rss->rss_caps & ECORE_RSS_IPV4)); - SET_FIELD(p_config->capabilities, + SET_FIELD(capabilities, ETH_VPORT_RSS_CONFIG_IPV6_CAPABILITY, !!(p_rss->rss_caps & ECORE_RSS_IPV6)); - SET_FIELD(p_config->capabilities, + SET_FIELD(capabilities, ETH_VPORT_RSS_CONFIG_IPV4_TCP_CAPABILITY, !!(p_rss->rss_caps & ECORE_RSS_IPV4_TCP)); - SET_FIELD(p_config->capabilities, + SET_FIELD(capabilities, ETH_VPORT_RSS_CONFIG_IPV6_TCP_CAPABILITY, !!(p_rss->rss_caps & ECORE_RSS_IPV6_TCP)); - SET_FIELD(p_config->capabilities, + SET_FIELD(capabilities, ETH_VPORT_RSS_CONFIG_IPV4_UDP_CAPABILITY, !!(p_rss->rss_caps & ECORE_RSS_IPV4_UDP)); - SET_FIELD(p_config->capabilities, + SET_FIELD(capabilities, ETH_VPORT_RSS_CONFIG_IPV6_UDP_CAPABILITY, !!(p_rss->rss_caps & ECORE_RSS_IPV6_UDP)); p_config->tbl_size = p_rss->rss_table_size_log; - p_config->capabilities = OSAL_CPU_TO_LE16(p_config->capabilities); + p_config->capabilities = OSAL_CPU_TO_LE16(capabilities); DP_VERBOSE(p_hwfn, ECORE_MSG_IFUP, "update rss flag %d, rss_mode = %d, update_caps = %d, capabilities = %d, update_ind = %d, update_rss_key = %d\n", @@ -181,16 +521,40 @@ ecore_sp_vport_update_rss(struct ecore_hwfn *p_hwfn, p_config->capabilities, p_config->update_rss_ind_table, p_config->update_rss_key); - for (i = 0; i < ECORE_RSS_IND_TABLE_SIZE; i++) { - rc = ecore_fw_l2_queue(p_hwfn, - (u8)p_rss->rss_ind_table[i], - &abs_l2_queue); - if (rc != ECORE_SUCCESS) - return rc; + table_size = OSAL_MIN_T(int, ECORE_RSS_IND_TABLE_SIZE, + 1 << p_config->tbl_size); + for (i = 0; i < table_size; i++) { + struct ecore_queue_cid *p_queue = p_rss->rss_ind_table[i]; - p_config->indirection_table[i] = OSAL_CPU_TO_LE16(abs_l2_queue); - DP_VERBOSE(p_hwfn, ECORE_MSG_IFUP, "i= %d, queue = %d\n", - i, p_config->indirection_table[i]); + if (!p_queue) + return ECORE_INVAL; + + p_config->indirection_table[i] = + OSAL_CPU_TO_LE16(p_queue->abs.queue_id); + } + + DP_VERBOSE(p_hwfn, ECORE_MSG_IFUP, + "Configured RSS indirection table [%d entries]:\n", + table_size); + for (i = 0; i < ECORE_RSS_IND_TABLE_SIZE; i += 0x10) { + DP_VERBOSE(p_hwfn, ECORE_MSG_IFUP, + "%04x %04x %04x %04x %04x %04x %04x %04x %04x %04x %04x %04x %04x %04x %04x %04x\n", + OSAL_LE16_TO_CPU(p_config->indirection_table[i]), + OSAL_LE16_TO_CPU(p_config->indirection_table[i + 1]), + OSAL_LE16_TO_CPU(p_config->indirection_table[i + 2]), + OSAL_LE16_TO_CPU(p_config->indirection_table[i + 3]), + OSAL_LE16_TO_CPU(p_config->indirection_table[i + 4]), + OSAL_LE16_TO_CPU(p_config->indirection_table[i + 5]), + OSAL_LE16_TO_CPU(p_config->indirection_table[i + 6]), + OSAL_LE16_TO_CPU(p_config->indirection_table[i + 7]), + OSAL_LE16_TO_CPU(p_config->indirection_table[i + 8]), + OSAL_LE16_TO_CPU(p_config->indirection_table[i + 9]), + OSAL_LE16_TO_CPU(p_config->indirection_table[i + 10]), + OSAL_LE16_TO_CPU(p_config->indirection_table[i + 11]), + OSAL_LE16_TO_CPU(p_config->indirection_table[i + 12]), + OSAL_LE16_TO_CPU(p_config->indirection_table[i + 13]), + OSAL_LE16_TO_CPU(p_config->indirection_table[i + 14]), + OSAL_LE16_TO_CPU(p_config->indirection_table[i + 15])); } for (i = 0; i < 10; i++) @@ -202,10 +566,12 @@ ecore_sp_vport_update_rss(struct ecore_hwfn *p_hwfn, static void ecore_sp_update_accept_mode(struct ecore_hwfn *p_hwfn, struct vport_update_ramrod_data *p_ramrod, - struct ecore_filter_accept_flags flags) + struct ecore_filter_accept_flags accept_flags) { - p_ramrod->common.update_rx_mode_flg = flags.update_rx_mode_config; - p_ramrod->common.update_tx_mode_flg = flags.update_tx_mode_config; + p_ramrod->common.update_rx_mode_flg = + accept_flags.update_rx_mode_config; + p_ramrod->common.update_tx_mode_flg = + accept_flags.update_tx_mode_config; #ifndef ASIC_ONLY /* On B0 emulation we cannot enable Tx, since this would cause writes @@ -220,83 +586,64 @@ ecore_sp_update_accept_mode(struct ecore_hwfn *p_hwfn, /* Set Rx mode accept flags */ if (p_ramrod->common.update_rx_mode_flg) { - __le16 *state = &p_ramrod->rx_mode.state; - u8 accept_filter = flags.rx_accept_filter; + u8 accept_filter = accept_flags.rx_accept_filter; + u16 state = 0; -/* - * SET_FIELD(*state, ETH_VPORT_RX_MODE_UCAST_DROP_ALL, - * !!(accept_filter & ECORE_ACCEPT_NONE)); - */ - - SET_FIELD(*state, ETH_VPORT_RX_MODE_UCAST_ACCEPT_ALL, - (!!(accept_filter & ECORE_ACCEPT_UCAST_MATCHED) && + SET_FIELD(state, ETH_VPORT_RX_MODE_UCAST_DROP_ALL, + !(!!(accept_filter & ECORE_ACCEPT_UCAST_MATCHED) || !!(accept_filter & ECORE_ACCEPT_UCAST_UNMATCHED))); - SET_FIELD(*state, ETH_VPORT_RX_MODE_UCAST_DROP_ALL, - !(!!(accept_filter & ECORE_ACCEPT_UCAST_MATCHED) || - !!(accept_filter & ECORE_ACCEPT_UCAST_UNMATCHED))); - - SET_FIELD(*state, ETH_VPORT_RX_MODE_UCAST_ACCEPT_UNMATCHED, + SET_FIELD(state, ETH_VPORT_RX_MODE_UCAST_ACCEPT_UNMATCHED, !!(accept_filter & ECORE_ACCEPT_UCAST_UNMATCHED)); -/* - * SET_FIELD(*state, ETH_VPORT_RX_MODE_MCAST_DROP_ALL, - * !!(accept_filter & ECORE_ACCEPT_NONE)); - */ - SET_FIELD(*state, ETH_VPORT_RX_MODE_MCAST_DROP_ALL, + + SET_FIELD(state, ETH_VPORT_RX_MODE_MCAST_DROP_ALL, !(!!(accept_filter & ECORE_ACCEPT_MCAST_MATCHED) || !!(accept_filter & ECORE_ACCEPT_MCAST_UNMATCHED))); - SET_FIELD(*state, ETH_VPORT_RX_MODE_MCAST_ACCEPT_ALL, + SET_FIELD(state, ETH_VPORT_RX_MODE_MCAST_ACCEPT_ALL, (!!(accept_filter & ECORE_ACCEPT_MCAST_MATCHED) && !!(accept_filter & ECORE_ACCEPT_MCAST_UNMATCHED))); - SET_FIELD(*state, ETH_VPORT_RX_MODE_BCAST_ACCEPT_ALL, + SET_FIELD(state, ETH_VPORT_RX_MODE_BCAST_ACCEPT_ALL, !!(accept_filter & ECORE_ACCEPT_BCAST)); + p_ramrod->rx_mode.state = OSAL_CPU_TO_LE16(state); DP_VERBOSE(p_hwfn, ECORE_MSG_SP, - "p_ramrod->rx_mode.state = 0x%x\n", - p_ramrod->rx_mode.state); + "vport[%02x] p_ramrod->rx_mode.state = 0x%x\n", + p_ramrod->common.vport_id, state); } /* Set Tx mode accept flags */ if (p_ramrod->common.update_tx_mode_flg) { - __le16 *state = &p_ramrod->tx_mode.state; - u8 accept_filter = flags.tx_accept_filter; + u8 accept_filter = accept_flags.tx_accept_filter; + u16 state = 0; - SET_FIELD(*state, ETH_VPORT_TX_MODE_UCAST_DROP_ALL, + SET_FIELD(state, ETH_VPORT_TX_MODE_UCAST_DROP_ALL, !!(accept_filter & ECORE_ACCEPT_NONE)); - SET_FIELD(*state, ETH_VPORT_TX_MODE_MCAST_DROP_ALL, + SET_FIELD(state, ETH_VPORT_TX_MODE_MCAST_DROP_ALL, !!(accept_filter & ECORE_ACCEPT_NONE)); - SET_FIELD(*state, ETH_VPORT_TX_MODE_MCAST_ACCEPT_ALL, + SET_FIELD(state, ETH_VPORT_TX_MODE_MCAST_ACCEPT_ALL, (!!(accept_filter & ECORE_ACCEPT_MCAST_MATCHED) && !!(accept_filter & ECORE_ACCEPT_MCAST_UNMATCHED))); - SET_FIELD(*state, ETH_VPORT_TX_MODE_BCAST_ACCEPT_ALL, + SET_FIELD(state, ETH_VPORT_TX_MODE_BCAST_ACCEPT_ALL, !!(accept_filter & ECORE_ACCEPT_BCAST)); - /* @DPDK */ - /* ETH_VPORT_RX_MODE_UCAST_ACCEPT_ALL and - * ETH_VPORT_TX_MODE_UCAST_ACCEPT_ALL - * needs to be set for VF-VF communication to work - * when dest macaddr is unknown. - */ - SET_FIELD(*state, ETH_VPORT_TX_MODE_UCAST_ACCEPT_ALL, - (!!(accept_filter & ECORE_ACCEPT_UCAST_MATCHED) && - !!(accept_filter & ECORE_ACCEPT_UCAST_UNMATCHED))); + p_ramrod->tx_mode.state = OSAL_CPU_TO_LE16(state); DP_VERBOSE(p_hwfn, ECORE_MSG_SP, - "p_ramrod->tx_mode.state = 0x%x\n", - p_ramrod->tx_mode.state); + "vport[%02x] p_ramrod->tx_mode.state = 0x%x\n", + p_ramrod->common.vport_id, state); } } static void -ecore_sp_vport_update_sge_tpa(struct ecore_hwfn *p_hwfn, - struct vport_update_ramrod_data *p_ramrod, +ecore_sp_vport_update_sge_tpa(struct vport_update_ramrod_data *p_ramrod, struct ecore_sge_tpa_params *p_params) { struct eth_vport_tpa_param *p_tpa; + u16 val; if (!p_params) { p_ramrod->common.update_tpa_param_flg = 0; @@ -318,14 +665,16 @@ ecore_sp_vport_update_sge_tpa(struct ecore_hwfn *p_hwfn, p_tpa->tpa_hdr_data_split_flg = p_params->tpa_hdr_data_split_flg; p_tpa->tpa_gro_consistent_flg = p_params->tpa_gro_consistent_flg; p_tpa->tpa_max_aggs_num = p_params->tpa_max_aggs_num; - p_tpa->tpa_max_size = p_params->tpa_max_size; - p_tpa->tpa_min_size_to_start = p_params->tpa_min_size_to_start; - p_tpa->tpa_min_size_to_cont = p_params->tpa_min_size_to_cont; + val = p_params->tpa_max_size; + p_tpa->tpa_max_size = OSAL_CPU_TO_LE16(val); + val = p_params->tpa_min_size_to_start; + p_tpa->tpa_min_size_to_start = OSAL_CPU_TO_LE16(val); + val = p_params->tpa_min_size_to_cont; + p_tpa->tpa_min_size_to_cont = OSAL_CPU_TO_LE16(val); } static void -ecore_sp_update_mcast_bin(struct ecore_hwfn *p_hwfn, - struct vport_update_ramrod_data *p_ramrod, +ecore_sp_update_mcast_bin(struct vport_update_ramrod_data *p_ramrod, struct ecore_sp_vport_update_params *p_params) { int i; @@ -351,12 +700,12 @@ ecore_sp_vport_update(struct ecore_hwfn *p_hwfn, struct ecore_spq_comp_cb *p_comp_data) { struct ecore_rss_params *p_rss_params = p_params->rss_params; + struct vport_update_ramrod_data_cmn *p_cmn; + struct ecore_sp_init_data init_data; struct vport_update_ramrod_data *p_ramrod = OSAL_NULL; struct ecore_spq_entry *p_ent = OSAL_NULL; - enum _ecore_status_t rc = ECORE_NOTIMPL; - struct ecore_sp_init_data init_data; u8 abs_vport_id = 0, val; - u16 wordval; + enum _ecore_status_t rc = ECORE_NOTIMPL; if (IS_VF(p_hwfn->p_dev)) { rc = ecore_vf_pf_vport_update(p_hwfn, p_params); @@ -382,30 +731,31 @@ ecore_sp_vport_update(struct ecore_hwfn *p_hwfn, /* Copy input params to ramrod according to FW struct */ p_ramrod = &p_ent->ramrod.vport_update; + p_cmn = &p_ramrod->common; - p_ramrod->common.vport_id = abs_vport_id; + p_cmn->vport_id = abs_vport_id; - p_ramrod->common.rx_active_flg = p_params->vport_active_rx_flg; - p_ramrod->common.tx_active_flg = p_params->vport_active_tx_flg; - val = p_params->update_vport_active_rx_flg; - p_ramrod->common.update_rx_active_flg = val; - val = p_params->update_vport_active_tx_flg; - p_ramrod->common.update_tx_active_flg = val; + p_cmn->rx_active_flg = p_params->vport_active_rx_flg; + p_cmn->update_rx_active_flg = p_params->update_vport_active_rx_flg; + p_cmn->tx_active_flg = p_params->vport_active_tx_flg; + p_cmn->update_tx_active_flg = p_params->update_vport_active_tx_flg; + + p_cmn->accept_any_vlan = p_params->accept_any_vlan; + val = p_params->update_accept_any_vlan_flg; + p_cmn->update_accept_any_vlan_flg = val; + + p_cmn->inner_vlan_removal_en = p_params->inner_vlan_removal_flg; val = p_params->update_inner_vlan_removal_flg; - p_ramrod->common.update_inner_vlan_removal_en_flg = val; - val = p_params->inner_vlan_removal_flg; - p_ramrod->common.inner_vlan_removal_en = val; - val = p_params->silent_vlan_removal_flg; - p_ramrod->common.silent_vlan_removal_en = val; - val = p_params->update_tx_switching_flg; - p_ramrod->common.update_tx_switching_en_flg = val; + p_cmn->update_inner_vlan_removal_en_flg = val; + + p_cmn->default_vlan_en = p_params->default_vlan_enable_flg; val = p_params->update_default_vlan_enable_flg; - p_ramrod->common.update_default_vlan_en_flg = val; - p_ramrod->common.default_vlan_en = p_params->default_vlan_enable_flg; - val = p_params->update_default_vlan_flg; - p_ramrod->common.update_default_vlan_flg = val; - wordval = p_params->default_vlan; - p_ramrod->common.default_vlan = OSAL_CPU_TO_LE16(wordval); + p_cmn->update_default_vlan_en_flg = val; + + p_cmn->default_vlan = OSAL_CPU_TO_LE16(p_params->default_vlan); + p_cmn->update_default_vlan_flg = p_params->update_default_vlan_flg; + + p_cmn->silent_vlan_removal_en = p_params->silent_vlan_removal_flg; p_ramrod->common.tx_switching_en = p_params->tx_switching_flg; @@ -419,27 +769,29 @@ ecore_sp_vport_update(struct ecore_hwfn *p_hwfn, p_ramrod->common.update_tx_switching_en_flg = 1; } #endif + p_cmn->update_tx_switching_en_flg = p_params->update_tx_switching_flg; + p_cmn->anti_spoofing_en = p_params->anti_spoofing_en; val = p_params->update_anti_spoofing_en_flg; p_ramrod->common.update_anti_spoofing_en_flg = val; - p_ramrod->common.anti_spoofing_en = p_params->anti_spoofing_en; - p_ramrod->common.accept_any_vlan = p_params->accept_any_vlan; - val = p_params->update_accept_any_vlan_flg; - p_ramrod->common.update_accept_any_vlan_flg = val; rc = ecore_sp_vport_update_rss(p_hwfn, p_ramrod, p_rss_params); if (rc != ECORE_SUCCESS) { - /* Return spq entry which is taken in ecore_sp_init_request() */ + /* Return spq entry which is taken in ecore_sp_init_request()*/ ecore_spq_return_entry(p_hwfn, p_ent); return rc; } /* Update mcast bins for VFs, PF doesn't use this functionality */ - ecore_sp_update_mcast_bin(p_hwfn, p_ramrod, p_params); + ecore_sp_update_mcast_bin(p_ramrod, p_params); ecore_sp_update_accept_mode(p_hwfn, p_ramrod, p_params->accept_flags); - ecore_sp_vport_update_sge_tpa(p_hwfn, p_ramrod, - p_params->sge_tpa_params); + ecore_sp_vport_update_sge_tpa(p_ramrod, p_params->sge_tpa_params); + if (p_params->mtu) { + p_ramrod->common.update_mtu_flg = 1; + p_ramrod->common.mtu = OSAL_CPU_TO_LE16(p_params->mtu); + } + return ecore_spq_post(p_hwfn, p_ent, OSAL_NULL); } @@ -449,8 +801,8 @@ enum _ecore_status_t ecore_sp_vport_stop(struct ecore_hwfn *p_hwfn, struct vport_stop_ramrod_data *p_ramrod; struct ecore_sp_init_data init_data; struct ecore_spq_entry *p_ent; - enum _ecore_status_t rc; u8 abs_vport_id = 0; + enum _ecore_status_t rc; if (IS_VF(p_hwfn->p_dev)) return ecore_vf_pf_vport_stop(p_hwfn); @@ -499,20 +851,20 @@ ecore_filter_accept_cmd(struct ecore_dev *p_dev, enum spq_mode comp_mode, struct ecore_spq_comp_cb *p_comp_data) { - struct ecore_sp_vport_update_params update_params; + struct ecore_sp_vport_update_params vport_update_params; int i, rc; /* Prepare and send the vport rx_mode change */ - OSAL_MEMSET(&update_params, 0, sizeof(update_params)); - update_params.vport_id = vport; - update_params.accept_flags = accept_flags; - update_params.update_accept_any_vlan_flg = update_accept_any_vlan; - update_params.accept_any_vlan = accept_any_vlan; + OSAL_MEMSET(&vport_update_params, 0, sizeof(vport_update_params)); + vport_update_params.vport_id = vport; + vport_update_params.accept_flags = accept_flags; + vport_update_params.update_accept_any_vlan_flg = update_accept_any_vlan; + vport_update_params.accept_any_vlan = accept_any_vlan; for_each_hwfn(p_dev, i) { struct ecore_hwfn *p_hwfn = &p_dev->hwfns[i]; - update_params.opaque_fid = p_hwfn->hw_info.opaque_fid; + vport_update_params.opaque_fid = p_hwfn->hw_info.opaque_fid; if (IS_VF(p_dev)) { rc = ecore_vf_pf_accept_flags(p_hwfn, &accept_flags); @@ -521,7 +873,7 @@ ecore_filter_accept_cmd(struct ecore_dev *p_dev, continue; } - rc = ecore_sp_vport_update(p_hwfn, &update_params, + rc = ecore_sp_vport_update(p_hwfn, &vport_update_params, comp_mode, p_comp_data); if (rc != ECORE_SUCCESS) { DP_ERR(p_dev, "Update rx_mode failed %d\n", rc); @@ -542,58 +894,28 @@ ecore_filter_accept_cmd(struct ecore_dev *p_dev, return 0; } -static void ecore_sp_release_queue_cid(struct ecore_hwfn *p_hwfn, - struct ecore_hw_cid_data *p_cid_data) -{ - if (!p_cid_data->b_cid_allocated) - return; - - ecore_cxt_release_cid(p_hwfn, p_cid_data->cid); - p_cid_data->b_cid_allocated = false; -} - enum _ecore_status_t -ecore_sp_eth_rxq_start_ramrod(struct ecore_hwfn *p_hwfn, - u16 opaque_fid, - u32 cid, - u16 rx_queue_id, - u8 vport_id, - u8 stats_id, - u16 sb, - u8 sb_index, - u16 bd_max_bytes, - dma_addr_t bd_chain_phys_addr, - dma_addr_t cqe_pbl_addr, u16 cqe_pbl_size) +ecore_eth_rxq_start_ramrod(struct ecore_hwfn *p_hwfn, + struct ecore_queue_cid *p_cid, + u16 bd_max_bytes, + dma_addr_t bd_chain_phys_addr, + dma_addr_t cqe_pbl_addr, + u16 cqe_pbl_size) { - struct ecore_hw_cid_data *p_rx_cid = &p_hwfn->p_rx_cids[rx_queue_id]; struct rx_queue_start_ramrod_data *p_ramrod = OSAL_NULL; struct ecore_spq_entry *p_ent = OSAL_NULL; - enum _ecore_status_t rc = ECORE_NOTIMPL; struct ecore_sp_init_data init_data; - u16 abs_rx_q_id = 0; - u8 abs_vport_id = 0; - - /* Store information for the stop */ - p_rx_cid->cid = cid; - p_rx_cid->opaque_fid = opaque_fid; - p_rx_cid->vport_id = vport_id; - - rc = ecore_fw_vport(p_hwfn, vport_id, &abs_vport_id); - if (rc != ECORE_SUCCESS) - return rc; - - rc = ecore_fw_l2_queue(p_hwfn, rx_queue_id, &abs_rx_q_id); - if (rc != ECORE_SUCCESS) - return rc; + enum _ecore_status_t rc = ECORE_NOTIMPL; DP_VERBOSE(p_hwfn, ECORE_MSG_SP, - "opaque_fid=0x%x, cid=0x%x, rx_qid=0x%x, vport_id=0x%x, sb_id=0x%x\n", - opaque_fid, cid, rx_queue_id, vport_id, sb); + "opaque_fid=0x%x, cid=0x%x, rx_qzone=0x%x, vport_id=0x%x, sb_id=0x%x\n", + p_cid->opaque_fid, p_cid->cid, p_cid->abs.queue_id, + p_cid->abs.vport_id, p_cid->sb_igu_id); /* Get SPQ entry */ OSAL_MEMSET(&init_data, 0, sizeof(init_data)); - init_data.cid = cid; - init_data.opaque_fid = opaque_fid; + init_data.cid = p_cid->cid; + init_data.opaque_fid = p_cid->opaque_fid; init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK; rc = ecore_sp_init_request(p_hwfn, &p_ent, @@ -604,11 +926,11 @@ ecore_sp_eth_rxq_start_ramrod(struct ecore_hwfn *p_hwfn, p_ramrod = &p_ent->ramrod.rx_queue_start; - p_ramrod->sb_id = OSAL_CPU_TO_LE16(sb); - p_ramrod->sb_index = sb_index; - p_ramrod->vport_id = abs_vport_id; - p_ramrod->stats_counter_id = stats_id; - p_ramrod->rx_queue_id = OSAL_CPU_TO_LE16(abs_rx_q_id); + p_ramrod->sb_id = OSAL_CPU_TO_LE16(p_cid->sb_igu_id); + p_ramrod->sb_index = p_cid->sb_idx; + p_ramrod->vport_id = p_cid->abs.vport_id; + p_ramrod->stats_counter_id = p_cid->abs.stats_id; + p_ramrod->rx_queue_id = OSAL_CPU_TO_LE16(p_cid->abs.queue_id); p_ramrod->complete_cqe_flg = 0; p_ramrod->complete_event_flg = 1; @@ -618,83 +940,91 @@ ecore_sp_eth_rxq_start_ramrod(struct ecore_hwfn *p_hwfn, p_ramrod->num_of_pbl_pages = OSAL_CPU_TO_LE16(cqe_pbl_size); DMA_REGPAIR_LE(p_ramrod->cqe_pbl_addr, cqe_pbl_addr); + if (p_cid->vfid != ECORE_QUEUE_CID_PF) { + bool b_legacy_vf = !!(p_cid->vf_legacy & + ECORE_QCID_LEGACY_VF_RX_PROD); + + p_ramrod->vf_rx_prod_index = p_cid->vf_qid; + DP_VERBOSE(p_hwfn, ECORE_MSG_SP, + "Queue%s is meant for VF rxq[%02x]\n", + b_legacy_vf ? " [legacy]" : "", + p_cid->vf_qid); + p_ramrod->vf_rx_prod_use_zone_a = b_legacy_vf; + } + return ecore_spq_post(p_hwfn, p_ent, OSAL_NULL); } -enum _ecore_status_t ecore_sp_eth_rx_queue_start(struct ecore_hwfn *p_hwfn, - u16 opaque_fid, - u8 rx_queue_id, - u8 vport_id, - u8 stats_id, - u16 sb, - u8 sb_index, - u16 bd_max_bytes, - dma_addr_t bd_chain_phys_addr, - dma_addr_t cqe_pbl_addr, - u16 cqe_pbl_size, - void OSAL_IOMEM * *pp_prod) +static enum _ecore_status_t +ecore_eth_pf_rx_queue_start(struct ecore_hwfn *p_hwfn, + struct ecore_queue_cid *p_cid, + u16 bd_max_bytes, + dma_addr_t bd_chain_phys_addr, + dma_addr_t cqe_pbl_addr, + u16 cqe_pbl_size, + void OSAL_IOMEM * *pp_prod) { - struct ecore_hw_cid_data *p_rx_cid = &p_hwfn->p_rx_cids[rx_queue_id]; - u8 abs_stats_id = 0; - u16 abs_l2_queue = 0; - enum _ecore_status_t rc; - u64 init_prod_val = 0; + u32 init_prod_val = 0; - if (IS_VF(p_hwfn->p_dev)) { - return ecore_vf_pf_rxq_start(p_hwfn, - rx_queue_id, - sb, - sb_index, - bd_max_bytes, - bd_chain_phys_addr, - cqe_pbl_addr, - cqe_pbl_size, pp_prod); - } - - rc = ecore_fw_l2_queue(p_hwfn, rx_queue_id, &abs_l2_queue); - if (rc != ECORE_SUCCESS) - return rc; - - rc = ecore_fw_vport(p_hwfn, stats_id, &abs_stats_id); - if (rc != ECORE_SUCCESS) - return rc; - - *pp_prod = (u8 OSAL_IOMEM *)p_hwfn->regview + - GTT_BAR0_MAP_REG_MSDM_RAM + MSTORM_PRODS_OFFSET(abs_l2_queue); + *pp_prod = (u8 OSAL_IOMEM *) + p_hwfn->regview + + GTT_BAR0_MAP_REG_MSDM_RAM + + MSTORM_ETH_PF_PRODS_OFFSET(p_cid->abs.queue_id); /* Init the rcq, rx bd and rx sge (if valid) producers to 0 */ - __internal_ram_wr(p_hwfn, *pp_prod, sizeof(u64), + __internal_ram_wr(p_hwfn, *pp_prod, sizeof(u32), (u32 *)(&init_prod_val)); - /* Allocate a CID for the queue */ - rc = ecore_cxt_acquire_cid(p_hwfn, PROTOCOLID_ETH, &p_rx_cid->cid); - if (rc != ECORE_SUCCESS) { - DP_NOTICE(p_hwfn, true, "Failed to acquire cid\n"); - return rc; - } - p_rx_cid->b_cid_allocated = true; + return ecore_eth_rxq_start_ramrod(p_hwfn, p_cid, + bd_max_bytes, + bd_chain_phys_addr, + cqe_pbl_addr, cqe_pbl_size); +} - rc = ecore_sp_eth_rxq_start_ramrod(p_hwfn, - opaque_fid, - p_rx_cid->cid, - rx_queue_id, - vport_id, - abs_stats_id, - sb, - sb_index, +enum _ecore_status_t +ecore_eth_rx_queue_start(struct ecore_hwfn *p_hwfn, + u16 opaque_fid, + struct ecore_queue_start_common_params *p_params, + u16 bd_max_bytes, + dma_addr_t bd_chain_phys_addr, + dma_addr_t cqe_pbl_addr, + u16 cqe_pbl_size, + struct ecore_rxq_start_ret_params *p_ret_params) +{ + struct ecore_queue_cid *p_cid; + enum _ecore_status_t rc; + + /* Allocate a CID for the queue */ + p_cid = ecore_eth_queue_to_cid_pf(p_hwfn, opaque_fid, true, p_params); + if (p_cid == OSAL_NULL) + return ECORE_NOMEM; + + if (IS_PF(p_hwfn->p_dev)) + rc = ecore_eth_pf_rx_queue_start(p_hwfn, p_cid, + bd_max_bytes, + bd_chain_phys_addr, + cqe_pbl_addr, cqe_pbl_size, + &p_ret_params->p_prod); + else + rc = ecore_vf_pf_rxq_start(p_hwfn, p_cid, bd_max_bytes, bd_chain_phys_addr, - cqe_pbl_addr, cqe_pbl_size); + cqe_pbl_addr, + cqe_pbl_size, + &p_ret_params->p_prod); + /* Provide the caller with a reference to as handler */ if (rc != ECORE_SUCCESS) - ecore_sp_release_queue_cid(p_hwfn, p_rx_cid); + ecore_eth_queue_cid_release(p_hwfn, p_cid); + else + p_ret_params->p_handle = (void *)p_cid; return rc; } enum _ecore_status_t ecore_sp_eth_rx_queues_update(struct ecore_hwfn *p_hwfn, - u16 rx_queue_id, + void **pp_rxq_handles, u8 num_rxqs, u8 complete_cqe_flg, u8 complete_event_flg, @@ -703,15 +1033,15 @@ ecore_sp_eth_rx_queues_update(struct ecore_hwfn *p_hwfn, { struct rx_queue_update_ramrod_data *p_ramrod = OSAL_NULL; struct ecore_spq_entry *p_ent = OSAL_NULL; - enum _ecore_status_t rc = ECORE_NOTIMPL; struct ecore_sp_init_data init_data; - struct ecore_hw_cid_data *p_rx_cid; - u16 qid, abs_rx_q_id = 0; + struct ecore_queue_cid *p_cid; + enum _ecore_status_t rc = ECORE_NOTIMPL; u8 i; if (IS_VF(p_hwfn->p_dev)) return ecore_vf_pf_rxqs_update(p_hwfn, - rx_queue_id, + (struct ecore_queue_cid **) + pp_rxq_handles, num_rxqs, complete_cqe_flg, complete_event_flg); @@ -721,12 +1051,11 @@ ecore_sp_eth_rx_queues_update(struct ecore_hwfn *p_hwfn, init_data.p_comp_data = p_comp_data; for (i = 0; i < num_rxqs; i++) { - qid = rx_queue_id + i; - p_rx_cid = &p_hwfn->p_rx_cids[qid]; + p_cid = ((struct ecore_queue_cid **)pp_rxq_handles)[i]; /* Get SPQ entry */ - init_data.cid = p_rx_cid->cid; - init_data.opaque_fid = p_rx_cid->opaque_fid; + init_data.cid = p_cid->cid; + init_data.opaque_fid = p_cid->opaque_fid; rc = ecore_sp_init_request(p_hwfn, &p_ent, ETH_RAMROD_RX_QUEUE_UPDATE, @@ -735,41 +1064,34 @@ ecore_sp_eth_rx_queues_update(struct ecore_hwfn *p_hwfn, return rc; p_ramrod = &p_ent->ramrod.rx_queue_update; + p_ramrod->vport_id = p_cid->abs.vport_id; - ecore_fw_vport(p_hwfn, p_rx_cid->vport_id, &p_ramrod->vport_id); - ecore_fw_l2_queue(p_hwfn, qid, &abs_rx_q_id); - p_ramrod->rx_queue_id = OSAL_CPU_TO_LE16(abs_rx_q_id); + p_ramrod->rx_queue_id = OSAL_CPU_TO_LE16(p_cid->abs.queue_id); p_ramrod->complete_cqe_flg = complete_cqe_flg; p_ramrod->complete_event_flg = complete_event_flg; rc = ecore_spq_post(p_hwfn, p_ent, OSAL_NULL); - if (rc) + if (rc != ECORE_SUCCESS) return rc; } return rc; } -enum _ecore_status_t -ecore_sp_eth_rx_queue_stop(struct ecore_hwfn *p_hwfn, - u16 rx_queue_id, - bool eq_completion_only, bool cqe_completion) +static enum _ecore_status_t +ecore_eth_pf_rx_queue_stop(struct ecore_hwfn *p_hwfn, + struct ecore_queue_cid *p_cid, + bool b_eq_completion_only, + bool b_cqe_completion) { - struct ecore_hw_cid_data *p_rx_cid = &p_hwfn->p_rx_cids[rx_queue_id]; struct rx_queue_stop_ramrod_data *p_ramrod = OSAL_NULL; struct ecore_spq_entry *p_ent = OSAL_NULL; - enum _ecore_status_t rc = ECORE_NOTIMPL; struct ecore_sp_init_data init_data; - u16 abs_rx_q_id = 0; + enum _ecore_status_t rc; - if (IS_VF(p_hwfn->p_dev)) - return ecore_vf_pf_rxq_stop(p_hwfn, rx_queue_id, - cqe_completion); - - /* Get SPQ entry */ OSAL_MEMSET(&init_data, 0, sizeof(init_data)); - init_data.cid = p_rx_cid->cid; - init_data.opaque_fid = p_rx_cid->opaque_fid; + init_data.cid = p_cid->cid; + init_data.opaque_fid = p_cid->opaque_fid; init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK; rc = ecore_sp_init_request(p_hwfn, &p_ent, @@ -779,67 +1101,56 @@ ecore_sp_eth_rx_queue_stop(struct ecore_hwfn *p_hwfn, return rc; p_ramrod = &p_ent->ramrod.rx_queue_stop; - - ecore_fw_vport(p_hwfn, p_rx_cid->vport_id, &p_ramrod->vport_id); - ecore_fw_l2_queue(p_hwfn, rx_queue_id, &abs_rx_q_id); - p_ramrod->rx_queue_id = OSAL_CPU_TO_LE16(abs_rx_q_id); + p_ramrod->vport_id = p_cid->abs.vport_id; + p_ramrod->rx_queue_id = OSAL_CPU_TO_LE16(p_cid->abs.queue_id); /* Cleaning the queue requires the completion to arrive there. * In addition, VFs require the answer to come as eqe to PF. */ - p_ramrod->complete_cqe_flg = (!!(p_rx_cid->opaque_fid == - p_hwfn->hw_info.opaque_fid) && - !eq_completion_only) || cqe_completion; - p_ramrod->complete_event_flg = !(p_rx_cid->opaque_fid == - p_hwfn->hw_info.opaque_fid) || - eq_completion_only; + p_ramrod->complete_cqe_flg = ((p_cid->vfid == ECORE_QUEUE_CID_PF) && + !b_eq_completion_only) || + b_cqe_completion; + p_ramrod->complete_event_flg = (p_cid->vfid != ECORE_QUEUE_CID_PF) || + b_eq_completion_only; - rc = ecore_spq_post(p_hwfn, p_ent, OSAL_NULL); - if (rc != ECORE_SUCCESS) - return rc; + return ecore_spq_post(p_hwfn, p_ent, OSAL_NULL); +} - ecore_sp_release_queue_cid(p_hwfn, p_rx_cid); +enum _ecore_status_t ecore_eth_rx_queue_stop(struct ecore_hwfn *p_hwfn, + void *p_rxq, + bool eq_completion_only, + bool cqe_completion) +{ + struct ecore_queue_cid *p_cid = (struct ecore_queue_cid *)p_rxq; + enum _ecore_status_t rc = ECORE_NOTIMPL; + if (IS_PF(p_hwfn->p_dev)) + rc = ecore_eth_pf_rx_queue_stop(p_hwfn, p_cid, + eq_completion_only, + cqe_completion); + else + rc = ecore_vf_pf_rxq_stop(p_hwfn, p_cid, cqe_completion); + + if (rc == ECORE_SUCCESS) + ecore_eth_queue_cid_release(p_hwfn, p_cid); return rc; } enum _ecore_status_t -ecore_sp_eth_txq_start_ramrod(struct ecore_hwfn *p_hwfn, - u16 opaque_fid, - u16 tx_queue_id, - u32 cid, - u8 vport_id, - u8 stats_id, - u16 sb, - u8 sb_index, - dma_addr_t pbl_addr, - u16 pbl_size, - union ecore_qm_pq_params *p_pq_params) +ecore_eth_txq_start_ramrod(struct ecore_hwfn *p_hwfn, + struct ecore_queue_cid *p_cid, + dma_addr_t pbl_addr, u16 pbl_size, + u16 pq_id) { - struct ecore_hw_cid_data *p_tx_cid = &p_hwfn->p_tx_cids[tx_queue_id]; struct tx_queue_start_ramrod_data *p_ramrod = OSAL_NULL; struct ecore_spq_entry *p_ent = OSAL_NULL; - enum _ecore_status_t rc = ECORE_NOTIMPL; struct ecore_sp_init_data init_data; - u16 pq_id, abs_tx_q_id = 0; - u8 abs_vport_id; - - /* Store information for the stop */ - p_tx_cid->cid = cid; - p_tx_cid->opaque_fid = opaque_fid; - - rc = ecore_fw_vport(p_hwfn, vport_id, &abs_vport_id); - if (rc != ECORE_SUCCESS) - return rc; - - rc = ecore_fw_l2_queue(p_hwfn, tx_queue_id, &abs_tx_q_id); - if (rc != ECORE_SUCCESS) - return rc; + enum _ecore_status_t rc = ECORE_NOTIMPL; /* Get SPQ entry */ OSAL_MEMSET(&init_data, 0, sizeof(init_data)); - init_data.cid = cid; - init_data.opaque_fid = opaque_fid; + init_data.cid = p_cid->cid; + init_data.opaque_fid = p_cid->opaque_fid; init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK; rc = ecore_sp_init_request(p_hwfn, &p_ent, @@ -849,108 +1160,89 @@ ecore_sp_eth_txq_start_ramrod(struct ecore_hwfn *p_hwfn, return rc; p_ramrod = &p_ent->ramrod.tx_queue_start; - p_ramrod->vport_id = abs_vport_id; + p_ramrod->vport_id = p_cid->abs.vport_id; - p_ramrod->sb_id = OSAL_CPU_TO_LE16(sb); - p_ramrod->sb_index = sb_index; - p_ramrod->stats_counter_id = stats_id; + p_ramrod->sb_id = OSAL_CPU_TO_LE16(p_cid->sb_igu_id); + p_ramrod->sb_index = p_cid->sb_idx; + p_ramrod->stats_counter_id = p_cid->abs.stats_id; - p_ramrod->queue_zone_id = OSAL_CPU_TO_LE16(abs_tx_q_id); + p_ramrod->queue_zone_id = OSAL_CPU_TO_LE16(p_cid->abs.queue_id); + p_ramrod->same_as_last_id = OSAL_CPU_TO_LE16(p_cid->abs.queue_id); p_ramrod->pbl_size = OSAL_CPU_TO_LE16(pbl_size); - p_ramrod->pbl_base_addr.hi = DMA_HI_LE(pbl_addr); - p_ramrod->pbl_base_addr.lo = DMA_LO_LE(pbl_addr); + DMA_REGPAIR_LE(p_ramrod->pbl_base_addr, pbl_addr); - pq_id = ecore_get_qm_pq(p_hwfn, PROTOCOLID_ETH, p_pq_params); p_ramrod->qm_pq_id = OSAL_CPU_TO_LE16(pq_id); return ecore_spq_post(p_hwfn, p_ent, OSAL_NULL); } -enum _ecore_status_t ecore_sp_eth_tx_queue_start(struct ecore_hwfn *p_hwfn, - u16 opaque_fid, - u16 tx_queue_id, - u8 vport_id, - u8 stats_id, - u16 sb, - u8 sb_index, - dma_addr_t pbl_addr, - u16 pbl_size, - void OSAL_IOMEM * *pp_doorbell) +static enum _ecore_status_t +ecore_eth_pf_tx_queue_start(struct ecore_hwfn *p_hwfn, + struct ecore_queue_cid *p_cid, + u8 tc, + dma_addr_t pbl_addr, u16 pbl_size, + void OSAL_IOMEM * *pp_doorbell) { - struct ecore_hw_cid_data *p_tx_cid = &p_hwfn->p_tx_cids[tx_queue_id]; - union ecore_qm_pq_params pq_params; enum _ecore_status_t rc; - u8 abs_stats_id = 0; - - if (IS_VF(p_hwfn->p_dev)) { - return ecore_vf_pf_txq_start(p_hwfn, - tx_queue_id, - sb, - sb_index, - pbl_addr, pbl_size, pp_doorbell); - } - - rc = ecore_fw_vport(p_hwfn, stats_id, &abs_stats_id); - if (rc != ECORE_SUCCESS) - return rc; - - OSAL_MEMSET(p_tx_cid, 0, sizeof(*p_tx_cid)); - OSAL_MEMSET(&pq_params, 0, sizeof(pq_params)); - - /* Allocate a CID for the queue */ - rc = ecore_cxt_acquire_cid(p_hwfn, PROTOCOLID_ETH, &p_tx_cid->cid); - if (rc != ECORE_SUCCESS) { - DP_NOTICE(p_hwfn, true, "Failed to acquire cid\n"); - return rc; - } - p_tx_cid->b_cid_allocated = true; - - DP_VERBOSE(p_hwfn, ECORE_MSG_SP, - "opaque_fid=0x%x, cid=0x%x, tx_qid=0x%x, vport_id=0x%x, sb_id=0x%x\n", - opaque_fid, p_tx_cid->cid, tx_queue_id, vport_id, sb); /* TODO - set tc in the pq_params for multi-cos */ - rc = ecore_sp_eth_txq_start_ramrod(p_hwfn, - opaque_fid, - tx_queue_id, - p_tx_cid->cid, - vport_id, - abs_stats_id, - sb, - sb_index, - pbl_addr, pbl_size, &pq_params); + rc = ecore_eth_txq_start_ramrod(p_hwfn, p_cid, + pbl_addr, pbl_size, + ecore_get_cm_pq_idx_mcos(p_hwfn, tc)); + if (rc != ECORE_SUCCESS) + return rc; - *pp_doorbell = (u8 OSAL_IOMEM *)p_hwfn->doorbells + - DB_ADDR(p_tx_cid->cid, DQ_DEMS_LEGACY); + /* Provide the caller with the necessary return values */ + *pp_doorbell = (u8 OSAL_IOMEM *) + p_hwfn->doorbells + + DB_ADDR(p_cid->cid, DQ_DEMS_LEGACY); + + return ECORE_SUCCESS; +} + +enum _ecore_status_t +ecore_eth_tx_queue_start(struct ecore_hwfn *p_hwfn, u16 opaque_fid, + struct ecore_queue_start_common_params *p_params, + u8 tc, + dma_addr_t pbl_addr, u16 pbl_size, + struct ecore_txq_start_ret_params *p_ret_params) +{ + struct ecore_queue_cid *p_cid; + enum _ecore_status_t rc; + + p_cid = ecore_eth_queue_to_cid_pf(p_hwfn, opaque_fid, false, p_params); + if (p_cid == OSAL_NULL) + return ECORE_INVAL; + + if (IS_PF(p_hwfn->p_dev)) + rc = ecore_eth_pf_tx_queue_start(p_hwfn, p_cid, tc, + pbl_addr, pbl_size, + &p_ret_params->p_doorbell); + else + rc = ecore_vf_pf_txq_start(p_hwfn, p_cid, + pbl_addr, pbl_size, + &p_ret_params->p_doorbell); if (rc != ECORE_SUCCESS) - ecore_sp_release_queue_cid(p_hwfn, p_tx_cid); + ecore_eth_queue_cid_release(p_hwfn, p_cid); + else + p_ret_params->p_handle = (void *)p_cid; return rc; } -enum _ecore_status_t ecore_sp_eth_tx_queue_update(struct ecore_hwfn *p_hwfn) +static enum _ecore_status_t +ecore_eth_pf_tx_queue_stop(struct ecore_hwfn *p_hwfn, + struct ecore_queue_cid *p_cid) { - return ECORE_NOTIMPL; -} - -enum _ecore_status_t ecore_sp_eth_tx_queue_stop(struct ecore_hwfn *p_hwfn, - u16 tx_queue_id) -{ - struct ecore_hw_cid_data *p_tx_cid = &p_hwfn->p_tx_cids[tx_queue_id]; - struct tx_queue_stop_ramrod_data *p_ramrod = OSAL_NULL; struct ecore_spq_entry *p_ent = OSAL_NULL; - enum _ecore_status_t rc = ECORE_NOTIMPL; struct ecore_sp_init_data init_data; + enum _ecore_status_t rc; - if (IS_VF(p_hwfn->p_dev)) - return ecore_vf_pf_txq_stop(p_hwfn, tx_queue_id); - - /* Get SPQ entry */ OSAL_MEMSET(&init_data, 0, sizeof(init_data)); - init_data.cid = p_tx_cid->cid; - init_data.opaque_fid = p_tx_cid->opaque_fid; + init_data.cid = p_cid->cid; + init_data.opaque_fid = p_cid->opaque_fid; init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK; rc = ecore_sp_init_request(p_hwfn, &p_ent, @@ -959,13 +1251,22 @@ enum _ecore_status_t ecore_sp_eth_tx_queue_stop(struct ecore_hwfn *p_hwfn, if (rc != ECORE_SUCCESS) return rc; - p_ramrod = &p_ent->ramrod.tx_queue_stop; + return ecore_spq_post(p_hwfn, p_ent, OSAL_NULL); +} - rc = ecore_spq_post(p_hwfn, p_ent, OSAL_NULL); - if (rc != ECORE_SUCCESS) - return rc; +enum _ecore_status_t ecore_eth_tx_queue_stop(struct ecore_hwfn *p_hwfn, + void *p_handle) +{ + struct ecore_queue_cid *p_cid = (struct ecore_queue_cid *)p_handle; + enum _ecore_status_t rc; - ecore_sp_release_queue_cid(p_hwfn, p_tx_cid); + if (IS_PF(p_hwfn->p_dev)) + rc = ecore_eth_pf_tx_queue_stop(p_hwfn, p_cid); + else + rc = ecore_vf_pf_txq_stop(p_hwfn, p_cid); + + if (rc == ECORE_SUCCESS) + ecore_eth_queue_cid_release(p_hwfn, p_cid); return rc; } @@ -991,17 +1292,6 @@ ecore_filter_action(enum ecore_filter_opcode opcode) return action; } -static void ecore_set_fw_mac_addr(__le16 *fw_msb, - __le16 *fw_mid, __le16 *fw_lsb, u8 *mac) -{ - ((u8 *)fw_msb)[0] = mac[1]; - ((u8 *)fw_msb)[1] = mac[0]; - ((u8 *)fw_mid)[0] = mac[3]; - ((u8 *)fw_mid)[1] = mac[2]; - ((u8 *)fw_lsb)[0] = mac[5]; - ((u8 *)fw_lsb)[1] = mac[4]; -} - static enum _ecore_status_t ecore_filter_ucast_common(struct ecore_hwfn *p_hwfn, u16 opaque_fid, @@ -1011,8 +1301,8 @@ ecore_filter_ucast_common(struct ecore_hwfn *p_hwfn, enum spq_mode comp_mode, struct ecore_spq_comp_cb *p_comp_data) { - struct vport_filter_update_ramrod_data *p_ramrod; u8 vport_to_add_to = 0, vport_to_remove_from = 0; + struct vport_filter_update_ramrod_data *p_ramrod; struct eth_filter_cmd *p_first_filter; struct eth_filter_cmd *p_second_filter; struct ecore_sp_init_data init_data; @@ -1096,6 +1386,9 @@ ecore_filter_ucast_common(struct ecore_hwfn *p_hwfn, case ECORE_FILTER_VNI: p_first_filter->type = ETH_FILTER_TYPE_VNI; break; + case ECORE_FILTER_UNUSED: /* @DPDK */ + p_first_filter->type = MAX_ETH_FILTER_TYPE; + break; } if ((p_first_filter->type == ETH_FILTER_TYPE_MAC) || @@ -1217,8 +1510,7 @@ ecore_sp_eth_filter_ucast(struct ecore_hwfn *p_hwfn, * Note: crc32_length MUST be aligned to 8 * Return: ******************************************************************************/ -static u32 ecore_calc_crc32c(u8 *crc32_packet, - u32 crc32_length, u32 crc32_seed, u8 complement) +static u32 ecore_calc_crc32c(u8 *crc32_packet, u32 crc32_length, u32 crc32_seed) { u32 byte = 0, bit = 0, crc32_result = crc32_seed; u8 msb = 0, current_byte = 0; @@ -1243,41 +1535,43 @@ static u32 ecore_calc_crc32c(u8 *crc32_packet, return crc32_result; } -static OSAL_INLINE u32 ecore_crc32c_le(u32 seed, u8 *mac, u32 len) +static u32 ecore_crc32c_le(u32 seed, u8 *mac) { u32 packet_buf[2] = { 0 }; OSAL_MEMCPY((u8 *)(&packet_buf[0]), &mac[0], 6); - return ecore_calc_crc32c((u8 *)packet_buf, 8, seed, 0); + return ecore_calc_crc32c((u8 *)packet_buf, 8, seed); } u8 ecore_mcast_bin_from_mac(u8 *mac) { - u32 crc = ecore_crc32c_le(ETH_MULTICAST_BIN_FROM_MAC_SEED, - mac, ETH_ALEN); + u32 crc = ecore_crc32c_le(ETH_MULTICAST_BIN_FROM_MAC_SEED, mac); return crc & 0xff; } static enum _ecore_status_t ecore_sp_eth_filter_mcast(struct ecore_hwfn *p_hwfn, - u16 opaque_fid, struct ecore_filter_mcast *p_filter_cmd, enum spq_mode comp_mode, struct ecore_spq_comp_cb *p_comp_data) { - struct vport_update_ramrod_data *p_ramrod = OSAL_NULL; unsigned long bins[ETH_MULTICAST_MAC_BINS_IN_REGS]; + struct vport_update_ramrod_data *p_ramrod = OSAL_NULL; struct ecore_spq_entry *p_ent = OSAL_NULL; struct ecore_sp_init_data init_data; - enum _ecore_status_t rc; u8 abs_vport_id = 0; + enum _ecore_status_t rc; int i; - rc = ecore_fw_vport(p_hwfn, - (p_filter_cmd->opcode == ECORE_FILTER_ADD) ? - p_filter_cmd->vport_to_add_to : - p_filter_cmd->vport_to_remove_from, &abs_vport_id); + if (p_filter_cmd->opcode == ECORE_FILTER_ADD) + rc = ecore_fw_vport(p_hwfn, + p_filter_cmd->vport_to_add_to, + &abs_vport_id); + else + rc = ecore_fw_vport(p_hwfn, + p_filter_cmd->vport_to_remove_from, + &abs_vport_id); if (rc != ECORE_SUCCESS) return rc; @@ -1304,11 +1598,10 @@ ecore_sp_eth_filter_mcast(struct ecore_hwfn *p_hwfn, 0, sizeof(p_ramrod->approx_mcast.bins)); OSAL_MEMSET(bins, 0, sizeof(unsigned long) * ETH_MULTICAST_MAC_BINS_IN_REGS); - + /* filter ADD op is explicit set op and it removes + * any existing filters for the vport. + */ if (p_filter_cmd->opcode == ECORE_FILTER_ADD) { - /* filter ADD op is explicit set op and it removes - * any existing filters for the vport. - */ for (i = 0; i < p_filter_cmd->num_mc_addrs; i++) { u32 bit; @@ -1360,7 +1653,6 @@ ecore_filter_mcast_cmd(struct ecore_dev *p_dev, } rc = ecore_sp_eth_filter_mcast(p_hwfn, - p_hwfn->hw_info.opaque_fid, p_filter_cmd, comp_mode, p_comp_data); if (rc != ECORE_SUCCESS) @@ -1381,14 +1673,16 @@ ecore_filter_ucast_cmd(struct ecore_dev *p_dev, for_each_hwfn(p_dev, i) { struct ecore_hwfn *p_hwfn = &p_dev->hwfns[i]; + u16 opaque_fid; if (IS_VF(p_dev)) { rc = ecore_vf_pf_filter_ucast(p_hwfn, p_filter_cmd); continue; } + opaque_fid = p_hwfn->hw_info.opaque_fid; rc = ecore_sp_eth_filter_ucast(p_hwfn, - p_hwfn->hw_info.opaque_fid, + opaque_fid, p_filter_cmd, comp_mode, p_comp_data); if (rc != ECORE_SUCCESS) @@ -1398,77 +1692,6 @@ ecore_filter_ucast_cmd(struct ecore_dev *p_dev, return rc; } -/* IOV related */ -enum _ecore_status_t ecore_sp_vf_start(struct ecore_hwfn *p_hwfn, - u32 concrete_vfid, u16 opaque_vfid) -{ - struct vf_start_ramrod_data *p_ramrod = OSAL_NULL; - struct ecore_spq_entry *p_ent = OSAL_NULL; - enum _ecore_status_t rc = ECORE_NOTIMPL; - struct ecore_sp_init_data init_data; - - /* Get SPQ entry */ - OSAL_MEMSET(&init_data, 0, sizeof(init_data)); - init_data.cid = ecore_spq_get_cid(p_hwfn); - init_data.opaque_fid = opaque_vfid; - init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK; - - rc = ecore_sp_init_request(p_hwfn, &p_ent, - COMMON_RAMROD_VF_START, - PROTOCOLID_COMMON, &init_data); - if (rc != ECORE_SUCCESS) - return rc; - - p_ramrod = &p_ent->ramrod.vf_start; - - p_ramrod->vf_id = GET_FIELD(concrete_vfid, PXP_CONCRETE_FID_VFID); - p_ramrod->opaque_fid = OSAL_CPU_TO_LE16(opaque_vfid); - - switch (p_hwfn->hw_info.personality) { - case ECORE_PCI_ETH: - p_ramrod->personality = PERSONALITY_ETH; - break; - default: - DP_NOTICE(p_hwfn, true, "Unknown VF personality %d\n", - p_hwfn->hw_info.personality); - return ECORE_INVAL; - } - - return ecore_spq_post(p_hwfn, p_ent, OSAL_NULL); -} - -enum _ecore_status_t ecore_sp_vf_update(struct ecore_hwfn *p_hwfn) -{ - return ECORE_NOTIMPL; -} - -enum _ecore_status_t ecore_sp_vf_stop(struct ecore_hwfn *p_hwfn, - u32 concrete_vfid, u16 opaque_vfid) -{ - enum _ecore_status_t rc = ECORE_NOTIMPL; - struct vf_stop_ramrod_data *p_ramrod = OSAL_NULL; - struct ecore_spq_entry *p_ent = OSAL_NULL; - struct ecore_sp_init_data init_data; - - /* Get SPQ entry */ - OSAL_MEMSET(&init_data, 0, sizeof(init_data)); - init_data.cid = ecore_spq_get_cid(p_hwfn); - init_data.opaque_fid = opaque_vfid; - init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK; - - rc = ecore_sp_init_request(p_hwfn, &p_ent, - COMMON_RAMROD_VF_STOP, - PROTOCOLID_COMMON, &init_data); - if (rc != ECORE_SUCCESS) - return rc; - - p_ramrod = &p_ent->ramrod.vf_stop; - - p_ramrod->vf_id = GET_FIELD(concrete_vfid, PXP_CONCRETE_FID_VFID); - - return ecore_spq_post(p_hwfn, p_ent, OSAL_NULL); -} - /* Statistics related code */ static void __ecore_get_vport_pstats_addrlen(struct ecore_hwfn *p_hwfn, u32 *p_addr, u32 *p_len, @@ -1501,19 +1724,25 @@ static void __ecore_get_vport_pstats(struct ecore_hwfn *p_hwfn, OSAL_MEMSET(&pstats, 0, sizeof(pstats)); ecore_memcpy_from(p_hwfn, p_ptt, &pstats, pstats_addr, pstats_len); - p_stats->tx_ucast_bytes += HILO_64_REGPAIR(pstats.sent_ucast_bytes); - p_stats->tx_mcast_bytes += HILO_64_REGPAIR(pstats.sent_mcast_bytes); - p_stats->tx_bcast_bytes += HILO_64_REGPAIR(pstats.sent_bcast_bytes); - p_stats->tx_ucast_pkts += HILO_64_REGPAIR(pstats.sent_ucast_pkts); - p_stats->tx_mcast_pkts += HILO_64_REGPAIR(pstats.sent_mcast_pkts); - p_stats->tx_bcast_pkts += HILO_64_REGPAIR(pstats.sent_bcast_pkts); - p_stats->tx_err_drop_pkts += HILO_64_REGPAIR(pstats.error_drop_pkts); + p_stats->common.tx_ucast_bytes += + HILO_64_REGPAIR(pstats.sent_ucast_bytes); + p_stats->common.tx_mcast_bytes += + HILO_64_REGPAIR(pstats.sent_mcast_bytes); + p_stats->common.tx_bcast_bytes += + HILO_64_REGPAIR(pstats.sent_bcast_bytes); + p_stats->common.tx_ucast_pkts += + HILO_64_REGPAIR(pstats.sent_ucast_pkts); + p_stats->common.tx_mcast_pkts += + HILO_64_REGPAIR(pstats.sent_mcast_pkts); + p_stats->common.tx_bcast_pkts += + HILO_64_REGPAIR(pstats.sent_bcast_pkts); + p_stats->common.tx_err_drop_pkts += + HILO_64_REGPAIR(pstats.error_drop_pkts); } static void __ecore_get_vport_tstats(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, - struct ecore_eth_stats *p_stats, - u16 statistics_bin) + struct ecore_eth_stats *p_stats) { struct tstorm_per_port_stat tstats; u32 tstats_addr, tstats_len; @@ -1533,10 +1762,10 @@ static void __ecore_get_vport_tstats(struct ecore_hwfn *p_hwfn, OSAL_MEMSET(&tstats, 0, sizeof(tstats)); ecore_memcpy_from(p_hwfn, p_ptt, &tstats, tstats_addr, tstats_len); - p_stats->mftag_filter_discards += - HILO_64_REGPAIR(tstats.mftag_filter_discard); - p_stats->mac_filter_discards += - HILO_64_REGPAIR(tstats.eth_mac_filter_discard); + p_stats->common.mftag_filter_discards += + HILO_64_REGPAIR(tstats.mftag_filter_discard); + p_stats->common.mac_filter_discards += + HILO_64_REGPAIR(tstats.eth_mac_filter_discard); } static void __ecore_get_vport_ustats_addrlen(struct ecore_hwfn *p_hwfn, @@ -1570,12 +1799,18 @@ static void __ecore_get_vport_ustats(struct ecore_hwfn *p_hwfn, OSAL_MEMSET(&ustats, 0, sizeof(ustats)); ecore_memcpy_from(p_hwfn, p_ptt, &ustats, ustats_addr, ustats_len); - p_stats->rx_ucast_bytes += HILO_64_REGPAIR(ustats.rcv_ucast_bytes); - p_stats->rx_mcast_bytes += HILO_64_REGPAIR(ustats.rcv_mcast_bytes); - p_stats->rx_bcast_bytes += HILO_64_REGPAIR(ustats.rcv_bcast_bytes); - p_stats->rx_ucast_pkts += HILO_64_REGPAIR(ustats.rcv_ucast_pkts); - p_stats->rx_mcast_pkts += HILO_64_REGPAIR(ustats.rcv_mcast_pkts); - p_stats->rx_bcast_pkts += HILO_64_REGPAIR(ustats.rcv_bcast_pkts); + p_stats->common.rx_ucast_bytes += + HILO_64_REGPAIR(ustats.rcv_ucast_bytes); + p_stats->common.rx_mcast_bytes += + HILO_64_REGPAIR(ustats.rcv_mcast_bytes); + p_stats->common.rx_bcast_bytes += + HILO_64_REGPAIR(ustats.rcv_bcast_bytes); + p_stats->common.rx_ucast_pkts += + HILO_64_REGPAIR(ustats.rcv_ucast_pkts); + p_stats->common.rx_mcast_pkts += + HILO_64_REGPAIR(ustats.rcv_mcast_pkts); + p_stats->common.rx_bcast_pkts += + HILO_64_REGPAIR(ustats.rcv_bcast_pkts); } static void __ecore_get_vport_mstats_addrlen(struct ecore_hwfn *p_hwfn, @@ -1609,23 +1844,27 @@ static void __ecore_get_vport_mstats(struct ecore_hwfn *p_hwfn, OSAL_MEMSET(&mstats, 0, sizeof(mstats)); ecore_memcpy_from(p_hwfn, p_ptt, &mstats, mstats_addr, mstats_len); - p_stats->no_buff_discards += HILO_64_REGPAIR(mstats.no_buff_discard); - p_stats->packet_too_big_discard += - HILO_64_REGPAIR(mstats.packet_too_big_discard); - p_stats->ttl0_discard += HILO_64_REGPAIR(mstats.ttl0_discard); - p_stats->tpa_coalesced_pkts += - HILO_64_REGPAIR(mstats.tpa_coalesced_pkts); - p_stats->tpa_coalesced_events += - HILO_64_REGPAIR(mstats.tpa_coalesced_events); - p_stats->tpa_aborts_num += HILO_64_REGPAIR(mstats.tpa_aborts_num); - p_stats->tpa_coalesced_bytes += - HILO_64_REGPAIR(mstats.tpa_coalesced_bytes); + p_stats->common.no_buff_discards += + HILO_64_REGPAIR(mstats.no_buff_discard); + p_stats->common.packet_too_big_discard += + HILO_64_REGPAIR(mstats.packet_too_big_discard); + p_stats->common.ttl0_discard += + HILO_64_REGPAIR(mstats.ttl0_discard); + p_stats->common.tpa_coalesced_pkts += + HILO_64_REGPAIR(mstats.tpa_coalesced_pkts); + p_stats->common.tpa_coalesced_events += + HILO_64_REGPAIR(mstats.tpa_coalesced_events); + p_stats->common.tpa_aborts_num += + HILO_64_REGPAIR(mstats.tpa_aborts_num); + p_stats->common.tpa_coalesced_bytes += + HILO_64_REGPAIR(mstats.tpa_coalesced_bytes); } static void __ecore_get_vport_port_stats(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, struct ecore_eth_stats *p_stats) { + struct ecore_eth_stats_common *p_common = &p_stats->common; struct port_stats port_stats; int j; @@ -1636,54 +1875,75 @@ static void __ecore_get_vport_port_stats(struct ecore_hwfn *p_hwfn, OFFSETOF(struct public_port, stats), sizeof(port_stats)); - p_stats->rx_64_byte_packets += port_stats.pmm.r64; - p_stats->rx_65_to_127_byte_packets += port_stats.pmm.r127; - p_stats->rx_128_to_255_byte_packets += port_stats.pmm.r255; - p_stats->rx_256_to_511_byte_packets += port_stats.pmm.r511; - p_stats->rx_512_to_1023_byte_packets += port_stats.pmm.r1023; - p_stats->rx_1024_to_1518_byte_packets += port_stats.pmm.r1518; - p_stats->rx_1519_to_1522_byte_packets += port_stats.pmm.r1522; - p_stats->rx_1519_to_2047_byte_packets += port_stats.pmm.r2047; - p_stats->rx_2048_to_4095_byte_packets += port_stats.pmm.r4095; - p_stats->rx_4096_to_9216_byte_packets += port_stats.pmm.r9216; - p_stats->rx_9217_to_16383_byte_packets += port_stats.pmm.r16383; - p_stats->rx_crc_errors += port_stats.pmm.rfcs; - p_stats->rx_mac_crtl_frames += port_stats.pmm.rxcf; - p_stats->rx_pause_frames += port_stats.pmm.rxpf; - p_stats->rx_pfc_frames += port_stats.pmm.rxpp; - p_stats->rx_align_errors += port_stats.pmm.raln; - p_stats->rx_carrier_errors += port_stats.pmm.rfcr; - p_stats->rx_oversize_packets += port_stats.pmm.rovr; - p_stats->rx_jabbers += port_stats.pmm.rjbr; - p_stats->rx_undersize_packets += port_stats.pmm.rund; - p_stats->rx_fragments += port_stats.pmm.rfrg; - p_stats->tx_64_byte_packets += port_stats.pmm.t64; - p_stats->tx_65_to_127_byte_packets += port_stats.pmm.t127; - p_stats->tx_128_to_255_byte_packets += port_stats.pmm.t255; - p_stats->tx_256_to_511_byte_packets += port_stats.pmm.t511; - p_stats->tx_512_to_1023_byte_packets += port_stats.pmm.t1023; - p_stats->tx_1024_to_1518_byte_packets += port_stats.pmm.t1518; - p_stats->tx_1519_to_2047_byte_packets += port_stats.pmm.t2047; - p_stats->tx_2048_to_4095_byte_packets += port_stats.pmm.t4095; - p_stats->tx_4096_to_9216_byte_packets += port_stats.pmm.t9216; - p_stats->tx_9217_to_16383_byte_packets += port_stats.pmm.t16383; - p_stats->tx_pause_frames += port_stats.pmm.txpf; - p_stats->tx_pfc_frames += port_stats.pmm.txpp; - p_stats->tx_lpi_entry_count += port_stats.pmm.tlpiec; - p_stats->tx_total_collisions += port_stats.pmm.tncl; - p_stats->rx_mac_bytes += port_stats.pmm.rbyte; - p_stats->rx_mac_uc_packets += port_stats.pmm.rxuca; - p_stats->rx_mac_mc_packets += port_stats.pmm.rxmca; - p_stats->rx_mac_bc_packets += port_stats.pmm.rxbca; - p_stats->rx_mac_frames_ok += port_stats.pmm.rxpok; - p_stats->tx_mac_bytes += port_stats.pmm.tbyte; - p_stats->tx_mac_uc_packets += port_stats.pmm.txuca; - p_stats->tx_mac_mc_packets += port_stats.pmm.txmca; - p_stats->tx_mac_bc_packets += port_stats.pmm.txbca; - p_stats->tx_mac_ctrl_frames += port_stats.pmm.txcf; + p_common->rx_64_byte_packets += port_stats.eth.r64; + p_common->rx_65_to_127_byte_packets += port_stats.eth.r127; + p_common->rx_128_to_255_byte_packets += port_stats.eth.r255; + p_common->rx_256_to_511_byte_packets += port_stats.eth.r511; + p_common->rx_512_to_1023_byte_packets += port_stats.eth.r1023; + p_common->rx_1024_to_1518_byte_packets += port_stats.eth.r1518; + p_common->rx_crc_errors += port_stats.eth.rfcs; + p_common->rx_mac_crtl_frames += port_stats.eth.rxcf; + p_common->rx_pause_frames += port_stats.eth.rxpf; + p_common->rx_pfc_frames += port_stats.eth.rxpp; + p_common->rx_align_errors += port_stats.eth.raln; + p_common->rx_carrier_errors += port_stats.eth.rfcr; + p_common->rx_oversize_packets += port_stats.eth.rovr; + p_common->rx_jabbers += port_stats.eth.rjbr; + p_common->rx_undersize_packets += port_stats.eth.rund; + p_common->rx_fragments += port_stats.eth.rfrg; + p_common->tx_64_byte_packets += port_stats.eth.t64; + p_common->tx_65_to_127_byte_packets += port_stats.eth.t127; + p_common->tx_128_to_255_byte_packets += port_stats.eth.t255; + p_common->tx_256_to_511_byte_packets += port_stats.eth.t511; + p_common->tx_512_to_1023_byte_packets += port_stats.eth.t1023; + p_common->tx_1024_to_1518_byte_packets += port_stats.eth.t1518; + p_common->tx_pause_frames += port_stats.eth.txpf; + p_common->tx_pfc_frames += port_stats.eth.txpp; + p_common->rx_mac_bytes += port_stats.eth.rbyte; + p_common->rx_mac_uc_packets += port_stats.eth.rxuca; + p_common->rx_mac_mc_packets += port_stats.eth.rxmca; + p_common->rx_mac_bc_packets += port_stats.eth.rxbca; + p_common->rx_mac_frames_ok += port_stats.eth.rxpok; + p_common->tx_mac_bytes += port_stats.eth.tbyte; + p_common->tx_mac_uc_packets += port_stats.eth.txuca; + p_common->tx_mac_mc_packets += port_stats.eth.txmca; + p_common->tx_mac_bc_packets += port_stats.eth.txbca; + p_common->tx_mac_ctrl_frames += port_stats.eth.txcf; for (j = 0; j < 8; j++) { - p_stats->brb_truncates += port_stats.brb.brb_truncate[j]; - p_stats->brb_discards += port_stats.brb.brb_discard[j]; + p_common->brb_truncates += port_stats.brb.brb_truncate[j]; + p_common->brb_discards += port_stats.brb.brb_discard[j]; + } + + if (ECORE_IS_BB(p_hwfn->p_dev)) { + struct ecore_eth_stats_bb *p_bb = &p_stats->bb; + + p_bb->rx_1519_to_1522_byte_packets += + port_stats.eth.u0.bb0.r1522; + p_bb->rx_1519_to_2047_byte_packets += + port_stats.eth.u0.bb0.r2047; + p_bb->rx_2048_to_4095_byte_packets += + port_stats.eth.u0.bb0.r4095; + p_bb->rx_4096_to_9216_byte_packets += + port_stats.eth.u0.bb0.r9216; + p_bb->rx_9217_to_16383_byte_packets += + port_stats.eth.u0.bb0.r16383; + p_bb->tx_1519_to_2047_byte_packets += + port_stats.eth.u1.bb1.t2047; + p_bb->tx_2048_to_4095_byte_packets += + port_stats.eth.u1.bb1.t4095; + p_bb->tx_4096_to_9216_byte_packets += + port_stats.eth.u1.bb1.t9216; + p_bb->tx_9217_to_16383_byte_packets += + port_stats.eth.u1.bb1.t16383; + p_bb->tx_lpi_entry_count += port_stats.eth.u2.bb2.tlpiec; + p_bb->tx_total_collisions += port_stats.eth.u2.bb2.tncl; + } else { + struct ecore_eth_stats_ah *p_ah = &p_stats->ah; + + p_ah->rx_1519_to_max_byte_packets += + port_stats.eth.u0.ah0.r1519_to_max; + p_ah->tx_1519_to_max_byte_packets = + port_stats.eth.u1.ah1.t1519_to_max; } } @@ -1694,7 +1954,7 @@ void __ecore_get_vport_stats(struct ecore_hwfn *p_hwfn, { __ecore_get_vport_mstats(p_hwfn, p_ptt, stats, statistics_bin); __ecore_get_vport_ustats(p_hwfn, p_ptt, stats, statistics_bin); - __ecore_get_vport_tstats(p_hwfn, p_ptt, stats, statistics_bin); + __ecore_get_vport_tstats(p_hwfn, p_ptt, stats); __ecore_get_vport_pstats(p_hwfn, p_ptt, stats, statistics_bin); #ifndef ASIC_ONLY @@ -1719,6 +1979,7 @@ static void _ecore_get_vport_stats(struct ecore_dev *p_dev, struct ecore_hwfn *p_hwfn = &p_dev->hwfns[i]; struct ecore_ptt *p_ptt = IS_PF(p_dev) ? ecore_ptt_acquire(p_hwfn) : OSAL_NULL; + bool b_get_port_stats; if (IS_PF(p_dev)) { /* The main vport index is relative first */ @@ -1733,11 +1994,12 @@ static void _ecore_get_vport_stats(struct ecore_dev *p_dev, continue; } + b_get_port_stats = IS_PF(p_dev) && IS_LEAD_HWFN(p_hwfn); __ecore_get_vport_stats(p_hwfn, p_ptt, stats, fw_vport, - IS_PF(p_dev) ? true : false); + b_get_port_stats); out: - if (IS_PF(p_dev)) + if (IS_PF(p_dev) && p_ptt) ecore_ptt_release(p_hwfn, p_ptt); } } @@ -1805,3 +2067,203 @@ void ecore_reset_vport_stats(struct ecore_dev *p_dev) else _ecore_get_vport_stats(p_dev, p_dev->reset_stats); } + +void ecore_arfs_mode_configure(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + struct ecore_arfs_config_params *p_cfg_params) +{ + if (OSAL_TEST_BIT(ECORE_MF_DISABLE_ARFS, &p_hwfn->p_dev->mf_bits)) + return; + + if (p_cfg_params->arfs_enable) { + ecore_gft_config(p_hwfn, p_ptt, p_hwfn->rel_pf_id, + p_cfg_params->tcp, + p_cfg_params->udp, + p_cfg_params->ipv4, + p_cfg_params->ipv6, + GFT_PROFILE_TYPE_4_TUPLE); + DP_VERBOSE(p_hwfn, ECORE_MSG_SP, + "tcp = %s, udp = %s, ipv4 = %s, ipv6 =%s\n", + p_cfg_params->tcp ? "Enable" : "Disable", + p_cfg_params->udp ? "Enable" : "Disable", + p_cfg_params->ipv4 ? "Enable" : "Disable", + p_cfg_params->ipv6 ? "Enable" : "Disable"); + } else { + ecore_gft_disable(p_hwfn, p_ptt, p_hwfn->rel_pf_id); + } + DP_VERBOSE(p_hwfn, ECORE_MSG_SP, "Configured ARFS mode : %s\n", + p_cfg_params->arfs_enable ? "Enable" : "Disable"); +} + +enum _ecore_status_t +ecore_configure_rfs_ntuple_filter(struct ecore_hwfn *p_hwfn, + struct ecore_spq_comp_cb *p_cb, + dma_addr_t p_addr, u16 length, + u16 qid, u8 vport_id, + bool b_is_add) +{ + struct rx_update_gft_filter_data *p_ramrod = OSAL_NULL; + struct ecore_spq_entry *p_ent = OSAL_NULL; + struct ecore_sp_init_data init_data; + u16 abs_rx_q_id = 0; + u8 abs_vport_id = 0; + enum _ecore_status_t rc = ECORE_NOTIMPL; + + rc = ecore_fw_vport(p_hwfn, vport_id, &abs_vport_id); + if (rc != ECORE_SUCCESS) + return rc; + + rc = ecore_fw_l2_queue(p_hwfn, qid, &abs_rx_q_id); + if (rc != ECORE_SUCCESS) + return rc; + + /* Get SPQ entry */ + OSAL_MEMSET(&init_data, 0, sizeof(init_data)); + init_data.cid = ecore_spq_get_cid(p_hwfn); + + init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; + + if (p_cb) { + init_data.comp_mode = ECORE_SPQ_MODE_CB; + init_data.p_comp_data = p_cb; + } else { + init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK; + } + + rc = ecore_sp_init_request(p_hwfn, &p_ent, + ETH_RAMROD_GFT_UPDATE_FILTER, + PROTOCOLID_ETH, &init_data); + if (rc != ECORE_SUCCESS) + return rc; + + p_ramrod = &p_ent->ramrod.rx_update_gft; + + DMA_REGPAIR_LE(p_ramrod->pkt_hdr_addr, p_addr); + p_ramrod->pkt_hdr_length = OSAL_CPU_TO_LE16(length); + + p_ramrod->action_icid_valid = 0; + p_ramrod->action_icid = 0; + + p_ramrod->rx_qid_valid = 1; + p_ramrod->rx_qid = OSAL_CPU_TO_LE16(abs_rx_q_id); + + p_ramrod->flow_id_valid = 0; + p_ramrod->flow_id = 0; + + p_ramrod->vport_id = abs_vport_id; + p_ramrod->filter_action = b_is_add ? GFT_ADD_FILTER + : GFT_DELETE_FILTER; + + DP_VERBOSE(p_hwfn, ECORE_MSG_SP, + "V[%0x], Q[%04x] - %s filter from 0x%lx [length %04xb]\n", + abs_vport_id, abs_rx_q_id, + b_is_add ? "Adding" : "Removing", + (unsigned long)p_addr, length); + + return ecore_spq_post(p_hwfn, p_ent, OSAL_NULL); +} + +int ecore_get_rxq_coalesce(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + struct ecore_queue_cid *p_cid, + u16 *p_rx_coal) +{ + u32 coalesce, address, is_valid; + struct cau_sb_entry sb_entry; + u8 timer_res; + enum _ecore_status_t rc; + + rc = ecore_dmae_grc2host(p_hwfn, p_ptt, CAU_REG_SB_VAR_MEMORY + + p_cid->sb_igu_id * sizeof(u64), + (u64)(osal_uintptr_t)&sb_entry, 2, 0); + if (rc != ECORE_SUCCESS) { + DP_ERR(p_hwfn, "dmae_grc2host failed %d\n", rc); + return rc; + } + + timer_res = GET_FIELD(sb_entry.params, CAU_SB_ENTRY_TIMER_RES0); + + address = BAR0_MAP_REG_USDM_RAM + + USTORM_ETH_QUEUE_ZONE_OFFSET(p_cid->abs.queue_id); + coalesce = ecore_rd(p_hwfn, p_ptt, address); + + is_valid = GET_FIELD(coalesce, COALESCING_TIMESET_VALID); + if (!is_valid) + return ECORE_INVAL; + + coalesce = GET_FIELD(coalesce, COALESCING_TIMESET_TIMESET); + *p_rx_coal = (u16)(coalesce << timer_res); + + return ECORE_SUCCESS; +} + +int ecore_get_txq_coalesce(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + struct ecore_queue_cid *p_cid, + u16 *p_tx_coal) +{ + u32 coalesce, address, is_valid; + struct cau_sb_entry sb_entry; + u8 timer_res; + enum _ecore_status_t rc; + + rc = ecore_dmae_grc2host(p_hwfn, p_ptt, CAU_REG_SB_VAR_MEMORY + + p_cid->sb_igu_id * sizeof(u64), + (u64)(osal_uintptr_t)&sb_entry, 2, 0); + if (rc != ECORE_SUCCESS) { + DP_ERR(p_hwfn, "dmae_grc2host failed %d\n", rc); + return rc; + } + + timer_res = GET_FIELD(sb_entry.params, CAU_SB_ENTRY_TIMER_RES1); + + address = BAR0_MAP_REG_XSDM_RAM + + XSTORM_ETH_QUEUE_ZONE_OFFSET(p_cid->abs.queue_id); + coalesce = ecore_rd(p_hwfn, p_ptt, address); + + is_valid = GET_FIELD(coalesce, COALESCING_TIMESET_VALID); + if (!is_valid) + return ECORE_INVAL; + + coalesce = GET_FIELD(coalesce, COALESCING_TIMESET_TIMESET); + *p_tx_coal = (u16)(coalesce << timer_res); + + return ECORE_SUCCESS; +} + +enum _ecore_status_t +ecore_get_queue_coalesce(struct ecore_hwfn *p_hwfn, u16 *p_coal, + void *handle) +{ + struct ecore_queue_cid *p_cid = (struct ecore_queue_cid *)handle; + enum _ecore_status_t rc = ECORE_SUCCESS; + struct ecore_ptt *p_ptt; + + if (IS_VF(p_hwfn->p_dev)) { + rc = ecore_vf_pf_get_coalesce(p_hwfn, p_coal, p_cid); + if (rc != ECORE_SUCCESS) + DP_NOTICE(p_hwfn, false, + "Unable to read queue calescing\n"); + + return rc; + } + + p_ptt = ecore_ptt_acquire(p_hwfn); + if (!p_ptt) + return ECORE_AGAIN; + + if (p_cid->b_is_rx) { + rc = ecore_get_rxq_coalesce(p_hwfn, p_ptt, p_cid, p_coal); + if (rc != ECORE_SUCCESS) + goto out; + } else { + rc = ecore_get_txq_coalesce(p_hwfn, p_ptt, p_cid, p_coal); + if (rc != ECORE_SUCCESS) + goto out; + } + +out: + ecore_ptt_release(p_hwfn, p_ptt); + + return rc; +} diff --git a/dpdk/drivers/net/qede/base/ecore_l2.h b/dpdk/drivers/net/qede/base/ecore_l2.h index b0850ca4..f4212cf2 100644 --- a/dpdk/drivers/net/qede/base/ecore_l2.h +++ b/dpdk/drivers/net/qede/base/ecore_l2.h @@ -9,124 +9,127 @@ #ifndef __ECORE_L2_H__ #define __ECORE_L2_H__ + #include "ecore.h" #include "ecore_hw.h" #include "ecore_spq.h" #include "ecore_l2_api.h" -/** - * @brief ecore_sp_vf_start - VF Function Start - * - * This ramrod is sent to initialize a virtual function (VF) is loaded. - * It will configure the function related parameters. - * - * @note Final phase API. - * - * @param p_hwfn - * @param concrete_vfid VF ID - * @param opaque_vfid - * - * @return enum _ecore_status_t +#define MAX_QUEUES_PER_QZONE (sizeof(unsigned long) * 8) +#define ECORE_QUEUE_CID_PF (0xff) + +/* Almost identical to the ecore_queue_start_common_params, + * but here we maintain the SB index in IGU CAM. */ +struct ecore_queue_cid_params { + u8 vport_id; + u16 queue_id; + u8 stats_id; +}; -enum _ecore_status_t ecore_sp_vf_start(struct ecore_hwfn *p_hwfn, - u32 concrete_vfid, u16 opaque_vfid); - -/** - * @brief ecore_sp_vf_update - VF Function Update Ramrod - * - * This ramrod performs updates of a virtual function (VF). - * It currently contains no functionality. - * - * @note Final phase API. - * - * @param p_hwfn - * - * @return enum _ecore_status_t + /* Additional parameters required for initialization of the queue_cid + * and are relevant only for a PF initializing one for its VFs. */ +struct ecore_queue_cid_vf_params { + /* Should match the VF's relative index */ + u8 vfid; -enum _ecore_status_t ecore_sp_vf_update(struct ecore_hwfn *p_hwfn); + /* 0-based queue index. Should reflect the relative qzone the + * VF thinks is associated with it [in its range]. + */ + u8 vf_qid; -/** - * @brief ecore_sp_vf_stop - VF Function Stop Ramrod - * - * This ramrod is sent to unload a virtual function (VF). - * - * @note Final phase API. - * - * @param p_hwfn - * @param concrete_vfid - * @param opaque_vfid - * - * @return enum _ecore_status_t - */ + /* Indicates a VF is legacy, making it differ in several things: + * - Producers would be placed in a different place. + * - Makes assumptions regarding the CIDs. + */ + u8 vf_legacy; -enum _ecore_status_t ecore_sp_vf_stop(struct ecore_hwfn *p_hwfn, - u32 concrete_vfid, u16 opaque_vfid); + /* For VFs, this index arrives via TLV to diffrentiate between + * different queues opened on the same qzone, and is passed + * [where the PF would have allocated it internally for its own]. + */ + u8 qid_usage_idx; +}; -/** - * @brief ecore_sp_eth_tx_queue_update - - * - * This ramrod updates a TX queue. It is used for setting the active - * state of the queue. - * - * @note Final phase API. - * - * @param p_hwfn - * - * @return enum _ecore_status_t - */ -enum _ecore_status_t ecore_sp_eth_tx_queue_update(struct ecore_hwfn *p_hwfn); +struct ecore_queue_cid { + /* For stats-id, the `rel' is actually absolute as well */ + struct ecore_queue_cid_params rel; + struct ecore_queue_cid_params abs; + + /* These have no 'relative' meaning */ + u16 sb_igu_id; + u8 sb_idx; + + u32 cid; + u16 opaque_fid; + + bool b_is_rx; + + /* VFs queues are mapped differently, so we need to know the + * relative queue associated with them [0-based]. + * Notice this is relevant on the *PF* queue-cid of its VF's queues, + * and not on the VF itself. + */ + u8 vfid; + u8 vf_qid; + + /* We need an additional index to diffrentiate between queues opened + * for same queue-zone, as VFs would have to communicate the info + * to the PF [otherwise PF has no way to diffrentiate]. + */ + u8 qid_usage_idx; + + /* Legacy VFs might have Rx producer located elsewhere */ + u8 vf_legacy; +#define ECORE_QCID_LEGACY_VF_RX_PROD (1 << 0) +#define ECORE_QCID_LEGACY_VF_CID (1 << 1) + + struct ecore_hwfn *p_owner; +}; + +enum _ecore_status_t ecore_l2_alloc(struct ecore_hwfn *p_hwfn); +void ecore_l2_setup(struct ecore_hwfn *p_hwfn); +void ecore_l2_free(struct ecore_hwfn *p_hwfn); + +void ecore_eth_queue_cid_release(struct ecore_hwfn *p_hwfn, + struct ecore_queue_cid *p_cid); + +struct ecore_queue_cid * +ecore_eth_queue_to_cid(struct ecore_hwfn *p_hwfn, u16 opaque_fid, + struct ecore_queue_start_common_params *p_params, + bool b_is_rx, + struct ecore_queue_cid_vf_params *p_vf_params); enum _ecore_status_t ecore_sp_eth_vport_start(struct ecore_hwfn *p_hwfn, struct ecore_sp_vport_start_params *p_params); /** - * @brief - Starts an Rx queue; Should be used where contexts are handled - * outside of the ramrod area [specifically iov scenarios] + * @brief - Starts an Rx queue, when queue_cid is already prepared * * @param p_hwfn - * @param opaque_fid - * @param cid - * @param rx_queue_id - * @param vport_id - * @param stats_id - * @param sb - * @param sb_index + * @param p_cid * @param bd_max_bytes * @param bd_chain_phys_addr * @param cqe_pbl_addr * @param cqe_pbl_size - * @param leading * * @return enum _ecore_status_t */ enum _ecore_status_t -ecore_sp_eth_rxq_start_ramrod(struct ecore_hwfn *p_hwfn, - u16 opaque_fid, - u32 cid, - u16 rx_queue_id, - u8 vport_id, - u8 stats_id, - u16 sb, - u8 sb_index, - u16 bd_max_bytes, - dma_addr_t bd_chain_phys_addr, - dma_addr_t cqe_pbl_addr, u16 cqe_pbl_size); +ecore_eth_rxq_start_ramrod(struct ecore_hwfn *p_hwfn, + struct ecore_queue_cid *p_cid, + u16 bd_max_bytes, + dma_addr_t bd_chain_phys_addr, + dma_addr_t cqe_pbl_addr, + u16 cqe_pbl_size); /** - * @brief - Starts a Tx queue; Should be used where contexts are handled - * outside of the ramrod area [specifically iov scenarios] + * @brief - Starts a Tx queue, where queue_cid is already prepared * * @param p_hwfn - * @param opaque_fid - * @param tx_queue_id - * @param cid - * @param vport_id - * @param stats_id - * @param sb - * @param sb_index + * @param p_cid * @param pbl_addr * @param pbl_size * @param p_pq_params - parameters for choosing the PQ for this Tx queue @@ -134,18 +137,31 @@ ecore_sp_eth_rxq_start_ramrod(struct ecore_hwfn *p_hwfn, * @return enum _ecore_status_t */ enum _ecore_status_t -ecore_sp_eth_txq_start_ramrod(struct ecore_hwfn *p_hwfn, - u16 opaque_fid, - u16 tx_queue_id, - u32 cid, - u8 vport_id, - u8 stats_id, - u16 sb, - u8 sb_index, - dma_addr_t pbl_addr, - u16 pbl_size, - union ecore_qm_pq_params *p_pq_params); +ecore_eth_txq_start_ramrod(struct ecore_hwfn *p_hwfn, + struct ecore_queue_cid *p_cid, + dma_addr_t pbl_addr, u16 pbl_size, + u16 pq_id); u8 ecore_mcast_bin_from_mac(u8 *mac); +enum _ecore_status_t ecore_set_rxq_coalesce(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + u16 coalesce, + struct ecore_queue_cid *p_cid); + +enum _ecore_status_t ecore_set_txq_coalesce(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + u16 coalesce, + struct ecore_queue_cid *p_cid); + +enum _ecore_status_t ecore_get_rxq_coalesce(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + struct ecore_queue_cid *p_cid, + u16 *p_hw_coal); + +enum _ecore_status_t ecore_get_txq_coalesce(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + struct ecore_queue_cid *p_cid, + u16 *p_hw_coal); + #endif diff --git a/dpdk/drivers/net/qede/base/ecore_l2_api.h b/dpdk/drivers/net/qede/base/ecore_l2_api.h index b41dd7fe..ed9837bf 100644 --- a/dpdk/drivers/net/qede/base/ecore_l2_api.h +++ b/dpdk/drivers/net/qede/base/ecore_l2_api.h @@ -11,22 +11,45 @@ #include "ecore_status.h" #include "ecore_sp_api.h" +#include "ecore_int_api.h" #ifndef __EXTRACT__LINUX__ enum ecore_rss_caps { - ECORE_RSS_IPV4 = 0x1, - ECORE_RSS_IPV6 = 0x2, - ECORE_RSS_IPV4_TCP = 0x4, - ECORE_RSS_IPV6_TCP = 0x8, - ECORE_RSS_IPV4_UDP = 0x10, - ECORE_RSS_IPV6_UDP = 0x20, + ECORE_RSS_IPV4 = 0x1, + ECORE_RSS_IPV6 = 0x2, + ECORE_RSS_IPV4_TCP = 0x4, + ECORE_RSS_IPV6_TCP = 0x8, + ECORE_RSS_IPV4_UDP = 0x10, + ECORE_RSS_IPV6_UDP = 0x20, }; /* Should be the same as ETH_RSS_IND_TABLE_ENTRIES_NUM */ #define ECORE_RSS_IND_TABLE_SIZE 128 -#define ECORE_RSS_KEY_SIZE 10 /* size in 32b chunks */ +#define ECORE_RSS_KEY_SIZE 10 /* size in 32b chunks */ #endif +struct ecore_queue_start_common_params { + /* Should always be relative to entity sending this. */ + u8 vport_id; + u16 queue_id; + + /* Relative, but relevant only for PFs */ + u8 stats_id; + + struct ecore_sb_info *p_sb; + u8 sb_idx; +}; + +struct ecore_rxq_start_ret_params { + void OSAL_IOMEM *p_prod; + void *p_handle; +}; + +struct ecore_txq_start_ret_params { + void OSAL_IOMEM *p_doorbell; + void *p_handle; +}; + struct ecore_rss_params { u8 update_rss_config; u8 rss_enable; @@ -35,8 +58,10 @@ struct ecore_rss_params { u8 update_rss_ind_table; u8 update_rss_key; u8 rss_caps; - u8 rss_table_size_log; /* The table size is 2 ^ rss_table_size_log */ - u16 rss_ind_table[ECORE_RSS_IND_TABLE_SIZE]; + u8 rss_table_size_log; /* The table size is 2 ^ rss_table_size_log */ + + /* Indirection table consist of rx queue handles */ + void *rss_ind_table[ECORE_RSS_IND_TABLE_SIZE]; u32 rss_key[ECORE_RSS_KEY_SIZE]; }; @@ -63,8 +88,8 @@ enum ecore_filter_opcode { ECORE_FILTER_ADD, ECORE_FILTER_REMOVE, ECORE_FILTER_MOVE, - ECORE_FILTER_REPLACE, /* Delete all MACs and add new one instead */ - ECORE_FILTER_FLUSH, /* Removes all filters */ + ECORE_FILTER_REPLACE, /* Delete all MACs and add new one instead */ + ECORE_FILTER_FLUSH, /* Removes all filters */ }; enum ecore_filter_ucast_type { @@ -77,6 +102,7 @@ enum ecore_filter_ucast_type { ECORE_FILTER_INNER_MAC_VNI_PAIR, ECORE_FILTER_MAC_VNI_PAIR, ECORE_FILTER_VNI, + ECORE_FILTER_UNUSED, /* @DPDK */ }; struct ecore_filter_ucast { @@ -97,7 +123,7 @@ struct ecore_filter_mcast { enum ecore_filter_opcode opcode; u8 vport_to_add_to; u8 vport_to_remove_from; - u8 num_mc_addrs; + u8 num_mc_addrs; #define ECORE_MAX_MC_ADDRS 64 unsigned char mac[ECORE_MAX_MC_ADDRS][ETH_ALEN]; }; @@ -115,6 +141,14 @@ struct ecore_filter_accept_flags { #define ECORE_ACCEPT_BCAST 0x20 }; +struct ecore_arfs_config_params { + bool tcp; + bool udp; + bool ipv4; + bool ipv6; + bool arfs_enable; /* Enable or disable arfs mode */ +}; + /* Add / remove / move / remove-all unicast MAC-VLAN filters. * FW will assert in the following cases, so driver should take care...: * 1. Adding a filter to a full table. @@ -137,61 +171,47 @@ ecore_filter_mcast_cmd(struct ecore_dev *p_dev, /* Set "accept" filters */ enum _ecore_status_t -ecore_filter_accept_cmd(struct ecore_dev *p_dev, - u8 vport, - struct ecore_filter_accept_flags accept_flags, - u8 update_accept_any_vlan, - u8 accept_any_vlan, - enum spq_mode comp_mode, - struct ecore_spq_comp_cb *p_comp_data); +ecore_filter_accept_cmd( + struct ecore_dev *p_dev, + u8 vport, + struct ecore_filter_accept_flags accept_flags, + u8 update_accept_any_vlan, + u8 accept_any_vlan, + enum spq_mode comp_mode, + struct ecore_spq_comp_cb *p_comp_data); /** - * @brief ecore_sp_eth_rx_queue_start - RX Queue Start Ramrod + * @brief ecore_eth_rx_queue_start - RX Queue Start Ramrod * * This ramrod initializes an RX Queue for a VPort. An Assert is generated if * the VPort ID is not currently initialized. * * @param p_hwfn * @param opaque_fid - * @param rx_queue_id RX Queue ID: Zero based, per VPort, allocated - * by assignment (=rssId) - * @param vport_id VPort ID - * @param u8 stats_id VPort ID which the queue stats - * will be added to - * @param sb Status Block of the Function Event Ring - * @param sb_index Index into the status block of the - * Function Event Ring + * @p_params Inputs; Relative for PF [SB being an exception] * @param bd_max_bytes Maximum bytes that can be placed on a BD * @param bd_chain_phys_addr Physical address of BDs for receive. * @param cqe_pbl_addr Physical address of the CQE PBL Table. * @param cqe_pbl_size Size of the CQE PBL Table - * @param pp_prod Pointer to place producer's - * address for the Rx Q (May be - * NULL). + * @param p_ret_params Pointed struct to be filled with outputs. * * @return enum _ecore_status_t */ -enum _ecore_status_t ecore_sp_eth_rx_queue_start(struct ecore_hwfn *p_hwfn, - u16 opaque_fid, - u8 rx_queue_id, - u8 vport_id, - u8 stats_id, - u16 sb, - u8 sb_index, - u16 bd_max_bytes, - dma_addr_t bd_chain_phys_addr, - dma_addr_t cqe_pbl_addr, - u16 cqe_pbl_size, - void OSAL_IOMEM * *pp_prod); +enum _ecore_status_t +ecore_eth_rx_queue_start(struct ecore_hwfn *p_hwfn, + u16 opaque_fid, + struct ecore_queue_start_common_params *p_params, + u16 bd_max_bytes, + dma_addr_t bd_chain_phys_addr, + dma_addr_t cqe_pbl_addr, + u16 cqe_pbl_size, + struct ecore_rxq_start_ret_params *p_ret_params); /** - * @brief ecore_sp_eth_rx_queue_stop - - * - * This ramrod closes an RX queue. It sends RX queue stop ramrod - * + CFC delete ramrod + * @brief ecore_eth_rx_queue_stop - This ramrod closes an Rx queue * * @param p_hwfn - * @param rx_queue_id RX Queue ID + * @param p_rxq Handler of queue to close * @param eq_completion_only If True completion will be on * EQe, if False completion will be * on EQe if p_hwfn opaque @@ -202,60 +222,48 @@ enum _ecore_status_t ecore_sp_eth_rx_queue_start(struct ecore_hwfn *p_hwfn, * @return enum _ecore_status_t */ enum _ecore_status_t -ecore_sp_eth_rx_queue_stop(struct ecore_hwfn *p_hwfn, - u16 rx_queue_id, - bool eq_completion_only, bool cqe_completion); +ecore_eth_rx_queue_stop(struct ecore_hwfn *p_hwfn, + void *p_rxq, + bool eq_completion_only, + bool cqe_completion); /** - * @brief ecore_sp_eth_tx_queue_start - TX Queue Start Ramrod + * @brief - TX Queue Start Ramrod * * This ramrod initializes a TX Queue for a VPort. An Assert is generated if * the VPort is not currently initialized. * * @param p_hwfn * @param opaque_fid - * @param tx_queue_id TX Queue ID - * @param vport_id VPort ID - * @param stats_id VPort ID which the queue stats - * will be added to - * @param sb Status Block of the Function Event Ring - * @param sb_index Index into the status block of the Function - * Event Ring + * @p_params + * @param tc traffic class to use with this L2 txq * @param pbl_addr address of the pbl array * @param pbl_size number of entries in pbl - * @param pp_doorbell Pointer to place doorbell pointer (May be NULL). - * This address should be used with the - * DIRECT_REG_WR macro. + * @param p_ret_params Pointer to fill the return parameters in. * * @return enum _ecore_status_t */ -enum _ecore_status_t ecore_sp_eth_tx_queue_start(struct ecore_hwfn *p_hwfn, - u16 opaque_fid, - u16 tx_queue_id, - u8 vport_id, - u8 stats_id, - u16 sb, - u8 sb_index, - dma_addr_t pbl_addr, - u16 pbl_size, - void OSAL_IOMEM * * - pp_doorbell); +enum _ecore_status_t +ecore_eth_tx_queue_start(struct ecore_hwfn *p_hwfn, + u16 opaque_fid, + struct ecore_queue_start_common_params *p_params, + u8 tc, + dma_addr_t pbl_addr, + u16 pbl_size, + struct ecore_txq_start_ret_params *p_ret_params); /** - * @brief ecore_sp_eth_tx_queue_stop - - * - * This ramrod closes a TX queue. It sends TX queue stop ramrod - * + CFC delete ramrod + * @brief ecore_eth_tx_queue_stop - closes a Tx queue * * @param p_hwfn - * @param tx_queue_id TX Queue ID + * @param p_txq - handle to Tx queue needed to be closed * * @return enum _ecore_status_t */ -enum _ecore_status_t ecore_sp_eth_tx_queue_stop(struct ecore_hwfn *p_hwfn, - u16 tx_queue_id); +enum _ecore_status_t ecore_eth_tx_queue_stop(struct ecore_hwfn *p_hwfn, + void *p_txq); -enum ecore_tpa_mode { +enum ecore_tpa_mode { ECORE_TPA_MODE_NONE, ECORE_TPA_MODE_RSC, ECORE_TPA_MODE_GRO, @@ -275,6 +283,17 @@ struct ecore_sp_vport_start_params { u8 vport_id; /* VPORT ID */ u16 mtu; /* VPORT MTU */ bool zero_placement_offset; + bool check_mac; + bool check_ethtype; + + /* Strict behavior on transmission errors */ + bool b_err_illegal_vlan_mode; + bool b_err_illegal_inband_mode; + bool b_err_vlan_insert_with_inband; + bool b_err_small_pkt; + bool b_err_big_pkt; + bool b_err_anti_spoof; + bool b_err_ctrl_frame; }; /** @@ -293,30 +312,34 @@ ecore_sp_vport_start(struct ecore_hwfn *p_hwfn, struct ecore_sp_vport_start_params *p_params); struct ecore_sp_vport_update_params { - u16 opaque_fid; - u8 vport_id; - u8 update_vport_active_rx_flg; - u8 vport_active_rx_flg; - u8 update_vport_active_tx_flg; - u8 vport_active_tx_flg; - u8 update_inner_vlan_removal_flg; - u8 inner_vlan_removal_flg; - u8 silent_vlan_removal_flg; - u8 update_default_vlan_enable_flg; - u8 default_vlan_enable_flg; - u8 update_default_vlan_flg; - u16 default_vlan; - u8 update_tx_switching_flg; - u8 tx_switching_flg; - u8 update_approx_mcast_flg; - u8 update_anti_spoofing_en_flg; - u8 anti_spoofing_en; - u8 update_accept_any_vlan_flg; - u8 accept_any_vlan; - unsigned long bins[8]; - struct ecore_rss_params *rss_params; + u16 opaque_fid; + u8 vport_id; + u8 update_vport_active_rx_flg; + u8 vport_active_rx_flg; + u8 update_vport_active_tx_flg; + u8 vport_active_tx_flg; + u8 update_inner_vlan_removal_flg; + u8 inner_vlan_removal_flg; + u8 silent_vlan_removal_flg; + u8 update_default_vlan_enable_flg; + u8 default_vlan_enable_flg; + u8 update_default_vlan_flg; + u16 default_vlan; + u8 update_tx_switching_flg; + u8 tx_switching_flg; + u8 update_approx_mcast_flg; + u8 update_anti_spoofing_en_flg; + u8 anti_spoofing_en; + u8 update_accept_any_vlan_flg; + u8 accept_any_vlan; + unsigned long bins[8]; + struct ecore_rss_params *rss_params; struct ecore_filter_accept_flags accept_flags; struct ecore_sge_tpa_params *sge_tpa_params; + /* MTU change - notice this requires the vport to be disabled. + * If non-zero, value would be used. + */ + u16 mtu; }; /** @@ -351,7 +374,8 @@ ecore_sp_vport_update(struct ecore_hwfn *p_hwfn, * @return enum _ecore_status_t */ enum _ecore_status_t ecore_sp_vport_stop(struct ecore_hwfn *p_hwfn, - u16 opaque_fid, u8 vport_id); + u16 opaque_fid, + u8 vport_id); enum _ecore_status_t ecore_sp_eth_filter_ucast(struct ecore_hwfn *p_hwfn, @@ -369,19 +393,19 @@ ecore_sp_eth_filter_ucast(struct ecore_hwfn *p_hwfn, * @note Final phase API. * * @param p_hwfn - * @param rx_queue_id RX Queue ID - * @param num_rxqs Allow to update multiple rx - * queues, from rx_queue_id to - * (rx_queue_id + num_rxqs) + * @param pp_rxq_handlers An array of queue handlers to be updated. + * @param num_rxqs number of queues to update. * @param complete_cqe_flg Post completion to the CQE Ring if set * @param complete_event_flg Post completion to the Event Ring if set + * @param comp_mode + * @param p_comp_data * * @return enum _ecore_status_t */ enum _ecore_status_t ecore_sp_eth_rx_queues_update(struct ecore_hwfn *p_hwfn, - u16 rx_queue_id, + void **pp_rxq_handlers, u8 num_rxqs, u8 complete_cqe_flg, u8 complete_event_flg, @@ -398,4 +422,44 @@ void ecore_get_vport_stats(struct ecore_dev *p_dev, void ecore_reset_vport_stats(struct ecore_dev *p_dev); +/** + *@brief ecore_arfs_mode_configure - + * + *Enable or disable rfs mode. It must accept atleast one of tcp or udp true + *and atleast one of ipv4 or ipv6 true to enable rfs mode. + * + *@param p_hwfn + *@param p_ptt + *@param p_cfg_params arfs mode configuration parameters. + * + */ +void ecore_arfs_mode_configure(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + struct ecore_arfs_config_params *p_cfg_params); + +/** + * @brief - ecore_configure_rfs_ntuple_filter + * + * This ramrod should be used to add or remove arfs hw filter + * + * @params p_hwfn + * @params p_cb Used for ECORE_SPQ_MODE_CB,where client would initialize + * it with cookie and callback function address, if not + * using this mode then client must pass NULL. + * @params p_addr p_addr is an actual packet header that needs to be + * filter. It has to mapped with IO to read prior to + * calling this, [contains 4 tuples- src ip, dest ip, + * src port, dest port]. + * @params length length of p_addr header up to past the transport header. + * @params qid receive packet will be directed to this queue. + * @params vport_id + * @params b_is_add flag to add or remove filter. + * + */ +enum _ecore_status_t +ecore_configure_rfs_ntuple_filter(struct ecore_hwfn *p_hwfn, + struct ecore_spq_comp_cb *p_cb, + dma_addr_t p_addr, u16 length, + u16 qid, u8 vport_id, + bool b_is_add); #endif diff --git a/dpdk/drivers/net/qede/base/ecore_mcp.c b/dpdk/drivers/net/qede/base/ecore_mcp.c index 9dd2eed3..8edd2e96 100644 --- a/dpdk/drivers/net/qede/base/ecore_mcp.c +++ b/dpdk/drivers/net/qede/base/ecore_mcp.c @@ -15,10 +15,13 @@ #include "ecore_hw.h" #include "ecore_init_fw_funcs.h" #include "ecore_sriov.h" +#include "ecore_vf.h" #include "ecore_iov_api.h" #include "ecore_gtt_reg_addr.h" #include "ecore_iro.h" #include "ecore_dcbx.h" +#include "ecore_sp_commands.h" +#include "ecore_cxt.h" #define CHIP_MCP_RESP_ITER_US 10 #define EMUL_MCP_RESP_ITER_US (1000 * 1000) @@ -42,9 +45,9 @@ OFFSETOF(struct public_drv_mb, _field)) #define PDA_COMP (((FW_MAJOR_VERSION) + (FW_MINOR_VERSION << 8)) << \ - DRV_ID_PDA_COMP_VER_SHIFT) + DRV_ID_PDA_COMP_VER_OFFSET) -#define MCP_BYTES_PER_MBIT_SHIFT 17 +#define MCP_BYTES_PER_MBIT_OFFSET 17 #ifndef ASIC_ONLY static int loaded; @@ -95,15 +98,82 @@ void ecore_mcp_read_mb(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt) } } +struct ecore_mcp_cmd_elem { + osal_list_entry_t list; + struct ecore_mcp_mb_params *p_mb_params; + u16 expected_seq_num; + bool b_is_completed; +}; + +/* Must be called while cmd_lock is acquired */ +static struct ecore_mcp_cmd_elem * +ecore_mcp_cmd_add_elem(struct ecore_hwfn *p_hwfn, + struct ecore_mcp_mb_params *p_mb_params, + u16 expected_seq_num) +{ + struct ecore_mcp_cmd_elem *p_cmd_elem = OSAL_NULL; + + p_cmd_elem = OSAL_ZALLOC(p_hwfn->p_dev, GFP_ATOMIC, + sizeof(*p_cmd_elem)); + if (!p_cmd_elem) { + DP_NOTICE(p_hwfn, false, + "Failed to allocate `struct ecore_mcp_cmd_elem'\n"); + goto out; + } + + p_cmd_elem->p_mb_params = p_mb_params; + p_cmd_elem->expected_seq_num = expected_seq_num; + OSAL_LIST_PUSH_HEAD(&p_cmd_elem->list, &p_hwfn->mcp_info->cmd_list); +out: + return p_cmd_elem; +} + +/* Must be called while cmd_lock is acquired */ +static void ecore_mcp_cmd_del_elem(struct ecore_hwfn *p_hwfn, + struct ecore_mcp_cmd_elem *p_cmd_elem) +{ + OSAL_LIST_REMOVE_ENTRY(&p_cmd_elem->list, &p_hwfn->mcp_info->cmd_list); + OSAL_FREE(p_hwfn->p_dev, p_cmd_elem); +} + +/* Must be called while cmd_lock is acquired */ +static struct ecore_mcp_cmd_elem * +ecore_mcp_cmd_get_elem(struct ecore_hwfn *p_hwfn, u16 seq_num) +{ + struct ecore_mcp_cmd_elem *p_cmd_elem = OSAL_NULL; + + OSAL_LIST_FOR_EACH_ENTRY(p_cmd_elem, &p_hwfn->mcp_info->cmd_list, list, + struct ecore_mcp_cmd_elem) { + if (p_cmd_elem->expected_seq_num == seq_num) + return p_cmd_elem; + } + + return OSAL_NULL; +} + enum _ecore_status_t ecore_mcp_free(struct ecore_hwfn *p_hwfn) { if (p_hwfn->mcp_info) { + struct ecore_mcp_cmd_elem *p_cmd_elem = OSAL_NULL, *p_tmp; + OSAL_FREE(p_hwfn->p_dev, p_hwfn->mcp_info->mfw_mb_cur); OSAL_FREE(p_hwfn->p_dev, p_hwfn->mcp_info->mfw_mb_shadow); - OSAL_SPIN_LOCK_DEALLOC(&p_hwfn->mcp_info->lock); + + OSAL_SPIN_LOCK(&p_hwfn->mcp_info->cmd_lock); + OSAL_LIST_FOR_EACH_ENTRY_SAFE(p_cmd_elem, p_tmp, + &p_hwfn->mcp_info->cmd_list, list, + struct ecore_mcp_cmd_elem) { + ecore_mcp_cmd_del_elem(p_hwfn, p_cmd_elem); + } + OSAL_SPIN_UNLOCK(&p_hwfn->mcp_info->cmd_lock); + +#ifdef CONFIG_ECORE_LOCK_ALLOC + OSAL_SPIN_LOCK_DEALLOC(&p_hwfn->mcp_info->cmd_lock); + OSAL_SPIN_LOCK_DEALLOC(&p_hwfn->mcp_info->link_lock); +#endif } + OSAL_FREE(p_hwfn->p_dev, p_hwfn->mcp_info); - p_hwfn->mcp_info = OSAL_NULL; return ECORE_SUCCESS; } @@ -135,7 +205,8 @@ static enum _ecore_status_t ecore_load_mcp_offsets(struct ecore_hwfn *p_hwfn, PUBLIC_DRV_MB)); p_info->drv_mb_addr = SECTION_ADDR(drv_mb_offsize, mcp_pf_id); DP_VERBOSE(p_hwfn, ECORE_MSG_SP, - "drv_mb_offsiz = 0x%x, drv_mb_addr = 0x%x mcp_pf_id = 0x%x\n", + "drv_mb_offsiz = 0x%x, drv_mb_addr = 0x%x" + " mcp_pf_id = 0x%x\n", drv_mb_offsize, p_info->drv_mb_addr, mcp_pf_id); /* Set the MFW MB address */ @@ -156,8 +227,7 @@ static enum _ecore_status_t ecore_load_mcp_offsets(struct ecore_hwfn *p_hwfn, p_info->drv_pulse_seq = DRV_MB_RD(p_hwfn, p_ptt, drv_pulse_mb) & DRV_PULSE_SEQ_MASK; - p_info->mcp_hist = (u16)ecore_rd(p_hwfn, p_ptt, - MISCS_REG_GENERIC_POR_0); + p_info->mcp_hist = ecore_rd(p_hwfn, p_ptt, MISCS_REG_GENERIC_POR_0); return ECORE_SUCCESS; } @@ -175,6 +245,16 @@ enum _ecore_status_t ecore_mcp_cmd_init(struct ecore_hwfn *p_hwfn, goto err; p_info = p_hwfn->mcp_info; + /* Initialize the MFW spinlocks */ +#ifdef CONFIG_ECORE_LOCK_ALLOC + OSAL_SPIN_LOCK_ALLOC(p_hwfn, &p_info->cmd_lock); + OSAL_SPIN_LOCK_ALLOC(p_hwfn, &p_info->link_lock); +#endif + OSAL_SPIN_LOCK_INIT(&p_info->cmd_lock); + OSAL_SPIN_LOCK_INIT(&p_info->link_lock); + + OSAL_LIST_INIT(&p_info->cmd_list); + if (ecore_load_mcp_offsets(p_hwfn, p_ptt) != ECORE_SUCCESS) { DP_NOTICE(p_hwfn, false, "MCP is not initialized\n"); /* Do not free mcp_info here, since public_base indicate that @@ -189,10 +269,6 @@ enum _ecore_status_t ecore_mcp_cmd_init(struct ecore_hwfn *p_hwfn, if (!p_info->mfw_mb_shadow || !p_info->mfw_mb_addr) goto err; - /* Initialize the MFW spinlock */ - OSAL_SPIN_LOCK_ALLOC(p_hwfn, &p_info->lock); - OSAL_SPIN_LOCK_INIT(&p_info->lock); - return ECORE_SUCCESS; err: @@ -201,12 +277,28 @@ err: return ECORE_NOMEM; } +static void ecore_mcp_reread_offsets(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt) +{ + u32 generic_por_0 = ecore_rd(p_hwfn, p_ptt, MISCS_REG_GENERIC_POR_0); + + /* Use MCP history register to check if MCP reset occurred between init + * time and now. + */ + if (p_hwfn->mcp_info->mcp_hist != generic_por_0) { + DP_VERBOSE(p_hwfn, ECORE_MSG_SP, + "Rereading MCP offsets [mcp_hist 0x%08x, generic_por_0 0x%08x]\n", + p_hwfn->mcp_info->mcp_hist, generic_por_0); + + ecore_load_mcp_offsets(p_hwfn, p_ptt); + ecore_mcp_cmd_port_init(p_hwfn, p_ptt); + } +} + enum _ecore_status_t ecore_mcp_reset(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt) { - u32 seq = ++p_hwfn->mcp_info->drv_mb_seq; - u32 delay = CHIP_MCP_RESP_ITER_US; - u32 org_mcp_reset_seq, cnt = 0; + u32 org_mcp_reset_seq, seq, delay = CHIP_MCP_RESP_ITER_US, cnt = 0; enum _ecore_status_t rc = ECORE_SUCCESS; #ifndef ASIC_ONLY @@ -214,10 +306,20 @@ enum _ecore_status_t ecore_mcp_reset(struct ecore_hwfn *p_hwfn, delay = EMUL_MCP_RESP_ITER_US; #endif - OSAL_SPIN_LOCK(&p_hwfn->mcp_info->lock); + if (p_hwfn->mcp_info->b_block_cmd) { + DP_NOTICE(p_hwfn, false, + "The MFW is not responsive. Avoid sending MCP_RESET mailbox command.\n"); + return ECORE_ABORTED; + } + + /* Ensure that only a single thread is accessing the mailbox */ + OSAL_SPIN_LOCK(&p_hwfn->mcp_info->cmd_lock); + + org_mcp_reset_seq = ecore_rd(p_hwfn, p_ptt, MISCS_REG_GENERIC_POR_0); /* Set drv command along with the updated sequence */ - org_mcp_reset_seq = ecore_rd(p_hwfn, p_ptt, MISCS_REG_GENERIC_POR_0); + ecore_mcp_reread_offsets(p_hwfn, p_ptt); + seq = ++p_hwfn->mcp_info->drv_mb_seq; DRV_MB_WR(p_hwfn, p_ptt, drv_mb_header, (DRV_MSG_CODE_MCP_RESET | seq)); do { @@ -237,85 +339,291 @@ enum _ecore_status_t ecore_mcp_reset(struct ecore_hwfn *p_hwfn, rc = ECORE_AGAIN; } - OSAL_SPIN_UNLOCK(&p_hwfn->mcp_info->lock); + OSAL_SPIN_UNLOCK(&p_hwfn->mcp_info->cmd_lock); return rc; } -/* Should be called while the dedicated spinlock is acquired */ -static enum _ecore_status_t ecore_do_mcp_cmd(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, - u32 cmd, u32 param, - u32 *o_mcp_resp, - u32 *o_mcp_param) +/* Must be called while cmd_lock is acquired */ +static bool ecore_mcp_has_pending_cmd(struct ecore_hwfn *p_hwfn) { - u32 delay = CHIP_MCP_RESP_ITER_US; - u32 seq, cnt = 1, actual_mb_seq; + struct ecore_mcp_cmd_elem *p_cmd_elem = OSAL_NULL; + + /* There is at most one pending command at a certain time, and if it + * exists - it is placed at the HEAD of the list. + */ + if (!OSAL_LIST_IS_EMPTY(&p_hwfn->mcp_info->cmd_list)) { + p_cmd_elem = OSAL_LIST_FIRST_ENTRY(&p_hwfn->mcp_info->cmd_list, + struct ecore_mcp_cmd_elem, + list); + return !p_cmd_elem->b_is_completed; + } + + return false; +} + +/* Must be called while cmd_lock is acquired */ +static enum _ecore_status_t +ecore_mcp_update_pending_cmd(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt) +{ + struct ecore_mcp_mb_params *p_mb_params; + struct ecore_mcp_cmd_elem *p_cmd_elem; + u32 mcp_resp; + u16 seq_num; + + mcp_resp = DRV_MB_RD(p_hwfn, p_ptt, fw_mb_header); + seq_num = (u16)(mcp_resp & FW_MSG_SEQ_NUMBER_MASK); + + /* Return if no new non-handled response has been received */ + if (seq_num != p_hwfn->mcp_info->drv_mb_seq) + return ECORE_AGAIN; + + p_cmd_elem = ecore_mcp_cmd_get_elem(p_hwfn, seq_num); + if (!p_cmd_elem) { + DP_ERR(p_hwfn, + "Failed to find a pending mailbox cmd that expects sequence number %d\n", + seq_num); + return ECORE_UNKNOWN_ERROR; + } + + p_mb_params = p_cmd_elem->p_mb_params; + + /* Get the MFW response along with the sequence number */ + p_mb_params->mcp_resp = mcp_resp; + + /* Get the MFW param */ + p_mb_params->mcp_param = DRV_MB_RD(p_hwfn, p_ptt, fw_mb_param); + + /* Get the union data */ + if (p_mb_params->p_data_dst != OSAL_NULL && + p_mb_params->data_dst_size) { + u32 union_data_addr = p_hwfn->mcp_info->drv_mb_addr + + OFFSETOF(struct public_drv_mb, + union_data); + ecore_memcpy_from(p_hwfn, p_ptt, p_mb_params->p_data_dst, + union_data_addr, p_mb_params->data_dst_size); + } + + p_cmd_elem->b_is_completed = true; + + return ECORE_SUCCESS; +} + +/* Must be called while cmd_lock is acquired */ +static void __ecore_mcp_cmd_and_union(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + struct ecore_mcp_mb_params *p_mb_params, + u16 seq_num) +{ + union drv_union_data union_data; + u32 union_data_addr; + + /* Set the union data */ + union_data_addr = p_hwfn->mcp_info->drv_mb_addr + + OFFSETOF(struct public_drv_mb, union_data); + OSAL_MEM_ZERO(&union_data, sizeof(union_data)); + if (p_mb_params->p_data_src != OSAL_NULL && p_mb_params->data_src_size) + OSAL_MEMCPY(&union_data, p_mb_params->p_data_src, + p_mb_params->data_src_size); + ecore_memcpy_to(p_hwfn, p_ptt, union_data_addr, &union_data, + sizeof(union_data)); + + /* Set the drv param */ + DRV_MB_WR(p_hwfn, p_ptt, drv_mb_param, p_mb_params->param); + + /* Set the drv command along with the sequence number */ + DRV_MB_WR(p_hwfn, p_ptt, drv_mb_header, (p_mb_params->cmd | seq_num)); + + DP_VERBOSE(p_hwfn, ECORE_MSG_SP, + "MFW mailbox: command 0x%08x param 0x%08x\n", + (p_mb_params->cmd | seq_num), p_mb_params->param); +} + +static void ecore_mcp_cmd_set_blocking(struct ecore_hwfn *p_hwfn, + bool block_cmd) +{ + p_hwfn->mcp_info->b_block_cmd = block_cmd; + + DP_INFO(p_hwfn, "%s sending of mailbox commands to the MFW\n", + block_cmd ? "Block" : "Unblock"); +} + +void ecore_mcp_print_cpu_info(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt) +{ + u32 cpu_mode, cpu_state, cpu_pc_0, cpu_pc_1, cpu_pc_2; + + cpu_mode = ecore_rd(p_hwfn, p_ptt, MCP_REG_CPU_MODE); + cpu_state = ecore_rd(p_hwfn, p_ptt, MCP_REG_CPU_STATE); + cpu_pc_0 = ecore_rd(p_hwfn, p_ptt, MCP_REG_CPU_PROGRAM_COUNTER); + OSAL_UDELAY(CHIP_MCP_RESP_ITER_US); + cpu_pc_1 = ecore_rd(p_hwfn, p_ptt, MCP_REG_CPU_PROGRAM_COUNTER); + OSAL_UDELAY(CHIP_MCP_RESP_ITER_US); + cpu_pc_2 = ecore_rd(p_hwfn, p_ptt, MCP_REG_CPU_PROGRAM_COUNTER); + + DP_NOTICE(p_hwfn, false, + "MCP CPU info: mode 0x%08x, state 0x%08x, pc {0x%08x, 0x%08x, 0x%08x}\n", + cpu_mode, cpu_state, cpu_pc_0, cpu_pc_1, cpu_pc_2); +} + +static enum _ecore_status_t +_ecore_mcp_cmd_and_union(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, + struct ecore_mcp_mb_params *p_mb_params, + u32 max_retries, u32 delay) +{ + struct ecore_mcp_cmd_elem *p_cmd_elem; + u32 cnt = 0; + u16 seq_num; enum _ecore_status_t rc = ECORE_SUCCESS; + /* Wait until the mailbox is non-occupied */ + do { + /* Exit the loop if there is no pending command, or if the + * pending command is completed during this iteration. + * The spinlock stays locked until the command is sent. + */ + + OSAL_SPIN_LOCK(&p_hwfn->mcp_info->cmd_lock); + + if (!ecore_mcp_has_pending_cmd(p_hwfn)) + break; + + rc = ecore_mcp_update_pending_cmd(p_hwfn, p_ptt); + if (rc == ECORE_SUCCESS) + break; + else if (rc != ECORE_AGAIN) + goto err; + + OSAL_SPIN_UNLOCK(&p_hwfn->mcp_info->cmd_lock); + OSAL_UDELAY(delay); + OSAL_MFW_CMD_PREEMPT(p_hwfn); + } while (++cnt < max_retries); + + if (cnt >= max_retries) { + DP_NOTICE(p_hwfn, false, + "The MFW mailbox is occupied by an uncompleted command. Failed to send command 0x%08x [param 0x%08x].\n", + p_mb_params->cmd, p_mb_params->param); + return ECORE_AGAIN; + } + + /* Send the mailbox command */ + ecore_mcp_reread_offsets(p_hwfn, p_ptt); + seq_num = ++p_hwfn->mcp_info->drv_mb_seq; + p_cmd_elem = ecore_mcp_cmd_add_elem(p_hwfn, p_mb_params, seq_num); + if (!p_cmd_elem) { + rc = ECORE_NOMEM; + goto err; + } + + __ecore_mcp_cmd_and_union(p_hwfn, p_ptt, p_mb_params, seq_num); + OSAL_SPIN_UNLOCK(&p_hwfn->mcp_info->cmd_lock); + + /* Wait for the MFW response */ + do { + /* Exit the loop if the command is already completed, or if the + * command is completed during this iteration. + * The spinlock stays locked until the list element is removed. + */ + + OSAL_UDELAY(delay); + OSAL_SPIN_LOCK(&p_hwfn->mcp_info->cmd_lock); + + if (p_cmd_elem->b_is_completed) + break; + + rc = ecore_mcp_update_pending_cmd(p_hwfn, p_ptt); + if (rc == ECORE_SUCCESS) + break; + else if (rc != ECORE_AGAIN) + goto err; + + OSAL_SPIN_UNLOCK(&p_hwfn->mcp_info->cmd_lock); + OSAL_MFW_CMD_PREEMPT(p_hwfn); + } while (++cnt < max_retries); + + if (cnt >= max_retries) { + DP_NOTICE(p_hwfn, false, + "The MFW failed to respond to command 0x%08x [param 0x%08x].\n", + p_mb_params->cmd, p_mb_params->param); + ecore_mcp_print_cpu_info(p_hwfn, p_ptt); + + OSAL_SPIN_LOCK(&p_hwfn->mcp_info->cmd_lock); + ecore_mcp_cmd_del_elem(p_hwfn, p_cmd_elem); + OSAL_SPIN_UNLOCK(&p_hwfn->mcp_info->cmd_lock); + + ecore_mcp_cmd_set_blocking(p_hwfn, true); + ecore_hw_err_notify(p_hwfn, ECORE_HW_ERR_MFW_RESP_FAIL); + return ECORE_AGAIN; + } + + ecore_mcp_cmd_del_elem(p_hwfn, p_cmd_elem); + OSAL_SPIN_UNLOCK(&p_hwfn->mcp_info->cmd_lock); + + DP_VERBOSE(p_hwfn, ECORE_MSG_SP, + "MFW mailbox: response 0x%08x param 0x%08x [after %d.%03d ms]\n", + p_mb_params->mcp_resp, p_mb_params->mcp_param, + (cnt * delay) / 1000, (cnt * delay) % 1000); + + /* Clear the sequence number from the MFW response */ + p_mb_params->mcp_resp &= FW_MSG_CODE_MASK; + + return ECORE_SUCCESS; + +err: + OSAL_SPIN_UNLOCK(&p_hwfn->mcp_info->cmd_lock); + return rc; +} + +static enum _ecore_status_t +ecore_mcp_cmd_and_union(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + struct ecore_mcp_mb_params *p_mb_params) +{ + osal_size_t union_data_size = sizeof(union drv_union_data); + u32 max_retries = ECORE_DRV_MB_MAX_RETRIES; + u32 delay = CHIP_MCP_RESP_ITER_US; + #ifndef ASIC_ONLY if (CHIP_REV_IS_EMUL(p_hwfn->p_dev)) delay = EMUL_MCP_RESP_ITER_US; + /* There is a built-in delay of 100usec in each MFW response read */ + if (CHIP_REV_IS_FPGA(p_hwfn->p_dev)) + max_retries /= 10; #endif - /* Get actual driver mailbox sequence */ - actual_mb_seq = DRV_MB_RD(p_hwfn, p_ptt, drv_mb_header) & - DRV_MSG_SEQ_NUMBER_MASK; - - /* Use MCP history register to check if MCP reset occurred between - * init time and now. - */ - if (p_hwfn->mcp_info->mcp_hist != - ecore_rd(p_hwfn, p_ptt, MISCS_REG_GENERIC_POR_0)) { - DP_VERBOSE(p_hwfn, ECORE_MSG_SP, "Rereading MCP offsets\n"); - ecore_load_mcp_offsets(p_hwfn, p_ptt); - ecore_mcp_cmd_port_init(p_hwfn, p_ptt); + /* MCP not initialized */ + if (!ecore_mcp_is_init(p_hwfn)) { + DP_NOTICE(p_hwfn, true, "MFW is not initialized !\n"); + return ECORE_BUSY; } - seq = ++p_hwfn->mcp_info->drv_mb_seq; - /* Set drv param */ - DRV_MB_WR(p_hwfn, p_ptt, drv_mb_param, param); - - /* Set drv command along with the updated sequence */ - DRV_MB_WR(p_hwfn, p_ptt, drv_mb_header, (cmd | seq)); - - DP_VERBOSE(p_hwfn, ECORE_MSG_SP, - "wrote command (%x) to MFW MB param 0x%08x\n", - (cmd | seq), param); - - do { - /* Wait for MFW response */ - OSAL_UDELAY(delay); - *o_mcp_resp = DRV_MB_RD(p_hwfn, p_ptt, fw_mb_header); - - /* Give the FW up to 5 second (500*10ms) */ - } while ((seq != (*o_mcp_resp & FW_MSG_SEQ_NUMBER_MASK)) && - (cnt++ < ECORE_DRV_MB_MAX_RETRIES)); - - DP_VERBOSE(p_hwfn, ECORE_MSG_SP, - "[after %d ms] read (%x) seq is (%x) from FW MB\n", - cnt * delay, *o_mcp_resp, seq); - - /* Is this a reply to our command? */ - if (seq == (*o_mcp_resp & FW_MSG_SEQ_NUMBER_MASK)) { - *o_mcp_resp &= FW_MSG_CODE_MASK; - /* Get the MCP param */ - *o_mcp_param = DRV_MB_RD(p_hwfn, p_ptt, fw_mb_param); - } else { - /* FW BUG! */ - DP_ERR(p_hwfn, "MFW failed to respond [cmd 0x%x param 0x%x]\n", - cmd, param); - *o_mcp_resp = 0; - rc = ECORE_AGAIN; - ecore_hw_err_notify(p_hwfn, ECORE_HW_ERR_MFW_RESP_FAIL); + if (p_mb_params->data_src_size > union_data_size || + p_mb_params->data_dst_size > union_data_size) { + DP_ERR(p_hwfn, + "The provided size is larger than the union data size [src_size %u, dst_size %u, union_data_size %zu]\n", + p_mb_params->data_src_size, p_mb_params->data_dst_size, + union_data_size); + return ECORE_INVAL; } - return rc; + + if (p_hwfn->mcp_info->b_block_cmd) { + DP_NOTICE(p_hwfn, false, + "The MFW is not responsive. Avoid sending mailbox command 0x%08x [param 0x%08x].\n", + p_mb_params->cmd, p_mb_params->param); + return ECORE_ABORTED; + } + + return _ecore_mcp_cmd_and_union(p_hwfn, p_ptt, p_mb_params, max_retries, + delay); } enum _ecore_status_t ecore_mcp_cmd(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, u32 cmd, u32 param, u32 *o_mcp_resp, u32 *o_mcp_param) { + struct ecore_mcp_mb_params mb_params; + enum _ecore_status_t rc; + #ifndef ASIC_ONLY if (CHIP_REV_IS_EMUL(p_hwfn->p_dev)) { if (cmd == DRV_MSG_CODE_UNLOAD_REQ) { @@ -328,44 +636,17 @@ enum _ecore_status_t ecore_mcp_cmd(struct ecore_hwfn *p_hwfn, } #endif - return ecore_mcp_cmd_and_union(p_hwfn, p_ptt, cmd, param, OSAL_NULL, - o_mcp_resp, o_mcp_param); -} + OSAL_MEM_ZERO(&mb_params, sizeof(mb_params)); + mb_params.cmd = cmd; + mb_params.param = param; + rc = ecore_mcp_cmd_and_union(p_hwfn, p_ptt, &mb_params); + if (rc != ECORE_SUCCESS) + return rc; -enum _ecore_status_t ecore_mcp_cmd_and_union(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, - u32 cmd, u32 param, - union drv_union_data *p_union_data, - u32 *o_mcp_resp, - u32 *o_mcp_param) -{ - u32 union_data_addr; - enum _ecore_status_t rc; + *o_mcp_resp = mb_params.mcp_resp; + *o_mcp_param = mb_params.mcp_param; - /* MCP not initialized */ - if (!ecore_mcp_is_init(p_hwfn)) { - DP_NOTICE(p_hwfn, true, "MFW is not initialized !\n"); - return ECORE_BUSY; - } - - /* Acquiring a spinlock is needed to ensure that only a single thread - * is accessing the mailbox at a certain time. - */ - OSAL_SPIN_LOCK(&p_hwfn->mcp_info->lock); - - if (p_union_data != OSAL_NULL) { - union_data_addr = p_hwfn->mcp_info->drv_mb_addr + - OFFSETOF(struct public_drv_mb, union_data); - ecore_memcpy_to(p_hwfn, p_ptt, union_data_addr, p_union_data, - sizeof(*p_union_data)); - } - - rc = ecore_do_mcp_cmd(p_hwfn, p_ptt, cmd, param, o_mcp_resp, - o_mcp_param); - - OSAL_SPIN_UNLOCK(&p_hwfn->mcp_info->lock); - - return rc; + return ECORE_SUCCESS; } enum _ecore_status_t ecore_mcp_nvm_wr_cmd(struct ecore_hwfn *p_hwfn, @@ -376,12 +657,22 @@ enum _ecore_status_t ecore_mcp_nvm_wr_cmd(struct ecore_hwfn *p_hwfn, u32 *o_mcp_param, u32 i_txn_size, u32 *i_buf) { - union drv_union_data union_data; + struct ecore_mcp_mb_params mb_params; + enum _ecore_status_t rc; - OSAL_MEMCPY((u32 *)&union_data.raw_data, i_buf, i_txn_size); + OSAL_MEM_ZERO(&mb_params, sizeof(mb_params)); + mb_params.cmd = cmd; + mb_params.param = param; + mb_params.p_data_src = i_buf; + mb_params.data_src_size = (u8)i_txn_size; + rc = ecore_mcp_cmd_and_union(p_hwfn, p_ptt, &mb_params); + if (rc != ECORE_SUCCESS) + return rc; - return ecore_mcp_cmd_and_union(p_hwfn, p_ptt, cmd, param, &union_data, - o_mcp_resp, o_mcp_param); + *o_mcp_resp = mb_params.mcp_resp; + *o_mcp_param = mb_params.mcp_param; + + return ECORE_SUCCESS; } enum _ecore_status_t ecore_mcp_nvm_rd_cmd(struct ecore_hwfn *p_hwfn, @@ -392,30 +683,30 @@ enum _ecore_status_t ecore_mcp_nvm_rd_cmd(struct ecore_hwfn *p_hwfn, u32 *o_mcp_param, u32 *o_txn_size, u32 *o_buf) { + struct ecore_mcp_mb_params mb_params; + u8 raw_data[MCP_DRV_NVM_BUF_LEN]; enum _ecore_status_t rc; - u32 i; - /* MCP not initialized */ - if (!ecore_mcp_is_init(p_hwfn)) { - DP_NOTICE(p_hwfn, true, "MFW is not initialized !\n"); - return ECORE_BUSY; - } + OSAL_MEM_ZERO(&mb_params, sizeof(mb_params)); + mb_params.cmd = cmd; + mb_params.param = param; + mb_params.p_data_dst = raw_data; - OSAL_SPIN_LOCK(&p_hwfn->mcp_info->lock); - rc = ecore_do_mcp_cmd(p_hwfn, p_ptt, cmd, param, o_mcp_resp, - o_mcp_param); + /* Use the maximal value since the actual one is part of the response */ + mb_params.data_dst_size = MCP_DRV_NVM_BUF_LEN; + + rc = ecore_mcp_cmd_and_union(p_hwfn, p_ptt, &mb_params); if (rc != ECORE_SUCCESS) - goto out; + return rc; + + *o_mcp_resp = mb_params.mcp_resp; + *o_mcp_param = mb_params.mcp_param; - /* Get payload after operation completes successfully */ *o_txn_size = *o_mcp_param; - for (i = 0; i < *o_txn_size; i += 4) - o_buf[i / sizeof(u32)] = DRV_MB_RD(p_hwfn, p_ptt, - union_data.raw_data[i]); + /* @DPDK */ + OSAL_MEMCPY(o_buf, raw_data, RTE_MIN(*o_txn_size, MCP_DRV_NVM_BUF_LEN)); -out: - OSAL_SPIN_UNLOCK(&p_hwfn->mcp_info->lock); - return rc; + return ECORE_SUCCESS; } #ifndef ASIC_ONLY @@ -432,7 +723,7 @@ static void ecore_mcp_mf_workaround(struct ecore_hwfn *p_hwfn, load_phase = FW_MSG_CODE_DRV_LOAD_FUNCTION; /* On CMT, always tell that it's engine */ - if (p_hwfn->p_dev->num_hwfns > 1) + if (ECORE_IS_CMT(p_hwfn->p_dev)) load_phase = FW_MSG_CODE_DRV_LOAD_ENGINE; *p_load_code = load_phase; @@ -446,54 +737,391 @@ static void ecore_mcp_mf_workaround(struct ecore_hwfn *p_hwfn, } #endif +static bool +ecore_mcp_can_force_load(u8 drv_role, u8 exist_drv_role, + enum ecore_override_force_load override_force_load) +{ + bool can_force_load = false; + + switch (override_force_load) { + case ECORE_OVERRIDE_FORCE_LOAD_ALWAYS: + can_force_load = true; + break; + case ECORE_OVERRIDE_FORCE_LOAD_NEVER: + can_force_load = false; + break; + default: + can_force_load = (drv_role == DRV_ROLE_OS && + exist_drv_role == DRV_ROLE_PREBOOT) || + (drv_role == DRV_ROLE_KDUMP && + exist_drv_role == DRV_ROLE_OS); + break; + } + + return can_force_load; +} + +static enum _ecore_status_t ecore_mcp_cancel_load_req(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt) +{ + u32 resp = 0, param = 0; + enum _ecore_status_t rc; + + rc = ecore_mcp_cmd(p_hwfn, p_ptt, DRV_MSG_CODE_CANCEL_LOAD_REQ, 0, + &resp, ¶m); + if (rc != ECORE_SUCCESS) + DP_NOTICE(p_hwfn, false, + "Failed to send cancel load request, rc = %d\n", rc); + + return rc; +} + +#define CONFIG_ECORE_L2_BITMAP_IDX (0x1 << 0) +#define CONFIG_ECORE_SRIOV_BITMAP_IDX (0x1 << 1) +#define CONFIG_ECORE_ROCE_BITMAP_IDX (0x1 << 2) +#define CONFIG_ECORE_IWARP_BITMAP_IDX (0x1 << 3) +#define CONFIG_ECORE_FCOE_BITMAP_IDX (0x1 << 4) +#define CONFIG_ECORE_ISCSI_BITMAP_IDX (0x1 << 5) +#define CONFIG_ECORE_LL2_BITMAP_IDX (0x1 << 6) + +static u32 ecore_get_config_bitmap(void) +{ + u32 config_bitmap = 0x0; + +#ifdef CONFIG_ECORE_L2 + config_bitmap |= CONFIG_ECORE_L2_BITMAP_IDX; +#endif +#ifdef CONFIG_ECORE_SRIOV + config_bitmap |= CONFIG_ECORE_SRIOV_BITMAP_IDX; +#endif +#ifdef CONFIG_ECORE_ROCE + config_bitmap |= CONFIG_ECORE_ROCE_BITMAP_IDX; +#endif +#ifdef CONFIG_ECORE_IWARP + config_bitmap |= CONFIG_ECORE_IWARP_BITMAP_IDX; +#endif +#ifdef CONFIG_ECORE_FCOE + config_bitmap |= CONFIG_ECORE_FCOE_BITMAP_IDX; +#endif +#ifdef CONFIG_ECORE_ISCSI + config_bitmap |= CONFIG_ECORE_ISCSI_BITMAP_IDX; +#endif +#ifdef CONFIG_ECORE_LL2 + config_bitmap |= CONFIG_ECORE_LL2_BITMAP_IDX; +#endif + + return config_bitmap; +} + +struct ecore_load_req_in_params { + u8 hsi_ver; +#define ECORE_LOAD_REQ_HSI_VER_DEFAULT 0 +#define ECORE_LOAD_REQ_HSI_VER_1 1 + u32 drv_ver_0; + u32 drv_ver_1; + u32 fw_ver; + u8 drv_role; + u8 timeout_val; + u8 force_cmd; + bool avoid_eng_reset; +}; + +struct ecore_load_req_out_params { + u32 load_code; + u32 exist_drv_ver_0; + u32 exist_drv_ver_1; + u32 exist_fw_ver; + u8 exist_drv_role; + u8 mfw_hsi_ver; + bool drv_exists; +}; + +static enum _ecore_status_t +__ecore_mcp_load_req(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, + struct ecore_load_req_in_params *p_in_params, + struct ecore_load_req_out_params *p_out_params) +{ + struct ecore_mcp_mb_params mb_params; + struct load_req_stc load_req; + struct load_rsp_stc load_rsp; + u32 hsi_ver; + enum _ecore_status_t rc; + + OSAL_MEM_ZERO(&load_req, sizeof(load_req)); + load_req.drv_ver_0 = p_in_params->drv_ver_0; + load_req.drv_ver_1 = p_in_params->drv_ver_1; + load_req.fw_ver = p_in_params->fw_ver; + SET_MFW_FIELD(load_req.misc0, LOAD_REQ_ROLE, p_in_params->drv_role); + SET_MFW_FIELD(load_req.misc0, LOAD_REQ_LOCK_TO, + p_in_params->timeout_val); + SET_MFW_FIELD(load_req.misc0, LOAD_REQ_FORCE, p_in_params->force_cmd); + SET_MFW_FIELD(load_req.misc0, LOAD_REQ_FLAGS0, + p_in_params->avoid_eng_reset); + + hsi_ver = (p_in_params->hsi_ver == ECORE_LOAD_REQ_HSI_VER_DEFAULT) ? + DRV_ID_MCP_HSI_VER_CURRENT : + (p_in_params->hsi_ver << DRV_ID_MCP_HSI_VER_OFFSET); + + OSAL_MEM_ZERO(&mb_params, sizeof(mb_params)); + mb_params.cmd = DRV_MSG_CODE_LOAD_REQ; + mb_params.param = PDA_COMP | hsi_ver | p_hwfn->p_dev->drv_type; + mb_params.p_data_src = &load_req; + mb_params.data_src_size = sizeof(load_req); + mb_params.p_data_dst = &load_rsp; + mb_params.data_dst_size = sizeof(load_rsp); + + DP_VERBOSE(p_hwfn, ECORE_MSG_SP, + "Load Request: param 0x%08x [init_hw %d, drv_type %d, hsi_ver %d, pda 0x%04x]\n", + mb_params.param, + GET_MFW_FIELD(mb_params.param, DRV_ID_DRV_INIT_HW), + GET_MFW_FIELD(mb_params.param, DRV_ID_DRV_TYPE), + GET_MFW_FIELD(mb_params.param, DRV_ID_MCP_HSI_VER), + GET_MFW_FIELD(mb_params.param, DRV_ID_PDA_COMP_VER)); + + if (p_in_params->hsi_ver != ECORE_LOAD_REQ_HSI_VER_1) + DP_VERBOSE(p_hwfn, ECORE_MSG_SP, + "Load Request: drv_ver 0x%08x_0x%08x, fw_ver 0x%08x, misc0 0x%08x [role %d, timeout %d, force %d, flags0 0x%x]\n", + load_req.drv_ver_0, load_req.drv_ver_1, + load_req.fw_ver, load_req.misc0, + GET_MFW_FIELD(load_req.misc0, LOAD_REQ_ROLE), + GET_MFW_FIELD(load_req.misc0, LOAD_REQ_LOCK_TO), + GET_MFW_FIELD(load_req.misc0, LOAD_REQ_FORCE), + GET_MFW_FIELD(load_req.misc0, LOAD_REQ_FLAGS0)); + + rc = ecore_mcp_cmd_and_union(p_hwfn, p_ptt, &mb_params); + if (rc != ECORE_SUCCESS) { + DP_NOTICE(p_hwfn, false, + "Failed to send load request, rc = %d\n", rc); + return rc; + } + + DP_VERBOSE(p_hwfn, ECORE_MSG_SP, + "Load Response: resp 0x%08x\n", mb_params.mcp_resp); + p_out_params->load_code = mb_params.mcp_resp; + + if (p_in_params->hsi_ver != ECORE_LOAD_REQ_HSI_VER_1 && + p_out_params->load_code != FW_MSG_CODE_DRV_LOAD_REFUSED_HSI_1) { + DP_VERBOSE(p_hwfn, ECORE_MSG_SP, + "Load Response: exist_drv_ver 0x%08x_0x%08x, exist_fw_ver 0x%08x, misc0 0x%08x [exist_role %d, mfw_hsi %d, flags0 0x%x]\n", + load_rsp.drv_ver_0, load_rsp.drv_ver_1, + load_rsp.fw_ver, load_rsp.misc0, + GET_MFW_FIELD(load_rsp.misc0, LOAD_RSP_ROLE), + GET_MFW_FIELD(load_rsp.misc0, LOAD_RSP_HSI), + GET_MFW_FIELD(load_rsp.misc0, LOAD_RSP_FLAGS0)); + + p_out_params->exist_drv_ver_0 = load_rsp.drv_ver_0; + p_out_params->exist_drv_ver_1 = load_rsp.drv_ver_1; + p_out_params->exist_fw_ver = load_rsp.fw_ver; + p_out_params->exist_drv_role = + GET_MFW_FIELD(load_rsp.misc0, LOAD_RSP_ROLE); + p_out_params->mfw_hsi_ver = + GET_MFW_FIELD(load_rsp.misc0, LOAD_RSP_HSI); + p_out_params->drv_exists = + GET_MFW_FIELD(load_rsp.misc0, LOAD_RSP_FLAGS0) & + LOAD_RSP_FLAGS0_DRV_EXISTS; + } + + return ECORE_SUCCESS; +} + +static void ecore_get_mfw_drv_role(enum ecore_drv_role drv_role, + u8 *p_mfw_drv_role) +{ + switch (drv_role) { + case ECORE_DRV_ROLE_OS: + *p_mfw_drv_role = DRV_ROLE_OS; + break; + case ECORE_DRV_ROLE_KDUMP: + *p_mfw_drv_role = DRV_ROLE_KDUMP; + break; + } +} + +enum ecore_load_req_force { + ECORE_LOAD_REQ_FORCE_NONE, + ECORE_LOAD_REQ_FORCE_PF, + ECORE_LOAD_REQ_FORCE_ALL, +}; + +static void ecore_get_mfw_force_cmd(enum ecore_load_req_force force_cmd, + u8 *p_mfw_force_cmd) +{ + switch (force_cmd) { + case ECORE_LOAD_REQ_FORCE_NONE: + *p_mfw_force_cmd = LOAD_REQ_FORCE_NONE; + break; + case ECORE_LOAD_REQ_FORCE_PF: + *p_mfw_force_cmd = LOAD_REQ_FORCE_PF; + break; + case ECORE_LOAD_REQ_FORCE_ALL: + *p_mfw_force_cmd = LOAD_REQ_FORCE_ALL; + break; + } +} + enum _ecore_status_t ecore_mcp_load_req(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, - u32 *p_load_code) + struct ecore_load_req_params *p_params) { - struct ecore_dev *p_dev = p_hwfn->p_dev; - union drv_union_data union_data; - u32 param; + struct ecore_load_req_out_params out_params; + struct ecore_load_req_in_params in_params; + u8 mfw_drv_role = 0, mfw_force_cmd; enum _ecore_status_t rc; #ifndef ASIC_ONLY if (CHIP_REV_IS_EMUL(p_hwfn->p_dev)) { - ecore_mcp_mf_workaround(p_hwfn, p_load_code); + ecore_mcp_mf_workaround(p_hwfn, &p_params->load_code); return ECORE_SUCCESS; } #endif - OSAL_MEMCPY(&union_data.ver_str, p_dev->ver_str, MCP_DRV_VER_STR_SIZE); + OSAL_MEM_ZERO(&in_params, sizeof(in_params)); + in_params.hsi_ver = ECORE_LOAD_REQ_HSI_VER_DEFAULT; + in_params.drv_ver_0 = ECORE_VERSION; + in_params.drv_ver_1 = ecore_get_config_bitmap(); + in_params.fw_ver = STORM_FW_VERSION; + ecore_get_mfw_drv_role(p_params->drv_role, &mfw_drv_role); + in_params.drv_role = mfw_drv_role; + in_params.timeout_val = p_params->timeout_val; + ecore_get_mfw_force_cmd(ECORE_LOAD_REQ_FORCE_NONE, &mfw_force_cmd); + in_params.force_cmd = mfw_force_cmd; + in_params.avoid_eng_reset = p_params->avoid_eng_reset; - rc = ecore_mcp_cmd_and_union(p_hwfn, p_ptt, DRV_MSG_CODE_LOAD_REQ, - (PDA_COMP | DRV_ID_MCP_HSI_VER_CURRENT | - p_dev->drv_type), - &union_data, p_load_code, ¶m); - - /* if mcp fails to respond we must abort */ - if (rc != ECORE_SUCCESS) { - DP_ERR(p_hwfn, "MCP response failure, aborting\n"); + OSAL_MEM_ZERO(&out_params, sizeof(out_params)); + rc = __ecore_mcp_load_req(p_hwfn, p_ptt, &in_params, &out_params); + if (rc != ECORE_SUCCESS) return rc; + + /* First handle cases where another load request should/might be sent: + * - MFW expects the old interface [HSI version = 1] + * - MFW responds that a force load request is required + */ + if (out_params.load_code == FW_MSG_CODE_DRV_LOAD_REFUSED_HSI_1) { + DP_INFO(p_hwfn, + "MFW refused a load request due to HSI > 1. Resending with HSI = 1.\n"); + + in_params.hsi_ver = ECORE_LOAD_REQ_HSI_VER_1; + OSAL_MEM_ZERO(&out_params, sizeof(out_params)); + rc = __ecore_mcp_load_req(p_hwfn, p_ptt, &in_params, + &out_params); + if (rc != ECORE_SUCCESS) + return rc; + } else if (out_params.load_code == + FW_MSG_CODE_DRV_LOAD_REFUSED_REQUIRES_FORCE) { + if (ecore_mcp_can_force_load(in_params.drv_role, + out_params.exist_drv_role, + p_params->override_force_load)) { + DP_INFO(p_hwfn, + "A force load is required [{role, fw_ver, drv_ver}: loading={%d, 0x%08x, 0x%08x_%08x}, existing={%d, 0x%08x, 0x%08x_%08x}]\n", + in_params.drv_role, in_params.fw_ver, + in_params.drv_ver_0, in_params.drv_ver_1, + out_params.exist_drv_role, + out_params.exist_fw_ver, + out_params.exist_drv_ver_0, + out_params.exist_drv_ver_1); + + ecore_get_mfw_force_cmd(ECORE_LOAD_REQ_FORCE_ALL, + &mfw_force_cmd); + + in_params.force_cmd = mfw_force_cmd; + OSAL_MEM_ZERO(&out_params, sizeof(out_params)); + rc = __ecore_mcp_load_req(p_hwfn, p_ptt, &in_params, + &out_params); + if (rc != ECORE_SUCCESS) + return rc; + } else { + DP_NOTICE(p_hwfn, false, + "A force load is required [{role, fw_ver, drv_ver}: loading={%d, 0x%08x, x%08x_0x%08x}, existing={%d, 0x%08x, 0x%08x_0x%08x}] - Avoid\n", + in_params.drv_role, in_params.fw_ver, + in_params.drv_ver_0, in_params.drv_ver_1, + out_params.exist_drv_role, + out_params.exist_fw_ver, + out_params.exist_drv_ver_0, + out_params.exist_drv_ver_1); + + ecore_mcp_cancel_load_req(p_hwfn, p_ptt); + return ECORE_BUSY; + } } - /* If MFW refused (e.g. other port is in diagnostic mode) we - * must abort. This can happen in the following cases: - * - Other port is in diagnostic mode - * - Previously loaded function on the engine is not compliant with - * the requester. - * - MFW cannot cope with the requester's DRV_MFW_HSI_VERSION. - * - + /* Now handle the other types of responses. + * The "REFUSED_HSI_1" and "REFUSED_REQUIRES_FORCE" responses are not + * expected here after the additional revised load requests were sent. */ - if (!(*p_load_code) || - ((*p_load_code) == FW_MSG_CODE_DRV_LOAD_REFUSED_HSI) || - ((*p_load_code) == FW_MSG_CODE_DRV_LOAD_REFUSED_PDA) || - ((*p_load_code) == FW_MSG_CODE_DRV_LOAD_REFUSED_DIAG)) { - DP_ERR(p_hwfn, "MCP refused load request, aborting\n"); + switch (out_params.load_code) { + case FW_MSG_CODE_DRV_LOAD_ENGINE: + case FW_MSG_CODE_DRV_LOAD_PORT: + case FW_MSG_CODE_DRV_LOAD_FUNCTION: + if (out_params.mfw_hsi_ver != ECORE_LOAD_REQ_HSI_VER_1 && + out_params.drv_exists) { + /* The role and fw/driver version match, but the PF is + * already loaded and has not been unloaded gracefully. + * This is unexpected since a quasi-FLR request was + * previously sent as part of ecore_hw_prepare(). + */ + DP_NOTICE(p_hwfn, false, + "PF is already loaded - shouldn't have got here since a quasi-FLR request was previously sent!\n"); + return ECORE_INVAL; + } + break; + default: + DP_NOTICE(p_hwfn, false, + "Unexpected refusal to load request [resp 0x%08x]. Aborting.\n", + out_params.load_code); return ECORE_BUSY; } + p_params->load_code = out_params.load_code; + return ECORE_SUCCESS; } +enum _ecore_status_t ecore_mcp_load_done(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt) +{ + u32 resp = 0, param = 0; + enum _ecore_status_t rc; + + rc = ecore_mcp_cmd(p_hwfn, p_ptt, DRV_MSG_CODE_LOAD_DONE, 0, &resp, + ¶m); + if (rc != ECORE_SUCCESS) { + DP_NOTICE(p_hwfn, false, + "Failed to send a LOAD_DONE command, rc = %d\n", rc); + return rc; + } + + /* Check if there is a DID mismatch between nvm-cfg/efuse */ + if (param & FW_MB_PARAM_LOAD_DONE_DID_EFUSE_ERROR) + DP_NOTICE(p_hwfn, false, + "warning: device configuration is not supported on this board type. The device may not function as expected.\n"); + + return ECORE_SUCCESS; +} + +enum _ecore_status_t ecore_mcp_unload_req(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt) +{ + u32 wol_param, mcp_resp, mcp_param; + + /* @DPDK */ + wol_param = DRV_MB_PARAM_UNLOAD_WOL_MCP; + + return ecore_mcp_cmd(p_hwfn, p_ptt, DRV_MSG_CODE_UNLOAD_REQ, wol_param, + &mcp_resp, &mcp_param); +} + +enum _ecore_status_t ecore_mcp_unload_done(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt) +{ + struct ecore_mcp_mb_params mb_params; + struct mcp_mac wol_mac; + + OSAL_MEM_ZERO(&mb_params, sizeof(mb_params)); + mb_params.cmd = DRV_MSG_CODE_UNLOAD_DONE; + + return ecore_mcp_cmd_and_union(p_hwfn, p_ptt, &mb_params); +} + static void ecore_mcp_handle_vf_flr(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt) { @@ -534,8 +1162,7 @@ enum _ecore_status_t ecore_mcp_ack_vf_flr(struct ecore_hwfn *p_hwfn, u32 mfw_func_offsize = ecore_rd(p_hwfn, p_ptt, addr); u32 func_addr = SECTION_ADDR(mfw_func_offsize, MCP_PF_ID(p_hwfn)); - union drv_union_data union_data; - u32 resp, param; + struct ecore_mcp_mb_params mb_params; enum _ecore_status_t rc; int i; @@ -544,13 +1171,14 @@ enum _ecore_status_t ecore_mcp_ack_vf_flr(struct ecore_hwfn *p_hwfn, "Acking VFs [%08x,...,%08x] - %08x\n", i * 32, (i + 1) * 32 - 1, vfs_to_ack[i]); - OSAL_MEMCPY(&union_data.ack_vf_disabled, vfs_to_ack, VF_MAX_STATIC / 8); - + OSAL_MEM_ZERO(&mb_params, sizeof(mb_params)); + mb_params.cmd = DRV_MSG_CODE_VF_DISABLED_DONE; + mb_params.p_data_src = vfs_to_ack; + mb_params.data_src_size = VF_MAX_STATIC / 8; rc = ecore_mcp_cmd_and_union(p_hwfn, p_ptt, - DRV_MSG_CODE_VF_DISABLED_DONE, 0, - &union_data, &resp, ¶m); + &mb_params); if (rc != ECORE_SUCCESS) { - DP_NOTICE(p_hwfn, (ECORE_MSG_SP | ECORE_MSG_IOV), + DP_NOTICE(p_hwfn, false, "Failed to pass ACK for VF flr to MFW\n"); return ECORE_TIMEOUT; } @@ -577,25 +1205,78 @@ static void ecore_mcp_handle_transceiver_change(struct ecore_hwfn *p_hwfn, DP_VERBOSE(p_hwfn, (ECORE_MSG_HW | ECORE_MSG_SP), "Received transceiver state update [0x%08x] from mfw" - "[Addr 0x%x]\n", + " [Addr 0x%x]\n", transceiver_state, (u32)(p_hwfn->mcp_info->port_addr + OFFSETOF(struct public_port, transceiver_data))); - transceiver_state = GET_FIELD(transceiver_state, PMM_TRANSCEIVER_STATE); + transceiver_state = GET_MFW_FIELD(transceiver_state, + ETH_TRANSCEIVER_STATE); - if (transceiver_state == PMM_TRANSCEIVER_STATE_PRESENT) + if (transceiver_state == ETH_TRANSCEIVER_STATE_PRESENT) DP_NOTICE(p_hwfn, false, "Transceiver is present.\n"); else DP_NOTICE(p_hwfn, false, "Transceiver is unplugged.\n"); + + OSAL_TRANSCEIVER_UPDATE(p_hwfn); +} + +static void ecore_mcp_read_eee_config(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + struct ecore_mcp_link_state *p_link) +{ + u32 eee_status, val; + + p_link->eee_adv_caps = 0; + p_link->eee_lp_adv_caps = 0; + eee_status = ecore_rd(p_hwfn, p_ptt, p_hwfn->mcp_info->port_addr + + OFFSETOF(struct public_port, eee_status)); + p_link->eee_active = !!(eee_status & EEE_ACTIVE_BIT); + val = (eee_status & EEE_LD_ADV_STATUS_MASK) >> EEE_LD_ADV_STATUS_OFFSET; + if (val & EEE_1G_ADV) + p_link->eee_adv_caps |= ECORE_EEE_1G_ADV; + if (val & EEE_10G_ADV) + p_link->eee_adv_caps |= ECORE_EEE_10G_ADV; + val = (eee_status & EEE_LP_ADV_STATUS_MASK) >> EEE_LP_ADV_STATUS_OFFSET; + if (val & EEE_1G_ADV) + p_link->eee_lp_adv_caps |= ECORE_EEE_1G_ADV; + if (val & EEE_10G_ADV) + p_link->eee_lp_adv_caps |= ECORE_EEE_10G_ADV; +} + +static u32 ecore_mcp_get_shmem_func(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + struct public_func *p_data, + int pfid) +{ + u32 addr = SECTION_OFFSIZE_ADDR(p_hwfn->mcp_info->public_base, + PUBLIC_FUNC); + u32 mfw_path_offsize = ecore_rd(p_hwfn, p_ptt, addr); + u32 func_addr = SECTION_ADDR(mfw_path_offsize, pfid); + u32 i, size; + + OSAL_MEM_ZERO(p_data, sizeof(*p_data)); + + size = OSAL_MIN_T(u32, sizeof(*p_data), + SECTION_SIZE(mfw_path_offsize)); + for (i = 0; i < size / sizeof(u32); i++) + ((u32 *)p_data)[i] = ecore_rd(p_hwfn, p_ptt, + func_addr + (i << 2)); + + return size; } static void ecore_mcp_handle_link_change(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, bool b_reset) + struct ecore_ptt *p_ptt, + bool b_reset) { struct ecore_mcp_link_state *p_link; + u8 max_bw, min_bw; u32 status = 0; + /* Prevent SW/attentions from doing this at the same time */ + OSAL_SPIN_LOCK(&p_hwfn->mcp_info->link_lock); + p_link = &p_hwfn->mcp_info->link_output; OSAL_MEMSET(p_link, 0, sizeof(*p_link)); if (!b_reset) { @@ -611,13 +1292,27 @@ static void ecore_mcp_handle_link_change(struct ecore_hwfn *p_hwfn, } else { DP_VERBOSE(p_hwfn, ECORE_MSG_LINK, "Resetting link indications\n"); - return; + goto out; } - if (p_hwfn->b_drv_link_init) - p_link->link_up = !!(status & LINK_STATUS_LINK_UP); - else + if (p_hwfn->b_drv_link_init) { + /* Link indication with modern MFW arrives as per-PF + * indication. + */ + if (p_hwfn->mcp_info->capabilities & + FW_MB_PARAM_FEATURE_SUPPORT_VLINK) { + struct public_func shmem_info; + + ecore_mcp_get_shmem_func(p_hwfn, p_ptt, &shmem_info, + MCP_PF_ID(p_hwfn)); + p_link->link_up = !!(shmem_info.status & + FUNC_STATUS_VIRTUAL_LINK_UP); + } else { + p_link->link_up = !!(status & LINK_STATUS_LINK_UP); + } + } else { p_link->link_up = false; + } p_link->full_duplex = true; switch ((status & LINK_STATUS_SPEED_AND_DUPLEX_MASK)) { @@ -657,23 +1352,18 @@ static void ecore_mcp_handle_link_change(struct ecore_hwfn *p_hwfn, else p_link->line_speed = 0; - /* Correct speed according to bandwidth allocation */ - if (p_hwfn->mcp_info->func_info.bandwidth_max && p_link->speed) { - u8 max_bw = p_hwfn->mcp_info->func_info.bandwidth_max; + max_bw = p_hwfn->mcp_info->func_info.bandwidth_max; + min_bw = p_hwfn->mcp_info->func_info.bandwidth_min; - __ecore_configure_pf_max_bandwidth(p_hwfn, p_ptt, - p_link, max_bw); - } + /* Max bandwidth configuration */ + __ecore_configure_pf_max_bandwidth(p_hwfn, p_ptt, + p_link, max_bw); - if (p_hwfn->mcp_info->func_info.bandwidth_min && p_link->speed) { - u8 min_bw = p_hwfn->mcp_info->func_info.bandwidth_min; - - __ecore_configure_pf_min_bandwidth(p_hwfn, p_ptt, - p_link, min_bw); - - ecore_configure_vp_wfq_on_link_change(p_hwfn->p_dev, - p_link->min_pf_rate); - } + /* Min bandwidth configuration */ + __ecore_configure_pf_min_bandwidth(p_hwfn, p_ptt, + p_link, min_bw); + ecore_configure_vp_wfq_on_link_change(p_hwfn->p_dev, p_ptt, + p_link->min_pf_rate); p_link->an = !!(status & LINK_STATUS_AUTO_NEGOTIATE_ENABLED); p_link->an_complete = !!(status & LINK_STATUS_AUTO_NEGOTIATE_COMPLETE); @@ -727,20 +1417,22 @@ static void ecore_mcp_handle_link_change(struct ecore_hwfn *p_hwfn, p_link->sfp_tx_fault = !!(status & LINK_STATUS_SFP_TX_FAULT); - if (p_link->link_up) - ecore_dcbx_eagle_workaround(p_hwfn, p_ptt, p_link->pfc_enabled); + if (p_hwfn->mcp_info->capabilities & FW_MB_PARAM_FEATURE_SUPPORT_EEE) + ecore_mcp_read_eee_config(p_hwfn, p_ptt, p_link); OSAL_LINK_UPDATE(p_hwfn); +out: + OSAL_SPIN_UNLOCK(&p_hwfn->mcp_info->link_lock); } enum _ecore_status_t ecore_mcp_set_link(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, bool b_up) { struct ecore_mcp_link_params *params = &p_hwfn->mcp_info->link_input; - union drv_union_data union_data; - struct pmm_phy_cfg *p_phy_cfg; - u32 param = 0, reply = 0, cmd; + struct ecore_mcp_mb_params mb_params; + struct eth_phy_cfg phy_cfg; enum _ecore_status_t rc = ECORE_SUCCESS; + u32 cmd; #ifndef ASIC_ONLY if (CHIP_REV_IS_EMUL(p_hwfn->p_dev)) @@ -748,41 +1440,51 @@ enum _ecore_status_t ecore_mcp_set_link(struct ecore_hwfn *p_hwfn, #endif /* Set the shmem configuration according to params */ - p_phy_cfg = &union_data.drv_phy_cfg; - OSAL_MEMSET(p_phy_cfg, 0, sizeof(*p_phy_cfg)); + OSAL_MEM_ZERO(&phy_cfg, sizeof(phy_cfg)); cmd = b_up ? DRV_MSG_CODE_INIT_PHY : DRV_MSG_CODE_LINK_RESET; if (!params->speed.autoneg) - p_phy_cfg->speed = params->speed.forced_speed; - p_phy_cfg->pause |= (params->pause.autoneg) ? PMM_PAUSE_AUTONEG : 0; - p_phy_cfg->pause |= (params->pause.forced_rx) ? PMM_PAUSE_RX : 0; - p_phy_cfg->pause |= (params->pause.forced_tx) ? PMM_PAUSE_TX : 0; - p_phy_cfg->adv_speed = params->speed.advertised_speeds; - p_phy_cfg->loopback_mode = params->loopback_mode; + phy_cfg.speed = params->speed.forced_speed; + phy_cfg.pause |= (params->pause.autoneg) ? ETH_PAUSE_AUTONEG : 0; + phy_cfg.pause |= (params->pause.forced_rx) ? ETH_PAUSE_RX : 0; + phy_cfg.pause |= (params->pause.forced_tx) ? ETH_PAUSE_TX : 0; + phy_cfg.adv_speed = params->speed.advertised_speeds; + phy_cfg.loopback_mode = params->loopback_mode; -#ifndef ASIC_ONLY - if (CHIP_REV_IS_FPGA(p_hwfn->p_dev)) { - DP_INFO(p_hwfn, - "Link on FPGA - Ask for loopback mode '5' at 10G\n"); - p_phy_cfg->loopback_mode = 5; - p_phy_cfg->speed = 10000; + /* There are MFWs that share this capability regardless of whether + * this is feasible or not. And given that at the very least adv_caps + * would be set internally by ecore, we want to make sure LFA would + * still work. + */ + if ((p_hwfn->mcp_info->capabilities & + FW_MB_PARAM_FEATURE_SUPPORT_EEE) && + params->eee.enable) { + phy_cfg.eee_cfg |= EEE_CFG_EEE_ENABLED; + if (params->eee.tx_lpi_enable) + phy_cfg.eee_cfg |= EEE_CFG_TX_LPI; + if (params->eee.adv_caps & ECORE_EEE_1G_ADV) + phy_cfg.eee_cfg |= EEE_CFG_ADV_SPEED_1G; + if (params->eee.adv_caps & ECORE_EEE_10G_ADV) + phy_cfg.eee_cfg |= EEE_CFG_ADV_SPEED_10G; + phy_cfg.eee_cfg |= (params->eee.tx_lpi_timer << + EEE_TX_TIMER_USEC_OFFSET) & + EEE_TX_TIMER_USEC_MASK; } -#endif p_hwfn->b_drv_link_init = b_up; if (b_up) DP_VERBOSE(p_hwfn, ECORE_MSG_LINK, - "Configuring Link: Speed 0x%08x, Pause 0x%08x," - " adv_speed 0x%08x, loopback 0x%08x," - " features 0x%08x\n", - p_phy_cfg->speed, p_phy_cfg->pause, - p_phy_cfg->adv_speed, p_phy_cfg->loopback_mode, - p_phy_cfg->feature_config_flags); + "Configuring Link: Speed 0x%08x, Pause 0x%08x, adv_speed 0x%08x, loopback 0x%08x\n", + phy_cfg.speed, phy_cfg.pause, phy_cfg.adv_speed, + phy_cfg.loopback_mode); else DP_VERBOSE(p_hwfn, ECORE_MSG_LINK, "Resetting link\n"); - rc = ecore_mcp_cmd_and_union(p_hwfn, p_ptt, cmd, 0, &union_data, &reply, - ¶m); + OSAL_MEM_ZERO(&mb_params, sizeof(mb_params)); + mb_params.cmd = cmd; + mb_params.p_data_src = &phy_cfg; + mb_params.data_src_size = sizeof(phy_cfg); + rc = ecore_mcp_cmd_and_union(p_hwfn, p_ptt, &mb_params); /* if mcp fails to respond we must abort */ if (rc != ECORE_SUCCESS) { @@ -790,11 +1492,15 @@ enum _ecore_status_t ecore_mcp_set_link(struct ecore_hwfn *p_hwfn, return rc; } - /* Reset the link status if needed */ - if (!b_up) - ecore_mcp_handle_link_change(p_hwfn, p_ptt, true); + /* Mimic link-change attention, done for several reasons: + * - On reset, there's no guarantee MFW would trigger + * an attention. + * - On initialization, older MFWs might not indicate link change + * during LFA, so we'll never get an UP indication. + */ + ecore_mcp_handle_link_change(p_hwfn, p_ptt, !b_up); - return rc; + return ECORE_SUCCESS; } u32 ecore_get_process_kill_counter(struct ecore_hwfn *p_hwfn, @@ -859,8 +1565,9 @@ static void ecore_mcp_send_protocol_stats(struct ecore_hwfn *p_hwfn, { enum ecore_mcp_protocol_type stats_type; union ecore_mcp_protocol_stats stats; - u32 hsi_param, param = 0, reply = 0; - union drv_union_data union_data; + struct ecore_mcp_mb_params mb_params; + u32 hsi_param; + enum _ecore_status_t rc; switch (type) { case MFW_DRV_MSG_GET_LAN_STATS: @@ -868,41 +1575,25 @@ static void ecore_mcp_send_protocol_stats(struct ecore_hwfn *p_hwfn, hsi_param = DRV_MSG_CODE_STATS_TYPE_LAN; break; default: - DP_NOTICE(p_hwfn, false, "Invalid protocol type %d\n", type); + DP_VERBOSE(p_hwfn, ECORE_MSG_SP, + "Invalid protocol type %d\n", type); return; } OSAL_GET_PROTOCOL_STATS(p_hwfn->p_dev, stats_type, &stats); - OSAL_MEMCPY(&union_data, &stats, sizeof(stats)); - - ecore_mcp_cmd_and_union(p_hwfn, p_ptt, DRV_MSG_CODE_GET_STATS, - hsi_param, &union_data, &reply, ¶m); + OSAL_MEM_ZERO(&mb_params, sizeof(mb_params)); + mb_params.cmd = DRV_MSG_CODE_GET_STATS; + mb_params.param = hsi_param; + mb_params.p_data_src = &stats; + mb_params.data_src_size = sizeof(stats); + rc = ecore_mcp_cmd_and_union(p_hwfn, p_ptt, &mb_params); + if (rc != ECORE_SUCCESS) + DP_ERR(p_hwfn, "Failed to send protocol stats, rc = %d\n", rc); } -static u32 ecore_mcp_get_shmem_func(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, - struct public_func *p_data, int pfid) -{ - u32 addr = SECTION_OFFSIZE_ADDR(p_hwfn->mcp_info->public_base, - PUBLIC_FUNC); - u32 mfw_path_offsize = ecore_rd(p_hwfn, p_ptt, addr); - u32 func_addr = SECTION_ADDR(mfw_path_offsize, pfid); - u32 i, size; - - OSAL_MEM_ZERO(p_data, sizeof(*p_data)); - - size = OSAL_MIN_T(u32, sizeof(*p_data), SECTION_SIZE(mfw_path_offsize)); - for (i = 0; i < size / sizeof(u32); i++) - ((u32 *)p_data)[i] = ecore_rd(p_hwfn, p_ptt, - func_addr + (i << 2)); - - return size; -} - -static void -ecore_read_pf_bandwidth(struct ecore_hwfn *p_hwfn, - struct public_func *p_shmem_info) +static void ecore_read_pf_bandwidth(struct ecore_hwfn *p_hwfn, + struct public_func *p_shmem_info) { struct ecore_mcp_function_info *p_info; @@ -916,7 +1607,7 @@ ecore_read_pf_bandwidth(struct ecore_hwfn *p_hwfn, */ p_info->bandwidth_min = (p_shmem_info->config & FUNC_MF_CFG_MIN_BW_MASK) >> - FUNC_MF_CFG_MIN_BW_SHIFT; + FUNC_MF_CFG_MIN_BW_OFFSET; if (p_info->bandwidth_min < 1 || p_info->bandwidth_min > 100) { DP_INFO(p_hwfn, "bandwidth minimum out of bounds [%02x]. Set to 1\n", @@ -926,7 +1617,7 @@ ecore_read_pf_bandwidth(struct ecore_hwfn *p_hwfn, p_info->bandwidth_max = (p_shmem_info->config & FUNC_MF_CFG_MAX_BW_MASK) >> - FUNC_MF_CFG_MAX_BW_SHIFT; + FUNC_MF_CFG_MAX_BW_OFFSET; if (p_info->bandwidth_max < 1 || p_info->bandwidth_max > 100) { DP_INFO(p_hwfn, "bandwidth maximum out of bounds [%02x]. Set to 100\n", @@ -957,8 +1648,7 @@ ecore_mcp_update_bw(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt) ¶m); } -static void ecore_mcp_handle_fan_failure(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt) +static void ecore_mcp_handle_fan_failure(struct ecore_hwfn *p_hwfn) { /* A single notification should be sent to upper driver in CMT mode */ if (p_hwfn != ECORE_LEADING_HWFN(p_hwfn->p_dev)) @@ -971,6 +1661,311 @@ static void ecore_mcp_handle_fan_failure(struct ecore_hwfn *p_hwfn, ecore_hw_err_notify(p_hwfn, ECORE_HW_ERR_FAN_FAIL); } +struct ecore_mdump_cmd_params { + u32 cmd; + void *p_data_src; + u8 data_src_size; + void *p_data_dst; + u8 data_dst_size; + u32 mcp_resp; +}; + +static enum _ecore_status_t +ecore_mcp_mdump_cmd(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, + struct ecore_mdump_cmd_params *p_mdump_cmd_params) +{ + struct ecore_mcp_mb_params mb_params; + enum _ecore_status_t rc; + + OSAL_MEM_ZERO(&mb_params, sizeof(mb_params)); + mb_params.cmd = DRV_MSG_CODE_MDUMP_CMD; + mb_params.param = p_mdump_cmd_params->cmd; + mb_params.p_data_src = p_mdump_cmd_params->p_data_src; + mb_params.data_src_size = p_mdump_cmd_params->data_src_size; + mb_params.p_data_dst = p_mdump_cmd_params->p_data_dst; + mb_params.data_dst_size = p_mdump_cmd_params->data_dst_size; + rc = ecore_mcp_cmd_and_union(p_hwfn, p_ptt, &mb_params); + if (rc != ECORE_SUCCESS) + return rc; + + p_mdump_cmd_params->mcp_resp = mb_params.mcp_resp; + + if (p_mdump_cmd_params->mcp_resp == FW_MSG_CODE_MDUMP_INVALID_CMD) { + DP_INFO(p_hwfn, + "The mdump sub command is unsupported by the MFW [mdump_cmd 0x%x]\n", + p_mdump_cmd_params->cmd); + rc = ECORE_NOTIMPL; + } else if (p_mdump_cmd_params->mcp_resp == FW_MSG_CODE_UNSUPPORTED) { + DP_INFO(p_hwfn, + "The mdump command is not supported by the MFW\n"); + rc = ECORE_NOTIMPL; + } + + return rc; +} + +static enum _ecore_status_t ecore_mcp_mdump_ack(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt) +{ + struct ecore_mdump_cmd_params mdump_cmd_params; + + OSAL_MEM_ZERO(&mdump_cmd_params, sizeof(mdump_cmd_params)); + mdump_cmd_params.cmd = DRV_MSG_CODE_MDUMP_ACK; + + return ecore_mcp_mdump_cmd(p_hwfn, p_ptt, &mdump_cmd_params); +} + +enum _ecore_status_t ecore_mcp_mdump_set_values(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + u32 epoch) +{ + struct ecore_mdump_cmd_params mdump_cmd_params; + + OSAL_MEM_ZERO(&mdump_cmd_params, sizeof(mdump_cmd_params)); + mdump_cmd_params.cmd = DRV_MSG_CODE_MDUMP_SET_VALUES; + mdump_cmd_params.p_data_src = &epoch; + mdump_cmd_params.data_src_size = sizeof(epoch); + + return ecore_mcp_mdump_cmd(p_hwfn, p_ptt, &mdump_cmd_params); +} + +enum _ecore_status_t ecore_mcp_mdump_trigger(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt) +{ + struct ecore_mdump_cmd_params mdump_cmd_params; + + OSAL_MEM_ZERO(&mdump_cmd_params, sizeof(mdump_cmd_params)); + mdump_cmd_params.cmd = DRV_MSG_CODE_MDUMP_TRIGGER; + + return ecore_mcp_mdump_cmd(p_hwfn, p_ptt, &mdump_cmd_params); +} + +static enum _ecore_status_t +ecore_mcp_mdump_get_config(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, + struct mdump_config_stc *p_mdump_config) +{ + struct ecore_mdump_cmd_params mdump_cmd_params; + enum _ecore_status_t rc; + + OSAL_MEM_ZERO(&mdump_cmd_params, sizeof(mdump_cmd_params)); + mdump_cmd_params.cmd = DRV_MSG_CODE_MDUMP_GET_CONFIG; + mdump_cmd_params.p_data_dst = p_mdump_config; + mdump_cmd_params.data_dst_size = sizeof(*p_mdump_config); + + rc = ecore_mcp_mdump_cmd(p_hwfn, p_ptt, &mdump_cmd_params); + if (rc != ECORE_SUCCESS) + return rc; + + if (mdump_cmd_params.mcp_resp != FW_MSG_CODE_OK) { + DP_INFO(p_hwfn, + "Failed to get the mdump configuration and logs info [mcp_resp 0x%x]\n", + mdump_cmd_params.mcp_resp); + rc = ECORE_UNKNOWN_ERROR; + } + + return rc; +} + +enum _ecore_status_t +ecore_mcp_mdump_get_info(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, + struct ecore_mdump_info *p_mdump_info) +{ + u32 addr, global_offsize, global_addr; + struct mdump_config_stc mdump_config; + enum _ecore_status_t rc; + + OSAL_MEMSET(p_mdump_info, 0, sizeof(*p_mdump_info)); + + addr = SECTION_OFFSIZE_ADDR(p_hwfn->mcp_info->public_base, + PUBLIC_GLOBAL); + global_offsize = ecore_rd(p_hwfn, p_ptt, addr); + global_addr = SECTION_ADDR(global_offsize, 0); + p_mdump_info->reason = ecore_rd(p_hwfn, p_ptt, + global_addr + + OFFSETOF(struct public_global, + mdump_reason)); + + if (p_mdump_info->reason) { + rc = ecore_mcp_mdump_get_config(p_hwfn, p_ptt, &mdump_config); + if (rc != ECORE_SUCCESS) + return rc; + + p_mdump_info->version = mdump_config.version; + p_mdump_info->config = mdump_config.config; + p_mdump_info->epoch = mdump_config.epoc; + p_mdump_info->num_of_logs = mdump_config.num_of_logs; + p_mdump_info->valid_logs = mdump_config.valid_logs; + + DP_VERBOSE(p_hwfn, ECORE_MSG_SP, + "MFW mdump info: reason %d, version 0x%x, config 0x%x, epoch 0x%x, num_of_logs 0x%x, valid_logs 0x%x\n", + p_mdump_info->reason, p_mdump_info->version, + p_mdump_info->config, p_mdump_info->epoch, + p_mdump_info->num_of_logs, p_mdump_info->valid_logs); + } else { + DP_VERBOSE(p_hwfn, ECORE_MSG_SP, + "MFW mdump info: reason %d\n", p_mdump_info->reason); + } + + return ECORE_SUCCESS; +} + +enum _ecore_status_t ecore_mcp_mdump_clear_logs(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt) +{ + struct ecore_mdump_cmd_params mdump_cmd_params; + + OSAL_MEM_ZERO(&mdump_cmd_params, sizeof(mdump_cmd_params)); + mdump_cmd_params.cmd = DRV_MSG_CODE_MDUMP_CLEAR_LOGS; + + return ecore_mcp_mdump_cmd(p_hwfn, p_ptt, &mdump_cmd_params); +} + +enum _ecore_status_t +ecore_mcp_mdump_get_retain(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, + struct ecore_mdump_retain_data *p_mdump_retain) +{ + struct ecore_mdump_cmd_params mdump_cmd_params; + struct mdump_retain_data_stc mfw_mdump_retain; + enum _ecore_status_t rc; + + OSAL_MEM_ZERO(&mdump_cmd_params, sizeof(mdump_cmd_params)); + mdump_cmd_params.cmd = DRV_MSG_CODE_MDUMP_GET_RETAIN; + mdump_cmd_params.p_data_dst = &mfw_mdump_retain; + mdump_cmd_params.data_dst_size = sizeof(mfw_mdump_retain); + + rc = ecore_mcp_mdump_cmd(p_hwfn, p_ptt, &mdump_cmd_params); + if (rc != ECORE_SUCCESS) + return rc; + + if (mdump_cmd_params.mcp_resp != FW_MSG_CODE_OK) { + DP_INFO(p_hwfn, + "Failed to get the mdump retained data [mcp_resp 0x%x]\n", + mdump_cmd_params.mcp_resp); + return ECORE_UNKNOWN_ERROR; + } + + p_mdump_retain->valid = mfw_mdump_retain.valid; + p_mdump_retain->epoch = mfw_mdump_retain.epoch; + p_mdump_retain->pf = mfw_mdump_retain.pf; + p_mdump_retain->status = mfw_mdump_retain.status; + + return ECORE_SUCCESS; +} + +enum _ecore_status_t ecore_mcp_mdump_clr_retain(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt) +{ + struct ecore_mdump_cmd_params mdump_cmd_params; + + OSAL_MEM_ZERO(&mdump_cmd_params, sizeof(mdump_cmd_params)); + mdump_cmd_params.cmd = DRV_MSG_CODE_MDUMP_CLR_RETAIN; + + return ecore_mcp_mdump_cmd(p_hwfn, p_ptt, &mdump_cmd_params); +} + +static void ecore_mcp_handle_critical_error(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt) +{ + struct ecore_mdump_retain_data mdump_retain; + enum _ecore_status_t rc; + + /* In CMT mode - no need for more than a single acknowledgment to the + * MFW, and no more than a single notification to the upper driver. + */ + if (p_hwfn != ECORE_LEADING_HWFN(p_hwfn->p_dev)) + return; + + rc = ecore_mcp_mdump_get_retain(p_hwfn, p_ptt, &mdump_retain); + if (rc == ECORE_SUCCESS && mdump_retain.valid) { + DP_NOTICE(p_hwfn, false, + "The MFW notified that a critical error occurred in the device [epoch 0x%08x, pf 0x%x, status 0x%08x]\n", + mdump_retain.epoch, mdump_retain.pf, + mdump_retain.status); + } else { + DP_NOTICE(p_hwfn, false, + "The MFW notified that a critical error occurred in the device\n"); + } + + if (p_hwfn->p_dev->allow_mdump) { + DP_NOTICE(p_hwfn, false, + "Not acknowledging the notification to allow the MFW crash dump\n"); + return; + } + + DP_NOTICE(p_hwfn, false, + "Acknowledging the notification to not allow the MFW crash dump [driver debug data collection is preferable]\n"); + ecore_mcp_mdump_ack(p_hwfn, p_ptt); + ecore_hw_err_notify(p_hwfn, ECORE_HW_ERR_HW_ATTN); +} + +void +ecore_mcp_read_ufp_config(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt) +{ + struct public_func shmem_info; + u32 port_cfg, val; + + if (!OSAL_TEST_BIT(ECORE_MF_UFP_SPECIFIC, &p_hwfn->p_dev->mf_bits)) + return; + + OSAL_MEMSET(&p_hwfn->ufp_info, 0, sizeof(p_hwfn->ufp_info)); + port_cfg = ecore_rd(p_hwfn, p_ptt, p_hwfn->mcp_info->port_addr + + OFFSETOF(struct public_port, oem_cfg_port)); + val = GET_MFW_FIELD(port_cfg, OEM_CFG_CHANNEL_TYPE); + if (val != OEM_CFG_CHANNEL_TYPE_STAGGED) + DP_NOTICE(p_hwfn, false, "Incorrect UFP Channel type %d\n", + val); + + val = GET_MFW_FIELD(port_cfg, OEM_CFG_SCHED_TYPE); + if (val == OEM_CFG_SCHED_TYPE_ETS) + p_hwfn->ufp_info.mode = ECORE_UFP_MODE_ETS; + else if (val == OEM_CFG_SCHED_TYPE_VNIC_BW) + p_hwfn->ufp_info.mode = ECORE_UFP_MODE_VNIC_BW; + else + DP_NOTICE(p_hwfn, false, "Unknown UFP scheduling mode %d\n", + val); + + ecore_mcp_get_shmem_func(p_hwfn, p_ptt, &shmem_info, + MCP_PF_ID(p_hwfn)); + val = GET_MFW_FIELD(shmem_info.oem_cfg_func, OEM_CFG_FUNC_TC); + p_hwfn->ufp_info.tc = (u8)val; + val = GET_MFW_FIELD(shmem_info.oem_cfg_func, + OEM_CFG_FUNC_HOST_PRI_CTRL); + if (val == OEM_CFG_FUNC_HOST_PRI_CTRL_VNIC) + p_hwfn->ufp_info.pri_type = ECORE_UFP_PRI_VNIC; + else if (val == OEM_CFG_FUNC_HOST_PRI_CTRL_OS) + p_hwfn->ufp_info.pri_type = ECORE_UFP_PRI_OS; + else + DP_NOTICE(p_hwfn, false, "Unknown Host priority control %d\n", + val); + + DP_VERBOSE(p_hwfn, ECORE_MSG_DCB, + "UFP shmem config: mode = %d tc = %d pri_type = %d\n", + p_hwfn->ufp_info.mode, p_hwfn->ufp_info.tc, + p_hwfn->ufp_info.pri_type); +} + +static enum _ecore_status_t +ecore_mcp_handle_ufp_event(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt) +{ + ecore_mcp_read_ufp_config(p_hwfn, p_ptt); + + if (p_hwfn->ufp_info.mode == ECORE_UFP_MODE_VNIC_BW) { + p_hwfn->qm_info.ooo_tc = p_hwfn->ufp_info.tc; + p_hwfn->hw_info.offload_tc = p_hwfn->ufp_info.tc; + + ecore_qm_reconf(p_hwfn, p_ptt); + } else { + /* Merge UFP TC with the dcbx TC data */ + ecore_dcbx_mib_update_event(p_hwfn, p_ptt, + ECORE_DCBX_OPERATIONAL_MIB); + } + + /* update storm FW with negotiation results */ + ecore_sp_pf_update_ufp(p_hwfn); + + return ECORE_SUCCESS; +} + enum _ecore_status_t ecore_mcp_handle_events(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt) { @@ -1013,6 +2008,18 @@ enum _ecore_status_t ecore_mcp_handle_events(struct ecore_hwfn *p_hwfn, case MFW_DRV_MSG_DCBX_OPERATIONAL_MIB_UPDATED: ecore_dcbx_mib_update_event(p_hwfn, p_ptt, ECORE_DCBX_OPERATIONAL_MIB); + /* clear the user-config cache */ + OSAL_MEMSET(&p_hwfn->p_dcbx_info->set, 0, + sizeof(struct ecore_dcbx_set)); + break; + case MFW_DRV_MSG_LLDP_RECEIVED_TLVS_UPDATED: + ecore_lldp_mib_update_event(p_hwfn, p_ptt); + break; + case MFW_DRV_MSG_OEM_CFG_UPDATE: + ecore_mcp_handle_ufp_event(p_hwfn, p_ptt); + break; + case MFW_DRV_MSG_TRANSCEIVER_STATE_CHANGE: + ecore_mcp_handle_transceiver_change(p_hwfn, p_ptt); break; case MFW_DRV_MSG_ERROR_RECOVERY: ecore_mcp_handle_process_kill(p_hwfn, p_ptt); @@ -1026,16 +2033,14 @@ enum _ecore_status_t ecore_mcp_handle_events(struct ecore_hwfn *p_hwfn, case MFW_DRV_MSG_BW_UPDATE: ecore_mcp_update_bw(p_hwfn, p_ptt); break; - case MFW_DRV_MSG_TRANSCEIVER_STATE_CHANGE: - ecore_mcp_handle_transceiver_change(p_hwfn, p_ptt); - break; case MFW_DRV_MSG_FAILURE_DETECTED: - ecore_mcp_handle_fan_failure(p_hwfn, p_ptt); + ecore_mcp_handle_fan_failure(p_hwfn); + break; + case MFW_DRV_MSG_CRITICAL_ERROR_OCCURRED: + ecore_mcp_handle_critical_error(p_hwfn, p_ptt); break; default: - /* @DPDK */ - DP_NOTICE(p_hwfn, false, - "Unimplemented MFW message %d\n", i); + DP_INFO(p_hwfn, "Unimplemented MFW message %d\n", i); rc = ECORE_INVAL; } } @@ -1064,33 +2069,32 @@ enum _ecore_status_t ecore_mcp_handle_events(struct ecore_hwfn *p_hwfn, return rc; } -enum _ecore_status_t ecore_mcp_get_mfw_ver(struct ecore_dev *p_dev, +enum _ecore_status_t ecore_mcp_get_mfw_ver(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, u32 *p_mfw_ver, u32 *p_running_bundle_id) { - struct ecore_hwfn *p_hwfn = ECORE_LEADING_HWFN(p_dev); u32 global_offsize; #ifndef ASIC_ONLY - if (CHIP_REV_IS_EMUL(p_dev)) { - DP_NOTICE(p_dev, false, "Emulation - can't get MFW version\n"); + if (CHIP_REV_IS_EMUL(p_hwfn->p_dev)) { + DP_NOTICE(p_hwfn, false, "Emulation - can't get MFW version\n"); return ECORE_SUCCESS; } #endif - if (IS_VF(p_dev)) { + if (IS_VF(p_hwfn->p_dev)) { if (p_hwfn->vf_iov_info) { struct pfvf_acquire_resp_tlv *p_resp; p_resp = &p_hwfn->vf_iov_info->acquire_resp; *p_mfw_ver = p_resp->pfdev_info.mfw_ver; return ECORE_SUCCESS; + } else { + DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, + "VF requested MFW version prior to ACQUIRE\n"); + return ECORE_INVAL; } - - DP_VERBOSE(p_dev, ECORE_MSG_IOV, - "VF requested MFW vers prior to ACQUIRE\n"); - return ECORE_INVAL; } global_offsize = ecore_rd(p_hwfn, p_ptt, @@ -1113,14 +2117,13 @@ enum _ecore_status_t ecore_mcp_get_mfw_ver(struct ecore_dev *p_dev, return ECORE_SUCCESS; } -enum _ecore_status_t ecore_mcp_get_media_type(struct ecore_dev *p_dev, +enum _ecore_status_t ecore_mcp_get_media_type(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, u32 *p_media_type) { - struct ecore_hwfn *p_hwfn = &p_dev->hwfns[0]; - struct ecore_ptt *p_ptt; /* TODO - Add support for VFs */ - if (IS_VF(p_dev)) + if (IS_VF(p_hwfn->p_dev)) return ECORE_INVAL; if (!ecore_mcp_is_init(p_hwfn)) { @@ -1128,30 +2131,60 @@ enum _ecore_status_t ecore_mcp_get_media_type(struct ecore_dev *p_dev, return ECORE_BUSY; } - *p_media_type = MEDIA_UNSPECIFIED; + if (!p_ptt) { + *p_media_type = MEDIA_UNSPECIFIED; + return ECORE_INVAL; + } else { + *p_media_type = ecore_rd(p_hwfn, p_ptt, + p_hwfn->mcp_info->port_addr + + OFFSETOF(struct public_port, + media_type)); + } - p_ptt = ecore_ptt_acquire(p_hwfn); - if (!p_ptt) - return ECORE_BUSY; + return ECORE_SUCCESS; +} - *p_media_type = ecore_rd(p_hwfn, p_ptt, p_hwfn->mcp_info->port_addr + - OFFSETOF(struct public_port, media_type)); +/* @DPDK */ +/* Old MFW has a global configuration for all PFs regarding RDMA support */ +static void +ecore_mcp_get_shmem_proto_legacy(struct ecore_hwfn *p_hwfn, + enum ecore_pci_personality *p_proto) +{ + *p_proto = ECORE_PCI_ETH; - ecore_ptt_release(p_hwfn, p_ptt); + DP_VERBOSE(p_hwfn, ECORE_MSG_IFUP, + "According to Legacy capabilities, L2 personality is %08x\n", + (u32)*p_proto); +} +/* @DPDK */ +static enum _ecore_status_t +ecore_mcp_get_shmem_proto_mfw(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + enum ecore_pci_personality *p_proto) +{ + u32 resp = 0, param = 0; + enum _ecore_status_t rc; + + DP_VERBOSE(p_hwfn, ECORE_MSG_IFUP, + "According to capabilities, L2 personality is %08x [resp %08x param %08x]\n", + (u32)*p_proto, resp, param); return ECORE_SUCCESS; } static enum _ecore_status_t ecore_mcp_get_shmem_proto(struct ecore_hwfn *p_hwfn, struct public_func *p_info, + struct ecore_ptt *p_ptt, enum ecore_pci_personality *p_proto) { enum _ecore_status_t rc = ECORE_SUCCESS; switch (p_info->config & FUNC_MF_CFG_PROTOCOL_MASK) { case FUNC_MF_CFG_PROTOCOL_ETHERNET: - *p_proto = ECORE_PCI_ETH; + if (ecore_mcp_get_shmem_proto_mfw(p_hwfn, p_ptt, p_proto) != + ECORE_SUCCESS) + ecore_mcp_get_shmem_proto_legacy(p_hwfn, p_proto); break; default: rc = ECORE_INVAL; @@ -1172,7 +2205,8 @@ enum _ecore_status_t ecore_mcp_fill_shmem_func_info(struct ecore_hwfn *p_hwfn, info->pause_on_host = (shmem_info.config & FUNC_MF_CFG_PAUSE_ON_HOST_RING) ? 1 : 0; - if (ecore_mcp_get_shmem_proto(p_hwfn, &shmem_info, &info->protocol)) { + if (ecore_mcp_get_shmem_proto(p_hwfn, &shmem_info, p_ptt, + &info->protocol)) { DP_ERR(p_hwfn, "Unknown personality %08x\n", (u32)(shmem_info.config & FUNC_MF_CFG_PROTOCOL_MASK)); return ECORE_INVAL; @@ -1192,18 +2226,32 @@ enum _ecore_status_t ecore_mcp_fill_shmem_func_info(struct ecore_hwfn *p_hwfn, DP_NOTICE(p_hwfn, false, "MAC is 0 in shmem\n"); } + /* TODO - are these calculations true for BE machine? */ + info->wwn_port = (u64)shmem_info.fcoe_wwn_port_name_upper | + (((u64)shmem_info.fcoe_wwn_port_name_lower) << 32); + info->wwn_node = (u64)shmem_info.fcoe_wwn_node_name_upper | + (((u64)shmem_info.fcoe_wwn_node_name_lower) << 32); + info->ovlan = (u16)(shmem_info.ovlan_stag & FUNC_MF_CFG_OV_STAG_MASK); + info->mtu = (u16)shmem_info.mtu_size; + + if (info->mtu == 0) + info->mtu = 1500; + + info->mtu = (u16)shmem_info.mtu_size; + DP_VERBOSE(p_hwfn, (ECORE_MSG_SP | ECORE_MSG_IFUP), "Read configuration from shmem: pause_on_host %02x" " protocol %02x BW [%02x - %02x]" - " MAC %02x:%02x:%02x:%02x:%02x:%02x wwn port %" PRIx64 - " node %" PRIx64 " ovlan %04x\n", + " MAC %02x:%02x:%02x:%02x:%02x:%02x wwn port %lx" + " node %lx ovlan %04x\n", info->pause_on_host, info->protocol, info->bandwidth_min, info->bandwidth_max, info->mac[0], info->mac[1], info->mac[2], info->mac[3], info->mac[4], info->mac[5], - info->wwn_port, info->wwn_node, info->ovlan); + (unsigned long)info->wwn_port, + (unsigned long)info->wwn_node, info->ovlan); return ECORE_SUCCESS; } @@ -1243,14 +2291,14 @@ struct ecore_mcp_link_capabilities enum _ecore_status_t ecore_mcp_drain(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt) { - enum _ecore_status_t rc; u32 resp = 0, param = 0; + enum _ecore_status_t rc; rc = ecore_mcp_cmd(p_hwfn, p_ptt, - DRV_MSG_CODE_NIG_DRAIN, 100, &resp, ¶m); + DRV_MSG_CODE_NIG_DRAIN, 1000, &resp, ¶m); /* Wait for the drain to complete before returning */ - OSAL_MSLEEP(120); + OSAL_MSLEEP(1020); return rc; } @@ -1263,42 +2311,6 @@ const struct ecore_mcp_function_info return &p_hwfn->mcp_info->func_info; } -enum _ecore_status_t ecore_mcp_nvm_command(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, - struct ecore_mcp_nvm_params *params) -{ - enum _ecore_status_t rc; - - switch (params->type) { - case ECORE_MCP_NVM_RD: - rc = ecore_mcp_nvm_rd_cmd(p_hwfn, p_ptt, params->nvm_common.cmd, - params->nvm_common.offset, - ¶ms->nvm_common.resp, - ¶ms->nvm_common.param, - params->nvm_rd.buf_size, - params->nvm_rd.buf); - break; - case ECORE_MCP_CMD: - rc = ecore_mcp_cmd(p_hwfn, p_ptt, params->nvm_common.cmd, - params->nvm_common.offset, - ¶ms->nvm_common.resp, - ¶ms->nvm_common.param); - break; - case ECORE_MCP_NVM_WR: - rc = ecore_mcp_nvm_wr_cmd(p_hwfn, p_ptt, params->nvm_common.cmd, - params->nvm_common.offset, - ¶ms->nvm_common.resp, - ¶ms->nvm_common.param, - params->nvm_wr.buf_size, - params->nvm_wr.buf); - break; - default: - rc = ECORE_NOTIMPL; - break; - } - return rc; -} - int ecore_mcp_get_personality_cnt(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, u32 personalities) { @@ -1314,8 +2326,9 @@ int ecore_mcp_get_personality_cnt(struct ecore_hwfn *p_hwfn, if (shmem_info.config & FUNC_MF_CFG_FUNC_HIDE) continue; - if (ecore_mcp_get_shmem_proto(p_hwfn, &shmem_info, - &protocol) != ECORE_SUCCESS) + if (ecore_mcp_get_shmem_proto(p_hwfn, &shmem_info, p_ptt, + &protocol) != + ECORE_SUCCESS) continue; if ((1 << ((u32)protocol)) & personalities) @@ -1343,8 +2356,8 @@ enum _ecore_status_t ecore_mcp_get_flash_size(struct ecore_hwfn *p_hwfn, flash_size = ecore_rd(p_hwfn, p_ptt, MCP_REG_NVM_CFG4); flash_size = (flash_size & MCP_REG_NVM_CFG4_FLASH_SIZE) >> - MCP_REG_NVM_CFG4_FLASH_SIZE_SHIFT; - flash_size = (1 << (flash_size + MCP_BYTES_PER_MBIT_SHIFT)); + MCP_REG_NVM_CFG4_FLASH_SIZE_SHIFT; + flash_size = (1 << (flash_size + MCP_BYTES_PER_MBIT_OFFSET)); *p_flash_size = flash_size; @@ -1369,16 +2382,23 @@ enum _ecore_status_t ecore_start_recovery_process(struct ecore_hwfn *p_hwfn, return ECORE_SUCCESS; } -enum _ecore_status_t ecore_mcp_config_vf_msix(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, - u8 vf_id, u8 num) +static enum _ecore_status_t +ecore_mcp_config_vf_msix_bb(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + u8 vf_id, u8 num) { u32 resp = 0, param = 0, rc_param = 0; enum _ecore_status_t rc; - param |= (vf_id << DRV_MB_PARAM_CFG_VF_MSIX_VF_ID_SHIFT) & +/* Only Leader can configure MSIX, and need to take CMT into account */ + + if (!IS_LEAD_HWFN(p_hwfn)) + return ECORE_SUCCESS; + num *= p_hwfn->p_dev->num_hwfns; + + param |= (vf_id << DRV_MB_PARAM_CFG_VF_MSIX_VF_ID_OFFSET) & DRV_MB_PARAM_CFG_VF_MSIX_VF_ID_MASK; - param |= (num << DRV_MB_PARAM_CFG_VF_MSIX_SB_NUM_SHIFT) & + param |= (num << DRV_MB_PARAM_CFG_VF_MSIX_SB_NUM_OFFSET) & DRV_MB_PARAM_CFG_VF_MSIX_SB_NUM_MASK; rc = ecore_mcp_cmd(p_hwfn, p_ptt, DRV_MSG_CODE_CFG_VF_MSIX, param, @@ -1388,18 +2408,55 @@ enum _ecore_status_t ecore_mcp_config_vf_msix(struct ecore_hwfn *p_hwfn, DP_NOTICE(p_hwfn, true, "VF[%d]: MFW failed to set MSI-X\n", vf_id); rc = ECORE_INVAL; + } else { + DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, + "Requested 0x%02x MSI-x interrupts from VF 0x%02x\n", + num, vf_id); } return rc; } +static enum _ecore_status_t +ecore_mcp_config_vf_msix_ah(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + u8 num) +{ + u32 resp = 0, param = num, rc_param = 0; + enum _ecore_status_t rc; + + rc = ecore_mcp_cmd(p_hwfn, p_ptt, DRV_MSG_CODE_CFG_PF_VFS_MSIX, + param, &resp, &rc_param); + + if (resp != FW_MSG_CODE_DRV_CFG_PF_VFS_MSIX_DONE) { + DP_NOTICE(p_hwfn, true, "MFW failed to set MSI-X for VFs\n"); + rc = ECORE_INVAL; + } else { + DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, + "Requested 0x%02x MSI-x interrupts for VFs\n", + num); + } + + return rc; +} + +enum _ecore_status_t ecore_mcp_config_vf_msix(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + u8 vf_id, u8 num) +{ + if (ECORE_IS_BB(p_hwfn->p_dev)) + return ecore_mcp_config_vf_msix_bb(p_hwfn, p_ptt, vf_id, num); + else + return ecore_mcp_config_vf_msix_ah(p_hwfn, p_ptt, num); +} + enum _ecore_status_t ecore_mcp_send_drv_version(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, struct ecore_mcp_drv_version *p_ver) { - u32 param = 0, reply = 0, num_words, i; - struct drv_version_stc *p_drv_version; - union drv_union_data union_data; + struct ecore_mcp_mb_params mb_params; + struct drv_version_stc drv_version; + u32 num_words, i; void *p_name; OSAL_BE32 val; enum _ecore_status_t rc; @@ -1409,71 +2466,112 @@ ecore_mcp_send_drv_version(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, return ECORE_SUCCESS; #endif - p_drv_version = &union_data.drv_version; - p_drv_version->version = p_ver->version; + OSAL_MEM_ZERO(&drv_version, sizeof(drv_version)); + drv_version.version = p_ver->version; num_words = (MCP_DRV_VER_STR_SIZE - 4) / 4; for (i = 0; i < num_words; i++) { + /* The driver name is expected to be in a big-endian format */ p_name = &p_ver->name[i * sizeof(u32)]; val = OSAL_CPU_TO_BE32(*(u32 *)p_name); - *(u32 *)&p_drv_version->name[i * sizeof(u32)] = val; + *(u32 *)&drv_version.name[i * sizeof(u32)] = val; } - rc = ecore_mcp_cmd_and_union(p_hwfn, p_ptt, DRV_MSG_CODE_SET_VERSION, 0, - &union_data, &reply, ¶m); + OSAL_MEM_ZERO(&mb_params, sizeof(mb_params)); + mb_params.cmd = DRV_MSG_CODE_SET_VERSION; + mb_params.p_data_src = &drv_version; + mb_params.data_src_size = sizeof(drv_version); + rc = ecore_mcp_cmd_and_union(p_hwfn, p_ptt, &mb_params); if (rc != ECORE_SUCCESS) DP_ERR(p_hwfn, "MCP response failure, aborting\n"); return rc; } +/* A maximal 100 msec waiting time for the MCP to halt */ +#define ECORE_MCP_HALT_SLEEP_MS 10 +#define ECORE_MCP_HALT_MAX_RETRIES 10 + enum _ecore_status_t ecore_mcp_halt(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt) { + u32 resp = 0, param = 0, cpu_state, cnt = 0; enum _ecore_status_t rc; - u32 resp = 0, param = 0; rc = ecore_mcp_cmd(p_hwfn, p_ptt, DRV_MSG_CODE_MCP_HALT, 0, &resp, ¶m); - if (rc != ECORE_SUCCESS) + if (rc != ECORE_SUCCESS) { DP_ERR(p_hwfn, "MCP response failure, aborting\n"); + return rc; + } - return rc; + do { + OSAL_MSLEEP(ECORE_MCP_HALT_SLEEP_MS); + cpu_state = ecore_rd(p_hwfn, p_ptt, MCP_REG_CPU_STATE); + if (cpu_state & MCP_REG_CPU_STATE_SOFT_HALTED) + break; + } while (++cnt < ECORE_MCP_HALT_MAX_RETRIES); + + if (cnt == ECORE_MCP_HALT_MAX_RETRIES) { + DP_NOTICE(p_hwfn, false, + "Failed to halt the MCP [CPU_MODE = 0x%08x, CPU_STATE = 0x%08x]\n", + ecore_rd(p_hwfn, p_ptt, MCP_REG_CPU_MODE), cpu_state); + return ECORE_BUSY; + } + + ecore_mcp_cmd_set_blocking(p_hwfn, true); + + return ECORE_SUCCESS; } +#define ECORE_MCP_RESUME_SLEEP_MS 10 + enum _ecore_status_t ecore_mcp_resume(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt) { - u32 value, cpu_mode; + u32 cpu_mode, cpu_state; ecore_wr(p_hwfn, p_ptt, MCP_REG_CPU_STATE, 0xffffffff); - value = ecore_rd(p_hwfn, p_ptt, MCP_REG_CPU_MODE); - value &= ~MCP_REG_CPU_MODE_SOFT_HALT; - ecore_wr(p_hwfn, p_ptt, MCP_REG_CPU_MODE, value); cpu_mode = ecore_rd(p_hwfn, p_ptt, MCP_REG_CPU_MODE); + cpu_mode &= ~MCP_REG_CPU_MODE_SOFT_HALT; + ecore_wr(p_hwfn, p_ptt, MCP_REG_CPU_MODE, cpu_mode); - return (cpu_mode & MCP_REG_CPU_MODE_SOFT_HALT) ? -1 : 0; + OSAL_MSLEEP(ECORE_MCP_RESUME_SLEEP_MS); + cpu_state = ecore_rd(p_hwfn, p_ptt, MCP_REG_CPU_STATE); + + if (cpu_state & MCP_REG_CPU_STATE_SOFT_HALTED) { + DP_NOTICE(p_hwfn, false, + "Failed to resume the MCP [CPU_MODE = 0x%08x, CPU_STATE = 0x%08x]\n", + cpu_mode, cpu_state); + return ECORE_BUSY; + } + + ecore_mcp_cmd_set_blocking(p_hwfn, false); + + return ECORE_SUCCESS; } enum _ecore_status_t ecore_mcp_ov_update_current_config(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, - enum ecore_ov_config_method config, enum ecore_ov_client client) { - enum _ecore_status_t rc; u32 resp = 0, param = 0; u32 drv_mb_param; + enum _ecore_status_t rc; - switch (config) { + switch (client) { case ECORE_OV_CLIENT_DRV: drv_mb_param = DRV_MB_PARAM_OV_CURR_CFG_OS; break; case ECORE_OV_CLIENT_USER: drv_mb_param = DRV_MB_PARAM_OV_CURR_CFG_OTHER; break; + case ECORE_OV_CLIENT_VENDOR_SPEC: + drv_mb_param = DRV_MB_PARAM_OV_CURR_CFG_VENDOR_SPEC; + break; default: - DP_NOTICE(p_hwfn, true, "Invalid client type %d\n", config); + DP_NOTICE(p_hwfn, true, "Invalid client type %d\n", client); return ECORE_INVAL; } @@ -1490,9 +2588,9 @@ ecore_mcp_ov_update_driver_state(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, enum ecore_ov_driver_state drv_state) { - enum _ecore_status_t rc; u32 resp = 0, param = 0; u32 drv_mb_param; + enum _ecore_status_t rc; switch (drv_state) { case ECORE_OV_DRIVER_STATE_NOT_LOADED: @@ -1510,9 +2608,9 @@ ecore_mcp_ov_update_driver_state(struct ecore_hwfn *p_hwfn, } rc = ecore_mcp_cmd(p_hwfn, p_ptt, DRV_MSG_CODE_OV_UPDATE_DRIVER_STATE, - drv_state, &resp, ¶m); + drv_mb_param, &resp, ¶m); if (rc != ECORE_SUCCESS) - DP_ERR(p_hwfn, "MCP response failure, aborting\n"); + DP_ERR(p_hwfn, "Failed to send driver state\n"); return rc; } @@ -1565,8 +2663,8 @@ enum _ecore_status_t ecore_mcp_mask_parities(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, u32 mask_parities) { - enum _ecore_status_t rc; u32 resp = 0, param = 0; + enum _ecore_status_t rc; rc = ecore_mcp_cmd(p_hwfn, p_ptt, DRV_MSG_CODE_MASK_PARITIES, mask_parities, &resp, ¶m); @@ -1588,7 +2686,7 @@ enum _ecore_status_t ecore_mcp_nvm_read(struct ecore_dev *p_dev, u32 addr, { struct ecore_hwfn *p_hwfn = ECORE_LEADING_HWFN(p_dev); u32 bytes_left, offset, bytes_to_copy, buf_size; - struct ecore_mcp_nvm_params params; + u32 nvm_offset, resp, param; struct ecore_ptt *p_ptt; enum _ecore_status_t rc = ECORE_SUCCESS; @@ -1596,29 +2694,44 @@ enum _ecore_status_t ecore_mcp_nvm_read(struct ecore_dev *p_dev, u32 addr, if (!p_ptt) return ECORE_BUSY; - OSAL_MEMSET(¶ms, 0, sizeof(struct ecore_mcp_nvm_params)); bytes_left = len; offset = 0; - params.type = ECORE_MCP_NVM_RD; - params.nvm_rd.buf_size = &buf_size; - params.nvm_common.cmd = DRV_MSG_CODE_NVM_READ_NVRAM; while (bytes_left > 0) { bytes_to_copy = OSAL_MIN_T(u32, bytes_left, MCP_DRV_NVM_BUF_LEN); - params.nvm_common.offset = (addr + offset) | - (bytes_to_copy << DRV_MB_PARAM_NVM_LEN_SHIFT); - params.nvm_rd.buf = (u32 *)(p_buf + offset); - rc = ecore_mcp_nvm_command(p_hwfn, p_ptt, ¶ms); - if (rc != ECORE_SUCCESS || (params.nvm_common.resp != - FW_MSG_CODE_NVM_OK)) { - DP_NOTICE(p_dev, false, "MCP command rc = %d\n", rc); + nvm_offset = (addr + offset) | (bytes_to_copy << + DRV_MB_PARAM_NVM_LEN_OFFSET); + rc = ecore_mcp_nvm_rd_cmd(p_hwfn, p_ptt, + DRV_MSG_CODE_NVM_READ_NVRAM, + nvm_offset, &resp, ¶m, &buf_size, + (u32 *)(p_buf + offset)); + if (rc != ECORE_SUCCESS) { + DP_NOTICE(p_dev, false, + "ecore_mcp_nvm_rd_cmd() failed, rc = %d\n", + rc); + resp = FW_MSG_CODE_ERROR; break; } - offset += *params.nvm_rd.buf_size; - bytes_left -= *params.nvm_rd.buf_size; + + if (resp != FW_MSG_CODE_NVM_OK) { + DP_NOTICE(p_dev, false, + "nvm read failed, resp = 0x%08x\n", resp); + rc = ECORE_UNKNOWN_ERROR; + break; + } + + /* This can be a lengthy process, and it's possible scheduler + * isn't preemptible. Sleep a bit to prevent CPU hogging. + */ + if (bytes_left % 0x1000 < + (bytes_left - buf_size) % 0x1000) + OSAL_MSLEEP(1); + + offset += buf_size; + bytes_left -= buf_size; } - p_dev->mcp_nvm_resp = params.nvm_common.resp; + p_dev->mcp_nvm_resp = resp; ecore_ptt_release(p_hwfn, p_ptt); return rc; @@ -1628,26 +2741,23 @@ enum _ecore_status_t ecore_mcp_phy_read(struct ecore_dev *p_dev, u32 cmd, u32 addr, u8 *p_buf, u32 len) { struct ecore_hwfn *p_hwfn = ECORE_LEADING_HWFN(p_dev); - struct ecore_mcp_nvm_params params; struct ecore_ptt *p_ptt; + u32 resp, param; enum _ecore_status_t rc; p_ptt = ecore_ptt_acquire(p_hwfn); if (!p_ptt) return ECORE_BUSY; - OSAL_MEMSET(¶ms, 0, sizeof(struct ecore_mcp_nvm_params)); - params.type = ECORE_MCP_NVM_RD; - params.nvm_rd.buf_size = &len; - params.nvm_common.cmd = (cmd == ECORE_PHY_CORE_READ) ? - DRV_MSG_CODE_PHY_CORE_READ : DRV_MSG_CODE_PHY_RAW_READ; - params.nvm_common.offset = addr; - params.nvm_rd.buf = (u32 *)p_buf; - rc = ecore_mcp_nvm_command(p_hwfn, p_ptt, ¶ms); + rc = ecore_mcp_nvm_rd_cmd(p_hwfn, p_ptt, + (cmd == ECORE_PHY_CORE_READ) ? + DRV_MSG_CODE_PHY_CORE_READ : + DRV_MSG_CODE_PHY_RAW_READ, + addr, &resp, ¶m, &len, (u32 *)p_buf); if (rc != ECORE_SUCCESS) DP_NOTICE(p_dev, false, "MCP command rc = %d\n", rc); - p_dev->mcp_nvm_resp = params.nvm_common.resp; + p_dev->mcp_nvm_resp = resp; ecore_ptt_release(p_hwfn, p_ptt); return rc; @@ -1656,14 +2766,12 @@ enum _ecore_status_t ecore_mcp_phy_read(struct ecore_dev *p_dev, u32 cmd, enum _ecore_status_t ecore_mcp_nvm_resp(struct ecore_dev *p_dev, u8 *p_buf) { struct ecore_hwfn *p_hwfn = ECORE_LEADING_HWFN(p_dev); - struct ecore_mcp_nvm_params params; struct ecore_ptt *p_ptt; p_ptt = ecore_ptt_acquire(p_hwfn); if (!p_ptt) return ECORE_BUSY; - OSAL_MEMSET(¶ms, 0, sizeof(struct ecore_mcp_nvm_params)); OSAL_MEMCPY(p_buf, &p_dev->mcp_nvm_resp, sizeof(p_dev->mcp_nvm_resp)); ecore_ptt_release(p_hwfn, p_ptt); @@ -1673,19 +2781,16 @@ enum _ecore_status_t ecore_mcp_nvm_resp(struct ecore_dev *p_dev, u8 *p_buf) enum _ecore_status_t ecore_mcp_nvm_del_file(struct ecore_dev *p_dev, u32 addr) { struct ecore_hwfn *p_hwfn = ECORE_LEADING_HWFN(p_dev); - struct ecore_mcp_nvm_params params; struct ecore_ptt *p_ptt; + u32 resp, param; enum _ecore_status_t rc; p_ptt = ecore_ptt_acquire(p_hwfn); if (!p_ptt) return ECORE_BUSY; - OSAL_MEMSET(¶ms, 0, sizeof(struct ecore_mcp_nvm_params)); - params.type = ECORE_MCP_CMD; - params.nvm_common.cmd = DRV_MSG_CODE_NVM_DEL_FILE; - params.nvm_common.offset = addr; - rc = ecore_mcp_nvm_command(p_hwfn, p_ptt, ¶ms); - p_dev->mcp_nvm_resp = params.nvm_common.resp; + rc = ecore_mcp_cmd(p_hwfn, p_ptt, DRV_MSG_CODE_NVM_DEL_FILE, addr, + &resp, ¶m); + p_dev->mcp_nvm_resp = resp; ecore_ptt_release(p_hwfn, p_ptt); return rc; @@ -1695,19 +2800,16 @@ enum _ecore_status_t ecore_mcp_nvm_put_file_begin(struct ecore_dev *p_dev, u32 addr) { struct ecore_hwfn *p_hwfn = ECORE_LEADING_HWFN(p_dev); - struct ecore_mcp_nvm_params params; struct ecore_ptt *p_ptt; + u32 resp, param; enum _ecore_status_t rc; p_ptt = ecore_ptt_acquire(p_hwfn); if (!p_ptt) return ECORE_BUSY; - OSAL_MEMSET(¶ms, 0, sizeof(struct ecore_mcp_nvm_params)); - params.type = ECORE_MCP_CMD; - params.nvm_common.cmd = DRV_MSG_CODE_NVM_PUT_FILE_BEGIN; - params.nvm_common.offset = addr; - rc = ecore_mcp_nvm_command(p_hwfn, p_ptt, ¶ms); - p_dev->mcp_nvm_resp = params.nvm_common.resp; + rc = ecore_mcp_cmd(p_hwfn, p_ptt, DRV_MSG_CODE_NVM_PUT_FILE_BEGIN, addr, + &resp, ¶m); + p_dev->mcp_nvm_resp = resp; ecore_ptt_release(p_hwfn, p_ptt); return rc; @@ -1719,42 +2821,71 @@ enum _ecore_status_t ecore_mcp_nvm_put_file_begin(struct ecore_dev *p_dev, enum _ecore_status_t ecore_mcp_nvm_write(struct ecore_dev *p_dev, u32 cmd, u32 addr, u8 *p_buf, u32 len) { + u32 buf_idx, buf_size, nvm_cmd, nvm_offset, resp, param; struct ecore_hwfn *p_hwfn = ECORE_LEADING_HWFN(p_dev); enum _ecore_status_t rc = ECORE_INVAL; - struct ecore_mcp_nvm_params params; struct ecore_ptt *p_ptt; - u32 buf_idx, buf_size; p_ptt = ecore_ptt_acquire(p_hwfn); if (!p_ptt) return ECORE_BUSY; - OSAL_MEMSET(¶ms, 0, sizeof(struct ecore_mcp_nvm_params)); - params.type = ECORE_MCP_NVM_WR; - if (cmd == ECORE_PUT_FILE_DATA) - params.nvm_common.cmd = DRV_MSG_CODE_NVM_PUT_FILE_DATA; - else - params.nvm_common.cmd = DRV_MSG_CODE_NVM_WRITE_NVRAM; + switch (cmd) { + case ECORE_PUT_FILE_DATA: + nvm_cmd = DRV_MSG_CODE_NVM_PUT_FILE_DATA; + break; + case ECORE_NVM_WRITE_NVRAM: + nvm_cmd = DRV_MSG_CODE_NVM_WRITE_NVRAM; + break; + case ECORE_EXT_PHY_FW_UPGRADE: + nvm_cmd = DRV_MSG_CODE_EXT_PHY_FW_UPGRADE; + break; + default: + DP_NOTICE(p_hwfn, true, "Invalid nvm write command 0x%x\n", + cmd); + rc = ECORE_INVAL; + goto out; + } + buf_idx = 0; while (buf_idx < len) { buf_size = OSAL_MIN_T(u32, (len - buf_idx), MCP_DRV_NVM_BUF_LEN); - params.nvm_common.offset = ((buf_size << - DRV_MB_PARAM_NVM_LEN_SHIFT) - | addr) + buf_idx; - params.nvm_wr.buf_size = buf_size; - params.nvm_wr.buf = (u32 *)&p_buf[buf_idx]; - rc = ecore_mcp_nvm_command(p_hwfn, p_ptt, ¶ms); - if (rc != ECORE_SUCCESS || - ((params.nvm_common.resp != FW_MSG_CODE_NVM_OK) && - (params.nvm_common.resp != - FW_MSG_CODE_NVM_PUT_FILE_FINISH_OK))) - DP_NOTICE(p_dev, false, "MCP command rc = %d\n", rc); + nvm_offset = ((buf_size << DRV_MB_PARAM_NVM_LEN_OFFSET) | + addr) + + buf_idx; + rc = ecore_mcp_nvm_wr_cmd(p_hwfn, p_ptt, nvm_cmd, nvm_offset, + &resp, ¶m, buf_size, + (u32 *)&p_buf[buf_idx]); + if (rc != ECORE_SUCCESS) { + DP_NOTICE(p_dev, false, + "ecore_mcp_nvm_write() failed, rc = %d\n", + rc); + resp = FW_MSG_CODE_ERROR; + break; + } + + if (resp != FW_MSG_CODE_OK && + resp != FW_MSG_CODE_NVM_OK && + resp != FW_MSG_CODE_NVM_PUT_FILE_FINISH_OK) { + DP_NOTICE(p_dev, false, + "nvm write failed, resp = 0x%08x\n", resp); + rc = ECORE_UNKNOWN_ERROR; + break; + } + + /* This can be a lengthy process, and it's possible scheduler + * isn't preemptible. Sleep a bit to prevent CPU hogging. + */ + if (buf_idx % 0x1000 > + (buf_idx + buf_size) % 0x1000) + OSAL_MSLEEP(1); buf_idx += buf_size; } - p_dev->mcp_nvm_resp = params.nvm_common.resp; + p_dev->mcp_nvm_resp = resp; +out: ecore_ptt_release(p_hwfn, p_ptt); return rc; @@ -1764,25 +2895,21 @@ enum _ecore_status_t ecore_mcp_phy_write(struct ecore_dev *p_dev, u32 cmd, u32 addr, u8 *p_buf, u32 len) { struct ecore_hwfn *p_hwfn = ECORE_LEADING_HWFN(p_dev); - struct ecore_mcp_nvm_params params; struct ecore_ptt *p_ptt; + u32 resp, param, nvm_cmd; enum _ecore_status_t rc; p_ptt = ecore_ptt_acquire(p_hwfn); if (!p_ptt) return ECORE_BUSY; - OSAL_MEMSET(¶ms, 0, sizeof(struct ecore_mcp_nvm_params)); - params.type = ECORE_MCP_NVM_WR; - params.nvm_wr.buf_size = len; - params.nvm_common.cmd = (cmd == ECORE_PHY_CORE_WRITE) ? - DRV_MSG_CODE_PHY_CORE_WRITE : DRV_MSG_CODE_PHY_RAW_WRITE; - params.nvm_common.offset = addr; - params.nvm_wr.buf = (u32 *)p_buf; - rc = ecore_mcp_nvm_command(p_hwfn, p_ptt, ¶ms); + nvm_cmd = (cmd == ECORE_PHY_CORE_WRITE) ? DRV_MSG_CODE_PHY_CORE_WRITE : + DRV_MSG_CODE_PHY_RAW_WRITE; + rc = ecore_mcp_nvm_wr_cmd(p_hwfn, p_ptt, nvm_cmd, addr, + &resp, ¶m, len, (u32 *)p_buf); if (rc != ECORE_SUCCESS) DP_NOTICE(p_dev, false, "MCP command rc = %d\n", rc); - p_dev->mcp_nvm_resp = params.nvm_common.resp; + p_dev->mcp_nvm_resp = resp; ecore_ptt_release(p_hwfn, p_ptt); return rc; @@ -1792,20 +2919,17 @@ enum _ecore_status_t ecore_mcp_nvm_set_secure_mode(struct ecore_dev *p_dev, u32 addr) { struct ecore_hwfn *p_hwfn = ECORE_LEADING_HWFN(p_dev); - struct ecore_mcp_nvm_params params; struct ecore_ptt *p_ptt; + u32 resp, param; enum _ecore_status_t rc; p_ptt = ecore_ptt_acquire(p_hwfn); if (!p_ptt) return ECORE_BUSY; - OSAL_MEMSET(¶ms, 0, sizeof(struct ecore_mcp_nvm_params)); - params.type = ECORE_MCP_CMD; - params.nvm_common.cmd = DRV_MSG_CODE_SET_SECURE_MODE; - params.nvm_common.offset = addr; - rc = ecore_mcp_nvm_command(p_hwfn, p_ptt, ¶ms); - p_dev->mcp_nvm_resp = params.nvm_common.resp; + rc = ecore_mcp_cmd(p_hwfn, p_ptt, DRV_MSG_CODE_SET_SECURE_MODE, addr, + &resp, ¶m); + p_dev->mcp_nvm_resp = resp; ecore_ptt_release(p_hwfn, p_ptt); return rc; @@ -1816,39 +2940,42 @@ enum _ecore_status_t ecore_mcp_phy_sfp_read(struct ecore_hwfn *p_hwfn, u32 port, u32 addr, u32 offset, u32 len, u8 *p_buf) { - struct ecore_mcp_nvm_params params; + u32 bytes_left, bytes_to_copy, buf_size, nvm_offset; + u32 resp, param; enum _ecore_status_t rc; - u32 bytes_left, bytes_to_copy, buf_size; - OSAL_MEMSET(¶ms, 0, sizeof(struct ecore_mcp_nvm_params)); - SET_FIELD(params.nvm_common.offset, - DRV_MB_PARAM_TRANSCEIVER_PORT, port); - SET_FIELD(params.nvm_common.offset, - DRV_MB_PARAM_TRANSCEIVER_I2C_ADDRESS, addr); + nvm_offset = (port << DRV_MB_PARAM_TRANSCEIVER_PORT_OFFSET) | + (addr << DRV_MB_PARAM_TRANSCEIVER_I2C_ADDRESS_OFFSET); addr = offset; offset = 0; bytes_left = len; - params.type = ECORE_MCP_NVM_RD; - params.nvm_rd.buf_size = &buf_size; - params.nvm_common.cmd = DRV_MSG_CODE_TRANSCEIVER_READ; while (bytes_left > 0) { bytes_to_copy = OSAL_MIN_T(u32, bytes_left, MAX_I2C_TRANSACTION_SIZE); - params.nvm_rd.buf = (u32 *)(p_buf + offset); - SET_FIELD(params.nvm_common.offset, - DRV_MB_PARAM_TRANSCEIVER_OFFSET, addr + offset); - SET_FIELD(params.nvm_common.offset, - DRV_MB_PARAM_TRANSCEIVER_SIZE, bytes_to_copy); - rc = ecore_mcp_nvm_command(p_hwfn, p_ptt, ¶ms); - if ((params.nvm_common.resp & FW_MSG_CODE_MASK) == - FW_MSG_CODE_TRANSCEIVER_NOT_PRESENT) { + nvm_offset &= (DRV_MB_PARAM_TRANSCEIVER_I2C_ADDRESS_MASK | + DRV_MB_PARAM_TRANSCEIVER_PORT_MASK); + nvm_offset |= ((addr + offset) << + DRV_MB_PARAM_TRANSCEIVER_OFFSET_OFFSET); + nvm_offset |= (bytes_to_copy << + DRV_MB_PARAM_TRANSCEIVER_SIZE_OFFSET); + rc = ecore_mcp_nvm_rd_cmd(p_hwfn, p_ptt, + DRV_MSG_CODE_TRANSCEIVER_READ, + nvm_offset, &resp, ¶m, &buf_size, + (u32 *)(p_buf + offset)); + if (rc != ECORE_SUCCESS) { + DP_NOTICE(p_hwfn, false, + "Failed to send a transceiver read command to the MFW. rc = %d.\n", + rc); + return rc; + } + + if (resp == FW_MSG_CODE_TRANSCEIVER_NOT_PRESENT) return ECORE_NODEV; - } else if ((params.nvm_common.resp & FW_MSG_CODE_MASK) != - FW_MSG_CODE_TRANSCEIVER_DIAG_OK) + else if (resp != FW_MSG_CODE_TRANSCEIVER_DIAG_OK) return ECORE_UNKNOWN_ERROR; - offset += *params.nvm_rd.buf_size; - bytes_left -= *params.nvm_rd.buf_size; + offset += buf_size; + bytes_left -= buf_size; } return ECORE_SUCCESS; @@ -1859,33 +2986,35 @@ enum _ecore_status_t ecore_mcp_phy_sfp_write(struct ecore_hwfn *p_hwfn, u32 port, u32 addr, u32 offset, u32 len, u8 *p_buf) { - struct ecore_mcp_nvm_params params; + u32 buf_idx, buf_size, nvm_offset, resp, param; enum _ecore_status_t rc; - u32 buf_idx, buf_size; - OSAL_MEMSET(¶ms, 0, sizeof(struct ecore_mcp_nvm_params)); - SET_FIELD(params.nvm_common.offset, - DRV_MB_PARAM_TRANSCEIVER_PORT, port); - SET_FIELD(params.nvm_common.offset, - DRV_MB_PARAM_TRANSCEIVER_I2C_ADDRESS, addr); - params.type = ECORE_MCP_NVM_WR; - params.nvm_common.cmd = DRV_MSG_CODE_TRANSCEIVER_WRITE; + nvm_offset = (port << DRV_MB_PARAM_TRANSCEIVER_PORT_OFFSET) | + (addr << DRV_MB_PARAM_TRANSCEIVER_I2C_ADDRESS_OFFSET); buf_idx = 0; while (buf_idx < len) { buf_size = OSAL_MIN_T(u32, (len - buf_idx), MAX_I2C_TRANSACTION_SIZE); - SET_FIELD(params.nvm_common.offset, - DRV_MB_PARAM_TRANSCEIVER_OFFSET, offset + buf_idx); - SET_FIELD(params.nvm_common.offset, - DRV_MB_PARAM_TRANSCEIVER_SIZE, buf_size); - params.nvm_wr.buf_size = buf_size; - params.nvm_wr.buf = (u32 *)&p_buf[buf_idx]; - rc = ecore_mcp_nvm_command(p_hwfn, p_ptt, ¶ms); - if ((params.nvm_common.resp & FW_MSG_CODE_MASK) == - FW_MSG_CODE_TRANSCEIVER_NOT_PRESENT) { + nvm_offset &= (DRV_MB_PARAM_TRANSCEIVER_I2C_ADDRESS_MASK | + DRV_MB_PARAM_TRANSCEIVER_PORT_MASK); + nvm_offset |= ((offset + buf_idx) << + DRV_MB_PARAM_TRANSCEIVER_OFFSET_OFFSET); + nvm_offset |= (buf_size << + DRV_MB_PARAM_TRANSCEIVER_SIZE_OFFSET); + rc = ecore_mcp_nvm_wr_cmd(p_hwfn, p_ptt, + DRV_MSG_CODE_TRANSCEIVER_WRITE, + nvm_offset, &resp, ¶m, buf_size, + (u32 *)&p_buf[buf_idx]); + if (rc != ECORE_SUCCESS) { + DP_NOTICE(p_hwfn, false, + "Failed to send a transceiver write command to the MFW. rc = %d.\n", + rc); + return rc; + } + + if (resp == FW_MSG_CODE_TRANSCEIVER_NOT_PRESENT) return ECORE_NODEV; - } else if ((params.nvm_common.resp & FW_MSG_CODE_MASK) != - FW_MSG_CODE_TRANSCEIVER_DIAG_OK) + else if (resp != FW_MSG_CODE_TRANSCEIVER_DIAG_OK) return ECORE_UNKNOWN_ERROR; buf_idx += buf_size; @@ -1901,11 +3030,14 @@ enum _ecore_status_t ecore_mcp_gpio_read(struct ecore_hwfn *p_hwfn, enum _ecore_status_t rc = ECORE_SUCCESS; u32 drv_mb_param = 0, rsp; - SET_FIELD(drv_mb_param, DRV_MB_PARAM_GPIO_NUMBER, gpio); + drv_mb_param = (gpio << DRV_MB_PARAM_GPIO_NUMBER_OFFSET); rc = ecore_mcp_cmd(p_hwfn, p_ptt, DRV_MSG_CODE_GPIO_READ, drv_mb_param, &rsp, gpio_val); + if (rc != ECORE_SUCCESS) + return rc; + if ((rsp & FW_MSG_CODE_MASK) != FW_MSG_CODE_GPIO_OK) return ECORE_UNKNOWN_ERROR; @@ -1919,14 +3051,779 @@ enum _ecore_status_t ecore_mcp_gpio_write(struct ecore_hwfn *p_hwfn, enum _ecore_status_t rc = ECORE_SUCCESS; u32 drv_mb_param = 0, param, rsp; - SET_FIELD(drv_mb_param, DRV_MB_PARAM_GPIO_NUMBER, gpio); - SET_FIELD(drv_mb_param, DRV_MB_PARAM_GPIO_VALUE, gpio_val); + drv_mb_param = (gpio << DRV_MB_PARAM_GPIO_NUMBER_OFFSET) | + (gpio_val << DRV_MB_PARAM_GPIO_VALUE_OFFSET); rc = ecore_mcp_cmd(p_hwfn, p_ptt, DRV_MSG_CODE_GPIO_WRITE, drv_mb_param, &rsp, ¶m); + if (rc != ECORE_SUCCESS) + return rc; + if ((rsp & FW_MSG_CODE_MASK) != FW_MSG_CODE_GPIO_OK) return ECORE_UNKNOWN_ERROR; return ECORE_SUCCESS; } + +enum _ecore_status_t ecore_mcp_gpio_info(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + u16 gpio, u32 *gpio_direction, + u32 *gpio_ctrl) +{ + u32 drv_mb_param = 0, rsp, val = 0; + enum _ecore_status_t rc = ECORE_SUCCESS; + + drv_mb_param = gpio << DRV_MB_PARAM_GPIO_NUMBER_OFFSET; + + rc = ecore_mcp_cmd(p_hwfn, p_ptt, DRV_MSG_CODE_GPIO_INFO, + drv_mb_param, &rsp, &val); + if (rc != ECORE_SUCCESS) + return rc; + + *gpio_direction = (val & DRV_MB_PARAM_GPIO_DIRECTION_MASK) >> + DRV_MB_PARAM_GPIO_DIRECTION_OFFSET; + *gpio_ctrl = (val & DRV_MB_PARAM_GPIO_CTRL_MASK) >> + DRV_MB_PARAM_GPIO_CTRL_OFFSET; + + if ((rsp & FW_MSG_CODE_MASK) != FW_MSG_CODE_GPIO_OK) + return ECORE_UNKNOWN_ERROR; + + return ECORE_SUCCESS; +} + +enum _ecore_status_t ecore_mcp_bist_register_test(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt) +{ + u32 drv_mb_param = 0, rsp, param; + enum _ecore_status_t rc = ECORE_SUCCESS; + + drv_mb_param = (DRV_MB_PARAM_BIST_REGISTER_TEST << + DRV_MB_PARAM_BIST_TEST_INDEX_OFFSET); + + rc = ecore_mcp_cmd(p_hwfn, p_ptt, DRV_MSG_CODE_BIST_TEST, + drv_mb_param, &rsp, ¶m); + + if (rc != ECORE_SUCCESS) + return rc; + + if (((rsp & FW_MSG_CODE_MASK) != FW_MSG_CODE_OK) || + (param != DRV_MB_PARAM_BIST_RC_PASSED)) + rc = ECORE_UNKNOWN_ERROR; + + return rc; +} + +enum _ecore_status_t ecore_mcp_bist_clock_test(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt) +{ + u32 drv_mb_param, rsp, param; + enum _ecore_status_t rc = ECORE_SUCCESS; + + drv_mb_param = (DRV_MB_PARAM_BIST_CLOCK_TEST << + DRV_MB_PARAM_BIST_TEST_INDEX_OFFSET); + + rc = ecore_mcp_cmd(p_hwfn, p_ptt, DRV_MSG_CODE_BIST_TEST, + drv_mb_param, &rsp, ¶m); + + if (rc != ECORE_SUCCESS) + return rc; + + if (((rsp & FW_MSG_CODE_MASK) != FW_MSG_CODE_OK) || + (param != DRV_MB_PARAM_BIST_RC_PASSED)) + rc = ECORE_UNKNOWN_ERROR; + + return rc; +} + +enum _ecore_status_t ecore_mcp_bist_nvm_test_get_num_images( + struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, u32 *num_images) +{ + u32 drv_mb_param = 0, rsp; + enum _ecore_status_t rc = ECORE_SUCCESS; + + drv_mb_param = (DRV_MB_PARAM_BIST_NVM_TEST_NUM_IMAGES << + DRV_MB_PARAM_BIST_TEST_INDEX_OFFSET); + + rc = ecore_mcp_cmd(p_hwfn, p_ptt, DRV_MSG_CODE_BIST_TEST, + drv_mb_param, &rsp, num_images); + + if (rc != ECORE_SUCCESS) + return rc; + + if (((rsp & FW_MSG_CODE_MASK) != FW_MSG_CODE_OK)) + rc = ECORE_UNKNOWN_ERROR; + + return rc; +} + +enum _ecore_status_t ecore_mcp_bist_nvm_test_get_image_att( + struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, + struct bist_nvm_image_att *p_image_att, u32 image_index) +{ + u32 buf_size, nvm_offset, resp, param; + enum _ecore_status_t rc; + + nvm_offset = (DRV_MB_PARAM_BIST_NVM_TEST_IMAGE_BY_INDEX << + DRV_MB_PARAM_BIST_TEST_INDEX_OFFSET); + nvm_offset |= (image_index << + DRV_MB_PARAM_BIST_TEST_IMAGE_INDEX_OFFSET); + rc = ecore_mcp_nvm_rd_cmd(p_hwfn, p_ptt, DRV_MSG_CODE_BIST_TEST, + nvm_offset, &resp, ¶m, &buf_size, + (u32 *)p_image_att); + if (rc != ECORE_SUCCESS) + return rc; + + if (((resp & FW_MSG_CODE_MASK) != FW_MSG_CODE_OK) || + (p_image_att->return_code != 1)) + rc = ECORE_UNKNOWN_ERROR; + + return rc; +} + +enum _ecore_status_t +ecore_mcp_get_temperature_info(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + struct ecore_temperature_info *p_temp_info) +{ + struct ecore_temperature_sensor *p_temp_sensor; + struct temperature_status_stc mfw_temp_info; + struct ecore_mcp_mb_params mb_params; + u32 val; + enum _ecore_status_t rc; + u8 i; + + OSAL_MEM_ZERO(&mb_params, sizeof(mb_params)); + mb_params.cmd = DRV_MSG_CODE_GET_TEMPERATURE; + mb_params.p_data_dst = &mfw_temp_info; + mb_params.data_dst_size = sizeof(mfw_temp_info); + rc = ecore_mcp_cmd_and_union(p_hwfn, p_ptt, &mb_params); + if (rc != ECORE_SUCCESS) + return rc; + + OSAL_BUILD_BUG_ON(ECORE_MAX_NUM_OF_SENSORS != MAX_NUM_OF_SENSORS); + p_temp_info->num_sensors = OSAL_MIN_T(u32, mfw_temp_info.num_of_sensors, + ECORE_MAX_NUM_OF_SENSORS); + for (i = 0; i < p_temp_info->num_sensors; i++) { + val = mfw_temp_info.sensor[i]; + p_temp_sensor = &p_temp_info->sensors[i]; + p_temp_sensor->sensor_location = (val & SENSOR_LOCATION_MASK) >> + SENSOR_LOCATION_OFFSET; + p_temp_sensor->threshold_high = (val & THRESHOLD_HIGH_MASK) >> + THRESHOLD_HIGH_OFFSET; + p_temp_sensor->critical = (val & CRITICAL_TEMPERATURE_MASK) >> + CRITICAL_TEMPERATURE_OFFSET; + p_temp_sensor->current_temp = (val & CURRENT_TEMP_MASK) >> + CURRENT_TEMP_OFFSET; + } + + return ECORE_SUCCESS; +} + +enum _ecore_status_t ecore_mcp_get_mba_versions( + struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + struct ecore_mba_vers *p_mba_vers) +{ + u32 buf_size, resp, param; + enum _ecore_status_t rc; + + rc = ecore_mcp_nvm_rd_cmd(p_hwfn, p_ptt, DRV_MSG_CODE_GET_MBA_VERSION, + 0, &resp, ¶m, &buf_size, + &p_mba_vers->mba_vers[0]); + + if (rc != ECORE_SUCCESS) + return rc; + + if ((resp & FW_MSG_CODE_MASK) != FW_MSG_CODE_NVM_OK) + rc = ECORE_UNKNOWN_ERROR; + + if (buf_size != MCP_DRV_NVM_BUF_LEN) + rc = ECORE_UNKNOWN_ERROR; + + return rc; +} + +enum _ecore_status_t ecore_mcp_mem_ecc_events(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + u64 *num_events) +{ + u32 rsp; + + return ecore_mcp_cmd(p_hwfn, p_ptt, DRV_MSG_CODE_MEM_ECC_EVENTS, + 0, &rsp, (u32 *)num_events); +} + +static enum resource_id_enum +ecore_mcp_get_mfw_res_id(enum ecore_resources res_id) +{ + enum resource_id_enum mfw_res_id = RESOURCE_NUM_INVALID; + + switch (res_id) { + case ECORE_SB: + mfw_res_id = RESOURCE_NUM_SB_E; + break; + case ECORE_L2_QUEUE: + mfw_res_id = RESOURCE_NUM_L2_QUEUE_E; + break; + case ECORE_VPORT: + mfw_res_id = RESOURCE_NUM_VPORT_E; + break; + case ECORE_RSS_ENG: + mfw_res_id = RESOURCE_NUM_RSS_ENGINES_E; + break; + case ECORE_PQ: + mfw_res_id = RESOURCE_NUM_PQ_E; + break; + case ECORE_RL: + mfw_res_id = RESOURCE_NUM_RL_E; + break; + case ECORE_MAC: + case ECORE_VLAN: + /* Each VFC resource can accommodate both a MAC and a VLAN */ + mfw_res_id = RESOURCE_VFC_FILTER_E; + break; + case ECORE_ILT: + mfw_res_id = RESOURCE_ILT_E; + break; + case ECORE_LL2_QUEUE: + mfw_res_id = RESOURCE_LL2_QUEUE_E; + break; + case ECORE_RDMA_CNQ_RAM: + case ECORE_CMDQS_CQS: + /* CNQ/CMDQS are the same resource */ + mfw_res_id = RESOURCE_CQS_E; + break; + case ECORE_RDMA_STATS_QUEUE: + mfw_res_id = RESOURCE_RDMA_STATS_QUEUE_E; + break; + case ECORE_BDQ: + mfw_res_id = RESOURCE_BDQ_E; + break; + default: + break; + } + + return mfw_res_id; +} + +#define ECORE_RESC_ALLOC_VERSION_MAJOR 2 +#define ECORE_RESC_ALLOC_VERSION_MINOR 0 +#define ECORE_RESC_ALLOC_VERSION \ + ((ECORE_RESC_ALLOC_VERSION_MAJOR << \ + DRV_MB_PARAM_RESOURCE_ALLOC_VERSION_MAJOR_OFFSET) | \ + (ECORE_RESC_ALLOC_VERSION_MINOR << \ + DRV_MB_PARAM_RESOURCE_ALLOC_VERSION_MINOR_OFFSET)) + +struct ecore_resc_alloc_in_params { + u32 cmd; + enum ecore_resources res_id; + u32 resc_max_val; +}; + +struct ecore_resc_alloc_out_params { + u32 mcp_resp; + u32 mcp_param; + u32 resc_num; + u32 resc_start; + u32 vf_resc_num; + u32 vf_resc_start; + u32 flags; +}; + +#define ECORE_RECOVERY_PROLOG_SLEEP_MS 100 + +enum _ecore_status_t ecore_recovery_prolog(struct ecore_dev *p_dev) +{ + struct ecore_hwfn *p_hwfn = ECORE_LEADING_HWFN(p_dev); + struct ecore_ptt *p_ptt = p_hwfn->p_main_ptt; + enum _ecore_status_t rc; + + /* Allow ongoing PCIe transactions to complete */ + OSAL_MSLEEP(ECORE_RECOVERY_PROLOG_SLEEP_MS); + + /* Clear the PF's internal FID_enable in the PXP */ + rc = ecore_pglueb_set_pfid_enable(p_hwfn, p_ptt, false); + if (rc != ECORE_SUCCESS) + DP_NOTICE(p_hwfn, false, + "ecore_pglueb_set_pfid_enable() failed. rc = %d.\n", + rc); + + return rc; +} + +static enum _ecore_status_t +ecore_mcp_resc_allocation_msg(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + struct ecore_resc_alloc_in_params *p_in_params, + struct ecore_resc_alloc_out_params *p_out_params) +{ + struct ecore_mcp_mb_params mb_params; + struct resource_info mfw_resc_info; + enum _ecore_status_t rc; + + OSAL_MEM_ZERO(&mfw_resc_info, sizeof(mfw_resc_info)); + + mfw_resc_info.res_id = ecore_mcp_get_mfw_res_id(p_in_params->res_id); + if (mfw_resc_info.res_id == RESOURCE_NUM_INVALID) { + DP_ERR(p_hwfn, + "Failed to match resource %d [%s] with the MFW resources\n", + p_in_params->res_id, + ecore_hw_get_resc_name(p_in_params->res_id)); + return ECORE_INVAL; + } + + switch (p_in_params->cmd) { + case DRV_MSG_SET_RESOURCE_VALUE_MSG: + mfw_resc_info.size = p_in_params->resc_max_val; + /* Fallthrough */ + case DRV_MSG_GET_RESOURCE_ALLOC_MSG: + break; + default: + DP_ERR(p_hwfn, "Unexpected resource alloc command [0x%08x]\n", + p_in_params->cmd); + return ECORE_INVAL; + } + + OSAL_MEM_ZERO(&mb_params, sizeof(mb_params)); + mb_params.cmd = p_in_params->cmd; + mb_params.param = ECORE_RESC_ALLOC_VERSION; + mb_params.p_data_src = &mfw_resc_info; + mb_params.data_src_size = sizeof(mfw_resc_info); + mb_params.p_data_dst = mb_params.p_data_src; + mb_params.data_dst_size = mb_params.data_src_size; + + DP_VERBOSE(p_hwfn, ECORE_MSG_SP, + "Resource message request: cmd 0x%08x, res_id %d [%s], hsi_version %d.%d, val 0x%x\n", + p_in_params->cmd, p_in_params->res_id, + ecore_hw_get_resc_name(p_in_params->res_id), + GET_MFW_FIELD(mb_params.param, + DRV_MB_PARAM_RESOURCE_ALLOC_VERSION_MAJOR), + GET_MFW_FIELD(mb_params.param, + DRV_MB_PARAM_RESOURCE_ALLOC_VERSION_MINOR), + p_in_params->resc_max_val); + + rc = ecore_mcp_cmd_and_union(p_hwfn, p_ptt, &mb_params); + if (rc != ECORE_SUCCESS) + return rc; + + p_out_params->mcp_resp = mb_params.mcp_resp; + p_out_params->mcp_param = mb_params.mcp_param; + p_out_params->resc_num = mfw_resc_info.size; + p_out_params->resc_start = mfw_resc_info.offset; + p_out_params->vf_resc_num = mfw_resc_info.vf_size; + p_out_params->vf_resc_start = mfw_resc_info.vf_offset; + p_out_params->flags = mfw_resc_info.flags; + + DP_VERBOSE(p_hwfn, ECORE_MSG_SP, + "Resource message response: mfw_hsi_version %d.%d, num 0x%x, start 0x%x, vf_num 0x%x, vf_start 0x%x, flags 0x%08x\n", + GET_MFW_FIELD(p_out_params->mcp_param, + FW_MB_PARAM_RESOURCE_ALLOC_VERSION_MAJOR), + GET_MFW_FIELD(p_out_params->mcp_param, + FW_MB_PARAM_RESOURCE_ALLOC_VERSION_MINOR), + p_out_params->resc_num, p_out_params->resc_start, + p_out_params->vf_resc_num, p_out_params->vf_resc_start, + p_out_params->flags); + + return ECORE_SUCCESS; +} + +enum _ecore_status_t +ecore_mcp_set_resc_max_val(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, + enum ecore_resources res_id, u32 resc_max_val, + u32 *p_mcp_resp) +{ + struct ecore_resc_alloc_out_params out_params; + struct ecore_resc_alloc_in_params in_params; + enum _ecore_status_t rc; + + OSAL_MEM_ZERO(&in_params, sizeof(in_params)); + in_params.cmd = DRV_MSG_SET_RESOURCE_VALUE_MSG; + in_params.res_id = res_id; + in_params.resc_max_val = resc_max_val; + OSAL_MEM_ZERO(&out_params, sizeof(out_params)); + rc = ecore_mcp_resc_allocation_msg(p_hwfn, p_ptt, &in_params, + &out_params); + if (rc != ECORE_SUCCESS) + return rc; + + *p_mcp_resp = out_params.mcp_resp; + + return ECORE_SUCCESS; +} + +enum _ecore_status_t +ecore_mcp_get_resc_info(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, + enum ecore_resources res_id, u32 *p_mcp_resp, + u32 *p_resc_num, u32 *p_resc_start) +{ + struct ecore_resc_alloc_out_params out_params; + struct ecore_resc_alloc_in_params in_params; + enum _ecore_status_t rc; + + OSAL_MEM_ZERO(&in_params, sizeof(in_params)); + in_params.cmd = DRV_MSG_GET_RESOURCE_ALLOC_MSG; + in_params.res_id = res_id; + OSAL_MEM_ZERO(&out_params, sizeof(out_params)); + rc = ecore_mcp_resc_allocation_msg(p_hwfn, p_ptt, &in_params, + &out_params); + if (rc != ECORE_SUCCESS) + return rc; + + *p_mcp_resp = out_params.mcp_resp; + + if (*p_mcp_resp == FW_MSG_CODE_RESOURCE_ALLOC_OK) { + *p_resc_num = out_params.resc_num; + *p_resc_start = out_params.resc_start; + } + + return ECORE_SUCCESS; +} + +enum _ecore_status_t ecore_mcp_initiate_pf_flr(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt) +{ + u32 mcp_resp, mcp_param; + + return ecore_mcp_cmd(p_hwfn, p_ptt, DRV_MSG_CODE_INITIATE_PF_FLR, 0, + &mcp_resp, &mcp_param); +} + +static enum _ecore_status_t ecore_mcp_resource_cmd(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + u32 param, u32 *p_mcp_resp, + u32 *p_mcp_param) +{ + enum _ecore_status_t rc; + + rc = ecore_mcp_cmd(p_hwfn, p_ptt, DRV_MSG_CODE_RESOURCE_CMD, param, + p_mcp_resp, p_mcp_param); + if (rc != ECORE_SUCCESS) + return rc; + + if (*p_mcp_resp == FW_MSG_CODE_UNSUPPORTED) { + DP_INFO(p_hwfn, + "The resource command is unsupported by the MFW\n"); + return ECORE_NOTIMPL; + } + + if (*p_mcp_param == RESOURCE_OPCODE_UNKNOWN_CMD) { + u8 opcode = GET_MFW_FIELD(param, RESOURCE_CMD_REQ_OPCODE); + + DP_NOTICE(p_hwfn, false, + "The resource command is unknown to the MFW [param 0x%08x, opcode %d]\n", + param, opcode); + return ECORE_INVAL; + } + + return rc; +} + +enum _ecore_status_t +__ecore_mcp_resc_lock(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, + struct ecore_resc_lock_params *p_params) +{ + u32 param = 0, mcp_resp, mcp_param; + u8 opcode; + enum _ecore_status_t rc; + + switch (p_params->timeout) { + case ECORE_MCP_RESC_LOCK_TO_DEFAULT: + opcode = RESOURCE_OPCODE_REQ; + p_params->timeout = 0; + break; + case ECORE_MCP_RESC_LOCK_TO_NONE: + opcode = RESOURCE_OPCODE_REQ_WO_AGING; + p_params->timeout = 0; + break; + default: + opcode = RESOURCE_OPCODE_REQ_W_AGING; + break; + } + + SET_MFW_FIELD(param, RESOURCE_CMD_REQ_RESC, p_params->resource); + SET_MFW_FIELD(param, RESOURCE_CMD_REQ_OPCODE, opcode); + SET_MFW_FIELD(param, RESOURCE_CMD_REQ_AGE, p_params->timeout); + + DP_VERBOSE(p_hwfn, ECORE_MSG_SP, + "Resource lock request: param 0x%08x [age %d, opcode %d, resource %d]\n", + param, p_params->timeout, opcode, p_params->resource); + + /* Attempt to acquire the resource */ + rc = ecore_mcp_resource_cmd(p_hwfn, p_ptt, param, &mcp_resp, + &mcp_param); + if (rc != ECORE_SUCCESS) + return rc; + + /* Analyze the response */ + p_params->owner = GET_MFW_FIELD(mcp_param, RESOURCE_CMD_RSP_OWNER); + opcode = GET_MFW_FIELD(mcp_param, RESOURCE_CMD_RSP_OPCODE); + + DP_VERBOSE(p_hwfn, ECORE_MSG_SP, + "Resource lock response: mcp_param 0x%08x [opcode %d, owner %d]\n", + mcp_param, opcode, p_params->owner); + + switch (opcode) { + case RESOURCE_OPCODE_GNT: + p_params->b_granted = true; + break; + case RESOURCE_OPCODE_BUSY: + p_params->b_granted = false; + break; + default: + DP_NOTICE(p_hwfn, false, + "Unexpected opcode in resource lock response [mcp_param 0x%08x, opcode %d]\n", + mcp_param, opcode); + return ECORE_INVAL; + } + + return ECORE_SUCCESS; +} + +enum _ecore_status_t +ecore_mcp_resc_lock(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, + struct ecore_resc_lock_params *p_params) +{ + u32 retry_cnt = 0; + enum _ecore_status_t rc; + + do { + /* No need for an interval before the first iteration */ + if (retry_cnt) { + if (p_params->sleep_b4_retry) { + u16 retry_interval_in_ms = + DIV_ROUND_UP(p_params->retry_interval, + 1000); + + OSAL_MSLEEP(retry_interval_in_ms); + } else { + OSAL_UDELAY(p_params->retry_interval); + } + } + + rc = __ecore_mcp_resc_lock(p_hwfn, p_ptt, p_params); + if (rc != ECORE_SUCCESS) + return rc; + + if (p_params->b_granted) + break; + } while (retry_cnt++ < p_params->retry_num); + + return ECORE_SUCCESS; +} + +void ecore_mcp_resc_lock_default_init(struct ecore_resc_lock_params *p_lock, + struct ecore_resc_unlock_params *p_unlock, + enum ecore_resc_lock resource, + bool b_is_permanent) +{ + if (p_lock != OSAL_NULL) { + OSAL_MEM_ZERO(p_lock, sizeof(*p_lock)); + + /* Permanent resources don't require aging, and there's no + * point in trying to acquire them more than once since it's + * unexpected another entity would release them. + */ + if (b_is_permanent) { + p_lock->timeout = ECORE_MCP_RESC_LOCK_TO_NONE; + } else { + p_lock->retry_num = ECORE_MCP_RESC_LOCK_RETRY_CNT_DFLT; + p_lock->retry_interval = + ECORE_MCP_RESC_LOCK_RETRY_VAL_DFLT; + p_lock->sleep_b4_retry = true; + } + + p_lock->resource = resource; + } + + if (p_unlock != OSAL_NULL) { + OSAL_MEM_ZERO(p_unlock, sizeof(*p_unlock)); + p_unlock->resource = resource; + } +} + +enum _ecore_status_t +ecore_mcp_resc_unlock(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, + struct ecore_resc_unlock_params *p_params) +{ + u32 param = 0, mcp_resp, mcp_param; + u8 opcode; + enum _ecore_status_t rc; + + opcode = p_params->b_force ? RESOURCE_OPCODE_FORCE_RELEASE + : RESOURCE_OPCODE_RELEASE; + SET_MFW_FIELD(param, RESOURCE_CMD_REQ_RESC, p_params->resource); + SET_MFW_FIELD(param, RESOURCE_CMD_REQ_OPCODE, opcode); + + DP_VERBOSE(p_hwfn, ECORE_MSG_SP, + "Resource unlock request: param 0x%08x [opcode %d, resource %d]\n", + param, opcode, p_params->resource); + + /* Attempt to release the resource */ + rc = ecore_mcp_resource_cmd(p_hwfn, p_ptt, param, &mcp_resp, + &mcp_param); + if (rc != ECORE_SUCCESS) + return rc; + + /* Analyze the response */ + opcode = GET_MFW_FIELD(mcp_param, RESOURCE_CMD_RSP_OPCODE); + + DP_VERBOSE(p_hwfn, ECORE_MSG_SP, + "Resource unlock response: mcp_param 0x%08x [opcode %d]\n", + mcp_param, opcode); + + switch (opcode) { + case RESOURCE_OPCODE_RELEASED_PREVIOUS: + DP_INFO(p_hwfn, + "Resource unlock request for an already released resource [%d]\n", + p_params->resource); + /* Fallthrough */ + case RESOURCE_OPCODE_RELEASED: + p_params->b_released = true; + break; + case RESOURCE_OPCODE_WRONG_OWNER: + p_params->b_released = false; + break; + default: + DP_NOTICE(p_hwfn, false, + "Unexpected opcode in resource unlock response [mcp_param 0x%08x, opcode %d]\n", + mcp_param, opcode); + return ECORE_INVAL; + } + + return ECORE_SUCCESS; +} + +bool ecore_mcp_is_smart_an_supported(struct ecore_hwfn *p_hwfn) +{ + return !!(p_hwfn->mcp_info->capabilities & + FW_MB_PARAM_FEATURE_SUPPORT_SMARTLINQ); +} + +enum _ecore_status_t ecore_mcp_get_capabilities(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt) +{ + u32 mcp_resp; + enum _ecore_status_t rc; + + rc = ecore_mcp_cmd(p_hwfn, p_ptt, DRV_MSG_CODE_GET_MFW_FEATURE_SUPPORT, + 0, &mcp_resp, &p_hwfn->mcp_info->capabilities); + if (rc == ECORE_SUCCESS) + DP_VERBOSE(p_hwfn, (ECORE_MSG_SP | ECORE_MSG_PROBE), + "MFW supported features: %08x\n", + p_hwfn->mcp_info->capabilities); + + return rc; +} + +enum _ecore_status_t ecore_mcp_set_capabilities(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt) +{ + u32 mcp_resp, mcp_param, features; + + features = DRV_MB_PARAM_FEATURE_SUPPORT_PORT_SMARTLINQ | + DRV_MB_PARAM_FEATURE_SUPPORT_PORT_EEE | + DRV_MB_PARAM_FEATURE_SUPPORT_FUNC_VLINK; + + return ecore_mcp_cmd(p_hwfn, p_ptt, DRV_MSG_CODE_FEATURE_SUPPORT, + features, &mcp_resp, &mcp_param); +} + +enum _ecore_status_t +ecore_mcp_drv_attribute(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, + struct ecore_mcp_drv_attr *p_drv_attr) +{ + struct attribute_cmd_write_stc attr_cmd_write; + enum _attribute_commands_e mfw_attr_cmd; + struct ecore_mcp_mb_params mb_params; + enum _ecore_status_t rc; + + switch (p_drv_attr->attr_cmd) { + case ECORE_MCP_DRV_ATTR_CMD_READ: + mfw_attr_cmd = ATTRIBUTE_CMD_READ; + break; + case ECORE_MCP_DRV_ATTR_CMD_WRITE: + mfw_attr_cmd = ATTRIBUTE_CMD_WRITE; + break; + case ECORE_MCP_DRV_ATTR_CMD_READ_CLEAR: + mfw_attr_cmd = ATTRIBUTE_CMD_READ_CLEAR; + break; + case ECORE_MCP_DRV_ATTR_CMD_CLEAR: + mfw_attr_cmd = ATTRIBUTE_CMD_CLEAR; + break; + default: + DP_NOTICE(p_hwfn, false, "Unknown attribute command %d\n", + p_drv_attr->attr_cmd); + return ECORE_INVAL; + } + + OSAL_MEM_ZERO(&mb_params, sizeof(mb_params)); + mb_params.cmd = DRV_MSG_CODE_ATTRIBUTE; + SET_MFW_FIELD(mb_params.param, DRV_MB_PARAM_ATTRIBUTE_KEY, + p_drv_attr->attr_num); + SET_MFW_FIELD(mb_params.param, DRV_MB_PARAM_ATTRIBUTE_CMD, + mfw_attr_cmd); + if (p_drv_attr->attr_cmd == ECORE_MCP_DRV_ATTR_CMD_WRITE) { + OSAL_MEM_ZERO(&attr_cmd_write, sizeof(attr_cmd_write)); + attr_cmd_write.val = p_drv_attr->val; + attr_cmd_write.mask = p_drv_attr->mask; + attr_cmd_write.offset = p_drv_attr->offset; + + mb_params.p_data_src = &attr_cmd_write; + mb_params.data_src_size = sizeof(attr_cmd_write); + } + + rc = ecore_mcp_cmd_and_union(p_hwfn, p_ptt, &mb_params); + if (rc != ECORE_SUCCESS) + return rc; + + if (mb_params.mcp_resp == FW_MSG_CODE_UNSUPPORTED) { + DP_INFO(p_hwfn, + "The attribute command is not supported by the MFW\n"); + return ECORE_NOTIMPL; + } else if (mb_params.mcp_resp != FW_MSG_CODE_OK) { + DP_INFO(p_hwfn, + "Failed to send an attribute command [mcp_resp 0x%x, attr_cmd %d, attr_num %d]\n", + mb_params.mcp_resp, p_drv_attr->attr_cmd, + p_drv_attr->attr_num); + return ECORE_INVAL; + } + + DP_VERBOSE(p_hwfn, ECORE_MSG_SP, + "Attribute Command: cmd %d [mfw_cmd %d], num %d, in={val 0x%08x, mask 0x%08x, offset 0x%08x}, out={val 0x%08x}\n", + p_drv_attr->attr_cmd, mfw_attr_cmd, p_drv_attr->attr_num, + p_drv_attr->val, p_drv_attr->mask, p_drv_attr->offset, + mb_params.mcp_param); + + if (p_drv_attr->attr_cmd == ECORE_MCP_DRV_ATTR_CMD_READ || + p_drv_attr->attr_cmd == ECORE_MCP_DRV_ATTR_CMD_READ_CLEAR) + p_drv_attr->val = mb_params.mcp_param; + + return ECORE_SUCCESS; +} + +void ecore_mcp_wol_wr(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, + u32 offset, u32 val) +{ + struct ecore_mcp_mb_params mb_params = {0}; + enum _ecore_status_t rc = ECORE_SUCCESS; + u32 dword = val; + + mb_params.cmd = DRV_MSG_CODE_WRITE_WOL_REG; + mb_params.param = offset; + mb_params.p_data_src = &dword; + mb_params.data_src_size = sizeof(dword); + + rc = ecore_mcp_cmd_and_union(p_hwfn, p_ptt, &mb_params); + if (rc != ECORE_SUCCESS) { + DP_NOTICE(p_hwfn, false, + "Failed to wol write request, rc = %d\n", rc); + } + + if (mb_params.mcp_resp != FW_MSG_CODE_WOL_READ_WRITE_OK) { + DP_NOTICE(p_hwfn, false, + "Failed to write value 0x%x to offset 0x%x [mcp_resp 0x%x]\n", + val, offset, mb_params.mcp_resp); + rc = ECORE_UNKNOWN_ERROR; + } +} diff --git a/dpdk/drivers/net/qede/base/ecore_mcp.h b/dpdk/drivers/net/qede/base/ecore_mcp.h index 448c30bb..6afaf7de 100644 --- a/dpdk/drivers/net/qede/base/ecore_mcp.h +++ b/dpdk/drivers/net/qede/base/ecore_mcp.h @@ -11,7 +11,9 @@ #include "bcm_osal.h" #include "mcp_public.h" +#include "ecore.h" #include "ecore_mcp_api.h" +#include "ecore_dev_api.h" /* Using hwfn number (and not pf_num) is required since in CMT mode, * same pf_num may be used by two different hwfn @@ -23,30 +25,74 @@ ((rel_pfid) | \ ((p_hwfn)->abs_pf_id & 1) << 3) : \ rel_pfid) -#define MCP_PF_ID(p_hwfn) MCP_PF_ID_BY_REL(p_hwfn, (p_hwfn)->rel_pf_id) +#define MCP_PF_ID(p_hwfn) MCP_PF_ID_BY_REL(p_hwfn, (p_hwfn)->rel_pf_id) -/* TODO - this is only correct as long as only BB is supported, and - * no port-swapping is implemented; Afterwards we'll need to fix it. - */ #define MFW_PORT(_p_hwfn) ((_p_hwfn)->abs_pf_id % \ - ((_p_hwfn)->p_dev->num_ports_in_engines * 2)) + ecore_device_num_ports((_p_hwfn)->p_dev)) + struct ecore_mcp_info { - osal_spinlock_t lock; /* Spinlock used for accessing MCP mailbox */ - u32 public_base; /* Address of the MCP public area */ - u32 drv_mb_addr; /* Address of the driver mailbox */ - u32 mfw_mb_addr; /* Address of the MFW mailbox */ - u32 port_addr; /* Address of the port configuration (link) */ - u16 drv_mb_seq; /* Current driver mailbox sequence */ - u16 drv_pulse_seq; /* Current driver pulse sequence */ - struct ecore_mcp_link_params link_input; - struct ecore_mcp_link_state link_output; + /* List for mailbox commands which were sent and wait for a response */ + osal_list_t cmd_list; + + /* Spinlock used for protecting the access to the mailbox commands list + * and the sending of the commands. + */ + osal_spinlock_t cmd_lock; + + /* Flag to indicate whether sending a MFW mailbox command is blocked */ + bool b_block_cmd; + + /* Spinlock used for syncing SW link-changes and link-changes + * originating from attention context. + */ + osal_spinlock_t link_lock; + + /* Address of the MCP public area */ + u32 public_base; + /* Address of the driver mailbox */ + u32 drv_mb_addr; + /* Address of the MFW mailbox */ + u32 mfw_mb_addr; + /* Address of the port configuration (link) */ + u32 port_addr; + + /* Current driver mailbox sequence */ + u16 drv_mb_seq; + /* Current driver pulse sequence */ + u16 drv_pulse_seq; + + struct ecore_mcp_link_params link_input; + struct ecore_mcp_link_state link_output; struct ecore_mcp_link_capabilities link_capabilities; - struct ecore_mcp_function_info func_info; + + struct ecore_mcp_function_info func_info; u8 *mfw_mb_cur; u8 *mfw_mb_shadow; u16 mfw_mb_length; - u16 mcp_hist; + u32 mcp_hist; + + /* Capabilties negotiated with the MFW */ + u32 capabilities; +}; + +struct ecore_mcp_mb_params { + u32 cmd; + u32 param; + void *p_data_src; + u8 data_src_size; + void *p_data_dst; + u8 data_dst_size; + u32 mcp_resp; + u32 mcp_param; +}; + +struct ecore_drv_tlv_hdr { + u8 tlv_type; /* According to the enum below */ + u8 tlv_length; /* In dwords - not including this header */ + u8 tlv_reserved; +#define ECORE_DRV_TLV_FLAGS_CHANGED 0x01 + u8 tlv_flags; }; /** @@ -65,7 +111,7 @@ enum _ecore_status_t ecore_mcp_cmd_init(struct ecore_hwfn *p_hwfn, * * @param p_hwfn * @param p_ptt - * Can only be called after `num_ports_in_engines' is set + * Can only be called after `num_ports_in_engine' is set */ void ecore_mcp_cmd_port_init(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt); @@ -107,32 +153,73 @@ enum _ecore_status_t ecore_mcp_handle_events(struct ecore_hwfn *p_hwfn, * @param p_hwfn - hw function * @param p_ptt - PTT required for register access * @return enum _ecore_status_t - ECORE_SUCCESS - operation - * was successul. + * was successful. */ enum _ecore_status_t ecore_issue_pulse(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt); +enum ecore_drv_role { + ECORE_DRV_ROLE_OS, + ECORE_DRV_ROLE_KDUMP, +}; + +struct ecore_load_req_params { + /* Input params */ + enum ecore_drv_role drv_role; + u8 timeout_val; /* 1..254, '0' - default value, '255' - no timeout */ + bool avoid_eng_reset; + enum ecore_override_force_load override_force_load; + + /* Output params */ + u32 load_code; +}; + /** - * @brief Sends a LOAD_REQ to the MFW, and in case operation - * succeed, returns whether this PF is the first on the - * chip/engine/port or function. This function should be - * called when driver is ready to accept MFW events after - * Storms initializations are done. + * @brief Sends a LOAD_REQ to the MFW, and in case the operation succeeds, + * returns whether this PF is the first on the engine/port or function. * - * @param p_hwfn - hw function - * @param p_ptt - PTT required for register access - * @param p_load_code - The MCP response param containing one - * of the following: - * FW_MSG_CODE_DRV_LOAD_ENGINE - * FW_MSG_CODE_DRV_LOAD_PORT - * FW_MSG_CODE_DRV_LOAD_FUNCTION - * @return enum _ecore_status_t - - * ECORE_SUCCESS - Operation was successul. - * ECORE_BUSY - Operation failed + * @param p_hwfn + * @param p_ptt + * @param p_params + * + * @return enum _ecore_status_t - ECORE_SUCCESS - Operation was successful. */ enum _ecore_status_t ecore_mcp_load_req(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, - u32 *p_load_code); + struct ecore_load_req_params *p_params); + +/** + * @brief Sends a LOAD_DONE message to the MFW + * + * @param p_hwfn + * @param p_ptt + * + * @return enum _ecore_status_t - ECORE_SUCCESS - Operation was successful. + */ +enum _ecore_status_t ecore_mcp_load_done(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt); + +/** + * @brief Sends a UNLOAD_REQ message to the MFW + * + * @param p_hwfn + * @param p_ptt + * + * @return enum _ecore_status_t - ECORE_SUCCESS - Operation was successful. + */ +enum _ecore_status_t ecore_mcp_unload_req(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt); + +/** + * @brief Sends a UNLOAD_DONE message to the MFW + * + * @param p_hwfn + * @param p_ptt + * + * @return enum _ecore_status_t - ECORE_SUCCESS - Operation was successful. + */ +enum _ecore_status_t ecore_mcp_unload_done(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt); /** * @brief Read the MFW mailbox into Current buffer. @@ -140,7 +227,8 @@ enum _ecore_status_t ecore_mcp_load_req(struct ecore_hwfn *p_hwfn, * @param p_hwfn * @param p_ptt */ -void ecore_mcp_read_mb(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt); +void ecore_mcp_read_mb(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt); /** * @brief Ack to mfw that driver finished FLR process for VFs @@ -176,76 +264,6 @@ enum _ecore_status_t ecore_mcp_fill_shmem_func_info(struct ecore_hwfn *p_hwfn, enum _ecore_status_t ecore_mcp_reset(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt); -/** - * @brief - Sets the union data in the MCP mailbox and sends a mailbox command. - * - * @param p_hwfn - hw function - * @param p_ptt - PTT required for register access - * @param cmd - command to be sent to the MCP - * @param param - optional param - * @param p_union_data - pointer to a drv_union_data - * @param o_mcp_resp - the MCP response code (exclude sequence) - * @param o_mcp_param - optional parameter provided by the MCP response - * - * @return enum _ecore_status_t - - * ECORE_SUCCESS - operation was successful - * ECORE_BUSY - operation failed - */ -enum _ecore_status_t ecore_mcp_cmd_and_union(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, - u32 cmd, u32 param, - union drv_union_data *p_union_data, - u32 *o_mcp_resp, - u32 *o_mcp_param); - -/** - * @brief - Sends an NVM write command request to the MFW with - * payload. - * - * @param p_hwfn - * @param p_ptt - * @param cmd - Command: Either DRV_MSG_CODE_NVM_WRITE_NVRAM or - * DRV_MSG_CODE_NVM_PUT_FILE_DATA - * @param param - [0:23] - Offset [24:31] - Size - * @param o_mcp_resp - MCP response - * @param o_mcp_param - MCP response param - * @param i_txn_size - Buffer size - * @param i_buf - Pointer to the buffer - * - * @param return ECORE_SUCCESS upon success. - */ -enum _ecore_status_t ecore_mcp_nvm_wr_cmd(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, - u32 cmd, - u32 param, - u32 *o_mcp_resp, - u32 *o_mcp_param, - u32 i_txn_size, u32 *i_buf); - -/** - * @brief - Sends an NVM read command request to the MFW to get - * a buffer. - * - * @param p_hwfn - * @param p_ptt - * @param cmd - Command: DRV_MSG_CODE_NVM_GET_FILE_DATA or - * DRV_MSG_CODE_NVM_READ_NVRAM commands - * @param param - [0:23] - Offset [24:31] - Size - * @param o_mcp_resp - MCP response - * @param o_mcp_param - MCP response param - * @param o_txn_size - Buffer size output - * @param o_buf - Pointer to the buffer returned by the MFW. - * - * @param return ECORE_SUCCESS upon success. - */ -enum _ecore_status_t ecore_mcp_nvm_rd_cmd(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, - u32 cmd, - u32 param, - u32 *o_mcp_resp, - u32 *o_mcp_param, - u32 *o_txn_size, u32 *o_buf); - /** * @brief indicates whether the MFW objects [under mcp_info] are accessible * @@ -301,4 +319,253 @@ int __ecore_configure_pf_min_bandwidth(struct ecore_hwfn *p_hwfn, enum _ecore_status_t ecore_mcp_mask_parities(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, u32 mask_parities); +/** + * @brief - Sends crash mdump related info to the MFW. + * + * @param p_hwfn + * @param p_ptt + * + * @param return ECORE_SUCCESS upon success. + */ +enum _ecore_status_t ecore_mcp_mdump_set_values(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + u32 epoch); + +/** + * @brief - Triggers a MFW crash dump procedure. + * + * @param p_hwfn + * @param p_ptt + * @param epoch + * + * @param return ECORE_SUCCESS upon success. + */ +enum _ecore_status_t ecore_mcp_mdump_trigger(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt); + +struct ecore_mdump_retain_data { + u32 valid; + u32 epoch; + u32 pf; + u32 status; +}; + +/** + * @brief - Gets the mdump retained data from the MFW. + * + * @param p_hwfn + * @param p_ptt + * @param p_mdump_retain + * + * @param return ECORE_SUCCESS upon success. + */ +enum _ecore_status_t +ecore_mcp_mdump_get_retain(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, + struct ecore_mdump_retain_data *p_mdump_retain); + +/** + * @brief - Sets the MFW's max value for the given resource + * + * @param p_hwfn + * @param p_ptt + * @param res_id + * @param resc_max_val + * @param p_mcp_resp + * + * @return enum _ecore_status_t - ECORE_SUCCESS - operation was successful. + */ +enum _ecore_status_t +ecore_mcp_set_resc_max_val(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, + enum ecore_resources res_id, u32 resc_max_val, + u32 *p_mcp_resp); + +/** + * @brief - Gets the MFW allocation info for the given resource + * + * @param p_hwfn + * @param p_ptt + * @param res_id + * @param p_mcp_resp + * @param p_resc_num + * @param p_resc_start + * + * @return enum _ecore_status_t - ECORE_SUCCESS - operation was successful. + */ +enum _ecore_status_t +ecore_mcp_get_resc_info(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, + enum ecore_resources res_id, u32 *p_mcp_resp, + u32 *p_resc_num, u32 *p_resc_start); + +/** + * @brief - Initiates PF FLR + * + * @param p_hwfn + * @param p_ptt + * + * @param return ECORE_SUCCESS upon success. + */ +enum _ecore_status_t ecore_mcp_initiate_pf_flr(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt); + +#define ECORE_MCP_RESC_LOCK_MIN_VAL RESOURCE_DUMP /* 0 */ +#define ECORE_MCP_RESC_LOCK_MAX_VAL 31 + +enum ecore_resc_lock { + ECORE_RESC_LOCK_DBG_DUMP = ECORE_MCP_RESC_LOCK_MIN_VAL, + /* Locks that the MFW is aware of should be added here downwards */ + + /* Ecore only locks should be added here upwards */ + ECORE_RESC_LOCK_RESC_ALLOC = ECORE_MCP_RESC_LOCK_MAX_VAL, + + /* A dummy value to be used for auxiliary functions in need of + * returning an 'error' value. + */ + ECORE_RESC_LOCK_RESC_INVALID, +}; + +struct ecore_resc_lock_params { + /* Resource number [valid values are 0..31] */ + u8 resource; + + /* Lock timeout value in seconds [default, none or 1..254] */ + u8 timeout; +#define ECORE_MCP_RESC_LOCK_TO_DEFAULT 0 +#define ECORE_MCP_RESC_LOCK_TO_NONE 255 + + /* Number of times to retry locking */ + u8 retry_num; +#define ECORE_MCP_RESC_LOCK_RETRY_CNT_DFLT 10 + + /* The interval in usec between retries */ + u16 retry_interval; +#define ECORE_MCP_RESC_LOCK_RETRY_VAL_DFLT 10000 + + /* Use sleep or delay between retries */ + bool sleep_b4_retry; + + /* Will be set as true if the resource is free and granted */ + bool b_granted; + + /* Will be filled with the resource owner. + * [0..15 = PF0-15, 16 = MFW, 17 = diag over serial] + */ + u8 owner; +}; + +/** + * @brief Acquires MFW generic resource lock + * + * @param p_hwfn + * @param p_ptt + * @param p_params + * + * @return enum _ecore_status_t - ECORE_SUCCESS - operation was successful. + */ +enum _ecore_status_t +ecore_mcp_resc_lock(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, + struct ecore_resc_lock_params *p_params); + +struct ecore_resc_unlock_params { + /* Resource number [valid values are 0..31] */ + u8 resource; + + /* Allow to release a resource even if belongs to another PF */ + bool b_force; + + /* Will be set as true if the resource is released */ + bool b_released; +}; + +/** + * @brief Releases MFW generic resource lock + * + * @param p_hwfn + * @param p_ptt + * @param p_params + * + * @return enum _ecore_status_t - ECORE_SUCCESS - operation was successful. + */ +enum _ecore_status_t +ecore_mcp_resc_unlock(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, + struct ecore_resc_unlock_params *p_params); + +/** + * @brief - default initialization for lock/unlock resource structs + * + * @param p_lock - lock params struct to be initialized; Can be OSAL_NULL + * @param p_unlock - unlock params struct to be initialized; Can be OSAL_NULL + * @param resource - the requested resource + * @paral b_is_permanent - disable retries & aging when set + */ +void ecore_mcp_resc_lock_default_init(struct ecore_resc_lock_params *p_lock, + struct ecore_resc_unlock_params *p_unlock, + enum ecore_resc_lock resource, + bool b_is_permanent); + +/** + * @brief Learn of supported MFW features; To be done during early init + * + * @param p_hwfn + * @param p_ptt + */ +enum _ecore_status_t ecore_mcp_get_capabilities(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt); + +/** + * @brief Inform MFW of set of features supported by driver. Should be done + * inside the contet of the LOAD_REQ. + * + * @param p_hwfn + * @param p_ptt + */ +enum _ecore_status_t ecore_mcp_set_capabilities(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt); + +enum ecore_mcp_drv_attr_cmd { + ECORE_MCP_DRV_ATTR_CMD_READ, + ECORE_MCP_DRV_ATTR_CMD_WRITE, + ECORE_MCP_DRV_ATTR_CMD_READ_CLEAR, + ECORE_MCP_DRV_ATTR_CMD_CLEAR, +}; + +struct ecore_mcp_drv_attr { + enum ecore_mcp_drv_attr_cmd attr_cmd; + u32 attr_num; + + /* R/RC - will be set with the read value + * W - should hold the required value to be written + * C - DC + */ + u32 val; + + /* W - mask/offset to be applied on the given value + * R/RC/C - DC + */ + u32 mask; + u32 offset; +}; + +/** + * @brief Handle the drivers' attributes that are kept by the MFW. + * + * @param p_hwfn + * @param p_ptt + * @param p_drv_attr + */ +enum _ecore_status_t +ecore_mcp_drv_attribute(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, + struct ecore_mcp_drv_attr *p_drv_attr); + +/** + * @brief Read ufp config from the shared memory. + * + * @param p_hwfn + * @param p_ptt + */ +void +ecore_mcp_read_ufp_config(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt); + +void ecore_mcp_wol_wr(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, + u32 offset, u32 val); + #endif /* __ECORE_MCP_H__ */ diff --git a/dpdk/drivers/net/qede/base/ecore_mcp_api.h b/dpdk/drivers/net/qede/base/ecore_mcp_api.h index 7360b356..be3e91f0 100644 --- a/dpdk/drivers/net/qede/base/ecore_mcp_api.h +++ b/dpdk/drivers/net/qede/base/ecore_mcp_api.h @@ -13,8 +13,8 @@ struct ecore_mcp_link_speed_params { bool autoneg; - u32 advertised_speeds; /* bitmask of DRV_SPEED_CAPABILITY */ - u32 forced_speed; /* In Mb/s */ + u32 advertised_speeds; /* bitmask of DRV_SPEED_CAPABILITY */ + u32 forced_speed; /* In Mb/s */ }; struct ecore_mcp_link_pause_params { @@ -23,22 +23,51 @@ struct ecore_mcp_link_pause_params { bool forced_tx; }; +enum ecore_mcp_eee_mode { + ECORE_MCP_EEE_DISABLED, + ECORE_MCP_EEE_ENABLED, + ECORE_MCP_EEE_UNSUPPORTED +}; + +struct ecore_link_eee_params { + u32 tx_lpi_timer; +#define ECORE_EEE_1G_ADV (1 << 0) +#define ECORE_EEE_10G_ADV (1 << 1) + /* Capabilities are represented using ECORE_EEE_*_ADV values */ + u8 adv_caps; + u8 lp_adv_caps; + bool enable; + bool tx_lpi_enable; +}; + struct ecore_mcp_link_params { struct ecore_mcp_link_speed_params speed; struct ecore_mcp_link_pause_params pause; - u32 loopback_mode; /* in PMM_LOOPBACK values */ + u32 loopback_mode; /* in PMM_LOOPBACK values */ + struct ecore_link_eee_params eee; }; struct ecore_mcp_link_capabilities { u32 speed_capabilities; + bool default_speed_autoneg; /* In Mb/s */ + u32 default_speed; /* In Mb/s */ + enum ecore_mcp_eee_mode default_eee; + u32 eee_lpi_timer; + u8 eee_speed_caps; }; struct ecore_mcp_link_state { bool link_up; - u32 line_speed; /* In Mb/s */ - u32 min_pf_rate; /* In Mb/s */ - u32 speed; /* In Mb/s */ + u32 min_pf_rate; /* In Mb/s */ + + /* Actual link speed in Mb/s */ + u32 line_speed; + + /* PF max speed in MB/s, deduced from line_speed + * according to PF max bandwidth configuration. + */ + u32 speed; bool full_duplex; bool an; @@ -65,6 +94,10 @@ struct ecore_mcp_link_state { u8 partner_adv_pause; bool sfp_tx_fault; + + bool eee_active; + u8 eee_adv_caps; + u8 eee_lp_adv_caps; }; struct ecore_mcp_function_info { @@ -82,38 +115,16 @@ struct ecore_mcp_function_info { #define ECORE_MCP_VLAN_UNSET (0xffff) u16 ovlan; + + u16 mtu; }; -struct ecore_mcp_nvm_common { - u32 offset; - u32 param; - u32 resp; - u32 cmd; -}; - -struct ecore_mcp_nvm_rd { - u32 *buf_size; - u32 *buf; -}; - -struct ecore_mcp_nvm_wr { - u32 buf_size; - u32 *buf; -}; - -struct ecore_mcp_nvm_params { -#define ECORE_MCP_CMD (1 << 0) -#define ECORE_MCP_NVM_RD (1 << 1) -#define ECORE_MCP_NVM_WR (1 << 2) - u8 type; - - struct ecore_mcp_nvm_common nvm_common; - - union { - struct ecore_mcp_nvm_rd nvm_rd; - struct ecore_mcp_nvm_wr nvm_wr; - }; +#ifndef __EXTRACT__LINUX__ +enum ecore_nvm_images { + ECORE_NVM_IMAGE_ISCSI_CFG, + ECORE_NVM_IMAGE_FCOE_CFG, }; +#endif struct ecore_mcp_drv_version { u32 version; @@ -128,25 +139,46 @@ struct ecore_mcp_lan_stats { #ifndef ECORE_PROTO_STATS #define ECORE_PROTO_STATS +struct ecore_mcp_fcoe_stats { + u64 rx_pkts; + u64 tx_pkts; + u32 fcs_err; + u32 login_failure; +}; + +struct ecore_mcp_iscsi_stats { + u64 rx_pdus; + u64 tx_pdus; + u64 rx_bytes; + u64 tx_bytes; +}; + +struct ecore_mcp_rdma_stats { + u64 rx_pkts; + u64 tx_pkts; + u64 rx_bytes; + u64 tx_byts; +}; enum ecore_mcp_protocol_type { ECORE_MCP_LAN_STATS, + ECORE_MCP_FCOE_STATS, + ECORE_MCP_ISCSI_STATS, + ECORE_MCP_RDMA_STATS }; union ecore_mcp_protocol_stats { struct ecore_mcp_lan_stats lan_stats; + struct ecore_mcp_fcoe_stats fcoe_stats; + struct ecore_mcp_iscsi_stats iscsi_stats; + struct ecore_mcp_rdma_stats rdma_stats; }; #endif -enum ecore_ov_config_method { - ECORE_OV_CONFIG_MTU, - ECORE_OV_CONFIG_MAC, - ECORE_OV_CONFIG_WOL -}; - enum ecore_ov_client { ECORE_OV_CLIENT_DRV, - ECORE_OV_CLIENT_USER + ECORE_OV_CLIENT_USER, + ECORE_OV_CLIENT_VENDOR_SPEC }; enum ecore_ov_driver_state { @@ -171,6 +203,326 @@ enum ecore_led_mode { }; #endif +struct ecore_temperature_sensor { + u8 sensor_location; + u8 threshold_high; + u8 critical; + u8 current_temp; +}; + +#define ECORE_MAX_NUM_OF_SENSORS 7 +struct ecore_temperature_info { + u32 num_sensors; + struct ecore_temperature_sensor sensors[ECORE_MAX_NUM_OF_SENSORS]; +}; + +enum ecore_mba_img_idx { + ECORE_MBA_LEGACY_IDX, + ECORE_MBA_PCI3CLP_IDX, + ECORE_MBA_PCI3_IDX, + ECORE_MBA_FCODE_IDX, + ECORE_EFI_X86_IDX, + ECORE_EFI_IPF_IDX, + ECORE_EFI_EBC_IDX, + ECORE_EFI_X64_IDX, + ECORE_MAX_NUM_OF_ROMIMG +}; + +struct ecore_mba_vers { + u32 mba_vers[ECORE_MAX_NUM_OF_ROMIMG]; +}; + +enum ecore_mfw_tlv_type { + ECORE_MFW_TLV_GENERIC = 0x1, /* Core driver TLVs */ + ECORE_MFW_TLV_ETH = 0x2, /* L2 driver TLVs */ + ECORE_MFW_TLV_FCOE = 0x4, /* FCoE protocol TLVs */ + ECORE_MFW_TLV_ISCSI = 0x8, /* SCSI protocol TLVs */ + ECORE_MFW_TLV_MAX = 0x16, +}; + +struct ecore_mfw_tlv_generic { + u16 feat_flags; + bool feat_flags_set; + u64 local_mac; + bool local_mac_set; + u64 additional_mac1; + bool additional_mac1_set; + u64 additional_mac2; + bool additional_mac2_set; + u8 drv_state; + bool drv_state_set; + u8 pxe_progress; + bool pxe_progress_set; + u64 rx_frames; + bool rx_frames_set; + u64 rx_bytes; + bool rx_bytes_set; + u64 tx_frames; + bool tx_frames_set; + u64 tx_bytes; + bool tx_bytes_set; +}; + +struct ecore_mfw_tlv_eth { + u16 lso_maxoff_size; + bool lso_maxoff_size_set; + u16 lso_minseg_size; + bool lso_minseg_size_set; + u8 prom_mode; + bool prom_mode_set; + u16 tx_descr_size; + bool tx_descr_size_set; + u16 rx_descr_size; + bool rx_descr_size_set; + u16 netq_count; + bool netq_count_set; + u32 tcp4_offloads; + bool tcp4_offloads_set; + u32 tcp6_offloads; + bool tcp6_offloads_set; + u16 tx_descr_qdepth; + bool tx_descr_qdepth_set; + u16 rx_descr_qdepth; + bool rx_descr_qdepth_set; + u8 iov_offload; + bool iov_offload_set; + u8 txqs_empty; + bool txqs_empty_set; + u8 rxqs_empty; + bool rxqs_empty_set; + u8 num_txqs_full; + bool num_txqs_full_set; + u8 num_rxqs_full; + bool num_rxqs_full_set; +}; + +struct ecore_mfw_tlv_fcoe { + u8 scsi_timeout; + bool scsi_timeout_set; + u32 rt_tov; + bool rt_tov_set; + u32 ra_tov; + bool ra_tov_set; + u32 ed_tov; + bool ed_tov_set; + u32 cr_tov; + bool cr_tov_set; + u8 boot_type; + bool boot_type_set; + u8 npiv_state; + bool npiv_state_set; + u32 num_npiv_ids; + bool num_npiv_ids_set; + u8 switch_name[8]; + bool switch_name_set; + u16 switch_portnum; + bool switch_portnum_set; + u8 switch_portid[3]; + bool switch_portid_set; + u8 vendor_name[8]; + bool vendor_name_set; + u8 switch_model[8]; + bool switch_model_set; + u8 switch_fw_version[8]; + bool switch_fw_version_set; + u8 qos_pri; + bool qos_pri_set; + u8 port_alias[3]; + bool port_alias_set; + u8 port_state; + bool port_state_set; + u16 fip_tx_descr_size; + bool fip_tx_descr_size_set; + u16 fip_rx_descr_size; + bool fip_rx_descr_size_set; + u16 link_failures; + bool link_failures_set; + u8 fcoe_boot_progress; + bool fcoe_boot_progress_set; + u64 rx_bcast; + bool rx_bcast_set; + u64 tx_bcast; + bool tx_bcast_set; + u16 fcoe_txq_depth; + bool fcoe_txq_depth_set; + u16 fcoe_rxq_depth; + bool fcoe_rxq_depth_set; + u64 fcoe_rx_frames; + bool fcoe_rx_frames_set; + u64 fcoe_rx_bytes; + bool fcoe_rx_bytes_set; + u64 fcoe_tx_frames; + bool fcoe_tx_frames_set; + u64 fcoe_tx_bytes; + bool fcoe_tx_bytes_set; + u16 crc_count; + bool crc_count_set; + u32 crc_err_src_fcid[5]; + bool crc_err_src_fcid_set[5]; + u8 crc_err_tstamp[5][14]; + bool crc_err_tstamp_set[5]; + u16 losync_err; + bool losync_err_set; + u16 losig_err; + bool losig_err_set; + u16 primtive_err; + bool primtive_err_set; + u16 disparity_err; + bool disparity_err_set; + u16 code_violation_err; + bool code_violation_err_set; + u32 flogi_param[4]; + bool flogi_param_set[4]; + u8 flogi_tstamp[14]; + bool flogi_tstamp_set; + u32 flogi_acc_param[4]; + bool flogi_acc_param_set[4]; + u8 flogi_acc_tstamp[14]; + bool flogi_acc_tstamp_set; + u32 flogi_rjt; + bool flogi_rjt_set; + u8 flogi_rjt_tstamp[14]; + bool flogi_rjt_tstamp_set; + u32 fdiscs; + bool fdiscs_set; + u8 fdisc_acc; + bool fdisc_acc_set; + u8 fdisc_rjt; + bool fdisc_rjt_set; + u8 plogi; + bool plogi_set; + u8 plogi_acc; + bool plogi_acc_set; + u8 plogi_rjt; + bool plogi_rjt_set; + u32 plogi_dst_fcid[5]; + bool plogi_dst_fcid_set[5]; + u8 plogi_tstamp[5][14]; + bool plogi_tstamp_set[5]; + u32 plogi_acc_src_fcid[5]; + bool plogi_acc_src_fcid_set[5]; + u8 plogi_acc_tstamp[5][14]; + bool plogi_acc_tstamp_set[5]; + u8 tx_plogos; + bool tx_plogos_set; + u8 plogo_acc; + bool plogo_acc_set; + u8 plogo_rjt; + bool plogo_rjt_set; + u32 plogo_src_fcid[5]; + bool plogo_src_fcid_set[5]; + u8 plogo_tstamp[5][14]; + bool plogo_tstamp_set[5]; + u8 rx_logos; + bool rx_logos_set; + u8 tx_accs; + bool tx_accs_set; + u8 tx_prlis; + bool tx_prlis_set; + u8 rx_accs; + bool rx_accs_set; + u8 tx_abts; + bool tx_abts_set; + u8 rx_abts_acc; + bool rx_abts_acc_set; + u8 rx_abts_rjt; + bool rx_abts_rjt_set; + u32 abts_dst_fcid[5]; + bool abts_dst_fcid_set[5]; + u8 abts_tstamp[5][14]; + bool abts_tstamp_set[5]; + u8 rx_rscn; + bool rx_rscn_set; + u32 rx_rscn_nport[4]; + bool rx_rscn_nport_set[4]; + u8 tx_lun_rst; + bool tx_lun_rst_set; + u8 abort_task_sets; + bool abort_task_sets_set; + u8 tx_tprlos; + bool tx_tprlos_set; + u8 tx_nos; + bool tx_nos_set; + u8 rx_nos; + bool rx_nos_set; + u8 ols; + bool ols_set; + u8 lr; + bool lr_set; + u8 lrr; + bool lrr_set; + u8 tx_lip; + bool tx_lip_set; + u8 rx_lip; + bool rx_lip_set; + u8 eofa; + bool eofa_set; + u8 eofni; + bool eofni_set; + u8 scsi_chks; + bool scsi_chks_set; + u8 scsi_cond_met; + bool scsi_cond_met_set; + u8 scsi_busy; + bool scsi_busy_set; + u8 scsi_inter; + bool scsi_inter_set; + u8 scsi_inter_cond_met; + bool scsi_inter_cond_met_set; + u8 scsi_rsv_conflicts; + bool scsi_rsv_conflicts_set; + u8 scsi_tsk_full; + bool scsi_tsk_full_set; + u8 scsi_aca_active; + bool scsi_aca_active_set; + u8 scsi_tsk_abort; + bool scsi_tsk_abort_set; + u32 scsi_rx_chk[5]; + bool scsi_rx_chk_set[5]; + u8 scsi_chk_tstamp[5][14]; + bool scsi_chk_tstamp_set[5]; +}; + +struct ecore_mfw_tlv_iscsi { + u8 target_llmnr; + bool target_llmnr_set; + u8 header_digest; + bool header_digest_set; + u8 data_digest; + bool data_digest_set; + u8 auth_method; + bool auth_method_set; + u16 boot_taget_portal; + bool boot_taget_portal_set; + u16 frame_size; + bool frame_size_set; + u16 tx_desc_size; + bool tx_desc_size_set; + u16 rx_desc_size; + bool rx_desc_size_set; + u8 boot_progress; + bool boot_progress_set; + u16 tx_desc_qdepth; + bool tx_desc_qdepth_set; + u16 rx_desc_qdepth; + bool rx_desc_qdepth_set; + u64 rx_frames; + bool rx_frames_set; + u64 rx_bytes; + bool rx_bytes_set; + u64 tx_frames; + bool tx_frames_set; + u64 tx_bytes; + bool tx_bytes_set; +}; + +union ecore_mfw_tlv_data { + struct ecore_mfw_tlv_generic generic; + struct ecore_mfw_tlv_eth eth; + struct ecore_mfw_tlv_fcoe fcoe; + struct ecore_mfw_tlv_iscsi iscsi; +}; + /** * @brief - returns the link params of the hw function * @@ -209,19 +561,20 @@ struct ecore_mcp_link_capabilities * @return enum _ecore_status_t */ enum _ecore_status_t ecore_mcp_set_link(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, bool b_up); + struct ecore_ptt *p_ptt, + bool b_up); /** * @brief Get the management firmware version value * - * @param p_dev - ecore dev pointer + * @param p_hwfn * @param p_ptt * @param p_mfw_ver - mfw version value * @param p_running_bundle_id - image id in nvram; Optional. * * @return enum _ecore_status_t - ECORE_SUCCESS - operation was successful. */ -enum _ecore_status_t ecore_mcp_get_mfw_ver(struct ecore_dev *p_dev, +enum _ecore_status_t ecore_mcp_get_mfw_ver(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, u32 *p_mfw_ver, u32 *p_running_bundle_id); @@ -230,13 +583,15 @@ enum _ecore_status_t ecore_mcp_get_mfw_ver(struct ecore_dev *p_dev, * @brief Get media type value of the port. * * @param p_dev - ecore dev pointer + * @param p_ptt * @param mfw_ver - media type value * * @return enum _ecore_status_t - * ECORE_SUCCESS - Operation was successful. * ECORE_BUSY - Operation failed */ -enum _ecore_status_t ecore_mcp_get_media_type(struct ecore_dev *p_dev, +enum _ecore_status_t ecore_mcp_get_media_type(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, u32 *media_type); /** @@ -245,9 +600,9 @@ enum _ecore_status_t ecore_mcp_get_media_type(struct ecore_dev *p_dev, * @param p_hwfn - hw function * @param p_ptt - PTT required for register access * @param cmd - command to be sent to the MCP - * @param param - optional param - * @param o_mcp_resp - the MCP response code (exclude sequence) - * @param o_mcp_param - optional parameter provided by the MCP response + * @param param - Optional param + * @param o_mcp_resp - The MCP response code (exclude sequence) + * @param o_mcp_param - Optional parameter provided by the MCP response * * @return enum _ecore_status_t - * ECORE_SUCCESS - operation was successful @@ -267,6 +622,7 @@ enum _ecore_status_t ecore_mcp_cmd(struct ecore_hwfn *p_hwfn, enum _ecore_status_t ecore_mcp_drain(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt); +#ifndef LINUX_REMOVE /** * @brief - return the mcp function info of the hw function * @@ -276,45 +632,9 @@ enum _ecore_status_t ecore_mcp_drain(struct ecore_hwfn *p_hwfn, */ const struct ecore_mcp_function_info *ecore_mcp_get_function_info(struct ecore_hwfn *p_hwfn); +#endif -/** - * @brief - Function for reading/manipulating the nvram. Following are supported - * functionalities. - * 1. Read: Read the specified nvram offset. - * input values: - * type - ECORE_MCP_NVM_RD - * cmd - command code (e.g. DRV_MSG_CODE_NVM_READ_NVRAM) - * offset - nvm offset - * - * output values: - * buf - buffer - * buf_size - buffer size - * - * 2. Write: Write the data at the specified nvram offset - * input values: - * type - ECORE_MCP_NVM_WR - * cmd - command code (e.g. DRV_MSG_CODE_NVM_WRITE_NVRAM) - * offset - nvm offset - * buf - buffer - * buf_size - buffer size - * - * 3. Command: Send the NVM command to MCP. - * input values: - * type - ECORE_MCP_CMD - * cmd - command code (e.g. DRV_MSG_CODE_NVM_DEL_FILE) - * offset - nvm offset - * - * - * @param p_hwfn - * @param p_ptt - * @param params - * - * @return ECORE_SUCCESS - operation was successful. - */ -enum _ecore_status_t ecore_mcp_nvm_command(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, - struct ecore_mcp_nvm_params *params); - +#ifndef LINUX_REMOVE /** * @brief - count number of function with a matching personality on engine. * @@ -326,7 +646,9 @@ enum _ecore_status_t ecore_mcp_nvm_command(struct ecore_hwfn *p_hwfn, * the bitsmasks. */ int ecore_mcp_get_personality_cnt(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, u32 personalities); + struct ecore_ptt *p_ptt, + u32 personalities); +#endif /** * @brief Get the flash size value @@ -377,12 +699,22 @@ u32 ecore_get_process_kill_counter(struct ecore_hwfn *p_hwfn, enum _ecore_status_t ecore_start_recovery_process(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt); +/** + * @brief A recovery handler must call this function as its first step. + * It is assumed that the handler is not run from an interrupt context. + * + * @param p_dev + * @param p_ptt + * + * @return enum _ecore_status_t + */ +enum _ecore_status_t ecore_recovery_prolog(struct ecore_dev *p_dev); + /** * @brief Notify MFW about the change in base device properties * * @param p_hwfn * @param p_ptt - * @param config - Configuation that has been updated * @param client - ecore client type * * @return enum _ecore_status_t - ECORE_SUCCESS - operation was successful. @@ -390,7 +722,6 @@ enum _ecore_status_t ecore_start_recovery_process(struct ecore_hwfn *p_hwfn, enum _ecore_status_t ecore_mcp_ov_update_current_config(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, - enum ecore_ov_config_method config, enum ecore_ov_client client); /** @@ -505,7 +836,8 @@ enum _ecore_status_t ecore_mcp_nvm_put_file_begin(struct ecore_dev *p_dev, * * @return enum _ecore_status_t - ECORE_SUCCESS - operation was successful. */ -enum _ecore_status_t ecore_mcp_nvm_del_file(struct ecore_dev *p_dev, u32 addr); +enum _ecore_status_t ecore_mcp_nvm_del_file(struct ecore_dev *p_dev, + u32 addr); /** * @brief Check latest response @@ -523,7 +855,7 @@ enum _ecore_status_t ecore_mcp_nvm_resp(struct ecore_dev *p_dev, u8 *p_buf); * @param p_dev * @param addr - nvm offset * @param cmd - nvm command - * @param p_buf - nvm write buffer + * @param p_buf - nvm read buffer * @param len - buffer len * * @return enum _ecore_status_t - ECORE_SUCCESS - operation was successful. @@ -536,13 +868,63 @@ enum _ecore_status_t ecore_mcp_phy_read(struct ecore_dev *p_dev, u32 cmd, * * @param p_dev * @param addr - nvm offset - * @param p_buf - nvm write buffer + * @param p_buf - nvm read buffer * @param len - buffer len * * @return enum _ecore_status_t - ECORE_SUCCESS - operation was successful. */ enum _ecore_status_t ecore_mcp_nvm_read(struct ecore_dev *p_dev, u32 addr, - u8 *p_buf, u32 len); + u8 *p_buf, u32 len); + +/** + * @brief - Sends an NVM write command request to the MFW with + * payload. + * + * @param p_hwfn + * @param p_ptt + * @param cmd - Command: Either DRV_MSG_CODE_NVM_WRITE_NVRAM or + * DRV_MSG_CODE_NVM_PUT_FILE_DATA + * @param param - [0:23] - Offset [24:31] - Size + * @param o_mcp_resp - MCP response + * @param o_mcp_param - MCP response param + * @param i_txn_size - Buffer size + * @param i_buf - Pointer to the buffer + * + * @param return ECORE_SUCCESS upon success. + */ +enum _ecore_status_t ecore_mcp_nvm_wr_cmd(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + u32 cmd, + u32 param, + u32 *o_mcp_resp, + u32 *o_mcp_param, + u32 i_txn_size, + u32 *i_buf); + +/** + * @brief - Sends an NVM read command request to the MFW to get + * a buffer. + * + * @param p_hwfn + * @param p_ptt + * @param cmd - Command: DRV_MSG_CODE_NVM_GET_FILE_DATA or + * DRV_MSG_CODE_NVM_READ_NVRAM commands + * @param param - [0:23] - Offset [24:31] - Size + * @param o_mcp_resp - MCP response + * @param o_mcp_param - MCP response param + * @param o_txn_size - Buffer size output + * @param o_buf - Pointer to the buffer returned by the MFW. + * + * @param return ECORE_SUCCESS upon success. + */ +enum _ecore_status_t ecore_mcp_nvm_rd_cmd(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + u32 cmd, + u32 param, + u32 *o_mcp_resp, + u32 *o_mcp_param, + u32 *o_txn_size, + u32 *o_buf); /** * @brief Read from sfp @@ -608,4 +990,182 @@ enum _ecore_status_t ecore_mcp_gpio_write(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, u16 gpio, u16 gpio_val); +/** + * @brief Gpio get information + * + * @param p_hwfn - hw function + * @param p_ptt - PTT required for register access + * @param gpio - gpio number + * @param gpio_direction - gpio is output (0) or input (1) + * @param gpio_ctrl - gpio control is uninitialized (0), + * path 0 (1), path 1 (2) or shared(3) + * + * @return enum _ecore_status_t - ECORE_SUCCESS - operation was successful. + */ +enum _ecore_status_t ecore_mcp_gpio_info(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + u16 gpio, u32 *gpio_direction, + u32 *gpio_ctrl); + +/** + * @brief Bist register test + * + * @param p_hwfn - hw function + * @param p_ptt - PTT required for register access + * + * @return enum _ecore_status_t - ECORE_SUCCESS - operation was successful. + */ +enum _ecore_status_t ecore_mcp_bist_register_test(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt); + +/** + * @brief Bist clock test + * + * @param p_hwfn - hw function + * @param p_ptt - PTT required for register access + * + * @return enum _ecore_status_t - ECORE_SUCCESS - operation was successful. + */ +enum _ecore_status_t ecore_mcp_bist_clock_test(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt); + +/** + * @brief Bist nvm test - get number of images + * + * @param p_hwfn - hw function + * @param p_ptt - PTT required for register access + * @param num_images - number of images if operation was + * successful. 0 if not. + * + * @return enum _ecore_status_t - ECORE_SUCCESS - operation was successful. + */ +enum _ecore_status_t ecore_mcp_bist_nvm_test_get_num_images( + struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + u32 *num_images); + +/** + * @brief Bist nvm test - get image attributes by index + * + * @param p_hwfn - hw function + * @param p_ptt - PTT required for register access + * @param p_image_att - Attributes of image + * @param image_index - Index of image to get information for + * + * @return enum _ecore_status_t - ECORE_SUCCESS - operation was successful. + */ +enum _ecore_status_t ecore_mcp_bist_nvm_test_get_image_att( + struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + struct bist_nvm_image_att *p_image_att, + u32 image_index); + +/** + * @brief ecore_mcp_get_temperature_info - get the status of the temperature + * sensors + * + * @param p_hwfn - hw function + * @param p_ptt - PTT required for register access + * @param p_temp_status - A pointer to an ecore_temperature_info structure to + * be filled with the temperature data + * + * @return enum _ecore_status_t - ECORE_SUCCESS - operation was successful. + */ +enum _ecore_status_t +ecore_mcp_get_temperature_info(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + struct ecore_temperature_info *p_temp_info); + +/** + * @brief Get MBA versions - get MBA sub images versions + * + * @param p_hwfn - hw function + * @param p_ptt - PTT required for register access + * @param p_mba_vers - MBA versions array to fill + * + * @return enum _ecore_status_t - ECORE_SUCCESS - operation was successful. + */ +enum _ecore_status_t ecore_mcp_get_mba_versions( + struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + struct ecore_mba_vers *p_mba_vers); + +/** + * @brief Count memory ecc events + * + * @param p_hwfn - hw function + * @param p_ptt - PTT required for register access + * @param num_events - number of memory ecc events + * + * @return enum _ecore_status_t - ECORE_SUCCESS - operation was successful. + */ +enum _ecore_status_t ecore_mcp_mem_ecc_events(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + u64 *num_events); + +struct ecore_mdump_info { + u32 reason; + u32 version; + u32 config; + u32 epoch; + u32 num_of_logs; + u32 valid_logs; +}; + +/** + * @brief - Gets the MFW crash dump configuration and logs info. + * + * @param p_hwfn + * @param p_ptt + * @param p_mdump_info + * + * @param return ECORE_SUCCESS upon success. + */ +enum _ecore_status_t +ecore_mcp_mdump_get_info(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, + struct ecore_mdump_info *p_mdump_info); + +/** + * @brief - Clears the MFW crash dump logs. + * + * @param p_hwfn + * @param p_ptt + * + * @param return ECORE_SUCCESS upon success. + */ +enum _ecore_status_t ecore_mcp_mdump_clear_logs(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt); + +/** + * @brief - Clear the mdump retained data. + * + * @param p_hwfn + * @param p_ptt + * + * @param return ECORE_SUCCESS upon success. + */ +enum _ecore_status_t ecore_mcp_mdump_clr_retain(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt); + +/** + * @brief - Processes the TLV request from MFW i.e., get the required TLV info + * from the ecore client and send it to the MFW. + * + * @param p_hwfn + * @param p_ptt + * + * @param return ECORE_SUCCESS upon success. + */ +enum _ecore_status_t ecore_mfw_process_tlv_req(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt); + + +/** + * @brief - Return whether management firmware support smart AN + * + * @param p_hwfn + * + * @return bool - true iff feature is supported. + */ +bool ecore_mcp_is_smart_an_supported(struct ecore_hwfn *p_hwfn); #endif diff --git a/dpdk/drivers/net/qede/base/ecore_mng_tlv.c b/dpdk/drivers/net/qede/base/ecore_mng_tlv.c new file mode 100644 index 00000000..3a1de094 --- /dev/null +++ b/dpdk/drivers/net/qede/base/ecore_mng_tlv.c @@ -0,0 +1,1534 @@ +#include "bcm_osal.h" +#include "ecore.h" +#include "ecore_status.h" +#include "ecore_mcp.h" +#include "ecore_hw.h" +#include "reg_addr.h" + +#define TLV_TYPE(p) (p[0]) +#define TLV_LENGTH(p) (p[1]) +#define TLV_FLAGS(p) (p[3]) + +static enum _ecore_status_t +ecore_mfw_get_tlv_group(u8 tlv_type, u8 *tlv_group) +{ + switch (tlv_type) { + case DRV_TLV_FEATURE_FLAGS: + case DRV_TLV_LOCAL_ADMIN_ADDR: + case DRV_TLV_ADDITIONAL_MAC_ADDR_1: + case DRV_TLV_ADDITIONAL_MAC_ADDR_2: + case DRV_TLV_OS_DRIVER_STATES: + case DRV_TLV_PXE_BOOT_PROGRESS: + case DRV_TLV_RX_FRAMES_RECEIVED: + case DRV_TLV_RX_BYTES_RECEIVED: + case DRV_TLV_TX_FRAMES_SENT: + case DRV_TLV_TX_BYTES_SENT: + *tlv_group |= ECORE_MFW_TLV_GENERIC; + break; + case DRV_TLV_LSO_MAX_OFFLOAD_SIZE: + case DRV_TLV_LSO_MIN_SEGMENT_COUNT: + case DRV_TLV_PROMISCUOUS_MODE: + case DRV_TLV_TX_DESCRIPTORS_QUEUE_SIZE: + case DRV_TLV_RX_DESCRIPTORS_QUEUE_SIZE: + case DRV_TLV_NUM_OF_NET_QUEUE_VMQ_CFG: + case DRV_TLV_NUM_OFFLOADED_CONNECTIONS_TCP_IPV4: + case DRV_TLV_NUM_OFFLOADED_CONNECTIONS_TCP_IPV6: + case DRV_TLV_TX_DESCRIPTOR_QUEUE_AVG_DEPTH: + case DRV_TLV_RX_DESCRIPTORS_QUEUE_AVG_DEPTH: + case DRV_TLV_IOV_OFFLOAD: + case DRV_TLV_TX_QUEUES_EMPTY: + case DRV_TLV_RX_QUEUES_EMPTY: + case DRV_TLV_TX_QUEUES_FULL: + case DRV_TLV_RX_QUEUES_FULL: + *tlv_group |= ECORE_MFW_TLV_ETH; + break; + case DRV_TLV_SCSI_TO: + case DRV_TLV_R_T_TOV: + case DRV_TLV_R_A_TOV: + case DRV_TLV_E_D_TOV: + case DRV_TLV_CR_TOV: + case DRV_TLV_BOOT_TYPE: + case DRV_TLV_NPIV_STATE: + case DRV_TLV_NUM_OF_NPIV_IDS: + case DRV_TLV_SWITCH_NAME: + case DRV_TLV_SWITCH_PORT_NUM: + case DRV_TLV_SWITCH_PORT_ID: + case DRV_TLV_VENDOR_NAME: + case DRV_TLV_SWITCH_MODEL: + case DRV_TLV_SWITCH_FW_VER: + case DRV_TLV_QOS_PRIORITY_PER_802_1P: + case DRV_TLV_PORT_ALIAS: + case DRV_TLV_PORT_STATE: + case DRV_TLV_FIP_TX_DESCRIPTORS_QUEUE_SIZE: + case DRV_TLV_FCOE_RX_DESCRIPTORS_QUEUE_SIZE: + case DRV_TLV_LINK_FAILURE_COUNT: + case DRV_TLV_FCOE_BOOT_PROGRESS: + case DRV_TLV_RX_BROADCAST_PACKETS: + case DRV_TLV_TX_BROADCAST_PACKETS: + case DRV_TLV_FCOE_TX_DESCRIPTOR_QUEUE_AVG_DEPTH: + case DRV_TLV_FCOE_RX_DESCRIPTORS_QUEUE_AVG_DEPTH: + case DRV_TLV_FCOE_RX_FRAMES_RECEIVED: + case DRV_TLV_FCOE_RX_BYTES_RECEIVED: + case DRV_TLV_FCOE_TX_FRAMES_SENT: + case DRV_TLV_FCOE_TX_BYTES_SENT: + case DRV_TLV_CRC_ERROR_COUNT: + case DRV_TLV_CRC_ERROR_1_RECEIVED_SOURCE_FC_ID: + case DRV_TLV_CRC_ERROR_1_TIMESTAMP: + case DRV_TLV_CRC_ERROR_2_RECEIVED_SOURCE_FC_ID: + case DRV_TLV_CRC_ERROR_2_TIMESTAMP: + case DRV_TLV_CRC_ERROR_3_RECEIVED_SOURCE_FC_ID: + case DRV_TLV_CRC_ERROR_3_TIMESTAMP: + case DRV_TLV_CRC_ERROR_4_RECEIVED_SOURCE_FC_ID: + case DRV_TLV_CRC_ERROR_4_TIMESTAMP: + case DRV_TLV_CRC_ERROR_5_RECEIVED_SOURCE_FC_ID: + case DRV_TLV_CRC_ERROR_5_TIMESTAMP: + case DRV_TLV_LOSS_OF_SYNC_ERROR_COUNT: + case DRV_TLV_LOSS_OF_SIGNAL_ERRORS: + case DRV_TLV_PRIMITIVE_SEQUENCE_PROTOCOL_ERROR_COUNT: + case DRV_TLV_DISPARITY_ERROR_COUNT: + case DRV_TLV_CODE_VIOLATION_ERROR_COUNT: + case DRV_TLV_LAST_FLOGI_ISSUED_COMMON_PARAMETERS_WORD_1: + case DRV_TLV_LAST_FLOGI_ISSUED_COMMON_PARAMETERS_WORD_2: + case DRV_TLV_LAST_FLOGI_ISSUED_COMMON_PARAMETERS_WORD_3: + case DRV_TLV_LAST_FLOGI_ISSUED_COMMON_PARAMETERS_WORD_4: + case DRV_TLV_LAST_FLOGI_TIMESTAMP: + case DRV_TLV_LAST_FLOGI_ACC_COMMON_PARAMETERS_WORD_1: + case DRV_TLV_LAST_FLOGI_ACC_COMMON_PARAMETERS_WORD_2: + case DRV_TLV_LAST_FLOGI_ACC_COMMON_PARAMETERS_WORD_3: + case DRV_TLV_LAST_FLOGI_ACC_COMMON_PARAMETERS_WORD_4: + case DRV_TLV_LAST_FLOGI_ACC_TIMESTAMP: + case DRV_TLV_LAST_FLOGI_RJT: + case DRV_TLV_LAST_FLOGI_RJT_TIMESTAMP: + case DRV_TLV_FDISCS_SENT_COUNT: + case DRV_TLV_FDISC_ACCS_RECEIVED: + case DRV_TLV_FDISC_RJTS_RECEIVED: + case DRV_TLV_PLOGI_SENT_COUNT: + case DRV_TLV_PLOGI_ACCS_RECEIVED: + case DRV_TLV_PLOGI_RJTS_RECEIVED: + case DRV_TLV_PLOGI_1_SENT_DESTINATION_FC_ID: + case DRV_TLV_PLOGI_1_TIMESTAMP: + case DRV_TLV_PLOGI_2_SENT_DESTINATION_FC_ID: + case DRV_TLV_PLOGI_2_TIMESTAMP: + case DRV_TLV_PLOGI_3_SENT_DESTINATION_FC_ID: + case DRV_TLV_PLOGI_3_TIMESTAMP: + case DRV_TLV_PLOGI_4_SENT_DESTINATION_FC_ID: + case DRV_TLV_PLOGI_4_TIMESTAMP: + case DRV_TLV_PLOGI_5_SENT_DESTINATION_FC_ID: + case DRV_TLV_PLOGI_5_TIMESTAMP: + case DRV_TLV_PLOGI_1_ACC_RECEIVED_SOURCE_FC_ID: + case DRV_TLV_PLOGI_1_ACC_TIMESTAMP: + case DRV_TLV_PLOGI_2_ACC_RECEIVED_SOURCE_FC_ID: + case DRV_TLV_PLOGI_2_ACC_TIMESTAMP: + case DRV_TLV_PLOGI_3_ACC_RECEIVED_SOURCE_FC_ID: + case DRV_TLV_PLOGI_3_ACC_TIMESTAMP: + case DRV_TLV_PLOGI_4_ACC_RECEIVED_SOURCE_FC_ID: + case DRV_TLV_PLOGI_4_ACC_TIMESTAMP: + case DRV_TLV_PLOGI_5_ACC_RECEIVED_SOURCE_FC_ID: + case DRV_TLV_PLOGI_5_ACC_TIMESTAMP: + case DRV_TLV_LOGOS_ISSUED: + case DRV_TLV_LOGO_ACCS_RECEIVED: + case DRV_TLV_LOGO_RJTS_RECEIVED: + case DRV_TLV_LOGO_1_RECEIVED_SOURCE_FC_ID: + case DRV_TLV_LOGO_1_TIMESTAMP: + case DRV_TLV_LOGO_2_RECEIVED_SOURCE_FC_ID: + case DRV_TLV_LOGO_2_TIMESTAMP: + case DRV_TLV_LOGO_3_RECEIVED_SOURCE_FC_ID: + case DRV_TLV_LOGO_3_TIMESTAMP: + case DRV_TLV_LOGO_4_RECEIVED_SOURCE_FC_ID: + case DRV_TLV_LOGO_4_TIMESTAMP: + case DRV_TLV_LOGO_5_RECEIVED_SOURCE_FC_ID: + case DRV_TLV_LOGO_5_TIMESTAMP: + case DRV_TLV_LOGOS_RECEIVED: + case DRV_TLV_ACCS_ISSUED: + case DRV_TLV_PRLIS_ISSUED: + case DRV_TLV_ACCS_RECEIVED: + case DRV_TLV_ABTS_SENT_COUNT: + case DRV_TLV_ABTS_ACCS_RECEIVED: + case DRV_TLV_ABTS_RJTS_RECEIVED: + case DRV_TLV_ABTS_1_SENT_DESTINATION_FC_ID: + case DRV_TLV_ABTS_1_TIMESTAMP: + case DRV_TLV_ABTS_2_SENT_DESTINATION_FC_ID: + case DRV_TLV_ABTS_2_TIMESTAMP: + case DRV_TLV_ABTS_3_SENT_DESTINATION_FC_ID: + case DRV_TLV_ABTS_3_TIMESTAMP: + case DRV_TLV_ABTS_4_SENT_DESTINATION_FC_ID: + case DRV_TLV_ABTS_4_TIMESTAMP: + case DRV_TLV_ABTS_5_SENT_DESTINATION_FC_ID: + case DRV_TLV_ABTS_5_TIMESTAMP: + case DRV_TLV_RSCNS_RECEIVED: + case DRV_TLV_LAST_RSCN_RECEIVED_N_PORT_1: + case DRV_TLV_LAST_RSCN_RECEIVED_N_PORT_2: + case DRV_TLV_LAST_RSCN_RECEIVED_N_PORT_3: + case DRV_TLV_LAST_RSCN_RECEIVED_N_PORT_4: + case DRV_TLV_LUN_RESETS_ISSUED: + case DRV_TLV_ABORT_TASK_SETS_ISSUED: + case DRV_TLV_TPRLOS_SENT: + case DRV_TLV_NOS_SENT_COUNT: + case DRV_TLV_NOS_RECEIVED_COUNT: + case DRV_TLV_OLS_COUNT: + case DRV_TLV_LR_COUNT: + case DRV_TLV_LRR_COUNT: + case DRV_TLV_LIP_SENT_COUNT: + case DRV_TLV_LIP_RECEIVED_COUNT: + case DRV_TLV_EOFA_COUNT: + case DRV_TLV_EOFNI_COUNT: + case DRV_TLV_SCSI_STATUS_CHECK_CONDITION_COUNT: + case DRV_TLV_SCSI_STATUS_CONDITION_MET_COUNT: + case DRV_TLV_SCSI_STATUS_BUSY_COUNT: + case DRV_TLV_SCSI_STATUS_INTERMEDIATE_COUNT: + case DRV_TLV_SCSI_STATUS_INTERMEDIATE_CONDITION_MET_COUNT: + case DRV_TLV_SCSI_STATUS_RESERVATION_CONFLICT_COUNT: + case DRV_TLV_SCSI_STATUS_TASK_SET_FULL_COUNT: + case DRV_TLV_SCSI_STATUS_ACA_ACTIVE_COUNT: + case DRV_TLV_SCSI_STATUS_TASK_ABORTED_COUNT: + case DRV_TLV_SCSI_CHECK_CONDITION_1_RECEIVED_SK_ASC_ASCQ: + case DRV_TLV_SCSI_CHECK_1_TIMESTAMP: + case DRV_TLV_SCSI_CHECK_CONDITION_2_RECEIVED_SK_ASC_ASCQ: + case DRV_TLV_SCSI_CHECK_2_TIMESTAMP: + case DRV_TLV_SCSI_CHECK_CONDITION_3_RECEIVED_SK_ASC_ASCQ: + case DRV_TLV_SCSI_CHECK_3_TIMESTAMP: + case DRV_TLV_SCSI_CHECK_CONDITION_4_RECEIVED_SK_ASC_ASCQ: + case DRV_TLV_SCSI_CHECK_4_TIMESTAMP: + case DRV_TLV_SCSI_CHECK_CONDITION_5_RECEIVED_SK_ASC_ASCQ: + case DRV_TLV_SCSI_CHECK_5_TIMESTAMP: + *tlv_group = ECORE_MFW_TLV_FCOE; + break; + case DRV_TLV_TARGET_LLMNR_ENABLED: + case DRV_TLV_HEADER_DIGEST_FLAG_ENABLED: + case DRV_TLV_DATA_DIGEST_FLAG_ENABLED: + case DRV_TLV_AUTHENTICATION_METHOD: + case DRV_TLV_ISCSI_BOOT_TARGET_PORTAL: + case DRV_TLV_MAX_FRAME_SIZE: + case DRV_TLV_PDU_TX_DESCRIPTORS_QUEUE_SIZE: + case DRV_TLV_PDU_RX_DESCRIPTORS_QUEUE_SIZE: + case DRV_TLV_ISCSI_BOOT_PROGRESS: + case DRV_TLV_PDU_TX_DESCRIPTOR_QUEUE_AVG_DEPTH: + case DRV_TLV_PDU_RX_DESCRIPTORS_QUEUE_AVG_DEPTH: + case DRV_TLV_ISCSI_PDU_RX_FRAMES_RECEIVED: + case DRV_TLV_ISCSI_PDU_RX_BYTES_RECEIVED: + case DRV_TLV_ISCSI_PDU_TX_FRAMES_SENT: + case DRV_TLV_ISCSI_PDU_TX_BYTES_SENT: + *tlv_group |= ECORE_MFW_TLV_ISCSI; + break; + default: + return ECORE_INVAL; + } + + return ECORE_SUCCESS; +} + +static int +ecore_mfw_get_gen_tlv_value(struct ecore_drv_tlv_hdr *p_tlv, + struct ecore_mfw_tlv_generic *p_drv_buf, + u8 **p_tlv_buf) +{ + switch (p_tlv->tlv_type) { + case DRV_TLV_FEATURE_FLAGS: + if (p_drv_buf->feat_flags_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->feat_flags; + return sizeof(p_drv_buf->feat_flags); + } + break; + case DRV_TLV_LOCAL_ADMIN_ADDR: + if (p_drv_buf->local_mac_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->local_mac; + return sizeof(p_drv_buf->local_mac); + } + break; + case DRV_TLV_ADDITIONAL_MAC_ADDR_1: + if (p_drv_buf->additional_mac1_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->additional_mac1; + return sizeof(p_drv_buf->additional_mac1); + } + break; + case DRV_TLV_ADDITIONAL_MAC_ADDR_2: + if (p_drv_buf->additional_mac2_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->additional_mac2; + return sizeof(p_drv_buf->additional_mac2); + } + break; + case DRV_TLV_OS_DRIVER_STATES: + if (p_drv_buf->drv_state_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->drv_state; + return sizeof(p_drv_buf->drv_state); + } + break; + case DRV_TLV_PXE_BOOT_PROGRESS: + if (p_drv_buf->pxe_progress_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->pxe_progress; + return sizeof(p_drv_buf->pxe_progress); + } + break; + case DRV_TLV_RX_FRAMES_RECEIVED: + if (p_drv_buf->rx_frames_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->rx_frames; + return sizeof(p_drv_buf->rx_frames); + } + break; + case DRV_TLV_RX_BYTES_RECEIVED: + if (p_drv_buf->rx_bytes_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->rx_bytes; + return sizeof(p_drv_buf->rx_bytes); + } + break; + case DRV_TLV_TX_FRAMES_SENT: + if (p_drv_buf->tx_frames_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->tx_frames; + return sizeof(p_drv_buf->tx_frames); + } + break; + case DRV_TLV_TX_BYTES_SENT: + if (p_drv_buf->tx_bytes_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->tx_bytes; + return sizeof(p_drv_buf->tx_bytes); + } + break; + default: + break; + } + + return -1; +} + +static int +ecore_mfw_get_eth_tlv_value(struct ecore_drv_tlv_hdr *p_tlv, + struct ecore_mfw_tlv_eth *p_drv_buf, + u8 **p_tlv_buf) +{ + switch (p_tlv->tlv_type) { + case DRV_TLV_LSO_MAX_OFFLOAD_SIZE: + if (p_drv_buf->lso_maxoff_size_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->lso_maxoff_size; + return sizeof(p_drv_buf->lso_maxoff_size); + } + break; + case DRV_TLV_LSO_MIN_SEGMENT_COUNT: + if (p_drv_buf->lso_minseg_size_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->lso_minseg_size; + return sizeof(p_drv_buf->lso_minseg_size); + } + break; + case DRV_TLV_PROMISCUOUS_MODE: + if (p_drv_buf->prom_mode_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->prom_mode; + return sizeof(p_drv_buf->prom_mode); + } + break; + case DRV_TLV_TX_DESCRIPTORS_QUEUE_SIZE: + if (p_drv_buf->tx_descr_size_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->tx_descr_size; + return sizeof(p_drv_buf->tx_descr_size); + } + break; + case DRV_TLV_RX_DESCRIPTORS_QUEUE_SIZE: + if (p_drv_buf->rx_descr_size_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->rx_descr_size; + return sizeof(p_drv_buf->rx_descr_size); + } + break; + case DRV_TLV_NUM_OF_NET_QUEUE_VMQ_CFG: + if (p_drv_buf->netq_count_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->netq_count; + return sizeof(p_drv_buf->netq_count); + } + break; + case DRV_TLV_NUM_OFFLOADED_CONNECTIONS_TCP_IPV4: + if (p_drv_buf->tcp4_offloads_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->tcp4_offloads; + return sizeof(p_drv_buf->tcp4_offloads); + } + break; + case DRV_TLV_NUM_OFFLOADED_CONNECTIONS_TCP_IPV6: + if (p_drv_buf->tcp6_offloads_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->tcp6_offloads; + return sizeof(p_drv_buf->tcp6_offloads); + } + break; + case DRV_TLV_TX_DESCRIPTOR_QUEUE_AVG_DEPTH: + if (p_drv_buf->tx_descr_qdepth_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->tx_descr_qdepth; + return sizeof(p_drv_buf->tx_descr_qdepth); + } + break; + case DRV_TLV_RX_DESCRIPTORS_QUEUE_AVG_DEPTH: + if (p_drv_buf->rx_descr_qdepth_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->rx_descr_qdepth; + return sizeof(p_drv_buf->rx_descr_qdepth); + } + break; + case DRV_TLV_IOV_OFFLOAD: + if (p_drv_buf->iov_offload_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->iov_offload; + return sizeof(p_drv_buf->iov_offload); + } + break; + case DRV_TLV_TX_QUEUES_EMPTY: + if (p_drv_buf->txqs_empty_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->txqs_empty; + return sizeof(p_drv_buf->txqs_empty); + } + break; + case DRV_TLV_RX_QUEUES_EMPTY: + if (p_drv_buf->rxqs_empty_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->rxqs_empty; + return sizeof(p_drv_buf->rxqs_empty); + } + break; + case DRV_TLV_TX_QUEUES_FULL: + if (p_drv_buf->num_txqs_full_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->num_txqs_full; + return sizeof(p_drv_buf->num_txqs_full); + } + break; + case DRV_TLV_RX_QUEUES_FULL: + if (p_drv_buf->num_rxqs_full_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->num_rxqs_full; + return sizeof(p_drv_buf->num_rxqs_full); + } + break; + default: + break; + } + + return -1; +} + +static int +ecore_mfw_get_fcoe_tlv_value(struct ecore_drv_tlv_hdr *p_tlv, + struct ecore_mfw_tlv_fcoe *p_drv_buf, + u8 **p_tlv_buf) +{ + switch (p_tlv->tlv_type) { + case DRV_TLV_SCSI_TO: + if (p_drv_buf->scsi_timeout_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->scsi_timeout; + return sizeof(p_drv_buf->scsi_timeout); + } + break; + case DRV_TLV_R_T_TOV: + if (p_drv_buf->rt_tov_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->rt_tov; + return sizeof(p_drv_buf->rt_tov); + } + break; + case DRV_TLV_R_A_TOV: + if (p_drv_buf->ra_tov_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->ra_tov; + return sizeof(p_drv_buf->ra_tov); + } + break; + case DRV_TLV_E_D_TOV: + if (p_drv_buf->ed_tov_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->ed_tov; + return sizeof(p_drv_buf->ed_tov); + } + break; + case DRV_TLV_CR_TOV: + if (p_drv_buf->cr_tov_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->cr_tov; + return sizeof(p_drv_buf->cr_tov); + } + break; + case DRV_TLV_BOOT_TYPE: + if (p_drv_buf->boot_type_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->boot_type; + return sizeof(p_drv_buf->boot_type); + } + break; + case DRV_TLV_NPIV_STATE: + if (p_drv_buf->npiv_state_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->npiv_state; + return sizeof(p_drv_buf->npiv_state); + } + break; + case DRV_TLV_NUM_OF_NPIV_IDS: + if (p_drv_buf->num_npiv_ids_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->num_npiv_ids; + return sizeof(p_drv_buf->num_npiv_ids); + } + break; + case DRV_TLV_SWITCH_NAME: + if (p_drv_buf->switch_name_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->switch_name; + return sizeof(p_drv_buf->switch_name); + } + break; + case DRV_TLV_SWITCH_PORT_NUM: + if (p_drv_buf->switch_portnum_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->switch_portnum; + return sizeof(p_drv_buf->switch_portnum); + } + break; + case DRV_TLV_SWITCH_PORT_ID: + if (p_drv_buf->switch_portid_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->switch_portid; + return sizeof(p_drv_buf->switch_portid); + } + break; + case DRV_TLV_VENDOR_NAME: + if (p_drv_buf->vendor_name_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->vendor_name; + return sizeof(p_drv_buf->vendor_name); + } + break; + case DRV_TLV_SWITCH_MODEL: + if (p_drv_buf->switch_model_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->switch_model; + return sizeof(p_drv_buf->switch_model); + } + break; + case DRV_TLV_SWITCH_FW_VER: + if (p_drv_buf->switch_fw_version_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->switch_fw_version; + return sizeof(p_drv_buf->switch_fw_version); + } + break; + case DRV_TLV_QOS_PRIORITY_PER_802_1P: + if (p_drv_buf->qos_pri_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->qos_pri; + return sizeof(p_drv_buf->qos_pri); + } + break; + case DRV_TLV_PORT_ALIAS: + if (p_drv_buf->port_alias_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->port_alias; + return sizeof(p_drv_buf->port_alias); + } + break; + case DRV_TLV_PORT_STATE: + if (p_drv_buf->port_state_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->port_state; + return sizeof(p_drv_buf->port_state); + } + break; + case DRV_TLV_FIP_TX_DESCRIPTORS_QUEUE_SIZE: + if (p_drv_buf->fip_tx_descr_size_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->fip_tx_descr_size; + return sizeof(p_drv_buf->fip_tx_descr_size); + } + break; + case DRV_TLV_FCOE_RX_DESCRIPTORS_QUEUE_SIZE: + if (p_drv_buf->fip_rx_descr_size_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->fip_rx_descr_size; + return sizeof(p_drv_buf->fip_rx_descr_size); + } + break; + case DRV_TLV_LINK_FAILURE_COUNT: + if (p_drv_buf->link_failures_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->link_failures; + return sizeof(p_drv_buf->link_failures); + } + break; + case DRV_TLV_FCOE_BOOT_PROGRESS: + if (p_drv_buf->fcoe_boot_progress_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->fcoe_boot_progress; + return sizeof(p_drv_buf->fcoe_boot_progress); + } + break; + case DRV_TLV_RX_BROADCAST_PACKETS: + if (p_drv_buf->rx_bcast_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->rx_bcast; + return sizeof(p_drv_buf->rx_bcast); + } + break; + case DRV_TLV_TX_BROADCAST_PACKETS: + if (p_drv_buf->tx_bcast_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->tx_bcast; + return sizeof(p_drv_buf->tx_bcast); + } + break; + case DRV_TLV_FCOE_TX_DESCRIPTOR_QUEUE_AVG_DEPTH: + if (p_drv_buf->fcoe_txq_depth_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->fcoe_txq_depth; + return sizeof(p_drv_buf->fcoe_txq_depth); + } + break; + case DRV_TLV_FCOE_RX_DESCRIPTORS_QUEUE_AVG_DEPTH: + if (p_drv_buf->fcoe_rxq_depth_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->fcoe_rxq_depth; + return sizeof(p_drv_buf->fcoe_rxq_depth); + } + break; + case DRV_TLV_FCOE_RX_FRAMES_RECEIVED: + if (p_drv_buf->fcoe_rx_frames_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->fcoe_rx_frames; + return sizeof(p_drv_buf->fcoe_rx_frames); + } + break; + case DRV_TLV_FCOE_RX_BYTES_RECEIVED: + if (p_drv_buf->fcoe_rx_bytes_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->fcoe_rx_bytes; + return sizeof(p_drv_buf->fcoe_rx_bytes); + } + break; + case DRV_TLV_FCOE_TX_FRAMES_SENT: + if (p_drv_buf->fcoe_tx_frames_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->fcoe_tx_frames; + return sizeof(p_drv_buf->fcoe_tx_frames); + } + break; + case DRV_TLV_FCOE_TX_BYTES_SENT: + if (p_drv_buf->fcoe_tx_bytes_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->fcoe_tx_bytes; + return sizeof(p_drv_buf->fcoe_tx_bytes); + } + break; + case DRV_TLV_CRC_ERROR_COUNT: + if (p_drv_buf->crc_count_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->crc_count; + return sizeof(p_drv_buf->crc_count); + } + break; + case DRV_TLV_CRC_ERROR_1_RECEIVED_SOURCE_FC_ID: + if (p_drv_buf->crc_err_src_fcid_set[0]) { + *p_tlv_buf = (u8 *)&p_drv_buf->crc_err_src_fcid[0]; + return sizeof(p_drv_buf->crc_err_src_fcid[0]); + } + break; + case DRV_TLV_CRC_ERROR_2_RECEIVED_SOURCE_FC_ID: + if (p_drv_buf->crc_err_src_fcid_set[1]) { + *p_tlv_buf = (u8 *)&p_drv_buf->crc_err_src_fcid[1]; + return sizeof(p_drv_buf->crc_err_src_fcid[1]); + } + break; + case DRV_TLV_CRC_ERROR_3_RECEIVED_SOURCE_FC_ID: + if (p_drv_buf->crc_err_src_fcid_set[2]) { + *p_tlv_buf = (u8 *)&p_drv_buf->crc_err_src_fcid[2]; + return sizeof(p_drv_buf->crc_err_src_fcid[2]); + } + break; + case DRV_TLV_CRC_ERROR_4_RECEIVED_SOURCE_FC_ID: + if (p_drv_buf->crc_err_src_fcid_set[3]) { + *p_tlv_buf = (u8 *)&p_drv_buf->crc_err_src_fcid[3]; + return sizeof(p_drv_buf->crc_err_src_fcid[3]); + } + break; + case DRV_TLV_CRC_ERROR_5_RECEIVED_SOURCE_FC_ID: + if (p_drv_buf->crc_err_src_fcid_set[4]) { + *p_tlv_buf = (u8 *)&p_drv_buf->crc_err_src_fcid[4]; + return sizeof(p_drv_buf->crc_err_src_fcid[4]); + } + break; + case DRV_TLV_CRC_ERROR_1_TIMESTAMP: + if (p_drv_buf->crc_err_tstamp_set[0]) { + *p_tlv_buf = (u8 *)&p_drv_buf->crc_err_tstamp[0]; + return sizeof(p_drv_buf->crc_err_tstamp[0]); + } + break; + case DRV_TLV_CRC_ERROR_2_TIMESTAMP: + if (p_drv_buf->crc_err_tstamp_set[1]) { + *p_tlv_buf = (u8 *)&p_drv_buf->crc_err_tstamp[1]; + return sizeof(p_drv_buf->crc_err_tstamp[1]); + } + break; + case DRV_TLV_CRC_ERROR_3_TIMESTAMP: + if (p_drv_buf->crc_err_tstamp_set[2]) { + *p_tlv_buf = (u8 *)&p_drv_buf->crc_err_tstamp[2]; + return sizeof(p_drv_buf->crc_err_tstamp[2]); + } + break; + case DRV_TLV_CRC_ERROR_4_TIMESTAMP: + if (p_drv_buf->crc_err_tstamp_set[3]) { + *p_tlv_buf = (u8 *)&p_drv_buf->crc_err_tstamp[3]; + return sizeof(p_drv_buf->crc_err_tstamp[3]); + } + break; + case DRV_TLV_CRC_ERROR_5_TIMESTAMP: + if (p_drv_buf->crc_err_tstamp_set[4]) { + *p_tlv_buf = (u8 *)&p_drv_buf->crc_err_tstamp[4]; + return sizeof(p_drv_buf->crc_err_tstamp[4]); + } + break; + case DRV_TLV_LOSS_OF_SYNC_ERROR_COUNT: + if (p_drv_buf->losync_err_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->losync_err; + return sizeof(p_drv_buf->losync_err); + } + break; + case DRV_TLV_LOSS_OF_SIGNAL_ERRORS: + if (p_drv_buf->losig_err_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->losig_err; + return sizeof(p_drv_buf->losig_err); + } + break; + case DRV_TLV_PRIMITIVE_SEQUENCE_PROTOCOL_ERROR_COUNT: + if (p_drv_buf->primtive_err_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->primtive_err; + return sizeof(p_drv_buf->primtive_err); + } + break; + case DRV_TLV_DISPARITY_ERROR_COUNT: + if (p_drv_buf->disparity_err_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->disparity_err; + return sizeof(p_drv_buf->disparity_err); + } + break; + case DRV_TLV_CODE_VIOLATION_ERROR_COUNT: + if (p_drv_buf->code_violation_err_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->code_violation_err; + return sizeof(p_drv_buf->code_violation_err); + } + break; + case DRV_TLV_LAST_FLOGI_ISSUED_COMMON_PARAMETERS_WORD_1: + if (p_drv_buf->flogi_param_set[0]) { + *p_tlv_buf = (u8 *)&p_drv_buf->flogi_param[0]; + return sizeof(p_drv_buf->flogi_param[0]); + } + break; + case DRV_TLV_LAST_FLOGI_ISSUED_COMMON_PARAMETERS_WORD_2: + if (p_drv_buf->flogi_param_set[1]) { + *p_tlv_buf = (u8 *)&p_drv_buf->flogi_param[1]; + return sizeof(p_drv_buf->flogi_param[1]); + } + break; + case DRV_TLV_LAST_FLOGI_ISSUED_COMMON_PARAMETERS_WORD_3: + if (p_drv_buf->flogi_param_set[2]) { + *p_tlv_buf = (u8 *)&p_drv_buf->flogi_param[2]; + return sizeof(p_drv_buf->flogi_param[2]); + } + break; + case DRV_TLV_LAST_FLOGI_ISSUED_COMMON_PARAMETERS_WORD_4: + if (p_drv_buf->flogi_param_set[3]) { + *p_tlv_buf = (u8 *)&p_drv_buf->flogi_param[3]; + return sizeof(p_drv_buf->flogi_param[3]); + } + break; + case DRV_TLV_LAST_FLOGI_TIMESTAMP: + if (p_drv_buf->flogi_tstamp_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->flogi_tstamp; + return sizeof(p_drv_buf->flogi_tstamp); + } + break; + case DRV_TLV_LAST_FLOGI_ACC_COMMON_PARAMETERS_WORD_1: + if (p_drv_buf->flogi_acc_param_set[0]) { + *p_tlv_buf = (u8 *)&p_drv_buf->flogi_acc_param[0]; + return sizeof(p_drv_buf->flogi_acc_param[0]); + } + break; + case DRV_TLV_LAST_FLOGI_ACC_COMMON_PARAMETERS_WORD_2: + if (p_drv_buf->flogi_acc_param_set[1]) { + *p_tlv_buf = (u8 *)&p_drv_buf->flogi_acc_param[1]; + return sizeof(p_drv_buf->flogi_acc_param[1]); + } + break; + case DRV_TLV_LAST_FLOGI_ACC_COMMON_PARAMETERS_WORD_3: + if (p_drv_buf->flogi_acc_param_set[2]) { + *p_tlv_buf = (u8 *)&p_drv_buf->flogi_acc_param[2]; + return sizeof(p_drv_buf->flogi_acc_param[2]); + } + break; + case DRV_TLV_LAST_FLOGI_ACC_COMMON_PARAMETERS_WORD_4: + if (p_drv_buf->flogi_acc_param_set[3]) { + *p_tlv_buf = (u8 *)&p_drv_buf->flogi_acc_param[3]; + return sizeof(p_drv_buf->flogi_acc_param[3]); + } + break; + case DRV_TLV_LAST_FLOGI_ACC_TIMESTAMP: + if (p_drv_buf->flogi_acc_tstamp_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->flogi_acc_tstamp; + return sizeof(p_drv_buf->flogi_acc_tstamp); + } + break; + case DRV_TLV_LAST_FLOGI_RJT: + if (p_drv_buf->flogi_rjt_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->flogi_rjt; + return sizeof(p_drv_buf->flogi_rjt); + } + break; + case DRV_TLV_LAST_FLOGI_RJT_TIMESTAMP: + if (p_drv_buf->flogi_rjt_tstamp_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->flogi_rjt_tstamp; + return sizeof(p_drv_buf->flogi_rjt_tstamp); + } + break; + case DRV_TLV_FDISCS_SENT_COUNT: + if (p_drv_buf->fdiscs_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->fdiscs; + return sizeof(p_drv_buf->fdiscs); + } + break; + case DRV_TLV_FDISC_ACCS_RECEIVED: + if (p_drv_buf->fdisc_acc_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->fdisc_acc; + return sizeof(p_drv_buf->fdisc_acc); + } + break; + case DRV_TLV_FDISC_RJTS_RECEIVED: + if (p_drv_buf->fdisc_rjt_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->fdisc_rjt; + return sizeof(p_drv_buf->fdisc_rjt); + } + break; + case DRV_TLV_PLOGI_SENT_COUNT: + if (p_drv_buf->plogi_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->plogi; + return sizeof(p_drv_buf->plogi); + } + break; + case DRV_TLV_PLOGI_ACCS_RECEIVED: + if (p_drv_buf->plogi_acc_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->plogi_acc; + return sizeof(p_drv_buf->plogi_acc); + } + break; + case DRV_TLV_PLOGI_RJTS_RECEIVED: + if (p_drv_buf->plogi_rjt_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->plogi_rjt; + return sizeof(p_drv_buf->plogi_rjt); + } + break; + case DRV_TLV_PLOGI_1_SENT_DESTINATION_FC_ID: + if (p_drv_buf->plogi_dst_fcid_set[0]) { + *p_tlv_buf = (u8 *)&p_drv_buf->plogi_dst_fcid[0]; + return sizeof(p_drv_buf->plogi_dst_fcid[0]); + } + break; + case DRV_TLV_PLOGI_2_SENT_DESTINATION_FC_ID: + if (p_drv_buf->plogi_dst_fcid_set[1]) { + *p_tlv_buf = (u8 *)&p_drv_buf->plogi_dst_fcid[1]; + return sizeof(p_drv_buf->plogi_dst_fcid[1]); + } + break; + case DRV_TLV_PLOGI_3_SENT_DESTINATION_FC_ID: + if (p_drv_buf->plogi_dst_fcid_set[2]) { + *p_tlv_buf = (u8 *)&p_drv_buf->plogi_dst_fcid[2]; + return sizeof(p_drv_buf->plogi_dst_fcid[2]); + } + break; + case DRV_TLV_PLOGI_4_SENT_DESTINATION_FC_ID: + if (p_drv_buf->plogi_dst_fcid_set[3]) { + *p_tlv_buf = (u8 *)&p_drv_buf->plogi_dst_fcid[3]; + return sizeof(p_drv_buf->plogi_dst_fcid[3]); + } + break; + case DRV_TLV_PLOGI_5_SENT_DESTINATION_FC_ID: + if (p_drv_buf->plogi_dst_fcid_set[4]) { + *p_tlv_buf = (u8 *)&p_drv_buf->plogi_dst_fcid[4]; + return sizeof(p_drv_buf->plogi_dst_fcid[4]); + } + break; + case DRV_TLV_PLOGI_1_TIMESTAMP: + if (p_drv_buf->plogi_tstamp_set[0]) { + *p_tlv_buf = (u8 *)&p_drv_buf->plogi_tstamp[0]; + return sizeof(p_drv_buf->plogi_tstamp[0]); + } + break; + case DRV_TLV_PLOGI_2_TIMESTAMP: + if (p_drv_buf->plogi_tstamp_set[1]) { + *p_tlv_buf = (u8 *)&p_drv_buf->plogi_tstamp[1]; + return sizeof(p_drv_buf->plogi_tstamp[1]); + } + break; + case DRV_TLV_PLOGI_3_TIMESTAMP: + if (p_drv_buf->plogi_tstamp_set[2]) { + *p_tlv_buf = (u8 *)&p_drv_buf->plogi_tstamp[2]; + return sizeof(p_drv_buf->plogi_tstamp[2]); + } + break; + case DRV_TLV_PLOGI_4_TIMESTAMP: + if (p_drv_buf->plogi_tstamp_set[3]) { + *p_tlv_buf = (u8 *)&p_drv_buf->plogi_tstamp[3]; + return sizeof(p_drv_buf->plogi_tstamp[3]); + } + break; + case DRV_TLV_PLOGI_5_TIMESTAMP: + if (p_drv_buf->plogi_tstamp_set[4]) { + *p_tlv_buf = (u8 *)&p_drv_buf->plogi_tstamp[4]; + return sizeof(p_drv_buf->plogi_tstamp[4]); + } + break; + case DRV_TLV_PLOGI_1_ACC_RECEIVED_SOURCE_FC_ID: + if (p_drv_buf->plogi_acc_src_fcid_set[0]) { + *p_tlv_buf = (u8 *)&p_drv_buf->plogi_acc_src_fcid[0]; + return sizeof(p_drv_buf->plogi_acc_src_fcid[0]); + } + break; + case DRV_TLV_PLOGI_2_ACC_RECEIVED_SOURCE_FC_ID: + if (p_drv_buf->plogi_acc_src_fcid_set[1]) { + *p_tlv_buf = (u8 *)&p_drv_buf->plogi_acc_src_fcid[1]; + return sizeof(p_drv_buf->plogi_acc_src_fcid[1]); + } + break; + case DRV_TLV_PLOGI_3_ACC_RECEIVED_SOURCE_FC_ID: + if (p_drv_buf->plogi_acc_src_fcid_set[2]) { + *p_tlv_buf = (u8 *)&p_drv_buf->plogi_acc_src_fcid[2]; + return sizeof(p_drv_buf->plogi_acc_src_fcid[2]); + } + break; + case DRV_TLV_PLOGI_4_ACC_RECEIVED_SOURCE_FC_ID: + if (p_drv_buf->plogi_acc_src_fcid_set[3]) { + *p_tlv_buf = (u8 *)&p_drv_buf->plogi_acc_src_fcid[3]; + return sizeof(p_drv_buf->plogi_acc_src_fcid[3]); + } + break; + case DRV_TLV_PLOGI_5_ACC_RECEIVED_SOURCE_FC_ID: + if (p_drv_buf->plogi_acc_src_fcid_set[4]) { + *p_tlv_buf = (u8 *)&p_drv_buf->plogi_acc_src_fcid[4]; + return sizeof(p_drv_buf->plogi_acc_src_fcid[4]); + } + break; + case DRV_TLV_PLOGI_1_ACC_TIMESTAMP: + if (p_drv_buf->plogi_acc_tstamp_set[0]) { + *p_tlv_buf = (u8 *)&p_drv_buf->plogi_acc_tstamp[0]; + return sizeof(p_drv_buf->plogi_acc_tstamp[0]); + } + break; + case DRV_TLV_PLOGI_2_ACC_TIMESTAMP: + if (p_drv_buf->plogi_acc_tstamp_set[1]) { + *p_tlv_buf = (u8 *)&p_drv_buf->plogi_acc_tstamp[1]; + return sizeof(p_drv_buf->plogi_acc_tstamp[1]); + } + break; + case DRV_TLV_PLOGI_3_ACC_TIMESTAMP: + if (p_drv_buf->plogi_acc_tstamp_set[2]) { + *p_tlv_buf = (u8 *)&p_drv_buf->plogi_acc_tstamp[2]; + return sizeof(p_drv_buf->plogi_acc_tstamp[2]); + } + break; + case DRV_TLV_PLOGI_4_ACC_TIMESTAMP: + if (p_drv_buf->plogi_acc_tstamp_set[3]) { + *p_tlv_buf = (u8 *)&p_drv_buf->plogi_acc_tstamp[3]; + return sizeof(p_drv_buf->plogi_acc_tstamp[3]); + } + break; + case DRV_TLV_PLOGI_5_ACC_TIMESTAMP: + if (p_drv_buf->plogi_acc_tstamp_set[4]) { + *p_tlv_buf = (u8 *)&p_drv_buf->plogi_acc_tstamp[4]; + return sizeof(p_drv_buf->plogi_acc_tstamp[4]); + } + break; + case DRV_TLV_LOGOS_ISSUED: + if (p_drv_buf->tx_plogos_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->tx_plogos; + return sizeof(p_drv_buf->tx_plogos); + } + break; + case DRV_TLV_LOGO_ACCS_RECEIVED: + if (p_drv_buf->plogo_acc_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->plogo_acc; + return sizeof(p_drv_buf->plogo_acc); + } + break; + case DRV_TLV_LOGO_RJTS_RECEIVED: + if (p_drv_buf->plogo_rjt_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->plogo_rjt; + return sizeof(p_drv_buf->plogo_rjt); + } + break; + case DRV_TLV_LOGO_1_RECEIVED_SOURCE_FC_ID: + if (p_drv_buf->plogo_src_fcid_set[0]) { + *p_tlv_buf = (u8 *)&p_drv_buf->plogo_src_fcid[0]; + return sizeof(p_drv_buf->plogo_src_fcid[0]); + } + break; + case DRV_TLV_LOGO_2_RECEIVED_SOURCE_FC_ID: + if (p_drv_buf->plogo_src_fcid_set[1]) { + *p_tlv_buf = (u8 *)&p_drv_buf->plogo_src_fcid[1]; + return sizeof(p_drv_buf->plogo_src_fcid[1]); + } + break; + case DRV_TLV_LOGO_3_RECEIVED_SOURCE_FC_ID: + if (p_drv_buf->plogo_src_fcid_set[2]) { + *p_tlv_buf = (u8 *)&p_drv_buf->plogo_src_fcid[2]; + return sizeof(p_drv_buf->plogo_src_fcid[2]); + } + break; + case DRV_TLV_LOGO_4_RECEIVED_SOURCE_FC_ID: + if (p_drv_buf->plogo_src_fcid_set[3]) { + *p_tlv_buf = (u8 *)&p_drv_buf->plogo_src_fcid[3]; + return sizeof(p_drv_buf->plogo_src_fcid[3]); + } + break; + case DRV_TLV_LOGO_5_RECEIVED_SOURCE_FC_ID: + if (p_drv_buf->plogo_src_fcid_set[4]) { + *p_tlv_buf = (u8 *)&p_drv_buf->plogo_src_fcid[4]; + return sizeof(p_drv_buf->plogo_src_fcid[4]); + } + break; + case DRV_TLV_LOGO_1_TIMESTAMP: + if (p_drv_buf->plogo_tstamp_set[0]) { + *p_tlv_buf = (u8 *)&p_drv_buf->plogo_tstamp[0]; + return sizeof(p_drv_buf->plogo_tstamp[0]); + } + break; + case DRV_TLV_LOGO_2_TIMESTAMP: + if (p_drv_buf->plogo_tstamp_set[1]) { + *p_tlv_buf = (u8 *)&p_drv_buf->plogo_tstamp[1]; + return sizeof(p_drv_buf->plogo_tstamp[1]); + } + break; + case DRV_TLV_LOGO_3_TIMESTAMP: + if (p_drv_buf->plogo_tstamp_set[2]) { + *p_tlv_buf = (u8 *)&p_drv_buf->plogo_tstamp[2]; + return sizeof(p_drv_buf->plogo_tstamp[2]); + } + break; + case DRV_TLV_LOGO_4_TIMESTAMP: + if (p_drv_buf->plogo_tstamp_set[3]) { + *p_tlv_buf = (u8 *)&p_drv_buf->plogo_tstamp[3]; + return sizeof(p_drv_buf->plogo_tstamp[3]); + } + break; + case DRV_TLV_LOGO_5_TIMESTAMP: + if (p_drv_buf->plogo_tstamp_set[4]) { + *p_tlv_buf = (u8 *)&p_drv_buf->plogo_tstamp[4]; + return sizeof(p_drv_buf->plogo_tstamp[4]); + } + break; + case DRV_TLV_LOGOS_RECEIVED: + if (p_drv_buf->rx_logos_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->rx_logos; + return sizeof(p_drv_buf->rx_logos); + } + break; + case DRV_TLV_ACCS_ISSUED: + if (p_drv_buf->tx_accs_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->tx_accs; + return sizeof(p_drv_buf->tx_accs); + } + break; + case DRV_TLV_PRLIS_ISSUED: + if (p_drv_buf->tx_prlis_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->tx_prlis; + return sizeof(p_drv_buf->tx_prlis); + } + break; + case DRV_TLV_ACCS_RECEIVED: + if (p_drv_buf->rx_accs_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->rx_accs; + return sizeof(p_drv_buf->rx_accs); + } + break; + case DRV_TLV_ABTS_SENT_COUNT: + if (p_drv_buf->tx_abts_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->tx_abts; + return sizeof(p_drv_buf->tx_abts); + } + break; + case DRV_TLV_ABTS_ACCS_RECEIVED: + if (p_drv_buf->rx_abts_acc_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->rx_abts_acc; + return sizeof(p_drv_buf->rx_abts_acc); + } + break; + case DRV_TLV_ABTS_RJTS_RECEIVED: + if (p_drv_buf->rx_abts_rjt_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->rx_abts_rjt; + return sizeof(p_drv_buf->rx_abts_rjt); + } + break; + case DRV_TLV_ABTS_1_SENT_DESTINATION_FC_ID: + if (p_drv_buf->abts_dst_fcid_set[0]) { + *p_tlv_buf = (u8 *)&p_drv_buf->abts_dst_fcid[0]; + return sizeof(p_drv_buf->abts_dst_fcid[0]); + } + break; + case DRV_TLV_ABTS_2_SENT_DESTINATION_FC_ID: + if (p_drv_buf->abts_dst_fcid_set[1]) { + *p_tlv_buf = (u8 *)&p_drv_buf->abts_dst_fcid[1]; + return sizeof(p_drv_buf->abts_dst_fcid[1]); + } + break; + case DRV_TLV_ABTS_3_SENT_DESTINATION_FC_ID: + if (p_drv_buf->abts_dst_fcid_set[2]) { + *p_tlv_buf = (u8 *)&p_drv_buf->abts_dst_fcid[2]; + return sizeof(p_drv_buf->abts_dst_fcid[2]); + } + break; + case DRV_TLV_ABTS_4_SENT_DESTINATION_FC_ID: + if (p_drv_buf->abts_dst_fcid_set[3]) { + *p_tlv_buf = (u8 *)&p_drv_buf->abts_dst_fcid[3]; + return sizeof(p_drv_buf->abts_dst_fcid[3]); + } + break; + case DRV_TLV_ABTS_5_SENT_DESTINATION_FC_ID: + if (p_drv_buf->abts_dst_fcid_set[4]) { + *p_tlv_buf = (u8 *)&p_drv_buf->abts_dst_fcid[4]; + return sizeof(p_drv_buf->abts_dst_fcid[4]); + } + break; + case DRV_TLV_ABTS_1_TIMESTAMP: + if (p_drv_buf->abts_tstamp_set[0]) { + *p_tlv_buf = (u8 *)&p_drv_buf->abts_tstamp[0]; + return sizeof(p_drv_buf->abts_tstamp[0]); + } + break; + case DRV_TLV_ABTS_2_TIMESTAMP: + if (p_drv_buf->abts_tstamp_set[1]) { + *p_tlv_buf = (u8 *)&p_drv_buf->abts_tstamp[1]; + return sizeof(p_drv_buf->abts_tstamp[1]); + } + break; + case DRV_TLV_ABTS_3_TIMESTAMP: + if (p_drv_buf->abts_tstamp_set[2]) { + *p_tlv_buf = (u8 *)&p_drv_buf->abts_tstamp[2]; + return sizeof(p_drv_buf->abts_tstamp[2]); + } + break; + case DRV_TLV_ABTS_4_TIMESTAMP: + if (p_drv_buf->abts_tstamp_set[3]) { + *p_tlv_buf = (u8 *)&p_drv_buf->abts_tstamp[3]; + return sizeof(p_drv_buf->abts_tstamp[3]); + } + break; + case DRV_TLV_ABTS_5_TIMESTAMP: + if (p_drv_buf->abts_tstamp_set[4]) { + *p_tlv_buf = (u8 *)&p_drv_buf->abts_tstamp[4]; + return sizeof(p_drv_buf->abts_tstamp[4]); + } + break; + case DRV_TLV_RSCNS_RECEIVED: + if (p_drv_buf->rx_rscn_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->rx_rscn; + return sizeof(p_drv_buf->rx_rscn); + } + break; + case DRV_TLV_LAST_RSCN_RECEIVED_N_PORT_1: + if (p_drv_buf->rx_rscn_nport_set[0]) { + *p_tlv_buf = (u8 *)&p_drv_buf->rx_rscn_nport[0]; + return sizeof(p_drv_buf->rx_rscn_nport[0]); + } + break; + case DRV_TLV_LAST_RSCN_RECEIVED_N_PORT_2: + if (p_drv_buf->rx_rscn_nport_set[1]) { + *p_tlv_buf = (u8 *)&p_drv_buf->rx_rscn_nport[1]; + return sizeof(p_drv_buf->rx_rscn_nport[1]); + } + break; + case DRV_TLV_LAST_RSCN_RECEIVED_N_PORT_3: + if (p_drv_buf->rx_rscn_nport_set[2]) { + *p_tlv_buf = (u8 *)&p_drv_buf->rx_rscn_nport[2]; + return sizeof(p_drv_buf->rx_rscn_nport[2]); + } + break; + case DRV_TLV_LAST_RSCN_RECEIVED_N_PORT_4: + if (p_drv_buf->rx_rscn_nport_set[3]) { + *p_tlv_buf = (u8 *)&p_drv_buf->rx_rscn_nport[3]; + return sizeof(p_drv_buf->rx_rscn_nport[3]); + } + break; + case DRV_TLV_LUN_RESETS_ISSUED: + if (p_drv_buf->tx_lun_rst_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->tx_lun_rst; + return sizeof(p_drv_buf->tx_lun_rst); + } + break; + case DRV_TLV_ABORT_TASK_SETS_ISSUED: + if (p_drv_buf->abort_task_sets_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->abort_task_sets; + return sizeof(p_drv_buf->abort_task_sets); + } + break; + case DRV_TLV_TPRLOS_SENT: + if (p_drv_buf->tx_tprlos_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->tx_tprlos; + return sizeof(p_drv_buf->tx_tprlos); + } + break; + case DRV_TLV_NOS_SENT_COUNT: + if (p_drv_buf->tx_nos_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->tx_nos; + return sizeof(p_drv_buf->tx_nos); + } + break; + case DRV_TLV_NOS_RECEIVED_COUNT: + if (p_drv_buf->rx_nos_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->rx_nos; + return sizeof(p_drv_buf->rx_nos); + } + break; + case DRV_TLV_OLS_COUNT: + if (p_drv_buf->ols_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->ols; + return sizeof(p_drv_buf->ols); + } + break; + case DRV_TLV_LR_COUNT: + if (p_drv_buf->lr_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->lr; + return sizeof(p_drv_buf->lr); + } + break; + case DRV_TLV_LRR_COUNT: + if (p_drv_buf->lrr_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->lrr; + return sizeof(p_drv_buf->lrr); + } + break; + case DRV_TLV_LIP_SENT_COUNT: + if (p_drv_buf->tx_lip_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->tx_lip; + return sizeof(p_drv_buf->tx_lip); + } + break; + case DRV_TLV_LIP_RECEIVED_COUNT: + if (p_drv_buf->rx_lip_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->rx_lip; + return sizeof(p_drv_buf->rx_lip); + } + break; + case DRV_TLV_EOFA_COUNT: + if (p_drv_buf->eofa_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->eofa; + return sizeof(p_drv_buf->eofa); + } + break; + case DRV_TLV_EOFNI_COUNT: + if (p_drv_buf->eofni_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->eofni; + return sizeof(p_drv_buf->eofni); + } + break; + case DRV_TLV_SCSI_STATUS_CHECK_CONDITION_COUNT: + if (p_drv_buf->scsi_chks_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->scsi_chks; + return sizeof(p_drv_buf->scsi_chks); + } + break; + case DRV_TLV_SCSI_STATUS_CONDITION_MET_COUNT: + if (p_drv_buf->scsi_cond_met_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->scsi_cond_met; + return sizeof(p_drv_buf->scsi_cond_met); + } + break; + case DRV_TLV_SCSI_STATUS_BUSY_COUNT: + if (p_drv_buf->scsi_busy_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->scsi_busy; + return sizeof(p_drv_buf->scsi_busy); + } + break; + case DRV_TLV_SCSI_STATUS_INTERMEDIATE_COUNT: + if (p_drv_buf->scsi_inter_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->scsi_inter; + return sizeof(p_drv_buf->scsi_inter); + } + break; + case DRV_TLV_SCSI_STATUS_INTERMEDIATE_CONDITION_MET_COUNT: + if (p_drv_buf->scsi_inter_cond_met_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->scsi_inter_cond_met; + return sizeof(p_drv_buf->scsi_inter_cond_met); + } + break; + case DRV_TLV_SCSI_STATUS_RESERVATION_CONFLICT_COUNT: + if (p_drv_buf->scsi_rsv_conflicts_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->scsi_rsv_conflicts; + return sizeof(p_drv_buf->scsi_rsv_conflicts); + } + break; + case DRV_TLV_SCSI_STATUS_TASK_SET_FULL_COUNT: + if (p_drv_buf->scsi_tsk_full_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->scsi_tsk_full; + return sizeof(p_drv_buf->scsi_tsk_full); + } + break; + case DRV_TLV_SCSI_STATUS_ACA_ACTIVE_COUNT: + if (p_drv_buf->scsi_aca_active_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->scsi_aca_active; + return sizeof(p_drv_buf->scsi_aca_active); + } + break; + case DRV_TLV_SCSI_STATUS_TASK_ABORTED_COUNT: + if (p_drv_buf->scsi_tsk_abort_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->scsi_tsk_abort; + return sizeof(p_drv_buf->scsi_tsk_abort); + } + break; + case DRV_TLV_SCSI_CHECK_CONDITION_1_RECEIVED_SK_ASC_ASCQ: + if (p_drv_buf->scsi_rx_chk_set[0]) { + *p_tlv_buf = (u8 *)&p_drv_buf->scsi_rx_chk[0]; + return sizeof(p_drv_buf->scsi_rx_chk[0]); + } + break; + case DRV_TLV_SCSI_CHECK_CONDITION_2_RECEIVED_SK_ASC_ASCQ: + if (p_drv_buf->scsi_rx_chk_set[1]) { + *p_tlv_buf = (u8 *)&p_drv_buf->scsi_rx_chk[1]; + return sizeof(p_drv_buf->scsi_rx_chk[1]); + } + break; + case DRV_TLV_SCSI_CHECK_CONDITION_3_RECEIVED_SK_ASC_ASCQ: + if (p_drv_buf->scsi_rx_chk_set[2]) { + *p_tlv_buf = (u8 *)&p_drv_buf->scsi_rx_chk[2]; + return sizeof(p_drv_buf->scsi_rx_chk[2]); + } + break; + case DRV_TLV_SCSI_CHECK_CONDITION_4_RECEIVED_SK_ASC_ASCQ: + if (p_drv_buf->scsi_rx_chk_set[3]) { + *p_tlv_buf = (u8 *)&p_drv_buf->scsi_rx_chk[3]; + return sizeof(p_drv_buf->scsi_rx_chk[4]); + } + break; + case DRV_TLV_SCSI_CHECK_CONDITION_5_RECEIVED_SK_ASC_ASCQ: + if (p_drv_buf->scsi_rx_chk_set[4]) { + *p_tlv_buf = (u8 *)&p_drv_buf->scsi_rx_chk[4]; + return sizeof(p_drv_buf->scsi_rx_chk[4]); + } + break; + case DRV_TLV_SCSI_CHECK_1_TIMESTAMP: + if (p_drv_buf->scsi_chk_tstamp_set[0]) { + *p_tlv_buf = (u8 *)&p_drv_buf->scsi_chk_tstamp[0]; + return sizeof(p_drv_buf->scsi_chk_tstamp[0]); + } + break; + case DRV_TLV_SCSI_CHECK_2_TIMESTAMP: + if (p_drv_buf->scsi_chk_tstamp_set[1]) { + *p_tlv_buf = (u8 *)&p_drv_buf->scsi_chk_tstamp[1]; + return sizeof(p_drv_buf->scsi_chk_tstamp[1]); + } + break; + case DRV_TLV_SCSI_CHECK_3_TIMESTAMP: + if (p_drv_buf->scsi_chk_tstamp_set[2]) { + *p_tlv_buf = (u8 *)&p_drv_buf->scsi_chk_tstamp[2]; + return sizeof(p_drv_buf->scsi_chk_tstamp[2]); + } + break; + case DRV_TLV_SCSI_CHECK_4_TIMESTAMP: + if (p_drv_buf->scsi_chk_tstamp_set[3]) { + *p_tlv_buf = (u8 *)&p_drv_buf->scsi_chk_tstamp[3]; + return sizeof(p_drv_buf->scsi_chk_tstamp[3]); + } + break; + case DRV_TLV_SCSI_CHECK_5_TIMESTAMP: + if (p_drv_buf->scsi_chk_tstamp_set[4]) { + *p_tlv_buf = (u8 *)&p_drv_buf->scsi_chk_tstamp[4]; + return sizeof(p_drv_buf->scsi_chk_tstamp[4]); + } + break; + default: + break; + } + + return -1; +} + +static int +ecore_mfw_get_iscsi_tlv_value(struct ecore_drv_tlv_hdr *p_tlv, + struct ecore_mfw_tlv_iscsi *p_drv_buf, + u8 **p_tlv_buf) +{ + switch (p_tlv->tlv_type) { + case DRV_TLV_TARGET_LLMNR_ENABLED: + if (p_drv_buf->target_llmnr_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->target_llmnr; + return sizeof(p_drv_buf->target_llmnr); + } + break; + case DRV_TLV_HEADER_DIGEST_FLAG_ENABLED: + if (p_drv_buf->header_digest_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->header_digest; + return sizeof(p_drv_buf->header_digest); + } + break; + case DRV_TLV_DATA_DIGEST_FLAG_ENABLED: + if (p_drv_buf->data_digest_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->data_digest; + return sizeof(p_drv_buf->data_digest); + } + break; + case DRV_TLV_AUTHENTICATION_METHOD: + if (p_drv_buf->auth_method_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->auth_method; + return sizeof(p_drv_buf->auth_method); + } + break; + case DRV_TLV_ISCSI_BOOT_TARGET_PORTAL: + if (p_drv_buf->boot_taget_portal_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->boot_taget_portal; + return sizeof(p_drv_buf->boot_taget_portal); + } + break; + case DRV_TLV_MAX_FRAME_SIZE: + if (p_drv_buf->frame_size_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->frame_size; + return sizeof(p_drv_buf->frame_size); + } + break; + case DRV_TLV_PDU_TX_DESCRIPTORS_QUEUE_SIZE: + if (p_drv_buf->tx_desc_size_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->tx_desc_size; + return sizeof(p_drv_buf->tx_desc_size); + } + break; + case DRV_TLV_PDU_RX_DESCRIPTORS_QUEUE_SIZE: + if (p_drv_buf->rx_desc_size_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->rx_desc_size; + return sizeof(p_drv_buf->rx_desc_size); + } + break; + case DRV_TLV_ISCSI_BOOT_PROGRESS: + if (p_drv_buf->boot_progress_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->boot_progress; + return sizeof(p_drv_buf->boot_progress); + } + break; + case DRV_TLV_PDU_TX_DESCRIPTOR_QUEUE_AVG_DEPTH: + if (p_drv_buf->tx_desc_qdepth_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->tx_desc_qdepth; + return sizeof(p_drv_buf->tx_desc_qdepth); + } + break; + case DRV_TLV_PDU_RX_DESCRIPTORS_QUEUE_AVG_DEPTH: + if (p_drv_buf->rx_desc_qdepth_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->rx_desc_qdepth; + return sizeof(p_drv_buf->rx_desc_qdepth); + } + break; + case DRV_TLV_ISCSI_PDU_RX_FRAMES_RECEIVED: + if (p_drv_buf->rx_frames_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->rx_frames; + return sizeof(p_drv_buf->rx_frames); + } + break; + case DRV_TLV_ISCSI_PDU_RX_BYTES_RECEIVED: + if (p_drv_buf->rx_bytes_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->rx_bytes; + return sizeof(p_drv_buf->rx_bytes); + } + break; + case DRV_TLV_ISCSI_PDU_TX_FRAMES_SENT: + if (p_drv_buf->tx_frames_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->tx_frames; + return sizeof(p_drv_buf->tx_frames); + } + break; + case DRV_TLV_ISCSI_PDU_TX_BYTES_SENT: + if (p_drv_buf->tx_bytes_set) { + *p_tlv_buf = (u8 *)&p_drv_buf->tx_bytes; + return sizeof(p_drv_buf->tx_bytes); + } + break; + default: + break; + } + + return -1; +} + +static enum _ecore_status_t ecore_mfw_update_tlvs(struct ecore_hwfn *p_hwfn, + u8 tlv_group, u8 *p_mfw_buf, + u32 size) +{ + union ecore_mfw_tlv_data *p_tlv_data; + struct ecore_drv_tlv_hdr tlv; + u8 *p_tlv_ptr = OSAL_NULL, *p_temp; + u32 offset; + int len; + + p_tlv_data = OSAL_VZALLOC(p_hwfn->p_dev, sizeof(*p_tlv_data)); + if (!p_tlv_data) + return ECORE_NOMEM; + + if (OSAL_MFW_FILL_TLV_DATA(p_hwfn, tlv_group, p_tlv_data)) { + OSAL_VFREE(p_hwfn->p_dev, p_tlv_data); + return ECORE_INVAL; + } + + offset = 0; + OSAL_MEMSET(&tlv, 0, sizeof(tlv)); + while (offset < size) { + p_temp = &p_mfw_buf[offset]; + tlv.tlv_type = TLV_TYPE(p_temp); + tlv.tlv_length = TLV_LENGTH(p_temp); + tlv.tlv_flags = TLV_FLAGS(p_temp); + DP_INFO(p_hwfn, "Type %d length = %d flags = 0x%x\n", + tlv.tlv_type, tlv.tlv_length, tlv.tlv_flags); + + offset += sizeof(tlv); + if (tlv_group == ECORE_MFW_TLV_GENERIC) + len = ecore_mfw_get_gen_tlv_value(&tlv, + &p_tlv_data->generic, &p_tlv_ptr); + else if (tlv_group == ECORE_MFW_TLV_ETH) + len = ecore_mfw_get_eth_tlv_value(&tlv, + &p_tlv_data->eth, &p_tlv_ptr); + else if (tlv_group == ECORE_MFW_TLV_FCOE) + len = ecore_mfw_get_fcoe_tlv_value(&tlv, + &p_tlv_data->fcoe, &p_tlv_ptr); + else + len = ecore_mfw_get_iscsi_tlv_value(&tlv, + &p_tlv_data->iscsi, &p_tlv_ptr); + + if (len > 0) { + OSAL_WARN(len > 4 * tlv.tlv_length, + "Incorrect MFW TLV length"); + len = OSAL_MIN_T(int, len, 4 * tlv.tlv_length); + tlv.tlv_flags |= ECORE_DRV_TLV_FLAGS_CHANGED; + /* TODO: Endianness handling? */ + OSAL_MEMCPY(p_mfw_buf, &tlv, sizeof(tlv)); + OSAL_MEMCPY(p_mfw_buf + offset, p_tlv_ptr, len); + } + + offset += sizeof(u32) * tlv.tlv_length; + } + + OSAL_VFREE(p_hwfn->p_dev, p_tlv_data); + + return ECORE_SUCCESS; +} + +enum _ecore_status_t +ecore_mfw_process_tlv_req(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt) +{ + u32 addr, size, offset, resp, param, val; + u8 tlv_group = 0, id, *p_mfw_buf = OSAL_NULL, *p_temp; + u32 global_offsize, global_addr; + enum _ecore_status_t rc; + struct ecore_drv_tlv_hdr tlv; + + addr = SECTION_OFFSIZE_ADDR(p_hwfn->mcp_info->public_base, + PUBLIC_GLOBAL); + global_offsize = ecore_rd(p_hwfn, p_ptt, addr); + global_addr = SECTION_ADDR(global_offsize, 0); + addr = global_addr + OFFSETOF(struct public_global, data_ptr); + size = ecore_rd(p_hwfn, p_ptt, global_addr + + OFFSETOF(struct public_global, data_size)); + + if (!size) { + DP_NOTICE(p_hwfn, false, "Invalid TLV req size = %d\n", size); + goto drv_done; + } + + p_mfw_buf = (void *)OSAL_VZALLOC(p_hwfn->p_dev, size); + if (!p_mfw_buf) { + DP_NOTICE(p_hwfn, false, + "Failed allocate memory for p_mfw_buf\n"); + goto drv_done; + } + + /* Read the TLV request to local buffer */ + for (offset = 0; offset < size; offset += sizeof(u32)) { + val = ecore_rd(p_hwfn, p_ptt, addr + offset); + OSAL_MEMCPY(&p_mfw_buf[offset], &val, sizeof(u32)); + } + + /* Parse the headers to enumerate the requested TLV groups */ + for (offset = 0; offset < size; + offset += sizeof(tlv) + sizeof(u32) * tlv.tlv_length) { + p_temp = &p_mfw_buf[offset]; + tlv.tlv_type = TLV_TYPE(p_temp); + tlv.tlv_length = TLV_LENGTH(p_temp); + if (ecore_mfw_get_tlv_group(tlv.tlv_type, &tlv_group)) + goto drv_done; + } + + /* Update the TLV values in the local buffer */ + for (id = ECORE_MFW_TLV_GENERIC; id < ECORE_MFW_TLV_MAX; id <<= 1) { + if (tlv_group & id) { + if (ecore_mfw_update_tlvs(p_hwfn, id, p_mfw_buf, size)) + goto drv_done; + } + } + + /* Write the TLV data to shared memory */ + for (offset = 0; offset < size; offset += sizeof(u32)) { + val = (u32)p_mfw_buf[offset]; + ecore_wr(p_hwfn, p_ptt, addr + offset, val); + offset += sizeof(u32); + } + +drv_done: + rc = ecore_mcp_cmd(p_hwfn, p_ptt, DRV_MSG_CODE_GET_TLV_DONE, 0, &resp, + ¶m); + + OSAL_VFREE(p_hwfn->p_dev, p_mfw_buf); + + return rc; +} diff --git a/dpdk/drivers/net/qede/base/ecore_proto_if.h b/dpdk/drivers/net/qede/base/ecore_proto_if.h index 2fecbc86..66622323 100644 --- a/dpdk/drivers/net/qede/base/ecore_proto_if.h +++ b/dpdk/drivers/net/qede/base/ecore_proto_if.h @@ -13,16 +13,96 @@ * PF parameters (according to personality/protocol) */ +#define ECORE_ROCE_PROTOCOL_INDEX (3) + struct ecore_eth_pf_params { /* The following parameters are used during HW-init * and these parameters need to be passed as arguments * to update_pf_params routine invoked before slowpath start */ - u16 num_cons; + u16 num_cons; + + /* per-VF number of CIDs */ + u8 num_vf_cons; +#define ETH_PF_PARAMS_VF_CONS_DEFAULT (32) + + /* To enable arfs, previous to HW-init a positive number needs to be + * set [as filters require allocated searcher ILT memory]. + * This will set the maximal number of configured steering-filters. + */ + u32 num_arfs_filters; +}; + +/* Most of the the parameters below are described in the FW iSCSI / TCP HSI */ +struct ecore_iscsi_pf_params { + u64 glbl_q_params_addr; + u64 bdq_pbl_base_addr[2]; + u16 cq_num_entries; + u16 cmdq_num_entries; + u32 two_msl_timer; + u16 tx_sws_timer; + /* The following parameters are used during HW-init + * and these parameters need to be passed as arguments + * to update_pf_params routine invoked before slowpath start + */ + u16 num_cons; + u16 num_tasks; + + /* The following parameters are used during protocol-init */ + u16 half_way_close_timeout; + u16 bdq_xoff_threshold[2]; + u16 bdq_xon_threshold[2]; + u16 cmdq_xoff_threshold; + u16 cmdq_xon_threshold; + u16 rq_buffer_size; + + u8 num_sq_pages_in_ring; + u8 num_r2tq_pages_in_ring; + u8 num_uhq_pages_in_ring; + u8 num_queues; + u8 log_page_size; + u8 rqe_log_size; + u8 max_fin_rt; + u8 gl_rq_pi; + u8 gl_cmd_pi; + u8 debug_mode; + u8 ll2_ooo_queue_id; + u8 ooo_enable; + + u8 is_target; + u8 bdq_pbl_num_entries[2]; + u8 disable_stats_collection; +}; + +enum ecore_rdma_protocol { + ECORE_RDMA_PROTOCOL_DEFAULT, + ECORE_RDMA_PROTOCOL_ROCE, + ECORE_RDMA_PROTOCOL_IWARP, +}; + +struct ecore_rdma_pf_params { + /* Supplied to ECORE during resource allocation (may affect the ILT and + * the doorbell BAR). + */ + u32 min_dpis; /* number of requested DPIs */ + u32 num_mrs; /* number of requested memory regions*/ + u32 num_qps; /* number of requested Queue Pairs */ + u32 num_srqs; /* number of requested SRQ */ + u8 roce_edpm_mode; /* see QED_ROCE_EDPM_MODE_ENABLE */ + u8 gl_pi; /* protocol index */ + + /* Will allocate rate limiters to be used with QPs */ + u8 enable_dcqcn; + + /* TCP port number used for the iwarp traffic */ + u16 iwarp_port; + enum ecore_rdma_protocol rdma_protocol; }; struct ecore_pf_params { - struct ecore_eth_pf_params eth_pf_params; + struct ecore_eth_pf_params eth_pf_params; + struct ecore_iscsi_pf_params iscsi_pf_params; + struct ecore_rdma_pf_params rdma_pf_params; }; #endif diff --git a/dpdk/drivers/net/qede/base/ecore_rt_defs.h b/dpdk/drivers/net/qede/base/ecore_rt_defs.h index 1f5139ea..1d085815 100644 --- a/dpdk/drivers/net/qede/base/ecore_rt_defs.h +++ b/dpdk/drivers/net/qede/base/ecore_rt_defs.h @@ -10,437 +10,524 @@ #define __RT_DEFS_H__ /* Runtime array offsets */ -#define DORQ_REG_PF_MAX_ICID_0_RT_OFFSET 0 -#define DORQ_REG_PF_MAX_ICID_1_RT_OFFSET 1 -#define DORQ_REG_PF_MAX_ICID_2_RT_OFFSET 2 -#define DORQ_REG_PF_MAX_ICID_3_RT_OFFSET 3 -#define DORQ_REG_PF_MAX_ICID_4_RT_OFFSET 4 -#define DORQ_REG_PF_MAX_ICID_5_RT_OFFSET 5 -#define DORQ_REG_PF_MAX_ICID_6_RT_OFFSET 6 -#define DORQ_REG_PF_MAX_ICID_7_RT_OFFSET 7 -#define DORQ_REG_VF_MAX_ICID_0_RT_OFFSET 8 -#define DORQ_REG_VF_MAX_ICID_1_RT_OFFSET 9 -#define DORQ_REG_VF_MAX_ICID_2_RT_OFFSET 10 -#define DORQ_REG_VF_MAX_ICID_3_RT_OFFSET 11 -#define DORQ_REG_VF_MAX_ICID_4_RT_OFFSET 12 -#define DORQ_REG_VF_MAX_ICID_5_RT_OFFSET 13 -#define DORQ_REG_VF_MAX_ICID_6_RT_OFFSET 14 -#define DORQ_REG_VF_MAX_ICID_7_RT_OFFSET 15 -#define DORQ_REG_PF_WAKE_ALL_RT_OFFSET 16 -#define DORQ_REG_TAG1_ETHERTYPE_RT_OFFSET 17 -#define IGU_REG_PF_CONFIGURATION_RT_OFFSET 18 -#define IGU_REG_VF_CONFIGURATION_RT_OFFSET 19 -#define IGU_REG_ATTN_MSG_ADDR_L_RT_OFFSET 20 -#define IGU_REG_ATTN_MSG_ADDR_H_RT_OFFSET 21 -#define IGU_REG_LEADING_EDGE_LATCH_RT_OFFSET 22 -#define IGU_REG_TRAILING_EDGE_LATCH_RT_OFFSET 23 -#define CAU_REG_CQE_AGG_UNIT_SIZE_RT_OFFSET 24 -#define CAU_REG_SB_VAR_MEMORY_RT_OFFSET 761 -#define CAU_REG_SB_VAR_MEMORY_RT_SIZE 736 -#define CAU_REG_SB_VAR_MEMORY_RT_OFFSET 761 -#define CAU_REG_SB_VAR_MEMORY_RT_SIZE 736 -#define CAU_REG_SB_ADDR_MEMORY_RT_OFFSET 1497 -#define CAU_REG_SB_ADDR_MEMORY_RT_SIZE 736 -#define CAU_REG_PI_MEMORY_RT_OFFSET 2233 -#define CAU_REG_PI_MEMORY_RT_SIZE 4416 -#define PRS_REG_SEARCH_RESP_INITIATOR_TYPE_RT_OFFSET 6649 -#define PRS_REG_TASK_ID_MAX_INITIATOR_PF_RT_OFFSET 6650 -#define PRS_REG_TASK_ID_MAX_INITIATOR_VF_RT_OFFSET 6651 -#define PRS_REG_TASK_ID_MAX_TARGET_PF_RT_OFFSET 6652 -#define PRS_REG_TASK_ID_MAX_TARGET_VF_RT_OFFSET 6653 -#define PRS_REG_SEARCH_TCP_RT_OFFSET 6654 -#define PRS_REG_SEARCH_OPENFLOW_RT_OFFSET 6659 -#define PRS_REG_SEARCH_NON_IP_AS_OPENFLOW_RT_OFFSET 6660 -#define PRS_REG_OPENFLOW_SUPPORT_ONLY_KNOWN_OVER_IP_RT_OFFSET 6661 -#define PRS_REG_OPENFLOW_SEARCH_KEY_MASK_RT_OFFSET 6662 -#define PRS_REG_TAG_ETHERTYPE_0_RT_OFFSET 6663 -#define PRS_REG_LIGHT_L2_ETHERTYPE_EN_RT_OFFSET 6664 -#define SRC_REG_FIRSTFREE_RT_OFFSET 6665 -#define SRC_REG_FIRSTFREE_RT_SIZE 2 -#define SRC_REG_LASTFREE_RT_OFFSET 6667 -#define SRC_REG_LASTFREE_RT_SIZE 2 -#define SRC_REG_COUNTFREE_RT_OFFSET 6669 -#define SRC_REG_NUMBER_HASH_BITS_RT_OFFSET 6670 -#define PSWRQ2_REG_CDUT_P_SIZE_RT_OFFSET 6671 -#define PSWRQ2_REG_CDUC_P_SIZE_RT_OFFSET 6672 -#define PSWRQ2_REG_TM_P_SIZE_RT_OFFSET 6673 -#define PSWRQ2_REG_QM_P_SIZE_RT_OFFSET 6674 -#define PSWRQ2_REG_SRC_P_SIZE_RT_OFFSET 6675 -#define PSWRQ2_REG_TSDM_P_SIZE_RT_OFFSET 6676 -#define PSWRQ2_REG_TM_FIRST_ILT_RT_OFFSET 6677 -#define PSWRQ2_REG_TM_LAST_ILT_RT_OFFSET 6678 -#define PSWRQ2_REG_QM_FIRST_ILT_RT_OFFSET 6679 -#define PSWRQ2_REG_QM_LAST_ILT_RT_OFFSET 6680 -#define PSWRQ2_REG_SRC_FIRST_ILT_RT_OFFSET 6681 -#define PSWRQ2_REG_SRC_LAST_ILT_RT_OFFSET 6682 -#define PSWRQ2_REG_CDUC_FIRST_ILT_RT_OFFSET 6683 -#define PSWRQ2_REG_CDUC_LAST_ILT_RT_OFFSET 6684 -#define PSWRQ2_REG_CDUT_FIRST_ILT_RT_OFFSET 6685 -#define PSWRQ2_REG_CDUT_LAST_ILT_RT_OFFSET 6686 -#define PSWRQ2_REG_TSDM_FIRST_ILT_RT_OFFSET 6687 -#define PSWRQ2_REG_TSDM_LAST_ILT_RT_OFFSET 6688 -#define PSWRQ2_REG_TM_NUMBER_OF_PF_BLOCKS_RT_OFFSET 6689 -#define PSWRQ2_REG_CDUT_NUMBER_OF_PF_BLOCKS_RT_OFFSET 6690 -#define PSWRQ2_REG_CDUC_NUMBER_OF_PF_BLOCKS_RT_OFFSET 6691 -#define PSWRQ2_REG_TM_VF_BLOCKS_RT_OFFSET 6692 -#define PSWRQ2_REG_CDUT_VF_BLOCKS_RT_OFFSET 6693 -#define PSWRQ2_REG_CDUC_VF_BLOCKS_RT_OFFSET 6694 -#define PSWRQ2_REG_TM_BLOCKS_FACTOR_RT_OFFSET 6695 -#define PSWRQ2_REG_CDUT_BLOCKS_FACTOR_RT_OFFSET 6696 -#define PSWRQ2_REG_CDUC_BLOCKS_FACTOR_RT_OFFSET 6697 -#define PSWRQ2_REG_VF_BASE_RT_OFFSET 6698 -#define PSWRQ2_REG_VF_LAST_ILT_RT_OFFSET 6699 -#define PSWRQ2_REG_WR_MBS0_RT_OFFSET 6700 -#define PSWRQ2_REG_RD_MBS0_RT_OFFSET 6701 -#define PSWRQ2_REG_DRAM_ALIGN_WR_RT_OFFSET 6702 -#define PSWRQ2_REG_DRAM_ALIGN_RD_RT_OFFSET 6703 -#define PSWRQ2_REG_ILT_MEMORY_RT_OFFSET 6704 -#define PSWRQ2_REG_ILT_MEMORY_RT_SIZE 22000 -#define PGLUE_REG_B_VF_BASE_RT_OFFSET 28704 -#define PGLUE_REG_B_CACHE_LINE_SIZE_RT_OFFSET 28705 -#define PGLUE_REG_B_PF_BAR0_SIZE_RT_OFFSET 28706 -#define PGLUE_REG_B_PF_BAR1_SIZE_RT_OFFSET 28707 -#define PGLUE_REG_B_VF_BAR1_SIZE_RT_OFFSET 28708 -#define TM_REG_VF_ENABLE_CONN_RT_OFFSET 28709 -#define TM_REG_PF_ENABLE_CONN_RT_OFFSET 28710 -#define TM_REG_PF_ENABLE_TASK_RT_OFFSET 28711 -#define TM_REG_GROUP_SIZE_RESOLUTION_CONN_RT_OFFSET 28712 -#define TM_REG_GROUP_SIZE_RESOLUTION_TASK_RT_OFFSET 28713 -#define TM_REG_CONFIG_CONN_MEM_RT_OFFSET 28714 -#define TM_REG_CONFIG_CONN_MEM_RT_SIZE 416 -#define TM_REG_CONFIG_TASK_MEM_RT_OFFSET 29130 -#define TM_REG_CONFIG_TASK_MEM_RT_SIZE 512 -#define QM_REG_MAXPQSIZE_0_RT_OFFSET 29642 -#define QM_REG_MAXPQSIZE_1_RT_OFFSET 29643 -#define QM_REG_MAXPQSIZE_2_RT_OFFSET 29644 -#define QM_REG_MAXPQSIZETXSEL_0_RT_OFFSET 29645 -#define QM_REG_MAXPQSIZETXSEL_1_RT_OFFSET 29646 -#define QM_REG_MAXPQSIZETXSEL_2_RT_OFFSET 29647 -#define QM_REG_MAXPQSIZETXSEL_3_RT_OFFSET 29648 -#define QM_REG_MAXPQSIZETXSEL_4_RT_OFFSET 29649 -#define QM_REG_MAXPQSIZETXSEL_5_RT_OFFSET 29650 -#define QM_REG_MAXPQSIZETXSEL_6_RT_OFFSET 29651 -#define QM_REG_MAXPQSIZETXSEL_7_RT_OFFSET 29652 -#define QM_REG_MAXPQSIZETXSEL_8_RT_OFFSET 29653 -#define QM_REG_MAXPQSIZETXSEL_9_RT_OFFSET 29654 -#define QM_REG_MAXPQSIZETXSEL_10_RT_OFFSET 29655 -#define QM_REG_MAXPQSIZETXSEL_11_RT_OFFSET 29656 -#define QM_REG_MAXPQSIZETXSEL_12_RT_OFFSET 29657 -#define QM_REG_MAXPQSIZETXSEL_13_RT_OFFSET 29658 -#define QM_REG_MAXPQSIZETXSEL_14_RT_OFFSET 29659 -#define QM_REG_MAXPQSIZETXSEL_15_RT_OFFSET 29660 -#define QM_REG_MAXPQSIZETXSEL_16_RT_OFFSET 29661 -#define QM_REG_MAXPQSIZETXSEL_17_RT_OFFSET 29662 -#define QM_REG_MAXPQSIZETXSEL_18_RT_OFFSET 29663 -#define QM_REG_MAXPQSIZETXSEL_19_RT_OFFSET 29664 -#define QM_REG_MAXPQSIZETXSEL_20_RT_OFFSET 29665 -#define QM_REG_MAXPQSIZETXSEL_21_RT_OFFSET 29666 -#define QM_REG_MAXPQSIZETXSEL_22_RT_OFFSET 29667 -#define QM_REG_MAXPQSIZETXSEL_23_RT_OFFSET 29668 -#define QM_REG_MAXPQSIZETXSEL_24_RT_OFFSET 29669 -#define QM_REG_MAXPQSIZETXSEL_25_RT_OFFSET 29670 -#define QM_REG_MAXPQSIZETXSEL_26_RT_OFFSET 29671 -#define QM_REG_MAXPQSIZETXSEL_27_RT_OFFSET 29672 -#define QM_REG_MAXPQSIZETXSEL_28_RT_OFFSET 29673 -#define QM_REG_MAXPQSIZETXSEL_29_RT_OFFSET 29674 -#define QM_REG_MAXPQSIZETXSEL_30_RT_OFFSET 29675 -#define QM_REG_MAXPQSIZETXSEL_31_RT_OFFSET 29676 -#define QM_REG_MAXPQSIZETXSEL_32_RT_OFFSET 29677 -#define QM_REG_MAXPQSIZETXSEL_33_RT_OFFSET 29678 -#define QM_REG_MAXPQSIZETXSEL_34_RT_OFFSET 29679 -#define QM_REG_MAXPQSIZETXSEL_35_RT_OFFSET 29680 -#define QM_REG_MAXPQSIZETXSEL_36_RT_OFFSET 29681 -#define QM_REG_MAXPQSIZETXSEL_37_RT_OFFSET 29682 -#define QM_REG_MAXPQSIZETXSEL_38_RT_OFFSET 29683 -#define QM_REG_MAXPQSIZETXSEL_39_RT_OFFSET 29684 -#define QM_REG_MAXPQSIZETXSEL_40_RT_OFFSET 29685 -#define QM_REG_MAXPQSIZETXSEL_41_RT_OFFSET 29686 -#define QM_REG_MAXPQSIZETXSEL_42_RT_OFFSET 29687 -#define QM_REG_MAXPQSIZETXSEL_43_RT_OFFSET 29688 -#define QM_REG_MAXPQSIZETXSEL_44_RT_OFFSET 29689 -#define QM_REG_MAXPQSIZETXSEL_45_RT_OFFSET 29690 -#define QM_REG_MAXPQSIZETXSEL_46_RT_OFFSET 29691 -#define QM_REG_MAXPQSIZETXSEL_47_RT_OFFSET 29692 -#define QM_REG_MAXPQSIZETXSEL_48_RT_OFFSET 29693 -#define QM_REG_MAXPQSIZETXSEL_49_RT_OFFSET 29694 -#define QM_REG_MAXPQSIZETXSEL_50_RT_OFFSET 29695 -#define QM_REG_MAXPQSIZETXSEL_51_RT_OFFSET 29696 -#define QM_REG_MAXPQSIZETXSEL_52_RT_OFFSET 29697 -#define QM_REG_MAXPQSIZETXSEL_53_RT_OFFSET 29698 -#define QM_REG_MAXPQSIZETXSEL_54_RT_OFFSET 29699 -#define QM_REG_MAXPQSIZETXSEL_55_RT_OFFSET 29700 -#define QM_REG_MAXPQSIZETXSEL_56_RT_OFFSET 29701 -#define QM_REG_MAXPQSIZETXSEL_57_RT_OFFSET 29702 -#define QM_REG_MAXPQSIZETXSEL_58_RT_OFFSET 29703 -#define QM_REG_MAXPQSIZETXSEL_59_RT_OFFSET 29704 -#define QM_REG_MAXPQSIZETXSEL_60_RT_OFFSET 29705 -#define QM_REG_MAXPQSIZETXSEL_61_RT_OFFSET 29706 -#define QM_REG_MAXPQSIZETXSEL_62_RT_OFFSET 29707 -#define QM_REG_MAXPQSIZETXSEL_63_RT_OFFSET 29708 -#define QM_REG_BASEADDROTHERPQ_RT_OFFSET 29709 -#define QM_REG_BASEADDROTHERPQ_RT_SIZE 128 -#define QM_REG_VOQCRDLINE_RT_OFFSET 29837 -#define QM_REG_VOQCRDLINE_RT_SIZE 20 -#define QM_REG_VOQINITCRDLINE_RT_OFFSET 29857 -#define QM_REG_VOQINITCRDLINE_RT_SIZE 20 -#define QM_REG_AFULLQMBYPTHRPFWFQ_RT_OFFSET 29877 -#define QM_REG_AFULLQMBYPTHRVPWFQ_RT_OFFSET 29878 -#define QM_REG_AFULLQMBYPTHRPFRL_RT_OFFSET 29879 -#define QM_REG_AFULLQMBYPTHRGLBLRL_RT_OFFSET 29880 -#define QM_REG_AFULLOPRTNSTCCRDMASK_RT_OFFSET 29881 -#define QM_REG_WRROTHERPQGRP_0_RT_OFFSET 29882 -#define QM_REG_WRROTHERPQGRP_1_RT_OFFSET 29883 -#define QM_REG_WRROTHERPQGRP_2_RT_OFFSET 29884 -#define QM_REG_WRROTHERPQGRP_3_RT_OFFSET 29885 -#define QM_REG_WRROTHERPQGRP_4_RT_OFFSET 29886 -#define QM_REG_WRROTHERPQGRP_5_RT_OFFSET 29887 -#define QM_REG_WRROTHERPQGRP_6_RT_OFFSET 29888 -#define QM_REG_WRROTHERPQGRP_7_RT_OFFSET 29889 -#define QM_REG_WRROTHERPQGRP_8_RT_OFFSET 29890 -#define QM_REG_WRROTHERPQGRP_9_RT_OFFSET 29891 -#define QM_REG_WRROTHERPQGRP_10_RT_OFFSET 29892 -#define QM_REG_WRROTHERPQGRP_11_RT_OFFSET 29893 -#define QM_REG_WRROTHERPQGRP_12_RT_OFFSET 29894 -#define QM_REG_WRROTHERPQGRP_13_RT_OFFSET 29895 -#define QM_REG_WRROTHERPQGRP_14_RT_OFFSET 29896 -#define QM_REG_WRROTHERPQGRP_15_RT_OFFSET 29897 -#define QM_REG_WRROTHERGRPWEIGHT_0_RT_OFFSET 29898 -#define QM_REG_WRROTHERGRPWEIGHT_1_RT_OFFSET 29899 -#define QM_REG_WRROTHERGRPWEIGHT_2_RT_OFFSET 29900 -#define QM_REG_WRROTHERGRPWEIGHT_3_RT_OFFSET 29901 -#define QM_REG_WRRTXGRPWEIGHT_0_RT_OFFSET 29902 -#define QM_REG_WRRTXGRPWEIGHT_1_RT_OFFSET 29903 -#define QM_REG_PQTX2PF_0_RT_OFFSET 29904 -#define QM_REG_PQTX2PF_1_RT_OFFSET 29905 -#define QM_REG_PQTX2PF_2_RT_OFFSET 29906 -#define QM_REG_PQTX2PF_3_RT_OFFSET 29907 -#define QM_REG_PQTX2PF_4_RT_OFFSET 29908 -#define QM_REG_PQTX2PF_5_RT_OFFSET 29909 -#define QM_REG_PQTX2PF_6_RT_OFFSET 29910 -#define QM_REG_PQTX2PF_7_RT_OFFSET 29911 -#define QM_REG_PQTX2PF_8_RT_OFFSET 29912 -#define QM_REG_PQTX2PF_9_RT_OFFSET 29913 -#define QM_REG_PQTX2PF_10_RT_OFFSET 29914 -#define QM_REG_PQTX2PF_11_RT_OFFSET 29915 -#define QM_REG_PQTX2PF_12_RT_OFFSET 29916 -#define QM_REG_PQTX2PF_13_RT_OFFSET 29917 -#define QM_REG_PQTX2PF_14_RT_OFFSET 29918 -#define QM_REG_PQTX2PF_15_RT_OFFSET 29919 -#define QM_REG_PQTX2PF_16_RT_OFFSET 29920 -#define QM_REG_PQTX2PF_17_RT_OFFSET 29921 -#define QM_REG_PQTX2PF_18_RT_OFFSET 29922 -#define QM_REG_PQTX2PF_19_RT_OFFSET 29923 -#define QM_REG_PQTX2PF_20_RT_OFFSET 29924 -#define QM_REG_PQTX2PF_21_RT_OFFSET 29925 -#define QM_REG_PQTX2PF_22_RT_OFFSET 29926 -#define QM_REG_PQTX2PF_23_RT_OFFSET 29927 -#define QM_REG_PQTX2PF_24_RT_OFFSET 29928 -#define QM_REG_PQTX2PF_25_RT_OFFSET 29929 -#define QM_REG_PQTX2PF_26_RT_OFFSET 29930 -#define QM_REG_PQTX2PF_27_RT_OFFSET 29931 -#define QM_REG_PQTX2PF_28_RT_OFFSET 29932 -#define QM_REG_PQTX2PF_29_RT_OFFSET 29933 -#define QM_REG_PQTX2PF_30_RT_OFFSET 29934 -#define QM_REG_PQTX2PF_31_RT_OFFSET 29935 -#define QM_REG_PQTX2PF_32_RT_OFFSET 29936 -#define QM_REG_PQTX2PF_33_RT_OFFSET 29937 -#define QM_REG_PQTX2PF_34_RT_OFFSET 29938 -#define QM_REG_PQTX2PF_35_RT_OFFSET 29939 -#define QM_REG_PQTX2PF_36_RT_OFFSET 29940 -#define QM_REG_PQTX2PF_37_RT_OFFSET 29941 -#define QM_REG_PQTX2PF_38_RT_OFFSET 29942 -#define QM_REG_PQTX2PF_39_RT_OFFSET 29943 -#define QM_REG_PQTX2PF_40_RT_OFFSET 29944 -#define QM_REG_PQTX2PF_41_RT_OFFSET 29945 -#define QM_REG_PQTX2PF_42_RT_OFFSET 29946 -#define QM_REG_PQTX2PF_43_RT_OFFSET 29947 -#define QM_REG_PQTX2PF_44_RT_OFFSET 29948 -#define QM_REG_PQTX2PF_45_RT_OFFSET 29949 -#define QM_REG_PQTX2PF_46_RT_OFFSET 29950 -#define QM_REG_PQTX2PF_47_RT_OFFSET 29951 -#define QM_REG_PQTX2PF_48_RT_OFFSET 29952 -#define QM_REG_PQTX2PF_49_RT_OFFSET 29953 -#define QM_REG_PQTX2PF_50_RT_OFFSET 29954 -#define QM_REG_PQTX2PF_51_RT_OFFSET 29955 -#define QM_REG_PQTX2PF_52_RT_OFFSET 29956 -#define QM_REG_PQTX2PF_53_RT_OFFSET 29957 -#define QM_REG_PQTX2PF_54_RT_OFFSET 29958 -#define QM_REG_PQTX2PF_55_RT_OFFSET 29959 -#define QM_REG_PQTX2PF_56_RT_OFFSET 29960 -#define QM_REG_PQTX2PF_57_RT_OFFSET 29961 -#define QM_REG_PQTX2PF_58_RT_OFFSET 29962 -#define QM_REG_PQTX2PF_59_RT_OFFSET 29963 -#define QM_REG_PQTX2PF_60_RT_OFFSET 29964 -#define QM_REG_PQTX2PF_61_RT_OFFSET 29965 -#define QM_REG_PQTX2PF_62_RT_OFFSET 29966 -#define QM_REG_PQTX2PF_63_RT_OFFSET 29967 -#define QM_REG_PQOTHER2PF_0_RT_OFFSET 29968 -#define QM_REG_PQOTHER2PF_1_RT_OFFSET 29969 -#define QM_REG_PQOTHER2PF_2_RT_OFFSET 29970 -#define QM_REG_PQOTHER2PF_3_RT_OFFSET 29971 -#define QM_REG_PQOTHER2PF_4_RT_OFFSET 29972 -#define QM_REG_PQOTHER2PF_5_RT_OFFSET 29973 -#define QM_REG_PQOTHER2PF_6_RT_OFFSET 29974 -#define QM_REG_PQOTHER2PF_7_RT_OFFSET 29975 -#define QM_REG_PQOTHER2PF_8_RT_OFFSET 29976 -#define QM_REG_PQOTHER2PF_9_RT_OFFSET 29977 -#define QM_REG_PQOTHER2PF_10_RT_OFFSET 29978 -#define QM_REG_PQOTHER2PF_11_RT_OFFSET 29979 -#define QM_REG_PQOTHER2PF_12_RT_OFFSET 29980 -#define QM_REG_PQOTHER2PF_13_RT_OFFSET 29981 -#define QM_REG_PQOTHER2PF_14_RT_OFFSET 29982 -#define QM_REG_PQOTHER2PF_15_RT_OFFSET 29983 -#define QM_REG_RLGLBLPERIOD_0_RT_OFFSET 29984 -#define QM_REG_RLGLBLPERIOD_1_RT_OFFSET 29985 -#define QM_REG_RLGLBLPERIODTIMER_0_RT_OFFSET 29986 -#define QM_REG_RLGLBLPERIODTIMER_1_RT_OFFSET 29987 -#define QM_REG_RLGLBLPERIODSEL_0_RT_OFFSET 29988 -#define QM_REG_RLGLBLPERIODSEL_1_RT_OFFSET 29989 -#define QM_REG_RLGLBLPERIODSEL_2_RT_OFFSET 29990 -#define QM_REG_RLGLBLPERIODSEL_3_RT_OFFSET 29991 -#define QM_REG_RLGLBLPERIODSEL_4_RT_OFFSET 29992 -#define QM_REG_RLGLBLPERIODSEL_5_RT_OFFSET 29993 -#define QM_REG_RLGLBLPERIODSEL_6_RT_OFFSET 29994 -#define QM_REG_RLGLBLPERIODSEL_7_RT_OFFSET 29995 -#define QM_REG_RLGLBLINCVAL_RT_OFFSET 29996 -#define QM_REG_RLGLBLINCVAL_RT_SIZE 256 -#define QM_REG_RLGLBLUPPERBOUND_RT_OFFSET 30252 -#define QM_REG_RLGLBLUPPERBOUND_RT_SIZE 256 -#define QM_REG_RLGLBLCRD_RT_OFFSET 30508 -#define QM_REG_RLGLBLCRD_RT_SIZE 256 -#define QM_REG_RLGLBLENABLE_RT_OFFSET 30764 -#define QM_REG_RLPFPERIOD_RT_OFFSET 30765 -#define QM_REG_RLPFPERIODTIMER_RT_OFFSET 30766 -#define QM_REG_RLPFINCVAL_RT_OFFSET 30767 -#define QM_REG_RLPFINCVAL_RT_SIZE 16 -#define QM_REG_RLPFUPPERBOUND_RT_OFFSET 30783 -#define QM_REG_RLPFUPPERBOUND_RT_SIZE 16 -#define QM_REG_RLPFCRD_RT_OFFSET 30799 -#define QM_REG_RLPFCRD_RT_SIZE 16 -#define QM_REG_RLPFENABLE_RT_OFFSET 30815 -#define QM_REG_RLPFVOQENABLE_RT_OFFSET 30816 -#define QM_REG_WFQPFWEIGHT_RT_OFFSET 30817 -#define QM_REG_WFQPFWEIGHT_RT_SIZE 16 -#define QM_REG_WFQPFUPPERBOUND_RT_OFFSET 30833 -#define QM_REG_WFQPFUPPERBOUND_RT_SIZE 16 -#define QM_REG_WFQPFCRD_RT_OFFSET 30849 -#define QM_REG_WFQPFCRD_RT_SIZE 160 -#define QM_REG_WFQPFENABLE_RT_OFFSET 31009 -#define QM_REG_WFQVPENABLE_RT_OFFSET 31010 -#define QM_REG_BASEADDRTXPQ_RT_OFFSET 31011 -#define QM_REG_BASEADDRTXPQ_RT_SIZE 512 -#define QM_REG_TXPQMAP_RT_OFFSET 31523 -#define QM_REG_TXPQMAP_RT_SIZE 512 -#define QM_REG_WFQVPWEIGHT_RT_OFFSET 32035 -#define QM_REG_WFQVPWEIGHT_RT_SIZE 512 -#define QM_REG_WFQVPCRD_RT_OFFSET 32547 -#define QM_REG_WFQVPCRD_RT_SIZE 512 -#define QM_REG_WFQVPMAP_RT_OFFSET 33059 -#define QM_REG_WFQVPMAP_RT_SIZE 512 -#define QM_REG_WFQPFCRD_MSB_RT_OFFSET 33571 -#define QM_REG_WFQPFCRD_MSB_RT_SIZE 160 -#define NIG_REG_TAG_ETHERTYPE_0_RT_OFFSET 33731 -#define NIG_REG_OUTER_TAG_VALUE_LIST0_RT_OFFSET 33732 -#define NIG_REG_OUTER_TAG_VALUE_LIST1_RT_OFFSET 33733 -#define NIG_REG_OUTER_TAG_VALUE_LIST2_RT_OFFSET 33734 -#define NIG_REG_OUTER_TAG_VALUE_LIST3_RT_OFFSET 33735 -#define NIG_REG_OUTER_TAG_VALUE_MASK_RT_OFFSET 33736 -#define NIG_REG_LLH_FUNC_TAGMAC_CLS_TYPE_RT_OFFSET 33737 -#define NIG_REG_LLH_FUNC_TAG_EN_RT_OFFSET 33738 -#define NIG_REG_LLH_FUNC_TAG_EN_RT_SIZE 4 -#define NIG_REG_LLH_FUNC_TAG_HDR_SEL_RT_OFFSET 33742 -#define NIG_REG_LLH_FUNC_TAG_HDR_SEL_RT_SIZE 4 -#define NIG_REG_LLH_FUNC_TAG_VALUE_RT_OFFSET 33746 -#define NIG_REG_LLH_FUNC_TAG_VALUE_RT_SIZE 4 -#define NIG_REG_LLH_FUNC_NO_TAG_RT_OFFSET 33750 -#define NIG_REG_LLH_FUNC_FILTER_VALUE_RT_OFFSET 33751 -#define NIG_REG_LLH_FUNC_FILTER_VALUE_RT_SIZE 32 -#define NIG_REG_LLH_FUNC_FILTER_EN_RT_OFFSET 33783 -#define NIG_REG_LLH_FUNC_FILTER_EN_RT_SIZE 16 -#define NIG_REG_LLH_FUNC_FILTER_MODE_RT_OFFSET 33799 -#define NIG_REG_LLH_FUNC_FILTER_MODE_RT_SIZE 16 -#define NIG_REG_LLH_FUNC_FILTER_PROTOCOL_TYPE_RT_OFFSET 33815 -#define NIG_REG_LLH_FUNC_FILTER_PROTOCOL_TYPE_RT_SIZE 16 -#define NIG_REG_LLH_FUNC_FILTER_HDR_SEL_RT_OFFSET 33831 -#define NIG_REG_LLH_FUNC_FILTER_HDR_SEL_RT_SIZE 16 -#define NIG_REG_TX_EDPM_CTRL_RT_OFFSET 33847 -#define CDU_REG_CID_ADDR_PARAMS_RT_OFFSET 33848 -#define CDU_REG_SEGMENT0_PARAMS_RT_OFFSET 33849 -#define CDU_REG_SEGMENT1_PARAMS_RT_OFFSET 33850 -#define CDU_REG_PF_SEG0_TYPE_OFFSET_RT_OFFSET 33851 -#define CDU_REG_PF_SEG1_TYPE_OFFSET_RT_OFFSET 33852 -#define CDU_REG_PF_SEG2_TYPE_OFFSET_RT_OFFSET 33853 -#define CDU_REG_PF_SEG3_TYPE_OFFSET_RT_OFFSET 33854 -#define CDU_REG_PF_FL_SEG0_TYPE_OFFSET_RT_OFFSET 33855 -#define CDU_REG_PF_FL_SEG1_TYPE_OFFSET_RT_OFFSET 33856 -#define CDU_REG_PF_FL_SEG2_TYPE_OFFSET_RT_OFFSET 33857 -#define CDU_REG_PF_FL_SEG3_TYPE_OFFSET_RT_OFFSET 33858 -#define CDU_REG_VF_SEG_TYPE_OFFSET_RT_OFFSET 33859 -#define CDU_REG_VF_FL_SEG_TYPE_OFFSET_RT_OFFSET 33860 -#define PBF_REG_TAG_ETHERTYPE_0_RT_OFFSET 33861 -#define PBF_REG_BTB_SHARED_AREA_SIZE_RT_OFFSET 33862 -#define PBF_REG_YCMD_QS_NUM_LINES_VOQ0_RT_OFFSET 33863 -#define PBF_REG_BTB_GUARANTEED_VOQ0_RT_OFFSET 33864 -#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ0_RT_OFFSET 33865 -#define PBF_REG_YCMD_QS_NUM_LINES_VOQ1_RT_OFFSET 33866 -#define PBF_REG_BTB_GUARANTEED_VOQ1_RT_OFFSET 33867 -#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ1_RT_OFFSET 33868 -#define PBF_REG_YCMD_QS_NUM_LINES_VOQ2_RT_OFFSET 33869 -#define PBF_REG_BTB_GUARANTEED_VOQ2_RT_OFFSET 33870 -#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ2_RT_OFFSET 33871 -#define PBF_REG_YCMD_QS_NUM_LINES_VOQ3_RT_OFFSET 33872 -#define PBF_REG_BTB_GUARANTEED_VOQ3_RT_OFFSET 33873 -#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ3_RT_OFFSET 33874 -#define PBF_REG_YCMD_QS_NUM_LINES_VOQ4_RT_OFFSET 33875 -#define PBF_REG_BTB_GUARANTEED_VOQ4_RT_OFFSET 33876 -#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ4_RT_OFFSET 33877 -#define PBF_REG_YCMD_QS_NUM_LINES_VOQ5_RT_OFFSET 33878 -#define PBF_REG_BTB_GUARANTEED_VOQ5_RT_OFFSET 33879 -#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ5_RT_OFFSET 33880 -#define PBF_REG_YCMD_QS_NUM_LINES_VOQ6_RT_OFFSET 33881 -#define PBF_REG_BTB_GUARANTEED_VOQ6_RT_OFFSET 33882 -#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ6_RT_OFFSET 33883 -#define PBF_REG_YCMD_QS_NUM_LINES_VOQ7_RT_OFFSET 33884 -#define PBF_REG_BTB_GUARANTEED_VOQ7_RT_OFFSET 33885 -#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ7_RT_OFFSET 33886 -#define PBF_REG_YCMD_QS_NUM_LINES_VOQ8_RT_OFFSET 33887 -#define PBF_REG_BTB_GUARANTEED_VOQ8_RT_OFFSET 33888 -#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ8_RT_OFFSET 33889 -#define PBF_REG_YCMD_QS_NUM_LINES_VOQ9_RT_OFFSET 33890 -#define PBF_REG_BTB_GUARANTEED_VOQ9_RT_OFFSET 33891 -#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ9_RT_OFFSET 33892 -#define PBF_REG_YCMD_QS_NUM_LINES_VOQ10_RT_OFFSET 33893 -#define PBF_REG_BTB_GUARANTEED_VOQ10_RT_OFFSET 33894 -#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ10_RT_OFFSET 33895 -#define PBF_REG_YCMD_QS_NUM_LINES_VOQ11_RT_OFFSET 33896 -#define PBF_REG_BTB_GUARANTEED_VOQ11_RT_OFFSET 33897 -#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ11_RT_OFFSET 33898 -#define PBF_REG_YCMD_QS_NUM_LINES_VOQ12_RT_OFFSET 33899 -#define PBF_REG_BTB_GUARANTEED_VOQ12_RT_OFFSET 33900 -#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ12_RT_OFFSET 33901 -#define PBF_REG_YCMD_QS_NUM_LINES_VOQ13_RT_OFFSET 33902 -#define PBF_REG_BTB_GUARANTEED_VOQ13_RT_OFFSET 33903 -#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ13_RT_OFFSET 33904 -#define PBF_REG_YCMD_QS_NUM_LINES_VOQ14_RT_OFFSET 33905 -#define PBF_REG_BTB_GUARANTEED_VOQ14_RT_OFFSET 33906 -#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ14_RT_OFFSET 33907 -#define PBF_REG_YCMD_QS_NUM_LINES_VOQ15_RT_OFFSET 33908 -#define PBF_REG_BTB_GUARANTEED_VOQ15_RT_OFFSET 33909 -#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ15_RT_OFFSET 33910 -#define PBF_REG_YCMD_QS_NUM_LINES_VOQ16_RT_OFFSET 33911 -#define PBF_REG_BTB_GUARANTEED_VOQ16_RT_OFFSET 33912 -#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ16_RT_OFFSET 33913 -#define PBF_REG_YCMD_QS_NUM_LINES_VOQ17_RT_OFFSET 33914 -#define PBF_REG_BTB_GUARANTEED_VOQ17_RT_OFFSET 33915 -#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ17_RT_OFFSET 33916 -#define PBF_REG_YCMD_QS_NUM_LINES_VOQ18_RT_OFFSET 33917 -#define PBF_REG_BTB_GUARANTEED_VOQ18_RT_OFFSET 33918 -#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ18_RT_OFFSET 33919 -#define PBF_REG_YCMD_QS_NUM_LINES_VOQ19_RT_OFFSET 33920 -#define PBF_REG_BTB_GUARANTEED_VOQ19_RT_OFFSET 33921 -#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ19_RT_OFFSET 33922 -#define XCM_REG_CON_PHY_Q3_RT_OFFSET 33923 +#define DORQ_REG_PF_MAX_ICID_0_RT_OFFSET 0 +#define DORQ_REG_PF_MAX_ICID_1_RT_OFFSET 1 +#define DORQ_REG_PF_MAX_ICID_2_RT_OFFSET 2 +#define DORQ_REG_PF_MAX_ICID_3_RT_OFFSET 3 +#define DORQ_REG_PF_MAX_ICID_4_RT_OFFSET 4 +#define DORQ_REG_PF_MAX_ICID_5_RT_OFFSET 5 +#define DORQ_REG_PF_MAX_ICID_6_RT_OFFSET 6 +#define DORQ_REG_PF_MAX_ICID_7_RT_OFFSET 7 +#define DORQ_REG_VF_MAX_ICID_0_RT_OFFSET 8 +#define DORQ_REG_VF_MAX_ICID_1_RT_OFFSET 9 +#define DORQ_REG_VF_MAX_ICID_2_RT_OFFSET 10 +#define DORQ_REG_VF_MAX_ICID_3_RT_OFFSET 11 +#define DORQ_REG_VF_MAX_ICID_4_RT_OFFSET 12 +#define DORQ_REG_VF_MAX_ICID_5_RT_OFFSET 13 +#define DORQ_REG_VF_MAX_ICID_6_RT_OFFSET 14 +#define DORQ_REG_VF_MAX_ICID_7_RT_OFFSET 15 +#define DORQ_REG_PF_WAKE_ALL_RT_OFFSET 16 +#define DORQ_REG_TAG1_ETHERTYPE_RT_OFFSET 17 +#define DORQ_REG_GLB_MAX_ICID_0_RT_OFFSET 18 +#define DORQ_REG_GLB_MAX_ICID_1_RT_OFFSET 19 +#define DORQ_REG_GLB_RANGE2CONN_TYPE_0_RT_OFFSET 20 +#define DORQ_REG_GLB_RANGE2CONN_TYPE_1_RT_OFFSET 21 +#define DORQ_REG_PRV_PF_MAX_ICID_2_RT_OFFSET 22 +#define DORQ_REG_PRV_PF_MAX_ICID_3_RT_OFFSET 23 +#define DORQ_REG_PRV_PF_MAX_ICID_4_RT_OFFSET 24 +#define DORQ_REG_PRV_PF_MAX_ICID_5_RT_OFFSET 25 +#define DORQ_REG_PRV_VF_MAX_ICID_2_RT_OFFSET 26 +#define DORQ_REG_PRV_VF_MAX_ICID_3_RT_OFFSET 27 +#define DORQ_REG_PRV_VF_MAX_ICID_4_RT_OFFSET 28 +#define DORQ_REG_PRV_VF_MAX_ICID_5_RT_OFFSET 29 +#define DORQ_REG_PRV_PF_RANGE2CONN_TYPE_2_RT_OFFSET 30 +#define DORQ_REG_PRV_PF_RANGE2CONN_TYPE_3_RT_OFFSET 31 +#define DORQ_REG_PRV_PF_RANGE2CONN_TYPE_4_RT_OFFSET 32 +#define DORQ_REG_PRV_PF_RANGE2CONN_TYPE_5_RT_OFFSET 33 +#define DORQ_REG_PRV_VF_RANGE2CONN_TYPE_2_RT_OFFSET 34 +#define DORQ_REG_PRV_VF_RANGE2CONN_TYPE_3_RT_OFFSET 35 +#define DORQ_REG_PRV_VF_RANGE2CONN_TYPE_4_RT_OFFSET 36 +#define DORQ_REG_PRV_VF_RANGE2CONN_TYPE_5_RT_OFFSET 37 +#define IGU_REG_PF_CONFIGURATION_RT_OFFSET 38 +#define IGU_REG_VF_CONFIGURATION_RT_OFFSET 39 +#define IGU_REG_ATTN_MSG_ADDR_L_RT_OFFSET 40 +#define IGU_REG_ATTN_MSG_ADDR_H_RT_OFFSET 41 +#define IGU_REG_LEADING_EDGE_LATCH_RT_OFFSET 42 +#define IGU_REG_TRAILING_EDGE_LATCH_RT_OFFSET 43 +#define CAU_REG_CQE_AGG_UNIT_SIZE_RT_OFFSET 44 +#define CAU_REG_SB_VAR_MEMORY_RT_OFFSET 45 +#define CAU_REG_SB_VAR_MEMORY_RT_SIZE 1024 +#define CAU_REG_SB_ADDR_MEMORY_RT_OFFSET 1069 +#define CAU_REG_SB_ADDR_MEMORY_RT_SIZE 1024 +#define CAU_REG_PI_MEMORY_RT_OFFSET 2093 +#define CAU_REG_PI_MEMORY_RT_SIZE 4416 +#define PRS_REG_SEARCH_RESP_INITIATOR_TYPE_RT_OFFSET 6509 +#define PRS_REG_TASK_ID_MAX_INITIATOR_PF_RT_OFFSET 6510 +#define PRS_REG_TASK_ID_MAX_INITIATOR_VF_RT_OFFSET 6511 +#define PRS_REG_TASK_ID_MAX_TARGET_PF_RT_OFFSET 6512 +#define PRS_REG_TASK_ID_MAX_TARGET_VF_RT_OFFSET 6513 +#define PRS_REG_SEARCH_TCP_RT_OFFSET 6514 +#define PRS_REG_SEARCH_FCOE_RT_OFFSET 6515 +#define PRS_REG_SEARCH_ROCE_RT_OFFSET 6516 +#define PRS_REG_ROCE_DEST_QP_MAX_VF_RT_OFFSET 6517 +#define PRS_REG_ROCE_DEST_QP_MAX_PF_RT_OFFSET 6518 +#define PRS_REG_SEARCH_OPENFLOW_RT_OFFSET 6519 +#define PRS_REG_SEARCH_NON_IP_AS_OPENFLOW_RT_OFFSET 6520 +#define PRS_REG_OPENFLOW_SUPPORT_ONLY_KNOWN_OVER_IP_RT_OFFSET 6521 +#define PRS_REG_OPENFLOW_SEARCH_KEY_MASK_RT_OFFSET 6522 +#define PRS_REG_TAG_ETHERTYPE_0_RT_OFFSET 6523 +#define PRS_REG_LIGHT_L2_ETHERTYPE_EN_RT_OFFSET 6524 +#define SRC_REG_FIRSTFREE_RT_OFFSET 6525 +#define SRC_REG_FIRSTFREE_RT_SIZE 2 +#define SRC_REG_LASTFREE_RT_OFFSET 6527 +#define SRC_REG_LASTFREE_RT_SIZE 2 +#define SRC_REG_COUNTFREE_RT_OFFSET 6529 +#define SRC_REG_NUMBER_HASH_BITS_RT_OFFSET 6530 +#define PSWRQ2_REG_CDUT_P_SIZE_RT_OFFSET 6531 +#define PSWRQ2_REG_CDUC_P_SIZE_RT_OFFSET 6532 +#define PSWRQ2_REG_TM_P_SIZE_RT_OFFSET 6533 +#define PSWRQ2_REG_QM_P_SIZE_RT_OFFSET 6534 +#define PSWRQ2_REG_SRC_P_SIZE_RT_OFFSET 6535 +#define PSWRQ2_REG_TSDM_P_SIZE_RT_OFFSET 6536 +#define PSWRQ2_REG_TM_FIRST_ILT_RT_OFFSET 6537 +#define PSWRQ2_REG_TM_LAST_ILT_RT_OFFSET 6538 +#define PSWRQ2_REG_QM_FIRST_ILT_RT_OFFSET 6539 +#define PSWRQ2_REG_QM_LAST_ILT_RT_OFFSET 6540 +#define PSWRQ2_REG_SRC_FIRST_ILT_RT_OFFSET 6541 +#define PSWRQ2_REG_SRC_LAST_ILT_RT_OFFSET 6542 +#define PSWRQ2_REG_CDUC_FIRST_ILT_RT_OFFSET 6543 +#define PSWRQ2_REG_CDUC_LAST_ILT_RT_OFFSET 6544 +#define PSWRQ2_REG_CDUT_FIRST_ILT_RT_OFFSET 6545 +#define PSWRQ2_REG_CDUT_LAST_ILT_RT_OFFSET 6546 +#define PSWRQ2_REG_TSDM_FIRST_ILT_RT_OFFSET 6547 +#define PSWRQ2_REG_TSDM_LAST_ILT_RT_OFFSET 6548 +#define PSWRQ2_REG_TM_NUMBER_OF_PF_BLOCKS_RT_OFFSET 6549 +#define PSWRQ2_REG_CDUT_NUMBER_OF_PF_BLOCKS_RT_OFFSET 6550 +#define PSWRQ2_REG_CDUC_NUMBER_OF_PF_BLOCKS_RT_OFFSET 6551 +#define PSWRQ2_REG_TM_VF_BLOCKS_RT_OFFSET 6552 +#define PSWRQ2_REG_CDUT_VF_BLOCKS_RT_OFFSET 6553 +#define PSWRQ2_REG_CDUC_VF_BLOCKS_RT_OFFSET 6554 +#define PSWRQ2_REG_TM_BLOCKS_FACTOR_RT_OFFSET 6555 +#define PSWRQ2_REG_CDUT_BLOCKS_FACTOR_RT_OFFSET 6556 +#define PSWRQ2_REG_CDUC_BLOCKS_FACTOR_RT_OFFSET 6557 +#define PSWRQ2_REG_VF_BASE_RT_OFFSET 6558 +#define PSWRQ2_REG_VF_LAST_ILT_RT_OFFSET 6559 +#define PSWRQ2_REG_DRAM_ALIGN_WR_RT_OFFSET 6560 +#define PSWRQ2_REG_DRAM_ALIGN_RD_RT_OFFSET 6561 +#define PSWRQ2_REG_TGSRC_FIRST_ILT_RT_OFFSET 6562 +#define PSWRQ2_REG_RGSRC_FIRST_ILT_RT_OFFSET 6563 +#define PSWRQ2_REG_TGSRC_LAST_ILT_RT_OFFSET 6564 +#define PSWRQ2_REG_RGSRC_LAST_ILT_RT_OFFSET 6565 +#define PSWRQ2_REG_ILT_MEMORY_RT_OFFSET 6566 +#define PSWRQ2_REG_ILT_MEMORY_RT_SIZE 26414 +#define PGLUE_REG_B_VF_BASE_RT_OFFSET 32980 +#define PGLUE_REG_B_MSDM_OFFSET_MASK_B_RT_OFFSET 32981 +#define PGLUE_REG_B_MSDM_VF_SHIFT_B_RT_OFFSET 32982 +#define PGLUE_REG_B_CACHE_LINE_SIZE_RT_OFFSET 32983 +#define PGLUE_REG_B_PF_BAR0_SIZE_RT_OFFSET 32984 +#define PGLUE_REG_B_PF_BAR1_SIZE_RT_OFFSET 32985 +#define PGLUE_REG_B_VF_BAR1_SIZE_RT_OFFSET 32986 +#define TM_REG_VF_ENABLE_CONN_RT_OFFSET 32987 +#define TM_REG_PF_ENABLE_CONN_RT_OFFSET 32988 +#define TM_REG_PF_ENABLE_TASK_RT_OFFSET 32989 +#define TM_REG_GROUP_SIZE_RESOLUTION_CONN_RT_OFFSET 32990 +#define TM_REG_GROUP_SIZE_RESOLUTION_TASK_RT_OFFSET 32991 +#define TM_REG_CONFIG_CONN_MEM_RT_OFFSET 32992 +#define TM_REG_CONFIG_CONN_MEM_RT_SIZE 416 +#define TM_REG_CONFIG_TASK_MEM_RT_OFFSET 33408 +#define TM_REG_CONFIG_TASK_MEM_RT_SIZE 608 +#define QM_REG_MAXPQSIZE_0_RT_OFFSET 34016 +#define QM_REG_MAXPQSIZE_1_RT_OFFSET 34017 +#define QM_REG_MAXPQSIZE_2_RT_OFFSET 34018 +#define QM_REG_MAXPQSIZETXSEL_0_RT_OFFSET 34019 +#define QM_REG_MAXPQSIZETXSEL_1_RT_OFFSET 34020 +#define QM_REG_MAXPQSIZETXSEL_2_RT_OFFSET 34021 +#define QM_REG_MAXPQSIZETXSEL_3_RT_OFFSET 34022 +#define QM_REG_MAXPQSIZETXSEL_4_RT_OFFSET 34023 +#define QM_REG_MAXPQSIZETXSEL_5_RT_OFFSET 34024 +#define QM_REG_MAXPQSIZETXSEL_6_RT_OFFSET 34025 +#define QM_REG_MAXPQSIZETXSEL_7_RT_OFFSET 34026 +#define QM_REG_MAXPQSIZETXSEL_8_RT_OFFSET 34027 +#define QM_REG_MAXPQSIZETXSEL_9_RT_OFFSET 34028 +#define QM_REG_MAXPQSIZETXSEL_10_RT_OFFSET 34029 +#define QM_REG_MAXPQSIZETXSEL_11_RT_OFFSET 34030 +#define QM_REG_MAXPQSIZETXSEL_12_RT_OFFSET 34031 +#define QM_REG_MAXPQSIZETXSEL_13_RT_OFFSET 34032 +#define QM_REG_MAXPQSIZETXSEL_14_RT_OFFSET 34033 +#define QM_REG_MAXPQSIZETXSEL_15_RT_OFFSET 34034 +#define QM_REG_MAXPQSIZETXSEL_16_RT_OFFSET 34035 +#define QM_REG_MAXPQSIZETXSEL_17_RT_OFFSET 34036 +#define QM_REG_MAXPQSIZETXSEL_18_RT_OFFSET 34037 +#define QM_REG_MAXPQSIZETXSEL_19_RT_OFFSET 34038 +#define QM_REG_MAXPQSIZETXSEL_20_RT_OFFSET 34039 +#define QM_REG_MAXPQSIZETXSEL_21_RT_OFFSET 34040 +#define QM_REG_MAXPQSIZETXSEL_22_RT_OFFSET 34041 +#define QM_REG_MAXPQSIZETXSEL_23_RT_OFFSET 34042 +#define QM_REG_MAXPQSIZETXSEL_24_RT_OFFSET 34043 +#define QM_REG_MAXPQSIZETXSEL_25_RT_OFFSET 34044 +#define QM_REG_MAXPQSIZETXSEL_26_RT_OFFSET 34045 +#define QM_REG_MAXPQSIZETXSEL_27_RT_OFFSET 34046 +#define QM_REG_MAXPQSIZETXSEL_28_RT_OFFSET 34047 +#define QM_REG_MAXPQSIZETXSEL_29_RT_OFFSET 34048 +#define QM_REG_MAXPQSIZETXSEL_30_RT_OFFSET 34049 +#define QM_REG_MAXPQSIZETXSEL_31_RT_OFFSET 34050 +#define QM_REG_MAXPQSIZETXSEL_32_RT_OFFSET 34051 +#define QM_REG_MAXPQSIZETXSEL_33_RT_OFFSET 34052 +#define QM_REG_MAXPQSIZETXSEL_34_RT_OFFSET 34053 +#define QM_REG_MAXPQSIZETXSEL_35_RT_OFFSET 34054 +#define QM_REG_MAXPQSIZETXSEL_36_RT_OFFSET 34055 +#define QM_REG_MAXPQSIZETXSEL_37_RT_OFFSET 34056 +#define QM_REG_MAXPQSIZETXSEL_38_RT_OFFSET 34057 +#define QM_REG_MAXPQSIZETXSEL_39_RT_OFFSET 34058 +#define QM_REG_MAXPQSIZETXSEL_40_RT_OFFSET 34059 +#define QM_REG_MAXPQSIZETXSEL_41_RT_OFFSET 34060 +#define QM_REG_MAXPQSIZETXSEL_42_RT_OFFSET 34061 +#define QM_REG_MAXPQSIZETXSEL_43_RT_OFFSET 34062 +#define QM_REG_MAXPQSIZETXSEL_44_RT_OFFSET 34063 +#define QM_REG_MAXPQSIZETXSEL_45_RT_OFFSET 34064 +#define QM_REG_MAXPQSIZETXSEL_46_RT_OFFSET 34065 +#define QM_REG_MAXPQSIZETXSEL_47_RT_OFFSET 34066 +#define QM_REG_MAXPQSIZETXSEL_48_RT_OFFSET 34067 +#define QM_REG_MAXPQSIZETXSEL_49_RT_OFFSET 34068 +#define QM_REG_MAXPQSIZETXSEL_50_RT_OFFSET 34069 +#define QM_REG_MAXPQSIZETXSEL_51_RT_OFFSET 34070 +#define QM_REG_MAXPQSIZETXSEL_52_RT_OFFSET 34071 +#define QM_REG_MAXPQSIZETXSEL_53_RT_OFFSET 34072 +#define QM_REG_MAXPQSIZETXSEL_54_RT_OFFSET 34073 +#define QM_REG_MAXPQSIZETXSEL_55_RT_OFFSET 34074 +#define QM_REG_MAXPQSIZETXSEL_56_RT_OFFSET 34075 +#define QM_REG_MAXPQSIZETXSEL_57_RT_OFFSET 34076 +#define QM_REG_MAXPQSIZETXSEL_58_RT_OFFSET 34077 +#define QM_REG_MAXPQSIZETXSEL_59_RT_OFFSET 34078 +#define QM_REG_MAXPQSIZETXSEL_60_RT_OFFSET 34079 +#define QM_REG_MAXPQSIZETXSEL_61_RT_OFFSET 34080 +#define QM_REG_MAXPQSIZETXSEL_62_RT_OFFSET 34081 +#define QM_REG_MAXPQSIZETXSEL_63_RT_OFFSET 34082 +#define QM_REG_BASEADDROTHERPQ_RT_OFFSET 34083 +#define QM_REG_BASEADDROTHERPQ_RT_SIZE 128 +#define QM_REG_AFULLQMBYPTHRPFWFQ_RT_OFFSET 34211 +#define QM_REG_AFULLQMBYPTHRVPWFQ_RT_OFFSET 34212 +#define QM_REG_AFULLQMBYPTHRPFRL_RT_OFFSET 34213 +#define QM_REG_AFULLQMBYPTHRGLBLRL_RT_OFFSET 34214 +#define QM_REG_AFULLOPRTNSTCCRDMASK_RT_OFFSET 34215 +#define QM_REG_WRROTHERPQGRP_0_RT_OFFSET 34216 +#define QM_REG_WRROTHERPQGRP_1_RT_OFFSET 34217 +#define QM_REG_WRROTHERPQGRP_2_RT_OFFSET 34218 +#define QM_REG_WRROTHERPQGRP_3_RT_OFFSET 34219 +#define QM_REG_WRROTHERPQGRP_4_RT_OFFSET 34220 +#define QM_REG_WRROTHERPQGRP_5_RT_OFFSET 34221 +#define QM_REG_WRROTHERPQGRP_6_RT_OFFSET 34222 +#define QM_REG_WRROTHERPQGRP_7_RT_OFFSET 34223 +#define QM_REG_WRROTHERPQGRP_8_RT_OFFSET 34224 +#define QM_REG_WRROTHERPQGRP_9_RT_OFFSET 34225 +#define QM_REG_WRROTHERPQGRP_10_RT_OFFSET 34226 +#define QM_REG_WRROTHERPQGRP_11_RT_OFFSET 34227 +#define QM_REG_WRROTHERPQGRP_12_RT_OFFSET 34228 +#define QM_REG_WRROTHERPQGRP_13_RT_OFFSET 34229 +#define QM_REG_WRROTHERPQGRP_14_RT_OFFSET 34230 +#define QM_REG_WRROTHERPQGRP_15_RT_OFFSET 34231 +#define QM_REG_WRROTHERGRPWEIGHT_0_RT_OFFSET 34232 +#define QM_REG_WRROTHERGRPWEIGHT_1_RT_OFFSET 34233 +#define QM_REG_WRROTHERGRPWEIGHT_2_RT_OFFSET 34234 +#define QM_REG_WRROTHERGRPWEIGHT_3_RT_OFFSET 34235 +#define QM_REG_WRRTXGRPWEIGHT_0_RT_OFFSET 34236 +#define QM_REG_WRRTXGRPWEIGHT_1_RT_OFFSET 34237 +#define QM_REG_PQTX2PF_0_RT_OFFSET 34238 +#define QM_REG_PQTX2PF_1_RT_OFFSET 34239 +#define QM_REG_PQTX2PF_2_RT_OFFSET 34240 +#define QM_REG_PQTX2PF_3_RT_OFFSET 34241 +#define QM_REG_PQTX2PF_4_RT_OFFSET 34242 +#define QM_REG_PQTX2PF_5_RT_OFFSET 34243 +#define QM_REG_PQTX2PF_6_RT_OFFSET 34244 +#define QM_REG_PQTX2PF_7_RT_OFFSET 34245 +#define QM_REG_PQTX2PF_8_RT_OFFSET 34246 +#define QM_REG_PQTX2PF_9_RT_OFFSET 34247 +#define QM_REG_PQTX2PF_10_RT_OFFSET 34248 +#define QM_REG_PQTX2PF_11_RT_OFFSET 34249 +#define QM_REG_PQTX2PF_12_RT_OFFSET 34250 +#define QM_REG_PQTX2PF_13_RT_OFFSET 34251 +#define QM_REG_PQTX2PF_14_RT_OFFSET 34252 +#define QM_REG_PQTX2PF_15_RT_OFFSET 34253 +#define QM_REG_PQTX2PF_16_RT_OFFSET 34254 +#define QM_REG_PQTX2PF_17_RT_OFFSET 34255 +#define QM_REG_PQTX2PF_18_RT_OFFSET 34256 +#define QM_REG_PQTX2PF_19_RT_OFFSET 34257 +#define QM_REG_PQTX2PF_20_RT_OFFSET 34258 +#define QM_REG_PQTX2PF_21_RT_OFFSET 34259 +#define QM_REG_PQTX2PF_22_RT_OFFSET 34260 +#define QM_REG_PQTX2PF_23_RT_OFFSET 34261 +#define QM_REG_PQTX2PF_24_RT_OFFSET 34262 +#define QM_REG_PQTX2PF_25_RT_OFFSET 34263 +#define QM_REG_PQTX2PF_26_RT_OFFSET 34264 +#define QM_REG_PQTX2PF_27_RT_OFFSET 34265 +#define QM_REG_PQTX2PF_28_RT_OFFSET 34266 +#define QM_REG_PQTX2PF_29_RT_OFFSET 34267 +#define QM_REG_PQTX2PF_30_RT_OFFSET 34268 +#define QM_REG_PQTX2PF_31_RT_OFFSET 34269 +#define QM_REG_PQTX2PF_32_RT_OFFSET 34270 +#define QM_REG_PQTX2PF_33_RT_OFFSET 34271 +#define QM_REG_PQTX2PF_34_RT_OFFSET 34272 +#define QM_REG_PQTX2PF_35_RT_OFFSET 34273 +#define QM_REG_PQTX2PF_36_RT_OFFSET 34274 +#define QM_REG_PQTX2PF_37_RT_OFFSET 34275 +#define QM_REG_PQTX2PF_38_RT_OFFSET 34276 +#define QM_REG_PQTX2PF_39_RT_OFFSET 34277 +#define QM_REG_PQTX2PF_40_RT_OFFSET 34278 +#define QM_REG_PQTX2PF_41_RT_OFFSET 34279 +#define QM_REG_PQTX2PF_42_RT_OFFSET 34280 +#define QM_REG_PQTX2PF_43_RT_OFFSET 34281 +#define QM_REG_PQTX2PF_44_RT_OFFSET 34282 +#define QM_REG_PQTX2PF_45_RT_OFFSET 34283 +#define QM_REG_PQTX2PF_46_RT_OFFSET 34284 +#define QM_REG_PQTX2PF_47_RT_OFFSET 34285 +#define QM_REG_PQTX2PF_48_RT_OFFSET 34286 +#define QM_REG_PQTX2PF_49_RT_OFFSET 34287 +#define QM_REG_PQTX2PF_50_RT_OFFSET 34288 +#define QM_REG_PQTX2PF_51_RT_OFFSET 34289 +#define QM_REG_PQTX2PF_52_RT_OFFSET 34290 +#define QM_REG_PQTX2PF_53_RT_OFFSET 34291 +#define QM_REG_PQTX2PF_54_RT_OFFSET 34292 +#define QM_REG_PQTX2PF_55_RT_OFFSET 34293 +#define QM_REG_PQTX2PF_56_RT_OFFSET 34294 +#define QM_REG_PQTX2PF_57_RT_OFFSET 34295 +#define QM_REG_PQTX2PF_58_RT_OFFSET 34296 +#define QM_REG_PQTX2PF_59_RT_OFFSET 34297 +#define QM_REG_PQTX2PF_60_RT_OFFSET 34298 +#define QM_REG_PQTX2PF_61_RT_OFFSET 34299 +#define QM_REG_PQTX2PF_62_RT_OFFSET 34300 +#define QM_REG_PQTX2PF_63_RT_OFFSET 34301 +#define QM_REG_PQOTHER2PF_0_RT_OFFSET 34302 +#define QM_REG_PQOTHER2PF_1_RT_OFFSET 34303 +#define QM_REG_PQOTHER2PF_2_RT_OFFSET 34304 +#define QM_REG_PQOTHER2PF_3_RT_OFFSET 34305 +#define QM_REG_PQOTHER2PF_4_RT_OFFSET 34306 +#define QM_REG_PQOTHER2PF_5_RT_OFFSET 34307 +#define QM_REG_PQOTHER2PF_6_RT_OFFSET 34308 +#define QM_REG_PQOTHER2PF_7_RT_OFFSET 34309 +#define QM_REG_PQOTHER2PF_8_RT_OFFSET 34310 +#define QM_REG_PQOTHER2PF_9_RT_OFFSET 34311 +#define QM_REG_PQOTHER2PF_10_RT_OFFSET 34312 +#define QM_REG_PQOTHER2PF_11_RT_OFFSET 34313 +#define QM_REG_PQOTHER2PF_12_RT_OFFSET 34314 +#define QM_REG_PQOTHER2PF_13_RT_OFFSET 34315 +#define QM_REG_PQOTHER2PF_14_RT_OFFSET 34316 +#define QM_REG_PQOTHER2PF_15_RT_OFFSET 34317 +#define QM_REG_RLGLBLPERIOD_0_RT_OFFSET 34318 +#define QM_REG_RLGLBLPERIOD_1_RT_OFFSET 34319 +#define QM_REG_RLGLBLPERIODTIMER_0_RT_OFFSET 34320 +#define QM_REG_RLGLBLPERIODTIMER_1_RT_OFFSET 34321 +#define QM_REG_RLGLBLPERIODSEL_0_RT_OFFSET 34322 +#define QM_REG_RLGLBLPERIODSEL_1_RT_OFFSET 34323 +#define QM_REG_RLGLBLPERIODSEL_2_RT_OFFSET 34324 +#define QM_REG_RLGLBLPERIODSEL_3_RT_OFFSET 34325 +#define QM_REG_RLGLBLPERIODSEL_4_RT_OFFSET 34326 +#define QM_REG_RLGLBLPERIODSEL_5_RT_OFFSET 34327 +#define QM_REG_RLGLBLPERIODSEL_6_RT_OFFSET 34328 +#define QM_REG_RLGLBLPERIODSEL_7_RT_OFFSET 34329 +#define QM_REG_RLGLBLINCVAL_RT_OFFSET 34330 +#define QM_REG_RLGLBLINCVAL_RT_SIZE 256 +#define QM_REG_RLGLBLUPPERBOUND_RT_OFFSET 34586 +#define QM_REG_RLGLBLUPPERBOUND_RT_SIZE 256 +#define QM_REG_RLGLBLCRD_RT_OFFSET 34842 +#define QM_REG_RLGLBLCRD_RT_SIZE 256 +#define QM_REG_RLGLBLENABLE_RT_OFFSET 35098 +#define QM_REG_RLPFPERIOD_RT_OFFSET 35099 +#define QM_REG_RLPFPERIODTIMER_RT_OFFSET 35100 +#define QM_REG_RLPFINCVAL_RT_OFFSET 35101 +#define QM_REG_RLPFINCVAL_RT_SIZE 16 +#define QM_REG_RLPFUPPERBOUND_RT_OFFSET 35117 +#define QM_REG_RLPFUPPERBOUND_RT_SIZE 16 +#define QM_REG_RLPFCRD_RT_OFFSET 35133 +#define QM_REG_RLPFCRD_RT_SIZE 16 +#define QM_REG_RLPFENABLE_RT_OFFSET 35149 +#define QM_REG_RLPFVOQENABLE_RT_OFFSET 35150 +#define QM_REG_WFQPFWEIGHT_RT_OFFSET 35151 +#define QM_REG_WFQPFWEIGHT_RT_SIZE 16 +#define QM_REG_WFQPFUPPERBOUND_RT_OFFSET 35167 +#define QM_REG_WFQPFUPPERBOUND_RT_SIZE 16 +#define QM_REG_WFQPFCRD_RT_OFFSET 35183 +#define QM_REG_WFQPFCRD_RT_SIZE 256 +#define QM_REG_WFQPFENABLE_RT_OFFSET 35439 +#define QM_REG_WFQVPENABLE_RT_OFFSET 35440 +#define QM_REG_BASEADDRTXPQ_RT_OFFSET 35441 +#define QM_REG_BASEADDRTXPQ_RT_SIZE 512 +#define QM_REG_TXPQMAP_RT_OFFSET 35953 +#define QM_REG_TXPQMAP_RT_SIZE 512 +#define QM_REG_WFQVPWEIGHT_RT_OFFSET 36465 +#define QM_REG_WFQVPWEIGHT_RT_SIZE 512 +#define QM_REG_WFQVPCRD_RT_OFFSET 36977 +#define QM_REG_WFQVPCRD_RT_SIZE 512 +#define QM_REG_WFQVPMAP_RT_OFFSET 37489 +#define QM_REG_WFQVPMAP_RT_SIZE 512 +#define QM_REG_WFQPFCRD_MSB_RT_OFFSET 38001 +#define QM_REG_WFQPFCRD_MSB_RT_SIZE 320 +#define QM_REG_VOQCRDLINE_RT_OFFSET 38321 +#define QM_REG_VOQCRDLINE_RT_SIZE 36 +#define QM_REG_VOQINITCRDLINE_RT_OFFSET 38357 +#define QM_REG_VOQINITCRDLINE_RT_SIZE 36 +#define QM_REG_RLPFVOQENABLE_MSB_RT_OFFSET 38393 +#define NIG_REG_TAG_ETHERTYPE_0_RT_OFFSET 38394 +#define NIG_REG_BRB_GATE_DNTFWD_PORT_RT_OFFSET 38395 +#define NIG_REG_OUTER_TAG_VALUE_LIST0_RT_OFFSET 38396 +#define NIG_REG_OUTER_TAG_VALUE_LIST1_RT_OFFSET 38397 +#define NIG_REG_OUTER_TAG_VALUE_LIST2_RT_OFFSET 38398 +#define NIG_REG_OUTER_TAG_VALUE_LIST3_RT_OFFSET 38399 +#define NIG_REG_LLH_FUNC_TAGMAC_CLS_TYPE_RT_OFFSET 38400 +#define NIG_REG_LLH_FUNC_TAG_EN_RT_OFFSET 38401 +#define NIG_REG_LLH_FUNC_TAG_EN_RT_SIZE 4 +#define NIG_REG_LLH_FUNC_TAG_VALUE_RT_OFFSET 38405 +#define NIG_REG_LLH_FUNC_TAG_VALUE_RT_SIZE 4 +#define NIG_REG_LLH_FUNC_FILTER_VALUE_RT_OFFSET 38409 +#define NIG_REG_LLH_FUNC_FILTER_VALUE_RT_SIZE 32 +#define NIG_REG_LLH_FUNC_FILTER_EN_RT_OFFSET 38441 +#define NIG_REG_LLH_FUNC_FILTER_EN_RT_SIZE 16 +#define NIG_REG_LLH_FUNC_FILTER_MODE_RT_OFFSET 38457 +#define NIG_REG_LLH_FUNC_FILTER_MODE_RT_SIZE 16 +#define NIG_REG_LLH_FUNC_FILTER_PROTOCOL_TYPE_RT_OFFSET 38473 +#define NIG_REG_LLH_FUNC_FILTER_PROTOCOL_TYPE_RT_SIZE 16 +#define NIG_REG_LLH_FUNC_FILTER_HDR_SEL_RT_OFFSET 38489 +#define NIG_REG_LLH_FUNC_FILTER_HDR_SEL_RT_SIZE 16 +#define NIG_REG_TX_EDPM_CTRL_RT_OFFSET 38505 +#define NIG_REG_ROCE_DUPLICATE_TO_HOST_RT_OFFSET 38506 +#define NIG_REG_PPF_TO_ENGINE_SEL_RT_OFFSET 38507 +#define NIG_REG_PPF_TO_ENGINE_SEL_RT_SIZE 8 +#define NIG_REG_LLH_PF_CLS_FUNC_FILTER_VALUE_RT_OFFSET 38515 +#define NIG_REG_LLH_PF_CLS_FUNC_FILTER_VALUE_RT_SIZE 1024 +#define NIG_REG_LLH_PF_CLS_FUNC_FILTER_EN_RT_OFFSET 39539 +#define NIG_REG_LLH_PF_CLS_FUNC_FILTER_EN_RT_SIZE 512 +#define NIG_REG_LLH_PF_CLS_FUNC_FILTER_MODE_RT_OFFSET 40051 +#define NIG_REG_LLH_PF_CLS_FUNC_FILTER_MODE_RT_SIZE 512 +#define NIG_REG_LLH_PF_CLS_FUNC_FILTER_PROTOCOL_TYPE_RT_OFFSET 40563 +#define NIG_REG_LLH_PF_CLS_FUNC_FILTER_PROTOCOL_TYPE_RT_SIZE 512 +#define NIG_REG_LLH_PF_CLS_FUNC_FILTER_HDR_SEL_RT_OFFSET 41075 +#define NIG_REG_LLH_PF_CLS_FUNC_FILTER_HDR_SEL_RT_SIZE 512 +#define NIG_REG_LLH_PF_CLS_FILTERS_MAP_RT_OFFSET 41587 +#define NIG_REG_LLH_PF_CLS_FILTERS_MAP_RT_SIZE 32 +#define CDU_REG_CID_ADDR_PARAMS_RT_OFFSET 41619 +#define CDU_REG_SEGMENT0_PARAMS_RT_OFFSET 41620 +#define CDU_REG_SEGMENT1_PARAMS_RT_OFFSET 41621 +#define CDU_REG_PF_SEG0_TYPE_OFFSET_RT_OFFSET 41622 +#define CDU_REG_PF_SEG1_TYPE_OFFSET_RT_OFFSET 41623 +#define CDU_REG_PF_SEG2_TYPE_OFFSET_RT_OFFSET 41624 +#define CDU_REG_PF_SEG3_TYPE_OFFSET_RT_OFFSET 41625 +#define CDU_REG_PF_FL_SEG0_TYPE_OFFSET_RT_OFFSET 41626 +#define CDU_REG_PF_FL_SEG1_TYPE_OFFSET_RT_OFFSET 41627 +#define CDU_REG_PF_FL_SEG2_TYPE_OFFSET_RT_OFFSET 41628 +#define CDU_REG_PF_FL_SEG3_TYPE_OFFSET_RT_OFFSET 41629 +#define CDU_REG_VF_SEG_TYPE_OFFSET_RT_OFFSET 41630 +#define CDU_REG_VF_FL_SEG_TYPE_OFFSET_RT_OFFSET 41631 +#define PBF_REG_TAG_ETHERTYPE_0_RT_OFFSET 41632 +#define PBF_REG_BTB_SHARED_AREA_SIZE_RT_OFFSET 41633 +#define PBF_REG_YCMD_QS_NUM_LINES_VOQ0_RT_OFFSET 41634 +#define PBF_REG_BTB_GUARANTEED_VOQ0_RT_OFFSET 41635 +#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ0_RT_OFFSET 41636 +#define PBF_REG_YCMD_QS_NUM_LINES_VOQ1_RT_OFFSET 41637 +#define PBF_REG_BTB_GUARANTEED_VOQ1_RT_OFFSET 41638 +#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ1_RT_OFFSET 41639 +#define PBF_REG_YCMD_QS_NUM_LINES_VOQ2_RT_OFFSET 41640 +#define PBF_REG_BTB_GUARANTEED_VOQ2_RT_OFFSET 41641 +#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ2_RT_OFFSET 41642 +#define PBF_REG_YCMD_QS_NUM_LINES_VOQ3_RT_OFFSET 41643 +#define PBF_REG_BTB_GUARANTEED_VOQ3_RT_OFFSET 41644 +#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ3_RT_OFFSET 41645 +#define PBF_REG_YCMD_QS_NUM_LINES_VOQ4_RT_OFFSET 41646 +#define PBF_REG_BTB_GUARANTEED_VOQ4_RT_OFFSET 41647 +#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ4_RT_OFFSET 41648 +#define PBF_REG_YCMD_QS_NUM_LINES_VOQ5_RT_OFFSET 41649 +#define PBF_REG_BTB_GUARANTEED_VOQ5_RT_OFFSET 41650 +#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ5_RT_OFFSET 41651 +#define PBF_REG_YCMD_QS_NUM_LINES_VOQ6_RT_OFFSET 41652 +#define PBF_REG_BTB_GUARANTEED_VOQ6_RT_OFFSET 41653 +#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ6_RT_OFFSET 41654 +#define PBF_REG_YCMD_QS_NUM_LINES_VOQ7_RT_OFFSET 41655 +#define PBF_REG_BTB_GUARANTEED_VOQ7_RT_OFFSET 41656 +#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ7_RT_OFFSET 41657 +#define PBF_REG_YCMD_QS_NUM_LINES_VOQ8_RT_OFFSET 41658 +#define PBF_REG_BTB_GUARANTEED_VOQ8_RT_OFFSET 41659 +#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ8_RT_OFFSET 41660 +#define PBF_REG_YCMD_QS_NUM_LINES_VOQ9_RT_OFFSET 41661 +#define PBF_REG_BTB_GUARANTEED_VOQ9_RT_OFFSET 41662 +#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ9_RT_OFFSET 41663 +#define PBF_REG_YCMD_QS_NUM_LINES_VOQ10_RT_OFFSET 41664 +#define PBF_REG_BTB_GUARANTEED_VOQ10_RT_OFFSET 41665 +#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ10_RT_OFFSET 41666 +#define PBF_REG_YCMD_QS_NUM_LINES_VOQ11_RT_OFFSET 41667 +#define PBF_REG_BTB_GUARANTEED_VOQ11_RT_OFFSET 41668 +#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ11_RT_OFFSET 41669 +#define PBF_REG_YCMD_QS_NUM_LINES_VOQ12_RT_OFFSET 41670 +#define PBF_REG_BTB_GUARANTEED_VOQ12_RT_OFFSET 41671 +#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ12_RT_OFFSET 41672 +#define PBF_REG_YCMD_QS_NUM_LINES_VOQ13_RT_OFFSET 41673 +#define PBF_REG_BTB_GUARANTEED_VOQ13_RT_OFFSET 41674 +#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ13_RT_OFFSET 41675 +#define PBF_REG_YCMD_QS_NUM_LINES_VOQ14_RT_OFFSET 41676 +#define PBF_REG_BTB_GUARANTEED_VOQ14_RT_OFFSET 41677 +#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ14_RT_OFFSET 41678 +#define PBF_REG_YCMD_QS_NUM_LINES_VOQ15_RT_OFFSET 41679 +#define PBF_REG_BTB_GUARANTEED_VOQ15_RT_OFFSET 41680 +#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ15_RT_OFFSET 41681 +#define PBF_REG_YCMD_QS_NUM_LINES_VOQ16_RT_OFFSET 41682 +#define PBF_REG_BTB_GUARANTEED_VOQ16_RT_OFFSET 41683 +#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ16_RT_OFFSET 41684 +#define PBF_REG_YCMD_QS_NUM_LINES_VOQ17_RT_OFFSET 41685 +#define PBF_REG_BTB_GUARANTEED_VOQ17_RT_OFFSET 41686 +#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ17_RT_OFFSET 41687 +#define PBF_REG_YCMD_QS_NUM_LINES_VOQ18_RT_OFFSET 41688 +#define PBF_REG_BTB_GUARANTEED_VOQ18_RT_OFFSET 41689 +#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ18_RT_OFFSET 41690 +#define PBF_REG_YCMD_QS_NUM_LINES_VOQ19_RT_OFFSET 41691 +#define PBF_REG_BTB_GUARANTEED_VOQ19_RT_OFFSET 41692 +#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ19_RT_OFFSET 41693 +#define PBF_REG_YCMD_QS_NUM_LINES_VOQ20_RT_OFFSET 41694 +#define PBF_REG_BTB_GUARANTEED_VOQ20_RT_OFFSET 41695 +#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ20_RT_OFFSET 41696 +#define PBF_REG_YCMD_QS_NUM_LINES_VOQ21_RT_OFFSET 41697 +#define PBF_REG_BTB_GUARANTEED_VOQ21_RT_OFFSET 41698 +#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ21_RT_OFFSET 41699 +#define PBF_REG_YCMD_QS_NUM_LINES_VOQ22_RT_OFFSET 41700 +#define PBF_REG_BTB_GUARANTEED_VOQ22_RT_OFFSET 41701 +#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ22_RT_OFFSET 41702 +#define PBF_REG_YCMD_QS_NUM_LINES_VOQ23_RT_OFFSET 41703 +#define PBF_REG_BTB_GUARANTEED_VOQ23_RT_OFFSET 41704 +#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ23_RT_OFFSET 41705 +#define PBF_REG_YCMD_QS_NUM_LINES_VOQ24_RT_OFFSET 41706 +#define PBF_REG_BTB_GUARANTEED_VOQ24_RT_OFFSET 41707 +#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ24_RT_OFFSET 41708 +#define PBF_REG_YCMD_QS_NUM_LINES_VOQ25_RT_OFFSET 41709 +#define PBF_REG_BTB_GUARANTEED_VOQ25_RT_OFFSET 41710 +#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ25_RT_OFFSET 41711 +#define PBF_REG_YCMD_QS_NUM_LINES_VOQ26_RT_OFFSET 41712 +#define PBF_REG_BTB_GUARANTEED_VOQ26_RT_OFFSET 41713 +#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ26_RT_OFFSET 41714 +#define PBF_REG_YCMD_QS_NUM_LINES_VOQ27_RT_OFFSET 41715 +#define PBF_REG_BTB_GUARANTEED_VOQ27_RT_OFFSET 41716 +#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ27_RT_OFFSET 41717 +#define PBF_REG_YCMD_QS_NUM_LINES_VOQ28_RT_OFFSET 41718 +#define PBF_REG_BTB_GUARANTEED_VOQ28_RT_OFFSET 41719 +#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ28_RT_OFFSET 41720 +#define PBF_REG_YCMD_QS_NUM_LINES_VOQ29_RT_OFFSET 41721 +#define PBF_REG_BTB_GUARANTEED_VOQ29_RT_OFFSET 41722 +#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ29_RT_OFFSET 41723 +#define PBF_REG_YCMD_QS_NUM_LINES_VOQ30_RT_OFFSET 41724 +#define PBF_REG_BTB_GUARANTEED_VOQ30_RT_OFFSET 41725 +#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ30_RT_OFFSET 41726 +#define PBF_REG_YCMD_QS_NUM_LINES_VOQ31_RT_OFFSET 41727 +#define PBF_REG_BTB_GUARANTEED_VOQ31_RT_OFFSET 41728 +#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ31_RT_OFFSET 41729 +#define PBF_REG_YCMD_QS_NUM_LINES_VOQ32_RT_OFFSET 41730 +#define PBF_REG_BTB_GUARANTEED_VOQ32_RT_OFFSET 41731 +#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ32_RT_OFFSET 41732 +#define PBF_REG_YCMD_QS_NUM_LINES_VOQ33_RT_OFFSET 41733 +#define PBF_REG_BTB_GUARANTEED_VOQ33_RT_OFFSET 41734 +#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ33_RT_OFFSET 41735 +#define PBF_REG_YCMD_QS_NUM_LINES_VOQ34_RT_OFFSET 41736 +#define PBF_REG_BTB_GUARANTEED_VOQ34_RT_OFFSET 41737 +#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ34_RT_OFFSET 41738 +#define PBF_REG_YCMD_QS_NUM_LINES_VOQ35_RT_OFFSET 41739 +#define PBF_REG_BTB_GUARANTEED_VOQ35_RT_OFFSET 41740 +#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ35_RT_OFFSET 41741 +#define XCM_REG_CON_PHY_Q3_RT_OFFSET 41742 -#define RUNTIME_ARRAY_SIZE 33924 +#define RUNTIME_ARRAY_SIZE 41743 #endif /* __RT_DEFS_H__ */ diff --git a/dpdk/drivers/net/qede/base/ecore_sp_api.h b/dpdk/drivers/net/qede/base/ecore_sp_api.h index e80f5ef3..86e84964 100644 --- a/dpdk/drivers/net/qede/base/ecore_sp_api.h +++ b/dpdk/drivers/net/qede/base/ecore_sp_api.h @@ -12,9 +12,9 @@ #include "ecore_status.h" enum spq_mode { - ECORE_SPQ_MODE_BLOCK, /* Client will poll a designated mem. address */ - ECORE_SPQ_MODE_CB, /* Client supplies a callback */ - ECORE_SPQ_MODE_EBLOCK, /* ECORE should block until completion */ + ECORE_SPQ_MODE_BLOCK, /* Client will poll a designated mem. address */ + ECORE_SPQ_MODE_CB, /* Client supplies a callback */ + ECORE_SPQ_MODE_EBLOCK, /* ECORE should block until completion */ }; struct ecore_hwfn; @@ -22,11 +22,14 @@ union event_ring_data; struct eth_slow_path_rx_cqe; struct ecore_spq_comp_cb { - void (*function)(struct ecore_hwfn *, - void *, union event_ring_data *, u8 fw_return_code); - void *cookie; + void (*function)(struct ecore_hwfn *, + void *, + union event_ring_data *, + u8 fw_return_code); + void *cookie; }; + /** * @brief ecore_eth_cqe_completion - handles the completion of a * ramrod on the cqe ring @@ -38,5 +41,26 @@ struct ecore_spq_comp_cb { */ enum _ecore_status_t ecore_eth_cqe_completion(struct ecore_hwfn *p_hwfn, struct eth_slow_path_rx_cqe *cqe); +/** + * @brief ecore_sp_pf_update_tunn_cfg - PF Function Tunnel configuration + * update Ramrod + * + * This ramrod is sent to update a tunneling configuration + * for a physical function (PF). + * + * @param p_hwfn + * @param p_ptt + * @param p_tunn - pf update tunneling parameters + * @param comp_mode - completion mode + * @param p_comp_data - callback function + * + * @return enum _ecore_status_t + */ +enum _ecore_status_t +ecore_sp_pf_update_tunn_cfg(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + struct ecore_tunnel_info *p_tunn, + enum spq_mode comp_mode, + struct ecore_spq_comp_cb *p_comp_data); #endif diff --git a/dpdk/drivers/net/qede/base/ecore_sp_commands.c b/dpdk/drivers/net/qede/base/ecore_sp_commands.c index e9ac8988..7598e7a6 100644 --- a/dpdk/drivers/net/qede/base/ecore_sp_commands.c +++ b/dpdk/drivers/net/qede/base/ecore_sp_commands.c @@ -21,6 +21,8 @@ #include "ecore_int.h" #include "ecore_hw.h" #include "ecore_dcbx.h" +#include "ecore_sriov.h" +#include "ecore_vf.h" enum _ecore_status_t ecore_sp_init_request(struct ecore_hwfn *p_hwfn, struct ecore_spq_entry **pp_ent, @@ -30,7 +32,10 @@ enum _ecore_status_t ecore_sp_init_request(struct ecore_hwfn *p_hwfn, { u32 opaque_cid = p_data->opaque_fid << 16 | p_data->cid; struct ecore_spq_entry *p_ent = OSAL_NULL; - enum _ecore_status_t rc = ECORE_NOTIMPL; + enum _ecore_status_t rc; + + if (!pp_ent) + return ECORE_INVAL; /* Get an SPQ entry */ rc = ecore_spq_get_entry(p_hwfn, pp_ent); @@ -84,7 +89,7 @@ enum _ecore_status_t ecore_sp_init_request(struct ecore_hwfn *p_hwfn, return ECORE_SUCCESS; } -static enum tunnel_clss ecore_tunn_get_clss_type(u8 type) +static enum tunnel_clss ecore_tunn_clss_to_fw_clss(u8 type) { switch (type) { case ECORE_TUNN_CLSS_MAC_VLAN: @@ -95,240 +100,215 @@ static enum tunnel_clss ecore_tunn_get_clss_type(u8 type) return TUNNEL_CLSS_INNER_MAC_VLAN; case ECORE_TUNN_CLSS_INNER_MAC_VNI: return TUNNEL_CLSS_INNER_MAC_VNI; + case ECORE_TUNN_CLSS_MAC_VLAN_DUAL_STAGE: + return TUNNEL_CLSS_MAC_VLAN_DUAL_STAGE; default: return TUNNEL_CLSS_MAC_VLAN; } } static void -ecore_tunn_set_pf_fix_tunn_mode(struct ecore_hwfn *p_hwfn, - struct ecore_tunn_update_params *p_src, - struct pf_update_tunnel_config *p_tunn_cfg) +ecore_set_pf_update_tunn_mode(struct ecore_tunnel_info *p_tun, + struct ecore_tunnel_info *p_src, + bool b_pf_start) { - unsigned long cached_tunn_mode = p_hwfn->p_dev->tunn_mode; - unsigned long update_mask = p_src->tunn_mode_update_mask; - unsigned long tunn_mode = p_src->tunn_mode; - unsigned long new_tunn_mode = 0; + if (p_src->vxlan.b_update_mode || b_pf_start) + p_tun->vxlan.b_mode_enabled = p_src->vxlan.b_mode_enabled; - if (OSAL_TEST_BIT(ECORE_MODE_L2GRE_TUNN, &update_mask)) { - if (OSAL_TEST_BIT(ECORE_MODE_L2GRE_TUNN, &tunn_mode)) - OSAL_SET_BIT(ECORE_MODE_L2GRE_TUNN, &new_tunn_mode); - } else { - if (OSAL_TEST_BIT(ECORE_MODE_L2GRE_TUNN, &cached_tunn_mode)) - OSAL_SET_BIT(ECORE_MODE_L2GRE_TUNN, &new_tunn_mode); - } + if (p_src->l2_gre.b_update_mode || b_pf_start) + p_tun->l2_gre.b_mode_enabled = p_src->l2_gre.b_mode_enabled; - if (OSAL_TEST_BIT(ECORE_MODE_IPGRE_TUNN, &update_mask)) { - if (OSAL_TEST_BIT(ECORE_MODE_IPGRE_TUNN, &tunn_mode)) - OSAL_SET_BIT(ECORE_MODE_IPGRE_TUNN, &new_tunn_mode); - } else { - if (OSAL_TEST_BIT(ECORE_MODE_IPGRE_TUNN, &cached_tunn_mode)) - OSAL_SET_BIT(ECORE_MODE_IPGRE_TUNN, &new_tunn_mode); - } + if (p_src->ip_gre.b_update_mode || b_pf_start) + p_tun->ip_gre.b_mode_enabled = p_src->ip_gre.b_mode_enabled; - if (OSAL_TEST_BIT(ECORE_MODE_VXLAN_TUNN, &update_mask)) { - if (OSAL_TEST_BIT(ECORE_MODE_VXLAN_TUNN, &tunn_mode)) - OSAL_SET_BIT(ECORE_MODE_VXLAN_TUNN, &new_tunn_mode); - } else { - if (OSAL_TEST_BIT(ECORE_MODE_VXLAN_TUNN, &cached_tunn_mode)) - OSAL_SET_BIT(ECORE_MODE_VXLAN_TUNN, &new_tunn_mode); - } + if (p_src->l2_geneve.b_update_mode || b_pf_start) + p_tun->l2_geneve.b_mode_enabled = + p_src->l2_geneve.b_mode_enabled; - if (ECORE_IS_BB_A0(p_hwfn->p_dev)) { - if (p_src->update_geneve_udp_port) - DP_NOTICE(p_hwfn, true, "Geneve not supported\n"); - p_src->update_geneve_udp_port = 0; - p_src->tunn_mode = new_tunn_mode; - return; - } + if (p_src->ip_geneve.b_update_mode || b_pf_start) + p_tun->ip_geneve.b_mode_enabled = + p_src->ip_geneve.b_mode_enabled; +} - if (p_src->update_geneve_udp_port) { - p_tunn_cfg->set_geneve_udp_port_flg = 1; - p_tunn_cfg->geneve_udp_port = - OSAL_CPU_TO_LE16(p_src->geneve_udp_port); - } +static void ecore_set_tunn_cls_info(struct ecore_tunnel_info *p_tun, + struct ecore_tunnel_info *p_src) +{ + enum tunnel_clss type; - if (OSAL_TEST_BIT(ECORE_MODE_L2GENEVE_TUNN, &update_mask)) { - if (OSAL_TEST_BIT(ECORE_MODE_L2GENEVE_TUNN, &tunn_mode)) - OSAL_SET_BIT(ECORE_MODE_L2GENEVE_TUNN, &new_tunn_mode); - } else { - if (OSAL_TEST_BIT(ECORE_MODE_L2GENEVE_TUNN, &cached_tunn_mode)) - OSAL_SET_BIT(ECORE_MODE_L2GENEVE_TUNN, &new_tunn_mode); - } + p_tun->b_update_rx_cls = p_src->b_update_rx_cls; + p_tun->b_update_tx_cls = p_src->b_update_tx_cls; - if (OSAL_TEST_BIT(ECORE_MODE_IPGENEVE_TUNN, &update_mask)) { - if (OSAL_TEST_BIT(ECORE_MODE_IPGENEVE_TUNN, &tunn_mode)) - OSAL_SET_BIT(ECORE_MODE_IPGENEVE_TUNN, &new_tunn_mode); - } else { - if (OSAL_TEST_BIT(ECORE_MODE_IPGENEVE_TUNN, &cached_tunn_mode)) - OSAL_SET_BIT(ECORE_MODE_IPGENEVE_TUNN, &new_tunn_mode); - } + /* @DPDK - typecast tunnul class */ + type = ecore_tunn_clss_to_fw_clss(p_src->vxlan.tun_cls); + p_tun->vxlan.tun_cls = (enum ecore_tunn_clss)type; + type = ecore_tunn_clss_to_fw_clss(p_src->l2_gre.tun_cls); + p_tun->l2_gre.tun_cls = (enum ecore_tunn_clss)type; + type = ecore_tunn_clss_to_fw_clss(p_src->ip_gre.tun_cls); + p_tun->ip_gre.tun_cls = (enum ecore_tunn_clss)type; + type = ecore_tunn_clss_to_fw_clss(p_src->l2_geneve.tun_cls); + p_tun->l2_geneve.tun_cls = (enum ecore_tunn_clss)type; + type = ecore_tunn_clss_to_fw_clss(p_src->ip_geneve.tun_cls); + p_tun->ip_geneve.tun_cls = (enum ecore_tunn_clss)type; +} - p_src->tunn_mode = new_tunn_mode; +static void ecore_set_tunn_ports(struct ecore_tunnel_info *p_tun, + struct ecore_tunnel_info *p_src) +{ + p_tun->geneve_port.b_update_port = p_src->geneve_port.b_update_port; + p_tun->vxlan_port.b_update_port = p_src->vxlan_port.b_update_port; + + if (p_src->geneve_port.b_update_port) + p_tun->geneve_port.port = p_src->geneve_port.port; + + if (p_src->vxlan_port.b_update_port) + p_tun->vxlan_port.port = p_src->vxlan_port.port; } static void -ecore_tunn_set_pf_update_params(struct ecore_hwfn *p_hwfn, - struct ecore_tunn_update_params *p_src, - struct pf_update_tunnel_config *p_tunn_cfg) +__ecore_set_ramrod_tunnel_param(u8 *p_tunn_cls, + struct ecore_tunn_update_type *tun_type) { - unsigned long tunn_mode = p_src->tunn_mode; - enum tunnel_clss type; + *p_tunn_cls = tun_type->tun_cls; +} - ecore_tunn_set_pf_fix_tunn_mode(p_hwfn, p_src, p_tunn_cfg); - p_tunn_cfg->update_rx_pf_clss = p_src->update_rx_pf_clss; - p_tunn_cfg->update_tx_pf_clss = p_src->update_tx_pf_clss; - - type = ecore_tunn_get_clss_type(p_src->tunn_clss_vxlan); - p_tunn_cfg->tunnel_clss_vxlan = type; - type = ecore_tunn_get_clss_type(p_src->tunn_clss_l2gre); - p_tunn_cfg->tunnel_clss_l2gre = type; - type = ecore_tunn_get_clss_type(p_src->tunn_clss_ipgre); - p_tunn_cfg->tunnel_clss_ipgre = type; - - if (p_src->update_vxlan_udp_port) { - p_tunn_cfg->set_vxlan_udp_port_flg = 1; - p_tunn_cfg->vxlan_udp_port = - OSAL_CPU_TO_LE16(p_src->vxlan_udp_port); +static void +ecore_set_ramrod_tunnel_param(u8 *p_tunn_cls, + struct ecore_tunn_update_type *tun_type, + u8 *p_update_port, __le16 *p_port, + struct ecore_tunn_update_udp_port *p_udp_port) +{ + __ecore_set_ramrod_tunnel_param(p_tunn_cls, tun_type); + if (p_udp_port->b_update_port) { + *p_update_port = 1; + *p_port = OSAL_CPU_TO_LE16(p_udp_port->port); } +} - if (OSAL_TEST_BIT(ECORE_MODE_L2GRE_TUNN, &tunn_mode)) - p_tunn_cfg->tx_enable_l2gre = 1; +static void +ecore_tunn_set_pf_update_params(struct ecore_hwfn *p_hwfn, + struct ecore_tunnel_info *p_src, + struct pf_update_tunnel_config *p_tunn_cfg) +{ + struct ecore_tunnel_info *p_tun = &p_hwfn->p_dev->tunnel; - if (OSAL_TEST_BIT(ECORE_MODE_IPGRE_TUNN, &tunn_mode)) - p_tunn_cfg->tx_enable_ipgre = 1; + ecore_set_pf_update_tunn_mode(p_tun, p_src, false); + ecore_set_tunn_cls_info(p_tun, p_src); + ecore_set_tunn_ports(p_tun, p_src); - if (OSAL_TEST_BIT(ECORE_MODE_VXLAN_TUNN, &tunn_mode)) - p_tunn_cfg->tx_enable_vxlan = 1; + ecore_set_ramrod_tunnel_param(&p_tunn_cfg->tunnel_clss_vxlan, + &p_tun->vxlan, + &p_tunn_cfg->set_vxlan_udp_port_flg, + &p_tunn_cfg->vxlan_udp_port, + &p_tun->vxlan_port); - if (ECORE_IS_BB_A0(p_hwfn->p_dev)) { - if (p_src->update_geneve_udp_port) - DP_NOTICE(p_hwfn, true, "Geneve not supported\n"); - p_src->update_geneve_udp_port = 0; - return; - } + ecore_set_ramrod_tunnel_param(&p_tunn_cfg->tunnel_clss_l2geneve, + &p_tun->l2_geneve, + &p_tunn_cfg->set_geneve_udp_port_flg, + &p_tunn_cfg->geneve_udp_port, + &p_tun->geneve_port); - if (p_src->update_geneve_udp_port) { - p_tunn_cfg->set_geneve_udp_port_flg = 1; - p_tunn_cfg->geneve_udp_port = - OSAL_CPU_TO_LE16(p_src->geneve_udp_port); - } + __ecore_set_ramrod_tunnel_param(&p_tunn_cfg->tunnel_clss_ipgeneve, + &p_tun->ip_geneve); - if (OSAL_TEST_BIT(ECORE_MODE_L2GENEVE_TUNN, &tunn_mode)) - p_tunn_cfg->tx_enable_l2geneve = 1; + __ecore_set_ramrod_tunnel_param(&p_tunn_cfg->tunnel_clss_l2gre, + &p_tun->l2_gre); - if (OSAL_TEST_BIT(ECORE_MODE_IPGENEVE_TUNN, &tunn_mode)) - p_tunn_cfg->tx_enable_ipgeneve = 1; + __ecore_set_ramrod_tunnel_param(&p_tunn_cfg->tunnel_clss_ipgre, + &p_tun->ip_gre); - type = ecore_tunn_get_clss_type(p_src->tunn_clss_l2geneve); - p_tunn_cfg->tunnel_clss_l2geneve = type; - type = ecore_tunn_get_clss_type(p_src->tunn_clss_ipgeneve); - p_tunn_cfg->tunnel_clss_ipgeneve = type; + p_tunn_cfg->update_rx_pf_clss = p_tun->b_update_rx_cls; } static void ecore_set_hw_tunn_mode(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, - unsigned long tunn_mode) + struct ecore_tunnel_info *p_tun) { - u8 l2gre_enable = 0, ipgre_enable = 0, vxlan_enable = 0; - u8 l2geneve_enable = 0, ipgeneve_enable = 0; + ecore_set_gre_enable(p_hwfn, p_ptt, p_tun->l2_gre.b_mode_enabled, + p_tun->ip_gre.b_mode_enabled); + ecore_set_vxlan_enable(p_hwfn, p_ptt, p_tun->vxlan.b_mode_enabled); - if (OSAL_TEST_BIT(ECORE_MODE_L2GRE_TUNN, &tunn_mode)) - l2gre_enable = 1; + ecore_set_geneve_enable(p_hwfn, p_ptt, p_tun->l2_geneve.b_mode_enabled, + p_tun->ip_geneve.b_mode_enabled); +} - if (OSAL_TEST_BIT(ECORE_MODE_IPGRE_TUNN, &tunn_mode)) - ipgre_enable = 1; - - if (OSAL_TEST_BIT(ECORE_MODE_VXLAN_TUNN, &tunn_mode)) - vxlan_enable = 1; - - ecore_set_gre_enable(p_hwfn, p_ptt, l2gre_enable, ipgre_enable); - ecore_set_vxlan_enable(p_hwfn, p_ptt, vxlan_enable); - - if (ECORE_IS_BB_A0(p_hwfn->p_dev)) +static void ecore_set_hw_tunn_mode_port(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + struct ecore_tunnel_info *p_tunn) +{ + if (ECORE_IS_BB_A0(p_hwfn->p_dev)) { + DP_NOTICE(p_hwfn, true, + "A0 chip: tunnel hw config is not supported\n"); return; + } - if (OSAL_TEST_BIT(ECORE_MODE_L2GENEVE_TUNN, &tunn_mode)) - l2geneve_enable = 1; + if (p_tunn->vxlan_port.b_update_port) + ecore_set_vxlan_dest_port(p_hwfn, p_ptt, + p_tunn->vxlan_port.port); - if (OSAL_TEST_BIT(ECORE_MODE_IPGENEVE_TUNN, &tunn_mode)) - ipgeneve_enable = 1; + if (p_tunn->geneve_port.b_update_port) + ecore_set_geneve_dest_port(p_hwfn, p_ptt, + p_tunn->geneve_port.port); - ecore_set_geneve_enable(p_hwfn, p_ptt, l2geneve_enable, - ipgeneve_enable); + ecore_set_hw_tunn_mode(p_hwfn, p_ptt, p_tunn); } static void ecore_tunn_set_pf_start_params(struct ecore_hwfn *p_hwfn, - struct ecore_tunn_start_params *p_src, + struct ecore_tunnel_info *p_src, struct pf_start_tunnel_config *p_tunn_cfg) { - unsigned long tunn_mode; - enum tunnel_clss type; + struct ecore_tunnel_info *p_tun = &p_hwfn->p_dev->tunnel; + + if (ECORE_IS_BB_A0(p_hwfn->p_dev)) { + DP_NOTICE(p_hwfn, true, + "A0 chip: tunnel pf start config is not supported\n"); + return; + } if (!p_src) return; - tunn_mode = p_src->tunn_mode; - type = ecore_tunn_get_clss_type(p_src->tunn_clss_vxlan); - p_tunn_cfg->tunnel_clss_vxlan = type; - type = ecore_tunn_get_clss_type(p_src->tunn_clss_l2gre); - p_tunn_cfg->tunnel_clss_l2gre = type; - type = ecore_tunn_get_clss_type(p_src->tunn_clss_ipgre); - p_tunn_cfg->tunnel_clss_ipgre = type; + ecore_set_pf_update_tunn_mode(p_tun, p_src, true); + ecore_set_tunn_cls_info(p_tun, p_src); + ecore_set_tunn_ports(p_tun, p_src); - if (p_src->update_vxlan_udp_port) { - p_tunn_cfg->set_vxlan_udp_port_flg = 1; - p_tunn_cfg->vxlan_udp_port = - OSAL_CPU_TO_LE16(p_src->vxlan_udp_port); - } + ecore_set_ramrod_tunnel_param(&p_tunn_cfg->tunnel_clss_vxlan, + &p_tun->vxlan, + &p_tunn_cfg->set_vxlan_udp_port_flg, + &p_tunn_cfg->vxlan_udp_port, + &p_tun->vxlan_port); - if (OSAL_TEST_BIT(ECORE_MODE_L2GRE_TUNN, &tunn_mode)) - p_tunn_cfg->tx_enable_l2gre = 1; + ecore_set_ramrod_tunnel_param(&p_tunn_cfg->tunnel_clss_l2geneve, + &p_tun->l2_geneve, + &p_tunn_cfg->set_geneve_udp_port_flg, + &p_tunn_cfg->geneve_udp_port, + &p_tun->geneve_port); - if (OSAL_TEST_BIT(ECORE_MODE_IPGRE_TUNN, &tunn_mode)) - p_tunn_cfg->tx_enable_ipgre = 1; + __ecore_set_ramrod_tunnel_param(&p_tunn_cfg->tunnel_clss_ipgeneve, + &p_tun->ip_geneve); - if (OSAL_TEST_BIT(ECORE_MODE_VXLAN_TUNN, &tunn_mode)) - p_tunn_cfg->tx_enable_vxlan = 1; + __ecore_set_ramrod_tunnel_param(&p_tunn_cfg->tunnel_clss_l2gre, + &p_tun->l2_gre); - if (ECORE_IS_BB_A0(p_hwfn->p_dev)) { - if (p_src->update_geneve_udp_port) - DP_NOTICE(p_hwfn, true, "Geneve not supported\n"); - p_src->update_geneve_udp_port = 0; - return; - } - - if (p_src->update_geneve_udp_port) { - p_tunn_cfg->set_geneve_udp_port_flg = 1; - p_tunn_cfg->geneve_udp_port = - OSAL_CPU_TO_LE16(p_src->geneve_udp_port); - } - - if (OSAL_TEST_BIT(ECORE_MODE_L2GENEVE_TUNN, &tunn_mode)) - p_tunn_cfg->tx_enable_l2geneve = 1; - - if (OSAL_TEST_BIT(ECORE_MODE_IPGENEVE_TUNN, &tunn_mode)) - p_tunn_cfg->tx_enable_ipgeneve = 1; - - type = ecore_tunn_get_clss_type(p_src->tunn_clss_l2geneve); - p_tunn_cfg->tunnel_clss_l2geneve = type; - type = ecore_tunn_get_clss_type(p_src->tunn_clss_ipgeneve); - p_tunn_cfg->tunnel_clss_ipgeneve = type; + __ecore_set_ramrod_tunnel_param(&p_tunn_cfg->tunnel_clss_ipgre, + &p_tun->ip_gre); } +#define ETH_P_8021Q 0x8100 + enum _ecore_status_t ecore_sp_pf_start(struct ecore_hwfn *p_hwfn, - struct ecore_tunn_start_params *p_tunn, - enum ecore_mf_mode mode, + struct ecore_ptt *p_ptt, + struct ecore_tunnel_info *p_tunn, bool allow_npar_tx_switch) { struct pf_start_ramrod_data *p_ramrod = OSAL_NULL; - struct ecore_spq_entry *p_ent = OSAL_NULL; u16 sb = ecore_int_get_sp_sb_id(p_hwfn); u8 sb_index = p_hwfn->p_eq->eq_sb_index; - enum _ecore_status_t rc = ECORE_NOTIMPL; + struct ecore_spq_entry *p_ent = OSAL_NULL; struct ecore_sp_init_data init_data; + enum _ecore_status_t rc = ECORE_NOTIMPL; u8 page_cnt; + int i; /* update initial eq producer */ ecore_eq_prod_update(p_hwfn, @@ -351,38 +331,45 @@ enum _ecore_status_t ecore_sp_pf_start(struct ecore_hwfn *p_hwfn, p_ramrod->event_ring_sb_id = OSAL_CPU_TO_LE16(sb); p_ramrod->event_ring_sb_index = sb_index; p_ramrod->path_id = ECORE_PATH_ID(p_hwfn); - p_ramrod->outer_tag = p_hwfn->hw_info.ovlan; /* For easier debugging */ p_ramrod->dont_log_ramrods = 0; - p_ramrod->log_type_mask = OSAL_CPU_TO_LE16(0xf); + p_ramrod->log_type_mask = OSAL_CPU_TO_LE16(0x8f); - switch (mode) { - case ECORE_MF_DEFAULT: - case ECORE_MF_NPAR: - p_ramrod->mf_mode = MF_NPAR; - break; - case ECORE_MF_OVLAN: + if (OSAL_TEST_BIT(ECORE_MF_OVLAN_CLSS, &p_hwfn->p_dev->mf_bits)) p_ramrod->mf_mode = MF_OVLAN; - break; - default: - DP_NOTICE(p_hwfn, true, - "Unsupported MF mode, init as DEFAULT\n"); + else p_ramrod->mf_mode = MF_NPAR; + + p_ramrod->outer_tag_config.outer_tag.tci = + OSAL_CPU_TO_LE16(p_hwfn->hw_info.ovlan); + + if (OSAL_TEST_BIT(ECORE_MF_UFP_SPECIFIC, &p_hwfn->p_dev->mf_bits)) { + p_ramrod->outer_tag_config.outer_tag.tpid = + OSAL_CPU_TO_LE16(ETH_P_8021Q); + if (p_hwfn->ufp_info.pri_type == ECORE_UFP_PRI_OS) + p_ramrod->outer_tag_config.enable_stag_pri_change = 1; + else + p_ramrod->outer_tag_config.enable_stag_pri_change = 0; + p_ramrod->outer_tag_config.pri_map_valid = 1; + for (i = 0; i < 8; i++) + p_ramrod->outer_tag_config.inner_to_outer_pri_map[i] = + (u8)i; } /* Place EQ address in RAMROD */ DMA_REGPAIR_LE(p_ramrod->event_ring_pbl_addr, - p_hwfn->p_eq->chain.pbl.p_phys_table); + p_hwfn->p_eq->chain.pbl_sp.p_phys_table); page_cnt = (u8)ecore_chain_get_page_cnt(&p_hwfn->p_eq->chain); p_ramrod->event_ring_num_pages = page_cnt; DMA_REGPAIR_LE(p_ramrod->consolid_q_pbl_addr, - p_hwfn->p_consq->chain.pbl.p_phys_table); + p_hwfn->p_consq->chain.pbl_sp.p_phys_table); ecore_tunn_set_pf_start_params(p_hwfn, p_tunn, &p_ramrod->tunnel_config); - if (IS_MF_SI(p_hwfn)) + if (OSAL_TEST_BIT(ECORE_MF_INTER_PF_SWITCH, + &p_hwfn->p_dev->mf_bits)) p_ramrod->allow_npar_tx_switching = allow_npar_tx_switch; switch (p_hwfn->hw_info.personality) { @@ -391,33 +378,41 @@ enum _ecore_status_t ecore_sp_pf_start(struct ecore_hwfn *p_hwfn, break; default: DP_NOTICE(p_hwfn, true, "Unknown personality %d\n", - p_hwfn->hw_info.personality); + p_hwfn->hw_info.personality); p_ramrod->personality = PERSONALITY_ETH; } - p_ramrod->base_vf_id = (u8)p_hwfn->hw_info.first_vf_in_pf; - p_ramrod->num_vfs = (u8)p_hwfn->p_dev->sriov_info.total_vfs; + if (p_hwfn->p_dev->p_iov_info) { + struct ecore_hw_sriov_info *p_iov = p_hwfn->p_dev->p_iov_info; + + p_ramrod->base_vf_id = (u8)p_iov->first_vf_in_pf; + p_ramrod->num_vfs = (u8)p_iov->total_vfs; + } + /* @@@TBD - update also the "ROCE_VER_KEY" entries when the FW RoCE HSI + * version is available. + */ + p_ramrod->hsi_fp_ver.major_ver_arr[ETH_VER_KEY] = ETH_HSI_VER_MAJOR; + p_ramrod->hsi_fp_ver.minor_ver_arr[ETH_VER_KEY] = ETH_HSI_VER_MINOR; DP_VERBOSE(p_hwfn, ECORE_MSG_SPQ, - "Setting event_ring_sb [id %04x index %02x], outer_tag [%d]\n", - sb, sb_index, p_ramrod->outer_tag); + "Setting event_ring_sb [id %04x index %02x], outer_tag.tpid [%d], outer_tag.tci [%d]\n", + sb, sb_index, p_ramrod->outer_tag_config.outer_tag.tpid, + p_ramrod->outer_tag_config.outer_tag.tci); rc = ecore_spq_post(p_hwfn, p_ent, OSAL_NULL); - if (p_tunn) { - ecore_set_hw_tunn_mode(p_hwfn, p_hwfn->p_main_ptt, - p_tunn->tunn_mode); - p_hwfn->p_dev->tunn_mode = p_tunn->tunn_mode; - } + if (p_tunn) + ecore_set_hw_tunn_mode_port(p_hwfn, p_ptt, + &p_hwfn->p_dev->tunnel); return rc; } -enum _ecore_status_t ecore_sp_pf_update(struct ecore_hwfn *p_hwfn) +enum _ecore_status_t ecore_sp_pf_update_dcbx(struct ecore_hwfn *p_hwfn) { struct ecore_spq_entry *p_ent = OSAL_NULL; - enum _ecore_status_t rc = ECORE_NOTIMPL; struct ecore_sp_init_data init_data; + enum _ecore_status_t rc = ECORE_NOTIMPL; /* Get SPQ entry */ OSAL_MEMSET(&init_data, 0, sizeof(init_data)); @@ -437,16 +432,131 @@ enum _ecore_status_t ecore_sp_pf_update(struct ecore_hwfn *p_hwfn) return ecore_spq_post(p_hwfn, p_ent, OSAL_NULL); } +enum _ecore_status_t ecore_sp_pf_update_ufp(struct ecore_hwfn *p_hwfn) +{ + struct ecore_spq_entry *p_ent = OSAL_NULL; + struct ecore_sp_init_data init_data; + enum _ecore_status_t rc = ECORE_NOTIMPL; + + /* Get SPQ entry */ + OSAL_MEMSET(&init_data, 0, sizeof(init_data)); + init_data.cid = ecore_spq_get_cid(p_hwfn); + init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; + init_data.comp_mode = ECORE_SPQ_MODE_CB; + + rc = ecore_sp_init_request(p_hwfn, &p_ent, + COMMON_RAMROD_PF_UPDATE, PROTOCOLID_COMMON, + &init_data); + if (rc != ECORE_SUCCESS) + return rc; + + p_ent->ramrod.pf_update.update_enable_stag_pri_change = true; + if (p_hwfn->ufp_info.pri_type == ECORE_UFP_PRI_OS) + p_ent->ramrod.pf_update.enable_stag_pri_change = 1; + else + p_ent->ramrod.pf_update.enable_stag_pri_change = 0; + + return ecore_spq_post(p_hwfn, p_ent, OSAL_NULL); +} + + +/* QM rate limiter resolution is 1.6Mbps */ +#define QM_RL_RESOLUTION(mb_val) ((mb_val) * 10 / 16) + +/* FW uses 1/64k to express gd */ +#define FW_GD_RESOLUTION(gd) (64 * 1024 / (gd)) + +u16 ecore_sp_rl_mb_to_qm(u32 mb_val) +{ + return (u16)OSAL_MIN_T(u32, (u16)(~0U), QM_RL_RESOLUTION(mb_val)); +} + +u16 ecore_sp_rl_gd_denom(u32 gd) +{ + return gd ? (u16)OSAL_MIN_T(u32, (u16)(~0U), FW_GD_RESOLUTION(gd)) : 0; +} + +enum _ecore_status_t ecore_sp_rl_update(struct ecore_hwfn *p_hwfn, + struct ecore_rl_update_params *params) +{ + struct ecore_spq_entry *p_ent = OSAL_NULL; + enum _ecore_status_t rc = ECORE_NOTIMPL; + struct rl_update_ramrod_data *rl_update; + struct ecore_sp_init_data init_data; + + /* Get SPQ entry */ + OSAL_MEMSET(&init_data, 0, sizeof(init_data)); + init_data.cid = ecore_spq_get_cid(p_hwfn); + init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; + init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK; + + rc = ecore_sp_init_request(p_hwfn, &p_ent, + COMMON_RAMROD_RL_UPDATE, PROTOCOLID_COMMON, + &init_data); + if (rc != ECORE_SUCCESS) + return rc; + + rl_update = &p_ent->ramrod.rl_update; + + rl_update->qcn_update_param_flg = params->qcn_update_param_flg; + rl_update->dcqcn_update_param_flg = params->dcqcn_update_param_flg; + rl_update->rl_init_flg = params->rl_init_flg; + rl_update->rl_start_flg = params->rl_start_flg; + rl_update->rl_stop_flg = params->rl_stop_flg; + rl_update->rl_id_first = params->rl_id_first; + rl_update->rl_id_last = params->rl_id_last; + rl_update->rl_dc_qcn_flg = params->rl_dc_qcn_flg; + rl_update->rl_bc_rate = OSAL_CPU_TO_LE32(params->rl_bc_rate); + rl_update->rl_max_rate = + OSAL_CPU_TO_LE16(ecore_sp_rl_mb_to_qm(params->rl_max_rate)); + rl_update->rl_r_ai = + OSAL_CPU_TO_LE16(ecore_sp_rl_mb_to_qm(params->rl_r_ai)); + rl_update->rl_r_hai = + OSAL_CPU_TO_LE16(ecore_sp_rl_mb_to_qm(params->rl_r_hai)); + rl_update->dcqcn_g = + OSAL_CPU_TO_LE16(ecore_sp_rl_gd_denom(params->dcqcn_gd)); + rl_update->dcqcn_k_us = OSAL_CPU_TO_LE32(params->dcqcn_k_us); + rl_update->dcqcn_timeuot_us = + OSAL_CPU_TO_LE32(params->dcqcn_timeuot_us); + rl_update->qcn_timeuot_us = OSAL_CPU_TO_LE32(params->qcn_timeuot_us); + + DP_VERBOSE(p_hwfn, ECORE_MSG_SPQ, "rl_params: qcn_update_param_flg %x, dcqcn_update_param_flg %x, rl_init_flg %x, rl_start_flg %x, rl_stop_flg %x, rl_id_first %x, rl_id_last %x, rl_dc_qcn_flg %x, rl_bc_rate %x, rl_max_rate %x, rl_r_ai %x, rl_r_hai %x, dcqcn_g %x, dcqcn_k_us %x, dcqcn_timeuot_us %x, qcn_timeuot_us %x\n", + rl_update->qcn_update_param_flg, + rl_update->dcqcn_update_param_flg, + rl_update->rl_init_flg, rl_update->rl_start_flg, + rl_update->rl_stop_flg, rl_update->rl_id_first, + rl_update->rl_id_last, rl_update->rl_dc_qcn_flg, + rl_update->rl_bc_rate, rl_update->rl_max_rate, + rl_update->rl_r_ai, rl_update->rl_r_hai, + rl_update->dcqcn_g, rl_update->dcqcn_k_us, + rl_update->dcqcn_timeuot_us, rl_update->qcn_timeuot_us); + + return ecore_spq_post(p_hwfn, p_ent, OSAL_NULL); +} + /* Set pf update ramrod command params */ enum _ecore_status_t ecore_sp_pf_update_tunn_cfg(struct ecore_hwfn *p_hwfn, - struct ecore_tunn_update_params *p_tunn, + struct ecore_ptt *p_ptt, + struct ecore_tunnel_info *p_tunn, enum spq_mode comp_mode, struct ecore_spq_comp_cb *p_comp_data) { struct ecore_spq_entry *p_ent = OSAL_NULL; - enum _ecore_status_t rc = ECORE_NOTIMPL; struct ecore_sp_init_data init_data; + enum _ecore_status_t rc = ECORE_NOTIMPL; + + if (IS_VF(p_hwfn->p_dev)) + return ecore_vf_pf_tunnel_param_update(p_hwfn, p_tunn); + + if (ECORE_IS_BB_A0(p_hwfn->p_dev)) { + DP_NOTICE(p_hwfn, true, + "A0 chip: tunnel pf update config is not supported\n"); + return rc; + } + + if (!p_tunn) + return ECORE_INVAL; /* Get SPQ entry */ OSAL_MEMSET(&init_data, 0, sizeof(init_data)); @@ -465,28 +575,19 @@ ecore_sp_pf_update_tunn_cfg(struct ecore_hwfn *p_hwfn, &p_ent->ramrod.pf_update.tunnel_config); rc = ecore_spq_post(p_hwfn, p_ent, OSAL_NULL); + if (rc != ECORE_SUCCESS) + return rc; - if ((rc == ECORE_SUCCESS) && p_tunn) { - if (p_tunn->update_vxlan_udp_port) - ecore_set_vxlan_dest_port(p_hwfn, p_hwfn->p_main_ptt, - p_tunn->vxlan_udp_port); - if (p_tunn->update_geneve_udp_port) - ecore_set_geneve_dest_port(p_hwfn, p_hwfn->p_main_ptt, - p_tunn->geneve_udp_port); - - ecore_set_hw_tunn_mode(p_hwfn, p_hwfn->p_main_ptt, - p_tunn->tunn_mode); - p_hwfn->p_dev->tunn_mode = p_tunn->tunn_mode; - } + ecore_set_hw_tunn_mode_port(p_hwfn, p_ptt, &p_hwfn->p_dev->tunnel); return rc; } enum _ecore_status_t ecore_sp_pf_stop(struct ecore_hwfn *p_hwfn) { - enum _ecore_status_t rc = ECORE_NOTIMPL; struct ecore_spq_entry *p_ent = OSAL_NULL; struct ecore_sp_init_data init_data; + enum _ecore_status_t rc = ECORE_NOTIMPL; /* Get SPQ entry */ OSAL_MEMSET(&init_data, 0, sizeof(init_data)); @@ -506,8 +607,8 @@ enum _ecore_status_t ecore_sp_pf_stop(struct ecore_hwfn *p_hwfn) enum _ecore_status_t ecore_sp_heartbeat_ramrod(struct ecore_hwfn *p_hwfn) { struct ecore_spq_entry *p_ent = OSAL_NULL; - enum _ecore_status_t rc = ECORE_NOTIMPL; struct ecore_sp_init_data init_data; + enum _ecore_status_t rc; /* Get SPQ entry */ OSAL_MEMSET(&init_data, 0, sizeof(init_data)); @@ -523,3 +624,28 @@ enum _ecore_status_t ecore_sp_heartbeat_ramrod(struct ecore_hwfn *p_hwfn) return ecore_spq_post(p_hwfn, p_ent, OSAL_NULL); } + +enum _ecore_status_t ecore_sp_pf_update_stag(struct ecore_hwfn *p_hwfn) +{ + struct ecore_spq_entry *p_ent = OSAL_NULL; + struct ecore_sp_init_data init_data; + enum _ecore_status_t rc = ECORE_NOTIMPL; + + /* Get SPQ entry */ + OSAL_MEMSET(&init_data, 0, sizeof(init_data)); + init_data.cid = ecore_spq_get_cid(p_hwfn); + init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; + init_data.comp_mode = ECORE_SPQ_MODE_CB; + + rc = ecore_sp_init_request(p_hwfn, &p_ent, + COMMON_RAMROD_PF_UPDATE, PROTOCOLID_COMMON, + &init_data); + if (rc != ECORE_SUCCESS) + return rc; + + p_ent->ramrod.pf_update.update_mf_vlan_flag = true; + p_ent->ramrod.pf_update.mf_vlan = + OSAL_CPU_TO_LE16(p_hwfn->hw_info.ovlan); + + return ecore_spq_post(p_hwfn, p_ent, OSAL_NULL); +} diff --git a/dpdk/drivers/net/qede/base/ecore_sp_commands.h b/dpdk/drivers/net/qede/base/ecore_sp_commands.h index e281ab0e..98009c65 100644 --- a/dpdk/drivers/net/qede/base/ecore_sp_commands.h +++ b/dpdk/drivers/net/qede/base/ecore_sp_commands.h @@ -21,12 +21,12 @@ struct ecore_sp_init_data { * e.g., in IOV scenarios. CID might defer between SPQ and * other elements. */ - u32 cid; - u16 opaque_fid; + u32 cid; + u16 opaque_fid; /* Information regarding operation upon sending & completion */ - enum spq_mode comp_mode; - struct ecore_spq_comp_cb *p_comp_data; + enum spq_mode comp_mode; + struct ecore_spq_comp_cb *p_comp_data; }; @@ -59,8 +59,8 @@ enum _ecore_status_t ecore_sp_init_request(struct ecore_hwfn *p_hwfn, * to the internal RAM of the UStorm by the Function Start Ramrod. * * @param p_hwfn + * @param p_ptt * @param p_tunn - pf start tunneling configuration - * @param mode * @param allow_npar_tx_switch - npar tx switching to be used * for vports configured for tx-switching. * @@ -68,31 +68,10 @@ enum _ecore_status_t ecore_sp_init_request(struct ecore_hwfn *p_hwfn, */ enum _ecore_status_t ecore_sp_pf_start(struct ecore_hwfn *p_hwfn, - struct ecore_tunn_start_params *p_tunn, - enum ecore_mf_mode mode, + struct ecore_ptt *p_ptt, + struct ecore_tunnel_info *p_tunn, bool allow_npar_tx_switch); -/** - * @brief ecore_sp_pf_update_tunn_cfg - PF Function Tunnel configuration - * update Ramrod - * - * This ramrod is sent to update a tunneling configuration - * for a physical function (PF). - * - * @param p_hwfn - * @param p_tunn - pf update tunneling parameters - * @param comp_mode - completion mode - * @param p_comp_data - callback function - * - * @return enum _ecore_status_t - */ - -enum _ecore_status_t -ecore_sp_pf_update_tunn_cfg(struct ecore_hwfn *p_hwfn, - struct ecore_tunn_update_params *p_tunn, - enum spq_mode comp_mode, - struct ecore_spq_comp_cb *p_comp_data); - /** * @brief ecore_sp_pf_update - PF Function Update Ramrod * @@ -106,7 +85,7 @@ ecore_sp_pf_update_tunn_cfg(struct ecore_hwfn *p_hwfn, * @return enum _ecore_status_t */ -enum _ecore_status_t ecore_sp_pf_update(struct ecore_hwfn *p_hwfn); +enum _ecore_status_t ecore_sp_pf_update_dcbx(struct ecore_hwfn *p_hwfn); /** * @brief ecore_sp_pf_stop - PF Function Stop Ramrod @@ -134,4 +113,53 @@ enum _ecore_status_t ecore_sp_pf_stop(struct ecore_hwfn *p_hwfn); enum _ecore_status_t ecore_sp_heartbeat_ramrod(struct ecore_hwfn *p_hwfn); +struct ecore_rl_update_params { + u8 qcn_update_param_flg; + u8 dcqcn_update_param_flg; + u8 rl_init_flg; + u8 rl_start_flg; + u8 rl_stop_flg; + u8 rl_id_first; + u8 rl_id_last; + u8 rl_dc_qcn_flg; /* If set, RL will used for DCQCN */ + u32 rl_bc_rate; /* Byte Counter Limit */ + u32 rl_max_rate; /* Maximum rate in Mbps resolution */ + u32 rl_r_ai; /* Active increase rate */ + u32 rl_r_hai; /* Hyper active increase rate */ + u32 dcqcn_gd; /* DCQCN Alpha update gain */ + u32 dcqcn_k_us; /* DCQCN Alpha update interval */ + u32 dcqcn_timeuot_us; + u32 qcn_timeuot_us; +}; + +/** + * @brief ecore_sp_rl_update - Update rate limiters + * + * @param p_hwfn + * @param params + * + * @return enum _ecore_status_t + */ +enum _ecore_status_t ecore_sp_rl_update(struct ecore_hwfn *p_hwfn, + struct ecore_rl_update_params *params); + +/** + * @brief ecore_sp_pf_update_stag - PF STAG value update Ramrod + * + * @param p_hwfn + * + * @return enum _ecore_status_t + */ + +enum _ecore_status_t ecore_sp_pf_update_stag(struct ecore_hwfn *p_hwfn); + +/** + * @brief ecore_sp_pf_update_ufp - PF ufp update Ramrod + * + * @param p_hwfn + * + * @return enum _ecore_status_t + */ +enum _ecore_status_t ecore_sp_pf_update_ufp(struct ecore_hwfn *p_hwfn); + #endif /*__ECORE_SP_COMMANDS_H__*/ diff --git a/dpdk/drivers/net/qede/base/ecore_spq.c b/dpdk/drivers/net/qede/base/ecore_spq.c index b263693f..70ffa8cd 100644 --- a/dpdk/drivers/net/qede/base/ecore_spq.c +++ b/dpdk/drivers/net/qede/base/ecore_spq.c @@ -27,14 +27,17 @@ ***************************************************************************/ #define SPQ_HIGH_PRI_RESERVE_DEFAULT (1) -#define SPQ_BLOCK_SLEEP_LENGTH (1000) + +#define SPQ_BLOCK_DELAY_MAX_ITER (10) +#define SPQ_BLOCK_DELAY_US (10) +#define SPQ_BLOCK_SLEEP_MAX_ITER (1000) +#define SPQ_BLOCK_SLEEP_MS (5) /*************************************************************************** * Blocking Imp. (BLOCK/EBLOCK mode) ***************************************************************************/ -static void ecore_spq_blocking_cb(struct ecore_hwfn *p_hwfn, - void *cookie, - union event_ring_data *data, +static void ecore_spq_blocking_cb(struct ecore_hwfn *p_hwfn, void *cookie, + union event_ring_data OSAL_UNUSED * data, u8 fw_return_code) { struct ecore_spq_comp_done *comp_done; @@ -48,53 +51,82 @@ static void ecore_spq_blocking_cb(struct ecore_hwfn *p_hwfn, OSAL_SMP_WMB(p_hwfn->p_dev); } -static enum _ecore_status_t ecore_spq_block(struct ecore_hwfn *p_hwfn, - struct ecore_spq_entry *p_ent, - u8 *p_fw_ret) +static enum _ecore_status_t __ecore_spq_block(struct ecore_hwfn *p_hwfn, + struct ecore_spq_entry *p_ent, + u8 *p_fw_ret, + bool sleep_between_iter) { - int sleep_count = SPQ_BLOCK_SLEEP_LENGTH; struct ecore_spq_comp_done *comp_done; - enum _ecore_status_t rc; + u32 iter_cnt; comp_done = (struct ecore_spq_comp_done *)p_ent->comp_cb.cookie; - while (sleep_count) { + iter_cnt = sleep_between_iter ? SPQ_BLOCK_SLEEP_MAX_ITER + : SPQ_BLOCK_DELAY_MAX_ITER; + + while (iter_cnt--) { OSAL_POLL_MODE_DPC(p_hwfn); - /* validate we receive completion update */ OSAL_SMP_RMB(p_hwfn->p_dev); if (comp_done->done == 1) { if (p_fw_ret) *p_fw_ret = comp_done->fw_return_code; return ECORE_SUCCESS; } - OSAL_MSLEEP(5); - sleep_count--; + + if (sleep_between_iter) + OSAL_MSLEEP(SPQ_BLOCK_SLEEP_MS); + else + OSAL_UDELAY(SPQ_BLOCK_DELAY_US); } + return ECORE_TIMEOUT; +} + +static enum _ecore_status_t ecore_spq_block(struct ecore_hwfn *p_hwfn, + struct ecore_spq_entry *p_ent, + u8 *p_fw_ret, bool skip_quick_poll) +{ + struct ecore_spq_comp_done *comp_done; + struct ecore_ptt *p_ptt; + enum _ecore_status_t rc; + + /* A relatively short polling period w/o sleeping, to allow the FW to + * complete the ramrod and thus possibly to avoid the following sleeps. + */ + if (!skip_quick_poll) { + rc = __ecore_spq_block(p_hwfn, p_ent, p_fw_ret, false); + if (rc == ECORE_SUCCESS) + return ECORE_SUCCESS; + } + + /* Move to polling with a sleeping period between iterations */ + rc = __ecore_spq_block(p_hwfn, p_ent, p_fw_ret, true); + if (rc == ECORE_SUCCESS) + return ECORE_SUCCESS; + + p_ptt = ecore_ptt_acquire(p_hwfn); + if (!p_ptt) + return ECORE_AGAIN; + DP_INFO(p_hwfn, "Ramrod is stuck, requesting MCP drain\n"); - rc = ecore_mcp_drain(p_hwfn, p_hwfn->p_main_ptt); - if (rc != ECORE_SUCCESS) + rc = ecore_mcp_drain(p_hwfn, p_ptt); + ecore_ptt_release(p_hwfn, p_ptt); + if (rc != ECORE_SUCCESS) { DP_NOTICE(p_hwfn, true, "MCP drain failed\n"); - - /* Retry after drain */ - sleep_count = SPQ_BLOCK_SLEEP_LENGTH; - while (sleep_count) { - /* validate we receive completion update */ - OSAL_SMP_RMB(p_hwfn->p_dev); - if (comp_done->done == 1) { - if (p_fw_ret) - *p_fw_ret = comp_done->fw_return_code; - return ECORE_SUCCESS; - } - OSAL_MSLEEP(5); - sleep_count--; + goto err; } + /* Retry after drain */ + rc = __ecore_spq_block(p_hwfn, p_ent, p_fw_ret, true); + if (rc == ECORE_SUCCESS) + return ECORE_SUCCESS; + + comp_done = (struct ecore_spq_comp_done *)p_ent->comp_cb.cookie; if (comp_done->done == 1) { if (p_fw_ret) *p_fw_ret = comp_done->fw_return_code; return ECORE_SUCCESS; } - +err: DP_NOTICE(p_hwfn, true, "Ramrod is stuck [CID %08x cmd %02x proto %02x echo %04x]\n", OSAL_LE32_TO_CPU(p_ent->elem.hdr.cid), @@ -146,10 +178,9 @@ ecore_spq_fill_entry(struct ecore_hwfn *p_hwfn, struct ecore_spq_entry *p_ent) static void ecore_spq_hw_initialize(struct ecore_hwfn *p_hwfn, struct ecore_spq *p_spq) { - u16 pq; + struct e4_core_conn_context *p_cxt; struct ecore_cxt_info cxt_info; - struct core_conn_context *p_cxt; - union ecore_qm_pq_params pq_params; + u16 physical_q; enum _ecore_status_t rc; cxt_info.iid = p_spq->cid; @@ -157,40 +188,41 @@ static void ecore_spq_hw_initialize(struct ecore_hwfn *p_hwfn, rc = ecore_cxt_get_cid_info(p_hwfn, &cxt_info); if (rc < 0) { - DP_NOTICE(p_hwfn, true, "Cannot find context info for cid=%d", + DP_NOTICE(p_hwfn, true, "Cannot find context info for cid=%d\n", p_spq->cid); return; } p_cxt = cxt_info.p_cxt; - SET_FIELD(p_cxt->xstorm_ag_context.flags10, - XSTORM_CORE_CONN_AG_CTX_DQ_CF_EN, 1); - SET_FIELD(p_cxt->xstorm_ag_context.flags1, - XSTORM_CORE_CONN_AG_CTX_DQ_CF_ACTIVE, 1); - /* SET_FIELD(p_cxt->xstorm_ag_context.flags10, - * XSTORM_CORE_CONN_AG_CTX_SLOW_PATH_EN, 1); - */ - SET_FIELD(p_cxt->xstorm_ag_context.flags9, - XSTORM_CORE_CONN_AG_CTX_CONSOLID_PROD_CF_EN, 1); + /* @@@TBD we zero the context until we have ilt_reset implemented. */ + OSAL_MEM_ZERO(p_cxt, sizeof(*p_cxt)); + + if (ECORE_IS_BB(p_hwfn->p_dev) || ECORE_IS_AH(p_hwfn->p_dev)) { + SET_FIELD(p_cxt->xstorm_ag_context.flags10, + E4_XSTORM_CORE_CONN_AG_CTX_DQ_CF_EN, 1); + SET_FIELD(p_cxt->xstorm_ag_context.flags1, + E4_XSTORM_CORE_CONN_AG_CTX_DQ_CF_ACTIVE, 1); + /* SET_FIELD(p_cxt->xstorm_ag_context.flags10, + * E4_XSTORM_CORE_CONN_AG_CTX_SLOW_PATH_EN, 1); + */ + SET_FIELD(p_cxt->xstorm_ag_context.flags9, + E4_XSTORM_CORE_CONN_AG_CTX_CONSOLID_PROD_CF_EN, 1); + } /* CDU validation - FIXME currently disabled */ /* QM physical queue */ - OSAL_MEMSET(&pq_params, 0, sizeof(pq_params)); - pq_params.core.tc = LB_TC; - pq = ecore_get_qm_pq(p_hwfn, PROTOCOLID_CORE, &pq_params); - p_cxt->xstorm_ag_context.physical_q0 = OSAL_CPU_TO_LE16(pq); + physical_q = ecore_get_cm_pq_idx(p_hwfn, PQ_FLAGS_LB); + p_cxt->xstorm_ag_context.physical_q0 = OSAL_CPU_TO_LE16(physical_q); p_cxt->xstorm_st_context.spq_base_lo = DMA_LO_LE(p_spq->chain.p_phys_addr); p_cxt->xstorm_st_context.spq_base_hi = DMA_HI_LE(p_spq->chain.p_phys_addr); - p_cxt->xstorm_st_context.consolid_base_addr.lo = - DMA_LO_LE(p_hwfn->p_consq->chain.p_phys_addr); - p_cxt->xstorm_st_context.consolid_base_addr.hi = - DMA_HI_LE(p_hwfn->p_consq->chain.p_phys_addr); + DMA_REGPAIR_LE(p_cxt->xstorm_st_context.consolid_base_addr, + p_hwfn->p_consq->chain.p_phys_addr); } static enum _ecore_status_t ecore_spq_hw_post(struct ecore_hwfn *p_hwfn, @@ -198,9 +230,9 @@ static enum _ecore_status_t ecore_spq_hw_post(struct ecore_hwfn *p_hwfn, struct ecore_spq_entry *p_ent) { struct ecore_chain *p_chain = &p_hwfn->p_spq->chain; + struct core_db_data *p_db_data = &p_spq->db_data; u16 echo = ecore_chain_get_prod_idx(p_chain); struct slow_path_element *elem; - struct core_db_data db; p_ent->elem.hdr.echo = OSAL_CPU_TO_LE16(echo); elem = ecore_chain_produce(p_chain); @@ -209,34 +241,24 @@ static enum _ecore_status_t ecore_spq_hw_post(struct ecore_hwfn *p_hwfn, return ECORE_INVAL; } - *elem = p_ent->elem; /* struct assignment */ + *elem = p_ent->elem; /* Struct assignment */ - /* send a doorbell on the slow hwfn session */ - OSAL_MEMSET(&db, 0, sizeof(db)); - SET_FIELD(db.params, CORE_DB_DATA_DEST, DB_DEST_XCM); - SET_FIELD(db.params, CORE_DB_DATA_AGG_CMD, DB_AGG_CMD_SET); - SET_FIELD(db.params, CORE_DB_DATA_AGG_VAL_SEL, - DQ_XCM_CORE_SPQ_PROD_CMD); - db.agg_flags = DQ_XCM_CORE_DQ_CF_CMD; + p_db_data->spq_prod = + OSAL_CPU_TO_LE16(ecore_chain_get_prod_idx(p_chain)); - /* validate producer is up to-date */ - OSAL_RMB(p_hwfn->p_dev); + /* Make sure the SPQE is updated before the doorbell */ + OSAL_WMB(p_hwfn->p_dev); - db.spq_prod = OSAL_CPU_TO_LE16(ecore_chain_get_prod_idx(p_chain)); + DOORBELL(p_hwfn, p_spq->db_addr_offset, *(u32 *)p_db_data); - /* do not reorder */ - OSAL_BARRIER(p_hwfn->p_dev); - - DOORBELL(p_hwfn, DB_ADDR(p_spq->cid, DQ_DEMS_LEGACY), *(u32 *)&db); - - /* make sure doorbell is rang */ - OSAL_MMIOWB(p_hwfn->p_dev); + /* Make sure doorbell is rang */ + OSAL_WMB(p_hwfn->p_dev); DP_VERBOSE(p_hwfn, ECORE_MSG_SPQ, "Doorbelled [0x%08x, CID 0x%08x] with Flags: %02x" " agg_params: %02x, prod: %04x\n", - DB_ADDR(p_spq->cid, DQ_DEMS_LEGACY), p_spq->cid, db.params, - db.agg_flags, ecore_chain_get_prod_idx(p_chain)); + p_spq->db_addr_offset, p_spq->cid, p_db_data->params, + p_db_data->agg_flags, ecore_chain_get_prod_idx(p_chain)); return ECORE_SUCCESS; } @@ -249,12 +271,16 @@ static enum _ecore_status_t ecore_async_event_completion(struct ecore_hwfn *p_hwfn, struct event_ring_entry *p_eqe) { - switch (p_eqe->protocol_id) { - case PROTOCOLID_COMMON: - return ecore_sriov_eqe_event(p_hwfn, - p_eqe->opcode, - p_eqe->echo, &p_eqe->data); - default: + ecore_spq_async_comp_cb cb; + + if (!p_hwfn->p_spq || (p_eqe->protocol_id >= MAX_PROTOCOL_TYPE)) + return ECORE_INVAL; + + cb = p_hwfn->p_spq->async_comp_cb[p_eqe->protocol_id]; + if (cb) { + return cb(p_hwfn, p_eqe->opcode, p_eqe->echo, + &p_eqe->data, p_eqe->fw_return_code); + } else { DP_NOTICE(p_hwfn, true, "Unknown Async completion for protocol: %d\n", p_eqe->protocol_id); @@ -262,6 +288,28 @@ ecore_async_event_completion(struct ecore_hwfn *p_hwfn, } } +enum _ecore_status_t +ecore_spq_register_async_cb(struct ecore_hwfn *p_hwfn, + enum protocol_type protocol_id, + ecore_spq_async_comp_cb cb) +{ + if (!p_hwfn->p_spq || (protocol_id >= MAX_PROTOCOL_TYPE)) + return ECORE_INVAL; + + p_hwfn->p_spq->async_comp_cb[protocol_id] = cb; + return ECORE_SUCCESS; +} + +void +ecore_spq_unregister_async_cb(struct ecore_hwfn *p_hwfn, + enum protocol_type protocol_id) +{ + if (!p_hwfn->p_spq || (protocol_id >= MAX_PROTOCOL_TYPE)) + return; + + p_hwfn->p_spq->async_comp_cb[protocol_id] = OSAL_NULL; +} + /*************************************************************************** * EQ API ***************************************************************************/ @@ -305,14 +353,15 @@ enum _ecore_status_t ecore_eq_completion(struct ecore_hwfn *p_hwfn, } DP_VERBOSE(p_hwfn, ECORE_MSG_SPQ, - "op %x prot %x res0 %x echo %x " - "fwret %x flags %x\n", p_eqe->opcode, + "op %x prot %x res0 %x echo %x fwret %x flags %x\n", + p_eqe->opcode, /* Event Opcode */ p_eqe->protocol_id, /* Event Protocol ID */ p_eqe->reserved0, /* Reserved */ + /* Echo value from ramrod data on the host */ OSAL_LE16_TO_CPU(p_eqe->echo), - p_eqe->fw_return_code, /* FW return code for SP - * ramrods - */ + p_eqe->fw_return_code, /* FW return code for SP + * ramrods + */ p_eqe->flags); if (GET_FIELD(p_eqe->flags, EVENT_RING_ENTRY_ASYNC)) { @@ -333,52 +382,56 @@ enum _ecore_status_t ecore_eq_completion(struct ecore_hwfn *p_hwfn, return rc; } -struct ecore_eq *ecore_eq_alloc(struct ecore_hwfn *p_hwfn, u16 num_elem) +enum _ecore_status_t ecore_eq_alloc(struct ecore_hwfn *p_hwfn, u16 num_elem) { struct ecore_eq *p_eq; /* Allocate EQ struct */ - p_eq = OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL, sizeof(struct ecore_eq)); + p_eq = OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL, sizeof(*p_eq)); if (!p_eq) { DP_NOTICE(p_hwfn, true, "Failed to allocate `struct ecore_eq'\n"); - return OSAL_NULL; + return ECORE_NOMEM; } - /* Allocate and initialize EQ chain */ + /* Allocate and initialize EQ chain*/ if (ecore_chain_alloc(p_hwfn->p_dev, ECORE_CHAIN_USE_TO_PRODUCE, ECORE_CHAIN_MODE_PBL, ECORE_CHAIN_CNT_TYPE_U16, num_elem, - sizeof(union event_ring_element), &p_eq->chain)) { - DP_NOTICE(p_hwfn, true, "Failed to allocate eq chain"); + sizeof(union event_ring_element), + &p_eq->chain, OSAL_NULL) != ECORE_SUCCESS) { + DP_NOTICE(p_hwfn, true, "Failed to allocate eq chain\n"); goto eq_allocate_fail; } /* register EQ completion on the SP SB */ - ecore_int_register_cb(p_hwfn, - ecore_eq_completion, + ecore_int_register_cb(p_hwfn, ecore_eq_completion, p_eq, &p_eq->eq_sb_index, &p_eq->p_fw_cons); - return p_eq; + p_hwfn->p_eq = p_eq; + return ECORE_SUCCESS; eq_allocate_fail: - ecore_eq_free(p_hwfn, p_eq); - return OSAL_NULL; -} - -void ecore_eq_setup(struct ecore_hwfn *p_hwfn, struct ecore_eq *p_eq) -{ - ecore_chain_reset(&p_eq->chain); -} - -void ecore_eq_free(struct ecore_hwfn *p_hwfn, struct ecore_eq *p_eq) -{ - if (!p_eq) - return; - ecore_chain_free(p_hwfn->p_dev, &p_eq->chain); OSAL_FREE(p_hwfn->p_dev, p_eq); + return ECORE_NOMEM; +} + +void ecore_eq_setup(struct ecore_hwfn *p_hwfn) +{ + ecore_chain_reset(&p_hwfn->p_eq->chain); +} + +void ecore_eq_free(struct ecore_hwfn *p_hwfn) +{ + if (!p_hwfn->p_eq) + return; + + ecore_chain_free(p_hwfn->p_dev, &p_hwfn->p_eq->chain); + + OSAL_FREE(p_hwfn->p_dev, p_hwfn->p_eq); + p_hwfn->p_eq = OSAL_NULL; } /*************************************************************************** @@ -419,10 +472,13 @@ enum _ecore_status_t ecore_eth_cqe_completion(struct ecore_hwfn *p_hwfn, ***************************************************************************/ void ecore_spq_setup(struct ecore_hwfn *p_hwfn) { - struct ecore_spq_entry *p_virt = OSAL_NULL; struct ecore_spq *p_spq = p_hwfn->p_spq; + struct ecore_spq_entry *p_virt = OSAL_NULL; + struct core_db_data *p_db_data; + void OSAL_IOMEM *db_addr; dma_addr_t p_phys = 0; u32 i, capacity; + enum _ecore_status_t rc; OSAL_LIST_INIT(&p_spq->pending); OSAL_LIST_INIT(&p_spq->completion_pending); @@ -436,8 +492,7 @@ void ecore_spq_setup(struct ecore_hwfn *p_hwfn) capacity = ecore_chain_get_capacity(&p_spq->chain); for (i = 0; i < capacity; i++) { - p_virt->elem.data_ptr.hi = DMA_HI_LE(p_phys); - p_virt->elem.data_ptr.lo = DMA_LO_LE(p_phys); + DMA_REGPAIR_LE(p_virt->elem.data_ptr, p_phys); OSAL_LIST_PUSH_TAIL(&p_virt->list, &p_spq->free_pool); @@ -461,6 +516,24 @@ void ecore_spq_setup(struct ecore_hwfn *p_hwfn) /* reset the chain itself */ ecore_chain_reset(&p_spq->chain); + + /* Initialize the address/data of the SPQ doorbell */ + p_spq->db_addr_offset = DB_ADDR(p_spq->cid, DQ_DEMS_LEGACY); + p_db_data = &p_spq->db_data; + OSAL_MEM_ZERO(p_db_data, sizeof(*p_db_data)); + SET_FIELD(p_db_data->params, CORE_DB_DATA_DEST, DB_DEST_XCM); + SET_FIELD(p_db_data->params, CORE_DB_DATA_AGG_CMD, DB_AGG_CMD_MAX); + SET_FIELD(p_db_data->params, CORE_DB_DATA_AGG_VAL_SEL, + DQ_XCM_CORE_SPQ_PROD_CMD); + p_db_data->agg_flags = DQ_XCM_CORE_DQ_CF_CMD; + + /* Register the SPQ doorbell with the doorbell recovery mechanism */ + db_addr = (void *)((u8 *)p_hwfn->doorbells + p_spq->db_addr_offset); + rc = ecore_db_recovery_add(p_hwfn->p_dev, db_addr, &p_spq->db_data, + DB_REC_WIDTH_32B, DB_REC_KERNEL); + if (rc != ECORE_SUCCESS) + DP_INFO(p_hwfn, + "Failed to register the SPQ doorbell with the doorbell recovery mechanism\n"); } enum _ecore_status_t ecore_spq_alloc(struct ecore_hwfn *p_hwfn) @@ -475,16 +548,19 @@ enum _ecore_status_t ecore_spq_alloc(struct ecore_hwfn *p_hwfn) OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL, sizeof(struct ecore_spq)); if (!p_spq) { DP_NOTICE(p_hwfn, true, - "Failed to allocate `struct ecore_spq'"); + "Failed to allocate `struct ecore_spq'\n"); return ECORE_NOMEM; } /* SPQ ring */ - if (ecore_chain_alloc(p_hwfn->p_dev, ECORE_CHAIN_USE_TO_PRODUCE, - ECORE_CHAIN_MODE_SINGLE, ECORE_CHAIN_CNT_TYPE_U16, 0, - /* N/A when the mode is SINGLE */ - sizeof(struct slow_path_element), &p_spq->chain)) { - DP_NOTICE(p_hwfn, true, "Failed to allocate spq chain"); + if (ecore_chain_alloc(p_hwfn->p_dev, + ECORE_CHAIN_USE_TO_PRODUCE, + ECORE_CHAIN_MODE_SINGLE, + ECORE_CHAIN_CNT_TYPE_U16, + 0, /* N/A when the mode is SINGLE */ + sizeof(struct slow_path_element), + &p_spq->chain, OSAL_NULL)) { + DP_NOTICE(p_hwfn, true, "Failed to allocate spq chain\n"); goto spq_allocate_fail; } @@ -499,7 +575,9 @@ enum _ecore_status_t ecore_spq_alloc(struct ecore_hwfn *p_hwfn) p_spq->p_virt = p_virt; p_spq->p_phys = p_phys; +#ifdef CONFIG_ECORE_LOCK_ALLOC OSAL_SPIN_LOCK_ALLOC(p_hwfn, &p_spq->lock); +#endif p_hwfn->p_spq = p_spq; return ECORE_SUCCESS; @@ -513,11 +591,16 @@ spq_allocate_fail: void ecore_spq_free(struct ecore_hwfn *p_hwfn) { struct ecore_spq *p_spq = p_hwfn->p_spq; + void OSAL_IOMEM *db_addr; u32 capacity; if (!p_spq) return; + /* Delete the SPQ doorbell from the doorbell recovery mechanism */ + db_addr = (void *)((u8 *)p_hwfn->doorbells + p_spq->db_addr_offset); + ecore_db_recovery_del(p_hwfn->p_dev, db_addr, &p_spq->db_data); + if (p_spq->p_virt) { capacity = ecore_chain_get_capacity(&p_spq->chain); OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev, @@ -528,7 +611,10 @@ void ecore_spq_free(struct ecore_hwfn *p_hwfn) } ecore_chain_free(p_hwfn->p_dev, &p_spq->chain); +#ifdef CONFIG_ECORE_LOCK_ALLOC OSAL_SPIN_LOCK_DEALLOC(&p_spq->lock); +#endif + OSAL_FREE(p_hwfn->p_dev, p_spq); } @@ -537,18 +623,18 @@ ecore_spq_get_entry(struct ecore_hwfn *p_hwfn, struct ecore_spq_entry **pp_ent) { struct ecore_spq *p_spq = p_hwfn->p_spq; struct ecore_spq_entry *p_ent = OSAL_NULL; + enum _ecore_status_t rc = ECORE_SUCCESS; OSAL_SPIN_LOCK(&p_spq->lock); if (OSAL_LIST_IS_EMPTY(&p_spq->free_pool)) { - p_ent = OSAL_ZALLOC(p_hwfn->p_dev, GFP_ATOMIC, - sizeof(struct ecore_spq_entry)); + p_ent = OSAL_ZALLOC(p_hwfn->p_dev, GFP_ATOMIC, sizeof(*p_ent)); if (!p_ent) { - OSAL_SPIN_UNLOCK(&p_spq->lock); DP_NOTICE(p_hwfn, true, - "Failed to allocate an SPQ entry" - " for a pending ramrod\n"); - return ECORE_NOMEM; + "Failed to allocate an SPQ entry for a pending" + " ramrod\n"); + rc = ECORE_NOMEM; + goto out_unlock; } p_ent->queue = &p_spq->unlimited_pending; } else { @@ -560,9 +646,9 @@ ecore_spq_get_entry(struct ecore_hwfn *p_hwfn, struct ecore_spq_entry **pp_ent) *pp_ent = p_ent; +out_unlock: OSAL_SPIN_UNLOCK(&p_spq->lock); - - return ECORE_SUCCESS; + return rc; } /* Locked variant; Should be called while the SPQ lock is taken */ @@ -607,32 +693,29 @@ ecore_spq_add_entry(struct ecore_hwfn *p_hwfn, p_spq->unlimited_pending_count++; return ECORE_SUCCESS; + + } else { + struct ecore_spq_entry *p_en2; + + p_en2 = OSAL_LIST_FIRST_ENTRY(&p_spq->free_pool, + struct ecore_spq_entry, + list); + OSAL_LIST_REMOVE_ENTRY(&p_en2->list, &p_spq->free_pool); + + /* Copy the ring element physical pointer to the new + * entry, since we are about to override the entire ring + * entry and don't want to lose the pointer. + */ + p_ent->elem.data_ptr = p_en2->elem.data_ptr; + + *p_en2 = *p_ent; + + /* EBLOCK responsible to free the allocated p_ent */ + if (p_ent->comp_mode != ECORE_SPQ_MODE_EBLOCK) + OSAL_FREE(p_hwfn->p_dev, p_ent); + + p_ent = p_en2; } - - struct ecore_spq_entry *p_en2; - - p_en2 = OSAL_LIST_FIRST_ENTRY(&p_spq->free_pool, - struct ecore_spq_entry, - list); - OSAL_LIST_REMOVE_ENTRY(&p_en2->list, &p_spq->free_pool); - - /* Copy the ring element physical pointer to the new - * entry, since we are about to override the entire ring - * entry and don't want to lose the pointer. - */ - p_ent->elem.data_ptr = p_en2->elem.data_ptr; - - /* Setting the cookie to the comp_done of the - * new element. - */ - if (p_ent->comp_cb.cookie == &p_ent->comp_done) - p_ent->comp_cb.cookie = &p_en2->comp_done; - - *p_en2 = *p_ent; - - OSAL_FREE(p_hwfn->p_dev, p_ent); - - p_ent = p_en2; } /* entry is to be placed in 'pending' queue */ @@ -682,16 +765,22 @@ static enum _ecore_status_t ecore_spq_post_list(struct ecore_hwfn *p_hwfn, !OSAL_LIST_IS_EMPTY(head)) { struct ecore_spq_entry *p_ent = OSAL_LIST_FIRST_ENTRY(head, struct ecore_spq_entry, list); - OSAL_LIST_REMOVE_ENTRY(&p_ent->list, head); - OSAL_LIST_PUSH_TAIL(&p_ent->list, &p_spq->completion_pending); - p_spq->comp_sent_count++; + if (p_ent != OSAL_NULL) { +#if defined(_NTDDK_) +#pragma warning(suppress : 6011 28182) +#endif + OSAL_LIST_REMOVE_ENTRY(&p_ent->list, head); + OSAL_LIST_PUSH_TAIL(&p_ent->list, + &p_spq->completion_pending); + p_spq->comp_sent_count++; - rc = ecore_spq_hw_post(p_hwfn, p_spq, p_ent); - if (rc) { - OSAL_LIST_REMOVE_ENTRY(&p_ent->list, - &p_spq->completion_pending); - __ecore_spq_return_entry(p_hwfn, p_ent); - return rc; + rc = ecore_spq_hw_post(p_hwfn, p_spq, p_ent); + if (rc) { + OSAL_LIST_REMOVE_ENTRY(&p_ent->list, + &p_spq->completion_pending); + __ecore_spq_return_entry(p_hwfn, p_ent); + return rc; + } } } @@ -700,7 +789,6 @@ static enum _ecore_status_t ecore_spq_post_list(struct ecore_hwfn *p_hwfn, static enum _ecore_status_t ecore_spq_pend_post(struct ecore_hwfn *p_hwfn) { - enum _ecore_status_t rc = ECORE_NOTIMPL; struct ecore_spq *p_spq = p_hwfn->p_spq; struct ecore_spq_entry *p_ent = OSAL_NULL; @@ -713,17 +801,16 @@ static enum _ecore_status_t ecore_spq_pend_post(struct ecore_hwfn *p_hwfn) if (!p_ent) return ECORE_INVAL; +#if defined(_NTDDK_) +#pragma warning(suppress : 6011) +#endif OSAL_LIST_REMOVE_ENTRY(&p_ent->list, &p_spq->unlimited_pending); ecore_spq_add_entry(p_hwfn, p_ent, p_ent->priority); } - rc = ecore_spq_post_list(p_hwfn, + return ecore_spq_post_list(p_hwfn, &p_spq->pending, SPQ_HIGH_PRI_RESERVE_DEFAULT); - if (rc) - return rc; - - return ECORE_SUCCESS; } enum _ecore_status_t ecore_spq_post(struct ecore_hwfn *p_hwfn, @@ -745,7 +832,7 @@ enum _ecore_status_t ecore_spq_post(struct ecore_hwfn *p_hwfn, if (p_hwfn->p_dev->recov_in_prog) { DP_VERBOSE(p_hwfn, ECORE_MSG_SPQ, "Recovery is in progress -> skip spq post" - " [cmd %02x protocol %02x]", + " [cmd %02x protocol %02x]\n", p_ent->elem.hdr.cmd_id, p_ent->elem.hdr.protocol_id); /* Return success to let the flows to be completed successfully * w/o any error handling. @@ -785,7 +872,21 @@ enum _ecore_status_t ecore_spq_post(struct ecore_hwfn *p_hwfn, * access p_ent here to see whether it's successful or not. * Thus, after gaining the answer perform the cleanup here. */ - rc = ecore_spq_block(p_hwfn, p_ent, fw_return_code); + rc = ecore_spq_block(p_hwfn, p_ent, fw_return_code, + p_ent->queue == &p_spq->unlimited_pending); + + if (p_ent->queue == &p_spq->unlimited_pending) { + /* This is an allocated p_ent which does not need to + * return to pool. + */ + OSAL_FREE(p_hwfn->p_dev, p_ent); + + /* TBD: handle error flow and remove p_ent from + * completion pending + */ + return rc; + } + if (rc) goto spq_post_fail2; @@ -884,11 +985,17 @@ enum _ecore_status_t ecore_spq_completion(struct ecore_hwfn *p_hwfn, if (found->comp_cb.function) found->comp_cb.function(p_hwfn, found->comp_cb.cookie, p_data, fw_return_code); + else + DP_VERBOSE(p_hwfn, ECORE_MSG_SPQ, + "Got a completion without a callback function\n"); - if (found->comp_mode != ECORE_SPQ_MODE_EBLOCK) { - /* EBLOCK is responsible for freeing its own entry */ + if ((found->comp_mode != ECORE_SPQ_MODE_EBLOCK) || + (found->queue == &p_spq->unlimited_pending)) + /* EBLOCK is responsible for returning its own entry into the + * free list, unless it originally added the entry into the + * unlimited pending list. + */ ecore_spq_return_entry(p_hwfn, found); - } /* Attempt to post pending requests */ OSAL_SPIN_LOCK(&p_spq->lock); @@ -898,17 +1005,17 @@ enum _ecore_status_t ecore_spq_completion(struct ecore_hwfn *p_hwfn, return rc; } -struct ecore_consq *ecore_consq_alloc(struct ecore_hwfn *p_hwfn) +enum _ecore_status_t ecore_consq_alloc(struct ecore_hwfn *p_hwfn) { struct ecore_consq *p_consq; /* Allocate ConsQ struct */ p_consq = - OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL, sizeof(struct ecore_consq)); + OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL, sizeof(*p_consq)); if (!p_consq) { DP_NOTICE(p_hwfn, true, "Failed to allocate `struct ecore_consq'\n"); - return OSAL_NULL; + return ECORE_NOMEM; } /* Allocate and initialize EQ chain */ @@ -917,27 +1024,30 @@ struct ecore_consq *ecore_consq_alloc(struct ecore_hwfn *p_hwfn) ECORE_CHAIN_MODE_PBL, ECORE_CHAIN_CNT_TYPE_U16, ECORE_CHAIN_PAGE_SIZE / 0x80, - 0x80, &p_consq->chain)) { + 0x80, + &p_consq->chain, OSAL_NULL) != ECORE_SUCCESS) { DP_NOTICE(p_hwfn, true, "Failed to allocate consq chain"); goto consq_allocate_fail; } - return p_consq; + p_hwfn->p_consq = p_consq; + return ECORE_SUCCESS; consq_allocate_fail: - ecore_consq_free(p_hwfn, p_consq); - return OSAL_NULL; -} - -void ecore_consq_setup(struct ecore_hwfn *p_hwfn, struct ecore_consq *p_consq) -{ - ecore_chain_reset(&p_consq->chain); -} - -void ecore_consq_free(struct ecore_hwfn *p_hwfn, struct ecore_consq *p_consq) -{ - if (!p_consq) - return; - ecore_chain_free(p_hwfn->p_dev, &p_consq->chain); OSAL_FREE(p_hwfn->p_dev, p_consq); + return ECORE_NOMEM; +} + +void ecore_consq_setup(struct ecore_hwfn *p_hwfn) +{ + ecore_chain_reset(&p_hwfn->p_consq->chain); +} + +void ecore_consq_free(struct ecore_hwfn *p_hwfn) +{ + if (!p_hwfn->p_consq) + return; + + ecore_chain_free(p_hwfn->p_dev, &p_hwfn->p_consq->chain); + OSAL_FREE(p_hwfn->p_dev, p_hwfn->p_consq); } diff --git a/dpdk/drivers/net/qede/base/ecore_spq.h b/dpdk/drivers/net/qede/base/ecore_spq.h index 5c168654..526cff08 100644 --- a/dpdk/drivers/net/qede/base/ecore_spq.h +++ b/dpdk/drivers/net/qede/base/ecore_spq.h @@ -16,24 +16,26 @@ #include "ecore_sp_api.h" union ramrod_data { - struct pf_start_ramrod_data pf_start; - struct pf_update_ramrod_data pf_update; - struct rx_queue_start_ramrod_data rx_queue_start; - struct rx_queue_update_ramrod_data rx_queue_update; - struct rx_queue_stop_ramrod_data rx_queue_stop; - struct tx_queue_start_ramrod_data tx_queue_start; - struct tx_queue_stop_ramrod_data tx_queue_stop; - struct vport_start_ramrod_data vport_start; - struct vport_stop_ramrod_data vport_stop; - struct vport_update_ramrod_data vport_update; - struct core_rx_start_ramrod_data core_rx_queue_start; - struct core_rx_stop_ramrod_data core_rx_queue_stop; - struct core_tx_start_ramrod_data core_tx_queue_start; - struct core_tx_stop_ramrod_data core_tx_queue_stop; - struct vport_filter_update_ramrod_data vport_filter_update; + struct pf_start_ramrod_data pf_start; + struct pf_update_ramrod_data pf_update; + struct rl_update_ramrod_data rl_update; + struct rx_queue_start_ramrod_data rx_queue_start; + struct rx_queue_update_ramrod_data rx_queue_update; + struct rx_queue_stop_ramrod_data rx_queue_stop; + struct tx_queue_start_ramrod_data tx_queue_start; + struct tx_queue_stop_ramrod_data tx_queue_stop; + struct vport_start_ramrod_data vport_start; + struct vport_stop_ramrod_data vport_stop; + struct rx_update_gft_filter_data rx_update_gft; + struct vport_update_ramrod_data vport_update; + struct core_rx_start_ramrod_data core_rx_queue_start; + struct core_rx_stop_ramrod_data core_rx_queue_stop; + struct core_tx_start_ramrod_data core_tx_queue_start; + struct core_tx_stop_ramrod_data core_tx_queue_stop; + struct vport_filter_update_ramrod_data vport_filter_update; - struct vf_start_ramrod_data vf_start; - struct vf_stop_ramrod_data vf_stop; + struct vf_start_ramrod_data vf_start; + struct vf_stop_ramrod_data vf_stop; }; #define EQ_MAX_CREDIT 0xffffffff @@ -45,83 +47,103 @@ enum spq_priority { union ecore_spq_req_comp { struct ecore_spq_comp_cb cb; - u64 *done_addr; + u64 *done_addr; }; /* SPQ_MODE_EBLOCK */ struct ecore_spq_comp_done { u64 done; - u8 fw_return_code; + u8 fw_return_code; }; struct ecore_spq_entry { - osal_list_entry_t list; + osal_list_entry_t list; - u8 flags; + u8 flags; /* HSI slow path element */ - struct slow_path_element elem; + struct slow_path_element elem; - union ramrod_data ramrod; + union ramrod_data ramrod; - enum spq_priority priority; + enum spq_priority priority; /* pending queue for this entry */ - osal_list_t *queue; + osal_list_t *queue; - enum spq_mode comp_mode; - struct ecore_spq_comp_cb comp_cb; - struct ecore_spq_comp_done comp_done; /* SPQ_MODE_EBLOCK */ + enum spq_mode comp_mode; + struct ecore_spq_comp_cb comp_cb; + struct ecore_spq_comp_done comp_done; /* SPQ_MODE_EBLOCK */ }; struct ecore_eq { - struct ecore_chain chain; - u8 eq_sb_index; /* index within the SB */ - __le16 *p_fw_cons; /* ptr to index value */ + struct ecore_chain chain; + u8 eq_sb_index; /* index within the SB */ + __le16 *p_fw_cons; /* ptr to index value */ }; struct ecore_consq { - struct ecore_chain chain; + struct ecore_chain chain; }; +typedef enum _ecore_status_t +(*ecore_spq_async_comp_cb)(struct ecore_hwfn *p_hwfn, + u8 opcode, + u16 echo, + union event_ring_data *data, + u8 fw_return_code); + +enum _ecore_status_t +ecore_spq_register_async_cb(struct ecore_hwfn *p_hwfn, + enum protocol_type protocol_id, + ecore_spq_async_comp_cb cb); + +void +ecore_spq_unregister_async_cb(struct ecore_hwfn *p_hwfn, + enum protocol_type protocol_id); + struct ecore_spq { - osal_spinlock_t lock; + osal_spinlock_t lock; - osal_list_t unlimited_pending; - osal_list_t pending; - osal_list_t completion_pending; - osal_list_t free_pool; + osal_list_t unlimited_pending; + osal_list_t pending; + osal_list_t completion_pending; + osal_list_t free_pool; - struct ecore_chain chain; + struct ecore_chain chain; /* allocated dma-able memory for spq entries (+ramrod data) */ - dma_addr_t p_phys; - struct ecore_spq_entry *p_virt; + dma_addr_t p_phys; + struct ecore_spq_entry *p_virt; /* Bitmap for handling out-of-order completions */ -#define SPQ_RING_SIZE \ +#define SPQ_RING_SIZE \ (CORE_SPQE_PAGE_SIZE_BYTES / sizeof(struct slow_path_element)) -#define SPQ_COMP_BMAP_SIZE \ -(SPQ_RING_SIZE / (sizeof(unsigned long) * 8 /* BITS_PER_LONG */)) - unsigned long p_comp_bitmap[SPQ_COMP_BMAP_SIZE]; - u8 comp_bitmap_idx; -#define SPQ_COMP_BMAP_SET_BIT(p_spq, idx) \ -(OSAL_SET_BIT(((idx) % SPQ_RING_SIZE), (p_spq)->p_comp_bitmap)) +/* BITS_PER_LONG */ +#define SPQ_COMP_BMAP_SIZE (SPQ_RING_SIZE / (sizeof(unsigned long) * 8)) + unsigned long p_comp_bitmap[SPQ_COMP_BMAP_SIZE]; + u8 comp_bitmap_idx; +#define SPQ_COMP_BMAP_SET_BIT(p_spq, idx) \ + (OSAL_SET_BIT(((idx) % SPQ_RING_SIZE), (p_spq)->p_comp_bitmap)) -#define SPQ_COMP_BMAP_CLEAR_BIT(p_spq, idx) \ -(OSAL_CLEAR_BIT(((idx) % SPQ_RING_SIZE), (p_spq)->p_comp_bitmap)) +#define SPQ_COMP_BMAP_CLEAR_BIT(p_spq, idx) \ + (OSAL_CLEAR_BIT(((idx) % SPQ_RING_SIZE), (p_spq)->p_comp_bitmap)) -#define SPQ_COMP_BMAP_TEST_BIT(p_spq, idx) \ -(OSAL_TEST_BIT(((idx) % SPQ_RING_SIZE), (p_spq)->p_comp_bitmap)) +#define SPQ_COMP_BMAP_TEST_BIT(p_spq, idx) \ + (OSAL_TEST_BIT(((idx) % SPQ_RING_SIZE), (p_spq)->p_comp_bitmap)) /* Statistics */ - u32 unlimited_pending_count; - u32 normal_count; - u32 high_count; - u32 comp_sent_count; - u32 comp_count; + u32 unlimited_pending_count; + u32 normal_count; + u32 high_count; + u32 comp_sent_count; + u32 comp_count; - u32 cid; + u32 cid; + + u32 db_addr_offset; + struct core_db_data db_data; + ecore_spq_async_comp_cb async_comp_cb[MAX_PROTOCOL_TYPE]; }; struct ecore_port; @@ -136,9 +158,9 @@ struct ecore_hwfn; * * @return enum _ecore_status_t */ -enum _ecore_status_t ecore_spq_post(struct ecore_hwfn *p_hwfn, +enum _ecore_status_t ecore_spq_post(struct ecore_hwfn *p_hwfn, struct ecore_spq_entry *p_ent, - u8 *fw_return_code); + u8 *fw_return_code); /** * @brief ecore_spq_allocate - Alloocates & initializes the SPQ and EQ. @@ -147,7 +169,7 @@ enum _ecore_status_t ecore_spq_post(struct ecore_hwfn *p_hwfn, * * @return enum _ecore_status_t */ -enum _ecore_status_t ecore_spq_alloc(struct ecore_hwfn *p_hwfn); +enum _ecore_status_t ecore_spq_alloc(struct ecore_hwfn *p_hwfn); /** * @brief ecore_spq_setup - Reset the SPQ to its start state. @@ -175,7 +197,8 @@ void ecore_spq_free(struct ecore_hwfn *p_hwfn); * @return enum _ecore_status_t */ enum _ecore_status_t -ecore_spq_get_entry(struct ecore_hwfn *p_hwfn, struct ecore_spq_entry **pp_ent); +ecore_spq_get_entry(struct ecore_hwfn *p_hwfn, + struct ecore_spq_entry **pp_ent); /** * @brief ecore_spq_return_entry - Return an entry to spq free @@ -184,33 +207,31 @@ ecore_spq_get_entry(struct ecore_hwfn *p_hwfn, struct ecore_spq_entry **pp_ent); * @param p_hwfn * @param p_ent */ -void ecore_spq_return_entry(struct ecore_hwfn *p_hwfn, - struct ecore_spq_entry *p_ent); +void ecore_spq_return_entry(struct ecore_hwfn *p_hwfn, + struct ecore_spq_entry *p_ent); /** * @brief ecore_eq_allocate - Allocates & initializes an EQ struct * * @param p_hwfn * @param num_elem number of elements in the eq * - * @return struct ecore_eq* - a newly allocated structure; NULL upon error. + * @return enum _ecore_status_t */ -struct ecore_eq *ecore_eq_alloc(struct ecore_hwfn *p_hwfn, u16 num_elem); +enum _ecore_status_t ecore_eq_alloc(struct ecore_hwfn *p_hwfn, u16 num_elem); /** - * @brief ecore_eq_setup - Reset the SPQ to its start state. + * @brief ecore_eq_setup - Reset the EQ to its start state. * * @param p_hwfn - * @param p_eq */ -void ecore_eq_setup(struct ecore_hwfn *p_hwfn, struct ecore_eq *p_eq); +void ecore_eq_setup(struct ecore_hwfn *p_hwfn); /** - * @brief ecore_eq_deallocate - deallocates the given EQ struct. + * @brief ecore_eq_free - deallocates the given EQ struct. * * @param p_hwfn - * @param p_eq */ -void ecore_eq_free(struct ecore_hwfn *p_hwfn, struct ecore_eq *p_eq); +void ecore_eq_free(struct ecore_hwfn *p_hwfn); /** * @brief ecore_eq_prod_update - update the FW with default EQ producer @@ -218,7 +239,8 @@ void ecore_eq_free(struct ecore_hwfn *p_hwfn, struct ecore_eq *p_eq); * @param p_hwfn * @param prod */ -void ecore_eq_prod_update(struct ecore_hwfn *p_hwfn, u16 prod); +void ecore_eq_prod_update(struct ecore_hwfn *p_hwfn, + u16 prod); /** * @brief ecore_eq_completion - Completes currently pending EQ elements @@ -228,8 +250,8 @@ void ecore_eq_prod_update(struct ecore_hwfn *p_hwfn, u16 prod); * * @return enum _ecore_status_t */ -enum _ecore_status_t ecore_eq_completion(struct ecore_hwfn *p_hwfn, - void *cookie); +enum _ecore_status_t ecore_eq_completion(struct ecore_hwfn *p_hwfn, + void *cookie); /** * @brief ecore_spq_completion - Completes a single event @@ -240,10 +262,10 @@ enum _ecore_status_t ecore_eq_completion(struct ecore_hwfn *p_hwfn, * * @return enum _ecore_status_t */ -enum _ecore_status_t ecore_spq_completion(struct ecore_hwfn *p_hwfn, - __le16 echo, - u8 fw_return_code, - union event_ring_data *p_data); +enum _ecore_status_t ecore_spq_completion(struct ecore_hwfn *p_hwfn, + __le16 echo, + u8 fw_return_code, + union event_ring_data *p_data); /** * @brief ecore_spq_get_cid - Given p_hwfn, return cid for the hwfn's SPQ @@ -255,30 +277,26 @@ enum _ecore_status_t ecore_spq_completion(struct ecore_hwfn *p_hwfn, u32 ecore_spq_get_cid(struct ecore_hwfn *p_hwfn); /** - * @brief ecore_consq_alloc - Allocates & initializes an ConsQ - * struct + * @brief ecore_consq_alloc - Allocates & initializes an ConsQ struct * * @param p_hwfn * - * @return struct ecore_eq* - a newly allocated structure; NULL upon error. + * @return enum _ecore_status_t */ -struct ecore_consq *ecore_consq_alloc(struct ecore_hwfn *p_hwfn); +enum _ecore_status_t ecore_consq_alloc(struct ecore_hwfn *p_hwfn); /** - * @brief ecore_consq_setup - Reset the ConsQ to its start - * state. + * @brief ecore_consq_setup - Reset the ConsQ to its start state. * * @param p_hwfn - * @param p_eq */ -void ecore_consq_setup(struct ecore_hwfn *p_hwfn, struct ecore_consq *p_consq); +void ecore_consq_setup(struct ecore_hwfn *p_hwfn); /** * @brief ecore_consq_free - deallocates the given ConsQ struct. * * @param p_hwfn - * @param p_eq */ -void ecore_consq_free(struct ecore_hwfn *p_hwfn, struct ecore_consq *p_consq); +void ecore_consq_free(struct ecore_hwfn *p_hwfn); #endif /* __ECORE_SPQ_H__ */ diff --git a/dpdk/drivers/net/qede/base/ecore_sriov.c b/dpdk/drivers/net/qede/base/ecore_sriov.c index 1b3119d2..b1e26d6f 100644 --- a/dpdk/drivers/net/qede/base/ecore_sriov.c +++ b/dpdk/drivers/net/qede/base/ecore_sriov.c @@ -25,8 +25,14 @@ #include "ecore_cxt.h" #include "ecore_vf.h" #include "ecore_init_fw_funcs.h" +#include "ecore_sp_commands.h" + +static enum _ecore_status_t ecore_sriov_eqe_event(struct ecore_hwfn *p_hwfn, + u8 opcode, + __le16 echo, + union event_ring_data *data, + u8 fw_return_code); -/* TEMPORARY until we implement print_enums... */ const char *ecore_channel_tlvs_string[] = { "CHANNEL_TLV_NONE", /* ends tlv sequence */ "CHANNEL_TLV_ACQUIRE", @@ -51,20 +57,288 @@ const char *ecore_channel_tlvs_string[] = { "CHANNEL_TLV_VPORT_UPDATE_RSS", "CHANNEL_TLV_VPORT_UPDATE_ACCEPT_ANY_VLAN", "CHANNEL_TLV_VPORT_UPDATE_SGE_TPA", + "CHANNEL_TLV_UPDATE_TUNN_PARAM", + "CHANNEL_TLV_COALESCE_UPDATE", + "CHANNEL_TLV_QID", + "CHANNEL_TLV_COALESCE_READ", "CHANNEL_TLV_MAX" }; -/* TODO - this is linux crc32; Need a way to ifdef it out for linux */ -u32 ecore_crc32(u32 crc, u8 *ptr, u32 length) +static u8 ecore_vf_calculate_legacy(struct ecore_vf_info *p_vf) +{ + u8 legacy = 0; + + if (p_vf->acquire.vfdev_info.eth_fp_hsi_minor == + ETH_HSI_VER_NO_PKT_LEN_TUNN) + legacy |= ECORE_QCID_LEGACY_VF_RX_PROD; + + if (!(p_vf->acquire.vfdev_info.capabilities & + VFPF_ACQUIRE_CAP_QUEUE_QIDS)) + legacy |= ECORE_QCID_LEGACY_VF_CID; + + return legacy; +} + +/* IOV ramrods */ +static enum _ecore_status_t ecore_sp_vf_start(struct ecore_hwfn *p_hwfn, + struct ecore_vf_info *p_vf) +{ + struct vf_start_ramrod_data *p_ramrod = OSAL_NULL; + struct ecore_spq_entry *p_ent = OSAL_NULL; + struct ecore_sp_init_data init_data; + enum _ecore_status_t rc = ECORE_NOTIMPL; + u8 fp_minor; + + /* Get SPQ entry */ + OSAL_MEMSET(&init_data, 0, sizeof(init_data)); + init_data.cid = ecore_spq_get_cid(p_hwfn); + init_data.opaque_fid = p_vf->opaque_fid; + init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK; + + rc = ecore_sp_init_request(p_hwfn, &p_ent, + COMMON_RAMROD_VF_START, + PROTOCOLID_COMMON, &init_data); + if (rc != ECORE_SUCCESS) + return rc; + + p_ramrod = &p_ent->ramrod.vf_start; + + p_ramrod->vf_id = GET_FIELD(p_vf->concrete_fid, PXP_CONCRETE_FID_VFID); + p_ramrod->opaque_fid = OSAL_CPU_TO_LE16(p_vf->opaque_fid); + + switch (p_hwfn->hw_info.personality) { + case ECORE_PCI_ETH: + p_ramrod->personality = PERSONALITY_ETH; + break; + case ECORE_PCI_ETH_ROCE: + case ECORE_PCI_ETH_IWARP: + p_ramrod->personality = PERSONALITY_RDMA_AND_ETH; + break; + default: + DP_NOTICE(p_hwfn, true, "Unknown VF personality %d\n", + p_hwfn->hw_info.personality); + return ECORE_INVAL; + } + + fp_minor = p_vf->acquire.vfdev_info.eth_fp_hsi_minor; + if (fp_minor > ETH_HSI_VER_MINOR && + fp_minor != ETH_HSI_VER_NO_PKT_LEN_TUNN) { + DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, + "VF [%d] - Requested fp hsi %02x.%02x which is" + " slightly newer than PF's %02x.%02x; Configuring" + " PFs version\n", + p_vf->abs_vf_id, + ETH_HSI_VER_MAJOR, fp_minor, + ETH_HSI_VER_MAJOR, ETH_HSI_VER_MINOR); + fp_minor = ETH_HSI_VER_MINOR; + } + + p_ramrod->hsi_fp_ver.major_ver_arr[ETH_VER_KEY] = ETH_HSI_VER_MAJOR; + p_ramrod->hsi_fp_ver.minor_ver_arr[ETH_VER_KEY] = fp_minor; + + DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, + "VF[%d] - Starting using HSI %02x.%02x\n", + p_vf->abs_vf_id, ETH_HSI_VER_MAJOR, fp_minor); + + return ecore_spq_post(p_hwfn, p_ent, OSAL_NULL); +} + +static enum _ecore_status_t ecore_sp_vf_stop(struct ecore_hwfn *p_hwfn, + u32 concrete_vfid, + u16 opaque_vfid) +{ + struct vf_stop_ramrod_data *p_ramrod = OSAL_NULL; + struct ecore_spq_entry *p_ent = OSAL_NULL; + struct ecore_sp_init_data init_data; + enum _ecore_status_t rc = ECORE_NOTIMPL; + + /* Get SPQ entry */ + OSAL_MEMSET(&init_data, 0, sizeof(init_data)); + init_data.cid = ecore_spq_get_cid(p_hwfn); + init_data.opaque_fid = opaque_vfid; + init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK; + + rc = ecore_sp_init_request(p_hwfn, &p_ent, + COMMON_RAMROD_VF_STOP, + PROTOCOLID_COMMON, &init_data); + if (rc != ECORE_SUCCESS) + return rc; + + p_ramrod = &p_ent->ramrod.vf_stop; + + p_ramrod->vf_id = GET_FIELD(concrete_vfid, PXP_CONCRETE_FID_VFID); + + return ecore_spq_post(p_hwfn, p_ent, OSAL_NULL); +} + +bool ecore_iov_is_valid_vfid(struct ecore_hwfn *p_hwfn, int rel_vf_id, + bool b_enabled_only, bool b_non_malicious) +{ + if (!p_hwfn->pf_iov_info) { + DP_NOTICE(p_hwfn->p_dev, true, "No iov info\n"); + return false; + } + + if ((rel_vf_id >= p_hwfn->p_dev->p_iov_info->total_vfs) || + (rel_vf_id < 0)) + return false; + + if ((!p_hwfn->pf_iov_info->vfs_array[rel_vf_id].b_init) && + b_enabled_only) + return false; + + if ((p_hwfn->pf_iov_info->vfs_array[rel_vf_id].b_malicious) && + b_non_malicious) + return false; + + return true; +} + +struct ecore_vf_info *ecore_iov_get_vf_info(struct ecore_hwfn *p_hwfn, + u16 relative_vf_id, + bool b_enabled_only) +{ + struct ecore_vf_info *vf = OSAL_NULL; + + if (!p_hwfn->pf_iov_info) { + DP_NOTICE(p_hwfn->p_dev, true, "No iov info\n"); + return OSAL_NULL; + } + + if (ecore_iov_is_valid_vfid(p_hwfn, relative_vf_id, + b_enabled_only, false)) + vf = &p_hwfn->pf_iov_info->vfs_array[relative_vf_id]; + else + DP_ERR(p_hwfn, "ecore_iov_get_vf_info: VF[%d] is not enabled\n", + relative_vf_id); + + return vf; +} + +static struct ecore_queue_cid * +ecore_iov_get_vf_rx_queue_cid(struct ecore_vf_queue *p_queue) { int i; - while (length--) { - crc ^= *ptr++; - for (i = 0; i < 8; i++) - crc = (crc >> 1) ^ ((crc & 1) ? 0xedb88320 : 0); + for (i = 0; i < MAX_QUEUES_PER_QZONE; i++) { + if (p_queue->cids[i].p_cid && + !p_queue->cids[i].b_is_tx) + return p_queue->cids[i].p_cid; } - return crc; + + return OSAL_NULL; +} + +enum ecore_iov_validate_q_mode { + ECORE_IOV_VALIDATE_Q_NA, + ECORE_IOV_VALIDATE_Q_ENABLE, + ECORE_IOV_VALIDATE_Q_DISABLE, +}; + +static bool ecore_iov_validate_queue_mode(struct ecore_vf_info *p_vf, + u16 qid, + enum ecore_iov_validate_q_mode mode, + bool b_is_tx) +{ + int i; + + if (mode == ECORE_IOV_VALIDATE_Q_NA) + return true; + + for (i = 0; i < MAX_QUEUES_PER_QZONE; i++) { + struct ecore_vf_queue_cid *p_qcid; + + p_qcid = &p_vf->vf_queues[qid].cids[i]; + + if (p_qcid->p_cid == OSAL_NULL) + continue; + + if (p_qcid->b_is_tx != b_is_tx) + continue; + + /* Found. It's enabled. */ + return (mode == ECORE_IOV_VALIDATE_Q_ENABLE); + } + + /* In case we haven't found any valid cid, then its disabled */ + return (mode == ECORE_IOV_VALIDATE_Q_DISABLE); +} + +static bool ecore_iov_validate_rxq(struct ecore_hwfn *p_hwfn, + struct ecore_vf_info *p_vf, + u16 rx_qid, + enum ecore_iov_validate_q_mode mode) +{ + if (rx_qid >= p_vf->num_rxqs) { + DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, + "VF[0x%02x] - can't touch Rx queue[%04x];" + " Only 0x%04x are allocated\n", + p_vf->abs_vf_id, rx_qid, p_vf->num_rxqs); + return false; + } + + return ecore_iov_validate_queue_mode(p_vf, rx_qid, mode, false); +} + +static bool ecore_iov_validate_txq(struct ecore_hwfn *p_hwfn, + struct ecore_vf_info *p_vf, + u16 tx_qid, + enum ecore_iov_validate_q_mode mode) +{ + if (tx_qid >= p_vf->num_txqs) { + DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, + "VF[0x%02x] - can't touch Tx queue[%04x];" + " Only 0x%04x are allocated\n", + p_vf->abs_vf_id, tx_qid, p_vf->num_txqs); + return false; + } + + return ecore_iov_validate_queue_mode(p_vf, tx_qid, mode, true); +} + +static bool ecore_iov_validate_sb(struct ecore_hwfn *p_hwfn, + struct ecore_vf_info *p_vf, + u16 sb_idx) +{ + int i; + + for (i = 0; i < p_vf->num_sbs; i++) + if (p_vf->igu_sbs[i] == sb_idx) + return true; + + DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, + "VF[0%02x] - tried using sb_idx %04x which doesn't exist as" + " one of its 0x%02x SBs\n", + p_vf->abs_vf_id, sb_idx, p_vf->num_sbs); + + return false; +} + +/* Is there at least 1 queue open? */ +static bool ecore_iov_validate_active_rxq(struct ecore_vf_info *p_vf) +{ + u8 i; + + for (i = 0; i < p_vf->num_rxqs; i++) + if (ecore_iov_validate_queue_mode(p_vf, i, + ECORE_IOV_VALIDATE_Q_ENABLE, + false)) + return true; + + return false; +} + +static bool ecore_iov_validate_active_txq(struct ecore_vf_info *p_vf) +{ + u8 i; + + for (i = 0; i < p_vf->num_txqs; i++) + if (ecore_iov_validate_queue_mode(p_vf, i, + ECORE_IOV_VALIDATE_Q_ENABLE, + true)) + return true; + + return false; } enum _ecore_status_t ecore_iov_post_vf_bulletin(struct ecore_hwfn *p_hwfn, @@ -72,9 +346,9 @@ enum _ecore_status_t ecore_iov_post_vf_bulletin(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt) { struct ecore_bulletin_content *p_bulletin; + int crc_size = sizeof(p_bulletin->crc); struct ecore_dmae_params params; struct ecore_vf_info *p_vf; - int crc_size = sizeof(p_bulletin->crc); p_vf = ecore_iov_get_vf_info(p_hwfn, (u16)vfid, true); if (!p_vf) @@ -88,8 +362,8 @@ enum _ecore_status_t ecore_iov_post_vf_bulletin(struct ecore_hwfn *p_hwfn, /* Increment bulletin board version and compute crc */ p_bulletin->version++; - p_bulletin->crc = ecore_crc32(0, (u8 *)p_bulletin + crc_size, - p_vf->bulletin.size - crc_size); + p_bulletin->crc = OSAL_CRC32(0, (u8 *)p_bulletin + crc_size, + p_vf->bulletin.size - crc_size); DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, "Posting Bulletin 0x%08x to VF[%d] (CRC 0x%08x)\n", @@ -106,7 +380,7 @@ enum _ecore_status_t ecore_iov_post_vf_bulletin(struct ecore_hwfn *p_hwfn, static enum _ecore_status_t ecore_iov_pci_cfg_info(struct ecore_dev *p_dev) { - struct ecore_hw_sriov_info *iov = &p_dev->sriov_info; + struct ecore_hw_sriov_info *iov = p_dev->p_iov_info; int pos = iov->pos; DP_VERBOSE(p_dev, ECORE_MSG_IOV, "sriov ext pos %d\n", pos); @@ -145,9 +419,9 @@ static enum _ecore_status_t ecore_iov_pci_cfg_info(struct ecore_dev *p_dev) OSAL_PCI_READ_CONFIG_BYTE(p_dev, pos + PCI_SRIOV_FUNC_LINK, &iov->link); - DP_VERBOSE(p_dev, ECORE_MSG_IOV, "IOV info[%d]: nres %d, cap 0x%x," + DP_VERBOSE(p_dev, ECORE_MSG_IOV, "IOV info: nres %d, cap 0x%x," "ctrl 0x%x, total %d, initial %d, num vfs %d, offset %d," - " stride %d, page size 0x%x\n", 0, + " stride %d, page size 0x%x\n", iov->nres, iov->cap, iov->ctrl, iov->total_vfs, iov->initial_vfs, iov->nr_virtfn, iov->offset, iov->stride, iov->pgsz); @@ -171,45 +445,16 @@ static enum _ecore_status_t ecore_iov_pci_cfg_info(struct ecore_dev *p_dev) return ECORE_SUCCESS; } -static void ecore_iov_clear_vf_igu_blocks(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt) -{ - struct ecore_igu_block *p_sb; - u16 sb_id; - u32 val; - - if (!p_hwfn->hw_info.p_igu_info) { - DP_ERR(p_hwfn, - "ecore_iov_clear_vf_igu_blocks IGU Info not inited\n"); - return; - } - - for (sb_id = 0; - sb_id < ECORE_MAPPING_MEMORY_SIZE(p_hwfn->p_dev); sb_id++) { - p_sb = &p_hwfn->hw_info.p_igu_info->igu_map.igu_blocks[sb_id]; - if ((p_sb->status & ECORE_IGU_STATUS_FREE) && - !(p_sb->status & ECORE_IGU_STATUS_PF)) { - val = ecore_rd(p_hwfn, p_ptt, - IGU_REG_MAPPING_MEMORY + sb_id * 4); - SET_FIELD(val, IGU_MAPPING_LINE_VALID, 0); - ecore_wr(p_hwfn, p_ptt, - IGU_REG_MAPPING_MEMORY + 4 * sb_id, val); - } - } -} - static void ecore_iov_setup_vfdb(struct ecore_hwfn *p_hwfn) { - u16 num_vfs = p_hwfn->p_dev->sriov_info.total_vfs; + struct ecore_hw_sriov_info *p_iov = p_hwfn->p_dev->p_iov_info; + struct ecore_pf_iov *p_iov_info = p_hwfn->pf_iov_info; + struct ecore_bulletin_content *p_bulletin_virt; + dma_addr_t req_p, rply_p, bulletin_p; union pfvf_tlvs *p_reply_virt_addr; union vfpf_tlvs *p_req_virt_addr; - struct ecore_bulletin_content *p_bulletin_virt; - struct ecore_pf_iov *p_iov_info; - dma_addr_t req_p, rply_p, bulletin_p; u8 idx = 0; - p_iov_info = p_hwfn->pf_iov_info; - OSAL_MEMSET(p_iov_info->vfs_array, 0, sizeof(p_iov_info->vfs_array)); p_req_virt_addr = p_iov_info->mbx_msg_virt_addr; @@ -224,9 +469,7 @@ static void ecore_iov_setup_vfdb(struct ecore_hwfn *p_hwfn) return; } - p_iov_info->base_vport_id = 1; /* @@@TBD resource allocation */ - - for (idx = 0; idx < num_vfs; idx++) { + for (idx = 0; idx < p_iov->total_vfs; idx++) { struct ecore_vf_info *vf = &p_iov_info->vfs_array[idx]; u32 concrete; @@ -240,6 +483,7 @@ static void ecore_iov_setup_vfdb(struct ecore_hwfn *p_hwfn) vf->vf_mbx.sw_mbx.mbx_state = VF_PF_WAIT_FOR_START_REQUEST; #endif vf->state = VF_STOPPED; + vf->b_init = false; vf->bulletin.phys = idx * sizeof(struct ecore_bulletin_content) + bulletin_p; @@ -247,14 +491,15 @@ static void ecore_iov_setup_vfdb(struct ecore_hwfn *p_hwfn) vf->bulletin.size = sizeof(struct ecore_bulletin_content); vf->relative_vf_id = idx; - vf->abs_vf_id = idx + p_hwfn->hw_info.first_vf_in_pf; + vf->abs_vf_id = idx + p_iov->first_vf_in_pf; concrete = ecore_vfid_to_concrete(p_hwfn, vf->abs_vf_id); vf->concrete_fid = concrete; /* TODO - need to devise a better way of getting opaque */ vf->opaque_fid = (p_hwfn->hw_info.opaque_fid & 0xff) | (vf->abs_vf_id << 8); - /* @@TBD MichalK - add base vport_id of VFs to equation */ - vf->vport_id = p_iov_info->base_vport_id + idx; + + vf->num_mac_filters = ECORE_ETH_VF_NUM_MAC_FILTERS; + vf->num_vlan_filters = ECORE_ETH_VF_NUM_VLAN_FILTERS; } } @@ -264,7 +509,7 @@ static enum _ecore_status_t ecore_iov_allocate_vfdb(struct ecore_hwfn *p_hwfn) void **p_v_addr; u16 num_vfs = 0; - num_vfs = p_hwfn->p_dev->sriov_info.total_vfs; + num_vfs = p_hwfn->p_dev->p_iov_info->total_vfs; DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, "ecore_iov_allocate_vfdb for %d VFs\n", num_vfs); @@ -297,16 +542,15 @@ static enum _ecore_status_t ecore_iov_allocate_vfdb(struct ecore_hwfn *p_hwfn) return ECORE_NOMEM; DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, - "PF's Requests mailbox [%p virt 0x%" PRIx64 " phys], " - "Response mailbox [%p virt 0x%" PRIx64 " phys] Bulletins" - " [%p virt 0x%" PRIx64 " phys]\n", + "PF's Requests mailbox [%p virt 0x%lx phys], " + "Response mailbox [%p virt 0x%lx phys] Bulletinsi" + " [%p virt 0x%lx phys]\n", p_iov_info->mbx_msg_virt_addr, - (u64)p_iov_info->mbx_msg_phys_addr, + (unsigned long)p_iov_info->mbx_msg_phys_addr, p_iov_info->mbx_reply_virt_addr, - (u64)p_iov_info->mbx_reply_phys_addr, - p_iov_info->p_bulletins, (u64)p_iov_info->bulletins_phys); - - /* @@@TBD MichalK - statistics / RSS */ + (unsigned long)p_iov_info->mbx_reply_phys_addr, + p_iov_info->p_bulletins, + (unsigned long)p_iov_info->bulletins_phys); return ECORE_SUCCESS; } @@ -332,141 +576,185 @@ static void ecore_iov_free_vfdb(struct ecore_hwfn *p_hwfn) p_iov_info->p_bulletins, p_iov_info->bulletins_phys, p_iov_info->bulletins_size); - - /* @@@TBD MichalK - statistics / RSS */ } enum _ecore_status_t ecore_iov_alloc(struct ecore_hwfn *p_hwfn) { - enum _ecore_status_t rc = ECORE_SUCCESS; struct ecore_pf_iov *p_sriov; if (!IS_PF_SRIOV(p_hwfn)) { DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, "No SR-IOV - no need for IOV db\n"); - return rc; + return ECORE_SUCCESS; } p_sriov = OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL, sizeof(*p_sriov)); if (!p_sriov) { DP_NOTICE(p_hwfn, true, - "Failed to allocate `struct ecore_sriov'"); + "Failed to allocate `struct ecore_sriov'\n"); return ECORE_NOMEM; } p_hwfn->pf_iov_info = p_sriov; - rc = ecore_iov_allocate_vfdb(p_hwfn); + ecore_spq_register_async_cb(p_hwfn, PROTOCOLID_COMMON, + ecore_sriov_eqe_event); - return rc; + return ecore_iov_allocate_vfdb(p_hwfn); } -void ecore_iov_setup(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt) +void ecore_iov_setup(struct ecore_hwfn *p_hwfn) { - if (!IS_PF_SRIOV(p_hwfn) || !p_hwfn->pf_iov_info) + if (!IS_PF_SRIOV(p_hwfn) || !IS_PF_SRIOV_ALLOC(p_hwfn)) return; ecore_iov_setup_vfdb(p_hwfn); - ecore_iov_clear_vf_igu_blocks(p_hwfn, p_ptt); } void ecore_iov_free(struct ecore_hwfn *p_hwfn) { - if (p_hwfn->pf_iov_info) { + ecore_spq_unregister_async_cb(p_hwfn, PROTOCOLID_COMMON); + + if (IS_PF_SRIOV_ALLOC(p_hwfn)) { ecore_iov_free_vfdb(p_hwfn); OSAL_FREE(p_hwfn->p_dev, p_hwfn->pf_iov_info); } } -enum _ecore_status_t ecore_iov_hw_info(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt) +void ecore_iov_free_hw_info(struct ecore_dev *p_dev) { + OSAL_FREE(p_dev, p_dev->p_iov_info); +} + +enum _ecore_status_t ecore_iov_hw_info(struct ecore_hwfn *p_hwfn) +{ + struct ecore_dev *p_dev = p_hwfn->p_dev; + int pos; enum _ecore_status_t rc; - /* @@@ TBD get this information from shmem / pci cfg */ if (IS_VF(p_hwfn->p_dev)) return ECORE_SUCCESS; - /* First hwfn should learn the PCI configuration */ - if (IS_LEAD_HWFN(p_hwfn)) { - struct ecore_dev *p_dev = p_hwfn->p_dev; - int *pos = &p_hwfn->p_dev->sriov_info.pos; - - *pos = OSAL_PCI_FIND_EXT_CAPABILITY(p_hwfn->p_dev, - PCI_EXT_CAP_ID_SRIOV); - if (!*pos) { - DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, - "No PCIe IOV support\n"); - return ECORE_SUCCESS; - } - - rc = ecore_iov_pci_cfg_info(p_dev); - if (rc) - return rc; - } else if (!p_hwfn->p_dev->sriov_info.pos) { + /* Learn the PCI configuration */ + pos = OSAL_PCI_FIND_EXT_CAPABILITY(p_hwfn->p_dev, + PCI_EXT_CAP_ID_SRIOV); + if (!pos) { DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, "No PCIe IOV support\n"); return ECORE_SUCCESS; } - /* Calculate the first VF index - this is a bit tricky; Basically, - * VFs start at offset 16 relative to PF0, and 2nd engine VFs begin - * after the first engine's VFs. + /* Allocate a new struct for IOV information */ + /* TODO - can change to VALLOC when its available */ + p_dev->p_iov_info = OSAL_ZALLOC(p_dev, GFP_KERNEL, + sizeof(*p_dev->p_iov_info)); + if (!p_dev->p_iov_info) { + DP_NOTICE(p_hwfn, true, + "Can't support IOV due to lack of memory\n"); + return ECORE_NOMEM; + } + p_dev->p_iov_info->pos = pos; + + rc = ecore_iov_pci_cfg_info(p_dev); + if (rc) + return rc; + + /* We want PF IOV to be synonemous with the existence of p_iov_info; + * In case the capability is published but there are no VFs, simply + * de-allocate the struct. */ - p_hwfn->hw_info.first_vf_in_pf = p_hwfn->p_dev->sriov_info.offset + - p_hwfn->abs_pf_id - 16; - if (ECORE_PATH_ID(p_hwfn)) - p_hwfn->hw_info.first_vf_in_pf -= MAX_NUM_VFS_BB; + if (!p_dev->p_iov_info->total_vfs) { + DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, + "IOV capabilities, but no VFs are published\n"); + OSAL_FREE(p_dev, p_dev->p_iov_info); + return ECORE_SUCCESS; + } + + /* First VF index based on offset is tricky: + * - If ARI is supported [likely], offset - (16 - pf_id) would + * provide the number for eng0. 2nd engine Vfs would begin + * after the first engine's VFs. + * - If !ARI, VFs would start on next device. + * so offset - (256 - pf_id) would provide the number. + * Utilize the fact that (256 - pf_id) is achieved only be later + * to diffrentiate between the two. + */ + + if (p_hwfn->p_dev->p_iov_info->offset < (256 - p_hwfn->abs_pf_id)) { + u32 first = p_hwfn->p_dev->p_iov_info->offset + + p_hwfn->abs_pf_id - 16; + + p_dev->p_iov_info->first_vf_in_pf = first; + + if (ECORE_PATH_ID(p_hwfn)) + p_dev->p_iov_info->first_vf_in_pf -= MAX_NUM_VFS_BB; + } else { + u32 first = p_hwfn->p_dev->p_iov_info->offset + + p_hwfn->abs_pf_id - 256; + + p_dev->p_iov_info->first_vf_in_pf = first; + } DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, - "First VF in hwfn 0x%08x\n", p_hwfn->hw_info.first_vf_in_pf); + "First VF in hwfn 0x%08x\n", + p_dev->p_iov_info->first_vf_in_pf); return ECORE_SUCCESS; } -struct ecore_vf_info *ecore_iov_get_vf_info(struct ecore_hwfn *p_hwfn, - u16 relative_vf_id, - bool b_enabled_only) +static bool _ecore_iov_pf_sanity_check(struct ecore_hwfn *p_hwfn, int vfid, + bool b_fail_malicious) { - struct ecore_vf_info *vf = OSAL_NULL; + /* Check PF supports sriov */ + if (IS_VF(p_hwfn->p_dev) || !IS_ECORE_SRIOV(p_hwfn->p_dev) || + !IS_PF_SRIOV_ALLOC(p_hwfn)) + return false; - if (!p_hwfn->pf_iov_info) { - DP_NOTICE(p_hwfn->p_dev, true, "No iov info\n"); - return OSAL_NULL; - } + /* Check VF validity */ + if (!ecore_iov_is_valid_vfid(p_hwfn, vfid, true, b_fail_malicious)) + return false; - if (ecore_iov_is_valid_vfid(p_hwfn, relative_vf_id, b_enabled_only)) - vf = &p_hwfn->pf_iov_info->vfs_array[relative_vf_id]; - else - DP_ERR(p_hwfn, "ecore_iov_get_vf_info: VF[%d] is not enabled\n", - relative_vf_id); - - return vf; + return true; } -void ecore_iov_set_vf_to_disable(struct ecore_hwfn *p_hwfn, +bool ecore_iov_pf_sanity_check(struct ecore_hwfn *p_hwfn, int vfid) +{ + return _ecore_iov_pf_sanity_check(p_hwfn, vfid, true); +} + +void ecore_iov_set_vf_to_disable(struct ecore_dev *p_dev, u16 rel_vf_id, u8 to_disable) { struct ecore_vf_info *vf; + int i; - vf = ecore_iov_get_vf_info(p_hwfn, rel_vf_id, false); - if (!vf) - return; + for_each_hwfn(p_dev, i) { + struct ecore_hwfn *p_hwfn = &p_dev->hwfns[i]; - vf->to_disable = to_disable; + vf = ecore_iov_get_vf_info(p_hwfn, rel_vf_id, false); + if (!vf) + continue; + + vf->to_disable = to_disable; + } } -void ecore_iov_set_vfs_to_disable(struct ecore_hwfn *p_hwfn, u8 to_disable) +void ecore_iov_set_vfs_to_disable(struct ecore_dev *p_dev, + u8 to_disable) { u16 i; - for (i = 0; i < p_hwfn->p_dev->sriov_info.total_vfs; i++) - ecore_iov_set_vf_to_disable(p_hwfn, i, to_disable); + if (!IS_ECORE_SRIOV(p_dev)) + return; + + for (i = 0; i < p_dev->p_iov_info->total_vfs; i++) + ecore_iov_set_vf_to_disable(p_dev, i, to_disable); } #ifndef LINUX_REMOVE /* @@@TBD Consider taking outside of ecore... */ enum _ecore_status_t ecore_iov_set_vf_ctx(struct ecore_hwfn *p_hwfn, - u16 vf_id, void *ctx) + u16 vf_id, + void *ctx) { enum _ecore_status_t rc = ECORE_SUCCESS; struct ecore_vf_info *vf = ecore_iov_get_vf_info(p_hwfn, vf_id, true); @@ -483,29 +771,9 @@ enum _ecore_status_t ecore_iov_set_vf_ctx(struct ecore_hwfn *p_hwfn, } #endif -/** - * VF enable primitives - * - * when pretend is required the caller is reponsible - * for calling pretend prioir to calling these routines - */ - -/* clears vf error in all semi blocks - * Assumption: called under VF pretend... - */ -static OSAL_INLINE void ecore_iov_vf_semi_clear_err(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt) -{ - ecore_wr(p_hwfn, p_ptt, TSEM_REG_VF_ERROR, 1); - ecore_wr(p_hwfn, p_ptt, USEM_REG_VF_ERROR, 1); - ecore_wr(p_hwfn, p_ptt, MSEM_REG_VF_ERROR, 1); - ecore_wr(p_hwfn, p_ptt, XSEM_REG_VF_ERROR, 1); - ecore_wr(p_hwfn, p_ptt, YSEM_REG_VF_ERROR, 1); - ecore_wr(p_hwfn, p_ptt, PSEM_REG_VF_ERROR, 1); -} - -static void ecore_iov_vf_pglue_clear_err(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, u8 abs_vfid) +static void ecore_iov_vf_pglue_clear_err(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + u8 abs_vfid) { ecore_wr(p_hwfn, p_ptt, PGLUE_B_REG_WAS_ERROR_VF_31_0_CLR + (abs_vfid >> 5) * 4, @@ -517,30 +785,20 @@ static void ecore_iov_vf_igu_reset(struct ecore_hwfn *p_hwfn, struct ecore_vf_info *vf) { int i; - u16 igu_sb_id; /* Set VF masks and configuration - pretend */ ecore_fid_pretend(p_hwfn, p_ptt, (u16)vf->concrete_fid); ecore_wr(p_hwfn, p_ptt, IGU_REG_STATISTIC_NUM_VF_MSG_SENT, 0); - DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, - "value in VF_CONFIGURATION of vf %d after write %x\n", - vf->abs_vf_id, - ecore_rd(p_hwfn, p_ptt, IGU_REG_VF_CONFIGURATION)); - /* unpretend */ ecore_fid_pretend(p_hwfn, p_ptt, (u16)p_hwfn->hw_info.concrete_fid); - /* iterate ove all queues, clear sb consumer */ - for (i = 0; i < vf->num_sbs; i++) { - igu_sb_id = vf->igu_sbs[i]; - /* Set then clear... */ - ecore_int_igu_cleanup_sb(p_hwfn, p_ptt, igu_sb_id, 1, - vf->opaque_fid); - ecore_int_igu_cleanup_sb(p_hwfn, p_ptt, igu_sb_id, 0, - vf->opaque_fid); - } + /* iterate over all queues, clear sb consumer */ + for (i = 0; i < vf->num_sbs; i++) + ecore_int_igu_init_pure_rt_single(p_hwfn, p_ptt, + vf->igu_sbs[i], + vf->opaque_fid, true); } static void ecore_iov_vf_igu_set_int(struct ecore_hwfn *p_hwfn, @@ -564,12 +822,53 @@ static void ecore_iov_vf_igu_set_int(struct ecore_hwfn *p_hwfn, ecore_fid_pretend(p_hwfn, p_ptt, (u16)p_hwfn->hw_info.concrete_fid); } +static enum _ecore_status_t +ecore_iov_enable_vf_access_msix(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + u8 abs_vf_id, + u8 num_sbs) +{ + u8 current_max = 0; + int i; + + /* If client overrides this, don't do anything */ + if (p_hwfn->p_dev->b_dont_override_vf_msix) + return ECORE_SUCCESS; + + /* For AH onward, configuration is per-PF. Find maximum of all + * the currently enabled child VFs, and set the number to be that. + */ + if (!ECORE_IS_BB(p_hwfn->p_dev)) { + ecore_for_each_vf(p_hwfn, i) { + struct ecore_vf_info *p_vf; + + p_vf = ecore_iov_get_vf_info(p_hwfn, (u16)i, true); + if (!p_vf) + continue; + + current_max = OSAL_MAX_T(u8, current_max, + p_vf->num_sbs); + } + } + + if (num_sbs > current_max) + return ecore_mcp_config_vf_msix(p_hwfn, p_ptt, + abs_vf_id, num_sbs); + + return ECORE_SUCCESS; +} + static enum _ecore_status_t ecore_iov_enable_vf_access(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, struct ecore_vf_info *vf) { u32 igu_vf_conf = IGU_VF_CONF_FUNC_EN; - enum _ecore_status_t rc; + enum _ecore_status_t rc = ECORE_SUCCESS; + + /* It's possible VF was previously considered malicious - + * clear the indication even if we're only going to disable VF. + */ + vf->b_malicious = false; if (vf->to_disable) return ECORE_SUCCESS; @@ -581,9 +880,11 @@ ecore_iov_enable_vf_access(struct ecore_hwfn *p_hwfn, ecore_iov_vf_pglue_clear_err(p_hwfn, p_ptt, ECORE_VF_ABS_ID(p_hwfn, vf)); - rc = ecore_mcp_config_vf_msix(p_hwfn, p_ptt, - vf->abs_vf_id, vf->num_sbs); - if (rc) + ecore_iov_vf_igu_reset(p_hwfn, p_ptt, vf); + + rc = ecore_iov_enable_vf_access_msix(p_hwfn, p_ptt, + vf->abs_vf_id, vf->num_sbs); + if (rc != ECORE_SUCCESS) return rc; ecore_fid_pretend(p_hwfn, p_ptt, (u16)vf->concrete_fid); @@ -597,18 +898,6 @@ ecore_iov_enable_vf_access(struct ecore_hwfn *p_hwfn, /* unpretend */ ecore_fid_pretend(p_hwfn, p_ptt, (u16)p_hwfn->hw_info.concrete_fid); - if (vf->state != VF_STOPPED) { - DP_NOTICE(p_hwfn, true, "VF[%02x] is already started\n", - vf->abs_vf_id); - return ECORE_INVAL; - } - - /* Start VF */ - rc = ecore_sp_vf_start(p_hwfn, vf->concrete_fid, vf->opaque_fid); - if (rc != ECORE_SUCCESS) - DP_NOTICE(p_hwfn, true, "Failed to start VF[%02x]\n", - vf->abs_vf_id); - vf->state = VF_FREE; return rc; @@ -631,8 +920,7 @@ static void ecore_iov_config_perm_table(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, struct ecore_vf_info *vf, u8 enable) { - u32 reg_addr; - u32 val; + u32 reg_addr, val; u16 qzone_id = 0; int qid; @@ -650,13 +938,13 @@ static void ecore_iov_enable_vf_traffic(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, struct ecore_vf_info *vf) { - /* Reset vf in IGU interrupts are still disabled */ + /* Reset vf in IGU - interrupts are still disabled */ ecore_iov_vf_igu_reset(p_hwfn, p_ptt, vf); - ecore_iov_vf_igu_set_int(p_hwfn, p_ptt, vf, 1 /* enable */); + ecore_iov_vf_igu_set_int(p_hwfn, p_ptt, vf, 1); /* Permission Table */ - ecore_iov_config_perm_table(p_hwfn, p_ptt, vf, true /* enable */); + ecore_iov_config_perm_table(p_hwfn, p_ptt, vf, true); } static u8 ecore_iov_alloc_vf_igu_sbs(struct ecore_hwfn *p_hwfn, @@ -664,46 +952,38 @@ static u8 ecore_iov_alloc_vf_igu_sbs(struct ecore_hwfn *p_hwfn, struct ecore_vf_info *vf, u16 num_rx_queues) { - int igu_id = 0; + struct ecore_igu_block *p_block; + struct cau_sb_entry sb_entry; int qid = 0; u32 val = 0; - struct ecore_igu_block *igu_blocks = - p_hwfn->hw_info.p_igu_info->igu_map.igu_blocks; - if (num_rx_queues > p_hwfn->hw_info.p_igu_info->free_blks) - num_rx_queues = p_hwfn->hw_info.p_igu_info->free_blks; - - p_hwfn->hw_info.p_igu_info->free_blks -= num_rx_queues; + if (num_rx_queues > p_hwfn->hw_info.p_igu_info->usage.free_cnt_iov) + num_rx_queues = + (u16)p_hwfn->hw_info.p_igu_info->usage.free_cnt_iov; + p_hwfn->hw_info.p_igu_info->usage.free_cnt_iov -= num_rx_queues; SET_FIELD(val, IGU_MAPPING_LINE_FUNCTION_NUMBER, vf->abs_vf_id); SET_FIELD(val, IGU_MAPPING_LINE_VALID, 1); SET_FIELD(val, IGU_MAPPING_LINE_PF_VALID, 0); - while ((qid < num_rx_queues) && - (igu_id < ECORE_MAPPING_MEMORY_SIZE(p_hwfn->p_dev))) { - if (igu_blocks[igu_id].status & ECORE_IGU_STATUS_FREE) { - struct cau_sb_entry sb_entry; + for (qid = 0; qid < num_rx_queues; qid++) { + p_block = ecore_get_igu_free_sb(p_hwfn, false); + vf->igu_sbs[qid] = p_block->igu_sb_id; + p_block->status &= ~ECORE_IGU_STATUS_FREE; + SET_FIELD(val, IGU_MAPPING_LINE_VECTOR_NUMBER, qid); - vf->igu_sbs[qid] = (u16)igu_id; - igu_blocks[igu_id].status &= ~ECORE_IGU_STATUS_FREE; + ecore_wr(p_hwfn, p_ptt, + IGU_REG_MAPPING_MEMORY + + sizeof(u32) * p_block->igu_sb_id, val); - SET_FIELD(val, IGU_MAPPING_LINE_VECTOR_NUMBER, qid); - - ecore_wr(p_hwfn, p_ptt, - IGU_REG_MAPPING_MEMORY + sizeof(u32) * igu_id, - val); - - /* Configure igu sb in CAU which were marked valid */ - ecore_init_cau_sb_entry(p_hwfn, &sb_entry, - p_hwfn->rel_pf_id, - vf->abs_vf_id, 1); - ecore_dmae_host2grc(p_hwfn, p_ptt, - (u64)(osal_uintptr_t)&sb_entry, - CAU_REG_SB_VAR_MEMORY + - igu_id * sizeof(u64), 2, 0); - qid++; - } - igu_id++; + /* Configure igu sb in CAU which were marked valid */ + ecore_init_cau_sb_entry(p_hwfn, &sb_entry, + p_hwfn->rel_pf_id, + vf->abs_vf_id, 1); + ecore_dmae_host2grc(p_hwfn, p_ptt, + (u64)(osal_uintptr_t)&sb_entry, + CAU_REG_SB_VAR_MEMORY + + p_block->igu_sb_id * sizeof(u64), 2, 0); } vf->num_sbs = (u8)num_rx_queues; @@ -739,49 +1019,141 @@ static void ecore_iov_free_vf_igu_sbs(struct ecore_hwfn *p_hwfn, SET_FIELD(val, IGU_MAPPING_LINE_VALID, 0); ecore_wr(p_hwfn, p_ptt, addr, val); - p_info->igu_map.igu_blocks[igu_id].status |= - ECORE_IGU_STATUS_FREE; - - p_hwfn->hw_info.p_igu_info->free_blks++; + p_info->entry[igu_id].status |= ECORE_IGU_STATUS_FREE; + p_hwfn->hw_info.p_igu_info->usage.free_cnt_iov++; } vf->num_sbs = 0; } -enum _ecore_status_t ecore_iov_init_hw_for_vf(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, - u16 rel_vf_id, u16 num_rx_queues) +void ecore_iov_set_link(struct ecore_hwfn *p_hwfn, + u16 vfid, + struct ecore_mcp_link_params *params, + struct ecore_mcp_link_state *link, + struct ecore_mcp_link_capabilities *p_caps) { - enum _ecore_status_t rc = ECORE_SUCCESS; + struct ecore_vf_info *p_vf = ecore_iov_get_vf_info(p_hwfn, vfid, false); + struct ecore_bulletin_content *p_bulletin; + + if (!p_vf) + return; + + p_bulletin = p_vf->bulletin.p_virt; + p_bulletin->req_autoneg = params->speed.autoneg; + p_bulletin->req_adv_speed = params->speed.advertised_speeds; + p_bulletin->req_forced_speed = params->speed.forced_speed; + p_bulletin->req_autoneg_pause = params->pause.autoneg; + p_bulletin->req_forced_rx = params->pause.forced_rx; + p_bulletin->req_forced_tx = params->pause.forced_tx; + p_bulletin->req_loopback = params->loopback_mode; + + p_bulletin->link_up = link->link_up; + p_bulletin->speed = link->speed; + p_bulletin->full_duplex = link->full_duplex; + p_bulletin->autoneg = link->an; + p_bulletin->autoneg_complete = link->an_complete; + p_bulletin->parallel_detection = link->parallel_detection; + p_bulletin->pfc_enabled = link->pfc_enabled; + p_bulletin->partner_adv_speed = link->partner_adv_speed; + p_bulletin->partner_tx_flow_ctrl_en = link->partner_tx_flow_ctrl_en; + p_bulletin->partner_rx_flow_ctrl_en = link->partner_rx_flow_ctrl_en; + p_bulletin->partner_adv_pause = link->partner_adv_pause; + p_bulletin->sfp_tx_fault = link->sfp_tx_fault; + + p_bulletin->capability_speed = p_caps->speed_capabilities; +} + +enum _ecore_status_t +ecore_iov_init_hw_for_vf(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + struct ecore_iov_vf_init_params *p_params) +{ + struct ecore_mcp_link_capabilities link_caps; + struct ecore_mcp_link_params link_params; + struct ecore_mcp_link_state link_state; + u8 num_of_vf_available_chains = 0; struct ecore_vf_info *vf = OSAL_NULL; - u8 num_of_vf_available_chains = 0; + u16 qid, num_irqs; + enum _ecore_status_t rc = ECORE_SUCCESS; u32 cids; u8 i; - if (ECORE_IS_VF_ACTIVE(p_hwfn->p_dev, rel_vf_id)) { - DP_NOTICE(p_hwfn, true, "VF[%d] is already active.\n", - rel_vf_id); - return ECORE_INVAL; - } - - vf = ecore_iov_get_vf_info(p_hwfn, rel_vf_id, false); + vf = ecore_iov_get_vf_info(p_hwfn, p_params->rel_vf_id, false); if (!vf) { DP_ERR(p_hwfn, "ecore_iov_init_hw_for_vf : vf is OSAL_NULL\n"); return ECORE_UNKNOWN_ERROR; } + if (vf->b_init) { + DP_NOTICE(p_hwfn, true, "VF[%d] is already active.\n", + p_params->rel_vf_id); + return ECORE_INVAL; + } + + /* Perform sanity checking on the requested vport/rss */ + if (p_params->vport_id >= RESC_NUM(p_hwfn, ECORE_VPORT)) { + DP_NOTICE(p_hwfn, true, "VF[%d] - can't use VPORT %02x\n", + p_params->rel_vf_id, p_params->vport_id); + return ECORE_INVAL; + } + + if ((p_params->num_queues > 1) && + (p_params->rss_eng_id >= RESC_NUM(p_hwfn, ECORE_RSS_ENG))) { + DP_NOTICE(p_hwfn, true, "VF[%d] - can't use RSS_ENG %02x\n", + p_params->rel_vf_id, p_params->rss_eng_id); + return ECORE_INVAL; + } + + /* TODO - remove this once we get confidence of change */ + if (!p_params->vport_id) { + DP_NOTICE(p_hwfn, false, + "VF[%d] - Unlikely that VF uses vport0. Forgotten?\n", + p_params->rel_vf_id); + } + if ((!p_params->rss_eng_id) && (p_params->num_queues > 1)) { + DP_NOTICE(p_hwfn, false, + "VF[%d] - Unlikely that VF uses RSS_eng0. Forgotten?\n", + p_params->rel_vf_id); + } + vf->vport_id = p_params->vport_id; + vf->rss_eng_id = p_params->rss_eng_id; + + /* Since it's possible to relocate SBs, it's a bit difficult to check + * things here. Simply check whether the index falls in the range + * belonging to the PF. + */ + for (i = 0; i < p_params->num_queues; i++) { + qid = p_params->req_rx_queue[i]; + if (qid > (u16)RESC_NUM(p_hwfn, ECORE_L2_QUEUE)) { + DP_NOTICE(p_hwfn, true, + "Can't enable Rx qid [%04x] for VF[%d]: qids [0,,...,0x%04x] available\n", + qid, p_params->rel_vf_id, + (u16)RESC_NUM(p_hwfn, ECORE_L2_QUEUE)); + return ECORE_INVAL; + } + + qid = p_params->req_tx_queue[i]; + if (qid > (u16)RESC_NUM(p_hwfn, ECORE_L2_QUEUE)) { + DP_NOTICE(p_hwfn, true, + "Can't enable Tx qid [%04x] for VF[%d]: qids [0,,...,0x%04x] available\n", + qid, p_params->rel_vf_id, + (u16)RESC_NUM(p_hwfn, ECORE_L2_QUEUE)); + return ECORE_INVAL; + } + } + /* Limit number of queues according to number of CIDs */ ecore_cxt_get_proto_cid_count(p_hwfn, PROTOCOLID_ETH, &cids); DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, "VF[%d] - requesting to initialize for 0x%04x queues" " [0x%04x CIDs available]\n", - vf->relative_vf_id, num_rx_queues, (u16)cids); - num_rx_queues = OSAL_MIN_T(u16, num_rx_queues, ((u16)cids)); + vf->relative_vf_id, p_params->num_queues, (u16)cids); + num_irqs = OSAL_MIN_T(u16, p_params->num_queues, ((u16)cids)); num_of_vf_available_chains = ecore_iov_alloc_vf_igu_sbs(p_hwfn, p_ptt, vf, - num_rx_queues); + num_irqs); if (num_of_vf_available_chains == 0) { DP_ERR(p_hwfn, "no available igu sbs\n"); return ECORE_NOMEM; @@ -792,36 +1164,37 @@ enum _ecore_status_t ecore_iov_init_hw_for_vf(struct ecore_hwfn *p_hwfn, vf->num_txqs = num_of_vf_available_chains; for (i = 0; i < vf->num_rxqs; i++) { - u16 queue_id = ecore_int_queue_id_from_sb_id(p_hwfn, - vf->igu_sbs[i]); + struct ecore_vf_queue *p_queue = &vf->vf_queues[i]; - if (queue_id > RESC_NUM(p_hwfn, ECORE_L2_QUEUE)) { - DP_NOTICE(p_hwfn, true, - "VF[%d] will require utilizing of" - " out-of-bounds queues - %04x\n", - vf->relative_vf_id, queue_id); - /* TODO - cleanup the already allocate SBs */ - return ECORE_INVAL; - } - - /* CIDs are per-VF, so no problem having them 0-based. */ - vf->vf_queues[i].fw_rx_qid = queue_id; - vf->vf_queues[i].fw_tx_qid = queue_id; - vf->vf_queues[i].fw_cid = i; + p_queue->fw_rx_qid = p_params->req_rx_queue[i]; + p_queue->fw_tx_qid = p_params->req_tx_queue[i]; DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, - "VF[%d] - [%d] SB %04x, Tx/Rx queue %04x CID %04x\n", - vf->relative_vf_id, i, vf->igu_sbs[i], queue_id, i); + "VF[%d] - Q[%d] SB %04x, qid [Rx %04x Tx %04x]\n", + vf->relative_vf_id, i, vf->igu_sbs[i], + p_queue->fw_rx_qid, p_queue->fw_tx_qid); } + /* Update the link configuration in bulletin. + */ + OSAL_MEMCPY(&link_params, ecore_mcp_get_link_params(p_hwfn), + sizeof(link_params)); + OSAL_MEMCPY(&link_state, ecore_mcp_get_link_state(p_hwfn), + sizeof(link_state)); + OSAL_MEMCPY(&link_caps, ecore_mcp_get_link_capabilities(p_hwfn), + sizeof(link_caps)); + ecore_iov_set_link(p_hwfn, p_params->rel_vf_id, + &link_params, &link_state, &link_caps); + rc = ecore_iov_enable_vf_access(p_hwfn, p_ptt, vf); if (rc == ECORE_SUCCESS) { - struct ecore_hw_sriov_info *p_iov = &p_hwfn->p_dev->sriov_info; - u16 vf_id = vf->relative_vf_id; + vf->b_init = true; + p_hwfn->pf_iov_info->active_vfs[vf->relative_vf_id / 64] |= + (1ULL << (vf->relative_vf_id % 64)); - p_iov->num_vfs++; - p_iov->active_vfs[vf_id / 64] |= (1ULL << (vf_id % 64)); + if (IS_LEAD_HWFN(p_hwfn)) + p_hwfn->p_dev->p_iov_info->num_vfs++; } return rc; @@ -831,8 +1204,10 @@ enum _ecore_status_t ecore_iov_release_hw_for_vf(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, u16 rel_vf_id) { + struct ecore_mcp_link_capabilities caps; + struct ecore_mcp_link_params params; + struct ecore_mcp_link_state link; struct ecore_vf_info *vf = OSAL_NULL; - enum _ecore_status_t rc = ECORE_SUCCESS; vf = ecore_iov_get_vf_info(p_hwfn, rel_vf_id, true); if (!vf) { @@ -840,38 +1215,45 @@ enum _ecore_status_t ecore_iov_release_hw_for_vf(struct ecore_hwfn *p_hwfn, return ECORE_UNKNOWN_ERROR; } - if (vf->state != VF_STOPPED) { - /* Stopping the VF */ - rc = ecore_sp_vf_stop(p_hwfn, vf->concrete_fid, vf->opaque_fid); + if (vf->bulletin.p_virt) + OSAL_MEMSET(vf->bulletin.p_virt, 0, + sizeof(*vf->bulletin.p_virt)); - if (rc != ECORE_SUCCESS) { - DP_ERR(p_hwfn, "ecore_sp_vf_stop returned error %d\n", - rc); - return rc; - } + OSAL_MEMSET(&vf->p_vf_info, 0, sizeof(vf->p_vf_info)); - vf->state = VF_STOPPED; - } + /* Get the link configuration back in bulletin so + * that when VFs are re-enabled they get the actual + * link configuration. + */ + OSAL_MEMCPY(¶ms, ecore_mcp_get_link_params(p_hwfn), sizeof(params)); + OSAL_MEMCPY(&link, ecore_mcp_get_link_state(p_hwfn), sizeof(link)); + OSAL_MEMCPY(&caps, ecore_mcp_get_link_capabilities(p_hwfn), + sizeof(caps)); + ecore_iov_set_link(p_hwfn, rel_vf_id, ¶ms, &link, &caps); + + /* Forget the VF's acquisition message */ + OSAL_MEMSET(&vf->acquire, 0, sizeof(vf->acquire)); /* disablng interrupts and resetting permission table was done during * vf-close, however, we could get here without going through vf_close */ /* Disable Interrupts for VF */ - ecore_iov_vf_igu_set_int(p_hwfn, p_ptt, vf, 0 /* disable */); + ecore_iov_vf_igu_set_int(p_hwfn, p_ptt, vf, 0); /* Reset Permission table */ - ecore_iov_config_perm_table(p_hwfn, p_ptt, vf, 0 /* disable */); + ecore_iov_config_perm_table(p_hwfn, p_ptt, vf, 0); vf->num_rxqs = 0; vf->num_txqs = 0; ecore_iov_free_vf_igu_sbs(p_hwfn, p_ptt, vf); - if (ECORE_IS_VF_ACTIVE(p_hwfn->p_dev, rel_vf_id)) { - struct ecore_hw_sriov_info *p_iov = &p_hwfn->p_dev->sriov_info; - u16 vf_id = vf->relative_vf_id; + if (vf->b_init) { + vf->b_init = false; + p_hwfn->pf_iov_info->active_vfs[vf->relative_vf_id / 64] &= + ~(1ULL << (vf->relative_vf_id / 64)); - p_iov->num_vfs--; - p_iov->active_vfs[vf_id / 64] &= ~(1ULL << (vf_id % 64)); + if (IS_LEAD_HWFN(p_hwfn)) + p_hwfn->p_dev->p_iov_info->num_vfs--; } return ECORE_SUCCESS; @@ -885,10 +1267,6 @@ static bool ecore_iov_tlv_supported(u16 tlvtype) static void ecore_iov_lock_vf_pf_channel(struct ecore_hwfn *p_hwfn, struct ecore_vf_info *vf, u16 tlv) { - /* we don't lock the channel for unsupported tlvs */ - if (!ecore_iov_tlv_supported(tlv)) - return; - /* lock the channel */ /* mutex_lock(&vf->op_mutex); @@@TBD MichalK - add lock... */ @@ -896,43 +1274,42 @@ static void ecore_iov_lock_vf_pf_channel(struct ecore_hwfn *p_hwfn, /* vf->op_current = tlv; @@@TBD MichalK */ /* log the lock */ - DP_VERBOSE(p_hwfn, - ECORE_MSG_IOV, - "VF[%d]: vf pf channel locked by %s\n", - vf->abs_vf_id, ecore_channel_tlvs_string[tlv]); + if (ecore_iov_tlv_supported(tlv)) + DP_VERBOSE(p_hwfn, + ECORE_MSG_IOV, + "VF[%d]: vf pf channel locked by %s\n", + vf->abs_vf_id, + ecore_channel_tlvs_string[tlv]); + else + DP_VERBOSE(p_hwfn, + ECORE_MSG_IOV, + "VF[%d]: vf pf channel locked by %04x\n", + vf->abs_vf_id, tlv); } static void ecore_iov_unlock_vf_pf_channel(struct ecore_hwfn *p_hwfn, struct ecore_vf_info *vf, u16 expected_tlv) { - /* we don't unlock the channel for unsupported tlvs */ - if (!ecore_iov_tlv_supported(expected_tlv)) - return; - - /* WARN(expected_tlv != vf->op_current, - * "lock mismatch: expected %s found %s", - * channel_tlvs_string[expected_tlv], - * channel_tlvs_string[vf->op_current]); - * @@@TBD MichalK - */ - - /* lock the channel */ - /* mutex_unlock(&vf->op_mutex); @@@TBD MichalK add the lock */ - /* log the unlock */ - DP_VERBOSE(p_hwfn, - ECORE_MSG_IOV, - "VF[%d]: vf pf channel unlocked by %s\n", - vf->abs_vf_id, ecore_channel_tlvs_string[expected_tlv]); + if (ecore_iov_tlv_supported(expected_tlv)) + DP_VERBOSE(p_hwfn, + ECORE_MSG_IOV, + "VF[%d]: vf pf channel unlocked by %s\n", + vf->abs_vf_id, + ecore_channel_tlvs_string[expected_tlv]); + else + DP_VERBOSE(p_hwfn, + ECORE_MSG_IOV, + "VF[%d]: vf pf channel unlocked by %04x\n", + vf->abs_vf_id, expected_tlv); /* record the locking op */ /* vf->op_current = CHANNEL_TLV_NONE; */ } /* place a given tlv on the tlv buffer, continuing current tlv list */ -void *ecore_add_tlv(struct ecore_hwfn *p_hwfn, - u8 **offset, u16 type, u16 length) +void *ecore_add_tlv(u8 **offset, u16 type, u16 length) { struct channel_tlv *tl = (struct channel_tlv *)*offset; @@ -988,7 +1365,12 @@ void ecore_dp_tlv_list(struct ecore_hwfn *p_hwfn, void *tlvs_list) static void ecore_iov_send_response(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, struct ecore_vf_info *p_vf, - u16 length, u8 status) +#ifdef CONFIG_ECORE_SW_CHANNEL + u16 length, +#else + u16 OSAL_UNUSED length, +#endif + u8 status) { struct ecore_iov_vf_mbx *mbx = &p_vf->vf_mbx; struct ecore_dmae_params params; @@ -996,15 +1378,15 @@ static void ecore_iov_send_response(struct ecore_hwfn *p_hwfn, mbx->reply_virt->default_resp.hdr.status = status; + ecore_dp_tlv_list(p_hwfn, mbx->reply_virt); + #ifdef CONFIG_ECORE_SW_CHANNEL mbx->sw_mbx.response_size = length + sizeof(struct channel_list_end_tlv); -#endif - ecore_dp_tlv_list(p_hwfn, mbx->reply_virt); - - if (!p_hwfn->p_dev->sriov_info.b_hw_channel) + if (!p_vf->b_hw_channel) return; +#endif eng_vf_id = p_vf->abs_vf_id; @@ -1018,17 +1400,22 @@ static void ecore_iov_send_response(struct ecore_hwfn *p_hwfn, (sizeof(union pfvf_tlvs) - sizeof(u64)) / 4, ¶ms); + /* Once PF copies the rc to the VF, the latter can continue and + * and send an additional message. So we have to make sure the + * channel would be re-set to ready prior to that. + */ + REG_WR(p_hwfn, + GTT_BAR0_MAP_REG_USDM_RAM + + USTORM_VF_PF_CHANNEL_READY_OFFSET(eng_vf_id), 1); + ecore_dmae_host2host(p_hwfn, p_ptt, mbx->reply_phys, mbx->req_virt->first_tlv.reply_address, sizeof(u64) / 4, ¶ms); - REG_WR(p_hwfn, - GTT_BAR0_MAP_REG_USDM_RAM + - USTORM_VF_PF_CHANNEL_READY_OFFSET(eng_vf_id), 1); + OSAL_IOV_PF_RESP_TYPE(p_hwfn, p_vf->relative_vf_id, status); } -static u16 ecore_iov_vport_to_tlv(struct ecore_hwfn *p_hwfn, - enum ecore_iov_vport_update_flag flag) +static u16 ecore_iov_vport_to_tlv(enum ecore_iov_vport_update_flag flag) { switch (flag) { case ECORE_IOV_VP_UPDATE_ACTIVATE: @@ -1062,19 +1449,19 @@ static u16 ecore_iov_prep_vp_update_resp_tlvs(struct ecore_hwfn *p_hwfn, u16 size, total_len, i; OSAL_MEMSET(p_mbx->reply_virt, 0, sizeof(union pfvf_tlvs)); - p_mbx->offset = (u8 *)(p_mbx->reply_virt); + p_mbx->offset = (u8 *)p_mbx->reply_virt; size = sizeof(struct pfvf_def_resp_tlv); total_len = size; - ecore_add_tlv(p_hwfn, &p_mbx->offset, CHANNEL_TLV_VPORT_UPDATE, size); + ecore_add_tlv(&p_mbx->offset, CHANNEL_TLV_VPORT_UPDATE, size); /* Prepare response for all extended tlvs if they are found by PF */ for (i = 0; i < ECORE_IOV_VP_UPDATE_MAX; i++) { if (!(tlvs_mask & (1 << i))) continue; - resp = ecore_add_tlv(p_hwfn, &p_mbx->offset, - ecore_iov_vport_to_tlv(p_hwfn, i), size); + resp = ecore_add_tlv(&p_mbx->offset, ecore_iov_vport_to_tlv(i), + size); if (tlvs_accepted & (1 << i)) resp->hdr.status = status; @@ -1084,12 +1471,13 @@ static u16 ecore_iov_prep_vp_update_resp_tlvs(struct ecore_hwfn *p_hwfn, DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, "VF[%d] - vport_update resp: TLV %d, status %02x\n", p_vf->relative_vf_id, - ecore_iov_vport_to_tlv(p_hwfn, i), resp->hdr.status); + ecore_iov_vport_to_tlv(i), + resp->hdr.status); total_len += size; } - ecore_add_tlv(p_hwfn, &p_mbx->offset, CHANNEL_TLV_LIST_END, + ecore_add_tlv(&p_mbx->offset, CHANNEL_TLV_LIST_END, sizeof(struct channel_list_end_tlv)); return total_len; @@ -1102,23 +1490,35 @@ static void ecore_iov_prepare_resp(struct ecore_hwfn *p_hwfn, { struct ecore_iov_vf_mbx *mbx = &vf_info->vf_mbx; - mbx->offset = (u8 *)(mbx->reply_virt); + mbx->offset = (u8 *)mbx->reply_virt; - ecore_add_tlv(p_hwfn, &mbx->offset, type, length); - ecore_add_tlv(p_hwfn, &mbx->offset, CHANNEL_TLV_LIST_END, + ecore_add_tlv(&mbx->offset, type, length); + ecore_add_tlv(&mbx->offset, CHANNEL_TLV_LIST_END, sizeof(struct channel_list_end_tlv)); ecore_iov_send_response(p_hwfn, p_ptt, vf_info, length, status); } +struct ecore_public_vf_info +*ecore_iov_get_public_vf_info(struct ecore_hwfn *p_hwfn, + u16 relative_vf_id, + bool b_enabled_only) +{ + struct ecore_vf_info *vf = OSAL_NULL; + + vf = ecore_iov_get_vf_info(p_hwfn, relative_vf_id, b_enabled_only); + if (!vf) + return OSAL_NULL; + + return &vf->p_vf_info; +} + static void ecore_iov_vf_cleanup(struct ecore_hwfn *p_hwfn, struct ecore_vf_info *p_vf) { + u32 i, j; p_vf->vf_bulletin = 0; p_vf->vport_instance = 0; - p_vf->num_mac_filters = 0; - p_vf->num_vlan_filters = 0; - p_vf->num_mc_filters = 0; p_vf->configured_features = 0; /* If VF previously requested less resources, go back to default */ @@ -1127,39 +1527,246 @@ static void ecore_iov_vf_cleanup(struct ecore_hwfn *p_hwfn, p_vf->num_active_rxqs = 0; + for (i = 0; i < ECORE_MAX_VF_CHAINS_PER_PF; i++) { + struct ecore_vf_queue *p_queue = &p_vf->vf_queues[i]; + + for (j = 0; j < MAX_QUEUES_PER_QZONE; j++) { + if (!p_queue->cids[j].p_cid) + continue; + + ecore_eth_queue_cid_release(p_hwfn, + p_queue->cids[j].p_cid); + p_queue->cids[j].p_cid = OSAL_NULL; + } + } + OSAL_MEMSET(&p_vf->shadow_config, 0, sizeof(p_vf->shadow_config)); + OSAL_MEMSET(&p_vf->acquire, 0, sizeof(p_vf->acquire)); OSAL_IOV_VF_CLEANUP(p_hwfn, p_vf->relative_vf_id); } -static void ecore_iov_vf_mbx_acquire(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt, - struct ecore_vf_info *vf) +/* Returns either 0, or log(size) */ +static u32 ecore_iov_vf_db_bar_size(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt) +{ + u32 val = ecore_rd(p_hwfn, p_ptt, PGLUE_B_REG_VF_BAR1_SIZE); + + if (val) + return val + 11; + return 0; +} + +static void +ecore_iov_vf_mbx_acquire_resc_cids(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + struct ecore_vf_info *p_vf, + struct vf_pf_resc_request *p_req, + struct pf_vf_resc *p_resp) +{ + u8 num_vf_cons = p_hwfn->pf_params.eth_pf_params.num_vf_cons; + u8 db_size = DB_ADDR_VF(1, DQ_DEMS_LEGACY) - + DB_ADDR_VF(0, DQ_DEMS_LEGACY); + u32 bar_size; + + p_resp->num_cids = OSAL_MIN_T(u8, p_req->num_cids, num_vf_cons); + + /* If VF didn't bother asking for QIDs than don't bother limiting + * number of CIDs. The VF doesn't care about the number, and this + * has the likely result of causing an additional acquisition. + */ + if (!(p_vf->acquire.vfdev_info.capabilities & + VFPF_ACQUIRE_CAP_QUEUE_QIDS)) + return; + + /* If doorbell bar was mapped by VF, limit the VF CIDs to an amount + * that would make sure doorbells for all CIDs fall within the bar. + * If it doesn't, make sure regview window is sufficient. + */ + if (p_vf->acquire.vfdev_info.capabilities & + VFPF_ACQUIRE_CAP_PHYSICAL_BAR) { + bar_size = ecore_iov_vf_db_bar_size(p_hwfn, p_ptt); + if (bar_size) + bar_size = 1 << bar_size; + + if (ECORE_IS_CMT(p_hwfn->p_dev)) + bar_size /= 2; + } else { + bar_size = PXP_VF_BAR0_DQ_LENGTH; + } + + if (bar_size / db_size < 256) + p_resp->num_cids = OSAL_MIN_T(u8, p_resp->num_cids, + (u8)(bar_size / db_size)); +} + +static u8 ecore_iov_vf_mbx_acquire_resc(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + struct ecore_vf_info *p_vf, + struct vf_pf_resc_request *p_req, + struct pf_vf_resc *p_resp) +{ + u8 i; + + /* Queue related information */ + p_resp->num_rxqs = p_vf->num_rxqs; + p_resp->num_txqs = p_vf->num_txqs; + p_resp->num_sbs = p_vf->num_sbs; + + for (i = 0; i < p_resp->num_sbs; i++) { + p_resp->hw_sbs[i].hw_sb_id = p_vf->igu_sbs[i]; + /* TODO - what's this sb_qid field? Is it deprecated? + * or is there an ecore_client that looks at this? + */ + p_resp->hw_sbs[i].sb_qid = 0; + } + + /* These fields are filled for backward compatibility. + * Unused by modern vfs. + */ + for (i = 0; i < p_resp->num_rxqs; i++) { + ecore_fw_l2_queue(p_hwfn, p_vf->vf_queues[i].fw_rx_qid, + (u16 *)&p_resp->hw_qid[i]); + p_resp->cid[i] = i; + } + + /* Filter related information */ + p_resp->num_mac_filters = OSAL_MIN_T(u8, p_vf->num_mac_filters, + p_req->num_mac_filters); + p_resp->num_vlan_filters = OSAL_MIN_T(u8, p_vf->num_vlan_filters, + p_req->num_vlan_filters); + + ecore_iov_vf_mbx_acquire_resc_cids(p_hwfn, p_ptt, p_vf, p_req, p_resp); + + /* This isn't really needed/enforced, but some legacy VFs might depend + * on the correct filling of this field. + */ + p_resp->num_mc_filters = ECORE_MAX_MC_ADDRS; + + /* Validate sufficient resources for VF */ + if (p_resp->num_rxqs < p_req->num_rxqs || + p_resp->num_txqs < p_req->num_txqs || + p_resp->num_sbs < p_req->num_sbs || + p_resp->num_mac_filters < p_req->num_mac_filters || + p_resp->num_vlan_filters < p_req->num_vlan_filters || + p_resp->num_mc_filters < p_req->num_mc_filters || + p_resp->num_cids < p_req->num_cids) { + DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, + "VF[%d] - Insufficient resources: rxq [%02x/%02x] txq [%02x/%02x] sbs [%02x/%02x] mac [%02x/%02x] vlan [%02x/%02x] mc [%02x/%02x] cids [%02x/%02x]\n", + p_vf->abs_vf_id, + p_req->num_rxqs, p_resp->num_rxqs, + p_req->num_rxqs, p_resp->num_txqs, + p_req->num_sbs, p_resp->num_sbs, + p_req->num_mac_filters, p_resp->num_mac_filters, + p_req->num_vlan_filters, p_resp->num_vlan_filters, + p_req->num_mc_filters, p_resp->num_mc_filters, + p_req->num_cids, p_resp->num_cids); + + /* Some legacy OSes are incapable of correctly handling this + * failure. + */ + if ((p_vf->acquire.vfdev_info.eth_fp_hsi_minor == + ETH_HSI_VER_NO_PKT_LEN_TUNN) && + (p_vf->acquire.vfdev_info.os_type == + VFPF_ACQUIRE_OS_WINDOWS)) + return PFVF_STATUS_SUCCESS; + + return PFVF_STATUS_NO_RESOURCE; + } + + return PFVF_STATUS_SUCCESS; +} + +static void ecore_iov_vf_mbx_acquire_stats(struct pfvf_stats_info *p_stats) +{ + p_stats->mstats.address = PXP_VF_BAR0_START_MSDM_ZONE_B + + OFFSETOF(struct mstorm_vf_zone, + non_trigger.eth_queue_stat); + p_stats->mstats.len = sizeof(struct eth_mstorm_per_queue_stat); + p_stats->ustats.address = PXP_VF_BAR0_START_USDM_ZONE_B + + OFFSETOF(struct ustorm_vf_zone, + non_trigger.eth_queue_stat); + p_stats->ustats.len = sizeof(struct eth_ustorm_per_queue_stat); + p_stats->pstats.address = PXP_VF_BAR0_START_PSDM_ZONE_B + + OFFSETOF(struct pstorm_vf_zone, + non_trigger.eth_queue_stat); + p_stats->pstats.len = sizeof(struct eth_pstorm_per_queue_stat); + p_stats->tstats.address = 0; + p_stats->tstats.len = 0; +} + +static void ecore_iov_vf_mbx_acquire(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + struct ecore_vf_info *vf) { struct ecore_iov_vf_mbx *mbx = &vf->vf_mbx; - struct vfpf_acquire_tlv *req = &mbx->req_virt->acquire; struct pfvf_acquire_resp_tlv *resp = &mbx->reply_virt->acquire_resp; - struct pf_vf_resc *resc = &resp->resc; struct pf_vf_pfdev_info *pfdev_info = &resp->pfdev_info; - u16 length; - u8 i, vfpf_status = PFVF_STATUS_SUCCESS; + struct vfpf_acquire_tlv *req = &mbx->req_virt->acquire; + u8 vfpf_status = PFVF_STATUS_NOT_SUPPORTED; + struct pf_vf_resc *resc = &resp->resc; + enum _ecore_status_t rc; - /* Validate FW compatibility */ - if (req->vfdev_info.fw_major != FW_MAJOR_VERSION || - req->vfdev_info.fw_minor != FW_MINOR_VERSION || - req->vfdev_info.fw_revision != FW_REVISION_VERSION || - req->vfdev_info.fw_engineering != FW_ENGINEERING_VERSION) { - DP_INFO(p_hwfn, - "VF[%d] is running an incompatible driver [VF needs" - " FW %02x:%02x:%02x:%02x but Hypervisor is" - " using %02x:%02x:%02x:%02x]\n", - vf->abs_vf_id, req->vfdev_info.fw_major, - req->vfdev_info.fw_minor, req->vfdev_info.fw_revision, - req->vfdev_info.fw_engineering, FW_MAJOR_VERSION, - FW_MINOR_VERSION, FW_REVISION_VERSION, - FW_ENGINEERING_VERSION); - vfpf_status = PFVF_STATUS_NOT_SUPPORTED; + OSAL_MEMSET(resp, 0, sizeof(*resp)); + + /* Write the PF version so that VF would know which version + * is supported - might be later overridden. This guarantees that + * VF could recognize legacy PF based on lack of versions in reply. + */ + pfdev_info->major_fp_hsi = ETH_HSI_VER_MAJOR; + pfdev_info->minor_fp_hsi = ETH_HSI_VER_MINOR; + + /* TODO - not doing anything is bad since we'll assert, but this isn't + * necessarily the right behavior - perhaps we should have allowed some + * versatility here. + */ + if (vf->state != VF_FREE && + vf->state != VF_STOPPED) { + DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, + "VF[%d] sent ACQUIRE but is already in state %d - fail request\n", + vf->abs_vf_id, vf->state); goto out; } + + /* Validate FW compatibility */ + if (req->vfdev_info.eth_fp_hsi_major != ETH_HSI_VER_MAJOR) { + if (req->vfdev_info.capabilities & + VFPF_ACQUIRE_CAP_PRE_FP_HSI) { + struct vf_pf_vfdev_info *p_vfdev = &req->vfdev_info; + + /* This legacy support would need to be removed once + * the major has changed. + */ + OSAL_BUILD_BUG_ON(ETH_HSI_VER_MAJOR != 3); + + DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, + "VF[%d] is pre-fastpath HSI\n", + vf->abs_vf_id); + p_vfdev->eth_fp_hsi_major = ETH_HSI_VER_MAJOR; + p_vfdev->eth_fp_hsi_minor = ETH_HSI_VER_NO_PKT_LEN_TUNN; + } else { + DP_INFO(p_hwfn, + "VF[%d] needs fastpath HSI %02x.%02x, which is" + " incompatible with loaded FW's faspath" + " HSI %02x.%02x\n", + vf->abs_vf_id, + req->vfdev_info.eth_fp_hsi_major, + req->vfdev_info.eth_fp_hsi_minor, + ETH_HSI_VER_MAJOR, ETH_HSI_VER_MINOR); + + goto out; + } + } + + /* On 100g PFs, prevent old VFs from loading */ + if (ECORE_IS_CMT(p_hwfn->p_dev) && + !(req->vfdev_info.capabilities & VFPF_ACQUIRE_CAP_100G)) { + DP_INFO(p_hwfn, + "VF[%d] is running an old driver that doesn't support" + " 100g\n", + vf->abs_vf_id); + goto out; + } + #ifndef __EXTRACT__LINUX__ if (OSAL_IOV_VF_ACQUIRE(p_hwfn, vf->relative_vf_id) != ECORE_SUCCESS) { vfpf_status = PFVF_STATUS_NOT_SUPPORTED; @@ -1167,13 +1774,10 @@ static void ecore_iov_vf_mbx_acquire(struct ecore_hwfn *p_hwfn, } #endif - OSAL_MEMSET(resp, 0, sizeof(*resp)); + /* Store the acquire message */ + OSAL_MEMCPY(&vf->acquire, req, sizeof(vf->acquire)); - /* Fill in vf info stuff : @@@TBD MichalK Hard Coded for now... */ vf->opaque_fid = req->vfdev_info.opaque_fid; - vf->num_mac_filters = 1; - vf->num_vlan_filters = ECORE_ETH_VF_NUM_VLAN_FILTERS; - vf->num_mc_filters = ECORE_MAX_MC_ADDRS; vf->vf_bulletin = req->bulletin_addr; vf->bulletin.size = (vf->bulletin.size < req->bulletin_size) ? @@ -1182,29 +1786,24 @@ static void ecore_iov_vf_mbx_acquire(struct ecore_hwfn *p_hwfn, /* fill in pfdev info */ pfdev_info->chip_num = p_hwfn->p_dev->chip_num; pfdev_info->db_size = 0; /* @@@ TBD MichalK Vf Doorbells */ - pfdev_info->indices_per_sb = PIS_PER_SB; - pfdev_info->capabilities = PFVF_ACQUIRE_CAP_DEFAULT_UNTAGGED; + pfdev_info->indices_per_sb = PIS_PER_SB_E4; - pfdev_info->stats_info.mstats.address = - PXP_VF_BAR0_START_MSDM_ZONE_B + - OFFSETOF(struct mstorm_vf_zone, non_trigger.eth_queue_stat); - pfdev_info->stats_info.mstats.len = - sizeof(struct eth_mstorm_per_queue_stat); + pfdev_info->capabilities = PFVF_ACQUIRE_CAP_DEFAULT_UNTAGGED | + PFVF_ACQUIRE_CAP_POST_FW_OVERRIDE; + if (ECORE_IS_CMT(p_hwfn->p_dev)) + pfdev_info->capabilities |= PFVF_ACQUIRE_CAP_100G; - pfdev_info->stats_info.ustats.address = - PXP_VF_BAR0_START_USDM_ZONE_B + - OFFSETOF(struct ustorm_vf_zone, non_trigger.eth_queue_stat); - pfdev_info->stats_info.ustats.len = - sizeof(struct eth_ustorm_per_queue_stat); + /* Share our ability to use multiple queue-ids only with VFs + * that request it. + */ + if (req->vfdev_info.capabilities & VFPF_ACQUIRE_CAP_QUEUE_QIDS) + pfdev_info->capabilities |= PFVF_ACQUIRE_CAP_QUEUE_QIDS; - pfdev_info->stats_info.pstats.address = - PXP_VF_BAR0_START_PSDM_ZONE_B + - OFFSETOF(struct pstorm_vf_zone, non_trigger.eth_queue_stat); - pfdev_info->stats_info.pstats.len = - sizeof(struct eth_pstorm_per_queue_stat); + /* Share the sizes of the bars with VF */ + resp->pfdev_info.bar_size = (u8)ecore_iov_vf_db_bar_size(p_hwfn, + p_ptt); - pfdev_info->stats_info.tstats.address = 0; - pfdev_info->stats_info.tstats.len = 0; + ecore_iov_vf_mbx_acquire_stats(&pfdev_info->stats_info); OSAL_MEMCPY(pfdev_info->port_mac, p_hwfn->hw_info.hw_mac_addr, ETH_ALEN); @@ -1213,35 +1812,36 @@ static void ecore_iov_vf_mbx_acquire(struct ecore_hwfn *p_hwfn, pfdev_info->fw_minor = FW_MINOR_VERSION; pfdev_info->fw_rev = FW_REVISION_VERSION; pfdev_info->fw_eng = FW_ENGINEERING_VERSION; + + /* Incorrect when legacy, but doesn't matter as legacy isn't reading + * this field. + */ + pfdev_info->minor_fp_hsi = OSAL_MIN_T(u8, ETH_HSI_VER_MINOR, + req->vfdev_info.eth_fp_hsi_minor); pfdev_info->os_type = OSAL_IOV_GET_OS_TYPE(); - ecore_mcp_get_mfw_ver(p_hwfn->p_dev, p_ptt, &pfdev_info->mfw_ver, + ecore_mcp_get_mfw_ver(p_hwfn, p_ptt, &pfdev_info->mfw_ver, OSAL_NULL); pfdev_info->dev_type = p_hwfn->p_dev->type; pfdev_info->chip_rev = p_hwfn->p_dev->chip_rev; - /* Fill in resc : @@@TBD MichalK Hard Coded for now... */ - resc->num_rxqs = vf->num_rxqs; - resc->num_txqs = vf->num_txqs; - resc->num_sbs = vf->num_sbs; - for (i = 0; i < resc->num_sbs; i++) { - resc->hw_sbs[i].hw_sb_id = vf->igu_sbs[i]; - resc->hw_sbs[i].sb_qid = 0; - } + /* Fill resources available to VF; Make sure there are enough to + * satisfy the VF's request. + */ + vfpf_status = ecore_iov_vf_mbx_acquire_resc(p_hwfn, p_ptt, vf, + &req->resc_request, resc); + if (vfpf_status != PFVF_STATUS_SUCCESS) + goto out; - for (i = 0; i < resc->num_rxqs; i++) { - ecore_fw_l2_queue(p_hwfn, vf->vf_queues[i].fw_rx_qid, - (u16 *)&resc->hw_qid[i]); - resc->cid[i] = vf->vf_queues[i].fw_cid; + /* Start the VF in FW */ + rc = ecore_sp_vf_start(p_hwfn, vf); + if (rc != ECORE_SUCCESS) { + DP_NOTICE(p_hwfn, true, "Failed to start VF[%02x]\n", + vf->abs_vf_id); + vfpf_status = PFVF_STATUS_FAILURE; + goto out; } - resc->num_mac_filters = OSAL_MIN_T(u8, vf->num_mac_filters, - req->resc_request.num_mac_filters); - resc->num_vlan_filters = OSAL_MIN_T(u8, vf->num_vlan_filters, - req->resc_request.num_vlan_filters); - resc->num_mc_filters = OSAL_MIN_T(u8, vf->num_mc_filters, - req->resc_request.num_mc_filters); - /* Fill agreed size of bulletin board in response, and post * an initial image to the bulletin board. */ @@ -1250,25 +1850,22 @@ static void ecore_iov_vf_mbx_acquire(struct ecore_hwfn *p_hwfn, DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, "VF[%d] ACQUIRE_RESPONSE: pfdev_info- chip_num=0x%x," - " db_size=%d, idx_per_sb=%d, pf_cap=0x%" PRIx64 "\n" + " db_size=%d, idx_per_sb=%d, pf_cap=0x%lx\n" "resources- n_rxq-%d, n_txq-%d, n_sbs-%d, n_macs-%d," - " n_vlans-%d, n_mcs-%d\n", + " n_vlans-%d\n", vf->abs_vf_id, resp->pfdev_info.chip_num, resp->pfdev_info.db_size, resp->pfdev_info.indices_per_sb, - resp->pfdev_info.capabilities, resc->num_rxqs, + (unsigned long)resp->pfdev_info.capabilities, resc->num_rxqs, resc->num_txqs, resc->num_sbs, resc->num_mac_filters, - resc->num_vlan_filters, resc->num_mc_filters); + resc->num_vlan_filters); vf->state = VF_ACQUIRED; - /* Prepare Response */ - length = sizeof(struct pfvf_acquire_resp_tlv); - out: + /* Prepare Response */ ecore_iov_prepare_resp(p_hwfn, p_ptt, vf, CHANNEL_TLV_ACQUIRE, - length, vfpf_status); - - /* @@@TBD Bulletin */ + sizeof(struct pfvf_acquire_resp_tlv), + vfpf_status); } static enum _ecore_status_t @@ -1310,8 +1907,8 @@ static enum _ecore_status_t ecore_iov_reconfigure_unicast_vlan(struct ecore_hwfn *p_hwfn, struct ecore_vf_info *p_vf) { - enum _ecore_status_t rc = ECORE_SUCCESS; struct ecore_filter_ucast filter; + enum _ecore_status_t rc = ECORE_SUCCESS; int i; OSAL_MEMSET(&filter, 0, sizeof(filter)); @@ -1322,24 +1919,23 @@ ecore_iov_reconfigure_unicast_vlan(struct ecore_hwfn *p_hwfn, /* Reconfigure vlans */ for (i = 0; i < ECORE_ETH_VF_NUM_VLAN_FILTERS + 1; i++) { - if (p_vf->shadow_config.vlans[i].used) { - filter.type = ECORE_FILTER_VLAN; - filter.vlan = p_vf->shadow_config.vlans[i].vid; - DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, - "Reconfig VLAN [0x%04x] for VF [%04x]\n", - filter.vlan, p_vf->relative_vf_id); - rc = ecore_sp_eth_filter_ucast(p_hwfn, - p_vf->opaque_fid, - &filter, - ECORE_SPQ_MODE_CB, - OSAL_NULL); - if (rc) { - DP_NOTICE(p_hwfn, true, - "Failed to configure VLAN [%04x]" - " to VF [%04x]\n", - filter.vlan, p_vf->relative_vf_id); - break; - } + if (!p_vf->shadow_config.vlans[i].used) + continue; + + filter.type = ECORE_FILTER_VLAN; + filter.vlan = p_vf->shadow_config.vlans[i].vid; + DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, + "Reconfiguring VLAN [0x%04x] for VF [%04x]\n", + filter.vlan, p_vf->relative_vf_id); + rc = ecore_sp_eth_filter_ucast(p_hwfn, p_vf->opaque_fid, + &filter, ECORE_SPQ_MODE_CB, + OSAL_NULL); + if (rc) { + DP_NOTICE(p_hwfn, true, + "Failed to configure VLAN [%04x]" + " to VF [%04x]\n", + filter.vlan, p_vf->relative_vf_id); + break; } } @@ -1361,9 +1957,10 @@ ecore_iov_reconfigure_unicast_shadow(struct ecore_hwfn *p_hwfn, return rc; } -static int ecore_iov_configure_vport_forced(struct ecore_hwfn *p_hwfn, - struct ecore_vf_info *p_vf, - u64 events) +static enum _ecore_status_t +ecore_iov_configure_vport_forced(struct ecore_hwfn *p_hwfn, + struct ecore_vf_info *p_vf, + u64 events) { enum _ecore_status_t rc = ECORE_SUCCESS; struct ecore_filter_ucast filter; @@ -1443,22 +2040,24 @@ static int ecore_iov_configure_vport_forced(struct ecore_hwfn *p_hwfn, /* Update all the Rx queues */ for (i = 0; i < ECORE_MAX_VF_CHAINS_PER_PF; i++) { - u16 qid; + struct ecore_vf_queue *p_queue = &p_vf->vf_queues[i]; + struct ecore_queue_cid *p_cid = OSAL_NULL; - if (!p_vf->vf_queues[i].rxq_active) + /* There can be at most 1 Rx queue on qzone. Find it */ + p_cid = ecore_iov_get_vf_rx_queue_cid(p_queue); + if (p_cid == OSAL_NULL) continue; - qid = p_vf->vf_queues[i].fw_rx_qid; - - rc = ecore_sp_eth_rx_queues_update(p_hwfn, qid, + rc = ecore_sp_eth_rx_queues_update(p_hwfn, + (void **)&p_cid, 1, 0, 1, ECORE_SPQ_MODE_EBLOCK, OSAL_NULL); if (rc) { DP_NOTICE(p_hwfn, true, "Failed to send Rx update" - " queue[0x%04x]\n", - qid); + " fo queue[0x%04x]\n", + p_cid->rel.queue_id); return rc; } } @@ -1482,14 +2081,14 @@ static void ecore_iov_vf_mbx_start_vport(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, struct ecore_vf_info *vf) { - struct ecore_iov_vf_mbx *mbx = &vf->vf_mbx; - struct vfpf_vport_start_tlv *start = &mbx->req_virt->start_vport; struct ecore_sp_vport_start_params params = { 0 }; + struct ecore_iov_vf_mbx *mbx = &vf->vf_mbx; + struct vfpf_vport_start_tlv *start; u8 status = PFVF_STATUS_SUCCESS; struct ecore_vf_info *vf_info; - enum _ecore_status_t rc; u64 *p_bitmap; int sb_id; + enum _ecore_status_t rc; vf_info = ecore_iov_get_vf_info(p_hwfn, (u16)vf->relative_vf_id, true); if (!vf_info) { @@ -1500,6 +2099,9 @@ static void ecore_iov_vf_mbx_start_vport(struct ecore_hwfn *p_hwfn, } vf->state = VF_ENABLED; + start = &mbx->req_virt->start_vport; + + ecore_iov_enable_vf_traffic(p_hwfn, p_ptt, vf); /* Initialize Status block in CAU */ for (sb_id = 0; sb_id < vf->num_sbs; sb_id++) { @@ -1513,9 +2115,8 @@ static void ecore_iov_vf_mbx_start_vport(struct ecore_hwfn *p_hwfn, ecore_int_cau_conf_sb(p_hwfn, p_ptt, start->sb_addr[sb_id], vf->igu_sbs[sb_id], - vf->abs_vf_id, 1 /* VF Valid */); + vf->abs_vf_id, 1); } - ecore_iov_enable_vf_traffic(p_hwfn, p_ptt, vf); vf->mtu = start->mtu; vf->shadow_config.inner_vlan_removal = start->inner_vlan_removal; @@ -1539,7 +2140,7 @@ static void ecore_iov_vf_mbx_start_vport(struct ecore_hwfn *p_hwfn, #ifndef ASIC_ONLY if (CHIP_REV_IS_FPGA(p_hwfn->p_dev)) { DP_NOTICE(p_hwfn, false, - "FPGA: Don't confi VF for Tx-switching [no pVFC]\n"); + "FPGA: Don't config VF for Tx-switching [no pVFC]\n"); params.tx_switching = false; } #endif @@ -1551,6 +2152,7 @@ static void ecore_iov_vf_mbx_start_vport(struct ecore_hwfn *p_hwfn, params.vport_id = vf->vport_id; params.max_buffers_per_cqe = start->max_buffers_per_cqe; params.mtu = vf->mtu; + params.check_mac = true; rc = ecore_sp_eth_vport_start(p_hwfn, ¶ms); if (rc != ECORE_SUCCESS) { @@ -1578,9 +2180,21 @@ static void ecore_iov_vf_mbx_stop_vport(struct ecore_hwfn *p_hwfn, u8 status = PFVF_STATUS_SUCCESS; enum _ecore_status_t rc; + OSAL_IOV_VF_VPORT_STOP(p_hwfn, vf); vf->vport_instance--; vf->spoof_chk = false; + if ((ecore_iov_validate_active_rxq(vf)) || + (ecore_iov_validate_active_txq(vf))) { + vf->b_malicious = true; + DP_NOTICE(p_hwfn, false, + "VF [%02x] - considered malicious;" + " Unable to stop RX/TX queuess\n", + vf->abs_vf_id); + status = PFVF_STATUS_MALICIOUS; + goto out; + } + rc = ecore_sp_vport_stop(p_hwfn, vf->opaque_fid, vf->vport_id); if (rc != ECORE_SUCCESS) { DP_ERR(p_hwfn, @@ -1592,151 +2206,539 @@ static void ecore_iov_vf_mbx_stop_vport(struct ecore_hwfn *p_hwfn, vf->configured_features = 0; OSAL_MEMSET(&vf->shadow_config, 0, sizeof(vf->shadow_config)); +out: ecore_iov_prepare_resp(p_hwfn, p_ptt, vf, CHANNEL_TLV_VPORT_TEARDOWN, sizeof(struct pfvf_def_resp_tlv), status); } +static void ecore_iov_vf_mbx_start_rxq_resp(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + struct ecore_vf_info *vf, + u8 status, bool b_legacy) +{ + struct ecore_iov_vf_mbx *mbx = &vf->vf_mbx; + struct pfvf_start_queue_resp_tlv *p_tlv; + struct vfpf_start_rxq_tlv *req; + u16 length; + + mbx->offset = (u8 *)mbx->reply_virt; + + /* Taking a bigger struct instead of adding a TLV to list was a + * mistake, but one which we're now stuck with, as some older + * clients assume the size of the previous response. + */ + if (!b_legacy) + length = sizeof(*p_tlv); + else + length = sizeof(struct pfvf_def_resp_tlv); + + p_tlv = ecore_add_tlv(&mbx->offset, CHANNEL_TLV_START_RXQ, length); + ecore_add_tlv(&mbx->offset, CHANNEL_TLV_LIST_END, + sizeof(struct channel_list_end_tlv)); + + /* Update the TLV with the response */ + if ((status == PFVF_STATUS_SUCCESS) && !b_legacy) { + req = &mbx->req_virt->start_rxq; + p_tlv->offset = PXP_VF_BAR0_START_MSDM_ZONE_B + + OFFSETOF(struct mstorm_vf_zone, + non_trigger.eth_rx_queue_producers) + + sizeof(struct eth_rx_prod_data) * req->rx_qid; + } + + ecore_iov_send_response(p_hwfn, p_ptt, vf, length, status); +} + +static u8 ecore_iov_vf_mbx_qid(struct ecore_hwfn *p_hwfn, + struct ecore_vf_info *p_vf, bool b_is_tx) +{ + struct ecore_iov_vf_mbx *p_mbx = &p_vf->vf_mbx; + struct vfpf_qid_tlv *p_qid_tlv; + + /* Search for the qid if the VF published if its going to provide it */ + if (!(p_vf->acquire.vfdev_info.capabilities & + VFPF_ACQUIRE_CAP_QUEUE_QIDS)) { + if (b_is_tx) + return ECORE_IOV_LEGACY_QID_TX; + else + return ECORE_IOV_LEGACY_QID_RX; + } + + p_qid_tlv = (struct vfpf_qid_tlv *) + ecore_iov_search_list_tlvs(p_hwfn, p_mbx->req_virt, + CHANNEL_TLV_QID); + if (p_qid_tlv == OSAL_NULL) { + DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, + "VF[%2x]: Failed to provide qid\n", + p_vf->relative_vf_id); + + return ECORE_IOV_QID_INVALID; + } + + if (p_qid_tlv->qid >= MAX_QUEUES_PER_QZONE) { + DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, + "VF[%02x]: Provided qid out-of-bounds %02x\n", + p_vf->relative_vf_id, p_qid_tlv->qid); + return ECORE_IOV_QID_INVALID; + } + + return p_qid_tlv->qid; +} + static void ecore_iov_vf_mbx_start_rxq(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, struct ecore_vf_info *vf) { + struct ecore_queue_start_common_params params; + struct ecore_queue_cid_vf_params vf_params; struct ecore_iov_vf_mbx *mbx = &vf->vf_mbx; - struct vfpf_start_rxq_tlv *req = &mbx->req_virt->start_rxq; - u16 length = sizeof(struct pfvf_def_resp_tlv); - u8 status = PFVF_STATUS_SUCCESS; + u8 status = PFVF_STATUS_NO_RESOURCE; + u8 qid_usage_idx, vf_legacy = 0; + struct ecore_vf_queue *p_queue; + struct vfpf_start_rxq_tlv *req; + struct ecore_queue_cid *p_cid; + struct ecore_sb_info sb_dummy; enum _ecore_status_t rc; - rc = ecore_sp_eth_rxq_start_ramrod(p_hwfn, vf->opaque_fid, - vf->vf_queues[req->rx_qid].fw_cid, - vf->vf_queues[req->rx_qid].fw_rx_qid, - vf->vport_id, - vf->abs_vf_id + 0x10, - req->hw_sb, - req->sb_index, - req->bd_max_bytes, - req->rxq_addr, - req->cqe_pbl_addr, - req->cqe_pbl_size); + req = &mbx->req_virt->start_rxq; - if (rc) { + if (!ecore_iov_validate_rxq(p_hwfn, vf, req->rx_qid, + ECORE_IOV_VALIDATE_Q_DISABLE) || + !ecore_iov_validate_sb(p_hwfn, vf, req->hw_sb)) + goto out; + + qid_usage_idx = ecore_iov_vf_mbx_qid(p_hwfn, vf, false); + if (qid_usage_idx == ECORE_IOV_QID_INVALID) + goto out; + + p_queue = &vf->vf_queues[req->rx_qid]; + if (p_queue->cids[qid_usage_idx].p_cid) + goto out; + + vf_legacy = ecore_vf_calculate_legacy(vf); + + /* Acquire a new queue-cid */ + OSAL_MEMSET(¶ms, 0, sizeof(params)); + params.queue_id = (u8)p_queue->fw_rx_qid; + params.vport_id = vf->vport_id; + params.stats_id = vf->abs_vf_id + 0x10; + + /* Since IGU index is passed via sb_info, construct a dummy one */ + OSAL_MEM_ZERO(&sb_dummy, sizeof(sb_dummy)); + sb_dummy.igu_sb_id = req->hw_sb; + params.p_sb = &sb_dummy; + params.sb_idx = req->sb_index; + + OSAL_MEM_ZERO(&vf_params, sizeof(vf_params)); + vf_params.vfid = vf->relative_vf_id; + vf_params.vf_qid = (u8)req->rx_qid; + vf_params.vf_legacy = vf_legacy; + vf_params.qid_usage_idx = qid_usage_idx; + + p_cid = ecore_eth_queue_to_cid(p_hwfn, vf->opaque_fid, + ¶ms, true, &vf_params); + if (p_cid == OSAL_NULL) + goto out; + + /* Legacy VFs have their Producers in a different location, which they + * calculate on their own and clean the producer prior to this. + */ + if (!(vf_legacy & ECORE_QCID_LEGACY_VF_RX_PROD)) + REG_WR(p_hwfn, + GTT_BAR0_MAP_REG_MSDM_RAM + + MSTORM_ETH_VF_PRODS_OFFSET(vf->abs_vf_id, req->rx_qid), + 0); + + rc = ecore_eth_rxq_start_ramrod(p_hwfn, p_cid, + req->bd_max_bytes, + req->rxq_addr, + req->cqe_pbl_addr, + req->cqe_pbl_size); + if (rc != ECORE_SUCCESS) { status = PFVF_STATUS_FAILURE; + ecore_eth_queue_cid_release(p_hwfn, p_cid); } else { - vf->vf_queues[req->rx_qid].rxq_active = true; + p_queue->cids[qid_usage_idx].p_cid = p_cid; + p_queue->cids[qid_usage_idx].b_is_tx = false; + status = PFVF_STATUS_SUCCESS; vf->num_active_rxqs++; } - ecore_iov_prepare_resp(p_hwfn, p_ptt, vf, CHANNEL_TLV_START_RXQ, - length, status); +out: + ecore_iov_vf_mbx_start_rxq_resp(p_hwfn, p_ptt, vf, status, + !!(vf_legacy & + ECORE_QCID_LEGACY_VF_RX_PROD)); +} + +static void +ecore_iov_pf_update_tun_response(struct pfvf_update_tunn_param_tlv *p_resp, + struct ecore_tunnel_info *p_tun, + u16 tunn_feature_mask) +{ + p_resp->tunn_feature_mask = tunn_feature_mask; + p_resp->vxlan_mode = p_tun->vxlan.b_mode_enabled; + p_resp->l2geneve_mode = p_tun->l2_geneve.b_mode_enabled; + p_resp->ipgeneve_mode = p_tun->ip_geneve.b_mode_enabled; + p_resp->l2gre_mode = p_tun->l2_gre.b_mode_enabled; + p_resp->ipgre_mode = p_tun->l2_gre.b_mode_enabled; + p_resp->vxlan_clss = p_tun->vxlan.tun_cls; + p_resp->l2gre_clss = p_tun->l2_gre.tun_cls; + p_resp->ipgre_clss = p_tun->ip_gre.tun_cls; + p_resp->l2geneve_clss = p_tun->l2_geneve.tun_cls; + p_resp->ipgeneve_clss = p_tun->ip_geneve.tun_cls; + p_resp->geneve_udp_port = p_tun->geneve_port.port; + p_resp->vxlan_udp_port = p_tun->vxlan_port.port; +} + +static void +__ecore_iov_pf_update_tun_param(struct vfpf_update_tunn_param_tlv *p_req, + struct ecore_tunn_update_type *p_tun, + enum ecore_tunn_mode mask, u8 tun_cls) +{ + if (p_req->tun_mode_update_mask & (1 << mask)) { + p_tun->b_update_mode = true; + + if (p_req->tunn_mode & (1 << mask)) + p_tun->b_mode_enabled = true; + } + + p_tun->tun_cls = tun_cls; +} + +static void +ecore_iov_pf_update_tun_param(struct vfpf_update_tunn_param_tlv *p_req, + struct ecore_tunn_update_type *p_tun, + struct ecore_tunn_update_udp_port *p_port, + enum ecore_tunn_mode mask, + u8 tun_cls, u8 update_port, u16 port) +{ + if (update_port) { + p_port->b_update_port = true; + p_port->port = port; + } + + __ecore_iov_pf_update_tun_param(p_req, p_tun, mask, tun_cls); +} + +static bool +ecore_iov_pf_validate_tunn_param(struct vfpf_update_tunn_param_tlv *p_req) +{ + bool b_update_requested = false; + + if (p_req->tun_mode_update_mask || p_req->update_tun_cls || + p_req->update_geneve_port || p_req->update_vxlan_port) + b_update_requested = true; + + return b_update_requested; +} + +static void ecore_iov_vf_mbx_update_tunn_param(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + struct ecore_vf_info *p_vf) +{ + struct ecore_tunnel_info *p_tun = &p_hwfn->p_dev->tunnel; + struct ecore_iov_vf_mbx *mbx = &p_vf->vf_mbx; + struct pfvf_update_tunn_param_tlv *p_resp; + struct vfpf_update_tunn_param_tlv *p_req; + enum _ecore_status_t rc = ECORE_SUCCESS; + u8 status = PFVF_STATUS_SUCCESS; + bool b_update_required = false; + struct ecore_tunnel_info tunn; + u16 tunn_feature_mask = 0; + int i; + + mbx->offset = (u8 *)mbx->reply_virt; + + OSAL_MEM_ZERO(&tunn, sizeof(tunn)); + p_req = &mbx->req_virt->tunn_param_update; + + if (!ecore_iov_pf_validate_tunn_param(p_req)) { + DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, + "No tunnel update requested by VF\n"); + status = PFVF_STATUS_FAILURE; + goto send_resp; + } + + tunn.b_update_rx_cls = p_req->update_tun_cls; + tunn.b_update_tx_cls = p_req->update_tun_cls; + + ecore_iov_pf_update_tun_param(p_req, &tunn.vxlan, &tunn.vxlan_port, + ECORE_MODE_VXLAN_TUNN, p_req->vxlan_clss, + p_req->update_vxlan_port, + p_req->vxlan_port); + ecore_iov_pf_update_tun_param(p_req, &tunn.l2_geneve, &tunn.geneve_port, + ECORE_MODE_L2GENEVE_TUNN, + p_req->l2geneve_clss, + p_req->update_geneve_port, + p_req->geneve_port); + __ecore_iov_pf_update_tun_param(p_req, &tunn.ip_geneve, + ECORE_MODE_IPGENEVE_TUNN, + p_req->ipgeneve_clss); + __ecore_iov_pf_update_tun_param(p_req, &tunn.l2_gre, + ECORE_MODE_L2GRE_TUNN, + p_req->l2gre_clss); + __ecore_iov_pf_update_tun_param(p_req, &tunn.ip_gre, + ECORE_MODE_IPGRE_TUNN, + p_req->ipgre_clss); + + /* If PF modifies VF's req then it should + * still return an error in case of partial configuration + * or modified configuration as opposed to requested one. + */ + rc = OSAL_PF_VALIDATE_MODIFY_TUNN_CONFIG(p_hwfn, &tunn_feature_mask, + &b_update_required, &tunn); + + if (rc != ECORE_SUCCESS) + status = PFVF_STATUS_FAILURE; + + /* If ECORE client is willing to update anything ? */ + if (b_update_required) { + u16 geneve_port; + + rc = ecore_sp_pf_update_tunn_cfg(p_hwfn, p_ptt, &tunn, + ECORE_SPQ_MODE_EBLOCK, + OSAL_NULL); + if (rc != ECORE_SUCCESS) + status = PFVF_STATUS_FAILURE; + + geneve_port = p_tun->geneve_port.port; + ecore_for_each_vf(p_hwfn, i) { + ecore_iov_bulletin_set_udp_ports(p_hwfn, i, + p_tun->vxlan_port.port, + geneve_port); + } + } + +send_resp: + p_resp = ecore_add_tlv(&mbx->offset, + CHANNEL_TLV_UPDATE_TUNN_PARAM, sizeof(*p_resp)); + + ecore_iov_pf_update_tun_response(p_resp, p_tun, tunn_feature_mask); + ecore_add_tlv(&mbx->offset, CHANNEL_TLV_LIST_END, + sizeof(struct channel_list_end_tlv)); + + ecore_iov_send_response(p_hwfn, p_ptt, p_vf, sizeof(*p_resp), status); +} + +static void ecore_iov_vf_mbx_start_txq_resp(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + struct ecore_vf_info *p_vf, + u32 cid, + u8 status) +{ + struct ecore_iov_vf_mbx *mbx = &p_vf->vf_mbx; + struct pfvf_start_queue_resp_tlv *p_tlv; + bool b_legacy = false; + u16 length; + + mbx->offset = (u8 *)mbx->reply_virt; + + /* Taking a bigger struct instead of adding a TLV to list was a + * mistake, but one which we're now stuck with, as some older + * clients assume the size of the previous response. + */ + if (p_vf->acquire.vfdev_info.eth_fp_hsi_minor == + ETH_HSI_VER_NO_PKT_LEN_TUNN) + b_legacy = true; + + if (!b_legacy) + length = sizeof(*p_tlv); + else + length = sizeof(struct pfvf_def_resp_tlv); + + p_tlv = ecore_add_tlv(&mbx->offset, CHANNEL_TLV_START_TXQ, length); + ecore_add_tlv(&mbx->offset, CHANNEL_TLV_LIST_END, + sizeof(struct channel_list_end_tlv)); + + /* Update the TLV with the response */ + if ((status == PFVF_STATUS_SUCCESS) && !b_legacy) + p_tlv->offset = DB_ADDR_VF(cid, DQ_DEMS_LEGACY); + + ecore_iov_send_response(p_hwfn, p_ptt, p_vf, length, status); } static void ecore_iov_vf_mbx_start_txq(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, struct ecore_vf_info *vf) { + struct ecore_queue_start_common_params params; + struct ecore_queue_cid_vf_params vf_params; struct ecore_iov_vf_mbx *mbx = &vf->vf_mbx; - struct vfpf_start_txq_tlv *req = &mbx->req_virt->start_txq; - u16 length = sizeof(struct pfvf_def_resp_tlv); - union ecore_qm_pq_params pq_params; - u8 status = PFVF_STATUS_SUCCESS; + u8 status = PFVF_STATUS_NO_RESOURCE; + struct ecore_vf_queue *p_queue; + struct vfpf_start_txq_tlv *req; + struct ecore_queue_cid *p_cid; + struct ecore_sb_info sb_dummy; + u8 qid_usage_idx, vf_legacy; + u32 cid = 0; enum _ecore_status_t rc; + u16 pq; - /* Prepare the parameters which would choose the right PQ */ - OSAL_MEMSET(&pq_params, 0, sizeof(pq_params)); - pq_params.eth.is_vf = 1; - pq_params.eth.vf_id = vf->relative_vf_id; + OSAL_MEMSET(¶ms, 0, sizeof(params)); + req = &mbx->req_virt->start_txq; - rc = ecore_sp_eth_txq_start_ramrod(p_hwfn, - vf->opaque_fid, - vf->vf_queues[req->tx_qid].fw_tx_qid, - vf->vf_queues[req->tx_qid].fw_cid, - vf->vport_id, - vf->abs_vf_id + 0x10, - req->hw_sb, - req->sb_index, - req->pbl_addr, - req->pbl_size, &pq_params); + if (!ecore_iov_validate_txq(p_hwfn, vf, req->tx_qid, + ECORE_IOV_VALIDATE_Q_NA) || + !ecore_iov_validate_sb(p_hwfn, vf, req->hw_sb)) + goto out; - if (rc) + qid_usage_idx = ecore_iov_vf_mbx_qid(p_hwfn, vf, true); + if (qid_usage_idx == ECORE_IOV_QID_INVALID) + goto out; + + p_queue = &vf->vf_queues[req->tx_qid]; + if (p_queue->cids[qid_usage_idx].p_cid) + goto out; + + vf_legacy = ecore_vf_calculate_legacy(vf); + + /* Acquire a new queue-cid */ + params.queue_id = p_queue->fw_tx_qid; + params.vport_id = vf->vport_id; + params.stats_id = vf->abs_vf_id + 0x10; + + /* Since IGU index is passed via sb_info, construct a dummy one */ + OSAL_MEM_ZERO(&sb_dummy, sizeof(sb_dummy)); + sb_dummy.igu_sb_id = req->hw_sb; + params.p_sb = &sb_dummy; + params.sb_idx = req->sb_index; + + OSAL_MEM_ZERO(&vf_params, sizeof(vf_params)); + vf_params.vfid = vf->relative_vf_id; + vf_params.vf_qid = (u8)req->tx_qid; + vf_params.vf_legacy = vf_legacy; + vf_params.qid_usage_idx = qid_usage_idx; + + p_cid = ecore_eth_queue_to_cid(p_hwfn, vf->opaque_fid, + ¶ms, false, &vf_params); + if (p_cid == OSAL_NULL) + goto out; + + pq = ecore_get_cm_pq_idx_vf(p_hwfn, + vf->relative_vf_id); + rc = ecore_eth_txq_start_ramrod(p_hwfn, p_cid, + req->pbl_addr, req->pbl_size, pq); + if (rc != ECORE_SUCCESS) { status = PFVF_STATUS_FAILURE; - else - vf->vf_queues[req->tx_qid].txq_active = true; + ecore_eth_queue_cid_release(p_hwfn, p_cid); + } else { + status = PFVF_STATUS_SUCCESS; + p_queue->cids[qid_usage_idx].p_cid = p_cid; + p_queue->cids[qid_usage_idx].b_is_tx = true; + cid = p_cid->cid; + } - ecore_iov_prepare_resp(p_hwfn, p_ptt, vf, CHANNEL_TLV_START_TXQ, - length, status); +out: + ecore_iov_vf_mbx_start_txq_resp(p_hwfn, p_ptt, vf, + cid, status); } static enum _ecore_status_t ecore_iov_vf_stop_rxqs(struct ecore_hwfn *p_hwfn, struct ecore_vf_info *vf, u16 rxq_id, - u8 num_rxqs, + u8 qid_usage_idx, bool cqe_completion) { + struct ecore_vf_queue *p_queue; enum _ecore_status_t rc = ECORE_SUCCESS; - int qid; - if (rxq_id + num_rxqs > OSAL_ARRAY_SIZE(vf->vf_queues)) + if (!ecore_iov_validate_rxq(p_hwfn, vf, rxq_id, + ECORE_IOV_VALIDATE_Q_NA)) { + DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, + "VF[%d] Tried Closing Rx 0x%04x.%02x which is inactive\n", + vf->relative_vf_id, rxq_id, qid_usage_idx); return ECORE_INVAL; - - for (qid = rxq_id; qid < rxq_id + num_rxqs; qid++) { - if (vf->vf_queues[qid].rxq_active) { - rc = ecore_sp_eth_rx_queue_stop(p_hwfn, - vf->vf_queues[qid]. - fw_rx_qid, false, - cqe_completion); - - if (rc) - return rc; - } - vf->vf_queues[qid].rxq_active = false; - vf->num_active_rxqs--; } - return rc; + p_queue = &vf->vf_queues[rxq_id]; + + /* We've validated the index and the existence of the active RXQ - + * now we need to make sure that it's using the correct qid. + */ + if (!p_queue->cids[qid_usage_idx].p_cid || + p_queue->cids[qid_usage_idx].b_is_tx) { + struct ecore_queue_cid *p_cid; + + p_cid = ecore_iov_get_vf_rx_queue_cid(p_queue); + DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, + "VF[%d] - Tried Closing Rx 0x%04x.%02x, but Rx is at %04x.%02x\n", + vf->relative_vf_id, rxq_id, qid_usage_idx, + rxq_id, p_cid->qid_usage_idx); + return ECORE_INVAL; + } + + /* Now that we know we have a valid Rx-queue - close it */ + rc = ecore_eth_rx_queue_stop(p_hwfn, + p_queue->cids[qid_usage_idx].p_cid, + false, cqe_completion); + if (rc != ECORE_SUCCESS) + return rc; + + p_queue->cids[qid_usage_idx].p_cid = OSAL_NULL; + vf->num_active_rxqs--; + + return ECORE_SUCCESS; } static enum _ecore_status_t ecore_iov_vf_stop_txqs(struct ecore_hwfn *p_hwfn, struct ecore_vf_info *vf, - u16 txq_id, u8 num_txqs) + u16 txq_id, + u8 qid_usage_idx) { + struct ecore_vf_queue *p_queue; enum _ecore_status_t rc = ECORE_SUCCESS; - int qid; - if (txq_id + num_txqs > OSAL_ARRAY_SIZE(vf->vf_queues)) + if (!ecore_iov_validate_txq(p_hwfn, vf, txq_id, + ECORE_IOV_VALIDATE_Q_NA)) return ECORE_INVAL; - for (qid = txq_id; qid < txq_id + num_txqs; qid++) { - if (vf->vf_queues[qid].txq_active) { - rc = ecore_sp_eth_tx_queue_stop(p_hwfn, - vf->vf_queues[qid]. - fw_tx_qid); + p_queue = &vf->vf_queues[txq_id]; + if (!p_queue->cids[qid_usage_idx].p_cid || + !p_queue->cids[qid_usage_idx].b_is_tx) + return ECORE_INVAL; - if (rc) - return rc; - } - vf->vf_queues[qid].txq_active = false; - } - return rc; + rc = ecore_eth_tx_queue_stop(p_hwfn, + p_queue->cids[qid_usage_idx].p_cid); + if (rc != ECORE_SUCCESS) + return rc; + + p_queue->cids[qid_usage_idx].p_cid = OSAL_NULL; + return ECORE_SUCCESS; } static void ecore_iov_vf_mbx_stop_rxqs(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, struct ecore_vf_info *vf) { - struct ecore_iov_vf_mbx *mbx = &vf->vf_mbx; - struct vfpf_stop_rxqs_tlv *req = &mbx->req_virt->stop_rxqs; u16 length = sizeof(struct pfvf_def_resp_tlv); - u8 status = PFVF_STATUS_SUCCESS; + struct ecore_iov_vf_mbx *mbx = &vf->vf_mbx; + u8 status = PFVF_STATUS_FAILURE; + struct vfpf_stop_rxqs_tlv *req; + u8 qid_usage_idx; enum _ecore_status_t rc; - /* We give the option of starting from qid != 0, in this case we - * need to make sure that qid + num_qs doesn't exceed the actual - * amount of queues that exist. + /* Starting with CHANNEL_TLV_QID, it's assumed the 'num_rxqs' + * would be one. Since no older ecore passed multiple queues + * using this API, sanitize on the value. */ - rc = ecore_iov_vf_stop_rxqs(p_hwfn, vf, req->rx_qid, - req->num_rxqs, req->cqe_completion); - if (rc) - status = PFVF_STATUS_FAILURE; + req = &mbx->req_virt->stop_rxqs; + if (req->num_rxqs != 1) { + DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, + "Odd; VF[%d] tried stopping multiple Rx queues\n", + vf->relative_vf_id); + status = PFVF_STATUS_NOT_SUPPORTED; + goto out; + } + /* Find which qid-index is associated with the queue */ + qid_usage_idx = ecore_iov_vf_mbx_qid(p_hwfn, vf, false); + if (qid_usage_idx == ECORE_IOV_QID_INVALID) + goto out; + + rc = ecore_iov_vf_stop_rxqs(p_hwfn, vf, req->rx_qid, + qid_usage_idx, req->cqe_completion); + if (rc == ECORE_SUCCESS) + status = PFVF_STATUS_SUCCESS; +out: ecore_iov_prepare_resp(p_hwfn, p_ptt, vf, CHANNEL_TLV_STOP_RXQS, length, status); } @@ -1745,20 +2747,37 @@ static void ecore_iov_vf_mbx_stop_txqs(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, struct ecore_vf_info *vf) { - struct ecore_iov_vf_mbx *mbx = &vf->vf_mbx; - struct vfpf_stop_txqs_tlv *req = &mbx->req_virt->stop_txqs; u16 length = sizeof(struct pfvf_def_resp_tlv); - u8 status = PFVF_STATUS_SUCCESS; + struct ecore_iov_vf_mbx *mbx = &vf->vf_mbx; + u8 status = PFVF_STATUS_FAILURE; + struct vfpf_stop_txqs_tlv *req; + u8 qid_usage_idx; enum _ecore_status_t rc; - /* We give the option of starting from qid != 0, in this case we - * need to make sure that qid + num_qs doesn't exceed the actual - * amount of queues that exist. + /* Starting with CHANNEL_TLV_QID, it's assumed the 'num_txqs' + * would be one. Since no older ecore passed multiple queues + * using this API, sanitize on the value. */ - rc = ecore_iov_vf_stop_txqs(p_hwfn, vf, req->tx_qid, req->num_txqs); - if (rc) - status = PFVF_STATUS_FAILURE; + req = &mbx->req_virt->stop_txqs; + if (req->num_txqs != 1) { + DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, + "Odd; VF[%d] tried stopping multiple Tx queues\n", + vf->relative_vf_id); + status = PFVF_STATUS_NOT_SUPPORTED; + goto out; + } + /* Find which qid-index is associated with the queue */ + qid_usage_idx = ecore_iov_vf_mbx_qid(p_hwfn, vf, true); + if (qid_usage_idx == ECORE_IOV_QID_INVALID) + goto out; + + rc = ecore_iov_vf_stop_txqs(p_hwfn, vf, req->tx_qid, + qid_usage_idx); + if (rc == ECORE_SUCCESS) + status = PFVF_STATUS_SUCCESS; + +out: ecore_iov_prepare_resp(p_hwfn, p_ptt, vf, CHANNEL_TLV_STOP_TXQS, length, status); } @@ -1767,43 +2786,70 @@ static void ecore_iov_vf_mbx_update_rxqs(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, struct ecore_vf_info *vf) { - struct ecore_iov_vf_mbx *mbx = &vf->vf_mbx; - struct vfpf_update_rxq_tlv *req = &mbx->req_virt->update_rxq; + struct ecore_queue_cid *handlers[ECORE_MAX_VF_CHAINS_PER_PF]; u16 length = sizeof(struct pfvf_def_resp_tlv); - u8 status = PFVF_STATUS_SUCCESS; + struct ecore_iov_vf_mbx *mbx = &vf->vf_mbx; + struct vfpf_update_rxq_tlv *req; + u8 status = PFVF_STATUS_FAILURE; u8 complete_event_flg; u8 complete_cqe_flg; + u8 qid_usage_idx; enum _ecore_status_t rc; - u16 qid; - u8 i; + u16 i; + req = &mbx->req_virt->update_rxq; complete_cqe_flg = !!(req->flags & VFPF_RXQ_UPD_COMPLETE_CQE_FLAG); complete_event_flg = !!(req->flags & VFPF_RXQ_UPD_COMPLETE_EVENT_FLAG); - for (i = 0; i < req->num_rxqs; i++) { - qid = req->rx_qid + i; + qid_usage_idx = ecore_iov_vf_mbx_qid(p_hwfn, vf, false); + if (qid_usage_idx == ECORE_IOV_QID_INVALID) + goto out; - if (!vf->vf_queues[qid].rxq_active) { - DP_NOTICE(p_hwfn, true, - "VF rx_qid = %d isn`t active!\n", qid); - status = PFVF_STATUS_FAILURE; - break; - } + /* Starting with the addition of CHANNEL_TLV_QID, this API started + * expecting a single queue at a time. Validate this. + */ + if ((vf->acquire.vfdev_info.capabilities & + VFPF_ACQUIRE_CAP_QUEUE_QIDS) && + req->num_rxqs != 1) { + DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, + "VF[%d] supports QIDs but sends multiple queues\n", + vf->relative_vf_id); + goto out; + } - rc = ecore_sp_eth_rx_queues_update(p_hwfn, - vf->vf_queues[qid].fw_rx_qid, - 1, - complete_cqe_flg, - complete_event_flg, - ECORE_SPQ_MODE_EBLOCK, - OSAL_NULL); - - if (rc) { - status = PFVF_STATUS_FAILURE; - break; + /* Validate inputs - for the legacy case this is still true since + * qid_usage_idx for each Rx queue would be LEGACY_QID_RX. + */ + for (i = req->rx_qid; i < req->rx_qid + req->num_rxqs; i++) { + if (!ecore_iov_validate_rxq(p_hwfn, vf, i, + ECORE_IOV_VALIDATE_Q_NA) || + !vf->vf_queues[i].cids[qid_usage_idx].p_cid || + vf->vf_queues[i].cids[qid_usage_idx].b_is_tx) { + DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, + "VF[%d]: Incorrect Rxqs [%04x, %02x]\n", + vf->relative_vf_id, req->rx_qid, + req->num_rxqs); + goto out; } } + for (i = 0; i < req->num_rxqs; i++) { + u16 qid = req->rx_qid + i; + + handlers[i] = vf->vf_queues[qid].cids[qid_usage_idx].p_cid; + } + + rc = ecore_sp_eth_rx_queues_update(p_hwfn, (void **)&handlers, + req->num_rxqs, + complete_cqe_flg, + complete_event_flg, + ECORE_SPQ_MODE_EBLOCK, + OSAL_NULL); + if (rc != ECORE_SUCCESS) + goto out; + + status = PFVF_STATUS_SUCCESS; +out: ecore_iov_prepare_resp(p_hwfn, p_ptt, vf, CHANNEL_TLV_UPDATE_RXQ, length, status); } @@ -1851,13 +2897,14 @@ ecore_iov_vp_update_act_param(struct ecore_hwfn *p_hwfn, p_act_tlv = (struct vfpf_vport_update_activate_tlv *) ecore_iov_search_list_tlvs(p_hwfn, p_mbx->req_virt, tlv); - if (p_act_tlv) { - p_data->update_vport_active_rx_flg = p_act_tlv->update_rx; - p_data->vport_active_rx_flg = p_act_tlv->active_rx; - p_data->update_vport_active_tx_flg = p_act_tlv->update_tx; - p_data->vport_active_tx_flg = p_act_tlv->active_tx; - *tlvs_mask |= 1 << ECORE_IOV_VP_UPDATE_ACTIVATE; - } + if (!p_act_tlv) + return; + + p_data->update_vport_active_rx_flg = p_act_tlv->update_rx; + p_data->vport_active_rx_flg = p_act_tlv->active_rx; + p_data->update_vport_active_tx_flg = p_act_tlv->update_tx; + p_data->vport_active_tx_flg = p_act_tlv->active_tx; + *tlvs_mask |= 1 << ECORE_IOV_VP_UPDATE_ACTIVATE; } static void @@ -1895,20 +2942,21 @@ ecore_iov_vp_update_tx_switch(struct ecore_hwfn *p_hwfn, p_tx_switch_tlv = (struct vfpf_vport_update_tx_switch_tlv *) ecore_iov_search_list_tlvs(p_hwfn, p_mbx->req_virt, tlv); + if (!p_tx_switch_tlv) + return; #ifndef ASIC_ONLY if (CHIP_REV_IS_FPGA(p_hwfn->p_dev)) { DP_NOTICE(p_hwfn, false, - "FPGA: Ignore tx-switching configuration originating from VFs\n"); + "FPGA: Ignore tx-switching configuration originating" + " from VFs\n"); return; } #endif - if (p_tx_switch_tlv) { - p_data->update_tx_switching_flg = 1; - p_data->tx_switching_flg = p_tx_switch_tlv->tx_switching; - *tlvs_mask |= 1 << ECORE_IOV_VP_UPDATE_TX_SWITCH; - } + p_data->update_tx_switching_flg = 1; + p_data->tx_switching_flg = p_tx_switch_tlv->tx_switching; + *tlvs_mask |= 1 << ECORE_IOV_VP_UPDATE_TX_SWITCH; } static void @@ -1922,14 +2970,14 @@ ecore_iov_vp_update_mcast_bin_param(struct ecore_hwfn *p_hwfn, p_mcast_tlv = (struct vfpf_vport_update_mcast_bin_tlv *) ecore_iov_search_list_tlvs(p_hwfn, p_mbx->req_virt, tlv); + if (!p_mcast_tlv) + return; - if (p_mcast_tlv) { - p_data->update_approx_mcast_flg = 1; - OSAL_MEMCPY(p_data->bins, p_mcast_tlv->bins, - sizeof(unsigned long) * - ETH_MULTICAST_MAC_BINS_IN_REGS); - *tlvs_mask |= 1 << ECORE_IOV_VP_UPDATE_MCAST; - } + p_data->update_approx_mcast_flg = 1; + OSAL_MEMCPY(p_data->bins, p_mcast_tlv->bins, + sizeof(unsigned long) * + ETH_MULTICAST_MAC_BINS_IN_REGS); + *tlvs_mask |= 1 << ECORE_IOV_VP_UPDATE_MCAST; } static void @@ -1937,23 +2985,20 @@ ecore_iov_vp_update_accept_flag(struct ecore_hwfn *p_hwfn, struct ecore_sp_vport_update_params *p_data, struct ecore_iov_vf_mbx *p_mbx, u16 *tlvs_mask) { + struct ecore_filter_accept_flags *p_flags = &p_data->accept_flags; struct vfpf_vport_update_accept_param_tlv *p_accept_tlv; u16 tlv = CHANNEL_TLV_VPORT_UPDATE_ACCEPT_PARAM; p_accept_tlv = (struct vfpf_vport_update_accept_param_tlv *) ecore_iov_search_list_tlvs(p_hwfn, p_mbx->req_virt, tlv); + if (!p_accept_tlv) + return; - if (p_accept_tlv) { - p_data->accept_flags.update_rx_mode_config = - p_accept_tlv->update_rx_mode; - p_data->accept_flags.rx_accept_filter = - p_accept_tlv->rx_accept_filter; - p_data->accept_flags.update_tx_mode_config = - p_accept_tlv->update_tx_mode; - p_data->accept_flags.tx_accept_filter = - p_accept_tlv->tx_accept_filter; - *tlvs_mask |= 1 << ECORE_IOV_VP_UPDATE_ACCEPT_PARAM; - } + p_flags->update_rx_mode_config = p_accept_tlv->update_rx_mode; + p_flags->rx_accept_filter = p_accept_tlv->rx_accept_filter; + p_flags->update_tx_mode_config = p_accept_tlv->update_tx_mode; + p_flags->tx_accept_filter = p_accept_tlv->tx_accept_filter; + *tlvs_mask |= 1 << ECORE_IOV_VP_UPDATE_ACCEPT_PARAM; } static void @@ -1967,13 +3012,13 @@ ecore_iov_vp_update_accept_any_vlan(struct ecore_hwfn *p_hwfn, p_accept_any_vlan = (struct vfpf_vport_update_accept_any_vlan_tlv *) ecore_iov_search_list_tlvs(p_hwfn, p_mbx->req_virt, tlv); + if (!p_accept_any_vlan) + return; - if (p_accept_any_vlan) { - p_data->accept_any_vlan = p_accept_any_vlan->accept_any_vlan; - p_data->update_accept_any_vlan_flg = - p_accept_any_vlan->update_accept_any_vlan_flg; - *tlvs_mask |= 1 << ECORE_IOV_VP_UPDATE_ACCEPT_ANY_VLAN; - } + p_data->accept_any_vlan = p_accept_any_vlan->accept_any_vlan; + p_data->update_accept_any_vlan_flg = + p_accept_any_vlan->update_accept_any_vlan_flg; + *tlvs_mask |= 1 << ECORE_IOV_VP_UPDATE_ACCEPT_ANY_VLAN; } static void @@ -1981,77 +3026,73 @@ ecore_iov_vp_update_rss_param(struct ecore_hwfn *p_hwfn, struct ecore_vf_info *vf, struct ecore_sp_vport_update_params *p_data, struct ecore_rss_params *p_rss, - struct ecore_iov_vf_mbx *p_mbx, u16 *tlvs_mask) + struct ecore_iov_vf_mbx *p_mbx, + u16 *tlvs_mask, u16 *tlvs_accepted) { struct vfpf_vport_update_rss_tlv *p_rss_tlv; u16 tlv = CHANNEL_TLV_VPORT_UPDATE_RSS; + bool b_reject = false; u16 table_size; - u16 i, q_idx, max_q_idx; + u16 i, q_idx; p_rss_tlv = (struct vfpf_vport_update_rss_tlv *) ecore_iov_search_list_tlvs(p_hwfn, p_mbx->req_virt, tlv); - if (p_rss_tlv) { - OSAL_MEMSET(p_rss, 0, sizeof(struct ecore_rss_params)); + if (!p_rss_tlv) { + p_data->rss_params = OSAL_NULL; + return; + } - p_rss->update_rss_config = - !!(p_rss_tlv->update_rss_flags & - VFPF_UPDATE_RSS_CONFIG_FLAG); - p_rss->update_rss_capabilities = - !!(p_rss_tlv->update_rss_flags & - VFPF_UPDATE_RSS_CAPS_FLAG); - p_rss->update_rss_ind_table = - !!(p_rss_tlv->update_rss_flags & - VFPF_UPDATE_RSS_IND_TABLE_FLAG); - p_rss->update_rss_key = - !!(p_rss_tlv->update_rss_flags & VFPF_UPDATE_RSS_KEY_FLAG); + OSAL_MEMSET(p_rss, 0, sizeof(struct ecore_rss_params)); - p_rss->rss_enable = p_rss_tlv->rss_enable; - p_rss->rss_eng_id = vf->relative_vf_id + 1; - p_rss->rss_caps = p_rss_tlv->rss_caps; - p_rss->rss_table_size_log = p_rss_tlv->rss_table_size_log; - OSAL_MEMCPY(p_rss->rss_ind_table, p_rss_tlv->rss_ind_table, - sizeof(p_rss->rss_ind_table)); - OSAL_MEMCPY(p_rss->rss_key, p_rss_tlv->rss_key, - sizeof(p_rss->rss_key)); + p_rss->update_rss_config = + !!(p_rss_tlv->update_rss_flags & + VFPF_UPDATE_RSS_CONFIG_FLAG); + p_rss->update_rss_capabilities = + !!(p_rss_tlv->update_rss_flags & + VFPF_UPDATE_RSS_CAPS_FLAG); + p_rss->update_rss_ind_table = + !!(p_rss_tlv->update_rss_flags & + VFPF_UPDATE_RSS_IND_TABLE_FLAG); + p_rss->update_rss_key = + !!(p_rss_tlv->update_rss_flags & + VFPF_UPDATE_RSS_KEY_FLAG); - table_size = OSAL_MIN_T(u16, - OSAL_ARRAY_SIZE(p_rss->rss_ind_table), - (1 << p_rss_tlv->rss_table_size_log)); + p_rss->rss_enable = p_rss_tlv->rss_enable; + p_rss->rss_eng_id = vf->rss_eng_id; + p_rss->rss_caps = p_rss_tlv->rss_caps; + p_rss->rss_table_size_log = p_rss_tlv->rss_table_size_log; + OSAL_MEMCPY(p_rss->rss_key, p_rss_tlv->rss_key, + sizeof(p_rss->rss_key)); - max_q_idx = OSAL_ARRAY_SIZE(vf->vf_queues); + table_size = OSAL_MIN_T(u16, OSAL_ARRAY_SIZE(p_rss->rss_ind_table), + (1 << p_rss_tlv->rss_table_size_log)); - for (i = 0; i < table_size; i++) { - q_idx = p_rss->rss_ind_table[i]; - if (q_idx >= max_q_idx) { - DP_NOTICE(p_hwfn, true, - "rss_ind_table[%d] = %d, rxq is out of range\n", - i, q_idx); - /* TBD: fail the request mark VF as malicious */ - p_rss->rss_ind_table[i] = - vf->vf_queues[0].fw_rx_qid; - } else if (!vf->vf_queues[q_idx].rxq_active) { - DP_NOTICE(p_hwfn, true, - "rss_ind_table[%d] = %d, rxq is not active\n", - i, q_idx); - /* TBD: fail the request mark VF as malicious */ - p_rss->rss_ind_table[i] = - vf->vf_queues[0].fw_rx_qid; - } else { - p_rss->rss_ind_table[i] = - vf->vf_queues[q_idx].fw_rx_qid; - } + for (i = 0; i < table_size; i++) { + struct ecore_queue_cid *p_cid; + + q_idx = p_rss_tlv->rss_ind_table[i]; + if (!ecore_iov_validate_rxq(p_hwfn, vf, q_idx, + ECORE_IOV_VALIDATE_Q_ENABLE)) { + DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, + "VF[%d]: Omitting RSS due to wrong queue %04x\n", + vf->relative_vf_id, q_idx); + b_reject = true; + goto out; } - p_data->rss_params = p_rss; - *tlvs_mask |= 1 << ECORE_IOV_VP_UPDATE_RSS; - } else { - p_data->rss_params = OSAL_NULL; + p_cid = ecore_iov_get_vf_rx_queue_cid(&vf->vf_queues[q_idx]); + p_rss->rss_ind_table[i] = p_cid; } + + p_data->rss_params = p_rss; +out: + *tlvs_mask |= 1 << ECORE_IOV_VP_UPDATE_RSS; + if (!b_reject) + *tlvs_accepted |= 1 << ECORE_IOV_VP_UPDATE_RSS; } static void ecore_iov_vp_update_sge_tpa_param(struct ecore_hwfn *p_hwfn, - struct ecore_vf_info *vf, struct ecore_sp_vport_update_params *p_data, struct ecore_sge_tpa_params *p_sge_tpa, struct ecore_iov_vf_mbx *p_mbx, @@ -2102,14 +3143,30 @@ static void ecore_iov_vf_mbx_vport_update(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, struct ecore_vf_info *vf) { + struct ecore_rss_params *p_rss_params = OSAL_NULL; struct ecore_sp_vport_update_params params; struct ecore_iov_vf_mbx *mbx = &vf->vf_mbx; struct ecore_sge_tpa_params sge_tpa_params; - struct ecore_rss_params rss_params; + u16 tlvs_mask = 0, tlvs_accepted = 0; u8 status = PFVF_STATUS_SUCCESS; - enum _ecore_status_t rc; - u16 tlvs_mask = 0, tlvs_accepted; u16 length; + enum _ecore_status_t rc; + + /* Valiate PF can send such a request */ + if (!vf->vport_instance) { + DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, + "No VPORT instance available for VF[%d]," + " failing vport update\n", + vf->abs_vf_id); + status = PFVF_STATUS_FAILURE; + goto out; + } + + p_rss_params = OSAL_VZALLOC(p_hwfn->p_dev, sizeof(*p_rss_params)); + if (p_rss_params == OSAL_NULL) { + status = PFVF_STATUS_FAILURE; + goto out; + } OSAL_MEMSET(¶ms, 0, sizeof(params)); params.opaque_fid = vf->opaque_fid; @@ -2124,20 +3181,24 @@ static void ecore_iov_vf_mbx_vport_update(struct ecore_hwfn *p_hwfn, ecore_iov_vp_update_tx_switch(p_hwfn, ¶ms, mbx, &tlvs_mask); ecore_iov_vp_update_mcast_bin_param(p_hwfn, ¶ms, mbx, &tlvs_mask); ecore_iov_vp_update_accept_flag(p_hwfn, ¶ms, mbx, &tlvs_mask); - ecore_iov_vp_update_rss_param(p_hwfn, vf, ¶ms, &rss_params, - mbx, &tlvs_mask); ecore_iov_vp_update_accept_any_vlan(p_hwfn, ¶ms, mbx, &tlvs_mask); - ecore_iov_vp_update_sge_tpa_param(p_hwfn, vf, ¶ms, + ecore_iov_vp_update_sge_tpa_param(p_hwfn, ¶ms, &sge_tpa_params, mbx, &tlvs_mask); + tlvs_accepted = tlvs_mask; + + /* Some of the extended TLVs need to be validated first; In that case, + * they can update the mask without updating the accepted [so that + * PF could communicate to VF it has rejected request]. + */ + ecore_iov_vp_update_rss_param(p_hwfn, vf, ¶ms, p_rss_params, + mbx, &tlvs_mask, &tlvs_accepted); + /* Just log a message if there is no single extended tlv in buffer. * When all features of vport update ramrod would be requested by VF * as extended TLVs in buffer then an error can be returned in response * if there is no extended TLV present in buffer. */ - tlvs_accepted = tlvs_mask; - -#ifndef __EXTRACT__LINUX__ if (OSAL_IOV_VF_VPORT_UPDATE(p_hwfn, vf->relative_vf_id, ¶ms, &tlvs_accepted) != ECORE_SUCCESS) { @@ -2145,15 +3206,15 @@ static void ecore_iov_vf_mbx_vport_update(struct ecore_hwfn *p_hwfn, status = PFVF_STATUS_NOT_SUPPORTED; goto out; } -#endif if (!tlvs_accepted) { if (tlvs_mask) DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, - "Upper-layer prevents said VF configuration\n"); + "Upper-layer prevents said VF" + " configuration\n"); else - DP_NOTICE(p_hwfn, true, - "No feature tlvs found for vport update\n"); + DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, + "No feature tlvs found for vport update\n"); status = PFVF_STATUS_NOT_SUPPORTED; goto out; } @@ -2165,22 +3226,19 @@ static void ecore_iov_vf_mbx_vport_update(struct ecore_hwfn *p_hwfn, status = PFVF_STATUS_FAILURE; out: + OSAL_VFREE(p_hwfn->p_dev, p_rss_params); length = ecore_iov_prep_vp_update_resp_tlvs(p_hwfn, vf, mbx, status, tlvs_mask, tlvs_accepted); ecore_iov_send_response(p_hwfn, p_ptt, vf, length, status); } static enum _ecore_status_t -ecore_iov_vf_update_unicast_shadow(struct ecore_hwfn *p_hwfn, - struct ecore_vf_info *p_vf, - struct ecore_filter_ucast *p_params) +ecore_iov_vf_update_vlan_shadow(struct ecore_hwfn *p_hwfn, + struct ecore_vf_info *p_vf, + struct ecore_filter_ucast *p_params) { int i; - /* TODO - do we need a MAC shadow registery? */ - if (p_params->type == ECORE_FILTER_MAC) - return ECORE_SUCCESS; - /* First remove entries and then add new ones */ if (p_params->opcode == ECORE_FILTER_REMOVE) { for (i = 0; i < ECORE_ETH_VF_NUM_VLAN_FILTERS + 1; i++) @@ -2192,7 +3250,8 @@ ecore_iov_vf_update_unicast_shadow(struct ecore_hwfn *p_hwfn, } if (i == ECORE_ETH_VF_NUM_VLAN_FILTERS + 1) { DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, - "VF [%d] - Tries to remove a non-existing vlan\n", + "VF [%d] - Tries to remove a non-existing" + " vlan\n", p_vf->relative_vf_id); return ECORE_INVAL; } @@ -2210,16 +3269,19 @@ ecore_iov_vf_update_unicast_shadow(struct ecore_hwfn *p_hwfn, if (p_params->opcode == ECORE_FILTER_ADD || p_params->opcode == ECORE_FILTER_REPLACE) { - for (i = 0; i < ECORE_ETH_VF_NUM_VLAN_FILTERS + 1; i++) - if (!p_vf->shadow_config.vlans[i].used) { - p_vf->shadow_config.vlans[i].used = true; - p_vf->shadow_config.vlans[i].vid = - p_params->vlan; - break; - } + for (i = 0; i < ECORE_ETH_VF_NUM_VLAN_FILTERS + 1; i++) { + if (p_vf->shadow_config.vlans[i].used) + continue; + + p_vf->shadow_config.vlans[i].used = true; + p_vf->shadow_config.vlans[i].vid = p_params->vlan; + break; + } + if (i == ECORE_ETH_VF_NUM_VLAN_FILTERS + 1) { DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, - "VF [%d] - Tries to configure more than %d vlan filters\n", + "VF [%d] - Tries to configure more than %d" + " vlan filters\n", p_vf->relative_vf_id, ECORE_ETH_VF_NUM_VLAN_FILTERS + 1); return ECORE_INVAL; @@ -2229,19 +3291,104 @@ ecore_iov_vf_update_unicast_shadow(struct ecore_hwfn *p_hwfn, return ECORE_SUCCESS; } +static enum _ecore_status_t +ecore_iov_vf_update_mac_shadow(struct ecore_hwfn *p_hwfn, + struct ecore_vf_info *p_vf, + struct ecore_filter_ucast *p_params) +{ + char empty_mac[ETH_ALEN]; + int i; + + OSAL_MEM_ZERO(empty_mac, ETH_ALEN); + + /* If we're in forced-mode, we don't allow any change */ + /* TODO - this would change if we were ever to implement logic for + * removing a forced MAC altogether [in which case, like for vlans, + * we should be able to re-trace previous configuration. + */ + if (p_vf->bulletin.p_virt->valid_bitmap & (1 << MAC_ADDR_FORCED)) + return ECORE_SUCCESS; + + /* First remove entries and then add new ones */ + if (p_params->opcode == ECORE_FILTER_REMOVE) { + for (i = 0; i < ECORE_ETH_VF_NUM_MAC_FILTERS; i++) { + if (!OSAL_MEMCMP(p_vf->shadow_config.macs[i], + p_params->mac, ETH_ALEN)) { + OSAL_MEM_ZERO(p_vf->shadow_config.macs[i], + ETH_ALEN); + break; + } + } + + if (i == ECORE_ETH_VF_NUM_MAC_FILTERS) { + DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, + "MAC isn't configured\n"); + return ECORE_INVAL; + } + } else if (p_params->opcode == ECORE_FILTER_REPLACE || + p_params->opcode == ECORE_FILTER_FLUSH) { + for (i = 0; i < ECORE_ETH_VF_NUM_MAC_FILTERS; i++) + OSAL_MEM_ZERO(p_vf->shadow_config.macs[i], ETH_ALEN); + } + + /* List the new MAC address */ + if (p_params->opcode != ECORE_FILTER_ADD && + p_params->opcode != ECORE_FILTER_REPLACE) + return ECORE_SUCCESS; + + for (i = 0; i < ECORE_ETH_VF_NUM_MAC_FILTERS; i++) { + if (!OSAL_MEMCMP(p_vf->shadow_config.macs[i], + empty_mac, ETH_ALEN)) { + OSAL_MEMCPY(p_vf->shadow_config.macs[i], + p_params->mac, ETH_ALEN); + DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, + "Added MAC at %d entry in shadow\n", i); + break; + } + } + + if (i == ECORE_ETH_VF_NUM_MAC_FILTERS) { + DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, + "No available place for MAC\n"); + return ECORE_INVAL; + } + + return ECORE_SUCCESS; +} + +static enum _ecore_status_t +ecore_iov_vf_update_unicast_shadow(struct ecore_hwfn *p_hwfn, + struct ecore_vf_info *p_vf, + struct ecore_filter_ucast *p_params) +{ + enum _ecore_status_t rc = ECORE_SUCCESS; + + if (p_params->type == ECORE_FILTER_MAC) { + rc = ecore_iov_vf_update_mac_shadow(p_hwfn, p_vf, p_params); + if (rc != ECORE_SUCCESS) + return rc; + } + + if (p_params->type == ECORE_FILTER_VLAN) + rc = ecore_iov_vf_update_vlan_shadow(p_hwfn, p_vf, p_params); + + return rc; +} + static void ecore_iov_vf_mbx_ucast_filter(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, struct ecore_vf_info *vf) { - struct ecore_iov_vf_mbx *mbx = &vf->vf_mbx; - struct vfpf_ucast_filter_tlv *req = &mbx->req_virt->ucast_filter; struct ecore_bulletin_content *p_bulletin = vf->bulletin.p_virt; - struct ecore_filter_ucast params; + struct ecore_iov_vf_mbx *mbx = &vf->vf_mbx; + struct vfpf_ucast_filter_tlv *req; u8 status = PFVF_STATUS_SUCCESS; + struct ecore_filter_ucast params; enum _ecore_status_t rc; /* Prepare the unicast filter params */ OSAL_MEMSET(¶ms, 0, sizeof(struct ecore_filter_ucast)); + req = &mbx->req_virt->ucast_filter; params.opcode = (enum ecore_filter_opcode)req->opcode; params.type = (enum ecore_filter_ucast_type)req->type; @@ -2254,7 +3401,8 @@ static void ecore_iov_vf_mbx_ucast_filter(struct ecore_hwfn *p_hwfn, params.vlan = req->vlan; DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, - "VF[%d]: opcode 0x%02x type 0x%02x [%s %s] [vport 0x%02x] MAC %02x:%02x:%02x:%02x:%02x:%02x, vlan 0x%04x\n", + "VF[%d]: opcode 0x%02x type 0x%02x [%s %s] [vport 0x%02x]" + " MAC %02x:%02x:%02x:%02x:%02x:%02x, vlan 0x%04x\n", vf->abs_vf_id, params.opcode, params.type, params.is_rx_filter ? "RX" : "", params.is_tx_filter ? "TX" : "", @@ -2264,18 +3412,20 @@ static void ecore_iov_vf_mbx_ucast_filter(struct ecore_hwfn *p_hwfn, if (!vf->vport_instance) { DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, - "No VPORT instance available for VF[%d], failing ucast MAC configuration\n", + "No VPORT instance available for VF[%d]," + " failing ucast MAC configuration\n", vf->abs_vf_id); status = PFVF_STATUS_FAILURE; goto out; } - /* Update shadow copy of the VF configuration */ + /* Update shadow copy of the VF configuration. In case shadow indicates + * the action should be blocked return success to VF to imitate the + * firmware behaviour in such case. + */ if (ecore_iov_vf_update_unicast_shadow(p_hwfn, vf, ¶ms) != - ECORE_SUCCESS) { - status = PFVF_STATUS_FAILURE; + ECORE_SUCCESS) goto out; - } /* Determine if the unicast filtering is acceptible by PF */ if ((p_bulletin->valid_bitmap & (1 << VLAN_ADDR_FORCED)) && @@ -2343,10 +3493,10 @@ static void ecore_iov_vf_mbx_close(struct ecore_hwfn *p_hwfn, u8 status = PFVF_STATUS_SUCCESS; /* Disable Interrupts for VF */ - ecore_iov_vf_igu_set_int(p_hwfn, p_ptt, vf, 0 /* disable */); + ecore_iov_vf_igu_set_int(p_hwfn, p_ptt, vf, 0); /* Reset Permission table */ - ecore_iov_config_perm_table(p_hwfn, p_ptt, vf, 0 /* disable */); + ecore_iov_config_perm_table(p_hwfn, p_ptt, vf, 0); ecore_iov_prepare_resp(p_hwfn, p_ptt, vf, CHANNEL_TLV_CLOSE, length, status); @@ -2357,11 +3507,266 @@ static void ecore_iov_vf_mbx_release(struct ecore_hwfn *p_hwfn, struct ecore_vf_info *p_vf) { u16 length = sizeof(struct pfvf_def_resp_tlv); + u8 status = PFVF_STATUS_SUCCESS; + enum _ecore_status_t rc = ECORE_SUCCESS; ecore_iov_vf_cleanup(p_hwfn, p_vf); + if (p_vf->state != VF_STOPPED && p_vf->state != VF_FREE) { + /* Stopping the VF */ + rc = ecore_sp_vf_stop(p_hwfn, p_vf->concrete_fid, + p_vf->opaque_fid); + + if (rc != ECORE_SUCCESS) { + DP_ERR(p_hwfn, "ecore_sp_vf_stop returned error %d\n", + rc); + status = PFVF_STATUS_FAILURE; + } + + p_vf->state = VF_STOPPED; + } + ecore_iov_prepare_resp(p_hwfn, p_ptt, p_vf, CHANNEL_TLV_RELEASE, - length, PFVF_STATUS_SUCCESS); + length, status); +} + +static void ecore_iov_vf_pf_get_coalesce(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + struct ecore_vf_info *p_vf) +{ + struct ecore_iov_vf_mbx *mbx = &p_vf->vf_mbx; + struct pfvf_read_coal_resp_tlv *p_resp; + struct vfpf_read_coal_req_tlv *req; + u8 status = PFVF_STATUS_FAILURE; + struct ecore_vf_queue *p_queue; + struct ecore_queue_cid *p_cid; + enum _ecore_status_t rc = ECORE_SUCCESS; + u16 coal = 0, qid, i; + bool b_is_rx; + + mbx->offset = (u8 *)mbx->reply_virt; + req = &mbx->req_virt->read_coal_req; + + qid = req->qid; + b_is_rx = req->is_rx ? true : false; + + if (b_is_rx) { + if (!ecore_iov_validate_rxq(p_hwfn, p_vf, qid, + ECORE_IOV_VALIDATE_Q_ENABLE)) { + DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, + "VF[%d]: Invalid Rx queue_id = %d\n", + p_vf->abs_vf_id, qid); + goto send_resp; + } + + p_cid = ecore_iov_get_vf_rx_queue_cid(&p_vf->vf_queues[qid]); + rc = ecore_get_rxq_coalesce(p_hwfn, p_ptt, p_cid, &coal); + if (rc != ECORE_SUCCESS) + goto send_resp; + } else { + if (!ecore_iov_validate_txq(p_hwfn, p_vf, qid, + ECORE_IOV_VALIDATE_Q_ENABLE)) { + DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, + "VF[%d]: Invalid Tx queue_id = %d\n", + p_vf->abs_vf_id, qid); + goto send_resp; + } + for (i = 0; i < MAX_QUEUES_PER_QZONE; i++) { + p_queue = &p_vf->vf_queues[qid]; + if ((p_queue->cids[i].p_cid == OSAL_NULL) || + (!p_queue->cids[i].b_is_tx)) + continue; + + p_cid = p_queue->cids[i].p_cid; + + rc = ecore_get_txq_coalesce(p_hwfn, p_ptt, + p_cid, &coal); + if (rc != ECORE_SUCCESS) + goto send_resp; + break; + } + } + + status = PFVF_STATUS_SUCCESS; + +send_resp: + p_resp = ecore_add_tlv(&mbx->offset, CHANNEL_TLV_COALESCE_READ, + sizeof(*p_resp)); + p_resp->coal = coal; + + ecore_add_tlv(&mbx->offset, CHANNEL_TLV_LIST_END, + sizeof(struct channel_list_end_tlv)); + + ecore_iov_send_response(p_hwfn, p_ptt, p_vf, sizeof(*p_resp), status); +} + +static void ecore_iov_vf_pf_set_coalesce(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + struct ecore_vf_info *vf) +{ + struct ecore_iov_vf_mbx *mbx = &vf->vf_mbx; + enum _ecore_status_t rc = ECORE_SUCCESS; + struct vfpf_update_coalesce *req; + u8 status = PFVF_STATUS_FAILURE; + struct ecore_queue_cid *p_cid; + u16 rx_coal, tx_coal; + u16 qid; + int i; + + req = &mbx->req_virt->update_coalesce; + + rx_coal = req->rx_coal; + tx_coal = req->tx_coal; + qid = req->qid; + + if (!ecore_iov_validate_rxq(p_hwfn, vf, qid, + ECORE_IOV_VALIDATE_Q_ENABLE) && + rx_coal) { + DP_ERR(p_hwfn, "VF[%d]: Invalid Rx queue_id = %d\n", + vf->abs_vf_id, qid); + goto out; + } + + if (!ecore_iov_validate_txq(p_hwfn, vf, qid, + ECORE_IOV_VALIDATE_Q_ENABLE) && + tx_coal) { + DP_ERR(p_hwfn, "VF[%d]: Invalid Tx queue_id = %d\n", + vf->abs_vf_id, qid); + goto out; + } + + DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, + "VF[%d]: Setting coalesce for VF rx_coal = %d, tx_coal = %d at queue = %d\n", + vf->abs_vf_id, rx_coal, tx_coal, qid); + + if (rx_coal) { + p_cid = ecore_iov_get_vf_rx_queue_cid(&vf->vf_queues[qid]); + + rc = ecore_set_rxq_coalesce(p_hwfn, p_ptt, rx_coal, p_cid); + if (rc != ECORE_SUCCESS) { + DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, + "VF[%d]: Unable to set rx queue = %d coalesce\n", + vf->abs_vf_id, vf->vf_queues[qid].fw_rx_qid); + goto out; + } + vf->rx_coal = rx_coal; + } + + /* TODO - in future, it might be possible to pass this in a per-cid + * granularity. For now, do this for all Tx queues. + */ + if (tx_coal) { + struct ecore_vf_queue *p_queue = &vf->vf_queues[qid]; + + for (i = 0; i < MAX_QUEUES_PER_QZONE; i++) { + if (p_queue->cids[i].p_cid == OSAL_NULL) + continue; + + if (!p_queue->cids[i].b_is_tx) + continue; + + rc = ecore_set_txq_coalesce(p_hwfn, p_ptt, tx_coal, + p_queue->cids[i].p_cid); + if (rc != ECORE_SUCCESS) { + DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, + "VF[%d]: Unable to set tx queue coalesce\n", + vf->abs_vf_id); + goto out; + } + } + vf->tx_coal = tx_coal; + } + + status = PFVF_STATUS_SUCCESS; +out: + ecore_iov_prepare_resp(p_hwfn, p_ptt, vf, CHANNEL_TLV_COALESCE_UPDATE, + sizeof(struct pfvf_def_resp_tlv), status); +} + +enum _ecore_status_t +ecore_iov_pf_configure_vf_queue_coalesce(struct ecore_hwfn *p_hwfn, + u16 rx_coal, u16 tx_coal, + u16 vf_id, u16 qid) +{ + struct ecore_queue_cid *p_cid; + struct ecore_vf_info *vf; + struct ecore_ptt *p_ptt; + int i, rc = 0; + + if (!ecore_iov_is_valid_vfid(p_hwfn, vf_id, true, true)) { + DP_NOTICE(p_hwfn, true, + "VF[%d] - Can not set coalescing: VF is not active\n", + vf_id); + return ECORE_INVAL; + } + + vf = &p_hwfn->pf_iov_info->vfs_array[vf_id]; + p_ptt = ecore_ptt_acquire(p_hwfn); + if (!p_ptt) + return ECORE_AGAIN; + + if (!ecore_iov_validate_rxq(p_hwfn, vf, qid, + ECORE_IOV_VALIDATE_Q_ENABLE) && + rx_coal) { + DP_ERR(p_hwfn, "VF[%d]: Invalid Rx queue_id = %d\n", + vf->abs_vf_id, qid); + goto out; + } + + if (!ecore_iov_validate_txq(p_hwfn, vf, qid, + ECORE_IOV_VALIDATE_Q_ENABLE) && + tx_coal) { + DP_ERR(p_hwfn, "VF[%d]: Invalid Tx queue_id = %d\n", + vf->abs_vf_id, qid); + goto out; + } + + DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, + "VF[%d]: Setting coalesce for VF rx_coal = %d, tx_coal = %d at queue = %d\n", + vf->abs_vf_id, rx_coal, tx_coal, qid); + + if (rx_coal) { + p_cid = ecore_iov_get_vf_rx_queue_cid(&vf->vf_queues[qid]); + + rc = ecore_set_rxq_coalesce(p_hwfn, p_ptt, rx_coal, p_cid); + if (rc != ECORE_SUCCESS) { + DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, + "VF[%d]: Unable to set rx queue = %d coalesce\n", + vf->abs_vf_id, vf->vf_queues[qid].fw_rx_qid); + goto out; + } + vf->rx_coal = rx_coal; + } + + /* TODO - in future, it might be possible to pass this in a per-cid + * granularity. For now, do this for all Tx queues. + */ + if (tx_coal) { + struct ecore_vf_queue *p_queue = &vf->vf_queues[qid]; + + for (i = 0; i < MAX_QUEUES_PER_QZONE; i++) { + if (p_queue->cids[i].p_cid == OSAL_NULL) + continue; + + if (!p_queue->cids[i].b_is_tx) + continue; + + rc = ecore_set_txq_coalesce(p_hwfn, p_ptt, tx_coal, + p_queue->cids[i].p_cid); + if (rc != ECORE_SUCCESS) { + DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, + "VF[%d]: Unable to set tx queue coalesce\n", + vf->abs_vf_id); + goto out; + } + } + vf->tx_coal = tx_coal; + } + +out: + ecore_ptt_release(p_hwfn, p_ptt); + + return rc; } static enum _ecore_status_t @@ -2395,11 +3800,11 @@ static enum _ecore_status_t ecore_iov_vf_flr_poll_pbf(struct ecore_hwfn *p_hwfn, struct ecore_vf_info *p_vf, struct ecore_ptt *p_ptt) { - u32 cons[MAX_NUM_VOQS], distance[MAX_NUM_VOQS]; + u32 cons[MAX_NUM_VOQS_E4], distance[MAX_NUM_VOQS_E4]; int i, cnt; /* Read initial consumers & producers */ - for (i = 0; i < MAX_NUM_VOQS; i++) { + for (i = 0; i < MAX_NUM_VOQS_E4; i++) { u32 prod; cons[i] = ecore_rd(p_hwfn, p_ptt, @@ -2414,7 +3819,7 @@ ecore_iov_vf_flr_poll_pbf(struct ecore_hwfn *p_hwfn, /* Wait for consumers to pass the producers */ i = 0; for (cnt = 0; cnt < 50; cnt++) { - for (; i < MAX_NUM_VOQS; i++) { + for (; i < MAX_NUM_VOQS_E4; i++) { u32 tmp; tmp = ecore_rd(p_hwfn, p_ptt, @@ -2424,7 +3829,7 @@ ecore_iov_vf_flr_poll_pbf(struct ecore_hwfn *p_hwfn, break; } - if (i == MAX_NUM_VOQS) + if (i == MAX_NUM_VOQS_E4) break; OSAL_MSLEEP(20); @@ -2439,61 +3844,6 @@ ecore_iov_vf_flr_poll_pbf(struct ecore_hwfn *p_hwfn, return ECORE_SUCCESS; } -static enum _ecore_status_t -ecore_iov_vf_flr_poll_prs(struct ecore_hwfn *p_hwfn, - struct ecore_vf_info *p_vf, struct ecore_ptt *p_ptt) -{ - u16 tc_cons[NUM_OF_TCS], tc_lb_cons[NUM_OF_TCS]; - u16 prod[NUM_OF_TCS]; - int i, cnt; - - /* Read initial consumers & producers */ - for (i = 0; i < NUM_OF_TCS; i++) { - tc_cons[i] = (u16)ecore_rd(p_hwfn, p_ptt, - PRS_REG_MSG_CT_MAIN_0 + i * 0x4); - tc_lb_cons[i] = (u16)ecore_rd(p_hwfn, p_ptt, - PRS_REG_MSG_CT_LB_0 + i * 0x4); - prod[i] = (u16)ecore_rd(p_hwfn, p_ptt, - BRB_REG_PER_TC_COUNTERS + - p_hwfn->port_id * 0x20 + i * 0x4); - } - - /* Wait for consumers to pass the producers */ - i = 0; - for (cnt = 0; cnt < 50; cnt++) { - for (; i < NUM_OF_TCS; i++) { - u16 cons; - - cons = (u16)ecore_rd(p_hwfn, p_ptt, - PRS_REG_MSG_CT_MAIN_0 + i * 0x4); - if (prod[i] - tc_cons[i] > cons - tc_cons[i]) - break; - - cons = (u16)ecore_rd(p_hwfn, p_ptt, - PRS_REG_MSG_CT_LB_0 + i * 0x4); - if (prod[i] - tc_lb_cons[i] > cons - tc_lb_cons[i]) - break; - } - - if (i == NUM_OF_TCS) - break; - - /* 16-bit counters; Delay instead of sleep... */ - OSAL_UDELAY(10); - } - - /* This is only optional polling for BB, since registers are only - * 16-bit wide and guarantee is not good enough. Don't fail things - * if polling didn't return the expected results. - */ - if (cnt == 50) - DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, - "VF[%d] - prs polling failed on TC %d\n", - p_vf->abs_vf_id, i); - - return ECORE_SUCCESS; -} - static enum _ecore_status_t ecore_iov_vf_flr_poll(struct ecore_hwfn *p_hwfn, struct ecore_vf_info *p_vf, struct ecore_ptt *p_ptt) @@ -2510,10 +3860,6 @@ static enum _ecore_status_t ecore_iov_vf_flr_poll(struct ecore_hwfn *p_hwfn, if (rc) return rc; - rc = ecore_iov_vf_flr_poll_prs(p_hwfn, p_vf, p_ptt); - if (rc) - return rc; - return ECORE_SUCCESS; } @@ -2522,8 +3868,8 @@ ecore_iov_execute_vf_flr_cleanup(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, u16 rel_vf_id, u32 *ack_vfs) { - enum _ecore_status_t rc = ECORE_SUCCESS; struct ecore_vf_info *p_vf; + enum _ecore_status_t rc = ECORE_SUCCESS; p_vf = ecore_iov_get_vf_info(p_hwfn, rel_vf_id, false); if (!p_vf) @@ -2541,7 +3887,7 @@ ecore_iov_execute_vf_flr_cleanup(struct ecore_hwfn *p_hwfn, ecore_iov_vf_cleanup(p_hwfn, p_vf); /* If VF isn't active, no need for anything but SW */ - if (!ECORE_IS_VF_ACTIVE(p_hwfn->p_dev, p_vf->relative_vf_id)) + if (!p_vf->b_init) goto cleanup; /* TODO - what to do in case of failure? */ @@ -2556,6 +3902,13 @@ ecore_iov_execute_vf_flr_cleanup(struct ecore_hwfn *p_hwfn, return rc; } + /* Workaround to make VF-PF channel ready, as FW + * doesn't do that as a part of FLR. + */ + REG_WR(p_hwfn, + GTT_BAR0_MAP_REG_USDM_RAM + + USTORM_VF_PF_CHANNEL_READY_OFFSET(vfid), 1); + /* VF_STOPPED has to be set only after final cleanup * but prior to re-enabling the VF. */ @@ -2575,8 +3928,7 @@ cleanup: ack_vfs[vfid / 32] |= (1 << (vfid % 32)); p_hwfn->pf_iov_info->pending_flr[rel_vf_id / 64] &= ~(1ULL << (rel_vf_id % 64)); - p_hwfn->pf_iov_info->pending_events[rel_vf_id / 64] &= - ~(1ULL << (rel_vf_id % 64)); + p_vf->vf_mbx.b_pending_msg = false; } return rc; @@ -2591,7 +3943,13 @@ enum _ecore_status_t ecore_iov_vf_flr_cleanup(struct ecore_hwfn *p_hwfn, OSAL_MEMSET(ack_vfs, 0, sizeof(u32) * (VF_MAX_STATIC / 32)); - for (i = 0; i < p_hwfn->p_dev->sriov_info.total_vfs; i++) + /* Since BRB <-> PRS interface can't be tested as part of the flr + * polling due to HW limitations, simply sleep a bit. And since + * there's no need to wait per-vf, do it before looping. + */ + OSAL_MSLEEP(100); + + for (i = 0; i < p_hwfn->p_dev->p_iov_info->total_vfs; i++) ecore_iov_execute_vf_flr_cleanup(p_hwfn, p_ptt, i, ack_vfs); rc = ecore_mcp_ack_vf_flr(p_hwfn, p_ptt, ack_vfs); @@ -2607,15 +3965,19 @@ ecore_iov_single_vf_flr_cleanup(struct ecore_hwfn *p_hwfn, OSAL_MEMSET(ack_vfs, 0, sizeof(u32) * (VF_MAX_STATIC / 32)); + /* Wait instead of polling the BRB <-> PRS interface */ + OSAL_MSLEEP(100); + ecore_iov_execute_vf_flr_cleanup(p_hwfn, p_ptt, rel_vf_id, ack_vfs); rc = ecore_mcp_ack_vf_flr(p_hwfn, p_ptt, ack_vfs); return rc; } -int ecore_iov_mark_vf_flr(struct ecore_hwfn *p_hwfn, u32 *p_disabled_vfs) +bool ecore_iov_mark_vf_flr(struct ecore_hwfn *p_hwfn, u32 *p_disabled_vfs) { - u16 i, found = 0; + bool found = false; + u16 i; DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, "Marking FLR-ed VFs\n"); for (i = 0; i < (VF_MAX_STATIC / 32); i++) @@ -2623,8 +3985,13 @@ int ecore_iov_mark_vf_flr(struct ecore_hwfn *p_hwfn, u32 *p_disabled_vfs) "[%08x,...,%08x]: %08x\n", i * 32, (i + 1) * 32 - 1, p_disabled_vfs[i]); + if (!p_hwfn->p_dev->p_iov_info) { + DP_NOTICE(p_hwfn, true, "VF flr but no IOV\n"); + return false; + } + /* Mark VFs */ - for (i = 0; i < p_hwfn->p_dev->sriov_info.total_vfs; i++) { + for (i = 0; i < p_hwfn->p_dev->p_iov_info->total_vfs; i++) { struct ecore_vf_info *p_vf; u8 vfid; @@ -2649,50 +4016,13 @@ int ecore_iov_mark_vf_flr(struct ecore_hwfn *p_hwfn, u32 *p_disabled_vfs) * VF flr until ACKs, we're safe. */ p_flr[rel_vf_id / 64] |= 1ULL << (rel_vf_id % 64); - found = 1; + found = true; } } return found; } -void ecore_iov_set_link(struct ecore_hwfn *p_hwfn, - u16 vfid, - struct ecore_mcp_link_params *params, - struct ecore_mcp_link_state *link, - struct ecore_mcp_link_capabilities *p_caps) -{ - struct ecore_vf_info *p_vf = ecore_iov_get_vf_info(p_hwfn, vfid, false); - struct ecore_bulletin_content *p_bulletin; - - if (!p_vf) - return; - - p_bulletin = p_vf->bulletin.p_virt; - p_bulletin->req_autoneg = params->speed.autoneg; - p_bulletin->req_adv_speed = params->speed.advertised_speeds; - p_bulletin->req_forced_speed = params->speed.forced_speed; - p_bulletin->req_autoneg_pause = params->pause.autoneg; - p_bulletin->req_forced_rx = params->pause.forced_rx; - p_bulletin->req_forced_tx = params->pause.forced_tx; - p_bulletin->req_loopback = params->loopback_mode; - - p_bulletin->link_up = link->link_up; - p_bulletin->speed = link->speed; - p_bulletin->full_duplex = link->full_duplex; - p_bulletin->autoneg = link->an; - p_bulletin->autoneg_complete = link->an_complete; - p_bulletin->parallel_detection = link->parallel_detection; - p_bulletin->pfc_enabled = link->pfc_enabled; - p_bulletin->partner_adv_speed = link->partner_adv_speed; - p_bulletin->partner_tx_flow_ctrl_en = link->partner_tx_flow_ctrl_en; - p_bulletin->partner_rx_flow_ctrl_en = link->partner_rx_flow_ctrl_en; - p_bulletin->partner_adv_pause = link->partner_adv_pause; - p_bulletin->sfp_tx_fault = link->sfp_tx_fault; - - p_bulletin->capability_speed = p_caps->speed_capabilities; -} - void ecore_iov_get_link(struct ecore_hwfn *p_hwfn, u16 vfid, struct ecore_mcp_link_params *p_params, @@ -2708,11 +4038,11 @@ void ecore_iov_get_link(struct ecore_hwfn *p_hwfn, p_bulletin = p_vf->bulletin.p_virt; if (p_params) - __ecore_vf_get_link_params(p_hwfn, p_params, p_bulletin); + __ecore_vf_get_link_params(p_params, p_bulletin); if (p_link) - __ecore_vf_get_link_state(p_hwfn, p_link, p_bulletin); + __ecore_vf_get_link_state(p_link, p_bulletin); if (p_caps) - __ecore_vf_get_link_caps(p_hwfn, p_caps, p_bulletin); + __ecore_vf_get_link_caps(p_caps, p_bulletin); } void ecore_iov_process_mbx_req(struct ecore_hwfn *p_hwfn, @@ -2720,7 +4050,6 @@ void ecore_iov_process_mbx_req(struct ecore_hwfn *p_hwfn, { struct ecore_iov_vf_mbx *mbx; struct ecore_vf_info *p_vf; - int i; p_vf = ecore_iov_get_vf_info(p_hwfn, (u16)vfid, true); if (!p_vf) @@ -2729,20 +4058,35 @@ void ecore_iov_process_mbx_req(struct ecore_hwfn *p_hwfn, mbx = &p_vf->vf_mbx; /* ecore_iov_process_mbx_request */ - DP_VERBOSE(p_hwfn, - ECORE_MSG_IOV, - "ecore_iov_process_mbx_req vfid %d\n", p_vf->abs_vf_id); +#ifndef CONFIG_ECORE_SW_CHANNEL + if (!mbx->b_pending_msg) { + DP_NOTICE(p_hwfn, true, + "VF[%02x]: Trying to process mailbox message when none is pending\n", + p_vf->abs_vf_id); + return; + } + mbx->b_pending_msg = false; +#endif mbx->first_tlv = mbx->req_virt->first_tlv; - /* check if tlv type is known */ - if (ecore_iov_tlv_supported(mbx->first_tlv.tl.type)) { - /* Lock the per vf op mutex and note the locker's identity. - * The unlock will take place in mbx response. - */ - ecore_iov_lock_vf_pf_channel(p_hwfn, - p_vf, mbx->first_tlv.tl.type); + DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, + "VF[%02x]: Processing mailbox message [type %04x]\n", + p_vf->abs_vf_id, mbx->first_tlv.tl.type); + OSAL_IOV_VF_MSG_TYPE(p_hwfn, + p_vf->relative_vf_id, + mbx->first_tlv.tl.type); + + /* Lock the per vf op mutex and note the locker's identity. + * The unlock will take place in mbx response. + */ + ecore_iov_lock_vf_pf_channel(p_hwfn, + p_vf, mbx->first_tlv.tl.type); + + /* check if tlv type is known */ + if (ecore_iov_tlv_supported(mbx->first_tlv.tl.type) && + !p_vf->b_malicious) { /* switch on the opcode */ switch (mbx->first_tlv.tl.type) { case CHANNEL_TLV_ACQUIRE: @@ -2784,11 +4128,37 @@ void ecore_iov_process_mbx_req(struct ecore_hwfn *p_hwfn, case CHANNEL_TLV_RELEASE: ecore_iov_vf_mbx_release(p_hwfn, p_ptt, p_vf); break; + case CHANNEL_TLV_UPDATE_TUNN_PARAM: + ecore_iov_vf_mbx_update_tunn_param(p_hwfn, p_ptt, p_vf); + break; + case CHANNEL_TLV_COALESCE_UPDATE: + ecore_iov_vf_pf_set_coalesce(p_hwfn, p_ptt, p_vf); + break; + case CHANNEL_TLV_COALESCE_READ: + ecore_iov_vf_pf_get_coalesce(p_hwfn, p_ptt, p_vf); + break; + } + } else if (ecore_iov_tlv_supported(mbx->first_tlv.tl.type)) { + /* If we've received a message from a VF we consider malicious + * we ignore the messasge unless it's one for RELEASE, in which + * case we'll let it have the benefit of doubt, allowing the + * next loaded driver to start again. + */ + if (mbx->first_tlv.tl.type == CHANNEL_TLV_RELEASE) { + /* TODO - initiate FLR, remove malicious indication */ + DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, + "VF [%02x] - considered malicious, but wanted to RELEASE. TODO\n", + p_vf->abs_vf_id); + } else { + DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, + "VF [%02x] - considered malicious; Ignoring TLV [%04x]\n", + p_vf->abs_vf_id, mbx->first_tlv.tl.type); } - ecore_iov_unlock_vf_pf_channel(p_hwfn, - p_vf, mbx->first_tlv.tl.type); - + ecore_iov_prepare_resp(p_hwfn, p_ptt, p_vf, + mbx->first_tlv.tl.type, + sizeof(struct pfvf_def_resp_tlv), + PFVF_STATUS_MALICIOUS); } else { /* unknown TLV - this may belong to a VF driver from the future * - a version written after this PF driver was written, which @@ -2796,76 +4166,135 @@ void ecore_iov_process_mbx_req(struct ecore_hwfn *p_hwfn, * support them. Or this may be because someone wrote a crappy * VF driver and is sending garbage over the channel. */ - DP_ERR(p_hwfn, - "unknown TLV. type %d length %d. first 20 bytes of mailbox buffer:\n", - mbx->first_tlv.tl.type, mbx->first_tlv.tl.length); + DP_NOTICE(p_hwfn, false, + "VF[%02x]: unknown TLV. type %04x length %04x" + " padding %08x reply address %lu\n", + p_vf->abs_vf_id, + mbx->first_tlv.tl.type, + mbx->first_tlv.tl.length, + mbx->first_tlv.padding, + (unsigned long)mbx->first_tlv.reply_address); - for (i = 0; i < 20; i++) { - DP_VERBOSE(p_hwfn, - ECORE_MSG_IOV, - "%x ", - mbx->req_virt->tlv_buf_size.tlv_buffer[i]); - } - - /* test whether we can respond to the VF (do we have an address - * for it?) + /* Try replying in case reply address matches the acquisition's + * posted address. */ - if (p_vf->state == VF_ACQUIRED) - DP_ERR(p_hwfn, "UNKNOWN TLV Not supported yet\n"); + if (p_vf->acquire.first_tlv.reply_address && + (mbx->first_tlv.reply_address == + p_vf->acquire.first_tlv.reply_address)) + ecore_iov_prepare_resp(p_hwfn, p_ptt, p_vf, + mbx->first_tlv.tl.type, + sizeof(struct pfvf_def_resp_tlv), + PFVF_STATUS_NOT_SUPPORTED); + else + DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, + "VF[%02x]: Can't respond to TLV -" + " no valid reply address\n", + p_vf->abs_vf_id); } + ecore_iov_unlock_vf_pf_channel(p_hwfn, p_vf, + mbx->first_tlv.tl.type); + #ifdef CONFIG_ECORE_SW_CHANNEL mbx->sw_mbx.mbx_state = VF_PF_RESPONSE_READY; mbx->sw_mbx.response_offset = 0; #endif } +void ecore_iov_pf_get_pending_events(struct ecore_hwfn *p_hwfn, + u64 *events) +{ + int i; + + OSAL_MEM_ZERO(events, sizeof(u64) * ECORE_VF_ARRAY_LENGTH); + + ecore_for_each_vf(p_hwfn, i) { + struct ecore_vf_info *p_vf; + + p_vf = &p_hwfn->pf_iov_info->vfs_array[i]; + if (p_vf->vf_mbx.b_pending_msg) + events[i / 64] |= 1ULL << (i % 64); + } +} + +static struct ecore_vf_info * +ecore_sriov_get_vf_from_absid(struct ecore_hwfn *p_hwfn, u16 abs_vfid) +{ + u8 min = (u8)p_hwfn->p_dev->p_iov_info->first_vf_in_pf; + + if (!_ecore_iov_pf_sanity_check(p_hwfn, (int)abs_vfid - min, false)) { + DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, + "Got indication for VF [abs 0x%08x] that cannot be" + " handled by PF\n", + abs_vfid); + return OSAL_NULL; + } + + return &p_hwfn->pf_iov_info->vfs_array[(u8)abs_vfid - min]; +} + static enum _ecore_status_t ecore_sriov_vfpf_msg(struct ecore_hwfn *p_hwfn, - __le16 vfid, + u16 abs_vfid, struct regpair *vf_msg) { - struct ecore_vf_info *p_vf; - u8 min, max; + struct ecore_vf_info *p_vf = ecore_sriov_get_vf_from_absid(p_hwfn, + abs_vfid); - if (!p_hwfn->pf_iov_info || !p_hwfn->pf_iov_info->vfs_array) { - DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, - "Got a message from VF while PF is not initialized for IOV support\n"); + if (!p_vf) return ECORE_SUCCESS; - } - - /* Find the VF record - message comes with realtive [engine] vfid */ - min = (u8)p_hwfn->hw_info.first_vf_in_pf; - max = min + p_hwfn->p_dev->sriov_info.total_vfs; - /* @@@TBD - for BE machines, should echo field be reversed? */ - if ((u8)vfid < min || (u8)vfid >= max) { - DP_INFO(p_hwfn, - "Got a message from VF with relative id 0x%08x, but PF's range is [0x%02x,...,0x%02x)\n", - (u8)vfid, min, max); - return ECORE_INVAL; - } - p_vf = &p_hwfn->pf_iov_info->vfs_array[(u8)vfid - min]; /* List the physical address of the request so that handler * could later on copy the message from it. */ p_vf->vf_mbx.pending_req = (((u64)vf_msg->hi) << 32) | vf_msg->lo; + p_vf->vf_mbx.b_pending_msg = true; + return OSAL_PF_VF_MSG(p_hwfn, p_vf->relative_vf_id); } -enum _ecore_status_t ecore_sriov_eqe_event(struct ecore_hwfn *p_hwfn, - u8 opcode, - __le16 echo, - union event_ring_data *data) +static void ecore_sriov_vfpf_malicious(struct ecore_hwfn *p_hwfn, + struct malicious_vf_eqe_data *p_data) +{ + struct ecore_vf_info *p_vf; + + p_vf = ecore_sriov_get_vf_from_absid(p_hwfn, p_data->vf_id); + + if (!p_vf) + return; + + if (!p_vf->b_malicious) { + DP_NOTICE(p_hwfn, false, + "VF [%d] - Malicious behavior [%02x]\n", + p_vf->abs_vf_id, p_data->err_id); + + p_vf->b_malicious = true; + } else { + DP_INFO(p_hwfn, + "VF [%d] - Malicious behavior [%02x]\n", + p_vf->abs_vf_id, p_data->err_id); + } + + OSAL_PF_VF_MALICIOUS(p_hwfn, p_vf->relative_vf_id); +} + +static enum _ecore_status_t ecore_sriov_eqe_event(struct ecore_hwfn *p_hwfn, + u8 opcode, + __le16 echo, + union event_ring_data *data, + u8 OSAL_UNUSED fw_return_code) { switch (opcode) { case COMMON_EVENT_VF_PF_CHANNEL: - return ecore_sriov_vfpf_msg(p_hwfn, echo, + return ecore_sriov_vfpf_msg(p_hwfn, OSAL_LE16_TO_CPU(echo), &data->vf_pf_channel.msg_addr); case COMMON_EVENT_VF_FLR: DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, "VF-FLR is still not supported\n"); return ECORE_SUCCESS; + case COMMON_EVENT_MALICIOUS_VF: + ecore_sriov_vfpf_malicious(p_hwfn, &data->malicious_vf); + return ECORE_SUCCESS; default: DP_INFO(p_hwfn->p_dev, "Unknown sriov eqe event 0x%02x\n", opcode); @@ -2879,51 +4308,20 @@ bool ecore_iov_is_vf_pending_flr(struct ecore_hwfn *p_hwfn, u16 rel_vf_id) (1ULL << (rel_vf_id % 64))); } -bool ecore_iov_is_valid_vfid(struct ecore_hwfn *p_hwfn, int rel_vf_id, - bool b_enabled_only) +u16 ecore_iov_get_next_active_vf(struct ecore_hwfn *p_hwfn, u16 rel_vf_id) { - if (!p_hwfn->pf_iov_info) { - DP_NOTICE(p_hwfn->p_dev, true, "No iov info\n"); - return false; - } + struct ecore_hw_sriov_info *p_iov = p_hwfn->p_dev->p_iov_info; + u16 i; - return b_enabled_only ? ECORE_IS_VF_ACTIVE(p_hwfn->p_dev, rel_vf_id) : - (rel_vf_id < p_hwfn->p_dev->sriov_info.total_vfs); -} + if (!p_iov) + goto out; -struct ecore_public_vf_info *ecore_iov_get_public_vf_info(struct ecore_hwfn - *p_hwfn, - u16 relative_vf_id, - bool b_enabled_only) -{ - struct ecore_vf_info *vf = OSAL_NULL; + for (i = rel_vf_id; i < p_iov->total_vfs; i++) + if (ecore_iov_is_valid_vfid(p_hwfn, rel_vf_id, true, false)) + return i; - vf = ecore_iov_get_vf_info(p_hwfn, relative_vf_id, b_enabled_only); - if (!vf) - return OSAL_NULL; - - return &vf->p_vf_info; -} - -void ecore_iov_pf_add_pending_events(struct ecore_hwfn *p_hwfn, u8 vfid) -{ - u64 add_bit = 1ULL << (vfid % 64); - - /* TODO - add locking mechanisms [no atomics in ecore, so we can't - * add the lock inside the ecore_pf_iov struct]. - */ - p_hwfn->pf_iov_info->pending_events[vfid / 64] |= add_bit; -} - -void ecore_iov_pf_get_and_clear_pending_events(struct ecore_hwfn *p_hwfn, - u64 *events) -{ - u64 *p_pending_events = p_hwfn->pf_iov_info->pending_events; - - /* TODO - Take a lock */ - OSAL_MEMCPY(events, p_pending_events, - sizeof(u64) * ECORE_VF_ARRAY_LENGTH); - OSAL_MEMSET(p_pending_events, 0, sizeof(u64) * ECORE_VF_ARRAY_LENGTH); +out: + return MAX_NUM_VFS_E4; } enum _ecore_status_t ecore_iov_copy_vf_msg(struct ecore_hwfn *p_hwfn, @@ -2965,6 +4363,12 @@ void ecore_iov_bulletin_set_forced_mac(struct ecore_hwfn *p_hwfn, "Can not set forced MAC, invalid vfid [%d]\n", vfid); return; } + if (vf_info->b_malicious) { + DP_NOTICE(p_hwfn->p_dev, false, + "Can't set forced MAC to malicious VF [%d]\n", + vfid); + return; + } feature = 1 << MAC_ADDR_FORCED; OSAL_MEMCPY(vf_info->bulletin.p_virt->mac, mac, ETH_ALEN); @@ -2989,6 +4393,12 @@ enum _ecore_status_t ecore_iov_bulletin_set_mac(struct ecore_hwfn *p_hwfn, "Can not set MAC, invalid vfid [%d]\n", vfid); return ECORE_INVAL; } + if (vf_info->b_malicious) { + DP_NOTICE(p_hwfn->p_dev, false, + "Can't set MAC to malicious VF [%d]\n", + vfid); + return ECORE_INVAL; + } if (vf_info->bulletin.p_virt->valid_bitmap & (1 << MAC_ADDR_FORCED)) { DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, @@ -3004,29 +4414,6 @@ enum _ecore_status_t ecore_iov_bulletin_set_mac(struct ecore_hwfn *p_hwfn, return ECORE_SUCCESS; } -void ecore_iov_bulletin_set_forced_vlan(struct ecore_hwfn *p_hwfn, - u16 pvid, int vfid) -{ - struct ecore_vf_info *vf_info; - u64 feature; - - vf_info = ecore_iov_get_vf_info(p_hwfn, (u16)vfid, true); - if (!vf_info) { - DP_NOTICE(p_hwfn->p_dev, true, - "Can not set forced MAC, invalid vfid [%d]\n", vfid); - return; - } - - feature = 1 << VLAN_ADDR_FORCED; - vf_info->bulletin.p_virt->pvid = pvid; - if (pvid) - vf_info->bulletin.p_virt->valid_bitmap |= feature; - else - vf_info->bulletin.p_virt->valid_bitmap &= ~feature; - - ecore_iov_configure_vport_forced(p_hwfn, vf_info, feature); -} - enum _ecore_status_t ecore_iov_bulletin_set_forced_untagged_default(struct ecore_hwfn *p_hwfn, bool b_untagged_only, int vfid) @@ -3037,7 +4424,14 @@ ecore_iov_bulletin_set_forced_untagged_default(struct ecore_hwfn *p_hwfn, vf_info = ecore_iov_get_vf_info(p_hwfn, (u16)vfid, true); if (!vf_info) { DP_NOTICE(p_hwfn->p_dev, true, - "Can not set forced MAC, invalid vfid [%d]\n", vfid); + "Can not set untagged default, invalid vfid [%d]\n", + vfid); + return ECORE_INVAL; + } + if (vf_info->b_malicious) { + DP_NOTICE(p_hwfn->p_dev, false, + "Can't set untagged default to malicious VF [%d]\n", + vfid); return ECORE_INVAL; } @@ -3046,7 +4440,8 @@ ecore_iov_bulletin_set_forced_untagged_default(struct ecore_hwfn *p_hwfn, */ if (vf_info->state == VF_ENABLED) { DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, - "Can't support untagged change for vfid[%d] - VF is already active\n", + "Can't support untagged change for vfid[%d] -" + " VF is already active\n", vfid); return ECORE_INVAL; } @@ -3076,16 +4471,57 @@ void ecore_iov_get_vfs_opaque_fid(struct ecore_hwfn *p_hwfn, int vfid, *opaque_fid = vf_info->opaque_fid; } -void ecore_iov_get_vfs_vport_id(struct ecore_hwfn *p_hwfn, int vfid, - u8 *p_vort_id) +void ecore_iov_bulletin_set_forced_vlan(struct ecore_hwfn *p_hwfn, + u16 pvid, int vfid) +{ + struct ecore_vf_info *vf_info; + u64 feature; + + vf_info = ecore_iov_get_vf_info(p_hwfn, (u16)vfid, true); + if (!vf_info) { + DP_NOTICE(p_hwfn->p_dev, true, + "Can not set forced MAC, invalid vfid [%d]\n", + vfid); + return; + } + if (vf_info->b_malicious) { + DP_NOTICE(p_hwfn->p_dev, false, + "Can't set forced vlan to malicious VF [%d]\n", + vfid); + return; + } + + feature = 1 << VLAN_ADDR_FORCED; + vf_info->bulletin.p_virt->pvid = pvid; + if (pvid) + vf_info->bulletin.p_virt->valid_bitmap |= feature; + else + vf_info->bulletin.p_virt->valid_bitmap &= ~feature; + + ecore_iov_configure_vport_forced(p_hwfn, vf_info, feature); +} + +void ecore_iov_bulletin_set_udp_ports(struct ecore_hwfn *p_hwfn, + int vfid, u16 vxlan_port, u16 geneve_port) { struct ecore_vf_info *vf_info; vf_info = ecore_iov_get_vf_info(p_hwfn, (u16)vfid, true); - if (!vf_info) + if (!vf_info) { + DP_NOTICE(p_hwfn->p_dev, true, + "Can not set udp ports, invalid vfid [%d]\n", vfid); return; + } - *p_vort_id = vf_info->vport_id; + if (vf_info->b_malicious) { + DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, + "Can not set udp ports to malicious VF [%d]\n", + vfid); + return; + } + + vf_info->bulletin.p_virt->vxlan_udp_port = vxlan_port; + vf_info->bulletin.p_virt->geneve_udp_port = geneve_port; } bool ecore_iov_vf_has_vport_instance(struct ecore_hwfn *p_hwfn, int vfid) @@ -3104,6 +4540,8 @@ bool ecore_iov_is_vf_stopped(struct ecore_hwfn *p_hwfn, int vfid) struct ecore_vf_info *p_vf_info; p_vf_info = ecore_iov_get_vf_info(p_hwfn, (u16)vfid, true); + if (!p_vf_info) + return true; return p_vf_info->state == VF_STOPPED; } @@ -3119,21 +4557,11 @@ bool ecore_iov_spoofchk_get(struct ecore_hwfn *p_hwfn, int vfid) return vf_info->spoof_chk; } -bool ecore_iov_pf_sanity_check(struct ecore_hwfn *p_hwfn, int vfid) -{ - if (IS_VF(p_hwfn->p_dev) || !IS_ECORE_SRIOV(p_hwfn->p_dev) || - !IS_PF_SRIOV_ALLOC(p_hwfn) || - !ECORE_IS_VF_ACTIVE(p_hwfn->p_dev, vfid)) - return false; - else - return true; -} - enum _ecore_status_t ecore_iov_spoofchk_set(struct ecore_hwfn *p_hwfn, int vfid, bool val) { - enum _ecore_status_t rc = ECORE_INVAL; struct ecore_vf_info *vf; + enum _ecore_status_t rc = ECORE_INVAL; if (!ecore_iov_pf_sanity_check(p_hwfn, vfid)) { DP_NOTICE(p_hwfn, true, @@ -3262,9 +4690,10 @@ enum _ecore_status_t ecore_iov_configure_tx_rate(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, int vfid, int val) { + struct ecore_mcp_link_state *p_link; struct ecore_vf_info *vf; - enum _ecore_status_t rc; u8 abs_vp_id = 0; + enum _ecore_status_t rc; vf = ecore_iov_get_vf_info(p_hwfn, (u16)vfid, true); @@ -3275,35 +4704,10 @@ enum _ecore_status_t ecore_iov_configure_tx_rate(struct ecore_hwfn *p_hwfn, if (rc != ECORE_SUCCESS) return rc; - rc = ecore_init_vport_rl(p_hwfn, p_ptt, abs_vp_id, (u32)val); + p_link = &ECORE_LEADING_HWFN(p_hwfn->p_dev)->mcp_info->link_output; - return rc; -} - -enum _ecore_status_t ecore_iov_configure_min_tx_rate(struct ecore_dev *p_dev, - int vfid, u32 rate) -{ - struct ecore_vf_info *vf; - enum _ecore_status_t rc; - u8 vport_id; - int i; - - for_each_hwfn(p_dev, i) { - struct ecore_hwfn *p_hwfn = &p_dev->hwfns[i]; - - if (!ecore_iov_pf_sanity_check(p_hwfn, vfid)) { - DP_NOTICE(p_hwfn, true, - "SR-IOV sanity check failed, can't set min rate\n"); - return ECORE_INVAL; - } - } - - vf = ecore_iov_get_vf_info(ECORE_LEADING_HWFN(p_dev), (u16)vfid, true); - vport_id = vf->vport_id; - - rc = ecore_configure_vport_wfq(p_dev, vport_id, rate); - - return rc; + return ecore_init_vport_rl(p_hwfn, p_ptt, abs_vp_id, (u32)val, + p_link->speed); } enum _ecore_status_t ecore_iov_get_vf_stats(struct ecore_hwfn *p_hwfn, @@ -3404,7 +4808,20 @@ bool ecore_iov_is_vf_initialized(struct ecore_hwfn *p_hwfn, u16 rel_vf_id) return (p_vf->state == VF_ENABLED); } -int ecore_iov_get_vf_min_rate(struct ecore_hwfn *p_hwfn, int vfid) +bool ecore_iov_is_vf_started(struct ecore_hwfn *p_hwfn, + u16 rel_vf_id) +{ + struct ecore_vf_info *p_vf; + + p_vf = ecore_iov_get_vf_info(p_hwfn, rel_vf_id, true); + if (!p_vf) + return false; + + return (p_vf->state != VF_FREE && p_vf->state != VF_STOPPED); +} + +enum _ecore_status_t +ecore_iov_get_vf_min_rate(struct ecore_hwfn *p_hwfn, int vfid) { struct ecore_wfq_data *vf_vp_wfq; struct ecore_vf_info *vf_info; @@ -3420,3 +4837,17 @@ int ecore_iov_get_vf_min_rate(struct ecore_hwfn *p_hwfn, int vfid) else return 0; } + +#ifdef CONFIG_ECORE_SW_CHANNEL +void ecore_iov_set_vf_hw_channel(struct ecore_hwfn *p_hwfn, int vfid, + bool b_is_hw) +{ + struct ecore_vf_info *vf_info; + + vf_info = ecore_iov_get_vf_info(p_hwfn, (u16)vfid, true); + if (!vf_info) + return; + + vf_info->b_hw_channel = b_is_hw; +} +#endif diff --git a/dpdk/drivers/net/qede/base/ecore_sriov.h b/dpdk/drivers/net/qede/base/ecore_sriov.h index 3471e5c6..850b1052 100644 --- a/dpdk/drivers/net/qede/base/ecore_sriov.h +++ b/dpdk/drivers/net/qede/base/ecore_sriov.h @@ -13,11 +13,10 @@ #include "ecore_vfpf_if.h" #include "ecore_iov_api.h" #include "ecore_hsi_common.h" - -#define ECORE_ETH_VF_NUM_VLAN_FILTERS 2 +#include "ecore_l2.h" #define ECORE_ETH_MAX_VF_NUM_VLAN_FILTERS \ - (MAX_NUM_VFS * ECORE_ETH_VF_NUM_VLAN_FILTERS) + (MAX_NUM_VFS_E4 * ECORE_ETH_VF_NUM_VLAN_FILTERS) /* Represents a full message. Both the request filled by VF * and the response filled by the PF. The VF needs one copy @@ -33,37 +32,21 @@ struct ecore_vf_mbx_msg { union pfvf_tlvs resp; }; -/* This data is held in the ecore_hwfn structure for VFs only. */ -struct ecore_vf_iov { - union vfpf_tlvs *vf2pf_request; - dma_addr_t vf2pf_request_phys; - union pfvf_tlvs *pf2vf_reply; - dma_addr_t pf2vf_reply_phys; - - /* Should be taken whenever the mailbox buffers are accessed */ - osal_mutex_t mutex; - u8 *offset; - - /* Bulletin Board */ - struct ecore_bulletin bulletin; - struct ecore_bulletin_content bulletin_shadow; - - /* we set aside a copy of the acquire response */ - struct pfvf_acquire_resp_tlv acquire_resp; -}; - /* This mailbox is maintained per VF in its PF * contains all information required for sending / receiving * a message */ struct ecore_iov_vf_mbx { - union vfpf_tlvs *req_virt; - dma_addr_t req_phys; - union pfvf_tlvs *reply_virt; - dma_addr_t reply_phys; + union vfpf_tlvs *req_virt; + dma_addr_t req_phys; + union pfvf_tlvs *reply_virt; + dma_addr_t reply_phys; /* Address in VF where a pending message is located */ - dma_addr_t pending_req; + dma_addr_t pending_req; + + /* Message from VF awaits handling */ + bool b_pending_msg; u8 *offset; @@ -72,40 +55,41 @@ struct ecore_iov_vf_mbx { #endif /* VF GPA address */ - u32 vf_addr_lo; - u32 vf_addr_hi; + u32 vf_addr_lo; + u32 vf_addr_hi; - struct vfpf_first_tlv first_tlv; /* saved VF request header */ + struct vfpf_first_tlv first_tlv; /* saved VF request header */ - u8 flags; + u8 flags; #define VF_MSG_INPROCESS 0x1 /* failsafe - the FW should prevent * more then one pending msg */ }; -struct ecore_vf_q_info { - u16 fw_rx_qid; - u16 fw_tx_qid; - u8 fw_cid; - u8 rxq_active; - u8 txq_active; +#define ECORE_IOV_LEGACY_QID_RX (0) +#define ECORE_IOV_LEGACY_QID_TX (1) +#define ECORE_IOV_QID_INVALID (0xFE) + +struct ecore_vf_queue_cid { + bool b_is_tx; + struct ecore_queue_cid *p_cid; }; -enum int_mod { - VPORT_INT_MOD_UNDEFINED = 0, - VPORT_INT_MOD_ADAPTIVE = 1, - VPORT_INT_MOD_OFF = 2, - VPORT_INT_MOD_LOW = 100, - VPORT_INT_MOD_MEDIUM = 200, - VPORT_INT_MOD_HIGH = 300 +/* Describes a qzone associated with the VF */ +struct ecore_vf_queue { + /* Input from upper-layer, mapping relateive queue to queue-zone */ + u16 fw_rx_qid; + u16 fw_tx_qid; + + struct ecore_vf_queue_cid cids[MAX_QUEUES_PER_QZONE]; }; enum vf_state { - VF_FREE = 0, /* VF ready to be acquired holds no resc */ - VF_ACQUIRED = 1, /* VF, acquired, but not initalized */ - VF_ENABLED = 2, /* VF, Enabled */ - VF_RESET = 3, /* VF, FLR'd, pending cleanup */ - VF_STOPPED = 4 /* VF, Stopped */ + VF_FREE = 0, /* VF ready to be acquired holds no resc */ + VF_ACQUIRED = 1, /* VF, acquired, but not initalized */ + VF_ENABLED = 2, /* VF, Enabled */ + VF_RESET = 3, /* VF, FLR'd, pending cleanup */ + VF_STOPPED = 4 /* VF, Stopped */ }; struct ecore_vf_vlan_shadow { @@ -117,6 +101,8 @@ struct ecore_vf_shadow_config { /* Shadow copy of all guest vlans */ struct ecore_vf_vlan_shadow vlans[ECORE_ETH_VF_NUM_VLAN_FILTERS + 1]; + /* Shadow copy of all configured MACs; Empty if forcing MACs */ + u8 macs[ECORE_ETH_VF_NUM_MAC_FILTERS][ETH_ALEN]; u8 inner_vlan_removal; }; @@ -124,34 +110,47 @@ struct ecore_vf_shadow_config { struct ecore_vf_info { struct ecore_iov_vf_mbx vf_mbx; enum vf_state state; - u8 to_disable; + bool b_init; + bool b_malicious; + u8 to_disable; - struct ecore_bulletin bulletin; - dma_addr_t vf_bulletin; + struct ecore_bulletin bulletin; + dma_addr_t vf_bulletin; - u32 concrete_fid; - u16 opaque_fid; - u16 mtu; +#ifdef CONFIG_ECORE_SW_CHANNEL + /* Determine whether PF communicate with VF using HW/SW channel */ + bool b_hw_channel; +#endif - u8 vport_id; - u8 relative_vf_id; - u8 abs_vf_id; + /* PF saves a copy of the last VF acquire message */ + struct vfpf_acquire_tlv acquire; + + u32 concrete_fid; + u16 opaque_fid; + u16 mtu; + + u8 vport_id; + u8 rss_eng_id; + u8 relative_vf_id; + u8 abs_vf_id; #define ECORE_VF_ABS_ID(p_hwfn, p_vf) (ECORE_PATH_ID(p_hwfn) ? \ (p_vf)->abs_vf_id + MAX_NUM_VFS_BB : \ (p_vf)->abs_vf_id) - u8 vport_instance; /* Number of active vports */ - u8 num_rxqs; - u8 num_txqs; + u8 vport_instance; /* Number of active vports */ + u8 num_rxqs; + u8 num_txqs; - u8 num_sbs; + u16 rx_coal; + u16 tx_coal; - u8 num_mac_filters; - u8 num_vlan_filters; - u8 num_mc_filters; + u8 num_sbs; - struct ecore_vf_q_info vf_queues[ECORE_MAX_VF_CHAINS_PER_PF]; - u16 igu_sbs[ECORE_MAX_VF_CHAINS_PER_PF]; + u8 num_mac_filters; + u8 num_vlan_filters; + + struct ecore_vf_queue vf_queues[ECORE_MAX_VF_CHAINS_PER_PF]; + u16 igu_sbs[ECORE_MAX_VF_CHAINS_PER_PF]; /* TODO - Only windows is using it - should be removed */ u8 was_malicious; @@ -159,7 +158,7 @@ struct ecore_vf_info { void *ctx; struct ecore_public_vf_info p_vf_info; bool spoof_chk; /* Current configured on HW */ - bool req_spoofchk_val; /* Requested value */ + bool req_spoofchk_val; /* Requested value */ /* Stores the configuration requested by VF */ struct ecore_vf_shadow_config shadow_config; @@ -176,49 +175,49 @@ struct ecore_vf_info { * capability enabled. */ struct ecore_pf_iov { - struct ecore_vf_info vfs_array[MAX_NUM_VFS]; - u64 pending_events[ECORE_VF_ARRAY_LENGTH]; - u64 pending_flr[ECORE_VF_ARRAY_LENGTH]; - u16 base_vport_id; + struct ecore_vf_info vfs_array[MAX_NUM_VFS_E4]; + u64 pending_flr[ECORE_VF_ARRAY_LENGTH]; + +#ifndef REMOVE_DBG + /* This doesn't serve anything functionally, but it makes windows + * debugging of IOV related issues easier. + */ + u64 active_vfs[ECORE_VF_ARRAY_LENGTH]; +#endif /* Allocate message address continuosuly and split to each VF */ - void *mbx_msg_virt_addr; - dma_addr_t mbx_msg_phys_addr; - u32 mbx_msg_size; - void *mbx_reply_virt_addr; - dma_addr_t mbx_reply_phys_addr; - u32 mbx_reply_size; - void *p_bulletins; - dma_addr_t bulletins_phys; - u32 bulletins_size; + void *mbx_msg_virt_addr; + dma_addr_t mbx_msg_phys_addr; + u32 mbx_msg_size; + void *mbx_reply_virt_addr; + dma_addr_t mbx_reply_phys_addr; + u32 mbx_reply_size; + void *p_bulletins; + dma_addr_t bulletins_phys; + u32 bulletins_size; }; #ifdef CONFIG_ECORE_SRIOV /** * @brief Read sriov related information and allocated resources - * reads from configuraiton space, shmem, and allocates the VF - * database in the PF. + * reads from configuraiton space, shmem, etc. * * @param p_hwfn - * @param p_ptt * * @return enum _ecore_status_t */ -enum _ecore_status_t ecore_iov_hw_info(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt); +enum _ecore_status_t ecore_iov_hw_info(struct ecore_hwfn *p_hwfn); /** * @brief ecore_add_tlv - place a given tlv on the tlv buffer at next offset * - * @param p_hwfn - * @param p_iov + * @param offset * @param type * @param length * * @return pointer to the newly placed tlv */ -void *ecore_add_tlv(struct ecore_hwfn *p_hwfn, - u8 **offset, u16 type, u16 length); +void *ecore_add_tlv(u8 **offset, u16 type, u16 length); /** * @brief list the types and lengths of the tlvs on the buffer @@ -226,7 +225,8 @@ void *ecore_add_tlv(struct ecore_hwfn *p_hwfn, * @param p_hwfn * @param tlvs_list */ -void ecore_dp_tlv_list(struct ecore_hwfn *p_hwfn, void *tlvs_list); +void ecore_dp_tlv_list(struct ecore_hwfn *p_hwfn, + void *tlvs_list); /** * @brief ecore_iov_alloc - allocate sriov related resources @@ -241,9 +241,8 @@ enum _ecore_status_t ecore_iov_alloc(struct ecore_hwfn *p_hwfn); * @brief ecore_iov_setup - setup sriov related resources * * @param p_hwfn - * @param p_ptt */ -void ecore_iov_setup(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt); +void ecore_iov_setup(struct ecore_hwfn *p_hwfn); /** * @brief ecore_iov_free - free sriov related resources @@ -253,28 +252,11 @@ void ecore_iov_setup(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt); void ecore_iov_free(struct ecore_hwfn *p_hwfn); /** - * @brief ecore_sriov_eqe_event - handle async sriov event arrived on eqe. + * @brief free sriov related memory that was allocated during hw_prepare * - * @param p_hwfn - * @param opcode - * @param echo - * @param data + * @param p_dev */ -enum _ecore_status_t ecore_sriov_eqe_event(struct ecore_hwfn *p_hwfn, - u8 opcode, - __le16 echo, - union event_ring_data *data); - -/** - * @brief calculate CRC for bulletin board validation - * - * @param basic crc seed - * @param ptr to beginning of buffer - * @length in bytes of buffer - * - * @return calculated crc over buffer [with respect to seed]. - */ -u32 ecore_crc32(u32 crc, u8 *ptr, u32 length); +void ecore_iov_free_hw_info(struct ecore_dev *p_dev); /** * @brief Mark structs of vfs that have been FLR-ed. @@ -282,9 +264,10 @@ u32 ecore_crc32(u32 crc, u8 *ptr, u32 length); * @param p_hwfn * @param disabled_vfs - bitmask of all VFs on path that were FLRed * - * @return 1 iff one of the PF's vfs got FLRed. 0 otherwise. + * @return true iff one of the PF's vfs got FLRed. false otherwise. */ -int ecore_iov_mark_vf_flr(struct ecore_hwfn *p_hwfn, u32 *disabled_vfs); +bool ecore_iov_mark_vf_flr(struct ecore_hwfn *p_hwfn, + u32 *disabled_vfs); /** * @brief Search extended TLVs in request/reply buffer. @@ -312,79 +295,5 @@ void *ecore_iov_search_list_tlvs(struct ecore_hwfn *p_hwfn, struct ecore_vf_info *ecore_iov_get_vf_info(struct ecore_hwfn *p_hwfn, u16 relative_vf_id, bool b_enabled_only); -#else -static OSAL_INLINE enum _ecore_status_t ecore_iov_hw_info(struct ecore_hwfn - *p_hwfn, - struct ecore_ptt - *p_ptt) -{ - return ECORE_SUCCESS; -} - -static OSAL_INLINE void *ecore_add_tlv(struct ecore_hwfn *p_hwfn, u8 **offset, - u16 type, u16 length) -{ - return OSAL_NULL; -} - -static OSAL_INLINE void ecore_dp_tlv_list(struct ecore_hwfn *p_hwfn, - void *tlvs_list) -{ -} - -static OSAL_INLINE enum _ecore_status_t ecore_iov_alloc(struct ecore_hwfn - *p_hwfn) -{ - return ECORE_SUCCESS; -} - -static OSAL_INLINE void ecore_iov_setup(struct ecore_hwfn *p_hwfn, - struct ecore_ptt *p_ptt) -{ -} - -static OSAL_INLINE void ecore_iov_free(struct ecore_hwfn *p_hwfn) -{ -} - -static OSAL_INLINE enum _ecore_status_t ecore_sriov_eqe_event(struct ecore_hwfn - *p_hwfn, - u8 opcode, - __le16 echo, - union - event_ring_data - * data) -{ - return ECORE_INVAL; -} - -static OSAL_INLINE u32 ecore_crc32(u32 crc, u8 *ptr, u32 length) -{ - return 0; -} - -static OSAL_INLINE int ecore_iov_mark_vf_flr(struct ecore_hwfn *p_hwfn, - u32 *disabled_vfs) -{ - return 0; -} - -static OSAL_INLINE void *ecore_iov_search_list_tlvs(struct ecore_hwfn *p_hwfn, - void *p_tlvs_list, - u16 req_type) -{ - return OSAL_NULL; -} - -static OSAL_INLINE struct ecore_vf_info *ecore_iov_get_vf_info(struct ecore_hwfn - *p_hwfn, - u16 - relative_vf_id, - bool - b_enabled_only) -{ - return OSAL_NULL; -} - #endif #endif /* __ECORE_SRIOV_H__ */ diff --git a/dpdk/drivers/net/qede/base/ecore_status.h b/dpdk/drivers/net/qede/base/ecore_status.h index 98d40bb5..c77ec260 100644 --- a/dpdk/drivers/net/qede/base/ecore_status.h +++ b/dpdk/drivers/net/qede/base/ecore_status.h @@ -10,18 +10,19 @@ #define __ECORE_STATUS_H__ enum _ecore_status_t { - ECORE_UNKNOWN_ERROR = -12, - ECORE_NORESOURCES = -11, - ECORE_NODEV = -10, + ECORE_CONN_RESET = -13, + ECORE_UNKNOWN_ERROR = -12, + ECORE_NORESOURCES = -11, + ECORE_NODEV = -10, ECORE_ABORTED = -9, - ECORE_AGAIN = -8, + ECORE_AGAIN = -8, ECORE_NOTIMPL = -7, - ECORE_EXISTS = -6, - ECORE_IO = -5, + ECORE_EXISTS = -6, + ECORE_IO = -5, ECORE_TIMEOUT = -4, - ECORE_INVAL = -3, - ECORE_BUSY = -2, - ECORE_NOMEM = -1, + ECORE_INVAL = -3, + ECORE_BUSY = -2, + ECORE_NOMEM = -1, ECORE_SUCCESS = 0, /* PENDING is not an error and should be positive */ ECORE_PENDING = 1, diff --git a/dpdk/drivers/net/qede/base/ecore_utils.h b/dpdk/drivers/net/qede/base/ecore_utils.h index 616b44c2..034cf1eb 100644 --- a/dpdk/drivers/net/qede/base/ecore_utils.h +++ b/dpdk/drivers/net/qede/base/ecore_utils.h @@ -10,6 +10,12 @@ #define __ECORE_UTILS_H__ /* dma_addr_t manip */ +/* Suppress "right shift count >= width of type" warning when that quantity is + * 32-bits rquires the >> 16) >> 16) + */ +#define PTR_LO(x) ((u32)(((osal_uintptr_t)(x)) & 0xffffffff)) +#define PTR_HI(x) ((u32)((((osal_uintptr_t)(x)) >> 16) >> 16)) + #define DMA_LO(x) ((u32)(((dma_addr_t)(x)) & 0xffffffff)) #define DMA_HI(x) ((u32)(((dma_addr_t)(x)) >> 32)) diff --git a/dpdk/drivers/net/qede/base/ecore_vf.c b/dpdk/drivers/net/qede/base/ecore_vf.c index d32fb35a..e0f2dd5a 100644 --- a/dpdk/drivers/net/qede/base/ecore_vf.c +++ b/dpdk/drivers/net/qede/base/ecore_vf.c @@ -44,7 +44,7 @@ static void *ecore_vf_pf_prep(struct ecore_hwfn *p_hwfn, u16 type, u16 length) OSAL_MEMSET(p_iov->pf2vf_reply, 0, sizeof(union pfvf_tlvs)); /* Init type and length */ - p_tlv = ecore_add_tlv(p_hwfn, &p_iov->offset, type, length); + p_tlv = ecore_add_tlv(&p_iov->offset, type, length); /* Init first tlv header */ ((struct vfpf_first_tlv *)p_tlv)->reply_address = @@ -53,15 +53,37 @@ static void *ecore_vf_pf_prep(struct ecore_hwfn *p_hwfn, u16 type, u16 length) return p_tlv; } -static int ecore_send_msg2pf(struct ecore_hwfn *p_hwfn, - u8 *done, u32 resp_size) +static void ecore_vf_pf_req_end(struct ecore_hwfn *p_hwfn, + enum _ecore_status_t req_status) +{ + union pfvf_tlvs *resp = p_hwfn->vf_iov_info->pf2vf_reply; + + DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, + "VF request status = 0x%x, PF reply status = 0x%x\n", + req_status, resp->default_resp.hdr.status); + + OSAL_MUTEX_RELEASE(&p_hwfn->vf_iov_info->mutex); +} + +#ifdef CONFIG_ECORE_SW_CHANNEL +/* The SW channel implementation of Windows needs to know the 'exact' + * response size of any given message. That means that for future + * messages we'd be unable to send TLVs to PF if he'll be unable to + * answer them if the |response| != |default response|. + * We'd need to handshake in acquire capabilities for any such. + */ +#endif +static enum _ecore_status_t +ecore_send_msg2pf(struct ecore_hwfn *p_hwfn, + u8 *done, u32 resp_size) { - struct ustorm_vf_zone *zone_data = (struct ustorm_vf_zone *) - ((u8 *)PXP_VF_BAR0_START_USDM_ZONE_B); union vfpf_tlvs *p_req = p_hwfn->vf_iov_info->vf2pf_request; struct ustorm_trigger_vf_zone trigger; - int rc = ECORE_SUCCESS, time = 100; - u8 pf_id; + struct ustorm_vf_zone *zone_data; + enum _ecore_status_t rc = ECORE_SUCCESS; + int time = 100; + + zone_data = (struct ustorm_vf_zone *)PXP_VF_BAR0_START_USDM_ZONE_B; /* output tlvs list */ ecore_dp_tlv_list(p_hwfn, p_req); @@ -69,25 +91,15 @@ static int ecore_send_msg2pf(struct ecore_hwfn *p_hwfn, /* need to add the END TLV to the message size */ resp_size += sizeof(struct channel_list_end_tlv); - if (!p_hwfn->p_dev->sriov_info.b_hw_channel) { - rc = OSAL_VF_SEND_MSG2PF(p_hwfn->p_dev, - done, - p_req, - p_hwfn->vf_iov_info->pf2vf_reply, - sizeof(union vfpf_tlvs), resp_size); - /* TODO - no prints about message ? */ - goto exit; - } - /* Send TLVs over HW channel */ OSAL_MEMSET(&trigger, 0, sizeof(struct ustorm_trigger_vf_zone)); trigger.vf_pf_msg_valid = 1; - /* TODO - FW should remove this requirement */ - pf_id = GET_FIELD(p_hwfn->hw_info.concrete_fid, PXP_CONCRETE_FID_PFID); DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, - "VF -> PF [%02x] message: [%08x, %08x] --> %p, %08x --> %p\n", - pf_id, + "VF -> PF [%02x] message: [%08x, %08x] --> %p," + " %08x --> %p\n", + GET_FIELD(p_hwfn->hw_info.concrete_fid, + PXP_CONCRETE_FID_PFID), U64_HI(p_hwfn->vf_iov_info->vf2pf_request_phys), U64_LO(p_hwfn->vf_iov_info->vf2pf_request_phys), &zone_data->non_trigger.vf_pf_msg_addr, @@ -109,32 +121,137 @@ static int ecore_send_msg2pf(struct ecore_hwfn *p_hwfn, REG_WR(p_hwfn, (osal_uintptr_t)&zone_data->trigger, *((u32 *)&trigger)); + /* When PF would be done with the response, it would write back to the + * `done' address. Poll until then. + */ while ((!*done) && time) { OSAL_MSLEEP(25); time--; } if (!*done) { - DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, - "VF <-- PF Timeout [Type %d]\n", - p_req->first_tlv.tl.type); + DP_NOTICE(p_hwfn, true, + "VF <-- PF Timeout [Type %d]\n", + p_req->first_tlv.tl.type); rc = ECORE_TIMEOUT; - goto exit; } else { - DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, - "PF response: %d [Type %d]\n", - *done, p_req->first_tlv.tl.type); + if ((*done != PFVF_STATUS_SUCCESS) && + (*done != PFVF_STATUS_NO_RESOURCE)) + DP_NOTICE(p_hwfn, false, + "PF response: %d [Type %d]\n", + *done, p_req->first_tlv.tl.type); + else + DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, + "PF response: %d [Type %d]\n", + *done, p_req->first_tlv.tl.type); } -exit: - OSAL_MUTEX_RELEASE(&p_hwfn->vf_iov_info->mutex); - return rc; } +static void ecore_vf_pf_add_qid(struct ecore_hwfn *p_hwfn, + struct ecore_queue_cid *p_cid) +{ + struct ecore_vf_iov *p_iov = p_hwfn->vf_iov_info; + struct vfpf_qid_tlv *p_qid_tlv; + + /* Only add QIDs for the queue if it was negotiated with PF */ + if (!(p_iov->acquire_resp.pfdev_info.capabilities & + PFVF_ACQUIRE_CAP_QUEUE_QIDS)) + return; + + p_qid_tlv = ecore_add_tlv(&p_iov->offset, + CHANNEL_TLV_QID, sizeof(*p_qid_tlv)); + p_qid_tlv->qid = p_cid->qid_usage_idx; +} + +enum _ecore_status_t _ecore_vf_pf_release(struct ecore_hwfn *p_hwfn, + bool b_final) +{ + struct ecore_vf_iov *p_iov = p_hwfn->vf_iov_info; + struct pfvf_def_resp_tlv *resp; + struct vfpf_first_tlv *req; + u32 size; + enum _ecore_status_t rc; + + /* clear mailbox and prep first tlv */ + req = ecore_vf_pf_prep(p_hwfn, CHANNEL_TLV_RELEASE, sizeof(*req)); + + /* add list termination tlv */ + ecore_add_tlv(&p_iov->offset, + CHANNEL_TLV_LIST_END, + sizeof(struct channel_list_end_tlv)); + + resp = &p_iov->pf2vf_reply->default_resp; + rc = ecore_send_msg2pf(p_hwfn, &resp->hdr.status, sizeof(*resp)); + + if (rc == ECORE_SUCCESS && resp->hdr.status != PFVF_STATUS_SUCCESS) + rc = ECORE_AGAIN; + + ecore_vf_pf_req_end(p_hwfn, rc); + if (!b_final) + return rc; + + p_hwfn->b_int_enabled = 0; + + if (p_iov->vf2pf_request) + OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev, + p_iov->vf2pf_request, + p_iov->vf2pf_request_phys, + sizeof(union vfpf_tlvs)); + if (p_iov->pf2vf_reply) + OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev, + p_iov->pf2vf_reply, + p_iov->pf2vf_reply_phys, + sizeof(union pfvf_tlvs)); + + if (p_iov->bulletin.p_virt) { + size = sizeof(struct ecore_bulletin_content); + OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev, + p_iov->bulletin.p_virt, + p_iov->bulletin.phys, + size); + } + +#ifdef CONFIG_ECORE_LOCK_ALLOC + OSAL_MUTEX_DEALLOC(&p_iov->mutex); +#endif + + OSAL_FREE(p_hwfn->p_dev, p_hwfn->vf_iov_info); + p_hwfn->vf_iov_info = OSAL_NULL; + + return rc; +} + +enum _ecore_status_t ecore_vf_pf_release(struct ecore_hwfn *p_hwfn) +{ + return _ecore_vf_pf_release(p_hwfn, true); +} + #define VF_ACQUIRE_THRESH 3 -#define VF_ACQUIRE_MAC_FILTERS 1 -#define VF_ACQUIRE_MC_FILTERS 10 +static void ecore_vf_pf_acquire_reduce_resc(struct ecore_hwfn *p_hwfn, + struct vf_pf_resc_request *p_req, + struct pf_vf_resc *p_resp) +{ + DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, + "PF unwilling to fullill resource request: rxq [%02x/%02x] txq [%02x/%02x] sbs [%02x/%02x] mac [%02x/%02x] vlan [%02x/%02x] mc [%02x/%02x] cids [%02x/%02x]. Try PF recommended amount\n", + p_req->num_rxqs, p_resp->num_rxqs, + p_req->num_rxqs, p_resp->num_txqs, + p_req->num_sbs, p_resp->num_sbs, + p_req->num_mac_filters, p_resp->num_mac_filters, + p_req->num_vlan_filters, p_resp->num_vlan_filters, + p_req->num_mc_filters, p_resp->num_mc_filters, + p_req->num_cids, p_resp->num_cids); + + /* humble our request */ + p_req->num_txqs = p_resp->num_txqs; + p_req->num_rxqs = p_resp->num_rxqs; + p_req->num_sbs = p_resp->num_sbs; + p_req->num_mac_filters = p_resp->num_mac_filters; + p_req->num_vlan_filters = p_resp->num_vlan_filters; + p_req->num_mc_filters = p_resp->num_mc_filters; + p_req->num_cids = p_resp->num_cids; +} static enum _ecore_status_t ecore_vf_pf_acquire(struct ecore_hwfn *p_hwfn) { @@ -142,26 +259,25 @@ static enum _ecore_status_t ecore_vf_pf_acquire(struct ecore_hwfn *p_hwfn) struct pfvf_acquire_resp_tlv *resp = &p_iov->pf2vf_reply->acquire_resp; struct pf_vf_pfdev_info *pfdev_info = &resp->pfdev_info; struct ecore_vf_acquire_sw_info vf_sw_info; - struct vfpf_acquire_tlv *req; - int rc = 0, attempts = 0; + struct vf_pf_resc_request *p_resc; bool resources_acquired = false; - - /* @@@ TBD: MichalK take this from somewhere else... */ - u8 rx_count = 1, tx_count = 1, num_sbs = 1; - u8 num_mac = VF_ACQUIRE_MAC_FILTERS, num_mc = VF_ACQUIRE_MC_FILTERS; + struct vfpf_acquire_tlv *req; + int attempts = 0; + enum _ecore_status_t rc = ECORE_SUCCESS; /* clear mailbox and prep first tlv */ req = ecore_vf_pf_prep(p_hwfn, CHANNEL_TLV_ACQUIRE, sizeof(*req)); + p_resc = &req->resc_request; /* @@@ TBD: PF may not be ready bnx2x_get_vf_id... */ req->vfdev_info.opaque_fid = p_hwfn->hw_info.opaque_fid; - req->resc_request.num_rxqs = rx_count; - req->resc_request.num_txqs = tx_count; - req->resc_request.num_sbs = num_sbs; - req->resc_request.num_mac_filters = num_mac; - req->resc_request.num_mc_filters = num_mc; - req->resc_request.num_vlan_filters = ECORE_ETH_VF_NUM_VLAN_FILTERS; + p_resc->num_rxqs = ECORE_MAX_VF_CHAINS_PER_PF; + p_resc->num_txqs = ECORE_MAX_VF_CHAINS_PER_PF; + p_resc->num_sbs = ECORE_MAX_VF_CHAINS_PER_PF; + p_resc->num_mac_filters = ECORE_ETH_VF_NUM_MAC_FILTERS; + p_resc->num_vlan_filters = ECORE_ETH_VF_NUM_VLAN_FILTERS; + p_resc->num_cids = ECORE_ETH_VF_DEFAULT_NUM_CIDS; OSAL_MEMSET(&vf_sw_info, 0, sizeof(vf_sw_info)); OSAL_VF_FILL_ACQUIRE_RESC_REQ(p_hwfn, &req->resc_request, &vf_sw_info); @@ -172,16 +288,23 @@ static enum _ecore_status_t ecore_vf_pf_acquire(struct ecore_hwfn *p_hwfn) req->vfdev_info.fw_minor = FW_MINOR_VERSION; req->vfdev_info.fw_revision = FW_REVISION_VERSION; req->vfdev_info.fw_engineering = FW_ENGINEERING_VERSION; + req->vfdev_info.eth_fp_hsi_major = ETH_HSI_VER_MAJOR; + req->vfdev_info.eth_fp_hsi_minor = ETH_HSI_VER_MINOR; - if (vf_sw_info.override_fw_version) - req->vfdev_info.capabilties |= VFPF_ACQUIRE_CAP_OVERRIDE_FW_VER; + /* Fill capability field with any non-deprecated config we support */ + req->vfdev_info.capabilities |= VFPF_ACQUIRE_CAP_100G; + + /* If we've mapped the doorbell bar, try using queue qids */ + if (p_iov->b_doorbell_bar) + req->vfdev_info.capabilities |= VFPF_ACQUIRE_CAP_PHYSICAL_BAR | + VFPF_ACQUIRE_CAP_QUEUE_QIDS; /* pf 2 vf bulletin board address */ req->bulletin_addr = p_iov->bulletin.phys; req->bulletin_size = p_iov->bulletin.size; /* add list termination tlv */ - ecore_add_tlv(p_hwfn, &p_iov->offset, + ecore_add_tlv(&p_iov->offset, CHANNEL_TLV_LIST_END, sizeof(struct channel_list_end_tlv)); @@ -189,13 +312,15 @@ static enum _ecore_status_t ecore_vf_pf_acquire(struct ecore_hwfn *p_hwfn) DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, "attempting to acquire resources\n"); + /* Clear response buffer, as this might be a re-send */ + OSAL_MEMSET(p_iov->pf2vf_reply, 0, + sizeof(union pfvf_tlvs)); + /* send acquire request */ rc = ecore_send_msg2pf(p_hwfn, &resp->hdr.status, sizeof(*resp)); - - /* PF timeout */ - if (rc) - return rc; + if (rc != ECORE_SUCCESS) + goto exit; /* copy acquire response from buffer to p_hwfn */ OSAL_MEMCPY(&p_iov->acquire_resp, @@ -203,46 +328,101 @@ static enum _ecore_status_t ecore_vf_pf_acquire(struct ecore_hwfn *p_hwfn) attempts++; - /* PF agrees to allocate our resources */ if (resp->hdr.status == PFVF_STATUS_SUCCESS) { + /* PF agrees to allocate our resources */ + if (!(resp->pfdev_info.capabilities & + PFVF_ACQUIRE_CAP_POST_FW_OVERRIDE)) { + /* It's possible legacy PF mistakenly accepted; + * but we don't care - simply mark it as + * legacy and continue. + */ + req->vfdev_info.capabilities |= + VFPF_ACQUIRE_CAP_PRE_FP_HSI; + } DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, "resources acquired\n"); resources_acquired = true; } /* PF refuses to allocate our resources */ - else if (resp->hdr.status == - PFVF_STATUS_NO_RESOURCE && + else if (resp->hdr.status == PFVF_STATUS_NO_RESOURCE && attempts < VF_ACQUIRE_THRESH) { - DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, - "PF unwilling to fullfill resource request. Try PF recommended amount\n"); + ecore_vf_pf_acquire_reduce_resc(p_hwfn, p_resc, + &resp->resc); - /* humble our request */ - req->resc_request.num_txqs = resp->resc.num_txqs; - req->resc_request.num_rxqs = resp->resc.num_rxqs; - req->resc_request.num_sbs = resp->resc.num_sbs; - req->resc_request.num_mac_filters = - resp->resc.num_mac_filters; - req->resc_request.num_vlan_filters = - resp->resc.num_vlan_filters; - req->resc_request.num_mc_filters = - resp->resc.num_mc_filters; + } else if (resp->hdr.status == PFVF_STATUS_NOT_SUPPORTED) { + if (pfdev_info->major_fp_hsi && + (pfdev_info->major_fp_hsi != ETH_HSI_VER_MAJOR)) { + DP_NOTICE(p_hwfn, false, + "PF uses an incompatible fastpath HSI" + " %02x.%02x [VF requires %02x.%02x]." + " Please change to a VF driver using" + " %02x.xx.\n", + pfdev_info->major_fp_hsi, + pfdev_info->minor_fp_hsi, + ETH_HSI_VER_MAJOR, ETH_HSI_VER_MINOR, + pfdev_info->major_fp_hsi); + rc = ECORE_INVAL; + goto exit; + } - /* Clear response buffer */ - OSAL_MEMSET(p_iov->pf2vf_reply, 0, - sizeof(union pfvf_tlvs)); + if (!pfdev_info->major_fp_hsi) { + if (req->vfdev_info.capabilities & + VFPF_ACQUIRE_CAP_PRE_FP_HSI) { + DP_NOTICE(p_hwfn, false, + "PF uses very old drivers." + " Please change to a VF" + " driver using no later than" + " 8.8.x.x.\n"); + rc = ECORE_INVAL; + goto exit; + } else { + DP_INFO(p_hwfn, + "PF is old - try re-acquire to" + " see if it supports FW-version" + " override\n"); + req->vfdev_info.capabilities |= + VFPF_ACQUIRE_CAP_PRE_FP_HSI; + continue; + } + } + + /* If PF/VF are using same Major, PF must have had + * it's reasons. Simply fail. + */ + DP_NOTICE(p_hwfn, false, + "PF rejected acquisition by VF\n"); + rc = ECORE_INVAL; + goto exit; } else { DP_ERR(p_hwfn, - "PF returned error %d to VF acquisition request\n", + "PF returned err %d to VF acquisition request\n", resp->hdr.status); - return ECORE_AGAIN; + rc = ECORE_AGAIN; + goto exit; } } + /* Mark the PF as legacy, if needed */ + if (req->vfdev_info.capabilities & + VFPF_ACQUIRE_CAP_PRE_FP_HSI) + p_iov->b_pre_fp_hsi = true; + + /* In case PF doesn't support multi-queue Tx, update the number of + * CIDs to reflect the number of queues [older PFs didn't fill that + * field]. + */ + if (!(resp->pfdev_info.capabilities & + PFVF_ACQUIRE_CAP_QUEUE_QIDS)) + resp->resc.num_cids = resp->resc.num_rxqs + + resp->resc.num_txqs; + rc = OSAL_VF_UPDATE_ACQUIRE_RESC_RESP(p_hwfn, &resp->resc); if (rc) { DP_NOTICE(p_hwfn, true, - "VF_UPDATE_ACQUIRE_RESC_RESP Failed: status = 0x%x.\n", + "VF_UPDATE_ACQUIRE_RESC_RESP Failed:" + " status = 0x%x.\n", rc); - return ECORE_AGAIN; + rc = ECORE_AGAIN; + goto exit; } /* Update bulletin board size with response from PF */ @@ -250,347 +430,629 @@ static enum _ecore_status_t ecore_vf_pf_acquire(struct ecore_hwfn *p_hwfn) /* get HW info */ p_hwfn->p_dev->type = resp->pfdev_info.dev_type; - p_hwfn->p_dev->chip_rev = resp->pfdev_info.chip_rev; + p_hwfn->p_dev->chip_rev = (u8)resp->pfdev_info.chip_rev; DP_INFO(p_hwfn, "Chip details - %s%d\n", ECORE_IS_BB(p_hwfn->p_dev) ? "BB" : "AH", CHIP_REV_IS_A0(p_hwfn->p_dev) ? 0 : 1); - /* @@@TBD MichalK: Fw ver... */ - /* strlcpy(p_hwfn->fw_ver, p_hwfn->acquire_resp.pfdev_info.fw_ver, - * sizeof(p_hwfn->fw_ver)); - */ - p_hwfn->p_dev->chip_num = pfdev_info->chip_num & 0xffff; + /* Learn of the possibility of CMT */ + if (IS_LEAD_HWFN(p_hwfn)) { + if (resp->pfdev_info.capabilities & PFVF_ACQUIRE_CAP_100G) { + DP_INFO(p_hwfn, "100g VF\n"); + p_hwfn->p_dev->num_hwfns = 2; + } + } + + /* @DPDK */ + if ((~p_iov->b_pre_fp_hsi & + ETH_HSI_VER_MINOR) && + (resp->pfdev_info.minor_fp_hsi < ETH_HSI_VER_MINOR)) + DP_INFO(p_hwfn, + "PF is using older fastpath HSI;" + " %02x.%02x is configured\n", + ETH_HSI_VER_MAJOR, + resp->pfdev_info.minor_fp_hsi); + +exit: + ecore_vf_pf_req_end(p_hwfn, rc); + + return rc; +} + +u32 ecore_vf_hw_bar_size(struct ecore_hwfn *p_hwfn, + enum BAR_ID bar_id) +{ + u32 bar_size; + + /* Regview size is fixed */ + if (bar_id == BAR_ID_0) + return 1 << 17; + + /* Doorbell is received from PF */ + bar_size = p_hwfn->vf_iov_info->acquire_resp.pfdev_info.bar_size; + if (bar_size) + return 1 << bar_size; return 0; } -enum _ecore_status_t ecore_vf_hw_prepare(struct ecore_dev *p_dev) +enum _ecore_status_t ecore_vf_hw_prepare(struct ecore_hwfn *p_hwfn) { - enum _ecore_status_t rc = ECORE_NOMEM; - struct ecore_vf_iov *p_sriov; - struct ecore_hwfn *p_hwfn = &p_dev->hwfns[0]; /* @@@TBD CMT */ + struct ecore_hwfn *p_lead = ECORE_LEADING_HWFN(p_hwfn->p_dev); + struct ecore_vf_iov *p_iov; + u32 reg; + enum _ecore_status_t rc; - p_dev->num_hwfns = 1; /* @@@TBD CMT must be fixed... */ + /* Set number of hwfns - might be overridden once leading hwfn learns + * actual configuration from PF. + */ + if (IS_LEAD_HWFN(p_hwfn)) + p_hwfn->p_dev->num_hwfns = 1; - p_hwfn->regview = p_dev->regview; - if (p_hwfn->regview == OSAL_NULL) { - DP_ERR(p_hwfn, - "regview should be initialized before" - " ecore_vf_hw_prepare is called\n"); - return ECORE_INVAL; - } + reg = PXP_VF_BAR0_ME_OPAQUE_ADDRESS; + p_hwfn->hw_info.opaque_fid = (u16)REG_RD(p_hwfn, reg); - /* Set the doorbell bar. Assumption: regview is set */ - p_hwfn->doorbells = (u8 OSAL_IOMEM *)p_hwfn->regview + - PXP_VF_BAR0_START_DQ; - - p_hwfn->hw_info.opaque_fid = (u16)REG_RD(p_hwfn, - PXP_VF_BAR0_ME_OPAQUE_ADDRESS); - - p_hwfn->hw_info.concrete_fid = REG_RD(p_hwfn, - PXP_VF_BAR0_ME_CONCRETE_ADDRESS); + reg = PXP_VF_BAR0_ME_CONCRETE_ADDRESS; + p_hwfn->hw_info.concrete_fid = REG_RD(p_hwfn, reg); /* Allocate vf sriov info */ - p_sriov = OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL, sizeof(*p_sriov)); - if (!p_sriov) { + p_iov = OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL, sizeof(*p_iov)); + if (!p_iov) { DP_NOTICE(p_hwfn, true, "Failed to allocate `struct ecore_sriov'\n"); return ECORE_NOMEM; } - OSAL_MEMSET(p_sriov, 0, sizeof(*p_sriov)); + /* Doorbells are tricky; Upper-layer has alreday set the hwfn doorbell + * value, but there are several incompatibily scenarios where that + * would be incorrect and we'd need to override it. + */ + if (p_hwfn->doorbells == OSAL_NULL) { + p_hwfn->doorbells = (u8 OSAL_IOMEM *)p_hwfn->regview + + PXP_VF_BAR0_START_DQ; + } else if (p_hwfn == p_lead) { + /* For leading hw-function, value is always correct, but need + * to handle scenario where legacy PF would not support 100g + * mapped bars later. + */ + p_iov->b_doorbell_bar = true; + } else { + /* here, value would be correct ONLY if the leading hwfn + * received indication that mapped-bars are supported. + */ + if (p_lead->vf_iov_info->b_doorbell_bar) + p_iov->b_doorbell_bar = true; + else + p_hwfn->doorbells = (u8 OSAL_IOMEM *) + p_hwfn->regview + + PXP_VF_BAR0_START_DQ; + } /* Allocate vf2pf msg */ - p_sriov->vf2pf_request = OSAL_DMA_ALLOC_COHERENT(p_hwfn->p_dev, - &p_sriov-> + p_iov->vf2pf_request = OSAL_DMA_ALLOC_COHERENT(p_hwfn->p_dev, + &p_iov-> vf2pf_request_phys, sizeof(union vfpf_tlvs)); - if (!p_sriov->vf2pf_request) { + if (!p_iov->vf2pf_request) { DP_NOTICE(p_hwfn, true, - "Failed to allocate `vf2pf_request' DMA memory\n"); - goto free_p_sriov; + "Failed to allocate `vf2pf_request' DMA memory\n"); + goto free_p_iov; } - p_sriov->pf2vf_reply = OSAL_DMA_ALLOC_COHERENT(p_hwfn->p_dev, - &p_sriov-> + p_iov->pf2vf_reply = OSAL_DMA_ALLOC_COHERENT(p_hwfn->p_dev, + &p_iov-> pf2vf_reply_phys, sizeof(union pfvf_tlvs)); - if (!p_sriov->pf2vf_reply) { + if (!p_iov->pf2vf_reply) { DP_NOTICE(p_hwfn, true, "Failed to allocate `pf2vf_reply' DMA memory\n"); goto free_vf2pf_request; } DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, - "VF's Request mailbox [%p virt 0x%" PRIx64 " phys], " - "Response mailbox [%p virt 0x%" PRIx64 " phys]\n", - p_sriov->vf2pf_request, - (u64)p_sriov->vf2pf_request_phys, - p_sriov->pf2vf_reply, (u64)p_sriov->pf2vf_reply_phys); + "VF's Request mailbox [%p virt 0x%lx phys], " + "Response mailbox [%p virt 0x%lx phys]\n", + p_iov->vf2pf_request, + (unsigned long)p_iov->vf2pf_request_phys, + p_iov->pf2vf_reply, + (unsigned long)p_iov->pf2vf_reply_phys); /* Allocate Bulletin board */ - p_sriov->bulletin.size = sizeof(struct ecore_bulletin_content); - p_sriov->bulletin.p_virt = OSAL_DMA_ALLOC_COHERENT(p_hwfn->p_dev, - &p_sriov->bulletin. + p_iov->bulletin.size = sizeof(struct ecore_bulletin_content); + p_iov->bulletin.p_virt = OSAL_DMA_ALLOC_COHERENT(p_hwfn->p_dev, + &p_iov->bulletin. phys, - p_sriov->bulletin. + p_iov->bulletin. size); DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, - "VF's bulletin Board [%p virt 0x%" PRIx64 " phys 0x%08x bytes]\n", - p_sriov->bulletin.p_virt, (u64)p_sriov->bulletin.phys, - p_sriov->bulletin.size); + "VF's bulletin Board [%p virt 0x%lx phys 0x%08x bytes]\n", + p_iov->bulletin.p_virt, (unsigned long)p_iov->bulletin.phys, + p_iov->bulletin.size); - OSAL_MUTEX_ALLOC(p_hwfn, &p_sriov->mutex); - OSAL_MUTEX_INIT(&p_sriov->mutex); +#ifdef CONFIG_ECORE_LOCK_ALLOC + OSAL_MUTEX_ALLOC(p_hwfn, &p_iov->mutex); +#endif + OSAL_MUTEX_INIT(&p_iov->mutex); - p_hwfn->vf_iov_info = p_sriov; + p_hwfn->vf_iov_info = p_iov; p_hwfn->hw_info.personality = ECORE_PCI_ETH; - /* First VF needs to query for information from PF */ - if (!p_hwfn->my_id) + rc = ecore_vf_pf_acquire(p_hwfn); + + /* If VF is 100g using a mapped bar and PF is too old to support that, + * acquisition would succeed - but the VF would have no way knowing + * the size of the doorbell bar configured in HW and thus will not + * know how to split it for 2nd hw-function. + * In this case we re-try without the indication of the mapped + * doorbell. + */ + if (rc == ECORE_SUCCESS && + p_iov->b_doorbell_bar && + !ecore_vf_hw_bar_size(p_hwfn, BAR_ID_1) && + ECORE_IS_CMT(p_hwfn->p_dev)) { + rc = _ecore_vf_pf_release(p_hwfn, false); + if (rc != ECORE_SUCCESS) + return rc; + + p_iov->b_doorbell_bar = false; + p_hwfn->doorbells = (u8 OSAL_IOMEM *)p_hwfn->regview + + PXP_VF_BAR0_START_DQ; rc = ecore_vf_pf_acquire(p_hwfn); + } + + DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, + "Regview [%p], Doorbell [%p], Device-doorbell [%p]\n", + p_hwfn->regview, p_hwfn->doorbells, + p_hwfn->p_dev->doorbells); return rc; free_vf2pf_request: - OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev, p_sriov->vf2pf_request, - p_sriov->vf2pf_request_phys, + OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev, p_iov->vf2pf_request, + p_iov->vf2pf_request_phys, sizeof(union vfpf_tlvs)); -free_p_sriov: - OSAL_FREE(p_hwfn->p_dev, p_sriov); +free_p_iov: + OSAL_FREE(p_hwfn->p_dev, p_iov); - return rc; + return ECORE_NOMEM; } -enum _ecore_status_t ecore_vf_pf_init(struct ecore_hwfn *p_hwfn) -{ - p_hwfn->b_int_enabled = 1; - - return 0; -} - -/* TEMP TEMP until in HSI */ #define TSTORM_QZONE_START PXP_VF_BAR0_START_SDM_ZONE_A #define MSTORM_QZONE_START(dev) (TSTORM_QZONE_START + \ (TSTORM_QZONE_SIZE * NUM_OF_L2_QUEUES(dev))) -#define USTORM_QZONE_START(dev) (MSTORM_QZONE_START + \ - (MSTORM_QZONE_SIZE * NUM_OF_L2_QUEUES(dev))) -enum _ecore_status_t ecore_vf_pf_rxq_start(struct ecore_hwfn *p_hwfn, - u8 rx_qid, - u16 sb, - u8 sb_index, - u16 bd_max_bytes, - dma_addr_t bd_chain_phys_addr, - dma_addr_t cqe_pbl_addr, - u16 cqe_pbl_size, - void OSAL_IOMEM * *pp_prod) +/* @DPDK - changed enum ecore_tunn_clss to enum ecore_tunn_mode */ +static void +__ecore_vf_prep_tunn_req_tlv(struct vfpf_update_tunn_param_tlv *p_req, + struct ecore_tunn_update_type *p_src, + enum ecore_tunn_mode mask, u8 *p_cls) +{ + if (p_src->b_update_mode) { + p_req->tun_mode_update_mask |= (1 << mask); + + if (p_src->b_mode_enabled) + p_req->tunn_mode |= (1 << mask); + } + + *p_cls = p_src->tun_cls; +} + +/* @DPDK - changed enum ecore_tunn_clss to enum ecore_tunn_mode */ +static void +ecore_vf_prep_tunn_req_tlv(struct vfpf_update_tunn_param_tlv *p_req, + struct ecore_tunn_update_type *p_src, + enum ecore_tunn_mode mask, u8 *p_cls, + struct ecore_tunn_update_udp_port *p_port, + u8 *p_update_port, u16 *p_udp_port) +{ + if (p_port->b_update_port) { + *p_update_port = 1; + *p_udp_port = p_port->port; + } + + __ecore_vf_prep_tunn_req_tlv(p_req, p_src, mask, p_cls); +} + +void ecore_vf_set_vf_start_tunn_update_param(struct ecore_tunnel_info *p_tun) +{ + if (p_tun->vxlan.b_mode_enabled) + p_tun->vxlan.b_update_mode = true; + if (p_tun->l2_geneve.b_mode_enabled) + p_tun->l2_geneve.b_update_mode = true; + if (p_tun->ip_geneve.b_mode_enabled) + p_tun->ip_geneve.b_update_mode = true; + if (p_tun->l2_gre.b_mode_enabled) + p_tun->l2_gre.b_update_mode = true; + if (p_tun->ip_gre.b_mode_enabled) + p_tun->ip_gre.b_update_mode = true; + + p_tun->b_update_rx_cls = true; + p_tun->b_update_tx_cls = true; +} + +static void +__ecore_vf_update_tunn_param(struct ecore_tunn_update_type *p_tun, + u16 feature_mask, u8 tunn_mode, u8 tunn_cls, + enum ecore_tunn_mode val) +{ + if (feature_mask & (1 << val)) { + p_tun->b_mode_enabled = tunn_mode; + p_tun->tun_cls = tunn_cls; + } else { + p_tun->b_mode_enabled = false; + } +} + +static void +ecore_vf_update_tunn_param(struct ecore_hwfn *p_hwfn, + struct ecore_tunnel_info *p_tun, + struct pfvf_update_tunn_param_tlv *p_resp) +{ + /* Update mode and classes provided by PF */ + u16 feat_mask = p_resp->tunn_feature_mask; + + __ecore_vf_update_tunn_param(&p_tun->vxlan, feat_mask, + p_resp->vxlan_mode, p_resp->vxlan_clss, + ECORE_MODE_VXLAN_TUNN); + __ecore_vf_update_tunn_param(&p_tun->l2_geneve, feat_mask, + p_resp->l2geneve_mode, + p_resp->l2geneve_clss, + ECORE_MODE_L2GENEVE_TUNN); + __ecore_vf_update_tunn_param(&p_tun->ip_geneve, feat_mask, + p_resp->ipgeneve_mode, + p_resp->ipgeneve_clss, + ECORE_MODE_IPGENEVE_TUNN); + __ecore_vf_update_tunn_param(&p_tun->l2_gre, feat_mask, + p_resp->l2gre_mode, p_resp->l2gre_clss, + ECORE_MODE_L2GRE_TUNN); + __ecore_vf_update_tunn_param(&p_tun->ip_gre, feat_mask, + p_resp->ipgre_mode, p_resp->ipgre_clss, + ECORE_MODE_IPGRE_TUNN); + p_tun->geneve_port.port = p_resp->geneve_udp_port; + p_tun->vxlan_port.port = p_resp->vxlan_udp_port; + + DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, + "tunn mode: vxlan=0x%x, l2geneve=0x%x, ipgeneve=0x%x, l2gre=0x%x, ipgre=0x%x", + p_tun->vxlan.b_mode_enabled, p_tun->l2_geneve.b_mode_enabled, + p_tun->ip_geneve.b_mode_enabled, + p_tun->l2_gre.b_mode_enabled, + p_tun->ip_gre.b_mode_enabled); +} + +enum _ecore_status_t +ecore_vf_pf_tunnel_param_update(struct ecore_hwfn *p_hwfn, + struct ecore_tunnel_info *p_src) +{ + struct ecore_tunnel_info *p_tun = &p_hwfn->p_dev->tunnel; + struct ecore_vf_iov *p_iov = p_hwfn->vf_iov_info; + struct pfvf_update_tunn_param_tlv *p_resp; + struct vfpf_update_tunn_param_tlv *p_req; + enum _ecore_status_t rc; + + p_req = ecore_vf_pf_prep(p_hwfn, CHANNEL_TLV_UPDATE_TUNN_PARAM, + sizeof(*p_req)); + + if (p_src->b_update_rx_cls && p_src->b_update_tx_cls) + p_req->update_tun_cls = 1; + + ecore_vf_prep_tunn_req_tlv(p_req, &p_src->vxlan, ECORE_MODE_VXLAN_TUNN, + &p_req->vxlan_clss, &p_src->vxlan_port, + &p_req->update_vxlan_port, + &p_req->vxlan_port); + ecore_vf_prep_tunn_req_tlv(p_req, &p_src->l2_geneve, + ECORE_MODE_L2GENEVE_TUNN, + &p_req->l2geneve_clss, &p_src->geneve_port, + &p_req->update_geneve_port, + &p_req->geneve_port); + __ecore_vf_prep_tunn_req_tlv(p_req, &p_src->ip_geneve, + ECORE_MODE_IPGENEVE_TUNN, + &p_req->ipgeneve_clss); + __ecore_vf_prep_tunn_req_tlv(p_req, &p_src->l2_gre, + ECORE_MODE_L2GRE_TUNN, &p_req->l2gre_clss); + __ecore_vf_prep_tunn_req_tlv(p_req, &p_src->ip_gre, + ECORE_MODE_IPGRE_TUNN, &p_req->ipgre_clss); + + /* add list termination tlv */ + ecore_add_tlv(&p_iov->offset, + CHANNEL_TLV_LIST_END, + sizeof(struct channel_list_end_tlv)); + + p_resp = &p_iov->pf2vf_reply->tunn_param_resp; + rc = ecore_send_msg2pf(p_hwfn, &p_resp->hdr.status, sizeof(*p_resp)); + + if (rc) + goto exit; + + if (p_resp->hdr.status != PFVF_STATUS_SUCCESS) { + DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, + "Failed to update tunnel parameters\n"); + rc = ECORE_INVAL; + } + + ecore_vf_update_tunn_param(p_hwfn, p_tun, p_resp); +exit: + ecore_vf_pf_req_end(p_hwfn, rc); + return rc; +} + +enum _ecore_status_t +ecore_vf_pf_rxq_start(struct ecore_hwfn *p_hwfn, + struct ecore_queue_cid *p_cid, + u16 bd_max_bytes, + dma_addr_t bd_chain_phys_addr, + dma_addr_t cqe_pbl_addr, + u16 cqe_pbl_size, + void OSAL_IOMEM **pp_prod) { struct ecore_vf_iov *p_iov = p_hwfn->vf_iov_info; + struct pfvf_start_queue_resp_tlv *resp; struct vfpf_start_rxq_tlv *req; - struct pfvf_def_resp_tlv *resp = &p_iov->pf2vf_reply->default_resp; - int rc; - u8 hw_qid; - u64 init_prod_val = 0; + u16 rx_qid = p_cid->rel.queue_id; + enum _ecore_status_t rc; /* clear mailbox and prep first tlv */ req = ecore_vf_pf_prep(p_hwfn, CHANNEL_TLV_START_RXQ, sizeof(*req)); - /* @@@TBD MichalK TPA */ - req->rx_qid = rx_qid; req->cqe_pbl_addr = cqe_pbl_addr; req->cqe_pbl_size = cqe_pbl_size; req->rxq_addr = bd_chain_phys_addr; - req->hw_sb = sb; - req->sb_index = sb_index; - req->hc_rate = 0; /* @@@TBD MichalK -> host coalescing! */ + req->hw_sb = p_cid->sb_igu_id; + req->sb_index = p_cid->sb_idx; req->bd_max_bytes = bd_max_bytes; - req->stat_id = -1; /* No stats at the moment */ + req->stat_id = -1; /* Keep initialized, for future compatibility */ - /* add list termination tlv */ - ecore_add_tlv(p_hwfn, &p_iov->offset, - CHANNEL_TLV_LIST_END, - sizeof(struct channel_list_end_tlv)); + /* If PF is legacy, we'll need to calculate producers ourselves + * as well as clean them. + */ + if (p_iov->b_pre_fp_hsi) { + u8 hw_qid = p_iov->acquire_resp.resc.hw_qid[rx_qid]; + u32 init_prod_val = 0; - if (pp_prod) { - hw_qid = p_iov->acquire_resp.resc.hw_qid[rx_qid]; - - *pp_prod = (u8 OSAL_IOMEM *)p_hwfn->regview + - MSTORM_QZONE_START(p_hwfn->p_dev) + - (hw_qid) * MSTORM_QZONE_SIZE + - OFFSETOF(struct mstorm_eth_queue_zone, rx_producers); + *pp_prod = (u8 OSAL_IOMEM *) + p_hwfn->regview + + MSTORM_QZONE_START(p_hwfn->p_dev) + + (hw_qid) * MSTORM_QZONE_SIZE; /* Init the rcq, rx bd and rx sge (if valid) producers to 0 */ - __internal_ram_wr(p_hwfn, *pp_prod, sizeof(u64), + __internal_ram_wr(p_hwfn, *pp_prod, sizeof(u32), (u32 *)(&init_prod_val)); } + ecore_vf_pf_add_qid(p_hwfn, p_cid); + + /* add list termination tlv */ + ecore_add_tlv(&p_iov->offset, + CHANNEL_TLV_LIST_END, + sizeof(struct channel_list_end_tlv)); + + resp = &p_iov->pf2vf_reply->queue_start; rc = ecore_send_msg2pf(p_hwfn, &resp->hdr.status, sizeof(*resp)); if (rc) - return rc; + goto exit; - if (resp->hdr.status != PFVF_STATUS_SUCCESS) - return ECORE_INVAL; + if (resp->hdr.status != PFVF_STATUS_SUCCESS) { + rc = ECORE_INVAL; + goto exit; + } + + /* Learn the address of the producer from the response */ + if (!p_iov->b_pre_fp_hsi) { + u32 init_prod_val = 0; + + *pp_prod = (u8 OSAL_IOMEM *)p_hwfn->regview + resp->offset; + DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, + "Rxq[0x%02x]: producer at %p [offset 0x%08x]\n", + rx_qid, *pp_prod, resp->offset); + + /* Init the rcq, rx bd and rx sge (if valid) producers to 0. + * It was actually the PF's responsibility, but since some + * old PFs might fail to do so, we do this as well. + */ + OSAL_BUILD_BUG_ON(ETH_HSI_VER_MAJOR != 3); + __internal_ram_wr(p_hwfn, *pp_prod, sizeof(u32), + (u32 *)&init_prod_val); + } + +exit: + ecore_vf_pf_req_end(p_hwfn, rc); return rc; } enum _ecore_status_t ecore_vf_pf_rxq_stop(struct ecore_hwfn *p_hwfn, - u16 rx_qid, bool cqe_completion) + struct ecore_queue_cid *p_cid, + bool cqe_completion) { struct ecore_vf_iov *p_iov = p_hwfn->vf_iov_info; struct vfpf_stop_rxqs_tlv *req; - struct pfvf_def_resp_tlv *resp = &p_iov->pf2vf_reply->default_resp; - int rc; + struct pfvf_def_resp_tlv *resp; + enum _ecore_status_t rc; /* clear mailbox and prep first tlv */ req = ecore_vf_pf_prep(p_hwfn, CHANNEL_TLV_STOP_RXQS, sizeof(*req)); - /* @@@TBD MichalK TPA */ - - /* @@@TBD MichalK - relevant ??? - * flags VFPF_QUEUE_FLG_OV VFPF_QUEUE_FLG_VLAN - */ - req->rx_qid = rx_qid; + req->rx_qid = p_cid->rel.queue_id; req->num_rxqs = 1; req->cqe_completion = cqe_completion; + ecore_vf_pf_add_qid(p_hwfn, p_cid); + /* add list termination tlv */ - ecore_add_tlv(p_hwfn, &p_iov->offset, + ecore_add_tlv(&p_iov->offset, CHANNEL_TLV_LIST_END, sizeof(struct channel_list_end_tlv)); + resp = &p_iov->pf2vf_reply->default_resp; rc = ecore_send_msg2pf(p_hwfn, &resp->hdr.status, sizeof(*resp)); if (rc) - return rc; + goto exit; - if (resp->hdr.status != PFVF_STATUS_SUCCESS) - return ECORE_INVAL; + if (resp->hdr.status != PFVF_STATUS_SUCCESS) { + rc = ECORE_INVAL; + goto exit; + } + +exit: + ecore_vf_pf_req_end(p_hwfn, rc); return rc; } -enum _ecore_status_t ecore_vf_pf_txq_start(struct ecore_hwfn *p_hwfn, - u16 tx_queue_id, - u16 sb, - u8 sb_index, - dma_addr_t pbl_addr, - u16 pbl_size, - void OSAL_IOMEM * *pp_doorbell) +enum _ecore_status_t +ecore_vf_pf_txq_start(struct ecore_hwfn *p_hwfn, + struct ecore_queue_cid *p_cid, + dma_addr_t pbl_addr, u16 pbl_size, + void OSAL_IOMEM **pp_doorbell) { struct ecore_vf_iov *p_iov = p_hwfn->vf_iov_info; + struct pfvf_start_queue_resp_tlv *resp; struct vfpf_start_txq_tlv *req; - struct pfvf_def_resp_tlv *resp = &p_iov->pf2vf_reply->default_resp; - int rc; + u16 qid = p_cid->rel.queue_id; + enum _ecore_status_t rc; /* clear mailbox and prep first tlv */ req = ecore_vf_pf_prep(p_hwfn, CHANNEL_TLV_START_TXQ, sizeof(*req)); - /* @@@TBD MichalK TPA */ - - req->tx_qid = tx_queue_id; + req->tx_qid = qid; /* Tx */ req->pbl_addr = pbl_addr; req->pbl_size = pbl_size; - req->hw_sb = sb; - req->sb_index = sb_index; - req->hc_rate = 0; /* @@@TBD MichalK -> host coalescing! */ - req->flags = 0; /* @@@TBD MichalK -> flags... */ + req->hw_sb = p_cid->sb_igu_id; + req->sb_index = p_cid->sb_idx; + + ecore_vf_pf_add_qid(p_hwfn, p_cid); /* add list termination tlv */ - ecore_add_tlv(p_hwfn, &p_iov->offset, + ecore_add_tlv(&p_iov->offset, CHANNEL_TLV_LIST_END, sizeof(struct channel_list_end_tlv)); + resp = &p_iov->pf2vf_reply->queue_start; rc = ecore_send_msg2pf(p_hwfn, &resp->hdr.status, sizeof(*resp)); if (rc) - return rc; + goto exit; - if (resp->hdr.status != PFVF_STATUS_SUCCESS) - return ECORE_INVAL; + if (resp->hdr.status != PFVF_STATUS_SUCCESS) { + rc = ECORE_INVAL; + goto exit; + } - if (pp_doorbell) { - u8 cid = p_iov->acquire_resp.resc.cid[tx_queue_id]; + /* Modern PFs provide the actual offsets, while legacy + * provided only the queue id. + */ + if (!p_iov->b_pre_fp_hsi) { + *pp_doorbell = (u8 OSAL_IOMEM *)p_hwfn->doorbells + + resp->offset; + } else { + u8 cid = p_iov->acquire_resp.resc.cid[qid]; *pp_doorbell = (u8 OSAL_IOMEM *)p_hwfn->doorbells + - DB_ADDR_VF(cid, DQ_DEMS_LEGACY); + DB_ADDR_VF(cid, DQ_DEMS_LEGACY); } + DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, + "Txq[0x%02x]: doorbell at %p [offset 0x%08x]\n", + qid, *pp_doorbell, resp->offset); +exit: + ecore_vf_pf_req_end(p_hwfn, rc); + return rc; } -enum _ecore_status_t ecore_vf_pf_txq_stop(struct ecore_hwfn *p_hwfn, u16 tx_qid) +enum _ecore_status_t ecore_vf_pf_txq_stop(struct ecore_hwfn *p_hwfn, + struct ecore_queue_cid *p_cid) { struct ecore_vf_iov *p_iov = p_hwfn->vf_iov_info; struct vfpf_stop_txqs_tlv *req; - struct pfvf_def_resp_tlv *resp = &p_iov->pf2vf_reply->default_resp; - int rc; + struct pfvf_def_resp_tlv *resp; + enum _ecore_status_t rc; /* clear mailbox and prep first tlv */ req = ecore_vf_pf_prep(p_hwfn, CHANNEL_TLV_STOP_TXQS, sizeof(*req)); - /* @@@TBD MichalK TPA */ - - /* @@@TBD MichalK - relevant ??? flags - * VFPF_QUEUE_FLG_OV VFPF_QUEUE_FLG_VLAN - */ - req->tx_qid = tx_qid; + req->tx_qid = p_cid->rel.queue_id; req->num_txqs = 1; + ecore_vf_pf_add_qid(p_hwfn, p_cid); + /* add list termination tlv */ - ecore_add_tlv(p_hwfn, &p_iov->offset, + ecore_add_tlv(&p_iov->offset, CHANNEL_TLV_LIST_END, sizeof(struct channel_list_end_tlv)); + resp = &p_iov->pf2vf_reply->default_resp; rc = ecore_send_msg2pf(p_hwfn, &resp->hdr.status, sizeof(*resp)); if (rc) - return rc; + goto exit; - if (resp->hdr.status != PFVF_STATUS_SUCCESS) - return ECORE_INVAL; + if (resp->hdr.status != PFVF_STATUS_SUCCESS) { + rc = ECORE_INVAL; + goto exit; + } + +exit: + ecore_vf_pf_req_end(p_hwfn, rc); return rc; } enum _ecore_status_t ecore_vf_pf_rxqs_update(struct ecore_hwfn *p_hwfn, - u16 rx_queue_id, + struct ecore_queue_cid **pp_cid, u8 num_rxqs, - u8 comp_cqe_flg, u8 comp_event_flg) + u8 comp_cqe_flg, + u8 comp_event_flg) { struct ecore_vf_iov *p_iov = p_hwfn->vf_iov_info; struct pfvf_def_resp_tlv *resp = &p_iov->pf2vf_reply->default_resp; struct vfpf_update_rxq_tlv *req; - int rc; + enum _ecore_status_t rc; + + /* Starting with CHANNEL_TLV_QID and the need for additional queue + * information, this API stopped supporting multiple rxqs. + * TODO - remove this and change the API to accept a single queue-cid + * in a follow-up patch. + */ + if (num_rxqs != 1) { + DP_NOTICE(p_hwfn, true, + "VFs can no longer update more than a single queue\n"); + return ECORE_INVAL; + } /* clear mailbox and prep first tlv */ req = ecore_vf_pf_prep(p_hwfn, CHANNEL_TLV_UPDATE_RXQ, sizeof(*req)); - req->rx_qid = rx_queue_id; - req->num_rxqs = num_rxqs; + req->rx_qid = (*pp_cid)->rel.queue_id; + req->num_rxqs = 1; if (comp_cqe_flg) req->flags |= VFPF_RXQ_UPD_COMPLETE_CQE_FLAG; if (comp_event_flg) req->flags |= VFPF_RXQ_UPD_COMPLETE_EVENT_FLAG; + ecore_vf_pf_add_qid(p_hwfn, *pp_cid); + /* add list termination tlv */ - ecore_add_tlv(p_hwfn, &p_iov->offset, + ecore_add_tlv(&p_iov->offset, CHANNEL_TLV_LIST_END, sizeof(struct channel_list_end_tlv)); rc = ecore_send_msg2pf(p_hwfn, &resp->hdr.status, sizeof(*resp)); if (rc) - return rc; + goto exit; - if (resp->hdr.status != PFVF_STATUS_SUCCESS) - return ECORE_INVAL; + if (resp->hdr.status != PFVF_STATUS_SUCCESS) { + rc = ECORE_INVAL; + goto exit; + } +exit: + ecore_vf_pf_req_end(p_hwfn, rc); return rc; } @@ -602,8 +1064,9 @@ ecore_vf_pf_vport_start(struct ecore_hwfn *p_hwfn, u8 vport_id, { struct ecore_vf_iov *p_iov = p_hwfn->vf_iov_info; struct vfpf_vport_start_tlv *req; - struct pfvf_def_resp_tlv *resp = &p_iov->pf2vf_reply->default_resp; - int rc, i; + struct pfvf_def_resp_tlv *resp; + enum _ecore_status_t rc; + int i; /* clear mailbox and prep first tlv */ req = ecore_vf_pf_prep(p_hwfn, CHANNEL_TLV_VPORT_START, sizeof(*req)); @@ -616,21 +1079,30 @@ ecore_vf_pf_vport_start(struct ecore_hwfn *p_hwfn, u8 vport_id, req->only_untagged = only_untagged; /* status blocks */ - for (i = 0; i < p_hwfn->vf_iov_info->acquire_resp.resc.num_sbs; i++) - if (p_hwfn->sbs_info[i]) - req->sb_addr[i] = p_hwfn->sbs_info[i]->sb_phys; + for (i = 0; i < p_hwfn->vf_iov_info->acquire_resp.resc.num_sbs; i++) { + struct ecore_sb_info *p_sb = p_hwfn->vf_iov_info->sbs_info[i]; + + if (p_sb) + req->sb_addr[i] = p_sb->sb_phys; + } /* add list termination tlv */ - ecore_add_tlv(p_hwfn, &p_iov->offset, + ecore_add_tlv(&p_iov->offset, CHANNEL_TLV_LIST_END, sizeof(struct channel_list_end_tlv)); + resp = &p_iov->pf2vf_reply->default_resp; rc = ecore_send_msg2pf(p_hwfn, &resp->hdr.status, sizeof(*resp)); if (rc) - return rc; + goto exit; - if (resp->hdr.status != PFVF_STATUS_SUCCESS) - return ECORE_INVAL; + if (resp->hdr.status != PFVF_STATUS_SUCCESS) { + rc = ECORE_INVAL; + goto exit; + } + +exit: + ecore_vf_pf_req_end(p_hwfn, rc); return rc; } @@ -639,27 +1111,69 @@ enum _ecore_status_t ecore_vf_pf_vport_stop(struct ecore_hwfn *p_hwfn) { struct ecore_vf_iov *p_iov = p_hwfn->vf_iov_info; struct pfvf_def_resp_tlv *resp = &p_iov->pf2vf_reply->default_resp; - int rc; + enum _ecore_status_t rc; /* clear mailbox and prep first tlv */ ecore_vf_pf_prep(p_hwfn, CHANNEL_TLV_VPORT_TEARDOWN, sizeof(struct vfpf_first_tlv)); /* add list termination tlv */ - ecore_add_tlv(p_hwfn, &p_iov->offset, + ecore_add_tlv(&p_iov->offset, CHANNEL_TLV_LIST_END, sizeof(struct channel_list_end_tlv)); rc = ecore_send_msg2pf(p_hwfn, &resp->hdr.status, sizeof(*resp)); if (rc) - return rc; + goto exit; - if (resp->hdr.status != PFVF_STATUS_SUCCESS) - return ECORE_INVAL; + if (resp->hdr.status != PFVF_STATUS_SUCCESS) { + rc = ECORE_INVAL; + goto exit; + } + +exit: + ecore_vf_pf_req_end(p_hwfn, rc); return rc; } +static bool +ecore_vf_handle_vp_update_is_needed(struct ecore_hwfn *p_hwfn, + struct ecore_sp_vport_update_params *p_data, + u16 tlv) +{ + switch (tlv) { + case CHANNEL_TLV_VPORT_UPDATE_ACTIVATE: + return !!(p_data->update_vport_active_rx_flg || + p_data->update_vport_active_tx_flg); + case CHANNEL_TLV_VPORT_UPDATE_TX_SWITCH: +#ifndef ASIC_ONLY + /* FPGA doesn't have PVFC and so can't support tx-switching */ + return !!(p_data->update_tx_switching_flg && + !CHIP_REV_IS_FPGA(p_hwfn->p_dev)); +#else + return !!p_data->update_tx_switching_flg; +#endif + case CHANNEL_TLV_VPORT_UPDATE_VLAN_STRIP: + return !!p_data->update_inner_vlan_removal_flg; + case CHANNEL_TLV_VPORT_UPDATE_ACCEPT_ANY_VLAN: + return !!p_data->update_accept_any_vlan_flg; + case CHANNEL_TLV_VPORT_UPDATE_MCAST: + return !!p_data->update_approx_mcast_flg; + case CHANNEL_TLV_VPORT_UPDATE_ACCEPT_PARAM: + return !!(p_data->accept_flags.update_rx_mode_config || + p_data->accept_flags.update_tx_mode_config); + case CHANNEL_TLV_VPORT_UPDATE_RSS: + return !!p_data->rss_params; + case CHANNEL_TLV_VPORT_UPDATE_SGE_TPA: + return !!p_data->sge_tpa_params; + default: + DP_INFO(p_hwfn, "Unexpected vport-update TLV[%d] %s\n", + tlv, ecore_channel_tlvs_string[tlv]); + return false; + } +} + static void ecore_vf_handle_vp_update_tlvs_resp(struct ecore_hwfn *p_hwfn, struct ecore_sp_vport_update_params *p_data) @@ -668,96 +1182,20 @@ ecore_vf_handle_vp_update_tlvs_resp(struct ecore_hwfn *p_hwfn, struct pfvf_def_resp_tlv *p_resp; u16 tlv; - if (p_data->update_vport_active_rx_flg || - p_data->update_vport_active_tx_flg) { - tlv = CHANNEL_TLV_VPORT_UPDATE_ACTIVATE; - p_resp = (struct pfvf_def_resp_tlv *) - ecore_iov_search_list_tlvs(p_hwfn, p_iov->pf2vf_reply, tlv); - if (p_resp && p_resp->hdr.status) - DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, - "VP update activate tlv configured\n"); - else - DP_NOTICE(p_hwfn, true, - "VP update activate tlv config failed\n"); - } + for (tlv = CHANNEL_TLV_VPORT_UPDATE_ACTIVATE; + tlv < CHANNEL_TLV_VPORT_UPDATE_MAX; + tlv++) { + if (!ecore_vf_handle_vp_update_is_needed(p_hwfn, p_data, tlv)) + continue; - if (p_data->update_tx_switching_flg) { - tlv = CHANNEL_TLV_VPORT_UPDATE_TX_SWITCH; p_resp = (struct pfvf_def_resp_tlv *) ecore_iov_search_list_tlvs(p_hwfn, p_iov->pf2vf_reply, tlv); if (p_resp && p_resp->hdr.status) DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, - "VP update tx switch tlv configured\n"); -#ifndef ASIC_ONLY - else if (CHIP_REV_IS_FPGA(p_hwfn->p_dev)) - DP_NOTICE(p_hwfn, false, - "FPGA: Skip checking whether PF" - " replied to Tx-switching request\n"); -#endif - else - DP_NOTICE(p_hwfn, true, - "VP update tx switch tlv config failed\n"); - } - - if (p_data->update_inner_vlan_removal_flg) { - tlv = CHANNEL_TLV_VPORT_UPDATE_VLAN_STRIP; - p_resp = (struct pfvf_def_resp_tlv *) - ecore_iov_search_list_tlvs(p_hwfn, p_iov->pf2vf_reply, tlv); - if (p_resp && p_resp->hdr.status) - DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, - "VP update vlan strip tlv configured\n"); - else - DP_NOTICE(p_hwfn, true, - "VP update vlan strip tlv config failed\n"); - } - - if (p_data->update_approx_mcast_flg) { - tlv = CHANNEL_TLV_VPORT_UPDATE_MCAST; - p_resp = (struct pfvf_def_resp_tlv *) - ecore_iov_search_list_tlvs(p_hwfn, p_iov->pf2vf_reply, tlv); - if (p_resp && p_resp->hdr.status) - DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, - "VP update mcast tlv configured\n"); - else - DP_NOTICE(p_hwfn, true, - "VP update mcast tlv config failed\n"); - } - - if (p_data->accept_flags.update_rx_mode_config || - p_data->accept_flags.update_tx_mode_config) { - tlv = CHANNEL_TLV_VPORT_UPDATE_ACCEPT_PARAM; - p_resp = (struct pfvf_def_resp_tlv *) - ecore_iov_search_list_tlvs(p_hwfn, p_iov->pf2vf_reply, tlv); - if (p_resp && p_resp->hdr.status) - DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, - "VP update accept_mode tlv configured\n"); - else - DP_NOTICE(p_hwfn, true, - "VP update accept_mode tlv config failed\n"); - } - - if (p_data->rss_params) { - tlv = CHANNEL_TLV_VPORT_UPDATE_RSS; - p_resp = (struct pfvf_def_resp_tlv *) - ecore_iov_search_list_tlvs(p_hwfn, p_iov->pf2vf_reply, tlv); - if (p_resp && p_resp->hdr.status) - DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, - "VP update rss tlv configured\n"); - else - DP_NOTICE(p_hwfn, true, - "VP update rss tlv config failed\n"); - } - - if (p_data->sge_tpa_params) { - tlv = CHANNEL_TLV_VPORT_UPDATE_SGE_TPA; - p_resp = (struct pfvf_def_resp_tlv *) - ecore_iov_search_list_tlvs(p_hwfn, p_iov->pf2vf_reply, tlv); - if (p_resp && p_resp->hdr.status) - DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, - "VP update sge tpa tlv configured\n"); - else - DP_NOTICE(p_hwfn, true, - "VP update sge tpa tlv config failed\n"); + "TLV[%d] type %s Configuration %s\n", + tlv, ecore_channel_tlvs_string[tlv], + (p_resp && p_resp->hdr.status) ? "succeeded" + : "failed"); } } @@ -765,21 +1203,13 @@ enum _ecore_status_t ecore_vf_pf_vport_update(struct ecore_hwfn *p_hwfn, struct ecore_sp_vport_update_params *p_params) { - struct vfpf_vport_update_accept_any_vlan_tlv *p_any_vlan_tlv; - struct vfpf_vport_update_accept_param_tlv *p_accept_tlv; - struct vfpf_vport_update_tx_switch_tlv *p_tx_switch_tlv; - struct vfpf_vport_update_mcast_bin_tlv *p_mcast_tlv; - struct vfpf_vport_update_vlan_strip_tlv *p_vlan_tlv; - struct vfpf_vport_update_sge_tpa_tlv *p_sge_tpa_tlv; - struct vfpf_vport_update_activate_tlv *p_act_tlv; struct ecore_vf_iov *p_iov = p_hwfn->vf_iov_info; - struct vfpf_vport_update_rss_tlv *p_rss_tlv; struct vfpf_vport_update_tlv *req; struct pfvf_def_resp_tlv *resp; u8 update_rx, update_tx; u32 resp_size = 0; u16 size, tlv; - int rc; + enum _ecore_status_t rc; resp = &p_iov->pf2vf_reply->default_resp; resp_size = sizeof(*resp); @@ -792,8 +1222,10 @@ ecore_vf_pf_vport_update(struct ecore_hwfn *p_hwfn, /* Prepare extended tlvs */ if (update_rx || update_tx) { + struct vfpf_vport_update_activate_tlv *p_act_tlv; + size = sizeof(struct vfpf_vport_update_activate_tlv); - p_act_tlv = ecore_add_tlv(p_hwfn, &p_iov->offset, + p_act_tlv = ecore_add_tlv(&p_iov->offset, CHANNEL_TLV_VPORT_UPDATE_ACTIVATE, size); resp_size += sizeof(struct pfvf_def_resp_tlv); @@ -810,8 +1242,10 @@ ecore_vf_pf_vport_update(struct ecore_hwfn *p_hwfn, } if (p_params->update_inner_vlan_removal_flg) { + struct vfpf_vport_update_vlan_strip_tlv *p_vlan_tlv; + size = sizeof(struct vfpf_vport_update_vlan_strip_tlv); - p_vlan_tlv = ecore_add_tlv(p_hwfn, &p_iov->offset, + p_vlan_tlv = ecore_add_tlv(&p_iov->offset, CHANNEL_TLV_VPORT_UPDATE_VLAN_STRIP, size); resp_size += sizeof(struct pfvf_def_resp_tlv); @@ -820,9 +1254,11 @@ ecore_vf_pf_vport_update(struct ecore_hwfn *p_hwfn, } if (p_params->update_tx_switching_flg) { + struct vfpf_vport_update_tx_switch_tlv *p_tx_switch_tlv; + size = sizeof(struct vfpf_vport_update_tx_switch_tlv); tlv = CHANNEL_TLV_VPORT_UPDATE_TX_SWITCH; - p_tx_switch_tlv = ecore_add_tlv(p_hwfn, &p_iov->offset, + p_tx_switch_tlv = ecore_add_tlv(&p_iov->offset, tlv, size); resp_size += sizeof(struct pfvf_def_resp_tlv); @@ -830,8 +1266,10 @@ ecore_vf_pf_vport_update(struct ecore_hwfn *p_hwfn, } if (p_params->update_approx_mcast_flg) { + struct vfpf_vport_update_mcast_bin_tlv *p_mcast_tlv; + size = sizeof(struct vfpf_vport_update_mcast_bin_tlv); - p_mcast_tlv = ecore_add_tlv(p_hwfn, &p_iov->offset, + p_mcast_tlv = ecore_add_tlv(&p_iov->offset, CHANNEL_TLV_VPORT_UPDATE_MCAST, size); resp_size += sizeof(struct pfvf_def_resp_tlv); @@ -845,9 +1283,11 @@ ecore_vf_pf_vport_update(struct ecore_hwfn *p_hwfn, update_tx = p_params->accept_flags.update_tx_mode_config; if (update_rx || update_tx) { + struct vfpf_vport_update_accept_param_tlv *p_accept_tlv; + tlv = CHANNEL_TLV_VPORT_UPDATE_ACCEPT_PARAM; size = sizeof(struct vfpf_vport_update_accept_param_tlv); - p_accept_tlv = ecore_add_tlv(p_hwfn, &p_iov->offset, tlv, size); + p_accept_tlv = ecore_add_tlv(&p_iov->offset, tlv, size); resp_size += sizeof(struct pfvf_def_resp_tlv); if (update_rx) { @@ -865,9 +1305,11 @@ ecore_vf_pf_vport_update(struct ecore_hwfn *p_hwfn, if (p_params->rss_params) { struct ecore_rss_params *rss_params = p_params->rss_params; + struct vfpf_vport_update_rss_tlv *p_rss_tlv; + int i, table_size; size = sizeof(struct vfpf_vport_update_rss_tlv); - p_rss_tlv = ecore_add_tlv(p_hwfn, &p_iov->offset, + p_rss_tlv = ecore_add_tlv(&p_iov->offset, CHANNEL_TLV_VPORT_UPDATE_RSS, size); resp_size += sizeof(struct pfvf_def_resp_tlv); @@ -886,17 +1328,26 @@ ecore_vf_pf_vport_update(struct ecore_hwfn *p_hwfn, p_rss_tlv->rss_enable = rss_params->rss_enable; p_rss_tlv->rss_caps = rss_params->rss_caps; p_rss_tlv->rss_table_size_log = rss_params->rss_table_size_log; - OSAL_MEMCPY(p_rss_tlv->rss_ind_table, rss_params->rss_ind_table, - sizeof(rss_params->rss_ind_table)); + + table_size = OSAL_MIN_T(int, T_ETH_INDIRECTION_TABLE_SIZE, + 1 << p_rss_tlv->rss_table_size_log); + for (i = 0; i < table_size; i++) { + struct ecore_queue_cid *p_queue; + + p_queue = rss_params->rss_ind_table[i]; + p_rss_tlv->rss_ind_table[i] = p_queue->rel.queue_id; + } + OSAL_MEMCPY(p_rss_tlv->rss_key, rss_params->rss_key, sizeof(rss_params->rss_key)); } if (p_params->update_accept_any_vlan_flg) { + struct vfpf_vport_update_accept_any_vlan_tlv *p_any_vlan_tlv; + size = sizeof(struct vfpf_vport_update_accept_any_vlan_tlv); tlv = CHANNEL_TLV_VPORT_UPDATE_ACCEPT_ANY_VLAN; - p_any_vlan_tlv = ecore_add_tlv(p_hwfn, &p_iov->offset, - tlv, size); + p_any_vlan_tlv = ecore_add_tlv(&p_iov->offset, tlv, size); resp_size += sizeof(struct pfvf_def_resp_tlv); p_any_vlan_tlv->accept_any_vlan = p_params->accept_any_vlan; @@ -905,11 +1356,12 @@ ecore_vf_pf_vport_update(struct ecore_hwfn *p_hwfn, } if (p_params->sge_tpa_params) { - struct ecore_sge_tpa_params *sge_tpa_params = - p_params->sge_tpa_params; + struct ecore_sge_tpa_params *sge_tpa_params; + struct vfpf_vport_update_sge_tpa_tlv *p_sge_tpa_tlv; + sge_tpa_params = p_params->sge_tpa_params; size = sizeof(struct vfpf_vport_update_sge_tpa_tlv); - p_sge_tpa_tlv = ecore_add_tlv(p_hwfn, &p_iov->offset, + p_sge_tpa_tlv = ecore_add_tlv(&p_iov->offset, CHANNEL_TLV_VPORT_UPDATE_SGE_TPA, size); resp_size += sizeof(struct pfvf_def_resp_tlv); @@ -933,6 +1385,12 @@ ecore_vf_pf_vport_update(struct ecore_hwfn *p_hwfn, if (sge_tpa_params->tpa_gro_consistent_flg) p_sge_tpa_tlv->sge_tpa_flags |= VFPF_TPA_GRO_CONSIST_FLAG; + if (sge_tpa_params->tpa_ipv4_tunn_en_flg) + p_sge_tpa_tlv->sge_tpa_flags |= + VFPF_TPA_TUNN_IPV4_EN_FLAG; + if (sge_tpa_params->tpa_ipv6_tunn_en_flg) + p_sge_tpa_tlv->sge_tpa_flags |= + VFPF_TPA_TUNN_IPV6_EN_FLAG; p_sge_tpa_tlv->tpa_max_aggs_num = sge_tpa_params->tpa_max_aggs_num; @@ -947,96 +1405,56 @@ ecore_vf_pf_vport_update(struct ecore_hwfn *p_hwfn, } /* add list termination tlv */ - ecore_add_tlv(p_hwfn, &p_iov->offset, + ecore_add_tlv(&p_iov->offset, CHANNEL_TLV_LIST_END, sizeof(struct channel_list_end_tlv)); rc = ecore_send_msg2pf(p_hwfn, &resp->hdr.status, resp_size); if (rc) - return rc; + goto exit; - if (resp->hdr.status != PFVF_STATUS_SUCCESS) - return ECORE_INVAL; + if (resp->hdr.status != PFVF_STATUS_SUCCESS) { + rc = ECORE_INVAL; + goto exit; + } ecore_vf_handle_vp_update_tlvs_resp(p_hwfn, p_params); +exit: + ecore_vf_pf_req_end(p_hwfn, rc); + return rc; } enum _ecore_status_t ecore_vf_pf_reset(struct ecore_hwfn *p_hwfn) { struct ecore_vf_iov *p_iov = p_hwfn->vf_iov_info; + struct pfvf_def_resp_tlv *resp; struct vfpf_first_tlv *req; - struct pfvf_def_resp_tlv *resp = &p_iov->pf2vf_reply->default_resp; - int rc; + enum _ecore_status_t rc; /* clear mailbox and prep first tlv */ req = ecore_vf_pf_prep(p_hwfn, CHANNEL_TLV_CLOSE, sizeof(*req)); /* add list termination tlv */ - ecore_add_tlv(p_hwfn, &p_iov->offset, + ecore_add_tlv(&p_iov->offset, CHANNEL_TLV_LIST_END, sizeof(struct channel_list_end_tlv)); + resp = &p_iov->pf2vf_reply->default_resp; rc = ecore_send_msg2pf(p_hwfn, &resp->hdr.status, sizeof(*resp)); if (rc) - return rc; + goto exit; - if (resp->hdr.status != PFVF_STATUS_SUCCESS) - return ECORE_AGAIN; - - p_hwfn->b_int_enabled = 0; - - return ECORE_SUCCESS; -} - -enum _ecore_status_t ecore_vf_pf_release(struct ecore_hwfn *p_hwfn) -{ - struct ecore_vf_iov *p_iov = p_hwfn->vf_iov_info; - struct vfpf_first_tlv *req; - struct pfvf_def_resp_tlv *resp = &p_iov->pf2vf_reply->default_resp; - u32 size; - int rc; - - /* clear mailbox and prep first tlv */ - req = ecore_vf_pf_prep(p_hwfn, CHANNEL_TLV_RELEASE, sizeof(*req)); - - /* add list termination tlv */ - ecore_add_tlv(p_hwfn, &p_iov->offset, - CHANNEL_TLV_LIST_END, - sizeof(struct channel_list_end_tlv)); - - rc = ecore_send_msg2pf(p_hwfn, &resp->hdr.status, sizeof(*resp)); - - if (rc == ECORE_SUCCESS && resp->hdr.status != PFVF_STATUS_SUCCESS) + if (resp->hdr.status != PFVF_STATUS_SUCCESS) { rc = ECORE_AGAIN; - - p_hwfn->b_int_enabled = 0; - - /* TODO - might need to revise this for 100g */ - if (IS_LEAD_HWFN(p_hwfn)) - OSAL_MUTEX_DEALLOC(&p_iov->mutex); - - if (p_iov->vf2pf_request) - OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev, - p_iov->vf2pf_request, - p_iov->vf2pf_request_phys, - sizeof(union vfpf_tlvs)); - if (p_iov->pf2vf_reply) - OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev, - p_iov->pf2vf_reply, - p_iov->pf2vf_reply_phys, - sizeof(union pfvf_tlvs)); - - if (p_iov->bulletin.p_virt) { - size = sizeof(struct ecore_bulletin_content); - OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev, - p_iov->bulletin.p_virt, - p_iov->bulletin.phys, size); + goto exit; } - OSAL_FREE(p_hwfn->p_dev, p_hwfn->vf_iov_info); - p_hwfn->vf_iov_info = OSAL_NULL; + p_hwfn->b_int_enabled = 0; + +exit: + ecore_vf_pf_req_end(p_hwfn, rc); return rc; } @@ -1068,8 +1486,8 @@ enum _ecore_status_t ecore_vf_pf_filter_ucast(struct ecore_hwfn *p_hwfn, { struct ecore_vf_iov *p_iov = p_hwfn->vf_iov_info; struct vfpf_ucast_filter_tlv *req; - struct pfvf_def_resp_tlv *resp = &p_iov->pf2vf_reply->default_resp; - int rc; + struct pfvf_def_resp_tlv *resp; + enum _ecore_status_t rc; /* Sanitize */ if (p_ucast->opcode == ECORE_FILTER_MOVE) { @@ -1086,52 +1504,170 @@ enum _ecore_status_t ecore_vf_pf_filter_ucast(struct ecore_hwfn *p_hwfn, req->vlan = p_ucast->vlan; /* add list termination tlv */ - ecore_add_tlv(p_hwfn, &p_iov->offset, + ecore_add_tlv(&p_iov->offset, CHANNEL_TLV_LIST_END, sizeof(struct channel_list_end_tlv)); + resp = &p_iov->pf2vf_reply->default_resp; rc = ecore_send_msg2pf(p_hwfn, &resp->hdr.status, sizeof(*resp)); if (rc) - return rc; + goto exit; - if (resp->hdr.status != PFVF_STATUS_SUCCESS) - return ECORE_AGAIN; + if (resp->hdr.status != PFVF_STATUS_SUCCESS) { + rc = ECORE_AGAIN; + goto exit; + } - return ECORE_SUCCESS; +exit: + ecore_vf_pf_req_end(p_hwfn, rc); + + return rc; } enum _ecore_status_t ecore_vf_pf_int_cleanup(struct ecore_hwfn *p_hwfn) { struct ecore_vf_iov *p_iov = p_hwfn->vf_iov_info; struct pfvf_def_resp_tlv *resp = &p_iov->pf2vf_reply->default_resp; - int rc; + enum _ecore_status_t rc; /* clear mailbox and prep first tlv */ ecore_vf_pf_prep(p_hwfn, CHANNEL_TLV_INT_CLEANUP, sizeof(struct vfpf_first_tlv)); /* add list termination tlv */ - ecore_add_tlv(p_hwfn, &p_iov->offset, + ecore_add_tlv(&p_iov->offset, CHANNEL_TLV_LIST_END, sizeof(struct channel_list_end_tlv)); rc = ecore_send_msg2pf(p_hwfn, &resp->hdr.status, sizeof(*resp)); if (rc) - return rc; + goto exit; + + if (resp->hdr.status != PFVF_STATUS_SUCCESS) { + rc = ECORE_INVAL; + goto exit; + } + +exit: + ecore_vf_pf_req_end(p_hwfn, rc); + + return rc; +} + +enum _ecore_status_t ecore_vf_pf_get_coalesce(struct ecore_hwfn *p_hwfn, + u16 *p_coal, + struct ecore_queue_cid *p_cid) +{ + struct ecore_vf_iov *p_iov = p_hwfn->vf_iov_info; + struct pfvf_read_coal_resp_tlv *resp; + struct vfpf_read_coal_req_tlv *req; + enum _ecore_status_t rc; + + /* clear mailbox and prep header tlv */ + req = ecore_vf_pf_prep(p_hwfn, CHANNEL_TLV_COALESCE_READ, + sizeof(*req)); + req->qid = p_cid->rel.queue_id; + req->is_rx = p_cid->b_is_rx ? 1 : 0; + + ecore_add_tlv(&p_iov->offset, CHANNEL_TLV_LIST_END, + sizeof(struct channel_list_end_tlv)); + resp = &p_iov->pf2vf_reply->read_coal_resp; + + rc = ecore_send_msg2pf(p_hwfn, &resp->hdr.status, sizeof(*resp)); + if (rc != ECORE_SUCCESS) + goto exit; if (resp->hdr.status != PFVF_STATUS_SUCCESS) - return ECORE_INVAL; + goto exit; - return ECORE_SUCCESS; + *p_coal = resp->coal; +exit: + ecore_vf_pf_req_end(p_hwfn, rc); + + return rc; +} + +enum _ecore_status_t +ecore_vf_pf_set_coalesce(struct ecore_hwfn *p_hwfn, u16 rx_coal, u16 tx_coal, + struct ecore_queue_cid *p_cid) +{ + struct ecore_vf_iov *p_iov = p_hwfn->vf_iov_info; + struct vfpf_update_coalesce *req; + struct pfvf_def_resp_tlv *resp; + enum _ecore_status_t rc; + + /* clear mailbox and prep header tlv */ + req = ecore_vf_pf_prep(p_hwfn, CHANNEL_TLV_COALESCE_UPDATE, + sizeof(*req)); + + req->rx_coal = rx_coal; + req->tx_coal = tx_coal; + req->qid = p_cid->rel.queue_id; + + DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, + "Setting coalesce rx_coal = %d, tx_coal = %d at queue = %d\n", + rx_coal, tx_coal, req->qid); + + /* add list termination tlv */ + ecore_add_tlv(&p_iov->offset, CHANNEL_TLV_LIST_END, + sizeof(struct channel_list_end_tlv)); + + resp = &p_iov->pf2vf_reply->default_resp; + rc = ecore_send_msg2pf(p_hwfn, &resp->hdr.status, sizeof(*resp)); + + if (rc != ECORE_SUCCESS) + goto exit; + + if (resp->hdr.status != PFVF_STATUS_SUCCESS) + goto exit; + + p_hwfn->p_dev->rx_coalesce_usecs = rx_coal; + p_hwfn->p_dev->tx_coalesce_usecs = tx_coal; + +exit: + ecore_vf_pf_req_end(p_hwfn, rc); + return rc; +} + +u16 ecore_vf_get_igu_sb_id(struct ecore_hwfn *p_hwfn, + u16 sb_id) +{ + struct ecore_vf_iov *p_iov = p_hwfn->vf_iov_info; + + if (!p_iov) { + DP_NOTICE(p_hwfn, true, "vf_sriov_info isn't initialized\n"); + return 0; + } + + return p_iov->acquire_resp.resc.hw_sbs[sb_id].hw_sb_id; +} + +void ecore_vf_set_sb_info(struct ecore_hwfn *p_hwfn, + u16 sb_id, struct ecore_sb_info *p_sb) +{ + struct ecore_vf_iov *p_iov = p_hwfn->vf_iov_info; + + if (!p_iov) { + DP_NOTICE(p_hwfn, true, "vf_sriov_info isn't initialized\n"); + return; + } + + if (sb_id >= PFVF_MAX_SBS_PER_VF) { + DP_NOTICE(p_hwfn, true, "Can't configure SB %04x\n", sb_id); + return; + } + + p_iov->sbs_info[sb_id] = p_sb; } enum _ecore_status_t ecore_vf_read_bulletin(struct ecore_hwfn *p_hwfn, u8 *p_change) { - struct ecore_bulletin_content shadow; struct ecore_vf_iov *p_iov = p_hwfn->vf_iov_info; - u32 crc, crc_size = sizeof(p_iov->bulletin.p_virt->crc); + struct ecore_bulletin_content shadow; + u32 crc, crc_size; + crc_size = sizeof(p_iov->bulletin.p_virt->crc); *p_change = 0; /* Need to guarantee PF is not in the middle of writing it */ @@ -1142,8 +1678,8 @@ enum _ecore_status_t ecore_vf_read_bulletin(struct ecore_hwfn *p_hwfn, return ECORE_SUCCESS; /* Verify the bulletin we see is valid */ - crc = ecore_crc32(0, (u8 *)&shadow + crc_size, - p_iov->bulletin.size - crc_size); + crc = OSAL_CRC32(0, (u8 *)&shadow + crc_size, + p_iov->bulletin.size - crc_size); if (crc != shadow.crc) return ECORE_AGAIN; @@ -1158,20 +1694,7 @@ enum _ecore_status_t ecore_vf_read_bulletin(struct ecore_hwfn *p_hwfn, return ECORE_SUCCESS; } -u16 ecore_vf_get_igu_sb_id(struct ecore_hwfn *p_hwfn, u16 sb_id) -{ - struct ecore_vf_iov *p_iov = p_hwfn->vf_iov_info; - - if (!p_iov) { - DP_NOTICE(p_hwfn, true, "vf_sriov_info isn't initialized\n"); - return 0; - } - - return p_iov->acquire_resp.resc.hw_sbs[sb_id].hw_sb_id; -} - -void __ecore_vf_get_link_params(struct ecore_hwfn *p_hwfn, - struct ecore_mcp_link_params *p_params, +void __ecore_vf_get_link_params(struct ecore_mcp_link_params *p_params, struct ecore_bulletin_content *p_bulletin) { OSAL_MEMSET(p_params, 0, sizeof(*p_params)); @@ -1188,12 +1711,11 @@ void __ecore_vf_get_link_params(struct ecore_hwfn *p_hwfn, void ecore_vf_get_link_params(struct ecore_hwfn *p_hwfn, struct ecore_mcp_link_params *params) { - __ecore_vf_get_link_params(p_hwfn, params, + __ecore_vf_get_link_params(params, &p_hwfn->vf_iov_info->bulletin_shadow); } -void __ecore_vf_get_link_state(struct ecore_hwfn *p_hwfn, - struct ecore_mcp_link_state *p_link, +void __ecore_vf_get_link_state(struct ecore_mcp_link_state *p_link, struct ecore_bulletin_content *p_bulletin) { OSAL_MEMSET(p_link, 0, sizeof(*p_link)); @@ -1215,12 +1737,11 @@ void __ecore_vf_get_link_state(struct ecore_hwfn *p_hwfn, void ecore_vf_get_link_state(struct ecore_hwfn *p_hwfn, struct ecore_mcp_link_state *link) { - __ecore_vf_get_link_state(p_hwfn, link, + __ecore_vf_get_link_state(link, &p_hwfn->vf_iov_info->bulletin_shadow); } -void __ecore_vf_get_link_caps(struct ecore_hwfn *p_hwfn, - struct ecore_mcp_link_capabilities *p_link_caps, +void __ecore_vf_get_link_caps(struct ecore_mcp_link_capabilities *p_link_caps, struct ecore_bulletin_content *p_bulletin) { OSAL_MEMSET(p_link_caps, 0, sizeof(*p_link_caps)); @@ -1230,7 +1751,7 @@ void __ecore_vf_get_link_caps(struct ecore_hwfn *p_hwfn, void ecore_vf_get_link_caps(struct ecore_hwfn *p_hwfn, struct ecore_mcp_link_capabilities *p_link_caps) { - __ecore_vf_get_link_caps(p_hwfn, p_link_caps, + __ecore_vf_get_link_caps(p_link_caps, &p_hwfn->vf_iov_info->bulletin_shadow); } @@ -1239,6 +1760,12 @@ void ecore_vf_get_num_rxqs(struct ecore_hwfn *p_hwfn, u8 *num_rxqs) *num_rxqs = p_hwfn->vf_iov_info->acquire_resp.resc.num_rxqs; } +void ecore_vf_get_num_txqs(struct ecore_hwfn *p_hwfn, + u8 *num_txqs) +{ + *num_txqs = p_hwfn->vf_iov_info->acquire_resp.resc.num_txqs; +} + void ecore_vf_get_port_mac(struct ecore_hwfn *p_hwfn, u8 *port_mac) { OSAL_MEMCPY(port_mac, @@ -1255,14 +1782,21 @@ void ecore_vf_get_num_vlan_filters(struct ecore_hwfn *p_hwfn, *num_vlan_filters = p_vf->acquire_resp.resc.num_vlan_filters; } -/* @DPDK */ -void ecore_vf_get_num_mac_filters(struct ecore_hwfn *p_hwfn, - u32 *num_mac) +void ecore_vf_get_num_sbs(struct ecore_hwfn *p_hwfn, + u32 *num_sbs) { struct ecore_vf_iov *p_vf; p_vf = p_hwfn->vf_iov_info; - *num_mac = p_vf->acquire_resp.resc.num_mac_filters; + *num_sbs = (u32)p_vf->acquire_resp.resc.num_sbs; +} + +void ecore_vf_get_num_mac_filters(struct ecore_hwfn *p_hwfn, + u32 *num_mac_filters) +{ + struct ecore_vf_iov *p_vf = p_hwfn->vf_iov_info; + + *num_mac_filters = p_vf->acquire_resp.resc.num_mac_filters; } bool ecore_vf_check_mac(struct ecore_hwfn *p_hwfn, u8 *mac) @@ -1302,6 +1836,18 @@ bool ecore_vf_bulletin_get_forced_mac(struct ecore_hwfn *hwfn, u8 *dst_mac, return true; } +void ecore_vf_bulletin_get_udp_ports(struct ecore_hwfn *p_hwfn, + u16 *p_vxlan_port, + u16 *p_geneve_port) +{ + struct ecore_bulletin_content *p_bulletin; + + p_bulletin = &p_hwfn->vf_iov_info->bulletin_shadow; + + *p_vxlan_port = p_bulletin->vxlan_udp_port; + *p_geneve_port = p_bulletin->geneve_udp_port; +} + bool ecore_vf_bulletin_get_forced_vlan(struct ecore_hwfn *hwfn, u16 *dst_pvid) { struct ecore_bulletin_content *bulletin; @@ -1317,6 +1863,11 @@ bool ecore_vf_bulletin_get_forced_vlan(struct ecore_hwfn *hwfn, u16 *dst_pvid) return true; } +bool ecore_vf_get_pre_fp_hsi(struct ecore_hwfn *p_hwfn) +{ + return p_hwfn->vf_iov_info->b_pre_fp_hsi; +} + void ecore_vf_get_fw_version(struct ecore_hwfn *p_hwfn, u16 *fw_major, u16 *fw_minor, u16 *fw_rev, u16 *fw_eng) @@ -1330,3 +1881,10 @@ void ecore_vf_get_fw_version(struct ecore_hwfn *p_hwfn, *fw_rev = info->fw_rev; *fw_eng = info->fw_eng; } + +#ifdef CONFIG_ECORE_SW_CHANNEL +void ecore_vf_set_hw_channel(struct ecore_hwfn *p_hwfn, bool b_is_hw) +{ + p_hwfn->vf_iov_info->b_hw_channel = b_is_hw; +} +#endif diff --git a/dpdk/drivers/net/qede/base/ecore_vf.h b/dpdk/drivers/net/qede/base/ecore_vf.h index 334b588c..de2758cb 100644 --- a/dpdk/drivers/net/qede/base/ecore_vf.h +++ b/dpdk/drivers/net/qede/base/ecore_vf.h @@ -14,126 +14,178 @@ #include "ecore_l2_api.h" #include "ecore_vfpf_if.h" +/* Default number of CIDs [total of both Rx and Tx] to be requested + * by default. + */ +#define ECORE_ETH_VF_DEFAULT_NUM_CIDS (32) + +/* This data is held in the ecore_hwfn structure for VFs only. */ +struct ecore_vf_iov { + union vfpf_tlvs *vf2pf_request; + dma_addr_t vf2pf_request_phys; + union pfvf_tlvs *pf2vf_reply; + dma_addr_t pf2vf_reply_phys; + + /* Should be taken whenever the mailbox buffers are accessed */ + osal_mutex_t mutex; + u8 *offset; + + /* Bulletin Board */ + struct ecore_bulletin bulletin; + struct ecore_bulletin_content bulletin_shadow; + + /* we set aside a copy of the acquire response */ + struct pfvf_acquire_resp_tlv acquire_resp; + + /* In case PF originates prior to the fp-hsi version comparison, + * this has to be propagated as it affects the fastpath. + */ + bool b_pre_fp_hsi; + + /* Current day VFs are passing the SBs physical address on vport + * start, and as they lack an IGU mapping they need to store the + * addresses of previously registered SBs. + * Even if we were to change configuration flow, due to backward + * compatibility [with older PFs] we'd still need to store these. + */ + struct ecore_sb_info *sbs_info[PFVF_MAX_SBS_PER_VF]; + +#ifdef CONFIG_ECORE_SW_CHANNEL + /* Would be set if the VF is to try communicating with it PF + * using a hw channel. + */ + bool b_hw_channel; +#endif + + /* Determines whether VF utilizes doorbells via limited register + * bar or via the doorbell bar. + */ + bool b_doorbell_bar; +}; + +/** + * @brief VF - Get coalesce per VF's relative queue. + * + * @param p_hwfn + * @param p_coal - coalesce value in micro second for VF queues. + * @param p_cid - queue cid + * + **/ +enum _ecore_status_t ecore_vf_pf_get_coalesce(struct ecore_hwfn *p_hwfn, + u16 *p_coal, + struct ecore_queue_cid *p_cid); +/** + * @brief VF - Set Rx/Tx coalesce per VF's relative queue. + * Coalesce value '0' will omit the configuration. + * + * @param p_hwfn + * @param rx_coal - coalesce value in micro second for rx queue + * @param tx_coal - coalesce value in micro second for tx queue + * @param p_cid - queue cid + * + **/ +enum _ecore_status_t ecore_vf_pf_set_coalesce(struct ecore_hwfn *p_hwfn, + u16 rx_coal, u16 tx_coal, + struct ecore_queue_cid *p_cid); + #ifdef CONFIG_ECORE_SRIOV /** - * * @brief hw preparation for VF * sends ACQUIRE message * - * @param p_dev - * - * @return enum _ecore_status_t - */ -enum _ecore_status_t ecore_vf_hw_prepare(struct ecore_dev *p_dev); - -/** - * - * @brief VF init in hw (equivalent to hw_init in PF) - * mark interrupts as enabled - * * @param p_hwfn * * @return enum _ecore_status_t */ -enum _ecore_status_t ecore_vf_pf_init(struct ecore_hwfn *p_hwfn); +enum _ecore_status_t ecore_vf_hw_prepare(struct ecore_hwfn *p_hwfn); /** - * * @brief VF - start the RX Queue by sending a message to the PF * * @param p_hwfn - * @param cid - zero based within the VF - * @param rx_queue_id - zero based within the VF - * @param sb - VF status block for this queue - * @param sb_index - Index within the status block + * @param p_cid - Only relative fields are relevant * @param bd_max_bytes - maximum number of bytes per bd * @param bd_chain_phys_addr - physical address of bd chain * @param cqe_pbl_addr - physical address of pbl * @param cqe_pbl_size - pbl size * @param pp_prod - pointer to the producer to be - * used in fasthwfn + * used in fasthpath * * @return enum _ecore_status_t */ enum _ecore_status_t ecore_vf_pf_rxq_start(struct ecore_hwfn *p_hwfn, - u8 rx_queue_id, - u16 sb, - u8 sb_index, + struct ecore_queue_cid *p_cid, u16 bd_max_bytes, dma_addr_t bd_chain_phys_addr, dma_addr_t cqe_pbl_addr, u16 cqe_pbl_size, - void OSAL_IOMEM * *pp_prod); + void OSAL_IOMEM **pp_prod); /** - * * @brief VF - start the TX queue by sending a message to the * PF. * * @param p_hwfn - * @param tx_queue_id - zero based within the VF - * @param sb - status block for this queue - * @param sb_index - index within the status block + * @param p_cid * @param bd_chain_phys_addr - physical address of tx chain * @param pp_doorbell - pointer to address to which to - * write the doorbell too.. + * write the doorbell too.. * * @return enum _ecore_status_t */ -enum _ecore_status_t ecore_vf_pf_txq_start(struct ecore_hwfn *p_hwfn, - u16 tx_queue_id, - u16 sb, - u8 sb_index, - dma_addr_t pbl_addr, - u16 pbl_size, - void OSAL_IOMEM * *pp_doorbell); +enum _ecore_status_t +ecore_vf_pf_txq_start(struct ecore_hwfn *p_hwfn, + struct ecore_queue_cid *p_cid, + dma_addr_t pbl_addr, u16 pbl_size, + void OSAL_IOMEM **pp_doorbell); /** - * * @brief VF - stop the RX queue by sending a message to the PF * * @param p_hwfn - * @param rx_qid + * @param p_cid * @param cqe_completion * * @return enum _ecore_status_t */ enum _ecore_status_t ecore_vf_pf_rxq_stop(struct ecore_hwfn *p_hwfn, - u16 rx_qid, bool cqe_completion); + struct ecore_queue_cid *p_cid, + bool cqe_completion); /** - * * @brief VF - stop the TX queue by sending a message to the PF * * @param p_hwfn - * @param tx_qid + * @param p_cid * * @return enum _ecore_status_t */ enum _ecore_status_t ecore_vf_pf_txq_stop(struct ecore_hwfn *p_hwfn, - u16 tx_qid); + struct ecore_queue_cid *p_cid); +/* TODO - fix all the !SRIOV prototypes */ + +#ifndef LINUX_REMOVE /** * @brief VF - update the RX queue by sending a message to the * PF * * @param p_hwfn - * @param rx_queue_id + * @param pp_cid - list of queue-cids which we want to update * @param num_rxqs - * @param init_sge_ring * @param comp_cqe_flg * @param comp_event_flg * * @return enum _ecore_status_t */ enum _ecore_status_t ecore_vf_pf_rxqs_update(struct ecore_hwfn *p_hwfn, - u16 rx_queue_id, + struct ecore_queue_cid **pp_cid, u8 num_rxqs, u8 comp_cqe_flg, u8 comp_event_flg); +#endif /** - * * @brief VF - send a vport update command * * @param p_hwfn @@ -146,7 +198,6 @@ ecore_vf_pf_vport_update(struct ecore_hwfn *p_hwfn, struct ecore_sp_vport_update_params *p_params); /** - * * @brief VF - send a close message to PF * * @param p_hwfn @@ -156,7 +207,6 @@ ecore_vf_pf_vport_update(struct ecore_hwfn *p_hwfn, enum _ecore_status_t ecore_vf_pf_reset(struct ecore_hwfn *p_hwfn); /** - * * @brief VF - free vf`s memories * * @param p_hwfn @@ -166,7 +216,6 @@ enum _ecore_status_t ecore_vf_pf_reset(struct ecore_hwfn *p_hwfn); enum _ecore_status_t ecore_vf_pf_release(struct ecore_hwfn *p_hwfn); /** - * * @brief ecore_vf_get_igu_sb_id - Get the IGU SB ID for a given * sb_id. For VFs igu sbs don't have to be contiguous * @@ -175,7 +224,18 @@ enum _ecore_status_t ecore_vf_pf_release(struct ecore_hwfn *p_hwfn); * * @return INLINE u16 */ -u16 ecore_vf_get_igu_sb_id(struct ecore_hwfn *p_hwfn, u16 sb_id); +u16 ecore_vf_get_igu_sb_id(struct ecore_hwfn *p_hwfn, + u16 sb_id); + +/** + * @brief Stores [or removes] a configured sb_info. + * + * @param p_hwfn + * @param sb_id - zero-based SB index [for fastpath] + * @param sb_info - may be OSAL_NULL [during removal]. + */ +void ecore_vf_set_sb_info(struct ecore_hwfn *p_hwfn, + u16 sb_id, struct ecore_sb_info *p_sb); /** * @brief ecore_vf_pf_vport_start - perform vport start for VF. @@ -190,13 +250,14 @@ u16 ecore_vf_get_igu_sb_id(struct ecore_hwfn *p_hwfn, u16 sb_id); * * @return enum _ecore_status */ -enum _ecore_status_t ecore_vf_pf_vport_start(struct ecore_hwfn *p_hwfn, - u8 vport_id, - u16 mtu, - u8 inner_vlan_removal, - enum ecore_tpa_mode tpa_mode, - u8 max_buffers_per_cqe, - u8 only_untagged); +enum _ecore_status_t ecore_vf_pf_vport_start( + struct ecore_hwfn *p_hwfn, + u8 vport_id, + u16 mtu, + u8 inner_vlan_removal, + enum ecore_tpa_mode tpa_mode, + u8 max_buffers_per_cqe, + u8 only_untagged); /** * @brief ecore_vf_pf_vport_stop - stop the VF's vport @@ -207,9 +268,9 @@ enum _ecore_status_t ecore_vf_pf_vport_start(struct ecore_hwfn *p_hwfn, */ enum _ecore_status_t ecore_vf_pf_vport_stop(struct ecore_hwfn *p_hwfn); -enum _ecore_status_t ecore_vf_pf_filter_ucast(struct ecore_hwfn *p_hwfn, - struct ecore_filter_ucast - *p_param); +enum _ecore_status_t ecore_vf_pf_filter_ucast( + struct ecore_hwfn *p_hwfn, + struct ecore_filter_ucast *p_param); void ecore_vf_pf_filter_mcast(struct ecore_hwfn *p_hwfn, struct ecore_filter_mcast *p_filter_cmd); @@ -226,190 +287,37 @@ enum _ecore_status_t ecore_vf_pf_int_cleanup(struct ecore_hwfn *p_hwfn); /** * @brief - return the link params in a given bulletin board * - * @param p_hwfn * @param p_params - pointer to a struct to fill with link params * @param p_bulletin */ -void __ecore_vf_get_link_params(struct ecore_hwfn *p_hwfn, - struct ecore_mcp_link_params *p_params, +void __ecore_vf_get_link_params(struct ecore_mcp_link_params *p_params, struct ecore_bulletin_content *p_bulletin); /** * @brief - return the link state in a given bulletin board * - * @param p_hwfn * @param p_link - pointer to a struct to fill with link state * @param p_bulletin */ -void __ecore_vf_get_link_state(struct ecore_hwfn *p_hwfn, - struct ecore_mcp_link_state *p_link, +void __ecore_vf_get_link_state(struct ecore_mcp_link_state *p_link, struct ecore_bulletin_content *p_bulletin); /** * @brief - return the link capabilities in a given bulletin board * - * @param p_hwfn * @param p_link - pointer to a struct to fill with link capabilities * @param p_bulletin */ -void __ecore_vf_get_link_caps(struct ecore_hwfn *p_hwfn, - struct ecore_mcp_link_capabilities *p_link_caps, +void __ecore_vf_get_link_caps(struct ecore_mcp_link_capabilities *p_link_caps, struct ecore_bulletin_content *p_bulletin); -#else -static OSAL_INLINE enum _ecore_status_t ecore_vf_hw_prepare(struct ecore_dev - *p_dev) -{ - return ECORE_INVAL; -} +enum _ecore_status_t +ecore_vf_pf_tunnel_param_update(struct ecore_hwfn *p_hwfn, + struct ecore_tunnel_info *p_tunn); -static OSAL_INLINE enum _ecore_status_t ecore_vf_pf_init(struct ecore_hwfn - *p_hwfn) -{ - return ECORE_INVAL; -} +void ecore_vf_set_vf_start_tunn_update_param(struct ecore_tunnel_info *p_tun); -static OSAL_INLINE enum _ecore_status_t ecore_vf_pf_rxq_start(struct ecore_hwfn - *p_hwfn, - u8 rx_queue_id, - u16 sb, - u8 sb_index, - u16 bd_max_bytes, - dma_addr_t - bd_chain_phys_adr, - dma_addr_t - cqe_pbl_addr, - u16 cqe_pbl_size, - void OSAL_IOMEM * - *pp_prod) -{ - return ECORE_INVAL; -} - -static OSAL_INLINE enum _ecore_status_t ecore_vf_pf_txq_start(struct ecore_hwfn - *p_hwfn, - u16 tx_queue_id, - u16 sb, - u8 sb_index, - dma_addr_t - pbl_addr, - u16 pbl_size, - void OSAL_IOMEM * - *pp_doorbell) -{ - return ECORE_INVAL; -} - -static OSAL_INLINE enum _ecore_status_t ecore_vf_pf_rxq_stop(struct ecore_hwfn - *p_hwfn, - u16 rx_qid, - bool - cqe_completion) -{ - return ECORE_INVAL; -} - -static OSAL_INLINE enum _ecore_status_t ecore_vf_pf_txq_stop(struct ecore_hwfn - *p_hwfn, - u16 tx_qid) -{ - return ECORE_INVAL; -} - -static OSAL_INLINE enum _ecore_status_t ecore_vf_pf_rxqs_update(struct - ecore_hwfn - * p_hwfn, - u16 rx_queue_id, - u8 num_rxqs, - u8 comp_cqe_flg, - u8 - comp_event_flg) -{ - return ECORE_INVAL; -} - -static OSAL_INLINE enum _ecore_status_t ecore_vf_pf_vport_update( - struct ecore_hwfn *p_hwfn, - struct ecore_sp_vport_update_params *p_params) -{ - return ECORE_INVAL; -} - -static OSAL_INLINE enum _ecore_status_t ecore_vf_pf_reset(struct ecore_hwfn - *p_hwfn) -{ - return ECORE_INVAL; -} - -static OSAL_INLINE enum _ecore_status_t ecore_vf_pf_release(struct ecore_hwfn - *p_hwfn) -{ - return ECORE_INVAL; -} - -static OSAL_INLINE u16 ecore_vf_get_igu_sb_id(struct ecore_hwfn *p_hwfn, - u16 sb_id) -{ - return 0; -} - -static OSAL_INLINE enum _ecore_status_t ecore_vf_pf_vport_start( - struct ecore_hwfn *p_hwfn, u8 vport_id, u16 mtu, - u8 inner_vlan_removal, enum ecore_tpa_mode tpa_mode, - u8 max_buffers_per_cqe, u8 only_untagged) -{ - return ECORE_INVAL; -} - -static OSAL_INLINE enum _ecore_status_t ecore_vf_pf_vport_stop( - struct ecore_hwfn *p_hwfn) -{ - return ECORE_INVAL; -} - -static OSAL_INLINE enum _ecore_status_t ecore_vf_pf_filter_ucast( - struct ecore_hwfn *p_hwfn, struct ecore_filter_ucast *p_param) -{ - return ECORE_INVAL; -} - -static OSAL_INLINE void ecore_vf_pf_filter_mcast(struct ecore_hwfn *p_hwfn, - struct ecore_filter_mcast - *p_filter_cmd) -{ -} - -static OSAL_INLINE enum _ecore_status_t ecore_vf_pf_int_cleanup(struct - ecore_hwfn - * p_hwfn) -{ - return ECORE_INVAL; -} - -static OSAL_INLINE void __ecore_vf_get_link_params(struct ecore_hwfn *p_hwfn, - struct ecore_mcp_link_params - *p_params, - struct ecore_bulletin_content - *p_bulletin) -{ -} - -static OSAL_INLINE void __ecore_vf_get_link_state(struct ecore_hwfn *p_hwfn, - struct ecore_mcp_link_state - *p_link, - struct ecore_bulletin_content - *p_bulletin) -{ -} - -static OSAL_INLINE void __ecore_vf_get_link_caps(struct ecore_hwfn *p_hwfn, - struct - ecore_mcp_link_capabilities - * p_link_caps, - struct ecore_bulletin_content - *p_bulletin) -{ -} +u32 ecore_vf_hw_bar_size(struct ecore_hwfn *p_hwfn, + enum BAR_ID bar_id); #endif - #endif /* __ECORE_VF_H__ */ diff --git a/dpdk/drivers/net/qede/base/ecore_vf_api.h b/dpdk/drivers/net/qede/base/ecore_vf_api.h index f28b6860..9815cf8a 100644 --- a/dpdk/drivers/net/qede/base/ecore_vf_api.h +++ b/dpdk/drivers/net/qede/base/ecore_vf_api.h @@ -57,7 +57,17 @@ void ecore_vf_get_link_caps(struct ecore_hwfn *p_hwfn, * @param p_hwfn * @param num_rxqs - allocated RX queues */ -void ecore_vf_get_num_rxqs(struct ecore_hwfn *p_hwfn, u8 *num_rxqs); +void ecore_vf_get_num_rxqs(struct ecore_hwfn *p_hwfn, + u8 *num_rxqs); + +/** + * @brief Get number of Rx queues allocated for VF by ecore + * + * @param p_hwfn + * @param num_txqs - allocated RX queues + */ +void ecore_vf_get_num_txqs(struct ecore_hwfn *p_hwfn, + u8 *num_txqs); /** * @brief Get port mac address for VF @@ -65,7 +75,8 @@ void ecore_vf_get_num_rxqs(struct ecore_hwfn *p_hwfn, u8 *num_rxqs); * @param p_hwfn * @param port_mac - destination location for port mac */ -void ecore_vf_get_port_mac(struct ecore_hwfn *p_hwfn, u8 *port_mac); +void ecore_vf_get_port_mac(struct ecore_hwfn *p_hwfn, + u8 *port_mac); /** * @brief Get number of VLAN filters allocated for VF by ecore @@ -76,11 +87,14 @@ void ecore_vf_get_port_mac(struct ecore_hwfn *p_hwfn, u8 *port_mac); void ecore_vf_get_num_vlan_filters(struct ecore_hwfn *p_hwfn, u8 *num_vlan_filters); +void ecore_vf_get_num_sbs(struct ecore_hwfn *p_hwfn, + u32 *num_sbs); + /** * @brief Get number of MAC filters allocated for VF by ecore * * @param p_hwfn - * @param num_mac - allocated MAC filters + * @param num_rxqs - allocated MAC filters */ void ecore_vf_get_num_mac_filters(struct ecore_hwfn *p_hwfn, u32 *num_mac_filters); @@ -95,13 +109,14 @@ void ecore_vf_get_num_mac_filters(struct ecore_hwfn *p_hwfn, */ bool ecore_vf_check_mac(struct ecore_hwfn *p_hwfn, u8 *mac); +#ifndef LINUX_REMOVE /** * @brief Copy forced MAC address from bulletin board * * @param hwfn * @param dst_mac * @param p_is_forced - out param which indicate in case mac - * exist if it forced or not. + * exist if it forced or not. * * @return bool - return true if mac exist and false if * not. @@ -120,8 +135,20 @@ bool ecore_vf_bulletin_get_forced_mac(struct ecore_hwfn *hwfn, u8 *dst_mac, bool ecore_vf_bulletin_get_forced_vlan(struct ecore_hwfn *hwfn, u16 *dst_pvid); /** - * @brief Set firmware version information in dev_info from VFs acquire response - * tlv + * @brief Check if VF is based on PF whose driver is pre-fp-hsi version; + * This affects the fastpath implementation of the driver. + * + * @param p_hwfn + * + * @return bool - true iff PF is pre-fp-hsi version. + */ +bool ecore_vf_get_pre_fp_hsi(struct ecore_hwfn *p_hwfn); + +#endif + +/** + * @brief Set firmware version information in dev_info from VFs acquire + * response tlv * * @param p_hwfn * @param fw_major @@ -131,70 +158,23 @@ bool ecore_vf_bulletin_get_forced_vlan(struct ecore_hwfn *hwfn, u16 *dst_pvid); */ void ecore_vf_get_fw_version(struct ecore_hwfn *p_hwfn, u16 *fw_major, - u16 *fw_minor, u16 *fw_rev, u16 *fw_eng); -#else -static OSAL_INLINE enum _ecore_status_t ecore_vf_read_bulletin(struct ecore_hwfn - *p_hwfn, - u8 *p_change) -{ - return ECORE_INVAL; -} + u16 *fw_minor, + u16 *fw_rev, + u16 *fw_eng); +void ecore_vf_bulletin_get_udp_ports(struct ecore_hwfn *p_hwfn, + u16 *p_vxlan_port, u16 *p_geneve_port); -static OSAL_INLINE void ecore_vf_get_link_params(struct ecore_hwfn *p_hwfn, - struct ecore_mcp_link_params - *params) -{ -} - -static OSAL_INLINE void ecore_vf_get_link_state(struct ecore_hwfn *p_hwfn, - struct ecore_mcp_link_state - *link) -{ -} - -static OSAL_INLINE void ecore_vf_get_link_caps(struct ecore_hwfn *p_hwfn, - struct - ecore_mcp_link_capabilities - * p_link_caps) -{ -} - -static OSAL_INLINE void ecore_vf_get_num_rxqs(struct ecore_hwfn *p_hwfn, - u8 *num_rxqs) -{ -} - -static OSAL_INLINE void ecore_vf_get_port_mac(struct ecore_hwfn *p_hwfn, - u8 *port_mac) -{ -} - -static OSAL_INLINE void ecore_vf_get_num_vlan_filters(struct ecore_hwfn *p_hwfn, - u8 *num_vlan_filters) -{ -} - -static OSAL_INLINE void ecore_vf_get_num_mac_filters(struct ecore_hwfn *p_hwfn, - u32 *num_mac) -{ -} - -static OSAL_INLINE bool ecore_vf_check_mac(struct ecore_hwfn *p_hwfn, u8 *mac) -{ - return false; -} - -static OSAL_INLINE bool ecore_vf_bulletin_get_forced_mac(struct ecore_hwfn - *hwfn, u8 *dst_mac, - u8 *p_is_forced) -{ - return false; -} - -static OSAL_INLINE void ecore_vf_get_fw_version(struct ecore_hwfn *p_hwfn, - u16 *fw_major, u16 *fw_minor, - u16 *fw_rev, u16 *fw_eng) -{ -} +#ifdef CONFIG_ECORE_SW_CHANNEL +/** + * @brief set the VF to use a SW/HW channel when communicating with PF. + * NOTICE: today the likely first place to call this from VF + * would be OSAL_VF_FILL_ACQUIRE_RESC_REQ(); Might want to consider + * something a bit more appropriate. + * + * @param p_hwfn + * @param b_is_hw - true iff VF is to use a HW-channel + */ +void ecore_vf_set_hw_channel(struct ecore_hwfn *p_hwfn, bool b_is_hw); +#endif #endif #endif diff --git a/dpdk/drivers/net/qede/base/ecore_vfpf_if.h b/dpdk/drivers/net/qede/base/ecore_vfpf_if.h index 2fa4d155..ecb00649 100644 --- a/dpdk/drivers/net/qede/base/ecore_vfpf_if.h +++ b/dpdk/drivers/net/qede/base/ecore_vfpf_if.h @@ -9,11 +9,9 @@ #ifndef __ECORE_VF_PF_IF_H__ #define __ECORE_VF_PF_IF_H__ +/* @@@ TBD MichalK this should be HSI? */ #define T_ETH_INDIRECTION_TABLE_SIZE 128 -#define T_ETH_RSS_KEY_SIZE 10 -#ifndef aligned_u64 -#define aligned_u64 u64 -#endif +#define T_ETH_RSS_KEY_SIZE 10 /* @@@ TBD this should be HSI? */ /*********************************************** * @@ -26,13 +24,14 @@ struct vf_pf_resc_request { u8 num_sbs; u8 num_mac_filters; u8 num_vlan_filters; - u8 num_mc_filters; /* No limit so superfluous */ - u16 padding; + u8 num_mc_filters; /* No limit so superfluous */ + u8 num_cids; + u8 padding; }; struct hw_sb_info { - u16 hw_sb_id; /* aka absolute igu id, used to ack the sb */ - u8 sb_qid; /* used to update DHC for sb */ + u16 hw_sb_id; /* aka absolute igu id, used to ack the sb */ + u8 sb_qid; /* used to update DHC for sb */ u8 padding[5]; }; @@ -44,29 +43,6 @@ struct hw_sb_info { * **/ #define TLV_BUFFER_SIZE 1024 -#define TLV_ALIGN sizeof(u64) -#define PF_VF_BULLETIN_SIZE 512 - -#define VFPF_RX_MASK_ACCEPT_NONE 0x00000000 -#define VFPF_RX_MASK_ACCEPT_MATCHED_UNICAST 0x00000001 -#define VFPF_RX_MASK_ACCEPT_MATCHED_MULTICAST 0x00000002 -#define VFPF_RX_MASK_ACCEPT_ALL_UNICAST 0x00000004 -#define VFPF_RX_MASK_ACCEPT_ALL_MULTICAST 0x00000008 -#define VFPF_RX_MASK_ACCEPT_BROADCAST 0x00000010 -/* TODO: #define VFPF_RX_MASK_ACCEPT_ANY_VLAN 0x00000020 */ - -#define BULLETIN_CONTENT_SIZE (sizeof(struct pf_vf_bulletin_content)) -#define BULLETIN_ATTEMPTS 5 /* crc failures before throwing towel */ -#define BULLETIN_CRC_SEED 0 - -enum { - PFVF_STATUS_WAITING = 0, - PFVF_STATUS_SUCCESS, - PFVF_STATUS_FAILURE, - PFVF_STATUS_NOT_SUPPORTED, - PFVF_STATUS_NO_RESOURCE, - PFVF_STATUS_FORCED, -}; /* vf pf channel tlvs */ /* general tlv header (used for both vf->pf request and pf->vf response) */ @@ -81,7 +57,7 @@ struct channel_tlv { struct vfpf_first_tlv { struct channel_tlv tl; u32 padding; - aligned_u64 reply_address; + u64 reply_address; }; /* header of pf->vf tlvs, carries the status of handling the request */ @@ -107,38 +83,63 @@ struct vfpf_acquire_tlv { struct vfpf_first_tlv first_tlv; struct vf_pf_vfdev_info { -#define VFPF_ACQUIRE_CAP_OVERRIDE_FW_VER (1 << 0) - aligned_u64 capabilties; +#ifndef LINUX_REMOVE + /* First bit was used on 8.7.x and 8.8.x versions, which had different + * FWs used but with the same faspath HSI. As this was prior to the + * fastpath versioning, wanted to have ability to override fw matching + * and allow them to interact. + */ +#endif +/* VF pre-FP hsi version */ +#define VFPF_ACQUIRE_CAP_PRE_FP_HSI (1 << 0) +#define VFPF_ACQUIRE_CAP_100G (1 << 1) /* VF can support 100g */ + + /* A requirement for supporting multi-Tx queues on a single queue-zone, + * VF would pass qids as additional information whenever passing queue + * references. + * TODO - due to the CID limitations in Bar0, VFs currently don't pass + * this, and use the legacy CID scheme. + */ +#define VFPF_ACQUIRE_CAP_QUEUE_QIDS (1 << 2) + + /* The VF is using the physical bar. While this is mostly internal + * to the VF, might affect the number of CIDs supported assuming + * QUEUE_QIDS is set. + */ +#define VFPF_ACQUIRE_CAP_PHYSICAL_BAR (1 << 3) + u64 capabilities; u8 fw_major; u8 fw_minor; u8 fw_revision; u8 fw_engineering; u32 driver_version; - u16 opaque_fid; /* ME register value */ - u8 os_type; /* VFPF_ACQUIRE_OS_* value */ - u8 padding[5]; + u16 opaque_fid; /* ME register value */ + u8 os_type; /* VFPF_ACQUIRE_OS_* value */ + u8 eth_fp_hsi_major; + u8 eth_fp_hsi_minor; + u8 padding[3]; } vfdev_info; struct vf_pf_resc_request resc_request; - aligned_u64 bulletin_addr; + u64 bulletin_addr; u32 bulletin_size; u32 padding; }; /* receive side scaling tlv */ struct vfpf_vport_update_rss_tlv { - struct channel_tlv tl; + struct channel_tlv tl; u8 update_rss_flags; -#define VFPF_UPDATE_RSS_CONFIG_FLAG (1 << 0) -#define VFPF_UPDATE_RSS_CAPS_FLAG (1 << 1) -#define VFPF_UPDATE_RSS_IND_TABLE_FLAG (1 << 2) -#define VFPF_UPDATE_RSS_KEY_FLAG (1 << 3) + #define VFPF_UPDATE_RSS_CONFIG_FLAG (1 << 0) + #define VFPF_UPDATE_RSS_CAPS_FLAG (1 << 1) + #define VFPF_UPDATE_RSS_IND_TABLE_FLAG (1 << 2) + #define VFPF_UPDATE_RSS_KEY_FLAG (1 << 3) u8 rss_enable; u8 rss_caps; - u8 rss_table_size_log; /* The table size is 2 ^ rss_table_size_log */ + u8 rss_table_size_log; /* The table size is 2 ^ rss_table_size_log */ u16 rss_ind_table[T_ETH_INDIRECTION_TABLE_SIZE]; u32 rss_key[T_ETH_RSS_KEY_SIZE]; }; @@ -168,23 +169,47 @@ struct pfvf_acquire_resp_tlv { u16 fw_rev; u16 fw_eng; - aligned_u64 capabilities; + u64 capabilities; #define PFVF_ACQUIRE_CAP_DEFAULT_UNTAGGED (1 << 0) +#define PFVF_ACQUIRE_CAP_100G (1 << 1) /* If set, 100g PF */ +/* There are old PF versions where the PF might mistakenly override the sanity + * mechanism [version-based] and allow a VF that can't be supported to pass + * the acquisition phase. + * To overcome this, PFs now indicate that they're past that point and the new + * VFs would fail probe on the older PFs that fail to do so. + */ +#ifndef LINUX_REMOVE +/* Said bug was in quest/serpens; Can't be certain no official release included + * the bug since the fix arrived very late in the programs. + */ +#endif +#define PFVF_ACQUIRE_CAP_POST_FW_OVERRIDE (1 << 2) + + /* PF expects queues to be received with additional qids */ +#define PFVF_ACQUIRE_CAP_QUEUE_QIDS (1 << 3) u16 db_size; - u8 indices_per_sb; + u8 indices_per_sb; u8 os_type; - /* Thesee should match the PF's ecore_dev values */ + /* These should match the PF's ecore_dev values */ u16 chip_rev; u8 dev_type; - u8 padding; + /* Doorbell bar size configured in HW: log(size) or 0 */ + u8 bar_size; struct pfvf_stats_info stats_info; u8 port_mac[ETH_ALEN]; - u8 padding2[2]; + + /* It's possible PF had to configure an older fastpath HSI + * [in case VF is newer than PF]. This is communicated back + * to the VF. It can also be used in case of error due to + * non-matching versions to shed light in VF about failure. + */ + u8 major_fp_hsi; + u8 minor_fp_hsi; } pfdev_info; struct pf_vf_resc { @@ -192,174 +217,194 @@ struct pfvf_acquire_resp_tlv { * this struct with suggested amount of resources for next * acquire request */ -#define PFVF_MAX_QUEUES_PER_VF 16 -#define PFVF_MAX_SBS_PER_VF 16 + #define PFVF_MAX_QUEUES_PER_VF 16 + #define PFVF_MAX_SBS_PER_VF 16 struct hw_sb_info hw_sbs[PFVF_MAX_SBS_PER_VF]; - u8 hw_qid[PFVF_MAX_QUEUES_PER_VF]; - u8 cid[PFVF_MAX_QUEUES_PER_VF]; + u8 hw_qid[PFVF_MAX_QUEUES_PER_VF]; + u8 cid[PFVF_MAX_QUEUES_PER_VF]; - u8 num_rxqs; - u8 num_txqs; - u8 num_sbs; - u8 num_mac_filters; - u8 num_vlan_filters; - u8 num_mc_filters; - u8 padding[2]; + u8 num_rxqs; + u8 num_txqs; + u8 num_sbs; + u8 num_mac_filters; + u8 num_vlan_filters; + u8 num_mc_filters; + u8 num_cids; + u8 padding; } resc; u32 bulletin_size; u32 padding; }; -/* Init VF */ -struct vfpf_init_tlv { - struct vfpf_first_tlv first_tlv; - aligned_u64 stats_addr; - - u16 rx_mask; - u16 tx_mask; - u8 drop_ttl0_flg; - u8 padding[3]; +struct pfvf_start_queue_resp_tlv { + struct pfvf_tlv hdr; + u32 offset; /* offset to consumer/producer of queue */ + u8 padding[4]; +}; +/* Extended queue information - additional index for reference inside qzone. + * If commmunicated between VF/PF, each TLV relating to queues should be + * extended by one such [or have a future base TLV that already contains info]. + */ +struct vfpf_qid_tlv { + struct channel_tlv tl; + u8 qid; + u8 padding[3]; }; /* Setup Queue */ struct vfpf_start_rxq_tlv { - struct vfpf_first_tlv first_tlv; + struct vfpf_first_tlv first_tlv; /* physical addresses */ - aligned_u64 rxq_addr; - aligned_u64 deprecated_sge_addr; - aligned_u64 cqe_pbl_addr; + u64 rxq_addr; + u64 deprecated_sge_addr; + u64 cqe_pbl_addr; - u16 cqe_pbl_size; - u16 hw_sb; - u16 rx_qid; - u16 hc_rate; /* desired interrupts per sec. */ + u16 cqe_pbl_size; + u16 hw_sb; + u16 rx_qid; + u16 hc_rate; /* desired interrupts per sec. */ - u16 bd_max_bytes; - u16 stat_id; - u8 sb_index; - u8 padding[3]; + u16 bd_max_bytes; + u16 stat_id; + u8 sb_index; + u8 padding[3]; }; struct vfpf_start_txq_tlv { - struct vfpf_first_tlv first_tlv; + struct vfpf_first_tlv first_tlv; /* physical addresses */ - aligned_u64 pbl_addr; - u16 pbl_size; - u16 stat_id; - u16 tx_qid; - u16 hw_sb; + u64 pbl_addr; + u16 pbl_size; + u16 stat_id; + u16 tx_qid; + u16 hw_sb; - u32 flags; /* VFPF_QUEUE_FLG_X flags */ - u16 hc_rate; /* desired interrupts per sec. */ - u8 sb_index; - u8 padding[3]; + u32 flags; /* VFPF_QUEUE_FLG_X flags */ + u16 hc_rate; /* desired interrupts per sec. */ + u8 sb_index; + u8 padding[3]; }; /* Stop RX Queue */ struct vfpf_stop_rxqs_tlv { - struct vfpf_first_tlv first_tlv; + struct vfpf_first_tlv first_tlv; - u16 rx_qid; - u8 num_rxqs; - u8 cqe_completion; - u8 padding[4]; + u16 rx_qid; + + /* While the API supports multiple Rx-queues on a single TLV + * message, in practice older VFs always used it as one [ecore]. + * And there are PFs [starting with the CHANNEL_TLV_QID] which + * would start assuming this is always a '1'. So in practice this + * field should be considered deprecated and *Always* set to '1'. + */ + u8 num_rxqs; + + u8 cqe_completion; + u8 padding[4]; }; /* Stop TX Queues */ struct vfpf_stop_txqs_tlv { - struct vfpf_first_tlv first_tlv; + struct vfpf_first_tlv first_tlv; - u16 tx_qid; - u8 num_txqs; - u8 padding[5]; + u16 tx_qid; + + /* While the API supports multiple Tx-queues on a single TLV + * message, in practice older VFs always used it as one [ecore]. + * And there are PFs [starting with the CHANNEL_TLV_QID] which + * would start assuming this is always a '1'. So in practice this + * field should be considered deprecated and *Always* set to '1'. + */ + u8 num_txqs; + u8 padding[5]; }; struct vfpf_update_rxq_tlv { - struct vfpf_first_tlv first_tlv; + struct vfpf_first_tlv first_tlv; - aligned_u64 deprecated_sge_addr[PFVF_MAX_QUEUES_PER_VF]; + u64 deprecated_sge_addr[PFVF_MAX_QUEUES_PER_VF]; - u16 rx_qid; - u8 num_rxqs; - u8 flags; -#define VFPF_RXQ_UPD_INIT_SGE_DEPRECATE_FLAG (1 << 0) -#define VFPF_RXQ_UPD_COMPLETE_CQE_FLAG (1 << 1) -#define VFPF_RXQ_UPD_COMPLETE_EVENT_FLAG (1 << 2) + u16 rx_qid; + u8 num_rxqs; + u8 flags; + #define VFPF_RXQ_UPD_INIT_SGE_DEPRECATE_FLAG (1 << 0) + #define VFPF_RXQ_UPD_COMPLETE_CQE_FLAG (1 << 1) + #define VFPF_RXQ_UPD_COMPLETE_EVENT_FLAG (1 << 2) - u8 padding[4]; + u8 padding[4]; }; /* Set Queue Filters */ struct vfpf_q_mac_vlan_filter { u32 flags; -#define VFPF_Q_FILTER_DEST_MAC_VALID 0x01 -#define VFPF_Q_FILTER_VLAN_TAG_VALID 0x02 -#define VFPF_Q_FILTER_SET_MAC 0x100 /* set/clear */ + #define VFPF_Q_FILTER_DEST_MAC_VALID 0x01 + #define VFPF_Q_FILTER_VLAN_TAG_VALID 0x02 + #define VFPF_Q_FILTER_SET_MAC 0x100 /* set/clear */ - u8 mac[ETH_ALEN]; + u8 mac[ETH_ALEN]; u16 vlan_tag; - u8 padding[4]; + u8 padding[4]; }; /* Start a vport */ struct vfpf_vport_start_tlv { - struct vfpf_first_tlv first_tlv; + struct vfpf_first_tlv first_tlv; - aligned_u64 sb_addr[PFVF_MAX_SBS_PER_VF]; + u64 sb_addr[PFVF_MAX_SBS_PER_VF]; - u32 tpa_mode; - u16 dep1; - u16 mtu; + u32 tpa_mode; + u16 dep1; + u16 mtu; - u8 vport_id; - u8 inner_vlan_removal; + u8 vport_id; + u8 inner_vlan_removal; - u8 only_untagged; - u8 max_buffers_per_cqe; + u8 only_untagged; + u8 max_buffers_per_cqe; - u8 padding[4]; + u8 padding[4]; }; /* Extended tlvs - need to add rss, mcast, accept mode tlvs */ struct vfpf_vport_update_activate_tlv { - struct channel_tlv tl; - u8 update_rx; - u8 update_tx; - u8 active_rx; - u8 active_tx; + struct channel_tlv tl; + u8 update_rx; + u8 update_tx; + u8 active_rx; + u8 active_tx; }; struct vfpf_vport_update_tx_switch_tlv { - struct channel_tlv tl; - u8 tx_switching; - u8 padding[3]; + struct channel_tlv tl; + u8 tx_switching; + u8 padding[3]; }; struct vfpf_vport_update_vlan_strip_tlv { - struct channel_tlv tl; - u8 remove_vlan; - u8 padding[3]; + struct channel_tlv tl; + u8 remove_vlan; + u8 padding[3]; }; struct vfpf_vport_update_mcast_bin_tlv { - struct channel_tlv tl; - u8 padding[4]; + struct channel_tlv tl; + u8 padding[4]; - aligned_u64 bins[8]; + u64 bins[8]; }; struct vfpf_vport_update_accept_param_tlv { struct channel_tlv tl; - u8 update_rx_mode; - u8 update_tx_mode; - u8 rx_accept_filter; - u8 tx_accept_filter; + u8 update_rx_mode; + u8 update_tx_mode; + u8 rx_accept_filter; + u8 tx_accept_filter; }; struct vfpf_vport_update_accept_any_vlan_tlv { @@ -371,29 +416,31 @@ struct vfpf_vport_update_accept_any_vlan_tlv { }; struct vfpf_vport_update_sge_tpa_tlv { - struct channel_tlv tl; + struct channel_tlv tl; - u16 sge_tpa_flags; -#define VFPF_TPA_IPV4_EN_FLAG (1 << 0) -#define VFPF_TPA_IPV6_EN_FLAG (1 << 1) -#define VFPF_TPA_PKT_SPLIT_FLAG (1 << 2) -#define VFPF_TPA_HDR_DATA_SPLIT_FLAG (1 << 3) -#define VFPF_TPA_GRO_CONSIST_FLAG (1 << 4) + u16 sge_tpa_flags; + #define VFPF_TPA_IPV4_EN_FLAG (1 << 0) + #define VFPF_TPA_IPV6_EN_FLAG (1 << 1) + #define VFPF_TPA_PKT_SPLIT_FLAG (1 << 2) + #define VFPF_TPA_HDR_DATA_SPLIT_FLAG (1 << 3) + #define VFPF_TPA_GRO_CONSIST_FLAG (1 << 4) + #define VFPF_TPA_TUNN_IPV4_EN_FLAG (1 << 5) + #define VFPF_TPA_TUNN_IPV6_EN_FLAG (1 << 6) - u8 update_sge_tpa_flags; -#define VFPF_UPDATE_SGE_DEPRECATED_FLAG (1 << 0) -#define VFPF_UPDATE_TPA_EN_FLAG (1 << 1) -#define VFPF_UPDATE_TPA_PARAM_FLAG (1 << 2) + u8 update_sge_tpa_flags; + #define VFPF_UPDATE_SGE_DEPRECATED_FLAG (1 << 0) + #define VFPF_UPDATE_TPA_EN_FLAG (1 << 1) + #define VFPF_UPDATE_TPA_PARAM_FLAG (1 << 2) - u8 max_buffers_per_cqe; + u8 max_buffers_per_cqe; - u16 deprecated_sge_buff_size; - u16 tpa_max_size; - u16 tpa_min_size_to_start; - u16 tpa_min_size_to_cont; + u16 deprecated_sge_buff_size; + u16 tpa_max_size; + u16 tpa_min_size_to_start; + u16 tpa_min_size_to_cont; - u8 tpa_max_aggs_num; - u8 padding[7]; + u8 tpa_max_aggs_num; + u8 padding[7]; }; @@ -405,42 +452,103 @@ struct vfpf_vport_update_tlv { }; struct vfpf_ucast_filter_tlv { - struct vfpf_first_tlv first_tlv; + struct vfpf_first_tlv first_tlv; - u8 opcode; - u8 type; + u8 opcode; + u8 type; - u8 mac[ETH_ALEN]; + u8 mac[ETH_ALEN]; - u16 vlan; - u16 padding[3]; + u16 vlan; + u16 padding[3]; +}; + +/* tunnel update param tlv */ +struct vfpf_update_tunn_param_tlv { + struct vfpf_first_tlv first_tlv; + + u8 tun_mode_update_mask; + u8 tunn_mode; + u8 update_tun_cls; + u8 vxlan_clss; + u8 l2gre_clss; + u8 ipgre_clss; + u8 l2geneve_clss; + u8 ipgeneve_clss; + u8 update_geneve_port; + u8 update_vxlan_port; + u16 geneve_port; + u16 vxlan_port; + u8 padding[2]; +}; + +struct pfvf_update_tunn_param_tlv { + struct pfvf_tlv hdr; + + u16 tunn_feature_mask; + u8 vxlan_mode; + u8 l2geneve_mode; + u8 ipgeneve_mode; + u8 l2gre_mode; + u8 ipgre_mode; + u8 vxlan_clss; + u8 l2gre_clss; + u8 ipgre_clss; + u8 l2geneve_clss; + u8 ipgeneve_clss; + u16 vxlan_udp_port; + u16 geneve_udp_port; }; struct tlv_buffer_size { u8 tlv_buffer[TLV_BUFFER_SIZE]; }; -union vfpf_tlvs { +struct vfpf_update_coalesce { struct vfpf_first_tlv first_tlv; - struct vfpf_acquire_tlv acquire; - struct vfpf_init_tlv init; - struct vfpf_start_rxq_tlv start_rxq; - struct vfpf_start_txq_tlv start_txq; - struct vfpf_stop_rxqs_tlv stop_rxqs; - struct vfpf_stop_txqs_tlv stop_txqs; - struct vfpf_update_rxq_tlv update_rxq; - struct vfpf_vport_start_tlv start_vport; - struct vfpf_vport_update_tlv vport_update; - struct vfpf_ucast_filter_tlv ucast_filter; - struct channel_list_end_tlv list_end; - struct tlv_buffer_size tlv_buf_size; + u16 rx_coal; + u16 tx_coal; + u16 qid; + u8 padding[2]; +}; + +struct vfpf_read_coal_req_tlv { + struct vfpf_first_tlv first_tlv; + u16 qid; + u8 is_rx; + u8 padding[5]; +}; + +struct pfvf_read_coal_resp_tlv { + struct pfvf_tlv hdr; + u16 coal; + u8 padding[6]; +}; + +union vfpf_tlvs { + struct vfpf_first_tlv first_tlv; + struct vfpf_acquire_tlv acquire; + struct vfpf_start_rxq_tlv start_rxq; + struct vfpf_start_txq_tlv start_txq; + struct vfpf_stop_rxqs_tlv stop_rxqs; + struct vfpf_stop_txqs_tlv stop_txqs; + struct vfpf_update_rxq_tlv update_rxq; + struct vfpf_vport_start_tlv start_vport; + struct vfpf_vport_update_tlv vport_update; + struct vfpf_ucast_filter_tlv ucast_filter; + struct vfpf_update_tunn_param_tlv tunn_param_update; + struct vfpf_update_coalesce update_coalesce; + struct vfpf_read_coal_req_tlv read_coal_req; + struct tlv_buffer_size tlv_buf_size; }; union pfvf_tlvs { - struct pfvf_def_resp_tlv default_resp; - struct pfvf_acquire_resp_tlv acquire_resp; - struct channel_list_end_tlv list_end; - struct tlv_buffer_size tlv_buf_size; + struct pfvf_def_resp_tlv default_resp; + struct pfvf_acquire_resp_tlv acquire_resp; + struct tlv_buffer_size tlv_buf_size; + struct pfvf_start_queue_resp_tlv queue_start; + struct pfvf_update_tunn_param_tlv tunn_param_resp; + struct pfvf_read_coal_resp_tlv read_coal_resp; }; /* This is a structure which is allocated in the VF, which the PF may update @@ -469,20 +577,19 @@ enum ecore_bulletin_bit { }; struct ecore_bulletin_content { - u32 crc; /* crc of structure to ensure is not in - * mid-update - */ + /* crc of structure to ensure is not in mid-update */ + u32 crc; + u32 version; - aligned_u64 valid_bitmap; /* bitmap indicating wich fields - * hold valid values - */ + /* bitmap indicating which fields hold valid values */ + u64 valid_bitmap; - u8 mac[ETH_ALEN]; /* used for MAC_ADDR or MAC_ADDR_FORCED */ + /* used for MAC_ADDR or MAC_ADDR_FORCED */ + u8 mac[ETH_ALEN]; - u8 default_only_untagged; /* If valid, 1 => only untagged Rx - * if no vlan filter is configured. - */ + /* If valid, 1 => only untagged Rx if no vlan is configured */ + u8 default_only_untagged; u8 padding; /* The following is a 'copy' of ecore_mcp_link_state, @@ -509,9 +616,12 @@ struct ecore_bulletin_content { u8 pfc_enabled; u8 partner_tx_flow_ctrl_en; u8 partner_rx_flow_ctrl_en; + u8 partner_adv_pause; u8 sfp_tx_fault; - u8 padding4[6]; + u16 vxlan_udp_port; + u16 geneve_udp_port; + u8 padding4[2]; u32 speed; u32 partner_adv_speed; @@ -529,11 +639,10 @@ struct ecore_bulletin { u32 size; }; -#ifndef print_enum enum { /*!!!!! Make sure to update STRINGS structure accordingly !!!!!*/ - CHANNEL_TLV_NONE, /* ends tlv sequence */ + CHANNEL_TLV_NONE, /* ends tlv sequence */ CHANNEL_TLV_ACQUIRE, CHANNEL_TLV_VPORT_START, CHANNEL_TLV_VPORT_UPDATE, @@ -556,35 +665,19 @@ enum { CHANNEL_TLV_VPORT_UPDATE_RSS, CHANNEL_TLV_VPORT_UPDATE_ACCEPT_ANY_VLAN, CHANNEL_TLV_VPORT_UPDATE_SGE_TPA, - CHANNEL_TLV_MAX + CHANNEL_TLV_UPDATE_TUNN_PARAM, + CHANNEL_TLV_COALESCE_UPDATE, + CHANNEL_TLV_QID, + CHANNEL_TLV_COALESCE_READ, + CHANNEL_TLV_MAX, + + /* Required for iterating over vport-update tlvs. + * Will break in case non-sequential vport-update tlvs. + */ + CHANNEL_TLV_VPORT_UPDATE_MAX = CHANNEL_TLV_VPORT_UPDATE_SGE_TPA + 1, + /*!!!!! Make sure to update STRINGS structure accordingly !!!!!*/ }; extern const char *ecore_channel_tlvs_string[]; -#else -print_enum(channel_tlvs, CHANNEL_TLV_NONE, /* ends tlv sequence */ - CHANNEL_TLV_ACQUIRE, - CHANNEL_TLV_VPORT_START, - CHANNEL_TLV_VPORT_UPDATE, - CHANNEL_TLV_VPORT_TEARDOWN, - CHANNEL_TLV_SETUP_RXQ, - CHANNEL_TLV_SETUP_TXQ, - CHANNEL_TLV_STOP_RXQS, - CHANNEL_TLV_STOP_TXQS, - CHANNEL_TLV_UPDATE_RXQ, - CHANNEL_TLV_INT_CLEANUP, - CHANNEL_TLV_CLOSE, - CHANNEL_TLV_RELEASE, - CHANNEL_TLV_LIST_END, - CHANNEL_TLV_UCAST_FILTER, - CHANNEL_TLV_VPORT_UPDATE_ACTIVATE, - CHANNEL_TLV_VPORT_UPDATE_TX_SWITCH, - CHANNEL_TLV_VPORT_UPDATE_VLAN_STRIP, - CHANNEL_TLV_VPORT_UPDATE_MCAST, - CHANNEL_TLV_VPORT_UPDATE_ACCEPT_PARAM, - CHANNEL_TLV_VPORT_UPDATE_RSS, - CHANNEL_TLV_VPORT_UPDATE_ACCEPT_ANY_VLAN, - CHANNEL_TLV_VPORT_UPDATE_SGE_TPA, CHANNEL_TLV_MAX); -#endif - #endif /* __ECORE_VF_PF_IF_H__ */ diff --git a/dpdk/drivers/net/qede/base/eth_common.h b/dpdk/drivers/net/qede/base/eth_common.h index 046bbb20..45a0356d 100644 --- a/dpdk/drivers/net/qede/base/eth_common.h +++ b/dpdk/drivers/net/qede/base/eth_common.h @@ -11,60 +11,115 @@ /********************/ /* ETH FW CONSTANTS */ /********************/ + +/* FP HSI version. FP HSI is compatible if (fwVer.major == drvVer.major && + * fwVer.minor >= drvVer.minor) + */ +/* ETH FP HSI Major version */ +#define ETH_HSI_VER_MAJOR 3 +/* ETH FP HSI Minor version */ +#define ETH_HSI_VER_MINOR 10 + +/* Alias for 8.7.x.x/8.8.x.x ETH FP HSI MINOR version. In this version driver + * is not required to set pkt_len field in eth_tx_1st_bd struct, and tunneling + * offload is not supported. + */ +#define ETH_HSI_VER_NO_PKT_LEN_TUNN 5 + #define ETH_CACHE_LINE_SIZE 64 -#define ETH_RX_CQE_GAP 32 -#define ETH_MAX_RAMROD_PER_CON 8 -#define ETH_TX_BD_PAGE_SIZE_BYTES 4096 -#define ETH_RX_BD_PAGE_SIZE_BYTES 4096 -#define ETH_RX_CQE_PAGE_SIZE_BYTES 4096 -#define ETH_RX_NUM_NEXT_PAGE_BDS 2 +#define ETH_RX_CQE_GAP 32 +#define ETH_MAX_RAMROD_PER_CON 8 +#define ETH_TX_BD_PAGE_SIZE_BYTES 4096 +#define ETH_RX_BD_PAGE_SIZE_BYTES 4096 +#define ETH_RX_CQE_PAGE_SIZE_BYTES 4096 +#define ETH_RX_NUM_NEXT_PAGE_BDS 2 -#define ETH_TX_MIN_BDS_PER_NON_LSO_PKT 1 -#define ETH_TX_MAX_BDS_PER_NON_LSO_PACKET 18 -#define ETH_TX_MAX_LSO_HDR_NBD 4 -#define ETH_TX_MIN_BDS_PER_LSO_PKT 3 -#define ETH_TX_MIN_BDS_PER_TUNN_IPV6_WITH_EXT_PKT 3 -#define ETH_TX_MIN_BDS_PER_IPV6_WITH_EXT_PKT 2 -#define ETH_TX_MIN_BDS_PER_PKT_W_LOOPBACK_MODE 2 -#define ETH_TX_MAX_NON_LSO_PKT_LEN (9700 - (4 + 12 + 8)) +/* Limitation for Tunneled LSO Packets on the offset (in bytes) of the inner IP + * header (relevant to LSO for tunneled packet): + */ +/* Offset is limited to 253 bytes (inclusive). */ +#define ETH_MAX_TUNN_LSO_INNER_IPV4_OFFSET 253 +/* Offset is limited to 251 bytes (inclusive). */ +#define ETH_MAX_TUNN_LSO_INNER_IPV6_OFFSET 251 + +#define ETH_TX_MIN_BDS_PER_NON_LSO_PKT 1 +#define ETH_TX_MAX_BDS_PER_NON_LSO_PACKET 18 +#define ETH_TX_MAX_BDS_PER_LSO_PACKET 255 +#define ETH_TX_MAX_LSO_HDR_NBD 4 +#define ETH_TX_MIN_BDS_PER_LSO_PKT 3 +#define ETH_TX_MIN_BDS_PER_TUNN_IPV6_WITH_EXT_PKT 3 +#define ETH_TX_MIN_BDS_PER_IPV6_WITH_EXT_PKT 2 +#define ETH_TX_MIN_BDS_PER_PKT_W_LOOPBACK_MODE 2 +/* (QM_REG_TASKBYTECRDCOST_0, QM_VOQ_BYTE_CRD_TASK_COST) - + * (VLAN-TAG + CRC + IPG + PREAMBLE) + */ +#define ETH_TX_MAX_NON_LSO_PKT_LEN (9700 - (4 + 4 + 12 + 8)) #define ETH_TX_MAX_LSO_HDR_BYTES 510 -#define ETH_TX_LSO_WINDOW_BDS_NUM 18 +/* Number of BDs to consider for LSO sliding window restriction is + * (ETH_TX_LSO_WINDOW_BDS_NUM - hdr_nbd) + */ +#define ETH_TX_LSO_WINDOW_BDS_NUM (18 - 1) +/* Minimum data length (in bytes) in LSO sliding window */ #define ETH_TX_LSO_WINDOW_MIN_LEN 9700 -#define ETH_TX_MAX_LSO_PAYLOAD_LEN 0xFFFF +/* Maximum LSO packet TCP payload length (in bytes) */ +#define ETH_TX_MAX_LSO_PAYLOAD_LEN 0xFE000 +/* Number of same-as-last resources in tx switching */ +#define ETH_TX_NUM_SAME_AS_LAST_ENTRIES 320 +/* Value for a connection for which same as last feature is disabled */ +#define ETH_TX_INACTIVE_SAME_AS_LAST 0xFFFF -#define ETH_NUM_STATISTIC_COUNTERS MAX_NUM_VPORTS +/* Maximum number of statistics counters */ +#define ETH_NUM_STATISTIC_COUNTERS MAX_NUM_VPORTS +/* Maximum number of statistics counters when doubled VF zone used */ +#define ETH_NUM_STATISTIC_COUNTERS_DOUBLE_VF_ZONE \ + (ETH_NUM_STATISTIC_COUNTERS - MAX_NUM_VFS / 2) +/* Maximum number of statistics counters when quad VF zone used */ +#define ETH_NUM_STATISTIC_COUNTERS_QUAD_VF_ZONE \ + (ETH_NUM_STATISTIC_COUNTERS - 3 * MAX_NUM_VFS / 4) +/* Maximum number of buffers, used for RX packet placement */ #define ETH_RX_MAX_BUFF_PER_PKT 5 +/* Minimum number of free BDs in RX ring, that guarantee receiving of at least + * one RX packet. + */ +#define ETH_RX_BD_THRESHOLD 12 /* num of MAC/VLAN filters */ -#define ETH_NUM_MAC_FILTERS 512 -#define ETH_NUM_VLAN_FILTERS 512 +#define ETH_NUM_MAC_FILTERS 512 +#define ETH_NUM_VLAN_FILTERS 512 /* approx. multicast constants */ -#define ETH_MULTICAST_BIN_FROM_MAC_SEED 0 -#define ETH_MULTICAST_MAC_BINS 256 -#define ETH_MULTICAST_MAC_BINS_IN_REGS (ETH_MULTICAST_MAC_BINS / 32) +/* CRC seed for multicast bin calculation */ +#define ETH_MULTICAST_BIN_FROM_MAC_SEED 0 +#define ETH_MULTICAST_MAC_BINS 256 +#define ETH_MULTICAST_MAC_BINS_IN_REGS (ETH_MULTICAST_MAC_BINS / 32) /* ethernet vport update constants */ -#define ETH_FILTER_RULES_COUNT 10 -#define ETH_RSS_IND_TABLE_ENTRIES_NUM 128 -#define ETH_RSS_KEY_SIZE_REGS 10 +#define ETH_FILTER_RULES_COUNT 10 +/* number of RSS indirection table entries, per Vport) */ +#define ETH_RSS_IND_TABLE_ENTRIES_NUM 128 +/* Length of RSS key (in regs) */ +#define ETH_RSS_KEY_SIZE_REGS 10 +/* number of available RSS engines in K2 */ #define ETH_RSS_ENGINE_NUM_K2 207 +/* number of available RSS engines in BB */ #define ETH_RSS_ENGINE_NUM_BB 127 /* TPA constants */ +/* Maximum number of open TPA aggregations */ #define ETH_TPA_MAX_AGGS_NUM 64 +/* Maximum number of additional buffers, reported by TPA-start CQE */ #define ETH_TPA_CQE_START_LEN_LIST_SIZE ETH_RX_MAX_BUFF_PER_PKT +/* Maximum number of buffers, reported by TPA-continue CQE */ #define ETH_TPA_CQE_CONT_LEN_LIST_SIZE 6 +/* Maximum number of buffers, reported by TPA-end CQE */ #define ETH_TPA_CQE_END_LEN_LIST_SIZE 4 -/* - * Interrupt coalescing TimeSet - */ -struct coalescing_timeset { - u8 timeset; - u8 valid /* Only if this flag is set, timeset will take effect */; -}; +/* Control frame check constants */ +/* Number of etherType values configured by driver for control frame check */ +#define ETH_CTL_FRAME_ETH_TYPE_NUM 4 + + /* * Destination port mode @@ -77,6 +132,7 @@ enum dest_port_mode { MAX_DEST_PORT_MODE }; + /* * Ethernet address type */ @@ -88,22 +144,38 @@ enum eth_addr_type { MAX_ETH_ADDR_TYPE }; + struct eth_tx_1st_bd_flags { u8 bitfields; +/* Set to 1 in the first BD. (for debug) */ #define ETH_TX_1ST_BD_FLAGS_START_BD_MASK 0x1 #define ETH_TX_1ST_BD_FLAGS_START_BD_SHIFT 0 +/* Do not allow additional VLAN manipulations on this packet. */ #define ETH_TX_1ST_BD_FLAGS_FORCE_VLAN_MODE_MASK 0x1 #define ETH_TX_1ST_BD_FLAGS_FORCE_VLAN_MODE_SHIFT 1 +/* Recalculate IP checksum. For tunneled packet - relevant to inner header. */ #define ETH_TX_1ST_BD_FLAGS_IP_CSUM_MASK 0x1 #define ETH_TX_1ST_BD_FLAGS_IP_CSUM_SHIFT 2 +/* Recalculate TCP/UDP checksum. + * For tunneled packet - relevant to inner header. + */ #define ETH_TX_1ST_BD_FLAGS_L4_CSUM_MASK 0x1 #define ETH_TX_1ST_BD_FLAGS_L4_CSUM_SHIFT 3 +/* If set, insert VLAN tag from vlan field to the packet. + * For tunneled packet - relevant to outer header. + */ #define ETH_TX_1ST_BD_FLAGS_VLAN_INSERTION_MASK 0x1 #define ETH_TX_1ST_BD_FLAGS_VLAN_INSERTION_SHIFT 4 +/* If set, this is an LSO packet. Note: For Tunneled LSO packets, the offset of + * the inner IPV4 (and IPV6) header is limited to 253 (and 251 respectively) + * bytes, inclusive. + */ #define ETH_TX_1ST_BD_FLAGS_LSO_MASK 0x1 #define ETH_TX_1ST_BD_FLAGS_LSO_SHIFT 5 +/* Recalculate Tunnel IP Checksum (if Tunnel IP Header is IPv4) */ #define ETH_TX_1ST_BD_FLAGS_TUNN_IP_CSUM_MASK 0x1 #define ETH_TX_1ST_BD_FLAGS_TUNN_IP_CSUM_SHIFT 6 +/* Recalculate Tunnel UDP/GRE Checksum (Depending on Tunnel Type) */ #define ETH_TX_1ST_BD_FLAGS_TUNN_L4_CSUM_MASK 0x1 #define ETH_TX_1ST_BD_FLAGS_TUNN_L4_CSUM_SHIFT 7 }; @@ -112,48 +184,76 @@ struct eth_tx_1st_bd_flags { * The parsing information data for the first tx bd of a given packet. */ struct eth_tx_data_1st_bd { - __le16 vlan /* VLAN to insert to packet (if needed). */; - /* Number of BDs in packet. Should be at least 2 in non-LSO - * packet and at least 3 in LSO (or Tunnel with IPv6+ext) packet. - */ +/* VLAN tag to insert to packet (if enabled by vlan_insertion flag). */ + __le16 vlan; +/* Number of BDs in packet. Should be at least 1 in non-LSO packet and at least + * 3 in LSO (or Tunnel with IPv6+ext) packet. + */ u8 nbds; struct eth_tx_1st_bd_flags bd_flags; __le16 bitfields; -#define ETH_TX_DATA_1ST_BD_TUNN_CFG_OVERRIDE_MASK 0x1 -#define ETH_TX_DATA_1ST_BD_TUNN_CFG_OVERRIDE_SHIFT 0 -#define ETH_TX_DATA_1ST_BD_RESERVED0_MASK 0x1 -#define ETH_TX_DATA_1ST_BD_RESERVED0_SHIFT 1 -#define ETH_TX_DATA_1ST_BD_FW_USE_ONLY_MASK 0x3FFF -#define ETH_TX_DATA_1ST_BD_FW_USE_ONLY_SHIFT 2 +/* Indicates a tunneled packet. Must be set for encapsulated packet. */ +#define ETH_TX_DATA_1ST_BD_TUNN_FLAG_MASK 0x1 +#define ETH_TX_DATA_1ST_BD_TUNN_FLAG_SHIFT 0 +#define ETH_TX_DATA_1ST_BD_RESERVED0_MASK 0x1 +#define ETH_TX_DATA_1ST_BD_RESERVED0_SHIFT 1 +/* Total packet length - must be filled for non-LSO packets. */ +#define ETH_TX_DATA_1ST_BD_PKT_LEN_MASK 0x3FFF +#define ETH_TX_DATA_1ST_BD_PKT_LEN_SHIFT 2 }; /* * The parsing information data for the second tx bd of a given packet. */ struct eth_tx_data_2nd_bd { +/* For tunnel with IPv6+ext - Tunnel header IP datagram length (in BYTEs) */ __le16 tunn_ip_size; __le16 bitfields1; +/* For Tunnel header with IPv6 ext. - Inner L2 Header Size (in 2-byte WORDs) */ #define ETH_TX_DATA_2ND_BD_TUNN_INNER_L2_HDR_SIZE_W_MASK 0xF #define ETH_TX_DATA_2ND_BD_TUNN_INNER_L2_HDR_SIZE_W_SHIFT 0 +/* For Tunnel header with IPv6 ext. - Inner L2 Header MAC DA Type + * (use enum eth_addr_type) + */ #define ETH_TX_DATA_2ND_BD_TUNN_INNER_ETH_TYPE_MASK 0x3 #define ETH_TX_DATA_2ND_BD_TUNN_INNER_ETH_TYPE_SHIFT 4 +/* Destination port mode. (use enum dest_port_mode) */ #define ETH_TX_DATA_2ND_BD_DEST_PORT_MODE_MASK 0x3 #define ETH_TX_DATA_2ND_BD_DEST_PORT_MODE_SHIFT 6 +/* Should be 0 in all the BDs, except the first one. (for debug) */ #define ETH_TX_DATA_2ND_BD_START_BD_MASK 0x1 #define ETH_TX_DATA_2ND_BD_START_BD_SHIFT 8 +/* For Tunnel header with IPv6 ext. - Tunnel Type (use enum eth_tx_tunn_type) */ #define ETH_TX_DATA_2ND_BD_TUNN_TYPE_MASK 0x3 #define ETH_TX_DATA_2ND_BD_TUNN_TYPE_SHIFT 9 +/* For LSO / Tunnel header with IPv6+ext - Set if inner header is IPv6 */ #define ETH_TX_DATA_2ND_BD_TUNN_INNER_IPV6_MASK 0x1 #define ETH_TX_DATA_2ND_BD_TUNN_INNER_IPV6_SHIFT 11 +/* In tunneling mode - Set to 1 when the Inner header is IPv6 with extension. + * Otherwise set to 1 if the header is IPv6 with extension. + */ #define ETH_TX_DATA_2ND_BD_IPV6_EXT_MASK 0x1 #define ETH_TX_DATA_2ND_BD_IPV6_EXT_SHIFT 12 +/* Set to 1 if Tunnel (outer = encapsulating) header has IPv6 ext. (Note: 3rd BD + * is required, hence EDPM does not support Tunnel [outer] header with Ipv6Ext) + */ #define ETH_TX_DATA_2ND_BD_TUNN_IPV6_EXT_MASK 0x1 #define ETH_TX_DATA_2ND_BD_TUNN_IPV6_EXT_SHIFT 13 +/* Set if (inner) L4 protocol is UDP. (Required when IPv6+ext (or tunnel with + * inner or outer Ipv6+ext) and l4_csum is set) + */ #define ETH_TX_DATA_2ND_BD_L4_UDP_MASK 0x1 #define ETH_TX_DATA_2ND_BD_L4_UDP_SHIFT 14 +/* The pseudo header checksum type in the L4 checksum field. Required when + * IPv6+ext and l4_csum is set. (use enum eth_l4_pseudo_checksum_mode) + */ #define ETH_TX_DATA_2ND_BD_L4_PSEUDO_CSUM_MODE_MASK 0x1 #define ETH_TX_DATA_2ND_BD_L4_PSEUDO_CSUM_MODE_SHIFT 15 __le16 bitfields2; +/* For inner/outer header IPv6+ext - (inner) L4 header offset (in 2-byte WORDs). + * For regular packet - offset from the beginning of the packet. For tunneled + * packet - offset from the beginning of the inner header + */ #define ETH_TX_DATA_2ND_BD_L4_HDR_START_OFFSET_W_MASK 0x1FFF #define ETH_TX_DATA_2ND_BD_L4_HDR_START_OFFSET_W_SHIFT 0 #define ETH_TX_DATA_2ND_BD_RESERVED0_MASK 0x7 @@ -164,36 +264,63 @@ struct eth_tx_data_2nd_bd { * Firmware data for L2-EDPM packet. */ struct eth_edpm_fw_data { - struct eth_tx_data_1st_bd data_1st_bd - /* Parsing information data from the 1st BD. */; - struct eth_tx_data_2nd_bd data_2nd_bd - /* Parsing information data from the 2nd BD. */; +/* Parsing information data from the 1st BD. */ + struct eth_tx_data_1st_bd data_1st_bd; +/* Parsing information data from the 2nd BD. */ + struct eth_tx_data_2nd_bd data_2nd_bd; __le32 reserved; }; + /* * FW debug. */ struct eth_fast_path_cqe_fw_debug { - u8 reserved0 /* FW reserved. */; - u8 reserved1 /* FW reserved. */; __le16 reserved2 /* FW reserved. */; }; -struct tunnel_parsing_flags { + +/* + * tunneling parsing flags + */ +struct eth_tunnel_parsing_flags { u8 flags; -#define TUNNEL_PARSING_FLAGS_TYPE_MASK 0x3 -#define TUNNEL_PARSING_FLAGS_TYPE_SHIFT 0 -#define TUNNEL_PARSING_FLAGS_TENNANT_ID_EXIST_MASK 0x1 -#define TUNNEL_PARSING_FLAGS_TENNANT_ID_EXIST_SHIFT 2 -#define TUNNEL_PARSING_FLAGS_NEXT_PROTOCOL_MASK 0x3 -#define TUNNEL_PARSING_FLAGS_NEXT_PROTOCOL_SHIFT 3 -#define TUNNEL_PARSING_FLAGS_FIRSTHDRIPMATCH_MASK 0x1 -#define TUNNEL_PARSING_FLAGS_FIRSTHDRIPMATCH_SHIFT 5 -#define TUNNEL_PARSING_FLAGS_IPV4_FRAGMENT_MASK 0x1 -#define TUNNEL_PARSING_FLAGS_IPV4_FRAGMENT_SHIFT 6 -#define TUNNEL_PARSING_FLAGS_IPV4_OPTIONS_MASK 0x1 -#define TUNNEL_PARSING_FLAGS_IPV4_OPTIONS_SHIFT 7 +/* 0 - no tunneling, 1 - GENEVE, 2 - GRE, 3 - VXLAN + * (use enum eth_rx_tunn_type) + */ +#define ETH_TUNNEL_PARSING_FLAGS_TYPE_MASK 0x3 +#define ETH_TUNNEL_PARSING_FLAGS_TYPE_SHIFT 0 +/* If it s not an encapsulated packet then put 0x0. If it s an encapsulated + * packet but the tenant-id doesn t exist then put 0x0. Else put 0x1 + * + */ +#define ETH_TUNNEL_PARSING_FLAGS_TENNANT_ID_EXIST_MASK 0x1 +#define ETH_TUNNEL_PARSING_FLAGS_TENNANT_ID_EXIST_SHIFT 2 +/* Type of the next header above the tunneling: 0 - unknown, 1 - L2, 2 - Ipv4, + * 3 - IPv6 (use enum tunnel_next_protocol) + */ +#define ETH_TUNNEL_PARSING_FLAGS_NEXT_PROTOCOL_MASK 0x3 +#define ETH_TUNNEL_PARSING_FLAGS_NEXT_PROTOCOL_SHIFT 3 +/* The result of comparing the DA-ip of the tunnel header. */ +#define ETH_TUNNEL_PARSING_FLAGS_FIRSTHDRIPMATCH_MASK 0x1 +#define ETH_TUNNEL_PARSING_FLAGS_FIRSTHDRIPMATCH_SHIFT 5 +#define ETH_TUNNEL_PARSING_FLAGS_IPV4_FRAGMENT_MASK 0x1 +#define ETH_TUNNEL_PARSING_FLAGS_IPV4_FRAGMENT_SHIFT 6 +#define ETH_TUNNEL_PARSING_FLAGS_IPV4_OPTIONS_MASK 0x1 +#define ETH_TUNNEL_PARSING_FLAGS_IPV4_OPTIONS_SHIFT 7 +}; + +/* + * PMD flow control bits + */ +struct eth_pmd_flow_flags { + u8 flags; +#define ETH_PMD_FLOW_FLAGS_VALID_MASK 0x1 /* CQE valid bit */ +#define ETH_PMD_FLOW_FLAGS_VALID_SHIFT 0 +#define ETH_PMD_FLOW_FLAGS_TOGGLE_MASK 0x1 /* CQE ring toggle bit */ +#define ETH_PMD_FLOW_FLAGS_TOGGLE_SHIFT 1 +#define ETH_PMD_FLOW_FLAGS_RESERVED_MASK 0x3F +#define ETH_PMD_FLOW_FLAGS_RESERVED_SHIFT 2 }; /* @@ -202,47 +329,47 @@ struct tunnel_parsing_flags { struct eth_fast_path_rx_reg_cqe { u8 type /* CQE type */; u8 bitfields; +/* Type of calculated RSS hash (use enum rss_hash_type) */ #define ETH_FAST_PATH_RX_REG_CQE_RSS_HASH_TYPE_MASK 0x7 #define ETH_FAST_PATH_RX_REG_CQE_RSS_HASH_TYPE_SHIFT 0 +/* Traffic Class */ #define ETH_FAST_PATH_RX_REG_CQE_TC_MASK 0xF #define ETH_FAST_PATH_RX_REG_CQE_TC_SHIFT 3 #define ETH_FAST_PATH_RX_REG_CQE_RESERVED0_MASK 0x1 #define ETH_FAST_PATH_RX_REG_CQE_RESERVED0_SHIFT 7 __le16 pkt_len /* Total packet length (from the parser) */; - struct parsing_and_err_flags pars_flags - /* Parsing and error flags from the parser */; +/* Parsing and error flags from the parser */ + struct parsing_and_err_flags pars_flags; __le16 vlan_tag /* 802.1q VLAN tag */; __le32 rss_hash /* RSS hash result */; __le16 len_on_first_bd /* Number of bytes placed on first BD */; u8 placement_offset /* Offset of placement from BD start */; - struct tunnel_parsing_flags tunnel_pars_flags /* Tunnel Parsing Flags */ - ; +/* Tunnel Parsing Flags */ + struct eth_tunnel_parsing_flags tunnel_pars_flags; u8 bd_num /* Number of BDs, used for packet */; - u8 reserved[7]; + u8 reserved[9]; struct eth_fast_path_cqe_fw_debug fw_debug /* FW reserved. */; u8 reserved1[3]; - u8 flags; -#define ETH_FAST_PATH_RX_REG_CQE_VALID_MASK 0x1 -#define ETH_FAST_PATH_RX_REG_CQE_VALID_SHIFT 0 -#define ETH_FAST_PATH_RX_REG_CQE_VALID_TOGGLE_MASK 0x1 -#define ETH_FAST_PATH_RX_REG_CQE_VALID_TOGGLE_SHIFT 1 -#define ETH_FAST_PATH_RX_REG_CQE_RESERVED2_MASK 0x3F -#define ETH_FAST_PATH_RX_REG_CQE_RESERVED2_SHIFT 2 + struct eth_pmd_flow_flags pmd_flags /* CQE valid and toggle bits */; }; + /* * TPA-continue ETH Rx FP CQE. */ struct eth_fast_path_rx_tpa_cont_cqe { u8 type /* CQE type */; u8 tpa_agg_index /* TPA aggregation index */; - __le16 len_list[ETH_TPA_CQE_CONT_LEN_LIST_SIZE] - /* List of the segment sizes */; - u8 reserved[5]; +/* List of the segment sizes */ + __le16 len_list[ETH_TPA_CQE_CONT_LEN_LIST_SIZE]; + u8 reserved; u8 reserved1 /* FW reserved. */; __le16 reserved2[ETH_TPA_CQE_CONT_LEN_LIST_SIZE] /* FW reserved. */; + u8 reserved3[3]; + struct eth_pmd_flow_flags pmd_flags /* CQE valid and toggle bits */; }; + /* * TPA-end ETH Rx FP CQE . */ @@ -251,64 +378,72 @@ struct eth_fast_path_rx_tpa_end_cqe { u8 tpa_agg_index /* TPA aggregation index */; __le16 total_packet_len /* Total aggregated packet length */; u8 num_of_bds /* Total number of BDs comprising the packet */; - u8 end_reason /* Aggregation end reason. Use enum eth_tpa_end_reason */ - ; +/* Aggregation end reason. Use enum eth_tpa_end_reason */ + u8 end_reason; __le16 num_of_coalesced_segs /* Number of coalesced TCP segments */; __le32 ts_delta /* TCP timestamp delta */; - __le16 len_list[ETH_TPA_CQE_END_LEN_LIST_SIZE] - /* List of the segment sizes */; - u8 reserved1[3]; - u8 reserved2 /* FW reserved. */; +/* List of the segment sizes */ + __le16 len_list[ETH_TPA_CQE_END_LEN_LIST_SIZE]; __le16 reserved3[ETH_TPA_CQE_END_LEN_LIST_SIZE] /* FW reserved. */; + __le16 reserved1; + u8 reserved2 /* FW reserved. */; + struct eth_pmd_flow_flags pmd_flags /* CQE valid and toggle bits */; }; + /* * TPA-start ETH Rx FP CQE. */ struct eth_fast_path_rx_tpa_start_cqe { u8 type /* CQE type */; u8 bitfields; +/* Type of calculated RSS hash (use enum rss_hash_type) */ #define ETH_FAST_PATH_RX_TPA_START_CQE_RSS_HASH_TYPE_MASK 0x7 #define ETH_FAST_PATH_RX_TPA_START_CQE_RSS_HASH_TYPE_SHIFT 0 +/* Traffic Class */ #define ETH_FAST_PATH_RX_TPA_START_CQE_TC_MASK 0xF #define ETH_FAST_PATH_RX_TPA_START_CQE_TC_SHIFT 3 #define ETH_FAST_PATH_RX_TPA_START_CQE_RESERVED0_MASK 0x1 #define ETH_FAST_PATH_RX_TPA_START_CQE_RESERVED0_SHIFT 7 __le16 seg_len /* Segment length (packetLen from the parser) */; - struct parsing_and_err_flags pars_flags - /* Parsing and error flags from the parser */; +/* Parsing and error flags from the parser */ + struct parsing_and_err_flags pars_flags; __le16 vlan_tag /* 802.1q VLAN tag */; __le32 rss_hash /* RSS hash result */; __le16 len_on_first_bd /* Number of bytes placed on first BD */; u8 placement_offset /* Offset of placement from BD start */; - struct tunnel_parsing_flags tunnel_pars_flags /* Tunnel Parsing Flags */ - ; +/* Tunnel Parsing Flags */ + struct eth_tunnel_parsing_flags tunnel_pars_flags; u8 tpa_agg_index /* TPA aggregation index */; u8 header_len /* Packet L2+L3+L4 header length */; - __le16 ext_bd_len_list[ETH_TPA_CQE_START_LEN_LIST_SIZE] - /* Additional BDs length list. */; +/* Additional BDs length list. */ + __le16 ext_bd_len_list[ETH_TPA_CQE_START_LEN_LIST_SIZE]; struct eth_fast_path_cqe_fw_debug fw_debug /* FW reserved. */; + u8 reserved; + struct eth_pmd_flow_flags pmd_flags /* CQE valid and toggle bits */; }; + /* * The L4 pseudo checksum mode for Ethernet */ enum eth_l4_pseudo_checksum_mode { - ETH_L4_PSEUDO_CSUM_CORRECT_LENGTH - /* Pseudo Header checksum on packet is calculated - * with the correct packet length field. - */ - , - ETH_L4_PSEUDO_CSUM_ZERO_LENGTH - /* Pseudo Hdr checksum on packet is calc with zero len field. */ - , +/* Pseudo Header checksum on packet is calculated with the correct packet length + * field. + */ + ETH_L4_PSEUDO_CSUM_CORRECT_LENGTH, +/* Pseudo Header checksum on packet is calculated with zero length field. */ + ETH_L4_PSEUDO_CSUM_ZERO_LENGTH, MAX_ETH_L4_PSEUDO_CHECKSUM_MODE }; + + struct eth_rx_bd { struct regpair addr /* single continues buffer */; }; + /* * regular ETH Rx SP CQE */ @@ -319,29 +454,25 @@ struct eth_slow_path_rx_cqe { u8 reserved[25]; __le16 echo; u8 reserved1; - u8 flags; -#define ETH_SLOW_PATH_RX_CQE_VALID_MASK 0x1 -#define ETH_SLOW_PATH_RX_CQE_VALID_SHIFT 0 -#define ETH_SLOW_PATH_RX_CQE_VALID_TOGGLE_MASK 0x1 -#define ETH_SLOW_PATH_RX_CQE_VALID_TOGGLE_SHIFT 1 -#define ETH_SLOW_PATH_RX_CQE_RESERVED2_MASK 0x3F -#define ETH_SLOW_PATH_RX_CQE_RESERVED2_SHIFT 2 + struct eth_pmd_flow_flags pmd_flags /* CQE valid and toggle bits */; }; /* * union for all ETH Rx CQE types */ union eth_rx_cqe { - struct eth_fast_path_rx_reg_cqe fast_path_regular /* Regular FP CQE */; - struct eth_fast_path_rx_tpa_start_cqe fast_path_tpa_start - /* TPA-start CQE */; - struct eth_fast_path_rx_tpa_cont_cqe fast_path_tpa_cont - /* TPA-continue CQE */; - struct eth_fast_path_rx_tpa_end_cqe fast_path_tpa_end /* TPA-end CQE */ - ; +/* Regular FP CQE */ + struct eth_fast_path_rx_reg_cqe fast_path_regular; +/* TPA-start CQE */ + struct eth_fast_path_rx_tpa_start_cqe fast_path_tpa_start; +/* TPA-continue CQE */ + struct eth_fast_path_rx_tpa_cont_cqe fast_path_tpa_cont; +/* TPA-end CQE */ + struct eth_fast_path_rx_tpa_end_cqe fast_path_tpa_end; struct eth_slow_path_rx_cqe slow_path /* SP CQE */; }; + /* * ETH Rx CQE type */ @@ -355,51 +486,59 @@ enum eth_rx_cqe_type { MAX_ETH_RX_CQE_TYPE }; + /* - * Wrapp for PD RX CQE used in order to cover full cache line when writing CQE + * Wrapper for PD RX CQE - used in order to cover full cache line when writing + * CQE */ struct eth_rx_pmd_cqe { union eth_rx_cqe cqe /* CQE data itself */; u8 reserved[ETH_RX_CQE_GAP]; }; + /* - * ETH Rx producers data + * Eth RX Tunnel Type */ -struct eth_rx_prod_data { - __le16 bd_prod /* BD producer */; - __le16 cqe_prod /* CQE producer */; - __le16 reserved; - __le16 reserved1 /* FW reserved. */; +enum eth_rx_tunn_type { + ETH_RX_NO_TUNN /* No Tunnel. */, + ETH_RX_TUNN_GENEVE /* GENEVE Tunnel. */, + ETH_RX_TUNN_GRE /* GRE Tunnel. */, + ETH_RX_TUNN_VXLAN /* VXLAN Tunnel. */, + MAX_ETH_RX_TUNN_TYPE }; + + /* * Aggregation end reason. */ enum eth_tpa_end_reason { ETH_AGG_END_UNUSED, ETH_AGG_END_SP_UPDATE /* SP configuration update */, - ETH_AGG_END_MAX_LEN - /* Maximum aggregation length or maximum buffer number used. */, - ETH_AGG_END_LAST_SEG - /* TCP PSH flag or TCP payload length below continue threshold. */, +/* Maximum aggregation length or maximum buffer number used. */ + ETH_AGG_END_MAX_LEN, +/* TCP PSH flag or TCP payload length below continue threshold. */ + ETH_AGG_END_LAST_SEG, ETH_AGG_END_TIMEOUT /* Timeout expiration. */, +/* Packet header not consistency: different IPv4 TOS, TTL or flags, IPv6 TC, + * Hop limit or Flow label, TCP header length or TS options. In GRO different + * TS value, SMAC, DMAC, ackNum, windowSize or VLAN + */ ETH_AGG_END_NOT_CONSISTENT, +/* Out of order or retransmission packet: sequence, ack or timestamp not + * consistent with previous segment. + */ ETH_AGG_END_OUT_OF_ORDER, +/* Next segment cant be aggregated due to LLC/SNAP, IP error, IP fragment, IPv4 + * options, IPv6 extension, IP ECN = CE, TCP errors, TCP options, zero TCP + * payload length , TCP flags or not supported tunnel header options. + */ ETH_AGG_END_NON_TPA_SEG, MAX_ETH_TPA_END_REASON }; -/* - * Eth Tunnel Type - */ -enum eth_tunn_type { - ETH_TUNN_GENEVE /* GENEVE Tunnel. */, - ETH_TUNN_TTAG /* T-Tag Tunnel. */, - ETH_TUNN_GRE /* GRE Tunnel. */, - ETH_TUNN_VXLAN /* VXLAN Tunnel. */, - MAX_ETH_TUNN_TYPE -}; + /* * The first tx bd of a given packet @@ -410,6 +549,8 @@ struct eth_tx_1st_bd { struct eth_tx_data_1st_bd data /* Parsing information data. */; }; + + /* * The second tx bd of a given packet */ @@ -419,21 +560,29 @@ struct eth_tx_2nd_bd { struct eth_tx_data_2nd_bd data /* Parsing information data. */; }; + /* * The parsing information data for the third tx bd of a given packet. */ struct eth_tx_data_3rd_bd { __le16 lso_mss /* For LSO packet - the MSS in bytes. */; __le16 bitfields; +/* For LSO with inner/outer IPv6+ext - TCP header length (in 4-byte WORDs) */ #define ETH_TX_DATA_3RD_BD_TCP_HDR_LEN_DW_MASK 0xF #define ETH_TX_DATA_3RD_BD_TCP_HDR_LEN_DW_SHIFT 0 +/* LSO - number of BDs which contain headers. value should be in range + * (1..ETH_TX_MAX_LSO_HDR_NBD). + */ #define ETH_TX_DATA_3RD_BD_HDR_NBD_MASK 0xF #define ETH_TX_DATA_3RD_BD_HDR_NBD_SHIFT 4 +/* Should be 0 in all the BDs, except the first one. (for debug) */ #define ETH_TX_DATA_3RD_BD_START_BD_MASK 0x1 #define ETH_TX_DATA_3RD_BD_START_BD_SHIFT 8 #define ETH_TX_DATA_3RD_BD_RESERVED0_MASK 0x7F #define ETH_TX_DATA_3RD_BD_RESERVED0_SHIFT 9 +/* For tunnel with IPv6+ext - Pointer to tunnel L4 Header (in 2-byte WORDs) */ u8 tunn_l4_hdr_start_offset_w; +/* For tunnel with IPv6+ext - Total size of Tunnel Header (in 2-byte WORDs) */ u8 tunn_hdr_size_w; }; @@ -446,6 +595,7 @@ struct eth_tx_3rd_bd { struct eth_tx_data_3rd_bd data /* Parsing information data. */; }; + /* * Complementary information for the regular tx bd of a given packet. */ @@ -454,6 +604,7 @@ struct eth_tx_data_bd { __le16 bitfields; #define ETH_TX_DATA_BD_RESERVED1_MASK 0xFF #define ETH_TX_DATA_BD_RESERVED1_SHIFT 0 +/* Should be 0 in all the BDs, except the first one. (for debug) */ #define ETH_TX_DATA_BD_START_BD_MASK 0x1 #define ETH_TX_DATA_BD_START_BD_SHIFT 8 #define ETH_TX_DATA_BD_RESERVED2_MASK 0x7F @@ -470,55 +621,61 @@ struct eth_tx_bd { struct eth_tx_data_bd data /* Complementary information. */; }; + union eth_tx_bd_types { struct eth_tx_1st_bd first_bd /* The first tx bd of a given packet */; - struct eth_tx_2nd_bd second_bd /* The second tx bd of a given packet */ - ; +/* The second tx bd of a given packet */ + struct eth_tx_2nd_bd second_bd; struct eth_tx_3rd_bd third_bd /* The third tx bd of a given packet */; struct eth_tx_bd reg_bd /* The common non-special bd */; }; -/* - * Mstorm Queue Zone - */ -struct mstorm_eth_queue_zone { - struct eth_rx_prod_data rx_producers; - __le32 reserved[2]; -}; + + + + /* - * Ustorm Queue Zone + * Eth Tx Tunnel Type */ -struct ustorm_eth_queue_zone { - struct coalescing_timeset int_coalescing_timeset - /* Rx interrupt coalescing TimeSet */; - __le16 reserved[3]; +enum eth_tx_tunn_type { + ETH_TX_TUNN_GENEVE /* GENEVE Tunnel. */, + ETH_TX_TUNN_TTAG /* T-Tag Tunnel. */, + ETH_TX_TUNN_GRE /* GRE Tunnel. */, + ETH_TX_TUNN_VXLAN /* VXLAN Tunnel. */, + MAX_ETH_TX_TUNN_TYPE }; + /* * Ystorm Queue Zone */ -struct ystorm_eth_queue_zone { - struct coalescing_timeset int_coalescing_timeset - /* Tx interrupt coalescing TimeSet */; - __le16 reserved[3]; +struct xstorm_eth_queue_zone { +/* Tx interrupt coalescing TimeSet */ + struct coalescing_timeset int_coalescing_timeset; + u8 reserved[7]; }; + /* * ETH doorbell data */ struct eth_db_data { u8 params; +/* destination of doorbell (use enum db_dest) */ #define ETH_DB_DATA_DEST_MASK 0x3 #define ETH_DB_DATA_DEST_SHIFT 0 +/* aggregative command to CM (use enum db_agg_cmd_sel) */ #define ETH_DB_DATA_AGG_CMD_MASK 0x3 #define ETH_DB_DATA_AGG_CMD_SHIFT 2 -#define ETH_DB_DATA_BYPASS_EN_MASK 0x1 +#define ETH_DB_DATA_BYPASS_EN_MASK 0x1 /* enable QM bypass */ #define ETH_DB_DATA_BYPASS_EN_SHIFT 4 #define ETH_DB_DATA_RESERVED_MASK 0x1 #define ETH_DB_DATA_RESERVED_SHIFT 5 +/* aggregative value selection */ #define ETH_DB_DATA_AGG_VAL_SEL_MASK 0x3 #define ETH_DB_DATA_AGG_VAL_SEL_SHIFT 6 +/* bit for every DQ counter flags in CM context that DQ can increment */ u8 agg_flags; __le16 bd_prod; }; diff --git a/dpdk/drivers/net/qede/base/mcp_public.h b/dpdk/drivers/net/qede/base/mcp_public.h index 71922654..81ca6634 100644 --- a/dpdk/drivers/net/qede/base/mcp_public.h +++ b/dpdk/drivers/net/qede/base/mcp_public.h @@ -26,59 +26,86 @@ #define MCP_GLOB_PORT_MAX 4 /* Global */ #define MCP_GLOB_FUNC_MAX 16 /* Global */ -typedef u32 offsize_t; /* In DWORDS !!! */ +typedef u32 offsize_t; /* In DWORDS !!! */ /* Offset from the beginning of the MCP scratchpad */ -#define OFFSIZE_OFFSET_SHIFT 0 +#define OFFSIZE_OFFSET_OFFSET 0 #define OFFSIZE_OFFSET_MASK 0x0000ffff /* Size of specific element (not the whole array if any) */ -#define OFFSIZE_SIZE_SHIFT 16 +#define OFFSIZE_SIZE_OFFSET 16 #define OFFSIZE_SIZE_MASK 0xffff0000 /* SECTION_OFFSET is calculating the offset in bytes out of offsize */ -#define SECTION_OFFSET(_offsize) \ -((((_offsize & OFFSIZE_OFFSET_MASK) >> OFFSIZE_OFFSET_SHIFT) << 2)) +#define SECTION_OFFSET(_offsize) \ + ((((_offsize & OFFSIZE_OFFSET_MASK) >> OFFSIZE_OFFSET_OFFSET) << 2)) /* SECTION_SIZE is calculating the size in bytes out of offsize */ -#define SECTION_SIZE(_offsize) \ -(((_offsize & OFFSIZE_SIZE_MASK) >> OFFSIZE_SIZE_SHIFT) << 2) +#define SECTION_SIZE(_offsize) \ + (((_offsize & OFFSIZE_SIZE_MASK) >> OFFSIZE_SIZE_OFFSET) << 2) -#define SECTION_ADDR(_offsize, idx) \ -(MCP_REG_SCRATCH + SECTION_OFFSET(_offsize) + (SECTION_SIZE(_offsize) * idx)) +/* SECTION_ADDR returns the GRC addr of a section, given offsize and index + * within section + */ +#define SECTION_ADDR(_offsize, idx) \ + (MCP_REG_SCRATCH + \ + SECTION_OFFSET(_offsize) + (SECTION_SIZE(_offsize) * idx)) +/* SECTION_OFFSIZE_ADDR returns the GRC addr to the offsize address. Use + * offsetof, since the OFFSETUP collide with the firmware definition + */ #define SECTION_OFFSIZE_ADDR(_pub_base, _section) \ -(_pub_base + offsetof(struct mcp_public_data, sections[_section])) - + (_pub_base + offsetof(struct mcp_public_data, sections[_section])) /* PHY configuration */ -struct pmm_phy_cfg { - u32 speed; /* 0 = autoneg, 1000/10000/20000/25000/40000/50000/100000 */ -#define PMM_SPEED_AUTONEG 0 -#define PMM_SPEED_SMARTLINQ 0x8 +struct eth_phy_cfg { +/* 0 = autoneg, 1000/10000/20000/25000/40000/50000/100000 */ + u32 speed; +#define ETH_SPEED_AUTONEG 0 +#define ETH_SPEED_SMARTLINQ 0x8 /* deprecated - use link_modes field instead */ - u32 pause; /* bitmask */ -#define PMM_PAUSE_NONE 0x0 -#define PMM_PAUSE_AUTONEG 0x1 -#define PMM_PAUSE_RX 0x2 -#define PMM_PAUSE_TX 0x4 + u32 pause; /* bitmask */ +#define ETH_PAUSE_NONE 0x0 +#define ETH_PAUSE_AUTONEG 0x1 +#define ETH_PAUSE_RX 0x2 +#define ETH_PAUSE_TX 0x4 - u32 adv_speed; /* Default should be the speed_cap_mask */ + u32 adv_speed; /* Default should be the speed_cap_mask */ u32 loopback_mode; -#define PMM_LOOPBACK_NONE 0 -#define PMM_LOOPBACK_INT_PHY 1 -#define PMM_LOOPBACK_EXT_PHY 2 -#define PMM_LOOPBACK_EXT 3 -#define PMM_LOOPBACK_MAC 4 -#define PMM_LOOPBACK_CNIG_AH_ONLY_0123 5 /* Port to itself */ -#define PMM_LOOPBACK_CNIG_AH_ONLY_2301 6 /* Port to Port */ +#define ETH_LOOPBACK_NONE (0) +/* Serdes loopback. In AH, it refers to Near End */ +#define ETH_LOOPBACK_INT_PHY (1) +#define ETH_LOOPBACK_EXT_PHY (2) /* External PHY Loopback */ +/* External Loopback (Require loopback plug) */ +#define ETH_LOOPBACK_EXT (3) +#define ETH_LOOPBACK_MAC (4) /* MAC Loopback - not supported */ +#define ETH_LOOPBACK_CNIG_AH_ONLY_0123 (5) /* Port to itself */ +#define ETH_LOOPBACK_CNIG_AH_ONLY_2301 (6) /* Port to Port */ +#define ETH_LOOPBACK_PCS_AH_ONLY (7) /* PCS loopback (TX to RX) */ +/* Loop RX packet from PCS to TX */ +#define ETH_LOOPBACK_REVERSE_MAC_AH_ONLY (8) +/* Remote Serdes Loopback (RX to TX) */ +#define ETH_LOOPBACK_INT_PHY_FEA_AH_ONLY (9) - /* features */ - u32 feature_config_flags; + u32 eee_cfg; +/* EEE is enabled (configuration). Refer to eee_status->active for negotiated + * status + */ +#define EEE_CFG_EEE_ENABLED (1 << 0) +#define EEE_CFG_TX_LPI (1 << 1) +#define EEE_CFG_ADV_SPEED_1G (1 << 2) +#define EEE_CFG_ADV_SPEED_10G (1 << 3) +#define EEE_TX_TIMER_USEC_MASK (0xfffffff0) +#define EEE_TX_TIMER_USEC_OFFSET 4 +#define EEE_TX_TIMER_USEC_BALANCED_TIME (0xa00) +#define EEE_TX_TIMER_USEC_AGGRESSIVE_TIME (0x100) +#define EEE_TX_TIMER_USEC_LATENCY_TIME (0x6000) + u32 link_modes; /* Additional link modes */ +#define LINK_MODE_SMARTLINQ_ENABLE 0x1 /* XXX Deprecate */ }; struct port_mf_cfg { - u32 dynamic_cfg; /* device control channel */ + u32 dynamic_cfg; /* device control channel */ #define PORT_MF_CFG_OV_TAG_MASK 0x0000ffff -#define PORT_MF_CFG_OV_TAG_SHIFT 0 +#define PORT_MF_CFG_OV_TAG_OFFSET 0 #define PORT_MF_CFG_OV_TAG_DEFAULT PORT_MF_CFG_OV_TAG_MASK u32 reserved[1]; @@ -87,52 +114,102 @@ struct port_mf_cfg { /* DO NOT add new fields in the middle * MUST be synced with struct pmm_stats_map */ -struct pmm_stats { - u64 r64; /* 0x00 (Offset 0x00 ) RX 64-byte frame counter */ - u64 r127; /* 0x01 (Offset 0x08 ) RX 65 to 127 byte frame counter */ - u64 r255; /* 0x02 (Offset 0x10 ) RX 128 to 255 byte frame counter */ - u64 r511; /* 0x03 (Offset 0x18 ) RX 256 to 511 byte frame counter */ - u64 r1023; /* 0x04 (Offset 0x20 ) RX 512 to 1023 byte frame counter */ - u64 r1518; /* 0x05 (Offset 0x28 ) RX 1024 to 1518 byte frame counter */ - u64 r1522; /* 0x06 (Offset 0x30 ) RX 1519 to 1522 byte VLAN-tagged */ - u64 r2047; /* 0x07 (Offset 0x38 ) RX 1519 to 2047 byte frame counter */ - u64 r4095; /* 0x08 (Offset 0x40 ) RX 2048 to 4095 byte frame counter */ - u64 r9216; /* 0x09 (Offset 0x48 ) RX 4096 to 9216 byte frame counter */ - u64 r16383; /* 0x0A (Offset 0x50 ) RX 9217 to 16383 byte frame ctr */ - u64 rfcs; /* 0x0F (Offset 0x58 ) RX FCS error frame counter */ - u64 rxcf; /* 0x10 (Offset 0x60 ) RX control frame counter */ - u64 rxpf; /* 0x11 (Offset 0x68 ) RX pause frame counter */ - u64 rxpp; /* 0x12 (Offset 0x70 ) RX PFC frame counter */ - u64 raln; /* 0x16 (Offset 0x78 ) RX alignment error counter */ - u64 rfcr; /* 0x19 (Offset 0x80 ) RX false carrier counter */ - u64 rovr; /* 0x1A (Offset 0x88 ) RX oversized frame counter */ - u64 rjbr; /* 0x1B (Offset 0x90 ) RX jabber frame counter */ - u64 rund; /* 0x34 (Offset 0x98 ) RX undersized frame counter */ - u64 rfrg; /* 0x35 (Offset 0xa0 ) RX fragment counter */ - u64 t64; /* 0x40 (Offset 0xa8 ) TX 64-byte frame counter */ +struct eth_stats { + u64 r64; /* 0x00 (Offset 0x00 ) RX 64-byte frame counter*/ + u64 r127; /* 0x01 (Offset 0x08 ) RX 65 to 127 byte frame counter*/ + u64 r255; /* 0x02 (Offset 0x10 ) RX 128 to 255 byte frame counter*/ + u64 r511; /* 0x03 (Offset 0x18 ) RX 256 to 511 byte frame counter*/ + u64 r1023; /* 0x04 (Offset 0x20 ) RX 512 to 1023 byte frame counter*/ +/* 0x05 (Offset 0x28 ) RX 1024 to 1518 byte frame counter */ + u64 r1518; + union { + struct { /* bb */ +/* 0x06 (Offset 0x30 ) RX 1519 to 1522 byte VLAN-tagged frame counter */ + u64 r1522; +/* 0x07 (Offset 0x38 ) RX 1519 to 2047 byte frame counter*/ + u64 r2047; +/* 0x08 (Offset 0x40 ) RX 2048 to 4095 byte frame counter*/ + u64 r4095; +/* 0x09 (Offset 0x48 ) RX 4096 to 9216 byte frame counter*/ + u64 r9216; +/* 0x0A (Offset 0x50 ) RX 9217 to 16383 byte frame counter */ + u64 r16383; + } bb0; + struct { /* ah */ + u64 unused1; +/* 0x07 (Offset 0x38 ) RX 1519 to max byte frame counter*/ + u64 r1519_to_max; + u64 unused2; + u64 unused3; + u64 unused4; + } ah0; + } u0; + u64 rfcs; /* 0x0F (Offset 0x58 ) RX FCS error frame counter*/ + u64 rxcf; /* 0x10 (Offset 0x60 ) RX control frame counter*/ + u64 rxpf; /* 0x11 (Offset 0x68 ) RX pause frame counter*/ + u64 rxpp; /* 0x12 (Offset 0x70 ) RX PFC frame counter*/ + u64 raln; /* 0x16 (Offset 0x78 ) RX alignment error counter*/ + u64 rfcr; /* 0x19 (Offset 0x80 ) RX false carrier counter */ + u64 rovr; /* 0x1A (Offset 0x88 ) RX oversized frame counter*/ + u64 rjbr; /* 0x1B (Offset 0x90 ) RX jabber frame counter */ + u64 rund; /* 0x34 (Offset 0x98 ) RX undersized frame counter */ + u64 rfrg; /* 0x35 (Offset 0xa0 ) RX fragment counter */ + u64 t64; /* 0x40 (Offset 0xa8 ) TX 64-byte frame counter */ u64 t127; /* 0x41 (Offset 0xb0 ) TX 65 to 127 byte frame counter */ - u64 t255; /* 0x42 (Offset 0xb8 ) TX 128 to 255 byte frame counter */ - u64 t511; /* 0x43 (Offset 0xc0 ) TX 256 to 511 byte frame counter */ - u64 t1023; /* 0x44 (Offset 0xc8 ) TX 512 to 1023 byte frame counter */ - u64 t1518; /* 0x45 (Offset 0xd0 ) TX 1024 to 1518 byte frame counter */ - u64 t2047; /* 0x47 (Offset 0xd8 ) TX 1519 to 2047 byte frame counter */ - u64 t4095; /* 0x48 (Offset 0xe0 ) TX 2048 to 4095 byte frame counter */ - u64 t9216; /* 0x49 (Offset 0xe8 ) TX 4096 to 9216 byte frame counter */ - u64 t16383; /* 0x4A (Offset 0xf0 ) TX 9217 to 16383 byte frame ctr */ - u64 txpf; /* 0x50 (Offset 0xf8 ) TX pause frame counter */ - u64 txpp; /* 0x51 (Offset 0x100) TX PFC frame counter */ - u64 tlpiec; /* 0x6C (Offset 0x108) Transmit Logical Type LLFC */ - u64 tncl; /* 0x6E (Offset 0x110) Transmit Total Collision Counter */ - u64 rbyte; /* 0x3d (Offset 0x118) RX byte counter */ - u64 rxuca; /* 0x0c (Offset 0x120) RX UC frame counter */ - u64 rxmca; /* 0x0d (Offset 0x128) RX MC frame counter */ - u64 rxbca; /* 0x0e (Offset 0x130) RX BC frame counter */ - u64 rxpok; /* 0x22 (Offset 0x138) RX good frame */ - u64 tbyte; /* 0x6f (Offset 0x140) TX byte counter */ - u64 txuca; /* 0x4d (Offset 0x148) TX UC frame counter */ - u64 txmca; /* 0x4e (Offset 0x150) TX MC frame counter */ - u64 txbca; /* 0x4f (Offset 0x158) TX BC frame counter */ - u64 txcf; /* 0x54 (Offset 0x160) TX control frame counter */ + u64 t255; /* 0x42 (Offset 0xb8 ) TX 128 to 255 byte frame counter*/ + u64 t511; /* 0x43 (Offset 0xc0 ) TX 256 to 511 byte frame counter*/ + u64 t1023; /* 0x44 (Offset 0xc8 ) TX 512 to 1023 byte frame counter*/ +/* 0x45 (Offset 0xd0 ) TX 1024 to 1518 byte frame counter */ + u64 t1518; + union { + struct { /* bb */ +/* 0x47 (Offset 0xd8 ) TX 1519 to 2047 byte frame counter */ + u64 t2047; +/* 0x48 (Offset 0xe0 ) TX 2048 to 4095 byte frame counter */ + u64 t4095; +/* 0x49 (Offset 0xe8 ) TX 4096 to 9216 byte frame counter */ + u64 t9216; +/* 0x4A (Offset 0xf0 ) TX 9217 to 16383 byte frame counter */ + u64 t16383; + } bb1; + struct { /* ah */ +/* 0x47 (Offset 0xd8 ) TX 1519 to max byte frame counter */ + u64 t1519_to_max; + u64 unused6; + u64 unused7; + u64 unused8; + } ah1; + } u1; + u64 txpf; /* 0x50 (Offset 0xf8 ) TX pause frame counter */ + u64 txpp; /* 0x51 (Offset 0x100) TX PFC frame counter */ +/* 0x6C (Offset 0x108) Transmit Logical Type LLFC message counter */ + union { + struct { /* bb */ +/* 0x6C (Offset 0x108) Transmit Logical Type LLFC message counter */ + u64 tlpiec; +/* 0x6E (Offset 0x110) Transmit Total Collision Counter */ + u64 tncl; + } bb2; + struct { /* ah */ + u64 unused9; + u64 unused10; + } ah2; + } u2; + u64 rbyte; /* 0x3d (Offset 0x118) RX byte counter */ + u64 rxuca; /* 0x0c (Offset 0x120) RX UC frame counter */ + u64 rxmca; /* 0x0d (Offset 0x128) RX MC frame counter */ + u64 rxbca; /* 0x0e (Offset 0x130) RX BC frame counter */ +/* 0x22 (Offset 0x138) RX good frame (good CRC, not oversized, no ERROR) */ + u64 rxpok; + u64 tbyte; /* 0x6f (Offset 0x140) TX byte counter */ + u64 txuca; /* 0x4d (Offset 0x148) TX UC frame counter */ + u64 txmca; /* 0x4e (Offset 0x150) TX MC frame counter */ + u64 txbca; /* 0x4f (Offset 0x158) TX BC frame counter */ + u64 txcf; /* 0x54 (Offset 0x160) TX control frame counter */ +/* HSI - Cannot add more stats to this struct. If needed, then need to open new + * struct + */ + }; struct brb_stats { @@ -142,28 +219,29 @@ struct brb_stats { struct port_stats { struct brb_stats brb; - struct pmm_stats pmm; + struct eth_stats eth; }; -/*-----+----------------------------------------------------------------------- - * Chip | Number and | Ports in| Ports in|2 PHY-s |# of ports|# of engines - * | rate of physical | team #1 | team #2 |are used|per path | (paths) - * | ports | | | | | - *======+==================+=========+=========+========+====================== - * BB | 1x100G | This is special mode, where there are 2 HW func - * BB | 2x10/20Gbps | 0,1 | NA | No | 1 | 1 - * BB | 2x40 Gbps | 0,1 | NA | Yes | 1 | 1 - * BB | 2x50Gbps | 0,1 | NA | No | 1 | 1 - * BB | 4x10Gbps | 0,2 | 1,3 | No | 1/2 | 1,2 - * BB | 4x10Gbps | 0,1 | 2,3 | No | 1/2 | 1,2 - * BB | 4x10Gbps | 0,3 | 1,2 | No | 1/2 | 1,2 - * BB | 4x10Gbps | 0,1,2,3 | NA | No | 1 | 1 - * AH | 2x10/20Gbps | 0,1 | NA | NA | 1 | NA - * AH | 4x10Gbps | 0,1 | 2,3 | NA | 2 | NA - * AH | 4x10Gbps | 0,2 | 1,3 | NA | 2 | NA - * AH | 4x10Gbps | 0,3 | 1,2 | NA | 2 | NA - * AH | 4x10Gbps | 0,1,2,3 | NA | NA | 1 | NA - *======+==================+=========+=========+========+======================= +/*----+------------------------------------------------------------------------ + * C | Number and | Ports in| Ports in|2 PHY-s |# of ports|# of engines + * h | rate of | team #1 | team #2 |are used|per path | (paths) + * i | physical | | | | | enabled + * p | ports | | | | | + *====+============+=========+=========+========+==========+=================== + * BB | 1x100G | This is special mode, where there are actually 2 HW func + * BB | 2x10/20Gbps| 0,1 | NA | No | 1 | 1 + * BB | 2x40 Gbps | 0,1 | NA | Yes | 1 | 1 + * BB | 2x50Gbps | 0,1 | NA | No | 1 | 1 + * BB | 4x10Gbps | 0,2 | 1,3 | No | 1/2 | 1,2 (2 is optional) + * BB | 4x10Gbps | 0,1 | 2,3 | No | 1/2 | 1,2 (2 is optional) + * BB | 4x10Gbps | 0,3 | 1,2 | No | 1/2 | 1,2 (2 is optional) + * BB | 4x10Gbps | 0,1,2,3 | NA | No | 1 | 1 + * AH | 2x10/20Gbps| 0,1 | NA | NA | 1 | NA + * AH | 4x10Gbps | 0,1 | 2,3 | NA | 2 | NA + * AH | 4x10Gbps | 0,2 | 1,3 | NA | 2 | NA + * AH | 4x10Gbps | 0,3 | 1,2 | NA | 2 | NA + * AH | 4x10Gbps | 0,1,2,3 | NA | NA | 1 | NA + *====+============+=========+=========+========+==========+=================== */ #define CMT_TEAM0 0 @@ -186,11 +264,11 @@ struct couple_mode_teaming { /************************************** * LLDP and DCBX HSI structures **************************************/ -#define LLDP_CHASSIS_ID_STAT_LEN 4 -#define LLDP_PORT_ID_STAT_LEN 4 +#define LLDP_CHASSIS_ID_STAT_LEN 4 +#define LLDP_PORT_ID_STAT_LEN 4 #define DCBX_MAX_APP_PROTOCOL 32 -#define MAX_SYSTEM_LLDP_TLV_DATA 32 - +#define MAX_SYSTEM_LLDP_TLV_DATA 32 /* In dwords. 128 in bytes*/ +#define MAX_TLV_BUFFER 128 /* In dwords. 512 in bytes*/ typedef enum _lldp_agent_e { LLDP_NEAREST_BRIDGE = 0, LLDP_NEAREST_NON_TPMR_BRIDGE, @@ -201,34 +279,31 @@ typedef enum _lldp_agent_e { struct lldp_config_params_s { u32 config; #define LLDP_CONFIG_TX_INTERVAL_MASK 0x000000ff -#define LLDP_CONFIG_TX_INTERVAL_SHIFT 0 +#define LLDP_CONFIG_TX_INTERVAL_OFFSET 0 #define LLDP_CONFIG_HOLD_MASK 0x00000f00 -#define LLDP_CONFIG_HOLD_SHIFT 8 +#define LLDP_CONFIG_HOLD_OFFSET 8 #define LLDP_CONFIG_MAX_CREDIT_MASK 0x0000f000 -#define LLDP_CONFIG_MAX_CREDIT_SHIFT 12 +#define LLDP_CONFIG_MAX_CREDIT_OFFSET 12 #define LLDP_CONFIG_ENABLE_RX_MASK 0x40000000 -#define LLDP_CONFIG_ENABLE_RX_SHIFT 30 +#define LLDP_CONFIG_ENABLE_RX_OFFSET 30 #define LLDP_CONFIG_ENABLE_TX_MASK 0x80000000 -#define LLDP_CONFIG_ENABLE_TX_SHIFT 31 +#define LLDP_CONFIG_ENABLE_TX_OFFSET 31 /* Holds local Chassis ID TLV header, subtype and 9B of payload. * If firtst byte is 0, then we will use default chassis ID */ u32 local_chassis_id[LLDP_CHASSIS_ID_STAT_LEN]; /* Holds local Port ID TLV header, subtype and 9B of payload. * If firtst byte is 0, then we will use default port ID - */ + */ u32 local_port_id[LLDP_PORT_ID_STAT_LEN]; }; struct lldp_status_params_s { u32 prefix_seq_num; - u32 status; /* TBD */ - /* Holds remote Chassis ID TLV header, subtype and 9B of payload. - */ - u32 local_port_id[LLDP_PORT_ID_STAT_LEN]; + u32 status; /* TBD */ + /* Holds remote Chassis ID TLV header, subtype and 9B of payload. */ u32 peer_chassis_id[LLDP_CHASSIS_ID_STAT_LEN]; - /* Holds remote Port ID TLV header, subtype and 9B of payload. - */ + /* Holds remote Port ID TLV header, subtype and 9B of payload. */ u32 peer_port_id[LLDP_PORT_ID_STAT_LEN]; u32 suffix_seq_num; }; @@ -236,20 +311,40 @@ struct lldp_status_params_s { struct dcbx_ets_feature { u32 flags; #define DCBX_ETS_ENABLED_MASK 0x00000001 -#define DCBX_ETS_ENABLED_SHIFT 0 +#define DCBX_ETS_ENABLED_OFFSET 0 #define DCBX_ETS_WILLING_MASK 0x00000002 -#define DCBX_ETS_WILLING_SHIFT 1 +#define DCBX_ETS_WILLING_OFFSET 1 #define DCBX_ETS_ERROR_MASK 0x00000004 -#define DCBX_ETS_ERROR_SHIFT 2 +#define DCBX_ETS_ERROR_OFFSET 2 #define DCBX_ETS_CBS_MASK 0x00000008 -#define DCBX_ETS_CBS_SHIFT 3 +#define DCBX_ETS_CBS_OFFSET 3 #define DCBX_ETS_MAX_TCS_MASK 0x000000f0 -#define DCBX_ETS_MAX_TCS_SHIFT 4 - u32 pri_tc_tbl[1]; +#define DCBX_ETS_MAX_TCS_OFFSET 4 +#define DCBX_OOO_TC_MASK 0x00000f00 +#define DCBX_OOO_TC_OFFSET 8 +/* Entries in tc table are orginized that the left most is pri 0, right most is + * prio 7 + */ + + u32 pri_tc_tbl[1]; +/* Fixed TCP OOO TC usage is deprecated and used only for driver backward + * compatibility + */ +#define DCBX_TCP_OOO_TC (4) +#define DCBX_TCP_OOO_K2_4PORT_TC (3) + +#define NIG_ETS_ISCSI_OOO_CLIENT_OFFSET (DCBX_TCP_OOO_TC + 1) #define DCBX_CEE_STRICT_PRIORITY 0xf -#define DCBX_CEE_STRICT_PRIORITY_TC 0x7 - u32 tc_bw_tbl[2]; - u32 tc_tsa_tbl[2]; +/* Entries in tc table are orginized that the left most is pri 0, right most is + * prio 7 + */ + + u32 tc_bw_tbl[2]; +/* Entries in tc table are orginized that the left most is pri 0, right most is + * prio 7 + */ + + u32 tc_tsa_tbl[2]; #define DCBX_ETS_TSA_STRICT 0 #define DCBX_ETS_TSA_CBS 1 #define DCBX_ETS_TSA_ETS 2 @@ -258,7 +353,7 @@ struct dcbx_ets_feature { struct dcbx_app_priority_entry { u32 entry; #define DCBX_APP_PRI_MAP_MASK 0x000000ff -#define DCBX_APP_PRI_MAP_SHIFT 0 +#define DCBX_APP_PRI_MAP_OFFSET 0 #define DCBX_APP_PRI_0 0x01 #define DCBX_APP_PRI_1 0x02 #define DCBX_APP_PRI_2 0x04 @@ -268,31 +363,40 @@ struct dcbx_app_priority_entry { #define DCBX_APP_PRI_6 0x40 #define DCBX_APP_PRI_7 0x80 #define DCBX_APP_SF_MASK 0x00000300 -#define DCBX_APP_SF_SHIFT 8 +#define DCBX_APP_SF_OFFSET 8 #define DCBX_APP_SF_ETHTYPE 0 #define DCBX_APP_SF_PORT 1 +#define DCBX_APP_SF_IEEE_MASK 0x0000f000 +#define DCBX_APP_SF_IEEE_OFFSET 12 +#define DCBX_APP_SF_IEEE_RESERVED 0 +#define DCBX_APP_SF_IEEE_ETHTYPE 1 +#define DCBX_APP_SF_IEEE_TCP_PORT 2 +#define DCBX_APP_SF_IEEE_UDP_PORT 3 +#define DCBX_APP_SF_IEEE_TCP_UDP_PORT 4 + #define DCBX_APP_PROTOCOL_ID_MASK 0xffff0000 -#define DCBX_APP_PROTOCOL_ID_SHIFT 16 +#define DCBX_APP_PROTOCOL_ID_OFFSET 16 }; + /* FW structure in BE */ struct dcbx_app_priority_feature { u32 flags; #define DCBX_APP_ENABLED_MASK 0x00000001 -#define DCBX_APP_ENABLED_SHIFT 0 +#define DCBX_APP_ENABLED_OFFSET 0 #define DCBX_APP_WILLING_MASK 0x00000002 -#define DCBX_APP_WILLING_SHIFT 1 +#define DCBX_APP_WILLING_OFFSET 1 #define DCBX_APP_ERROR_MASK 0x00000004 -#define DCBX_APP_ERROR_SHIFT 2 +#define DCBX_APP_ERROR_OFFSET 2 /* Not in use - * #define DCBX_APP_DEFAULT_PRI_MASK 0x00000f00 - * #define DCBX_APP_DEFAULT_PRI_SHIFT 8 - */ + #define DCBX_APP_DEFAULT_PRI_MASK 0x00000f00 + #define DCBX_APP_DEFAULT_PRI_OFFSET 8 + */ #define DCBX_APP_MAX_TCS_MASK 0x0000f000 -#define DCBX_APP_MAX_TCS_SHIFT 12 +#define DCBX_APP_MAX_TCS_OFFSET 12 #define DCBX_APP_NUM_ENTRIES_MASK 0x00ff0000 -#define DCBX_APP_NUM_ENTRIES_SHIFT 16 - struct dcbx_app_priority_entry app_pri_tbl[DCBX_MAX_APP_PROTOCOL]; +#define DCBX_APP_NUM_ENTRIES_OFFSET 16 + struct dcbx_app_priority_entry app_pri_tbl[DCBX_MAX_APP_PROTOCOL]; }; /* FW structure in BE */ @@ -302,7 +406,7 @@ struct dcbx_features { /* PFC feature */ u32 pfc; #define DCBX_PFC_PRI_EN_BITMAP_MASK 0x000000ff -#define DCBX_PFC_PRI_EN_BITMAP_SHIFT 0 +#define DCBX_PFC_PRI_EN_BITMAP_OFFSET 0 #define DCBX_PFC_PRI_EN_BITMAP_PRI_0 0x01 #define DCBX_PFC_PRI_EN_BITMAP_PRI_1 0x02 #define DCBX_PFC_PRI_EN_BITMAP_PRI_2 0x04 @@ -313,17 +417,17 @@ struct dcbx_features { #define DCBX_PFC_PRI_EN_BITMAP_PRI_7 0x80 #define DCBX_PFC_FLAGS_MASK 0x0000ff00 -#define DCBX_PFC_FLAGS_SHIFT 8 +#define DCBX_PFC_FLAGS_OFFSET 8 #define DCBX_PFC_CAPS_MASK 0x00000f00 -#define DCBX_PFC_CAPS_SHIFT 8 +#define DCBX_PFC_CAPS_OFFSET 8 #define DCBX_PFC_MBC_MASK 0x00004000 -#define DCBX_PFC_MBC_SHIFT 14 +#define DCBX_PFC_MBC_OFFSET 14 #define DCBX_PFC_WILLING_MASK 0x00008000 -#define DCBX_PFC_WILLING_SHIFT 15 +#define DCBX_PFC_WILLING_OFFSET 15 #define DCBX_PFC_ENABLED_MASK 0x00010000 -#define DCBX_PFC_ENABLED_SHIFT 16 +#define DCBX_PFC_ENABLED_OFFSET 16 #define DCBX_PFC_ERROR_MASK 0x00020000 -#define DCBX_PFC_ERROR_SHIFT 17 +#define DCBX_PFC_ERROR_OFFSET 17 /* APP feature */ struct dcbx_app_priority_feature app; @@ -331,11 +435,14 @@ struct dcbx_features { struct dcbx_local_params { u32 config; -#define DCBX_CONFIG_VERSION_MASK 0x00000003 -#define DCBX_CONFIG_VERSION_SHIFT 0 +#define DCBX_CONFIG_VERSION_MASK 0x00000007 +#define DCBX_CONFIG_VERSION_OFFSET 0 #define DCBX_CONFIG_VERSION_DISABLED 0 #define DCBX_CONFIG_VERSION_IEEE 1 #define DCBX_CONFIG_VERSION_CEE 2 +#define DCBX_CONFIG_VERSION_DYNAMIC \ + (DCBX_CONFIG_VERSION_IEEE | DCBX_CONFIG_VERSION_CEE) +#define DCBX_CONFIG_VERSION_STATIC 4 u32 flags; struct dcbx_features features; @@ -345,41 +452,94 @@ struct dcbx_mib { u32 prefix_seq_num; u32 flags; /* - * #define DCBX_CONFIG_VERSION_MASK 0x00000003 - * #define DCBX_CONFIG_VERSION_SHIFT 0 - * #define DCBX_CONFIG_VERSION_DISABLED 0 - * #define DCBX_CONFIG_VERSION_IEEE 1 - * #define DCBX_CONFIG_VERSION_CEE 2 - */ + #define DCBX_CONFIG_VERSION_MASK 0x00000007 + #define DCBX_CONFIG_VERSION_OFFSET 0 + #define DCBX_CONFIG_VERSION_DISABLED 0 + #define DCBX_CONFIG_VERSION_IEEE 1 + #define DCBX_CONFIG_VERSION_CEE 2 + #define DCBX_CONFIG_VERSION_STATIC 4 + */ struct dcbx_features features; u32 suffix_seq_num; }; struct lldp_system_tlvs_buffer_s { - u16 valid; - u16 length; + u32 flags; +#define LLDP_SYSTEM_TLV_VALID_MASK 0x1 +#define LLDP_SYSTEM_TLV_VALID_OFFSET 0 +/* This bit defines if system TLVs are instead of mandatory TLVS or in + * addition to them. Set 1 for replacing mandatory TLVs + */ +#define LLDP_SYSTEM_TLV_MANDATORY_MASK 0x2 +#define LLDP_SYSTEM_TLV_MANDATORY_OFFSET 1 +#define LLDP_SYSTEM_TLV_LENGTH_MASK 0xffff0000 +#define LLDP_SYSTEM_TLV_LENGTH_OFFSET 16 u32 data[MAX_SYSTEM_LLDP_TLV_DATA]; }; +/* Since this struct is written by MFW and read by driver need to add + * sequence guards (as in case of DCBX MIB) + */ +struct lldp_received_tlvs_s { + u32 prefix_seq_num; + u32 length; + u32 tlvs_buffer[MAX_TLV_BUFFER]; + u32 suffix_seq_num; +}; + +struct dcb_dscp_map { + u32 flags; +#define DCB_DSCP_ENABLE_MASK 0x1 +#define DCB_DSCP_ENABLE_OFFSET 0 +#define DCB_DSCP_ENABLE 1 + u32 dscp_pri_map[8]; +}; + +/************************************** + * Attributes commands + **************************************/ + +enum _attribute_commands_e { + ATTRIBUTE_CMD_READ = 0, + ATTRIBUTE_CMD_WRITE, + ATTRIBUTE_CMD_READ_CLEAR, + ATTRIBUTE_CMD_CLEAR, + ATTRIBUTE_NUM_OF_COMMANDS +}; + /**************************************/ /* */ /* P U B L I C G L O B A L */ /* */ /**************************************/ struct public_global { - u32 max_path; /* 32bit is wasty, but this will be used often */ - u32 max_ports; /* (Global) 32bit is wasty, this will be used often */ -#define MODE_1P 1 /* TBD - NEED TO THINK OF A BETTER NAME */ + u32 max_path; /* 32bit is wasty, but this will be used often */ +/* (Global) 32bit is wasty, but this will be used often */ + u32 max_ports; +#define MODE_1P 1 /* TBD - NEED TO THINK OF A BETTER NAME */ #define MODE_2P 2 #define MODE_3P 3 #define MODE_4P 4 u32 debug_mb_offset; u32 phymod_dbg_mb_offset; struct couple_mode_teaming cmt; +/* Temperature in Celcius (-255C / +255C), measured every second. */ s32 internal_temperature; u32 mfw_ver; u32 running_bundle_id; s32 external_temperature; + u32 mdump_reason; +#define MDUMP_REASON_INTERNAL_ERROR (1 << 0) +#define MDUMP_REASON_EXTERNAL_TRIGGER (1 << 1) +#define MDUMP_REASON_DUMP_AGED (1 << 2) + u32 ext_phy_upgrade_fw; +#define EXT_PHY_FW_UPGRADE_STATUS_MASK (0x0000ffff) +#define EXT_PHY_FW_UPGRADE_STATUS_OFFSET (0) +#define EXT_PHY_FW_UPGRADE_STATUS_IN_PROGRESS (1) +#define EXT_PHY_FW_UPGRADE_STATUS_FAILED (2) +#define EXT_PHY_FW_UPGRADE_STATUS_SUCCESS (3) +#define EXT_PHY_FW_UPGRADE_TYPE_MASK (0xffff0000) +#define EXT_PHY_FW_UPGRADE_TYPE_OFFSET (16) }; /**************************************/ @@ -406,7 +566,7 @@ struct public_global { struct fw_flr_mb { u32 aggint; u32 opgen_addr; - u32 accum_ack; /* 0..15:PF, 16..207:VF, 256..271:IOV_DIS */ + u32 accum_ack; /* 0..15:PF, 16..207:VF, 256..271:IOV_DIS */ #define ACCUM_ACK_PF_BASE 0 #define ACCUM_ACK_PF_SHIFT 0 @@ -424,14 +584,14 @@ struct public_path { * mcp_vf_disabled is set by the MCP to indicate the driver about VFs * which were disabled/flred */ - u32 mcp_vf_disabled[VF_MAX_STATIC / 32]; /* 0x003c */ + u32 mcp_vf_disabled[VF_MAX_STATIC / 32]; /* 0x003c */ +/* Reset on mcp reset, and incremented for eveny process kill event. */ u32 process_kill; - /* Reset on mcp reset, and incremented for eveny process kill event. */ #define PROCESS_KILL_COUNTER_MASK 0x0000ffff -#define PROCESS_KILL_COUNTER_SHIFT 0 +#define PROCESS_KILL_COUNTER_OFFSET 0 #define PROCESS_KILL_GLOB_AEU_BIT_MASK 0xffff0000 -#define PROCESS_KILL_GLOB_AEU_BIT_SHIFT 16 +#define PROCESS_KILL_GLOB_AEU_BIT_OFFSET 16 #define GLOBAL_AEU_BIT(aeu_reg_id, aeu_bit) (aeu_reg_id * 32 + aeu_bit) }; @@ -464,7 +624,7 @@ struct dci_fc_npiv_tbl { ****************************************************************************/ struct public_port { - u32 validity_map; /* 0x0 (4*2 = 0x8) */ + u32 validity_map; /* 0x0 (4*2 = 0x8) */ /* validity bits */ #define MCP_VALIDITY_PCI_CFG 0x00100000 @@ -473,7 +633,8 @@ struct public_port { #define MCP_VALIDITY_RESERVED 0x00000007 /* One licensing bit should be set */ -#define MCP_VALIDITY_LIC_KEY_IN_EFFECT_MASK 0x00000038 /* yaniv - tbd */ +/* yaniv - tbd ? license */ +#define MCP_VALIDITY_LIC_KEY_IN_EFFECT_MASK 0x00000038 #define MCP_VALIDITY_LIC_MANUF_KEY_IN_EFFECT 0x00000008 #define MCP_VALIDITY_LIC_UPGRADE_KEY_IN_EFFECT 0x00000010 #define MCP_VALIDITY_LIC_NO_KEY_IN_EFFECT 0x00000020 @@ -485,22 +646,19 @@ struct public_port { #define MCP_VALIDITY_ACTIVE_MFW_NONE 0x000001c0 u32 link_status; -#define LINK_STATUS_LINK_UP 0x00000001 -#define LINK_STATUS_SPEED_AND_DUPLEX_MASK 0x0000001e +#define LINK_STATUS_LINK_UP 0x00000001 +#define LINK_STATUS_SPEED_AND_DUPLEX_MASK 0x0000001e #define LINK_STATUS_SPEED_AND_DUPLEX_1000THD (1 << 1) #define LINK_STATUS_SPEED_AND_DUPLEX_1000TFD (2 << 1) -#define LINK_STATUS_SPEED_AND_DUPLEX_10G (3 << 1) -#define LINK_STATUS_SPEED_AND_DUPLEX_20G (4 << 1) -#define LINK_STATUS_SPEED_AND_DUPLEX_40G (5 << 1) -#define LINK_STATUS_SPEED_AND_DUPLEX_50G (6 << 1) -#define LINK_STATUS_SPEED_AND_DUPLEX_100G (7 << 1) -#define LINK_STATUS_SPEED_AND_DUPLEX_25G (8 << 1) - -#define LINK_STATUS_AUTO_NEGOTIATE_ENABLED 0x00000020 - -#define LINK_STATUS_AUTO_NEGOTIATE_COMPLETE 0x00000040 -#define LINK_STATUS_PARALLEL_DETECTION_USED 0x00000080 - +#define LINK_STATUS_SPEED_AND_DUPLEX_10G (3 << 1) +#define LINK_STATUS_SPEED_AND_DUPLEX_20G (4 << 1) +#define LINK_STATUS_SPEED_AND_DUPLEX_40G (5 << 1) +#define LINK_STATUS_SPEED_AND_DUPLEX_50G (6 << 1) +#define LINK_STATUS_SPEED_AND_DUPLEX_100G (7 << 1) +#define LINK_STATUS_SPEED_AND_DUPLEX_25G (8 << 1) +#define LINK_STATUS_AUTO_NEGOTIATE_ENABLED 0x00000020 +#define LINK_STATUS_AUTO_NEGOTIATE_COMPLETE 0x00000040 +#define LINK_STATUS_PARALLEL_DETECTION_USED 0x00000080 #define LINK_STATUS_PFC_ENABLED 0x00000100 #define LINK_STATUS_LINK_PARTNER_1000TFD_CAPABLE 0x00000200 #define LINK_STATUS_LINK_PARTNER_1000THD_CAPABLE 0x00000400 @@ -510,24 +668,28 @@ struct public_port { #define LINK_STATUS_LINK_PARTNER_50G_CAPABLE 0x00004000 #define LINK_STATUS_LINK_PARTNER_100G_CAPABLE 0x00008000 #define LINK_STATUS_LINK_PARTNER_25G_CAPABLE 0x00010000 - #define LINK_STATUS_LINK_PARTNER_FLOW_CONTROL_MASK 0x000C0000 #define LINK_STATUS_LINK_PARTNER_NOT_PAUSE_CAPABLE (0 << 18) #define LINK_STATUS_LINK_PARTNER_SYMMETRIC_PAUSE (1 << 18) #define LINK_STATUS_LINK_PARTNER_ASYMMETRIC_PAUSE (2 << 18) -#define LINK_STATUS_LINK_PARTNER_BOTH_PAUSE (3 << 18) - -#define LINK_STATUS_SFP_TX_FAULT 0x00100000 -#define LINK_STATUS_TX_FLOW_CONTROL_ENABLED 0x00200000 -#define LINK_STATUS_RX_FLOW_CONTROL_ENABLED 0x00400000 -#define LINK_STATUS_RX_SIGNAL_PRESENT 0x00800000 -#define LINK_STATUS_MAC_LOCAL_FAULT 0x01000000 -#define LINK_STATUS_MAC_REMOTE_FAULT 0x02000000 -#define LINK_STATUS_UNSUPPORTED_SPD_REQ 0x04000000 +#define LINK_STATUS_LINK_PARTNER_BOTH_PAUSE (3 << 18) +#define LINK_STATUS_SFP_TX_FAULT 0x00100000 +#define LINK_STATUS_TX_FLOW_CONTROL_ENABLED 0x00200000 +#define LINK_STATUS_RX_FLOW_CONTROL_ENABLED 0x00400000 +#define LINK_STATUS_RX_SIGNAL_PRESENT 0x00800000 +#define LINK_STATUS_MAC_LOCAL_FAULT 0x01000000 +#define LINK_STATUS_MAC_REMOTE_FAULT 0x02000000 +#define LINK_STATUS_UNSUPPORTED_SPD_REQ 0x04000000 +#define LINK_STATUS_FEC_MODE_MASK 0x38000000 +#define LINK_STATUS_FEC_MODE_NONE (0 << 27) +#define LINK_STATUS_FEC_MODE_FIRECODE_CL74 (1 << 27) +#define LINK_STATUS_FEC_MODE_RS_CL91 (2 << 27) +#define LINK_STATUS_EXT_PHY_LINK_UP 0x40000000 u32 link_status1; u32 ext_phy_fw_version; - u32 drv_phy_cfg_addr; /* Points to pmm_phy_cfg (For READ-ONLY) */ +/* Points to struct eth_phy_cfg (For READ-ONLY) */ + u32 drv_phy_cfg_addr; u32 port_stx; @@ -537,15 +699,15 @@ struct public_port { struct port_stats stats; u32 media_type; -#define MEDIA_UNSPECIFIED 0x0 -#define MEDIA_SFPP_10G_FIBER 0x1 -#define MEDIA_XFP_FIBER 0x2 -#define MEDIA_DA_TWINAX 0x3 -#define MEDIA_BASE_T 0x4 -#define MEDIA_SFP_1G_FIBER 0x5 -#define MEDIA_MODULE_FIBER 0x6 -#define MEDIA_KR 0xf0 -#define MEDIA_NOT_PRESENT 0xff +#define MEDIA_UNSPECIFIED 0x0 +#define MEDIA_SFPP_10G_FIBER 0x1 /* Use MEDIA_MODULE_FIBER instead */ +#define MEDIA_XFP_FIBER 0x2 /* Use MEDIA_MODULE_FIBER instead */ +#define MEDIA_DA_TWINAX 0x3 +#define MEDIA_BASE_T 0x4 +#define MEDIA_SFP_1G_FIBER 0x5 /* Use MEDIA_MODULE_FIBER instead */ +#define MEDIA_MODULE_FIBER 0x6 +#define MEDIA_KR 0xf0 +#define MEDIA_NOT_PRESENT 0xff u32 lfa_status; #define LFA_LINK_FLAP_REASON_OFFSET 0 @@ -557,6 +719,8 @@ struct public_port { #define LFA_SPEED_MISMATCH (1 << 3) #define LFA_FLOW_CTRL_MISMATCH (1 << 4) #define LFA_ADV_SPEED_MISMATCH (1 << 5) +#define LFA_EEE_MISMATCH (1 << 6) +#define LFA_LINK_MODES_MISMATCH (1 << 7) #define LINK_FLAP_AVOIDANCE_COUNT_OFFSET 8 #define LINK_FLAP_AVOIDANCE_COUNT_MASK 0x0000ff00 #define LINK_FLAP_COUNT_OFFSET 16 @@ -565,6 +729,7 @@ struct public_port { u32 link_change_count; /* LLDP params */ +/* offset: 536 bytes? */ struct lldp_config_params_s lldp_config_params[LLDP_MAX_LLDP_AGENTS]; struct lldp_status_params_s lldp_status_params[LLDP_MAX_LLDP_AGENTS]; struct lldp_system_tlvs_buffer_s system_lldp_tlvs_buf; @@ -574,60 +739,128 @@ struct public_port { struct dcbx_mib remote_dcbx_mib; struct dcbx_mib operational_dcbx_mib; - /* FC_NPIV table offset & size in NVRAM value of 0 means not present */ +/* FC_NPIV table offset & size in NVRAM value of 0 means not present */ + u32 fc_npiv_nvram_tbl_addr; u32 fc_npiv_nvram_tbl_size; u32 transceiver_data; -#define PMM_TRANSCEIVER_STATE_MASK 0x000000FF -#define PMM_TRANSCEIVER_STATE_SHIFT 0x00000000 -#define PMM_TRANSCEIVER_STATE_UNPLUGGED 0x00000000 -#define PMM_TRANSCEIVER_STATE_PRESENT 0x00000001 -#define PMM_TRANSCEIVER_STATE_VALID 0x00000003 -#define PMM_TRANSCEIVER_STATE_UPDATING 0x00000008 -#define PMM_TRANSCEIVER_TYPE_MASK 0x0000FF00 -#define PMM_TRANSCEIVER_TYPE_SHIFT 0x00000008 -#define PMM_TRANSCEIVER_TYPE_NONE 0x00000000 -#define PMM_TRANSCEIVER_TYPE_UNKNOWN 0x000000FF -#define PMM_TRANSCEIVER_TYPE_1G_PCC 0x01 /* 1G Passive copper cable */ -#define PMM_TRANSCEIVER_TYPE_1G_ACC 0x02 /* 1G Active copper cable */ -#define PMM_TRANSCEIVER_TYPE_1G_LX 0x03 -#define PMM_TRANSCEIVER_TYPE_1G_SX 0x04 -#define PMM_TRANSCEIVER_TYPE_10G_SR 0x05 -#define PMM_TRANSCEIVER_TYPE_10G_LR 0x06 -#define PMM_TRANSCEIVER_TYPE_10G_LRM 0x07 -#define PMM_TRANSCEIVER_TYPE_10G_ER 0x08 -#define PMM_TRANSCEIVER_TYPE_10G_PCC 0x09 /* 10G Passive copper cable */ -#define PMM_TRANSCEIVER_TYPE_10G_ACC 0x0a /* 10G Active copper cable */ -#define PMM_TRANSCEIVER_TYPE_XLPPI 0x0b -#define PMM_TRANSCEIVER_TYPE_40G_LR4 0x0c -#define PMM_TRANSCEIVER_TYPE_40G_SR4 0x0d -#define PMM_TRANSCEIVER_TYPE_40G_CR4 0x0e -#define PMM_TRANSCEIVER_TYPE_100G_AOC 0x0f /* Active optical cable */ -#define PMM_TRANSCEIVER_TYPE_100G_SR4 0x10 -#define PMM_TRANSCEIVER_TYPE_100G_LR4 0x11 -#define PMM_TRANSCEIVER_TYPE_100G_ER4 0x12 -#define PMM_TRANSCEIVER_TYPE_100G_ACC 0x13 /* Active copper cable */ -#define PMM_TRANSCEIVER_TYPE_100G_CR4 0x14 -#define PMM_TRANSCEIVER_TYPE_4x10G_SR 0x15 -#define PMM_TRANSCEIVER_TYPE_25G_PCC_S 0x16 -#define PMM_TRANSCEIVER_TYPE_25G_ACC_S 0x17 -#define PMM_TRANSCEIVER_TYPE_25G_PCC_M 0x18 -#define PMM_TRANSCEIVER_TYPE_25G_ACC_M 0x19 -#define PMM_TRANSCEIVER_TYPE_25G_PCC_L 0x1a -#define PMM_TRANSCEIVER_TYPE_25G_ACC_L 0x1b -#define PMM_TRANSCEIVER_TYPE_25G_SR 0x1c -#define PMM_TRANSCEIVER_TYPE_25G_LR 0x1d -#define PMM_TRANSCEIVER_TYPE_25G_AOC 0x1e +#define ETH_TRANSCEIVER_STATE_MASK 0x000000FF +#define ETH_TRANSCEIVER_STATE_OFFSET 0x00000000 +#define ETH_TRANSCEIVER_STATE_UNPLUGGED 0x00000000 +#define ETH_TRANSCEIVER_STATE_PRESENT 0x00000001 +#define ETH_TRANSCEIVER_STATE_VALID 0x00000003 +#define ETH_TRANSCEIVER_STATE_UPDATING 0x00000008 +#define ETH_TRANSCEIVER_TYPE_MASK 0x0000FF00 +#define ETH_TRANSCEIVER_TYPE_OFFSET 0x00000008 +#define ETH_TRANSCEIVER_TYPE_NONE 0x00000000 +#define ETH_TRANSCEIVER_TYPE_UNKNOWN 0x000000FF +/* 1G Passive copper cable */ +#define ETH_TRANSCEIVER_TYPE_1G_PCC 0x01 +/* 1G Active copper cable */ +#define ETH_TRANSCEIVER_TYPE_1G_ACC 0x02 +#define ETH_TRANSCEIVER_TYPE_1G_LX 0x03 +#define ETH_TRANSCEIVER_TYPE_1G_SX 0x04 +#define ETH_TRANSCEIVER_TYPE_10G_SR 0x05 +#define ETH_TRANSCEIVER_TYPE_10G_LR 0x06 +#define ETH_TRANSCEIVER_TYPE_10G_LRM 0x07 +#define ETH_TRANSCEIVER_TYPE_10G_ER 0x08 +/* 10G Passive copper cable */ +#define ETH_TRANSCEIVER_TYPE_10G_PCC 0x09 +/* 10G Active copper cable */ +#define ETH_TRANSCEIVER_TYPE_10G_ACC 0x0a +#define ETH_TRANSCEIVER_TYPE_XLPPI 0x0b +#define ETH_TRANSCEIVER_TYPE_40G_LR4 0x0c +#define ETH_TRANSCEIVER_TYPE_40G_SR4 0x0d +#define ETH_TRANSCEIVER_TYPE_40G_CR4 0x0e +/* Active optical cable */ +#define ETH_TRANSCEIVER_TYPE_100G_AOC 0x0f +#define ETH_TRANSCEIVER_TYPE_100G_SR4 0x10 +#define ETH_TRANSCEIVER_TYPE_100G_LR4 0x11 +#define ETH_TRANSCEIVER_TYPE_100G_ER4 0x12 +/* Active copper cable */ +#define ETH_TRANSCEIVER_TYPE_100G_ACC 0x13 +#define ETH_TRANSCEIVER_TYPE_100G_CR4 0x14 +#define ETH_TRANSCEIVER_TYPE_4x10G_SR 0x15 +/* 25G Passive copper cable - short */ +#define ETH_TRANSCEIVER_TYPE_25G_CA_N 0x16 +/* 25G Active copper cable - short */ +#define ETH_TRANSCEIVER_TYPE_25G_ACC_S 0x17 +/* 25G Passive copper cable - medium */ +#define ETH_TRANSCEIVER_TYPE_25G_CA_S 0x18 +/* 25G Active copper cable - medium */ +#define ETH_TRANSCEIVER_TYPE_25G_ACC_M 0x19 +/* 25G Passive copper cable - long */ +#define ETH_TRANSCEIVER_TYPE_25G_CA_L 0x1a +/* 25G Active copper cable - long */ +#define ETH_TRANSCEIVER_TYPE_25G_ACC_L 0x1b +#define ETH_TRANSCEIVER_TYPE_25G_SR 0x1c +#define ETH_TRANSCEIVER_TYPE_25G_LR 0x1d +#define ETH_TRANSCEIVER_TYPE_25G_AOC 0x1e -#define PMM_TRANSCEIVER_TYPE_4x10G 0x1d -#define PMM_TRANSCEIVER_TYPE_4x25G_CR 0x1e -#define PMM_TRANSCEIVER_TYPE_MULTI_RATE_10G_40GR 0x30 -#define PMM_TRANSCEIVER_TYPE_MULTI_RATE_10G_40G_CR 0x31 -#define PMM_TRANSCEIVER_TYPE_MULTI_RATE_10G_40G_LR 0x32 -#define PMM_TRANSCEIVER_TYPE_MULTI_RATE_40G_100G_SR 0x33 -#define PMM_TRANSCEIVER_TYPE_MULTI_RATE_40G_100G_CR 0x34 -#define PMM_TRANSCEIVER_TYPE_MULTI_RATE_40G_100G_LR 0x35 -#define PMM_TRANSCEIVER_TYPE_MULTI_RATE_40G_100G_AOC 0x36 +#define ETH_TRANSCEIVER_TYPE_4x10G 0x1f +#define ETH_TRANSCEIVER_TYPE_4x25G_CR 0x20 +#define ETH_TRANSCEIVER_TYPE_1000BASET 0x21 +#define ETH_TRANSCEIVER_TYPE_MULTI_RATE_10G_40G_SR 0x30 +#define ETH_TRANSCEIVER_TYPE_MULTI_RATE_10G_40G_CR 0x31 +#define ETH_TRANSCEIVER_TYPE_MULTI_RATE_10G_40G_LR 0x32 +#define ETH_TRANSCEIVER_TYPE_MULTI_RATE_40G_100G_SR 0x33 +#define ETH_TRANSCEIVER_TYPE_MULTI_RATE_40G_100G_CR 0x34 +#define ETH_TRANSCEIVER_TYPE_MULTI_RATE_40G_100G_LR 0x35 +#define ETH_TRANSCEIVER_TYPE_MULTI_RATE_40G_100G_AOC 0x36 + u32 wol_info; + u32 wol_pkt_len; + u32 wol_pkt_details; + struct dcb_dscp_map dcb_dscp_map; + + u32 eee_status; +/* Set when EEE negotiation is complete. */ +#define EEE_ACTIVE_BIT (1 << 0) + +/* Shows the Local Device EEE capabilities */ +#define EEE_LD_ADV_STATUS_MASK 0x000000f0 +#define EEE_LD_ADV_STATUS_OFFSET 4 + #define EEE_1G_ADV (1 << 1) + #define EEE_10G_ADV (1 << 2) +/* Same values as in EEE_LD_ADV, but for Link Parter */ +#define EEE_LP_ADV_STATUS_MASK 0x00000f00 +#define EEE_LP_ADV_STATUS_OFFSET 8 + +/* Supported speeds for EEE */ +#define EEE_SUPPORTED_SPEED_MASK 0x0000f000 +#define EEE_SUPPORTED_SPEED_OFFSET 12 + #define EEE_1G_SUPPORTED (1 << 1) + #define EEE_10G_SUPPORTED (1 << 2) + + u32 eee_remote; /* Used for EEE in LLDP */ +#define EEE_REMOTE_TW_TX_MASK 0x0000ffff +#define EEE_REMOTE_TW_TX_OFFSET 0 +#define EEE_REMOTE_TW_RX_MASK 0xffff0000 +#define EEE_REMOTE_TW_RX_OFFSET 16 + + u32 module_info; +#define ETH_TRANSCEIVER_MONITORING_TYPE_MASK 0x000000FF +#define ETH_TRANSCEIVER_MONITORING_TYPE_OFFSET 0 +#define ETH_TRANSCEIVER_ADDR_CHNG_REQUIRED (1 << 2) +#define ETH_TRANSCEIVER_RCV_PWR_MEASURE_TYPE (1 << 3) +#define ETH_TRANSCEIVER_EXTERNALLY_CALIBRATED (1 << 4) +#define ETH_TRANSCEIVER_INTERNALLY_CALIBRATED (1 << 5) +#define ETH_TRANSCEIVER_HAS_DIAGNOSTIC (1 << 6) +#define ETH_TRANSCEIVER_IDENT_MASK 0x0000ff00 +#define ETH_TRANSCEIVER_IDENT_OFFSET 8 + + u32 oem_cfg_port; +#define OEM_CFG_CHANNEL_TYPE_MASK 0x00000003 +#define OEM_CFG_CHANNEL_TYPE_OFFSET 0 +#define OEM_CFG_CHANNEL_TYPE_VLAN_PARTITION 0x1 +#define OEM_CFG_CHANNEL_TYPE_STAGGED 0x2 + +#define OEM_CFG_SCHED_TYPE_MASK 0x0000000C +#define OEM_CFG_SCHED_TYPE_OFFSET 2 +#define OEM_CFG_SCHED_TYPE_ETS 0x1 +#define OEM_CFG_SCHED_TYPE_VNIC_BW 0x2 + + struct lldp_received_tlvs_s lldp_received_tlvs[LLDP_MAX_LLDP_AGENTS]; + u32 system_lldp_tlvs_buf2[MAX_SYSTEM_LLDP_TLV_DATA]; }; /**************************************/ @@ -637,11 +870,13 @@ struct public_port { /**************************************/ struct public_func { - u32 dpdk_rsvd1[2]; + u32 iscsi_boot_signature; + u32 iscsi_boot_block_offset; /* MTU size per funciton is needed for the OV feature */ u32 mtu_size; - /* 9 entires for the C2S PCP map for each inner VLAN PCP + 1 default */ +/* 9 entires for the C2S PCP map for each inner VLAN PCP + 1 default */ + /* For PCP values 0-3 use the map lower */ /* 0xFF000000 - PCP 0, 0x00FF0000 - PCP 1, * 0x0000FF00 - PCP 2, 0x000000FF PCP 3 @@ -650,7 +885,7 @@ struct public_func { /* For PCP values 4-7 use the map upper */ /* 0xFF000000 - PCP 4, 0x00FF0000 - PCP 5, * 0x0000FF00 - PCP 6, 0x000000FF PCP 7 - */ + */ u32 c2s_pcp_map_upper; /* For PCP default value get the MSB byte of the map default */ @@ -664,42 +899,52 @@ struct public_func { /* function 0 of each port cannot be hidden */ #define FUNC_MF_CFG_FUNC_HIDE 0x00000001 #define FUNC_MF_CFG_PAUSE_ON_HOST_RING 0x00000002 -#define FUNC_MF_CFG_PAUSE_ON_HOST_RING_SHIFT 0x00000001 +#define FUNC_MF_CFG_PAUSE_ON_HOST_RING_OFFSET 0x00000001 + #define FUNC_MF_CFG_PROTOCOL_MASK 0x000000f0 -#define FUNC_MF_CFG_PROTOCOL_SHIFT 4 +#define FUNC_MF_CFG_PROTOCOL_OFFSET 4 #define FUNC_MF_CFG_PROTOCOL_ETHERNET 0x00000000 -#define FUNC_MF_CFG_PROTOCOL_MAX 0x00000000 +#define FUNC_MF_CFG_PROTOCOL_ISCSI 0x00000010 +#define FUNC_MF_CFG_PROTOCOL_FCOE 0x00000020 +#define FUNC_MF_CFG_PROTOCOL_ROCE 0x00000030 +#define FUNC_MF_CFG_PROTOCOL_MAX 0x00000030 /* MINBW, MAXBW */ /* value range - 0..100, increments in 1 % */ #define FUNC_MF_CFG_MIN_BW_MASK 0x0000ff00 -#define FUNC_MF_CFG_MIN_BW_SHIFT 8 +#define FUNC_MF_CFG_MIN_BW_OFFSET 8 #define FUNC_MF_CFG_MIN_BW_DEFAULT 0x00000000 #define FUNC_MF_CFG_MAX_BW_MASK 0x00ff0000 -#define FUNC_MF_CFG_MAX_BW_SHIFT 16 +#define FUNC_MF_CFG_MAX_BW_OFFSET 16 #define FUNC_MF_CFG_MAX_BW_DEFAULT 0x00640000 u32 status; -#define FUNC_STATUS_VLINK_DOWN 0x00000001 +#define FUNC_STATUS_VIRTUAL_LINK_UP 0x00000001 +#define FUNC_STATUS_LOGICAL_LINK_UP 0x00000002 +#define FUNC_STATUS_FORCED_LINK 0x00000004 - u32 mac_upper; /* MAC */ + u32 mac_upper; /* MAC */ #define FUNC_MF_CFG_UPPERMAC_MASK 0x0000ffff -#define FUNC_MF_CFG_UPPERMAC_SHIFT 0 +#define FUNC_MF_CFG_UPPERMAC_OFFSET 0 #define FUNC_MF_CFG_UPPERMAC_DEFAULT FUNC_MF_CFG_UPPERMAC_MASK u32 mac_lower; #define FUNC_MF_CFG_LOWERMAC_DEFAULT 0xffffffff - u32 dpdk_rsvd2[4]; + u32 fcoe_wwn_port_name_upper; + u32 fcoe_wwn_port_name_lower; - u32 ovlan_stag; /* tags */ + u32 fcoe_wwn_node_name_upper; + u32 fcoe_wwn_node_name_lower; + + u32 ovlan_stag; /* tags */ #define FUNC_MF_CFG_OV_STAG_MASK 0x0000ffff -#define FUNC_MF_CFG_OV_STAG_SHIFT 0 +#define FUNC_MF_CFG_OV_STAG_OFFSET 0 #define FUNC_MF_CFG_OV_STAG_DEFAULT FUNC_MF_CFG_OV_STAG_MASK - u32 pf_allocation; /* vf per pf */ + u32 pf_allocation; /* vf per pf */ - u32 preserve_data; /* Will be used bt CCM */ + u32 preserve_data; /* Will be used bt CCM */ u32 driver_last_activity_ts; @@ -707,31 +952,50 @@ struct public_func { * drv_ack_vf_disabled is set by the PF driver to ack handled disabled * VFs */ - u32 drv_ack_vf_disabled[VF_MAX_STATIC / 32]; /* 0x0044 */ + u32 drv_ack_vf_disabled[VF_MAX_STATIC / 32]; /* 0x0044 */ u32 drv_id; #define DRV_ID_PDA_COMP_VER_MASK 0x0000ffff -#define DRV_ID_PDA_COMP_VER_SHIFT 0 +#define DRV_ID_PDA_COMP_VER_OFFSET 0 +#define LOAD_REQ_HSI_VERSION 2 #define DRV_ID_MCP_HSI_VER_MASK 0x00ff0000 -#define DRV_ID_MCP_HSI_VER_SHIFT 16 -#define DRV_ID_MCP_HSI_VER_CURRENT (1 << DRV_ID_MCP_HSI_VER_SHIFT) +#define DRV_ID_MCP_HSI_VER_OFFSET 16 +#define DRV_ID_MCP_HSI_VER_CURRENT (LOAD_REQ_HSI_VERSION << \ + DRV_ID_MCP_HSI_VER_OFFSET) #define DRV_ID_DRV_TYPE_MASK 0x7f000000 -#define DRV_ID_DRV_TYPE_SHIFT 24 -#define DRV_ID_DRV_TYPE_UNKNOWN (0 << DRV_ID_DRV_TYPE_SHIFT) -#define DRV_ID_DRV_TYPE_LINUX (1 << DRV_ID_DRV_TYPE_SHIFT) -#define DRV_ID_DRV_TYPE_WINDOWS (2 << DRV_ID_DRV_TYPE_SHIFT) -#define DRV_ID_DRV_TYPE_DIAG (3 << DRV_ID_DRV_TYPE_SHIFT) -#define DRV_ID_DRV_TYPE_PREBOOT (4 << DRV_ID_DRV_TYPE_SHIFT) -#define DRV_ID_DRV_TYPE_SOLARIS (5 << DRV_ID_DRV_TYPE_SHIFT) -#define DRV_ID_DRV_TYPE_VMWARE (6 << DRV_ID_DRV_TYPE_SHIFT) -#define DRV_ID_DRV_TYPE_FREEBSD (7 << DRV_ID_DRV_TYPE_SHIFT) -#define DRV_ID_DRV_TYPE_AIX (8 << DRV_ID_DRV_TYPE_SHIFT) +#define DRV_ID_DRV_TYPE_OFFSET 24 +#define DRV_ID_DRV_TYPE_UNKNOWN (0 << DRV_ID_DRV_TYPE_OFFSET) +#define DRV_ID_DRV_TYPE_LINUX (1 << DRV_ID_DRV_TYPE_OFFSET) +#define DRV_ID_DRV_TYPE_WINDOWS (2 << DRV_ID_DRV_TYPE_OFFSET) +#define DRV_ID_DRV_TYPE_DIAG (3 << DRV_ID_DRV_TYPE_OFFSET) +#define DRV_ID_DRV_TYPE_PREBOOT (4 << DRV_ID_DRV_TYPE_OFFSET) +#define DRV_ID_DRV_TYPE_SOLARIS (5 << DRV_ID_DRV_TYPE_OFFSET) +#define DRV_ID_DRV_TYPE_VMWARE (6 << DRV_ID_DRV_TYPE_OFFSET) +#define DRV_ID_DRV_TYPE_FREEBSD (7 << DRV_ID_DRV_TYPE_OFFSET) +#define DRV_ID_DRV_TYPE_AIX (8 << DRV_ID_DRV_TYPE_OFFSET) #define DRV_ID_DRV_INIT_HW_MASK 0x80000000 -#define DRV_ID_DRV_INIT_HW_SHIFT 31 -#define DRV_ID_DRV_INIT_HW_FLAG (1 << DRV_ID_DRV_INIT_HW_SHIFT) +#define DRV_ID_DRV_INIT_HW_OFFSET 31 +#define DRV_ID_DRV_INIT_HW_FLAG (1 << DRV_ID_DRV_INIT_HW_OFFSET) + + u32 oem_cfg_func; +#define OEM_CFG_FUNC_TC_MASK 0x0000000F +#define OEM_CFG_FUNC_TC_OFFSET 0 +#define OEM_CFG_FUNC_TC_0 0x0 +#define OEM_CFG_FUNC_TC_1 0x1 +#define OEM_CFG_FUNC_TC_2 0x2 +#define OEM_CFG_FUNC_TC_3 0x3 +#define OEM_CFG_FUNC_TC_4 0x4 +#define OEM_CFG_FUNC_TC_5 0x5 +#define OEM_CFG_FUNC_TC_6 0x6 +#define OEM_CFG_FUNC_TC_7 0x7 + +#define OEM_CFG_FUNC_HOST_PRI_CTRL_MASK 0x00000030 +#define OEM_CFG_FUNC_HOST_PRI_CTRL_OFFSET 4 +#define OEM_CFG_FUNC_HOST_PRI_CTRL_VNIC 0x1 +#define OEM_CFG_FUNC_HOST_PRI_CTRL_OS 0x2 }; /**************************************/ @@ -747,7 +1011,7 @@ struct public_func { */ struct mcp_mac { - u32 mac_upper; /* Upper 16 bits are always zeroes */ + u32 mac_upper; /* Upper 16 bits are always zeroes */ u32 mac_lower; }; @@ -761,6 +1025,13 @@ struct mcp_file_att { u32 len; }; +struct bist_nvm_image_att { + u32 return_code; + u32 image_type; /* Image type */ + u32 nvm_start_addr; /* NVM address of the image */ + u32 len; /* Include CRC */ +}; + #define MCP_DRV_VER_STR_SIZE 16 #define MCP_DRV_VER_STR_SIZE_DWORD (MCP_DRV_VER_STR_SIZE / sizeof(u32)) #define MCP_DRV_NVM_BUF_LEN 32 @@ -777,19 +1048,159 @@ struct lan_stats_stc { u32 rserved; }; +struct fcoe_stats_stc { + u64 rx_pkts; + u64 tx_pkts; + u32 fcs_err; + u32 login_failure; +}; + +struct iscsi_stats_stc { + u64 rx_pdus; + u64 tx_pdus; + u64 rx_bytes; + u64 tx_bytes; +}; + +struct rdma_stats_stc { + u64 rx_pkts; + u64 tx_pkts; + u64 rx_bytes; + u64 tx_bytes; +}; + struct ocbb_data_stc { u32 ocbb_host_addr; u32 ocsd_host_addr; u32 ocsd_req_update_interval; }; +#define MAX_NUM_OF_SENSORS 7 +#define MFW_SENSOR_LOCATION_INTERNAL 1 +#define MFW_SENSOR_LOCATION_EXTERNAL 2 +#define MFW_SENSOR_LOCATION_SFP 3 + +#define SENSOR_LOCATION_OFFSET 0 +#define SENSOR_LOCATION_MASK 0x000000ff +#define THRESHOLD_HIGH_OFFSET 8 +#define THRESHOLD_HIGH_MASK 0x0000ff00 +#define CRITICAL_TEMPERATURE_OFFSET 16 +#define CRITICAL_TEMPERATURE_MASK 0x00ff0000 +#define CURRENT_TEMP_OFFSET 24 +#define CURRENT_TEMP_MASK 0xff000000 +struct temperature_status_stc { + u32 num_of_sensors; + u32 sensor[MAX_NUM_OF_SENSORS]; +}; + +/* crash dump configuration header */ +struct mdump_config_stc { + u32 version; + u32 config; + u32 epoc; + u32 num_of_logs; + u32 valid_logs; +}; + +enum resource_id_enum { + RESOURCE_NUM_SB_E = 0, + RESOURCE_NUM_L2_QUEUE_E = 1, + RESOURCE_NUM_VPORT_E = 2, + RESOURCE_NUM_VMQ_E = 3, +/* Not a real resource!! it's a factor used to calculate others */ + RESOURCE_FACTOR_NUM_RSS_PF_E = 4, +/* Not a real resource!! it's a factor used to calculate others */ + RESOURCE_FACTOR_RSS_PER_VF_E = 5, + RESOURCE_NUM_RL_E = 6, + RESOURCE_NUM_PQ_E = 7, + RESOURCE_NUM_VF_E = 8, + RESOURCE_VFC_FILTER_E = 9, + RESOURCE_ILT_E = 10, + RESOURCE_CQS_E = 11, + RESOURCE_GFT_PROFILES_E = 12, + RESOURCE_NUM_TC_E = 13, + RESOURCE_NUM_RSS_ENGINES_E = 14, + RESOURCE_LL2_QUEUE_E = 15, + RESOURCE_RDMA_STATS_QUEUE_E = 16, + RESOURCE_BDQ_E = 17, + RESOURCE_MAX_NUM, + RESOURCE_NUM_INVALID = 0xFFFFFFFF +}; + +/* Resource ID is to be filled by the driver in the MB request + * Size, offset & flags to be filled by the MFW in the MB response + */ +struct resource_info { + enum resource_id_enum res_id; + u32 size; /* number of allocated resources */ + u32 offset; /* Offset of the 1st resource */ + u32 vf_size; + u32 vf_offset; + u32 flags; +#define RESOURCE_ELEMENT_STRICT (1 << 0) +}; + +#define DRV_ROLE_NONE 0 +#define DRV_ROLE_PREBOOT 1 +#define DRV_ROLE_OS 2 +#define DRV_ROLE_KDUMP 3 + +struct load_req_stc { + u32 drv_ver_0; + u32 drv_ver_1; + u32 fw_ver; + u32 misc0; +#define LOAD_REQ_ROLE_MASK 0x000000FF +#define LOAD_REQ_ROLE_OFFSET 0 +#define LOAD_REQ_LOCK_TO_MASK 0x0000FF00 +#define LOAD_REQ_LOCK_TO_OFFSET 8 +#define LOAD_REQ_LOCK_TO_DEFAULT 0 +#define LOAD_REQ_LOCK_TO_NONE 255 +#define LOAD_REQ_FORCE_MASK 0x000F0000 +#define LOAD_REQ_FORCE_OFFSET 16 +#define LOAD_REQ_FORCE_NONE 0 +#define LOAD_REQ_FORCE_PF 1 +#define LOAD_REQ_FORCE_ALL 2 +#define LOAD_REQ_FLAGS0_MASK 0x00F00000 +#define LOAD_REQ_FLAGS0_OFFSET 20 +#define LOAD_REQ_FLAGS0_AVOID_RESET (0x1 << 0) +}; + +struct load_rsp_stc { + u32 drv_ver_0; + u32 drv_ver_1; + u32 fw_ver; + u32 misc0; +#define LOAD_RSP_ROLE_MASK 0x000000FF +#define LOAD_RSP_ROLE_OFFSET 0 +#define LOAD_RSP_HSI_MASK 0x0000FF00 +#define LOAD_RSP_HSI_OFFSET 8 +#define LOAD_RSP_FLAGS0_MASK 0x000F0000 +#define LOAD_RSP_FLAGS0_OFFSET 16 +#define LOAD_RSP_FLAGS0_DRV_EXISTS (0x1 << 0) +}; + +struct mdump_retain_data_stc { + u32 valid; + u32 epoch; + u32 pf; + u32 status; +}; + +struct attribute_cmd_write_stc { + u32 val; + u32 mask; + u32 offset; +}; + union drv_union_data { - u32 ver_str[MCP_DRV_VER_STR_SIZE_DWORD]; /* LOAD_REQ */ - struct mcp_mac wol_mac; /* UNLOAD_DONE */ + struct mcp_mac wol_mac; /* UNLOAD_DONE */ - struct pmm_phy_cfg drv_phy_cfg; +/* This configuration should be set by the driver for the LINK_SET command. */ - struct mcp_val64 val64; /* For PHY / AVS commands */ + struct eth_phy_cfg drv_phy_cfg; + + struct mcp_val64 val64; /* For PHY / AVS commands */ u8 raw_data[MCP_DRV_NVM_BUF_LEN]; @@ -800,9 +1211,20 @@ union drv_union_data { struct drv_version_stc drv_version; struct lan_stats_stc lan_stats; - u32 dpdk_rsvd[3]; + struct fcoe_stats_stc fcoe_stats; + struct iscsi_stats_stc iscsi_stats; + struct rdma_stats_stc rdma_stats; struct ocbb_data_stc ocbb_info; + struct temperature_status_stc temp_info; + struct resource_info resource; + struct bist_nvm_image_att nvm_image_att; + struct mdump_config_stc mdump_config; + u32 dword; + struct load_req_stc load_req; + struct load_rsp_stc load_rsp; + struct mdump_retain_data_stc mdump_retain; + struct attribute_cmd_write_stc attribute_cmd_write; /* ... */ }; @@ -812,6 +1234,7 @@ struct public_drv_mb { #define DRV_MSG_CODE_LOAD_REQ 0x10000000 #define DRV_MSG_CODE_LOAD_DONE 0x11000000 #define DRV_MSG_CODE_INIT_HW 0x12000000 +#define DRV_MSG_CODE_CANCEL_LOAD_REQ 0x13000000 #define DRV_MSG_CODE_UNLOAD_REQ 0x20000000 #define DRV_MSG_CODE_UNLOAD_DONE 0x21000000 #define DRV_MSG_CODE_INIT_PHY 0x22000000 @@ -819,64 +1242,233 @@ struct public_drv_mb { /* - DONT_CARE - Don't flap the link if up */ #define DRV_MSG_CODE_LINK_RESET 0x23000000 - /* Vitaly: LLDP commands */ #define DRV_MSG_CODE_SET_LLDP 0x24000000 +#define DRV_MSG_CODE_REGISTER_LLDP_TLVS_RX 0x24100000 #define DRV_MSG_CODE_SET_DCBX 0x25000000 - /* OneView feature driver HSI */ + /* OneView feature driver HSI*/ #define DRV_MSG_CODE_OV_UPDATE_CURR_CFG 0x26000000 #define DRV_MSG_CODE_OV_UPDATE_BUS_NUM 0x27000000 #define DRV_MSG_CODE_OV_UPDATE_BOOT_PROGRESS 0x28000000 #define DRV_MSG_CODE_OV_UPDATE_STORM_FW_VER 0x29000000 +#define DRV_MSG_CODE_NIG_DRAIN 0x30000000 #define DRV_MSG_CODE_OV_UPDATE_DRIVER_STATE 0x31000000 #define DRV_MSG_CODE_BW_UPDATE_ACK 0x32000000 #define DRV_MSG_CODE_OV_UPDATE_MTU 0x33000000 +/* DRV_MB Param: driver version supp, FW_MB param: MFW version supp, + * data: struct resource_info + */ +#define DRV_MSG_GET_RESOURCE_ALLOC_MSG 0x34000000 +#define DRV_MSG_SET_RESOURCE_VALUE_MSG 0x35000000 -#define DRV_MSG_CODE_NIG_DRAIN 0x30000000 - -#define DRV_MSG_CODE_INITIATE_FLR 0x02000000 +/*deprecated don't use*/ +#define DRV_MSG_CODE_INITIATE_FLR_DEPRECATED 0x02000000 +#define DRV_MSG_CODE_INITIATE_PF_FLR 0x02010000 #define DRV_MSG_CODE_VF_DISABLED_DONE 0xc0000000 #define DRV_MSG_CODE_CFG_VF_MSIX 0xc0010000 +#define DRV_MSG_CODE_CFG_PF_VFS_MSIX 0xc0020000 +/* Param is either DRV_MB_PARAM_NVM_PUT_FILE_BEGIN_MFW/IMAGE */ #define DRV_MSG_CODE_NVM_PUT_FILE_BEGIN 0x00010000 +/* Param should be set to the transaction size (up to 64 bytes) */ #define DRV_MSG_CODE_NVM_PUT_FILE_DATA 0x00020000 +/* MFW will place the file offset and len in file_att struct */ #define DRV_MSG_CODE_NVM_GET_FILE_ATT 0x00030000 +/* Read 32bytes of nvram data. Param is [0:23] ??? Offset [24:31] - + * ??? Len in Bytes + */ #define DRV_MSG_CODE_NVM_READ_NVRAM 0x00050000 +/* Writes up to 32Bytes to nvram. Param is [0:23] ??? Offset [24:31] + * ??? Len in Bytes. In case this address is in the range of secured file in + * secured mode, the operation will fail + */ #define DRV_MSG_CODE_NVM_WRITE_NVRAM 0x00060000 +/* Delete a file from nvram. Param is image_type. */ #define DRV_MSG_CODE_NVM_DEL_FILE 0x00080000 +/* Reset MCP when no NVM operation is going on, and no drivers are loaded. + * In case operation succeed, MCP will not ack back. + */ #define DRV_MSG_CODE_MCP_RESET 0x00090000 +/* Temporary command to set secure mode, where the param is 0 (None secure) / + * 1 (Secure) / 2 (Full-Secure) + */ #define DRV_MSG_CODE_SET_SECURE_MODE 0x000a0000 +/* Param: [0:15] - Address, [16:18] - lane# (0/1/2/3 - for single lane, + * 4/5 - for dual lanes, 6 - for all lanes, [28] - PMD reg, [29] - select port, + * [30:31] - port + */ #define DRV_MSG_CODE_PHY_RAW_READ 0x000b0000 +/* Param: [0:15] - Address, [16:18] - lane# (0/1/2/3 - for single lane, + * 4/5 - for dual lanes, 6 - for all lanes, [28] - PMD reg, [29] - select port, + * [30:31] - port + */ #define DRV_MSG_CODE_PHY_RAW_WRITE 0x000c0000 +/* Param: [0:15] - Address, [30:31] - port */ #define DRV_MSG_CODE_PHY_CORE_READ 0x000d0000 +/* Param: [0:15] - Address, [30:31] - port */ #define DRV_MSG_CODE_PHY_CORE_WRITE 0x000e0000 +/* Param: [0:3] - version, [4:15] - name (null terminated) */ #define DRV_MSG_CODE_SET_VERSION 0x000f0000 +/* Halts the MCP. To resume MCP, user will need to use + * MCP_REG_CPU_STATE/MCP_REG_CPU_MODE registers. + */ #define DRV_MSG_CODE_MCP_HALT 0x00100000 -#define DRV_MSG_CODE_PMD_DIAG_DUMP 0x00140000 -#define DRV_MSG_CODE_PMD_DIAG_EYE 0x00150000 -#define DRV_MSG_CODE_TRANSCEIVER_READ 0x00160000 -#define DRV_MSG_CODE_TRANSCEIVER_WRITE 0x00170000 - +/* Set virtual mac address, params [31:6] - reserved, [5:4] - type, + * [3:0] - func, drv_data[7:0] - MAC/WWNN/WWPN + */ #define DRV_MSG_CODE_SET_VMAC 0x00110000 +/* Set virtual mac address, params [31:6] - reserved, [5:4] - type, + * [3:0] - func, drv_data[7:0] - MAC/WWNN/WWPN + */ #define DRV_MSG_CODE_GET_VMAC 0x00120000 +#define DRV_MSG_CODE_VMAC_TYPE_OFFSET 4 +#define DRV_MSG_CODE_VMAC_TYPE_MASK 0x30 #define DRV_MSG_CODE_VMAC_TYPE_MAC 1 #define DRV_MSG_CODE_VMAC_TYPE_WWNN 2 #define DRV_MSG_CODE_VMAC_TYPE_WWPN 3 - +/* Get statistics from pf, params [31:4] - reserved, [3:0] - stats type */ #define DRV_MSG_CODE_GET_STATS 0x00130000 #define DRV_MSG_CODE_STATS_TYPE_LAN 1 - +#define DRV_MSG_CODE_STATS_TYPE_FCOE 2 +#define DRV_MSG_CODE_STATS_TYPE_ISCSI 3 +#define DRV_MSG_CODE_STATS_TYPE_RDMA 4 +/* Host shall provide buffer and size for MFW */ +#define DRV_MSG_CODE_PMD_DIAG_DUMP 0x00140000 +/* Host shall provide buffer and size for MFW */ +#define DRV_MSG_CODE_PMD_DIAG_EYE 0x00150000 +/* Param: [0:1] - Port, [2:7] - read size, [8:15] - I2C address, + * [16:31] - offset + */ +#define DRV_MSG_CODE_TRANSCEIVER_READ 0x00160000 +/* Param: [0:1] - Port, [2:7] - write size, [8:15] - I2C address, + * [16:31] - offset + */ +#define DRV_MSG_CODE_TRANSCEIVER_WRITE 0x00170000 +/* indicate OCBB related information */ #define DRV_MSG_CODE_OCBB_DATA 0x00180000 +/* Set function BW, params[15:8] - min, params[7:0] - max */ #define DRV_MSG_CODE_SET_BW 0x00190000 +#define BW_MAX_MASK 0x000000ff +#define BW_MAX_OFFSET 0 +#define BW_MIN_MASK 0x0000ff00 +#define BW_MIN_OFFSET 8 + +/* When param is set to 1, all parities will be masked(disabled). When params + * are set to 0, parities will be unmasked again. + */ #define DRV_MSG_CODE_MASK_PARITIES 0x001a0000 +/* param[0] - Simulate fan failure, param[1] - simulate over temp. */ #define DRV_MSG_CODE_INDUCE_FAILURE 0x001b0000 #define DRV_MSG_FAN_FAILURE_TYPE (1 << 0) #define DRV_MSG_TEMPERATURE_FAILURE_TYPE (1 << 1) - +/* Param: [0:15] - gpio number */ #define DRV_MSG_CODE_GPIO_READ 0x001c0000 +/* Param: [0:15] - gpio number, [16:31] - gpio value */ #define DRV_MSG_CODE_GPIO_WRITE 0x001d0000 +/* Param: [0:7] - test enum, [8:15] - image index, [16:31] - reserved */ +#define DRV_MSG_CODE_BIST_TEST 0x001e0000 +#define DRV_MSG_CODE_GET_TEMPERATURE 0x001f0000 +/* Set LED mode params :0 operational, 1 LED turn ON, 2 LED turn OFF */ #define DRV_MSG_CODE_SET_LED_MODE 0x00200000 +/* drv_data[7:0] - EPOC in seconds, drv_data[15:8] - + * driver version (MAJ MIN BUILD SUB) + */ +#define DRV_MSG_CODE_TIMESTAMP 0x00210000 +/* This is an empty mailbox just return OK*/ #define DRV_MSG_CODE_EMPTY_MB 0x00220000 +/* Param[0:4] - resource number (0-31), Param[5:7] - opcode, + * param[15:8] - age + */ +#define DRV_MSG_CODE_RESOURCE_CMD 0x00230000 + +#define RESOURCE_CMD_REQ_RESC_MASK 0x0000001F +#define RESOURCE_CMD_REQ_RESC_OFFSET 0 +#define RESOURCE_CMD_REQ_OPCODE_MASK 0x000000E0 +#define RESOURCE_CMD_REQ_OPCODE_OFFSET 5 +/* request resource ownership with default aging */ +#define RESOURCE_OPCODE_REQ 1 +/* request resource ownership without aging */ +#define RESOURCE_OPCODE_REQ_WO_AGING 2 +/* request resource ownership with specific aging timer (in seconds) */ +#define RESOURCE_OPCODE_REQ_W_AGING 3 +#define RESOURCE_OPCODE_RELEASE 4 /* release resource */ +/* force resource release */ +#define RESOURCE_OPCODE_FORCE_RELEASE 5 +#define RESOURCE_CMD_REQ_AGE_MASK 0x0000FF00 +#define RESOURCE_CMD_REQ_AGE_OFFSET 8 + +#define RESOURCE_CMD_RSP_OWNER_MASK 0x000000FF +#define RESOURCE_CMD_RSP_OWNER_OFFSET 0 +#define RESOURCE_CMD_RSP_OPCODE_MASK 0x00000700 +#define RESOURCE_CMD_RSP_OPCODE_OFFSET 8 +/* resource is free and granted to requester */ +#define RESOURCE_OPCODE_GNT 1 +/* resource is busy, param[7:0] indicates owner as follow 0-15 = PF0-15, + * 16 = MFW, 17 = diag over serial + */ +#define RESOURCE_OPCODE_BUSY 2 +/* indicate release request was acknowledged */ +#define RESOURCE_OPCODE_RELEASED 3 +/* indicate release request was previously received by other owner */ +#define RESOURCE_OPCODE_RELEASED_PREVIOUS 4 +/* indicate wrong owner during release */ +#define RESOURCE_OPCODE_WRONG_OWNER 5 +#define RESOURCE_OPCODE_UNKNOWN_CMD 255 + +/* dedicate resource 0 for dump */ +#define RESOURCE_DUMP 0 + +#define DRV_MSG_CODE_GET_MBA_VERSION 0x00240000 /* Get MBA version */ +/* Send crash dump commands with param[3:0] - opcode */ +#define DRV_MSG_CODE_MDUMP_CMD 0x00250000 +#define MDUMP_DRV_PARAM_OPCODE_MASK 0x0000000f +/* acknowledge reception of error indication */ +#define DRV_MSG_CODE_MDUMP_ACK 0x01 +/* set epoc and personality as follow: drv_data[3:0] - epoch, + * drv_data[7:4] - personality + */ +#define DRV_MSG_CODE_MDUMP_SET_VALUES 0x02 +/* trigger crash dump procedure */ +#define DRV_MSG_CODE_MDUMP_TRIGGER 0x03 +/* Request valid logs and config words */ +#define DRV_MSG_CODE_MDUMP_GET_CONFIG 0x04 +/* Set triggers mask. drv_mb_param should indicate (bitwise) which + * trigger enabled + */ +#define DRV_MSG_CODE_MDUMP_SET_ENABLE 0x05 +/* Clear all logs */ +#define DRV_MSG_CODE_MDUMP_CLEAR_LOGS 0x06 +#define DRV_MSG_CODE_MDUMP_GET_RETAIN 0x07 /* Get retained data */ +#define DRV_MSG_CODE_MDUMP_CLR_RETAIN 0x08 /* Clear retain data */ +#define DRV_MSG_CODE_MEM_ECC_EVENTS 0x00260000 /* Param: None */ +/* Param: [0:15] - gpio number */ +#define DRV_MSG_CODE_GPIO_INFO 0x00270000 +/* Value will be placed in union */ +#define DRV_MSG_CODE_EXT_PHY_READ 0x00280000 +/* Value should be placed in union */ +#define DRV_MSG_CODE_EXT_PHY_WRITE 0x00290000 +#define DRV_MB_PARAM_ADDR_OFFSET 0 +#define DRV_MB_PARAM_ADDR_MASK 0x0000FFFF +#define DRV_MB_PARAM_DEVAD_OFFSET 16 +#define DRV_MB_PARAM_DEVAD_MASK 0x001F0000 +#define DRV_MB_PARAM_PORT_OFFSET 21 +#define DRV_MB_PARAM_PORT_MASK 0x00600000 +#define DRV_MSG_CODE_EXT_PHY_FW_UPGRADE 0x002a0000 + +#define DRV_MSG_CODE_GET_TLV_DONE 0x002f0000 /* Param: None */ +/* Param: Set DRV_MB_PARAM_FEATURE_SUPPORT_* */ +#define DRV_MSG_CODE_FEATURE_SUPPORT 0x00300000 +/* return FW_MB_PARAM_FEATURE_SUPPORT_* */ +#define DRV_MSG_CODE_GET_MFW_FEATURE_SUPPORT 0x00310000 +#define DRV_MSG_CODE_READ_WOL_REG 0X00320000 +#define DRV_MSG_CODE_WRITE_WOL_REG 0X00330000 +#define DRV_MSG_CODE_GET_WOL_BUFFER 0X00340000 +/* Param: [0:23] Attribute key, [24:31] Attribute sub command */ +#define DRV_MSG_CODE_ATTRIBUTE 0x00350000 + +/* Param: Password len. Union: Plain Password */ +#define DRV_MSG_CODE_ENCRYPT_PASSWORD 0x00360000 + #define DRV_MSG_SEQ_NUMBER_MASK 0x0000ffff u32 drv_mb_param; @@ -893,46 +1485,57 @@ struct public_drv_mb { #define DRV_MB_PARAM_INIT_PHY_FORCE 0x00000001 #define DRV_MB_PARAM_INIT_PHY_DONT_CARE 0x00000002 - /* LLDP / DCBX params */ + /* LLDP / DCBX params*/ + /* To be used with SET_LLDP command */ #define DRV_MB_PARAM_LLDP_SEND_MASK 0x00000001 -#define DRV_MB_PARAM_LLDP_SEND_SHIFT 0 +#define DRV_MB_PARAM_LLDP_SEND_OFFSET 0 + /* To be used with SET_LLDP and REGISTER_LLDP_TLVS_RX commands */ #define DRV_MB_PARAM_LLDP_AGENT_MASK 0x00000006 -#define DRV_MB_PARAM_LLDP_AGENT_SHIFT 1 +#define DRV_MB_PARAM_LLDP_AGENT_OFFSET 1 + /* To be used with REGISTER_LLDP_TLVS_RX command */ +#define DRV_MB_PARAM_LLDP_TLV_RX_VALID_MASK 0x00000001 +#define DRV_MB_PARAM_LLDP_TLV_RX_VALID_OFFSET 0 +#define DRV_MB_PARAM_LLDP_TLV_RX_TYPE_MASK 0x000007f0 +#define DRV_MB_PARAM_LLDP_TLV_RX_TYPE_OFFSET 4 + /* To be used with SET_DCBX command */ #define DRV_MB_PARAM_DCBX_NOTIFY_MASK 0x00000008 -#define DRV_MB_PARAM_DCBX_NOTIFY_SHIFT 3 +#define DRV_MB_PARAM_DCBX_NOTIFY_OFFSET 3 #define DRV_MB_PARAM_NIG_DRAIN_PERIOD_MS_MASK 0x000000FF -#define DRV_MB_PARAM_NIG_DRAIN_PERIOD_MS_SHIFT 0 +#define DRV_MB_PARAM_NIG_DRAIN_PERIOD_MS_OFFSET 0 #define DRV_MB_PARAM_NVM_PUT_FILE_BEGIN_MFW 0x1 #define DRV_MB_PARAM_NVM_PUT_FILE_BEGIN_IMAGE 0x2 -#define DRV_MB_PARAM_NVM_OFFSET_SHIFT 0 +#define DRV_MB_PARAM_NVM_OFFSET_OFFSET 0 #define DRV_MB_PARAM_NVM_OFFSET_MASK 0x00FFFFFF -#define DRV_MB_PARAM_NVM_LEN_SHIFT 24 +#define DRV_MB_PARAM_NVM_LEN_OFFSET 24 #define DRV_MB_PARAM_NVM_LEN_MASK 0xFF000000 -#define DRV_MB_PARAM_PHY_ADDR_SHIFT 0 +#define DRV_MB_PARAM_PHY_ADDR_OFFSET 0 #define DRV_MB_PARAM_PHY_ADDR_MASK 0x1FF0FFFF -#define DRV_MB_PARAM_PHY_LANE_SHIFT 16 +#define DRV_MB_PARAM_PHY_LANE_OFFSET 16 #define DRV_MB_PARAM_PHY_LANE_MASK 0x000F0000 -#define DRV_MB_PARAM_PHY_SELECT_PORT_SHIFT 29 +#define DRV_MB_PARAM_PHY_SELECT_PORT_OFFSET 29 #define DRV_MB_PARAM_PHY_SELECT_PORT_MASK 0x20000000 -#define DRV_MB_PARAM_PHY_PORT_SHIFT 30 +#define DRV_MB_PARAM_PHY_PORT_OFFSET 30 #define DRV_MB_PARAM_PHY_PORT_MASK 0xc0000000 -#define DRV_MB_PARAM_PHYMOD_LANE_SHIFT 0 +#define DRV_MB_PARAM_PHYMOD_LANE_OFFSET 0 #define DRV_MB_PARAM_PHYMOD_LANE_MASK 0x000000FF -#define DRV_MB_PARAM_PHYMOD_SIZE_SHIFT 8 +#define DRV_MB_PARAM_PHYMOD_SIZE_OFFSET 8 #define DRV_MB_PARAM_PHYMOD_SIZE_MASK 0x000FFF00 - /* configure vf MSIX params */ -#define DRV_MB_PARAM_CFG_VF_MSIX_VF_ID_SHIFT 0 + /* configure vf MSIX params BB */ +#define DRV_MB_PARAM_CFG_VF_MSIX_VF_ID_OFFSET 0 #define DRV_MB_PARAM_CFG_VF_MSIX_VF_ID_MASK 0x000000FF -#define DRV_MB_PARAM_CFG_VF_MSIX_SB_NUM_SHIFT 8 +#define DRV_MB_PARAM_CFG_VF_MSIX_SB_NUM_OFFSET 8 #define DRV_MB_PARAM_CFG_VF_MSIX_SB_NUM_MASK 0x0000FF00 + /* configure vf MSIX for PF params AH*/ +#define DRV_MB_PARAM_CFG_PF_VFS_MSIX_SB_NUM_OFFSET 0 +#define DRV_MB_PARAM_CFG_PF_VFS_MSIX_SB_NUM_MASK 0x000000FF /* OneView configuration parametres */ -#define DRV_MB_PARAM_OV_CURR_CFG_SHIFT 0 +#define DRV_MB_PARAM_OV_CURR_CFG_OFFSET 0 #define DRV_MB_PARAM_OV_CURR_CFG_MASK 0x0000000F #define DRV_MB_PARAM_OV_CURR_CFG_NONE 0 #define DRV_MB_PARAM_OV_CURR_CFG_OS 1 @@ -943,62 +1546,115 @@ struct public_drv_mb { #define DRV_MB_PARAM_OV_CURR_CFG_DCI 6 #define DRV_MB_PARAM_OV_CURR_CFG_HII 7 -#define DRV_MB_PARAM_OV_UPDATE_BOOT_PROG_SHIFT 0 +#define DRV_MB_PARAM_OV_UPDATE_BOOT_PROG_OFFSET 0 #define DRV_MB_PARAM_OV_UPDATE_BOOT_PROG_MASK 0x000000FF -#define DRV_MB_PARAM_OV_UPDATE_BOOT_PROG_NONE (1 << 0) +#define DRV_MB_PARAM_OV_UPDATE_BOOT_PROG_NONE (1 << 0) +#define DRV_MB_PARAM_OV_UPDATE_BOOT_PROG_ISCSI_IP_ACQUIRED (1 << 1) +#define DRV_MB_PARAM_OV_UPDATE_BOOT_PROG_FCOE_FABRIC_LOGIN_SUCCESS (1 << 1) #define DRV_MB_PARAM_OV_UPDATE_BOOT_PROG_TRARGET_FOUND (1 << 2) +#define DRV_MB_PARAM_OV_UPDATE_BOOT_PROG_ISCSI_CHAP_SUCCESS (1 << 3) +#define DRV_MB_PARAM_OV_UPDATE_BOOT_PROG_FCOE_LUN_FOUND (1 << 3) #define DRV_MB_PARAM_OV_UPDATE_BOOT_PROG_LOGGED_INTO_TGT (1 << 4) #define DRV_MB_PARAM_OV_UPDATE_BOOT_PROG_IMG_DOWNLOADED (1 << 5) #define DRV_MB_PARAM_OV_UPDATE_BOOT_PROG_OS_HANDOFF (1 << 6) #define DRV_MB_PARAM_OV_UPDATE_BOOT_COMPLETED 0 -#define DRV_MB_PARAM_OV_PCI_BUS_NUM_SHIFT 0 +#define DRV_MB_PARAM_OV_PCI_BUS_NUM_OFFSET 0 #define DRV_MB_PARAM_OV_PCI_BUS_NUM_MASK 0x000000FF -#define DRV_MB_PARAM_OV_STORM_FW_VER_SHIFT 0 +#define DRV_MB_PARAM_OV_STORM_FW_VER_OFFSET 0 #define DRV_MB_PARAM_OV_STORM_FW_VER_MASK 0xFFFFFFFF #define DRV_MB_PARAM_OV_STORM_FW_VER_MAJOR_MASK 0xFF000000 #define DRV_MB_PARAM_OV_STORM_FW_VER_MINOR_MASK 0x00FF0000 #define DRV_MB_PARAM_OV_STORM_FW_VER_BUILD_MASK 0x0000FF00 #define DRV_MB_PARAM_OV_STORM_FW_VER_DROP_MASK 0x000000FF -#define DRV_MSG_CODE_OV_UPDATE_DRIVER_STATE_SHIFT 0 +#define DRV_MSG_CODE_OV_UPDATE_DRIVER_STATE_OFFSET 0 #define DRV_MSG_CODE_OV_UPDATE_DRIVER_STATE_MASK 0xF #define DRV_MSG_CODE_OV_UPDATE_DRIVER_STATE_UNKNOWN 0x1 +/* Not Installed*/ #define DRV_MSG_CODE_OV_UPDATE_DRIVER_STATE_NOT_LOADED 0x2 #define DRV_MSG_CODE_OV_UPDATE_DRIVER_STATE_LOADING 0x3 +/* installed but disabled by user/admin/OS */ #define DRV_MSG_CODE_OV_UPDATE_DRIVER_STATE_DISABLED 0x4 +/* installed and active */ #define DRV_MSG_CODE_OV_UPDATE_DRIVER_STATE_ACTIVE 0x5 -#define DRV_MB_PARAM_OV_MTU_SIZE_SHIFT 0 +#define DRV_MB_PARAM_OV_MTU_SIZE_OFFSET 0 #define DRV_MB_PARAM_OV_MTU_SIZE_MASK 0xFFFFFFFF #define DRV_MB_PARAM_SET_LED_MODE_OPER 0x0 #define DRV_MB_PARAM_SET_LED_MODE_ON 0x1 #define DRV_MB_PARAM_SET_LED_MODE_OFF 0x2 -#define DRV_MB_PARAM_TRANSCEIVER_PORT_SHIFT 0 +#define DRV_MB_PARAM_TRANSCEIVER_PORT_OFFSET 0 #define DRV_MB_PARAM_TRANSCEIVER_PORT_MASK 0x00000003 -#define DRV_MB_PARAM_TRANSCEIVER_SIZE_SHIFT 2 +#define DRV_MB_PARAM_TRANSCEIVER_SIZE_OFFSET 2 #define DRV_MB_PARAM_TRANSCEIVER_SIZE_MASK 0x000000FC -#define DRV_MB_PARAM_TRANSCEIVER_I2C_ADDRESS_SHIFT 8 +#define DRV_MB_PARAM_TRANSCEIVER_I2C_ADDRESS_OFFSET 8 #define DRV_MB_PARAM_TRANSCEIVER_I2C_ADDRESS_MASK 0x0000FF00 -#define DRV_MB_PARAM_TRANSCEIVER_OFFSET_SHIFT 16 +#define DRV_MB_PARAM_TRANSCEIVER_OFFSET_OFFSET 16 #define DRV_MB_PARAM_TRANSCEIVER_OFFSET_MASK 0xFFFF0000 -#define DRV_MB_PARAM_GPIO_NUMBER_SHIFT 0 +#define DRV_MB_PARAM_GPIO_NUMBER_OFFSET 0 #define DRV_MB_PARAM_GPIO_NUMBER_MASK 0x0000FFFF -#define DRV_MB_PARAM_GPIO_VALUE_SHIFT 16 +#define DRV_MB_PARAM_GPIO_VALUE_OFFSET 16 #define DRV_MB_PARAM_GPIO_VALUE_MASK 0xFFFF0000 +#define DRV_MB_PARAM_GPIO_DIRECTION_OFFSET 16 +#define DRV_MB_PARAM_GPIO_DIRECTION_MASK 0x00FF0000 +#define DRV_MB_PARAM_GPIO_CTRL_OFFSET 24 +#define DRV_MB_PARAM_GPIO_CTRL_MASK 0xFF000000 + + /* Resource Allocation params - Driver version support*/ +#define DRV_MB_PARAM_RESOURCE_ALLOC_VERSION_MAJOR_MASK 0xFFFF0000 +#define DRV_MB_PARAM_RESOURCE_ALLOC_VERSION_MAJOR_OFFSET 16 +#define DRV_MB_PARAM_RESOURCE_ALLOC_VERSION_MINOR_MASK 0x0000FFFF +#define DRV_MB_PARAM_RESOURCE_ALLOC_VERSION_MINOR_OFFSET 0 + +#define DRV_MB_PARAM_BIST_UNKNOWN_TEST 0 +#define DRV_MB_PARAM_BIST_REGISTER_TEST 1 +#define DRV_MB_PARAM_BIST_CLOCK_TEST 2 +#define DRV_MB_PARAM_BIST_NVM_TEST_NUM_IMAGES 3 +#define DRV_MB_PARAM_BIST_NVM_TEST_IMAGE_BY_INDEX 4 + +#define DRV_MB_PARAM_BIST_RC_UNKNOWN 0 +#define DRV_MB_PARAM_BIST_RC_PASSED 1 +#define DRV_MB_PARAM_BIST_RC_FAILED 2 +#define DRV_MB_PARAM_BIST_RC_INVALID_PARAMETER 3 + +#define DRV_MB_PARAM_BIST_TEST_INDEX_OFFSET 0 +#define DRV_MB_PARAM_BIST_TEST_INDEX_MASK 0x000000FF +#define DRV_MB_PARAM_BIST_TEST_IMAGE_INDEX_OFFSET 8 +#define DRV_MB_PARAM_BIST_TEST_IMAGE_INDEX_MASK 0x0000FF00 + +#define DRV_MB_PARAM_FEATURE_SUPPORT_PORT_MASK 0x0000FFFF +#define DRV_MB_PARAM_FEATURE_SUPPORT_PORT_OFFSET 0 +/* driver supports SmartLinQ parameter */ +#define DRV_MB_PARAM_FEATURE_SUPPORT_PORT_SMARTLINQ 0x00000001 +/* driver supports EEE parameter */ +#define DRV_MB_PARAM_FEATURE_SUPPORT_PORT_EEE 0x00000002 +#define DRV_MB_PARAM_FEATURE_SUPPORT_FUNC_MASK 0xFFFF0000 +#define DRV_MB_PARAM_FEATURE_SUPPORT_FUNC_OFFSET 16 +/* driver supports virtual link parameter */ +#define DRV_MB_PARAM_FEATURE_SUPPORT_FUNC_VLINK 0x00010000 + /* Driver attributes params */ +#define DRV_MB_PARAM_ATTRIBUTE_KEY_OFFSET 0 +#define DRV_MB_PARAM_ATTRIBUTE_KEY_MASK 0x00FFFFFF +#define DRV_MB_PARAM_ATTRIBUTE_CMD_OFFSET 24 +#define DRV_MB_PARAM_ATTRIBUTE_CMD_MASK 0xFF000000 u32 fw_mb_header; #define FW_MSG_CODE_MASK 0xffff0000 +#define FW_MSG_CODE_UNSUPPORTED 0x00000000 #define FW_MSG_CODE_DRV_LOAD_ENGINE 0x10100000 #define FW_MSG_CODE_DRV_LOAD_PORT 0x10110000 #define FW_MSG_CODE_DRV_LOAD_FUNCTION 0x10120000 #define FW_MSG_CODE_DRV_LOAD_REFUSED_PDA 0x10200000 -#define FW_MSG_CODE_DRV_LOAD_REFUSED_HSI 0x10210000 +#define FW_MSG_CODE_DRV_LOAD_REFUSED_HSI_1 0x10210000 #define FW_MSG_CODE_DRV_LOAD_REFUSED_DIAG 0x10220000 +#define FW_MSG_CODE_DRV_LOAD_REFUSED_HSI 0x10230000 +#define FW_MSG_CODE_DRV_LOAD_REFUSED_REQUIRES_FORCE 0x10300000 +#define FW_MSG_CODE_DRV_LOAD_REFUSED_REJECT 0x10310000 #define FW_MSG_CODE_DRV_LOAD_DONE 0x11100000 #define FW_MSG_CODE_DRV_UNLOAD_ENGINE 0x20110000 #define FW_MSG_CODE_DRV_UNLOAD_PORT 0x20120000 @@ -1009,6 +1665,7 @@ struct public_drv_mb { #define FW_MSG_CODE_LINK_RESET_DONE 0x23000000 #define FW_MSG_CODE_SET_LLDP_DONE 0x24000000 #define FW_MSG_CODE_SET_LLDP_UNSUPPORTED_AGENT 0x24010000 +#define FW_MSG_CODE_REGISTER_LLDP_TLVS_RX_DONE 0x24100000 #define FW_MSG_CODE_SET_DCBX_DONE 0x25000000 #define FW_MSG_CODE_UPDATE_CURR_CFG_DONE 0x26000000 #define FW_MSG_CODE_UPDATE_BUS_NUM_DONE 0x27000000 @@ -1017,6 +1674,10 @@ struct public_drv_mb { #define FW_MSG_CODE_UPDATE_DRIVER_STATE_DONE 0x31000000 #define FW_MSG_CODE_DRV_MSG_CODE_BW_UPDATE_DONE 0x32000000 #define FW_MSG_CODE_DRV_MSG_CODE_MTU_SIZE_DONE 0x33000000 +#define FW_MSG_CODE_RESOURCE_ALLOC_OK 0x34000000 +#define FW_MSG_CODE_RESOURCE_ALLOC_UNKNOWN 0x35000000 +#define FW_MSG_CODE_RESOURCE_ALLOC_DEPRECATED 0x36000000 +#define FW_MSG_CODE_RESOURCE_ALLOC_GEN_ERR 0x37000000 #define FW_MSG_CODE_NIG_DRAIN_DONE 0x30000000 #define FW_MSG_CODE_VF_DISABLED_DONE 0xb0000000 #define FW_MSG_CODE_DRV_CFG_VF_MSIX_DONE 0xb0010000 @@ -1045,34 +1706,81 @@ struct public_drv_mb { #define FW_MSG_CODE_NVM_FILE_READ_ONLY 0x00200000 #define FW_MSG_CODE_NVM_UNKNOWN_FILE 0x00300000 #define FW_MSG_CODE_NVM_PUT_FILE_FINISH_OK 0x00400000 +/* MFW reject "mcp reset" command if one of the drivers is up */ #define FW_MSG_CODE_MCP_RESET_REJECT 0x00600000 +#define FW_MSG_CODE_NVM_FAILED_CALC_HASH 0x00310000 +#define FW_MSG_CODE_NVM_PUBLIC_KEY_MISSING 0x00320000 +#define FW_MSG_CODE_NVM_INVALID_PUBLIC_KEY 0x00330000 + #define FW_MSG_CODE_PHY_OK 0x00110000 #define FW_MSG_CODE_PHY_ERROR 0x00120000 #define FW_MSG_CODE_SET_SECURE_MODE_ERROR 0x00130000 #define FW_MSG_CODE_SET_SECURE_MODE_OK 0x00140000 #define FW_MSG_MODE_PHY_PRIVILEGE_ERROR 0x00150000 #define FW_MSG_CODE_OK 0x00160000 +#define FW_MSG_CODE_ERROR 0x00170000 #define FW_MSG_CODE_LED_MODE_INVALID 0x00170000 -#define FW_MSG_CODE_PHY_DIAG_OK 0x00160000 -#define FW_MSG_CODE_PHY_DIAG_ERROR 0x00170000 +#define FW_MSG_CODE_PHY_DIAG_OK 0x00160000 +#define FW_MSG_CODE_PHY_DIAG_ERROR 0x00170000 #define FW_MSG_CODE_INIT_HW_FAILED_TO_ALLOCATE_PAGE 0x00040000 #define FW_MSG_CODE_INIT_HW_FAILED_BAD_STATE 0x00170000 #define FW_MSG_CODE_INIT_HW_FAILED_TO_SET_WINDOW 0x000d0000 #define FW_MSG_CODE_INIT_HW_FAILED_NO_IMAGE 0x000c0000 #define FW_MSG_CODE_INIT_HW_FAILED_VERSION_MISMATCH 0x00100000 -#define FW_MSG_CODE_TRANSCEIVER_DIAG_OK 0x00160000 -#define FW_MSG_CODE_TRANSCEIVER_DIAG_ERROR 0x00170000 +#define FW_MSG_CODE_TRANSCEIVER_DIAG_OK 0x00160000 +#define FW_MSG_CODE_TRANSCEIVER_DIAG_ERROR 0x00170000 #define FW_MSG_CODE_TRANSCEIVER_NOT_PRESENT 0x00020000 #define FW_MSG_CODE_TRANSCEIVER_BAD_BUFFER_SIZE 0x000f0000 -#define FW_MSG_CODE_GPIO_OK 0x00160000 -#define FW_MSG_CODE_GPIO_DIRECTION_ERR 0x00170000 +#define FW_MSG_CODE_GPIO_OK 0x00160000 +#define FW_MSG_CODE_GPIO_DIRECTION_ERR 0x00170000 #define FW_MSG_CODE_GPIO_CTRL_ERR 0x00020000 #define FW_MSG_CODE_GPIO_INVALID 0x000f0000 -#define FW_MSG_CODE_GPIO_INVALID_VALUE 0x00050000 +#define FW_MSG_CODE_GPIO_INVALID_VALUE 0x00050000 +#define FW_MSG_CODE_BIST_TEST_INVALID 0x000f0000 +#define FW_MSG_CODE_EXTPHY_INVALID_IMAGE_HEADER 0x00700000 +#define FW_MSG_CODE_EXTPHY_INVALID_PHY_TYPE 0x00710000 +#define FW_MSG_CODE_EXTPHY_OPERATION_FAILED 0x00720000 +#define FW_MSG_CODE_EXTPHY_NO_PHY_DETECTED 0x00730000 +#define FW_MSG_CODE_RECOVERY_MODE 0x00740000 + + /* mdump related response codes */ +#define FW_MSG_CODE_MDUMP_NO_IMAGE_FOUND 0x00010000 +#define FW_MSG_CODE_MDUMP_ALLOC_FAILED 0x00020000 +#define FW_MSG_CODE_MDUMP_INVALID_CMD 0x00030000 +#define FW_MSG_CODE_MDUMP_IN_PROGRESS 0x00040000 +#define FW_MSG_CODE_MDUMP_WRITE_FAILED 0x00050000 + + +#define FW_MSG_CODE_DRV_CFG_PF_VFS_MSIX_DONE 0x00870000 +#define FW_MSG_CODE_DRV_CFG_PF_VFS_MSIX_BAD_ASIC 0x00880000 + +#define FW_MSG_CODE_WOL_READ_WRITE_OK 0x00820000 +#define FW_MSG_CODE_WOL_READ_WRITE_INVALID_VAL 0x00830000 +#define FW_MSG_CODE_WOL_READ_WRITE_INVALID_ADDR 0x00840000 +#define FW_MSG_CODE_WOL_READ_BUFFER_OK 0x00850000 +#define FW_MSG_CODE_WOL_READ_BUFFER_INVALID_VAL 0x00860000 #define FW_MSG_SEQ_NUMBER_MASK 0x0000ffff +#define FW_MSG_CODE_ATTRIBUTE_INVALID_KEY 0x00020000 +#define FW_MSG_CODE_ATTRIBUTE_INVALID_CMD 0x00030000 + u32 fw_mb_param; +/* Resource Allocation params - MFW version support */ +#define FW_MB_PARAM_RESOURCE_ALLOC_VERSION_MAJOR_MASK 0xFFFF0000 +#define FW_MB_PARAM_RESOURCE_ALLOC_VERSION_MAJOR_OFFSET 16 +#define FW_MB_PARAM_RESOURCE_ALLOC_VERSION_MINOR_MASK 0x0000FFFF +#define FW_MB_PARAM_RESOURCE_ALLOC_VERSION_MINOR_OFFSET 0 + +/* get MFW feature support response */ +/* MFW supports SmartLinQ */ +#define FW_MB_PARAM_FEATURE_SUPPORT_SMARTLINQ 0x00000001 +/* MFW supports EEE */ +#define FW_MB_PARAM_FEATURE_SUPPORT_EEE 0x00000002 +/* MFW supports virtual link */ +#define FW_MB_PARAM_FEATURE_SUPPORT_VLINK 0x00010000 + +#define FW_MB_PARAM_LOAD_DONE_DID_EFUSE_ERROR (1 << 0) u32 drv_pulse_mb; #define DRV_PULSE_SEQ_MASK 0x00007fff @@ -1097,7 +1805,10 @@ struct public_drv_mb { #define MCP_EVENT_MASK 0xffff0000 #define MCP_EVENT_OTHER_DRIVER_RESET_REQ 0x00010000 +/* The union data is used by the driver to pass parameters to the scratchpad. */ + union drv_union_data union_data; + }; /* MFW - DRV MB */ @@ -1132,6 +1843,11 @@ enum MFW_DRV_MSG_TYPE { MFW_DRV_MSG_GET_RDMA_STATS, MFW_DRV_MSG_FAILURE_DETECTED, MFW_DRV_MSG_TRANSCEIVER_STATE_CHANGE, + MFW_DRV_MSG_CRITICAL_ERROR_OCCURRED, + MFW_DRV_MSG_EEE_NEGOTIATION_COMPLETE, + MFW_DRV_MSG_GET_TLV_REQ, + MFW_DRV_MSG_OEM_CFG_UPDATE, + MFW_DRV_MSG_LLDP_RECEIVED_TLVS_UPDATED, MFW_DRV_MSG_MAX }; @@ -1152,8 +1868,10 @@ enum MFW_DRV_MSG_TYPE { ((u8)((u8 *)(MFW_MB_P(shmem_func)->msg))[msg_id]++;) struct public_mfw_mb { - u32 sup_msgs; /* Assigend with MFW_DRV_MSG_MAX */ + u32 sup_msgs; /* Assigend with MFW_DRV_MSG_MAX */ +/* Incremented by the MFW */ u32 msg[MFW_DRV_MSG_MAX_DWORDS(MFW_DRV_MSG_MAX)]; +/* Incremented by the driver */ u32 ack[MFW_DRV_MSG_MAX_DWORDS(MFW_DRV_MSG_MAX)]; }; @@ -1163,8 +1881,8 @@ struct public_mfw_mb { /* */ /**************************************/ enum public_sections { - PUBLIC_DRV_MB, /* Points to the first drv_mb of path0 */ - PUBLIC_MFW_MB, /* Points to the first mfw_mb of path0 */ + PUBLIC_DRV_MB, /* Points to the first drv_mb of path0 */ + PUBLIC_MFW_MB, /* Points to the first mfw_mb of path0 */ PUBLIC_GLOBAL, PUBLIC_PATH, PUBLIC_PORT, @@ -1202,4 +1920,4 @@ struct mcp_public_data { #define MAX_I2C_TRANSACTION_SIZE 16 #define MAX_I2C_TRANSCEIVER_PAGE_SIZE 256 -#endif /* MCP_PUBLIC_H */ +#endif /* MCP_PUBLIC_H */ diff --git a/dpdk/drivers/net/qede/base/nvm_cfg.h b/dpdk/drivers/net/qede/base/nvm_cfg.h index 7f1a60dd..c99e805d 100644 --- a/dpdk/drivers/net/qede/base/nvm_cfg.h +++ b/dpdk/drivers/net/qede/base/nvm_cfg.h @@ -13,17 +13,25 @@ * Description: NVM config file - Generated file from nvm cfg excel. * DO NOT MODIFY !!! * - * Created: 1/14/2016 + * Created: 5/8/2017 * ****************************************************************************/ #ifndef NVM_CFG_H #define NVM_CFG_H +#define NVM_CFG_version 0x83000 + +#define NVM_CFG_new_option_seq 23 + +#define NVM_CFG_removed_option_seq 1 + +#define NVM_CFG_updated_value_seq 4 + struct nvm_cfg_mac_address { u32 mac_addr_hi; -#define NVM_CFG_MAC_ADDRESS_HI_MASK 0x0000FFFF -#define NVM_CFG_MAC_ADDRESS_HI_OFFSET 0 + #define NVM_CFG_MAC_ADDRESS_HI_MASK 0x0000FFFF + #define NVM_CFG_MAC_ADDRESS_HI_OFFSET 0 u32 mac_addr_lo; }; @@ -31,875 +39,1933 @@ struct nvm_cfg_mac_address { * nvm_cfg1 structs ******************************************/ struct nvm_cfg1_glob { - u32 generic_cont0; /* 0x0 */ -#define NVM_CFG1_GLOB_BOARD_SWAP_MASK 0x0000000F -#define NVM_CFG1_GLOB_BOARD_SWAP_OFFSET 0 -#define NVM_CFG1_GLOB_BOARD_SWAP_NONE 0x0 -#define NVM_CFG1_GLOB_BOARD_SWAP_PATH 0x1 -#define NVM_CFG1_GLOB_BOARD_SWAP_PORT 0x2 -#define NVM_CFG1_GLOB_BOARD_SWAP_BOTH 0x3 -#define NVM_CFG1_GLOB_MF_MODE_MASK 0x00000FF0 -#define NVM_CFG1_GLOB_MF_MODE_OFFSET 4 -#define NVM_CFG1_GLOB_MF_MODE_MF_ALLOWED 0x0 -#define NVM_CFG1_GLOB_MF_MODE_DEFAULT 0x1 -#define NVM_CFG1_GLOB_MF_MODE_SPIO4 0x2 -#define NVM_CFG1_GLOB_MF_MODE_NPAR1_0 0x3 -#define NVM_CFG1_GLOB_MF_MODE_NPAR1_5 0x4 -#define NVM_CFG1_GLOB_MF_MODE_NPAR2_0 0x5 -#define NVM_CFG1_GLOB_MF_MODE_BD 0x6 -#define NVM_CFG1_GLOB_MF_MODE_UFP 0x7 -#define NVM_CFG1_GLOB_FAN_FAILURE_ENFORCEMENT_MASK 0x00001000 -#define NVM_CFG1_GLOB_FAN_FAILURE_ENFORCEMENT_OFFSET 12 -#define NVM_CFG1_GLOB_FAN_FAILURE_ENFORCEMENT_DISABLED 0x0 -#define NVM_CFG1_GLOB_FAN_FAILURE_ENFORCEMENT_ENABLED 0x1 -#define NVM_CFG1_GLOB_AVS_MARGIN_LOW_MASK 0x001FE000 -#define NVM_CFG1_GLOB_AVS_MARGIN_LOW_OFFSET 13 -#define NVM_CFG1_GLOB_AVS_MARGIN_HIGH_MASK 0x1FE00000 -#define NVM_CFG1_GLOB_AVS_MARGIN_HIGH_OFFSET 21 -#define NVM_CFG1_GLOB_ENABLE_SRIOV_MASK 0x20000000 -#define NVM_CFG1_GLOB_ENABLE_SRIOV_OFFSET 29 -#define NVM_CFG1_GLOB_ENABLE_SRIOV_DISABLED 0x0 -#define NVM_CFG1_GLOB_ENABLE_SRIOV_ENABLED 0x1 -#define NVM_CFG1_GLOB_ENABLE_ATC_MASK 0x40000000 -#define NVM_CFG1_GLOB_ENABLE_ATC_OFFSET 30 -#define NVM_CFG1_GLOB_ENABLE_ATC_DISABLED 0x0 -#define NVM_CFG1_GLOB_ENABLE_ATC_ENABLED 0x1 -#define NVM_CFG1_GLOB_CLOCK_SLOWDOWN_MASK 0x80000000 -#define NVM_CFG1_GLOB_CLOCK_SLOWDOWN_OFFSET 31 -#define NVM_CFG1_GLOB_CLOCK_SLOWDOWN_DISABLED 0x0 -#define NVM_CFG1_GLOB_CLOCK_SLOWDOWN_ENABLED 0x1 - u32 engineering_change[3]; /* 0x4 */ - u32 manufacturing_id; /* 0x10 */ - u32 serial_number[4]; /* 0x14 */ - u32 pcie_cfg; /* 0x24 */ -#define NVM_CFG1_GLOB_PCI_GEN_MASK 0x00000003 -#define NVM_CFG1_GLOB_PCI_GEN_OFFSET 0 -#define NVM_CFG1_GLOB_PCI_GEN_PCI_GEN1 0x0 -#define NVM_CFG1_GLOB_PCI_GEN_PCI_GEN2 0x1 -#define NVM_CFG1_GLOB_PCI_GEN_PCI_GEN3 0x2 -#define NVM_CFG1_GLOB_BEACON_WOL_ENABLED_MASK 0x00000004 -#define NVM_CFG1_GLOB_BEACON_WOL_ENABLED_OFFSET 2 -#define NVM_CFG1_GLOB_BEACON_WOL_ENABLED_DISABLED 0x0 -#define NVM_CFG1_GLOB_BEACON_WOL_ENABLED_ENABLED 0x1 -#define NVM_CFG1_GLOB_ASPM_SUPPORT_MASK 0x00000018 -#define NVM_CFG1_GLOB_ASPM_SUPPORT_OFFSET 3 -#define NVM_CFG1_GLOB_ASPM_SUPPORT_L0S_L1_ENABLED 0x0 -#define NVM_CFG1_GLOB_ASPM_SUPPORT_L1_DISABLED 0x2 -#define NVM_CFG1_GLOB_RESERVED_MPREVENT_PCIE_L1_MENTRY_MASK 0x00000020 -#define NVM_CFG1_GLOB_RESERVED_MPREVENT_PCIE_L1_MENTRY_OFFSET 5 -#define NVM_CFG1_GLOB_PCIE_G2_TX_AMPLITUDE_MASK 0x000003C0 -#define NVM_CFG1_GLOB_PCIE_G2_TX_AMPLITUDE_OFFSET 6 -#define NVM_CFG1_GLOB_PCIE_PREEMPHASIS_MASK 0x00001C00 -#define NVM_CFG1_GLOB_PCIE_PREEMPHASIS_OFFSET 10 -#define NVM_CFG1_GLOB_PCIE_PREEMPHASIS_HW 0x0 -#define NVM_CFG1_GLOB_PCIE_PREEMPHASIS_0DB 0x1 -#define NVM_CFG1_GLOB_PCIE_PREEMPHASIS_3_5DB 0x2 -#define NVM_CFG1_GLOB_PCIE_PREEMPHASIS_6_0DB 0x3 -#define NVM_CFG1_GLOB_WWN_NODE_PREFIX0_MASK 0x001FE000 -#define NVM_CFG1_GLOB_WWN_NODE_PREFIX0_OFFSET 13 -#define NVM_CFG1_GLOB_WWN_NODE_PREFIX1_MASK 0x1FE00000 -#define NVM_CFG1_GLOB_WWN_NODE_PREFIX1_OFFSET 21 -#define NVM_CFG1_GLOB_NCSI_PACKAGE_ID_MASK 0x60000000 -#define NVM_CFG1_GLOB_NCSI_PACKAGE_ID_OFFSET 29 - /* Set the duration, in seconds, fan failure signal should be - * sampled - */ -#define NVM_CFG1_GLOB_RESERVED_FAN_FAILURE_DURATION_MASK 0x80000000 -#define NVM_CFG1_GLOB_RESERVED_FAN_FAILURE_DURATION_OFFSET 31 - u32 mgmt_traffic; /* 0x28 */ -#define NVM_CFG1_GLOB_RESERVED60_MASK 0x00000001 -#define NVM_CFG1_GLOB_RESERVED60_OFFSET 0 -#define NVM_CFG1_GLOB_WWN_PORT_PREFIX0_MASK 0x000001FE -#define NVM_CFG1_GLOB_WWN_PORT_PREFIX0_OFFSET 1 -#define NVM_CFG1_GLOB_WWN_PORT_PREFIX1_MASK 0x0001FE00 -#define NVM_CFG1_GLOB_WWN_PORT_PREFIX1_OFFSET 9 -#define NVM_CFG1_GLOB_SMBUS_ADDRESS_MASK 0x01FE0000 -#define NVM_CFG1_GLOB_SMBUS_ADDRESS_OFFSET 17 -#define NVM_CFG1_GLOB_SIDEBAND_MODE_MASK 0x06000000 -#define NVM_CFG1_GLOB_SIDEBAND_MODE_OFFSET 25 -#define NVM_CFG1_GLOB_SIDEBAND_MODE_DISABLED 0x0 -#define NVM_CFG1_GLOB_SIDEBAND_MODE_RMII 0x1 -#define NVM_CFG1_GLOB_SIDEBAND_MODE_SGMII 0x2 -#define NVM_CFG1_GLOB_AUX_MODE_MASK 0x78000000 -#define NVM_CFG1_GLOB_AUX_MODE_OFFSET 27 -#define NVM_CFG1_GLOB_AUX_MODE_DEFAULT 0x0 -#define NVM_CFG1_GLOB_AUX_MODE_SMBUS_ONLY 0x1 + u32 generic_cont0; /* 0x0 */ + #define NVM_CFG1_GLOB_BOARD_SWAP_MASK 0x0000000F + #define NVM_CFG1_GLOB_BOARD_SWAP_OFFSET 0 + #define NVM_CFG1_GLOB_BOARD_SWAP_NONE 0x0 + #define NVM_CFG1_GLOB_BOARD_SWAP_PATH 0x1 + #define NVM_CFG1_GLOB_BOARD_SWAP_PORT 0x2 + #define NVM_CFG1_GLOB_BOARD_SWAP_BOTH 0x3 + #define NVM_CFG1_GLOB_MF_MODE_MASK 0x00000FF0 + #define NVM_CFG1_GLOB_MF_MODE_OFFSET 4 + #define NVM_CFG1_GLOB_MF_MODE_MF_ALLOWED 0x0 + #define NVM_CFG1_GLOB_MF_MODE_DEFAULT 0x1 + #define NVM_CFG1_GLOB_MF_MODE_SPIO4 0x2 + #define NVM_CFG1_GLOB_MF_MODE_NPAR1_0 0x3 + #define NVM_CFG1_GLOB_MF_MODE_NPAR1_5 0x4 + #define NVM_CFG1_GLOB_MF_MODE_NPAR2_0 0x5 + #define NVM_CFG1_GLOB_MF_MODE_BD 0x6 + #define NVM_CFG1_GLOB_MF_MODE_UFP 0x7 + #define NVM_CFG1_GLOB_FAN_FAILURE_ENFORCEMENT_MASK 0x00001000 + #define NVM_CFG1_GLOB_FAN_FAILURE_ENFORCEMENT_OFFSET 12 + #define NVM_CFG1_GLOB_FAN_FAILURE_ENFORCEMENT_DISABLED 0x0 + #define NVM_CFG1_GLOB_FAN_FAILURE_ENFORCEMENT_ENABLED 0x1 + #define NVM_CFG1_GLOB_AVS_MARGIN_LOW_MASK 0x001FE000 + #define NVM_CFG1_GLOB_AVS_MARGIN_LOW_OFFSET 13 + #define NVM_CFG1_GLOB_AVS_MARGIN_HIGH_MASK 0x1FE00000 + #define NVM_CFG1_GLOB_AVS_MARGIN_HIGH_OFFSET 21 + #define NVM_CFG1_GLOB_ENABLE_SRIOV_MASK 0x20000000 + #define NVM_CFG1_GLOB_ENABLE_SRIOV_OFFSET 29 + #define NVM_CFG1_GLOB_ENABLE_SRIOV_DISABLED 0x0 + #define NVM_CFG1_GLOB_ENABLE_SRIOV_ENABLED 0x1 + #define NVM_CFG1_GLOB_ENABLE_ATC_MASK 0x40000000 + #define NVM_CFG1_GLOB_ENABLE_ATC_OFFSET 30 + #define NVM_CFG1_GLOB_ENABLE_ATC_DISABLED 0x0 + #define NVM_CFG1_GLOB_ENABLE_ATC_ENABLED 0x1 + #define NVM_CFG1_GLOB_RESERVED__M_WAS_CLOCK_SLOWDOWN_MASK \ + 0x80000000 + #define NVM_CFG1_GLOB_RESERVED__M_WAS_CLOCK_SLOWDOWN_OFFSET 31 + #define NVM_CFG1_GLOB_RESERVED__M_WAS_CLOCK_SLOWDOWN_DISABLED \ + 0x0 + #define NVM_CFG1_GLOB_RESERVED__M_WAS_CLOCK_SLOWDOWN_ENABLED 0x1 + u32 engineering_change[3]; /* 0x4 */ + u32 manufacturing_id; /* 0x10 */ + u32 serial_number[4]; /* 0x14 */ + u32 pcie_cfg; /* 0x24 */ + #define NVM_CFG1_GLOB_PCI_GEN_MASK 0x00000003 + #define NVM_CFG1_GLOB_PCI_GEN_OFFSET 0 + #define NVM_CFG1_GLOB_PCI_GEN_PCI_GEN1 0x0 + #define NVM_CFG1_GLOB_PCI_GEN_PCI_GEN2 0x1 + #define NVM_CFG1_GLOB_PCI_GEN_PCI_GEN3 0x2 + #define NVM_CFG1_GLOB_BEACON_WOL_ENABLED_MASK 0x00000004 + #define NVM_CFG1_GLOB_BEACON_WOL_ENABLED_OFFSET 2 + #define NVM_CFG1_GLOB_BEACON_WOL_ENABLED_DISABLED 0x0 + #define NVM_CFG1_GLOB_BEACON_WOL_ENABLED_ENABLED 0x1 + #define NVM_CFG1_GLOB_ASPM_SUPPORT_MASK 0x00000018 + #define NVM_CFG1_GLOB_ASPM_SUPPORT_OFFSET 3 + #define NVM_CFG1_GLOB_ASPM_SUPPORT_L0S_L1_ENABLED 0x0 + #define NVM_CFG1_GLOB_ASPM_SUPPORT_L0S_DISABLED 0x1 + #define NVM_CFG1_GLOB_ASPM_SUPPORT_L1_DISABLED 0x2 + #define NVM_CFG1_GLOB_ASPM_SUPPORT_L0S_L1_DISABLED 0x3 + #define NVM_CFG1_GLOB_RESERVED_MPREVENT_PCIE_L1_MENTRY_MASK \ + 0x00000020 + #define NVM_CFG1_GLOB_RESERVED_MPREVENT_PCIE_L1_MENTRY_OFFSET 5 + #define NVM_CFG1_GLOB_PCIE_G2_TX_AMPLITUDE_MASK 0x000003C0 + #define NVM_CFG1_GLOB_PCIE_G2_TX_AMPLITUDE_OFFSET 6 + #define NVM_CFG1_GLOB_PCIE_PREEMPHASIS_MASK 0x00001C00 + #define NVM_CFG1_GLOB_PCIE_PREEMPHASIS_OFFSET 10 + #define NVM_CFG1_GLOB_PCIE_PREEMPHASIS_HW 0x0 + #define NVM_CFG1_GLOB_PCIE_PREEMPHASIS_0DB 0x1 + #define NVM_CFG1_GLOB_PCIE_PREEMPHASIS_3_5DB 0x2 + #define NVM_CFG1_GLOB_PCIE_PREEMPHASIS_6_0DB 0x3 + #define NVM_CFG1_GLOB_WWN_NODE_PREFIX0_MASK 0x001FE000 + #define NVM_CFG1_GLOB_WWN_NODE_PREFIX0_OFFSET 13 + #define NVM_CFG1_GLOB_WWN_NODE_PREFIX1_MASK 0x1FE00000 + #define NVM_CFG1_GLOB_WWN_NODE_PREFIX1_OFFSET 21 + #define NVM_CFG1_GLOB_NCSI_PACKAGE_ID_MASK 0x60000000 + #define NVM_CFG1_GLOB_NCSI_PACKAGE_ID_OFFSET 29 + /* Set the duration, in sec, fan failure signal should be sampled */ + #define NVM_CFG1_GLOB_RESERVED_FAN_FAILURE_DURATION_MASK \ + 0x80000000 + #define NVM_CFG1_GLOB_RESERVED_FAN_FAILURE_DURATION_OFFSET 31 + u32 mgmt_traffic; /* 0x28 */ + #define NVM_CFG1_GLOB_RESERVED60_MASK 0x00000001 + #define NVM_CFG1_GLOB_RESERVED60_OFFSET 0 + #define NVM_CFG1_GLOB_WWN_PORT_PREFIX0_MASK 0x000001FE + #define NVM_CFG1_GLOB_WWN_PORT_PREFIX0_OFFSET 1 + #define NVM_CFG1_GLOB_WWN_PORT_PREFIX1_MASK 0x0001FE00 + #define NVM_CFG1_GLOB_WWN_PORT_PREFIX1_OFFSET 9 + #define NVM_CFG1_GLOB_SMBUS_ADDRESS_MASK 0x01FE0000 + #define NVM_CFG1_GLOB_SMBUS_ADDRESS_OFFSET 17 + #define NVM_CFG1_GLOB_SIDEBAND_MODE_MASK 0x06000000 + #define NVM_CFG1_GLOB_SIDEBAND_MODE_OFFSET 25 + #define NVM_CFG1_GLOB_SIDEBAND_MODE_DISABLED 0x0 + #define NVM_CFG1_GLOB_SIDEBAND_MODE_RMII 0x1 + #define NVM_CFG1_GLOB_SIDEBAND_MODE_SGMII 0x2 + #define NVM_CFG1_GLOB_AUX_MODE_MASK 0x78000000 + #define NVM_CFG1_GLOB_AUX_MODE_OFFSET 27 + #define NVM_CFG1_GLOB_AUX_MODE_DEFAULT 0x0 + #define NVM_CFG1_GLOB_AUX_MODE_SMBUS_ONLY 0x1 /* Indicates whether external thermal sonsor is available */ -#define NVM_CFG1_GLOB_EXTERNAL_THERMAL_SENSOR_MASK 0x80000000 -#define NVM_CFG1_GLOB_EXTERNAL_THERMAL_SENSOR_OFFSET 31 -#define NVM_CFG1_GLOB_EXTERNAL_THERMAL_SENSOR_DISABLED 0x0 -#define NVM_CFG1_GLOB_EXTERNAL_THERMAL_SENSOR_ENABLED 0x1 - u32 core_cfg; /* 0x2C */ -#define NVM_CFG1_GLOB_NETWORK_PORT_MODE_MASK 0x000000FF -#define NVM_CFG1_GLOB_NETWORK_PORT_MODE_OFFSET 0 -#define NVM_CFG1_GLOB_NETWORK_PORT_MODE_DE_2X40G 0x0 -#define NVM_CFG1_GLOB_NETWORK_PORT_MODE_DE_2X50G 0x1 -#define NVM_CFG1_GLOB_NETWORK_PORT_MODE_DE_1X100G 0x2 -#define NVM_CFG1_GLOB_NETWORK_PORT_MODE_DE_4X10G_F 0x3 -#define NVM_CFG1_GLOB_NETWORK_PORT_MODE_DE_4X10G_E 0x4 -#define NVM_CFG1_GLOB_NETWORK_PORT_MODE_DE_4X20G 0x5 -#define NVM_CFG1_GLOB_NETWORK_PORT_MODE_DE_1X40G 0xB -#define NVM_CFG1_GLOB_NETWORK_PORT_MODE_DE_2X25G 0xC -#define NVM_CFG1_GLOB_NETWORK_PORT_MODE_DE_1X25G 0xD -#define NVM_CFG1_GLOB_EAGLE_ENFORCE_TX_FIR_CFG_MASK 0x00000100 -#define NVM_CFG1_GLOB_EAGLE_ENFORCE_TX_FIR_CFG_OFFSET 8 -#define NVM_CFG1_GLOB_EAGLE_ENFORCE_TX_FIR_CFG_DISABLED 0x0 -#define NVM_CFG1_GLOB_EAGLE_ENFORCE_TX_FIR_CFG_ENABLED 0x1 -#define NVM_CFG1_GLOB_FALCON_ENFORCE_TX_FIR_CFG_MASK 0x00000200 -#define NVM_CFG1_GLOB_FALCON_ENFORCE_TX_FIR_CFG_OFFSET 9 -#define NVM_CFG1_GLOB_FALCON_ENFORCE_TX_FIR_CFG_DISABLED 0x0 -#define NVM_CFG1_GLOB_FALCON_ENFORCE_TX_FIR_CFG_ENABLED 0x1 -#define NVM_CFG1_GLOB_EAGLE_CORE_ADDR_MASK 0x0003FC00 -#define NVM_CFG1_GLOB_EAGLE_CORE_ADDR_OFFSET 10 -#define NVM_CFG1_GLOB_FALCON_CORE_ADDR_MASK 0x03FC0000 -#define NVM_CFG1_GLOB_FALCON_CORE_ADDR_OFFSET 18 -#define NVM_CFG1_GLOB_AVS_MODE_MASK 0x1C000000 -#define NVM_CFG1_GLOB_AVS_MODE_OFFSET 26 -#define NVM_CFG1_GLOB_AVS_MODE_CLOSE_LOOP 0x0 -#define NVM_CFG1_GLOB_AVS_MODE_OPEN_LOOP_CFG 0x1 -#define NVM_CFG1_GLOB_AVS_MODE_OPEN_LOOP_OTP 0x2 -#define NVM_CFG1_GLOB_AVS_MODE_DISABLED 0x3 -#define NVM_CFG1_GLOB_OVERRIDE_SECURE_MODE_MASK 0x60000000 -#define NVM_CFG1_GLOB_OVERRIDE_SECURE_MODE_OFFSET 29 -#define NVM_CFG1_GLOB_OVERRIDE_SECURE_MODE_DISABLED 0x0 -#define NVM_CFG1_GLOB_OVERRIDE_SECURE_MODE_ENABLED 0x1 - u32 e_lane_cfg1; /* 0x30 */ -#define NVM_CFG1_GLOB_RX_LANE0_SWAP_MASK 0x0000000F -#define NVM_CFG1_GLOB_RX_LANE0_SWAP_OFFSET 0 -#define NVM_CFG1_GLOB_RX_LANE1_SWAP_MASK 0x000000F0 -#define NVM_CFG1_GLOB_RX_LANE1_SWAP_OFFSET 4 -#define NVM_CFG1_GLOB_RX_LANE2_SWAP_MASK 0x00000F00 -#define NVM_CFG1_GLOB_RX_LANE2_SWAP_OFFSET 8 -#define NVM_CFG1_GLOB_RX_LANE3_SWAP_MASK 0x0000F000 -#define NVM_CFG1_GLOB_RX_LANE3_SWAP_OFFSET 12 -#define NVM_CFG1_GLOB_TX_LANE0_SWAP_MASK 0x000F0000 -#define NVM_CFG1_GLOB_TX_LANE0_SWAP_OFFSET 16 -#define NVM_CFG1_GLOB_TX_LANE1_SWAP_MASK 0x00F00000 -#define NVM_CFG1_GLOB_TX_LANE1_SWAP_OFFSET 20 -#define NVM_CFG1_GLOB_TX_LANE2_SWAP_MASK 0x0F000000 -#define NVM_CFG1_GLOB_TX_LANE2_SWAP_OFFSET 24 -#define NVM_CFG1_GLOB_TX_LANE3_SWAP_MASK 0xF0000000 -#define NVM_CFG1_GLOB_TX_LANE3_SWAP_OFFSET 28 - u32 e_lane_cfg2; /* 0x34 */ -#define NVM_CFG1_GLOB_RX_LANE0_POL_FLIP_MASK 0x00000001 -#define NVM_CFG1_GLOB_RX_LANE0_POL_FLIP_OFFSET 0 -#define NVM_CFG1_GLOB_RX_LANE1_POL_FLIP_MASK 0x00000002 -#define NVM_CFG1_GLOB_RX_LANE1_POL_FLIP_OFFSET 1 -#define NVM_CFG1_GLOB_RX_LANE2_POL_FLIP_MASK 0x00000004 -#define NVM_CFG1_GLOB_RX_LANE2_POL_FLIP_OFFSET 2 -#define NVM_CFG1_GLOB_RX_LANE3_POL_FLIP_MASK 0x00000008 -#define NVM_CFG1_GLOB_RX_LANE3_POL_FLIP_OFFSET 3 -#define NVM_CFG1_GLOB_TX_LANE0_POL_FLIP_MASK 0x00000010 -#define NVM_CFG1_GLOB_TX_LANE0_POL_FLIP_OFFSET 4 -#define NVM_CFG1_GLOB_TX_LANE1_POL_FLIP_MASK 0x00000020 -#define NVM_CFG1_GLOB_TX_LANE1_POL_FLIP_OFFSET 5 -#define NVM_CFG1_GLOB_TX_LANE2_POL_FLIP_MASK 0x00000040 -#define NVM_CFG1_GLOB_TX_LANE2_POL_FLIP_OFFSET 6 -#define NVM_CFG1_GLOB_TX_LANE3_POL_FLIP_MASK 0x00000080 -#define NVM_CFG1_GLOB_TX_LANE3_POL_FLIP_OFFSET 7 -#define NVM_CFG1_GLOB_SMBUS_MODE_MASK 0x00000F00 -#define NVM_CFG1_GLOB_SMBUS_MODE_OFFSET 8 -#define NVM_CFG1_GLOB_SMBUS_MODE_DISABLED 0x0 -#define NVM_CFG1_GLOB_SMBUS_MODE_100KHZ 0x1 -#define NVM_CFG1_GLOB_SMBUS_MODE_400KHZ 0x2 -#define NVM_CFG1_GLOB_NCSI_MASK 0x0000F000 -#define NVM_CFG1_GLOB_NCSI_OFFSET 12 -#define NVM_CFG1_GLOB_NCSI_DISABLED 0x0 -#define NVM_CFG1_GLOB_NCSI_ENABLED 0x1 + #define NVM_CFG1_GLOB_EXTERNAL_THERMAL_SENSOR_MASK 0x80000000 + #define NVM_CFG1_GLOB_EXTERNAL_THERMAL_SENSOR_OFFSET 31 + #define NVM_CFG1_GLOB_EXTERNAL_THERMAL_SENSOR_DISABLED 0x0 + #define NVM_CFG1_GLOB_EXTERNAL_THERMAL_SENSOR_ENABLED 0x1 + u32 core_cfg; /* 0x2C */ + #define NVM_CFG1_GLOB_NETWORK_PORT_MODE_MASK 0x000000FF + #define NVM_CFG1_GLOB_NETWORK_PORT_MODE_OFFSET 0 + #define NVM_CFG1_GLOB_NETWORK_PORT_MODE_BB_2X40G 0x0 + #define NVM_CFG1_GLOB_NETWORK_PORT_MODE_2X50G 0x1 + #define NVM_CFG1_GLOB_NETWORK_PORT_MODE_BB_1X100G 0x2 + #define NVM_CFG1_GLOB_NETWORK_PORT_MODE_4X10G_F 0x3 + #define NVM_CFG1_GLOB_NETWORK_PORT_MODE_BB_4X10G_E 0x4 + #define NVM_CFG1_GLOB_NETWORK_PORT_MODE_BB_4X20G 0x5 + #define NVM_CFG1_GLOB_NETWORK_PORT_MODE_1X40G 0xB + #define NVM_CFG1_GLOB_NETWORK_PORT_MODE_2X25G 0xC + #define NVM_CFG1_GLOB_NETWORK_PORT_MODE_1X25G 0xD + #define NVM_CFG1_GLOB_NETWORK_PORT_MODE_4X25G 0xE + #define NVM_CFG1_GLOB_NETWORK_PORT_MODE_2X10G 0xF + #define NVM_CFG1_GLOB_MPS10_ENFORCE_TX_FIR_CFG_MASK 0x00000100 + #define NVM_CFG1_GLOB_MPS10_ENFORCE_TX_FIR_CFG_OFFSET 8 + #define NVM_CFG1_GLOB_MPS10_ENFORCE_TX_FIR_CFG_DISABLED 0x0 + #define NVM_CFG1_GLOB_MPS10_ENFORCE_TX_FIR_CFG_ENABLED 0x1 + #define NVM_CFG1_GLOB_MPS25_ENFORCE_TX_FIR_CFG_MASK 0x00000200 + #define NVM_CFG1_GLOB_MPS25_ENFORCE_TX_FIR_CFG_OFFSET 9 + #define NVM_CFG1_GLOB_MPS25_ENFORCE_TX_FIR_CFG_DISABLED 0x0 + #define NVM_CFG1_GLOB_MPS25_ENFORCE_TX_FIR_CFG_ENABLED 0x1 + #define NVM_CFG1_GLOB_MPS10_CORE_ADDR_MASK 0x0003FC00 + #define NVM_CFG1_GLOB_MPS10_CORE_ADDR_OFFSET 10 + #define NVM_CFG1_GLOB_MPS25_CORE_ADDR_MASK 0x03FC0000 + #define NVM_CFG1_GLOB_MPS25_CORE_ADDR_OFFSET 18 + #define NVM_CFG1_GLOB_AVS_MODE_MASK 0x1C000000 + #define NVM_CFG1_GLOB_AVS_MODE_OFFSET 26 + #define NVM_CFG1_GLOB_AVS_MODE_CLOSE_LOOP 0x0 + #define NVM_CFG1_GLOB_AVS_MODE_OPEN_LOOP_CFG 0x1 + #define NVM_CFG1_GLOB_AVS_MODE_OPEN_LOOP_OTP 0x2 + #define NVM_CFG1_GLOB_AVS_MODE_DISABLED 0x3 + #define NVM_CFG1_GLOB_OVERRIDE_SECURE_MODE_MASK 0x60000000 + #define NVM_CFG1_GLOB_OVERRIDE_SECURE_MODE_OFFSET 29 + #define NVM_CFG1_GLOB_OVERRIDE_SECURE_MODE_DISABLED 0x0 + #define NVM_CFG1_GLOB_OVERRIDE_SECURE_MODE_ENABLED 0x1 + u32 e_lane_cfg1; /* 0x30 */ + #define NVM_CFG1_GLOB_RX_LANE0_SWAP_MASK 0x0000000F + #define NVM_CFG1_GLOB_RX_LANE0_SWAP_OFFSET 0 + #define NVM_CFG1_GLOB_RX_LANE1_SWAP_MASK 0x000000F0 + #define NVM_CFG1_GLOB_RX_LANE1_SWAP_OFFSET 4 + #define NVM_CFG1_GLOB_RX_LANE2_SWAP_MASK 0x00000F00 + #define NVM_CFG1_GLOB_RX_LANE2_SWAP_OFFSET 8 + #define NVM_CFG1_GLOB_RX_LANE3_SWAP_MASK 0x0000F000 + #define NVM_CFG1_GLOB_RX_LANE3_SWAP_OFFSET 12 + #define NVM_CFG1_GLOB_TX_LANE0_SWAP_MASK 0x000F0000 + #define NVM_CFG1_GLOB_TX_LANE0_SWAP_OFFSET 16 + #define NVM_CFG1_GLOB_TX_LANE1_SWAP_MASK 0x00F00000 + #define NVM_CFG1_GLOB_TX_LANE1_SWAP_OFFSET 20 + #define NVM_CFG1_GLOB_TX_LANE2_SWAP_MASK 0x0F000000 + #define NVM_CFG1_GLOB_TX_LANE2_SWAP_OFFSET 24 + #define NVM_CFG1_GLOB_TX_LANE3_SWAP_MASK 0xF0000000 + #define NVM_CFG1_GLOB_TX_LANE3_SWAP_OFFSET 28 + u32 e_lane_cfg2; /* 0x34 */ + #define NVM_CFG1_GLOB_RX_LANE0_POL_FLIP_MASK 0x00000001 + #define NVM_CFG1_GLOB_RX_LANE0_POL_FLIP_OFFSET 0 + #define NVM_CFG1_GLOB_RX_LANE1_POL_FLIP_MASK 0x00000002 + #define NVM_CFG1_GLOB_RX_LANE1_POL_FLIP_OFFSET 1 + #define NVM_CFG1_GLOB_RX_LANE2_POL_FLIP_MASK 0x00000004 + #define NVM_CFG1_GLOB_RX_LANE2_POL_FLIP_OFFSET 2 + #define NVM_CFG1_GLOB_RX_LANE3_POL_FLIP_MASK 0x00000008 + #define NVM_CFG1_GLOB_RX_LANE3_POL_FLIP_OFFSET 3 + #define NVM_CFG1_GLOB_TX_LANE0_POL_FLIP_MASK 0x00000010 + #define NVM_CFG1_GLOB_TX_LANE0_POL_FLIP_OFFSET 4 + #define NVM_CFG1_GLOB_TX_LANE1_POL_FLIP_MASK 0x00000020 + #define NVM_CFG1_GLOB_TX_LANE1_POL_FLIP_OFFSET 5 + #define NVM_CFG1_GLOB_TX_LANE2_POL_FLIP_MASK 0x00000040 + #define NVM_CFG1_GLOB_TX_LANE2_POL_FLIP_OFFSET 6 + #define NVM_CFG1_GLOB_TX_LANE3_POL_FLIP_MASK 0x00000080 + #define NVM_CFG1_GLOB_TX_LANE3_POL_FLIP_OFFSET 7 + #define NVM_CFG1_GLOB_SMBUS_MODE_MASK 0x00000F00 + #define NVM_CFG1_GLOB_SMBUS_MODE_OFFSET 8 + #define NVM_CFG1_GLOB_SMBUS_MODE_DISABLED 0x0 + #define NVM_CFG1_GLOB_SMBUS_MODE_100KHZ 0x1 + #define NVM_CFG1_GLOB_SMBUS_MODE_400KHZ 0x2 + #define NVM_CFG1_GLOB_NCSI_MASK 0x0000F000 + #define NVM_CFG1_GLOB_NCSI_OFFSET 12 + #define NVM_CFG1_GLOB_NCSI_DISABLED 0x0 + #define NVM_CFG1_GLOB_NCSI_ENABLED 0x1 /* Maximum advertised pcie link width */ -#define NVM_CFG1_GLOB_MAX_LINK_WIDTH_MASK 0x000F0000 -#define NVM_CFG1_GLOB_MAX_LINK_WIDTH_OFFSET 16 -#define NVM_CFG1_GLOB_MAX_LINK_WIDTH_16_LANES 0x0 -#define NVM_CFG1_GLOB_MAX_LINK_WIDTH_1_LANE 0x1 -#define NVM_CFG1_GLOB_MAX_LINK_WIDTH_2_LANES 0x2 -#define NVM_CFG1_GLOB_MAX_LINK_WIDTH_4_LANES 0x3 -#define NVM_CFG1_GLOB_MAX_LINK_WIDTH_8_LANES 0x4 + #define NVM_CFG1_GLOB_MAX_LINK_WIDTH_MASK 0x000F0000 + #define NVM_CFG1_GLOB_MAX_LINK_WIDTH_OFFSET 16 + #define NVM_CFG1_GLOB_MAX_LINK_WIDTH_BB_16_LANES 0x0 + #define NVM_CFG1_GLOB_MAX_LINK_WIDTH_1_LANE 0x1 + #define NVM_CFG1_GLOB_MAX_LINK_WIDTH_2_LANES 0x2 + #define NVM_CFG1_GLOB_MAX_LINK_WIDTH_4_LANES 0x3 + #define NVM_CFG1_GLOB_MAX_LINK_WIDTH_8_LANES 0x4 /* ASPM L1 mode */ -#define NVM_CFG1_GLOB_ASPM_L1_MODE_MASK 0x00300000 -#define NVM_CFG1_GLOB_ASPM_L1_MODE_OFFSET 20 -#define NVM_CFG1_GLOB_ASPM_L1_MODE_FORCED 0x0 -#define NVM_CFG1_GLOB_ASPM_L1_MODE_DYNAMIC_LOW_LATENCY 0x1 -#define NVM_CFG1_GLOB_ON_CHIP_SENSOR_MODE_MASK 0x01C00000 -#define NVM_CFG1_GLOB_ON_CHIP_SENSOR_MODE_OFFSET 22 -#define NVM_CFG1_GLOB_ON_CHIP_SENSOR_MODE_DISABLED 0x0 -#define NVM_CFG1_GLOB_ON_CHIP_SENSOR_MODE_INT_EXT_I2C 0x1 -#define NVM_CFG1_GLOB_ON_CHIP_SENSOR_MODE_INT_ONLY 0x2 -#define NVM_CFG1_GLOB_ON_CHIP_SENSOR_MODE_INT_EXT_SMBUS 0x3 -#define NVM_CFG1_GLOB_TEMPERATURE_MONITORING_MODE_MASK 0x06000000 -#define NVM_CFG1_GLOB_TEMPERATURE_MONITORING_MODE_OFFSET 25 -#define NVM_CFG1_GLOB_TEMPERATURE_MONITORING_MODE_DISABLE 0x0 -#define NVM_CFG1_GLOB_TEMPERATURE_MONITORING_MODE_INTERNAL 0x1 -#define NVM_CFG1_GLOB_TEMPERATURE_MONITORING_MODE_EXTERNAL 0x2 -#define NVM_CFG1_GLOB_TEMPERATURE_MONITORING_MODE_BOTH 0x3 + #define NVM_CFG1_GLOB_ASPM_L1_MODE_MASK 0x00300000 + #define NVM_CFG1_GLOB_ASPM_L1_MODE_OFFSET 20 + #define NVM_CFG1_GLOB_ASPM_L1_MODE_FORCED 0x0 + #define NVM_CFG1_GLOB_ASPM_L1_MODE_DYNAMIC_LOW_LATENCY 0x1 + #define NVM_CFG1_GLOB_ON_CHIP_SENSOR_MODE_MASK 0x01C00000 + #define NVM_CFG1_GLOB_ON_CHIP_SENSOR_MODE_OFFSET 22 + #define NVM_CFG1_GLOB_ON_CHIP_SENSOR_MODE_DISABLED 0x0 + #define NVM_CFG1_GLOB_ON_CHIP_SENSOR_MODE_INT_EXT_I2C 0x1 + #define NVM_CFG1_GLOB_ON_CHIP_SENSOR_MODE_INT_ONLY 0x2 + #define NVM_CFG1_GLOB_ON_CHIP_SENSOR_MODE_INT_EXT_SMBUS 0x3 + #define NVM_CFG1_GLOB_TEMPERATURE_MONITORING_MODE_MASK \ + 0x06000000 + #define NVM_CFG1_GLOB_TEMPERATURE_MONITORING_MODE_OFFSET 25 + #define NVM_CFG1_GLOB_TEMPERATURE_MONITORING_MODE_DISABLE 0x0 + #define NVM_CFG1_GLOB_TEMPERATURE_MONITORING_MODE_INTERNAL 0x1 + #define NVM_CFG1_GLOB_TEMPERATURE_MONITORING_MODE_EXTERNAL 0x2 + #define NVM_CFG1_GLOB_TEMPERATURE_MONITORING_MODE_BOTH 0x3 /* Set the PLDM sensor modes */ -#define NVM_CFG1_GLOB_PLDM_SENSOR_MODE_MASK 0x38000000 -#define NVM_CFG1_GLOB_PLDM_SENSOR_MODE_OFFSET 27 -#define NVM_CFG1_GLOB_PLDM_SENSOR_MODE_INTERNAL 0x0 -#define NVM_CFG1_GLOB_PLDM_SENSOR_MODE_EXTERNAL 0x1 -#define NVM_CFG1_GLOB_PLDM_SENSOR_MODE_BOTH 0x2 - u32 f_lane_cfg1; /* 0x38 */ -#define NVM_CFG1_GLOB_RX_LANE0_SWAP_MASK 0x0000000F -#define NVM_CFG1_GLOB_RX_LANE0_SWAP_OFFSET 0 -#define NVM_CFG1_GLOB_RX_LANE1_SWAP_MASK 0x000000F0 -#define NVM_CFG1_GLOB_RX_LANE1_SWAP_OFFSET 4 -#define NVM_CFG1_GLOB_RX_LANE2_SWAP_MASK 0x00000F00 -#define NVM_CFG1_GLOB_RX_LANE2_SWAP_OFFSET 8 -#define NVM_CFG1_GLOB_RX_LANE3_SWAP_MASK 0x0000F000 -#define NVM_CFG1_GLOB_RX_LANE3_SWAP_OFFSET 12 -#define NVM_CFG1_GLOB_TX_LANE0_SWAP_MASK 0x000F0000 -#define NVM_CFG1_GLOB_TX_LANE0_SWAP_OFFSET 16 -#define NVM_CFG1_GLOB_TX_LANE1_SWAP_MASK 0x00F00000 -#define NVM_CFG1_GLOB_TX_LANE1_SWAP_OFFSET 20 -#define NVM_CFG1_GLOB_TX_LANE2_SWAP_MASK 0x0F000000 -#define NVM_CFG1_GLOB_TX_LANE2_SWAP_OFFSET 24 -#define NVM_CFG1_GLOB_TX_LANE3_SWAP_MASK 0xF0000000 -#define NVM_CFG1_GLOB_TX_LANE3_SWAP_OFFSET 28 - u32 f_lane_cfg2; /* 0x3C */ -#define NVM_CFG1_GLOB_RX_LANE0_POL_FLIP_MASK 0x00000001 -#define NVM_CFG1_GLOB_RX_LANE0_POL_FLIP_OFFSET 0 -#define NVM_CFG1_GLOB_RX_LANE1_POL_FLIP_MASK 0x00000002 -#define NVM_CFG1_GLOB_RX_LANE1_POL_FLIP_OFFSET 1 -#define NVM_CFG1_GLOB_RX_LANE2_POL_FLIP_MASK 0x00000004 -#define NVM_CFG1_GLOB_RX_LANE2_POL_FLIP_OFFSET 2 -#define NVM_CFG1_GLOB_RX_LANE3_POL_FLIP_MASK 0x00000008 -#define NVM_CFG1_GLOB_RX_LANE3_POL_FLIP_OFFSET 3 -#define NVM_CFG1_GLOB_TX_LANE0_POL_FLIP_MASK 0x00000010 -#define NVM_CFG1_GLOB_TX_LANE0_POL_FLIP_OFFSET 4 -#define NVM_CFG1_GLOB_TX_LANE1_POL_FLIP_MASK 0x00000020 -#define NVM_CFG1_GLOB_TX_LANE1_POL_FLIP_OFFSET 5 -#define NVM_CFG1_GLOB_TX_LANE2_POL_FLIP_MASK 0x00000040 -#define NVM_CFG1_GLOB_TX_LANE2_POL_FLIP_OFFSET 6 -#define NVM_CFG1_GLOB_TX_LANE3_POL_FLIP_MASK 0x00000080 -#define NVM_CFG1_GLOB_TX_LANE3_POL_FLIP_OFFSET 7 + #define NVM_CFG1_GLOB_PLDM_SENSOR_MODE_MASK 0x38000000 + #define NVM_CFG1_GLOB_PLDM_SENSOR_MODE_OFFSET 27 + #define NVM_CFG1_GLOB_PLDM_SENSOR_MODE_INTERNAL 0x0 + #define NVM_CFG1_GLOB_PLDM_SENSOR_MODE_EXTERNAL 0x1 + #define NVM_CFG1_GLOB_PLDM_SENSOR_MODE_BOTH 0x2 + /* ROL enable */ + #define NVM_CFG1_GLOB_RESET_ON_LAN_MASK 0x80000000 + #define NVM_CFG1_GLOB_RESET_ON_LAN_OFFSET 31 + #define NVM_CFG1_GLOB_RESET_ON_LAN_DISABLED 0x0 + #define NVM_CFG1_GLOB_RESET_ON_LAN_ENABLED 0x1 + u32 f_lane_cfg1; /* 0x38 */ + #define NVM_CFG1_GLOB_RX_LANE0_SWAP_MASK 0x0000000F + #define NVM_CFG1_GLOB_RX_LANE0_SWAP_OFFSET 0 + #define NVM_CFG1_GLOB_RX_LANE1_SWAP_MASK 0x000000F0 + #define NVM_CFG1_GLOB_RX_LANE1_SWAP_OFFSET 4 + #define NVM_CFG1_GLOB_RX_LANE2_SWAP_MASK 0x00000F00 + #define NVM_CFG1_GLOB_RX_LANE2_SWAP_OFFSET 8 + #define NVM_CFG1_GLOB_RX_LANE3_SWAP_MASK 0x0000F000 + #define NVM_CFG1_GLOB_RX_LANE3_SWAP_OFFSET 12 + #define NVM_CFG1_GLOB_TX_LANE0_SWAP_MASK 0x000F0000 + #define NVM_CFG1_GLOB_TX_LANE0_SWAP_OFFSET 16 + #define NVM_CFG1_GLOB_TX_LANE1_SWAP_MASK 0x00F00000 + #define NVM_CFG1_GLOB_TX_LANE1_SWAP_OFFSET 20 + #define NVM_CFG1_GLOB_TX_LANE2_SWAP_MASK 0x0F000000 + #define NVM_CFG1_GLOB_TX_LANE2_SWAP_OFFSET 24 + #define NVM_CFG1_GLOB_TX_LANE3_SWAP_MASK 0xF0000000 + #define NVM_CFG1_GLOB_TX_LANE3_SWAP_OFFSET 28 + u32 f_lane_cfg2; /* 0x3C */ + #define NVM_CFG1_GLOB_RX_LANE0_POL_FLIP_MASK 0x00000001 + #define NVM_CFG1_GLOB_RX_LANE0_POL_FLIP_OFFSET 0 + #define NVM_CFG1_GLOB_RX_LANE1_POL_FLIP_MASK 0x00000002 + #define NVM_CFG1_GLOB_RX_LANE1_POL_FLIP_OFFSET 1 + #define NVM_CFG1_GLOB_RX_LANE2_POL_FLIP_MASK 0x00000004 + #define NVM_CFG1_GLOB_RX_LANE2_POL_FLIP_OFFSET 2 + #define NVM_CFG1_GLOB_RX_LANE3_POL_FLIP_MASK 0x00000008 + #define NVM_CFG1_GLOB_RX_LANE3_POL_FLIP_OFFSET 3 + #define NVM_CFG1_GLOB_TX_LANE0_POL_FLIP_MASK 0x00000010 + #define NVM_CFG1_GLOB_TX_LANE0_POL_FLIP_OFFSET 4 + #define NVM_CFG1_GLOB_TX_LANE1_POL_FLIP_MASK 0x00000020 + #define NVM_CFG1_GLOB_TX_LANE1_POL_FLIP_OFFSET 5 + #define NVM_CFG1_GLOB_TX_LANE2_POL_FLIP_MASK 0x00000040 + #define NVM_CFG1_GLOB_TX_LANE2_POL_FLIP_OFFSET 6 + #define NVM_CFG1_GLOB_TX_LANE3_POL_FLIP_MASK 0x00000080 + #define NVM_CFG1_GLOB_TX_LANE3_POL_FLIP_OFFSET 7 /* Control the period between two successive checks */ -#define NVM_CFG1_GLOB_TEMPERATURE_PERIOD_BETWEEN_CHECKS_MASK 0x0000FF00 -#define NVM_CFG1_GLOB_TEMPERATURE_PERIOD_BETWEEN_CHECKS_OFFSET 8 + #define NVM_CFG1_GLOB_TEMPERATURE_PERIOD_BETWEEN_CHECKS_MASK \ + 0x0000FF00 + #define NVM_CFG1_GLOB_TEMPERATURE_PERIOD_BETWEEN_CHECKS_OFFSET 8 /* Set shutdown temperature */ -#define NVM_CFG1_GLOB_SHUTDOWN_THRESHOLD_TEMPERATURE_MASK 0x00FF0000 -#define NVM_CFG1_GLOB_SHUTDOWN_THRESHOLD_TEMPERATURE_OFFSET 16 + #define NVM_CFG1_GLOB_SHUTDOWN_THRESHOLD_TEMPERATURE_MASK \ + 0x00FF0000 + #define NVM_CFG1_GLOB_SHUTDOWN_THRESHOLD_TEMPERATURE_OFFSET 16 /* Set max. count for over operational temperature */ -#define NVM_CFG1_GLOB_MAX_COUNT_OPER_THRESHOLD_MASK 0xFF000000 -#define NVM_CFG1_GLOB_MAX_COUNT_OPER_THRESHOLD_OFFSET 24 - u32 eagle_preemphasis; /* 0x40 */ -#define NVM_CFG1_GLOB_LANE0_PREEMP_MASK 0x000000FF -#define NVM_CFG1_GLOB_LANE0_PREEMP_OFFSET 0 -#define NVM_CFG1_GLOB_LANE1_PREEMP_MASK 0x0000FF00 -#define NVM_CFG1_GLOB_LANE1_PREEMP_OFFSET 8 -#define NVM_CFG1_GLOB_LANE2_PREEMP_MASK 0x00FF0000 -#define NVM_CFG1_GLOB_LANE2_PREEMP_OFFSET 16 -#define NVM_CFG1_GLOB_LANE3_PREEMP_MASK 0xFF000000 -#define NVM_CFG1_GLOB_LANE3_PREEMP_OFFSET 24 - u32 eagle_driver_current; /* 0x44 */ -#define NVM_CFG1_GLOB_LANE0_AMP_MASK 0x000000FF -#define NVM_CFG1_GLOB_LANE0_AMP_OFFSET 0 -#define NVM_CFG1_GLOB_LANE1_AMP_MASK 0x0000FF00 -#define NVM_CFG1_GLOB_LANE1_AMP_OFFSET 8 -#define NVM_CFG1_GLOB_LANE2_AMP_MASK 0x00FF0000 -#define NVM_CFG1_GLOB_LANE2_AMP_OFFSET 16 -#define NVM_CFG1_GLOB_LANE3_AMP_MASK 0xFF000000 -#define NVM_CFG1_GLOB_LANE3_AMP_OFFSET 24 - u32 falcon_preemphasis; /* 0x48 */ -#define NVM_CFG1_GLOB_LANE0_PREEMP_MASK 0x000000FF -#define NVM_CFG1_GLOB_LANE0_PREEMP_OFFSET 0 -#define NVM_CFG1_GLOB_LANE1_PREEMP_MASK 0x0000FF00 -#define NVM_CFG1_GLOB_LANE1_PREEMP_OFFSET 8 -#define NVM_CFG1_GLOB_LANE2_PREEMP_MASK 0x00FF0000 -#define NVM_CFG1_GLOB_LANE2_PREEMP_OFFSET 16 -#define NVM_CFG1_GLOB_LANE3_PREEMP_MASK 0xFF000000 -#define NVM_CFG1_GLOB_LANE3_PREEMP_OFFSET 24 - u32 falcon_driver_current; /* 0x4C */ -#define NVM_CFG1_GLOB_LANE0_AMP_MASK 0x000000FF -#define NVM_CFG1_GLOB_LANE0_AMP_OFFSET 0 -#define NVM_CFG1_GLOB_LANE1_AMP_MASK 0x0000FF00 -#define NVM_CFG1_GLOB_LANE1_AMP_OFFSET 8 -#define NVM_CFG1_GLOB_LANE2_AMP_MASK 0x00FF0000 -#define NVM_CFG1_GLOB_LANE2_AMP_OFFSET 16 -#define NVM_CFG1_GLOB_LANE3_AMP_MASK 0xFF000000 -#define NVM_CFG1_GLOB_LANE3_AMP_OFFSET 24 - u32 pci_id; /* 0x50 */ -#define NVM_CFG1_GLOB_VENDOR_ID_MASK 0x0000FFFF -#define NVM_CFG1_GLOB_VENDOR_ID_OFFSET 0 + #define NVM_CFG1_GLOB_MAX_COUNT_OPER_THRESHOLD_MASK 0xFF000000 + #define NVM_CFG1_GLOB_MAX_COUNT_OPER_THRESHOLD_OFFSET 24 + u32 mps10_preemphasis; /* 0x40 */ + #define NVM_CFG1_GLOB_LANE0_PREEMP_MASK 0x000000FF + #define NVM_CFG1_GLOB_LANE0_PREEMP_OFFSET 0 + #define NVM_CFG1_GLOB_LANE1_PREEMP_MASK 0x0000FF00 + #define NVM_CFG1_GLOB_LANE1_PREEMP_OFFSET 8 + #define NVM_CFG1_GLOB_LANE2_PREEMP_MASK 0x00FF0000 + #define NVM_CFG1_GLOB_LANE2_PREEMP_OFFSET 16 + #define NVM_CFG1_GLOB_LANE3_PREEMP_MASK 0xFF000000 + #define NVM_CFG1_GLOB_LANE3_PREEMP_OFFSET 24 + u32 mps10_driver_current; /* 0x44 */ + #define NVM_CFG1_GLOB_LANE0_AMP_MASK 0x000000FF + #define NVM_CFG1_GLOB_LANE0_AMP_OFFSET 0 + #define NVM_CFG1_GLOB_LANE1_AMP_MASK 0x0000FF00 + #define NVM_CFG1_GLOB_LANE1_AMP_OFFSET 8 + #define NVM_CFG1_GLOB_LANE2_AMP_MASK 0x00FF0000 + #define NVM_CFG1_GLOB_LANE2_AMP_OFFSET 16 + #define NVM_CFG1_GLOB_LANE3_AMP_MASK 0xFF000000 + #define NVM_CFG1_GLOB_LANE3_AMP_OFFSET 24 + u32 mps25_preemphasis; /* 0x48 */ + #define NVM_CFG1_GLOB_LANE0_PREEMP_MASK 0x000000FF + #define NVM_CFG1_GLOB_LANE0_PREEMP_OFFSET 0 + #define NVM_CFG1_GLOB_LANE1_PREEMP_MASK 0x0000FF00 + #define NVM_CFG1_GLOB_LANE1_PREEMP_OFFSET 8 + #define NVM_CFG1_GLOB_LANE2_PREEMP_MASK 0x00FF0000 + #define NVM_CFG1_GLOB_LANE2_PREEMP_OFFSET 16 + #define NVM_CFG1_GLOB_LANE3_PREEMP_MASK 0xFF000000 + #define NVM_CFG1_GLOB_LANE3_PREEMP_OFFSET 24 + u32 mps25_driver_current; /* 0x4C */ + #define NVM_CFG1_GLOB_LANE0_AMP_MASK 0x000000FF + #define NVM_CFG1_GLOB_LANE0_AMP_OFFSET 0 + #define NVM_CFG1_GLOB_LANE1_AMP_MASK 0x0000FF00 + #define NVM_CFG1_GLOB_LANE1_AMP_OFFSET 8 + #define NVM_CFG1_GLOB_LANE2_AMP_MASK 0x00FF0000 + #define NVM_CFG1_GLOB_LANE2_AMP_OFFSET 16 + #define NVM_CFG1_GLOB_LANE3_AMP_MASK 0xFF000000 + #define NVM_CFG1_GLOB_LANE3_AMP_OFFSET 24 + u32 pci_id; /* 0x50 */ + #define NVM_CFG1_GLOB_VENDOR_ID_MASK 0x0000FFFF + #define NVM_CFG1_GLOB_VENDOR_ID_OFFSET 0 /* Set caution temperature */ -#define NVM_CFG1_GLOB_CAUTION_THRESHOLD_TEMPERATURE_MASK 0x00FF0000 -#define NVM_CFG1_GLOB_CAUTION_THRESHOLD_TEMPERATURE_OFFSET 16 + #define NVM_CFG1_GLOB_DEAD_TEMP_TH_TEMPERATURE_MASK 0x00FF0000 + #define NVM_CFG1_GLOB_DEAD_TEMP_TH_TEMPERATURE_OFFSET 16 /* Set external thermal sensor I2C address */ -#define NVM_CFG1_GLOB_EXTERNAL_THERMAL_SENSOR_ADDRESS_MASK 0xFF000000 -#define NVM_CFG1_GLOB_EXTERNAL_THERMAL_SENSOR_ADDRESS_OFFSET 24 - u32 pci_subsys_id; /* 0x54 */ -#define NVM_CFG1_GLOB_SUBSYSTEM_VENDOR_ID_MASK 0x0000FFFF -#define NVM_CFG1_GLOB_SUBSYSTEM_VENDOR_ID_OFFSET 0 -#define NVM_CFG1_GLOB_SUBSYSTEM_DEVICE_ID_MASK 0xFFFF0000 -#define NVM_CFG1_GLOB_SUBSYSTEM_DEVICE_ID_OFFSET 16 - u32 bar; /* 0x58 */ -#define NVM_CFG1_GLOB_EXPANSION_ROM_SIZE_MASK 0x0000000F -#define NVM_CFG1_GLOB_EXPANSION_ROM_SIZE_OFFSET 0 -#define NVM_CFG1_GLOB_EXPANSION_ROM_SIZE_DISABLED 0x0 -#define NVM_CFG1_GLOB_EXPANSION_ROM_SIZE_2K 0x1 -#define NVM_CFG1_GLOB_EXPANSION_ROM_SIZE_4K 0x2 -#define NVM_CFG1_GLOB_EXPANSION_ROM_SIZE_8K 0x3 -#define NVM_CFG1_GLOB_EXPANSION_ROM_SIZE_16K 0x4 -#define NVM_CFG1_GLOB_EXPANSION_ROM_SIZE_32K 0x5 -#define NVM_CFG1_GLOB_EXPANSION_ROM_SIZE_64K 0x6 -#define NVM_CFG1_GLOB_EXPANSION_ROM_SIZE_128K 0x7 -#define NVM_CFG1_GLOB_EXPANSION_ROM_SIZE_256K 0x8 -#define NVM_CFG1_GLOB_EXPANSION_ROM_SIZE_512K 0x9 -#define NVM_CFG1_GLOB_EXPANSION_ROM_SIZE_1M 0xA -#define NVM_CFG1_GLOB_EXPANSION_ROM_SIZE_2M 0xB -#define NVM_CFG1_GLOB_EXPANSION_ROM_SIZE_4M 0xC -#define NVM_CFG1_GLOB_EXPANSION_ROM_SIZE_8M 0xD -#define NVM_CFG1_GLOB_EXPANSION_ROM_SIZE_16M 0xE -#define NVM_CFG1_GLOB_EXPANSION_ROM_SIZE_32M 0xF -#define NVM_CFG1_GLOB_VF_PCI_BAR2_SIZE_MASK 0x000000F0 -#define NVM_CFG1_GLOB_VF_PCI_BAR2_SIZE_OFFSET 4 -#define NVM_CFG1_GLOB_VF_PCI_BAR2_SIZE_DISABLED 0x0 -#define NVM_CFG1_GLOB_VF_PCI_BAR2_SIZE_4K 0x1 -#define NVM_CFG1_GLOB_VF_PCI_BAR2_SIZE_8K 0x2 -#define NVM_CFG1_GLOB_VF_PCI_BAR2_SIZE_16K 0x3 -#define NVM_CFG1_GLOB_VF_PCI_BAR2_SIZE_32K 0x4 -#define NVM_CFG1_GLOB_VF_PCI_BAR2_SIZE_64K 0x5 -#define NVM_CFG1_GLOB_VF_PCI_BAR2_SIZE_128K 0x6 -#define NVM_CFG1_GLOB_VF_PCI_BAR2_SIZE_256K 0x7 -#define NVM_CFG1_GLOB_VF_PCI_BAR2_SIZE_512K 0x8 -#define NVM_CFG1_GLOB_VF_PCI_BAR2_SIZE_1M 0x9 -#define NVM_CFG1_GLOB_VF_PCI_BAR2_SIZE_2M 0xA -#define NVM_CFG1_GLOB_VF_PCI_BAR2_SIZE_4M 0xB -#define NVM_CFG1_GLOB_VF_PCI_BAR2_SIZE_8M 0xC -#define NVM_CFG1_GLOB_VF_PCI_BAR2_SIZE_16M 0xD -#define NVM_CFG1_GLOB_VF_PCI_BAR2_SIZE_32M 0xE -#define NVM_CFG1_GLOB_VF_PCI_BAR2_SIZE_64M 0xF -#define NVM_CFG1_GLOB_BAR2_SIZE_MASK 0x00000F00 -#define NVM_CFG1_GLOB_BAR2_SIZE_OFFSET 8 -#define NVM_CFG1_GLOB_BAR2_SIZE_DISABLED 0x0 -#define NVM_CFG1_GLOB_BAR2_SIZE_64K 0x1 -#define NVM_CFG1_GLOB_BAR2_SIZE_128K 0x2 -#define NVM_CFG1_GLOB_BAR2_SIZE_256K 0x3 -#define NVM_CFG1_GLOB_BAR2_SIZE_512K 0x4 -#define NVM_CFG1_GLOB_BAR2_SIZE_1M 0x5 -#define NVM_CFG1_GLOB_BAR2_SIZE_2M 0x6 -#define NVM_CFG1_GLOB_BAR2_SIZE_4M 0x7 -#define NVM_CFG1_GLOB_BAR2_SIZE_8M 0x8 -#define NVM_CFG1_GLOB_BAR2_SIZE_16M 0x9 -#define NVM_CFG1_GLOB_BAR2_SIZE_32M 0xA -#define NVM_CFG1_GLOB_BAR2_SIZE_64M 0xB -#define NVM_CFG1_GLOB_BAR2_SIZE_128M 0xC -#define NVM_CFG1_GLOB_BAR2_SIZE_256M 0xD -#define NVM_CFG1_GLOB_BAR2_SIZE_512M 0xE -#define NVM_CFG1_GLOB_BAR2_SIZE_1G 0xF - /* Set the duration, in seconds, fan failure signal should be - * sampled + #define NVM_CFG1_GLOB_EXTERNAL_THERMAL_SENSOR_ADDRESS_MASK \ + 0xFF000000 + #define NVM_CFG1_GLOB_EXTERNAL_THERMAL_SENSOR_ADDRESS_OFFSET 24 + u32 pci_subsys_id; /* 0x54 */ + #define NVM_CFG1_GLOB_SUBSYSTEM_VENDOR_ID_MASK 0x0000FFFF + #define NVM_CFG1_GLOB_SUBSYSTEM_VENDOR_ID_OFFSET 0 + #define NVM_CFG1_GLOB_SUBSYSTEM_DEVICE_ID_MASK 0xFFFF0000 + #define NVM_CFG1_GLOB_SUBSYSTEM_DEVICE_ID_OFFSET 16 + u32 bar; /* 0x58 */ + #define NVM_CFG1_GLOB_EXPANSION_ROM_SIZE_MASK 0x0000000F + #define NVM_CFG1_GLOB_EXPANSION_ROM_SIZE_OFFSET 0 + #define NVM_CFG1_GLOB_EXPANSION_ROM_SIZE_DISABLED 0x0 + #define NVM_CFG1_GLOB_EXPANSION_ROM_SIZE_2K 0x1 + #define NVM_CFG1_GLOB_EXPANSION_ROM_SIZE_4K 0x2 + #define NVM_CFG1_GLOB_EXPANSION_ROM_SIZE_8K 0x3 + #define NVM_CFG1_GLOB_EXPANSION_ROM_SIZE_16K 0x4 + #define NVM_CFG1_GLOB_EXPANSION_ROM_SIZE_32K 0x5 + #define NVM_CFG1_GLOB_EXPANSION_ROM_SIZE_64K 0x6 + #define NVM_CFG1_GLOB_EXPANSION_ROM_SIZE_128K 0x7 + #define NVM_CFG1_GLOB_EXPANSION_ROM_SIZE_256K 0x8 + #define NVM_CFG1_GLOB_EXPANSION_ROM_SIZE_512K 0x9 + #define NVM_CFG1_GLOB_EXPANSION_ROM_SIZE_1M 0xA + #define NVM_CFG1_GLOB_EXPANSION_ROM_SIZE_2M 0xB + #define NVM_CFG1_GLOB_EXPANSION_ROM_SIZE_4M 0xC + #define NVM_CFG1_GLOB_EXPANSION_ROM_SIZE_8M 0xD + #define NVM_CFG1_GLOB_EXPANSION_ROM_SIZE_16M 0xE + #define NVM_CFG1_GLOB_EXPANSION_ROM_SIZE_32M 0xF + /* BB VF BAR2 size */ + #define NVM_CFG1_GLOB_VF_PCI_BAR2_SIZE_MASK 0x000000F0 + #define NVM_CFG1_GLOB_VF_PCI_BAR2_SIZE_OFFSET 4 + #define NVM_CFG1_GLOB_VF_PCI_BAR2_SIZE_DISABLED 0x0 + #define NVM_CFG1_GLOB_VF_PCI_BAR2_SIZE_4K 0x1 + #define NVM_CFG1_GLOB_VF_PCI_BAR2_SIZE_8K 0x2 + #define NVM_CFG1_GLOB_VF_PCI_BAR2_SIZE_16K 0x3 + #define NVM_CFG1_GLOB_VF_PCI_BAR2_SIZE_32K 0x4 + #define NVM_CFG1_GLOB_VF_PCI_BAR2_SIZE_64K 0x5 + #define NVM_CFG1_GLOB_VF_PCI_BAR2_SIZE_128K 0x6 + #define NVM_CFG1_GLOB_VF_PCI_BAR2_SIZE_256K 0x7 + #define NVM_CFG1_GLOB_VF_PCI_BAR2_SIZE_512K 0x8 + #define NVM_CFG1_GLOB_VF_PCI_BAR2_SIZE_1M 0x9 + #define NVM_CFG1_GLOB_VF_PCI_BAR2_SIZE_2M 0xA + #define NVM_CFG1_GLOB_VF_PCI_BAR2_SIZE_4M 0xB + #define NVM_CFG1_GLOB_VF_PCI_BAR2_SIZE_8M 0xC + #define NVM_CFG1_GLOB_VF_PCI_BAR2_SIZE_16M 0xD + #define NVM_CFG1_GLOB_VF_PCI_BAR2_SIZE_32M 0xE + #define NVM_CFG1_GLOB_VF_PCI_BAR2_SIZE_64M 0xF + /* BB BAR2 size (global) */ + #define NVM_CFG1_GLOB_BAR2_SIZE_MASK 0x00000F00 + #define NVM_CFG1_GLOB_BAR2_SIZE_OFFSET 8 + #define NVM_CFG1_GLOB_BAR2_SIZE_DISABLED 0x0 + #define NVM_CFG1_GLOB_BAR2_SIZE_64K 0x1 + #define NVM_CFG1_GLOB_BAR2_SIZE_128K 0x2 + #define NVM_CFG1_GLOB_BAR2_SIZE_256K 0x3 + #define NVM_CFG1_GLOB_BAR2_SIZE_512K 0x4 + #define NVM_CFG1_GLOB_BAR2_SIZE_1M 0x5 + #define NVM_CFG1_GLOB_BAR2_SIZE_2M 0x6 + #define NVM_CFG1_GLOB_BAR2_SIZE_4M 0x7 + #define NVM_CFG1_GLOB_BAR2_SIZE_8M 0x8 + #define NVM_CFG1_GLOB_BAR2_SIZE_16M 0x9 + #define NVM_CFG1_GLOB_BAR2_SIZE_32M 0xA + #define NVM_CFG1_GLOB_BAR2_SIZE_64M 0xB + #define NVM_CFG1_GLOB_BAR2_SIZE_128M 0xC + #define NVM_CFG1_GLOB_BAR2_SIZE_256M 0xD + #define NVM_CFG1_GLOB_BAR2_SIZE_512M 0xE + #define NVM_CFG1_GLOB_BAR2_SIZE_1G 0xF + /* Set the duration, in secs, fan failure signal should be sampled */ + #define NVM_CFG1_GLOB_FAN_FAILURE_DURATION_MASK 0x0000F000 + #define NVM_CFG1_GLOB_FAN_FAILURE_DURATION_OFFSET 12 + /* This field defines the board total budget for bar2 when disabled + * the regular bar size is used. */ -#define NVM_CFG1_GLOB_FAN_FAILURE_DURATION_MASK 0x0000F000 -#define NVM_CFG1_GLOB_FAN_FAILURE_DURATION_OFFSET 12 - u32 eagle_txfir_main; /* 0x5C */ -#define NVM_CFG1_GLOB_LANE0_TXFIR_MAIN_MASK 0x000000FF -#define NVM_CFG1_GLOB_LANE0_TXFIR_MAIN_OFFSET 0 -#define NVM_CFG1_GLOB_LANE1_TXFIR_MAIN_MASK 0x0000FF00 -#define NVM_CFG1_GLOB_LANE1_TXFIR_MAIN_OFFSET 8 -#define NVM_CFG1_GLOB_LANE2_TXFIR_MAIN_MASK 0x00FF0000 -#define NVM_CFG1_GLOB_LANE2_TXFIR_MAIN_OFFSET 16 -#define NVM_CFG1_GLOB_LANE3_TXFIR_MAIN_MASK 0xFF000000 -#define NVM_CFG1_GLOB_LANE3_TXFIR_MAIN_OFFSET 24 - u32 eagle_txfir_post; /* 0x60 */ -#define NVM_CFG1_GLOB_LANE0_TXFIR_POST_MASK 0x000000FF -#define NVM_CFG1_GLOB_LANE0_TXFIR_POST_OFFSET 0 -#define NVM_CFG1_GLOB_LANE1_TXFIR_POST_MASK 0x0000FF00 -#define NVM_CFG1_GLOB_LANE1_TXFIR_POST_OFFSET 8 -#define NVM_CFG1_GLOB_LANE2_TXFIR_POST_MASK 0x00FF0000 -#define NVM_CFG1_GLOB_LANE2_TXFIR_POST_OFFSET 16 -#define NVM_CFG1_GLOB_LANE3_TXFIR_POST_MASK 0xFF000000 -#define NVM_CFG1_GLOB_LANE3_TXFIR_POST_OFFSET 24 - u32 falcon_txfir_main; /* 0x64 */ -#define NVM_CFG1_GLOB_LANE0_TXFIR_MAIN_MASK 0x000000FF -#define NVM_CFG1_GLOB_LANE0_TXFIR_MAIN_OFFSET 0 -#define NVM_CFG1_GLOB_LANE1_TXFIR_MAIN_MASK 0x0000FF00 -#define NVM_CFG1_GLOB_LANE1_TXFIR_MAIN_OFFSET 8 -#define NVM_CFG1_GLOB_LANE2_TXFIR_MAIN_MASK 0x00FF0000 -#define NVM_CFG1_GLOB_LANE2_TXFIR_MAIN_OFFSET 16 -#define NVM_CFG1_GLOB_LANE3_TXFIR_MAIN_MASK 0xFF000000 -#define NVM_CFG1_GLOB_LANE3_TXFIR_MAIN_OFFSET 24 - u32 falcon_txfir_post; /* 0x68 */ -#define NVM_CFG1_GLOB_LANE0_TXFIR_POST_MASK 0x000000FF -#define NVM_CFG1_GLOB_LANE0_TXFIR_POST_OFFSET 0 -#define NVM_CFG1_GLOB_LANE1_TXFIR_POST_MASK 0x0000FF00 -#define NVM_CFG1_GLOB_LANE1_TXFIR_POST_OFFSET 8 -#define NVM_CFG1_GLOB_LANE2_TXFIR_POST_MASK 0x00FF0000 -#define NVM_CFG1_GLOB_LANE2_TXFIR_POST_OFFSET 16 -#define NVM_CFG1_GLOB_LANE3_TXFIR_POST_MASK 0xFF000000 -#define NVM_CFG1_GLOB_LANE3_TXFIR_POST_OFFSET 24 - u32 manufacture_ver; /* 0x6C */ -#define NVM_CFG1_GLOB_MANUF0_VER_MASK 0x0000003F -#define NVM_CFG1_GLOB_MANUF0_VER_OFFSET 0 -#define NVM_CFG1_GLOB_MANUF1_VER_MASK 0x00000FC0 -#define NVM_CFG1_GLOB_MANUF1_VER_OFFSET 6 -#define NVM_CFG1_GLOB_MANUF2_VER_MASK 0x0003F000 -#define NVM_CFG1_GLOB_MANUF2_VER_OFFSET 12 -#define NVM_CFG1_GLOB_MANUF3_VER_MASK 0x00FC0000 -#define NVM_CFG1_GLOB_MANUF3_VER_OFFSET 18 -#define NVM_CFG1_GLOB_MANUF4_VER_MASK 0x3F000000 -#define NVM_CFG1_GLOB_MANUF4_VER_OFFSET 24 - u32 manufacture_time; /* 0x70 */ -#define NVM_CFG1_GLOB_MANUF0_TIME_MASK 0x0000003F -#define NVM_CFG1_GLOB_MANUF0_TIME_OFFSET 0 -#define NVM_CFG1_GLOB_MANUF1_TIME_MASK 0x00000FC0 -#define NVM_CFG1_GLOB_MANUF1_TIME_OFFSET 6 -#define NVM_CFG1_GLOB_MANUF2_TIME_MASK 0x0003F000 -#define NVM_CFG1_GLOB_MANUF2_TIME_OFFSET 12 - u32 led_global_settings; /* 0x74 */ -#define NVM_CFG1_GLOB_LED_SWAP_0_MASK 0x0000000F -#define NVM_CFG1_GLOB_LED_SWAP_0_OFFSET 0 -#define NVM_CFG1_GLOB_LED_SWAP_1_MASK 0x000000F0 -#define NVM_CFG1_GLOB_LED_SWAP_1_OFFSET 4 -#define NVM_CFG1_GLOB_LED_SWAP_2_MASK 0x00000F00 -#define NVM_CFG1_GLOB_LED_SWAP_2_OFFSET 8 -#define NVM_CFG1_GLOB_LED_SWAP_3_MASK 0x0000F000 -#define NVM_CFG1_GLOB_LED_SWAP_3_OFFSET 12 - u32 generic_cont1; /* 0x78 */ -#define NVM_CFG1_GLOB_AVS_DAC_CODE_MASK 0x000003FF -#define NVM_CFG1_GLOB_AVS_DAC_CODE_OFFSET 0 - u32 mbi_version; /* 0x7C */ -#define NVM_CFG1_GLOB_MBI_VERSION_0_MASK 0x000000FF -#define NVM_CFG1_GLOB_MBI_VERSION_0_OFFSET 0 -#define NVM_CFG1_GLOB_MBI_VERSION_1_MASK 0x0000FF00 -#define NVM_CFG1_GLOB_MBI_VERSION_1_OFFSET 8 -#define NVM_CFG1_GLOB_MBI_VERSION_2_MASK 0x00FF0000 -#define NVM_CFG1_GLOB_MBI_VERSION_2_OFFSET 16 - u32 mbi_date; /* 0x80 */ - u32 misc_sig; /* 0x84 */ + #define NVM_CFG1_GLOB_BAR2_TOTAL_BUDGET_MASK 0x00FF0000 + #define NVM_CFG1_GLOB_BAR2_TOTAL_BUDGET_OFFSET 16 + #define NVM_CFG1_GLOB_BAR2_TOTAL_BUDGET_DISABLED 0x0 + #define NVM_CFG1_GLOB_BAR2_TOTAL_BUDGET_64K 0x1 + #define NVM_CFG1_GLOB_BAR2_TOTAL_BUDGET_128K 0x2 + #define NVM_CFG1_GLOB_BAR2_TOTAL_BUDGET_256K 0x3 + #define NVM_CFG1_GLOB_BAR2_TOTAL_BUDGET_512K 0x4 + #define NVM_CFG1_GLOB_BAR2_TOTAL_BUDGET_1M 0x5 + #define NVM_CFG1_GLOB_BAR2_TOTAL_BUDGET_2M 0x6 + #define NVM_CFG1_GLOB_BAR2_TOTAL_BUDGET_4M 0x7 + #define NVM_CFG1_GLOB_BAR2_TOTAL_BUDGET_8M 0x8 + #define NVM_CFG1_GLOB_BAR2_TOTAL_BUDGET_16M 0x9 + #define NVM_CFG1_GLOB_BAR2_TOTAL_BUDGET_32M 0xA + #define NVM_CFG1_GLOB_BAR2_TOTAL_BUDGET_64M 0xB + #define NVM_CFG1_GLOB_BAR2_TOTAL_BUDGET_128M 0xC + #define NVM_CFG1_GLOB_BAR2_TOTAL_BUDGET_256M 0xD + #define NVM_CFG1_GLOB_BAR2_TOTAL_BUDGET_512M 0xE + #define NVM_CFG1_GLOB_BAR2_TOTAL_BUDGET_1G 0xF + /* Enable/Disable Crash dump triggers */ + #define NVM_CFG1_GLOB_CRASH_DUMP_TRIGGER_ENABLE_MASK 0xFF000000 + #define NVM_CFG1_GLOB_CRASH_DUMP_TRIGGER_ENABLE_OFFSET 24 + u32 mps10_txfir_main; /* 0x5C */ + #define NVM_CFG1_GLOB_LANE0_TXFIR_MAIN_MASK 0x000000FF + #define NVM_CFG1_GLOB_LANE0_TXFIR_MAIN_OFFSET 0 + #define NVM_CFG1_GLOB_LANE1_TXFIR_MAIN_MASK 0x0000FF00 + #define NVM_CFG1_GLOB_LANE1_TXFIR_MAIN_OFFSET 8 + #define NVM_CFG1_GLOB_LANE2_TXFIR_MAIN_MASK 0x00FF0000 + #define NVM_CFG1_GLOB_LANE2_TXFIR_MAIN_OFFSET 16 + #define NVM_CFG1_GLOB_LANE3_TXFIR_MAIN_MASK 0xFF000000 + #define NVM_CFG1_GLOB_LANE3_TXFIR_MAIN_OFFSET 24 + u32 mps10_txfir_post; /* 0x60 */ + #define NVM_CFG1_GLOB_LANE0_TXFIR_POST_MASK 0x000000FF + #define NVM_CFG1_GLOB_LANE0_TXFIR_POST_OFFSET 0 + #define NVM_CFG1_GLOB_LANE1_TXFIR_POST_MASK 0x0000FF00 + #define NVM_CFG1_GLOB_LANE1_TXFIR_POST_OFFSET 8 + #define NVM_CFG1_GLOB_LANE2_TXFIR_POST_MASK 0x00FF0000 + #define NVM_CFG1_GLOB_LANE2_TXFIR_POST_OFFSET 16 + #define NVM_CFG1_GLOB_LANE3_TXFIR_POST_MASK 0xFF000000 + #define NVM_CFG1_GLOB_LANE3_TXFIR_POST_OFFSET 24 + u32 mps25_txfir_main; /* 0x64 */ + #define NVM_CFG1_GLOB_LANE0_TXFIR_MAIN_MASK 0x000000FF + #define NVM_CFG1_GLOB_LANE0_TXFIR_MAIN_OFFSET 0 + #define NVM_CFG1_GLOB_LANE1_TXFIR_MAIN_MASK 0x0000FF00 + #define NVM_CFG1_GLOB_LANE1_TXFIR_MAIN_OFFSET 8 + #define NVM_CFG1_GLOB_LANE2_TXFIR_MAIN_MASK 0x00FF0000 + #define NVM_CFG1_GLOB_LANE2_TXFIR_MAIN_OFFSET 16 + #define NVM_CFG1_GLOB_LANE3_TXFIR_MAIN_MASK 0xFF000000 + #define NVM_CFG1_GLOB_LANE3_TXFIR_MAIN_OFFSET 24 + u32 mps25_txfir_post; /* 0x68 */ + #define NVM_CFG1_GLOB_LANE0_TXFIR_POST_MASK 0x000000FF + #define NVM_CFG1_GLOB_LANE0_TXFIR_POST_OFFSET 0 + #define NVM_CFG1_GLOB_LANE1_TXFIR_POST_MASK 0x0000FF00 + #define NVM_CFG1_GLOB_LANE1_TXFIR_POST_OFFSET 8 + #define NVM_CFG1_GLOB_LANE2_TXFIR_POST_MASK 0x00FF0000 + #define NVM_CFG1_GLOB_LANE2_TXFIR_POST_OFFSET 16 + #define NVM_CFG1_GLOB_LANE3_TXFIR_POST_MASK 0xFF000000 + #define NVM_CFG1_GLOB_LANE3_TXFIR_POST_OFFSET 24 + u32 manufacture_ver; /* 0x6C */ + #define NVM_CFG1_GLOB_MANUF0_VER_MASK 0x0000003F + #define NVM_CFG1_GLOB_MANUF0_VER_OFFSET 0 + #define NVM_CFG1_GLOB_MANUF1_VER_MASK 0x00000FC0 + #define NVM_CFG1_GLOB_MANUF1_VER_OFFSET 6 + #define NVM_CFG1_GLOB_MANUF2_VER_MASK 0x0003F000 + #define NVM_CFG1_GLOB_MANUF2_VER_OFFSET 12 + #define NVM_CFG1_GLOB_MANUF3_VER_MASK 0x00FC0000 + #define NVM_CFG1_GLOB_MANUF3_VER_OFFSET 18 + #define NVM_CFG1_GLOB_MANUF4_VER_MASK 0x3F000000 + #define NVM_CFG1_GLOB_MANUF4_VER_OFFSET 24 + /* Select package id method */ + #define NVM_CFG1_GLOB_NCSI_PACKAGE_ID_IO_MASK 0x40000000 + #define NVM_CFG1_GLOB_NCSI_PACKAGE_ID_IO_OFFSET 30 + #define NVM_CFG1_GLOB_NCSI_PACKAGE_ID_IO_NVRAM 0x0 + #define NVM_CFG1_GLOB_NCSI_PACKAGE_ID_IO_IO_PINS 0x1 + #define NVM_CFG1_GLOB_RECOVERY_MODE_MASK 0x80000000 + #define NVM_CFG1_GLOB_RECOVERY_MODE_OFFSET 31 + #define NVM_CFG1_GLOB_RECOVERY_MODE_DISABLED 0x0 + #define NVM_CFG1_GLOB_RECOVERY_MODE_ENABLED 0x1 + u32 manufacture_time; /* 0x70 */ + #define NVM_CFG1_GLOB_MANUF0_TIME_MASK 0x0000003F + #define NVM_CFG1_GLOB_MANUF0_TIME_OFFSET 0 + #define NVM_CFG1_GLOB_MANUF1_TIME_MASK 0x00000FC0 + #define NVM_CFG1_GLOB_MANUF1_TIME_OFFSET 6 + #define NVM_CFG1_GLOB_MANUF2_TIME_MASK 0x0003F000 + #define NVM_CFG1_GLOB_MANUF2_TIME_OFFSET 12 + /* Max MSIX for Ethernet in default mode */ + #define NVM_CFG1_GLOB_MAX_MSIX_MASK 0x03FC0000 + #define NVM_CFG1_GLOB_MAX_MSIX_OFFSET 18 + /* PF Mapping */ + #define NVM_CFG1_GLOB_PF_MAPPING_MASK 0x0C000000 + #define NVM_CFG1_GLOB_PF_MAPPING_OFFSET 26 + #define NVM_CFG1_GLOB_PF_MAPPING_CONTINUOUS 0x0 + #define NVM_CFG1_GLOB_PF_MAPPING_FIXED 0x1 + #define NVM_CFG1_GLOB_VOLTAGE_REGULATOR_TYPE_MASK 0x30000000 + #define NVM_CFG1_GLOB_VOLTAGE_REGULATOR_TYPE_OFFSET 28 + #define NVM_CFG1_GLOB_VOLTAGE_REGULATOR_TYPE_DISABLED 0x0 + #define NVM_CFG1_GLOB_VOLTAGE_REGULATOR_TYPE_TI 0x1 + u32 led_global_settings; /* 0x74 */ + #define NVM_CFG1_GLOB_LED_SWAP_0_MASK 0x0000000F + #define NVM_CFG1_GLOB_LED_SWAP_0_OFFSET 0 + #define NVM_CFG1_GLOB_LED_SWAP_1_MASK 0x000000F0 + #define NVM_CFG1_GLOB_LED_SWAP_1_OFFSET 4 + #define NVM_CFG1_GLOB_LED_SWAP_2_MASK 0x00000F00 + #define NVM_CFG1_GLOB_LED_SWAP_2_OFFSET 8 + #define NVM_CFG1_GLOB_LED_SWAP_3_MASK 0x0000F000 + #define NVM_CFG1_GLOB_LED_SWAP_3_OFFSET 12 + /* Max. continues operating temperature */ + #define NVM_CFG1_GLOB_MAX_CONT_OPERATING_TEMP_MASK 0x00FF0000 + #define NVM_CFG1_GLOB_MAX_CONT_OPERATING_TEMP_OFFSET 16 + /* GPIO which triggers run-time port swap according to the map + * specified in option 205 + */ + #define NVM_CFG1_GLOB_RUNTIME_PORT_SWAP_GPIO_MASK 0xFF000000 + #define NVM_CFG1_GLOB_RUNTIME_PORT_SWAP_GPIO_OFFSET 24 + #define NVM_CFG1_GLOB_RUNTIME_PORT_SWAP_GPIO_NA 0x0 + #define NVM_CFG1_GLOB_RUNTIME_PORT_SWAP_GPIO_GPIO0 0x1 + #define NVM_CFG1_GLOB_RUNTIME_PORT_SWAP_GPIO_GPIO1 0x2 + #define NVM_CFG1_GLOB_RUNTIME_PORT_SWAP_GPIO_GPIO2 0x3 + #define NVM_CFG1_GLOB_RUNTIME_PORT_SWAP_GPIO_GPIO3 0x4 + #define NVM_CFG1_GLOB_RUNTIME_PORT_SWAP_GPIO_GPIO4 0x5 + #define NVM_CFG1_GLOB_RUNTIME_PORT_SWAP_GPIO_GPIO5 0x6 + #define NVM_CFG1_GLOB_RUNTIME_PORT_SWAP_GPIO_GPIO6 0x7 + #define NVM_CFG1_GLOB_RUNTIME_PORT_SWAP_GPIO_GPIO7 0x8 + #define NVM_CFG1_GLOB_RUNTIME_PORT_SWAP_GPIO_GPIO8 0x9 + #define NVM_CFG1_GLOB_RUNTIME_PORT_SWAP_GPIO_GPIO9 0xA + #define NVM_CFG1_GLOB_RUNTIME_PORT_SWAP_GPIO_GPIO10 0xB + #define NVM_CFG1_GLOB_RUNTIME_PORT_SWAP_GPIO_GPIO11 0xC + #define NVM_CFG1_GLOB_RUNTIME_PORT_SWAP_GPIO_GPIO12 0xD + #define NVM_CFG1_GLOB_RUNTIME_PORT_SWAP_GPIO_GPIO13 0xE + #define NVM_CFG1_GLOB_RUNTIME_PORT_SWAP_GPIO_GPIO14 0xF + #define NVM_CFG1_GLOB_RUNTIME_PORT_SWAP_GPIO_GPIO15 0x10 + #define NVM_CFG1_GLOB_RUNTIME_PORT_SWAP_GPIO_GPIO16 0x11 + #define NVM_CFG1_GLOB_RUNTIME_PORT_SWAP_GPIO_GPIO17 0x12 + #define NVM_CFG1_GLOB_RUNTIME_PORT_SWAP_GPIO_GPIO18 0x13 + #define NVM_CFG1_GLOB_RUNTIME_PORT_SWAP_GPIO_GPIO19 0x14 + #define NVM_CFG1_GLOB_RUNTIME_PORT_SWAP_GPIO_GPIO20 0x15 + #define NVM_CFG1_GLOB_RUNTIME_PORT_SWAP_GPIO_GPIO21 0x16 + #define NVM_CFG1_GLOB_RUNTIME_PORT_SWAP_GPIO_GPIO22 0x17 + #define NVM_CFG1_GLOB_RUNTIME_PORT_SWAP_GPIO_GPIO23 0x18 + #define NVM_CFG1_GLOB_RUNTIME_PORT_SWAP_GPIO_GPIO24 0x19 + #define NVM_CFG1_GLOB_RUNTIME_PORT_SWAP_GPIO_GPIO25 0x1A + #define NVM_CFG1_GLOB_RUNTIME_PORT_SWAP_GPIO_GPIO26 0x1B + #define NVM_CFG1_GLOB_RUNTIME_PORT_SWAP_GPIO_GPIO27 0x1C + #define NVM_CFG1_GLOB_RUNTIME_PORT_SWAP_GPIO_GPIO28 0x1D + #define NVM_CFG1_GLOB_RUNTIME_PORT_SWAP_GPIO_GPIO29 0x1E + #define NVM_CFG1_GLOB_RUNTIME_PORT_SWAP_GPIO_GPIO30 0x1F + #define NVM_CFG1_GLOB_RUNTIME_PORT_SWAP_GPIO_GPIO31 0x20 + u32 generic_cont1; /* 0x78 */ + #define NVM_CFG1_GLOB_AVS_DAC_CODE_MASK 0x000003FF + #define NVM_CFG1_GLOB_AVS_DAC_CODE_OFFSET 0 + #define NVM_CFG1_GLOB_LANE0_SWAP_MASK 0x00000C00 + #define NVM_CFG1_GLOB_LANE0_SWAP_OFFSET 10 + #define NVM_CFG1_GLOB_LANE1_SWAP_MASK 0x00003000 + #define NVM_CFG1_GLOB_LANE1_SWAP_OFFSET 12 + #define NVM_CFG1_GLOB_LANE2_SWAP_MASK 0x0000C000 + #define NVM_CFG1_GLOB_LANE2_SWAP_OFFSET 14 + #define NVM_CFG1_GLOB_LANE3_SWAP_MASK 0x00030000 + #define NVM_CFG1_GLOB_LANE3_SWAP_OFFSET 16 + /* Enable option 195 - Overriding the PCIe Preset value */ + #define NVM_CFG1_GLOB_OVERRIDE_PCIE_PRESET_EQUAL_MASK 0x00040000 + #define NVM_CFG1_GLOB_OVERRIDE_PCIE_PRESET_EQUAL_OFFSET 18 + #define NVM_CFG1_GLOB_OVERRIDE_PCIE_PRESET_EQUAL_DISABLED 0x0 + #define NVM_CFG1_GLOB_OVERRIDE_PCIE_PRESET_EQUAL_ENABLED 0x1 + /* PCIe Preset value - applies only if option 194 is enabled */ + #define NVM_CFG1_GLOB_PCIE_PRESET_VALUE_MASK 0x00780000 + #define NVM_CFG1_GLOB_PCIE_PRESET_VALUE_OFFSET 19 + /* Port mapping to be used when the run-time GPIO for port-swap is + * defined and set. + */ + #define NVM_CFG1_GLOB_RUNTIME_PORT0_SWAP_MAP_MASK 0x01800000 + #define NVM_CFG1_GLOB_RUNTIME_PORT0_SWAP_MAP_OFFSET 23 + #define NVM_CFG1_GLOB_RUNTIME_PORT1_SWAP_MAP_MASK 0x06000000 + #define NVM_CFG1_GLOB_RUNTIME_PORT1_SWAP_MAP_OFFSET 25 + #define NVM_CFG1_GLOB_RUNTIME_PORT2_SWAP_MAP_MASK 0x18000000 + #define NVM_CFG1_GLOB_RUNTIME_PORT2_SWAP_MAP_OFFSET 27 + #define NVM_CFG1_GLOB_RUNTIME_PORT3_SWAP_MAP_MASK 0x60000000 + #define NVM_CFG1_GLOB_RUNTIME_PORT3_SWAP_MAP_OFFSET 29 + u32 mbi_version; /* 0x7C */ + #define NVM_CFG1_GLOB_MBI_VERSION_0_MASK 0x000000FF + #define NVM_CFG1_GLOB_MBI_VERSION_0_OFFSET 0 + #define NVM_CFG1_GLOB_MBI_VERSION_1_MASK 0x0000FF00 + #define NVM_CFG1_GLOB_MBI_VERSION_1_OFFSET 8 + #define NVM_CFG1_GLOB_MBI_VERSION_2_MASK 0x00FF0000 + #define NVM_CFG1_GLOB_MBI_VERSION_2_OFFSET 16 + /* If set to other than NA, 0 - Normal operation, 1 - Thermal event + * occurred + */ + #define NVM_CFG1_GLOB_THERMAL_EVENT_GPIO_MASK 0xFF000000 + #define NVM_CFG1_GLOB_THERMAL_EVENT_GPIO_OFFSET 24 + #define NVM_CFG1_GLOB_THERMAL_EVENT_GPIO_NA 0x0 + #define NVM_CFG1_GLOB_THERMAL_EVENT_GPIO_GPIO0 0x1 + #define NVM_CFG1_GLOB_THERMAL_EVENT_GPIO_GPIO1 0x2 + #define NVM_CFG1_GLOB_THERMAL_EVENT_GPIO_GPIO2 0x3 + #define NVM_CFG1_GLOB_THERMAL_EVENT_GPIO_GPIO3 0x4 + #define NVM_CFG1_GLOB_THERMAL_EVENT_GPIO_GPIO4 0x5 + #define NVM_CFG1_GLOB_THERMAL_EVENT_GPIO_GPIO5 0x6 + #define NVM_CFG1_GLOB_THERMAL_EVENT_GPIO_GPIO6 0x7 + #define NVM_CFG1_GLOB_THERMAL_EVENT_GPIO_GPIO7 0x8 + #define NVM_CFG1_GLOB_THERMAL_EVENT_GPIO_GPIO8 0x9 + #define NVM_CFG1_GLOB_THERMAL_EVENT_GPIO_GPIO9 0xA + #define NVM_CFG1_GLOB_THERMAL_EVENT_GPIO_GPIO10 0xB + #define NVM_CFG1_GLOB_THERMAL_EVENT_GPIO_GPIO11 0xC + #define NVM_CFG1_GLOB_THERMAL_EVENT_GPIO_GPIO12 0xD + #define NVM_CFG1_GLOB_THERMAL_EVENT_GPIO_GPIO13 0xE + #define NVM_CFG1_GLOB_THERMAL_EVENT_GPIO_GPIO14 0xF + #define NVM_CFG1_GLOB_THERMAL_EVENT_GPIO_GPIO15 0x10 + #define NVM_CFG1_GLOB_THERMAL_EVENT_GPIO_GPIO16 0x11 + #define NVM_CFG1_GLOB_THERMAL_EVENT_GPIO_GPIO17 0x12 + #define NVM_CFG1_GLOB_THERMAL_EVENT_GPIO_GPIO18 0x13 + #define NVM_CFG1_GLOB_THERMAL_EVENT_GPIO_GPIO19 0x14 + #define NVM_CFG1_GLOB_THERMAL_EVENT_GPIO_GPIO20 0x15 + #define NVM_CFG1_GLOB_THERMAL_EVENT_GPIO_GPIO21 0x16 + #define NVM_CFG1_GLOB_THERMAL_EVENT_GPIO_GPIO22 0x17 + #define NVM_CFG1_GLOB_THERMAL_EVENT_GPIO_GPIO23 0x18 + #define NVM_CFG1_GLOB_THERMAL_EVENT_GPIO_GPIO24 0x19 + #define NVM_CFG1_GLOB_THERMAL_EVENT_GPIO_GPIO25 0x1A + #define NVM_CFG1_GLOB_THERMAL_EVENT_GPIO_GPIO26 0x1B + #define NVM_CFG1_GLOB_THERMAL_EVENT_GPIO_GPIO27 0x1C + #define NVM_CFG1_GLOB_THERMAL_EVENT_GPIO_GPIO28 0x1D + #define NVM_CFG1_GLOB_THERMAL_EVENT_GPIO_GPIO29 0x1E + #define NVM_CFG1_GLOB_THERMAL_EVENT_GPIO_GPIO30 0x1F + #define NVM_CFG1_GLOB_THERMAL_EVENT_GPIO_GPIO31 0x20 + u32 mbi_date; /* 0x80 */ + u32 misc_sig; /* 0x84 */ /* Define the GPIO mapping to switch i2c mux */ -#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO_0_MASK 0x000000FF -#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO_0_OFFSET 0 -#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO_1_MASK 0x0000FF00 -#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO_1_OFFSET 8 -#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__NA 0x0 -#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO0 0x1 -#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO1 0x2 -#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO2 0x3 -#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO3 0x4 -#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO4 0x5 -#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO5 0x6 -#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO6 0x7 -#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO7 0x8 -#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO8 0x9 -#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO9 0xA -#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO10 0xB -#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO11 0xC -#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO12 0xD -#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO13 0xE -#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO14 0xF -#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO15 0x10 -#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO16 0x11 -#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO17 0x12 -#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO18 0x13 -#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO19 0x14 -#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO20 0x15 -#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO21 0x16 -#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO22 0x17 -#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO23 0x18 -#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO24 0x19 -#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO25 0x1A -#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO26 0x1B -#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO27 0x1C -#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO28 0x1D -#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO29 0x1E -#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO30 0x1F -#define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO31 0x20 - u32 device_capabilities; /* 0x88 */ -#define NVM_CFG1_GLOB_DEVICE_CAPABILITIES_ETHERNET 0x1 - u32 power_dissipated; /* 0x8C */ -#define NVM_CFG1_GLOB_POWER_DIS_D0_MASK 0x000000FF -#define NVM_CFG1_GLOB_POWER_DIS_D0_OFFSET 0 -#define NVM_CFG1_GLOB_POWER_DIS_D1_MASK 0x0000FF00 -#define NVM_CFG1_GLOB_POWER_DIS_D1_OFFSET 8 -#define NVM_CFG1_GLOB_POWER_DIS_D2_MASK 0x00FF0000 -#define NVM_CFG1_GLOB_POWER_DIS_D2_OFFSET 16 -#define NVM_CFG1_GLOB_POWER_DIS_D3_MASK 0xFF000000 -#define NVM_CFG1_GLOB_POWER_DIS_D3_OFFSET 24 - u32 power_consumed; /* 0x90 */ -#define NVM_CFG1_GLOB_POWER_CONS_D0_MASK 0x000000FF -#define NVM_CFG1_GLOB_POWER_CONS_D0_OFFSET 0 -#define NVM_CFG1_GLOB_POWER_CONS_D1_MASK 0x0000FF00 -#define NVM_CFG1_GLOB_POWER_CONS_D1_OFFSET 8 -#define NVM_CFG1_GLOB_POWER_CONS_D2_MASK 0x00FF0000 -#define NVM_CFG1_GLOB_POWER_CONS_D2_OFFSET 16 -#define NVM_CFG1_GLOB_POWER_CONS_D3_MASK 0xFF000000 -#define NVM_CFG1_GLOB_POWER_CONS_D3_OFFSET 24 - u32 efi_version; /* 0x94 */ - u32 reserved[42]; /* 0x98 */ + #define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO_0_MASK 0x000000FF + #define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO_0_OFFSET 0 + #define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO_1_MASK 0x0000FF00 + #define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO_1_OFFSET 8 + #define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__NA 0x0 + #define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO0 0x1 + #define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO1 0x2 + #define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO2 0x3 + #define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO3 0x4 + #define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO4 0x5 + #define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO5 0x6 + #define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO6 0x7 + #define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO7 0x8 + #define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO8 0x9 + #define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO9 0xA + #define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO10 0xB + #define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO11 0xC + #define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO12 0xD + #define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO13 0xE + #define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO14 0xF + #define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO15 0x10 + #define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO16 0x11 + #define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO17 0x12 + #define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO18 0x13 + #define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO19 0x14 + #define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO20 0x15 + #define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO21 0x16 + #define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO22 0x17 + #define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO23 0x18 + #define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO24 0x19 + #define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO25 0x1A + #define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO26 0x1B + #define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO27 0x1C + #define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO28 0x1D + #define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO29 0x1E + #define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO30 0x1F + #define NVM_CFG1_GLOB_I2C_MUX_SEL_GPIO__GPIO31 0x20 + /* Interrupt signal used for SMBus/I2C management interface + * 0 = Interrupt event occurred + * 1 = Normal + */ + #define NVM_CFG1_GLOB_I2C_INTERRUPT_GPIO_MASK 0x00FF0000 + #define NVM_CFG1_GLOB_I2C_INTERRUPT_GPIO_OFFSET 16 + #define NVM_CFG1_GLOB_I2C_INTERRUPT_GPIO_NA 0x0 + #define NVM_CFG1_GLOB_I2C_INTERRUPT_GPIO_GPIO0 0x1 + #define NVM_CFG1_GLOB_I2C_INTERRUPT_GPIO_GPIO1 0x2 + #define NVM_CFG1_GLOB_I2C_INTERRUPT_GPIO_GPIO2 0x3 + #define NVM_CFG1_GLOB_I2C_INTERRUPT_GPIO_GPIO3 0x4 + #define NVM_CFG1_GLOB_I2C_INTERRUPT_GPIO_GPIO4 0x5 + #define NVM_CFG1_GLOB_I2C_INTERRUPT_GPIO_GPIO5 0x6 + #define NVM_CFG1_GLOB_I2C_INTERRUPT_GPIO_GPIO6 0x7 + #define NVM_CFG1_GLOB_I2C_INTERRUPT_GPIO_GPIO7 0x8 + #define NVM_CFG1_GLOB_I2C_INTERRUPT_GPIO_GPIO8 0x9 + #define NVM_CFG1_GLOB_I2C_INTERRUPT_GPIO_GPIO9 0xA + #define NVM_CFG1_GLOB_I2C_INTERRUPT_GPIO_GPIO10 0xB + #define NVM_CFG1_GLOB_I2C_INTERRUPT_GPIO_GPIO11 0xC + #define NVM_CFG1_GLOB_I2C_INTERRUPT_GPIO_GPIO12 0xD + #define NVM_CFG1_GLOB_I2C_INTERRUPT_GPIO_GPIO13 0xE + #define NVM_CFG1_GLOB_I2C_INTERRUPT_GPIO_GPIO14 0xF + #define NVM_CFG1_GLOB_I2C_INTERRUPT_GPIO_GPIO15 0x10 + #define NVM_CFG1_GLOB_I2C_INTERRUPT_GPIO_GPIO16 0x11 + #define NVM_CFG1_GLOB_I2C_INTERRUPT_GPIO_GPIO17 0x12 + #define NVM_CFG1_GLOB_I2C_INTERRUPT_GPIO_GPIO18 0x13 + #define NVM_CFG1_GLOB_I2C_INTERRUPT_GPIO_GPIO19 0x14 + #define NVM_CFG1_GLOB_I2C_INTERRUPT_GPIO_GPIO20 0x15 + #define NVM_CFG1_GLOB_I2C_INTERRUPT_GPIO_GPIO21 0x16 + #define NVM_CFG1_GLOB_I2C_INTERRUPT_GPIO_GPIO22 0x17 + #define NVM_CFG1_GLOB_I2C_INTERRUPT_GPIO_GPIO23 0x18 + #define NVM_CFG1_GLOB_I2C_INTERRUPT_GPIO_GPIO24 0x19 + #define NVM_CFG1_GLOB_I2C_INTERRUPT_GPIO_GPIO25 0x1A + #define NVM_CFG1_GLOB_I2C_INTERRUPT_GPIO_GPIO26 0x1B + #define NVM_CFG1_GLOB_I2C_INTERRUPT_GPIO_GPIO27 0x1C + #define NVM_CFG1_GLOB_I2C_INTERRUPT_GPIO_GPIO28 0x1D + #define NVM_CFG1_GLOB_I2C_INTERRUPT_GPIO_GPIO29 0x1E + #define NVM_CFG1_GLOB_I2C_INTERRUPT_GPIO_GPIO30 0x1F + #define NVM_CFG1_GLOB_I2C_INTERRUPT_GPIO_GPIO31 0x20 + /* Set aLOM FAN on GPIO */ + #define NVM_CFG1_GLOB_ALOM_FAN_ON_AUX_GPIO_MASK 0xFF000000 + #define NVM_CFG1_GLOB_ALOM_FAN_ON_AUX_GPIO_OFFSET 24 + #define NVM_CFG1_GLOB_ALOM_FAN_ON_AUX_GPIO_NA 0x0 + #define NVM_CFG1_GLOB_ALOM_FAN_ON_AUX_GPIO_GPIO0 0x1 + #define NVM_CFG1_GLOB_ALOM_FAN_ON_AUX_GPIO_GPIO1 0x2 + #define NVM_CFG1_GLOB_ALOM_FAN_ON_AUX_GPIO_GPIO2 0x3 + #define NVM_CFG1_GLOB_ALOM_FAN_ON_AUX_GPIO_GPIO3 0x4 + #define NVM_CFG1_GLOB_ALOM_FAN_ON_AUX_GPIO_GPIO4 0x5 + #define NVM_CFG1_GLOB_ALOM_FAN_ON_AUX_GPIO_GPIO5 0x6 + #define NVM_CFG1_GLOB_ALOM_FAN_ON_AUX_GPIO_GPIO6 0x7 + #define NVM_CFG1_GLOB_ALOM_FAN_ON_AUX_GPIO_GPIO7 0x8 + #define NVM_CFG1_GLOB_ALOM_FAN_ON_AUX_GPIO_GPIO8 0x9 + #define NVM_CFG1_GLOB_ALOM_FAN_ON_AUX_GPIO_GPIO9 0xA + #define NVM_CFG1_GLOB_ALOM_FAN_ON_AUX_GPIO_GPIO10 0xB + #define NVM_CFG1_GLOB_ALOM_FAN_ON_AUX_GPIO_GPIO11 0xC + #define NVM_CFG1_GLOB_ALOM_FAN_ON_AUX_GPIO_GPIO12 0xD + #define NVM_CFG1_GLOB_ALOM_FAN_ON_AUX_GPIO_GPIO13 0xE + #define NVM_CFG1_GLOB_ALOM_FAN_ON_AUX_GPIO_GPIO14 0xF + #define NVM_CFG1_GLOB_ALOM_FAN_ON_AUX_GPIO_GPIO15 0x10 + #define NVM_CFG1_GLOB_ALOM_FAN_ON_AUX_GPIO_GPIO16 0x11 + #define NVM_CFG1_GLOB_ALOM_FAN_ON_AUX_GPIO_GPIO17 0x12 + #define NVM_CFG1_GLOB_ALOM_FAN_ON_AUX_GPIO_GPIO18 0x13 + #define NVM_CFG1_GLOB_ALOM_FAN_ON_AUX_GPIO_GPIO19 0x14 + #define NVM_CFG1_GLOB_ALOM_FAN_ON_AUX_GPIO_GPIO20 0x15 + #define NVM_CFG1_GLOB_ALOM_FAN_ON_AUX_GPIO_GPIO21 0x16 + #define NVM_CFG1_GLOB_ALOM_FAN_ON_AUX_GPIO_GPIO22 0x17 + #define NVM_CFG1_GLOB_ALOM_FAN_ON_AUX_GPIO_GPIO23 0x18 + #define NVM_CFG1_GLOB_ALOM_FAN_ON_AUX_GPIO_GPIO24 0x19 + #define NVM_CFG1_GLOB_ALOM_FAN_ON_AUX_GPIO_GPIO25 0x1A + #define NVM_CFG1_GLOB_ALOM_FAN_ON_AUX_GPIO_GPIO26 0x1B + #define NVM_CFG1_GLOB_ALOM_FAN_ON_AUX_GPIO_GPIO27 0x1C + #define NVM_CFG1_GLOB_ALOM_FAN_ON_AUX_GPIO_GPIO28 0x1D + #define NVM_CFG1_GLOB_ALOM_FAN_ON_AUX_GPIO_GPIO29 0x1E + #define NVM_CFG1_GLOB_ALOM_FAN_ON_AUX_GPIO_GPIO30 0x1F + #define NVM_CFG1_GLOB_ALOM_FAN_ON_AUX_GPIO_GPIO31 0x20 + u32 device_capabilities; /* 0x88 */ + #define NVM_CFG1_GLOB_DEVICE_CAPABILITIES_ETHERNET 0x1 + #define NVM_CFG1_GLOB_DEVICE_CAPABILITIES_FCOE 0x2 + #define NVM_CFG1_GLOB_DEVICE_CAPABILITIES_ISCSI 0x4 + #define NVM_CFG1_GLOB_DEVICE_CAPABILITIES_ROCE 0x8 + #define NVM_CFG1_GLOB_DEVICE_CAPABILITIES_IWARP 0x10 + u32 power_dissipated; /* 0x8C */ + #define NVM_CFG1_GLOB_POWER_DIS_D0_MASK 0x000000FF + #define NVM_CFG1_GLOB_POWER_DIS_D0_OFFSET 0 + #define NVM_CFG1_GLOB_POWER_DIS_D1_MASK 0x0000FF00 + #define NVM_CFG1_GLOB_POWER_DIS_D1_OFFSET 8 + #define NVM_CFG1_GLOB_POWER_DIS_D2_MASK 0x00FF0000 + #define NVM_CFG1_GLOB_POWER_DIS_D2_OFFSET 16 + #define NVM_CFG1_GLOB_POWER_DIS_D3_MASK 0xFF000000 + #define NVM_CFG1_GLOB_POWER_DIS_D3_OFFSET 24 + u32 power_consumed; /* 0x90 */ + #define NVM_CFG1_GLOB_POWER_CONS_D0_MASK 0x000000FF + #define NVM_CFG1_GLOB_POWER_CONS_D0_OFFSET 0 + #define NVM_CFG1_GLOB_POWER_CONS_D1_MASK 0x0000FF00 + #define NVM_CFG1_GLOB_POWER_CONS_D1_OFFSET 8 + #define NVM_CFG1_GLOB_POWER_CONS_D2_MASK 0x00FF0000 + #define NVM_CFG1_GLOB_POWER_CONS_D2_OFFSET 16 + #define NVM_CFG1_GLOB_POWER_CONS_D3_MASK 0xFF000000 + #define NVM_CFG1_GLOB_POWER_CONS_D3_OFFSET 24 + u32 efi_version; /* 0x94 */ + u32 multi_network_modes_capability; /* 0x98 */ + #define NVM_CFG1_GLOB_MULTI_NETWORK_MODES_CAPABILITY_4X10G 0x1 + #define NVM_CFG1_GLOB_MULTI_NETWORK_MODES_CAPABILITY_1X25G 0x2 + #define NVM_CFG1_GLOB_MULTI_NETWORK_MODES_CAPABILITY_2X25G 0x4 + #define NVM_CFG1_GLOB_MULTI_NETWORK_MODES_CAPABILITY_4X25G 0x8 + #define NVM_CFG1_GLOB_MULTI_NETWORK_MODES_CAPABILITY_1X40G 0x10 + #define NVM_CFG1_GLOB_MULTI_NETWORK_MODES_CAPABILITY_2X40G 0x20 + #define NVM_CFG1_GLOB_MULTI_NETWORK_MODES_CAPABILITY_2X50G 0x40 + #define NVM_CFG1_GLOB_MULTI_NETWORK_MODES_CAPABILITY_BB_1X100G \ + 0x80 + #define NVM_CFG1_GLOB_MULTI_NETWORK_MODES_CAPABILITY_2X10G 0x100 + /* @DPDK */ + u32 reserved1[12]; /* 0x9C */ + u32 oem1_number[8]; /* 0xCC */ + u32 oem2_number[8]; /* 0xEC */ + u32 mps25_active_txfir_pre; /* 0x10C */ + #define NVM_CFG1_GLOB_LANE0_ACT_TXFIR_PRE_MASK 0x000000FF + #define NVM_CFG1_GLOB_LANE0_ACT_TXFIR_PRE_OFFSET 0 + #define NVM_CFG1_GLOB_LANE1_ACT_TXFIR_PRE_MASK 0x0000FF00 + #define NVM_CFG1_GLOB_LANE1_ACT_TXFIR_PRE_OFFSET 8 + #define NVM_CFG1_GLOB_LANE2_ACT_TXFIR_PRE_MASK 0x00FF0000 + #define NVM_CFG1_GLOB_LANE2_ACT_TXFIR_PRE_OFFSET 16 + #define NVM_CFG1_GLOB_LANE3_ACT_TXFIR_PRE_MASK 0xFF000000 + #define NVM_CFG1_GLOB_LANE3_ACT_TXFIR_PRE_OFFSET 24 + u32 mps25_active_txfir_main; /* 0x110 */ + #define NVM_CFG1_GLOB_LANE0_ACT_TXFIR_MAIN_MASK 0x000000FF + #define NVM_CFG1_GLOB_LANE0_ACT_TXFIR_MAIN_OFFSET 0 + #define NVM_CFG1_GLOB_LANE1_ACT_TXFIR_MAIN_MASK 0x0000FF00 + #define NVM_CFG1_GLOB_LANE1_ACT_TXFIR_MAIN_OFFSET 8 + #define NVM_CFG1_GLOB_LANE2_ACT_TXFIR_MAIN_MASK 0x00FF0000 + #define NVM_CFG1_GLOB_LANE2_ACT_TXFIR_MAIN_OFFSET 16 + #define NVM_CFG1_GLOB_LANE3_ACT_TXFIR_MAIN_MASK 0xFF000000 + #define NVM_CFG1_GLOB_LANE3_ACT_TXFIR_MAIN_OFFSET 24 + u32 mps25_active_txfir_post; /* 0x114 */ + #define NVM_CFG1_GLOB_LANE0_ACT_TXFIR_POST_MASK 0x000000FF + #define NVM_CFG1_GLOB_LANE0_ACT_TXFIR_POST_OFFSET 0 + #define NVM_CFG1_GLOB_LANE1_ACT_TXFIR_POST_MASK 0x0000FF00 + #define NVM_CFG1_GLOB_LANE1_ACT_TXFIR_POST_OFFSET 8 + #define NVM_CFG1_GLOB_LANE2_ACT_TXFIR_POST_MASK 0x00FF0000 + #define NVM_CFG1_GLOB_LANE2_ACT_TXFIR_POST_OFFSET 16 + #define NVM_CFG1_GLOB_LANE3_ACT_TXFIR_POST_MASK 0xFF000000 + #define NVM_CFG1_GLOB_LANE3_ACT_TXFIR_POST_OFFSET 24 + u32 features; /* 0x118 */ + /* Set the Aux Fan on temperature */ + #define NVM_CFG1_GLOB_ALOM_FAN_ON_AUX_VALUE_MASK 0x000000FF + #define NVM_CFG1_GLOB_ALOM_FAN_ON_AUX_VALUE_OFFSET 0 + /* Set NC-SI package ID */ + #define NVM_CFG1_GLOB_SLOT_ID_GPIO_MASK 0x0000FF00 + #define NVM_CFG1_GLOB_SLOT_ID_GPIO_OFFSET 8 + #define NVM_CFG1_GLOB_SLOT_ID_GPIO_NA 0x0 + #define NVM_CFG1_GLOB_SLOT_ID_GPIO_GPIO0 0x1 + #define NVM_CFG1_GLOB_SLOT_ID_GPIO_GPIO1 0x2 + #define NVM_CFG1_GLOB_SLOT_ID_GPIO_GPIO2 0x3 + #define NVM_CFG1_GLOB_SLOT_ID_GPIO_GPIO3 0x4 + #define NVM_CFG1_GLOB_SLOT_ID_GPIO_GPIO4 0x5 + #define NVM_CFG1_GLOB_SLOT_ID_GPIO_GPIO5 0x6 + #define NVM_CFG1_GLOB_SLOT_ID_GPIO_GPIO6 0x7 + #define NVM_CFG1_GLOB_SLOT_ID_GPIO_GPIO7 0x8 + #define NVM_CFG1_GLOB_SLOT_ID_GPIO_GPIO8 0x9 + #define NVM_CFG1_GLOB_SLOT_ID_GPIO_GPIO9 0xA + #define NVM_CFG1_GLOB_SLOT_ID_GPIO_GPIO10 0xB + #define NVM_CFG1_GLOB_SLOT_ID_GPIO_GPIO11 0xC + #define NVM_CFG1_GLOB_SLOT_ID_GPIO_GPIO12 0xD + #define NVM_CFG1_GLOB_SLOT_ID_GPIO_GPIO13 0xE + #define NVM_CFG1_GLOB_SLOT_ID_GPIO_GPIO14 0xF + #define NVM_CFG1_GLOB_SLOT_ID_GPIO_GPIO15 0x10 + #define NVM_CFG1_GLOB_SLOT_ID_GPIO_GPIO16 0x11 + #define NVM_CFG1_GLOB_SLOT_ID_GPIO_GPIO17 0x12 + #define NVM_CFG1_GLOB_SLOT_ID_GPIO_GPIO18 0x13 + #define NVM_CFG1_GLOB_SLOT_ID_GPIO_GPIO19 0x14 + #define NVM_CFG1_GLOB_SLOT_ID_GPIO_GPIO20 0x15 + #define NVM_CFG1_GLOB_SLOT_ID_GPIO_GPIO21 0x16 + #define NVM_CFG1_GLOB_SLOT_ID_GPIO_GPIO22 0x17 + #define NVM_CFG1_GLOB_SLOT_ID_GPIO_GPIO23 0x18 + #define NVM_CFG1_GLOB_SLOT_ID_GPIO_GPIO24 0x19 + #define NVM_CFG1_GLOB_SLOT_ID_GPIO_GPIO25 0x1A + #define NVM_CFG1_GLOB_SLOT_ID_GPIO_GPIO26 0x1B + #define NVM_CFG1_GLOB_SLOT_ID_GPIO_GPIO27 0x1C + #define NVM_CFG1_GLOB_SLOT_ID_GPIO_GPIO28 0x1D + #define NVM_CFG1_GLOB_SLOT_ID_GPIO_GPIO29 0x1E + #define NVM_CFG1_GLOB_SLOT_ID_GPIO_GPIO30 0x1F + #define NVM_CFG1_GLOB_SLOT_ID_GPIO_GPIO31 0x20 + /* PMBUS Clock GPIO */ + #define NVM_CFG1_GLOB_PMBUS_SCL_GPIO_MASK 0x00FF0000 + #define NVM_CFG1_GLOB_PMBUS_SCL_GPIO_OFFSET 16 + #define NVM_CFG1_GLOB_PMBUS_SCL_GPIO_NA 0x0 + #define NVM_CFG1_GLOB_PMBUS_SCL_GPIO_GPIO0 0x1 + #define NVM_CFG1_GLOB_PMBUS_SCL_GPIO_GPIO1 0x2 + #define NVM_CFG1_GLOB_PMBUS_SCL_GPIO_GPIO2 0x3 + #define NVM_CFG1_GLOB_PMBUS_SCL_GPIO_GPIO3 0x4 + #define NVM_CFG1_GLOB_PMBUS_SCL_GPIO_GPIO4 0x5 + #define NVM_CFG1_GLOB_PMBUS_SCL_GPIO_GPIO5 0x6 + #define NVM_CFG1_GLOB_PMBUS_SCL_GPIO_GPIO6 0x7 + #define NVM_CFG1_GLOB_PMBUS_SCL_GPIO_GPIO7 0x8 + #define NVM_CFG1_GLOB_PMBUS_SCL_GPIO_GPIO8 0x9 + #define NVM_CFG1_GLOB_PMBUS_SCL_GPIO_GPIO9 0xA + #define NVM_CFG1_GLOB_PMBUS_SCL_GPIO_GPIO10 0xB + #define NVM_CFG1_GLOB_PMBUS_SCL_GPIO_GPIO11 0xC + #define NVM_CFG1_GLOB_PMBUS_SCL_GPIO_GPIO12 0xD + #define NVM_CFG1_GLOB_PMBUS_SCL_GPIO_GPIO13 0xE + #define NVM_CFG1_GLOB_PMBUS_SCL_GPIO_GPIO14 0xF + #define NVM_CFG1_GLOB_PMBUS_SCL_GPIO_GPIO15 0x10 + #define NVM_CFG1_GLOB_PMBUS_SCL_GPIO_GPIO16 0x11 + #define NVM_CFG1_GLOB_PMBUS_SCL_GPIO_GPIO17 0x12 + #define NVM_CFG1_GLOB_PMBUS_SCL_GPIO_GPIO18 0x13 + #define NVM_CFG1_GLOB_PMBUS_SCL_GPIO_GPIO19 0x14 + #define NVM_CFG1_GLOB_PMBUS_SCL_GPIO_GPIO20 0x15 + #define NVM_CFG1_GLOB_PMBUS_SCL_GPIO_GPIO21 0x16 + #define NVM_CFG1_GLOB_PMBUS_SCL_GPIO_GPIO22 0x17 + #define NVM_CFG1_GLOB_PMBUS_SCL_GPIO_GPIO23 0x18 + #define NVM_CFG1_GLOB_PMBUS_SCL_GPIO_GPIO24 0x19 + #define NVM_CFG1_GLOB_PMBUS_SCL_GPIO_GPIO25 0x1A + #define NVM_CFG1_GLOB_PMBUS_SCL_GPIO_GPIO26 0x1B + #define NVM_CFG1_GLOB_PMBUS_SCL_GPIO_GPIO27 0x1C + #define NVM_CFG1_GLOB_PMBUS_SCL_GPIO_GPIO28 0x1D + #define NVM_CFG1_GLOB_PMBUS_SCL_GPIO_GPIO29 0x1E + #define NVM_CFG1_GLOB_PMBUS_SCL_GPIO_GPIO30 0x1F + #define NVM_CFG1_GLOB_PMBUS_SCL_GPIO_GPIO31 0x20 + /* PMBUS Data GPIO */ + #define NVM_CFG1_GLOB_PMBUS_SDA_GPIO_MASK 0xFF000000 + #define NVM_CFG1_GLOB_PMBUS_SDA_GPIO_OFFSET 24 + #define NVM_CFG1_GLOB_PMBUS_SDA_GPIO_NA 0x0 + #define NVM_CFG1_GLOB_PMBUS_SDA_GPIO_GPIO0 0x1 + #define NVM_CFG1_GLOB_PMBUS_SDA_GPIO_GPIO1 0x2 + #define NVM_CFG1_GLOB_PMBUS_SDA_GPIO_GPIO2 0x3 + #define NVM_CFG1_GLOB_PMBUS_SDA_GPIO_GPIO3 0x4 + #define NVM_CFG1_GLOB_PMBUS_SDA_GPIO_GPIO4 0x5 + #define NVM_CFG1_GLOB_PMBUS_SDA_GPIO_GPIO5 0x6 + #define NVM_CFG1_GLOB_PMBUS_SDA_GPIO_GPIO6 0x7 + #define NVM_CFG1_GLOB_PMBUS_SDA_GPIO_GPIO7 0x8 + #define NVM_CFG1_GLOB_PMBUS_SDA_GPIO_GPIO8 0x9 + #define NVM_CFG1_GLOB_PMBUS_SDA_GPIO_GPIO9 0xA + #define NVM_CFG1_GLOB_PMBUS_SDA_GPIO_GPIO10 0xB + #define NVM_CFG1_GLOB_PMBUS_SDA_GPIO_GPIO11 0xC + #define NVM_CFG1_GLOB_PMBUS_SDA_GPIO_GPIO12 0xD + #define NVM_CFG1_GLOB_PMBUS_SDA_GPIO_GPIO13 0xE + #define NVM_CFG1_GLOB_PMBUS_SDA_GPIO_GPIO14 0xF + #define NVM_CFG1_GLOB_PMBUS_SDA_GPIO_GPIO15 0x10 + #define NVM_CFG1_GLOB_PMBUS_SDA_GPIO_GPIO16 0x11 + #define NVM_CFG1_GLOB_PMBUS_SDA_GPIO_GPIO17 0x12 + #define NVM_CFG1_GLOB_PMBUS_SDA_GPIO_GPIO18 0x13 + #define NVM_CFG1_GLOB_PMBUS_SDA_GPIO_GPIO19 0x14 + #define NVM_CFG1_GLOB_PMBUS_SDA_GPIO_GPIO20 0x15 + #define NVM_CFG1_GLOB_PMBUS_SDA_GPIO_GPIO21 0x16 + #define NVM_CFG1_GLOB_PMBUS_SDA_GPIO_GPIO22 0x17 + #define NVM_CFG1_GLOB_PMBUS_SDA_GPIO_GPIO23 0x18 + #define NVM_CFG1_GLOB_PMBUS_SDA_GPIO_GPIO24 0x19 + #define NVM_CFG1_GLOB_PMBUS_SDA_GPIO_GPIO25 0x1A + #define NVM_CFG1_GLOB_PMBUS_SDA_GPIO_GPIO26 0x1B + #define NVM_CFG1_GLOB_PMBUS_SDA_GPIO_GPIO27 0x1C + #define NVM_CFG1_GLOB_PMBUS_SDA_GPIO_GPIO28 0x1D + #define NVM_CFG1_GLOB_PMBUS_SDA_GPIO_GPIO29 0x1E + #define NVM_CFG1_GLOB_PMBUS_SDA_GPIO_GPIO30 0x1F + #define NVM_CFG1_GLOB_PMBUS_SDA_GPIO_GPIO31 0x20 + u32 tx_rx_eq_25g_hlpc; /* 0x11C */ + #define NVM_CFG1_GLOB_INDEX0_RX_TX_EQ_25G_HLPC_MASK 0x000000FF + #define NVM_CFG1_GLOB_INDEX0_RX_TX_EQ_25G_HLPC_OFFSET 0 + #define NVM_CFG1_GLOB_INDEX1_RX_TX_EQ_25G_HLPC_MASK 0x0000FF00 + #define NVM_CFG1_GLOB_INDEX1_RX_TX_EQ_25G_HLPC_OFFSET 8 + #define NVM_CFG1_GLOB_INDEX2_RX_TX_EQ_25G_HLPC_MASK 0x00FF0000 + #define NVM_CFG1_GLOB_INDEX2_RX_TX_EQ_25G_HLPC_OFFSET 16 + #define NVM_CFG1_GLOB_INDEX3_RX_TX_EQ_25G_HLPC_MASK 0xFF000000 + #define NVM_CFG1_GLOB_INDEX3_RX_TX_EQ_25G_HLPC_OFFSET 24 + u32 tx_rx_eq_25g_llpc; /* 0x120 */ + #define NVM_CFG1_GLOB_INDEX0_RX_TX_EQ_25G_LLPC_MASK 0x000000FF + #define NVM_CFG1_GLOB_INDEX0_RX_TX_EQ_25G_LLPC_OFFSET 0 + #define NVM_CFG1_GLOB_INDEX1_RX_TX_EQ_25G_LLPC_MASK 0x0000FF00 + #define NVM_CFG1_GLOB_INDEX1_RX_TX_EQ_25G_LLPC_OFFSET 8 + #define NVM_CFG1_GLOB_INDEX2_RX_TX_EQ_25G_LLPC_MASK 0x00FF0000 + #define NVM_CFG1_GLOB_INDEX2_RX_TX_EQ_25G_LLPC_OFFSET 16 + #define NVM_CFG1_GLOB_INDEX3_RX_TX_EQ_25G_LLPC_MASK 0xFF000000 + #define NVM_CFG1_GLOB_INDEX3_RX_TX_EQ_25G_LLPC_OFFSET 24 + u32 tx_rx_eq_25g_ac; /* 0x124 */ + #define NVM_CFG1_GLOB_INDEX0_RX_TX_EQ_25G_AC_MASK 0x000000FF + #define NVM_CFG1_GLOB_INDEX0_RX_TX_EQ_25G_AC_OFFSET 0 + #define NVM_CFG1_GLOB_INDEX1_RX_TX_EQ_25G_AC_MASK 0x0000FF00 + #define NVM_CFG1_GLOB_INDEX1_RX_TX_EQ_25G_AC_OFFSET 8 + #define NVM_CFG1_GLOB_INDEX2_RX_TX_EQ_25G_AC_MASK 0x00FF0000 + #define NVM_CFG1_GLOB_INDEX2_RX_TX_EQ_25G_AC_OFFSET 16 + #define NVM_CFG1_GLOB_INDEX3_RX_TX_EQ_25G_AC_MASK 0xFF000000 + #define NVM_CFG1_GLOB_INDEX3_RX_TX_EQ_25G_AC_OFFSET 24 + u32 tx_rx_eq_10g_pc; /* 0x128 */ + #define NVM_CFG1_GLOB_INDEX0_RX_TX_EQ_10G_PC_MASK 0x000000FF + #define NVM_CFG1_GLOB_INDEX0_RX_TX_EQ_10G_PC_OFFSET 0 + #define NVM_CFG1_GLOB_INDEX1_RX_TX_EQ_10G_PC_MASK 0x0000FF00 + #define NVM_CFG1_GLOB_INDEX1_RX_TX_EQ_10G_PC_OFFSET 8 + #define NVM_CFG1_GLOB_INDEX2_RX_TX_EQ_10G_PC_MASK 0x00FF0000 + #define NVM_CFG1_GLOB_INDEX2_RX_TX_EQ_10G_PC_OFFSET 16 + #define NVM_CFG1_GLOB_INDEX3_RX_TX_EQ_10G_PC_MASK 0xFF000000 + #define NVM_CFG1_GLOB_INDEX3_RX_TX_EQ_10G_PC_OFFSET 24 + u32 tx_rx_eq_10g_ac; /* 0x12C */ + #define NVM_CFG1_GLOB_INDEX0_RX_TX_EQ_10G_AC_MASK 0x000000FF + #define NVM_CFG1_GLOB_INDEX0_RX_TX_EQ_10G_AC_OFFSET 0 + #define NVM_CFG1_GLOB_INDEX1_RX_TX_EQ_10G_AC_MASK 0x0000FF00 + #define NVM_CFG1_GLOB_INDEX1_RX_TX_EQ_10G_AC_OFFSET 8 + #define NVM_CFG1_GLOB_INDEX2_RX_TX_EQ_10G_AC_MASK 0x00FF0000 + #define NVM_CFG1_GLOB_INDEX2_RX_TX_EQ_10G_AC_OFFSET 16 + #define NVM_CFG1_GLOB_INDEX3_RX_TX_EQ_10G_AC_MASK 0xFF000000 + #define NVM_CFG1_GLOB_INDEX3_RX_TX_EQ_10G_AC_OFFSET 24 + u32 tx_rx_eq_1g; /* 0x130 */ + #define NVM_CFG1_GLOB_INDEX0_RX_TX_EQ_1G_MASK 0x000000FF + #define NVM_CFG1_GLOB_INDEX0_RX_TX_EQ_1G_OFFSET 0 + #define NVM_CFG1_GLOB_INDEX1_RX_TX_EQ_1G_MASK 0x0000FF00 + #define NVM_CFG1_GLOB_INDEX1_RX_TX_EQ_1G_OFFSET 8 + #define NVM_CFG1_GLOB_INDEX2_RX_TX_EQ_1G_MASK 0x00FF0000 + #define NVM_CFG1_GLOB_INDEX2_RX_TX_EQ_1G_OFFSET 16 + #define NVM_CFG1_GLOB_INDEX3_RX_TX_EQ_1G_MASK 0xFF000000 + #define NVM_CFG1_GLOB_INDEX3_RX_TX_EQ_1G_OFFSET 24 + u32 tx_rx_eq_25g_bt; /* 0x134 */ + #define NVM_CFG1_GLOB_INDEX0_RX_TX_EQ_25G_BT_MASK 0x000000FF + #define NVM_CFG1_GLOB_INDEX0_RX_TX_EQ_25G_BT_OFFSET 0 + #define NVM_CFG1_GLOB_INDEX1_RX_TX_EQ_25G_BT_MASK 0x0000FF00 + #define NVM_CFG1_GLOB_INDEX1_RX_TX_EQ_25G_BT_OFFSET 8 + #define NVM_CFG1_GLOB_INDEX2_RX_TX_EQ_25G_BT_MASK 0x00FF0000 + #define NVM_CFG1_GLOB_INDEX2_RX_TX_EQ_25G_BT_OFFSET 16 + #define NVM_CFG1_GLOB_INDEX3_RX_TX_EQ_25G_BT_MASK 0xFF000000 + #define NVM_CFG1_GLOB_INDEX3_RX_TX_EQ_25G_BT_OFFSET 24 + u32 tx_rx_eq_10g_bt; /* 0x138 */ + #define NVM_CFG1_GLOB_INDEX0_RX_TX_EQ_10G_BT_MASK 0x000000FF + #define NVM_CFG1_GLOB_INDEX0_RX_TX_EQ_10G_BT_OFFSET 0 + #define NVM_CFG1_GLOB_INDEX1_RX_TX_EQ_10G_BT_MASK 0x0000FF00 + #define NVM_CFG1_GLOB_INDEX1_RX_TX_EQ_10G_BT_OFFSET 8 + #define NVM_CFG1_GLOB_INDEX2_RX_TX_EQ_10G_BT_MASK 0x00FF0000 + #define NVM_CFG1_GLOB_INDEX2_RX_TX_EQ_10G_BT_OFFSET 16 + #define NVM_CFG1_GLOB_INDEX3_RX_TX_EQ_10G_BT_MASK 0xFF000000 + #define NVM_CFG1_GLOB_INDEX3_RX_TX_EQ_10G_BT_OFFSET 24 + u32 generic_cont4; /* 0x13C */ + #define NVM_CFG1_GLOB_THERMAL_ALARM_GPIO_MASK 0x000000FF + #define NVM_CFG1_GLOB_THERMAL_ALARM_GPIO_OFFSET 0 + #define NVM_CFG1_GLOB_THERMAL_ALARM_GPIO_NA 0x0 + #define NVM_CFG1_GLOB_THERMAL_ALARM_GPIO_GPIO0 0x1 + #define NVM_CFG1_GLOB_THERMAL_ALARM_GPIO_GPIO1 0x2 + #define NVM_CFG1_GLOB_THERMAL_ALARM_GPIO_GPIO2 0x3 + #define NVM_CFG1_GLOB_THERMAL_ALARM_GPIO_GPIO3 0x4 + #define NVM_CFG1_GLOB_THERMAL_ALARM_GPIO_GPIO4 0x5 + #define NVM_CFG1_GLOB_THERMAL_ALARM_GPIO_GPIO5 0x6 + #define NVM_CFG1_GLOB_THERMAL_ALARM_GPIO_GPIO6 0x7 + #define NVM_CFG1_GLOB_THERMAL_ALARM_GPIO_GPIO7 0x8 + #define NVM_CFG1_GLOB_THERMAL_ALARM_GPIO_GPIO8 0x9 + #define NVM_CFG1_GLOB_THERMAL_ALARM_GPIO_GPIO9 0xA + #define NVM_CFG1_GLOB_THERMAL_ALARM_GPIO_GPIO10 0xB + #define NVM_CFG1_GLOB_THERMAL_ALARM_GPIO_GPIO11 0xC + #define NVM_CFG1_GLOB_THERMAL_ALARM_GPIO_GPIO12 0xD + #define NVM_CFG1_GLOB_THERMAL_ALARM_GPIO_GPIO13 0xE + #define NVM_CFG1_GLOB_THERMAL_ALARM_GPIO_GPIO14 0xF + #define NVM_CFG1_GLOB_THERMAL_ALARM_GPIO_GPIO15 0x10 + #define NVM_CFG1_GLOB_THERMAL_ALARM_GPIO_GPIO16 0x11 + #define NVM_CFG1_GLOB_THERMAL_ALARM_GPIO_GPIO17 0x12 + #define NVM_CFG1_GLOB_THERMAL_ALARM_GPIO_GPIO18 0x13 + #define NVM_CFG1_GLOB_THERMAL_ALARM_GPIO_GPIO19 0x14 + #define NVM_CFG1_GLOB_THERMAL_ALARM_GPIO_GPIO20 0x15 + #define NVM_CFG1_GLOB_THERMAL_ALARM_GPIO_GPIO21 0x16 + #define NVM_CFG1_GLOB_THERMAL_ALARM_GPIO_GPIO22 0x17 + #define NVM_CFG1_GLOB_THERMAL_ALARM_GPIO_GPIO23 0x18 + #define NVM_CFG1_GLOB_THERMAL_ALARM_GPIO_GPIO24 0x19 + #define NVM_CFG1_GLOB_THERMAL_ALARM_GPIO_GPIO25 0x1A + #define NVM_CFG1_GLOB_THERMAL_ALARM_GPIO_GPIO26 0x1B + #define NVM_CFG1_GLOB_THERMAL_ALARM_GPIO_GPIO27 0x1C + #define NVM_CFG1_GLOB_THERMAL_ALARM_GPIO_GPIO28 0x1D + #define NVM_CFG1_GLOB_THERMAL_ALARM_GPIO_GPIO29 0x1E + #define NVM_CFG1_GLOB_THERMAL_ALARM_GPIO_GPIO30 0x1F + #define NVM_CFG1_GLOB_THERMAL_ALARM_GPIO_GPIO31 0x20 + u32 preboot_debug_mode_std; /* 0x140 */ + u32 preboot_debug_mode_ext; /* 0x144 */ + u32 ext_phy_cfg1; /* 0x148 */ + /* Ext PHY MDI pair swap value */ + #define NVM_CFG1_GLOB_EXT_PHY_MDI_PAIR_SWAP_MASK 0x0000FFFF + #define NVM_CFG1_GLOB_EXT_PHY_MDI_PAIR_SWAP_OFFSET 0 + u32 reserved[55]; /* 0x14C */ }; struct nvm_cfg1_path { - u32 reserved[30]; /* 0x0 */ + u32 reserved[1]; /* 0x0 */ }; struct nvm_cfg1_port { - u32 reserved__m_relocated_to_option_123; /* 0x0 */ - u32 reserved__m_relocated_to_option_124; /* 0x4 */ - u32 generic_cont0; /* 0x8 */ -#define NVM_CFG1_PORT_LED_MODE_MASK 0x000000FF -#define NVM_CFG1_PORT_LED_MODE_OFFSET 0 -#define NVM_CFG1_PORT_LED_MODE_MAC1 0x0 -#define NVM_CFG1_PORT_LED_MODE_PHY1 0x1 -#define NVM_CFG1_PORT_LED_MODE_PHY2 0x2 -#define NVM_CFG1_PORT_LED_MODE_PHY3 0x3 -#define NVM_CFG1_PORT_LED_MODE_MAC2 0x4 -#define NVM_CFG1_PORT_LED_MODE_PHY4 0x5 -#define NVM_CFG1_PORT_LED_MODE_PHY5 0x6 -#define NVM_CFG1_PORT_LED_MODE_PHY6 0x7 -#define NVM_CFG1_PORT_LED_MODE_MAC3 0x8 -#define NVM_CFG1_PORT_LED_MODE_PHY7 0x9 -#define NVM_CFG1_PORT_LED_MODE_PHY8 0xA -#define NVM_CFG1_PORT_LED_MODE_PHY9 0xB -#define NVM_CFG1_PORT_LED_MODE_MAC4 0xC -#define NVM_CFG1_PORT_LED_MODE_PHY10 0xD -#define NVM_CFG1_PORT_LED_MODE_PHY11 0xE -#define NVM_CFG1_PORT_LED_MODE_PHY12 0xF -#define NVM_CFG1_PORT_DCBX_MODE_MASK 0x000F0000 -#define NVM_CFG1_PORT_DCBX_MODE_OFFSET 16 -#define NVM_CFG1_PORT_DCBX_MODE_DISABLED 0x0 -#define NVM_CFG1_PORT_DCBX_MODE_IEEE 0x1 -#define NVM_CFG1_PORT_DCBX_MODE_CEE 0x2 -#define NVM_CFG1_PORT_DCBX_MODE_DYNAMIC 0x3 -#define NVM_CFG1_PORT_DEFAULT_ENABLED_PROTOCOLS_MASK 0x00F00000 -#define NVM_CFG1_PORT_DEFAULT_ENABLED_PROTOCOLS_OFFSET 20 -#define NVM_CFG1_PORT_DEFAULT_ENABLED_PROTOCOLS_ETHERNET 0x1 - u32 pcie_cfg; /* 0xC */ -#define NVM_CFG1_PORT_RESERVED15_MASK 0x00000007 -#define NVM_CFG1_PORT_RESERVED15_OFFSET 0 - u32 features; /* 0x10 */ -#define NVM_CFG1_PORT_ENABLE_WOL_ON_ACPI_PATTERN_MASK 0x00000001 -#define NVM_CFG1_PORT_ENABLE_WOL_ON_ACPI_PATTERN_OFFSET 0 -#define NVM_CFG1_PORT_ENABLE_WOL_ON_ACPI_PATTERN_DISABLED 0x0 -#define NVM_CFG1_PORT_ENABLE_WOL_ON_ACPI_PATTERN_ENABLED 0x1 -#define NVM_CFG1_PORT_MAGIC_PACKET_WOL_MASK 0x00000002 -#define NVM_CFG1_PORT_MAGIC_PACKET_WOL_OFFSET 1 -#define NVM_CFG1_PORT_MAGIC_PACKET_WOL_DISABLED 0x0 -#define NVM_CFG1_PORT_MAGIC_PACKET_WOL_ENABLED 0x1 - u32 speed_cap_mask; /* 0x14 */ -#define NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_MASK 0x0000FFFF -#define NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_OFFSET 0 -#define NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_1G 0x1 -#define NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_10G 0x2 -#define NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_25G 0x8 -#define NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_40G 0x10 -#define NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_50G 0x20 -#define NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_100G 0x40 -#define NVM_CFG1_PORT_MFW_SPEED_CAPABILITY_MASK_MASK 0xFFFF0000 -#define NVM_CFG1_PORT_MFW_SPEED_CAPABILITY_MASK_OFFSET 16 -#define NVM_CFG1_PORT_MFW_SPEED_CAPABILITY_MASK_1G 0x1 -#define NVM_CFG1_PORT_MFW_SPEED_CAPABILITY_MASK_10G 0x2 -#define NVM_CFG1_PORT_MFW_SPEED_CAPABILITY_MASK_25G 0x8 -#define NVM_CFG1_PORT_MFW_SPEED_CAPABILITY_MASK_40G 0x10 -#define NVM_CFG1_PORT_MFW_SPEED_CAPABILITY_MASK_50G 0x20 -#define NVM_CFG1_PORT_MFW_SPEED_CAPABILITY_MASK_100G 0x40 - u32 link_settings; /* 0x18 */ -#define NVM_CFG1_PORT_DRV_LINK_SPEED_MASK 0x0000000F -#define NVM_CFG1_PORT_DRV_LINK_SPEED_OFFSET 0 -#define NVM_CFG1_PORT_DRV_LINK_SPEED_AUTONEG 0x0 -#define NVM_CFG1_PORT_DRV_LINK_SPEED_1G 0x1 -#define NVM_CFG1_PORT_DRV_LINK_SPEED_10G 0x2 -#define NVM_CFG1_PORT_DRV_LINK_SPEED_25G 0x4 -#define NVM_CFG1_PORT_DRV_LINK_SPEED_40G 0x5 -#define NVM_CFG1_PORT_DRV_LINK_SPEED_50G 0x6 -#define NVM_CFG1_PORT_DRV_LINK_SPEED_100G 0x7 -#define NVM_CFG1_PORT_DRV_LINK_SPEED_SMARTLINQ 0x8 -#define NVM_CFG1_PORT_DRV_FLOW_CONTROL_MASK 0x00000070 -#define NVM_CFG1_PORT_DRV_FLOW_CONTROL_OFFSET 4 -#define NVM_CFG1_PORT_DRV_FLOW_CONTROL_AUTONEG 0x1 -#define NVM_CFG1_PORT_DRV_FLOW_CONTROL_RX 0x2 -#define NVM_CFG1_PORT_DRV_FLOW_CONTROL_TX 0x4 -#define NVM_CFG1_PORT_MFW_LINK_SPEED_MASK 0x00000780 -#define NVM_CFG1_PORT_MFW_LINK_SPEED_OFFSET 7 -#define NVM_CFG1_PORT_MFW_LINK_SPEED_AUTONEG 0x0 -#define NVM_CFG1_PORT_MFW_LINK_SPEED_1G 0x1 -#define NVM_CFG1_PORT_MFW_LINK_SPEED_10G 0x2 -#define NVM_CFG1_PORT_MFW_LINK_SPEED_25G 0x4 -#define NVM_CFG1_PORT_MFW_LINK_SPEED_40G 0x5 -#define NVM_CFG1_PORT_MFW_LINK_SPEED_50G 0x6 -#define NVM_CFG1_PORT_MFW_LINK_SPEED_100G 0x7 -#define NVM_CFG1_PORT_MFW_LINK_SPEED_SMARTLINQ 0x8 -#define NVM_CFG1_PORT_MFW_FLOW_CONTROL_MASK 0x00003800 -#define NVM_CFG1_PORT_MFW_FLOW_CONTROL_OFFSET 11 -#define NVM_CFG1_PORT_MFW_FLOW_CONTROL_AUTONEG 0x1 -#define NVM_CFG1_PORT_MFW_FLOW_CONTROL_RX 0x2 -#define NVM_CFG1_PORT_MFW_FLOW_CONTROL_TX 0x4 -#define NVM_CFG1_PORT_OPTIC_MODULE_VENDOR_ENFORCEMENT_MASK 0x00004000 -#define NVM_CFG1_PORT_OPTIC_MODULE_VENDOR_ENFORCEMENT_OFFSET 14 -#define NVM_CFG1_PORT_OPTIC_MODULE_VENDOR_ENFORCEMENT_DISABLED 0x0 -#define NVM_CFG1_PORT_OPTIC_MODULE_VENDOR_ENFORCEMENT_ENABLED 0x1 -#define NVM_CFG1_PORT_AN_25G_50G_OUI_MASK 0x00018000 -#define NVM_CFG1_PORT_AN_25G_50G_OUI_OFFSET 15 -#define NVM_CFG1_PORT_AN_25G_50G_OUI_CONSORTIUM 0x0 -#define NVM_CFG1_PORT_AN_25G_50G_OUI_BAM 0x1 -#define NVM_CFG1_PORT_FEC_FORCE_MODE_MASK 0x000E0000 -#define NVM_CFG1_PORT_FEC_FORCE_MODE_OFFSET 17 -#define NVM_CFG1_PORT_FEC_FORCE_MODE_FEC_FORCE_NONE 0x0 -#define NVM_CFG1_PORT_FEC_FORCE_MODE_FEC_FORCE_FIRECODE 0x1 -#define NVM_CFG1_PORT_FEC_FORCE_MODE_FEC_FORCE_RS 0x2 - u32 phy_cfg; /* 0x1C */ -#define NVM_CFG1_PORT_OPTIONAL_LINK_MODES_MASK 0x0000FFFF -#define NVM_CFG1_PORT_OPTIONAL_LINK_MODES_OFFSET 0 -#define NVM_CFG1_PORT_OPTIONAL_LINK_MODES_HIGIG 0x1 -#define NVM_CFG1_PORT_OPTIONAL_LINK_MODES_SCRAMBLER 0x2 -#define NVM_CFG1_PORT_OPTIONAL_LINK_MODES_FIBER 0x4 -#define NVM_CFG1_PORT_OPTIONAL_LINK_MODES_DISABLE_CL72_AN 0x8 -#define NVM_CFG1_PORT_OPTIONAL_LINK_MODES_DISABLE_FEC_AN 0x10 -#define NVM_CFG1_PORT_SERDES_NET_INTERFACE_MASK 0x00FF0000 -#define NVM_CFG1_PORT_SERDES_NET_INTERFACE_OFFSET 16 -#define NVM_CFG1_PORT_SERDES_NET_INTERFACE_BYPASS 0x0 -#define NVM_CFG1_PORT_SERDES_NET_INTERFACE_KR 0x2 -#define NVM_CFG1_PORT_SERDES_NET_INTERFACE_KR2 0x3 -#define NVM_CFG1_PORT_SERDES_NET_INTERFACE_KR4 0x4 -#define NVM_CFG1_PORT_SERDES_NET_INTERFACE_XFI 0x8 -#define NVM_CFG1_PORT_SERDES_NET_INTERFACE_SFI 0x9 -#define NVM_CFG1_PORT_SERDES_NET_INTERFACE_1000X 0xB -#define NVM_CFG1_PORT_SERDES_NET_INTERFACE_SGMII 0xC -#define NVM_CFG1_PORT_SERDES_NET_INTERFACE_XLAUI 0x11 -#define NVM_CFG1_PORT_SERDES_NET_INTERFACE_XLPPI 0x12 -#define NVM_CFG1_PORT_SERDES_NET_INTERFACE_CAUI 0x21 -#define NVM_CFG1_PORT_SERDES_NET_INTERFACE_CPPI 0x22 -#define NVM_CFG1_PORT_SERDES_NET_INTERFACE_25GAUI 0x31 -#define NVM_CFG1_PORT_AN_MODE_MASK 0xFF000000 -#define NVM_CFG1_PORT_AN_MODE_OFFSET 24 -#define NVM_CFG1_PORT_AN_MODE_NONE 0x0 -#define NVM_CFG1_PORT_AN_MODE_CL73 0x1 -#define NVM_CFG1_PORT_AN_MODE_CL37 0x2 -#define NVM_CFG1_PORT_AN_MODE_CL73_BAM 0x3 -#define NVM_CFG1_PORT_AN_MODE_CL37_BAM 0x4 -#define NVM_CFG1_PORT_AN_MODE_HPAM 0x5 -#define NVM_CFG1_PORT_AN_MODE_SGMII 0x6 - u32 mgmt_traffic; /* 0x20 */ -#define NVM_CFG1_PORT_RESERVED61_MASK 0x0000000F -#define NVM_CFG1_PORT_RESERVED61_OFFSET 0 - u32 ext_phy; /* 0x24 */ -#define NVM_CFG1_PORT_EXTERNAL_PHY_TYPE_MASK 0x000000FF -#define NVM_CFG1_PORT_EXTERNAL_PHY_TYPE_OFFSET 0 -#define NVM_CFG1_PORT_EXTERNAL_PHY_TYPE_NONE 0x0 -#define NVM_CFG1_PORT_EXTERNAL_PHY_TYPE_BCM84844 0x1 -#define NVM_CFG1_PORT_EXTERNAL_PHY_ADDRESS_MASK 0x0000FF00 -#define NVM_CFG1_PORT_EXTERNAL_PHY_ADDRESS_OFFSET 8 - u32 mba_cfg1; /* 0x28 */ -#define NVM_CFG1_PORT_PREBOOT_OPROM_MASK 0x00000001 -#define NVM_CFG1_PORT_PREBOOT_OPROM_OFFSET 0 -#define NVM_CFG1_PORT_PREBOOT_OPROM_DISABLED 0x0 -#define NVM_CFG1_PORT_PREBOOT_OPROM_ENABLED 0x1 -#define NVM_CFG1_PORT_RESERVED__M_MBA_BOOT_TYPE_MASK 0x00000006 -#define NVM_CFG1_PORT_RESERVED__M_MBA_BOOT_TYPE_OFFSET 1 -#define NVM_CFG1_PORT_MBA_DELAY_TIME_MASK 0x00000078 -#define NVM_CFG1_PORT_MBA_DELAY_TIME_OFFSET 3 -#define NVM_CFG1_PORT_MBA_SETUP_HOT_KEY_MASK 0x00000080 -#define NVM_CFG1_PORT_MBA_SETUP_HOT_KEY_OFFSET 7 -#define NVM_CFG1_PORT_MBA_SETUP_HOT_KEY_CTRL_S 0x0 -#define NVM_CFG1_PORT_MBA_SETUP_HOT_KEY_CTRL_B 0x1 -#define NVM_CFG1_PORT_MBA_HIDE_SETUP_PROMPT_MASK 0x00000100 -#define NVM_CFG1_PORT_MBA_HIDE_SETUP_PROMPT_OFFSET 8 -#define NVM_CFG1_PORT_MBA_HIDE_SETUP_PROMPT_DISABLED 0x0 -#define NVM_CFG1_PORT_MBA_HIDE_SETUP_PROMPT_ENABLED 0x1 -#define NVM_CFG1_PORT_RESERVED5_MASK 0x0001FE00 -#define NVM_CFG1_PORT_RESERVED5_OFFSET 9 -#define NVM_CFG1_PORT_PREBOOT_LINK_SPEED_MASK 0x001E0000 -#define NVM_CFG1_PORT_PREBOOT_LINK_SPEED_OFFSET 17 -#define NVM_CFG1_PORT_PREBOOT_LINK_SPEED_AUTONEG 0x0 -#define NVM_CFG1_PORT_PREBOOT_LINK_SPEED_1G 0x1 -#define NVM_CFG1_PORT_PREBOOT_LINK_SPEED_10G 0x2 -#define NVM_CFG1_PORT_PREBOOT_LINK_SPEED_25G 0x4 -#define NVM_CFG1_PORT_PREBOOT_LINK_SPEED_40G 0x5 -#define NVM_CFG1_PORT_PREBOOT_LINK_SPEED_50G 0x6 -#define NVM_CFG1_PORT_PREBOOT_LINK_SPEED_100G 0x7 -#define NVM_CFG1_PORT_PREBOOT_LINK_SPEED_SMARTLINQ 0x8 -#define NVM_CFG1_PORT_RESERVED__M_MBA_BOOT_RETRY_COUNT_MASK 0x00E00000 -#define NVM_CFG1_PORT_RESERVED__M_MBA_BOOT_RETRY_COUNT_OFFSET 21 - u32 mba_cfg2; /* 0x2C */ -#define NVM_CFG1_PORT_RESERVED65_MASK 0x0000FFFF -#define NVM_CFG1_PORT_RESERVED65_OFFSET 0 -#define NVM_CFG1_PORT_RESERVED66_MASK 0x00010000 -#define NVM_CFG1_PORT_RESERVED66_OFFSET 16 - u32 vf_cfg; /* 0x30 */ -#define NVM_CFG1_PORT_RESERVED8_MASK 0x0000FFFF -#define NVM_CFG1_PORT_RESERVED8_OFFSET 0 -#define NVM_CFG1_PORT_RESERVED6_MASK 0x000F0000 -#define NVM_CFG1_PORT_RESERVED6_OFFSET 16 - struct nvm_cfg_mac_address lldp_mac_address; /* 0x34 */ - u32 led_port_settings; /* 0x3C */ -#define NVM_CFG1_PORT_LANE_LED_SPD_0_SEL_MASK 0x000000FF -#define NVM_CFG1_PORT_LANE_LED_SPD_0_SEL_OFFSET 0 -#define NVM_CFG1_PORT_LANE_LED_SPD_1_SEL_MASK 0x0000FF00 -#define NVM_CFG1_PORT_LANE_LED_SPD_1_SEL_OFFSET 8 -#define NVM_CFG1_PORT_LANE_LED_SPD_2_SEL_MASK 0x00FF0000 -#define NVM_CFG1_PORT_LANE_LED_SPD_2_SEL_OFFSET 16 -#define NVM_CFG1_PORT_LANE_LED_SPD__SEL_1G 0x1 -#define NVM_CFG1_PORT_LANE_LED_SPD__SEL_10G 0x2 -#define NVM_CFG1_PORT_LANE_LED_SPD__SEL_25G 0x8 -#define NVM_CFG1_PORT_LANE_LED_SPD__SEL_40G 0x10 -#define NVM_CFG1_PORT_LANE_LED_SPD__SEL_50G 0x20 -#define NVM_CFG1_PORT_LANE_LED_SPD__SEL_100G 0x40 - u32 transceiver_00; /* 0x40 */ + u32 reserved__m_relocated_to_option_123; /* 0x0 */ + u32 reserved__m_relocated_to_option_124; /* 0x4 */ + u32 generic_cont0; /* 0x8 */ + #define NVM_CFG1_PORT_LED_MODE_MASK 0x000000FF + #define NVM_CFG1_PORT_LED_MODE_OFFSET 0 + #define NVM_CFG1_PORT_LED_MODE_MAC1 0x0 + #define NVM_CFG1_PORT_LED_MODE_PHY1 0x1 + #define NVM_CFG1_PORT_LED_MODE_PHY2 0x2 + #define NVM_CFG1_PORT_LED_MODE_PHY3 0x3 + #define NVM_CFG1_PORT_LED_MODE_MAC2 0x4 + #define NVM_CFG1_PORT_LED_MODE_PHY4 0x5 + #define NVM_CFG1_PORT_LED_MODE_PHY5 0x6 + #define NVM_CFG1_PORT_LED_MODE_PHY6 0x7 + #define NVM_CFG1_PORT_LED_MODE_MAC3 0x8 + #define NVM_CFG1_PORT_LED_MODE_PHY7 0x9 + #define NVM_CFG1_PORT_LED_MODE_PHY8 0xA + #define NVM_CFG1_PORT_LED_MODE_PHY9 0xB + #define NVM_CFG1_PORT_LED_MODE_MAC4 0xC + #define NVM_CFG1_PORT_LED_MODE_PHY10 0xD + #define NVM_CFG1_PORT_LED_MODE_PHY11 0xE + #define NVM_CFG1_PORT_LED_MODE_PHY12 0xF + #define NVM_CFG1_PORT_LED_MODE_BREAKOUT 0x10 + #define NVM_CFG1_PORT_ROCE_PRIORITY_MASK 0x0000FF00 + #define NVM_CFG1_PORT_ROCE_PRIORITY_OFFSET 8 + #define NVM_CFG1_PORT_DCBX_MODE_MASK 0x000F0000 + #define NVM_CFG1_PORT_DCBX_MODE_OFFSET 16 + #define NVM_CFG1_PORT_DCBX_MODE_DISABLED 0x0 + #define NVM_CFG1_PORT_DCBX_MODE_IEEE 0x1 + #define NVM_CFG1_PORT_DCBX_MODE_CEE 0x2 + #define NVM_CFG1_PORT_DCBX_MODE_DYNAMIC 0x3 + #define NVM_CFG1_PORT_DEFAULT_ENABLED_PROTOCOLS_MASK 0x00F00000 + #define NVM_CFG1_PORT_DEFAULT_ENABLED_PROTOCOLS_OFFSET 20 + #define NVM_CFG1_PORT_DEFAULT_ENABLED_PROTOCOLS_ETHERNET 0x1 + #define NVM_CFG1_PORT_DEFAULT_ENABLED_PROTOCOLS_FCOE 0x2 + #define NVM_CFG1_PORT_DEFAULT_ENABLED_PROTOCOLS_ISCSI 0x4 + /* GPIO for HW reset the PHY. In case it is the same for all ports, + * need to set same value for all ports + */ + #define NVM_CFG1_PORT_EXT_PHY_RESET_MASK 0xFF000000 + #define NVM_CFG1_PORT_EXT_PHY_RESET_OFFSET 24 + #define NVM_CFG1_PORT_EXT_PHY_RESET_NA 0x0 + #define NVM_CFG1_PORT_EXT_PHY_RESET_GPIO0 0x1 + #define NVM_CFG1_PORT_EXT_PHY_RESET_GPIO1 0x2 + #define NVM_CFG1_PORT_EXT_PHY_RESET_GPIO2 0x3 + #define NVM_CFG1_PORT_EXT_PHY_RESET_GPIO3 0x4 + #define NVM_CFG1_PORT_EXT_PHY_RESET_GPIO4 0x5 + #define NVM_CFG1_PORT_EXT_PHY_RESET_GPIO5 0x6 + #define NVM_CFG1_PORT_EXT_PHY_RESET_GPIO6 0x7 + #define NVM_CFG1_PORT_EXT_PHY_RESET_GPIO7 0x8 + #define NVM_CFG1_PORT_EXT_PHY_RESET_GPIO8 0x9 + #define NVM_CFG1_PORT_EXT_PHY_RESET_GPIO9 0xA + #define NVM_CFG1_PORT_EXT_PHY_RESET_GPIO10 0xB + #define NVM_CFG1_PORT_EXT_PHY_RESET_GPIO11 0xC + #define NVM_CFG1_PORT_EXT_PHY_RESET_GPIO12 0xD + #define NVM_CFG1_PORT_EXT_PHY_RESET_GPIO13 0xE + #define NVM_CFG1_PORT_EXT_PHY_RESET_GPIO14 0xF + #define NVM_CFG1_PORT_EXT_PHY_RESET_GPIO15 0x10 + #define NVM_CFG1_PORT_EXT_PHY_RESET_GPIO16 0x11 + #define NVM_CFG1_PORT_EXT_PHY_RESET_GPIO17 0x12 + #define NVM_CFG1_PORT_EXT_PHY_RESET_GPIO18 0x13 + #define NVM_CFG1_PORT_EXT_PHY_RESET_GPIO19 0x14 + #define NVM_CFG1_PORT_EXT_PHY_RESET_GPIO20 0x15 + #define NVM_CFG1_PORT_EXT_PHY_RESET_GPIO21 0x16 + #define NVM_CFG1_PORT_EXT_PHY_RESET_GPIO22 0x17 + #define NVM_CFG1_PORT_EXT_PHY_RESET_GPIO23 0x18 + #define NVM_CFG1_PORT_EXT_PHY_RESET_GPIO24 0x19 + #define NVM_CFG1_PORT_EXT_PHY_RESET_GPIO25 0x1A + #define NVM_CFG1_PORT_EXT_PHY_RESET_GPIO26 0x1B + #define NVM_CFG1_PORT_EXT_PHY_RESET_GPIO27 0x1C + #define NVM_CFG1_PORT_EXT_PHY_RESET_GPIO28 0x1D + #define NVM_CFG1_PORT_EXT_PHY_RESET_GPIO29 0x1E + #define NVM_CFG1_PORT_EXT_PHY_RESET_GPIO30 0x1F + #define NVM_CFG1_PORT_EXT_PHY_RESET_GPIO31 0x20 + u32 pcie_cfg; /* 0xC */ + #define NVM_CFG1_PORT_RESERVED15_MASK 0x00000007 + #define NVM_CFG1_PORT_RESERVED15_OFFSET 0 + u32 features; /* 0x10 */ + #define NVM_CFG1_PORT_ENABLE_WOL_ON_ACPI_PATTERN_MASK 0x00000001 + #define NVM_CFG1_PORT_ENABLE_WOL_ON_ACPI_PATTERN_OFFSET 0 + #define NVM_CFG1_PORT_ENABLE_WOL_ON_ACPI_PATTERN_DISABLED 0x0 + #define NVM_CFG1_PORT_ENABLE_WOL_ON_ACPI_PATTERN_ENABLED 0x1 + #define NVM_CFG1_PORT_MAGIC_PACKET_WOL_MASK 0x00000002 + #define NVM_CFG1_PORT_MAGIC_PACKET_WOL_OFFSET 1 + #define NVM_CFG1_PORT_MAGIC_PACKET_WOL_DISABLED 0x0 + #define NVM_CFG1_PORT_MAGIC_PACKET_WOL_ENABLED 0x1 + u32 speed_cap_mask; /* 0x14 */ + #define NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_MASK 0x0000FFFF + #define NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_OFFSET 0 + #define NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_1G 0x1 + #define NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_10G 0x2 + #define NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_20G 0x4 + #define NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_25G 0x8 + #define NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_40G 0x10 + #define NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_50G 0x20 + #define NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_BB_100G 0x40 + #define NVM_CFG1_PORT_MFW_SPEED_CAPABILITY_MASK_MASK 0xFFFF0000 + #define NVM_CFG1_PORT_MFW_SPEED_CAPABILITY_MASK_OFFSET 16 + #define NVM_CFG1_PORT_MFW_SPEED_CAPABILITY_MASK_1G 0x1 + #define NVM_CFG1_PORT_MFW_SPEED_CAPABILITY_MASK_10G 0x2 + #define NVM_CFG1_PORT_MFW_SPEED_CAPABILITY_MASK_20G 0x4 + #define NVM_CFG1_PORT_MFW_SPEED_CAPABILITY_MASK_25G 0x8 + #define NVM_CFG1_PORT_MFW_SPEED_CAPABILITY_MASK_40G 0x10 + #define NVM_CFG1_PORT_MFW_SPEED_CAPABILITY_MASK_50G 0x20 + #define NVM_CFG1_PORT_MFW_SPEED_CAPABILITY_MASK_BB_100G 0x40 + u32 link_settings; /* 0x18 */ + #define NVM_CFG1_PORT_DRV_LINK_SPEED_MASK 0x0000000F + #define NVM_CFG1_PORT_DRV_LINK_SPEED_OFFSET 0 + #define NVM_CFG1_PORT_DRV_LINK_SPEED_AUTONEG 0x0 + #define NVM_CFG1_PORT_DRV_LINK_SPEED_1G 0x1 + #define NVM_CFG1_PORT_DRV_LINK_SPEED_10G 0x2 + #define NVM_CFG1_PORT_DRV_LINK_SPEED_20G 0x3 + #define NVM_CFG1_PORT_DRV_LINK_SPEED_25G 0x4 + #define NVM_CFG1_PORT_DRV_LINK_SPEED_40G 0x5 + #define NVM_CFG1_PORT_DRV_LINK_SPEED_50G 0x6 + #define NVM_CFG1_PORT_DRV_LINK_SPEED_BB_100G 0x7 + #define NVM_CFG1_PORT_DRV_FLOW_CONTROL_MASK 0x00000070 + #define NVM_CFG1_PORT_DRV_FLOW_CONTROL_OFFSET 4 + #define NVM_CFG1_PORT_DRV_FLOW_CONTROL_AUTONEG 0x1 + #define NVM_CFG1_PORT_DRV_FLOW_CONTROL_RX 0x2 + #define NVM_CFG1_PORT_DRV_FLOW_CONTROL_TX 0x4 + #define NVM_CFG1_PORT_MFW_LINK_SPEED_MASK 0x00000780 + #define NVM_CFG1_PORT_MFW_LINK_SPEED_OFFSET 7 + #define NVM_CFG1_PORT_MFW_LINK_SPEED_AUTONEG 0x0 + #define NVM_CFG1_PORT_MFW_LINK_SPEED_1G 0x1 + #define NVM_CFG1_PORT_MFW_LINK_SPEED_10G 0x2 + #define NVM_CFG1_PORT_MFW_LINK_SPEED_20G 0x3 + #define NVM_CFG1_PORT_MFW_LINK_SPEED_25G 0x4 + #define NVM_CFG1_PORT_MFW_LINK_SPEED_40G 0x5 + #define NVM_CFG1_PORT_MFW_LINK_SPEED_50G 0x6 + #define NVM_CFG1_PORT_MFW_LINK_SPEED_BB_100G 0x7 + #define NVM_CFG1_PORT_MFW_FLOW_CONTROL_MASK 0x00003800 + #define NVM_CFG1_PORT_MFW_FLOW_CONTROL_OFFSET 11 + #define NVM_CFG1_PORT_MFW_FLOW_CONTROL_AUTONEG 0x1 + #define NVM_CFG1_PORT_MFW_FLOW_CONTROL_RX 0x2 + #define NVM_CFG1_PORT_MFW_FLOW_CONTROL_TX 0x4 + #define NVM_CFG1_PORT_OPTIC_MODULE_VENDOR_ENFORCEMENT_MASK \ + 0x00004000 + #define NVM_CFG1_PORT_OPTIC_MODULE_VENDOR_ENFORCEMENT_OFFSET 14 + #define NVM_CFG1_PORT_OPTIC_MODULE_VENDOR_ENFORCEMENT_DISABLED \ + 0x0 + #define NVM_CFG1_PORT_OPTIC_MODULE_VENDOR_ENFORCEMENT_ENABLED \ + 0x1 + #define NVM_CFG1_PORT_AN_25G_50G_OUI_MASK 0x00018000 + #define NVM_CFG1_PORT_AN_25G_50G_OUI_OFFSET 15 + #define NVM_CFG1_PORT_AN_25G_50G_OUI_CONSORTIUM 0x0 + #define NVM_CFG1_PORT_AN_25G_50G_OUI_BAM 0x1 + #define NVM_CFG1_PORT_FEC_FORCE_MODE_MASK 0x000E0000 + #define NVM_CFG1_PORT_FEC_FORCE_MODE_OFFSET 17 + #define NVM_CFG1_PORT_FEC_FORCE_MODE_NONE 0x0 + #define NVM_CFG1_PORT_FEC_FORCE_MODE_FIRECODE 0x1 + #define NVM_CFG1_PORT_FEC_FORCE_MODE_RS 0x2 + #define NVM_CFG1_PORT_FEC_FORCE_MODE_AUTO 0x7 + #define NVM_CFG1_PORT_FEC_AN_MODE_MASK 0x00700000 + #define NVM_CFG1_PORT_FEC_AN_MODE_OFFSET 20 + #define NVM_CFG1_PORT_FEC_AN_MODE_NONE 0x0 + #define NVM_CFG1_PORT_FEC_AN_MODE_10G_FIRECODE 0x1 + #define NVM_CFG1_PORT_FEC_AN_MODE_25G_FIRECODE 0x2 + #define NVM_CFG1_PORT_FEC_AN_MODE_10G_AND_25G_FIRECODE 0x3 + #define NVM_CFG1_PORT_FEC_AN_MODE_25G_RS 0x4 + #define NVM_CFG1_PORT_FEC_AN_MODE_25G_FIRECODE_AND_RS 0x5 + #define NVM_CFG1_PORT_FEC_AN_MODE_ALL 0x6 + #define NVM_CFG1_PORT_SMARTLINQ_MODE_MASK 0x00800000 + #define NVM_CFG1_PORT_SMARTLINQ_MODE_OFFSET 23 + #define NVM_CFG1_PORT_SMARTLINQ_MODE_DISABLED 0x0 + #define NVM_CFG1_PORT_SMARTLINQ_MODE_ENABLED 0x1 + #define NVM_CFG1_PORT_RESERVED_WAS_MFW_SMARTLINQ_MASK 0x01000000 + #define NVM_CFG1_PORT_RESERVED_WAS_MFW_SMARTLINQ_OFFSET 24 + #define NVM_CFG1_PORT_RESERVED_WAS_MFW_SMARTLINQ_DISABLED 0x0 + #define NVM_CFG1_PORT_RESERVED_WAS_MFW_SMARTLINQ_ENABLED 0x1 + u32 phy_cfg; /* 0x1C */ + #define NVM_CFG1_PORT_OPTIONAL_LINK_MODES_MASK 0x0000FFFF + #define NVM_CFG1_PORT_OPTIONAL_LINK_MODES_OFFSET 0 + #define NVM_CFG1_PORT_OPTIONAL_LINK_MODES_HIGIG 0x1 + #define NVM_CFG1_PORT_OPTIONAL_LINK_MODES_SCRAMBLER 0x2 + #define NVM_CFG1_PORT_OPTIONAL_LINK_MODES_FIBER 0x4 + #define NVM_CFG1_PORT_OPTIONAL_LINK_MODES_DISABLE_CL72_AN 0x8 + #define NVM_CFG1_PORT_OPTIONAL_LINK_MODES_DISABLE_FEC_AN 0x10 + #define NVM_CFG1_PORT_SERDES_NET_INTERFACE_MASK 0x00FF0000 + #define NVM_CFG1_PORT_SERDES_NET_INTERFACE_OFFSET 16 + #define NVM_CFG1_PORT_SERDES_NET_INTERFACE_BYPASS 0x0 + #define NVM_CFG1_PORT_SERDES_NET_INTERFACE_KR 0x2 + #define NVM_CFG1_PORT_SERDES_NET_INTERFACE_KR2 0x3 + #define NVM_CFG1_PORT_SERDES_NET_INTERFACE_KR4 0x4 + #define NVM_CFG1_PORT_SERDES_NET_INTERFACE_XFI 0x8 + #define NVM_CFG1_PORT_SERDES_NET_INTERFACE_SFI 0x9 + #define NVM_CFG1_PORT_SERDES_NET_INTERFACE_1000X 0xB + #define NVM_CFG1_PORT_SERDES_NET_INTERFACE_SGMII 0xC + #define NVM_CFG1_PORT_SERDES_NET_INTERFACE_XLAUI 0x11 + #define NVM_CFG1_PORT_SERDES_NET_INTERFACE_XLPPI 0x12 + #define NVM_CFG1_PORT_SERDES_NET_INTERFACE_CAUI 0x21 + #define NVM_CFG1_PORT_SERDES_NET_INTERFACE_CPPI 0x22 + #define NVM_CFG1_PORT_SERDES_NET_INTERFACE_25GAUI 0x31 + #define NVM_CFG1_PORT_AN_MODE_MASK 0xFF000000 + #define NVM_CFG1_PORT_AN_MODE_OFFSET 24 + #define NVM_CFG1_PORT_AN_MODE_NONE 0x0 + #define NVM_CFG1_PORT_AN_MODE_CL73 0x1 + #define NVM_CFG1_PORT_AN_MODE_CL37 0x2 + #define NVM_CFG1_PORT_AN_MODE_CL73_BAM 0x3 + #define NVM_CFG1_PORT_AN_MODE_BB_CL37_BAM 0x4 + #define NVM_CFG1_PORT_AN_MODE_BB_HPAM 0x5 + #define NVM_CFG1_PORT_AN_MODE_BB_SGMII 0x6 + u32 mgmt_traffic; /* 0x20 */ + #define NVM_CFG1_PORT_RESERVED61_MASK 0x0000000F + #define NVM_CFG1_PORT_RESERVED61_OFFSET 0 + u32 ext_phy; /* 0x24 */ + #define NVM_CFG1_PORT_EXTERNAL_PHY_TYPE_MASK 0x000000FF + #define NVM_CFG1_PORT_EXTERNAL_PHY_TYPE_OFFSET 0 + #define NVM_CFG1_PORT_EXTERNAL_PHY_TYPE_NONE 0x0 + #define NVM_CFG1_PORT_EXTERNAL_PHY_TYPE_BCM8485X 0x1 + #define NVM_CFG1_PORT_EXTERNAL_PHY_TYPE_BCM5422X 0x2 + #define NVM_CFG1_PORT_EXTERNAL_PHY_ADDRESS_MASK 0x0000FF00 + #define NVM_CFG1_PORT_EXTERNAL_PHY_ADDRESS_OFFSET 8 + /* EEE power saving mode */ + #define NVM_CFG1_PORT_EEE_POWER_SAVING_MODE_MASK 0x00FF0000 + #define NVM_CFG1_PORT_EEE_POWER_SAVING_MODE_OFFSET 16 + #define NVM_CFG1_PORT_EEE_POWER_SAVING_MODE_DISABLED 0x0 + #define NVM_CFG1_PORT_EEE_POWER_SAVING_MODE_BALANCED 0x1 + #define NVM_CFG1_PORT_EEE_POWER_SAVING_MODE_AGGRESSIVE 0x2 + #define NVM_CFG1_PORT_EEE_POWER_SAVING_MODE_LOW_LATENCY 0x3 + u32 mba_cfg1; /* 0x28 */ + #define NVM_CFG1_PORT_PREBOOT_OPROM_MASK 0x00000001 + #define NVM_CFG1_PORT_PREBOOT_OPROM_OFFSET 0 + #define NVM_CFG1_PORT_PREBOOT_OPROM_DISABLED 0x0 + #define NVM_CFG1_PORT_PREBOOT_OPROM_ENABLED 0x1 + #define NVM_CFG1_PORT_RESERVED__M_MBA_BOOT_TYPE_MASK 0x00000006 + #define NVM_CFG1_PORT_RESERVED__M_MBA_BOOT_TYPE_OFFSET 1 + #define NVM_CFG1_PORT_MBA_DELAY_TIME_MASK 0x00000078 + #define NVM_CFG1_PORT_MBA_DELAY_TIME_OFFSET 3 + #define NVM_CFG1_PORT_MBA_SETUP_HOT_KEY_MASK 0x00000080 + #define NVM_CFG1_PORT_MBA_SETUP_HOT_KEY_OFFSET 7 + #define NVM_CFG1_PORT_MBA_SETUP_HOT_KEY_CTRL_S 0x0 + #define NVM_CFG1_PORT_MBA_SETUP_HOT_KEY_CTRL_B 0x1 + #define NVM_CFG1_PORT_MBA_HIDE_SETUP_PROMPT_MASK 0x00000100 + #define NVM_CFG1_PORT_MBA_HIDE_SETUP_PROMPT_OFFSET 8 + #define NVM_CFG1_PORT_MBA_HIDE_SETUP_PROMPT_DISABLED 0x0 + #define NVM_CFG1_PORT_MBA_HIDE_SETUP_PROMPT_ENABLED 0x1 + #define NVM_CFG1_PORT_RESERVED5_MASK 0x0001FE00 + #define NVM_CFG1_PORT_RESERVED5_OFFSET 9 + #define NVM_CFG1_PORT_PREBOOT_LINK_SPEED_MASK 0x001E0000 + #define NVM_CFG1_PORT_PREBOOT_LINK_SPEED_OFFSET 17 + #define NVM_CFG1_PORT_PREBOOT_LINK_SPEED_AUTONEG 0x0 + #define NVM_CFG1_PORT_PREBOOT_LINK_SPEED_1G 0x1 + #define NVM_CFG1_PORT_PREBOOT_LINK_SPEED_10G 0x2 + #define NVM_CFG1_PORT_PREBOOT_LINK_SPEED_20G 0x3 + #define NVM_CFG1_PORT_PREBOOT_LINK_SPEED_25G 0x4 + #define NVM_CFG1_PORT_PREBOOT_LINK_SPEED_40G 0x5 + #define NVM_CFG1_PORT_PREBOOT_LINK_SPEED_50G 0x6 + #define NVM_CFG1_PORT_PREBOOT_LINK_SPEED_BB_100G 0x7 + #define NVM_CFG1_PORT_RESERVED__M_MBA_BOOT_RETRY_COUNT_MASK \ + 0x00E00000 + #define NVM_CFG1_PORT_RESERVED__M_MBA_BOOT_RETRY_COUNT_OFFSET 21 + #define NVM_CFG1_PORT_RESERVED_WAS_PREBOOT_SMARTLINQ_MASK \ + 0x01000000 + #define NVM_CFG1_PORT_RESERVED_WAS_PREBOOT_SMARTLINQ_OFFSET 24 + #define NVM_CFG1_PORT_RESERVED_WAS_PREBOOT_SMARTLINQ_DISABLED \ + 0x0 + #define NVM_CFG1_PORT_RESERVED_WAS_PREBOOT_SMARTLINQ_ENABLED 0x1 + u32 mba_cfg2; /* 0x2C */ + #define NVM_CFG1_PORT_RESERVED65_MASK 0x0000FFFF + #define NVM_CFG1_PORT_RESERVED65_OFFSET 0 + #define NVM_CFG1_PORT_RESERVED66_MASK 0x00010000 + #define NVM_CFG1_PORT_RESERVED66_OFFSET 16 + #define NVM_CFG1_PORT_PREBOOT_LINK_UP_DELAY_MASK 0x01FE0000 + #define NVM_CFG1_PORT_PREBOOT_LINK_UP_DELAY_OFFSET 17 + u32 vf_cfg; /* 0x30 */ + #define NVM_CFG1_PORT_RESERVED8_MASK 0x0000FFFF + #define NVM_CFG1_PORT_RESERVED8_OFFSET 0 + #define NVM_CFG1_PORT_RESERVED6_MASK 0x000F0000 + #define NVM_CFG1_PORT_RESERVED6_OFFSET 16 + struct nvm_cfg_mac_address lldp_mac_address; /* 0x34 */ + u32 led_port_settings; /* 0x3C */ + #define NVM_CFG1_PORT_LANE_LED_SPD_0_SEL_MASK 0x000000FF + #define NVM_CFG1_PORT_LANE_LED_SPD_0_SEL_OFFSET 0 + #define NVM_CFG1_PORT_LANE_LED_SPD_1_SEL_MASK 0x0000FF00 + #define NVM_CFG1_PORT_LANE_LED_SPD_1_SEL_OFFSET 8 + #define NVM_CFG1_PORT_LANE_LED_SPD_2_SEL_MASK 0x00FF0000 + #define NVM_CFG1_PORT_LANE_LED_SPD_2_SEL_OFFSET 16 + #define NVM_CFG1_PORT_LANE_LED_SPD__SEL_1G 0x1 + #define NVM_CFG1_PORT_LANE_LED_SPD__SEL_10G 0x2 + #define NVM_CFG1_PORT_LANE_LED_SPD__SEL_AH_25G 0x4 + #define NVM_CFG1_PORT_LANE_LED_SPD__SEL_BB_25G 0x8 + #define NVM_CFG1_PORT_LANE_LED_SPD__SEL_AH_40G 0x8 + #define NVM_CFG1_PORT_LANE_LED_SPD__SEL_BB_40G 0x10 + #define NVM_CFG1_PORT_LANE_LED_SPD__SEL_AH_50G 0x10 + #define NVM_CFG1_PORT_LANE_LED_SPD__SEL_BB_50G 0x20 + #define NVM_CFG1_PORT_LANE_LED_SPD__SEL_BB_100G 0x40 + u32 transceiver_00; /* 0x40 */ /* Define for mapping of transceiver signal module absent */ -#define NVM_CFG1_PORT_TRANS_MODULE_ABS_MASK 0x000000FF -#define NVM_CFG1_PORT_TRANS_MODULE_ABS_OFFSET 0 -#define NVM_CFG1_PORT_TRANS_MODULE_ABS_NA 0x0 -#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO0 0x1 -#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO1 0x2 -#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO2 0x3 -#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO3 0x4 -#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO4 0x5 -#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO5 0x6 -#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO6 0x7 -#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO7 0x8 -#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO8 0x9 -#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO9 0xA -#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO10 0xB -#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO11 0xC -#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO12 0xD -#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO13 0xE -#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO14 0xF -#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO15 0x10 -#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO16 0x11 -#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO17 0x12 -#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO18 0x13 -#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO19 0x14 -#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO20 0x15 -#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO21 0x16 -#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO22 0x17 -#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO23 0x18 -#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO24 0x19 -#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO25 0x1A -#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO26 0x1B -#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO27 0x1C -#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO28 0x1D -#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO29 0x1E -#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO30 0x1F -#define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO31 0x20 + #define NVM_CFG1_PORT_TRANS_MODULE_ABS_MASK 0x000000FF + #define NVM_CFG1_PORT_TRANS_MODULE_ABS_OFFSET 0 + #define NVM_CFG1_PORT_TRANS_MODULE_ABS_NA 0x0 + #define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO0 0x1 + #define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO1 0x2 + #define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO2 0x3 + #define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO3 0x4 + #define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO4 0x5 + #define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO5 0x6 + #define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO6 0x7 + #define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO7 0x8 + #define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO8 0x9 + #define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO9 0xA + #define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO10 0xB + #define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO11 0xC + #define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO12 0xD + #define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO13 0xE + #define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO14 0xF + #define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO15 0x10 + #define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO16 0x11 + #define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO17 0x12 + #define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO18 0x13 + #define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO19 0x14 + #define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO20 0x15 + #define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO21 0x16 + #define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO22 0x17 + #define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO23 0x18 + #define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO24 0x19 + #define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO25 0x1A + #define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO26 0x1B + #define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO27 0x1C + #define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO28 0x1D + #define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO29 0x1E + #define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO30 0x1F + #define NVM_CFG1_PORT_TRANS_MODULE_ABS_GPIO31 0x20 /* Define the GPIO mux settings to switch i2c mux to this port */ -#define NVM_CFG1_PORT_I2C_MUX_SEL_VALUE_0_MASK 0x00000F00 -#define NVM_CFG1_PORT_I2C_MUX_SEL_VALUE_0_OFFSET 8 -#define NVM_CFG1_PORT_I2C_MUX_SEL_VALUE_1_MASK 0x0000F000 -#define NVM_CFG1_PORT_I2C_MUX_SEL_VALUE_1_OFFSET 12 - u32 device_ids; /* 0x44 */ -#define NVM_CFG1_PORT_ETH_DID_SUFFIX_MASK 0x000000FF -#define NVM_CFG1_PORT_ETH_DID_SUFFIX_OFFSET 0 -#define NVM_CFG1_PORT_RESERVED_DID_SUFFIX_MASK 0xFF000000 -#define NVM_CFG1_PORT_RESERVED_DID_SUFFIX_OFFSET 24 - u32 board_cfg; /* 0x48 */ - /* This field defines the board technology + #define NVM_CFG1_PORT_I2C_MUX_SEL_VALUE_0_MASK 0x00000F00 + #define NVM_CFG1_PORT_I2C_MUX_SEL_VALUE_0_OFFSET 8 + #define NVM_CFG1_PORT_I2C_MUX_SEL_VALUE_1_MASK 0x0000F000 + #define NVM_CFG1_PORT_I2C_MUX_SEL_VALUE_1_OFFSET 12 + u32 device_ids; /* 0x44 */ + #define NVM_CFG1_PORT_ETH_DID_SUFFIX_MASK 0x000000FF + #define NVM_CFG1_PORT_ETH_DID_SUFFIX_OFFSET 0 + #define NVM_CFG1_PORT_FCOE_DID_SUFFIX_MASK 0x0000FF00 + #define NVM_CFG1_PORT_FCOE_DID_SUFFIX_OFFSET 8 + #define NVM_CFG1_PORT_ISCSI_DID_SUFFIX_MASK 0x00FF0000 + #define NVM_CFG1_PORT_ISCSI_DID_SUFFIX_OFFSET 16 + #define NVM_CFG1_PORT_RESERVED_DID_SUFFIX_MASK 0xFF000000 + #define NVM_CFG1_PORT_RESERVED_DID_SUFFIX_OFFSET 24 + u32 board_cfg; /* 0x48 */ + /* This field defines the board technology * (backpane,transceiver,external PHY) */ -#define NVM_CFG1_PORT_PORT_TYPE_MASK 0x000000FF -#define NVM_CFG1_PORT_PORT_TYPE_OFFSET 0 -#define NVM_CFG1_PORT_PORT_TYPE_UNDEFINED 0x0 -#define NVM_CFG1_PORT_PORT_TYPE_MODULE 0x1 -#define NVM_CFG1_PORT_PORT_TYPE_BACKPLANE 0x2 -#define NVM_CFG1_PORT_PORT_TYPE_EXT_PHY 0x3 -#define NVM_CFG1_PORT_PORT_TYPE_MODULE_SLAVE 0x4 + #define NVM_CFG1_PORT_PORT_TYPE_MASK 0x000000FF + #define NVM_CFG1_PORT_PORT_TYPE_OFFSET 0 + #define NVM_CFG1_PORT_PORT_TYPE_UNDEFINED 0x0 + #define NVM_CFG1_PORT_PORT_TYPE_MODULE 0x1 + #define NVM_CFG1_PORT_PORT_TYPE_BACKPLANE 0x2 + #define NVM_CFG1_PORT_PORT_TYPE_EXT_PHY 0x3 + #define NVM_CFG1_PORT_PORT_TYPE_MODULE_SLAVE 0x4 /* This field defines the GPIO mapped to tx_disable signal in SFP */ -#define NVM_CFG1_PORT_TX_DISABLE_MASK 0x0000FF00 -#define NVM_CFG1_PORT_TX_DISABLE_OFFSET 8 -#define NVM_CFG1_PORT_TX_DISABLE_NA 0x0 -#define NVM_CFG1_PORT_TX_DISABLE_GPIO0 0x1 -#define NVM_CFG1_PORT_TX_DISABLE_GPIO1 0x2 -#define NVM_CFG1_PORT_TX_DISABLE_GPIO2 0x3 -#define NVM_CFG1_PORT_TX_DISABLE_GPIO3 0x4 -#define NVM_CFG1_PORT_TX_DISABLE_GPIO4 0x5 -#define NVM_CFG1_PORT_TX_DISABLE_GPIO5 0x6 -#define NVM_CFG1_PORT_TX_DISABLE_GPIO6 0x7 -#define NVM_CFG1_PORT_TX_DISABLE_GPIO7 0x8 -#define NVM_CFG1_PORT_TX_DISABLE_GPIO8 0x9 -#define NVM_CFG1_PORT_TX_DISABLE_GPIO9 0xA -#define NVM_CFG1_PORT_TX_DISABLE_GPIO10 0xB -#define NVM_CFG1_PORT_TX_DISABLE_GPIO11 0xC -#define NVM_CFG1_PORT_TX_DISABLE_GPIO12 0xD -#define NVM_CFG1_PORT_TX_DISABLE_GPIO13 0xE -#define NVM_CFG1_PORT_TX_DISABLE_GPIO14 0xF -#define NVM_CFG1_PORT_TX_DISABLE_GPIO15 0x10 -#define NVM_CFG1_PORT_TX_DISABLE_GPIO16 0x11 -#define NVM_CFG1_PORT_TX_DISABLE_GPIO17 0x12 -#define NVM_CFG1_PORT_TX_DISABLE_GPIO18 0x13 -#define NVM_CFG1_PORT_TX_DISABLE_GPIO19 0x14 -#define NVM_CFG1_PORT_TX_DISABLE_GPIO20 0x15 -#define NVM_CFG1_PORT_TX_DISABLE_GPIO21 0x16 -#define NVM_CFG1_PORT_TX_DISABLE_GPIO22 0x17 -#define NVM_CFG1_PORT_TX_DISABLE_GPIO23 0x18 -#define NVM_CFG1_PORT_TX_DISABLE_GPIO24 0x19 -#define NVM_CFG1_PORT_TX_DISABLE_GPIO25 0x1A -#define NVM_CFG1_PORT_TX_DISABLE_GPIO26 0x1B -#define NVM_CFG1_PORT_TX_DISABLE_GPIO27 0x1C -#define NVM_CFG1_PORT_TX_DISABLE_GPIO28 0x1D -#define NVM_CFG1_PORT_TX_DISABLE_GPIO29 0x1E -#define NVM_CFG1_PORT_TX_DISABLE_GPIO30 0x1F -#define NVM_CFG1_PORT_TX_DISABLE_GPIO31 0x20 - u32 reserved[131]; /* 0x4C */ + #define NVM_CFG1_PORT_TX_DISABLE_MASK 0x0000FF00 + #define NVM_CFG1_PORT_TX_DISABLE_OFFSET 8 + #define NVM_CFG1_PORT_TX_DISABLE_NA 0x0 + #define NVM_CFG1_PORT_TX_DISABLE_GPIO0 0x1 + #define NVM_CFG1_PORT_TX_DISABLE_GPIO1 0x2 + #define NVM_CFG1_PORT_TX_DISABLE_GPIO2 0x3 + #define NVM_CFG1_PORT_TX_DISABLE_GPIO3 0x4 + #define NVM_CFG1_PORT_TX_DISABLE_GPIO4 0x5 + #define NVM_CFG1_PORT_TX_DISABLE_GPIO5 0x6 + #define NVM_CFG1_PORT_TX_DISABLE_GPIO6 0x7 + #define NVM_CFG1_PORT_TX_DISABLE_GPIO7 0x8 + #define NVM_CFG1_PORT_TX_DISABLE_GPIO8 0x9 + #define NVM_CFG1_PORT_TX_DISABLE_GPIO9 0xA + #define NVM_CFG1_PORT_TX_DISABLE_GPIO10 0xB + #define NVM_CFG1_PORT_TX_DISABLE_GPIO11 0xC + #define NVM_CFG1_PORT_TX_DISABLE_GPIO12 0xD + #define NVM_CFG1_PORT_TX_DISABLE_GPIO13 0xE + #define NVM_CFG1_PORT_TX_DISABLE_GPIO14 0xF + #define NVM_CFG1_PORT_TX_DISABLE_GPIO15 0x10 + #define NVM_CFG1_PORT_TX_DISABLE_GPIO16 0x11 + #define NVM_CFG1_PORT_TX_DISABLE_GPIO17 0x12 + #define NVM_CFG1_PORT_TX_DISABLE_GPIO18 0x13 + #define NVM_CFG1_PORT_TX_DISABLE_GPIO19 0x14 + #define NVM_CFG1_PORT_TX_DISABLE_GPIO20 0x15 + #define NVM_CFG1_PORT_TX_DISABLE_GPIO21 0x16 + #define NVM_CFG1_PORT_TX_DISABLE_GPIO22 0x17 + #define NVM_CFG1_PORT_TX_DISABLE_GPIO23 0x18 + #define NVM_CFG1_PORT_TX_DISABLE_GPIO24 0x19 + #define NVM_CFG1_PORT_TX_DISABLE_GPIO25 0x1A + #define NVM_CFG1_PORT_TX_DISABLE_GPIO26 0x1B + #define NVM_CFG1_PORT_TX_DISABLE_GPIO27 0x1C + #define NVM_CFG1_PORT_TX_DISABLE_GPIO28 0x1D + #define NVM_CFG1_PORT_TX_DISABLE_GPIO29 0x1E + #define NVM_CFG1_PORT_TX_DISABLE_GPIO30 0x1F + #define NVM_CFG1_PORT_TX_DISABLE_GPIO31 0x20 + u32 mnm_10g_cap; /* 0x4C */ + #define NVM_CFG1_PORT_MNM_10G_DRV_SPEED_CAPABILITY_MASK_MASK \ + 0x0000FFFF + #define NVM_CFG1_PORT_MNM_10G_DRV_SPEED_CAPABILITY_MASK_OFFSET 0 + #define NVM_CFG1_PORT_MNM_10G_DRV_SPEED_CAPABILITY_MASK_1G 0x1 + #define NVM_CFG1_PORT_MNM_10G_DRV_SPEED_CAPABILITY_MASK_10G 0x2 + #define NVM_CFG1_PORT_MNM_10G_DRV_SPEED_CAPABILITY_MASK_20G 0x4 + #define NVM_CFG1_PORT_MNM_10G_DRV_SPEED_CAPABILITY_MASK_25G 0x8 + #define NVM_CFG1_PORT_MNM_10G_DRV_SPEED_CAPABILITY_MASK_40G 0x10 + #define NVM_CFG1_PORT_MNM_10G_DRV_SPEED_CAPABILITY_MASK_50G 0x20 + #define \ + NVM_CFG1_PORT_MNM_10G_DRV_SPEED_CAPABILITY_MASK_BB_100G 0x40 + #define NVM_CFG1_PORT_MNM_10G_MFW_SPEED_CAPABILITY_MASK_MASK \ + 0xFFFF0000 + #define NVM_CFG1_PORT_MNM_10G_MFW_SPEED_CAPABILITY_MASK_OFFSET \ + 16 + #define NVM_CFG1_PORT_MNM_10G_MFW_SPEED_CAPABILITY_MASK_1G 0x1 + #define NVM_CFG1_PORT_MNM_10G_MFW_SPEED_CAPABILITY_MASK_10G 0x2 + #define NVM_CFG1_PORT_MNM_10G_MFW_SPEED_CAPABILITY_MASK_20G 0x4 + #define NVM_CFG1_PORT_MNM_10G_MFW_SPEED_CAPABILITY_MASK_25G 0x8 + #define NVM_CFG1_PORT_MNM_10G_MFW_SPEED_CAPABILITY_MASK_40G 0x10 + #define NVM_CFG1_PORT_MNM_10G_MFW_SPEED_CAPABILITY_MASK_50G 0x20 + #define \ + NVM_CFG1_PORT_MNM_10G_MFW_SPEED_CAPABILITY_MASK_BB_100G 0x40 + u32 mnm_10g_ctrl; /* 0x50 */ + #define NVM_CFG1_PORT_MNM_10G_DRV_LINK_SPEED_MASK 0x0000000F + #define NVM_CFG1_PORT_MNM_10G_DRV_LINK_SPEED_OFFSET 0 + #define NVM_CFG1_PORT_MNM_10G_DRV_LINK_SPEED_AUTONEG 0x0 + #define NVM_CFG1_PORT_MNM_10G_DRV_LINK_SPEED_1G 0x1 + #define NVM_CFG1_PORT_MNM_10G_DRV_LINK_SPEED_10G 0x2 + #define NVM_CFG1_PORT_MNM_10G_DRV_LINK_SPEED_20G 0x3 + #define NVM_CFG1_PORT_MNM_10G_DRV_LINK_SPEED_25G 0x4 + #define NVM_CFG1_PORT_MNM_10G_DRV_LINK_SPEED_40G 0x5 + #define NVM_CFG1_PORT_MNM_10G_DRV_LINK_SPEED_50G 0x6 + #define NVM_CFG1_PORT_MNM_10G_DRV_LINK_SPEED_BB_100G 0x7 + #define NVM_CFG1_PORT_MNM_10G_MFW_LINK_SPEED_MASK 0x000000F0 + #define NVM_CFG1_PORT_MNM_10G_MFW_LINK_SPEED_OFFSET 4 + #define NVM_CFG1_PORT_MNM_10G_MFW_LINK_SPEED_AUTONEG 0x0 + #define NVM_CFG1_PORT_MNM_10G_MFW_LINK_SPEED_1G 0x1 + #define NVM_CFG1_PORT_MNM_10G_MFW_LINK_SPEED_10G 0x2 + #define NVM_CFG1_PORT_MNM_10G_MFW_LINK_SPEED_20G 0x3 + #define NVM_CFG1_PORT_MNM_10G_MFW_LINK_SPEED_25G 0x4 + #define NVM_CFG1_PORT_MNM_10G_MFW_LINK_SPEED_40G 0x5 + #define NVM_CFG1_PORT_MNM_10G_MFW_LINK_SPEED_50G 0x6 + #define NVM_CFG1_PORT_MNM_10G_MFW_LINK_SPEED_BB_100G 0x7 + /* This field defines the board technology + * (backpane,transceiver,external PHY) + */ + #define NVM_CFG1_PORT_MNM_10G_PORT_TYPE_MASK 0x0000FF00 + #define NVM_CFG1_PORT_MNM_10G_PORT_TYPE_OFFSET 8 + #define NVM_CFG1_PORT_MNM_10G_PORT_TYPE_UNDEFINED 0x0 + #define NVM_CFG1_PORT_MNM_10G_PORT_TYPE_MODULE 0x1 + #define NVM_CFG1_PORT_MNM_10G_PORT_TYPE_BACKPLANE 0x2 + #define NVM_CFG1_PORT_MNM_10G_PORT_TYPE_EXT_PHY 0x3 + #define NVM_CFG1_PORT_MNM_10G_PORT_TYPE_MODULE_SLAVE 0x4 + #define NVM_CFG1_PORT_MNM_10G_SERDES_NET_INTERFACE_MASK \ + 0x00FF0000 + #define NVM_CFG1_PORT_MNM_10G_SERDES_NET_INTERFACE_OFFSET 16 + #define NVM_CFG1_PORT_MNM_10G_SERDES_NET_INTERFACE_BYPASS 0x0 + #define NVM_CFG1_PORT_MNM_10G_SERDES_NET_INTERFACE_KR 0x2 + #define NVM_CFG1_PORT_MNM_10G_SERDES_NET_INTERFACE_KR2 0x3 + #define NVM_CFG1_PORT_MNM_10G_SERDES_NET_INTERFACE_KR4 0x4 + #define NVM_CFG1_PORT_MNM_10G_SERDES_NET_INTERFACE_XFI 0x8 + #define NVM_CFG1_PORT_MNM_10G_SERDES_NET_INTERFACE_SFI 0x9 + #define NVM_CFG1_PORT_MNM_10G_SERDES_NET_INTERFACE_1000X 0xB + #define NVM_CFG1_PORT_MNM_10G_SERDES_NET_INTERFACE_SGMII 0xC + #define NVM_CFG1_PORT_MNM_10G_SERDES_NET_INTERFACE_XLAUI 0x11 + #define NVM_CFG1_PORT_MNM_10G_SERDES_NET_INTERFACE_XLPPI 0x12 + #define NVM_CFG1_PORT_MNM_10G_SERDES_NET_INTERFACE_CAUI 0x21 + #define NVM_CFG1_PORT_MNM_10G_SERDES_NET_INTERFACE_CPPI 0x22 + #define NVM_CFG1_PORT_MNM_10G_SERDES_NET_INTERFACE_25GAUI 0x31 + #define NVM_CFG1_PORT_MNM_10G_ETH_DID_SUFFIX_MASK 0xFF000000 + #define NVM_CFG1_PORT_MNM_10G_ETH_DID_SUFFIX_OFFSET 24 + u32 mnm_10g_misc; /* 0x54 */ + #define NVM_CFG1_PORT_MNM_10G_FEC_FORCE_MODE_MASK 0x00000007 + #define NVM_CFG1_PORT_MNM_10G_FEC_FORCE_MODE_OFFSET 0 + #define NVM_CFG1_PORT_MNM_10G_FEC_FORCE_MODE_NONE 0x0 + #define NVM_CFG1_PORT_MNM_10G_FEC_FORCE_MODE_FIRECODE 0x1 + #define NVM_CFG1_PORT_MNM_10G_FEC_FORCE_MODE_RS 0x2 + #define NVM_CFG1_PORT_MNM_10G_FEC_FORCE_MODE_AUTO 0x7 + u32 mnm_25g_cap; /* 0x58 */ + #define NVM_CFG1_PORT_MNM_25G_DRV_SPEED_CAPABILITY_MASK_MASK \ + 0x0000FFFF + #define NVM_CFG1_PORT_MNM_25G_DRV_SPEED_CAPABILITY_MASK_OFFSET 0 + #define NVM_CFG1_PORT_MNM_25G_DRV_SPEED_CAPABILITY_MASK_1G 0x1 + #define NVM_CFG1_PORT_MNM_25G_DRV_SPEED_CAPABILITY_MASK_10G 0x2 + #define NVM_CFG1_PORT_MNM_25G_DRV_SPEED_CAPABILITY_MASK_20G 0x4 + #define NVM_CFG1_PORT_MNM_25G_DRV_SPEED_CAPABILITY_MASK_25G 0x8 + #define NVM_CFG1_PORT_MNM_25G_DRV_SPEED_CAPABILITY_MASK_40G 0x10 + #define NVM_CFG1_PORT_MNM_25G_DRV_SPEED_CAPABILITY_MASK_50G 0x20 + #define \ + NVM_CFG1_PORT_MNM_25G_DRV_SPEED_CAPABILITY_MASK_BB_100G 0x40 + #define NVM_CFG1_PORT_MNM_25G_MFW_SPEED_CAPABILITY_MASK_MASK \ + 0xFFFF0000 + #define NVM_CFG1_PORT_MNM_25G_MFW_SPEED_CAPABILITY_MASK_OFFSET \ + 16 + #define NVM_CFG1_PORT_MNM_25G_MFW_SPEED_CAPABILITY_MASK_1G 0x1 + #define NVM_CFG1_PORT_MNM_25G_MFW_SPEED_CAPABILITY_MASK_10G 0x2 + #define NVM_CFG1_PORT_MNM_25G_MFW_SPEED_CAPABILITY_MASK_20G 0x4 + #define NVM_CFG1_PORT_MNM_25G_MFW_SPEED_CAPABILITY_MASK_25G 0x8 + #define NVM_CFG1_PORT_MNM_25G_MFW_SPEED_CAPABILITY_MASK_40G 0x10 + #define NVM_CFG1_PORT_MNM_25G_MFW_SPEED_CAPABILITY_MASK_50G 0x20 + #define \ + NVM_CFG1_PORT_MNM_25G_MFW_SPEED_CAPABILITY_MASK_BB_100G 0x40 + u32 mnm_25g_ctrl; /* 0x5C */ + #define NVM_CFG1_PORT_MNM_25G_DRV_LINK_SPEED_MASK 0x0000000F + #define NVM_CFG1_PORT_MNM_25G_DRV_LINK_SPEED_OFFSET 0 + #define NVM_CFG1_PORT_MNM_25G_DRV_LINK_SPEED_AUTONEG 0x0 + #define NVM_CFG1_PORT_MNM_25G_DRV_LINK_SPEED_1G 0x1 + #define NVM_CFG1_PORT_MNM_25G_DRV_LINK_SPEED_10G 0x2 + #define NVM_CFG1_PORT_MNM_25G_DRV_LINK_SPEED_20G 0x3 + #define NVM_CFG1_PORT_MNM_25G_DRV_LINK_SPEED_25G 0x4 + #define NVM_CFG1_PORT_MNM_25G_DRV_LINK_SPEED_40G 0x5 + #define NVM_CFG1_PORT_MNM_25G_DRV_LINK_SPEED_50G 0x6 + #define NVM_CFG1_PORT_MNM_25G_DRV_LINK_SPEED_BB_100G 0x7 + #define NVM_CFG1_PORT_MNM_25G_MFW_LINK_SPEED_MASK 0x000000F0 + #define NVM_CFG1_PORT_MNM_25G_MFW_LINK_SPEED_OFFSET 4 + #define NVM_CFG1_PORT_MNM_25G_MFW_LINK_SPEED_AUTONEG 0x0 + #define NVM_CFG1_PORT_MNM_25G_MFW_LINK_SPEED_1G 0x1 + #define NVM_CFG1_PORT_MNM_25G_MFW_LINK_SPEED_10G 0x2 + #define NVM_CFG1_PORT_MNM_25G_MFW_LINK_SPEED_20G 0x3 + #define NVM_CFG1_PORT_MNM_25G_MFW_LINK_SPEED_25G 0x4 + #define NVM_CFG1_PORT_MNM_25G_MFW_LINK_SPEED_40G 0x5 + #define NVM_CFG1_PORT_MNM_25G_MFW_LINK_SPEED_50G 0x6 + #define NVM_CFG1_PORT_MNM_25G_MFW_LINK_SPEED_BB_100G 0x7 + /* This field defines the board technology + * (backpane,transceiver,external PHY) + */ + #define NVM_CFG1_PORT_MNM_25G_PORT_TYPE_MASK 0x0000FF00 + #define NVM_CFG1_PORT_MNM_25G_PORT_TYPE_OFFSET 8 + #define NVM_CFG1_PORT_MNM_25G_PORT_TYPE_UNDEFINED 0x0 + #define NVM_CFG1_PORT_MNM_25G_PORT_TYPE_MODULE 0x1 + #define NVM_CFG1_PORT_MNM_25G_PORT_TYPE_BACKPLANE 0x2 + #define NVM_CFG1_PORT_MNM_25G_PORT_TYPE_EXT_PHY 0x3 + #define NVM_CFG1_PORT_MNM_25G_PORT_TYPE_MODULE_SLAVE 0x4 + #define NVM_CFG1_PORT_MNM_25G_SERDES_NET_INTERFACE_MASK \ + 0x00FF0000 + #define NVM_CFG1_PORT_MNM_25G_SERDES_NET_INTERFACE_OFFSET 16 + #define NVM_CFG1_PORT_MNM_25G_SERDES_NET_INTERFACE_BYPASS 0x0 + #define NVM_CFG1_PORT_MNM_25G_SERDES_NET_INTERFACE_KR 0x2 + #define NVM_CFG1_PORT_MNM_25G_SERDES_NET_INTERFACE_KR2 0x3 + #define NVM_CFG1_PORT_MNM_25G_SERDES_NET_INTERFACE_KR4 0x4 + #define NVM_CFG1_PORT_MNM_25G_SERDES_NET_INTERFACE_XFI 0x8 + #define NVM_CFG1_PORT_MNM_25G_SERDES_NET_INTERFACE_SFI 0x9 + #define NVM_CFG1_PORT_MNM_25G_SERDES_NET_INTERFACE_1000X 0xB + #define NVM_CFG1_PORT_MNM_25G_SERDES_NET_INTERFACE_SGMII 0xC + #define NVM_CFG1_PORT_MNM_25G_SERDES_NET_INTERFACE_XLAUI 0x11 + #define NVM_CFG1_PORT_MNM_25G_SERDES_NET_INTERFACE_XLPPI 0x12 + #define NVM_CFG1_PORT_MNM_25G_SERDES_NET_INTERFACE_CAUI 0x21 + #define NVM_CFG1_PORT_MNM_25G_SERDES_NET_INTERFACE_CPPI 0x22 + #define NVM_CFG1_PORT_MNM_25G_SERDES_NET_INTERFACE_25GAUI 0x31 + #define NVM_CFG1_PORT_MNM_25G_ETH_DID_SUFFIX_MASK 0xFF000000 + #define NVM_CFG1_PORT_MNM_25G_ETH_DID_SUFFIX_OFFSET 24 + u32 mnm_25g_misc; /* 0x60 */ + #define NVM_CFG1_PORT_MNM_25G_FEC_FORCE_MODE_MASK 0x00000007 + #define NVM_CFG1_PORT_MNM_25G_FEC_FORCE_MODE_OFFSET 0 + #define NVM_CFG1_PORT_MNM_25G_FEC_FORCE_MODE_NONE 0x0 + #define NVM_CFG1_PORT_MNM_25G_FEC_FORCE_MODE_FIRECODE 0x1 + #define NVM_CFG1_PORT_MNM_25G_FEC_FORCE_MODE_RS 0x2 + #define NVM_CFG1_PORT_MNM_25G_FEC_FORCE_MODE_AUTO 0x7 + u32 mnm_40g_cap; /* 0x64 */ + #define NVM_CFG1_PORT_MNM_40G_DRV_SPEED_CAPABILITY_MASK_MASK \ + 0x0000FFFF + #define NVM_CFG1_PORT_MNM_40G_DRV_SPEED_CAPABILITY_MASK_OFFSET 0 + #define NVM_CFG1_PORT_MNM_40G_DRV_SPEED_CAPABILITY_MASK_1G 0x1 + #define NVM_CFG1_PORT_MNM_40G_DRV_SPEED_CAPABILITY_MASK_10G 0x2 + #define NVM_CFG1_PORT_MNM_40G_DRV_SPEED_CAPABILITY_MASK_20G 0x4 + #define NVM_CFG1_PORT_MNM_40G_DRV_SPEED_CAPABILITY_MASK_25G 0x8 + #define NVM_CFG1_PORT_MNM_40G_DRV_SPEED_CAPABILITY_MASK_40G 0x10 + #define NVM_CFG1_PORT_MNM_40G_DRV_SPEED_CAPABILITY_MASK_50G 0x20 + #define \ + NVM_CFG1_PORT_MNM_40G_DRV_SPEED_CAPABILITY_MASK_BB_100G 0x40 + #define NVM_CFG1_PORT_MNM_40G_MFW_SPEED_CAPABILITY_MASK_MASK \ + 0xFFFF0000 + #define NVM_CFG1_PORT_MNM_40G_MFW_SPEED_CAPABILITY_MASK_OFFSET \ + 16 + #define NVM_CFG1_PORT_MNM_40G_MFW_SPEED_CAPABILITY_MASK_1G 0x1 + #define NVM_CFG1_PORT_MNM_40G_MFW_SPEED_CAPABILITY_MASK_10G 0x2 + #define NVM_CFG1_PORT_MNM_40G_MFW_SPEED_CAPABILITY_MASK_20G 0x4 + #define NVM_CFG1_PORT_MNM_40G_MFW_SPEED_CAPABILITY_MASK_25G 0x8 + #define NVM_CFG1_PORT_MNM_40G_MFW_SPEED_CAPABILITY_MASK_40G 0x10 + #define NVM_CFG1_PORT_MNM_40G_MFW_SPEED_CAPABILITY_MASK_50G 0x20 + #define \ + NVM_CFG1_PORT_MNM_40G_MFW_SPEED_CAPABILITY_MASK_BB_100G 0x40 + u32 mnm_40g_ctrl; /* 0x68 */ + #define NVM_CFG1_PORT_MNM_40G_DRV_LINK_SPEED_MASK 0x0000000F + #define NVM_CFG1_PORT_MNM_40G_DRV_LINK_SPEED_OFFSET 0 + #define NVM_CFG1_PORT_MNM_40G_DRV_LINK_SPEED_AUTONEG 0x0 + #define NVM_CFG1_PORT_MNM_40G_DRV_LINK_SPEED_1G 0x1 + #define NVM_CFG1_PORT_MNM_40G_DRV_LINK_SPEED_10G 0x2 + #define NVM_CFG1_PORT_MNM_40G_DRV_LINK_SPEED_20G 0x3 + #define NVM_CFG1_PORT_MNM_40G_DRV_LINK_SPEED_25G 0x4 + #define NVM_CFG1_PORT_MNM_40G_DRV_LINK_SPEED_40G 0x5 + #define NVM_CFG1_PORT_MNM_40G_DRV_LINK_SPEED_50G 0x6 + #define NVM_CFG1_PORT_MNM_40G_DRV_LINK_SPEED_BB_100G 0x7 + #define NVM_CFG1_PORT_MNM_40G_MFW_LINK_SPEED_MASK 0x000000F0 + #define NVM_CFG1_PORT_MNM_40G_MFW_LINK_SPEED_OFFSET 4 + #define NVM_CFG1_PORT_MNM_40G_MFW_LINK_SPEED_AUTONEG 0x0 + #define NVM_CFG1_PORT_MNM_40G_MFW_LINK_SPEED_1G 0x1 + #define NVM_CFG1_PORT_MNM_40G_MFW_LINK_SPEED_10G 0x2 + #define NVM_CFG1_PORT_MNM_40G_MFW_LINK_SPEED_20G 0x3 + #define NVM_CFG1_PORT_MNM_40G_MFW_LINK_SPEED_25G 0x4 + #define NVM_CFG1_PORT_MNM_40G_MFW_LINK_SPEED_40G 0x5 + #define NVM_CFG1_PORT_MNM_40G_MFW_LINK_SPEED_50G 0x6 + #define NVM_CFG1_PORT_MNM_40G_MFW_LINK_SPEED_BB_100G 0x7 + /* This field defines the board technology + * (backpane,transceiver,external PHY) + */ + #define NVM_CFG1_PORT_MNM_40G_PORT_TYPE_MASK 0x0000FF00 + #define NVM_CFG1_PORT_MNM_40G_PORT_TYPE_OFFSET 8 + #define NVM_CFG1_PORT_MNM_40G_PORT_TYPE_UNDEFINED 0x0 + #define NVM_CFG1_PORT_MNM_40G_PORT_TYPE_MODULE 0x1 + #define NVM_CFG1_PORT_MNM_40G_PORT_TYPE_BACKPLANE 0x2 + #define NVM_CFG1_PORT_MNM_40G_PORT_TYPE_EXT_PHY 0x3 + #define NVM_CFG1_PORT_MNM_40G_PORT_TYPE_MODULE_SLAVE 0x4 + #define NVM_CFG1_PORT_MNM_40G_SERDES_NET_INTERFACE_MASK \ + 0x00FF0000 + #define NVM_CFG1_PORT_MNM_40G_SERDES_NET_INTERFACE_OFFSET 16 + #define NVM_CFG1_PORT_MNM_40G_SERDES_NET_INTERFACE_BYPASS 0x0 + #define NVM_CFG1_PORT_MNM_40G_SERDES_NET_INTERFACE_KR 0x2 + #define NVM_CFG1_PORT_MNM_40G_SERDES_NET_INTERFACE_KR2 0x3 + #define NVM_CFG1_PORT_MNM_40G_SERDES_NET_INTERFACE_KR4 0x4 + #define NVM_CFG1_PORT_MNM_40G_SERDES_NET_INTERFACE_XFI 0x8 + #define NVM_CFG1_PORT_MNM_40G_SERDES_NET_INTERFACE_SFI 0x9 + #define NVM_CFG1_PORT_MNM_40G_SERDES_NET_INTERFACE_1000X 0xB + #define NVM_CFG1_PORT_MNM_40G_SERDES_NET_INTERFACE_SGMII 0xC + #define NVM_CFG1_PORT_MNM_40G_SERDES_NET_INTERFACE_XLAUI 0x11 + #define NVM_CFG1_PORT_MNM_40G_SERDES_NET_INTERFACE_XLPPI 0x12 + #define NVM_CFG1_PORT_MNM_40G_SERDES_NET_INTERFACE_CAUI 0x21 + #define NVM_CFG1_PORT_MNM_40G_SERDES_NET_INTERFACE_CPPI 0x22 + #define NVM_CFG1_PORT_MNM_40G_SERDES_NET_INTERFACE_25GAUI 0x31 + #define NVM_CFG1_PORT_MNM_40G_ETH_DID_SUFFIX_MASK 0xFF000000 + #define NVM_CFG1_PORT_MNM_40G_ETH_DID_SUFFIX_OFFSET 24 + u32 mnm_40g_misc; /* 0x6C */ + #define NVM_CFG1_PORT_MNM_40G_FEC_FORCE_MODE_MASK 0x00000007 + #define NVM_CFG1_PORT_MNM_40G_FEC_FORCE_MODE_OFFSET 0 + #define NVM_CFG1_PORT_MNM_40G_FEC_FORCE_MODE_NONE 0x0 + #define NVM_CFG1_PORT_MNM_40G_FEC_FORCE_MODE_FIRECODE 0x1 + #define NVM_CFG1_PORT_MNM_40G_FEC_FORCE_MODE_RS 0x2 + #define NVM_CFG1_PORT_MNM_40G_FEC_FORCE_MODE_AUTO 0x7 + u32 mnm_50g_cap; /* 0x70 */ + #define NVM_CFG1_PORT_MNM_50G_DRV_SPEED_CAPABILITY_MASK_MASK \ + 0x0000FFFF + #define NVM_CFG1_PORT_MNM_50G_DRV_SPEED_CAPABILITY_MASK_OFFSET 0 + #define NVM_CFG1_PORT_MNM_50G_DRV_SPEED_CAPABILITY_MASK_1G 0x1 + #define NVM_CFG1_PORT_MNM_50G_DRV_SPEED_CAPABILITY_MASK_10G 0x2 + #define NVM_CFG1_PORT_MNM_50G_DRV_SPEED_CAPABILITY_MASK_20G 0x4 + #define NVM_CFG1_PORT_MNM_50G_DRV_SPEED_CAPABILITY_MASK_25G 0x8 + #define NVM_CFG1_PORT_MNM_50G_DRV_SPEED_CAPABILITY_MASK_40G 0x10 + #define NVM_CFG1_PORT_MNM_50G_DRV_SPEED_CAPABILITY_MASK_50G 0x20 + #define \ + NVM_CFG1_PORT_MNM_50G_DRV_SPEED_CAPABILITY_MASK_BB_100G \ + 0x40 + #define NVM_CFG1_PORT_MNM_50G_MFW_SPEED_CAPABILITY_MASK_MASK \ + 0xFFFF0000 + #define NVM_CFG1_PORT_MNM_50G_MFW_SPEED_CAPABILITY_MASK_OFFSET \ + 16 + #define NVM_CFG1_PORT_MNM_50G_MFW_SPEED_CAPABILITY_MASK_1G 0x1 + #define NVM_CFG1_PORT_MNM_50G_MFW_SPEED_CAPABILITY_MASK_10G 0x2 + #define NVM_CFG1_PORT_MNM_50G_MFW_SPEED_CAPABILITY_MASK_20G 0x4 + #define NVM_CFG1_PORT_MNM_50G_MFW_SPEED_CAPABILITY_MASK_25G 0x8 + #define NVM_CFG1_PORT_MNM_50G_MFW_SPEED_CAPABILITY_MASK_40G 0x10 + #define NVM_CFG1_PORT_MNM_50G_MFW_SPEED_CAPABILITY_MASK_50G 0x20 + #define \ + NVM_CFG1_PORT_MNM_50G_MFW_SPEED_CAPABILITY_MASK_BB_100G \ + 0x40 + u32 mnm_50g_ctrl; /* 0x74 */ + #define NVM_CFG1_PORT_MNM_50G_DRV_LINK_SPEED_MASK 0x0000000F + #define NVM_CFG1_PORT_MNM_50G_DRV_LINK_SPEED_OFFSET 0 + #define NVM_CFG1_PORT_MNM_50G_DRV_LINK_SPEED_AUTONEG 0x0 + #define NVM_CFG1_PORT_MNM_50G_DRV_LINK_SPEED_1G 0x1 + #define NVM_CFG1_PORT_MNM_50G_DRV_LINK_SPEED_10G 0x2 + #define NVM_CFG1_PORT_MNM_50G_DRV_LINK_SPEED_20G 0x3 + #define NVM_CFG1_PORT_MNM_50G_DRV_LINK_SPEED_25G 0x4 + #define NVM_CFG1_PORT_MNM_50G_DRV_LINK_SPEED_40G 0x5 + #define NVM_CFG1_PORT_MNM_50G_DRV_LINK_SPEED_50G 0x6 + #define NVM_CFG1_PORT_MNM_50G_DRV_LINK_SPEED_BB_100G 0x7 + #define NVM_CFG1_PORT_MNM_50G_MFW_LINK_SPEED_MASK 0x000000F0 + #define NVM_CFG1_PORT_MNM_50G_MFW_LINK_SPEED_OFFSET 4 + #define NVM_CFG1_PORT_MNM_50G_MFW_LINK_SPEED_AUTONEG 0x0 + #define NVM_CFG1_PORT_MNM_50G_MFW_LINK_SPEED_1G 0x1 + #define NVM_CFG1_PORT_MNM_50G_MFW_LINK_SPEED_10G 0x2 + #define NVM_CFG1_PORT_MNM_50G_MFW_LINK_SPEED_20G 0x3 + #define NVM_CFG1_PORT_MNM_50G_MFW_LINK_SPEED_25G 0x4 + #define NVM_CFG1_PORT_MNM_50G_MFW_LINK_SPEED_40G 0x5 + #define NVM_CFG1_PORT_MNM_50G_MFW_LINK_SPEED_50G 0x6 + #define NVM_CFG1_PORT_MNM_50G_MFW_LINK_SPEED_BB_100G 0x7 + /* This field defines the board technology + * (backpane,transceiver,external PHY) + */ + #define NVM_CFG1_PORT_MNM_50G_PORT_TYPE_MASK 0x0000FF00 + #define NVM_CFG1_PORT_MNM_50G_PORT_TYPE_OFFSET 8 + #define NVM_CFG1_PORT_MNM_50G_PORT_TYPE_UNDEFINED 0x0 + #define NVM_CFG1_PORT_MNM_50G_PORT_TYPE_MODULE 0x1 + #define NVM_CFG1_PORT_MNM_50G_PORT_TYPE_BACKPLANE 0x2 + #define NVM_CFG1_PORT_MNM_50G_PORT_TYPE_EXT_PHY 0x3 + #define NVM_CFG1_PORT_MNM_50G_PORT_TYPE_MODULE_SLAVE 0x4 + #define NVM_CFG1_PORT_MNM_50G_SERDES_NET_INTERFACE_MASK \ + 0x00FF0000 + #define NVM_CFG1_PORT_MNM_50G_SERDES_NET_INTERFACE_OFFSET 16 + #define NVM_CFG1_PORT_MNM_50G_SERDES_NET_INTERFACE_BYPASS 0x0 + #define NVM_CFG1_PORT_MNM_50G_SERDES_NET_INTERFACE_KR 0x2 + #define NVM_CFG1_PORT_MNM_50G_SERDES_NET_INTERFACE_KR2 0x3 + #define NVM_CFG1_PORT_MNM_50G_SERDES_NET_INTERFACE_KR4 0x4 + #define NVM_CFG1_PORT_MNM_50G_SERDES_NET_INTERFACE_XFI 0x8 + #define NVM_CFG1_PORT_MNM_50G_SERDES_NET_INTERFACE_SFI 0x9 + #define NVM_CFG1_PORT_MNM_50G_SERDES_NET_INTERFACE_1000X 0xB + #define NVM_CFG1_PORT_MNM_50G_SERDES_NET_INTERFACE_SGMII 0xC + #define NVM_CFG1_PORT_MNM_50G_SERDES_NET_INTERFACE_XLAUI 0x11 + #define NVM_CFG1_PORT_MNM_50G_SERDES_NET_INTERFACE_XLPPI 0x12 + #define NVM_CFG1_PORT_MNM_50G_SERDES_NET_INTERFACE_CAUI 0x21 + #define NVM_CFG1_PORT_MNM_50G_SERDES_NET_INTERFACE_CPPI 0x22 + #define NVM_CFG1_PORT_MNM_50G_SERDES_NET_INTERFACE_25GAUI 0x31 + #define NVM_CFG1_PORT_MNM_50G_ETH_DID_SUFFIX_MASK 0xFF000000 + #define NVM_CFG1_PORT_MNM_50G_ETH_DID_SUFFIX_OFFSET 24 + u32 mnm_50g_misc; /* 0x78 */ + #define NVM_CFG1_PORT_MNM_50G_FEC_FORCE_MODE_MASK 0x00000007 + #define NVM_CFG1_PORT_MNM_50G_FEC_FORCE_MODE_OFFSET 0 + #define NVM_CFG1_PORT_MNM_50G_FEC_FORCE_MODE_NONE 0x0 + #define NVM_CFG1_PORT_MNM_50G_FEC_FORCE_MODE_FIRECODE 0x1 + #define NVM_CFG1_PORT_MNM_50G_FEC_FORCE_MODE_RS 0x2 + #define NVM_CFG1_PORT_MNM_50G_FEC_FORCE_MODE_AUTO 0x7 + u32 mnm_100g_cap; /* 0x7C */ + #define NVM_CFG1_PORT_MNM_100G_DRV_SPEED_CAP_MASK_MASK \ + 0x0000FFFF + #define NVM_CFG1_PORT_MNM_100G_DRV_SPEED_CAP_MASK_OFFSET 0 + #define NVM_CFG1_PORT_MNM_100G_DRV_SPEED_CAP_MASK_1G 0x1 + #define NVM_CFG1_PORT_MNM_100G_DRV_SPEED_CAP_MASK_10G 0x2 + #define NVM_CFG1_PORT_MNM_100G_DRV_SPEED_CAP_MASK_20G 0x4 + #define NVM_CFG1_PORT_MNM_100G_DRV_SPEED_CAP_MASK_25G 0x8 + #define NVM_CFG1_PORT_MNM_100G_DRV_SPEED_CAP_MASK_40G 0x10 + #define NVM_CFG1_PORT_MNM_100G_DRV_SPEED_CAP_MASK_50G 0x20 + #define NVM_CFG1_PORT_MNM_100G_DRV_SPEED_CAP_MASK_BB_100G 0x40 + #define NVM_CFG1_PORT_MNM_100G_MFW_SPEED_CAP_MASK_MASK \ + 0xFFFF0000 + #define NVM_CFG1_PORT_MNM_100G_MFW_SPEED_CAP_MASK_OFFSET 16 + #define NVM_CFG1_PORT_MNM_100G_MFW_SPEED_CAP_MASK_1G 0x1 + #define NVM_CFG1_PORT_MNM_100G_MFW_SPEED_CAP_MASK_10G 0x2 + #define NVM_CFG1_PORT_MNM_100G_MFW_SPEED_CAP_MASK_20G 0x4 + #define NVM_CFG1_PORT_MNM_100G_MFW_SPEED_CAP_MASK_25G 0x8 + #define NVM_CFG1_PORT_MNM_100G_MFW_SPEED_CAP_MASK_40G 0x10 + #define NVM_CFG1_PORT_MNM_100G_MFW_SPEED_CAP_MASK_50G 0x20 + #define NVM_CFG1_PORT_MNM_100G_MFW_SPEED_CAP_MASK_BB_100G 0x40 + u32 mnm_100g_ctrl; /* 0x80 */ + #define NVM_CFG1_PORT_MNM_100G_DRV_LINK_SPEED_MASK 0x0000000F + #define NVM_CFG1_PORT_MNM_100G_DRV_LINK_SPEED_OFFSET 0 + #define NVM_CFG1_PORT_MNM_100G_DRV_LINK_SPEED_AUTONEG 0x0 + #define NVM_CFG1_PORT_MNM_100G_DRV_LINK_SPEED_1G 0x1 + #define NVM_CFG1_PORT_MNM_100G_DRV_LINK_SPEED_10G 0x2 + #define NVM_CFG1_PORT_MNM_100G_DRV_LINK_SPEED_20G 0x3 + #define NVM_CFG1_PORT_MNM_100G_DRV_LINK_SPEED_25G 0x4 + #define NVM_CFG1_PORT_MNM_100G_DRV_LINK_SPEED_40G 0x5 + #define NVM_CFG1_PORT_MNM_100G_DRV_LINK_SPEED_50G 0x6 + #define NVM_CFG1_PORT_MNM_100G_DRV_LINK_SPEED_BB_100G 0x7 + #define NVM_CFG1_PORT_MNM_100G_MFW_LINK_SPEED_MASK 0x000000F0 + #define NVM_CFG1_PORT_MNM_100G_MFW_LINK_SPEED_OFFSET 4 + #define NVM_CFG1_PORT_MNM_100G_MFW_LINK_SPEED_AUTONEG 0x0 + #define NVM_CFG1_PORT_MNM_100G_MFW_LINK_SPEED_1G 0x1 + #define NVM_CFG1_PORT_MNM_100G_MFW_LINK_SPEED_10G 0x2 + #define NVM_CFG1_PORT_MNM_100G_MFW_LINK_SPEED_20G 0x3 + #define NVM_CFG1_PORT_MNM_100G_MFW_LINK_SPEED_25G 0x4 + #define NVM_CFG1_PORT_MNM_100G_MFW_LINK_SPEED_40G 0x5 + #define NVM_CFG1_PORT_MNM_100G_MFW_LINK_SPEED_50G 0x6 + #define NVM_CFG1_PORT_MNM_100G_MFW_LINK_SPEED_BB_100G 0x7 + /* This field defines the board technology + * (backpane,transceiver,external PHY) + */ + #define NVM_CFG1_PORT_MNM_100G_PORT_TYPE_MASK 0x0000FF00 + #define NVM_CFG1_PORT_MNM_100G_PORT_TYPE_OFFSET 8 + #define NVM_CFG1_PORT_MNM_100G_PORT_TYPE_UNDEFINED 0x0 + #define NVM_CFG1_PORT_MNM_100G_PORT_TYPE_MODULE 0x1 + #define NVM_CFG1_PORT_MNM_100G_PORT_TYPE_BACKPLANE 0x2 + #define NVM_CFG1_PORT_MNM_100G_PORT_TYPE_EXT_PHY 0x3 + #define NVM_CFG1_PORT_MNM_100G_PORT_TYPE_MODULE_SLAVE 0x4 + #define NVM_CFG1_PORT_MNM_100G_SERDES_NET_INTERFACE_MASK \ + 0x00FF0000 + #define NVM_CFG1_PORT_MNM_100G_SERDES_NET_INTERFACE_OFFSET 16 + #define NVM_CFG1_PORT_MNM_100G_SERDES_NET_INTERFACE_BYPASS 0x0 + #define NVM_CFG1_PORT_MNM_100G_SERDES_NET_INTERFACE_KR 0x2 + #define NVM_CFG1_PORT_MNM_100G_SERDES_NET_INTERFACE_KR2 0x3 + #define NVM_CFG1_PORT_MNM_100G_SERDES_NET_INTERFACE_KR4 0x4 + #define NVM_CFG1_PORT_MNM_100G_SERDES_NET_INTERFACE_XFI 0x8 + #define NVM_CFG1_PORT_MNM_100G_SERDES_NET_INTERFACE_SFI 0x9 + #define NVM_CFG1_PORT_MNM_100G_SERDES_NET_INTERFACE_1000X 0xB + #define NVM_CFG1_PORT_MNM_100G_SERDES_NET_INTERFACE_SGMII 0xC + #define NVM_CFG1_PORT_MNM_100G_SERDES_NET_INTERFACE_XLAUI 0x11 + #define NVM_CFG1_PORT_MNM_100G_SERDES_NET_INTERFACE_XLPPI 0x12 + #define NVM_CFG1_PORT_MNM_100G_SERDES_NET_INTERFACE_CAUI 0x21 + #define NVM_CFG1_PORT_MNM_100G_SERDES_NET_INTERFACE_CPPI 0x22 + #define NVM_CFG1_PORT_MNM_100G_SERDES_NET_INTERFACE_25GAUI 0x31 + #define NVM_CFG1_PORT_MNM_100G_ETH_DID_SUFFIX_MASK 0xFF000000 + #define NVM_CFG1_PORT_MNM_100G_ETH_DID_SUFFIX_OFFSET 24 + u32 mnm_100g_misc; /* 0x84 */ + #define NVM_CFG1_PORT_MNM_100G_FEC_FORCE_MODE_MASK 0x00000007 + #define NVM_CFG1_PORT_MNM_100G_FEC_FORCE_MODE_OFFSET 0 + #define NVM_CFG1_PORT_MNM_100G_FEC_FORCE_MODE_NONE 0x0 + #define NVM_CFG1_PORT_MNM_100G_FEC_FORCE_MODE_FIRECODE 0x1 + #define NVM_CFG1_PORT_MNM_100G_FEC_FORCE_MODE_RS 0x2 + #define NVM_CFG1_PORT_MNM_100G_FEC_FORCE_MODE_AUTO 0x7 + u32 temperature; /* 0x88 */ + #define NVM_CFG1_PORT_PHY_MODULE_DEAD_TEMP_TH_MASK 0x000000FF + #define NVM_CFG1_PORT_PHY_MODULE_DEAD_TEMP_TH_OFFSET 0 + #define NVM_CFG1_PORT_PHY_MODULE_ALOM_FAN_ON_TEMP_TH_MASK \ + 0x0000FF00 + #define NVM_CFG1_PORT_PHY_MODULE_ALOM_FAN_ON_TEMP_TH_OFFSET 8 + u32 reserved[115]; /* 0x8C */ }; struct nvm_cfg1_func { - struct nvm_cfg_mac_address mac_address; /* 0x0 */ - u32 rsrv1; /* 0x8 */ -#define NVM_CFG1_FUNC_RESERVED1_MASK 0x0000FFFF -#define NVM_CFG1_FUNC_RESERVED1_OFFSET 0 -#define NVM_CFG1_FUNC_RESERVED2_MASK 0xFFFF0000 -#define NVM_CFG1_FUNC_RESERVED2_OFFSET 16 - u32 rsrv2; /* 0xC */ -#define NVM_CFG1_FUNC_RESERVED3_MASK 0x0000FFFF -#define NVM_CFG1_FUNC_RESERVED3_OFFSET 0 -#define NVM_CFG1_FUNC_RESERVED4_MASK 0xFFFF0000 -#define NVM_CFG1_FUNC_RESERVED4_OFFSET 16 - u32 device_id; /* 0x10 */ -#define NVM_CFG1_FUNC_MF_VENDOR_DEVICE_ID_MASK 0x0000FFFF -#define NVM_CFG1_FUNC_MF_VENDOR_DEVICE_ID_OFFSET 0 -#define NVM_CFG1_FUNC_RESERVED77_MASK 0xFFFF0000 -#define NVM_CFG1_FUNC_RESERVED77_OFFSET 16 - u32 cmn_cfg; /* 0x14 */ -#define NVM_CFG1_FUNC_PREBOOT_BOOT_PROTOCOL_MASK 0x00000007 -#define NVM_CFG1_FUNC_PREBOOT_BOOT_PROTOCOL_OFFSET 0 -#define NVM_CFG1_FUNC_PREBOOT_BOOT_PROTOCOL_PXE 0x0 -#define NVM_CFG1_FUNC_PREBOOT_BOOT_PROTOCOL_NONE 0x7 -#define NVM_CFG1_FUNC_VF_PCI_DEVICE_ID_MASK 0x0007FFF8 -#define NVM_CFG1_FUNC_VF_PCI_DEVICE_ID_OFFSET 3 -#define NVM_CFG1_FUNC_PERSONALITY_MASK 0x00780000 -#define NVM_CFG1_FUNC_PERSONALITY_OFFSET 19 -#define NVM_CFG1_FUNC_PERSONALITY_ETHERNET 0x0 -#define NVM_CFG1_FUNC_BANDWIDTH_WEIGHT_MASK 0x7F800000 -#define NVM_CFG1_FUNC_BANDWIDTH_WEIGHT_OFFSET 23 -#define NVM_CFG1_FUNC_PAUSE_ON_HOST_RING_MASK 0x80000000 -#define NVM_CFG1_FUNC_PAUSE_ON_HOST_RING_OFFSET 31 -#define NVM_CFG1_FUNC_PAUSE_ON_HOST_RING_DISABLED 0x0 -#define NVM_CFG1_FUNC_PAUSE_ON_HOST_RING_ENABLED 0x1 - u32 pci_cfg; /* 0x18 */ -#define NVM_CFG1_FUNC_NUMBER_OF_VFS_PER_PF_MASK 0x0000007F -#define NVM_CFG1_FUNC_NUMBER_OF_VFS_PER_PF_OFFSET 0 -#define NVM_CFG1_FUNC_RESERVESD12_MASK 0x00003F80 -#define NVM_CFG1_FUNC_RESERVESD12_OFFSET 7 -#define NVM_CFG1_FUNC_BAR1_SIZE_MASK 0x0003C000 -#define NVM_CFG1_FUNC_BAR1_SIZE_OFFSET 14 -#define NVM_CFG1_FUNC_BAR1_SIZE_DISABLED 0x0 -#define NVM_CFG1_FUNC_BAR1_SIZE_64K 0x1 -#define NVM_CFG1_FUNC_BAR1_SIZE_128K 0x2 -#define NVM_CFG1_FUNC_BAR1_SIZE_256K 0x3 -#define NVM_CFG1_FUNC_BAR1_SIZE_512K 0x4 -#define NVM_CFG1_FUNC_BAR1_SIZE_1M 0x5 -#define NVM_CFG1_FUNC_BAR1_SIZE_2M 0x6 -#define NVM_CFG1_FUNC_BAR1_SIZE_4M 0x7 -#define NVM_CFG1_FUNC_BAR1_SIZE_8M 0x8 -#define NVM_CFG1_FUNC_BAR1_SIZE_16M 0x9 -#define NVM_CFG1_FUNC_BAR1_SIZE_32M 0xA -#define NVM_CFG1_FUNC_BAR1_SIZE_64M 0xB -#define NVM_CFG1_FUNC_BAR1_SIZE_128M 0xC -#define NVM_CFG1_FUNC_BAR1_SIZE_256M 0xD -#define NVM_CFG1_FUNC_BAR1_SIZE_512M 0xE -#define NVM_CFG1_FUNC_BAR1_SIZE_1G 0xF -#define NVM_CFG1_FUNC_MAX_BANDWIDTH_MASK 0x03FC0000 -#define NVM_CFG1_FUNC_MAX_BANDWIDTH_OFFSET 18 - u32 preboot_generic_cfg; /* 0x2C */ -#define NVM_CFG1_FUNC_PREBOOT_VLAN_VALUE_MASK 0x0000FFFF -#define NVM_CFG1_FUNC_PREBOOT_VLAN_VALUE_OFFSET 0 -#define NVM_CFG1_FUNC_PREBOOT_VLAN_MASK 0x00010000 -#define NVM_CFG1_FUNC_PREBOOT_VLAN_OFFSET 16 - u32 reserved[8]; /* 0x30 */ + struct nvm_cfg_mac_address mac_address; /* 0x0 */ + u32 rsrv1; /* 0x8 */ + #define NVM_CFG1_FUNC_RESERVED1_MASK 0x0000FFFF + #define NVM_CFG1_FUNC_RESERVED1_OFFSET 0 + #define NVM_CFG1_FUNC_RESERVED2_MASK 0xFFFF0000 + #define NVM_CFG1_FUNC_RESERVED2_OFFSET 16 + u32 rsrv2; /* 0xC */ + #define NVM_CFG1_FUNC_RESERVED3_MASK 0x0000FFFF + #define NVM_CFG1_FUNC_RESERVED3_OFFSET 0 + #define NVM_CFG1_FUNC_RESERVED4_MASK 0xFFFF0000 + #define NVM_CFG1_FUNC_RESERVED4_OFFSET 16 + u32 device_id; /* 0x10 */ + #define NVM_CFG1_FUNC_MF_VENDOR_DEVICE_ID_MASK 0x0000FFFF + #define NVM_CFG1_FUNC_MF_VENDOR_DEVICE_ID_OFFSET 0 + #define NVM_CFG1_FUNC_RESERVED77_MASK 0xFFFF0000 + #define NVM_CFG1_FUNC_RESERVED77_OFFSET 16 + u32 cmn_cfg; /* 0x14 */ + #define NVM_CFG1_FUNC_PREBOOT_BOOT_PROTOCOL_MASK 0x00000007 + #define NVM_CFG1_FUNC_PREBOOT_BOOT_PROTOCOL_OFFSET 0 + #define NVM_CFG1_FUNC_PREBOOT_BOOT_PROTOCOL_PXE 0x0 + #define NVM_CFG1_FUNC_PREBOOT_BOOT_PROTOCOL_ISCSI_BOOT 0x3 + #define NVM_CFG1_FUNC_PREBOOT_BOOT_PROTOCOL_FCOE_BOOT 0x4 + #define NVM_CFG1_FUNC_PREBOOT_BOOT_PROTOCOL_NONE 0x7 + #define NVM_CFG1_FUNC_VF_PCI_DEVICE_ID_MASK 0x0007FFF8 + #define NVM_CFG1_FUNC_VF_PCI_DEVICE_ID_OFFSET 3 + #define NVM_CFG1_FUNC_PERSONALITY_MASK 0x00780000 + #define NVM_CFG1_FUNC_PERSONALITY_OFFSET 19 + #define NVM_CFG1_FUNC_PERSONALITY_ETHERNET 0x0 + #define NVM_CFG1_FUNC_PERSONALITY_ISCSI 0x1 + #define NVM_CFG1_FUNC_PERSONALITY_FCOE 0x2 + #define NVM_CFG1_FUNC_PERSONALITY_ROCE 0x3 + #define NVM_CFG1_FUNC_BANDWIDTH_WEIGHT_MASK 0x7F800000 + #define NVM_CFG1_FUNC_BANDWIDTH_WEIGHT_OFFSET 23 + #define NVM_CFG1_FUNC_PAUSE_ON_HOST_RING_MASK 0x80000000 + #define NVM_CFG1_FUNC_PAUSE_ON_HOST_RING_OFFSET 31 + #define NVM_CFG1_FUNC_PAUSE_ON_HOST_RING_DISABLED 0x0 + #define NVM_CFG1_FUNC_PAUSE_ON_HOST_RING_ENABLED 0x1 + u32 pci_cfg; /* 0x18 */ + #define NVM_CFG1_FUNC_NUMBER_OF_VFS_PER_PF_MASK 0x0000007F + #define NVM_CFG1_FUNC_NUMBER_OF_VFS_PER_PF_OFFSET 0 + /* AH VF BAR2 size */ + #define NVM_CFG1_FUNC_VF_PCI_BAR2_SIZE_MASK 0x00003F80 + #define NVM_CFG1_FUNC_VF_PCI_BAR2_SIZE_OFFSET 7 + #define NVM_CFG1_FUNC_VF_PCI_BAR2_SIZE_DISABLED 0x0 + #define NVM_CFG1_FUNC_VF_PCI_BAR2_SIZE_4K 0x1 + #define NVM_CFG1_FUNC_VF_PCI_BAR2_SIZE_8K 0x2 + #define NVM_CFG1_FUNC_VF_PCI_BAR2_SIZE_16K 0x3 + #define NVM_CFG1_FUNC_VF_PCI_BAR2_SIZE_32K 0x4 + #define NVM_CFG1_FUNC_VF_PCI_BAR2_SIZE_64K 0x5 + #define NVM_CFG1_FUNC_VF_PCI_BAR2_SIZE_128K 0x6 + #define NVM_CFG1_FUNC_VF_PCI_BAR2_SIZE_256K 0x7 + #define NVM_CFG1_FUNC_VF_PCI_BAR2_SIZE_512K 0x8 + #define NVM_CFG1_FUNC_VF_PCI_BAR2_SIZE_1M 0x9 + #define NVM_CFG1_FUNC_VF_PCI_BAR2_SIZE_2M 0xA + #define NVM_CFG1_FUNC_VF_PCI_BAR2_SIZE_4M 0xB + #define NVM_CFG1_FUNC_VF_PCI_BAR2_SIZE_8M 0xC + #define NVM_CFG1_FUNC_VF_PCI_BAR2_SIZE_16M 0xD + #define NVM_CFG1_FUNC_VF_PCI_BAR2_SIZE_32M 0xE + #define NVM_CFG1_FUNC_VF_PCI_BAR2_SIZE_64M 0xF + #define NVM_CFG1_FUNC_BAR1_SIZE_MASK 0x0003C000 + #define NVM_CFG1_FUNC_BAR1_SIZE_OFFSET 14 + #define NVM_CFG1_FUNC_BAR1_SIZE_DISABLED 0x0 + #define NVM_CFG1_FUNC_BAR1_SIZE_64K 0x1 + #define NVM_CFG1_FUNC_BAR1_SIZE_128K 0x2 + #define NVM_CFG1_FUNC_BAR1_SIZE_256K 0x3 + #define NVM_CFG1_FUNC_BAR1_SIZE_512K 0x4 + #define NVM_CFG1_FUNC_BAR1_SIZE_1M 0x5 + #define NVM_CFG1_FUNC_BAR1_SIZE_2M 0x6 + #define NVM_CFG1_FUNC_BAR1_SIZE_4M 0x7 + #define NVM_CFG1_FUNC_BAR1_SIZE_8M 0x8 + #define NVM_CFG1_FUNC_BAR1_SIZE_16M 0x9 + #define NVM_CFG1_FUNC_BAR1_SIZE_32M 0xA + #define NVM_CFG1_FUNC_BAR1_SIZE_64M 0xB + #define NVM_CFG1_FUNC_BAR1_SIZE_128M 0xC + #define NVM_CFG1_FUNC_BAR1_SIZE_256M 0xD + #define NVM_CFG1_FUNC_BAR1_SIZE_512M 0xE + #define NVM_CFG1_FUNC_BAR1_SIZE_1G 0xF + #define NVM_CFG1_FUNC_MAX_BANDWIDTH_MASK 0x03FC0000 + #define NVM_CFG1_FUNC_MAX_BANDWIDTH_OFFSET 18 + /* Hide function in npar mode */ + #define NVM_CFG1_FUNC_FUNCTION_HIDE_MASK 0x04000000 + #define NVM_CFG1_FUNC_FUNCTION_HIDE_OFFSET 26 + #define NVM_CFG1_FUNC_FUNCTION_HIDE_DISABLED 0x0 + #define NVM_CFG1_FUNC_FUNCTION_HIDE_ENABLED 0x1 + /* AH BAR2 size (per function) */ + #define NVM_CFG1_FUNC_BAR2_SIZE_MASK 0x78000000 + #define NVM_CFG1_FUNC_BAR2_SIZE_OFFSET 27 + #define NVM_CFG1_FUNC_BAR2_SIZE_DISABLED 0x0 + #define NVM_CFG1_FUNC_BAR2_SIZE_1M 0x5 + #define NVM_CFG1_FUNC_BAR2_SIZE_2M 0x6 + #define NVM_CFG1_FUNC_BAR2_SIZE_4M 0x7 + #define NVM_CFG1_FUNC_BAR2_SIZE_8M 0x8 + #define NVM_CFG1_FUNC_BAR2_SIZE_16M 0x9 + #define NVM_CFG1_FUNC_BAR2_SIZE_32M 0xA + #define NVM_CFG1_FUNC_BAR2_SIZE_64M 0xB + #define NVM_CFG1_FUNC_BAR2_SIZE_128M 0xC + #define NVM_CFG1_FUNC_BAR2_SIZE_256M 0xD + #define NVM_CFG1_FUNC_BAR2_SIZE_512M 0xE + #define NVM_CFG1_FUNC_BAR2_SIZE_1G 0xF + struct nvm_cfg_mac_address fcoe_node_wwn_mac_addr; /* 0x1C */ + struct nvm_cfg_mac_address fcoe_port_wwn_mac_addr; /* 0x24 */ + u32 preboot_generic_cfg; /* 0x2C */ + #define NVM_CFG1_FUNC_PREBOOT_VLAN_VALUE_MASK 0x0000FFFF + #define NVM_CFG1_FUNC_PREBOOT_VLAN_VALUE_OFFSET 0 + #define NVM_CFG1_FUNC_PREBOOT_VLAN_MASK 0x00010000 + #define NVM_CFG1_FUNC_PREBOOT_VLAN_OFFSET 16 + #define NVM_CFG1_FUNC_NPAR_ENABLED_PROTOCOL_MASK 0x001E0000 + #define NVM_CFG1_FUNC_NPAR_ENABLED_PROTOCOL_OFFSET 17 + #define NVM_CFG1_FUNC_NPAR_ENABLED_PROTOCOL_ETHERNET 0x1 + #define NVM_CFG1_FUNC_NPAR_ENABLED_PROTOCOL_FCOE 0x2 + #define NVM_CFG1_FUNC_NPAR_ENABLED_PROTOCOL_ISCSI 0x4 + #define NVM_CFG1_FUNC_NPAR_ENABLED_PROTOCOL_RDMA 0x8 + u32 reserved[8]; /* 0x30 */ }; struct nvm_cfg1 { - struct nvm_cfg1_glob glob; /* 0x0 */ - struct nvm_cfg1_path path[MCP_GLOB_PATH_MAX]; /* 0x140 */ - struct nvm_cfg1_port port[MCP_GLOB_PORT_MAX]; /* 0x230 */ - struct nvm_cfg1_func func[MCP_GLOB_FUNC_MAX]; /* 0xB90 */ + struct nvm_cfg1_glob glob; /* 0x0 */ + struct nvm_cfg1_path path[MCP_GLOB_PATH_MAX]; /* 0x228 */ + struct nvm_cfg1_port port[MCP_GLOB_PORT_MAX]; /* 0x230 */ + struct nvm_cfg1_func func[MCP_GLOB_FUNC_MAX]; /* 0xB90 */ }; /****************************************** diff --git a/dpdk/drivers/net/qede/base/reg_addr.h b/dpdk/drivers/net/qede/base/reg_addr.h index 3b25e1a8..ad15d28a 100644 --- a/dpdk/drivers/net/qede/base/reg_addr.h +++ b/dpdk/drivers/net/qede/base/reg_addr.h @@ -6,6 +6,14 @@ * See LICENSE.qede_pmd for copyright and licensing details. */ +/* + * Copyright (c) 2016 QLogic Corporation. + * All rights reserved. + * www.qlogic.com + * + * See LICENSE.qede_pmd for copyright and licensing details. + */ + #define CDU_REG_CID_ADDR_PARAMS_CONTEXT_SIZE_SHIFT \ 0 @@ -22,7 +30,7 @@ 24 #define CDU_REG_CID_ADDR_PARAMS_NCIB ( \ - 0xff << 24) + 0xffUL << 24) /* @DPDK */ #define XSDM_REG_OPERATION_GEN \ 0xf80408UL @@ -428,11 +436,11 @@ #define NIG_REG_LLH_FUNC_FILTER_VALUE 0x501a00UL #define XMAC_REG_CTRL_TX_EN (0x1 << 0) #define XMAC_REG_CTRL_RX_EN (0x1 << 1) -#define CDU_REG_SEGMENT0_PARAMS_T0_TID_SIZE (0xff << 24) +#define CDU_REG_SEGMENT0_PARAMS_T0_TID_SIZE (0xffUL << 24) /* @DPDK */ #define CDU_REG_SEGMENT0_PARAMS_T0_TID_BLOCK_WASTE (0xff << 16) #define CDU_REG_SEGMENT0_PARAMS_T0_TID_BLOCK_WASTE_SHIFT 16 #define CDU_REG_SEGMENT1_PARAMS_T1_TID_BLOCK_WASTE (0xff << 16) -#define CDU_REG_SEGMENT1_PARAMS_T1_TID_SIZE (0xff << 24) +#define CDU_REG_SEGMENT1_PARAMS_T1_TID_SIZE (0xffUL << 24) /* @DPDK */ #define CDU_REG_SEGMENT1_PARAMS_T1_NUM_TIDS_IN_BLOCK (0xfff << 0) #define CDU_REG_SEGMENT1_PARAMS_T1_NUM_TIDS_IN_BLOCK_SHIFT 0 #define CDU_REG_SEGMENT0_PARAMS_T0_NUM_TIDS_IN_BLOCK (0xfff << 0) @@ -1105,3 +1113,112 @@ #define CDU_REG_SEGMENT1_PARAMS_T1_TID_SIZE_SHIFT 24 #define CDU_REG_SEGMENT1_PARAMS_T1_TID_BLOCK_WASTE_SHIFT 16 #define DORQ_REG_DB_DROP_DETAILS_ADDRESS 0x100a1cUL + +/* 8.10.9.0 FW */ +#define NIG_REG_VXLAN_CTRL 0x50105cUL +#define PRS_REG_SEARCH_ROCE 0x1f040cUL +#define PRS_REG_CM_HDR_GFT 0x1f11c8UL +#define PRS_REG_CM_HDR_GFT_EVENT_ID_SHIFT 0 +#define PRS_REG_CM_HDR_GFT_CM_HDR_SHIFT 8 +#define CCFC_REG_WEAK_ENABLE_VF 0x2e0704UL +#define TCFC_REG_STRONG_ENABLE_VF 0x2d070cUL +#define TCFC_REG_WEAK_ENABLE_VF 0x2d0704UL +#define PRS_REG_SEARCH_GFT 0x1f11bcUL +#define PRS_REG_LOAD_L2_FILTER 0x1f0198UL +#define PRS_REG_GFT_CAM 0x1f1100UL +#define PRS_REG_GFT_PROFILE_MASK_RAM 0x1f1000UL +#define PGLUE_B_REG_MSDM_VF_SHIFT_B 0x2aa1c4UL +#define PGLUE_B_REG_MSDM_OFFSET_MASK_B 0x2aa1c0UL +#define PRS_REG_PKT_LEN_STAT_TAGS_NOT_COUNTED_FIRST 0x1f0a0cUL +#define PRS_REG_SEARCH_FCOE 0x1f0408UL +#define PGLUE_B_REG_PGL_ADDR_E8_F0 0x2aaf98UL +#define NIG_REG_DSCP_TO_TC_MAP_ENABLE 0x5088f8UL +#define PGLUE_B_REG_PGL_ADDR_EC_F0 0x2aaf9cUL +#define PGLUE_B_REG_PGL_ADDR_F0_F0 0x2aafa0UL +#define PRS_REG_ROCE_DEST_QP_MAX_PF 0x1f0430UL +#define PGLUE_B_REG_PGL_ADDR_F4_F0 0x2aafa4UL +#define IGU_REG_WRITE_DONE_PENDING 0x180900UL +#define NIG_REG_LLH_TAGMAC_DEF_PF_VECTOR 0x50196cUL +#define PRS_REG_MSG_INFO 0x1f0a1cUL +#define BAR0_MAP_REG_XSDM_RAM 0x1e00000UL + +/* 8.18.7.0 FW */ +#define BRB_REG_INT_MASK_10 0x3401b8UL + +#define IGU_REG_PRODUCER_MEMORY 0x182000UL +#define IGU_REG_CONSUMER_MEM 0x183000UL + +#define CDU_REG_CCFC_CTX_VALID0 0x580400UL +#define CDU_REG_CCFC_CTX_VALID1 0x580404UL +#define CDU_REG_TCFC_CTX_VALID0 0x580408UL + +#define DORQ_REG_L2_EDPM_TUNNEL_NGE_IP_EN_K2_E5 0x10092cUL +#define DORQ_REG_L2_EDPM_TUNNEL_NGE_ETH_EN_K2_E5 0x100930UL +#define MISCS_REG_RESET_PL_HV_2_K2_E5 0x009150UL +#define CNIG_REG_NW_PORT_MODE_BB 0x218200UL +#define CNIG_REG_PMEG_IF_CMD_BB 0x21821cUL +#define CNIG_REG_PMEG_IF_ADDR_BB 0x218224UL +#define CNIG_REG_PMEG_IF_WRDATA_BB 0x218228UL +#define NWM_REG_MAC0_K2_E5 0x800400UL +#define CNIG_REG_NIG_PORT0_CONF_K2_E5 0x218200UL +#define CNIG_REG_NIG_PORT0_CONF_NIG_PORT_ENABLE_0_K2_E5_SHIFT 0 +#define CNIG_REG_NIG_PORT0_CONF_NIG_PORT_NWM_PORT_MAP_0_K2_E5_SHIFT 1 +#define CNIG_REG_NIG_PORT0_CONF_NIG_PORT_RATE_0_K2_E5_SHIFT 3 +#define ETH_MAC_REG_XIF_MODE_K2_E5 0x000080UL +#define ETH_MAC_REG_XIF_MODE_XGMII_K2_E5_SHIFT 0 +#define ETH_MAC_REG_FRM_LENGTH_K2_E5 0x000014UL +#define ETH_MAC_REG_FRM_LENGTH_FRM_LENGTH_K2_E5_SHIFT 0 +#define ETH_MAC_REG_TX_IPG_LENGTH_K2_E5 0x000044UL +#define ETH_MAC_REG_TX_IPG_LENGTH_TXIPG_K2_E5_SHIFT 0 +#define ETH_MAC_REG_RX_FIFO_SECTIONS_K2_E5 0x00001cUL +#define ETH_MAC_REG_RX_FIFO_SECTIONS_RX_SECTION_FULL_K2_E5_SHIFT 0 +#define ETH_MAC_REG_TX_FIFO_SECTIONS_K2_E5 0x000020UL +#define ETH_MAC_REG_TX_FIFO_SECTIONS_TX_SECTION_EMPTY_K2_E5_SHIFT 16 +#define ETH_MAC_REG_TX_FIFO_SECTIONS_TX_SECTION_FULL_K2_E5_SHIFT 0 +#define ETH_MAC_REG_COMMAND_CONFIG_K2_E5 0x000008UL +#define MISC_REG_XMAC_CORE_PORT_MODE_BB 0x008c08UL +#define MISC_REG_XMAC_PHY_PORT_MODE_BB 0x008c04UL +#define XMAC_REG_MODE_BB 0x210008UL +#define XMAC_REG_RX_MAX_SIZE_BB 0x210040UL +#define XMAC_REG_TX_CTRL_LO_BB 0x210020UL +#define XMAC_REG_CTRL_BB 0x210000UL +#define XMAC_REG_CTRL_TX_EN_BB (0x1 << 0) +#define XMAC_REG_CTRL_RX_EN_BB (0x1 << 1) +#define XMAC_REG_RX_CTRL_BB 0x210030UL +#define XMAC_REG_RX_CTRL_PROCESS_VARIABLE_PREAMBLE_BB (0x1 << 12) + +#define PGLUE_B_REG_PGL_ADDR_E8_F0_K2_E5 0x2aaf98UL +#define PGLUE_B_REG_PGL_ADDR_EC_F0_K2_E5 0x2aaf9cUL +#define PGLUE_B_REG_PGL_ADDR_F0_F0_K2_E5 0x2aafa0UL +#define PGLUE_B_REG_PGL_ADDR_F4_F0_K2_E5 0x2aafa4UL +#define PGLUE_B_REG_PGL_ADDR_88_F0_BB 0x2aa404UL +#define PGLUE_B_REG_PGL_ADDR_8C_F0_BB 0x2aa408UL +#define PGLUE_B_REG_PGL_ADDR_90_F0_BB 0x2aa40cUL +#define PGLUE_B_REG_PGL_ADDR_94_F0_BB 0x2aa410UL +#define MISCS_REG_FUNCTION_HIDE_BB_K2 0x0096f0UL +#define PCIE_REG_PRTY_MASK_K2_E5 0x0547b4UL +#define PGLUE_B_REG_VF_BAR0_SIZE_K2_E5 0x2aaeb4UL + +#define PRS_REG_OUTPUT_FORMAT_4_0_BB_K2 0x1f099cUL + +#define NIG_REG_LLH_FUNC_FILTER_VALUE_BB_K2 0x501a00UL +#define NIG_REG_LLH_FUNC_FILTER_EN_BB_K2 0x501a80UL +#define NIG_REG_LLH_FUNC_FILTER_MODE_BB_K2 0x501ac0UL +#define NIG_REG_LLH_FUNC_FILTER_PROTOCOL_TYPE_BB_K2 0x501b00UL + +#define PSWRQ2_REG_WR_MBS0 0x240400UL +#define PGLUE_B_REG_MASTER_WRITE_PAD_ENABLE 0x2aae30UL +#define DORQ_REG_PF_USAGE_CNT 0x1009c0UL +#define DORQ_REG_DPM_FORCE_ABORT 0x1009d8UL +#define DORQ_REG_PF_OVFL_STICKY 0x1009d0UL +#define DORQ_REG_INT_STS 0x100180UL + #define DORQ_REG_INT_STS_DB_DROP (0x1 << 1) + #define DORQ_REG_INT_STS_DORQ_FIFO_OVFL_ERR (0x1 << 2) + #define DORQ_REG_INT_STS_DORQ_FIFO_AFULL (0x1 << 3) +#define DORQ_REG_DB_DROP_DETAILS_REL 0x100a28UL +#define DORQ_REG_INT_STS_WR 0x100188UL +#define DORQ_REG_DB_DROP_DETAILS_REASON 0x100a20UL +#define MCP_REG_CPU_PROGRAM_COUNTER 0xe0501cUL + #define MCP_REG_CPU_STATE_SOFT_HALTED (0x1 << 10) +#define PRS_REG_SEARCH_TENANT_ID 0x1f044cUL +#define PGLUE_B_REG_VF_BAR1_SIZE 0x2aae68UL diff --git a/dpdk/drivers/net/qede/qede_eth_if.c b/dpdk/drivers/net/qede/qede_eth_if.c deleted file mode 100644 index b6f6487d..00000000 --- a/dpdk/drivers/net/qede/qede_eth_if.c +++ /dev/null @@ -1,463 +0,0 @@ -/* - * Copyright (c) 2016 QLogic Corporation. - * All rights reserved. - * www.qlogic.com - * - * See LICENSE.qede_pmd for copyright and licensing details. - */ - -#include "qede_ethdev.h" - -static int -qed_start_vport(struct ecore_dev *edev, struct qed_start_vport_params *p_params) -{ - int rc, i; - - for_each_hwfn(edev, i) { - struct ecore_hwfn *p_hwfn = &edev->hwfns[i]; - u8 tx_switching = 0; - struct ecore_sp_vport_start_params start = { 0 }; - - start.tpa_mode = p_params->gro_enable ? ECORE_TPA_MODE_GRO : - ECORE_TPA_MODE_NONE; - start.remove_inner_vlan = p_params->remove_inner_vlan; - start.tx_switching = tx_switching; - start.only_untagged = false; /* untagged only */ - start.drop_ttl0 = p_params->drop_ttl0; - start.concrete_fid = p_hwfn->hw_info.concrete_fid; - start.opaque_fid = p_hwfn->hw_info.opaque_fid; - start.concrete_fid = p_hwfn->hw_info.concrete_fid; - start.handle_ptp_pkts = p_params->handle_ptp_pkts; - start.vport_id = p_params->vport_id; - start.max_buffers_per_cqe = 16; /* TODO-is this right */ - start.mtu = p_params->mtu; - /* @DPDK - Disable FW placement */ - start.zero_placement_offset = 1; - - rc = ecore_sp_vport_start(p_hwfn, &start); - if (rc) { - DP_ERR(edev, "Failed to start VPORT\n"); - return rc; - } - - ecore_hw_start_fastpath(p_hwfn); - - DP_VERBOSE(edev, ECORE_MSG_SPQ, - "Started V-PORT %d with MTU %d\n", - p_params->vport_id, p_params->mtu); - } - - ecore_reset_vport_stats(edev); - - return 0; -} - -static int qed_stop_vport(struct ecore_dev *edev, uint8_t vport_id) -{ - int rc, i; - - for_each_hwfn(edev, i) { - struct ecore_hwfn *p_hwfn = &edev->hwfns[i]; - rc = ecore_sp_vport_stop(p_hwfn, - p_hwfn->hw_info.opaque_fid, vport_id); - - if (rc) { - DP_ERR(edev, "Failed to stop VPORT\n"); - return rc; - } - } - - return 0; -} - -static int -qed_update_vport(struct ecore_dev *edev, struct qed_update_vport_params *params) -{ - struct ecore_sp_vport_update_params sp_params; - struct ecore_rss_params sp_rss_params; - int rc, i; - - memset(&sp_params, 0, sizeof(sp_params)); - memset(&sp_rss_params, 0, sizeof(sp_rss_params)); - - /* Translate protocol params into sp params */ - sp_params.vport_id = params->vport_id; - sp_params.update_vport_active_rx_flg = params->update_vport_active_flg; - sp_params.update_vport_active_tx_flg = params->update_vport_active_flg; - sp_params.vport_active_rx_flg = params->vport_active_flg; - sp_params.vport_active_tx_flg = params->vport_active_flg; - sp_params.update_inner_vlan_removal_flg = - params->update_inner_vlan_removal_flg; - sp_params.inner_vlan_removal_flg = params->inner_vlan_removal_flg; - sp_params.update_tx_switching_flg = params->update_tx_switching_flg; - sp_params.tx_switching_flg = params->tx_switching_flg; - sp_params.accept_any_vlan = params->accept_any_vlan; - sp_params.update_accept_any_vlan_flg = - params->update_accept_any_vlan_flg; - - /* RSS - is a bit tricky, since upper-layer isn't familiar with hwfns. - * We need to re-fix the rss values per engine for CMT. - */ - - if (edev->num_hwfns > 1 && params->update_rss_flg) { - struct qed_update_vport_rss_params *rss = ¶ms->rss_params; - int k, max = 0; - - /* Find largest entry, since it's possible RSS needs to - * be disabled [in case only 1 queue per-hwfn] - */ - for (k = 0; k < ECORE_RSS_IND_TABLE_SIZE; k++) - max = (max > rss->rss_ind_table[k]) ? - max : rss->rss_ind_table[k]; - - /* Either fix RSS values or disable RSS */ - if (edev->num_hwfns < max + 1) { - int divisor = (max + edev->num_hwfns - 1) / - edev->num_hwfns; - - DP_VERBOSE(edev, ECORE_MSG_SPQ, - "CMT - fixing RSS values (modulo %02x)\n", - divisor); - - for (k = 0; k < ECORE_RSS_IND_TABLE_SIZE; k++) - rss->rss_ind_table[k] = - rss->rss_ind_table[k] % divisor; - } else { - DP_VERBOSE(edev, ECORE_MSG_SPQ, - "CMT - 1 queue per-hwfn; Disabling RSS\n"); - params->update_rss_flg = 0; - } - } - - /* Now, update the RSS configuration for actual configuration */ - if (params->update_rss_flg) { - sp_rss_params.update_rss_config = 1; - sp_rss_params.rss_enable = 1; - sp_rss_params.update_rss_capabilities = 1; - sp_rss_params.update_rss_ind_table = 1; - sp_rss_params.update_rss_key = 1; - sp_rss_params.rss_caps = ECORE_RSS_IPV4 | ECORE_RSS_IPV6 | - ECORE_RSS_IPV4_TCP | ECORE_RSS_IPV6_TCP; - sp_rss_params.rss_table_size_log = 7; /* 2^7 = 128 */ - rte_memcpy(sp_rss_params.rss_ind_table, - params->rss_params.rss_ind_table, - ECORE_RSS_IND_TABLE_SIZE * sizeof(uint16_t)); - rte_memcpy(sp_rss_params.rss_key, params->rss_params.rss_key, - ECORE_RSS_KEY_SIZE * sizeof(uint32_t)); - } - sp_params.rss_params = &sp_rss_params; - - for_each_hwfn(edev, i) { - struct ecore_hwfn *p_hwfn = &edev->hwfns[i]; - - sp_params.opaque_fid = p_hwfn->hw_info.opaque_fid; - rc = ecore_sp_vport_update(p_hwfn, &sp_params, - ECORE_SPQ_MODE_EBLOCK, NULL); - if (rc) { - DP_ERR(edev, "Failed to update VPORT\n"); - return rc; - } - - DP_VERBOSE(edev, ECORE_MSG_SPQ, - "Updated V-PORT %d: active_flag %d [update %d]\n", - params->vport_id, params->vport_active_flg, - params->update_vport_active_flg); - } - - return 0; -} - -static int -qed_start_rxq(struct ecore_dev *edev, - uint8_t rss_id, uint8_t rx_queue_id, - uint8_t vport_id, uint16_t sb, - uint8_t sb_index, uint16_t bd_max_bytes, - dma_addr_t bd_chain_phys_addr, - dma_addr_t cqe_pbl_addr, - uint16_t cqe_pbl_size, void OSAL_IOMEM * *pp_prod) -{ - struct ecore_hwfn *p_hwfn; - int rc, hwfn_index; - - hwfn_index = rss_id % edev->num_hwfns; - p_hwfn = &edev->hwfns[hwfn_index]; - - rc = ecore_sp_eth_rx_queue_start(p_hwfn, - p_hwfn->hw_info.opaque_fid, - rx_queue_id / edev->num_hwfns, - vport_id, - vport_id, - sb, - sb_index, - bd_max_bytes, - bd_chain_phys_addr, - cqe_pbl_addr, cqe_pbl_size, pp_prod); - - if (rc) { - DP_ERR(edev, "Failed to start RXQ#%d\n", rx_queue_id); - return rc; - } - - DP_VERBOSE(edev, ECORE_MSG_SPQ, - "Started RX-Q %d [rss %d] on V-PORT %d and SB %d\n", - rx_queue_id, rss_id, vport_id, sb); - - return 0; -} - -static int -qed_stop_rxq(struct ecore_dev *edev, struct qed_stop_rxq_params *params) -{ - int rc, hwfn_index; - struct ecore_hwfn *p_hwfn; - - hwfn_index = params->rss_id % edev->num_hwfns; - p_hwfn = &edev->hwfns[hwfn_index]; - - rc = ecore_sp_eth_rx_queue_stop(p_hwfn, - params->rx_queue_id / edev->num_hwfns, - params->eq_completion_only, false); - if (rc) { - DP_ERR(edev, "Failed to stop RXQ#%d\n", params->rx_queue_id); - return rc; - } - - return 0; -} - -static int -qed_start_txq(struct ecore_dev *edev, - uint8_t rss_id, uint16_t tx_queue_id, - uint8_t vport_id, uint16_t sb, - uint8_t sb_index, - dma_addr_t pbl_addr, - uint16_t pbl_size, void OSAL_IOMEM * *pp_doorbell) -{ - struct ecore_hwfn *p_hwfn; - int rc, hwfn_index; - - hwfn_index = rss_id % edev->num_hwfns; - p_hwfn = &edev->hwfns[hwfn_index]; - - rc = ecore_sp_eth_tx_queue_start(p_hwfn, - p_hwfn->hw_info.opaque_fid, - tx_queue_id / edev->num_hwfns, - vport_id, - vport_id, - sb, - sb_index, - pbl_addr, pbl_size, pp_doorbell); - - if (rc) { - DP_ERR(edev, "Failed to start TXQ#%d\n", tx_queue_id); - return rc; - } - - DP_VERBOSE(edev, ECORE_MSG_SPQ, - "Started TX-Q %d [rss %d] on V-PORT %d and SB %d\n", - tx_queue_id, rss_id, vport_id, sb); - - return 0; -} - -static int -qed_stop_txq(struct ecore_dev *edev, struct qed_stop_txq_params *params) -{ - struct ecore_hwfn *p_hwfn; - int rc, hwfn_index; - - hwfn_index = params->rss_id % edev->num_hwfns; - p_hwfn = &edev->hwfns[hwfn_index]; - - rc = ecore_sp_eth_tx_queue_stop(p_hwfn, - params->tx_queue_id / edev->num_hwfns); - if (rc) { - DP_ERR(edev, "Failed to stop TXQ#%d\n", params->tx_queue_id); - return rc; - } - - return 0; -} - -static int -qed_fp_cqe_completion(struct ecore_dev *edev, - uint8_t rss_id, struct eth_slow_path_rx_cqe *cqe) -{ - return ecore_eth_cqe_completion(&edev->hwfns[rss_id % edev->num_hwfns], - cqe); -} - -static int qed_fastpath_stop(struct ecore_dev *edev) -{ - ecore_hw_stop_fastpath(edev); - - return 0; -} - -static void -qed_get_vport_stats(struct ecore_dev *edev, struct ecore_eth_stats *stats) -{ - ecore_get_vport_stats(edev, stats); -} - -static int -qed_configure_filter_ucast(struct ecore_dev *edev, - struct qed_filter_ucast_params *params) -{ - struct ecore_filter_ucast ucast; - - if (!params->vlan_valid && !params->mac_valid) { - DP_NOTICE(edev, true, - "Tried configuring a unicast filter," - "but both MAC and VLAN are not set\n"); - return -EINVAL; - } - - memset(&ucast, 0, sizeof(ucast)); - switch (params->type) { - case QED_FILTER_XCAST_TYPE_ADD: - ucast.opcode = ECORE_FILTER_ADD; - break; - case QED_FILTER_XCAST_TYPE_DEL: - ucast.opcode = ECORE_FILTER_REMOVE; - break; - case QED_FILTER_XCAST_TYPE_REPLACE: - ucast.opcode = ECORE_FILTER_REPLACE; - break; - default: - DP_NOTICE(edev, true, "Unknown unicast filter type %d\n", - params->type); - } - - if (params->vlan_valid && params->mac_valid) { - ucast.type = ECORE_FILTER_MAC_VLAN; - ether_addr_copy((struct ether_addr *)¶ms->mac, - (struct ether_addr *)&ucast.mac); - ucast.vlan = params->vlan; - } else if (params->mac_valid) { - ucast.type = ECORE_FILTER_MAC; - ether_addr_copy((struct ether_addr *)¶ms->mac, - (struct ether_addr *)&ucast.mac); - } else { - ucast.type = ECORE_FILTER_VLAN; - ucast.vlan = params->vlan; - } - - ucast.is_rx_filter = true; - ucast.is_tx_filter = true; - - return ecore_filter_ucast_cmd(edev, &ucast, ECORE_SPQ_MODE_CB, NULL); -} - -static int -qed_configure_filter_mcast(struct ecore_dev *edev, - struct qed_filter_mcast_params *params) -{ - struct ecore_filter_mcast mcast; - int i; - - memset(&mcast, 0, sizeof(mcast)); - switch (params->type) { - case QED_FILTER_XCAST_TYPE_ADD: - mcast.opcode = ECORE_FILTER_ADD; - break; - case QED_FILTER_XCAST_TYPE_DEL: - mcast.opcode = ECORE_FILTER_REMOVE; - break; - default: - DP_NOTICE(edev, true, "Unknown multicast filter type %d\n", - params->type); - } - - mcast.num_mc_addrs = params->num; - for (i = 0; i < mcast.num_mc_addrs; i++) - ether_addr_copy((struct ether_addr *)¶ms->mac[i], - (struct ether_addr *)&mcast.mac[i]); - - return ecore_filter_mcast_cmd(edev, &mcast, ECORE_SPQ_MODE_CB, NULL); -} - -int qed_configure_filter_rx_mode(struct ecore_dev *edev, - enum qed_filter_rx_mode_type type) -{ - struct ecore_filter_accept_flags flags; - - memset(&flags, 0, sizeof(flags)); - - flags.update_rx_mode_config = 1; - flags.update_tx_mode_config = 1; - flags.rx_accept_filter = ECORE_ACCEPT_UCAST_MATCHED | - ECORE_ACCEPT_MCAST_MATCHED | - ECORE_ACCEPT_BCAST; - - flags.tx_accept_filter = ECORE_ACCEPT_UCAST_MATCHED | - ECORE_ACCEPT_MCAST_MATCHED | - ECORE_ACCEPT_BCAST; - - if (type == QED_FILTER_RX_MODE_TYPE_PROMISC) { - flags.rx_accept_filter |= ECORE_ACCEPT_UCAST_UNMATCHED; - if (IS_VF(edev)) { - flags.tx_accept_filter |= ECORE_ACCEPT_UCAST_UNMATCHED; - DP_INFO(edev, "Enabling Tx unmatched flag for VF\n"); - } - } else if (type == QED_FILTER_RX_MODE_TYPE_MULTI_PROMISC) { - flags.rx_accept_filter |= ECORE_ACCEPT_MCAST_UNMATCHED; - } else if (type == (QED_FILTER_RX_MODE_TYPE_MULTI_PROMISC | - QED_FILTER_RX_MODE_TYPE_PROMISC)) { - flags.rx_accept_filter |= ECORE_ACCEPT_UCAST_UNMATCHED | - ECORE_ACCEPT_MCAST_UNMATCHED; - } - - return ecore_filter_accept_cmd(edev, 0, flags, false, false, - ECORE_SPQ_MODE_CB, NULL); -} - -static int -qed_configure_filter(struct ecore_dev *edev, struct qed_filter_params *params) -{ - switch (params->type) { - case QED_FILTER_TYPE_UCAST: - return qed_configure_filter_ucast(edev, ¶ms->filter.ucast); - case QED_FILTER_TYPE_MCAST: - return qed_configure_filter_mcast(edev, ¶ms->filter.mcast); - case QED_FILTER_TYPE_RX_MODE: - return qed_configure_filter_rx_mode(edev, - params->filter. - accept_flags); - default: - DP_NOTICE(edev, true, "Unknown filter type %d\n", - (int)params->type); - return -EINVAL; - } -} - -static const struct qed_eth_ops qed_eth_ops_pass = { - INIT_STRUCT_FIELD(common, &qed_common_ops_pass), - INIT_STRUCT_FIELD(fill_dev_info, &qed_fill_eth_dev_info), - INIT_STRUCT_FIELD(vport_start, &qed_start_vport), - INIT_STRUCT_FIELD(vport_stop, &qed_stop_vport), - INIT_STRUCT_FIELD(vport_update, &qed_update_vport), - INIT_STRUCT_FIELD(q_rx_start, &qed_start_rxq), - INIT_STRUCT_FIELD(q_tx_start, &qed_start_txq), - INIT_STRUCT_FIELD(q_rx_stop, &qed_stop_rxq), - INIT_STRUCT_FIELD(q_tx_stop, &qed_stop_txq), - INIT_STRUCT_FIELD(eth_cqe_completion, &qed_fp_cqe_completion), - INIT_STRUCT_FIELD(fastpath_stop, &qed_fastpath_stop), - INIT_STRUCT_FIELD(get_vport_stats, &qed_get_vport_stats), - INIT_STRUCT_FIELD(filter_config, &qed_configure_filter), -}; - -uint32_t qed_get_protocol_version(enum qed_protocol protocol) -{ - switch (protocol) { - case QED_PROTOCOL_ETH: - return QED_ETH_INTERFACE_VERSION; - default: - return 0; - } -} - -const struct qed_eth_ops *qed_get_eth_ops(void) -{ - return &qed_eth_ops_pass; -} diff --git a/dpdk/drivers/net/qede/qede_eth_if.h b/dpdk/drivers/net/qede/qede_eth_if.h deleted file mode 100644 index 26968eb0..00000000 --- a/dpdk/drivers/net/qede/qede_eth_if.h +++ /dev/null @@ -1,177 +0,0 @@ -/* - * Copyright (c) 2016 QLogic Corporation. - * All rights reserved. - * www.qlogic.com - * - * See LICENSE.qede_pmd for copyright and licensing details. - */ - -#ifndef _QEDE_ETH_IF_H -#define _QEDE_ETH_IF_H - -#include "qede_if.h" - -/*forward decl */ -struct eth_slow_path_rx_cqe; - -#define INIT_STRUCT_FIELD(field, value) .field = value - -#define QED_ETH_INTERFACE_VERSION 609 - -#define QEDE_MAX_MCAST_FILTERS 64 - -enum qed_filter_rx_mode_type { - QED_FILTER_RX_MODE_TYPE_REGULAR, - QED_FILTER_RX_MODE_TYPE_MULTI_PROMISC, - QED_FILTER_RX_MODE_TYPE_PROMISC, -}; - -enum qed_filter_xcast_params_type { - QED_FILTER_XCAST_TYPE_ADD, - QED_FILTER_XCAST_TYPE_DEL, - QED_FILTER_XCAST_TYPE_REPLACE, -}; - -enum qed_filter_type { - QED_FILTER_TYPE_UCAST, - QED_FILTER_TYPE_MCAST, - QED_FILTER_TYPE_RX_MODE, - QED_MAX_FILTER_TYPES, -}; - -struct qed_dev_eth_info { - struct qed_dev_info common; - - uint8_t num_queues; - uint8_t num_tc; - - struct ether_addr port_mac; - uint8_t num_vlan_filters; - uint32_t num_mac_addrs; -}; - -struct qed_update_vport_rss_params { - uint16_t rss_ind_table[128]; - uint32_t rss_key[10]; - u8 rss_caps; -}; - -struct qed_stop_rxq_params { - uint8_t rss_id; - uint8_t rx_queue_id; - uint8_t vport_id; - bool eq_completion_only; -}; - -struct qed_update_vport_params { - uint8_t vport_id; - uint8_t update_vport_active_flg; - uint8_t vport_active_flg; - uint8_t update_inner_vlan_removal_flg; - uint8_t inner_vlan_removal_flg; - uint8_t update_tx_switching_flg; - uint8_t tx_switching_flg; - uint8_t update_accept_any_vlan_flg; - uint8_t accept_any_vlan; - uint8_t update_rss_flg; - struct qed_update_vport_rss_params rss_params; -}; - -struct qed_start_vport_params { - bool remove_inner_vlan; - bool handle_ptp_pkts; - bool gro_enable; - bool drop_ttl0; - uint8_t vport_id; - uint16_t mtu; - bool clear_stats; -}; - -struct qed_stop_txq_params { - uint8_t rss_id; - uint8_t tx_queue_id; -}; - -struct qed_filter_ucast_params { - enum qed_filter_xcast_params_type type; - uint8_t vlan_valid; - uint16_t vlan; - uint8_t mac_valid; - unsigned char mac[ETHER_ADDR_LEN]; -}; - -struct qed_filter_mcast_params { - enum qed_filter_xcast_params_type type; - uint8_t num; - unsigned char mac[QEDE_MAX_MCAST_FILTERS][ETHER_ADDR_LEN]; -}; - -union qed_filter_type_params { - enum qed_filter_rx_mode_type accept_flags; - struct qed_filter_ucast_params ucast; - struct qed_filter_mcast_params mcast; -}; - -struct qed_filter_params { - enum qed_filter_type type; - union qed_filter_type_params filter; -}; - -struct qed_eth_ops { - const struct qed_common_ops *common; - - int (*fill_dev_info)(struct ecore_dev *edev, - struct qed_dev_eth_info *info); - - int (*vport_start)(struct ecore_dev *edev, - struct qed_start_vport_params *params); - - int (*vport_stop)(struct ecore_dev *edev, uint8_t vport_id); - - int (*vport_update)(struct ecore_dev *edev, - struct qed_update_vport_params *params); - - int (*q_rx_start)(struct ecore_dev *cdev, - uint8_t rss_id, uint8_t rx_queue_id, - uint8_t vport_id, uint16_t sb, - uint8_t sb_index, uint16_t bd_max_bytes, - dma_addr_t bd_chain_phys_addr, - dma_addr_t cqe_pbl_addr, - uint16_t cqe_pbl_size, void OSAL_IOMEM * *pp_prod); - - int (*q_rx_stop)(struct ecore_dev *edev, - struct qed_stop_rxq_params *params); - - int (*q_tx_start)(struct ecore_dev *edev, - uint8_t rss_id, uint16_t tx_queue_id, - uint8_t vport_id, uint16_t sb, - uint8_t sb_index, - dma_addr_t pbl_addr, - uint16_t pbl_size, void OSAL_IOMEM * *pp_doorbell); - - int (*q_tx_stop)(struct ecore_dev *edev, - struct qed_stop_txq_params *params); - - int (*eth_cqe_completion)(struct ecore_dev *edev, - uint8_t rss_id, - struct eth_slow_path_rx_cqe *cqe); - - int (*fastpath_stop)(struct ecore_dev *edev); - - void (*get_vport_stats)(struct ecore_dev *edev, - struct ecore_eth_stats *stats); - - int (*filter_config)(struct ecore_dev *edev, - struct qed_filter_params *params); -}; - -/* externs */ - -extern const struct qed_common_ops qed_common_ops_pass; - -const struct qed_eth_ops *qed_get_eth_ops(); - -int qed_configure_filter_rx_mode(struct ecore_dev *edev, - enum qed_filter_rx_mode_type type); - -#endif /* _QEDE_ETH_IF_H */ diff --git a/dpdk/drivers/net/qede/qede_ethdev.c b/dpdk/drivers/net/qede/qede_ethdev.c index 82e44b8f..73764e9a 100644 --- a/dpdk/drivers/net/qede/qede_ethdev.c +++ b/dpdk/drivers/net/qede/qede_ethdev.c @@ -8,155 +8,326 @@ #include "qede_ethdev.h" #include +#include +#include /* Globals */ static const struct qed_eth_ops *qed_ops; -static const char *drivername = "qede pmd"; static int64_t timer_period = 1; +/* VXLAN tunnel classification mapping */ +const struct _qede_vxlan_tunn_types { + uint16_t rte_filter_type; + enum ecore_filter_ucast_type qede_type; + enum ecore_tunn_clss qede_tunn_clss; + const char *string; +} qede_tunn_types[] = { + { + ETH_TUNNEL_FILTER_OMAC, + ECORE_FILTER_MAC, + ECORE_TUNN_CLSS_MAC_VLAN, + "outer-mac" + }, + { + ETH_TUNNEL_FILTER_TENID, + ECORE_FILTER_VNI, + ECORE_TUNN_CLSS_MAC_VNI, + "vni" + }, + { + ETH_TUNNEL_FILTER_IMAC, + ECORE_FILTER_INNER_MAC, + ECORE_TUNN_CLSS_INNER_MAC_VLAN, + "inner-mac" + }, + { + ETH_TUNNEL_FILTER_IVLAN, + ECORE_FILTER_INNER_VLAN, + ECORE_TUNN_CLSS_INNER_MAC_VLAN, + "inner-vlan" + }, + { + ETH_TUNNEL_FILTER_OMAC | ETH_TUNNEL_FILTER_TENID, + ECORE_FILTER_MAC_VNI_PAIR, + ECORE_TUNN_CLSS_MAC_VNI, + "outer-mac and vni" + }, + { + ETH_TUNNEL_FILTER_OMAC | ETH_TUNNEL_FILTER_IMAC, + ECORE_FILTER_UNUSED, + MAX_ECORE_TUNN_CLSS, + "outer-mac and inner-mac" + }, + { + ETH_TUNNEL_FILTER_OMAC | ETH_TUNNEL_FILTER_IVLAN, + ECORE_FILTER_UNUSED, + MAX_ECORE_TUNN_CLSS, + "outer-mac and inner-vlan" + }, + { + ETH_TUNNEL_FILTER_TENID | ETH_TUNNEL_FILTER_IMAC, + ECORE_FILTER_INNER_MAC_VNI_PAIR, + ECORE_TUNN_CLSS_INNER_MAC_VNI, + "vni and inner-mac", + }, + { + ETH_TUNNEL_FILTER_TENID | ETH_TUNNEL_FILTER_IVLAN, + ECORE_FILTER_UNUSED, + MAX_ECORE_TUNN_CLSS, + "vni and inner-vlan", + }, + { + ETH_TUNNEL_FILTER_IMAC | ETH_TUNNEL_FILTER_IVLAN, + ECORE_FILTER_INNER_PAIR, + ECORE_TUNN_CLSS_INNER_MAC_VLAN, + "inner-mac and inner-vlan", + }, + { + ETH_TUNNEL_FILTER_OIP, + ECORE_FILTER_UNUSED, + MAX_ECORE_TUNN_CLSS, + "outer-IP" + }, + { + ETH_TUNNEL_FILTER_IIP, + ECORE_FILTER_UNUSED, + MAX_ECORE_TUNN_CLSS, + "inner-IP" + }, + { + RTE_TUNNEL_FILTER_IMAC_IVLAN, + ECORE_FILTER_UNUSED, + MAX_ECORE_TUNN_CLSS, + "IMAC_IVLAN" + }, + { + RTE_TUNNEL_FILTER_IMAC_IVLAN_TENID, + ECORE_FILTER_UNUSED, + MAX_ECORE_TUNN_CLSS, + "IMAC_IVLAN_TENID" + }, + { + RTE_TUNNEL_FILTER_IMAC_TENID, + ECORE_FILTER_UNUSED, + MAX_ECORE_TUNN_CLSS, + "IMAC_TENID" + }, + { + RTE_TUNNEL_FILTER_OMAC_TENID_IMAC, + ECORE_FILTER_UNUSED, + MAX_ECORE_TUNN_CLSS, + "OMAC_TENID_IMAC" + }, +}; + struct rte_qede_xstats_name_off { char name[RTE_ETH_XSTATS_NAME_SIZE]; uint64_t offset; }; static const struct rte_qede_xstats_name_off qede_xstats_strings[] = { - {"rx_unicast_bytes", offsetof(struct ecore_eth_stats, rx_ucast_bytes)}, + {"rx_unicast_bytes", + offsetof(struct ecore_eth_stats_common, rx_ucast_bytes)}, {"rx_multicast_bytes", - offsetof(struct ecore_eth_stats, rx_mcast_bytes)}, + offsetof(struct ecore_eth_stats_common, rx_mcast_bytes)}, {"rx_broadcast_bytes", - offsetof(struct ecore_eth_stats, rx_bcast_bytes)}, - {"rx_unicast_packets", offsetof(struct ecore_eth_stats, rx_ucast_pkts)}, + offsetof(struct ecore_eth_stats_common, rx_bcast_bytes)}, + {"rx_unicast_packets", + offsetof(struct ecore_eth_stats_common, rx_ucast_pkts)}, {"rx_multicast_packets", - offsetof(struct ecore_eth_stats, rx_mcast_pkts)}, + offsetof(struct ecore_eth_stats_common, rx_mcast_pkts)}, {"rx_broadcast_packets", - offsetof(struct ecore_eth_stats, rx_bcast_pkts)}, + offsetof(struct ecore_eth_stats_common, rx_bcast_pkts)}, - {"tx_unicast_bytes", offsetof(struct ecore_eth_stats, tx_ucast_bytes)}, + {"tx_unicast_bytes", + offsetof(struct ecore_eth_stats_common, tx_ucast_bytes)}, {"tx_multicast_bytes", - offsetof(struct ecore_eth_stats, tx_mcast_bytes)}, + offsetof(struct ecore_eth_stats_common, tx_mcast_bytes)}, {"tx_broadcast_bytes", - offsetof(struct ecore_eth_stats, tx_bcast_bytes)}, - {"tx_unicast_packets", offsetof(struct ecore_eth_stats, tx_ucast_pkts)}, + offsetof(struct ecore_eth_stats_common, tx_bcast_bytes)}, + {"tx_unicast_packets", + offsetof(struct ecore_eth_stats_common, tx_ucast_pkts)}, {"tx_multicast_packets", - offsetof(struct ecore_eth_stats, tx_mcast_pkts)}, + offsetof(struct ecore_eth_stats_common, tx_mcast_pkts)}, {"tx_broadcast_packets", - offsetof(struct ecore_eth_stats, tx_bcast_pkts)}, + offsetof(struct ecore_eth_stats_common, tx_bcast_pkts)}, {"rx_64_byte_packets", - offsetof(struct ecore_eth_stats, rx_64_byte_packets)}, + offsetof(struct ecore_eth_stats_common, rx_64_byte_packets)}, {"rx_65_to_127_byte_packets", - offsetof(struct ecore_eth_stats, rx_65_to_127_byte_packets)}, + offsetof(struct ecore_eth_stats_common, + rx_65_to_127_byte_packets)}, {"rx_128_to_255_byte_packets", - offsetof(struct ecore_eth_stats, rx_128_to_255_byte_packets)}, + offsetof(struct ecore_eth_stats_common, + rx_128_to_255_byte_packets)}, {"rx_256_to_511_byte_packets", - offsetof(struct ecore_eth_stats, rx_256_to_511_byte_packets)}, + offsetof(struct ecore_eth_stats_common, + rx_256_to_511_byte_packets)}, {"rx_512_to_1023_byte_packets", - offsetof(struct ecore_eth_stats, rx_512_to_1023_byte_packets)}, + offsetof(struct ecore_eth_stats_common, + rx_512_to_1023_byte_packets)}, {"rx_1024_to_1518_byte_packets", - offsetof(struct ecore_eth_stats, rx_1024_to_1518_byte_packets)}, - {"rx_1519_to_1522_byte_packets", - offsetof(struct ecore_eth_stats, rx_1519_to_1522_byte_packets)}, - {"rx_1519_to_2047_byte_packets", - offsetof(struct ecore_eth_stats, rx_1519_to_2047_byte_packets)}, - {"rx_2048_to_4095_byte_packets", - offsetof(struct ecore_eth_stats, rx_2048_to_4095_byte_packets)}, - {"rx_4096_to_9216_byte_packets", - offsetof(struct ecore_eth_stats, rx_4096_to_9216_byte_packets)}, - {"rx_9217_to_16383_byte_packets", - offsetof(struct ecore_eth_stats, - rx_9217_to_16383_byte_packets)}, + offsetof(struct ecore_eth_stats_common, + rx_1024_to_1518_byte_packets)}, {"tx_64_byte_packets", - offsetof(struct ecore_eth_stats, tx_64_byte_packets)}, + offsetof(struct ecore_eth_stats_common, tx_64_byte_packets)}, {"tx_65_to_127_byte_packets", - offsetof(struct ecore_eth_stats, tx_65_to_127_byte_packets)}, + offsetof(struct ecore_eth_stats_common, + tx_65_to_127_byte_packets)}, {"tx_128_to_255_byte_packets", - offsetof(struct ecore_eth_stats, tx_128_to_255_byte_packets)}, + offsetof(struct ecore_eth_stats_common, + tx_128_to_255_byte_packets)}, {"tx_256_to_511_byte_packets", - offsetof(struct ecore_eth_stats, tx_256_to_511_byte_packets)}, + offsetof(struct ecore_eth_stats_common, + tx_256_to_511_byte_packets)}, {"tx_512_to_1023_byte_packets", - offsetof(struct ecore_eth_stats, tx_512_to_1023_byte_packets)}, + offsetof(struct ecore_eth_stats_common, + tx_512_to_1023_byte_packets)}, {"tx_1024_to_1518_byte_packets", - offsetof(struct ecore_eth_stats, tx_1024_to_1518_byte_packets)}, - {"trx_1519_to_1522_byte_packets", - offsetof(struct ecore_eth_stats, tx_1519_to_2047_byte_packets)}, - {"tx_2048_to_4095_byte_packets", - offsetof(struct ecore_eth_stats, tx_2048_to_4095_byte_packets)}, - {"tx_4096_to_9216_byte_packets", - offsetof(struct ecore_eth_stats, tx_4096_to_9216_byte_packets)}, - {"tx_9217_to_16383_byte_packets", - offsetof(struct ecore_eth_stats, - tx_9217_to_16383_byte_packets)}, + offsetof(struct ecore_eth_stats_common, + tx_1024_to_1518_byte_packets)}, {"rx_mac_crtl_frames", - offsetof(struct ecore_eth_stats, rx_mac_crtl_frames)}, + offsetof(struct ecore_eth_stats_common, rx_mac_crtl_frames)}, {"tx_mac_control_frames", - offsetof(struct ecore_eth_stats, tx_mac_ctrl_frames)}, - {"rx_pause_frames", offsetof(struct ecore_eth_stats, rx_pause_frames)}, - {"tx_pause_frames", offsetof(struct ecore_eth_stats, tx_pause_frames)}, + offsetof(struct ecore_eth_stats_common, tx_mac_ctrl_frames)}, + {"rx_pause_frames", + offsetof(struct ecore_eth_stats_common, rx_pause_frames)}, + {"tx_pause_frames", + offsetof(struct ecore_eth_stats_common, tx_pause_frames)}, {"rx_priority_flow_control_frames", - offsetof(struct ecore_eth_stats, rx_pfc_frames)}, + offsetof(struct ecore_eth_stats_common, rx_pfc_frames)}, {"tx_priority_flow_control_frames", - offsetof(struct ecore_eth_stats, tx_pfc_frames)}, + offsetof(struct ecore_eth_stats_common, tx_pfc_frames)}, - {"rx_crc_errors", offsetof(struct ecore_eth_stats, rx_crc_errors)}, - {"rx_align_errors", offsetof(struct ecore_eth_stats, rx_align_errors)}, + {"rx_crc_errors", + offsetof(struct ecore_eth_stats_common, rx_crc_errors)}, + {"rx_align_errors", + offsetof(struct ecore_eth_stats_common, rx_align_errors)}, {"rx_carrier_errors", - offsetof(struct ecore_eth_stats, rx_carrier_errors)}, + offsetof(struct ecore_eth_stats_common, rx_carrier_errors)}, {"rx_oversize_packet_errors", - offsetof(struct ecore_eth_stats, rx_oversize_packets)}, - {"rx_jabber_errors", offsetof(struct ecore_eth_stats, rx_jabbers)}, + offsetof(struct ecore_eth_stats_common, rx_oversize_packets)}, + {"rx_jabber_errors", + offsetof(struct ecore_eth_stats_common, rx_jabbers)}, {"rx_undersize_packet_errors", - offsetof(struct ecore_eth_stats, rx_undersize_packets)}, - {"rx_fragments", offsetof(struct ecore_eth_stats, rx_fragments)}, + offsetof(struct ecore_eth_stats_common, rx_undersize_packets)}, + {"rx_fragments", offsetof(struct ecore_eth_stats_common, rx_fragments)}, {"rx_host_buffer_not_available", - offsetof(struct ecore_eth_stats, no_buff_discards)}, + offsetof(struct ecore_eth_stats_common, no_buff_discards)}, /* Number of packets discarded because they are bigger than MTU */ {"rx_packet_too_big_discards", - offsetof(struct ecore_eth_stats, packet_too_big_discard)}, + offsetof(struct ecore_eth_stats_common, + packet_too_big_discard)}, {"rx_ttl_zero_discards", - offsetof(struct ecore_eth_stats, ttl0_discard)}, + offsetof(struct ecore_eth_stats_common, ttl0_discard)}, {"rx_multi_function_tag_filter_discards", - offsetof(struct ecore_eth_stats, mftag_filter_discards)}, + offsetof(struct ecore_eth_stats_common, mftag_filter_discards)}, {"rx_mac_filter_discards", - offsetof(struct ecore_eth_stats, mac_filter_discards)}, + offsetof(struct ecore_eth_stats_common, mac_filter_discards)}, {"rx_hw_buffer_truncates", - offsetof(struct ecore_eth_stats, brb_truncates)}, + offsetof(struct ecore_eth_stats_common, brb_truncates)}, {"rx_hw_buffer_discards", - offsetof(struct ecore_eth_stats, brb_discards)}, - {"tx_lpi_entry_count", - offsetof(struct ecore_eth_stats, tx_lpi_entry_count)}, - {"tx_total_collisions", - offsetof(struct ecore_eth_stats, tx_total_collisions)}, + offsetof(struct ecore_eth_stats_common, brb_discards)}, {"tx_error_drop_packets", - offsetof(struct ecore_eth_stats, tx_err_drop_pkts)}, + offsetof(struct ecore_eth_stats_common, tx_err_drop_pkts)}, - {"rx_mac_bytes", offsetof(struct ecore_eth_stats, rx_mac_bytes)}, + {"rx_mac_bytes", offsetof(struct ecore_eth_stats_common, rx_mac_bytes)}, {"rx_mac_unicast_packets", - offsetof(struct ecore_eth_stats, rx_mac_uc_packets)}, + offsetof(struct ecore_eth_stats_common, rx_mac_uc_packets)}, {"rx_mac_multicast_packets", - offsetof(struct ecore_eth_stats, rx_mac_mc_packets)}, + offsetof(struct ecore_eth_stats_common, rx_mac_mc_packets)}, {"rx_mac_broadcast_packets", - offsetof(struct ecore_eth_stats, rx_mac_bc_packets)}, + offsetof(struct ecore_eth_stats_common, rx_mac_bc_packets)}, {"rx_mac_frames_ok", - offsetof(struct ecore_eth_stats, rx_mac_frames_ok)}, - {"tx_mac_bytes", offsetof(struct ecore_eth_stats, tx_mac_bytes)}, + offsetof(struct ecore_eth_stats_common, rx_mac_frames_ok)}, + {"tx_mac_bytes", offsetof(struct ecore_eth_stats_common, tx_mac_bytes)}, {"tx_mac_unicast_packets", - offsetof(struct ecore_eth_stats, tx_mac_uc_packets)}, + offsetof(struct ecore_eth_stats_common, tx_mac_uc_packets)}, {"tx_mac_multicast_packets", - offsetof(struct ecore_eth_stats, tx_mac_mc_packets)}, + offsetof(struct ecore_eth_stats_common, tx_mac_mc_packets)}, {"tx_mac_broadcast_packets", - offsetof(struct ecore_eth_stats, tx_mac_bc_packets)}, + offsetof(struct ecore_eth_stats_common, tx_mac_bc_packets)}, {"lro_coalesced_packets", - offsetof(struct ecore_eth_stats, tpa_coalesced_pkts)}, + offsetof(struct ecore_eth_stats_common, tpa_coalesced_pkts)}, {"lro_coalesced_events", - offsetof(struct ecore_eth_stats, tpa_coalesced_events)}, + offsetof(struct ecore_eth_stats_common, tpa_coalesced_events)}, {"lro_aborts_num", - offsetof(struct ecore_eth_stats, tpa_aborts_num)}, + offsetof(struct ecore_eth_stats_common, tpa_aborts_num)}, {"lro_not_coalesced_packets", - offsetof(struct ecore_eth_stats, tpa_not_coalesced_pkts)}, + offsetof(struct ecore_eth_stats_common, + tpa_not_coalesced_pkts)}, {"lro_coalesced_bytes", - offsetof(struct ecore_eth_stats, tpa_coalesced_bytes)}, + offsetof(struct ecore_eth_stats_common, + tpa_coalesced_bytes)}, +}; + +static const struct rte_qede_xstats_name_off qede_bb_xstats_strings[] = { + {"rx_1519_to_1522_byte_packets", + offsetof(struct ecore_eth_stats, bb) + + offsetof(struct ecore_eth_stats_bb, + rx_1519_to_1522_byte_packets)}, + {"rx_1519_to_2047_byte_packets", + offsetof(struct ecore_eth_stats, bb) + + offsetof(struct ecore_eth_stats_bb, + rx_1519_to_2047_byte_packets)}, + {"rx_2048_to_4095_byte_packets", + offsetof(struct ecore_eth_stats, bb) + + offsetof(struct ecore_eth_stats_bb, + rx_2048_to_4095_byte_packets)}, + {"rx_4096_to_9216_byte_packets", + offsetof(struct ecore_eth_stats, bb) + + offsetof(struct ecore_eth_stats_bb, + rx_4096_to_9216_byte_packets)}, + {"rx_9217_to_16383_byte_packets", + offsetof(struct ecore_eth_stats, bb) + + offsetof(struct ecore_eth_stats_bb, + rx_9217_to_16383_byte_packets)}, + + {"tx_1519_to_2047_byte_packets", + offsetof(struct ecore_eth_stats, bb) + + offsetof(struct ecore_eth_stats_bb, + tx_1519_to_2047_byte_packets)}, + {"tx_2048_to_4095_byte_packets", + offsetof(struct ecore_eth_stats, bb) + + offsetof(struct ecore_eth_stats_bb, + tx_2048_to_4095_byte_packets)}, + {"tx_4096_to_9216_byte_packets", + offsetof(struct ecore_eth_stats, bb) + + offsetof(struct ecore_eth_stats_bb, + tx_4096_to_9216_byte_packets)}, + {"tx_9217_to_16383_byte_packets", + offsetof(struct ecore_eth_stats, bb) + + offsetof(struct ecore_eth_stats_bb, + tx_9217_to_16383_byte_packets)}, + + {"tx_lpi_entry_count", + offsetof(struct ecore_eth_stats, bb) + + offsetof(struct ecore_eth_stats_bb, tx_lpi_entry_count)}, + {"tx_total_collisions", + offsetof(struct ecore_eth_stats, bb) + + offsetof(struct ecore_eth_stats_bb, tx_total_collisions)}, +}; + +static const struct rte_qede_xstats_name_off qede_ah_xstats_strings[] = { + {"rx_1519_to_max_byte_packets", + offsetof(struct ecore_eth_stats, ah) + + offsetof(struct ecore_eth_stats_ah, + rx_1519_to_max_byte_packets)}, + {"tx_1519_to_max_byte_packets", + offsetof(struct ecore_eth_stats, ah) + + offsetof(struct ecore_eth_stats_ah, + tx_1519_to_max_byte_packets)}, +}; + +static const struct rte_qede_xstats_name_off qede_rxq_xstats_strings[] = { + {"rx_q_segments", + offsetof(struct qede_rx_queue, rx_segs)}, + {"rx_q_hw_errors", + offsetof(struct qede_rx_queue, rx_hw_errors)}, + {"rx_q_allocation_errors", + offsetof(struct qede_rx_queue, rx_alloc_errors)} }; static void qede_interrupt_action(struct ecore_hwfn *p_hwfn) @@ -165,14 +336,14 @@ static void qede_interrupt_action(struct ecore_hwfn *p_hwfn) } static void -qede_interrupt_handler(__rte_unused struct rte_intr_handle *handle, void *param) +qede_interrupt_handler(void *param) { struct rte_eth_dev *eth_dev = (struct rte_eth_dev *)param; struct qede_dev *qdev = eth_dev->data->dev_private; struct ecore_dev *edev = &qdev->edev; qede_interrupt_action(ECORE_LEADING_HWFN(edev)); - if (rte_intr_enable(ð_dev->pci_dev->intr_handle)) + if (rte_intr_enable(eth_dev->intr_handle)) DP_ERR(edev, "rte_intr_enable failed\n"); } @@ -180,83 +351,546 @@ static void qede_alloc_etherdev(struct qede_dev *qdev, struct qed_dev_eth_info *info) { rte_memcpy(&qdev->dev_info, info, sizeof(*info)); - qdev->num_tc = qdev->dev_info.num_tc; qdev->ops = qed_ops; } +#ifdef RTE_LIBRTE_QEDE_DEBUG_INFO static void qede_print_adapter_info(struct qede_dev *qdev) { struct ecore_dev *edev = &qdev->edev; struct qed_dev_info *info = &qdev->dev_info.common; - static char ver_str[QED_DRV_VER_STR_SIZE]; + static char drv_ver[QEDE_PMD_DRV_VER_STR_SIZE]; + static char ver_str[QEDE_PMD_DRV_VER_STR_SIZE]; DP_INFO(edev, "*********************************\n"); - DP_INFO(edev, " Chip details : %s%d\n", - ECORE_IS_BB(edev) ? "BB" : "AH", - CHIP_REV_IS_A0(edev) ? 0 : 1); - - sprintf(ver_str, "%s %s_%d.%d.%d.%d", QEDE_PMD_VER_PREFIX, - edev->ver_str, QEDE_PMD_VERSION_MAJOR, QEDE_PMD_VERSION_MINOR, - QEDE_PMD_VERSION_REVISION, QEDE_PMD_VERSION_PATCH); - strcpy(qdev->drv_ver, ver_str); - DP_INFO(edev, " Driver version : %s\n", ver_str); - - sprintf(ver_str, "%d.%d.%d.%d", info->fw_major, info->fw_minor, - info->fw_rev, info->fw_eng); + DP_INFO(edev, " DPDK version:%s\n", rte_version()); + DP_INFO(edev, " Chip details : %s %c%d\n", + ECORE_IS_BB(edev) ? "BB" : "AH", + 'A' + edev->chip_rev, + (int)edev->chip_metal); + snprintf(ver_str, QEDE_PMD_DRV_VER_STR_SIZE, "%d.%d.%d.%d", + info->fw_major, info->fw_minor, info->fw_rev, info->fw_eng); + snprintf(drv_ver, QEDE_PMD_DRV_VER_STR_SIZE, "%s_%s", + ver_str, QEDE_PMD_VERSION); + DP_INFO(edev, " Driver version : %s\n", drv_ver); DP_INFO(edev, " Firmware version : %s\n", ver_str); - sprintf(ver_str, "%d.%d.%d.%d", + snprintf(ver_str, MCP_DRV_VER_STR_SIZE, + "%d.%d.%d.%d", (info->mfw_rev >> 24) & 0xff, (info->mfw_rev >> 16) & 0xff, (info->mfw_rev >> 8) & 0xff, (info->mfw_rev) & 0xff); - DP_INFO(edev, " Management firmware version : %s\n", ver_str); - + DP_INFO(edev, " Management Firmware version : %s\n", ver_str); DP_INFO(edev, " Firmware file : %s\n", fw_file); - DP_INFO(edev, "*********************************\n"); } +#endif + +static void qede_reset_queue_stats(struct qede_dev *qdev, bool xstats) +{ +#ifdef RTE_LIBRTE_QEDE_DEBUG_DRIVER + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); +#endif + unsigned int i = 0, j = 0, qid; + unsigned int rxq_stat_cntrs, txq_stat_cntrs; + struct qede_tx_queue *txq; + + DP_VERBOSE(edev, ECORE_MSG_DEBUG, "Clearing queue stats\n"); + + rxq_stat_cntrs = RTE_MIN(QEDE_RSS_COUNT(qdev), + RTE_ETHDEV_QUEUE_STAT_CNTRS); + txq_stat_cntrs = RTE_MIN(QEDE_TSS_COUNT(qdev), + RTE_ETHDEV_QUEUE_STAT_CNTRS); + + for_each_rss(qid) { + OSAL_MEMSET(((char *)(qdev->fp_array[qid].rxq)) + + offsetof(struct qede_rx_queue, rcv_pkts), 0, + sizeof(uint64_t)); + OSAL_MEMSET(((char *)(qdev->fp_array[qid].rxq)) + + offsetof(struct qede_rx_queue, rx_hw_errors), 0, + sizeof(uint64_t)); + OSAL_MEMSET(((char *)(qdev->fp_array[qid].rxq)) + + offsetof(struct qede_rx_queue, rx_alloc_errors), 0, + sizeof(uint64_t)); + + if (xstats) + for (j = 0; j < RTE_DIM(qede_rxq_xstats_strings); j++) + OSAL_MEMSET((((char *) + (qdev->fp_array[qid].rxq)) + + qede_rxq_xstats_strings[j].offset), + 0, + sizeof(uint64_t)); + + i++; + if (i == rxq_stat_cntrs) + break; + } + + i = 0; + + for_each_tss(qid) { + txq = qdev->fp_array[qid].txq; + + OSAL_MEMSET((uint64_t *)(uintptr_t) + (((uint64_t)(uintptr_t)(txq)) + + offsetof(struct qede_tx_queue, xmit_pkts)), 0, + sizeof(uint64_t)); + + i++; + if (i == txq_stat_cntrs) + break; + } +} static int -qede_set_ucast_rx_mac(struct qede_dev *qdev, - enum qed_filter_xcast_params_type opcode, - uint8_t mac[ETHER_ADDR_LEN]) +qede_start_vport(struct qede_dev *qdev, uint16_t mtu) { - struct ecore_dev *edev = &qdev->edev; - struct qed_filter_params filter_cmd; + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); + struct ecore_sp_vport_start_params params; + struct ecore_hwfn *p_hwfn; + int rc; + int i; - memset(&filter_cmd, 0, sizeof(filter_cmd)); - filter_cmd.type = QED_FILTER_TYPE_UCAST; - filter_cmd.filter.ucast.type = opcode; - filter_cmd.filter.ucast.mac_valid = 1; - rte_memcpy(&filter_cmd.filter.ucast.mac[0], &mac[0], ETHER_ADDR_LEN); - return qdev->ops->filter_config(edev, &filter_cmd); + memset(¶ms, 0, sizeof(params)); + params.vport_id = 0; + params.mtu = mtu; + /* @DPDK - Disable FW placement */ + params.zero_placement_offset = 1; + for_each_hwfn(edev, i) { + p_hwfn = &edev->hwfns[i]; + params.concrete_fid = p_hwfn->hw_info.concrete_fid; + params.opaque_fid = p_hwfn->hw_info.opaque_fid; + rc = ecore_sp_vport_start(p_hwfn, ¶ms); + if (rc != ECORE_SUCCESS) { + DP_ERR(edev, "Start V-PORT failed %d\n", rc); + return rc; + } + } + ecore_reset_vport_stats(edev); + if (IS_PF(edev)) + qede_reset_queue_stats(qdev, true); + DP_INFO(edev, "VPORT started with MTU = %u\n", mtu); + + return 0; +} + +static int +qede_stop_vport(struct ecore_dev *edev) +{ + struct ecore_hwfn *p_hwfn; + uint8_t vport_id; + int rc; + int i; + + vport_id = 0; + for_each_hwfn(edev, i) { + p_hwfn = &edev->hwfns[i]; + rc = ecore_sp_vport_stop(p_hwfn, p_hwfn->hw_info.opaque_fid, + vport_id); + if (rc != ECORE_SUCCESS) { + DP_ERR(edev, "Stop V-PORT failed rc = %d\n", rc); + return rc; + } + } + + return 0; +} + +/* Activate or deactivate vport via vport-update */ +int qede_activate_vport(struct rte_eth_dev *eth_dev, bool flg) +{ + struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); + struct ecore_sp_vport_update_params params; + struct ecore_hwfn *p_hwfn; + uint8_t i; + int rc = -1; + + memset(¶ms, 0, sizeof(struct ecore_sp_vport_update_params)); + params.vport_id = 0; + params.update_vport_active_rx_flg = 1; + params.update_vport_active_tx_flg = 1; + params.vport_active_rx_flg = flg; + params.vport_active_tx_flg = flg; + if (!qdev->enable_tx_switching) { + if (IS_VF(edev)) { + params.update_tx_switching_flg = 1; + params.tx_switching_flg = !flg; + DP_INFO(edev, "VF tx-switching is disabled\n"); + } + } + for_each_hwfn(edev, i) { + p_hwfn = &edev->hwfns[i]; + params.opaque_fid = p_hwfn->hw_info.opaque_fid; + rc = ecore_sp_vport_update(p_hwfn, ¶ms, + ECORE_SPQ_MODE_EBLOCK, NULL); + if (rc != ECORE_SUCCESS) { + DP_ERR(edev, "Failed to update vport\n"); + break; + } + } + DP_INFO(edev, "vport is %s\n", flg ? "activated" : "deactivated"); + + return rc; } static void -qede_mac_addr_add(struct rte_eth_dev *eth_dev, struct ether_addr *mac_addr, - uint32_t index, __rte_unused uint32_t pool) +qede_update_sge_tpa_params(struct ecore_sge_tpa_params *sge_tpa_params, + uint16_t mtu, bool enable) { - struct qede_dev *qdev = eth_dev->data->dev_private; - struct ecore_dev *edev = &qdev->edev; + /* Enable LRO in split mode */ + sge_tpa_params->tpa_ipv4_en_flg = enable; + sge_tpa_params->tpa_ipv6_en_flg = enable; + sge_tpa_params->tpa_ipv4_tunn_en_flg = enable; + sge_tpa_params->tpa_ipv6_tunn_en_flg = enable; + /* set if tpa enable changes */ + sge_tpa_params->update_tpa_en_flg = 1; + /* set if tpa parameters should be handled */ + sge_tpa_params->update_tpa_param_flg = enable; + + sge_tpa_params->max_buffers_per_cqe = 20; + /* Enable TPA in split mode. In this mode each TPA segment + * starts on the new BD, so there is one BD per segment. + */ + sge_tpa_params->tpa_pkt_split_flg = 1; + sge_tpa_params->tpa_hdr_data_split_flg = 0; + sge_tpa_params->tpa_gro_consistent_flg = 0; + sge_tpa_params->tpa_max_aggs_num = ETH_TPA_MAX_AGGS_NUM; + sge_tpa_params->tpa_max_size = 0x7FFF; + sge_tpa_params->tpa_min_size_to_start = mtu / 2; + sge_tpa_params->tpa_min_size_to_cont = mtu / 2; +} + +/* Enable/disable LRO via vport-update */ +int qede_enable_tpa(struct rte_eth_dev *eth_dev, bool flg) +{ + struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); + struct ecore_sp_vport_update_params params; + struct ecore_sge_tpa_params tpa_params; + struct ecore_hwfn *p_hwfn; int rc; + int i; - PMD_INIT_FUNC_TRACE(edev); + memset(¶ms, 0, sizeof(struct ecore_sp_vport_update_params)); + memset(&tpa_params, 0, sizeof(struct ecore_sge_tpa_params)); + qede_update_sge_tpa_params(&tpa_params, qdev->mtu, flg); + params.vport_id = 0; + params.sge_tpa_params = &tpa_params; + for_each_hwfn(edev, i) { + p_hwfn = &edev->hwfns[i]; + params.opaque_fid = p_hwfn->hw_info.opaque_fid; + rc = ecore_sp_vport_update(p_hwfn, ¶ms, + ECORE_SPQ_MODE_EBLOCK, NULL); + if (rc != ECORE_SUCCESS) { + DP_ERR(edev, "Failed to update LRO\n"); + return -1; + } + } + qdev->enable_lro = flg; + DP_INFO(edev, "LRO is %s\n", flg ? "enabled" : "disabled"); - if (index >= qdev->dev_info.num_mac_addrs) { - DP_ERR(edev, "Index %u is above MAC filter limit %u\n", - index, qdev->dev_info.num_mac_addrs); - return; + return 0; +} + +/* Update MTU via vport-update without doing port restart. + * The vport must be deactivated before calling this API. + */ +int qede_update_mtu(struct rte_eth_dev *eth_dev, uint16_t mtu) +{ + struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); + struct ecore_sp_vport_update_params params; + struct ecore_hwfn *p_hwfn; + int rc; + int i; + + memset(¶ms, 0, sizeof(struct ecore_sp_vport_update_params)); + params.vport_id = 0; + params.mtu = mtu; + params.vport_id = 0; + for_each_hwfn(edev, i) { + p_hwfn = &edev->hwfns[i]; + params.opaque_fid = p_hwfn->hw_info.opaque_fid; + rc = ecore_sp_vport_update(p_hwfn, ¶ms, + ECORE_SPQ_MODE_EBLOCK, NULL); + if (rc != ECORE_SUCCESS) { + DP_ERR(edev, "Failed to update MTU\n"); + return -1; + } + } + DP_INFO(edev, "MTU updated to %u\n", mtu); + + return 0; +} + +static void qede_set_ucast_cmn_params(struct ecore_filter_ucast *ucast) +{ + memset(ucast, 0, sizeof(struct ecore_filter_ucast)); + ucast->is_rx_filter = true; + ucast->is_tx_filter = true; + /* ucast->assert_on_error = true; - For debug */ +} + +static int +qed_configure_filter_rx_mode(struct rte_eth_dev *eth_dev, + enum qed_filter_rx_mode_type type) +{ + struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); + struct ecore_filter_accept_flags flags; + + memset(&flags, 0, sizeof(flags)); + + flags.update_rx_mode_config = 1; + flags.update_tx_mode_config = 1; + flags.rx_accept_filter = ECORE_ACCEPT_UCAST_MATCHED | + ECORE_ACCEPT_MCAST_MATCHED | + ECORE_ACCEPT_BCAST; + + flags.tx_accept_filter = ECORE_ACCEPT_UCAST_MATCHED | + ECORE_ACCEPT_MCAST_MATCHED | + ECORE_ACCEPT_BCAST; + + if (type == QED_FILTER_RX_MODE_TYPE_PROMISC) { + flags.rx_accept_filter |= ECORE_ACCEPT_UCAST_UNMATCHED; + if (IS_VF(edev)) { + flags.tx_accept_filter |= ECORE_ACCEPT_UCAST_UNMATCHED; + DP_INFO(edev, "Enabling Tx unmatched flag for VF\n"); + } + } else if (type == QED_FILTER_RX_MODE_TYPE_MULTI_PROMISC) { + flags.rx_accept_filter |= ECORE_ACCEPT_MCAST_UNMATCHED; + } else if (type == (QED_FILTER_RX_MODE_TYPE_MULTI_PROMISC | + QED_FILTER_RX_MODE_TYPE_PROMISC)) { + flags.rx_accept_filter |= ECORE_ACCEPT_UCAST_UNMATCHED | + ECORE_ACCEPT_MCAST_UNMATCHED; } - /* Adding macaddr even though promiscuous mode is set */ - if (rte_eth_promiscuous_get(eth_dev->data->port_id) == 1) - DP_INFO(edev, "Port is in promisc mode, yet adding it\n"); + return ecore_filter_accept_cmd(edev, 0, flags, false, false, + ECORE_SPQ_MODE_CB, NULL); +} - /* Add MAC filters according to the unicast secondary macs */ - rc = qede_set_ucast_rx_mac(qdev, QED_FILTER_XCAST_TYPE_ADD, - mac_addr->addr_bytes); - if (rc) - DP_ERR(edev, "Unable to add macaddr rc=%d\n", rc); +static int +qede_vxlan_enable(struct rte_eth_dev *eth_dev, uint8_t clss, + bool enable, bool mask) +{ + struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); + enum _ecore_status_t rc = ECORE_INVAL; + struct ecore_ptt *p_ptt; + struct ecore_tunnel_info tunn; + struct ecore_hwfn *p_hwfn; + int i; + + memset(&tunn, 0, sizeof(struct ecore_tunnel_info)); + tunn.vxlan.b_update_mode = enable; + tunn.vxlan.b_mode_enabled = mask; + tunn.b_update_rx_cls = true; + tunn.b_update_tx_cls = true; + tunn.vxlan.tun_cls = clss; + + for_each_hwfn(edev, i) { + p_hwfn = &edev->hwfns[i]; + if (IS_PF(edev)) { + p_ptt = ecore_ptt_acquire(p_hwfn); + if (!p_ptt) + return -EAGAIN; + } else { + p_ptt = NULL; + } + rc = ecore_sp_pf_update_tunn_cfg(p_hwfn, p_ptt, + &tunn, ECORE_SPQ_MODE_CB, NULL); + if (rc != ECORE_SUCCESS) { + DP_ERR(edev, "Failed to update tunn_clss %u\n", + tunn.vxlan.tun_cls); + if (IS_PF(edev)) + ecore_ptt_release(p_hwfn, p_ptt); + break; + } + } + + if (rc == ECORE_SUCCESS) { + qdev->vxlan.enable = enable; + qdev->vxlan.udp_port = (enable) ? QEDE_VXLAN_DEF_PORT : 0; + DP_INFO(edev, "vxlan is %s\n", enable ? "enabled" : "disabled"); + } + + return rc; +} + +static int +qede_ucast_filter(struct rte_eth_dev *eth_dev, struct ecore_filter_ucast *ucast, + bool add) +{ + struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); + struct qede_ucast_entry *tmp = NULL; + struct qede_ucast_entry *u; + struct ether_addr *mac_addr; + + mac_addr = (struct ether_addr *)ucast->mac; + if (add) { + SLIST_FOREACH(tmp, &qdev->uc_list_head, list) { + if ((memcmp(mac_addr, &tmp->mac, + ETHER_ADDR_LEN) == 0) && + ucast->vni == tmp->vni && + ucast->vlan == tmp->vlan) { + DP_ERR(edev, "Unicast MAC is already added" + " with vlan = %u, vni = %u\n", + ucast->vlan, ucast->vni); + return -EEXIST; + } + } + u = rte_malloc(NULL, sizeof(struct qede_ucast_entry), + RTE_CACHE_LINE_SIZE); + if (!u) { + DP_ERR(edev, "Did not allocate memory for ucast\n"); + return -ENOMEM; + } + ether_addr_copy(mac_addr, &u->mac); + u->vlan = ucast->vlan; + u->vni = ucast->vni; + SLIST_INSERT_HEAD(&qdev->uc_list_head, u, list); + qdev->num_uc_addr++; + } else { + SLIST_FOREACH(tmp, &qdev->uc_list_head, list) { + if ((memcmp(mac_addr, &tmp->mac, + ETHER_ADDR_LEN) == 0) && + ucast->vlan == tmp->vlan && + ucast->vni == tmp->vni) + break; + } + if (tmp == NULL) { + DP_INFO(edev, "Unicast MAC is not found\n"); + return -EINVAL; + } + SLIST_REMOVE(&qdev->uc_list_head, tmp, qede_ucast_entry, list); + qdev->num_uc_addr--; + } + + return 0; +} + +static int +qede_mcast_filter(struct rte_eth_dev *eth_dev, struct ecore_filter_ucast *mcast, + bool add) +{ + struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); + struct ether_addr *mac_addr; + struct qede_mcast_entry *tmp = NULL; + struct qede_mcast_entry *m; + + mac_addr = (struct ether_addr *)mcast->mac; + if (add) { + SLIST_FOREACH(tmp, &qdev->mc_list_head, list) { + if (memcmp(mac_addr, &tmp->mac, ETHER_ADDR_LEN) == 0) { + DP_ERR(edev, + "Multicast MAC is already added\n"); + return -EEXIST; + } + } + m = rte_malloc(NULL, sizeof(struct qede_mcast_entry), + RTE_CACHE_LINE_SIZE); + if (!m) { + DP_ERR(edev, + "Did not allocate memory for mcast\n"); + return -ENOMEM; + } + ether_addr_copy(mac_addr, &m->mac); + SLIST_INSERT_HEAD(&qdev->mc_list_head, m, list); + qdev->num_mc_addr++; + } else { + SLIST_FOREACH(tmp, &qdev->mc_list_head, list) { + if (memcmp(mac_addr, &tmp->mac, ETHER_ADDR_LEN) == 0) + break; + } + if (tmp == NULL) { + DP_INFO(edev, "Multicast mac is not found\n"); + return -EINVAL; + } + SLIST_REMOVE(&qdev->mc_list_head, tmp, + qede_mcast_entry, list); + qdev->num_mc_addr--; + } + + return 0; +} + +static enum _ecore_status_t +qede_mac_int_ops(struct rte_eth_dev *eth_dev, struct ecore_filter_ucast *ucast, + bool add) +{ + struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); + enum _ecore_status_t rc; + struct ecore_filter_mcast mcast; + struct qede_mcast_entry *tmp; + uint16_t j = 0; + + /* Multicast */ + if (is_multicast_ether_addr((struct ether_addr *)ucast->mac)) { + if (add) { + if (qdev->num_mc_addr >= ECORE_MAX_MC_ADDRS) { + DP_ERR(edev, + "Mcast filter table limit exceeded, " + "Please enable mcast promisc mode\n"); + return -ECORE_INVAL; + } + } + rc = qede_mcast_filter(eth_dev, ucast, add); + if (rc == 0) { + DP_INFO(edev, "num_mc_addrs = %u\n", qdev->num_mc_addr); + memset(&mcast, 0, sizeof(mcast)); + mcast.num_mc_addrs = qdev->num_mc_addr; + mcast.opcode = ECORE_FILTER_ADD; + SLIST_FOREACH(tmp, &qdev->mc_list_head, list) { + ether_addr_copy(&tmp->mac, + (struct ether_addr *)&mcast.mac[j]); + j++; + } + rc = ecore_filter_mcast_cmd(edev, &mcast, + ECORE_SPQ_MODE_CB, NULL); + } + if (rc != ECORE_SUCCESS) { + DP_ERR(edev, "Failed to add multicast filter" + " rc = %d, op = %d\n", rc, add); + } + } else { /* Unicast */ + if (add) { + if (qdev->num_uc_addr >= + qdev->dev_info.num_mac_filters) { + DP_ERR(edev, + "Ucast filter table limit exceeded," + " Please enable promisc mode\n"); + return -ECORE_INVAL; + } + } + rc = qede_ucast_filter(eth_dev, ucast, add); + if (rc == 0) + rc = ecore_filter_ucast_cmd(edev, ucast, + ECORE_SPQ_MODE_CB, NULL); + if (rc != ECORE_SUCCESS) { + DP_ERR(edev, "MAC filter failed, rc = %d, op = %d\n", + rc, add); + } + } + + return rc; +} + +static int +qede_mac_addr_add(struct rte_eth_dev *eth_dev, struct ether_addr *mac_addr, + __rte_unused uint32_t index, __rte_unused uint32_t pool) +{ + struct ecore_filter_ucast ucast; + int re; + + qede_set_ucast_cmn_params(&ucast); + ucast.type = ECORE_FILTER_MAC; + ether_addr_copy(mac_addr, (struct ether_addr *)&ucast.mac); + re = (int)qede_mac_int_ops(eth_dev, &ucast, 1); + return re; } static void @@ -264,23 +898,25 @@ qede_mac_addr_remove(struct rte_eth_dev *eth_dev, uint32_t index) { struct qede_dev *qdev = eth_dev->data->dev_private; struct ecore_dev *edev = &qdev->edev; - struct ether_addr mac_addr; - int rc; + struct ecore_filter_ucast ucast; PMD_INIT_FUNC_TRACE(edev); - if (index >= qdev->dev_info.num_mac_addrs) { + if (index >= qdev->dev_info.num_mac_filters) { DP_ERR(edev, "Index %u is above MAC filter limit %u\n", - index, qdev->dev_info.num_mac_addrs); + index, qdev->dev_info.num_mac_filters); return; } + qede_set_ucast_cmn_params(&ucast); + ucast.opcode = ECORE_FILTER_REMOVE; + ucast.type = ECORE_FILTER_MAC; + /* Use the index maintained by rte */ - ether_addr_copy(ð_dev->data->mac_addrs[index], &mac_addr); - rc = qede_set_ucast_rx_mac(qdev, QED_FILTER_XCAST_TYPE_DEL, - mac_addr.addr_bytes); - if (rc) - DP_ERR(edev, "Unable to remove macaddr rc=%d\n", rc); + ether_addr_copy(ð_dev->data->mac_addrs[index], + (struct ether_addr *)&ucast.mac); + + ecore_filter_ucast_cmd(edev, &ucast, ECORE_SPQ_MODE_CB, NULL); } static void @@ -288,7 +924,6 @@ qede_mac_addr_set(struct rte_eth_dev *eth_dev, struct ether_addr *mac_addr) { struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); - int rc; if (IS_VF(edev) && !ecore_vf_check_mac(ECORE_LEADING_HWFN(edev), mac_addr->addr_bytes)) { @@ -298,277 +933,524 @@ qede_mac_addr_set(struct rte_eth_dev *eth_dev, struct ether_addr *mac_addr) return; } - /* First remove the primary mac */ - rc = qede_set_ucast_rx_mac(qdev, QED_FILTER_XCAST_TYPE_DEL, - qdev->primary_mac.addr_bytes); - - if (rc) { - DP_ERR(edev, "Unable to remove current macaddr" - " Reverting to previous default mac\n"); - ether_addr_copy(&qdev->primary_mac, - ð_dev->data->mac_addrs[0]); - return; - } - - /* Add new MAC */ - rc = qede_set_ucast_rx_mac(qdev, QED_FILTER_XCAST_TYPE_ADD, - mac_addr->addr_bytes); - - if (rc) - DP_ERR(edev, "Unable to add new default mac\n"); - else - ether_addr_copy(mac_addr, &qdev->primary_mac); + qede_mac_addr_add(eth_dev, mac_addr, 0, 0); } - - - -static void qede_config_accept_any_vlan(struct qede_dev *qdev, bool action) +static void qede_config_accept_any_vlan(struct qede_dev *qdev, bool flg) { - struct ecore_dev *edev = &qdev->edev; - struct qed_update_vport_params params = { - .vport_id = 0, - .accept_any_vlan = action, - .update_accept_any_vlan_flg = 1, - }; + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); + struct ecore_sp_vport_update_params params; + struct ecore_hwfn *p_hwfn; + uint8_t i; int rc; - /* Proceed only if action actually needs to be performed */ - if (qdev->accept_any_vlan == action) - return; - - rc = qdev->ops->vport_update(edev, ¶ms); - if (rc) { - DP_ERR(edev, "Failed to %s accept-any-vlan\n", - action ? "enable" : "disable"); - } else { - DP_INFO(edev, "%s accept-any-vlan\n", - action ? "enabled" : "disabled"); - qdev->accept_any_vlan = action; - } -} - -void qede_config_rx_mode(struct rte_eth_dev *eth_dev) -{ - struct qede_dev *qdev = eth_dev->data->dev_private; - struct ecore_dev *edev = &qdev->edev; - /* TODO: - QED_FILTER_TYPE_UCAST */ - enum qed_filter_rx_mode_type accept_flags = - QED_FILTER_RX_MODE_TYPE_REGULAR; - struct qed_filter_params rx_mode; - int rc; - - /* Configure the struct for the Rx mode */ - memset(&rx_mode, 0, sizeof(struct qed_filter_params)); - rx_mode.type = QED_FILTER_TYPE_RX_MODE; - - rc = qede_set_ucast_rx_mac(qdev, QED_FILTER_XCAST_TYPE_REPLACE, - eth_dev->data->mac_addrs[0].addr_bytes); - if (rte_eth_promiscuous_get(eth_dev->data->port_id) == 1) { - accept_flags = QED_FILTER_RX_MODE_TYPE_PROMISC; - } else { - rc = qede_set_ucast_rx_mac(qdev, QED_FILTER_XCAST_TYPE_ADD, - eth_dev->data-> - mac_addrs[0].addr_bytes); - if (rc) { - DP_ERR(edev, "Unable to add filter\n"); + memset(¶ms, 0, sizeof(struct ecore_sp_vport_update_params)); + params.vport_id = 0; + params.update_accept_any_vlan_flg = 1; + params.accept_any_vlan = flg; + for_each_hwfn(edev, i) { + p_hwfn = &edev->hwfns[i]; + params.opaque_fid = p_hwfn->hw_info.opaque_fid; + rc = ecore_sp_vport_update(p_hwfn, ¶ms, + ECORE_SPQ_MODE_EBLOCK, NULL); + if (rc != ECORE_SUCCESS) { + DP_ERR(edev, "Failed to configure accept-any-vlan\n"); return; } } - /* take care of VLAN mode */ - if (rte_eth_promiscuous_get(eth_dev->data->port_id) == 1) { - qede_config_accept_any_vlan(qdev, true); - } else if (!qdev->non_configured_vlans) { - /* If we dont have non-configured VLANs and promisc - * is not set, then check if we need to disable - * accept_any_vlan mode. - * Because in this case, accept_any_vlan mode is set - * as part of IFF_RPOMISC flag handling. - */ - qede_config_accept_any_vlan(qdev, false); - } - rx_mode.filter.accept_flags = accept_flags; - rc = qdev->ops->filter_config(edev, &rx_mode); - if (rc) - DP_ERR(edev, "Filter config failed rc=%d\n", rc); + DP_INFO(edev, "%s accept-any-vlan\n", flg ? "enabled" : "disabled"); } -static int qede_vlan_stripping(struct rte_eth_dev *eth_dev, bool set_stripping) +static int qede_vlan_stripping(struct rte_eth_dev *eth_dev, bool flg) { - struct qed_update_vport_params vport_update_params; struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); + struct ecore_sp_vport_update_params params; + struct ecore_hwfn *p_hwfn; + uint8_t i; int rc; - memset(&vport_update_params, 0, sizeof(vport_update_params)); - vport_update_params.vport_id = 0; - vport_update_params.update_inner_vlan_removal_flg = 1; - vport_update_params.inner_vlan_removal_flg = set_stripping; - rc = qdev->ops->vport_update(edev, &vport_update_params); - if (rc) { - DP_ERR(edev, "Update V-PORT failed %d\n", rc); - return rc; + memset(¶ms, 0, sizeof(struct ecore_sp_vport_update_params)); + params.vport_id = 0; + params.update_inner_vlan_removal_flg = 1; + params.inner_vlan_removal_flg = flg; + for_each_hwfn(edev, i) { + p_hwfn = &edev->hwfns[i]; + params.opaque_fid = p_hwfn->hw_info.opaque_fid; + rc = ecore_sp_vport_update(p_hwfn, ¶ms, + ECORE_SPQ_MODE_EBLOCK, NULL); + if (rc != ECORE_SUCCESS) { + DP_ERR(edev, "Failed to update vport\n"); + return -1; + } } + DP_INFO(edev, "VLAN stripping %s\n", flg ? "enabled" : "disabled"); return 0; } -static void qede_vlan_offload_set(struct rte_eth_dev *eth_dev, int mask) -{ - struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); - struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); - - if (mask & ETH_VLAN_STRIP_MASK) { - if (eth_dev->data->dev_conf.rxmode.hw_vlan_strip) - (void)qede_vlan_stripping(eth_dev, 1); - else - (void)qede_vlan_stripping(eth_dev, 0); - } - - DP_INFO(edev, "vlan offload mask %d vlan-strip %d\n", - mask, eth_dev->data->dev_conf.rxmode.hw_vlan_strip); -} - -static int qede_set_ucast_rx_vlan(struct qede_dev *qdev, - enum qed_filter_xcast_params_type opcode, - uint16_t vid) -{ - struct qed_filter_params filter_cmd; - struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); - - memset(&filter_cmd, 0, sizeof(filter_cmd)); - filter_cmd.type = QED_FILTER_TYPE_UCAST; - filter_cmd.filter.ucast.type = opcode; - filter_cmd.filter.ucast.vlan_valid = 1; - filter_cmd.filter.ucast.vlan = vid; - - return qdev->ops->filter_config(edev, &filter_cmd); -} - static int qede_vlan_filter_set(struct rte_eth_dev *eth_dev, uint16_t vlan_id, int on) { struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); struct qed_dev_eth_info *dev_info = &qdev->dev_info; + struct qede_vlan_entry *tmp = NULL; + struct qede_vlan_entry *vlan; + struct ecore_filter_ucast ucast; int rc; - if (vlan_id != 0 && - qdev->configured_vlans == dev_info->num_vlan_filters) { - DP_NOTICE(edev, false, "Reached max VLAN filter limit" - " enabling accept_any_vlan\n"); - qede_config_accept_any_vlan(qdev, true); - return 0; - } - if (on) { - rc = qede_set_ucast_rx_vlan(qdev, QED_FILTER_XCAST_TYPE_ADD, - vlan_id); - if (rc) + if (qdev->configured_vlans == dev_info->num_vlan_filters) { + DP_ERR(edev, "Reached max VLAN filter limit" + " enabling accept_any_vlan\n"); + qede_config_accept_any_vlan(qdev, true); + return 0; + } + + SLIST_FOREACH(tmp, &qdev->vlan_list_head, list) { + if (tmp->vid == vlan_id) { + DP_ERR(edev, "VLAN %u already configured\n", + vlan_id); + return -EEXIST; + } + } + + vlan = rte_malloc(NULL, sizeof(struct qede_vlan_entry), + RTE_CACHE_LINE_SIZE); + + if (!vlan) { + DP_ERR(edev, "Did not allocate memory for VLAN\n"); + return -ENOMEM; + } + + qede_set_ucast_cmn_params(&ucast); + ucast.opcode = ECORE_FILTER_ADD; + ucast.type = ECORE_FILTER_VLAN; + ucast.vlan = vlan_id; + rc = ecore_filter_ucast_cmd(edev, &ucast, ECORE_SPQ_MODE_CB, + NULL); + if (rc != 0) { DP_ERR(edev, "Failed to add VLAN %u rc %d\n", vlan_id, rc); - else - if (vlan_id != 0) - qdev->configured_vlans++; + rte_free(vlan); + } else { + vlan->vid = vlan_id; + SLIST_INSERT_HEAD(&qdev->vlan_list_head, vlan, list); + qdev->configured_vlans++; + DP_INFO(edev, "VLAN %u added, configured_vlans %u\n", + vlan_id, qdev->configured_vlans); + } } else { - rc = qede_set_ucast_rx_vlan(qdev, QED_FILTER_XCAST_TYPE_DEL, - vlan_id); - if (rc) + SLIST_FOREACH(tmp, &qdev->vlan_list_head, list) { + if (tmp->vid == vlan_id) + break; + } + + if (!tmp) { + if (qdev->configured_vlans == 0) { + DP_INFO(edev, + "No VLAN filters configured yet\n"); + return 0; + } + + DP_ERR(edev, "VLAN %u not configured\n", vlan_id); + return -EINVAL; + } + + SLIST_REMOVE(&qdev->vlan_list_head, tmp, qede_vlan_entry, list); + + qede_set_ucast_cmn_params(&ucast); + ucast.opcode = ECORE_FILTER_REMOVE; + ucast.type = ECORE_FILTER_VLAN; + ucast.vlan = vlan_id; + rc = ecore_filter_ucast_cmd(edev, &ucast, ECORE_SPQ_MODE_CB, + NULL); + if (rc != 0) { DP_ERR(edev, "Failed to delete VLAN %u rc %d\n", vlan_id, rc); - else - if (vlan_id != 0) - qdev->configured_vlans--; + } else { + qdev->configured_vlans--; + DP_INFO(edev, "VLAN %u removed configured_vlans %u\n", + vlan_id, qdev->configured_vlans); + } } - DP_INFO(edev, "vlan_id %u on %u rc %d configured_vlans %u\n", - vlan_id, on, rc, qdev->configured_vlans); - return rc; } -static int qede_dev_configure(struct rte_eth_dev *eth_dev) +static int qede_vlan_offload_set(struct rte_eth_dev *eth_dev, int mask) { - struct qede_dev *qdev = eth_dev->data->dev_private; - struct ecore_dev *edev = &qdev->edev; + struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); struct rte_eth_rxmode *rxmode = ð_dev->data->dev_conf.rxmode; + if (mask & ETH_VLAN_STRIP_MASK) { + if (rxmode->hw_vlan_strip) + (void)qede_vlan_stripping(eth_dev, 1); + else + (void)qede_vlan_stripping(eth_dev, 0); + } + + if (mask & ETH_VLAN_FILTER_MASK) { + /* VLAN filtering kicks in when a VLAN is added */ + if (rxmode->hw_vlan_filter) { + qede_vlan_filter_set(eth_dev, 0, 1); + } else { + if (qdev->configured_vlans > 1) { /* Excluding VLAN0 */ + DP_ERR(edev, + " Please remove existing VLAN filters" + " before disabling VLAN filtering\n"); + /* Signal app that VLAN filtering is still + * enabled + */ + rxmode->hw_vlan_filter = true; + } else { + qede_vlan_filter_set(eth_dev, 0, 0); + } + } + } + + if (mask & ETH_VLAN_EXTEND_MASK) + DP_INFO(edev, "No offloads are supported with VLAN Q-in-Q" + " and classification is based on outer tag only\n"); + + DP_INFO(edev, "vlan offload mask %d vlan-strip %d vlan-filter %d\n", + mask, rxmode->hw_vlan_strip, rxmode->hw_vlan_filter); + + return 0; +} + +static void qede_prandom_bytes(uint32_t *buff) +{ + uint8_t i; + + srand((unsigned int)time(NULL)); + for (i = 0; i < ECORE_RSS_KEY_SIZE; i++) + buff[i] = rand(); +} + +int qede_config_rss(struct rte_eth_dev *eth_dev) +{ + struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); +#ifdef RTE_LIBRTE_QEDE_DEBUG_INFO + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); +#endif + uint32_t def_rss_key[ECORE_RSS_KEY_SIZE]; + struct rte_eth_rss_reta_entry64 reta_conf[2]; + struct rte_eth_rss_conf rss_conf; + uint32_t i, id, pos, q; + + rss_conf = eth_dev->data->dev_conf.rx_adv_conf.rss_conf; + if (!rss_conf.rss_key) { + DP_INFO(edev, "Applying driver default key\n"); + rss_conf.rss_key_len = ECORE_RSS_KEY_SIZE * sizeof(uint32_t); + qede_prandom_bytes(&def_rss_key[0]); + rss_conf.rss_key = (uint8_t *)&def_rss_key[0]; + } + + /* Configure RSS hash */ + if (qede_rss_hash_update(eth_dev, &rss_conf)) + return -EINVAL; + + /* Configure default RETA */ + memset(reta_conf, 0, sizeof(reta_conf)); + for (i = 0; i < ECORE_RSS_IND_TABLE_SIZE; i++) + reta_conf[i / RTE_RETA_GROUP_SIZE].mask = UINT64_MAX; + + for (i = 0; i < ECORE_RSS_IND_TABLE_SIZE; i++) { + id = i / RTE_RETA_GROUP_SIZE; + pos = i % RTE_RETA_GROUP_SIZE; + q = i % QEDE_RSS_COUNT(qdev); + reta_conf[id].reta[pos] = q; + } + if (qede_rss_reta_update(eth_dev, &reta_conf[0], + ECORE_RSS_IND_TABLE_SIZE)) + return -EINVAL; + + return 0; +} + +static void qede_fastpath_start(struct ecore_dev *edev) +{ + struct ecore_hwfn *p_hwfn; + int i; + + for_each_hwfn(edev, i) { + p_hwfn = &edev->hwfns[i]; + ecore_hw_start_fastpath(p_hwfn); + } +} + +static int qede_dev_start(struct rte_eth_dev *eth_dev) +{ + struct rte_eth_rxmode *rxmode = ð_dev->data->dev_conf.rxmode; + struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); + PMD_INIT_FUNC_TRACE(edev); - if (eth_dev->data->nb_rx_queues != eth_dev->data->nb_tx_queues) { - DP_NOTICE(edev, false, - "Unequal number of rx/tx queues " - "is not supported RX=%u TX=%u\n", - eth_dev->data->nb_rx_queues, - eth_dev->data->nb_tx_queues); - return -EINVAL; + /* Update MTU only if it has changed */ + if (qdev->mtu != qdev->new_mtu) { + if (qede_update_mtu(eth_dev, qdev->new_mtu)) + goto err; + qdev->mtu = qdev->new_mtu; } + /* Configure TPA parameters */ + if (rxmode->enable_lro) { + if (qede_enable_tpa(eth_dev, true)) + return -EINVAL; + /* Enable scatter mode for LRO */ + if (!rxmode->enable_scatter) + eth_dev->data->scattered_rx = 1; + } + + /* Start queues */ + if (qede_start_queues(eth_dev)) + goto err; + + /* Newer SR-IOV PF driver expects RX/TX queues to be started before + * enabling RSS. Hence RSS configuration is deferred upto this point. + * Also, we would like to retain similar behavior in PF case, so we + * don't do PF/VF specific check here. + */ + if (rxmode->mq_mode == ETH_MQ_RX_RSS) + if (qede_config_rss(eth_dev)) + goto err; + + /* Enable vport*/ + if (qede_activate_vport(eth_dev, true)) + goto err; + + /* Bring-up the link */ + qede_dev_set_link_state(eth_dev, true); + + /* Update link status */ + qede_link_update(eth_dev, 0); + + /* Start/resume traffic */ + qede_fastpath_start(edev); + + DP_INFO(edev, "Device started\n"); + + return 0; +err: + DP_ERR(edev, "Device start fails\n"); + return -1; /* common error code is < 0 */ +} + +static void qede_dev_stop(struct rte_eth_dev *eth_dev) +{ + struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); + + PMD_INIT_FUNC_TRACE(edev); + + /* Disable vport */ + if (qede_activate_vport(eth_dev, false)) + return; + + if (qdev->enable_lro) + qede_enable_tpa(eth_dev, false); + + /* Stop queues */ + qede_stop_queues(eth_dev); + + /* Disable traffic */ + ecore_hw_stop_fastpath(edev); /* TBD - loop */ + + /* Bring the link down */ + qede_dev_set_link_state(eth_dev, false); + + DP_INFO(edev, "Device is stopped\n"); +} + +#define QEDE_TX_SWITCHING "vf_txswitch" + +const char *valid_args[] = { + QEDE_TX_SWITCHING, + NULL, +}; + +static int qede_args_check(const char *key, const char *val, void *opaque) +{ + unsigned long tmp; + int ret = 0; + struct rte_eth_dev *eth_dev = opaque; + struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); +#ifdef RTE_LIBRTE_QEDE_DEBUG_INFO + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); +#endif + + errno = 0; + tmp = strtoul(val, NULL, 0); + if (errno) { + DP_INFO(edev, "%s: \"%s\" is not a valid integer", key, val); + return errno; + } + + if (strcmp(QEDE_TX_SWITCHING, key) == 0) + qdev->enable_tx_switching = !!tmp; + + return ret; +} + +static int qede_args(struct rte_eth_dev *eth_dev) +{ + struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(eth_dev->device); + struct rte_kvargs *kvlist; + struct rte_devargs *devargs; + int ret; + int i; + + devargs = pci_dev->device.devargs; + if (!devargs) + return 0; /* return success */ + + kvlist = rte_kvargs_parse(devargs->args, valid_args); + if (kvlist == NULL) + return -EINVAL; + + /* Process parameters. */ + for (i = 0; (valid_args[i] != NULL); ++i) { + if (rte_kvargs_count(kvlist, valid_args[i])) { + ret = rte_kvargs_process(kvlist, valid_args[i], + qede_args_check, eth_dev); + if (ret != ECORE_SUCCESS) { + rte_kvargs_free(kvlist); + return ret; + } + } + } + rte_kvargs_free(kvlist); + + return 0; +} + +static int qede_dev_configure(struct rte_eth_dev *eth_dev) +{ + struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); + struct rte_eth_rxmode *rxmode = ð_dev->data->dev_conf.rxmode; + int ret; + + PMD_INIT_FUNC_TRACE(edev); + /* Check requirements for 100G mode */ - if (edev->num_hwfns > 1) { - if (eth_dev->data->nb_rx_queues < 2) { - DP_NOTICE(edev, false, - "100G mode requires minimum two queues\n"); + if (ECORE_IS_CMT(edev)) { + if (eth_dev->data->nb_rx_queues < 2 || + eth_dev->data->nb_tx_queues < 2) { + DP_ERR(edev, "100G mode needs min. 2 RX/TX queues\n"); return -EINVAL; } - if ((eth_dev->data->nb_rx_queues % 2) != 0) { - DP_NOTICE(edev, false, - "100G mode requires even number of queues\n"); + if ((eth_dev->data->nb_rx_queues % 2 != 0) || + (eth_dev->data->nb_tx_queues % 2 != 0)) { + DP_ERR(edev, + "100G mode needs even no. of RX/TX queues\n"); return -EINVAL; } } - qdev->num_rss = eth_dev->data->nb_rx_queues; + /* We need to have min 1 RX queue.There is no min check in + * rte_eth_dev_configure(), so we are checking it here. + */ + if (eth_dev->data->nb_rx_queues == 0) { + DP_ERR(edev, "Minimum one RX queue is required\n"); + return -EINVAL; + } - /* Initial state */ - qdev->state = QEDE_CLOSE; + /* Enable Tx switching by default */ + qdev->enable_tx_switching = 1; + + /* Parse devargs and fix up rxmode */ + if (qede_args(eth_dev)) + return -ENOTSUP; /* Sanity checks and throw warnings */ - - if (rxmode->enable_scatter == 1) { - DP_ERR(edev, "RX scatter packets is not supported\n"); - return -EINVAL; - } - - if (rxmode->enable_lro == 1) { - DP_INFO(edev, "LRO is not supported\n"); - return -EINVAL; - } + if (rxmode->enable_scatter) + eth_dev->data->scattered_rx = 1; if (!rxmode->hw_strip_crc) DP_INFO(edev, "L2 CRC stripping is always enabled in hw\n"); if (!rxmode->hw_ip_checksum) DP_INFO(edev, "IP/UDP/TCP checksum offload is always enabled " - "in hw\n"); + "in hw\n"); + if (rxmode->header_split) + DP_INFO(edev, "Header split enable is not supported\n"); + if (!(rxmode->mq_mode == ETH_MQ_RX_NONE || rxmode->mq_mode == + ETH_MQ_RX_RSS)) { + DP_ERR(edev, "Unsupported multi-queue mode\n"); + return -ENOTSUP; + } + /* Flow director mode check */ + if (qede_check_fdir_support(eth_dev)) + return -ENOTSUP; + /* Deallocate resources if held previously. It is needed only if the + * queue count has been changed from previous configuration. If its + * going to change then it means RX/TX queue setup will be called + * again and the fastpath pointers will be reinitialized there. + */ + if (qdev->num_tx_queues != eth_dev->data->nb_tx_queues || + qdev->num_rx_queues != eth_dev->data->nb_rx_queues) { + qede_dealloc_fp_resc(eth_dev); + /* Proceed with updated queue count */ + qdev->num_tx_queues = eth_dev->data->nb_tx_queues; + qdev->num_rx_queues = eth_dev->data->nb_rx_queues; + if (qede_alloc_fp_resc(qdev)) + return -ENOMEM; + } - DP_INFO(edev, "Allocated %d RSS queues on %d TC/s\n", - QEDE_RSS_CNT(qdev), qdev->num_tc); + /* If jumbo enabled adjust MTU */ + if (eth_dev->data->dev_conf.rxmode.jumbo_frame) + eth_dev->data->mtu = + eth_dev->data->dev_conf.rxmode.max_rx_pkt_len - + ETHER_HDR_LEN - ETHER_CRC_LEN; - DP_INFO(edev, "my_id %u rel_pf_id %u abs_pf_id %u" - " port %u first_on_engine %d\n", - edev->hwfns[0].my_id, - edev->hwfns[0].rel_pf_id, - edev->hwfns[0].abs_pf_id, - edev->hwfns[0].port_id, edev->hwfns[0].first_on_engine); + /* VF's MTU has to be set using vport-start where as + * PF's MTU can be updated via vport-update. + */ + if (IS_VF(edev)) { + if (qede_start_vport(qdev, eth_dev->data->mtu)) + return -1; + } else { + if (qede_update_mtu(eth_dev, eth_dev->data->mtu)) + return -1; + } + + qdev->mtu = eth_dev->data->mtu; + qdev->new_mtu = qdev->mtu; + + /* Enable VLAN offloads by default */ + ret = qede_vlan_offload_set(eth_dev, ETH_VLAN_STRIP_MASK | + ETH_VLAN_FILTER_MASK | + ETH_VLAN_EXTEND_MASK); + if (ret) + return ret; + + DP_INFO(edev, "Device configured with RSS=%d TSS=%d\n", + QEDE_RSS_COUNT(qdev), QEDE_TSS_COUNT(qdev)); return 0; } /* Info about HW descriptor ring limitations */ static const struct rte_eth_desc_lim qede_rx_desc_lim = { - .nb_max = NUM_RX_BDS_MAX, + .nb_max = 0x8000, /* 32K */ .nb_min = 128, - .nb_align = 128 /* lowest common multiple */ + .nb_align = 128 /* lowest common multiple */ }; static const struct rte_eth_desc_lim qede_tx_desc_lim = { - .nb_max = NUM_TX_BDS_MAX, + .nb_max = 0x8000, /* 32K */ .nb_min = 256, - .nb_align = 256 + .nb_align = 256, + .nb_seg_max = ETH_TX_MAX_BDS_PER_LSO_PACKET, + .nb_mtu_seg_max = ETH_TX_MAX_BDS_PER_NON_LSO_PACKET }; static void @@ -577,43 +1459,69 @@ qede_dev_info_get(struct rte_eth_dev *eth_dev, { struct qede_dev *qdev = eth_dev->data->dev_private; struct ecore_dev *edev = &qdev->edev; + struct qed_link_output link; + uint32_t speed_cap = 0; PMD_INIT_FUNC_TRACE(edev); - dev_info->min_rx_bufsize = (uint32_t)(ETHER_MIN_MTU + - QEDE_ETH_OVERHEAD); + dev_info->pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); + dev_info->min_rx_bufsize = (uint32_t)QEDE_MIN_RX_BUFF_SIZE; dev_info->max_rx_pktlen = (uint32_t)ETH_TX_MAX_NON_LSO_PKT_LEN; dev_info->rx_desc_lim = qede_rx_desc_lim; dev_info->tx_desc_lim = qede_tx_desc_lim; - dev_info->max_rx_queues = (uint16_t)QEDE_MAX_RSS_CNT(qdev); - dev_info->max_tx_queues = dev_info->max_rx_queues; - dev_info->max_mac_addrs = qdev->dev_info.num_mac_addrs; - if (IS_VF(edev)) - dev_info->max_vfs = 0; + + if (IS_PF(edev)) + dev_info->max_rx_queues = (uint16_t)RTE_MIN( + QEDE_MAX_RSS_CNT(qdev), QEDE_PF_NUM_CONNS / 2); else - dev_info->max_vfs = (uint16_t)NUM_OF_VFS(&qdev->edev); - dev_info->driver_name = qdev->drv_ver; + dev_info->max_rx_queues = (uint16_t)RTE_MIN( + QEDE_MAX_RSS_CNT(qdev), ECORE_MAX_VF_CHAINS_PER_PF); + dev_info->max_tx_queues = dev_info->max_rx_queues; + + dev_info->max_mac_addrs = qdev->dev_info.num_mac_filters; + dev_info->max_vfs = 0; dev_info->reta_size = ECORE_RSS_IND_TABLE_SIZE; + dev_info->hash_key_size = ECORE_RSS_KEY_SIZE * sizeof(uint32_t); dev_info->flow_type_rss_offloads = (uint64_t)QEDE_RSS_OFFLOAD_ALL; dev_info->default_txconf = (struct rte_eth_txconf) { .txq_flags = QEDE_TXQ_FLAGS, }; - dev_info->rx_offload_capa = (DEV_RX_OFFLOAD_VLAN_STRIP | - DEV_RX_OFFLOAD_IPV4_CKSUM | - DEV_RX_OFFLOAD_UDP_CKSUM | - DEV_RX_OFFLOAD_TCP_CKSUM); - dev_info->tx_offload_capa = (DEV_TX_OFFLOAD_VLAN_INSERT | - DEV_TX_OFFLOAD_IPV4_CKSUM | - DEV_TX_OFFLOAD_UDP_CKSUM | - DEV_TX_OFFLOAD_TCP_CKSUM); + dev_info->rx_offload_capa = (DEV_RX_OFFLOAD_VLAN_STRIP | + DEV_RX_OFFLOAD_IPV4_CKSUM | + DEV_RX_OFFLOAD_UDP_CKSUM | + DEV_RX_OFFLOAD_TCP_CKSUM | + DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM | + DEV_RX_OFFLOAD_TCP_LRO); - dev_info->speed_capa = ETH_LINK_SPEED_25G | ETH_LINK_SPEED_40G; + dev_info->tx_offload_capa = (DEV_TX_OFFLOAD_VLAN_INSERT | + DEV_TX_OFFLOAD_IPV4_CKSUM | + DEV_TX_OFFLOAD_UDP_CKSUM | + DEV_TX_OFFLOAD_TCP_CKSUM | + DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM | + DEV_TX_OFFLOAD_TCP_TSO | + DEV_TX_OFFLOAD_VXLAN_TNL_TSO); + + memset(&link, 0, sizeof(struct qed_link_output)); + qdev->ops->common->get_link(edev, &link); + if (link.adv_speed & NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_1G) + speed_cap |= ETH_LINK_SPEED_1G; + if (link.adv_speed & NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_10G) + speed_cap |= ETH_LINK_SPEED_10G; + if (link.adv_speed & NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_25G) + speed_cap |= ETH_LINK_SPEED_25G; + if (link.adv_speed & NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_40G) + speed_cap |= ETH_LINK_SPEED_40G; + if (link.adv_speed & NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_50G) + speed_cap |= ETH_LINK_SPEED_50G; + if (link.adv_speed & NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_BB_100G) + speed_cap |= ETH_LINK_SPEED_100G; + dev_info->speed_capa = speed_cap; } /* return 0 means link status changed, -1 means not changed */ -static int +int qede_link_update(struct rte_eth_dev *eth_dev, __rte_unused int wait_to_complete) { struct qede_dev *qdev = eth_dev->data->dev_private; @@ -657,49 +1565,38 @@ qede_link_update(struct rte_eth_dev *eth_dev, __rte_unused int wait_to_complete) return ((curr->link_status == link.link_up) ? -1 : 0); } -static void -qede_rx_mode_setting(struct rte_eth_dev *eth_dev, - enum qed_filter_rx_mode_type accept_flags) -{ - struct qede_dev *qdev = eth_dev->data->dev_private; - struct ecore_dev *edev = &qdev->edev; - struct qed_filter_params rx_mode; - - DP_INFO(edev, "%s mode %u\n", __func__, accept_flags); - - memset(&rx_mode, 0, sizeof(struct qed_filter_params)); - rx_mode.type = QED_FILTER_TYPE_RX_MODE; - rx_mode.filter.accept_flags = accept_flags; - qdev->ops->filter_config(edev, &rx_mode); -} - static void qede_promiscuous_enable(struct rte_eth_dev *eth_dev) { +#ifdef RTE_LIBRTE_QEDE_DEBUG_INIT struct qede_dev *qdev = eth_dev->data->dev_private; struct ecore_dev *edev = &qdev->edev; PMD_INIT_FUNC_TRACE(edev); +#endif enum qed_filter_rx_mode_type type = QED_FILTER_RX_MODE_TYPE_PROMISC; if (rte_eth_allmulticast_get(eth_dev->data->port_id) == 1) type |= QED_FILTER_RX_MODE_TYPE_MULTI_PROMISC; - qede_rx_mode_setting(eth_dev, type); + qed_configure_filter_rx_mode(eth_dev, type); } static void qede_promiscuous_disable(struct rte_eth_dev *eth_dev) { +#ifdef RTE_LIBRTE_QEDE_DEBUG_INIT struct qede_dev *qdev = eth_dev->data->dev_private; struct ecore_dev *edev = &qdev->edev; PMD_INIT_FUNC_TRACE(edev); +#endif if (rte_eth_allmulticast_get(eth_dev->data->port_id) == 1) - qede_rx_mode_setting(eth_dev, - QED_FILTER_RX_MODE_TYPE_MULTI_PROMISC); + qed_configure_filter_rx_mode(eth_dev, + QED_FILTER_RX_MODE_TYPE_MULTI_PROMISC); else - qede_rx_mode_setting(eth_dev, QED_FILTER_RX_MODE_TYPE_REGULAR); + qed_configure_filter_rx_mode(eth_dev, + QED_FILTER_RX_MODE_TYPE_REGULAR); } static void qede_poll_sp_sb_cb(void *param) @@ -724,8 +1621,9 @@ static void qede_poll_sp_sb_cb(void *param) static void qede_dev_close(struct rte_eth_dev *eth_dev) { - struct qede_dev *qdev = eth_dev->data->dev_private; - struct ecore_dev *edev = &qdev->edev; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); + struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); PMD_INIT_FUNC_TRACE(edev); @@ -734,82 +1632,180 @@ static void qede_dev_close(struct rte_eth_dev *eth_dev) * by the app without reconfiguration. However, in dev_close() we * can release all the resources and device can be brought up newly */ - if (qdev->state != QEDE_STOP) + if (eth_dev->data->dev_started) qede_dev_stop(eth_dev); - else - DP_INFO(edev, "Device is already stopped\n"); - qede_free_mem_load(qdev); + qede_stop_vport(edev); + qede_fdir_dealloc_resc(eth_dev); + qede_dealloc_fp_resc(eth_dev); - qede_free_fp_arrays(qdev); - - qede_dev_set_link_state(eth_dev, false); + eth_dev->data->nb_rx_queues = 0; + eth_dev->data->nb_tx_queues = 0; qdev->ops->common->slowpath_stop(edev); - qdev->ops->common->remove(edev); - - rte_intr_disable(ð_dev->pci_dev->intr_handle); - - rte_intr_callback_unregister(ð_dev->pci_dev->intr_handle, + rte_intr_disable(&pci_dev->intr_handle); + rte_intr_callback_unregister(&pci_dev->intr_handle, qede_interrupt_handler, (void *)eth_dev); - - if (edev->num_hwfns > 1) + if (ECORE_IS_CMT(edev)) rte_eal_alarm_cancel(qede_poll_sp_sb_cb, (void *)eth_dev); - - qdev->state = QEDE_CLOSE; } -static void +static int qede_get_stats(struct rte_eth_dev *eth_dev, struct rte_eth_stats *eth_stats) { struct qede_dev *qdev = eth_dev->data->dev_private; struct ecore_dev *edev = &qdev->edev; struct ecore_eth_stats stats; + unsigned int i = 0, j = 0, qid; + unsigned int rxq_stat_cntrs, txq_stat_cntrs; + struct qede_tx_queue *txq; - qdev->ops->get_vport_stats(edev, &stats); + ecore_get_vport_stats(edev, &stats); /* RX Stats */ - eth_stats->ipackets = stats.rx_ucast_pkts + - stats.rx_mcast_pkts + stats.rx_bcast_pkts; + eth_stats->ipackets = stats.common.rx_ucast_pkts + + stats.common.rx_mcast_pkts + stats.common.rx_bcast_pkts; - eth_stats->ibytes = stats.rx_ucast_bytes + - stats.rx_mcast_bytes + stats.rx_bcast_bytes; + eth_stats->ibytes = stats.common.rx_ucast_bytes + + stats.common.rx_mcast_bytes + stats.common.rx_bcast_bytes; - eth_stats->ierrors = stats.rx_crc_errors + - stats.rx_align_errors + - stats.rx_carrier_errors + - stats.rx_oversize_packets + - stats.rx_jabbers + stats.rx_undersize_packets; + eth_stats->ierrors = stats.common.rx_crc_errors + + stats.common.rx_align_errors + + stats.common.rx_carrier_errors + + stats.common.rx_oversize_packets + + stats.common.rx_jabbers + stats.common.rx_undersize_packets; - eth_stats->rx_nombuf = stats.no_buff_discards; + eth_stats->rx_nombuf = stats.common.no_buff_discards; - eth_stats->imissed = stats.mftag_filter_discards + - stats.mac_filter_discards + - stats.no_buff_discards + stats.brb_truncates + stats.brb_discards; + eth_stats->imissed = stats.common.mftag_filter_discards + + stats.common.mac_filter_discards + + stats.common.no_buff_discards + + stats.common.brb_truncates + stats.common.brb_discards; /* TX stats */ - eth_stats->opackets = stats.tx_ucast_pkts + - stats.tx_mcast_pkts + stats.tx_bcast_pkts; + eth_stats->opackets = stats.common.tx_ucast_pkts + + stats.common.tx_mcast_pkts + stats.common.tx_bcast_pkts; - eth_stats->obytes = stats.tx_ucast_bytes + - stats.tx_mcast_bytes + stats.tx_bcast_bytes; + eth_stats->obytes = stats.common.tx_ucast_bytes + + stats.common.tx_mcast_bytes + stats.common.tx_bcast_bytes; - eth_stats->oerrors = stats.tx_err_drop_pkts; + eth_stats->oerrors = stats.common.tx_err_drop_pkts; + + /* Queue stats */ + rxq_stat_cntrs = RTE_MIN(QEDE_RSS_COUNT(qdev), + RTE_ETHDEV_QUEUE_STAT_CNTRS); + txq_stat_cntrs = RTE_MIN(QEDE_TSS_COUNT(qdev), + RTE_ETHDEV_QUEUE_STAT_CNTRS); + if ((rxq_stat_cntrs != (unsigned int)QEDE_RSS_COUNT(qdev)) || + (txq_stat_cntrs != (unsigned int)QEDE_TSS_COUNT(qdev))) + DP_VERBOSE(edev, ECORE_MSG_DEBUG, + "Not all the queue stats will be displayed. Set" + " RTE_ETHDEV_QUEUE_STAT_CNTRS config param" + " appropriately and retry.\n"); + + for_each_rss(qid) { + eth_stats->q_ipackets[i] = + *(uint64_t *)( + ((char *)(qdev->fp_array[qid].rxq)) + + offsetof(struct qede_rx_queue, + rcv_pkts)); + eth_stats->q_errors[i] = + *(uint64_t *)( + ((char *)(qdev->fp_array[qid].rxq)) + + offsetof(struct qede_rx_queue, + rx_hw_errors)) + + *(uint64_t *)( + ((char *)(qdev->fp_array[qid].rxq)) + + offsetof(struct qede_rx_queue, + rx_alloc_errors)); + i++; + if (i == rxq_stat_cntrs) + break; + } + + for_each_tss(qid) { + txq = qdev->fp_array[qid].txq; + eth_stats->q_opackets[j] = + *((uint64_t *)(uintptr_t) + (((uint64_t)(uintptr_t)(txq)) + + offsetof(struct qede_tx_queue, + xmit_pkts))); + j++; + if (j == txq_stat_cntrs) + break; + } + + return 0; +} + +static unsigned +qede_get_xstats_count(struct qede_dev *qdev) { + if (ECORE_IS_BB(&qdev->edev)) + return RTE_DIM(qede_xstats_strings) + + RTE_DIM(qede_bb_xstats_strings) + + (RTE_DIM(qede_rxq_xstats_strings) * + RTE_MIN(QEDE_RSS_COUNT(qdev), + RTE_ETHDEV_QUEUE_STAT_CNTRS)); + else + return RTE_DIM(qede_xstats_strings) + + RTE_DIM(qede_ah_xstats_strings) + + (RTE_DIM(qede_rxq_xstats_strings) * + RTE_MIN(QEDE_RSS_COUNT(qdev), + RTE_ETHDEV_QUEUE_STAT_CNTRS)); } static int -qede_get_xstats_names(__rte_unused struct rte_eth_dev *dev, - struct rte_eth_xstat_name *xstats_names, unsigned limit) +qede_get_xstats_names(struct rte_eth_dev *dev, + struct rte_eth_xstat_name *xstats_names, + __rte_unused unsigned int limit) { - unsigned int i, stat_cnt = RTE_DIM(qede_xstats_strings); + struct qede_dev *qdev = dev->data->dev_private; + struct ecore_dev *edev = &qdev->edev; + const unsigned int stat_cnt = qede_get_xstats_count(qdev); + unsigned int i, qid, stat_idx = 0; + unsigned int rxq_stat_cntrs; - if (xstats_names != NULL) - for (i = 0; i < stat_cnt; i++) - snprintf(xstats_names[i].name, - sizeof(xstats_names[i].name), + if (xstats_names != NULL) { + for (i = 0; i < RTE_DIM(qede_xstats_strings); i++) { + snprintf(xstats_names[stat_idx].name, + sizeof(xstats_names[stat_idx].name), "%s", qede_xstats_strings[i].name); + stat_idx++; + } + + if (ECORE_IS_BB(edev)) { + for (i = 0; i < RTE_DIM(qede_bb_xstats_strings); i++) { + snprintf(xstats_names[stat_idx].name, + sizeof(xstats_names[stat_idx].name), + "%s", + qede_bb_xstats_strings[i].name); + stat_idx++; + } + } else { + for (i = 0; i < RTE_DIM(qede_ah_xstats_strings); i++) { + snprintf(xstats_names[stat_idx].name, + sizeof(xstats_names[stat_idx].name), + "%s", + qede_ah_xstats_strings[i].name); + stat_idx++; + } + } + + rxq_stat_cntrs = RTE_MIN(QEDE_RSS_COUNT(qdev), + RTE_ETHDEV_QUEUE_STAT_CNTRS); + for (qid = 0; qid < rxq_stat_cntrs; qid++) { + for (i = 0; i < RTE_DIM(qede_rxq_xstats_strings); i++) { + snprintf(xstats_names[stat_idx].name, + sizeof(xstats_names[stat_idx].name), + "%.4s%d%s", + qede_rxq_xstats_strings[i].name, qid, + qede_rxq_xstats_strings[i].name + 4); + stat_idx++; + } + } + } return stat_cnt; } @@ -821,18 +1817,55 @@ qede_get_xstats(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, struct qede_dev *qdev = dev->data->dev_private; struct ecore_dev *edev = &qdev->edev; struct ecore_eth_stats stats; - unsigned int num = RTE_DIM(qede_xstats_strings); + const unsigned int num = qede_get_xstats_count(qdev); + unsigned int i, qid, stat_idx = 0; + unsigned int rxq_stat_cntrs; if (n < num) return num; - qdev->ops->get_vport_stats(edev, &stats); + ecore_get_vport_stats(edev, &stats); - for (num = 0; num < n; num++) - xstats[num].value = *(u64 *)(((char *)&stats) + - qede_xstats_strings[num].offset); + for (i = 0; i < RTE_DIM(qede_xstats_strings); i++) { + xstats[stat_idx].value = *(uint64_t *)(((char *)&stats) + + qede_xstats_strings[i].offset); + xstats[stat_idx].id = stat_idx; + stat_idx++; + } - return num; + if (ECORE_IS_BB(edev)) { + for (i = 0; i < RTE_DIM(qede_bb_xstats_strings); i++) { + xstats[stat_idx].value = + *(uint64_t *)(((char *)&stats) + + qede_bb_xstats_strings[i].offset); + xstats[stat_idx].id = stat_idx; + stat_idx++; + } + } else { + for (i = 0; i < RTE_DIM(qede_ah_xstats_strings); i++) { + xstats[stat_idx].value = + *(uint64_t *)(((char *)&stats) + + qede_ah_xstats_strings[i].offset); + xstats[stat_idx].id = stat_idx; + stat_idx++; + } + } + + rxq_stat_cntrs = RTE_MIN(QEDE_RSS_COUNT(qdev), + RTE_ETHDEV_QUEUE_STAT_CNTRS); + for (qid = 0; qid < rxq_stat_cntrs; qid++) { + for_each_rss(qid) { + for (i = 0; i < RTE_DIM(qede_rxq_xstats_strings); i++) { + xstats[stat_idx].value = *(uint64_t *)( + ((char *)(qdev->fp_array[qid].rxq)) + + qede_rxq_xstats_strings[i].offset); + xstats[stat_idx].id = stat_idx; + stat_idx++; + } + } + } + + return stat_idx; } static void @@ -842,6 +1875,7 @@ qede_reset_xstats(struct rte_eth_dev *dev) struct ecore_dev *edev = &qdev->edev; ecore_reset_vport_stats(edev); + qede_reset_queue_stats(qdev, true); } int qede_dev_set_link_state(struct rte_eth_dev *eth_dev, bool link_up) @@ -877,6 +1911,7 @@ static void qede_reset_stats(struct rte_eth_dev *eth_dev) struct ecore_dev *edev = &qdev->edev; ecore_reset_vport_stats(edev); + qede_reset_queue_stats(qdev, false); } static void qede_allmulticast_enable(struct rte_eth_dev *eth_dev) @@ -887,15 +1922,17 @@ static void qede_allmulticast_enable(struct rte_eth_dev *eth_dev) if (rte_eth_promiscuous_get(eth_dev->data->port_id) == 1) type |= QED_FILTER_RX_MODE_TYPE_PROMISC; - qede_rx_mode_setting(eth_dev, type); + qed_configure_filter_rx_mode(eth_dev, type); } static void qede_allmulticast_disable(struct rte_eth_dev *eth_dev) { if (rte_eth_promiscuous_get(eth_dev->data->port_id) == 1) - qede_rx_mode_setting(eth_dev, QED_FILTER_RX_MODE_TYPE_PROMISC); + qed_configure_filter_rx_mode(eth_dev, + QED_FILTER_RX_MODE_TYPE_PROMISC); else - qede_rx_mode_setting(eth_dev, QED_FILTER_RX_MODE_TYPE_REGULAR); + qed_configure_filter_rx_mode(eth_dev, + QED_FILTER_RX_MODE_TYPE_REGULAR); } static int qede_flow_ctrl_set(struct rte_eth_dev *eth_dev, @@ -964,8 +2001,22 @@ static const uint32_t * qede_dev_supported_ptypes_get(struct rte_eth_dev *eth_dev) { static const uint32_t ptypes[] = { + RTE_PTYPE_L2_ETHER, + RTE_PTYPE_L2_ETHER_VLAN, RTE_PTYPE_L3_IPV4, RTE_PTYPE_L3_IPV6, + RTE_PTYPE_L4_TCP, + RTE_PTYPE_L4_UDP, + RTE_PTYPE_TUNNEL_VXLAN, + RTE_PTYPE_L4_FRAG, + /* Inner */ + RTE_PTYPE_INNER_L2_ETHER, + RTE_PTYPE_INNER_L2_ETHER_VLAN, + RTE_PTYPE_INNER_L3_IPV4, + RTE_PTYPE_INNER_L3_IPV6, + RTE_PTYPE_INNER_L4_TCP, + RTE_PTYPE_INNER_L4_UDP, + RTE_PTYPE_INNER_L4_FRAG, RTE_PTYPE_UNKNOWN }; @@ -975,87 +2026,191 @@ qede_dev_supported_ptypes_get(struct rte_eth_dev *eth_dev) return NULL; } +static void qede_init_rss_caps(uint8_t *rss_caps, uint64_t hf) +{ + *rss_caps = 0; + *rss_caps |= (hf & ETH_RSS_IPV4) ? ECORE_RSS_IPV4 : 0; + *rss_caps |= (hf & ETH_RSS_IPV6) ? ECORE_RSS_IPV6 : 0; + *rss_caps |= (hf & ETH_RSS_IPV6_EX) ? ECORE_RSS_IPV6 : 0; + *rss_caps |= (hf & ETH_RSS_NONFRAG_IPV4_TCP) ? ECORE_RSS_IPV4_TCP : 0; + *rss_caps |= (hf & ETH_RSS_NONFRAG_IPV6_TCP) ? ECORE_RSS_IPV6_TCP : 0; + *rss_caps |= (hf & ETH_RSS_IPV6_TCP_EX) ? ECORE_RSS_IPV6_TCP : 0; + *rss_caps |= (hf & ETH_RSS_NONFRAG_IPV4_UDP) ? ECORE_RSS_IPV4_UDP : 0; + *rss_caps |= (hf & ETH_RSS_NONFRAG_IPV6_UDP) ? ECORE_RSS_IPV6_UDP : 0; +} + int qede_rss_hash_update(struct rte_eth_dev *eth_dev, struct rte_eth_rss_conf *rss_conf) { - struct qed_update_vport_params vport_update_params; - struct qede_dev *qdev = eth_dev->data->dev_private; - struct ecore_dev *edev = &qdev->edev; - uint8_t rss_caps; + struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); + struct ecore_sp_vport_update_params vport_update_params; + struct ecore_rss_params rss_params; + struct ecore_hwfn *p_hwfn; uint32_t *key = (uint32_t *)rss_conf->rss_key; uint64_t hf = rss_conf->rss_hf; - int i; - - if (hf == 0) - DP_ERR(edev, "hash function 0 will disable RSS\n"); - - rss_caps = 0; - rss_caps |= (hf & ETH_RSS_IPV4) ? ECORE_RSS_IPV4 : 0; - rss_caps |= (hf & ETH_RSS_IPV6) ? ECORE_RSS_IPV6 : 0; - rss_caps |= (hf & ETH_RSS_IPV6_EX) ? ECORE_RSS_IPV6 : 0; - rss_caps |= (hf & ETH_RSS_NONFRAG_IPV4_TCP) ? ECORE_RSS_IPV4_TCP : 0; - rss_caps |= (hf & ETH_RSS_NONFRAG_IPV6_TCP) ? ECORE_RSS_IPV6_TCP : 0; - rss_caps |= (hf & ETH_RSS_IPV6_TCP_EX) ? ECORE_RSS_IPV6_TCP : 0; - - /* If the mapping doesn't fit any supported, return */ - if (rss_caps == 0 && hf != 0) - return -EINVAL; + uint8_t len = rss_conf->rss_key_len; + uint8_t idx; + uint8_t i; + int rc; memset(&vport_update_params, 0, sizeof(vport_update_params)); + memset(&rss_params, 0, sizeof(rss_params)); - if (key != NULL) - memcpy(qdev->rss_params.rss_key, rss_conf->rss_key, - rss_conf->rss_key_len); + DP_INFO(edev, "RSS hf = 0x%lx len = %u key = %p\n", + (unsigned long)hf, len, key); - qdev->rss_params.rss_caps = rss_caps; - memcpy(&vport_update_params.rss_params, &qdev->rss_params, - sizeof(vport_update_params.rss_params)); - vport_update_params.update_rss_flg = 1; + if (hf != 0) { + /* Enabling RSS */ + DP_INFO(edev, "Enabling rss\n"); + + /* RSS caps */ + qede_init_rss_caps(&rss_params.rss_caps, hf); + rss_params.update_rss_capabilities = 1; + + /* RSS hash key */ + if (key) { + if (len > (ECORE_RSS_KEY_SIZE * sizeof(uint32_t))) { + DP_ERR(edev, "RSS key length exceeds limit\n"); + return -EINVAL; + } + DP_INFO(edev, "Applying user supplied hash key\n"); + rss_params.update_rss_key = 1; + memcpy(&rss_params.rss_key, key, len); + } + rss_params.rss_enable = 1; + } + + rss_params.update_rss_config = 1; + /* tbl_size has to be set with capabilities */ + rss_params.rss_table_size_log = 7; vport_update_params.vport_id = 0; + /* pass the L2 handles instead of qids */ + for (i = 0 ; i < ECORE_RSS_IND_TABLE_SIZE ; i++) { + idx = qdev->rss_ind_table[i]; + rss_params.rss_ind_table[i] = qdev->fp_array[idx].rxq->handle; + } + vport_update_params.rss_params = &rss_params; - return qdev->ops->vport_update(edev, &vport_update_params); -} + for_each_hwfn(edev, i) { + p_hwfn = &edev->hwfns[i]; + vport_update_params.opaque_fid = p_hwfn->hw_info.opaque_fid; + rc = ecore_sp_vport_update(p_hwfn, &vport_update_params, + ECORE_SPQ_MODE_EBLOCK, NULL); + if (rc) { + DP_ERR(edev, "vport-update for RSS failed\n"); + return rc; + } + } + qdev->rss_enable = rss_params.rss_enable; -int qede_rss_hash_conf_get(struct rte_eth_dev *eth_dev, - struct rte_eth_rss_conf *rss_conf) -{ - struct qede_dev *qdev = eth_dev->data->dev_private; - uint64_t hf; - - if (rss_conf->rss_key_len < sizeof(qdev->rss_params.rss_key)) - return -EINVAL; - - if (rss_conf->rss_key) - memcpy(rss_conf->rss_key, qdev->rss_params.rss_key, - sizeof(qdev->rss_params.rss_key)); - - hf = 0; - hf |= (qdev->rss_params.rss_caps & ECORE_RSS_IPV4) ? - ETH_RSS_IPV4 : 0; - hf |= (qdev->rss_params.rss_caps & ECORE_RSS_IPV6) ? - ETH_RSS_IPV6 : 0; - hf |= (qdev->rss_params.rss_caps & ECORE_RSS_IPV6) ? - ETH_RSS_IPV6_EX : 0; - hf |= (qdev->rss_params.rss_caps & ECORE_RSS_IPV4_TCP) ? - ETH_RSS_NONFRAG_IPV4_TCP : 0; - hf |= (qdev->rss_params.rss_caps & ECORE_RSS_IPV6_TCP) ? - ETH_RSS_NONFRAG_IPV6_TCP : 0; - hf |= (qdev->rss_params.rss_caps & ECORE_RSS_IPV6_TCP) ? - ETH_RSS_IPV6_TCP_EX : 0; - - rss_conf->rss_hf = hf; + /* Update local structure for hash query */ + qdev->rss_conf.rss_hf = hf; + qdev->rss_conf.rss_key_len = len; + if (qdev->rss_enable) { + if (qdev->rss_conf.rss_key == NULL) { + qdev->rss_conf.rss_key = (uint8_t *)malloc(len); + if (qdev->rss_conf.rss_key == NULL) { + DP_ERR(edev, "No memory to store RSS key\n"); + return -ENOMEM; + } + } + if (key && len) { + DP_INFO(edev, "Storing RSS key\n"); + memcpy(qdev->rss_conf.rss_key, key, len); + } + } else if (!qdev->rss_enable && len == 0) { + if (qdev->rss_conf.rss_key) { + free(qdev->rss_conf.rss_key); + qdev->rss_conf.rss_key = NULL; + DP_INFO(edev, "Free RSS key\n"); + } + } return 0; } +static int qede_rss_hash_conf_get(struct rte_eth_dev *eth_dev, + struct rte_eth_rss_conf *rss_conf) +{ + struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); + + rss_conf->rss_hf = qdev->rss_conf.rss_hf; + rss_conf->rss_key_len = qdev->rss_conf.rss_key_len; + + if (rss_conf->rss_key && qdev->rss_conf.rss_key) + memcpy(rss_conf->rss_key, qdev->rss_conf.rss_key, + rss_conf->rss_key_len); + return 0; +} + +static bool qede_update_rss_parm_cmt(struct ecore_dev *edev, + struct ecore_rss_params *rss) +{ + int i, fn; + bool rss_mode = 1; /* enable */ + struct ecore_queue_cid *cid; + struct ecore_rss_params *t_rss; + + /* In regular scenario, we'd simply need to take input handlers. + * But in CMT, we'd have to split the handlers according to the + * engine they were configured on. We'd then have to understand + * whether RSS is really required, since 2-queues on CMT doesn't + * require RSS. + */ + + /* CMT should be round-robin */ + for (i = 0; i < ECORE_RSS_IND_TABLE_SIZE; i++) { + cid = rss->rss_ind_table[i]; + + if (cid->p_owner == ECORE_LEADING_HWFN(edev)) + t_rss = &rss[0]; + else + t_rss = &rss[1]; + + t_rss->rss_ind_table[i / edev->num_hwfns] = cid; + } + + t_rss = &rss[1]; + t_rss->update_rss_ind_table = 1; + t_rss->rss_table_size_log = 7; + t_rss->update_rss_config = 1; + + /* Make sure RSS is actually required */ + for_each_hwfn(edev, fn) { + for (i = 1; i < ECORE_RSS_IND_TABLE_SIZE / edev->num_hwfns; + i++) { + if (rss[fn].rss_ind_table[i] != + rss[fn].rss_ind_table[0]) + break; + } + + if (i == ECORE_RSS_IND_TABLE_SIZE / edev->num_hwfns) { + DP_INFO(edev, + "CMT - 1 queue per-hwfn; Disabling RSS\n"); + rss_mode = 0; + goto out; + } + } + +out: + t_rss->rss_enable = rss_mode; + + return rss_mode; +} + int qede_rss_reta_update(struct rte_eth_dev *eth_dev, struct rte_eth_rss_reta_entry64 *reta_conf, uint16_t reta_size) { - struct qed_update_vport_params vport_update_params; - struct qede_dev *qdev = eth_dev->data->dev_private; - struct ecore_dev *edev = &qdev->edev; + struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); + struct ecore_sp_vport_update_params vport_update_params; + struct ecore_rss_params *params; + struct ecore_hwfn *p_hwfn; uint16_t i, idx, shift; + uint8_t entry; + int rc = 0; if (reta_size > ETH_RSS_RETA_SIZE_128) { DP_ERR(edev, "reta_size %d is not supported by hardware\n", @@ -1064,42 +2219,75 @@ int qede_rss_reta_update(struct rte_eth_dev *eth_dev, } memset(&vport_update_params, 0, sizeof(vport_update_params)); - memcpy(&vport_update_params.rss_params, &qdev->rss_params, - sizeof(vport_update_params.rss_params)); + params = rte_zmalloc("qede_rss", sizeof(*params) * edev->num_hwfns, + RTE_CACHE_LINE_SIZE); + if (params == NULL) { + DP_ERR(edev, "failed to allocate memory\n"); + return -ENOMEM; + } for (i = 0; i < reta_size; i++) { idx = i / RTE_RETA_GROUP_SIZE; shift = i % RTE_RETA_GROUP_SIZE; if (reta_conf[idx].mask & (1ULL << shift)) { - uint8_t entry = reta_conf[idx].reta[shift]; - qdev->rss_params.rss_ind_table[i] = entry; + entry = reta_conf[idx].reta[shift]; + /* Pass rxq handles to ecore */ + params->rss_ind_table[i] = + qdev->fp_array[entry].rxq->handle; + /* Update the local copy for RETA query command */ + qdev->rss_ind_table[i] = entry; } } - vport_update_params.update_rss_flg = 1; - vport_update_params.vport_id = 0; + params->update_rss_ind_table = 1; + params->rss_table_size_log = 7; + params->update_rss_config = 1; - return qdev->ops->vport_update(edev, &vport_update_params); + /* Fix up RETA for CMT mode device */ + if (ECORE_IS_CMT(edev)) + qdev->rss_enable = qede_update_rss_parm_cmt(edev, + params); + vport_update_params.vport_id = 0; + /* Use the current value of rss_enable */ + params->rss_enable = qdev->rss_enable; + vport_update_params.rss_params = params; + + for_each_hwfn(edev, i) { + p_hwfn = &edev->hwfns[i]; + vport_update_params.opaque_fid = p_hwfn->hw_info.opaque_fid; + rc = ecore_sp_vport_update(p_hwfn, &vport_update_params, + ECORE_SPQ_MODE_EBLOCK, NULL); + if (rc) { + DP_ERR(edev, "vport-update for RSS failed\n"); + goto out; + } + } + +out: + rte_free(params); + return rc; } -int qede_rss_reta_query(struct rte_eth_dev *eth_dev, - struct rte_eth_rss_reta_entry64 *reta_conf, - uint16_t reta_size) +static int qede_rss_reta_query(struct rte_eth_dev *eth_dev, + struct rte_eth_rss_reta_entry64 *reta_conf, + uint16_t reta_size) { struct qede_dev *qdev = eth_dev->data->dev_private; + struct ecore_dev *edev = &qdev->edev; uint16_t i, idx, shift; + uint8_t entry; if (reta_size > ETH_RSS_RETA_SIZE_128) { - struct ecore_dev *edev = &qdev->edev; DP_ERR(edev, "reta_size %d is not supported\n", reta_size); + return -EINVAL; } for (i = 0; i < reta_size; i++) { idx = i / RTE_RETA_GROUP_SIZE; shift = i % RTE_RETA_GROUP_SIZE; if (reta_conf[idx].mask & (1ULL << shift)) { - uint8_t entry = qdev->rss_params.rss_ind_table[i]; + entry = qdev->rss_ind_table[i]; reta_conf[idx].reta[shift] = entry; } } @@ -1107,34 +2295,407 @@ int qede_rss_reta_query(struct rte_eth_dev *eth_dev, return 0; } -int qede_set_mtu(struct rte_eth_dev *dev, uint16_t mtu) + + +static int qede_set_mtu(struct rte_eth_dev *dev, uint16_t mtu) { - uint32_t frame_size; - struct qede_dev *qdev = dev->data->dev_private; + struct qede_dev *qdev = QEDE_INIT_QDEV(dev); + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); struct rte_eth_dev_info dev_info = {0}; + struct qede_fastpath *fp; + uint32_t max_rx_pkt_len; + uint32_t frame_size; + uint16_t rx_buf_size; + uint16_t bufsz; + bool restart = false; + int i; + PMD_INIT_FUNC_TRACE(edev); + if (IS_VF(edev)) + return -ENOTSUP; qede_dev_info_get(dev, &dev_info); - - /* VLAN_TAG = 4 */ - frame_size = mtu + ETHER_HDR_LEN + ETHER_CRC_LEN + 4; - - if ((mtu < ETHER_MIN_MTU) || (frame_size > dev_info.max_rx_pktlen)) + max_rx_pkt_len = mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; + frame_size = max_rx_pkt_len + QEDE_ETH_OVERHEAD; + if ((mtu < ETHER_MIN_MTU) || (frame_size > dev_info.max_rx_pktlen)) { + DP_ERR(edev, "MTU %u out of range, %u is maximum allowable\n", + mtu, dev_info.max_rx_pktlen - ETHER_HDR_LEN - + ETHER_CRC_LEN - QEDE_ETH_OVERHEAD); return -EINVAL; - + } if (!dev->data->scattered_rx && - frame_size > dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM) + frame_size > dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM) { + DP_INFO(edev, "MTU greater than minimum RX buffer size of %u\n", + dev->data->min_rx_buf_size); return -EINVAL; - - if (frame_size > ETHER_MAX_LEN) + } + /* Temporarily replace I/O functions with dummy ones. It cannot + * be set to NULL because rte_eth_rx_burst() doesn't check for NULL. + */ + dev->rx_pkt_burst = qede_rxtx_pkts_dummy; + dev->tx_pkt_burst = qede_rxtx_pkts_dummy; + if (dev->data->dev_started) { + dev->data->dev_started = 0; + qede_dev_stop(dev); + restart = true; + } + rte_delay_ms(1000); + qdev->new_mtu = mtu; + /* Fix up RX buf size for all queues of the port */ + for_each_rss(i) { + fp = &qdev->fp_array[i]; + if (fp->rxq != NULL) { + bufsz = (uint16_t)rte_pktmbuf_data_room_size( + fp->rxq->mb_pool) - RTE_PKTMBUF_HEADROOM; + if (dev->data->scattered_rx) + rx_buf_size = bufsz + ETHER_HDR_LEN + + ETHER_CRC_LEN + QEDE_ETH_OVERHEAD; + else + rx_buf_size = frame_size; + rx_buf_size = QEDE_CEIL_TO_CACHE_LINE_SIZE(rx_buf_size); + fp->rxq->rx_buf_size = rx_buf_size; + DP_INFO(edev, "buf_size adjusted to %u\n", rx_buf_size); + } + } + if (max_rx_pkt_len > ETHER_MAX_LEN) dev->data->dev_conf.rxmode.jumbo_frame = 1; else dev->data->dev_conf.rxmode.jumbo_frame = 0; - + if (!dev->data->dev_started && restart) { + qede_dev_start(dev); + dev->data->dev_started = 1; + } /* update max frame size */ - dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size; - qdev->mtu = mtu; - qede_dev_stop(dev); - qede_dev_start(dev); + dev->data->dev_conf.rxmode.max_rx_pkt_len = max_rx_pkt_len; + /* Reassign back */ + dev->rx_pkt_burst = qede_recv_pkts; + dev->tx_pkt_burst = qede_xmit_pkts; + + return 0; +} + +static int +qede_conf_udp_dst_port(struct rte_eth_dev *eth_dev, + struct rte_eth_udp_tunnel *tunnel_udp, + bool add) +{ + struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); + struct ecore_tunnel_info tunn; /* @DPDK */ + struct ecore_hwfn *p_hwfn; + struct ecore_ptt *p_ptt; + uint16_t udp_port; + int rc, i; + + PMD_INIT_FUNC_TRACE(edev); + + memset(&tunn, 0, sizeof(tunn)); + if (tunnel_udp->prot_type == RTE_TUNNEL_TYPE_VXLAN) { + /* Enable VxLAN tunnel if needed before UDP port update using + * default MAC/VLAN classification. + */ + if (add) { + if (qdev->vxlan.udp_port == tunnel_udp->udp_port) { + DP_INFO(edev, + "UDP port %u was already configured\n", + tunnel_udp->udp_port); + return ECORE_SUCCESS; + } + /* Enable VXLAN if it was not enabled while adding + * VXLAN filter. + */ + if (!qdev->vxlan.enable) { + rc = qede_vxlan_enable(eth_dev, + ECORE_TUNN_CLSS_MAC_VLAN, true, true); + if (rc != ECORE_SUCCESS) { + DP_ERR(edev, "Failed to enable VXLAN " + "prior to updating UDP port\n"); + return rc; + } + } + udp_port = tunnel_udp->udp_port; + } else { + if (qdev->vxlan.udp_port != tunnel_udp->udp_port) { + DP_ERR(edev, "UDP port %u doesn't exist\n", + tunnel_udp->udp_port); + return ECORE_INVAL; + } + udp_port = 0; + } + + tunn.vxlan_port.b_update_port = true; + tunn.vxlan_port.port = udp_port; + for_each_hwfn(edev, i) { + p_hwfn = &edev->hwfns[i]; + if (IS_PF(edev)) { + p_ptt = ecore_ptt_acquire(p_hwfn); + if (!p_ptt) + return -EAGAIN; + } else { + p_ptt = NULL; + } + rc = ecore_sp_pf_update_tunn_cfg(p_hwfn, p_ptt, &tunn, + ECORE_SPQ_MODE_CB, NULL); + if (rc != ECORE_SUCCESS) { + DP_ERR(edev, "Unable to config UDP port %u\n", + tunn.vxlan_port.port); + if (IS_PF(edev)) + ecore_ptt_release(p_hwfn, p_ptt); + return rc; + } + } + + qdev->vxlan.udp_port = udp_port; + /* If the request is to delete UDP port and if the number of + * VXLAN filters have reached 0 then VxLAN offload can be be + * disabled. + */ + if (!add && qdev->vxlan.enable && qdev->vxlan.num_filters == 0) + return qede_vxlan_enable(eth_dev, + ECORE_TUNN_CLSS_MAC_VLAN, false, true); + } + + return 0; +} + +static int +qede_udp_dst_port_del(struct rte_eth_dev *eth_dev, + struct rte_eth_udp_tunnel *tunnel_udp) +{ + return qede_conf_udp_dst_port(eth_dev, tunnel_udp, false); +} + +static int +qede_udp_dst_port_add(struct rte_eth_dev *eth_dev, + struct rte_eth_udp_tunnel *tunnel_udp) +{ + return qede_conf_udp_dst_port(eth_dev, tunnel_udp, true); +} + +static void qede_get_ecore_tunn_params(uint32_t filter, uint32_t *type, + uint32_t *clss, char *str) +{ + uint16_t j; + *clss = MAX_ECORE_TUNN_CLSS; + + for (j = 0; j < RTE_DIM(qede_tunn_types); j++) { + if (filter == qede_tunn_types[j].rte_filter_type) { + *type = qede_tunn_types[j].qede_type; + *clss = qede_tunn_types[j].qede_tunn_clss; + strcpy(str, qede_tunn_types[j].string); + return; + } + } +} + +static int +qede_set_ucast_tunn_cmn_param(struct ecore_filter_ucast *ucast, + const struct rte_eth_tunnel_filter_conf *conf, + uint32_t type) +{ + /* Init commmon ucast params first */ + qede_set_ucast_cmn_params(ucast); + + /* Copy out the required fields based on classification type */ + ucast->type = type; + + switch (type) { + case ECORE_FILTER_VNI: + ucast->vni = conf->tenant_id; + break; + case ECORE_FILTER_INNER_VLAN: + ucast->vlan = conf->inner_vlan; + break; + case ECORE_FILTER_MAC: + memcpy(ucast->mac, conf->outer_mac.addr_bytes, + ETHER_ADDR_LEN); + break; + case ECORE_FILTER_INNER_MAC: + memcpy(ucast->mac, conf->inner_mac.addr_bytes, + ETHER_ADDR_LEN); + break; + case ECORE_FILTER_MAC_VNI_PAIR: + memcpy(ucast->mac, conf->outer_mac.addr_bytes, + ETHER_ADDR_LEN); + ucast->vni = conf->tenant_id; + break; + case ECORE_FILTER_INNER_MAC_VNI_PAIR: + memcpy(ucast->mac, conf->inner_mac.addr_bytes, + ETHER_ADDR_LEN); + ucast->vni = conf->tenant_id; + break; + case ECORE_FILTER_INNER_PAIR: + memcpy(ucast->mac, conf->inner_mac.addr_bytes, + ETHER_ADDR_LEN); + ucast->vlan = conf->inner_vlan; + break; + default: + return -EINVAL; + } + + return ECORE_SUCCESS; +} + +static int qede_vxlan_tunn_config(struct rte_eth_dev *eth_dev, + enum rte_filter_op filter_op, + const struct rte_eth_tunnel_filter_conf *conf) +{ + struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); + enum ecore_filter_ucast_type type; + enum ecore_tunn_clss clss = MAX_ECORE_TUNN_CLSS; + struct ecore_filter_ucast ucast = {0}; + char str[80]; + uint16_t filter_type = 0; + int rc; + + PMD_INIT_FUNC_TRACE(edev); + + switch (filter_op) { + case RTE_ETH_FILTER_ADD: + if (IS_VF(edev)) + return qede_vxlan_enable(eth_dev, + ECORE_TUNN_CLSS_MAC_VLAN, true, true); + + filter_type = conf->filter_type; + /* Determine if the given filter classification is supported */ + qede_get_ecore_tunn_params(filter_type, &type, &clss, str); + if (clss == MAX_ECORE_TUNN_CLSS) { + DP_ERR(edev, "Unsupported filter type\n"); + return -EINVAL; + } + /* Init tunnel ucast params */ + rc = qede_set_ucast_tunn_cmn_param(&ucast, conf, type); + if (rc != ECORE_SUCCESS) { + DP_ERR(edev, "Unsupported VxLAN filter type 0x%x\n", + conf->filter_type); + return rc; + } + DP_INFO(edev, "Rule: \"%s\", op %d, type 0x%x\n", + str, filter_op, ucast.type); + + ucast.opcode = ECORE_FILTER_ADD; + + /* Skip MAC/VLAN if filter is based on VNI */ + if (!(filter_type & ETH_TUNNEL_FILTER_TENID)) { + rc = qede_mac_int_ops(eth_dev, &ucast, 1); + if (rc == 0) { + /* Enable accept anyvlan */ + qede_config_accept_any_vlan(qdev, true); + } + } else { + rc = qede_ucast_filter(eth_dev, &ucast, 1); + if (rc == 0) + rc = ecore_filter_ucast_cmd(edev, &ucast, + ECORE_SPQ_MODE_CB, NULL); + } + + if (rc != ECORE_SUCCESS) + return rc; + + qdev->vxlan.num_filters++; + qdev->vxlan.filter_type = filter_type; + if (!qdev->vxlan.enable) + return qede_vxlan_enable(eth_dev, clss, true, true); + + break; + case RTE_ETH_FILTER_DELETE: + if (IS_VF(edev)) + return qede_vxlan_enable(eth_dev, + ECORE_TUNN_CLSS_MAC_VLAN, false, true); + + filter_type = conf->filter_type; + /* Determine if the given filter classification is supported */ + qede_get_ecore_tunn_params(filter_type, &type, &clss, str); + if (clss == MAX_ECORE_TUNN_CLSS) { + DP_ERR(edev, "Unsupported filter type\n"); + return -EINVAL; + } + /* Init tunnel ucast params */ + rc = qede_set_ucast_tunn_cmn_param(&ucast, conf, type); + if (rc != ECORE_SUCCESS) { + DP_ERR(edev, "Unsupported VxLAN filter type 0x%x\n", + conf->filter_type); + return rc; + } + DP_INFO(edev, "Rule: \"%s\", op %d, type 0x%x\n", + str, filter_op, ucast.type); + + ucast.opcode = ECORE_FILTER_REMOVE; + + if (!(filter_type & ETH_TUNNEL_FILTER_TENID)) { + rc = qede_mac_int_ops(eth_dev, &ucast, 0); + } else { + rc = qede_ucast_filter(eth_dev, &ucast, 0); + if (rc == 0) + rc = ecore_filter_ucast_cmd(edev, &ucast, + ECORE_SPQ_MODE_CB, NULL); + } + if (rc != ECORE_SUCCESS) + return rc; + + qdev->vxlan.num_filters--; + + /* Disable VXLAN if VXLAN filters become 0 */ + if (qdev->vxlan.num_filters == 0) + return qede_vxlan_enable(eth_dev, clss, false, true); + break; + default: + DP_ERR(edev, "Unsupported operation %d\n", filter_op); + return -EINVAL; + } + + return 0; +} + +int qede_dev_filter_ctrl(struct rte_eth_dev *eth_dev, + enum rte_filter_type filter_type, + enum rte_filter_op filter_op, + void *arg) +{ + struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); + struct rte_eth_tunnel_filter_conf *filter_conf = + (struct rte_eth_tunnel_filter_conf *)arg; + + switch (filter_type) { + case RTE_ETH_FILTER_TUNNEL: + switch (filter_conf->tunnel_type) { + case RTE_TUNNEL_TYPE_VXLAN: + DP_INFO(edev, + "Packet steering to the specified Rx queue" + " is not supported with VXLAN tunneling"); + return(qede_vxlan_tunn_config(eth_dev, filter_op, + filter_conf)); + /* Place holders for future tunneling support */ + case RTE_TUNNEL_TYPE_GENEVE: + case RTE_TUNNEL_TYPE_TEREDO: + case RTE_TUNNEL_TYPE_NVGRE: + case RTE_TUNNEL_TYPE_IP_IN_GRE: + case RTE_L2_TUNNEL_TYPE_E_TAG: + DP_ERR(edev, "Unsupported tunnel type %d\n", + filter_conf->tunnel_type); + return -EINVAL; + case RTE_TUNNEL_TYPE_NONE: + default: + return 0; + } + break; + case RTE_ETH_FILTER_FDIR: + return qede_fdir_filter_conf(eth_dev, filter_op, arg); + case RTE_ETH_FILTER_NTUPLE: + return qede_ntuple_filter_conf(eth_dev, filter_op, arg); + case RTE_ETH_FILTER_MACVLAN: + case RTE_ETH_FILTER_ETHERTYPE: + case RTE_ETH_FILTER_FLEXIBLE: + case RTE_ETH_FILTER_SYN: + case RTE_ETH_FILTER_HASH: + case RTE_ETH_FILTER_L2_TUNNEL: + case RTE_ETH_FILTER_MAX: + default: + DP_ERR(edev, "Unsupported filter type %d\n", + filter_type); + return -EINVAL; + } return 0; } @@ -1174,6 +2735,9 @@ static const struct eth_dev_ops qede_eth_dev_ops = { .reta_update = qede_rss_reta_update, .reta_query = qede_rss_reta_query, .mtu_set = qede_set_mtu, + .filter_ctrl = qede_dev_filter_ctrl, + .udp_tunnel_port_add = qede_udp_dst_port_add, + .udp_tunnel_port_del = qede_udp_dst_port_del, }; static const struct eth_dev_ops qede_eth_vf_dev_ops = { @@ -1206,14 +2770,17 @@ static const struct eth_dev_ops qede_eth_vf_dev_ops = { .reta_update = qede_rss_reta_update, .reta_query = qede_rss_reta_query, .mtu_set = qede_set_mtu, + .udp_tunnel_port_add = qede_udp_dst_port_add, + .udp_tunnel_port_del = qede_udp_dst_port_del, }; static void qede_update_pf_params(struct ecore_dev *edev) { struct ecore_pf_params pf_params; - /* 32 rx + 32 tx */ + memset(&pf_params, 0, sizeof(struct ecore_pf_params)); - pf_params.eth_pf_params.num_cons = 64; + pf_params.eth_pf_params.num_cons = QEDE_PF_NUM_CONNS; + pf_params.eth_pf_params.num_arfs_filters = QEDE_RFS_MAX_FLTR; qed_ops->common->update_pf_params(edev, &pf_params); } @@ -1225,7 +2792,6 @@ static int qede_common_dev_init(struct rte_eth_dev *eth_dev, bool is_vf) struct ecore_dev *edev; struct qed_dev_eth_info dev_info; struct qed_slowpath_params params; - uint32_t qed_ver; static bool do_once = true; uint8_t bulletin_change; uint8_t vf_mac[ETHER_ADDR_LEN]; @@ -1234,13 +2800,14 @@ static int qede_common_dev_init(struct rte_eth_dev *eth_dev, bool is_vf) /* Fix up ecore debug level */ uint32_t dp_module = ~0 & ~ECORE_MSG_HW; uint8_t dp_level = ECORE_LEVEL_VERBOSE; - uint32_t max_mac_addrs; int rc; /* Extract key data structures */ adapter = eth_dev->data->dev_private; + adapter->ethdev = eth_dev; edev = &adapter->edev; - pci_addr = eth_dev->pci_dev->addr; + pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); + pci_addr = pci_dev->addr; PMD_INIT_FUNC_TRACE(edev); @@ -1250,18 +2817,18 @@ static int qede_common_dev_init(struct rte_eth_dev *eth_dev, bool is_vf) eth_dev->rx_pkt_burst = qede_recv_pkts; eth_dev->tx_pkt_burst = qede_xmit_pkts; + eth_dev->tx_pkt_prepare = qede_xmit_prep_pkts; if (rte_eal_process_type() != RTE_PROC_PRIMARY) { - DP_NOTICE(edev, false, - "Skipping device init from secondary process\n"); + DP_ERR(edev, "Skipping device init from secondary process\n"); return 0; } - pci_dev = eth_dev->pci_dev; - rte_eth_copy_pci_info(eth_dev, pci_dev); - qed_ver = qed_get_protocol_version(QED_PROTOCOL_ETH); + /* @DPDK */ + edev->vendor_id = pci_dev->id.vendor_id; + edev->device_id = pci_dev->id.device_id; qed_ops = qed_get_eth_ops(); if (!qed_ops) { @@ -1270,21 +2837,16 @@ static int qede_common_dev_init(struct rte_eth_dev *eth_dev, bool is_vf) } DP_INFO(edev, "Starting qede probe\n"); - - rc = qed_ops->common->probe(edev, pci_dev, QED_PROTOCOL_ETH, - dp_module, dp_level, is_vf); - + rc = qed_ops->common->probe(edev, pci_dev, dp_module, + dp_level, is_vf); if (rc != 0) { DP_ERR(edev, "qede probe failed rc %d\n", rc); return -ENODEV; } - qede_update_pf_params(edev); - - rte_intr_callback_register(ð_dev->pci_dev->intr_handle, + rte_intr_callback_register(&pci_dev->intr_handle, qede_interrupt_handler, (void *)eth_dev); - - if (rte_intr_enable(ð_dev->pci_dev->intr_handle)) { + if (rte_intr_enable(&pci_dev->intr_handle)) { DP_ERR(edev, "rte_intr_enable() failed\n"); return -ENODEV; } @@ -1292,17 +2854,18 @@ static int qede_common_dev_init(struct rte_eth_dev *eth_dev, bool is_vf) /* Start the Slowpath-process */ memset(¶ms, 0, sizeof(struct qed_slowpath_params)); params.int_mode = ECORE_INT_MODE_MSIX; - params.drv_major = QEDE_MAJOR_VERSION; - params.drv_minor = QEDE_MINOR_VERSION; - params.drv_rev = QEDE_REVISION_VERSION; - params.drv_eng = QEDE_ENGINEERING_VERSION; - strncpy((char *)params.name, "qede LAN", QED_DRV_VER_STR_SIZE); + params.drv_major = QEDE_PMD_VERSION_MAJOR; + params.drv_minor = QEDE_PMD_VERSION_MINOR; + params.drv_rev = QEDE_PMD_VERSION_REVISION; + params.drv_eng = QEDE_PMD_VERSION_PATCH; + strncpy((char *)params.name, QEDE_PMD_VER_PREFIX, + QEDE_PMD_DRV_VER_STR_SIZE); /* For CMT mode device do periodic polling for slowpath events. * This is required since uio device uses only one MSI-x * interrupt vector but we need one for each engine. */ - if (edev->num_hwfns > 1) { + if (ECORE_IS_CMT(edev) && IS_PF(edev)) { rc = rte_eal_alarm_set(timer_period * US_PER_S, qede_poll_sp_sb_cb, (void *)eth_dev); @@ -1333,20 +2896,20 @@ static int qede_common_dev_init(struct rte_eth_dev *eth_dev, bool is_vf) qede_alloc_etherdev(adapter, &dev_info); - adapter->ops->common->set_id(edev, edev->name, QEDE_DRV_MODULE_VERSION); + adapter->ops->common->set_name(edev, edev->name); if (!is_vf) - adapter->dev_info.num_mac_addrs = + adapter->dev_info.num_mac_filters = (uint32_t)RESC_NUM(ECORE_LEADING_HWFN(edev), ECORE_MAC); else ecore_vf_get_num_mac_filters(ECORE_LEADING_HWFN(edev), - &adapter->dev_info.num_mac_addrs); + (uint32_t *)&adapter->dev_info.num_mac_filters); /* Allocate memory for storing MAC addr */ eth_dev->data->mac_addrs = rte_zmalloc(edev->name, (ETHER_ADDR_LEN * - adapter->dev_info.num_mac_addrs), + adapter->dev_info.num_mac_filters), RTE_CACHE_LINE_SIZE); if (eth_dev->data->mac_addrs == NULL) { @@ -1380,8 +2943,7 @@ static int qede_common_dev_init(struct rte_eth_dev *eth_dev, bool is_vf) ether_addr_copy(ð_dev->data->mac_addrs[0], &adapter->primary_mac); } else { - DP_NOTICE(edev, false, - "No VF macaddr assigned\n"); + DP_ERR(edev, "No VF macaddr assigned\n"); } } } @@ -1389,19 +2951,34 @@ static int qede_common_dev_init(struct rte_eth_dev *eth_dev, bool is_vf) eth_dev->dev_ops = (is_vf) ? &qede_eth_vf_dev_ops : &qede_eth_dev_ops; if (do_once) { +#ifdef RTE_LIBRTE_QEDE_DEBUG_INFO qede_print_adapter_info(adapter); +#endif do_once = false; } - DP_NOTICE(edev, false, "MAC address : %02x:%02x:%02x:%02x:%02x:%02x\n", - adapter->primary_mac.addr_bytes[0], - adapter->primary_mac.addr_bytes[1], - adapter->primary_mac.addr_bytes[2], - adapter->primary_mac.addr_bytes[3], - adapter->primary_mac.addr_bytes[4], - adapter->primary_mac.addr_bytes[5]); + adapter->num_tx_queues = 0; + adapter->num_rx_queues = 0; + SLIST_INIT(&adapter->fdir_info.fdir_list_head); + SLIST_INIT(&adapter->vlan_list_head); + SLIST_INIT(&adapter->uc_list_head); + adapter->mtu = ETHER_MTU; + adapter->new_mtu = ETHER_MTU; + if (!is_vf) + if (qede_start_vport(adapter, adapter->mtu)) + return -1; - return rc; + DP_INFO(edev, "MAC address : %02x:%02x:%02x:%02x:%02x:%02x\n", + adapter->primary_mac.addr_bytes[0], + adapter->primary_mac.addr_bytes[1], + adapter->primary_mac.addr_bytes[2], + adapter->primary_mac.addr_bytes[3], + adapter->primary_mac.addr_bytes[4], + adapter->primary_mac.addr_bytes[5]); + + DP_INFO(edev, "Device initialized\n"); + + return 0; } static int qedevf_eth_dev_init(struct rte_eth_dev *eth_dev) @@ -1416,6 +2993,13 @@ static int qede_eth_dev_init(struct rte_eth_dev *eth_dev) static int qede_dev_common_uninit(struct rte_eth_dev *eth_dev) { +#ifdef RTE_LIBRTE_QEDE_DEBUG_INIT + struct qede_dev *qdev = eth_dev->data->dev_private; + struct ecore_dev *edev = &qdev->edev; + + PMD_INIT_FUNC_TRACE(edev); +#endif + /* only uninitialize in the primary process */ if (rte_eal_process_type() != RTE_PROC_PRIMARY) return 0; @@ -1445,90 +3029,96 @@ static int qedevf_eth_dev_uninit(struct rte_eth_dev *eth_dev) return qede_dev_common_uninit(eth_dev); } -static struct rte_pci_id pci_id_qedevf_map[] = { +static const struct rte_pci_id pci_id_qedevf_map[] = { #define QEDEVF_RTE_PCI_DEVICE(dev) RTE_PCI_DEVICE(PCI_VENDOR_ID_QLOGIC, dev) { - QEDEVF_RTE_PCI_DEVICE(PCI_DEVICE_ID_NX2_VF) + QEDEVF_RTE_PCI_DEVICE(PCI_DEVICE_ID_QLOGIC_NX2_VF) }, { - QEDEVF_RTE_PCI_DEVICE(PCI_DEVICE_ID_57980S_IOV) + QEDEVF_RTE_PCI_DEVICE(PCI_DEVICE_ID_QLOGIC_57980S_IOV) + }, + { + QEDEVF_RTE_PCI_DEVICE(PCI_DEVICE_ID_QLOGIC_AH_IOV) }, {.vendor_id = 0,} }; -static struct rte_pci_id pci_id_qede_map[] = { +static const struct rte_pci_id pci_id_qede_map[] = { #define QEDE_RTE_PCI_DEVICE(dev) RTE_PCI_DEVICE(PCI_VENDOR_ID_QLOGIC, dev) { - QEDE_RTE_PCI_DEVICE(PCI_DEVICE_ID_NX2_57980E) + QEDE_RTE_PCI_DEVICE(PCI_DEVICE_ID_QLOGIC_NX2_57980E) }, { - QEDE_RTE_PCI_DEVICE(PCI_DEVICE_ID_NX2_57980S) + QEDE_RTE_PCI_DEVICE(PCI_DEVICE_ID_QLOGIC_NX2_57980S) }, { - QEDE_RTE_PCI_DEVICE(PCI_DEVICE_ID_57980S_40) + QEDE_RTE_PCI_DEVICE(PCI_DEVICE_ID_QLOGIC_57980S_40) }, { - QEDE_RTE_PCI_DEVICE(PCI_DEVICE_ID_57980S_25) + QEDE_RTE_PCI_DEVICE(PCI_DEVICE_ID_QLOGIC_57980S_25) }, { - QEDE_RTE_PCI_DEVICE(PCI_DEVICE_ID_57980S_100) + QEDE_RTE_PCI_DEVICE(PCI_DEVICE_ID_QLOGIC_57980S_100) + }, + { + QEDE_RTE_PCI_DEVICE(PCI_DEVICE_ID_QLOGIC_57980S_50) + }, + { + QEDE_RTE_PCI_DEVICE(PCI_DEVICE_ID_QLOGIC_AH_50G) + }, + { + QEDE_RTE_PCI_DEVICE(PCI_DEVICE_ID_QLOGIC_AH_10G) + }, + { + QEDE_RTE_PCI_DEVICE(PCI_DEVICE_ID_QLOGIC_AH_40G) + }, + { + QEDE_RTE_PCI_DEVICE(PCI_DEVICE_ID_QLOGIC_AH_25G) }, {.vendor_id = 0,} }; -static struct eth_driver rte_qedevf_pmd = { - .pci_drv = { - .name = "rte_qedevf_pmd", - .id_table = pci_id_qedevf_map, - .drv_flags = - RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC, - }, - .eth_dev_init = qedevf_eth_dev_init, - .eth_dev_uninit = qedevf_eth_dev_uninit, - .dev_private_size = sizeof(struct qede_dev), -}; - -static struct eth_driver rte_qede_pmd = { - .pci_drv = { - .name = "rte_qede_pmd", - .id_table = pci_id_qede_map, - .drv_flags = - RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC, - }, - .eth_dev_init = qede_eth_dev_init, - .eth_dev_uninit = qede_eth_dev_uninit, - .dev_private_size = sizeof(struct qede_dev), -}; - -static int -rte_qedevf_pmd_init(const char *name __rte_unused, - const char *params __rte_unused) +static int qedevf_eth_dev_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, + struct rte_pci_device *pci_dev) { - rte_eth_driver_register(&rte_qedevf_pmd); - - return 0; + return rte_eth_dev_pci_generic_probe(pci_dev, + sizeof(struct qede_dev), qedevf_eth_dev_init); } -static int -rte_qede_pmd_init(const char *name __rte_unused, - const char *params __rte_unused) +static int qedevf_eth_dev_pci_remove(struct rte_pci_device *pci_dev) { - rte_eth_driver_register(&rte_qede_pmd); - - return 0; + return rte_eth_dev_pci_generic_remove(pci_dev, qedevf_eth_dev_uninit); } -static struct rte_driver rte_qedevf_driver = { - .type = PMD_PDEV, - .init = rte_qede_pmd_init +static struct rte_pci_driver rte_qedevf_pmd = { + .id_table = pci_id_qedevf_map, + .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC, + .probe = qedevf_eth_dev_pci_probe, + .remove = qedevf_eth_dev_pci_remove, }; -static struct rte_driver rte_qede_driver = { - .type = PMD_PDEV, - .init = rte_qedevf_pmd_init +static int qede_eth_dev_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, + struct rte_pci_device *pci_dev) +{ + return rte_eth_dev_pci_generic_probe(pci_dev, + sizeof(struct qede_dev), qede_eth_dev_init); +} + +static int qede_eth_dev_pci_remove(struct rte_pci_device *pci_dev) +{ + return rte_eth_dev_pci_generic_remove(pci_dev, qede_eth_dev_uninit); +} + +static struct rte_pci_driver rte_qede_pmd = { + .id_table = pci_id_qede_map, + .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC, + .probe = qede_eth_dev_pci_probe, + .remove = qede_eth_dev_pci_remove, }; -PMD_REGISTER_DRIVER(rte_qede_driver, qede); -DRIVER_REGISTER_PCI_TABLE(qede, pci_id_qede_map); -PMD_REGISTER_DRIVER(rte_qedevf_driver, qedevf); -DRIVER_REGISTER_PCI_TABLE(qedevf, pci_id_qedevf_map); +RTE_PMD_REGISTER_PCI(net_qede, rte_qede_pmd); +RTE_PMD_REGISTER_PCI_TABLE(net_qede, pci_id_qede_map); +RTE_PMD_REGISTER_KMOD_DEP(net_qede, "* igb_uio | uio_pci_generic | vfio-pci"); +RTE_PMD_REGISTER_PCI(net_qede_vf, rte_qedevf_pmd); +RTE_PMD_REGISTER_PCI_TABLE(net_qede_vf, pci_id_qedevf_map); +RTE_PMD_REGISTER_KMOD_DEP(net_qede_vf, "* igb_uio | vfio-pci"); diff --git a/dpdk/drivers/net/qede/qede_ethdev.h b/dpdk/drivers/net/qede/qede_ethdev.h index abb33af1..8f21b338 100644 --- a/dpdk/drivers/net/qede/qede_ethdev.h +++ b/dpdk/drivers/net/qede/qede_ethdev.h @@ -10,9 +10,13 @@ #ifndef _QEDE_ETHDEV_H_ #define _QEDE_ETHDEV_H_ +#include + #include #include +#include #include +#include /* ecore includes */ #include "base/bcm_osal.h" @@ -25,13 +29,14 @@ #include "base/ecore_chain.h" #include "base/ecore_status.h" #include "base/ecore_hsi_eth.h" -#include "base/ecore_dev_api.h" #include "base/ecore_iov_api.h" +#include "base/ecore_cxt.h" +#include "base/nvm_cfg.h" +#include "base/ecore_sp_commands.h" +#include "base/ecore_l2.h" #include "qede_logs.h" #include "qede_if.h" -#include "qede_eth_if.h" - #include "qede_rxtx.h" #define qede_stringify1(x...) #x @@ -39,20 +44,19 @@ /* Driver versions */ #define QEDE_PMD_VER_PREFIX "QEDE PMD" -#define QEDE_PMD_VERSION_MAJOR 1 -#define QEDE_PMD_VERSION_MINOR 1 +#define QEDE_PMD_VERSION_MAJOR 2 +#define QEDE_PMD_VERSION_MINOR 6 #define QEDE_PMD_VERSION_REVISION 0 #define QEDE_PMD_VERSION_PATCH 1 -#define QEDE_MAJOR_VERSION 8 -#define QEDE_MINOR_VERSION 7 -#define QEDE_REVISION_VERSION 9 -#define QEDE_ENGINEERING_VERSION 0 +#define QEDE_PMD_VERSION qede_stringify(QEDE_PMD_VERSION_MAJOR) "." \ + qede_stringify(QEDE_PMD_VERSION_MINOR) "." \ + qede_stringify(QEDE_PMD_VERSION_REVISION) "." \ + qede_stringify(QEDE_PMD_VERSION_PATCH) + +#define QEDE_PMD_DRV_VER_STR_SIZE NAME_SIZE +#define QEDE_PMD_VER_PREFIX "QEDE PMD" -#define QEDE_DRV_MODULE_VERSION qede_stringify(QEDE_MAJOR_VERSION) "." \ - qede_stringify(QEDE_MINOR_VERSION) "." \ - qede_stringify(QEDE_REVISION_VERSION) "." \ - qede_stringify(QEDE_ENGINEERING_VERSION) #define QEDE_RSS_INDIR_INITED (1 << 0) #define QEDE_RSS_KEY_INITED (1 << 1) @@ -62,8 +66,9 @@ #define QEDE_MAX_TSS_CNT(edev) ((edev)->dev_info.num_queues * \ (edev)->dev_info.num_tc) -#define QEDE_RSS_CNT(edev) ((edev)->num_rss) -#define QEDE_TSS_CNT(edev) ((edev)->num_rss * (edev)->num_tc) +#define QEDE_QUEUE_CNT(qdev) ((qdev)->num_queues) +#define QEDE_RSS_COUNT(qdev) ((qdev)->num_rx_queues) +#define QEDE_TSS_COUNT(qdev) ((qdev)->num_tx_queues) #define QEDE_DUPLEX_FULL 1 #define QEDE_DUPLEX_HALF 2 @@ -81,44 +86,92 @@ struct ecore_dev *edev = &qdev->edev; \ } -/************* QLogic 25G/40G/100G vendor/devices ids *************/ -#define PCI_VENDOR_ID_QLOGIC 0x1077 +/************* QLogic 10G/25G/40G/50G/100G vendor/devices ids *************/ +#define PCI_VENDOR_ID_QLOGIC 0x1077 + +#define CHIP_NUM_57980E 0x1634 +#define CHIP_NUM_57980S 0x1629 +#define CHIP_NUM_VF 0x1630 +#define CHIP_NUM_57980S_40 0x1634 +#define CHIP_NUM_57980S_25 0x1656 +#define CHIP_NUM_57980S_IOV 0x1664 +#define CHIP_NUM_57980S_100 0x1644 +#define CHIP_NUM_57980S_50 0x1654 +#define CHIP_NUM_AH_50G 0x8070 +#define CHIP_NUM_AH_10G 0x8071 +#define CHIP_NUM_AH_40G 0x8072 +#define CHIP_NUM_AH_25G 0x8073 +#define CHIP_NUM_AH_IOV 0x8090 + +#define PCI_DEVICE_ID_QLOGIC_NX2_57980E CHIP_NUM_57980E +#define PCI_DEVICE_ID_QLOGIC_NX2_57980S CHIP_NUM_57980S +#define PCI_DEVICE_ID_QLOGIC_NX2_VF CHIP_NUM_VF +#define PCI_DEVICE_ID_QLOGIC_57980S_40 CHIP_NUM_57980S_40 +#define PCI_DEVICE_ID_QLOGIC_57980S_25 CHIP_NUM_57980S_25 +#define PCI_DEVICE_ID_QLOGIC_57980S_IOV CHIP_NUM_57980S_IOV +#define PCI_DEVICE_ID_QLOGIC_57980S_100 CHIP_NUM_57980S_100 +#define PCI_DEVICE_ID_QLOGIC_57980S_50 CHIP_NUM_57980S_50 +#define PCI_DEVICE_ID_QLOGIC_AH_50G CHIP_NUM_AH_50G +#define PCI_DEVICE_ID_QLOGIC_AH_10G CHIP_NUM_AH_10G +#define PCI_DEVICE_ID_QLOGIC_AH_40G CHIP_NUM_AH_40G +#define PCI_DEVICE_ID_QLOGIC_AH_25G CHIP_NUM_AH_25G +#define PCI_DEVICE_ID_QLOGIC_AH_IOV CHIP_NUM_AH_IOV -#define CHIP_NUM_57980E 0x1634 -#define CHIP_NUM_57980S 0x1629 -#define CHIP_NUM_VF 0x1630 -#define CHIP_NUM_57980S_40 0x1634 -#define CHIP_NUM_57980S_25 0x1656 -#define CHIP_NUM_57980S_IOV 0x1664 -#define CHIP_NUM_57980S_100 0x1644 -#define PCI_DEVICE_ID_NX2_57980E CHIP_NUM_57980E -#define PCI_DEVICE_ID_NX2_57980S CHIP_NUM_57980S -#define PCI_DEVICE_ID_NX2_VF CHIP_NUM_VF -#define PCI_DEVICE_ID_57980S_40 CHIP_NUM_57980S_40 -#define PCI_DEVICE_ID_57980S_25 CHIP_NUM_57980S_25 -#define PCI_DEVICE_ID_57980S_IOV CHIP_NUM_57980S_IOV -#define PCI_DEVICE_ID_57980S_100 CHIP_NUM_57980S_100 extern char fw_file[]; -/* Port/function states */ -enum dev_state { - QEDE_START, - QEDE_STOP, - QEDE_CLOSE +/* Number of PF connections - 32 RX + 32 TX */ +#define QEDE_PF_NUM_CONNS (64) + +/* Maximum number of flowdir filters */ +#define QEDE_RFS_MAX_FLTR (256) + +#define QEDE_MAX_MCAST_FILTERS (64) + +enum qed_filter_rx_mode_type { + QED_FILTER_RX_MODE_TYPE_REGULAR, + QED_FILTER_RX_MODE_TYPE_MULTI_PROMISC, + QED_FILTER_RX_MODE_TYPE_PROMISC, }; -struct qed_int_param { - uint32_t int_mode; - uint8_t num_vectors; - uint8_t min_msix_cnt; +struct qede_vlan_entry { + SLIST_ENTRY(qede_vlan_entry) list; + uint16_t vid; }; -struct qed_int_params { - struct qed_int_param in; - struct qed_int_param out; - bool fp_initialized; +struct qede_mcast_entry { + struct ether_addr mac; + SLIST_ENTRY(qede_mcast_entry) list; +}; + +struct qede_ucast_entry { + struct ether_addr mac; + uint16_t vlan; + uint16_t vni; + SLIST_ENTRY(qede_ucast_entry) list; +}; + +struct qede_fdir_entry { + uint32_t soft_id; /* unused for now */ + uint16_t pkt_len; /* actual packet length to match */ + uint16_t rx_queue; /* queue to be steered to */ + const struct rte_memzone *mz; /* mz used to hold L2 frame */ + SLIST_ENTRY(qede_fdir_entry) list; +}; + +struct qede_fdir_info { + struct ecore_arfs_config_params arfs; + uint16_t filter_count; + SLIST_HEAD(fdir_list_head, qede_fdir_entry)fdir_list_head; +}; + +struct qede_vxlan_tunn { + bool enable; + uint16_t num_filters; + uint16_t filter_type; +#define QEDE_VXLAN_DEF_PORT (4789) + uint16_t udp_port; }; /* @@ -126,37 +179,76 @@ struct qed_int_params { */ struct qede_dev { struct ecore_dev edev; - uint8_t protocol; const struct qed_eth_ops *ops; struct qed_dev_eth_info dev_info; struct ecore_sb_info *sb_array; struct qede_fastpath *fp_array; - uint16_t num_rss; - uint8_t num_tc; uint16_t mtu; - bool rss_enabled; - struct qed_update_vport_rss_params rss_params; - uint32_t flags; - bool gro_disable; - struct qede_rx_queue **rx_queues; - struct qede_tx_queue **tx_queues; - enum dev_state state; - - /* Vlans */ - osal_list_t vlan_list; + uint16_t new_mtu; + bool enable_tx_switching; + bool rss_enable; + struct rte_eth_rss_conf rss_conf; + uint16_t rss_ind_table[ECORE_RSS_IND_TABLE_SIZE]; + uint64_t rss_hf; + uint8_t rss_key_len; + bool enable_lro; + uint8_t num_rx_queues; + uint8_t num_tx_queues; + SLIST_HEAD(vlan_list_head, qede_vlan_entry)vlan_list_head; uint16_t configured_vlans; - uint16_t non_configured_vlans; bool accept_any_vlan; - uint16_t vxlan_dst_port; - struct ether_addr primary_mac; + SLIST_HEAD(mc_list_head, qede_mcast_entry) mc_list_head; + uint16_t num_mc_addr; + SLIST_HEAD(uc_list_head, qede_ucast_entry) uc_list_head; + uint16_t num_uc_addr; bool handle_hw_err; - char drv_ver[QED_DRV_VER_STR_SIZE]; + struct qede_vxlan_tunn vxlan; + struct qede_fdir_info fdir_info; + bool vlan_strip_flg; + char drv_ver[QEDE_PMD_DRV_VER_STR_SIZE]; + void *ethdev; }; +/* Non-static functions */ +int qede_config_rss(struct rte_eth_dev *eth_dev); + +int qede_rss_hash_update(struct rte_eth_dev *eth_dev, + struct rte_eth_rss_conf *rss_conf); + +int qede_rss_reta_update(struct rte_eth_dev *eth_dev, + struct rte_eth_rss_reta_entry64 *reta_conf, + uint16_t reta_size); + int qed_fill_eth_dev_info(struct ecore_dev *edev, struct qed_dev_eth_info *info); int qede_dev_set_link_state(struct rte_eth_dev *eth_dev, bool link_up); -void qede_config_rx_mode(struct rte_eth_dev *eth_dev); + +int qede_link_update(struct rte_eth_dev *eth_dev, + __rte_unused int wait_to_complete); + +int qede_dev_filter_ctrl(struct rte_eth_dev *dev, enum rte_filter_type type, + enum rte_filter_op op, void *arg); + +int qede_fdir_filter_conf(struct rte_eth_dev *eth_dev, + enum rte_filter_op filter_op, void *arg); + +int qede_ntuple_filter_conf(struct rte_eth_dev *eth_dev, + enum rte_filter_op filter_op, void *arg); + +int qede_check_fdir_support(struct rte_eth_dev *eth_dev); + +uint16_t qede_fdir_construct_pkt(struct rte_eth_dev *eth_dev, + struct rte_eth_fdir_filter *fdir, + void *buff, + struct ecore_arfs_config_params *params); + +void qede_fdir_dealloc_resc(struct rte_eth_dev *eth_dev); + +int qede_activate_vport(struct rte_eth_dev *eth_dev, bool flg); + +int qede_update_mtu(struct rte_eth_dev *eth_dev, uint16_t mtu); + +int qede_enable_tpa(struct rte_eth_dev *eth_dev, bool flg); #endif /* _QEDE_ETHDEV_H_ */ diff --git a/dpdk/drivers/net/qede/qede_fdir.c b/dpdk/drivers/net/qede/qede_fdir.c new file mode 100644 index 00000000..da6364ee --- /dev/null +++ b/dpdk/drivers/net/qede/qede_fdir.c @@ -0,0 +1,469 @@ +/* + * Copyright (c) 2017 QLogic Corporation. + * All rights reserved. + * www.qlogic.com + * + * See LICENSE.qede_pmd for copyright and licensing details. + */ + +#include +#include +#include +#include + +#include "qede_ethdev.h" + +#define IP_VERSION (0x40) +#define IP_HDRLEN (0x5) +#define QEDE_FDIR_IP_DEFAULT_VERSION_IHL (IP_VERSION | IP_HDRLEN) +#define QEDE_FDIR_TCP_DEFAULT_DATAOFF (0x50) +#define QEDE_FDIR_IPV4_DEF_TTL (64) + +/* Sum of length of header types of L2, L3, L4. + * L2 : ether_hdr + vlan_hdr + vxlan_hdr + * L3 : ipv6_hdr + * L4 : tcp_hdr + */ +#define QEDE_MAX_FDIR_PKT_LEN (86) + +#ifndef IPV6_ADDR_LEN +#define IPV6_ADDR_LEN (16) +#endif + +#define QEDE_VALID_FLOW(flow_type) \ + ((flow_type) == RTE_ETH_FLOW_NONFRAG_IPV4_TCP || \ + (flow_type) == RTE_ETH_FLOW_NONFRAG_IPV4_UDP || \ + (flow_type) == RTE_ETH_FLOW_NONFRAG_IPV6_TCP || \ + (flow_type) == RTE_ETH_FLOW_NONFRAG_IPV6_UDP) + +/* Note: Flowdir support is only partial. + * For ex: drop_queue, FDIR masks, flex_conf are not supported. + * Parameters like pballoc/status fields are irrelevant here. + */ +int qede_check_fdir_support(struct rte_eth_dev *eth_dev) +{ + struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); + struct rte_fdir_conf *fdir = ð_dev->data->dev_conf.fdir_conf; + + /* check FDIR modes */ + switch (fdir->mode) { + case RTE_FDIR_MODE_NONE: + qdev->fdir_info.arfs.arfs_enable = false; + DP_INFO(edev, "flowdir is disabled\n"); + break; + case RTE_FDIR_MODE_PERFECT: + if (ECORE_IS_CMT(edev)) { + DP_ERR(edev, "flowdir is not supported in 100G mode\n"); + qdev->fdir_info.arfs.arfs_enable = false; + return -ENOTSUP; + } + qdev->fdir_info.arfs.arfs_enable = true; + DP_INFO(edev, "flowdir is enabled\n"); + break; + case RTE_FDIR_MODE_PERFECT_TUNNEL: + case RTE_FDIR_MODE_SIGNATURE: + case RTE_FDIR_MODE_PERFECT_MAC_VLAN: + DP_ERR(edev, "Unsupported flowdir mode %d\n", fdir->mode); + return -ENOTSUP; + } + + return 0; +} + +void qede_fdir_dealloc_resc(struct rte_eth_dev *eth_dev) +{ + struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); + struct qede_fdir_entry *tmp = NULL; + + SLIST_FOREACH(tmp, &qdev->fdir_info.fdir_list_head, list) { + if (tmp) { + if (tmp->mz) + rte_memzone_free(tmp->mz); + SLIST_REMOVE(&qdev->fdir_info.fdir_list_head, tmp, + qede_fdir_entry, list); + rte_free(tmp); + } + } +} + +static int +qede_config_cmn_fdir_filter(struct rte_eth_dev *eth_dev, + struct rte_eth_fdir_filter *fdir_filter, + bool add) +{ + struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); + char mz_name[RTE_MEMZONE_NAMESIZE] = {0}; + struct qede_fdir_entry *tmp = NULL; + struct qede_fdir_entry *fdir = NULL; + const struct rte_memzone *mz; + struct ecore_hwfn *p_hwfn; + enum _ecore_status_t rc; + uint16_t pkt_len; + void *pkt; + + if (add) { + if (qdev->fdir_info.filter_count == QEDE_RFS_MAX_FLTR - 1) { + DP_ERR(edev, "Reached max flowdir filter limit\n"); + return -EINVAL; + } + fdir = rte_malloc(NULL, sizeof(struct qede_fdir_entry), + RTE_CACHE_LINE_SIZE); + if (!fdir) { + DP_ERR(edev, "Did not allocate memory for fdir\n"); + return -ENOMEM; + } + } + /* soft_id could have been used as memzone string, but soft_id is + * not currently used so it has no significance. + */ + snprintf(mz_name, sizeof(mz_name) - 1, "%lx", + (unsigned long)rte_get_timer_cycles()); + mz = rte_memzone_reserve_aligned(mz_name, QEDE_MAX_FDIR_PKT_LEN, + SOCKET_ID_ANY, 0, RTE_CACHE_LINE_SIZE); + if (!mz) { + DP_ERR(edev, "Failed to allocate memzone for fdir, err = %s\n", + rte_strerror(rte_errno)); + rc = -rte_errno; + goto err1; + } + + pkt = mz->addr; + memset(pkt, 0, QEDE_MAX_FDIR_PKT_LEN); + pkt_len = qede_fdir_construct_pkt(eth_dev, fdir_filter, pkt, + &qdev->fdir_info.arfs); + if (pkt_len == 0) { + rc = -EINVAL; + goto err2; + } + DP_INFO(edev, "pkt_len = %u memzone = %s\n", pkt_len, mz_name); + if (add) { + SLIST_FOREACH(tmp, &qdev->fdir_info.fdir_list_head, list) { + if (memcmp(tmp->mz->addr, pkt, pkt_len) == 0) { + DP_ERR(edev, "flowdir filter exist\n"); + rc = -EEXIST; + goto err2; + } + } + } else { + SLIST_FOREACH(tmp, &qdev->fdir_info.fdir_list_head, list) { + if (memcmp(tmp->mz->addr, pkt, pkt_len) == 0) + break; + } + if (!tmp) { + DP_ERR(edev, "flowdir filter does not exist\n"); + rc = -EEXIST; + goto err2; + } + } + p_hwfn = ECORE_LEADING_HWFN(edev); + if (add) { + if (!qdev->fdir_info.arfs.arfs_enable) { + /* Force update */ + eth_dev->data->dev_conf.fdir_conf.mode = + RTE_FDIR_MODE_PERFECT; + qdev->fdir_info.arfs.arfs_enable = true; + DP_INFO(edev, "Force enable flowdir in perfect mode\n"); + } + /* Enable ARFS searcher with updated flow_types */ + ecore_arfs_mode_configure(p_hwfn, p_hwfn->p_arfs_ptt, + &qdev->fdir_info.arfs); + } + /* configure filter with ECORE_SPQ_MODE_EBLOCK */ + rc = ecore_configure_rfs_ntuple_filter(p_hwfn, NULL, + (dma_addr_t)mz->iova, + pkt_len, + fdir_filter->action.rx_queue, + 0, add); + if (rc == ECORE_SUCCESS) { + if (add) { + fdir->rx_queue = fdir_filter->action.rx_queue; + fdir->pkt_len = pkt_len; + fdir->mz = mz; + SLIST_INSERT_HEAD(&qdev->fdir_info.fdir_list_head, + fdir, list); + qdev->fdir_info.filter_count++; + DP_INFO(edev, "flowdir filter added, count = %d\n", + qdev->fdir_info.filter_count); + } else { + rte_memzone_free(tmp->mz); + SLIST_REMOVE(&qdev->fdir_info.fdir_list_head, tmp, + qede_fdir_entry, list); + rte_free(tmp); /* the node deleted */ + rte_memzone_free(mz); /* temp node allocated */ + qdev->fdir_info.filter_count--; + DP_INFO(edev, "Fdir filter deleted, count = %d\n", + qdev->fdir_info.filter_count); + } + } else { + DP_ERR(edev, "flowdir filter failed, rc=%d filter_count=%d\n", + rc, qdev->fdir_info.filter_count); + } + + /* Disable ARFS searcher if there are no more filters */ + if (qdev->fdir_info.filter_count == 0) { + memset(&qdev->fdir_info.arfs, 0, + sizeof(struct ecore_arfs_config_params)); + DP_INFO(edev, "Disabling flowdir\n"); + qdev->fdir_info.arfs.arfs_enable = false; + ecore_arfs_mode_configure(p_hwfn, p_hwfn->p_arfs_ptt, + &qdev->fdir_info.arfs); + } + return 0; + +err2: + rte_memzone_free(mz); +err1: + if (add) + rte_free(fdir); + return rc; +} + +static int +qede_fdir_filter_add(struct rte_eth_dev *eth_dev, + struct rte_eth_fdir_filter *fdir, + bool add) +{ + struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); + + if (!QEDE_VALID_FLOW(fdir->input.flow_type)) { + DP_ERR(edev, "invalid flow_type input\n"); + return -EINVAL; + } + + if (fdir->action.rx_queue >= QEDE_RSS_COUNT(qdev)) { + DP_ERR(edev, "invalid queue number %u\n", + fdir->action.rx_queue); + return -EINVAL; + } + + if (fdir->input.flow_ext.is_vf) { + DP_ERR(edev, "flowdir is not supported over VF\n"); + return -EINVAL; + } + + return qede_config_cmn_fdir_filter(eth_dev, fdir, add); +} + +/* Fills the L3/L4 headers and returns the actual length of flowdir packet */ +uint16_t +qede_fdir_construct_pkt(struct rte_eth_dev *eth_dev, + struct rte_eth_fdir_filter *fdir, + void *buff, + struct ecore_arfs_config_params *params) + +{ + struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); + uint16_t *ether_type; + uint8_t *raw_pkt; + struct rte_eth_fdir_input *input; + static uint8_t vlan_frame[] = {0x81, 0, 0, 0}; + struct ipv4_hdr *ip; + struct ipv6_hdr *ip6; + struct udp_hdr *udp; + struct tcp_hdr *tcp; + uint16_t len; + static const uint8_t next_proto[] = { + [RTE_ETH_FLOW_NONFRAG_IPV4_TCP] = IPPROTO_TCP, + [RTE_ETH_FLOW_NONFRAG_IPV4_UDP] = IPPROTO_UDP, + [RTE_ETH_FLOW_NONFRAG_IPV6_TCP] = IPPROTO_TCP, + [RTE_ETH_FLOW_NONFRAG_IPV6_UDP] = IPPROTO_UDP, + }; + raw_pkt = (uint8_t *)buff; + input = &fdir->input; + DP_INFO(edev, "flow_type %d\n", input->flow_type); + + len = 2 * sizeof(struct ether_addr); + raw_pkt += 2 * sizeof(struct ether_addr); + if (input->flow_ext.vlan_tci) { + DP_INFO(edev, "adding VLAN header\n"); + rte_memcpy(raw_pkt, vlan_frame, sizeof(vlan_frame)); + rte_memcpy(raw_pkt + sizeof(uint16_t), + &input->flow_ext.vlan_tci, + sizeof(uint16_t)); + raw_pkt += sizeof(vlan_frame); + len += sizeof(vlan_frame); + } + ether_type = (uint16_t *)raw_pkt; + raw_pkt += sizeof(uint16_t); + len += sizeof(uint16_t); + + switch (input->flow_type) { + case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: + case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: + /* fill the common ip header */ + ip = (struct ipv4_hdr *)raw_pkt; + *ether_type = rte_cpu_to_be_16(ETHER_TYPE_IPv4); + ip->version_ihl = QEDE_FDIR_IP_DEFAULT_VERSION_IHL; + ip->total_length = sizeof(struct ipv4_hdr); + ip->next_proto_id = input->flow.ip4_flow.proto ? + input->flow.ip4_flow.proto : + next_proto[input->flow_type]; + ip->time_to_live = input->flow.ip4_flow.ttl ? + input->flow.ip4_flow.ttl : + QEDE_FDIR_IPV4_DEF_TTL; + ip->type_of_service = input->flow.ip4_flow.tos; + ip->dst_addr = input->flow.ip4_flow.dst_ip; + ip->src_addr = input->flow.ip4_flow.src_ip; + len += sizeof(struct ipv4_hdr); + params->ipv4 = true; + + raw_pkt = (uint8_t *)buff; + /* UDP */ + if (input->flow_type == RTE_ETH_FLOW_NONFRAG_IPV4_UDP) { + udp = (struct udp_hdr *)(raw_pkt + len); + udp->dst_port = input->flow.udp4_flow.dst_port; + udp->src_port = input->flow.udp4_flow.src_port; + udp->dgram_len = sizeof(struct udp_hdr); + len += sizeof(struct udp_hdr); + /* adjust ip total_length */ + ip->total_length += sizeof(struct udp_hdr); + params->udp = true; + } else { /* TCP */ + tcp = (struct tcp_hdr *)(raw_pkt + len); + tcp->src_port = input->flow.tcp4_flow.src_port; + tcp->dst_port = input->flow.tcp4_flow.dst_port; + tcp->data_off = QEDE_FDIR_TCP_DEFAULT_DATAOFF; + len += sizeof(struct tcp_hdr); + /* adjust ip total_length */ + ip->total_length += sizeof(struct tcp_hdr); + params->tcp = true; + } + break; + case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: + case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: + ip6 = (struct ipv6_hdr *)raw_pkt; + *ether_type = rte_cpu_to_be_16(ETHER_TYPE_IPv6); + ip6->proto = input->flow.ipv6_flow.proto ? + input->flow.ipv6_flow.proto : + next_proto[input->flow_type]; + rte_memcpy(&ip6->src_addr, &input->flow.ipv6_flow.dst_ip, + IPV6_ADDR_LEN); + rte_memcpy(&ip6->dst_addr, &input->flow.ipv6_flow.src_ip, + IPV6_ADDR_LEN); + len += sizeof(struct ipv6_hdr); + + raw_pkt = (uint8_t *)buff; + /* UDP */ + if (input->flow_type == RTE_ETH_FLOW_NONFRAG_IPV6_UDP) { + udp = (struct udp_hdr *)(raw_pkt + len); + udp->src_port = input->flow.udp6_flow.dst_port; + udp->dst_port = input->flow.udp6_flow.src_port; + len += sizeof(struct udp_hdr); + params->udp = true; + } else { /* TCP */ + tcp = (struct tcp_hdr *)(raw_pkt + len); + tcp->src_port = input->flow.tcp4_flow.src_port; + tcp->dst_port = input->flow.tcp4_flow.dst_port; + tcp->data_off = QEDE_FDIR_TCP_DEFAULT_DATAOFF; + len += sizeof(struct tcp_hdr); + params->tcp = true; + } + break; + default: + DP_ERR(edev, "Unsupported flow_type %u\n", + input->flow_type); + return 0; + } + + return len; +} + +int +qede_fdir_filter_conf(struct rte_eth_dev *eth_dev, + enum rte_filter_op filter_op, + void *arg) +{ + struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); + struct rte_eth_fdir_filter *fdir; + int ret; + + fdir = (struct rte_eth_fdir_filter *)arg; + switch (filter_op) { + case RTE_ETH_FILTER_NOP: + /* Typically used to query flowdir support */ + if (ECORE_IS_CMT(edev)) { + DP_ERR(edev, "flowdir is not supported in 100G mode\n"); + return -ENOTSUP; + } + return 0; /* means supported */ + case RTE_ETH_FILTER_ADD: + ret = qede_fdir_filter_add(eth_dev, fdir, 1); + break; + case RTE_ETH_FILTER_DELETE: + ret = qede_fdir_filter_add(eth_dev, fdir, 0); + break; + case RTE_ETH_FILTER_FLUSH: + case RTE_ETH_FILTER_UPDATE: + case RTE_ETH_FILTER_INFO: + return -ENOTSUP; + break; + default: + DP_ERR(edev, "unknown operation %u", filter_op); + ret = -EINVAL; + } + + return ret; +} + +int qede_ntuple_filter_conf(struct rte_eth_dev *eth_dev, + enum rte_filter_op filter_op, + void *arg) +{ + struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); + struct rte_eth_ntuple_filter *ntuple; + struct rte_eth_fdir_filter fdir_entry; + struct rte_eth_tcpv4_flow *tcpv4_flow; + struct rte_eth_udpv4_flow *udpv4_flow; + bool add = false; + + switch (filter_op) { + case RTE_ETH_FILTER_NOP: + /* Typically used to query fdir support */ + if (ECORE_IS_CMT(edev)) { + DP_ERR(edev, "flowdir is not supported in 100G mode\n"); + return -ENOTSUP; + } + return 0; /* means supported */ + case RTE_ETH_FILTER_ADD: + add = true; + break; + case RTE_ETH_FILTER_DELETE: + break; + case RTE_ETH_FILTER_INFO: + case RTE_ETH_FILTER_GET: + case RTE_ETH_FILTER_UPDATE: + case RTE_ETH_FILTER_FLUSH: + case RTE_ETH_FILTER_SET: + case RTE_ETH_FILTER_STATS: + case RTE_ETH_FILTER_OP_MAX: + DP_ERR(edev, "Unsupported filter_op %d\n", filter_op); + return -ENOTSUP; + } + ntuple = (struct rte_eth_ntuple_filter *)arg; + /* Internally convert ntuple to fdir entry */ + memset(&fdir_entry, 0, sizeof(fdir_entry)); + if (ntuple->proto == IPPROTO_TCP) { + fdir_entry.input.flow_type = RTE_ETH_FLOW_NONFRAG_IPV4_TCP; + tcpv4_flow = &fdir_entry.input.flow.tcp4_flow; + tcpv4_flow->ip.src_ip = ntuple->src_ip; + tcpv4_flow->ip.dst_ip = ntuple->dst_ip; + tcpv4_flow->ip.proto = IPPROTO_TCP; + tcpv4_flow->src_port = ntuple->src_port; + tcpv4_flow->dst_port = ntuple->dst_port; + } else { + fdir_entry.input.flow_type = RTE_ETH_FLOW_NONFRAG_IPV4_UDP; + udpv4_flow = &fdir_entry.input.flow.udp4_flow; + udpv4_flow->ip.src_ip = ntuple->src_ip; + udpv4_flow->ip.dst_ip = ntuple->dst_ip; + udpv4_flow->ip.proto = IPPROTO_TCP; + udpv4_flow->src_port = ntuple->src_port; + udpv4_flow->dst_port = ntuple->dst_port; + } + return qede_config_cmn_fdir_filter(eth_dev, &fdir_entry, add); +} diff --git a/dpdk/drivers/net/qede/qede_if.h b/dpdk/drivers/net/qede/qede_if.h index 1b05ff86..246f0fd3 100644 --- a/dpdk/drivers/net/qede/qede_if.h +++ b/dpdk/drivers/net/qede/qede_if.h @@ -30,21 +30,51 @@ struct qed_dev_info { /* MFW version */ uint32_t mfw_rev; +#define QED_MFW_VERSION_0_MASK 0x000000FF +#define QED_MFW_VERSION_0_OFFSET 0 +#define QED_MFW_VERSION_1_MASK 0x0000FF00 +#define QED_MFW_VERSION_1_OFFSET 8 +#define QED_MFW_VERSION_2_MASK 0x00FF0000 +#define QED_MFW_VERSION_2_OFFSET 16 +#define QED_MFW_VERSION_3_MASK 0xFF000000 +#define QED_MFW_VERSION_3_OFFSET 24 uint32_t flash_size; - uint8_t mf_mode; + bool b_arfs_capable; + bool b_inter_pf_switch; bool tx_switching; - /* To be added... */ + u16 mtu; + + bool smart_an; + + /* Out param for qede */ + bool vxlan_enable; + bool gre_enable; + bool geneve_enable; + + enum ecore_dev_type dev_type; }; -enum qed_sb_type { - QED_SB_TYPE_L2_QUEUE, - QED_SB_TYPE_STORAGE, - QED_SB_TYPE_CNQ, +struct qed_dev_eth_info { + struct qed_dev_info common; + + uint8_t num_queues; + uint8_t num_tc; + + struct ether_addr port_mac; + uint16_t num_vlan_filters; + uint32_t num_mac_filters; + + /* Legacy VF - this affects the datapath */ + bool is_legacy; }; -enum qed_protocol { - QED_PROTOCOL_ETH, +#define INIT_STRUCT_FIELD(field, value) .field = value + +struct qed_eth_ops { + const struct qed_common_ops *common; + int (*fill_dev_info)(struct ecore_dev *edev, + struct qed_dev_eth_info *info); }; struct qed_link_params { @@ -54,6 +84,7 @@ struct qed_link_params { #define QED_LINK_OVERRIDE_SPEED_ADV_SPEEDS (1 << 1) #define QED_LINK_OVERRIDE_SPEED_FORCED_SPEED (1 << 2) #define QED_LINK_OVERRIDE_PAUSE_CONFIG (1 << 3) +#define QED_LINK_OVERRIDE_EEE_CONFIG (1 << 5) uint32_t override_flags; bool autoneg; uint32_t adv_speeds; @@ -62,6 +93,7 @@ struct qed_link_params { #define QED_LINK_PAUSE_RX_ENABLE (1 << 1) #define QED_LINK_PAUSE_TX_ENABLE (1 << 2) uint32_t pause_config; + struct ecore_link_eee_params eee; }; struct qed_link_output { @@ -70,54 +102,47 @@ struct qed_link_output { uint32_t advertised_caps; /* In ADVERTISED defs */ uint32_t lp_caps; /* In ADVERTISED defs */ uint32_t speed; /* In Mb/s */ + uint32_t adv_speed; /* Speed mask */ uint8_t duplex; /* In DUPLEX defs */ - uint8_t port; /* In PORT defs */ + uint16_t port; /* In PORT defs */ bool autoneg; uint32_t pause_config; + + /* EEE - capability & param */ + bool eee_supported; + bool eee_active; + u8 sup_caps; + struct ecore_link_eee_params eee; }; -#define QED_DRV_VER_STR_SIZE 80 struct qed_slowpath_params { uint32_t int_mode; uint8_t drv_major; uint8_t drv_minor; uint8_t drv_rev; uint8_t drv_eng; - uint8_t name[QED_DRV_VER_STR_SIZE]; + uint8_t name[NAME_SIZE]; }; -#define ILT_PAGE_SIZE_TCFC 0x8000 /* 32KB */ - struct qed_common_cb_ops { void (*link_update)(void *dev, struct qed_link_output *link); }; -struct qed_selftest_ops { -/** - * @brief registers - Perform register tests - * - * @param edev - * - * @return 0 on success, error otherwise. - */ - int (*registers)(struct ecore_dev *edev); -}; - struct qed_common_ops { int (*probe)(struct ecore_dev *edev, struct rte_pci_device *pci_dev, - enum qed_protocol protocol, uint32_t dp_module, uint8_t dp_level, bool is_vf); - void (*set_id)(struct ecore_dev *edev, - char name[], const char ver_str[]); - enum _ecore_status_t (*chain_alloc)(struct ecore_dev *edev, - enum ecore_chain_use_mode - intended_use, - enum ecore_chain_mode mode, - enum ecore_chain_cnt_type cnt_type, - uint32_t num_elems, - osal_size_t elem_size, - struct ecore_chain *p_chain); + void (*set_name)(struct ecore_dev *edev, char name[]); + enum _ecore_status_t + (*chain_alloc)(struct ecore_dev *edev, + enum ecore_chain_use_mode + intended_use, + enum ecore_chain_mode mode, + enum ecore_chain_cnt_type cnt_type, + uint32_t num_elems, + osal_size_t elem_size, + struct ecore_chain *p_chain, + struct ecore_chain_ext_pbl *ext_pbl); void (*chain_free)(struct ecore_dev *edev, struct ecore_chain *p_chain); @@ -145,20 +170,22 @@ struct qed_common_ops { struct ecore_sb_info *sb_info, void *sb_virt_addr, dma_addr_t sb_phy_addr, - uint16_t sb_id, enum qed_sb_type type); + uint16_t sb_id); + + int (*get_sb_info)(struct ecore_dev *edev, + struct ecore_sb_info *sb, u16 qid, + struct ecore_sb_info_dbg *sb_dbg); bool (*can_link_change)(struct ecore_dev *edev); + void (*update_msglvl)(struct ecore_dev *edev, uint32_t dp_module, uint8_t dp_level); + + int (*send_drv_state)(struct ecore_dev *edev, bool active); }; -/** - * @brief qed_get_protocol_version - * - * @param protocol - * - * @return version supported by qed for given protocol driver - */ -uint32_t qed_get_protocol_version(enum qed_protocol protocol); +/* Externs */ + +const struct qed_eth_ops *qed_get_eth_ops(void); #endif /* _QEDE_IF_H */ diff --git a/dpdk/drivers/net/qede/qede_logs.h b/dpdk/drivers/net/qede/qede_logs.h index 45c4af09..15821151 100644 --- a/dpdk/drivers/net/qede/qede_logs.h +++ b/dpdk/drivers/net/qede/qede_logs.h @@ -17,14 +17,22 @@ ##__VA_ARGS__) #define DP_NOTICE(p_dev, is_assert, fmt, ...) \ - rte_log(RTE_LOG_NOTICE, RTE_LOGTYPE_PMD,\ - "[QEDE PMD: (%s)]%s:" fmt, \ - (p_dev)->name ? (p_dev)->name : "", \ - __func__, \ - ##__VA_ARGS__) +do { \ + if (is_assert) \ + rte_log(RTE_LOG_ERR, RTE_LOGTYPE_PMD,\ + "[QEDE PMD: (%s)]%s:" fmt, \ + (p_dev)->name ? (p_dev)->name : "", \ + __func__, \ + ##__VA_ARGS__); \ + else \ + rte_log(RTE_LOG_NOTICE, RTE_LOGTYPE_PMD,\ + "[QEDE PMD: (%s)]%s:" fmt, \ + (p_dev)->name ? (p_dev)->name : "", \ + __func__, \ + ##__VA_ARGS__); \ +} while (0) #ifdef RTE_LIBRTE_QEDE_DEBUG_INFO - #define DP_INFO(p_dev, fmt, ...) \ rte_log(RTE_LOG_INFO, RTE_LOGTYPE_PMD, \ "[%s:%d(%s)]" fmt, \ @@ -33,7 +41,6 @@ ##__VA_ARGS__) #else #define DP_INFO(p_dev, fmt, ...) do { } while (0) - #endif #ifdef RTE_LIBRTE_QEDE_DEBUG_DRIVER @@ -77,14 +84,4 @@ do { \ #define PMD_RX_LOG(level, q, fmt, args...) do { } while (0) #endif -#ifdef RTE_LIBRTE_QEDE_DEBUG_DRIVER -#define PMD_DRV_LOG_RAW(level, fmt, args...) \ - RTE_LOG(level, PMD, "%s(): " fmt, __func__, ## args) -#else -#define PMD_DRV_LOG_RAW(level, fmt, args...) do { } while (0) -#endif - -#define PMD_DRV_LOG(level, fmt, args...) \ - PMD_DRV_LOG_RAW(level, fmt "\n", ## args) - #endif /* _QEDE_LOGS_H_ */ diff --git a/dpdk/drivers/net/qede/qede_main.c b/dpdk/drivers/net/qede/qede_main.c index 73608c69..ae187321 100644 --- a/dpdk/drivers/net/qede/qede_main.c +++ b/dpdk/drivers/net/qede/qede_main.c @@ -6,26 +6,20 @@ * See LICENSE.qede_pmd for copyright and licensing details. */ -#include -#include -#include -#include #include +#include #include #include "qede_ethdev.h" -static uint8_t npar_tx_switching = 1; - /* Alarm timeout. */ #define QEDE_ALARM_TIMEOUT_US 100000 -#define CONFIG_QED_BINARY_FW /* Global variable to hold absolute path of fw file */ char fw_file[PATH_MAX]; const char *QEDE_DEFAULT_FIRMWARE = - "/lib/firmware/qed/qed_init_values_zipped-8.7.7.0.bin"; + "/lib/firmware/qed/qed_init_values-8.30.12.0.bin"; static void qed_update_pf_params(struct ecore_dev *edev, struct ecore_pf_params *params) @@ -42,25 +36,34 @@ static void qed_init_pci(struct ecore_dev *edev, struct rte_pci_device *pci_dev) { edev->regview = pci_dev->mem_resource[0].addr; edev->doorbells = pci_dev->mem_resource[2].addr; + edev->db_size = pci_dev->mem_resource[2].len; } static int qed_probe(struct ecore_dev *edev, struct rte_pci_device *pci_dev, - enum qed_protocol protocol, uint32_t dp_module, - uint8_t dp_level, bool is_vf) + uint32_t dp_module, uint8_t dp_level, bool is_vf) { - struct qede_dev *qdev = (struct qede_dev *)edev; + struct ecore_hw_prepare_params hw_prepare_params; int rc; ecore_init_struct(edev); - qdev->protocol = protocol; - if (is_vf) { + edev->drv_type = DRV_ID_DRV_TYPE_LINUX; + /* Protocol type is always fixed to PROTOCOL_ETH */ + + if (is_vf) edev->b_is_vf = true; - edev->sriov_info.b_hw_channel = true; - } + ecore_init_dp(edev, dp_module, dp_level, NULL); qed_init_pci(edev, pci_dev); - rc = ecore_hw_prepare(edev, ECORE_PCI_DEFAULT); + + memset(&hw_prepare_params, 0, sizeof(hw_prepare_params)); + hw_prepare_params.personality = ECORE_PCI_ETH; + hw_prepare_params.drv_resc_alloc = false; + hw_prepare_params.chk_reg_fifo = false; + hw_prepare_params.initiate_pf_flr = true; + hw_prepare_params.allow_mdump = false; + hw_prepare_params.epoch = (u32)time(NULL); + rc = ecore_hw_prepare(edev, &hw_prepare_params); if (rc) { DP_ERR(edev, "hw prepare failed\n"); return rc; @@ -71,7 +74,7 @@ qed_probe(struct ecore_dev *edev, struct rte_pci_device *pci_dev, static int qed_nic_setup(struct ecore_dev *edev) { - int rc, i; + int rc; rc = ecore_resc_alloc(edev); if (rc) @@ -83,6 +86,7 @@ static int qed_nic_setup(struct ecore_dev *edev) return rc; } +#ifdef CONFIG_ECORE_ZIPPED_FW static int qed_alloc_stream_mem(struct ecore_dev *edev) { int i; @@ -112,7 +116,9 @@ static void qed_free_stream_mem(struct ecore_dev *edev) OSAL_FREE(p_hwfn->p_dev, p_hwfn->stream); } } +#endif +#ifdef CONFIG_ECORE_BINARY_FW static int qed_load_firmware_data(struct ecore_dev *edev) { int fd; @@ -126,38 +132,42 @@ static int qed_load_firmware_data(struct ecore_dev *edev) fd = open(fw_file, O_RDONLY); if (fd < 0) { - DP_NOTICE(edev, false, "Can't open firmware file\n"); + DP_ERR(edev, "Can't open firmware file\n"); return -ENOENT; } if (fstat(fd, &st) < 0) { - DP_NOTICE(edev, false, "Can't stat firmware file\n"); + DP_ERR(edev, "Can't stat firmware file\n"); + close(fd); return -1; } edev->firmware = rte_zmalloc("qede_fw", st.st_size, RTE_CACHE_LINE_SIZE); if (!edev->firmware) { - DP_NOTICE(edev, false, "Can't allocate memory for firmware\n"); + DP_ERR(edev, "Can't allocate memory for firmware\n"); close(fd); return -ENOMEM; } if (read(fd, edev->firmware, st.st_size) != st.st_size) { - DP_NOTICE(edev, false, "Can't read firmware data\n"); + DP_ERR(edev, "Can't read firmware data\n"); close(fd); return -1; } edev->fw_len = st.st_size; if (edev->fw_len < 104) { - DP_NOTICE(edev, false, "Invalid fw size: %" PRIu64 "\n", + DP_ERR(edev, "Invalid fw size: %" PRIu64 "\n", edev->fw_len); + close(fd); return -EINVAL; } + close(fd); return 0; } +#endif static void qed_handle_bulletin_change(struct ecore_hwfn *hwfn) { @@ -212,26 +222,34 @@ static void qed_stop_iov_task(struct ecore_dev *edev) static int qed_slowpath_start(struct ecore_dev *edev, struct qed_slowpath_params *params) { - bool allow_npar_tx_switching; + struct ecore_drv_load_params drv_load_params; + struct ecore_hw_init_params hw_init_params; + struct ecore_mcp_drv_version drv_version; const uint8_t *data = NULL; struct ecore_hwfn *hwfn; - struct ecore_mcp_drv_version drv_version; - struct qede_dev *qdev = (struct qede_dev *)edev; + struct ecore_ptt *p_ptt; int rc; -#ifdef QED_ENC_SUPPORTED - struct ecore_tunn_start_params tunn_info; -#endif -#ifdef CONFIG_QED_BINARY_FW if (IS_PF(edev)) { +#ifdef CONFIG_ECORE_BINARY_FW rc = qed_load_firmware_data(edev); if (rc) { - DP_NOTICE(edev, true, - "Failed to find fw file %s\n", fw_file); + DP_ERR(edev, "Failed to find fw file %s\n", fw_file); goto err; } - } #endif + hwfn = ECORE_LEADING_HWFN(edev); + if (edev->num_hwfns == 1) { /* skip aRFS for 100G device */ + p_ptt = ecore_ptt_acquire(hwfn); + if (p_ptt) { + ECORE_LEADING_HWFN(edev)->p_arfs_ptt = p_ptt; + } else { + DP_ERR(edev, "Failed to acquire PTT for flowdir\n"); + rc = -ENOMEM; + goto err; + } + } + } rc = qed_nic_setup(edev); if (rc) @@ -240,41 +258,38 @@ static int qed_slowpath_start(struct ecore_dev *edev, /* set int_coalescing_mode */ edev->int_coalescing_mode = ECORE_COAL_MODE_ENABLE; - /* Should go with CONFIG_QED_BINARY_FW */ +#ifdef CONFIG_ECORE_ZIPPED_FW if (IS_PF(edev)) { /* Allocate stream for unzipping */ rc = qed_alloc_stream_mem(edev); if (rc) { - DP_NOTICE(edev, true, - "Failed to allocate stream memory\n"); - goto err2; + DP_ERR(edev, "Failed to allocate stream memory\n"); + goto err1; } } +#endif qed_start_iov_task(edev); - /* Start the slowpath */ -#ifdef CONFIG_QED_BINARY_FW +#ifdef CONFIG_ECORE_BINARY_FW if (IS_PF(edev)) - data = edev->firmware; + data = (const uint8_t *)edev->firmware + sizeof(u32); #endif - allow_npar_tx_switching = npar_tx_switching ? true : false; -#ifdef QED_ENC_SUPPORTED - memset(&tunn_info, 0, sizeof(tunn_info)); - tunn_info.tunn_mode |= 1 << QED_MODE_VXLAN_TUNN | - 1 << QED_MODE_L2GRE_TUNN | - 1 << QED_MODE_IPGRE_TUNN | - 1 << QED_MODE_L2GENEVE_TUNN | 1 << QED_MODE_IPGENEVE_TUNN; - tunn_info.tunn_clss_vxlan = QED_TUNN_CLSS_MAC_VLAN; - tunn_info.tunn_clss_l2gre = QED_TUNN_CLSS_MAC_VLAN; - tunn_info.tunn_clss_ipgre = QED_TUNN_CLSS_MAC_VLAN; - rc = ecore_hw_init(edev, &tunn_info, true, ECORE_INT_MODE_MSIX, - allow_npar_tx_switching, data); -#else - rc = ecore_hw_init(edev, NULL, true, ECORE_INT_MODE_MSIX, - allow_npar_tx_switching, data); -#endif + /* Start the slowpath */ + memset(&hw_init_params, 0, sizeof(hw_init_params)); + hw_init_params.b_hw_start = true; + hw_init_params.int_mode = ECORE_INT_MODE_MSIX; + hw_init_params.allow_npar_tx_switch = true; + hw_init_params.bin_fw_data = data; + + memset(&drv_load_params, 0, sizeof(drv_load_params)); + drv_load_params.mfw_timeout_val = ECORE_LOAD_REQ_LOCK_TO_DEFAULT; + drv_load_params.avoid_eng_reset = false; + drv_load_params.override_force_load = ECORE_OVERRIDE_FORCE_LOAD_ALWAYS; + hw_init_params.p_drv_load_params = &drv_load_params; + + rc = ecore_hw_init(edev, &hw_init_params); if (rc) { DP_ERR(edev, "ecore_hw_init failed\n"); goto err2; @@ -293,9 +308,8 @@ static int qed_slowpath_start(struct ecore_dev *edev, rc = ecore_mcp_send_drv_version(hwfn, hwfn->p_main_ptt, &drv_version); if (rc) { - DP_NOTICE(edev, true, - "Failed sending drv version command\n"); - return rc; + DP_ERR(edev, "Failed sending drv version command\n"); + goto err3; } } @@ -303,11 +317,17 @@ static int qed_slowpath_start(struct ecore_dev *edev, return 0; +err3: ecore_hw_stop(edev); err2: + qed_stop_iov_task(edev); +#ifdef CONFIG_ECORE_ZIPPED_FW + qed_free_stream_mem(edev); +err1: +#endif ecore_resc_free(edev); err: -#ifdef CONFIG_QED_BINARY_FW +#ifdef CONFIG_ECORE_BINARY_FW if (IS_PF(edev)) { if (edev->firmware) rte_free(edev->firmware); @@ -322,31 +342,51 @@ err: static int qed_fill_dev_info(struct ecore_dev *edev, struct qed_dev_info *dev_info) { + struct ecore_hwfn *p_hwfn = ECORE_LEADING_HWFN(edev); struct ecore_ptt *ptt = NULL; + struct ecore_tunnel_info *tun = &edev->tunnel; memset(dev_info, 0, sizeof(struct qed_dev_info)); + + if (tun->vxlan.tun_cls == ECORE_TUNN_CLSS_MAC_VLAN && + tun->vxlan.b_mode_enabled) + dev_info->vxlan_enable = true; + + if (tun->l2_gre.b_mode_enabled && tun->ip_gre.b_mode_enabled && + tun->l2_gre.tun_cls == ECORE_TUNN_CLSS_MAC_VLAN && + tun->ip_gre.tun_cls == ECORE_TUNN_CLSS_MAC_VLAN) + dev_info->gre_enable = true; + + if (tun->l2_geneve.b_mode_enabled && tun->ip_geneve.b_mode_enabled && + tun->l2_geneve.tun_cls == ECORE_TUNN_CLSS_MAC_VLAN && + tun->ip_geneve.tun_cls == ECORE_TUNN_CLSS_MAC_VLAN) + dev_info->geneve_enable = true; + dev_info->num_hwfns = edev->num_hwfns; dev_info->is_mf_default = IS_MF_DEFAULT(&edev->hwfns[0]); + dev_info->mtu = ECORE_LEADING_HWFN(edev)->hw_info.mtu; + dev_info->dev_type = edev->type; + rte_memcpy(&dev_info->hw_mac, &edev->hwfns[0].hw_info.hw_mac_addr, ETHER_ADDR_LEN); - if (IS_PF(edev)) { - dev_info->fw_major = FW_MAJOR_VERSION; - dev_info->fw_minor = FW_MINOR_VERSION; - dev_info->fw_rev = FW_REVISION_VERSION; - dev_info->fw_eng = FW_ENGINEERING_VERSION; - dev_info->mf_mode = edev->mf_mode; - dev_info->tx_switching = false; - } else { - ecore_vf_get_fw_version(&edev->hwfns[0], &dev_info->fw_major, - &dev_info->fw_minor, &dev_info->fw_rev, - &dev_info->fw_eng); - } + dev_info->fw_major = FW_MAJOR_VERSION; + dev_info->fw_minor = FW_MINOR_VERSION; + dev_info->fw_rev = FW_REVISION_VERSION; + dev_info->fw_eng = FW_ENGINEERING_VERSION; if (IS_PF(edev)) { + dev_info->b_inter_pf_switch = + OSAL_TEST_BIT(ECORE_MF_INTER_PF_SWITCH, &edev->mf_bits); + if (!OSAL_TEST_BIT(ECORE_MF_DISABLE_ARFS, &edev->mf_bits)) + dev_info->b_arfs_capable = true; + dev_info->tx_switching = false; + + dev_info->smart_an = ecore_mcp_is_smart_an_supported(p_hwfn); + ptt = ecore_ptt_acquire(ECORE_LEADING_HWFN(edev)); if (ptt) { - ecore_mcp_get_mfw_ver(edev, ptt, + ecore_mcp_get_mfw_ver(ECORE_LEADING_HWFN(edev), ptt, &dev_info->mfw_rev, NULL); ecore_mcp_get_flash_size(ECORE_LEADING_HWFN(edev), ptt, @@ -361,7 +401,8 @@ qed_fill_dev_info(struct ecore_dev *edev, struct qed_dev_info *dev_info) ecore_ptt_release(ECORE_LEADING_HWFN(edev), ptt); } } else { - ecore_mcp_get_mfw_ver(edev, ptt, &dev_info->mfw_rev, NULL); + ecore_mcp_get_mfw_ver(ECORE_LEADING_HWFN(edev), ptt, + &dev_info->mfw_rev, NULL); } return 0; @@ -370,7 +411,7 @@ qed_fill_dev_info(struct ecore_dev *edev, struct qed_dev_info *dev_info) int qed_fill_eth_dev_info(struct ecore_dev *edev, struct qed_dev_eth_info *info) { - struct qede_dev *qdev = (struct qede_dev *)edev; + uint8_t queues = 0; int i; memset(info, 0, sizeof(*info)); @@ -378,23 +419,36 @@ qed_fill_eth_dev_info(struct ecore_dev *edev, struct qed_dev_eth_info *info) info->num_tc = 1 /* @@@TBD aelior MULTI_COS */; if (IS_PF(edev)) { + int max_vf_vlan_filters = 0; + info->num_queues = 0; for_each_hwfn(edev, i) info->num_queues += FEAT_NUM(&edev->hwfns[i], ECORE_PF_L2_QUE); - info->num_vlan_filters = RESC_NUM(&edev->hwfns[0], ECORE_VLAN); + if (IS_ECORE_SRIOV(edev)) + max_vf_vlan_filters = edev->p_iov_info->total_vfs * + ECORE_ETH_VF_NUM_VLAN_FILTERS; + info->num_vlan_filters = RESC_NUM(&edev->hwfns[0], ECORE_VLAN) - + max_vf_vlan_filters; rte_memcpy(&info->port_mac, &edev->hwfns[0].hw_info.hw_mac_addr, ETHER_ADDR_LEN); } else { - ecore_vf_get_num_rxqs(&edev->hwfns[0], &info->num_queues); + ecore_vf_get_num_rxqs(ECORE_LEADING_HWFN(edev), + &info->num_queues); + if (ECORE_IS_CMT(edev)) { + ecore_vf_get_num_rxqs(&edev->hwfns[1], &queues); + info->num_queues += queues; + } ecore_vf_get_num_vlan_filters(&edev->hwfns[0], - &info->num_vlan_filters); + (u8 *)&info->num_vlan_filters); ecore_vf_get_port_mac(&edev->hwfns[0], (uint8_t *)&info->port_mac); + + info->is_legacy = ecore_vf_get_pre_fp_hsi(&edev->hwfns[0]); } qed_fill_dev_info(edev, &info->common); @@ -405,9 +459,7 @@ qed_fill_eth_dev_info(struct ecore_dev *edev, struct qed_dev_eth_info *info) return 0; } -static void -qed_set_id(struct ecore_dev *edev, char name[NAME_SIZE], - const char ver_str[VER_SIZE]) +static void qed_set_name(struct ecore_dev *edev, char name[NAME_SIZE]) { int i; @@ -415,29 +467,18 @@ qed_set_id(struct ecore_dev *edev, char name[NAME_SIZE], for_each_hwfn(edev, i) { snprintf(edev->hwfns[i].name, NAME_SIZE, "%s-%d", name, i); } - rte_memcpy(edev->ver_str, ver_str, VER_SIZE); - edev->drv_type = DRV_ID_DRV_TYPE_LINUX; } static uint32_t qed_sb_init(struct ecore_dev *edev, struct ecore_sb_info *sb_info, - void *sb_virt_addr, dma_addr_t sb_phy_addr, - uint16_t sb_id, enum qed_sb_type type) + void *sb_virt_addr, dma_addr_t sb_phy_addr, uint16_t sb_id) { struct ecore_hwfn *p_hwfn; int hwfn_index; uint16_t rel_sb_id; - uint8_t n_hwfns; + uint8_t n_hwfns = edev->num_hwfns; uint32_t rc; - /* RoCE uses single engine and CMT uses two engines. When using both - * we force only a single engine. Storage uses only engine 0 too. - */ - if (type == QED_SB_TYPE_L2_QUEUE) - n_hwfns = edev->num_hwfns; - else - n_hwfns = 1; - hwfn_index = sb_id % n_hwfns; p_hwfn = &edev->hwfns[hwfn_index]; rel_sb_id = sb_id / n_hwfns; @@ -452,12 +493,12 @@ qed_sb_init(struct ecore_dev *edev, struct ecore_sb_info *sb_info, } static void qed_fill_link(struct ecore_hwfn *hwfn, + __rte_unused struct ecore_ptt *ptt, struct qed_link_output *if_link) { struct ecore_mcp_link_params params; struct ecore_mcp_link_state link; struct ecore_mcp_link_capabilities link_caps; - uint32_t media_type; uint8_t change = 0; memset(if_link, 0, sizeof(*if_link)); @@ -485,6 +526,9 @@ static void qed_fill_link(struct ecore_hwfn *hwfn, if_link->duplex = QEDE_DUPLEX_FULL; + /* Fill up the native advertised speed cap mask */ + if_link->adv_speed = params.speed.advertised_speeds; + if (params.speed.autoneg) if_link->supported_caps |= QEDE_SUPPORTED_AUTONEG; @@ -500,17 +544,42 @@ static void qed_fill_link(struct ecore_hwfn *hwfn, if (params.pause.forced_tx) if_link->pause_config |= QED_LINK_PAUSE_TX_ENABLE; + + if (link_caps.default_eee == ECORE_MCP_EEE_UNSUPPORTED) { + if_link->eee_supported = false; + } else { + if_link->eee_supported = true; + if_link->eee_active = link.eee_active; + if_link->sup_caps = link_caps.eee_speed_caps; + /* MFW clears adv_caps on eee disable; use configured value */ + if_link->eee.adv_caps = link.eee_adv_caps ? link.eee_adv_caps : + params.eee.adv_caps; + if_link->eee.lp_adv_caps = link.eee_lp_adv_caps; + if_link->eee.enable = params.eee.enable; + if_link->eee.tx_lpi_enable = params.eee.tx_lpi_enable; + if_link->eee.tx_lpi_timer = params.eee.tx_lpi_timer; + } } static void qed_get_current_link(struct ecore_dev *edev, struct qed_link_output *if_link) { - qed_fill_link(&edev->hwfns[0], if_link); + struct ecore_hwfn *hwfn; + struct ecore_ptt *ptt; -#ifdef CONFIG_QED_SRIOV - for_each_hwfn(cdev, i) - qed_inform_vf_link_state(&cdev->hwfns[i]); -#endif + hwfn = &edev->hwfns[0]; + if (IS_PF(edev)) { + ptt = ecore_ptt_acquire(hwfn); + if (!ptt) + DP_NOTICE(hwfn, true, "Failed to fill link; No PTT\n"); + + qed_fill_link(hwfn, ptt, if_link); + + if (ptt) + ecore_ptt_release(hwfn, ptt); + } else { + qed_fill_link(hwfn, NULL, if_link); + } } static int qed_set_link(struct ecore_dev *edev, struct qed_link_params *params) @@ -549,6 +618,10 @@ static int qed_set_link(struct ecore_dev *edev, struct qed_link_params *params) link_params->pause.forced_tx = false; } + if (params->override_flags & QED_LINK_OVERRIDE_EEE_CONFIG) + memcpy(&link_params->eee, ¶ms->eee, + sizeof(link_params->eee)); + rc = ecore_mcp_set_link(hwfn, ptt, params->link_up); ecore_ptt_release(hwfn, ptt); @@ -558,9 +631,10 @@ static int qed_set_link(struct ecore_dev *edev, struct qed_link_params *params) void qed_link_update(struct ecore_hwfn *hwfn) { - struct qed_link_output if_link; + struct ecore_dev *edev = hwfn->p_dev; + struct qede_dev *qdev = (struct qede_dev *)edev; - qed_fill_link(hwfn, &if_link); + qede_link_update((struct rte_eth_dev *)qdev->ethdev, 0); } static int qed_drain(struct ecore_dev *edev) @@ -576,7 +650,7 @@ static int qed_drain(struct ecore_dev *edev) hwfn = &edev->hwfns[i]; ptt = ecore_ptt_acquire(hwfn); if (!ptt) { - DP_NOTICE(hwfn, true, "Failed to drain NIG; No PTT\n"); + DP_ERR(hwfn, "Failed to drain NIG; No PTT\n"); return -EBUSY; } rc = ecore_mcp_drain(hwfn, ptt); @@ -602,19 +676,6 @@ static int qed_nic_stop(struct ecore_dev *edev) return rc; } -static int qed_nic_reset(struct ecore_dev *edev) -{ - int rc; - - rc = ecore_hw_reset(edev); - if (rc) - return rc; - - ecore_resc_free(edev); - - return 0; -} - static int qed_slowpath_stop(struct ecore_dev *edev) { #ifdef CONFIG_QED_SRIOV @@ -625,16 +686,19 @@ static int qed_slowpath_stop(struct ecore_dev *edev) return -ENODEV; if (IS_PF(edev)) { +#ifdef CONFIG_ECORE_ZIPPED_FW qed_free_stream_mem(edev); +#endif #ifdef CONFIG_QED_SRIOV if (IS_QED_ETH_IF(edev)) qed_sriov_disable(edev, true); #endif - qed_nic_stop(edev); } - qed_nic_reset(edev); + qed_nic_stop(edev); + + ecore_resc_free(edev); qed_stop_iov_task(edev); return 0; @@ -648,17 +712,71 @@ static void qed_remove(struct ecore_dev *edev) ecore_hw_remove(edev); } +static int qed_send_drv_state(struct ecore_dev *edev, bool active) +{ + struct ecore_hwfn *hwfn = ECORE_LEADING_HWFN(edev); + struct ecore_ptt *ptt; + int status = 0; + + ptt = ecore_ptt_acquire(hwfn); + if (!ptt) + return -EAGAIN; + + status = ecore_mcp_ov_update_driver_state(hwfn, ptt, active ? + ECORE_OV_DRIVER_STATE_ACTIVE : + ECORE_OV_DRIVER_STATE_DISABLED); + + ecore_ptt_release(hwfn, ptt); + + return status; +} + +static int qed_get_sb_info(struct ecore_dev *edev, struct ecore_sb_info *sb, + u16 qid, struct ecore_sb_info_dbg *sb_dbg) +{ + struct ecore_hwfn *hwfn = &edev->hwfns[qid % edev->num_hwfns]; + struct ecore_ptt *ptt; + int rc; + + if (IS_VF(edev)) + return -EINVAL; + + ptt = ecore_ptt_acquire(hwfn); + if (!ptt) { + DP_ERR(hwfn, "Can't acquire PTT\n"); + return -EAGAIN; + } + + memset(sb_dbg, 0, sizeof(*sb_dbg)); + rc = ecore_int_get_sb_dbg(hwfn, ptt, sb, sb_dbg); + + ecore_ptt_release(hwfn, ptt); + return rc; +} + const struct qed_common_ops qed_common_ops_pass = { INIT_STRUCT_FIELD(probe, &qed_probe), INIT_STRUCT_FIELD(update_pf_params, &qed_update_pf_params), INIT_STRUCT_FIELD(slowpath_start, &qed_slowpath_start), - INIT_STRUCT_FIELD(set_id, &qed_set_id), + INIT_STRUCT_FIELD(set_name, &qed_set_name), INIT_STRUCT_FIELD(chain_alloc, &ecore_chain_alloc), INIT_STRUCT_FIELD(chain_free, &ecore_chain_free), INIT_STRUCT_FIELD(sb_init, &qed_sb_init), + INIT_STRUCT_FIELD(get_sb_info, &qed_get_sb_info), INIT_STRUCT_FIELD(get_link, &qed_get_current_link), INIT_STRUCT_FIELD(set_link, &qed_set_link), INIT_STRUCT_FIELD(drain, &qed_drain), INIT_STRUCT_FIELD(slowpath_stop, &qed_slowpath_stop), INIT_STRUCT_FIELD(remove, &qed_remove), + INIT_STRUCT_FIELD(send_drv_state, &qed_send_drv_state), }; + +const struct qed_eth_ops qed_eth_ops_pass = { + INIT_STRUCT_FIELD(common, &qed_common_ops_pass), + INIT_STRUCT_FIELD(fill_dev_info, &qed_fill_eth_dev_info), +}; + +const struct qed_eth_ops *qed_get_eth_ops(void) +{ + return &qed_eth_ops_pass; +} diff --git a/dpdk/drivers/net/qede/qede_rxtx.c b/dpdk/drivers/net/qede/qede_rxtx.c index b5a40fe4..31132ceb 100644 --- a/dpdk/drivers/net/qede/qede_rxtx.c +++ b/dpdk/drivers/net/qede/qede_rxtx.c @@ -6,10 +6,9 @@ * See LICENSE.qede_pmd for copyright and licensing details. */ +#include #include "qede_rxtx.h" -static bool gro_disable = 1; /* mod_param */ - static inline int qede_alloc_rx_buffer(struct qede_rx_queue *rxq) { struct rte_mbuf *new_mb = NULL; @@ -29,7 +28,7 @@ static inline int qede_alloc_rx_buffer(struct qede_rx_queue *rxq) } rxq->sw_rx_ring[idx].mbuf = new_mb; rxq->sw_rx_ring[idx].page_offset = 0; - mapping = rte_mbuf_data_dma_addr_default(new_mb); + mapping = rte_mbuf_data_iova_default(new_mb); /* Advance PROD and get BD pointer */ rx_bd = (struct eth_rx_bd *)ecore_chain_produce(&rxq->rx_bd_ring); rx_bd->addr.hi = rte_cpu_to_le_32(U64_HI(mapping)); @@ -38,48 +37,20 @@ static inline int qede_alloc_rx_buffer(struct qede_rx_queue *rxq) return 0; } -static void qede_rx_queue_release_mbufs(struct qede_rx_queue *rxq) -{ - uint16_t i; - - if (rxq->sw_rx_ring != NULL) { - for (i = 0; i < rxq->nb_rx_desc; i++) { - if (rxq->sw_rx_ring[i].mbuf != NULL) { - rte_pktmbuf_free(rxq->sw_rx_ring[i].mbuf); - rxq->sw_rx_ring[i].mbuf = NULL; - } - } - } -} - -void qede_rx_queue_release(void *rx_queue) -{ - struct qede_rx_queue *rxq = rx_queue; - - if (rxq != NULL) { - qede_rx_queue_release_mbufs(rxq); - rte_free(rxq->sw_rx_ring); - rxq->sw_rx_ring = NULL; - rte_free(rxq); - rx_queue = NULL; - } -} - int qede_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, uint16_t nb_desc, unsigned int socket_id, - const struct rte_eth_rxconf *rx_conf, + __rte_unused const struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp) { - struct qede_dev *qdev = dev->data->dev_private; - struct ecore_dev *edev = &qdev->edev; - struct rte_eth_dev_data *eth_data = dev->data; + struct qede_dev *qdev = QEDE_INIT_QDEV(dev); + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); + struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode; struct qede_rx_queue *rxq; - uint16_t pkt_len = (uint16_t)dev->data->dev_conf.rxmode.max_rx_pkt_len; + uint16_t max_rx_pkt_len; + uint16_t bufsz; size_t size; - uint16_t data_size; int rc; - int i; PMD_INIT_FUNC_TRACE(edev); @@ -112,40 +83,37 @@ qede_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, rxq->queue_id = queue_idx; rxq->port_id = dev->data->port_id; - /* Sanity check */ - data_size = (uint16_t)rte_pktmbuf_data_room_size(mp) - - RTE_PKTMBUF_HEADROOM; + max_rx_pkt_len = (uint16_t)rxmode->max_rx_pkt_len; - if (pkt_len > data_size) { - DP_ERR(edev, "MTU %u should not exceed dataroom %u\n", - pkt_len, data_size); - rte_free(rxq); - return -EINVAL; + /* Fix up RX buffer size */ + bufsz = (uint16_t)rte_pktmbuf_data_room_size(mp) - RTE_PKTMBUF_HEADROOM; + if ((rxmode->enable_scatter) || + (max_rx_pkt_len + QEDE_ETH_OVERHEAD) > bufsz) { + if (!dev->data->scattered_rx) { + DP_INFO(edev, "Forcing scatter-gather mode\n"); + dev->data->scattered_rx = 1; + } } - qdev->mtu = pkt_len; - rxq->rx_buf_size = pkt_len + QEDE_ETH_OVERHEAD; + if (dev->data->scattered_rx) + rxq->rx_buf_size = bufsz + ETHER_HDR_LEN + + ETHER_CRC_LEN + QEDE_ETH_OVERHEAD; + else + rxq->rx_buf_size = max_rx_pkt_len + QEDE_ETH_OVERHEAD; + /* Align to cache-line size if needed */ + rxq->rx_buf_size = QEDE_CEIL_TO_CACHE_LINE_SIZE(rxq->rx_buf_size); - DP_INFO(edev, "MTU = %u ; RX buffer = %u\n", - qdev->mtu, rxq->rx_buf_size); - - if (pkt_len > ETHER_MAX_LEN) { - dev->data->dev_conf.rxmode.jumbo_frame = 1; - DP_NOTICE(edev, false, "jumbo frame enabled\n"); - } else { - dev->data->dev_conf.rxmode.jumbo_frame = 0; - } + DP_INFO(edev, "mtu %u mbufsz %u bd_max_bytes %u scatter_mode %d\n", + qdev->mtu, bufsz, rxq->rx_buf_size, dev->data->scattered_rx); /* Allocate the parallel driver ring for Rx buffers */ size = sizeof(*rxq->sw_rx_ring) * rxq->nb_rx_desc; rxq->sw_rx_ring = rte_zmalloc_socket("sw_rx_ring", size, RTE_CACHE_LINE_SIZE, socket_id); if (!rxq->sw_rx_ring) { - DP_NOTICE(edev, false, - "Unable to alloc memory for sw_rx_ring on socket %u\n", - socket_id); + DP_ERR(edev, "Memory allocation fails for sw_rx_ring on" + " socket %u\n", socket_id); rte_free(rxq); - rxq = NULL; return -ENOMEM; } @@ -156,16 +124,15 @@ qede_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, ECORE_CHAIN_CNT_TYPE_U16, rxq->nb_rx_desc, sizeof(struct eth_rx_bd), - &rxq->rx_bd_ring); + &rxq->rx_bd_ring, + NULL); if (rc != ECORE_SUCCESS) { - DP_NOTICE(edev, false, - "Unable to alloc memory for rxbd ring on socket %u\n", - socket_id); + DP_ERR(edev, "Memory allocation fails for RX BD ring" + " on socket %u\n", socket_id); rte_free(rxq->sw_rx_ring); - rxq->sw_rx_ring = NULL; rte_free(rxq); - rxq = NULL; + return -ENOMEM; } /* Allocate FW completion ring */ @@ -175,70 +142,95 @@ qede_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, ECORE_CHAIN_CNT_TYPE_U16, rxq->nb_rx_desc, sizeof(union eth_rx_cqe), - &rxq->rx_comp_ring); + &rxq->rx_comp_ring, + NULL); if (rc != ECORE_SUCCESS) { - DP_NOTICE(edev, false, - "Unable to alloc memory for cqe ring on socket %u\n", - socket_id); - /* TBD: Freeing RX BD ring */ + DP_ERR(edev, "Memory allocation fails for RX CQE ring" + " on socket %u\n", socket_id); + qdev->ops->common->chain_free(edev, &rxq->rx_bd_ring); rte_free(rxq->sw_rx_ring); - rxq->sw_rx_ring = NULL; rte_free(rxq); - } - - /* Allocate buffers for the Rx ring */ - for (i = 0; i < rxq->nb_rx_desc; i++) { - rc = qede_alloc_rx_buffer(rxq); - if (rc) { - DP_NOTICE(edev, false, - "RX buffer allocation failed at idx=%d\n", i); - goto err4; - } + return -ENOMEM; } dev->data->rx_queues[queue_idx] = rxq; - if (!qdev->rx_queues) - qdev->rx_queues = (struct qede_rx_queue **)dev->data->rx_queues; + qdev->fp_array[queue_idx].rxq = rxq; DP_INFO(edev, "rxq %d num_desc %u rx_buf_size=%u socket %u\n", - queue_idx, nb_desc, qdev->mtu, socket_id); + queue_idx, nb_desc, rxq->rx_buf_size, socket_id); return 0; -err4: - qede_rx_queue_release(rxq); - return -ENOMEM; } -static void qede_tx_queue_release_mbufs(struct qede_tx_queue *txq) +static void +qede_rx_queue_reset(__rte_unused struct qede_dev *qdev, + struct qede_rx_queue *rxq) { - unsigned int i; + DP_INFO(&qdev->edev, "Reset RX queue %u\n", rxq->queue_id); + ecore_chain_reset(&rxq->rx_bd_ring); + ecore_chain_reset(&rxq->rx_comp_ring); + rxq->sw_rx_prod = 0; + rxq->sw_rx_cons = 0; + *rxq->hw_cons_ptr = 0; +} - PMD_TX_LOG(DEBUG, txq, "releasing %u mbufs\n", txq->nb_tx_desc); +static void qede_rx_queue_release_mbufs(struct qede_rx_queue *rxq) +{ + uint16_t i; - if (txq->sw_tx_ring != NULL) { - for (i = 0; i < txq->nb_tx_desc; i++) { - if (txq->sw_tx_ring[i].mbuf != NULL) { - rte_pktmbuf_free(txq->sw_tx_ring[i].mbuf); - txq->sw_tx_ring[i].mbuf = NULL; + if (rxq->sw_rx_ring) { + for (i = 0; i < rxq->nb_rx_desc; i++) { + if (rxq->sw_rx_ring[i].mbuf) { + rte_pktmbuf_free(rxq->sw_rx_ring[i].mbuf); + rxq->sw_rx_ring[i].mbuf = NULL; } } } } -void qede_tx_queue_release(void *tx_queue) +void qede_rx_queue_release(void *rx_queue) { - struct qede_tx_queue *txq = tx_queue; + struct qede_rx_queue *rxq = rx_queue; - if (txq != NULL) { - qede_tx_queue_release_mbufs(txq); - if (txq->sw_tx_ring) { - rte_free(txq->sw_tx_ring); - txq->sw_tx_ring = NULL; - } - rte_free(txq); + if (rxq) { + qede_rx_queue_release_mbufs(rxq); + rte_free(rxq->sw_rx_ring); + rte_free(rxq); } - tx_queue = NULL; +} + +/* Stops a given RX queue in the HW */ +static int qede_rx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id) +{ + struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); + struct ecore_hwfn *p_hwfn; + struct qede_rx_queue *rxq; + int hwfn_index; + int rc; + + if (rx_queue_id < eth_dev->data->nb_rx_queues) { + rxq = eth_dev->data->rx_queues[rx_queue_id]; + hwfn_index = rx_queue_id % edev->num_hwfns; + p_hwfn = &edev->hwfns[hwfn_index]; + rc = ecore_eth_rx_queue_stop(p_hwfn, rxq->handle, + true, false); + if (rc != ECORE_SUCCESS) { + DP_ERR(edev, "RX queue %u stop fails\n", rx_queue_id); + return -1; + } + qede_rx_queue_release_mbufs(rxq); + qede_rx_queue_reset(qdev, rxq); + eth_dev->data->rx_queue_state[rx_queue_id] = + RTE_ETH_QUEUE_STATE_STOPPED; + DP_INFO(edev, "RX queue %u stopped\n", rx_queue_id); + } else { + DP_ERR(edev, "RX queue %u is not in range\n", rx_queue_id); + rc = -EINVAL; + } + + return rc; } int @@ -287,7 +279,8 @@ qede_tx_queue_setup(struct rte_eth_dev *dev, ECORE_CHAIN_CNT_TYPE_U16, txq->nb_tx_desc, sizeof(union eth_tx_bd_types), - &txq->tx_pbl); + &txq->tx_pbl, + NULL); if (rc != ECORE_SUCCESS) { DP_ERR(edev, "Unable to allocate memory for txbd ring on socket %u", @@ -306,6 +299,7 @@ qede_tx_queue_setup(struct rte_eth_dev *dev, DP_ERR(edev, "Unable to allocate memory for txbd ring on socket %u", socket_id); + qdev->ops->common->chain_free(edev, &txq->tx_pbl); qede_tx_queue_release(txq); return -ENOMEM; } @@ -319,10 +313,7 @@ qede_tx_queue_setup(struct rte_eth_dev *dev, (txq->nb_tx_desc - QEDE_DEFAULT_TX_FREE_THRESH); dev->data->tx_queues[queue_idx] = txq; - if (!qdev->tx_queues) - qdev->tx_queues = (struct qede_tx_queue **)dev->data->tx_queues; - - txq->txq_counter = 0; + qdev->fp_array[queue_idx].txq = txq; DP_INFO(edev, "txq %u num_desc %u tx_free_thresh %u socket %u\n", @@ -331,81 +322,40 @@ qede_tx_queue_setup(struct rte_eth_dev *dev, return 0; } -/* This function inits fp content and resets the SB, RXQ and TXQ arrays */ -static void qede_init_fp(struct qede_dev *qdev) +static void +qede_tx_queue_reset(__rte_unused struct qede_dev *qdev, + struct qede_tx_queue *txq) { - struct qede_fastpath *fp; - int rss_id, txq_index, tc; + DP_INFO(&qdev->edev, "Reset TX queue %u\n", txq->queue_id); + ecore_chain_reset(&txq->tx_pbl); + txq->sw_tx_cons = 0; + txq->sw_tx_prod = 0; + *txq->hw_cons_ptr = 0; +} - memset((void *)qdev->fp_array, 0, (QEDE_RSS_CNT(qdev) * - sizeof(*qdev->fp_array))); - memset((void *)qdev->sb_array, 0, (QEDE_RSS_CNT(qdev) * - sizeof(*qdev->sb_array))); - for_each_rss(rss_id) { - fp = &qdev->fp_array[rss_id]; +static void qede_tx_queue_release_mbufs(struct qede_tx_queue *txq) +{ + uint16_t i; - fp->qdev = qdev; - fp->rss_id = rss_id; - - /* Point rxq to generic rte queues that was created - * as part of queue creation. - */ - fp->rxq = qdev->rx_queues[rss_id]; - fp->sb_info = &qdev->sb_array[rss_id]; - - for (tc = 0; tc < qdev->num_tc; tc++) { - txq_index = tc * QEDE_RSS_CNT(qdev) + rss_id; - fp->txqs[tc] = qdev->tx_queues[txq_index]; - fp->txqs[tc]->queue_id = txq_index; - /* Updating it to main structure */ - snprintf(fp->name, sizeof(fp->name), "%s-fp-%d", - "qdev", rss_id); + if (txq->sw_tx_ring) { + for (i = 0; i < txq->nb_tx_desc; i++) { + if (txq->sw_tx_ring[i].mbuf) { + rte_pktmbuf_free(txq->sw_tx_ring[i].mbuf); + txq->sw_tx_ring[i].mbuf = NULL; + } } } - - qdev->gro_disable = gro_disable; } -void qede_free_fp_arrays(struct qede_dev *qdev) +void qede_tx_queue_release(void *tx_queue) { - /* It asseumes qede_free_mem_load() is called before */ - if (qdev->fp_array != NULL) { - rte_free(qdev->fp_array); - qdev->fp_array = NULL; + struct qede_tx_queue *txq = tx_queue; + + if (txq) { + qede_tx_queue_release_mbufs(txq); + rte_free(txq->sw_tx_ring); + rte_free(txq); } - - if (qdev->sb_array != NULL) { - rte_free(qdev->sb_array); - qdev->sb_array = NULL; - } -} - -int qede_alloc_fp_array(struct qede_dev *qdev) -{ - struct qede_fastpath *fp; - struct ecore_dev *edev = &qdev->edev; - int i; - - qdev->fp_array = rte_calloc("fp", QEDE_RSS_CNT(qdev), - sizeof(*qdev->fp_array), - RTE_CACHE_LINE_SIZE); - - if (!qdev->fp_array) { - DP_ERR(edev, "fp array allocation failed\n"); - return -ENOMEM; - } - - qdev->sb_array = rte_calloc("sb", QEDE_RSS_CNT(qdev), - sizeof(*qdev->sb_array), - RTE_CACHE_LINE_SIZE); - - if (!qdev->sb_array) { - DP_ERR(edev, "sb array allocation failed\n"); - rte_free(qdev->fp_array); - return -ENOMEM; - } - - return 0; } /* This function allocates fast-path status block memory */ @@ -413,76 +363,133 @@ static int qede_alloc_mem_sb(struct qede_dev *qdev, struct ecore_sb_info *sb_info, uint16_t sb_id) { - struct ecore_dev *edev = &qdev->edev; - struct status_block *sb_virt; + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); + struct status_block_e4 *sb_virt; dma_addr_t sb_phys; int rc; - sb_virt = OSAL_DMA_ALLOC_COHERENT(edev, &sb_phys, sizeof(*sb_virt)); - + sb_virt = OSAL_DMA_ALLOC_COHERENT(edev, &sb_phys, + sizeof(struct status_block_e4)); if (!sb_virt) { DP_ERR(edev, "Status block allocation failed\n"); return -ENOMEM; } - - rc = qdev->ops->common->sb_init(edev, sb_info, - sb_virt, sb_phys, sb_id, - QED_SB_TYPE_L2_QUEUE); + rc = qdev->ops->common->sb_init(edev, sb_info, sb_virt, + sb_phys, sb_id); if (rc) { DP_ERR(edev, "Status block initialization failed\n"); - /* TBD: No dma_free_coherent possible */ + OSAL_DMA_FREE_COHERENT(edev, sb_virt, sb_phys, + sizeof(struct status_block_e4)); return rc; } return 0; } -static int qede_alloc_mem_fp(struct qede_dev *qdev, struct qede_fastpath *fp) +int qede_alloc_fp_resc(struct qede_dev *qdev) { - return qede_alloc_mem_sb(qdev, fp->sb_info, fp->rss_id); -} - -static void qede_shrink_txq(struct qede_dev *qdev, uint16_t num_rss) -{ - /* @@@TBD - this should also re-set the qed interrupts */ -} - -/* This function allocates all qede memory at NIC load. */ -static int qede_alloc_mem_load(struct qede_dev *qdev) -{ - int rc = 0, rss_id; struct ecore_dev *edev = &qdev->edev; + struct qede_fastpath *fp; + uint32_t num_sbs; + uint16_t sb_idx; - for (rss_id = 0; rss_id < QEDE_RSS_CNT(qdev); rss_id++) { - struct qede_fastpath *fp = &qdev->fp_array[rss_id]; + if (IS_VF(edev)) + ecore_vf_get_num_sbs(ECORE_LEADING_HWFN(edev), &num_sbs); + else + num_sbs = ecore_cxt_get_proto_cid_count + (ECORE_LEADING_HWFN(edev), PROTOCOLID_ETH, NULL); - rc = qede_alloc_mem_fp(qdev, fp); - if (rc) - break; + if (num_sbs == 0) { + DP_ERR(edev, "No status blocks available\n"); + return -EINVAL; } - if (rss_id != QEDE_RSS_CNT(qdev)) { - /* Failed allocating memory for all the queues */ - if (!rss_id) { - DP_ERR(edev, - "Failed to alloc memory for leading queue\n"); - rc = -ENOMEM; - } else { - DP_NOTICE(edev, false, - "Failed to allocate memory for all of " - "RSS queues\n" - "Desired: %d queues, allocated: %d queues\n", - QEDE_RSS_CNT(qdev), rss_id); - qede_shrink_txq(qdev, rss_id); + qdev->fp_array = rte_calloc("fp", QEDE_RXTX_MAX(qdev), + sizeof(*qdev->fp_array), RTE_CACHE_LINE_SIZE); + + if (!qdev->fp_array) { + DP_ERR(edev, "fp array allocation failed\n"); + return -ENOMEM; + } + + memset((void *)qdev->fp_array, 0, QEDE_RXTX_MAX(qdev) * + sizeof(*qdev->fp_array)); + + for (sb_idx = 0; sb_idx < QEDE_RXTX_MAX(qdev); sb_idx++) { + fp = &qdev->fp_array[sb_idx]; + fp->sb_info = rte_calloc("sb", 1, sizeof(struct ecore_sb_info), + RTE_CACHE_LINE_SIZE); + if (!fp->sb_info) { + DP_ERR(edev, "FP sb_info allocation fails\n"); + return -1; } - qdev->num_rss = rss_id; + if (qede_alloc_mem_sb(qdev, fp->sb_info, sb_idx)) { + DP_ERR(edev, "FP status block allocation fails\n"); + return -1; + } + DP_INFO(edev, "sb_info idx 0x%x initialized\n", + fp->sb_info->igu_sb_id); } return 0; } +void qede_dealloc_fp_resc(struct rte_eth_dev *eth_dev) +{ + struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); + struct qede_fastpath *fp; + struct qede_rx_queue *rxq; + struct qede_tx_queue *txq; + uint16_t sb_idx; + uint8_t i; + + PMD_INIT_FUNC_TRACE(edev); + + for (sb_idx = 0; sb_idx < QEDE_RXTX_MAX(qdev); sb_idx++) { + fp = &qdev->fp_array[sb_idx]; + DP_INFO(edev, "Free sb_info index 0x%x\n", + fp->sb_info->igu_sb_id); + if (fp->sb_info) { + OSAL_DMA_FREE_COHERENT(edev, fp->sb_info->sb_virt, + fp->sb_info->sb_phys, + sizeof(struct status_block_e4)); + rte_free(fp->sb_info); + fp->sb_info = NULL; + } + } + + /* Free packet buffers and ring memories */ + for (i = 0; i < eth_dev->data->nb_rx_queues; i++) { + if (eth_dev->data->rx_queues[i]) { + qede_rx_queue_release(eth_dev->data->rx_queues[i]); + rxq = eth_dev->data->rx_queues[i]; + qdev->ops->common->chain_free(edev, + &rxq->rx_bd_ring); + qdev->ops->common->chain_free(edev, + &rxq->rx_comp_ring); + eth_dev->data->rx_queues[i] = NULL; + } + } + + for (i = 0; i < eth_dev->data->nb_tx_queues; i++) { + if (eth_dev->data->tx_queues[i]) { + txq = eth_dev->data->tx_queues[i]; + qede_tx_queue_release(eth_dev->data->tx_queues[i]); + qdev->ops->common->chain_free(edev, + &txq->tx_pbl); + eth_dev->data->tx_queues[i] = NULL; + } + } + + if (qdev->fp_array) + rte_free(qdev->fp_array); + qdev->fp_array = NULL; +} + static inline void -qede_update_rx_prod(struct qede_dev *edev, struct qede_rx_queue *rxq) +qede_update_rx_prod(__rte_unused struct qede_dev *edev, + struct qede_rx_queue *rxq) { uint16_t bd_prod = ecore_chain_get_prod_idx(&rxq->rx_bd_ring); uint16_t cqe_prod = ecore_chain_get_prod_idx(&rxq->rx_comp_ring); @@ -509,294 +516,558 @@ qede_update_rx_prod(struct qede_dev *edev, struct qede_rx_queue *rxq) */ rte_wmb(); - PMD_RX_LOG(DEBUG, rxq, "bd_prod %u cqe_prod %u\n", bd_prod, cqe_prod); + PMD_RX_LOG(DEBUG, rxq, "bd_prod %u cqe_prod %u", bd_prod, cqe_prod); } -static inline uint32_t -qede_rxfh_indir_default(uint32_t index, uint32_t n_rx_rings) +/* Starts a given RX queue in HW */ +static int +qede_rx_queue_start(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id) { - return index % n_rx_rings; -} + struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); + struct ecore_queue_start_common_params params; + struct ecore_rxq_start_ret_params ret_params; + struct qede_rx_queue *rxq; + struct qede_fastpath *fp; + struct ecore_hwfn *p_hwfn; + dma_addr_t p_phys_table; + uint16_t page_cnt; + uint16_t j; + int hwfn_index; + int rc; -static void qede_prandom_bytes(uint32_t *buff, size_t bytes) -{ - unsigned int i; + if (rx_queue_id < eth_dev->data->nb_rx_queues) { + fp = &qdev->fp_array[rx_queue_id]; + rxq = eth_dev->data->rx_queues[rx_queue_id]; + /* Allocate buffers for the Rx ring */ + for (j = 0; j < rxq->nb_rx_desc; j++) { + rc = qede_alloc_rx_buffer(rxq); + if (rc) { + DP_ERR(edev, "RX buffer allocation failed" + " for rxq = %u\n", rx_queue_id); + return -ENOMEM; + } + } + /* disable interrupts */ + ecore_sb_ack(fp->sb_info, IGU_INT_DISABLE, 0); + /* Prepare ramrod */ + memset(¶ms, 0, sizeof(params)); + params.queue_id = rx_queue_id / edev->num_hwfns; + params.vport_id = 0; + params.stats_id = params.vport_id; + params.p_sb = fp->sb_info; + DP_INFO(edev, "rxq %u igu_sb_id 0x%x\n", + fp->rxq->queue_id, fp->sb_info->igu_sb_id); + params.sb_idx = RX_PI; + hwfn_index = rx_queue_id % edev->num_hwfns; + p_hwfn = &edev->hwfns[hwfn_index]; + p_phys_table = ecore_chain_get_pbl_phys(&fp->rxq->rx_comp_ring); + page_cnt = ecore_chain_get_page_cnt(&fp->rxq->rx_comp_ring); + memset(&ret_params, 0, sizeof(ret_params)); + rc = ecore_eth_rx_queue_start(p_hwfn, + p_hwfn->hw_info.opaque_fid, + ¶ms, fp->rxq->rx_buf_size, + fp->rxq->rx_bd_ring.p_phys_addr, + p_phys_table, page_cnt, + &ret_params); + if (rc) { + DP_ERR(edev, "RX queue %u could not be started, rc = %d\n", + rx_queue_id, rc); + return -1; + } + /* Update with the returned parameters */ + fp->rxq->hw_rxq_prod_addr = ret_params.p_prod; + fp->rxq->handle = ret_params.p_handle; - srand((unsigned int)time(NULL)); + fp->rxq->hw_cons_ptr = &fp->sb_info->sb_virt->pi_array[RX_PI]; + qede_update_rx_prod(qdev, fp->rxq); + eth_dev->data->rx_queue_state[rx_queue_id] = + RTE_ETH_QUEUE_STATE_STARTED; + DP_INFO(edev, "RX queue %u started\n", rx_queue_id); + } else { + DP_ERR(edev, "RX queue %u is not in range\n", rx_queue_id); + rc = -EINVAL; + } - for (i = 0; i < ECORE_RSS_KEY_SIZE; i++) - buff[i] = rand(); + return rc; } static int -qede_config_rss(struct rte_eth_dev *eth_dev, - struct qed_update_vport_rss_params *rss_params) +qede_tx_queue_start(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id) { - struct rte_eth_rss_conf rss_conf; - enum rte_eth_rx_mq_mode mode = eth_dev->data->dev_conf.rxmode.mq_mode; - struct qede_dev *qdev = eth_dev->data->dev_private; - struct ecore_dev *edev = &qdev->edev; - uint8_t rss_caps; - unsigned int i; - uint64_t hf; - uint32_t *key; + struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); + struct ecore_queue_start_common_params params; + struct ecore_txq_start_ret_params ret_params; + struct ecore_hwfn *p_hwfn; + dma_addr_t p_phys_table; + struct qede_tx_queue *txq; + struct qede_fastpath *fp; + uint16_t page_cnt; + int hwfn_index; + int rc; - rss_conf = eth_dev->data->dev_conf.rx_adv_conf.rss_conf; - key = (uint32_t *)rss_conf.rss_key; - hf = rss_conf.rss_hf; - PMD_INIT_FUNC_TRACE(edev); - - /* Check if RSS conditions are met. - * Note: Even though its meaningless to enable RSS with one queue, it - * could be used to produce RSS Hash, so skipping that check. - */ - - if (!(mode & ETH_MQ_RX_RSS)) { - DP_INFO(edev, "RSS flag is not set\n"); - return -EINVAL; - } - - DP_INFO(edev, "RSS flag is set\n"); - - if (rss_conf.rss_hf == 0) - DP_NOTICE(edev, false, "RSS hash function = 0, disables RSS\n"); - - if (rss_conf.rss_key != NULL) - memcpy(qdev->rss_params.rss_key, rss_conf.rss_key, - rss_conf.rss_key_len); - - memset(rss_params, 0, sizeof(*rss_params)); - - for (i = 0; i < ECORE_RSS_IND_TABLE_SIZE; i++) - rss_params->rss_ind_table[i] = qede_rxfh_indir_default(i, - QEDE_RSS_CNT(qdev)); - - /* key and protocols */ - if (rss_conf.rss_key == NULL) - qede_prandom_bytes(rss_params->rss_key, - sizeof(rss_params->rss_key)); - else - memcpy(rss_params->rss_key, rss_conf.rss_key, - rss_conf.rss_key_len); - - rss_caps = 0; - rss_caps |= (hf & ETH_RSS_IPV4) ? ECORE_RSS_IPV4 : 0; - rss_caps |= (hf & ETH_RSS_IPV6) ? ECORE_RSS_IPV6 : 0; - rss_caps |= (hf & ETH_RSS_IPV6_EX) ? ECORE_RSS_IPV6 : 0; - rss_caps |= (hf & ETH_RSS_NONFRAG_IPV4_TCP) ? ECORE_RSS_IPV4_TCP : 0; - rss_caps |= (hf & ETH_RSS_NONFRAG_IPV6_TCP) ? ECORE_RSS_IPV6_TCP : 0; - rss_caps |= (hf & ETH_RSS_IPV6_TCP_EX) ? ECORE_RSS_IPV6_TCP : 0; - - rss_params->rss_caps = rss_caps; - - DP_INFO(edev, "RSS check passes\n"); - - return 0; -} - -static int qede_start_queues(struct rte_eth_dev *eth_dev, bool clear_stats) -{ - struct qede_dev *qdev = eth_dev->data->dev_private; - struct ecore_dev *edev = &qdev->edev; - struct qed_update_vport_rss_params *rss_params = &qdev->rss_params; - struct qed_dev_info *qed_info = &qdev->dev_info.common; - struct qed_update_vport_params vport_update_params; - struct qed_start_vport_params start = { 0 }; - int vlan_removal_en = 1; - int rc, tc, i; - - if (!qdev->num_rss) { - DP_ERR(edev, - "Cannot update V-VPORT as active as " - "there are no Rx queues\n"); - return -EINVAL; - } - - start.remove_inner_vlan = vlan_removal_en; - start.gro_enable = !qdev->gro_disable; - start.mtu = qdev->mtu; - start.vport_id = 0; - start.drop_ttl0 = true; - start.clear_stats = clear_stats; - - rc = qdev->ops->vport_start(edev, &start); - if (rc) { - DP_ERR(edev, "Start V-PORT failed %d\n", rc); - return rc; - } - - DP_INFO(edev, - "Start vport ramrod passed, vport_id = %d," - " MTU = %d, vlan_removal_en = %d\n", - start.vport_id, qdev->mtu, vlan_removal_en); - - for_each_rss(i) { - struct qede_fastpath *fp = &qdev->fp_array[i]; - dma_addr_t p_phys_table; - uint16_t page_cnt; - - p_phys_table = ecore_chain_get_pbl_phys(&fp->rxq->rx_comp_ring); - page_cnt = ecore_chain_get_page_cnt(&fp->rxq->rx_comp_ring); - - ecore_sb_ack(fp->sb_info, IGU_INT_DISABLE, 0); /* @DPDK */ - - rc = qdev->ops->q_rx_start(edev, i, i, 0, - fp->sb_info->igu_sb_id, - RX_PI, - fp->rxq->rx_buf_size, - fp->rxq->rx_bd_ring.p_phys_addr, - p_phys_table, - page_cnt, - &fp->rxq->hw_rxq_prod_addr); - if (rc) { - DP_ERR(edev, "Start RXQ #%d failed %d\n", i, rc); - return rc; + if (tx_queue_id < eth_dev->data->nb_tx_queues) { + txq = eth_dev->data->tx_queues[tx_queue_id]; + fp = &qdev->fp_array[tx_queue_id]; + memset(¶ms, 0, sizeof(params)); + params.queue_id = tx_queue_id / edev->num_hwfns; + params.vport_id = 0; + params.stats_id = params.vport_id; + params.p_sb = fp->sb_info; + DP_INFO(edev, "txq %u igu_sb_id 0x%x\n", + fp->txq->queue_id, fp->sb_info->igu_sb_id); + params.sb_idx = TX_PI(0); /* tc = 0 */ + p_phys_table = ecore_chain_get_pbl_phys(&txq->tx_pbl); + page_cnt = ecore_chain_get_page_cnt(&txq->tx_pbl); + hwfn_index = tx_queue_id % edev->num_hwfns; + p_hwfn = &edev->hwfns[hwfn_index]; + if (qdev->dev_info.is_legacy) + fp->txq->is_legacy = true; + rc = ecore_eth_tx_queue_start(p_hwfn, + p_hwfn->hw_info.opaque_fid, + ¶ms, 0 /* tc */, + p_phys_table, page_cnt, + &ret_params); + if (rc != ECORE_SUCCESS) { + DP_ERR(edev, "TX queue %u couldn't be started, rc=%d\n", + tx_queue_id, rc); + return -1; } + txq->doorbell_addr = ret_params.p_doorbell; + txq->handle = ret_params.p_handle; - fp->rxq->hw_cons_ptr = &fp->sb_info->sb_virt->pi_array[RX_PI]; - - qede_update_rx_prod(qdev, fp->rxq); - - for (tc = 0; tc < qdev->num_tc; tc++) { - struct qede_tx_queue *txq = fp->txqs[tc]; - int txq_index = tc * QEDE_RSS_CNT(qdev) + i; - - p_phys_table = ecore_chain_get_pbl_phys(&txq->tx_pbl); - page_cnt = ecore_chain_get_page_cnt(&txq->tx_pbl); - rc = qdev->ops->q_tx_start(edev, i, txq_index, - 0, - fp->sb_info->igu_sb_id, - TX_PI(tc), - p_phys_table, page_cnt, - &txq->doorbell_addr); - if (rc) { - DP_ERR(edev, "Start txq %u failed %d\n", - txq_index, rc); - return rc; - } - - txq->hw_cons_ptr = - &fp->sb_info->sb_virt->pi_array[TX_PI(tc)]; - SET_FIELD(txq->tx_db.data.params, - ETH_DB_DATA_DEST, DB_DEST_XCM); - SET_FIELD(txq->tx_db.data.params, ETH_DB_DATA_AGG_CMD, - DB_AGG_CMD_SET); - SET_FIELD(txq->tx_db.data.params, - ETH_DB_DATA_AGG_VAL_SEL, - DQ_XCM_ETH_TX_BD_PROD_CMD); - - txq->tx_db.data.agg_flags = DQ_XCM_ETH_DQ_CF_CMD; - } - } - - /* Prepare and send the vport enable */ - memset(&vport_update_params, 0, sizeof(vport_update_params)); - vport_update_params.vport_id = start.vport_id; - vport_update_params.update_vport_active_flg = 1; - vport_update_params.vport_active_flg = 1; - - /* @DPDK */ - if (qed_info->mf_mode == MF_NPAR && qed_info->tx_switching) { - /* TBD: Check SRIOV enabled for VF */ - vport_update_params.update_tx_switching_flg = 1; - vport_update_params.tx_switching_flg = 1; - } - - if (!qede_config_rss(eth_dev, rss_params)) { - vport_update_params.update_rss_flg = 1; - - qdev->rss_enabled = 1; - DP_INFO(edev, "Updating RSS flag\n"); + txq->hw_cons_ptr = &fp->sb_info->sb_virt->pi_array[TX_PI(0)]; + SET_FIELD(txq->tx_db.data.params, ETH_DB_DATA_DEST, + DB_DEST_XCM); + SET_FIELD(txq->tx_db.data.params, ETH_DB_DATA_AGG_CMD, + DB_AGG_CMD_SET); + SET_FIELD(txq->tx_db.data.params, + ETH_DB_DATA_AGG_VAL_SEL, + DQ_XCM_ETH_TX_BD_PROD_CMD); + txq->tx_db.data.agg_flags = DQ_XCM_ETH_DQ_CF_CMD; + eth_dev->data->tx_queue_state[tx_queue_id] = + RTE_ETH_QUEUE_STATE_STARTED; + DP_INFO(edev, "TX queue %u started\n", tx_queue_id); } else { - qdev->rss_enabled = 0; - DP_INFO(edev, "Not Updating RSS flag\n"); + DP_ERR(edev, "TX queue %u is not in range\n", tx_queue_id); + rc = -EINVAL; } - rte_memcpy(&vport_update_params.rss_params, rss_params, - sizeof(*rss_params)); + return rc; +} - rc = qdev->ops->vport_update(edev, &vport_update_params); - if (rc) { - DP_ERR(edev, "Update V-PORT failed %d\n", rc); - return rc; +static inline void +qede_free_tx_pkt(struct qede_tx_queue *txq) +{ + struct rte_mbuf *mbuf; + uint16_t nb_segs; + uint16_t idx; + + idx = TX_CONS(txq); + mbuf = txq->sw_tx_ring[idx].mbuf; + if (mbuf) { + nb_segs = mbuf->nb_segs; + PMD_TX_LOG(DEBUG, txq, "nb_segs to free %u\n", nb_segs); + while (nb_segs) { + /* It's like consuming rxbuf in recv() */ + ecore_chain_consume(&txq->tx_pbl); + txq->nb_tx_avail++; + nb_segs--; + } + rte_pktmbuf_free(mbuf); + txq->sw_tx_ring[idx].mbuf = NULL; + txq->sw_tx_cons++; + PMD_TX_LOG(DEBUG, txq, "Freed tx packet\n"); + } else { + ecore_chain_consume(&txq->tx_pbl); + txq->nb_tx_avail++; } +} + +static inline void +qede_process_tx_compl(__rte_unused struct ecore_dev *edev, + struct qede_tx_queue *txq) +{ + uint16_t hw_bd_cons; +#ifdef RTE_LIBRTE_QEDE_DEBUG_TX + uint16_t sw_tx_cons; +#endif + + rte_compiler_barrier(); + hw_bd_cons = rte_le_to_cpu_16(*txq->hw_cons_ptr); +#ifdef RTE_LIBRTE_QEDE_DEBUG_TX + sw_tx_cons = ecore_chain_get_cons_idx(&txq->tx_pbl); + PMD_TX_LOG(DEBUG, txq, "Tx Completions = %u\n", + abs(hw_bd_cons - sw_tx_cons)); +#endif + while (hw_bd_cons != ecore_chain_get_cons_idx(&txq->tx_pbl)) + qede_free_tx_pkt(txq); +} + +static int qede_drain_txq(struct qede_dev *qdev, + struct qede_tx_queue *txq, bool allow_drain) +{ + struct ecore_dev *edev = &qdev->edev; + int rc, cnt = 1000; + + while (txq->sw_tx_cons != txq->sw_tx_prod) { + qede_process_tx_compl(edev, txq); + if (!cnt) { + if (allow_drain) { + DP_ERR(edev, "Tx queue[%u] is stuck," + "requesting MCP to drain\n", + txq->queue_id); + rc = qdev->ops->common->drain(edev); + if (rc) + return rc; + return qede_drain_txq(qdev, txq, false); + } + DP_ERR(edev, "Timeout waiting for tx queue[%d]:" + "PROD=%d, CONS=%d\n", + txq->queue_id, txq->sw_tx_prod, + txq->sw_tx_cons); + return -1; + } + cnt--; + DELAY(1000); + rte_compiler_barrier(); + } + + /* FW finished processing, wait for HW to transmit all tx packets */ + DELAY(2000); return 0; } -#ifdef ENC_SUPPORTED -static bool qede_tunn_exist(uint16_t flag) +/* Stops a given TX queue in the HW */ +static int qede_tx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id) +{ + struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); + struct ecore_hwfn *p_hwfn; + struct qede_tx_queue *txq; + int hwfn_index; + int rc; + + if (tx_queue_id < eth_dev->data->nb_tx_queues) { + txq = eth_dev->data->tx_queues[tx_queue_id]; + /* Drain txq */ + if (qede_drain_txq(qdev, txq, true)) + return -1; /* For the lack of retcodes */ + /* Stop txq */ + hwfn_index = tx_queue_id % edev->num_hwfns; + p_hwfn = &edev->hwfns[hwfn_index]; + rc = ecore_eth_tx_queue_stop(p_hwfn, txq->handle); + if (rc != ECORE_SUCCESS) { + DP_ERR(edev, "TX queue %u stop fails\n", tx_queue_id); + return -1; + } + qede_tx_queue_release_mbufs(txq); + qede_tx_queue_reset(qdev, txq); + eth_dev->data->tx_queue_state[tx_queue_id] = + RTE_ETH_QUEUE_STATE_STOPPED; + DP_INFO(edev, "TX queue %u stopped\n", tx_queue_id); + } else { + DP_ERR(edev, "TX queue %u is not in range\n", tx_queue_id); + rc = -EINVAL; + } + + return rc; +} + +int qede_start_queues(struct rte_eth_dev *eth_dev) +{ + struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); + uint8_t id; + int rc = -1; + + for_each_rss(id) { + rc = qede_rx_queue_start(eth_dev, id); + if (rc != ECORE_SUCCESS) + return -1; + } + + for_each_tss(id) { + rc = qede_tx_queue_start(eth_dev, id); + if (rc != ECORE_SUCCESS) + return -1; + } + + return rc; +} + +void qede_stop_queues(struct rte_eth_dev *eth_dev) +{ + struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); + uint8_t id; + + /* Stopping RX/TX queues */ + for_each_tss(id) { + qede_tx_queue_stop(eth_dev, id); + } + + for_each_rss(id) { + qede_rx_queue_stop(eth_dev, id); + } +} + +static inline bool qede_tunn_exist(uint16_t flag) { return !!((PARSING_AND_ERR_FLAGS_TUNNELEXIST_MASK << PARSING_AND_ERR_FLAGS_TUNNELEXIST_SHIFT) & flag); } -static inline uint8_t qede_check_tunn_csum(uint16_t flag) +static inline uint8_t qede_check_tunn_csum_l3(uint16_t flag) { - uint8_t tcsum = 0; - uint16_t csum_flag = 0; + return !!((PARSING_AND_ERR_FLAGS_TUNNELIPHDRERROR_MASK << + PARSING_AND_ERR_FLAGS_TUNNELIPHDRERROR_SHIFT) & flag); +} +/* + * qede_check_tunn_csum_l4: + * Returns: + * 1 : If L4 csum is enabled AND if the validation has failed. + * 0 : Otherwise + */ +static inline uint8_t qede_check_tunn_csum_l4(uint16_t flag) +{ if ((PARSING_AND_ERR_FLAGS_TUNNELL4CHKSMWASCALCULATED_MASK << PARSING_AND_ERR_FLAGS_TUNNELL4CHKSMWASCALCULATED_SHIFT) & flag) - csum_flag |= PARSING_AND_ERR_FLAGS_TUNNELL4CHKSMERROR_MASK << - PARSING_AND_ERR_FLAGS_TUNNELL4CHKSMERROR_SHIFT; + return !!((PARSING_AND_ERR_FLAGS_TUNNELL4CHKSMERROR_MASK << + PARSING_AND_ERR_FLAGS_TUNNELL4CHKSMERROR_SHIFT) & flag); - if ((PARSING_AND_ERR_FLAGS_L4CHKSMWASCALCULATED_MASK << - PARSING_AND_ERR_FLAGS_L4CHKSMWASCALCULATED_SHIFT) & flag) { - csum_flag |= PARSING_AND_ERR_FLAGS_L4CHKSMERROR_MASK << - PARSING_AND_ERR_FLAGS_L4CHKSMERROR_SHIFT; - tcsum = QEDE_TUNN_CSUM_UNNECESSARY; - } - - csum_flag |= PARSING_AND_ERR_FLAGS_TUNNELIPHDRERROR_MASK << - PARSING_AND_ERR_FLAGS_TUNNELIPHDRERROR_SHIFT | - PARSING_AND_ERR_FLAGS_IPHDRERROR_MASK << - PARSING_AND_ERR_FLAGS_IPHDRERROR_SHIFT; - - if (csum_flag & flag) - return QEDE_CSUM_ERROR; - - return QEDE_CSUM_UNNECESSARY | tcsum; -} -#else -static inline uint8_t qede_tunn_exist(uint16_t flag) -{ return 0; } -static inline uint8_t qede_check_tunn_csum(uint16_t flag) +static inline uint8_t qede_check_notunn_csum_l4(uint16_t flag) { + if ((PARSING_AND_ERR_FLAGS_L4CHKSMWASCALCULATED_MASK << + PARSING_AND_ERR_FLAGS_L4CHKSMWASCALCULATED_SHIFT) & flag) + return !!((PARSING_AND_ERR_FLAGS_L4CHKSMERROR_MASK << + PARSING_AND_ERR_FLAGS_L4CHKSMERROR_SHIFT) & flag); + return 0; } -#endif -static inline uint8_t qede_check_notunn_csum(uint16_t flag) +/* Returns outer L2, L3 and L4 packet_type for tunneled packets */ +static inline uint32_t qede_rx_cqe_to_pkt_type_outer(struct rte_mbuf *m) { - uint8_t csum = 0; - uint16_t csum_flag = 0; + uint32_t packet_type = RTE_PTYPE_UNKNOWN; + struct ether_hdr *eth_hdr; + struct ipv4_hdr *ipv4_hdr; + struct ipv6_hdr *ipv6_hdr; + struct vlan_hdr *vlan_hdr; + uint16_t ethertype; + bool vlan_tagged = 0; + uint16_t len; - if ((PARSING_AND_ERR_FLAGS_L4CHKSMWASCALCULATED_MASK << - PARSING_AND_ERR_FLAGS_L4CHKSMWASCALCULATED_SHIFT) & flag) { - csum_flag |= PARSING_AND_ERR_FLAGS_L4CHKSMERROR_MASK << - PARSING_AND_ERR_FLAGS_L4CHKSMERROR_SHIFT; - csum = QEDE_CSUM_UNNECESSARY; + eth_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *); + len = sizeof(struct ether_hdr); + ethertype = rte_cpu_to_be_16(eth_hdr->ether_type); + + /* Note: Valid only if VLAN stripping is disabled */ + if (ethertype == ETHER_TYPE_VLAN) { + vlan_tagged = 1; + vlan_hdr = (struct vlan_hdr *)(eth_hdr + 1); + len += sizeof(struct vlan_hdr); + ethertype = rte_cpu_to_be_16(vlan_hdr->eth_proto); } - csum_flag |= PARSING_AND_ERR_FLAGS_IPHDRERROR_MASK << - PARSING_AND_ERR_FLAGS_IPHDRERROR_SHIFT; + if (ethertype == ETHER_TYPE_IPv4) { + packet_type |= RTE_PTYPE_L3_IPV4; + ipv4_hdr = rte_pktmbuf_mtod_offset(m, struct ipv4_hdr *, len); + if (ipv4_hdr->next_proto_id == IPPROTO_TCP) + packet_type |= RTE_PTYPE_L4_TCP; + else if (ipv4_hdr->next_proto_id == IPPROTO_UDP) + packet_type |= RTE_PTYPE_L4_UDP; + } else if (ethertype == ETHER_TYPE_IPv6) { + packet_type |= RTE_PTYPE_L3_IPV6; + ipv6_hdr = rte_pktmbuf_mtod_offset(m, struct ipv6_hdr *, len); + if (ipv6_hdr->proto == IPPROTO_TCP) + packet_type |= RTE_PTYPE_L4_TCP; + else if (ipv6_hdr->proto == IPPROTO_UDP) + packet_type |= RTE_PTYPE_L4_UDP; + } - if (csum_flag & flag) - return QEDE_CSUM_ERROR; - - return csum; -} - -static inline uint8_t qede_check_csum(uint16_t flag) -{ - if (likely(!qede_tunn_exist(flag))) - return qede_check_notunn_csum(flag); + if (vlan_tagged) + packet_type |= RTE_PTYPE_L2_ETHER_VLAN; else - return qede_check_tunn_csum(flag); + packet_type |= RTE_PTYPE_L2_ETHER; + + return packet_type; +} + +static inline uint32_t qede_rx_cqe_to_pkt_type_inner(uint16_t flags) +{ + uint16_t val; + + /* Lookup table */ + static const uint32_t + ptype_lkup_tbl[QEDE_PKT_TYPE_MAX] __rte_cache_aligned = { + [QEDE_PKT_TYPE_IPV4] = RTE_PTYPE_INNER_L3_IPV4 | + RTE_PTYPE_INNER_L2_ETHER, + [QEDE_PKT_TYPE_IPV6] = RTE_PTYPE_INNER_L3_IPV6 | + RTE_PTYPE_INNER_L2_ETHER, + [QEDE_PKT_TYPE_IPV4_TCP] = RTE_PTYPE_INNER_L3_IPV4 | + RTE_PTYPE_INNER_L4_TCP | + RTE_PTYPE_INNER_L2_ETHER, + [QEDE_PKT_TYPE_IPV6_TCP] = RTE_PTYPE_INNER_L3_IPV6 | + RTE_PTYPE_INNER_L4_TCP | + RTE_PTYPE_INNER_L2_ETHER, + [QEDE_PKT_TYPE_IPV4_UDP] = RTE_PTYPE_INNER_L3_IPV4 | + RTE_PTYPE_INNER_L4_UDP | + RTE_PTYPE_INNER_L2_ETHER, + [QEDE_PKT_TYPE_IPV6_UDP] = RTE_PTYPE_INNER_L3_IPV6 | + RTE_PTYPE_INNER_L4_UDP | + RTE_PTYPE_INNER_L2_ETHER, + /* Frags with no VLAN */ + [QEDE_PKT_TYPE_IPV4_FRAG] = RTE_PTYPE_INNER_L3_IPV4 | + RTE_PTYPE_INNER_L4_FRAG | + RTE_PTYPE_INNER_L2_ETHER, + [QEDE_PKT_TYPE_IPV6_FRAG] = RTE_PTYPE_INNER_L3_IPV6 | + RTE_PTYPE_INNER_L4_FRAG | + RTE_PTYPE_INNER_L2_ETHER, + /* VLANs */ + [QEDE_PKT_TYPE_IPV4_VLAN] = RTE_PTYPE_INNER_L3_IPV4 | + RTE_PTYPE_INNER_L2_ETHER_VLAN, + [QEDE_PKT_TYPE_IPV6_VLAN] = RTE_PTYPE_INNER_L3_IPV6 | + RTE_PTYPE_INNER_L2_ETHER_VLAN, + [QEDE_PKT_TYPE_IPV4_TCP_VLAN] = RTE_PTYPE_INNER_L3_IPV4 | + RTE_PTYPE_INNER_L4_TCP | + RTE_PTYPE_INNER_L2_ETHER_VLAN, + [QEDE_PKT_TYPE_IPV6_TCP_VLAN] = RTE_PTYPE_INNER_L3_IPV6 | + RTE_PTYPE_INNER_L4_TCP | + RTE_PTYPE_INNER_L2_ETHER_VLAN, + [QEDE_PKT_TYPE_IPV4_UDP_VLAN] = RTE_PTYPE_INNER_L3_IPV4 | + RTE_PTYPE_INNER_L4_UDP | + RTE_PTYPE_INNER_L2_ETHER_VLAN, + [QEDE_PKT_TYPE_IPV6_UDP_VLAN] = RTE_PTYPE_INNER_L3_IPV6 | + RTE_PTYPE_INNER_L4_UDP | + RTE_PTYPE_INNER_L2_ETHER_VLAN, + /* Frags with VLAN */ + [QEDE_PKT_TYPE_IPV4_VLAN_FRAG] = RTE_PTYPE_INNER_L3_IPV4 | + RTE_PTYPE_INNER_L4_FRAG | + RTE_PTYPE_INNER_L2_ETHER_VLAN, + [QEDE_PKT_TYPE_IPV6_VLAN_FRAG] = RTE_PTYPE_INNER_L3_IPV6 | + RTE_PTYPE_INNER_L4_FRAG | + RTE_PTYPE_INNER_L2_ETHER_VLAN, + }; + + /* Bits (0..3) provides L3/L4 protocol type */ + /* Bits (4,5) provides frag and VLAN info */ + val = ((PARSING_AND_ERR_FLAGS_L3TYPE_MASK << + PARSING_AND_ERR_FLAGS_L3TYPE_SHIFT) | + (PARSING_AND_ERR_FLAGS_L4PROTOCOL_MASK << + PARSING_AND_ERR_FLAGS_L4PROTOCOL_SHIFT) | + (PARSING_AND_ERR_FLAGS_IPV4FRAG_MASK << + PARSING_AND_ERR_FLAGS_IPV4FRAG_SHIFT) | + (PARSING_AND_ERR_FLAGS_TAG8021QEXIST_MASK << + PARSING_AND_ERR_FLAGS_TAG8021QEXIST_SHIFT)) & flags; + + if (val < QEDE_PKT_TYPE_MAX) + return ptype_lkup_tbl[val]; + + return RTE_PTYPE_UNKNOWN; +} + +static inline uint32_t qede_rx_cqe_to_pkt_type(uint16_t flags) +{ + uint16_t val; + + /* Lookup table */ + static const uint32_t + ptype_lkup_tbl[QEDE_PKT_TYPE_MAX] __rte_cache_aligned = { + [QEDE_PKT_TYPE_IPV4] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L2_ETHER, + [QEDE_PKT_TYPE_IPV6] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L2_ETHER, + [QEDE_PKT_TYPE_IPV4_TCP] = RTE_PTYPE_L3_IPV4 | + RTE_PTYPE_L4_TCP | + RTE_PTYPE_L2_ETHER, + [QEDE_PKT_TYPE_IPV6_TCP] = RTE_PTYPE_L3_IPV6 | + RTE_PTYPE_L4_TCP | + RTE_PTYPE_L2_ETHER, + [QEDE_PKT_TYPE_IPV4_UDP] = RTE_PTYPE_L3_IPV4 | + RTE_PTYPE_L4_UDP | + RTE_PTYPE_L2_ETHER, + [QEDE_PKT_TYPE_IPV6_UDP] = RTE_PTYPE_L3_IPV6 | + RTE_PTYPE_L4_UDP | + RTE_PTYPE_L2_ETHER, + /* Frags with no VLAN */ + [QEDE_PKT_TYPE_IPV4_FRAG] = RTE_PTYPE_L3_IPV4 | + RTE_PTYPE_L4_FRAG | + RTE_PTYPE_L2_ETHER, + [QEDE_PKT_TYPE_IPV6_FRAG] = RTE_PTYPE_L3_IPV6 | + RTE_PTYPE_L4_FRAG | + RTE_PTYPE_L2_ETHER, + /* VLANs */ + [QEDE_PKT_TYPE_IPV4_VLAN] = RTE_PTYPE_L3_IPV4 | + RTE_PTYPE_L2_ETHER_VLAN, + [QEDE_PKT_TYPE_IPV6_VLAN] = RTE_PTYPE_L3_IPV6 | + RTE_PTYPE_L2_ETHER_VLAN, + [QEDE_PKT_TYPE_IPV4_TCP_VLAN] = RTE_PTYPE_L3_IPV4 | + RTE_PTYPE_L4_TCP | + RTE_PTYPE_L2_ETHER_VLAN, + [QEDE_PKT_TYPE_IPV6_TCP_VLAN] = RTE_PTYPE_L3_IPV6 | + RTE_PTYPE_L4_TCP | + RTE_PTYPE_L2_ETHER_VLAN, + [QEDE_PKT_TYPE_IPV4_UDP_VLAN] = RTE_PTYPE_L3_IPV4 | + RTE_PTYPE_L4_UDP | + RTE_PTYPE_L2_ETHER_VLAN, + [QEDE_PKT_TYPE_IPV6_UDP_VLAN] = RTE_PTYPE_L3_IPV6 | + RTE_PTYPE_L4_UDP | + RTE_PTYPE_L2_ETHER_VLAN, + /* Frags with VLAN */ + [QEDE_PKT_TYPE_IPV4_VLAN_FRAG] = RTE_PTYPE_L3_IPV4 | + RTE_PTYPE_L4_FRAG | + RTE_PTYPE_L2_ETHER_VLAN, + [QEDE_PKT_TYPE_IPV6_VLAN_FRAG] = RTE_PTYPE_L3_IPV6 | + RTE_PTYPE_L4_FRAG | + RTE_PTYPE_L2_ETHER_VLAN, + }; + + /* Bits (0..3) provides L3/L4 protocol type */ + /* Bits (4,5) provides frag and VLAN info */ + val = ((PARSING_AND_ERR_FLAGS_L3TYPE_MASK << + PARSING_AND_ERR_FLAGS_L3TYPE_SHIFT) | + (PARSING_AND_ERR_FLAGS_L4PROTOCOL_MASK << + PARSING_AND_ERR_FLAGS_L4PROTOCOL_SHIFT) | + (PARSING_AND_ERR_FLAGS_IPV4FRAG_MASK << + PARSING_AND_ERR_FLAGS_IPV4FRAG_SHIFT) | + (PARSING_AND_ERR_FLAGS_TAG8021QEXIST_MASK << + PARSING_AND_ERR_FLAGS_TAG8021QEXIST_SHIFT)) & flags; + + if (val < QEDE_PKT_TYPE_MAX) + return ptype_lkup_tbl[val]; + + return RTE_PTYPE_UNKNOWN; +} + +static inline uint8_t +qede_check_notunn_csum_l3(struct rte_mbuf *m, uint16_t flag) +{ + struct ipv4_hdr *ip; + uint16_t pkt_csum; + uint16_t calc_csum; + uint16_t val; + + val = ((PARSING_AND_ERR_FLAGS_IPHDRERROR_MASK << + PARSING_AND_ERR_FLAGS_IPHDRERROR_SHIFT) & flag); + + if (unlikely(val)) { + m->packet_type = qede_rx_cqe_to_pkt_type(flag); + if (RTE_ETH_IS_IPV4_HDR(m->packet_type)) { + ip = rte_pktmbuf_mtod_offset(m, struct ipv4_hdr *, + sizeof(struct ether_hdr)); + pkt_csum = ip->hdr_checksum; + ip->hdr_checksum = 0; + calc_csum = rte_ipv4_cksum(ip); + ip->hdr_checksum = pkt_csum; + return (calc_csum != pkt_csum); + } else if (RTE_ETH_IS_IPV6_HDR(m->packet_type)) { + return 1; + } + } + return 0; } static inline void qede_rx_bd_ring_consume(struct qede_rx_queue *rxq) @@ -806,7 +1077,7 @@ static inline void qede_rx_bd_ring_consume(struct qede_rx_queue *rxq) } static inline void -qede_reuse_page(struct qede_dev *qdev, +qede_reuse_page(__rte_unused struct qede_dev *qdev, struct qede_rx_queue *rxq, struct qede_rx_entry *curr_cons) { struct eth_rx_bd *rx_bd_prod = ecore_chain_produce(&rxq->rx_bd_ring); @@ -817,7 +1088,7 @@ qede_reuse_page(struct qede_dev *qdev, curr_prod = &rxq->sw_rx_ring[idx]; *curr_prod = *curr_cons; - new_mapping = rte_mbuf_data_dma_addr_default(curr_prod->mbuf) + + new_mapping = rte_mbuf_data_iova_default(curr_prod->mbuf) + curr_prod->page_offset; rx_bd_prod->addr.hi = rte_cpu_to_le_32(U64_HI(new_mapping)); @@ -839,42 +1110,219 @@ qede_recycle_rx_bd_ring(struct qede_rx_queue *rxq, } } -static inline uint32_t qede_rx_cqe_to_pkt_type(uint16_t flags) +static inline void +qede_rx_process_tpa_cmn_cont_end_cqe(__rte_unused struct qede_dev *qdev, + struct qede_rx_queue *rxq, + uint8_t agg_index, uint16_t len) { - uint32_t p_type; - /* TBD - L4 indications needed ? */ - uint16_t protocol = ((PARSING_AND_ERR_FLAGS_L3TYPE_MASK << - PARSING_AND_ERR_FLAGS_L3TYPE_SHIFT) & flags); + struct qede_agg_info *tpa_info; + struct rte_mbuf *curr_frag; /* Pointer to currently filled TPA seg */ + uint16_t cons_idx; - /* protocol = 3 means LLC/SNAP over Ethernet */ - if (unlikely(protocol == 0 || protocol == 3)) - p_type = RTE_PTYPE_UNKNOWN; - else if (protocol == 1) - p_type = RTE_PTYPE_L3_IPV4; - else if (protocol == 2) - p_type = RTE_PTYPE_L3_IPV6; - - return RTE_PTYPE_L2_ETHER | p_type; + /* Under certain conditions it is possible that FW may not consume + * additional or new BD. So decision to consume the BD must be made + * based on len_list[0]. + */ + if (rte_le_to_cpu_16(len)) { + tpa_info = &rxq->tpa_info[agg_index]; + cons_idx = rxq->sw_rx_cons & NUM_RX_BDS(rxq); + curr_frag = rxq->sw_rx_ring[cons_idx].mbuf; + assert(curr_frag); + curr_frag->nb_segs = 1; + curr_frag->pkt_len = rte_le_to_cpu_16(len); + curr_frag->data_len = curr_frag->pkt_len; + tpa_info->tpa_tail->next = curr_frag; + tpa_info->tpa_tail = curr_frag; + qede_rx_bd_ring_consume(rxq); + if (unlikely(qede_alloc_rx_buffer(rxq) != 0)) { + PMD_RX_LOG(ERR, rxq, "mbuf allocation fails\n"); + rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed++; + rxq->rx_alloc_errors++; + } + } } +static inline void +qede_rx_process_tpa_cont_cqe(struct qede_dev *qdev, + struct qede_rx_queue *rxq, + struct eth_fast_path_rx_tpa_cont_cqe *cqe) +{ + PMD_RX_LOG(INFO, rxq, "TPA cont[%d] - len [%d]\n", + cqe->tpa_agg_index, rte_le_to_cpu_16(cqe->len_list[0])); + /* only len_list[0] will have value */ + qede_rx_process_tpa_cmn_cont_end_cqe(qdev, rxq, cqe->tpa_agg_index, + cqe->len_list[0]); +} + +static inline void +qede_rx_process_tpa_end_cqe(struct qede_dev *qdev, + struct qede_rx_queue *rxq, + struct eth_fast_path_rx_tpa_end_cqe *cqe) +{ + struct rte_mbuf *rx_mb; /* Pointer to head of the chained agg */ + + qede_rx_process_tpa_cmn_cont_end_cqe(qdev, rxq, cqe->tpa_agg_index, + cqe->len_list[0]); + /* Update total length and frags based on end TPA */ + rx_mb = rxq->tpa_info[cqe->tpa_agg_index].tpa_head; + /* TODO: Add Sanity Checks */ + rx_mb->nb_segs = cqe->num_of_bds; + rx_mb->pkt_len = cqe->total_packet_len; + + PMD_RX_LOG(INFO, rxq, "TPA End[%d] reason %d cqe_len %d nb_segs %d" + " pkt_len %d\n", cqe->tpa_agg_index, cqe->end_reason, + rte_le_to_cpu_16(cqe->len_list[0]), rx_mb->nb_segs, + rx_mb->pkt_len); +} + +static inline uint32_t qede_rx_cqe_to_tunn_pkt_type(uint16_t flags) +{ + uint32_t val; + + /* Lookup table */ + static const uint32_t + ptype_tunn_lkup_tbl[QEDE_PKT_TYPE_TUNN_MAX_TYPE] __rte_cache_aligned = { + [QEDE_PKT_TYPE_UNKNOWN] = RTE_PTYPE_UNKNOWN, + [QEDE_PKT_TYPE_TUNN_GENEVE] = RTE_PTYPE_TUNNEL_GENEVE, + [QEDE_PKT_TYPE_TUNN_GRE] = RTE_PTYPE_TUNNEL_GRE, + [QEDE_PKT_TYPE_TUNN_VXLAN] = RTE_PTYPE_TUNNEL_VXLAN, + [QEDE_PKT_TYPE_TUNN_L2_TENID_NOEXIST_GENEVE] = + RTE_PTYPE_TUNNEL_GENEVE, + [QEDE_PKT_TYPE_TUNN_L2_TENID_NOEXIST_GRE] = + RTE_PTYPE_TUNNEL_GRE, + [QEDE_PKT_TYPE_TUNN_L2_TENID_NOEXIST_VXLAN] = + RTE_PTYPE_TUNNEL_VXLAN, + [QEDE_PKT_TYPE_TUNN_L2_TENID_EXIST_GENEVE] = + RTE_PTYPE_TUNNEL_GENEVE, + [QEDE_PKT_TYPE_TUNN_L2_TENID_EXIST_GRE] = + RTE_PTYPE_TUNNEL_GRE, + [QEDE_PKT_TYPE_TUNN_L2_TENID_EXIST_VXLAN] = + RTE_PTYPE_TUNNEL_VXLAN, + [QEDE_PKT_TYPE_TUNN_IPV4_TENID_NOEXIST_GENEVE] = + RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L3_IPV4, + [QEDE_PKT_TYPE_TUNN_IPV4_TENID_NOEXIST_GRE] = + RTE_PTYPE_TUNNEL_GRE | RTE_PTYPE_L3_IPV4, + [QEDE_PKT_TYPE_TUNN_IPV4_TENID_NOEXIST_VXLAN] = + RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L3_IPV4, + [QEDE_PKT_TYPE_TUNN_IPV4_TENID_EXIST_GENEVE] = + RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L3_IPV4, + [QEDE_PKT_TYPE_TUNN_IPV4_TENID_EXIST_GRE] = + RTE_PTYPE_TUNNEL_GRE | RTE_PTYPE_L3_IPV4, + [QEDE_PKT_TYPE_TUNN_IPV4_TENID_EXIST_VXLAN] = + RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L3_IPV4, + [QEDE_PKT_TYPE_TUNN_IPV6_TENID_NOEXIST_GENEVE] = + RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L3_IPV6, + [QEDE_PKT_TYPE_TUNN_IPV6_TENID_NOEXIST_GRE] = + RTE_PTYPE_TUNNEL_GRE | RTE_PTYPE_L3_IPV6, + [QEDE_PKT_TYPE_TUNN_IPV6_TENID_NOEXIST_VXLAN] = + RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L3_IPV6, + [QEDE_PKT_TYPE_TUNN_IPV6_TENID_EXIST_GENEVE] = + RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L3_IPV6, + [QEDE_PKT_TYPE_TUNN_IPV6_TENID_EXIST_GRE] = + RTE_PTYPE_TUNNEL_GRE | RTE_PTYPE_L3_IPV6, + [QEDE_PKT_TYPE_TUNN_IPV6_TENID_EXIST_VXLAN] = + RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L3_IPV6, + }; + + /* Cover bits[4-0] to include tunn_type and next protocol */ + val = ((ETH_TUNNEL_PARSING_FLAGS_TYPE_MASK << + ETH_TUNNEL_PARSING_FLAGS_TYPE_SHIFT) | + (ETH_TUNNEL_PARSING_FLAGS_NEXT_PROTOCOL_MASK << + ETH_TUNNEL_PARSING_FLAGS_NEXT_PROTOCOL_SHIFT)) & flags; + + if (val < QEDE_PKT_TYPE_TUNN_MAX_TYPE) + return ptype_tunn_lkup_tbl[val]; + else + return RTE_PTYPE_UNKNOWN; +} + +static inline int +qede_process_sg_pkts(void *p_rxq, struct rte_mbuf *rx_mb, + uint8_t num_segs, uint16_t pkt_len) +{ + struct qede_rx_queue *rxq = p_rxq; + struct qede_dev *qdev = rxq->qdev; + register struct rte_mbuf *seg1 = NULL; + register struct rte_mbuf *seg2 = NULL; + uint16_t sw_rx_index; + uint16_t cur_size; + + seg1 = rx_mb; + while (num_segs) { + cur_size = pkt_len > rxq->rx_buf_size ? rxq->rx_buf_size : + pkt_len; + if (unlikely(!cur_size)) { + PMD_RX_LOG(ERR, rxq, "Length is 0 while %u BDs" + " left for mapping jumbo\n", num_segs); + qede_recycle_rx_bd_ring(rxq, qdev, num_segs); + return -EINVAL; + } + sw_rx_index = rxq->sw_rx_cons & NUM_RX_BDS(rxq); + seg2 = rxq->sw_rx_ring[sw_rx_index].mbuf; + qede_rx_bd_ring_consume(rxq); + pkt_len -= cur_size; + seg2->data_len = cur_size; + seg1->next = seg2; + seg1 = seg1->next; + num_segs--; + rxq->rx_segs++; + } + + return 0; +} + +#ifdef RTE_LIBRTE_QEDE_DEBUG_RX +static inline void +print_rx_bd_info(struct rte_mbuf *m, struct qede_rx_queue *rxq, + uint8_t bitfield) +{ + PMD_RX_LOG(INFO, rxq, + "len 0x%04x bf 0x%04x hash_val 0x%x" + " ol_flags 0x%04lx l2=%s l3=%s l4=%s tunn=%s" + " inner_l2=%s inner_l3=%s inner_l4=%s\n", + m->data_len, bitfield, m->hash.rss, + (unsigned long)m->ol_flags, + rte_get_ptype_l2_name(m->packet_type), + rte_get_ptype_l3_name(m->packet_type), + rte_get_ptype_l4_name(m->packet_type), + rte_get_ptype_tunnel_name(m->packet_type), + rte_get_ptype_inner_l2_name(m->packet_type), + rte_get_ptype_inner_l3_name(m->packet_type), + rte_get_ptype_inner_l4_name(m->packet_type)); +} +#endif + uint16_t qede_recv_pkts(void *p_rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) { struct qede_rx_queue *rxq = p_rxq; struct qede_dev *qdev = rxq->qdev; struct ecore_dev *edev = &qdev->edev; - struct qede_fastpath *fp = &qdev->fp_array[rxq->queue_id]; uint16_t hw_comp_cons, sw_comp_cons, sw_rx_index; uint16_t rx_pkt = 0; union eth_rx_cqe *cqe; - struct eth_fast_path_rx_reg_cqe *fp_cqe; + struct eth_fast_path_rx_reg_cqe *fp_cqe = NULL; register struct rte_mbuf *rx_mb = NULL; + register struct rte_mbuf *seg1 = NULL; enum eth_rx_cqe_type cqe_type; - uint16_t len, pad; + uint16_t pkt_len = 0; /* Sum of all BD segments */ + uint16_t len; /* Length of first BD */ + uint8_t num_segs = 1; uint16_t preload_idx; - uint8_t csum_flag; uint16_t parse_flag; - enum rss_hash_type htype; +#ifdef RTE_LIBRTE_QEDE_DEBUG_RX + uint8_t bitfield_val; +#endif + uint8_t tunn_parse_flag; + uint8_t j; + struct eth_fast_path_rx_tpa_start_cqe *cqe_start_tpa; + uint64_t ol_flags; + uint32_t packet_type; + uint16_t vlan_tci; + bool tpa_start_flg; + uint8_t offset, tpa_agg_idx, flags; + struct qede_agg_info *tpa_info = NULL; + uint32_t rss_hash; hw_comp_cons = rte_le_to_cpu_16(*rxq->hw_cons_ptr); sw_comp_cons = ecore_chain_get_cons_idx(&rxq->rx_comp_ring); @@ -885,16 +1333,60 @@ qede_recv_pkts(void *p_rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) return 0; while (sw_comp_cons != hw_comp_cons) { + ol_flags = 0; + packet_type = RTE_PTYPE_UNKNOWN; + vlan_tci = 0; + tpa_start_flg = false; + rss_hash = 0; + /* Get the CQE from the completion ring */ cqe = (union eth_rx_cqe *)ecore_chain_consume(&rxq->rx_comp_ring); cqe_type = cqe->fast_path_regular.type; + PMD_RX_LOG(INFO, rxq, "Rx CQE type %d\n", cqe_type); - if (unlikely(cqe_type == ETH_RX_CQE_TYPE_SLOW_PATH)) { - PMD_RX_LOG(DEBUG, rxq, "Got a slowath CQE\n"); + switch (cqe_type) { + case ETH_RX_CQE_TYPE_REGULAR: + fp_cqe = &cqe->fast_path_regular; + break; + case ETH_RX_CQE_TYPE_TPA_START: + cqe_start_tpa = &cqe->fast_path_tpa_start; + tpa_info = &rxq->tpa_info[cqe_start_tpa->tpa_agg_index]; + tpa_start_flg = true; + /* Mark it as LRO packet */ + ol_flags |= PKT_RX_LRO; + /* In split mode, seg_len is same as len_on_first_bd + * and ext_bd_len_list will be empty since there are + * no additional buffers + */ + PMD_RX_LOG(INFO, rxq, + "TPA start[%d] - len_on_first_bd %d header %d" + " [bd_list[0] %d], [seg_len %d]\n", + cqe_start_tpa->tpa_agg_index, + rte_le_to_cpu_16(cqe_start_tpa->len_on_first_bd), + cqe_start_tpa->header_len, + rte_le_to_cpu_16(cqe_start_tpa->ext_bd_len_list[0]), + rte_le_to_cpu_16(cqe_start_tpa->seg_len)); - qdev->ops->eth_cqe_completion(edev, fp->rss_id, + break; + case ETH_RX_CQE_TYPE_TPA_CONT: + qede_rx_process_tpa_cont_cqe(qdev, rxq, + &cqe->fast_path_tpa_cont); + goto next_cqe; + case ETH_RX_CQE_TYPE_TPA_END: + qede_rx_process_tpa_end_cqe(qdev, rxq, + &cqe->fast_path_tpa_end); + tpa_agg_idx = cqe->fast_path_tpa_end.tpa_agg_index; + tpa_info = &rxq->tpa_info[tpa_agg_idx]; + rx_mb = rxq->tpa_info[tpa_agg_idx].tpa_head; + goto tpa_end; + case ETH_RX_CQE_TYPE_SLOW_PATH: + PMD_RX_LOG(INFO, rxq, "Got unexpected slowpath CQE\n"); + ecore_eth_cqe_completion( + &edev->hwfns[rxq->queue_id % edev->num_hwfns], (struct eth_slow_path_rx_cqe *)cqe); + /* fall-thru */ + default: goto next_cqe; } @@ -903,32 +1395,111 @@ qede_recv_pkts(void *p_rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) rx_mb = rxq->sw_rx_ring[sw_rx_index].mbuf; assert(rx_mb != NULL); - /* non GRO */ - fp_cqe = &cqe->fast_path_regular; + /* Handle regular CQE or TPA start CQE */ + if (!tpa_start_flg) { + parse_flag = rte_le_to_cpu_16(fp_cqe->pars_flags.flags); + offset = fp_cqe->placement_offset; + len = rte_le_to_cpu_16(fp_cqe->len_on_first_bd); + pkt_len = rte_le_to_cpu_16(fp_cqe->pkt_len); + vlan_tci = rte_le_to_cpu_16(fp_cqe->vlan_tag); + rss_hash = rte_le_to_cpu_32(fp_cqe->rss_hash); +#ifdef RTE_LIBRTE_QEDE_DEBUG_RX + bitfield_val = fp_cqe->bitfields; +#endif + } else { + parse_flag = + rte_le_to_cpu_16(cqe_start_tpa->pars_flags.flags); + offset = cqe_start_tpa->placement_offset; + /* seg_len = len_on_first_bd */ + len = rte_le_to_cpu_16(cqe_start_tpa->len_on_first_bd); + vlan_tci = rte_le_to_cpu_16(cqe_start_tpa->vlan_tag); +#ifdef RTE_LIBRTE_QEDE_DEBUG_RX + bitfield_val = cqe_start_tpa->bitfields; +#endif + rss_hash = rte_le_to_cpu_32(cqe_start_tpa->rss_hash); + } + if (qede_tunn_exist(parse_flag)) { + PMD_RX_LOG(INFO, rxq, "Rx tunneled packet\n"); + if (unlikely(qede_check_tunn_csum_l4(parse_flag))) { + PMD_RX_LOG(ERR, rxq, + "L4 csum failed, flags = 0x%x\n", + parse_flag); + rxq->rx_hw_errors++; + ol_flags |= PKT_RX_L4_CKSUM_BAD; + } else { + ol_flags |= PKT_RX_L4_CKSUM_GOOD; + } - len = rte_le_to_cpu_16(fp_cqe->len_on_first_bd); - pad = fp_cqe->placement_offset; - assert((len + pad) <= rx_mb->buf_len); + if (unlikely(qede_check_tunn_csum_l3(parse_flag))) { + PMD_RX_LOG(ERR, rxq, + "Outer L3 csum failed, flags = 0x%x\n", + parse_flag); + rxq->rx_hw_errors++; + ol_flags |= PKT_RX_EIP_CKSUM_BAD; + } else { + ol_flags |= PKT_RX_IP_CKSUM_GOOD; + } - PMD_RX_LOG(DEBUG, rxq, - "CQE type = 0x%x, flags = 0x%x, vlan = 0x%x" - " len = %u, parsing_flags = %d\n", - cqe_type, fp_cqe->bitfields, - rte_le_to_cpu_16(fp_cqe->vlan_tag), - len, rte_le_to_cpu_16(fp_cqe->pars_flags.flags)); + if (tpa_start_flg) + flags = cqe_start_tpa->tunnel_pars_flags.flags; + else + flags = fp_cqe->tunnel_pars_flags.flags; + tunn_parse_flag = flags; - /* If this is an error packet then drop it */ - parse_flag = - rte_le_to_cpu_16(cqe->fast_path_regular.pars_flags.flags); - csum_flag = qede_check_csum(parse_flag); - if (unlikely(csum_flag == QEDE_CSUM_ERROR)) { + /* Tunnel_type */ + packet_type = + qede_rx_cqe_to_tunn_pkt_type(tunn_parse_flag); + + /* Inner header */ + packet_type |= + qede_rx_cqe_to_pkt_type_inner(parse_flag); + + /* Outer L3/L4 types is not available in CQE */ + packet_type |= qede_rx_cqe_to_pkt_type_outer(rx_mb); + + /* Outer L3/L4 types is not available in CQE. + * Need to add offset to parse correctly, + */ + rx_mb->data_off = offset + RTE_PKTMBUF_HEADROOM; + packet_type |= qede_rx_cqe_to_pkt_type_outer(rx_mb); + } + + /* Common handling for non-tunnel packets and for inner + * headers in the case of tunnel. + */ + if (unlikely(qede_check_notunn_csum_l4(parse_flag))) { PMD_RX_LOG(ERR, rxq, - "CQE in CONS = %u has error, flags = 0x%x " - "dropping incoming packet\n", - sw_comp_cons, parse_flag); + "L4 csum failed, flags = 0x%x\n", + parse_flag); rxq->rx_hw_errors++; - qede_recycle_rx_bd_ring(rxq, qdev, fp_cqe->bd_num); - goto next_cqe; + ol_flags |= PKT_RX_L4_CKSUM_BAD; + } else { + ol_flags |= PKT_RX_L4_CKSUM_GOOD; + } + if (unlikely(qede_check_notunn_csum_l3(rx_mb, parse_flag))) { + PMD_RX_LOG(ERR, rxq, "IP csum failed, flags = 0x%x\n", + parse_flag); + rxq->rx_hw_errors++; + ol_flags |= PKT_RX_IP_CKSUM_BAD; + } else { + ol_flags |= PKT_RX_IP_CKSUM_GOOD; + packet_type |= qede_rx_cqe_to_pkt_type(parse_flag); + } + + if (CQE_HAS_VLAN(parse_flag) || + CQE_HAS_OUTER_VLAN(parse_flag)) { + /* Note: FW doesn't indicate Q-in-Q packet */ + ol_flags |= PKT_RX_VLAN; + if (qdev->vlan_strip_flg) { + ol_flags |= PKT_RX_VLAN_STRIPPED; + rx_mb->vlan_tci = vlan_tci; + } + } + + /* RSS Hash */ + if (qdev->rss_enable) { + ol_flags |= PKT_RX_RSS_HASH; + rx_mb->hash.rss = rss_hash; } if (unlikely(qede_alloc_rx_buffer(rxq) != 0)) { @@ -941,59 +1512,64 @@ qede_recv_pkts(void *p_rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) rxq->rx_alloc_errors++; break; } - qede_rx_bd_ring_consume(rxq); + if (!tpa_start_flg && fp_cqe->bd_num > 1) { + PMD_RX_LOG(DEBUG, rxq, "Jumbo-over-BD packet: %02x BDs" + " len on first: %04x Total Len: %04x", + fp_cqe->bd_num, len, pkt_len); + num_segs = fp_cqe->bd_num - 1; + seg1 = rx_mb; + if (qede_process_sg_pkts(p_rxq, seg1, num_segs, + pkt_len - len)) + goto next_cqe; + for (j = 0; j < num_segs; j++) { + if (qede_alloc_rx_buffer(rxq)) { + PMD_RX_LOG(ERR, rxq, + "Buffer allocation failed"); + rte_eth_devices[rxq->port_id]. + data->rx_mbuf_alloc_failed++; + rxq->rx_alloc_errors++; + break; + } + rxq->rx_segs++; + } + } + rxq->rx_segs++; /* for the first segment */ + /* Prefetch next mbuf while processing current one. */ preload_idx = rxq->sw_rx_cons & NUM_RX_BDS(rxq); rte_prefetch0(rxq->sw_rx_ring[preload_idx].mbuf); - if (fp_cqe->bd_num != 1) - PMD_RX_LOG(DEBUG, rxq, - "Jumbo-over-BD packet not supported\n"); - - /* Update MBUF fields */ - rx_mb->ol_flags = 0; - rx_mb->data_off = pad + RTE_PKTMBUF_HEADROOM; - rx_mb->nb_segs = 1; - rx_mb->data_len = len; - rx_mb->pkt_len = len; + /* Update rest of the MBUF fields */ + rx_mb->data_off = offset + RTE_PKTMBUF_HEADROOM; rx_mb->port = rxq->port_id; - rx_mb->packet_type = qede_rx_cqe_to_pkt_type(parse_flag); - - htype = (uint8_t)GET_FIELD(fp_cqe->bitfields, - ETH_FAST_PATH_RX_REG_CQE_RSS_HASH_TYPE); - if (qdev->rss_enabled && htype) { - rx_mb->ol_flags |= PKT_RX_RSS_HASH; - rx_mb->hash.rss = rte_le_to_cpu_32(fp_cqe->rss_hash); - PMD_RX_LOG(DEBUG, rxq, "Hash result 0x%x\n", - rx_mb->hash.rss); + rx_mb->ol_flags = ol_flags; + rx_mb->data_len = len; + rx_mb->packet_type = packet_type; +#ifdef RTE_LIBRTE_QEDE_DEBUG_RX + print_rx_bd_info(rx_mb, rxq, bitfield_val); +#endif + if (!tpa_start_flg) { + rx_mb->nb_segs = fp_cqe->bd_num; + rx_mb->pkt_len = pkt_len; + } else { + /* store ref to the updated mbuf */ + tpa_info->tpa_head = rx_mb; + tpa_info->tpa_tail = tpa_info->tpa_head; } - rte_prefetch1(rte_pktmbuf_mtod(rx_mb, void *)); - - if (CQE_HAS_VLAN(parse_flag)) { - rx_mb->vlan_tci = rte_le_to_cpu_16(fp_cqe->vlan_tag); - rx_mb->ol_flags |= PKT_RX_VLAN_PKT; +tpa_end: + if (!tpa_start_flg) { + rx_pkts[rx_pkt] = rx_mb; + rx_pkt++; } - - if (CQE_HAS_OUTER_VLAN(parse_flag)) { - /* FW does not provide indication of Outer VLAN tag, - * which is always stripped, so vlan_tci_outer is set - * to 0. Here vlan_tag represents inner VLAN tag. - */ - rx_mb->vlan_tci = rte_le_to_cpu_16(fp_cqe->vlan_tag); - rx_mb->ol_flags |= PKT_RX_QINQ_PKT; - } - - rx_pkts[rx_pkt] = rx_mb; - rx_pkt++; next_cqe: ecore_chain_recycle_consumed(&rxq->rx_comp_ring); sw_comp_cons = ecore_chain_get_cons_idx(&rxq->rx_comp_ring); if (rx_pkt == nb_pkts) { PMD_RX_LOG(DEBUG, rxq, - "Budget reached nb_pkts=%u received=%u\n", + "Budget reached nb_pkts=%u received=%u", rx_pkt, nb_pkts); break; } @@ -1001,389 +1577,527 @@ next_cqe: qede_update_rx_prod(qdev, rxq); - PMD_RX_LOG(DEBUG, rxq, "rx_pkts=%u core=%d\n", rx_pkt, rte_lcore_id()); + rxq->rcv_pkts += rx_pkt; + + PMD_RX_LOG(DEBUG, rxq, "rx_pkts=%u core=%d", rx_pkt, rte_lcore_id()); return rx_pkt; } -static inline int -qede_free_tx_pkt(struct ecore_dev *edev, struct qede_tx_queue *txq) -{ - uint16_t idx = TX_CONS(txq); - struct eth_tx_bd *tx_data_bd; - struct rte_mbuf *mbuf = txq->sw_tx_ring[idx].mbuf; - if (unlikely(!mbuf)) { - PMD_TX_LOG(ERR, txq, - "null mbuf nb_tx_desc %u nb_tx_avail %u " - "sw_tx_cons %u sw_tx_prod %u\n", - txq->nb_tx_desc, txq->nb_tx_avail, idx, - TX_PROD(txq)); - return -1; +/* Populate scatter gather buffer descriptor fields */ +static inline uint16_t +qede_encode_sg_bd(struct qede_tx_queue *p_txq, struct rte_mbuf *m_seg, + struct eth_tx_2nd_bd **bd2, struct eth_tx_3rd_bd **bd3, + uint16_t start_seg) +{ + struct qede_tx_queue *txq = p_txq; + struct eth_tx_bd *tx_bd = NULL; + dma_addr_t mapping; + uint16_t nb_segs = 0; + + /* Check for scattered buffers */ + while (m_seg) { + if (start_seg == 0) { + if (!*bd2) { + *bd2 = (struct eth_tx_2nd_bd *) + ecore_chain_produce(&txq->tx_pbl); + memset(*bd2, 0, sizeof(struct eth_tx_2nd_bd)); + nb_segs++; + } + mapping = rte_mbuf_data_iova(m_seg); + QEDE_BD_SET_ADDR_LEN(*bd2, mapping, m_seg->data_len); + PMD_TX_LOG(DEBUG, txq, "BD2 len %04x", m_seg->data_len); + } else if (start_seg == 1) { + if (!*bd3) { + *bd3 = (struct eth_tx_3rd_bd *) + ecore_chain_produce(&txq->tx_pbl); + memset(*bd3, 0, sizeof(struct eth_tx_3rd_bd)); + nb_segs++; + } + mapping = rte_mbuf_data_iova(m_seg); + QEDE_BD_SET_ADDR_LEN(*bd3, mapping, m_seg->data_len); + PMD_TX_LOG(DEBUG, txq, "BD3 len %04x", m_seg->data_len); + } else { + tx_bd = (struct eth_tx_bd *) + ecore_chain_produce(&txq->tx_pbl); + memset(tx_bd, 0, sizeof(*tx_bd)); + nb_segs++; + mapping = rte_mbuf_data_iova(m_seg); + QEDE_BD_SET_ADDR_LEN(tx_bd, mapping, m_seg->data_len); + PMD_TX_LOG(DEBUG, txq, "BD len %04x", m_seg->data_len); + } + m_seg = m_seg->next; } - /* Free now */ - rte_pktmbuf_free_seg(mbuf); - txq->sw_tx_ring[idx].mbuf = NULL; - ecore_chain_consume(&txq->tx_pbl); - txq->nb_tx_avail++; - - return 0; + /* Return total scattered buffers */ + return nb_segs; } -static inline uint16_t -qede_process_tx_compl(struct ecore_dev *edev, struct qede_tx_queue *txq) +#ifdef RTE_LIBRTE_QEDE_DEBUG_TX +static inline void +print_tx_bd_info(struct qede_tx_queue *txq, + struct eth_tx_1st_bd *bd1, + struct eth_tx_2nd_bd *bd2, + struct eth_tx_3rd_bd *bd3, + uint64_t tx_ol_flags) { - uint16_t tx_compl = 0; - uint16_t hw_bd_cons; - int rc; + char ol_buf[256] = { 0 }; /* for verbose prints */ - hw_bd_cons = rte_le_to_cpu_16(*txq->hw_cons_ptr); - rte_compiler_barrier(); + if (bd1) + PMD_TX_LOG(INFO, txq, + "BD1: nbytes=0x%04x nbds=0x%04x bd_flags=0x%04x bf=0x%04x", + rte_cpu_to_le_16(bd1->nbytes), bd1->data.nbds, + bd1->data.bd_flags.bitfields, + rte_cpu_to_le_16(bd1->data.bitfields)); + if (bd2) + PMD_TX_LOG(INFO, txq, + "BD2: nbytes=0x%04x bf1=0x%04x bf2=0x%04x tunn_ip=0x%04x\n", + rte_cpu_to_le_16(bd2->nbytes), bd2->data.bitfields1, + bd2->data.bitfields2, bd2->data.tunn_ip_size); + if (bd3) + PMD_TX_LOG(INFO, txq, + "BD3: nbytes=0x%04x bf=0x%04x MSS=0x%04x " + "tunn_l4_hdr_start_offset_w=0x%04x tunn_hdr_size=0x%04x\n", + rte_cpu_to_le_16(bd3->nbytes), + rte_cpu_to_le_16(bd3->data.bitfields), + rte_cpu_to_le_16(bd3->data.lso_mss), + bd3->data.tunn_l4_hdr_start_offset_w, + bd3->data.tunn_hdr_size_w); - while (hw_bd_cons != ecore_chain_get_cons_idx(&txq->tx_pbl)) { - rc = qede_free_tx_pkt(edev, txq); - if (rc) { - DP_NOTICE(edev, false, - "hw_bd_cons = %d, chain_cons=%d\n", - hw_bd_cons, - ecore_chain_get_cons_idx(&txq->tx_pbl)); + rte_get_tx_ol_flag_list(tx_ol_flags, ol_buf, sizeof(ol_buf)); + PMD_TX_LOG(INFO, txq, "TX offloads = %s\n", ol_buf); +} +#endif + +/* TX prepare to check packets meets TX conditions */ +uint16_t +#ifdef RTE_LIBRTE_QEDE_DEBUG_TX +qede_xmit_prep_pkts(void *p_txq, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts) +{ + struct qede_tx_queue *txq = p_txq; +#else +qede_xmit_prep_pkts(__rte_unused void *p_txq, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts) +{ +#endif + uint64_t ol_flags; + struct rte_mbuf *m; + uint16_t i; +#ifdef RTE_LIBRTE_ETHDEV_DEBUG + int ret; +#endif + + for (i = 0; i < nb_pkts; i++) { + m = tx_pkts[i]; + ol_flags = m->ol_flags; + if (ol_flags & PKT_TX_TCP_SEG) { + if (m->nb_segs >= ETH_TX_MAX_BDS_PER_LSO_PACKET) { + rte_errno = -EINVAL; + break; + } + /* TBD: confirm its ~9700B for both ? */ + if (m->tso_segsz > ETH_TX_MAX_NON_LSO_PKT_LEN) { + rte_errno = -EINVAL; + break; + } + } else { + if (m->nb_segs >= ETH_TX_MAX_BDS_PER_NON_LSO_PACKET) { + rte_errno = -EINVAL; + break; + } + } + if (ol_flags & QEDE_TX_OFFLOAD_NOTSUP_MASK) { + rte_errno = -ENOTSUP; break; } - txq->sw_tx_cons++; /* Making TXD available */ - tx_compl++; + +#ifdef RTE_LIBRTE_ETHDEV_DEBUG + ret = rte_validate_tx_offload(m); + if (ret != 0) { + rte_errno = ret; + break; + } +#endif } - PMD_TX_LOG(DEBUG, txq, "Tx compl %u sw_tx_cons %u avail %u\n", - tx_compl, txq->sw_tx_cons, txq->nb_tx_avail); - return tx_compl; +#ifdef RTE_LIBRTE_QEDE_DEBUG_TX + if (unlikely(i != nb_pkts)) + PMD_TX_LOG(ERR, txq, "TX prepare failed for %u\n", + nb_pkts - i); +#endif + return i; } +#define MPLSINUDP_HDR_SIZE (12) + +#ifdef RTE_LIBRTE_QEDE_DEBUG_TX +static inline void +qede_mpls_tunn_tx_sanity_check(struct rte_mbuf *mbuf, + struct qede_tx_queue *txq) +{ + if (((mbuf->outer_l2_len + mbuf->outer_l3_len) / 2) > 0xff) + PMD_TX_LOG(ERR, txq, "tunn_l4_hdr_start_offset overflow\n"); + if (((mbuf->outer_l2_len + mbuf->outer_l3_len + + MPLSINUDP_HDR_SIZE) / 2) > 0xff) + PMD_TX_LOG(ERR, txq, "tunn_hdr_size overflow\n"); + if (((mbuf->l2_len - MPLSINUDP_HDR_SIZE) / 2) > + ETH_TX_DATA_2ND_BD_TUNN_INNER_L2_HDR_SIZE_W_MASK) + PMD_TX_LOG(ERR, txq, "inner_l2_hdr_size overflow\n"); + if (((mbuf->l2_len - MPLSINUDP_HDR_SIZE + mbuf->l3_len) / 2) > + ETH_TX_DATA_2ND_BD_L4_HDR_START_OFFSET_W_MASK) + PMD_TX_LOG(ERR, txq, "inner_l2_hdr_size overflow\n"); +} +#endif + uint16_t qede_xmit_pkts(void *p_txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { struct qede_tx_queue *txq = p_txq; struct qede_dev *qdev = txq->qdev; struct ecore_dev *edev = &qdev->edev; - struct qede_fastpath *fp = &qdev->fp_array[txq->queue_id]; - struct eth_tx_1st_bd *first_bd; + struct rte_mbuf *mbuf; + struct rte_mbuf *m_seg = NULL; uint16_t nb_tx_pkts; - uint16_t nb_pkt_sent = 0; uint16_t bd_prod; uint16_t idx; - uint16_t tx_count; + uint16_t nb_frags; + uint16_t nb_pkt_sent = 0; + uint8_t nbds; + bool lso_flg; + bool mplsoudp_flg; + __rte_unused bool tunn_flg; + bool tunn_ipv6_ext_flg; + struct eth_tx_1st_bd *bd1; + struct eth_tx_2nd_bd *bd2; + struct eth_tx_3rd_bd *bd3; + uint64_t tx_ol_flags; + uint16_t hdr_size; + /* BD1 */ + uint16_t bd1_bf; + uint8_t bd1_bd_flags_bf; + uint16_t vlan; + /* BD2 */ + uint16_t bd2_bf1; + uint16_t bd2_bf2; + /* BD3 */ + uint16_t mss; + uint16_t bd3_bf; + + uint8_t tunn_l4_hdr_start_offset; + uint8_t tunn_hdr_size; + uint8_t inner_l2_hdr_size; + uint16_t inner_l4_hdr_offset; if (unlikely(txq->nb_tx_avail < txq->tx_free_thresh)) { - PMD_TX_LOG(DEBUG, txq, "send=%u avail=%u free_thresh=%u\n", + PMD_TX_LOG(DEBUG, txq, "send=%u avail=%u free_thresh=%u", nb_pkts, txq->nb_tx_avail, txq->tx_free_thresh); - (void)qede_process_tx_compl(edev, txq); + qede_process_tx_compl(edev, txq); } - nb_tx_pkts = RTE_MIN(nb_pkts, (txq->nb_tx_avail / MAX_NUM_TX_BDS)); - if (unlikely(nb_tx_pkts == 0)) { - PMD_TX_LOG(DEBUG, txq, "Out of BDs nb_pkts=%u avail=%u\n", - nb_pkts, txq->nb_tx_avail); - return 0; - } - - tx_count = nb_tx_pkts; + nb_tx_pkts = nb_pkts; + bd_prod = rte_cpu_to_le_16(ecore_chain_get_prod_idx(&txq->tx_pbl)); while (nb_tx_pkts--) { - /* Fill the entry in the SW ring and the BDs in the FW ring */ - idx = TX_PROD(txq); - struct rte_mbuf *mbuf = *tx_pkts++; - txq->sw_tx_ring[idx].mbuf = mbuf; - first_bd = (struct eth_tx_1st_bd *) - ecore_chain_produce(&txq->tx_pbl); - first_bd->data.bd_flags.bitfields = - 1 << ETH_TX_1ST_BD_FLAGS_START_BD_SHIFT; - /* Map MBUF linear data for DMA and set in the first BD */ - QEDE_BD_SET_ADDR_LEN(first_bd, rte_mbuf_data_dma_addr(mbuf), - mbuf->data_len); + /* Init flags/values */ + tunn_flg = false; + lso_flg = false; + nbds = 0; + vlan = 0; + bd1 = NULL; + bd2 = NULL; + bd3 = NULL; + hdr_size = 0; + bd1_bf = 0; + bd1_bd_flags_bf = 0; + bd2_bf1 = 0; + bd2_bf2 = 0; + mss = 0; + bd3_bf = 0; + mplsoudp_flg = false; + tunn_ipv6_ext_flg = false; + tunn_hdr_size = 0; + tunn_l4_hdr_start_offset = 0; + + mbuf = *tx_pkts++; + assert(mbuf); + + /* Check minimum TX BDS availability against available BDs */ + if (unlikely(txq->nb_tx_avail < mbuf->nb_segs)) + break; + + tx_ol_flags = mbuf->ol_flags; + bd1_bd_flags_bf |= 1 << ETH_TX_1ST_BD_FLAGS_START_BD_SHIFT; + + /* TX prepare would have already checked supported tunnel Tx + * offloads. Don't rely on pkt_type marked by Rx, instead use + * tx_ol_flags to decide. + */ + if (((tx_ol_flags & PKT_TX_TUNNEL_MASK) == + PKT_TX_TUNNEL_VXLAN) || + ((tx_ol_flags & PKT_TX_TUNNEL_MASK) == + PKT_TX_TUNNEL_MPLSINUDP)) { + /* Check against max which is Tunnel IPv6 + ext */ + if (unlikely(txq->nb_tx_avail < + ETH_TX_MIN_BDS_PER_TUNN_IPV6_WITH_EXT_PKT)) + break; + tunn_flg = true; + /* First indicate its a tunnel pkt */ + bd1_bf |= ETH_TX_DATA_1ST_BD_TUNN_FLAG_MASK << + ETH_TX_DATA_1ST_BD_TUNN_FLAG_SHIFT; + /* Legacy FW had flipped behavior in regard to this bit + * i.e. it needed to set to prevent FW from touching + * encapsulated packets when it didn't need to. + */ + if (unlikely(txq->is_legacy)) { + bd1_bf ^= 1 << + ETH_TX_DATA_1ST_BD_TUNN_FLAG_SHIFT; + } + + /* Outer IP checksum offload */ + if (tx_ol_flags & (PKT_TX_OUTER_IP_CKSUM | + PKT_TX_OUTER_IPV4)) { + bd1_bd_flags_bf |= + ETH_TX_1ST_BD_FLAGS_TUNN_IP_CSUM_MASK << + ETH_TX_1ST_BD_FLAGS_TUNN_IP_CSUM_SHIFT; + } + + /** + * Currently, only inner checksum offload in MPLS-in-UDP + * tunnel with one MPLS label is supported. Both outer + * and inner layers lengths need to be provided in + * mbuf. + */ + if ((tx_ol_flags & PKT_TX_TUNNEL_MASK) == + PKT_TX_TUNNEL_MPLSINUDP) { + mplsoudp_flg = true; +#ifdef RTE_LIBRTE_QEDE_DEBUG_TX + qede_mpls_tunn_tx_sanity_check(mbuf, txq); +#endif + /* Outer L4 offset in two byte words */ + tunn_l4_hdr_start_offset = + (mbuf->outer_l2_len + mbuf->outer_l3_len) / 2; + /* Tunnel header size in two byte words */ + tunn_hdr_size = (mbuf->outer_l2_len + + mbuf->outer_l3_len + + MPLSINUDP_HDR_SIZE) / 2; + /* Inner L2 header size in two byte words */ + inner_l2_hdr_size = (mbuf->l2_len - + MPLSINUDP_HDR_SIZE) / 2; + /* Inner L4 header offset from the beggining + * of inner packet in two byte words + */ + inner_l4_hdr_offset = (mbuf->l2_len - + MPLSINUDP_HDR_SIZE + mbuf->l3_len) / 2; + + /* Inner L2 size and address type */ + bd2_bf1 |= (inner_l2_hdr_size & + ETH_TX_DATA_2ND_BD_TUNN_INNER_L2_HDR_SIZE_W_MASK) << + ETH_TX_DATA_2ND_BD_TUNN_INNER_L2_HDR_SIZE_W_SHIFT; + bd2_bf1 |= (UNICAST_ADDRESS & + ETH_TX_DATA_2ND_BD_TUNN_INNER_ETH_TYPE_MASK) << + ETH_TX_DATA_2ND_BD_TUNN_INNER_ETH_TYPE_SHIFT; + /* Treated as IPv6+Ext */ + bd2_bf1 |= + 1 << ETH_TX_DATA_2ND_BD_TUNN_IPV6_EXT_SHIFT; + + /* Mark inner IPv6 if present */ + if (tx_ol_flags & PKT_TX_IPV6) + bd2_bf1 |= + 1 << ETH_TX_DATA_2ND_BD_TUNN_INNER_IPV6_SHIFT; + + /* Inner L4 offsets */ + if ((tx_ol_flags & (PKT_TX_IPV4 | PKT_TX_IPV6)) && + (tx_ol_flags & (PKT_TX_UDP_CKSUM | + PKT_TX_TCP_CKSUM))) { + /* Determines if BD3 is needed */ + tunn_ipv6_ext_flg = true; + if ((tx_ol_flags & PKT_TX_L4_MASK) == + PKT_TX_UDP_CKSUM) { + bd2_bf1 |= + 1 << ETH_TX_DATA_2ND_BD_L4_UDP_SHIFT; + } + + /* TODO other pseudo checksum modes are + * not supported + */ + bd2_bf1 |= + ETH_L4_PSEUDO_CSUM_CORRECT_LENGTH << + ETH_TX_DATA_2ND_BD_L4_PSEUDO_CSUM_MODE_SHIFT; + bd2_bf2 |= (inner_l4_hdr_offset & + ETH_TX_DATA_2ND_BD_L4_HDR_START_OFFSET_W_MASK) << + ETH_TX_DATA_2ND_BD_L4_HDR_START_OFFSET_W_SHIFT; + } + } /* End MPLSoUDP */ + } /* End Tunnel handling */ + + if (tx_ol_flags & PKT_TX_TCP_SEG) { + lso_flg = true; + if (unlikely(txq->nb_tx_avail < + ETH_TX_MIN_BDS_PER_LSO_PKT)) + break; + /* For LSO, packet header and payload must reside on + * buffers pointed by different BDs. Using BD1 for HDR + * and BD2 onwards for data. + */ + hdr_size = mbuf->l2_len + mbuf->l3_len + mbuf->l4_len; + if (tunn_flg) + hdr_size += mbuf->outer_l2_len + + mbuf->outer_l3_len; + + bd1_bd_flags_bf |= 1 << ETH_TX_1ST_BD_FLAGS_LSO_SHIFT; + bd1_bd_flags_bf |= + 1 << ETH_TX_1ST_BD_FLAGS_IP_CSUM_SHIFT; + /* PKT_TX_TCP_SEG implies PKT_TX_TCP_CKSUM */ + bd1_bd_flags_bf |= + 1 << ETH_TX_1ST_BD_FLAGS_L4_CSUM_SHIFT; + mss = rte_cpu_to_le_16(mbuf->tso_segsz); + /* Using one header BD */ + bd3_bf |= rte_cpu_to_le_16(1 << + ETH_TX_DATA_3RD_BD_HDR_NBD_SHIFT); + } else { + if (unlikely(txq->nb_tx_avail < + ETH_TX_MIN_BDS_PER_NON_LSO_PKT)) + break; + bd1_bf |= + (mbuf->pkt_len & ETH_TX_DATA_1ST_BD_PKT_LEN_MASK) + << ETH_TX_DATA_1ST_BD_PKT_LEN_SHIFT; + } /* Descriptor based VLAN insertion */ - if (mbuf->ol_flags & (PKT_TX_VLAN_PKT | PKT_TX_QINQ_PKT)) { - first_bd->data.vlan = rte_cpu_to_le_16(mbuf->vlan_tci); - first_bd->data.bd_flags.bitfields |= + if (tx_ol_flags & (PKT_TX_VLAN_PKT | PKT_TX_QINQ_PKT)) { + vlan = rte_cpu_to_le_16(mbuf->vlan_tci); + bd1_bd_flags_bf |= 1 << ETH_TX_1ST_BD_FLAGS_VLAN_INSERTION_SHIFT; } /* Offload the IP checksum in the hardware */ - if (mbuf->ol_flags & PKT_TX_IP_CKSUM) { - first_bd->data.bd_flags.bitfields |= - 1 << ETH_TX_1ST_BD_FLAGS_IP_CSUM_SHIFT; + if (tx_ol_flags & PKT_TX_IP_CKSUM) { + bd1_bd_flags_bf |= + 1 << ETH_TX_1ST_BD_FLAGS_IP_CSUM_SHIFT; + /* There's no DPDK flag to request outer-L4 csum + * offload. But in the case of tunnel if inner L3 or L4 + * csum offload is requested then we need to force + * recalculation of L4 tunnel header csum also. + */ + if (tunn_flg) { + bd1_bd_flags_bf |= + ETH_TX_1ST_BD_FLAGS_TUNN_L4_CSUM_MASK << + ETH_TX_1ST_BD_FLAGS_TUNN_L4_CSUM_SHIFT; + } } /* L4 checksum offload (tcp or udp) */ - if (mbuf->ol_flags & (PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM)) { - first_bd->data.bd_flags.bitfields |= - 1 << ETH_TX_1ST_BD_FLAGS_L4_CSUM_SHIFT; - /* IPv6 + extn. -> later */ + if ((tx_ol_flags & (PKT_TX_IPV4 | PKT_TX_IPV6)) && + (tx_ol_flags & (PKT_TX_UDP_CKSUM | PKT_TX_TCP_CKSUM))) { + bd1_bd_flags_bf |= + 1 << ETH_TX_1ST_BD_FLAGS_L4_CSUM_SHIFT; + /* There's no DPDK flag to request outer-L4 csum + * offload. But in the case of tunnel if inner L3 or L4 + * csum offload is requested then we need to force + * recalculation of L4 tunnel header csum also. + */ + if (tunn_flg) { + bd1_bd_flags_bf |= + ETH_TX_1ST_BD_FLAGS_TUNN_L4_CSUM_MASK << + ETH_TX_1ST_BD_FLAGS_TUNN_L4_CSUM_SHIFT; + } } - first_bd->data.nbds = MAX_NUM_TX_BDS; + + /* Fill the entry in the SW ring and the BDs in the FW ring */ + idx = TX_PROD(txq); + txq->sw_tx_ring[idx].mbuf = mbuf; + + /* BD1 */ + bd1 = (struct eth_tx_1st_bd *)ecore_chain_produce(&txq->tx_pbl); + memset(bd1, 0, sizeof(struct eth_tx_1st_bd)); + nbds++; + + /* Map MBUF linear data for DMA and set in the BD1 */ + QEDE_BD_SET_ADDR_LEN(bd1, rte_mbuf_data_iova(mbuf), + mbuf->data_len); + bd1->data.bitfields = rte_cpu_to_le_16(bd1_bf); + bd1->data.bd_flags.bitfields = bd1_bd_flags_bf; + bd1->data.vlan = vlan; + + if (lso_flg || mplsoudp_flg) { + bd2 = (struct eth_tx_2nd_bd *)ecore_chain_produce + (&txq->tx_pbl); + memset(bd2, 0, sizeof(struct eth_tx_2nd_bd)); + nbds++; + + /* BD1 */ + QEDE_BD_SET_ADDR_LEN(bd1, rte_mbuf_data_iova(mbuf), + hdr_size); + /* BD2 */ + QEDE_BD_SET_ADDR_LEN(bd2, (hdr_size + + rte_mbuf_data_iova(mbuf)), + mbuf->data_len - hdr_size); + bd2->data.bitfields1 = rte_cpu_to_le_16(bd2_bf1); + if (mplsoudp_flg) { + bd2->data.bitfields2 = + rte_cpu_to_le_16(bd2_bf2); + /* Outer L3 size */ + bd2->data.tunn_ip_size = + rte_cpu_to_le_16(mbuf->outer_l3_len); + } + /* BD3 */ + if (lso_flg || (mplsoudp_flg && tunn_ipv6_ext_flg)) { + bd3 = (struct eth_tx_3rd_bd *) + ecore_chain_produce(&txq->tx_pbl); + memset(bd3, 0, sizeof(struct eth_tx_3rd_bd)); + nbds++; + bd3->data.bitfields = rte_cpu_to_le_16(bd3_bf); + if (lso_flg) + bd3->data.lso_mss = mss; + if (mplsoudp_flg) { + bd3->data.tunn_l4_hdr_start_offset_w = + tunn_l4_hdr_start_offset; + bd3->data.tunn_hdr_size_w = + tunn_hdr_size; + } + } + } + + /* Handle fragmented MBUF */ + m_seg = mbuf->next; + + /* Encode scatter gather buffer descriptors if required */ + nb_frags = qede_encode_sg_bd(txq, m_seg, &bd2, &bd3, nbds - 1); + bd1->data.nbds = nbds + nb_frags; + + txq->nb_tx_avail -= bd1->data.nbds; txq->sw_tx_prod++; rte_prefetch0(txq->sw_tx_ring[TX_PROD(txq)].mbuf); - txq->nb_tx_avail--; bd_prod = rte_cpu_to_le_16(ecore_chain_get_prod_idx(&txq->tx_pbl)); +#ifdef RTE_LIBRTE_QEDE_DEBUG_TX + print_tx_bd_info(txq, bd1, bd2, bd3, tx_ol_flags); +#endif nb_pkt_sent++; + txq->xmit_pkts++; } /* Write value of prod idx into bd_prod */ txq->tx_db.data.bd_prod = bd_prod; rte_wmb(); rte_compiler_barrier(); - DIRECT_REG_WR(edev, txq->doorbell_addr, txq->tx_db.raw); + DIRECT_REG_WR_RELAXED(edev, txq->doorbell_addr, txq->tx_db.raw); rte_wmb(); /* Check again for Tx completions */ - (void)qede_process_tx_compl(edev, txq); + qede_process_tx_compl(edev, txq); - PMD_TX_LOG(DEBUG, txq, "to_send=%u can_send=%u sent=%u core=%d\n", - nb_pkts, tx_count, nb_pkt_sent, rte_lcore_id()); + PMD_TX_LOG(DEBUG, txq, "to_send=%u sent=%u bd_prod=%u core=%d", + nb_pkts, nb_pkt_sent, TX_PROD(txq), rte_lcore_id()); return nb_pkt_sent; } -int qede_dev_start(struct rte_eth_dev *eth_dev) +uint16_t +qede_rxtx_pkts_dummy(__rte_unused void *p_rxq, + __rte_unused struct rte_mbuf **pkts, + __rte_unused uint16_t nb_pkts) { - struct qede_dev *qdev = eth_dev->data->dev_private; - struct ecore_dev *edev = &qdev->edev; - struct qed_link_output link_output; - int rc; - - DP_INFO(edev, "port %u\n", eth_dev->data->port_id); - - if (qdev->state == QEDE_START) { - DP_INFO(edev, "device already started\n"); - return 0; - } - - if (qdev->state == QEDE_CLOSE) { - rc = qede_alloc_fp_array(qdev); - qede_init_fp(qdev); - rc = qede_alloc_mem_load(qdev); - DP_INFO(edev, "Allocated %d RSS queues on %d TC/s\n", - QEDE_RSS_CNT(qdev), qdev->num_tc); - } else if (qdev->state == QEDE_STOP) { - DP_INFO(edev, "restarting port %u\n", eth_dev->data->port_id); - } else { - DP_INFO(edev, "unknown state port %u\n", - eth_dev->data->port_id); - return -EINVAL; - } - - rc = qede_start_queues(eth_dev, true); - - if (rc) { - DP_ERR(edev, "Failed to start queues\n"); - /* TBD: free */ - return rc; - } - - DP_INFO(edev, "Start VPORT, RXQ and TXQ succeeded\n"); - - qede_dev_set_link_state(eth_dev, true); - - /* Query whether link is already-up */ - memset(&link_output, 0, sizeof(link_output)); - qdev->ops->common->get_link(edev, &link_output); - DP_NOTICE(edev, false, "link status: %s\n", - link_output.link_up ? "up" : "down"); - - qdev->state = QEDE_START; - - qede_config_rx_mode(eth_dev); - - DP_INFO(edev, "dev_state is QEDE_START\n"); - return 0; } - -static int qede_drain_txq(struct qede_dev *qdev, - struct qede_tx_queue *txq, bool allow_drain) -{ - struct ecore_dev *edev = &qdev->edev; - int rc, cnt = 1000; - - while (txq->sw_tx_cons != txq->sw_tx_prod) { - qede_process_tx_compl(edev, txq); - if (!cnt) { - if (allow_drain) { - DP_NOTICE(edev, false, - "Tx queue[%u] is stuck," - "requesting MCP to drain\n", - txq->queue_id); - rc = qdev->ops->common->drain(edev); - if (rc) - return rc; - return qede_drain_txq(qdev, txq, false); - } - - DP_NOTICE(edev, false, - "Timeout waiting for tx queue[%d]:" - "PROD=%d, CONS=%d\n", - txq->queue_id, txq->sw_tx_prod, - txq->sw_tx_cons); - return -ENODEV; - } - cnt--; - DELAY(1000); - rte_compiler_barrier(); - } - - /* FW finished processing, wait for HW to transmit all tx packets */ - DELAY(2000); - - return 0; -} - -static int qede_stop_queues(struct qede_dev *qdev) -{ - struct qed_update_vport_params vport_update_params; - struct ecore_dev *edev = &qdev->edev; - int rc, tc, i; - - /* Disable the vport */ - memset(&vport_update_params, 0, sizeof(vport_update_params)); - vport_update_params.vport_id = 0; - vport_update_params.update_vport_active_flg = 1; - vport_update_params.vport_active_flg = 0; - vport_update_params.update_rss_flg = 0; - - DP_INFO(edev, "vport_update\n"); - - rc = qdev->ops->vport_update(edev, &vport_update_params); - if (rc) { - DP_ERR(edev, "Failed to update vport\n"); - return rc; - } - - DP_INFO(edev, "Flushing tx queues\n"); - - /* Flush Tx queues. If needed, request drain from MCP */ - for_each_rss(i) { - struct qede_fastpath *fp = &qdev->fp_array[i]; - for (tc = 0; tc < qdev->num_tc; tc++) { - struct qede_tx_queue *txq = fp->txqs[tc]; - rc = qede_drain_txq(qdev, txq, true); - if (rc) - return rc; - } - } - - /* Stop all Queues in reverse order */ - for (i = QEDE_RSS_CNT(qdev) - 1; i >= 0; i--) { - struct qed_stop_rxq_params rx_params; - - /* Stop the Tx Queue(s) */ - for (tc = 0; tc < qdev->num_tc; tc++) { - struct qed_stop_txq_params tx_params; - - tx_params.rss_id = i; - tx_params.tx_queue_id = tc * QEDE_RSS_CNT(qdev) + i; - - DP_INFO(edev, "Stopping tx queues\n"); - rc = qdev->ops->q_tx_stop(edev, &tx_params); - if (rc) { - DP_ERR(edev, "Failed to stop TXQ #%d\n", - tx_params.tx_queue_id); - return rc; - } - } - - /* Stop the Rx Queue */ - memset(&rx_params, 0, sizeof(rx_params)); - rx_params.rss_id = i; - rx_params.rx_queue_id = i; - rx_params.eq_completion_only = 1; - - DP_INFO(edev, "Stopping rx queues\n"); - - rc = qdev->ops->q_rx_stop(edev, &rx_params); - if (rc) { - DP_ERR(edev, "Failed to stop RXQ #%d\n", i); - return rc; - } - } - - DP_INFO(edev, "Stopping vports\n"); - - /* Stop the vport */ - rc = qdev->ops->vport_stop(edev, 0); - if (rc) - DP_ERR(edev, "Failed to stop VPORT\n"); - - return rc; -} - -void qede_reset_fp_rings(struct qede_dev *qdev) -{ - uint16_t rss_id; - uint8_t tc; - - for_each_rss(rss_id) { - DP_INFO(&qdev->edev, "reset fp chain for rss %u\n", rss_id); - struct qede_fastpath *fp = &qdev->fp_array[rss_id]; - ecore_chain_reset(&fp->rxq->rx_bd_ring); - ecore_chain_reset(&fp->rxq->rx_comp_ring); - for (tc = 0; tc < qdev->num_tc; tc++) { - struct qede_tx_queue *txq = fp->txqs[tc]; - ecore_chain_reset(&txq->tx_pbl); - } - } -} - -/* This function frees all memory of a single fp */ -static void qede_free_mem_fp(struct qede_dev *qdev, struct qede_fastpath *fp) -{ - uint8_t tc; - - qede_rx_queue_release(fp->rxq); - for (tc = 0; tc < qdev->num_tc; tc++) - qede_tx_queue_release(fp->txqs[tc]); -} - -void qede_free_mem_load(struct qede_dev *qdev) -{ - uint8_t rss_id; - - for_each_rss(rss_id) { - struct qede_fastpath *fp = &qdev->fp_array[rss_id]; - qede_free_mem_fp(qdev, fp); - } - /* qdev->num_rss = 0; */ -} - -/* - * Stop an Ethernet device. The device can be restarted with a call to - * rte_eth_dev_start(). - * Do not change link state and do not release sw structures. - */ -void qede_dev_stop(struct rte_eth_dev *eth_dev) -{ - struct qede_dev *qdev = eth_dev->data->dev_private; - struct ecore_dev *edev = &qdev->edev; - int rc; - - DP_INFO(edev, "port %u\n", eth_dev->data->port_id); - - if (qdev->state != QEDE_START) { - DP_INFO(edev, "device not yet started\n"); - return; - } - - rc = qede_stop_queues(qdev); - - if (rc) - DP_ERR(edev, "Didn't succeed to close queues\n"); - - DP_INFO(edev, "Stopped queues\n"); - - qdev->ops->fastpath_stop(edev); - - qede_reset_fp_rings(qdev); - - qdev->state = QEDE_STOP; - - DP_INFO(edev, "dev_state is QEDE_STOP\n"); -} diff --git a/dpdk/drivers/net/qede/qede_rxtx.h b/dpdk/drivers/net/qede/qede_rxtx.h index 34eaf8b6..ae88206d 100644 --- a/dpdk/drivers/net/qede/qede_rxtx.h +++ b/dpdk/drivers/net/qede/qede_rxtx.h @@ -30,9 +30,6 @@ #define TX_CONS(txq) (txq->sw_tx_cons & NUM_TX_BDS(txq)) #define TX_PROD(txq) (txq->sw_tx_prod & NUM_TX_BDS(txq)) -/* Number of TX BDs per packet used currently */ -#define MAX_NUM_TX_BDS 1 - #define QEDE_DEFAULT_TX_FREE_THRESH 32 #define QEDE_CSUM_ERROR (1 << 0) @@ -54,24 +51,110 @@ ((flags) & (PARSING_AND_ERR_FLAGS_TUNNEL8021QTAGEXIST_MASK \ << PARSING_AND_ERR_FLAGS_TUNNEL8021QTAGEXIST_SHIFT)) +#define QEDE_MIN_RX_BUFF_SIZE (1024) +#define QEDE_VLAN_TAG_SIZE (4) +#define QEDE_LLC_SNAP_HDR_LEN (8) + /* Max supported alignment is 256 (8 shift) * minimal alignment shift 6 is optimal for 57xxx HW performance */ #define QEDE_L1_CACHE_SHIFT 6 #define QEDE_RX_ALIGN_SHIFT (RTE_MAX(6, RTE_MIN(8, QEDE_L1_CACHE_SHIFT))) #define QEDE_FW_RX_ALIGN_END (1UL << QEDE_RX_ALIGN_SHIFT) +#define QEDE_CEIL_TO_CACHE_LINE_SIZE(n) (((n) + (QEDE_FW_RX_ALIGN_END - 1)) & \ + ~(QEDE_FW_RX_ALIGN_END - 1)) +/* Note: QEDE_LLC_SNAP_HDR_LEN is optional */ +#define QEDE_ETH_OVERHEAD (((2 * QEDE_VLAN_TAG_SIZE)) - (ETHER_CRC_LEN) \ + + (QEDE_LLC_SNAP_HDR_LEN)) -#define QEDE_ETH_OVERHEAD (ETHER_HDR_LEN + 8 + 8 + QEDE_FW_RX_ALIGN_END) - -/* TBD: Excluding IPV6 */ -#define QEDE_RSS_OFFLOAD_ALL (ETH_RSS_IPV4 | ETH_RSS_NONFRAG_IPV4_TCP | \ - ETH_RSS_NONFRAG_IPV4_UDP) +#define QEDE_RSS_OFFLOAD_ALL (ETH_RSS_IPV4 |\ + ETH_RSS_NONFRAG_IPV4_TCP |\ + ETH_RSS_NONFRAG_IPV4_UDP |\ + ETH_RSS_IPV6 |\ + ETH_RSS_NONFRAG_IPV6_TCP |\ + ETH_RSS_NONFRAG_IPV6_UDP |\ + ETH_RSS_VXLAN) #define QEDE_TXQ_FLAGS ((uint32_t)ETH_TXQ_FLAGS_NOMULTSEGS) -#define MAX_NUM_TC 8 +#define for_each_rss(i) for (i = 0; i < qdev->num_rx_queues; i++) +#define for_each_tss(i) for (i = 0; i < qdev->num_tx_queues; i++) +#define QEDE_RXTX_MAX(qdev) \ + (RTE_MAX(QEDE_RSS_COUNT(qdev), QEDE_TSS_COUNT(qdev))) -#define for_each_rss(i) for (i = 0; i < qdev->num_rss; i++) +/* Macros for non-tunnel packet types lkup table */ +#define QEDE_PKT_TYPE_UNKNOWN 0x0 +#define QEDE_PKT_TYPE_MAX 0x3f + +#define QEDE_PKT_TYPE_IPV4 0x1 +#define QEDE_PKT_TYPE_IPV6 0x2 +#define QEDE_PKT_TYPE_IPV4_TCP 0x5 +#define QEDE_PKT_TYPE_IPV6_TCP 0x6 +#define QEDE_PKT_TYPE_IPV4_UDP 0x9 +#define QEDE_PKT_TYPE_IPV6_UDP 0xa + +/* For frag pkts, corresponding IP bits is set */ +#define QEDE_PKT_TYPE_IPV4_FRAG 0x11 +#define QEDE_PKT_TYPE_IPV6_FRAG 0x12 + +#define QEDE_PKT_TYPE_IPV4_VLAN 0x21 +#define QEDE_PKT_TYPE_IPV6_VLAN 0x22 +#define QEDE_PKT_TYPE_IPV4_TCP_VLAN 0x25 +#define QEDE_PKT_TYPE_IPV6_TCP_VLAN 0x26 +#define QEDE_PKT_TYPE_IPV4_UDP_VLAN 0x29 +#define QEDE_PKT_TYPE_IPV6_UDP_VLAN 0x2a + +#define QEDE_PKT_TYPE_IPV4_VLAN_FRAG 0x31 +#define QEDE_PKT_TYPE_IPV6_VLAN_FRAG 0x32 + +/* Macros for tunneled packets with next protocol lkup table */ +#define QEDE_PKT_TYPE_TUNN_GENEVE 0x1 +#define QEDE_PKT_TYPE_TUNN_GRE 0x2 +#define QEDE_PKT_TYPE_TUNN_VXLAN 0x3 + +/* Bit 2 is don't care bit */ +#define QEDE_PKT_TYPE_TUNN_L2_TENID_NOEXIST_GENEVE 0x9 +#define QEDE_PKT_TYPE_TUNN_L2_TENID_NOEXIST_GRE 0xa +#define QEDE_PKT_TYPE_TUNN_L2_TENID_NOEXIST_VXLAN 0xb + +#define QEDE_PKT_TYPE_TUNN_L2_TENID_EXIST_GENEVE 0xd +#define QEDE_PKT_TYPE_TUNN_L2_TENID_EXIST_GRE 0xe +#define QEDE_PKT_TYPE_TUNN_L2_TENID_EXIST_VXLAN 0xf + + +#define QEDE_PKT_TYPE_TUNN_IPV4_TENID_NOEXIST_GENEVE 0x11 +#define QEDE_PKT_TYPE_TUNN_IPV4_TENID_NOEXIST_GRE 0x12 +#define QEDE_PKT_TYPE_TUNN_IPV4_TENID_NOEXIST_VXLAN 0x13 + +#define QEDE_PKT_TYPE_TUNN_IPV4_TENID_EXIST_GENEVE 0x15 +#define QEDE_PKT_TYPE_TUNN_IPV4_TENID_EXIST_GRE 0x16 +#define QEDE_PKT_TYPE_TUNN_IPV4_TENID_EXIST_VXLAN 0x17 + + +#define QEDE_PKT_TYPE_TUNN_IPV6_TENID_NOEXIST_GENEVE 0x19 +#define QEDE_PKT_TYPE_TUNN_IPV6_TENID_NOEXIST_GRE 0x1a +#define QEDE_PKT_TYPE_TUNN_IPV6_TENID_NOEXIST_VXLAN 0x1b + +#define QEDE_PKT_TYPE_TUNN_IPV6_TENID_EXIST_GENEVE 0x1d +#define QEDE_PKT_TYPE_TUNN_IPV6_TENID_EXIST_GRE 0x1e +#define QEDE_PKT_TYPE_TUNN_IPV6_TENID_EXIST_VXLAN 0x1f + +#define QEDE_PKT_TYPE_TUNN_MAX_TYPE 0x20 /* 2^5 */ + +#define QEDE_TX_CSUM_OFFLOAD_MASK (PKT_TX_IP_CKSUM | \ + PKT_TX_TCP_CKSUM | \ + PKT_TX_UDP_CKSUM | \ + PKT_TX_OUTER_IP_CKSUM | \ + PKT_TX_TCP_SEG) + +#define QEDE_TX_OFFLOAD_MASK (QEDE_TX_CSUM_OFFLOAD_MASK | \ + PKT_TX_QINQ_PKT | \ + PKT_TX_VLAN_PKT | \ + PKT_TX_TUNNEL_VXLAN | \ + PKT_TX_TUNNEL_MPLSINUDP) + +#define QEDE_TX_OFFLOAD_NOTSUP_MASK \ + (PKT_TX_OFFLOAD_MASK ^ QEDE_TX_OFFLOAD_MASK) /* * RX BD descriptor ring @@ -82,6 +165,12 @@ struct qede_rx_entry { /* allows expansion .. */ }; +/* TPA related structures */ +struct qede_agg_info { + struct rte_mbuf *tpa_head; /* Pointer to first TPA segment */ + struct rte_mbuf *tpa_tail; /* Pointer to last TPA segment */ +}; + /* * Structure associated with each RX queue. */ @@ -92,15 +181,20 @@ struct qede_rx_queue { uint16_t *hw_cons_ptr; void OSAL_IOMEM *hw_rxq_prod_addr; struct qede_rx_entry *sw_rx_ring; + struct ecore_sb_info *sb_info; uint16_t sw_rx_cons; uint16_t sw_rx_prod; uint16_t nb_rx_desc; uint16_t queue_id; uint16_t port_id; uint16_t rx_buf_size; + uint64_t rcv_pkts; + uint64_t rx_segs; uint64_t rx_hw_errors; uint64_t rx_alloc_errors; + struct qede_agg_info tpa_info[ETH_TPA_MAX_AGGS_NUM]; struct qede_dev *qdev; + void *handle; }; /* @@ -129,17 +223,16 @@ struct qede_tx_queue { void OSAL_IOMEM *doorbell_addr; volatile union db_prod tx_db; uint16_t port_id; - uint64_t txq_counter; + uint64_t xmit_pkts; + bool is_legacy; struct qede_dev *qdev; + void *handle; }; struct qede_fastpath { - struct qede_dev *qdev; - uint8_t rss_id; struct ecore_sb_info *sb_info; struct qede_rx_queue *rxq; - struct qede_tx_queue *txqs[MAX_NUM_TC]; - char name[80]; + struct qede_tx_queue *txq; }; /* @@ -160,20 +253,26 @@ void qede_rx_queue_release(void *rx_queue); void qede_tx_queue_release(void *tx_queue); -int qede_dev_start(struct rte_eth_dev *eth_dev); - -void qede_dev_stop(struct rte_eth_dev *eth_dev); - -void qede_reset_fp_rings(struct qede_dev *qdev); - -void qede_free_fp_arrays(struct qede_dev *qdev); - -void qede_free_mem_load(struct qede_dev *qdev); - uint16_t qede_xmit_pkts(void *p_txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts); +uint16_t qede_xmit_prep_pkts(void *p_txq, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts); + uint16_t qede_recv_pkts(void *p_rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts); +uint16_t qede_rxtx_pkts_dummy(void *p_rxq, + struct rte_mbuf **pkts, + uint16_t nb_pkts); + +int qede_start_queues(struct rte_eth_dev *eth_dev); + +void qede_stop_queues(struct rte_eth_dev *eth_dev); + +/* Fastpath resource alloc/dealloc helpers */ +int qede_alloc_fp_resc(struct qede_dev *qdev); + +void qede_dealloc_fp_resc(struct rte_eth_dev *eth_dev); + #endif /* _QEDE_RXTX_H_ */ diff --git a/dpdk/drivers/net/ring/Makefile b/dpdk/drivers/net/ring/Makefile index ae835052..085ffa57 100644 --- a/dpdk/drivers/net/ring/Makefile +++ b/dpdk/drivers/net/ring/Makefile @@ -38,8 +38,11 @@ LIB = librte_pmd_ring.a CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs +LDLIBS += -lrte_bus_vdev -EXPORT_MAP := rte_eth_ring_version.map +EXPORT_MAP := rte_pmd_ring_version.map LIBABIVER := 2 @@ -53,9 +56,4 @@ SRCS-$(CONFIG_RTE_LIBRTE_PMD_RING) += rte_eth_ring.c # SYMLINK-y-include += rte_eth_ring.h -# this lib depends upon: -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_RING) += lib/librte_eal lib/librte_ring -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_RING) += lib/librte_mbuf lib/librte_ether -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_RING) += lib/librte_kvargs - include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/net/ring/rte_eth_ring.c b/dpdk/drivers/net/ring/rte_eth_ring.c index 5551e187..8583a673 100644 --- a/dpdk/drivers/net/ring/rte_eth_ring.c +++ b/dpdk/drivers/net/ring/rte_eth_ring.c @@ -36,21 +36,31 @@ #include #include #include -#include #include -#include +#include #include #include #define ETH_RING_NUMA_NODE_ACTION_ARG "nodeaction" #define ETH_RING_ACTION_CREATE "CREATE" #define ETH_RING_ACTION_ATTACH "ATTACH" +#define ETH_RING_INTERNAL_ARG "internal" static const char *valid_arguments[] = { ETH_RING_NUMA_NODE_ACTION_ARG, + ETH_RING_INTERNAL_ARG, NULL }; +struct ring_internal_args { + struct rte_ring * const *rx_queues; + const unsigned int nb_rx_queues; + struct rte_ring * const *tx_queues; + const unsigned int nb_tx_queues; + const unsigned int numa_node; + void *addr; /* self addr for sanity check */ +}; + enum dev_action { DEV_CREATE, DEV_ATTACH @@ -75,12 +85,11 @@ struct pmd_internals { }; -static const char *drivername = "Rings PMD"; static struct rte_eth_link pmd_link = { .link_speed = ETH_SPEED_NUM_10G, .link_duplex = ETH_LINK_FULL_DUPLEX, .link_status = ETH_LINK_DOWN, - .link_autoneg = ETH_LINK_SPEED_AUTONEG + .link_autoneg = ETH_LINK_AUTONEG }; static uint16_t @@ -89,7 +98,7 @@ eth_ring_rx(void *q, struct rte_mbuf **bufs, uint16_t nb_bufs) void **ptrs = (void *)&bufs[0]; struct ring_queue *r = q; const uint16_t nb_rx = (uint16_t)rte_ring_dequeue_burst(r->rng, - ptrs, nb_bufs); + ptrs, nb_bufs, NULL); if (r->rng->flags & RING_F_SC_DEQ) r->rx_pkts.cnt += nb_rx; else @@ -103,7 +112,7 @@ eth_ring_tx(void *q, struct rte_mbuf **bufs, uint16_t nb_bufs) void **ptrs = (void *)&bufs[0]; struct ring_queue *r = q; const uint16_t nb_tx = (uint16_t)rte_ring_enqueue_burst(r->rng, - ptrs, nb_bufs); + ptrs, nb_bufs, NULL); if (r->rng->flags & RING_F_SP_ENQ) { r->tx_pkts.cnt += nb_tx; r->err_pkts.cnt += nb_bufs - nb_tx; @@ -173,16 +182,14 @@ eth_dev_info(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) { struct pmd_internals *internals = dev->data->dev_private; - dev_info->driver_name = drivername; dev_info->max_mac_addrs = 1; dev_info->max_rx_pktlen = (uint32_t)-1; dev_info->max_rx_queues = (uint16_t)internals->max_rx_queues; dev_info->max_tx_queues = (uint16_t)internals->max_tx_queues; dev_info->min_rx_bufsize = 0; - dev_info->pci_dev = NULL; } -static void +static int eth_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) { unsigned i; @@ -206,6 +213,8 @@ eth_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) stats->ipackets = rx_total; stats->opackets = tx_total; stats->oerrors = tx_err_total; + + return 0; } static void @@ -227,12 +236,13 @@ eth_mac_addr_remove(struct rte_eth_dev *dev __rte_unused, { } -static void +static int eth_mac_addr_add(struct rte_eth_dev *dev __rte_unused, struct ether_addr *mac_addr __rte_unused, uint32_t index __rte_unused, uint32_t vmdq __rte_unused) { + return 0; } static void @@ -259,15 +269,20 @@ static const struct eth_dev_ops ops = { .mac_addr_add = eth_mac_addr_add, }; +static struct rte_vdev_driver pmd_ring_drv; + static int do_eth_dev_ring_create(const char *name, struct rte_ring * const rx_queues[], const unsigned nb_rx_queues, struct rte_ring *const tx_queues[], const unsigned nb_tx_queues, - const unsigned numa_node, enum dev_action action) + const unsigned int numa_node, enum dev_action action, + struct rte_eth_dev **eth_dev_p) { struct rte_eth_dev_data *data = NULL; struct pmd_internals *internals = NULL; struct rte_eth_dev *eth_dev = NULL; + void **rx_queues_local = NULL; + void **tx_queues_local = NULL; unsigned i; RTE_LOG(INFO, PMD, "Creating rings-backed ethdev on numa socket %u\n", @@ -282,16 +297,16 @@ do_eth_dev_ring_create(const char *name, goto error; } - data->rx_queues = rte_zmalloc_socket(name, + rx_queues_local = rte_zmalloc_socket(name, sizeof(void *) * nb_rx_queues, 0, numa_node); - if (data->rx_queues == NULL) { + if (rx_queues_local == NULL) { rte_errno = ENOMEM; goto error; } - data->tx_queues = rte_zmalloc_socket(name, + tx_queues_local = rte_zmalloc_socket(name, sizeof(void *) * nb_tx_queues, 0, numa_node); - if (data->tx_queues == NULL) { + if (tx_queues_local == NULL) { rte_errno = ENOMEM; goto error; } @@ -303,7 +318,7 @@ do_eth_dev_ring_create(const char *name, } /* reserve an ethdev entry */ - eth_dev = rte_eth_dev_allocate(name, RTE_ETH_DEV_VIRTUAL); + eth_dev = rte_eth_dev_allocate(name); if (eth_dev == NULL) { rte_errno = ENOSPC; goto error; @@ -318,6 +333,10 @@ do_eth_dev_ring_create(const char *name, /* NOTE: we'll replace the data element, of originally allocated eth_dev * so the rings are local per-process */ + rte_memcpy(data, eth_dev->data, sizeof(*data)); + data->rx_queues = rx_queues_local; + data->tx_queues = tx_queues_local; + internals->action = action; internals->max_rx_queues = nb_rx_queues; internals->max_tx_queues = nb_tx_queues; @@ -331,34 +350,27 @@ do_eth_dev_ring_create(const char *name, } data->dev_private = internals; - data->port_id = eth_dev->data->port_id; - memmove(data->name, eth_dev->data->name, sizeof(data->name)); data->nb_rx_queues = (uint16_t)nb_rx_queues; data->nb_tx_queues = (uint16_t)nb_tx_queues; data->dev_link = pmd_link; data->mac_addrs = &internals->address; eth_dev->data = data; - eth_dev->driver = NULL; eth_dev->dev_ops = &ops; - data->dev_flags = RTE_ETH_DEV_DETACHABLE; data->kdrv = RTE_KDRV_NONE; - data->drv_name = drivername; data->numa_node = numa_node; - TAILQ_INIT(&(eth_dev->link_intr_cbs)); - /* finally assign rx and tx ops */ eth_dev->rx_pkt_burst = eth_ring_rx; eth_dev->tx_pkt_burst = eth_ring_tx; + *eth_dev_p = eth_dev; + return data->port_id; error: - if (data) { - rte_free(data->rx_queues); - rte_free(data->tx_queues); - } + rte_free(rx_queues_local); + rte_free(tx_queues_local); rte_free(data); rte_free(internals); @@ -372,6 +384,19 @@ rte_eth_from_rings(const char *name, struct rte_ring *const rx_queues[], const unsigned nb_tx_queues, const unsigned numa_node) { + struct ring_internal_args args = { + .rx_queues = rx_queues, + .nb_rx_queues = nb_rx_queues, + .tx_queues = tx_queues, + .nb_tx_queues = nb_tx_queues, + .numa_node = numa_node, + .addr = &args, + }; + char args_str[32] = { 0 }; + char ring_name[32] = { 0 }; + uint16_t port_id = RTE_MAX_ETHPORTS; + int ret; + /* do some parameter checking */ if (rx_queues == NULL && nb_rx_queues > 0) { rte_errno = EINVAL; @@ -386,8 +411,18 @@ rte_eth_from_rings(const char *name, struct rte_ring *const rx_queues[], return -1; } - return do_eth_dev_ring_create(name, rx_queues, nb_rx_queues, - tx_queues, nb_tx_queues, numa_node, DEV_ATTACH); + snprintf(args_str, 32, "%s=%p", ETH_RING_INTERNAL_ARG, &args); + snprintf(ring_name, 32, "net_ring_%s", name); + + ret = rte_vdev_init(ring_name, args_str); + if (ret) { + rte_errno = EINVAL; + return -1; + } + + rte_eth_dev_get_port_by_name(ring_name, &port_id); + + return port_id; } int @@ -399,7 +434,7 @@ rte_eth_from_ring(struct rte_ring *r) static int eth_dev_ring_create(const char *name, const unsigned numa_node, - enum dev_action action) + enum dev_action action, struct rte_eth_dev **eth_dev) { /* rx and tx are so-called from point of view of first port. * They are inverted from the point of view of second port @@ -421,7 +456,7 @@ eth_dev_ring_create(const char *name, const unsigned numa_node, } if (do_eth_dev_ring_create(name, rxtx, num_rings, rxtx, num_rings, - numa_node, action) < 0) + numa_node, action, eth_dev) < 0) return -1; return 0; @@ -453,13 +488,14 @@ static int parse_kvlist (const char *key __rte_unused, const char *value, void * ret = -EINVAL; if (!name) { - RTE_LOG(WARNING, PMD, "command line paramter is empty for ring pmd!\n"); + RTE_LOG(WARNING, PMD, "command line parameter is empty for ring pmd!\n"); goto out; } node = strchr(name, ':'); if (!node) { - RTE_LOG(WARNING, PMD, "could not parse node value from %s", name); + RTE_LOG(WARNING, PMD, "could not parse node value from %s\n", + name); goto out; } @@ -468,7 +504,8 @@ static int parse_kvlist (const char *key __rte_unused, const char *value, void * action = strchr(node, ':'); if (!action) { - RTE_LOG(WARNING, PMD, "could not action value from %s", node); + RTE_LOG(WARNING, PMD, "could not parse action value from %s\n", + node); goto out; } @@ -505,39 +542,85 @@ out: } static int -rte_pmd_ring_devinit(const char *name, const char *params) +parse_internal_args(const char *key __rte_unused, const char *value, + void *data) { + struct ring_internal_args **internal_args = data; + void *args; + + sscanf(value, "%p", &args); + + *internal_args = args; + + if ((*internal_args)->addr != args) + return -1; + + return 0; +} + +static int +rte_pmd_ring_probe(struct rte_vdev_device *dev) +{ + const char *name, *params; struct rte_kvargs *kvlist = NULL; int ret = 0; struct node_action_list *info = NULL; + struct rte_eth_dev *eth_dev = NULL; + struct ring_internal_args *internal_args; + + name = rte_vdev_device_name(dev); + params = rte_vdev_device_args(dev); RTE_LOG(INFO, PMD, "Initializing pmd_ring for %s\n", name); if (params == NULL || params[0] == '\0') { - ret = eth_dev_ring_create(name, rte_socket_id(), DEV_CREATE); + ret = eth_dev_ring_create(name, rte_socket_id(), DEV_CREATE, + ð_dev); if (ret == -1) { RTE_LOG(INFO, PMD, "Attach to pmd_ring for %s\n", name); ret = eth_dev_ring_create(name, rte_socket_id(), - DEV_ATTACH); + DEV_ATTACH, ð_dev); } - } - else { + } else { kvlist = rte_kvargs_parse(params, valid_arguments); if (!kvlist) { RTE_LOG(INFO, PMD, "Ignoring unsupported parameters when creating" " rings-backed ethernet device\n"); ret = eth_dev_ring_create(name, rte_socket_id(), - DEV_CREATE); + DEV_CREATE, ð_dev); if (ret == -1) { RTE_LOG(INFO, PMD, "Attach to pmd_ring for %s\n", name); ret = eth_dev_ring_create(name, rte_socket_id(), - DEV_ATTACH); + DEV_ATTACH, ð_dev); } + + if (eth_dev) + eth_dev->device = &dev->device; + return ret; + } + + if (rte_kvargs_count(kvlist, ETH_RING_INTERNAL_ARG) == 1) { + ret = rte_kvargs_process(kvlist, ETH_RING_INTERNAL_ARG, + parse_internal_args, + &internal_args); + if (ret < 0) + goto out_free; + + ret = do_eth_dev_ring_create(name, + internal_args->rx_queues, + internal_args->nb_rx_queues, + internal_args->tx_queues, + internal_args->nb_tx_queues, + internal_args->numa_node, + DEV_ATTACH, + ð_dev); + if (ret >= 0) + ret = 0; } else { ret = rte_kvargs_count(kvlist, ETH_RING_NUMA_NODE_ACTION_ARG); info = rte_zmalloc("struct node_action_list", @@ -559,7 +642,8 @@ rte_pmd_ring_devinit(const char *name, const char *params) for (info->count = 0; info->count < info->total; info->count++) { ret = eth_dev_ring_create(info->list[info->count].name, info->list[info->count].node, - info->list[info->count].action); + info->list[info->count].action, + ð_dev); if ((ret == -1) && (info->list[info->count].action == DEV_CREATE)) { RTE_LOG(INFO, PMD, @@ -567,12 +651,16 @@ rte_pmd_ring_devinit(const char *name, const char *params) name); ret = eth_dev_ring_create(name, info->list[info->count].node, - DEV_ATTACH); + DEV_ATTACH, + ð_dev); } } } } + if (eth_dev) + eth_dev->device = &dev->device; + out_free: rte_kvargs_free(kvlist); rte_free(info); @@ -580,8 +668,9 @@ out_free: } static int -rte_pmd_ring_devuninit(const char *name) +rte_pmd_ring_remove(struct rte_vdev_device *dev) { + const char *name = rte_vdev_device_name(dev); struct rte_eth_dev *eth_dev = NULL; struct pmd_internals *internals = NULL; struct ring_queue *r = NULL; @@ -599,36 +688,34 @@ rte_pmd_ring_devuninit(const char *name) eth_dev_stop(eth_dev); - if (eth_dev->data) { - internals = eth_dev->data->dev_private; - if (internals->action == DEV_CREATE) { - /* - * it is only necessary to delete the rings in rx_queues because - * they are the same used in tx_queues - */ - for (i = 0; i < eth_dev->data->nb_rx_queues; i++) { - r = eth_dev->data->rx_queues[i]; - rte_ring_free(r->rng); - } + internals = eth_dev->data->dev_private; + if (internals->action == DEV_CREATE) { + /* + * it is only necessary to delete the rings in rx_queues because + * they are the same used in tx_queues + */ + for (i = 0; i < eth_dev->data->nb_rx_queues; i++) { + r = eth_dev->data->rx_queues[i]; + rte_ring_free(r->rng); } - - rte_free(eth_dev->data->rx_queues); - rte_free(eth_dev->data->tx_queues); - rte_free(eth_dev->data->dev_private); } + rte_free(eth_dev->data->rx_queues); + rte_free(eth_dev->data->tx_queues); + rte_free(eth_dev->data->dev_private); + rte_free(eth_dev->data); rte_eth_dev_release_port(eth_dev); return 0; } -static struct rte_driver pmd_ring_drv = { - .type = PMD_VDEV, - .init = rte_pmd_ring_devinit, - .uninit = rte_pmd_ring_devuninit, +static struct rte_vdev_driver pmd_ring_drv = { + .probe = rte_pmd_ring_probe, + .remove = rte_pmd_ring_remove, }; -PMD_REGISTER_DRIVER(pmd_ring_drv, eth_ring); -DRIVER_REGISTER_PARAM_STRING(eth_ring, - "nodeaction=[attach|detach]"); +RTE_PMD_REGISTER_VDEV(net_ring, pmd_ring_drv); +RTE_PMD_REGISTER_ALIAS(net_ring, eth_ring); +RTE_PMD_REGISTER_PARAM_STRING(net_ring, + ETH_RING_NUMA_NODE_ACTION_ARG "=name:node:action(ATTACH|CREATE)"); diff --git a/dpdk/drivers/net/ring/rte_eth_ring_version.map b/dpdk/drivers/net/ring/rte_pmd_ring_version.map similarity index 100% rename from dpdk/drivers/net/ring/rte_eth_ring_version.map rename to dpdk/drivers/net/ring/rte_pmd_ring_version.map diff --git a/dpdk/drivers/net/sfc/Makefile b/dpdk/drivers/net/sfc/Makefile new file mode 100644 index 00000000..2cfd62a2 --- /dev/null +++ b/dpdk/drivers/net/sfc/Makefile @@ -0,0 +1,149 @@ +# +# BSD LICENSE +# +# Copyright (c) 2016-2017 Solarflare Communications Inc. +# All rights reserved. +# +# This software was jointly developed between OKTET Labs (under contract +# for Solarflare) and Solarflare Communications, Inc. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; +# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +include $(RTE_SDK)/mk/rte.vars.mk + +# +# library name +# +LIB = librte_pmd_sfc_efx.a + +CFLAGS += -I$(SRCDIR)/base/ +CFLAGS += -I$(SRCDIR) +CFLAGS += -O3 +CFLAGS += $(WERROR_FLAGS) +# Strict-aliasing rules are violated by rte_eth_link to uint64_t casts +CFLAGS += -Wno-strict-aliasing + +# Enable extra warnings +CFLAGS += -Wextra + +# More warnings not enabled by above aggregators +CFLAGS += -Wdisabled-optimization + +# Extra CFLAGS for base driver files +CFLAGS_BASE_DRIVER += -Wno-sign-compare +CFLAGS_BASE_DRIVER += -Wno-unused-parameter +CFLAGS_BASE_DRIVER += -Wno-unused-variable + +# Compiler and version dependent flags +ifeq ($(CONFIG_RTE_TOOLCHAIN_GCC),y) +CFLAGS += -Waggregate-return +CFLAGS += -Wnested-externs +CFLAGS_BASE_DRIVER += -Wno-empty-body +CFLAGS_BASE_DRIVER += -Wno-unused-but-set-variable +else ifeq ($(CONFIG_RTE_TOOLCHAIN_CLANG),y) +CFLAGS += -Waggregate-return +CFLAGS += -Wbad-function-cast +CFLAGS_BASE_DRIVER += -Wno-empty-body +else ifeq ($(CONFIG_RTE_TOOLCHAIN_ICC),y) +CFLAGS_BASE_DRIVER += -Wno-unused-but-set-variable +# Suppress ICC false positive warning on 'bulk' may be used before its +# value is set +CFLAGS_sfc_ef10_tx.o += -wd3656 +endif +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs +LDLIBS += -lrte_bus_pci + +# +# List of base driver object files for which +# special CFLAGS above should be applied +# +BASE_DRIVER_OBJS=$(sort $(patsubst %.c,%.o,$(notdir $(wildcard $(SRCDIR)/base/*.c)))) +$(foreach obj, $(BASE_DRIVER_OBJS), \ + $(eval CFLAGS_$(obj)+=$(CFLAGS_BASE_DRIVER))) + +EXPORT_MAP := rte_pmd_sfc_efx_version.map + +LIBABIVER := 1 + +# +# all source are stored in SRCS-y +# +SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += sfc_ethdev.c +SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += sfc_kvargs.c +SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += sfc.c +SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += sfc_mcdi.c +SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += sfc_intr.c +SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += sfc_ev.c +SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += sfc_port.c +SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += sfc_rx.c +SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += sfc_tx.c +SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += sfc_tso.c +SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += sfc_filter.c +SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += sfc_flow.c +SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += sfc_dp.c +SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += sfc_ef10_rx.c +SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += sfc_ef10_tx.c + +VPATH += $(SRCDIR)/base + +SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += efx_bootcfg.c +SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += efx_crc32.c +SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += efx_ev.c +SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += efx_filter.c +SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += efx_hash.c +SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += efx_intr.c +SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += efx_lic.c +SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += efx_mac.c +SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += efx_mcdi.c +SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += efx_mon.c +SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += efx_nic.c +SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += efx_nvram.c +SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += efx_phy.c +SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += efx_port.c +SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += efx_rx.c +SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += efx_sram.c +SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += efx_tx.c +SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += efx_vpd.c +SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += mcdi_mon.c +SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += siena_mac.c +SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += siena_mcdi.c +SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += siena_nic.c +SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += siena_nvram.c +SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += siena_phy.c +SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += siena_sram.c +SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += siena_vpd.c +SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += ef10_ev.c +SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += ef10_filter.c +SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += ef10_intr.c +SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += ef10_mac.c +SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += ef10_mcdi.c +SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += ef10_nic.c +SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += ef10_nvram.c +SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += ef10_phy.c +SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += ef10_rx.c +SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += ef10_tx.c +SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += ef10_vpd.c +SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += hunt_nic.c +SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += medford_nic.c + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/net/sfc/base/README b/dpdk/drivers/net/sfc/base/README new file mode 100644 index 00000000..9019e8ba --- /dev/null +++ b/dpdk/drivers/net/sfc/base/README @@ -0,0 +1,36 @@ + + Copyright (c) 2006-2016 Solarflare Communications Inc. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Solarflare libefx driver library +================================ + +This directory contains source code of Solarflare Communications libefx +driver library of version v4.10.0.1012. + +Updating +======== + +The source code in this directory should not be modified. +Please contact the driver maintainers to request changes. diff --git a/dpdk/drivers/net/sfc/base/ef10_ev.c b/dpdk/drivers/net/sfc/base/ef10_ev.c new file mode 100644 index 00000000..d9389dab --- /dev/null +++ b/dpdk/drivers/net/sfc/base/ef10_ev.c @@ -0,0 +1,1406 @@ +/* + * Copyright (c) 2012-2016 Solarflare Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are + * those of the authors and should not be interpreted as representing official + * policies, either expressed or implied, of the FreeBSD Project. + */ + +#include "efx.h" +#include "efx_impl.h" +#if EFSYS_OPT_MON_STATS +#include "mcdi_mon.h" +#endif + +#if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD + +#if EFSYS_OPT_QSTATS +#define EFX_EV_QSTAT_INCR(_eep, _stat) \ + do { \ + (_eep)->ee_stat[_stat]++; \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) +#else +#define EFX_EV_QSTAT_INCR(_eep, _stat) +#endif + +/* + * Non-interrupting event queue requires interrrupting event queue to + * refer to for wake-up events even if wake ups are never used. + * It could be even non-allocated event queue. + */ +#define EFX_EF10_ALWAYS_INTERRUPTING_EVQ_INDEX (0) + +static __checkReturn boolean_t +ef10_ev_rx( + __in efx_evq_t *eep, + __in efx_qword_t *eqp, + __in const efx_ev_callbacks_t *eecp, + __in_opt void *arg); + +static __checkReturn boolean_t +ef10_ev_tx( + __in efx_evq_t *eep, + __in efx_qword_t *eqp, + __in const efx_ev_callbacks_t *eecp, + __in_opt void *arg); + +static __checkReturn boolean_t +ef10_ev_driver( + __in efx_evq_t *eep, + __in efx_qword_t *eqp, + __in const efx_ev_callbacks_t *eecp, + __in_opt void *arg); + +static __checkReturn boolean_t +ef10_ev_drv_gen( + __in efx_evq_t *eep, + __in efx_qword_t *eqp, + __in const efx_ev_callbacks_t *eecp, + __in_opt void *arg); + +static __checkReturn boolean_t +ef10_ev_mcdi( + __in efx_evq_t *eep, + __in efx_qword_t *eqp, + __in const efx_ev_callbacks_t *eecp, + __in_opt void *arg); + + +static __checkReturn efx_rc_t +efx_mcdi_set_evq_tmr( + __in efx_nic_t *enp, + __in uint32_t instance, + __in uint32_t mode, + __in uint32_t timer_ns) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_SET_EVQ_TMR_IN_LEN, + MC_CMD_SET_EVQ_TMR_OUT_LEN)]; + efx_rc_t rc; + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_SET_EVQ_TMR; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_SET_EVQ_TMR_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_SET_EVQ_TMR_OUT_LEN; + + MCDI_IN_SET_DWORD(req, SET_EVQ_TMR_IN_INSTANCE, instance); + MCDI_IN_SET_DWORD(req, SET_EVQ_TMR_IN_TMR_LOAD_REQ_NS, timer_ns); + MCDI_IN_SET_DWORD(req, SET_EVQ_TMR_IN_TMR_RELOAD_REQ_NS, timer_ns); + MCDI_IN_SET_DWORD(req, SET_EVQ_TMR_IN_TMR_MODE, mode); + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + if (req.emr_out_length_used < MC_CMD_SET_EVQ_TMR_OUT_LEN) { + rc = EMSGSIZE; + goto fail2; + } + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +static __checkReturn efx_rc_t +efx_mcdi_init_evq( + __in efx_nic_t *enp, + __in unsigned int instance, + __in efsys_mem_t *esmp, + __in size_t nevs, + __in uint32_t irq, + __in uint32_t us, + __in uint32_t flags, + __in boolean_t low_latency) +{ + efx_mcdi_req_t req; + uint8_t payload[ + MAX(MC_CMD_INIT_EVQ_IN_LEN(EFX_EVQ_NBUFS(EFX_EVQ_MAXNEVS)), + MC_CMD_INIT_EVQ_OUT_LEN)]; + efx_qword_t *dma_addr; + uint64_t addr; + int npages; + int i; + boolean_t interrupting; + int ev_cut_through; + efx_rc_t rc; + + npages = EFX_EVQ_NBUFS(nevs); + if (MC_CMD_INIT_EVQ_IN_LEN(npages) > MC_CMD_INIT_EVQ_IN_LENMAX) { + rc = EINVAL; + goto fail1; + } + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_INIT_EVQ; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_INIT_EVQ_IN_LEN(npages); + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_INIT_EVQ_OUT_LEN; + + MCDI_IN_SET_DWORD(req, INIT_EVQ_IN_SIZE, nevs); + MCDI_IN_SET_DWORD(req, INIT_EVQ_IN_INSTANCE, instance); + MCDI_IN_SET_DWORD(req, INIT_EVQ_IN_IRQ_NUM, irq); + + interrupting = ((flags & EFX_EVQ_FLAGS_NOTIFY_MASK) == + EFX_EVQ_FLAGS_NOTIFY_INTERRUPT); + + /* + * On Huntington RX and TX event batching can only be requested together + * (even if the datapath firmware doesn't actually support RX + * batching). If event cut through is enabled no RX batching will occur. + * + * So always enable RX and TX event batching, and enable event cut + * through if we want low latency operation. + */ + switch (flags & EFX_EVQ_FLAGS_TYPE_MASK) { + case EFX_EVQ_FLAGS_TYPE_AUTO: + ev_cut_through = low_latency ? 1 : 0; + break; + case EFX_EVQ_FLAGS_TYPE_THROUGHPUT: + ev_cut_through = 0; + break; + case EFX_EVQ_FLAGS_TYPE_LOW_LATENCY: + ev_cut_through = 1; + break; + default: + rc = EINVAL; + goto fail2; + } + MCDI_IN_POPULATE_DWORD_6(req, INIT_EVQ_IN_FLAGS, + INIT_EVQ_IN_FLAG_INTERRUPTING, interrupting, + INIT_EVQ_IN_FLAG_RPTR_DOS, 0, + INIT_EVQ_IN_FLAG_INT_ARMD, 0, + INIT_EVQ_IN_FLAG_CUT_THRU, ev_cut_through, + INIT_EVQ_IN_FLAG_RX_MERGE, 1, + INIT_EVQ_IN_FLAG_TX_MERGE, 1); + + /* If the value is zero then disable the timer */ + if (us == 0) { + MCDI_IN_SET_DWORD(req, INIT_EVQ_IN_TMR_MODE, + MC_CMD_INIT_EVQ_IN_TMR_MODE_DIS); + MCDI_IN_SET_DWORD(req, INIT_EVQ_IN_TMR_LOAD, 0); + MCDI_IN_SET_DWORD(req, INIT_EVQ_IN_TMR_RELOAD, 0); + } else { + unsigned int ticks; + + if ((rc = efx_ev_usecs_to_ticks(enp, us, &ticks)) != 0) + goto fail3; + + MCDI_IN_SET_DWORD(req, INIT_EVQ_IN_TMR_MODE, + MC_CMD_INIT_EVQ_IN_TMR_INT_HLDOFF); + MCDI_IN_SET_DWORD(req, INIT_EVQ_IN_TMR_LOAD, ticks); + MCDI_IN_SET_DWORD(req, INIT_EVQ_IN_TMR_RELOAD, ticks); + } + + MCDI_IN_SET_DWORD(req, INIT_EVQ_IN_COUNT_MODE, + MC_CMD_INIT_EVQ_IN_COUNT_MODE_DIS); + MCDI_IN_SET_DWORD(req, INIT_EVQ_IN_COUNT_THRSHLD, 0); + + dma_addr = MCDI_IN2(req, efx_qword_t, INIT_EVQ_IN_DMA_ADDR); + addr = EFSYS_MEM_ADDR(esmp); + + for (i = 0; i < npages; i++) { + EFX_POPULATE_QWORD_2(*dma_addr, + EFX_DWORD_1, (uint32_t)(addr >> 32), + EFX_DWORD_0, (uint32_t)(addr & 0xffffffff)); + + dma_addr++; + addr += EFX_BUF_SIZE; + } + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail4; + } + + if (req.emr_out_length_used < MC_CMD_INIT_EVQ_OUT_LEN) { + rc = EMSGSIZE; + goto fail5; + } + + /* NOTE: ignore the returned IRQ param as firmware does not set it. */ + + return (0); + +fail5: + EFSYS_PROBE(fail5); +fail4: + EFSYS_PROBE(fail4); +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + +static __checkReturn efx_rc_t +efx_mcdi_init_evq_v2( + __in efx_nic_t *enp, + __in unsigned int instance, + __in efsys_mem_t *esmp, + __in size_t nevs, + __in uint32_t irq, + __in uint32_t us, + __in uint32_t flags) +{ + efx_mcdi_req_t req; + uint8_t payload[ + MAX(MC_CMD_INIT_EVQ_V2_IN_LEN(EFX_EVQ_NBUFS(EFX_EVQ_MAXNEVS)), + MC_CMD_INIT_EVQ_V2_OUT_LEN)]; + boolean_t interrupting; + unsigned int evq_type; + efx_qword_t *dma_addr; + uint64_t addr; + int npages; + int i; + efx_rc_t rc; + + npages = EFX_EVQ_NBUFS(nevs); + if (MC_CMD_INIT_EVQ_V2_IN_LEN(npages) > MC_CMD_INIT_EVQ_V2_IN_LENMAX) { + rc = EINVAL; + goto fail1; + } + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_INIT_EVQ; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_INIT_EVQ_V2_IN_LEN(npages); + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_INIT_EVQ_V2_OUT_LEN; + + MCDI_IN_SET_DWORD(req, INIT_EVQ_V2_IN_SIZE, nevs); + MCDI_IN_SET_DWORD(req, INIT_EVQ_V2_IN_INSTANCE, instance); + MCDI_IN_SET_DWORD(req, INIT_EVQ_V2_IN_IRQ_NUM, irq); + + interrupting = ((flags & EFX_EVQ_FLAGS_NOTIFY_MASK) == + EFX_EVQ_FLAGS_NOTIFY_INTERRUPT); + + switch (flags & EFX_EVQ_FLAGS_TYPE_MASK) { + case EFX_EVQ_FLAGS_TYPE_AUTO: + evq_type = MC_CMD_INIT_EVQ_V2_IN_FLAG_TYPE_AUTO; + break; + case EFX_EVQ_FLAGS_TYPE_THROUGHPUT: + evq_type = MC_CMD_INIT_EVQ_V2_IN_FLAG_TYPE_THROUGHPUT; + break; + case EFX_EVQ_FLAGS_TYPE_LOW_LATENCY: + evq_type = MC_CMD_INIT_EVQ_V2_IN_FLAG_TYPE_LOW_LATENCY; + break; + default: + rc = EINVAL; + goto fail2; + } + MCDI_IN_POPULATE_DWORD_4(req, INIT_EVQ_V2_IN_FLAGS, + INIT_EVQ_V2_IN_FLAG_INTERRUPTING, interrupting, + INIT_EVQ_V2_IN_FLAG_RPTR_DOS, 0, + INIT_EVQ_V2_IN_FLAG_INT_ARMD, 0, + INIT_EVQ_V2_IN_FLAG_TYPE, evq_type); + + /* If the value is zero then disable the timer */ + if (us == 0) { + MCDI_IN_SET_DWORD(req, INIT_EVQ_V2_IN_TMR_MODE, + MC_CMD_INIT_EVQ_V2_IN_TMR_MODE_DIS); + MCDI_IN_SET_DWORD(req, INIT_EVQ_V2_IN_TMR_LOAD, 0); + MCDI_IN_SET_DWORD(req, INIT_EVQ_V2_IN_TMR_RELOAD, 0); + } else { + unsigned int ticks; + + if ((rc = efx_ev_usecs_to_ticks(enp, us, &ticks)) != 0) + goto fail3; + + MCDI_IN_SET_DWORD(req, INIT_EVQ_V2_IN_TMR_MODE, + MC_CMD_INIT_EVQ_V2_IN_TMR_INT_HLDOFF); + MCDI_IN_SET_DWORD(req, INIT_EVQ_V2_IN_TMR_LOAD, ticks); + MCDI_IN_SET_DWORD(req, INIT_EVQ_V2_IN_TMR_RELOAD, ticks); + } + + MCDI_IN_SET_DWORD(req, INIT_EVQ_V2_IN_COUNT_MODE, + MC_CMD_INIT_EVQ_V2_IN_COUNT_MODE_DIS); + MCDI_IN_SET_DWORD(req, INIT_EVQ_V2_IN_COUNT_THRSHLD, 0); + + dma_addr = MCDI_IN2(req, efx_qword_t, INIT_EVQ_V2_IN_DMA_ADDR); + addr = EFSYS_MEM_ADDR(esmp); + + for (i = 0; i < npages; i++) { + EFX_POPULATE_QWORD_2(*dma_addr, + EFX_DWORD_1, (uint32_t)(addr >> 32), + EFX_DWORD_0, (uint32_t)(addr & 0xffffffff)); + + dma_addr++; + addr += EFX_BUF_SIZE; + } + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail4; + } + + if (req.emr_out_length_used < MC_CMD_INIT_EVQ_V2_OUT_LEN) { + rc = EMSGSIZE; + goto fail5; + } + + /* NOTE: ignore the returned IRQ param as firmware does not set it. */ + + EFSYS_PROBE1(mcdi_evq_flags, uint32_t, + MCDI_OUT_DWORD(req, INIT_EVQ_V2_OUT_FLAGS)); + + return (0); + +fail5: + EFSYS_PROBE(fail5); +fail4: + EFSYS_PROBE(fail4); +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +static __checkReturn efx_rc_t +efx_mcdi_fini_evq( + __in efx_nic_t *enp, + __in uint32_t instance) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_FINI_EVQ_IN_LEN, + MC_CMD_FINI_EVQ_OUT_LEN)]; + efx_rc_t rc; + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_FINI_EVQ; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_FINI_EVQ_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_FINI_EVQ_OUT_LEN; + + MCDI_IN_SET_DWORD(req, FINI_EVQ_IN_INSTANCE, instance); + + efx_mcdi_execute_quiet(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + return (0); + +fail1: + /* + * EALREADY is not an error, but indicates that the MC has rebooted and + * that the EVQ has already been destroyed. + */ + if (rc != EALREADY) + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + + + __checkReturn efx_rc_t +ef10_ev_init( + __in efx_nic_t *enp) +{ + _NOTE(ARGUNUSED(enp)) + return (0); +} + + void +ef10_ev_fini( + __in efx_nic_t *enp) +{ + _NOTE(ARGUNUSED(enp)) +} + + __checkReturn efx_rc_t +ef10_ev_qcreate( + __in efx_nic_t *enp, + __in unsigned int index, + __in efsys_mem_t *esmp, + __in size_t n, + __in uint32_t id, + __in uint32_t us, + __in uint32_t flags, + __in efx_evq_t *eep) +{ + efx_nic_cfg_t *encp = &(enp->en_nic_cfg); + uint32_t irq; + efx_rc_t rc; + + _NOTE(ARGUNUSED(id)) /* buftbl id managed by MC */ + EFX_STATIC_ASSERT(ISP2(EFX_EVQ_MAXNEVS)); + EFX_STATIC_ASSERT(ISP2(EFX_EVQ_MINNEVS)); + + if (!ISP2(n) || (n < EFX_EVQ_MINNEVS) || (n > EFX_EVQ_MAXNEVS)) { + rc = EINVAL; + goto fail1; + } + + if (index >= encp->enc_evq_limit) { + rc = EINVAL; + goto fail2; + } + + if (us > encp->enc_evq_timer_max_us) { + rc = EINVAL; + goto fail3; + } + + /* Set up the handler table */ + eep->ee_rx = ef10_ev_rx; + eep->ee_tx = ef10_ev_tx; + eep->ee_driver = ef10_ev_driver; + eep->ee_drv_gen = ef10_ev_drv_gen; + eep->ee_mcdi = ef10_ev_mcdi; + + /* Set up the event queue */ + /* INIT_EVQ expects function-relative vector number */ + if ((flags & EFX_EVQ_FLAGS_NOTIFY_MASK) == + EFX_EVQ_FLAGS_NOTIFY_INTERRUPT) { + irq = index; + } else if (index == EFX_EF10_ALWAYS_INTERRUPTING_EVQ_INDEX) { + irq = index; + flags = (flags & ~EFX_EVQ_FLAGS_NOTIFY_MASK) | + EFX_EVQ_FLAGS_NOTIFY_INTERRUPT; + } else { + irq = EFX_EF10_ALWAYS_INTERRUPTING_EVQ_INDEX; + } + + /* + * Interrupts may be raised for events immediately after the queue is + * created. See bug58606. + */ + + if (encp->enc_init_evq_v2_supported) { + /* + * On Medford the low latency license is required to enable RX + * and event cut through and to disable RX batching. If event + * queue type in flags is auto, we let the firmware decide the + * settings to use. If the adapter has a low latency license, + * it will choose the best settings for low latency, otherwise + * it will choose the best settings for throughput. + */ + rc = efx_mcdi_init_evq_v2(enp, index, esmp, n, irq, us, flags); + if (rc != 0) + goto fail4; + } else { + /* + * On Huntington we need to specify the settings to use. + * If event queue type in flags is auto, we favour throughput + * if the adapter is running virtualization supporting firmware + * (i.e. the full featured firmware variant) + * and latency otherwise. The Ethernet Virtual Bridging + * capability is used to make this decision. (Note though that + * the low latency firmware variant is also best for + * throughput and corresponding type should be specified + * to choose it.) + */ + boolean_t low_latency = encp->enc_datapath_cap_evb ? 0 : 1; + rc = efx_mcdi_init_evq(enp, index, esmp, n, irq, us, flags, + low_latency); + if (rc != 0) + goto fail5; + } + + return (0); + +fail5: + EFSYS_PROBE(fail5); +fail4: + EFSYS_PROBE(fail4); +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + void +ef10_ev_qdestroy( + __in efx_evq_t *eep) +{ + efx_nic_t *enp = eep->ee_enp; + + EFSYS_ASSERT(enp->en_family == EFX_FAMILY_HUNTINGTON || + enp->en_family == EFX_FAMILY_MEDFORD); + + (void) efx_mcdi_fini_evq(eep->ee_enp, eep->ee_index); +} + + __checkReturn efx_rc_t +ef10_ev_qprime( + __in efx_evq_t *eep, + __in unsigned int count) +{ + efx_nic_t *enp = eep->ee_enp; + uint32_t rptr; + efx_dword_t dword; + + rptr = count & eep->ee_mask; + + if (enp->en_nic_cfg.enc_bug35388_workaround) { + EFX_STATIC_ASSERT(EFX_EVQ_MINNEVS > + (1 << ERF_DD_EVQ_IND_RPTR_WIDTH)); + EFX_STATIC_ASSERT(EFX_EVQ_MAXNEVS < + (1 << 2 * ERF_DD_EVQ_IND_RPTR_WIDTH)); + + EFX_POPULATE_DWORD_2(dword, + ERF_DD_EVQ_IND_RPTR_FLAGS, + EFE_DD_EVQ_IND_RPTR_FLAGS_HIGH, + ERF_DD_EVQ_IND_RPTR, + (rptr >> ERF_DD_EVQ_IND_RPTR_WIDTH)); + EFX_BAR_TBL_WRITED(enp, ER_DD_EVQ_INDIRECT, eep->ee_index, + &dword, B_FALSE); + + EFX_POPULATE_DWORD_2(dword, + ERF_DD_EVQ_IND_RPTR_FLAGS, + EFE_DD_EVQ_IND_RPTR_FLAGS_LOW, + ERF_DD_EVQ_IND_RPTR, + rptr & ((1 << ERF_DD_EVQ_IND_RPTR_WIDTH) - 1)); + EFX_BAR_TBL_WRITED(enp, ER_DD_EVQ_INDIRECT, eep->ee_index, + &dword, B_FALSE); + } else { + EFX_POPULATE_DWORD_1(dword, ERF_DZ_EVQ_RPTR, rptr); + EFX_BAR_TBL_WRITED(enp, ER_DZ_EVQ_RPTR_REG, eep->ee_index, + &dword, B_FALSE); + } + + return (0); +} + +static __checkReturn efx_rc_t +efx_mcdi_driver_event( + __in efx_nic_t *enp, + __in uint32_t evq, + __in efx_qword_t data) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_DRIVER_EVENT_IN_LEN, + MC_CMD_DRIVER_EVENT_OUT_LEN)]; + efx_rc_t rc; + + req.emr_cmd = MC_CMD_DRIVER_EVENT; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_DRIVER_EVENT_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_DRIVER_EVENT_OUT_LEN; + + MCDI_IN_SET_DWORD(req, DRIVER_EVENT_IN_EVQ, evq); + + MCDI_IN_SET_DWORD(req, DRIVER_EVENT_IN_DATA_LO, + EFX_QWORD_FIELD(data, EFX_DWORD_0)); + MCDI_IN_SET_DWORD(req, DRIVER_EVENT_IN_DATA_HI, + EFX_QWORD_FIELD(data, EFX_DWORD_1)); + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + void +ef10_ev_qpost( + __in efx_evq_t *eep, + __in uint16_t data) +{ + efx_nic_t *enp = eep->ee_enp; + efx_qword_t event; + + EFX_POPULATE_QWORD_3(event, + ESF_DZ_DRV_CODE, ESE_DZ_EV_CODE_DRV_GEN_EV, + ESF_DZ_DRV_SUB_CODE, 0, + ESF_DZ_DRV_SUB_DATA_DW0, (uint32_t)data); + + (void) efx_mcdi_driver_event(enp, eep->ee_index, event); +} + + __checkReturn efx_rc_t +ef10_ev_qmoderate( + __in efx_evq_t *eep, + __in unsigned int us) +{ + efx_nic_t *enp = eep->ee_enp; + efx_nic_cfg_t *encp = &(enp->en_nic_cfg); + efx_dword_t dword; + uint32_t mode; + efx_rc_t rc; + + /* Check that hardware and MCDI use the same timer MODE values */ + EFX_STATIC_ASSERT(FFE_CZ_TIMER_MODE_DIS == + MC_CMD_SET_EVQ_TMR_IN_TIMER_MODE_DIS); + EFX_STATIC_ASSERT(FFE_CZ_TIMER_MODE_IMMED_START == + MC_CMD_SET_EVQ_TMR_IN_TIMER_MODE_IMMED_START); + EFX_STATIC_ASSERT(FFE_CZ_TIMER_MODE_TRIG_START == + MC_CMD_SET_EVQ_TMR_IN_TIMER_MODE_TRIG_START); + EFX_STATIC_ASSERT(FFE_CZ_TIMER_MODE_INT_HLDOFF == + MC_CMD_SET_EVQ_TMR_IN_TIMER_MODE_INT_HLDOFF); + + if (us > encp->enc_evq_timer_max_us) { + rc = EINVAL; + goto fail1; + } + + /* If the value is zero then disable the timer */ + if (us == 0) { + mode = FFE_CZ_TIMER_MODE_DIS; + } else { + mode = FFE_CZ_TIMER_MODE_INT_HLDOFF; + } + + if (encp->enc_bug61265_workaround) { + uint32_t ns = us * 1000; + + rc = efx_mcdi_set_evq_tmr(enp, eep->ee_index, mode, ns); + if (rc != 0) + goto fail2; + } else { + unsigned int ticks; + + if ((rc = efx_ev_usecs_to_ticks(enp, us, &ticks)) != 0) + goto fail3; + + if (encp->enc_bug35388_workaround) { + EFX_POPULATE_DWORD_3(dword, + ERF_DD_EVQ_IND_TIMER_FLAGS, + EFE_DD_EVQ_IND_TIMER_FLAGS, + ERF_DD_EVQ_IND_TIMER_MODE, mode, + ERF_DD_EVQ_IND_TIMER_VAL, ticks); + EFX_BAR_TBL_WRITED(enp, ER_DD_EVQ_INDIRECT, + eep->ee_index, &dword, 0); + } else { + EFX_POPULATE_DWORD_2(dword, + ERF_DZ_TC_TIMER_MODE, mode, + ERF_DZ_TC_TIMER_VAL, ticks); + EFX_BAR_TBL_WRITED(enp, ER_DZ_EVQ_TMR_REG, + eep->ee_index, &dword, 0); + } + } + + return (0); + +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + +#if EFSYS_OPT_QSTATS + void +ef10_ev_qstats_update( + __in efx_evq_t *eep, + __inout_ecount(EV_NQSTATS) efsys_stat_t *stat) +{ + unsigned int id; + + for (id = 0; id < EV_NQSTATS; id++) { + efsys_stat_t *essp = &stat[id]; + + EFSYS_STAT_INCR(essp, eep->ee_stat[id]); + eep->ee_stat[id] = 0; + } +} +#endif /* EFSYS_OPT_QSTATS */ + +#if EFSYS_OPT_RX_PACKED_STREAM + +static __checkReturn boolean_t +ef10_ev_rx_packed_stream( + __in efx_evq_t *eep, + __in efx_qword_t *eqp, + __in const efx_ev_callbacks_t *eecp, + __in_opt void *arg) +{ + uint32_t label; + uint32_t next_read_lbits; + uint16_t flags; + boolean_t should_abort; + efx_evq_rxq_state_t *eersp; + unsigned int pkt_count; + unsigned int current_id; + boolean_t new_buffer; + + next_read_lbits = EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_DSC_PTR_LBITS); + label = EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_QLABEL); + new_buffer = EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_EV_ROTATE); + + flags = 0; + + eersp = &eep->ee_rxq_state[label]; + pkt_count = (EFX_MASK32(ESF_DZ_RX_DSC_PTR_LBITS) + 1 + + next_read_lbits - eersp->eers_rx_stream_npackets) & + EFX_MASK32(ESF_DZ_RX_DSC_PTR_LBITS); + eersp->eers_rx_stream_npackets += pkt_count; + + if (new_buffer) { + flags |= EFX_PKT_PACKED_STREAM_NEW_BUFFER; + if (eersp->eers_rx_packed_stream_credits < + EFX_RX_PACKED_STREAM_MAX_CREDITS) + eersp->eers_rx_packed_stream_credits++; + eersp->eers_rx_read_ptr++; + } + current_id = eersp->eers_rx_read_ptr & eersp->eers_rx_mask; + + /* Check for errors that invalidate checksum and L3/L4 fields */ + if (EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_ECC_ERR) != 0) { + /* RX frame truncated (error flag is misnamed) */ + EFX_EV_QSTAT_INCR(eep, EV_RX_FRM_TRUNC); + flags |= EFX_DISCARD; + goto deliver; + } + if (EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_ECRC_ERR) != 0) { + /* Bad Ethernet frame CRC */ + EFX_EV_QSTAT_INCR(eep, EV_RX_ETH_CRC_ERR); + flags |= EFX_DISCARD; + goto deliver; + } + + if (EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_PARSE_INCOMPLETE)) { + flags |= EFX_PKT_PACKED_STREAM_PARSE_INCOMPLETE; + goto deliver; + } + + if (EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_IPCKSUM_ERR)) + EFX_EV_QSTAT_INCR(eep, EV_RX_IPV4_HDR_CHKSUM_ERR); + + if (EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_TCPUDP_CKSUM_ERR)) + EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_UDP_CHKSUM_ERR); + +deliver: + /* If we're not discarding the packet then it is ok */ + if (~flags & EFX_DISCARD) + EFX_EV_QSTAT_INCR(eep, EV_RX_OK); + + EFSYS_ASSERT(eecp->eec_rx_ps != NULL); + should_abort = eecp->eec_rx_ps(arg, label, current_id, pkt_count, + flags); + + return (should_abort); +} + +#endif /* EFSYS_OPT_RX_PACKED_STREAM */ + +static __checkReturn boolean_t +ef10_ev_rx( + __in efx_evq_t *eep, + __in efx_qword_t *eqp, + __in const efx_ev_callbacks_t *eecp, + __in_opt void *arg) +{ + efx_nic_t *enp = eep->ee_enp; + uint32_t size; + uint32_t label; + uint32_t mac_class; + uint32_t eth_tag_class; + uint32_t l3_class; + uint32_t l4_class; + uint32_t next_read_lbits; + uint16_t flags; + boolean_t cont; + boolean_t should_abort; + efx_evq_rxq_state_t *eersp; + unsigned int desc_count; + unsigned int last_used_id; + + EFX_EV_QSTAT_INCR(eep, EV_RX); + + /* Discard events after RXQ/TXQ errors */ + if (enp->en_reset_flags & (EFX_RESET_RXQ_ERR | EFX_RESET_TXQ_ERR)) + return (B_FALSE); + + /* Basic packet information */ + label = EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_QLABEL); + eersp = &eep->ee_rxq_state[label]; + +#if EFSYS_OPT_RX_PACKED_STREAM + /* + * Packed stream events are very different, + * so handle them separately + */ + if (eersp->eers_rx_packed_stream) + return (ef10_ev_rx_packed_stream(eep, eqp, eecp, arg)); +#endif + + size = EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_BYTES); + next_read_lbits = EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_DSC_PTR_LBITS); + eth_tag_class = EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_ETH_TAG_CLASS); + mac_class = EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_MAC_CLASS); + l3_class = EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_L3_CLASS); + l4_class = EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_L4_CLASS); + cont = EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_CONT); + + if (EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_DROP_EVENT) != 0) { + /* Drop this event */ + return (B_FALSE); + } + flags = 0; + + if (cont != 0) { + /* + * This may be part of a scattered frame, or it may be a + * truncated frame if scatter is disabled on this RXQ. + * Overlength frames can be received if e.g. a VF is configured + * for 1500 MTU but connected to a port set to 9000 MTU + * (see bug56567). + * FIXME: There is not yet any driver that supports scatter on + * Huntington. Scatter support is required for OSX. + */ + flags |= EFX_PKT_CONT; + } + + if (mac_class == ESE_DZ_MAC_CLASS_UCAST) + flags |= EFX_PKT_UNICAST; + + /* Increment the count of descriptors read */ + desc_count = (next_read_lbits - eersp->eers_rx_read_ptr) & + EFX_MASK32(ESF_DZ_RX_DSC_PTR_LBITS); + eersp->eers_rx_read_ptr += desc_count; + + /* + * FIXME: add error checking to make sure this a batched event. + * This could also be an aborted scatter, see Bug36629. + */ + if (desc_count > 1) { + EFX_EV_QSTAT_INCR(eep, EV_RX_BATCH); + flags |= EFX_PKT_PREFIX_LEN; + } + + /* Calculate the index of the last descriptor consumed */ + last_used_id = (eersp->eers_rx_read_ptr - 1) & eersp->eers_rx_mask; + + /* Check for errors that invalidate checksum and L3/L4 fields */ + if (EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_ECC_ERR) != 0) { + /* RX frame truncated (error flag is misnamed) */ + EFX_EV_QSTAT_INCR(eep, EV_RX_FRM_TRUNC); + flags |= EFX_DISCARD; + goto deliver; + } + if (EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_ECRC_ERR) != 0) { + /* Bad Ethernet frame CRC */ + EFX_EV_QSTAT_INCR(eep, EV_RX_ETH_CRC_ERR); + flags |= EFX_DISCARD; + goto deliver; + } + if (EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_PARSE_INCOMPLETE)) { + /* + * Hardware parse failed, due to malformed headers + * or headers that are too long for the parser. + * Headers and checksums must be validated by the host. + */ + /* TODO: EFX_EV_QSTAT_INCR(eep, EV_RX_PARSE_INCOMPLETE); */ + goto deliver; + } + + if ((eth_tag_class == ESE_DZ_ETH_TAG_CLASS_VLAN1) || + (eth_tag_class == ESE_DZ_ETH_TAG_CLASS_VLAN2)) { + flags |= EFX_PKT_VLAN_TAGGED; + } + + switch (l3_class) { + case ESE_DZ_L3_CLASS_IP4: + case ESE_DZ_L3_CLASS_IP4_FRAG: + flags |= EFX_PKT_IPV4; + if (EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_IPCKSUM_ERR)) { + EFX_EV_QSTAT_INCR(eep, EV_RX_IPV4_HDR_CHKSUM_ERR); + } else { + flags |= EFX_CKSUM_IPV4; + } + + if (l4_class == ESE_DZ_L4_CLASS_TCP) { + EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_IPV4); + flags |= EFX_PKT_TCP; + } else if (l4_class == ESE_DZ_L4_CLASS_UDP) { + EFX_EV_QSTAT_INCR(eep, EV_RX_UDP_IPV4); + flags |= EFX_PKT_UDP; + } else { + EFX_EV_QSTAT_INCR(eep, EV_RX_OTHER_IPV4); + } + break; + + case ESE_DZ_L3_CLASS_IP6: + case ESE_DZ_L3_CLASS_IP6_FRAG: + flags |= EFX_PKT_IPV6; + + if (l4_class == ESE_DZ_L4_CLASS_TCP) { + EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_IPV6); + flags |= EFX_PKT_TCP; + } else if (l4_class == ESE_DZ_L4_CLASS_UDP) { + EFX_EV_QSTAT_INCR(eep, EV_RX_UDP_IPV6); + flags |= EFX_PKT_UDP; + } else { + EFX_EV_QSTAT_INCR(eep, EV_RX_OTHER_IPV6); + } + break; + + default: + EFX_EV_QSTAT_INCR(eep, EV_RX_NON_IP); + break; + } + + if (flags & (EFX_PKT_TCP | EFX_PKT_UDP)) { + if (EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_TCPUDP_CKSUM_ERR)) { + EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_UDP_CHKSUM_ERR); + } else { + flags |= EFX_CKSUM_TCPUDP; + } + } + +deliver: + /* If we're not discarding the packet then it is ok */ + if (~flags & EFX_DISCARD) + EFX_EV_QSTAT_INCR(eep, EV_RX_OK); + + EFSYS_ASSERT(eecp->eec_rx != NULL); + should_abort = eecp->eec_rx(arg, label, last_used_id, size, flags); + + return (should_abort); +} + +static __checkReturn boolean_t +ef10_ev_tx( + __in efx_evq_t *eep, + __in efx_qword_t *eqp, + __in const efx_ev_callbacks_t *eecp, + __in_opt void *arg) +{ + efx_nic_t *enp = eep->ee_enp; + uint32_t id; + uint32_t label; + boolean_t should_abort; + + EFX_EV_QSTAT_INCR(eep, EV_TX); + + /* Discard events after RXQ/TXQ errors */ + if (enp->en_reset_flags & (EFX_RESET_RXQ_ERR | EFX_RESET_TXQ_ERR)) + return (B_FALSE); + + if (EFX_QWORD_FIELD(*eqp, ESF_DZ_TX_DROP_EVENT) != 0) { + /* Drop this event */ + return (B_FALSE); + } + + /* Per-packet TX completion (was per-descriptor for Falcon/Siena) */ + id = EFX_QWORD_FIELD(*eqp, ESF_DZ_TX_DESCR_INDX); + label = EFX_QWORD_FIELD(*eqp, ESF_DZ_TX_QLABEL); + + EFSYS_PROBE2(tx_complete, uint32_t, label, uint32_t, id); + + EFSYS_ASSERT(eecp->eec_tx != NULL); + should_abort = eecp->eec_tx(arg, label, id); + + return (should_abort); +} + +static __checkReturn boolean_t +ef10_ev_driver( + __in efx_evq_t *eep, + __in efx_qword_t *eqp, + __in const efx_ev_callbacks_t *eecp, + __in_opt void *arg) +{ + unsigned int code; + boolean_t should_abort; + + EFX_EV_QSTAT_INCR(eep, EV_DRIVER); + should_abort = B_FALSE; + + code = EFX_QWORD_FIELD(*eqp, ESF_DZ_DRV_SUB_CODE); + switch (code) { + case ESE_DZ_DRV_TIMER_EV: { + uint32_t id; + + id = EFX_QWORD_FIELD(*eqp, ESF_DZ_DRV_TMR_ID); + + EFSYS_ASSERT(eecp->eec_timer != NULL); + should_abort = eecp->eec_timer(arg, id); + break; + } + + case ESE_DZ_DRV_WAKE_UP_EV: { + uint32_t id; + + id = EFX_QWORD_FIELD(*eqp, ESF_DZ_DRV_EVQ_ID); + + EFSYS_ASSERT(eecp->eec_wake_up != NULL); + should_abort = eecp->eec_wake_up(arg, id); + break; + } + + case ESE_DZ_DRV_START_UP_EV: + EFSYS_ASSERT(eecp->eec_initialized != NULL); + should_abort = eecp->eec_initialized(arg); + break; + + default: + EFSYS_PROBE3(bad_event, unsigned int, eep->ee_index, + uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_1), + uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_0)); + break; + } + + return (should_abort); +} + +static __checkReturn boolean_t +ef10_ev_drv_gen( + __in efx_evq_t *eep, + __in efx_qword_t *eqp, + __in const efx_ev_callbacks_t *eecp, + __in_opt void *arg) +{ + uint32_t data; + boolean_t should_abort; + + EFX_EV_QSTAT_INCR(eep, EV_DRV_GEN); + should_abort = B_FALSE; + + data = EFX_QWORD_FIELD(*eqp, ESF_DZ_DRV_SUB_DATA_DW0); + if (data >= ((uint32_t)1 << 16)) { + EFSYS_PROBE3(bad_event, unsigned int, eep->ee_index, + uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_1), + uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_0)); + + return (B_TRUE); + } + + EFSYS_ASSERT(eecp->eec_software != NULL); + should_abort = eecp->eec_software(arg, (uint16_t)data); + + return (should_abort); +} + +static __checkReturn boolean_t +ef10_ev_mcdi( + __in efx_evq_t *eep, + __in efx_qword_t *eqp, + __in const efx_ev_callbacks_t *eecp, + __in_opt void *arg) +{ + efx_nic_t *enp = eep->ee_enp; + unsigned int code; + boolean_t should_abort = B_FALSE; + + EFX_EV_QSTAT_INCR(eep, EV_MCDI_RESPONSE); + + code = EFX_QWORD_FIELD(*eqp, MCDI_EVENT_CODE); + switch (code) { + case MCDI_EVENT_CODE_BADSSERT: + efx_mcdi_ev_death(enp, EINTR); + break; + + case MCDI_EVENT_CODE_CMDDONE: + efx_mcdi_ev_cpl(enp, + MCDI_EV_FIELD(eqp, CMDDONE_SEQ), + MCDI_EV_FIELD(eqp, CMDDONE_DATALEN), + MCDI_EV_FIELD(eqp, CMDDONE_ERRNO)); + break; + +#if EFSYS_OPT_MCDI_PROXY_AUTH + case MCDI_EVENT_CODE_PROXY_RESPONSE: + /* + * This event notifies a function that an authorization request + * has been processed. If the request was authorized then the + * function can now re-send the original MCDI request. + * See SF-113652-SW "SR-IOV Proxied Network Access Control". + */ + efx_mcdi_ev_proxy_response(enp, + MCDI_EV_FIELD(eqp, PROXY_RESPONSE_HANDLE), + MCDI_EV_FIELD(eqp, PROXY_RESPONSE_RC)); + break; +#endif /* EFSYS_OPT_MCDI_PROXY_AUTH */ + + case MCDI_EVENT_CODE_LINKCHANGE: { + efx_link_mode_t link_mode; + + ef10_phy_link_ev(enp, eqp, &link_mode); + should_abort = eecp->eec_link_change(arg, link_mode); + break; + } + + case MCDI_EVENT_CODE_SENSOREVT: { +#if EFSYS_OPT_MON_STATS + efx_mon_stat_t id; + efx_mon_stat_value_t value; + efx_rc_t rc; + + /* Decode monitor stat for MCDI sensor (if supported) */ + if ((rc = mcdi_mon_ev(enp, eqp, &id, &value)) == 0) { + /* Report monitor stat change */ + should_abort = eecp->eec_monitor(arg, id, value); + } else if (rc == ENOTSUP) { + should_abort = eecp->eec_exception(arg, + EFX_EXCEPTION_UNKNOWN_SENSOREVT, + MCDI_EV_FIELD(eqp, DATA)); + } else { + EFSYS_ASSERT(rc == ENODEV); /* Wrong port */ + } +#endif + break; + } + + case MCDI_EVENT_CODE_SCHEDERR: + /* Informational only */ + break; + + case MCDI_EVENT_CODE_REBOOT: + /* Falcon/Siena only (should not been seen with Huntington). */ + efx_mcdi_ev_death(enp, EIO); + break; + + case MCDI_EVENT_CODE_MC_REBOOT: + /* MC_REBOOT event is used for Huntington (EF10) and later. */ + efx_mcdi_ev_death(enp, EIO); + break; + + case MCDI_EVENT_CODE_MAC_STATS_DMA: +#if EFSYS_OPT_MAC_STATS + if (eecp->eec_mac_stats != NULL) { + eecp->eec_mac_stats(arg, + MCDI_EV_FIELD(eqp, MAC_STATS_DMA_GENERATION)); + } +#endif + break; + + case MCDI_EVENT_CODE_FWALERT: { + uint32_t reason = MCDI_EV_FIELD(eqp, FWALERT_REASON); + + if (reason == MCDI_EVENT_FWALERT_REASON_SRAM_ACCESS) + should_abort = eecp->eec_exception(arg, + EFX_EXCEPTION_FWALERT_SRAM, + MCDI_EV_FIELD(eqp, FWALERT_DATA)); + else + should_abort = eecp->eec_exception(arg, + EFX_EXCEPTION_UNKNOWN_FWALERT, + MCDI_EV_FIELD(eqp, DATA)); + break; + } + + case MCDI_EVENT_CODE_TX_ERR: { + /* + * After a TXQ error is detected, firmware sends a TX_ERR event. + * This may be followed by TX completions (which we discard), + * and then finally by a TX_FLUSH event. Firmware destroys the + * TXQ automatically after sending the TX_FLUSH event. + */ + enp->en_reset_flags |= EFX_RESET_TXQ_ERR; + + EFSYS_PROBE2(tx_descq_err, + uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_1), + uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_0)); + + /* Inform the driver that a reset is required. */ + eecp->eec_exception(arg, EFX_EXCEPTION_TX_ERROR, + MCDI_EV_FIELD(eqp, TX_ERR_DATA)); + break; + } + + case MCDI_EVENT_CODE_TX_FLUSH: { + uint32_t txq_index = MCDI_EV_FIELD(eqp, TX_FLUSH_TXQ); + + /* + * EF10 firmware sends two TX_FLUSH events: one to the txq's + * event queue, and one to evq 0 (with TX_FLUSH_TO_DRIVER set). + * We want to wait for all completions, so ignore the events + * with TX_FLUSH_TO_DRIVER. + */ + if (MCDI_EV_FIELD(eqp, TX_FLUSH_TO_DRIVER) != 0) { + should_abort = B_FALSE; + break; + } + + EFX_EV_QSTAT_INCR(eep, EV_DRIVER_TX_DESCQ_FLS_DONE); + + EFSYS_PROBE1(tx_descq_fls_done, uint32_t, txq_index); + + EFSYS_ASSERT(eecp->eec_txq_flush_done != NULL); + should_abort = eecp->eec_txq_flush_done(arg, txq_index); + break; + } + + case MCDI_EVENT_CODE_RX_ERR: { + /* + * After an RXQ error is detected, firmware sends an RX_ERR + * event. This may be followed by RX events (which we discard), + * and then finally by an RX_FLUSH event. Firmware destroys the + * RXQ automatically after sending the RX_FLUSH event. + */ + enp->en_reset_flags |= EFX_RESET_RXQ_ERR; + + EFSYS_PROBE2(rx_descq_err, + uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_1), + uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_0)); + + /* Inform the driver that a reset is required. */ + eecp->eec_exception(arg, EFX_EXCEPTION_RX_ERROR, + MCDI_EV_FIELD(eqp, RX_ERR_DATA)); + break; + } + + case MCDI_EVENT_CODE_RX_FLUSH: { + uint32_t rxq_index = MCDI_EV_FIELD(eqp, RX_FLUSH_RXQ); + + /* + * EF10 firmware sends two RX_FLUSH events: one to the rxq's + * event queue, and one to evq 0 (with RX_FLUSH_TO_DRIVER set). + * We want to wait for all completions, so ignore the events + * with RX_FLUSH_TO_DRIVER. + */ + if (MCDI_EV_FIELD(eqp, RX_FLUSH_TO_DRIVER) != 0) { + should_abort = B_FALSE; + break; + } + + EFX_EV_QSTAT_INCR(eep, EV_DRIVER_RX_DESCQ_FLS_DONE); + + EFSYS_PROBE1(rx_descq_fls_done, uint32_t, rxq_index); + + EFSYS_ASSERT(eecp->eec_rxq_flush_done != NULL); + should_abort = eecp->eec_rxq_flush_done(arg, rxq_index); + break; + } + + default: + EFSYS_PROBE3(bad_event, unsigned int, eep->ee_index, + uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_1), + uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_0)); + break; + } + + return (should_abort); +} + + void +ef10_ev_rxlabel_init( + __in efx_evq_t *eep, + __in efx_rxq_t *erp, + __in unsigned int label, + __in boolean_t packed_stream) +{ + efx_evq_rxq_state_t *eersp; + + EFSYS_ASSERT3U(label, <, EFX_ARRAY_SIZE(eep->ee_rxq_state)); + eersp = &eep->ee_rxq_state[label]; + + EFSYS_ASSERT3U(eersp->eers_rx_mask, ==, 0); + +#if EFSYS_OPT_RX_PACKED_STREAM + /* + * For packed stream modes, the very first event will + * have a new buffer flag set, so it will be incremented, + * yielding the correct pointer. That results in a simpler + * code than trying to detect start-of-the-world condition + * in the event handler. + */ + eersp->eers_rx_read_ptr = packed_stream ? ~0 : 0; +#else + eersp->eers_rx_read_ptr = 0; +#endif + eersp->eers_rx_mask = erp->er_mask; +#if EFSYS_OPT_RX_PACKED_STREAM + eersp->eers_rx_stream_npackets = 0; + eersp->eers_rx_packed_stream = packed_stream; + if (packed_stream) { + eersp->eers_rx_packed_stream_credits = (eep->ee_mask + 1) / + (EFX_RX_PACKED_STREAM_MEM_PER_CREDIT / + EFX_RX_PACKED_STREAM_MIN_PACKET_SPACE); + EFSYS_ASSERT3U(eersp->eers_rx_packed_stream_credits, !=, 0); + /* + * A single credit is allocated to the queue when it is started. + * It is immediately spent by the first packet which has NEW + * BUFFER flag set, though, but still we shall take into + * account, as to not wrap around the maximum number of credits + * accidentally + */ + eersp->eers_rx_packed_stream_credits--; + EFSYS_ASSERT3U(eersp->eers_rx_packed_stream_credits, <=, + EFX_RX_PACKED_STREAM_MAX_CREDITS); + } +#else + EFSYS_ASSERT(!packed_stream); +#endif +} + + void +ef10_ev_rxlabel_fini( + __in efx_evq_t *eep, + __in unsigned int label) +{ + efx_evq_rxq_state_t *eersp; + + EFSYS_ASSERT3U(label, <, EFX_ARRAY_SIZE(eep->ee_rxq_state)); + eersp = &eep->ee_rxq_state[label]; + + EFSYS_ASSERT3U(eersp->eers_rx_mask, !=, 0); + + eersp->eers_rx_read_ptr = 0; + eersp->eers_rx_mask = 0; +#if EFSYS_OPT_RX_PACKED_STREAM + eersp->eers_rx_stream_npackets = 0; + eersp->eers_rx_packed_stream = B_FALSE; + eersp->eers_rx_packed_stream_credits = 0; +#endif +} + +#endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */ diff --git a/dpdk/drivers/net/sfc/base/ef10_filter.c b/dpdk/drivers/net/sfc/base/ef10_filter.c new file mode 100644 index 00000000..e1faf1dd --- /dev/null +++ b/dpdk/drivers/net/sfc/base/ef10_filter.c @@ -0,0 +1,1663 @@ +/* + * Copyright (c) 2007-2016 Solarflare Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are + * those of the authors and should not be interpreted as representing official + * policies, either expressed or implied, of the FreeBSD Project. + */ + +#include "efx.h" +#include "efx_impl.h" + +#if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD + +#if EFSYS_OPT_FILTER + +#define EFE_SPEC(eftp, index) ((eftp)->eft_entry[(index)].efe_spec) + +static efx_filter_spec_t * +ef10_filter_entry_spec( + __in const ef10_filter_table_t *eftp, + __in unsigned int index) +{ + return ((efx_filter_spec_t *)(EFE_SPEC(eftp, index) & + ~(uintptr_t)EFX_EF10_FILTER_FLAGS)); +} + +static boolean_t +ef10_filter_entry_is_busy( + __in const ef10_filter_table_t *eftp, + __in unsigned int index) +{ + if (EFE_SPEC(eftp, index) & EFX_EF10_FILTER_FLAG_BUSY) + return (B_TRUE); + else + return (B_FALSE); +} + +static boolean_t +ef10_filter_entry_is_auto_old( + __in const ef10_filter_table_t *eftp, + __in unsigned int index) +{ + if (EFE_SPEC(eftp, index) & EFX_EF10_FILTER_FLAG_AUTO_OLD) + return (B_TRUE); + else + return (B_FALSE); +} + +static void +ef10_filter_set_entry( + __inout ef10_filter_table_t *eftp, + __in unsigned int index, + __in_opt const efx_filter_spec_t *efsp) +{ + EFE_SPEC(eftp, index) = (uintptr_t)efsp; +} + +static void +ef10_filter_set_entry_busy( + __inout ef10_filter_table_t *eftp, + __in unsigned int index) +{ + EFE_SPEC(eftp, index) |= (uintptr_t)EFX_EF10_FILTER_FLAG_BUSY; +} + +static void +ef10_filter_set_entry_not_busy( + __inout ef10_filter_table_t *eftp, + __in unsigned int index) +{ + EFE_SPEC(eftp, index) &= ~(uintptr_t)EFX_EF10_FILTER_FLAG_BUSY; +} + +static void +ef10_filter_set_entry_auto_old( + __inout ef10_filter_table_t *eftp, + __in unsigned int index) +{ + EFSYS_ASSERT(ef10_filter_entry_spec(eftp, index) != NULL); + EFE_SPEC(eftp, index) |= (uintptr_t)EFX_EF10_FILTER_FLAG_AUTO_OLD; +} + +static void +ef10_filter_set_entry_not_auto_old( + __inout ef10_filter_table_t *eftp, + __in unsigned int index) +{ + EFE_SPEC(eftp, index) &= ~(uintptr_t)EFX_EF10_FILTER_FLAG_AUTO_OLD; + EFSYS_ASSERT(ef10_filter_entry_spec(eftp, index) != NULL); +} + + __checkReturn efx_rc_t +ef10_filter_init( + __in efx_nic_t *enp) +{ + efx_rc_t rc; + ef10_filter_table_t *eftp; + + EFSYS_ASSERT(enp->en_family == EFX_FAMILY_HUNTINGTON || + enp->en_family == EFX_FAMILY_MEDFORD); + +#define MATCH_MASK(match) (EFX_MASK32(match) << EFX_LOW_BIT(match)) + EFX_STATIC_ASSERT(EFX_FILTER_MATCH_REM_HOST == + MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_SRC_IP)); + EFX_STATIC_ASSERT(EFX_FILTER_MATCH_LOC_HOST == + MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_DST_IP)); + EFX_STATIC_ASSERT(EFX_FILTER_MATCH_REM_MAC == + MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_SRC_MAC)); + EFX_STATIC_ASSERT(EFX_FILTER_MATCH_REM_PORT == + MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_SRC_PORT)); + EFX_STATIC_ASSERT(EFX_FILTER_MATCH_LOC_MAC == + MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_DST_MAC)); + EFX_STATIC_ASSERT(EFX_FILTER_MATCH_LOC_PORT == + MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_DST_PORT)); + EFX_STATIC_ASSERT(EFX_FILTER_MATCH_ETHER_TYPE == + MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_ETHER_TYPE)); + EFX_STATIC_ASSERT(EFX_FILTER_MATCH_INNER_VID == + MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_INNER_VLAN)); + EFX_STATIC_ASSERT(EFX_FILTER_MATCH_OUTER_VID == + MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_OUTER_VLAN)); + EFX_STATIC_ASSERT(EFX_FILTER_MATCH_IP_PROTO == + MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_IP_PROTO)); + EFX_STATIC_ASSERT(EFX_FILTER_MATCH_IFRM_UNKNOWN_MCAST_DST == + MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_UNKNOWN_MCAST_DST)); + EFX_STATIC_ASSERT(EFX_FILTER_MATCH_IFRM_UNKNOWN_UCAST_DST == + MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_UNKNOWN_UCAST_DST)); + EFX_STATIC_ASSERT(EFX_FILTER_MATCH_UNKNOWN_MCAST_DST == + MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_UNKNOWN_MCAST_DST)); + EFX_STATIC_ASSERT((uint32_t)EFX_FILTER_MATCH_UNKNOWN_UCAST_DST == + MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_UNKNOWN_UCAST_DST)); +#undef MATCH_MASK + + EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (ef10_filter_table_t), eftp); + + if (!eftp) { + rc = ENOMEM; + goto fail1; + } + + enp->en_filter.ef_ef10_filter_table = eftp; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + void +ef10_filter_fini( + __in efx_nic_t *enp) +{ + EFSYS_ASSERT(enp->en_family == EFX_FAMILY_HUNTINGTON || + enp->en_family == EFX_FAMILY_MEDFORD); + + if (enp->en_filter.ef_ef10_filter_table != NULL) { + EFSYS_KMEM_FREE(enp->en_esip, sizeof (ef10_filter_table_t), + enp->en_filter.ef_ef10_filter_table); + } +} + +static __checkReturn efx_rc_t +efx_mcdi_filter_op_add( + __in efx_nic_t *enp, + __in efx_filter_spec_t *spec, + __in unsigned int filter_op, + __inout ef10_filter_handle_t *handle) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_FILTER_OP_EXT_IN_LEN, + MC_CMD_FILTER_OP_EXT_OUT_LEN)]; + efx_rc_t rc; + + memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_FILTER_OP; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_FILTER_OP_EXT_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_FILTER_OP_EXT_OUT_LEN; + + switch (filter_op) { + case MC_CMD_FILTER_OP_IN_OP_REPLACE: + MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_HANDLE_LO, + handle->efh_lo); + MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_HANDLE_HI, + handle->efh_hi); + /* Fall through */ + case MC_CMD_FILTER_OP_IN_OP_INSERT: + case MC_CMD_FILTER_OP_IN_OP_SUBSCRIBE: + MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_OP, filter_op); + break; + default: + EFSYS_ASSERT(0); + rc = EINVAL; + goto fail1; + } + + MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_PORT_ID, + EVB_PORT_ID_ASSIGNED); + MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_MATCH_FIELDS, + spec->efs_match_flags); + MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_RX_DEST, + MC_CMD_FILTER_OP_EXT_IN_RX_DEST_HOST); + MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_RX_QUEUE, + spec->efs_dmaq_id); + +#if EFSYS_OPT_RX_SCALE + if (spec->efs_flags & EFX_FILTER_FLAG_RX_RSS) { + uint32_t rss_context; + + if (spec->efs_rss_context == EFX_RSS_CONTEXT_DEFAULT) + rss_context = enp->en_rss_context; + else + rss_context = spec->efs_rss_context; + MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_RX_CONTEXT, + rss_context); + } +#endif + + MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_RX_MODE, + spec->efs_flags & EFX_FILTER_FLAG_RX_RSS ? + MC_CMD_FILTER_OP_EXT_IN_RX_MODE_RSS : + MC_CMD_FILTER_OP_EXT_IN_RX_MODE_SIMPLE); + MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_TX_DEST, + MC_CMD_FILTER_OP_EXT_IN_TX_DEST_DEFAULT); + + if (filter_op != MC_CMD_FILTER_OP_IN_OP_REPLACE) { + /* + * NOTE: Unlike most MCDI requests, the filter fields + * are presented in network (big endian) byte order. + */ + memcpy(MCDI_IN2(req, uint8_t, FILTER_OP_EXT_IN_SRC_MAC), + spec->efs_rem_mac, EFX_MAC_ADDR_LEN); + memcpy(MCDI_IN2(req, uint8_t, FILTER_OP_EXT_IN_DST_MAC), + spec->efs_loc_mac, EFX_MAC_ADDR_LEN); + + MCDI_IN_SET_WORD(req, FILTER_OP_EXT_IN_SRC_PORT, + __CPU_TO_BE_16(spec->efs_rem_port)); + MCDI_IN_SET_WORD(req, FILTER_OP_EXT_IN_DST_PORT, + __CPU_TO_BE_16(spec->efs_loc_port)); + + MCDI_IN_SET_WORD(req, FILTER_OP_EXT_IN_ETHER_TYPE, + __CPU_TO_BE_16(spec->efs_ether_type)); + + MCDI_IN_SET_WORD(req, FILTER_OP_EXT_IN_INNER_VLAN, + __CPU_TO_BE_16(spec->efs_inner_vid)); + MCDI_IN_SET_WORD(req, FILTER_OP_EXT_IN_OUTER_VLAN, + __CPU_TO_BE_16(spec->efs_outer_vid)); + + /* IP protocol (in low byte, high byte is zero) */ + MCDI_IN_SET_BYTE(req, FILTER_OP_EXT_IN_IP_PROTO, + spec->efs_ip_proto); + + EFX_STATIC_ASSERT(sizeof (spec->efs_rem_host) == + MC_CMD_FILTER_OP_EXT_IN_SRC_IP_LEN); + EFX_STATIC_ASSERT(sizeof (spec->efs_loc_host) == + MC_CMD_FILTER_OP_EXT_IN_DST_IP_LEN); + + memcpy(MCDI_IN2(req, uint8_t, FILTER_OP_EXT_IN_SRC_IP), + &spec->efs_rem_host.eo_byte[0], + MC_CMD_FILTER_OP_EXT_IN_SRC_IP_LEN); + memcpy(MCDI_IN2(req, uint8_t, FILTER_OP_EXT_IN_DST_IP), + &spec->efs_loc_host.eo_byte[0], + MC_CMD_FILTER_OP_EXT_IN_DST_IP_LEN); + + /* + * On Medford, filters for encapsulated packets match based on + * the ether type and IP protocol in the outer frame. In + * addition we need to fill in the VNI or VSID type field. + */ + switch (spec->efs_encap_type) { + case EFX_TUNNEL_PROTOCOL_NONE: + break; + case EFX_TUNNEL_PROTOCOL_VXLAN: + case EFX_TUNNEL_PROTOCOL_GENEVE: + MCDI_IN_POPULATE_DWORD_1(req, + FILTER_OP_EXT_IN_VNI_OR_VSID, + FILTER_OP_EXT_IN_VNI_TYPE, + spec->efs_encap_type == EFX_TUNNEL_PROTOCOL_VXLAN ? + MC_CMD_FILTER_OP_EXT_IN_VNI_TYPE_VXLAN : + MC_CMD_FILTER_OP_EXT_IN_VNI_TYPE_GENEVE); + break; + case EFX_TUNNEL_PROTOCOL_NVGRE: + MCDI_IN_POPULATE_DWORD_1(req, + FILTER_OP_EXT_IN_VNI_OR_VSID, + FILTER_OP_EXT_IN_VSID_TYPE, + MC_CMD_FILTER_OP_EXT_IN_VSID_TYPE_NVGRE); + break; + default: + EFSYS_ASSERT(0); + rc = EINVAL; + goto fail2; + } + } + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail3; + } + + if (req.emr_out_length_used < MC_CMD_FILTER_OP_EXT_OUT_LEN) { + rc = EMSGSIZE; + goto fail4; + } + + handle->efh_lo = MCDI_OUT_DWORD(req, FILTER_OP_EXT_OUT_HANDLE_LO); + handle->efh_hi = MCDI_OUT_DWORD(req, FILTER_OP_EXT_OUT_HANDLE_HI); + + return (0); + +fail4: + EFSYS_PROBE(fail4); +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); + +} + +static __checkReturn efx_rc_t +efx_mcdi_filter_op_delete( + __in efx_nic_t *enp, + __in unsigned int filter_op, + __inout ef10_filter_handle_t *handle) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_FILTER_OP_EXT_IN_LEN, + MC_CMD_FILTER_OP_EXT_OUT_LEN)]; + efx_rc_t rc; + + memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_FILTER_OP; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_FILTER_OP_EXT_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_FILTER_OP_EXT_OUT_LEN; + + switch (filter_op) { + case MC_CMD_FILTER_OP_IN_OP_REMOVE: + MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_OP, + MC_CMD_FILTER_OP_IN_OP_REMOVE); + break; + case MC_CMD_FILTER_OP_IN_OP_UNSUBSCRIBE: + MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_OP, + MC_CMD_FILTER_OP_IN_OP_UNSUBSCRIBE); + break; + default: + EFSYS_ASSERT(0); + rc = EINVAL; + goto fail1; + } + + MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_HANDLE_LO, handle->efh_lo); + MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_HANDLE_HI, handle->efh_hi); + + efx_mcdi_execute_quiet(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail2; + } + + if (req.emr_out_length_used < MC_CMD_FILTER_OP_EXT_OUT_LEN) { + rc = EMSGSIZE; + goto fail3; + } + + return (0); + +fail3: + EFSYS_PROBE(fail3); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +static __checkReturn boolean_t +ef10_filter_equal( + __in const efx_filter_spec_t *left, + __in const efx_filter_spec_t *right) +{ + /* FIXME: Consider rx vs tx filters (look at efs_flags) */ + if (left->efs_match_flags != right->efs_match_flags) + return (B_FALSE); + if (!EFX_OWORD_IS_EQUAL(left->efs_rem_host, right->efs_rem_host)) + return (B_FALSE); + if (!EFX_OWORD_IS_EQUAL(left->efs_loc_host, right->efs_loc_host)) + return (B_FALSE); + if (memcmp(left->efs_rem_mac, right->efs_rem_mac, EFX_MAC_ADDR_LEN)) + return (B_FALSE); + if (memcmp(left->efs_loc_mac, right->efs_loc_mac, EFX_MAC_ADDR_LEN)) + return (B_FALSE); + if (left->efs_rem_port != right->efs_rem_port) + return (B_FALSE); + if (left->efs_loc_port != right->efs_loc_port) + return (B_FALSE); + if (left->efs_inner_vid != right->efs_inner_vid) + return (B_FALSE); + if (left->efs_outer_vid != right->efs_outer_vid) + return (B_FALSE); + if (left->efs_ether_type != right->efs_ether_type) + return (B_FALSE); + if (left->efs_ip_proto != right->efs_ip_proto) + return (B_FALSE); + if (left->efs_encap_type != right->efs_encap_type) + return (B_FALSE); + + return (B_TRUE); + +} + +static __checkReturn boolean_t +ef10_filter_same_dest( + __in const efx_filter_spec_t *left, + __in const efx_filter_spec_t *right) +{ + if ((left->efs_flags & EFX_FILTER_FLAG_RX_RSS) && + (right->efs_flags & EFX_FILTER_FLAG_RX_RSS)) { + if (left->efs_rss_context == right->efs_rss_context) + return (B_TRUE); + } else if ((~(left->efs_flags) & EFX_FILTER_FLAG_RX_RSS) && + (~(right->efs_flags) & EFX_FILTER_FLAG_RX_RSS)) { + if (left->efs_dmaq_id == right->efs_dmaq_id) + return (B_TRUE); + } + return (B_FALSE); +} + +static __checkReturn uint32_t +ef10_filter_hash( + __in efx_filter_spec_t *spec) +{ + EFX_STATIC_ASSERT((sizeof (efx_filter_spec_t) % sizeof (uint32_t)) + == 0); + EFX_STATIC_ASSERT((EFX_FIELD_OFFSET(efx_filter_spec_t, efs_outer_vid) % + sizeof (uint32_t)) == 0); + + /* + * As the area of the efx_filter_spec_t we need to hash is DWORD + * aligned and an exact number of DWORDs in size we can use the + * optimised efx_hash_dwords() rather than efx_hash_bytes() + */ + return (efx_hash_dwords((const uint32_t *)&spec->efs_outer_vid, + (sizeof (efx_filter_spec_t) - + EFX_FIELD_OFFSET(efx_filter_spec_t, efs_outer_vid)) / + sizeof (uint32_t), 0)); +} + +/* + * Decide whether a filter should be exclusive or else should allow + * delivery to additional recipients. Currently we decide that + * filters for specific local unicast MAC and IP addresses are + * exclusive. + */ +static __checkReturn boolean_t +ef10_filter_is_exclusive( + __in efx_filter_spec_t *spec) +{ + if ((spec->efs_match_flags & EFX_FILTER_MATCH_LOC_MAC) && + !EFX_MAC_ADDR_IS_MULTICAST(spec->efs_loc_mac)) + return (B_TRUE); + + if ((spec->efs_match_flags & + (EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_LOC_HOST)) == + (EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_LOC_HOST)) { + if ((spec->efs_ether_type == EFX_ETHER_TYPE_IPV4) && + ((spec->efs_loc_host.eo_u8[0] & 0xf) != 0xe)) + return (B_TRUE); + if ((spec->efs_ether_type == EFX_ETHER_TYPE_IPV6) && + (spec->efs_loc_host.eo_u8[0] != 0xff)) + return (B_TRUE); + } + + return (B_FALSE); +} + + __checkReturn efx_rc_t +ef10_filter_restore( + __in efx_nic_t *enp) +{ + int tbl_id; + efx_filter_spec_t *spec; + ef10_filter_table_t *eftp = enp->en_filter.ef_ef10_filter_table; + boolean_t restoring; + efsys_lock_state_t state; + efx_rc_t rc; + + EFSYS_ASSERT(enp->en_family == EFX_FAMILY_HUNTINGTON || + enp->en_family == EFX_FAMILY_MEDFORD); + + for (tbl_id = 0; tbl_id < EFX_EF10_FILTER_TBL_ROWS; tbl_id++) { + + EFSYS_LOCK(enp->en_eslp, state); + + spec = ef10_filter_entry_spec(eftp, tbl_id); + if (spec == NULL) { + restoring = B_FALSE; + } else if (ef10_filter_entry_is_busy(eftp, tbl_id)) { + /* Ignore busy entries. */ + restoring = B_FALSE; + } else { + ef10_filter_set_entry_busy(eftp, tbl_id); + restoring = B_TRUE; + } + + EFSYS_UNLOCK(enp->en_eslp, state); + + if (restoring == B_FALSE) + continue; + + if (ef10_filter_is_exclusive(spec)) { + rc = efx_mcdi_filter_op_add(enp, spec, + MC_CMD_FILTER_OP_IN_OP_INSERT, + &eftp->eft_entry[tbl_id].efe_handle); + } else { + rc = efx_mcdi_filter_op_add(enp, spec, + MC_CMD_FILTER_OP_IN_OP_SUBSCRIBE, + &eftp->eft_entry[tbl_id].efe_handle); + } + + if (rc != 0) + goto fail1; + + EFSYS_LOCK(enp->en_eslp, state); + + ef10_filter_set_entry_not_busy(eftp, tbl_id); + + EFSYS_UNLOCK(enp->en_eslp, state); + } + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +/* + * An arbitrary search limit for the software hash table. As per the linux net + * driver. + */ +#define EF10_FILTER_SEARCH_LIMIT 200 + +static __checkReturn efx_rc_t +ef10_filter_add_internal( + __in efx_nic_t *enp, + __inout efx_filter_spec_t *spec, + __in boolean_t may_replace, + __out_opt uint32_t *filter_id) +{ + efx_rc_t rc; + ef10_filter_table_t *eftp = enp->en_filter.ef_ef10_filter_table; + efx_filter_spec_t *saved_spec; + uint32_t hash; + unsigned int depth; + int ins_index; + boolean_t replacing = B_FALSE; + unsigned int i; + efsys_lock_state_t state; + boolean_t locked = B_FALSE; + + EFSYS_ASSERT(enp->en_family == EFX_FAMILY_HUNTINGTON || + enp->en_family == EFX_FAMILY_MEDFORD); + + hash = ef10_filter_hash(spec); + + /* + * FIXME: Add support for inserting filters of different priorities + * and removing lower priority multicast filters (bug 42378) + */ + + /* + * Find any existing filters with the same match tuple or + * else a free slot to insert at. If any of them are busy, + * we have to wait and retry. + */ + for (;;) { + ins_index = -1; + depth = 1; + EFSYS_LOCK(enp->en_eslp, state); + locked = B_TRUE; + + for (;;) { + i = (hash + depth) & (EFX_EF10_FILTER_TBL_ROWS - 1); + saved_spec = ef10_filter_entry_spec(eftp, i); + + if (!saved_spec) { + if (ins_index < 0) { + ins_index = i; + } + } else if (ef10_filter_equal(spec, saved_spec)) { + if (ef10_filter_entry_is_busy(eftp, i)) + break; + if (saved_spec->efs_priority + == EFX_FILTER_PRI_AUTO) { + ins_index = i; + goto found; + } else if (ef10_filter_is_exclusive(spec)) { + if (may_replace) { + ins_index = i; + goto found; + } else { + rc = EEXIST; + goto fail1; + } + } + + /* Leave existing */ + } + + /* + * Once we reach the maximum search depth, use + * the first suitable slot or return EBUSY if + * there was none. + */ + if (depth == EF10_FILTER_SEARCH_LIMIT) { + if (ins_index < 0) { + rc = EBUSY; + goto fail2; + } + goto found; + } + depth++; + } + EFSYS_UNLOCK(enp->en_eslp, state); + locked = B_FALSE; + } + +found: + /* + * Create a software table entry if necessary, and mark it + * busy. We might yet fail to insert, but any attempt to + * insert a conflicting filter while we're waiting for the + * firmware must find the busy entry. + */ + saved_spec = ef10_filter_entry_spec(eftp, ins_index); + if (saved_spec) { + if (saved_spec->efs_priority == EFX_FILTER_PRI_AUTO) { + /* This is a filter we are refreshing */ + ef10_filter_set_entry_not_auto_old(eftp, ins_index); + goto out_unlock; + + } + replacing = B_TRUE; + } else { + EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (*spec), saved_spec); + if (!saved_spec) { + rc = ENOMEM; + goto fail3; + } + *saved_spec = *spec; + ef10_filter_set_entry(eftp, ins_index, saved_spec); + } + ef10_filter_set_entry_busy(eftp, ins_index); + + EFSYS_UNLOCK(enp->en_eslp, state); + locked = B_FALSE; + + /* + * On replacing the filter handle may change after after a successful + * replace operation. + */ + if (replacing) { + rc = efx_mcdi_filter_op_add(enp, spec, + MC_CMD_FILTER_OP_IN_OP_REPLACE, + &eftp->eft_entry[ins_index].efe_handle); + } else if (ef10_filter_is_exclusive(spec)) { + rc = efx_mcdi_filter_op_add(enp, spec, + MC_CMD_FILTER_OP_IN_OP_INSERT, + &eftp->eft_entry[ins_index].efe_handle); + } else { + rc = efx_mcdi_filter_op_add(enp, spec, + MC_CMD_FILTER_OP_IN_OP_SUBSCRIBE, + &eftp->eft_entry[ins_index].efe_handle); + } + + if (rc != 0) + goto fail4; + + EFSYS_LOCK(enp->en_eslp, state); + locked = B_TRUE; + + if (replacing) { + /* Update the fields that may differ */ + saved_spec->efs_priority = spec->efs_priority; + saved_spec->efs_flags = spec->efs_flags; + saved_spec->efs_rss_context = spec->efs_rss_context; + saved_spec->efs_dmaq_id = spec->efs_dmaq_id; + } + + ef10_filter_set_entry_not_busy(eftp, ins_index); + +out_unlock: + + EFSYS_UNLOCK(enp->en_eslp, state); + locked = B_FALSE; + + if (filter_id) + *filter_id = ins_index; + + return (0); + +fail4: + EFSYS_PROBE(fail4); + + if (!replacing) { + EFSYS_KMEM_FREE(enp->en_esip, sizeof (*spec), saved_spec); + saved_spec = NULL; + } + ef10_filter_set_entry_not_busy(eftp, ins_index); + ef10_filter_set_entry(eftp, ins_index, NULL); + +fail3: + EFSYS_PROBE(fail3); + +fail2: + EFSYS_PROBE(fail2); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + if (locked) + EFSYS_UNLOCK(enp->en_eslp, state); + + return (rc); +} + + __checkReturn efx_rc_t +ef10_filter_add( + __in efx_nic_t *enp, + __inout efx_filter_spec_t *spec, + __in boolean_t may_replace) +{ + efx_rc_t rc; + + rc = ef10_filter_add_internal(enp, spec, may_replace, NULL); + if (rc != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + +static __checkReturn efx_rc_t +ef10_filter_delete_internal( + __in efx_nic_t *enp, + __in uint32_t filter_id) +{ + efx_rc_t rc; + ef10_filter_table_t *table = enp->en_filter.ef_ef10_filter_table; + efx_filter_spec_t *spec; + efsys_lock_state_t state; + uint32_t filter_idx = filter_id % EFX_EF10_FILTER_TBL_ROWS; + + /* + * Find the software table entry and mark it busy. Don't + * remove it yet; any attempt to update while we're waiting + * for the firmware must find the busy entry. + * + * FIXME: What if the busy flag is never cleared? + */ + EFSYS_LOCK(enp->en_eslp, state); + while (ef10_filter_entry_is_busy(table, filter_idx)) { + EFSYS_UNLOCK(enp->en_eslp, state); + EFSYS_SPIN(1); + EFSYS_LOCK(enp->en_eslp, state); + } + if ((spec = ef10_filter_entry_spec(table, filter_idx)) != NULL) { + ef10_filter_set_entry_busy(table, filter_idx); + } + EFSYS_UNLOCK(enp->en_eslp, state); + + if (spec == NULL) { + rc = ENOENT; + goto fail1; + } + + /* + * Try to remove the hardware filter. This may fail if the MC has + * rebooted (which frees all hardware filter resources). + */ + if (ef10_filter_is_exclusive(spec)) { + rc = efx_mcdi_filter_op_delete(enp, + MC_CMD_FILTER_OP_IN_OP_REMOVE, + &table->eft_entry[filter_idx].efe_handle); + } else { + rc = efx_mcdi_filter_op_delete(enp, + MC_CMD_FILTER_OP_IN_OP_UNSUBSCRIBE, + &table->eft_entry[filter_idx].efe_handle); + } + + /* Free the software table entry */ + EFSYS_LOCK(enp->en_eslp, state); + ef10_filter_set_entry_not_busy(table, filter_idx); + ef10_filter_set_entry(table, filter_idx, NULL); + EFSYS_UNLOCK(enp->en_eslp, state); + + EFSYS_KMEM_FREE(enp->en_esip, sizeof (*spec), spec); + + /* Check result of hardware filter removal */ + if (rc != 0) + goto fail2; + + return (0); + +fail2: + EFSYS_PROBE(fail2); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +ef10_filter_delete( + __in efx_nic_t *enp, + __inout efx_filter_spec_t *spec) +{ + efx_rc_t rc; + ef10_filter_table_t *table = enp->en_filter.ef_ef10_filter_table; + efx_filter_spec_t *saved_spec; + unsigned int hash; + unsigned int depth; + unsigned int i; + efsys_lock_state_t state; + boolean_t locked = B_FALSE; + + EFSYS_ASSERT(enp->en_family == EFX_FAMILY_HUNTINGTON || + enp->en_family == EFX_FAMILY_MEDFORD); + + hash = ef10_filter_hash(spec); + + EFSYS_LOCK(enp->en_eslp, state); + locked = B_TRUE; + + depth = 1; + for (;;) { + i = (hash + depth) & (EFX_EF10_FILTER_TBL_ROWS - 1); + saved_spec = ef10_filter_entry_spec(table, i); + if (saved_spec && ef10_filter_equal(spec, saved_spec) && + ef10_filter_same_dest(spec, saved_spec)) { + break; + } + if (depth == EF10_FILTER_SEARCH_LIMIT) { + rc = ENOENT; + goto fail1; + } + depth++; + } + + EFSYS_UNLOCK(enp->en_eslp, state); + locked = B_FALSE; + + rc = ef10_filter_delete_internal(enp, i); + if (rc != 0) + goto fail2; + + return (0); + +fail2: + EFSYS_PROBE(fail2); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + if (locked) + EFSYS_UNLOCK(enp->en_eslp, state); + + return (rc); +} + +static __checkReturn efx_rc_t +efx_mcdi_get_parser_disp_info( + __in efx_nic_t *enp, + __out_ecount(buffer_length) uint32_t *buffer, + __in size_t buffer_length, + __out size_t *list_lengthp) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_GET_PARSER_DISP_INFO_IN_LEN, + MC_CMD_GET_PARSER_DISP_INFO_OUT_LENMAX)]; + size_t matches_count; + size_t list_size; + efx_rc_t rc; + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_GET_PARSER_DISP_INFO; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_GET_PARSER_DISP_INFO_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_GET_PARSER_DISP_INFO_OUT_LENMAX; + + MCDI_IN_SET_DWORD(req, GET_PARSER_DISP_INFO_OUT_OP, + MC_CMD_GET_PARSER_DISP_INFO_IN_OP_GET_SUPPORTED_RX_MATCHES); + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + matches_count = MCDI_OUT_DWORD(req, + GET_PARSER_DISP_INFO_OUT_NUM_SUPPORTED_MATCHES); + + if (req.emr_out_length_used < + MC_CMD_GET_PARSER_DISP_INFO_OUT_LEN(matches_count)) { + rc = EMSGSIZE; + goto fail2; + } + + *list_lengthp = matches_count; + + if (buffer_length < matches_count) { + rc = ENOSPC; + goto fail3; + } + + /* + * Check that the elements in the list in the MCDI response are the size + * we expect, so we can just copy them directly. Any conversion of the + * flags is handled by the caller. + */ + EFX_STATIC_ASSERT(sizeof (uint32_t) == + MC_CMD_GET_PARSER_DISP_INFO_OUT_SUPPORTED_MATCHES_LEN); + + list_size = matches_count * + MC_CMD_GET_PARSER_DISP_INFO_OUT_SUPPORTED_MATCHES_LEN; + memcpy(buffer, + MCDI_OUT2(req, uint32_t, + GET_PARSER_DISP_INFO_OUT_SUPPORTED_MATCHES), + list_size); + + return (0); + +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +ef10_filter_supported_filters( + __in efx_nic_t *enp, + __out_ecount(buffer_length) uint32_t *buffer, + __in size_t buffer_length, + __out size_t *list_lengthp) +{ + + size_t mcdi_list_length; + size_t list_length; + uint32_t i; + efx_rc_t rc; + uint32_t all_filter_flags = + (EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_LOC_HOST | + EFX_FILTER_MATCH_REM_MAC | EFX_FILTER_MATCH_REM_PORT | + EFX_FILTER_MATCH_LOC_MAC | EFX_FILTER_MATCH_LOC_PORT | + EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_INNER_VID | + EFX_FILTER_MATCH_OUTER_VID | EFX_FILTER_MATCH_IP_PROTO | + EFX_FILTER_MATCH_UNKNOWN_MCAST_DST | + EFX_FILTER_MATCH_UNKNOWN_UCAST_DST); + + rc = efx_mcdi_get_parser_disp_info(enp, buffer, buffer_length, + &mcdi_list_length); + if (rc != 0) { + if (rc == ENOSPC) { + /* Pass through mcdi_list_length for the list length */ + *list_lengthp = mcdi_list_length; + } + goto fail1; + } + + /* + * The static assertions in ef10_filter_init() ensure that the values of + * the EFX_FILTER_MATCH flags match those used by MCDI, so they don't + * need to be converted. + * + * In case support is added to MCDI for additional flags, remove any + * matches from the list which include flags we don't support. The order + * of the matches is preserved as they are ordered from highest to + * lowest priority. + */ + EFSYS_ASSERT(mcdi_list_length <= buffer_length); + list_length = 0; + for (i = 0; i < mcdi_list_length; i++) { + if ((buffer[i] & ~all_filter_flags) == 0) { + buffer[list_length] = buffer[i]; + list_length++; + } + } + + *list_lengthp = list_length; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +static __checkReturn efx_rc_t +ef10_filter_insert_unicast( + __in efx_nic_t *enp, + __in_ecount(6) uint8_t const *addr, + __in efx_filter_flags_t filter_flags) +{ + ef10_filter_table_t *eftp = enp->en_filter.ef_ef10_filter_table; + efx_filter_spec_t spec; + efx_rc_t rc; + + /* Insert the filter for the local station address */ + efx_filter_spec_init_rx(&spec, EFX_FILTER_PRI_AUTO, + filter_flags, + eftp->eft_default_rxq); + efx_filter_spec_set_eth_local(&spec, EFX_FILTER_SPEC_VID_UNSPEC, addr); + + rc = ef10_filter_add_internal(enp, &spec, B_TRUE, + &eftp->eft_unicst_filter_indexes[eftp->eft_unicst_filter_count]); + if (rc != 0) + goto fail1; + + eftp->eft_unicst_filter_count++; + EFSYS_ASSERT(eftp->eft_unicst_filter_count <= + EFX_EF10_FILTER_UNICAST_FILTERS_MAX); + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + return (rc); +} + +static __checkReturn efx_rc_t +ef10_filter_insert_all_unicast( + __in efx_nic_t *enp, + __in efx_filter_flags_t filter_flags) +{ + ef10_filter_table_t *eftp = enp->en_filter.ef_ef10_filter_table; + efx_filter_spec_t spec; + efx_rc_t rc; + + /* Insert the unknown unicast filter */ + efx_filter_spec_init_rx(&spec, EFX_FILTER_PRI_AUTO, + filter_flags, + eftp->eft_default_rxq); + efx_filter_spec_set_uc_def(&spec); + rc = ef10_filter_add_internal(enp, &spec, B_TRUE, + &eftp->eft_unicst_filter_indexes[eftp->eft_unicst_filter_count]); + if (rc != 0) + goto fail1; + + eftp->eft_unicst_filter_count++; + EFSYS_ASSERT(eftp->eft_unicst_filter_count <= + EFX_EF10_FILTER_UNICAST_FILTERS_MAX); + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + return (rc); +} + +static __checkReturn efx_rc_t +ef10_filter_insert_multicast_list( + __in efx_nic_t *enp, + __in boolean_t mulcst, + __in boolean_t brdcst, + __in_ecount(6*count) uint8_t const *addrs, + __in uint32_t count, + __in efx_filter_flags_t filter_flags, + __in boolean_t rollback) +{ + ef10_filter_table_t *eftp = enp->en_filter.ef_ef10_filter_table; + efx_filter_spec_t spec; + uint8_t addr[6]; + uint32_t i; + uint32_t filter_index; + uint32_t filter_count; + efx_rc_t rc; + + if (mulcst == B_FALSE) + count = 0; + + if (count + (brdcst ? 1 : 0) > + EFX_ARRAY_SIZE(eftp->eft_mulcst_filter_indexes)) { + /* Too many MAC addresses */ + rc = EINVAL; + goto fail1; + } + + /* Insert/renew multicast address list filters */ + filter_count = 0; + for (i = 0; i < count; i++) { + efx_filter_spec_init_rx(&spec, + EFX_FILTER_PRI_AUTO, + filter_flags, + eftp->eft_default_rxq); + + efx_filter_spec_set_eth_local(&spec, + EFX_FILTER_SPEC_VID_UNSPEC, + &addrs[i * EFX_MAC_ADDR_LEN]); + + rc = ef10_filter_add_internal(enp, &spec, B_TRUE, + &filter_index); + + if (rc == 0) { + eftp->eft_mulcst_filter_indexes[filter_count] = + filter_index; + filter_count++; + } else if (rollback == B_TRUE) { + /* Only stop upon failure if told to rollback */ + goto rollback; + } + + } + + if (brdcst == B_TRUE) { + /* Insert/renew broadcast address filter */ + efx_filter_spec_init_rx(&spec, EFX_FILTER_PRI_AUTO, + filter_flags, + eftp->eft_default_rxq); + + EFX_MAC_BROADCAST_ADDR_SET(addr); + efx_filter_spec_set_eth_local(&spec, EFX_FILTER_SPEC_VID_UNSPEC, + addr); + + rc = ef10_filter_add_internal(enp, &spec, B_TRUE, + &filter_index); + + if (rc == 0) { + eftp->eft_mulcst_filter_indexes[filter_count] = + filter_index; + filter_count++; + } else if (rollback == B_TRUE) { + /* Only stop upon failure if told to rollback */ + goto rollback; + } + } + + eftp->eft_mulcst_filter_count = filter_count; + eftp->eft_using_all_mulcst = B_FALSE; + + return (0); + +rollback: + /* Remove any filters we have inserted */ + i = filter_count; + while (i--) { + (void) ef10_filter_delete_internal(enp, + eftp->eft_mulcst_filter_indexes[i]); + } + eftp->eft_mulcst_filter_count = 0; + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +static __checkReturn efx_rc_t +ef10_filter_insert_all_multicast( + __in efx_nic_t *enp, + __in efx_filter_flags_t filter_flags) +{ + ef10_filter_table_t *eftp = enp->en_filter.ef_ef10_filter_table; + efx_filter_spec_t spec; + efx_rc_t rc; + + /* Insert the unknown multicast filter */ + efx_filter_spec_init_rx(&spec, EFX_FILTER_PRI_AUTO, + filter_flags, + eftp->eft_default_rxq); + efx_filter_spec_set_mc_def(&spec); + + rc = ef10_filter_add_internal(enp, &spec, B_TRUE, + &eftp->eft_mulcst_filter_indexes[0]); + if (rc != 0) + goto fail1; + + eftp->eft_mulcst_filter_count = 1; + eftp->eft_using_all_mulcst = B_TRUE; + + /* + * FIXME: If brdcst == B_FALSE, add a filter to drop broadcast traffic. + */ + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +typedef struct ef10_filter_encap_entry_s { + uint16_t ether_type; + efx_tunnel_protocol_t encap_type; + uint32_t inner_frame_match; +} ef10_filter_encap_entry_t; + +#define EF10_ENCAP_FILTER_ENTRY(ipv, encap_type, inner_frame_match) \ + { EFX_ETHER_TYPE_##ipv, EFX_TUNNEL_PROTOCOL_##encap_type, \ + EFX_FILTER_INNER_FRAME_MATCH_UNKNOWN_##inner_frame_match } + +static ef10_filter_encap_entry_t ef10_filter_encap_list[] = { + EF10_ENCAP_FILTER_ENTRY(IPV4, VXLAN, UCAST_DST), + EF10_ENCAP_FILTER_ENTRY(IPV4, VXLAN, MCAST_DST), + EF10_ENCAP_FILTER_ENTRY(IPV6, VXLAN, UCAST_DST), + EF10_ENCAP_FILTER_ENTRY(IPV6, VXLAN, MCAST_DST), + + EF10_ENCAP_FILTER_ENTRY(IPV4, GENEVE, UCAST_DST), + EF10_ENCAP_FILTER_ENTRY(IPV4, GENEVE, MCAST_DST), + EF10_ENCAP_FILTER_ENTRY(IPV6, GENEVE, UCAST_DST), + EF10_ENCAP_FILTER_ENTRY(IPV6, GENEVE, MCAST_DST), + + EF10_ENCAP_FILTER_ENTRY(IPV4, NVGRE, UCAST_DST), + EF10_ENCAP_FILTER_ENTRY(IPV4, NVGRE, MCAST_DST), + EF10_ENCAP_FILTER_ENTRY(IPV6, NVGRE, UCAST_DST), + EF10_ENCAP_FILTER_ENTRY(IPV6, NVGRE, MCAST_DST), +}; + +#undef EF10_ENCAP_FILTER_ENTRY + +static __checkReturn efx_rc_t +ef10_filter_insert_encap_filters( + __in efx_nic_t *enp, + __in boolean_t mulcst, + __in efx_filter_flags_t filter_flags) +{ + ef10_filter_table_t *table = enp->en_filter.ef_ef10_filter_table; + uint32_t i; + efx_rc_t rc; + + EFX_STATIC_ASSERT(EFX_ARRAY_SIZE(ef10_filter_encap_list) <= + EFX_ARRAY_SIZE(table->eft_encap_filter_indexes)); + + /* + * On Medford, full-featured firmware can identify packets as being + * tunnel encapsulated, even if no encapsulated packet offloads are in + * use. When packets are identified as such, ordinary filters are not + * applied, only ones specific to encapsulated packets. Hence we need to + * insert filters for encapsulated packets in order to receive them. + * + * Separate filters need to be inserted for each ether type, + * encapsulation type, and inner frame type (unicast or multicast). To + * keep things simple and reduce the number of filters needed, catch-all + * filters for all combinations of types are inserted, even if + * all_unicst or all_mulcst have not been set. (These catch-all filters + * may well, however, fail to insert on unprivileged functions.) + */ + table->eft_encap_filter_count = 0; + for (i = 0; i < EFX_ARRAY_SIZE(ef10_filter_encap_list); i++) { + efx_filter_spec_t spec; + ef10_filter_encap_entry_t *encap_filter = + &ef10_filter_encap_list[i]; + + /* + * Skip multicast filters if we've not been asked for + * any multicast traffic. + */ + if ((mulcst == B_FALSE) && + (encap_filter->inner_frame_match == + EFX_FILTER_INNER_FRAME_MATCH_UNKNOWN_MCAST_DST)) + continue; + + efx_filter_spec_init_rx(&spec, EFX_FILTER_PRI_AUTO, + filter_flags, + table->eft_default_rxq); + efx_filter_spec_set_ether_type(&spec, encap_filter->ether_type); + rc = efx_filter_spec_set_encap_type(&spec, + encap_filter->encap_type, + encap_filter->inner_frame_match); + if (rc != 0) + goto fail1; + + rc = ef10_filter_add_internal(enp, &spec, B_TRUE, + &table->eft_encap_filter_indexes[ + table->eft_encap_filter_count]); + if (rc != 0) { + if (rc != EACCES) + goto fail2; + } else { + table->eft_encap_filter_count++; + } + } + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +static void +ef10_filter_remove_old( + __in efx_nic_t *enp) +{ + ef10_filter_table_t *table = enp->en_filter.ef_ef10_filter_table; + uint32_t i; + + for (i = 0; i < EFX_ARRAY_SIZE(table->eft_entry); i++) { + if (ef10_filter_entry_is_auto_old(table, i)) { + (void) ef10_filter_delete_internal(enp, i); + } + } +} + + +static __checkReturn efx_rc_t +ef10_filter_get_workarounds( + __in efx_nic_t *enp) +{ + efx_nic_cfg_t *encp = &enp->en_nic_cfg; + uint32_t implemented = 0; + uint32_t enabled = 0; + efx_rc_t rc; + + rc = efx_mcdi_get_workarounds(enp, &implemented, &enabled); + if (rc == 0) { + /* Check if chained multicast filter support is enabled */ + if (implemented & enabled & MC_CMD_GET_WORKAROUNDS_OUT_BUG26807) + encp->enc_bug26807_workaround = B_TRUE; + else + encp->enc_bug26807_workaround = B_FALSE; + } else if (rc == ENOTSUP) { + /* + * Firmware is too old to support GET_WORKAROUNDS, and support + * for this workaround was implemented later. + */ + encp->enc_bug26807_workaround = B_FALSE; + } else { + goto fail1; + } + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); + +} + + +/* + * Reconfigure all filters. + * If all_unicst and/or all mulcst filters cannot be applied then + * return ENOTSUP (Note the filters for the specified addresses are + * still applied in this case). + */ + __checkReturn efx_rc_t +ef10_filter_reconfigure( + __in efx_nic_t *enp, + __in_ecount(6) uint8_t const *mac_addr, + __in boolean_t all_unicst, + __in boolean_t mulcst, + __in boolean_t all_mulcst, + __in boolean_t brdcst, + __in_ecount(6*count) uint8_t const *addrs, + __in uint32_t count) +{ + efx_nic_cfg_t *encp = &enp->en_nic_cfg; + ef10_filter_table_t *table = enp->en_filter.ef_ef10_filter_table; + efx_filter_flags_t filter_flags; + unsigned int i; + efx_rc_t all_unicst_rc = 0; + efx_rc_t all_mulcst_rc = 0; + efx_rc_t rc; + + if (table->eft_default_rxq == NULL) { + /* + * Filters direct traffic to the default RXQ, and so cannot be + * inserted until it is available. Any currently configured + * filters must be removed (ignore errors in case the MC + * has rebooted, which removes hardware filters). + */ + for (i = 0; i < table->eft_unicst_filter_count; i++) { + (void) ef10_filter_delete_internal(enp, + table->eft_unicst_filter_indexes[i]); + } + table->eft_unicst_filter_count = 0; + + for (i = 0; i < table->eft_mulcst_filter_count; i++) { + (void) ef10_filter_delete_internal(enp, + table->eft_mulcst_filter_indexes[i]); + } + table->eft_mulcst_filter_count = 0; + + for (i = 0; i < table->eft_encap_filter_count; i++) { + (void) ef10_filter_delete_internal(enp, + table->eft_encap_filter_indexes[i]); + } + table->eft_encap_filter_count = 0; + + return (0); + } + + if (table->eft_using_rss) + filter_flags = EFX_FILTER_FLAG_RX_RSS; + else + filter_flags = 0; + + /* Mark old filters which may need to be removed */ + for (i = 0; i < table->eft_unicst_filter_count; i++) { + ef10_filter_set_entry_auto_old(table, + table->eft_unicst_filter_indexes[i]); + } + for (i = 0; i < table->eft_mulcst_filter_count; i++) { + ef10_filter_set_entry_auto_old(table, + table->eft_mulcst_filter_indexes[i]); + } + for (i = 0; i < table->eft_encap_filter_count; i++) { + ef10_filter_set_entry_auto_old(table, + table->eft_encap_filter_indexes[i]); + } + + /* + * Insert or renew unicast filters. + * + * Frimware does not perform chaining on unicast filters. As traffic is + * therefore only delivered to the first matching filter, we should + * always insert the specific filter for our MAC address, to try and + * ensure we get that traffic. + * + * (If the filter for our MAC address has already been inserted by + * another function, we won't receive traffic sent to us, even if we + * insert a unicast mismatch filter. To prevent traffic stealing, this + * therefore relies on the privilege model only allowing functions to + * insert filters for their own MAC address unless explicitly given + * additional privileges by the user. This also means that, even on a + * priviliged function, inserting a unicast mismatch filter may not + * catch all traffic in multi PCI function scenarios.) + */ + table->eft_unicst_filter_count = 0; + rc = ef10_filter_insert_unicast(enp, mac_addr, filter_flags); + if (all_unicst || (rc != 0)) { + all_unicst_rc = ef10_filter_insert_all_unicast(enp, + filter_flags); + if ((rc != 0) && (all_unicst_rc != 0)) + goto fail1; + } + + /* + * WORKAROUND_BUG26807 controls firmware support for chained multicast + * filters, and can only be enabled or disabled when the hardware filter + * table is empty. + * + * Chained multicast filters require support from the datapath firmware, + * and may not be available (e.g. low-latency variants or old Huntington + * firmware). + * + * Firmware will reset (FLR) functions which have inserted filters in + * the hardware filter table when the workaround is enabled/disabled. + * Functions without any hardware filters are not reset. + * + * Re-check if the workaround is enabled after adding unicast hardware + * filters. This ensures that encp->enc_bug26807_workaround matches the + * firmware state, and that later changes to enable/disable the + * workaround will result in this function seeing a reset (FLR). + * + * In common-code drivers, we only support multiple PCI function + * scenarios with firmware that supports multicast chaining, so we can + * assume it is enabled for such cases and hence simplify the filter + * insertion logic. Firmware that does not support multicast chaining + * does not support multiple PCI function configurations either, so + * filter insertion is much simpler and the same strategies can still be + * used. + */ + if ((rc = ef10_filter_get_workarounds(enp)) != 0) + goto fail2; + + if ((table->eft_using_all_mulcst != all_mulcst) && + (encp->enc_bug26807_workaround == B_TRUE)) { + /* + * Multicast filter chaining is enabled, so traffic that matches + * more than one multicast filter will be replicated and + * delivered to multiple recipients. To avoid this duplicate + * delivery, remove old multicast filters before inserting new + * multicast filters. + */ + ef10_filter_remove_old(enp); + } + + /* Insert or renew multicast filters */ + if (all_mulcst == B_TRUE) { + /* + * Insert the all multicast filter. If that fails, try to insert + * all of our multicast filters (but without rollback on + * failure). + */ + all_mulcst_rc = ef10_filter_insert_all_multicast(enp, + filter_flags); + if (all_mulcst_rc != 0) { + rc = ef10_filter_insert_multicast_list(enp, B_TRUE, + brdcst, addrs, count, filter_flags, B_FALSE); + if (rc != 0) + goto fail3; + } + } else { + /* + * Insert filters for multicast addresses. + * If any insertion fails, then rollback and try to insert the + * all multicast filter instead. + * If that also fails, try to insert all of the multicast + * filters (but without rollback on failure). + */ + rc = ef10_filter_insert_multicast_list(enp, mulcst, brdcst, + addrs, count, filter_flags, B_TRUE); + if (rc != 0) { + if ((table->eft_using_all_mulcst == B_FALSE) && + (encp->enc_bug26807_workaround == B_TRUE)) { + /* + * Multicast filter chaining is on, so remove + * old filters before inserting the multicast + * all filter to avoid duplicate delivery caused + * by packets matching multiple filters. + */ + ef10_filter_remove_old(enp); + } + + rc = ef10_filter_insert_all_multicast(enp, + filter_flags); + if (rc != 0) { + rc = ef10_filter_insert_multicast_list(enp, + mulcst, brdcst, + addrs, count, filter_flags, B_FALSE); + if (rc != 0) + goto fail4; + } + } + } + + if (encp->enc_tunnel_encapsulations_supported != 0) { + /* Try to insert filters for encapsulated packets. */ + (void) ef10_filter_insert_encap_filters(enp, + mulcst || all_mulcst || brdcst, + filter_flags); + } + + /* Remove old filters which were not renewed */ + ef10_filter_remove_old(enp); + + /* report if any optional flags were rejected */ + if (((all_unicst != B_FALSE) && (all_unicst_rc != 0)) || + ((all_mulcst != B_FALSE) && (all_mulcst_rc != 0))) { + rc = ENOTSUP; + } + + return (rc); + +fail4: + EFSYS_PROBE(fail4); +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + /* Clear auto old flags */ + for (i = 0; i < EFX_ARRAY_SIZE(table->eft_entry); i++) { + if (ef10_filter_entry_is_auto_old(table, i)) { + ef10_filter_set_entry_not_auto_old(table, i); + } + } + + return (rc); +} + + void +ef10_filter_get_default_rxq( + __in efx_nic_t *enp, + __out efx_rxq_t **erpp, + __out boolean_t *using_rss) +{ + ef10_filter_table_t *table = enp->en_filter.ef_ef10_filter_table; + + *erpp = table->eft_default_rxq; + *using_rss = table->eft_using_rss; +} + + + void +ef10_filter_default_rxq_set( + __in efx_nic_t *enp, + __in efx_rxq_t *erp, + __in boolean_t using_rss) +{ + ef10_filter_table_t *table = enp->en_filter.ef_ef10_filter_table; + +#if EFSYS_OPT_RX_SCALE + EFSYS_ASSERT((using_rss == B_FALSE) || + (enp->en_rss_context != EF10_RSS_CONTEXT_INVALID)); + table->eft_using_rss = using_rss; +#else + EFSYS_ASSERT(using_rss == B_FALSE); + table->eft_using_rss = B_FALSE; +#endif + table->eft_default_rxq = erp; +} + + void +ef10_filter_default_rxq_clear( + __in efx_nic_t *enp) +{ + ef10_filter_table_t *table = enp->en_filter.ef_ef10_filter_table; + + table->eft_default_rxq = NULL; + table->eft_using_rss = B_FALSE; +} + + +#endif /* EFSYS_OPT_FILTER */ + +#endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */ diff --git a/dpdk/drivers/net/sfc/base/ef10_impl.h b/dpdk/drivers/net/sfc/base/ef10_impl.h new file mode 100644 index 00000000..8f9eb7a3 --- /dev/null +++ b/dpdk/drivers/net/sfc/base/ef10_impl.h @@ -0,0 +1,1208 @@ +/* + * Copyright (c) 2015-2016 Solarflare Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are + * those of the authors and should not be interpreted as representing official + * policies, either expressed or implied, of the FreeBSD Project. + */ + +#ifndef _SYS_EF10_IMPL_H +#define _SYS_EF10_IMPL_H + +#ifdef __cplusplus +extern "C" { +#endif + +#if (EFSYS_OPT_HUNTINGTON && EFSYS_OPT_MEDFORD) +#define EF10_MAX_PIOBUF_NBUFS MAX(HUNT_PIOBUF_NBUFS, MEDFORD_PIOBUF_NBUFS) +#elif EFSYS_OPT_HUNTINGTON +#define EF10_MAX_PIOBUF_NBUFS HUNT_PIOBUF_NBUFS +#elif EFSYS_OPT_MEDFORD +#define EF10_MAX_PIOBUF_NBUFS MEDFORD_PIOBUF_NBUFS +#endif + +/* + * FIXME: This is just a power of 2 which fits in an MCDI v1 message, and could + * possibly be increased, or the write size reported by newer firmware used + * instead. + */ +#define EF10_NVRAM_CHUNK 0x80 + +/* Alignment requirement for value written to RX WPTR: + * the WPTR must be aligned to an 8 descriptor boundary + */ +#define EF10_RX_WPTR_ALIGN 8 + +/* + * Max byte offset into the packet the TCP header must start for the hardware + * to be able to parse the packet correctly. + */ +#define EF10_TCP_HEADER_OFFSET_LIMIT 208 + +/* Invalid RSS context handle */ +#define EF10_RSS_CONTEXT_INVALID (0xffffffff) + + +/* EV */ + + __checkReturn efx_rc_t +ef10_ev_init( + __in efx_nic_t *enp); + + void +ef10_ev_fini( + __in efx_nic_t *enp); + + __checkReturn efx_rc_t +ef10_ev_qcreate( + __in efx_nic_t *enp, + __in unsigned int index, + __in efsys_mem_t *esmp, + __in size_t n, + __in uint32_t id, + __in uint32_t us, + __in uint32_t flags, + __in efx_evq_t *eep); + + void +ef10_ev_qdestroy( + __in efx_evq_t *eep); + + __checkReturn efx_rc_t +ef10_ev_qprime( + __in efx_evq_t *eep, + __in unsigned int count); + + void +ef10_ev_qpost( + __in efx_evq_t *eep, + __in uint16_t data); + + __checkReturn efx_rc_t +ef10_ev_qmoderate( + __in efx_evq_t *eep, + __in unsigned int us); + +#if EFSYS_OPT_QSTATS + void +ef10_ev_qstats_update( + __in efx_evq_t *eep, + __inout_ecount(EV_NQSTATS) efsys_stat_t *stat); +#endif /* EFSYS_OPT_QSTATS */ + + void +ef10_ev_rxlabel_init( + __in efx_evq_t *eep, + __in efx_rxq_t *erp, + __in unsigned int label, + __in boolean_t packed_stream); + + void +ef10_ev_rxlabel_fini( + __in efx_evq_t *eep, + __in unsigned int label); + +/* INTR */ + + __checkReturn efx_rc_t +ef10_intr_init( + __in efx_nic_t *enp, + __in efx_intr_type_t type, + __in efsys_mem_t *esmp); + + void +ef10_intr_enable( + __in efx_nic_t *enp); + + void +ef10_intr_disable( + __in efx_nic_t *enp); + + void +ef10_intr_disable_unlocked( + __in efx_nic_t *enp); + + __checkReturn efx_rc_t +ef10_intr_trigger( + __in efx_nic_t *enp, + __in unsigned int level); + + void +ef10_intr_status_line( + __in efx_nic_t *enp, + __out boolean_t *fatalp, + __out uint32_t *qmaskp); + + void +ef10_intr_status_message( + __in efx_nic_t *enp, + __in unsigned int message, + __out boolean_t *fatalp); + + void +ef10_intr_fatal( + __in efx_nic_t *enp); + void +ef10_intr_fini( + __in efx_nic_t *enp); + +/* NIC */ + +extern __checkReturn efx_rc_t +ef10_nic_probe( + __in efx_nic_t *enp); + +extern __checkReturn efx_rc_t +ef10_nic_set_drv_limits( + __inout efx_nic_t *enp, + __in efx_drv_limits_t *edlp); + +extern __checkReturn efx_rc_t +ef10_nic_get_vi_pool( + __in efx_nic_t *enp, + __out uint32_t *vi_countp); + +extern __checkReturn efx_rc_t +ef10_nic_get_bar_region( + __in efx_nic_t *enp, + __in efx_nic_region_t region, + __out uint32_t *offsetp, + __out size_t *sizep); + +extern __checkReturn efx_rc_t +ef10_nic_reset( + __in efx_nic_t *enp); + +extern __checkReturn efx_rc_t +ef10_nic_init( + __in efx_nic_t *enp); + +#if EFSYS_OPT_DIAG + +extern __checkReturn efx_rc_t +ef10_nic_register_test( + __in efx_nic_t *enp); + +#endif /* EFSYS_OPT_DIAG */ + +extern void +ef10_nic_fini( + __in efx_nic_t *enp); + +extern void +ef10_nic_unprobe( + __in efx_nic_t *enp); + + +/* MAC */ + +extern __checkReturn efx_rc_t +ef10_mac_poll( + __in efx_nic_t *enp, + __out efx_link_mode_t *link_modep); + +extern __checkReturn efx_rc_t +ef10_mac_up( + __in efx_nic_t *enp, + __out boolean_t *mac_upp); + +extern __checkReturn efx_rc_t +ef10_mac_addr_set( + __in efx_nic_t *enp); + +extern __checkReturn efx_rc_t +ef10_mac_pdu_set( + __in efx_nic_t *enp); + +extern __checkReturn efx_rc_t +ef10_mac_pdu_get( + __in efx_nic_t *enp, + __out size_t *pdu); + +extern __checkReturn efx_rc_t +ef10_mac_reconfigure( + __in efx_nic_t *enp); + +extern __checkReturn efx_rc_t +ef10_mac_multicast_list_set( + __in efx_nic_t *enp); + +extern __checkReturn efx_rc_t +ef10_mac_filter_default_rxq_set( + __in efx_nic_t *enp, + __in efx_rxq_t *erp, + __in boolean_t using_rss); + +extern void +ef10_mac_filter_default_rxq_clear( + __in efx_nic_t *enp); + +#if EFSYS_OPT_LOOPBACK + +extern __checkReturn efx_rc_t +ef10_mac_loopback_set( + __in efx_nic_t *enp, + __in efx_link_mode_t link_mode, + __in efx_loopback_type_t loopback_type); + +#endif /* EFSYS_OPT_LOOPBACK */ + +#if EFSYS_OPT_MAC_STATS + +extern __checkReturn efx_rc_t +ef10_mac_stats_get_mask( + __in efx_nic_t *enp, + __inout_bcount(mask_size) uint32_t *maskp, + __in size_t mask_size); + +extern __checkReturn efx_rc_t +ef10_mac_stats_update( + __in efx_nic_t *enp, + __in efsys_mem_t *esmp, + __inout_ecount(EFX_MAC_NSTATS) efsys_stat_t *stat, + __inout_opt uint32_t *generationp); + +#endif /* EFSYS_OPT_MAC_STATS */ + + +/* MCDI */ + +#if EFSYS_OPT_MCDI + +extern __checkReturn efx_rc_t +ef10_mcdi_init( + __in efx_nic_t *enp, + __in const efx_mcdi_transport_t *mtp); + +extern void +ef10_mcdi_fini( + __in efx_nic_t *enp); + +extern void +ef10_mcdi_send_request( + __in efx_nic_t *enp, + __in_bcount(hdr_len) void *hdrp, + __in size_t hdr_len, + __in_bcount(sdu_len) void *sdup, + __in size_t sdu_len); + +extern __checkReturn boolean_t +ef10_mcdi_poll_response( + __in efx_nic_t *enp); + +extern void +ef10_mcdi_read_response( + __in efx_nic_t *enp, + __out_bcount(length) void *bufferp, + __in size_t offset, + __in size_t length); + +extern efx_rc_t +ef10_mcdi_poll_reboot( + __in efx_nic_t *enp); + +extern __checkReturn efx_rc_t +ef10_mcdi_feature_supported( + __in efx_nic_t *enp, + __in efx_mcdi_feature_id_t id, + __out boolean_t *supportedp); + +extern void +ef10_mcdi_get_timeout( + __in efx_nic_t *enp, + __in efx_mcdi_req_t *emrp, + __out uint32_t *timeoutp); + +#endif /* EFSYS_OPT_MCDI */ + +/* NVRAM */ + +#if EFSYS_OPT_NVRAM || EFSYS_OPT_VPD + +extern __checkReturn efx_rc_t +ef10_nvram_buf_read_tlv( + __in efx_nic_t *enp, + __in_bcount(max_seg_size) caddr_t seg_data, + __in size_t max_seg_size, + __in uint32_t tag, + __deref_out_bcount_opt(*sizep) caddr_t *datap, + __out size_t *sizep); + +extern __checkReturn efx_rc_t +ef10_nvram_buf_write_tlv( + __inout_bcount(partn_size) caddr_t partn_data, + __in size_t partn_size, + __in uint32_t tag, + __in_bcount(tag_size) caddr_t tag_data, + __in size_t tag_size, + __out size_t *total_lengthp); + +extern __checkReturn efx_rc_t +ef10_nvram_partn_read_tlv( + __in efx_nic_t *enp, + __in uint32_t partn, + __in uint32_t tag, + __deref_out_bcount_opt(*sizep) caddr_t *datap, + __out size_t *sizep); + +extern __checkReturn efx_rc_t +ef10_nvram_partn_write_tlv( + __in efx_nic_t *enp, + __in uint32_t partn, + __in uint32_t tag, + __in_bcount(size) caddr_t data, + __in size_t size); + +extern __checkReturn efx_rc_t +ef10_nvram_partn_write_segment_tlv( + __in efx_nic_t *enp, + __in uint32_t partn, + __in uint32_t tag, + __in_bcount(size) caddr_t data, + __in size_t size, + __in boolean_t all_segments); + +extern __checkReturn efx_rc_t +ef10_nvram_partn_lock( + __in efx_nic_t *enp, + __in uint32_t partn); + +extern __checkReturn efx_rc_t +ef10_nvram_partn_unlock( + __in efx_nic_t *enp, + __in uint32_t partn, + __out_opt uint32_t *resultp); + +#endif /* EFSYS_OPT_NVRAM || EFSYS_OPT_VPD */ + +#if EFSYS_OPT_NVRAM + +#if EFSYS_OPT_DIAG + +extern __checkReturn efx_rc_t +ef10_nvram_test( + __in efx_nic_t *enp); + +#endif /* EFSYS_OPT_DIAG */ + +extern __checkReturn efx_rc_t +ef10_nvram_type_to_partn( + __in efx_nic_t *enp, + __in efx_nvram_type_t type, + __out uint32_t *partnp); + +extern __checkReturn efx_rc_t +ef10_nvram_partn_size( + __in efx_nic_t *enp, + __in uint32_t partn, + __out size_t *sizep); + +extern __checkReturn efx_rc_t +ef10_nvram_partn_rw_start( + __in efx_nic_t *enp, + __in uint32_t partn, + __out size_t *chunk_sizep); + +extern __checkReturn efx_rc_t +ef10_nvram_partn_read_mode( + __in efx_nic_t *enp, + __in uint32_t partn, + __in unsigned int offset, + __out_bcount(size) caddr_t data, + __in size_t size, + __in uint32_t mode); + +extern __checkReturn efx_rc_t +ef10_nvram_partn_read( + __in efx_nic_t *enp, + __in uint32_t partn, + __in unsigned int offset, + __out_bcount(size) caddr_t data, + __in size_t size); + +extern __checkReturn efx_rc_t +ef10_nvram_partn_erase( + __in efx_nic_t *enp, + __in uint32_t partn, + __in unsigned int offset, + __in size_t size); + +extern __checkReturn efx_rc_t +ef10_nvram_partn_write( + __in efx_nic_t *enp, + __in uint32_t partn, + __in unsigned int offset, + __out_bcount(size) caddr_t data, + __in size_t size); + +extern __checkReturn efx_rc_t +ef10_nvram_partn_rw_finish( + __in efx_nic_t *enp, + __in uint32_t partn); + +extern __checkReturn efx_rc_t +ef10_nvram_partn_get_version( + __in efx_nic_t *enp, + __in uint32_t partn, + __out uint32_t *subtypep, + __out_ecount(4) uint16_t version[4]); + +extern __checkReturn efx_rc_t +ef10_nvram_partn_set_version( + __in efx_nic_t *enp, + __in uint32_t partn, + __in_ecount(4) uint16_t version[4]); + +extern __checkReturn efx_rc_t +ef10_nvram_buffer_validate( + __in efx_nic_t *enp, + __in uint32_t partn, + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size); + +extern __checkReturn efx_rc_t +ef10_nvram_buffer_create( + __in efx_nic_t *enp, + __in uint16_t partn_type, + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size); + +extern __checkReturn efx_rc_t +ef10_nvram_buffer_find_item_start( + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size, + __out uint32_t *startp + ); + +extern __checkReturn efx_rc_t +ef10_nvram_buffer_find_end( + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size, + __in uint32_t offset, + __out uint32_t *endp + ); + +extern __checkReturn __success(return != B_FALSE) boolean_t +ef10_nvram_buffer_find_item( + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size, + __in uint32_t offset, + __out uint32_t *startp, + __out uint32_t *lengthp + ); + +extern __checkReturn efx_rc_t +ef10_nvram_buffer_get_item( + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size, + __in uint32_t offset, + __in uint32_t length, + __out_bcount_part(item_max_size, *lengthp) + caddr_t itemp, + __in size_t item_max_size, + __out uint32_t *lengthp + ); + +extern __checkReturn efx_rc_t +ef10_nvram_buffer_insert_item( + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size, + __in uint32_t offset, + __in_bcount(length) caddr_t keyp, + __in uint32_t length, + __out uint32_t *lengthp + ); + +extern __checkReturn efx_rc_t +ef10_nvram_buffer_delete_item( + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size, + __in uint32_t offset, + __in uint32_t length, + __in uint32_t end + ); + +extern __checkReturn efx_rc_t +ef10_nvram_buffer_finish( + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size + ); + +#endif /* EFSYS_OPT_NVRAM */ + + +/* PHY */ + +typedef struct ef10_link_state_s { + uint32_t els_adv_cap_mask; + uint32_t els_lp_cap_mask; + unsigned int els_fcntl; + efx_link_mode_t els_link_mode; +#if EFSYS_OPT_LOOPBACK + efx_loopback_type_t els_loopback; +#endif + boolean_t els_mac_up; +} ef10_link_state_t; + +extern void +ef10_phy_link_ev( + __in efx_nic_t *enp, + __in efx_qword_t *eqp, + __out efx_link_mode_t *link_modep); + +extern __checkReturn efx_rc_t +ef10_phy_get_link( + __in efx_nic_t *enp, + __out ef10_link_state_t *elsp); + +extern __checkReturn efx_rc_t +ef10_phy_power( + __in efx_nic_t *enp, + __in boolean_t on); + +extern __checkReturn efx_rc_t +ef10_phy_reconfigure( + __in efx_nic_t *enp); + +extern __checkReturn efx_rc_t +ef10_phy_verify( + __in efx_nic_t *enp); + +extern __checkReturn efx_rc_t +ef10_phy_oui_get( + __in efx_nic_t *enp, + __out uint32_t *ouip); + +#if EFSYS_OPT_PHY_STATS + +extern __checkReturn efx_rc_t +ef10_phy_stats_update( + __in efx_nic_t *enp, + __in efsys_mem_t *esmp, + __inout_ecount(EFX_PHY_NSTATS) uint32_t *stat); + +#endif /* EFSYS_OPT_PHY_STATS */ + +#if EFSYS_OPT_BIST + +extern __checkReturn efx_rc_t +ef10_bist_enable_offline( + __in efx_nic_t *enp); + +extern __checkReturn efx_rc_t +ef10_bist_start( + __in efx_nic_t *enp, + __in efx_bist_type_t type); + +extern __checkReturn efx_rc_t +ef10_bist_poll( + __in efx_nic_t *enp, + __in efx_bist_type_t type, + __out efx_bist_result_t *resultp, + __out_opt __drv_when(count > 0, __notnull) + uint32_t *value_maskp, + __out_ecount_opt(count) __drv_when(count > 0, __notnull) + unsigned long *valuesp, + __in size_t count); + +extern void +ef10_bist_stop( + __in efx_nic_t *enp, + __in efx_bist_type_t type); + +#endif /* EFSYS_OPT_BIST */ + +/* TX */ + +extern __checkReturn efx_rc_t +ef10_tx_init( + __in efx_nic_t *enp); + +extern void +ef10_tx_fini( + __in efx_nic_t *enp); + +extern __checkReturn efx_rc_t +ef10_tx_qcreate( + __in efx_nic_t *enp, + __in unsigned int index, + __in unsigned int label, + __in efsys_mem_t *esmp, + __in size_t n, + __in uint32_t id, + __in uint16_t flags, + __in efx_evq_t *eep, + __in efx_txq_t *etp, + __out unsigned int *addedp); + +extern void +ef10_tx_qdestroy( + __in efx_txq_t *etp); + +extern __checkReturn efx_rc_t +ef10_tx_qpost( + __in efx_txq_t *etp, + __in_ecount(n) efx_buffer_t *eb, + __in unsigned int n, + __in unsigned int completed, + __inout unsigned int *addedp); + +extern void +ef10_tx_qpush( + __in efx_txq_t *etp, + __in unsigned int added, + __in unsigned int pushed); + +#if EFSYS_OPT_RX_PACKED_STREAM +extern void +ef10_rx_qps_update_credits( + __in efx_rxq_t *erp); + +extern __checkReturn uint8_t * +ef10_rx_qps_packet_info( + __in efx_rxq_t *erp, + __in uint8_t *buffer, + __in uint32_t buffer_length, + __in uint32_t current_offset, + __out uint16_t *lengthp, + __out uint32_t *next_offsetp, + __out uint32_t *timestamp); +#endif + +extern __checkReturn efx_rc_t +ef10_tx_qpace( + __in efx_txq_t *etp, + __in unsigned int ns); + +extern __checkReturn efx_rc_t +ef10_tx_qflush( + __in efx_txq_t *etp); + +extern void +ef10_tx_qenable( + __in efx_txq_t *etp); + +extern __checkReturn efx_rc_t +ef10_tx_qpio_enable( + __in efx_txq_t *etp); + +extern void +ef10_tx_qpio_disable( + __in efx_txq_t *etp); + +extern __checkReturn efx_rc_t +ef10_tx_qpio_write( + __in efx_txq_t *etp, + __in_ecount(buf_length) uint8_t *buffer, + __in size_t buf_length, + __in size_t pio_buf_offset); + +extern __checkReturn efx_rc_t +ef10_tx_qpio_post( + __in efx_txq_t *etp, + __in size_t pkt_length, + __in unsigned int completed, + __inout unsigned int *addedp); + +extern __checkReturn efx_rc_t +ef10_tx_qdesc_post( + __in efx_txq_t *etp, + __in_ecount(n) efx_desc_t *ed, + __in unsigned int n, + __in unsigned int completed, + __inout unsigned int *addedp); + +extern void +ef10_tx_qdesc_dma_create( + __in efx_txq_t *etp, + __in efsys_dma_addr_t addr, + __in size_t size, + __in boolean_t eop, + __out efx_desc_t *edp); + +extern void +ef10_tx_qdesc_tso_create( + __in efx_txq_t *etp, + __in uint16_t ipv4_id, + __in uint32_t tcp_seq, + __in uint8_t tcp_flags, + __out efx_desc_t *edp); + +extern void +ef10_tx_qdesc_tso2_create( + __in efx_txq_t *etp, + __in uint16_t ipv4_id, + __in uint32_t tcp_seq, + __in uint16_t tcp_mss, + __out_ecount(count) efx_desc_t *edp, + __in int count); + +extern void +ef10_tx_qdesc_vlantci_create( + __in efx_txq_t *etp, + __in uint16_t vlan_tci, + __out efx_desc_t *edp); + + +#if EFSYS_OPT_QSTATS + +extern void +ef10_tx_qstats_update( + __in efx_txq_t *etp, + __inout_ecount(TX_NQSTATS) efsys_stat_t *stat); + +#endif /* EFSYS_OPT_QSTATS */ + +typedef uint32_t efx_piobuf_handle_t; + +#define EFX_PIOBUF_HANDLE_INVALID ((efx_piobuf_handle_t) -1) + +extern __checkReturn efx_rc_t +ef10_nic_pio_alloc( + __inout efx_nic_t *enp, + __out uint32_t *bufnump, + __out efx_piobuf_handle_t *handlep, + __out uint32_t *blknump, + __out uint32_t *offsetp, + __out size_t *sizep); + +extern __checkReturn efx_rc_t +ef10_nic_pio_free( + __inout efx_nic_t *enp, + __in uint32_t bufnum, + __in uint32_t blknum); + +extern __checkReturn efx_rc_t +ef10_nic_pio_link( + __inout efx_nic_t *enp, + __in uint32_t vi_index, + __in efx_piobuf_handle_t handle); + +extern __checkReturn efx_rc_t +ef10_nic_pio_unlink( + __inout efx_nic_t *enp, + __in uint32_t vi_index); + + +/* VPD */ + +#if EFSYS_OPT_VPD + +extern __checkReturn efx_rc_t +ef10_vpd_init( + __in efx_nic_t *enp); + +extern __checkReturn efx_rc_t +ef10_vpd_size( + __in efx_nic_t *enp, + __out size_t *sizep); + +extern __checkReturn efx_rc_t +ef10_vpd_read( + __in efx_nic_t *enp, + __out_bcount(size) caddr_t data, + __in size_t size); + +extern __checkReturn efx_rc_t +ef10_vpd_verify( + __in efx_nic_t *enp, + __in_bcount(size) caddr_t data, + __in size_t size); + +extern __checkReturn efx_rc_t +ef10_vpd_reinit( + __in efx_nic_t *enp, + __in_bcount(size) caddr_t data, + __in size_t size); + +extern __checkReturn efx_rc_t +ef10_vpd_get( + __in efx_nic_t *enp, + __in_bcount(size) caddr_t data, + __in size_t size, + __inout efx_vpd_value_t *evvp); + +extern __checkReturn efx_rc_t +ef10_vpd_set( + __in efx_nic_t *enp, + __in_bcount(size) caddr_t data, + __in size_t size, + __in efx_vpd_value_t *evvp); + +extern __checkReturn efx_rc_t +ef10_vpd_next( + __in efx_nic_t *enp, + __in_bcount(size) caddr_t data, + __in size_t size, + __out efx_vpd_value_t *evvp, + __inout unsigned int *contp); + +extern __checkReturn efx_rc_t +ef10_vpd_write( + __in efx_nic_t *enp, + __in_bcount(size) caddr_t data, + __in size_t size); + +extern void +ef10_vpd_fini( + __in efx_nic_t *enp); + +#endif /* EFSYS_OPT_VPD */ + + +/* RX */ + +extern __checkReturn efx_rc_t +ef10_rx_init( + __in efx_nic_t *enp); + +#if EFSYS_OPT_RX_SCATTER +extern __checkReturn efx_rc_t +ef10_rx_scatter_enable( + __in efx_nic_t *enp, + __in unsigned int buf_size); +#endif /* EFSYS_OPT_RX_SCATTER */ + + +#if EFSYS_OPT_RX_SCALE + +extern __checkReturn efx_rc_t +ef10_rx_scale_context_alloc( + __in efx_nic_t *enp, + __in efx_rx_scale_context_type_t type, + __in uint32_t num_queues, + __out uint32_t *rss_contextp); + +extern __checkReturn efx_rc_t +ef10_rx_scale_context_free( + __in efx_nic_t *enp, + __in uint32_t rss_context); + +extern __checkReturn efx_rc_t +ef10_rx_scale_mode_set( + __in efx_nic_t *enp, + __in uint32_t rss_context, + __in efx_rx_hash_alg_t alg, + __in efx_rx_hash_type_t type, + __in boolean_t insert); + +extern __checkReturn efx_rc_t +ef10_rx_scale_key_set( + __in efx_nic_t *enp, + __in uint32_t rss_context, + __in_ecount(n) uint8_t *key, + __in size_t n); + +extern __checkReturn efx_rc_t +ef10_rx_scale_tbl_set( + __in efx_nic_t *enp, + __in uint32_t rss_context, + __in_ecount(n) unsigned int *table, + __in size_t n); + +extern __checkReturn uint32_t +ef10_rx_prefix_hash( + __in efx_nic_t *enp, + __in efx_rx_hash_alg_t func, + __in uint8_t *buffer); + +#endif /* EFSYS_OPT_RX_SCALE */ + +extern __checkReturn efx_rc_t +ef10_rx_prefix_pktlen( + __in efx_nic_t *enp, + __in uint8_t *buffer, + __out uint16_t *lengthp); + +extern void +ef10_rx_qpost( + __in efx_rxq_t *erp, + __in_ecount(n) efsys_dma_addr_t *addrp, + __in size_t size, + __in unsigned int n, + __in unsigned int completed, + __in unsigned int added); + +extern void +ef10_rx_qpush( + __in efx_rxq_t *erp, + __in unsigned int added, + __inout unsigned int *pushedp); + +extern __checkReturn efx_rc_t +ef10_rx_qflush( + __in efx_rxq_t *erp); + +extern void +ef10_rx_qenable( + __in efx_rxq_t *erp); + +extern __checkReturn efx_rc_t +ef10_rx_qcreate( + __in efx_nic_t *enp, + __in unsigned int index, + __in unsigned int label, + __in efx_rxq_type_t type, + __in efsys_mem_t *esmp, + __in size_t n, + __in uint32_t id, + __in efx_evq_t *eep, + __in efx_rxq_t *erp); + +extern void +ef10_rx_qdestroy( + __in efx_rxq_t *erp); + +extern void +ef10_rx_fini( + __in efx_nic_t *enp); + +#if EFSYS_OPT_FILTER + +typedef struct ef10_filter_handle_s { + uint32_t efh_lo; + uint32_t efh_hi; +} ef10_filter_handle_t; + +typedef struct ef10_filter_entry_s { + uintptr_t efe_spec; /* pointer to filter spec plus busy bit */ + ef10_filter_handle_t efe_handle; +} ef10_filter_entry_t; + +/* + * BUSY flag indicates that an update is in progress. + * AUTO_OLD flag is used to mark and sweep MAC packet filters. + */ +#define EFX_EF10_FILTER_FLAG_BUSY 1U +#define EFX_EF10_FILTER_FLAG_AUTO_OLD 2U +#define EFX_EF10_FILTER_FLAGS 3U + +/* + * Size of the hash table used by the driver. Doesn't need to be the + * same size as the hardware's table. + */ +#define EFX_EF10_FILTER_TBL_ROWS 8192 + +/* Only need to allow for one directed and one unknown unicast filter */ +#define EFX_EF10_FILTER_UNICAST_FILTERS_MAX 2 + +/* Allow for the broadcast address to be added to the multicast list */ +#define EFX_EF10_FILTER_MULTICAST_FILTERS_MAX (EFX_MAC_MULTICAST_LIST_MAX + 1) + +/* + * For encapsulated packets, there is one filter each for each combination of + * IPv4 or IPv6 outer frame, VXLAN, GENEVE or NVGRE packet type, and unicast or + * multicast inner frames. + */ +#define EFX_EF10_FILTER_ENCAP_FILTERS_MAX 12 + +typedef struct ef10_filter_table_s { + ef10_filter_entry_t eft_entry[EFX_EF10_FILTER_TBL_ROWS]; + efx_rxq_t *eft_default_rxq; + boolean_t eft_using_rss; + uint32_t eft_unicst_filter_indexes[ + EFX_EF10_FILTER_UNICAST_FILTERS_MAX]; + uint32_t eft_unicst_filter_count; + uint32_t eft_mulcst_filter_indexes[ + EFX_EF10_FILTER_MULTICAST_FILTERS_MAX]; + uint32_t eft_mulcst_filter_count; + boolean_t eft_using_all_mulcst; + uint32_t eft_encap_filter_indexes[ + EFX_EF10_FILTER_ENCAP_FILTERS_MAX]; + uint32_t eft_encap_filter_count; +} ef10_filter_table_t; + + __checkReturn efx_rc_t +ef10_filter_init( + __in efx_nic_t *enp); + + void +ef10_filter_fini( + __in efx_nic_t *enp); + + __checkReturn efx_rc_t +ef10_filter_restore( + __in efx_nic_t *enp); + + __checkReturn efx_rc_t +ef10_filter_add( + __in efx_nic_t *enp, + __inout efx_filter_spec_t *spec, + __in boolean_t may_replace); + + __checkReturn efx_rc_t +ef10_filter_delete( + __in efx_nic_t *enp, + __inout efx_filter_spec_t *spec); + +extern __checkReturn efx_rc_t +ef10_filter_supported_filters( + __in efx_nic_t *enp, + __out_ecount(buffer_length) uint32_t *buffer, + __in size_t buffer_length, + __out size_t *list_lengthp); + +extern __checkReturn efx_rc_t +ef10_filter_reconfigure( + __in efx_nic_t *enp, + __in_ecount(6) uint8_t const *mac_addr, + __in boolean_t all_unicst, + __in boolean_t mulcst, + __in boolean_t all_mulcst, + __in boolean_t brdcst, + __in_ecount(6*count) uint8_t const *addrs, + __in uint32_t count); + +extern void +ef10_filter_get_default_rxq( + __in efx_nic_t *enp, + __out efx_rxq_t **erpp, + __out boolean_t *using_rss); + +extern void +ef10_filter_default_rxq_set( + __in efx_nic_t *enp, + __in efx_rxq_t *erp, + __in boolean_t using_rss); + +extern void +ef10_filter_default_rxq_clear( + __in efx_nic_t *enp); + + +#endif /* EFSYS_OPT_FILTER */ + +extern __checkReturn efx_rc_t +efx_mcdi_get_function_info( + __in efx_nic_t *enp, + __out uint32_t *pfp, + __out_opt uint32_t *vfp); + +extern __checkReturn efx_rc_t +efx_mcdi_privilege_mask( + __in efx_nic_t *enp, + __in uint32_t pf, + __in uint32_t vf, + __out uint32_t *maskp); + +extern __checkReturn efx_rc_t +efx_mcdi_get_port_assignment( + __in efx_nic_t *enp, + __out uint32_t *portp); + +extern __checkReturn efx_rc_t +efx_mcdi_get_port_modes( + __in efx_nic_t *enp, + __out uint32_t *modesp, + __out_opt uint32_t *current_modep); + +extern __checkReturn efx_rc_t +ef10_nic_get_port_mode_bandwidth( + __in uint32_t port_mode, + __out uint32_t *bandwidth_mbpsp); + +extern __checkReturn efx_rc_t +efx_mcdi_get_mac_address_pf( + __in efx_nic_t *enp, + __out_ecount_opt(6) uint8_t mac_addrp[6]); + +extern __checkReturn efx_rc_t +efx_mcdi_get_mac_address_vf( + __in efx_nic_t *enp, + __out_ecount_opt(6) uint8_t mac_addrp[6]); + +extern __checkReturn efx_rc_t +efx_mcdi_get_clock( + __in efx_nic_t *enp, + __out uint32_t *sys_freqp, + __out uint32_t *dpcpu_freqp); + + +extern __checkReturn efx_rc_t +efx_mcdi_get_vector_cfg( + __in efx_nic_t *enp, + __out_opt uint32_t *vec_basep, + __out_opt uint32_t *pf_nvecp, + __out_opt uint32_t *vf_nvecp); + +extern __checkReturn efx_rc_t +ef10_get_datapath_caps( + __in efx_nic_t *enp); + +extern __checkReturn efx_rc_t +ef10_get_privilege_mask( + __in efx_nic_t *enp, + __out uint32_t *maskp); + +extern __checkReturn efx_rc_t +ef10_external_port_mapping( + __in efx_nic_t *enp, + __in uint32_t port, + __out uint8_t *external_portp); + +#if EFSYS_OPT_RX_PACKED_STREAM + +/* Data space per credit in packed stream mode */ +#define EFX_RX_PACKED_STREAM_MEM_PER_CREDIT (1 << 16) + +/* + * Received packets are always aligned at this boundary. Also there always + * exists a gap of this size between packets. + * (see SF-112241-TC, 4.5) + */ +#define EFX_RX_PACKED_STREAM_ALIGNMENT 64 + +/* + * Size of a pseudo-header prepended to received packets + * in packed stream mode + */ +#define EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE 8 + +/* Minimum space for packet in packed stream mode */ +#define EFX_RX_PACKED_STREAM_MIN_PACKET_SPACE \ + P2ROUNDUP(EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE + \ + EFX_MAC_PDU_MIN + \ + EFX_RX_PACKED_STREAM_ALIGNMENT, \ + EFX_RX_PACKED_STREAM_ALIGNMENT) + +/* Maximum number of credits */ +#define EFX_RX_PACKED_STREAM_MAX_CREDITS 127 + +#endif /* EFSYS_OPT_RX_PACKED_STREAM */ + +#ifdef __cplusplus +} +#endif + +#endif /* _SYS_EF10_IMPL_H */ diff --git a/dpdk/drivers/net/sfc/base/ef10_intr.c b/dpdk/drivers/net/sfc/base/ef10_intr.c new file mode 100644 index 00000000..16be3d8c --- /dev/null +++ b/dpdk/drivers/net/sfc/base/ef10_intr.c @@ -0,0 +1,197 @@ +/* + * Copyright (c) 2012-2016 Solarflare Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are + * those of the authors and should not be interpreted as representing official + * policies, either expressed or implied, of the FreeBSD Project. + */ + +#include "efx.h" +#include "efx_impl.h" + + +#if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD + + __checkReturn efx_rc_t +ef10_intr_init( + __in efx_nic_t *enp, + __in efx_intr_type_t type, + __in efsys_mem_t *esmp) +{ + _NOTE(ARGUNUSED(enp, type, esmp)) + return (0); +} + + + void +ef10_intr_enable( + __in efx_nic_t *enp) +{ + _NOTE(ARGUNUSED(enp)) +} + + + void +ef10_intr_disable( + __in efx_nic_t *enp) +{ + _NOTE(ARGUNUSED(enp)) +} + + + void +ef10_intr_disable_unlocked( + __in efx_nic_t *enp) +{ + _NOTE(ARGUNUSED(enp)) +} + + +static __checkReturn efx_rc_t +efx_mcdi_trigger_interrupt( + __in efx_nic_t *enp, + __in unsigned int level) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_TRIGGER_INTERRUPT_IN_LEN, + MC_CMD_TRIGGER_INTERRUPT_OUT_LEN)]; + efx_rc_t rc; + + EFSYS_ASSERT(enp->en_family == EFX_FAMILY_HUNTINGTON || + enp->en_family == EFX_FAMILY_MEDFORD); + + if (level >= enp->en_nic_cfg.enc_intr_limit) { + rc = EINVAL; + goto fail1; + } + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_TRIGGER_INTERRUPT; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_TRIGGER_INTERRUPT_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_TRIGGER_INTERRUPT_OUT_LEN; + + MCDI_IN_SET_DWORD(req, TRIGGER_INTERRUPT_IN_INTR_LEVEL, level); + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail2; + } + + return (0); + +fail2: + EFSYS_PROBE(fail2); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +ef10_intr_trigger( + __in efx_nic_t *enp, + __in unsigned int level) +{ + efx_nic_cfg_t *encp = &(enp->en_nic_cfg); + efx_rc_t rc; + + if (encp->enc_bug41750_workaround) { + /* + * bug 41750: Test interrupts don't work on Greenport + * bug 50084: Test interrupts don't work on VFs + */ + rc = ENOTSUP; + goto fail1; + } + + if ((rc = efx_mcdi_trigger_interrupt(enp, level)) != 0) + goto fail2; + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + void +ef10_intr_status_line( + __in efx_nic_t *enp, + __out boolean_t *fatalp, + __out uint32_t *qmaskp) +{ + efx_dword_t dword; + + EFSYS_ASSERT(enp->en_family == EFX_FAMILY_HUNTINGTON || + enp->en_family == EFX_FAMILY_MEDFORD); + + /* Read the queue mask and implicitly acknowledge the interrupt. */ + EFX_BAR_READD(enp, ER_DZ_BIU_INT_ISR_REG, &dword, B_FALSE); + *qmaskp = EFX_DWORD_FIELD(dword, EFX_DWORD_0); + + EFSYS_PROBE1(qmask, uint32_t, *qmaskp); + + *fatalp = B_FALSE; +} + + void +ef10_intr_status_message( + __in efx_nic_t *enp, + __in unsigned int message, + __out boolean_t *fatalp) +{ + EFSYS_ASSERT(enp->en_family == EFX_FAMILY_HUNTINGTON || + enp->en_family == EFX_FAMILY_MEDFORD); + + _NOTE(ARGUNUSED(enp, message)) + + /* EF10 fatal errors are reported via events */ + *fatalp = B_FALSE; +} + + void +ef10_intr_fatal( + __in efx_nic_t *enp) +{ + /* EF10 fatal errors are reported via events */ + _NOTE(ARGUNUSED(enp)) +} + + void +ef10_intr_fini( + __in efx_nic_t *enp) +{ + _NOTE(ARGUNUSED(enp)) +} + +#endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */ diff --git a/dpdk/drivers/net/sfc/base/ef10_mac.c b/dpdk/drivers/net/sfc/base/ef10_mac.c new file mode 100644 index 00000000..488633f5 --- /dev/null +++ b/dpdk/drivers/net/sfc/base/ef10_mac.c @@ -0,0 +1,897 @@ +/* + * Copyright (c) 2012-2016 Solarflare Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are + * those of the authors and should not be interpreted as representing official + * policies, either expressed or implied, of the FreeBSD Project. + */ + +#include "efx.h" +#include "efx_impl.h" + + +#if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD + + __checkReturn efx_rc_t +ef10_mac_poll( + __in efx_nic_t *enp, + __out efx_link_mode_t *link_modep) +{ + efx_port_t *epp = &(enp->en_port); + ef10_link_state_t els; + efx_rc_t rc; + + if ((rc = ef10_phy_get_link(enp, &els)) != 0) + goto fail1; + + epp->ep_adv_cap_mask = els.els_adv_cap_mask; + epp->ep_fcntl = els.els_fcntl; + + *link_modep = els.els_link_mode; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + *link_modep = EFX_LINK_UNKNOWN; + + return (rc); +} + + __checkReturn efx_rc_t +ef10_mac_up( + __in efx_nic_t *enp, + __out boolean_t *mac_upp) +{ + ef10_link_state_t els; + efx_rc_t rc; + + /* + * Because EF10 doesn't *require* polling, we can't rely on + * ef10_mac_poll() being executed to populate epp->ep_mac_up. + */ + if ((rc = ef10_phy_get_link(enp, &els)) != 0) + goto fail1; + + *mac_upp = els.els_mac_up; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +/* + * EF10 adapters use MC_CMD_VADAPTOR_SET_MAC to set the + * MAC address; the address field in MC_CMD_SET_MAC has no + * effect. + * MC_CMD_VADAPTOR_SET_MAC requires mac-spoofing privilege and + * the port to have no filters or queues active. + */ +static __checkReturn efx_rc_t +efx_mcdi_vadapter_set_mac( + __in efx_nic_t *enp) +{ + efx_port_t *epp = &(enp->en_port); + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_VADAPTOR_SET_MAC_IN_LEN, + MC_CMD_VADAPTOR_SET_MAC_OUT_LEN)]; + efx_rc_t rc; + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_VADAPTOR_SET_MAC; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_VADAPTOR_SET_MAC_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_VADAPTOR_SET_MAC_OUT_LEN; + + MCDI_IN_SET_DWORD(req, VADAPTOR_SET_MAC_IN_UPSTREAM_PORT_ID, + enp->en_vport_id); + EFX_MAC_ADDR_COPY(MCDI_IN2(req, uint8_t, VADAPTOR_SET_MAC_IN_MACADDR), + epp->ep_mac_addr); + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +ef10_mac_addr_set( + __in efx_nic_t *enp) +{ + efx_rc_t rc; + + if ((rc = efx_mcdi_vadapter_set_mac(enp)) != 0) { + if (rc != ENOTSUP) + goto fail1; + + /* + * Fallback for older Huntington firmware without Vadapter + * support. + */ + if ((rc = ef10_mac_reconfigure(enp)) != 0) + goto fail2; + } + + return (0); + +fail2: + EFSYS_PROBE(fail2); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +static __checkReturn efx_rc_t +efx_mcdi_mtu_set( + __in efx_nic_t *enp, + __in uint32_t mtu) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_SET_MAC_EXT_IN_LEN, + MC_CMD_SET_MAC_OUT_LEN)]; + efx_rc_t rc; + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_SET_MAC; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_SET_MAC_EXT_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_SET_MAC_OUT_LEN; + + /* Only configure the MTU in this call to MC_CMD_SET_MAC */ + MCDI_IN_SET_DWORD(req, SET_MAC_EXT_IN_MTU, mtu); + MCDI_IN_POPULATE_DWORD_1(req, SET_MAC_EXT_IN_CONTROL, + SET_MAC_EXT_IN_CFG_MTU, 1); + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +static __checkReturn efx_rc_t +efx_mcdi_mtu_get( + __in efx_nic_t *enp, + __out size_t *mtu) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_SET_MAC_EXT_IN_LEN, + MC_CMD_SET_MAC_V2_OUT_LEN)]; + efx_rc_t rc; + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_SET_MAC; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_SET_MAC_EXT_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_SET_MAC_V2_OUT_LEN; + + /* + * With MC_CMD_SET_MAC_EXT_IN_CONTROL set to 0, this just queries the + * MTU. This should always be supported on Medford, but it is not + * supported on older Huntington firmware. + */ + MCDI_IN_SET_DWORD(req, SET_MAC_EXT_IN_CONTROL, 0); + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + if (req.emr_out_length_used < MC_CMD_SET_MAC_V2_OUT_MTU_OFST + 4) { + rc = EMSGSIZE; + goto fail2; + } + + *mtu = MCDI_OUT_DWORD(req, SET_MAC_V2_OUT_MTU); + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +ef10_mac_pdu_set( + __in efx_nic_t *enp) +{ + efx_port_t *epp = &(enp->en_port); + efx_nic_cfg_t *encp = &(enp->en_nic_cfg); + efx_rc_t rc; + + if (encp->enc_enhanced_set_mac_supported) { + if ((rc = efx_mcdi_mtu_set(enp, epp->ep_mac_pdu)) != 0) + goto fail1; + } else { + /* + * Fallback for older Huntington firmware, which always + * configure all of the parameters to MC_CMD_SET_MAC. This isn't + * suitable for setting the MTU on unpriviliged functions. + */ + if ((rc = ef10_mac_reconfigure(enp)) != 0) + goto fail2; + } + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +ef10_mac_pdu_get( + __in efx_nic_t *enp, + __out size_t *pdu) +{ + efx_rc_t rc; + + if ((rc = efx_mcdi_mtu_get(enp, pdu)) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +__checkReturn efx_rc_t +ef10_mac_reconfigure( + __in efx_nic_t *enp) +{ + efx_port_t *epp = &(enp->en_port); + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_SET_MAC_IN_LEN, + MC_CMD_SET_MAC_OUT_LEN)]; + efx_rc_t rc; + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_SET_MAC; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_SET_MAC_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_SET_MAC_OUT_LEN; + + MCDI_IN_SET_DWORD(req, SET_MAC_IN_MTU, epp->ep_mac_pdu); + MCDI_IN_SET_DWORD(req, SET_MAC_IN_DRAIN, epp->ep_mac_drain ? 1 : 0); + EFX_MAC_ADDR_COPY(MCDI_IN2(req, uint8_t, SET_MAC_IN_ADDR), + epp->ep_mac_addr); + + /* + * Note: The Huntington MAC does not support REJECT_BRDCST. + * The REJECT_UNCST flag will also prevent multicast traffic + * from reaching the filters. As Huntington filters drop any + * traffic that does not match a filter it is ok to leave the + * MAC running in promiscuous mode. See bug41141. + * + * FIXME: Does REJECT_UNCST behave the same way on Medford? + */ + MCDI_IN_POPULATE_DWORD_2(req, SET_MAC_IN_REJECT, + SET_MAC_IN_REJECT_UNCST, 0, + SET_MAC_IN_REJECT_BRDCST, 0); + + /* + * Flow control, whether it is auto-negotiated or not, + * is set via the PHY advertised capabilities. When set to + * automatic the MAC will use the PHY settings to determine + * the flow control settings. + */ + MCDI_IN_SET_DWORD(req, SET_MAC_IN_FCNTL, MC_CMD_FCNTL_AUTO); + + /* Do not include the Ethernet frame checksum in RX packets */ + MCDI_IN_POPULATE_DWORD_1(req, SET_MAC_IN_FLAGS, + SET_MAC_IN_FLAG_INCLUDE_FCS, 0); + + efx_mcdi_execute_quiet(enp, &req); + + if (req.emr_rc != 0) { + /* + * Unprivileged functions cannot control link state, + * but still need to configure filters. + */ + if (req.emr_rc != EACCES) { + rc = req.emr_rc; + goto fail1; + } + } + + /* + * Apply the filters for the MAC configuration. + * If the NIC isn't ready to accept filters this may + * return success without setting anything. + */ + rc = efx_filter_reconfigure(enp, epp->ep_mac_addr, + epp->ep_all_unicst, epp->ep_mulcst, + epp->ep_all_mulcst, epp->ep_brdcst, + epp->ep_mulcst_addr_list, + epp->ep_mulcst_addr_count); + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +ef10_mac_multicast_list_set( + __in efx_nic_t *enp) +{ + efx_port_t *epp = &(enp->en_port); + const efx_mac_ops_t *emop = epp->ep_emop; + efx_rc_t rc; + + EFSYS_ASSERT(enp->en_family == EFX_FAMILY_HUNTINGTON || + enp->en_family == EFX_FAMILY_MEDFORD); + + if ((rc = emop->emo_reconfigure(enp)) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +ef10_mac_filter_default_rxq_set( + __in efx_nic_t *enp, + __in efx_rxq_t *erp, + __in boolean_t using_rss) +{ + efx_port_t *epp = &(enp->en_port); + efx_rxq_t *old_rxq; + boolean_t old_using_rss; + efx_rc_t rc; + + ef10_filter_get_default_rxq(enp, &old_rxq, &old_using_rss); + + ef10_filter_default_rxq_set(enp, erp, using_rss); + + rc = efx_filter_reconfigure(enp, epp->ep_mac_addr, + epp->ep_all_unicst, epp->ep_mulcst, + epp->ep_all_mulcst, epp->ep_brdcst, + epp->ep_mulcst_addr_list, + epp->ep_mulcst_addr_count); + + if (rc != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + ef10_filter_default_rxq_set(enp, old_rxq, old_using_rss); + + return (rc); +} + + void +ef10_mac_filter_default_rxq_clear( + __in efx_nic_t *enp) +{ + efx_port_t *epp = &(enp->en_port); + + ef10_filter_default_rxq_clear(enp); + + efx_filter_reconfigure(enp, epp->ep_mac_addr, + epp->ep_all_unicst, epp->ep_mulcst, + epp->ep_all_mulcst, epp->ep_brdcst, + epp->ep_mulcst_addr_list, + epp->ep_mulcst_addr_count); +} + + +#if EFSYS_OPT_LOOPBACK + + __checkReturn efx_rc_t +ef10_mac_loopback_set( + __in efx_nic_t *enp, + __in efx_link_mode_t link_mode, + __in efx_loopback_type_t loopback_type) +{ + efx_port_t *epp = &(enp->en_port); + const efx_phy_ops_t *epop = epp->ep_epop; + efx_loopback_type_t old_loopback_type; + efx_link_mode_t old_loopback_link_mode; + efx_rc_t rc; + + /* The PHY object handles this on EF10 */ + old_loopback_type = epp->ep_loopback_type; + old_loopback_link_mode = epp->ep_loopback_link_mode; + epp->ep_loopback_type = loopback_type; + epp->ep_loopback_link_mode = link_mode; + + if ((rc = epop->epo_reconfigure(enp)) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + epp->ep_loopback_type = old_loopback_type; + epp->ep_loopback_link_mode = old_loopback_link_mode; + + return (rc); +} + +#endif /* EFSYS_OPT_LOOPBACK */ + +#if EFSYS_OPT_MAC_STATS + + __checkReturn efx_rc_t +ef10_mac_stats_get_mask( + __in efx_nic_t *enp, + __inout_bcount(mask_size) uint32_t *maskp, + __in size_t mask_size) +{ + const struct efx_mac_stats_range ef10_common[] = { + { EFX_MAC_RX_OCTETS, EFX_MAC_RX_GE_15XX_PKTS }, + { EFX_MAC_RX_FCS_ERRORS, EFX_MAC_RX_DROP_EVENTS }, + { EFX_MAC_RX_JABBER_PKTS, EFX_MAC_RX_JABBER_PKTS }, + { EFX_MAC_RX_NODESC_DROP_CNT, EFX_MAC_TX_PAUSE_PKTS }, + }; + const struct efx_mac_stats_range ef10_tx_size_bins[] = { + { EFX_MAC_TX_LE_64_PKTS, EFX_MAC_TX_GE_15XX_PKTS }, + }; + efx_nic_cfg_t *encp = &(enp->en_nic_cfg); + efx_port_t *epp = &(enp->en_port); + efx_rc_t rc; + + if ((rc = efx_mac_stats_mask_add_ranges(maskp, mask_size, + ef10_common, EFX_ARRAY_SIZE(ef10_common))) != 0) + goto fail1; + + if (epp->ep_phy_cap_mask & (1 << MC_CMD_PHY_CAP_40000FDX_LBN)) { + const struct efx_mac_stats_range ef10_40g_extra[] = { + { EFX_MAC_RX_ALIGN_ERRORS, EFX_MAC_RX_ALIGN_ERRORS }, + }; + + if ((rc = efx_mac_stats_mask_add_ranges(maskp, mask_size, + ef10_40g_extra, EFX_ARRAY_SIZE(ef10_40g_extra))) != 0) + goto fail2; + + if (encp->enc_mac_stats_40g_tx_size_bins) { + if ((rc = efx_mac_stats_mask_add_ranges(maskp, + mask_size, ef10_tx_size_bins, + EFX_ARRAY_SIZE(ef10_tx_size_bins))) != 0) + goto fail3; + } + } else { + if ((rc = efx_mac_stats_mask_add_ranges(maskp, mask_size, + ef10_tx_size_bins, EFX_ARRAY_SIZE(ef10_tx_size_bins))) != 0) + goto fail4; + } + + if (encp->enc_pm_and_rxdp_counters) { + const struct efx_mac_stats_range ef10_pm_and_rxdp[] = { + { EFX_MAC_PM_TRUNC_BB_OVERFLOW, EFX_MAC_RXDP_HLB_WAIT }, + }; + + if ((rc = efx_mac_stats_mask_add_ranges(maskp, mask_size, + ef10_pm_and_rxdp, EFX_ARRAY_SIZE(ef10_pm_and_rxdp))) != 0) + goto fail5; + } + + if (encp->enc_datapath_cap_evb) { + const struct efx_mac_stats_range ef10_vadaptor[] = { + { EFX_MAC_VADAPTER_RX_UNICAST_PACKETS, + EFX_MAC_VADAPTER_TX_OVERFLOW }, + }; + + if ((rc = efx_mac_stats_mask_add_ranges(maskp, mask_size, + ef10_vadaptor, EFX_ARRAY_SIZE(ef10_vadaptor))) != 0) + goto fail6; + } + + return (0); + +fail6: + EFSYS_PROBE(fail6); +fail5: + EFSYS_PROBE(fail5); +fail4: + EFSYS_PROBE(fail4); +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +#define EF10_MAC_STAT_READ(_esmp, _field, _eqp) \ + EFSYS_MEM_READQ((_esmp), (_field) * sizeof (efx_qword_t), _eqp) + + + __checkReturn efx_rc_t +ef10_mac_stats_update( + __in efx_nic_t *enp, + __in efsys_mem_t *esmp, + __inout_ecount(EFX_MAC_NSTATS) efsys_stat_t *stat, + __inout_opt uint32_t *generationp) +{ + efx_qword_t value; + efx_qword_t generation_start; + efx_qword_t generation_end; + + _NOTE(ARGUNUSED(enp)) + + /* Read END first so we don't race with the MC */ + EFSYS_DMA_SYNC_FOR_KERNEL(esmp, 0, EFX_MAC_STATS_SIZE); + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_GENERATION_END, + &generation_end); + EFSYS_MEM_READ_BARRIER(); + + /* TX */ + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_PKTS]), &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_CONTROL_PKTS, &value); + EFSYS_STAT_SUBR_QWORD(&(stat[EFX_MAC_TX_PKTS]), &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_PAUSE_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_PAUSE_PKTS]), &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_UNICAST_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_UNICST_PKTS]), &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_MULTICAST_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_MULTICST_PKTS]), &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_BROADCAST_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_BRDCST_PKTS]), &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_BYTES, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_OCTETS]), &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_LT64_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_LE_64_PKTS]), &value); + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_64_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_LE_64_PKTS]), &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_65_TO_127_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_65_TO_127_PKTS]), &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_128_TO_255_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_128_TO_255_PKTS]), &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_256_TO_511_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_256_TO_511_PKTS]), &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_512_TO_1023_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_512_TO_1023_PKTS]), &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_1024_TO_15XX_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_1024_TO_15XX_PKTS]), &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_15XX_TO_JUMBO_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_GE_15XX_PKTS]), &value); + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_GTJUMBO_PKTS, &value); + EFSYS_STAT_INCR_QWORD(&(stat[EFX_MAC_TX_GE_15XX_PKTS]), &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_BAD_FCS_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_ERRORS]), &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_SINGLE_COLLISION_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_SGL_COL_PKTS]), &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_MULTIPLE_COLLISION_PKTS, + &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_MULT_COL_PKTS]), &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_EXCESSIVE_COLLISION_PKTS, + &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_EX_COL_PKTS]), &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_LATE_COLLISION_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_LATE_COL_PKTS]), &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_DEFERRED_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_DEF_PKTS]), &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_EXCESSIVE_DEFERRED_PKTS, + &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_EX_DEF_PKTS]), &value); + + /* RX */ + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_BYTES, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_OCTETS]), &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_PKTS]), &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_UNICAST_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_UNICST_PKTS]), &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_MULTICAST_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_MULTICST_PKTS]), &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_BROADCAST_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_BRDCST_PKTS]), &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_PAUSE_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_PAUSE_PKTS]), &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_UNDERSIZE_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_LE_64_PKTS]), &value); + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_64_PKTS, &value); + EFSYS_STAT_INCR_QWORD(&(stat[EFX_MAC_RX_LE_64_PKTS]), &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_65_TO_127_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_65_TO_127_PKTS]), &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_128_TO_255_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_128_TO_255_PKTS]), &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_256_TO_511_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_256_TO_511_PKTS]), &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_512_TO_1023_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_512_TO_1023_PKTS]), &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_1024_TO_15XX_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_1024_TO_15XX_PKTS]), &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_15XX_TO_JUMBO_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_GE_15XX_PKTS]), &value); + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_GTJUMBO_PKTS, &value); + EFSYS_STAT_INCR_QWORD(&(stat[EFX_MAC_RX_GE_15XX_PKTS]), &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_BAD_FCS_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_FCS_ERRORS]), &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_OVERFLOW_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_DROP_EVENTS]), &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_FALSE_CARRIER_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_FALSE_CARRIER_ERRORS]), &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_SYMBOL_ERROR_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_SYMBOL_ERRORS]), &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_ALIGN_ERROR_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_ALIGN_ERRORS]), &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_INTERNAL_ERROR_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_INTERNAL_ERRORS]), &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_JABBER_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_JABBER_PKTS]), &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_LANES01_CHAR_ERR, &value); + EFSYS_STAT_SET_DWORD(&(stat[EFX_MAC_RX_LANE0_CHAR_ERR]), + &(value.eq_dword[0])); + EFSYS_STAT_SET_DWORD(&(stat[EFX_MAC_RX_LANE1_CHAR_ERR]), + &(value.eq_dword[1])); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_LANES23_CHAR_ERR, &value); + EFSYS_STAT_SET_DWORD(&(stat[EFX_MAC_RX_LANE2_CHAR_ERR]), + &(value.eq_dword[0])); + EFSYS_STAT_SET_DWORD(&(stat[EFX_MAC_RX_LANE3_CHAR_ERR]), + &(value.eq_dword[1])); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_LANES01_DISP_ERR, &value); + EFSYS_STAT_SET_DWORD(&(stat[EFX_MAC_RX_LANE0_DISP_ERR]), + &(value.eq_dword[0])); + EFSYS_STAT_SET_DWORD(&(stat[EFX_MAC_RX_LANE1_DISP_ERR]), + &(value.eq_dword[1])); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_LANES23_DISP_ERR, &value); + EFSYS_STAT_SET_DWORD(&(stat[EFX_MAC_RX_LANE2_DISP_ERR]), + &(value.eq_dword[0])); + EFSYS_STAT_SET_DWORD(&(stat[EFX_MAC_RX_LANE3_DISP_ERR]), + &(value.eq_dword[1])); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_MATCH_FAULT, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_MATCH_FAULT]), &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_NODESC_DROPS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_NODESC_DROP_CNT]), &value); + + /* Packet memory (EF10 only) */ + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_PM_TRUNC_BB_OVERFLOW, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_PM_TRUNC_BB_OVERFLOW]), &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_PM_DISCARD_BB_OVERFLOW, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_PM_DISCARD_BB_OVERFLOW]), &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_PM_TRUNC_VFIFO_FULL, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_PM_TRUNC_VFIFO_FULL]), &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_PM_DISCARD_VFIFO_FULL, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_PM_DISCARD_VFIFO_FULL]), &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_PM_TRUNC_QBB, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_PM_TRUNC_QBB]), &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_PM_DISCARD_QBB, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_PM_DISCARD_QBB]), &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_PM_DISCARD_MAPPING, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_PM_DISCARD_MAPPING]), &value); + + /* RX datapath */ + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RXDP_Q_DISABLED_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RXDP_Q_DISABLED_PKTS]), &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RXDP_DI_DROPPED_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RXDP_DI_DROPPED_PKTS]), &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RXDP_STREAMING_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RXDP_STREAMING_PKTS]), &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RXDP_HLB_FETCH_CONDITIONS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RXDP_HLB_FETCH]), &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_RXDP_HLB_WAIT_CONDITIONS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RXDP_HLB_WAIT]), &value); + + + /* VADAPTER RX */ + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_VADAPTER_RX_UNICAST_PACKETS, + &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_VADAPTER_RX_UNICAST_PACKETS]), + &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_VADAPTER_RX_UNICAST_BYTES, + &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_VADAPTER_RX_UNICAST_BYTES]), + &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_VADAPTER_RX_MULTICAST_PACKETS, + &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_VADAPTER_RX_MULTICAST_PACKETS]), + &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_VADAPTER_RX_MULTICAST_BYTES, + &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_VADAPTER_RX_MULTICAST_BYTES]), + &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_VADAPTER_RX_BROADCAST_PACKETS, + &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_VADAPTER_RX_BROADCAST_PACKETS]), + &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_VADAPTER_RX_BROADCAST_BYTES, + &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_VADAPTER_RX_BROADCAST_BYTES]), + &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_VADAPTER_RX_BAD_PACKETS, + &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_VADAPTER_RX_BAD_PACKETS]), + &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_VADAPTER_RX_BAD_BYTES, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_VADAPTER_RX_BAD_BYTES]), &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_VADAPTER_RX_OVERFLOW, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_VADAPTER_RX_OVERFLOW]), &value); + + /* VADAPTER TX */ + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_VADAPTER_TX_UNICAST_PACKETS, + &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_VADAPTER_TX_UNICAST_PACKETS]), + &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_VADAPTER_TX_UNICAST_BYTES, + &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_VADAPTER_TX_UNICAST_BYTES]), + &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_VADAPTER_TX_MULTICAST_PACKETS, + &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_VADAPTER_TX_MULTICAST_PACKETS]), + &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_VADAPTER_TX_MULTICAST_BYTES, + &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_VADAPTER_TX_MULTICAST_BYTES]), + &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_VADAPTER_TX_BROADCAST_PACKETS, + &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_VADAPTER_TX_BROADCAST_PACKETS]), + &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_VADAPTER_TX_BROADCAST_BYTES, + &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_VADAPTER_TX_BROADCAST_BYTES]), + &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_VADAPTER_TX_BAD_PACKETS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_VADAPTER_TX_BAD_PACKETS]), &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_VADAPTER_TX_BAD_BYTES, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_VADAPTER_TX_BAD_BYTES]), &value); + + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_VADAPTER_TX_OVERFLOW, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_VADAPTER_TX_OVERFLOW]), &value); + + + EFSYS_DMA_SYNC_FOR_KERNEL(esmp, 0, EFX_MAC_STATS_SIZE); + EFSYS_MEM_READ_BARRIER(); + EF10_MAC_STAT_READ(esmp, MC_CMD_MAC_GENERATION_START, + &generation_start); + + /* Check that we didn't read the stats in the middle of a DMA */ + /* Not a good enough check ? */ + if (memcmp(&generation_start, &generation_end, + sizeof (generation_start))) + return (EAGAIN); + + if (generationp) + *generationp = EFX_QWORD_FIELD(generation_start, EFX_DWORD_0); + + return (0); +} + +#endif /* EFSYS_OPT_MAC_STATS */ + +#endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */ diff --git a/dpdk/drivers/net/sfc/base/ef10_mcdi.c b/dpdk/drivers/net/sfc/base/ef10_mcdi.c new file mode 100644 index 00000000..5a26bda2 --- /dev/null +++ b/dpdk/drivers/net/sfc/base/ef10_mcdi.c @@ -0,0 +1,342 @@ +/* + * Copyright (c) 2012-2016 Solarflare Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are + * those of the authors and should not be interpreted as representing official + * policies, either expressed or implied, of the FreeBSD Project. + */ + +#include "efx.h" +#include "efx_impl.h" + + +#if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD + +#if EFSYS_OPT_MCDI + +#ifndef WITH_MCDI_V2 +#error "WITH_MCDI_V2 required for EF10 MCDIv2 commands." +#endif + + + __checkReturn efx_rc_t +ef10_mcdi_init( + __in efx_nic_t *enp, + __in const efx_mcdi_transport_t *emtp) +{ + efx_mcdi_iface_t *emip = &(enp->en_mcdi.em_emip); + efsys_mem_t *esmp = emtp->emt_dma_mem; + efx_dword_t dword; + efx_rc_t rc; + + EFSYS_ASSERT(enp->en_family == EFX_FAMILY_HUNTINGTON || + enp->en_family == EFX_FAMILY_MEDFORD); + EFSYS_ASSERT(enp->en_features & EFX_FEATURE_MCDI_DMA); + + /* + * All EF10 firmware supports MCDIv2 and MCDIv1. + * Medford BootROM supports MCDIv2 and MCDIv1. + * Huntington BootROM supports MCDIv1 only. + */ + emip->emi_max_version = 2; + + /* A host DMA buffer is required for EF10 MCDI */ + if (esmp == NULL) { + rc = EINVAL; + goto fail1; + } + + /* + * Ensure that the MC doorbell is in a known state before issuing MCDI + * commands. The recovery algorithm requires that the MC command buffer + * must be 256 byte aligned. See bug24769. + */ + if ((EFSYS_MEM_ADDR(esmp) & 0xFF) != 0) { + rc = EINVAL; + goto fail2; + } + EFX_POPULATE_DWORD_1(dword, EFX_DWORD_0, 1); + EFX_BAR_WRITED(enp, ER_DZ_MC_DB_HWRD_REG, &dword, B_FALSE); + + /* Save initial MC reboot status */ + (void) ef10_mcdi_poll_reboot(enp); + + /* Start a new epoch (allow fresh MCDI requests to succeed) */ + efx_mcdi_new_epoch(enp); + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + void +ef10_mcdi_fini( + __in efx_nic_t *enp) +{ + efx_mcdi_iface_t *emip = &(enp->en_mcdi.em_emip); + + emip->emi_new_epoch = B_FALSE; +} + +/* + * In older firmware all commands are processed in a single thread, so a long + * running command for one PCIe function can block processing for another + * function (see bug 61269). + * + * In newer firmware that supports multithreaded MCDI processing, we can extend + * the timeout for long-running requests which we know firmware may choose to + * process in a background thread. + */ +#define EF10_MCDI_CMD_TIMEOUT_US (10 * 1000 * 1000) +#define EF10_MCDI_CMD_LONG_TIMEOUT_US (60 * 1000 * 1000) + + void +ef10_mcdi_get_timeout( + __in efx_nic_t *enp, + __in efx_mcdi_req_t *emrp, + __out uint32_t *timeoutp) +{ + efx_nic_cfg_t *encp = &(enp->en_nic_cfg); + + switch (emrp->emr_cmd) { + case MC_CMD_POLL_BIST: + case MC_CMD_NVRAM_ERASE: + case MC_CMD_LICENSING_V3: + case MC_CMD_NVRAM_UPDATE_FINISH: + if (encp->enc_fw_verified_nvram_update_required != B_FALSE) { + /* + * Potentially longer running commands, which firmware + * may choose to process in a background thread. + */ + *timeoutp = EF10_MCDI_CMD_LONG_TIMEOUT_US; + break; + } + /* FALLTHRU */ + default: + *timeoutp = EF10_MCDI_CMD_TIMEOUT_US; + break; + } +} + + void +ef10_mcdi_send_request( + __in efx_nic_t *enp, + __in_bcount(hdr_len) void *hdrp, + __in size_t hdr_len, + __in_bcount(sdu_len) void *sdup, + __in size_t sdu_len) +{ + const efx_mcdi_transport_t *emtp = enp->en_mcdi.em_emtp; + efsys_mem_t *esmp = emtp->emt_dma_mem; + efx_dword_t dword; + unsigned int pos; + + EFSYS_ASSERT(enp->en_family == EFX_FAMILY_HUNTINGTON || + enp->en_family == EFX_FAMILY_MEDFORD); + + /* Write the header */ + for (pos = 0; pos < hdr_len; pos += sizeof (efx_dword_t)) { + dword = *(efx_dword_t *)((uint8_t *)hdrp + pos); + EFSYS_MEM_WRITED(esmp, pos, &dword); + } + + /* Write the payload */ + for (pos = 0; pos < sdu_len; pos += sizeof (efx_dword_t)) { + dword = *(efx_dword_t *)((uint8_t *)sdup + pos); + EFSYS_MEM_WRITED(esmp, hdr_len + pos, &dword); + } + + /* Guarantee ordering of memory (MCDI request) and PIO (MC doorbell) */ + EFSYS_DMA_SYNC_FOR_DEVICE(esmp, 0, hdr_len + sdu_len); + EFSYS_PIO_WRITE_BARRIER(); + + /* Ring the doorbell to post the command DMA address to the MC */ + EFX_POPULATE_DWORD_1(dword, EFX_DWORD_0, + EFSYS_MEM_ADDR(esmp) >> 32); + EFX_BAR_WRITED(enp, ER_DZ_MC_DB_LWRD_REG, &dword, B_FALSE); + + EFX_POPULATE_DWORD_1(dword, EFX_DWORD_0, + EFSYS_MEM_ADDR(esmp) & 0xffffffff); + EFX_BAR_WRITED(enp, ER_DZ_MC_DB_HWRD_REG, &dword, B_FALSE); +} + + __checkReturn boolean_t +ef10_mcdi_poll_response( + __in efx_nic_t *enp) +{ + const efx_mcdi_transport_t *emtp = enp->en_mcdi.em_emtp; + efsys_mem_t *esmp = emtp->emt_dma_mem; + efx_dword_t hdr; + + EFSYS_MEM_READD(esmp, 0, &hdr); + EFSYS_MEM_READ_BARRIER(); + + return (EFX_DWORD_FIELD(hdr, MCDI_HEADER_RESPONSE) ? B_TRUE : B_FALSE); +} + + void +ef10_mcdi_read_response( + __in efx_nic_t *enp, + __out_bcount(length) void *bufferp, + __in size_t offset, + __in size_t length) +{ + const efx_mcdi_transport_t *emtp = enp->en_mcdi.em_emtp; + efsys_mem_t *esmp = emtp->emt_dma_mem; + unsigned int pos; + efx_dword_t data; + + for (pos = 0; pos < length; pos += sizeof (efx_dword_t)) { + EFSYS_MEM_READD(esmp, offset + pos, &data); + memcpy((uint8_t *)bufferp + pos, &data, + MIN(sizeof (data), length - pos)); + } +} + + efx_rc_t +ef10_mcdi_poll_reboot( + __in efx_nic_t *enp) +{ + efx_mcdi_iface_t *emip = &(enp->en_mcdi.em_emip); + efx_dword_t dword; + uint32_t old_status; + uint32_t new_status; + efx_rc_t rc; + + old_status = emip->emi_mc_reboot_status; + + /* Update MC reboot status word */ + EFX_BAR_TBL_READD(enp, ER_DZ_BIU_MC_SFT_STATUS_REG, 0, &dword, B_FALSE); + new_status = dword.ed_u32[0]; + + /* MC has rebooted if the value has changed */ + if (new_status != old_status) { + emip->emi_mc_reboot_status = new_status; + + /* + * FIXME: Ignore detected MC REBOOT for now. + * + * The Siena support for checking for MC reboot from status + * flags is broken - see comments in siena_mcdi_poll_reboot(). + * As the generic MCDI code is shared the EF10 reboot + * detection suffers similar problems. + * + * Do not report an error when the boot status changes until + * this can be handled by common code drivers (and reworked to + * support Siena too). + */ + _NOTE(CONSTANTCONDITION) + if (B_FALSE) { + rc = EIO; + goto fail1; + } + } + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +ef10_mcdi_feature_supported( + __in efx_nic_t *enp, + __in efx_mcdi_feature_id_t id, + __out boolean_t *supportedp) +{ + efx_nic_cfg_t *encp = &(enp->en_nic_cfg); + uint32_t privilege_mask = encp->enc_privilege_mask; + efx_rc_t rc; + + EFSYS_ASSERT(enp->en_family == EFX_FAMILY_HUNTINGTON || + enp->en_family == EFX_FAMILY_MEDFORD); + + /* + * Use privilege mask state at MCDI attach. + */ + + switch (id) { + case EFX_MCDI_FEATURE_FW_UPDATE: + /* + * Admin privilege must be used prior to introduction of + * specific flag. + */ + *supportedp = + EFX_MCDI_HAVE_PRIVILEGE(privilege_mask, ADMIN); + break; + case EFX_MCDI_FEATURE_LINK_CONTROL: + /* + * Admin privilege used prior to introduction of + * specific flag. + */ + *supportedp = + EFX_MCDI_HAVE_PRIVILEGE(privilege_mask, LINK) || + EFX_MCDI_HAVE_PRIVILEGE(privilege_mask, ADMIN); + break; + case EFX_MCDI_FEATURE_MACADDR_CHANGE: + /* + * Admin privilege must be used prior to introduction of + * mac spoofing privilege (at v4.6), which is used up to + * introduction of change mac spoofing privilege (at v4.7) + */ + *supportedp = + EFX_MCDI_HAVE_PRIVILEGE(privilege_mask, CHANGE_MAC) || + EFX_MCDI_HAVE_PRIVILEGE(privilege_mask, MAC_SPOOFING) || + EFX_MCDI_HAVE_PRIVILEGE(privilege_mask, ADMIN); + break; + case EFX_MCDI_FEATURE_MAC_SPOOFING: + /* + * Admin privilege must be used prior to introduction of + * mac spoofing privilege (at v4.6), which is used up to + * introduction of mac spoofing TX privilege (at v4.7) + */ + *supportedp = + EFX_MCDI_HAVE_PRIVILEGE(privilege_mask, MAC_SPOOFING_TX) || + EFX_MCDI_HAVE_PRIVILEGE(privilege_mask, MAC_SPOOFING) || + EFX_MCDI_HAVE_PRIVILEGE(privilege_mask, ADMIN); + break; + default: + rc = ENOTSUP; + goto fail1; + } + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +#endif /* EFSYS_OPT_MCDI */ + +#endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */ diff --git a/dpdk/drivers/net/sfc/base/ef10_nic.c b/dpdk/drivers/net/sfc/base/ef10_nic.c new file mode 100644 index 00000000..58d1b0af --- /dev/null +++ b/dpdk/drivers/net/sfc/base/ef10_nic.c @@ -0,0 +1,1790 @@ +/* + * Copyright (c) 2012-2016 Solarflare Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are + * those of the authors and should not be interpreted as representing official + * policies, either expressed or implied, of the FreeBSD Project. + */ + +#include "efx.h" +#include "efx_impl.h" +#if EFSYS_OPT_MON_MCDI +#include "mcdi_mon.h" +#endif + +#if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD + +#include "ef10_tlv_layout.h" + + __checkReturn efx_rc_t +efx_mcdi_get_port_assignment( + __in efx_nic_t *enp, + __out uint32_t *portp) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_GET_PORT_ASSIGNMENT_IN_LEN, + MC_CMD_GET_PORT_ASSIGNMENT_OUT_LEN)]; + efx_rc_t rc; + + EFSYS_ASSERT(enp->en_family == EFX_FAMILY_HUNTINGTON || + enp->en_family == EFX_FAMILY_MEDFORD); + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_GET_PORT_ASSIGNMENT; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_GET_PORT_ASSIGNMENT_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_GET_PORT_ASSIGNMENT_OUT_LEN; + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + if (req.emr_out_length_used < MC_CMD_GET_PORT_ASSIGNMENT_OUT_LEN) { + rc = EMSGSIZE; + goto fail2; + } + + *portp = MCDI_OUT_DWORD(req, GET_PORT_ASSIGNMENT_OUT_PORT); + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_mcdi_get_port_modes( + __in efx_nic_t *enp, + __out uint32_t *modesp, + __out_opt uint32_t *current_modep) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_GET_PORT_MODES_IN_LEN, + MC_CMD_GET_PORT_MODES_OUT_LEN)]; + efx_rc_t rc; + + EFSYS_ASSERT(enp->en_family == EFX_FAMILY_HUNTINGTON || + enp->en_family == EFX_FAMILY_MEDFORD); + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_GET_PORT_MODES; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_GET_PORT_MODES_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_GET_PORT_MODES_OUT_LEN; + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + /* + * Require only Modes and DefaultMode fields, unless the current mode + * was requested (CurrentMode field was added for Medford). + */ + if (req.emr_out_length_used < + MC_CMD_GET_PORT_MODES_OUT_CURRENT_MODE_OFST) { + rc = EMSGSIZE; + goto fail2; + } + if ((current_modep != NULL) && (req.emr_out_length_used < + MC_CMD_GET_PORT_MODES_OUT_CURRENT_MODE_OFST + 4)) { + rc = EMSGSIZE; + goto fail3; + } + + *modesp = MCDI_OUT_DWORD(req, GET_PORT_MODES_OUT_MODES); + + if (current_modep != NULL) { + *current_modep = MCDI_OUT_DWORD(req, + GET_PORT_MODES_OUT_CURRENT_MODE); + } + + return (0); + +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +ef10_nic_get_port_mode_bandwidth( + __in uint32_t port_mode, + __out uint32_t *bandwidth_mbpsp) +{ + uint32_t bandwidth; + efx_rc_t rc; + + switch (port_mode) { + case TLV_PORT_MODE_10G: + bandwidth = 10000; + break; + case TLV_PORT_MODE_10G_10G: + bandwidth = 10000 * 2; + break; + case TLV_PORT_MODE_10G_10G_10G_10G: + case TLV_PORT_MODE_10G_10G_10G_10G_Q: + case TLV_PORT_MODE_10G_10G_10G_10G_Q1_Q2: + case TLV_PORT_MODE_10G_10G_10G_10G_Q2: + bandwidth = 10000 * 4; + break; + case TLV_PORT_MODE_40G: + bandwidth = 40000; + break; + case TLV_PORT_MODE_40G_40G: + bandwidth = 40000 * 2; + break; + case TLV_PORT_MODE_40G_10G_10G: + case TLV_PORT_MODE_10G_10G_40G: + bandwidth = 40000 + (10000 * 2); + break; + default: + rc = EINVAL; + goto fail1; + } + + *bandwidth_mbpsp = bandwidth; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +static __checkReturn efx_rc_t +efx_mcdi_vadaptor_alloc( + __in efx_nic_t *enp, + __in uint32_t port_id) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_VADAPTOR_ALLOC_IN_LEN, + MC_CMD_VADAPTOR_ALLOC_OUT_LEN)]; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_vport_id, ==, EVB_PORT_ID_NULL); + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_VADAPTOR_ALLOC; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_VADAPTOR_ALLOC_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_VADAPTOR_ALLOC_OUT_LEN; + + MCDI_IN_SET_DWORD(req, VADAPTOR_ALLOC_IN_UPSTREAM_PORT_ID, port_id); + MCDI_IN_POPULATE_DWORD_1(req, VADAPTOR_ALLOC_IN_FLAGS, + VADAPTOR_ALLOC_IN_FLAG_PERMIT_SET_MAC_WHEN_FILTERS_INSTALLED, + enp->en_nic_cfg.enc_allow_set_mac_with_installed_filters ? 1 : 0); + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +static __checkReturn efx_rc_t +efx_mcdi_vadaptor_free( + __in efx_nic_t *enp, + __in uint32_t port_id) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_VADAPTOR_FREE_IN_LEN, + MC_CMD_VADAPTOR_FREE_OUT_LEN)]; + efx_rc_t rc; + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_VADAPTOR_FREE; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_VADAPTOR_FREE_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_VADAPTOR_FREE_OUT_LEN; + + MCDI_IN_SET_DWORD(req, VADAPTOR_FREE_IN_UPSTREAM_PORT_ID, port_id); + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_mcdi_get_mac_address_pf( + __in efx_nic_t *enp, + __out_ecount_opt(6) uint8_t mac_addrp[6]) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_GET_MAC_ADDRESSES_IN_LEN, + MC_CMD_GET_MAC_ADDRESSES_OUT_LEN)]; + efx_rc_t rc; + + EFSYS_ASSERT(enp->en_family == EFX_FAMILY_HUNTINGTON || + enp->en_family == EFX_FAMILY_MEDFORD); + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_GET_MAC_ADDRESSES; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_GET_MAC_ADDRESSES_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_GET_MAC_ADDRESSES_OUT_LEN; + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + if (req.emr_out_length_used < MC_CMD_GET_MAC_ADDRESSES_OUT_LEN) { + rc = EMSGSIZE; + goto fail2; + } + + if (MCDI_OUT_DWORD(req, GET_MAC_ADDRESSES_OUT_MAC_COUNT) < 1) { + rc = ENOENT; + goto fail3; + } + + if (mac_addrp != NULL) { + uint8_t *addrp; + + addrp = MCDI_OUT2(req, uint8_t, + GET_MAC_ADDRESSES_OUT_MAC_ADDR_BASE); + + EFX_MAC_ADDR_COPY(mac_addrp, addrp); + } + + return (0); + +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_mcdi_get_mac_address_vf( + __in efx_nic_t *enp, + __out_ecount_opt(6) uint8_t mac_addrp[6]) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_VPORT_GET_MAC_ADDRESSES_IN_LEN, + MC_CMD_VPORT_GET_MAC_ADDRESSES_OUT_LENMAX)]; + efx_rc_t rc; + + EFSYS_ASSERT(enp->en_family == EFX_FAMILY_HUNTINGTON || + enp->en_family == EFX_FAMILY_MEDFORD); + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_VPORT_GET_MAC_ADDRESSES; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_VPORT_GET_MAC_ADDRESSES_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_VPORT_GET_MAC_ADDRESSES_OUT_LENMAX; + + MCDI_IN_SET_DWORD(req, VPORT_GET_MAC_ADDRESSES_IN_VPORT_ID, + EVB_PORT_ID_ASSIGNED); + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + if (req.emr_out_length_used < + MC_CMD_VPORT_GET_MAC_ADDRESSES_OUT_LENMIN) { + rc = EMSGSIZE; + goto fail2; + } + + if (MCDI_OUT_DWORD(req, + VPORT_GET_MAC_ADDRESSES_OUT_MACADDR_COUNT) < 1) { + rc = ENOENT; + goto fail3; + } + + if (mac_addrp != NULL) { + uint8_t *addrp; + + addrp = MCDI_OUT2(req, uint8_t, + VPORT_GET_MAC_ADDRESSES_OUT_MACADDR); + + EFX_MAC_ADDR_COPY(mac_addrp, addrp); + } + + return (0); + +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_mcdi_get_clock( + __in efx_nic_t *enp, + __out uint32_t *sys_freqp, + __out uint32_t *dpcpu_freqp) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_GET_CLOCK_IN_LEN, + MC_CMD_GET_CLOCK_OUT_LEN)]; + efx_rc_t rc; + + EFSYS_ASSERT(enp->en_family == EFX_FAMILY_HUNTINGTON || + enp->en_family == EFX_FAMILY_MEDFORD); + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_GET_CLOCK; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_GET_CLOCK_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_GET_CLOCK_OUT_LEN; + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + if (req.emr_out_length_used < MC_CMD_GET_CLOCK_OUT_LEN) { + rc = EMSGSIZE; + goto fail2; + } + + *sys_freqp = MCDI_OUT_DWORD(req, GET_CLOCK_OUT_SYS_FREQ); + if (*sys_freqp == 0) { + rc = EINVAL; + goto fail3; + } + *dpcpu_freqp = MCDI_OUT_DWORD(req, GET_CLOCK_OUT_DPCPU_FREQ); + if (*dpcpu_freqp == 0) { + rc = EINVAL; + goto fail4; + } + + return (0); + +fail4: + EFSYS_PROBE(fail4); +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_mcdi_get_vector_cfg( + __in efx_nic_t *enp, + __out_opt uint32_t *vec_basep, + __out_opt uint32_t *pf_nvecp, + __out_opt uint32_t *vf_nvecp) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_GET_VECTOR_CFG_IN_LEN, + MC_CMD_GET_VECTOR_CFG_OUT_LEN)]; + efx_rc_t rc; + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_GET_VECTOR_CFG; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_GET_VECTOR_CFG_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_GET_VECTOR_CFG_OUT_LEN; + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + if (req.emr_out_length_used < MC_CMD_GET_VECTOR_CFG_OUT_LEN) { + rc = EMSGSIZE; + goto fail2; + } + + if (vec_basep != NULL) + *vec_basep = MCDI_OUT_DWORD(req, GET_VECTOR_CFG_OUT_VEC_BASE); + if (pf_nvecp != NULL) + *pf_nvecp = MCDI_OUT_DWORD(req, GET_VECTOR_CFG_OUT_VECS_PER_PF); + if (vf_nvecp != NULL) + *vf_nvecp = MCDI_OUT_DWORD(req, GET_VECTOR_CFG_OUT_VECS_PER_VF); + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +static __checkReturn efx_rc_t +efx_mcdi_alloc_vis( + __in efx_nic_t *enp, + __in uint32_t min_vi_count, + __in uint32_t max_vi_count, + __out uint32_t *vi_basep, + __out uint32_t *vi_countp, + __out uint32_t *vi_shiftp) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_ALLOC_VIS_IN_LEN, + MC_CMD_ALLOC_VIS_EXT_OUT_LEN)]; + efx_rc_t rc; + + if (vi_countp == NULL) { + rc = EINVAL; + goto fail1; + } + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_ALLOC_VIS; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_ALLOC_VIS_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_ALLOC_VIS_EXT_OUT_LEN; + + MCDI_IN_SET_DWORD(req, ALLOC_VIS_IN_MIN_VI_COUNT, min_vi_count); + MCDI_IN_SET_DWORD(req, ALLOC_VIS_IN_MAX_VI_COUNT, max_vi_count); + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail2; + } + + if (req.emr_out_length_used < MC_CMD_ALLOC_VIS_OUT_LEN) { + rc = EMSGSIZE; + goto fail3; + } + + *vi_basep = MCDI_OUT_DWORD(req, ALLOC_VIS_OUT_VI_BASE); + *vi_countp = MCDI_OUT_DWORD(req, ALLOC_VIS_OUT_VI_COUNT); + + /* Report VI_SHIFT if available (always zero for Huntington) */ + if (req.emr_out_length_used < MC_CMD_ALLOC_VIS_EXT_OUT_LEN) + *vi_shiftp = 0; + else + *vi_shiftp = MCDI_OUT_DWORD(req, ALLOC_VIS_EXT_OUT_VI_SHIFT); + + return (0); + +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + +static __checkReturn efx_rc_t +efx_mcdi_free_vis( + __in efx_nic_t *enp) +{ + efx_mcdi_req_t req; + efx_rc_t rc; + + EFX_STATIC_ASSERT(MC_CMD_FREE_VIS_IN_LEN == 0); + EFX_STATIC_ASSERT(MC_CMD_FREE_VIS_OUT_LEN == 0); + + req.emr_cmd = MC_CMD_FREE_VIS; + req.emr_in_buf = NULL; + req.emr_in_length = 0; + req.emr_out_buf = NULL; + req.emr_out_length = 0; + + efx_mcdi_execute_quiet(enp, &req); + + /* Ignore ELREADY (no allocated VIs, so nothing to free) */ + if ((req.emr_rc != 0) && (req.emr_rc != EALREADY)) { + rc = req.emr_rc; + goto fail1; + } + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + +static __checkReturn efx_rc_t +efx_mcdi_alloc_piobuf( + __in efx_nic_t *enp, + __out efx_piobuf_handle_t *handlep) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_ALLOC_PIOBUF_IN_LEN, + MC_CMD_ALLOC_PIOBUF_OUT_LEN)]; + efx_rc_t rc; + + if (handlep == NULL) { + rc = EINVAL; + goto fail1; + } + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_ALLOC_PIOBUF; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_ALLOC_PIOBUF_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_ALLOC_PIOBUF_OUT_LEN; + + efx_mcdi_execute_quiet(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail2; + } + + if (req.emr_out_length_used < MC_CMD_ALLOC_PIOBUF_OUT_LEN) { + rc = EMSGSIZE; + goto fail3; + } + + *handlep = MCDI_OUT_DWORD(req, ALLOC_PIOBUF_OUT_PIOBUF_HANDLE); + + return (0); + +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +static __checkReturn efx_rc_t +efx_mcdi_free_piobuf( + __in efx_nic_t *enp, + __in efx_piobuf_handle_t handle) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_FREE_PIOBUF_IN_LEN, + MC_CMD_FREE_PIOBUF_OUT_LEN)]; + efx_rc_t rc; + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_FREE_PIOBUF; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_FREE_PIOBUF_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_FREE_PIOBUF_OUT_LEN; + + MCDI_IN_SET_DWORD(req, FREE_PIOBUF_IN_PIOBUF_HANDLE, handle); + + efx_mcdi_execute_quiet(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +static __checkReturn efx_rc_t +efx_mcdi_link_piobuf( + __in efx_nic_t *enp, + __in uint32_t vi_index, + __in efx_piobuf_handle_t handle) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_LINK_PIOBUF_IN_LEN, + MC_CMD_LINK_PIOBUF_OUT_LEN)]; + efx_rc_t rc; + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_LINK_PIOBUF; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_LINK_PIOBUF_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_LINK_PIOBUF_OUT_LEN; + + MCDI_IN_SET_DWORD(req, LINK_PIOBUF_IN_PIOBUF_HANDLE, handle); + MCDI_IN_SET_DWORD(req, LINK_PIOBUF_IN_TXQ_INSTANCE, vi_index); + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +static __checkReturn efx_rc_t +efx_mcdi_unlink_piobuf( + __in efx_nic_t *enp, + __in uint32_t vi_index) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_UNLINK_PIOBUF_IN_LEN, + MC_CMD_UNLINK_PIOBUF_OUT_LEN)]; + efx_rc_t rc; + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_UNLINK_PIOBUF; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_UNLINK_PIOBUF_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_UNLINK_PIOBUF_OUT_LEN; + + MCDI_IN_SET_DWORD(req, UNLINK_PIOBUF_IN_TXQ_INSTANCE, vi_index); + + efx_mcdi_execute_quiet(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +static void +ef10_nic_alloc_piobufs( + __in efx_nic_t *enp, + __in uint32_t max_piobuf_count) +{ + efx_piobuf_handle_t *handlep; + unsigned int i; + + EFSYS_ASSERT3U(max_piobuf_count, <=, + EFX_ARRAY_SIZE(enp->en_arch.ef10.ena_piobuf_handle)); + + enp->en_arch.ef10.ena_piobuf_count = 0; + + for (i = 0; i < max_piobuf_count; i++) { + handlep = &enp->en_arch.ef10.ena_piobuf_handle[i]; + + if (efx_mcdi_alloc_piobuf(enp, handlep) != 0) + goto fail1; + + enp->en_arch.ef10.ena_pio_alloc_map[i] = 0; + enp->en_arch.ef10.ena_piobuf_count++; + } + + return; + +fail1: + for (i = 0; i < enp->en_arch.ef10.ena_piobuf_count; i++) { + handlep = &enp->en_arch.ef10.ena_piobuf_handle[i]; + + efx_mcdi_free_piobuf(enp, *handlep); + *handlep = EFX_PIOBUF_HANDLE_INVALID; + } + enp->en_arch.ef10.ena_piobuf_count = 0; +} + + +static void +ef10_nic_free_piobufs( + __in efx_nic_t *enp) +{ + efx_piobuf_handle_t *handlep; + unsigned int i; + + for (i = 0; i < enp->en_arch.ef10.ena_piobuf_count; i++) { + handlep = &enp->en_arch.ef10.ena_piobuf_handle[i]; + + efx_mcdi_free_piobuf(enp, *handlep); + *handlep = EFX_PIOBUF_HANDLE_INVALID; + } + enp->en_arch.ef10.ena_piobuf_count = 0; +} + +/* Sub-allocate a block from a piobuf */ + __checkReturn efx_rc_t +ef10_nic_pio_alloc( + __inout efx_nic_t *enp, + __out uint32_t *bufnump, + __out efx_piobuf_handle_t *handlep, + __out uint32_t *blknump, + __out uint32_t *offsetp, + __out size_t *sizep) +{ + efx_nic_cfg_t *encp = &enp->en_nic_cfg; + efx_drv_cfg_t *edcp = &enp->en_drv_cfg; + uint32_t blk_per_buf; + uint32_t buf, blk; + efx_rc_t rc; + + EFSYS_ASSERT(enp->en_family == EFX_FAMILY_HUNTINGTON || + enp->en_family == EFX_FAMILY_MEDFORD); + EFSYS_ASSERT(bufnump); + EFSYS_ASSERT(handlep); + EFSYS_ASSERT(blknump); + EFSYS_ASSERT(offsetp); + EFSYS_ASSERT(sizep); + + if ((edcp->edc_pio_alloc_size == 0) || + (enp->en_arch.ef10.ena_piobuf_count == 0)) { + rc = ENOMEM; + goto fail1; + } + blk_per_buf = encp->enc_piobuf_size / edcp->edc_pio_alloc_size; + + for (buf = 0; buf < enp->en_arch.ef10.ena_piobuf_count; buf++) { + uint32_t *map = &enp->en_arch.ef10.ena_pio_alloc_map[buf]; + + if (~(*map) == 0) + continue; + + EFSYS_ASSERT3U(blk_per_buf, <=, (8 * sizeof (*map))); + for (blk = 0; blk < blk_per_buf; blk++) { + if ((*map & (1u << blk)) == 0) { + *map |= (1u << blk); + goto done; + } + } + } + rc = ENOMEM; + goto fail2; + +done: + *handlep = enp->en_arch.ef10.ena_piobuf_handle[buf]; + *bufnump = buf; + *blknump = blk; + *sizep = edcp->edc_pio_alloc_size; + *offsetp = blk * (*sizep); + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +/* Free a piobuf sub-allocated block */ + __checkReturn efx_rc_t +ef10_nic_pio_free( + __inout efx_nic_t *enp, + __in uint32_t bufnum, + __in uint32_t blknum) +{ + uint32_t *map; + efx_rc_t rc; + + if ((bufnum >= enp->en_arch.ef10.ena_piobuf_count) || + (blknum >= (8 * sizeof (*map)))) { + rc = EINVAL; + goto fail1; + } + + map = &enp->en_arch.ef10.ena_pio_alloc_map[bufnum]; + if ((*map & (1u << blknum)) == 0) { + rc = ENOENT; + goto fail2; + } + *map &= ~(1u << blknum); + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +ef10_nic_pio_link( + __inout efx_nic_t *enp, + __in uint32_t vi_index, + __in efx_piobuf_handle_t handle) +{ + return (efx_mcdi_link_piobuf(enp, vi_index, handle)); +} + + __checkReturn efx_rc_t +ef10_nic_pio_unlink( + __inout efx_nic_t *enp, + __in uint32_t vi_index) +{ + return (efx_mcdi_unlink_piobuf(enp, vi_index)); +} + +static __checkReturn efx_rc_t +ef10_mcdi_get_pf_count( + __in efx_nic_t *enp, + __out uint32_t *pf_countp) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_GET_PF_COUNT_IN_LEN, + MC_CMD_GET_PF_COUNT_OUT_LEN)]; + efx_rc_t rc; + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_GET_PF_COUNT; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_GET_PF_COUNT_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_GET_PF_COUNT_OUT_LEN; + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + if (req.emr_out_length_used < MC_CMD_GET_PF_COUNT_OUT_LEN) { + rc = EMSGSIZE; + goto fail2; + } + + *pf_countp = *MCDI_OUT(req, uint8_t, + MC_CMD_GET_PF_COUNT_OUT_PF_COUNT_OFST); + + EFSYS_ASSERT(*pf_countp != 0); + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +ef10_get_datapath_caps( + __in efx_nic_t *enp) +{ + efx_nic_cfg_t *encp = &(enp->en_nic_cfg); + uint32_t flags; + uint32_t flags2; + uint32_t tso2nc; + efx_rc_t rc; + + if ((rc = efx_mcdi_get_capabilities(enp, &flags, NULL, NULL, + &flags2, &tso2nc)) != 0) + goto fail1; + + if ((rc = ef10_mcdi_get_pf_count(enp, &encp->enc_hw_pf_count)) != 0) + goto fail1; + +#define CAP_FLAG(flags1, field) \ + ((flags1) & (1 << (MC_CMD_GET_CAPABILITIES_V2_OUT_ ## field ## _LBN))) + +#define CAP_FLAG2(flags2, field) \ + ((flags2) & (1 << (MC_CMD_GET_CAPABILITIES_V2_OUT_ ## field ## _LBN))) + + /* + * Huntington RXDP firmware inserts a 0 or 14 byte prefix. + * We only support the 14 byte prefix here. + */ + if (CAP_FLAG(flags, RX_PREFIX_LEN_14) == 0) { + rc = ENOTSUP; + goto fail2; + } + encp->enc_rx_prefix_size = 14; + + /* Check if the firmware supports TSO */ + encp->enc_fw_assisted_tso_enabled = + CAP_FLAG(flags, TX_TSO) ? B_TRUE : B_FALSE; + + /* Check if the firmware supports FATSOv2 */ + encp->enc_fw_assisted_tso_v2_enabled = + CAP_FLAG2(flags2, TX_TSO_V2) ? B_TRUE : B_FALSE; + + /* Get the number of TSO contexts (FATSOv2) */ + encp->enc_fw_assisted_tso_v2_n_contexts = + CAP_FLAG2(flags2, TX_TSO_V2) ? tso2nc : 0; + + /* Check if the firmware has vadapter/vport/vswitch support */ + encp->enc_datapath_cap_evb = + CAP_FLAG(flags, EVB) ? B_TRUE : B_FALSE; + + /* Check if the firmware supports VLAN insertion */ + encp->enc_hw_tx_insert_vlan_enabled = + CAP_FLAG(flags, TX_VLAN_INSERTION) ? B_TRUE : B_FALSE; + + /* Check if the firmware supports RX event batching */ + encp->enc_rx_batching_enabled = + CAP_FLAG(flags, RX_BATCHING) ? B_TRUE : B_FALSE; + + /* + * Even if batching isn't reported as supported, we may still get + * batched events (see bug61153). + */ + encp->enc_rx_batch_max = 16; + + /* Check if the firmware supports disabling scatter on RXQs */ + encp->enc_rx_disable_scatter_supported = + CAP_FLAG(flags, RX_DISABLE_SCATTER) ? B_TRUE : B_FALSE; + + /* Check if the firmware supports packed stream mode */ + encp->enc_rx_packed_stream_supported = + CAP_FLAG(flags, RX_PACKED_STREAM) ? B_TRUE : B_FALSE; + + /* + * Check if the firmware supports configurable buffer sizes + * for packed stream mode (otherwise buffer size is 1Mbyte) + */ + encp->enc_rx_var_packed_stream_supported = + CAP_FLAG(flags, RX_PACKED_STREAM_VAR_BUFFERS) ? B_TRUE : B_FALSE; + + /* Check if the firmware supports set mac with running filters */ + encp->enc_allow_set_mac_with_installed_filters = + CAP_FLAG(flags, VADAPTOR_PERMIT_SET_MAC_WHEN_FILTERS_INSTALLED) ? + B_TRUE : B_FALSE; + + /* + * Check if firmware supports the extended MC_CMD_SET_MAC, which allows + * specifying which parameters to configure. + */ + encp->enc_enhanced_set_mac_supported = + CAP_FLAG(flags, SET_MAC_ENHANCED) ? B_TRUE : B_FALSE; + + /* + * Check if firmware supports version 2 of MC_CMD_INIT_EVQ, which allows + * us to let the firmware choose the settings to use on an EVQ. + */ + encp->enc_init_evq_v2_supported = + CAP_FLAG2(flags2, INIT_EVQ_V2) ? B_TRUE : B_FALSE; + + /* + * Check if firmware-verified NVRAM updates must be used. + * + * The firmware trusted installer requires all NVRAM updates to use + * version 2 of MC_CMD_NVRAM_UPDATE_START (to enable verified update) + * and version 2 of MC_CMD_NVRAM_UPDATE_FINISH (to verify the updated + * partition and report the result). + */ + encp->enc_fw_verified_nvram_update_required = + CAP_FLAG2(flags2, NVRAM_UPDATE_REPORT_VERIFY_RESULT) ? + B_TRUE : B_FALSE; + + /* + * Check if firmware provides packet memory and Rx datapath + * counters. + */ + encp->enc_pm_and_rxdp_counters = + CAP_FLAG(flags, PM_AND_RXDP_COUNTERS) ? B_TRUE : B_FALSE; + + /* + * Check if the 40G MAC hardware is capable of reporting + * statistics for Tx size bins. + */ + encp->enc_mac_stats_40g_tx_size_bins = + CAP_FLAG2(flags2, MAC_STATS_40G_TX_SIZE_BINS) ? B_TRUE : B_FALSE; + + /* + * Check if firmware supports VXLAN and NVGRE tunnels. + * The capability indicates Geneve protocol support as well. + */ + if (CAP_FLAG(flags, VXLAN_NVGRE)) + encp->enc_tunnel_encapsulations_supported = + (1u << EFX_TUNNEL_PROTOCOL_VXLAN) | + (1u << EFX_TUNNEL_PROTOCOL_GENEVE) | + (1u << EFX_TUNNEL_PROTOCOL_NVGRE); + +#undef CAP_FLAG +#undef CAP_FLAG2 + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + +#define EF10_LEGACY_PF_PRIVILEGE_MASK \ + (MC_CMD_PRIVILEGE_MASK_IN_GRP_ADMIN | \ + MC_CMD_PRIVILEGE_MASK_IN_GRP_LINK | \ + MC_CMD_PRIVILEGE_MASK_IN_GRP_ONLOAD | \ + MC_CMD_PRIVILEGE_MASK_IN_GRP_PTP | \ + MC_CMD_PRIVILEGE_MASK_IN_GRP_INSECURE_FILTERS | \ + MC_CMD_PRIVILEGE_MASK_IN_GRP_MAC_SPOOFING | \ + MC_CMD_PRIVILEGE_MASK_IN_GRP_UNICAST | \ + MC_CMD_PRIVILEGE_MASK_IN_GRP_MULTICAST | \ + MC_CMD_PRIVILEGE_MASK_IN_GRP_BROADCAST | \ + MC_CMD_PRIVILEGE_MASK_IN_GRP_ALL_MULTICAST | \ + MC_CMD_PRIVILEGE_MASK_IN_GRP_PROMISCUOUS) + +#define EF10_LEGACY_VF_PRIVILEGE_MASK 0 + + + __checkReturn efx_rc_t +ef10_get_privilege_mask( + __in efx_nic_t *enp, + __out uint32_t *maskp) +{ + efx_nic_cfg_t *encp = &(enp->en_nic_cfg); + uint32_t mask; + efx_rc_t rc; + + if ((rc = efx_mcdi_privilege_mask(enp, encp->enc_pf, encp->enc_vf, + &mask)) != 0) { + if (rc != ENOTSUP) + goto fail1; + + /* Fallback for old firmware without privilege mask support */ + if (EFX_PCI_FUNCTION_IS_PF(encp)) { + /* Assume PF has admin privilege */ + mask = EF10_LEGACY_PF_PRIVILEGE_MASK; + } else { + /* VF is always unprivileged by default */ + mask = EF10_LEGACY_VF_PRIVILEGE_MASK; + } + } + + *maskp = mask; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + +/* + * Table of mapping schemes from port number to the number of the external + * connector on the board. The external numbering does not distinguish + * off-board separated outputs such as from multi-headed cables. + * + * The count of adjacent port numbers that map to each external port + * and the offset in the numbering, is determined by the chip family and + * current port mode. + * + * For the Huntington family, the current port mode cannot be discovered, + * so the mapping used is instead the last match in the table to the full + * set of port modes to which the NIC can be configured. Therefore the + * ordering of entries in the the mapping table is significant. + */ +static struct { + efx_family_t family; + uint32_t modes_mask; + int32_t count; + int32_t offset; +} __ef10_external_port_mappings[] = { + /* Supported modes with 1 output per external port */ + { + EFX_FAMILY_HUNTINGTON, + (1 << TLV_PORT_MODE_10G) | + (1 << TLV_PORT_MODE_10G_10G) | + (1 << TLV_PORT_MODE_10G_10G_10G_10G), + 1, + 1 + }, + { + EFX_FAMILY_MEDFORD, + (1 << TLV_PORT_MODE_10G) | + (1 << TLV_PORT_MODE_10G_10G), + 1, + 1 + }, + /* Supported modes with 2 outputs per external port */ + { + EFX_FAMILY_HUNTINGTON, + (1 << TLV_PORT_MODE_40G) | + (1 << TLV_PORT_MODE_40G_40G) | + (1 << TLV_PORT_MODE_40G_10G_10G) | + (1 << TLV_PORT_MODE_10G_10G_40G), + 2, + 1 + }, + { + EFX_FAMILY_MEDFORD, + (1 << TLV_PORT_MODE_40G) | + (1 << TLV_PORT_MODE_40G_40G) | + (1 << TLV_PORT_MODE_40G_10G_10G) | + (1 << TLV_PORT_MODE_10G_10G_40G) | + (1 << TLV_PORT_MODE_10G_10G_10G_10G_Q1_Q2), + 2, + 1 + }, + /* Supported modes with 4 outputs per external port */ + { + EFX_FAMILY_MEDFORD, + (1 << TLV_PORT_MODE_10G_10G_10G_10G_Q) | + (1 << TLV_PORT_MODE_10G_10G_10G_10G_Q1), + 4, + 1, + }, + { + EFX_FAMILY_MEDFORD, + (1 << TLV_PORT_MODE_10G_10G_10G_10G_Q2), + 4, + 2 + }, +}; + + __checkReturn efx_rc_t +ef10_external_port_mapping( + __in efx_nic_t *enp, + __in uint32_t port, + __out uint8_t *external_portp) +{ + efx_rc_t rc; + int i; + uint32_t port_modes; + uint32_t matches; + uint32_t current; + int32_t count = 1; /* Default 1-1 mapping */ + int32_t offset = 1; /* Default starting external port number */ + + if ((rc = efx_mcdi_get_port_modes(enp, &port_modes, ¤t)) != 0) { + /* + * No current port mode information + * - infer mapping from available modes + */ + if ((rc = efx_mcdi_get_port_modes(enp, + &port_modes, NULL)) != 0) { + /* + * No port mode information available + * - use default mapping + */ + goto out; + } + } else { + /* Only need to scan the current mode */ + port_modes = 1 << current; + } + + /* + * Infer the internal port -> external port mapping from + * the possible port modes for this NIC. + */ + for (i = 0; i < EFX_ARRAY_SIZE(__ef10_external_port_mappings); ++i) { + if (__ef10_external_port_mappings[i].family != + enp->en_family) + continue; + matches = (__ef10_external_port_mappings[i].modes_mask & + port_modes); + if (matches != 0) { + count = __ef10_external_port_mappings[i].count; + offset = __ef10_external_port_mappings[i].offset; + port_modes &= ~matches; + } + } + + if (port_modes != 0) { + /* Some advertised modes are not supported */ + rc = ENOTSUP; + goto fail1; + } + +out: + /* + * Scale as required by last matched mode and then convert to + * correctly offset numbering + */ + *external_portp = (uint8_t)((port / count) + offset); + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + + __checkReturn efx_rc_t +ef10_nic_probe( + __in efx_nic_t *enp) +{ + const efx_nic_ops_t *enop = enp->en_enop; + efx_nic_cfg_t *encp = &(enp->en_nic_cfg); + efx_drv_cfg_t *edcp = &(enp->en_drv_cfg); + efx_rc_t rc; + + EFSYS_ASSERT(enp->en_family == EFX_FAMILY_HUNTINGTON || + enp->en_family == EFX_FAMILY_MEDFORD); + + /* Read and clear any assertion state */ + if ((rc = efx_mcdi_read_assertion(enp)) != 0) + goto fail1; + + /* Exit the assertion handler */ + if ((rc = efx_mcdi_exit_assertion_handler(enp)) != 0) + if (rc != EACCES) + goto fail2; + + if ((rc = efx_mcdi_drv_attach(enp, B_TRUE)) != 0) + goto fail3; + + if ((rc = enop->eno_board_cfg(enp)) != 0) + if (rc != EACCES) + goto fail4; + + /* + * Set default driver config limits (based on board config). + * + * FIXME: For now allocate a fixed number of VIs which is likely to be + * sufficient and small enough to allow multiple functions on the same + * port. + */ + edcp->edc_min_vi_count = edcp->edc_max_vi_count = + MIN(128, MAX(encp->enc_rxq_limit, encp->enc_txq_limit)); + + /* The client driver must configure and enable PIO buffer support */ + edcp->edc_max_piobuf_count = 0; + edcp->edc_pio_alloc_size = 0; + +#if EFSYS_OPT_MAC_STATS + /* Wipe the MAC statistics */ + if ((rc = efx_mcdi_mac_stats_clear(enp)) != 0) + goto fail5; +#endif + +#if EFSYS_OPT_LOOPBACK + if ((rc = efx_mcdi_get_loopback_modes(enp)) != 0) + goto fail6; +#endif + +#if EFSYS_OPT_MON_STATS + if ((rc = mcdi_mon_cfg_build(enp)) != 0) { + /* Unprivileged functions do not have access to sensors */ + if (rc != EACCES) + goto fail7; + } +#endif + + encp->enc_features = enp->en_features; + + return (0); + +#if EFSYS_OPT_MON_STATS +fail7: + EFSYS_PROBE(fail7); +#endif +#if EFSYS_OPT_LOOPBACK +fail6: + EFSYS_PROBE(fail6); +#endif +#if EFSYS_OPT_MAC_STATS +fail5: + EFSYS_PROBE(fail5); +#endif +fail4: + EFSYS_PROBE(fail4); +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +ef10_nic_set_drv_limits( + __inout efx_nic_t *enp, + __in efx_drv_limits_t *edlp) +{ + efx_nic_cfg_t *encp = &(enp->en_nic_cfg); + efx_drv_cfg_t *edcp = &(enp->en_drv_cfg); + uint32_t min_evq_count, max_evq_count; + uint32_t min_rxq_count, max_rxq_count; + uint32_t min_txq_count, max_txq_count; + efx_rc_t rc; + + if (edlp == NULL) { + rc = EINVAL; + goto fail1; + } + + /* Get minimum required and maximum usable VI limits */ + min_evq_count = MIN(edlp->edl_min_evq_count, encp->enc_evq_limit); + min_rxq_count = MIN(edlp->edl_min_rxq_count, encp->enc_rxq_limit); + min_txq_count = MIN(edlp->edl_min_txq_count, encp->enc_txq_limit); + + edcp->edc_min_vi_count = + MAX(min_evq_count, MAX(min_rxq_count, min_txq_count)); + + max_evq_count = MIN(edlp->edl_max_evq_count, encp->enc_evq_limit); + max_rxq_count = MIN(edlp->edl_max_rxq_count, encp->enc_rxq_limit); + max_txq_count = MIN(edlp->edl_max_txq_count, encp->enc_txq_limit); + + edcp->edc_max_vi_count = + MAX(max_evq_count, MAX(max_rxq_count, max_txq_count)); + + /* + * Check limits for sub-allocated piobuf blocks. + * PIO is optional, so don't fail if the limits are incorrect. + */ + if ((encp->enc_piobuf_size == 0) || + (encp->enc_piobuf_limit == 0) || + (edlp->edl_min_pio_alloc_size == 0) || + (edlp->edl_min_pio_alloc_size > encp->enc_piobuf_size)) { + /* Disable PIO */ + edcp->edc_max_piobuf_count = 0; + edcp->edc_pio_alloc_size = 0; + } else { + uint32_t blk_size, blk_count, blks_per_piobuf; + + blk_size = + MAX(edlp->edl_min_pio_alloc_size, + encp->enc_piobuf_min_alloc_size); + + blks_per_piobuf = encp->enc_piobuf_size / blk_size; + EFSYS_ASSERT3U(blks_per_piobuf, <=, 32); + + blk_count = (encp->enc_piobuf_limit * blks_per_piobuf); + + /* A zero max pio alloc count means unlimited */ + if ((edlp->edl_max_pio_alloc_count > 0) && + (edlp->edl_max_pio_alloc_count < blk_count)) { + blk_count = edlp->edl_max_pio_alloc_count; + } + + edcp->edc_pio_alloc_size = blk_size; + edcp->edc_max_piobuf_count = + (blk_count + (blks_per_piobuf - 1)) / blks_per_piobuf; + } + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + + __checkReturn efx_rc_t +ef10_nic_reset( + __in efx_nic_t *enp) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_ENTITY_RESET_IN_LEN, + MC_CMD_ENTITY_RESET_OUT_LEN)]; + efx_rc_t rc; + + /* ef10_nic_reset() is called to recover from BADASSERT failures. */ + if ((rc = efx_mcdi_read_assertion(enp)) != 0) + goto fail1; + if ((rc = efx_mcdi_exit_assertion_handler(enp)) != 0) + goto fail2; + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_ENTITY_RESET; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_ENTITY_RESET_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_ENTITY_RESET_OUT_LEN; + + MCDI_IN_POPULATE_DWORD_1(req, ENTITY_RESET_IN_FLAG, + ENTITY_RESET_IN_FUNCTION_RESOURCE_RESET, 1); + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail3; + } + + /* Clear RX/TX DMA queue errors */ + enp->en_reset_flags &= ~(EFX_RESET_RXQ_ERR | EFX_RESET_TXQ_ERR); + + return (0); + +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +ef10_nic_init( + __in efx_nic_t *enp) +{ + efx_drv_cfg_t *edcp = &(enp->en_drv_cfg); + uint32_t min_vi_count, max_vi_count; + uint32_t vi_count, vi_base, vi_shift; + uint32_t i; + uint32_t retry; + uint32_t delay_us; + efx_rc_t rc; + + EFSYS_ASSERT(enp->en_family == EFX_FAMILY_HUNTINGTON || + enp->en_family == EFX_FAMILY_MEDFORD); + + /* Enable reporting of some events (e.g. link change) */ + if ((rc = efx_mcdi_log_ctrl(enp)) != 0) + goto fail1; + + /* Allocate (optional) on-chip PIO buffers */ + ef10_nic_alloc_piobufs(enp, edcp->edc_max_piobuf_count); + + /* + * For best performance, PIO writes should use a write-combined + * (WC) memory mapping. Using a separate WC mapping for the PIO + * aperture of each VI would be a burden to drivers (and not + * possible if the host page size is >4Kbyte). + * + * To avoid this we use a single uncached (UC) mapping for VI + * register access, and a single WC mapping for extra VIs used + * for PIO writes. + * + * Each piobuf must be linked to a VI in the WC mapping, and to + * each VI that is using a sub-allocated block from the piobuf. + */ + min_vi_count = edcp->edc_min_vi_count; + max_vi_count = + edcp->edc_max_vi_count + enp->en_arch.ef10.ena_piobuf_count; + + /* Ensure that the previously attached driver's VIs are freed */ + if ((rc = efx_mcdi_free_vis(enp)) != 0) + goto fail2; + + /* + * Reserve VI resources (EVQ+RXQ+TXQ) for this PCIe function. If this + * fails then retrying the request for fewer VI resources may succeed. + */ + vi_count = 0; + if ((rc = efx_mcdi_alloc_vis(enp, min_vi_count, max_vi_count, + &vi_base, &vi_count, &vi_shift)) != 0) + goto fail3; + + EFSYS_PROBE2(vi_alloc, uint32_t, vi_base, uint32_t, vi_count); + + if (vi_count < min_vi_count) { + rc = ENOMEM; + goto fail4; + } + + enp->en_arch.ef10.ena_vi_base = vi_base; + enp->en_arch.ef10.ena_vi_count = vi_count; + enp->en_arch.ef10.ena_vi_shift = vi_shift; + + if (vi_count < min_vi_count + enp->en_arch.ef10.ena_piobuf_count) { + /* Not enough extra VIs to map piobufs */ + ef10_nic_free_piobufs(enp); + } + + enp->en_arch.ef10.ena_pio_write_vi_base = + vi_count - enp->en_arch.ef10.ena_piobuf_count; + + /* Save UC memory mapping details */ + enp->en_arch.ef10.ena_uc_mem_map_offset = 0; + if (enp->en_arch.ef10.ena_piobuf_count > 0) { + enp->en_arch.ef10.ena_uc_mem_map_size = + (ER_DZ_TX_PIOBUF_STEP * + enp->en_arch.ef10.ena_pio_write_vi_base); + } else { + enp->en_arch.ef10.ena_uc_mem_map_size = + (ER_DZ_TX_PIOBUF_STEP * + enp->en_arch.ef10.ena_vi_count); + } + + /* Save WC memory mapping details */ + enp->en_arch.ef10.ena_wc_mem_map_offset = + enp->en_arch.ef10.ena_uc_mem_map_offset + + enp->en_arch.ef10.ena_uc_mem_map_size; + + enp->en_arch.ef10.ena_wc_mem_map_size = + (ER_DZ_TX_PIOBUF_STEP * + enp->en_arch.ef10.ena_piobuf_count); + + /* Link piobufs to extra VIs in WC mapping */ + if (enp->en_arch.ef10.ena_piobuf_count > 0) { + for (i = 0; i < enp->en_arch.ef10.ena_piobuf_count; i++) { + rc = efx_mcdi_link_piobuf(enp, + enp->en_arch.ef10.ena_pio_write_vi_base + i, + enp->en_arch.ef10.ena_piobuf_handle[i]); + if (rc != 0) + break; + } + } + + /* + * Allocate a vAdaptor attached to our upstream vPort/pPort. + * + * On a VF, this may fail with MC_CMD_ERR_NO_EVB_PORT (ENOENT) if the PF + * driver has yet to bring up the EVB port. See bug 56147. In this case, + * retry the request several times after waiting a while. The wait time + * between retries starts small (10ms) and exponentially increases. + * Total wait time is a little over two seconds. Retry logic in the + * client driver may mean this whole loop is repeated if it continues to + * fail. + */ + retry = 0; + delay_us = 10000; + while ((rc = efx_mcdi_vadaptor_alloc(enp, EVB_PORT_ID_ASSIGNED)) != 0) { + if (EFX_PCI_FUNCTION_IS_PF(&enp->en_nic_cfg) || + (rc != ENOENT)) { + /* + * Do not retry alloc for PF, or for other errors on + * a VF. + */ + goto fail5; + } + + /* VF startup before PF is ready. Retry allocation. */ + if (retry > 5) { + /* Too many attempts */ + rc = EINVAL; + goto fail6; + } + EFSYS_PROBE1(mcdi_no_evb_port_retry, int, retry); + EFSYS_SLEEP(delay_us); + retry++; + if (delay_us < 500000) + delay_us <<= 2; + } + + enp->en_vport_id = EVB_PORT_ID_ASSIGNED; + enp->en_nic_cfg.enc_mcdi_max_payload_length = MCDI_CTL_SDU_LEN_MAX_V2; + + return (0); + +fail6: + EFSYS_PROBE(fail6); +fail5: + EFSYS_PROBE(fail5); +fail4: + EFSYS_PROBE(fail4); +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); + + ef10_nic_free_piobufs(enp); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +ef10_nic_get_vi_pool( + __in efx_nic_t *enp, + __out uint32_t *vi_countp) +{ + EFSYS_ASSERT(enp->en_family == EFX_FAMILY_HUNTINGTON || + enp->en_family == EFX_FAMILY_MEDFORD); + + /* + * Report VIs that the client driver can use. + * Do not include VIs used for PIO buffer writes. + */ + *vi_countp = enp->en_arch.ef10.ena_pio_write_vi_base; + + return (0); +} + + __checkReturn efx_rc_t +ef10_nic_get_bar_region( + __in efx_nic_t *enp, + __in efx_nic_region_t region, + __out uint32_t *offsetp, + __out size_t *sizep) +{ + efx_rc_t rc; + + EFSYS_ASSERT(enp->en_family == EFX_FAMILY_HUNTINGTON || + enp->en_family == EFX_FAMILY_MEDFORD); + + /* + * TODO: Specify host memory mapping alignment and granularity + * in efx_drv_limits_t so that they can be taken into account + * when allocating extra VIs for PIO writes. + */ + switch (region) { + case EFX_REGION_VI: + /* UC mapped memory BAR region for VI registers */ + *offsetp = enp->en_arch.ef10.ena_uc_mem_map_offset; + *sizep = enp->en_arch.ef10.ena_uc_mem_map_size; + break; + + case EFX_REGION_PIO_WRITE_VI: + /* WC mapped memory BAR region for piobuf writes */ + *offsetp = enp->en_arch.ef10.ena_wc_mem_map_offset; + *sizep = enp->en_arch.ef10.ena_wc_mem_map_size; + break; + + default: + rc = EINVAL; + goto fail1; + } + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + void +ef10_nic_fini( + __in efx_nic_t *enp) +{ + uint32_t i; + efx_rc_t rc; + + (void) efx_mcdi_vadaptor_free(enp, enp->en_vport_id); + enp->en_vport_id = 0; + + /* Unlink piobufs from extra VIs in WC mapping */ + if (enp->en_arch.ef10.ena_piobuf_count > 0) { + for (i = 0; i < enp->en_arch.ef10.ena_piobuf_count; i++) { + rc = efx_mcdi_unlink_piobuf(enp, + enp->en_arch.ef10.ena_pio_write_vi_base + i); + if (rc != 0) + break; + } + } + + ef10_nic_free_piobufs(enp); + + (void) efx_mcdi_free_vis(enp); + enp->en_arch.ef10.ena_vi_count = 0; +} + + void +ef10_nic_unprobe( + __in efx_nic_t *enp) +{ +#if EFSYS_OPT_MON_STATS + mcdi_mon_cfg_free(enp); +#endif /* EFSYS_OPT_MON_STATS */ + (void) efx_mcdi_drv_attach(enp, B_FALSE); +} + +#if EFSYS_OPT_DIAG + + __checkReturn efx_rc_t +ef10_nic_register_test( + __in efx_nic_t *enp) +{ + efx_rc_t rc; + + /* FIXME */ + _NOTE(ARGUNUSED(enp)) + _NOTE(CONSTANTCONDITION) + if (B_FALSE) { + rc = ENOTSUP; + goto fail1; + } + /* FIXME */ + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +#endif /* EFSYS_OPT_DIAG */ + + +#endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */ diff --git a/dpdk/drivers/net/sfc/base/ef10_nvram.c b/dpdk/drivers/net/sfc/base/ef10_nvram.c new file mode 100644 index 00000000..3f9d3750 --- /dev/null +++ b/dpdk/drivers/net/sfc/base/ef10_nvram.c @@ -0,0 +1,2385 @@ +/* + * Copyright (c) 2012-2016 Solarflare Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are + * those of the authors and should not be interpreted as representing official + * policies, either expressed or implied, of the FreeBSD Project. + */ + +#include "efx.h" +#include "efx_impl.h" + +#if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD + +#if EFSYS_OPT_VPD || EFSYS_OPT_NVRAM + +#include "ef10_tlv_layout.h" + +/* Cursor for TLV partition format */ +typedef struct tlv_cursor_s { + uint32_t *block; /* Base of data block */ + uint32_t *current; /* Cursor position */ + uint32_t *end; /* End tag position */ + uint32_t *limit; /* Last dword of data block */ +} tlv_cursor_t; + +typedef struct nvram_partition_s { + uint16_t type; + uint8_t chip_select; + uint8_t flags; + /* + * The full length of the NVRAM partition. + * This is different from tlv_partition_header.total_length, + * which can be smaller. + */ + uint32_t length; + uint32_t erase_size; + uint32_t *data; + tlv_cursor_t tlv_cursor; +} nvram_partition_t; + + +static __checkReturn efx_rc_t +tlv_validate_state( + __inout tlv_cursor_t *cursor); + + +static void +tlv_init_block( + __out uint32_t *block) +{ + *block = __CPU_TO_LE_32(TLV_TAG_END); +} + +static uint32_t +tlv_tag( + __in tlv_cursor_t *cursor) +{ + uint32_t dword, tag; + + dword = cursor->current[0]; + tag = __LE_TO_CPU_32(dword); + + return (tag); +} + +static size_t +tlv_length( + __in tlv_cursor_t *cursor) +{ + uint32_t dword, length; + + if (tlv_tag(cursor) == TLV_TAG_END) + return (0); + + dword = cursor->current[1]; + length = __LE_TO_CPU_32(dword); + + return ((size_t)length); +} + +static uint8_t * +tlv_value( + __in tlv_cursor_t *cursor) +{ + if (tlv_tag(cursor) == TLV_TAG_END) + return (NULL); + + return ((uint8_t *)(&cursor->current[2])); +} + +static uint8_t * +tlv_item( + __in tlv_cursor_t *cursor) +{ + if (tlv_tag(cursor) == TLV_TAG_END) + return (NULL); + + return ((uint8_t *)cursor->current); +} + +/* + * TLV item DWORD length is tag + length + value (rounded up to DWORD) + * equivalent to tlv_n_words_for_len in mc-comms tlv.c + */ +#define TLV_DWORD_COUNT(length) \ + (1 + 1 + (((length) + sizeof (uint32_t) - 1) / sizeof (uint32_t))) + + +static uint32_t * +tlv_next_item_ptr( + __in tlv_cursor_t *cursor) +{ + uint32_t length; + + length = tlv_length(cursor); + + return (cursor->current + TLV_DWORD_COUNT(length)); +} + +static __checkReturn efx_rc_t +tlv_advance( + __inout tlv_cursor_t *cursor) +{ + efx_rc_t rc; + + if ((rc = tlv_validate_state(cursor)) != 0) + goto fail1; + + if (cursor->current == cursor->end) { + /* No more tags after END tag */ + cursor->current = NULL; + rc = ENOENT; + goto fail2; + } + + /* Advance to next item and validate */ + cursor->current = tlv_next_item_ptr(cursor); + + if ((rc = tlv_validate_state(cursor)) != 0) + goto fail3; + + return (0); + +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +static efx_rc_t +tlv_rewind( + __in tlv_cursor_t *cursor) +{ + efx_rc_t rc; + + cursor->current = cursor->block; + + if ((rc = tlv_validate_state(cursor)) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +static efx_rc_t +tlv_find( + __inout tlv_cursor_t *cursor, + __in uint32_t tag) +{ + efx_rc_t rc; + + rc = tlv_rewind(cursor); + while (rc == 0) { + if (tlv_tag(cursor) == tag) + break; + + rc = tlv_advance(cursor); + } + return (rc); +} + +static __checkReturn efx_rc_t +tlv_validate_state( + __inout tlv_cursor_t *cursor) +{ + efx_rc_t rc; + + /* Check cursor position */ + if (cursor->current < cursor->block) { + rc = EINVAL; + goto fail1; + } + if (cursor->current > cursor->limit) { + rc = EINVAL; + goto fail2; + } + + if (tlv_tag(cursor) != TLV_TAG_END) { + /* Check current item has space for tag and length */ + if (cursor->current > (cursor->limit - 2)) { + cursor->current = NULL; + rc = EFAULT; + goto fail3; + } + + /* Check we have value data for current item and another tag */ + if (tlv_next_item_ptr(cursor) > (cursor->limit - 1)) { + cursor->current = NULL; + rc = EFAULT; + goto fail4; + } + } + + return (0); + +fail4: + EFSYS_PROBE(fail4); +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +static efx_rc_t +tlv_init_cursor( + __out tlv_cursor_t *cursor, + __in uint32_t *block, + __in uint32_t *limit, + __in uint32_t *current) +{ + cursor->block = block; + cursor->limit = limit; + + cursor->current = current; + cursor->end = NULL; + + return (tlv_validate_state(cursor)); +} + +static __checkReturn efx_rc_t +tlv_init_cursor_from_size( + __out tlv_cursor_t *cursor, + __in_bcount(size) + uint8_t *block, + __in size_t size) +{ + uint32_t *limit; + limit = (uint32_t *)(block + size - sizeof (uint32_t)); + return (tlv_init_cursor(cursor, (uint32_t *)block, + limit, (uint32_t *)block)); +} + +static __checkReturn efx_rc_t +tlv_init_cursor_at_offset( + __out tlv_cursor_t *cursor, + __in_bcount(size) + uint8_t *block, + __in size_t size, + __in size_t offset) +{ + uint32_t *limit; + uint32_t *current; + limit = (uint32_t *)(block + size - sizeof (uint32_t)); + current = (uint32_t *)(block + offset); + return (tlv_init_cursor(cursor, (uint32_t *)block, limit, current)); +} + +static __checkReturn efx_rc_t +tlv_require_end( + __inout tlv_cursor_t *cursor) +{ + uint32_t *pos; + efx_rc_t rc; + + if (cursor->end == NULL) { + pos = cursor->current; + if ((rc = tlv_find(cursor, TLV_TAG_END)) != 0) + goto fail1; + + cursor->end = cursor->current; + cursor->current = pos; + } + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +static size_t +tlv_block_length_used( + __inout tlv_cursor_t *cursor) +{ + efx_rc_t rc; + + if ((rc = tlv_validate_state(cursor)) != 0) + goto fail1; + + if ((rc = tlv_require_end(cursor)) != 0) + goto fail2; + + /* Return space used (including the END tag) */ + return (cursor->end + 1 - cursor->block) * sizeof (uint32_t); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (0); +} + +static uint32_t * +tlv_last_segment_end( + __in tlv_cursor_t *cursor) +{ + tlv_cursor_t segment_cursor; + uint32_t *last_segment_end = cursor->block; + uint32_t *segment_start = cursor->block; + + /* + * Go through each segment and check that it has an end tag. If there + * is no end tag then the previous segment was the last valid one, + * so return the pointer to its end tag. + */ + for (;;) { + if (tlv_init_cursor(&segment_cursor, segment_start, + cursor->limit, segment_start) != 0) + break; + if (tlv_require_end(&segment_cursor) != 0) + break; + last_segment_end = segment_cursor.end; + segment_start = segment_cursor.end + 1; + } + + return (last_segment_end); +} + + +static uint32_t * +tlv_write( + __in tlv_cursor_t *cursor, + __in uint32_t tag, + __in_bcount(size) uint8_t *data, + __in size_t size) +{ + uint32_t len = size; + uint32_t *ptr; + + ptr = cursor->current; + + *ptr++ = __CPU_TO_LE_32(tag); + *ptr++ = __CPU_TO_LE_32(len); + + if (len > 0) { + ptr[(len - 1) / sizeof (uint32_t)] = 0; + memcpy(ptr, data, len); + ptr += P2ROUNDUP(len, sizeof (uint32_t)) / sizeof (*ptr); + } + + return (ptr); +} + +static __checkReturn efx_rc_t +tlv_insert( + __inout tlv_cursor_t *cursor, + __in uint32_t tag, + __in_bcount(size) + uint8_t *data, + __in size_t size) +{ + unsigned int delta; + uint32_t *last_segment_end; + efx_rc_t rc; + + if ((rc = tlv_validate_state(cursor)) != 0) + goto fail1; + + if ((rc = tlv_require_end(cursor)) != 0) + goto fail2; + + if (tag == TLV_TAG_END) { + rc = EINVAL; + goto fail3; + } + + last_segment_end = tlv_last_segment_end(cursor); + + delta = TLV_DWORD_COUNT(size); + if (last_segment_end + 1 + delta > cursor->limit) { + rc = ENOSPC; + goto fail4; + } + + /* Move data up: new space at cursor->current */ + memmove(cursor->current + delta, cursor->current, + (last_segment_end + 1 - cursor->current) * sizeof (uint32_t)); + + /* Adjust the end pointer */ + cursor->end += delta; + + /* Write new TLV item */ + tlv_write(cursor, tag, data, size); + + return (0); + +fail4: + EFSYS_PROBE(fail4); +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +static __checkReturn efx_rc_t +tlv_delete( + __inout tlv_cursor_t *cursor) +{ + unsigned int delta; + uint32_t *last_segment_end; + efx_rc_t rc; + + if ((rc = tlv_validate_state(cursor)) != 0) + goto fail1; + + if (tlv_tag(cursor) == TLV_TAG_END) { + rc = EINVAL; + goto fail2; + } + + delta = TLV_DWORD_COUNT(tlv_length(cursor)); + + if ((rc = tlv_require_end(cursor)) != 0) + goto fail3; + + last_segment_end = tlv_last_segment_end(cursor); + + /* Shuffle things down, destroying the item at cursor->current */ + memmove(cursor->current, cursor->current + delta, + (last_segment_end + 1 - cursor->current) * sizeof (uint32_t)); + /* Zero the new space at the end of the TLV chain */ + memset(last_segment_end + 1 - delta, 0, delta * sizeof (uint32_t)); + /* Adjust the end pointer */ + cursor->end -= delta; + + return (0); + +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +static __checkReturn efx_rc_t +tlv_modify( + __inout tlv_cursor_t *cursor, + __in uint32_t tag, + __in_bcount(size) + uint8_t *data, + __in size_t size) +{ + uint32_t *pos; + unsigned int old_ndwords; + unsigned int new_ndwords; + unsigned int delta; + uint32_t *last_segment_end; + efx_rc_t rc; + + if ((rc = tlv_validate_state(cursor)) != 0) + goto fail1; + + if (tlv_tag(cursor) == TLV_TAG_END) { + rc = EINVAL; + goto fail2; + } + if (tlv_tag(cursor) != tag) { + rc = EINVAL; + goto fail3; + } + + old_ndwords = TLV_DWORD_COUNT(tlv_length(cursor)); + new_ndwords = TLV_DWORD_COUNT(size); + + if ((rc = tlv_require_end(cursor)) != 0) + goto fail4; + + last_segment_end = tlv_last_segment_end(cursor); + + if (new_ndwords > old_ndwords) { + /* Expand space used for TLV item */ + delta = new_ndwords - old_ndwords; + pos = cursor->current + old_ndwords; + + if (last_segment_end + 1 + delta > cursor->limit) { + rc = ENOSPC; + goto fail5; + } + + /* Move up: new space at (cursor->current + old_ndwords) */ + memmove(pos + delta, pos, + (last_segment_end + 1 - pos) * sizeof (uint32_t)); + + /* Adjust the end pointer */ + cursor->end += delta; + + } else if (new_ndwords < old_ndwords) { + /* Shrink space used for TLV item */ + delta = old_ndwords - new_ndwords; + pos = cursor->current + new_ndwords; + + /* Move down: remove words at (cursor->current + new_ndwords) */ + memmove(pos, pos + delta, + (last_segment_end + 1 - pos) * sizeof (uint32_t)); + + /* Zero the new space at the end of the TLV chain */ + memset(last_segment_end + 1 - delta, 0, + delta * sizeof (uint32_t)); + + /* Adjust the end pointer */ + cursor->end -= delta; + } + + /* Write new data */ + tlv_write(cursor, tag, data, size); + + return (0); + +fail5: + EFSYS_PROBE(fail5); +fail4: + EFSYS_PROBE(fail4); +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +static uint32_t checksum_tlv_partition( + __in nvram_partition_t *partition) +{ + tlv_cursor_t *cursor; + uint32_t *ptr; + uint32_t *end; + uint32_t csum; + size_t len; + + cursor = &partition->tlv_cursor; + len = tlv_block_length_used(cursor); + EFSYS_ASSERT3U((len & 3), ==, 0); + + csum = 0; + ptr = partition->data; + end = &ptr[len >> 2]; + + while (ptr < end) + csum += __LE_TO_CPU_32(*ptr++); + + return (csum); +} + +static __checkReturn efx_rc_t +tlv_update_partition_len_and_cks( + __in tlv_cursor_t *cursor) +{ + efx_rc_t rc; + nvram_partition_t partition; + struct tlv_partition_header *header; + struct tlv_partition_trailer *trailer; + size_t new_len; + + /* + * We just modified the partition, so the total length may not be + * valid. Don't use tlv_find(), which performs some sanity checks + * that may fail here. + */ + partition.data = cursor->block; + memcpy(&partition.tlv_cursor, cursor, sizeof (*cursor)); + header = (struct tlv_partition_header *)partition.data; + /* Sanity check. */ + if (__LE_TO_CPU_32(header->tag) != TLV_TAG_PARTITION_HEADER) { + rc = EFAULT; + goto fail1; + } + new_len = tlv_block_length_used(&partition.tlv_cursor); + if (new_len == 0) { + rc = EFAULT; + goto fail2; + } + header->total_length = __CPU_TO_LE_32(new_len); + /* Ensure the modified partition always has a new generation count. */ + header->generation = __CPU_TO_LE_32( + __LE_TO_CPU_32(header->generation) + 1); + + trailer = (struct tlv_partition_trailer *)((uint8_t *)header + + new_len - sizeof (*trailer) - sizeof (uint32_t)); + trailer->generation = header->generation; + trailer->checksum = __CPU_TO_LE_32( + __LE_TO_CPU_32(trailer->checksum) - + checksum_tlv_partition(&partition)); + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +/* Validate buffer contents (before writing to flash) */ + __checkReturn efx_rc_t +ef10_nvram_buffer_validate( + __in efx_nic_t *enp, + __in uint32_t partn, + __in_bcount(partn_size) caddr_t partn_data, + __in size_t partn_size) +{ + tlv_cursor_t cursor; + struct tlv_partition_header *header; + struct tlv_partition_trailer *trailer; + size_t total_length; + uint32_t cksum; + int pos; + efx_rc_t rc; + + EFX_STATIC_ASSERT(sizeof (*header) <= EF10_NVRAM_CHUNK); + + if ((partn_data == NULL) || (partn_size == 0)) { + rc = EINVAL; + goto fail1; + } + + /* The partition header must be the first item (at offset zero) */ + if ((rc = tlv_init_cursor_from_size(&cursor, (uint8_t *)partn_data, + partn_size)) != 0) { + rc = EFAULT; + goto fail2; + } + if (tlv_tag(&cursor) != TLV_TAG_PARTITION_HEADER) { + rc = EINVAL; + goto fail3; + } + header = (struct tlv_partition_header *)tlv_item(&cursor); + + /* Check TLV partition length (includes the END tag) */ + total_length = __LE_TO_CPU_32(header->total_length); + if (total_length > partn_size) { + rc = EFBIG; + goto fail4; + } + + /* Check partition ends with PARTITION_TRAILER and END tags */ + if ((rc = tlv_find(&cursor, TLV_TAG_PARTITION_TRAILER)) != 0) { + rc = EINVAL; + goto fail5; + } + trailer = (struct tlv_partition_trailer *)tlv_item(&cursor); + + if ((rc = tlv_advance(&cursor)) != 0) { + rc = EINVAL; + goto fail6; + } + if (tlv_tag(&cursor) != TLV_TAG_END) { + rc = EINVAL; + goto fail7; + } + + /* Check generation counts are consistent */ + if (trailer->generation != header->generation) { + rc = EINVAL; + goto fail8; + } + + /* Verify partition checksum */ + cksum = 0; + for (pos = 0; (size_t)pos < total_length; pos += sizeof (uint32_t)) { + cksum += *((uint32_t *)(partn_data + pos)); + } + if (cksum != 0) { + rc = EINVAL; + goto fail9; + } + + return (0); + +fail9: + EFSYS_PROBE(fail9); +fail8: + EFSYS_PROBE(fail8); +fail7: + EFSYS_PROBE(fail7); +fail6: + EFSYS_PROBE(fail6); +fail5: + EFSYS_PROBE(fail5); +fail4: + EFSYS_PROBE(fail4); +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + + + __checkReturn efx_rc_t +ef10_nvram_buffer_create( + __in efx_nic_t *enp, + __in uint16_t partn_type, + __in_bcount(partn_size) caddr_t partn_data, + __in size_t partn_size) +{ + uint32_t *buf = (uint32_t *)partn_data; + efx_rc_t rc; + tlv_cursor_t cursor; + struct tlv_partition_header header; + struct tlv_partition_trailer trailer; + + unsigned int min_buf_size = sizeof (struct tlv_partition_header) + + sizeof (struct tlv_partition_trailer); + if (partn_size < min_buf_size) { + rc = EINVAL; + goto fail1; + } + + memset(buf, 0xff, partn_size); + + tlv_init_block(buf); + if ((rc = tlv_init_cursor(&cursor, buf, + (uint32_t *)((uint8_t *)buf + partn_size), + buf)) != 0) { + goto fail2; + } + + header.tag = __CPU_TO_LE_32(TLV_TAG_PARTITION_HEADER); + header.length = __CPU_TO_LE_32(sizeof (header) - 8); + header.type_id = __CPU_TO_LE_16(partn_type); + header.preset = 0; + header.generation = __CPU_TO_LE_32(1); + header.total_length = 0; /* This will be fixed below. */ + if ((rc = tlv_insert( + &cursor, TLV_TAG_PARTITION_HEADER, + (uint8_t *)&header.type_id, sizeof (header) - 8)) != 0) + goto fail3; + if ((rc = tlv_advance(&cursor)) != 0) + goto fail4; + + trailer.tag = __CPU_TO_LE_32(TLV_TAG_PARTITION_TRAILER); + trailer.length = __CPU_TO_LE_32(sizeof (trailer) - 8); + trailer.generation = header.generation; + trailer.checksum = 0; /* This will be fixed below. */ + if ((rc = tlv_insert(&cursor, TLV_TAG_PARTITION_TRAILER, + (uint8_t *)&trailer.generation, sizeof (trailer) - 8)) != 0) + goto fail5; + + if ((rc = tlv_update_partition_len_and_cks(&cursor)) != 0) + goto fail6; + + /* Check that the partition is valid. */ + if ((rc = ef10_nvram_buffer_validate(enp, partn_type, + partn_data, partn_size)) != 0) + goto fail7; + + return (0); + +fail7: + EFSYS_PROBE(fail7); +fail6: + EFSYS_PROBE(fail6); +fail5: + EFSYS_PROBE(fail5); +fail4: + EFSYS_PROBE(fail4); +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +static uint32_t +byte_offset( + __in uint32_t *position, + __in uint32_t *base) +{ + return (uint32_t)((uint8_t *)position - (uint8_t *)base); +} + + __checkReturn efx_rc_t +ef10_nvram_buffer_find_item_start( + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size, + __out uint32_t *startp) +{ + /* Read past partition header to find start address of the first key */ + tlv_cursor_t cursor; + efx_rc_t rc; + + /* A PARTITION_HEADER tag must be the first item (at offset zero) */ + if ((rc = tlv_init_cursor_from_size(&cursor, (uint8_t *)bufferp, + buffer_size)) != 0) { + rc = EFAULT; + goto fail1; + } + if (tlv_tag(&cursor) != TLV_TAG_PARTITION_HEADER) { + rc = EINVAL; + goto fail2; + } + + if ((rc = tlv_advance(&cursor)) != 0) { + rc = EINVAL; + goto fail3; + } + *startp = byte_offset(cursor.current, cursor.block); + + if ((rc = tlv_require_end(&cursor)) != 0) + goto fail4; + + return (0); + +fail4: + EFSYS_PROBE(fail4); +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +ef10_nvram_buffer_find_end( + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size, + __in uint32_t offset, + __out uint32_t *endp) +{ + /* Read to end of partition */ + tlv_cursor_t cursor; + efx_rc_t rc; + uint32_t *segment_used; + + _NOTE(ARGUNUSED(offset)) + + if ((rc = tlv_init_cursor_from_size(&cursor, (uint8_t *)bufferp, + buffer_size)) != 0) { + rc = EFAULT; + goto fail1; + } + + segment_used = cursor.block; + + /* + * Go through each segment and check that it has an end tag. If there + * is no end tag then the previous segment was the last valid one, + * so return the used space including that end tag. + */ + while (tlv_tag(&cursor) == TLV_TAG_PARTITION_HEADER) { + if (tlv_require_end(&cursor) != 0) { + if (segment_used == cursor.block) { + /* + * First segment is corrupt, so there is + * no valid data in partition. + */ + rc = EINVAL; + goto fail2; + } + break; + } + segment_used = cursor.end + 1; + + cursor.current = segment_used; + } + /* Return space used (including the END tag) */ + *endp = (segment_used - cursor.block) * sizeof (uint32_t); + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn __success(return != B_FALSE) boolean_t +ef10_nvram_buffer_find_item( + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size, + __in uint32_t offset, + __out uint32_t *startp, + __out uint32_t *lengthp) +{ + /* Find TLV at offset and return key start and length */ + tlv_cursor_t cursor; + uint8_t *key; + uint32_t tag; + + if (tlv_init_cursor_at_offset(&cursor, (uint8_t *)bufferp, + buffer_size, offset) != 0) { + return (B_FALSE); + } + + while ((key = tlv_item(&cursor)) != NULL) { + tag = tlv_tag(&cursor); + if (tag == TLV_TAG_PARTITION_HEADER || + tag == TLV_TAG_PARTITION_TRAILER) { + if (tlv_advance(&cursor) != 0) { + break; + } + continue; + } + *startp = byte_offset(cursor.current, cursor.block); + *lengthp = byte_offset(tlv_next_item_ptr(&cursor), + cursor.current); + return (B_TRUE); + } + + return (B_FALSE); +} + + __checkReturn efx_rc_t +ef10_nvram_buffer_get_item( + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size, + __in uint32_t offset, + __in uint32_t length, + __out_bcount_part(item_max_size, *lengthp) + caddr_t itemp, + __in size_t item_max_size, + __out uint32_t *lengthp) +{ + efx_rc_t rc; + tlv_cursor_t cursor; + uint32_t item_length; + + if (item_max_size < length) { + rc = ENOSPC; + goto fail1; + } + + if ((rc = tlv_init_cursor_at_offset(&cursor, (uint8_t *)bufferp, + buffer_size, offset)) != 0) { + goto fail2; + } + + item_length = tlv_length(&cursor); + if (length < item_length) { + rc = ENOSPC; + goto fail3; + } + memcpy(itemp, tlv_value(&cursor), item_length); + + *lengthp = item_length; + + return (0); + +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +ef10_nvram_buffer_insert_item( + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size, + __in uint32_t offset, + __in_bcount(length) caddr_t keyp, + __in uint32_t length, + __out uint32_t *lengthp) +{ + efx_rc_t rc; + tlv_cursor_t cursor; + + if ((rc = tlv_init_cursor_at_offset(&cursor, (uint8_t *)bufferp, + buffer_size, offset)) != 0) { + goto fail1; + } + + rc = tlv_insert(&cursor, TLV_TAG_LICENSE, (uint8_t *)keyp, length); + + if (rc != 0) { + goto fail2; + } + + *lengthp = byte_offset(tlv_next_item_ptr(&cursor), + cursor.current); + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +ef10_nvram_buffer_delete_item( + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size, + __in uint32_t offset, + __in uint32_t length, + __in uint32_t end) +{ + efx_rc_t rc; + tlv_cursor_t cursor; + + _NOTE(ARGUNUSED(length, end)) + + if ((rc = tlv_init_cursor_at_offset(&cursor, (uint8_t *)bufferp, + buffer_size, offset)) != 0) { + goto fail1; + } + + if ((rc = tlv_delete(&cursor)) != 0) + goto fail2; + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +ef10_nvram_buffer_finish( + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size) +{ + efx_rc_t rc; + tlv_cursor_t cursor; + + if ((rc = tlv_init_cursor_from_size(&cursor, (uint8_t *)bufferp, + buffer_size)) != 0) { + rc = EFAULT; + goto fail1; + } + + if ((rc = tlv_require_end(&cursor)) != 0) + goto fail2; + + if ((rc = tlv_update_partition_len_and_cks(&cursor)) != 0) + goto fail3; + + return (0); + +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + + +/* + * Read and validate a segment from a partition. A segment is a complete + * tlv chain between PARTITION_HEADER and PARTITION_END tags. There may + * be multiple segments in a partition, so seg_offset allows segments + * beyond the first to be read. + */ +static __checkReturn efx_rc_t +ef10_nvram_read_tlv_segment( + __in efx_nic_t *enp, + __in uint32_t partn, + __in size_t seg_offset, + __in_bcount(max_seg_size) caddr_t seg_data, + __in size_t max_seg_size) +{ + tlv_cursor_t cursor; + struct tlv_partition_header *header; + struct tlv_partition_trailer *trailer; + size_t total_length; + uint32_t cksum; + int pos; + efx_rc_t rc; + + EFX_STATIC_ASSERT(sizeof (*header) <= EF10_NVRAM_CHUNK); + + if ((seg_data == NULL) || (max_seg_size == 0)) { + rc = EINVAL; + goto fail1; + } + + /* Read initial chunk of the segment, starting at offset */ + if ((rc = ef10_nvram_partn_read_mode(enp, partn, seg_offset, seg_data, + EF10_NVRAM_CHUNK, + MC_CMD_NVRAM_READ_IN_V2_TARGET_CURRENT)) != 0) { + goto fail2; + } + + /* A PARTITION_HEADER tag must be the first item at the given offset */ + if ((rc = tlv_init_cursor_from_size(&cursor, (uint8_t *)seg_data, + max_seg_size)) != 0) { + rc = EFAULT; + goto fail3; + } + if (tlv_tag(&cursor) != TLV_TAG_PARTITION_HEADER) { + rc = EINVAL; + goto fail4; + } + header = (struct tlv_partition_header *)tlv_item(&cursor); + + /* Check TLV segment length (includes the END tag) */ + total_length = __LE_TO_CPU_32(header->total_length); + if (total_length > max_seg_size) { + rc = EFBIG; + goto fail5; + } + + /* Read the remaining segment content */ + if (total_length > EF10_NVRAM_CHUNK) { + if ((rc = ef10_nvram_partn_read_mode(enp, partn, + seg_offset + EF10_NVRAM_CHUNK, + seg_data + EF10_NVRAM_CHUNK, + total_length - EF10_NVRAM_CHUNK, + MC_CMD_NVRAM_READ_IN_V2_TARGET_CURRENT)) != 0) + goto fail6; + } + + /* Check segment ends with PARTITION_TRAILER and END tags */ + if ((rc = tlv_find(&cursor, TLV_TAG_PARTITION_TRAILER)) != 0) { + rc = EINVAL; + goto fail7; + } + trailer = (struct tlv_partition_trailer *)tlv_item(&cursor); + + if ((rc = tlv_advance(&cursor)) != 0) { + rc = EINVAL; + goto fail8; + } + if (tlv_tag(&cursor) != TLV_TAG_END) { + rc = EINVAL; + goto fail9; + } + + /* Check data read from segment is consistent */ + if (trailer->generation != header->generation) { + /* + * The partition data may have been modified between successive + * MCDI NVRAM_READ requests by the MC or another PCI function. + * + * The caller must retry to obtain consistent partition data. + */ + rc = EAGAIN; + goto fail10; + } + + /* Verify segment checksum */ + cksum = 0; + for (pos = 0; (size_t)pos < total_length; pos += sizeof (uint32_t)) { + cksum += *((uint32_t *)(seg_data + pos)); + } + if (cksum != 0) { + rc = EINVAL; + goto fail11; + } + + return (0); + +fail11: + EFSYS_PROBE(fail11); +fail10: + EFSYS_PROBE(fail10); +fail9: + EFSYS_PROBE(fail9); +fail8: + EFSYS_PROBE(fail8); +fail7: + EFSYS_PROBE(fail7); +fail6: + EFSYS_PROBE(fail6); +fail5: + EFSYS_PROBE(fail5); +fail4: + EFSYS_PROBE(fail4); +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +/* + * Read a single TLV item from a host memory + * buffer containing a TLV formatted segment. + */ + __checkReturn efx_rc_t +ef10_nvram_buf_read_tlv( + __in efx_nic_t *enp, + __in_bcount(max_seg_size) caddr_t seg_data, + __in size_t max_seg_size, + __in uint32_t tag, + __deref_out_bcount_opt(*sizep) caddr_t *datap, + __out size_t *sizep) +{ + tlv_cursor_t cursor; + caddr_t data; + size_t length; + caddr_t value; + efx_rc_t rc; + + if ((seg_data == NULL) || (max_seg_size == 0)) { + rc = EINVAL; + goto fail1; + } + + /* Find requested TLV tag in segment data */ + if ((rc = tlv_init_cursor_from_size(&cursor, (uint8_t *)seg_data, + max_seg_size)) != 0) { + rc = EFAULT; + goto fail2; + } + if ((rc = tlv_find(&cursor, tag)) != 0) { + rc = ENOENT; + goto fail3; + } + value = (caddr_t)tlv_value(&cursor); + length = tlv_length(&cursor); + + if (length == 0) + data = NULL; + else { + /* Copy out data from TLV item */ + EFSYS_KMEM_ALLOC(enp->en_esip, length, data); + if (data == NULL) { + rc = ENOMEM; + goto fail4; + } + memcpy(data, value, length); + } + + *datap = data; + *sizep = length; + + return (0); + +fail4: + EFSYS_PROBE(fail4); +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +/* Read a single TLV item from the first segment in a TLV formatted partition */ + __checkReturn efx_rc_t +ef10_nvram_partn_read_tlv( + __in efx_nic_t *enp, + __in uint32_t partn, + __in uint32_t tag, + __deref_out_bcount_opt(*seg_sizep) caddr_t *seg_datap, + __out size_t *seg_sizep) +{ + caddr_t seg_data = NULL; + size_t partn_size = 0; + size_t length; + caddr_t data; + int retry; + efx_rc_t rc; + + /* Allocate sufficient memory for the entire partition */ + if ((rc = ef10_nvram_partn_size(enp, partn, &partn_size)) != 0) + goto fail1; + + if (partn_size == 0) { + rc = ENOENT; + goto fail2; + } + + EFSYS_KMEM_ALLOC(enp->en_esip, partn_size, seg_data); + if (seg_data == NULL) { + rc = ENOMEM; + goto fail3; + } + + /* + * Read the first segment in a TLV partition. Retry until consistent + * segment contents are returned. Inconsistent data may be read if: + * a) the segment contents are invalid + * b) the MC has rebooted while we were reading the partition + * c) the partition has been modified while we were reading it + * Limit retry attempts to ensure forward progress. + */ + retry = 10; + do { + rc = ef10_nvram_read_tlv_segment(enp, partn, 0, + seg_data, partn_size); + } while ((rc == EAGAIN) && (--retry > 0)); + + if (rc != 0) { + /* Failed to obtain consistent segment data */ + goto fail4; + } + + if ((rc = ef10_nvram_buf_read_tlv(enp, seg_data, partn_size, + tag, &data, &length)) != 0) + goto fail5; + + EFSYS_KMEM_FREE(enp->en_esip, partn_size, seg_data); + + *seg_datap = data; + *seg_sizep = length; + + return (0); + +fail5: + EFSYS_PROBE(fail5); +fail4: + EFSYS_PROBE(fail4); + + EFSYS_KMEM_FREE(enp->en_esip, partn_size, seg_data); +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +/* Compute the size of a segment. */ + static __checkReturn efx_rc_t +ef10_nvram_buf_segment_size( + __in caddr_t seg_data, + __in size_t max_seg_size, + __out size_t *seg_sizep) +{ + efx_rc_t rc; + tlv_cursor_t cursor; + struct tlv_partition_header *header; + uint32_t cksum; + int pos; + uint32_t *end_tag_position; + uint32_t segment_length; + + /* A PARTITION_HEADER tag must be the first item at the given offset */ + if ((rc = tlv_init_cursor_from_size(&cursor, (uint8_t *)seg_data, + max_seg_size)) != 0) { + rc = EFAULT; + goto fail1; + } + if (tlv_tag(&cursor) != TLV_TAG_PARTITION_HEADER) { + rc = EINVAL; + goto fail2; + } + header = (struct tlv_partition_header *)tlv_item(&cursor); + + /* Check TLV segment length (includes the END tag) */ + *seg_sizep = __LE_TO_CPU_32(header->total_length); + if (*seg_sizep > max_seg_size) { + rc = EFBIG; + goto fail3; + } + + /* Check segment ends with PARTITION_TRAILER and END tags */ + if ((rc = tlv_find(&cursor, TLV_TAG_PARTITION_TRAILER)) != 0) { + rc = EINVAL; + goto fail4; + } + + if ((rc = tlv_advance(&cursor)) != 0) { + rc = EINVAL; + goto fail5; + } + if (tlv_tag(&cursor) != TLV_TAG_END) { + rc = EINVAL; + goto fail6; + } + end_tag_position = cursor.current; + + /* Verify segment checksum */ + cksum = 0; + for (pos = 0; (size_t)pos < *seg_sizep; pos += sizeof (uint32_t)) { + cksum += *((uint32_t *)(seg_data + pos)); + } + if (cksum != 0) { + rc = EINVAL; + goto fail7; + } + + /* + * Calculate total length from HEADER to END tags and compare to + * max_seg_size and the total_length field in the HEADER tag. + */ + segment_length = tlv_block_length_used(&cursor); + + if (segment_length > max_seg_size) { + rc = EINVAL; + goto fail8; + } + + if (segment_length != *seg_sizep) { + rc = EINVAL; + goto fail9; + } + + /* Skip over the first HEADER tag. */ + rc = tlv_rewind(&cursor); + rc = tlv_advance(&cursor); + + while (rc == 0) { + if (tlv_tag(&cursor) == TLV_TAG_END) { + /* Check that the END tag is the one found earlier. */ + if (cursor.current != end_tag_position) + goto fail10; + break; + } + /* Check for duplicate HEADER tags before the END tag. */ + if (tlv_tag(&cursor) == TLV_TAG_PARTITION_HEADER) { + rc = EINVAL; + goto fail11; + } + + rc = tlv_advance(&cursor); + } + if (rc != 0) + goto fail12; + + return (0); + +fail12: + EFSYS_PROBE(fail12); +fail11: + EFSYS_PROBE(fail11); +fail10: + EFSYS_PROBE(fail10); +fail9: + EFSYS_PROBE(fail9); +fail8: + EFSYS_PROBE(fail8); +fail7: + EFSYS_PROBE(fail7); +fail6: + EFSYS_PROBE(fail6); +fail5: + EFSYS_PROBE(fail5); +fail4: + EFSYS_PROBE(fail4); +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +/* + * Add or update a single TLV item in a host memory buffer containing a TLV + * formatted segment. Historically partitions consisted of only one segment. + */ + __checkReturn efx_rc_t +ef10_nvram_buf_write_tlv( + __inout_bcount(max_seg_size) caddr_t seg_data, + __in size_t max_seg_size, + __in uint32_t tag, + __in_bcount(tag_size) caddr_t tag_data, + __in size_t tag_size, + __out size_t *total_lengthp) +{ + tlv_cursor_t cursor; + struct tlv_partition_header *header; + struct tlv_partition_trailer *trailer; + uint32_t generation; + uint32_t cksum; + int pos; + efx_rc_t rc; + + /* A PARTITION_HEADER tag must be the first item (at offset zero) */ + if ((rc = tlv_init_cursor_from_size(&cursor, (uint8_t *)seg_data, + max_seg_size)) != 0) { + rc = EFAULT; + goto fail1; + } + if (tlv_tag(&cursor) != TLV_TAG_PARTITION_HEADER) { + rc = EINVAL; + goto fail2; + } + header = (struct tlv_partition_header *)tlv_item(&cursor); + + /* Update the TLV chain to contain the new data */ + if ((rc = tlv_find(&cursor, tag)) == 0) { + /* Modify existing TLV item */ + if ((rc = tlv_modify(&cursor, tag, + (uint8_t *)tag_data, tag_size)) != 0) + goto fail3; + } else { + /* Insert a new TLV item before the PARTITION_TRAILER */ + rc = tlv_find(&cursor, TLV_TAG_PARTITION_TRAILER); + if (rc != 0) { + rc = EINVAL; + goto fail4; + } + if ((rc = tlv_insert(&cursor, tag, + (uint8_t *)tag_data, tag_size)) != 0) { + rc = EINVAL; + goto fail5; + } + } + + /* Find the trailer tag */ + if ((rc = tlv_find(&cursor, TLV_TAG_PARTITION_TRAILER)) != 0) { + rc = EINVAL; + goto fail6; + } + trailer = (struct tlv_partition_trailer *)tlv_item(&cursor); + + /* Update PARTITION_HEADER and PARTITION_TRAILER fields */ + *total_lengthp = tlv_block_length_used(&cursor); + if (*total_lengthp > max_seg_size) { + rc = ENOSPC; + goto fail7; + } + generation = __LE_TO_CPU_32(header->generation) + 1; + + header->total_length = __CPU_TO_LE_32(*total_lengthp); + header->generation = __CPU_TO_LE_32(generation); + trailer->generation = __CPU_TO_LE_32(generation); + + /* Recompute PARTITION_TRAILER checksum */ + trailer->checksum = 0; + cksum = 0; + for (pos = 0; (size_t)pos < *total_lengthp; pos += sizeof (uint32_t)) { + cksum += *((uint32_t *)(seg_data + pos)); + } + trailer->checksum = ~cksum + 1; + + return (0); + +fail7: + EFSYS_PROBE(fail7); +fail6: + EFSYS_PROBE(fail6); +fail5: + EFSYS_PROBE(fail5); +fail4: + EFSYS_PROBE(fail4); +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +/* + * Add or update a single TLV item in the first segment of a TLV formatted + * dynamic config partition. The first segment is the current active + * configuration. + */ + __checkReturn efx_rc_t +ef10_nvram_partn_write_tlv( + __in efx_nic_t *enp, + __in uint32_t partn, + __in uint32_t tag, + __in_bcount(size) caddr_t data, + __in size_t size) +{ + return ef10_nvram_partn_write_segment_tlv(enp, partn, tag, data, + size, B_FALSE); +} + +/* + * Read a segment from nvram at the given offset into a buffer (segment_data) + * and optionally write a new tag to it. + */ +static __checkReturn efx_rc_t +ef10_nvram_segment_write_tlv( + __in efx_nic_t *enp, + __in uint32_t partn, + __in uint32_t tag, + __in_bcount(size) caddr_t data, + __in size_t size, + __inout caddr_t *seg_datap, + __inout size_t *partn_offsetp, + __inout size_t *src_remain_lenp, + __inout size_t *dest_remain_lenp, + __in boolean_t write) +{ + efx_rc_t rc; + efx_rc_t status; + size_t original_segment_size; + size_t modified_segment_size; + + /* + * Read the segment from NVRAM into the segment_data buffer and validate + * it, returning if it does not validate. This is not a failure unless + * this is the first segment in a partition. In this case the caller + * must propagate the error. + */ + status = ef10_nvram_read_tlv_segment(enp, partn, *partn_offsetp, + *seg_datap, *src_remain_lenp); + if (status != 0) { + rc = EINVAL; + goto fail1; + } + + status = ef10_nvram_buf_segment_size(*seg_datap, + *src_remain_lenp, &original_segment_size); + if (status != 0) { + rc = EINVAL; + goto fail2; + } + + if (write) { + /* Update the contents of the segment in the buffer */ + if ((rc = ef10_nvram_buf_write_tlv(*seg_datap, + *dest_remain_lenp, tag, data, size, + &modified_segment_size)) != 0) { + goto fail3; + } + *dest_remain_lenp -= modified_segment_size; + *seg_datap += modified_segment_size; + } else { + /* + * We won't modify this segment, but still need to update the + * remaining lengths and pointers. + */ + *dest_remain_lenp -= original_segment_size; + *seg_datap += original_segment_size; + } + + *partn_offsetp += original_segment_size; + *src_remain_lenp -= original_segment_size; + + return (0); + +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +/* + * Add or update a single TLV item in either the first segment or in all + * segments in a TLV formatted dynamic config partition. Dynamic config + * partitions on boards that support RFID are divided into a number of segments, + * each formatted like a partition, with header, trailer and end tags. The first + * segment is the current active configuration. + * + * The segments are initialised by manftest and each contain a different + * configuration e.g. firmware variant. The firmware can be instructed + * via RFID to copy a segment to replace the first segment, hence changing the + * active configuration. This allows ops to change the configuration of a board + * prior to shipment using RFID. + * + * Changes to the dynamic config may need to be written to all segments (e.g. + * firmware versions) or just the first segment (changes to the active + * configuration). See SF-111324-SW "The use of RFID in Solarflare Products". + * If only the first segment is written the code still needs to be aware of the + * possible presence of subsequent segments as writing to a segment may cause + * its size to increase, which would overwrite the subsequent segments and + * invalidate them. + */ + __checkReturn efx_rc_t +ef10_nvram_partn_write_segment_tlv( + __in efx_nic_t *enp, + __in uint32_t partn, + __in uint32_t tag, + __in_bcount(size) caddr_t data, + __in size_t size, + __in boolean_t all_segments) +{ + size_t partn_size = 0; + caddr_t partn_data; + size_t total_length = 0; + efx_rc_t rc; + size_t current_offset = 0; + size_t remaining_original_length; + size_t remaining_modified_length; + caddr_t segment_data; + + EFSYS_ASSERT3U(partn, ==, NVRAM_PARTITION_TYPE_DYNAMIC_CONFIG); + + /* Allocate sufficient memory for the entire partition */ + if ((rc = ef10_nvram_partn_size(enp, partn, &partn_size)) != 0) + goto fail1; + + EFSYS_KMEM_ALLOC(enp->en_esip, partn_size, partn_data); + if (partn_data == NULL) { + rc = ENOMEM; + goto fail2; + } + + remaining_original_length = partn_size; + remaining_modified_length = partn_size; + segment_data = partn_data; + + /* Lock the partition */ + if ((rc = ef10_nvram_partn_lock(enp, partn)) != 0) + goto fail3; + + /* Iterate over each (potential) segment to update it. */ + do { + boolean_t write = all_segments || current_offset == 0; + + rc = ef10_nvram_segment_write_tlv(enp, partn, tag, data, size, + &segment_data, ¤t_offset, &remaining_original_length, + &remaining_modified_length, write); + if (rc != 0) { + if (current_offset == 0) { + /* + * If no data has been read then the first + * segment is invalid, which is an error. + */ + goto fail4; + } + break; + } + } while (current_offset < partn_size); + + total_length = segment_data - partn_data; + + /* + * We've run out of space. This should actually be dealt with by + * ef10_nvram_buf_write_tlv returning ENOSPC. + */ + if (total_length > partn_size) { + rc = ENOSPC; + goto fail5; + } + + /* Erase the whole partition in NVRAM */ + if ((rc = ef10_nvram_partn_erase(enp, partn, 0, partn_size)) != 0) + goto fail6; + + /* Write new partition contents from the buffer to NVRAM */ + if ((rc = ef10_nvram_partn_write(enp, partn, 0, partn_data, + total_length)) != 0) + goto fail7; + + /* Unlock the partition */ + ef10_nvram_partn_unlock(enp, partn, NULL); + + EFSYS_KMEM_FREE(enp->en_esip, partn_size, partn_data); + + return (0); + +fail7: + EFSYS_PROBE(fail7); +fail6: + EFSYS_PROBE(fail6); +fail5: + EFSYS_PROBE(fail5); +fail4: + EFSYS_PROBE(fail4); + + ef10_nvram_partn_unlock(enp, partn, NULL); +fail3: + EFSYS_PROBE(fail3); + + EFSYS_KMEM_FREE(enp->en_esip, partn_size, partn_data); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +/* + * Get the size of a NVRAM partition. This is the total size allocated in nvram, + * not the data used by the segments in the partition. + */ + __checkReturn efx_rc_t +ef10_nvram_partn_size( + __in efx_nic_t *enp, + __in uint32_t partn, + __out size_t *sizep) +{ + efx_rc_t rc; + + if ((rc = efx_mcdi_nvram_info(enp, partn, sizep, + NULL, NULL, NULL)) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +ef10_nvram_partn_lock( + __in efx_nic_t *enp, + __in uint32_t partn) +{ + efx_rc_t rc; + + if ((rc = efx_mcdi_nvram_update_start(enp, partn)) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +ef10_nvram_partn_read_mode( + __in efx_nic_t *enp, + __in uint32_t partn, + __in unsigned int offset, + __out_bcount(size) caddr_t data, + __in size_t size, + __in uint32_t mode) +{ + size_t chunk; + efx_rc_t rc; + + while (size > 0) { + chunk = MIN(size, EF10_NVRAM_CHUNK); + + if ((rc = efx_mcdi_nvram_read(enp, partn, offset, + data, chunk, mode)) != 0) { + goto fail1; + } + + size -= chunk; + data += chunk; + offset += chunk; + } + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +ef10_nvram_partn_read( + __in efx_nic_t *enp, + __in uint32_t partn, + __in unsigned int offset, + __out_bcount(size) caddr_t data, + __in size_t size) +{ + /* + * Read requests which come in through the EFX API expect to + * read the current, active partition. + */ + return ef10_nvram_partn_read_mode(enp, partn, offset, data, size, + MC_CMD_NVRAM_READ_IN_V2_TARGET_CURRENT); +} + + __checkReturn efx_rc_t +ef10_nvram_partn_erase( + __in efx_nic_t *enp, + __in uint32_t partn, + __in unsigned int offset, + __in size_t size) +{ + efx_rc_t rc; + uint32_t erase_size; + + if ((rc = efx_mcdi_nvram_info(enp, partn, NULL, NULL, + &erase_size, NULL)) != 0) + goto fail1; + + if (erase_size == 0) { + if ((rc = efx_mcdi_nvram_erase(enp, partn, offset, size)) != 0) + goto fail2; + } else { + if (size % erase_size != 0) { + rc = EINVAL; + goto fail3; + } + while (size > 0) { + if ((rc = efx_mcdi_nvram_erase(enp, partn, offset, + erase_size)) != 0) + goto fail4; + offset += erase_size; + size -= erase_size; + } + } + + return (0); + +fail4: + EFSYS_PROBE(fail4); +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +ef10_nvram_partn_write( + __in efx_nic_t *enp, + __in uint32_t partn, + __in unsigned int offset, + __out_bcount(size) caddr_t data, + __in size_t size) +{ + size_t chunk; + uint32_t write_size; + efx_rc_t rc; + + if ((rc = efx_mcdi_nvram_info(enp, partn, NULL, NULL, + NULL, &write_size)) != 0) + goto fail1; + + if (write_size != 0) { + /* + * Check that the size is a multiple of the write chunk size if + * the write chunk size is available. + */ + if (size % write_size != 0) { + rc = EINVAL; + goto fail2; + } + } else { + write_size = EF10_NVRAM_CHUNK; + } + + while (size > 0) { + chunk = MIN(size, write_size); + + if ((rc = efx_mcdi_nvram_write(enp, partn, offset, + data, chunk)) != 0) { + goto fail3; + } + + size -= chunk; + data += chunk; + offset += chunk; + } + + return (0); + +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +ef10_nvram_partn_unlock( + __in efx_nic_t *enp, + __in uint32_t partn, + __out_opt uint32_t *resultp) +{ + boolean_t reboot = B_FALSE; + efx_rc_t rc; + + if (resultp != NULL) + *resultp = MC_CMD_NVRAM_VERIFY_RC_UNKNOWN; + + rc = efx_mcdi_nvram_update_finish(enp, partn, reboot, resultp); + if (rc != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +ef10_nvram_partn_set_version( + __in efx_nic_t *enp, + __in uint32_t partn, + __in_ecount(4) uint16_t version[4]) +{ + struct tlv_partition_version partn_version; + size_t size; + efx_rc_t rc; + + /* Add or modify partition version TLV item */ + partn_version.version_w = __CPU_TO_LE_16(version[0]); + partn_version.version_x = __CPU_TO_LE_16(version[1]); + partn_version.version_y = __CPU_TO_LE_16(version[2]); + partn_version.version_z = __CPU_TO_LE_16(version[3]); + + size = sizeof (partn_version) - (2 * sizeof (uint32_t)); + + /* Write the version number to all segments in the partition */ + if ((rc = ef10_nvram_partn_write_segment_tlv(enp, + NVRAM_PARTITION_TYPE_DYNAMIC_CONFIG, + TLV_TAG_PARTITION_VERSION(partn), + (caddr_t)&partn_version.version_w, size, B_TRUE)) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +#endif /* EFSYS_OPT_VPD || EFSYS_OPT_NVRAM */ + +#if EFSYS_OPT_NVRAM + +typedef struct ef10_parttbl_entry_s { + unsigned int partn; + unsigned int port; + efx_nvram_type_t nvtype; +} ef10_parttbl_entry_t; + +/* Translate EFX NVRAM types to firmware partition types */ +static ef10_parttbl_entry_t hunt_parttbl[] = { + {NVRAM_PARTITION_TYPE_MC_FIRMWARE, 1, EFX_NVRAM_MC_FIRMWARE}, + {NVRAM_PARTITION_TYPE_MC_FIRMWARE, 2, EFX_NVRAM_MC_FIRMWARE}, + {NVRAM_PARTITION_TYPE_MC_FIRMWARE, 3, EFX_NVRAM_MC_FIRMWARE}, + {NVRAM_PARTITION_TYPE_MC_FIRMWARE, 4, EFX_NVRAM_MC_FIRMWARE}, + {NVRAM_PARTITION_TYPE_MC_FIRMWARE_BACKUP, 1, EFX_NVRAM_MC_GOLDEN}, + {NVRAM_PARTITION_TYPE_MC_FIRMWARE_BACKUP, 2, EFX_NVRAM_MC_GOLDEN}, + {NVRAM_PARTITION_TYPE_MC_FIRMWARE_BACKUP, 3, EFX_NVRAM_MC_GOLDEN}, + {NVRAM_PARTITION_TYPE_MC_FIRMWARE_BACKUP, 4, EFX_NVRAM_MC_GOLDEN}, + {NVRAM_PARTITION_TYPE_EXPANSION_ROM, 1, EFX_NVRAM_BOOTROM}, + {NVRAM_PARTITION_TYPE_EXPANSION_ROM, 2, EFX_NVRAM_BOOTROM}, + {NVRAM_PARTITION_TYPE_EXPANSION_ROM, 3, EFX_NVRAM_BOOTROM}, + {NVRAM_PARTITION_TYPE_EXPANSION_ROM, 4, EFX_NVRAM_BOOTROM}, + {NVRAM_PARTITION_TYPE_EXPROM_CONFIG_PORT0, 1, EFX_NVRAM_BOOTROM_CFG}, + {NVRAM_PARTITION_TYPE_EXPROM_CONFIG_PORT1, 2, EFX_NVRAM_BOOTROM_CFG}, + {NVRAM_PARTITION_TYPE_EXPROM_CONFIG_PORT2, 3, EFX_NVRAM_BOOTROM_CFG}, + {NVRAM_PARTITION_TYPE_EXPROM_CONFIG_PORT3, 4, EFX_NVRAM_BOOTROM_CFG}, + {NVRAM_PARTITION_TYPE_DYNAMIC_CONFIG, 1, EFX_NVRAM_DYNAMIC_CFG}, + {NVRAM_PARTITION_TYPE_DYNAMIC_CONFIG, 2, EFX_NVRAM_DYNAMIC_CFG}, + {NVRAM_PARTITION_TYPE_DYNAMIC_CONFIG, 3, EFX_NVRAM_DYNAMIC_CFG}, + {NVRAM_PARTITION_TYPE_DYNAMIC_CONFIG, 4, EFX_NVRAM_DYNAMIC_CFG}, + {NVRAM_PARTITION_TYPE_FPGA, 1, EFX_NVRAM_FPGA}, + {NVRAM_PARTITION_TYPE_FPGA, 2, EFX_NVRAM_FPGA}, + {NVRAM_PARTITION_TYPE_FPGA, 3, EFX_NVRAM_FPGA}, + {NVRAM_PARTITION_TYPE_FPGA, 4, EFX_NVRAM_FPGA}, + {NVRAM_PARTITION_TYPE_FPGA_BACKUP, 1, EFX_NVRAM_FPGA_BACKUP}, + {NVRAM_PARTITION_TYPE_FPGA_BACKUP, 2, EFX_NVRAM_FPGA_BACKUP}, + {NVRAM_PARTITION_TYPE_FPGA_BACKUP, 3, EFX_NVRAM_FPGA_BACKUP}, + {NVRAM_PARTITION_TYPE_FPGA_BACKUP, 4, EFX_NVRAM_FPGA_BACKUP}, + {NVRAM_PARTITION_TYPE_LICENSE, 1, EFX_NVRAM_LICENSE}, + {NVRAM_PARTITION_TYPE_LICENSE, 2, EFX_NVRAM_LICENSE}, + {NVRAM_PARTITION_TYPE_LICENSE, 3, EFX_NVRAM_LICENSE}, + {NVRAM_PARTITION_TYPE_LICENSE, 4, EFX_NVRAM_LICENSE} +}; + +static ef10_parttbl_entry_t medford_parttbl[] = { + {NVRAM_PARTITION_TYPE_MC_FIRMWARE, 1, EFX_NVRAM_MC_FIRMWARE}, + {NVRAM_PARTITION_TYPE_MC_FIRMWARE, 2, EFX_NVRAM_MC_FIRMWARE}, + {NVRAM_PARTITION_TYPE_MC_FIRMWARE, 3, EFX_NVRAM_MC_FIRMWARE}, + {NVRAM_PARTITION_TYPE_MC_FIRMWARE, 4, EFX_NVRAM_MC_FIRMWARE}, + {NVRAM_PARTITION_TYPE_MC_FIRMWARE_BACKUP, 1, EFX_NVRAM_MC_GOLDEN}, + {NVRAM_PARTITION_TYPE_MC_FIRMWARE_BACKUP, 2, EFX_NVRAM_MC_GOLDEN}, + {NVRAM_PARTITION_TYPE_MC_FIRMWARE_BACKUP, 3, EFX_NVRAM_MC_GOLDEN}, + {NVRAM_PARTITION_TYPE_MC_FIRMWARE_BACKUP, 4, EFX_NVRAM_MC_GOLDEN}, + {NVRAM_PARTITION_TYPE_EXPANSION_ROM, 1, EFX_NVRAM_BOOTROM}, + {NVRAM_PARTITION_TYPE_EXPANSION_ROM, 2, EFX_NVRAM_BOOTROM}, + {NVRAM_PARTITION_TYPE_EXPANSION_ROM, 3, EFX_NVRAM_BOOTROM}, + {NVRAM_PARTITION_TYPE_EXPANSION_ROM, 4, EFX_NVRAM_BOOTROM}, + {NVRAM_PARTITION_TYPE_EXPROM_CONFIG_PORT0, 1, EFX_NVRAM_BOOTROM_CFG}, + {NVRAM_PARTITION_TYPE_EXPROM_CONFIG_PORT0, 2, EFX_NVRAM_BOOTROM_CFG}, + {NVRAM_PARTITION_TYPE_EXPROM_CONFIG_PORT0, 3, EFX_NVRAM_BOOTROM_CFG}, + {NVRAM_PARTITION_TYPE_EXPROM_CONFIG_PORT0, 4, EFX_NVRAM_BOOTROM_CFG}, + {NVRAM_PARTITION_TYPE_DYNAMIC_CONFIG, 1, EFX_NVRAM_DYNAMIC_CFG}, + {NVRAM_PARTITION_TYPE_DYNAMIC_CONFIG, 2, EFX_NVRAM_DYNAMIC_CFG}, + {NVRAM_PARTITION_TYPE_DYNAMIC_CONFIG, 3, EFX_NVRAM_DYNAMIC_CFG}, + {NVRAM_PARTITION_TYPE_DYNAMIC_CONFIG, 4, EFX_NVRAM_DYNAMIC_CFG}, + {NVRAM_PARTITION_TYPE_FPGA, 1, EFX_NVRAM_FPGA}, + {NVRAM_PARTITION_TYPE_FPGA, 2, EFX_NVRAM_FPGA}, + {NVRAM_PARTITION_TYPE_FPGA, 3, EFX_NVRAM_FPGA}, + {NVRAM_PARTITION_TYPE_FPGA, 4, EFX_NVRAM_FPGA}, + {NVRAM_PARTITION_TYPE_FPGA_BACKUP, 1, EFX_NVRAM_FPGA_BACKUP}, + {NVRAM_PARTITION_TYPE_FPGA_BACKUP, 2, EFX_NVRAM_FPGA_BACKUP}, + {NVRAM_PARTITION_TYPE_FPGA_BACKUP, 3, EFX_NVRAM_FPGA_BACKUP}, + {NVRAM_PARTITION_TYPE_FPGA_BACKUP, 4, EFX_NVRAM_FPGA_BACKUP}, + {NVRAM_PARTITION_TYPE_LICENSE, 1, EFX_NVRAM_LICENSE}, + {NVRAM_PARTITION_TYPE_LICENSE, 2, EFX_NVRAM_LICENSE}, + {NVRAM_PARTITION_TYPE_LICENSE, 3, EFX_NVRAM_LICENSE}, + {NVRAM_PARTITION_TYPE_LICENSE, 4, EFX_NVRAM_LICENSE}, + {NVRAM_PARTITION_TYPE_EXPANSION_UEFI, 1, EFX_NVRAM_UEFIROM}, + {NVRAM_PARTITION_TYPE_EXPANSION_UEFI, 2, EFX_NVRAM_UEFIROM}, + {NVRAM_PARTITION_TYPE_EXPANSION_UEFI, 3, EFX_NVRAM_UEFIROM}, + {NVRAM_PARTITION_TYPE_EXPANSION_UEFI, 4, EFX_NVRAM_UEFIROM} +}; + +static __checkReturn efx_rc_t +ef10_parttbl_get( + __in efx_nic_t *enp, + __out ef10_parttbl_entry_t **parttblp, + __out size_t *parttbl_rowsp) +{ + switch (enp->en_family) { + case EFX_FAMILY_HUNTINGTON: + *parttblp = hunt_parttbl; + *parttbl_rowsp = EFX_ARRAY_SIZE(hunt_parttbl); + break; + + case EFX_FAMILY_MEDFORD: + *parttblp = medford_parttbl; + *parttbl_rowsp = EFX_ARRAY_SIZE(medford_parttbl); + break; + + default: + EFSYS_ASSERT(B_FALSE); + return (EINVAL); + } + return (0); +} + + __checkReturn efx_rc_t +ef10_nvram_type_to_partn( + __in efx_nic_t *enp, + __in efx_nvram_type_t type, + __out uint32_t *partnp) +{ + efx_mcdi_iface_t *emip = &(enp->en_mcdi.em_emip); + ef10_parttbl_entry_t *parttbl = NULL; + size_t parttbl_rows = 0; + unsigned int i; + + EFSYS_ASSERT3U(type, <, EFX_NVRAM_NTYPES); + EFSYS_ASSERT(partnp != NULL); + + if (ef10_parttbl_get(enp, &parttbl, &parttbl_rows) == 0) { + for (i = 0; i < parttbl_rows; i++) { + ef10_parttbl_entry_t *entry = &parttbl[i]; + + if (entry->nvtype == type && + entry->port == emip->emi_port) { + *partnp = entry->partn; + return (0); + } + } + } + + return (ENOTSUP); +} + +#if EFSYS_OPT_DIAG + +static __checkReturn efx_rc_t +ef10_nvram_partn_to_type( + __in efx_nic_t *enp, + __in uint32_t partn, + __out efx_nvram_type_t *typep) +{ + efx_mcdi_iface_t *emip = &(enp->en_mcdi.em_emip); + ef10_parttbl_entry_t *parttbl = NULL; + size_t parttbl_rows = 0; + unsigned int i; + + EFSYS_ASSERT(typep != NULL); + + if (ef10_parttbl_get(enp, &parttbl, &parttbl_rows) == 0) { + for (i = 0; i < parttbl_rows; i++) { + ef10_parttbl_entry_t *entry = &parttbl[i]; + + if (entry->partn == partn && + entry->port == emip->emi_port) { + *typep = entry->nvtype; + return (0); + } + } + } + + return (ENOTSUP); +} + + __checkReturn efx_rc_t +ef10_nvram_test( + __in efx_nic_t *enp) +{ + efx_nvram_type_t type; + unsigned int npartns = 0; + uint32_t *partns = NULL; + size_t size; + unsigned int i; + efx_rc_t rc; + + /* Read available partitions from NVRAM partition map */ + size = MC_CMD_NVRAM_PARTITIONS_OUT_TYPE_ID_MAXNUM * sizeof (uint32_t); + EFSYS_KMEM_ALLOC(enp->en_esip, size, partns); + if (partns == NULL) { + rc = ENOMEM; + goto fail1; + } + + if ((rc = efx_mcdi_nvram_partitions(enp, (caddr_t)partns, size, + &npartns)) != 0) { + goto fail2; + } + + for (i = 0; i < npartns; i++) { + /* Check if the partition is supported for this port */ + if ((rc = ef10_nvram_partn_to_type(enp, partns[i], &type)) != 0) + continue; + + if ((rc = efx_mcdi_nvram_test(enp, partns[i])) != 0) + goto fail3; + } + + EFSYS_KMEM_FREE(enp->en_esip, size, partns); + return (0); + +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); + EFSYS_KMEM_FREE(enp->en_esip, size, partns); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + return (rc); +} + +#endif /* EFSYS_OPT_DIAG */ + + __checkReturn efx_rc_t +ef10_nvram_partn_get_version( + __in efx_nic_t *enp, + __in uint32_t partn, + __out uint32_t *subtypep, + __out_ecount(4) uint16_t version[4]) +{ + efx_rc_t rc; + + /* FIXME: get highest partn version from all ports */ + /* FIXME: return partn description if available */ + + if ((rc = efx_mcdi_nvram_metadata(enp, partn, subtypep, + version, NULL, 0)) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +ef10_nvram_partn_rw_start( + __in efx_nic_t *enp, + __in uint32_t partn, + __out size_t *chunk_sizep) +{ + efx_rc_t rc; + + if ((rc = ef10_nvram_partn_lock(enp, partn)) != 0) + goto fail1; + + if (chunk_sizep != NULL) + *chunk_sizep = EF10_NVRAM_CHUNK; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +ef10_nvram_partn_rw_finish( + __in efx_nic_t *enp, + __in uint32_t partn) +{ + efx_rc_t rc; + + if ((rc = ef10_nvram_partn_unlock(enp, partn, NULL)) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +#endif /* EFSYS_OPT_NVRAM */ + +#endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */ diff --git a/dpdk/drivers/net/sfc/base/ef10_phy.c b/dpdk/drivers/net/sfc/base/ef10_phy.c new file mode 100644 index 00000000..81309f29 --- /dev/null +++ b/dpdk/drivers/net/sfc/base/ef10_phy.c @@ -0,0 +1,631 @@ +/* + * Copyright (c) 2012-2016 Solarflare Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are + * those of the authors and should not be interpreted as representing official + * policies, either expressed or implied, of the FreeBSD Project. + */ + +#include "efx.h" +#include "efx_impl.h" + +#if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD + +static void +mcdi_phy_decode_cap( + __in uint32_t mcdi_cap, + __out uint32_t *maskp) +{ + uint32_t mask; + + mask = 0; + if (mcdi_cap & (1 << MC_CMD_PHY_CAP_10HDX_LBN)) + mask |= (1 << EFX_PHY_CAP_10HDX); + if (mcdi_cap & (1 << MC_CMD_PHY_CAP_10FDX_LBN)) + mask |= (1 << EFX_PHY_CAP_10FDX); + if (mcdi_cap & (1 << MC_CMD_PHY_CAP_100HDX_LBN)) + mask |= (1 << EFX_PHY_CAP_100HDX); + if (mcdi_cap & (1 << MC_CMD_PHY_CAP_100FDX_LBN)) + mask |= (1 << EFX_PHY_CAP_100FDX); + if (mcdi_cap & (1 << MC_CMD_PHY_CAP_1000HDX_LBN)) + mask |= (1 << EFX_PHY_CAP_1000HDX); + if (mcdi_cap & (1 << MC_CMD_PHY_CAP_1000FDX_LBN)) + mask |= (1 << EFX_PHY_CAP_1000FDX); + if (mcdi_cap & (1 << MC_CMD_PHY_CAP_10000FDX_LBN)) + mask |= (1 << EFX_PHY_CAP_10000FDX); + if (mcdi_cap & (1 << MC_CMD_PHY_CAP_40000FDX_LBN)) + mask |= (1 << EFX_PHY_CAP_40000FDX); + if (mcdi_cap & (1 << MC_CMD_PHY_CAP_PAUSE_LBN)) + mask |= (1 << EFX_PHY_CAP_PAUSE); + if (mcdi_cap & (1 << MC_CMD_PHY_CAP_ASYM_LBN)) + mask |= (1 << EFX_PHY_CAP_ASYM); + if (mcdi_cap & (1 << MC_CMD_PHY_CAP_AN_LBN)) + mask |= (1 << EFX_PHY_CAP_AN); + + *maskp = mask; +} + +static void +mcdi_phy_decode_link_mode( + __in efx_nic_t *enp, + __in uint32_t link_flags, + __in unsigned int speed, + __in unsigned int fcntl, + __out efx_link_mode_t *link_modep, + __out unsigned int *fcntlp) +{ + boolean_t fd = !!(link_flags & + (1 << MC_CMD_GET_LINK_OUT_FULL_DUPLEX_LBN)); + boolean_t up = !!(link_flags & + (1 << MC_CMD_GET_LINK_OUT_LINK_UP_LBN)); + + _NOTE(ARGUNUSED(enp)) + + if (!up) + *link_modep = EFX_LINK_DOWN; + else if (speed == 40000 && fd) + *link_modep = EFX_LINK_40000FDX; + else if (speed == 10000 && fd) + *link_modep = EFX_LINK_10000FDX; + else if (speed == 1000) + *link_modep = fd ? EFX_LINK_1000FDX : EFX_LINK_1000HDX; + else if (speed == 100) + *link_modep = fd ? EFX_LINK_100FDX : EFX_LINK_100HDX; + else if (speed == 10) + *link_modep = fd ? EFX_LINK_10FDX : EFX_LINK_10HDX; + else + *link_modep = EFX_LINK_UNKNOWN; + + if (fcntl == MC_CMD_FCNTL_OFF) + *fcntlp = 0; + else if (fcntl == MC_CMD_FCNTL_RESPOND) + *fcntlp = EFX_FCNTL_RESPOND; + else if (fcntl == MC_CMD_FCNTL_GENERATE) + *fcntlp = EFX_FCNTL_GENERATE; + else if (fcntl == MC_CMD_FCNTL_BIDIR) + *fcntlp = EFX_FCNTL_RESPOND | EFX_FCNTL_GENERATE; + else { + EFSYS_PROBE1(mc_pcol_error, int, fcntl); + *fcntlp = 0; + } +} + + + void +ef10_phy_link_ev( + __in efx_nic_t *enp, + __in efx_qword_t *eqp, + __out efx_link_mode_t *link_modep) +{ + efx_port_t *epp = &(enp->en_port); + unsigned int link_flags; + unsigned int speed; + unsigned int fcntl; + efx_link_mode_t link_mode; + uint32_t lp_cap_mask; + + /* + * Convert the LINKCHANGE speed enumeration into mbit/s, in the + * same way as GET_LINK encodes the speed + */ + switch (MCDI_EV_FIELD(eqp, LINKCHANGE_SPEED)) { + case MCDI_EVENT_LINKCHANGE_SPEED_100M: + speed = 100; + break; + case MCDI_EVENT_LINKCHANGE_SPEED_1G: + speed = 1000; + break; + case MCDI_EVENT_LINKCHANGE_SPEED_10G: + speed = 10000; + break; + case MCDI_EVENT_LINKCHANGE_SPEED_40G: + speed = 40000; + break; + default: + speed = 0; + break; + } + + link_flags = MCDI_EV_FIELD(eqp, LINKCHANGE_LINK_FLAGS); + mcdi_phy_decode_link_mode(enp, link_flags, speed, + MCDI_EV_FIELD(eqp, LINKCHANGE_FCNTL), + &link_mode, &fcntl); + mcdi_phy_decode_cap(MCDI_EV_FIELD(eqp, LINKCHANGE_LP_CAP), + &lp_cap_mask); + + /* + * It's safe to update ep_lp_cap_mask without the driver's port lock + * because presumably any concurrently running efx_port_poll() is + * only going to arrive at the same value. + * + * ep_fcntl has two meanings. It's either the link common fcntl + * (if the PHY supports AN), or it's the forced link state. If + * the former, it's safe to update the value for the same reason as + * for ep_lp_cap_mask. If the latter, then just ignore the value, + * because we can race with efx_mac_fcntl_set(). + */ + epp->ep_lp_cap_mask = lp_cap_mask; + epp->ep_fcntl = fcntl; + + *link_modep = link_mode; +} + + __checkReturn efx_rc_t +ef10_phy_power( + __in efx_nic_t *enp, + __in boolean_t power) +{ + efx_rc_t rc; + + if (!power) + return (0); + + /* Check if the PHY is a zombie */ + if ((rc = ef10_phy_verify(enp)) != 0) + goto fail1; + + enp->en_reset_flags |= EFX_RESET_PHY; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +ef10_phy_get_link( + __in efx_nic_t *enp, + __out ef10_link_state_t *elsp) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_GET_LINK_IN_LEN, + MC_CMD_GET_LINK_OUT_LEN)]; + efx_rc_t rc; + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_GET_LINK; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_GET_LINK_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_GET_LINK_OUT_LEN; + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + if (req.emr_out_length_used < MC_CMD_GET_LINK_OUT_LEN) { + rc = EMSGSIZE; + goto fail2; + } + + mcdi_phy_decode_cap(MCDI_OUT_DWORD(req, GET_LINK_OUT_CAP), + &elsp->els_adv_cap_mask); + mcdi_phy_decode_cap(MCDI_OUT_DWORD(req, GET_LINK_OUT_LP_CAP), + &elsp->els_lp_cap_mask); + + mcdi_phy_decode_link_mode(enp, MCDI_OUT_DWORD(req, GET_LINK_OUT_FLAGS), + MCDI_OUT_DWORD(req, GET_LINK_OUT_LINK_SPEED), + MCDI_OUT_DWORD(req, GET_LINK_OUT_FCNTL), + &elsp->els_link_mode, &elsp->els_fcntl); + +#if EFSYS_OPT_LOOPBACK + /* Assert the MC_CMD_LOOPBACK and EFX_LOOPBACK namespace agree */ + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_NONE == EFX_LOOPBACK_OFF); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_DATA == EFX_LOOPBACK_DATA); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_GMAC == EFX_LOOPBACK_GMAC); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_XGMII == EFX_LOOPBACK_XGMII); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_XGXS == EFX_LOOPBACK_XGXS); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_XAUI == EFX_LOOPBACK_XAUI); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_GMII == EFX_LOOPBACK_GMII); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_SGMII == EFX_LOOPBACK_SGMII); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_XGBR == EFX_LOOPBACK_XGBR); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_XFI == EFX_LOOPBACK_XFI); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_XAUI_FAR == EFX_LOOPBACK_XAUI_FAR); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_GMII_FAR == EFX_LOOPBACK_GMII_FAR); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_SGMII_FAR == EFX_LOOPBACK_SGMII_FAR); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_XFI_FAR == EFX_LOOPBACK_XFI_FAR); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_GPHY == EFX_LOOPBACK_GPHY); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_PHYXS == EFX_LOOPBACK_PHY_XS); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_PCS == EFX_LOOPBACK_PCS); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_PMAPMD == EFX_LOOPBACK_PMA_PMD); + + elsp->els_loopback = MCDI_OUT_DWORD(req, GET_LINK_OUT_LOOPBACK_MODE); +#endif /* EFSYS_OPT_LOOPBACK */ + + elsp->els_mac_up = MCDI_OUT_DWORD(req, GET_LINK_OUT_MAC_FAULT) == 0; + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +ef10_phy_reconfigure( + __in efx_nic_t *enp) +{ + efx_port_t *epp = &(enp->en_port); + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_SET_LINK_IN_LEN, + MC_CMD_SET_LINK_OUT_LEN)]; + uint32_t cap_mask; + unsigned int led_mode; + unsigned int speed; + boolean_t supported; + efx_rc_t rc; + + if ((rc = efx_mcdi_link_control_supported(enp, &supported)) != 0) + goto fail1; + if (supported == B_FALSE) + goto out; + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_SET_LINK; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_SET_LINK_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_SET_LINK_OUT_LEN; + + cap_mask = epp->ep_adv_cap_mask; + MCDI_IN_POPULATE_DWORD_10(req, SET_LINK_IN_CAP, + PHY_CAP_10HDX, (cap_mask >> EFX_PHY_CAP_10HDX) & 0x1, + PHY_CAP_10FDX, (cap_mask >> EFX_PHY_CAP_10FDX) & 0x1, + PHY_CAP_100HDX, (cap_mask >> EFX_PHY_CAP_100HDX) & 0x1, + PHY_CAP_100FDX, (cap_mask >> EFX_PHY_CAP_100FDX) & 0x1, + PHY_CAP_1000HDX, (cap_mask >> EFX_PHY_CAP_1000HDX) & 0x1, + PHY_CAP_1000FDX, (cap_mask >> EFX_PHY_CAP_1000FDX) & 0x1, + PHY_CAP_10000FDX, (cap_mask >> EFX_PHY_CAP_10000FDX) & 0x1, + PHY_CAP_PAUSE, (cap_mask >> EFX_PHY_CAP_PAUSE) & 0x1, + PHY_CAP_ASYM, (cap_mask >> EFX_PHY_CAP_ASYM) & 0x1, + PHY_CAP_AN, (cap_mask >> EFX_PHY_CAP_AN) & 0x1); + /* Too many fields for for POPULATE macros, so insert this afterwards */ + MCDI_IN_SET_DWORD_FIELD(req, SET_LINK_IN_CAP, + PHY_CAP_40000FDX, (cap_mask >> EFX_PHY_CAP_40000FDX) & 0x1); + +#if EFSYS_OPT_LOOPBACK + MCDI_IN_SET_DWORD(req, SET_LINK_IN_LOOPBACK_MODE, + epp->ep_loopback_type); + switch (epp->ep_loopback_link_mode) { + case EFX_LINK_100FDX: + speed = 100; + break; + case EFX_LINK_1000FDX: + speed = 1000; + break; + case EFX_LINK_10000FDX: + speed = 10000; + break; + case EFX_LINK_40000FDX: + speed = 40000; + break; + default: + speed = 0; + } +#else + MCDI_IN_SET_DWORD(req, SET_LINK_IN_LOOPBACK_MODE, MC_CMD_LOOPBACK_NONE); + speed = 0; +#endif /* EFSYS_OPT_LOOPBACK */ + MCDI_IN_SET_DWORD(req, SET_LINK_IN_LOOPBACK_SPEED, speed); + +#if EFSYS_OPT_PHY_FLAGS + MCDI_IN_SET_DWORD(req, SET_LINK_IN_FLAGS, epp->ep_phy_flags); +#else + MCDI_IN_SET_DWORD(req, SET_LINK_IN_FLAGS, 0); +#endif /* EFSYS_OPT_PHY_FLAGS */ + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail2; + } + + /* And set the blink mode */ + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_SET_ID_LED; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_SET_ID_LED_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_SET_ID_LED_OUT_LEN; + +#if EFSYS_OPT_PHY_LED_CONTROL + switch (epp->ep_phy_led_mode) { + case EFX_PHY_LED_DEFAULT: + led_mode = MC_CMD_LED_DEFAULT; + break; + case EFX_PHY_LED_OFF: + led_mode = MC_CMD_LED_OFF; + break; + case EFX_PHY_LED_ON: + led_mode = MC_CMD_LED_ON; + break; + default: + EFSYS_ASSERT(0); + led_mode = MC_CMD_LED_DEFAULT; + } + + MCDI_IN_SET_DWORD(req, SET_ID_LED_IN_STATE, led_mode); +#else + MCDI_IN_SET_DWORD(req, SET_ID_LED_IN_STATE, MC_CMD_LED_DEFAULT); +#endif /* EFSYS_OPT_PHY_LED_CONTROL */ + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail3; + } +out: + return (0); + +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +ef10_phy_verify( + __in efx_nic_t *enp) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_GET_PHY_STATE_IN_LEN, + MC_CMD_GET_PHY_STATE_OUT_LEN)]; + uint32_t state; + efx_rc_t rc; + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_GET_PHY_STATE; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_GET_PHY_STATE_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_GET_PHY_STATE_OUT_LEN; + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + if (req.emr_out_length_used < MC_CMD_GET_PHY_STATE_OUT_LEN) { + rc = EMSGSIZE; + goto fail2; + } + + state = MCDI_OUT_DWORD(req, GET_PHY_STATE_OUT_STATE); + if (state != MC_CMD_PHY_STATE_OK) { + if (state != MC_CMD_PHY_STATE_ZOMBIE) + EFSYS_PROBE1(mc_pcol_error, int, state); + rc = ENOTACTIVE; + goto fail3; + } + + return (0); + +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +ef10_phy_oui_get( + __in efx_nic_t *enp, + __out uint32_t *ouip) +{ + _NOTE(ARGUNUSED(enp, ouip)) + + return (ENOTSUP); +} + +#if EFSYS_OPT_PHY_STATS + + __checkReturn efx_rc_t +ef10_phy_stats_update( + __in efx_nic_t *enp, + __in efsys_mem_t *esmp, + __inout_ecount(EFX_PHY_NSTATS) uint32_t *stat) +{ + /* TBD: no stats support in firmware yet */ + _NOTE(ARGUNUSED(enp, esmp)) + memset(stat, 0, EFX_PHY_NSTATS * sizeof (*stat)); + + return (0); +} + +#endif /* EFSYS_OPT_PHY_STATS */ + +#if EFSYS_OPT_BIST + + __checkReturn efx_rc_t +ef10_bist_enable_offline( + __in efx_nic_t *enp) +{ + efx_rc_t rc; + + if ((rc = efx_mcdi_bist_enable_offline(enp)) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +ef10_bist_start( + __in efx_nic_t *enp, + __in efx_bist_type_t type) +{ + efx_rc_t rc; + + if ((rc = efx_mcdi_bist_start(enp, type)) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +ef10_bist_poll( + __in efx_nic_t *enp, + __in efx_bist_type_t type, + __out efx_bist_result_t *resultp, + __out_opt __drv_when(count > 0, __notnull) + uint32_t *value_maskp, + __out_ecount_opt(count) __drv_when(count > 0, __notnull) + unsigned long *valuesp, + __in size_t count) +{ + efx_nic_cfg_t *encp = &(enp->en_nic_cfg); + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_POLL_BIST_IN_LEN, + MCDI_CTL_SDU_LEN_MAX)]; + uint32_t value_mask = 0; + uint32_t result; + efx_rc_t rc; + + _NOTE(ARGUNUSED(type)) + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_POLL_BIST; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_POLL_BIST_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MCDI_CTL_SDU_LEN_MAX; + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + if (req.emr_out_length_used < MC_CMD_POLL_BIST_OUT_RESULT_OFST + 4) { + rc = EMSGSIZE; + goto fail2; + } + + if (count > 0) + (void) memset(valuesp, '\0', count * sizeof (unsigned long)); + + result = MCDI_OUT_DWORD(req, POLL_BIST_OUT_RESULT); + + if (result == MC_CMD_POLL_BIST_FAILED && + req.emr_out_length >= MC_CMD_POLL_BIST_OUT_MEM_LEN && + count > EFX_BIST_MEM_ECC_FATAL) { + if (valuesp != NULL) { + valuesp[EFX_BIST_MEM_TEST] = + MCDI_OUT_DWORD(req, POLL_BIST_OUT_MEM_TEST); + valuesp[EFX_BIST_MEM_ADDR] = + MCDI_OUT_DWORD(req, POLL_BIST_OUT_MEM_ADDR); + valuesp[EFX_BIST_MEM_BUS] = + MCDI_OUT_DWORD(req, POLL_BIST_OUT_MEM_BUS); + valuesp[EFX_BIST_MEM_EXPECT] = + MCDI_OUT_DWORD(req, POLL_BIST_OUT_MEM_EXPECT); + valuesp[EFX_BIST_MEM_ACTUAL] = + MCDI_OUT_DWORD(req, POLL_BIST_OUT_MEM_ACTUAL); + valuesp[EFX_BIST_MEM_ECC] = + MCDI_OUT_DWORD(req, POLL_BIST_OUT_MEM_ECC); + valuesp[EFX_BIST_MEM_ECC_PARITY] = + MCDI_OUT_DWORD(req, POLL_BIST_OUT_MEM_ECC_PARITY); + valuesp[EFX_BIST_MEM_ECC_FATAL] = + MCDI_OUT_DWORD(req, POLL_BIST_OUT_MEM_ECC_FATAL); + } + value_mask |= (1 << EFX_BIST_MEM_TEST) | + (1 << EFX_BIST_MEM_ADDR) | + (1 << EFX_BIST_MEM_BUS) | + (1 << EFX_BIST_MEM_EXPECT) | + (1 << EFX_BIST_MEM_ACTUAL) | + (1 << EFX_BIST_MEM_ECC) | + (1 << EFX_BIST_MEM_ECC_PARITY) | + (1 << EFX_BIST_MEM_ECC_FATAL); + } else if (result == MC_CMD_POLL_BIST_FAILED && + encp->enc_phy_type == EFX_PHY_XFI_FARMI && + req.emr_out_length >= MC_CMD_POLL_BIST_OUT_MRSFP_LEN && + count > EFX_BIST_FAULT_CODE) { + if (valuesp != NULL) + valuesp[EFX_BIST_FAULT_CODE] = + MCDI_OUT_DWORD(req, POLL_BIST_OUT_MRSFP_TEST); + value_mask |= 1 << EFX_BIST_FAULT_CODE; + } + + if (value_maskp != NULL) + *value_maskp = value_mask; + + EFSYS_ASSERT(resultp != NULL); + if (result == MC_CMD_POLL_BIST_RUNNING) + *resultp = EFX_BIST_RESULT_RUNNING; + else if (result == MC_CMD_POLL_BIST_PASSED) + *resultp = EFX_BIST_RESULT_PASSED; + else + *resultp = EFX_BIST_RESULT_FAILED; + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + void +ef10_bist_stop( + __in efx_nic_t *enp, + __in efx_bist_type_t type) +{ + /* There is no way to stop BIST on EF10. */ + _NOTE(ARGUNUSED(enp, type)) +} + +#endif /* EFSYS_OPT_BIST */ + +#endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */ diff --git a/dpdk/drivers/net/sfc/base/ef10_rx.c b/dpdk/drivers/net/sfc/base/ef10_rx.c new file mode 100644 index 00000000..849f674c --- /dev/null +++ b/dpdk/drivers/net/sfc/base/ef10_rx.c @@ -0,0 +1,1033 @@ +/* + * Copyright (c) 2012-2016 Solarflare Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are + * those of the authors and should not be interpreted as representing official + * policies, either expressed or implied, of the FreeBSD Project. + */ + +#include "efx.h" +#include "efx_impl.h" + + +#if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD + + +static __checkReturn efx_rc_t +efx_mcdi_init_rxq( + __in efx_nic_t *enp, + __in uint32_t size, + __in uint32_t target_evq, + __in uint32_t label, + __in uint32_t instance, + __in efsys_mem_t *esmp, + __in boolean_t disable_scatter, + __in uint32_t ps_bufsize) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_INIT_RXQ_EXT_IN_LEN, + MC_CMD_INIT_RXQ_EXT_OUT_LEN)]; + int npages = EFX_RXQ_NBUFS(size); + int i; + efx_qword_t *dma_addr; + uint64_t addr; + efx_rc_t rc; + uint32_t dma_mode; + + /* If this changes, then the payload size might need to change. */ + EFSYS_ASSERT3U(MC_CMD_INIT_RXQ_OUT_LEN, ==, 0); + EFSYS_ASSERT3U(size, <=, EFX_RXQ_MAXNDESCS); + + if (ps_bufsize > 0) + dma_mode = MC_CMD_INIT_RXQ_EXT_IN_PACKED_STREAM; + else + dma_mode = MC_CMD_INIT_RXQ_EXT_IN_SINGLE_PACKET; + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_INIT_RXQ; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_INIT_RXQ_EXT_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_INIT_RXQ_EXT_OUT_LEN; + + MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_SIZE, size); + MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_TARGET_EVQ, target_evq); + MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_LABEL, label); + MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_INSTANCE, instance); + MCDI_IN_POPULATE_DWORD_8(req, INIT_RXQ_EXT_IN_FLAGS, + INIT_RXQ_EXT_IN_FLAG_BUFF_MODE, 0, + INIT_RXQ_EXT_IN_FLAG_HDR_SPLIT, 0, + INIT_RXQ_EXT_IN_FLAG_TIMESTAMP, 0, + INIT_RXQ_EXT_IN_CRC_MODE, 0, + INIT_RXQ_EXT_IN_FLAG_PREFIX, 1, + INIT_RXQ_EXT_IN_FLAG_DISABLE_SCATTER, disable_scatter, + INIT_RXQ_EXT_IN_DMA_MODE, + dma_mode, + INIT_RXQ_EXT_IN_PACKED_STREAM_BUFF_SIZE, ps_bufsize); + MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_OWNER_ID, 0); + MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_PORT_ID, EVB_PORT_ID_ASSIGNED); + + dma_addr = MCDI_IN2(req, efx_qword_t, INIT_RXQ_IN_DMA_ADDR); + addr = EFSYS_MEM_ADDR(esmp); + + for (i = 0; i < npages; i++) { + EFX_POPULATE_QWORD_2(*dma_addr, + EFX_DWORD_1, (uint32_t)(addr >> 32), + EFX_DWORD_0, (uint32_t)(addr & 0xffffffff)); + + dma_addr++; + addr += EFX_BUF_SIZE; + } + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +static __checkReturn efx_rc_t +efx_mcdi_fini_rxq( + __in efx_nic_t *enp, + __in uint32_t instance) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_FINI_RXQ_IN_LEN, + MC_CMD_FINI_RXQ_OUT_LEN)]; + efx_rc_t rc; + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_FINI_RXQ; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_FINI_RXQ_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_FINI_RXQ_OUT_LEN; + + MCDI_IN_SET_DWORD(req, FINI_RXQ_IN_INSTANCE, instance); + + efx_mcdi_execute_quiet(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + return (0); + +fail1: + /* + * EALREADY is not an error, but indicates that the MC has rebooted and + * that the RXQ has already been destroyed. + */ + if (rc != EALREADY) + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +#if EFSYS_OPT_RX_SCALE +static __checkReturn efx_rc_t +efx_mcdi_rss_context_alloc( + __in efx_nic_t *enp, + __in efx_rx_scale_context_type_t type, + __in uint32_t num_queues, + __out uint32_t *rss_contextp) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN, + MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN)]; + uint32_t rss_context; + uint32_t context_type; + efx_rc_t rc; + + if (num_queues > EFX_MAXRSS) { + rc = EINVAL; + goto fail1; + } + + switch (type) { + case EFX_RX_SCALE_EXCLUSIVE: + context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_EXCLUSIVE; + break; + case EFX_RX_SCALE_SHARED: + context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_SHARED; + break; + default: + rc = EINVAL; + goto fail2; + } + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_RSS_CONTEXT_ALLOC; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN; + + MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_UPSTREAM_PORT_ID, + EVB_PORT_ID_ASSIGNED); + MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_TYPE, context_type); + /* NUM_QUEUES is only used to validate indirection table offsets */ + MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_NUM_QUEUES, num_queues); + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail3; + } + + if (req.emr_out_length_used < MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN) { + rc = EMSGSIZE; + goto fail4; + } + + rss_context = MCDI_OUT_DWORD(req, RSS_CONTEXT_ALLOC_OUT_RSS_CONTEXT_ID); + if (rss_context == EF10_RSS_CONTEXT_INVALID) { + rc = ENOENT; + goto fail5; + } + + *rss_contextp = rss_context; + + return (0); + +fail5: + EFSYS_PROBE(fail5); +fail4: + EFSYS_PROBE(fail4); +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} +#endif /* EFSYS_OPT_RX_SCALE */ + +#if EFSYS_OPT_RX_SCALE +static efx_rc_t +efx_mcdi_rss_context_free( + __in efx_nic_t *enp, + __in uint32_t rss_context) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_FREE_IN_LEN, + MC_CMD_RSS_CONTEXT_FREE_OUT_LEN)]; + efx_rc_t rc; + + if (rss_context == EF10_RSS_CONTEXT_INVALID) { + rc = EINVAL; + goto fail1; + } + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_RSS_CONTEXT_FREE; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_RSS_CONTEXT_FREE_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_RSS_CONTEXT_FREE_OUT_LEN; + + MCDI_IN_SET_DWORD(req, RSS_CONTEXT_FREE_IN_RSS_CONTEXT_ID, rss_context); + + efx_mcdi_execute_quiet(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail2; + } + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} +#endif /* EFSYS_OPT_RX_SCALE */ + +#if EFSYS_OPT_RX_SCALE +static efx_rc_t +efx_mcdi_rss_context_set_flags( + __in efx_nic_t *enp, + __in uint32_t rss_context, + __in efx_rx_hash_type_t type) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN, + MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN)]; + efx_rc_t rc; + + if (rss_context == EF10_RSS_CONTEXT_INVALID) { + rc = EINVAL; + goto fail1; + } + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_FLAGS; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN; + + MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_FLAGS_IN_RSS_CONTEXT_ID, + rss_context); + + MCDI_IN_POPULATE_DWORD_4(req, RSS_CONTEXT_SET_FLAGS_IN_FLAGS, + RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV4_EN, + (type & EFX_RX_HASH_IPV4) ? 1 : 0, + RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV4_EN, + (type & EFX_RX_HASH_TCPIPV4) ? 1 : 0, + RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV6_EN, + (type & EFX_RX_HASH_IPV6) ? 1 : 0, + RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV6_EN, + (type & EFX_RX_HASH_TCPIPV6) ? 1 : 0); + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail2; + } + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} +#endif /* EFSYS_OPT_RX_SCALE */ + +#if EFSYS_OPT_RX_SCALE +static efx_rc_t +efx_mcdi_rss_context_set_key( + __in efx_nic_t *enp, + __in uint32_t rss_context, + __in_ecount(n) uint8_t *key, + __in size_t n) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN, + MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN)]; + efx_rc_t rc; + + if (rss_context == EF10_RSS_CONTEXT_INVALID) { + rc = EINVAL; + goto fail1; + } + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_KEY; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN; + + MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_KEY_IN_RSS_CONTEXT_ID, + rss_context); + + EFSYS_ASSERT3U(n, ==, MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN); + if (n != MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN) { + rc = EINVAL; + goto fail2; + } + + memcpy(MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY), + key, n); + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail3; + } + + return (0); + +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} +#endif /* EFSYS_OPT_RX_SCALE */ + +#if EFSYS_OPT_RX_SCALE +static efx_rc_t +efx_mcdi_rss_context_set_table( + __in efx_nic_t *enp, + __in uint32_t rss_context, + __in_ecount(n) unsigned int *table, + __in size_t n) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN, + MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN)]; + uint8_t *req_table; + int i, rc; + + if (rss_context == EF10_RSS_CONTEXT_INVALID) { + rc = EINVAL; + goto fail1; + } + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_TABLE; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN; + + MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_TABLE_IN_RSS_CONTEXT_ID, + rss_context); + + req_table = + MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE); + + for (i = 0; + i < MC_CMD_RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE_LEN; + i++) { + req_table[i] = (n > 0) ? (uint8_t)table[i % n] : 0; + } + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail2; + } + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} +#endif /* EFSYS_OPT_RX_SCALE */ + + + __checkReturn efx_rc_t +ef10_rx_init( + __in efx_nic_t *enp) +{ +#if EFSYS_OPT_RX_SCALE + + if (efx_mcdi_rss_context_alloc(enp, EFX_RX_SCALE_EXCLUSIVE, EFX_MAXRSS, + &enp->en_rss_context) == 0) { + /* + * Allocated an exclusive RSS context, which allows both the + * indirection table and key to be modified. + */ + enp->en_rss_context_type = EFX_RX_SCALE_EXCLUSIVE; + enp->en_hash_support = EFX_RX_HASH_AVAILABLE; + } else { + /* + * Failed to allocate an exclusive RSS context. Continue + * operation without support for RSS. The pseudo-header in + * received packets will not contain a Toeplitz hash value. + */ + enp->en_rss_context_type = EFX_RX_SCALE_UNAVAILABLE; + enp->en_hash_support = EFX_RX_HASH_UNAVAILABLE; + } + +#endif /* EFSYS_OPT_RX_SCALE */ + + return (0); +} + +#if EFSYS_OPT_RX_SCATTER + __checkReturn efx_rc_t +ef10_rx_scatter_enable( + __in efx_nic_t *enp, + __in unsigned int buf_size) +{ + _NOTE(ARGUNUSED(enp, buf_size)) + return (0); +} +#endif /* EFSYS_OPT_RX_SCATTER */ + +#if EFSYS_OPT_RX_SCALE + __checkReturn efx_rc_t +ef10_rx_scale_context_alloc( + __in efx_nic_t *enp, + __in efx_rx_scale_context_type_t type, + __in uint32_t num_queues, + __out uint32_t *rss_contextp) +{ + efx_rc_t rc; + + rc = efx_mcdi_rss_context_alloc(enp, type, num_queues, rss_contextp); + if (rc != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + return (rc); +} +#endif /* EFSYS_OPT_RX_SCALE */ + +#if EFSYS_OPT_RX_SCALE + __checkReturn efx_rc_t +ef10_rx_scale_context_free( + __in efx_nic_t *enp, + __in uint32_t rss_context) +{ + efx_rc_t rc; + + rc = efx_mcdi_rss_context_free(enp, rss_context); + if (rc != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + return (rc); +} +#endif /* EFSYS_OPT_RX_SCALE */ + +#if EFSYS_OPT_RX_SCALE + __checkReturn efx_rc_t +ef10_rx_scale_mode_set( + __in efx_nic_t *enp, + __in uint32_t rss_context, + __in efx_rx_hash_alg_t alg, + __in efx_rx_hash_type_t type, + __in boolean_t insert) +{ + efx_rc_t rc; + + EFSYS_ASSERT3U(alg, ==, EFX_RX_HASHALG_TOEPLITZ); + EFSYS_ASSERT3U(insert, ==, B_TRUE); + + if ((alg != EFX_RX_HASHALG_TOEPLITZ) || (insert == B_FALSE)) { + rc = EINVAL; + goto fail1; + } + + if (rss_context == EFX_RSS_CONTEXT_DEFAULT) { + if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) { + rc = ENOTSUP; + goto fail2; + } + rss_context = enp->en_rss_context; + } + + if ((rc = efx_mcdi_rss_context_set_flags(enp, + rss_context, type)) != 0) + goto fail3; + + return (0); + +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} +#endif /* EFSYS_OPT_RX_SCALE */ + +#if EFSYS_OPT_RX_SCALE + __checkReturn efx_rc_t +ef10_rx_scale_key_set( + __in efx_nic_t *enp, + __in uint32_t rss_context, + __in_ecount(n) uint8_t *key, + __in size_t n) +{ + efx_rc_t rc; + + EFX_STATIC_ASSERT(EFX_RSS_KEY_SIZE == + MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN); + + if (rss_context == EFX_RSS_CONTEXT_DEFAULT) { + if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) { + rc = ENOTSUP; + goto fail1; + } + rss_context = enp->en_rss_context; + } + + if ((rc = efx_mcdi_rss_context_set_key(enp, rss_context, key, n)) != 0) + goto fail2; + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} +#endif /* EFSYS_OPT_RX_SCALE */ + +#if EFSYS_OPT_RX_SCALE + __checkReturn efx_rc_t +ef10_rx_scale_tbl_set( + __in efx_nic_t *enp, + __in uint32_t rss_context, + __in_ecount(n) unsigned int *table, + __in size_t n) +{ + efx_rc_t rc; + + + if (rss_context == EFX_RSS_CONTEXT_DEFAULT) { + if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) { + rc = ENOTSUP; + goto fail1; + } + rss_context = enp->en_rss_context; + } + + if ((rc = efx_mcdi_rss_context_set_table(enp, + rss_context, table, n)) != 0) + goto fail2; + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} +#endif /* EFSYS_OPT_RX_SCALE */ + + +/* + * EF10 RX pseudo-header + * --------------------- + * + * Receive packets are prefixed by an (optional) 14 byte pseudo-header: + * + * +00: Toeplitz hash value. + * (32bit little-endian) + * +04: Outer VLAN tag. Zero if the packet did not have an outer VLAN tag. + * (16bit big-endian) + * +06: Inner VLAN tag. Zero if the packet did not have an inner VLAN tag. + * (16bit big-endian) + * +08: Packet Length. Zero if the RX datapath was in cut-through mode. + * (16bit little-endian) + * +10: MAC timestamp. Zero if timestamping is not enabled. + * (32bit little-endian) + * + * See "The RX Pseudo-header" in SF-109306-TC. + */ + + __checkReturn efx_rc_t +ef10_rx_prefix_pktlen( + __in efx_nic_t *enp, + __in uint8_t *buffer, + __out uint16_t *lengthp) +{ + _NOTE(ARGUNUSED(enp)) + + /* + * The RX pseudo-header contains the packet length, excluding the + * pseudo-header. If the hardware receive datapath was operating in + * cut-through mode then the length in the RX pseudo-header will be + * zero, and the packet length must be obtained from the DMA length + * reported in the RX event. + */ + *lengthp = buffer[8] | (buffer[9] << 8); + return (0); +} + +#if EFSYS_OPT_RX_SCALE + __checkReturn uint32_t +ef10_rx_prefix_hash( + __in efx_nic_t *enp, + __in efx_rx_hash_alg_t func, + __in uint8_t *buffer) +{ + _NOTE(ARGUNUSED(enp)) + + switch (func) { + case EFX_RX_HASHALG_TOEPLITZ: + return (buffer[0] | + (buffer[1] << 8) | + (buffer[2] << 16) | + (buffer[3] << 24)); + + default: + EFSYS_ASSERT(0); + return (0); + } +} +#endif /* EFSYS_OPT_RX_SCALE */ + + void +ef10_rx_qpost( + __in efx_rxq_t *erp, + __in_ecount(n) efsys_dma_addr_t *addrp, + __in size_t size, + __in unsigned int n, + __in unsigned int completed, + __in unsigned int added) +{ + efx_qword_t qword; + unsigned int i; + unsigned int offset; + unsigned int id; + + /* The client driver must not overfill the queue */ + EFSYS_ASSERT3U(added - completed + n, <=, + EFX_RXQ_LIMIT(erp->er_mask + 1)); + + id = added & (erp->er_mask); + for (i = 0; i < n; i++) { + EFSYS_PROBE4(rx_post, unsigned int, erp->er_index, + unsigned int, id, efsys_dma_addr_t, addrp[i], + size_t, size); + + EFX_POPULATE_QWORD_3(qword, + ESF_DZ_RX_KER_BYTE_CNT, (uint32_t)(size), + ESF_DZ_RX_KER_BUF_ADDR_DW0, + (uint32_t)(addrp[i] & 0xffffffff), + ESF_DZ_RX_KER_BUF_ADDR_DW1, + (uint32_t)(addrp[i] >> 32)); + + offset = id * sizeof (efx_qword_t); + EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword); + + id = (id + 1) & (erp->er_mask); + } +} + + void +ef10_rx_qpush( + __in efx_rxq_t *erp, + __in unsigned int added, + __inout unsigned int *pushedp) +{ + efx_nic_t *enp = erp->er_enp; + unsigned int pushed = *pushedp; + uint32_t wptr; + efx_dword_t dword; + + /* Hardware has alignment restriction for WPTR */ + wptr = P2ALIGN(added, EF10_RX_WPTR_ALIGN); + if (pushed == wptr) + return; + + *pushedp = wptr; + + /* Push the populated descriptors out */ + wptr &= erp->er_mask; + + EFX_POPULATE_DWORD_1(dword, ERF_DZ_RX_DESC_WPTR, wptr); + + /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */ + EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1, + wptr, pushed & erp->er_mask); + EFSYS_PIO_WRITE_BARRIER(); + EFX_BAR_TBL_WRITED(enp, ER_DZ_RX_DESC_UPD_REG, + erp->er_index, &dword, B_FALSE); +} + +#if EFSYS_OPT_RX_PACKED_STREAM + + void +ef10_rx_qps_update_credits( + __in efx_rxq_t *erp) +{ + efx_nic_t *enp = erp->er_enp; + efx_dword_t dword; + efx_evq_rxq_state_t *rxq_state = + &erp->er_eep->ee_rxq_state[erp->er_label]; + + EFSYS_ASSERT(rxq_state->eers_rx_packed_stream); + + if (rxq_state->eers_rx_packed_stream_credits == 0) + return; + + EFX_POPULATE_DWORD_3(dword, + ERF_DZ_RX_DESC_MAGIC_DOORBELL, 1, + ERF_DZ_RX_DESC_MAGIC_CMD, + ERE_DZ_RX_DESC_MAGIC_CMD_PS_CREDITS, + ERF_DZ_RX_DESC_MAGIC_DATA, + rxq_state->eers_rx_packed_stream_credits); + EFX_BAR_TBL_WRITED(enp, ER_DZ_RX_DESC_UPD_REG, + erp->er_index, &dword, B_FALSE); + + rxq_state->eers_rx_packed_stream_credits = 0; +} + + __checkReturn uint8_t * +ef10_rx_qps_packet_info( + __in efx_rxq_t *erp, + __in uint8_t *buffer, + __in uint32_t buffer_length, + __in uint32_t current_offset, + __out uint16_t *lengthp, + __out uint32_t *next_offsetp, + __out uint32_t *timestamp) +{ + uint16_t buf_len; + uint8_t *pkt_start; + efx_qword_t *qwordp; + efx_evq_rxq_state_t *rxq_state = + &erp->er_eep->ee_rxq_state[erp->er_label]; + + EFSYS_ASSERT(rxq_state->eers_rx_packed_stream); + + buffer += current_offset; + pkt_start = buffer + EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE; + + qwordp = (efx_qword_t *)buffer; + *timestamp = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_TSTAMP); + *lengthp = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_ORIG_LEN); + buf_len = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_CAP_LEN); + + buf_len = P2ROUNDUP(buf_len + EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE, + EFX_RX_PACKED_STREAM_ALIGNMENT); + *next_offsetp = + current_offset + buf_len + EFX_RX_PACKED_STREAM_ALIGNMENT; + + EFSYS_ASSERT3U(*next_offsetp, <=, buffer_length); + EFSYS_ASSERT3U(current_offset + *lengthp, <, *next_offsetp); + + if ((*next_offsetp ^ current_offset) & + EFX_RX_PACKED_STREAM_MEM_PER_CREDIT) { + if (rxq_state->eers_rx_packed_stream_credits < + EFX_RX_PACKED_STREAM_MAX_CREDITS) + rxq_state->eers_rx_packed_stream_credits++; + } + + return (pkt_start); +} + + +#endif + + __checkReturn efx_rc_t +ef10_rx_qflush( + __in efx_rxq_t *erp) +{ + efx_nic_t *enp = erp->er_enp; + efx_rc_t rc; + + if ((rc = efx_mcdi_fini_rxq(enp, erp->er_index)) != 0) + goto fail1; + + return (0); + +fail1: + /* + * EALREADY is not an error, but indicates that the MC has rebooted and + * that the RXQ has already been destroyed. Callers need to know that + * the RXQ flush has completed to avoid waiting until timeout for a + * flush done event that will not be delivered. + */ + if (rc != EALREADY) + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + void +ef10_rx_qenable( + __in efx_rxq_t *erp) +{ + /* FIXME */ + _NOTE(ARGUNUSED(erp)) + /* FIXME */ +} + + __checkReturn efx_rc_t +ef10_rx_qcreate( + __in efx_nic_t *enp, + __in unsigned int index, + __in unsigned int label, + __in efx_rxq_type_t type, + __in efsys_mem_t *esmp, + __in size_t n, + __in uint32_t id, + __in efx_evq_t *eep, + __in efx_rxq_t *erp) +{ + efx_nic_cfg_t *encp = &(enp->en_nic_cfg); + efx_rc_t rc; + boolean_t disable_scatter; + unsigned int ps_buf_size; + + _NOTE(ARGUNUSED(id, erp)) + + EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS == (1 << ESF_DZ_RX_QLABEL_WIDTH)); + EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS); + EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit); + + EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MAXNDESCS)); + EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MINNDESCS)); + + if (!ISP2(n) || (n < EFX_RXQ_MINNDESCS) || (n > EFX_RXQ_MAXNDESCS)) { + rc = EINVAL; + goto fail1; + } + if (index >= encp->enc_rxq_limit) { + rc = EINVAL; + goto fail2; + } + + switch (type) { + case EFX_RXQ_TYPE_DEFAULT: + case EFX_RXQ_TYPE_SCATTER: + ps_buf_size = 0; + break; +#if EFSYS_OPT_RX_PACKED_STREAM + case EFX_RXQ_TYPE_PACKED_STREAM_1M: + ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_1M; + break; + case EFX_RXQ_TYPE_PACKED_STREAM_512K: + ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_512K; + break; + case EFX_RXQ_TYPE_PACKED_STREAM_256K: + ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_256K; + break; + case EFX_RXQ_TYPE_PACKED_STREAM_128K: + ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_128K; + break; + case EFX_RXQ_TYPE_PACKED_STREAM_64K: + ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_64K; + break; +#endif /* EFSYS_OPT_RX_PACKED_STREAM */ + default: + rc = ENOTSUP; + goto fail3; + } + +#if EFSYS_OPT_RX_PACKED_STREAM + if (ps_buf_size != 0) { + /* Check if datapath firmware supports packed stream mode */ + if (encp->enc_rx_packed_stream_supported == B_FALSE) { + rc = ENOTSUP; + goto fail4; + } + /* Check if packed stream allows configurable buffer sizes */ + if ((type != EFX_RXQ_TYPE_PACKED_STREAM_1M) && + (encp->enc_rx_var_packed_stream_supported == B_FALSE)) { + rc = ENOTSUP; + goto fail5; + } + } +#else /* EFSYS_OPT_RX_PACKED_STREAM */ + EFSYS_ASSERT(ps_buf_size == 0); +#endif /* EFSYS_OPT_RX_PACKED_STREAM */ + + /* Scatter can only be disabled if the firmware supports doing so */ + if (type == EFX_RXQ_TYPE_SCATTER) + disable_scatter = B_FALSE; + else + disable_scatter = encp->enc_rx_disable_scatter_supported; + + if ((rc = efx_mcdi_init_rxq(enp, n, eep->ee_index, label, index, + esmp, disable_scatter, ps_buf_size)) != 0) + goto fail6; + + erp->er_eep = eep; + erp->er_label = label; + + ef10_ev_rxlabel_init(eep, erp, label, ps_buf_size != 0); + + return (0); + +fail6: + EFSYS_PROBE(fail6); +#if EFSYS_OPT_RX_PACKED_STREAM +fail5: + EFSYS_PROBE(fail5); +fail4: + EFSYS_PROBE(fail4); +#endif /* EFSYS_OPT_RX_PACKED_STREAM */ +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + void +ef10_rx_qdestroy( + __in efx_rxq_t *erp) +{ + efx_nic_t *enp = erp->er_enp; + efx_evq_t *eep = erp->er_eep; + unsigned int label = erp->er_label; + + ef10_ev_rxlabel_fini(eep, label); + + EFSYS_ASSERT(enp->en_rx_qcount != 0); + --enp->en_rx_qcount; + + EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp); +} + + void +ef10_rx_fini( + __in efx_nic_t *enp) +{ +#if EFSYS_OPT_RX_SCALE + if (enp->en_rss_context_type != EFX_RX_SCALE_UNAVAILABLE) + (void) efx_mcdi_rss_context_free(enp, enp->en_rss_context); + enp->en_rss_context = 0; + enp->en_rss_context_type = EFX_RX_SCALE_UNAVAILABLE; +#else + _NOTE(ARGUNUSED(enp)) +#endif /* EFSYS_OPT_RX_SCALE */ +} + +#endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */ diff --git a/dpdk/drivers/net/sfc/base/ef10_tlv_layout.h b/dpdk/drivers/net/sfc/base/ef10_tlv_layout.h new file mode 100644 index 00000000..7d099b81 --- /dev/null +++ b/dpdk/drivers/net/sfc/base/ef10_tlv_layout.h @@ -0,0 +1,941 @@ +/**************************************************************************\ +*//*! \file +** +** \author mjs +** \brief TLV item layouts for EF10 static and dynamic config in NVRAM +** \date 2012/11/20 +** \cop (c) Solarflare Communications Inc. +** +*//* +\**************************************************************************/ + +/* These structures define the layouts for the TLV items stored in static and + * dynamic configuration partitions in NVRAM for EF10 (Huntington etc.). + * + * They contain the same sort of information that was kept in the + * siena_mc_static_config_hdr_t and siena_mc_dynamic_config_hdr_t structures + * (defined in and ) for + * Siena. + * + * These are used directly by the MC and should also be usable directly on host + * systems which are little-endian and do not do strange things with structure + * padding. (Big-endian host systems will require some byte-swapping.) + * + * ----- + * + * Please refer to SF-108797-SW for a general overview of the TLV partition + * format. + * + * ----- + * + * The current tag IDs have a general structure: with the exception of the + * special values defined in the document, they are of the form 0xLTTTNNNN, + * where: + * + * - L is a location, indicating where this tag is expected to be found: + * 0: static configuration + * 1: dynamic configuration + * 2: firmware internal use + * 3: license partition + * + * - TTT is a type, which is just a unique value. The same type value + * might appear in both locations, indicating a relationship between + * the items (e.g. static and dynamic VPD below). + * + * - NNNN is an index of some form. Some item types are per-port, some + * are per-PF, some are per-partition-type. + * + * ----- + * + * As with the previous Siena structures, each structure here is laid out + * carefully: values are aligned to their natural boundary, with explicit + * padding fields added where necessary. (No, technically this does not + * absolutely guarantee portability. But, in practice, compilers are generally + * sensible enough not to introduce completely pointless padding, and it works + * well enough.) + */ + + +#ifndef CI_MGMT_TLV_LAYOUT_H +#define CI_MGMT_TLV_LAYOUT_H + + +/* ---------------------------------------------------------------------------- + * General structure (defined by SF-108797-SW) + * ---------------------------------------------------------------------------- + */ + + +/* The "end" tag. + * + * (Note that this is *not* followed by length or value fields: anything after + * the tag itself is irrelevant.) + */ + +#define TLV_TAG_END (0xEEEEEEEE) + + +/* Other special reserved tag values. + */ + +#define TLV_TAG_SKIP (0x00000000) +#define TLV_TAG_INVALID (0xFFFFFFFF) + + +/* TLV partition header. + * + * In a TLV partition, this must be the first item in the sequence, at offset + * 0. + */ + +#define TLV_TAG_PARTITION_HEADER (0xEF10DA7A) + +struct tlv_partition_header { + uint32_t tag; + uint32_t length; + uint16_t type_id; +/* 0 indicates the default segment (always located at offset 0), while other values + * are for RFID-selectable presets that should immediately follow the default segment. + * The default segment may also have preset > 0, which means that it is a preset + * selected through an RFID command and copied by FW to the location at offset 0. */ + uint16_t preset; + uint32_t generation; + uint32_t total_length; +}; + + +/* TLV partition trailer. + * + * In a TLV partition, this must be the last item in the sequence, immediately + * preceding the TLV_TAG_END word. + */ + +#define TLV_TAG_PARTITION_TRAILER (0xEF101A57) + +struct tlv_partition_trailer { + uint32_t tag; + uint32_t length; + uint32_t generation; + uint32_t checksum; +}; + + +/* Appendable TLV partition header. + * + * In an appendable TLV partition, this must be the first item in the sequence, + * at offset 0. (Note that, unlike the configuration partitions, there is no + * trailer before the TLV_TAG_END word.) + */ + +#define TLV_TAG_APPENDABLE_PARTITION_HEADER (0xEF10ADA7) + +struct tlv_appendable_partition_header { + uint32_t tag; + uint32_t length; + uint16_t type_id; + uint16_t reserved; +}; + + +/* ---------------------------------------------------------------------------- + * Configuration items + * ---------------------------------------------------------------------------- + */ + + +/* NIC global capabilities. + */ + +#define TLV_TAG_GLOBAL_CAPABILITIES (0x00010000) + +struct tlv_global_capabilities { + uint32_t tag; + uint32_t length; + uint32_t flags; +}; + + +/* Siena-style per-port MAC address allocation. + * + * There are addresses, starting at and incrementing + * by adding to the low-order byte(s). + * + * (See also TLV_TAG_GLOBAL_MAC for an alternative, specifying a global pool + * of contiguous MAC addresses for the firmware to allocate as it sees fit.) + */ + +#define TLV_TAG_PORT_MAC(port) (0x00020000 + (port)) + +struct tlv_port_mac { + uint32_t tag; + uint32_t length; + uint8_t base_address[6]; + uint16_t reserved; + uint16_t count; + uint16_t stride; +}; + + +/* Static VPD. + * + * This is the portion of VPD which is set at manufacturing time and not + * expected to change. It is formatted as a standard PCI VPD block. There are + * global and per-pf TLVs for this, the global TLV is new for Medford and is + * used in preference to the per-pf TLV. + */ + +#define TLV_TAG_PF_STATIC_VPD(pf) (0x00030000 + (pf)) + +struct tlv_pf_static_vpd { + uint32_t tag; + uint32_t length; + uint8_t bytes[]; +}; + +#define TLV_TAG_GLOBAL_STATIC_VPD (0x001f0000) + +struct tlv_global_static_vpd { + uint32_t tag; + uint32_t length; + uint8_t bytes[]; +}; + + +/* Dynamic VPD. + * + * This is the portion of VPD which may be changed (e.g. by firmware updates). + * It is formatted as a standard PCI VPD block. There are global and per-pf TLVs + * for this, the global TLV is new for Medford and is used in preference to the + * per-pf TLV. + */ + +#define TLV_TAG_PF_DYNAMIC_VPD(pf) (0x10030000 + (pf)) + +struct tlv_pf_dynamic_vpd { + uint32_t tag; + uint32_t length; + uint8_t bytes[]; +}; + +#define TLV_TAG_GLOBAL_DYNAMIC_VPD (0x10200000) + +struct tlv_global_dynamic_vpd { + uint32_t tag; + uint32_t length; + uint8_t bytes[]; +}; + + +/* "DBI" PCI config space changes. + * + * This is a set of edits made to the default PCI config space values before + * the device is allowed to enumerate. There are global and per-pf TLVs for + * this, the global TLV is new for Medford and is used in preference to the + * per-pf TLV. + */ + +#define TLV_TAG_PF_DBI(pf) (0x00040000 + (pf)) + +struct tlv_pf_dbi { + uint32_t tag; + uint32_t length; + struct { + uint16_t addr; + uint16_t byte_enables; + uint32_t value; + } items[]; +}; + + +#define TLV_TAG_GLOBAL_DBI (0x00210000) + +struct tlv_global_dbi { + uint32_t tag; + uint32_t length; + struct { + uint16_t addr; + uint16_t byte_enables; + uint32_t value; + } items[]; +}; + + +/* Partition subtype codes. + * + * A subtype may optionally be stored for each type of partition present in + * the NVRAM. For example, this may be used to allow a generic firmware update + * utility to select a specific variant of firmware for a specific variant of + * board. + * + * The description[] field is an optional string which is returned in the + * MC_CMD_NVRAM_METADATA response if present. + */ + +#define TLV_TAG_PARTITION_SUBTYPE(type) (0x00050000 + (type)) + +struct tlv_partition_subtype { + uint32_t tag; + uint32_t length; + uint32_t subtype; + uint8_t description[]; +}; + + +/* Partition version codes. + * + * A version may optionally be stored for each type of partition present in + * the NVRAM. This provides a standard way of tracking the currently stored + * version of each of the various component images. + */ + +#define TLV_TAG_PARTITION_VERSION(type) (0x10060000 + (type)) + +struct tlv_partition_version { + uint32_t tag; + uint32_t length; + uint16_t version_w; + uint16_t version_x; + uint16_t version_y; + uint16_t version_z; +}; + +/* Global PCIe configuration */ + +#define TLV_TAG_GLOBAL_PCIE_CONFIG (0x10070000) + +struct tlv_pcie_config { + uint32_t tag; + uint32_t length; + int16_t max_pf_number; /**< Largest PF RID (lower PFs may be hidden) */ + uint16_t pf_aper; /**< BIU aperture for PF BAR2 */ + uint16_t vf_aper; /**< BIU aperture for VF BAR0 */ + uint16_t int_aper; /**< BIU aperture for PF BAR4 and VF BAR2 */ +#define TLV_MAX_PF_DEFAULT (-1) /* Use FW default for largest PF RID */ +#define TLV_APER_DEFAULT (0xFFFF) /* Use FW default for a given aperture */ +}; + +/* Per-PF configuration. Note that not all these fields are necessarily useful + * as the apertures are constrained by the BIU settings (the one case we do + * use is to make BAR2 bigger than the BIU thinks to reserve space), but we can + * tidy things up later */ + +#define TLV_TAG_PF_PCIE_CONFIG(pf) (0x10080000 + (pf)) + +struct tlv_per_pf_pcie_config { + uint32_t tag; + uint32_t length; + uint8_t vfs_total; + uint8_t port_allocation; + uint16_t vectors_per_pf; + uint16_t vectors_per_vf; + uint8_t pf_bar0_aperture; + uint8_t pf_bar2_aperture; + uint8_t vf_bar0_aperture; + uint8_t vf_base; + uint16_t supp_pagesz; + uint16_t msix_vec_base; +}; + + +/* Development ONLY. This is a single TLV tag for all the gubbins + * that can be set through the MC command-line other than the PCIe + * settings. This is a temporary measure. */ +#define TLV_TAG_TMP_GUBBINS (0x10090000) /* legacy symbol - do not use */ +#define TLV_TAG_TMP_GUBBINS_HUNT TLV_TAG_TMP_GUBBINS + +struct tlv_tmp_gubbins { + uint32_t tag; + uint32_t length; + /* Consumed by dpcpu.c */ + uint64_t tx0_tags; /* Bitmap */ + uint64_t tx1_tags; /* Bitmap */ + uint64_t dl_tags; /* Bitmap */ + uint32_t flags; +#define TLV_DPCPU_TX_STRIPE (1) /* No longer used, has no effect */ +#define TLV_DPCPU_BIU_TAGS (2) /* Use BIU tag manager */ +#define TLV_DPCPU_TX0_TAGS (4) /* tx0_tags is valid */ +#define TLV_DPCPU_TX1_TAGS (8) /* tx1_tags is valid */ +#define TLV_DPCPU_DL_TAGS (16) /* dl_tags is valid */ + /* Consumed by features.c */ + uint32_t dut_features; /* All 1s -> leave alone */ + int8_t with_rmon; /* 0 -> off, 1 -> on, -1 -> leave alone */ + /* Consumed by clocks_hunt.c */ + int8_t clk_mode; /* 0 -> off, 1 -> on, -1 -> leave alone */ + /* No longer used, superseded by TLV_TAG_DESCRIPTOR_CACHE_CONFIG. */ + int8_t rx_dc_size; /* -1 -> leave alone */ + int8_t tx_dc_size; + int16_t num_q_allocs; +}; + +/* Global port configuration + * + * This is now deprecated in favour of a platform-provided default + * and dynamic config override via tlv_global_port_options. + */ +#define TLV_TAG_GLOBAL_PORT_CONFIG (0x000a0000) + +struct tlv_global_port_config { + uint32_t tag; + uint32_t length; + uint32_t ports_per_core; + uint32_t max_port_speed; +}; + + +/* Firmware options. + * + * This is intended for user-configurable selection of optional firmware + * features and variants. + * + * Initially, this consists only of the satellite CPU firmware variant + * selection, but this tag could be extended in the future (using the + * tag length to determine whether additional fields are present). + */ + +#define TLV_TAG_FIRMWARE_OPTIONS (0x100b0000) + +struct tlv_firmware_options { + uint32_t tag; + uint32_t length; + uint32_t firmware_variant; +#define TLV_FIRMWARE_VARIANT_DRIVER_SELECTED (0xffffffff) + +/* These are the values for overriding the driver's choice; the definitions + * are taken from MCDI so that they don't get out of step. Include + * or the equivalent from your driver's tree if + * you need to use these constants. + */ +#define TLV_FIRMWARE_VARIANT_FULL_FEATURED MC_CMD_FW_FULL_FEATURED +#define TLV_FIRMWARE_VARIANT_LOW_LATENCY MC_CMD_FW_LOW_LATENCY +#define TLV_FIRMWARE_VARIANT_PACKED_STREAM MC_CMD_FW_PACKED_STREAM +#define TLV_FIRMWARE_VARIANT_HIGH_TX_RATE MC_CMD_FW_HIGH_TX_RATE +#define TLV_FIRMWARE_VARIANT_PACKED_STREAM_HASH_MODE_1 \ + MC_CMD_FW_PACKED_STREAM_HASH_MODE_1 +#define TLV_FIRMWARE_VARIANT_RULES_ENGINE MC_CMD_FW_RULES_ENGINE +}; + +/* Voltage settings + * + * Intended for boards with A0 silicon where the core voltage may + * need tweaking. Most likely set once when the pass voltage is + * determined. */ + +#define TLV_TAG_0V9_SETTINGS (0x000c0000) + +struct tlv_0v9_settings { + uint32_t tag; + uint32_t length; + uint16_t flags; /* Boards with high 0v9 settings may need active cooling */ +#define TLV_TAG_0V9_REQUIRES_FAN (1) + uint16_t target_voltage; /* In millivolts */ + /* Since the limits are meant to be centred to the target (and must at least + * contain it) they need setting as well. */ + uint16_t warn_low; /* In millivolts */ + uint16_t warn_high; /* In millivolts */ + uint16_t panic_low; /* In millivolts */ + uint16_t panic_high; /* In millivolts */ +}; + + +/* Clock configuration */ + +#define TLV_TAG_CLOCK_CONFIG (0x000d0000) /* legacy symbol - do not use */ +#define TLV_TAG_CLOCK_CONFIG_HUNT TLV_TAG_CLOCK_CONFIG + +struct tlv_clock_config { + uint32_t tag; + uint32_t length; + uint16_t clk_sys; /* MHz */ + uint16_t clk_dpcpu; /* MHz */ + uint16_t clk_icore; /* MHz */ + uint16_t clk_pcs; /* MHz */ +}; + +#define TLV_TAG_CLOCK_CONFIG_MEDFORD (0x00100000) + +struct tlv_clock_config_medford { + uint32_t tag; + uint32_t length; + uint16_t clk_sys; /* MHz */ + uint16_t clk_mc; /* MHz */ + uint16_t clk_rmon; /* MHz */ + uint16_t clk_vswitch; /* MHz */ + uint16_t clk_dpcpu; /* MHz */ + uint16_t clk_pcs; /* MHz */ +}; + + +/* EF10-style global pool of MAC addresses. + * + * There are addresses, starting at , which are + * contiguous. Firmware is responsible for allocating addresses from this + * pool to ports / PFs as appropriate. + */ + +#define TLV_TAG_GLOBAL_MAC (0x000e0000) + +struct tlv_global_mac { + uint32_t tag; + uint32_t length; + uint8_t base_address[6]; + uint16_t reserved1; + uint16_t count; + uint16_t reserved2; +}; + +#define TLV_TAG_ATB_0V9_TARGET (0x000f0000) /* legacy symbol - do not use */ +#define TLV_TAG_ATB_0V9_TARGET_HUNT TLV_TAG_ATB_0V9_TARGET + +/* The target value for the 0v9 power rail measured on-chip at the + * analogue test bus */ +struct tlv_0v9_atb_target { + uint32_t tag; + uint32_t length; + uint16_t millivolts; + uint16_t reserved; +}; + +/* Factory settings for amplitude calibration of the PCIE TX serdes */ +#define TLV_TAG_TX_PCIE_AMP_CONFIG (0x00220000) +struct tlv_pcie_tx_amp_config { + uint32_t tag; + uint32_t length; + uint8_t quad_tx_imp2k[4]; + uint8_t quad_tx_imp50[4]; + uint8_t lane_amp[16]; +}; + + +/* Global PCIe configuration, second revision. This represents the visible PFs + * by a bitmap rather than having the number of the highest visible one. As such + * it can (for a 16-PF chip) represent a superset of what TLV_TAG_GLOBAL_PCIE_CONFIG + * can and it should be used in place of that tag in future (but compatibility with + * the old tag will be left in the firmware indefinitely). */ + +#define TLV_TAG_GLOBAL_PCIE_CONFIG_R2 (0x10100000) + +struct tlv_pcie_config_r2 { + uint32_t tag; + uint32_t length; + uint16_t visible_pfs; /**< Bitmap of visible PFs */ + uint16_t pf_aper; /**< BIU aperture for PF BAR2 */ + uint16_t vf_aper; /**< BIU aperture for VF BAR0 */ + uint16_t int_aper; /**< BIU aperture for PF BAR4 and VF BAR2 */ +}; + +/* Dynamic port mode. + * + * Allows selecting alternate port configuration for platforms that support it + * (e.g. 1x40G vs 2x10G on Milano, 1x40G vs 4x10G on Medford). This affects the + * number of externally visible ports (and, hence, PF to port mapping), so must + * be done at boot time. + * + * This tag supercedes tlv_global_port_config. + */ + +#define TLV_TAG_GLOBAL_PORT_MODE (0x10110000) + +struct tlv_global_port_mode { + uint32_t tag; + uint32_t length; + uint32_t port_mode; +#define TLV_PORT_MODE_DEFAULT (0xffffffff) /* Default for given platform */ +#define TLV_PORT_MODE_10G (0) /* 10G, single SFP/10G-KR */ +#define TLV_PORT_MODE_40G (1) /* 40G, single QSFP/40G-KR */ +#define TLV_PORT_MODE_10G_10G (2) /* 2x10G, dual SFP/10G-KR or single QSFP */ +#define TLV_PORT_MODE_40G_40G (3) /* 40G + 40G, dual QSFP/40G-KR (Greenport, Medford) */ +#define TLV_PORT_MODE_10G_10G_10G_10G (4) /* 2x10G + 2x10G, quad SFP/10G-KR or dual QSFP (Greenport) */ +#define TLV_PORT_MODE_10G_10G_10G_10G_Q1 (4) /* 4x10G, single QSFP, cage 0 (Medford) */ +#define TLV_PORT_MODE_10G_10G_10G_10G_Q (5) /* 4x10G, single QSFP, cage 0 (Medford) OBSOLETE DO NOT USE */ +#define TLV_PORT_MODE_40G_10G_10G (6) /* 1x40G + 2x10G, dual QSFP (Greenport, Medford) */ +#define TLV_PORT_MODE_10G_10G_40G (7) /* 2x10G + 1x40G, dual QSFP (Greenport, Medford) */ +#define TLV_PORT_MODE_10G_10G_10G_10G_Q2 (8) /* 4x10G, single QSFP, cage 1 (Medford) */ +#define TLV_PORT_MODE_10G_10G_10G_10G_Q1_Q2 (9) /* 2x10G + 2x10G, dual QSFP (Medford) */ +#define TLV_PORT_MODE_MAX TLV_PORT_MODE_10G_10G_10G_10G_Q1_Q2 +}; + +/* Type of the v-switch created implicitly by the firmware */ + +#define TLV_TAG_VSWITCH_TYPE(port) (0x10120000 + (port)) + +struct tlv_vswitch_type { + uint32_t tag; + uint32_t length; + uint32_t vswitch_type; +#define TLV_VSWITCH_TYPE_DEFAULT (0xffffffff) /* Firmware default; equivalent to no TLV present for a given port */ +#define TLV_VSWITCH_TYPE_NONE (0) +#define TLV_VSWITCH_TYPE_VLAN (1) +#define TLV_VSWITCH_TYPE_VEB (2) +#define TLV_VSWITCH_TYPE_VEPA (3) +#define TLV_VSWITCH_TYPE_MUX (4) +#define TLV_VSWITCH_TYPE_TEST (5) +}; + +/* A VLAN tag for the v-port created implicitly by the firmware */ + +#define TLV_TAG_VPORT_VLAN_TAG(pf) (0x10130000 + (pf)) + +struct tlv_vport_vlan_tag { + uint32_t tag; + uint32_t length; + uint32_t vlan_tag; +#define TLV_VPORT_NO_VLAN_TAG (0xFFFFFFFF) /* Default in the absence of TLV for a given PF */ +}; + +/* Offset to be applied to the 0v9 setting, wherever it came from */ + +#define TLV_TAG_ATB_0V9_OFFSET (0x10140000) + +struct tlv_0v9_atb_offset { + uint32_t tag; + uint32_t length; + int16_t offset_millivolts; + uint16_t reserved; +}; + +/* A privilege mask given on reset to all non-admin PCIe functions (that is other than first-PF-per-port). + * The meaning of particular bits is defined in mcdi_ef10.yml under MC_CMD_PRIVILEGE_MASK, see also bug 44583. + * TLV_TAG_PRIVILEGE_MASK_ADD specifies bits that should be added (ORed) to firmware default while + * TLV_TAG_PRIVILEGE_MASK_REM specifies bits that should be removed (ANDed) from firmware default: + * Initial_privilege_mask = (firmware_default_mask | privilege_mask_add) & ~privilege_mask_rem */ + +#define TLV_TAG_PRIVILEGE_MASK (0x10150000) /* legacy symbol - do not use */ + +struct tlv_privilege_mask { /* legacy structure - do not use */ + uint32_t tag; + uint32_t length; + uint32_t privilege_mask; +}; + +#define TLV_TAG_PRIVILEGE_MASK_ADD (0x10150000) + +struct tlv_privilege_mask_add { + uint32_t tag; + uint32_t length; + uint32_t privilege_mask_add; +}; + +#define TLV_TAG_PRIVILEGE_MASK_REM (0x10160000) + +struct tlv_privilege_mask_rem { + uint32_t tag; + uint32_t length; + uint32_t privilege_mask_rem; +}; + +/* Additional privileges given to all PFs. + * This tag takes precedence over TLV_TAG_PRIVILEGE_MASK_REM. */ + +#define TLV_TAG_PRIVILEGE_MASK_ADD_ALL_PFS (0x10190000) + +struct tlv_privilege_mask_add_all_pfs { + uint32_t tag; + uint32_t length; + uint32_t privilege_mask_add; +}; + +/* Additional privileges given to a selected PF. + * This tag takes precedence over TLV_TAG_PRIVILEGE_MASK_REM. */ + +#define TLV_TAG_PRIVILEGE_MASK_ADD_SINGLE_PF(pf) (0x101A0000 + (pf)) + +struct tlv_privilege_mask_add_single_pf { + uint32_t tag; + uint32_t length; + uint32_t privilege_mask_add; +}; + +/* Turning on/off the PFIOV mode. + * This tag only takes effect if TLV_TAG_VSWITCH_TYPE is missing or set to DEFAULT. */ + +#define TLV_TAG_PFIOV(port) (0x10170000 + (port)) + +struct tlv_pfiov { + uint32_t tag; + uint32_t length; + uint32_t pfiov; +#define TLV_PFIOV_OFF (0) /* Default */ +#define TLV_PFIOV_ON (1) +}; + +/* Multicast filter chaining mode selection. + * + * When enabled, multicast packets are delivered to all recipients of all + * matching multicast filters, with the exception that IP multicast filters + * will steal traffic from MAC multicast filters on a per-function basis. + * (New behaviour.) + * + * When disabled, multicast packets will always be delivered only to the + * recipients of the highest priority matching multicast filter. + * (Legacy behaviour.) + * + * The DEFAULT mode (which is the same as the tag not being present at all) + * is equivalent to ENABLED in production builds, and DISABLED in eftest + * builds. + * + * This option is intended to provide run-time control over this feature + * while it is being stabilised and may be withdrawn at some point in the + * future; the new behaviour is intended to become the standard behaviour. + */ + +#define TLV_TAG_MCAST_FILTER_CHAINING (0x10180000) + +struct tlv_mcast_filter_chaining { + uint32_t tag; + uint32_t length; + uint32_t mode; +#define TLV_MCAST_FILTER_CHAINING_DEFAULT (0xffffffff) +#define TLV_MCAST_FILTER_CHAINING_DISABLED (0) +#define TLV_MCAST_FILTER_CHAINING_ENABLED (1) +}; + +/* Pacer rate limit per PF */ +#define TLV_TAG_RATE_LIMIT(pf) (0x101b0000 + (pf)) + +struct tlv_rate_limit { + uint32_t tag; + uint32_t length; + uint32_t rate_mbps; +}; + +/* OCSD Enable/Disable + * + * This setting allows OCSD to be disabled. This is a requirement for HP + * servers to support PCI passthrough for virtualization. + * + * The DEFAULT mode (which is the same as the tag not being present) is + * equivalent to ENABLED. + * + * This option is not used by the MCFW, and is entirely handled by the various + * drivers that support OCSD, by reading the setting before they attempt + * to enable OCSD. + * + * bit0: OCSD Disabled/Enabled + */ + +#define TLV_TAG_OCSD (0x101C0000) + +struct tlv_ocsd { + uint32_t tag; + uint32_t length; + uint32_t mode; +#define TLV_OCSD_DISABLED 0 +#define TLV_OCSD_ENABLED 1 /* Default */ +}; + +/* Descriptor cache config. + * + * Sets the sizes of the TX and RX descriptor caches as a power of 2. It also + * sets the total number of VIs. When the number of VIs is reduced VIs are taken + * away from the highest numbered port first, so a vi_count of 1024 means 1024 + * VIs on the first port and 0 on the second (on a Torino). + */ + +#define TLV_TAG_DESCRIPTOR_CACHE_CONFIG (0x101d0000) + +struct tlv_descriptor_cache_config { + uint32_t tag; + uint32_t length; + uint8_t rx_desc_cache_size; + uint8_t tx_desc_cache_size; + uint16_t vi_count; +}; +#define TLV_DESC_CACHE_DEFAULT (0xff) +#define TLV_VI_COUNT_DEFAULT (0xffff) + +/* RX event merging config (read batching). + * + * Sets the global maximum number of events for the merging bins, and the + * global timeout configuration for the bins. + */ + +#define TLV_TAG_RX_EVENT_MERGING_CONFIG (0x101e0000) + +struct tlv_rx_event_merging_config { + uint32_t tag; + uint32_t length; + uint32_t max_events; +#define TLV_RX_EVENT_MERGING_CONFIG_MAX_EVENTS_MAX ((1 << 4) - 1) + uint32_t timeout_ns; +}; +#define TLV_RX_EVENT_MERGING_MAX_EVENTS_DEFAULT (0xffffffff) +#define TLV_RX_EVENT_MERGING_TIMEOUT_NS_DEFAULT (0xffffffff) + +#define TLV_TAG_PCIE_LINK_SETTINGS (0x101f0000) +struct tlv_pcie_link_settings { + uint32_t tag; + uint32_t length; + uint16_t gen; /* Target PCIe generation: 1, 2, 3 */ + uint16_t width; /* Number of lanes */ +}; + +/* TX event merging config. + * + * Sets the global maximum number of events for the merging bins, and the + * global timeout configuration for the bins, and the global timeout for + * empty queues. + */ +#define TLV_TAG_TX_EVENT_MERGING_CONFIG (0x10210000) +struct tlv_tx_event_merging_config { + uint32_t tag; + uint32_t length; + uint32_t max_events; +#define TLV_TX_EVENT_MERGING_CONFIG_MAX_EVENTS_MAX ((1 << 4) - 1) + uint32_t timeout_ns; + uint32_t qempty_timeout_ns; /* Medford only */ +}; +#define TLV_TX_EVENT_MERGING_MAX_EVENTS_DEFAULT (0xffffffff) +#define TLV_TX_EVENT_MERGING_TIMEOUT_NS_DEFAULT (0xffffffff) +#define TLV_TX_EVENT_MERGING_QEMPTY_TIMEOUT_NS_DEFAULT (0xffffffff) + +#define TLV_TAG_LICENSE (0x30800000) + +typedef struct tlv_license { + uint32_t tag; + uint32_t length; + uint8_t data[]; +} tlv_license_t; + +/* TSA NIC IP address configuration + * + * Sets the TSA NIC IP address statically via configuration tool or dynamically + * via DHCP via snooping based on the mode selection (0=Static, 1=DHCP, 2=Snoop) + * + * NOTE: This TAG is temporarily placed in the dynamic config partition and will + * be moved to a private partition during TSA development. It is not used in any + * released code yet. + */ + +#define TLV_TAG_TMP_TSAN_CONFIG (0x10220000) + +#define TLV_TSAN_IP_MODE_STATIC (0) +#define TLV_TSAN_IP_MODE_DHCP (1) +#define TLV_TSAN_IP_MODE_SNOOP (2) +typedef struct tlv_tsan_config { + uint32_t tag; + uint32_t length; + uint32_t mode; + uint32_t ip; + uint32_t netmask; + uint32_t gateway; + uint32_t port; + uint32_t bind_retry; /* DEPRECATED */ + uint32_t bind_bkout; /* DEPRECATED */ +} tlv_tsan_config_t; + +/* TSA Controller IP address configuration + * + * Sets the TSA Controller IP address statically via configuration tool + * + * NOTE: This TAG is temporarily placed in the dynamic config partition and will + * be moved to a private partition during TSA development. It is not used in any + * released code yet. + */ + +#define TLV_TAG_TMP_TSAC_CONFIG (0x10230000) + +#define TLV_MAX_TSACS (4) +typedef struct tlv_tsac_config { + uint32_t tag; + uint32_t length; + uint32_t num_tsacs; + uint32_t ip[TLV_MAX_TSACS]; + uint32_t port[TLV_MAX_TSACS]; +} tlv_tsac_config_t; + +/* Binding ticket + * + * Sets the TSA NIC binding ticket used for binding process between the TSA NIC + * and the TSA Controller + * + * NOTE: This TAG is temporarily placed in the dynamic config partition and will + * be moved to a private partition during TSA development. It is not used in any + * released code yet. + */ + +#define TLV_TAG_TMP_BINDING_TICKET (0x10240000) + +typedef struct tlv_binding_ticket { + uint32_t tag; + uint32_t length; + uint8_t bytes[]; +} tlv_binding_ticket_t; + +/* Solarflare private key (DEPRECATED) + * + * Sets the Solareflare private key used for signing during the binding process + * + * NOTE: This TAG is temporarily placed in the dynamic config partition and will + * be moved to a private partition during TSA development. It is not used in any + * released code yet. + */ + +#define TLV_TAG_TMP_PIK_SF (0x10250000) /* DEPRECATED */ + +typedef struct tlv_pik_sf { + uint32_t tag; + uint32_t length; + uint8_t bytes[]; +} tlv_pik_sf_t; + +/* CA root certificate + * + * Sets the CA root certificate used for TSA Controller verfication during + * TLS connection setup between the TSA NIC and the TSA Controller + * + * NOTE: This TAG is temporarily placed in the dynamic config partition and will + * be moved to a private partition during TSA development. It is not used in any + * released code yet. + */ + +#define TLV_TAG_TMP_CA_ROOT_CERT (0x10260000) + +typedef struct tlv_ca_root_cert { + uint32_t tag; + uint32_t length; + uint8_t bytes[]; +} tlv_ca_root_cert_t; + +/* Tx vFIFO Low latency configuration + * + * To keep the desired booting behaviour for the switch, it just requires to + * know if the low latency mode is enabled. + */ + +#define TLV_TAG_TX_VFIFO_ULL_MODE (0x10270000) +struct tlv_tx_vfifo_ull_mode { + uint32_t tag; + uint32_t length; + uint8_t mode; +#define TLV_TX_VFIFO_ULL_MODE_DEFAULT 0 +}; + +/* BIU mode + * + * Medford2 tag for selecting VI window decode (see values below) + */ +#define TLV_TAG_BIU_VI_WINDOW_MODE (0x10280000) +struct tlv_biu_vi_window_mode { + uint32_t tag; + uint32_t length; + uint8_t mode; +#define TLV_BIU_VI_WINDOW_MODE_8K 0 /* 8k per VI, CTPIO not mapped, medford/hunt compatible */ +#define TLV_BIU_VI_WINDOW_MODE_16K 1 /* 16k per VI, CTPIO mapped */ +#define TLV_BIU_VI_WINDOW_MODE_64K 2 /* 64k per VI, CTPIO mapped, POWER-friendly */ +}; + +/* FastPD mode + * + * Medford2 tag for configuring the FastPD mode (see values below) + */ +#define TLV_TAG_FASTPD_MODE(port) (0x10290000 + (port)) +struct tlv_fastpd_mode { + uint32_t tag; + uint32_t length; + uint8_t mode; +#define TLV_FASTPD_MODE_SOFT_ALL 0 /* All packets to the SoftPD */ +#define TLV_FASTPD_MODE_FAST_ALL 1 /* All packets to the FastPD */ +#define TLV_FASTPD_MODE_FAST_SUPPORTED 2 /* Supported packet types to the FastPD; everything else to the SoftPD */ +}; + +#endif /* CI_MGMT_TLV_LAYOUT_H */ diff --git a/dpdk/drivers/net/sfc/base/ef10_tx.c b/dpdk/drivers/net/sfc/base/ef10_tx.c new file mode 100644 index 00000000..211d2655 --- /dev/null +++ b/dpdk/drivers/net/sfc/base/ef10_tx.c @@ -0,0 +1,722 @@ +/* + * Copyright (c) 2012-2016 Solarflare Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are + * those of the authors and should not be interpreted as representing official + * policies, either expressed or implied, of the FreeBSD Project. + */ + +#include "efx.h" +#include "efx_impl.h" + + +#if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD + +#if EFSYS_OPT_QSTATS +#define EFX_TX_QSTAT_INCR(_etp, _stat) \ + do { \ + (_etp)->et_stat[_stat]++; \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) +#else +#define EFX_TX_QSTAT_INCR(_etp, _stat) +#endif + +static __checkReturn efx_rc_t +efx_mcdi_init_txq( + __in efx_nic_t *enp, + __in uint32_t size, + __in uint32_t target_evq, + __in uint32_t label, + __in uint32_t instance, + __in uint16_t flags, + __in efsys_mem_t *esmp) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_INIT_TXQ_IN_LEN(EFX_TXQ_MAX_BUFS), + MC_CMD_INIT_TXQ_OUT_LEN)]; + efx_qword_t *dma_addr; + uint64_t addr; + int npages; + int i; + efx_rc_t rc; + + EFSYS_ASSERT(EFX_TXQ_MAX_BUFS >= + EFX_TXQ_NBUFS(enp->en_nic_cfg.enc_txq_max_ndescs)); + + npages = EFX_TXQ_NBUFS(size); + if (MC_CMD_INIT_TXQ_IN_LEN(npages) > sizeof (payload)) { + rc = EINVAL; + goto fail1; + } + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_INIT_TXQ; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_INIT_TXQ_IN_LEN(npages); + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_INIT_TXQ_OUT_LEN; + + MCDI_IN_SET_DWORD(req, INIT_TXQ_IN_SIZE, size); + MCDI_IN_SET_DWORD(req, INIT_TXQ_IN_TARGET_EVQ, target_evq); + MCDI_IN_SET_DWORD(req, INIT_TXQ_IN_LABEL, label); + MCDI_IN_SET_DWORD(req, INIT_TXQ_IN_INSTANCE, instance); + + MCDI_IN_POPULATE_DWORD_7(req, INIT_TXQ_IN_FLAGS, + INIT_TXQ_IN_FLAG_BUFF_MODE, 0, + INIT_TXQ_IN_FLAG_IP_CSUM_DIS, + (flags & EFX_TXQ_CKSUM_IPV4) ? 0 : 1, + INIT_TXQ_IN_FLAG_TCP_CSUM_DIS, + (flags & EFX_TXQ_CKSUM_TCPUDP) ? 0 : 1, + INIT_TXQ_EXT_IN_FLAG_TSOV2_EN, (flags & EFX_TXQ_FATSOV2) ? 1 : 0, + INIT_TXQ_IN_FLAG_TCP_UDP_ONLY, 0, + INIT_TXQ_IN_CRC_MODE, 0, + INIT_TXQ_IN_FLAG_TIMESTAMP, 0); + + MCDI_IN_SET_DWORD(req, INIT_TXQ_IN_OWNER_ID, 0); + MCDI_IN_SET_DWORD(req, INIT_TXQ_IN_PORT_ID, EVB_PORT_ID_ASSIGNED); + + dma_addr = MCDI_IN2(req, efx_qword_t, INIT_TXQ_IN_DMA_ADDR); + addr = EFSYS_MEM_ADDR(esmp); + + for (i = 0; i < npages; i++) { + EFX_POPULATE_QWORD_2(*dma_addr, + EFX_DWORD_1, (uint32_t)(addr >> 32), + EFX_DWORD_0, (uint32_t)(addr & 0xffffffff)); + + dma_addr++; + addr += EFX_BUF_SIZE; + } + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail2; + } + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +static __checkReturn efx_rc_t +efx_mcdi_fini_txq( + __in efx_nic_t *enp, + __in uint32_t instance) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_FINI_TXQ_IN_LEN, + MC_CMD_FINI_TXQ_OUT_LEN)]; + efx_rc_t rc; + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_FINI_TXQ; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_FINI_TXQ_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_FINI_TXQ_OUT_LEN; + + MCDI_IN_SET_DWORD(req, FINI_TXQ_IN_INSTANCE, instance); + + efx_mcdi_execute_quiet(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + return (0); + +fail1: + /* + * EALREADY is not an error, but indicates that the MC has rebooted and + * that the TXQ has already been destroyed. + */ + if (rc != EALREADY) + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +ef10_tx_init( + __in efx_nic_t *enp) +{ + _NOTE(ARGUNUSED(enp)) + return (0); +} + + void +ef10_tx_fini( + __in efx_nic_t *enp) +{ + _NOTE(ARGUNUSED(enp)) +} + + __checkReturn efx_rc_t +ef10_tx_qcreate( + __in efx_nic_t *enp, + __in unsigned int index, + __in unsigned int label, + __in efsys_mem_t *esmp, + __in size_t n, + __in uint32_t id, + __in uint16_t flags, + __in efx_evq_t *eep, + __in efx_txq_t *etp, + __out unsigned int *addedp) +{ + efx_qword_t desc; + efx_rc_t rc; + + _NOTE(ARGUNUSED(id)) + + if ((rc = efx_mcdi_init_txq(enp, n, eep->ee_index, label, index, flags, + esmp)) != 0) + goto fail1; + + /* + * A previous user of this TX queue may have written a descriptor to the + * TX push collector, but not pushed the doorbell (e.g. after a crash). + * The next doorbell write would then push the stale descriptor. + * + * Ensure the (per network port) TX push collector is cleared by writing + * a no-op TX option descriptor. See bug29981 for details. + */ + *addedp = 1; + EFX_POPULATE_QWORD_4(desc, + ESF_DZ_TX_DESC_IS_OPT, 1, + ESF_DZ_TX_OPTION_TYPE, ESE_DZ_TX_OPTION_DESC_CRC_CSUM, + ESF_DZ_TX_OPTION_UDP_TCP_CSUM, + (flags & EFX_TXQ_CKSUM_TCPUDP) ? 1 : 0, + ESF_DZ_TX_OPTION_IP_CSUM, + (flags & EFX_TXQ_CKSUM_IPV4) ? 1 : 0); + + EFSYS_MEM_WRITEQ(etp->et_esmp, 0, &desc); + ef10_tx_qpush(etp, *addedp, 0); + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + void +ef10_tx_qdestroy( + __in efx_txq_t *etp) +{ + /* FIXME */ + _NOTE(ARGUNUSED(etp)) + /* FIXME */ +} + + __checkReturn efx_rc_t +ef10_tx_qpio_enable( + __in efx_txq_t *etp) +{ + efx_nic_t *enp = etp->et_enp; + efx_piobuf_handle_t handle; + efx_rc_t rc; + + if (etp->et_pio_size != 0) { + rc = EALREADY; + goto fail1; + } + + /* Sub-allocate a PIO block from a piobuf */ + if ((rc = ef10_nic_pio_alloc(enp, + &etp->et_pio_bufnum, + &handle, + &etp->et_pio_blknum, + &etp->et_pio_offset, + &etp->et_pio_size)) != 0) { + goto fail2; + } + EFSYS_ASSERT3U(etp->et_pio_size, !=, 0); + + /* Link the piobuf to this TXQ */ + if ((rc = ef10_nic_pio_link(enp, etp->et_index, handle)) != 0) { + goto fail3; + } + + /* + * et_pio_offset is the offset of the sub-allocated block within the + * hardware PIO buffer. It is used as the buffer address in the PIO + * option descriptor. + * + * et_pio_write_offset is the offset of the sub-allocated block from the + * start of the write-combined memory mapping, and is used for writing + * data into the PIO buffer. + */ + etp->et_pio_write_offset = + (etp->et_pio_bufnum * ER_DZ_TX_PIOBUF_STEP) + + ER_DZ_TX_PIOBUF_OFST + etp->et_pio_offset; + + return (0); + +fail3: + EFSYS_PROBE(fail3); + ef10_nic_pio_free(enp, etp->et_pio_bufnum, etp->et_pio_blknum); +fail2: + EFSYS_PROBE(fail2); + etp->et_pio_size = 0; +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + void +ef10_tx_qpio_disable( + __in efx_txq_t *etp) +{ + efx_nic_t *enp = etp->et_enp; + + if (etp->et_pio_size != 0) { + /* Unlink the piobuf from this TXQ */ + ef10_nic_pio_unlink(enp, etp->et_index); + + /* Free the sub-allocated PIO block */ + ef10_nic_pio_free(enp, etp->et_pio_bufnum, etp->et_pio_blknum); + etp->et_pio_size = 0; + etp->et_pio_write_offset = 0; + } +} + + __checkReturn efx_rc_t +ef10_tx_qpio_write( + __in efx_txq_t *etp, + __in_ecount(length) uint8_t *buffer, + __in size_t length, + __in size_t offset) +{ + efx_nic_t *enp = etp->et_enp; + efsys_bar_t *esbp = enp->en_esbp; + uint32_t write_offset; + uint32_t write_offset_limit; + efx_qword_t *eqp; + efx_rc_t rc; + + EFSYS_ASSERT(length % sizeof (efx_qword_t) == 0); + + if (etp->et_pio_size == 0) { + rc = ENOENT; + goto fail1; + } + if (offset + length > etp->et_pio_size) { + rc = ENOSPC; + goto fail2; + } + + /* + * Writes to PIO buffers must be 64 bit aligned, and multiples of + * 64 bits. + */ + write_offset = etp->et_pio_write_offset + offset; + write_offset_limit = write_offset + length; + eqp = (efx_qword_t *)buffer; + while (write_offset < write_offset_limit) { + EFSYS_BAR_WC_WRITEQ(esbp, write_offset, eqp); + eqp++; + write_offset += sizeof (efx_qword_t); + } + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +ef10_tx_qpio_post( + __in efx_txq_t *etp, + __in size_t pkt_length, + __in unsigned int completed, + __inout unsigned int *addedp) +{ + efx_qword_t pio_desc; + unsigned int id; + size_t offset; + unsigned int added = *addedp; + efx_rc_t rc; + + + if (added - completed + 1 > EFX_TXQ_LIMIT(etp->et_mask + 1)) { + rc = ENOSPC; + goto fail1; + } + + if (etp->et_pio_size == 0) { + rc = ENOENT; + goto fail2; + } + + id = added++ & etp->et_mask; + offset = id * sizeof (efx_qword_t); + + EFSYS_PROBE4(tx_pio_post, unsigned int, etp->et_index, + unsigned int, id, uint32_t, etp->et_pio_offset, + size_t, pkt_length); + + EFX_POPULATE_QWORD_5(pio_desc, + ESF_DZ_TX_DESC_IS_OPT, 1, + ESF_DZ_TX_OPTION_TYPE, 1, + ESF_DZ_TX_PIO_CONT, 0, + ESF_DZ_TX_PIO_BYTE_CNT, pkt_length, + ESF_DZ_TX_PIO_BUF_ADDR, etp->et_pio_offset); + + EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &pio_desc); + + EFX_TX_QSTAT_INCR(etp, TX_POST_PIO); + + *addedp = added; + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +ef10_tx_qpost( + __in efx_txq_t *etp, + __in_ecount(n) efx_buffer_t *eb, + __in unsigned int n, + __in unsigned int completed, + __inout unsigned int *addedp) +{ + unsigned int added = *addedp; + unsigned int i; + efx_rc_t rc; + + if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1)) { + rc = ENOSPC; + goto fail1; + } + + for (i = 0; i < n; i++) { + efx_buffer_t *ebp = &eb[i]; + efsys_dma_addr_t addr = ebp->eb_addr; + size_t size = ebp->eb_size; + boolean_t eop = ebp->eb_eop; + unsigned int id; + size_t offset; + efx_qword_t qword; + + /* No limitations on boundary crossing */ + EFSYS_ASSERT(size <= + etp->et_enp->en_nic_cfg.enc_tx_dma_desc_size_max); + + id = added++ & etp->et_mask; + offset = id * sizeof (efx_qword_t); + + EFSYS_PROBE5(tx_post, unsigned int, etp->et_index, + unsigned int, id, efsys_dma_addr_t, addr, + size_t, size, boolean_t, eop); + + EFX_POPULATE_QWORD_5(qword, + ESF_DZ_TX_KER_TYPE, 0, + ESF_DZ_TX_KER_CONT, (eop) ? 0 : 1, + ESF_DZ_TX_KER_BYTE_CNT, (uint32_t)(size), + ESF_DZ_TX_KER_BUF_ADDR_DW0, (uint32_t)(addr & 0xffffffff), + ESF_DZ_TX_KER_BUF_ADDR_DW1, (uint32_t)(addr >> 32)); + + EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &qword); + } + + EFX_TX_QSTAT_INCR(etp, TX_POST); + + *addedp = added; + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +/* + * This improves performance by pushing a TX descriptor at the same time as the + * doorbell. The descriptor must be added to the TXQ, so that can be used if the + * hardware decides not to use the pushed descriptor. + */ + void +ef10_tx_qpush( + __in efx_txq_t *etp, + __in unsigned int added, + __in unsigned int pushed) +{ + efx_nic_t *enp = etp->et_enp; + unsigned int wptr; + unsigned int id; + size_t offset; + efx_qword_t desc; + efx_oword_t oword; + + wptr = added & etp->et_mask; + id = pushed & etp->et_mask; + offset = id * sizeof (efx_qword_t); + + EFSYS_MEM_READQ(etp->et_esmp, offset, &desc); + EFX_POPULATE_OWORD_3(oword, + ERF_DZ_TX_DESC_WPTR, wptr, + ERF_DZ_TX_DESC_HWORD, EFX_QWORD_FIELD(desc, EFX_DWORD_1), + ERF_DZ_TX_DESC_LWORD, EFX_QWORD_FIELD(desc, EFX_DWORD_0)); + + /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */ + EFX_DMA_SYNC_QUEUE_FOR_DEVICE(etp->et_esmp, etp->et_mask + 1, wptr, id); + EFSYS_PIO_WRITE_BARRIER(); + EFX_BAR_TBL_DOORBELL_WRITEO(enp, ER_DZ_TX_DESC_UPD_REG, etp->et_index, + &oword); +} + + __checkReturn efx_rc_t +ef10_tx_qdesc_post( + __in efx_txq_t *etp, + __in_ecount(n) efx_desc_t *ed, + __in unsigned int n, + __in unsigned int completed, + __inout unsigned int *addedp) +{ + unsigned int added = *addedp; + unsigned int i; + efx_rc_t rc; + + if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1)) { + rc = ENOSPC; + goto fail1; + } + + for (i = 0; i < n; i++) { + efx_desc_t *edp = &ed[i]; + unsigned int id; + size_t offset; + + id = added++ & etp->et_mask; + offset = id * sizeof (efx_desc_t); + + EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &edp->ed_eq); + } + + EFSYS_PROBE3(tx_desc_post, unsigned int, etp->et_index, + unsigned int, added, unsigned int, n); + + EFX_TX_QSTAT_INCR(etp, TX_POST); + + *addedp = added; + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + void +ef10_tx_qdesc_dma_create( + __in efx_txq_t *etp, + __in efsys_dma_addr_t addr, + __in size_t size, + __in boolean_t eop, + __out efx_desc_t *edp) +{ + /* No limitations on boundary crossing */ + EFSYS_ASSERT(size <= etp->et_enp->en_nic_cfg.enc_tx_dma_desc_size_max); + + EFSYS_PROBE4(tx_desc_dma_create, unsigned int, etp->et_index, + efsys_dma_addr_t, addr, + size_t, size, boolean_t, eop); + + EFX_POPULATE_QWORD_5(edp->ed_eq, + ESF_DZ_TX_KER_TYPE, 0, + ESF_DZ_TX_KER_CONT, (eop) ? 0 : 1, + ESF_DZ_TX_KER_BYTE_CNT, (uint32_t)(size), + ESF_DZ_TX_KER_BUF_ADDR_DW0, (uint32_t)(addr & 0xffffffff), + ESF_DZ_TX_KER_BUF_ADDR_DW1, (uint32_t)(addr >> 32)); +} + + void +ef10_tx_qdesc_tso_create( + __in efx_txq_t *etp, + __in uint16_t ipv4_id, + __in uint32_t tcp_seq, + __in uint8_t tcp_flags, + __out efx_desc_t *edp) +{ + EFSYS_PROBE4(tx_desc_tso_create, unsigned int, etp->et_index, + uint16_t, ipv4_id, uint32_t, tcp_seq, + uint8_t, tcp_flags); + + EFX_POPULATE_QWORD_5(edp->ed_eq, + ESF_DZ_TX_DESC_IS_OPT, 1, + ESF_DZ_TX_OPTION_TYPE, + ESE_DZ_TX_OPTION_DESC_TSO, + ESF_DZ_TX_TSO_TCP_FLAGS, tcp_flags, + ESF_DZ_TX_TSO_IP_ID, ipv4_id, + ESF_DZ_TX_TSO_TCP_SEQNO, tcp_seq); +} + + void +ef10_tx_qdesc_tso2_create( + __in efx_txq_t *etp, + __in uint16_t ipv4_id, + __in uint32_t tcp_seq, + __in uint16_t tcp_mss, + __out_ecount(count) efx_desc_t *edp, + __in int count) +{ + EFSYS_PROBE4(tx_desc_tso2_create, unsigned int, etp->et_index, + uint16_t, ipv4_id, uint32_t, tcp_seq, + uint16_t, tcp_mss); + + EFSYS_ASSERT(count >= EFX_TX_FATSOV2_OPT_NDESCS); + + EFX_POPULATE_QWORD_5(edp[0].ed_eq, + ESF_DZ_TX_DESC_IS_OPT, 1, + ESF_DZ_TX_OPTION_TYPE, + ESE_DZ_TX_OPTION_DESC_TSO, + ESF_DZ_TX_TSO_OPTION_TYPE, + ESE_DZ_TX_TSO_OPTION_DESC_FATSO2A, + ESF_DZ_TX_TSO_IP_ID, ipv4_id, + ESF_DZ_TX_TSO_TCP_SEQNO, tcp_seq); + EFX_POPULATE_QWORD_4(edp[1].ed_eq, + ESF_DZ_TX_DESC_IS_OPT, 1, + ESF_DZ_TX_OPTION_TYPE, + ESE_DZ_TX_OPTION_DESC_TSO, + ESF_DZ_TX_TSO_OPTION_TYPE, + ESE_DZ_TX_TSO_OPTION_DESC_FATSO2B, + ESF_DZ_TX_TSO_TCP_MSS, tcp_mss); +} + + void +ef10_tx_qdesc_vlantci_create( + __in efx_txq_t *etp, + __in uint16_t tci, + __out efx_desc_t *edp) +{ + EFSYS_PROBE2(tx_desc_vlantci_create, unsigned int, etp->et_index, + uint16_t, tci); + + EFX_POPULATE_QWORD_4(edp->ed_eq, + ESF_DZ_TX_DESC_IS_OPT, 1, + ESF_DZ_TX_OPTION_TYPE, + ESE_DZ_TX_OPTION_DESC_VLAN, + ESF_DZ_TX_VLAN_OP, tci ? 1 : 0, + ESF_DZ_TX_VLAN_TAG1, tci); +} + + + __checkReturn efx_rc_t +ef10_tx_qpace( + __in efx_txq_t *etp, + __in unsigned int ns) +{ + efx_rc_t rc; + + /* FIXME */ + _NOTE(ARGUNUSED(etp, ns)) + _NOTE(CONSTANTCONDITION) + if (B_FALSE) { + rc = ENOTSUP; + goto fail1; + } + /* FIXME */ + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +ef10_tx_qflush( + __in efx_txq_t *etp) +{ + efx_nic_t *enp = etp->et_enp; + efx_rc_t rc; + + if ((rc = efx_mcdi_fini_txq(enp, etp->et_index)) != 0) + goto fail1; + + return (0); + +fail1: + /* + * EALREADY is not an error, but indicates that the MC has rebooted and + * that the TXQ has already been destroyed. Callers need to know that + * the TXQ flush has completed to avoid waiting until timeout for a + * flush done event that will not be delivered. + */ + if (rc != EALREADY) + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + void +ef10_tx_qenable( + __in efx_txq_t *etp) +{ + /* FIXME */ + _NOTE(ARGUNUSED(etp)) + /* FIXME */ +} + +#if EFSYS_OPT_QSTATS + void +ef10_tx_qstats_update( + __in efx_txq_t *etp, + __inout_ecount(TX_NQSTATS) efsys_stat_t *stat) +{ + unsigned int id; + + for (id = 0; id < TX_NQSTATS; id++) { + efsys_stat_t *essp = &stat[id]; + + EFSYS_STAT_INCR(essp, etp->et_stat[id]); + etp->et_stat[id] = 0; + } +} + +#endif /* EFSYS_OPT_QSTATS */ + +#endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */ diff --git a/dpdk/drivers/net/sfc/base/ef10_vpd.c b/dpdk/drivers/net/sfc/base/ef10_vpd.c new file mode 100644 index 00000000..71123a90 --- /dev/null +++ b/dpdk/drivers/net/sfc/base/ef10_vpd.c @@ -0,0 +1,463 @@ +/* + * Copyright (c) 2009-2016 Solarflare Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are + * those of the authors and should not be interpreted as representing official + * policies, either expressed or implied, of the FreeBSD Project. + */ + +#include "efx.h" +#include "efx_impl.h" + + +#if EFSYS_OPT_VPD + +#if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD + +#include "ef10_tlv_layout.h" + + __checkReturn efx_rc_t +ef10_vpd_init( + __in efx_nic_t *enp) +{ + caddr_t svpd; + size_t svpd_size; + uint32_t pci_pf; + uint32_t tag; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE); + EFSYS_ASSERT(enp->en_family == EFX_FAMILY_HUNTINGTON || + enp->en_family == EFX_FAMILY_MEDFORD); + + if (enp->en_nic_cfg.enc_vpd_is_global) { + tag = TLV_TAG_GLOBAL_STATIC_VPD; + } else { + pci_pf = enp->en_nic_cfg.enc_pf; + tag = TLV_TAG_PF_STATIC_VPD(pci_pf); + } + + /* + * The VPD interface exposes VPD resources from the combined static and + * dynamic VPD storage. As the static VPD configuration should *never* + * change, we can cache it. + */ + svpd = NULL; + svpd_size = 0; + rc = ef10_nvram_partn_read_tlv(enp, + NVRAM_PARTITION_TYPE_STATIC_CONFIG, + tag, &svpd, &svpd_size); + if (rc != 0) { + if (rc == EACCES) { + /* Unprivileged functions cannot access VPD */ + goto out; + } + goto fail1; + } + + if (svpd != NULL && svpd_size > 0) { + if ((rc = efx_vpd_hunk_verify(svpd, svpd_size, NULL)) != 0) + goto fail2; + } + + enp->en_arch.ef10.ena_svpd = svpd; + enp->en_arch.ef10.ena_svpd_length = svpd_size; + +out: + return (0); + +fail2: + EFSYS_PROBE(fail2); + + EFSYS_KMEM_FREE(enp->en_esip, svpd_size, svpd); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +ef10_vpd_size( + __in efx_nic_t *enp, + __out size_t *sizep) +{ + efx_rc_t rc; + + EFSYS_ASSERT(enp->en_family == EFX_FAMILY_HUNTINGTON || + enp->en_family == EFX_FAMILY_MEDFORD); + + /* + * This function returns the total size the user should allocate + * for all VPD operations. We've already cached the static vpd, + * so we just need to return an upper bound on the dynamic vpd, + * which is the size of the DYNAMIC_CONFIG partition. + */ + if ((rc = efx_mcdi_nvram_info(enp, NVRAM_PARTITION_TYPE_DYNAMIC_CONFIG, + sizep, NULL, NULL, NULL)) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +ef10_vpd_read( + __in efx_nic_t *enp, + __out_bcount(size) caddr_t data, + __in size_t size) +{ + caddr_t dvpd; + size_t dvpd_size; + uint32_t pci_pf; + uint32_t tag; + efx_rc_t rc; + + EFSYS_ASSERT(enp->en_family == EFX_FAMILY_HUNTINGTON || + enp->en_family == EFX_FAMILY_MEDFORD); + + if (enp->en_nic_cfg.enc_vpd_is_global) { + tag = TLV_TAG_GLOBAL_DYNAMIC_VPD; + } else { + pci_pf = enp->en_nic_cfg.enc_pf; + tag = TLV_TAG_PF_DYNAMIC_VPD(pci_pf); + } + + if ((rc = ef10_nvram_partn_read_tlv(enp, + NVRAM_PARTITION_TYPE_DYNAMIC_CONFIG, + tag, &dvpd, &dvpd_size)) != 0) + goto fail1; + + if (dvpd_size > size) { + rc = ENOSPC; + goto fail2; + } + memcpy(data, dvpd, dvpd_size); + + /* Pad data with all-1s, consistent with update operations */ + memset(data + dvpd_size, 0xff, size - dvpd_size); + + EFSYS_KMEM_FREE(enp->en_esip, dvpd_size, dvpd); + + return (0); + +fail2: + EFSYS_PROBE(fail2); + + EFSYS_KMEM_FREE(enp->en_esip, dvpd_size, dvpd); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +ef10_vpd_verify( + __in efx_nic_t *enp, + __in_bcount(size) caddr_t data, + __in size_t size) +{ + efx_vpd_tag_t stag; + efx_vpd_tag_t dtag; + efx_vpd_keyword_t skey; + efx_vpd_keyword_t dkey; + unsigned int scont; + unsigned int dcont; + efx_rc_t rc; + + EFSYS_ASSERT(enp->en_family == EFX_FAMILY_HUNTINGTON || + enp->en_family == EFX_FAMILY_MEDFORD); + + /* + * Strictly you could take the view that dynamic vpd is optional. + * Instead, to conform more closely to the read/verify/reinit() + * paradigm, we require dynamic vpd. ef10_vpd_reinit() will + * reinitialize it as required. + */ + if ((rc = efx_vpd_hunk_verify(data, size, NULL)) != 0) + goto fail1; + + /* + * Verify that there is no duplication between the static and + * dynamic cfg sectors. + */ + if (enp->en_arch.ef10.ena_svpd_length == 0) + goto done; + + dcont = 0; + _NOTE(CONSTANTCONDITION) + while (1) { + if ((rc = efx_vpd_hunk_next(data, size, &dtag, + &dkey, NULL, NULL, &dcont)) != 0) + goto fail2; + if (dcont == 0) + break; + + /* + * Skip the RV keyword. It should be present in both the static + * and dynamic cfg sectors. + */ + if (dtag == EFX_VPD_RO && dkey == EFX_VPD_KEYWORD('R', 'V')) + continue; + + scont = 0; + _NOTE(CONSTANTCONDITION) + while (1) { + if ((rc = efx_vpd_hunk_next( + enp->en_arch.ef10.ena_svpd, + enp->en_arch.ef10.ena_svpd_length, &stag, &skey, + NULL, NULL, &scont)) != 0) + goto fail3; + if (scont == 0) + break; + + if (stag == dtag && skey == dkey) { + rc = EEXIST; + goto fail4; + } + } + } + +done: + return (0); + +fail4: + EFSYS_PROBE(fail4); +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +ef10_vpd_reinit( + __in efx_nic_t *enp, + __in_bcount(size) caddr_t data, + __in size_t size) +{ + boolean_t wantpid; + efx_rc_t rc; + + /* + * Only create an ID string if the dynamic cfg doesn't have one + */ + if (enp->en_arch.ef10.ena_svpd_length == 0) + wantpid = B_TRUE; + else { + unsigned int offset; + uint8_t length; + + rc = efx_vpd_hunk_get(enp->en_arch.ef10.ena_svpd, + enp->en_arch.ef10.ena_svpd_length, + EFX_VPD_ID, 0, &offset, &length); + if (rc == 0) + wantpid = B_FALSE; + else if (rc == ENOENT) + wantpid = B_TRUE; + else + goto fail1; + } + + if ((rc = efx_vpd_hunk_reinit(data, size, wantpid)) != 0) + goto fail2; + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +ef10_vpd_get( + __in efx_nic_t *enp, + __in_bcount(size) caddr_t data, + __in size_t size, + __inout efx_vpd_value_t *evvp) +{ + unsigned int offset; + uint8_t length; + efx_rc_t rc; + + EFSYS_ASSERT(enp->en_family == EFX_FAMILY_HUNTINGTON || + enp->en_family == EFX_FAMILY_MEDFORD); + + /* Attempt to satisfy the request from svpd first */ + if (enp->en_arch.ef10.ena_svpd_length > 0) { + if ((rc = efx_vpd_hunk_get(enp->en_arch.ef10.ena_svpd, + enp->en_arch.ef10.ena_svpd_length, evvp->evv_tag, + evvp->evv_keyword, &offset, &length)) == 0) { + evvp->evv_length = length; + memcpy(evvp->evv_value, + enp->en_arch.ef10.ena_svpd + offset, length); + return (0); + } else if (rc != ENOENT) + goto fail1; + } + + /* And then from the provided data buffer */ + if ((rc = efx_vpd_hunk_get(data, size, evvp->evv_tag, + evvp->evv_keyword, &offset, &length)) != 0) { + if (rc == ENOENT) + return (rc); + goto fail2; + } + + evvp->evv_length = length; + memcpy(evvp->evv_value, data + offset, length); + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +ef10_vpd_set( + __in efx_nic_t *enp, + __in_bcount(size) caddr_t data, + __in size_t size, + __in efx_vpd_value_t *evvp) +{ + efx_rc_t rc; + + EFSYS_ASSERT(enp->en_family == EFX_FAMILY_HUNTINGTON || + enp->en_family == EFX_FAMILY_MEDFORD); + + /* If the provided (tag,keyword) exists in svpd, then it is readonly */ + if (enp->en_arch.ef10.ena_svpd_length > 0) { + unsigned int offset; + uint8_t length; + + if ((rc = efx_vpd_hunk_get(enp->en_arch.ef10.ena_svpd, + enp->en_arch.ef10.ena_svpd_length, evvp->evv_tag, + evvp->evv_keyword, &offset, &length)) == 0) { + rc = EACCES; + goto fail1; + } + } + + if ((rc = efx_vpd_hunk_set(data, size, evvp)) != 0) + goto fail2; + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +ef10_vpd_next( + __in efx_nic_t *enp, + __in_bcount(size) caddr_t data, + __in size_t size, + __out efx_vpd_value_t *evvp, + __inout unsigned int *contp) +{ + _NOTE(ARGUNUSED(enp, data, size, evvp, contp)) + + return (ENOTSUP); +} + + __checkReturn efx_rc_t +ef10_vpd_write( + __in efx_nic_t *enp, + __in_bcount(size) caddr_t data, + __in size_t size) +{ + size_t vpd_length; + uint32_t pci_pf; + uint32_t tag; + efx_rc_t rc; + + EFSYS_ASSERT(enp->en_family == EFX_FAMILY_HUNTINGTON || + enp->en_family == EFX_FAMILY_MEDFORD); + + if (enp->en_nic_cfg.enc_vpd_is_global) { + tag = TLV_TAG_GLOBAL_DYNAMIC_VPD; + } else { + pci_pf = enp->en_nic_cfg.enc_pf; + tag = TLV_TAG_PF_DYNAMIC_VPD(pci_pf); + } + + /* Determine total length of new dynamic VPD */ + if ((rc = efx_vpd_hunk_length(data, size, &vpd_length)) != 0) + goto fail1; + + /* Store new dynamic VPD in all segments in DYNAMIC_CONFIG partition */ + if ((rc = ef10_nvram_partn_write_segment_tlv(enp, + NVRAM_PARTITION_TYPE_DYNAMIC_CONFIG, + tag, data, vpd_length, B_TRUE)) != 0) { + goto fail2; + } + + return (0); + +fail2: + EFSYS_PROBE(fail2); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + void +ef10_vpd_fini( + __in efx_nic_t *enp) +{ + EFSYS_ASSERT(enp->en_family == EFX_FAMILY_HUNTINGTON || + enp->en_family == EFX_FAMILY_MEDFORD); + + if (enp->en_arch.ef10.ena_svpd_length > 0) { + EFSYS_KMEM_FREE(enp->en_esip, enp->en_arch.ef10.ena_svpd_length, + enp->en_arch.ef10.ena_svpd); + + enp->en_arch.ef10.ena_svpd = NULL; + enp->en_arch.ef10.ena_svpd_length = 0; + } +} + +#endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */ + +#endif /* EFSYS_OPT_VPD */ diff --git a/dpdk/drivers/net/sfc/base/efx.h b/dpdk/drivers/net/sfc/base/efx.h new file mode 100644 index 00000000..57fba052 --- /dev/null +++ b/dpdk/drivers/net/sfc/base/efx.h @@ -0,0 +1,2596 @@ +/* + * Copyright (c) 2006-2016 Solarflare Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are + * those of the authors and should not be interpreted as representing official + * policies, either expressed or implied, of the FreeBSD Project. + */ + +#ifndef _SYS_EFX_H +#define _SYS_EFX_H + +#include "efsys.h" +#include "efx_check.h" +#include "efx_phy_ids.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define EFX_STATIC_ASSERT(_cond) \ + ((void)sizeof(char[(_cond) ? 1 : -1])) + +#define EFX_ARRAY_SIZE(_array) \ + (sizeof(_array) / sizeof((_array)[0])) + +#define EFX_FIELD_OFFSET(_type, _field) \ + ((size_t) &(((_type *)0)->_field)) + +/* Return codes */ + +typedef __success(return == 0) int efx_rc_t; + + +/* Chip families */ + +typedef enum efx_family_e { + EFX_FAMILY_INVALID, + EFX_FAMILY_FALCON, /* Obsolete and not supported */ + EFX_FAMILY_SIENA, + EFX_FAMILY_HUNTINGTON, + EFX_FAMILY_MEDFORD, + EFX_FAMILY_NTYPES +} efx_family_t; + +extern __checkReturn efx_rc_t +efx_family( + __in uint16_t venid, + __in uint16_t devid, + __out efx_family_t *efp); + + +#define EFX_PCI_VENID_SFC 0x1924 + +#define EFX_PCI_DEVID_FALCON 0x0710 /* SFC4000 */ + +#define EFX_PCI_DEVID_BETHPAGE 0x0803 /* SFC9020 */ +#define EFX_PCI_DEVID_SIENA 0x0813 /* SFL9021 */ +#define EFX_PCI_DEVID_SIENA_F1_UNINIT 0x0810 + +#define EFX_PCI_DEVID_HUNTINGTON_PF_UNINIT 0x0901 +#define EFX_PCI_DEVID_FARMINGDALE 0x0903 /* SFC9120 PF */ +#define EFX_PCI_DEVID_GREENPORT 0x0923 /* SFC9140 PF */ + +#define EFX_PCI_DEVID_FARMINGDALE_VF 0x1903 /* SFC9120 VF */ +#define EFX_PCI_DEVID_GREENPORT_VF 0x1923 /* SFC9140 VF */ + +#define EFX_PCI_DEVID_MEDFORD_PF_UNINIT 0x0913 +#define EFX_PCI_DEVID_MEDFORD 0x0A03 /* SFC9240 PF */ +#define EFX_PCI_DEVID_MEDFORD_VF 0x1A03 /* SFC9240 VF */ + +#define EFX_MEM_BAR 2 + +/* Error codes */ + +enum { + EFX_ERR_INVALID, + EFX_ERR_SRAM_OOB, + EFX_ERR_BUFID_DC_OOB, + EFX_ERR_MEM_PERR, + EFX_ERR_RBUF_OWN, + EFX_ERR_TBUF_OWN, + EFX_ERR_RDESQ_OWN, + EFX_ERR_TDESQ_OWN, + EFX_ERR_EVQ_OWN, + EFX_ERR_EVFF_OFLO, + EFX_ERR_ILL_ADDR, + EFX_ERR_SRAM_PERR, + EFX_ERR_NCODES +}; + +/* Calculate the IEEE 802.3 CRC32 of a MAC addr */ +extern __checkReturn uint32_t +efx_crc32_calculate( + __in uint32_t crc_init, + __in_ecount(length) uint8_t const *input, + __in int length); + + +/* Type prototypes */ + +typedef struct efx_rxq_s efx_rxq_t; + +/* NIC */ + +typedef struct efx_nic_s efx_nic_t; + +extern __checkReturn efx_rc_t +efx_nic_create( + __in efx_family_t family, + __in efsys_identifier_t *esip, + __in efsys_bar_t *esbp, + __in efsys_lock_t *eslp, + __deref_out efx_nic_t **enpp); + +extern __checkReturn efx_rc_t +efx_nic_probe( + __in efx_nic_t *enp); + +extern __checkReturn efx_rc_t +efx_nic_init( + __in efx_nic_t *enp); + +extern __checkReturn efx_rc_t +efx_nic_reset( + __in efx_nic_t *enp); + +#if EFSYS_OPT_DIAG + +extern __checkReturn efx_rc_t +efx_nic_register_test( + __in efx_nic_t *enp); + +#endif /* EFSYS_OPT_DIAG */ + +extern void +efx_nic_fini( + __in efx_nic_t *enp); + +extern void +efx_nic_unprobe( + __in efx_nic_t *enp); + +extern void +efx_nic_destroy( + __in efx_nic_t *enp); + +#define EFX_PCIE_LINK_SPEED_GEN1 1 +#define EFX_PCIE_LINK_SPEED_GEN2 2 +#define EFX_PCIE_LINK_SPEED_GEN3 3 + +typedef enum efx_pcie_link_performance_e { + EFX_PCIE_LINK_PERFORMANCE_UNKNOWN_BANDWIDTH, + EFX_PCIE_LINK_PERFORMANCE_SUBOPTIMAL_BANDWIDTH, + EFX_PCIE_LINK_PERFORMANCE_SUBOPTIMAL_LATENCY, + EFX_PCIE_LINK_PERFORMANCE_OPTIMAL +} efx_pcie_link_performance_t; + +extern __checkReturn efx_rc_t +efx_nic_calculate_pcie_link_bandwidth( + __in uint32_t pcie_link_width, + __in uint32_t pcie_link_gen, + __out uint32_t *bandwidth_mbpsp); + +extern __checkReturn efx_rc_t +efx_nic_check_pcie_link_speed( + __in efx_nic_t *enp, + __in uint32_t pcie_link_width, + __in uint32_t pcie_link_gen, + __out efx_pcie_link_performance_t *resultp); + +#if EFSYS_OPT_MCDI + +#if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD +/* Huntington and Medford require MCDIv2 commands */ +#define WITH_MCDI_V2 1 +#endif + +typedef struct efx_mcdi_req_s efx_mcdi_req_t; + +typedef enum efx_mcdi_exception_e { + EFX_MCDI_EXCEPTION_MC_REBOOT, + EFX_MCDI_EXCEPTION_MC_BADASSERT, +} efx_mcdi_exception_t; + +#if EFSYS_OPT_MCDI_LOGGING +typedef enum efx_log_msg_e { + EFX_LOG_INVALID, + EFX_LOG_MCDI_REQUEST, + EFX_LOG_MCDI_RESPONSE, +} efx_log_msg_t; +#endif /* EFSYS_OPT_MCDI_LOGGING */ + +typedef struct efx_mcdi_transport_s { + void *emt_context; + efsys_mem_t *emt_dma_mem; + void (*emt_execute)(void *, efx_mcdi_req_t *); + void (*emt_ev_cpl)(void *); + void (*emt_exception)(void *, efx_mcdi_exception_t); +#if EFSYS_OPT_MCDI_LOGGING + void (*emt_logger)(void *, efx_log_msg_t, + void *, size_t, void *, size_t); +#endif /* EFSYS_OPT_MCDI_LOGGING */ +#if EFSYS_OPT_MCDI_PROXY_AUTH + void (*emt_ev_proxy_response)(void *, uint32_t, efx_rc_t); +#endif /* EFSYS_OPT_MCDI_PROXY_AUTH */ +} efx_mcdi_transport_t; + +extern __checkReturn efx_rc_t +efx_mcdi_init( + __in efx_nic_t *enp, + __in const efx_mcdi_transport_t *mtp); + +extern __checkReturn efx_rc_t +efx_mcdi_reboot( + __in efx_nic_t *enp); + + void +efx_mcdi_new_epoch( + __in efx_nic_t *enp); + +extern void +efx_mcdi_get_timeout( + __in efx_nic_t *enp, + __in efx_mcdi_req_t *emrp, + __out uint32_t *usec_timeoutp); + +extern void +efx_mcdi_request_start( + __in efx_nic_t *enp, + __in efx_mcdi_req_t *emrp, + __in boolean_t ev_cpl); + +extern __checkReturn boolean_t +efx_mcdi_request_poll( + __in efx_nic_t *enp); + +extern __checkReturn boolean_t +efx_mcdi_request_abort( + __in efx_nic_t *enp); + +extern void +efx_mcdi_fini( + __in efx_nic_t *enp); + +#endif /* EFSYS_OPT_MCDI */ + +/* INTR */ + +#define EFX_NINTR_SIENA 1024 + +typedef enum efx_intr_type_e { + EFX_INTR_INVALID = 0, + EFX_INTR_LINE, + EFX_INTR_MESSAGE, + EFX_INTR_NTYPES +} efx_intr_type_t; + +#define EFX_INTR_SIZE (sizeof (efx_oword_t)) + +extern __checkReturn efx_rc_t +efx_intr_init( + __in efx_nic_t *enp, + __in efx_intr_type_t type, + __in efsys_mem_t *esmp); + +extern void +efx_intr_enable( + __in efx_nic_t *enp); + +extern void +efx_intr_disable( + __in efx_nic_t *enp); + +extern void +efx_intr_disable_unlocked( + __in efx_nic_t *enp); + +#define EFX_INTR_NEVQS 32 + +extern __checkReturn efx_rc_t +efx_intr_trigger( + __in efx_nic_t *enp, + __in unsigned int level); + +extern void +efx_intr_status_line( + __in efx_nic_t *enp, + __out boolean_t *fatalp, + __out uint32_t *maskp); + +extern void +efx_intr_status_message( + __in efx_nic_t *enp, + __in unsigned int message, + __out boolean_t *fatalp); + +extern void +efx_intr_fatal( + __in efx_nic_t *enp); + +extern void +efx_intr_fini( + __in efx_nic_t *enp); + +/* MAC */ + +#if EFSYS_OPT_MAC_STATS + +/* START MKCONFIG GENERATED EfxHeaderMacBlock e323546097fd7c65 */ +typedef enum efx_mac_stat_e { + EFX_MAC_RX_OCTETS, + EFX_MAC_RX_PKTS, + EFX_MAC_RX_UNICST_PKTS, + EFX_MAC_RX_MULTICST_PKTS, + EFX_MAC_RX_BRDCST_PKTS, + EFX_MAC_RX_PAUSE_PKTS, + EFX_MAC_RX_LE_64_PKTS, + EFX_MAC_RX_65_TO_127_PKTS, + EFX_MAC_RX_128_TO_255_PKTS, + EFX_MAC_RX_256_TO_511_PKTS, + EFX_MAC_RX_512_TO_1023_PKTS, + EFX_MAC_RX_1024_TO_15XX_PKTS, + EFX_MAC_RX_GE_15XX_PKTS, + EFX_MAC_RX_ERRORS, + EFX_MAC_RX_FCS_ERRORS, + EFX_MAC_RX_DROP_EVENTS, + EFX_MAC_RX_FALSE_CARRIER_ERRORS, + EFX_MAC_RX_SYMBOL_ERRORS, + EFX_MAC_RX_ALIGN_ERRORS, + EFX_MAC_RX_INTERNAL_ERRORS, + EFX_MAC_RX_JABBER_PKTS, + EFX_MAC_RX_LANE0_CHAR_ERR, + EFX_MAC_RX_LANE1_CHAR_ERR, + EFX_MAC_RX_LANE2_CHAR_ERR, + EFX_MAC_RX_LANE3_CHAR_ERR, + EFX_MAC_RX_LANE0_DISP_ERR, + EFX_MAC_RX_LANE1_DISP_ERR, + EFX_MAC_RX_LANE2_DISP_ERR, + EFX_MAC_RX_LANE3_DISP_ERR, + EFX_MAC_RX_MATCH_FAULT, + EFX_MAC_RX_NODESC_DROP_CNT, + EFX_MAC_TX_OCTETS, + EFX_MAC_TX_PKTS, + EFX_MAC_TX_UNICST_PKTS, + EFX_MAC_TX_MULTICST_PKTS, + EFX_MAC_TX_BRDCST_PKTS, + EFX_MAC_TX_PAUSE_PKTS, + EFX_MAC_TX_LE_64_PKTS, + EFX_MAC_TX_65_TO_127_PKTS, + EFX_MAC_TX_128_TO_255_PKTS, + EFX_MAC_TX_256_TO_511_PKTS, + EFX_MAC_TX_512_TO_1023_PKTS, + EFX_MAC_TX_1024_TO_15XX_PKTS, + EFX_MAC_TX_GE_15XX_PKTS, + EFX_MAC_TX_ERRORS, + EFX_MAC_TX_SGL_COL_PKTS, + EFX_MAC_TX_MULT_COL_PKTS, + EFX_MAC_TX_EX_COL_PKTS, + EFX_MAC_TX_LATE_COL_PKTS, + EFX_MAC_TX_DEF_PKTS, + EFX_MAC_TX_EX_DEF_PKTS, + EFX_MAC_PM_TRUNC_BB_OVERFLOW, + EFX_MAC_PM_DISCARD_BB_OVERFLOW, + EFX_MAC_PM_TRUNC_VFIFO_FULL, + EFX_MAC_PM_DISCARD_VFIFO_FULL, + EFX_MAC_PM_TRUNC_QBB, + EFX_MAC_PM_DISCARD_QBB, + EFX_MAC_PM_DISCARD_MAPPING, + EFX_MAC_RXDP_Q_DISABLED_PKTS, + EFX_MAC_RXDP_DI_DROPPED_PKTS, + EFX_MAC_RXDP_STREAMING_PKTS, + EFX_MAC_RXDP_HLB_FETCH, + EFX_MAC_RXDP_HLB_WAIT, + EFX_MAC_VADAPTER_RX_UNICAST_PACKETS, + EFX_MAC_VADAPTER_RX_UNICAST_BYTES, + EFX_MAC_VADAPTER_RX_MULTICAST_PACKETS, + EFX_MAC_VADAPTER_RX_MULTICAST_BYTES, + EFX_MAC_VADAPTER_RX_BROADCAST_PACKETS, + EFX_MAC_VADAPTER_RX_BROADCAST_BYTES, + EFX_MAC_VADAPTER_RX_BAD_PACKETS, + EFX_MAC_VADAPTER_RX_BAD_BYTES, + EFX_MAC_VADAPTER_RX_OVERFLOW, + EFX_MAC_VADAPTER_TX_UNICAST_PACKETS, + EFX_MAC_VADAPTER_TX_UNICAST_BYTES, + EFX_MAC_VADAPTER_TX_MULTICAST_PACKETS, + EFX_MAC_VADAPTER_TX_MULTICAST_BYTES, + EFX_MAC_VADAPTER_TX_BROADCAST_PACKETS, + EFX_MAC_VADAPTER_TX_BROADCAST_BYTES, + EFX_MAC_VADAPTER_TX_BAD_PACKETS, + EFX_MAC_VADAPTER_TX_BAD_BYTES, + EFX_MAC_VADAPTER_TX_OVERFLOW, + EFX_MAC_NSTATS +} efx_mac_stat_t; + +/* END MKCONFIG GENERATED EfxHeaderMacBlock */ + +#endif /* EFSYS_OPT_MAC_STATS */ + +typedef enum efx_link_mode_e { + EFX_LINK_UNKNOWN = 0, + EFX_LINK_DOWN, + EFX_LINK_10HDX, + EFX_LINK_10FDX, + EFX_LINK_100HDX, + EFX_LINK_100FDX, + EFX_LINK_1000HDX, + EFX_LINK_1000FDX, + EFX_LINK_10000FDX, + EFX_LINK_40000FDX, + EFX_LINK_NMODES +} efx_link_mode_t; + +#define EFX_MAC_ADDR_LEN 6 + +#define EFX_MAC_ADDR_IS_MULTICAST(_address) (((uint8_t *)_address)[0] & 0x01) + +#define EFX_MAC_MULTICAST_LIST_MAX 256 + +#define EFX_MAC_SDU_MAX 9202 + +#define EFX_MAC_PDU_ADJUSTMENT \ + (/* EtherII */ 14 \ + + /* VLAN */ 4 \ + + /* CRC */ 4 \ + + /* bug16011 */ 16) \ + +#define EFX_MAC_PDU(_sdu) \ + P2ROUNDUP((_sdu) + EFX_MAC_PDU_ADJUSTMENT, 8) + +/* + * Due to the P2ROUNDUP in EFX_MAC_PDU(), EFX_MAC_SDU_FROM_PDU() may give + * the SDU rounded up slightly. + */ +#define EFX_MAC_SDU_FROM_PDU(_pdu) ((_pdu) - EFX_MAC_PDU_ADJUSTMENT) + +#define EFX_MAC_PDU_MIN 60 +#define EFX_MAC_PDU_MAX EFX_MAC_PDU(EFX_MAC_SDU_MAX) + +extern __checkReturn efx_rc_t +efx_mac_pdu_get( + __in efx_nic_t *enp, + __out size_t *pdu); + +extern __checkReturn efx_rc_t +efx_mac_pdu_set( + __in efx_nic_t *enp, + __in size_t pdu); + +extern __checkReturn efx_rc_t +efx_mac_addr_set( + __in efx_nic_t *enp, + __in uint8_t *addr); + +extern __checkReturn efx_rc_t +efx_mac_filter_set( + __in efx_nic_t *enp, + __in boolean_t all_unicst, + __in boolean_t mulcst, + __in boolean_t all_mulcst, + __in boolean_t brdcst); + +extern __checkReturn efx_rc_t +efx_mac_multicast_list_set( + __in efx_nic_t *enp, + __in_ecount(6*count) uint8_t const *addrs, + __in int count); + +extern __checkReturn efx_rc_t +efx_mac_filter_default_rxq_set( + __in efx_nic_t *enp, + __in efx_rxq_t *erp, + __in boolean_t using_rss); + +extern void +efx_mac_filter_default_rxq_clear( + __in efx_nic_t *enp); + +extern __checkReturn efx_rc_t +efx_mac_drain( + __in efx_nic_t *enp, + __in boolean_t enabled); + +extern __checkReturn efx_rc_t +efx_mac_up( + __in efx_nic_t *enp, + __out boolean_t *mac_upp); + +#define EFX_FCNTL_RESPOND 0x00000001 +#define EFX_FCNTL_GENERATE 0x00000002 + +extern __checkReturn efx_rc_t +efx_mac_fcntl_set( + __in efx_nic_t *enp, + __in unsigned int fcntl, + __in boolean_t autoneg); + +extern void +efx_mac_fcntl_get( + __in efx_nic_t *enp, + __out unsigned int *fcntl_wantedp, + __out unsigned int *fcntl_linkp); + + +#if EFSYS_OPT_MAC_STATS + +#if EFSYS_OPT_NAMES + +extern __checkReturn const char * +efx_mac_stat_name( + __in efx_nic_t *enp, + __in unsigned int id); + +#endif /* EFSYS_OPT_NAMES */ + +#define EFX_MAC_STATS_MASK_BITS_PER_PAGE (8 * sizeof (uint32_t)) + +#define EFX_MAC_STATS_MASK_NPAGES \ + (P2ROUNDUP(EFX_MAC_NSTATS, EFX_MAC_STATS_MASK_BITS_PER_PAGE) / \ + EFX_MAC_STATS_MASK_BITS_PER_PAGE) + +/* + * Get mask of MAC statistics supported by the hardware. + * + * If mask_size is insufficient to return the mask, EINVAL error is + * returned. EFX_MAC_STATS_MASK_NPAGES multiplied by size of the page + * (which is sizeof (uint32_t)) is sufficient. + */ +extern __checkReturn efx_rc_t +efx_mac_stats_get_mask( + __in efx_nic_t *enp, + __out_bcount(mask_size) uint32_t *maskp, + __in size_t mask_size); + +#define EFX_MAC_STAT_SUPPORTED(_mask, _stat) \ + ((_mask)[(_stat) / EFX_MAC_STATS_MASK_BITS_PER_PAGE] & \ + (1ULL << ((_stat) & (EFX_MAC_STATS_MASK_BITS_PER_PAGE - 1)))) + +#define EFX_MAC_STATS_SIZE 0x400 + +extern __checkReturn efx_rc_t +efx_mac_stats_clear( + __in efx_nic_t *enp); + +/* + * Upload mac statistics supported by the hardware into the given buffer. + * + * The reference buffer must be at least %EFX_MAC_STATS_SIZE bytes, + * and page aligned. + * + * The hardware will only DMA statistics that it understands (of course). + * Drivers should not make any assumptions about which statistics are + * supported, especially when the statistics are generated by firmware. + * + * Thus, drivers should zero this buffer before use, so that not-understood + * statistics read back as zero. + */ +extern __checkReturn efx_rc_t +efx_mac_stats_upload( + __in efx_nic_t *enp, + __in efsys_mem_t *esmp); + +extern __checkReturn efx_rc_t +efx_mac_stats_periodic( + __in efx_nic_t *enp, + __in efsys_mem_t *esmp, + __in uint16_t period_ms, + __in boolean_t events); + +extern __checkReturn efx_rc_t +efx_mac_stats_update( + __in efx_nic_t *enp, + __in efsys_mem_t *esmp, + __inout_ecount(EFX_MAC_NSTATS) efsys_stat_t *stat, + __inout_opt uint32_t *generationp); + +#endif /* EFSYS_OPT_MAC_STATS */ + +/* MON */ + +typedef enum efx_mon_type_e { + EFX_MON_INVALID = 0, + EFX_MON_SFC90X0, + EFX_MON_SFC91X0, + EFX_MON_SFC92X0, + EFX_MON_NTYPES +} efx_mon_type_t; + +#if EFSYS_OPT_NAMES + +extern const char * +efx_mon_name( + __in efx_nic_t *enp); + +#endif /* EFSYS_OPT_NAMES */ + +extern __checkReturn efx_rc_t +efx_mon_init( + __in efx_nic_t *enp); + +#if EFSYS_OPT_MON_STATS + +#define EFX_MON_STATS_PAGE_SIZE 0x100 +#define EFX_MON_MASK_ELEMENT_SIZE 32 + +/* START MKCONFIG GENERATED MonitorHeaderStatsBlock 5d4ee5185e419abe */ +typedef enum efx_mon_stat_e { + EFX_MON_STAT_2_5V, + EFX_MON_STAT_VCCP1, + EFX_MON_STAT_VCC, + EFX_MON_STAT_5V, + EFX_MON_STAT_12V, + EFX_MON_STAT_VCCP2, + EFX_MON_STAT_EXT_TEMP, + EFX_MON_STAT_INT_TEMP, + EFX_MON_STAT_AIN1, + EFX_MON_STAT_AIN2, + EFX_MON_STAT_INT_COOLING, + EFX_MON_STAT_EXT_COOLING, + EFX_MON_STAT_1V, + EFX_MON_STAT_1_2V, + EFX_MON_STAT_1_8V, + EFX_MON_STAT_3_3V, + EFX_MON_STAT_1_2VA, + EFX_MON_STAT_VREF, + EFX_MON_STAT_VAOE, + EFX_MON_STAT_AOE_TEMP, + EFX_MON_STAT_PSU_AOE_TEMP, + EFX_MON_STAT_PSU_TEMP, + EFX_MON_STAT_FAN0, + EFX_MON_STAT_FAN1, + EFX_MON_STAT_FAN2, + EFX_MON_STAT_FAN3, + EFX_MON_STAT_FAN4, + EFX_MON_STAT_VAOE_IN, + EFX_MON_STAT_IAOE, + EFX_MON_STAT_IAOE_IN, + EFX_MON_STAT_NIC_POWER, + EFX_MON_STAT_0_9V, + EFX_MON_STAT_I0_9V, + EFX_MON_STAT_I1_2V, + EFX_MON_STAT_0_9V_ADC, + EFX_MON_STAT_INT_TEMP2, + EFX_MON_STAT_VREG_TEMP, + EFX_MON_STAT_VREG_0_9V_TEMP, + EFX_MON_STAT_VREG_1_2V_TEMP, + EFX_MON_STAT_INT_VPTAT, + EFX_MON_STAT_INT_ADC_TEMP, + EFX_MON_STAT_EXT_VPTAT, + EFX_MON_STAT_EXT_ADC_TEMP, + EFX_MON_STAT_AMBIENT_TEMP, + EFX_MON_STAT_AIRFLOW, + EFX_MON_STAT_VDD08D_VSS08D_CSR, + EFX_MON_STAT_VDD08D_VSS08D_CSR_EXTADC, + EFX_MON_STAT_HOTPOINT_TEMP, + EFX_MON_STAT_PHY_POWER_SWITCH_PORT0, + EFX_MON_STAT_PHY_POWER_SWITCH_PORT1, + EFX_MON_STAT_MUM_VCC, + EFX_MON_STAT_0V9_A, + EFX_MON_STAT_I0V9_A, + EFX_MON_STAT_0V9_A_TEMP, + EFX_MON_STAT_0V9_B, + EFX_MON_STAT_I0V9_B, + EFX_MON_STAT_0V9_B_TEMP, + EFX_MON_STAT_CCOM_AVREG_1V2_SUPPLY, + EFX_MON_STAT_CCOM_AVREG_1V2_SUPPLY_EXT_ADC, + EFX_MON_STAT_CCOM_AVREG_1V8_SUPPLY, + EFX_MON_STAT_CCOM_AVREG_1V8_SUPPLY_EXT_ADC, + EFX_MON_STAT_CONTROLLER_MASTER_VPTAT, + EFX_MON_STAT_CONTROLLER_MASTER_INTERNAL_TEMP, + EFX_MON_STAT_CONTROLLER_MASTER_VPTAT_EXT_ADC, + EFX_MON_STAT_CONTROLLER_MASTER_INTERNAL_TEMP_EXT_ADC, + EFX_MON_STAT_CONTROLLER_SLAVE_VPTAT, + EFX_MON_STAT_CONTROLLER_SLAVE_INTERNAL_TEMP, + EFX_MON_STAT_CONTROLLER_SLAVE_VPTAT_EXT_ADC, + EFX_MON_STAT_CONTROLLER_SLAVE_INTERNAL_TEMP_EXT_ADC, + EFX_MON_STAT_SODIMM_VOUT, + EFX_MON_STAT_SODIMM_0_TEMP, + EFX_MON_STAT_SODIMM_1_TEMP, + EFX_MON_STAT_PHY0_VCC, + EFX_MON_STAT_PHY1_VCC, + EFX_MON_STAT_CONTROLLER_TDIODE_TEMP, + EFX_MON_STAT_BOARD_FRONT_TEMP, + EFX_MON_STAT_BOARD_BACK_TEMP, + EFX_MON_NSTATS +} efx_mon_stat_t; + +/* END MKCONFIG GENERATED MonitorHeaderStatsBlock */ + +typedef enum efx_mon_stat_state_e { + EFX_MON_STAT_STATE_OK = 0, + EFX_MON_STAT_STATE_WARNING = 1, + EFX_MON_STAT_STATE_FATAL = 2, + EFX_MON_STAT_STATE_BROKEN = 3, + EFX_MON_STAT_STATE_NO_READING = 4, +} efx_mon_stat_state_t; + +typedef struct efx_mon_stat_value_s { + uint16_t emsv_value; + uint16_t emsv_state; +} efx_mon_stat_value_t; + +#if EFSYS_OPT_NAMES + +extern const char * +efx_mon_stat_name( + __in efx_nic_t *enp, + __in efx_mon_stat_t id); + +#endif /* EFSYS_OPT_NAMES */ + +extern __checkReturn efx_rc_t +efx_mon_stats_update( + __in efx_nic_t *enp, + __in efsys_mem_t *esmp, + __inout_ecount(EFX_MON_NSTATS) efx_mon_stat_value_t *values); + +#endif /* EFSYS_OPT_MON_STATS */ + +extern void +efx_mon_fini( + __in efx_nic_t *enp); + +/* PHY */ + +extern __checkReturn efx_rc_t +efx_phy_verify( + __in efx_nic_t *enp); + +#if EFSYS_OPT_PHY_LED_CONTROL + +typedef enum efx_phy_led_mode_e { + EFX_PHY_LED_DEFAULT = 0, + EFX_PHY_LED_OFF, + EFX_PHY_LED_ON, + EFX_PHY_LED_FLASH, + EFX_PHY_LED_NMODES +} efx_phy_led_mode_t; + +extern __checkReturn efx_rc_t +efx_phy_led_set( + __in efx_nic_t *enp, + __in efx_phy_led_mode_t mode); + +#endif /* EFSYS_OPT_PHY_LED_CONTROL */ + +extern __checkReturn efx_rc_t +efx_port_init( + __in efx_nic_t *enp); + +#if EFSYS_OPT_LOOPBACK + +typedef enum efx_loopback_type_e { + EFX_LOOPBACK_OFF = 0, + EFX_LOOPBACK_DATA = 1, + EFX_LOOPBACK_GMAC = 2, + EFX_LOOPBACK_XGMII = 3, + EFX_LOOPBACK_XGXS = 4, + EFX_LOOPBACK_XAUI = 5, + EFX_LOOPBACK_GMII = 6, + EFX_LOOPBACK_SGMII = 7, + EFX_LOOPBACK_XGBR = 8, + EFX_LOOPBACK_XFI = 9, + EFX_LOOPBACK_XAUI_FAR = 10, + EFX_LOOPBACK_GMII_FAR = 11, + EFX_LOOPBACK_SGMII_FAR = 12, + EFX_LOOPBACK_XFI_FAR = 13, + EFX_LOOPBACK_GPHY = 14, + EFX_LOOPBACK_PHY_XS = 15, + EFX_LOOPBACK_PCS = 16, + EFX_LOOPBACK_PMA_PMD = 17, + EFX_LOOPBACK_XPORT = 18, + EFX_LOOPBACK_XGMII_WS = 19, + EFX_LOOPBACK_XAUI_WS = 20, + EFX_LOOPBACK_XAUI_WS_FAR = 21, + EFX_LOOPBACK_XAUI_WS_NEAR = 22, + EFX_LOOPBACK_GMII_WS = 23, + EFX_LOOPBACK_XFI_WS = 24, + EFX_LOOPBACK_XFI_WS_FAR = 25, + EFX_LOOPBACK_PHYXS_WS = 26, + EFX_LOOPBACK_PMA_INT = 27, + EFX_LOOPBACK_SD_NEAR = 28, + EFX_LOOPBACK_SD_FAR = 29, + EFX_LOOPBACK_PMA_INT_WS = 30, + EFX_LOOPBACK_SD_FEP2_WS = 31, + EFX_LOOPBACK_SD_FEP1_5_WS = 32, + EFX_LOOPBACK_SD_FEP_WS = 33, + EFX_LOOPBACK_SD_FES_WS = 34, + EFX_LOOPBACK_NTYPES +} efx_loopback_type_t; + +typedef enum efx_loopback_kind_e { + EFX_LOOPBACK_KIND_OFF = 0, + EFX_LOOPBACK_KIND_ALL, + EFX_LOOPBACK_KIND_MAC, + EFX_LOOPBACK_KIND_PHY, + EFX_LOOPBACK_NKINDS +} efx_loopback_kind_t; + +extern void +efx_loopback_mask( + __in efx_loopback_kind_t loopback_kind, + __out efx_qword_t *maskp); + +extern __checkReturn efx_rc_t +efx_port_loopback_set( + __in efx_nic_t *enp, + __in efx_link_mode_t link_mode, + __in efx_loopback_type_t type); + +#if EFSYS_OPT_NAMES + +extern __checkReturn const char * +efx_loopback_type_name( + __in efx_nic_t *enp, + __in efx_loopback_type_t type); + +#endif /* EFSYS_OPT_NAMES */ + +#endif /* EFSYS_OPT_LOOPBACK */ + +extern __checkReturn efx_rc_t +efx_port_poll( + __in efx_nic_t *enp, + __out_opt efx_link_mode_t *link_modep); + +extern void +efx_port_fini( + __in efx_nic_t *enp); + +typedef enum efx_phy_cap_type_e { + EFX_PHY_CAP_INVALID = 0, + EFX_PHY_CAP_10HDX, + EFX_PHY_CAP_10FDX, + EFX_PHY_CAP_100HDX, + EFX_PHY_CAP_100FDX, + EFX_PHY_CAP_1000HDX, + EFX_PHY_CAP_1000FDX, + EFX_PHY_CAP_10000FDX, + EFX_PHY_CAP_PAUSE, + EFX_PHY_CAP_ASYM, + EFX_PHY_CAP_AN, + EFX_PHY_CAP_40000FDX, + EFX_PHY_CAP_NTYPES +} efx_phy_cap_type_t; + + +#define EFX_PHY_CAP_CURRENT 0x00000000 +#define EFX_PHY_CAP_DEFAULT 0x00000001 +#define EFX_PHY_CAP_PERM 0x00000002 + +extern void +efx_phy_adv_cap_get( + __in efx_nic_t *enp, + __in uint32_t flag, + __out uint32_t *maskp); + +extern __checkReturn efx_rc_t +efx_phy_adv_cap_set( + __in efx_nic_t *enp, + __in uint32_t mask); + +extern void +efx_phy_lp_cap_get( + __in efx_nic_t *enp, + __out uint32_t *maskp); + +extern __checkReturn efx_rc_t +efx_phy_oui_get( + __in efx_nic_t *enp, + __out uint32_t *ouip); + +typedef enum efx_phy_media_type_e { + EFX_PHY_MEDIA_INVALID = 0, + EFX_PHY_MEDIA_XAUI, + EFX_PHY_MEDIA_CX4, + EFX_PHY_MEDIA_KX4, + EFX_PHY_MEDIA_XFP, + EFX_PHY_MEDIA_SFP_PLUS, + EFX_PHY_MEDIA_BASE_T, + EFX_PHY_MEDIA_QSFP_PLUS, + EFX_PHY_MEDIA_NTYPES +} efx_phy_media_type_t; + +/* Get the type of medium currently used. If the board has ports for + * modules, a module is present, and we recognise the media type of + * the module, then this will be the media type of the module. + * Otherwise it will be the media type of the port. + */ +extern void +efx_phy_media_type_get( + __in efx_nic_t *enp, + __out efx_phy_media_type_t *typep); + +extern efx_rc_t +efx_phy_module_get_info( + __in efx_nic_t *enp, + __in uint8_t dev_addr, + __in uint8_t offset, + __in uint8_t len, + __out_bcount(len) uint8_t *data); + +#if EFSYS_OPT_PHY_STATS + +/* START MKCONFIG GENERATED PhyHeaderStatsBlock 30ed56ad501f8e36 */ +typedef enum efx_phy_stat_e { + EFX_PHY_STAT_OUI, + EFX_PHY_STAT_PMA_PMD_LINK_UP, + EFX_PHY_STAT_PMA_PMD_RX_FAULT, + EFX_PHY_STAT_PMA_PMD_TX_FAULT, + EFX_PHY_STAT_PMA_PMD_REV_A, + EFX_PHY_STAT_PMA_PMD_REV_B, + EFX_PHY_STAT_PMA_PMD_REV_C, + EFX_PHY_STAT_PMA_PMD_REV_D, + EFX_PHY_STAT_PCS_LINK_UP, + EFX_PHY_STAT_PCS_RX_FAULT, + EFX_PHY_STAT_PCS_TX_FAULT, + EFX_PHY_STAT_PCS_BER, + EFX_PHY_STAT_PCS_BLOCK_ERRORS, + EFX_PHY_STAT_PHY_XS_LINK_UP, + EFX_PHY_STAT_PHY_XS_RX_FAULT, + EFX_PHY_STAT_PHY_XS_TX_FAULT, + EFX_PHY_STAT_PHY_XS_ALIGN, + EFX_PHY_STAT_PHY_XS_SYNC_A, + EFX_PHY_STAT_PHY_XS_SYNC_B, + EFX_PHY_STAT_PHY_XS_SYNC_C, + EFX_PHY_STAT_PHY_XS_SYNC_D, + EFX_PHY_STAT_AN_LINK_UP, + EFX_PHY_STAT_AN_MASTER, + EFX_PHY_STAT_AN_LOCAL_RX_OK, + EFX_PHY_STAT_AN_REMOTE_RX_OK, + EFX_PHY_STAT_CL22EXT_LINK_UP, + EFX_PHY_STAT_SNR_A, + EFX_PHY_STAT_SNR_B, + EFX_PHY_STAT_SNR_C, + EFX_PHY_STAT_SNR_D, + EFX_PHY_STAT_PMA_PMD_SIGNAL_A, + EFX_PHY_STAT_PMA_PMD_SIGNAL_B, + EFX_PHY_STAT_PMA_PMD_SIGNAL_C, + EFX_PHY_STAT_PMA_PMD_SIGNAL_D, + EFX_PHY_STAT_AN_COMPLETE, + EFX_PHY_STAT_PMA_PMD_REV_MAJOR, + EFX_PHY_STAT_PMA_PMD_REV_MINOR, + EFX_PHY_STAT_PMA_PMD_REV_MICRO, + EFX_PHY_STAT_PCS_FW_VERSION_0, + EFX_PHY_STAT_PCS_FW_VERSION_1, + EFX_PHY_STAT_PCS_FW_VERSION_2, + EFX_PHY_STAT_PCS_FW_VERSION_3, + EFX_PHY_STAT_PCS_FW_BUILD_YY, + EFX_PHY_STAT_PCS_FW_BUILD_MM, + EFX_PHY_STAT_PCS_FW_BUILD_DD, + EFX_PHY_STAT_PCS_OP_MODE, + EFX_PHY_NSTATS +} efx_phy_stat_t; + +/* END MKCONFIG GENERATED PhyHeaderStatsBlock */ + +#if EFSYS_OPT_NAMES + +extern const char * +efx_phy_stat_name( + __in efx_nic_t *enp, + __in efx_phy_stat_t stat); + +#endif /* EFSYS_OPT_NAMES */ + +#define EFX_PHY_STATS_SIZE 0x100 + +extern __checkReturn efx_rc_t +efx_phy_stats_update( + __in efx_nic_t *enp, + __in efsys_mem_t *esmp, + __inout_ecount(EFX_PHY_NSTATS) uint32_t *stat); + +#endif /* EFSYS_OPT_PHY_STATS */ + + +#if EFSYS_OPT_BIST + +typedef enum efx_bist_type_e { + EFX_BIST_TYPE_UNKNOWN, + EFX_BIST_TYPE_PHY_NORMAL, + EFX_BIST_TYPE_PHY_CABLE_SHORT, + EFX_BIST_TYPE_PHY_CABLE_LONG, + EFX_BIST_TYPE_MC_MEM, /* Test the MC DMEM and IMEM */ + EFX_BIST_TYPE_SAT_MEM, /* Test the DMEM and IMEM of satellite cpus*/ + EFX_BIST_TYPE_REG, /* Test the register memories */ + EFX_BIST_TYPE_NTYPES, +} efx_bist_type_t; + +typedef enum efx_bist_result_e { + EFX_BIST_RESULT_UNKNOWN, + EFX_BIST_RESULT_RUNNING, + EFX_BIST_RESULT_PASSED, + EFX_BIST_RESULT_FAILED, +} efx_bist_result_t; + +typedef enum efx_phy_cable_status_e { + EFX_PHY_CABLE_STATUS_OK, + EFX_PHY_CABLE_STATUS_INVALID, + EFX_PHY_CABLE_STATUS_OPEN, + EFX_PHY_CABLE_STATUS_INTRAPAIRSHORT, + EFX_PHY_CABLE_STATUS_INTERPAIRSHORT, + EFX_PHY_CABLE_STATUS_BUSY, +} efx_phy_cable_status_t; + +typedef enum efx_bist_value_e { + EFX_BIST_PHY_CABLE_LENGTH_A, + EFX_BIST_PHY_CABLE_LENGTH_B, + EFX_BIST_PHY_CABLE_LENGTH_C, + EFX_BIST_PHY_CABLE_LENGTH_D, + EFX_BIST_PHY_CABLE_STATUS_A, + EFX_BIST_PHY_CABLE_STATUS_B, + EFX_BIST_PHY_CABLE_STATUS_C, + EFX_BIST_PHY_CABLE_STATUS_D, + EFX_BIST_FAULT_CODE, + /* Memory BIST specific values. These match to the MC_CMD_BIST_POLL + * response. */ + EFX_BIST_MEM_TEST, + EFX_BIST_MEM_ADDR, + EFX_BIST_MEM_BUS, + EFX_BIST_MEM_EXPECT, + EFX_BIST_MEM_ACTUAL, + EFX_BIST_MEM_ECC, + EFX_BIST_MEM_ECC_PARITY, + EFX_BIST_MEM_ECC_FATAL, + EFX_BIST_NVALUES, +} efx_bist_value_t; + +extern __checkReturn efx_rc_t +efx_bist_enable_offline( + __in efx_nic_t *enp); + +extern __checkReturn efx_rc_t +efx_bist_start( + __in efx_nic_t *enp, + __in efx_bist_type_t type); + +extern __checkReturn efx_rc_t +efx_bist_poll( + __in efx_nic_t *enp, + __in efx_bist_type_t type, + __out efx_bist_result_t *resultp, + __out_opt uint32_t *value_maskp, + __out_ecount_opt(count) unsigned long *valuesp, + __in size_t count); + +extern void +efx_bist_stop( + __in efx_nic_t *enp, + __in efx_bist_type_t type); + +#endif /* EFSYS_OPT_BIST */ + +#define EFX_FEATURE_IPV6 0x00000001 +#define EFX_FEATURE_LFSR_HASH_INSERT 0x00000002 +#define EFX_FEATURE_LINK_EVENTS 0x00000004 +#define EFX_FEATURE_PERIODIC_MAC_STATS 0x00000008 +#define EFX_FEATURE_MCDI 0x00000020 +#define EFX_FEATURE_LOOKAHEAD_SPLIT 0x00000040 +#define EFX_FEATURE_MAC_HEADER_FILTERS 0x00000080 +#define EFX_FEATURE_TURBO 0x00000100 +#define EFX_FEATURE_MCDI_DMA 0x00000200 +#define EFX_FEATURE_TX_SRC_FILTERS 0x00000400 +#define EFX_FEATURE_PIO_BUFFERS 0x00000800 +#define EFX_FEATURE_FW_ASSISTED_TSO 0x00001000 +#define EFX_FEATURE_FW_ASSISTED_TSO_V2 0x00002000 +#define EFX_FEATURE_PACKED_STREAM 0x00004000 + +typedef enum efx_tunnel_protocol_e { + EFX_TUNNEL_PROTOCOL_NONE = 0, + EFX_TUNNEL_PROTOCOL_VXLAN, + EFX_TUNNEL_PROTOCOL_GENEVE, + EFX_TUNNEL_PROTOCOL_NVGRE, + EFX_TUNNEL_NPROTOS +} efx_tunnel_protocol_t; + +typedef struct efx_nic_cfg_s { + uint32_t enc_board_type; + uint32_t enc_phy_type; +#if EFSYS_OPT_NAMES + char enc_phy_name[21]; +#endif + char enc_phy_revision[21]; + efx_mon_type_t enc_mon_type; +#if EFSYS_OPT_MON_STATS + uint32_t enc_mon_stat_dma_buf_size; + uint32_t enc_mon_stat_mask[(EFX_MON_NSTATS + 31) / 32]; +#endif + unsigned int enc_features; + uint8_t enc_mac_addr[6]; + uint8_t enc_port; /* PHY port number */ + uint32_t enc_intr_vec_base; + uint32_t enc_intr_limit; + uint32_t enc_evq_limit; + uint32_t enc_txq_limit; + uint32_t enc_rxq_limit; + uint32_t enc_txq_max_ndescs; + uint32_t enc_buftbl_limit; + uint32_t enc_piobuf_limit; + uint32_t enc_piobuf_size; + uint32_t enc_piobuf_min_alloc_size; + uint32_t enc_evq_timer_quantum_ns; + uint32_t enc_evq_timer_max_us; + uint32_t enc_clk_mult; + uint32_t enc_rx_prefix_size; + uint32_t enc_rx_buf_align_start; + uint32_t enc_rx_buf_align_end; + uint32_t enc_rx_scale_max_exclusive_contexts; +#if EFSYS_OPT_LOOPBACK + efx_qword_t enc_loopback_types[EFX_LINK_NMODES]; +#endif /* EFSYS_OPT_LOOPBACK */ +#if EFSYS_OPT_PHY_FLAGS + uint32_t enc_phy_flags_mask; +#endif /* EFSYS_OPT_PHY_FLAGS */ +#if EFSYS_OPT_PHY_LED_CONTROL + uint32_t enc_led_mask; +#endif /* EFSYS_OPT_PHY_LED_CONTROL */ +#if EFSYS_OPT_PHY_STATS + uint64_t enc_phy_stat_mask; +#endif /* EFSYS_OPT_PHY_STATS */ +#if EFSYS_OPT_MCDI + uint8_t enc_mcdi_mdio_channel; +#if EFSYS_OPT_PHY_STATS + uint32_t enc_mcdi_phy_stat_mask; +#endif /* EFSYS_OPT_PHY_STATS */ +#if EFSYS_OPT_MON_STATS + uint32_t *enc_mcdi_sensor_maskp; + uint32_t enc_mcdi_sensor_mask_size; +#endif /* EFSYS_OPT_MON_STATS */ +#endif /* EFSYS_OPT_MCDI */ +#if EFSYS_OPT_BIST + uint32_t enc_bist_mask; +#endif /* EFSYS_OPT_BIST */ +#if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD + uint32_t enc_pf; + uint32_t enc_vf; + uint32_t enc_privilege_mask; +#endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */ + boolean_t enc_bug26807_workaround; + boolean_t enc_bug35388_workaround; + boolean_t enc_bug41750_workaround; + boolean_t enc_bug61265_workaround; + boolean_t enc_rx_batching_enabled; + /* Maximum number of descriptors completed in an rx event. */ + uint32_t enc_rx_batch_max; + /* Number of rx descriptors the hardware requires for a push. */ + uint32_t enc_rx_push_align; + /* Maximum amount of data in DMA descriptor */ + uint32_t enc_tx_dma_desc_size_max; + /* + * Boundary which DMA descriptor data must not cross or 0 if no + * limitation. + */ + uint32_t enc_tx_dma_desc_boundary; + /* + * Maximum number of bytes into the packet the TCP header can start for + * the hardware to apply TSO packet edits. + */ + uint32_t enc_tx_tso_tcp_header_offset_limit; + boolean_t enc_fw_assisted_tso_enabled; + boolean_t enc_fw_assisted_tso_v2_enabled; + /* Number of TSO contexts on the NIC (FATSOv2) */ + uint32_t enc_fw_assisted_tso_v2_n_contexts; + boolean_t enc_hw_tx_insert_vlan_enabled; + /* Number of PFs on the NIC */ + uint32_t enc_hw_pf_count; + /* Datapath firmware vadapter/vport/vswitch support */ + boolean_t enc_datapath_cap_evb; + boolean_t enc_rx_disable_scatter_supported; + boolean_t enc_allow_set_mac_with_installed_filters; + boolean_t enc_enhanced_set_mac_supported; + boolean_t enc_init_evq_v2_supported; + boolean_t enc_rx_packed_stream_supported; + boolean_t enc_rx_var_packed_stream_supported; + boolean_t enc_pm_and_rxdp_counters; + boolean_t enc_mac_stats_40g_tx_size_bins; + uint32_t enc_tunnel_encapsulations_supported; + /* External port identifier */ + uint8_t enc_external_port; + uint32_t enc_mcdi_max_payload_length; + /* VPD may be per-PF or global */ + boolean_t enc_vpd_is_global; + /* Minimum unidirectional bandwidth in Mb/s to max out all ports */ + uint32_t enc_required_pcie_bandwidth_mbps; + uint32_t enc_max_pcie_link_gen; + /* Firmware verifies integrity of NVRAM updates */ + uint32_t enc_fw_verified_nvram_update_required; +} efx_nic_cfg_t; + +#define EFX_PCI_FUNCTION_IS_PF(_encp) ((_encp)->enc_vf == 0xffff) +#define EFX_PCI_FUNCTION_IS_VF(_encp) ((_encp)->enc_vf != 0xffff) + +#define EFX_PCI_FUNCTION(_encp) \ + (EFX_PCI_FUNCTION_IS_PF(_encp) ? (_encp)->enc_pf : (_encp)->enc_vf) + +#define EFX_PCI_VF_PARENT(_encp) ((_encp)->enc_pf) + +extern const efx_nic_cfg_t * +efx_nic_cfg_get( + __in efx_nic_t *enp); + +typedef struct efx_nic_fw_info_s { + /* Basic FW version information */ + uint16_t enfi_mc_fw_version[4]; + /* + * If datapath capabilities can be detected, + * additional FW information is to be shown + */ + boolean_t enfi_dpcpu_fw_ids_valid; + /* Rx and Tx datapath CPU FW IDs */ + uint16_t enfi_rx_dpcpu_fw_id; + uint16_t enfi_tx_dpcpu_fw_id; +} efx_nic_fw_info_t; + +extern __checkReturn efx_rc_t +efx_nic_get_fw_version( + __in efx_nic_t *enp, + __out efx_nic_fw_info_t *enfip); + +/* Driver resource limits (minimum required/maximum usable). */ +typedef struct efx_drv_limits_s { + uint32_t edl_min_evq_count; + uint32_t edl_max_evq_count; + + uint32_t edl_min_rxq_count; + uint32_t edl_max_rxq_count; + + uint32_t edl_min_txq_count; + uint32_t edl_max_txq_count; + + /* PIO blocks (sub-allocated from piobuf) */ + uint32_t edl_min_pio_alloc_size; + uint32_t edl_max_pio_alloc_count; +} efx_drv_limits_t; + +extern __checkReturn efx_rc_t +efx_nic_set_drv_limits( + __inout efx_nic_t *enp, + __in efx_drv_limits_t *edlp); + +typedef enum efx_nic_region_e { + EFX_REGION_VI, /* Memory BAR UC mapping */ + EFX_REGION_PIO_WRITE_VI, /* Memory BAR WC mapping */ +} efx_nic_region_t; + +extern __checkReturn efx_rc_t +efx_nic_get_bar_region( + __in efx_nic_t *enp, + __in efx_nic_region_t region, + __out uint32_t *offsetp, + __out size_t *sizep); + +extern __checkReturn efx_rc_t +efx_nic_get_vi_pool( + __in efx_nic_t *enp, + __out uint32_t *evq_countp, + __out uint32_t *rxq_countp, + __out uint32_t *txq_countp); + + +#if EFSYS_OPT_VPD + +typedef enum efx_vpd_tag_e { + EFX_VPD_ID = 0x02, + EFX_VPD_END = 0x0f, + EFX_VPD_RO = 0x10, + EFX_VPD_RW = 0x11, +} efx_vpd_tag_t; + +typedef uint16_t efx_vpd_keyword_t; + +typedef struct efx_vpd_value_s { + efx_vpd_tag_t evv_tag; + efx_vpd_keyword_t evv_keyword; + uint8_t evv_length; + uint8_t evv_value[0x100]; +} efx_vpd_value_t; + + +#define EFX_VPD_KEYWORD(x, y) ((x) | ((y) << 8)) + +extern __checkReturn efx_rc_t +efx_vpd_init( + __in efx_nic_t *enp); + +extern __checkReturn efx_rc_t +efx_vpd_size( + __in efx_nic_t *enp, + __out size_t *sizep); + +extern __checkReturn efx_rc_t +efx_vpd_read( + __in efx_nic_t *enp, + __out_bcount(size) caddr_t data, + __in size_t size); + +extern __checkReturn efx_rc_t +efx_vpd_verify( + __in efx_nic_t *enp, + __in_bcount(size) caddr_t data, + __in size_t size); + +extern __checkReturn efx_rc_t +efx_vpd_reinit( + __in efx_nic_t *enp, + __in_bcount(size) caddr_t data, + __in size_t size); + +extern __checkReturn efx_rc_t +efx_vpd_get( + __in efx_nic_t *enp, + __in_bcount(size) caddr_t data, + __in size_t size, + __inout efx_vpd_value_t *evvp); + +extern __checkReturn efx_rc_t +efx_vpd_set( + __in efx_nic_t *enp, + __inout_bcount(size) caddr_t data, + __in size_t size, + __in efx_vpd_value_t *evvp); + +extern __checkReturn efx_rc_t +efx_vpd_next( + __in efx_nic_t *enp, + __inout_bcount(size) caddr_t data, + __in size_t size, + __out efx_vpd_value_t *evvp, + __inout unsigned int *contp); + +extern __checkReturn efx_rc_t +efx_vpd_write( + __in efx_nic_t *enp, + __in_bcount(size) caddr_t data, + __in size_t size); + +extern void +efx_vpd_fini( + __in efx_nic_t *enp); + +#endif /* EFSYS_OPT_VPD */ + +/* NVRAM */ + +#if EFSYS_OPT_NVRAM + +typedef enum efx_nvram_type_e { + EFX_NVRAM_INVALID = 0, + EFX_NVRAM_BOOTROM, + EFX_NVRAM_BOOTROM_CFG, + EFX_NVRAM_MC_FIRMWARE, + EFX_NVRAM_MC_GOLDEN, + EFX_NVRAM_PHY, + EFX_NVRAM_NULLPHY, + EFX_NVRAM_FPGA, + EFX_NVRAM_FCFW, + EFX_NVRAM_CPLD, + EFX_NVRAM_FPGA_BACKUP, + EFX_NVRAM_DYNAMIC_CFG, + EFX_NVRAM_LICENSE, + EFX_NVRAM_UEFIROM, + EFX_NVRAM_NTYPES, +} efx_nvram_type_t; + +extern __checkReturn efx_rc_t +efx_nvram_init( + __in efx_nic_t *enp); + +#if EFSYS_OPT_DIAG + +extern __checkReturn efx_rc_t +efx_nvram_test( + __in efx_nic_t *enp); + +#endif /* EFSYS_OPT_DIAG */ + +extern __checkReturn efx_rc_t +efx_nvram_size( + __in efx_nic_t *enp, + __in efx_nvram_type_t type, + __out size_t *sizep); + +extern __checkReturn efx_rc_t +efx_nvram_rw_start( + __in efx_nic_t *enp, + __in efx_nvram_type_t type, + __out_opt size_t *pref_chunkp); + +extern __checkReturn efx_rc_t +efx_nvram_rw_finish( + __in efx_nic_t *enp, + __in efx_nvram_type_t type); + +extern __checkReturn efx_rc_t +efx_nvram_get_version( + __in efx_nic_t *enp, + __in efx_nvram_type_t type, + __out uint32_t *subtypep, + __out_ecount(4) uint16_t version[4]); + +extern __checkReturn efx_rc_t +efx_nvram_read_chunk( + __in efx_nic_t *enp, + __in efx_nvram_type_t type, + __in unsigned int offset, + __out_bcount(size) caddr_t data, + __in size_t size); + +extern __checkReturn efx_rc_t +efx_nvram_set_version( + __in efx_nic_t *enp, + __in efx_nvram_type_t type, + __in_ecount(4) uint16_t version[4]); + +extern __checkReturn efx_rc_t +efx_nvram_validate( + __in efx_nic_t *enp, + __in efx_nvram_type_t type, + __in_bcount(partn_size) caddr_t partn_data, + __in size_t partn_size); + +extern __checkReturn efx_rc_t +efx_nvram_erase( + __in efx_nic_t *enp, + __in efx_nvram_type_t type); + +extern __checkReturn efx_rc_t +efx_nvram_write_chunk( + __in efx_nic_t *enp, + __in efx_nvram_type_t type, + __in unsigned int offset, + __in_bcount(size) caddr_t data, + __in size_t size); + +extern void +efx_nvram_fini( + __in efx_nic_t *enp); + +#endif /* EFSYS_OPT_NVRAM */ + +#if EFSYS_OPT_BOOTCFG + +/* Report size and offset of bootcfg sector in NVRAM partition. */ +extern __checkReturn efx_rc_t +efx_bootcfg_sector_info( + __in efx_nic_t *enp, + __in uint32_t pf, + __out_opt uint32_t *sector_countp, + __out size_t *offsetp, + __out size_t *max_sizep); + +/* + * Copy bootcfg sector data to a target buffer which may differ in size. + * Optionally corrects format errors in source buffer. + */ +extern efx_rc_t +efx_bootcfg_copy_sector( + __in efx_nic_t *enp, + __inout_bcount(sector_length) + uint8_t *sector, + __in size_t sector_length, + __out_bcount(data_size) uint8_t *data, + __in size_t data_size, + __in boolean_t handle_format_errors); + +extern efx_rc_t +efx_bootcfg_read( + __in efx_nic_t *enp, + __out_bcount(size) caddr_t data, + __in size_t size); + +extern efx_rc_t +efx_bootcfg_write( + __in efx_nic_t *enp, + __in_bcount(size) caddr_t data, + __in size_t size); + +#endif /* EFSYS_OPT_BOOTCFG */ + +#if EFSYS_OPT_DIAG + +typedef enum efx_pattern_type_t { + EFX_PATTERN_BYTE_INCREMENT = 0, + EFX_PATTERN_ALL_THE_SAME, + EFX_PATTERN_BIT_ALTERNATE, + EFX_PATTERN_BYTE_ALTERNATE, + EFX_PATTERN_BYTE_CHANGING, + EFX_PATTERN_BIT_SWEEP, + EFX_PATTERN_NTYPES +} efx_pattern_type_t; + +typedef void +(*efx_sram_pattern_fn_t)( + __in size_t row, + __in boolean_t negate, + __out efx_qword_t *eqp); + +extern __checkReturn efx_rc_t +efx_sram_test( + __in efx_nic_t *enp, + __in efx_pattern_type_t type); + +#endif /* EFSYS_OPT_DIAG */ + +extern __checkReturn efx_rc_t +efx_sram_buf_tbl_set( + __in efx_nic_t *enp, + __in uint32_t id, + __in efsys_mem_t *esmp, + __in size_t n); + +extern void +efx_sram_buf_tbl_clear( + __in efx_nic_t *enp, + __in uint32_t id, + __in size_t n); + +#define EFX_BUF_TBL_SIZE 0x20000 + +#define EFX_BUF_SIZE 4096 + +/* EV */ + +typedef struct efx_evq_s efx_evq_t; + +#if EFSYS_OPT_QSTATS + +/* START MKCONFIG GENERATED EfxHeaderEventQueueBlock 6f3843f5fe7cc843 */ +typedef enum efx_ev_qstat_e { + EV_ALL, + EV_RX, + EV_RX_OK, + EV_RX_FRM_TRUNC, + EV_RX_TOBE_DISC, + EV_RX_PAUSE_FRM_ERR, + EV_RX_BUF_OWNER_ID_ERR, + EV_RX_IPV4_HDR_CHKSUM_ERR, + EV_RX_TCP_UDP_CHKSUM_ERR, + EV_RX_ETH_CRC_ERR, + EV_RX_IP_FRAG_ERR, + EV_RX_MCAST_PKT, + EV_RX_MCAST_HASH_MATCH, + EV_RX_TCP_IPV4, + EV_RX_TCP_IPV6, + EV_RX_UDP_IPV4, + EV_RX_UDP_IPV6, + EV_RX_OTHER_IPV4, + EV_RX_OTHER_IPV6, + EV_RX_NON_IP, + EV_RX_BATCH, + EV_TX, + EV_TX_WQ_FF_FULL, + EV_TX_PKT_ERR, + EV_TX_PKT_TOO_BIG, + EV_TX_UNEXPECTED, + EV_GLOBAL, + EV_GLOBAL_MNT, + EV_DRIVER, + EV_DRIVER_SRM_UPD_DONE, + EV_DRIVER_TX_DESCQ_FLS_DONE, + EV_DRIVER_RX_DESCQ_FLS_DONE, + EV_DRIVER_RX_DESCQ_FLS_FAILED, + EV_DRIVER_RX_DSC_ERROR, + EV_DRIVER_TX_DSC_ERROR, + EV_DRV_GEN, + EV_MCDI_RESPONSE, + EV_NQSTATS +} efx_ev_qstat_t; + +/* END MKCONFIG GENERATED EfxHeaderEventQueueBlock */ + +#endif /* EFSYS_OPT_QSTATS */ + +extern __checkReturn efx_rc_t +efx_ev_init( + __in efx_nic_t *enp); + +extern void +efx_ev_fini( + __in efx_nic_t *enp); + +#define EFX_EVQ_MAXNEVS 32768 +#define EFX_EVQ_MINNEVS 512 + +#define EFX_EVQ_SIZE(_nevs) ((_nevs) * sizeof (efx_qword_t)) +#define EFX_EVQ_NBUFS(_nevs) (EFX_EVQ_SIZE(_nevs) / EFX_BUF_SIZE) + +#define EFX_EVQ_FLAGS_TYPE_MASK (0x3) +#define EFX_EVQ_FLAGS_TYPE_AUTO (0x0) +#define EFX_EVQ_FLAGS_TYPE_THROUGHPUT (0x1) +#define EFX_EVQ_FLAGS_TYPE_LOW_LATENCY (0x2) + +#define EFX_EVQ_FLAGS_NOTIFY_MASK (0xC) +#define EFX_EVQ_FLAGS_NOTIFY_INTERRUPT (0x0) /* Interrupting (default) */ +#define EFX_EVQ_FLAGS_NOTIFY_DISABLED (0x4) /* Non-interrupting */ + +extern __checkReturn efx_rc_t +efx_ev_qcreate( + __in efx_nic_t *enp, + __in unsigned int index, + __in efsys_mem_t *esmp, + __in size_t n, + __in uint32_t id, + __in uint32_t us, + __in uint32_t flags, + __deref_out efx_evq_t **eepp); + +extern void +efx_ev_qpost( + __in efx_evq_t *eep, + __in uint16_t data); + +typedef __checkReturn boolean_t +(*efx_initialized_ev_t)( + __in_opt void *arg); + +#define EFX_PKT_UNICAST 0x0004 +#define EFX_PKT_START 0x0008 + +#define EFX_PKT_VLAN_TAGGED 0x0010 +#define EFX_CKSUM_TCPUDP 0x0020 +#define EFX_CKSUM_IPV4 0x0040 +#define EFX_PKT_CONT 0x0080 + +#define EFX_CHECK_VLAN 0x0100 +#define EFX_PKT_TCP 0x0200 +#define EFX_PKT_UDP 0x0400 +#define EFX_PKT_IPV4 0x0800 + +#define EFX_PKT_IPV6 0x1000 +#define EFX_PKT_PREFIX_LEN 0x2000 +#define EFX_ADDR_MISMATCH 0x4000 +#define EFX_DISCARD 0x8000 + +/* + * The following flags are used only for packed stream + * mode. The values for the flags are reused to fit into 16 bit, + * since EFX_PKT_START and EFX_PKT_CONT are never used in + * packed stream mode + */ +#define EFX_PKT_PACKED_STREAM_NEW_BUFFER EFX_PKT_START +#define EFX_PKT_PACKED_STREAM_PARSE_INCOMPLETE EFX_PKT_CONT + + +#define EFX_EV_RX_NLABELS 32 +#define EFX_EV_TX_NLABELS 32 + +typedef __checkReturn boolean_t +(*efx_rx_ev_t)( + __in_opt void *arg, + __in uint32_t label, + __in uint32_t id, + __in uint32_t size, + __in uint16_t flags); + +#if EFSYS_OPT_RX_PACKED_STREAM + +/* + * Packed stream mode is documented in SF-112241-TC. + * The general idea is that, instead of putting each incoming + * packet into a separate buffer which is specified in a RX + * descriptor, a large buffer is provided to the hardware and + * packets are put there in a continuous stream. + * The main advantage of such an approach is that RX queue refilling + * happens much less frequently. + */ + +typedef __checkReturn boolean_t +(*efx_rx_ps_ev_t)( + __in_opt void *arg, + __in uint32_t label, + __in uint32_t id, + __in uint32_t pkt_count, + __in uint16_t flags); + +#endif + +typedef __checkReturn boolean_t +(*efx_tx_ev_t)( + __in_opt void *arg, + __in uint32_t label, + __in uint32_t id); + +#define EFX_EXCEPTION_RX_RECOVERY 0x00000001 +#define EFX_EXCEPTION_RX_DSC_ERROR 0x00000002 +#define EFX_EXCEPTION_TX_DSC_ERROR 0x00000003 +#define EFX_EXCEPTION_UNKNOWN_SENSOREVT 0x00000004 +#define EFX_EXCEPTION_FWALERT_SRAM 0x00000005 +#define EFX_EXCEPTION_UNKNOWN_FWALERT 0x00000006 +#define EFX_EXCEPTION_RX_ERROR 0x00000007 +#define EFX_EXCEPTION_TX_ERROR 0x00000008 +#define EFX_EXCEPTION_EV_ERROR 0x00000009 + +typedef __checkReturn boolean_t +(*efx_exception_ev_t)( + __in_opt void *arg, + __in uint32_t label, + __in uint32_t data); + +typedef __checkReturn boolean_t +(*efx_rxq_flush_done_ev_t)( + __in_opt void *arg, + __in uint32_t rxq_index); + +typedef __checkReturn boolean_t +(*efx_rxq_flush_failed_ev_t)( + __in_opt void *arg, + __in uint32_t rxq_index); + +typedef __checkReturn boolean_t +(*efx_txq_flush_done_ev_t)( + __in_opt void *arg, + __in uint32_t txq_index); + +typedef __checkReturn boolean_t +(*efx_software_ev_t)( + __in_opt void *arg, + __in uint16_t magic); + +typedef __checkReturn boolean_t +(*efx_sram_ev_t)( + __in_opt void *arg, + __in uint32_t code); + +#define EFX_SRAM_CLEAR 0 +#define EFX_SRAM_UPDATE 1 +#define EFX_SRAM_ILLEGAL_CLEAR 2 + +typedef __checkReturn boolean_t +(*efx_wake_up_ev_t)( + __in_opt void *arg, + __in uint32_t label); + +typedef __checkReturn boolean_t +(*efx_timer_ev_t)( + __in_opt void *arg, + __in uint32_t label); + +typedef __checkReturn boolean_t +(*efx_link_change_ev_t)( + __in_opt void *arg, + __in efx_link_mode_t link_mode); + +#if EFSYS_OPT_MON_STATS + +typedef __checkReturn boolean_t +(*efx_monitor_ev_t)( + __in_opt void *arg, + __in efx_mon_stat_t id, + __in efx_mon_stat_value_t value); + +#endif /* EFSYS_OPT_MON_STATS */ + +#if EFSYS_OPT_MAC_STATS + +typedef __checkReturn boolean_t +(*efx_mac_stats_ev_t)( + __in_opt void *arg, + __in uint32_t generation + ); + +#endif /* EFSYS_OPT_MAC_STATS */ + +typedef struct efx_ev_callbacks_s { + efx_initialized_ev_t eec_initialized; + efx_rx_ev_t eec_rx; +#if EFSYS_OPT_RX_PACKED_STREAM + efx_rx_ps_ev_t eec_rx_ps; +#endif + efx_tx_ev_t eec_tx; + efx_exception_ev_t eec_exception; + efx_rxq_flush_done_ev_t eec_rxq_flush_done; + efx_rxq_flush_failed_ev_t eec_rxq_flush_failed; + efx_txq_flush_done_ev_t eec_txq_flush_done; + efx_software_ev_t eec_software; + efx_sram_ev_t eec_sram; + efx_wake_up_ev_t eec_wake_up; + efx_timer_ev_t eec_timer; + efx_link_change_ev_t eec_link_change; +#if EFSYS_OPT_MON_STATS + efx_monitor_ev_t eec_monitor; +#endif /* EFSYS_OPT_MON_STATS */ +#if EFSYS_OPT_MAC_STATS + efx_mac_stats_ev_t eec_mac_stats; +#endif /* EFSYS_OPT_MAC_STATS */ +} efx_ev_callbacks_t; + +extern __checkReturn boolean_t +efx_ev_qpending( + __in efx_evq_t *eep, + __in unsigned int count); + +#if EFSYS_OPT_EV_PREFETCH + +extern void +efx_ev_qprefetch( + __in efx_evq_t *eep, + __in unsigned int count); + +#endif /* EFSYS_OPT_EV_PREFETCH */ + +extern void +efx_ev_qpoll( + __in efx_evq_t *eep, + __inout unsigned int *countp, + __in const efx_ev_callbacks_t *eecp, + __in_opt void *arg); + +extern __checkReturn efx_rc_t +efx_ev_usecs_to_ticks( + __in efx_nic_t *enp, + __in unsigned int usecs, + __out unsigned int *ticksp); + +extern __checkReturn efx_rc_t +efx_ev_qmoderate( + __in efx_evq_t *eep, + __in unsigned int us); + +extern __checkReturn efx_rc_t +efx_ev_qprime( + __in efx_evq_t *eep, + __in unsigned int count); + +#if EFSYS_OPT_QSTATS + +#if EFSYS_OPT_NAMES + +extern const char * +efx_ev_qstat_name( + __in efx_nic_t *enp, + __in unsigned int id); + +#endif /* EFSYS_OPT_NAMES */ + +extern void +efx_ev_qstats_update( + __in efx_evq_t *eep, + __inout_ecount(EV_NQSTATS) efsys_stat_t *stat); + +#endif /* EFSYS_OPT_QSTATS */ + +extern void +efx_ev_qdestroy( + __in efx_evq_t *eep); + +/* RX */ + +extern __checkReturn efx_rc_t +efx_rx_init( + __inout efx_nic_t *enp); + +extern void +efx_rx_fini( + __in efx_nic_t *enp); + +#if EFSYS_OPT_RX_SCATTER + __checkReturn efx_rc_t +efx_rx_scatter_enable( + __in efx_nic_t *enp, + __in unsigned int buf_size); +#endif /* EFSYS_OPT_RX_SCATTER */ + +/* Handle to represent use of the default RSS context. */ +#define EFX_RSS_CONTEXT_DEFAULT 0xffffffff + +#if EFSYS_OPT_RX_SCALE + +typedef enum efx_rx_hash_alg_e { + EFX_RX_HASHALG_LFSR = 0, + EFX_RX_HASHALG_TOEPLITZ +} efx_rx_hash_alg_t; + +#define EFX_RX_HASH_IPV4 (1U << 0) +#define EFX_RX_HASH_TCPIPV4 (1U << 1) +#define EFX_RX_HASH_IPV6 (1U << 2) +#define EFX_RX_HASH_TCPIPV6 (1U << 3) + +typedef unsigned int efx_rx_hash_type_t; + +typedef enum efx_rx_hash_support_e { + EFX_RX_HASH_UNAVAILABLE = 0, /* Hardware hash not inserted */ + EFX_RX_HASH_AVAILABLE /* Insert hash with/without RSS */ +} efx_rx_hash_support_t; + +#define EFX_RSS_KEY_SIZE 40 /* RSS key size (bytes) */ +#define EFX_RSS_TBL_SIZE 128 /* Rows in RX indirection table */ +#define EFX_MAXRSS 64 /* RX indirection entry range */ +#define EFX_MAXRSS_LEGACY 16 /* See bug16611 and bug17213 */ + +typedef enum efx_rx_scale_context_type_e { + EFX_RX_SCALE_UNAVAILABLE = 0, /* No RX scale context */ + EFX_RX_SCALE_EXCLUSIVE, /* Writable key/indirection table */ + EFX_RX_SCALE_SHARED /* Read-only key/indirection table */ +} efx_rx_scale_context_type_t; + +extern __checkReturn efx_rc_t +efx_rx_hash_default_support_get( + __in efx_nic_t *enp, + __out efx_rx_hash_support_t *supportp); + + +extern __checkReturn efx_rc_t +efx_rx_scale_default_support_get( + __in efx_nic_t *enp, + __out efx_rx_scale_context_type_t *typep); + +extern __checkReturn efx_rc_t +efx_rx_scale_context_alloc( + __in efx_nic_t *enp, + __in efx_rx_scale_context_type_t type, + __in uint32_t num_queues, + __out uint32_t *rss_contextp); + +extern __checkReturn efx_rc_t +efx_rx_scale_context_free( + __in efx_nic_t *enp, + __in uint32_t rss_context); + +extern __checkReturn efx_rc_t +efx_rx_scale_mode_set( + __in efx_nic_t *enp, + __in uint32_t rss_context, + __in efx_rx_hash_alg_t alg, + __in efx_rx_hash_type_t type, + __in boolean_t insert); + +extern __checkReturn efx_rc_t +efx_rx_scale_tbl_set( + __in efx_nic_t *enp, + __in uint32_t rss_context, + __in_ecount(n) unsigned int *table, + __in size_t n); + +extern __checkReturn efx_rc_t +efx_rx_scale_key_set( + __in efx_nic_t *enp, + __in uint32_t rss_context, + __in_ecount(n) uint8_t *key, + __in size_t n); + +extern __checkReturn uint32_t +efx_pseudo_hdr_hash_get( + __in efx_rxq_t *erp, + __in efx_rx_hash_alg_t func, + __in uint8_t *buffer); + +#endif /* EFSYS_OPT_RX_SCALE */ + +extern __checkReturn efx_rc_t +efx_pseudo_hdr_pkt_length_get( + __in efx_rxq_t *erp, + __in uint8_t *buffer, + __out uint16_t *pkt_lengthp); + +#define EFX_RXQ_MAXNDESCS 4096 +#define EFX_RXQ_MINNDESCS 512 + +#define EFX_RXQ_SIZE(_ndescs) ((_ndescs) * sizeof (efx_qword_t)) +#define EFX_RXQ_NBUFS(_ndescs) (EFX_RXQ_SIZE(_ndescs) / EFX_BUF_SIZE) +#define EFX_RXQ_LIMIT(_ndescs) ((_ndescs) - 16) +#define EFX_RXQ_DC_NDESCS(_dcsize) (8 << _dcsize) + +typedef enum efx_rxq_type_e { + EFX_RXQ_TYPE_DEFAULT, + EFX_RXQ_TYPE_SCATTER, + EFX_RXQ_TYPE_PACKED_STREAM_1M, + EFX_RXQ_TYPE_PACKED_STREAM_512K, + EFX_RXQ_TYPE_PACKED_STREAM_256K, + EFX_RXQ_TYPE_PACKED_STREAM_128K, + EFX_RXQ_TYPE_PACKED_STREAM_64K, + EFX_RXQ_NTYPES +} efx_rxq_type_t; + +extern __checkReturn efx_rc_t +efx_rx_qcreate( + __in efx_nic_t *enp, + __in unsigned int index, + __in unsigned int label, + __in efx_rxq_type_t type, + __in efsys_mem_t *esmp, + __in size_t n, + __in uint32_t id, + __in efx_evq_t *eep, + __deref_out efx_rxq_t **erpp); + +typedef struct efx_buffer_s { + efsys_dma_addr_t eb_addr; + size_t eb_size; + boolean_t eb_eop; +} efx_buffer_t; + +typedef struct efx_desc_s { + efx_qword_t ed_eq; +} efx_desc_t; + +extern void +efx_rx_qpost( + __in efx_rxq_t *erp, + __in_ecount(n) efsys_dma_addr_t *addrp, + __in size_t size, + __in unsigned int n, + __in unsigned int completed, + __in unsigned int added); + +extern void +efx_rx_qpush( + __in efx_rxq_t *erp, + __in unsigned int added, + __inout unsigned int *pushedp); + +#if EFSYS_OPT_RX_PACKED_STREAM + +/* + * Fake length for RXQ descriptors in packed stream mode + * to make hardware happy + */ +#define EFX_RXQ_PACKED_STREAM_FAKE_BUF_SIZE 32 + +extern void +efx_rx_qps_update_credits( + __in efx_rxq_t *erp); + +extern __checkReturn uint8_t * +efx_rx_qps_packet_info( + __in efx_rxq_t *erp, + __in uint8_t *buffer, + __in uint32_t buffer_length, + __in uint32_t current_offset, + __out uint16_t *lengthp, + __out uint32_t *next_offsetp, + __out uint32_t *timestamp); +#endif + +extern __checkReturn efx_rc_t +efx_rx_qflush( + __in efx_rxq_t *erp); + +extern void +efx_rx_qenable( + __in efx_rxq_t *erp); + +extern void +efx_rx_qdestroy( + __in efx_rxq_t *erp); + +/* TX */ + +typedef struct efx_txq_s efx_txq_t; + +#if EFSYS_OPT_QSTATS + +/* START MKCONFIG GENERATED EfxHeaderTransmitQueueBlock 12dff8778598b2db */ +typedef enum efx_tx_qstat_e { + TX_POST, + TX_POST_PIO, + TX_NQSTATS +} efx_tx_qstat_t; + +/* END MKCONFIG GENERATED EfxHeaderTransmitQueueBlock */ + +#endif /* EFSYS_OPT_QSTATS */ + +extern __checkReturn efx_rc_t +efx_tx_init( + __in efx_nic_t *enp); + +extern void +efx_tx_fini( + __in efx_nic_t *enp); + +#define EFX_TXQ_MINNDESCS 512 + +#define EFX_TXQ_SIZE(_ndescs) ((_ndescs) * sizeof (efx_qword_t)) +#define EFX_TXQ_NBUFS(_ndescs) (EFX_TXQ_SIZE(_ndescs) / EFX_BUF_SIZE) +#define EFX_TXQ_LIMIT(_ndescs) ((_ndescs) - 16) +#define EFX_TXQ_DC_NDESCS(_dcsize) (8 << _dcsize) + +#define EFX_TXQ_MAX_BUFS 8 /* Maximum independent of EFX_BUG35388_WORKAROUND. */ + +#define EFX_TXQ_CKSUM_IPV4 0x0001 +#define EFX_TXQ_CKSUM_TCPUDP 0x0002 +#define EFX_TXQ_FATSOV2 0x0004 + +extern __checkReturn efx_rc_t +efx_tx_qcreate( + __in efx_nic_t *enp, + __in unsigned int index, + __in unsigned int label, + __in efsys_mem_t *esmp, + __in size_t n, + __in uint32_t id, + __in uint16_t flags, + __in efx_evq_t *eep, + __deref_out efx_txq_t **etpp, + __out unsigned int *addedp); + +extern __checkReturn efx_rc_t +efx_tx_qpost( + __in efx_txq_t *etp, + __in_ecount(n) efx_buffer_t *eb, + __in unsigned int n, + __in unsigned int completed, + __inout unsigned int *addedp); + +extern __checkReturn efx_rc_t +efx_tx_qpace( + __in efx_txq_t *etp, + __in unsigned int ns); + +extern void +efx_tx_qpush( + __in efx_txq_t *etp, + __in unsigned int added, + __in unsigned int pushed); + +extern __checkReturn efx_rc_t +efx_tx_qflush( + __in efx_txq_t *etp); + +extern void +efx_tx_qenable( + __in efx_txq_t *etp); + +extern __checkReturn efx_rc_t +efx_tx_qpio_enable( + __in efx_txq_t *etp); + +extern void +efx_tx_qpio_disable( + __in efx_txq_t *etp); + +extern __checkReturn efx_rc_t +efx_tx_qpio_write( + __in efx_txq_t *etp, + __in_ecount(buf_length) uint8_t *buffer, + __in size_t buf_length, + __in size_t pio_buf_offset); + +extern __checkReturn efx_rc_t +efx_tx_qpio_post( + __in efx_txq_t *etp, + __in size_t pkt_length, + __in unsigned int completed, + __inout unsigned int *addedp); + +extern __checkReturn efx_rc_t +efx_tx_qdesc_post( + __in efx_txq_t *etp, + __in_ecount(n) efx_desc_t *ed, + __in unsigned int n, + __in unsigned int completed, + __inout unsigned int *addedp); + +extern void +efx_tx_qdesc_dma_create( + __in efx_txq_t *etp, + __in efsys_dma_addr_t addr, + __in size_t size, + __in boolean_t eop, + __out efx_desc_t *edp); + +extern void +efx_tx_qdesc_tso_create( + __in efx_txq_t *etp, + __in uint16_t ipv4_id, + __in uint32_t tcp_seq, + __in uint8_t tcp_flags, + __out efx_desc_t *edp); + +/* Number of FATSOv2 option descriptors */ +#define EFX_TX_FATSOV2_OPT_NDESCS 2 + +/* Maximum number of DMA segments per TSO packet (not superframe) */ +#define EFX_TX_FATSOV2_DMA_SEGS_PER_PKT_MAX 24 + +extern void +efx_tx_qdesc_tso2_create( + __in efx_txq_t *etp, + __in uint16_t ipv4_id, + __in uint32_t tcp_seq, + __in uint16_t tcp_mss, + __out_ecount(count) efx_desc_t *edp, + __in int count); + +extern void +efx_tx_qdesc_vlantci_create( + __in efx_txq_t *etp, + __in uint16_t tci, + __out efx_desc_t *edp); + +#if EFSYS_OPT_QSTATS + +#if EFSYS_OPT_NAMES + +extern const char * +efx_tx_qstat_name( + __in efx_nic_t *etp, + __in unsigned int id); + +#endif /* EFSYS_OPT_NAMES */ + +extern void +efx_tx_qstats_update( + __in efx_txq_t *etp, + __inout_ecount(TX_NQSTATS) efsys_stat_t *stat); + +#endif /* EFSYS_OPT_QSTATS */ + +extern void +efx_tx_qdestroy( + __in efx_txq_t *etp); + + +/* FILTER */ + +#if EFSYS_OPT_FILTER + +#define EFX_ETHER_TYPE_IPV4 0x0800 +#define EFX_ETHER_TYPE_IPV6 0x86DD + +#define EFX_IPPROTO_TCP 6 +#define EFX_IPPROTO_UDP 17 +#define EFX_IPPROTO_GRE 47 + +/* Use RSS to spread across multiple queues */ +#define EFX_FILTER_FLAG_RX_RSS 0x01 +/* Enable RX scatter */ +#define EFX_FILTER_FLAG_RX_SCATTER 0x02 +/* + * Override an automatic filter (priority EFX_FILTER_PRI_AUTO). + * May only be set by the filter implementation for each type. + * A removal request will restore the automatic filter in its place. + */ +#define EFX_FILTER_FLAG_RX_OVER_AUTO 0x04 +/* Filter is for RX */ +#define EFX_FILTER_FLAG_RX 0x08 +/* Filter is for TX */ +#define EFX_FILTER_FLAG_TX 0x10 + +typedef unsigned int efx_filter_flags_t; + +/* + * Flags which specify the fields to match on. The values are the same as in the + * MC_CMD_FILTER_OP/MC_CMD_FILTER_OP_EXT commands. + */ +typedef enum efx_filter_match_flags_e { + EFX_FILTER_MATCH_REM_HOST = 0x0001, /* Match by remote IP host + * address */ + EFX_FILTER_MATCH_LOC_HOST = 0x0002, /* Match by local IP host + * address */ + EFX_FILTER_MATCH_REM_MAC = 0x0004, /* Match by remote MAC address */ + EFX_FILTER_MATCH_REM_PORT = 0x0008, /* Match by remote TCP/UDP port */ + EFX_FILTER_MATCH_LOC_MAC = 0x0010, /* Match by remote TCP/UDP port */ + EFX_FILTER_MATCH_LOC_PORT = 0x0020, /* Match by local TCP/UDP port */ + EFX_FILTER_MATCH_ETHER_TYPE = 0x0040, /* Match by Ether-type */ + EFX_FILTER_MATCH_INNER_VID = 0x0080, /* Match by inner VLAN ID */ + EFX_FILTER_MATCH_OUTER_VID = 0x0100, /* Match by outer VLAN ID */ + EFX_FILTER_MATCH_IP_PROTO = 0x0200, /* Match by IP transport + * protocol */ + /* For encapsulated packets, match all multicast inner frames */ + EFX_FILTER_MATCH_IFRM_UNKNOWN_MCAST_DST = 0x01000000, + /* For encapsulated packets, match all unicast inner frames */ + EFX_FILTER_MATCH_IFRM_UNKNOWN_UCAST_DST = 0x02000000, + /* Match otherwise-unmatched multicast and broadcast packets */ + EFX_FILTER_MATCH_UNKNOWN_MCAST_DST = 0x40000000, + /* Match otherwise-unmatched unicast packets */ + EFX_FILTER_MATCH_UNKNOWN_UCAST_DST = 0x80000000, +} efx_filter_match_flags_t; + +typedef enum efx_filter_priority_s { + EFX_FILTER_PRI_HINT = 0, /* Performance hint */ + EFX_FILTER_PRI_AUTO, /* Automatic filter based on device + * address list or hardware + * requirements. This may only be used + * by the filter implementation for + * each NIC type. */ + EFX_FILTER_PRI_MANUAL, /* Manually configured filter */ + EFX_FILTER_PRI_REQUIRED, /* Required for correct behaviour of the + * client (e.g. SR-IOV, HyperV VMQ etc.) + */ +} efx_filter_priority_t; + +/* + * FIXME: All these fields are assumed to be in little-endian byte order. + * It may be better for some to be big-endian. See bug42804. + */ + +typedef struct efx_filter_spec_s { + uint32_t efs_match_flags; + uint32_t efs_priority:2; + uint32_t efs_flags:6; + uint32_t efs_dmaq_id:12; + uint32_t efs_rss_context; + uint16_t efs_outer_vid; + uint16_t efs_inner_vid; + uint8_t efs_loc_mac[EFX_MAC_ADDR_LEN]; + uint8_t efs_rem_mac[EFX_MAC_ADDR_LEN]; + uint16_t efs_ether_type; + uint8_t efs_ip_proto; + efx_tunnel_protocol_t efs_encap_type; + uint16_t efs_loc_port; + uint16_t efs_rem_port; + efx_oword_t efs_rem_host; + efx_oword_t efs_loc_host; +} efx_filter_spec_t; + + +/* Default values for use in filter specifications */ +#define EFX_FILTER_SPEC_RX_DMAQ_ID_DROP 0xfff +#define EFX_FILTER_SPEC_VID_UNSPEC 0xffff + +extern __checkReturn efx_rc_t +efx_filter_init( + __in efx_nic_t *enp); + +extern void +efx_filter_fini( + __in efx_nic_t *enp); + +extern __checkReturn efx_rc_t +efx_filter_insert( + __in efx_nic_t *enp, + __inout efx_filter_spec_t *spec); + +extern __checkReturn efx_rc_t +efx_filter_remove( + __in efx_nic_t *enp, + __inout efx_filter_spec_t *spec); + +extern __checkReturn efx_rc_t +efx_filter_restore( + __in efx_nic_t *enp); + +extern __checkReturn efx_rc_t +efx_filter_supported_filters( + __in efx_nic_t *enp, + __out_ecount(buffer_length) uint32_t *buffer, + __in size_t buffer_length, + __out size_t *list_lengthp); + +extern void +efx_filter_spec_init_rx( + __out efx_filter_spec_t *spec, + __in efx_filter_priority_t priority, + __in efx_filter_flags_t flags, + __in efx_rxq_t *erp); + +extern void +efx_filter_spec_init_tx( + __out efx_filter_spec_t *spec, + __in efx_txq_t *etp); + +extern __checkReturn efx_rc_t +efx_filter_spec_set_ipv4_local( + __inout efx_filter_spec_t *spec, + __in uint8_t proto, + __in uint32_t host, + __in uint16_t port); + +extern __checkReturn efx_rc_t +efx_filter_spec_set_ipv4_full( + __inout efx_filter_spec_t *spec, + __in uint8_t proto, + __in uint32_t lhost, + __in uint16_t lport, + __in uint32_t rhost, + __in uint16_t rport); + +extern __checkReturn efx_rc_t +efx_filter_spec_set_eth_local( + __inout efx_filter_spec_t *spec, + __in uint16_t vid, + __in const uint8_t *addr); + +extern void +efx_filter_spec_set_ether_type( + __inout efx_filter_spec_t *spec, + __in uint16_t ether_type); + +extern __checkReturn efx_rc_t +efx_filter_spec_set_uc_def( + __inout efx_filter_spec_t *spec); + +extern __checkReturn efx_rc_t +efx_filter_spec_set_mc_def( + __inout efx_filter_spec_t *spec); + +typedef enum efx_filter_inner_frame_match_e { + EFX_FILTER_INNER_FRAME_MATCH_OTHER = 0, + EFX_FILTER_INNER_FRAME_MATCH_UNKNOWN_MCAST_DST, + EFX_FILTER_INNER_FRAME_MATCH_UNKNOWN_UCAST_DST +} efx_filter_inner_frame_match_t; + +extern __checkReturn efx_rc_t +efx_filter_spec_set_encap_type( + __inout efx_filter_spec_t *spec, + __in efx_tunnel_protocol_t encap_type, + __in efx_filter_inner_frame_match_t inner_frame_match); + +#if EFSYS_OPT_RX_SCALE +extern __checkReturn efx_rc_t +efx_filter_spec_set_rss_context( + __inout efx_filter_spec_t *spec, + __in uint32_t rss_context); +#endif +#endif /* EFSYS_OPT_FILTER */ + +/* HASH */ + +extern __checkReturn uint32_t +efx_hash_dwords( + __in_ecount(count) uint32_t const *input, + __in size_t count, + __in uint32_t init); + +extern __checkReturn uint32_t +efx_hash_bytes( + __in_ecount(length) uint8_t const *input, + __in size_t length, + __in uint32_t init); + +#if EFSYS_OPT_LICENSING + +/* LICENSING */ + +typedef struct efx_key_stats_s { + uint32_t eks_valid; + uint32_t eks_invalid; + uint32_t eks_blacklisted; + uint32_t eks_unverifiable; + uint32_t eks_wrong_node; + uint32_t eks_licensed_apps_lo; + uint32_t eks_licensed_apps_hi; + uint32_t eks_licensed_features_lo; + uint32_t eks_licensed_features_hi; +} efx_key_stats_t; + +extern __checkReturn efx_rc_t +efx_lic_init( + __in efx_nic_t *enp); + +extern void +efx_lic_fini( + __in efx_nic_t *enp); + +extern __checkReturn boolean_t +efx_lic_check_support( + __in efx_nic_t *enp); + +extern __checkReturn efx_rc_t +efx_lic_update_licenses( + __in efx_nic_t *enp); + +extern __checkReturn efx_rc_t +efx_lic_get_key_stats( + __in efx_nic_t *enp, + __out efx_key_stats_t *ksp); + +extern __checkReturn efx_rc_t +efx_lic_app_state( + __in efx_nic_t *enp, + __in uint64_t app_id, + __out boolean_t *licensedp); + +extern __checkReturn efx_rc_t +efx_lic_get_id( + __in efx_nic_t *enp, + __in size_t buffer_size, + __out uint32_t *typep, + __out size_t *lengthp, + __out_opt uint8_t *bufferp); + + +extern __checkReturn efx_rc_t +efx_lic_find_start( + __in efx_nic_t *enp, + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size, + __out uint32_t *startp + ); + +extern __checkReturn efx_rc_t +efx_lic_find_end( + __in efx_nic_t *enp, + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size, + __in uint32_t offset, + __out uint32_t *endp + ); + +extern __checkReturn __success(return != B_FALSE) boolean_t +efx_lic_find_key( + __in efx_nic_t *enp, + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size, + __in uint32_t offset, + __out uint32_t *startp, + __out uint32_t *lengthp + ); + +extern __checkReturn __success(return != B_FALSE) boolean_t +efx_lic_validate_key( + __in efx_nic_t *enp, + __in_bcount(length) caddr_t keyp, + __in uint32_t length + ); + +extern __checkReturn efx_rc_t +efx_lic_read_key( + __in efx_nic_t *enp, + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size, + __in uint32_t offset, + __in uint32_t length, + __out_bcount_part(key_max_size, *lengthp) + caddr_t keyp, + __in size_t key_max_size, + __out uint32_t *lengthp + ); + +extern __checkReturn efx_rc_t +efx_lic_write_key( + __in efx_nic_t *enp, + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size, + __in uint32_t offset, + __in_bcount(length) caddr_t keyp, + __in uint32_t length, + __out uint32_t *lengthp + ); + + __checkReturn efx_rc_t +efx_lic_delete_key( + __in efx_nic_t *enp, + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size, + __in uint32_t offset, + __in uint32_t length, + __in uint32_t end, + __out uint32_t *deltap + ); + +extern __checkReturn efx_rc_t +efx_lic_create_partition( + __in efx_nic_t *enp, + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size + ); + +extern __checkReturn efx_rc_t +efx_lic_finish_partition( + __in efx_nic_t *enp, + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size + ); + +#endif /* EFSYS_OPT_LICENSING */ + + + +#ifdef __cplusplus +} +#endif + +#endif /* _SYS_EFX_H */ diff --git a/dpdk/drivers/net/sfc/base/efx_bootcfg.c b/dpdk/drivers/net/sfc/base/efx_bootcfg.c new file mode 100644 index 00000000..d589c86a --- /dev/null +++ b/dpdk/drivers/net/sfc/base/efx_bootcfg.c @@ -0,0 +1,563 @@ +/* + * Copyright (c) 2009-2016 Solarflare Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are + * those of the authors and should not be interpreted as representing official + * policies, either expressed or implied, of the FreeBSD Project. + */ + +#include "efx.h" +#include "efx_impl.h" + +#if EFSYS_OPT_BOOTCFG + +/* + * Maximum size of BOOTCFG block across all nics as understood by SFCgPXE. + * NOTE: This is larger than the Medford per-PF bootcfg sector. + */ +#define BOOTCFG_MAX_SIZE 0x1000 + +/* Medford per-PF bootcfg sector */ +#define BOOTCFG_PER_PF 0x800 +#define BOOTCFG_PF_COUNT 16 + +#define DHCP_END ((uint8_t)0xff) +#define DHCP_PAD ((uint8_t)0) + + +/* Report the layout of bootcfg sectors in NVRAM partition. */ + __checkReturn efx_rc_t +efx_bootcfg_sector_info( + __in efx_nic_t *enp, + __in uint32_t pf, + __out_opt uint32_t *sector_countp, + __out size_t *offsetp, + __out size_t *max_sizep) +{ + uint32_t count; + size_t max_size; + size_t offset; + int rc; + + switch (enp->en_family) { +#if EFSYS_OPT_SIENA + case EFX_FAMILY_SIENA: + max_size = BOOTCFG_MAX_SIZE; + offset = 0; + count = 1; + break; +#endif /* EFSYS_OPT_SIENA */ + +#if EFSYS_OPT_HUNTINGTON + case EFX_FAMILY_HUNTINGTON: + max_size = BOOTCFG_MAX_SIZE; + offset = 0; + count = 1; + break; +#endif /* EFSYS_OPT_HUNTINGTON */ + +#if EFSYS_OPT_MEDFORD + case EFX_FAMILY_MEDFORD: { + /* Shared partition (array indexed by PF) */ + max_size = BOOTCFG_PER_PF; + count = BOOTCFG_PF_COUNT; + if (pf >= count) { + rc = EINVAL; + goto fail2; + } + offset = max_size * pf; + break; + } +#endif /* EFSYS_OPT_MEDFORD */ + + default: + EFSYS_ASSERT(0); + rc = ENOTSUP; + goto fail1; + } + EFSYS_ASSERT3U(max_size, <=, BOOTCFG_MAX_SIZE); + + if (sector_countp != NULL) + *sector_countp = count; + *offsetp = offset; + *max_sizep = max_size; + + return (0); + +#if EFSYS_OPT_MEDFORD +fail2: + EFSYS_PROBE(fail2); +#endif +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + return (rc); +} + + +static __checkReturn uint8_t +efx_bootcfg_csum( + __in efx_nic_t *enp, + __in_bcount(size) uint8_t const *data, + __in size_t size) +{ + _NOTE(ARGUNUSED(enp)) + + unsigned int pos; + uint8_t checksum = 0; + + for (pos = 0; pos < size; pos++) + checksum += data[pos]; + return (checksum); +} + +static __checkReturn efx_rc_t +efx_bootcfg_verify( + __in efx_nic_t *enp, + __in_bcount(size) uint8_t const *data, + __in size_t size, + __out_opt size_t *usedp) +{ + size_t offset = 0; + size_t used = 0; + efx_rc_t rc; + + /* Start parsing tags immediately after the checksum */ + for (offset = 1; offset < size; ) { + uint8_t tag; + uint8_t length; + + /* Consume tag */ + tag = data[offset]; + if (tag == DHCP_END) { + offset++; + used = offset; + break; + } + if (tag == DHCP_PAD) { + offset++; + continue; + } + + /* Consume length */ + if (offset + 1 >= size) { + rc = ENOSPC; + goto fail1; + } + length = data[offset + 1]; + + /* Consume *length */ + if (offset + 1 + length >= size) { + rc = ENOSPC; + goto fail2; + } + + offset += 2 + length; + used = offset; + } + + /* Checksum the entire sector, including bytes after any DHCP_END */ + if (efx_bootcfg_csum(enp, data, size) != 0) { + rc = EINVAL; + goto fail3; + } + + if (usedp != NULL) + *usedp = used; + + return (0); + +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +/* + * Copy bootcfg sector data to a target buffer which may differ in size. + * Optionally corrects format errors in source buffer. + */ + efx_rc_t +efx_bootcfg_copy_sector( + __in efx_nic_t *enp, + __inout_bcount(sector_length) + uint8_t *sector, + __in size_t sector_length, + __out_bcount(data_size) uint8_t *data, + __in size_t data_size, + __in boolean_t handle_format_errors) +{ + size_t used_bytes; + efx_rc_t rc; + + /* Verify that the area is correctly formatted and checksummed */ + rc = efx_bootcfg_verify(enp, sector, sector_length, + &used_bytes); + + if (!handle_format_errors) { + if (rc != 0) + goto fail1; + + if ((used_bytes < 2) || + (sector[used_bytes - 1] != DHCP_END)) { + /* Block too short, or DHCP_END missing */ + rc = ENOENT; + goto fail2; + } + } + + /* Synthesize empty format on verification failure */ + if (rc != 0 || used_bytes == 0) { + sector[0] = 0; + sector[1] = DHCP_END; + used_bytes = 2; + } + EFSYS_ASSERT(used_bytes >= 2); /* checksum and DHCP_END */ + EFSYS_ASSERT(used_bytes <= sector_length); + EFSYS_ASSERT(sector_length >= 2); + + /* + * Legacy bootcfg sectors don't terminate with a DHCP_END character. + * Modify the returned payload so it does. + * Reinitialise the sector if there isn't room for the character. + */ + if (sector[used_bytes - 1] != DHCP_END) { + if (used_bytes >= sector_length) { + sector[0] = 0; + used_bytes = 1; + } + sector[used_bytes] = DHCP_END; + ++used_bytes; + } + + /* + * Verify that the target buffer is large enough for the + * entire used bootcfg area, then copy into the target buffer. + */ + if (used_bytes > data_size) { + rc = ENOSPC; + goto fail3; + } + memcpy(data, sector, used_bytes); + + /* Zero out the unused portion of the target buffer */ + if (used_bytes < data_size) + (void) memset(data + used_bytes, 0, data_size - used_bytes); + + /* + * The checksum includes trailing data after any DHCP_END character, + * which we've just modified (by truncation or appending DHCP_END). + */ + data[0] -= efx_bootcfg_csum(enp, data, data_size); + + return (0); + +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + efx_rc_t +efx_bootcfg_read( + __in efx_nic_t *enp, + __out_bcount(size) caddr_t data, + __in size_t size) +{ + uint8_t *payload = NULL; + size_t used_bytes; + size_t partn_length; + size_t sector_length; + size_t sector_offset; + efx_rc_t rc; + uint32_t sector_number; + +#if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD + sector_number = enp->en_nic_cfg.enc_pf; +#else + sector_number = 0; +#endif + rc = efx_nvram_size(enp, EFX_NVRAM_BOOTROM_CFG, &partn_length); + if (rc != 0) + goto fail1; + + /* The bootcfg sector may be stored in a (larger) shared partition */ + rc = efx_bootcfg_sector_info(enp, sector_number, + NULL, §or_offset, §or_length); + if (rc != 0) + goto fail2; + + if (sector_length > BOOTCFG_MAX_SIZE) + sector_length = BOOTCFG_MAX_SIZE; + + if (sector_offset + sector_length > partn_length) { + /* Partition is too small */ + rc = EFBIG; + goto fail3; + } + + /* + * We need to read the entire BOOTCFG sector to ensure we read all the + * tags, because legacy bootcfg sectors are not guaranteed to end with + * a DHCP_END character. If the user hasn't supplied a sufficiently + * large buffer then use our own buffer. + */ + if (sector_length > size) { + EFSYS_KMEM_ALLOC(enp->en_esip, sector_length, payload); + if (payload == NULL) { + rc = ENOMEM; + goto fail4; + } + } else + payload = (uint8_t *)data; + + if ((rc = efx_nvram_rw_start(enp, EFX_NVRAM_BOOTROM_CFG, NULL)) != 0) + goto fail5; + + if ((rc = efx_nvram_read_chunk(enp, EFX_NVRAM_BOOTROM_CFG, + sector_offset, (caddr_t)payload, sector_length)) != 0) { + (void) efx_nvram_rw_finish(enp, EFX_NVRAM_BOOTROM_CFG); + goto fail6; + } + + if ((rc = efx_nvram_rw_finish(enp, EFX_NVRAM_BOOTROM_CFG)) != 0) + goto fail7; + + /* Verify that the area is correctly formatted and checksummed */ + rc = efx_bootcfg_verify(enp, (caddr_t)payload, sector_length, + &used_bytes); + if (rc != 0 || used_bytes == 0) { + payload[0] = (uint8_t)~DHCP_END; + payload[1] = DHCP_END; + used_bytes = 2; + } + + EFSYS_ASSERT(used_bytes >= 2); /* checksum and DHCP_END */ + EFSYS_ASSERT(used_bytes <= sector_length); + + /* + * Legacy bootcfg sectors don't terminate with a DHCP_END character. + * Modify the returned payload so it does. BOOTCFG_MAX_SIZE is by + * definition large enough for any valid (per-port) bootcfg sector, + * so reinitialise the sector if there isn't room for the character. + */ + if (payload[used_bytes - 1] != DHCP_END) { + if (used_bytes + 1 > sector_length) { + payload[0] = 0; + used_bytes = 1; + } + + payload[used_bytes] = DHCP_END; + ++used_bytes; + } + + /* + * Verify that the user supplied buffer is large enough for the + * entire used bootcfg area, then copy into the user supplied buffer. + */ + if (used_bytes > size) { + rc = ENOSPC; + goto fail8; + } + if (sector_length > size) { + memcpy(data, payload, used_bytes); + EFSYS_KMEM_FREE(enp->en_esip, sector_length, payload); + } + + /* Zero out the unused portion of the user buffer */ + if (used_bytes < size) + (void) memset(data + used_bytes, 0, size - used_bytes); + + /* + * The checksum includes trailing data after any DHCP_END character, + * which we've just modified (by truncation or appending DHCP_END). + */ + data[0] -= efx_bootcfg_csum(enp, data, size); + + return (0); + +fail8: + EFSYS_PROBE(fail8); +fail7: + EFSYS_PROBE(fail7); +fail6: + EFSYS_PROBE(fail6); +fail5: + EFSYS_PROBE(fail5); + if (sector_length > size) + EFSYS_KMEM_FREE(enp->en_esip, sector_length, payload); +fail4: + EFSYS_PROBE(fail4); +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + efx_rc_t +efx_bootcfg_write( + __in efx_nic_t *enp, + __in_bcount(size) caddr_t data, + __in size_t size) +{ + uint8_t *partn_data; + uint8_t checksum; + size_t partn_length; + size_t sector_length; + size_t sector_offset; + size_t used_bytes; + efx_rc_t rc; + uint32_t sector_number; + +#if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD + sector_number = enp->en_nic_cfg.enc_pf; +#else + sector_number = 0; +#endif + + rc = efx_nvram_size(enp, EFX_NVRAM_BOOTROM_CFG, &partn_length); + if (rc != 0) + goto fail1; + + /* The bootcfg sector may be stored in a (larger) shared partition */ + rc = efx_bootcfg_sector_info(enp, sector_number, + NULL, §or_offset, §or_length); + if (rc != 0) + goto fail2; + + if (sector_length > BOOTCFG_MAX_SIZE) + sector_length = BOOTCFG_MAX_SIZE; + + if (sector_offset + sector_length > partn_length) { + /* Partition is too small */ + rc = EFBIG; + goto fail3; + } + + if ((rc = efx_bootcfg_verify(enp, data, size, &used_bytes)) != 0) + goto fail4; + + /* The caller *must* terminate their block with a DHCP_END character */ + if ((used_bytes < 2) || ((uint8_t)data[used_bytes - 1] != DHCP_END)) { + /* Block too short or DHCP_END missing */ + rc = ENOENT; + goto fail5; + } + + /* Check that the hardware has support for this much data */ + if (used_bytes > MIN(sector_length, BOOTCFG_MAX_SIZE)) { + rc = ENOSPC; + goto fail6; + } + + /* + * If the BOOTCFG sector is stored in a shared partition, then we must + * read the whole partition and insert the updated bootcfg sector at the + * correct offset. + */ + EFSYS_KMEM_ALLOC(enp->en_esip, partn_length, partn_data); + if (partn_data == NULL) { + rc = ENOMEM; + goto fail7; + } + + rc = efx_nvram_rw_start(enp, EFX_NVRAM_BOOTROM_CFG, NULL); + if (rc != 0) + goto fail8; + + /* Read the entire partition */ + rc = efx_nvram_read_chunk(enp, EFX_NVRAM_BOOTROM_CFG, 0, + (caddr_t)partn_data, partn_length); + if (rc != 0) + goto fail9; + + /* + * Insert the BOOTCFG sector into the partition, Zero out all data after + * the DHCP_END tag, and adjust the checksum. + */ + (void) memset(partn_data + sector_offset, 0x0, sector_length); + (void) memcpy(partn_data + sector_offset, data, used_bytes); + + checksum = efx_bootcfg_csum(enp, data, used_bytes); + partn_data[sector_offset] -= checksum; + + if ((rc = efx_nvram_erase(enp, EFX_NVRAM_BOOTROM_CFG)) != 0) + goto fail10; + + if ((rc = efx_nvram_write_chunk(enp, EFX_NVRAM_BOOTROM_CFG, + 0, (caddr_t)partn_data, partn_length)) != 0) + goto fail11; + + if ((rc = efx_nvram_rw_finish(enp, EFX_NVRAM_BOOTROM_CFG)) != 0) + goto fail12; + + EFSYS_KMEM_FREE(enp->en_esip, partn_length, partn_data); + + return (0); + +fail12: + EFSYS_PROBE(fail12); +fail11: + EFSYS_PROBE(fail11); +fail10: + EFSYS_PROBE(fail10); +fail9: + EFSYS_PROBE(fail9); + + (void) efx_nvram_rw_finish(enp, EFX_NVRAM_BOOTROM_CFG); +fail8: + EFSYS_PROBE(fail8); + + EFSYS_KMEM_FREE(enp->en_esip, partn_length, partn_data); +fail7: + EFSYS_PROBE(fail7); +fail6: + EFSYS_PROBE(fail6); +fail5: + EFSYS_PROBE(fail5); +fail4: + EFSYS_PROBE(fail4); +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +#endif /* EFSYS_OPT_BOOTCFG */ diff --git a/dpdk/drivers/net/sfc/base/efx_check.h b/dpdk/drivers/net/sfc/base/efx_check.h new file mode 100644 index 00000000..c8548c04 --- /dev/null +++ b/dpdk/drivers/net/sfc/base/efx_check.h @@ -0,0 +1,346 @@ +/* + * Copyright (c) 2012-2016 Solarflare Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are + * those of the authors and should not be interpreted as representing official + * policies, either expressed or implied, of the FreeBSD Project. + */ + +#ifndef _SYS_EFX_CHECK_H +#define _SYS_EFX_CHECK_H + +#include "efsys.h" + +/* + * Check that the efsys.h header in client code has a valid combination of + * EFSYS_OPT_xxx options. + * + * NOTE: Keep checks for obsolete options here to ensure that they are removed + * from client code (and do not reappear in merges from other branches). + */ + +#ifdef EFSYS_OPT_FALCON +# error "FALCON is obsolete and is not supported." +#endif + +#if EFSYS_OPT_BOOTCFG +/* Support NVRAM based boot config */ +# if !EFSYS_OPT_NVRAM +# error "BOOTCFG requires NVRAM" +# endif +#endif /* EFSYS_OPT_BOOTCFG */ + +#if EFSYS_OPT_CHECK_REG +/* Verify chip implements accessed registers */ +# if !(EFSYS_OPT_SIENA || EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD) +# error "CHECK_REG requires SIENA or HUNTINGTON or MEDFORD" +# endif +#endif /* EFSYS_OPT_CHECK_REG */ + +#if EFSYS_OPT_DECODE_INTR_FATAL +/* Decode fatal errors */ +# if !EFSYS_OPT_SIENA +# error "INTR_FATAL requires SIENA" +# endif +#endif /* EFSYS_OPT_DECODE_INTR_FATAL */ + +#if EFSYS_OPT_DIAG +/* Support diagnostic hardware tests */ +# if !(EFSYS_OPT_SIENA || EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD) +# error "DIAG requires SIENA or HUNTINGTON or MEDFORD" +# endif +#endif /* EFSYS_OPT_DIAG */ + +#if EFSYS_OPT_EV_PREFETCH +/* Support optimized EVQ data access */ +# if !(EFSYS_OPT_SIENA || EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD) +# error "EV_PREFETCH requires SIENA or HUNTINGTON or MEDFORD" +# endif +#endif /* EFSYS_OPT_EV_PREFETCH */ + +#ifdef EFSYS_OPT_FALCON_NIC_CFG_OVERRIDE +# error "FALCON_NIC_CFG_OVERRIDE is obsolete and is not supported." +#endif + +#if EFSYS_OPT_FILTER +/* Support hardware packet filters */ +# if !(EFSYS_OPT_SIENA || EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD) +# error "FILTER requires SIENA or HUNTINGTON or MEDFORD" +# endif +#endif /* EFSYS_OPT_FILTER */ + +#if (EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD) +# if !EFSYS_OPT_FILTER +# error "HUNTINGTON or MEDFORD requires FILTER" +# endif +#endif /* EFSYS_OPT_HUNTINGTON */ + +#if EFSYS_OPT_LOOPBACK +/* Support hardware loopback modes */ +# if !(EFSYS_OPT_SIENA || EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD) +# error "LOOPBACK requires SIENA or HUNTINGTON or MEDFORD" +# endif +#endif /* EFSYS_OPT_LOOPBACK */ + +#ifdef EFSYS_OPT_MAC_FALCON_GMAC +# error "MAC_FALCON_GMAC is obsolete and is not supported." +#endif + +#ifdef EFSYS_OPT_MAC_FALCON_XMAC +# error "MAC_FALCON_XMAC is obsolete and is not supported." +#endif + +#if EFSYS_OPT_MAC_STATS +/* Support MAC statistics */ +# if !(EFSYS_OPT_SIENA || EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD) +# error "MAC_STATS requires SIENA or HUNTINGTON or MEDFORD" +# endif +#endif /* EFSYS_OPT_MAC_STATS */ + +#if EFSYS_OPT_MCDI +/* Support management controller messages */ +# if !(EFSYS_OPT_SIENA || EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD) +# error "MCDI requires SIENA or HUNTINGTON or MEDFORD" +# endif +#endif /* EFSYS_OPT_MCDI */ + +#if (EFSYS_OPT_SIENA || EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD) +# if !EFSYS_OPT_MCDI +# error "SIENA or HUNTINGTON or MEDFORD requires MCDI" +# endif +#endif + +#if EFSYS_OPT_MCDI_LOGGING +/* Support MCDI logging */ +# if !EFSYS_OPT_MCDI +# error "MCDI_LOGGING requires MCDI" +# endif +#endif /* EFSYS_OPT_MCDI_LOGGING */ + +#if EFSYS_OPT_MCDI_PROXY_AUTH +/* Support MCDI proxy authorization */ +# if !EFSYS_OPT_MCDI +# error "MCDI_PROXY_AUTH requires MCDI" +# endif +#endif /* EFSYS_OPT_MCDI_PROXY_AUTH */ + +#ifdef EFSYS_OPT_MON_LM87 +# error "MON_LM87 is obsolete and is not supported." +#endif + +#ifdef EFSYS_OPT_MON_MAX6647 +# error "MON_MAX6647 is obsolete and is not supported." +#endif + +#ifdef EFSYS_OPT_MON_NULL +# error "MON_NULL is obsolete and is not supported." +#endif + +#ifdef EFSYS_OPT_MON_SIENA +# error "MON_SIENA is obsolete (replaced by MON_MCDI)." +#endif + +#ifdef EFSYS_OPT_MON_HUNTINGTON +# error "MON_HUNTINGTON is obsolete (replaced by MON_MCDI)." +#endif + +#if EFSYS_OPT_MON_STATS +/* Support monitor statistics (voltage/temperature) */ +# if !(EFSYS_OPT_SIENA || EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD) +# error "MON_STATS requires SIENA or HUNTINGTON or MEDFORD" +# endif +#endif /* EFSYS_OPT_MON_STATS */ + +#if EFSYS_OPT_MON_MCDI +/* Support Monitor via mcdi */ +# if !(EFSYS_OPT_SIENA || EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD) +# error "MON_MCDI requires SIENA or HUNTINGTON or MEDFORD" +# endif +#endif /* EFSYS_OPT_MON_MCDI*/ + +#if EFSYS_OPT_NAMES +/* Support printable names for statistics */ +# if !(EFSYS_OPT_LOOPBACK || EFSYS_OPT_MAC_STATS || EFSYS_OPT_MCDI || \ + EFSYS_MON_STATS || EFSYS_OPT_PHY_STATS || EFSYS_OPT_QSTATS) +# error "NAMES requires LOOPBACK or xxxSTATS or MCDI" +# endif +#endif /* EFSYS_OPT_NAMES */ + +#if EFSYS_OPT_NVRAM +/* Support non volatile configuration */ +# if !(EFSYS_OPT_SIENA || EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD) +# error "NVRAM requires SIENA or HUNTINGTON or MEDFORD" +# endif +#endif /* EFSYS_OPT_NVRAM */ + +#ifdef EFSYS_OPT_NVRAM_FALCON_BOOTROM +# error "NVRAM_FALCON_BOOTROM is obsolete and is not supported." +#endif + +#ifdef EFSYS_OPT_NVRAM_SFT9001 +# error "NVRAM_SFT9001 is obsolete and is not supported." +#endif + +#ifdef EFSYS_OPT_NVRAM_SFX7101 +# error "NVRAM_SFX7101 is obsolete and is not supported." +#endif + +#ifdef EFSYS_OPT_PCIE_TUNE +# error "PCIE_TUNE is obsolete and is not supported." +#endif + +#ifdef EFSYS_OPT_PHY_BIST +# error "PHY_BIST is obsolete (replaced by BIST)." +#endif + +#if EFSYS_OPT_PHY_FLAGS +/* Support PHY flags */ +# if !EFSYS_OPT_SIENA +# error "PHY_FLAGS requires SIENA" +# endif +#endif /* EFSYS_OPT_PHY_FLAGS */ + +#if EFSYS_OPT_PHY_LED_CONTROL +/* Support for PHY LED control */ +# if !EFSYS_OPT_SIENA +# error "PHY_LED_CONTROL requires SIENA" +# endif +#endif /* EFSYS_OPT_PHY_LED_CONTROL */ + +#ifdef EFSYS_OPT_PHY_NULL +# error "PHY_NULL is obsolete and is not supported." +#endif + +#ifdef EFSYS_OPT_PHY_PM8358 +# error "PHY_PM8358 is obsolete and is not supported." +#endif + +#ifdef EFSYS_OPT_PHY_PROPS +# error "PHY_PROPS is obsolete and is not supported." +#endif + +#ifdef EFSYS_OPT_PHY_QT2022C2 +# error "PHY_QT2022C2 is obsolete and is not supported." +#endif + +#ifdef EFSYS_OPT_PHY_QT2025C +# error "PHY_QT2025C is obsolete and is not supported." +#endif + +#ifdef EFSYS_OPT_PHY_SFT9001 +# error "PHY_SFT9001 is obsolete and is not supported." +#endif + +#ifdef EFSYS_OPT_PHY_SFX7101 +# error "PHY_SFX7101 is obsolete and is not supported." +#endif + +#if EFSYS_OPT_PHY_STATS +/* Support PHY statistics */ +# if !EFSYS_OPT_SIENA +# error "PHY_STATS requires SIENA" +# endif +#endif /* EFSYS_OPT_PHY_STATS */ + +#ifdef EFSYS_OPT_PHY_TXC43128 +# error "PHY_TXC43128 is obsolete and is not supported." +#endif + +#if EFSYS_OPT_QSTATS +/* Support EVQ/RXQ/TXQ statistics */ +# if !(EFSYS_OPT_SIENA || EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD) +# error "QSTATS requires SIENA or HUNTINGTON or MEDFORD" +# endif +#endif /* EFSYS_OPT_QSTATS */ + +#ifdef EFSYS_OPT_RX_HDR_SPLIT +# error "RX_HDR_SPLIT is obsolete and is not supported" +#endif + +#if EFSYS_OPT_RX_SCALE +/* Support receive scaling (RSS) */ +# if !(EFSYS_OPT_SIENA || EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD) +# error "RX_SCALE requires SIENA or HUNTINGTON or MEDFORD" +# endif +#endif /* EFSYS_OPT_RX_SCALE */ + +#if EFSYS_OPT_RX_SCATTER +/* Support receive scatter DMA */ +# if !(EFSYS_OPT_SIENA || EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD) +# error "RX_SCATTER requires SIENA or HUNTINGTON or MEDFORD" +# endif +#endif /* EFSYS_OPT_RX_SCATTER */ + +#ifdef EFSYS_OPT_STAT_NAME +# error "STAT_NAME is obsolete (replaced by NAMES)." +#endif + +#if EFSYS_OPT_VPD +/* Support PCI Vital Product Data (VPD) */ +# if !(EFSYS_OPT_SIENA || EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD) +# error "VPD requires SIENA or HUNTINGTON or MEDFORD" +# endif +#endif /* EFSYS_OPT_VPD */ + +#ifdef EFSYS_OPT_WOL +# error "WOL is obsolete and is not supported" +#endif /* EFSYS_OPT_WOL */ + +#ifdef EFSYS_OPT_MCAST_FILTER_LIST +# error "MCAST_FILTER_LIST is obsolete and is not supported" +#endif + +#if EFSYS_OPT_BIST +/* Support BIST */ +# if !(EFSYS_OPT_SIENA || EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD) +# error "BIST requires SIENA or HUNTINGTON or MEDFORD" +# endif +#endif /* EFSYS_OPT_BIST */ + +#if EFSYS_OPT_LICENSING +/* Support MCDI licensing API */ +# if !EFSYS_OPT_MCDI +# error "LICENSING requires MCDI" +# endif +# if !EFSYS_HAS_UINT64 +# error "LICENSING requires UINT64" +# endif +#endif /* EFSYS_OPT_LICENSING */ + +#if EFSYS_OPT_ALLOW_UNCONFIGURED_NIC +/* Support adapters with missing static config (for factory use only) */ +# if !EFSYS_OPT_MEDFORD +# error "ALLOW_UNCONFIGURED_NIC requires MEDFORD" +# endif +#endif /* EFSYS_OPT_ALLOW_UNCONFIGURED_NIC */ + +#if EFSYS_OPT_RX_PACKED_STREAM +/* Support packed stream mode */ +# if !(EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD) +# error "PACKED_STREAM requires HUNTINGTON or MEDFORD" +# endif +#endif + +#endif /* _SYS_EFX_CHECK_H */ diff --git a/dpdk/drivers/net/sfc/base/efx_crc32.c b/dpdk/drivers/net/sfc/base/efx_crc32.c new file mode 100644 index 00000000..27e2708a --- /dev/null +++ b/dpdk/drivers/net/sfc/base/efx_crc32.c @@ -0,0 +1,122 @@ +/* + * Copyright (c) 2013-2016 Solarflare Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are + * those of the authors and should not be interpreted as representing official + * policies, either expressed or implied, of the FreeBSD Project. + */ + +#include "efx.h" +#include "efx_impl.h" + +/* + * Precomputed table for computing IEEE 802.3 CRC32 + * with polynomial 0x04c11db7 (bit-reversed 0xedb88320) + */ + +static const uint32_t efx_crc32_table[256] = { + 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, + 0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3, + 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, + 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, + 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, + 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, + 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, + 0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5, + 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, + 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, + 0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, + 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59, + 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, + 0x21b4f4b5, 0x56b3c423, 0xcfba9599, 0xb8bda50f, + 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, + 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, + 0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a, + 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433, + 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, + 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01, + 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, + 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, + 0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea, 0xfcb9887c, + 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65, + 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, + 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb, + 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, + 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, + 0x5005713c, 0x270241aa, 0xbe0b1010, 0xc90c2086, + 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, + 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, + 0x59b33d17, 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad, + 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, + 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, + 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, + 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, + 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, + 0xf762575d, 0x806567cb, 0x196c3671, 0x6e6b06e7, + 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, + 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, + 0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, + 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b, + 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, + 0xdf60efc3, 0xa867df55, 0x316e8eef, 0x4669be79, + 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, + 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, + 0xc5ba3bbe, 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, + 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d, + 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, + 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713, + 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, + 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, + 0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e, + 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777, + 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, + 0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45, + 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, + 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, + 0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0, + 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, + 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, + 0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf, + 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, + 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d +}; + +/* Calculate the IEEE 802.3 CRC32 of a MAC addr */ + __checkReturn uint32_t +efx_crc32_calculate( + __in uint32_t crc_init, + __in_ecount(length) uint8_t const *input, + __in int length) +{ + int index; + uint32_t crc = crc_init; + + for (index = 0; index < length; index++) { + uint32_t data = *(input++); + crc = (crc >> 8) ^ efx_crc32_table[(crc ^ data) & 0xff]; + } + + return (crc); +} diff --git a/dpdk/drivers/net/sfc/base/efx_ev.c b/dpdk/drivers/net/sfc/base/efx_ev.c new file mode 100644 index 00000000..42ded5aa --- /dev/null +++ b/dpdk/drivers/net/sfc/base/efx_ev.c @@ -0,0 +1,1470 @@ +/* + * Copyright (c) 2007-2016 Solarflare Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are + * those of the authors and should not be interpreted as representing official + * policies, either expressed or implied, of the FreeBSD Project. + */ + +#include "efx.h" +#include "efx_impl.h" +#if EFSYS_OPT_MON_MCDI +#include "mcdi_mon.h" +#endif + +#if EFSYS_OPT_QSTATS +#define EFX_EV_QSTAT_INCR(_eep, _stat) \ + do { \ + (_eep)->ee_stat[_stat]++; \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) +#else +#define EFX_EV_QSTAT_INCR(_eep, _stat) +#endif + +#define EFX_EV_PRESENT(_qword) \ + (EFX_QWORD_FIELD((_qword), EFX_DWORD_0) != 0xffffffff && \ + EFX_QWORD_FIELD((_qword), EFX_DWORD_1) != 0xffffffff) + + + +#if EFSYS_OPT_SIENA + +static __checkReturn efx_rc_t +siena_ev_init( + __in efx_nic_t *enp); + +static void +siena_ev_fini( + __in efx_nic_t *enp); + +static __checkReturn efx_rc_t +siena_ev_qcreate( + __in efx_nic_t *enp, + __in unsigned int index, + __in efsys_mem_t *esmp, + __in size_t n, + __in uint32_t id, + __in uint32_t us, + __in uint32_t flags, + __in efx_evq_t *eep); + +static void +siena_ev_qdestroy( + __in efx_evq_t *eep); + +static __checkReturn efx_rc_t +siena_ev_qprime( + __in efx_evq_t *eep, + __in unsigned int count); + +static void +siena_ev_qpost( + __in efx_evq_t *eep, + __in uint16_t data); + +static __checkReturn efx_rc_t +siena_ev_qmoderate( + __in efx_evq_t *eep, + __in unsigned int us); + +#if EFSYS_OPT_QSTATS +static void +siena_ev_qstats_update( + __in efx_evq_t *eep, + __inout_ecount(EV_NQSTATS) efsys_stat_t *stat); + +#endif + +#endif /* EFSYS_OPT_SIENA */ + +#if EFSYS_OPT_SIENA +static const efx_ev_ops_t __efx_ev_siena_ops = { + siena_ev_init, /* eevo_init */ + siena_ev_fini, /* eevo_fini */ + siena_ev_qcreate, /* eevo_qcreate */ + siena_ev_qdestroy, /* eevo_qdestroy */ + siena_ev_qprime, /* eevo_qprime */ + siena_ev_qpost, /* eevo_qpost */ + siena_ev_qmoderate, /* eevo_qmoderate */ +#if EFSYS_OPT_QSTATS + siena_ev_qstats_update, /* eevo_qstats_update */ +#endif +}; +#endif /* EFSYS_OPT_SIENA */ + +#if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD +static const efx_ev_ops_t __efx_ev_ef10_ops = { + ef10_ev_init, /* eevo_init */ + ef10_ev_fini, /* eevo_fini */ + ef10_ev_qcreate, /* eevo_qcreate */ + ef10_ev_qdestroy, /* eevo_qdestroy */ + ef10_ev_qprime, /* eevo_qprime */ + ef10_ev_qpost, /* eevo_qpost */ + ef10_ev_qmoderate, /* eevo_qmoderate */ +#if EFSYS_OPT_QSTATS + ef10_ev_qstats_update, /* eevo_qstats_update */ +#endif +}; +#endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */ + + + __checkReturn efx_rc_t +efx_ev_init( + __in efx_nic_t *enp) +{ + const efx_ev_ops_t *eevop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR); + + if (enp->en_mod_flags & EFX_MOD_EV) { + rc = EINVAL; + goto fail1; + } + + switch (enp->en_family) { +#if EFSYS_OPT_SIENA + case EFX_FAMILY_SIENA: + eevop = &__efx_ev_siena_ops; + break; +#endif /* EFSYS_OPT_SIENA */ + +#if EFSYS_OPT_HUNTINGTON + case EFX_FAMILY_HUNTINGTON: + eevop = &__efx_ev_ef10_ops; + break; +#endif /* EFSYS_OPT_HUNTINGTON */ + +#if EFSYS_OPT_MEDFORD + case EFX_FAMILY_MEDFORD: + eevop = &__efx_ev_ef10_ops; + break; +#endif /* EFSYS_OPT_MEDFORD */ + + default: + EFSYS_ASSERT(0); + rc = ENOTSUP; + goto fail1; + } + + EFSYS_ASSERT3U(enp->en_ev_qcount, ==, 0); + + if ((rc = eevop->eevo_init(enp)) != 0) + goto fail2; + + enp->en_eevop = eevop; + enp->en_mod_flags |= EFX_MOD_EV; + return (0); + +fail2: + EFSYS_PROBE(fail2); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + enp->en_eevop = NULL; + enp->en_mod_flags &= ~EFX_MOD_EV; + return (rc); +} + + void +efx_ev_fini( + __in efx_nic_t *enp) +{ + const efx_ev_ops_t *eevop = enp->en_eevop; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_EV); + EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_RX)); + EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_TX)); + EFSYS_ASSERT3U(enp->en_ev_qcount, ==, 0); + + eevop->eevo_fini(enp); + + enp->en_eevop = NULL; + enp->en_mod_flags &= ~EFX_MOD_EV; +} + + + __checkReturn efx_rc_t +efx_ev_qcreate( + __in efx_nic_t *enp, + __in unsigned int index, + __in efsys_mem_t *esmp, + __in size_t n, + __in uint32_t id, + __in uint32_t us, + __in uint32_t flags, + __deref_out efx_evq_t **eepp) +{ + const efx_ev_ops_t *eevop = enp->en_eevop; + efx_nic_cfg_t *encp = &(enp->en_nic_cfg); + efx_evq_t *eep; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_EV); + + EFSYS_ASSERT3U(enp->en_ev_qcount + 1, <, encp->enc_evq_limit); + + switch (flags & EFX_EVQ_FLAGS_NOTIFY_MASK) { + case EFX_EVQ_FLAGS_NOTIFY_INTERRUPT: + break; + case EFX_EVQ_FLAGS_NOTIFY_DISABLED: + if (us != 0) { + rc = EINVAL; + goto fail1; + } + break; + default: + rc = EINVAL; + goto fail2; + } + + /* Allocate an EVQ object */ + EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_evq_t), eep); + if (eep == NULL) { + rc = ENOMEM; + goto fail3; + } + + eep->ee_magic = EFX_EVQ_MAGIC; + eep->ee_enp = enp; + eep->ee_index = index; + eep->ee_mask = n - 1; + eep->ee_flags = flags; + eep->ee_esmp = esmp; + + /* + * Set outputs before the queue is created because interrupts may be + * raised for events immediately after the queue is created, before the + * function call below returns. See bug58606. + * + * The eepp pointer passed in by the client must therefore point to data + * shared with the client's event processing context. + */ + enp->en_ev_qcount++; + *eepp = eep; + + if ((rc = eevop->eevo_qcreate(enp, index, esmp, n, id, us, flags, + eep)) != 0) + goto fail4; + + return (0); + +fail4: + EFSYS_PROBE(fail4); + + *eepp = NULL; + enp->en_ev_qcount--; + EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_evq_t), eep); +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + return (rc); +} + + void +efx_ev_qdestroy( + __in efx_evq_t *eep) +{ + efx_nic_t *enp = eep->ee_enp; + const efx_ev_ops_t *eevop = enp->en_eevop; + + EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC); + + EFSYS_ASSERT(enp->en_ev_qcount != 0); + --enp->en_ev_qcount; + + eevop->eevo_qdestroy(eep); + + /* Free the EVQ object */ + EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_evq_t), eep); +} + + __checkReturn efx_rc_t +efx_ev_qprime( + __in efx_evq_t *eep, + __in unsigned int count) +{ + efx_nic_t *enp = eep->ee_enp; + const efx_ev_ops_t *eevop = enp->en_eevop; + efx_rc_t rc; + + EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC); + + if (!(enp->en_mod_flags & EFX_MOD_INTR)) { + rc = EINVAL; + goto fail1; + } + + if ((rc = eevop->eevo_qprime(eep, count)) != 0) + goto fail2; + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + return (rc); +} + + __checkReturn boolean_t +efx_ev_qpending( + __in efx_evq_t *eep, + __in unsigned int count) +{ + size_t offset; + efx_qword_t qword; + + EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC); + + offset = (count & eep->ee_mask) * sizeof (efx_qword_t); + EFSYS_MEM_READQ(eep->ee_esmp, offset, &qword); + + return (EFX_EV_PRESENT(qword)); +} + +#if EFSYS_OPT_EV_PREFETCH + + void +efx_ev_qprefetch( + __in efx_evq_t *eep, + __in unsigned int count) +{ + unsigned int offset; + + EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC); + + offset = (count & eep->ee_mask) * sizeof (efx_qword_t); + EFSYS_MEM_PREFETCH(eep->ee_esmp, offset); +} + +#endif /* EFSYS_OPT_EV_PREFETCH */ + +#define EFX_EV_BATCH 8 + + void +efx_ev_qpoll( + __in efx_evq_t *eep, + __inout unsigned int *countp, + __in const efx_ev_callbacks_t *eecp, + __in_opt void *arg) +{ + efx_qword_t ev[EFX_EV_BATCH]; + unsigned int batch; + unsigned int total; + unsigned int count; + unsigned int index; + size_t offset; + + /* Ensure events codes match for EF10 (Huntington/Medford) and Siena */ + EFX_STATIC_ASSERT(ESF_DZ_EV_CODE_LBN == FSF_AZ_EV_CODE_LBN); + EFX_STATIC_ASSERT(ESF_DZ_EV_CODE_WIDTH == FSF_AZ_EV_CODE_WIDTH); + + EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_RX_EV == FSE_AZ_EV_CODE_RX_EV); + EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_TX_EV == FSE_AZ_EV_CODE_TX_EV); + EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_DRIVER_EV == FSE_AZ_EV_CODE_DRIVER_EV); + EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_DRV_GEN_EV == + FSE_AZ_EV_CODE_DRV_GEN_EV); +#if EFSYS_OPT_MCDI + EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_MCDI_EV == + FSE_AZ_EV_CODE_MCDI_EVRESPONSE); +#endif + + EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC); + EFSYS_ASSERT(countp != NULL); + EFSYS_ASSERT(eecp != NULL); + + count = *countp; + do { + /* Read up until the end of the batch period */ + batch = EFX_EV_BATCH - (count & (EFX_EV_BATCH - 1)); + offset = (count & eep->ee_mask) * sizeof (efx_qword_t); + for (total = 0; total < batch; ++total) { + EFSYS_MEM_READQ(eep->ee_esmp, offset, &(ev[total])); + + if (!EFX_EV_PRESENT(ev[total])) + break; + + EFSYS_PROBE3(event, unsigned int, eep->ee_index, + uint32_t, EFX_QWORD_FIELD(ev[total], EFX_DWORD_1), + uint32_t, EFX_QWORD_FIELD(ev[total], EFX_DWORD_0)); + + offset += sizeof (efx_qword_t); + } + +#if EFSYS_OPT_EV_PREFETCH && (EFSYS_OPT_EV_PREFETCH_PERIOD > 1) + /* + * Prefetch the next batch when we get within PREFETCH_PERIOD + * of a completed batch. If the batch is smaller, then prefetch + * immediately. + */ + if (total == batch && total < EFSYS_OPT_EV_PREFETCH_PERIOD) + EFSYS_MEM_PREFETCH(eep->ee_esmp, offset); +#endif /* EFSYS_OPT_EV_PREFETCH */ + + /* Process the batch of events */ + for (index = 0; index < total; ++index) { + boolean_t should_abort; + uint32_t code; + +#if EFSYS_OPT_EV_PREFETCH + /* Prefetch if we've now reached the batch period */ + if (total == batch && + index + EFSYS_OPT_EV_PREFETCH_PERIOD == total) { + offset = (count + batch) & eep->ee_mask; + offset *= sizeof (efx_qword_t); + + EFSYS_MEM_PREFETCH(eep->ee_esmp, offset); + } +#endif /* EFSYS_OPT_EV_PREFETCH */ + + EFX_EV_QSTAT_INCR(eep, EV_ALL); + + code = EFX_QWORD_FIELD(ev[index], FSF_AZ_EV_CODE); + switch (code) { + case FSE_AZ_EV_CODE_RX_EV: + should_abort = eep->ee_rx(eep, + &(ev[index]), eecp, arg); + break; + case FSE_AZ_EV_CODE_TX_EV: + should_abort = eep->ee_tx(eep, + &(ev[index]), eecp, arg); + break; + case FSE_AZ_EV_CODE_DRIVER_EV: + should_abort = eep->ee_driver(eep, + &(ev[index]), eecp, arg); + break; + case FSE_AZ_EV_CODE_DRV_GEN_EV: + should_abort = eep->ee_drv_gen(eep, + &(ev[index]), eecp, arg); + break; +#if EFSYS_OPT_MCDI + case FSE_AZ_EV_CODE_MCDI_EVRESPONSE: + should_abort = eep->ee_mcdi(eep, + &(ev[index]), eecp, arg); + break; +#endif + case FSE_AZ_EV_CODE_GLOBAL_EV: + if (eep->ee_global) { + should_abort = eep->ee_global(eep, + &(ev[index]), eecp, arg); + break; + } + /* else fallthrough */ + default: + EFSYS_PROBE3(bad_event, + unsigned int, eep->ee_index, + uint32_t, + EFX_QWORD_FIELD(ev[index], EFX_DWORD_1), + uint32_t, + EFX_QWORD_FIELD(ev[index], EFX_DWORD_0)); + + EFSYS_ASSERT(eecp->eec_exception != NULL); + (void) eecp->eec_exception(arg, + EFX_EXCEPTION_EV_ERROR, code); + should_abort = B_TRUE; + } + if (should_abort) { + /* Ignore subsequent events */ + total = index + 1; + break; + } + } + + /* + * Now that the hardware has most likely moved onto dma'ing + * into the next cache line, clear the processed events. Take + * care to only clear out events that we've processed + */ + EFX_SET_QWORD(ev[0]); + offset = (count & eep->ee_mask) * sizeof (efx_qword_t); + for (index = 0; index < total; ++index) { + EFSYS_MEM_WRITEQ(eep->ee_esmp, offset, &(ev[0])); + offset += sizeof (efx_qword_t); + } + + count += total; + + } while (total == batch); + + *countp = count; +} + + void +efx_ev_qpost( + __in efx_evq_t *eep, + __in uint16_t data) +{ + efx_nic_t *enp = eep->ee_enp; + const efx_ev_ops_t *eevop = enp->en_eevop; + + EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC); + + EFSYS_ASSERT(eevop != NULL && + eevop->eevo_qpost != NULL); + + eevop->eevo_qpost(eep, data); +} + + __checkReturn efx_rc_t +efx_ev_usecs_to_ticks( + __in efx_nic_t *enp, + __in unsigned int us, + __out unsigned int *ticksp) +{ + efx_nic_cfg_t *encp = &(enp->en_nic_cfg); + unsigned int ticks; + + /* Convert microseconds to a timer tick count */ + if (us == 0) + ticks = 0; + else if (us * 1000 < encp->enc_evq_timer_quantum_ns) + ticks = 1; /* Never round down to zero */ + else + ticks = us * 1000 / encp->enc_evq_timer_quantum_ns; + + *ticksp = ticks; + return (0); +} + + __checkReturn efx_rc_t +efx_ev_qmoderate( + __in efx_evq_t *eep, + __in unsigned int us) +{ + efx_nic_t *enp = eep->ee_enp; + const efx_ev_ops_t *eevop = enp->en_eevop; + efx_rc_t rc; + + EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC); + + if ((eep->ee_flags & EFX_EVQ_FLAGS_NOTIFY_MASK) == + EFX_EVQ_FLAGS_NOTIFY_DISABLED) { + rc = EINVAL; + goto fail1; + } + + if ((rc = eevop->eevo_qmoderate(eep, us)) != 0) + goto fail2; + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + return (rc); +} + +#if EFSYS_OPT_QSTATS + void +efx_ev_qstats_update( + __in efx_evq_t *eep, + __inout_ecount(EV_NQSTATS) efsys_stat_t *stat) + +{ efx_nic_t *enp = eep->ee_enp; + const efx_ev_ops_t *eevop = enp->en_eevop; + + EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC); + + eevop->eevo_qstats_update(eep, stat); +} + +#endif /* EFSYS_OPT_QSTATS */ + +#if EFSYS_OPT_SIENA + +static __checkReturn efx_rc_t +siena_ev_init( + __in efx_nic_t *enp) +{ + efx_oword_t oword; + + /* + * Program the event queue for receive and transmit queue + * flush events. + */ + EFX_BAR_READO(enp, FR_AZ_DP_CTRL_REG, &oword); + EFX_SET_OWORD_FIELD(oword, FRF_AZ_FLS_EVQ_ID, 0); + EFX_BAR_WRITEO(enp, FR_AZ_DP_CTRL_REG, &oword); + + return (0); + +} + +static __checkReturn boolean_t +siena_ev_rx_not_ok( + __in efx_evq_t *eep, + __in efx_qword_t *eqp, + __in uint32_t label, + __in uint32_t id, + __inout uint16_t *flagsp) +{ + boolean_t ignore = B_FALSE; + + if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_TOBE_DISC) != 0) { + EFX_EV_QSTAT_INCR(eep, EV_RX_TOBE_DISC); + EFSYS_PROBE(tobe_disc); + /* + * Assume this is a unicast address mismatch, unless below + * we find either FSF_AZ_RX_EV_ETH_CRC_ERR or + * EV_RX_PAUSE_FRM_ERR is set. + */ + (*flagsp) |= EFX_ADDR_MISMATCH; + } + + if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_FRM_TRUNC) != 0) { + EFSYS_PROBE2(frm_trunc, uint32_t, label, uint32_t, id); + EFX_EV_QSTAT_INCR(eep, EV_RX_FRM_TRUNC); + (*flagsp) |= EFX_DISCARD; + +#if EFSYS_OPT_RX_SCATTER + /* + * Lookout for payload queue ran dry errors and ignore them. + * + * Sadly for the header/data split cases, the descriptor + * pointer in this event refers to the header queue and + * therefore cannot be easily detected as duplicate. + * So we drop these and rely on the receive processing seeing + * a subsequent packet with FSF_AZ_RX_EV_SOP set to discard + * the partially received packet. + */ + if ((EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_SOP) == 0) && + (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_JUMBO_CONT) == 0) && + (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_BYTE_CNT) == 0)) + ignore = B_TRUE; +#endif /* EFSYS_OPT_RX_SCATTER */ + } + + if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_ETH_CRC_ERR) != 0) { + EFX_EV_QSTAT_INCR(eep, EV_RX_ETH_CRC_ERR); + EFSYS_PROBE(crc_err); + (*flagsp) &= ~EFX_ADDR_MISMATCH; + (*flagsp) |= EFX_DISCARD; + } + + if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_PAUSE_FRM_ERR) != 0) { + EFX_EV_QSTAT_INCR(eep, EV_RX_PAUSE_FRM_ERR); + EFSYS_PROBE(pause_frm_err); + (*flagsp) &= ~EFX_ADDR_MISMATCH; + (*flagsp) |= EFX_DISCARD; + } + + if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_BUF_OWNER_ID_ERR) != 0) { + EFX_EV_QSTAT_INCR(eep, EV_RX_BUF_OWNER_ID_ERR); + EFSYS_PROBE(owner_id_err); + (*flagsp) |= EFX_DISCARD; + } + + if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_IP_HDR_CHKSUM_ERR) != 0) { + EFX_EV_QSTAT_INCR(eep, EV_RX_IPV4_HDR_CHKSUM_ERR); + EFSYS_PROBE(ipv4_err); + (*flagsp) &= ~EFX_CKSUM_IPV4; + } + + if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_TCP_UDP_CHKSUM_ERR) != 0) { + EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_UDP_CHKSUM_ERR); + EFSYS_PROBE(udp_chk_err); + (*flagsp) &= ~EFX_CKSUM_TCPUDP; + } + + if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_IP_FRAG_ERR) != 0) { + EFX_EV_QSTAT_INCR(eep, EV_RX_IP_FRAG_ERR); + + /* + * If IP is fragmented FSF_AZ_RX_EV_IP_FRAG_ERR is set. This + * causes FSF_AZ_RX_EV_PKT_OK to be clear. This is not an error + * condition. + */ + (*flagsp) &= ~(EFX_PKT_TCP | EFX_PKT_UDP | EFX_CKSUM_TCPUDP); + } + + return (ignore); +} + +static __checkReturn boolean_t +siena_ev_rx( + __in efx_evq_t *eep, + __in efx_qword_t *eqp, + __in const efx_ev_callbacks_t *eecp, + __in_opt void *arg) +{ + uint32_t id; + uint32_t size; + uint32_t label; + boolean_t ok; +#if EFSYS_OPT_RX_SCATTER + boolean_t sop; + boolean_t jumbo_cont; +#endif /* EFSYS_OPT_RX_SCATTER */ + uint32_t hdr_type; + boolean_t is_v6; + uint16_t flags; + boolean_t ignore; + boolean_t should_abort; + + EFX_EV_QSTAT_INCR(eep, EV_RX); + + /* Basic packet information */ + id = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_DESC_PTR); + size = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_BYTE_CNT); + label = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_Q_LABEL); + ok = (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_PKT_OK) != 0); + +#if EFSYS_OPT_RX_SCATTER + sop = (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_SOP) != 0); + jumbo_cont = (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_JUMBO_CONT) != 0); +#endif /* EFSYS_OPT_RX_SCATTER */ + + hdr_type = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_HDR_TYPE); + + is_v6 = (EFX_QWORD_FIELD(*eqp, FSF_CZ_RX_EV_IPV6_PKT) != 0); + + /* + * If packet is marked as OK and packet type is TCP/IP or + * UDP/IP or other IP, then we can rely on the hardware checksums. + */ + switch (hdr_type) { + case FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_TCP: + flags = EFX_PKT_TCP | EFX_CKSUM_TCPUDP; + if (is_v6) { + EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_IPV6); + flags |= EFX_PKT_IPV6; + } else { + EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_IPV4); + flags |= EFX_PKT_IPV4 | EFX_CKSUM_IPV4; + } + break; + + case FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_UDP: + flags = EFX_PKT_UDP | EFX_CKSUM_TCPUDP; + if (is_v6) { + EFX_EV_QSTAT_INCR(eep, EV_RX_UDP_IPV6); + flags |= EFX_PKT_IPV6; + } else { + EFX_EV_QSTAT_INCR(eep, EV_RX_UDP_IPV4); + flags |= EFX_PKT_IPV4 | EFX_CKSUM_IPV4; + } + break; + + case FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_OTHER: + if (is_v6) { + EFX_EV_QSTAT_INCR(eep, EV_RX_OTHER_IPV6); + flags = EFX_PKT_IPV6; + } else { + EFX_EV_QSTAT_INCR(eep, EV_RX_OTHER_IPV4); + flags = EFX_PKT_IPV4 | EFX_CKSUM_IPV4; + } + break; + + case FSE_AZ_RX_EV_HDR_TYPE_OTHER: + EFX_EV_QSTAT_INCR(eep, EV_RX_NON_IP); + flags = 0; + break; + + default: + EFSYS_ASSERT(B_FALSE); + flags = 0; + break; + } + +#if EFSYS_OPT_RX_SCATTER + /* Report scatter and header/lookahead split buffer flags */ + if (sop) + flags |= EFX_PKT_START; + if (jumbo_cont) + flags |= EFX_PKT_CONT; +#endif /* EFSYS_OPT_RX_SCATTER */ + + /* Detect errors included in the FSF_AZ_RX_EV_PKT_OK indication */ + if (!ok) { + ignore = siena_ev_rx_not_ok(eep, eqp, label, id, &flags); + if (ignore) { + EFSYS_PROBE4(rx_complete, uint32_t, label, uint32_t, id, + uint32_t, size, uint16_t, flags); + + return (B_FALSE); + } + } + + /* If we're not discarding the packet then it is ok */ + if (~flags & EFX_DISCARD) + EFX_EV_QSTAT_INCR(eep, EV_RX_OK); + + /* Detect multicast packets that didn't match the filter */ + if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_MCAST_PKT) != 0) { + EFX_EV_QSTAT_INCR(eep, EV_RX_MCAST_PKT); + + if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_MCAST_HASH_MATCH) != 0) { + EFX_EV_QSTAT_INCR(eep, EV_RX_MCAST_HASH_MATCH); + } else { + EFSYS_PROBE(mcast_mismatch); + flags |= EFX_ADDR_MISMATCH; + } + } else { + flags |= EFX_PKT_UNICAST; + } + + /* + * The packet parser in Siena can abort parsing packets under + * certain error conditions, setting the PKT_NOT_PARSED bit + * (which clears PKT_OK). If this is set, then don't trust + * the PKT_TYPE field. + */ + if (!ok) { + uint32_t parse_err; + + parse_err = EFX_QWORD_FIELD(*eqp, FSF_CZ_RX_EV_PKT_NOT_PARSED); + if (parse_err != 0) + flags |= EFX_CHECK_VLAN; + } + + if (~flags & EFX_CHECK_VLAN) { + uint32_t pkt_type; + + pkt_type = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_PKT_TYPE); + if (pkt_type >= FSE_AZ_RX_EV_PKT_TYPE_VLAN) + flags |= EFX_PKT_VLAN_TAGGED; + } + + EFSYS_PROBE4(rx_complete, uint32_t, label, uint32_t, id, + uint32_t, size, uint16_t, flags); + + EFSYS_ASSERT(eecp->eec_rx != NULL); + should_abort = eecp->eec_rx(arg, label, id, size, flags); + + return (should_abort); +} + +static __checkReturn boolean_t +siena_ev_tx( + __in efx_evq_t *eep, + __in efx_qword_t *eqp, + __in const efx_ev_callbacks_t *eecp, + __in_opt void *arg) +{ + uint32_t id; + uint32_t label; + boolean_t should_abort; + + EFX_EV_QSTAT_INCR(eep, EV_TX); + + if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_COMP) != 0 && + EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_ERR) == 0 && + EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_TOO_BIG) == 0 && + EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_WQ_FF_FULL) == 0) { + + id = EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_DESC_PTR); + label = EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_Q_LABEL); + + EFSYS_PROBE2(tx_complete, uint32_t, label, uint32_t, id); + + EFSYS_ASSERT(eecp->eec_tx != NULL); + should_abort = eecp->eec_tx(arg, label, id); + + return (should_abort); + } + + if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_COMP) != 0) + EFSYS_PROBE3(bad_event, unsigned int, eep->ee_index, + uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_1), + uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_0)); + + if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_ERR) != 0) + EFX_EV_QSTAT_INCR(eep, EV_TX_PKT_ERR); + + if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_TOO_BIG) != 0) + EFX_EV_QSTAT_INCR(eep, EV_TX_PKT_TOO_BIG); + + if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_WQ_FF_FULL) != 0) + EFX_EV_QSTAT_INCR(eep, EV_TX_WQ_FF_FULL); + + EFX_EV_QSTAT_INCR(eep, EV_TX_UNEXPECTED); + return (B_FALSE); +} + +static __checkReturn boolean_t +siena_ev_global( + __in efx_evq_t *eep, + __in efx_qword_t *eqp, + __in const efx_ev_callbacks_t *eecp, + __in_opt void *arg) +{ + _NOTE(ARGUNUSED(eqp, eecp, arg)) + + EFX_EV_QSTAT_INCR(eep, EV_GLOBAL); + + return (B_FALSE); +} + +static __checkReturn boolean_t +siena_ev_driver( + __in efx_evq_t *eep, + __in efx_qword_t *eqp, + __in const efx_ev_callbacks_t *eecp, + __in_opt void *arg) +{ + boolean_t should_abort; + + EFX_EV_QSTAT_INCR(eep, EV_DRIVER); + should_abort = B_FALSE; + + switch (EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBCODE)) { + case FSE_AZ_TX_DESCQ_FLS_DONE_EV: { + uint32_t txq_index; + + EFX_EV_QSTAT_INCR(eep, EV_DRIVER_TX_DESCQ_FLS_DONE); + + txq_index = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA); + + EFSYS_PROBE1(tx_descq_fls_done, uint32_t, txq_index); + + EFSYS_ASSERT(eecp->eec_txq_flush_done != NULL); + should_abort = eecp->eec_txq_flush_done(arg, txq_index); + + break; + } + case FSE_AZ_RX_DESCQ_FLS_DONE_EV: { + uint32_t rxq_index; + uint32_t failed; + + rxq_index = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_RX_DESCQ_ID); + failed = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_RX_FLUSH_FAIL); + + EFSYS_ASSERT(eecp->eec_rxq_flush_done != NULL); + EFSYS_ASSERT(eecp->eec_rxq_flush_failed != NULL); + + if (failed) { + EFX_EV_QSTAT_INCR(eep, EV_DRIVER_RX_DESCQ_FLS_FAILED); + + EFSYS_PROBE1(rx_descq_fls_failed, uint32_t, rxq_index); + + should_abort = eecp->eec_rxq_flush_failed(arg, + rxq_index); + } else { + EFX_EV_QSTAT_INCR(eep, EV_DRIVER_RX_DESCQ_FLS_DONE); + + EFSYS_PROBE1(rx_descq_fls_done, uint32_t, rxq_index); + + should_abort = eecp->eec_rxq_flush_done(arg, rxq_index); + } + + break; + } + case FSE_AZ_EVQ_INIT_DONE_EV: + EFSYS_ASSERT(eecp->eec_initialized != NULL); + should_abort = eecp->eec_initialized(arg); + + break; + + case FSE_AZ_EVQ_NOT_EN_EV: + EFSYS_PROBE(evq_not_en); + break; + + case FSE_AZ_SRM_UPD_DONE_EV: { + uint32_t code; + + EFX_EV_QSTAT_INCR(eep, EV_DRIVER_SRM_UPD_DONE); + + code = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA); + + EFSYS_ASSERT(eecp->eec_sram != NULL); + should_abort = eecp->eec_sram(arg, code); + + break; + } + case FSE_AZ_WAKE_UP_EV: { + uint32_t id; + + id = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA); + + EFSYS_ASSERT(eecp->eec_wake_up != NULL); + should_abort = eecp->eec_wake_up(arg, id); + + break; + } + case FSE_AZ_TX_PKT_NON_TCP_UDP: + EFSYS_PROBE(tx_pkt_non_tcp_udp); + break; + + case FSE_AZ_TIMER_EV: { + uint32_t id; + + id = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA); + + EFSYS_ASSERT(eecp->eec_timer != NULL); + should_abort = eecp->eec_timer(arg, id); + + break; + } + case FSE_AZ_RX_DSC_ERROR_EV: + EFX_EV_QSTAT_INCR(eep, EV_DRIVER_RX_DSC_ERROR); + + EFSYS_PROBE(rx_dsc_error); + + EFSYS_ASSERT(eecp->eec_exception != NULL); + should_abort = eecp->eec_exception(arg, + EFX_EXCEPTION_RX_DSC_ERROR, 0); + + break; + + case FSE_AZ_TX_DSC_ERROR_EV: + EFX_EV_QSTAT_INCR(eep, EV_DRIVER_TX_DSC_ERROR); + + EFSYS_PROBE(tx_dsc_error); + + EFSYS_ASSERT(eecp->eec_exception != NULL); + should_abort = eecp->eec_exception(arg, + EFX_EXCEPTION_TX_DSC_ERROR, 0); + + break; + + default: + break; + } + + return (should_abort); +} + +static __checkReturn boolean_t +siena_ev_drv_gen( + __in efx_evq_t *eep, + __in efx_qword_t *eqp, + __in const efx_ev_callbacks_t *eecp, + __in_opt void *arg) +{ + uint32_t data; + boolean_t should_abort; + + EFX_EV_QSTAT_INCR(eep, EV_DRV_GEN); + + data = EFX_QWORD_FIELD(*eqp, FSF_AZ_EV_DATA_DW0); + if (data >= ((uint32_t)1 << 16)) { + EFSYS_PROBE3(bad_event, unsigned int, eep->ee_index, + uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_1), + uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_0)); + return (B_TRUE); + } + + EFSYS_ASSERT(eecp->eec_software != NULL); + should_abort = eecp->eec_software(arg, (uint16_t)data); + + return (should_abort); +} + +#if EFSYS_OPT_MCDI + +static __checkReturn boolean_t +siena_ev_mcdi( + __in efx_evq_t *eep, + __in efx_qword_t *eqp, + __in const efx_ev_callbacks_t *eecp, + __in_opt void *arg) +{ + efx_nic_t *enp = eep->ee_enp; + unsigned int code; + boolean_t should_abort = B_FALSE; + + EFSYS_ASSERT3U(enp->en_family, ==, EFX_FAMILY_SIENA); + + if (enp->en_family != EFX_FAMILY_SIENA) + goto out; + + EFSYS_ASSERT(eecp->eec_link_change != NULL); + EFSYS_ASSERT(eecp->eec_exception != NULL); +#if EFSYS_OPT_MON_STATS + EFSYS_ASSERT(eecp->eec_monitor != NULL); +#endif + + EFX_EV_QSTAT_INCR(eep, EV_MCDI_RESPONSE); + + code = EFX_QWORD_FIELD(*eqp, MCDI_EVENT_CODE); + switch (code) { + case MCDI_EVENT_CODE_BADSSERT: + efx_mcdi_ev_death(enp, EINTR); + break; + + case MCDI_EVENT_CODE_CMDDONE: + efx_mcdi_ev_cpl(enp, + MCDI_EV_FIELD(eqp, CMDDONE_SEQ), + MCDI_EV_FIELD(eqp, CMDDONE_DATALEN), + MCDI_EV_FIELD(eqp, CMDDONE_ERRNO)); + break; + + case MCDI_EVENT_CODE_LINKCHANGE: { + efx_link_mode_t link_mode; + + siena_phy_link_ev(enp, eqp, &link_mode); + should_abort = eecp->eec_link_change(arg, link_mode); + break; + } + case MCDI_EVENT_CODE_SENSOREVT: { +#if EFSYS_OPT_MON_STATS + efx_mon_stat_t id; + efx_mon_stat_value_t value; + efx_rc_t rc; + + if ((rc = mcdi_mon_ev(enp, eqp, &id, &value)) == 0) + should_abort = eecp->eec_monitor(arg, id, value); + else if (rc == ENOTSUP) { + should_abort = eecp->eec_exception(arg, + EFX_EXCEPTION_UNKNOWN_SENSOREVT, + MCDI_EV_FIELD(eqp, DATA)); + } else + EFSYS_ASSERT(rc == ENODEV); /* Wrong port */ +#else + should_abort = B_FALSE; +#endif + break; + } + case MCDI_EVENT_CODE_SCHEDERR: + /* Informational only */ + break; + + case MCDI_EVENT_CODE_REBOOT: + efx_mcdi_ev_death(enp, EIO); + break; + + case MCDI_EVENT_CODE_MAC_STATS_DMA: +#if EFSYS_OPT_MAC_STATS + if (eecp->eec_mac_stats != NULL) { + eecp->eec_mac_stats(arg, + MCDI_EV_FIELD(eqp, MAC_STATS_DMA_GENERATION)); + } +#endif + break; + + case MCDI_EVENT_CODE_FWALERT: { + uint32_t reason = MCDI_EV_FIELD(eqp, FWALERT_REASON); + + if (reason == MCDI_EVENT_FWALERT_REASON_SRAM_ACCESS) + should_abort = eecp->eec_exception(arg, + EFX_EXCEPTION_FWALERT_SRAM, + MCDI_EV_FIELD(eqp, FWALERT_DATA)); + else + should_abort = eecp->eec_exception(arg, + EFX_EXCEPTION_UNKNOWN_FWALERT, + MCDI_EV_FIELD(eqp, DATA)); + break; + } + + default: + EFSYS_PROBE1(mc_pcol_error, int, code); + break; + } + +out: + return (should_abort); +} + +#endif /* EFSYS_OPT_MCDI */ + +static __checkReturn efx_rc_t +siena_ev_qprime( + __in efx_evq_t *eep, + __in unsigned int count) +{ + efx_nic_t *enp = eep->ee_enp; + uint32_t rptr; + efx_dword_t dword; + + rptr = count & eep->ee_mask; + + EFX_POPULATE_DWORD_1(dword, FRF_AZ_EVQ_RPTR, rptr); + + EFX_BAR_TBL_WRITED(enp, FR_AZ_EVQ_RPTR_REG, eep->ee_index, + &dword, B_FALSE); + + return (0); +} + +static void +siena_ev_qpost( + __in efx_evq_t *eep, + __in uint16_t data) +{ + efx_nic_t *enp = eep->ee_enp; + efx_qword_t ev; + efx_oword_t oword; + + EFX_POPULATE_QWORD_2(ev, FSF_AZ_EV_CODE, FSE_AZ_EV_CODE_DRV_GEN_EV, + FSF_AZ_EV_DATA_DW0, (uint32_t)data); + + EFX_POPULATE_OWORD_3(oword, FRF_AZ_DRV_EV_QID, eep->ee_index, + EFX_DWORD_0, EFX_QWORD_FIELD(ev, EFX_DWORD_0), + EFX_DWORD_1, EFX_QWORD_FIELD(ev, EFX_DWORD_1)); + + EFX_BAR_WRITEO(enp, FR_AZ_DRV_EV_REG, &oword); +} + +static __checkReturn efx_rc_t +siena_ev_qmoderate( + __in efx_evq_t *eep, + __in unsigned int us) +{ + efx_nic_t *enp = eep->ee_enp; + efx_nic_cfg_t *encp = &(enp->en_nic_cfg); + unsigned int locked; + efx_dword_t dword; + efx_rc_t rc; + + if (us > encp->enc_evq_timer_max_us) { + rc = EINVAL; + goto fail1; + } + + /* If the value is zero then disable the timer */ + if (us == 0) { + EFX_POPULATE_DWORD_2(dword, + FRF_CZ_TC_TIMER_MODE, FFE_CZ_TIMER_MODE_DIS, + FRF_CZ_TC_TIMER_VAL, 0); + } else { + unsigned int ticks; + + if ((rc = efx_ev_usecs_to_ticks(enp, us, &ticks)) != 0) + goto fail2; + + EFSYS_ASSERT(ticks > 0); + EFX_POPULATE_DWORD_2(dword, + FRF_CZ_TC_TIMER_MODE, FFE_CZ_TIMER_MODE_INT_HLDOFF, + FRF_CZ_TC_TIMER_VAL, ticks - 1); + } + + locked = (eep->ee_index == 0) ? 1 : 0; + + EFX_BAR_TBL_WRITED(enp, FR_BZ_TIMER_COMMAND_REGP0, + eep->ee_index, &dword, locked); + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +static __checkReturn efx_rc_t +siena_ev_qcreate( + __in efx_nic_t *enp, + __in unsigned int index, + __in efsys_mem_t *esmp, + __in size_t n, + __in uint32_t id, + __in uint32_t us, + __in uint32_t flags, + __in efx_evq_t *eep) +{ + efx_nic_cfg_t *encp = &(enp->en_nic_cfg); + uint32_t size; + efx_oword_t oword; + efx_rc_t rc; + boolean_t notify_mode; + + _NOTE(ARGUNUSED(esmp)) + + EFX_STATIC_ASSERT(ISP2(EFX_EVQ_MAXNEVS)); + EFX_STATIC_ASSERT(ISP2(EFX_EVQ_MINNEVS)); + + if (!ISP2(n) || (n < EFX_EVQ_MINNEVS) || (n > EFX_EVQ_MAXNEVS)) { + rc = EINVAL; + goto fail1; + } + if (index >= encp->enc_evq_limit) { + rc = EINVAL; + goto fail2; + } +#if EFSYS_OPT_RX_SCALE + if (enp->en_intr.ei_type == EFX_INTR_LINE && + index >= EFX_MAXRSS_LEGACY) { + rc = EINVAL; + goto fail3; + } +#endif + for (size = 0; (1 << size) <= (EFX_EVQ_MAXNEVS / EFX_EVQ_MINNEVS); + size++) + if ((1 << size) == (int)(n / EFX_EVQ_MINNEVS)) + break; + if (id + (1 << size) >= encp->enc_buftbl_limit) { + rc = EINVAL; + goto fail4; + } + + /* Set up the handler table */ + eep->ee_rx = siena_ev_rx; + eep->ee_tx = siena_ev_tx; + eep->ee_driver = siena_ev_driver; + eep->ee_global = siena_ev_global; + eep->ee_drv_gen = siena_ev_drv_gen; +#if EFSYS_OPT_MCDI + eep->ee_mcdi = siena_ev_mcdi; +#endif /* EFSYS_OPT_MCDI */ + + notify_mode = ((flags & EFX_EVQ_FLAGS_NOTIFY_MASK) != + EFX_EVQ_FLAGS_NOTIFY_INTERRUPT); + + /* Set up the new event queue */ + EFX_POPULATE_OWORD_3(oword, FRF_CZ_TIMER_Q_EN, 1, + FRF_CZ_HOST_NOTIFY_MODE, notify_mode, + FRF_CZ_TIMER_MODE, FFE_CZ_TIMER_MODE_DIS); + EFX_BAR_TBL_WRITEO(enp, FR_AZ_TIMER_TBL, index, &oword, B_TRUE); + + EFX_POPULATE_OWORD_3(oword, FRF_AZ_EVQ_EN, 1, FRF_AZ_EVQ_SIZE, size, + FRF_AZ_EVQ_BUF_BASE_ID, id); + + EFX_BAR_TBL_WRITEO(enp, FR_AZ_EVQ_PTR_TBL, index, &oword, B_TRUE); + + /* Set initial interrupt moderation */ + siena_ev_qmoderate(eep, us); + + return (0); + +fail4: + EFSYS_PROBE(fail4); +#if EFSYS_OPT_RX_SCALE +fail3: + EFSYS_PROBE(fail3); +#endif +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +#endif /* EFSYS_OPT_SIENA */ + +#if EFSYS_OPT_QSTATS +#if EFSYS_OPT_NAMES +/* START MKCONFIG GENERATED EfxEventQueueStatNamesBlock c0f3bc5083b40532 */ +static const char * const __efx_ev_qstat_name[] = { + "all", + "rx", + "rx_ok", + "rx_frm_trunc", + "rx_tobe_disc", + "rx_pause_frm_err", + "rx_buf_owner_id_err", + "rx_ipv4_hdr_chksum_err", + "rx_tcp_udp_chksum_err", + "rx_eth_crc_err", + "rx_ip_frag_err", + "rx_mcast_pkt", + "rx_mcast_hash_match", + "rx_tcp_ipv4", + "rx_tcp_ipv6", + "rx_udp_ipv4", + "rx_udp_ipv6", + "rx_other_ipv4", + "rx_other_ipv6", + "rx_non_ip", + "rx_batch", + "tx", + "tx_wq_ff_full", + "tx_pkt_err", + "tx_pkt_too_big", + "tx_unexpected", + "global", + "global_mnt", + "driver", + "driver_srm_upd_done", + "driver_tx_descq_fls_done", + "driver_rx_descq_fls_done", + "driver_rx_descq_fls_failed", + "driver_rx_dsc_error", + "driver_tx_dsc_error", + "drv_gen", + "mcdi_response", +}; +/* END MKCONFIG GENERATED EfxEventQueueStatNamesBlock */ + + const char * +efx_ev_qstat_name( + __in efx_nic_t *enp, + __in unsigned int id) +{ + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(id, <, EV_NQSTATS); + + return (__efx_ev_qstat_name[id]); +} +#endif /* EFSYS_OPT_NAMES */ +#endif /* EFSYS_OPT_QSTATS */ + +#if EFSYS_OPT_SIENA + +#if EFSYS_OPT_QSTATS +static void +siena_ev_qstats_update( + __in efx_evq_t *eep, + __inout_ecount(EV_NQSTATS) efsys_stat_t *stat) +{ + unsigned int id; + + for (id = 0; id < EV_NQSTATS; id++) { + efsys_stat_t *essp = &stat[id]; + + EFSYS_STAT_INCR(essp, eep->ee_stat[id]); + eep->ee_stat[id] = 0; + } +} +#endif /* EFSYS_OPT_QSTATS */ + +static void +siena_ev_qdestroy( + __in efx_evq_t *eep) +{ + efx_nic_t *enp = eep->ee_enp; + efx_oword_t oword; + + /* Purge event queue */ + EFX_ZERO_OWORD(oword); + + EFX_BAR_TBL_WRITEO(enp, FR_AZ_EVQ_PTR_TBL, + eep->ee_index, &oword, B_TRUE); + + EFX_ZERO_OWORD(oword); + EFX_BAR_TBL_WRITEO(enp, FR_AZ_TIMER_TBL, eep->ee_index, &oword, B_TRUE); +} + +static void +siena_ev_fini( + __in efx_nic_t *enp) +{ + _NOTE(ARGUNUSED(enp)) +} + +#endif /* EFSYS_OPT_SIENA */ diff --git a/dpdk/drivers/net/sfc/base/efx_filter.c b/dpdk/drivers/net/sfc/base/efx_filter.c new file mode 100644 index 00000000..5cab7d87 --- /dev/null +++ b/dpdk/drivers/net/sfc/base/efx_filter.c @@ -0,0 +1,1513 @@ +/* + * Copyright (c) 2007-2016 Solarflare Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are + * those of the authors and should not be interpreted as representing official + * policies, either expressed or implied, of the FreeBSD Project. + */ + +#include "efx.h" +#include "efx_impl.h" + + +#if EFSYS_OPT_FILTER + +#if EFSYS_OPT_SIENA + +static __checkReturn efx_rc_t +siena_filter_init( + __in efx_nic_t *enp); + +static void +siena_filter_fini( + __in efx_nic_t *enp); + +static __checkReturn efx_rc_t +siena_filter_restore( + __in efx_nic_t *enp); + +static __checkReturn efx_rc_t +siena_filter_add( + __in efx_nic_t *enp, + __inout efx_filter_spec_t *spec, + __in boolean_t may_replace); + +static __checkReturn efx_rc_t +siena_filter_delete( + __in efx_nic_t *enp, + __inout efx_filter_spec_t *spec); + +static __checkReturn efx_rc_t +siena_filter_supported_filters( + __in efx_nic_t *enp, + __out_ecount(buffer_length) uint32_t *buffer, + __in size_t buffer_length, + __out size_t *list_lengthp); + +#endif /* EFSYS_OPT_SIENA */ + +#if EFSYS_OPT_SIENA +static const efx_filter_ops_t __efx_filter_siena_ops = { + siena_filter_init, /* efo_init */ + siena_filter_fini, /* efo_fini */ + siena_filter_restore, /* efo_restore */ + siena_filter_add, /* efo_add */ + siena_filter_delete, /* efo_delete */ + siena_filter_supported_filters, /* efo_supported_filters */ + NULL, /* efo_reconfigure */ +}; +#endif /* EFSYS_OPT_SIENA */ + +#if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD +static const efx_filter_ops_t __efx_filter_ef10_ops = { + ef10_filter_init, /* efo_init */ + ef10_filter_fini, /* efo_fini */ + ef10_filter_restore, /* efo_restore */ + ef10_filter_add, /* efo_add */ + ef10_filter_delete, /* efo_delete */ + ef10_filter_supported_filters, /* efo_supported_filters */ + ef10_filter_reconfigure, /* efo_reconfigure */ +}; +#endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */ + + __checkReturn efx_rc_t +efx_filter_insert( + __in efx_nic_t *enp, + __inout efx_filter_spec_t *spec) +{ + const efx_filter_ops_t *efop = enp->en_efop; + + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER); + EFSYS_ASSERT3P(spec, !=, NULL); + EFSYS_ASSERT3U(spec->efs_flags, &, EFX_FILTER_FLAG_RX); + + return (efop->efo_add(enp, spec, B_FALSE)); +} + + __checkReturn efx_rc_t +efx_filter_remove( + __in efx_nic_t *enp, + __inout efx_filter_spec_t *spec) +{ + const efx_filter_ops_t *efop = enp->en_efop; + + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER); + EFSYS_ASSERT3P(spec, !=, NULL); + EFSYS_ASSERT3U(spec->efs_flags, &, EFX_FILTER_FLAG_RX); + + return (efop->efo_delete(enp, spec)); +} + + __checkReturn efx_rc_t +efx_filter_restore( + __in efx_nic_t *enp) +{ + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER); + + if ((rc = enp->en_efop->efo_restore(enp)) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_filter_init( + __in efx_nic_t *enp) +{ + const efx_filter_ops_t *efop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE); + EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_FILTER)); + + switch (enp->en_family) { +#if EFSYS_OPT_SIENA + case EFX_FAMILY_SIENA: + efop = &__efx_filter_siena_ops; + break; +#endif /* EFSYS_OPT_SIENA */ + +#if EFSYS_OPT_HUNTINGTON + case EFX_FAMILY_HUNTINGTON: + efop = &__efx_filter_ef10_ops; + break; +#endif /* EFSYS_OPT_HUNTINGTON */ + +#if EFSYS_OPT_MEDFORD + case EFX_FAMILY_MEDFORD: + efop = &__efx_filter_ef10_ops; + break; +#endif /* EFSYS_OPT_MEDFORD */ + + default: + EFSYS_ASSERT(0); + rc = ENOTSUP; + goto fail1; + } + + if ((rc = efop->efo_init(enp)) != 0) + goto fail2; + + enp->en_efop = efop; + enp->en_mod_flags |= EFX_MOD_FILTER; + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + enp->en_efop = NULL; + enp->en_mod_flags &= ~EFX_MOD_FILTER; + return (rc); +} + + void +efx_filter_fini( + __in efx_nic_t *enp) +{ + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER); + + enp->en_efop->efo_fini(enp); + + enp->en_efop = NULL; + enp->en_mod_flags &= ~EFX_MOD_FILTER; +} + +/* + * Query the possible combinations of match flags which can be filtered on. + * These are returned as a list, of which each 32 bit element is a bitmask + * formed of EFX_FILTER_MATCH flags. + * + * The combinations are ordered in priority from highest to lowest. + * + * If the provided buffer is too short to hold the list, the call with fail with + * ENOSPC and *list_lengthp will be set to the buffer length required. + */ + __checkReturn efx_rc_t +efx_filter_supported_filters( + __in efx_nic_t *enp, + __out_ecount(buffer_length) uint32_t *buffer, + __in size_t buffer_length, + __out size_t *list_lengthp) +{ + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER); + EFSYS_ASSERT(enp->en_efop->efo_supported_filters != NULL); + + if (buffer == NULL) { + rc = EINVAL; + goto fail1; + } + + rc = enp->en_efop->efo_supported_filters(enp, buffer, buffer_length, + list_lengthp); + if (rc != 0) + goto fail2; + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_filter_reconfigure( + __in efx_nic_t *enp, + __in_ecount(6) uint8_t const *mac_addr, + __in boolean_t all_unicst, + __in boolean_t mulcst, + __in boolean_t all_mulcst, + __in boolean_t brdcst, + __in_ecount(6*count) uint8_t const *addrs, + __in uint32_t count) +{ + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER); + + if (enp->en_efop->efo_reconfigure != NULL) { + if ((rc = enp->en_efop->efo_reconfigure(enp, mac_addr, + all_unicst, mulcst, + all_mulcst, brdcst, + addrs, count)) != 0) + goto fail1; + } + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + void +efx_filter_spec_init_rx( + __out efx_filter_spec_t *spec, + __in efx_filter_priority_t priority, + __in efx_filter_flags_t flags, + __in efx_rxq_t *erp) +{ + EFSYS_ASSERT3P(spec, !=, NULL); + EFSYS_ASSERT3P(erp, !=, NULL); + EFSYS_ASSERT((flags & ~(EFX_FILTER_FLAG_RX_RSS | + EFX_FILTER_FLAG_RX_SCATTER)) == 0); + + memset(spec, 0, sizeof (*spec)); + spec->efs_priority = priority; + spec->efs_flags = EFX_FILTER_FLAG_RX | flags; + spec->efs_rss_context = EFX_RSS_CONTEXT_DEFAULT; + spec->efs_dmaq_id = (uint16_t)erp->er_index; +} + + void +efx_filter_spec_init_tx( + __out efx_filter_spec_t *spec, + __in efx_txq_t *etp) +{ + EFSYS_ASSERT3P(spec, !=, NULL); + EFSYS_ASSERT3P(etp, !=, NULL); + + memset(spec, 0, sizeof (*spec)); + spec->efs_priority = EFX_FILTER_PRI_REQUIRED; + spec->efs_flags = EFX_FILTER_FLAG_TX; + spec->efs_dmaq_id = (uint16_t)etp->et_index; +} + + +/* + * Specify IPv4 host, transport protocol and port in a filter specification + */ +__checkReturn efx_rc_t +efx_filter_spec_set_ipv4_local( + __inout efx_filter_spec_t *spec, + __in uint8_t proto, + __in uint32_t host, + __in uint16_t port) +{ + EFSYS_ASSERT3P(spec, !=, NULL); + + spec->efs_match_flags |= + EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO | + EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT; + spec->efs_ether_type = EFX_ETHER_TYPE_IPV4; + spec->efs_ip_proto = proto; + spec->efs_loc_host.eo_u32[0] = host; + spec->efs_loc_port = port; + return (0); +} + +/* + * Specify IPv4 hosts, transport protocol and ports in a filter specification + */ +__checkReturn efx_rc_t +efx_filter_spec_set_ipv4_full( + __inout efx_filter_spec_t *spec, + __in uint8_t proto, + __in uint32_t lhost, + __in uint16_t lport, + __in uint32_t rhost, + __in uint16_t rport) +{ + EFSYS_ASSERT3P(spec, !=, NULL); + + spec->efs_match_flags |= + EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO | + EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT | + EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_REM_PORT; + spec->efs_ether_type = EFX_ETHER_TYPE_IPV4; + spec->efs_ip_proto = proto; + spec->efs_loc_host.eo_u32[0] = lhost; + spec->efs_loc_port = lport; + spec->efs_rem_host.eo_u32[0] = rhost; + spec->efs_rem_port = rport; + return (0); +} + +/* + * Specify local Ethernet address and/or VID in filter specification + */ +__checkReturn efx_rc_t +efx_filter_spec_set_eth_local( + __inout efx_filter_spec_t *spec, + __in uint16_t vid, + __in const uint8_t *addr) +{ + EFSYS_ASSERT3P(spec, !=, NULL); + EFSYS_ASSERT3P(addr, !=, NULL); + + if (vid == EFX_FILTER_SPEC_VID_UNSPEC && addr == NULL) + return (EINVAL); + + if (vid != EFX_FILTER_SPEC_VID_UNSPEC) { + spec->efs_match_flags |= EFX_FILTER_MATCH_OUTER_VID; + spec->efs_outer_vid = vid; + } + if (addr != NULL) { + spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_MAC; + memcpy(spec->efs_loc_mac, addr, EFX_MAC_ADDR_LEN); + } + return (0); +} + + void +efx_filter_spec_set_ether_type( + __inout efx_filter_spec_t *spec, + __in uint16_t ether_type) +{ + EFSYS_ASSERT3P(spec, !=, NULL); + + spec->efs_ether_type = ether_type; + spec->efs_match_flags |= EFX_FILTER_MATCH_ETHER_TYPE; +} + +/* + * Specify matching otherwise-unmatched unicast in a filter specification + */ +__checkReturn efx_rc_t +efx_filter_spec_set_uc_def( + __inout efx_filter_spec_t *spec) +{ + EFSYS_ASSERT3P(spec, !=, NULL); + + spec->efs_match_flags |= EFX_FILTER_MATCH_UNKNOWN_UCAST_DST; + return (0); +} + +/* + * Specify matching otherwise-unmatched multicast in a filter specification + */ +__checkReturn efx_rc_t +efx_filter_spec_set_mc_def( + __inout efx_filter_spec_t *spec) +{ + EFSYS_ASSERT3P(spec, !=, NULL); + + spec->efs_match_flags |= EFX_FILTER_MATCH_UNKNOWN_MCAST_DST; + return (0); +} + + +__checkReturn efx_rc_t +efx_filter_spec_set_encap_type( + __inout efx_filter_spec_t *spec, + __in efx_tunnel_protocol_t encap_type, + __in efx_filter_inner_frame_match_t inner_frame_match) +{ + uint32_t match_flags = 0; + uint8_t ip_proto; + efx_rc_t rc; + + EFSYS_ASSERT3P(spec, !=, NULL); + + switch (encap_type) { + case EFX_TUNNEL_PROTOCOL_VXLAN: + case EFX_TUNNEL_PROTOCOL_GENEVE: + ip_proto = EFX_IPPROTO_UDP; + break; + case EFX_TUNNEL_PROTOCOL_NVGRE: + ip_proto = EFX_IPPROTO_GRE; + break; + default: + EFSYS_ASSERT(0); + rc = EINVAL; + goto fail1; + } + + switch (inner_frame_match) { + case EFX_FILTER_INNER_FRAME_MATCH_UNKNOWN_MCAST_DST: + match_flags |= EFX_FILTER_MATCH_IFRM_UNKNOWN_MCAST_DST; + break; + case EFX_FILTER_INNER_FRAME_MATCH_UNKNOWN_UCAST_DST: + match_flags |= EFX_FILTER_MATCH_IFRM_UNKNOWN_UCAST_DST; + break; + case EFX_FILTER_INNER_FRAME_MATCH_OTHER: + /* This is for when specific inner frames are to be matched. */ + break; + default: + EFSYS_ASSERT(0); + rc = EINVAL; + goto fail2; + } + + spec->efs_encap_type = encap_type; + spec->efs_ip_proto = ip_proto; + spec->efs_match_flags |= (match_flags | EFX_FILTER_MATCH_IP_PROTO); + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +#if EFSYS_OPT_RX_SCALE + __checkReturn efx_rc_t +efx_filter_spec_set_rss_context( + __inout efx_filter_spec_t *spec, + __in uint32_t rss_context) +{ + efx_rc_t rc; + + EFSYS_ASSERT3P(spec, !=, NULL); + + /* The filter must have been created with EFX_FILTER_FLAG_RX_RSS. */ + if ((spec->efs_flags & EFX_FILTER_FLAG_RX_RSS) == 0) { + rc = EINVAL; + goto fail1; + } + + spec->efs_rss_context = rss_context; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} +#endif + +#if EFSYS_OPT_SIENA + +/* + * "Fudge factors" - difference between programmed value and actual depth. + * Due to pipelined implementation we need to program H/W with a value that + * is larger than the hop limit we want. + */ +#define FILTER_CTL_SRCH_FUDGE_WILD 3 +#define FILTER_CTL_SRCH_FUDGE_FULL 1 + +/* + * Hard maximum hop limit. Hardware will time-out beyond 200-something. + * We also need to avoid infinite loops in efx_filter_search() when the + * table is full. + */ +#define FILTER_CTL_SRCH_MAX 200 + +static __checkReturn efx_rc_t +siena_filter_spec_from_gen_spec( + __out siena_filter_spec_t *sf_spec, + __in efx_filter_spec_t *gen_spec) +{ + efx_rc_t rc; + boolean_t is_full = B_FALSE; + + if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) + EFSYS_ASSERT3U(gen_spec->efs_flags, ==, EFX_FILTER_FLAG_TX); + else + EFSYS_ASSERT3U(gen_spec->efs_flags, &, EFX_FILTER_FLAG_RX); + + /* Siena only has one RSS context */ + if ((gen_spec->efs_flags & EFX_FILTER_FLAG_RX_RSS) && + gen_spec->efs_rss_context != EFX_RSS_CONTEXT_DEFAULT) { + rc = EINVAL; + goto fail1; + } + + sf_spec->sfs_flags = gen_spec->efs_flags; + sf_spec->sfs_dmaq_id = gen_spec->efs_dmaq_id; + + switch (gen_spec->efs_match_flags) { + case EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO | + EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT | + EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_REM_PORT: + is_full = B_TRUE; + /* Fall through */ + case EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO | + EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT: { + uint32_t rhost, host1, host2; + uint16_t rport, port1, port2; + + if (gen_spec->efs_ether_type != EFX_ETHER_TYPE_IPV4) { + rc = ENOTSUP; + goto fail2; + } + if (gen_spec->efs_loc_port == 0 || + (is_full && gen_spec->efs_rem_port == 0)) { + rc = EINVAL; + goto fail3; + } + switch (gen_spec->efs_ip_proto) { + case EFX_IPPROTO_TCP: + if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) { + sf_spec->sfs_type = (is_full ? + EFX_SIENA_FILTER_TX_TCP_FULL : + EFX_SIENA_FILTER_TX_TCP_WILD); + } else { + sf_spec->sfs_type = (is_full ? + EFX_SIENA_FILTER_RX_TCP_FULL : + EFX_SIENA_FILTER_RX_TCP_WILD); + } + break; + case EFX_IPPROTO_UDP: + if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) { + sf_spec->sfs_type = (is_full ? + EFX_SIENA_FILTER_TX_UDP_FULL : + EFX_SIENA_FILTER_TX_UDP_WILD); + } else { + sf_spec->sfs_type = (is_full ? + EFX_SIENA_FILTER_RX_UDP_FULL : + EFX_SIENA_FILTER_RX_UDP_WILD); + } + break; + default: + rc = ENOTSUP; + goto fail4; + } + /* + * The filter is constructed in terms of source and destination, + * with the odd wrinkle that the ports are swapped in a UDP + * wildcard filter. We need to convert from local and remote + * addresses (zero for a wildcard). + */ + rhost = is_full ? gen_spec->efs_rem_host.eo_u32[0] : 0; + rport = is_full ? gen_spec->efs_rem_port : 0; + if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) { + host1 = gen_spec->efs_loc_host.eo_u32[0]; + host2 = rhost; + } else { + host1 = rhost; + host2 = gen_spec->efs_loc_host.eo_u32[0]; + } + if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) { + if (sf_spec->sfs_type == + EFX_SIENA_FILTER_TX_UDP_WILD) { + port1 = rport; + port2 = gen_spec->efs_loc_port; + } else { + port1 = gen_spec->efs_loc_port; + port2 = rport; + } + } else { + if (sf_spec->sfs_type == + EFX_SIENA_FILTER_RX_UDP_WILD) { + port1 = gen_spec->efs_loc_port; + port2 = rport; + } else { + port1 = rport; + port2 = gen_spec->efs_loc_port; + } + } + sf_spec->sfs_dword[0] = (host1 << 16) | port1; + sf_spec->sfs_dword[1] = (port2 << 16) | (host1 >> 16); + sf_spec->sfs_dword[2] = host2; + break; + } + + case EFX_FILTER_MATCH_LOC_MAC | EFX_FILTER_MATCH_OUTER_VID: + is_full = B_TRUE; + /* Fall through */ + case EFX_FILTER_MATCH_LOC_MAC: + if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) { + sf_spec->sfs_type = (is_full ? + EFX_SIENA_FILTER_TX_MAC_FULL : + EFX_SIENA_FILTER_TX_MAC_WILD); + } else { + sf_spec->sfs_type = (is_full ? + EFX_SIENA_FILTER_RX_MAC_FULL : + EFX_SIENA_FILTER_RX_MAC_WILD); + } + sf_spec->sfs_dword[0] = is_full ? gen_spec->efs_outer_vid : 0; + sf_spec->sfs_dword[1] = + gen_spec->efs_loc_mac[2] << 24 | + gen_spec->efs_loc_mac[3] << 16 | + gen_spec->efs_loc_mac[4] << 8 | + gen_spec->efs_loc_mac[5]; + sf_spec->sfs_dword[2] = + gen_spec->efs_loc_mac[0] << 8 | + gen_spec->efs_loc_mac[1]; + break; + + default: + EFSYS_ASSERT(B_FALSE); + rc = ENOTSUP; + goto fail5; + } + + return (0); + +fail5: + EFSYS_PROBE(fail5); +fail4: + EFSYS_PROBE(fail4); +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +/* + * The filter hash function is LFSR polynomial x^16 + x^3 + 1 of a 32-bit + * key derived from the n-tuple. + */ +static uint16_t +siena_filter_tbl_hash( + __in uint32_t key) +{ + uint16_t tmp; + + /* First 16 rounds */ + tmp = 0x1fff ^ (uint16_t)(key >> 16); + tmp = tmp ^ tmp >> 3 ^ tmp >> 6; + tmp = tmp ^ tmp >> 9; + + /* Last 16 rounds */ + tmp = tmp ^ tmp << 13 ^ (uint16_t)(key & 0xffff); + tmp = tmp ^ tmp >> 3 ^ tmp >> 6; + tmp = tmp ^ tmp >> 9; + + return (tmp); +} + +/* + * To allow for hash collisions, filter search continues at these + * increments from the first possible entry selected by the hash. + */ +static uint16_t +siena_filter_tbl_increment( + __in uint32_t key) +{ + return ((uint16_t)(key * 2 - 1)); +} + +static __checkReturn boolean_t +siena_filter_test_used( + __in siena_filter_tbl_t *sftp, + __in unsigned int index) +{ + EFSYS_ASSERT3P(sftp->sft_bitmap, !=, NULL); + return ((sftp->sft_bitmap[index / 32] & (1 << (index % 32))) != 0); +} + +static void +siena_filter_set_used( + __in siena_filter_tbl_t *sftp, + __in unsigned int index) +{ + EFSYS_ASSERT3P(sftp->sft_bitmap, !=, NULL); + sftp->sft_bitmap[index / 32] |= (1 << (index % 32)); + ++sftp->sft_used; +} + +static void +siena_filter_clear_used( + __in siena_filter_tbl_t *sftp, + __in unsigned int index) +{ + EFSYS_ASSERT3P(sftp->sft_bitmap, !=, NULL); + sftp->sft_bitmap[index / 32] &= ~(1 << (index % 32)); + + --sftp->sft_used; + EFSYS_ASSERT3U(sftp->sft_used, >=, 0); +} + + +static siena_filter_tbl_id_t +siena_filter_tbl_id( + __in siena_filter_type_t type) +{ + siena_filter_tbl_id_t tbl_id; + + switch (type) { + case EFX_SIENA_FILTER_RX_TCP_FULL: + case EFX_SIENA_FILTER_RX_TCP_WILD: + case EFX_SIENA_FILTER_RX_UDP_FULL: + case EFX_SIENA_FILTER_RX_UDP_WILD: + tbl_id = EFX_SIENA_FILTER_TBL_RX_IP; + break; + + case EFX_SIENA_FILTER_RX_MAC_FULL: + case EFX_SIENA_FILTER_RX_MAC_WILD: + tbl_id = EFX_SIENA_FILTER_TBL_RX_MAC; + break; + + case EFX_SIENA_FILTER_TX_TCP_FULL: + case EFX_SIENA_FILTER_TX_TCP_WILD: + case EFX_SIENA_FILTER_TX_UDP_FULL: + case EFX_SIENA_FILTER_TX_UDP_WILD: + tbl_id = EFX_SIENA_FILTER_TBL_TX_IP; + break; + + case EFX_SIENA_FILTER_TX_MAC_FULL: + case EFX_SIENA_FILTER_TX_MAC_WILD: + tbl_id = EFX_SIENA_FILTER_TBL_TX_MAC; + break; + + default: + EFSYS_ASSERT(B_FALSE); + tbl_id = EFX_SIENA_FILTER_NTBLS; + break; + } + return (tbl_id); +} + +static void +siena_filter_reset_search_depth( + __inout siena_filter_t *sfp, + __in siena_filter_tbl_id_t tbl_id) +{ + switch (tbl_id) { + case EFX_SIENA_FILTER_TBL_RX_IP: + sfp->sf_depth[EFX_SIENA_FILTER_RX_TCP_FULL] = 0; + sfp->sf_depth[EFX_SIENA_FILTER_RX_TCP_WILD] = 0; + sfp->sf_depth[EFX_SIENA_FILTER_RX_UDP_FULL] = 0; + sfp->sf_depth[EFX_SIENA_FILTER_RX_UDP_WILD] = 0; + break; + + case EFX_SIENA_FILTER_TBL_RX_MAC: + sfp->sf_depth[EFX_SIENA_FILTER_RX_MAC_FULL] = 0; + sfp->sf_depth[EFX_SIENA_FILTER_RX_MAC_WILD] = 0; + break; + + case EFX_SIENA_FILTER_TBL_TX_IP: + sfp->sf_depth[EFX_SIENA_FILTER_TX_TCP_FULL] = 0; + sfp->sf_depth[EFX_SIENA_FILTER_TX_TCP_WILD] = 0; + sfp->sf_depth[EFX_SIENA_FILTER_TX_UDP_FULL] = 0; + sfp->sf_depth[EFX_SIENA_FILTER_TX_UDP_WILD] = 0; + break; + + case EFX_SIENA_FILTER_TBL_TX_MAC: + sfp->sf_depth[EFX_SIENA_FILTER_TX_MAC_FULL] = 0; + sfp->sf_depth[EFX_SIENA_FILTER_TX_MAC_WILD] = 0; + break; + + default: + EFSYS_ASSERT(B_FALSE); + break; + } +} + +static void +siena_filter_push_rx_limits( + __in efx_nic_t *enp) +{ + siena_filter_t *sfp = enp->en_filter.ef_siena_filter; + efx_oword_t oword; + + EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword); + + EFX_SET_OWORD_FIELD(oword, FRF_AZ_TCP_FULL_SRCH_LIMIT, + sfp->sf_depth[EFX_SIENA_FILTER_RX_TCP_FULL] + + FILTER_CTL_SRCH_FUDGE_FULL); + EFX_SET_OWORD_FIELD(oword, FRF_AZ_TCP_WILD_SRCH_LIMIT, + sfp->sf_depth[EFX_SIENA_FILTER_RX_TCP_WILD] + + FILTER_CTL_SRCH_FUDGE_WILD); + EFX_SET_OWORD_FIELD(oword, FRF_AZ_UDP_FULL_SRCH_LIMIT, + sfp->sf_depth[EFX_SIENA_FILTER_RX_UDP_FULL] + + FILTER_CTL_SRCH_FUDGE_FULL); + EFX_SET_OWORD_FIELD(oword, FRF_AZ_UDP_WILD_SRCH_LIMIT, + sfp->sf_depth[EFX_SIENA_FILTER_RX_UDP_WILD] + + FILTER_CTL_SRCH_FUDGE_WILD); + + if (sfp->sf_tbl[EFX_SIENA_FILTER_TBL_RX_MAC].sft_size) { + EFX_SET_OWORD_FIELD(oword, + FRF_CZ_ETHERNET_FULL_SEARCH_LIMIT, + sfp->sf_depth[EFX_SIENA_FILTER_RX_MAC_FULL] + + FILTER_CTL_SRCH_FUDGE_FULL); + EFX_SET_OWORD_FIELD(oword, + FRF_CZ_ETHERNET_WILDCARD_SEARCH_LIMIT, + sfp->sf_depth[EFX_SIENA_FILTER_RX_MAC_WILD] + + FILTER_CTL_SRCH_FUDGE_WILD); + } + + EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword); +} + +static void +siena_filter_push_tx_limits( + __in efx_nic_t *enp) +{ + siena_filter_t *sfp = enp->en_filter.ef_siena_filter; + efx_oword_t oword; + + EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword); + + if (sfp->sf_tbl[EFX_SIENA_FILTER_TBL_TX_IP].sft_size != 0) { + EFX_SET_OWORD_FIELD(oword, + FRF_CZ_TX_TCPIP_FILTER_FULL_SEARCH_RANGE, + sfp->sf_depth[EFX_SIENA_FILTER_TX_TCP_FULL] + + FILTER_CTL_SRCH_FUDGE_FULL); + EFX_SET_OWORD_FIELD(oword, + FRF_CZ_TX_TCPIP_FILTER_WILD_SEARCH_RANGE, + sfp->sf_depth[EFX_SIENA_FILTER_TX_TCP_WILD] + + FILTER_CTL_SRCH_FUDGE_WILD); + EFX_SET_OWORD_FIELD(oword, + FRF_CZ_TX_UDPIP_FILTER_FULL_SEARCH_RANGE, + sfp->sf_depth[EFX_SIENA_FILTER_TX_UDP_FULL] + + FILTER_CTL_SRCH_FUDGE_FULL); + EFX_SET_OWORD_FIELD(oword, + FRF_CZ_TX_UDPIP_FILTER_WILD_SEARCH_RANGE, + sfp->sf_depth[EFX_SIENA_FILTER_TX_UDP_WILD] + + FILTER_CTL_SRCH_FUDGE_WILD); + } + + if (sfp->sf_tbl[EFX_SIENA_FILTER_TBL_TX_MAC].sft_size != 0) { + EFX_SET_OWORD_FIELD( + oword, FRF_CZ_TX_ETH_FILTER_FULL_SEARCH_RANGE, + sfp->sf_depth[EFX_SIENA_FILTER_TX_MAC_FULL] + + FILTER_CTL_SRCH_FUDGE_FULL); + EFX_SET_OWORD_FIELD( + oword, FRF_CZ_TX_ETH_FILTER_WILD_SEARCH_RANGE, + sfp->sf_depth[EFX_SIENA_FILTER_TX_MAC_WILD] + + FILTER_CTL_SRCH_FUDGE_WILD); + } + + EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword); +} + +/* Build a filter entry and return its n-tuple key. */ +static __checkReturn uint32_t +siena_filter_build( + __out efx_oword_t *filter, + __in siena_filter_spec_t *spec) +{ + uint32_t dword3; + uint32_t key; + uint8_t type = spec->sfs_type; + uint32_t flags = spec->sfs_flags; + + switch (siena_filter_tbl_id(type)) { + case EFX_SIENA_FILTER_TBL_RX_IP: { + boolean_t is_udp = (type == EFX_SIENA_FILTER_RX_UDP_FULL || + type == EFX_SIENA_FILTER_RX_UDP_WILD); + EFX_POPULATE_OWORD_7(*filter, + FRF_BZ_RSS_EN, + (flags & EFX_FILTER_FLAG_RX_RSS) ? 1 : 0, + FRF_BZ_SCATTER_EN, + (flags & EFX_FILTER_FLAG_RX_SCATTER) ? 1 : 0, + FRF_AZ_TCP_UDP, is_udp, + FRF_AZ_RXQ_ID, spec->sfs_dmaq_id, + EFX_DWORD_2, spec->sfs_dword[2], + EFX_DWORD_1, spec->sfs_dword[1], + EFX_DWORD_0, spec->sfs_dword[0]); + dword3 = is_udp; + break; + } + + case EFX_SIENA_FILTER_TBL_RX_MAC: { + boolean_t is_wild = (type == EFX_SIENA_FILTER_RX_MAC_WILD); + EFX_POPULATE_OWORD_7(*filter, + FRF_CZ_RMFT_RSS_EN, + (flags & EFX_FILTER_FLAG_RX_RSS) ? 1 : 0, + FRF_CZ_RMFT_SCATTER_EN, + (flags & EFX_FILTER_FLAG_RX_SCATTER) ? 1 : 0, + FRF_CZ_RMFT_RXQ_ID, spec->sfs_dmaq_id, + FRF_CZ_RMFT_WILDCARD_MATCH, is_wild, + FRF_CZ_RMFT_DEST_MAC_DW1, spec->sfs_dword[2], + FRF_CZ_RMFT_DEST_MAC_DW0, spec->sfs_dword[1], + FRF_CZ_RMFT_VLAN_ID, spec->sfs_dword[0]); + dword3 = is_wild; + break; + } + + case EFX_SIENA_FILTER_TBL_TX_IP: { + boolean_t is_udp = (type == EFX_SIENA_FILTER_TX_UDP_FULL || + type == EFX_SIENA_FILTER_TX_UDP_WILD); + EFX_POPULATE_OWORD_5(*filter, + FRF_CZ_TIFT_TCP_UDP, is_udp, + FRF_CZ_TIFT_TXQ_ID, spec->sfs_dmaq_id, + EFX_DWORD_2, spec->sfs_dword[2], + EFX_DWORD_1, spec->sfs_dword[1], + EFX_DWORD_0, spec->sfs_dword[0]); + dword3 = is_udp | spec->sfs_dmaq_id << 1; + break; + } + + case EFX_SIENA_FILTER_TBL_TX_MAC: { + boolean_t is_wild = (type == EFX_SIENA_FILTER_TX_MAC_WILD); + EFX_POPULATE_OWORD_5(*filter, + FRF_CZ_TMFT_TXQ_ID, spec->sfs_dmaq_id, + FRF_CZ_TMFT_WILDCARD_MATCH, is_wild, + FRF_CZ_TMFT_SRC_MAC_DW1, spec->sfs_dword[2], + FRF_CZ_TMFT_SRC_MAC_DW0, spec->sfs_dword[1], + FRF_CZ_TMFT_VLAN_ID, spec->sfs_dword[0]); + dword3 = is_wild | spec->sfs_dmaq_id << 1; + break; + } + + default: + EFSYS_ASSERT(B_FALSE); + return (0); + } + + key = + spec->sfs_dword[0] ^ + spec->sfs_dword[1] ^ + spec->sfs_dword[2] ^ + dword3; + + return (key); +} + +static __checkReturn efx_rc_t +siena_filter_push_entry( + __inout efx_nic_t *enp, + __in siena_filter_type_t type, + __in int index, + __in efx_oword_t *eop) +{ + efx_rc_t rc; + + switch (type) { + case EFX_SIENA_FILTER_RX_TCP_FULL: + case EFX_SIENA_FILTER_RX_TCP_WILD: + case EFX_SIENA_FILTER_RX_UDP_FULL: + case EFX_SIENA_FILTER_RX_UDP_WILD: + EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_FILTER_TBL0, index, + eop, B_TRUE); + break; + + case EFX_SIENA_FILTER_RX_MAC_FULL: + case EFX_SIENA_FILTER_RX_MAC_WILD: + EFX_BAR_TBL_WRITEO(enp, FR_CZ_RX_MAC_FILTER_TBL0, index, + eop, B_TRUE); + break; + + case EFX_SIENA_FILTER_TX_TCP_FULL: + case EFX_SIENA_FILTER_TX_TCP_WILD: + case EFX_SIENA_FILTER_TX_UDP_FULL: + case EFX_SIENA_FILTER_TX_UDP_WILD: + EFX_BAR_TBL_WRITEO(enp, FR_CZ_TX_FILTER_TBL0, index, + eop, B_TRUE); + break; + + case EFX_SIENA_FILTER_TX_MAC_FULL: + case EFX_SIENA_FILTER_TX_MAC_WILD: + EFX_BAR_TBL_WRITEO(enp, FR_CZ_TX_MAC_FILTER_TBL0, index, + eop, B_TRUE); + break; + + default: + EFSYS_ASSERT(B_FALSE); + rc = ENOTSUP; + goto fail1; + } + return (0); + +fail1: + return (rc); +} + + +static __checkReturn boolean_t +siena_filter_equal( + __in const siena_filter_spec_t *left, + __in const siena_filter_spec_t *right) +{ + siena_filter_tbl_id_t tbl_id; + + tbl_id = siena_filter_tbl_id(left->sfs_type); + + + if (left->sfs_type != right->sfs_type) + return (B_FALSE); + + if (memcmp(left->sfs_dword, right->sfs_dword, + sizeof (left->sfs_dword))) + return (B_FALSE); + + if ((tbl_id == EFX_SIENA_FILTER_TBL_TX_IP || + tbl_id == EFX_SIENA_FILTER_TBL_TX_MAC) && + left->sfs_dmaq_id != right->sfs_dmaq_id) + return (B_FALSE); + + return (B_TRUE); +} + +static __checkReturn efx_rc_t +siena_filter_search( + __in siena_filter_tbl_t *sftp, + __in siena_filter_spec_t *spec, + __in uint32_t key, + __in boolean_t for_insert, + __out int *filter_index, + __out unsigned int *depth_required) +{ + unsigned int hash, incr, filter_idx, depth; + + hash = siena_filter_tbl_hash(key); + incr = siena_filter_tbl_increment(key); + + filter_idx = hash & (sftp->sft_size - 1); + depth = 1; + + for (;;) { + /* + * Return success if entry is used and matches this spec + * or entry is unused and we are trying to insert. + */ + if (siena_filter_test_used(sftp, filter_idx) ? + siena_filter_equal(spec, + &sftp->sft_spec[filter_idx]) : + for_insert) { + *filter_index = filter_idx; + *depth_required = depth; + return (0); + } + + /* Return failure if we reached the maximum search depth */ + if (depth == FILTER_CTL_SRCH_MAX) + return (for_insert ? EBUSY : ENOENT); + + filter_idx = (filter_idx + incr) & (sftp->sft_size - 1); + ++depth; + } +} + +static void +siena_filter_clear_entry( + __in efx_nic_t *enp, + __in siena_filter_tbl_t *sftp, + __in int index) +{ + efx_oword_t filter; + + if (siena_filter_test_used(sftp, index)) { + siena_filter_clear_used(sftp, index); + + EFX_ZERO_OWORD(filter); + siena_filter_push_entry(enp, + sftp->sft_spec[index].sfs_type, + index, &filter); + + memset(&sftp->sft_spec[index], + 0, sizeof (sftp->sft_spec[0])); + } +} + + void +siena_filter_tbl_clear( + __in efx_nic_t *enp, + __in siena_filter_tbl_id_t tbl_id) +{ + siena_filter_t *sfp = enp->en_filter.ef_siena_filter; + siena_filter_tbl_t *sftp = &sfp->sf_tbl[tbl_id]; + int index; + efsys_lock_state_t state; + + EFSYS_LOCK(enp->en_eslp, state); + + for (index = 0; index < sftp->sft_size; ++index) { + siena_filter_clear_entry(enp, sftp, index); + } + + if (sftp->sft_used == 0) + siena_filter_reset_search_depth(sfp, tbl_id); + + EFSYS_UNLOCK(enp->en_eslp, state); +} + +static __checkReturn efx_rc_t +siena_filter_init( + __in efx_nic_t *enp) +{ + siena_filter_t *sfp; + siena_filter_tbl_t *sftp; + int tbl_id; + efx_rc_t rc; + + EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (siena_filter_t), sfp); + + if (!sfp) { + rc = ENOMEM; + goto fail1; + } + + enp->en_filter.ef_siena_filter = sfp; + + switch (enp->en_family) { + case EFX_FAMILY_SIENA: + sftp = &sfp->sf_tbl[EFX_SIENA_FILTER_TBL_RX_IP]; + sftp->sft_size = FR_AZ_RX_FILTER_TBL0_ROWS; + + sftp = &sfp->sf_tbl[EFX_SIENA_FILTER_TBL_RX_MAC]; + sftp->sft_size = FR_CZ_RX_MAC_FILTER_TBL0_ROWS; + + sftp = &sfp->sf_tbl[EFX_SIENA_FILTER_TBL_TX_IP]; + sftp->sft_size = FR_CZ_TX_FILTER_TBL0_ROWS; + + sftp = &sfp->sf_tbl[EFX_SIENA_FILTER_TBL_TX_MAC]; + sftp->sft_size = FR_CZ_TX_MAC_FILTER_TBL0_ROWS; + break; + + default: + rc = ENOTSUP; + goto fail2; + } + + for (tbl_id = 0; tbl_id < EFX_SIENA_FILTER_NTBLS; tbl_id++) { + unsigned int bitmap_size; + + sftp = &sfp->sf_tbl[tbl_id]; + if (sftp->sft_size == 0) + continue; + + EFX_STATIC_ASSERT(sizeof (sftp->sft_bitmap[0]) == + sizeof (uint32_t)); + bitmap_size = + (sftp->sft_size + (sizeof (uint32_t) * 8) - 1) / 8; + + EFSYS_KMEM_ALLOC(enp->en_esip, bitmap_size, sftp->sft_bitmap); + if (!sftp->sft_bitmap) { + rc = ENOMEM; + goto fail3; + } + + EFSYS_KMEM_ALLOC(enp->en_esip, + sftp->sft_size * sizeof (*sftp->sft_spec), + sftp->sft_spec); + if (!sftp->sft_spec) { + rc = ENOMEM; + goto fail4; + } + memset(sftp->sft_spec, 0, + sftp->sft_size * sizeof (*sftp->sft_spec)); + } + + return (0); + +fail4: + EFSYS_PROBE(fail4); + +fail3: + EFSYS_PROBE(fail3); + +fail2: + EFSYS_PROBE(fail2); + siena_filter_fini(enp); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + return (rc); +} + +static void +siena_filter_fini( + __in efx_nic_t *enp) +{ + siena_filter_t *sfp = enp->en_filter.ef_siena_filter; + siena_filter_tbl_id_t tbl_id; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE); + + if (sfp == NULL) + return; + + for (tbl_id = 0; tbl_id < EFX_SIENA_FILTER_NTBLS; tbl_id++) { + siena_filter_tbl_t *sftp = &sfp->sf_tbl[tbl_id]; + unsigned int bitmap_size; + + EFX_STATIC_ASSERT(sizeof (sftp->sft_bitmap[0]) == + sizeof (uint32_t)); + bitmap_size = + (sftp->sft_size + (sizeof (uint32_t) * 8) - 1) / 8; + + if (sftp->sft_bitmap != NULL) { + EFSYS_KMEM_FREE(enp->en_esip, bitmap_size, + sftp->sft_bitmap); + sftp->sft_bitmap = NULL; + } + + if (sftp->sft_spec != NULL) { + EFSYS_KMEM_FREE(enp->en_esip, sftp->sft_size * + sizeof (*sftp->sft_spec), sftp->sft_spec); + sftp->sft_spec = NULL; + } + } + + EFSYS_KMEM_FREE(enp->en_esip, sizeof (siena_filter_t), + enp->en_filter.ef_siena_filter); +} + +/* Restore filter state after a reset */ +static __checkReturn efx_rc_t +siena_filter_restore( + __in efx_nic_t *enp) +{ + siena_filter_t *sfp = enp->en_filter.ef_siena_filter; + siena_filter_tbl_id_t tbl_id; + siena_filter_tbl_t *sftp; + siena_filter_spec_t *spec; + efx_oword_t filter; + int filter_idx; + efsys_lock_state_t state; + uint32_t key; + efx_rc_t rc; + + EFSYS_LOCK(enp->en_eslp, state); + + for (tbl_id = 0; tbl_id < EFX_SIENA_FILTER_NTBLS; tbl_id++) { + sftp = &sfp->sf_tbl[tbl_id]; + for (filter_idx = 0; + filter_idx < sftp->sft_size; + filter_idx++) { + if (!siena_filter_test_used(sftp, filter_idx)) + continue; + + spec = &sftp->sft_spec[filter_idx]; + if ((key = siena_filter_build(&filter, spec)) == 0) { + rc = EINVAL; + goto fail1; + } + if ((rc = siena_filter_push_entry(enp, + spec->sfs_type, filter_idx, &filter)) != 0) + goto fail2; + } + } + + siena_filter_push_rx_limits(enp); + siena_filter_push_tx_limits(enp); + + EFSYS_UNLOCK(enp->en_eslp, state); + + return (0); + +fail2: + EFSYS_PROBE(fail2); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + EFSYS_UNLOCK(enp->en_eslp, state); + + return (rc); +} + +static __checkReturn efx_rc_t +siena_filter_add( + __in efx_nic_t *enp, + __inout efx_filter_spec_t *spec, + __in boolean_t may_replace) +{ + efx_rc_t rc; + siena_filter_spec_t sf_spec; + siena_filter_t *sfp = enp->en_filter.ef_siena_filter; + siena_filter_tbl_id_t tbl_id; + siena_filter_tbl_t *sftp; + siena_filter_spec_t *saved_sf_spec; + efx_oword_t filter; + int filter_idx; + unsigned int depth; + efsys_lock_state_t state; + uint32_t key; + + + EFSYS_ASSERT3P(spec, !=, NULL); + + if ((rc = siena_filter_spec_from_gen_spec(&sf_spec, spec)) != 0) + goto fail1; + + tbl_id = siena_filter_tbl_id(sf_spec.sfs_type); + sftp = &sfp->sf_tbl[tbl_id]; + + if (sftp->sft_size == 0) { + rc = EINVAL; + goto fail2; + } + + key = siena_filter_build(&filter, &sf_spec); + + EFSYS_LOCK(enp->en_eslp, state); + + rc = siena_filter_search(sftp, &sf_spec, key, B_TRUE, + &filter_idx, &depth); + if (rc != 0) + goto fail3; + + EFSYS_ASSERT3U(filter_idx, <, sftp->sft_size); + saved_sf_spec = &sftp->sft_spec[filter_idx]; + + if (siena_filter_test_used(sftp, filter_idx)) { + if (may_replace == B_FALSE) { + rc = EEXIST; + goto fail4; + } + } + siena_filter_set_used(sftp, filter_idx); + *saved_sf_spec = sf_spec; + + if (sfp->sf_depth[sf_spec.sfs_type] < depth) { + sfp->sf_depth[sf_spec.sfs_type] = depth; + if (tbl_id == EFX_SIENA_FILTER_TBL_TX_IP || + tbl_id == EFX_SIENA_FILTER_TBL_TX_MAC) + siena_filter_push_tx_limits(enp); + else + siena_filter_push_rx_limits(enp); + } + + siena_filter_push_entry(enp, sf_spec.sfs_type, + filter_idx, &filter); + + EFSYS_UNLOCK(enp->en_eslp, state); + return (0); + +fail4: + EFSYS_PROBE(fail4); + +fail3: + EFSYS_UNLOCK(enp->en_eslp, state); + EFSYS_PROBE(fail3); + +fail2: + EFSYS_PROBE(fail2); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + return (rc); +} + +static __checkReturn efx_rc_t +siena_filter_delete( + __in efx_nic_t *enp, + __inout efx_filter_spec_t *spec) +{ + efx_rc_t rc; + siena_filter_spec_t sf_spec; + siena_filter_t *sfp = enp->en_filter.ef_siena_filter; + siena_filter_tbl_id_t tbl_id; + siena_filter_tbl_t *sftp; + efx_oword_t filter; + int filter_idx; + unsigned int depth; + efsys_lock_state_t state; + uint32_t key; + + EFSYS_ASSERT3P(spec, !=, NULL); + + if ((rc = siena_filter_spec_from_gen_spec(&sf_spec, spec)) != 0) + goto fail1; + + tbl_id = siena_filter_tbl_id(sf_spec.sfs_type); + sftp = &sfp->sf_tbl[tbl_id]; + + key = siena_filter_build(&filter, &sf_spec); + + EFSYS_LOCK(enp->en_eslp, state); + + rc = siena_filter_search(sftp, &sf_spec, key, B_FALSE, + &filter_idx, &depth); + if (rc != 0) + goto fail2; + + siena_filter_clear_entry(enp, sftp, filter_idx); + if (sftp->sft_used == 0) + siena_filter_reset_search_depth(sfp, tbl_id); + + EFSYS_UNLOCK(enp->en_eslp, state); + return (0); + +fail2: + EFSYS_UNLOCK(enp->en_eslp, state); + EFSYS_PROBE(fail2); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + return (rc); +} + +#define SIENA_MAX_SUPPORTED_MATCHES 4 + +static __checkReturn efx_rc_t +siena_filter_supported_filters( + __in efx_nic_t *enp, + __out_ecount(buffer_length) uint32_t *buffer, + __in size_t buffer_length, + __out size_t *list_lengthp) +{ + uint32_t index = 0; + uint32_t rx_matches[SIENA_MAX_SUPPORTED_MATCHES]; + size_t list_length; + efx_rc_t rc; + + rx_matches[index++] = + EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO | + EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT | + EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_REM_PORT; + + rx_matches[index++] = + EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO | + EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT; + + if (enp->en_features & EFX_FEATURE_MAC_HEADER_FILTERS) { + rx_matches[index++] = + EFX_FILTER_MATCH_OUTER_VID | EFX_FILTER_MATCH_LOC_MAC; + + rx_matches[index++] = EFX_FILTER_MATCH_LOC_MAC; + } + + EFSYS_ASSERT3U(index, <=, SIENA_MAX_SUPPORTED_MATCHES); + list_length = index; + + *list_lengthp = list_length; + + if (buffer_length < list_length) { + rc = ENOSPC; + goto fail1; + } + + memcpy(buffer, rx_matches, list_length * sizeof (rx_matches[0])); + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +#undef MAX_SUPPORTED + +#endif /* EFSYS_OPT_SIENA */ + +#endif /* EFSYS_OPT_FILTER */ diff --git a/dpdk/drivers/net/sfc/base/efx_hash.c b/dpdk/drivers/net/sfc/base/efx_hash.c new file mode 100644 index 00000000..3cc0d200 --- /dev/null +++ b/dpdk/drivers/net/sfc/base/efx_hash.c @@ -0,0 +1,328 @@ +/* + * Copyright 2006 Bob Jenkins + * + * Derived from public domain source, see + * : + * + * "lookup3.c, by Bob Jenkins, May 2006, Public Domain. + * + * These are functions for producing 32-bit hashes for hash table lookup... + * ...You can use this free for any purpose. It's in the public domain. + * It has no warranty." + * + * Copyright (c) 2014-2016 Solarflare Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are + * those of the authors and should not be interpreted as representing official + * policies, either expressed or implied, of the FreeBSD Project. + */ + +#include "efx.h" +#include "efx_impl.h" + +/* Hash initial value */ +#define EFX_HASH_INITIAL_VALUE 0xdeadbeef + +/* + * Rotate a 32-bit value left + * + * Allow platform to provide an intrinsic or optimised routine and + * fall-back to a simple shift based implementation. + */ +#if EFSYS_HAS_ROTL_DWORD + +#define EFX_HASH_ROTATE(_value, _shift) \ + EFSYS_ROTL_DWORD(_value, _shift) + +#else + +#define EFX_HASH_ROTATE(_value, _shift) \ + (((_value) << (_shift)) | ((_value) >> (32 - (_shift)))) + +#endif + +/* Mix three 32-bit values reversibly */ +#define EFX_HASH_MIX(_a, _b, _c) \ + do { \ + _a -= _c; \ + _a ^= EFX_HASH_ROTATE(_c, 4); \ + _c += _b; \ + _b -= _a; \ + _b ^= EFX_HASH_ROTATE(_a, 6); \ + _a += _c; \ + _c -= _b; \ + _c ^= EFX_HASH_ROTATE(_b, 8); \ + _b += _a; \ + _a -= _c; \ + _a ^= EFX_HASH_ROTATE(_c, 16); \ + _c += _b; \ + _b -= _a; \ + _b ^= EFX_HASH_ROTATE(_a, 19); \ + _a += _c; \ + _c -= _b; \ + _c ^= EFX_HASH_ROTATE(_b, 4); \ + _b += _a; \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +/* Final mixing of three 32-bit values into one (_c) */ +#define EFX_HASH_FINALISE(_a, _b, _c) \ + do { \ + _c ^= _b; \ + _c -= EFX_HASH_ROTATE(_b, 14); \ + _a ^= _c; \ + _a -= EFX_HASH_ROTATE(_c, 11); \ + _b ^= _a; \ + _b -= EFX_HASH_ROTATE(_a, 25); \ + _c ^= _b; \ + _c -= EFX_HASH_ROTATE(_b, 16); \ + _a ^= _c; \ + _a -= EFX_HASH_ROTATE(_c, 4); \ + _b ^= _a; \ + _b -= EFX_HASH_ROTATE(_a, 14); \ + _c ^= _b; \ + _c -= EFX_HASH_ROTATE(_b, 24); \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + + +/* Produce a 32-bit hash from 32-bit aligned input */ + __checkReturn uint32_t +efx_hash_dwords( + __in_ecount(count) uint32_t const *input, + __in size_t count, + __in uint32_t init) +{ + uint32_t a; + uint32_t b; + uint32_t c; + + /* Set up the initial internal state */ + a = b = c = EFX_HASH_INITIAL_VALUE + + (((uint32_t)count) * sizeof (uint32_t)) + init; + + /* Handle all but the last three dwords of the input */ + while (count > 3) { + a += input[0]; + b += input[1]; + c += input[2]; + EFX_HASH_MIX(a, b, c); + + count -= 3; + input += 3; + } + + /* Handle the left-overs */ + switch (count) { + case 3: + c += input[2]; + /* Fall-through */ + case 2: + b += input[1]; + /* Fall-through */ + case 1: + a += input[0]; + EFX_HASH_FINALISE(a, b, c); + break; + + case 0: + /* Should only get here if count parameter was zero */ + break; + } + + return (c); +} + +#if EFSYS_IS_BIG_ENDIAN + +/* Produce a 32-bit hash from arbitrarily aligned input */ + __checkReturn uint32_t +efx_hash_bytes( + __in_ecount(length) uint8_t const *input, + __in size_t length, + __in uint32_t init) +{ + uint32_t a; + uint32_t b; + uint32_t c; + + /* Set up the initial internal state */ + a = b = c = EFX_HASH_INITIAL_VALUE + (uint32_t)length + init; + + /* Handle all but the last twelve bytes of the input */ + while (length > 12) { + a += ((uint32_t)input[0]) << 24; + a += ((uint32_t)input[1]) << 16; + a += ((uint32_t)input[2]) << 8; + a += ((uint32_t)input[3]); + b += ((uint32_t)input[4]) << 24; + b += ((uint32_t)input[5]) << 16; + b += ((uint32_t)input[6]) << 8; + b += ((uint32_t)input[7]); + c += ((uint32_t)input[8]) << 24; + c += ((uint32_t)input[9]) << 16; + c += ((uint32_t)input[10]) << 8; + c += ((uint32_t)input[11]); + EFX_HASH_MIX(a, b, c); + length -= 12; + input += 12; + } + + /* Handle the left-overs */ + switch (length) { + case 12: + c += ((uint32_t)input[11]); + /* Fall-through */ + case 11: + c += ((uint32_t)input[10]) << 8; + /* Fall-through */ + case 10: + c += ((uint32_t)input[9]) << 16; + /* Fall-through */ + case 9: + c += ((uint32_t)input[8]) << 24; + /* Fall-through */ + case 8: + b += ((uint32_t)input[7]); + /* Fall-through */ + case 7: + b += ((uint32_t)input[6]) << 8; + /* Fall-through */ + case 6: + b += ((uint32_t)input[5]) << 16; + /* Fall-through */ + case 5: + b += ((uint32_t)input[4]) << 24; + /* Fall-through */ + case 4: + a += ((uint32_t)input[3]); + /* Fall-through */ + case 3: + a += ((uint32_t)input[2]) << 8; + /* Fall-through */ + case 2: + a += ((uint32_t)input[1]) << 16; + /* Fall-through */ + case 1: + a += ((uint32_t)input[0]) << 24; + EFX_HASH_FINALISE(a, b, c); + break; + + case 0: + /* Should only get here if length parameter was zero */ + break; + } + + return (c); +} + +#elif EFSYS_IS_LITTLE_ENDIAN + +/* Produce a 32-bit hash from arbitrarily aligned input */ + __checkReturn uint32_t +efx_hash_bytes( + __in_ecount(length) uint8_t const *input, + __in size_t length, + __in uint32_t init) +{ + uint32_t a; + uint32_t b; + uint32_t c; + + /* Set up the initial internal state */ + a = b = c = EFX_HASH_INITIAL_VALUE + (uint32_t)length + init; + + /* Handle all but the last twelve bytes of the input */ + while (length > 12) { + a += ((uint32_t)input[0]); + a += ((uint32_t)input[1]) << 8; + a += ((uint32_t)input[2]) << 16; + a += ((uint32_t)input[3]) << 24; + b += ((uint32_t)input[4]); + b += ((uint32_t)input[5]) << 8; + b += ((uint32_t)input[6]) << 16; + b += ((uint32_t)input[7]) << 24; + c += ((uint32_t)input[8]); + c += ((uint32_t)input[9]) << 8; + c += ((uint32_t)input[10]) << 16; + c += ((uint32_t)input[11]) << 24; + EFX_HASH_MIX(a, b, c); + length -= 12; + input += 12; + } + + /* Handle the left-overs */ + switch (length) { + case 12: + c += ((uint32_t)input[11]) << 24; + /* Fall-through */ + case 11: + c += ((uint32_t)input[10]) << 16; + /* Fall-through */ + case 10: + c += ((uint32_t)input[9]) << 8; + /* Fall-through */ + case 9: + c += ((uint32_t)input[8]); + /* Fall-through */ + case 8: + b += ((uint32_t)input[7]) << 24; + /* Fall-through */ + case 7: + b += ((uint32_t)input[6]) << 16; + /* Fall-through */ + case 6: + b += ((uint32_t)input[5]) << 8; + /* Fall-through */ + case 5: + b += ((uint32_t)input[4]); + /* Fall-through */ + case 4: + a += ((uint32_t)input[3]) << 24; + /* Fall-through */ + case 3: + a += ((uint32_t)input[2]) << 16; + /* Fall-through */ + case 2: + a += ((uint32_t)input[1]) << 8; + /* Fall-through */ + case 1: + a += ((uint32_t)input[0]); + EFX_HASH_FINALISE(a, b, c); + break; + + case 0: + /* Should only get here if length parameter was zero */ + break; + } + + return (c); +} + +#else + +#error "Neither of EFSYS_IS_{BIG,LITTLE}_ENDIAN is set" + +#endif diff --git a/dpdk/drivers/net/sfc/base/efx_impl.h b/dpdk/drivers/net/sfc/base/efx_impl.h new file mode 100644 index 00000000..53fa37ac --- /dev/null +++ b/dpdk/drivers/net/sfc/base/efx_impl.h @@ -0,0 +1,1214 @@ +/* + * Copyright (c) 2007-2016 Solarflare Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are + * those of the authors and should not be interpreted as representing official + * policies, either expressed or implied, of the FreeBSD Project. + */ + +#ifndef _SYS_EFX_IMPL_H +#define _SYS_EFX_IMPL_H + +#include "efx.h" +#include "efx_regs.h" +#include "efx_regs_ef10.h" + +/* FIXME: Add definition for driver generated software events */ +#ifndef ESE_DZ_EV_CODE_DRV_GEN_EV +#define ESE_DZ_EV_CODE_DRV_GEN_EV FSE_AZ_EV_CODE_DRV_GEN_EV +#endif + + +#if EFSYS_OPT_SIENA +#include "siena_impl.h" +#endif /* EFSYS_OPT_SIENA */ + +#if EFSYS_OPT_HUNTINGTON +#include "hunt_impl.h" +#endif /* EFSYS_OPT_HUNTINGTON */ + +#if EFSYS_OPT_MEDFORD +#include "medford_impl.h" +#endif /* EFSYS_OPT_MEDFORD */ + +#if (EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD) +#include "ef10_impl.h" +#endif /* (EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD) */ + +#ifdef __cplusplus +extern "C" { +#endif + +#define EFX_MOD_MCDI 0x00000001 +#define EFX_MOD_PROBE 0x00000002 +#define EFX_MOD_NVRAM 0x00000004 +#define EFX_MOD_VPD 0x00000008 +#define EFX_MOD_NIC 0x00000010 +#define EFX_MOD_INTR 0x00000020 +#define EFX_MOD_EV 0x00000040 +#define EFX_MOD_RX 0x00000080 +#define EFX_MOD_TX 0x00000100 +#define EFX_MOD_PORT 0x00000200 +#define EFX_MOD_MON 0x00000400 +#define EFX_MOD_FILTER 0x00001000 +#define EFX_MOD_LIC 0x00002000 + +#define EFX_RESET_PHY 0x00000001 +#define EFX_RESET_RXQ_ERR 0x00000002 +#define EFX_RESET_TXQ_ERR 0x00000004 + +typedef enum efx_mac_type_e { + EFX_MAC_INVALID = 0, + EFX_MAC_SIENA, + EFX_MAC_HUNTINGTON, + EFX_MAC_MEDFORD, + EFX_MAC_NTYPES +} efx_mac_type_t; + +typedef struct efx_ev_ops_s { + efx_rc_t (*eevo_init)(efx_nic_t *); + void (*eevo_fini)(efx_nic_t *); + efx_rc_t (*eevo_qcreate)(efx_nic_t *, unsigned int, + efsys_mem_t *, size_t, uint32_t, + uint32_t, uint32_t, efx_evq_t *); + void (*eevo_qdestroy)(efx_evq_t *); + efx_rc_t (*eevo_qprime)(efx_evq_t *, unsigned int); + void (*eevo_qpost)(efx_evq_t *, uint16_t); + efx_rc_t (*eevo_qmoderate)(efx_evq_t *, unsigned int); +#if EFSYS_OPT_QSTATS + void (*eevo_qstats_update)(efx_evq_t *, efsys_stat_t *); +#endif +} efx_ev_ops_t; + +typedef struct efx_tx_ops_s { + efx_rc_t (*etxo_init)(efx_nic_t *); + void (*etxo_fini)(efx_nic_t *); + efx_rc_t (*etxo_qcreate)(efx_nic_t *, + unsigned int, unsigned int, + efsys_mem_t *, size_t, + uint32_t, uint16_t, + efx_evq_t *, efx_txq_t *, + unsigned int *); + void (*etxo_qdestroy)(efx_txq_t *); + efx_rc_t (*etxo_qpost)(efx_txq_t *, efx_buffer_t *, + unsigned int, unsigned int, + unsigned int *); + void (*etxo_qpush)(efx_txq_t *, unsigned int, unsigned int); + efx_rc_t (*etxo_qpace)(efx_txq_t *, unsigned int); + efx_rc_t (*etxo_qflush)(efx_txq_t *); + void (*etxo_qenable)(efx_txq_t *); + efx_rc_t (*etxo_qpio_enable)(efx_txq_t *); + void (*etxo_qpio_disable)(efx_txq_t *); + efx_rc_t (*etxo_qpio_write)(efx_txq_t *, uint8_t *, size_t, + size_t); + efx_rc_t (*etxo_qpio_post)(efx_txq_t *, size_t, unsigned int, + unsigned int *); + efx_rc_t (*etxo_qdesc_post)(efx_txq_t *, efx_desc_t *, + unsigned int, unsigned int, + unsigned int *); + void (*etxo_qdesc_dma_create)(efx_txq_t *, efsys_dma_addr_t, + size_t, boolean_t, + efx_desc_t *); + void (*etxo_qdesc_tso_create)(efx_txq_t *, uint16_t, + uint32_t, uint8_t, + efx_desc_t *); + void (*etxo_qdesc_tso2_create)(efx_txq_t *, uint16_t, + uint32_t, uint16_t, + efx_desc_t *, int); + void (*etxo_qdesc_vlantci_create)(efx_txq_t *, uint16_t, + efx_desc_t *); +#if EFSYS_OPT_QSTATS + void (*etxo_qstats_update)(efx_txq_t *, + efsys_stat_t *); +#endif +} efx_tx_ops_t; + +typedef struct efx_rx_ops_s { + efx_rc_t (*erxo_init)(efx_nic_t *); + void (*erxo_fini)(efx_nic_t *); +#if EFSYS_OPT_RX_SCATTER + efx_rc_t (*erxo_scatter_enable)(efx_nic_t *, unsigned int); +#endif +#if EFSYS_OPT_RX_SCALE + efx_rc_t (*erxo_scale_context_alloc)(efx_nic_t *, + efx_rx_scale_context_type_t, + uint32_t, uint32_t *); + efx_rc_t (*erxo_scale_context_free)(efx_nic_t *, uint32_t); + efx_rc_t (*erxo_scale_mode_set)(efx_nic_t *, uint32_t, + efx_rx_hash_alg_t, + efx_rx_hash_type_t, boolean_t); + efx_rc_t (*erxo_scale_key_set)(efx_nic_t *, uint32_t, + uint8_t *, size_t); + efx_rc_t (*erxo_scale_tbl_set)(efx_nic_t *, uint32_t, + unsigned int *, size_t); + uint32_t (*erxo_prefix_hash)(efx_nic_t *, efx_rx_hash_alg_t, + uint8_t *); +#endif /* EFSYS_OPT_RX_SCALE */ + efx_rc_t (*erxo_prefix_pktlen)(efx_nic_t *, uint8_t *, + uint16_t *); + void (*erxo_qpost)(efx_rxq_t *, efsys_dma_addr_t *, size_t, + unsigned int, unsigned int, + unsigned int); + void (*erxo_qpush)(efx_rxq_t *, unsigned int, unsigned int *); +#if EFSYS_OPT_RX_PACKED_STREAM + void (*erxo_qps_update_credits)(efx_rxq_t *); + uint8_t * (*erxo_qps_packet_info)(efx_rxq_t *, uint8_t *, + uint32_t, uint32_t, + uint16_t *, uint32_t *, uint32_t *); +#endif + efx_rc_t (*erxo_qflush)(efx_rxq_t *); + void (*erxo_qenable)(efx_rxq_t *); + efx_rc_t (*erxo_qcreate)(efx_nic_t *enp, unsigned int, + unsigned int, efx_rxq_type_t, + efsys_mem_t *, size_t, uint32_t, + efx_evq_t *, efx_rxq_t *); + void (*erxo_qdestroy)(efx_rxq_t *); +} efx_rx_ops_t; + +typedef struct efx_mac_ops_s { + efx_rc_t (*emo_poll)(efx_nic_t *, efx_link_mode_t *); + efx_rc_t (*emo_up)(efx_nic_t *, boolean_t *); + efx_rc_t (*emo_addr_set)(efx_nic_t *); + efx_rc_t (*emo_pdu_set)(efx_nic_t *); + efx_rc_t (*emo_pdu_get)(efx_nic_t *, size_t *); + efx_rc_t (*emo_reconfigure)(efx_nic_t *); + efx_rc_t (*emo_multicast_list_set)(efx_nic_t *); + efx_rc_t (*emo_filter_default_rxq_set)(efx_nic_t *, + efx_rxq_t *, boolean_t); + void (*emo_filter_default_rxq_clear)(efx_nic_t *); +#if EFSYS_OPT_LOOPBACK + efx_rc_t (*emo_loopback_set)(efx_nic_t *, efx_link_mode_t, + efx_loopback_type_t); +#endif /* EFSYS_OPT_LOOPBACK */ +#if EFSYS_OPT_MAC_STATS + efx_rc_t (*emo_stats_get_mask)(efx_nic_t *, uint32_t *, size_t); + efx_rc_t (*emo_stats_clear)(efx_nic_t *); + efx_rc_t (*emo_stats_upload)(efx_nic_t *, efsys_mem_t *); + efx_rc_t (*emo_stats_periodic)(efx_nic_t *, efsys_mem_t *, + uint16_t, boolean_t); + efx_rc_t (*emo_stats_update)(efx_nic_t *, efsys_mem_t *, + efsys_stat_t *, uint32_t *); +#endif /* EFSYS_OPT_MAC_STATS */ +} efx_mac_ops_t; + +typedef struct efx_phy_ops_s { + efx_rc_t (*epo_power)(efx_nic_t *, boolean_t); /* optional */ + efx_rc_t (*epo_reset)(efx_nic_t *); + efx_rc_t (*epo_reconfigure)(efx_nic_t *); + efx_rc_t (*epo_verify)(efx_nic_t *); + efx_rc_t (*epo_oui_get)(efx_nic_t *, uint32_t *); +#if EFSYS_OPT_PHY_STATS + efx_rc_t (*epo_stats_update)(efx_nic_t *, efsys_mem_t *, + uint32_t *); +#endif /* EFSYS_OPT_PHY_STATS */ +#if EFSYS_OPT_BIST + efx_rc_t (*epo_bist_enable_offline)(efx_nic_t *); + efx_rc_t (*epo_bist_start)(efx_nic_t *, efx_bist_type_t); + efx_rc_t (*epo_bist_poll)(efx_nic_t *, efx_bist_type_t, + efx_bist_result_t *, uint32_t *, + unsigned long *, size_t); + void (*epo_bist_stop)(efx_nic_t *, efx_bist_type_t); +#endif /* EFSYS_OPT_BIST */ +} efx_phy_ops_t; + +#if EFSYS_OPT_FILTER +typedef struct efx_filter_ops_s { + efx_rc_t (*efo_init)(efx_nic_t *); + void (*efo_fini)(efx_nic_t *); + efx_rc_t (*efo_restore)(efx_nic_t *); + efx_rc_t (*efo_add)(efx_nic_t *, efx_filter_spec_t *, + boolean_t may_replace); + efx_rc_t (*efo_delete)(efx_nic_t *, efx_filter_spec_t *); + efx_rc_t (*efo_supported_filters)(efx_nic_t *, uint32_t *, + size_t, size_t *); + efx_rc_t (*efo_reconfigure)(efx_nic_t *, uint8_t const *, boolean_t, + boolean_t, boolean_t, boolean_t, + uint8_t const *, uint32_t); +} efx_filter_ops_t; + +extern __checkReturn efx_rc_t +efx_filter_reconfigure( + __in efx_nic_t *enp, + __in_ecount(6) uint8_t const *mac_addr, + __in boolean_t all_unicst, + __in boolean_t mulcst, + __in boolean_t all_mulcst, + __in boolean_t brdcst, + __in_ecount(6*count) uint8_t const *addrs, + __in uint32_t count); + +#endif /* EFSYS_OPT_FILTER */ + + +typedef struct efx_port_s { + efx_mac_type_t ep_mac_type; + uint32_t ep_phy_type; + uint8_t ep_port; + uint32_t ep_mac_pdu; + uint8_t ep_mac_addr[6]; + efx_link_mode_t ep_link_mode; + boolean_t ep_all_unicst; + boolean_t ep_mulcst; + boolean_t ep_all_mulcst; + boolean_t ep_brdcst; + unsigned int ep_fcntl; + boolean_t ep_fcntl_autoneg; + efx_oword_t ep_multicst_hash[2]; + uint8_t ep_mulcst_addr_list[EFX_MAC_ADDR_LEN * + EFX_MAC_MULTICAST_LIST_MAX]; + uint32_t ep_mulcst_addr_count; +#if EFSYS_OPT_LOOPBACK + efx_loopback_type_t ep_loopback_type; + efx_link_mode_t ep_loopback_link_mode; +#endif /* EFSYS_OPT_LOOPBACK */ +#if EFSYS_OPT_PHY_FLAGS + uint32_t ep_phy_flags; +#endif /* EFSYS_OPT_PHY_FLAGS */ +#if EFSYS_OPT_PHY_LED_CONTROL + efx_phy_led_mode_t ep_phy_led_mode; +#endif /* EFSYS_OPT_PHY_LED_CONTROL */ + efx_phy_media_type_t ep_fixed_port_type; + efx_phy_media_type_t ep_module_type; + uint32_t ep_adv_cap_mask; + uint32_t ep_lp_cap_mask; + uint32_t ep_default_adv_cap_mask; + uint32_t ep_phy_cap_mask; + boolean_t ep_mac_drain; + boolean_t ep_mac_stats_pending; +#if EFSYS_OPT_BIST + efx_bist_type_t ep_current_bist; +#endif + const efx_mac_ops_t *ep_emop; + const efx_phy_ops_t *ep_epop; +} efx_port_t; + +typedef struct efx_mon_ops_s { +#if EFSYS_OPT_MON_STATS + efx_rc_t (*emo_stats_update)(efx_nic_t *, efsys_mem_t *, + efx_mon_stat_value_t *); +#endif /* EFSYS_OPT_MON_STATS */ +} efx_mon_ops_t; + +typedef struct efx_mon_s { + efx_mon_type_t em_type; + const efx_mon_ops_t *em_emop; +} efx_mon_t; + +typedef struct efx_intr_ops_s { + efx_rc_t (*eio_init)(efx_nic_t *, efx_intr_type_t, efsys_mem_t *); + void (*eio_enable)(efx_nic_t *); + void (*eio_disable)(efx_nic_t *); + void (*eio_disable_unlocked)(efx_nic_t *); + efx_rc_t (*eio_trigger)(efx_nic_t *, unsigned int); + void (*eio_status_line)(efx_nic_t *, boolean_t *, uint32_t *); + void (*eio_status_message)(efx_nic_t *, unsigned int, + boolean_t *); + void (*eio_fatal)(efx_nic_t *); + void (*eio_fini)(efx_nic_t *); +} efx_intr_ops_t; + +typedef struct efx_intr_s { + const efx_intr_ops_t *ei_eiop; + efsys_mem_t *ei_esmp; + efx_intr_type_t ei_type; + unsigned int ei_level; +} efx_intr_t; + +typedef struct efx_nic_ops_s { + efx_rc_t (*eno_probe)(efx_nic_t *); + efx_rc_t (*eno_board_cfg)(efx_nic_t *); + efx_rc_t (*eno_set_drv_limits)(efx_nic_t *, efx_drv_limits_t*); + efx_rc_t (*eno_reset)(efx_nic_t *); + efx_rc_t (*eno_init)(efx_nic_t *); + efx_rc_t (*eno_get_vi_pool)(efx_nic_t *, uint32_t *); + efx_rc_t (*eno_get_bar_region)(efx_nic_t *, efx_nic_region_t, + uint32_t *, size_t *); +#if EFSYS_OPT_DIAG + efx_rc_t (*eno_register_test)(efx_nic_t *); +#endif /* EFSYS_OPT_DIAG */ + void (*eno_fini)(efx_nic_t *); + void (*eno_unprobe)(efx_nic_t *); +} efx_nic_ops_t; + +#ifndef EFX_TXQ_LIMIT_TARGET +#define EFX_TXQ_LIMIT_TARGET 259 +#endif +#ifndef EFX_RXQ_LIMIT_TARGET +#define EFX_RXQ_LIMIT_TARGET 512 +#endif +#ifndef EFX_TXQ_DC_SIZE +#define EFX_TXQ_DC_SIZE 1 /* 16 descriptors */ +#endif +#ifndef EFX_RXQ_DC_SIZE +#define EFX_RXQ_DC_SIZE 3 /* 64 descriptors */ +#endif + +#if EFSYS_OPT_FILTER + +#if EFSYS_OPT_SIENA + +typedef struct siena_filter_spec_s { + uint8_t sfs_type; + uint32_t sfs_flags; + uint32_t sfs_dmaq_id; + uint32_t sfs_dword[3]; +} siena_filter_spec_t; + +typedef enum siena_filter_type_e { + EFX_SIENA_FILTER_RX_TCP_FULL, /* TCP/IPv4 {dIP,dTCP,sIP,sTCP} */ + EFX_SIENA_FILTER_RX_TCP_WILD, /* TCP/IPv4 {dIP,dTCP, -, -} */ + EFX_SIENA_FILTER_RX_UDP_FULL, /* UDP/IPv4 {dIP,dUDP,sIP,sUDP} */ + EFX_SIENA_FILTER_RX_UDP_WILD, /* UDP/IPv4 {dIP,dUDP, -, -} */ + EFX_SIENA_FILTER_RX_MAC_FULL, /* Ethernet {dMAC,VLAN} */ + EFX_SIENA_FILTER_RX_MAC_WILD, /* Ethernet {dMAC, -} */ + + EFX_SIENA_FILTER_TX_TCP_FULL, /* TCP/IPv4 {dIP,dTCP,sIP,sTCP} */ + EFX_SIENA_FILTER_TX_TCP_WILD, /* TCP/IPv4 { -, -,sIP,sTCP} */ + EFX_SIENA_FILTER_TX_UDP_FULL, /* UDP/IPv4 {dIP,dTCP,sIP,sTCP} */ + EFX_SIENA_FILTER_TX_UDP_WILD, /* UDP/IPv4 { -, -,sIP,sUDP} */ + EFX_SIENA_FILTER_TX_MAC_FULL, /* Ethernet {sMAC,VLAN} */ + EFX_SIENA_FILTER_TX_MAC_WILD, /* Ethernet {sMAC, -} */ + + EFX_SIENA_FILTER_NTYPES +} siena_filter_type_t; + +typedef enum siena_filter_tbl_id_e { + EFX_SIENA_FILTER_TBL_RX_IP = 0, + EFX_SIENA_FILTER_TBL_RX_MAC, + EFX_SIENA_FILTER_TBL_TX_IP, + EFX_SIENA_FILTER_TBL_TX_MAC, + EFX_SIENA_FILTER_NTBLS +} siena_filter_tbl_id_t; + +typedef struct siena_filter_tbl_s { + int sft_size; /* number of entries */ + int sft_used; /* active count */ + uint32_t *sft_bitmap; /* active bitmap */ + siena_filter_spec_t *sft_spec; /* array of saved specs */ +} siena_filter_tbl_t; + +typedef struct siena_filter_s { + siena_filter_tbl_t sf_tbl[EFX_SIENA_FILTER_NTBLS]; + unsigned int sf_depth[EFX_SIENA_FILTER_NTYPES]; +} siena_filter_t; + +#endif /* EFSYS_OPT_SIENA */ + +typedef struct efx_filter_s { +#if EFSYS_OPT_SIENA + siena_filter_t *ef_siena_filter; +#endif /* EFSYS_OPT_SIENA */ +#if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD + ef10_filter_table_t *ef_ef10_filter_table; +#endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */ +} efx_filter_t; + +#if EFSYS_OPT_SIENA + +extern void +siena_filter_tbl_clear( + __in efx_nic_t *enp, + __in siena_filter_tbl_id_t tbl); + +#endif /* EFSYS_OPT_SIENA */ + +#endif /* EFSYS_OPT_FILTER */ + +#if EFSYS_OPT_MCDI + +typedef struct efx_mcdi_ops_s { + efx_rc_t (*emco_init)(efx_nic_t *, const efx_mcdi_transport_t *); + void (*emco_send_request)(efx_nic_t *, void *, size_t, + void *, size_t); + efx_rc_t (*emco_poll_reboot)(efx_nic_t *); + boolean_t (*emco_poll_response)(efx_nic_t *); + void (*emco_read_response)(efx_nic_t *, void *, size_t, size_t); + void (*emco_fini)(efx_nic_t *); + efx_rc_t (*emco_feature_supported)(efx_nic_t *, + efx_mcdi_feature_id_t, boolean_t *); + void (*emco_get_timeout)(efx_nic_t *, efx_mcdi_req_t *, + uint32_t *); +} efx_mcdi_ops_t; + +typedef struct efx_mcdi_s { + const efx_mcdi_ops_t *em_emcop; + const efx_mcdi_transport_t *em_emtp; + efx_mcdi_iface_t em_emip; +} efx_mcdi_t; + +#endif /* EFSYS_OPT_MCDI */ + +#if EFSYS_OPT_NVRAM +typedef struct efx_nvram_ops_s { +#if EFSYS_OPT_DIAG + efx_rc_t (*envo_test)(efx_nic_t *); +#endif /* EFSYS_OPT_DIAG */ + efx_rc_t (*envo_type_to_partn)(efx_nic_t *, efx_nvram_type_t, + uint32_t *); + efx_rc_t (*envo_partn_size)(efx_nic_t *, uint32_t, size_t *); + efx_rc_t (*envo_partn_rw_start)(efx_nic_t *, uint32_t, size_t *); + efx_rc_t (*envo_partn_read)(efx_nic_t *, uint32_t, + unsigned int, caddr_t, size_t); + efx_rc_t (*envo_partn_erase)(efx_nic_t *, uint32_t, + unsigned int, size_t); + efx_rc_t (*envo_partn_write)(efx_nic_t *, uint32_t, + unsigned int, caddr_t, size_t); + efx_rc_t (*envo_partn_rw_finish)(efx_nic_t *, uint32_t); + efx_rc_t (*envo_partn_get_version)(efx_nic_t *, uint32_t, + uint32_t *, uint16_t *); + efx_rc_t (*envo_partn_set_version)(efx_nic_t *, uint32_t, + uint16_t *); + efx_rc_t (*envo_buffer_validate)(efx_nic_t *, uint32_t, + caddr_t, size_t); +} efx_nvram_ops_t; +#endif /* EFSYS_OPT_NVRAM */ + +#if EFSYS_OPT_VPD +typedef struct efx_vpd_ops_s { + efx_rc_t (*evpdo_init)(efx_nic_t *); + efx_rc_t (*evpdo_size)(efx_nic_t *, size_t *); + efx_rc_t (*evpdo_read)(efx_nic_t *, caddr_t, size_t); + efx_rc_t (*evpdo_verify)(efx_nic_t *, caddr_t, size_t); + efx_rc_t (*evpdo_reinit)(efx_nic_t *, caddr_t, size_t); + efx_rc_t (*evpdo_get)(efx_nic_t *, caddr_t, size_t, + efx_vpd_value_t *); + efx_rc_t (*evpdo_set)(efx_nic_t *, caddr_t, size_t, + efx_vpd_value_t *); + efx_rc_t (*evpdo_next)(efx_nic_t *, caddr_t, size_t, + efx_vpd_value_t *, unsigned int *); + efx_rc_t (*evpdo_write)(efx_nic_t *, caddr_t, size_t); + void (*evpdo_fini)(efx_nic_t *); +} efx_vpd_ops_t; +#endif /* EFSYS_OPT_VPD */ + +#if EFSYS_OPT_VPD || EFSYS_OPT_NVRAM + + __checkReturn efx_rc_t +efx_mcdi_nvram_partitions( + __in efx_nic_t *enp, + __out_bcount(size) caddr_t data, + __in size_t size, + __out unsigned int *npartnp); + + __checkReturn efx_rc_t +efx_mcdi_nvram_metadata( + __in efx_nic_t *enp, + __in uint32_t partn, + __out uint32_t *subtypep, + __out_ecount(4) uint16_t version[4], + __out_bcount_opt(size) char *descp, + __in size_t size); + + __checkReturn efx_rc_t +efx_mcdi_nvram_info( + __in efx_nic_t *enp, + __in uint32_t partn, + __out_opt size_t *sizep, + __out_opt uint32_t *addressp, + __out_opt uint32_t *erase_sizep, + __out_opt uint32_t *write_sizep); + + __checkReturn efx_rc_t +efx_mcdi_nvram_update_start( + __in efx_nic_t *enp, + __in uint32_t partn); + + __checkReturn efx_rc_t +efx_mcdi_nvram_read( + __in efx_nic_t *enp, + __in uint32_t partn, + __in uint32_t offset, + __out_bcount(size) caddr_t data, + __in size_t size, + __in uint32_t mode); + + __checkReturn efx_rc_t +efx_mcdi_nvram_erase( + __in efx_nic_t *enp, + __in uint32_t partn, + __in uint32_t offset, + __in size_t size); + + __checkReturn efx_rc_t +efx_mcdi_nvram_write( + __in efx_nic_t *enp, + __in uint32_t partn, + __in uint32_t offset, + __out_bcount(size) caddr_t data, + __in size_t size); + + __checkReturn efx_rc_t +efx_mcdi_nvram_update_finish( + __in efx_nic_t *enp, + __in uint32_t partn, + __in boolean_t reboot, + __out_opt uint32_t *resultp); + +#if EFSYS_OPT_DIAG + + __checkReturn efx_rc_t +efx_mcdi_nvram_test( + __in efx_nic_t *enp, + __in uint32_t partn); + +#endif /* EFSYS_OPT_DIAG */ + +#endif /* EFSYS_OPT_VPD || EFSYS_OPT_NVRAM */ + +#if EFSYS_OPT_LICENSING + +typedef struct efx_lic_ops_s { + efx_rc_t (*elo_update_licenses)(efx_nic_t *); + efx_rc_t (*elo_get_key_stats)(efx_nic_t *, efx_key_stats_t *); + efx_rc_t (*elo_app_state)(efx_nic_t *, uint64_t, boolean_t *); + efx_rc_t (*elo_get_id)(efx_nic_t *, size_t, uint32_t *, + size_t *, uint8_t *); + efx_rc_t (*elo_find_start) + (efx_nic_t *, caddr_t, size_t, uint32_t *); + efx_rc_t (*elo_find_end)(efx_nic_t *, caddr_t, size_t, + uint32_t, uint32_t *); + boolean_t (*elo_find_key)(efx_nic_t *, caddr_t, size_t, + uint32_t, uint32_t *, uint32_t *); + boolean_t (*elo_validate_key)(efx_nic_t *, + caddr_t, uint32_t); + efx_rc_t (*elo_read_key)(efx_nic_t *, + caddr_t, size_t, uint32_t, uint32_t, + caddr_t, size_t, uint32_t *); + efx_rc_t (*elo_write_key)(efx_nic_t *, + caddr_t, size_t, uint32_t, + caddr_t, uint32_t, uint32_t *); + efx_rc_t (*elo_delete_key)(efx_nic_t *, + caddr_t, size_t, uint32_t, + uint32_t, uint32_t, uint32_t *); + efx_rc_t (*elo_create_partition)(efx_nic_t *, + caddr_t, size_t); + efx_rc_t (*elo_finish_partition)(efx_nic_t *, + caddr_t, size_t); +} efx_lic_ops_t; + +#endif + +typedef struct efx_drv_cfg_s { + uint32_t edc_min_vi_count; + uint32_t edc_max_vi_count; + + uint32_t edc_max_piobuf_count; + uint32_t edc_pio_alloc_size; +} efx_drv_cfg_t; + +struct efx_nic_s { + uint32_t en_magic; + efx_family_t en_family; + uint32_t en_features; + efsys_identifier_t *en_esip; + efsys_lock_t *en_eslp; + efsys_bar_t *en_esbp; + unsigned int en_mod_flags; + unsigned int en_reset_flags; + efx_nic_cfg_t en_nic_cfg; + efx_drv_cfg_t en_drv_cfg; + efx_port_t en_port; + efx_mon_t en_mon; + efx_intr_t en_intr; + uint32_t en_ev_qcount; + uint32_t en_rx_qcount; + uint32_t en_tx_qcount; + const efx_nic_ops_t *en_enop; + const efx_ev_ops_t *en_eevop; + const efx_tx_ops_t *en_etxop; + const efx_rx_ops_t *en_erxop; +#if EFSYS_OPT_FILTER + efx_filter_t en_filter; + const efx_filter_ops_t *en_efop; +#endif /* EFSYS_OPT_FILTER */ +#if EFSYS_OPT_MCDI + efx_mcdi_t en_mcdi; +#endif /* EFSYS_OPT_MCDI */ +#if EFSYS_OPT_NVRAM + efx_nvram_type_t en_nvram_locked; + const efx_nvram_ops_t *en_envop; +#endif /* EFSYS_OPT_NVRAM */ +#if EFSYS_OPT_VPD + const efx_vpd_ops_t *en_evpdop; +#endif /* EFSYS_OPT_VPD */ +#if EFSYS_OPT_RX_SCALE + efx_rx_hash_support_t en_hash_support; + efx_rx_scale_context_type_t en_rss_context_type; + uint32_t en_rss_context; +#endif /* EFSYS_OPT_RX_SCALE */ + uint32_t en_vport_id; +#if EFSYS_OPT_LICENSING + const efx_lic_ops_t *en_elop; + boolean_t en_licensing_supported; +#endif + union { +#if EFSYS_OPT_SIENA + struct { +#if EFSYS_OPT_NVRAM || EFSYS_OPT_VPD + unsigned int enu_partn_mask; +#endif /* EFSYS_OPT_NVRAM || EFSYS_OPT_VPD */ +#if EFSYS_OPT_VPD + caddr_t enu_svpd; + size_t enu_svpd_length; +#endif /* EFSYS_OPT_VPD */ + int enu_unused; + } siena; +#endif /* EFSYS_OPT_SIENA */ + int enu_unused; + } en_u; +#if (EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD) + union en_arch { + struct { + int ena_vi_base; + int ena_vi_count; + int ena_vi_shift; +#if EFSYS_OPT_VPD + caddr_t ena_svpd; + size_t ena_svpd_length; +#endif /* EFSYS_OPT_VPD */ + efx_piobuf_handle_t ena_piobuf_handle[EF10_MAX_PIOBUF_NBUFS]; + uint32_t ena_piobuf_count; + uint32_t ena_pio_alloc_map[EF10_MAX_PIOBUF_NBUFS]; + uint32_t ena_pio_write_vi_base; + /* Memory BAR mapping regions */ + uint32_t ena_uc_mem_map_offset; + size_t ena_uc_mem_map_size; + uint32_t ena_wc_mem_map_offset; + size_t ena_wc_mem_map_size; + } ef10; + } en_arch; +#endif /* (EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD) */ +}; + + +#define EFX_NIC_MAGIC 0x02121996 + +typedef boolean_t (*efx_ev_handler_t)(efx_evq_t *, efx_qword_t *, + const efx_ev_callbacks_t *, void *); + +typedef struct efx_evq_rxq_state_s { + unsigned int eers_rx_read_ptr; + unsigned int eers_rx_mask; +#if EFSYS_OPT_RX_PACKED_STREAM + unsigned int eers_rx_stream_npackets; + boolean_t eers_rx_packed_stream; + unsigned int eers_rx_packed_stream_credits; +#endif +} efx_evq_rxq_state_t; + +struct efx_evq_s { + uint32_t ee_magic; + efx_nic_t *ee_enp; + unsigned int ee_index; + unsigned int ee_mask; + efsys_mem_t *ee_esmp; +#if EFSYS_OPT_QSTATS + uint32_t ee_stat[EV_NQSTATS]; +#endif /* EFSYS_OPT_QSTATS */ + + efx_ev_handler_t ee_rx; + efx_ev_handler_t ee_tx; + efx_ev_handler_t ee_driver; + efx_ev_handler_t ee_global; + efx_ev_handler_t ee_drv_gen; +#if EFSYS_OPT_MCDI + efx_ev_handler_t ee_mcdi; +#endif /* EFSYS_OPT_MCDI */ + + efx_evq_rxq_state_t ee_rxq_state[EFX_EV_RX_NLABELS]; + + uint32_t ee_flags; +}; + +#define EFX_EVQ_MAGIC 0x08081997 + +#define EFX_EVQ_SIENA_TIMER_QUANTUM_NS 6144 /* 768 cycles */ + +struct efx_rxq_s { + uint32_t er_magic; + efx_nic_t *er_enp; + efx_evq_t *er_eep; + unsigned int er_index; + unsigned int er_label; + unsigned int er_mask; + efsys_mem_t *er_esmp; +}; + +#define EFX_RXQ_MAGIC 0x15022005 + +struct efx_txq_s { + uint32_t et_magic; + efx_nic_t *et_enp; + unsigned int et_index; + unsigned int et_mask; + efsys_mem_t *et_esmp; +#if EFSYS_OPT_HUNTINGTON + uint32_t et_pio_bufnum; + uint32_t et_pio_blknum; + uint32_t et_pio_write_offset; + uint32_t et_pio_offset; + size_t et_pio_size; +#endif +#if EFSYS_OPT_QSTATS + uint32_t et_stat[TX_NQSTATS]; +#endif /* EFSYS_OPT_QSTATS */ +}; + +#define EFX_TXQ_MAGIC 0x05092005 + +#define EFX_MAC_ADDR_COPY(_dst, _src) \ + do { \ + (_dst)[0] = (_src)[0]; \ + (_dst)[1] = (_src)[1]; \ + (_dst)[2] = (_src)[2]; \ + (_dst)[3] = (_src)[3]; \ + (_dst)[4] = (_src)[4]; \ + (_dst)[5] = (_src)[5]; \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#define EFX_MAC_BROADCAST_ADDR_SET(_dst) \ + do { \ + uint16_t *_d = (uint16_t *)(_dst); \ + _d[0] = 0xffff; \ + _d[1] = 0xffff; \ + _d[2] = 0xffff; \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#if EFSYS_OPT_CHECK_REG +#define EFX_CHECK_REG(_enp, _reg) \ + do { \ + const char *name = #_reg; \ + char min = name[4]; \ + char max = name[5]; \ + char rev; \ + \ + switch ((_enp)->en_family) { \ + case EFX_FAMILY_SIENA: \ + rev = 'C'; \ + break; \ + \ + case EFX_FAMILY_HUNTINGTON: \ + rev = 'D'; \ + break; \ + \ + case EFX_FAMILY_MEDFORD: \ + rev = 'E'; \ + break; \ + \ + default: \ + rev = '?'; \ + break; \ + } \ + \ + EFSYS_ASSERT3S(rev, >=, min); \ + EFSYS_ASSERT3S(rev, <=, max); \ + \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) +#else +#define EFX_CHECK_REG(_enp, _reg) do { \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) +#endif + +#define EFX_BAR_READD(_enp, _reg, _edp, _lock) \ + do { \ + EFX_CHECK_REG((_enp), (_reg)); \ + EFSYS_BAR_READD((_enp)->en_esbp, _reg ## _OFST, \ + (_edp), (_lock)); \ + EFSYS_PROBE3(efx_bar_readd, const char *, #_reg, \ + uint32_t, _reg ## _OFST, \ + uint32_t, (_edp)->ed_u32[0]); \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#define EFX_BAR_WRITED(_enp, _reg, _edp, _lock) \ + do { \ + EFX_CHECK_REG((_enp), (_reg)); \ + EFSYS_PROBE3(efx_bar_writed, const char *, #_reg, \ + uint32_t, _reg ## _OFST, \ + uint32_t, (_edp)->ed_u32[0]); \ + EFSYS_BAR_WRITED((_enp)->en_esbp, _reg ## _OFST, \ + (_edp), (_lock)); \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#define EFX_BAR_READQ(_enp, _reg, _eqp) \ + do { \ + EFX_CHECK_REG((_enp), (_reg)); \ + EFSYS_BAR_READQ((_enp)->en_esbp, _reg ## _OFST, \ + (_eqp)); \ + EFSYS_PROBE4(efx_bar_readq, const char *, #_reg, \ + uint32_t, _reg ## _OFST, \ + uint32_t, (_eqp)->eq_u32[1], \ + uint32_t, (_eqp)->eq_u32[0]); \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#define EFX_BAR_WRITEQ(_enp, _reg, _eqp) \ + do { \ + EFX_CHECK_REG((_enp), (_reg)); \ + EFSYS_PROBE4(efx_bar_writeq, const char *, #_reg, \ + uint32_t, _reg ## _OFST, \ + uint32_t, (_eqp)->eq_u32[1], \ + uint32_t, (_eqp)->eq_u32[0]); \ + EFSYS_BAR_WRITEQ((_enp)->en_esbp, _reg ## _OFST, \ + (_eqp)); \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#define EFX_BAR_READO(_enp, _reg, _eop) \ + do { \ + EFX_CHECK_REG((_enp), (_reg)); \ + EFSYS_BAR_READO((_enp)->en_esbp, _reg ## _OFST, \ + (_eop), B_TRUE); \ + EFSYS_PROBE6(efx_bar_reado, const char *, #_reg, \ + uint32_t, _reg ## _OFST, \ + uint32_t, (_eop)->eo_u32[3], \ + uint32_t, (_eop)->eo_u32[2], \ + uint32_t, (_eop)->eo_u32[1], \ + uint32_t, (_eop)->eo_u32[0]); \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#define EFX_BAR_WRITEO(_enp, _reg, _eop) \ + do { \ + EFX_CHECK_REG((_enp), (_reg)); \ + EFSYS_PROBE6(efx_bar_writeo, const char *, #_reg, \ + uint32_t, _reg ## _OFST, \ + uint32_t, (_eop)->eo_u32[3], \ + uint32_t, (_eop)->eo_u32[2], \ + uint32_t, (_eop)->eo_u32[1], \ + uint32_t, (_eop)->eo_u32[0]); \ + EFSYS_BAR_WRITEO((_enp)->en_esbp, _reg ## _OFST, \ + (_eop), B_TRUE); \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#define EFX_BAR_TBL_READD(_enp, _reg, _index, _edp, _lock) \ + do { \ + EFX_CHECK_REG((_enp), (_reg)); \ + EFSYS_BAR_READD((_enp)->en_esbp, \ + (_reg ## _OFST + ((_index) * _reg ## _STEP)), \ + (_edp), (_lock)); \ + EFSYS_PROBE4(efx_bar_tbl_readd, const char *, #_reg, \ + uint32_t, (_index), \ + uint32_t, _reg ## _OFST, \ + uint32_t, (_edp)->ed_u32[0]); \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#define EFX_BAR_TBL_WRITED(_enp, _reg, _index, _edp, _lock) \ + do { \ + EFX_CHECK_REG((_enp), (_reg)); \ + EFSYS_PROBE4(efx_bar_tbl_writed, const char *, #_reg, \ + uint32_t, (_index), \ + uint32_t, _reg ## _OFST, \ + uint32_t, (_edp)->ed_u32[0]); \ + EFSYS_BAR_WRITED((_enp)->en_esbp, \ + (_reg ## _OFST + ((_index) * _reg ## _STEP)), \ + (_edp), (_lock)); \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#define EFX_BAR_TBL_WRITED2(_enp, _reg, _index, _edp, _lock) \ + do { \ + EFX_CHECK_REG((_enp), (_reg)); \ + EFSYS_PROBE4(efx_bar_tbl_writed, const char *, #_reg, \ + uint32_t, (_index), \ + uint32_t, _reg ## _OFST, \ + uint32_t, (_edp)->ed_u32[0]); \ + EFSYS_BAR_WRITED((_enp)->en_esbp, \ + (_reg ## _OFST + \ + (2 * sizeof (efx_dword_t)) + \ + ((_index) * _reg ## _STEP)), \ + (_edp), (_lock)); \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#define EFX_BAR_TBL_WRITED3(_enp, _reg, _index, _edp, _lock) \ + do { \ + EFX_CHECK_REG((_enp), (_reg)); \ + EFSYS_PROBE4(efx_bar_tbl_writed, const char *, #_reg, \ + uint32_t, (_index), \ + uint32_t, _reg ## _OFST, \ + uint32_t, (_edp)->ed_u32[0]); \ + EFSYS_BAR_WRITED((_enp)->en_esbp, \ + (_reg ## _OFST + \ + (3 * sizeof (efx_dword_t)) + \ + ((_index) * _reg ## _STEP)), \ + (_edp), (_lock)); \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#define EFX_BAR_TBL_READQ(_enp, _reg, _index, _eqp) \ + do { \ + EFX_CHECK_REG((_enp), (_reg)); \ + EFSYS_BAR_READQ((_enp)->en_esbp, \ + (_reg ## _OFST + ((_index) * _reg ## _STEP)), \ + (_eqp)); \ + EFSYS_PROBE5(efx_bar_tbl_readq, const char *, #_reg, \ + uint32_t, (_index), \ + uint32_t, _reg ## _OFST, \ + uint32_t, (_eqp)->eq_u32[1], \ + uint32_t, (_eqp)->eq_u32[0]); \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#define EFX_BAR_TBL_WRITEQ(_enp, _reg, _index, _eqp) \ + do { \ + EFX_CHECK_REG((_enp), (_reg)); \ + EFSYS_PROBE5(efx_bar_tbl_writeq, const char *, #_reg, \ + uint32_t, (_index), \ + uint32_t, _reg ## _OFST, \ + uint32_t, (_eqp)->eq_u32[1], \ + uint32_t, (_eqp)->eq_u32[0]); \ + EFSYS_BAR_WRITEQ((_enp)->en_esbp, \ + (_reg ## _OFST + ((_index) * _reg ## _STEP)), \ + (_eqp)); \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#define EFX_BAR_TBL_READO(_enp, _reg, _index, _eop, _lock) \ + do { \ + EFX_CHECK_REG((_enp), (_reg)); \ + EFSYS_BAR_READO((_enp)->en_esbp, \ + (_reg ## _OFST + ((_index) * _reg ## _STEP)), \ + (_eop), (_lock)); \ + EFSYS_PROBE7(efx_bar_tbl_reado, const char *, #_reg, \ + uint32_t, (_index), \ + uint32_t, _reg ## _OFST, \ + uint32_t, (_eop)->eo_u32[3], \ + uint32_t, (_eop)->eo_u32[2], \ + uint32_t, (_eop)->eo_u32[1], \ + uint32_t, (_eop)->eo_u32[0]); \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#define EFX_BAR_TBL_WRITEO(_enp, _reg, _index, _eop, _lock) \ + do { \ + EFX_CHECK_REG((_enp), (_reg)); \ + EFSYS_PROBE7(efx_bar_tbl_writeo, const char *, #_reg, \ + uint32_t, (_index), \ + uint32_t, _reg ## _OFST, \ + uint32_t, (_eop)->eo_u32[3], \ + uint32_t, (_eop)->eo_u32[2], \ + uint32_t, (_eop)->eo_u32[1], \ + uint32_t, (_eop)->eo_u32[0]); \ + EFSYS_BAR_WRITEO((_enp)->en_esbp, \ + (_reg ## _OFST + ((_index) * _reg ## _STEP)), \ + (_eop), (_lock)); \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +/* + * Allow drivers to perform optimised 128-bit doorbell writes. + * The DMA descriptor pointers (RX_DESC_UPD and TX_DESC_UPD) are + * special-cased in the BIU on the Falcon/Siena and EF10 architectures to avoid + * the need for locking in the host, and are the only ones known to be safe to + * use 128-bites write with. + */ +#define EFX_BAR_TBL_DOORBELL_WRITEO(_enp, _reg, _index, _eop) \ + do { \ + EFX_CHECK_REG((_enp), (_reg)); \ + EFSYS_PROBE7(efx_bar_tbl_doorbell_writeo, \ + const char *, \ + #_reg, \ + uint32_t, (_index), \ + uint32_t, _reg ## _OFST, \ + uint32_t, (_eop)->eo_u32[3], \ + uint32_t, (_eop)->eo_u32[2], \ + uint32_t, (_eop)->eo_u32[1], \ + uint32_t, (_eop)->eo_u32[0]); \ + EFSYS_BAR_DOORBELL_WRITEO((_enp)->en_esbp, \ + (_reg ## _OFST + ((_index) * _reg ## _STEP)), \ + (_eop)); \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#define EFX_DMA_SYNC_QUEUE_FOR_DEVICE(_esmp, _entries, _wptr, _owptr) \ + do { \ + unsigned int _new = (_wptr); \ + unsigned int _old = (_owptr); \ + \ + if ((_new) >= (_old)) \ + EFSYS_DMA_SYNC_FOR_DEVICE((_esmp), \ + (_old) * sizeof (efx_desc_t), \ + ((_new) - (_old)) * sizeof (efx_desc_t)); \ + else \ + /* \ + * It is cheaper to sync entire map than sync \ + * two parts especially when offset/size are \ + * ignored and entire map is synced in any case.\ + */ \ + EFSYS_DMA_SYNC_FOR_DEVICE((_esmp), \ + 0, \ + (_entries) * sizeof (efx_desc_t)); \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +extern __checkReturn efx_rc_t +efx_nic_biu_test( + __in efx_nic_t *enp); + +extern __checkReturn efx_rc_t +efx_mac_select( + __in efx_nic_t *enp); + +extern void +efx_mac_multicast_hash_compute( + __in_ecount(6*count) uint8_t const *addrs, + __in int count, + __out efx_oword_t *hash_low, + __out efx_oword_t *hash_high); + +extern __checkReturn efx_rc_t +efx_phy_probe( + __in efx_nic_t *enp); + +extern void +efx_phy_unprobe( + __in efx_nic_t *enp); + +#if EFSYS_OPT_VPD + +/* VPD utility functions */ + +extern __checkReturn efx_rc_t +efx_vpd_hunk_length( + __in_bcount(size) caddr_t data, + __in size_t size, + __out size_t *lengthp); + +extern __checkReturn efx_rc_t +efx_vpd_hunk_verify( + __in_bcount(size) caddr_t data, + __in size_t size, + __out_opt boolean_t *cksummedp); + +extern __checkReturn efx_rc_t +efx_vpd_hunk_reinit( + __in_bcount(size) caddr_t data, + __in size_t size, + __in boolean_t wantpid); + +extern __checkReturn efx_rc_t +efx_vpd_hunk_get( + __in_bcount(size) caddr_t data, + __in size_t size, + __in efx_vpd_tag_t tag, + __in efx_vpd_keyword_t keyword, + __out unsigned int *payloadp, + __out uint8_t *paylenp); + +extern __checkReturn efx_rc_t +efx_vpd_hunk_next( + __in_bcount(size) caddr_t data, + __in size_t size, + __out efx_vpd_tag_t *tagp, + __out efx_vpd_keyword_t *keyword, + __out_opt unsigned int *payloadp, + __out_opt uint8_t *paylenp, + __inout unsigned int *contp); + +extern __checkReturn efx_rc_t +efx_vpd_hunk_set( + __in_bcount(size) caddr_t data, + __in size_t size, + __in efx_vpd_value_t *evvp); + +#endif /* EFSYS_OPT_VPD */ + +#if EFSYS_OPT_DIAG + +extern efx_sram_pattern_fn_t __efx_sram_pattern_fns[]; + +typedef struct efx_register_set_s { + unsigned int address; + unsigned int step; + unsigned int rows; + efx_oword_t mask; +} efx_register_set_t; + +extern __checkReturn efx_rc_t +efx_nic_test_registers( + __in efx_nic_t *enp, + __in efx_register_set_t *rsp, + __in size_t count); + +extern __checkReturn efx_rc_t +efx_nic_test_tables( + __in efx_nic_t *enp, + __in efx_register_set_t *rsp, + __in efx_pattern_type_t pattern, + __in size_t count); + +#endif /* EFSYS_OPT_DIAG */ + +#if EFSYS_OPT_MCDI + +extern __checkReturn efx_rc_t +efx_mcdi_set_workaround( + __in efx_nic_t *enp, + __in uint32_t type, + __in boolean_t enabled, + __out_opt uint32_t *flagsp); + +extern __checkReturn efx_rc_t +efx_mcdi_get_workarounds( + __in efx_nic_t *enp, + __out_opt uint32_t *implementedp, + __out_opt uint32_t *enabledp); + +#endif /* EFSYS_OPT_MCDI */ + +#if EFSYS_OPT_MAC_STATS + +/* + * Closed range of stats (i.e. the first and the last are included). + * The last must be greater or equal (if the range is one item only) to + * the first. + */ +struct efx_mac_stats_range { + efx_mac_stat_t first; + efx_mac_stat_t last; +}; + +extern efx_rc_t +efx_mac_stats_mask_add_ranges( + __inout_bcount(mask_size) uint32_t *maskp, + __in size_t mask_size, + __in_ecount(rng_count) const struct efx_mac_stats_range *rngp, + __in unsigned int rng_count); + +#endif /* EFSYS_OPT_MAC_STATS */ + +#ifdef __cplusplus +} +#endif + +#endif /* _SYS_EFX_IMPL_H */ diff --git a/dpdk/drivers/net/sfc/base/efx_intr.c b/dpdk/drivers/net/sfc/base/efx_intr.c new file mode 100644 index 00000000..f0422d53 --- /dev/null +++ b/dpdk/drivers/net/sfc/base/efx_intr.c @@ -0,0 +1,572 @@ +/* + * Copyright (c) 2007-2016 Solarflare Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are + * those of the authors and should not be interpreted as representing official + * policies, either expressed or implied, of the FreeBSD Project. + */ + +#include "efx.h" +#include "efx_impl.h" + + +#if EFSYS_OPT_SIENA + +static __checkReturn efx_rc_t +siena_intr_init( + __in efx_nic_t *enp, + __in efx_intr_type_t type, + __in efsys_mem_t *esmp); + +static void +siena_intr_enable( + __in efx_nic_t *enp); + +static void +siena_intr_disable( + __in efx_nic_t *enp); + +static void +siena_intr_disable_unlocked( + __in efx_nic_t *enp); + +static __checkReturn efx_rc_t +siena_intr_trigger( + __in efx_nic_t *enp, + __in unsigned int level); + +static void +siena_intr_fini( + __in efx_nic_t *enp); + +static void +siena_intr_status_line( + __in efx_nic_t *enp, + __out boolean_t *fatalp, + __out uint32_t *qmaskp); + +static void +siena_intr_status_message( + __in efx_nic_t *enp, + __in unsigned int message, + __out boolean_t *fatalp); + +static void +siena_intr_fatal( + __in efx_nic_t *enp); + +static __checkReturn boolean_t +siena_intr_check_fatal( + __in efx_nic_t *enp); + + +#endif /* EFSYS_OPT_SIENA */ + + +#if EFSYS_OPT_SIENA +static const efx_intr_ops_t __efx_intr_siena_ops = { + siena_intr_init, /* eio_init */ + siena_intr_enable, /* eio_enable */ + siena_intr_disable, /* eio_disable */ + siena_intr_disable_unlocked, /* eio_disable_unlocked */ + siena_intr_trigger, /* eio_trigger */ + siena_intr_status_line, /* eio_status_line */ + siena_intr_status_message, /* eio_status_message */ + siena_intr_fatal, /* eio_fatal */ + siena_intr_fini, /* eio_fini */ +}; +#endif /* EFSYS_OPT_SIENA */ + +#if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD +static const efx_intr_ops_t __efx_intr_ef10_ops = { + ef10_intr_init, /* eio_init */ + ef10_intr_enable, /* eio_enable */ + ef10_intr_disable, /* eio_disable */ + ef10_intr_disable_unlocked, /* eio_disable_unlocked */ + ef10_intr_trigger, /* eio_trigger */ + ef10_intr_status_line, /* eio_status_line */ + ef10_intr_status_message, /* eio_status_message */ + ef10_intr_fatal, /* eio_fatal */ + ef10_intr_fini, /* eio_fini */ +}; +#endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */ + + __checkReturn efx_rc_t +efx_intr_init( + __in efx_nic_t *enp, + __in efx_intr_type_t type, + __in efsys_mem_t *esmp) +{ + efx_intr_t *eip = &(enp->en_intr); + const efx_intr_ops_t *eiop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); + + if (enp->en_mod_flags & EFX_MOD_INTR) { + rc = EINVAL; + goto fail1; + } + + eip->ei_esmp = esmp; + eip->ei_type = type; + eip->ei_level = 0; + + enp->en_mod_flags |= EFX_MOD_INTR; + + switch (enp->en_family) { +#if EFSYS_OPT_SIENA + case EFX_FAMILY_SIENA: + eiop = &__efx_intr_siena_ops; + break; +#endif /* EFSYS_OPT_SIENA */ + +#if EFSYS_OPT_HUNTINGTON + case EFX_FAMILY_HUNTINGTON: + eiop = &__efx_intr_ef10_ops; + break; +#endif /* EFSYS_OPT_HUNTINGTON */ + +#if EFSYS_OPT_MEDFORD + case EFX_FAMILY_MEDFORD: + eiop = &__efx_intr_ef10_ops; + break; +#endif /* EFSYS_OPT_MEDFORD */ + + default: + EFSYS_ASSERT(B_FALSE); + rc = ENOTSUP; + goto fail2; + } + + if ((rc = eiop->eio_init(enp, type, esmp)) != 0) + goto fail3; + + eip->ei_eiop = eiop; + + return (0); + +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + void +efx_intr_fini( + __in efx_nic_t *enp) +{ + efx_intr_t *eip = &(enp->en_intr); + const efx_intr_ops_t *eiop = eip->ei_eiop; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR); + + eiop->eio_fini(enp); + + enp->en_mod_flags &= ~EFX_MOD_INTR; +} + + void +efx_intr_enable( + __in efx_nic_t *enp) +{ + efx_intr_t *eip = &(enp->en_intr); + const efx_intr_ops_t *eiop = eip->ei_eiop; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR); + + eiop->eio_enable(enp); +} + + void +efx_intr_disable( + __in efx_nic_t *enp) +{ + efx_intr_t *eip = &(enp->en_intr); + const efx_intr_ops_t *eiop = eip->ei_eiop; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR); + + eiop->eio_disable(enp); +} + + void +efx_intr_disable_unlocked( + __in efx_nic_t *enp) +{ + efx_intr_t *eip = &(enp->en_intr); + const efx_intr_ops_t *eiop = eip->ei_eiop; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR); + + eiop->eio_disable_unlocked(enp); +} + + + __checkReturn efx_rc_t +efx_intr_trigger( + __in efx_nic_t *enp, + __in unsigned int level) +{ + efx_intr_t *eip = &(enp->en_intr); + const efx_intr_ops_t *eiop = eip->ei_eiop; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR); + + return (eiop->eio_trigger(enp, level)); +} + + void +efx_intr_status_line( + __in efx_nic_t *enp, + __out boolean_t *fatalp, + __out uint32_t *qmaskp) +{ + efx_intr_t *eip = &(enp->en_intr); + const efx_intr_ops_t *eiop = eip->ei_eiop; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR); + + eiop->eio_status_line(enp, fatalp, qmaskp); +} + + void +efx_intr_status_message( + __in efx_nic_t *enp, + __in unsigned int message, + __out boolean_t *fatalp) +{ + efx_intr_t *eip = &(enp->en_intr); + const efx_intr_ops_t *eiop = eip->ei_eiop; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR); + + eiop->eio_status_message(enp, message, fatalp); +} + + void +efx_intr_fatal( + __in efx_nic_t *enp) +{ + efx_intr_t *eip = &(enp->en_intr); + const efx_intr_ops_t *eiop = eip->ei_eiop; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR); + + eiop->eio_fatal(enp); +} + + +/* ************************************************************************* */ +/* ************************************************************************* */ +/* ************************************************************************* */ + +#if EFSYS_OPT_SIENA + +static __checkReturn efx_rc_t +siena_intr_init( + __in efx_nic_t *enp, + __in efx_intr_type_t type, + __in efsys_mem_t *esmp) +{ + efx_intr_t *eip = &(enp->en_intr); + efx_oword_t oword; + + /* + * bug17213 workaround. + * + * Under legacy interrupts, don't share a level between fatal + * interrupts and event queue interrupts. Under MSI-X, they + * must share, or we won't get an interrupt. + */ + if (enp->en_family == EFX_FAMILY_SIENA && + eip->ei_type == EFX_INTR_LINE) + eip->ei_level = 0x1f; + else + eip->ei_level = 0; + + /* Enable all the genuinely fatal interrupts */ + EFX_SET_OWORD(oword); + EFX_SET_OWORD_FIELD(oword, FRF_AZ_ILL_ADR_INT_KER_EN, 0); + EFX_SET_OWORD_FIELD(oword, FRF_AZ_RBUF_OWN_INT_KER_EN, 0); + EFX_SET_OWORD_FIELD(oword, FRF_AZ_TBUF_OWN_INT_KER_EN, 0); + if (enp->en_family >= EFX_FAMILY_SIENA) + EFX_SET_OWORD_FIELD(oword, FRF_CZ_SRAM_PERR_INT_P_KER_EN, 0); + EFX_BAR_WRITEO(enp, FR_AZ_FATAL_INTR_REG_KER, &oword); + + /* Set up the interrupt address register */ + EFX_POPULATE_OWORD_3(oword, + FRF_AZ_NORM_INT_VEC_DIS_KER, (type == EFX_INTR_MESSAGE) ? 1 : 0, + FRF_AZ_INT_ADR_KER_DW0, EFSYS_MEM_ADDR(esmp) & 0xffffffff, + FRF_AZ_INT_ADR_KER_DW1, EFSYS_MEM_ADDR(esmp) >> 32); + EFX_BAR_WRITEO(enp, FR_AZ_INT_ADR_REG_KER, &oword); + + return (0); +} + +static void +siena_intr_enable( + __in efx_nic_t *enp) +{ + efx_intr_t *eip = &(enp->en_intr); + efx_oword_t oword; + + EFX_BAR_READO(enp, FR_AZ_INT_EN_REG_KER, &oword); + + EFX_SET_OWORD_FIELD(oword, FRF_AZ_KER_INT_LEVE_SEL, eip->ei_level); + EFX_SET_OWORD_FIELD(oword, FRF_AZ_DRV_INT_EN_KER, 1); + EFX_BAR_WRITEO(enp, FR_AZ_INT_EN_REG_KER, &oword); +} + +static void +siena_intr_disable( + __in efx_nic_t *enp) +{ + efx_oword_t oword; + + EFX_BAR_READO(enp, FR_AZ_INT_EN_REG_KER, &oword); + EFX_SET_OWORD_FIELD(oword, FRF_AZ_DRV_INT_EN_KER, 0); + EFX_BAR_WRITEO(enp, FR_AZ_INT_EN_REG_KER, &oword); + + EFSYS_SPIN(10); +} + +static void +siena_intr_disable_unlocked( + __in efx_nic_t *enp) +{ + efx_oword_t oword; + + EFSYS_BAR_READO(enp->en_esbp, FR_AZ_INT_EN_REG_KER_OFST, + &oword, B_FALSE); + EFX_SET_OWORD_FIELD(oword, FRF_AZ_DRV_INT_EN_KER, 0); + EFSYS_BAR_WRITEO(enp->en_esbp, FR_AZ_INT_EN_REG_KER_OFST, + &oword, B_FALSE); +} + +static __checkReturn efx_rc_t +siena_intr_trigger( + __in efx_nic_t *enp, + __in unsigned int level) +{ + efx_intr_t *eip = &(enp->en_intr); + efx_oword_t oword; + unsigned int count; + uint32_t sel; + efx_rc_t rc; + + /* bug16757: No event queues can be initialized */ + EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_EV)); + + if (level >= EFX_NINTR_SIENA) { + rc = EINVAL; + goto fail1; + } + + if (level > EFX_MASK32(FRF_AZ_KER_INT_LEVE_SEL)) + return (ENOTSUP); /* avoid EFSYS_PROBE() */ + + sel = level; + + /* Trigger a test interrupt */ + EFX_BAR_READO(enp, FR_AZ_INT_EN_REG_KER, &oword); + EFX_SET_OWORD_FIELD(oword, FRF_AZ_KER_INT_LEVE_SEL, sel); + EFX_SET_OWORD_FIELD(oword, FRF_AZ_KER_INT_KER, 1); + EFX_BAR_WRITEO(enp, FR_AZ_INT_EN_REG_KER, &oword); + + /* + * Wait up to 100ms for the interrupt to be raised before restoring + * KER_INT_LEVE_SEL. Ignore a failure to raise (the caller will + * observe this soon enough anyway), but always reset KER_INT_LEVE_SEL + */ + count = 0; + do { + EFSYS_SPIN(100); /* 100us */ + + EFX_BAR_READO(enp, FR_AZ_INT_EN_REG_KER, &oword); + } while (EFX_OWORD_FIELD(oword, FRF_AZ_KER_INT_KER) && ++count < 1000); + + EFX_SET_OWORD_FIELD(oword, FRF_AZ_KER_INT_LEVE_SEL, eip->ei_level); + EFX_BAR_WRITEO(enp, FR_AZ_INT_EN_REG_KER, &oword); + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +static __checkReturn boolean_t +siena_intr_check_fatal( + __in efx_nic_t *enp) +{ + efx_intr_t *eip = &(enp->en_intr); + efsys_mem_t *esmp = eip->ei_esmp; + efx_oword_t oword; + + /* Read the syndrome */ + EFSYS_MEM_READO(esmp, 0, &oword); + + if (EFX_OWORD_FIELD(oword, FSF_AZ_NET_IVEC_FATAL_INT) != 0) { + EFSYS_PROBE(fatal); + + /* Clear the fatal interrupt condition */ + EFX_SET_OWORD_FIELD(oword, FSF_AZ_NET_IVEC_FATAL_INT, 0); + EFSYS_MEM_WRITEO(esmp, 0, &oword); + + return (B_TRUE); + } + + return (B_FALSE); +} + +static void +siena_intr_status_line( + __in efx_nic_t *enp, + __out boolean_t *fatalp, + __out uint32_t *qmaskp) +{ + efx_intr_t *eip = &(enp->en_intr); + efx_dword_t dword; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR); + + /* + * Read the queue mask and implicitly acknowledge the + * interrupt. + */ + EFX_BAR_READD(enp, FR_BZ_INT_ISR0_REG, &dword, B_FALSE); + *qmaskp = EFX_DWORD_FIELD(dword, EFX_DWORD_0); + + EFSYS_PROBE1(qmask, uint32_t, *qmaskp); + + if (*qmaskp & (1U << eip->ei_level)) + *fatalp = siena_intr_check_fatal(enp); + else + *fatalp = B_FALSE; +} + +static void +siena_intr_status_message( + __in efx_nic_t *enp, + __in unsigned int message, + __out boolean_t *fatalp) +{ + efx_intr_t *eip = &(enp->en_intr); + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR); + + if (message == eip->ei_level) + *fatalp = siena_intr_check_fatal(enp); + else + *fatalp = B_FALSE; +} + + +static void +siena_intr_fatal( + __in efx_nic_t *enp) +{ +#if EFSYS_OPT_DECODE_INTR_FATAL + efx_oword_t fatal; + efx_oword_t mem_per; + + EFX_BAR_READO(enp, FR_AZ_FATAL_INTR_REG_KER, &fatal); + EFX_ZERO_OWORD(mem_per); + + if (EFX_OWORD_FIELD(fatal, FRF_AZ_SRM_PERR_INT_KER) != 0 || + EFX_OWORD_FIELD(fatal, FRF_AZ_MEM_PERR_INT_KER) != 0) + EFX_BAR_READO(enp, FR_AZ_MEM_STAT_REG, &mem_per); + + if (EFX_OWORD_FIELD(fatal, FRF_AZ_SRAM_OOB_INT_KER) != 0) + EFSYS_ERR(enp->en_esip, EFX_ERR_SRAM_OOB, 0, 0); + + if (EFX_OWORD_FIELD(fatal, FRF_AZ_BUFID_DC_OOB_INT_KER) != 0) + EFSYS_ERR(enp->en_esip, EFX_ERR_BUFID_DC_OOB, 0, 0); + + if (EFX_OWORD_FIELD(fatal, FRF_AZ_MEM_PERR_INT_KER) != 0) + EFSYS_ERR(enp->en_esip, EFX_ERR_MEM_PERR, + EFX_OWORD_FIELD(mem_per, EFX_DWORD_0), + EFX_OWORD_FIELD(mem_per, EFX_DWORD_1)); + + if (EFX_OWORD_FIELD(fatal, FRF_AZ_RBUF_OWN_INT_KER) != 0) + EFSYS_ERR(enp->en_esip, EFX_ERR_RBUF_OWN, 0, 0); + + if (EFX_OWORD_FIELD(fatal, FRF_AZ_TBUF_OWN_INT_KER) != 0) + EFSYS_ERR(enp->en_esip, EFX_ERR_TBUF_OWN, 0, 0); + + if (EFX_OWORD_FIELD(fatal, FRF_AZ_RDESCQ_OWN_INT_KER) != 0) + EFSYS_ERR(enp->en_esip, EFX_ERR_RDESQ_OWN, 0, 0); + + if (EFX_OWORD_FIELD(fatal, FRF_AZ_TDESCQ_OWN_INT_KER) != 0) + EFSYS_ERR(enp->en_esip, EFX_ERR_TDESQ_OWN, 0, 0); + + if (EFX_OWORD_FIELD(fatal, FRF_AZ_EVQ_OWN_INT_KER) != 0) + EFSYS_ERR(enp->en_esip, EFX_ERR_EVQ_OWN, 0, 0); + + if (EFX_OWORD_FIELD(fatal, FRF_AZ_EVF_OFLO_INT_KER) != 0) + EFSYS_ERR(enp->en_esip, EFX_ERR_EVFF_OFLO, 0, 0); + + if (EFX_OWORD_FIELD(fatal, FRF_AZ_ILL_ADR_INT_KER) != 0) + EFSYS_ERR(enp->en_esip, EFX_ERR_ILL_ADDR, 0, 0); + + if (EFX_OWORD_FIELD(fatal, FRF_AZ_SRM_PERR_INT_KER) != 0) + EFSYS_ERR(enp->en_esip, EFX_ERR_SRAM_PERR, + EFX_OWORD_FIELD(mem_per, EFX_DWORD_0), + EFX_OWORD_FIELD(mem_per, EFX_DWORD_1)); +#else + EFSYS_ASSERT(0); +#endif +} + +static void +siena_intr_fini( + __in efx_nic_t *enp) +{ + efx_oword_t oword; + + /* Clear the interrupt address register */ + EFX_ZERO_OWORD(oword); + EFX_BAR_WRITEO(enp, FR_AZ_INT_ADR_REG_KER, &oword); +} + +#endif /* EFSYS_OPT_SIENA */ diff --git a/dpdk/drivers/net/sfc/base/efx_lic.c b/dpdk/drivers/net/sfc/base/efx_lic.c new file mode 100644 index 00000000..2cd05cc8 --- /dev/null +++ b/dpdk/drivers/net/sfc/base/efx_lic.c @@ -0,0 +1,1751 @@ +/* + * Copyright (c) 2009-2016 Solarflare Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are + * those of the authors and should not be interpreted as representing official + * policies, either expressed or implied, of the FreeBSD Project. + */ + +#include "efx.h" +#include "efx_impl.h" + +#if EFSYS_OPT_LICENSING + +#include "ef10_tlv_layout.h" + +#if EFSYS_OPT_SIENA | EFSYS_OPT_HUNTINGTON + + __checkReturn efx_rc_t +efx_lic_v1v2_find_start( + __in efx_nic_t *enp, + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size, + __out uint32_t *startp + ); + + __checkReturn efx_rc_t +efx_lic_v1v2_find_end( + __in efx_nic_t *enp, + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size, + __in uint32_t offset, + __out uint32_t *endp + ); + + __checkReturn __success(return != B_FALSE) boolean_t +efx_lic_v1v2_find_key( + __in efx_nic_t *enp, + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size, + __in uint32_t offset, + __out uint32_t *startp, + __out uint32_t *lengthp + ); + + __checkReturn __success(return != B_FALSE) boolean_t +efx_lic_v1v2_validate_key( + __in efx_nic_t *enp, + __in_bcount(length) caddr_t keyp, + __in uint32_t length + ); + + __checkReturn efx_rc_t +efx_lic_v1v2_read_key( + __in efx_nic_t *enp, + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size, + __in uint32_t offset, + __in uint32_t length, + __out_bcount_part(key_max_size, *lengthp) + caddr_t keyp, + __in size_t key_max_size, + __out uint32_t *lengthp + ); + + __checkReturn efx_rc_t +efx_lic_v1v2_write_key( + __in efx_nic_t *enp, + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size, + __in uint32_t offset, + __in_bcount(length) caddr_t keyp, + __in uint32_t length, + __out uint32_t *lengthp + ); + + __checkReturn efx_rc_t +efx_lic_v1v2_delete_key( + __in efx_nic_t *enp, + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size, + __in uint32_t offset, + __in uint32_t length, + __in uint32_t end, + __out uint32_t *deltap + ); + + __checkReturn efx_rc_t +efx_lic_v1v2_create_partition( + __in efx_nic_t *enp, + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size + ); + + __checkReturn efx_rc_t +efx_lic_v1v2_finish_partition( + __in efx_nic_t *enp, + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size + ); + +#endif /* EFSYS_OPT_HUNTINGTON | EFSYS_OPT_SIENA */ + + +#if EFSYS_OPT_SIENA + +static __checkReturn efx_rc_t +efx_mcdi_fc_license_update_license( + __in efx_nic_t *enp); + +static __checkReturn efx_rc_t +efx_mcdi_fc_license_get_key_stats( + __in efx_nic_t *enp, + __out efx_key_stats_t *eksp); + +static const efx_lic_ops_t __efx_lic_v1_ops = { + efx_mcdi_fc_license_update_license, /* elo_update_licenses */ + efx_mcdi_fc_license_get_key_stats, /* elo_get_key_stats */ + NULL, /* elo_app_state */ + NULL, /* elo_get_id */ + efx_lic_v1v2_find_start, /* elo_find_start */ + efx_lic_v1v2_find_end, /* elo_find_end */ + efx_lic_v1v2_find_key, /* elo_find_key */ + efx_lic_v1v2_validate_key, /* elo_validate_key */ + efx_lic_v1v2_read_key, /* elo_read_key */ + efx_lic_v1v2_write_key, /* elo_write_key */ + efx_lic_v1v2_delete_key, /* elo_delete_key */ + efx_lic_v1v2_create_partition, /* elo_create_partition */ + efx_lic_v1v2_finish_partition, /* elo_finish_partition */ +}; + +#endif /* EFSYS_OPT_SIENA */ + +#if EFSYS_OPT_HUNTINGTON + +static __checkReturn efx_rc_t +efx_mcdi_licensing_update_licenses( + __in efx_nic_t *enp); + +static __checkReturn efx_rc_t +efx_mcdi_licensing_get_key_stats( + __in efx_nic_t *enp, + __out efx_key_stats_t *eksp); + +static __checkReturn efx_rc_t +efx_mcdi_licensed_app_state( + __in efx_nic_t *enp, + __in uint64_t app_id, + __out boolean_t *licensedp); + +static const efx_lic_ops_t __efx_lic_v2_ops = { + efx_mcdi_licensing_update_licenses, /* elo_update_licenses */ + efx_mcdi_licensing_get_key_stats, /* elo_get_key_stats */ + efx_mcdi_licensed_app_state, /* elo_app_state */ + NULL, /* elo_get_id */ + efx_lic_v1v2_find_start, /* elo_find_start */ + efx_lic_v1v2_find_end, /* elo_find_end */ + efx_lic_v1v2_find_key, /* elo_find_key */ + efx_lic_v1v2_validate_key, /* elo_validate_key */ + efx_lic_v1v2_read_key, /* elo_read_key */ + efx_lic_v1v2_write_key, /* elo_write_key */ + efx_lic_v1v2_delete_key, /* elo_delete_key */ + efx_lic_v1v2_create_partition, /* elo_create_partition */ + efx_lic_v1v2_finish_partition, /* elo_finish_partition */ +}; + +#endif /* EFSYS_OPT_HUNTINGTON */ + +#if EFSYS_OPT_MEDFORD + +static __checkReturn efx_rc_t +efx_mcdi_licensing_v3_update_licenses( + __in efx_nic_t *enp); + +static __checkReturn efx_rc_t +efx_mcdi_licensing_v3_report_license( + __in efx_nic_t *enp, + __out efx_key_stats_t *eksp); + +static __checkReturn efx_rc_t +efx_mcdi_licensing_v3_app_state( + __in efx_nic_t *enp, + __in uint64_t app_id, + __out boolean_t *licensedp); + +static __checkReturn efx_rc_t +efx_mcdi_licensing_v3_get_id( + __in efx_nic_t *enp, + __in size_t buffer_size, + __out uint32_t *typep, + __out size_t *lengthp, + __out_bcount_part_opt(buffer_size, *lengthp) + uint8_t *bufferp); + + __checkReturn efx_rc_t +efx_lic_v3_find_start( + __in efx_nic_t *enp, + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size, + __out uint32_t *startp + ); + + __checkReturn efx_rc_t +efx_lic_v3_find_end( + __in efx_nic_t *enp, + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size, + __in uint32_t offset, + __out uint32_t *endp + ); + + __checkReturn __success(return != B_FALSE) boolean_t +efx_lic_v3_find_key( + __in efx_nic_t *enp, + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size, + __in uint32_t offset, + __out uint32_t *startp, + __out uint32_t *lengthp + ); + + __checkReturn __success(return != B_FALSE) boolean_t +efx_lic_v3_validate_key( + __in efx_nic_t *enp, + __in_bcount(length) caddr_t keyp, + __in uint32_t length + ); + + __checkReturn efx_rc_t +efx_lic_v3_read_key( + __in efx_nic_t *enp, + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size, + __in uint32_t offset, + __in uint32_t length, + __out_bcount_part(key_max_size, *lengthp) + caddr_t keyp, + __in size_t key_max_size, + __out uint32_t *lengthp + ); + + __checkReturn efx_rc_t +efx_lic_v3_write_key( + __in efx_nic_t *enp, + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size, + __in uint32_t offset, + __in_bcount(length) caddr_t keyp, + __in uint32_t length, + __out uint32_t *lengthp + ); + + __checkReturn efx_rc_t +efx_lic_v3_delete_key( + __in efx_nic_t *enp, + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size, + __in uint32_t offset, + __in uint32_t length, + __in uint32_t end, + __out uint32_t *deltap + ); + + __checkReturn efx_rc_t +efx_lic_v3_create_partition( + __in efx_nic_t *enp, + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size + ); + + __checkReturn efx_rc_t +efx_lic_v3_finish_partition( + __in efx_nic_t *enp, + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size + ); + +static const efx_lic_ops_t __efx_lic_v3_ops = { + efx_mcdi_licensing_v3_update_licenses, /* elo_update_licenses */ + efx_mcdi_licensing_v3_report_license, /* elo_get_key_stats */ + efx_mcdi_licensing_v3_app_state, /* elo_app_state */ + efx_mcdi_licensing_v3_get_id, /* elo_get_id */ + efx_lic_v3_find_start, /* elo_find_start*/ + efx_lic_v3_find_end, /* elo_find_end */ + efx_lic_v3_find_key, /* elo_find_key */ + efx_lic_v3_validate_key, /* elo_validate_key */ + efx_lic_v3_read_key, /* elo_read_key */ + efx_lic_v3_write_key, /* elo_write_key */ + efx_lic_v3_delete_key, /* elo_delete_key */ + efx_lic_v3_create_partition, /* elo_create_partition */ + efx_lic_v3_finish_partition, /* elo_finish_partition */ +}; + +#endif /* EFSYS_OPT_MEDFORD */ + + +/* V1 Licensing - used in Siena Modena only */ + +#if EFSYS_OPT_SIENA + +static __checkReturn efx_rc_t +efx_mcdi_fc_license_update_license( + __in efx_nic_t *enp) +{ + efx_mcdi_req_t req; + uint8_t payload[MC_CMD_FC_IN_LICENSE_LEN]; + efx_rc_t rc; + + EFSYS_ASSERT(enp->en_family == EFX_FAMILY_SIENA); + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_FC; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_FC_IN_LICENSE_LEN; + req.emr_out_buf = payload; + req.emr_out_length = 0; + + MCDI_IN_SET_DWORD(req, FC_IN_CMD, + MC_CMD_FC_OP_LICENSE); + + MCDI_IN_SET_DWORD(req, FC_IN_LICENSE_OP, + MC_CMD_FC_IN_LICENSE_UPDATE_LICENSE); + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + if (req.emr_out_length_used != 0) { + rc = EIO; + goto fail2; + } + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +static __checkReturn efx_rc_t +efx_mcdi_fc_license_get_key_stats( + __in efx_nic_t *enp, + __out efx_key_stats_t *eksp) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_FC_IN_LICENSE_LEN, + MC_CMD_FC_OUT_LICENSE_LEN)]; + efx_rc_t rc; + + EFSYS_ASSERT(enp->en_family == EFX_FAMILY_SIENA); + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_FC; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_FC_IN_LICENSE_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_FC_OUT_LICENSE_LEN; + + MCDI_IN_SET_DWORD(req, FC_IN_CMD, + MC_CMD_FC_OP_LICENSE); + + MCDI_IN_SET_DWORD(req, FC_IN_LICENSE_OP, + MC_CMD_FC_IN_LICENSE_GET_KEY_STATS); + + efx_mcdi_execute_quiet(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + if (req.emr_out_length_used < MC_CMD_FC_OUT_LICENSE_LEN) { + rc = EMSGSIZE; + goto fail2; + } + + eksp->eks_valid = + MCDI_OUT_DWORD(req, FC_OUT_LICENSE_VALID_KEYS); + eksp->eks_invalid = + MCDI_OUT_DWORD(req, FC_OUT_LICENSE_INVALID_KEYS); + eksp->eks_blacklisted = + MCDI_OUT_DWORD(req, FC_OUT_LICENSE_BLACKLISTED_KEYS); + eksp->eks_unverifiable = 0; + eksp->eks_wrong_node = 0; + eksp->eks_licensed_apps_lo = 0; + eksp->eks_licensed_apps_hi = 0; + eksp->eks_licensed_features_lo = 0; + eksp->eks_licensed_features_hi = 0; + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +#endif /* EFSYS_OPT_SIENA */ + +/* V1 and V2 Partition format - based on a 16-bit TLV format */ + +#if EFSYS_OPT_SIENA | EFSYS_OPT_HUNTINGTON + +/* + * V1/V2 format - defined in SF-108542-TC section 4.2: + * Type (T): 16bit - revision/HMAC algorithm + * Length (L): 16bit - value length in bytes + * Value (V): L bytes - payload + */ +#define EFX_LICENSE_V1V2_PAYLOAD_LENGTH_MAX (256) +#define EFX_LICENSE_V1V2_HEADER_LENGTH (2 * sizeof(uint16_t)) + + __checkReturn efx_rc_t +efx_lic_v1v2_find_start( + __in efx_nic_t *enp, + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size, + __out uint32_t *startp + ) +{ + _NOTE(ARGUNUSED(enp, bufferp, buffer_size)) + + *startp = 0; + return (0); +} + + __checkReturn efx_rc_t +efx_lic_v1v2_find_end( + __in efx_nic_t *enp, + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size, + __in uint32_t offset, + __out uint32_t *endp + ) +{ + _NOTE(ARGUNUSED(enp, bufferp, buffer_size)) + + *endp = offset + EFX_LICENSE_V1V2_HEADER_LENGTH; + return (0); +} + + __checkReturn __success(return != B_FALSE) boolean_t +efx_lic_v1v2_find_key( + __in efx_nic_t *enp, + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size, + __in uint32_t offset, + __out uint32_t *startp, + __out uint32_t *lengthp + ) +{ + boolean_t found; + uint16_t tlv_type; + uint16_t tlv_length; + + _NOTE(ARGUNUSED(enp)) + + if ((size_t)buffer_size - offset < EFX_LICENSE_V1V2_HEADER_LENGTH) + goto fail1; + + tlv_type = __LE_TO_CPU_16(((uint16_t *)&bufferp[offset])[0]); + tlv_length = __LE_TO_CPU_16(((uint16_t *)&bufferp[offset])[1]); + if ((tlv_length > EFX_LICENSE_V1V2_PAYLOAD_LENGTH_MAX) || + (tlv_type == 0 && tlv_length == 0)) { + found = B_FALSE; + } else { + *startp = offset; + *lengthp = tlv_length + EFX_LICENSE_V1V2_HEADER_LENGTH; + found = B_TRUE; + } + return (found); + +fail1: + EFSYS_PROBE(fail1); + + return (B_FALSE); +} + + __checkReturn __success(return != B_FALSE) boolean_t +efx_lic_v1v2_validate_key( + __in efx_nic_t *enp, + __in_bcount(length) caddr_t keyp, + __in uint32_t length + ) +{ + uint16_t tlv_type; + uint16_t tlv_length; + + _NOTE(ARGUNUSED(enp)) + + if (length < EFX_LICENSE_V1V2_HEADER_LENGTH) { + goto fail1; + } + + tlv_type = __LE_TO_CPU_16(((uint16_t *)keyp)[0]); + tlv_length = __LE_TO_CPU_16(((uint16_t *)keyp)[1]); + + if (tlv_length > EFX_LICENSE_V1V2_PAYLOAD_LENGTH_MAX) { + goto fail2; + } + if (tlv_type == 0) { + goto fail3; + } + if ((tlv_length + EFX_LICENSE_V1V2_HEADER_LENGTH) != length) { + goto fail4; + } + + return (B_TRUE); + +fail4: + EFSYS_PROBE(fail4); +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE(fail1); + + return (B_FALSE); +} + + + __checkReturn efx_rc_t +efx_lic_v1v2_read_key( + __in efx_nic_t *enp, + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size, + __in uint32_t offset, + __in uint32_t length, + __out_bcount_part(key_max_size, *lengthp) + caddr_t keyp, + __in size_t key_max_size, + __out uint32_t *lengthp + ) +{ + efx_rc_t rc; + + _NOTE(ARGUNUSED(enp)) + EFSYS_ASSERT(length <= (EFX_LICENSE_V1V2_PAYLOAD_LENGTH_MAX + + EFX_LICENSE_V1V2_HEADER_LENGTH)); + + if (key_max_size < length) { + rc = ENOSPC; + goto fail1; + } + memcpy(keyp, &bufferp[offset], length); + + *lengthp = length; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_lic_v1v2_write_key( + __in efx_nic_t *enp, + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size, + __in uint32_t offset, + __in_bcount(length) caddr_t keyp, + __in uint32_t length, + __out uint32_t *lengthp + ) +{ + efx_rc_t rc; + + _NOTE(ARGUNUSED(enp)) + EFSYS_ASSERT(length <= (EFX_LICENSE_V1V2_PAYLOAD_LENGTH_MAX + + EFX_LICENSE_V1V2_HEADER_LENGTH)); + + /* Ensure space for terminator remains */ + if ((offset + length) > + (buffer_size - EFX_LICENSE_V1V2_HEADER_LENGTH)) { + rc = ENOSPC; + goto fail1; + } + + memcpy(bufferp + offset, keyp, length); + + *lengthp = length; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_lic_v1v2_delete_key( + __in efx_nic_t *enp, + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size, + __in uint32_t offset, + __in uint32_t length, + __in uint32_t end, + __out uint32_t *deltap + ) +{ + uint32_t move_start = offset + length; + uint32_t move_length = end - move_start; + + _NOTE(ARGUNUSED(enp)) + EFSYS_ASSERT(end <= buffer_size); + + /* Shift everything after the key down */ + memmove(bufferp + offset, bufferp + move_start, move_length); + + *deltap = length; + + return (0); +} + + __checkReturn efx_rc_t +efx_lic_v1v2_create_partition( + __in efx_nic_t *enp, + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size + ) +{ + _NOTE(ARGUNUSED(enp)) + EFSYS_ASSERT(EFX_LICENSE_V1V2_HEADER_LENGTH <= buffer_size); + + /* Write terminator */ + memset(bufferp, '\0', EFX_LICENSE_V1V2_HEADER_LENGTH); + return (0); +} + + + __checkReturn efx_rc_t +efx_lic_v1v2_finish_partition( + __in efx_nic_t *enp, + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size + ) +{ + _NOTE(ARGUNUSED(enp, bufferp, buffer_size)) + + return (0); +} + +#endif /* EFSYS_OPT_HUNTINGTON | EFSYS_OPT_SIENA */ + + +/* V2 Licensing - used by Huntington family only. See SF-113611-TC */ + +#if EFSYS_OPT_HUNTINGTON + +static __checkReturn efx_rc_t +efx_mcdi_licensed_app_state( + __in efx_nic_t *enp, + __in uint64_t app_id, + __out boolean_t *licensedp) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_GET_LICENSED_APP_STATE_IN_LEN, + MC_CMD_GET_LICENSED_APP_STATE_OUT_LEN)]; + uint32_t app_state; + efx_rc_t rc; + + EFSYS_ASSERT(enp->en_family == EFX_FAMILY_HUNTINGTON); + + /* V2 licensing supports 32bit app id only */ + if ((app_id >> 32) != 0) { + rc = EINVAL; + goto fail1; + } + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_GET_LICENSED_APP_STATE; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_GET_LICENSED_APP_STATE_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_GET_LICENSED_APP_STATE_OUT_LEN; + + MCDI_IN_SET_DWORD(req, GET_LICENSED_APP_STATE_IN_APP_ID, + app_id & 0xffffffff); + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail2; + } + + if (req.emr_out_length_used < MC_CMD_GET_LICENSED_APP_STATE_OUT_LEN) { + rc = EMSGSIZE; + goto fail3; + } + + app_state = (MCDI_OUT_DWORD(req, GET_LICENSED_APP_STATE_OUT_STATE)); + if (app_state != MC_CMD_GET_LICENSED_APP_STATE_OUT_NOT_LICENSED) { + *licensedp = B_TRUE; + } else { + *licensedp = B_FALSE; + } + + return (0); + +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +static __checkReturn efx_rc_t +efx_mcdi_licensing_update_licenses( + __in efx_nic_t *enp) +{ + efx_mcdi_req_t req; + uint8_t payload[MC_CMD_LICENSING_IN_LEN]; + efx_rc_t rc; + + EFSYS_ASSERT(enp->en_family == EFX_FAMILY_HUNTINGTON); + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_LICENSING; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_LICENSING_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = 0; + + MCDI_IN_SET_DWORD(req, LICENSING_IN_OP, + MC_CMD_LICENSING_IN_OP_UPDATE_LICENSE); + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + if (req.emr_out_length_used != 0) { + rc = EIO; + goto fail2; + } + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +static __checkReturn efx_rc_t +efx_mcdi_licensing_get_key_stats( + __in efx_nic_t *enp, + __out efx_key_stats_t *eksp) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_LICENSING_IN_LEN, + MC_CMD_LICENSING_OUT_LEN)]; + efx_rc_t rc; + + EFSYS_ASSERT(enp->en_family == EFX_FAMILY_HUNTINGTON); + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_LICENSING; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_LICENSING_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_LICENSING_OUT_LEN; + + MCDI_IN_SET_DWORD(req, LICENSING_IN_OP, + MC_CMD_LICENSING_IN_OP_GET_KEY_STATS); + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + if (req.emr_out_length_used < MC_CMD_LICENSING_OUT_LEN) { + rc = EMSGSIZE; + goto fail2; + } + + eksp->eks_valid = + MCDI_OUT_DWORD(req, LICENSING_OUT_VALID_APP_KEYS); + eksp->eks_invalid = + MCDI_OUT_DWORD(req, LICENSING_OUT_INVALID_APP_KEYS); + eksp->eks_blacklisted = + MCDI_OUT_DWORD(req, LICENSING_OUT_BLACKLISTED_APP_KEYS); + eksp->eks_unverifiable = + MCDI_OUT_DWORD(req, LICENSING_OUT_UNVERIFIABLE_APP_KEYS); + eksp->eks_wrong_node = + MCDI_OUT_DWORD(req, LICENSING_OUT_WRONG_NODE_APP_KEYS); + eksp->eks_licensed_apps_lo = 0; + eksp->eks_licensed_apps_hi = 0; + eksp->eks_licensed_features_lo = 0; + eksp->eks_licensed_features_hi = 0; + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +#endif /* EFSYS_OPT_HUNTINGTON */ + +/* V3 Licensing - used starting from Medford family. See SF-114884-SW */ + +#if EFSYS_OPT_MEDFORD + +static __checkReturn efx_rc_t +efx_mcdi_licensing_v3_update_licenses( + __in efx_nic_t *enp) +{ + efx_mcdi_req_t req; + uint8_t payload[MC_CMD_LICENSING_V3_IN_LEN]; + efx_rc_t rc; + + EFSYS_ASSERT(enp->en_family == EFX_FAMILY_MEDFORD); + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_LICENSING_V3; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_LICENSING_V3_IN_LEN; + req.emr_out_buf = NULL; + req.emr_out_length = 0; + + MCDI_IN_SET_DWORD(req, LICENSING_V3_IN_OP, + MC_CMD_LICENSING_V3_IN_OP_UPDATE_LICENSE); + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +static __checkReturn efx_rc_t +efx_mcdi_licensing_v3_report_license( + __in efx_nic_t *enp, + __out efx_key_stats_t *eksp) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_LICENSING_V3_IN_LEN, + MC_CMD_LICENSING_V3_OUT_LEN)]; + efx_rc_t rc; + + EFSYS_ASSERT(enp->en_family == EFX_FAMILY_MEDFORD); + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_LICENSING_V3; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_LICENSING_V3_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_LICENSING_V3_OUT_LEN; + + MCDI_IN_SET_DWORD(req, LICENSING_V3_IN_OP, + MC_CMD_LICENSING_V3_IN_OP_REPORT_LICENSE); + + efx_mcdi_execute_quiet(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + if (req.emr_out_length_used < MC_CMD_LICENSING_V3_OUT_LEN) { + rc = EMSGSIZE; + goto fail2; + } + + eksp->eks_valid = + MCDI_OUT_DWORD(req, LICENSING_V3_OUT_VALID_KEYS); + eksp->eks_invalid = + MCDI_OUT_DWORD(req, LICENSING_V3_OUT_INVALID_KEYS); + eksp->eks_blacklisted = 0; + eksp->eks_unverifiable = + MCDI_OUT_DWORD(req, LICENSING_V3_OUT_UNVERIFIABLE_KEYS); + eksp->eks_wrong_node = + MCDI_OUT_DWORD(req, LICENSING_V3_OUT_WRONG_NODE_KEYS); + eksp->eks_licensed_apps_lo = + MCDI_OUT_DWORD(req, LICENSING_V3_OUT_LICENSED_APPS_LO); + eksp->eks_licensed_apps_hi = + MCDI_OUT_DWORD(req, LICENSING_V3_OUT_LICENSED_APPS_HI); + eksp->eks_licensed_features_lo = + MCDI_OUT_DWORD(req, LICENSING_V3_OUT_LICENSED_FEATURES_LO); + eksp->eks_licensed_features_hi = + MCDI_OUT_DWORD(req, LICENSING_V3_OUT_LICENSED_FEATURES_HI); + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +static __checkReturn efx_rc_t +efx_mcdi_licensing_v3_app_state( + __in efx_nic_t *enp, + __in uint64_t app_id, + __out boolean_t *licensedp) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_GET_LICENSED_V3_APP_STATE_IN_LEN, + MC_CMD_GET_LICENSED_V3_APP_STATE_OUT_LEN)]; + uint32_t app_state; + efx_rc_t rc; + + EFSYS_ASSERT(enp->en_family == EFX_FAMILY_MEDFORD); + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_GET_LICENSED_V3_APP_STATE; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_GET_LICENSED_V3_APP_STATE_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_GET_LICENSED_V3_APP_STATE_OUT_LEN; + + MCDI_IN_SET_DWORD(req, GET_LICENSED_V3_APP_STATE_IN_APP_ID_LO, + app_id & 0xffffffff); + MCDI_IN_SET_DWORD(req, GET_LICENSED_V3_APP_STATE_IN_APP_ID_HI, + app_id >> 32); + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + if (req.emr_out_length_used < MC_CMD_GET_LICENSED_V3_APP_STATE_OUT_LEN) { + rc = EMSGSIZE; + goto fail2; + } + + app_state = (MCDI_OUT_DWORD(req, GET_LICENSED_V3_APP_STATE_OUT_STATE)); + if (app_state != MC_CMD_GET_LICENSED_V3_APP_STATE_OUT_NOT_LICENSED) { + *licensedp = B_TRUE; + } else { + *licensedp = B_FALSE; + } + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +static __checkReturn efx_rc_t +efx_mcdi_licensing_v3_get_id( + __in efx_nic_t *enp, + __in size_t buffer_size, + __out uint32_t *typep, + __out size_t *lengthp, + __out_bcount_part_opt(buffer_size, *lengthp) + uint8_t *bufferp) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_LICENSING_GET_ID_V3_IN_LEN, + MC_CMD_LICENSING_GET_ID_V3_OUT_LENMIN)]; + efx_rc_t rc; + + req.emr_cmd = MC_CMD_LICENSING_GET_ID_V3; + + if (bufferp == NULL) { + /* Request id type and length only */ + req.emr_in_buf = bufferp; + req.emr_in_length = MC_CMD_LICENSING_GET_ID_V3_IN_LEN; + req.emr_out_buf = bufferp; + req.emr_out_length = MC_CMD_LICENSING_GET_ID_V3_OUT_LENMIN; + (void) memset(payload, 0, sizeof (payload)); + } else { + /* Request full buffer */ + req.emr_in_buf = bufferp; + req.emr_in_length = MC_CMD_LICENSING_GET_ID_V3_IN_LEN; + req.emr_out_buf = bufferp; + req.emr_out_length = MIN(buffer_size, MC_CMD_LICENSING_GET_ID_V3_OUT_LENMAX); + (void) memset(bufferp, 0, req.emr_out_length); + } + + efx_mcdi_execute_quiet(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + if (req.emr_out_length_used < MC_CMD_LICENSING_GET_ID_V3_OUT_LENMIN) { + rc = EMSGSIZE; + goto fail2; + } + + *typep = MCDI_OUT_DWORD(req, LICENSING_GET_ID_V3_OUT_LICENSE_TYPE); + *lengthp = MCDI_OUT_DWORD(req, LICENSING_GET_ID_V3_OUT_LICENSE_ID_LENGTH); + + if (bufferp == NULL) { + /* modify length requirements to indicate to caller the extra buffering + ** needed to read the complete output. + */ + *lengthp += MC_CMD_LICENSING_GET_ID_V3_OUT_LENMIN; + } else { + /* Shift ID down to start of buffer */ + memmove(bufferp, + bufferp + MC_CMD_LICENSING_GET_ID_V3_OUT_LICENSE_ID_OFST, + *lengthp); + memset(bufferp + (*lengthp), 0, + MC_CMD_LICENSING_GET_ID_V3_OUT_LICENSE_ID_OFST); + } + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +/* V3 format uses Huntington TLV format partition. See SF-108797-SW */ +#define EFX_LICENSE_V3_KEY_LENGTH_MIN (64) +#define EFX_LICENSE_V3_KEY_LENGTH_MAX (160) + + __checkReturn efx_rc_t +efx_lic_v3_find_start( + __in efx_nic_t *enp, + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size, + __out uint32_t *startp + ) +{ + _NOTE(ARGUNUSED(enp)) + + return ef10_nvram_buffer_find_item_start(bufferp, buffer_size, startp); +} + + __checkReturn efx_rc_t +efx_lic_v3_find_end( + __in efx_nic_t *enp, + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size, + __in uint32_t offset, + __out uint32_t *endp + ) +{ + _NOTE(ARGUNUSED(enp)) + + return ef10_nvram_buffer_find_end(bufferp, buffer_size, offset, endp); +} + + __checkReturn __success(return != B_FALSE) boolean_t +efx_lic_v3_find_key( + __in efx_nic_t *enp, + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size, + __in uint32_t offset, + __out uint32_t *startp, + __out uint32_t *lengthp + ) +{ + _NOTE(ARGUNUSED(enp)) + + return ef10_nvram_buffer_find_item(bufferp, buffer_size, + offset, startp, lengthp); +} + + __checkReturn __success(return != B_FALSE) boolean_t +efx_lic_v3_validate_key( + __in efx_nic_t *enp, + __in_bcount(length) caddr_t keyp, + __in uint32_t length + ) +{ + /* Check key is a valid V3 key */ + uint8_t key_type; + uint8_t key_length; + + _NOTE(ARGUNUSED(enp)) + + if (length < EFX_LICENSE_V3_KEY_LENGTH_MIN) { + goto fail1; + } + + if (length > EFX_LICENSE_V3_KEY_LENGTH_MAX) { + goto fail2; + } + + key_type = ((uint8_t *)keyp)[0]; + key_length = ((uint8_t *)keyp)[1]; + + if (key_type < 3) { + goto fail3; + } + if (key_length > length) { + goto fail4; + } + return (B_TRUE); + +fail4: + EFSYS_PROBE(fail4); +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE(fail1); + + return (B_FALSE); +} + + __checkReturn efx_rc_t +efx_lic_v3_read_key( + __in efx_nic_t *enp, + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size, + __in uint32_t offset, + __in uint32_t length, + __out_bcount_part(key_max_size, *lengthp) + caddr_t keyp, + __in size_t key_max_size, + __out uint32_t *lengthp + ) +{ + _NOTE(ARGUNUSED(enp)) + + return ef10_nvram_buffer_get_item(bufferp, buffer_size, + offset, length, keyp, key_max_size, lengthp); +} + + __checkReturn efx_rc_t +efx_lic_v3_write_key( + __in efx_nic_t *enp, + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size, + __in uint32_t offset, + __in_bcount(length) caddr_t keyp, + __in uint32_t length, + __out uint32_t *lengthp + ) +{ + _NOTE(ARGUNUSED(enp)) + EFSYS_ASSERT(length <= EFX_LICENSE_V3_KEY_LENGTH_MAX); + + return ef10_nvram_buffer_insert_item(bufferp, buffer_size, + offset, keyp, length, lengthp); +} + + __checkReturn efx_rc_t +efx_lic_v3_delete_key( + __in efx_nic_t *enp, + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size, + __in uint32_t offset, + __in uint32_t length, + __in uint32_t end, + __out uint32_t *deltap + ) +{ + efx_rc_t rc; + + _NOTE(ARGUNUSED(enp)) + + if ((rc = ef10_nvram_buffer_delete_item(bufferp, + buffer_size, offset, length, end)) != 0) { + goto fail1; + } + + *deltap = length; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_lic_v3_create_partition( + __in efx_nic_t *enp, + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size + ) +{ + efx_rc_t rc; + + /* Construct empty partition */ + if ((rc = ef10_nvram_buffer_create(enp, + NVRAM_PARTITION_TYPE_LICENSE, + bufferp, buffer_size)) != 0) { + rc = EFAULT; + goto fail1; + } + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_lic_v3_finish_partition( + __in efx_nic_t *enp, + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size + ) +{ + efx_rc_t rc; + + if ((rc = ef10_nvram_buffer_finish(bufferp, + buffer_size)) != 0) { + goto fail1; + } + + /* Validate completed partition */ + if ((rc = ef10_nvram_buffer_validate(enp, NVRAM_PARTITION_TYPE_LICENSE, + bufferp, buffer_size)) != 0) { + goto fail2; + } + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + +#endif /* EFSYS_OPT_MEDFORD */ + + __checkReturn efx_rc_t +efx_lic_init( + __in efx_nic_t *enp) +{ + const efx_lic_ops_t *elop; + efx_key_stats_t eks; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE); + EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_LIC)); + + switch (enp->en_family) { + +#if EFSYS_OPT_SIENA + case EFX_FAMILY_SIENA: + elop = &__efx_lic_v1_ops; + break; +#endif /* EFSYS_OPT_SIENA */ + +#if EFSYS_OPT_HUNTINGTON + case EFX_FAMILY_HUNTINGTON: + elop = &__efx_lic_v2_ops; + break; +#endif /* EFSYS_OPT_HUNTINGTON */ + +#if EFSYS_OPT_MEDFORD + case EFX_FAMILY_MEDFORD: + elop = &__efx_lic_v3_ops; + break; +#endif /* EFSYS_OPT_MEDFORD */ + + default: + EFSYS_ASSERT(0); + rc = ENOTSUP; + goto fail1; + } + + enp->en_elop = elop; + enp->en_mod_flags |= EFX_MOD_LIC; + + /* Probe for support */ + if (efx_lic_get_key_stats(enp, &eks) == 0) { + enp->en_licensing_supported = B_TRUE; + } else { + enp->en_licensing_supported = B_FALSE; + } + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +extern __checkReturn boolean_t +efx_lic_check_support( + __in efx_nic_t *enp) +{ + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_LIC); + + return enp->en_licensing_supported; +} + + void +efx_lic_fini( + __in efx_nic_t *enp) +{ + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_LIC); + + enp->en_elop = NULL; + enp->en_mod_flags &= ~EFX_MOD_LIC; +} + + + __checkReturn efx_rc_t +efx_lic_update_licenses( + __in efx_nic_t *enp) +{ + const efx_lic_ops_t *elop = enp->en_elop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_LIC); + + if ((rc = elop->elo_update_licenses(enp)) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_lic_get_key_stats( + __in efx_nic_t *enp, + __out efx_key_stats_t *eksp) +{ + const efx_lic_ops_t *elop = enp->en_elop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_LIC); + + if ((rc = elop->elo_get_key_stats(enp, eksp)) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_lic_app_state( + __in efx_nic_t *enp, + __in uint64_t app_id, + __out boolean_t *licensedp) +{ + const efx_lic_ops_t *elop = enp->en_elop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_LIC); + + if (elop->elo_app_state == NULL) + return (ENOTSUP); + + if ((rc = elop->elo_app_state(enp, app_id, licensedp)) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_lic_get_id( + __in efx_nic_t *enp, + __in size_t buffer_size, + __out uint32_t *typep, + __out size_t *lengthp, + __out_opt uint8_t *bufferp + ) +{ + const efx_lic_ops_t *elop = enp->en_elop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_LIC); + + if (elop->elo_get_id == NULL) + return (ENOTSUP); + + if ((rc = elop->elo_get_id(enp, buffer_size, typep, + lengthp, bufferp)) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +/* Buffer management API - abstracts varying TLV format used for License partition */ + + __checkReturn efx_rc_t +efx_lic_find_start( + __in efx_nic_t *enp, + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size, + __out uint32_t *startp + ) +{ + const efx_lic_ops_t *elop = enp->en_elop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_LIC); + + if ((rc = elop->elo_find_start(enp, bufferp, buffer_size, startp)) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_lic_find_end( + __in efx_nic_t *enp, + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size, + __in uint32_t offset, + __out uint32_t *endp + ) +{ + const efx_lic_ops_t *elop = enp->en_elop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_LIC); + + if ((rc = elop->elo_find_end(enp, bufferp, buffer_size, offset, endp)) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn __success(return != B_FALSE) boolean_t +efx_lic_find_key( + __in efx_nic_t *enp, + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size, + __in uint32_t offset, + __out uint32_t *startp, + __out uint32_t *lengthp + ) +{ + const efx_lic_ops_t *elop = enp->en_elop; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_LIC); + + EFSYS_ASSERT(bufferp); + EFSYS_ASSERT(startp); + EFSYS_ASSERT(lengthp); + + return (elop->elo_find_key(enp, bufferp, buffer_size, offset, + startp, lengthp)); +} + + +/* Validate that the buffer contains a single key in a recognised format. +** An empty or terminator buffer is not accepted as a valid key. +*/ + __checkReturn __success(return != B_FALSE) boolean_t +efx_lic_validate_key( + __in efx_nic_t *enp, + __in_bcount(length) caddr_t keyp, + __in uint32_t length + ) +{ + const efx_lic_ops_t *elop = enp->en_elop; + boolean_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_LIC); + + if ((rc = elop->elo_validate_key(enp, keyp, length)) == B_FALSE) + goto fail1; + + return (B_TRUE); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_lic_read_key( + __in efx_nic_t *enp, + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size, + __in uint32_t offset, + __in uint32_t length, + __out_bcount_part(key_max_size, *lengthp) + caddr_t keyp, + __in size_t key_max_size, + __out uint32_t *lengthp + ) +{ + const efx_lic_ops_t *elop = enp->en_elop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_LIC); + + if ((rc = elop->elo_read_key(enp, bufferp, buffer_size, offset, + length, keyp, key_max_size, lengthp)) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_lic_write_key( + __in efx_nic_t *enp, + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size, + __in uint32_t offset, + __in_bcount(length) caddr_t keyp, + __in uint32_t length, + __out uint32_t *lengthp + ) +{ + const efx_lic_ops_t *elop = enp->en_elop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_LIC); + + if ((rc = elop->elo_write_key(enp, bufferp, buffer_size, offset, + keyp, length, lengthp)) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_lic_delete_key( + __in efx_nic_t *enp, + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size, + __in uint32_t offset, + __in uint32_t length, + __in uint32_t end, + __out uint32_t *deltap + ) +{ + const efx_lic_ops_t *elop = enp->en_elop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_LIC); + + if ((rc = elop->elo_delete_key(enp, bufferp, buffer_size, offset, + length, end, deltap)) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_lic_create_partition( + __in efx_nic_t *enp, + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size + ) +{ + const efx_lic_ops_t *elop = enp->en_elop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_LIC); + + if ((rc = elop->elo_create_partition(enp, bufferp, buffer_size)) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + + __checkReturn efx_rc_t +efx_lic_finish_partition( + __in efx_nic_t *enp, + __in_bcount(buffer_size) + caddr_t bufferp, + __in size_t buffer_size + ) +{ + const efx_lic_ops_t *elop = enp->en_elop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_LIC); + + if ((rc = elop->elo_finish_partition(enp, bufferp, buffer_size)) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +#endif /* EFSYS_OPT_LICENSING */ diff --git a/dpdk/drivers/net/sfc/base/efx_mac.c b/dpdk/drivers/net/sfc/base/efx_mac.c new file mode 100644 index 00000000..752e7205 --- /dev/null +++ b/dpdk/drivers/net/sfc/base/efx_mac.c @@ -0,0 +1,951 @@ +/* + * Copyright (c) 2007-2016 Solarflare Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are + * those of the authors and should not be interpreted as representing official + * policies, either expressed or implied, of the FreeBSD Project. + */ + +#include "efx.h" +#include "efx_impl.h" + +#if EFSYS_OPT_SIENA + +static __checkReturn efx_rc_t +siena_mac_multicast_list_set( + __in efx_nic_t *enp); + +#endif /* EFSYS_OPT_SIENA */ + +#if EFSYS_OPT_SIENA +static const efx_mac_ops_t __efx_siena_mac_ops = { + siena_mac_poll, /* emo_poll */ + siena_mac_up, /* emo_up */ + siena_mac_reconfigure, /* emo_addr_set */ + siena_mac_reconfigure, /* emo_pdu_set */ + siena_mac_pdu_get, /* emo_pdu_get */ + siena_mac_reconfigure, /* emo_reconfigure */ + siena_mac_multicast_list_set, /* emo_multicast_list_set */ + NULL, /* emo_filter_set_default_rxq */ + NULL, /* emo_filter_default_rxq_clear */ +#if EFSYS_OPT_LOOPBACK + siena_mac_loopback_set, /* emo_loopback_set */ +#endif /* EFSYS_OPT_LOOPBACK */ +#if EFSYS_OPT_MAC_STATS + siena_mac_stats_get_mask, /* emo_stats_get_mask */ + efx_mcdi_mac_stats_clear, /* emo_stats_clear */ + efx_mcdi_mac_stats_upload, /* emo_stats_upload */ + efx_mcdi_mac_stats_periodic, /* emo_stats_periodic */ + siena_mac_stats_update /* emo_stats_update */ +#endif /* EFSYS_OPT_MAC_STATS */ +}; +#endif /* EFSYS_OPT_SIENA */ + +#if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD +static const efx_mac_ops_t __efx_ef10_mac_ops = { + ef10_mac_poll, /* emo_poll */ + ef10_mac_up, /* emo_up */ + ef10_mac_addr_set, /* emo_addr_set */ + ef10_mac_pdu_set, /* emo_pdu_set */ + ef10_mac_pdu_get, /* emo_pdu_get */ + ef10_mac_reconfigure, /* emo_reconfigure */ + ef10_mac_multicast_list_set, /* emo_multicast_list_set */ + ef10_mac_filter_default_rxq_set, /* emo_filter_default_rxq_set */ + ef10_mac_filter_default_rxq_clear, + /* emo_filter_default_rxq_clear */ +#if EFSYS_OPT_LOOPBACK + ef10_mac_loopback_set, /* emo_loopback_set */ +#endif /* EFSYS_OPT_LOOPBACK */ +#if EFSYS_OPT_MAC_STATS + ef10_mac_stats_get_mask, /* emo_stats_get_mask */ + efx_mcdi_mac_stats_clear, /* emo_stats_clear */ + efx_mcdi_mac_stats_upload, /* emo_stats_upload */ + efx_mcdi_mac_stats_periodic, /* emo_stats_periodic */ + ef10_mac_stats_update /* emo_stats_update */ +#endif /* EFSYS_OPT_MAC_STATS */ +}; +#endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */ + + __checkReturn efx_rc_t +efx_mac_pdu_set( + __in efx_nic_t *enp, + __in size_t pdu) +{ + efx_port_t *epp = &(enp->en_port); + const efx_mac_ops_t *emop = epp->ep_emop; + uint32_t old_pdu; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT); + EFSYS_ASSERT(emop != NULL); + + if (pdu < EFX_MAC_PDU_MIN) { + rc = EINVAL; + goto fail1; + } + + if (pdu > EFX_MAC_PDU_MAX) { + rc = EINVAL; + goto fail2; + } + + old_pdu = epp->ep_mac_pdu; + epp->ep_mac_pdu = (uint32_t)pdu; + if ((rc = emop->emo_pdu_set(enp)) != 0) + goto fail3; + + return (0); + +fail3: + EFSYS_PROBE(fail3); + + epp->ep_mac_pdu = old_pdu; + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_mac_pdu_get( + __in efx_nic_t *enp, + __out size_t *pdu) +{ + efx_port_t *epp = &(enp->en_port); + const efx_mac_ops_t *emop = epp->ep_emop; + efx_rc_t rc; + + if ((rc = emop->emo_pdu_get(enp, pdu)) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_mac_addr_set( + __in efx_nic_t *enp, + __in uint8_t *addr) +{ + efx_port_t *epp = &(enp->en_port); + const efx_mac_ops_t *emop = epp->ep_emop; + uint8_t old_addr[6]; + uint32_t oui; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT); + + if (EFX_MAC_ADDR_IS_MULTICAST(addr)) { + rc = EINVAL; + goto fail1; + } + + oui = addr[0] << 16 | addr[1] << 8 | addr[2]; + if (oui == 0x000000) { + rc = EINVAL; + goto fail2; + } + + EFX_MAC_ADDR_COPY(old_addr, epp->ep_mac_addr); + EFX_MAC_ADDR_COPY(epp->ep_mac_addr, addr); + if ((rc = emop->emo_addr_set(enp)) != 0) + goto fail3; + + return (0); + +fail3: + EFSYS_PROBE(fail3); + + EFX_MAC_ADDR_COPY(epp->ep_mac_addr, old_addr); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_mac_filter_set( + __in efx_nic_t *enp, + __in boolean_t all_unicst, + __in boolean_t mulcst, + __in boolean_t all_mulcst, + __in boolean_t brdcst) +{ + efx_port_t *epp = &(enp->en_port); + const efx_mac_ops_t *emop = epp->ep_emop; + boolean_t old_all_unicst; + boolean_t old_mulcst; + boolean_t old_all_mulcst; + boolean_t old_brdcst; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT); + + old_all_unicst = epp->ep_all_unicst; + old_mulcst = epp->ep_mulcst; + old_all_mulcst = epp->ep_all_mulcst; + old_brdcst = epp->ep_brdcst; + + epp->ep_all_unicst = all_unicst; + epp->ep_mulcst = mulcst; + epp->ep_all_mulcst = all_mulcst; + epp->ep_brdcst = brdcst; + + if ((rc = emop->emo_reconfigure(enp)) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + epp->ep_all_unicst = old_all_unicst; + epp->ep_mulcst = old_mulcst; + epp->ep_all_mulcst = old_all_mulcst; + epp->ep_brdcst = old_brdcst; + + return (rc); +} + + __checkReturn efx_rc_t +efx_mac_drain( + __in efx_nic_t *enp, + __in boolean_t enabled) +{ + efx_port_t *epp = &(enp->en_port); + const efx_mac_ops_t *emop = epp->ep_emop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT); + EFSYS_ASSERT(emop != NULL); + + if (epp->ep_mac_drain == enabled) + return (0); + + epp->ep_mac_drain = enabled; + + if ((rc = emop->emo_reconfigure(enp)) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_mac_up( + __in efx_nic_t *enp, + __out boolean_t *mac_upp) +{ + efx_port_t *epp = &(enp->en_port); + const efx_mac_ops_t *emop = epp->ep_emop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT); + + if ((rc = emop->emo_up(enp, mac_upp)) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_mac_fcntl_set( + __in efx_nic_t *enp, + __in unsigned int fcntl, + __in boolean_t autoneg) +{ + efx_port_t *epp = &(enp->en_port); + const efx_mac_ops_t *emop = epp->ep_emop; + const efx_phy_ops_t *epop = epp->ep_epop; + unsigned int old_fcntl; + boolean_t old_autoneg; + unsigned int old_adv_cap; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT); + + if ((fcntl & ~(EFX_FCNTL_RESPOND | EFX_FCNTL_GENERATE)) != 0) { + rc = EINVAL; + goto fail1; + } + + /* + * Ignore a request to set flow control auto-negotiation + * if the PHY doesn't support it. + */ + if (~epp->ep_phy_cap_mask & (1 << EFX_PHY_CAP_AN)) + autoneg = B_FALSE; + + old_fcntl = epp->ep_fcntl; + old_autoneg = epp->ep_fcntl_autoneg; + old_adv_cap = epp->ep_adv_cap_mask; + + epp->ep_fcntl = fcntl; + epp->ep_fcntl_autoneg = autoneg; + + /* + * Always encode the flow control settings in the advertised + * capabilities even if we are not trying to auto-negotiate + * them and reconfigure both the PHY and the MAC. + */ + if (fcntl & EFX_FCNTL_RESPOND) + epp->ep_adv_cap_mask |= (1 << EFX_PHY_CAP_PAUSE | + 1 << EFX_PHY_CAP_ASYM); + else + epp->ep_adv_cap_mask &= ~(1 << EFX_PHY_CAP_PAUSE | + 1 << EFX_PHY_CAP_ASYM); + + if (fcntl & EFX_FCNTL_GENERATE) + epp->ep_adv_cap_mask ^= (1 << EFX_PHY_CAP_ASYM); + + if ((rc = epop->epo_reconfigure(enp)) != 0) + goto fail2; + + if ((rc = emop->emo_reconfigure(enp)) != 0) + goto fail3; + + return (0); + +fail3: + EFSYS_PROBE(fail3); + +fail2: + EFSYS_PROBE(fail2); + + epp->ep_fcntl = old_fcntl; + epp->ep_fcntl_autoneg = old_autoneg; + epp->ep_adv_cap_mask = old_adv_cap; + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + void +efx_mac_fcntl_get( + __in efx_nic_t *enp, + __out unsigned int *fcntl_wantedp, + __out unsigned int *fcntl_linkp) +{ + efx_port_t *epp = &(enp->en_port); + unsigned int wanted = 0; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT); + + /* + * Decode the requested flow control settings from the PHY + * advertised capabilities. + */ + if (epp->ep_adv_cap_mask & (1 << EFX_PHY_CAP_PAUSE)) + wanted = EFX_FCNTL_RESPOND | EFX_FCNTL_GENERATE; + if (epp->ep_adv_cap_mask & (1 << EFX_PHY_CAP_ASYM)) + wanted ^= EFX_FCNTL_GENERATE; + + *fcntl_linkp = epp->ep_fcntl; + *fcntl_wantedp = wanted; +} + + __checkReturn efx_rc_t +efx_mac_multicast_list_set( + __in efx_nic_t *enp, + __in_ecount(6*count) uint8_t const *addrs, + __in int count) +{ + efx_port_t *epp = &(enp->en_port); + const efx_mac_ops_t *emop = epp->ep_emop; + uint8_t *old_mulcst_addr_list = NULL; + uint32_t old_mulcst_addr_count; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT); + + if (count > EFX_MAC_MULTICAST_LIST_MAX) { + rc = EINVAL; + goto fail1; + } + + old_mulcst_addr_count = epp->ep_mulcst_addr_count; + if (old_mulcst_addr_count > 0) { + /* Allocate memory to store old list (instead of using stack) */ + EFSYS_KMEM_ALLOC(enp->en_esip, + old_mulcst_addr_count * EFX_MAC_ADDR_LEN, + old_mulcst_addr_list); + if (old_mulcst_addr_list == NULL) { + rc = ENOMEM; + goto fail2; + } + + /* Save the old list in case we need to rollback */ + memcpy(old_mulcst_addr_list, epp->ep_mulcst_addr_list, + old_mulcst_addr_count * EFX_MAC_ADDR_LEN); + } + + /* Store the new list */ + memcpy(epp->ep_mulcst_addr_list, addrs, + count * EFX_MAC_ADDR_LEN); + epp->ep_mulcst_addr_count = count; + + if ((rc = emop->emo_multicast_list_set(enp)) != 0) + goto fail3; + + if (old_mulcst_addr_count > 0) { + EFSYS_KMEM_FREE(enp->en_esip, + old_mulcst_addr_count * EFX_MAC_ADDR_LEN, + old_mulcst_addr_list); + } + + return (0); + +fail3: + EFSYS_PROBE(fail3); + + /* Restore original list on failure */ + epp->ep_mulcst_addr_count = old_mulcst_addr_count; + if (old_mulcst_addr_count > 0) { + memcpy(epp->ep_mulcst_addr_list, old_mulcst_addr_list, + old_mulcst_addr_count * EFX_MAC_ADDR_LEN); + + EFSYS_KMEM_FREE(enp->en_esip, + old_mulcst_addr_count * EFX_MAC_ADDR_LEN, + old_mulcst_addr_list); + } + +fail2: + EFSYS_PROBE(fail2); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); + +} + + __checkReturn efx_rc_t +efx_mac_filter_default_rxq_set( + __in efx_nic_t *enp, + __in efx_rxq_t *erp, + __in boolean_t using_rss) +{ + efx_port_t *epp = &(enp->en_port); + const efx_mac_ops_t *emop = epp->ep_emop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT); + + if (emop->emo_filter_default_rxq_set != NULL) { + rc = emop->emo_filter_default_rxq_set(enp, erp, using_rss); + if (rc != 0) + goto fail1; + } + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + void +efx_mac_filter_default_rxq_clear( + __in efx_nic_t *enp) +{ + efx_port_t *epp = &(enp->en_port); + const efx_mac_ops_t *emop = epp->ep_emop; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT); + + if (emop->emo_filter_default_rxq_clear != NULL) + emop->emo_filter_default_rxq_clear(enp); +} + + +#if EFSYS_OPT_MAC_STATS + +#if EFSYS_OPT_NAMES + +/* START MKCONFIG GENERATED EfxMacStatNamesBlock c11b91b42f922516 */ +static const char * const __efx_mac_stat_name[] = { + "rx_octets", + "rx_pkts", + "rx_unicst_pkts", + "rx_multicst_pkts", + "rx_brdcst_pkts", + "rx_pause_pkts", + "rx_le_64_pkts", + "rx_65_to_127_pkts", + "rx_128_to_255_pkts", + "rx_256_to_511_pkts", + "rx_512_to_1023_pkts", + "rx_1024_to_15xx_pkts", + "rx_ge_15xx_pkts", + "rx_errors", + "rx_fcs_errors", + "rx_drop_events", + "rx_false_carrier_errors", + "rx_symbol_errors", + "rx_align_errors", + "rx_internal_errors", + "rx_jabber_pkts", + "rx_lane0_char_err", + "rx_lane1_char_err", + "rx_lane2_char_err", + "rx_lane3_char_err", + "rx_lane0_disp_err", + "rx_lane1_disp_err", + "rx_lane2_disp_err", + "rx_lane3_disp_err", + "rx_match_fault", + "rx_nodesc_drop_cnt", + "tx_octets", + "tx_pkts", + "tx_unicst_pkts", + "tx_multicst_pkts", + "tx_brdcst_pkts", + "tx_pause_pkts", + "tx_le_64_pkts", + "tx_65_to_127_pkts", + "tx_128_to_255_pkts", + "tx_256_to_511_pkts", + "tx_512_to_1023_pkts", + "tx_1024_to_15xx_pkts", + "tx_ge_15xx_pkts", + "tx_errors", + "tx_sgl_col_pkts", + "tx_mult_col_pkts", + "tx_ex_col_pkts", + "tx_late_col_pkts", + "tx_def_pkts", + "tx_ex_def_pkts", + "pm_trunc_bb_overflow", + "pm_discard_bb_overflow", + "pm_trunc_vfifo_full", + "pm_discard_vfifo_full", + "pm_trunc_qbb", + "pm_discard_qbb", + "pm_discard_mapping", + "rxdp_q_disabled_pkts", + "rxdp_di_dropped_pkts", + "rxdp_streaming_pkts", + "rxdp_hlb_fetch", + "rxdp_hlb_wait", + "vadapter_rx_unicast_packets", + "vadapter_rx_unicast_bytes", + "vadapter_rx_multicast_packets", + "vadapter_rx_multicast_bytes", + "vadapter_rx_broadcast_packets", + "vadapter_rx_broadcast_bytes", + "vadapter_rx_bad_packets", + "vadapter_rx_bad_bytes", + "vadapter_rx_overflow", + "vadapter_tx_unicast_packets", + "vadapter_tx_unicast_bytes", + "vadapter_tx_multicast_packets", + "vadapter_tx_multicast_bytes", + "vadapter_tx_broadcast_packets", + "vadapter_tx_broadcast_bytes", + "vadapter_tx_bad_packets", + "vadapter_tx_bad_bytes", + "vadapter_tx_overflow", +}; +/* END MKCONFIG GENERATED EfxMacStatNamesBlock */ + + __checkReturn const char * +efx_mac_stat_name( + __in efx_nic_t *enp, + __in unsigned int id) +{ + _NOTE(ARGUNUSED(enp)) + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + + EFSYS_ASSERT3U(id, <, EFX_MAC_NSTATS); + return (__efx_mac_stat_name[id]); +} + +#endif /* EFSYS_OPT_NAMES */ + +static efx_rc_t +efx_mac_stats_mask_add_range( + __inout_bcount(mask_size) uint32_t *maskp, + __in size_t mask_size, + __in const struct efx_mac_stats_range *rngp) +{ + unsigned int mask_npages = mask_size / sizeof (*maskp); + unsigned int el; + unsigned int el_min; + unsigned int el_max; + unsigned int low; + unsigned int high; + unsigned int width; + efx_rc_t rc; + + if ((mask_npages * EFX_MAC_STATS_MASK_BITS_PER_PAGE) <= + (unsigned int)rngp->last) { + rc = EINVAL; + goto fail1; + } + + EFSYS_ASSERT3U(rngp->first, <=, rngp->last); + EFSYS_ASSERT3U(rngp->last, <, EFX_MAC_NSTATS); + + for (el = 0; el < mask_npages; ++el) { + el_min = el * EFX_MAC_STATS_MASK_BITS_PER_PAGE; + el_max = + el_min + (EFX_MAC_STATS_MASK_BITS_PER_PAGE - 1); + if ((unsigned int)rngp->first > el_max || + (unsigned int)rngp->last < el_min) + continue; + low = MAX((unsigned int)rngp->first, el_min); + high = MIN((unsigned int)rngp->last, el_max); + width = high - low + 1; + maskp[el] |= + (width == EFX_MAC_STATS_MASK_BITS_PER_PAGE) ? + (~0ULL) : (((1ULL << width) - 1) << (low - el_min)); + } + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + efx_rc_t +efx_mac_stats_mask_add_ranges( + __inout_bcount(mask_size) uint32_t *maskp, + __in size_t mask_size, + __in_ecount(rng_count) const struct efx_mac_stats_range *rngp, + __in unsigned int rng_count) +{ + unsigned int i; + efx_rc_t rc; + + for (i = 0; i < rng_count; ++i) { + if ((rc = efx_mac_stats_mask_add_range(maskp, mask_size, + &rngp[i])) != 0) + goto fail1; + } + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_mac_stats_get_mask( + __in efx_nic_t *enp, + __out_bcount(mask_size) uint32_t *maskp, + __in size_t mask_size) +{ + efx_port_t *epp = &(enp->en_port); + const efx_mac_ops_t *emop = epp->ep_emop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE); + EFSYS_ASSERT(maskp != NULL); + EFSYS_ASSERT(mask_size % sizeof (maskp[0]) == 0); + + (void) memset(maskp, 0, mask_size); + + if ((rc = emop->emo_stats_get_mask(enp, maskp, mask_size)) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_mac_stats_clear( + __in efx_nic_t *enp) +{ + efx_port_t *epp = &(enp->en_port); + const efx_mac_ops_t *emop = epp->ep_emop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT); + EFSYS_ASSERT(emop != NULL); + + if ((rc = emop->emo_stats_clear(enp)) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_mac_stats_upload( + __in efx_nic_t *enp, + __in efsys_mem_t *esmp) +{ + efx_port_t *epp = &(enp->en_port); + const efx_mac_ops_t *emop = epp->ep_emop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT); + EFSYS_ASSERT(emop != NULL); + + /* + * Don't assert !ep_mac_stats_pending, because the client might + * have failed to finalise statistics when previously stopping + * the port. + */ + if ((rc = emop->emo_stats_upload(enp, esmp)) != 0) + goto fail1; + + epp->ep_mac_stats_pending = B_TRUE; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_mac_stats_periodic( + __in efx_nic_t *enp, + __in efsys_mem_t *esmp, + __in uint16_t period_ms, + __in boolean_t events) +{ + efx_port_t *epp = &(enp->en_port); + const efx_mac_ops_t *emop = epp->ep_emop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT); + + EFSYS_ASSERT(emop != NULL); + + if (emop->emo_stats_periodic == NULL) { + rc = EINVAL; + goto fail1; + } + + if ((rc = emop->emo_stats_periodic(enp, esmp, period_ms, events)) != 0) + goto fail2; + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + + __checkReturn efx_rc_t +efx_mac_stats_update( + __in efx_nic_t *enp, + __in efsys_mem_t *esmp, + __inout_ecount(EFX_MAC_NSTATS) efsys_stat_t *essp, + __inout_opt uint32_t *generationp) +{ + efx_port_t *epp = &(enp->en_port); + const efx_mac_ops_t *emop = epp->ep_emop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT); + EFSYS_ASSERT(emop != NULL); + + rc = emop->emo_stats_update(enp, esmp, essp, generationp); + if (rc == 0) + epp->ep_mac_stats_pending = B_FALSE; + + return (rc); +} + +#endif /* EFSYS_OPT_MAC_STATS */ + + __checkReturn efx_rc_t +efx_mac_select( + __in efx_nic_t *enp) +{ + efx_port_t *epp = &(enp->en_port); + efx_mac_type_t type = EFX_MAC_INVALID; + const efx_mac_ops_t *emop; + int rc = EINVAL; + + switch (enp->en_family) { +#if EFSYS_OPT_SIENA + case EFX_FAMILY_SIENA: + emop = &__efx_siena_mac_ops; + type = EFX_MAC_SIENA; + break; +#endif /* EFSYS_OPT_SIENA */ + +#if EFSYS_OPT_HUNTINGTON + case EFX_FAMILY_HUNTINGTON: + emop = &__efx_ef10_mac_ops; + type = EFX_MAC_HUNTINGTON; + break; +#endif /* EFSYS_OPT_HUNTINGTON */ + +#if EFSYS_OPT_MEDFORD + case EFX_FAMILY_MEDFORD: + emop = &__efx_ef10_mac_ops; + type = EFX_MAC_MEDFORD; + break; +#endif /* EFSYS_OPT_MEDFORD */ + + default: + rc = EINVAL; + goto fail1; + } + + EFSYS_ASSERT(type != EFX_MAC_INVALID); + EFSYS_ASSERT3U(type, <, EFX_MAC_NTYPES); + EFSYS_ASSERT(emop != NULL); + + epp->ep_emop = emop; + epp->ep_mac_type = type; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + +#if EFSYS_OPT_SIENA + +#define EFX_MAC_HASH_BITS (1 << 8) + +/* Compute the multicast hash as used on Falcon and Siena. */ +static void +siena_mac_multicast_hash_compute( + __in_ecount(6*count) uint8_t const *addrs, + __in int count, + __out efx_oword_t *hash_low, + __out efx_oword_t *hash_high) +{ + uint32_t crc, index; + int i; + + EFSYS_ASSERT(hash_low != NULL); + EFSYS_ASSERT(hash_high != NULL); + + EFX_ZERO_OWORD(*hash_low); + EFX_ZERO_OWORD(*hash_high); + + for (i = 0; i < count; i++) { + /* Calculate hash bucket (IEEE 802.3 CRC32 of the MAC addr) */ + crc = efx_crc32_calculate(0xffffffff, addrs, EFX_MAC_ADDR_LEN); + index = crc % EFX_MAC_HASH_BITS; + if (index < 128) { + EFX_SET_OWORD_BIT(*hash_low, index); + } else { + EFX_SET_OWORD_BIT(*hash_high, index - 128); + } + + addrs += EFX_MAC_ADDR_LEN; + } +} + +static __checkReturn efx_rc_t +siena_mac_multicast_list_set( + __in efx_nic_t *enp) +{ + efx_port_t *epp = &(enp->en_port); + const efx_mac_ops_t *emop = epp->ep_emop; + efx_oword_t old_hash[2]; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT); + + memcpy(old_hash, epp->ep_multicst_hash, sizeof (old_hash)); + + siena_mac_multicast_hash_compute( + epp->ep_mulcst_addr_list, + epp->ep_mulcst_addr_count, + &epp->ep_multicst_hash[0], + &epp->ep_multicst_hash[1]); + + if ((rc = emop->emo_reconfigure(enp)) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + memcpy(epp->ep_multicst_hash, old_hash, sizeof (old_hash)); + + return (rc); +} + +#endif /* EFSYS_OPT_SIENA */ diff --git a/dpdk/drivers/net/sfc/base/efx_mcdi.c b/dpdk/drivers/net/sfc/base/efx_mcdi.c new file mode 100644 index 00000000..c61b943c --- /dev/null +++ b/dpdk/drivers/net/sfc/base/efx_mcdi.c @@ -0,0 +1,2346 @@ +/* + * Copyright (c) 2008-2016 Solarflare Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are + * those of the authors and should not be interpreted as representing official + * policies, either expressed or implied, of the FreeBSD Project. + */ + +#include "efx.h" +#include "efx_impl.h" + +#if EFSYS_OPT_MCDI + +/* + * There are three versions of the MCDI interface: + * - MCDIv0: Siena BootROM. Transport uses MCDIv1 headers. + * - MCDIv1: Siena firmware and Huntington BootROM. + * - MCDIv2: EF10 firmware (Huntington/Medford) and Medford BootROM. + * Transport uses MCDIv2 headers. + * + * MCDIv2 Header NOT_EPOCH flag + * ---------------------------- + * A new epoch begins at initial startup or after an MC reboot, and defines when + * the MC should reject stale MCDI requests. + * + * The first MCDI request sent by the host should contain NOT_EPOCH=0, and all + * subsequent requests (until the next MC reboot) should contain NOT_EPOCH=1. + * + * After rebooting the MC will fail all requests with NOT_EPOCH=1 by writing a + * response with ERROR=1 and DATALEN=0 until a request is seen with NOT_EPOCH=0. + */ + + + +#if EFSYS_OPT_SIENA + +static const efx_mcdi_ops_t __efx_mcdi_siena_ops = { + siena_mcdi_init, /* emco_init */ + siena_mcdi_send_request, /* emco_send_request */ + siena_mcdi_poll_reboot, /* emco_poll_reboot */ + siena_mcdi_poll_response, /* emco_poll_response */ + siena_mcdi_read_response, /* emco_read_response */ + siena_mcdi_fini, /* emco_fini */ + siena_mcdi_feature_supported, /* emco_feature_supported */ + siena_mcdi_get_timeout, /* emco_get_timeout */ +}; + +#endif /* EFSYS_OPT_SIENA */ + +#if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD + +static const efx_mcdi_ops_t __efx_mcdi_ef10_ops = { + ef10_mcdi_init, /* emco_init */ + ef10_mcdi_send_request, /* emco_send_request */ + ef10_mcdi_poll_reboot, /* emco_poll_reboot */ + ef10_mcdi_poll_response, /* emco_poll_response */ + ef10_mcdi_read_response, /* emco_read_response */ + ef10_mcdi_fini, /* emco_fini */ + ef10_mcdi_feature_supported, /* emco_feature_supported */ + ef10_mcdi_get_timeout, /* emco_get_timeout */ +}; + +#endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */ + + + + __checkReturn efx_rc_t +efx_mcdi_init( + __in efx_nic_t *enp, + __in const efx_mcdi_transport_t *emtp) +{ + const efx_mcdi_ops_t *emcop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, ==, 0); + + switch (enp->en_family) { +#if EFSYS_OPT_SIENA + case EFX_FAMILY_SIENA: + emcop = &__efx_mcdi_siena_ops; + break; +#endif /* EFSYS_OPT_SIENA */ + +#if EFSYS_OPT_HUNTINGTON + case EFX_FAMILY_HUNTINGTON: + emcop = &__efx_mcdi_ef10_ops; + break; +#endif /* EFSYS_OPT_HUNTINGTON */ + +#if EFSYS_OPT_MEDFORD + case EFX_FAMILY_MEDFORD: + emcop = &__efx_mcdi_ef10_ops; + break; +#endif /* EFSYS_OPT_MEDFORD */ + + default: + EFSYS_ASSERT(0); + rc = ENOTSUP; + goto fail1; + } + + if (enp->en_features & EFX_FEATURE_MCDI_DMA) { + /* MCDI requires a DMA buffer in host memory */ + if ((emtp == NULL) || (emtp->emt_dma_mem) == NULL) { + rc = EINVAL; + goto fail2; + } + } + enp->en_mcdi.em_emtp = emtp; + + if (emcop != NULL && emcop->emco_init != NULL) { + if ((rc = emcop->emco_init(enp, emtp)) != 0) + goto fail3; + } + + enp->en_mcdi.em_emcop = emcop; + enp->en_mod_flags |= EFX_MOD_MCDI; + + return (0); + +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + enp->en_mcdi.em_emcop = NULL; + enp->en_mcdi.em_emtp = NULL; + enp->en_mod_flags &= ~EFX_MOD_MCDI; + + return (rc); +} + + void +efx_mcdi_fini( + __in efx_nic_t *enp) +{ + efx_mcdi_iface_t *emip = &(enp->en_mcdi.em_emip); + const efx_mcdi_ops_t *emcop = enp->en_mcdi.em_emcop; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, ==, EFX_MOD_MCDI); + + if (emcop != NULL && emcop->emco_fini != NULL) + emcop->emco_fini(enp); + + emip->emi_port = 0; + emip->emi_aborted = 0; + + enp->en_mcdi.em_emcop = NULL; + enp->en_mod_flags &= ~EFX_MOD_MCDI; +} + + void +efx_mcdi_new_epoch( + __in efx_nic_t *enp) +{ + efx_mcdi_iface_t *emip = &(enp->en_mcdi.em_emip); + efsys_lock_state_t state; + + /* Start a new epoch (allow fresh MCDI requests to succeed) */ + EFSYS_LOCK(enp->en_eslp, state); + emip->emi_new_epoch = B_TRUE; + EFSYS_UNLOCK(enp->en_eslp, state); +} + +static void +efx_mcdi_send_request( + __in efx_nic_t *enp, + __in void *hdrp, + __in size_t hdr_len, + __in void *sdup, + __in size_t sdu_len) +{ + const efx_mcdi_ops_t *emcop = enp->en_mcdi.em_emcop; + + emcop->emco_send_request(enp, hdrp, hdr_len, sdup, sdu_len); +} + +static efx_rc_t +efx_mcdi_poll_reboot( + __in efx_nic_t *enp) +{ + const efx_mcdi_ops_t *emcop = enp->en_mcdi.em_emcop; + efx_rc_t rc; + + rc = emcop->emco_poll_reboot(enp); + return (rc); +} + +static boolean_t +efx_mcdi_poll_response( + __in efx_nic_t *enp) +{ + const efx_mcdi_ops_t *emcop = enp->en_mcdi.em_emcop; + boolean_t available; + + available = emcop->emco_poll_response(enp); + return (available); +} + +static void +efx_mcdi_read_response( + __in efx_nic_t *enp, + __out void *bufferp, + __in size_t offset, + __in size_t length) +{ + const efx_mcdi_ops_t *emcop = enp->en_mcdi.em_emcop; + + emcop->emco_read_response(enp, bufferp, offset, length); +} + + void +efx_mcdi_request_start( + __in efx_nic_t *enp, + __in efx_mcdi_req_t *emrp, + __in boolean_t ev_cpl) +{ +#if EFSYS_OPT_MCDI_LOGGING + const efx_mcdi_transport_t *emtp = enp->en_mcdi.em_emtp; +#endif + efx_mcdi_iface_t *emip = &(enp->en_mcdi.em_emip); + efx_dword_t hdr[2]; + size_t hdr_len; + unsigned int max_version; + unsigned int seq; + unsigned int xflags; + boolean_t new_epoch; + efsys_lock_state_t state; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_MCDI); + EFSYS_ASSERT3U(enp->en_features, &, EFX_FEATURE_MCDI); + + /* + * efx_mcdi_request_start() is naturally serialised against both + * efx_mcdi_request_poll() and efx_mcdi_ev_cpl()/efx_mcdi_ev_death(), + * by virtue of there only being one outstanding MCDI request. + * Unfortunately, upper layers may also call efx_mcdi_request_abort() + * at any time, to timeout a pending mcdi request, That request may + * then subsequently complete, meaning efx_mcdi_ev_cpl() or + * efx_mcdi_ev_death() may end up running in parallel with + * efx_mcdi_request_start(). This race is handled by ensuring that + * %emi_pending_req, %emi_ev_cpl and %emi_seq are protected by the + * en_eslp lock. + */ + EFSYS_LOCK(enp->en_eslp, state); + EFSYS_ASSERT(emip->emi_pending_req == NULL); + emip->emi_pending_req = emrp; + emip->emi_ev_cpl = ev_cpl; + emip->emi_poll_cnt = 0; + seq = emip->emi_seq++ & EFX_MASK32(MCDI_HEADER_SEQ); + new_epoch = emip->emi_new_epoch; + max_version = emip->emi_max_version; + EFSYS_UNLOCK(enp->en_eslp, state); + + xflags = 0; + if (ev_cpl) + xflags |= MCDI_HEADER_XFLAGS_EVREQ; + + /* + * Huntington firmware supports MCDIv2, but the Huntington BootROM only + * supports MCDIv1. Use MCDIv1 headers for MCDIv1 commands where + * possible to support this. + */ + if ((max_version >= 2) && + ((emrp->emr_cmd > MC_CMD_CMD_SPACE_ESCAPE_7) || + (emrp->emr_in_length > MCDI_CTL_SDU_LEN_MAX_V1))) { + /* Construct MCDI v2 header */ + hdr_len = sizeof (hdr); + EFX_POPULATE_DWORD_8(hdr[0], + MCDI_HEADER_CODE, MC_CMD_V2_EXTN, + MCDI_HEADER_RESYNC, 1, + MCDI_HEADER_DATALEN, 0, + MCDI_HEADER_SEQ, seq, + MCDI_HEADER_NOT_EPOCH, new_epoch ? 0 : 1, + MCDI_HEADER_ERROR, 0, + MCDI_HEADER_RESPONSE, 0, + MCDI_HEADER_XFLAGS, xflags); + + EFX_POPULATE_DWORD_2(hdr[1], + MC_CMD_V2_EXTN_IN_EXTENDED_CMD, emrp->emr_cmd, + MC_CMD_V2_EXTN_IN_ACTUAL_LEN, emrp->emr_in_length); + } else { + /* Construct MCDI v1 header */ + hdr_len = sizeof (hdr[0]); + EFX_POPULATE_DWORD_8(hdr[0], + MCDI_HEADER_CODE, emrp->emr_cmd, + MCDI_HEADER_RESYNC, 1, + MCDI_HEADER_DATALEN, emrp->emr_in_length, + MCDI_HEADER_SEQ, seq, + MCDI_HEADER_NOT_EPOCH, new_epoch ? 0 : 1, + MCDI_HEADER_ERROR, 0, + MCDI_HEADER_RESPONSE, 0, + MCDI_HEADER_XFLAGS, xflags); + } + +#if EFSYS_OPT_MCDI_LOGGING + if (emtp->emt_logger != NULL) { + emtp->emt_logger(emtp->emt_context, EFX_LOG_MCDI_REQUEST, + &hdr[0], hdr_len, + emrp->emr_in_buf, emrp->emr_in_length); + } +#endif /* EFSYS_OPT_MCDI_LOGGING */ + + efx_mcdi_send_request(enp, &hdr[0], hdr_len, + emrp->emr_in_buf, emrp->emr_in_length); +} + + +static void +efx_mcdi_read_response_header( + __in efx_nic_t *enp, + __inout efx_mcdi_req_t *emrp) +{ +#if EFSYS_OPT_MCDI_LOGGING + const efx_mcdi_transport_t *emtp = enp->en_mcdi.em_emtp; +#endif /* EFSYS_OPT_MCDI_LOGGING */ + efx_mcdi_iface_t *emip = &(enp->en_mcdi.em_emip); + efx_dword_t hdr[2]; + unsigned int hdr_len; + unsigned int data_len; + unsigned int seq; + unsigned int cmd; + unsigned int error; + efx_rc_t rc; + + EFSYS_ASSERT(emrp != NULL); + + efx_mcdi_read_response(enp, &hdr[0], 0, sizeof (hdr[0])); + hdr_len = sizeof (hdr[0]); + + cmd = EFX_DWORD_FIELD(hdr[0], MCDI_HEADER_CODE); + seq = EFX_DWORD_FIELD(hdr[0], MCDI_HEADER_SEQ); + error = EFX_DWORD_FIELD(hdr[0], MCDI_HEADER_ERROR); + + if (cmd != MC_CMD_V2_EXTN) { + data_len = EFX_DWORD_FIELD(hdr[0], MCDI_HEADER_DATALEN); + } else { + efx_mcdi_read_response(enp, &hdr[1], hdr_len, sizeof (hdr[1])); + hdr_len += sizeof (hdr[1]); + + cmd = EFX_DWORD_FIELD(hdr[1], MC_CMD_V2_EXTN_IN_EXTENDED_CMD); + data_len = + EFX_DWORD_FIELD(hdr[1], MC_CMD_V2_EXTN_IN_ACTUAL_LEN); + } + + if (error && (data_len == 0)) { + /* The MC has rebooted since the request was sent. */ + EFSYS_SPIN(EFX_MCDI_STATUS_SLEEP_US); + efx_mcdi_poll_reboot(enp); + rc = EIO; + goto fail1; + } + if ((cmd != emrp->emr_cmd) || + (seq != ((emip->emi_seq - 1) & EFX_MASK32(MCDI_HEADER_SEQ)))) { + /* Response is for a different request */ + rc = EIO; + goto fail2; + } + if (error) { + efx_dword_t err[2]; + unsigned int err_len = MIN(data_len, sizeof (err)); + int err_code = MC_CMD_ERR_EPROTO; + int err_arg = 0; + + /* Read error code (and arg num for MCDI v2 commands) */ + efx_mcdi_read_response(enp, &err, hdr_len, err_len); + + if (err_len >= (MC_CMD_ERR_CODE_OFST + sizeof (efx_dword_t))) + err_code = EFX_DWORD_FIELD(err[0], EFX_DWORD_0); +#ifdef WITH_MCDI_V2 + if (err_len >= (MC_CMD_ERR_ARG_OFST + sizeof (efx_dword_t))) + err_arg = EFX_DWORD_FIELD(err[1], EFX_DWORD_0); +#endif + emrp->emr_err_code = err_code; + emrp->emr_err_arg = err_arg; + +#if EFSYS_OPT_MCDI_PROXY_AUTH + if ((err_code == MC_CMD_ERR_PROXY_PENDING) && + (err_len == sizeof (err))) { + /* + * The MCDI request would normally fail with EPERM, but + * firmware has forwarded it to an authorization agent + * attached to a privileged PF. + * + * Save the authorization request handle. The client + * must wait for a PROXY_RESPONSE event, or timeout. + */ + emrp->emr_proxy_handle = err_arg; + } +#endif /* EFSYS_OPT_MCDI_PROXY_AUTH */ + +#if EFSYS_OPT_MCDI_LOGGING + if (emtp->emt_logger != NULL) { + emtp->emt_logger(emtp->emt_context, + EFX_LOG_MCDI_RESPONSE, + &hdr[0], hdr_len, + &err[0], err_len); + } +#endif /* EFSYS_OPT_MCDI_LOGGING */ + + if (!emrp->emr_quiet) { + EFSYS_PROBE3(mcdi_err_arg, int, emrp->emr_cmd, + int, err_code, int, err_arg); + } + + rc = efx_mcdi_request_errcode(err_code); + goto fail3; + } + + emrp->emr_rc = 0; + emrp->emr_out_length_used = data_len; +#if EFSYS_OPT_MCDI_PROXY_AUTH + emrp->emr_proxy_handle = 0; +#endif /* EFSYS_OPT_MCDI_PROXY_AUTH */ + return; + +fail3: +fail2: +fail1: + emrp->emr_rc = rc; + emrp->emr_out_length_used = 0; +} + +static void +efx_mcdi_finish_response( + __in efx_nic_t *enp, + __in efx_mcdi_req_t *emrp) +{ +#if EFSYS_OPT_MCDI_LOGGING + const efx_mcdi_transport_t *emtp = enp->en_mcdi.em_emtp; +#endif /* EFSYS_OPT_MCDI_LOGGING */ + efx_dword_t hdr[2]; + unsigned int hdr_len; + size_t bytes; + + if (emrp->emr_out_buf == NULL) + return; + + /* Read the command header to detect MCDI response format */ + hdr_len = sizeof (hdr[0]); + efx_mcdi_read_response(enp, &hdr[0], 0, hdr_len); + if (EFX_DWORD_FIELD(hdr[0], MCDI_HEADER_CODE) == MC_CMD_V2_EXTN) { + /* + * Read the actual payload length. The length given in the event + * is only correct for responses with the V1 format. + */ + efx_mcdi_read_response(enp, &hdr[1], hdr_len, sizeof (hdr[1])); + hdr_len += sizeof (hdr[1]); + + emrp->emr_out_length_used = EFX_DWORD_FIELD(hdr[1], + MC_CMD_V2_EXTN_IN_ACTUAL_LEN); + } + + /* Copy payload out into caller supplied buffer */ + bytes = MIN(emrp->emr_out_length_used, emrp->emr_out_length); + efx_mcdi_read_response(enp, emrp->emr_out_buf, hdr_len, bytes); + +#if EFSYS_OPT_MCDI_LOGGING + if (emtp->emt_logger != NULL) { + emtp->emt_logger(emtp->emt_context, + EFX_LOG_MCDI_RESPONSE, + &hdr[0], hdr_len, + emrp->emr_out_buf, bytes); + } +#endif /* EFSYS_OPT_MCDI_LOGGING */ +} + + + __checkReturn boolean_t +efx_mcdi_request_poll( + __in efx_nic_t *enp) +{ + efx_mcdi_iface_t *emip = &(enp->en_mcdi.em_emip); + efx_mcdi_req_t *emrp; + efsys_lock_state_t state; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_MCDI); + EFSYS_ASSERT3U(enp->en_features, &, EFX_FEATURE_MCDI); + + /* Serialise against post-watchdog efx_mcdi_ev* */ + EFSYS_LOCK(enp->en_eslp, state); + + EFSYS_ASSERT(emip->emi_pending_req != NULL); + EFSYS_ASSERT(!emip->emi_ev_cpl); + emrp = emip->emi_pending_req; + + /* Check for reboot atomically w.r.t efx_mcdi_request_start */ + if (emip->emi_poll_cnt++ == 0) { + if ((rc = efx_mcdi_poll_reboot(enp)) != 0) { + emip->emi_pending_req = NULL; + EFSYS_UNLOCK(enp->en_eslp, state); + + /* Reboot/Assertion */ + if (rc == EIO || rc == EINTR) + efx_mcdi_raise_exception(enp, emrp, rc); + + goto fail1; + } + } + + /* Check if a response is available */ + if (efx_mcdi_poll_response(enp) == B_FALSE) { + EFSYS_UNLOCK(enp->en_eslp, state); + return (B_FALSE); + } + + /* Read the response header */ + efx_mcdi_read_response_header(enp, emrp); + + /* Request complete */ + emip->emi_pending_req = NULL; + + /* Ensure stale MCDI requests fail after an MC reboot. */ + emip->emi_new_epoch = B_FALSE; + + EFSYS_UNLOCK(enp->en_eslp, state); + + if ((rc = emrp->emr_rc) != 0) + goto fail2; + + efx_mcdi_finish_response(enp, emrp); + return (B_TRUE); + +fail2: + if (!emrp->emr_quiet) + EFSYS_PROBE(fail2); +fail1: + if (!emrp->emr_quiet) + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (B_TRUE); +} + + __checkReturn boolean_t +efx_mcdi_request_abort( + __in efx_nic_t *enp) +{ + efx_mcdi_iface_t *emip = &(enp->en_mcdi.em_emip); + efx_mcdi_req_t *emrp; + boolean_t aborted; + efsys_lock_state_t state; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_MCDI); + EFSYS_ASSERT3U(enp->en_features, &, EFX_FEATURE_MCDI); + + /* + * efx_mcdi_ev_* may have already completed this event, and be + * spinning/blocked on the upper layer lock. So it *is* legitimate + * to for emi_pending_req to be NULL. If there is a pending event + * completed request, then provide a "credit" to allow + * efx_mcdi_ev_cpl() to accept a single spurious completion. + */ + EFSYS_LOCK(enp->en_eslp, state); + emrp = emip->emi_pending_req; + aborted = (emrp != NULL); + if (aborted) { + emip->emi_pending_req = NULL; + + /* Error the request */ + emrp->emr_out_length_used = 0; + emrp->emr_rc = ETIMEDOUT; + + /* Provide a credit for seqno/emr_pending_req mismatches */ + if (emip->emi_ev_cpl) + ++emip->emi_aborted; + + /* + * The upper layer has called us, so we don't + * need to complete the request. + */ + } + EFSYS_UNLOCK(enp->en_eslp, state); + + return (aborted); +} + + void +efx_mcdi_get_timeout( + __in efx_nic_t *enp, + __in efx_mcdi_req_t *emrp, + __out uint32_t *timeoutp) +{ + const efx_mcdi_ops_t *emcop = enp->en_mcdi.em_emcop; + + emcop->emco_get_timeout(enp, emrp, timeoutp); +} + + __checkReturn efx_rc_t +efx_mcdi_request_errcode( + __in unsigned int err) +{ + + switch (err) { + /* MCDI v1 */ + case MC_CMD_ERR_EPERM: + return (EACCES); + case MC_CMD_ERR_ENOENT: + return (ENOENT); + case MC_CMD_ERR_EINTR: + return (EINTR); + case MC_CMD_ERR_EACCES: + return (EACCES); + case MC_CMD_ERR_EBUSY: + return (EBUSY); + case MC_CMD_ERR_EINVAL: + return (EINVAL); + case MC_CMD_ERR_EDEADLK: + return (EDEADLK); + case MC_CMD_ERR_ENOSYS: + return (ENOTSUP); + case MC_CMD_ERR_ETIME: + return (ETIMEDOUT); + case MC_CMD_ERR_ENOTSUP: + return (ENOTSUP); + case MC_CMD_ERR_EALREADY: + return (EALREADY); + + /* MCDI v2 */ + case MC_CMD_ERR_EEXIST: + return (EEXIST); +#ifdef MC_CMD_ERR_EAGAIN + case MC_CMD_ERR_EAGAIN: + return (EAGAIN); +#endif +#ifdef MC_CMD_ERR_ENOSPC + case MC_CMD_ERR_ENOSPC: + return (ENOSPC); +#endif + case MC_CMD_ERR_ERANGE: + return (ERANGE); + + case MC_CMD_ERR_ALLOC_FAIL: + return (ENOMEM); + case MC_CMD_ERR_NO_VADAPTOR: + return (ENOENT); + case MC_CMD_ERR_NO_EVB_PORT: + return (ENOENT); + case MC_CMD_ERR_NO_VSWITCH: + return (ENODEV); + case MC_CMD_ERR_VLAN_LIMIT: + return (EINVAL); + case MC_CMD_ERR_BAD_PCI_FUNC: + return (ENODEV); + case MC_CMD_ERR_BAD_VLAN_MODE: + return (EINVAL); + case MC_CMD_ERR_BAD_VSWITCH_TYPE: + return (EINVAL); + case MC_CMD_ERR_BAD_VPORT_TYPE: + return (EINVAL); + case MC_CMD_ERR_MAC_EXIST: + return (EEXIST); + + case MC_CMD_ERR_PROXY_PENDING: + return (EAGAIN); + + default: + EFSYS_PROBE1(mc_pcol_error, int, err); + return (EIO); + } +} + + void +efx_mcdi_raise_exception( + __in efx_nic_t *enp, + __in_opt efx_mcdi_req_t *emrp, + __in int rc) +{ + const efx_mcdi_transport_t *emtp = enp->en_mcdi.em_emtp; + efx_mcdi_exception_t exception; + + /* Reboot or Assertion failure only */ + EFSYS_ASSERT(rc == EIO || rc == EINTR); + + /* + * If MC_CMD_REBOOT causes a reboot (dependent on parameters), + * then the EIO is not worthy of an exception. + */ + if (emrp != NULL && emrp->emr_cmd == MC_CMD_REBOOT && rc == EIO) + return; + + exception = (rc == EIO) + ? EFX_MCDI_EXCEPTION_MC_REBOOT + : EFX_MCDI_EXCEPTION_MC_BADASSERT; + + emtp->emt_exception(emtp->emt_context, exception); +} + + void +efx_mcdi_execute( + __in efx_nic_t *enp, + __inout efx_mcdi_req_t *emrp) +{ + const efx_mcdi_transport_t *emtp = enp->en_mcdi.em_emtp; + + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_MCDI); + EFSYS_ASSERT3U(enp->en_features, &, EFX_FEATURE_MCDI); + + emrp->emr_quiet = B_FALSE; + emtp->emt_execute(emtp->emt_context, emrp); +} + + void +efx_mcdi_execute_quiet( + __in efx_nic_t *enp, + __inout efx_mcdi_req_t *emrp) +{ + const efx_mcdi_transport_t *emtp = enp->en_mcdi.em_emtp; + + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_MCDI); + EFSYS_ASSERT3U(enp->en_features, &, EFX_FEATURE_MCDI); + + emrp->emr_quiet = B_TRUE; + emtp->emt_execute(emtp->emt_context, emrp); +} + + void +efx_mcdi_ev_cpl( + __in efx_nic_t *enp, + __in unsigned int seq, + __in unsigned int outlen, + __in int errcode) +{ + efx_mcdi_iface_t *emip = &(enp->en_mcdi.em_emip); + const efx_mcdi_transport_t *emtp = enp->en_mcdi.em_emtp; + efx_mcdi_req_t *emrp; + efsys_lock_state_t state; + + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_MCDI); + EFSYS_ASSERT3U(enp->en_features, &, EFX_FEATURE_MCDI); + + /* + * Serialise against efx_mcdi_request_poll()/efx_mcdi_request_start() + * when we're completing an aborted request. + */ + EFSYS_LOCK(enp->en_eslp, state); + if (emip->emi_pending_req == NULL || !emip->emi_ev_cpl || + (seq != ((emip->emi_seq - 1) & EFX_MASK32(MCDI_HEADER_SEQ)))) { + EFSYS_ASSERT(emip->emi_aborted > 0); + if (emip->emi_aborted > 0) + --emip->emi_aborted; + EFSYS_UNLOCK(enp->en_eslp, state); + return; + } + + emrp = emip->emi_pending_req; + emip->emi_pending_req = NULL; + EFSYS_UNLOCK(enp->en_eslp, state); + + if (emip->emi_max_version >= 2) { + /* MCDIv2 response details do not fit into an event. */ + efx_mcdi_read_response_header(enp, emrp); + } else { + if (errcode != 0) { + if (!emrp->emr_quiet) { + EFSYS_PROBE2(mcdi_err, int, emrp->emr_cmd, + int, errcode); + } + emrp->emr_out_length_used = 0; + emrp->emr_rc = efx_mcdi_request_errcode(errcode); + } else { + emrp->emr_out_length_used = outlen; + emrp->emr_rc = 0; + } + } + if (errcode == 0) { + efx_mcdi_finish_response(enp, emrp); + } + + emtp->emt_ev_cpl(emtp->emt_context); +} + +#if EFSYS_OPT_MCDI_PROXY_AUTH + + __checkReturn efx_rc_t +efx_mcdi_get_proxy_handle( + __in efx_nic_t *enp, + __in efx_mcdi_req_t *emrp, + __out uint32_t *handlep) +{ + efx_rc_t rc; + + /* + * Return proxy handle from MCDI request that returned with error + * MC_MCD_ERR_PROXY_PENDING. This handle is used to wait for a matching + * PROXY_RESPONSE event. + */ + if ((emrp == NULL) || (handlep == NULL)) { + rc = EINVAL; + goto fail1; + } + if ((emrp->emr_rc != 0) && + (emrp->emr_err_code == MC_CMD_ERR_PROXY_PENDING)) { + *handlep = emrp->emr_proxy_handle; + rc = 0; + } else { + *handlep = 0; + rc = ENOENT; + } + return (rc); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + return (rc); +} + + void +efx_mcdi_ev_proxy_response( + __in efx_nic_t *enp, + __in unsigned int handle, + __in unsigned int status) +{ + const efx_mcdi_transport_t *emtp = enp->en_mcdi.em_emtp; + efx_rc_t rc; + + /* + * Handle results of an authorization request for a privileged MCDI + * command. If authorization was granted then we must re-issue the + * original MCDI request. If authorization failed or timed out, + * then the original MCDI request should be completed with the + * result code from this event. + */ + rc = (status == 0) ? 0 : efx_mcdi_request_errcode(status); + + emtp->emt_ev_proxy_response(emtp->emt_context, handle, rc); +} +#endif /* EFSYS_OPT_MCDI_PROXY_AUTH */ + + void +efx_mcdi_ev_death( + __in efx_nic_t *enp, + __in int rc) +{ + efx_mcdi_iface_t *emip = &(enp->en_mcdi.em_emip); + const efx_mcdi_transport_t *emtp = enp->en_mcdi.em_emtp; + efx_mcdi_req_t *emrp = NULL; + boolean_t ev_cpl; + efsys_lock_state_t state; + + /* + * The MCDI request (if there is one) has been terminated, either + * by a BADASSERT or REBOOT event. + * + * If there is an outstanding event-completed MCDI operation, then we + * will never receive the completion event (because both MCDI + * completions and BADASSERT events are sent to the same evq). So + * complete this MCDI op. + * + * This function might run in parallel with efx_mcdi_request_poll() + * for poll completed mcdi requests, and also with + * efx_mcdi_request_start() for post-watchdog completions. + */ + EFSYS_LOCK(enp->en_eslp, state); + emrp = emip->emi_pending_req; + ev_cpl = emip->emi_ev_cpl; + if (emrp != NULL && emip->emi_ev_cpl) { + emip->emi_pending_req = NULL; + + emrp->emr_out_length_used = 0; + emrp->emr_rc = rc; + ++emip->emi_aborted; + } + + /* + * Since we're running in parallel with a request, consume the + * status word before dropping the lock. + */ + if (rc == EIO || rc == EINTR) { + EFSYS_SPIN(EFX_MCDI_STATUS_SLEEP_US); + (void) efx_mcdi_poll_reboot(enp); + emip->emi_new_epoch = B_TRUE; + } + + EFSYS_UNLOCK(enp->en_eslp, state); + + efx_mcdi_raise_exception(enp, emrp, rc); + + if (emrp != NULL && ev_cpl) + emtp->emt_ev_cpl(emtp->emt_context); +} + + __checkReturn efx_rc_t +efx_mcdi_version( + __in efx_nic_t *enp, + __out_ecount_opt(4) uint16_t versionp[4], + __out_opt uint32_t *buildp, + __out_opt efx_mcdi_boot_t *statusp) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MAX(MC_CMD_GET_VERSION_IN_LEN, + MC_CMD_GET_VERSION_OUT_LEN), + MAX(MC_CMD_GET_BOOT_STATUS_IN_LEN, + MC_CMD_GET_BOOT_STATUS_OUT_LEN))]; + efx_word_t *ver_words; + uint16_t version[4]; + uint32_t build; + efx_mcdi_boot_t status; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_features, &, EFX_FEATURE_MCDI); + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_GET_VERSION; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_GET_VERSION_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_GET_VERSION_OUT_LEN; + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + /* bootrom support */ + if (req.emr_out_length_used == MC_CMD_GET_VERSION_V0_OUT_LEN) { + version[0] = version[1] = version[2] = version[3] = 0; + build = MCDI_OUT_DWORD(req, GET_VERSION_OUT_FIRMWARE); + + goto version; + } + + if (req.emr_out_length_used < MC_CMD_GET_VERSION_OUT_LEN) { + rc = EMSGSIZE; + goto fail2; + } + + ver_words = MCDI_OUT2(req, efx_word_t, GET_VERSION_OUT_VERSION); + version[0] = EFX_WORD_FIELD(ver_words[0], EFX_WORD_0); + version[1] = EFX_WORD_FIELD(ver_words[1], EFX_WORD_0); + version[2] = EFX_WORD_FIELD(ver_words[2], EFX_WORD_0); + version[3] = EFX_WORD_FIELD(ver_words[3], EFX_WORD_0); + build = MCDI_OUT_DWORD(req, GET_VERSION_OUT_FIRMWARE); + +version: + /* The bootrom doesn't understand BOOT_STATUS */ + if (MC_FW_VERSION_IS_BOOTLOADER(build)) { + status = EFX_MCDI_BOOT_ROM; + goto out; + } + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_GET_BOOT_STATUS; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_GET_BOOT_STATUS_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_GET_BOOT_STATUS_OUT_LEN; + + efx_mcdi_execute_quiet(enp, &req); + + if (req.emr_rc == EACCES) { + /* Unprivileged functions cannot access BOOT_STATUS */ + status = EFX_MCDI_BOOT_PRIMARY; + version[0] = version[1] = version[2] = version[3] = 0; + build = 0; + goto out; + } + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail3; + } + + if (req.emr_out_length_used < MC_CMD_GET_BOOT_STATUS_OUT_LEN) { + rc = EMSGSIZE; + goto fail4; + } + + if (MCDI_OUT_DWORD_FIELD(req, GET_BOOT_STATUS_OUT_FLAGS, + GET_BOOT_STATUS_OUT_FLAGS_PRIMARY)) + status = EFX_MCDI_BOOT_PRIMARY; + else + status = EFX_MCDI_BOOT_SECONDARY; + +out: + if (versionp != NULL) + memcpy(versionp, version, sizeof (version)); + if (buildp != NULL) + *buildp = build; + if (statusp != NULL) + *statusp = status; + + return (0); + +fail4: + EFSYS_PROBE(fail4); +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_mcdi_get_capabilities( + __in efx_nic_t *enp, + __out_opt uint32_t *flagsp, + __out_opt uint16_t *rx_dpcpu_fw_idp, + __out_opt uint16_t *tx_dpcpu_fw_idp, + __out_opt uint32_t *flags2p, + __out_opt uint32_t *tso2ncp) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_GET_CAPABILITIES_IN_LEN, + MC_CMD_GET_CAPABILITIES_V2_OUT_LEN)]; + boolean_t v2_capable; + efx_rc_t rc; + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_GET_CAPABILITIES; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_GET_CAPABILITIES_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_GET_CAPABILITIES_V2_OUT_LEN; + + efx_mcdi_execute_quiet(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + if (req.emr_out_length_used < MC_CMD_GET_CAPABILITIES_OUT_LEN) { + rc = EMSGSIZE; + goto fail2; + } + + if (flagsp != NULL) + *flagsp = MCDI_OUT_DWORD(req, GET_CAPABILITIES_OUT_FLAGS1); + + if (rx_dpcpu_fw_idp != NULL) + *rx_dpcpu_fw_idp = MCDI_OUT_WORD(req, + GET_CAPABILITIES_OUT_RX_DPCPU_FW_ID); + + if (tx_dpcpu_fw_idp != NULL) + *tx_dpcpu_fw_idp = MCDI_OUT_WORD(req, + GET_CAPABILITIES_OUT_TX_DPCPU_FW_ID); + + if (req.emr_out_length_used < MC_CMD_GET_CAPABILITIES_V2_OUT_LEN) + v2_capable = B_FALSE; + else + v2_capable = B_TRUE; + + if (flags2p != NULL) { + *flags2p = (v2_capable) ? + MCDI_OUT_DWORD(req, GET_CAPABILITIES_V2_OUT_FLAGS2) : + 0; + } + + if (tso2ncp != NULL) { + *tso2ncp = (v2_capable) ? + MCDI_OUT_WORD(req, + GET_CAPABILITIES_V2_OUT_TX_TSO_V2_N_CONTEXTS) : + 0; + } + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +static __checkReturn efx_rc_t +efx_mcdi_do_reboot( + __in efx_nic_t *enp, + __in boolean_t after_assertion) +{ + uint8_t payload[MAX(MC_CMD_REBOOT_IN_LEN, MC_CMD_REBOOT_OUT_LEN)]; + efx_mcdi_req_t req; + efx_rc_t rc; + + /* + * We could require the caller to have caused en_mod_flags=0 to + * call this function. This doesn't help the other port though, + * who's about to get the MC ripped out from underneath them. + * Since they have to cope with the subsequent fallout of MCDI + * failures, we should as well. + */ + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_REBOOT; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_REBOOT_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_REBOOT_OUT_LEN; + + MCDI_IN_SET_DWORD(req, REBOOT_IN_FLAGS, + (after_assertion ? MC_CMD_REBOOT_FLAGS_AFTER_ASSERTION : 0)); + + efx_mcdi_execute_quiet(enp, &req); + + if (req.emr_rc == EACCES) { + /* Unprivileged functions cannot reboot the MC. */ + goto out; + } + + /* A successful reboot request returns EIO. */ + if (req.emr_rc != 0 && req.emr_rc != EIO) { + rc = req.emr_rc; + goto fail1; + } + +out: + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_mcdi_reboot( + __in efx_nic_t *enp) +{ + return (efx_mcdi_do_reboot(enp, B_FALSE)); +} + + __checkReturn efx_rc_t +efx_mcdi_exit_assertion_handler( + __in efx_nic_t *enp) +{ + return (efx_mcdi_do_reboot(enp, B_TRUE)); +} + + __checkReturn efx_rc_t +efx_mcdi_read_assertion( + __in efx_nic_t *enp) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_GET_ASSERTS_IN_LEN, + MC_CMD_GET_ASSERTS_OUT_LEN)]; + const char *reason; + unsigned int flags; + unsigned int index; + unsigned int ofst; + int retry; + efx_rc_t rc; + + /* + * Before we attempt to chat to the MC, we should verify that the MC + * isn't in it's assertion handler, either due to a previous reboot, + * or because we're reinitializing due to an eec_exception(). + * + * Use GET_ASSERTS to read any assertion state that may be present. + * Retry this command twice. Once because a boot-time assertion failure + * might cause the 1st MCDI request to fail. And once again because + * we might race with efx_mcdi_exit_assertion_handler() running on + * partner port(s) on the same NIC. + */ + retry = 2; + do { + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_GET_ASSERTS; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_GET_ASSERTS_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_GET_ASSERTS_OUT_LEN; + + MCDI_IN_SET_DWORD(req, GET_ASSERTS_IN_CLEAR, 1); + efx_mcdi_execute_quiet(enp, &req); + + } while ((req.emr_rc == EINTR || req.emr_rc == EIO) && retry-- > 0); + + if (req.emr_rc != 0) { + if (req.emr_rc == EACCES) { + /* Unprivileged functions cannot clear assertions. */ + goto out; + } + rc = req.emr_rc; + goto fail1; + } + + if (req.emr_out_length_used < MC_CMD_GET_ASSERTS_OUT_LEN) { + rc = EMSGSIZE; + goto fail2; + } + + /* Print out any assertion state recorded */ + flags = MCDI_OUT_DWORD(req, GET_ASSERTS_OUT_GLOBAL_FLAGS); + if (flags == MC_CMD_GET_ASSERTS_FLAGS_NO_FAILS) + return (0); + + reason = (flags == MC_CMD_GET_ASSERTS_FLAGS_SYS_FAIL) + ? "system-level assertion" + : (flags == MC_CMD_GET_ASSERTS_FLAGS_THR_FAIL) + ? "thread-level assertion" + : (flags == MC_CMD_GET_ASSERTS_FLAGS_WDOG_FIRED) + ? "watchdog reset" + : (flags == MC_CMD_GET_ASSERTS_FLAGS_ADDR_TRAP) + ? "illegal address trap" + : "unknown assertion"; + EFSYS_PROBE3(mcpu_assertion, + const char *, reason, unsigned int, + MCDI_OUT_DWORD(req, GET_ASSERTS_OUT_SAVED_PC_OFFS), + unsigned int, + MCDI_OUT_DWORD(req, GET_ASSERTS_OUT_THREAD_OFFS)); + + /* Print out the registers (r1 ... r31) */ + ofst = MC_CMD_GET_ASSERTS_OUT_GP_REGS_OFFS_OFST; + for (index = 1; + index < 1 + MC_CMD_GET_ASSERTS_OUT_GP_REGS_OFFS_NUM; + index++) { + EFSYS_PROBE2(mcpu_register, unsigned int, index, unsigned int, + EFX_DWORD_FIELD(*MCDI_OUT(req, efx_dword_t, ofst), + EFX_DWORD_0)); + ofst += sizeof (efx_dword_t); + } + EFSYS_ASSERT(ofst <= MC_CMD_GET_ASSERTS_OUT_LEN); + +out: + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + +/* + * Internal routines for for specific MCDI requests. + */ + + __checkReturn efx_rc_t +efx_mcdi_drv_attach( + __in efx_nic_t *enp, + __in boolean_t attach) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_DRV_ATTACH_IN_LEN, + MC_CMD_DRV_ATTACH_EXT_OUT_LEN)]; + efx_rc_t rc; + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_DRV_ATTACH; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_DRV_ATTACH_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_DRV_ATTACH_EXT_OUT_LEN; + + /* + * Use DONT_CARE for the datapath firmware type to ensure that the + * driver can attach to an unprivileged function. The datapath firmware + * type to use is controlled by the 'sfboot' utility. + */ + MCDI_IN_SET_DWORD(req, DRV_ATTACH_IN_NEW_STATE, attach ? 1 : 0); + MCDI_IN_SET_DWORD(req, DRV_ATTACH_IN_UPDATE, 1); + MCDI_IN_SET_DWORD(req, DRV_ATTACH_IN_FIRMWARE_ID, MC_CMD_FW_DONT_CARE); + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + if (req.emr_out_length_used < MC_CMD_DRV_ATTACH_OUT_LEN) { + rc = EMSGSIZE; + goto fail2; + } + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_mcdi_get_board_cfg( + __in efx_nic_t *enp, + __out_opt uint32_t *board_typep, + __out_opt efx_dword_t *capabilitiesp, + __out_ecount_opt(6) uint8_t mac_addrp[6]) +{ + efx_mcdi_iface_t *emip = &(enp->en_mcdi.em_emip); + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_GET_BOARD_CFG_IN_LEN, + MC_CMD_GET_BOARD_CFG_OUT_LENMIN)]; + efx_rc_t rc; + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_GET_BOARD_CFG; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_GET_BOARD_CFG_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_GET_BOARD_CFG_OUT_LENMIN; + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + if (req.emr_out_length_used < MC_CMD_GET_BOARD_CFG_OUT_LENMIN) { + rc = EMSGSIZE; + goto fail2; + } + + if (mac_addrp != NULL) { + uint8_t *addrp; + + if (emip->emi_port == 1) { + addrp = MCDI_OUT2(req, uint8_t, + GET_BOARD_CFG_OUT_MAC_ADDR_BASE_PORT0); + } else if (emip->emi_port == 2) { + addrp = MCDI_OUT2(req, uint8_t, + GET_BOARD_CFG_OUT_MAC_ADDR_BASE_PORT1); + } else { + rc = EINVAL; + goto fail3; + } + + EFX_MAC_ADDR_COPY(mac_addrp, addrp); + } + + if (capabilitiesp != NULL) { + if (emip->emi_port == 1) { + *capabilitiesp = *MCDI_OUT2(req, efx_dword_t, + GET_BOARD_CFG_OUT_CAPABILITIES_PORT0); + } else if (emip->emi_port == 2) { + *capabilitiesp = *MCDI_OUT2(req, efx_dword_t, + GET_BOARD_CFG_OUT_CAPABILITIES_PORT1); + } else { + rc = EINVAL; + goto fail4; + } + } + + if (board_typep != NULL) { + *board_typep = MCDI_OUT_DWORD(req, + GET_BOARD_CFG_OUT_BOARD_TYPE); + } + + return (0); + +fail4: + EFSYS_PROBE(fail4); +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_mcdi_get_resource_limits( + __in efx_nic_t *enp, + __out_opt uint32_t *nevqp, + __out_opt uint32_t *nrxqp, + __out_opt uint32_t *ntxqp) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_GET_RESOURCE_LIMITS_IN_LEN, + MC_CMD_GET_RESOURCE_LIMITS_OUT_LEN)]; + efx_rc_t rc; + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_GET_RESOURCE_LIMITS; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_GET_RESOURCE_LIMITS_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_GET_RESOURCE_LIMITS_OUT_LEN; + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + if (req.emr_out_length_used < MC_CMD_GET_RESOURCE_LIMITS_OUT_LEN) { + rc = EMSGSIZE; + goto fail2; + } + + if (nevqp != NULL) + *nevqp = MCDI_OUT_DWORD(req, GET_RESOURCE_LIMITS_OUT_EVQ); + if (nrxqp != NULL) + *nrxqp = MCDI_OUT_DWORD(req, GET_RESOURCE_LIMITS_OUT_RXQ); + if (ntxqp != NULL) + *ntxqp = MCDI_OUT_DWORD(req, GET_RESOURCE_LIMITS_OUT_TXQ); + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_mcdi_get_phy_cfg( + __in efx_nic_t *enp) +{ + efx_port_t *epp = &(enp->en_port); + efx_nic_cfg_t *encp = &(enp->en_nic_cfg); + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_GET_PHY_CFG_IN_LEN, + MC_CMD_GET_PHY_CFG_OUT_LEN)]; + efx_rc_t rc; + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_GET_PHY_CFG; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_GET_PHY_CFG_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_GET_PHY_CFG_OUT_LEN; + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + if (req.emr_out_length_used < MC_CMD_GET_PHY_CFG_OUT_LEN) { + rc = EMSGSIZE; + goto fail2; + } + + encp->enc_phy_type = MCDI_OUT_DWORD(req, GET_PHY_CFG_OUT_TYPE); +#if EFSYS_OPT_NAMES + (void) strncpy(encp->enc_phy_name, + MCDI_OUT2(req, char, GET_PHY_CFG_OUT_NAME), + MIN(sizeof (encp->enc_phy_name) - 1, + MC_CMD_GET_PHY_CFG_OUT_NAME_LEN)); +#endif /* EFSYS_OPT_NAMES */ + (void) memset(encp->enc_phy_revision, 0, + sizeof (encp->enc_phy_revision)); + memcpy(encp->enc_phy_revision, + MCDI_OUT2(req, char, GET_PHY_CFG_OUT_REVISION), + MIN(sizeof (encp->enc_phy_revision) - 1, + MC_CMD_GET_PHY_CFG_OUT_REVISION_LEN)); +#if EFSYS_OPT_PHY_LED_CONTROL + encp->enc_led_mask = ((1 << EFX_PHY_LED_DEFAULT) | + (1 << EFX_PHY_LED_OFF) | + (1 << EFX_PHY_LED_ON)); +#endif /* EFSYS_OPT_PHY_LED_CONTROL */ + + /* Get the media type of the fixed port, if recognised. */ + EFX_STATIC_ASSERT(MC_CMD_MEDIA_XAUI == EFX_PHY_MEDIA_XAUI); + EFX_STATIC_ASSERT(MC_CMD_MEDIA_CX4 == EFX_PHY_MEDIA_CX4); + EFX_STATIC_ASSERT(MC_CMD_MEDIA_KX4 == EFX_PHY_MEDIA_KX4); + EFX_STATIC_ASSERT(MC_CMD_MEDIA_XFP == EFX_PHY_MEDIA_XFP); + EFX_STATIC_ASSERT(MC_CMD_MEDIA_SFP_PLUS == EFX_PHY_MEDIA_SFP_PLUS); + EFX_STATIC_ASSERT(MC_CMD_MEDIA_BASE_T == EFX_PHY_MEDIA_BASE_T); + EFX_STATIC_ASSERT(MC_CMD_MEDIA_QSFP_PLUS == EFX_PHY_MEDIA_QSFP_PLUS); + epp->ep_fixed_port_type = + (efx_phy_media_type_t) MCDI_OUT_DWORD(req, GET_PHY_CFG_OUT_MEDIA_TYPE); + if (epp->ep_fixed_port_type >= EFX_PHY_MEDIA_NTYPES) + epp->ep_fixed_port_type = EFX_PHY_MEDIA_INVALID; + + epp->ep_phy_cap_mask = + MCDI_OUT_DWORD(req, GET_PHY_CFG_OUT_SUPPORTED_CAP); +#if EFSYS_OPT_PHY_FLAGS + encp->enc_phy_flags_mask = MCDI_OUT_DWORD(req, GET_PHY_CFG_OUT_FLAGS); +#endif /* EFSYS_OPT_PHY_FLAGS */ + + encp->enc_port = (uint8_t)MCDI_OUT_DWORD(req, GET_PHY_CFG_OUT_PRT); + + /* Populate internal state */ + encp->enc_mcdi_mdio_channel = + (uint8_t)MCDI_OUT_DWORD(req, GET_PHY_CFG_OUT_CHANNEL); + +#if EFSYS_OPT_PHY_STATS + encp->enc_mcdi_phy_stat_mask = + MCDI_OUT_DWORD(req, GET_PHY_CFG_OUT_STATS_MASK); +#endif /* EFSYS_OPT_PHY_STATS */ + +#if EFSYS_OPT_BIST + encp->enc_bist_mask = 0; + if (MCDI_OUT_DWORD_FIELD(req, GET_PHY_CFG_OUT_FLAGS, + GET_PHY_CFG_OUT_BIST_CABLE_SHORT)) + encp->enc_bist_mask |= (1 << EFX_BIST_TYPE_PHY_CABLE_SHORT); + if (MCDI_OUT_DWORD_FIELD(req, GET_PHY_CFG_OUT_FLAGS, + GET_PHY_CFG_OUT_BIST_CABLE_LONG)) + encp->enc_bist_mask |= (1 << EFX_BIST_TYPE_PHY_CABLE_LONG); + if (MCDI_OUT_DWORD_FIELD(req, GET_PHY_CFG_OUT_FLAGS, + GET_PHY_CFG_OUT_BIST)) + encp->enc_bist_mask |= (1 << EFX_BIST_TYPE_PHY_NORMAL); +#endif /* EFSYS_OPT_BIST */ + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_mcdi_firmware_update_supported( + __in efx_nic_t *enp, + __out boolean_t *supportedp) +{ + const efx_mcdi_ops_t *emcop = enp->en_mcdi.em_emcop; + efx_rc_t rc; + + if (emcop != NULL) { + if ((rc = emcop->emco_feature_supported(enp, + EFX_MCDI_FEATURE_FW_UPDATE, supportedp)) != 0) + goto fail1; + } else { + /* Earlier devices always supported updates */ + *supportedp = B_TRUE; + } + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_mcdi_macaddr_change_supported( + __in efx_nic_t *enp, + __out boolean_t *supportedp) +{ + const efx_mcdi_ops_t *emcop = enp->en_mcdi.em_emcop; + efx_rc_t rc; + + if (emcop != NULL) { + if ((rc = emcop->emco_feature_supported(enp, + EFX_MCDI_FEATURE_MACADDR_CHANGE, supportedp)) != 0) + goto fail1; + } else { + /* Earlier devices always supported MAC changes */ + *supportedp = B_TRUE; + } + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_mcdi_link_control_supported( + __in efx_nic_t *enp, + __out boolean_t *supportedp) +{ + const efx_mcdi_ops_t *emcop = enp->en_mcdi.em_emcop; + efx_rc_t rc; + + if (emcop != NULL) { + if ((rc = emcop->emco_feature_supported(enp, + EFX_MCDI_FEATURE_LINK_CONTROL, supportedp)) != 0) + goto fail1; + } else { + /* Earlier devices always supported link control */ + *supportedp = B_TRUE; + } + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_mcdi_mac_spoofing_supported( + __in efx_nic_t *enp, + __out boolean_t *supportedp) +{ + const efx_mcdi_ops_t *emcop = enp->en_mcdi.em_emcop; + efx_rc_t rc; + + if (emcop != NULL) { + if ((rc = emcop->emco_feature_supported(enp, + EFX_MCDI_FEATURE_MAC_SPOOFING, supportedp)) != 0) + goto fail1; + } else { + /* Earlier devices always supported MAC spoofing */ + *supportedp = B_TRUE; + } + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +#if EFSYS_OPT_BIST + +#if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD +/* + * Enter bist offline mode. This is a fw mode which puts the NIC into a state + * where memory BIST tests can be run and not much else can interfere or happen. + * A reboot is required to exit this mode. + */ + __checkReturn efx_rc_t +efx_mcdi_bist_enable_offline( + __in efx_nic_t *enp) +{ + efx_mcdi_req_t req; + efx_rc_t rc; + + EFX_STATIC_ASSERT(MC_CMD_ENABLE_OFFLINE_BIST_IN_LEN == 0); + EFX_STATIC_ASSERT(MC_CMD_ENABLE_OFFLINE_BIST_OUT_LEN == 0); + + req.emr_cmd = MC_CMD_ENABLE_OFFLINE_BIST; + req.emr_in_buf = NULL; + req.emr_in_length = 0; + req.emr_out_buf = NULL; + req.emr_out_length = 0; + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} +#endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */ + + __checkReturn efx_rc_t +efx_mcdi_bist_start( + __in efx_nic_t *enp, + __in efx_bist_type_t type) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_START_BIST_IN_LEN, + MC_CMD_START_BIST_OUT_LEN)]; + efx_rc_t rc; + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_START_BIST; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_START_BIST_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_START_BIST_OUT_LEN; + + switch (type) { + case EFX_BIST_TYPE_PHY_NORMAL: + MCDI_IN_SET_DWORD(req, START_BIST_IN_TYPE, MC_CMD_PHY_BIST); + break; + case EFX_BIST_TYPE_PHY_CABLE_SHORT: + MCDI_IN_SET_DWORD(req, START_BIST_IN_TYPE, + MC_CMD_PHY_BIST_CABLE_SHORT); + break; + case EFX_BIST_TYPE_PHY_CABLE_LONG: + MCDI_IN_SET_DWORD(req, START_BIST_IN_TYPE, + MC_CMD_PHY_BIST_CABLE_LONG); + break; + case EFX_BIST_TYPE_MC_MEM: + MCDI_IN_SET_DWORD(req, START_BIST_IN_TYPE, + MC_CMD_MC_MEM_BIST); + break; + case EFX_BIST_TYPE_SAT_MEM: + MCDI_IN_SET_DWORD(req, START_BIST_IN_TYPE, + MC_CMD_PORT_MEM_BIST); + break; + case EFX_BIST_TYPE_REG: + MCDI_IN_SET_DWORD(req, START_BIST_IN_TYPE, + MC_CMD_REG_BIST); + break; + default: + EFSYS_ASSERT(0); + } + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +#endif /* EFSYS_OPT_BIST */ + + +/* Enable logging of some events (e.g. link state changes) */ + __checkReturn efx_rc_t +efx_mcdi_log_ctrl( + __in efx_nic_t *enp) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_LOG_CTRL_IN_LEN, + MC_CMD_LOG_CTRL_OUT_LEN)]; + efx_rc_t rc; + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_LOG_CTRL; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_LOG_CTRL_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_LOG_CTRL_OUT_LEN; + + MCDI_IN_SET_DWORD(req, LOG_CTRL_IN_LOG_DEST, + MC_CMD_LOG_CTRL_IN_LOG_DEST_EVQ); + MCDI_IN_SET_DWORD(req, LOG_CTRL_IN_LOG_DEST_EVQ, 0); + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + +#if EFSYS_OPT_MAC_STATS + +typedef enum efx_stats_action_e { + EFX_STATS_CLEAR, + EFX_STATS_UPLOAD, + EFX_STATS_ENABLE_NOEVENTS, + EFX_STATS_ENABLE_EVENTS, + EFX_STATS_DISABLE, +} efx_stats_action_t; + +static __checkReturn efx_rc_t +efx_mcdi_mac_stats( + __in efx_nic_t *enp, + __in_opt efsys_mem_t *esmp, + __in efx_stats_action_t action, + __in uint16_t period_ms) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_MAC_STATS_IN_LEN, + MC_CMD_MAC_STATS_OUT_DMA_LEN)]; + int clear = (action == EFX_STATS_CLEAR); + int upload = (action == EFX_STATS_UPLOAD); + int enable = (action == EFX_STATS_ENABLE_NOEVENTS); + int events = (action == EFX_STATS_ENABLE_EVENTS); + int disable = (action == EFX_STATS_DISABLE); + efx_rc_t rc; + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_MAC_STATS; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_MAC_STATS_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_MAC_STATS_OUT_DMA_LEN; + + MCDI_IN_POPULATE_DWORD_6(req, MAC_STATS_IN_CMD, + MAC_STATS_IN_DMA, upload, + MAC_STATS_IN_CLEAR, clear, + MAC_STATS_IN_PERIODIC_CHANGE, enable | events | disable, + MAC_STATS_IN_PERIODIC_ENABLE, enable | events, + MAC_STATS_IN_PERIODIC_NOEVENT, !events, + MAC_STATS_IN_PERIOD_MS, (enable | events) ? period_ms : 0); + + if (esmp != NULL) { + int bytes = MC_CMD_MAC_NSTATS * sizeof (uint64_t); + + EFX_STATIC_ASSERT(MC_CMD_MAC_NSTATS * sizeof (uint64_t) <= + EFX_MAC_STATS_SIZE); + + MCDI_IN_SET_DWORD(req, MAC_STATS_IN_DMA_ADDR_LO, + EFSYS_MEM_ADDR(esmp) & 0xffffffff); + MCDI_IN_SET_DWORD(req, MAC_STATS_IN_DMA_ADDR_HI, + EFSYS_MEM_ADDR(esmp) >> 32); + MCDI_IN_SET_DWORD(req, MAC_STATS_IN_DMA_LEN, bytes); + } else { + EFSYS_ASSERT(!upload && !enable && !events); + } + + /* + * NOTE: Do not use EVB_PORT_ID_ASSIGNED when disabling periodic stats, + * as this may fail (and leave periodic DMA enabled) if the + * vadapter has already been deleted. + */ + MCDI_IN_SET_DWORD(req, MAC_STATS_IN_PORT_ID, + (disable ? EVB_PORT_ID_NULL : enp->en_vport_id)); + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + /* EF10: Expect ENOENT if no DMA queues are initialised */ + if ((req.emr_rc != ENOENT) || + (enp->en_rx_qcount + enp->en_tx_qcount != 0)) { + rc = req.emr_rc; + goto fail1; + } + } + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_mcdi_mac_stats_clear( + __in efx_nic_t *enp) +{ + efx_rc_t rc; + + if ((rc = efx_mcdi_mac_stats(enp, NULL, EFX_STATS_CLEAR, 0)) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_mcdi_mac_stats_upload( + __in efx_nic_t *enp, + __in efsys_mem_t *esmp) +{ + efx_rc_t rc; + + /* + * The MC DMAs aggregate statistics for our convenience, so we can + * avoid having to pull the statistics buffer into the cache to + * maintain cumulative statistics. + */ + if ((rc = efx_mcdi_mac_stats(enp, esmp, EFX_STATS_UPLOAD, 0)) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_mcdi_mac_stats_periodic( + __in efx_nic_t *enp, + __in efsys_mem_t *esmp, + __in uint16_t period_ms, + __in boolean_t events) +{ + efx_rc_t rc; + + /* + * The MC DMAs aggregate statistics for our convenience, so we can + * avoid having to pull the statistics buffer into the cache to + * maintain cumulative statistics. + * Huntington uses a fixed 1sec period. + * Medford uses a fixed 1sec period before v6.2.1.1033 firmware. + */ + if (period_ms == 0) + rc = efx_mcdi_mac_stats(enp, NULL, EFX_STATS_DISABLE, 0); + else if (events) + rc = efx_mcdi_mac_stats(enp, esmp, EFX_STATS_ENABLE_EVENTS, + period_ms); + else + rc = efx_mcdi_mac_stats(enp, esmp, EFX_STATS_ENABLE_NOEVENTS, + period_ms); + + if (rc != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +#endif /* EFSYS_OPT_MAC_STATS */ + +#if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD + +/* + * This function returns the pf and vf number of a function. If it is a pf the + * vf number is 0xffff. The vf number is the index of the vf on that + * function. So if you have 3 vfs on pf 0 the 3 vfs will return (pf=0,vf=0), + * (pf=0,vf=1), (pf=0,vf=2) aand the pf will return (pf=0, vf=0xffff). + */ + __checkReturn efx_rc_t +efx_mcdi_get_function_info( + __in efx_nic_t *enp, + __out uint32_t *pfp, + __out_opt uint32_t *vfp) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_GET_FUNCTION_INFO_IN_LEN, + MC_CMD_GET_FUNCTION_INFO_OUT_LEN)]; + efx_rc_t rc; + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_GET_FUNCTION_INFO; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_GET_FUNCTION_INFO_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_GET_FUNCTION_INFO_OUT_LEN; + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + if (req.emr_out_length_used < MC_CMD_GET_FUNCTION_INFO_OUT_LEN) { + rc = EMSGSIZE; + goto fail2; + } + + *pfp = MCDI_OUT_DWORD(req, GET_FUNCTION_INFO_OUT_PF); + if (vfp != NULL) + *vfp = MCDI_OUT_DWORD(req, GET_FUNCTION_INFO_OUT_VF); + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_mcdi_privilege_mask( + __in efx_nic_t *enp, + __in uint32_t pf, + __in uint32_t vf, + __out uint32_t *maskp) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_PRIVILEGE_MASK_IN_LEN, + MC_CMD_PRIVILEGE_MASK_OUT_LEN)]; + efx_rc_t rc; + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_PRIVILEGE_MASK; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_PRIVILEGE_MASK_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_PRIVILEGE_MASK_OUT_LEN; + + MCDI_IN_POPULATE_DWORD_2(req, PRIVILEGE_MASK_IN_FUNCTION, + PRIVILEGE_MASK_IN_FUNCTION_PF, pf, + PRIVILEGE_MASK_IN_FUNCTION_VF, vf); + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + if (req.emr_out_length_used < MC_CMD_PRIVILEGE_MASK_OUT_LEN) { + rc = EMSGSIZE; + goto fail2; + } + + *maskp = MCDI_OUT_DWORD(req, PRIVILEGE_MASK_OUT_OLD_MASK); + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +#endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */ + + __checkReturn efx_rc_t +efx_mcdi_set_workaround( + __in efx_nic_t *enp, + __in uint32_t type, + __in boolean_t enabled, + __out_opt uint32_t *flagsp) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_WORKAROUND_IN_LEN, + MC_CMD_WORKAROUND_EXT_OUT_LEN)]; + efx_rc_t rc; + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_WORKAROUND; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_WORKAROUND_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_WORKAROUND_OUT_LEN; + + MCDI_IN_SET_DWORD(req, WORKAROUND_IN_TYPE, type); + MCDI_IN_SET_DWORD(req, WORKAROUND_IN_ENABLED, enabled ? 1 : 0); + + efx_mcdi_execute_quiet(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + if (flagsp != NULL) { + if (req.emr_out_length_used >= MC_CMD_WORKAROUND_EXT_OUT_LEN) + *flagsp = MCDI_OUT_DWORD(req, WORKAROUND_EXT_OUT_FLAGS); + else + *flagsp = 0; + } + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + + __checkReturn efx_rc_t +efx_mcdi_get_workarounds( + __in efx_nic_t *enp, + __out_opt uint32_t *implementedp, + __out_opt uint32_t *enabledp) +{ + efx_mcdi_req_t req; + uint8_t payload[MC_CMD_GET_WORKAROUNDS_OUT_LEN]; + efx_rc_t rc; + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_GET_WORKAROUNDS; + req.emr_in_buf = NULL; + req.emr_in_length = 0; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_GET_WORKAROUNDS_OUT_LEN; + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + if (implementedp != NULL) { + *implementedp = + MCDI_OUT_DWORD(req, GET_WORKAROUNDS_OUT_IMPLEMENTED); + } + + if (enabledp != NULL) { + *enabledp = MCDI_OUT_DWORD(req, GET_WORKAROUNDS_OUT_ENABLED); + } + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +/* + * Size of media information page in accordance with SFF-8472 and SFF-8436. + * It is used in MCDI interface as well. + */ +#define EFX_PHY_MEDIA_INFO_PAGE_SIZE 0x80 + +static __checkReturn efx_rc_t +efx_mcdi_get_phy_media_info( + __in efx_nic_t *enp, + __in uint32_t mcdi_page, + __in uint8_t offset, + __in uint8_t len, + __out_bcount(len) uint8_t *data) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_GET_PHY_MEDIA_INFO_IN_LEN, + MC_CMD_GET_PHY_MEDIA_INFO_OUT_LEN( + EFX_PHY_MEDIA_INFO_PAGE_SIZE))]; + efx_rc_t rc; + + EFSYS_ASSERT((uint32_t)offset + len <= EFX_PHY_MEDIA_INFO_PAGE_SIZE); + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_GET_PHY_MEDIA_INFO; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_GET_PHY_MEDIA_INFO_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = + MC_CMD_GET_PHY_MEDIA_INFO_OUT_LEN(EFX_PHY_MEDIA_INFO_PAGE_SIZE); + + MCDI_IN_SET_DWORD(req, GET_PHY_MEDIA_INFO_IN_PAGE, mcdi_page); + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + if (req.emr_out_length_used != + MC_CMD_GET_PHY_MEDIA_INFO_OUT_LEN(EFX_PHY_MEDIA_INFO_PAGE_SIZE)) { + rc = EMSGSIZE; + goto fail2; + } + + if (MCDI_OUT_DWORD(req, GET_PHY_MEDIA_INFO_OUT_DATALEN) != + EFX_PHY_MEDIA_INFO_PAGE_SIZE) { + rc = EIO; + goto fail3; + } + + memcpy(data, + MCDI_OUT2(req, uint8_t, GET_PHY_MEDIA_INFO_OUT_DATA) + offset, + len); + + return (0); + +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +/* + * 2-wire device address of the base information in accordance with SFF-8472 + * Diagnostic Monitoring Interface for Optical Transceivers section + * 4 Memory Organization. + */ +#define EFX_PHY_MEDIA_INFO_DEV_ADDR_SFP_BASE 0xA0 + +/* + * 2-wire device address of the digital diagnostics monitoring interface + * in accordance with SFF-8472 Diagnostic Monitoring Interface for Optical + * Transceivers section 4 Memory Organization. + */ +#define EFX_PHY_MEDIA_INFO_DEV_ADDR_SFP_DDM 0xA2 + +/* + * Hard wired 2-wire device address for QSFP+ in accordance with SFF-8436 + * QSFP+ 10 Gbs 4X PLUGGABLE TRANSCEIVER section 7.4 Device Addressing and + * Operation. + */ +#define EFX_PHY_MEDIA_INFO_DEV_ADDR_QSFP 0xA0 + + __checkReturn efx_rc_t +efx_mcdi_phy_module_get_info( + __in efx_nic_t *enp, + __in uint8_t dev_addr, + __in uint8_t offset, + __in uint8_t len, + __out_bcount(len) uint8_t *data) +{ + efx_port_t *epp = &(enp->en_port); + efx_rc_t rc; + uint32_t mcdi_lower_page; + uint32_t mcdi_upper_page; + + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE); + + /* + * Map device address to MC_CMD_GET_PHY_MEDIA_INFO pages. + * Offset plus length interface allows to access page 0 only. + * I.e. non-zero upper pages are not accessible. + * See SFF-8472 section 4 Memory Organization and SFF-8436 section 7.6 + * QSFP+ Memory Map for details on how information is structured + * and accessible. + */ + switch (epp->ep_fixed_port_type) { + case EFX_PHY_MEDIA_SFP_PLUS: + /* + * In accordance with SFF-8472 Diagnostic Monitoring + * Interface for Optical Transceivers section 4 Memory + * Organization two 2-wire addresses are defined. + */ + switch (dev_addr) { + /* Base information */ + case EFX_PHY_MEDIA_INFO_DEV_ADDR_SFP_BASE: + /* + * MCDI page 0 should be used to access lower + * page 0 (0x00 - 0x7f) at the device address 0xA0. + */ + mcdi_lower_page = 0; + /* + * MCDI page 1 should be used to access upper + * page 0 (0x80 - 0xff) at the device address 0xA0. + */ + mcdi_upper_page = 1; + break; + /* Diagnostics */ + case EFX_PHY_MEDIA_INFO_DEV_ADDR_SFP_DDM: + /* + * MCDI page 2 should be used to access lower + * page 0 (0x00 - 0x7f) at the device address 0xA2. + */ + mcdi_lower_page = 2; + /* + * MCDI page 3 should be used to access upper + * page 0 (0x80 - 0xff) at the device address 0xA2. + */ + mcdi_upper_page = 3; + break; + default: + rc = ENOTSUP; + goto fail1; + } + break; + case EFX_PHY_MEDIA_QSFP_PLUS: + switch (dev_addr) { + case EFX_PHY_MEDIA_INFO_DEV_ADDR_QSFP: + /* + * MCDI page -1 should be used to access lower page 0 + * (0x00 - 0x7f). + */ + mcdi_lower_page = (uint32_t)-1; + /* + * MCDI page 0 should be used to access upper page 0 + * (0x80h - 0xff). + */ + mcdi_upper_page = 0; + break; + default: + rc = ENOTSUP; + goto fail1; + } + break; + default: + rc = ENOTSUP; + goto fail1; + } + + if (offset < EFX_PHY_MEDIA_INFO_PAGE_SIZE) { + uint8_t read_len = + MIN(len, EFX_PHY_MEDIA_INFO_PAGE_SIZE - offset); + + rc = efx_mcdi_get_phy_media_info(enp, + mcdi_lower_page, offset, read_len, data); + if (rc != 0) + goto fail2; + + data += read_len; + len -= read_len; + + offset = 0; + } else { + offset -= EFX_PHY_MEDIA_INFO_PAGE_SIZE; + } + + if (len > 0) { + EFSYS_ASSERT3U(len, <=, EFX_PHY_MEDIA_INFO_PAGE_SIZE); + EFSYS_ASSERT3U(offset, <, EFX_PHY_MEDIA_INFO_PAGE_SIZE); + + rc = efx_mcdi_get_phy_media_info(enp, + mcdi_upper_page, offset, len, data); + if (rc != 0) + goto fail3; + } + + return (0); + +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +#endif /* EFSYS_OPT_MCDI */ diff --git a/dpdk/drivers/net/sfc/base/efx_mcdi.h b/dpdk/drivers/net/sfc/base/efx_mcdi.h new file mode 100644 index 00000000..21727713 --- /dev/null +++ b/dpdk/drivers/net/sfc/base/efx_mcdi.h @@ -0,0 +1,415 @@ +/* + * Copyright (c) 2009-2016 Solarflare Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are + * those of the authors and should not be interpreted as representing official + * policies, either expressed or implied, of the FreeBSD Project. + */ + +#ifndef _SYS_EFX_MCDI_H +#define _SYS_EFX_MCDI_H + +#include "efx.h" +#include "efx_regs_mcdi.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * A reboot/assertion causes the MCDI status word to be set after the + * command word is set or a REBOOT event is sent. If we notice a reboot + * via these mechanisms then wait 10ms for the status word to be set. + */ +#define EFX_MCDI_STATUS_SLEEP_US 10000 + +struct efx_mcdi_req_s { + boolean_t emr_quiet; + /* Inputs: Command #, input buffer and length */ + unsigned int emr_cmd; + uint8_t *emr_in_buf; + size_t emr_in_length; + /* Outputs: retcode, buffer, length, and length used*/ + efx_rc_t emr_rc; + uint8_t *emr_out_buf; + size_t emr_out_length; + size_t emr_out_length_used; + /* Internals: low level transport details */ + unsigned int emr_err_code; + unsigned int emr_err_arg; +#if EFSYS_OPT_MCDI_PROXY_AUTH + uint32_t emr_proxy_handle; +#endif +}; + +typedef struct efx_mcdi_iface_s { + unsigned int emi_port; + unsigned int emi_max_version; + unsigned int emi_seq; + efx_mcdi_req_t *emi_pending_req; + boolean_t emi_ev_cpl; + boolean_t emi_new_epoch; + int emi_aborted; + uint32_t emi_poll_cnt; + uint32_t emi_mc_reboot_status; +} efx_mcdi_iface_t; + +extern void +efx_mcdi_execute( + __in efx_nic_t *enp, + __inout efx_mcdi_req_t *emrp); + +extern void +efx_mcdi_execute_quiet( + __in efx_nic_t *enp, + __inout efx_mcdi_req_t *emrp); + +extern void +efx_mcdi_ev_cpl( + __in efx_nic_t *enp, + __in unsigned int seq, + __in unsigned int outlen, + __in int errcode); + +#if EFSYS_OPT_MCDI_PROXY_AUTH +extern __checkReturn efx_rc_t +efx_mcdi_get_proxy_handle( + __in efx_nic_t *enp, + __in efx_mcdi_req_t *emrp, + __out uint32_t *handlep); + +extern void +efx_mcdi_ev_proxy_response( + __in efx_nic_t *enp, + __in unsigned int handle, + __in unsigned int status); +#endif + +extern void +efx_mcdi_ev_death( + __in efx_nic_t *enp, + __in int rc); + +extern __checkReturn efx_rc_t +efx_mcdi_request_errcode( + __in unsigned int err); + +extern void +efx_mcdi_raise_exception( + __in efx_nic_t *enp, + __in_opt efx_mcdi_req_t *emrp, + __in int rc); + +typedef enum efx_mcdi_boot_e { + EFX_MCDI_BOOT_PRIMARY, + EFX_MCDI_BOOT_SECONDARY, + EFX_MCDI_BOOT_ROM, +} efx_mcdi_boot_t; + +extern __checkReturn efx_rc_t +efx_mcdi_version( + __in efx_nic_t *enp, + __out_ecount_opt(4) uint16_t versionp[4], + __out_opt uint32_t *buildp, + __out_opt efx_mcdi_boot_t *statusp); + +extern __checkReturn efx_rc_t +efx_mcdi_get_capabilities( + __in efx_nic_t *enp, + __out_opt uint32_t *flagsp, + __out_opt uint16_t *rx_dpcpu_fw_idp, + __out_opt uint16_t *tx_dpcpu_fw_idp, + __out_opt uint32_t *flags2p, + __out_opt uint32_t *tso2ncp); + +extern __checkReturn efx_rc_t +efx_mcdi_read_assertion( + __in efx_nic_t *enp); + +extern __checkReturn efx_rc_t +efx_mcdi_exit_assertion_handler( + __in efx_nic_t *enp); + +extern __checkReturn efx_rc_t +efx_mcdi_drv_attach( + __in efx_nic_t *enp, + __in boolean_t attach); + +extern __checkReturn efx_rc_t +efx_mcdi_get_board_cfg( + __in efx_nic_t *enp, + __out_opt uint32_t *board_typep, + __out_opt efx_dword_t *capabilitiesp, + __out_ecount_opt(6) uint8_t mac_addrp[6]); + +extern __checkReturn efx_rc_t +efx_mcdi_get_phy_cfg( + __in efx_nic_t *enp); + +extern __checkReturn efx_rc_t +efx_mcdi_firmware_update_supported( + __in efx_nic_t *enp, + __out boolean_t *supportedp); + +extern __checkReturn efx_rc_t +efx_mcdi_macaddr_change_supported( + __in efx_nic_t *enp, + __out boolean_t *supportedp); + +extern __checkReturn efx_rc_t +efx_mcdi_link_control_supported( + __in efx_nic_t *enp, + __out boolean_t *supportedp); + +extern __checkReturn efx_rc_t +efx_mcdi_mac_spoofing_supported( + __in efx_nic_t *enp, + __out boolean_t *supportedp); + + +#if EFSYS_OPT_BIST +#if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD +extern __checkReturn efx_rc_t +efx_mcdi_bist_enable_offline( + __in efx_nic_t *enp); +#endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */ +extern __checkReturn efx_rc_t +efx_mcdi_bist_start( + __in efx_nic_t *enp, + __in efx_bist_type_t type); +#endif /* EFSYS_OPT_BIST */ + +extern __checkReturn efx_rc_t +efx_mcdi_get_resource_limits( + __in efx_nic_t *enp, + __out_opt uint32_t *nevqp, + __out_opt uint32_t *nrxqp, + __out_opt uint32_t *ntxqp); + +extern __checkReturn efx_rc_t +efx_mcdi_log_ctrl( + __in efx_nic_t *enp); + +extern __checkReturn efx_rc_t +efx_mcdi_mac_stats_clear( + __in efx_nic_t *enp); + +extern __checkReturn efx_rc_t +efx_mcdi_mac_stats_upload( + __in efx_nic_t *enp, + __in efsys_mem_t *esmp); + +extern __checkReturn efx_rc_t +efx_mcdi_mac_stats_periodic( + __in efx_nic_t *enp, + __in efsys_mem_t *esmp, + __in uint16_t period_ms, + __in boolean_t events); + + +#if EFSYS_OPT_LOOPBACK +extern __checkReturn efx_rc_t +efx_mcdi_get_loopback_modes( + __in efx_nic_t *enp); +#endif /* EFSYS_OPT_LOOPBACK */ + +extern __checkReturn efx_rc_t +efx_mcdi_phy_module_get_info( + __in efx_nic_t *enp, + __in uint8_t dev_addr, + __in uint8_t offset, + __in uint8_t len, + __out_bcount(len) uint8_t *data); + +#define MCDI_IN(_emr, _type, _ofst) \ + ((_type *)((_emr).emr_in_buf + (_ofst))) + +#define MCDI_IN2(_emr, _type, _ofst) \ + MCDI_IN(_emr, _type, MC_CMD_ ## _ofst ## _OFST) + +#define MCDI_IN_SET_BYTE(_emr, _ofst, _value) \ + EFX_POPULATE_BYTE_1(*MCDI_IN2(_emr, efx_byte_t, _ofst), \ + EFX_BYTE_0, _value) + +#define MCDI_IN_SET_WORD(_emr, _ofst, _value) \ + EFX_POPULATE_WORD_1(*MCDI_IN2(_emr, efx_word_t, _ofst), \ + EFX_WORD_0, _value) + +#define MCDI_IN_SET_DWORD(_emr, _ofst, _value) \ + EFX_POPULATE_DWORD_1(*MCDI_IN2(_emr, efx_dword_t, _ofst), \ + EFX_DWORD_0, _value) + +#define MCDI_IN_SET_DWORD_FIELD(_emr, _ofst, _field, _value) \ + EFX_SET_DWORD_FIELD(*MCDI_IN2(_emr, efx_dword_t, _ofst), \ + MC_CMD_ ## _field, _value) + +#define MCDI_IN_POPULATE_DWORD_1(_emr, _ofst, _field1, _value1) \ + EFX_POPULATE_DWORD_1(*MCDI_IN2(_emr, efx_dword_t, _ofst), \ + MC_CMD_ ## _field1, _value1) + +#define MCDI_IN_POPULATE_DWORD_2(_emr, _ofst, _field1, _value1, \ + _field2, _value2) \ + EFX_POPULATE_DWORD_2(*MCDI_IN2(_emr, efx_dword_t, _ofst), \ + MC_CMD_ ## _field1, _value1, \ + MC_CMD_ ## _field2, _value2) + +#define MCDI_IN_POPULATE_DWORD_3(_emr, _ofst, _field1, _value1, \ + _field2, _value2, _field3, _value3) \ + EFX_POPULATE_DWORD_3(*MCDI_IN2(_emr, efx_dword_t, _ofst), \ + MC_CMD_ ## _field1, _value1, \ + MC_CMD_ ## _field2, _value2, \ + MC_CMD_ ## _field3, _value3) + +#define MCDI_IN_POPULATE_DWORD_4(_emr, _ofst, _field1, _value1, \ + _field2, _value2, _field3, _value3, _field4, _value4) \ + EFX_POPULATE_DWORD_4(*MCDI_IN2(_emr, efx_dword_t, _ofst), \ + MC_CMD_ ## _field1, _value1, \ + MC_CMD_ ## _field2, _value2, \ + MC_CMD_ ## _field3, _value3, \ + MC_CMD_ ## _field4, _value4) + +#define MCDI_IN_POPULATE_DWORD_5(_emr, _ofst, _field1, _value1, \ + _field2, _value2, _field3, _value3, _field4, _value4, \ + _field5, _value5) \ + EFX_POPULATE_DWORD_5(*MCDI_IN2(_emr, efx_dword_t, _ofst), \ + MC_CMD_ ## _field1, _value1, \ + MC_CMD_ ## _field2, _value2, \ + MC_CMD_ ## _field3, _value3, \ + MC_CMD_ ## _field4, _value4, \ + MC_CMD_ ## _field5, _value5) + +#define MCDI_IN_POPULATE_DWORD_6(_emr, _ofst, _field1, _value1, \ + _field2, _value2, _field3, _value3, _field4, _value4, \ + _field5, _value5, _field6, _value6) \ + EFX_POPULATE_DWORD_6(*MCDI_IN2(_emr, efx_dword_t, _ofst), \ + MC_CMD_ ## _field1, _value1, \ + MC_CMD_ ## _field2, _value2, \ + MC_CMD_ ## _field3, _value3, \ + MC_CMD_ ## _field4, _value4, \ + MC_CMD_ ## _field5, _value5, \ + MC_CMD_ ## _field6, _value6) + +#define MCDI_IN_POPULATE_DWORD_7(_emr, _ofst, _field1, _value1, \ + _field2, _value2, _field3, _value3, _field4, _value4, \ + _field5, _value5, _field6, _value6, _field7, _value7) \ + EFX_POPULATE_DWORD_7(*MCDI_IN2(_emr, efx_dword_t, _ofst), \ + MC_CMD_ ## _field1, _value1, \ + MC_CMD_ ## _field2, _value2, \ + MC_CMD_ ## _field3, _value3, \ + MC_CMD_ ## _field4, _value4, \ + MC_CMD_ ## _field5, _value5, \ + MC_CMD_ ## _field6, _value6, \ + MC_CMD_ ## _field7, _value7) + +#define MCDI_IN_POPULATE_DWORD_8(_emr, _ofst, _field1, _value1, \ + _field2, _value2, _field3, _value3, _field4, _value4, \ + _field5, _value5, _field6, _value6, _field7, _value7, \ + _field8, _value8) \ + EFX_POPULATE_DWORD_8(*MCDI_IN2(_emr, efx_dword_t, _ofst), \ + MC_CMD_ ## _field1, _value1, \ + MC_CMD_ ## _field2, _value2, \ + MC_CMD_ ## _field3, _value3, \ + MC_CMD_ ## _field4, _value4, \ + MC_CMD_ ## _field5, _value5, \ + MC_CMD_ ## _field6, _value6, \ + MC_CMD_ ## _field7, _value7, \ + MC_CMD_ ## _field8, _value8) + +#define MCDI_IN_POPULATE_DWORD_9(_emr, _ofst, _field1, _value1, \ + _field2, _value2, _field3, _value3, _field4, _value4, \ + _field5, _value5, _field6, _value6, _field7, _value7, \ + _field8, _value8, _field9, _value9) \ + EFX_POPULATE_DWORD_9(*MCDI_IN2(_emr, efx_dword_t, _ofst), \ + MC_CMD_ ## _field1, _value1, \ + MC_CMD_ ## _field2, _value2, \ + MC_CMD_ ## _field3, _value3, \ + MC_CMD_ ## _field4, _value4, \ + MC_CMD_ ## _field5, _value5, \ + MC_CMD_ ## _field6, _value6, \ + MC_CMD_ ## _field7, _value7, \ + MC_CMD_ ## _field8, _value8, \ + MC_CMD_ ## _field9, _value9) + +#define MCDI_IN_POPULATE_DWORD_10(_emr, _ofst, _field1, _value1, \ + _field2, _value2, _field3, _value3, _field4, _value4, \ + _field5, _value5, _field6, _value6, _field7, _value7, \ + _field8, _value8, _field9, _value9, _field10, _value10) \ + EFX_POPULATE_DWORD_10(*MCDI_IN2(_emr, efx_dword_t, _ofst), \ + MC_CMD_ ## _field1, _value1, \ + MC_CMD_ ## _field2, _value2, \ + MC_CMD_ ## _field3, _value3, \ + MC_CMD_ ## _field4, _value4, \ + MC_CMD_ ## _field5, _value5, \ + MC_CMD_ ## _field6, _value6, \ + MC_CMD_ ## _field7, _value7, \ + MC_CMD_ ## _field8, _value8, \ + MC_CMD_ ## _field9, _value9, \ + MC_CMD_ ## _field10, _value10) + +#define MCDI_OUT(_emr, _type, _ofst) \ + ((_type *)((_emr).emr_out_buf + (_ofst))) + +#define MCDI_OUT2(_emr, _type, _ofst) \ + MCDI_OUT(_emr, _type, MC_CMD_ ## _ofst ## _OFST) + +#define MCDI_OUT_BYTE(_emr, _ofst) \ + EFX_BYTE_FIELD(*MCDI_OUT2(_emr, efx_byte_t, _ofst), \ + EFX_BYTE_0) + +#define MCDI_OUT_WORD(_emr, _ofst) \ + EFX_WORD_FIELD(*MCDI_OUT2(_emr, efx_word_t, _ofst), \ + EFX_WORD_0) + +#define MCDI_OUT_DWORD(_emr, _ofst) \ + EFX_DWORD_FIELD(*MCDI_OUT2(_emr, efx_dword_t, _ofst), \ + EFX_DWORD_0) + +#define MCDI_OUT_DWORD_FIELD(_emr, _ofst, _field) \ + EFX_DWORD_FIELD(*MCDI_OUT2(_emr, efx_dword_t, _ofst), \ + MC_CMD_ ## _field) + +#define MCDI_EV_FIELD(_eqp, _field) \ + EFX_QWORD_FIELD(*_eqp, MCDI_EVENT_ ## _field) + +#define MCDI_CMD_DWORD_FIELD(_edp, _field) \ + EFX_DWORD_FIELD(*_edp, MC_CMD_ ## _field) + +#define EFX_MCDI_HAVE_PRIVILEGE(mask, priv) \ + (((mask) & (MC_CMD_PRIVILEGE_MASK_IN_GRP_ ## priv)) == \ + (MC_CMD_PRIVILEGE_MASK_IN_GRP_ ## priv)) + +typedef enum efx_mcdi_feature_id_e { + EFX_MCDI_FEATURE_FW_UPDATE = 0, + EFX_MCDI_FEATURE_LINK_CONTROL, + EFX_MCDI_FEATURE_MACADDR_CHANGE, + EFX_MCDI_FEATURE_MAC_SPOOFING, + EFX_MCDI_FEATURE_NIDS +} efx_mcdi_feature_id_t; + +#ifdef __cplusplus +} +#endif + +#endif /* _SYS_EFX_MCDI_H */ diff --git a/dpdk/drivers/net/sfc/base/efx_mon.c b/dpdk/drivers/net/sfc/base/efx_mon.c new file mode 100644 index 00000000..c2f1e97e --- /dev/null +++ b/dpdk/drivers/net/sfc/base/efx_mon.c @@ -0,0 +1,255 @@ +/* + * Copyright (c) 2007-2016 Solarflare Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are + * those of the authors and should not be interpreted as representing official + * policies, either expressed or implied, of the FreeBSD Project. + */ + +#include "efx.h" +#include "efx_impl.h" + +#if EFSYS_OPT_MON_MCDI +#include "mcdi_mon.h" +#endif + +#if EFSYS_OPT_NAMES + +static const char * const __efx_mon_name[] = { + "", + "sfx90x0", + "sfx91x0", + "sfx92x0" +}; + + const char * +efx_mon_name( + __in efx_nic_t *enp) +{ + efx_nic_cfg_t *encp = &(enp->en_nic_cfg); + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + + EFSYS_ASSERT(encp->enc_mon_type != EFX_MON_INVALID); + EFSYS_ASSERT3U(encp->enc_mon_type, <, EFX_MON_NTYPES); + return (__efx_mon_name[encp->enc_mon_type]); +} + +#endif /* EFSYS_OPT_NAMES */ + +#if EFSYS_OPT_MON_MCDI +static const efx_mon_ops_t __efx_mon_mcdi_ops = { +#if EFSYS_OPT_MON_STATS + mcdi_mon_stats_update /* emo_stats_update */ +#endif /* EFSYS_OPT_MON_STATS */ +}; +#endif + + + __checkReturn efx_rc_t +efx_mon_init( + __in efx_nic_t *enp) +{ + efx_nic_cfg_t *encp = &(enp->en_nic_cfg); + efx_mon_t *emp = &(enp->en_mon); + const efx_mon_ops_t *emop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE); + + if (enp->en_mod_flags & EFX_MOD_MON) { + rc = EINVAL; + goto fail1; + } + + enp->en_mod_flags |= EFX_MOD_MON; + + emp->em_type = encp->enc_mon_type; + + EFSYS_ASSERT(encp->enc_mon_type != EFX_MON_INVALID); + switch (emp->em_type) { +#if EFSYS_OPT_MON_MCDI + case EFX_MON_SFC90X0: + case EFX_MON_SFC91X0: + case EFX_MON_SFC92X0: + emop = &__efx_mon_mcdi_ops; + break; +#endif + default: + rc = ENOTSUP; + goto fail2; + } + + emp->em_emop = emop; + return (0); + +fail2: + EFSYS_PROBE(fail2); + + emp->em_type = EFX_MON_INVALID; + + enp->en_mod_flags &= ~EFX_MOD_MON; + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +#if EFSYS_OPT_MON_STATS + +#if EFSYS_OPT_NAMES + +/* START MKCONFIG GENERATED MonitorStatNamesBlock 5daa2a5725ba734b */ +static const char * const __mon_stat_name[] = { + "value_2_5v", + "value_vccp1", + "value_vcc", + "value_5v", + "value_12v", + "value_vccp2", + "value_ext_temp", + "value_int_temp", + "value_ain1", + "value_ain2", + "controller_cooling", + "ext_cooling", + "1v", + "1_2v", + "1_8v", + "3_3v", + "1_2va", + "vref", + "vaoe", + "aoe_temperature", + "psu_aoe_temperature", + "psu_temperature", + "fan0", + "fan1", + "fan2", + "fan3", + "fan4", + "vaoe_in", + "iaoe", + "iaoe_in", + "nic_power", + "0_9v", + "i0_9v", + "i1_2v", + "0_9v_adc", + "controller_temperature2", + "vreg_temperature", + "vreg_0_9v_temperature", + "vreg_1_2v_temperature", + "int_vptat", + "controller_internal_adc_temperature", + "ext_vptat", + "controller_external_adc_temperature", + "ambient_temperature", + "airflow", + "vdd08d_vss08d_csr", + "vdd08d_vss08d_csr_extadc", + "hotpoint_temperature", + "phy_power_switch_port0", + "phy_power_switch_port1", + "mum_vcc", + "0v9_a", + "i0v9_a", + "0v9_a_temp", + "0v9_b", + "i0v9_b", + "0v9_b_temp", + "ccom_avreg_1v2_supply", + "ccom_avreg_1v2_supply_ext_adc", + "ccom_avreg_1v8_supply", + "ccom_avreg_1v8_supply_ext_adc", + "controller_master_vptat", + "controller_master_internal_temp", + "controller_master_vptat_ext_adc", + "controller_master_internal_temp_ext_adc", + "controller_slave_vptat", + "controller_slave_internal_temp", + "controller_slave_vptat_ext_adc", + "controller_slave_internal_temp_ext_adc", + "sodimm_vout", + "sodimm_0_temp", + "sodimm_1_temp", + "phy0_vcc", + "phy1_vcc", + "controller_tdiode_temp", + "board_front_temp", + "board_back_temp", +}; + +/* END MKCONFIG GENERATED MonitorStatNamesBlock */ + +extern const char * +efx_mon_stat_name( + __in efx_nic_t *enp, + __in efx_mon_stat_t id) +{ + _NOTE(ARGUNUSED(enp)) + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + + EFSYS_ASSERT3U(id, <, EFX_MON_NSTATS); + return (__mon_stat_name[id]); +} + +#endif /* EFSYS_OPT_NAMES */ + + __checkReturn efx_rc_t +efx_mon_stats_update( + __in efx_nic_t *enp, + __in efsys_mem_t *esmp, + __inout_ecount(EFX_MON_NSTATS) efx_mon_stat_value_t *values) +{ + efx_mon_t *emp = &(enp->en_mon); + const efx_mon_ops_t *emop = emp->em_emop; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_MON); + + return (emop->emo_stats_update(enp, esmp, values)); +} + +#endif /* EFSYS_OPT_MON_STATS */ + + void +efx_mon_fini( + __in efx_nic_t *enp) +{ + efx_mon_t *emp = &(enp->en_mon); + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_MON); + + emp->em_emop = NULL; + + emp->em_type = EFX_MON_INVALID; + + enp->en_mod_flags &= ~EFX_MOD_MON; +} diff --git a/dpdk/drivers/net/sfc/base/efx_nic.c b/dpdk/drivers/net/sfc/base/efx_nic.c new file mode 100644 index 00000000..76caa744 --- /dev/null +++ b/dpdk/drivers/net/sfc/base/efx_nic.c @@ -0,0 +1,1110 @@ +/* + * Copyright (c) 2007-2016 Solarflare Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are + * those of the authors and should not be interpreted as representing official + * policies, either expressed or implied, of the FreeBSD Project. + */ + +#include "efx.h" +#include "efx_impl.h" + + __checkReturn efx_rc_t +efx_family( + __in uint16_t venid, + __in uint16_t devid, + __out efx_family_t *efp) +{ + if (venid == EFX_PCI_VENID_SFC) { + switch (devid) { +#if EFSYS_OPT_SIENA + case EFX_PCI_DEVID_SIENA_F1_UNINIT: + /* + * Hardware default for PF0 of uninitialised Siena. + * manftest must be able to cope with this device id. + */ + *efp = EFX_FAMILY_SIENA; + return (0); + + case EFX_PCI_DEVID_BETHPAGE: + case EFX_PCI_DEVID_SIENA: + *efp = EFX_FAMILY_SIENA; + return (0); +#endif /* EFSYS_OPT_SIENA */ + +#if EFSYS_OPT_HUNTINGTON + case EFX_PCI_DEVID_HUNTINGTON_PF_UNINIT: + /* + * Hardware default for PF0 of uninitialised Huntington. + * manftest must be able to cope with this device id. + */ + *efp = EFX_FAMILY_HUNTINGTON; + return (0); + + case EFX_PCI_DEVID_FARMINGDALE: + case EFX_PCI_DEVID_GREENPORT: + *efp = EFX_FAMILY_HUNTINGTON; + return (0); + + case EFX_PCI_DEVID_FARMINGDALE_VF: + case EFX_PCI_DEVID_GREENPORT_VF: + *efp = EFX_FAMILY_HUNTINGTON; + return (0); +#endif /* EFSYS_OPT_HUNTINGTON */ + +#if EFSYS_OPT_MEDFORD + case EFX_PCI_DEVID_MEDFORD_PF_UNINIT: + /* + * Hardware default for PF0 of uninitialised Medford. + * manftest must be able to cope with this device id. + */ + *efp = EFX_FAMILY_MEDFORD; + return (0); + + case EFX_PCI_DEVID_MEDFORD: + *efp = EFX_FAMILY_MEDFORD; + return (0); + + case EFX_PCI_DEVID_MEDFORD_VF: + *efp = EFX_FAMILY_MEDFORD; + return (0); +#endif /* EFSYS_OPT_MEDFORD */ + + case EFX_PCI_DEVID_FALCON: /* Obsolete, not supported */ + default: + break; + } + } + + *efp = EFX_FAMILY_INVALID; + return (ENOTSUP); +} + + +#define EFX_BIU_MAGIC0 0x01234567 +#define EFX_BIU_MAGIC1 0xfedcba98 + + __checkReturn efx_rc_t +efx_nic_biu_test( + __in efx_nic_t *enp) +{ + efx_oword_t oword; + efx_rc_t rc; + + /* + * Write magic values to scratch registers 0 and 1, then + * verify that the values were written correctly. Interleave + * the accesses to ensure that the BIU is not just reading + * back the cached value that was last written. + */ + EFX_POPULATE_OWORD_1(oword, FRF_AZ_DRIVER_DW0, EFX_BIU_MAGIC0); + EFX_BAR_TBL_WRITEO(enp, FR_AZ_DRIVER_REG, 0, &oword, B_TRUE); + + EFX_POPULATE_OWORD_1(oword, FRF_AZ_DRIVER_DW0, EFX_BIU_MAGIC1); + EFX_BAR_TBL_WRITEO(enp, FR_AZ_DRIVER_REG, 1, &oword, B_TRUE); + + EFX_BAR_TBL_READO(enp, FR_AZ_DRIVER_REG, 0, &oword, B_TRUE); + if (EFX_OWORD_FIELD(oword, FRF_AZ_DRIVER_DW0) != EFX_BIU_MAGIC0) { + rc = EIO; + goto fail1; + } + + EFX_BAR_TBL_READO(enp, FR_AZ_DRIVER_REG, 1, &oword, B_TRUE); + if (EFX_OWORD_FIELD(oword, FRF_AZ_DRIVER_DW0) != EFX_BIU_MAGIC1) { + rc = EIO; + goto fail2; + } + + /* + * Perform the same test, with the values swapped. This + * ensures that subsequent tests don't start with the correct + * values already written into the scratch registers. + */ + EFX_POPULATE_OWORD_1(oword, FRF_AZ_DRIVER_DW0, EFX_BIU_MAGIC1); + EFX_BAR_TBL_WRITEO(enp, FR_AZ_DRIVER_REG, 0, &oword, B_TRUE); + + EFX_POPULATE_OWORD_1(oword, FRF_AZ_DRIVER_DW0, EFX_BIU_MAGIC0); + EFX_BAR_TBL_WRITEO(enp, FR_AZ_DRIVER_REG, 1, &oword, B_TRUE); + + EFX_BAR_TBL_READO(enp, FR_AZ_DRIVER_REG, 0, &oword, B_TRUE); + if (EFX_OWORD_FIELD(oword, FRF_AZ_DRIVER_DW0) != EFX_BIU_MAGIC1) { + rc = EIO; + goto fail3; + } + + EFX_BAR_TBL_READO(enp, FR_AZ_DRIVER_REG, 1, &oword, B_TRUE); + if (EFX_OWORD_FIELD(oword, FRF_AZ_DRIVER_DW0) != EFX_BIU_MAGIC0) { + rc = EIO; + goto fail4; + } + + return (0); + +fail4: + EFSYS_PROBE(fail4); +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +#if EFSYS_OPT_SIENA + +static const efx_nic_ops_t __efx_nic_siena_ops = { + siena_nic_probe, /* eno_probe */ + NULL, /* eno_board_cfg */ + NULL, /* eno_set_drv_limits */ + siena_nic_reset, /* eno_reset */ + siena_nic_init, /* eno_init */ + NULL, /* eno_get_vi_pool */ + NULL, /* eno_get_bar_region */ +#if EFSYS_OPT_DIAG + siena_nic_register_test, /* eno_register_test */ +#endif /* EFSYS_OPT_DIAG */ + siena_nic_fini, /* eno_fini */ + siena_nic_unprobe, /* eno_unprobe */ +}; + +#endif /* EFSYS_OPT_SIENA */ + +#if EFSYS_OPT_HUNTINGTON + +static const efx_nic_ops_t __efx_nic_hunt_ops = { + ef10_nic_probe, /* eno_probe */ + hunt_board_cfg, /* eno_board_cfg */ + ef10_nic_set_drv_limits, /* eno_set_drv_limits */ + ef10_nic_reset, /* eno_reset */ + ef10_nic_init, /* eno_init */ + ef10_nic_get_vi_pool, /* eno_get_vi_pool */ + ef10_nic_get_bar_region, /* eno_get_bar_region */ +#if EFSYS_OPT_DIAG + ef10_nic_register_test, /* eno_register_test */ +#endif /* EFSYS_OPT_DIAG */ + ef10_nic_fini, /* eno_fini */ + ef10_nic_unprobe, /* eno_unprobe */ +}; + +#endif /* EFSYS_OPT_HUNTINGTON */ + +#if EFSYS_OPT_MEDFORD + +static const efx_nic_ops_t __efx_nic_medford_ops = { + ef10_nic_probe, /* eno_probe */ + medford_board_cfg, /* eno_board_cfg */ + ef10_nic_set_drv_limits, /* eno_set_drv_limits */ + ef10_nic_reset, /* eno_reset */ + ef10_nic_init, /* eno_init */ + ef10_nic_get_vi_pool, /* eno_get_vi_pool */ + ef10_nic_get_bar_region, /* eno_get_bar_region */ +#if EFSYS_OPT_DIAG + ef10_nic_register_test, /* eno_register_test */ +#endif /* EFSYS_OPT_DIAG */ + ef10_nic_fini, /* eno_fini */ + ef10_nic_unprobe, /* eno_unprobe */ +}; + +#endif /* EFSYS_OPT_MEDFORD */ + + + __checkReturn efx_rc_t +efx_nic_create( + __in efx_family_t family, + __in efsys_identifier_t *esip, + __in efsys_bar_t *esbp, + __in efsys_lock_t *eslp, + __deref_out efx_nic_t **enpp) +{ + efx_nic_t *enp; + efx_rc_t rc; + + EFSYS_ASSERT3U(family, >, EFX_FAMILY_INVALID); + EFSYS_ASSERT3U(family, <, EFX_FAMILY_NTYPES); + + /* Allocate a NIC object */ + EFSYS_KMEM_ALLOC(esip, sizeof (efx_nic_t), enp); + + if (enp == NULL) { + rc = ENOMEM; + goto fail1; + } + + enp->en_magic = EFX_NIC_MAGIC; + + switch (family) { +#if EFSYS_OPT_SIENA + case EFX_FAMILY_SIENA: + enp->en_enop = &__efx_nic_siena_ops; + enp->en_features = + EFX_FEATURE_IPV6 | + EFX_FEATURE_LFSR_HASH_INSERT | + EFX_FEATURE_LINK_EVENTS | + EFX_FEATURE_PERIODIC_MAC_STATS | + EFX_FEATURE_MCDI | + EFX_FEATURE_LOOKAHEAD_SPLIT | + EFX_FEATURE_MAC_HEADER_FILTERS | + EFX_FEATURE_TX_SRC_FILTERS; + break; +#endif /* EFSYS_OPT_SIENA */ + +#if EFSYS_OPT_HUNTINGTON + case EFX_FAMILY_HUNTINGTON: + enp->en_enop = &__efx_nic_hunt_ops; + enp->en_features = + EFX_FEATURE_IPV6 | + EFX_FEATURE_LINK_EVENTS | + EFX_FEATURE_PERIODIC_MAC_STATS | + EFX_FEATURE_MCDI | + EFX_FEATURE_MAC_HEADER_FILTERS | + EFX_FEATURE_MCDI_DMA | + EFX_FEATURE_PIO_BUFFERS | + EFX_FEATURE_FW_ASSISTED_TSO | + EFX_FEATURE_FW_ASSISTED_TSO_V2 | + EFX_FEATURE_PACKED_STREAM; + break; +#endif /* EFSYS_OPT_HUNTINGTON */ + +#if EFSYS_OPT_MEDFORD + case EFX_FAMILY_MEDFORD: + enp->en_enop = &__efx_nic_medford_ops; + /* + * FW_ASSISTED_TSO omitted as Medford only supports firmware + * assisted TSO version 2, not the v1 scheme used on Huntington. + */ + enp->en_features = + EFX_FEATURE_IPV6 | + EFX_FEATURE_LINK_EVENTS | + EFX_FEATURE_PERIODIC_MAC_STATS | + EFX_FEATURE_MCDI | + EFX_FEATURE_MAC_HEADER_FILTERS | + EFX_FEATURE_MCDI_DMA | + EFX_FEATURE_PIO_BUFFERS | + EFX_FEATURE_FW_ASSISTED_TSO_V2 | + EFX_FEATURE_PACKED_STREAM; + break; +#endif /* EFSYS_OPT_MEDFORD */ + + default: + rc = ENOTSUP; + goto fail2; + } + + enp->en_family = family; + enp->en_esip = esip; + enp->en_esbp = esbp; + enp->en_eslp = eslp; + + *enpp = enp; + + return (0); + +fail2: + EFSYS_PROBE(fail2); + + enp->en_magic = 0; + + /* Free the NIC object */ + EFSYS_KMEM_FREE(esip, sizeof (efx_nic_t), enp); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_nic_probe( + __in efx_nic_t *enp) +{ + const efx_nic_ops_t *enop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); +#if EFSYS_OPT_MCDI + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_MCDI); +#endif /* EFSYS_OPT_MCDI */ + EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_PROBE)); + + enop = enp->en_enop; + if ((rc = enop->eno_probe(enp)) != 0) + goto fail1; + + if ((rc = efx_phy_probe(enp)) != 0) + goto fail2; + + enp->en_mod_flags |= EFX_MOD_PROBE; + + return (0); + +fail2: + EFSYS_PROBE(fail2); + + enop->eno_unprobe(enp); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_nic_set_drv_limits( + __inout efx_nic_t *enp, + __in efx_drv_limits_t *edlp) +{ + const efx_nic_ops_t *enop = enp->en_enop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE); + + if (enop->eno_set_drv_limits != NULL) { + if ((rc = enop->eno_set_drv_limits(enp, edlp)) != 0) + goto fail1; + } + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_nic_get_bar_region( + __in efx_nic_t *enp, + __in efx_nic_region_t region, + __out uint32_t *offsetp, + __out size_t *sizep) +{ + const efx_nic_ops_t *enop = enp->en_enop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); + + if (enop->eno_get_bar_region == NULL) { + rc = ENOTSUP; + goto fail1; + } + if ((rc = (enop->eno_get_bar_region)(enp, + region, offsetp, sizep)) != 0) { + goto fail2; + } + + return (0); + +fail2: + EFSYS_PROBE(fail2); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + + __checkReturn efx_rc_t +efx_nic_get_vi_pool( + __in efx_nic_t *enp, + __out uint32_t *evq_countp, + __out uint32_t *rxq_countp, + __out uint32_t *txq_countp) +{ + const efx_nic_ops_t *enop = enp->en_enop; + efx_nic_cfg_t *encp = &enp->en_nic_cfg; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); + + if (enop->eno_get_vi_pool != NULL) { + uint32_t vi_count = 0; + + if ((rc = (enop->eno_get_vi_pool)(enp, &vi_count)) != 0) + goto fail1; + + *evq_countp = vi_count; + *rxq_countp = vi_count; + *txq_countp = vi_count; + } else { + /* Use NIC limits as default value */ + *evq_countp = encp->enc_evq_limit; + *rxq_countp = encp->enc_rxq_limit; + *txq_countp = encp->enc_txq_limit; + } + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + + __checkReturn efx_rc_t +efx_nic_init( + __in efx_nic_t *enp) +{ + const efx_nic_ops_t *enop = enp->en_enop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE); + + if (enp->en_mod_flags & EFX_MOD_NIC) { + rc = EINVAL; + goto fail1; + } + + if ((rc = enop->eno_init(enp)) != 0) + goto fail2; + + enp->en_mod_flags |= EFX_MOD_NIC; + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + void +efx_nic_fini( + __in efx_nic_t *enp) +{ + const efx_nic_ops_t *enop = enp->en_enop; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT(enp->en_mod_flags & EFX_MOD_PROBE); + EFSYS_ASSERT(enp->en_mod_flags & EFX_MOD_NIC); + EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_INTR)); + EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_EV)); + EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_RX)); + EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_TX)); + + enop->eno_fini(enp); + + enp->en_mod_flags &= ~EFX_MOD_NIC; +} + + void +efx_nic_unprobe( + __in efx_nic_t *enp) +{ + const efx_nic_ops_t *enop = enp->en_enop; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); +#if EFSYS_OPT_MCDI + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_MCDI); +#endif /* EFSYS_OPT_MCDI */ + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE); + EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_NIC)); + EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_INTR)); + EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_EV)); + EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_RX)); + EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_TX)); + + efx_phy_unprobe(enp); + + enop->eno_unprobe(enp); + + enp->en_mod_flags &= ~EFX_MOD_PROBE; +} + + void +efx_nic_destroy( + __in efx_nic_t *enp) +{ + efsys_identifier_t *esip = enp->en_esip; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, ==, 0); + + enp->en_family = EFX_FAMILY_INVALID; + enp->en_esip = NULL; + enp->en_esbp = NULL; + enp->en_eslp = NULL; + + enp->en_enop = NULL; + + enp->en_magic = 0; + + /* Free the NIC object */ + EFSYS_KMEM_FREE(esip, sizeof (efx_nic_t), enp); +} + + __checkReturn efx_rc_t +efx_nic_reset( + __in efx_nic_t *enp) +{ + const efx_nic_ops_t *enop = enp->en_enop; + unsigned int mod_flags; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT(enp->en_mod_flags & EFX_MOD_PROBE); + /* + * All modules except the MCDI, PROBE, NVRAM, VPD, MON + * (which we do not reset here) must have been shut down or never + * initialized. + * + * A rule of thumb here is: If the controller or MC reboots, is *any* + * state lost. If it's lost and needs reapplying, then the module + * *must* not be initialised during the reset. + */ + mod_flags = enp->en_mod_flags; + mod_flags &= ~(EFX_MOD_MCDI | EFX_MOD_PROBE | EFX_MOD_NVRAM | + EFX_MOD_VPD | EFX_MOD_MON); + EFSYS_ASSERT3U(mod_flags, ==, 0); + if (mod_flags != 0) { + rc = EINVAL; + goto fail1; + } + + if ((rc = enop->eno_reset(enp)) != 0) + goto fail2; + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + const efx_nic_cfg_t * +efx_nic_cfg_get( + __in efx_nic_t *enp) +{ + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + + return (&(enp->en_nic_cfg)); +} + + __checkReturn efx_rc_t +efx_nic_get_fw_version( + __in efx_nic_t *enp, + __out efx_nic_fw_info_t *enfip) +{ + uint16_t mc_fw_version[4]; + efx_rc_t rc; + + if (enfip == NULL) { + rc = EINVAL; + goto fail1; + } + + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_MCDI); + EFSYS_ASSERT3U(enp->en_features, &, EFX_FEATURE_MCDI); + + rc = efx_mcdi_version(enp, mc_fw_version, NULL, NULL); + if (rc != 0) + goto fail2; + + rc = efx_mcdi_get_capabilities(enp, NULL, + &enfip->enfi_rx_dpcpu_fw_id, + &enfip->enfi_tx_dpcpu_fw_id, + NULL, NULL); + if (rc == 0) { + enfip->enfi_dpcpu_fw_ids_valid = B_TRUE; + } else if (rc == ENOTSUP) { + enfip->enfi_dpcpu_fw_ids_valid = B_FALSE; + enfip->enfi_rx_dpcpu_fw_id = 0; + enfip->enfi_tx_dpcpu_fw_id = 0; + } else { + goto fail3; + } + + memcpy(enfip->enfi_mc_fw_version, mc_fw_version, sizeof(mc_fw_version)); + + return (0); + +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +#if EFSYS_OPT_DIAG + + __checkReturn efx_rc_t +efx_nic_register_test( + __in efx_nic_t *enp) +{ + const efx_nic_ops_t *enop = enp->en_enop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE); + EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_NIC)); + + if ((rc = enop->eno_register_test(enp)) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_nic_test_registers( + __in efx_nic_t *enp, + __in efx_register_set_t *rsp, + __in size_t count) +{ + unsigned int bit; + efx_oword_t original; + efx_oword_t reg; + efx_oword_t buf; + efx_rc_t rc; + + while (count > 0) { + /* This function is only suitable for registers */ + EFSYS_ASSERT(rsp->rows == 1); + + /* bit sweep on and off */ + EFSYS_BAR_READO(enp->en_esbp, rsp->address, &original, + B_TRUE); + for (bit = 0; bit < 128; bit++) { + /* Is this bit in the mask? */ + if (~(rsp->mask.eo_u32[bit >> 5]) & (1 << bit)) + continue; + + /* Test this bit can be set in isolation */ + reg = original; + EFX_AND_OWORD(reg, rsp->mask); + EFX_SET_OWORD_BIT(reg, bit); + + EFSYS_BAR_WRITEO(enp->en_esbp, rsp->address, ®, + B_TRUE); + EFSYS_BAR_READO(enp->en_esbp, rsp->address, &buf, + B_TRUE); + + EFX_AND_OWORD(buf, rsp->mask); + if (memcmp(®, &buf, sizeof (reg))) { + rc = EIO; + goto fail1; + } + + /* Test this bit can be cleared in isolation */ + EFX_OR_OWORD(reg, rsp->mask); + EFX_CLEAR_OWORD_BIT(reg, bit); + + EFSYS_BAR_WRITEO(enp->en_esbp, rsp->address, ®, + B_TRUE); + EFSYS_BAR_READO(enp->en_esbp, rsp->address, &buf, + B_TRUE); + + EFX_AND_OWORD(buf, rsp->mask); + if (memcmp(®, &buf, sizeof (reg))) { + rc = EIO; + goto fail2; + } + } + + /* Restore the old value */ + EFSYS_BAR_WRITEO(enp->en_esbp, rsp->address, &original, + B_TRUE); + + --count; + ++rsp; + } + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + /* Restore the old value */ + EFSYS_BAR_WRITEO(enp->en_esbp, rsp->address, &original, B_TRUE); + + return (rc); +} + + __checkReturn efx_rc_t +efx_nic_test_tables( + __in efx_nic_t *enp, + __in efx_register_set_t *rsp, + __in efx_pattern_type_t pattern, + __in size_t count) +{ + efx_sram_pattern_fn_t func; + unsigned int index; + unsigned int address; + efx_oword_t reg; + efx_oword_t buf; + efx_rc_t rc; + + EFSYS_ASSERT(pattern < EFX_PATTERN_NTYPES); + func = __efx_sram_pattern_fns[pattern]; + + while (count > 0) { + /* Write */ + address = rsp->address; + for (index = 0; index < rsp->rows; ++index) { + func(2 * index + 0, B_FALSE, ®.eo_qword[0]); + func(2 * index + 1, B_FALSE, ®.eo_qword[1]); + EFX_AND_OWORD(reg, rsp->mask); + EFSYS_BAR_WRITEO(enp->en_esbp, address, ®, B_TRUE); + + address += rsp->step; + } + + /* Read */ + address = rsp->address; + for (index = 0; index < rsp->rows; ++index) { + func(2 * index + 0, B_FALSE, ®.eo_qword[0]); + func(2 * index + 1, B_FALSE, ®.eo_qword[1]); + EFX_AND_OWORD(reg, rsp->mask); + EFSYS_BAR_READO(enp->en_esbp, address, &buf, B_TRUE); + if (memcmp(®, &buf, sizeof (reg))) { + rc = EIO; + goto fail1; + } + + address += rsp->step; + } + + ++rsp; + --count; + } + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +#endif /* EFSYS_OPT_DIAG */ + +#if EFSYS_OPT_LOOPBACK + +extern void +efx_loopback_mask( + __in efx_loopback_kind_t loopback_kind, + __out efx_qword_t *maskp) +{ + efx_qword_t mask; + + EFSYS_ASSERT3U(loopback_kind, <, EFX_LOOPBACK_NKINDS); + EFSYS_ASSERT(maskp != NULL); + + /* Assert the MC_CMD_LOOPBACK and EFX_LOOPBACK namespace agree */ + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_NONE == EFX_LOOPBACK_OFF); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_DATA == EFX_LOOPBACK_DATA); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_GMAC == EFX_LOOPBACK_GMAC); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_XGMII == EFX_LOOPBACK_XGMII); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_XGXS == EFX_LOOPBACK_XGXS); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_XAUI == EFX_LOOPBACK_XAUI); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_GMII == EFX_LOOPBACK_GMII); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_SGMII == EFX_LOOPBACK_SGMII); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_XGBR == EFX_LOOPBACK_XGBR); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_XFI == EFX_LOOPBACK_XFI); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_XAUI_FAR == EFX_LOOPBACK_XAUI_FAR); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_GMII_FAR == EFX_LOOPBACK_GMII_FAR); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_SGMII_FAR == EFX_LOOPBACK_SGMII_FAR); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_XFI_FAR == EFX_LOOPBACK_XFI_FAR); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_GPHY == EFX_LOOPBACK_GPHY); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_PHYXS == EFX_LOOPBACK_PHY_XS); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_PCS == EFX_LOOPBACK_PCS); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_PMAPMD == EFX_LOOPBACK_PMA_PMD); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_XPORT == EFX_LOOPBACK_XPORT); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_XGMII_WS == EFX_LOOPBACK_XGMII_WS); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_XAUI_WS == EFX_LOOPBACK_XAUI_WS); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_XAUI_WS_FAR == + EFX_LOOPBACK_XAUI_WS_FAR); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_XAUI_WS_NEAR == + EFX_LOOPBACK_XAUI_WS_NEAR); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_GMII_WS == EFX_LOOPBACK_GMII_WS); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_XFI_WS == EFX_LOOPBACK_XFI_WS); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_XFI_WS_FAR == + EFX_LOOPBACK_XFI_WS_FAR); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_PHYXS_WS == EFX_LOOPBACK_PHYXS_WS); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_PMA_INT == EFX_LOOPBACK_PMA_INT); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_SD_NEAR == EFX_LOOPBACK_SD_NEAR); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_SD_FAR == EFX_LOOPBACK_SD_FAR); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_PMA_INT_WS == + EFX_LOOPBACK_PMA_INT_WS); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_SD_FEP2_WS == + EFX_LOOPBACK_SD_FEP2_WS); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_SD_FEP1_5_WS == + EFX_LOOPBACK_SD_FEP1_5_WS); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_SD_FEP_WS == EFX_LOOPBACK_SD_FEP_WS); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_SD_FES_WS == EFX_LOOPBACK_SD_FES_WS); + + /* Build bitmask of possible loopback types */ + EFX_ZERO_QWORD(mask); + + if ((loopback_kind == EFX_LOOPBACK_KIND_OFF) || + (loopback_kind == EFX_LOOPBACK_KIND_ALL)) { + EFX_SET_QWORD_BIT(mask, EFX_LOOPBACK_OFF); + } + + if ((loopback_kind == EFX_LOOPBACK_KIND_MAC) || + (loopback_kind == EFX_LOOPBACK_KIND_ALL)) { + /* + * The "MAC" grouping has historically been used by drivers to + * mean loopbacks supported by on-chip hardware. Keep that + * meaning here, and include on-chip PHY layer loopbacks. + */ + EFX_SET_QWORD_BIT(mask, EFX_LOOPBACK_DATA); + EFX_SET_QWORD_BIT(mask, EFX_LOOPBACK_GMAC); + EFX_SET_QWORD_BIT(mask, EFX_LOOPBACK_XGMII); + EFX_SET_QWORD_BIT(mask, EFX_LOOPBACK_XGXS); + EFX_SET_QWORD_BIT(mask, EFX_LOOPBACK_XAUI); + EFX_SET_QWORD_BIT(mask, EFX_LOOPBACK_GMII); + EFX_SET_QWORD_BIT(mask, EFX_LOOPBACK_SGMII); + EFX_SET_QWORD_BIT(mask, EFX_LOOPBACK_XGBR); + EFX_SET_QWORD_BIT(mask, EFX_LOOPBACK_XFI); + EFX_SET_QWORD_BIT(mask, EFX_LOOPBACK_XAUI_FAR); + EFX_SET_QWORD_BIT(mask, EFX_LOOPBACK_GMII_FAR); + EFX_SET_QWORD_BIT(mask, EFX_LOOPBACK_SGMII_FAR); + EFX_SET_QWORD_BIT(mask, EFX_LOOPBACK_XFI_FAR); + EFX_SET_QWORD_BIT(mask, EFX_LOOPBACK_PMA_INT); + EFX_SET_QWORD_BIT(mask, EFX_LOOPBACK_SD_NEAR); + EFX_SET_QWORD_BIT(mask, EFX_LOOPBACK_SD_FAR); + } + + if ((loopback_kind == EFX_LOOPBACK_KIND_PHY) || + (loopback_kind == EFX_LOOPBACK_KIND_ALL)) { + /* + * The "PHY" grouping has historically been used by drivers to + * mean loopbacks supported by off-chip hardware. Keep that + * meaning here. + */ + EFX_SET_QWORD_BIT(mask, EFX_LOOPBACK_GPHY); + EFX_SET_QWORD_BIT(mask, EFX_LOOPBACK_PHY_XS); + EFX_SET_QWORD_BIT(mask, EFX_LOOPBACK_PCS); + EFX_SET_QWORD_BIT(mask, EFX_LOOPBACK_PMA_PMD); + } + + *maskp = mask; +} + + __checkReturn efx_rc_t +efx_mcdi_get_loopback_modes( + __in efx_nic_t *enp) +{ + efx_nic_cfg_t *encp = &(enp->en_nic_cfg); + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_GET_LOOPBACK_MODES_IN_LEN, + MC_CMD_GET_LOOPBACK_MODES_OUT_LEN)]; + efx_qword_t mask; + efx_qword_t modes; + efx_rc_t rc; + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_GET_LOOPBACK_MODES; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_GET_LOOPBACK_MODES_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_GET_LOOPBACK_MODES_OUT_LEN; + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + if (req.emr_out_length_used < + MC_CMD_GET_LOOPBACK_MODES_OUT_SUGGESTED_OFST + + MC_CMD_GET_LOOPBACK_MODES_OUT_SUGGESTED_LEN) { + rc = EMSGSIZE; + goto fail2; + } + + /* + * We assert the MC_CMD_LOOPBACK and EFX_LOOPBACK namespaces agree + * in efx_loopback_mask() and in siena_phy.c:siena_phy_get_link(). + */ + efx_loopback_mask(EFX_LOOPBACK_KIND_ALL, &mask); + + EFX_AND_QWORD(mask, + *MCDI_OUT2(req, efx_qword_t, GET_LOOPBACK_MODES_OUT_SUGGESTED)); + + modes = *MCDI_OUT2(req, efx_qword_t, GET_LOOPBACK_MODES_OUT_100M); + EFX_AND_QWORD(modes, mask); + encp->enc_loopback_types[EFX_LINK_100FDX] = modes; + + modes = *MCDI_OUT2(req, efx_qword_t, GET_LOOPBACK_MODES_OUT_1G); + EFX_AND_QWORD(modes, mask); + encp->enc_loopback_types[EFX_LINK_1000FDX] = modes; + + modes = *MCDI_OUT2(req, efx_qword_t, GET_LOOPBACK_MODES_OUT_10G); + EFX_AND_QWORD(modes, mask); + encp->enc_loopback_types[EFX_LINK_10000FDX] = modes; + + if (req.emr_out_length_used >= + MC_CMD_GET_LOOPBACK_MODES_OUT_40G_OFST + + MC_CMD_GET_LOOPBACK_MODES_OUT_40G_LEN) { + /* Response includes 40G loopback modes */ + modes = + *MCDI_OUT2(req, efx_qword_t, GET_LOOPBACK_MODES_OUT_40G); + EFX_AND_QWORD(modes, mask); + encp->enc_loopback_types[EFX_LINK_40000FDX] = modes; + } + + EFX_ZERO_QWORD(modes); + EFX_SET_QWORD_BIT(modes, EFX_LOOPBACK_OFF); + EFX_OR_QWORD(modes, encp->enc_loopback_types[EFX_LINK_100FDX]); + EFX_OR_QWORD(modes, encp->enc_loopback_types[EFX_LINK_1000FDX]); + EFX_OR_QWORD(modes, encp->enc_loopback_types[EFX_LINK_10000FDX]); + EFX_OR_QWORD(modes, encp->enc_loopback_types[EFX_LINK_40000FDX]); + encp->enc_loopback_types[EFX_LINK_UNKNOWN] = modes; + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +#endif /* EFSYS_OPT_LOOPBACK */ + + __checkReturn efx_rc_t +efx_nic_calculate_pcie_link_bandwidth( + __in uint32_t pcie_link_width, + __in uint32_t pcie_link_gen, + __out uint32_t *bandwidth_mbpsp) +{ + uint32_t lane_bandwidth; + uint32_t total_bandwidth; + efx_rc_t rc; + + if ((pcie_link_width == 0) || (pcie_link_width > 16) || + !ISP2(pcie_link_width)) { + rc = EINVAL; + goto fail1; + } + + switch (pcie_link_gen) { + case EFX_PCIE_LINK_SPEED_GEN1: + /* 2.5 Gb/s raw bandwidth with 8b/10b encoding */ + lane_bandwidth = 2000; + break; + case EFX_PCIE_LINK_SPEED_GEN2: + /* 5.0 Gb/s raw bandwidth with 8b/10b encoding */ + lane_bandwidth = 4000; + break; + case EFX_PCIE_LINK_SPEED_GEN3: + /* 8.0 Gb/s raw bandwidth with 128b/130b encoding */ + lane_bandwidth = 7877; + break; + default: + rc = EINVAL; + goto fail2; + } + + total_bandwidth = lane_bandwidth * pcie_link_width; + *bandwidth_mbpsp = total_bandwidth; + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + + __checkReturn efx_rc_t +efx_nic_check_pcie_link_speed( + __in efx_nic_t *enp, + __in uint32_t pcie_link_width, + __in uint32_t pcie_link_gen, + __out efx_pcie_link_performance_t *resultp) +{ + efx_nic_cfg_t *encp = &(enp->en_nic_cfg); + uint32_t bandwidth; + efx_pcie_link_performance_t result; + efx_rc_t rc; + + if ((encp->enc_required_pcie_bandwidth_mbps == 0) || + (pcie_link_width == 0) || (pcie_link_width == 32) || + (pcie_link_gen == 0)) { + /* + * No usable info on what is required and/or in use. In virtual + * machines, sometimes the PCIe link width is reported as 0 or + * 32, or the speed as 0. + */ + result = EFX_PCIE_LINK_PERFORMANCE_UNKNOWN_BANDWIDTH; + goto out; + } + + /* Calculate the available bandwidth in megabits per second */ + rc = efx_nic_calculate_pcie_link_bandwidth(pcie_link_width, + pcie_link_gen, &bandwidth); + if (rc != 0) + goto fail1; + + if (bandwidth < encp->enc_required_pcie_bandwidth_mbps) { + result = EFX_PCIE_LINK_PERFORMANCE_SUBOPTIMAL_BANDWIDTH; + } else if (pcie_link_gen < encp->enc_max_pcie_link_gen) { + /* The link provides enough bandwidth but not optimal latency */ + result = EFX_PCIE_LINK_PERFORMANCE_SUBOPTIMAL_LATENCY; + } else { + result = EFX_PCIE_LINK_PERFORMANCE_OPTIMAL; + } + +out: + *resultp = result; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} diff --git a/dpdk/drivers/net/sfc/base/efx_nvram.c b/dpdk/drivers/net/sfc/base/efx_nvram.c new file mode 100644 index 00000000..6ee2a71d --- /dev/null +++ b/dpdk/drivers/net/sfc/base/efx_nvram.c @@ -0,0 +1,1044 @@ +/* + * Copyright (c) 2009-2016 Solarflare Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are + * those of the authors and should not be interpreted as representing official + * policies, either expressed or implied, of the FreeBSD Project. + */ + +#include "efx.h" +#include "efx_impl.h" + +#if EFSYS_OPT_NVRAM + +#if EFSYS_OPT_SIENA + +static const efx_nvram_ops_t __efx_nvram_siena_ops = { +#if EFSYS_OPT_DIAG + siena_nvram_test, /* envo_test */ +#endif /* EFSYS_OPT_DIAG */ + siena_nvram_type_to_partn, /* envo_type_to_partn */ + siena_nvram_partn_size, /* envo_partn_size */ + siena_nvram_partn_rw_start, /* envo_partn_rw_start */ + siena_nvram_partn_read, /* envo_partn_read */ + siena_nvram_partn_erase, /* envo_partn_erase */ + siena_nvram_partn_write, /* envo_partn_write */ + siena_nvram_partn_rw_finish, /* envo_partn_rw_finish */ + siena_nvram_partn_get_version, /* envo_partn_get_version */ + siena_nvram_partn_set_version, /* envo_partn_set_version */ + NULL, /* envo_partn_validate */ +}; + +#endif /* EFSYS_OPT_SIENA */ + +#if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD + +static const efx_nvram_ops_t __efx_nvram_ef10_ops = { +#if EFSYS_OPT_DIAG + ef10_nvram_test, /* envo_test */ +#endif /* EFSYS_OPT_DIAG */ + ef10_nvram_type_to_partn, /* envo_type_to_partn */ + ef10_nvram_partn_size, /* envo_partn_size */ + ef10_nvram_partn_rw_start, /* envo_partn_rw_start */ + ef10_nvram_partn_read, /* envo_partn_read */ + ef10_nvram_partn_erase, /* envo_partn_erase */ + ef10_nvram_partn_write, /* envo_partn_write */ + ef10_nvram_partn_rw_finish, /* envo_partn_rw_finish */ + ef10_nvram_partn_get_version, /* envo_partn_get_version */ + ef10_nvram_partn_set_version, /* envo_partn_set_version */ + ef10_nvram_buffer_validate, /* envo_buffer_validate */ +}; + +#endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */ + + __checkReturn efx_rc_t +efx_nvram_init( + __in efx_nic_t *enp) +{ + const efx_nvram_ops_t *envop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE); + EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_NVRAM)); + + switch (enp->en_family) { +#if EFSYS_OPT_SIENA + case EFX_FAMILY_SIENA: + envop = &__efx_nvram_siena_ops; + break; +#endif /* EFSYS_OPT_SIENA */ + +#if EFSYS_OPT_HUNTINGTON + case EFX_FAMILY_HUNTINGTON: + envop = &__efx_nvram_ef10_ops; + break; +#endif /* EFSYS_OPT_HUNTINGTON */ + +#if EFSYS_OPT_MEDFORD + case EFX_FAMILY_MEDFORD: + envop = &__efx_nvram_ef10_ops; + break; +#endif /* EFSYS_OPT_MEDFORD */ + + default: + EFSYS_ASSERT(0); + rc = ENOTSUP; + goto fail1; + } + + enp->en_envop = envop; + enp->en_mod_flags |= EFX_MOD_NVRAM; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +#if EFSYS_OPT_DIAG + + __checkReturn efx_rc_t +efx_nvram_test( + __in efx_nic_t *enp) +{ + const efx_nvram_ops_t *envop = enp->en_envop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NVRAM); + + if ((rc = envop->envo_test(enp)) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +#endif /* EFSYS_OPT_DIAG */ + + __checkReturn efx_rc_t +efx_nvram_size( + __in efx_nic_t *enp, + __in efx_nvram_type_t type, + __out size_t *sizep) +{ + const efx_nvram_ops_t *envop = enp->en_envop; + uint32_t partn; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NVRAM); + + EFSYS_ASSERT3U(type, <, EFX_NVRAM_NTYPES); + + if ((rc = envop->envo_type_to_partn(enp, type, &partn)) != 0) + goto fail1; + + if ((rc = envop->envo_partn_size(enp, partn, sizep)) != 0) + goto fail2; + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + *sizep = 0; + + return (rc); +} + + __checkReturn efx_rc_t +efx_nvram_get_version( + __in efx_nic_t *enp, + __in efx_nvram_type_t type, + __out uint32_t *subtypep, + __out_ecount(4) uint16_t version[4]) +{ + const efx_nvram_ops_t *envop = enp->en_envop; + uint32_t partn; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NVRAM); + + EFSYS_ASSERT3U(type, <, EFX_NVRAM_NTYPES); + + if ((rc = envop->envo_type_to_partn(enp, type, &partn)) != 0) + goto fail1; + + if ((rc = envop->envo_partn_get_version(enp, partn, + subtypep, version)) != 0) + goto fail2; + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_nvram_rw_start( + __in efx_nic_t *enp, + __in efx_nvram_type_t type, + __out_opt size_t *chunk_sizep) +{ + const efx_nvram_ops_t *envop = enp->en_envop; + uint32_t partn; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NVRAM); + + EFSYS_ASSERT3U(type, <, EFX_NVRAM_NTYPES); + EFSYS_ASSERT3U(type, !=, EFX_NVRAM_INVALID); + + EFSYS_ASSERT3U(enp->en_nvram_locked, ==, EFX_NVRAM_INVALID); + + if ((rc = envop->envo_type_to_partn(enp, type, &partn)) != 0) + goto fail1; + + if ((rc = envop->envo_partn_rw_start(enp, partn, chunk_sizep)) != 0) + goto fail2; + + enp->en_nvram_locked = type; + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_nvram_read_chunk( + __in efx_nic_t *enp, + __in efx_nvram_type_t type, + __in unsigned int offset, + __out_bcount(size) caddr_t data, + __in size_t size) +{ + const efx_nvram_ops_t *envop = enp->en_envop; + uint32_t partn; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NVRAM); + + EFSYS_ASSERT3U(type, <, EFX_NVRAM_NTYPES); + EFSYS_ASSERT3U(type, !=, EFX_NVRAM_INVALID); + + EFSYS_ASSERT3U(enp->en_nvram_locked, ==, type); + + if ((rc = envop->envo_type_to_partn(enp, type, &partn)) != 0) + goto fail1; + + if ((rc = envop->envo_partn_read(enp, partn, offset, data, size)) != 0) + goto fail2; + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_nvram_erase( + __in efx_nic_t *enp, + __in efx_nvram_type_t type) +{ + const efx_nvram_ops_t *envop = enp->en_envop; + unsigned int offset = 0; + size_t size = 0; + uint32_t partn; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NVRAM); + + EFSYS_ASSERT3U(type, <, EFX_NVRAM_NTYPES); + EFSYS_ASSERT3U(type, !=, EFX_NVRAM_INVALID); + + EFSYS_ASSERT3U(enp->en_nvram_locked, ==, type); + + if ((rc = envop->envo_type_to_partn(enp, type, &partn)) != 0) + goto fail1; + + if ((rc = envop->envo_partn_size(enp, partn, &size)) != 0) + goto fail2; + + if ((rc = envop->envo_partn_erase(enp, partn, offset, size)) != 0) + goto fail3; + + return (0); + +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_nvram_write_chunk( + __in efx_nic_t *enp, + __in efx_nvram_type_t type, + __in unsigned int offset, + __in_bcount(size) caddr_t data, + __in size_t size) +{ + const efx_nvram_ops_t *envop = enp->en_envop; + uint32_t partn; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NVRAM); + + EFSYS_ASSERT3U(type, <, EFX_NVRAM_NTYPES); + EFSYS_ASSERT3U(type, !=, EFX_NVRAM_INVALID); + + EFSYS_ASSERT3U(enp->en_nvram_locked, ==, type); + + if ((rc = envop->envo_type_to_partn(enp, type, &partn)) != 0) + goto fail1; + + if ((rc = envop->envo_partn_write(enp, partn, offset, data, size)) != 0) + goto fail2; + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_nvram_rw_finish( + __in efx_nic_t *enp, + __in efx_nvram_type_t type) +{ + const efx_nvram_ops_t *envop = enp->en_envop; + uint32_t partn; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NVRAM); + + EFSYS_ASSERT3U(type, <, EFX_NVRAM_NTYPES); + EFSYS_ASSERT3U(type, !=, EFX_NVRAM_INVALID); + + EFSYS_ASSERT3U(enp->en_nvram_locked, ==, type); + + if ((rc = envop->envo_type_to_partn(enp, type, &partn)) != 0) + goto fail1; + + if ((rc = envop->envo_partn_rw_finish(enp, partn)) != 0) + goto fail2; + + enp->en_nvram_locked = EFX_NVRAM_INVALID; + + return (0); + +fail2: + EFSYS_PROBE(fail2); + enp->en_nvram_locked = EFX_NVRAM_INVALID; + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_nvram_set_version( + __in efx_nic_t *enp, + __in efx_nvram_type_t type, + __in_ecount(4) uint16_t version[4]) +{ + const efx_nvram_ops_t *envop = enp->en_envop; + uint32_t partn; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NVRAM); + + EFSYS_ASSERT3U(type, <, EFX_NVRAM_NTYPES); + + /* + * The Siena implementation of envo_set_version() will attempt to + * acquire the NVRAM_UPDATE lock for the DYNAMIC_CONFIG sector. + * Therefore, you can't have already acquired the NVRAM_UPDATE lock. + */ + EFSYS_ASSERT3U(enp->en_nvram_locked, ==, EFX_NVRAM_INVALID); + + if ((rc = envop->envo_type_to_partn(enp, type, &partn)) != 0) + goto fail1; + + if ((rc = envop->envo_partn_set_version(enp, partn, version)) != 0) + goto fail2; + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +/* Validate buffer contents (before writing to flash) */ + __checkReturn efx_rc_t +efx_nvram_validate( + __in efx_nic_t *enp, + __in efx_nvram_type_t type, + __in_bcount(partn_size) caddr_t partn_data, + __in size_t partn_size) +{ + const efx_nvram_ops_t *envop = enp->en_envop; + uint32_t partn; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NVRAM); + + EFSYS_ASSERT3U(type, <, EFX_NVRAM_NTYPES); + + + if ((rc = envop->envo_type_to_partn(enp, type, &partn)) != 0) + goto fail1; + + if (envop->envo_type_to_partn != NULL && + ((rc = envop->envo_buffer_validate(enp, partn, + partn_data, partn_size)) != 0)) + goto fail2; + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + +void +efx_nvram_fini( + __in efx_nic_t *enp) +{ + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NVRAM); + + EFSYS_ASSERT3U(enp->en_nvram_locked, ==, EFX_NVRAM_INVALID); + + enp->en_envop = NULL; + enp->en_mod_flags &= ~EFX_MOD_NVRAM; +} + +#endif /* EFSYS_OPT_NVRAM */ + +#if EFSYS_OPT_NVRAM || EFSYS_OPT_VPD + +/* + * Internal MCDI request handling + */ + + __checkReturn efx_rc_t +efx_mcdi_nvram_partitions( + __in efx_nic_t *enp, + __out_bcount(size) caddr_t data, + __in size_t size, + __out unsigned int *npartnp) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_NVRAM_PARTITIONS_IN_LEN, + MC_CMD_NVRAM_PARTITIONS_OUT_LENMAX)]; + unsigned int npartn; + efx_rc_t rc; + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_NVRAM_PARTITIONS; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_NVRAM_PARTITIONS_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_NVRAM_PARTITIONS_OUT_LENMAX; + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + if (req.emr_out_length_used < MC_CMD_NVRAM_PARTITIONS_OUT_LENMIN) { + rc = EMSGSIZE; + goto fail2; + } + npartn = MCDI_OUT_DWORD(req, NVRAM_PARTITIONS_OUT_NUM_PARTITIONS); + + if (req.emr_out_length_used < MC_CMD_NVRAM_PARTITIONS_OUT_LEN(npartn)) { + rc = ENOENT; + goto fail3; + } + + if (size < npartn * sizeof (uint32_t)) { + rc = ENOSPC; + goto fail3; + } + + *npartnp = npartn; + + memcpy(data, + MCDI_OUT2(req, uint32_t, NVRAM_PARTITIONS_OUT_TYPE_ID), + (npartn * sizeof (uint32_t))); + + return (0); + +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_mcdi_nvram_metadata( + __in efx_nic_t *enp, + __in uint32_t partn, + __out uint32_t *subtypep, + __out_ecount(4) uint16_t version[4], + __out_bcount_opt(size) char *descp, + __in size_t size) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_NVRAM_METADATA_IN_LEN, + MC_CMD_NVRAM_METADATA_OUT_LENMAX)]; + efx_rc_t rc; + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_NVRAM_METADATA; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_NVRAM_METADATA_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_NVRAM_METADATA_OUT_LENMAX; + + MCDI_IN_SET_DWORD(req, NVRAM_METADATA_IN_TYPE, partn); + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + if (req.emr_out_length_used < MC_CMD_NVRAM_METADATA_OUT_LENMIN) { + rc = EMSGSIZE; + goto fail2; + } + + if (MCDI_OUT_DWORD_FIELD(req, NVRAM_METADATA_OUT_FLAGS, + NVRAM_METADATA_OUT_SUBTYPE_VALID)) { + *subtypep = MCDI_OUT_DWORD(req, NVRAM_METADATA_OUT_SUBTYPE); + } else { + *subtypep = 0; + } + + if (MCDI_OUT_DWORD_FIELD(req, NVRAM_METADATA_OUT_FLAGS, + NVRAM_METADATA_OUT_VERSION_VALID)) { + version[0] = MCDI_OUT_WORD(req, NVRAM_METADATA_OUT_VERSION_W); + version[1] = MCDI_OUT_WORD(req, NVRAM_METADATA_OUT_VERSION_X); + version[2] = MCDI_OUT_WORD(req, NVRAM_METADATA_OUT_VERSION_Y); + version[3] = MCDI_OUT_WORD(req, NVRAM_METADATA_OUT_VERSION_Z); + } else { + version[0] = version[1] = version[2] = version[3] = 0; + } + + if (MCDI_OUT_DWORD_FIELD(req, NVRAM_METADATA_OUT_FLAGS, + NVRAM_METADATA_OUT_DESCRIPTION_VALID)) { + /* Return optional descrition string */ + if ((descp != NULL) && (size > 0)) { + size_t desclen; + + descp[0] = '\0'; + desclen = (req.emr_out_length_used + - MC_CMD_NVRAM_METADATA_OUT_LEN(0)); + + EFSYS_ASSERT3U(desclen, <=, + MC_CMD_NVRAM_METADATA_OUT_DESCRIPTION_MAXNUM); + + if (size < desclen) { + rc = ENOSPC; + goto fail3; + } + + memcpy(descp, MCDI_OUT2(req, char, + NVRAM_METADATA_OUT_DESCRIPTION), + desclen); + + /* Ensure string is NUL terminated */ + descp[desclen] = '\0'; + } + } + + return (0); + +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_mcdi_nvram_info( + __in efx_nic_t *enp, + __in uint32_t partn, + __out_opt size_t *sizep, + __out_opt uint32_t *addressp, + __out_opt uint32_t *erase_sizep, + __out_opt uint32_t *write_sizep) +{ + uint8_t payload[MAX(MC_CMD_NVRAM_INFO_IN_LEN, + MC_CMD_NVRAM_INFO_V2_OUT_LEN)]; + efx_mcdi_req_t req; + efx_rc_t rc; + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_NVRAM_INFO; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_NVRAM_INFO_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_NVRAM_INFO_V2_OUT_LEN; + + MCDI_IN_SET_DWORD(req, NVRAM_INFO_IN_TYPE, partn); + + efx_mcdi_execute_quiet(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + if (req.emr_out_length_used < MC_CMD_NVRAM_INFO_OUT_LEN) { + rc = EMSGSIZE; + goto fail2; + } + + if (sizep) + *sizep = MCDI_OUT_DWORD(req, NVRAM_INFO_OUT_SIZE); + + if (addressp) + *addressp = MCDI_OUT_DWORD(req, NVRAM_INFO_OUT_PHYSADDR); + + if (erase_sizep) + *erase_sizep = MCDI_OUT_DWORD(req, NVRAM_INFO_OUT_ERASESIZE); + + if (write_sizep) { + *write_sizep = + (req.emr_out_length_used < + MC_CMD_NVRAM_INFO_V2_OUT_LEN) ? + 0 : MCDI_OUT_DWORD(req, NVRAM_INFO_V2_OUT_WRITESIZE); + } + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +/* + * MC_CMD_NVRAM_UPDATE_START_V2 must be used to support firmware-verified + * NVRAM updates. Older firmware will ignore the flags field in the request. + */ + __checkReturn efx_rc_t +efx_mcdi_nvram_update_start( + __in efx_nic_t *enp, + __in uint32_t partn) +{ + uint8_t payload[MAX(MC_CMD_NVRAM_UPDATE_START_V2_IN_LEN, + MC_CMD_NVRAM_UPDATE_START_OUT_LEN)]; + efx_mcdi_req_t req; + efx_rc_t rc; + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_NVRAM_UPDATE_START; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_NVRAM_UPDATE_START_V2_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_NVRAM_UPDATE_START_OUT_LEN; + + MCDI_IN_SET_DWORD(req, NVRAM_UPDATE_START_V2_IN_TYPE, partn); + + MCDI_IN_POPULATE_DWORD_1(req, NVRAM_UPDATE_START_V2_IN_FLAGS, + NVRAM_UPDATE_START_V2_IN_FLAG_REPORT_VERIFY_RESULT, 1); + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_mcdi_nvram_read( + __in efx_nic_t *enp, + __in uint32_t partn, + __in uint32_t offset, + __out_bcount(size) caddr_t data, + __in size_t size, + __in uint32_t mode) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_NVRAM_READ_IN_V2_LEN, + MC_CMD_NVRAM_READ_OUT_LENMAX)]; + efx_rc_t rc; + + if (size > MC_CMD_NVRAM_READ_OUT_LENMAX) { + rc = EINVAL; + goto fail1; + } + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_NVRAM_READ; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_NVRAM_READ_IN_V2_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_NVRAM_READ_OUT_LENMAX; + + MCDI_IN_SET_DWORD(req, NVRAM_READ_IN_V2_TYPE, partn); + MCDI_IN_SET_DWORD(req, NVRAM_READ_IN_V2_OFFSET, offset); + MCDI_IN_SET_DWORD(req, NVRAM_READ_IN_V2_LENGTH, size); + MCDI_IN_SET_DWORD(req, NVRAM_READ_IN_V2_MODE, mode); + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + if (req.emr_out_length_used < MC_CMD_NVRAM_READ_OUT_LEN(size)) { + rc = EMSGSIZE; + goto fail2; + } + + memcpy(data, + MCDI_OUT2(req, uint8_t, NVRAM_READ_OUT_READ_BUFFER), + size); + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_mcdi_nvram_erase( + __in efx_nic_t *enp, + __in uint32_t partn, + __in uint32_t offset, + __in size_t size) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_NVRAM_ERASE_IN_LEN, + MC_CMD_NVRAM_ERASE_OUT_LEN)]; + efx_rc_t rc; + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_NVRAM_ERASE; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_NVRAM_ERASE_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_NVRAM_ERASE_OUT_LEN; + + MCDI_IN_SET_DWORD(req, NVRAM_ERASE_IN_TYPE, partn); + MCDI_IN_SET_DWORD(req, NVRAM_ERASE_IN_OFFSET, offset); + MCDI_IN_SET_DWORD(req, NVRAM_ERASE_IN_LENGTH, size); + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +/* + * The NVRAM_WRITE MCDI command is a V1 command and so is supported by both + * Sienna and EF10 based boards. However EF10 based boards support the use + * of this command with payloads up to the maximum MCDI V2 payload length. + */ + __checkReturn efx_rc_t +efx_mcdi_nvram_write( + __in efx_nic_t *enp, + __in uint32_t partn, + __in uint32_t offset, + __out_bcount(size) caddr_t data, + __in size_t size) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MCDI_CTL_SDU_LEN_MAX_V1, + MCDI_CTL_SDU_LEN_MAX_V2)]; + efx_rc_t rc; + size_t max_data_size; + + max_data_size = enp->en_nic_cfg.enc_mcdi_max_payload_length + - MC_CMD_NVRAM_WRITE_IN_LEN(0); + EFSYS_ASSERT3U(enp->en_nic_cfg.enc_mcdi_max_payload_length, >, 0); + EFSYS_ASSERT3U(max_data_size, <, + enp->en_nic_cfg.enc_mcdi_max_payload_length); + + if (size > max_data_size) { + rc = EINVAL; + goto fail1; + } + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_NVRAM_WRITE; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_NVRAM_WRITE_IN_LEN(size); + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_NVRAM_WRITE_OUT_LEN; + + MCDI_IN_SET_DWORD(req, NVRAM_WRITE_IN_TYPE, partn); + MCDI_IN_SET_DWORD(req, NVRAM_WRITE_IN_OFFSET, offset); + MCDI_IN_SET_DWORD(req, NVRAM_WRITE_IN_LENGTH, size); + + memcpy(MCDI_IN2(req, uint8_t, NVRAM_WRITE_IN_WRITE_BUFFER), + data, size); + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail2; + } + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + +/* + * MC_CMD_NVRAM_UPDATE_FINISH_V2 must be used to support firmware-verified + * NVRAM updates. Older firmware will ignore the flags field in the request. + */ + __checkReturn efx_rc_t +efx_mcdi_nvram_update_finish( + __in efx_nic_t *enp, + __in uint32_t partn, + __in boolean_t reboot, + __out_opt uint32_t *resultp) +{ + const efx_nic_cfg_t *encp = &enp->en_nic_cfg; + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_NVRAM_UPDATE_FINISH_V2_IN_LEN, + MC_CMD_NVRAM_UPDATE_FINISH_V2_OUT_LEN)]; + uint32_t result = 0; /* FIXME: use MC_CMD_NVRAM_VERIFY_RC_UNKNOWN */ + efx_rc_t rc; + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_NVRAM_UPDATE_FINISH; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_NVRAM_UPDATE_FINISH_V2_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_NVRAM_UPDATE_FINISH_V2_OUT_LEN; + + MCDI_IN_SET_DWORD(req, NVRAM_UPDATE_FINISH_V2_IN_TYPE, partn); + MCDI_IN_SET_DWORD(req, NVRAM_UPDATE_FINISH_V2_IN_REBOOT, reboot); + + MCDI_IN_POPULATE_DWORD_1(req, NVRAM_UPDATE_FINISH_V2_IN_FLAGS, + NVRAM_UPDATE_FINISH_V2_IN_FLAG_REPORT_VERIFY_RESULT, 1); + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + if (encp->enc_fw_verified_nvram_update_required == B_FALSE) { + /* Report success if verified updates are not supported. */ + result = MC_CMD_NVRAM_VERIFY_RC_SUCCESS; + } else { + /* Firmware-verified NVRAM updates are required */ + if (req.emr_out_length_used < + MC_CMD_NVRAM_UPDATE_FINISH_V2_OUT_LEN) { + rc = EMSGSIZE; + goto fail2; + } + result = + MCDI_OUT_DWORD(req, NVRAM_UPDATE_FINISH_V2_OUT_RESULT_CODE); + + if (result != MC_CMD_NVRAM_VERIFY_RC_SUCCESS) { + /* Mandatory verification failed */ + rc = EINVAL; + goto fail3; + } + } + + if (resultp != NULL) + *resultp = result; + + return (0); + +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + /* Always report verification result */ + if (resultp != NULL) + *resultp = result; + + return (rc); +} + +#if EFSYS_OPT_DIAG + + __checkReturn efx_rc_t +efx_mcdi_nvram_test( + __in efx_nic_t *enp, + __in uint32_t partn) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_NVRAM_TEST_IN_LEN, + MC_CMD_NVRAM_TEST_OUT_LEN)]; + int result; + efx_rc_t rc; + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_NVRAM_TEST; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_NVRAM_TEST_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_NVRAM_TEST_OUT_LEN; + + MCDI_IN_SET_DWORD(req, NVRAM_TEST_IN_TYPE, partn); + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + if (req.emr_out_length_used < MC_CMD_NVRAM_TEST_OUT_LEN) { + rc = EMSGSIZE; + goto fail2; + } + + result = MCDI_OUT_DWORD(req, NVRAM_TEST_OUT_RESULT); + if (result == MC_CMD_NVRAM_TEST_FAIL) { + + EFSYS_PROBE1(nvram_test_failure, int, partn); + + rc = (EINVAL); + goto fail3; + } + + return (0); + +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +#endif /* EFSYS_OPT_DIAG */ + + +#endif /* EFSYS_OPT_NVRAM || EFSYS_OPT_VPD */ diff --git a/dpdk/drivers/net/sfc/base/efx_phy.c b/dpdk/drivers/net/sfc/base/efx_phy.c new file mode 100644 index 00000000..752cd52e --- /dev/null +++ b/dpdk/drivers/net/sfc/base/efx_phy.c @@ -0,0 +1,561 @@ +/* + * Copyright (c) 2007-2016 Solarflare Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are + * those of the authors and should not be interpreted as representing official + * policies, either expressed or implied, of the FreeBSD Project. + */ + +#include "efx.h" +#include "efx_impl.h" + + +#if EFSYS_OPT_SIENA +static const efx_phy_ops_t __efx_phy_siena_ops = { + siena_phy_power, /* epo_power */ + NULL, /* epo_reset */ + siena_phy_reconfigure, /* epo_reconfigure */ + siena_phy_verify, /* epo_verify */ + siena_phy_oui_get, /* epo_oui_get */ +#if EFSYS_OPT_PHY_STATS + siena_phy_stats_update, /* epo_stats_update */ +#endif /* EFSYS_OPT_PHY_STATS */ +#if EFSYS_OPT_BIST + NULL, /* epo_bist_enable_offline */ + siena_phy_bist_start, /* epo_bist_start */ + siena_phy_bist_poll, /* epo_bist_poll */ + siena_phy_bist_stop, /* epo_bist_stop */ +#endif /* EFSYS_OPT_BIST */ +}; +#endif /* EFSYS_OPT_SIENA */ + +#if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD +static const efx_phy_ops_t __efx_phy_ef10_ops = { + ef10_phy_power, /* epo_power */ + NULL, /* epo_reset */ + ef10_phy_reconfigure, /* epo_reconfigure */ + ef10_phy_verify, /* epo_verify */ + ef10_phy_oui_get, /* epo_oui_get */ +#if EFSYS_OPT_PHY_STATS + ef10_phy_stats_update, /* epo_stats_update */ +#endif /* EFSYS_OPT_PHY_STATS */ +#if EFSYS_OPT_BIST + ef10_bist_enable_offline, /* epo_bist_enable_offline */ + ef10_bist_start, /* epo_bist_start */ + ef10_bist_poll, /* epo_bist_poll */ + ef10_bist_stop, /* epo_bist_stop */ +#endif /* EFSYS_OPT_BIST */ +}; +#endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */ + + __checkReturn efx_rc_t +efx_phy_probe( + __in efx_nic_t *enp) +{ + efx_port_t *epp = &(enp->en_port); + efx_nic_cfg_t *encp = &(enp->en_nic_cfg); + const efx_phy_ops_t *epop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + + epp->ep_port = encp->enc_port; + epp->ep_phy_type = encp->enc_phy_type; + + /* Hook in operations structure */ + switch (enp->en_family) { +#if EFSYS_OPT_SIENA + case EFX_FAMILY_SIENA: + epop = &__efx_phy_siena_ops; + break; +#endif /* EFSYS_OPT_SIENA */ +#if EFSYS_OPT_HUNTINGTON + case EFX_FAMILY_HUNTINGTON: + epop = &__efx_phy_ef10_ops; + break; +#endif /* EFSYS_OPT_HUNTINGTON */ +#if EFSYS_OPT_MEDFORD + case EFX_FAMILY_MEDFORD: + epop = &__efx_phy_ef10_ops; + break; +#endif /* EFSYS_OPT_MEDFORD */ + default: + rc = ENOTSUP; + goto fail1; + } + + epp->ep_epop = epop; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + epp->ep_port = 0; + epp->ep_phy_type = 0; + + return (rc); +} + + __checkReturn efx_rc_t +efx_phy_verify( + __in efx_nic_t *enp) +{ + efx_port_t *epp = &(enp->en_port); + const efx_phy_ops_t *epop = epp->ep_epop; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT); + + return (epop->epo_verify(enp)); +} + +#if EFSYS_OPT_PHY_LED_CONTROL + + __checkReturn efx_rc_t +efx_phy_led_set( + __in efx_nic_t *enp, + __in efx_phy_led_mode_t mode) +{ + efx_nic_cfg_t *encp = (&enp->en_nic_cfg); + efx_port_t *epp = &(enp->en_port); + const efx_phy_ops_t *epop = epp->ep_epop; + uint32_t mask; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT); + + if (epp->ep_phy_led_mode == mode) + goto done; + + mask = (1 << EFX_PHY_LED_DEFAULT); + mask |= encp->enc_led_mask; + + if (!((1 << mode) & mask)) { + rc = ENOTSUP; + goto fail1; + } + + EFSYS_ASSERT3U(mode, <, EFX_PHY_LED_NMODES); + epp->ep_phy_led_mode = mode; + + if ((rc = epop->epo_reconfigure(enp)) != 0) + goto fail2; + +done: + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} +#endif /* EFSYS_OPT_PHY_LED_CONTROL */ + + void +efx_phy_adv_cap_get( + __in efx_nic_t *enp, + __in uint32_t flag, + __out uint32_t *maskp) +{ + efx_port_t *epp = &(enp->en_port); + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE); + + switch (flag) { + case EFX_PHY_CAP_CURRENT: + *maskp = epp->ep_adv_cap_mask; + break; + case EFX_PHY_CAP_DEFAULT: + *maskp = epp->ep_default_adv_cap_mask; + break; + case EFX_PHY_CAP_PERM: + *maskp = epp->ep_phy_cap_mask; + break; + default: + EFSYS_ASSERT(B_FALSE); + break; + } +} + + __checkReturn efx_rc_t +efx_phy_adv_cap_set( + __in efx_nic_t *enp, + __in uint32_t mask) +{ + efx_port_t *epp = &(enp->en_port); + const efx_phy_ops_t *epop = epp->ep_epop; + uint32_t old_mask; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT); + + if ((mask & ~epp->ep_phy_cap_mask) != 0) { + rc = ENOTSUP; + goto fail1; + } + + if (epp->ep_adv_cap_mask == mask) + goto done; + + old_mask = epp->ep_adv_cap_mask; + epp->ep_adv_cap_mask = mask; + + if ((rc = epop->epo_reconfigure(enp)) != 0) + goto fail2; + +done: + return (0); + +fail2: + EFSYS_PROBE(fail2); + + epp->ep_adv_cap_mask = old_mask; + /* Reconfigure for robustness */ + if (epop->epo_reconfigure(enp) != 0) { + /* + * We may have an inconsistent view of our advertised speed + * capabilities. + */ + EFSYS_ASSERT(0); + } + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + void +efx_phy_lp_cap_get( + __in efx_nic_t *enp, + __out uint32_t *maskp) +{ + efx_port_t *epp = &(enp->en_port); + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT); + + *maskp = epp->ep_lp_cap_mask; +} + + __checkReturn efx_rc_t +efx_phy_oui_get( + __in efx_nic_t *enp, + __out uint32_t *ouip) +{ + efx_port_t *epp = &(enp->en_port); + const efx_phy_ops_t *epop = epp->ep_epop; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT); + + return (epop->epo_oui_get(enp, ouip)); +} + + void +efx_phy_media_type_get( + __in efx_nic_t *enp, + __out efx_phy_media_type_t *typep) +{ + efx_port_t *epp = &(enp->en_port); + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT); + + if (epp->ep_module_type != EFX_PHY_MEDIA_INVALID) + *typep = epp->ep_module_type; + else + *typep = epp->ep_fixed_port_type; +} + + __checkReturn efx_rc_t +efx_phy_module_get_info( + __in efx_nic_t *enp, + __in uint8_t dev_addr, + __in uint8_t offset, + __in uint8_t len, + __out_bcount(len) uint8_t *data) +{ + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT(data != NULL); + + if ((uint32_t)offset + len > 0xff) { + rc = EINVAL; + goto fail1; + } + + if ((rc = efx_mcdi_phy_module_get_info(enp, dev_addr, + offset, len, data)) != 0) + goto fail2; + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +#if EFSYS_OPT_PHY_STATS + +#if EFSYS_OPT_NAMES + +/* START MKCONFIG GENERATED PhyStatNamesBlock af9ffa24da3bc100 */ +static const char * const __efx_phy_stat_name[] = { + "oui", + "pma_pmd_link_up", + "pma_pmd_rx_fault", + "pma_pmd_tx_fault", + "pma_pmd_rev_a", + "pma_pmd_rev_b", + "pma_pmd_rev_c", + "pma_pmd_rev_d", + "pcs_link_up", + "pcs_rx_fault", + "pcs_tx_fault", + "pcs_ber", + "pcs_block_errors", + "phy_xs_link_up", + "phy_xs_rx_fault", + "phy_xs_tx_fault", + "phy_xs_align", + "phy_xs_sync_a", + "phy_xs_sync_b", + "phy_xs_sync_c", + "phy_xs_sync_d", + "an_link_up", + "an_master", + "an_local_rx_ok", + "an_remote_rx_ok", + "cl22ext_link_up", + "snr_a", + "snr_b", + "snr_c", + "snr_d", + "pma_pmd_signal_a", + "pma_pmd_signal_b", + "pma_pmd_signal_c", + "pma_pmd_signal_d", + "an_complete", + "pma_pmd_rev_major", + "pma_pmd_rev_minor", + "pma_pmd_rev_micro", + "pcs_fw_version_0", + "pcs_fw_version_1", + "pcs_fw_version_2", + "pcs_fw_version_3", + "pcs_fw_build_yy", + "pcs_fw_build_mm", + "pcs_fw_build_dd", + "pcs_op_mode", +}; + +/* END MKCONFIG GENERATED PhyStatNamesBlock */ + + const char * +efx_phy_stat_name( + __in efx_nic_t *enp, + __in efx_phy_stat_t type) +{ + _NOTE(ARGUNUSED(enp)) + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(type, <, EFX_PHY_NSTATS); + + return (__efx_phy_stat_name[type]); +} + +#endif /* EFSYS_OPT_NAMES */ + + __checkReturn efx_rc_t +efx_phy_stats_update( + __in efx_nic_t *enp, + __in efsys_mem_t *esmp, + __inout_ecount(EFX_PHY_NSTATS) uint32_t *stat) +{ + efx_port_t *epp = &(enp->en_port); + const efx_phy_ops_t *epop = epp->ep_epop; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT); + + return (epop->epo_stats_update(enp, esmp, stat)); +} + +#endif /* EFSYS_OPT_PHY_STATS */ + + +#if EFSYS_OPT_BIST + + __checkReturn efx_rc_t +efx_bist_enable_offline( + __in efx_nic_t *enp) +{ + efx_port_t *epp = &(enp->en_port); + const efx_phy_ops_t *epop = epp->ep_epop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + + if (epop->epo_bist_enable_offline == NULL) { + rc = ENOTSUP; + goto fail1; + } + + if ((rc = epop->epo_bist_enable_offline(enp)) != 0) + goto fail2; + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); + +} + + __checkReturn efx_rc_t +efx_bist_start( + __in efx_nic_t *enp, + __in efx_bist_type_t type) +{ + efx_port_t *epp = &(enp->en_port); + const efx_phy_ops_t *epop = epp->ep_epop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + + EFSYS_ASSERT3U(type, !=, EFX_BIST_TYPE_UNKNOWN); + EFSYS_ASSERT3U(type, <, EFX_BIST_TYPE_NTYPES); + EFSYS_ASSERT3U(epp->ep_current_bist, ==, EFX_BIST_TYPE_UNKNOWN); + + if (epop->epo_bist_start == NULL) { + rc = ENOTSUP; + goto fail1; + } + + if ((rc = epop->epo_bist_start(enp, type)) != 0) + goto fail2; + + epp->ep_current_bist = type; + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_bist_poll( + __in efx_nic_t *enp, + __in efx_bist_type_t type, + __out efx_bist_result_t *resultp, + __out_opt uint32_t *value_maskp, + __out_ecount_opt(count) unsigned long *valuesp, + __in size_t count) +{ + efx_port_t *epp = &(enp->en_port); + const efx_phy_ops_t *epop = epp->ep_epop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + + EFSYS_ASSERT3U(type, !=, EFX_BIST_TYPE_UNKNOWN); + EFSYS_ASSERT3U(type, <, EFX_BIST_TYPE_NTYPES); + EFSYS_ASSERT3U(epp->ep_current_bist, ==, type); + + EFSYS_ASSERT(epop->epo_bist_poll != NULL); + if (epop->epo_bist_poll == NULL) { + rc = ENOTSUP; + goto fail1; + } + + if ((rc = epop->epo_bist_poll(enp, type, resultp, value_maskp, + valuesp, count)) != 0) + goto fail2; + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + void +efx_bist_stop( + __in efx_nic_t *enp, + __in efx_bist_type_t type) +{ + efx_port_t *epp = &(enp->en_port); + const efx_phy_ops_t *epop = epp->ep_epop; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + + EFSYS_ASSERT3U(type, !=, EFX_BIST_TYPE_UNKNOWN); + EFSYS_ASSERT3U(type, <, EFX_BIST_TYPE_NTYPES); + EFSYS_ASSERT3U(epp->ep_current_bist, ==, type); + + EFSYS_ASSERT(epop->epo_bist_stop != NULL); + + if (epop->epo_bist_stop != NULL) + epop->epo_bist_stop(enp, type); + + epp->ep_current_bist = EFX_BIST_TYPE_UNKNOWN; +} + +#endif /* EFSYS_OPT_BIST */ + void +efx_phy_unprobe( + __in efx_nic_t *enp) +{ + efx_port_t *epp = &(enp->en_port); + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + + epp->ep_epop = NULL; + + epp->ep_adv_cap_mask = 0; + + epp->ep_port = 0; + epp->ep_phy_type = 0; +} diff --git a/dpdk/drivers/net/sfc/base/efx_phy_ids.h b/dpdk/drivers/net/sfc/base/efx_phy_ids.h new file mode 100644 index 00000000..9d9a0f90 --- /dev/null +++ b/dpdk/drivers/net/sfc/base/efx_phy_ids.h @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2013-2016 Solarflare Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are + * those of the authors and should not be interpreted as representing official + * policies, either expressed or implied, of the FreeBSD Project. + */ + +#ifndef _SYS_EFX_PHY_IDS_H +#define _SYS_EFX_PHY_IDS_H + +#define EFX_PHY_NULL 0 + +typedef enum efx_phy_type_e { /* GENERATED BY scripts/genfwdef */ + EFX_PHY_TXC43128 = 1, + EFX_PHY_SFX7101 = 3, + EFX_PHY_QT2022C2 = 4, + EFX_PHY_PM8358 = 6, + EFX_PHY_SFT9001A = 8, + EFX_PHY_QT2025C = 9, + EFX_PHY_SFT9001B = 10, + EFX_PHY_QLX111V = 12, + EFX_PHY_QT2025_KR = 17, + EFX_PHY_AEL3020 = 18, + EFX_PHY_XFI_FARMI = 19, +} efx_phy_type_t; + + +#endif /* _SYS_EFX_PHY_IDS_H */ diff --git a/dpdk/drivers/net/sfc/base/efx_port.c b/dpdk/drivers/net/sfc/base/efx_port.c new file mode 100644 index 00000000..518c2a22 --- /dev/null +++ b/dpdk/drivers/net/sfc/base/efx_port.c @@ -0,0 +1,252 @@ +/* + * Copyright (c) 2009-2016 Solarflare Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are + * those of the authors and should not be interpreted as representing official + * policies, either expressed or implied, of the FreeBSD Project. + */ + +#include "efx.h" +#include "efx_impl.h" + + __checkReturn efx_rc_t +efx_port_init( + __in efx_nic_t *enp) +{ + efx_port_t *epp = &(enp->en_port); + const efx_phy_ops_t *epop = epp->ep_epop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); + + if (enp->en_mod_flags & EFX_MOD_PORT) { + rc = EINVAL; + goto fail1; + } + + enp->en_mod_flags |= EFX_MOD_PORT; + + epp->ep_mac_type = EFX_MAC_INVALID; + epp->ep_link_mode = EFX_LINK_UNKNOWN; + epp->ep_mac_drain = B_TRUE; + + /* Configure the MAC */ + if ((rc = efx_mac_select(enp)) != 0) + goto fail1; + + epp->ep_emop->emo_reconfigure(enp); + + /* Pick up current phy capababilities */ + efx_port_poll(enp, NULL); + + /* + * Turn on the PHY if available, otherwise reset it, and + * reconfigure it with the current configuration. + */ + if (epop->epo_power != NULL) { + if ((rc = epop->epo_power(enp, B_TRUE)) != 0) + goto fail2; + } else { + if ((rc = epop->epo_reset(enp)) != 0) + goto fail2; + } + + EFSYS_ASSERT(enp->en_reset_flags & EFX_RESET_PHY); + enp->en_reset_flags &= ~EFX_RESET_PHY; + + if ((rc = epop->epo_reconfigure(enp)) != 0) + goto fail3; + + return (0); + +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + enp->en_mod_flags &= ~EFX_MOD_PORT; + + return (rc); +} + + __checkReturn efx_rc_t +efx_port_poll( + __in efx_nic_t *enp, + __out_opt efx_link_mode_t *link_modep) +{ + efx_port_t *epp = &(enp->en_port); + const efx_mac_ops_t *emop = epp->ep_emop; + efx_link_mode_t ignore_link_mode; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT); + + EFSYS_ASSERT(emop != NULL); + EFSYS_ASSERT(!epp->ep_mac_stats_pending); + + if (link_modep == NULL) + link_modep = &ignore_link_mode; + + if ((rc = emop->emo_poll(enp, link_modep)) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +#if EFSYS_OPT_LOOPBACK + + __checkReturn efx_rc_t +efx_port_loopback_set( + __in efx_nic_t *enp, + __in efx_link_mode_t link_mode, + __in efx_loopback_type_t loopback_type) +{ + efx_port_t *epp = &(enp->en_port); + efx_nic_cfg_t *encp = &(enp->en_nic_cfg); + const efx_mac_ops_t *emop = epp->ep_emop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT); + EFSYS_ASSERT(emop != NULL); + + EFSYS_ASSERT(link_mode < EFX_LINK_NMODES); + + if (EFX_TEST_QWORD_BIT(encp->enc_loopback_types[link_mode], + loopback_type) == 0) { + rc = ENOTSUP; + goto fail1; + } + + if (epp->ep_loopback_type == loopback_type && + epp->ep_loopback_link_mode == link_mode) + return (0); + + if ((rc = emop->emo_loopback_set(enp, link_mode, loopback_type)) != 0) + goto fail2; + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +#if EFSYS_OPT_NAMES + +static const char * const __efx_loopback_type_name[] = { + "OFF", + "DATA", + "GMAC", + "XGMII", + "XGXS", + "XAUI", + "GMII", + "SGMII", + "XGBR", + "XFI", + "XAUI_FAR", + "GMII_FAR", + "SGMII_FAR", + "XFI_FAR", + "GPHY", + "PHY_XS", + "PCS", + "PMA_PMD", + "XPORT", + "XGMII_WS", + "XAUI_WS", + "XAUI_WS_FAR", + "XAUI_WS_NEAR", + "GMII_WS", + "XFI_WS", + "XFI_WS_FAR", + "PHYXS_WS", + "PMA_INT", + "SD_NEAR", + "SD_FAR", + "PMA_INT_WS", + "SD_FEP2_WS", + "SD_FEP1_5_WS", + "SD_FEP_WS", + "SD_FES_WS", +}; + + __checkReturn const char * +efx_loopback_type_name( + __in efx_nic_t *enp, + __in efx_loopback_type_t type) +{ + EFX_STATIC_ASSERT(EFX_ARRAY_SIZE(__efx_loopback_type_name) == + EFX_LOOPBACK_NTYPES); + + _NOTE(ARGUNUSED(enp)) + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(type, <, EFX_LOOPBACK_NTYPES); + + return (__efx_loopback_type_name[type]); +} + +#endif /* EFSYS_OPT_NAMES */ + +#endif /* EFSYS_OPT_LOOPBACK */ + + void +efx_port_fini( + __in efx_nic_t *enp) +{ + efx_port_t *epp = &(enp->en_port); + const efx_phy_ops_t *epop = epp->ep_epop; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT); + + EFSYS_ASSERT(epp->ep_mac_drain); + + epp->ep_emop = NULL; + epp->ep_mac_type = EFX_MAC_INVALID; + epp->ep_mac_drain = B_FALSE; + + /* Turn off the PHY */ + if (epop->epo_power != NULL) + (void) epop->epo_power(enp, B_FALSE); + + enp->en_mod_flags &= ~EFX_MOD_PORT; +} diff --git a/dpdk/drivers/net/sfc/base/efx_regs.h b/dpdk/drivers/net/sfc/base/efx_regs.h new file mode 100644 index 00000000..a1a7f9da --- /dev/null +++ b/dpdk/drivers/net/sfc/base/efx_regs.h @@ -0,0 +1,3870 @@ +/* + * Copyright (c) 2007-2016 Solarflare Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are + * those of the authors and should not be interpreted as representing official + * policies, either expressed or implied, of the FreeBSD Project. + */ + +#ifndef _SYS_EFX_REGS_H +#define _SYS_EFX_REGS_H + + +#ifdef __cplusplus +extern "C" { +#endif + + +/************************************************************************** + * + * Falcon/Siena registers and descriptors + * + ************************************************************************** + */ + +/* + * FR_AB_EE_VPD_CFG0_REG_SF(128bit): + * SPI/VPD configuration register 0 + */ +#define FR_AB_EE_VPD_CFG0_REG_SF_OFST 0x00000300 +/* falcona0,falconb0=eeprom_flash */ +/* + * FR_AB_EE_VPD_CFG0_REG(128bit): + * SPI/VPD configuration register 0 + */ +#define FR_AB_EE_VPD_CFG0_REG_OFST 0x00000140 +/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_EE_SF_FASTRD_EN_LBN 127 +#define FRF_AB_EE_SF_FASTRD_EN_WIDTH 1 +#define FRF_AB_EE_SF_CLOCK_DIV_LBN 120 +#define FRF_AB_EE_SF_CLOCK_DIV_WIDTH 7 +#define FRF_AB_EE_VPD_WIP_POLL_LBN 119 +#define FRF_AB_EE_VPD_WIP_POLL_WIDTH 1 +#define FRF_AB_EE_EE_CLOCK_DIV_LBN 112 +#define FRF_AB_EE_EE_CLOCK_DIV_WIDTH 7 +#define FRF_AB_EE_EE_WR_TMR_VALUE_LBN 96 +#define FRF_AB_EE_EE_WR_TMR_VALUE_WIDTH 16 +#define FRF_AB_EE_VPDW_LENGTH_LBN 80 +#define FRF_AB_EE_VPDW_LENGTH_WIDTH 15 +#define FRF_AB_EE_VPDW_BASE_LBN 64 +#define FRF_AB_EE_VPDW_BASE_WIDTH 15 +#define FRF_AB_EE_VPD_WR_CMD_EN_LBN 56 +#define FRF_AB_EE_VPD_WR_CMD_EN_WIDTH 8 +#define FRF_AB_EE_VPD_BASE_LBN 32 +#define FRF_AB_EE_VPD_BASE_WIDTH 24 +#define FRF_AB_EE_VPD_LENGTH_LBN 16 +#define FRF_AB_EE_VPD_LENGTH_WIDTH 15 +#define FRF_AB_EE_VPD_AD_SIZE_LBN 8 +#define FRF_AB_EE_VPD_AD_SIZE_WIDTH 5 +#define FRF_AB_EE_VPD_ACCESS_ON_LBN 5 +#define FRF_AB_EE_VPD_ACCESS_ON_WIDTH 1 +#define FRF_AB_EE_VPD_ACCESS_BLOCK_LBN 4 +#define FRF_AB_EE_VPD_ACCESS_BLOCK_WIDTH 1 +#define FRF_AB_EE_VPD_DEV_SF_SEL_LBN 2 +#define FRF_AB_EE_VPD_DEV_SF_SEL_WIDTH 1 +#define FRF_AB_EE_VPD_EN_AD9_MODE_LBN 1 +#define FRF_AB_EE_VPD_EN_AD9_MODE_WIDTH 1 +#define FRF_AB_EE_VPD_EN_LBN 0 +#define FRF_AB_EE_VPD_EN_WIDTH 1 + + +/* + * FR_AB_PCIE_SD_CTL0123_REG_SF(128bit): + * PCIE SerDes control register 0 to 3 + */ +#define FR_AB_PCIE_SD_CTL0123_REG_SF_OFST 0x00000320 +/* falcona0,falconb0=eeprom_flash */ +/* + * FR_AB_PCIE_SD_CTL0123_REG(128bit): + * PCIE SerDes control register 0 to 3 + */ +#define FR_AB_PCIE_SD_CTL0123_REG_OFST 0x00000320 +/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_PCIE_TESTSIG_H_LBN 96 +#define FRF_AB_PCIE_TESTSIG_H_WIDTH 19 +#define FRF_AB_PCIE_TESTSIG_L_LBN 64 +#define FRF_AB_PCIE_TESTSIG_L_WIDTH 19 +#define FRF_AB_PCIE_OFFSET_LBN 56 +#define FRF_AB_PCIE_OFFSET_WIDTH 8 +#define FRF_AB_PCIE_OFFSETEN_H_LBN 55 +#define FRF_AB_PCIE_OFFSETEN_H_WIDTH 1 +#define FRF_AB_PCIE_OFFSETEN_L_LBN 54 +#define FRF_AB_PCIE_OFFSETEN_L_WIDTH 1 +#define FRF_AB_PCIE_HIVMODE_H_LBN 53 +#define FRF_AB_PCIE_HIVMODE_H_WIDTH 1 +#define FRF_AB_PCIE_HIVMODE_L_LBN 52 +#define FRF_AB_PCIE_HIVMODE_L_WIDTH 1 +#define FRF_AB_PCIE_PARRESET_H_LBN 51 +#define FRF_AB_PCIE_PARRESET_H_WIDTH 1 +#define FRF_AB_PCIE_PARRESET_L_LBN 50 +#define FRF_AB_PCIE_PARRESET_L_WIDTH 1 +#define FRF_AB_PCIE_LPBKWDRV_H_LBN 49 +#define FRF_AB_PCIE_LPBKWDRV_H_WIDTH 1 +#define FRF_AB_PCIE_LPBKWDRV_L_LBN 48 +#define FRF_AB_PCIE_LPBKWDRV_L_WIDTH 1 +#define FRF_AB_PCIE_LPBK_LBN 40 +#define FRF_AB_PCIE_LPBK_WIDTH 8 +#define FRF_AB_PCIE_PARLPBK_LBN 32 +#define FRF_AB_PCIE_PARLPBK_WIDTH 8 +#define FRF_AB_PCIE_RXTERMADJ_H_LBN 30 +#define FRF_AB_PCIE_RXTERMADJ_H_WIDTH 2 +#define FRF_AB_PCIE_RXTERMADJ_L_LBN 28 +#define FRF_AB_PCIE_RXTERMADJ_L_WIDTH 2 +#define FFE_AB_PCIE_RXTERMADJ_MIN15PCNT 3 +#define FFE_AB_PCIE_RXTERMADJ_PL10PCNT 2 +#define FFE_AB_PCIE_RXTERMADJ_MIN17PCNT 1 +#define FFE_AB_PCIE_RXTERMADJ_NOMNL 0 +#define FRF_AB_PCIE_TXTERMADJ_H_LBN 26 +#define FRF_AB_PCIE_TXTERMADJ_H_WIDTH 2 +#define FRF_AB_PCIE_TXTERMADJ_L_LBN 24 +#define FRF_AB_PCIE_TXTERMADJ_L_WIDTH 2 +#define FFE_AB_PCIE_TXTERMADJ_MIN15PCNT 3 +#define FFE_AB_PCIE_TXTERMADJ_PL10PCNT 2 +#define FFE_AB_PCIE_TXTERMADJ_MIN17PCNT 1 +#define FFE_AB_PCIE_TXTERMADJ_NOMNL 0 +#define FRF_AB_PCIE_RXEQCTL_H_LBN 18 +#define FRF_AB_PCIE_RXEQCTL_H_WIDTH 2 +#define FRF_AB_PCIE_RXEQCTL_L_LBN 16 +#define FRF_AB_PCIE_RXEQCTL_L_WIDTH 2 +#define FFE_AB_PCIE_RXEQCTL_OFF_ALT 3 +#define FFE_AB_PCIE_RXEQCTL_OFF 2 +#define FFE_AB_PCIE_RXEQCTL_MIN 1 +#define FFE_AB_PCIE_RXEQCTL_MAX 0 +#define FRF_AB_PCIE_HIDRV_LBN 8 +#define FRF_AB_PCIE_HIDRV_WIDTH 8 +#define FRF_AB_PCIE_LODRV_LBN 0 +#define FRF_AB_PCIE_LODRV_WIDTH 8 + + +/* + * FR_AB_PCIE_SD_CTL45_REG_SF(128bit): + * PCIE SerDes control register 4 and 5 + */ +#define FR_AB_PCIE_SD_CTL45_REG_SF_OFST 0x00000330 +/* falcona0,falconb0=eeprom_flash */ +/* + * FR_AB_PCIE_SD_CTL45_REG(128bit): + * PCIE SerDes control register 4 and 5 + */ +#define FR_AB_PCIE_SD_CTL45_REG_OFST 0x00000330 +/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_PCIE_DTX7_LBN 60 +#define FRF_AB_PCIE_DTX7_WIDTH 4 +#define FRF_AB_PCIE_DTX6_LBN 56 +#define FRF_AB_PCIE_DTX6_WIDTH 4 +#define FRF_AB_PCIE_DTX5_LBN 52 +#define FRF_AB_PCIE_DTX5_WIDTH 4 +#define FRF_AB_PCIE_DTX4_LBN 48 +#define FRF_AB_PCIE_DTX4_WIDTH 4 +#define FRF_AB_PCIE_DTX3_LBN 44 +#define FRF_AB_PCIE_DTX3_WIDTH 4 +#define FRF_AB_PCIE_DTX2_LBN 40 +#define FRF_AB_PCIE_DTX2_WIDTH 4 +#define FRF_AB_PCIE_DTX1_LBN 36 +#define FRF_AB_PCIE_DTX1_WIDTH 4 +#define FRF_AB_PCIE_DTX0_LBN 32 +#define FRF_AB_PCIE_DTX0_WIDTH 4 +#define FRF_AB_PCIE_DEQ7_LBN 28 +#define FRF_AB_PCIE_DEQ7_WIDTH 4 +#define FRF_AB_PCIE_DEQ6_LBN 24 +#define FRF_AB_PCIE_DEQ6_WIDTH 4 +#define FRF_AB_PCIE_DEQ5_LBN 20 +#define FRF_AB_PCIE_DEQ5_WIDTH 4 +#define FRF_AB_PCIE_DEQ4_LBN 16 +#define FRF_AB_PCIE_DEQ4_WIDTH 4 +#define FRF_AB_PCIE_DEQ3_LBN 12 +#define FRF_AB_PCIE_DEQ3_WIDTH 4 +#define FRF_AB_PCIE_DEQ2_LBN 8 +#define FRF_AB_PCIE_DEQ2_WIDTH 4 +#define FRF_AB_PCIE_DEQ1_LBN 4 +#define FRF_AB_PCIE_DEQ1_WIDTH 4 +#define FRF_AB_PCIE_DEQ0_LBN 0 +#define FRF_AB_PCIE_DEQ0_WIDTH 4 + + +/* + * FR_AB_PCIE_PCS_CTL_STAT_REG_SF(128bit): + * PCIE PCS control and status register + */ +#define FR_AB_PCIE_PCS_CTL_STAT_REG_SF_OFST 0x00000340 +/* falcona0,falconb0=eeprom_flash */ +/* + * FR_AB_PCIE_PCS_CTL_STAT_REG(128bit): + * PCIE PCS control and status register + */ +#define FR_AB_PCIE_PCS_CTL_STAT_REG_OFST 0x00000340 +/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_PCIE_PRBSERRCOUNT0_H_LBN 52 +#define FRF_AB_PCIE_PRBSERRCOUNT0_H_WIDTH 4 +#define FRF_AB_PCIE_PRBSERRCOUNT0_L_LBN 48 +#define FRF_AB_PCIE_PRBSERRCOUNT0_L_WIDTH 4 +#define FRF_AB_PCIE_PRBSERR_LBN 40 +#define FRF_AB_PCIE_PRBSERR_WIDTH 8 +#define FRF_AB_PCIE_PRBSERRH0_LBN 32 +#define FRF_AB_PCIE_PRBSERRH0_WIDTH 8 +#define FRF_AB_PCIE_FASTINIT_H_LBN 15 +#define FRF_AB_PCIE_FASTINIT_H_WIDTH 1 +#define FRF_AB_PCIE_FASTINIT_L_LBN 14 +#define FRF_AB_PCIE_FASTINIT_L_WIDTH 1 +#define FRF_AB_PCIE_CTCDISABLE_H_LBN 13 +#define FRF_AB_PCIE_CTCDISABLE_H_WIDTH 1 +#define FRF_AB_PCIE_CTCDISABLE_L_LBN 12 +#define FRF_AB_PCIE_CTCDISABLE_L_WIDTH 1 +#define FRF_AB_PCIE_PRBSSYNC_H_LBN 11 +#define FRF_AB_PCIE_PRBSSYNC_H_WIDTH 1 +#define FRF_AB_PCIE_PRBSSYNC_L_LBN 10 +#define FRF_AB_PCIE_PRBSSYNC_L_WIDTH 1 +#define FRF_AB_PCIE_PRBSERRACK_H_LBN 9 +#define FRF_AB_PCIE_PRBSERRACK_H_WIDTH 1 +#define FRF_AB_PCIE_PRBSERRACK_L_LBN 8 +#define FRF_AB_PCIE_PRBSERRACK_L_WIDTH 1 +#define FRF_AB_PCIE_PRBSSEL_LBN 0 +#define FRF_AB_PCIE_PRBSSEL_WIDTH 8 + + +/* + * FR_AB_HW_INIT_REG_SF(128bit): + * Hardware initialization register + */ +#define FR_AB_HW_INIT_REG_SF_OFST 0x00000350 +/* falcona0,falconb0=eeprom_flash */ +/* + * FR_AZ_HW_INIT_REG(128bit): + * Hardware initialization register + */ +#define FR_AZ_HW_INIT_REG_OFST 0x000000c0 +/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_BB_BDMRD_CPLF_FULL_LBN 124 +#define FRF_BB_BDMRD_CPLF_FULL_WIDTH 1 +#define FRF_BB_PCIE_CPL_TIMEOUT_CTRL_LBN 121 +#define FRF_BB_PCIE_CPL_TIMEOUT_CTRL_WIDTH 3 +#define FRF_CZ_TX_MRG_TAGS_LBN 120 +#define FRF_CZ_TX_MRG_TAGS_WIDTH 1 +#define FRF_AZ_TRGT_MASK_ALL_LBN 100 +#define FRF_AZ_TRGT_MASK_ALL_WIDTH 1 +#define FRF_AZ_DOORBELL_DROP_LBN 92 +#define FRF_AZ_DOORBELL_DROP_WIDTH 8 +#define FRF_AB_TX_RREQ_MASK_EN_LBN 76 +#define FRF_AB_TX_RREQ_MASK_EN_WIDTH 1 +#define FRF_AB_PE_EIDLE_DIS_LBN 75 +#define FRF_AB_PE_EIDLE_DIS_WIDTH 1 +#define FRF_AZ_FC_BLOCKING_EN_LBN 45 +#define FRF_AZ_FC_BLOCKING_EN_WIDTH 1 +#define FRF_AZ_B2B_REQ_EN_LBN 44 +#define FRF_AZ_B2B_REQ_EN_WIDTH 1 +#define FRF_AZ_POST_WR_MASK_LBN 40 +#define FRF_AZ_POST_WR_MASK_WIDTH 4 +#define FRF_AZ_TLP_TC_LBN 34 +#define FRF_AZ_TLP_TC_WIDTH 3 +#define FRF_AZ_TLP_ATTR_LBN 32 +#define FRF_AZ_TLP_ATTR_WIDTH 2 +#define FRF_AB_INTB_VEC_LBN 24 +#define FRF_AB_INTB_VEC_WIDTH 5 +#define FRF_AB_INTA_VEC_LBN 16 +#define FRF_AB_INTA_VEC_WIDTH 5 +#define FRF_AZ_WD_TIMER_LBN 8 +#define FRF_AZ_WD_TIMER_WIDTH 8 +#define FRF_AZ_US_DISABLE_LBN 5 +#define FRF_AZ_US_DISABLE_WIDTH 1 +#define FRF_AZ_TLP_EP_LBN 4 +#define FRF_AZ_TLP_EP_WIDTH 1 +#define FRF_AZ_ATTR_SEL_LBN 3 +#define FRF_AZ_ATTR_SEL_WIDTH 1 +#define FRF_AZ_TD_SEL_LBN 1 +#define FRF_AZ_TD_SEL_WIDTH 1 +#define FRF_AZ_TLP_TD_LBN 0 +#define FRF_AZ_TLP_TD_WIDTH 1 + + +/* + * FR_AB_NIC_STAT_REG_SF(128bit): + * NIC status register + */ +#define FR_AB_NIC_STAT_REG_SF_OFST 0x00000360 +/* falcona0,falconb0=eeprom_flash */ +/* + * FR_AB_NIC_STAT_REG(128bit): + * NIC status register + */ +#define FR_AB_NIC_STAT_REG_OFST 0x00000200 +/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_BB_AER_DIS_LBN 34 +#define FRF_BB_AER_DIS_WIDTH 1 +#define FRF_BB_EE_STRAP_EN_LBN 31 +#define FRF_BB_EE_STRAP_EN_WIDTH 1 +#define FRF_BB_EE_STRAP_LBN 24 +#define FRF_BB_EE_STRAP_WIDTH 4 +#define FRF_BB_REVISION_ID_LBN 17 +#define FRF_BB_REVISION_ID_WIDTH 7 +#define FRF_AB_ONCHIP_SRAM_LBN 16 +#define FRF_AB_ONCHIP_SRAM_WIDTH 1 +#define FRF_AB_SF_PRST_LBN 9 +#define FRF_AB_SF_PRST_WIDTH 1 +#define FRF_AB_EE_PRST_LBN 8 +#define FRF_AB_EE_PRST_WIDTH 1 +#define FRF_AB_ATE_MODE_LBN 3 +#define FRF_AB_ATE_MODE_WIDTH 1 +#define FRF_AB_STRAP_PINS_LBN 0 +#define FRF_AB_STRAP_PINS_WIDTH 3 + + +/* + * FR_AB_GLB_CTL_REG_SF(128bit): + * Global control register + */ +#define FR_AB_GLB_CTL_REG_SF_OFST 0x00000370 +/* falcona0,falconb0=eeprom_flash */ +/* + * FR_AB_GLB_CTL_REG(128bit): + * Global control register + */ +#define FR_AB_GLB_CTL_REG_OFST 0x00000220 +/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_EXT_PHY_RST_CTL_LBN 63 +#define FRF_AB_EXT_PHY_RST_CTL_WIDTH 1 +#define FRF_AB_XAUI_SD_RST_CTL_LBN 62 +#define FRF_AB_XAUI_SD_RST_CTL_WIDTH 1 +#define FRF_AB_PCIE_SD_RST_CTL_LBN 61 +#define FRF_AB_PCIE_SD_RST_CTL_WIDTH 1 +#define FRF_AA_PCIX_RST_CTL_LBN 60 +#define FRF_AA_PCIX_RST_CTL_WIDTH 1 +#define FRF_BB_BIU_RST_CTL_LBN 60 +#define FRF_BB_BIU_RST_CTL_WIDTH 1 +#define FRF_AB_PCIE_STKY_RST_CTL_LBN 59 +#define FRF_AB_PCIE_STKY_RST_CTL_WIDTH 1 +#define FRF_AB_PCIE_NSTKY_RST_CTL_LBN 58 +#define FRF_AB_PCIE_NSTKY_RST_CTL_WIDTH 1 +#define FRF_AB_PCIE_CORE_RST_CTL_LBN 57 +#define FRF_AB_PCIE_CORE_RST_CTL_WIDTH 1 +#define FRF_AB_XGRX_RST_CTL_LBN 56 +#define FRF_AB_XGRX_RST_CTL_WIDTH 1 +#define FRF_AB_XGTX_RST_CTL_LBN 55 +#define FRF_AB_XGTX_RST_CTL_WIDTH 1 +#define FRF_AB_EM_RST_CTL_LBN 54 +#define FRF_AB_EM_RST_CTL_WIDTH 1 +#define FRF_AB_EV_RST_CTL_LBN 53 +#define FRF_AB_EV_RST_CTL_WIDTH 1 +#define FRF_AB_SR_RST_CTL_LBN 52 +#define FRF_AB_SR_RST_CTL_WIDTH 1 +#define FRF_AB_RX_RST_CTL_LBN 51 +#define FRF_AB_RX_RST_CTL_WIDTH 1 +#define FRF_AB_TX_RST_CTL_LBN 50 +#define FRF_AB_TX_RST_CTL_WIDTH 1 +#define FRF_AB_EE_RST_CTL_LBN 49 +#define FRF_AB_EE_RST_CTL_WIDTH 1 +#define FRF_AB_CS_RST_CTL_LBN 48 +#define FRF_AB_CS_RST_CTL_WIDTH 1 +#define FRF_AB_HOT_RST_CTL_LBN 40 +#define FRF_AB_HOT_RST_CTL_WIDTH 2 +#define FRF_AB_RST_EXT_PHY_LBN 31 +#define FRF_AB_RST_EXT_PHY_WIDTH 1 +#define FRF_AB_RST_XAUI_SD_LBN 30 +#define FRF_AB_RST_XAUI_SD_WIDTH 1 +#define FRF_AB_RST_PCIE_SD_LBN 29 +#define FRF_AB_RST_PCIE_SD_WIDTH 1 +#define FRF_AA_RST_PCIX_LBN 28 +#define FRF_AA_RST_PCIX_WIDTH 1 +#define FRF_BB_RST_BIU_LBN 28 +#define FRF_BB_RST_BIU_WIDTH 1 +#define FRF_AB_RST_PCIE_STKY_LBN 27 +#define FRF_AB_RST_PCIE_STKY_WIDTH 1 +#define FRF_AB_RST_PCIE_NSTKY_LBN 26 +#define FRF_AB_RST_PCIE_NSTKY_WIDTH 1 +#define FRF_AB_RST_PCIE_CORE_LBN 25 +#define FRF_AB_RST_PCIE_CORE_WIDTH 1 +#define FRF_AB_RST_XGRX_LBN 24 +#define FRF_AB_RST_XGRX_WIDTH 1 +#define FRF_AB_RST_XGTX_LBN 23 +#define FRF_AB_RST_XGTX_WIDTH 1 +#define FRF_AB_RST_EM_LBN 22 +#define FRF_AB_RST_EM_WIDTH 1 +#define FRF_AB_RST_EV_LBN 21 +#define FRF_AB_RST_EV_WIDTH 1 +#define FRF_AB_RST_SR_LBN 20 +#define FRF_AB_RST_SR_WIDTH 1 +#define FRF_AB_RST_RX_LBN 19 +#define FRF_AB_RST_RX_WIDTH 1 +#define FRF_AB_RST_TX_LBN 18 +#define FRF_AB_RST_TX_WIDTH 1 +#define FRF_AB_RST_SF_LBN 17 +#define FRF_AB_RST_SF_WIDTH 1 +#define FRF_AB_RST_CS_LBN 16 +#define FRF_AB_RST_CS_WIDTH 1 +#define FRF_AB_INT_RST_DUR_LBN 4 +#define FRF_AB_INT_RST_DUR_WIDTH 3 +#define FRF_AB_EXT_PHY_RST_DUR_LBN 1 +#define FRF_AB_EXT_PHY_RST_DUR_WIDTH 3 +#define FFE_AB_EXT_PHY_RST_DUR_10240US 7 +#define FFE_AB_EXT_PHY_RST_DUR_5120US 6 +#define FFE_AB_EXT_PHY_RST_DUR_2560US 5 +#define FFE_AB_EXT_PHY_RST_DUR_1280US 4 +#define FFE_AB_EXT_PHY_RST_DUR_640US 3 +#define FFE_AB_EXT_PHY_RST_DUR_320US 2 +#define FFE_AB_EXT_PHY_RST_DUR_160US 1 +#define FFE_AB_EXT_PHY_RST_DUR_80US 0 +#define FRF_AB_SWRST_LBN 0 +#define FRF_AB_SWRST_WIDTH 1 + + +/* + * FR_AZ_IOM_IND_ADR_REG(32bit): + * IO-mapped indirect access address register + */ +#define FR_AZ_IOM_IND_ADR_REG_OFST 0x00000000 +/* falcona0,falconb0,sienaa0=net_func_bar0 */ + +#define FRF_AZ_IOM_AUTO_ADR_INC_EN_LBN 24 +#define FRF_AZ_IOM_AUTO_ADR_INC_EN_WIDTH 1 +#define FRF_AZ_IOM_IND_ADR_LBN 0 +#define FRF_AZ_IOM_IND_ADR_WIDTH 24 + + +/* + * FR_AZ_IOM_IND_DAT_REG(32bit): + * IO-mapped indirect access data register + */ +#define FR_AZ_IOM_IND_DAT_REG_OFST 0x00000004 +/* falcona0,falconb0,sienaa0=net_func_bar0 */ + +#define FRF_AZ_IOM_IND_DAT_LBN 0 +#define FRF_AZ_IOM_IND_DAT_WIDTH 32 + + +/* + * FR_AZ_ADR_REGION_REG(128bit): + * Address region register + */ +#define FR_AZ_ADR_REGION_REG_OFST 0x00000000 +/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AZ_ADR_REGION3_LBN 96 +#define FRF_AZ_ADR_REGION3_WIDTH 18 +#define FRF_AZ_ADR_REGION2_LBN 64 +#define FRF_AZ_ADR_REGION2_WIDTH 18 +#define FRF_AZ_ADR_REGION1_LBN 32 +#define FRF_AZ_ADR_REGION1_WIDTH 18 +#define FRF_AZ_ADR_REGION0_LBN 0 +#define FRF_AZ_ADR_REGION0_WIDTH 18 + + +/* + * FR_AZ_INT_EN_REG_KER(128bit): + * Kernel driver Interrupt enable register + */ +#define FR_AZ_INT_EN_REG_KER_OFST 0x00000010 +/* falcona0,falconb0,sienaa0=net_func_bar2 */ + +#define FRF_AZ_KER_INT_LEVE_SEL_LBN 8 +#define FRF_AZ_KER_INT_LEVE_SEL_WIDTH 6 +#define FRF_AZ_KER_INT_CHAR_LBN 4 +#define FRF_AZ_KER_INT_CHAR_WIDTH 1 +#define FRF_AZ_KER_INT_KER_LBN 3 +#define FRF_AZ_KER_INT_KER_WIDTH 1 +#define FRF_AZ_DRV_INT_EN_KER_LBN 0 +#define FRF_AZ_DRV_INT_EN_KER_WIDTH 1 + + +/* + * FR_AZ_INT_EN_REG_CHAR(128bit): + * Char Driver interrupt enable register + */ +#define FR_AZ_INT_EN_REG_CHAR_OFST 0x00000020 +/* falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AZ_CHAR_INT_LEVE_SEL_LBN 8 +#define FRF_AZ_CHAR_INT_LEVE_SEL_WIDTH 6 +#define FRF_AZ_CHAR_INT_CHAR_LBN 4 +#define FRF_AZ_CHAR_INT_CHAR_WIDTH 1 +#define FRF_AZ_CHAR_INT_KER_LBN 3 +#define FRF_AZ_CHAR_INT_KER_WIDTH 1 +#define FRF_AZ_DRV_INT_EN_CHAR_LBN 0 +#define FRF_AZ_DRV_INT_EN_CHAR_WIDTH 1 + + +/* + * FR_AZ_INT_ADR_REG_KER(128bit): + * Interrupt host address for Kernel driver + */ +#define FR_AZ_INT_ADR_REG_KER_OFST 0x00000030 +/* falcona0,falconb0,sienaa0=net_func_bar2 */ + +#define FRF_AZ_NORM_INT_VEC_DIS_KER_LBN 64 +#define FRF_AZ_NORM_INT_VEC_DIS_KER_WIDTH 1 +#define FRF_AZ_INT_ADR_KER_LBN 0 +#define FRF_AZ_INT_ADR_KER_WIDTH 64 +#define FRF_AZ_INT_ADR_KER_DW0_LBN 0 +#define FRF_AZ_INT_ADR_KER_DW0_WIDTH 32 +#define FRF_AZ_INT_ADR_KER_DW1_LBN 32 +#define FRF_AZ_INT_ADR_KER_DW1_WIDTH 32 + + +/* + * FR_AZ_INT_ADR_REG_CHAR(128bit): + * Interrupt host address for Char driver + */ +#define FR_AZ_INT_ADR_REG_CHAR_OFST 0x00000040 +/* falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AZ_NORM_INT_VEC_DIS_CHAR_LBN 64 +#define FRF_AZ_NORM_INT_VEC_DIS_CHAR_WIDTH 1 +#define FRF_AZ_INT_ADR_CHAR_LBN 0 +#define FRF_AZ_INT_ADR_CHAR_WIDTH 64 +#define FRF_AZ_INT_ADR_CHAR_DW0_LBN 0 +#define FRF_AZ_INT_ADR_CHAR_DW0_WIDTH 32 +#define FRF_AZ_INT_ADR_CHAR_DW1_LBN 32 +#define FRF_AZ_INT_ADR_CHAR_DW1_WIDTH 32 + + +/* + * FR_AA_INT_ACK_KER(32bit): + * Kernel interrupt acknowledge register + */ +#define FR_AA_INT_ACK_KER_OFST 0x00000050 +/* falcona0=net_func_bar2 */ + +#define FRF_AA_INT_ACK_KER_FIELD_LBN 0 +#define FRF_AA_INT_ACK_KER_FIELD_WIDTH 32 + + +/* + * FR_BZ_INT_ISR0_REG(128bit): + * Function 0 Interrupt Acknowlege Status register + */ +#define FR_BZ_INT_ISR0_REG_OFST 0x00000090 +/* falconb0,sienaa0=net_func_bar2 */ + +#define FRF_BZ_INT_ISR_REG_LBN 0 +#define FRF_BZ_INT_ISR_REG_WIDTH 64 +#define FRF_BZ_INT_ISR_REG_DW0_LBN 0 +#define FRF_BZ_INT_ISR_REG_DW0_WIDTH 32 +#define FRF_BZ_INT_ISR_REG_DW1_LBN 32 +#define FRF_BZ_INT_ISR_REG_DW1_WIDTH 32 + + +/* + * FR_AB_EE_SPI_HCMD_REG(128bit): + * SPI host command register + */ +#define FR_AB_EE_SPI_HCMD_REG_OFST 0x00000100 +/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_EE_SPI_HCMD_CMD_EN_LBN 31 +#define FRF_AB_EE_SPI_HCMD_CMD_EN_WIDTH 1 +#define FRF_AB_EE_WR_TIMER_ACTIVE_LBN 28 +#define FRF_AB_EE_WR_TIMER_ACTIVE_WIDTH 1 +#define FRF_AB_EE_SPI_HCMD_SF_SEL_LBN 24 +#define FRF_AB_EE_SPI_HCMD_SF_SEL_WIDTH 1 +#define FRF_AB_EE_SPI_HCMD_DABCNT_LBN 16 +#define FRF_AB_EE_SPI_HCMD_DABCNT_WIDTH 5 +#define FRF_AB_EE_SPI_HCMD_READ_LBN 15 +#define FRF_AB_EE_SPI_HCMD_READ_WIDTH 1 +#define FRF_AB_EE_SPI_HCMD_DUBCNT_LBN 12 +#define FRF_AB_EE_SPI_HCMD_DUBCNT_WIDTH 2 +#define FRF_AB_EE_SPI_HCMD_ADBCNT_LBN 8 +#define FRF_AB_EE_SPI_HCMD_ADBCNT_WIDTH 2 +#define FRF_AB_EE_SPI_HCMD_ENC_LBN 0 +#define FRF_AB_EE_SPI_HCMD_ENC_WIDTH 8 + + +/* + * FR_CZ_USR_EV_CFG(32bit): + * User Level Event Configuration register + */ +#define FR_CZ_USR_EV_CFG_OFST 0x00000100 +/* sienaa0=net_func_bar2 */ + +#define FRF_CZ_USREV_DIS_LBN 16 +#define FRF_CZ_USREV_DIS_WIDTH 1 +#define FRF_CZ_DFLT_EVQ_LBN 0 +#define FRF_CZ_DFLT_EVQ_WIDTH 10 + + +/* + * FR_AB_EE_SPI_HADR_REG(128bit): + * SPI host address register + */ +#define FR_AB_EE_SPI_HADR_REG_OFST 0x00000110 +/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_EE_SPI_HADR_DUBYTE_LBN 24 +#define FRF_AB_EE_SPI_HADR_DUBYTE_WIDTH 8 +#define FRF_AB_EE_SPI_HADR_ADR_LBN 0 +#define FRF_AB_EE_SPI_HADR_ADR_WIDTH 24 + + +/* + * FR_AB_EE_SPI_HDATA_REG(128bit): + * SPI host data register + */ +#define FR_AB_EE_SPI_HDATA_REG_OFST 0x00000120 +/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_EE_SPI_HDATA3_LBN 96 +#define FRF_AB_EE_SPI_HDATA3_WIDTH 32 +#define FRF_AB_EE_SPI_HDATA2_LBN 64 +#define FRF_AB_EE_SPI_HDATA2_WIDTH 32 +#define FRF_AB_EE_SPI_HDATA1_LBN 32 +#define FRF_AB_EE_SPI_HDATA1_WIDTH 32 +#define FRF_AB_EE_SPI_HDATA0_LBN 0 +#define FRF_AB_EE_SPI_HDATA0_WIDTH 32 + + +/* + * FR_AB_EE_BASE_PAGE_REG(128bit): + * Expansion ROM base mirror register + */ +#define FR_AB_EE_BASE_PAGE_REG_OFST 0x00000130 +/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_EE_EXPROM_MASK_LBN 16 +#define FRF_AB_EE_EXPROM_MASK_WIDTH 13 +#define FRF_AB_EE_EXP_ROM_WINDOW_BASE_LBN 0 +#define FRF_AB_EE_EXP_ROM_WINDOW_BASE_WIDTH 13 + + +/* + * FR_AB_EE_VPD_SW_CNTL_REG(128bit): + * VPD access SW control register + */ +#define FR_AB_EE_VPD_SW_CNTL_REG_OFST 0x00000150 +/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_EE_VPD_CYCLE_PENDING_LBN 31 +#define FRF_AB_EE_VPD_CYCLE_PENDING_WIDTH 1 +#define FRF_AB_EE_VPD_CYC_WRITE_LBN 28 +#define FRF_AB_EE_VPD_CYC_WRITE_WIDTH 1 +#define FRF_AB_EE_VPD_CYC_ADR_LBN 0 +#define FRF_AB_EE_VPD_CYC_ADR_WIDTH 15 + + +/* + * FR_AB_EE_VPD_SW_DATA_REG(128bit): + * VPD access SW data register + */ +#define FR_AB_EE_VPD_SW_DATA_REG_OFST 0x00000160 +/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_EE_VPD_CYC_DAT_LBN 0 +#define FRF_AB_EE_VPD_CYC_DAT_WIDTH 32 + + +/* + * FR_BB_PCIE_CORE_INDIRECT_REG(64bit): + * Indirect Access to PCIE Core registers + */ +#define FR_BB_PCIE_CORE_INDIRECT_REG_OFST 0x000001f0 +/* falconb0=net_func_bar2 */ + +#define FRF_BB_PCIE_CORE_TARGET_DATA_LBN 32 +#define FRF_BB_PCIE_CORE_TARGET_DATA_WIDTH 32 +#define FRF_BB_PCIE_CORE_INDIRECT_ACCESS_DIR_LBN 15 +#define FRF_BB_PCIE_CORE_INDIRECT_ACCESS_DIR_WIDTH 1 +#define FRF_BB_PCIE_CORE_TARGET_REG_ADRS_LBN 0 +#define FRF_BB_PCIE_CORE_TARGET_REG_ADRS_WIDTH 12 + + +/* + * FR_AB_GPIO_CTL_REG(128bit): + * GPIO control register + */ +#define FR_AB_GPIO_CTL_REG_OFST 0x00000210 +/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_GPIO15_OEN_LBN 63 +#define FRF_AB_GPIO15_OEN_WIDTH 1 +#define FRF_AB_GPIO14_OEN_LBN 62 +#define FRF_AB_GPIO14_OEN_WIDTH 1 +#define FRF_AB_GPIO13_OEN_LBN 61 +#define FRF_AB_GPIO13_OEN_WIDTH 1 +#define FRF_AB_GPIO12_OEN_LBN 60 +#define FRF_AB_GPIO12_OEN_WIDTH 1 +#define FRF_AB_GPIO11_OEN_LBN 59 +#define FRF_AB_GPIO11_OEN_WIDTH 1 +#define FRF_AB_GPIO10_OEN_LBN 58 +#define FRF_AB_GPIO10_OEN_WIDTH 1 +#define FRF_AB_GPIO9_OEN_LBN 57 +#define FRF_AB_GPIO9_OEN_WIDTH 1 +#define FRF_AB_GPIO8_OEN_LBN 56 +#define FRF_AB_GPIO8_OEN_WIDTH 1 +#define FRF_AB_GPIO15_OUT_LBN 55 +#define FRF_AB_GPIO15_OUT_WIDTH 1 +#define FRF_AB_GPIO14_OUT_LBN 54 +#define FRF_AB_GPIO14_OUT_WIDTH 1 +#define FRF_AB_GPIO13_OUT_LBN 53 +#define FRF_AB_GPIO13_OUT_WIDTH 1 +#define FRF_AB_GPIO12_OUT_LBN 52 +#define FRF_AB_GPIO12_OUT_WIDTH 1 +#define FRF_AB_GPIO11_OUT_LBN 51 +#define FRF_AB_GPIO11_OUT_WIDTH 1 +#define FRF_AB_GPIO10_OUT_LBN 50 +#define FRF_AB_GPIO10_OUT_WIDTH 1 +#define FRF_AB_GPIO9_OUT_LBN 49 +#define FRF_AB_GPIO9_OUT_WIDTH 1 +#define FRF_AB_GPIO8_OUT_LBN 48 +#define FRF_AB_GPIO8_OUT_WIDTH 1 +#define FRF_AB_GPIO15_IN_LBN 47 +#define FRF_AB_GPIO15_IN_WIDTH 1 +#define FRF_AB_GPIO14_IN_LBN 46 +#define FRF_AB_GPIO14_IN_WIDTH 1 +#define FRF_AB_GPIO13_IN_LBN 45 +#define FRF_AB_GPIO13_IN_WIDTH 1 +#define FRF_AB_GPIO12_IN_LBN 44 +#define FRF_AB_GPIO12_IN_WIDTH 1 +#define FRF_AB_GPIO11_IN_LBN 43 +#define FRF_AB_GPIO11_IN_WIDTH 1 +#define FRF_AB_GPIO10_IN_LBN 42 +#define FRF_AB_GPIO10_IN_WIDTH 1 +#define FRF_AB_GPIO9_IN_LBN 41 +#define FRF_AB_GPIO9_IN_WIDTH 1 +#define FRF_AB_GPIO8_IN_LBN 40 +#define FRF_AB_GPIO8_IN_WIDTH 1 +#define FRF_AB_GPIO15_PWRUP_VALUE_LBN 39 +#define FRF_AB_GPIO15_PWRUP_VALUE_WIDTH 1 +#define FRF_AB_GPIO14_PWRUP_VALUE_LBN 38 +#define FRF_AB_GPIO14_PWRUP_VALUE_WIDTH 1 +#define FRF_AB_GPIO13_PWRUP_VALUE_LBN 37 +#define FRF_AB_GPIO13_PWRUP_VALUE_WIDTH 1 +#define FRF_AB_GPIO12_PWRUP_VALUE_LBN 36 +#define FRF_AB_GPIO12_PWRUP_VALUE_WIDTH 1 +#define FRF_AB_GPIO11_PWRUP_VALUE_LBN 35 +#define FRF_AB_GPIO11_PWRUP_VALUE_WIDTH 1 +#define FRF_AB_GPIO10_PWRUP_VALUE_LBN 34 +#define FRF_AB_GPIO10_PWRUP_VALUE_WIDTH 1 +#define FRF_AB_GPIO9_PWRUP_VALUE_LBN 33 +#define FRF_AB_GPIO9_PWRUP_VALUE_WIDTH 1 +#define FRF_AB_GPIO8_PWRUP_VALUE_LBN 32 +#define FRF_AB_GPIO8_PWRUP_VALUE_WIDTH 1 +#define FRF_BB_CLK156_OUT_EN_LBN 31 +#define FRF_BB_CLK156_OUT_EN_WIDTH 1 +#define FRF_BB_USE_NIC_CLK_LBN 30 +#define FRF_BB_USE_NIC_CLK_WIDTH 1 +#define FRF_AB_GPIO5_OEN_LBN 29 +#define FRF_AB_GPIO5_OEN_WIDTH 1 +#define FRF_AB_GPIO4_OEN_LBN 28 +#define FRF_AB_GPIO4_OEN_WIDTH 1 +#define FRF_AB_GPIO3_OEN_LBN 27 +#define FRF_AB_GPIO3_OEN_WIDTH 1 +#define FRF_AB_GPIO2_OEN_LBN 26 +#define FRF_AB_GPIO2_OEN_WIDTH 1 +#define FRF_AB_GPIO1_OEN_LBN 25 +#define FRF_AB_GPIO1_OEN_WIDTH 1 +#define FRF_AB_GPIO0_OEN_LBN 24 +#define FRF_AB_GPIO0_OEN_WIDTH 1 +#define FRF_AB_GPIO5_OUT_LBN 21 +#define FRF_AB_GPIO5_OUT_WIDTH 1 +#define FRF_AB_GPIO4_OUT_LBN 20 +#define FRF_AB_GPIO4_OUT_WIDTH 1 +#define FRF_AB_GPIO3_OUT_LBN 19 +#define FRF_AB_GPIO3_OUT_WIDTH 1 +#define FRF_AB_GPIO2_OUT_LBN 18 +#define FRF_AB_GPIO2_OUT_WIDTH 1 +#define FRF_AB_GPIO1_OUT_LBN 17 +#define FRF_AB_GPIO1_OUT_WIDTH 1 +#define FRF_AB_GPIO0_OUT_LBN 16 +#define FRF_AB_GPIO0_OUT_WIDTH 1 +#define FRF_AB_GPIO5_IN_LBN 13 +#define FRF_AB_GPIO5_IN_WIDTH 1 +#define FRF_AB_GPIO4_IN_LBN 12 +#define FRF_AB_GPIO4_IN_WIDTH 1 +#define FRF_AB_GPIO3_IN_LBN 11 +#define FRF_AB_GPIO3_IN_WIDTH 1 +#define FRF_AB_GPIO2_IN_LBN 10 +#define FRF_AB_GPIO2_IN_WIDTH 1 +#define FRF_AB_GPIO1_IN_LBN 9 +#define FRF_AB_GPIO1_IN_WIDTH 1 +#define FRF_AB_GPIO0_IN_LBN 8 +#define FRF_AB_GPIO0_IN_WIDTH 1 +#define FRF_AB_GPIO5_PWRUP_VALUE_LBN 5 +#define FRF_AB_GPIO5_PWRUP_VALUE_WIDTH 1 +#define FRF_AB_GPIO4_PWRUP_VALUE_LBN 4 +#define FRF_AB_GPIO4_PWRUP_VALUE_WIDTH 1 +#define FRF_AB_GPIO3_PWRUP_VALUE_LBN 3 +#define FRF_AB_GPIO3_PWRUP_VALUE_WIDTH 1 +#define FRF_AB_GPIO2_PWRUP_VALUE_LBN 2 +#define FRF_AB_GPIO2_PWRUP_VALUE_WIDTH 1 +#define FRF_AB_GPIO1_PWRUP_VALUE_LBN 1 +#define FRF_AB_GPIO1_PWRUP_VALUE_WIDTH 1 +#define FRF_AB_GPIO0_PWRUP_VALUE_LBN 0 +#define FRF_AB_GPIO0_PWRUP_VALUE_WIDTH 1 + + +/* + * FR_AZ_FATAL_INTR_REG_KER(128bit): + * Fatal interrupt register for Kernel + */ +#define FR_AZ_FATAL_INTR_REG_KER_OFST 0x00000230 +/* falcona0,falconb0,sienaa0=net_func_bar2 */ + +#define FRF_CZ_SRAM_PERR_INT_P_KER_EN_LBN 44 +#define FRF_CZ_SRAM_PERR_INT_P_KER_EN_WIDTH 1 +#define FRF_AB_PCI_BUSERR_INT_KER_EN_LBN 43 +#define FRF_AB_PCI_BUSERR_INT_KER_EN_WIDTH 1 +#define FRF_CZ_MBU_PERR_INT_KER_EN_LBN 43 +#define FRF_CZ_MBU_PERR_INT_KER_EN_WIDTH 1 +#define FRF_AZ_SRAM_OOB_INT_KER_EN_LBN 42 +#define FRF_AZ_SRAM_OOB_INT_KER_EN_WIDTH 1 +#define FRF_AZ_BUFID_OOB_INT_KER_EN_LBN 41 +#define FRF_AZ_BUFID_OOB_INT_KER_EN_WIDTH 1 +#define FRF_AZ_MEM_PERR_INT_KER_EN_LBN 40 +#define FRF_AZ_MEM_PERR_INT_KER_EN_WIDTH 1 +#define FRF_AZ_RBUF_OWN_INT_KER_EN_LBN 39 +#define FRF_AZ_RBUF_OWN_INT_KER_EN_WIDTH 1 +#define FRF_AZ_TBUF_OWN_INT_KER_EN_LBN 38 +#define FRF_AZ_TBUF_OWN_INT_KER_EN_WIDTH 1 +#define FRF_AZ_RDESCQ_OWN_INT_KER_EN_LBN 37 +#define FRF_AZ_RDESCQ_OWN_INT_KER_EN_WIDTH 1 +#define FRF_AZ_TDESCQ_OWN_INT_KER_EN_LBN 36 +#define FRF_AZ_TDESCQ_OWN_INT_KER_EN_WIDTH 1 +#define FRF_AZ_EVQ_OWN_INT_KER_EN_LBN 35 +#define FRF_AZ_EVQ_OWN_INT_KER_EN_WIDTH 1 +#define FRF_AZ_EVF_OFLO_INT_KER_EN_LBN 34 +#define FRF_AZ_EVF_OFLO_INT_KER_EN_WIDTH 1 +#define FRF_AZ_ILL_ADR_INT_KER_EN_LBN 33 +#define FRF_AZ_ILL_ADR_INT_KER_EN_WIDTH 1 +#define FRF_AZ_SRM_PERR_INT_KER_EN_LBN 32 +#define FRF_AZ_SRM_PERR_INT_KER_EN_WIDTH 1 +#define FRF_CZ_SRAM_PERR_INT_P_KER_LBN 12 +#define FRF_CZ_SRAM_PERR_INT_P_KER_WIDTH 1 +#define FRF_AB_PCI_BUSERR_INT_KER_LBN 11 +#define FRF_AB_PCI_BUSERR_INT_KER_WIDTH 1 +#define FRF_CZ_MBU_PERR_INT_KER_LBN 11 +#define FRF_CZ_MBU_PERR_INT_KER_WIDTH 1 +#define FRF_AZ_SRAM_OOB_INT_KER_LBN 10 +#define FRF_AZ_SRAM_OOB_INT_KER_WIDTH 1 +#define FRF_AZ_BUFID_DC_OOB_INT_KER_LBN 9 +#define FRF_AZ_BUFID_DC_OOB_INT_KER_WIDTH 1 +#define FRF_AZ_MEM_PERR_INT_KER_LBN 8 +#define FRF_AZ_MEM_PERR_INT_KER_WIDTH 1 +#define FRF_AZ_RBUF_OWN_INT_KER_LBN 7 +#define FRF_AZ_RBUF_OWN_INT_KER_WIDTH 1 +#define FRF_AZ_TBUF_OWN_INT_KER_LBN 6 +#define FRF_AZ_TBUF_OWN_INT_KER_WIDTH 1 +#define FRF_AZ_RDESCQ_OWN_INT_KER_LBN 5 +#define FRF_AZ_RDESCQ_OWN_INT_KER_WIDTH 1 +#define FRF_AZ_TDESCQ_OWN_INT_KER_LBN 4 +#define FRF_AZ_TDESCQ_OWN_INT_KER_WIDTH 1 +#define FRF_AZ_EVQ_OWN_INT_KER_LBN 3 +#define FRF_AZ_EVQ_OWN_INT_KER_WIDTH 1 +#define FRF_AZ_EVF_OFLO_INT_KER_LBN 2 +#define FRF_AZ_EVF_OFLO_INT_KER_WIDTH 1 +#define FRF_AZ_ILL_ADR_INT_KER_LBN 1 +#define FRF_AZ_ILL_ADR_INT_KER_WIDTH 1 +#define FRF_AZ_SRM_PERR_INT_KER_LBN 0 +#define FRF_AZ_SRM_PERR_INT_KER_WIDTH 1 + + +/* + * FR_AZ_FATAL_INTR_REG_CHAR(128bit): + * Fatal interrupt register for Char + */ +#define FR_AZ_FATAL_INTR_REG_CHAR_OFST 0x00000240 +/* falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_CZ_SRAM_PERR_INT_P_CHAR_EN_LBN 44 +#define FRF_CZ_SRAM_PERR_INT_P_CHAR_EN_WIDTH 1 +#define FRF_AB_PCI_BUSERR_INT_CHAR_EN_LBN 43 +#define FRF_AB_PCI_BUSERR_INT_CHAR_EN_WIDTH 1 +#define FRF_CZ_MBU_PERR_INT_CHAR_EN_LBN 43 +#define FRF_CZ_MBU_PERR_INT_CHAR_EN_WIDTH 1 +#define FRF_AZ_SRAM_OOB_INT_CHAR_EN_LBN 42 +#define FRF_AZ_SRAM_OOB_INT_CHAR_EN_WIDTH 1 +#define FRF_AZ_BUFID_OOB_INT_CHAR_EN_LBN 41 +#define FRF_AZ_BUFID_OOB_INT_CHAR_EN_WIDTH 1 +#define FRF_AZ_MEM_PERR_INT_CHAR_EN_LBN 40 +#define FRF_AZ_MEM_PERR_INT_CHAR_EN_WIDTH 1 +#define FRF_AZ_RBUF_OWN_INT_CHAR_EN_LBN 39 +#define FRF_AZ_RBUF_OWN_INT_CHAR_EN_WIDTH 1 +#define FRF_AZ_TBUF_OWN_INT_CHAR_EN_LBN 38 +#define FRF_AZ_TBUF_OWN_INT_CHAR_EN_WIDTH 1 +#define FRF_AZ_RDESCQ_OWN_INT_CHAR_EN_LBN 37 +#define FRF_AZ_RDESCQ_OWN_INT_CHAR_EN_WIDTH 1 +#define FRF_AZ_TDESCQ_OWN_INT_CHAR_EN_LBN 36 +#define FRF_AZ_TDESCQ_OWN_INT_CHAR_EN_WIDTH 1 +#define FRF_AZ_EVQ_OWN_INT_CHAR_EN_LBN 35 +#define FRF_AZ_EVQ_OWN_INT_CHAR_EN_WIDTH 1 +#define FRF_AZ_EVF_OFLO_INT_CHAR_EN_LBN 34 +#define FRF_AZ_EVF_OFLO_INT_CHAR_EN_WIDTH 1 +#define FRF_AZ_ILL_ADR_INT_CHAR_EN_LBN 33 +#define FRF_AZ_ILL_ADR_INT_CHAR_EN_WIDTH 1 +#define FRF_AZ_SRM_PERR_INT_CHAR_EN_LBN 32 +#define FRF_AZ_SRM_PERR_INT_CHAR_EN_WIDTH 1 +#define FRF_CZ_SRAM_PERR_INT_P_CHAR_LBN 12 +#define FRF_CZ_SRAM_PERR_INT_P_CHAR_WIDTH 1 +#define FRF_AB_PCI_BUSERR_INT_CHAR_LBN 11 +#define FRF_AB_PCI_BUSERR_INT_CHAR_WIDTH 1 +#define FRF_CZ_MBU_PERR_INT_CHAR_LBN 11 +#define FRF_CZ_MBU_PERR_INT_CHAR_WIDTH 1 +#define FRF_AZ_SRAM_OOB_INT_CHAR_LBN 10 +#define FRF_AZ_SRAM_OOB_INT_CHAR_WIDTH 1 +#define FRF_AZ_BUFID_DC_OOB_INT_CHAR_LBN 9 +#define FRF_AZ_BUFID_DC_OOB_INT_CHAR_WIDTH 1 +#define FRF_AZ_MEM_PERR_INT_CHAR_LBN 8 +#define FRF_AZ_MEM_PERR_INT_CHAR_WIDTH 1 +#define FRF_AZ_RBUF_OWN_INT_CHAR_LBN 7 +#define FRF_AZ_RBUF_OWN_INT_CHAR_WIDTH 1 +#define FRF_AZ_TBUF_OWN_INT_CHAR_LBN 6 +#define FRF_AZ_TBUF_OWN_INT_CHAR_WIDTH 1 +#define FRF_AZ_RDESCQ_OWN_INT_CHAR_LBN 5 +#define FRF_AZ_RDESCQ_OWN_INT_CHAR_WIDTH 1 +#define FRF_AZ_TDESCQ_OWN_INT_CHAR_LBN 4 +#define FRF_AZ_TDESCQ_OWN_INT_CHAR_WIDTH 1 +#define FRF_AZ_EVQ_OWN_INT_CHAR_LBN 3 +#define FRF_AZ_EVQ_OWN_INT_CHAR_WIDTH 1 +#define FRF_AZ_EVF_OFLO_INT_CHAR_LBN 2 +#define FRF_AZ_EVF_OFLO_INT_CHAR_WIDTH 1 +#define FRF_AZ_ILL_ADR_INT_CHAR_LBN 1 +#define FRF_AZ_ILL_ADR_INT_CHAR_WIDTH 1 +#define FRF_AZ_SRM_PERR_INT_CHAR_LBN 0 +#define FRF_AZ_SRM_PERR_INT_CHAR_WIDTH 1 + + +/* + * FR_AZ_DP_CTRL_REG(128bit): + * Datapath control register + */ +#define FR_AZ_DP_CTRL_REG_OFST 0x00000250 +/* falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AZ_FLS_EVQ_ID_LBN 0 +#define FRF_AZ_FLS_EVQ_ID_WIDTH 12 + + +/* + * FR_AZ_MEM_STAT_REG(128bit): + * Memory status register + */ +#define FR_AZ_MEM_STAT_REG_OFST 0x00000260 +/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_MEM_PERR_VEC_LBN 53 +#define FRF_AB_MEM_PERR_VEC_WIDTH 40 +#define FRF_AB_MEM_PERR_VEC_DW0_LBN 53 +#define FRF_AB_MEM_PERR_VEC_DW0_WIDTH 32 +#define FRF_AB_MEM_PERR_VEC_DW1_LBN 85 +#define FRF_AB_MEM_PERR_VEC_DW1_WIDTH 6 +#define FRF_AB_MBIST_CORR_LBN 38 +#define FRF_AB_MBIST_CORR_WIDTH 15 +#define FRF_AB_MBIST_ERR_LBN 0 +#define FRF_AB_MBIST_ERR_WIDTH 40 +#define FRF_AB_MBIST_ERR_DW0_LBN 0 +#define FRF_AB_MBIST_ERR_DW0_WIDTH 32 +#define FRF_AB_MBIST_ERR_DW1_LBN 32 +#define FRF_AB_MBIST_ERR_DW1_WIDTH 6 +#define FRF_CZ_MEM_PERR_VEC_LBN 0 +#define FRF_CZ_MEM_PERR_VEC_WIDTH 35 +#define FRF_CZ_MEM_PERR_VEC_DW0_LBN 0 +#define FRF_CZ_MEM_PERR_VEC_DW0_WIDTH 32 +#define FRF_CZ_MEM_PERR_VEC_DW1_LBN 32 +#define FRF_CZ_MEM_PERR_VEC_DW1_WIDTH 3 + + +/* + * FR_PORT0_CS_DEBUG_REG(128bit): + * Debug register + */ + +#define FR_AZ_CS_DEBUG_REG_OFST 0x00000270 +/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_GLB_DEBUG2_SEL_LBN 50 +#define FRF_AB_GLB_DEBUG2_SEL_WIDTH 3 +#define FRF_AB_DEBUG_BLK_SEL2_LBN 47 +#define FRF_AB_DEBUG_BLK_SEL2_WIDTH 3 +#define FRF_AB_DEBUG_BLK_SEL1_LBN 44 +#define FRF_AB_DEBUG_BLK_SEL1_WIDTH 3 +#define FRF_AB_DEBUG_BLK_SEL0_LBN 41 +#define FRF_AB_DEBUG_BLK_SEL0_WIDTH 3 +#define FRF_CZ_CS_PORT_NUM_LBN 40 +#define FRF_CZ_CS_PORT_NUM_WIDTH 2 +#define FRF_AB_MISC_DEBUG_ADDR_LBN 36 +#define FRF_AB_MISC_DEBUG_ADDR_WIDTH 5 +#define FRF_CZ_CS_RESERVED_LBN 36 +#define FRF_CZ_CS_RESERVED_WIDTH 4 +#define FRF_AB_SERDES_DEBUG_ADDR_LBN 31 +#define FRF_AB_SERDES_DEBUG_ADDR_WIDTH 5 +#define FRF_CZ_CS_PORT_FPE_DW0_LBN 1 +#define FRF_CZ_CS_PORT_FPE_DW0_WIDTH 32 +#define FRF_CZ_CS_PORT_FPE_DW1_LBN 33 +#define FRF_CZ_CS_PORT_FPE_DW1_WIDTH 3 +#define FRF_CZ_CS_PORT_FPE_LBN 1 +#define FRF_CZ_CS_PORT_FPE_WIDTH 35 +#define FRF_AB_EM_DEBUG_ADDR_LBN 26 +#define FRF_AB_EM_DEBUG_ADDR_WIDTH 5 +#define FRF_AB_SR_DEBUG_ADDR_LBN 21 +#define FRF_AB_SR_DEBUG_ADDR_WIDTH 5 +#define FRF_AB_EV_DEBUG_ADDR_LBN 16 +#define FRF_AB_EV_DEBUG_ADDR_WIDTH 5 +#define FRF_AB_RX_DEBUG_ADDR_LBN 11 +#define FRF_AB_RX_DEBUG_ADDR_WIDTH 5 +#define FRF_AB_TX_DEBUG_ADDR_LBN 6 +#define FRF_AB_TX_DEBUG_ADDR_WIDTH 5 +#define FRF_AB_CS_BIU_DEBUG_ADDR_LBN 1 +#define FRF_AB_CS_BIU_DEBUG_ADDR_WIDTH 5 +#define FRF_AZ_CS_DEBUG_EN_LBN 0 +#define FRF_AZ_CS_DEBUG_EN_WIDTH 1 + + +/* + * FR_AZ_DRIVER_REG(128bit): + * Driver scratch register [0-7] + */ +#define FR_AZ_DRIVER_REG_OFST 0x00000280 +/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */ +#define FR_AZ_DRIVER_REG_STEP 16 +#define FR_AZ_DRIVER_REG_ROWS 8 + +#define FRF_AZ_DRIVER_DW0_LBN 0 +#define FRF_AZ_DRIVER_DW0_WIDTH 32 + + +/* + * FR_AZ_ALTERA_BUILD_REG(128bit): + * Altera build register + */ +#define FR_AZ_ALTERA_BUILD_REG_OFST 0x00000300 +/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AZ_ALTERA_BUILD_VER_LBN 0 +#define FRF_AZ_ALTERA_BUILD_VER_WIDTH 32 + + +/* + * FR_AZ_CSR_SPARE_REG(128bit): + * Spare register + */ +#define FR_AZ_CSR_SPARE_REG_OFST 0x00000310 +/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AZ_MEM_PERR_EN_TX_DATA_LBN 72 +#define FRF_AZ_MEM_PERR_EN_TX_DATA_WIDTH 2 +#define FRF_AZ_MEM_PERR_EN_LBN 64 +#define FRF_AZ_MEM_PERR_EN_WIDTH 38 +#define FRF_AZ_MEM_PERR_EN_DW0_LBN 64 +#define FRF_AZ_MEM_PERR_EN_DW0_WIDTH 32 +#define FRF_AZ_MEM_PERR_EN_DW1_LBN 96 +#define FRF_AZ_MEM_PERR_EN_DW1_WIDTH 6 +#define FRF_AZ_CSR_SPARE_BITS_LBN 0 +#define FRF_AZ_CSR_SPARE_BITS_WIDTH 32 + + +/* + * FR_BZ_DEBUG_DATA_OUT_REG(128bit): + * Live Debug and Debug 2 out ports + */ +#define FR_BZ_DEBUG_DATA_OUT_REG_OFST 0x00000350 +/* falconb0,sienaa0=net_func_bar2 */ + +#define FRF_BZ_DEBUG2_PORT_LBN 25 +#define FRF_BZ_DEBUG2_PORT_WIDTH 15 +#define FRF_BZ_DEBUG1_PORT_LBN 0 +#define FRF_BZ_DEBUG1_PORT_WIDTH 25 + + +/* + * FR_BZ_EVQ_RPTR_REGP0(32bit): + * Event queue read pointer register + */ +#define FR_BZ_EVQ_RPTR_REGP0_OFST 0x00000400 +/* falconb0,sienaa0=net_func_bar2 */ +#define FR_BZ_EVQ_RPTR_REGP0_STEP 8192 +#define FR_BZ_EVQ_RPTR_REGP0_ROWS 1024 +/* + * FR_AA_EVQ_RPTR_REG_KER(32bit): + * Event queue read pointer register + */ +#define FR_AA_EVQ_RPTR_REG_KER_OFST 0x00011b00 +/* falcona0=net_func_bar2 */ +#define FR_AA_EVQ_RPTR_REG_KER_STEP 4 +#define FR_AA_EVQ_RPTR_REG_KER_ROWS 4 +/* + * FR_AZ_EVQ_RPTR_REG(32bit): + * Event queue read pointer register + */ +#define FR_AZ_EVQ_RPTR_REG_OFST 0x00fa0000 +/* falconb0=net_func_bar2,sienaa0=net_func_bar2,falcona0=char_func_bar0 */ +#define FR_AZ_EVQ_RPTR_REG_STEP 16 +#define FR_AB_EVQ_RPTR_REG_ROWS 4096 +#define FR_CZ_EVQ_RPTR_REG_ROWS 1024 +/* + * FR_BB_EVQ_RPTR_REGP123(32bit): + * Event queue read pointer register + */ +#define FR_BB_EVQ_RPTR_REGP123_OFST 0x01000400 +/* falconb0=net_func_bar2 */ +#define FR_BB_EVQ_RPTR_REGP123_STEP 8192 +#define FR_BB_EVQ_RPTR_REGP123_ROWS 3072 + +#define FRF_AZ_EVQ_RPTR_VLD_LBN 15 +#define FRF_AZ_EVQ_RPTR_VLD_WIDTH 1 +#define FRF_AZ_EVQ_RPTR_LBN 0 +#define FRF_AZ_EVQ_RPTR_WIDTH 15 + + +/* + * FR_BZ_TIMER_COMMAND_REGP0(128bit): + * Timer Command Registers + */ +#define FR_BZ_TIMER_COMMAND_REGP0_OFST 0x00000420 +/* falconb0,sienaa0=net_func_bar2 */ +#define FR_BZ_TIMER_COMMAND_REGP0_STEP 8192 +#define FR_BZ_TIMER_COMMAND_REGP0_ROWS 1024 +/* + * FR_AA_TIMER_COMMAND_REG_KER(128bit): + * Timer Command Registers + */ +#define FR_AA_TIMER_COMMAND_REG_KER_OFST 0x00000420 +/* falcona0=net_func_bar2 */ +#define FR_AA_TIMER_COMMAND_REG_KER_STEP 8192 +#define FR_AA_TIMER_COMMAND_REG_KER_ROWS 4 +/* + * FR_AB_TIMER_COMMAND_REGP123(128bit): + * Timer Command Registers + */ +#define FR_AB_TIMER_COMMAND_REGP123_OFST 0x01000420 +/* falconb0=net_func_bar2,falcona0=char_func_bar0 */ +#define FR_AB_TIMER_COMMAND_REGP123_STEP 8192 +#define FR_AB_TIMER_COMMAND_REGP123_ROWS 3072 +/* + * FR_AA_TIMER_COMMAND_REGP0(128bit): + * Timer Command Registers + */ +#define FR_AA_TIMER_COMMAND_REGP0_OFST 0x00008420 +/* falcona0=char_func_bar0 */ +#define FR_AA_TIMER_COMMAND_REGP0_STEP 8192 +#define FR_AA_TIMER_COMMAND_REGP0_ROWS 1020 + +#define FRF_CZ_TC_TIMER_MODE_LBN 14 +#define FRF_CZ_TC_TIMER_MODE_WIDTH 2 +#define FRF_AB_TC_TIMER_MODE_LBN 12 +#define FRF_AB_TC_TIMER_MODE_WIDTH 2 +#define FRF_CZ_TC_TIMER_VAL_LBN 0 +#define FRF_CZ_TC_TIMER_VAL_WIDTH 14 +#define FRF_AB_TC_TIMER_VAL_LBN 0 +#define FRF_AB_TC_TIMER_VAL_WIDTH 12 + + +/* + * FR_AZ_DRV_EV_REG(128bit): + * Driver generated event register + */ +#define FR_AZ_DRV_EV_REG_OFST 0x00000440 +/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AZ_DRV_EV_QID_LBN 64 +#define FRF_AZ_DRV_EV_QID_WIDTH 12 +#define FRF_AZ_DRV_EV_DATA_LBN 0 +#define FRF_AZ_DRV_EV_DATA_WIDTH 64 +#define FRF_AZ_DRV_EV_DATA_DW0_LBN 0 +#define FRF_AZ_DRV_EV_DATA_DW0_WIDTH 32 +#define FRF_AZ_DRV_EV_DATA_DW1_LBN 32 +#define FRF_AZ_DRV_EV_DATA_DW1_WIDTH 32 + + +/* + * FR_AZ_EVQ_CTL_REG(128bit): + * Event queue control register + */ +#define FR_AZ_EVQ_CTL_REG_OFST 0x00000450 +/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_CZ_RX_EVQ_WAKEUP_MASK_LBN 15 +#define FRF_CZ_RX_EVQ_WAKEUP_MASK_WIDTH 10 +#define FRF_BB_RX_EVQ_WAKEUP_MASK_LBN 15 +#define FRF_BB_RX_EVQ_WAKEUP_MASK_WIDTH 6 +#define FRF_AZ_EVQ_OWNERR_CTL_LBN 14 +#define FRF_AZ_EVQ_OWNERR_CTL_WIDTH 1 +#define FRF_AZ_EVQ_FIFO_AF_TH_LBN 7 +#define FRF_AZ_EVQ_FIFO_AF_TH_WIDTH 7 +#define FRF_AZ_EVQ_FIFO_NOTAF_TH_LBN 0 +#define FRF_AZ_EVQ_FIFO_NOTAF_TH_WIDTH 7 + + +/* + * FR_AZ_EVQ_CNT1_REG(128bit): + * Event counter 1 register + */ +#define FR_AZ_EVQ_CNT1_REG_OFST 0x00000460 +/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AZ_EVQ_CNT_PRE_FIFO_LBN 120 +#define FRF_AZ_EVQ_CNT_PRE_FIFO_WIDTH 7 +#define FRF_AZ_EVQ_CNT_TOBIU_LBN 100 +#define FRF_AZ_EVQ_CNT_TOBIU_WIDTH 20 +#define FRF_AZ_EVQ_TX_REQ_CNT_LBN 80 +#define FRF_AZ_EVQ_TX_REQ_CNT_WIDTH 20 +#define FRF_AZ_EVQ_RX_REQ_CNT_LBN 60 +#define FRF_AZ_EVQ_RX_REQ_CNT_WIDTH 20 +#define FRF_AZ_EVQ_EM_REQ_CNT_LBN 40 +#define FRF_AZ_EVQ_EM_REQ_CNT_WIDTH 20 +#define FRF_AZ_EVQ_CSR_REQ_CNT_LBN 20 +#define FRF_AZ_EVQ_CSR_REQ_CNT_WIDTH 20 +#define FRF_AZ_EVQ_ERR_REQ_CNT_LBN 0 +#define FRF_AZ_EVQ_ERR_REQ_CNT_WIDTH 20 + + +/* + * FR_AZ_EVQ_CNT2_REG(128bit): + * Event counter 2 register + */ +#define FR_AZ_EVQ_CNT2_REG_OFST 0x00000470 +/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AZ_EVQ_UPD_REQ_CNT_LBN 104 +#define FRF_AZ_EVQ_UPD_REQ_CNT_WIDTH 20 +#define FRF_AZ_EVQ_CLR_REQ_CNT_LBN 84 +#define FRF_AZ_EVQ_CLR_REQ_CNT_WIDTH 20 +#define FRF_AZ_EVQ_RDY_CNT_LBN 80 +#define FRF_AZ_EVQ_RDY_CNT_WIDTH 4 +#define FRF_AZ_EVQ_WU_REQ_CNT_LBN 60 +#define FRF_AZ_EVQ_WU_REQ_CNT_WIDTH 20 +#define FRF_AZ_EVQ_WET_REQ_CNT_LBN 40 +#define FRF_AZ_EVQ_WET_REQ_CNT_WIDTH 20 +#define FRF_AZ_EVQ_INIT_REQ_CNT_LBN 20 +#define FRF_AZ_EVQ_INIT_REQ_CNT_WIDTH 20 +#define FRF_AZ_EVQ_TM_REQ_CNT_LBN 0 +#define FRF_AZ_EVQ_TM_REQ_CNT_WIDTH 20 + + +/* + * FR_CZ_USR_EV_REG(32bit): + * Event mailbox register + */ +#define FR_CZ_USR_EV_REG_OFST 0x00000540 +/* sienaa0=net_func_bar2 */ +#define FR_CZ_USR_EV_REG_STEP 8192 +#define FR_CZ_USR_EV_REG_ROWS 1024 + +#define FRF_CZ_USR_EV_DATA_LBN 0 +#define FRF_CZ_USR_EV_DATA_WIDTH 32 + + +/* + * FR_AZ_BUF_TBL_CFG_REG(128bit): + * Buffer table configuration register + */ +#define FR_AZ_BUF_TBL_CFG_REG_OFST 0x00000600 +/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AZ_BUF_TBL_MODE_LBN 3 +#define FRF_AZ_BUF_TBL_MODE_WIDTH 1 + + +/* + * FR_AZ_SRM_RX_DC_CFG_REG(128bit): + * SRAM receive descriptor cache configuration register + */ +#define FR_AZ_SRM_RX_DC_CFG_REG_OFST 0x00000610 +/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AZ_SRM_CLK_TMP_EN_LBN 21 +#define FRF_AZ_SRM_CLK_TMP_EN_WIDTH 1 +#define FRF_AZ_SRM_RX_DC_BASE_ADR_LBN 0 +#define FRF_AZ_SRM_RX_DC_BASE_ADR_WIDTH 21 + + +/* + * FR_AZ_SRM_TX_DC_CFG_REG(128bit): + * SRAM transmit descriptor cache configuration register + */ +#define FR_AZ_SRM_TX_DC_CFG_REG_OFST 0x00000620 +/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AZ_SRM_TX_DC_BASE_ADR_LBN 0 +#define FRF_AZ_SRM_TX_DC_BASE_ADR_WIDTH 21 + + +/* + * FR_AZ_SRM_CFG_REG(128bit): + * SRAM configuration register + */ +#define FR_AZ_SRM_CFG_REG_SF_OFST 0x00000380 +/* falcona0,falconb0=eeprom_flash */ +/* + * FR_AZ_SRM_CFG_REG(128bit): + * SRAM configuration register + */ +#define FR_AZ_SRM_CFG_REG_OFST 0x00000630 +/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AZ_SRM_OOB_ADR_INTEN_LBN 5 +#define FRF_AZ_SRM_OOB_ADR_INTEN_WIDTH 1 +#define FRF_AZ_SRM_OOB_BUF_INTEN_LBN 4 +#define FRF_AZ_SRM_OOB_BUF_INTEN_WIDTH 1 +#define FRF_AZ_SRM_INIT_EN_LBN 3 +#define FRF_AZ_SRM_INIT_EN_WIDTH 1 +#define FRF_AZ_SRM_NUM_BANK_LBN 2 +#define FRF_AZ_SRM_NUM_BANK_WIDTH 1 +#define FRF_AZ_SRM_BANK_SIZE_LBN 0 +#define FRF_AZ_SRM_BANK_SIZE_WIDTH 2 + + +/* + * FR_AZ_BUF_TBL_UPD_REG(128bit): + * Buffer table update register + */ +#define FR_AZ_BUF_TBL_UPD_REG_OFST 0x00000650 +/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AZ_BUF_UPD_CMD_LBN 63 +#define FRF_AZ_BUF_UPD_CMD_WIDTH 1 +#define FRF_AZ_BUF_CLR_CMD_LBN 62 +#define FRF_AZ_BUF_CLR_CMD_WIDTH 1 +#define FRF_AZ_BUF_CLR_END_ID_LBN 32 +#define FRF_AZ_BUF_CLR_END_ID_WIDTH 20 +#define FRF_AZ_BUF_CLR_START_ID_LBN 0 +#define FRF_AZ_BUF_CLR_START_ID_WIDTH 20 + + +/* + * FR_AZ_SRM_UPD_EVQ_REG(128bit): + * Buffer table update register + */ +#define FR_AZ_SRM_UPD_EVQ_REG_OFST 0x00000660 +/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AZ_SRM_UPD_EVQ_ID_LBN 0 +#define FRF_AZ_SRM_UPD_EVQ_ID_WIDTH 12 + + +/* + * FR_AZ_SRAM_PARITY_REG(128bit): + * SRAM parity register. + */ +#define FR_AZ_SRAM_PARITY_REG_OFST 0x00000670 +/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_CZ_BYPASS_ECC_LBN 3 +#define FRF_CZ_BYPASS_ECC_WIDTH 1 +#define FRF_CZ_SEC_INT_LBN 2 +#define FRF_CZ_SEC_INT_WIDTH 1 +#define FRF_CZ_FORCE_SRAM_DOUBLE_ERR_LBN 1 +#define FRF_CZ_FORCE_SRAM_DOUBLE_ERR_WIDTH 1 +#define FRF_CZ_FORCE_SRAM_SINGLE_ERR_LBN 0 +#define FRF_CZ_FORCE_SRAM_SINGLE_ERR_WIDTH 1 +#define FRF_AB_FORCE_SRAM_PERR_LBN 0 +#define FRF_AB_FORCE_SRAM_PERR_WIDTH 1 + + +/* + * FR_AZ_RX_CFG_REG(128bit): + * Receive configuration register + */ +#define FR_AZ_RX_CFG_REG_OFST 0x00000800 +/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_CZ_RX_HDR_SPLIT_EN_LBN 71 +#define FRF_CZ_RX_HDR_SPLIT_EN_WIDTH 1 +#define FRF_CZ_RX_HDR_SPLIT_PLD_BUF_SIZE_LBN 62 +#define FRF_CZ_RX_HDR_SPLIT_PLD_BUF_SIZE_WIDTH 9 +#define FRF_CZ_RX_HDR_SPLIT_HDR_BUF_SIZE_LBN 53 +#define FRF_CZ_RX_HDR_SPLIT_HDR_BUF_SIZE_WIDTH 9 +#define FRF_CZ_RX_PRE_RFF_IPG_LBN 49 +#define FRF_CZ_RX_PRE_RFF_IPG_WIDTH 4 +#define FRF_BZ_RX_TCP_SUP_LBN 48 +#define FRF_BZ_RX_TCP_SUP_WIDTH 1 +#define FRF_BZ_RX_INGR_EN_LBN 47 +#define FRF_BZ_RX_INGR_EN_WIDTH 1 +#define FRF_BZ_RX_IP_HASH_LBN 46 +#define FRF_BZ_RX_IP_HASH_WIDTH 1 +#define FRF_BZ_RX_HASH_ALG_LBN 45 +#define FRF_BZ_RX_HASH_ALG_WIDTH 1 +#define FRF_BZ_RX_HASH_INSRT_HDR_LBN 44 +#define FRF_BZ_RX_HASH_INSRT_HDR_WIDTH 1 +#define FRF_BZ_RX_DESC_PUSH_EN_LBN 43 +#define FRF_BZ_RX_DESC_PUSH_EN_WIDTH 1 +#define FRF_BZ_RX_RDW_PATCH_EN_LBN 42 +#define FRF_BZ_RX_RDW_PATCH_EN_WIDTH 1 +#define FRF_BB_RX_PCI_BURST_SIZE_LBN 39 +#define FRF_BB_RX_PCI_BURST_SIZE_WIDTH 3 +#define FRF_BZ_RX_OWNERR_CTL_LBN 38 +#define FRF_BZ_RX_OWNERR_CTL_WIDTH 1 +#define FRF_BZ_RX_XON_TX_TH_LBN 33 +#define FRF_BZ_RX_XON_TX_TH_WIDTH 5 +#define FRF_AA_RX_DESC_PUSH_EN_LBN 35 +#define FRF_AA_RX_DESC_PUSH_EN_WIDTH 1 +#define FRF_AA_RX_RDW_PATCH_EN_LBN 34 +#define FRF_AA_RX_RDW_PATCH_EN_WIDTH 1 +#define FRF_AA_RX_PCI_BURST_SIZE_LBN 31 +#define FRF_AA_RX_PCI_BURST_SIZE_WIDTH 3 +#define FRF_BZ_RX_XOFF_TX_TH_LBN 28 +#define FRF_BZ_RX_XOFF_TX_TH_WIDTH 5 +#define FRF_AA_RX_OWNERR_CTL_LBN 30 +#define FRF_AA_RX_OWNERR_CTL_WIDTH 1 +#define FRF_AA_RX_XON_TX_TH_LBN 25 +#define FRF_AA_RX_XON_TX_TH_WIDTH 5 +#define FRF_BZ_RX_USR_BUF_SIZE_LBN 19 +#define FRF_BZ_RX_USR_BUF_SIZE_WIDTH 9 +#define FRF_AA_RX_XOFF_TX_TH_LBN 20 +#define FRF_AA_RX_XOFF_TX_TH_WIDTH 5 +#define FRF_AA_RX_USR_BUF_SIZE_LBN 11 +#define FRF_AA_RX_USR_BUF_SIZE_WIDTH 9 +#define FRF_BZ_RX_XON_MAC_TH_LBN 10 +#define FRF_BZ_RX_XON_MAC_TH_WIDTH 9 +#define FRF_AA_RX_XON_MAC_TH_LBN 6 +#define FRF_AA_RX_XON_MAC_TH_WIDTH 5 +#define FRF_BZ_RX_XOFF_MAC_TH_LBN 1 +#define FRF_BZ_RX_XOFF_MAC_TH_WIDTH 9 +#define FRF_AA_RX_XOFF_MAC_TH_LBN 1 +#define FRF_AA_RX_XOFF_MAC_TH_WIDTH 5 +#define FRF_AZ_RX_XOFF_MAC_EN_LBN 0 +#define FRF_AZ_RX_XOFF_MAC_EN_WIDTH 1 + + +/* + * FR_AZ_RX_FILTER_CTL_REG(128bit): + * Receive filter control registers + */ +#define FR_AZ_RX_FILTER_CTL_REG_OFST 0x00000810 +/* falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_CZ_ETHERNET_WILDCARD_SEARCH_LIMIT_LBN 94 +#define FRF_CZ_ETHERNET_WILDCARD_SEARCH_LIMIT_WIDTH 8 +#define FRF_CZ_ETHERNET_FULL_SEARCH_LIMIT_LBN 86 +#define FRF_CZ_ETHERNET_FULL_SEARCH_LIMIT_WIDTH 8 +#define FRF_CZ_RX_FILTER_ALL_VLAN_ETHERTYPES_LBN 85 +#define FRF_CZ_RX_FILTER_ALL_VLAN_ETHERTYPES_WIDTH 1 +#define FRF_CZ_RX_VLAN_MATCH_ETHERTYPE_LBN 69 +#define FRF_CZ_RX_VLAN_MATCH_ETHERTYPE_WIDTH 16 +#define FRF_CZ_MULTICAST_NOMATCH_Q_ID_LBN 57 +#define FRF_CZ_MULTICAST_NOMATCH_Q_ID_WIDTH 12 +#define FRF_CZ_MULTICAST_NOMATCH_RSS_ENABLED_LBN 56 +#define FRF_CZ_MULTICAST_NOMATCH_RSS_ENABLED_WIDTH 1 +#define FRF_CZ_MULTICAST_NOMATCH_IP_OVERRIDE_LBN 55 +#define FRF_CZ_MULTICAST_NOMATCH_IP_OVERRIDE_WIDTH 1 +#define FRF_CZ_UNICAST_NOMATCH_Q_ID_LBN 43 +#define FRF_CZ_UNICAST_NOMATCH_Q_ID_WIDTH 12 +#define FRF_CZ_UNICAST_NOMATCH_RSS_ENABLED_LBN 42 +#define FRF_CZ_UNICAST_NOMATCH_RSS_ENABLED_WIDTH 1 +#define FRF_CZ_UNICAST_NOMATCH_IP_OVERRIDE_LBN 41 +#define FRF_CZ_UNICAST_NOMATCH_IP_OVERRIDE_WIDTH 1 +#define FRF_BZ_SCATTER_ENBL_NO_MATCH_Q_LBN 40 +#define FRF_BZ_SCATTER_ENBL_NO_MATCH_Q_WIDTH 1 +#define FRF_AZ_UDP_FULL_SRCH_LIMIT_LBN 32 +#define FRF_AZ_UDP_FULL_SRCH_LIMIT_WIDTH 8 +#define FRF_AZ_NUM_KER_LBN 24 +#define FRF_AZ_NUM_KER_WIDTH 2 +#define FRF_AZ_UDP_WILD_SRCH_LIMIT_LBN 16 +#define FRF_AZ_UDP_WILD_SRCH_LIMIT_WIDTH 8 +#define FRF_AZ_TCP_WILD_SRCH_LIMIT_LBN 8 +#define FRF_AZ_TCP_WILD_SRCH_LIMIT_WIDTH 8 +#define FRF_AZ_TCP_FULL_SRCH_LIMIT_LBN 0 +#define FRF_AZ_TCP_FULL_SRCH_LIMIT_WIDTH 8 + + +/* + * FR_AZ_RX_FLUSH_DESCQ_REG(128bit): + * Receive flush descriptor queue register + */ +#define FR_AZ_RX_FLUSH_DESCQ_REG_OFST 0x00000820 +/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AZ_RX_FLUSH_DESCQ_CMD_LBN 24 +#define FRF_AZ_RX_FLUSH_DESCQ_CMD_WIDTH 1 +#define FRF_AZ_RX_FLUSH_DESCQ_LBN 0 +#define FRF_AZ_RX_FLUSH_DESCQ_WIDTH 12 + + +/* + * FR_BZ_RX_DESC_UPD_REGP0(128bit): + * Receive descriptor update register. + */ +#define FR_BZ_RX_DESC_UPD_REGP0_OFST 0x00000830 +/* falconb0,sienaa0=net_func_bar2 */ +#define FR_BZ_RX_DESC_UPD_REGP0_STEP 8192 +#define FR_BZ_RX_DESC_UPD_REGP0_ROWS 1024 +/* + * FR_AA_RX_DESC_UPD_REG_KER(128bit): + * Receive descriptor update register. + */ +#define FR_AA_RX_DESC_UPD_REG_KER_OFST 0x00000830 +/* falcona0=net_func_bar2 */ +#define FR_AA_RX_DESC_UPD_REG_KER_STEP 8192 +#define FR_AA_RX_DESC_UPD_REG_KER_ROWS 4 +/* + * FR_AB_RX_DESC_UPD_REGP123(128bit): + * Receive descriptor update register. + */ +#define FR_AB_RX_DESC_UPD_REGP123_OFST 0x01000830 +/* falconb0=net_func_bar2,falcona0=char_func_bar0 */ +#define FR_AB_RX_DESC_UPD_REGP123_STEP 8192 +#define FR_AB_RX_DESC_UPD_REGP123_ROWS 3072 +/* + * FR_AA_RX_DESC_UPD_REGP0(128bit): + * Receive descriptor update register. + */ +#define FR_AA_RX_DESC_UPD_REGP0_OFST 0x00008830 +/* falcona0=char_func_bar0 */ +#define FR_AA_RX_DESC_UPD_REGP0_STEP 8192 +#define FR_AA_RX_DESC_UPD_REGP0_ROWS 1020 + +#define FRF_AZ_RX_DESC_WPTR_LBN 96 +#define FRF_AZ_RX_DESC_WPTR_WIDTH 12 +#define FRF_AZ_RX_DESC_PUSH_CMD_LBN 95 +#define FRF_AZ_RX_DESC_PUSH_CMD_WIDTH 1 +#define FRF_AZ_RX_DESC_LBN 0 +#define FRF_AZ_RX_DESC_WIDTH 64 +#define FRF_AZ_RX_DESC_DW0_LBN 0 +#define FRF_AZ_RX_DESC_DW0_WIDTH 32 +#define FRF_AZ_RX_DESC_DW1_LBN 32 +#define FRF_AZ_RX_DESC_DW1_WIDTH 32 + + +/* + * FR_AZ_RX_DC_CFG_REG(128bit): + * Receive descriptor cache configuration register + */ +#define FR_AZ_RX_DC_CFG_REG_OFST 0x00000840 +/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AZ_RX_MAX_PF_LBN 2 +#define FRF_AZ_RX_MAX_PF_WIDTH 2 +#define FRF_AZ_RX_DC_SIZE_LBN 0 +#define FRF_AZ_RX_DC_SIZE_WIDTH 2 +#define FFE_AZ_RX_DC_SIZE_64 3 +#define FFE_AZ_RX_DC_SIZE_32 2 +#define FFE_AZ_RX_DC_SIZE_16 1 +#define FFE_AZ_RX_DC_SIZE_8 0 + + +/* + * FR_AZ_RX_DC_PF_WM_REG(128bit): + * Receive descriptor cache pre-fetch watermark register + */ +#define FR_AZ_RX_DC_PF_WM_REG_OFST 0x00000850 +/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AZ_RX_DC_PF_HWM_LBN 6 +#define FRF_AZ_RX_DC_PF_HWM_WIDTH 6 +#define FRF_AZ_RX_DC_PF_LWM_LBN 0 +#define FRF_AZ_RX_DC_PF_LWM_WIDTH 6 + + +/* + * FR_BZ_RX_RSS_TKEY_REG(128bit): + * RSS Toeplitz hash key + */ +#define FR_BZ_RX_RSS_TKEY_REG_OFST 0x00000860 +/* falconb0,sienaa0=net_func_bar2 */ + +#define FRF_BZ_RX_RSS_TKEY_LBN 96 +#define FRF_BZ_RX_RSS_TKEY_WIDTH 32 +#define FRF_BZ_RX_RSS_TKEY_DW3_LBN 96 +#define FRF_BZ_RX_RSS_TKEY_DW3_WIDTH 32 +#define FRF_BZ_RX_RSS_TKEY_DW2_LBN 64 +#define FRF_BZ_RX_RSS_TKEY_DW2_WIDTH 32 +#define FRF_BZ_RX_RSS_TKEY_DW1_LBN 32 +#define FRF_BZ_RX_RSS_TKEY_DW1_WIDTH 32 +#define FRF_BZ_RX_RSS_TKEY_DW0_LBN 0 +#define FRF_BZ_RX_RSS_TKEY_DW0_WIDTH 32 + + +/* + * FR_AZ_RX_NODESC_DROP_REG(128bit): + * Receive dropped packet counter register + */ +#define FR_AZ_RX_NODESC_DROP_REG_OFST 0x00000880 +/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AZ_RX_NODESC_DROP_CNT_LBN 0 +#define FRF_AZ_RX_NODESC_DROP_CNT_WIDTH 16 + + +/* + * FR_AZ_RX_SELF_RST_REG(128bit): + * Receive self reset register + */ +#define FR_AZ_RX_SELF_RST_REG_OFST 0x00000890 +/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AZ_RX_ISCSI_DIS_LBN 17 +#define FRF_AZ_RX_ISCSI_DIS_WIDTH 1 +#define FRF_AB_RX_SW_RST_REG_LBN 16 +#define FRF_AB_RX_SW_RST_REG_WIDTH 1 +#define FRF_AB_RX_SELF_RST_EN_LBN 8 +#define FRF_AB_RX_SELF_RST_EN_WIDTH 1 +#define FRF_AZ_RX_MAX_PF_LAT_LBN 4 +#define FRF_AZ_RX_MAX_PF_LAT_WIDTH 4 +#define FRF_AZ_RX_MAX_LU_LAT_LBN 0 +#define FRF_AZ_RX_MAX_LU_LAT_WIDTH 4 + + +/* + * FR_AZ_RX_DEBUG_REG(128bit): + * undocumented register + */ +#define FR_AZ_RX_DEBUG_REG_OFST 0x000008a0 +/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AZ_RX_DEBUG_LBN 0 +#define FRF_AZ_RX_DEBUG_WIDTH 64 +#define FRF_AZ_RX_DEBUG_DW0_LBN 0 +#define FRF_AZ_RX_DEBUG_DW0_WIDTH 32 +#define FRF_AZ_RX_DEBUG_DW1_LBN 32 +#define FRF_AZ_RX_DEBUG_DW1_WIDTH 32 + + +/* + * FR_AZ_RX_PUSH_DROP_REG(128bit): + * Receive descriptor push dropped counter register + */ +#define FR_AZ_RX_PUSH_DROP_REG_OFST 0x000008b0 +/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AZ_RX_PUSH_DROP_CNT_LBN 0 +#define FRF_AZ_RX_PUSH_DROP_CNT_WIDTH 32 + + +/* + * FR_CZ_RX_RSS_IPV6_REG1(128bit): + * IPv6 RSS Toeplitz hash key low bytes + */ +#define FR_CZ_RX_RSS_IPV6_REG1_OFST 0x000008d0 +/* sienaa0=net_func_bar2 */ + +#define FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN 0 +#define FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH 128 +#define FRF_CZ_RX_RSS_IPV6_TKEY_LO_DW0_LBN 0 +#define FRF_CZ_RX_RSS_IPV6_TKEY_LO_DW0_WIDTH 32 +#define FRF_CZ_RX_RSS_IPV6_TKEY_LO_DW1_LBN 32 +#define FRF_CZ_RX_RSS_IPV6_TKEY_LO_DW1_WIDTH 32 +#define FRF_CZ_RX_RSS_IPV6_TKEY_LO_DW2_LBN 64 +#define FRF_CZ_RX_RSS_IPV6_TKEY_LO_DW2_WIDTH 32 +#define FRF_CZ_RX_RSS_IPV6_TKEY_LO_DW3_LBN 96 +#define FRF_CZ_RX_RSS_IPV6_TKEY_LO_DW3_WIDTH 32 + + +/* + * FR_CZ_RX_RSS_IPV6_REG2(128bit): + * IPv6 RSS Toeplitz hash key middle bytes + */ +#define FR_CZ_RX_RSS_IPV6_REG2_OFST 0x000008e0 +/* sienaa0=net_func_bar2 */ + +#define FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN 0 +#define FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH 128 +#define FRF_CZ_RX_RSS_IPV6_TKEY_MID_DW0_LBN 0 +#define FRF_CZ_RX_RSS_IPV6_TKEY_MID_DW0_WIDTH 32 +#define FRF_CZ_RX_RSS_IPV6_TKEY_MID_DW1_LBN 32 +#define FRF_CZ_RX_RSS_IPV6_TKEY_MID_DW1_WIDTH 32 +#define FRF_CZ_RX_RSS_IPV6_TKEY_MID_DW2_LBN 64 +#define FRF_CZ_RX_RSS_IPV6_TKEY_MID_DW2_WIDTH 32 +#define FRF_CZ_RX_RSS_IPV6_TKEY_MID_DW3_LBN 96 +#define FRF_CZ_RX_RSS_IPV6_TKEY_MID_DW3_WIDTH 32 + + +/* + * FR_CZ_RX_RSS_IPV6_REG3(128bit): + * IPv6 RSS Toeplitz hash key upper bytes and IPv6 RSS settings + */ +#define FR_CZ_RX_RSS_IPV6_REG3_OFST 0x000008f0 +/* sienaa0=net_func_bar2 */ + +#define FRF_CZ_RX_RSS_IPV6_THASH_ENABLE_LBN 66 +#define FRF_CZ_RX_RSS_IPV6_THASH_ENABLE_WIDTH 1 +#define FRF_CZ_RX_RSS_IPV6_IP_THASH_ENABLE_LBN 65 +#define FRF_CZ_RX_RSS_IPV6_IP_THASH_ENABLE_WIDTH 1 +#define FRF_CZ_RX_RSS_IPV6_TCP_SUPPRESS_LBN 64 +#define FRF_CZ_RX_RSS_IPV6_TCP_SUPPRESS_WIDTH 1 +#define FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN 0 +#define FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH 64 +#define FRF_CZ_RX_RSS_IPV6_TKEY_HI_DW0_LBN 0 +#define FRF_CZ_RX_RSS_IPV6_TKEY_HI_DW0_WIDTH 32 +#define FRF_CZ_RX_RSS_IPV6_TKEY_HI_DW1_LBN 32 +#define FRF_CZ_RX_RSS_IPV6_TKEY_HI_DW1_WIDTH 32 + + +/* + * FR_AZ_TX_FLUSH_DESCQ_REG(128bit): + * Transmit flush descriptor queue register + */ +#define FR_AZ_TX_FLUSH_DESCQ_REG_OFST 0x00000a00 +/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AZ_TX_FLUSH_DESCQ_CMD_LBN 12 +#define FRF_AZ_TX_FLUSH_DESCQ_CMD_WIDTH 1 +#define FRF_AZ_TX_FLUSH_DESCQ_LBN 0 +#define FRF_AZ_TX_FLUSH_DESCQ_WIDTH 12 + + +/* + * FR_BZ_TX_DESC_UPD_REGP0(128bit): + * Transmit descriptor update register. + */ +#define FR_BZ_TX_DESC_UPD_REGP0_OFST 0x00000a10 +/* falconb0,sienaa0=net_func_bar2 */ +#define FR_BZ_TX_DESC_UPD_REGP0_STEP 8192 +#define FR_BZ_TX_DESC_UPD_REGP0_ROWS 1024 +/* + * FR_AA_TX_DESC_UPD_REG_KER(128bit): + * Transmit descriptor update register. + */ +#define FR_AA_TX_DESC_UPD_REG_KER_OFST 0x00000a10 +/* falcona0=net_func_bar2 */ +#define FR_AA_TX_DESC_UPD_REG_KER_STEP 8192 +#define FR_AA_TX_DESC_UPD_REG_KER_ROWS 8 +/* + * FR_AB_TX_DESC_UPD_REGP123(128bit): + * Transmit descriptor update register. + */ +#define FR_AB_TX_DESC_UPD_REGP123_OFST 0x01000a10 +/* falconb0=net_func_bar2,falcona0=char_func_bar0 */ +#define FR_AB_TX_DESC_UPD_REGP123_STEP 8192 +#define FR_AB_TX_DESC_UPD_REGP123_ROWS 3072 +/* + * FR_AA_TX_DESC_UPD_REGP0(128bit): + * Transmit descriptor update register. + */ +#define FR_AA_TX_DESC_UPD_REGP0_OFST 0x00008a10 +/* falcona0=char_func_bar0 */ +#define FR_AA_TX_DESC_UPD_REGP0_STEP 8192 +#define FR_AA_TX_DESC_UPD_REGP0_ROWS 1020 + +#define FRF_AZ_TX_DESC_WPTR_LBN 96 +#define FRF_AZ_TX_DESC_WPTR_WIDTH 12 +#define FRF_AZ_TX_DESC_PUSH_CMD_LBN 95 +#define FRF_AZ_TX_DESC_PUSH_CMD_WIDTH 1 +#define FRF_AZ_TX_DESC_LBN 0 +#define FRF_AZ_TX_DESC_WIDTH 95 +#define FRF_AZ_TX_DESC_DW0_LBN 0 +#define FRF_AZ_TX_DESC_DW0_WIDTH 32 +#define FRF_AZ_TX_DESC_DW1_LBN 32 +#define FRF_AZ_TX_DESC_DW1_WIDTH 32 +#define FRF_AZ_TX_DESC_DW2_LBN 64 +#define FRF_AZ_TX_DESC_DW2_WIDTH 31 + + +/* + * FR_AZ_TX_DC_CFG_REG(128bit): + * Transmit descriptor cache configuration register + */ +#define FR_AZ_TX_DC_CFG_REG_OFST 0x00000a20 +/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AZ_TX_DC_SIZE_LBN 0 +#define FRF_AZ_TX_DC_SIZE_WIDTH 2 +#define FFE_AZ_TX_DC_SIZE_32 2 +#define FFE_AZ_TX_DC_SIZE_16 1 +#define FFE_AZ_TX_DC_SIZE_8 0 + + +/* + * FR_AA_TX_CHKSM_CFG_REG(128bit): + * Transmit checksum configuration register + */ +#define FR_AA_TX_CHKSM_CFG_REG_OFST 0x00000a30 +/* falcona0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AA_TX_Q_CHKSM_DIS_96_127_LBN 96 +#define FRF_AA_TX_Q_CHKSM_DIS_96_127_WIDTH 32 +#define FRF_AA_TX_Q_CHKSM_DIS_64_95_LBN 64 +#define FRF_AA_TX_Q_CHKSM_DIS_64_95_WIDTH 32 +#define FRF_AA_TX_Q_CHKSM_DIS_32_63_LBN 32 +#define FRF_AA_TX_Q_CHKSM_DIS_32_63_WIDTH 32 +#define FRF_AA_TX_Q_CHKSM_DIS_0_31_LBN 0 +#define FRF_AA_TX_Q_CHKSM_DIS_0_31_WIDTH 32 + + +/* + * FR_AZ_TX_CFG_REG(128bit): + * Transmit configuration register + */ +#define FR_AZ_TX_CFG_REG_OFST 0x00000a50 +/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_CZ_TX_CONT_LOOKUP_THRESH_RANGE_LBN 114 +#define FRF_CZ_TX_CONT_LOOKUP_THRESH_RANGE_WIDTH 8 +#define FRF_CZ_TX_FILTER_TEST_MODE_BIT_LBN 113 +#define FRF_CZ_TX_FILTER_TEST_MODE_BIT_WIDTH 1 +#define FRF_CZ_TX_ETH_FILTER_WILD_SEARCH_RANGE_LBN 105 +#define FRF_CZ_TX_ETH_FILTER_WILD_SEARCH_RANGE_WIDTH 8 +#define FRF_CZ_TX_ETH_FILTER_FULL_SEARCH_RANGE_LBN 97 +#define FRF_CZ_TX_ETH_FILTER_FULL_SEARCH_RANGE_WIDTH 8 +#define FRF_CZ_TX_UDPIP_FILTER_WILD_SEARCH_RANGE_LBN 89 +#define FRF_CZ_TX_UDPIP_FILTER_WILD_SEARCH_RANGE_WIDTH 8 +#define FRF_CZ_TX_UDPIP_FILTER_FULL_SEARCH_RANGE_LBN 81 +#define FRF_CZ_TX_UDPIP_FILTER_FULL_SEARCH_RANGE_WIDTH 8 +#define FRF_CZ_TX_TCPIP_FILTER_WILD_SEARCH_RANGE_LBN 73 +#define FRF_CZ_TX_TCPIP_FILTER_WILD_SEARCH_RANGE_WIDTH 8 +#define FRF_CZ_TX_TCPIP_FILTER_FULL_SEARCH_RANGE_LBN 65 +#define FRF_CZ_TX_TCPIP_FILTER_FULL_SEARCH_RANGE_WIDTH 8 +#define FRF_CZ_TX_FILTER_ALL_VLAN_ETHERTYPES_BIT_LBN 64 +#define FRF_CZ_TX_FILTER_ALL_VLAN_ETHERTYPES_BIT_WIDTH 1 +#define FRF_CZ_TX_VLAN_MATCH_ETHERTYPE_RANGE_LBN 48 +#define FRF_CZ_TX_VLAN_MATCH_ETHERTYPE_RANGE_WIDTH 16 +#define FRF_CZ_TX_FILTER_EN_BIT_LBN 47 +#define FRF_CZ_TX_FILTER_EN_BIT_WIDTH 1 +#define FRF_AZ_TX_IP_ID_P0_OFS_LBN 16 +#define FRF_AZ_TX_IP_ID_P0_OFS_WIDTH 15 +#define FRF_AZ_TX_NO_EOP_DISC_EN_LBN 5 +#define FRF_AZ_TX_NO_EOP_DISC_EN_WIDTH 1 +#define FRF_AZ_TX_P1_PRI_EN_LBN 4 +#define FRF_AZ_TX_P1_PRI_EN_WIDTH 1 +#define FRF_AZ_TX_OWNERR_CTL_LBN 2 +#define FRF_AZ_TX_OWNERR_CTL_WIDTH 1 +#define FRF_AA_TX_NON_IP_DROP_DIS_LBN 1 +#define FRF_AA_TX_NON_IP_DROP_DIS_WIDTH 1 +#define FRF_AZ_TX_IP_ID_REP_EN_LBN 0 +#define FRF_AZ_TX_IP_ID_REP_EN_WIDTH 1 + + +/* + * FR_AZ_TX_PUSH_DROP_REG(128bit): + * Transmit push dropped register + */ +#define FR_AZ_TX_PUSH_DROP_REG_OFST 0x00000a60 +/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AZ_TX_PUSH_DROP_CNT_LBN 0 +#define FRF_AZ_TX_PUSH_DROP_CNT_WIDTH 32 + + +/* + * FR_AZ_TX_RESERVED_REG(128bit): + * Transmit configuration register + */ +#define FR_AZ_TX_RESERVED_REG_OFST 0x00000a80 +/* falcona0,falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AZ_TX_EVT_CNT_LBN 121 +#define FRF_AZ_TX_EVT_CNT_WIDTH 7 +#define FRF_AZ_TX_PREF_AGE_CNT_LBN 119 +#define FRF_AZ_TX_PREF_AGE_CNT_WIDTH 2 +#define FRF_AZ_TX_RD_COMP_TMR_LBN 96 +#define FRF_AZ_TX_RD_COMP_TMR_WIDTH 23 +#define FRF_AZ_TX_PUSH_EN_LBN 89 +#define FRF_AZ_TX_PUSH_EN_WIDTH 1 +#define FRF_AZ_TX_PUSH_CHK_DIS_LBN 88 +#define FRF_AZ_TX_PUSH_CHK_DIS_WIDTH 1 +#define FRF_AZ_TX_D_FF_FULL_P0_LBN 85 +#define FRF_AZ_TX_D_FF_FULL_P0_WIDTH 1 +#define FRF_AZ_TX_DMAR_ST_P0_LBN 81 +#define FRF_AZ_TX_DMAR_ST_P0_WIDTH 1 +#define FRF_AZ_TX_DMAQ_ST_LBN 78 +#define FRF_AZ_TX_DMAQ_ST_WIDTH 1 +#define FRF_AZ_TX_RX_SPACER_LBN 64 +#define FRF_AZ_TX_RX_SPACER_WIDTH 8 +#define FRF_AZ_TX_DROP_ABORT_EN_LBN 60 +#define FRF_AZ_TX_DROP_ABORT_EN_WIDTH 1 +#define FRF_AZ_TX_SOFT_EVT_EN_LBN 59 +#define FRF_AZ_TX_SOFT_EVT_EN_WIDTH 1 +#define FRF_AZ_TX_PS_EVT_DIS_LBN 58 +#define FRF_AZ_TX_PS_EVT_DIS_WIDTH 1 +#define FRF_AZ_TX_RX_SPACER_EN_LBN 57 +#define FRF_AZ_TX_RX_SPACER_EN_WIDTH 1 +#define FRF_AZ_TX_XP_TIMER_LBN 52 +#define FRF_AZ_TX_XP_TIMER_WIDTH 5 +#define FRF_AZ_TX_PREF_SPACER_LBN 44 +#define FRF_AZ_TX_PREF_SPACER_WIDTH 8 +#define FRF_AZ_TX_PREF_WD_TMR_LBN 22 +#define FRF_AZ_TX_PREF_WD_TMR_WIDTH 22 +#define FRF_AZ_TX_ONLY1TAG_LBN 21 +#define FRF_AZ_TX_ONLY1TAG_WIDTH 1 +#define FRF_AZ_TX_PREF_THRESHOLD_LBN 19 +#define FRF_AZ_TX_PREF_THRESHOLD_WIDTH 2 +#define FRF_AZ_TX_ONE_PKT_PER_Q_LBN 18 +#define FRF_AZ_TX_ONE_PKT_PER_Q_WIDTH 1 +#define FRF_AZ_TX_DIS_NON_IP_EV_LBN 17 +#define FRF_AZ_TX_DIS_NON_IP_EV_WIDTH 1 +#define FRF_AA_TX_DMA_FF_THR_LBN 16 +#define FRF_AA_TX_DMA_FF_THR_WIDTH 1 +#define FRF_AZ_TX_DMA_SPACER_LBN 8 +#define FRF_AZ_TX_DMA_SPACER_WIDTH 8 +#define FRF_AA_TX_TCP_DIS_LBN 7 +#define FRF_AA_TX_TCP_DIS_WIDTH 1 +#define FRF_BZ_TX_FLUSH_MIN_LEN_EN_LBN 7 +#define FRF_BZ_TX_FLUSH_MIN_LEN_EN_WIDTH 1 +#define FRF_AA_TX_IP_DIS_LBN 6 +#define FRF_AA_TX_IP_DIS_WIDTH 1 +#define FRF_AZ_TX_MAX_CPL_LBN 2 +#define FRF_AZ_TX_MAX_CPL_WIDTH 2 +#define FFE_AZ_TX_MAX_CPL_16 3 +#define FFE_AZ_TX_MAX_CPL_8 2 +#define FFE_AZ_TX_MAX_CPL_4 1 +#define FFE_AZ_TX_MAX_CPL_NOLIMIT 0 +#define FRF_AZ_TX_MAX_PREF_LBN 0 +#define FRF_AZ_TX_MAX_PREF_WIDTH 2 +#define FFE_AZ_TX_MAX_PREF_32 3 +#define FFE_AZ_TX_MAX_PREF_16 2 +#define FFE_AZ_TX_MAX_PREF_8 1 +#define FFE_AZ_TX_MAX_PREF_OFF 0 + + +/* + * FR_BZ_TX_PACE_REG(128bit): + * Transmit pace control register + */ +#define FR_BZ_TX_PACE_REG_OFST 0x00000a90 +/* falconb0,sienaa0=net_func_bar2 */ +/* + * FR_AA_TX_PACE_REG(128bit): + * Transmit pace control register + */ +#define FR_AA_TX_PACE_REG_OFST 0x00f80000 +/* falcona0=char_func_bar0 */ + +#define FRF_AZ_TX_PACE_SB_NOT_AF_LBN 19 +#define FRF_AZ_TX_PACE_SB_NOT_AF_WIDTH 10 +#define FRF_AZ_TX_PACE_SB_AF_LBN 9 +#define FRF_AZ_TX_PACE_SB_AF_WIDTH 10 +#define FRF_AZ_TX_PACE_FB_BASE_LBN 5 +#define FRF_AZ_TX_PACE_FB_BASE_WIDTH 4 +#define FRF_AZ_TX_PACE_BIN_TH_LBN 0 +#define FRF_AZ_TX_PACE_BIN_TH_WIDTH 5 + + +/* + * FR_AZ_TX_PACE_DROP_QID_REG(128bit): + * PACE Drop QID Counter + */ +#define FR_AZ_TX_PACE_DROP_QID_REG_OFST 0x00000aa0 +/* falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AZ_TX_PACE_QID_DRP_CNT_LBN 0 +#define FRF_AZ_TX_PACE_QID_DRP_CNT_WIDTH 16 + + +/* + * FR_AB_TX_VLAN_REG(128bit): + * Transmit VLAN tag register + */ +#define FR_AB_TX_VLAN_REG_OFST 0x00000ae0 +/* falconb0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_TX_VLAN_EN_LBN 127 +#define FRF_AB_TX_VLAN_EN_WIDTH 1 +#define FRF_AB_TX_VLAN7_PORT1_EN_LBN 125 +#define FRF_AB_TX_VLAN7_PORT1_EN_WIDTH 1 +#define FRF_AB_TX_VLAN7_PORT0_EN_LBN 124 +#define FRF_AB_TX_VLAN7_PORT0_EN_WIDTH 1 +#define FRF_AB_TX_VLAN7_LBN 112 +#define FRF_AB_TX_VLAN7_WIDTH 12 +#define FRF_AB_TX_VLAN6_PORT1_EN_LBN 109 +#define FRF_AB_TX_VLAN6_PORT1_EN_WIDTH 1 +#define FRF_AB_TX_VLAN6_PORT0_EN_LBN 108 +#define FRF_AB_TX_VLAN6_PORT0_EN_WIDTH 1 +#define FRF_AB_TX_VLAN6_LBN 96 +#define FRF_AB_TX_VLAN6_WIDTH 12 +#define FRF_AB_TX_VLAN5_PORT1_EN_LBN 93 +#define FRF_AB_TX_VLAN5_PORT1_EN_WIDTH 1 +#define FRF_AB_TX_VLAN5_PORT0_EN_LBN 92 +#define FRF_AB_TX_VLAN5_PORT0_EN_WIDTH 1 +#define FRF_AB_TX_VLAN5_LBN 80 +#define FRF_AB_TX_VLAN5_WIDTH 12 +#define FRF_AB_TX_VLAN4_PORT1_EN_LBN 77 +#define FRF_AB_TX_VLAN4_PORT1_EN_WIDTH 1 +#define FRF_AB_TX_VLAN4_PORT0_EN_LBN 76 +#define FRF_AB_TX_VLAN4_PORT0_EN_WIDTH 1 +#define FRF_AB_TX_VLAN4_LBN 64 +#define FRF_AB_TX_VLAN4_WIDTH 12 +#define FRF_AB_TX_VLAN3_PORT1_EN_LBN 61 +#define FRF_AB_TX_VLAN3_PORT1_EN_WIDTH 1 +#define FRF_AB_TX_VLAN3_PORT0_EN_LBN 60 +#define FRF_AB_TX_VLAN3_PORT0_EN_WIDTH 1 +#define FRF_AB_TX_VLAN3_LBN 48 +#define FRF_AB_TX_VLAN3_WIDTH 12 +#define FRF_AB_TX_VLAN2_PORT1_EN_LBN 45 +#define FRF_AB_TX_VLAN2_PORT1_EN_WIDTH 1 +#define FRF_AB_TX_VLAN2_PORT0_EN_LBN 44 +#define FRF_AB_TX_VLAN2_PORT0_EN_WIDTH 1 +#define FRF_AB_TX_VLAN2_LBN 32 +#define FRF_AB_TX_VLAN2_WIDTH 12 +#define FRF_AB_TX_VLAN1_PORT1_EN_LBN 29 +#define FRF_AB_TX_VLAN1_PORT1_EN_WIDTH 1 +#define FRF_AB_TX_VLAN1_PORT0_EN_LBN 28 +#define FRF_AB_TX_VLAN1_PORT0_EN_WIDTH 1 +#define FRF_AB_TX_VLAN1_LBN 16 +#define FRF_AB_TX_VLAN1_WIDTH 12 +#define FRF_AB_TX_VLAN0_PORT1_EN_LBN 13 +#define FRF_AB_TX_VLAN0_PORT1_EN_WIDTH 1 +#define FRF_AB_TX_VLAN0_PORT0_EN_LBN 12 +#define FRF_AB_TX_VLAN0_PORT0_EN_WIDTH 1 +#define FRF_AB_TX_VLAN0_LBN 0 +#define FRF_AB_TX_VLAN0_WIDTH 12 + + +/* + * FR_AZ_TX_IPFIL_PORTEN_REG(128bit): + * Transmit filter control register + */ +#define FR_AZ_TX_IPFIL_PORTEN_REG_OFST 0x00000af0 +/* falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AZ_TX_MADR0_FIL_EN_LBN 64 +#define FRF_AZ_TX_MADR0_FIL_EN_WIDTH 1 +#define FRF_AB_TX_IPFIL31_PORT_EN_LBN 62 +#define FRF_AB_TX_IPFIL31_PORT_EN_WIDTH 1 +#define FRF_AB_TX_IPFIL30_PORT_EN_LBN 60 +#define FRF_AB_TX_IPFIL30_PORT_EN_WIDTH 1 +#define FRF_AB_TX_IPFIL29_PORT_EN_LBN 58 +#define FRF_AB_TX_IPFIL29_PORT_EN_WIDTH 1 +#define FRF_AB_TX_IPFIL28_PORT_EN_LBN 56 +#define FRF_AB_TX_IPFIL28_PORT_EN_WIDTH 1 +#define FRF_AB_TX_IPFIL27_PORT_EN_LBN 54 +#define FRF_AB_TX_IPFIL27_PORT_EN_WIDTH 1 +#define FRF_AB_TX_IPFIL26_PORT_EN_LBN 52 +#define FRF_AB_TX_IPFIL26_PORT_EN_WIDTH 1 +#define FRF_AB_TX_IPFIL25_PORT_EN_LBN 50 +#define FRF_AB_TX_IPFIL25_PORT_EN_WIDTH 1 +#define FRF_AB_TX_IPFIL24_PORT_EN_LBN 48 +#define FRF_AB_TX_IPFIL24_PORT_EN_WIDTH 1 +#define FRF_AB_TX_IPFIL23_PORT_EN_LBN 46 +#define FRF_AB_TX_IPFIL23_PORT_EN_WIDTH 1 +#define FRF_AB_TX_IPFIL22_PORT_EN_LBN 44 +#define FRF_AB_TX_IPFIL22_PORT_EN_WIDTH 1 +#define FRF_AB_TX_IPFIL21_PORT_EN_LBN 42 +#define FRF_AB_TX_IPFIL21_PORT_EN_WIDTH 1 +#define FRF_AB_TX_IPFIL20_PORT_EN_LBN 40 +#define FRF_AB_TX_IPFIL20_PORT_EN_WIDTH 1 +#define FRF_AB_TX_IPFIL19_PORT_EN_LBN 38 +#define FRF_AB_TX_IPFIL19_PORT_EN_WIDTH 1 +#define FRF_AB_TX_IPFIL18_PORT_EN_LBN 36 +#define FRF_AB_TX_IPFIL18_PORT_EN_WIDTH 1 +#define FRF_AB_TX_IPFIL17_PORT_EN_LBN 34 +#define FRF_AB_TX_IPFIL17_PORT_EN_WIDTH 1 +#define FRF_AB_TX_IPFIL16_PORT_EN_LBN 32 +#define FRF_AB_TX_IPFIL16_PORT_EN_WIDTH 1 +#define FRF_AB_TX_IPFIL15_PORT_EN_LBN 30 +#define FRF_AB_TX_IPFIL15_PORT_EN_WIDTH 1 +#define FRF_AB_TX_IPFIL14_PORT_EN_LBN 28 +#define FRF_AB_TX_IPFIL14_PORT_EN_WIDTH 1 +#define FRF_AB_TX_IPFIL13_PORT_EN_LBN 26 +#define FRF_AB_TX_IPFIL13_PORT_EN_WIDTH 1 +#define FRF_AB_TX_IPFIL12_PORT_EN_LBN 24 +#define FRF_AB_TX_IPFIL12_PORT_EN_WIDTH 1 +#define FRF_AB_TX_IPFIL11_PORT_EN_LBN 22 +#define FRF_AB_TX_IPFIL11_PORT_EN_WIDTH 1 +#define FRF_AB_TX_IPFIL10_PORT_EN_LBN 20 +#define FRF_AB_TX_IPFIL10_PORT_EN_WIDTH 1 +#define FRF_AB_TX_IPFIL9_PORT_EN_LBN 18 +#define FRF_AB_TX_IPFIL9_PORT_EN_WIDTH 1 +#define FRF_AB_TX_IPFIL8_PORT_EN_LBN 16 +#define FRF_AB_TX_IPFIL8_PORT_EN_WIDTH 1 +#define FRF_AB_TX_IPFIL7_PORT_EN_LBN 14 +#define FRF_AB_TX_IPFIL7_PORT_EN_WIDTH 1 +#define FRF_AB_TX_IPFIL6_PORT_EN_LBN 12 +#define FRF_AB_TX_IPFIL6_PORT_EN_WIDTH 1 +#define FRF_AB_TX_IPFIL5_PORT_EN_LBN 10 +#define FRF_AB_TX_IPFIL5_PORT_EN_WIDTH 1 +#define FRF_AB_TX_IPFIL4_PORT_EN_LBN 8 +#define FRF_AB_TX_IPFIL4_PORT_EN_WIDTH 1 +#define FRF_AB_TX_IPFIL3_PORT_EN_LBN 6 +#define FRF_AB_TX_IPFIL3_PORT_EN_WIDTH 1 +#define FRF_AB_TX_IPFIL2_PORT_EN_LBN 4 +#define FRF_AB_TX_IPFIL2_PORT_EN_WIDTH 1 +#define FRF_AB_TX_IPFIL1_PORT_EN_LBN 2 +#define FRF_AB_TX_IPFIL1_PORT_EN_WIDTH 1 +#define FRF_AB_TX_IPFIL0_PORT_EN_LBN 0 +#define FRF_AB_TX_IPFIL0_PORT_EN_WIDTH 1 + + +/* + * FR_AB_TX_IPFIL_TBL(128bit): + * Transmit IP source address filter table + */ +#define FR_AB_TX_IPFIL_TBL_OFST 0x00000b00 +/* falconb0=net_func_bar2,falcona0=char_func_bar0 */ +#define FR_AB_TX_IPFIL_TBL_STEP 16 +#define FR_AB_TX_IPFIL_TBL_ROWS 16 + +#define FRF_AB_TX_IPFIL_MASK_1_LBN 96 +#define FRF_AB_TX_IPFIL_MASK_1_WIDTH 32 +#define FRF_AB_TX_IP_SRC_ADR_1_LBN 64 +#define FRF_AB_TX_IP_SRC_ADR_1_WIDTH 32 +#define FRF_AB_TX_IPFIL_MASK_0_LBN 32 +#define FRF_AB_TX_IPFIL_MASK_0_WIDTH 32 +#define FRF_AB_TX_IP_SRC_ADR_0_LBN 0 +#define FRF_AB_TX_IP_SRC_ADR_0_WIDTH 32 + + +/* + * FR_AB_MD_TXD_REG(128bit): + * PHY management transmit data register + */ +#define FR_AB_MD_TXD_REG_OFST 0x00000c00 +/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_MD_TXD_LBN 0 +#define FRF_AB_MD_TXD_WIDTH 16 + + +/* + * FR_AB_MD_RXD_REG(128bit): + * PHY management receive data register + */ +#define FR_AB_MD_RXD_REG_OFST 0x00000c10 +/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_MD_RXD_LBN 0 +#define FRF_AB_MD_RXD_WIDTH 16 + + +/* + * FR_AB_MD_CS_REG(128bit): + * PHY management configuration & status register + */ +#define FR_AB_MD_CS_REG_OFST 0x00000c20 +/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_MD_RD_EN_LBN 15 +#define FRF_AB_MD_RD_EN_WIDTH 1 +#define FRF_AB_MD_WR_EN_LBN 14 +#define FRF_AB_MD_WR_EN_WIDTH 1 +#define FRF_AB_MD_ADDR_CMD_LBN 13 +#define FRF_AB_MD_ADDR_CMD_WIDTH 1 +#define FRF_AB_MD_PT_LBN 7 +#define FRF_AB_MD_PT_WIDTH 3 +#define FRF_AB_MD_PL_LBN 6 +#define FRF_AB_MD_PL_WIDTH 1 +#define FRF_AB_MD_INT_CLR_LBN 5 +#define FRF_AB_MD_INT_CLR_WIDTH 1 +#define FRF_AB_MD_GC_LBN 4 +#define FRF_AB_MD_GC_WIDTH 1 +#define FRF_AB_MD_PRSP_LBN 3 +#define FRF_AB_MD_PRSP_WIDTH 1 +#define FRF_AB_MD_RIC_LBN 2 +#define FRF_AB_MD_RIC_WIDTH 1 +#define FRF_AB_MD_RDC_LBN 1 +#define FRF_AB_MD_RDC_WIDTH 1 +#define FRF_AB_MD_WRC_LBN 0 +#define FRF_AB_MD_WRC_WIDTH 1 + + +/* + * FR_AB_MD_PHY_ADR_REG(128bit): + * PHY management PHY address register + */ +#define FR_AB_MD_PHY_ADR_REG_OFST 0x00000c30 +/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_MD_PHY_ADR_LBN 0 +#define FRF_AB_MD_PHY_ADR_WIDTH 16 + + +/* + * FR_AB_MD_ID_REG(128bit): + * PHY management ID register + */ +#define FR_AB_MD_ID_REG_OFST 0x00000c40 +/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_MD_PRT_ADR_LBN 11 +#define FRF_AB_MD_PRT_ADR_WIDTH 5 +#define FRF_AB_MD_DEV_ADR_LBN 6 +#define FRF_AB_MD_DEV_ADR_WIDTH 5 + + +/* + * FR_AB_MD_STAT_REG(128bit): + * PHY management status & mask register + */ +#define FR_AB_MD_STAT_REG_OFST 0x00000c50 +/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_MD_PINT_LBN 4 +#define FRF_AB_MD_PINT_WIDTH 1 +#define FRF_AB_MD_DONE_LBN 3 +#define FRF_AB_MD_DONE_WIDTH 1 +#define FRF_AB_MD_BSERR_LBN 2 +#define FRF_AB_MD_BSERR_WIDTH 1 +#define FRF_AB_MD_LNFL_LBN 1 +#define FRF_AB_MD_LNFL_WIDTH 1 +#define FRF_AB_MD_BSY_LBN 0 +#define FRF_AB_MD_BSY_WIDTH 1 + + +/* + * FR_AB_MAC_STAT_DMA_REG(128bit): + * Port MAC statistical counter DMA register + */ +#define FR_AB_MAC_STAT_DMA_REG_OFST 0x00000c60 +/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_MAC_STAT_DMA_CMD_LBN 48 +#define FRF_AB_MAC_STAT_DMA_CMD_WIDTH 1 +#define FRF_AB_MAC_STAT_DMA_ADR_LBN 0 +#define FRF_AB_MAC_STAT_DMA_ADR_WIDTH 48 +#define FRF_AB_MAC_STAT_DMA_ADR_DW0_LBN 0 +#define FRF_AB_MAC_STAT_DMA_ADR_DW0_WIDTH 32 +#define FRF_AB_MAC_STAT_DMA_ADR_DW1_LBN 32 +#define FRF_AB_MAC_STAT_DMA_ADR_DW1_WIDTH 16 + + +/* + * FR_AB_MAC_CTRL_REG(128bit): + * Port MAC control register + */ +#define FR_AB_MAC_CTRL_REG_OFST 0x00000c80 +/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_MAC_XOFF_VAL_LBN 16 +#define FRF_AB_MAC_XOFF_VAL_WIDTH 16 +#define FRF_BB_TXFIFO_DRAIN_EN_LBN 7 +#define FRF_BB_TXFIFO_DRAIN_EN_WIDTH 1 +#define FRF_AB_MAC_XG_DISTXCRC_LBN 5 +#define FRF_AB_MAC_XG_DISTXCRC_WIDTH 1 +#define FRF_AB_MAC_BCAD_ACPT_LBN 4 +#define FRF_AB_MAC_BCAD_ACPT_WIDTH 1 +#define FRF_AB_MAC_UC_PROM_LBN 3 +#define FRF_AB_MAC_UC_PROM_WIDTH 1 +#define FRF_AB_MAC_LINK_STATUS_LBN 2 +#define FRF_AB_MAC_LINK_STATUS_WIDTH 1 +#define FRF_AB_MAC_SPEED_LBN 0 +#define FRF_AB_MAC_SPEED_WIDTH 2 +#define FRF_AB_MAC_SPEED_10M 0 +#define FRF_AB_MAC_SPEED_100M 1 +#define FRF_AB_MAC_SPEED_1G 2 +#define FRF_AB_MAC_SPEED_10G 3 + +/* + * FR_BB_GEN_MODE_REG(128bit): + * General Purpose mode register (external interrupt mask) + */ +#define FR_BB_GEN_MODE_REG_OFST 0x00000c90 +/* falconb0=net_func_bar2 */ + +#define FRF_BB_XFP_PHY_INT_POL_SEL_LBN 3 +#define FRF_BB_XFP_PHY_INT_POL_SEL_WIDTH 1 +#define FRF_BB_XG_PHY_INT_POL_SEL_LBN 2 +#define FRF_BB_XG_PHY_INT_POL_SEL_WIDTH 1 +#define FRF_BB_XFP_PHY_INT_MASK_LBN 1 +#define FRF_BB_XFP_PHY_INT_MASK_WIDTH 1 +#define FRF_BB_XG_PHY_INT_MASK_LBN 0 +#define FRF_BB_XG_PHY_INT_MASK_WIDTH 1 + + +/* + * FR_AB_MAC_MC_HASH_REG0(128bit): + * Multicast address hash table + */ +#define FR_AB_MAC_MC_HASH0_REG_OFST 0x00000ca0 +/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_MAC_MCAST_HASH0_LBN 0 +#define FRF_AB_MAC_MCAST_HASH0_WIDTH 128 +#define FRF_AB_MAC_MCAST_HASH0_DW0_LBN 0 +#define FRF_AB_MAC_MCAST_HASH0_DW0_WIDTH 32 +#define FRF_AB_MAC_MCAST_HASH0_DW1_LBN 32 +#define FRF_AB_MAC_MCAST_HASH0_DW1_WIDTH 32 +#define FRF_AB_MAC_MCAST_HASH0_DW2_LBN 64 +#define FRF_AB_MAC_MCAST_HASH0_DW2_WIDTH 32 +#define FRF_AB_MAC_MCAST_HASH0_DW3_LBN 96 +#define FRF_AB_MAC_MCAST_HASH0_DW3_WIDTH 32 + + +/* + * FR_AB_MAC_MC_HASH_REG1(128bit): + * Multicast address hash table + */ +#define FR_AB_MAC_MC_HASH1_REG_OFST 0x00000cb0 +/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_MAC_MCAST_HASH1_LBN 0 +#define FRF_AB_MAC_MCAST_HASH1_WIDTH 128 +#define FRF_AB_MAC_MCAST_HASH1_DW0_LBN 0 +#define FRF_AB_MAC_MCAST_HASH1_DW0_WIDTH 32 +#define FRF_AB_MAC_MCAST_HASH1_DW1_LBN 32 +#define FRF_AB_MAC_MCAST_HASH1_DW1_WIDTH 32 +#define FRF_AB_MAC_MCAST_HASH1_DW2_LBN 64 +#define FRF_AB_MAC_MCAST_HASH1_DW2_WIDTH 32 +#define FRF_AB_MAC_MCAST_HASH1_DW3_LBN 96 +#define FRF_AB_MAC_MCAST_HASH1_DW3_WIDTH 32 + + +/* + * FR_AB_GM_CFG1_REG(32bit): + * GMAC configuration register 1 + */ +#define FR_AB_GM_CFG1_REG_OFST 0x00000e00 +/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_GM_SW_RST_LBN 31 +#define FRF_AB_GM_SW_RST_WIDTH 1 +#define FRF_AB_GM_SIM_RST_LBN 30 +#define FRF_AB_GM_SIM_RST_WIDTH 1 +#define FRF_AB_GM_RST_RX_MAC_CTL_LBN 19 +#define FRF_AB_GM_RST_RX_MAC_CTL_WIDTH 1 +#define FRF_AB_GM_RST_TX_MAC_CTL_LBN 18 +#define FRF_AB_GM_RST_TX_MAC_CTL_WIDTH 1 +#define FRF_AB_GM_RST_RX_FUNC_LBN 17 +#define FRF_AB_GM_RST_RX_FUNC_WIDTH 1 +#define FRF_AB_GM_RST_TX_FUNC_LBN 16 +#define FRF_AB_GM_RST_TX_FUNC_WIDTH 1 +#define FRF_AB_GM_LOOP_LBN 8 +#define FRF_AB_GM_LOOP_WIDTH 1 +#define FRF_AB_GM_RX_FC_EN_LBN 5 +#define FRF_AB_GM_RX_FC_EN_WIDTH 1 +#define FRF_AB_GM_TX_FC_EN_LBN 4 +#define FRF_AB_GM_TX_FC_EN_WIDTH 1 +#define FRF_AB_GM_SYNC_RXEN_LBN 3 +#define FRF_AB_GM_SYNC_RXEN_WIDTH 1 +#define FRF_AB_GM_RX_EN_LBN 2 +#define FRF_AB_GM_RX_EN_WIDTH 1 +#define FRF_AB_GM_SYNC_TXEN_LBN 1 +#define FRF_AB_GM_SYNC_TXEN_WIDTH 1 +#define FRF_AB_GM_TX_EN_LBN 0 +#define FRF_AB_GM_TX_EN_WIDTH 1 + + +/* + * FR_AB_GM_CFG2_REG(32bit): + * GMAC configuration register 2 + */ +#define FR_AB_GM_CFG2_REG_OFST 0x00000e10 +/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_GM_PAMBL_LEN_LBN 12 +#define FRF_AB_GM_PAMBL_LEN_WIDTH 4 +#define FRF_AB_GM_IF_MODE_LBN 8 +#define FRF_AB_GM_IF_MODE_WIDTH 2 +#define FRF_AB_GM_IF_MODE_BYTE_MODE 2 +#define FRF_AB_GM_IF_MODE_NIBBLE_MODE 1 +#define FRF_AB_GM_HUGE_FRM_EN_LBN 5 +#define FRF_AB_GM_HUGE_FRM_EN_WIDTH 1 +#define FRF_AB_GM_LEN_CHK_LBN 4 +#define FRF_AB_GM_LEN_CHK_WIDTH 1 +#define FRF_AB_GM_PAD_CRC_EN_LBN 2 +#define FRF_AB_GM_PAD_CRC_EN_WIDTH 1 +#define FRF_AB_GM_CRC_EN_LBN 1 +#define FRF_AB_GM_CRC_EN_WIDTH 1 +#define FRF_AB_GM_FD_LBN 0 +#define FRF_AB_GM_FD_WIDTH 1 + + +/* + * FR_AB_GM_IPG_REG(32bit): + * GMAC IPG register + */ +#define FR_AB_GM_IPG_REG_OFST 0x00000e20 +/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_GM_NONB2B_IPG1_LBN 24 +#define FRF_AB_GM_NONB2B_IPG1_WIDTH 7 +#define FRF_AB_GM_NONB2B_IPG2_LBN 16 +#define FRF_AB_GM_NONB2B_IPG2_WIDTH 7 +#define FRF_AB_GM_MIN_IPG_ENF_LBN 8 +#define FRF_AB_GM_MIN_IPG_ENF_WIDTH 8 +#define FRF_AB_GM_B2B_IPG_LBN 0 +#define FRF_AB_GM_B2B_IPG_WIDTH 7 + + +/* + * FR_AB_GM_HD_REG(32bit): + * GMAC half duplex register + */ +#define FR_AB_GM_HD_REG_OFST 0x00000e30 +/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_GM_ALT_BOFF_VAL_LBN 20 +#define FRF_AB_GM_ALT_BOFF_VAL_WIDTH 4 +#define FRF_AB_GM_ALT_BOFF_EN_LBN 19 +#define FRF_AB_GM_ALT_BOFF_EN_WIDTH 1 +#define FRF_AB_GM_BP_NO_BOFF_LBN 18 +#define FRF_AB_GM_BP_NO_BOFF_WIDTH 1 +#define FRF_AB_GM_DIS_BOFF_LBN 17 +#define FRF_AB_GM_DIS_BOFF_WIDTH 1 +#define FRF_AB_GM_EXDEF_TX_EN_LBN 16 +#define FRF_AB_GM_EXDEF_TX_EN_WIDTH 1 +#define FRF_AB_GM_RTRY_LIMIT_LBN 12 +#define FRF_AB_GM_RTRY_LIMIT_WIDTH 4 +#define FRF_AB_GM_COL_WIN_LBN 0 +#define FRF_AB_GM_COL_WIN_WIDTH 10 + + +/* + * FR_AB_GM_MAX_FLEN_REG(32bit): + * GMAC maximum frame length register + */ +#define FR_AB_GM_MAX_FLEN_REG_OFST 0x00000e40 +/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_GM_MAX_FLEN_LBN 0 +#define FRF_AB_GM_MAX_FLEN_WIDTH 16 + + +/* + * FR_AB_GM_TEST_REG(32bit): + * GMAC test register + */ +#define FR_AB_GM_TEST_REG_OFST 0x00000e70 +/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_GM_MAX_BOFF_LBN 3 +#define FRF_AB_GM_MAX_BOFF_WIDTH 1 +#define FRF_AB_GM_REG_TX_FLOW_EN_LBN 2 +#define FRF_AB_GM_REG_TX_FLOW_EN_WIDTH 1 +#define FRF_AB_GM_TEST_PAUSE_LBN 1 +#define FRF_AB_GM_TEST_PAUSE_WIDTH 1 +#define FRF_AB_GM_SHORT_SLOT_LBN 0 +#define FRF_AB_GM_SHORT_SLOT_WIDTH 1 + + +/* + * FR_AB_GM_ADR1_REG(32bit): + * GMAC station address register 1 + */ +#define FR_AB_GM_ADR1_REG_OFST 0x00000f00 +/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_GM_ADR_B0_LBN 24 +#define FRF_AB_GM_ADR_B0_WIDTH 8 +#define FRF_AB_GM_ADR_B1_LBN 16 +#define FRF_AB_GM_ADR_B1_WIDTH 8 +#define FRF_AB_GM_ADR_B2_LBN 8 +#define FRF_AB_GM_ADR_B2_WIDTH 8 +#define FRF_AB_GM_ADR_B3_LBN 0 +#define FRF_AB_GM_ADR_B3_WIDTH 8 + + +/* + * FR_AB_GM_ADR2_REG(32bit): + * GMAC station address register 2 + */ +#define FR_AB_GM_ADR2_REG_OFST 0x00000f10 +/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_GM_ADR_B4_LBN 24 +#define FRF_AB_GM_ADR_B4_WIDTH 8 +#define FRF_AB_GM_ADR_B5_LBN 16 +#define FRF_AB_GM_ADR_B5_WIDTH 8 + + +/* + * FR_AB_GMF_CFG0_REG(32bit): + * GMAC FIFO configuration register 0 + */ +#define FR_AB_GMF_CFG0_REG_OFST 0x00000f20 +/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_GMF_FTFENRPLY_LBN 20 +#define FRF_AB_GMF_FTFENRPLY_WIDTH 1 +#define FRF_AB_GMF_STFENRPLY_LBN 19 +#define FRF_AB_GMF_STFENRPLY_WIDTH 1 +#define FRF_AB_GMF_FRFENRPLY_LBN 18 +#define FRF_AB_GMF_FRFENRPLY_WIDTH 1 +#define FRF_AB_GMF_SRFENRPLY_LBN 17 +#define FRF_AB_GMF_SRFENRPLY_WIDTH 1 +#define FRF_AB_GMF_WTMENRPLY_LBN 16 +#define FRF_AB_GMF_WTMENRPLY_WIDTH 1 +#define FRF_AB_GMF_FTFENREQ_LBN 12 +#define FRF_AB_GMF_FTFENREQ_WIDTH 1 +#define FRF_AB_GMF_STFENREQ_LBN 11 +#define FRF_AB_GMF_STFENREQ_WIDTH 1 +#define FRF_AB_GMF_FRFENREQ_LBN 10 +#define FRF_AB_GMF_FRFENREQ_WIDTH 1 +#define FRF_AB_GMF_SRFENREQ_LBN 9 +#define FRF_AB_GMF_SRFENREQ_WIDTH 1 +#define FRF_AB_GMF_WTMENREQ_LBN 8 +#define FRF_AB_GMF_WTMENREQ_WIDTH 1 +#define FRF_AB_GMF_HSTRSTFT_LBN 4 +#define FRF_AB_GMF_HSTRSTFT_WIDTH 1 +#define FRF_AB_GMF_HSTRSTST_LBN 3 +#define FRF_AB_GMF_HSTRSTST_WIDTH 1 +#define FRF_AB_GMF_HSTRSTFR_LBN 2 +#define FRF_AB_GMF_HSTRSTFR_WIDTH 1 +#define FRF_AB_GMF_HSTRSTSR_LBN 1 +#define FRF_AB_GMF_HSTRSTSR_WIDTH 1 +#define FRF_AB_GMF_HSTRSTWT_LBN 0 +#define FRF_AB_GMF_HSTRSTWT_WIDTH 1 + + +/* + * FR_AB_GMF_CFG1_REG(32bit): + * GMAC FIFO configuration register 1 + */ +#define FR_AB_GMF_CFG1_REG_OFST 0x00000f30 +/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_GMF_CFGFRTH_LBN 16 +#define FRF_AB_GMF_CFGFRTH_WIDTH 5 +#define FRF_AB_GMF_CFGXOFFRTX_LBN 0 +#define FRF_AB_GMF_CFGXOFFRTX_WIDTH 16 + + +/* + * FR_AB_GMF_CFG2_REG(32bit): + * GMAC FIFO configuration register 2 + */ +#define FR_AB_GMF_CFG2_REG_OFST 0x00000f40 +/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_GMF_CFGHWM_LBN 16 +#define FRF_AB_GMF_CFGHWM_WIDTH 6 +#define FRF_AB_GMF_CFGLWM_LBN 0 +#define FRF_AB_GMF_CFGLWM_WIDTH 6 + + +/* + * FR_AB_GMF_CFG3_REG(32bit): + * GMAC FIFO configuration register 3 + */ +#define FR_AB_GMF_CFG3_REG_OFST 0x00000f50 +/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_GMF_CFGHWMFT_LBN 16 +#define FRF_AB_GMF_CFGHWMFT_WIDTH 6 +#define FRF_AB_GMF_CFGFTTH_LBN 0 +#define FRF_AB_GMF_CFGFTTH_WIDTH 6 + + +/* + * FR_AB_GMF_CFG4_REG(32bit): + * GMAC FIFO configuration register 4 + */ +#define FR_AB_GMF_CFG4_REG_OFST 0x00000f60 +/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_GMF_HSTFLTRFRM_LBN 0 +#define FRF_AB_GMF_HSTFLTRFRM_WIDTH 18 + + +/* + * FR_AB_GMF_CFG5_REG(32bit): + * GMAC FIFO configuration register 5 + */ +#define FR_AB_GMF_CFG5_REG_OFST 0x00000f70 +/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_GMF_CFGHDPLX_LBN 22 +#define FRF_AB_GMF_CFGHDPLX_WIDTH 1 +#define FRF_AB_GMF_SRFULL_LBN 21 +#define FRF_AB_GMF_SRFULL_WIDTH 1 +#define FRF_AB_GMF_HSTSRFULLCLR_LBN 20 +#define FRF_AB_GMF_HSTSRFULLCLR_WIDTH 1 +#define FRF_AB_GMF_CFGBYTMODE_LBN 19 +#define FRF_AB_GMF_CFGBYTMODE_WIDTH 1 +#define FRF_AB_GMF_HSTDRPLT64_LBN 18 +#define FRF_AB_GMF_HSTDRPLT64_WIDTH 1 +#define FRF_AB_GMF_HSTFLTRFRMDC_LBN 0 +#define FRF_AB_GMF_HSTFLTRFRMDC_WIDTH 18 + + +/* + * FR_BB_TX_SRC_MAC_TBL(128bit): + * Transmit IP source address filter table + */ +#define FR_BB_TX_SRC_MAC_TBL_OFST 0x00001000 +/* falconb0=net_func_bar2 */ +#define FR_BB_TX_SRC_MAC_TBL_STEP 16 +#define FR_BB_TX_SRC_MAC_TBL_ROWS 16 + +#define FRF_BB_TX_SRC_MAC_ADR_1_LBN 64 +#define FRF_BB_TX_SRC_MAC_ADR_1_WIDTH 48 +#define FRF_BB_TX_SRC_MAC_ADR_1_DW0_LBN 64 +#define FRF_BB_TX_SRC_MAC_ADR_1_DW0_WIDTH 32 +#define FRF_BB_TX_SRC_MAC_ADR_1_DW1_LBN 96 +#define FRF_BB_TX_SRC_MAC_ADR_1_DW1_WIDTH 16 +#define FRF_BB_TX_SRC_MAC_ADR_0_LBN 0 +#define FRF_BB_TX_SRC_MAC_ADR_0_WIDTH 48 +#define FRF_BB_TX_SRC_MAC_ADR_0_DW0_LBN 0 +#define FRF_BB_TX_SRC_MAC_ADR_0_DW0_WIDTH 32 +#define FRF_BB_TX_SRC_MAC_ADR_0_DW1_LBN 32 +#define FRF_BB_TX_SRC_MAC_ADR_0_DW1_WIDTH 16 + + +/* + * FR_BB_TX_SRC_MAC_CTL_REG(128bit): + * Transmit MAC source address filter control + */ +#define FR_BB_TX_SRC_MAC_CTL_REG_OFST 0x00001100 +/* falconb0=net_func_bar2 */ + +#define FRF_BB_TX_SRC_DROP_CTR_LBN 16 +#define FRF_BB_TX_SRC_DROP_CTR_WIDTH 16 +#define FRF_BB_TX_SRC_FLTR_EN_LBN 15 +#define FRF_BB_TX_SRC_FLTR_EN_WIDTH 1 +#define FRF_BB_TX_DROP_CTR_CLR_LBN 12 +#define FRF_BB_TX_DROP_CTR_CLR_WIDTH 1 +#define FRF_BB_TX_MAC_QID_SEL_LBN 0 +#define FRF_BB_TX_MAC_QID_SEL_WIDTH 3 + + +/* + * FR_AB_XM_ADR_LO_REG(128bit): + * XGMAC address register low + */ +#define FR_AB_XM_ADR_LO_REG_OFST 0x00001200 +/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_XM_ADR_LO_LBN 0 +#define FRF_AB_XM_ADR_LO_WIDTH 32 + + +/* + * FR_AB_XM_ADR_HI_REG(128bit): + * XGMAC address register high + */ +#define FR_AB_XM_ADR_HI_REG_OFST 0x00001210 +/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_XM_ADR_HI_LBN 0 +#define FRF_AB_XM_ADR_HI_WIDTH 16 + + +/* + * FR_AB_XM_GLB_CFG_REG(128bit): + * XGMAC global configuration + */ +#define FR_AB_XM_GLB_CFG_REG_OFST 0x00001220 +/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_XM_RMTFLT_GEN_LBN 17 +#define FRF_AB_XM_RMTFLT_GEN_WIDTH 1 +#define FRF_AB_XM_DEBUG_MODE_LBN 16 +#define FRF_AB_XM_DEBUG_MODE_WIDTH 1 +#define FRF_AB_XM_RX_STAT_EN_LBN 11 +#define FRF_AB_XM_RX_STAT_EN_WIDTH 1 +#define FRF_AB_XM_TX_STAT_EN_LBN 10 +#define FRF_AB_XM_TX_STAT_EN_WIDTH 1 +#define FRF_AB_XM_RX_JUMBO_MODE_LBN 6 +#define FRF_AB_XM_RX_JUMBO_MODE_WIDTH 1 +#define FRF_AB_XM_WAN_MODE_LBN 5 +#define FRF_AB_XM_WAN_MODE_WIDTH 1 +#define FRF_AB_XM_INTCLR_MODE_LBN 3 +#define FRF_AB_XM_INTCLR_MODE_WIDTH 1 +#define FRF_AB_XM_CORE_RST_LBN 0 +#define FRF_AB_XM_CORE_RST_WIDTH 1 + + +/* + * FR_AB_XM_TX_CFG_REG(128bit): + * XGMAC transmit configuration + */ +#define FR_AB_XM_TX_CFG_REG_OFST 0x00001230 +/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_XM_TX_PROG_LBN 24 +#define FRF_AB_XM_TX_PROG_WIDTH 1 +#define FRF_AB_XM_IPG_LBN 16 +#define FRF_AB_XM_IPG_WIDTH 4 +#define FRF_AB_XM_FCNTL_LBN 10 +#define FRF_AB_XM_FCNTL_WIDTH 1 +#define FRF_AB_XM_TXCRC_LBN 8 +#define FRF_AB_XM_TXCRC_WIDTH 1 +#define FRF_AB_XM_EDRC_LBN 6 +#define FRF_AB_XM_EDRC_WIDTH 1 +#define FRF_AB_XM_AUTO_PAD_LBN 5 +#define FRF_AB_XM_AUTO_PAD_WIDTH 1 +#define FRF_AB_XM_TX_PRMBL_LBN 2 +#define FRF_AB_XM_TX_PRMBL_WIDTH 1 +#define FRF_AB_XM_TXEN_LBN 1 +#define FRF_AB_XM_TXEN_WIDTH 1 +#define FRF_AB_XM_TX_RST_LBN 0 +#define FRF_AB_XM_TX_RST_WIDTH 1 + + +/* + * FR_AB_XM_RX_CFG_REG(128bit): + * XGMAC receive configuration + */ +#define FR_AB_XM_RX_CFG_REG_OFST 0x00001240 +/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_XM_PASS_LENERR_LBN 26 +#define FRF_AB_XM_PASS_LENERR_WIDTH 1 +#define FRF_AB_XM_PASS_CRC_ERR_LBN 25 +#define FRF_AB_XM_PASS_CRC_ERR_WIDTH 1 +#define FRF_AB_XM_PASS_PRMBLE_ERR_LBN 24 +#define FRF_AB_XM_PASS_PRMBLE_ERR_WIDTH 1 +#define FRF_AB_XM_REJ_BCAST_LBN 20 +#define FRF_AB_XM_REJ_BCAST_WIDTH 1 +#define FRF_AB_XM_ACPT_ALL_MCAST_LBN 11 +#define FRF_AB_XM_ACPT_ALL_MCAST_WIDTH 1 +#define FRF_AB_XM_ACPT_ALL_UCAST_LBN 9 +#define FRF_AB_XM_ACPT_ALL_UCAST_WIDTH 1 +#define FRF_AB_XM_AUTO_DEPAD_LBN 8 +#define FRF_AB_XM_AUTO_DEPAD_WIDTH 1 +#define FRF_AB_XM_RXCRC_LBN 3 +#define FRF_AB_XM_RXCRC_WIDTH 1 +#define FRF_AB_XM_RX_PRMBL_LBN 2 +#define FRF_AB_XM_RX_PRMBL_WIDTH 1 +#define FRF_AB_XM_RXEN_LBN 1 +#define FRF_AB_XM_RXEN_WIDTH 1 +#define FRF_AB_XM_RX_RST_LBN 0 +#define FRF_AB_XM_RX_RST_WIDTH 1 + + +/* + * FR_AB_XM_MGT_INT_MASK(128bit): + * documentation to be written for sum_XM_MGT_INT_MASK + */ +#define FR_AB_XM_MGT_INT_MASK_OFST 0x00001250 +/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_XM_MSK_STA_INTR_LBN 16 +#define FRF_AB_XM_MSK_STA_INTR_WIDTH 1 +#define FRF_AB_XM_MSK_STAT_CNTR_HF_LBN 9 +#define FRF_AB_XM_MSK_STAT_CNTR_HF_WIDTH 1 +#define FRF_AB_XM_MSK_STAT_CNTR_OF_LBN 8 +#define FRF_AB_XM_MSK_STAT_CNTR_OF_WIDTH 1 +#define FRF_AB_XM_MSK_PRMBLE_ERR_LBN 2 +#define FRF_AB_XM_MSK_PRMBLE_ERR_WIDTH 1 +#define FRF_AB_XM_MSK_RMTFLT_LBN 1 +#define FRF_AB_XM_MSK_RMTFLT_WIDTH 1 +#define FRF_AB_XM_MSK_LCLFLT_LBN 0 +#define FRF_AB_XM_MSK_LCLFLT_WIDTH 1 + + +/* + * FR_AB_XM_FC_REG(128bit): + * XGMAC flow control register + */ +#define FR_AB_XM_FC_REG_OFST 0x00001270 +/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_XM_PAUSE_TIME_LBN 16 +#define FRF_AB_XM_PAUSE_TIME_WIDTH 16 +#define FRF_AB_XM_RX_MAC_STAT_LBN 11 +#define FRF_AB_XM_RX_MAC_STAT_WIDTH 1 +#define FRF_AB_XM_TX_MAC_STAT_LBN 10 +#define FRF_AB_XM_TX_MAC_STAT_WIDTH 1 +#define FRF_AB_XM_MCNTL_PASS_LBN 8 +#define FRF_AB_XM_MCNTL_PASS_WIDTH 2 +#define FRF_AB_XM_REJ_CNTL_UCAST_LBN 6 +#define FRF_AB_XM_REJ_CNTL_UCAST_WIDTH 1 +#define FRF_AB_XM_REJ_CNTL_MCAST_LBN 5 +#define FRF_AB_XM_REJ_CNTL_MCAST_WIDTH 1 +#define FRF_AB_XM_ZPAUSE_LBN 2 +#define FRF_AB_XM_ZPAUSE_WIDTH 1 +#define FRF_AB_XM_XMIT_PAUSE_LBN 1 +#define FRF_AB_XM_XMIT_PAUSE_WIDTH 1 +#define FRF_AB_XM_DIS_FCNTL_LBN 0 +#define FRF_AB_XM_DIS_FCNTL_WIDTH 1 + + +/* + * FR_AB_XM_PAUSE_TIME_REG(128bit): + * XGMAC pause time register + */ +#define FR_AB_XM_PAUSE_TIME_REG_OFST 0x00001290 +/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_XM_TX_PAUSE_CNT_LBN 16 +#define FRF_AB_XM_TX_PAUSE_CNT_WIDTH 16 +#define FRF_AB_XM_RX_PAUSE_CNT_LBN 0 +#define FRF_AB_XM_RX_PAUSE_CNT_WIDTH 16 + + +/* + * FR_AB_XM_TX_PARAM_REG(128bit): + * XGMAC transmit parameter register + */ +#define FR_AB_XM_TX_PARAM_REG_OFST 0x000012d0 +/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_XM_TX_JUMBO_MODE_LBN 31 +#define FRF_AB_XM_TX_JUMBO_MODE_WIDTH 1 +#define FRF_AB_XM_MAX_TX_FRM_SIZE_HI_LBN 19 +#define FRF_AB_XM_MAX_TX_FRM_SIZE_HI_WIDTH 11 +#define FRF_AB_XM_MAX_TX_FRM_SIZE_LO_LBN 16 +#define FRF_AB_XM_MAX_TX_FRM_SIZE_LO_WIDTH 3 +#define FRF_AB_XM_PAD_CHAR_LBN 0 +#define FRF_AB_XM_PAD_CHAR_WIDTH 8 + + +/* + * FR_AB_XM_RX_PARAM_REG(128bit): + * XGMAC receive parameter register + */ +#define FR_AB_XM_RX_PARAM_REG_OFST 0x000012e0 +/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_XM_MAX_RX_FRM_SIZE_HI_LBN 3 +#define FRF_AB_XM_MAX_RX_FRM_SIZE_HI_WIDTH 11 +#define FRF_AB_XM_MAX_RX_FRM_SIZE_LO_LBN 0 +#define FRF_AB_XM_MAX_RX_FRM_SIZE_LO_WIDTH 3 + + +/* + * FR_AB_XM_MGT_INT_MSK_REG(128bit): + * XGMAC management interrupt mask register + */ +#define FR_AB_XM_MGT_INT_REG_OFST 0x000012f0 +/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_XM_STAT_CNTR_OF_LBN 9 +#define FRF_AB_XM_STAT_CNTR_OF_WIDTH 1 +#define FRF_AB_XM_STAT_CNTR_HF_LBN 8 +#define FRF_AB_XM_STAT_CNTR_HF_WIDTH 1 +#define FRF_AB_XM_PRMBLE_ERR_LBN 2 +#define FRF_AB_XM_PRMBLE_ERR_WIDTH 1 +#define FRF_AB_XM_RMTFLT_LBN 1 +#define FRF_AB_XM_RMTFLT_WIDTH 1 +#define FRF_AB_XM_LCLFLT_LBN 0 +#define FRF_AB_XM_LCLFLT_WIDTH 1 + + +/* + * FR_AB_XX_PWR_RST_REG(128bit): + * XGXS/XAUI powerdown/reset register + */ +#define FR_AB_XX_PWR_RST_REG_OFST 0x00001300 +/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_XX_PWRDND_SIG_LBN 31 +#define FRF_AB_XX_PWRDND_SIG_WIDTH 1 +#define FRF_AB_XX_PWRDNC_SIG_LBN 30 +#define FRF_AB_XX_PWRDNC_SIG_WIDTH 1 +#define FRF_AB_XX_PWRDNB_SIG_LBN 29 +#define FRF_AB_XX_PWRDNB_SIG_WIDTH 1 +#define FRF_AB_XX_PWRDNA_SIG_LBN 28 +#define FRF_AB_XX_PWRDNA_SIG_WIDTH 1 +#define FRF_AB_XX_SIM_MODE_LBN 27 +#define FRF_AB_XX_SIM_MODE_WIDTH 1 +#define FRF_AB_XX_RSTPLLCD_SIG_LBN 25 +#define FRF_AB_XX_RSTPLLCD_SIG_WIDTH 1 +#define FRF_AB_XX_RSTPLLAB_SIG_LBN 24 +#define FRF_AB_XX_RSTPLLAB_SIG_WIDTH 1 +#define FRF_AB_XX_RESETD_SIG_LBN 23 +#define FRF_AB_XX_RESETD_SIG_WIDTH 1 +#define FRF_AB_XX_RESETC_SIG_LBN 22 +#define FRF_AB_XX_RESETC_SIG_WIDTH 1 +#define FRF_AB_XX_RESETB_SIG_LBN 21 +#define FRF_AB_XX_RESETB_SIG_WIDTH 1 +#define FRF_AB_XX_RESETA_SIG_LBN 20 +#define FRF_AB_XX_RESETA_SIG_WIDTH 1 +#define FRF_AB_XX_RSTXGXSRX_SIG_LBN 18 +#define FRF_AB_XX_RSTXGXSRX_SIG_WIDTH 1 +#define FRF_AB_XX_RSTXGXSTX_SIG_LBN 17 +#define FRF_AB_XX_RSTXGXSTX_SIG_WIDTH 1 +#define FRF_AB_XX_SD_RST_ACT_LBN 16 +#define FRF_AB_XX_SD_RST_ACT_WIDTH 1 +#define FRF_AB_XX_PWRDND_EN_LBN 15 +#define FRF_AB_XX_PWRDND_EN_WIDTH 1 +#define FRF_AB_XX_PWRDNC_EN_LBN 14 +#define FRF_AB_XX_PWRDNC_EN_WIDTH 1 +#define FRF_AB_XX_PWRDNB_EN_LBN 13 +#define FRF_AB_XX_PWRDNB_EN_WIDTH 1 +#define FRF_AB_XX_PWRDNA_EN_LBN 12 +#define FRF_AB_XX_PWRDNA_EN_WIDTH 1 +#define FRF_AB_XX_RSTPLLCD_EN_LBN 9 +#define FRF_AB_XX_RSTPLLCD_EN_WIDTH 1 +#define FRF_AB_XX_RSTPLLAB_EN_LBN 8 +#define FRF_AB_XX_RSTPLLAB_EN_WIDTH 1 +#define FRF_AB_XX_RESETD_EN_LBN 7 +#define FRF_AB_XX_RESETD_EN_WIDTH 1 +#define FRF_AB_XX_RESETC_EN_LBN 6 +#define FRF_AB_XX_RESETC_EN_WIDTH 1 +#define FRF_AB_XX_RESETB_EN_LBN 5 +#define FRF_AB_XX_RESETB_EN_WIDTH 1 +#define FRF_AB_XX_RESETA_EN_LBN 4 +#define FRF_AB_XX_RESETA_EN_WIDTH 1 +#define FRF_AB_XX_RSTXGXSRX_EN_LBN 2 +#define FRF_AB_XX_RSTXGXSRX_EN_WIDTH 1 +#define FRF_AB_XX_RSTXGXSTX_EN_LBN 1 +#define FRF_AB_XX_RSTXGXSTX_EN_WIDTH 1 +#define FRF_AB_XX_RST_XX_EN_LBN 0 +#define FRF_AB_XX_RST_XX_EN_WIDTH 1 + + +/* + * FR_AB_XX_SD_CTL_REG(128bit): + * XGXS/XAUI powerdown/reset control register + */ +#define FR_AB_XX_SD_CTL_REG_OFST 0x00001310 +/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_XX_TERMADJ1_LBN 17 +#define FRF_AB_XX_TERMADJ1_WIDTH 1 +#define FRF_AB_XX_TERMADJ0_LBN 16 +#define FRF_AB_XX_TERMADJ0_WIDTH 1 +#define FRF_AB_XX_HIDRVD_LBN 15 +#define FRF_AB_XX_HIDRVD_WIDTH 1 +#define FRF_AB_XX_LODRVD_LBN 14 +#define FRF_AB_XX_LODRVD_WIDTH 1 +#define FRF_AB_XX_HIDRVC_LBN 13 +#define FRF_AB_XX_HIDRVC_WIDTH 1 +#define FRF_AB_XX_LODRVC_LBN 12 +#define FRF_AB_XX_LODRVC_WIDTH 1 +#define FRF_AB_XX_HIDRVB_LBN 11 +#define FRF_AB_XX_HIDRVB_WIDTH 1 +#define FRF_AB_XX_LODRVB_LBN 10 +#define FRF_AB_XX_LODRVB_WIDTH 1 +#define FRF_AB_XX_HIDRVA_LBN 9 +#define FRF_AB_XX_HIDRVA_WIDTH 1 +#define FRF_AB_XX_LODRVA_LBN 8 +#define FRF_AB_XX_LODRVA_WIDTH 1 +#define FRF_AB_XX_LPBKD_LBN 3 +#define FRF_AB_XX_LPBKD_WIDTH 1 +#define FRF_AB_XX_LPBKC_LBN 2 +#define FRF_AB_XX_LPBKC_WIDTH 1 +#define FRF_AB_XX_LPBKB_LBN 1 +#define FRF_AB_XX_LPBKB_WIDTH 1 +#define FRF_AB_XX_LPBKA_LBN 0 +#define FRF_AB_XX_LPBKA_WIDTH 1 + + +/* + * FR_AB_XX_TXDRV_CTL_REG(128bit): + * XAUI SerDes transmit drive control register + */ +#define FR_AB_XX_TXDRV_CTL_REG_OFST 0x00001320 +/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_XX_DEQD_LBN 28 +#define FRF_AB_XX_DEQD_WIDTH 4 +#define FRF_AB_XX_DEQC_LBN 24 +#define FRF_AB_XX_DEQC_WIDTH 4 +#define FRF_AB_XX_DEQB_LBN 20 +#define FRF_AB_XX_DEQB_WIDTH 4 +#define FRF_AB_XX_DEQA_LBN 16 +#define FRF_AB_XX_DEQA_WIDTH 4 +#define FRF_AB_XX_DTXD_LBN 12 +#define FRF_AB_XX_DTXD_WIDTH 4 +#define FRF_AB_XX_DTXC_LBN 8 +#define FRF_AB_XX_DTXC_WIDTH 4 +#define FRF_AB_XX_DTXB_LBN 4 +#define FRF_AB_XX_DTXB_WIDTH 4 +#define FRF_AB_XX_DTXA_LBN 0 +#define FRF_AB_XX_DTXA_WIDTH 4 + + +/* + * FR_AB_XX_PRBS_CTL_REG(128bit): + * documentation to be written for sum_XX_PRBS_CTL_REG + */ +#define FR_AB_XX_PRBS_CTL_REG_OFST 0x00001330 +/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_XX_CH3_RX_PRBS_SEL_LBN 30 +#define FRF_AB_XX_CH3_RX_PRBS_SEL_WIDTH 2 +#define FRF_AB_XX_CH3_RX_PRBS_INV_LBN 29 +#define FRF_AB_XX_CH3_RX_PRBS_INV_WIDTH 1 +#define FRF_AB_XX_CH3_RX_PRBS_CHKEN_LBN 28 +#define FRF_AB_XX_CH3_RX_PRBS_CHKEN_WIDTH 1 +#define FRF_AB_XX_CH2_RX_PRBS_SEL_LBN 26 +#define FRF_AB_XX_CH2_RX_PRBS_SEL_WIDTH 2 +#define FRF_AB_XX_CH2_RX_PRBS_INV_LBN 25 +#define FRF_AB_XX_CH2_RX_PRBS_INV_WIDTH 1 +#define FRF_AB_XX_CH2_RX_PRBS_CHKEN_LBN 24 +#define FRF_AB_XX_CH2_RX_PRBS_CHKEN_WIDTH 1 +#define FRF_AB_XX_CH1_RX_PRBS_SEL_LBN 22 +#define FRF_AB_XX_CH1_RX_PRBS_SEL_WIDTH 2 +#define FRF_AB_XX_CH1_RX_PRBS_INV_LBN 21 +#define FRF_AB_XX_CH1_RX_PRBS_INV_WIDTH 1 +#define FRF_AB_XX_CH1_RX_PRBS_CHKEN_LBN 20 +#define FRF_AB_XX_CH1_RX_PRBS_CHKEN_WIDTH 1 +#define FRF_AB_XX_CH0_RX_PRBS_SEL_LBN 18 +#define FRF_AB_XX_CH0_RX_PRBS_SEL_WIDTH 2 +#define FRF_AB_XX_CH0_RX_PRBS_INV_LBN 17 +#define FRF_AB_XX_CH0_RX_PRBS_INV_WIDTH 1 +#define FRF_AB_XX_CH0_RX_PRBS_CHKEN_LBN 16 +#define FRF_AB_XX_CH0_RX_PRBS_CHKEN_WIDTH 1 +#define FRF_AB_XX_CH3_TX_PRBS_SEL_LBN 14 +#define FRF_AB_XX_CH3_TX_PRBS_SEL_WIDTH 2 +#define FRF_AB_XX_CH3_TX_PRBS_INV_LBN 13 +#define FRF_AB_XX_CH3_TX_PRBS_INV_WIDTH 1 +#define FRF_AB_XX_CH3_TX_PRBS_CHKEN_LBN 12 +#define FRF_AB_XX_CH3_TX_PRBS_CHKEN_WIDTH 1 +#define FRF_AB_XX_CH2_TX_PRBS_SEL_LBN 10 +#define FRF_AB_XX_CH2_TX_PRBS_SEL_WIDTH 2 +#define FRF_AB_XX_CH2_TX_PRBS_INV_LBN 9 +#define FRF_AB_XX_CH2_TX_PRBS_INV_WIDTH 1 +#define FRF_AB_XX_CH2_TX_PRBS_CHKEN_LBN 8 +#define FRF_AB_XX_CH2_TX_PRBS_CHKEN_WIDTH 1 +#define FRF_AB_XX_CH1_TX_PRBS_SEL_LBN 6 +#define FRF_AB_XX_CH1_TX_PRBS_SEL_WIDTH 2 +#define FRF_AB_XX_CH1_TX_PRBS_INV_LBN 5 +#define FRF_AB_XX_CH1_TX_PRBS_INV_WIDTH 1 +#define FRF_AB_XX_CH1_TX_PRBS_CHKEN_LBN 4 +#define FRF_AB_XX_CH1_TX_PRBS_CHKEN_WIDTH 1 +#define FRF_AB_XX_CH0_TX_PRBS_SEL_LBN 2 +#define FRF_AB_XX_CH0_TX_PRBS_SEL_WIDTH 2 +#define FRF_AB_XX_CH0_TX_PRBS_INV_LBN 1 +#define FRF_AB_XX_CH0_TX_PRBS_INV_WIDTH 1 +#define FRF_AB_XX_CH0_TX_PRBS_CHKEN_LBN 0 +#define FRF_AB_XX_CH0_TX_PRBS_CHKEN_WIDTH 1 + + +/* + * FR_AB_XX_PRBS_CHK_REG(128bit): + * documentation to be written for sum_XX_PRBS_CHK_REG + */ +#define FR_AB_XX_PRBS_CHK_REG_OFST 0x00001340 +/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_XX_REV_LB_EN_LBN 16 +#define FRF_AB_XX_REV_LB_EN_WIDTH 1 +#define FRF_AB_XX_CH3_DEG_DET_LBN 15 +#define FRF_AB_XX_CH3_DEG_DET_WIDTH 1 +#define FRF_AB_XX_CH3_LFSR_LOCK_IND_LBN 14 +#define FRF_AB_XX_CH3_LFSR_LOCK_IND_WIDTH 1 +#define FRF_AB_XX_CH3_PRBS_FRUN_LBN 13 +#define FRF_AB_XX_CH3_PRBS_FRUN_WIDTH 1 +#define FRF_AB_XX_CH3_ERR_CHK_LBN 12 +#define FRF_AB_XX_CH3_ERR_CHK_WIDTH 1 +#define FRF_AB_XX_CH2_DEG_DET_LBN 11 +#define FRF_AB_XX_CH2_DEG_DET_WIDTH 1 +#define FRF_AB_XX_CH2_LFSR_LOCK_IND_LBN 10 +#define FRF_AB_XX_CH2_LFSR_LOCK_IND_WIDTH 1 +#define FRF_AB_XX_CH2_PRBS_FRUN_LBN 9 +#define FRF_AB_XX_CH2_PRBS_FRUN_WIDTH 1 +#define FRF_AB_XX_CH2_ERR_CHK_LBN 8 +#define FRF_AB_XX_CH2_ERR_CHK_WIDTH 1 +#define FRF_AB_XX_CH1_DEG_DET_LBN 7 +#define FRF_AB_XX_CH1_DEG_DET_WIDTH 1 +#define FRF_AB_XX_CH1_LFSR_LOCK_IND_LBN 6 +#define FRF_AB_XX_CH1_LFSR_LOCK_IND_WIDTH 1 +#define FRF_AB_XX_CH1_PRBS_FRUN_LBN 5 +#define FRF_AB_XX_CH1_PRBS_FRUN_WIDTH 1 +#define FRF_AB_XX_CH1_ERR_CHK_LBN 4 +#define FRF_AB_XX_CH1_ERR_CHK_WIDTH 1 +#define FRF_AB_XX_CH0_DEG_DET_LBN 3 +#define FRF_AB_XX_CH0_DEG_DET_WIDTH 1 +#define FRF_AB_XX_CH0_LFSR_LOCK_IND_LBN 2 +#define FRF_AB_XX_CH0_LFSR_LOCK_IND_WIDTH 1 +#define FRF_AB_XX_CH0_PRBS_FRUN_LBN 1 +#define FRF_AB_XX_CH0_PRBS_FRUN_WIDTH 1 +#define FRF_AB_XX_CH0_ERR_CHK_LBN 0 +#define FRF_AB_XX_CH0_ERR_CHK_WIDTH 1 + + +/* + * FR_AB_XX_PRBS_ERR_REG(128bit): + * documentation to be written for sum_XX_PRBS_ERR_REG + */ +#define FR_AB_XX_PRBS_ERR_REG_OFST 0x00001350 +/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_XX_CH3_PRBS_ERR_CNT_LBN 24 +#define FRF_AB_XX_CH3_PRBS_ERR_CNT_WIDTH 8 +#define FRF_AB_XX_CH2_PRBS_ERR_CNT_LBN 16 +#define FRF_AB_XX_CH2_PRBS_ERR_CNT_WIDTH 8 +#define FRF_AB_XX_CH1_PRBS_ERR_CNT_LBN 8 +#define FRF_AB_XX_CH1_PRBS_ERR_CNT_WIDTH 8 +#define FRF_AB_XX_CH0_PRBS_ERR_CNT_LBN 0 +#define FRF_AB_XX_CH0_PRBS_ERR_CNT_WIDTH 8 + + +/* + * FR_AB_XX_CORE_STAT_REG(128bit): + * XAUI XGXS core status register + */ +#define FR_AB_XX_CORE_STAT_REG_OFST 0x00001360 +/* falcona0,falconb0=net_func_bar2,falcona0=char_func_bar0 */ + +#define FRF_AB_XX_FORCE_SIG3_LBN 31 +#define FRF_AB_XX_FORCE_SIG3_WIDTH 1 +#define FRF_AB_XX_FORCE_SIG3_VAL_LBN 30 +#define FRF_AB_XX_FORCE_SIG3_VAL_WIDTH 1 +#define FRF_AB_XX_FORCE_SIG2_LBN 29 +#define FRF_AB_XX_FORCE_SIG2_WIDTH 1 +#define FRF_AB_XX_FORCE_SIG2_VAL_LBN 28 +#define FRF_AB_XX_FORCE_SIG2_VAL_WIDTH 1 +#define FRF_AB_XX_FORCE_SIG1_LBN 27 +#define FRF_AB_XX_FORCE_SIG1_WIDTH 1 +#define FRF_AB_XX_FORCE_SIG1_VAL_LBN 26 +#define FRF_AB_XX_FORCE_SIG1_VAL_WIDTH 1 +#define FRF_AB_XX_FORCE_SIG0_LBN 25 +#define FRF_AB_XX_FORCE_SIG0_WIDTH 1 +#define FRF_AB_XX_FORCE_SIG0_VAL_LBN 24 +#define FRF_AB_XX_FORCE_SIG0_VAL_WIDTH 1 +#define FRF_AB_XX_XGXS_LB_EN_LBN 23 +#define FRF_AB_XX_XGXS_LB_EN_WIDTH 1 +#define FRF_AB_XX_XGMII_LB_EN_LBN 22 +#define FRF_AB_XX_XGMII_LB_EN_WIDTH 1 +#define FRF_AB_XX_MATCH_FAULT_LBN 21 +#define FRF_AB_XX_MATCH_FAULT_WIDTH 1 +#define FRF_AB_XX_ALIGN_DONE_LBN 20 +#define FRF_AB_XX_ALIGN_DONE_WIDTH 1 +#define FRF_AB_XX_SYNC_STAT3_LBN 19 +#define FRF_AB_XX_SYNC_STAT3_WIDTH 1 +#define FRF_AB_XX_SYNC_STAT2_LBN 18 +#define FRF_AB_XX_SYNC_STAT2_WIDTH 1 +#define FRF_AB_XX_SYNC_STAT1_LBN 17 +#define FRF_AB_XX_SYNC_STAT1_WIDTH 1 +#define FRF_AB_XX_SYNC_STAT0_LBN 16 +#define FRF_AB_XX_SYNC_STAT0_WIDTH 1 +#define FRF_AB_XX_COMMA_DET_CH3_LBN 15 +#define FRF_AB_XX_COMMA_DET_CH3_WIDTH 1 +#define FRF_AB_XX_COMMA_DET_CH2_LBN 14 +#define FRF_AB_XX_COMMA_DET_CH2_WIDTH 1 +#define FRF_AB_XX_COMMA_DET_CH1_LBN 13 +#define FRF_AB_XX_COMMA_DET_CH1_WIDTH 1 +#define FRF_AB_XX_COMMA_DET_CH0_LBN 12 +#define FRF_AB_XX_COMMA_DET_CH0_WIDTH 1 +#define FRF_AB_XX_CGRP_ALIGN_CH3_LBN 11 +#define FRF_AB_XX_CGRP_ALIGN_CH3_WIDTH 1 +#define FRF_AB_XX_CGRP_ALIGN_CH2_LBN 10 +#define FRF_AB_XX_CGRP_ALIGN_CH2_WIDTH 1 +#define FRF_AB_XX_CGRP_ALIGN_CH1_LBN 9 +#define FRF_AB_XX_CGRP_ALIGN_CH1_WIDTH 1 +#define FRF_AB_XX_CGRP_ALIGN_CH0_LBN 8 +#define FRF_AB_XX_CGRP_ALIGN_CH0_WIDTH 1 +#define FRF_AB_XX_CHAR_ERR_CH3_LBN 7 +#define FRF_AB_XX_CHAR_ERR_CH3_WIDTH 1 +#define FRF_AB_XX_CHAR_ERR_CH2_LBN 6 +#define FRF_AB_XX_CHAR_ERR_CH2_WIDTH 1 +#define FRF_AB_XX_CHAR_ERR_CH1_LBN 5 +#define FRF_AB_XX_CHAR_ERR_CH1_WIDTH 1 +#define FRF_AB_XX_CHAR_ERR_CH0_LBN 4 +#define FRF_AB_XX_CHAR_ERR_CH0_WIDTH 1 +#define FRF_AB_XX_DISPERR_CH3_LBN 3 +#define FRF_AB_XX_DISPERR_CH3_WIDTH 1 +#define FRF_AB_XX_DISPERR_CH2_LBN 2 +#define FRF_AB_XX_DISPERR_CH2_WIDTH 1 +#define FRF_AB_XX_DISPERR_CH1_LBN 1 +#define FRF_AB_XX_DISPERR_CH1_WIDTH 1 +#define FRF_AB_XX_DISPERR_CH0_LBN 0 +#define FRF_AB_XX_DISPERR_CH0_WIDTH 1 + + +/* + * FR_AA_RX_DESC_PTR_TBL_KER(128bit): + * Receive descriptor pointer table + */ +#define FR_AA_RX_DESC_PTR_TBL_KER_OFST 0x00011800 +/* falcona0=net_func_bar2 */ +#define FR_AA_RX_DESC_PTR_TBL_KER_STEP 16 +#define FR_AA_RX_DESC_PTR_TBL_KER_ROWS 4 +/* + * FR_AZ_RX_DESC_PTR_TBL(128bit): + * Receive descriptor pointer table + */ +#define FR_AZ_RX_DESC_PTR_TBL_OFST 0x00f40000 +/* sienaa0=net_func_bar2,falconb0=net_func_bar2,falcona0=char_func_bar0 */ +#define FR_AZ_RX_DESC_PTR_TBL_STEP 16 +#define FR_CZ_RX_DESC_PTR_TBL_ROWS 1024 +#define FR_AB_RX_DESC_PTR_TBL_ROWS 4096 + +#define FRF_CZ_RX_HDR_SPLIT_LBN 90 +#define FRF_CZ_RX_HDR_SPLIT_WIDTH 1 +#define FRF_AZ_RX_RESET_LBN 89 +#define FRF_AZ_RX_RESET_WIDTH 1 +#define FRF_AZ_RX_ISCSI_DDIG_EN_LBN 88 +#define FRF_AZ_RX_ISCSI_DDIG_EN_WIDTH 1 +#define FRF_AZ_RX_ISCSI_HDIG_EN_LBN 87 +#define FRF_AZ_RX_ISCSI_HDIG_EN_WIDTH 1 +#define FRF_AZ_RX_DESC_PREF_ACT_LBN 86 +#define FRF_AZ_RX_DESC_PREF_ACT_WIDTH 1 +#define FRF_AZ_RX_DC_HW_RPTR_LBN 80 +#define FRF_AZ_RX_DC_HW_RPTR_WIDTH 6 +#define FRF_AZ_RX_DESCQ_HW_RPTR_LBN 68 +#define FRF_AZ_RX_DESCQ_HW_RPTR_WIDTH 12 +#define FRF_AZ_RX_DESCQ_SW_WPTR_LBN 56 +#define FRF_AZ_RX_DESCQ_SW_WPTR_WIDTH 12 +#define FRF_AZ_RX_DESCQ_BUF_BASE_ID_LBN 36 +#define FRF_AZ_RX_DESCQ_BUF_BASE_ID_WIDTH 20 +#define FRF_AZ_RX_DESCQ_EVQ_ID_LBN 24 +#define FRF_AZ_RX_DESCQ_EVQ_ID_WIDTH 12 +#define FRF_AZ_RX_DESCQ_OWNER_ID_LBN 10 +#define FRF_AZ_RX_DESCQ_OWNER_ID_WIDTH 14 +#define FRF_AZ_RX_DESCQ_LABEL_LBN 5 +#define FRF_AZ_RX_DESCQ_LABEL_WIDTH 5 +#define FRF_AZ_RX_DESCQ_SIZE_LBN 3 +#define FRF_AZ_RX_DESCQ_SIZE_WIDTH 2 +#define FFE_AZ_RX_DESCQ_SIZE_4K 3 +#define FFE_AZ_RX_DESCQ_SIZE_2K 2 +#define FFE_AZ_RX_DESCQ_SIZE_1K 1 +#define FFE_AZ_RX_DESCQ_SIZE_512 0 +#define FRF_AZ_RX_DESCQ_TYPE_LBN 2 +#define FRF_AZ_RX_DESCQ_TYPE_WIDTH 1 +#define FRF_AZ_RX_DESCQ_JUMBO_LBN 1 +#define FRF_AZ_RX_DESCQ_JUMBO_WIDTH 1 +#define FRF_AZ_RX_DESCQ_EN_LBN 0 +#define FRF_AZ_RX_DESCQ_EN_WIDTH 1 + + +/* + * FR_AA_TX_DESC_PTR_TBL_KER(128bit): + * Transmit descriptor pointer + */ +#define FR_AA_TX_DESC_PTR_TBL_KER_OFST 0x00011900 +/* falcona0=net_func_bar2 */ +#define FR_AA_TX_DESC_PTR_TBL_KER_STEP 16 +#define FR_AA_TX_DESC_PTR_TBL_KER_ROWS 8 +/* + * FR_AZ_TX_DESC_PTR_TBL(128bit): + * Transmit descriptor pointer + */ +#define FR_AZ_TX_DESC_PTR_TBL_OFST 0x00f50000 +/* falconb0=net_func_bar2,sienaa0=net_func_bar2,falcona0=char_func_bar0 */ +#define FR_AZ_TX_DESC_PTR_TBL_STEP 16 +#define FR_AB_TX_DESC_PTR_TBL_ROWS 4096 +#define FR_CZ_TX_DESC_PTR_TBL_ROWS 1024 + +#define FRF_CZ_TX_DPT_Q_MASK_WIDTH_LBN 94 +#define FRF_CZ_TX_DPT_Q_MASK_WIDTH_WIDTH 2 +#define FRF_CZ_TX_DPT_ETH_FILT_EN_LBN 93 +#define FRF_CZ_TX_DPT_ETH_FILT_EN_WIDTH 1 +#define FRF_CZ_TX_DPT_IP_FILT_EN_LBN 92 +#define FRF_CZ_TX_DPT_IP_FILT_EN_WIDTH 1 +#define FRF_BZ_TX_NON_IP_DROP_DIS_LBN 91 +#define FRF_BZ_TX_NON_IP_DROP_DIS_WIDTH 1 +#define FRF_BZ_TX_IP_CHKSM_DIS_LBN 90 +#define FRF_BZ_TX_IP_CHKSM_DIS_WIDTH 1 +#define FRF_BZ_TX_TCP_CHKSM_DIS_LBN 89 +#define FRF_BZ_TX_TCP_CHKSM_DIS_WIDTH 1 +#define FRF_AZ_TX_DESCQ_EN_LBN 88 +#define FRF_AZ_TX_DESCQ_EN_WIDTH 1 +#define FRF_AZ_TX_ISCSI_DDIG_EN_LBN 87 +#define FRF_AZ_TX_ISCSI_DDIG_EN_WIDTH 1 +#define FRF_AZ_TX_ISCSI_HDIG_EN_LBN 86 +#define FRF_AZ_TX_ISCSI_HDIG_EN_WIDTH 1 +#define FRF_AZ_TX_DC_HW_RPTR_LBN 80 +#define FRF_AZ_TX_DC_HW_RPTR_WIDTH 6 +#define FRF_AZ_TX_DESCQ_HW_RPTR_LBN 68 +#define FRF_AZ_TX_DESCQ_HW_RPTR_WIDTH 12 +#define FRF_AZ_TX_DESCQ_SW_WPTR_LBN 56 +#define FRF_AZ_TX_DESCQ_SW_WPTR_WIDTH 12 +#define FRF_AZ_TX_DESCQ_BUF_BASE_ID_LBN 36 +#define FRF_AZ_TX_DESCQ_BUF_BASE_ID_WIDTH 20 +#define FRF_AZ_TX_DESCQ_EVQ_ID_LBN 24 +#define FRF_AZ_TX_DESCQ_EVQ_ID_WIDTH 12 +#define FRF_AZ_TX_DESCQ_OWNER_ID_LBN 10 +#define FRF_AZ_TX_DESCQ_OWNER_ID_WIDTH 14 +#define FRF_AZ_TX_DESCQ_LABEL_LBN 5 +#define FRF_AZ_TX_DESCQ_LABEL_WIDTH 5 +#define FRF_AZ_TX_DESCQ_SIZE_LBN 3 +#define FRF_AZ_TX_DESCQ_SIZE_WIDTH 2 +#define FFE_AZ_TX_DESCQ_SIZE_4K 3 +#define FFE_AZ_TX_DESCQ_SIZE_2K 2 +#define FFE_AZ_TX_DESCQ_SIZE_1K 1 +#define FFE_AZ_TX_DESCQ_SIZE_512 0 +#define FRF_AZ_TX_DESCQ_TYPE_LBN 1 +#define FRF_AZ_TX_DESCQ_TYPE_WIDTH 2 +#define FRF_AZ_TX_DESCQ_FLUSH_LBN 0 +#define FRF_AZ_TX_DESCQ_FLUSH_WIDTH 1 + + +/* + * FR_AA_EVQ_PTR_TBL_KER(128bit): + * Event queue pointer table + */ +#define FR_AA_EVQ_PTR_TBL_KER_OFST 0x00011a00 +/* falcona0=net_func_bar2 */ +#define FR_AA_EVQ_PTR_TBL_KER_STEP 16 +#define FR_AA_EVQ_PTR_TBL_KER_ROWS 4 +/* + * FR_AZ_EVQ_PTR_TBL(128bit): + * Event queue pointer table + */ +#define FR_AZ_EVQ_PTR_TBL_OFST 0x00f60000 +/* sienaa0=net_func_bar2,falconb0=net_func_bar2,falcona0=char_func_bar0 */ +#define FR_AZ_EVQ_PTR_TBL_STEP 16 +#define FR_CZ_EVQ_PTR_TBL_ROWS 1024 +#define FR_AB_EVQ_PTR_TBL_ROWS 4096 + +#define FRF_BZ_EVQ_RPTR_IGN_LBN 40 +#define FRF_BZ_EVQ_RPTR_IGN_WIDTH 1 +#define FRF_AZ_EVQ_WKUP_OR_INT_EN_LBN 39 +#define FRF_AZ_EVQ_WKUP_OR_INT_EN_WIDTH 1 +#define FRF_AZ_EVQ_NXT_WPTR_LBN 24 +#define FRF_AZ_EVQ_NXT_WPTR_WIDTH 15 +#define FRF_AZ_EVQ_EN_LBN 23 +#define FRF_AZ_EVQ_EN_WIDTH 1 +#define FRF_AZ_EVQ_SIZE_LBN 20 +#define FRF_AZ_EVQ_SIZE_WIDTH 3 +#define FFE_AZ_EVQ_SIZE_32K 6 +#define FFE_AZ_EVQ_SIZE_16K 5 +#define FFE_AZ_EVQ_SIZE_8K 4 +#define FFE_AZ_EVQ_SIZE_4K 3 +#define FFE_AZ_EVQ_SIZE_2K 2 +#define FFE_AZ_EVQ_SIZE_1K 1 +#define FFE_AZ_EVQ_SIZE_512 0 +#define FRF_AZ_EVQ_BUF_BASE_ID_LBN 0 +#define FRF_AZ_EVQ_BUF_BASE_ID_WIDTH 20 + + +/* + * FR_AA_BUF_HALF_TBL_KER(64bit): + * Buffer table in half buffer table mode direct access by driver + */ +#define FR_AA_BUF_HALF_TBL_KER_OFST 0x00018000 +/* falcona0=net_func_bar2 */ +#define FR_AA_BUF_HALF_TBL_KER_STEP 8 +#define FR_AA_BUF_HALF_TBL_KER_ROWS 4096 +/* + * FR_AZ_BUF_HALF_TBL(64bit): + * Buffer table in half buffer table mode direct access by driver + */ +#define FR_AZ_BUF_HALF_TBL_OFST 0x00800000 +/* sienaa0=net_func_bar2,falconb0=net_func_bar2,falcona0=char_func_bar0 */ +#define FR_AZ_BUF_HALF_TBL_STEP 8 +#define FR_CZ_BUF_HALF_TBL_ROWS 147456 +#define FR_AB_BUF_HALF_TBL_ROWS 524288 + +#define FRF_AZ_BUF_ADR_HBUF_ODD_LBN 44 +#define FRF_AZ_BUF_ADR_HBUF_ODD_WIDTH 20 +#define FRF_AZ_BUF_OWNER_ID_HBUF_ODD_LBN 32 +#define FRF_AZ_BUF_OWNER_ID_HBUF_ODD_WIDTH 12 +#define FRF_AZ_BUF_ADR_HBUF_EVEN_LBN 12 +#define FRF_AZ_BUF_ADR_HBUF_EVEN_WIDTH 20 +#define FRF_AZ_BUF_OWNER_ID_HBUF_EVEN_LBN 0 +#define FRF_AZ_BUF_OWNER_ID_HBUF_EVEN_WIDTH 12 + + +/* + * FR_AA_BUF_FULL_TBL_KER(64bit): + * Buffer table in full buffer table mode direct access by driver + */ +#define FR_AA_BUF_FULL_TBL_KER_OFST 0x00018000 +/* falcona0=net_func_bar2 */ +#define FR_AA_BUF_FULL_TBL_KER_STEP 8 +#define FR_AA_BUF_FULL_TBL_KER_ROWS 4096 +/* + * FR_AZ_BUF_FULL_TBL(64bit): + * Buffer table in full buffer table mode direct access by driver + */ +#define FR_AZ_BUF_FULL_TBL_OFST 0x00800000 +/* sienaa0=net_func_bar2,falconb0=net_func_bar2,falcona0=char_func_bar0 */ +#define FR_AZ_BUF_FULL_TBL_STEP 8 + +#define FR_CZ_BUF_FULL_TBL_ROWS 147456 +#define FR_AB_BUF_FULL_TBL_ROWS 917504 + +#define FRF_AZ_BUF_FULL_UNUSED_LBN 51 +#define FRF_AZ_BUF_FULL_UNUSED_WIDTH 13 +#define FRF_AZ_IP_DAT_BUF_SIZE_LBN 50 +#define FRF_AZ_IP_DAT_BUF_SIZE_WIDTH 1 +#define FRF_AZ_BUF_ADR_REGION_LBN 48 +#define FRF_AZ_BUF_ADR_REGION_WIDTH 2 +#define FFE_AZ_BUF_ADR_REGN3 3 +#define FFE_AZ_BUF_ADR_REGN2 2 +#define FFE_AZ_BUF_ADR_REGN1 1 +#define FFE_AZ_BUF_ADR_REGN0 0 +#define FRF_AZ_BUF_ADR_FBUF_LBN 14 +#define FRF_AZ_BUF_ADR_FBUF_WIDTH 34 +#define FRF_AZ_BUF_ADR_FBUF_DW0_LBN 14 +#define FRF_AZ_BUF_ADR_FBUF_DW0_WIDTH 32 +#define FRF_AZ_BUF_ADR_FBUF_DW1_LBN 46 +#define FRF_AZ_BUF_ADR_FBUF_DW1_WIDTH 2 +#define FRF_AZ_BUF_OWNER_ID_FBUF_LBN 0 +#define FRF_AZ_BUF_OWNER_ID_FBUF_WIDTH 14 + + +/* + * FR_AZ_RX_FILTER_TBL0(128bit): + * TCP/IPv4 Receive filter table + */ +#define FR_AZ_RX_FILTER_TBL0_OFST 0x00f00000 +/* falconb0,sienaa0=net_func_bar2,falcona0=char_func_bar0 */ +#define FR_AZ_RX_FILTER_TBL0_STEP 32 +#define FR_AZ_RX_FILTER_TBL0_ROWS 8192 +/* + * FR_AB_RX_FILTER_TBL1(128bit): + * TCP/IPv4 Receive filter table + */ +#define FR_AB_RX_FILTER_TBL1_OFST 0x00f00010 +/* falconb0=net_func_bar2,falcona0=char_func_bar0 */ +#define FR_AB_RX_FILTER_TBL1_STEP 32 +#define FR_AB_RX_FILTER_TBL1_ROWS 8192 + +#define FRF_BZ_RSS_EN_LBN 110 +#define FRF_BZ_RSS_EN_WIDTH 1 +#define FRF_BZ_SCATTER_EN_LBN 109 +#define FRF_BZ_SCATTER_EN_WIDTH 1 +#define FRF_AZ_TCP_UDP_LBN 108 +#define FRF_AZ_TCP_UDP_WIDTH 1 +#define FRF_AZ_RXQ_ID_LBN 96 +#define FRF_AZ_RXQ_ID_WIDTH 12 +#define FRF_AZ_DEST_IP_LBN 64 +#define FRF_AZ_DEST_IP_WIDTH 32 +#define FRF_AZ_DEST_PORT_TCP_LBN 48 +#define FRF_AZ_DEST_PORT_TCP_WIDTH 16 +#define FRF_AZ_SRC_IP_LBN 16 +#define FRF_AZ_SRC_IP_WIDTH 32 +#define FRF_AZ_SRC_TCP_DEST_UDP_LBN 0 +#define FRF_AZ_SRC_TCP_DEST_UDP_WIDTH 16 + + +/* + * FR_CZ_RX_MAC_FILTER_TBL0(128bit): + * Receive Ethernet filter table + */ +#define FR_CZ_RX_MAC_FILTER_TBL0_OFST 0x00f00010 +/* sienaa0=net_func_bar2 */ +#define FR_CZ_RX_MAC_FILTER_TBL0_STEP 32 +#define FR_CZ_RX_MAC_FILTER_TBL0_ROWS 512 + +#define FRF_CZ_RMFT_RSS_EN_LBN 75 +#define FRF_CZ_RMFT_RSS_EN_WIDTH 1 +#define FRF_CZ_RMFT_SCATTER_EN_LBN 74 +#define FRF_CZ_RMFT_SCATTER_EN_WIDTH 1 +#define FRF_CZ_RMFT_IP_OVERRIDE_LBN 73 +#define FRF_CZ_RMFT_IP_OVERRIDE_WIDTH 1 +#define FRF_CZ_RMFT_RXQ_ID_LBN 61 +#define FRF_CZ_RMFT_RXQ_ID_WIDTH 12 +#define FRF_CZ_RMFT_WILDCARD_MATCH_LBN 60 +#define FRF_CZ_RMFT_WILDCARD_MATCH_WIDTH 1 +#define FRF_CZ_RMFT_DEST_MAC_LBN 12 +#define FRF_CZ_RMFT_DEST_MAC_WIDTH 48 +#define FRF_CZ_RMFT_DEST_MAC_DW0_LBN 12 +#define FRF_CZ_RMFT_DEST_MAC_DW0_WIDTH 32 +#define FRF_CZ_RMFT_DEST_MAC_DW1_LBN 44 +#define FRF_CZ_RMFT_DEST_MAC_DW1_WIDTH 16 +#define FRF_CZ_RMFT_VLAN_ID_LBN 0 +#define FRF_CZ_RMFT_VLAN_ID_WIDTH 12 + + +/* + * FR_AZ_TIMER_TBL(128bit): + * Timer table + */ +#define FR_AZ_TIMER_TBL_OFST 0x00f70000 +/* sienaa0=net_func_bar2,falconb0=net_func_bar2,falcona0=char_func_bar0 */ +#define FR_AZ_TIMER_TBL_STEP 16 +#define FR_CZ_TIMER_TBL_ROWS 1024 +#define FR_AB_TIMER_TBL_ROWS 4096 + +#define FRF_CZ_TIMER_Q_EN_LBN 33 +#define FRF_CZ_TIMER_Q_EN_WIDTH 1 +#define FRF_CZ_INT_ARMD_LBN 32 +#define FRF_CZ_INT_ARMD_WIDTH 1 +#define FRF_CZ_INT_PEND_LBN 31 +#define FRF_CZ_INT_PEND_WIDTH 1 +#define FRF_CZ_HOST_NOTIFY_MODE_LBN 30 +#define FRF_CZ_HOST_NOTIFY_MODE_WIDTH 1 +#define FRF_CZ_RELOAD_TIMER_VAL_LBN 16 +#define FRF_CZ_RELOAD_TIMER_VAL_WIDTH 14 +#define FRF_CZ_TIMER_MODE_LBN 14 +#define FRF_CZ_TIMER_MODE_WIDTH 2 +#define FFE_CZ_TIMER_MODE_INT_HLDOFF 3 +#define FFE_CZ_TIMER_MODE_TRIG_START 2 +#define FFE_CZ_TIMER_MODE_IMMED_START 1 +#define FFE_CZ_TIMER_MODE_DIS 0 +#define FRF_AB_TIMER_MODE_LBN 12 +#define FRF_AB_TIMER_MODE_WIDTH 2 +#define FFE_AB_TIMER_MODE_INT_HLDOFF 2 +#define FFE_AB_TIMER_MODE_TRIG_START 2 +#define FFE_AB_TIMER_MODE_IMMED_START 1 +#define FFE_AB_TIMER_MODE_DIS 0 +#define FRF_CZ_TIMER_VAL_LBN 0 +#define FRF_CZ_TIMER_VAL_WIDTH 14 +#define FRF_AB_TIMER_VAL_LBN 0 +#define FRF_AB_TIMER_VAL_WIDTH 12 + + +/* + * FR_BZ_TX_PACE_TBL(128bit): + * Transmit pacing table + */ +#define FR_BZ_TX_PACE_TBL_OFST 0x00f80000 +/* sienaa0=net_func_bar2,falconb0=net_func_bar2 */ +#define FR_AZ_TX_PACE_TBL_STEP 16 +#define FR_CZ_TX_PACE_TBL_ROWS 1024 +#define FR_BB_TX_PACE_TBL_ROWS 4096 +/* + * FR_AA_TX_PACE_TBL(128bit): + * Transmit pacing table + */ +#define FR_AA_TX_PACE_TBL_OFST 0x00f80040 +/* falcona0=char_func_bar0 */ +/* FR_AZ_TX_PACE_TBL_STEP 16 */ +#define FR_AA_TX_PACE_TBL_ROWS 4092 + +#define FRF_AZ_TX_PACE_LBN 0 +#define FRF_AZ_TX_PACE_WIDTH 5 + + +/* + * FR_BZ_RX_INDIRECTION_TBL(7bit): + * RX Indirection Table + */ +#define FR_BZ_RX_INDIRECTION_TBL_OFST 0x00fb0000 +/* falconb0,sienaa0=net_func_bar2 */ +#define FR_BZ_RX_INDIRECTION_TBL_STEP 16 +#define FR_BZ_RX_INDIRECTION_TBL_ROWS 128 + +#define FRF_BZ_IT_QUEUE_LBN 0 +#define FRF_BZ_IT_QUEUE_WIDTH 6 + + +/* + * FR_CZ_TX_FILTER_TBL0(128bit): + * TCP/IPv4 Transmit filter table + */ +#define FR_CZ_TX_FILTER_TBL0_OFST 0x00fc0000 +/* sienaa0=net_func_bar2 */ +#define FR_CZ_TX_FILTER_TBL0_STEP 16 +#define FR_CZ_TX_FILTER_TBL0_ROWS 8192 + +#define FRF_CZ_TIFT_TCP_UDP_LBN 108 +#define FRF_CZ_TIFT_TCP_UDP_WIDTH 1 +#define FRF_CZ_TIFT_TXQ_ID_LBN 96 +#define FRF_CZ_TIFT_TXQ_ID_WIDTH 12 +#define FRF_CZ_TIFT_DEST_IP_LBN 64 +#define FRF_CZ_TIFT_DEST_IP_WIDTH 32 +#define FRF_CZ_TIFT_DEST_PORT_TCP_LBN 48 +#define FRF_CZ_TIFT_DEST_PORT_TCP_WIDTH 16 +#define FRF_CZ_TIFT_SRC_IP_LBN 16 +#define FRF_CZ_TIFT_SRC_IP_WIDTH 32 +#define FRF_CZ_TIFT_SRC_TCP_DEST_UDP_LBN 0 +#define FRF_CZ_TIFT_SRC_TCP_DEST_UDP_WIDTH 16 + + +/* + * FR_CZ_TX_MAC_FILTER_TBL0(128bit): + * Transmit Ethernet filter table + */ +#define FR_CZ_TX_MAC_FILTER_TBL0_OFST 0x00fe0000 +/* sienaa0=net_func_bar2 */ +#define FR_CZ_TX_MAC_FILTER_TBL0_STEP 16 +#define FR_CZ_TX_MAC_FILTER_TBL0_ROWS 512 + +#define FRF_CZ_TMFT_TXQ_ID_LBN 61 +#define FRF_CZ_TMFT_TXQ_ID_WIDTH 12 +#define FRF_CZ_TMFT_WILDCARD_MATCH_LBN 60 +#define FRF_CZ_TMFT_WILDCARD_MATCH_WIDTH 1 +#define FRF_CZ_TMFT_SRC_MAC_LBN 12 +#define FRF_CZ_TMFT_SRC_MAC_WIDTH 48 +#define FRF_CZ_TMFT_SRC_MAC_DW0_LBN 12 +#define FRF_CZ_TMFT_SRC_MAC_DW0_WIDTH 32 +#define FRF_CZ_TMFT_SRC_MAC_DW1_LBN 44 +#define FRF_CZ_TMFT_SRC_MAC_DW1_WIDTH 16 +#define FRF_CZ_TMFT_VLAN_ID_LBN 0 +#define FRF_CZ_TMFT_VLAN_ID_WIDTH 12 + + +/* + * FR_CZ_MC_TREG_SMEM(32bit): + * MC Shared Memory + */ +#define FR_CZ_MC_TREG_SMEM_OFST 0x00ff0000 +/* sienaa0=net_func_bar2 */ +#define FR_CZ_MC_TREG_SMEM_STEP 4 +#define FR_CZ_MC_TREG_SMEM_ROWS 512 + +#define FRF_CZ_MC_TREG_SMEM_ROW_LBN 0 +#define FRF_CZ_MC_TREG_SMEM_ROW_WIDTH 32 + + +/* + * FR_BB_MSIX_VECTOR_TABLE(128bit): + * MSIX Vector Table + */ +#define FR_BB_MSIX_VECTOR_TABLE_OFST 0x00ff0000 +/* falconb0=net_func_bar2 */ +#define FR_BZ_MSIX_VECTOR_TABLE_STEP 16 +#define FR_BB_MSIX_VECTOR_TABLE_ROWS 64 +/* + * FR_CZ_MSIX_VECTOR_TABLE(128bit): + * MSIX Vector Table + */ +#define FR_CZ_MSIX_VECTOR_TABLE_OFST 0x00000000 +/* sienaa0=pci_f0_bar4 */ +/* FR_BZ_MSIX_VECTOR_TABLE_STEP 16 */ +#define FR_CZ_MSIX_VECTOR_TABLE_ROWS 1024 + +#define FRF_BZ_MSIX_VECTOR_RESERVED_LBN 97 +#define FRF_BZ_MSIX_VECTOR_RESERVED_WIDTH 31 +#define FRF_BZ_MSIX_VECTOR_MASK_LBN 96 +#define FRF_BZ_MSIX_VECTOR_MASK_WIDTH 1 +#define FRF_BZ_MSIX_MESSAGE_DATA_LBN 64 +#define FRF_BZ_MSIX_MESSAGE_DATA_WIDTH 32 +#define FRF_BZ_MSIX_MESSAGE_ADDRESS_HI_LBN 32 +#define FRF_BZ_MSIX_MESSAGE_ADDRESS_HI_WIDTH 32 +#define FRF_BZ_MSIX_MESSAGE_ADDRESS_LO_LBN 0 +#define FRF_BZ_MSIX_MESSAGE_ADDRESS_LO_WIDTH 32 + + +/* + * FR_BB_MSIX_PBA_TABLE(32bit): + * MSIX Pending Bit Array + */ +#define FR_BB_MSIX_PBA_TABLE_OFST 0x00ff2000 +/* falconb0=net_func_bar2 */ +#define FR_BZ_MSIX_PBA_TABLE_STEP 4 +#define FR_BB_MSIX_PBA_TABLE_ROWS 2 +/* + * FR_CZ_MSIX_PBA_TABLE(32bit): + * MSIX Pending Bit Array + */ +#define FR_CZ_MSIX_PBA_TABLE_OFST 0x00008000 +/* sienaa0=pci_f0_bar4 */ +/* FR_BZ_MSIX_PBA_TABLE_STEP 4 */ +#define FR_CZ_MSIX_PBA_TABLE_ROWS 32 + +#define FRF_BZ_MSIX_PBA_PEND_DWORD_LBN 0 +#define FRF_BZ_MSIX_PBA_PEND_DWORD_WIDTH 32 + + +/* + * FR_AZ_SRM_DBG_REG(64bit): + * SRAM debug access + */ +#define FR_AZ_SRM_DBG_REG_OFST 0x03000000 +/* sienaa0=net_func_bar2,falconb0=net_func_bar2,falcona0=char_func_bar0 */ +#define FR_AZ_SRM_DBG_REG_STEP 8 + +#define FR_CZ_SRM_DBG_REG_ROWS 262144 +#define FR_AB_SRM_DBG_REG_ROWS 2097152 + +#define FRF_AZ_SRM_DBG_LBN 0 +#define FRF_AZ_SRM_DBG_WIDTH 64 +#define FRF_AZ_SRM_DBG_DW0_LBN 0 +#define FRF_AZ_SRM_DBG_DW0_WIDTH 32 +#define FRF_AZ_SRM_DBG_DW1_LBN 32 +#define FRF_AZ_SRM_DBG_DW1_WIDTH 32 + + +/* + * FR_AA_INT_ACK_CHAR(32bit): + * CHAR interrupt acknowledge register + */ +#define FR_AA_INT_ACK_CHAR_OFST 0x00000060 +/* falcona0=char_func_bar0 */ + +#define FRF_AA_INT_ACK_CHAR_FIELD_LBN 0 +#define FRF_AA_INT_ACK_CHAR_FIELD_WIDTH 32 + + +/* FS_DRIVER_EV */ +#define FSF_AZ_DRIVER_EV_SUBCODE_LBN 56 +#define FSF_AZ_DRIVER_EV_SUBCODE_WIDTH 4 +#define FSE_AZ_TX_DSC_ERROR_EV 15 +#define FSE_AZ_RX_DSC_ERROR_EV 14 +#define FSE_AZ_RX_RECOVER_EV 11 +#define FSE_AZ_TIMER_EV 10 +#define FSE_AZ_TX_PKT_NON_TCP_UDP 9 +#define FSE_AZ_WAKE_UP_EV 6 +#define FSE_AZ_SRM_UPD_DONE_EV 5 +#define FSE_AZ_EVQ_NOT_EN_EV 3 +#define FSE_AZ_EVQ_INIT_DONE_EV 2 +#define FSE_AZ_RX_DESCQ_FLS_DONE_EV 1 +#define FSE_AZ_TX_DESCQ_FLS_DONE_EV 0 +#define FSF_AZ_DRIVER_EV_SUBDATA_LBN 0 +#define FSF_AZ_DRIVER_EV_SUBDATA_WIDTH 14 + + +/* FS_EVENT_ENTRY */ +#define FSF_AZ_EV_CODE_LBN 60 +#define FSF_AZ_EV_CODE_WIDTH 4 +#define FSE_AZ_EV_CODE_USER_EV 8 +#define FSE_AZ_EV_CODE_DRV_GEN_EV 7 +#define FSE_AZ_EV_CODE_GLOBAL_EV 6 +#define FSE_AZ_EV_CODE_DRIVER_EV 5 +#define FSE_AZ_EV_CODE_TX_EV 2 +#define FSE_AZ_EV_CODE_RX_EV 0 +#define FSF_AZ_EV_DATA_LBN 0 +#define FSF_AZ_EV_DATA_WIDTH 60 +#define FSF_AZ_EV_DATA_DW0_LBN 0 +#define FSF_AZ_EV_DATA_DW0_WIDTH 32 +#define FSF_AZ_EV_DATA_DW1_LBN 32 +#define FSF_AZ_EV_DATA_DW1_WIDTH 28 + + +/* FS_GLOBAL_EV */ +#define FSF_AA_GLB_EV_RX_RECOVERY_LBN 12 +#define FSF_AA_GLB_EV_RX_RECOVERY_WIDTH 1 +#define FSF_BZ_GLB_EV_XG_MNT_INTR_LBN 11 +#define FSF_BZ_GLB_EV_XG_MNT_INTR_WIDTH 1 +#define FSF_AZ_GLB_EV_XFP_PHY0_INTR_LBN 10 +#define FSF_AZ_GLB_EV_XFP_PHY0_INTR_WIDTH 1 +#define FSF_AZ_GLB_EV_XG_PHY0_INTR_LBN 9 +#define FSF_AZ_GLB_EV_XG_PHY0_INTR_WIDTH 1 +#define FSF_AZ_GLB_EV_G_PHY0_INTR_LBN 7 +#define FSF_AZ_GLB_EV_G_PHY0_INTR_WIDTH 1 + + +/* FS_RX_EV */ +#define FSF_CZ_RX_EV_PKT_NOT_PARSED_LBN 58 +#define FSF_CZ_RX_EV_PKT_NOT_PARSED_WIDTH 1 +#define FSF_CZ_RX_EV_IPV6_PKT_LBN 57 +#define FSF_CZ_RX_EV_IPV6_PKT_WIDTH 1 +#define FSF_AZ_RX_EV_PKT_OK_LBN 56 +#define FSF_AZ_RX_EV_PKT_OK_WIDTH 1 +#define FSF_AZ_RX_EV_PAUSE_FRM_ERR_LBN 55 +#define FSF_AZ_RX_EV_PAUSE_FRM_ERR_WIDTH 1 +#define FSF_AZ_RX_EV_BUF_OWNER_ID_ERR_LBN 54 +#define FSF_AZ_RX_EV_BUF_OWNER_ID_ERR_WIDTH 1 +#define FSF_AZ_RX_EV_IP_FRAG_ERR_LBN 53 +#define FSF_AZ_RX_EV_IP_FRAG_ERR_WIDTH 1 +#define FSF_AZ_RX_EV_IP_HDR_CHKSUM_ERR_LBN 52 +#define FSF_AZ_RX_EV_IP_HDR_CHKSUM_ERR_WIDTH 1 +#define FSF_AZ_RX_EV_TCP_UDP_CHKSUM_ERR_LBN 51 +#define FSF_AZ_RX_EV_TCP_UDP_CHKSUM_ERR_WIDTH 1 +#define FSF_AZ_RX_EV_ETH_CRC_ERR_LBN 50 +#define FSF_AZ_RX_EV_ETH_CRC_ERR_WIDTH 1 +#define FSF_AZ_RX_EV_FRM_TRUNC_LBN 49 +#define FSF_AZ_RX_EV_FRM_TRUNC_WIDTH 1 +#define FSF_AZ_RX_EV_TOBE_DISC_LBN 47 +#define FSF_AZ_RX_EV_TOBE_DISC_WIDTH 1 +#define FSF_AZ_RX_EV_PKT_TYPE_LBN 44 +#define FSF_AZ_RX_EV_PKT_TYPE_WIDTH 3 +#define FSE_AZ_RX_EV_PKT_TYPE_VLAN_JUMBO 5 +#define FSE_AZ_RX_EV_PKT_TYPE_VLAN_LLC 4 +#define FSE_AZ_RX_EV_PKT_TYPE_VLAN 3 +#define FSE_AZ_RX_EV_PKT_TYPE_JUMBO 2 +#define FSE_AZ_RX_EV_PKT_TYPE_LLC 1 +#define FSE_AZ_RX_EV_PKT_TYPE_ETH 0 +#define FSF_AZ_RX_EV_HDR_TYPE_LBN 42 +#define FSF_AZ_RX_EV_HDR_TYPE_WIDTH 2 +#define FSE_AZ_RX_EV_HDR_TYPE_OTHER 3 +#define FSE_AZ_RX_EV_HDR_TYPE_IPV4_OTHER 2 +#define FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_OTHER 2 +#define FSE_AZ_RX_EV_HDR_TYPE_IPV4_UDP 1 +#define FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_UDP 1 +#define FSE_AZ_RX_EV_HDR_TYPE_IPV4_TCP 0 +#define FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_TCP 0 +#define FSF_AZ_RX_EV_DESC_Q_EMPTY_LBN 41 +#define FSF_AZ_RX_EV_DESC_Q_EMPTY_WIDTH 1 +#define FSF_AZ_RX_EV_MCAST_HASH_MATCH_LBN 40 +#define FSF_AZ_RX_EV_MCAST_HASH_MATCH_WIDTH 1 +#define FSF_AZ_RX_EV_MCAST_PKT_LBN 39 +#define FSF_AZ_RX_EV_MCAST_PKT_WIDTH 1 +#define FSF_AA_RX_EV_RECOVERY_FLAG_LBN 37 +#define FSF_AA_RX_EV_RECOVERY_FLAG_WIDTH 1 +#define FSF_AZ_RX_EV_Q_LABEL_LBN 32 +#define FSF_AZ_RX_EV_Q_LABEL_WIDTH 5 +#define FSF_AZ_RX_EV_JUMBO_CONT_LBN 31 +#define FSF_AZ_RX_EV_JUMBO_CONT_WIDTH 1 +#define FSF_AZ_RX_EV_PORT_LBN 30 +#define FSF_AZ_RX_EV_PORT_WIDTH 1 +#define FSF_AZ_RX_EV_BYTE_CNT_LBN 16 +#define FSF_AZ_RX_EV_BYTE_CNT_WIDTH 14 +#define FSF_AZ_RX_EV_SOP_LBN 15 +#define FSF_AZ_RX_EV_SOP_WIDTH 1 +#define FSF_AZ_RX_EV_ISCSI_PKT_OK_LBN 14 +#define FSF_AZ_RX_EV_ISCSI_PKT_OK_WIDTH 1 +#define FSF_AZ_RX_EV_ISCSI_DDIG_ERR_LBN 13 +#define FSF_AZ_RX_EV_ISCSI_DDIG_ERR_WIDTH 1 +#define FSF_AZ_RX_EV_ISCSI_HDIG_ERR_LBN 12 +#define FSF_AZ_RX_EV_ISCSI_HDIG_ERR_WIDTH 1 +#define FSF_AZ_RX_EV_DESC_PTR_LBN 0 +#define FSF_AZ_RX_EV_DESC_PTR_WIDTH 12 + + +/* FS_RX_KER_DESC */ +#define FSF_AZ_RX_KER_BUF_SIZE_LBN 48 +#define FSF_AZ_RX_KER_BUF_SIZE_WIDTH 14 +#define FSF_AZ_RX_KER_BUF_REGION_LBN 46 +#define FSF_AZ_RX_KER_BUF_REGION_WIDTH 2 +#define FSF_AZ_RX_KER_BUF_ADDR_LBN 0 +#define FSF_AZ_RX_KER_BUF_ADDR_WIDTH 46 +#define FSF_AZ_RX_KER_BUF_ADDR_DW0_LBN 0 +#define FSF_AZ_RX_KER_BUF_ADDR_DW0_WIDTH 32 +#define FSF_AZ_RX_KER_BUF_ADDR_DW1_LBN 32 +#define FSF_AZ_RX_KER_BUF_ADDR_DW1_WIDTH 14 + + +/* FS_RX_USER_DESC */ +#define FSF_AZ_RX_USER_2BYTE_OFFSET_LBN 20 +#define FSF_AZ_RX_USER_2BYTE_OFFSET_WIDTH 12 +#define FSF_AZ_RX_USER_BUF_ID_LBN 0 +#define FSF_AZ_RX_USER_BUF_ID_WIDTH 20 + + +/* FS_TX_EV */ +#define FSF_AZ_TX_EV_PKT_ERR_LBN 38 +#define FSF_AZ_TX_EV_PKT_ERR_WIDTH 1 +#define FSF_AZ_TX_EV_PKT_TOO_BIG_LBN 37 +#define FSF_AZ_TX_EV_PKT_TOO_BIG_WIDTH 1 +#define FSF_AZ_TX_EV_Q_LABEL_LBN 32 +#define FSF_AZ_TX_EV_Q_LABEL_WIDTH 5 +#define FSF_AZ_TX_EV_PORT_LBN 16 +#define FSF_AZ_TX_EV_PORT_WIDTH 1 +#define FSF_AZ_TX_EV_WQ_FF_FULL_LBN 15 +#define FSF_AZ_TX_EV_WQ_FF_FULL_WIDTH 1 +#define FSF_AZ_TX_EV_BUF_OWNER_ID_ERR_LBN 14 +#define FSF_AZ_TX_EV_BUF_OWNER_ID_ERR_WIDTH 1 +#define FSF_AZ_TX_EV_COMP_LBN 12 +#define FSF_AZ_TX_EV_COMP_WIDTH 1 +#define FSF_AZ_TX_EV_DESC_PTR_LBN 0 +#define FSF_AZ_TX_EV_DESC_PTR_WIDTH 12 + + +/* FS_TX_KER_DESC */ +#define FSF_AZ_TX_KER_CONT_LBN 62 +#define FSF_AZ_TX_KER_CONT_WIDTH 1 +#define FSF_AZ_TX_KER_BYTE_COUNT_LBN 48 +#define FSF_AZ_TX_KER_BYTE_COUNT_WIDTH 14 +#define FSF_AZ_TX_KER_BUF_REGION_LBN 46 +#define FSF_AZ_TX_KER_BUF_REGION_WIDTH 2 +#define FSF_AZ_TX_KER_BUF_ADDR_LBN 0 +#define FSF_AZ_TX_KER_BUF_ADDR_WIDTH 46 +#define FSF_AZ_TX_KER_BUF_ADDR_DW0_LBN 0 +#define FSF_AZ_TX_KER_BUF_ADDR_DW0_WIDTH 32 +#define FSF_AZ_TX_KER_BUF_ADDR_DW1_LBN 32 +#define FSF_AZ_TX_KER_BUF_ADDR_DW1_WIDTH 14 + + +/* FS_TX_USER_DESC */ +#define FSF_AZ_TX_USER_SW_EV_EN_LBN 48 +#define FSF_AZ_TX_USER_SW_EV_EN_WIDTH 1 +#define FSF_AZ_TX_USER_CONT_LBN 46 +#define FSF_AZ_TX_USER_CONT_WIDTH 1 +#define FSF_AZ_TX_USER_BYTE_CNT_LBN 33 +#define FSF_AZ_TX_USER_BYTE_CNT_WIDTH 13 +#define FSF_AZ_TX_USER_BUF_ID_LBN 13 +#define FSF_AZ_TX_USER_BUF_ID_WIDTH 20 +#define FSF_AZ_TX_USER_BYTE_OFS_LBN 0 +#define FSF_AZ_TX_USER_BYTE_OFS_WIDTH 13 + + +/* FS_USER_EV */ +#define FSF_CZ_USER_QID_LBN 32 +#define FSF_CZ_USER_QID_WIDTH 10 +#define FSF_CZ_USER_EV_REG_VALUE_LBN 0 +#define FSF_CZ_USER_EV_REG_VALUE_WIDTH 32 + + +/* FS_NET_IVEC */ +#define FSF_AZ_NET_IVEC_FATAL_INT_LBN 64 +#define FSF_AZ_NET_IVEC_FATAL_INT_WIDTH 1 +#define FSF_AZ_NET_IVEC_INT_Q_LBN 40 +#define FSF_AZ_NET_IVEC_INT_Q_WIDTH 4 +#define FSF_AZ_NET_IVEC_INT_FLAG_LBN 32 +#define FSF_AZ_NET_IVEC_INT_FLAG_WIDTH 1 +#define FSF_AZ_NET_IVEC_EVQ_FIFO_HF_LBN 1 +#define FSF_AZ_NET_IVEC_EVQ_FIFO_HF_WIDTH 1 +#define FSF_AZ_NET_IVEC_EVQ_FIFO_AF_LBN 0 +#define FSF_AZ_NET_IVEC_EVQ_FIFO_AF_WIDTH 1 + + +/* DRIVER_EV */ +/* Sub-fields of an RX flush completion event */ +#define FSF_AZ_DRIVER_EV_RX_FLUSH_FAIL_LBN 12 +#define FSF_AZ_DRIVER_EV_RX_FLUSH_FAIL_WIDTH 1 +#define FSF_AZ_DRIVER_EV_RX_DESCQ_ID_LBN 0 +#define FSF_AZ_DRIVER_EV_RX_DESCQ_ID_WIDTH 12 + + + +/************************************************************************** + * + * Falcon non-volatile configuration + * + ************************************************************************** + */ + + +#define FR_AZ_TX_PACE_TBL_OFST FR_BZ_TX_PACE_TBL_OFST + + +#ifdef __cplusplus +} +#endif + + + + +#endif /* _SYS_EFX_REGS_H */ diff --git a/dpdk/drivers/net/sfc/base/efx_regs_ef10.h b/dpdk/drivers/net/sfc/base/efx_regs_ef10.h new file mode 100644 index 00000000..11a91848 --- /dev/null +++ b/dpdk/drivers/net/sfc/base/efx_regs_ef10.h @@ -0,0 +1,571 @@ +/* + * Copyright (c) 2007-2016 Solarflare Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are + * those of the authors and should not be interpreted as representing official + * policies, either expressed or implied, of the FreeBSD Project. + */ + +#ifndef _SYS_EFX_EF10_REGS_H +#define _SYS_EFX_EF10_REGS_H + +#ifdef __cplusplus +extern "C" { +#endif + +/************************************************************************** + * NOTE: the line below marks the start of the autogenerated section + * EF10 registers and descriptors + * + ************************************************************************** + */ + +/* + * BIU_HW_REV_ID_REG(32bit): + * + */ + +#define ER_DZ_BIU_HW_REV_ID_REG_OFST 0x00000000 +/* hunta0,medforda0=pcie_pf_bar2 */ +#define ER_DZ_BIU_HW_REV_ID_REG_RESET 0xeb14face + + +#define ERF_DZ_HW_REV_ID_LBN 0 +#define ERF_DZ_HW_REV_ID_WIDTH 32 + + +/* + * BIU_MC_SFT_STATUS_REG(32bit): + * + */ + +#define ER_DZ_BIU_MC_SFT_STATUS_REG_OFST 0x00000010 +/* hunta0,medforda0=pcie_pf_bar2 */ +#define ER_DZ_BIU_MC_SFT_STATUS_REG_STEP 4 +#define ER_DZ_BIU_MC_SFT_STATUS_REG_ROWS 8 +#define ER_DZ_BIU_MC_SFT_STATUS_REG_RESET 0x1111face + + +#define ERF_DZ_MC_SFT_STATUS_LBN 0 +#define ERF_DZ_MC_SFT_STATUS_WIDTH 32 + + +/* + * BIU_INT_ISR_REG(32bit): + * + */ + +#define ER_DZ_BIU_INT_ISR_REG_OFST 0x00000090 +/* hunta0,medforda0=pcie_pf_bar2 */ +#define ER_DZ_BIU_INT_ISR_REG_RESET 0x0 + + +#define ERF_DZ_ISR_REG_LBN 0 +#define ERF_DZ_ISR_REG_WIDTH 32 + + +/* + * MC_DB_LWRD_REG(32bit): + * + */ + +#define ER_DZ_MC_DB_LWRD_REG_OFST 0x00000200 +/* hunta0,medforda0=pcie_pf_bar2 */ +#define ER_DZ_MC_DB_LWRD_REG_RESET 0x0 + + +#define ERF_DZ_MC_DOORBELL_L_LBN 0 +#define ERF_DZ_MC_DOORBELL_L_WIDTH 32 + + +/* + * MC_DB_HWRD_REG(32bit): + * + */ + +#define ER_DZ_MC_DB_HWRD_REG_OFST 0x00000204 +/* hunta0,medforda0=pcie_pf_bar2 */ +#define ER_DZ_MC_DB_HWRD_REG_RESET 0x0 + + +#define ERF_DZ_MC_DOORBELL_H_LBN 0 +#define ERF_DZ_MC_DOORBELL_H_WIDTH 32 + + +/* + * EVQ_RPTR_REG(32bit): + * + */ + +#define ER_DZ_EVQ_RPTR_REG_OFST 0x00000400 +/* hunta0,medforda0=pcie_pf_bar2 */ +#define ER_DZ_EVQ_RPTR_REG_STEP 8192 +#define ER_DZ_EVQ_RPTR_REG_ROWS 2048 +#define ER_DZ_EVQ_RPTR_REG_RESET 0x0 + + +#define ERF_DZ_EVQ_RPTR_VLD_LBN 15 +#define ERF_DZ_EVQ_RPTR_VLD_WIDTH 1 +#define ERF_DZ_EVQ_RPTR_LBN 0 +#define ERF_DZ_EVQ_RPTR_WIDTH 15 + + +/* + * EVQ_TMR_REG(32bit): + * + */ + +#define ER_DZ_EVQ_TMR_REG_OFST 0x00000420 +/* hunta0,medforda0=pcie_pf_bar2 */ +#define ER_DZ_EVQ_TMR_REG_STEP 8192 +#define ER_DZ_EVQ_TMR_REG_ROWS 2048 +#define ER_DZ_EVQ_TMR_REG_RESET 0x0 + + +#define ERF_DZ_TC_TIMER_MODE_LBN 14 +#define ERF_DZ_TC_TIMER_MODE_WIDTH 2 +#define ERF_DZ_TC_TIMER_VAL_LBN 0 +#define ERF_DZ_TC_TIMER_VAL_WIDTH 14 + + +/* + * RX_DESC_UPD_REG(32bit): + * + */ + +#define ER_DZ_RX_DESC_UPD_REG_OFST 0x00000830 +/* hunta0,medforda0=pcie_pf_bar2 */ +#define ER_DZ_RX_DESC_UPD_REG_STEP 8192 +#define ER_DZ_RX_DESC_UPD_REG_ROWS 2048 +#define ER_DZ_RX_DESC_UPD_REG_RESET 0x0 + + +#define ERF_DZ_RX_DESC_WPTR_LBN 0 +#define ERF_DZ_RX_DESC_WPTR_WIDTH 12 + +/* + * TX_DESC_UPD_REG(96bit): + * + */ + +#define ER_DZ_TX_DESC_UPD_REG_OFST 0x00000a10 +/* hunta0,medforda0=pcie_pf_bar2 */ +#define ER_DZ_TX_DESC_UPD_REG_STEP 8192 +#define ER_DZ_TX_DESC_UPD_REG_ROWS 2048 +#define ER_DZ_TX_DESC_UPD_REG_RESET 0x0 + + +#define ERF_DZ_RSVD_LBN 76 +#define ERF_DZ_RSVD_WIDTH 20 +#define ERF_DZ_TX_DESC_WPTR_LBN 64 +#define ERF_DZ_TX_DESC_WPTR_WIDTH 12 +#define ERF_DZ_TX_DESC_HWORD_LBN 32 +#define ERF_DZ_TX_DESC_HWORD_WIDTH 32 +#define ERF_DZ_TX_DESC_LWORD_LBN 0 +#define ERF_DZ_TX_DESC_LWORD_WIDTH 32 + + +/* ES_DRIVER_EV */ +#define ESF_DZ_DRV_CODE_LBN 60 +#define ESF_DZ_DRV_CODE_WIDTH 4 +#define ESF_DZ_DRV_SUB_CODE_LBN 56 +#define ESF_DZ_DRV_SUB_CODE_WIDTH 4 +#define ESE_DZ_DRV_TIMER_EV 3 +#define ESE_DZ_DRV_START_UP_EV 2 +#define ESE_DZ_DRV_WAKE_UP_EV 1 +#define ESF_DZ_DRV_SUB_DATA_DW0_LBN 0 +#define ESF_DZ_DRV_SUB_DATA_DW0_WIDTH 32 +#define ESF_DZ_DRV_SUB_DATA_DW1_LBN 32 +#define ESF_DZ_DRV_SUB_DATA_DW1_WIDTH 24 +#define ESF_DZ_DRV_SUB_DATA_LBN 0 +#define ESF_DZ_DRV_SUB_DATA_WIDTH 56 +#define ESF_DZ_DRV_EVQ_ID_LBN 0 +#define ESF_DZ_DRV_EVQ_ID_WIDTH 14 +#define ESF_DZ_DRV_TMR_ID_LBN 0 +#define ESF_DZ_DRV_TMR_ID_WIDTH 14 + + +/* ES_EVENT_ENTRY */ +#define ESF_DZ_EV_CODE_LBN 60 +#define ESF_DZ_EV_CODE_WIDTH 4 +#define ESE_DZ_EV_CODE_MCDI_EV 12 +#define ESE_DZ_EV_CODE_DRIVER_EV 5 +#define ESE_DZ_EV_CODE_TX_EV 2 +#define ESE_DZ_EV_CODE_RX_EV 0 +#define ESE_DZ_OTHER other +#define ESF_DZ_EV_DATA_DW0_LBN 0 +#define ESF_DZ_EV_DATA_DW0_WIDTH 32 +#define ESF_DZ_EV_DATA_DW1_LBN 32 +#define ESF_DZ_EV_DATA_DW1_WIDTH 28 +#define ESF_DZ_EV_DATA_LBN 0 +#define ESF_DZ_EV_DATA_WIDTH 60 + + +/* ES_MC_EVENT */ +#define ESF_DZ_MC_CODE_LBN 60 +#define ESF_DZ_MC_CODE_WIDTH 4 +#define ESF_DZ_MC_OVERRIDE_HOLDOFF_LBN 59 +#define ESF_DZ_MC_OVERRIDE_HOLDOFF_WIDTH 1 +#define ESF_DZ_MC_DROP_EVENT_LBN 58 +#define ESF_DZ_MC_DROP_EVENT_WIDTH 1 +#define ESF_DZ_MC_SOFT_DW0_LBN 0 +#define ESF_DZ_MC_SOFT_DW0_WIDTH 32 +#define ESF_DZ_MC_SOFT_DW1_LBN 32 +#define ESF_DZ_MC_SOFT_DW1_WIDTH 26 +#define ESF_DZ_MC_SOFT_LBN 0 +#define ESF_DZ_MC_SOFT_WIDTH 58 + + +/* ES_RX_EVENT */ +#define ESF_DZ_RX_CODE_LBN 60 +#define ESF_DZ_RX_CODE_WIDTH 4 +#define ESF_DZ_RX_OVERRIDE_HOLDOFF_LBN 59 +#define ESF_DZ_RX_OVERRIDE_HOLDOFF_WIDTH 1 +#define ESF_DZ_RX_DROP_EVENT_LBN 58 +#define ESF_DZ_RX_DROP_EVENT_WIDTH 1 +#define ESF_DD_RX_EV_RSVD2_LBN 54 +#define ESF_DD_RX_EV_RSVD2_WIDTH 4 +#define ESF_EZ_RX_TCP_UDP_INNER_CHKSUM_ERR_LBN 57 +#define ESF_EZ_RX_TCP_UDP_INNER_CHKSUM_ERR_WIDTH 1 +#define ESF_EZ_RX_IP_INNER_CHKSUM_ERR_LBN 56 +#define ESF_EZ_RX_IP_INNER_CHKSUM_ERR_WIDTH 1 +#define ESF_EZ_RX_EV_RSVD2_LBN 54 +#define ESF_EZ_RX_EV_RSVD2_WIDTH 2 +#define ESF_DZ_RX_EV_SOFT2_LBN 52 +#define ESF_DZ_RX_EV_SOFT2_WIDTH 2 +#define ESF_DZ_RX_DSC_PTR_LBITS_LBN 48 +#define ESF_DZ_RX_DSC_PTR_LBITS_WIDTH 4 +#define ESF_DZ_RX_L4_CLASS_LBN 45 +#define ESF_DZ_RX_L4_CLASS_WIDTH 3 +#define ESE_DZ_L4_CLASS_RSVD7 7 +#define ESE_DZ_L4_CLASS_RSVD6 6 +#define ESE_DZ_L4_CLASS_RSVD5 5 +#define ESE_DZ_L4_CLASS_RSVD4 4 +#define ESE_DZ_L4_CLASS_RSVD3 3 +#define ESE_DZ_L4_CLASS_UDP 2 +#define ESE_DZ_L4_CLASS_TCP 1 +#define ESE_DZ_L4_CLASS_UNKNOWN 0 +#define ESF_DZ_RX_L3_CLASS_LBN 42 +#define ESF_DZ_RX_L3_CLASS_WIDTH 3 +#define ESE_DZ_L3_CLASS_RSVD7 7 +#define ESE_DZ_L3_CLASS_IP6_FRAG 6 +#define ESE_DZ_L3_CLASS_ARP 5 +#define ESE_DZ_L3_CLASS_IP4_FRAG 4 +#define ESE_DZ_L3_CLASS_FCOE 3 +#define ESE_DZ_L3_CLASS_IP6 2 +#define ESE_DZ_L3_CLASS_IP4 1 +#define ESE_DZ_L3_CLASS_UNKNOWN 0 +#define ESF_DZ_RX_ETH_TAG_CLASS_LBN 39 +#define ESF_DZ_RX_ETH_TAG_CLASS_WIDTH 3 +#define ESE_DZ_ETH_TAG_CLASS_RSVD7 7 +#define ESE_DZ_ETH_TAG_CLASS_RSVD6 6 +#define ESE_DZ_ETH_TAG_CLASS_RSVD5 5 +#define ESE_DZ_ETH_TAG_CLASS_RSVD4 4 +#define ESE_DZ_ETH_TAG_CLASS_RSVD3 3 +#define ESE_DZ_ETH_TAG_CLASS_VLAN2 2 +#define ESE_DZ_ETH_TAG_CLASS_VLAN1 1 +#define ESE_DZ_ETH_TAG_CLASS_NONE 0 +#define ESF_DZ_RX_ETH_BASE_CLASS_LBN 36 +#define ESF_DZ_RX_ETH_BASE_CLASS_WIDTH 3 +#define ESE_DZ_ETH_BASE_CLASS_LLC_SNAP 2 +#define ESE_DZ_ETH_BASE_CLASS_LLC 1 +#define ESE_DZ_ETH_BASE_CLASS_ETH2 0 +#define ESF_DZ_RX_MAC_CLASS_LBN 35 +#define ESF_DZ_RX_MAC_CLASS_WIDTH 1 +#define ESE_DZ_MAC_CLASS_MCAST 1 +#define ESE_DZ_MAC_CLASS_UCAST 0 +#define ESF_DD_RX_EV_SOFT1_LBN 32 +#define ESF_DD_RX_EV_SOFT1_WIDTH 3 +#define ESF_EZ_RX_EV_SOFT1_LBN 34 +#define ESF_EZ_RX_EV_SOFT1_WIDTH 1 +#define ESF_EZ_RX_ENCAP_HDR_LBN 32 +#define ESF_EZ_RX_ENCAP_HDR_WIDTH 2 +#define ESE_EZ_ENCAP_HDR_GRE 2 +#define ESE_EZ_ENCAP_HDR_VXLAN 1 +#define ESE_EZ_ENCAP_HDR_NONE 0 +#define ESF_DD_RX_EV_RSVD1_LBN 30 +#define ESF_DD_RX_EV_RSVD1_WIDTH 2 +#define ESF_EZ_RX_EV_RSVD1_LBN 31 +#define ESF_EZ_RX_EV_RSVD1_WIDTH 1 +#define ESF_EZ_RX_ABORT_LBN 30 +#define ESF_EZ_RX_ABORT_WIDTH 1 +#define ESF_DZ_RX_ECC_ERR_LBN 29 +#define ESF_DZ_RX_ECC_ERR_WIDTH 1 +#define ESF_DZ_RX_CRC1_ERR_LBN 28 +#define ESF_DZ_RX_CRC1_ERR_WIDTH 1 +#define ESF_DZ_RX_CRC0_ERR_LBN 27 +#define ESF_DZ_RX_CRC0_ERR_WIDTH 1 +#define ESF_DZ_RX_TCPUDP_CKSUM_ERR_LBN 26 +#define ESF_DZ_RX_TCPUDP_CKSUM_ERR_WIDTH 1 +#define ESF_DZ_RX_IPCKSUM_ERR_LBN 25 +#define ESF_DZ_RX_IPCKSUM_ERR_WIDTH 1 +#define ESF_DZ_RX_ECRC_ERR_LBN 24 +#define ESF_DZ_RX_ECRC_ERR_WIDTH 1 +#define ESF_DZ_RX_QLABEL_LBN 16 +#define ESF_DZ_RX_QLABEL_WIDTH 5 +#define ESF_DZ_RX_PARSE_INCOMPLETE_LBN 15 +#define ESF_DZ_RX_PARSE_INCOMPLETE_WIDTH 1 +#define ESF_DZ_RX_CONT_LBN 14 +#define ESF_DZ_RX_CONT_WIDTH 1 +#define ESF_DZ_RX_BYTES_LBN 0 +#define ESF_DZ_RX_BYTES_WIDTH 14 + + +/* ES_RX_KER_DESC */ +#define ESF_DZ_RX_KER_RESERVED_LBN 62 +#define ESF_DZ_RX_KER_RESERVED_WIDTH 2 +#define ESF_DZ_RX_KER_BYTE_CNT_LBN 48 +#define ESF_DZ_RX_KER_BYTE_CNT_WIDTH 14 +#define ESF_DZ_RX_KER_BUF_ADDR_DW0_LBN 0 +#define ESF_DZ_RX_KER_BUF_ADDR_DW0_WIDTH 32 +#define ESF_DZ_RX_KER_BUF_ADDR_DW1_LBN 32 +#define ESF_DZ_RX_KER_BUF_ADDR_DW1_WIDTH 16 +#define ESF_DZ_RX_KER_BUF_ADDR_LBN 0 +#define ESF_DZ_RX_KER_BUF_ADDR_WIDTH 48 + + +/* ES_TX_CSUM_TSTAMP_DESC */ +#define ESF_DZ_TX_DESC_IS_OPT_LBN 63 +#define ESF_DZ_TX_DESC_IS_OPT_WIDTH 1 +#define ESF_DZ_TX_OPTION_TYPE_LBN 60 +#define ESF_DZ_TX_OPTION_TYPE_WIDTH 3 +#define ESE_DZ_TX_OPTION_DESC_TSO 7 +#define ESE_DZ_TX_OPTION_DESC_VLAN 6 +#define ESE_DZ_TX_OPTION_DESC_CRC_CSUM 0 +#define ESF_DZ_TX_OPTION_TS_AT_TXDP_LBN 8 +#define ESF_DZ_TX_OPTION_TS_AT_TXDP_WIDTH 1 +#define ESF_DZ_TX_OPTION_INNER_UDP_TCP_CSUM_LBN 7 +#define ESF_DZ_TX_OPTION_INNER_UDP_TCP_CSUM_WIDTH 1 +#define ESF_DZ_TX_OPTION_INNER_IP_CSUM_LBN 6 +#define ESF_DZ_TX_OPTION_INNER_IP_CSUM_WIDTH 1 +#define ESF_DZ_TX_TIMESTAMP_LBN 5 +#define ESF_DZ_TX_TIMESTAMP_WIDTH 1 +#define ESF_DZ_TX_OPTION_CRC_MODE_LBN 2 +#define ESF_DZ_TX_OPTION_CRC_MODE_WIDTH 3 +#define ESE_DZ_TX_OPTION_CRC_FCOIP_MPA 5 +#define ESE_DZ_TX_OPTION_CRC_FCOIP_FCOE 4 +#define ESE_DZ_TX_OPTION_CRC_ISCSI_HDR_AND_PYLD 3 +#define ESE_DZ_TX_OPTION_CRC_ISCSI_HDR 2 +#define ESE_DZ_TX_OPTION_CRC_FCOE 1 +#define ESE_DZ_TX_OPTION_CRC_OFF 0 +#define ESF_DZ_TX_OPTION_UDP_TCP_CSUM_LBN 1 +#define ESF_DZ_TX_OPTION_UDP_TCP_CSUM_WIDTH 1 +#define ESF_DZ_TX_OPTION_IP_CSUM_LBN 0 +#define ESF_DZ_TX_OPTION_IP_CSUM_WIDTH 1 + + +/* ES_TX_EVENT */ +#define ESF_DZ_TX_CODE_LBN 60 +#define ESF_DZ_TX_CODE_WIDTH 4 +#define ESF_DZ_TX_OVERRIDE_HOLDOFF_LBN 59 +#define ESF_DZ_TX_OVERRIDE_HOLDOFF_WIDTH 1 +#define ESF_DZ_TX_DROP_EVENT_LBN 58 +#define ESF_DZ_TX_DROP_EVENT_WIDTH 1 +#define ESF_DD_TX_EV_RSVD_LBN 48 +#define ESF_DD_TX_EV_RSVD_WIDTH 10 +#define ESF_EZ_TCP_UDP_INNER_CHKSUM_ERR_LBN 57 +#define ESF_EZ_TCP_UDP_INNER_CHKSUM_ERR_WIDTH 1 +#define ESF_EZ_IP_INNER_CHKSUM_ERR_LBN 56 +#define ESF_EZ_IP_INNER_CHKSUM_ERR_WIDTH 1 +#define ESF_EZ_TX_EV_RSVD_LBN 48 +#define ESF_EZ_TX_EV_RSVD_WIDTH 8 +#define ESF_DZ_TX_SOFT2_LBN 32 +#define ESF_DZ_TX_SOFT2_WIDTH 16 +#define ESF_DD_TX_SOFT1_LBN 24 +#define ESF_DD_TX_SOFT1_WIDTH 8 +#define ESF_EZ_TX_CAN_MERGE_LBN 31 +#define ESF_EZ_TX_CAN_MERGE_WIDTH 1 +#define ESF_EZ_TX_SOFT1_LBN 24 +#define ESF_EZ_TX_SOFT1_WIDTH 7 +#define ESF_DZ_TX_QLABEL_LBN 16 +#define ESF_DZ_TX_QLABEL_WIDTH 5 +#define ESF_DZ_TX_DESCR_INDX_LBN 0 +#define ESF_DZ_TX_DESCR_INDX_WIDTH 16 + + +/* ES_TX_KER_DESC */ +#define ESF_DZ_TX_KER_TYPE_LBN 63 +#define ESF_DZ_TX_KER_TYPE_WIDTH 1 +#define ESF_DZ_TX_KER_CONT_LBN 62 +#define ESF_DZ_TX_KER_CONT_WIDTH 1 +#define ESF_DZ_TX_KER_BYTE_CNT_LBN 48 +#define ESF_DZ_TX_KER_BYTE_CNT_WIDTH 14 +#define ESF_DZ_TX_KER_BUF_ADDR_DW0_LBN 0 +#define ESF_DZ_TX_KER_BUF_ADDR_DW0_WIDTH 32 +#define ESF_DZ_TX_KER_BUF_ADDR_DW1_LBN 32 +#define ESF_DZ_TX_KER_BUF_ADDR_DW1_WIDTH 16 +#define ESF_DZ_TX_KER_BUF_ADDR_LBN 0 +#define ESF_DZ_TX_KER_BUF_ADDR_WIDTH 48 + + +/* ES_TX_PIO_DESC */ +#define ESF_DZ_TX_PIO_TYPE_LBN 63 +#define ESF_DZ_TX_PIO_TYPE_WIDTH 1 +#define ESF_DZ_TX_PIO_OPT_LBN 60 +#define ESF_DZ_TX_PIO_OPT_WIDTH 3 +#define ESF_DZ_TX_PIO_CONT_LBN 59 +#define ESF_DZ_TX_PIO_CONT_WIDTH 1 +#define ESF_DZ_TX_PIO_BYTE_CNT_LBN 32 +#define ESF_DZ_TX_PIO_BYTE_CNT_WIDTH 12 +#define ESF_DZ_TX_PIO_BUF_ADDR_LBN 0 +#define ESF_DZ_TX_PIO_BUF_ADDR_WIDTH 12 + + +/* ES_TX_TSO_DESC */ +#define ESF_DZ_TX_DESC_IS_OPT_LBN 63 +#define ESF_DZ_TX_DESC_IS_OPT_WIDTH 1 +#define ESF_DZ_TX_OPTION_TYPE_LBN 60 +#define ESF_DZ_TX_OPTION_TYPE_WIDTH 3 +#define ESE_DZ_TX_OPTION_DESC_TSO 7 +#define ESE_DZ_TX_OPTION_DESC_VLAN 6 +#define ESE_DZ_TX_OPTION_DESC_CRC_CSUM 0 +#define ESF_DZ_TX_TSO_OPTION_TYPE_LBN 56 +#define ESF_DZ_TX_TSO_OPTION_TYPE_WIDTH 4 +#define ESE_DZ_TX_TSO_OPTION_DESC_ENCAP 1 +#define ESE_DZ_TX_TSO_OPTION_DESC_NORMAL 0 +#define ESF_DZ_TX_TSO_TCP_FLAGS_LBN 48 +#define ESF_DZ_TX_TSO_TCP_FLAGS_WIDTH 8 +#define ESF_DZ_TX_TSO_IP_ID_LBN 32 +#define ESF_DZ_TX_TSO_IP_ID_WIDTH 16 +#define ESF_DZ_TX_TSO_TCP_SEQNO_LBN 0 +#define ESF_DZ_TX_TSO_TCP_SEQNO_WIDTH 32 + + +/* TX_TSO_FATSO2A_DESC */ +#define ESF_DZ_TX_DESC_IS_OPT_LBN 63 +#define ESF_DZ_TX_DESC_IS_OPT_WIDTH 1 +#define ESF_DZ_TX_OPTION_TYPE_LBN 60 +#define ESF_DZ_TX_OPTION_TYPE_WIDTH 3 +#define ESE_DZ_TX_OPTION_DESC_TSO 7 +#define ESE_DZ_TX_OPTION_DESC_VLAN 6 +#define ESE_DZ_TX_OPTION_DESC_CRC_CSUM 0 +#define ESF_DZ_TX_TSO_OPTION_TYPE_LBN 56 +#define ESF_DZ_TX_TSO_OPTION_TYPE_WIDTH 4 +#define ESE_DZ_TX_TSO_OPTION_DESC_FATSO2B 3 +#define ESE_DZ_TX_TSO_OPTION_DESC_FATSO2A 2 +#define ESE_DZ_TX_TSO_OPTION_DESC_ENCAP 1 +#define ESE_DZ_TX_TSO_OPTION_DESC_NORMAL 0 +#define ESF_DZ_TX_TSO_IP_ID_LBN 32 +#define ESF_DZ_TX_TSO_IP_ID_WIDTH 16 +#define ESF_DZ_TX_TSO_TCP_SEQNO_LBN 0 +#define ESF_DZ_TX_TSO_TCP_SEQNO_WIDTH 32 + + +/* TX_TSO_FATSO2B_DESC */ +#define ESF_DZ_TX_DESC_IS_OPT_LBN 63 +#define ESF_DZ_TX_DESC_IS_OPT_WIDTH 1 +#define ESF_DZ_TX_OPTION_TYPE_LBN 60 +#define ESF_DZ_TX_OPTION_TYPE_WIDTH 3 +#define ESE_DZ_TX_OPTION_DESC_TSO 7 +#define ESE_DZ_TX_OPTION_DESC_VLAN 6 +#define ESE_DZ_TX_OPTION_DESC_CRC_CSUM 0 +#define ESF_DZ_TX_TSO_OPTION_TYPE_LBN 56 +#define ESF_DZ_TX_TSO_OPTION_TYPE_WIDTH 4 +#define ESE_DZ_TX_TSO_OPTION_DESC_FATSO2B 3 +#define ESE_DZ_TX_TSO_OPTION_DESC_FATSO2A 2 +#define ESE_DZ_TX_TSO_OPTION_DESC_ENCAP 1 +#define ESE_DZ_TX_TSO_OPTION_DESC_NORMAL 0 +#define ESF_DZ_TX_TSO_OUTER_IP_ID_LBN 16 +#define ESF_DZ_TX_TSO_OUTER_IP_ID_WIDTH 16 +#define ESF_DZ_TX_TSO_TCP_MSS_LBN 32 +#define ESF_DZ_TX_TSO_TCP_MSS_WIDTH 16 +#define ESF_DZ_TX_TSO_INNER_PE_CSUM_LBN 0 +#define ESF_DZ_TX_TSO_INNER_PE_CSUM_WIDTH 16 + + +/* ES_TX_VLAN_DESC */ +#define ESF_DZ_TX_DESC_IS_OPT_LBN 63 +#define ESF_DZ_TX_DESC_IS_OPT_WIDTH 1 +#define ESF_DZ_TX_OPTION_TYPE_LBN 60 +#define ESF_DZ_TX_OPTION_TYPE_WIDTH 3 +#define ESE_DZ_TX_OPTION_DESC_TSO 7 +#define ESE_DZ_TX_OPTION_DESC_VLAN 6 +#define ESE_DZ_TX_OPTION_DESC_CRC_CSUM 0 +#define ESF_DZ_TX_VLAN_OP_LBN 32 +#define ESF_DZ_TX_VLAN_OP_WIDTH 2 +#define ESF_DZ_TX_VLAN_TAG2_LBN 16 +#define ESF_DZ_TX_VLAN_TAG2_WIDTH 16 +#define ESF_DZ_TX_VLAN_TAG1_LBN 0 +#define ESF_DZ_TX_VLAN_TAG1_WIDTH 16 + + +/************************************************************************* + * NOTE: the comment line above marks the end of the autogenerated section + */ + +/* + * The workaround for bug 35388 requires multiplexing writes through + * the ERF_DZ_TX_DESC_WPTR address. + * TX_DESC_UPD: 0ppppppppppp (bit 11 lost) + * EVQ_RPTR: 1000hhhhhhhh, 1001llllllll (split into high and low bits) + * EVQ_TMR: 11mmvvvvvvvv (bits 8:13 of value lost) + */ +#define ER_DD_EVQ_INDIRECT_OFST (ER_DZ_TX_DESC_UPD_REG_OFST + 2 * 4) +#define ER_DD_EVQ_INDIRECT_STEP ER_DZ_TX_DESC_UPD_REG_STEP +#define ERF_DD_EVQ_IND_RPTR_FLAGS_LBN 8 +#define ERF_DD_EVQ_IND_RPTR_FLAGS_WIDTH 4 +#define EFE_DD_EVQ_IND_RPTR_FLAGS_HIGH 8 +#define EFE_DD_EVQ_IND_RPTR_FLAGS_LOW 9 +#define ERF_DD_EVQ_IND_RPTR_LBN 0 +#define ERF_DD_EVQ_IND_RPTR_WIDTH 8 +#define ERF_DD_EVQ_IND_TIMER_FLAGS_LBN 10 +#define ERF_DD_EVQ_IND_TIMER_FLAGS_WIDTH 2 +#define EFE_DD_EVQ_IND_TIMER_FLAGS 3 +#define ERF_DD_EVQ_IND_TIMER_MODE_LBN 8 +#define ERF_DD_EVQ_IND_TIMER_MODE_WIDTH 2 +#define ERF_DD_EVQ_IND_TIMER_VAL_LBN 0 +#define ERF_DD_EVQ_IND_TIMER_VAL_WIDTH 8 + +/* Packed stream magic doorbell command */ +#define ERF_DZ_RX_DESC_MAGIC_DOORBELL_LBN 11 +#define ERF_DZ_RX_DESC_MAGIC_DOORBELL_WIDTH 1 + +#define ERF_DZ_RX_DESC_MAGIC_CMD_LBN 8 +#define ERF_DZ_RX_DESC_MAGIC_CMD_WIDTH 3 +#define ERE_DZ_RX_DESC_MAGIC_CMD_PS_CREDITS 0 + +#define ERF_DZ_RX_DESC_MAGIC_DATA_LBN 0 +#define ERF_DZ_RX_DESC_MAGIC_DATA_WIDTH 8 + +/* Packed stream RX packet prefix */ +#define ES_DZ_PS_RX_PREFIX_TSTAMP_LBN 0 +#define ES_DZ_PS_RX_PREFIX_TSTAMP_WIDTH 32 +#define ES_DZ_PS_RX_PREFIX_CAP_LEN_LBN 32 +#define ES_DZ_PS_RX_PREFIX_CAP_LEN_WIDTH 16 +#define ES_DZ_PS_RX_PREFIX_ORIG_LEN_LBN 48 +#define ES_DZ_PS_RX_PREFIX_ORIG_LEN_WIDTH 16 + +/* + * An extra flag for the packed stream mode, + * signalling the start of a new buffer + */ +#define ESF_DZ_RX_EV_ROTATE_LBN 53 +#define ESF_DZ_RX_EV_ROTATE_WIDTH 1 + +#ifdef __cplusplus +} +#endif + +#endif /* _SYS_EFX_EF10_REGS_H */ diff --git a/dpdk/drivers/net/sfc/base/efx_regs_mcdi.h b/dpdk/drivers/net/sfc/base/efx_regs_mcdi.h new file mode 100644 index 00000000..66896fbb --- /dev/null +++ b/dpdk/drivers/net/sfc/base/efx_regs_mcdi.h @@ -0,0 +1,15690 @@ +/*- + * Copyright 2008-2013 Solarflare Communications Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/*! \cidoxg_firmware_mc_cmd */ + +#ifndef _SIENA_MC_DRIVER_PCOL_H +#define _SIENA_MC_DRIVER_PCOL_H + + +/* Values to be written into FMCR_CZ_RESET_STATE_REG to control boot. */ +/* Power-on reset state */ +#define MC_FW_STATE_POR (1) +/* If this is set in MC_RESET_STATE_REG then it should be + * possible to jump into IMEM without loading code from flash. */ +#define MC_FW_WARM_BOOT_OK (2) +/* The MC main image has started to boot. */ +#define MC_FW_STATE_BOOTING (4) +/* The Scheduler has started. */ +#define MC_FW_STATE_SCHED (8) +/* If this is set in MC_RESET_STATE_REG then it should be + * possible to jump into IMEM without loading code from flash. + * Unlike a warm boot, assume DMEM has been reloaded, so that + * the MC persistent data must be reinitialised. */ +#define MC_FW_TEPID_BOOT_OK (16) +/* We have entered the main firmware via recovery mode. This + * means that MC persistent data must be reinitialised, but that + * we shouldn't touch PCIe config. */ +#define MC_FW_RECOVERY_MODE_PCIE_INIT_OK (32) +/* BIST state has been initialized */ +#define MC_FW_BIST_INIT_OK (128) + +/* Siena MC shared memmory offsets */ +/* The 'doorbell' addresses are hard-wired to alert the MC when written */ +#define MC_SMEM_P0_DOORBELL_OFST 0x000 +#define MC_SMEM_P1_DOORBELL_OFST 0x004 +/* The rest of these are firmware-defined */ +#define MC_SMEM_P0_PDU_OFST 0x008 +#define MC_SMEM_P1_PDU_OFST 0x108 +#define MC_SMEM_PDU_LEN 0x100 +#define MC_SMEM_P0_PTP_TIME_OFST 0x7f0 +#define MC_SMEM_P0_STATUS_OFST 0x7f8 +#define MC_SMEM_P1_STATUS_OFST 0x7fc + +/* Values to be written to the per-port status dword in shared + * memory on reboot and assert */ +#define MC_STATUS_DWORD_REBOOT (0xb007b007) +#define MC_STATUS_DWORD_ASSERT (0xdeaddead) + +/* Check whether an mcfw version (in host order) belongs to a bootloader */ +#define MC_FW_VERSION_IS_BOOTLOADER(_v) (((_v) >> 16) == 0xb007) + +/* The current version of the MCDI protocol. + * + * Note that the ROM burnt into the card only talks V0, so at the very + * least every driver must support version 0 and MCDI_PCOL_VERSION + */ +#ifdef WITH_MCDI_V2 +#define MCDI_PCOL_VERSION 2 +#else +#define MCDI_PCOL_VERSION 1 +#endif + +/* Unused commands: 0x23, 0x27, 0x30, 0x31 */ + +/* MCDI version 1 + * + * Each MCDI request starts with an MCDI_HEADER, which is a 32bit + * structure, filled in by the client. + * + * 0 7 8 16 20 22 23 24 31 + * | CODE | R | LEN | SEQ | Rsvd | E | R | XFLAGS | + * | | | + * | | \--- Response + * | \------- Error + * \------------------------------ Resync (always set) + * + * The client writes it's request into MC shared memory, and rings the + * doorbell. Each request is completed by either by the MC writting + * back into shared memory, or by writting out an event. + * + * All MCDI commands support completion by shared memory response. Each + * request may also contain additional data (accounted for by HEADER.LEN), + * and some response's may also contain additional data (again, accounted + * for by HEADER.LEN). + * + * Some MCDI commands support completion by event, in which any associated + * response data is included in the event. + * + * The protocol requires one response to be delivered for every request, a + * request should not be sent unless the response for the previous request + * has been received (either by polling shared memory, or by receiving + * an event). + */ + +/** Request/Response structure */ +#define MCDI_HEADER_OFST 0 +#define MCDI_HEADER_CODE_LBN 0 +#define MCDI_HEADER_CODE_WIDTH 7 +#define MCDI_HEADER_RESYNC_LBN 7 +#define MCDI_HEADER_RESYNC_WIDTH 1 +#define MCDI_HEADER_DATALEN_LBN 8 +#define MCDI_HEADER_DATALEN_WIDTH 8 +#define MCDI_HEADER_SEQ_LBN 16 +#define MCDI_HEADER_SEQ_WIDTH 4 +#define MCDI_HEADER_RSVD_LBN 20 +#define MCDI_HEADER_RSVD_WIDTH 1 +#define MCDI_HEADER_NOT_EPOCH_LBN 21 +#define MCDI_HEADER_NOT_EPOCH_WIDTH 1 +#define MCDI_HEADER_ERROR_LBN 22 +#define MCDI_HEADER_ERROR_WIDTH 1 +#define MCDI_HEADER_RESPONSE_LBN 23 +#define MCDI_HEADER_RESPONSE_WIDTH 1 +#define MCDI_HEADER_XFLAGS_LBN 24 +#define MCDI_HEADER_XFLAGS_WIDTH 8 +/* Request response using event */ +#define MCDI_HEADER_XFLAGS_EVREQ 0x01 +/* Request (and signal) early doorbell return */ +#define MCDI_HEADER_XFLAGS_DBRET 0x02 + +/* Maximum number of payload bytes */ +#define MCDI_CTL_SDU_LEN_MAX_V1 0xfc +#define MCDI_CTL_SDU_LEN_MAX_V2 0x400 + +#ifdef WITH_MCDI_V2 +#define MCDI_CTL_SDU_LEN_MAX MCDI_CTL_SDU_LEN_MAX_V2 +#else +#define MCDI_CTL_SDU_LEN_MAX MCDI_CTL_SDU_LEN_MAX_V1 +#endif + + +/* The MC can generate events for two reasons: + * - To advance a shared memory request if XFLAGS_EVREQ was set + * - As a notification (link state, i2c event), controlled + * via MC_CMD_LOG_CTRL + * + * Both events share a common structure: + * + * 0 32 33 36 44 52 60 + * | Data | Cont | Level | Src | Code | Rsvd | + * | + * \ There is another event pending in this notification + * + * If Code==CMDDONE, then the fields are further interpreted as: + * + * - LEVEL==INFO Command succeeded + * - LEVEL==ERR Command failed + * + * 0 8 16 24 32 + * | Seq | Datalen | Errno | Rsvd | + * + * These fields are taken directly out of the standard MCDI header, i.e., + * LEVEL==ERR, Datalen == 0 => Reboot + * + * Events can be squirted out of the UART (using LOG_CTRL) without a + * MCDI header. An event can be distinguished from a MCDI response by + * examining the first byte which is 0xc0. This corresponds to the + * non-existent MCDI command MC_CMD_DEBUG_LOG. + * + * 0 7 8 + * | command | Resync | = 0xc0 + * + * Since the event is written in big-endian byte order, this works + * providing bits 56-63 of the event are 0xc0. + * + * 56 60 63 + * | Rsvd | Code | = 0xc0 + * + * Which means for convenience the event code is 0xc for all MC + * generated events. + */ +#define FSE_AZ_EV_CODE_MCDI_EVRESPONSE 0xc + + +/* Operation not permitted. */ +#define MC_CMD_ERR_EPERM 1 +/* Non-existent command target */ +#define MC_CMD_ERR_ENOENT 2 +/* assert() has killed the MC */ +#define MC_CMD_ERR_EINTR 4 +/* I/O failure */ +#define MC_CMD_ERR_EIO 5 +/* Already exists */ +#define MC_CMD_ERR_EEXIST 6 +/* Try again */ +#define MC_CMD_ERR_EAGAIN 11 +/* Out of memory */ +#define MC_CMD_ERR_ENOMEM 12 +/* Caller does not hold required locks */ +#define MC_CMD_ERR_EACCES 13 +/* Resource is currently unavailable (e.g. lock contention) */ +#define MC_CMD_ERR_EBUSY 16 +/* No such device */ +#define MC_CMD_ERR_ENODEV 19 +/* Invalid argument to target */ +#define MC_CMD_ERR_EINVAL 22 +/* Broken pipe */ +#define MC_CMD_ERR_EPIPE 32 +/* Read-only */ +#define MC_CMD_ERR_EROFS 30 +/* Out of range */ +#define MC_CMD_ERR_ERANGE 34 +/* Non-recursive resource is already acquired */ +#define MC_CMD_ERR_EDEADLK 35 +/* Operation not implemented */ +#define MC_CMD_ERR_ENOSYS 38 +/* Operation timed out */ +#define MC_CMD_ERR_ETIME 62 +/* Link has been severed */ +#define MC_CMD_ERR_ENOLINK 67 +/* Protocol error */ +#define MC_CMD_ERR_EPROTO 71 +/* Operation not supported */ +#define MC_CMD_ERR_ENOTSUP 95 +/* Address not available */ +#define MC_CMD_ERR_EADDRNOTAVAIL 99 +/* Not connected */ +#define MC_CMD_ERR_ENOTCONN 107 +/* Operation already in progress */ +#define MC_CMD_ERR_EALREADY 114 + +/* Resource allocation failed. */ +#define MC_CMD_ERR_ALLOC_FAIL 0x1000 +/* V-adaptor not found. */ +#define MC_CMD_ERR_NO_VADAPTOR 0x1001 +/* EVB port not found. */ +#define MC_CMD_ERR_NO_EVB_PORT 0x1002 +/* V-switch not found. */ +#define MC_CMD_ERR_NO_VSWITCH 0x1003 +/* Too many VLAN tags. */ +#define MC_CMD_ERR_VLAN_LIMIT 0x1004 +/* Bad PCI function number. */ +#define MC_CMD_ERR_BAD_PCI_FUNC 0x1005 +/* Invalid VLAN mode. */ +#define MC_CMD_ERR_BAD_VLAN_MODE 0x1006 +/* Invalid v-switch type. */ +#define MC_CMD_ERR_BAD_VSWITCH_TYPE 0x1007 +/* Invalid v-port type. */ +#define MC_CMD_ERR_BAD_VPORT_TYPE 0x1008 +/* MAC address exists. */ +#define MC_CMD_ERR_MAC_EXIST 0x1009 +/* Slave core not present */ +#define MC_CMD_ERR_SLAVE_NOT_PRESENT 0x100a +/* The datapath is disabled. */ +#define MC_CMD_ERR_DATAPATH_DISABLED 0x100b +/* The requesting client is not a function */ +#define MC_CMD_ERR_CLIENT_NOT_FN 0x100c +/* The requested operation might require the + command to be passed between MCs, and the + transport doesn't support that. Should + only ever been seen over the UART. */ +#define MC_CMD_ERR_TRANSPORT_NOPROXY 0x100d +/* VLAN tag(s) exists */ +#define MC_CMD_ERR_VLAN_EXIST 0x100e +/* No MAC address assigned to an EVB port */ +#define MC_CMD_ERR_NO_MAC_ADDR 0x100f +/* Notifies the driver that the request has been relayed + * to an admin function for authorization. The driver should + * wait for a PROXY_RESPONSE event and then resend its request. + * This error code is followed by a 32-bit handle that + * helps matching it with the respective PROXY_RESPONSE event. */ +#define MC_CMD_ERR_PROXY_PENDING 0x1010 +#define MC_CMD_ERR_PROXY_PENDING_HANDLE_OFST 4 +/* The request cannot be passed for authorization because + * another request from the same function is currently being + * authorized. The drvier should try again later. */ +#define MC_CMD_ERR_PROXY_INPROGRESS 0x1011 +/* Returned by MC_CMD_PROXY_COMPLETE if the caller is not the function + * that has enabled proxying or BLOCK_INDEX points to a function that + * doesn't await an authorization. */ +#define MC_CMD_ERR_PROXY_UNEXPECTED 0x1012 +/* This code is currently only used internally in FW. Its meaning is that + * an operation failed due to lack of SR-IOV privilege. + * Normally it is translated to EPERM by send_cmd_err(), + * but it may also be used to trigger some special mechanism + * for handling such case, e.g. to relay the failed request + * to a designated admin function for authorization. */ +#define MC_CMD_ERR_NO_PRIVILEGE 0x1013 +/* Workaround 26807 could not be turned on/off because some functions + * have already installed filters. See the comment at + * MC_CMD_WORKAROUND_BUG26807. */ +#define MC_CMD_ERR_FILTERS_PRESENT 0x1014 +/* The clock whose frequency you've attempted to set set + * doesn't exist on this NIC */ +#define MC_CMD_ERR_NO_CLOCK 0x1015 +/* Returned by MC_CMD_TESTASSERT if the action that should + * have caused an assertion failed to do so. */ +#define MC_CMD_ERR_UNREACHABLE 0x1016 +/* This command needs to be processed in the background but there were no + * resources to do so. Send it again after a command has completed. */ +#define MC_CMD_ERR_QUEUE_FULL 0x1017 + +#define MC_CMD_ERR_CODE_OFST 0 + +/* We define 8 "escape" commands to allow + for command number space extension */ + +#define MC_CMD_CMD_SPACE_ESCAPE_0 0x78 +#define MC_CMD_CMD_SPACE_ESCAPE_1 0x79 +#define MC_CMD_CMD_SPACE_ESCAPE_2 0x7A +#define MC_CMD_CMD_SPACE_ESCAPE_3 0x7B +#define MC_CMD_CMD_SPACE_ESCAPE_4 0x7C +#define MC_CMD_CMD_SPACE_ESCAPE_5 0x7D +#define MC_CMD_CMD_SPACE_ESCAPE_6 0x7E +#define MC_CMD_CMD_SPACE_ESCAPE_7 0x7F + +/* Vectors in the boot ROM */ +/* Point to the copycode entry point. */ +#define SIENA_MC_BOOTROM_COPYCODE_VEC (0x800 - 3 * 0x4) +#define HUNT_MC_BOOTROM_COPYCODE_VEC (0x8000 - 3 * 0x4) +#define MEDFORD_MC_BOOTROM_COPYCODE_VEC (0x10000 - 3 * 0x4) +/* Points to the recovery mode entry point. */ +#define SIENA_MC_BOOTROM_NOFLASH_VEC (0x800 - 2 * 0x4) +#define HUNT_MC_BOOTROM_NOFLASH_VEC (0x8000 - 2 * 0x4) +#define MEDFORD_MC_BOOTROM_NOFLASH_VEC (0x10000 - 2 * 0x4) + +/* The command set exported by the boot ROM (MCDI v0) */ +#define MC_CMD_GET_VERSION_V0_SUPPORTED_FUNCS { \ + (1 << MC_CMD_READ32) | \ + (1 << MC_CMD_WRITE32) | \ + (1 << MC_CMD_COPYCODE) | \ + (1 << MC_CMD_GET_VERSION), \ + 0, 0, 0 } + +#define MC_CMD_SENSOR_INFO_OUT_OFFSET_OFST(_x) \ + (MC_CMD_SENSOR_ENTRY_OFST + (_x)) + +#define MC_CMD_DBI_WRITE_IN_ADDRESS_OFST(n) \ + (MC_CMD_DBI_WRITE_IN_DBIWROP_OFST + \ + MC_CMD_DBIWROP_TYPEDEF_ADDRESS_OFST + \ + (n) * MC_CMD_DBIWROP_TYPEDEF_LEN) + +#define MC_CMD_DBI_WRITE_IN_BYTE_MASK_OFST(n) \ + (MC_CMD_DBI_WRITE_IN_DBIWROP_OFST + \ + MC_CMD_DBIWROP_TYPEDEF_BYTE_MASK_OFST + \ + (n) * MC_CMD_DBIWROP_TYPEDEF_LEN) + +#define MC_CMD_DBI_WRITE_IN_VALUE_OFST(n) \ + (MC_CMD_DBI_WRITE_IN_DBIWROP_OFST + \ + MC_CMD_DBIWROP_TYPEDEF_VALUE_OFST + \ + (n) * MC_CMD_DBIWROP_TYPEDEF_LEN) + +/* This may be ORed with an EVB_PORT_ID_xxx constant to pass a non-default + * stack ID (which must be in the range 1-255) along with an EVB port ID. + */ +#define EVB_STACK_ID(n) (((n) & 0xff) << 16) + + +#ifdef WITH_MCDI_V2 + +/* Version 2 adds an optional argument to error returns: the errno value + * may be followed by the (0-based) number of the first argument that + * could not be processed. + */ +#define MC_CMD_ERR_ARG_OFST 4 + +/* No space */ +#define MC_CMD_ERR_ENOSPC 28 + +#endif + +/* MCDI_EVENT structuredef */ +#define MCDI_EVENT_LEN 8 +#define MCDI_EVENT_CONT_LBN 32 +#define MCDI_EVENT_CONT_WIDTH 1 +#define MCDI_EVENT_LEVEL_LBN 33 +#define MCDI_EVENT_LEVEL_WIDTH 3 +/* enum: Info. */ +#define MCDI_EVENT_LEVEL_INFO 0x0 +/* enum: Warning. */ +#define MCDI_EVENT_LEVEL_WARN 0x1 +/* enum: Error. */ +#define MCDI_EVENT_LEVEL_ERR 0x2 +/* enum: Fatal. */ +#define MCDI_EVENT_LEVEL_FATAL 0x3 +#define MCDI_EVENT_DATA_OFST 0 +#define MCDI_EVENT_CMDDONE_SEQ_LBN 0 +#define MCDI_EVENT_CMDDONE_SEQ_WIDTH 8 +#define MCDI_EVENT_CMDDONE_DATALEN_LBN 8 +#define MCDI_EVENT_CMDDONE_DATALEN_WIDTH 8 +#define MCDI_EVENT_CMDDONE_ERRNO_LBN 16 +#define MCDI_EVENT_CMDDONE_ERRNO_WIDTH 8 +#define MCDI_EVENT_LINKCHANGE_LP_CAP_LBN 0 +#define MCDI_EVENT_LINKCHANGE_LP_CAP_WIDTH 16 +#define MCDI_EVENT_LINKCHANGE_SPEED_LBN 16 +#define MCDI_EVENT_LINKCHANGE_SPEED_WIDTH 4 +/* enum: 100Mbs */ +#define MCDI_EVENT_LINKCHANGE_SPEED_100M 0x1 +/* enum: 1Gbs */ +#define MCDI_EVENT_LINKCHANGE_SPEED_1G 0x2 +/* enum: 10Gbs */ +#define MCDI_EVENT_LINKCHANGE_SPEED_10G 0x3 +/* enum: 40Gbs */ +#define MCDI_EVENT_LINKCHANGE_SPEED_40G 0x4 +#define MCDI_EVENT_LINKCHANGE_FCNTL_LBN 20 +#define MCDI_EVENT_LINKCHANGE_FCNTL_WIDTH 4 +#define MCDI_EVENT_LINKCHANGE_LINK_FLAGS_LBN 24 +#define MCDI_EVENT_LINKCHANGE_LINK_FLAGS_WIDTH 8 +#define MCDI_EVENT_SENSOREVT_MONITOR_LBN 0 +#define MCDI_EVENT_SENSOREVT_MONITOR_WIDTH 8 +#define MCDI_EVENT_SENSOREVT_STATE_LBN 8 +#define MCDI_EVENT_SENSOREVT_STATE_WIDTH 8 +#define MCDI_EVENT_SENSOREVT_VALUE_LBN 16 +#define MCDI_EVENT_SENSOREVT_VALUE_WIDTH 16 +#define MCDI_EVENT_FWALERT_DATA_LBN 8 +#define MCDI_EVENT_FWALERT_DATA_WIDTH 24 +#define MCDI_EVENT_FWALERT_REASON_LBN 0 +#define MCDI_EVENT_FWALERT_REASON_WIDTH 8 +/* enum: SRAM Access. */ +#define MCDI_EVENT_FWALERT_REASON_SRAM_ACCESS 0x1 +#define MCDI_EVENT_FLR_VF_LBN 0 +#define MCDI_EVENT_FLR_VF_WIDTH 8 +#define MCDI_EVENT_TX_ERR_TXQ_LBN 0 +#define MCDI_EVENT_TX_ERR_TXQ_WIDTH 12 +#define MCDI_EVENT_TX_ERR_TYPE_LBN 12 +#define MCDI_EVENT_TX_ERR_TYPE_WIDTH 4 +/* enum: Descriptor loader reported failure */ +#define MCDI_EVENT_TX_ERR_DL_FAIL 0x1 +/* enum: Descriptor ring empty and no EOP seen for packet */ +#define MCDI_EVENT_TX_ERR_NO_EOP 0x2 +/* enum: Overlength packet */ +#define MCDI_EVENT_TX_ERR_2BIG 0x3 +/* enum: Malformed option descriptor */ +#define MCDI_EVENT_TX_BAD_OPTDESC 0x5 +/* enum: Option descriptor part way through a packet */ +#define MCDI_EVENT_TX_OPT_IN_PKT 0x8 +/* enum: DMA or PIO data access error */ +#define MCDI_EVENT_TX_ERR_BAD_DMA_OR_PIO 0x9 +#define MCDI_EVENT_TX_ERR_INFO_LBN 16 +#define MCDI_EVENT_TX_ERR_INFO_WIDTH 16 +#define MCDI_EVENT_TX_FLUSH_TO_DRIVER_LBN 12 +#define MCDI_EVENT_TX_FLUSH_TO_DRIVER_WIDTH 1 +#define MCDI_EVENT_TX_FLUSH_TXQ_LBN 0 +#define MCDI_EVENT_TX_FLUSH_TXQ_WIDTH 12 +#define MCDI_EVENT_PTP_ERR_TYPE_LBN 0 +#define MCDI_EVENT_PTP_ERR_TYPE_WIDTH 8 +/* enum: PLL lost lock */ +#define MCDI_EVENT_PTP_ERR_PLL_LOST 0x1 +/* enum: Filter overflow (PDMA) */ +#define MCDI_EVENT_PTP_ERR_FILTER 0x2 +/* enum: FIFO overflow (FPGA) */ +#define MCDI_EVENT_PTP_ERR_FIFO 0x3 +/* enum: Merge queue overflow */ +#define MCDI_EVENT_PTP_ERR_QUEUE 0x4 +#define MCDI_EVENT_AOE_ERR_TYPE_LBN 0 +#define MCDI_EVENT_AOE_ERR_TYPE_WIDTH 8 +/* enum: AOE failed to load - no valid image? */ +#define MCDI_EVENT_AOE_NO_LOAD 0x1 +/* enum: AOE FC reported an exception */ +#define MCDI_EVENT_AOE_FC_ASSERT 0x2 +/* enum: AOE FC watchdogged */ +#define MCDI_EVENT_AOE_FC_WATCHDOG 0x3 +/* enum: AOE FC failed to start */ +#define MCDI_EVENT_AOE_FC_NO_START 0x4 +/* enum: Generic AOE fault - likely to have been reported via other means too + * but intended for use by aoex driver. + */ +#define MCDI_EVENT_AOE_FAULT 0x5 +/* enum: Results of reprogramming the CPLD (status in AOE_ERR_DATA) */ +#define MCDI_EVENT_AOE_CPLD_REPROGRAMMED 0x6 +/* enum: AOE loaded successfully */ +#define MCDI_EVENT_AOE_LOAD 0x7 +/* enum: AOE DMA operation completed (LSB of HOST_HANDLE in AOE_ERR_DATA) */ +#define MCDI_EVENT_AOE_DMA 0x8 +/* enum: AOE byteblaster connected/disconnected (Connection status in + * AOE_ERR_DATA) + */ +#define MCDI_EVENT_AOE_BYTEBLASTER 0x9 +/* enum: DDR ECC status update */ +#define MCDI_EVENT_AOE_DDR_ECC_STATUS 0xa +/* enum: PTP status update */ +#define MCDI_EVENT_AOE_PTP_STATUS 0xb +/* enum: FPGA header incorrect */ +#define MCDI_EVENT_AOE_FPGA_LOAD_HEADER_ERR 0xc +/* enum: FPGA Powered Off due to error in powering up FPGA */ +#define MCDI_EVENT_AOE_FPGA_POWER_OFF 0xd +/* enum: AOE FPGA load failed due to MC to MUM communication failure */ +#define MCDI_EVENT_AOE_FPGA_LOAD_FAILED 0xe +/* enum: Notify that invalid flash type detected */ +#define MCDI_EVENT_AOE_INVALID_FPGA_FLASH_TYPE 0xf +/* enum: Notify that the attempt to run FPGA Controller firmware timedout */ +#define MCDI_EVENT_AOE_FC_RUN_TIMEDOUT 0x10 +#define MCDI_EVENT_AOE_ERR_DATA_LBN 8 +#define MCDI_EVENT_AOE_ERR_DATA_WIDTH 8 +#define MCDI_EVENT_AOE_ERR_CODE_FPGA_HEADER_VERIFY_FAILED_LBN 8 +#define MCDI_EVENT_AOE_ERR_CODE_FPGA_HEADER_VERIFY_FAILED_WIDTH 8 +/* enum: Reading from NV failed */ +#define MCDI_EVENT_AOE_ERR_FPGA_HEADER_NV_READ_FAIL 0x0 +/* enum: Invalid Magic Number if FPGA header */ +#define MCDI_EVENT_AOE_ERR_FPGA_HEADER_MAGIC_FAIL 0x1 +/* enum: Invalid Silicon type detected in header */ +#define MCDI_EVENT_AOE_ERR_FPGA_HEADER_SILICON_TYPE 0x2 +/* enum: Unsupported VRatio */ +#define MCDI_EVENT_AOE_ERR_FPGA_HEADER_VRATIO 0x3 +/* enum: Unsupported DDR Type */ +#define MCDI_EVENT_AOE_ERR_FPGA_HEADER_DDR_TYPE 0x4 +/* enum: DDR Voltage out of supported range */ +#define MCDI_EVENT_AOE_ERR_FPGA_HEADER_DDR_VOLTAGE 0x5 +/* enum: Unsupported DDR speed */ +#define MCDI_EVENT_AOE_ERR_FPGA_HEADER_DDR_SPEED 0x6 +/* enum: Unsupported DDR size */ +#define MCDI_EVENT_AOE_ERR_FPGA_HEADER_DDR_SIZE 0x7 +/* enum: Unsupported DDR rank */ +#define MCDI_EVENT_AOE_ERR_FPGA_HEADER_DDR_RANK 0x8 +#define MCDI_EVENT_AOE_ERR_CODE_INVALID_FPGA_FLASH_TYPE_INFO_LBN 8 +#define MCDI_EVENT_AOE_ERR_CODE_INVALID_FPGA_FLASH_TYPE_INFO_WIDTH 8 +/* enum: Primary boot flash */ +#define MCDI_EVENT_AOE_FLASH_TYPE_BOOT_PRIMARY 0x0 +/* enum: Secondary boot flash */ +#define MCDI_EVENT_AOE_FLASH_TYPE_BOOT_SECONDARY 0x1 +#define MCDI_EVENT_AOE_ERR_CODE_FPGA_POWER_OFF_LBN 8 +#define MCDI_EVENT_AOE_ERR_CODE_FPGA_POWER_OFF_WIDTH 8 +#define MCDI_EVENT_AOE_ERR_CODE_FPGA_LOAD_FAILED_LBN 8 +#define MCDI_EVENT_AOE_ERR_CODE_FPGA_LOAD_FAILED_WIDTH 8 +#define MCDI_EVENT_RX_ERR_RXQ_LBN 0 +#define MCDI_EVENT_RX_ERR_RXQ_WIDTH 12 +#define MCDI_EVENT_RX_ERR_TYPE_LBN 12 +#define MCDI_EVENT_RX_ERR_TYPE_WIDTH 4 +#define MCDI_EVENT_RX_ERR_INFO_LBN 16 +#define MCDI_EVENT_RX_ERR_INFO_WIDTH 16 +#define MCDI_EVENT_RX_FLUSH_TO_DRIVER_LBN 12 +#define MCDI_EVENT_RX_FLUSH_TO_DRIVER_WIDTH 1 +#define MCDI_EVENT_RX_FLUSH_RXQ_LBN 0 +#define MCDI_EVENT_RX_FLUSH_RXQ_WIDTH 12 +#define MCDI_EVENT_MC_REBOOT_COUNT_LBN 0 +#define MCDI_EVENT_MC_REBOOT_COUNT_WIDTH 16 +#define MCDI_EVENT_MUM_ERR_TYPE_LBN 0 +#define MCDI_EVENT_MUM_ERR_TYPE_WIDTH 8 +/* enum: MUM failed to load - no valid image? */ +#define MCDI_EVENT_MUM_NO_LOAD 0x1 +/* enum: MUM f/w reported an exception */ +#define MCDI_EVENT_MUM_ASSERT 0x2 +/* enum: MUM not kicking watchdog */ +#define MCDI_EVENT_MUM_WATCHDOG 0x3 +#define MCDI_EVENT_MUM_ERR_DATA_LBN 8 +#define MCDI_EVENT_MUM_ERR_DATA_WIDTH 8 +#define MCDI_EVENT_DATA_LBN 0 +#define MCDI_EVENT_DATA_WIDTH 32 +#define MCDI_EVENT_SRC_LBN 36 +#define MCDI_EVENT_SRC_WIDTH 8 +#define MCDI_EVENT_EV_CODE_LBN 60 +#define MCDI_EVENT_EV_CODE_WIDTH 4 +#define MCDI_EVENT_CODE_LBN 44 +#define MCDI_EVENT_CODE_WIDTH 8 +/* enum: Event generated by host software */ +#define MCDI_EVENT_SW_EVENT 0x0 +/* enum: Bad assert. */ +#define MCDI_EVENT_CODE_BADSSERT 0x1 +/* enum: PM Notice. */ +#define MCDI_EVENT_CODE_PMNOTICE 0x2 +/* enum: Command done. */ +#define MCDI_EVENT_CODE_CMDDONE 0x3 +/* enum: Link change. */ +#define MCDI_EVENT_CODE_LINKCHANGE 0x4 +/* enum: Sensor Event. */ +#define MCDI_EVENT_CODE_SENSOREVT 0x5 +/* enum: Schedule error. */ +#define MCDI_EVENT_CODE_SCHEDERR 0x6 +/* enum: Reboot. */ +#define MCDI_EVENT_CODE_REBOOT 0x7 +/* enum: Mac stats DMA. */ +#define MCDI_EVENT_CODE_MAC_STATS_DMA 0x8 +/* enum: Firmware alert. */ +#define MCDI_EVENT_CODE_FWALERT 0x9 +/* enum: Function level reset. */ +#define MCDI_EVENT_CODE_FLR 0xa +/* enum: Transmit error */ +#define MCDI_EVENT_CODE_TX_ERR 0xb +/* enum: Tx flush has completed */ +#define MCDI_EVENT_CODE_TX_FLUSH 0xc +/* enum: PTP packet received timestamp */ +#define MCDI_EVENT_CODE_PTP_RX 0xd +/* enum: PTP NIC failure */ +#define MCDI_EVENT_CODE_PTP_FAULT 0xe +/* enum: PTP PPS event */ +#define MCDI_EVENT_CODE_PTP_PPS 0xf +/* enum: Rx flush has completed */ +#define MCDI_EVENT_CODE_RX_FLUSH 0x10 +/* enum: Receive error */ +#define MCDI_EVENT_CODE_RX_ERR 0x11 +/* enum: AOE fault */ +#define MCDI_EVENT_CODE_AOE 0x12 +/* enum: Network port calibration failed (VCAL). */ +#define MCDI_EVENT_CODE_VCAL_FAIL 0x13 +/* enum: HW PPS event */ +#define MCDI_EVENT_CODE_HW_PPS 0x14 +/* enum: The MC has rebooted (huntington and later, siena uses CODE_REBOOT and + * a different format) + */ +#define MCDI_EVENT_CODE_MC_REBOOT 0x15 +/* enum: the MC has detected a parity error */ +#define MCDI_EVENT_CODE_PAR_ERR 0x16 +/* enum: the MC has detected a correctable error */ +#define MCDI_EVENT_CODE_ECC_CORR_ERR 0x17 +/* enum: the MC has detected an uncorrectable error */ +#define MCDI_EVENT_CODE_ECC_FATAL_ERR 0x18 +/* enum: The MC has entered offline BIST mode */ +#define MCDI_EVENT_CODE_MC_BIST 0x19 +/* enum: PTP tick event providing current NIC time */ +#define MCDI_EVENT_CODE_PTP_TIME 0x1a +/* enum: MUM fault */ +#define MCDI_EVENT_CODE_MUM 0x1b +/* enum: notify the designated PF of a new authorization request */ +#define MCDI_EVENT_CODE_PROXY_REQUEST 0x1c +/* enum: notify a function that awaits an authorization that its request has + * been processed and it may now resend the command + */ +#define MCDI_EVENT_CODE_PROXY_RESPONSE 0x1d +/* enum: Artificial event generated by host and posted via MC for test + * purposes. + */ +#define MCDI_EVENT_CODE_TESTGEN 0xfa +#define MCDI_EVENT_CMDDONE_DATA_OFST 0 +#define MCDI_EVENT_CMDDONE_DATA_LBN 0 +#define MCDI_EVENT_CMDDONE_DATA_WIDTH 32 +#define MCDI_EVENT_LINKCHANGE_DATA_OFST 0 +#define MCDI_EVENT_LINKCHANGE_DATA_LBN 0 +#define MCDI_EVENT_LINKCHANGE_DATA_WIDTH 32 +#define MCDI_EVENT_SENSOREVT_DATA_OFST 0 +#define MCDI_EVENT_SENSOREVT_DATA_LBN 0 +#define MCDI_EVENT_SENSOREVT_DATA_WIDTH 32 +#define MCDI_EVENT_MAC_STATS_DMA_GENERATION_OFST 0 +#define MCDI_EVENT_MAC_STATS_DMA_GENERATION_LBN 0 +#define MCDI_EVENT_MAC_STATS_DMA_GENERATION_WIDTH 32 +#define MCDI_EVENT_TX_ERR_DATA_OFST 0 +#define MCDI_EVENT_TX_ERR_DATA_LBN 0 +#define MCDI_EVENT_TX_ERR_DATA_WIDTH 32 +/* For CODE_PTP_RX, CODE_PTP_PPS and CODE_HW_PPS events the seconds field of + * timestamp + */ +#define MCDI_EVENT_PTP_SECONDS_OFST 0 +#define MCDI_EVENT_PTP_SECONDS_LBN 0 +#define MCDI_EVENT_PTP_SECONDS_WIDTH 32 +/* For CODE_PTP_RX, CODE_PTP_PPS and CODE_HW_PPS events the major field of + * timestamp + */ +#define MCDI_EVENT_PTP_MAJOR_OFST 0 +#define MCDI_EVENT_PTP_MAJOR_LBN 0 +#define MCDI_EVENT_PTP_MAJOR_WIDTH 32 +/* For CODE_PTP_RX, CODE_PTP_PPS and CODE_HW_PPS events the nanoseconds field + * of timestamp + */ +#define MCDI_EVENT_PTP_NANOSECONDS_OFST 0 +#define MCDI_EVENT_PTP_NANOSECONDS_LBN 0 +#define MCDI_EVENT_PTP_NANOSECONDS_WIDTH 32 +/* For CODE_PTP_RX, CODE_PTP_PPS and CODE_HW_PPS events the minor field of + * timestamp + */ +#define MCDI_EVENT_PTP_MINOR_OFST 0 +#define MCDI_EVENT_PTP_MINOR_LBN 0 +#define MCDI_EVENT_PTP_MINOR_WIDTH 32 +/* For CODE_PTP_RX events, the lowest four bytes of sourceUUID from PTP packet + */ +#define MCDI_EVENT_PTP_UUID_OFST 0 +#define MCDI_EVENT_PTP_UUID_LBN 0 +#define MCDI_EVENT_PTP_UUID_WIDTH 32 +#define MCDI_EVENT_RX_ERR_DATA_OFST 0 +#define MCDI_EVENT_RX_ERR_DATA_LBN 0 +#define MCDI_EVENT_RX_ERR_DATA_WIDTH 32 +#define MCDI_EVENT_PAR_ERR_DATA_OFST 0 +#define MCDI_EVENT_PAR_ERR_DATA_LBN 0 +#define MCDI_EVENT_PAR_ERR_DATA_WIDTH 32 +#define MCDI_EVENT_ECC_CORR_ERR_DATA_OFST 0 +#define MCDI_EVENT_ECC_CORR_ERR_DATA_LBN 0 +#define MCDI_EVENT_ECC_CORR_ERR_DATA_WIDTH 32 +#define MCDI_EVENT_ECC_FATAL_ERR_DATA_OFST 0 +#define MCDI_EVENT_ECC_FATAL_ERR_DATA_LBN 0 +#define MCDI_EVENT_ECC_FATAL_ERR_DATA_WIDTH 32 +/* For CODE_PTP_TIME events, the major value of the PTP clock */ +#define MCDI_EVENT_PTP_TIME_MAJOR_OFST 0 +#define MCDI_EVENT_PTP_TIME_MAJOR_LBN 0 +#define MCDI_EVENT_PTP_TIME_MAJOR_WIDTH 32 +/* For CODE_PTP_TIME events, bits 19-26 of the minor value of the PTP clock */ +#define MCDI_EVENT_PTP_TIME_MINOR_26_19_LBN 36 +#define MCDI_EVENT_PTP_TIME_MINOR_26_19_WIDTH 8 +/* For CODE_PTP_TIME events where report sync status is enabled, indicates + * whether the NIC clock has ever been set + */ +#define MCDI_EVENT_PTP_TIME_NIC_CLOCK_VALID_LBN 36 +#define MCDI_EVENT_PTP_TIME_NIC_CLOCK_VALID_WIDTH 1 +/* For CODE_PTP_TIME events where report sync status is enabled, indicates + * whether the NIC and System clocks are in sync + */ +#define MCDI_EVENT_PTP_TIME_HOST_NIC_IN_SYNC_LBN 37 +#define MCDI_EVENT_PTP_TIME_HOST_NIC_IN_SYNC_WIDTH 1 +/* For CODE_PTP_TIME events where report sync status is enabled, bits 21-26 of + * the minor value of the PTP clock + */ +#define MCDI_EVENT_PTP_TIME_MINOR_26_21_LBN 38 +#define MCDI_EVENT_PTP_TIME_MINOR_26_21_WIDTH 6 +#define MCDI_EVENT_PROXY_REQUEST_BUFF_INDEX_OFST 0 +#define MCDI_EVENT_PROXY_REQUEST_BUFF_INDEX_LBN 0 +#define MCDI_EVENT_PROXY_REQUEST_BUFF_INDEX_WIDTH 32 +#define MCDI_EVENT_PROXY_RESPONSE_HANDLE_OFST 0 +#define MCDI_EVENT_PROXY_RESPONSE_HANDLE_LBN 0 +#define MCDI_EVENT_PROXY_RESPONSE_HANDLE_WIDTH 32 +/* Zero means that the request has been completed or authorized, and the driver + * should resend it. A non-zero value means that the authorization has been + * denied, and gives the reason. Typically it will be EPERM. + */ +#define MCDI_EVENT_PROXY_RESPONSE_RC_LBN 36 +#define MCDI_EVENT_PROXY_RESPONSE_RC_WIDTH 8 + +/* FCDI_EVENT structuredef */ +#define FCDI_EVENT_LEN 8 +#define FCDI_EVENT_CONT_LBN 32 +#define FCDI_EVENT_CONT_WIDTH 1 +#define FCDI_EVENT_LEVEL_LBN 33 +#define FCDI_EVENT_LEVEL_WIDTH 3 +/* enum: Info. */ +#define FCDI_EVENT_LEVEL_INFO 0x0 +/* enum: Warning. */ +#define FCDI_EVENT_LEVEL_WARN 0x1 +/* enum: Error. */ +#define FCDI_EVENT_LEVEL_ERR 0x2 +/* enum: Fatal. */ +#define FCDI_EVENT_LEVEL_FATAL 0x3 +#define FCDI_EVENT_DATA_OFST 0 +#define FCDI_EVENT_LINK_STATE_STATUS_LBN 0 +#define FCDI_EVENT_LINK_STATE_STATUS_WIDTH 1 +#define FCDI_EVENT_LINK_DOWN 0x0 /* enum */ +#define FCDI_EVENT_LINK_UP 0x1 /* enum */ +#define FCDI_EVENT_DATA_LBN 0 +#define FCDI_EVENT_DATA_WIDTH 32 +#define FCDI_EVENT_SRC_LBN 36 +#define FCDI_EVENT_SRC_WIDTH 8 +#define FCDI_EVENT_EV_CODE_LBN 60 +#define FCDI_EVENT_EV_CODE_WIDTH 4 +#define FCDI_EVENT_CODE_LBN 44 +#define FCDI_EVENT_CODE_WIDTH 8 +/* enum: The FC was rebooted. */ +#define FCDI_EVENT_CODE_REBOOT 0x1 +/* enum: Bad assert. */ +#define FCDI_EVENT_CODE_ASSERT 0x2 +/* enum: DDR3 test result. */ +#define FCDI_EVENT_CODE_DDR_TEST_RESULT 0x3 +/* enum: Link status. */ +#define FCDI_EVENT_CODE_LINK_STATE 0x4 +/* enum: A timed read is ready to be serviced. */ +#define FCDI_EVENT_CODE_TIMED_READ 0x5 +/* enum: One or more PPS IN events */ +#define FCDI_EVENT_CODE_PPS_IN 0x6 +/* enum: Tick event from PTP clock */ +#define FCDI_EVENT_CODE_PTP_TICK 0x7 +/* enum: ECC error counters */ +#define FCDI_EVENT_CODE_DDR_ECC_STATUS 0x8 +/* enum: Current status of PTP */ +#define FCDI_EVENT_CODE_PTP_STATUS 0x9 +/* enum: Port id config to map MC-FC port idx */ +#define FCDI_EVENT_CODE_PORT_CONFIG 0xa +/* enum: Boot result or error code */ +#define FCDI_EVENT_CODE_BOOT_RESULT 0xb +#define FCDI_EVENT_REBOOT_SRC_LBN 36 +#define FCDI_EVENT_REBOOT_SRC_WIDTH 8 +#define FCDI_EVENT_REBOOT_FC_FW 0x0 /* enum */ +#define FCDI_EVENT_REBOOT_FC_BOOTLOADER 0x1 /* enum */ +#define FCDI_EVENT_ASSERT_INSTR_ADDRESS_OFST 0 +#define FCDI_EVENT_ASSERT_INSTR_ADDRESS_LBN 0 +#define FCDI_EVENT_ASSERT_INSTR_ADDRESS_WIDTH 32 +#define FCDI_EVENT_ASSERT_TYPE_LBN 36 +#define FCDI_EVENT_ASSERT_TYPE_WIDTH 8 +#define FCDI_EVENT_DDR_TEST_RESULT_STATUS_CODE_LBN 36 +#define FCDI_EVENT_DDR_TEST_RESULT_STATUS_CODE_WIDTH 8 +#define FCDI_EVENT_DDR_TEST_RESULT_RESULT_OFST 0 +#define FCDI_EVENT_DDR_TEST_RESULT_RESULT_LBN 0 +#define FCDI_EVENT_DDR_TEST_RESULT_RESULT_WIDTH 32 +#define FCDI_EVENT_LINK_STATE_DATA_OFST 0 +#define FCDI_EVENT_LINK_STATE_DATA_LBN 0 +#define FCDI_EVENT_LINK_STATE_DATA_WIDTH 32 +#define FCDI_EVENT_PTP_STATE_OFST 0 +#define FCDI_EVENT_PTP_UNDEFINED 0x0 /* enum */ +#define FCDI_EVENT_PTP_SETUP_FAILED 0x1 /* enum */ +#define FCDI_EVENT_PTP_OPERATIONAL 0x2 /* enum */ +#define FCDI_EVENT_PTP_STATE_LBN 0 +#define FCDI_EVENT_PTP_STATE_WIDTH 32 +#define FCDI_EVENT_DDR_ECC_STATUS_BANK_ID_LBN 36 +#define FCDI_EVENT_DDR_ECC_STATUS_BANK_ID_WIDTH 8 +#define FCDI_EVENT_DDR_ECC_STATUS_STATUS_OFST 0 +#define FCDI_EVENT_DDR_ECC_STATUS_STATUS_LBN 0 +#define FCDI_EVENT_DDR_ECC_STATUS_STATUS_WIDTH 32 +/* Index of MC port being referred to */ +#define FCDI_EVENT_PORT_CONFIG_SRC_LBN 36 +#define FCDI_EVENT_PORT_CONFIG_SRC_WIDTH 8 +/* FC Port index that matches the MC port index in SRC */ +#define FCDI_EVENT_PORT_CONFIG_DATA_OFST 0 +#define FCDI_EVENT_PORT_CONFIG_DATA_LBN 0 +#define FCDI_EVENT_PORT_CONFIG_DATA_WIDTH 32 +#define FCDI_EVENT_BOOT_RESULT_OFST 0 +/* Enum values, see field(s): */ +/* MC_CMD_AOE/MC_CMD_AOE_OUT_INFO/FC_BOOT_RESULT */ +#define FCDI_EVENT_BOOT_RESULT_LBN 0 +#define FCDI_EVENT_BOOT_RESULT_WIDTH 32 + +/* FCDI_EXTENDED_EVENT_PPS structuredef: Extended FCDI event to send PPS events + * to the MC. Note that this structure | is overlayed over a normal FCDI event + * such that bits 32-63 containing | event code, level, source etc remain the + * same. In this case the data | field of the header is defined to be the + * number of timestamps + */ +#define FCDI_EXTENDED_EVENT_PPS_LENMIN 16 +#define FCDI_EXTENDED_EVENT_PPS_LENMAX 248 +#define FCDI_EXTENDED_EVENT_PPS_LEN(num) (8+8*(num)) +/* Number of timestamps following */ +#define FCDI_EXTENDED_EVENT_PPS_COUNT_OFST 0 +#define FCDI_EXTENDED_EVENT_PPS_COUNT_LBN 0 +#define FCDI_EXTENDED_EVENT_PPS_COUNT_WIDTH 32 +/* Seconds field of a timestamp record */ +#define FCDI_EXTENDED_EVENT_PPS_SECONDS_OFST 8 +#define FCDI_EXTENDED_EVENT_PPS_SECONDS_LBN 64 +#define FCDI_EXTENDED_EVENT_PPS_SECONDS_WIDTH 32 +/* Nanoseconds field of a timestamp record */ +#define FCDI_EXTENDED_EVENT_PPS_NANOSECONDS_OFST 12 +#define FCDI_EXTENDED_EVENT_PPS_NANOSECONDS_LBN 96 +#define FCDI_EXTENDED_EVENT_PPS_NANOSECONDS_WIDTH 32 +/* Timestamp records comprising the event */ +#define FCDI_EXTENDED_EVENT_PPS_TIMESTAMPS_OFST 8 +#define FCDI_EXTENDED_EVENT_PPS_TIMESTAMPS_LEN 8 +#define FCDI_EXTENDED_EVENT_PPS_TIMESTAMPS_LO_OFST 8 +#define FCDI_EXTENDED_EVENT_PPS_TIMESTAMPS_HI_OFST 12 +#define FCDI_EXTENDED_EVENT_PPS_TIMESTAMPS_MINNUM 1 +#define FCDI_EXTENDED_EVENT_PPS_TIMESTAMPS_MAXNUM 30 +#define FCDI_EXTENDED_EVENT_PPS_TIMESTAMPS_LBN 64 +#define FCDI_EXTENDED_EVENT_PPS_TIMESTAMPS_WIDTH 64 + +/* MUM_EVENT structuredef */ +#define MUM_EVENT_LEN 8 +#define MUM_EVENT_CONT_LBN 32 +#define MUM_EVENT_CONT_WIDTH 1 +#define MUM_EVENT_LEVEL_LBN 33 +#define MUM_EVENT_LEVEL_WIDTH 3 +/* enum: Info. */ +#define MUM_EVENT_LEVEL_INFO 0x0 +/* enum: Warning. */ +#define MUM_EVENT_LEVEL_WARN 0x1 +/* enum: Error. */ +#define MUM_EVENT_LEVEL_ERR 0x2 +/* enum: Fatal. */ +#define MUM_EVENT_LEVEL_FATAL 0x3 +#define MUM_EVENT_DATA_OFST 0 +#define MUM_EVENT_SENSOR_ID_LBN 0 +#define MUM_EVENT_SENSOR_ID_WIDTH 8 +/* Enum values, see field(s): */ +/* MC_CMD_SENSOR_INFO/MC_CMD_SENSOR_INFO_OUT/MASK */ +#define MUM_EVENT_SENSOR_STATE_LBN 8 +#define MUM_EVENT_SENSOR_STATE_WIDTH 8 +#define MUM_EVENT_PORT_PHY_READY_LBN 0 +#define MUM_EVENT_PORT_PHY_READY_WIDTH 1 +#define MUM_EVENT_PORT_PHY_LINK_UP_LBN 1 +#define MUM_EVENT_PORT_PHY_LINK_UP_WIDTH 1 +#define MUM_EVENT_PORT_PHY_TX_LOL_LBN 2 +#define MUM_EVENT_PORT_PHY_TX_LOL_WIDTH 1 +#define MUM_EVENT_PORT_PHY_RX_LOL_LBN 3 +#define MUM_EVENT_PORT_PHY_RX_LOL_WIDTH 1 +#define MUM_EVENT_PORT_PHY_TX_LOS_LBN 4 +#define MUM_EVENT_PORT_PHY_TX_LOS_WIDTH 1 +#define MUM_EVENT_PORT_PHY_RX_LOS_LBN 5 +#define MUM_EVENT_PORT_PHY_RX_LOS_WIDTH 1 +#define MUM_EVENT_PORT_PHY_TX_FAULT_LBN 6 +#define MUM_EVENT_PORT_PHY_TX_FAULT_WIDTH 1 +#define MUM_EVENT_DATA_LBN 0 +#define MUM_EVENT_DATA_WIDTH 32 +#define MUM_EVENT_SRC_LBN 36 +#define MUM_EVENT_SRC_WIDTH 8 +#define MUM_EVENT_EV_CODE_LBN 60 +#define MUM_EVENT_EV_CODE_WIDTH 4 +#define MUM_EVENT_CODE_LBN 44 +#define MUM_EVENT_CODE_WIDTH 8 +/* enum: The MUM was rebooted. */ +#define MUM_EVENT_CODE_REBOOT 0x1 +/* enum: Bad assert. */ +#define MUM_EVENT_CODE_ASSERT 0x2 +/* enum: Sensor failure. */ +#define MUM_EVENT_CODE_SENSOR 0x3 +/* enum: Link fault has been asserted, or has cleared. */ +#define MUM_EVENT_CODE_QSFP_LASI_INTERRUPT 0x4 +#define MUM_EVENT_SENSOR_DATA_OFST 0 +#define MUM_EVENT_SENSOR_DATA_LBN 0 +#define MUM_EVENT_SENSOR_DATA_WIDTH 32 +#define MUM_EVENT_PORT_PHY_FLAGS_OFST 0 +#define MUM_EVENT_PORT_PHY_FLAGS_LBN 0 +#define MUM_EVENT_PORT_PHY_FLAGS_WIDTH 32 +#define MUM_EVENT_PORT_PHY_COPPER_LEN_OFST 0 +#define MUM_EVENT_PORT_PHY_COPPER_LEN_LBN 0 +#define MUM_EVENT_PORT_PHY_COPPER_LEN_WIDTH 32 +#define MUM_EVENT_PORT_PHY_CAPS_OFST 0 +#define MUM_EVENT_PORT_PHY_CAPS_LBN 0 +#define MUM_EVENT_PORT_PHY_CAPS_WIDTH 32 +#define MUM_EVENT_PORT_PHY_TECH_OFST 0 +#define MUM_EVENT_PORT_PHY_STATE_QSFP_MODULE_TECH_UNKNOWN 0x0 /* enum */ +#define MUM_EVENT_PORT_PHY_STATE_QSFP_MODULE_TECH_OPTICAL 0x1 /* enum */ +#define MUM_EVENT_PORT_PHY_STATE_QSFP_MODULE_TECH_COPPER_PASSIVE 0x2 /* enum */ +#define MUM_EVENT_PORT_PHY_STATE_QSFP_MODULE_TECH_COPPER_PASSIVE_EQUALIZED 0x3 /* enum */ +#define MUM_EVENT_PORT_PHY_STATE_QSFP_MODULE_TECH_COPPER_ACTIVE_LIMITING 0x4 /* enum */ +#define MUM_EVENT_PORT_PHY_STATE_QSFP_MODULE_TECH_COPPER_ACTIVE_LINEAR 0x5 /* enum */ +#define MUM_EVENT_PORT_PHY_STATE_QSFP_MODULE_TECH_BASE_T 0x6 /* enum */ +#define MUM_EVENT_PORT_PHY_STATE_QSFP_MODULE_TECH_LOOPBACK_PASSIVE 0x7 /* enum */ +#define MUM_EVENT_PORT_PHY_TECH_LBN 0 +#define MUM_EVENT_PORT_PHY_TECH_WIDTH 32 +#define MUM_EVENT_PORT_PHY_SRC_DATA_ID_LBN 36 +#define MUM_EVENT_PORT_PHY_SRC_DATA_ID_WIDTH 4 +#define MUM_EVENT_PORT_PHY_SRC_DATA_ID_FLAGS 0x0 /* enum */ +#define MUM_EVENT_PORT_PHY_SRC_DATA_ID_COPPER_LEN 0x1 /* enum */ +#define MUM_EVENT_PORT_PHY_SRC_DATA_ID_CAPS 0x2 /* enum */ +#define MUM_EVENT_PORT_PHY_SRC_DATA_ID_TECH 0x3 /* enum */ +#define MUM_EVENT_PORT_PHY_SRC_DATA_ID_MAX 0x4 /* enum */ +#define MUM_EVENT_PORT_PHY_SRC_PORT_NO_LBN 40 +#define MUM_EVENT_PORT_PHY_SRC_PORT_NO_WIDTH 4 + + +/***********************************/ +/* MC_CMD_READ32 + * Read multiple 32byte words from MC memory. + */ +#define MC_CMD_READ32 0x1 +#undef MC_CMD_0x1_PRIVILEGE_CTG + +#define MC_CMD_0x1_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_READ32_IN msgrequest */ +#define MC_CMD_READ32_IN_LEN 8 +#define MC_CMD_READ32_IN_ADDR_OFST 0 +#define MC_CMD_READ32_IN_NUMWORDS_OFST 4 + +/* MC_CMD_READ32_OUT msgresponse */ +#define MC_CMD_READ32_OUT_LENMIN 4 +#define MC_CMD_READ32_OUT_LENMAX 252 +#define MC_CMD_READ32_OUT_LEN(num) (0+4*(num)) +#define MC_CMD_READ32_OUT_BUFFER_OFST 0 +#define MC_CMD_READ32_OUT_BUFFER_LEN 4 +#define MC_CMD_READ32_OUT_BUFFER_MINNUM 1 +#define MC_CMD_READ32_OUT_BUFFER_MAXNUM 63 + + +/***********************************/ +/* MC_CMD_WRITE32 + * Write multiple 32byte words to MC memory. + */ +#define MC_CMD_WRITE32 0x2 +#undef MC_CMD_0x2_PRIVILEGE_CTG + +#define MC_CMD_0x2_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_WRITE32_IN msgrequest */ +#define MC_CMD_WRITE32_IN_LENMIN 8 +#define MC_CMD_WRITE32_IN_LENMAX 252 +#define MC_CMD_WRITE32_IN_LEN(num) (4+4*(num)) +#define MC_CMD_WRITE32_IN_ADDR_OFST 0 +#define MC_CMD_WRITE32_IN_BUFFER_OFST 4 +#define MC_CMD_WRITE32_IN_BUFFER_LEN 4 +#define MC_CMD_WRITE32_IN_BUFFER_MINNUM 1 +#define MC_CMD_WRITE32_IN_BUFFER_MAXNUM 62 + +/* MC_CMD_WRITE32_OUT msgresponse */ +#define MC_CMD_WRITE32_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_COPYCODE + * Copy MC code between two locations and jump. + */ +#define MC_CMD_COPYCODE 0x3 +#undef MC_CMD_0x3_PRIVILEGE_CTG + +#define MC_CMD_0x3_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_COPYCODE_IN msgrequest */ +#define MC_CMD_COPYCODE_IN_LEN 16 +/* Source address + * + * The main image should be entered via a copy of a single word from and to a + * magic address, which controls various aspects of the boot. The magic address + * is a bitfield, with each bit as documented below. + */ +#define MC_CMD_COPYCODE_IN_SRC_ADDR_OFST 0 +/* enum: Deprecated; equivalent to setting BOOT_MAGIC_PRESENT (see below) */ +#define MC_CMD_COPYCODE_HUNT_NO_MAGIC_ADDR 0x10000 +/* enum: Deprecated; equivalent to setting BOOT_MAGIC_PRESENT and + * BOOT_MAGIC_SATELLITE_CPUS_NOT_LOADED (see below) + */ +#define MC_CMD_COPYCODE_HUNT_NO_DATAPATH_MAGIC_ADDR 0x1d0d0 +/* enum: Deprecated; equivalent to setting BOOT_MAGIC_PRESENT, + * BOOT_MAGIC_SATELLITE_CPUS_NOT_LOADED and BOOT_MAGIC_IGNORE_CONFIG (see + * below) + */ +#define MC_CMD_COPYCODE_HUNT_IGNORE_CONFIG_MAGIC_ADDR 0x1badc +#define MC_CMD_COPYCODE_IN_BOOT_MAGIC_PRESENT_LBN 17 +#define MC_CMD_COPYCODE_IN_BOOT_MAGIC_PRESENT_WIDTH 1 +#define MC_CMD_COPYCODE_IN_BOOT_MAGIC_SATELLITE_CPUS_NOT_LOADED_LBN 2 +#define MC_CMD_COPYCODE_IN_BOOT_MAGIC_SATELLITE_CPUS_NOT_LOADED_WIDTH 1 +#define MC_CMD_COPYCODE_IN_BOOT_MAGIC_IGNORE_CONFIG_LBN 3 +#define MC_CMD_COPYCODE_IN_BOOT_MAGIC_IGNORE_CONFIG_WIDTH 1 +#define MC_CMD_COPYCODE_IN_BOOT_MAGIC_SKIP_BOOT_ICORE_SYNC_LBN 4 +#define MC_CMD_COPYCODE_IN_BOOT_MAGIC_SKIP_BOOT_ICORE_SYNC_WIDTH 1 +#define MC_CMD_COPYCODE_IN_BOOT_MAGIC_FORCE_STANDALONE_LBN 5 +#define MC_CMD_COPYCODE_IN_BOOT_MAGIC_FORCE_STANDALONE_WIDTH 1 +#define MC_CMD_COPYCODE_IN_BOOT_MAGIC_DISABLE_XIP_LBN 6 +#define MC_CMD_COPYCODE_IN_BOOT_MAGIC_DISABLE_XIP_WIDTH 1 +/* Destination address */ +#define MC_CMD_COPYCODE_IN_DEST_ADDR_OFST 4 +#define MC_CMD_COPYCODE_IN_NUMWORDS_OFST 8 +/* Address of where to jump after copy. */ +#define MC_CMD_COPYCODE_IN_JUMP_OFST 12 +/* enum: Control should return to the caller rather than jumping */ +#define MC_CMD_COPYCODE_JUMP_NONE 0x1 + +/* MC_CMD_COPYCODE_OUT msgresponse */ +#define MC_CMD_COPYCODE_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_SET_FUNC + * Select function for function-specific commands. + */ +#define MC_CMD_SET_FUNC 0x4 +#undef MC_CMD_0x4_PRIVILEGE_CTG + +#define MC_CMD_0x4_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_SET_FUNC_IN msgrequest */ +#define MC_CMD_SET_FUNC_IN_LEN 4 +/* Set function */ +#define MC_CMD_SET_FUNC_IN_FUNC_OFST 0 + +/* MC_CMD_SET_FUNC_OUT msgresponse */ +#define MC_CMD_SET_FUNC_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_GET_BOOT_STATUS + * Get the instruction address from which the MC booted. + */ +#define MC_CMD_GET_BOOT_STATUS 0x5 +#undef MC_CMD_0x5_PRIVILEGE_CTG + +#define MC_CMD_0x5_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_GET_BOOT_STATUS_IN msgrequest */ +#define MC_CMD_GET_BOOT_STATUS_IN_LEN 0 + +/* MC_CMD_GET_BOOT_STATUS_OUT msgresponse */ +#define MC_CMD_GET_BOOT_STATUS_OUT_LEN 8 +/* ?? */ +#define MC_CMD_GET_BOOT_STATUS_OUT_BOOT_OFFSET_OFST 0 +/* enum: indicates that the MC wasn't flash booted */ +#define MC_CMD_GET_BOOT_STATUS_OUT_BOOT_OFFSET_NULL 0xdeadbeef +#define MC_CMD_GET_BOOT_STATUS_OUT_FLAGS_OFST 4 +#define MC_CMD_GET_BOOT_STATUS_OUT_FLAGS_WATCHDOG_LBN 0 +#define MC_CMD_GET_BOOT_STATUS_OUT_FLAGS_WATCHDOG_WIDTH 1 +#define MC_CMD_GET_BOOT_STATUS_OUT_FLAGS_PRIMARY_LBN 1 +#define MC_CMD_GET_BOOT_STATUS_OUT_FLAGS_PRIMARY_WIDTH 1 +#define MC_CMD_GET_BOOT_STATUS_OUT_FLAGS_BACKUP_LBN 2 +#define MC_CMD_GET_BOOT_STATUS_OUT_FLAGS_BACKUP_WIDTH 1 + + +/***********************************/ +/* MC_CMD_GET_ASSERTS + * Get (and optionally clear) the current assertion status. Only + * OUT.GLOBAL_FLAGS is guaranteed to exist in the completion payload. The other + * fields will only be present if OUT.GLOBAL_FLAGS != NO_FAILS + */ +#define MC_CMD_GET_ASSERTS 0x6 +#undef MC_CMD_0x6_PRIVILEGE_CTG + +#define MC_CMD_0x6_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_GET_ASSERTS_IN msgrequest */ +#define MC_CMD_GET_ASSERTS_IN_LEN 4 +/* Set to clear assertion */ +#define MC_CMD_GET_ASSERTS_IN_CLEAR_OFST 0 + +/* MC_CMD_GET_ASSERTS_OUT msgresponse */ +#define MC_CMD_GET_ASSERTS_OUT_LEN 140 +/* Assertion status flag. */ +#define MC_CMD_GET_ASSERTS_OUT_GLOBAL_FLAGS_OFST 0 +/* enum: No assertions have failed. */ +#define MC_CMD_GET_ASSERTS_FLAGS_NO_FAILS 0x1 +/* enum: A system-level assertion has failed. */ +#define MC_CMD_GET_ASSERTS_FLAGS_SYS_FAIL 0x2 +/* enum: A thread-level assertion has failed. */ +#define MC_CMD_GET_ASSERTS_FLAGS_THR_FAIL 0x3 +/* enum: The system was reset by the watchdog. */ +#define MC_CMD_GET_ASSERTS_FLAGS_WDOG_FIRED 0x4 +/* enum: An illegal address trap stopped the system (huntington and later) */ +#define MC_CMD_GET_ASSERTS_FLAGS_ADDR_TRAP 0x5 +/* Failing PC value */ +#define MC_CMD_GET_ASSERTS_OUT_SAVED_PC_OFFS_OFST 4 +/* Saved GP regs */ +#define MC_CMD_GET_ASSERTS_OUT_GP_REGS_OFFS_OFST 8 +#define MC_CMD_GET_ASSERTS_OUT_GP_REGS_OFFS_LEN 4 +#define MC_CMD_GET_ASSERTS_OUT_GP_REGS_OFFS_NUM 31 +/* enum: A magic value hinting that the value in this register at the time of + * the failure has likely been lost. + */ +#define MC_CMD_GET_ASSERTS_REG_NO_DATA 0xda7a1057 +/* Failing thread address */ +#define MC_CMD_GET_ASSERTS_OUT_THREAD_OFFS_OFST 132 +#define MC_CMD_GET_ASSERTS_OUT_RESERVED_OFST 136 + + +/***********************************/ +/* MC_CMD_LOG_CTRL + * Configure the output stream for log events such as link state changes, + * sensor notifications and MCDI completions + */ +#define MC_CMD_LOG_CTRL 0x7 +#undef MC_CMD_0x7_PRIVILEGE_CTG + +#define MC_CMD_0x7_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_LOG_CTRL_IN msgrequest */ +#define MC_CMD_LOG_CTRL_IN_LEN 8 +/* Log destination */ +#define MC_CMD_LOG_CTRL_IN_LOG_DEST_OFST 0 +/* enum: UART. */ +#define MC_CMD_LOG_CTRL_IN_LOG_DEST_UART 0x1 +/* enum: Event queue. */ +#define MC_CMD_LOG_CTRL_IN_LOG_DEST_EVQ 0x2 +/* Legacy argument. Must be zero. */ +#define MC_CMD_LOG_CTRL_IN_LOG_DEST_EVQ_OFST 4 + +/* MC_CMD_LOG_CTRL_OUT msgresponse */ +#define MC_CMD_LOG_CTRL_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_GET_VERSION + * Get version information about the MC firmware. + */ +#define MC_CMD_GET_VERSION 0x8 +#undef MC_CMD_0x8_PRIVILEGE_CTG + +#define MC_CMD_0x8_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_GET_VERSION_IN msgrequest */ +#define MC_CMD_GET_VERSION_IN_LEN 0 + +/* MC_CMD_GET_VERSION_EXT_IN msgrequest: Asks for the extended version */ +#define MC_CMD_GET_VERSION_EXT_IN_LEN 4 +/* placeholder, set to 0 */ +#define MC_CMD_GET_VERSION_EXT_IN_EXT_FLAGS_OFST 0 + +/* MC_CMD_GET_VERSION_V0_OUT msgresponse: deprecated version format */ +#define MC_CMD_GET_VERSION_V0_OUT_LEN 4 +#define MC_CMD_GET_VERSION_OUT_FIRMWARE_OFST 0 +/* enum: Reserved version number to indicate "any" version. */ +#define MC_CMD_GET_VERSION_OUT_FIRMWARE_ANY 0xffffffff +/* enum: Bootrom version value for Siena. */ +#define MC_CMD_GET_VERSION_OUT_FIRMWARE_SIENA_BOOTROM 0xb0070000 +/* enum: Bootrom version value for Huntington. */ +#define MC_CMD_GET_VERSION_OUT_FIRMWARE_HUNT_BOOTROM 0xb0070001 + +/* MC_CMD_GET_VERSION_OUT msgresponse */ +#define MC_CMD_GET_VERSION_OUT_LEN 32 +/* MC_CMD_GET_VERSION_OUT_FIRMWARE_OFST 0 */ +/* Enum values, see field(s): */ +/* MC_CMD_GET_VERSION_V0_OUT/MC_CMD_GET_VERSION_OUT_FIRMWARE */ +#define MC_CMD_GET_VERSION_OUT_PCOL_OFST 4 +/* 128bit mask of functions supported by the current firmware */ +#define MC_CMD_GET_VERSION_OUT_SUPPORTED_FUNCS_OFST 8 +#define MC_CMD_GET_VERSION_OUT_SUPPORTED_FUNCS_LEN 16 +#define MC_CMD_GET_VERSION_OUT_VERSION_OFST 24 +#define MC_CMD_GET_VERSION_OUT_VERSION_LEN 8 +#define MC_CMD_GET_VERSION_OUT_VERSION_LO_OFST 24 +#define MC_CMD_GET_VERSION_OUT_VERSION_HI_OFST 28 + +/* MC_CMD_GET_VERSION_EXT_OUT msgresponse */ +#define MC_CMD_GET_VERSION_EXT_OUT_LEN 48 +/* MC_CMD_GET_VERSION_OUT_FIRMWARE_OFST 0 */ +/* Enum values, see field(s): */ +/* MC_CMD_GET_VERSION_V0_OUT/MC_CMD_GET_VERSION_OUT_FIRMWARE */ +#define MC_CMD_GET_VERSION_EXT_OUT_PCOL_OFST 4 +/* 128bit mask of functions supported by the current firmware */ +#define MC_CMD_GET_VERSION_EXT_OUT_SUPPORTED_FUNCS_OFST 8 +#define MC_CMD_GET_VERSION_EXT_OUT_SUPPORTED_FUNCS_LEN 16 +#define MC_CMD_GET_VERSION_EXT_OUT_VERSION_OFST 24 +#define MC_CMD_GET_VERSION_EXT_OUT_VERSION_LEN 8 +#define MC_CMD_GET_VERSION_EXT_OUT_VERSION_LO_OFST 24 +#define MC_CMD_GET_VERSION_EXT_OUT_VERSION_HI_OFST 28 +/* extra info */ +#define MC_CMD_GET_VERSION_EXT_OUT_EXTRA_OFST 32 +#define MC_CMD_GET_VERSION_EXT_OUT_EXTRA_LEN 16 + + +/***********************************/ +/* MC_CMD_FC + * Perform an FC operation + */ +#define MC_CMD_FC 0x9 + +/* MC_CMD_FC_IN msgrequest */ +#define MC_CMD_FC_IN_LEN 4 +#define MC_CMD_FC_IN_OP_HDR_OFST 0 +#define MC_CMD_FC_IN_OP_LBN 0 +#define MC_CMD_FC_IN_OP_WIDTH 8 +/* enum: NULL MCDI command to FC. */ +#define MC_CMD_FC_OP_NULL 0x1 +/* enum: Unused opcode */ +#define MC_CMD_FC_OP_UNUSED 0x2 +/* enum: MAC driver commands */ +#define MC_CMD_FC_OP_MAC 0x3 +/* enum: Read FC memory */ +#define MC_CMD_FC_OP_READ32 0x4 +/* enum: Write to FC memory */ +#define MC_CMD_FC_OP_WRITE32 0x5 +/* enum: Read FC memory */ +#define MC_CMD_FC_OP_TRC_READ 0x6 +/* enum: Write to FC memory */ +#define MC_CMD_FC_OP_TRC_WRITE 0x7 +/* enum: FC firmware Version */ +#define MC_CMD_FC_OP_GET_VERSION 0x8 +/* enum: Read FC memory */ +#define MC_CMD_FC_OP_TRC_RX_READ 0x9 +/* enum: Write to FC memory */ +#define MC_CMD_FC_OP_TRC_RX_WRITE 0xa +/* enum: SFP parameters */ +#define MC_CMD_FC_OP_SFP 0xb +/* enum: DDR3 test */ +#define MC_CMD_FC_OP_DDR_TEST 0xc +/* enum: Get Crash context from FC */ +#define MC_CMD_FC_OP_GET_ASSERT 0xd +/* enum: Get FPGA Build registers */ +#define MC_CMD_FC_OP_FPGA_BUILD 0xe +/* enum: Read map support commands */ +#define MC_CMD_FC_OP_READ_MAP 0xf +/* enum: FC Capabilities */ +#define MC_CMD_FC_OP_CAPABILITIES 0x10 +/* enum: FC Global flags */ +#define MC_CMD_FC_OP_GLOBAL_FLAGS 0x11 +/* enum: FC IO using relative addressing modes */ +#define MC_CMD_FC_OP_IO_REL 0x12 +/* enum: FPGA link information */ +#define MC_CMD_FC_OP_UHLINK 0x13 +/* enum: Configure loopbacks and link on FPGA ports */ +#define MC_CMD_FC_OP_SET_LINK 0x14 +/* enum: Licensing operations relating to AOE */ +#define MC_CMD_FC_OP_LICENSE 0x15 +/* enum: Startup information to the FC */ +#define MC_CMD_FC_OP_STARTUP 0x16 +/* enum: Configure a DMA read */ +#define MC_CMD_FC_OP_DMA 0x17 +/* enum: Configure a timed read */ +#define MC_CMD_FC_OP_TIMED_READ 0x18 +/* enum: Control UART logging */ +#define MC_CMD_FC_OP_LOG 0x19 +/* enum: Get the value of a given clock_id */ +#define MC_CMD_FC_OP_CLOCK 0x1a +/* enum: DDR3/QDR3 parameters */ +#define MC_CMD_FC_OP_DDR 0x1b +/* enum: PTP and timestamp control */ +#define MC_CMD_FC_OP_TIMESTAMP 0x1c +/* enum: Commands for SPI Flash interface */ +#define MC_CMD_FC_OP_SPI 0x1d +/* enum: Commands for diagnostic components */ +#define MC_CMD_FC_OP_DIAG 0x1e +/* enum: External AOE port. */ +#define MC_CMD_FC_IN_PORT_EXT_OFST 0x0 +/* enum: Internal AOE port. */ +#define MC_CMD_FC_IN_PORT_INT_OFST 0x40 + +/* MC_CMD_FC_IN_NULL msgrequest */ +#define MC_CMD_FC_IN_NULL_LEN 4 +#define MC_CMD_FC_IN_CMD_OFST 0 + +/* MC_CMD_FC_IN_PHY msgrequest */ +#define MC_CMD_FC_IN_PHY_LEN 5 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +/* FC PHY driver operation code */ +#define MC_CMD_FC_IN_PHY_OP_OFST 4 +#define MC_CMD_FC_IN_PHY_OP_LEN 1 +/* enum: PHY init handler */ +#define MC_CMD_FC_OP_PHY_OP_INIT 0x1 +/* enum: PHY reconfigure handler */ +#define MC_CMD_FC_OP_PHY_OP_RECONFIGURE 0x2 +/* enum: PHY reboot handler */ +#define MC_CMD_FC_OP_PHY_OP_REBOOT 0x3 +/* enum: PHY get_supported_cap handler */ +#define MC_CMD_FC_OP_PHY_OP_GET_SUPPORTED_CAP 0x4 +/* enum: PHY get_config handler */ +#define MC_CMD_FC_OP_PHY_OP_GET_CONFIG 0x5 +/* enum: PHY get_media_info handler */ +#define MC_CMD_FC_OP_PHY_OP_GET_MEDIA_INFO 0x6 +/* enum: PHY set_led handler */ +#define MC_CMD_FC_OP_PHY_OP_SET_LED 0x7 +/* enum: PHY lasi_interrupt handler */ +#define MC_CMD_FC_OP_PHY_OP_LASI_INTERRUPT 0x8 +/* enum: PHY check_link handler */ +#define MC_CMD_FC_OP_PHY_OP_CHECK_LINK 0x9 +/* enum: PHY fill_stats handler */ +#define MC_CMD_FC_OP_PHY_OP_FILL_STATS 0xa +/* enum: PHY bpx_link_state_changed handler */ +#define MC_CMD_FC_OP_PHY_OP_BPX_LINK_STATE_CHANGED 0xb +/* enum: PHY get_state handler */ +#define MC_CMD_FC_OP_PHY_OP_GET_STATE 0xc +/* enum: PHY start_bist handler */ +#define MC_CMD_FC_OP_PHY_OP_START_BIST 0xd +/* enum: PHY poll_bist handler */ +#define MC_CMD_FC_OP_PHY_OP_POLL_BIST 0xe +/* enum: PHY nvram_test handler */ +#define MC_CMD_FC_OP_PHY_OP_NVRAM_TEST 0xf +/* enum: PHY relinquish handler */ +#define MC_CMD_FC_OP_PHY_OP_RELINQUISH_SPI 0x10 +/* enum: PHY read connection from FC - may be not required */ +#define MC_CMD_FC_OP_PHY_OP_GET_CONNECTION 0x11 +/* enum: PHY read flags from FC - may be not required */ +#define MC_CMD_FC_OP_PHY_OP_GET_FLAGS 0x12 + +/* MC_CMD_FC_IN_PHY_INIT msgrequest */ +#define MC_CMD_FC_IN_PHY_INIT_LEN 4 +#define MC_CMD_FC_IN_PHY_CMD_OFST 0 + +/* MC_CMD_FC_IN_MAC msgrequest */ +#define MC_CMD_FC_IN_MAC_LEN 8 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +#define MC_CMD_FC_IN_MAC_HEADER_OFST 4 +#define MC_CMD_FC_IN_MAC_OP_LBN 0 +#define MC_CMD_FC_IN_MAC_OP_WIDTH 8 +/* enum: MAC reconfigure handler */ +#define MC_CMD_FC_OP_MAC_OP_RECONFIGURE 0x1 +/* enum: MAC Set command - same as MC_CMD_SET_MAC */ +#define MC_CMD_FC_OP_MAC_OP_SET_LINK 0x2 +/* enum: MAC statistics */ +#define MC_CMD_FC_OP_MAC_OP_GET_STATS 0x3 +/* enum: MAC RX statistics */ +#define MC_CMD_FC_OP_MAC_OP_GET_RX_STATS 0x6 +/* enum: MAC TX statistics */ +#define MC_CMD_FC_OP_MAC_OP_GET_TX_STATS 0x7 +/* enum: MAC Read status */ +#define MC_CMD_FC_OP_MAC_OP_READ_STATUS 0x8 +#define MC_CMD_FC_IN_MAC_PORT_TYPE_LBN 8 +#define MC_CMD_FC_IN_MAC_PORT_TYPE_WIDTH 8 +/* enum: External FPGA port. */ +#define MC_CMD_FC_PORT_EXT 0x0 +/* enum: Internal Siena-facing FPGA ports. */ +#define MC_CMD_FC_PORT_INT 0x1 +#define MC_CMD_FC_IN_MAC_PORT_IDX_LBN 16 +#define MC_CMD_FC_IN_MAC_PORT_IDX_WIDTH 8 +#define MC_CMD_FC_IN_MAC_CMD_FORMAT_LBN 24 +#define MC_CMD_FC_IN_MAC_CMD_FORMAT_WIDTH 8 +/* enum: Default FC command format; the fields PORT_TYPE and PORT_IDX are + * irrelevant. Port number is derived from pci_fn; passed in FC header. + */ +#define MC_CMD_FC_OP_MAC_CMD_FORMAT_DEFAULT 0x0 +/* enum: Override default port number. Port number determined by fields + * PORT_TYPE and PORT_IDX. + */ +#define MC_CMD_FC_OP_MAC_CMD_FORMAT_PORT_OVERRIDE 0x1 + +/* MC_CMD_FC_IN_MAC_RECONFIGURE msgrequest */ +#define MC_CMD_FC_IN_MAC_RECONFIGURE_LEN 8 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +/* MC_CMD_FC_IN_MAC_HEADER_OFST 4 */ + +/* MC_CMD_FC_IN_MAC_SET_LINK msgrequest */ +#define MC_CMD_FC_IN_MAC_SET_LINK_LEN 32 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +/* MC_CMD_FC_IN_MAC_HEADER_OFST 4 */ +/* MTU size */ +#define MC_CMD_FC_IN_MAC_SET_LINK_MTU_OFST 8 +/* Drain Tx FIFO */ +#define MC_CMD_FC_IN_MAC_SET_LINK_DRAIN_OFST 12 +#define MC_CMD_FC_IN_MAC_SET_LINK_ADDR_OFST 16 +#define MC_CMD_FC_IN_MAC_SET_LINK_ADDR_LEN 8 +#define MC_CMD_FC_IN_MAC_SET_LINK_ADDR_LO_OFST 16 +#define MC_CMD_FC_IN_MAC_SET_LINK_ADDR_HI_OFST 20 +#define MC_CMD_FC_IN_MAC_SET_LINK_REJECT_OFST 24 +#define MC_CMD_FC_IN_MAC_SET_LINK_REJECT_UNICAST_LBN 0 +#define MC_CMD_FC_IN_MAC_SET_LINK_REJECT_UNICAST_WIDTH 1 +#define MC_CMD_FC_IN_MAC_SET_LINK_REJECT_BRDCAST_LBN 1 +#define MC_CMD_FC_IN_MAC_SET_LINK_REJECT_BRDCAST_WIDTH 1 +#define MC_CMD_FC_IN_MAC_SET_LINK_FCNTL_OFST 28 + +/* MC_CMD_FC_IN_MAC_READ_STATUS msgrequest */ +#define MC_CMD_FC_IN_MAC_READ_STATUS_LEN 8 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +/* MC_CMD_FC_IN_MAC_HEADER_OFST 4 */ + +/* MC_CMD_FC_IN_MAC_GET_RX_STATS msgrequest */ +#define MC_CMD_FC_IN_MAC_GET_RX_STATS_LEN 8 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +/* MC_CMD_FC_IN_MAC_HEADER_OFST 4 */ + +/* MC_CMD_FC_IN_MAC_GET_TX_STATS msgrequest */ +#define MC_CMD_FC_IN_MAC_GET_TX_STATS_LEN 8 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +/* MC_CMD_FC_IN_MAC_HEADER_OFST 4 */ + +/* MC_CMD_FC_IN_MAC_GET_STATS msgrequest */ +#define MC_CMD_FC_IN_MAC_GET_STATS_LEN 20 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +/* MC_CMD_FC_IN_MAC_HEADER_OFST 4 */ +/* MC Statistics index */ +#define MC_CMD_FC_IN_MAC_GET_STATS_STATS_INDEX_OFST 8 +#define MC_CMD_FC_IN_MAC_GET_STATS_FLAGS_OFST 12 +#define MC_CMD_FC_IN_MAC_GET_STATS_CLEAR_ALL_LBN 0 +#define MC_CMD_FC_IN_MAC_GET_STATS_CLEAR_ALL_WIDTH 1 +#define MC_CMD_FC_IN_MAC_GET_STATS_CLEAR_LBN 1 +#define MC_CMD_FC_IN_MAC_GET_STATS_CLEAR_WIDTH 1 +#define MC_CMD_FC_IN_MAC_GET_STATS_UPDATE_LBN 2 +#define MC_CMD_FC_IN_MAC_GET_STATS_UPDATE_WIDTH 1 +/* Number of statistics to read */ +#define MC_CMD_FC_IN_MAC_GET_STATS_NUM_OFST 16 +#define MC_CMD_FC_MAC_NSTATS_PER_BLOCK 0x1e /* enum */ +#define MC_CMD_FC_MAC_NBYTES_PER_STAT 0x8 /* enum */ + +/* MC_CMD_FC_IN_READ32 msgrequest */ +#define MC_CMD_FC_IN_READ32_LEN 16 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +#define MC_CMD_FC_IN_READ32_ADDR_HI_OFST 4 +#define MC_CMD_FC_IN_READ32_ADDR_LO_OFST 8 +#define MC_CMD_FC_IN_READ32_NUMWORDS_OFST 12 + +/* MC_CMD_FC_IN_WRITE32 msgrequest */ +#define MC_CMD_FC_IN_WRITE32_LENMIN 16 +#define MC_CMD_FC_IN_WRITE32_LENMAX 252 +#define MC_CMD_FC_IN_WRITE32_LEN(num) (12+4*(num)) +/* MC_CMD_FC_IN_CMD_OFST 0 */ +#define MC_CMD_FC_IN_WRITE32_ADDR_HI_OFST 4 +#define MC_CMD_FC_IN_WRITE32_ADDR_LO_OFST 8 +#define MC_CMD_FC_IN_WRITE32_BUFFER_OFST 12 +#define MC_CMD_FC_IN_WRITE32_BUFFER_LEN 4 +#define MC_CMD_FC_IN_WRITE32_BUFFER_MINNUM 1 +#define MC_CMD_FC_IN_WRITE32_BUFFER_MAXNUM 60 + +/* MC_CMD_FC_IN_TRC_READ msgrequest */ +#define MC_CMD_FC_IN_TRC_READ_LEN 12 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +#define MC_CMD_FC_IN_TRC_READ_TRC_OFST 4 +#define MC_CMD_FC_IN_TRC_READ_CHANNEL_OFST 8 + +/* MC_CMD_FC_IN_TRC_WRITE msgrequest */ +#define MC_CMD_FC_IN_TRC_WRITE_LEN 28 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +#define MC_CMD_FC_IN_TRC_WRITE_TRC_OFST 4 +#define MC_CMD_FC_IN_TRC_WRITE_CHANNEL_OFST 8 +#define MC_CMD_FC_IN_TRC_WRITE_DATA_OFST 12 +#define MC_CMD_FC_IN_TRC_WRITE_DATA_LEN 4 +#define MC_CMD_FC_IN_TRC_WRITE_DATA_NUM 4 + +/* MC_CMD_FC_IN_GET_VERSION msgrequest */ +#define MC_CMD_FC_IN_GET_VERSION_LEN 4 +/* MC_CMD_FC_IN_CMD_OFST 0 */ + +/* MC_CMD_FC_IN_TRC_RX_READ msgrequest */ +#define MC_CMD_FC_IN_TRC_RX_READ_LEN 12 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +#define MC_CMD_FC_IN_TRC_RX_READ_TRC_OFST 4 +#define MC_CMD_FC_IN_TRC_RX_READ_CHANNEL_OFST 8 + +/* MC_CMD_FC_IN_TRC_RX_WRITE msgrequest */ +#define MC_CMD_FC_IN_TRC_RX_WRITE_LEN 20 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +#define MC_CMD_FC_IN_TRC_RX_WRITE_TRC_OFST 4 +#define MC_CMD_FC_IN_TRC_RX_WRITE_CHANNEL_OFST 8 +#define MC_CMD_FC_IN_TRC_RX_WRITE_DATA_OFST 12 +#define MC_CMD_FC_IN_TRC_RX_WRITE_DATA_LEN 4 +#define MC_CMD_FC_IN_TRC_RX_WRITE_DATA_NUM 2 + +/* MC_CMD_FC_IN_SFP msgrequest */ +#define MC_CMD_FC_IN_SFP_LEN 28 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +/* Link speed is 100, 1000, 10000, 40000 */ +#define MC_CMD_FC_IN_SFP_SPEED_OFST 4 +/* Length of copper cable - zero when not relevant (e.g. if cable is fibre) */ +#define MC_CMD_FC_IN_SFP_COPPER_LEN_OFST 8 +/* Not relevant for cards with QSFP modules. For older cards, true if module is + * a dual speed SFP+ module. + */ +#define MC_CMD_FC_IN_SFP_DUAL_SPEED_OFST 12 +/* True if an SFP Module is present (other fields valid when true) */ +#define MC_CMD_FC_IN_SFP_PRESENT_OFST 16 +/* The type of the SFP+ Module. For later cards with QSFP modules, this field + * is unused and the type is communicated by other means. + */ +#define MC_CMD_FC_IN_SFP_TYPE_OFST 20 +/* Capabilities corresponding to 1 bits. */ +#define MC_CMD_FC_IN_SFP_CAPS_OFST 24 + +/* MC_CMD_FC_IN_DDR_TEST msgrequest */ +#define MC_CMD_FC_IN_DDR_TEST_LEN 8 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +#define MC_CMD_FC_IN_DDR_TEST_HEADER_OFST 4 +#define MC_CMD_FC_IN_DDR_TEST_OP_LBN 0 +#define MC_CMD_FC_IN_DDR_TEST_OP_WIDTH 8 +/* enum: DRAM Test Start */ +#define MC_CMD_FC_OP_DDR_TEST_START 0x1 +/* enum: DRAM Test Poll */ +#define MC_CMD_FC_OP_DDR_TEST_POLL 0x2 + +/* MC_CMD_FC_IN_DDR_TEST_START msgrequest */ +#define MC_CMD_FC_IN_DDR_TEST_START_LEN 12 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +/* MC_CMD_FC_IN_DDR_TEST_HEADER_OFST 4 */ +#define MC_CMD_FC_IN_DDR_TEST_START_MASK_OFST 8 +#define MC_CMD_FC_IN_DDR_TEST_START_T0_LBN 0 +#define MC_CMD_FC_IN_DDR_TEST_START_T0_WIDTH 1 +#define MC_CMD_FC_IN_DDR_TEST_START_T1_LBN 1 +#define MC_CMD_FC_IN_DDR_TEST_START_T1_WIDTH 1 +#define MC_CMD_FC_IN_DDR_TEST_START_B0_LBN 2 +#define MC_CMD_FC_IN_DDR_TEST_START_B0_WIDTH 1 +#define MC_CMD_FC_IN_DDR_TEST_START_B1_LBN 3 +#define MC_CMD_FC_IN_DDR_TEST_START_B1_WIDTH 1 + +/* MC_CMD_FC_IN_DDR_TEST_POLL msgrequest */ +#define MC_CMD_FC_IN_DDR_TEST_POLL_LEN 12 +#define MC_CMD_FC_IN_DDR_TEST_CMD_OFST 0 +/* MC_CMD_FC_IN_DDR_TEST_HEADER_OFST 4 */ +/* Clear previous test result and prepare for restarting DDR test */ +#define MC_CMD_FC_IN_DDR_TEST_POLL_CLEAR_RESULT_FOR_DDR_TEST_OFST 8 + +/* MC_CMD_FC_IN_GET_ASSERT msgrequest */ +#define MC_CMD_FC_IN_GET_ASSERT_LEN 4 +/* MC_CMD_FC_IN_CMD_OFST 0 */ + +/* MC_CMD_FC_IN_FPGA_BUILD msgrequest */ +#define MC_CMD_FC_IN_FPGA_BUILD_LEN 8 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +/* FPGA build info operation code */ +#define MC_CMD_FC_IN_FPGA_BUILD_OP_OFST 4 +/* enum: Get the build registers */ +#define MC_CMD_FC_IN_FPGA_BUILD_BUILD 0x1 +/* enum: Get the services registers */ +#define MC_CMD_FC_IN_FPGA_BUILD_SERVICES 0x2 +/* enum: Get the BSP version */ +#define MC_CMD_FC_IN_FPGA_BUILD_BSP_VERSION 0x3 +/* enum: Get build register for V2 (SFA974X) */ +#define MC_CMD_FC_IN_FPGA_BUILD_BUILD_V2 0x4 +/* enum: GEt the services register for V2 (SFA974X) */ +#define MC_CMD_FC_IN_FPGA_BUILD_SERVICES_V2 0x5 + +/* MC_CMD_FC_IN_READ_MAP msgrequest */ +#define MC_CMD_FC_IN_READ_MAP_LEN 8 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +#define MC_CMD_FC_IN_READ_MAP_HEADER_OFST 4 +#define MC_CMD_FC_IN_READ_MAP_OP_LBN 0 +#define MC_CMD_FC_IN_READ_MAP_OP_WIDTH 8 +/* enum: Get the number of map regions */ +#define MC_CMD_FC_OP_READ_MAP_COUNT 0x1 +/* enum: Get the specified map */ +#define MC_CMD_FC_OP_READ_MAP_INDEX 0x2 + +/* MC_CMD_FC_IN_READ_MAP_COUNT msgrequest */ +#define MC_CMD_FC_IN_READ_MAP_COUNT_LEN 8 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +/* MC_CMD_FC_IN_READ_MAP_HEADER_OFST 4 */ + +/* MC_CMD_FC_IN_READ_MAP_INDEX msgrequest */ +#define MC_CMD_FC_IN_READ_MAP_INDEX_LEN 12 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +/* MC_CMD_FC_IN_READ_MAP_HEADER_OFST 4 */ +#define MC_CMD_FC_IN_MAP_INDEX_OFST 8 + +/* MC_CMD_FC_IN_CAPABILITIES msgrequest */ +#define MC_CMD_FC_IN_CAPABILITIES_LEN 4 +/* MC_CMD_FC_IN_CMD_OFST 0 */ + +/* MC_CMD_FC_IN_GLOBAL_FLAGS msgrequest */ +#define MC_CMD_FC_IN_GLOBAL_FLAGS_LEN 8 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +#define MC_CMD_FC_IN_GLOBAL_FLAGS_FLAGS_OFST 4 +#define MC_CMD_FC_IN_GLOBAL_FLAGS_RX_TUNING_CABLE_PLUGGED_IN_LBN 0 +#define MC_CMD_FC_IN_GLOBAL_FLAGS_RX_TUNING_CABLE_PLUGGED_IN_WIDTH 1 +#define MC_CMD_FC_IN_GLOBAL_FLAGS_RX_TUNING_LINK_MONITORING_LBN 1 +#define MC_CMD_FC_IN_GLOBAL_FLAGS_RX_TUNING_LINK_MONITORING_WIDTH 1 +#define MC_CMD_FC_IN_GLOBAL_FLAGS_DFE_ENABLE_LBN 2 +#define MC_CMD_FC_IN_GLOBAL_FLAGS_DFE_ENABLE_WIDTH 1 +#define MC_CMD_FC_IN_GLOBAL_FLAGS_1D_EYE_ENABLE_LBN 3 +#define MC_CMD_FC_IN_GLOBAL_FLAGS_1D_EYE_ENABLE_WIDTH 1 +#define MC_CMD_FC_IN_GLOBAL_FLAGS_1D_TUNING_ENABLE_LBN 4 +#define MC_CMD_FC_IN_GLOBAL_FLAGS_1D_TUNING_ENABLE_WIDTH 1 +#define MC_CMD_FC_IN_GLOBAL_FLAGS_OFFCAL_ENABLE_LBN 5 +#define MC_CMD_FC_IN_GLOBAL_FLAGS_OFFCAL_ENABLE_WIDTH 1 + +/* MC_CMD_FC_IN_IO_REL msgrequest */ +#define MC_CMD_FC_IN_IO_REL_LEN 8 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +#define MC_CMD_FC_IN_IO_REL_HEADER_OFST 4 +#define MC_CMD_FC_IN_IO_REL_OP_LBN 0 +#define MC_CMD_FC_IN_IO_REL_OP_WIDTH 8 +/* enum: Get the base address that the FC applies to relative commands */ +#define MC_CMD_FC_IN_IO_REL_GET_ADDR 0x1 +/* enum: Read data */ +#define MC_CMD_FC_IN_IO_REL_READ32 0x2 +/* enum: Write data */ +#define MC_CMD_FC_IN_IO_REL_WRITE32 0x3 +#define MC_CMD_FC_IN_IO_REL_COMP_TYPE_LBN 8 +#define MC_CMD_FC_IN_IO_REL_COMP_TYPE_WIDTH 8 +/* enum: Application address space */ +#define MC_CMD_FC_COMP_TYPE_APP_ADDR_SPACE 0x1 +/* enum: Flash address space */ +#define MC_CMD_FC_COMP_TYPE_FLASH 0x2 + +/* MC_CMD_FC_IN_IO_REL_GET_ADDR msgrequest */ +#define MC_CMD_FC_IN_IO_REL_GET_ADDR_LEN 8 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +/* MC_CMD_FC_IN_IO_REL_HEADER_OFST 4 */ + +/* MC_CMD_FC_IN_IO_REL_READ32 msgrequest */ +#define MC_CMD_FC_IN_IO_REL_READ32_LEN 20 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +/* MC_CMD_FC_IN_IO_REL_HEADER_OFST 4 */ +#define MC_CMD_FC_IN_IO_REL_READ32_ADDR_HI_OFST 8 +#define MC_CMD_FC_IN_IO_REL_READ32_ADDR_LO_OFST 12 +#define MC_CMD_FC_IN_IO_REL_READ32_NUMWORDS_OFST 16 + +/* MC_CMD_FC_IN_IO_REL_WRITE32 msgrequest */ +#define MC_CMD_FC_IN_IO_REL_WRITE32_LENMIN 20 +#define MC_CMD_FC_IN_IO_REL_WRITE32_LENMAX 252 +#define MC_CMD_FC_IN_IO_REL_WRITE32_LEN(num) (16+4*(num)) +/* MC_CMD_FC_IN_CMD_OFST 0 */ +/* MC_CMD_FC_IN_IO_REL_HEADER_OFST 4 */ +#define MC_CMD_FC_IN_IO_REL_WRITE32_ADDR_HI_OFST 8 +#define MC_CMD_FC_IN_IO_REL_WRITE32_ADDR_LO_OFST 12 +#define MC_CMD_FC_IN_IO_REL_WRITE32_BUFFER_OFST 16 +#define MC_CMD_FC_IN_IO_REL_WRITE32_BUFFER_LEN 4 +#define MC_CMD_FC_IN_IO_REL_WRITE32_BUFFER_MINNUM 1 +#define MC_CMD_FC_IN_IO_REL_WRITE32_BUFFER_MAXNUM 59 + +/* MC_CMD_FC_IN_UHLINK msgrequest */ +#define MC_CMD_FC_IN_UHLINK_LEN 8 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +#define MC_CMD_FC_IN_UHLINK_HEADER_OFST 4 +#define MC_CMD_FC_IN_UHLINK_OP_LBN 0 +#define MC_CMD_FC_IN_UHLINK_OP_WIDTH 8 +/* enum: Get PHY configuration info */ +#define MC_CMD_FC_OP_UHLINK_PHY 0x1 +/* enum: Get MAC configuration info */ +#define MC_CMD_FC_OP_UHLINK_MAC 0x2 +/* enum: Get Rx eye table */ +#define MC_CMD_FC_OP_UHLINK_RX_EYE 0x3 +/* enum: Get Rx eye plot */ +#define MC_CMD_FC_OP_UHLINK_DUMP_RX_EYE_PLOT 0x4 +/* enum: Get Rx eye plot */ +#define MC_CMD_FC_OP_UHLINK_READ_RX_EYE_PLOT 0x5 +/* enum: Retune Rx settings */ +#define MC_CMD_FC_OP_UHLINK_RX_TUNE 0x6 +/* enum: Set loopback mode on fpga port */ +#define MC_CMD_FC_OP_UHLINK_LOOPBACK_SET 0x7 +/* enum: Get loopback mode config state on fpga port */ +#define MC_CMD_FC_OP_UHLINK_LOOPBACK_GET 0x8 +#define MC_CMD_FC_IN_UHLINK_PORT_TYPE_LBN 8 +#define MC_CMD_FC_IN_UHLINK_PORT_TYPE_WIDTH 8 +#define MC_CMD_FC_IN_UHLINK_PORT_IDX_LBN 16 +#define MC_CMD_FC_IN_UHLINK_PORT_IDX_WIDTH 8 +#define MC_CMD_FC_IN_UHLINK_CMD_FORMAT_LBN 24 +#define MC_CMD_FC_IN_UHLINK_CMD_FORMAT_WIDTH 8 +/* enum: Default FC command format; the fields PORT_TYPE and PORT_IDX are + * irrelevant. Port number is derived from pci_fn; passed in FC header. + */ +#define MC_CMD_FC_OP_UHLINK_CMD_FORMAT_DEFAULT 0x0 +/* enum: Override default port number. Port number determined by fields + * PORT_TYPE and PORT_IDX. + */ +#define MC_CMD_FC_OP_UHLINK_CMD_FORMAT_PORT_OVERRIDE 0x1 + +/* MC_CMD_FC_OP_UHLINK_PHY msgrequest */ +#define MC_CMD_FC_OP_UHLINK_PHY_LEN 8 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +/* MC_CMD_FC_IN_UHLINK_HEADER_OFST 4 */ + +/* MC_CMD_FC_OP_UHLINK_MAC msgrequest */ +#define MC_CMD_FC_OP_UHLINK_MAC_LEN 8 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +/* MC_CMD_FC_IN_UHLINK_HEADER_OFST 4 */ + +/* MC_CMD_FC_OP_UHLINK_RX_EYE msgrequest */ +#define MC_CMD_FC_OP_UHLINK_RX_EYE_LEN 12 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +/* MC_CMD_FC_IN_UHLINK_HEADER_OFST 4 */ +#define MC_CMD_FC_OP_UHLINK_RX_EYE_INDEX_OFST 8 +#define MC_CMD_FC_UHLINK_RX_EYE_PER_BLOCK 0x30 /* enum */ + +/* MC_CMD_FC_OP_UHLINK_DUMP_RX_EYE_PLOT msgrequest */ +#define MC_CMD_FC_OP_UHLINK_DUMP_RX_EYE_PLOT_LEN 8 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +/* MC_CMD_FC_IN_UHLINK_HEADER_OFST 4 */ + +/* MC_CMD_FC_OP_UHLINK_READ_RX_EYE_PLOT msgrequest */ +#define MC_CMD_FC_OP_UHLINK_READ_RX_EYE_PLOT_LEN 20 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +/* MC_CMD_FC_IN_UHLINK_HEADER_OFST 4 */ +#define MC_CMD_FC_OP_UHLINK_READ_RX_EYE_PLOT_DC_GAIN_OFST 8 +#define MC_CMD_FC_OP_UHLINK_READ_RX_EYE_PLOT_EQ_CONTROL_OFST 12 +#define MC_CMD_FC_OP_UHLINK_READ_RX_EYE_PLOT_INDEX_OFST 16 +#define MC_CMD_FC_UHLINK_RX_EYE_PLOT_ROWS_PER_BLOCK 0x1e /* enum */ + +/* MC_CMD_FC_OP_UHLINK_RX_TUNE msgrequest */ +#define MC_CMD_FC_OP_UHLINK_RX_TUNE_LEN 8 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +/* MC_CMD_FC_IN_UHLINK_HEADER_OFST 4 */ + +/* MC_CMD_FC_OP_UHLINK_LOOPBACK_SET msgrequest */ +#define MC_CMD_FC_OP_UHLINK_LOOPBACK_SET_LEN 16 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +/* MC_CMD_FC_IN_UHLINK_HEADER_OFST 4 */ +#define MC_CMD_FC_OP_UHLINK_LOOPBACK_SET_TYPE_OFST 8 +#define MC_CMD_FC_UHLINK_LOOPBACK_TYPE_PCS_SERIAL 0x0 /* enum */ +#define MC_CMD_FC_UHLINK_LOOPBACK_TYPE_PMA_PRE_CDR 0x1 /* enum */ +#define MC_CMD_FC_UHLINK_LOOPBACK_TYPE_PMA_POST_CDR 0x2 /* enum */ +#define MC_CMD_FC_OP_UHLINK_LOOPBACK_SET_STATE_OFST 12 +#define MC_CMD_FC_UHLINK_LOOPBACK_STATE_OFF 0x0 /* enum */ +#define MC_CMD_FC_UHLINK_LOOPBACK_STATE_ON 0x1 /* enum */ + +/* MC_CMD_FC_OP_UHLINK_LOOPBACK_GET msgrequest */ +#define MC_CMD_FC_OP_UHLINK_LOOPBACK_GET_LEN 12 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +/* MC_CMD_FC_IN_UHLINK_HEADER_OFST 4 */ +#define MC_CMD_FC_OP_UHLINK_LOOPBACK_GET_TYPE_OFST 8 + +/* MC_CMD_FC_IN_SET_LINK msgrequest */ +#define MC_CMD_FC_IN_SET_LINK_LEN 16 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +/* See MC_CMD_GET_LOOPBACK_MODES/MC_CMD_GET_LOOPBACK_MODES_OUT/100M */ +#define MC_CMD_FC_IN_SET_LINK_MODE_OFST 4 +#define MC_CMD_FC_IN_SET_LINK_SPEED_OFST 8 +#define MC_CMD_FC_IN_SET_LINK_FLAGS_OFST 12 +#define MC_CMD_FC_IN_SET_LINK_LOWPOWER_LBN 0 +#define MC_CMD_FC_IN_SET_LINK_LOWPOWER_WIDTH 1 +#define MC_CMD_FC_IN_SET_LINK_POWEROFF_LBN 1 +#define MC_CMD_FC_IN_SET_LINK_POWEROFF_WIDTH 1 +#define MC_CMD_FC_IN_SET_LINK_TXDIS_LBN 2 +#define MC_CMD_FC_IN_SET_LINK_TXDIS_WIDTH 1 + +/* MC_CMD_FC_IN_LICENSE msgrequest */ +#define MC_CMD_FC_IN_LICENSE_LEN 8 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +#define MC_CMD_FC_IN_LICENSE_OP_OFST 4 +#define MC_CMD_FC_IN_LICENSE_UPDATE_LICENSE 0x0 /* enum */ +#define MC_CMD_FC_IN_LICENSE_GET_KEY_STATS 0x1 /* enum */ + +/* MC_CMD_FC_IN_STARTUP msgrequest */ +#define MC_CMD_FC_IN_STARTUP_LEN 40 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +#define MC_CMD_FC_IN_STARTUP_BASE_OFST 4 +#define MC_CMD_FC_IN_STARTUP_LENGTH_OFST 8 +/* Length of identifier */ +#define MC_CMD_FC_IN_STARTUP_IDLENGTH_OFST 12 +/* Identifier for AOE FPGA */ +#define MC_CMD_FC_IN_STARTUP_ID_OFST 16 +#define MC_CMD_FC_IN_STARTUP_ID_LEN 1 +#define MC_CMD_FC_IN_STARTUP_ID_NUM 24 + +/* MC_CMD_FC_IN_DMA msgrequest */ +#define MC_CMD_FC_IN_DMA_LEN 8 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +#define MC_CMD_FC_IN_DMA_OP_OFST 4 +#define MC_CMD_FC_IN_DMA_STOP 0x0 /* enum */ +#define MC_CMD_FC_IN_DMA_READ 0x1 /* enum */ + +/* MC_CMD_FC_IN_DMA_STOP msgrequest */ +#define MC_CMD_FC_IN_DMA_STOP_LEN 12 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +/* MC_CMD_FC_IN_DMA_OP_OFST 4 */ +/* FC supplied handle */ +#define MC_CMD_FC_IN_DMA_STOP_FC_HANDLE_OFST 8 + +/* MC_CMD_FC_IN_DMA_READ msgrequest */ +#define MC_CMD_FC_IN_DMA_READ_LEN 16 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +/* MC_CMD_FC_IN_DMA_OP_OFST 4 */ +#define MC_CMD_FC_IN_DMA_READ_OFFSET_OFST 8 +#define MC_CMD_FC_IN_DMA_READ_LENGTH_OFST 12 + +/* MC_CMD_FC_IN_TIMED_READ msgrequest */ +#define MC_CMD_FC_IN_TIMED_READ_LEN 8 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +#define MC_CMD_FC_IN_TIMED_READ_OP_OFST 4 +#define MC_CMD_FC_IN_TIMED_READ_SET 0x0 /* enum */ +#define MC_CMD_FC_IN_TIMED_READ_GET 0x1 /* enum */ +#define MC_CMD_FC_IN_TIMED_READ_CLEAR 0x2 /* enum */ + +/* MC_CMD_FC_IN_TIMED_READ_SET msgrequest */ +#define MC_CMD_FC_IN_TIMED_READ_SET_LEN 52 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +/* MC_CMD_FC_IN_TIMED_READ_OP_OFST 4 */ +/* Host supplied handle (unique) */ +#define MC_CMD_FC_IN_TIMED_READ_SET_HOST_HANDLE_OFST 8 +/* Address into which to transfer data in host */ +#define MC_CMD_FC_IN_TIMED_READ_SET_HOST_DMA_ADDRESS_OFST 12 +#define MC_CMD_FC_IN_TIMED_READ_SET_HOST_DMA_ADDRESS_LEN 8 +#define MC_CMD_FC_IN_TIMED_READ_SET_HOST_DMA_ADDRESS_LO_OFST 12 +#define MC_CMD_FC_IN_TIMED_READ_SET_HOST_DMA_ADDRESS_HI_OFST 16 +/* AOE address from which to transfer data */ +#define MC_CMD_FC_IN_TIMED_READ_SET_AOE_ADDRESS_OFST 20 +#define MC_CMD_FC_IN_TIMED_READ_SET_AOE_ADDRESS_LEN 8 +#define MC_CMD_FC_IN_TIMED_READ_SET_AOE_ADDRESS_LO_OFST 20 +#define MC_CMD_FC_IN_TIMED_READ_SET_AOE_ADDRESS_HI_OFST 24 +/* Length of AOE transfer (total) */ +#define MC_CMD_FC_IN_TIMED_READ_SET_AOE_LENGTH_OFST 28 +/* Length of host transfer (total) */ +#define MC_CMD_FC_IN_TIMED_READ_SET_HOST_LENGTH_OFST 32 +/* Offset back from aoe_address to apply operation to */ +#define MC_CMD_FC_IN_TIMED_READ_SET_OFFSET_OFST 36 +/* Data to apply at offset */ +#define MC_CMD_FC_IN_TIMED_READ_SET_DATA_OFST 40 +#define MC_CMD_FC_IN_TIMED_READ_SET_FLAGS_OFST 44 +#define MC_CMD_FC_IN_TIMED_READ_SET_INDIRECT_LBN 0 +#define MC_CMD_FC_IN_TIMED_READ_SET_INDIRECT_WIDTH 1 +#define MC_CMD_FC_IN_TIMED_READ_SET_DOUBLE_LBN 1 +#define MC_CMD_FC_IN_TIMED_READ_SET_DOUBLE_WIDTH 1 +#define MC_CMD_FC_IN_TIMED_READ_SET_EVENT_LBN 2 +#define MC_CMD_FC_IN_TIMED_READ_SET_EVENT_WIDTH 1 +#define MC_CMD_FC_IN_TIMED_READ_SET_PREREAD_LBN 3 +#define MC_CMD_FC_IN_TIMED_READ_SET_PREREAD_WIDTH 2 +#define MC_CMD_FC_IN_TIMED_READ_SET_NONE 0x0 /* enum */ +#define MC_CMD_FC_IN_TIMED_READ_SET_READ 0x1 /* enum */ +#define MC_CMD_FC_IN_TIMED_READ_SET_WRITE 0x2 /* enum */ +#define MC_CMD_FC_IN_TIMED_READ_SET_READWRITE 0x3 /* enum */ +/* Period at which reads are performed (100ms units) */ +#define MC_CMD_FC_IN_TIMED_READ_SET_PERIOD_OFST 48 + +/* MC_CMD_FC_IN_TIMED_READ_GET msgrequest */ +#define MC_CMD_FC_IN_TIMED_READ_GET_LEN 12 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +/* MC_CMD_FC_IN_TIMED_READ_OP_OFST 4 */ +/* FC supplied handle */ +#define MC_CMD_FC_IN_TIMED_READ_GET_FC_HANDLE_OFST 8 + +/* MC_CMD_FC_IN_TIMED_READ_CLEAR msgrequest */ +#define MC_CMD_FC_IN_TIMED_READ_CLEAR_LEN 12 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +/* MC_CMD_FC_IN_TIMED_READ_OP_OFST 4 */ +/* FC supplied handle */ +#define MC_CMD_FC_IN_TIMED_READ_CLEAR_FC_HANDLE_OFST 8 + +/* MC_CMD_FC_IN_LOG msgrequest */ +#define MC_CMD_FC_IN_LOG_LEN 8 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +#define MC_CMD_FC_IN_LOG_OP_OFST 4 +#define MC_CMD_FC_IN_LOG_ADDR_RANGE 0x0 /* enum */ +#define MC_CMD_FC_IN_LOG_JTAG_UART 0x1 /* enum */ + +/* MC_CMD_FC_IN_LOG_ADDR_RANGE msgrequest */ +#define MC_CMD_FC_IN_LOG_ADDR_RANGE_LEN 20 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +/* MC_CMD_FC_IN_LOG_OP_OFST 4 */ +/* Partition offset into flash */ +#define MC_CMD_FC_IN_LOG_ADDR_RANGE_OFFSET_OFST 8 +/* Partition length */ +#define MC_CMD_FC_IN_LOG_ADDR_RANGE_LENGTH_OFST 12 +/* Partition erase size */ +#define MC_CMD_FC_IN_LOG_ADDR_RANGE_ERASE_SIZE_OFST 16 + +/* MC_CMD_FC_IN_LOG_JTAG_UART msgrequest */ +#define MC_CMD_FC_IN_LOG_JTAG_UART_LEN 12 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +/* MC_CMD_FC_IN_LOG_OP_OFST 4 */ +/* Enable/disable printing to JTAG UART */ +#define MC_CMD_FC_IN_LOG_JTAG_UART_ENABLE_OFST 8 + +/* MC_CMD_FC_IN_CLOCK msgrequest */ +#define MC_CMD_FC_IN_CLOCK_LEN 12 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +#define MC_CMD_FC_IN_CLOCK_OP_OFST 4 +#define MC_CMD_FC_IN_CLOCK_GET_TIME 0x0 /* enum */ +#define MC_CMD_FC_IN_CLOCK_SET_TIME 0x1 /* enum */ +/* Perform a clock operation */ +#define MC_CMD_FC_IN_CLOCK_ID_OFST 8 +#define MC_CMD_FC_IN_CLOCK_STATS 0x0 /* enum */ +#define MC_CMD_FC_IN_CLOCK_MAC 0x1 /* enum */ + +/* MC_CMD_FC_IN_CLOCK_GET_TIME msgrequest */ +#define MC_CMD_FC_IN_CLOCK_GET_TIME_LEN 12 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +/* MC_CMD_FC_IN_CLOCK_OP_OFST 4 */ +/* Retrieve the clock value of the specified clock */ +/* MC_CMD_FC_IN_CLOCK_ID_OFST 8 */ + +/* MC_CMD_FC_IN_CLOCK_SET_TIME msgrequest */ +#define MC_CMD_FC_IN_CLOCK_SET_TIME_LEN 24 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +/* MC_CMD_FC_IN_CLOCK_OP_OFST 4 */ +/* MC_CMD_FC_IN_CLOCK_ID_OFST 8 */ +#define MC_CMD_FC_IN_CLOCK_SET_TIME_SECONDS_OFST 12 +#define MC_CMD_FC_IN_CLOCK_SET_TIME_SECONDS_LEN 8 +#define MC_CMD_FC_IN_CLOCK_SET_TIME_SECONDS_LO_OFST 12 +#define MC_CMD_FC_IN_CLOCK_SET_TIME_SECONDS_HI_OFST 16 +/* Set the clock value of the specified clock */ +#define MC_CMD_FC_IN_CLOCK_SET_TIME_NANOSECONDS_OFST 20 + +/* MC_CMD_FC_IN_DDR msgrequest */ +#define MC_CMD_FC_IN_DDR_LEN 12 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +#define MC_CMD_FC_IN_DDR_OP_OFST 4 +#define MC_CMD_FC_IN_DDR_SET_SPD 0x0 /* enum */ +#define MC_CMD_FC_IN_DDR_GET_STATUS 0x1 /* enum */ +#define MC_CMD_FC_IN_DDR_SET_INFO 0x2 /* enum */ +#define MC_CMD_FC_IN_DDR_BANK_OFST 8 +#define MC_CMD_FC_IN_DDR_BANK_B0 0x0 /* enum */ +#define MC_CMD_FC_IN_DDR_BANK_B1 0x1 /* enum */ +#define MC_CMD_FC_IN_DDR_BANK_T0 0x2 /* enum */ +#define MC_CMD_FC_IN_DDR_BANK_T1 0x3 /* enum */ +#define MC_CMD_FC_IN_DDR_NUM_BANKS 0x4 /* enum */ + +/* MC_CMD_FC_IN_DDR_SET_SPD msgrequest */ +#define MC_CMD_FC_IN_DDR_SET_SPD_LEN 148 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +/* MC_CMD_FC_IN_DDR_OP_OFST 4 */ +/* Affected bank */ +/* MC_CMD_FC_IN_DDR_BANK_OFST 8 */ +/* Flags */ +#define MC_CMD_FC_IN_DDR_FLAGS_OFST 12 +#define MC_CMD_FC_IN_DDR_SET_SPD_ACTIVE 0x1 /* enum */ +/* 128-byte page of serial presence detect data read from module's EEPROM */ +#define MC_CMD_FC_IN_DDR_SPD_OFST 16 +#define MC_CMD_FC_IN_DDR_SPD_LEN 1 +#define MC_CMD_FC_IN_DDR_SPD_NUM 128 +/* Page index of the spd data copied into MC_CMD_FC_IN_DDR_SPD */ +#define MC_CMD_FC_IN_DDR_SPD_PAGE_ID_OFST 144 + +/* MC_CMD_FC_IN_DDR_SET_INFO msgrequest */ +#define MC_CMD_FC_IN_DDR_SET_INFO_LEN 16 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +/* MC_CMD_FC_IN_DDR_OP_OFST 4 */ +/* Affected bank */ +/* MC_CMD_FC_IN_DDR_BANK_OFST 8 */ +/* Size of DDR */ +#define MC_CMD_FC_IN_DDR_SIZE_OFST 12 + +/* MC_CMD_FC_IN_DDR_GET_STATUS msgrequest */ +#define MC_CMD_FC_IN_DDR_GET_STATUS_LEN 12 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +/* MC_CMD_FC_IN_DDR_OP_OFST 4 */ +/* Affected bank */ +/* MC_CMD_FC_IN_DDR_BANK_OFST 8 */ + +/* MC_CMD_FC_IN_TIMESTAMP msgrequest */ +#define MC_CMD_FC_IN_TIMESTAMP_LEN 8 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +/* FC timestamp operation code */ +#define MC_CMD_FC_IN_TIMESTAMP_OP_OFST 4 +/* enum: Read transmit timestamp(s) */ +#define MC_CMD_FC_IN_TIMESTAMP_READ_TRANSMIT 0x0 +/* enum: Read snapshot timestamps */ +#define MC_CMD_FC_IN_TIMESTAMP_READ_SNAPSHOT 0x1 +/* enum: Clear all transmit timestamps */ +#define MC_CMD_FC_IN_TIMESTAMP_CLEAR_TRANSMIT 0x2 + +/* MC_CMD_FC_IN_TIMESTAMP_READ_TRANSMIT msgrequest */ +#define MC_CMD_FC_IN_TIMESTAMP_READ_TRANSMIT_LEN 28 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +#define MC_CMD_FC_IN_TIMESTAMP_READ_TRANSMIT_OP_OFST 4 +/* Control filtering of the returned timestamp and sequence number specified + * here + */ +#define MC_CMD_FC_IN_TIMESTAMP_READ_TRANSMIT_FILTER_OFST 8 +/* enum: Return most recent timestamp. No filtering */ +#define MC_CMD_FC_IN_TIMESTAMP_READ_TRANSMIT_LATEST 0x0 +/* enum: Match timestamp against the PTP clock ID, port number and sequence + * number specified + */ +#define MC_CMD_FC_IN_TIMESTAMP_READ_TRANSMIT_MATCH 0x1 +/* Clock identity of PTP packet for which timestamp required */ +#define MC_CMD_FC_IN_TIMESTAMP_READ_TRANSMIT_CLOCK_ID_OFST 12 +#define MC_CMD_FC_IN_TIMESTAMP_READ_TRANSMIT_CLOCK_ID_LEN 8 +#define MC_CMD_FC_IN_TIMESTAMP_READ_TRANSMIT_CLOCK_ID_LO_OFST 12 +#define MC_CMD_FC_IN_TIMESTAMP_READ_TRANSMIT_CLOCK_ID_HI_OFST 16 +/* Port number of PTP packet for which timestamp required */ +#define MC_CMD_FC_IN_TIMESTAMP_READ_TRANSMIT_PORT_NUM_OFST 20 +/* Sequence number of PTP packet for which timestamp required */ +#define MC_CMD_FC_IN_TIMESTAMP_READ_TRANSMIT_SEQ_NUM_OFST 24 + +/* MC_CMD_FC_IN_TIMESTAMP_READ_SNAPSHOT msgrequest */ +#define MC_CMD_FC_IN_TIMESTAMP_READ_SNAPSHOT_LEN 8 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +#define MC_CMD_FC_IN_TIMESTAMP_READ_SNAPSHOT_OP_OFST 4 + +/* MC_CMD_FC_IN_TIMESTAMP_CLEAR_TRANSMIT msgrequest */ +#define MC_CMD_FC_IN_TIMESTAMP_CLEAR_TRANSMIT_LEN 8 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +#define MC_CMD_FC_IN_TIMESTAMP_CLEAR_TRANSMIT_OP_OFST 4 + +/* MC_CMD_FC_IN_SPI msgrequest */ +#define MC_CMD_FC_IN_SPI_LEN 8 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +/* Basic commands for SPI Flash. */ +#define MC_CMD_FC_IN_SPI_OP_OFST 4 +/* enum: SPI Flash read */ +#define MC_CMD_FC_IN_SPI_READ 0x0 +/* enum: SPI Flash write */ +#define MC_CMD_FC_IN_SPI_WRITE 0x1 +/* enum: SPI Flash erase */ +#define MC_CMD_FC_IN_SPI_ERASE 0x2 + +/* MC_CMD_FC_IN_SPI_READ msgrequest */ +#define MC_CMD_FC_IN_SPI_READ_LEN 16 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +#define MC_CMD_FC_IN_SPI_READ_OP_OFST 4 +#define MC_CMD_FC_IN_SPI_READ_ADDR_OFST 8 +#define MC_CMD_FC_IN_SPI_READ_NUMBYTES_OFST 12 + +/* MC_CMD_FC_IN_SPI_WRITE msgrequest */ +#define MC_CMD_FC_IN_SPI_WRITE_LENMIN 16 +#define MC_CMD_FC_IN_SPI_WRITE_LENMAX 252 +#define MC_CMD_FC_IN_SPI_WRITE_LEN(num) (12+4*(num)) +/* MC_CMD_FC_IN_CMD_OFST 0 */ +#define MC_CMD_FC_IN_SPI_WRITE_OP_OFST 4 +#define MC_CMD_FC_IN_SPI_WRITE_ADDR_OFST 8 +#define MC_CMD_FC_IN_SPI_WRITE_BUFFER_OFST 12 +#define MC_CMD_FC_IN_SPI_WRITE_BUFFER_LEN 4 +#define MC_CMD_FC_IN_SPI_WRITE_BUFFER_MINNUM 1 +#define MC_CMD_FC_IN_SPI_WRITE_BUFFER_MAXNUM 60 + +/* MC_CMD_FC_IN_SPI_ERASE msgrequest */ +#define MC_CMD_FC_IN_SPI_ERASE_LEN 16 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +#define MC_CMD_FC_IN_SPI_ERASE_OP_OFST 4 +#define MC_CMD_FC_IN_SPI_ERASE_ADDR_OFST 8 +#define MC_CMD_FC_IN_SPI_ERASE_NUMBYTES_OFST 12 + +/* MC_CMD_FC_IN_DIAG msgrequest */ +#define MC_CMD_FC_IN_DIAG_LEN 8 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +/* Operation code indicating component type */ +#define MC_CMD_FC_IN_DIAG_OP_OFST 4 +/* enum: Power noise generator. */ +#define MC_CMD_FC_IN_DIAG_POWER_NOISE 0x0 +/* enum: DDR soak test component. */ +#define MC_CMD_FC_IN_DIAG_DDR_SOAK 0x1 +/* enum: Diagnostics datapath control component. */ +#define MC_CMD_FC_IN_DIAG_DATAPATH_CTRL 0x2 + +/* MC_CMD_FC_IN_DIAG_POWER_NOISE msgrequest */ +#define MC_CMD_FC_IN_DIAG_POWER_NOISE_LEN 12 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +#define MC_CMD_FC_IN_DIAG_POWER_NOISE_OP_OFST 4 +/* Sub-opcode describing the operation to be carried out */ +#define MC_CMD_FC_IN_DIAG_POWER_NOISE_SUB_OP_OFST 8 +/* enum: Read the configuration (the 32-bit values in each of the clock enable + * count and toggle count registers) + */ +#define MC_CMD_FC_IN_DIAG_POWER_NOISE_READ_CONFIG 0x0 +/* enum: Write a new configuration to the clock enable count and toggle count + * registers + */ +#define MC_CMD_FC_IN_DIAG_POWER_NOISE_WRITE_CONFIG 0x1 + +/* MC_CMD_FC_IN_DIAG_POWER_NOISE_READ_CONFIG msgrequest */ +#define MC_CMD_FC_IN_DIAG_POWER_NOISE_READ_CONFIG_LEN 12 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +#define MC_CMD_FC_IN_DIAG_POWER_NOISE_READ_CONFIG_OP_OFST 4 +#define MC_CMD_FC_IN_DIAG_POWER_NOISE_READ_CONFIG_SUB_OP_OFST 8 + +/* MC_CMD_FC_IN_DIAG_POWER_NOISE_WRITE_CONFIG msgrequest */ +#define MC_CMD_FC_IN_DIAG_POWER_NOISE_WRITE_CONFIG_LEN 20 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +#define MC_CMD_FC_IN_DIAG_POWER_NOISE_WRITE_CONFIG_OP_OFST 4 +#define MC_CMD_FC_IN_DIAG_POWER_NOISE_WRITE_CONFIG_SUB_OP_OFST 8 +/* The 32-bit value to be written to the toggle count register */ +#define MC_CMD_FC_IN_DIAG_POWER_NOISE_WRITE_CONFIG_TOGGLE_COUNT_OFST 12 +/* The 32-bit value to be written to the clock enable count register */ +#define MC_CMD_FC_IN_DIAG_POWER_NOISE_WRITE_CONFIG_CLKEN_COUNT_OFST 16 + +/* MC_CMD_FC_IN_DIAG_DDR_SOAK msgrequest */ +#define MC_CMD_FC_IN_DIAG_DDR_SOAK_LEN 12 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +#define MC_CMD_FC_IN_DIAG_DDR_SOAK_OP_OFST 4 +/* Sub-opcode describing the operation to be carried out */ +#define MC_CMD_FC_IN_DIAG_DDR_SOAK_SUB_OP_OFST 8 +/* enum: Starts DDR soak test on selected banks */ +#define MC_CMD_FC_IN_DIAG_DDR_SOAK_START 0x0 +/* enum: Read status of DDR soak test */ +#define MC_CMD_FC_IN_DIAG_DDR_SOAK_RESULT 0x1 +/* enum: Stop test */ +#define MC_CMD_FC_IN_DIAG_DDR_SOAK_STOP 0x2 +/* enum: Set or clear bit that triggers fake errors. These cause subsequent + * tests to fail until the bit is cleared. + */ +#define MC_CMD_FC_IN_DIAG_DDR_SOAK_ERROR 0x3 + +/* MC_CMD_FC_IN_DIAG_DDR_SOAK_START msgrequest */ +#define MC_CMD_FC_IN_DIAG_DDR_SOAK_START_LEN 24 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +#define MC_CMD_FC_IN_DIAG_DDR_SOAK_START_OP_OFST 4 +#define MC_CMD_FC_IN_DIAG_DDR_SOAK_START_SUB_OP_OFST 8 +/* Mask of DDR banks to be tested */ +#define MC_CMD_FC_IN_DIAG_DDR_SOAK_START_BANK_MASK_OFST 12 +/* Pattern to use in the soak test */ +#define MC_CMD_FC_IN_DIAG_DDR_SOAK_START_TEST_PATTERN_OFST 16 +#define MC_CMD_FC_IN_DIAG_DDR_SOAK_START_ZEROS 0x0 /* enum */ +#define MC_CMD_FC_IN_DIAG_DDR_SOAK_START_ONES 0x1 /* enum */ +/* Either multiple automatic tests until a STOP command is issued, or one + * single test + */ +#define MC_CMD_FC_IN_DIAG_DDR_SOAK_START_TEST_TYPE_OFST 20 +#define MC_CMD_FC_IN_DIAG_DDR_SOAK_START_ONGOING_TEST 0x0 /* enum */ +#define MC_CMD_FC_IN_DIAG_DDR_SOAK_START_SINGLE_TEST 0x1 /* enum */ + +/* MC_CMD_FC_IN_DIAG_DDR_SOAK_RESULT msgrequest */ +#define MC_CMD_FC_IN_DIAG_DDR_SOAK_RESULT_LEN 16 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +#define MC_CMD_FC_IN_DIAG_DDR_SOAK_RESULT_OP_OFST 4 +#define MC_CMD_FC_IN_DIAG_DDR_SOAK_RESULT_SUB_OP_OFST 8 +/* DDR bank to read status from */ +#define MC_CMD_FC_IN_DIAG_DDR_SOAK_RESULT_BANK_ID_OFST 12 +#define MC_CMD_FC_DDR_BANK0 0x0 /* enum */ +#define MC_CMD_FC_DDR_BANK1 0x1 /* enum */ +#define MC_CMD_FC_DDR_BANK2 0x2 /* enum */ +#define MC_CMD_FC_DDR_BANK3 0x3 /* enum */ +#define MC_CMD_FC_DDR_AOEMEM_MAX_BANKS 0x4 /* enum */ + +/* MC_CMD_FC_IN_DIAG_DDR_SOAK_STOP msgrequest */ +#define MC_CMD_FC_IN_DIAG_DDR_SOAK_STOP_LEN 16 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +#define MC_CMD_FC_IN_DIAG_DDR_SOAK_STOP_OP_OFST 4 +#define MC_CMD_FC_IN_DIAG_DDR_SOAK_STOP_SUB_OP_OFST 8 +/* Mask of DDR banks to be tested */ +#define MC_CMD_FC_IN_DIAG_DDR_SOAK_STOP_BANK_MASK_OFST 12 + +/* MC_CMD_FC_IN_DIAG_DDR_SOAK_ERROR msgrequest */ +#define MC_CMD_FC_IN_DIAG_DDR_SOAK_ERROR_LEN 20 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +#define MC_CMD_FC_IN_DIAG_DDR_SOAK_ERROR_OP_OFST 4 +#define MC_CMD_FC_IN_DIAG_DDR_SOAK_ERROR_SUB_OP_OFST 8 +/* Mask of DDR banks to set/clear error flag on */ +#define MC_CMD_FC_IN_DIAG_DDR_SOAK_ERROR_BANK_MASK_OFST 12 +#define MC_CMD_FC_IN_DIAG_DDR_SOAK_ERROR_FLAG_ACTION_OFST 16 +#define MC_CMD_FC_IN_DIAG_DDR_SOAK_ERROR_CLEAR 0x0 /* enum */ +#define MC_CMD_FC_IN_DIAG_DDR_SOAK_ERROR_SET 0x1 /* enum */ + +/* MC_CMD_FC_IN_DIAG_DATAPATH_CTRL msgrequest */ +#define MC_CMD_FC_IN_DIAG_DATAPATH_CTRL_LEN 12 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +#define MC_CMD_FC_IN_DIAG_DATAPATH_CTRL_OP_OFST 4 +/* Sub-opcode describing the operation to be carried out */ +#define MC_CMD_FC_IN_DIAG_DATAPATH_CTRL_SUB_OP_OFST 8 +/* enum: Set a known datapath configuration */ +#define MC_CMD_FC_IN_DIAG_DATAPATH_CTRL_SET_MODE 0x0 +/* enum: Apply raw config to datapath control registers */ +#define MC_CMD_FC_IN_DIAG_DATAPATH_CTRL_RAW_CONFIG 0x1 + +/* MC_CMD_FC_IN_DIAG_DATAPATH_CTRL_SET_MODE msgrequest */ +#define MC_CMD_FC_IN_DIAG_DATAPATH_CTRL_SET_MODE_LEN 16 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +#define MC_CMD_FC_IN_DIAG_DATAPATH_CTRL_SET_MODE_OP_OFST 4 +#define MC_CMD_FC_IN_DIAG_DATAPATH_CTRL_SET_MODE_SUB_OP_OFST 8 +/* Datapath configuration identifier */ +#define MC_CMD_FC_IN_DIAG_DATAPATH_CTRL_SET_MODE_MODE_OFST 12 +#define MC_CMD_FC_IN_DIAG_DATAPATH_CTRL_SET_MODE_PASSTHROUGH 0x0 /* enum */ +#define MC_CMD_FC_IN_DIAG_DATAPATH_CTRL_SET_MODE_SNAKE 0x1 /* enum */ + +/* MC_CMD_FC_IN_DIAG_DATAPATH_CTRL_RAW_CONFIG msgrequest */ +#define MC_CMD_FC_IN_DIAG_DATAPATH_CTRL_RAW_CONFIG_LEN 24 +/* MC_CMD_FC_IN_CMD_OFST 0 */ +#define MC_CMD_FC_IN_DIAG_DATAPATH_CTRL_RAW_CONFIG_OP_OFST 4 +#define MC_CMD_FC_IN_DIAG_DATAPATH_CTRL_RAW_CONFIG_SUB_OP_OFST 8 +/* Value to write into control register 1 */ +#define MC_CMD_FC_IN_DIAG_DATAPATH_CTRL_RAW_CONFIG_CONTROL1_OFST 12 +/* Value to write into control register 2 */ +#define MC_CMD_FC_IN_DIAG_DATAPATH_CTRL_RAW_CONFIG_CONTROL2_OFST 16 +/* Value to write into control register 3 */ +#define MC_CMD_FC_IN_DIAG_DATAPATH_CTRL_RAW_CONFIG_CONTROL3_OFST 20 + +/* MC_CMD_FC_OUT msgresponse */ +#define MC_CMD_FC_OUT_LEN 0 + +/* MC_CMD_FC_OUT_NULL msgresponse */ +#define MC_CMD_FC_OUT_NULL_LEN 0 + +/* MC_CMD_FC_OUT_READ32 msgresponse */ +#define MC_CMD_FC_OUT_READ32_LENMIN 4 +#define MC_CMD_FC_OUT_READ32_LENMAX 252 +#define MC_CMD_FC_OUT_READ32_LEN(num) (0+4*(num)) +#define MC_CMD_FC_OUT_READ32_BUFFER_OFST 0 +#define MC_CMD_FC_OUT_READ32_BUFFER_LEN 4 +#define MC_CMD_FC_OUT_READ32_BUFFER_MINNUM 1 +#define MC_CMD_FC_OUT_READ32_BUFFER_MAXNUM 63 + +/* MC_CMD_FC_OUT_WRITE32 msgresponse */ +#define MC_CMD_FC_OUT_WRITE32_LEN 0 + +/* MC_CMD_FC_OUT_TRC_READ msgresponse */ +#define MC_CMD_FC_OUT_TRC_READ_LEN 16 +#define MC_CMD_FC_OUT_TRC_READ_DATA_OFST 0 +#define MC_CMD_FC_OUT_TRC_READ_DATA_LEN 4 +#define MC_CMD_FC_OUT_TRC_READ_DATA_NUM 4 + +/* MC_CMD_FC_OUT_TRC_WRITE msgresponse */ +#define MC_CMD_FC_OUT_TRC_WRITE_LEN 0 + +/* MC_CMD_FC_OUT_GET_VERSION msgresponse */ +#define MC_CMD_FC_OUT_GET_VERSION_LEN 12 +#define MC_CMD_FC_OUT_GET_VERSION_FIRMWARE_OFST 0 +#define MC_CMD_FC_OUT_GET_VERSION_VERSION_OFST 4 +#define MC_CMD_FC_OUT_GET_VERSION_VERSION_LEN 8 +#define MC_CMD_FC_OUT_GET_VERSION_VERSION_LO_OFST 4 +#define MC_CMD_FC_OUT_GET_VERSION_VERSION_HI_OFST 8 + +/* MC_CMD_FC_OUT_TRC_RX_READ msgresponse */ +#define MC_CMD_FC_OUT_TRC_RX_READ_LEN 8 +#define MC_CMD_FC_OUT_TRC_RX_READ_DATA_OFST 0 +#define MC_CMD_FC_OUT_TRC_RX_READ_DATA_LEN 4 +#define MC_CMD_FC_OUT_TRC_RX_READ_DATA_NUM 2 + +/* MC_CMD_FC_OUT_TRC_RX_WRITE msgresponse */ +#define MC_CMD_FC_OUT_TRC_RX_WRITE_LEN 0 + +/* MC_CMD_FC_OUT_MAC_RECONFIGURE msgresponse */ +#define MC_CMD_FC_OUT_MAC_RECONFIGURE_LEN 0 + +/* MC_CMD_FC_OUT_MAC_SET_LINK msgresponse */ +#define MC_CMD_FC_OUT_MAC_SET_LINK_LEN 0 + +/* MC_CMD_FC_OUT_MAC_READ_STATUS msgresponse */ +#define MC_CMD_FC_OUT_MAC_READ_STATUS_LEN 4 +#define MC_CMD_FC_OUT_MAC_READ_STATUS_STATUS_OFST 0 + +/* MC_CMD_FC_OUT_MAC_GET_RX_STATS msgresponse */ +#define MC_CMD_FC_OUT_MAC_GET_RX_STATS_LEN ((((0-1+(64*MC_CMD_FC_MAC_RX_NSTATS))+1))>>3) +#define MC_CMD_FC_OUT_MAC_GET_RX_STATS_STATISTICS_OFST 0 +#define MC_CMD_FC_OUT_MAC_GET_RX_STATS_STATISTICS_LEN 8 +#define MC_CMD_FC_OUT_MAC_GET_RX_STATS_STATISTICS_LO_OFST 0 +#define MC_CMD_FC_OUT_MAC_GET_RX_STATS_STATISTICS_HI_OFST 4 +#define MC_CMD_FC_OUT_MAC_GET_RX_STATS_STATISTICS_NUM MC_CMD_FC_MAC_RX_NSTATS +#define MC_CMD_FC_MAC_RX_STATS_OCTETS 0x0 /* enum */ +#define MC_CMD_FC_MAC_RX_OCTETS_OK 0x1 /* enum */ +#define MC_CMD_FC_MAC_RX_ALIGNMENT_ERRORS 0x2 /* enum */ +#define MC_CMD_FC_MAC_RX_PAUSE_MAC_CTRL_FRAMES 0x3 /* enum */ +#define MC_CMD_FC_MAC_RX_FRAMES_OK 0x4 /* enum */ +#define MC_CMD_FC_MAC_RX_CRC_ERRORS 0x5 /* enum */ +#define MC_CMD_FC_MAC_RX_VLAN_OK 0x6 /* enum */ +#define MC_CMD_FC_MAC_RX_ERRORS 0x7 /* enum */ +#define MC_CMD_FC_MAC_RX_UCAST_PKTS 0x8 /* enum */ +#define MC_CMD_FC_MAC_RX_MULTICAST_PKTS 0x9 /* enum */ +#define MC_CMD_FC_MAC_RX_BROADCAST_PKTS 0xa /* enum */ +#define MC_CMD_FC_MAC_RX_STATS_DROP_EVENTS 0xb /* enum */ +#define MC_CMD_FC_MAC_RX_STATS_PKTS 0xc /* enum */ +#define MC_CMD_FC_MAC_RX_STATS_UNDERSIZE_PKTS 0xd /* enum */ +#define MC_CMD_FC_MAC_RX_STATS_PKTS_64 0xe /* enum */ +#define MC_CMD_FC_MAC_RX_STATS_PKTS_65_127 0xf /* enum */ +#define MC_CMD_FC_MAC_RX_STATS_PKTS_128_255 0x10 /* enum */ +#define MC_CMD_FC_MAC_RX_STATS_PKTS_256_511 0x11 /* enum */ +#define MC_CMD_FC_MAC_RX_STATS_PKTS_512_1023 0x12 /* enum */ +#define MC_CMD_FC_MAC_RX_STATS_PKTS_1024_1518 0x13 /* enum */ +#define MC_CMD_FC_MAC_RX_STATS_PKTS_1519_MAX 0x14 /* enum */ +#define MC_CMD_FC_MAC_RX_STATS_OVERSIZE_PKTS 0x15 /* enum */ +#define MC_CMD_FC_MAC_RX_STATS_JABBERS 0x16 /* enum */ +#define MC_CMD_FC_MAC_RX_STATS_FRAGMENTS 0x17 /* enum */ +#define MC_CMD_FC_MAC_RX_MAC_CONTROL_FRAMES 0x18 /* enum */ +/* enum: (Last entry) */ +#define MC_CMD_FC_MAC_RX_NSTATS 0x19 + +/* MC_CMD_FC_OUT_MAC_GET_TX_STATS msgresponse */ +#define MC_CMD_FC_OUT_MAC_GET_TX_STATS_LEN ((((0-1+(64*MC_CMD_FC_MAC_TX_NSTATS))+1))>>3) +#define MC_CMD_FC_OUT_MAC_GET_TX_STATS_STATISTICS_OFST 0 +#define MC_CMD_FC_OUT_MAC_GET_TX_STATS_STATISTICS_LEN 8 +#define MC_CMD_FC_OUT_MAC_GET_TX_STATS_STATISTICS_LO_OFST 0 +#define MC_CMD_FC_OUT_MAC_GET_TX_STATS_STATISTICS_HI_OFST 4 +#define MC_CMD_FC_OUT_MAC_GET_TX_STATS_STATISTICS_NUM MC_CMD_FC_MAC_TX_NSTATS +#define MC_CMD_FC_MAC_TX_STATS_OCTETS 0x0 /* enum */ +#define MC_CMD_FC_MAC_TX_OCTETS_OK 0x1 /* enum */ +#define MC_CMD_FC_MAC_TX_ALIGNMENT_ERRORS 0x2 /* enum */ +#define MC_CMD_FC_MAC_TX_PAUSE_MAC_CTRL_FRAMES 0x3 /* enum */ +#define MC_CMD_FC_MAC_TX_FRAMES_OK 0x4 /* enum */ +#define MC_CMD_FC_MAC_TX_CRC_ERRORS 0x5 /* enum */ +#define MC_CMD_FC_MAC_TX_VLAN_OK 0x6 /* enum */ +#define MC_CMD_FC_MAC_TX_ERRORS 0x7 /* enum */ +#define MC_CMD_FC_MAC_TX_UCAST_PKTS 0x8 /* enum */ +#define MC_CMD_FC_MAC_TX_MULTICAST_PKTS 0x9 /* enum */ +#define MC_CMD_FC_MAC_TX_BROADCAST_PKTS 0xa /* enum */ +#define MC_CMD_FC_MAC_TX_STATS_DROP_EVENTS 0xb /* enum */ +#define MC_CMD_FC_MAC_TX_STATS_PKTS 0xc /* enum */ +#define MC_CMD_FC_MAC_TX_STATS_UNDERSIZE_PKTS 0xd /* enum */ +#define MC_CMD_FC_MAC_TX_STATS_PKTS_64 0xe /* enum */ +#define MC_CMD_FC_MAC_TX_STATS_PKTS_65_127 0xf /* enum */ +#define MC_CMD_FC_MAC_TX_STATS_PKTS_128_255 0x10 /* enum */ +#define MC_CMD_FC_MAC_TX_STATS_PKTS_256_511 0x11 /* enum */ +#define MC_CMD_FC_MAC_TX_STATS_PKTS_512_1023 0x12 /* enum */ +#define MC_CMD_FC_MAC_TX_STATS_PKTS_1024_1518 0x13 /* enum */ +#define MC_CMD_FC_MAC_TX_STATS_PKTS_1519_TX_MTU 0x14 /* enum */ +#define MC_CMD_FC_MAC_TX_MAC_CONTROL_FRAMES 0x15 /* enum */ +/* enum: (Last entry) */ +#define MC_CMD_FC_MAC_TX_NSTATS 0x16 + +/* MC_CMD_FC_OUT_MAC_GET_STATS msgresponse */ +#define MC_CMD_FC_OUT_MAC_GET_STATS_LEN ((((0-1+(64*MC_CMD_FC_MAC_NSTATS_PER_BLOCK))+1))>>3) +/* MAC Statistics */ +#define MC_CMD_FC_OUT_MAC_GET_STATS_STATISTICS_OFST 0 +#define MC_CMD_FC_OUT_MAC_GET_STATS_STATISTICS_LEN 8 +#define MC_CMD_FC_OUT_MAC_GET_STATS_STATISTICS_LO_OFST 0 +#define MC_CMD_FC_OUT_MAC_GET_STATS_STATISTICS_HI_OFST 4 +#define MC_CMD_FC_OUT_MAC_GET_STATS_STATISTICS_NUM MC_CMD_FC_MAC_NSTATS_PER_BLOCK + +/* MC_CMD_FC_OUT_MAC msgresponse */ +#define MC_CMD_FC_OUT_MAC_LEN 0 + +/* MC_CMD_FC_OUT_SFP msgresponse */ +#define MC_CMD_FC_OUT_SFP_LEN 0 + +/* MC_CMD_FC_OUT_DDR_TEST_START msgresponse */ +#define MC_CMD_FC_OUT_DDR_TEST_START_LEN 0 + +/* MC_CMD_FC_OUT_DDR_TEST_POLL msgresponse */ +#define MC_CMD_FC_OUT_DDR_TEST_POLL_LEN 8 +#define MC_CMD_FC_OUT_DDR_TEST_POLL_STATUS_OFST 0 +#define MC_CMD_FC_OUT_DDR_TEST_POLL_CODE_LBN 0 +#define MC_CMD_FC_OUT_DDR_TEST_POLL_CODE_WIDTH 8 +/* enum: Test not yet initiated */ +#define MC_CMD_FC_OP_DDR_TEST_NONE 0x0 +/* enum: Test is in progress */ +#define MC_CMD_FC_OP_DDR_TEST_INPROGRESS 0x1 +/* enum: Timed completed */ +#define MC_CMD_FC_OP_DDR_TEST_SUCCESS 0x2 +/* enum: Test did not complete in specified time */ +#define MC_CMD_FC_OP_DDR_TEST_TIMER_EXPIRED 0x3 +#define MC_CMD_FC_OUT_DDR_TEST_POLL_PRESENT_T0_LBN 11 +#define MC_CMD_FC_OUT_DDR_TEST_POLL_PRESENT_T0_WIDTH 1 +#define MC_CMD_FC_OUT_DDR_TEST_POLL_PRESENT_T1_LBN 10 +#define MC_CMD_FC_OUT_DDR_TEST_POLL_PRESENT_T1_WIDTH 1 +#define MC_CMD_FC_OUT_DDR_TEST_POLL_PRESENT_B0_LBN 9 +#define MC_CMD_FC_OUT_DDR_TEST_POLL_PRESENT_B0_WIDTH 1 +#define MC_CMD_FC_OUT_DDR_TEST_POLL_PRESENT_B1_LBN 8 +#define MC_CMD_FC_OUT_DDR_TEST_POLL_PRESENT_B1_WIDTH 1 +/* Test result from FPGA */ +#define MC_CMD_FC_OUT_DDR_TEST_POLL_RESULT_OFST 4 +#define MC_CMD_FC_OUT_DDR_TEST_POLL_FPGA_SUPPORTS_T0_LBN 31 +#define MC_CMD_FC_OUT_DDR_TEST_POLL_FPGA_SUPPORTS_T0_WIDTH 1 +#define MC_CMD_FC_OUT_DDR_TEST_POLL_FPGA_SUPPORTS_T1_LBN 30 +#define MC_CMD_FC_OUT_DDR_TEST_POLL_FPGA_SUPPORTS_T1_WIDTH 1 +#define MC_CMD_FC_OUT_DDR_TEST_POLL_FPGA_SUPPORTS_B0_LBN 29 +#define MC_CMD_FC_OUT_DDR_TEST_POLL_FPGA_SUPPORTS_B0_WIDTH 1 +#define MC_CMD_FC_OUT_DDR_TEST_POLL_FPGA_SUPPORTS_B1_LBN 28 +#define MC_CMD_FC_OUT_DDR_TEST_POLL_FPGA_SUPPORTS_B1_WIDTH 1 +#define MC_CMD_FC_OUT_DDR_TEST_POLL_T0_LBN 15 +#define MC_CMD_FC_OUT_DDR_TEST_POLL_T0_WIDTH 5 +#define MC_CMD_FC_OUT_DDR_TEST_POLL_T1_LBN 10 +#define MC_CMD_FC_OUT_DDR_TEST_POLL_T1_WIDTH 5 +#define MC_CMD_FC_OUT_DDR_TEST_POLL_B0_LBN 5 +#define MC_CMD_FC_OUT_DDR_TEST_POLL_B0_WIDTH 5 +#define MC_CMD_FC_OUT_DDR_TEST_POLL_B1_LBN 0 +#define MC_CMD_FC_OUT_DDR_TEST_POLL_B1_WIDTH 5 +#define MC_CMD_FC_OUT_DDR_TEST_POLL_TEST_COMPLETE 0x0 /* enum */ +#define MC_CMD_FC_OUT_DDR_TEST_POLL_TEST_FAIL 0x1 /* enum */ +#define MC_CMD_FC_OUT_DDR_TEST_POLL_TEST_PASS 0x2 /* enum */ +#define MC_CMD_FC_OUT_DDR_TEST_POLL_CAL_FAIL 0x3 /* enum */ +#define MC_CMD_FC_OUT_DDR_TEST_POLL_CAL_SUCCESS 0x4 /* enum */ + +/* MC_CMD_FC_OUT_DDR_TEST msgresponse */ +#define MC_CMD_FC_OUT_DDR_TEST_LEN 0 + +/* MC_CMD_FC_OUT_GET_ASSERT msgresponse */ +#define MC_CMD_FC_OUT_GET_ASSERT_LEN 144 +/* Assertion status flag. */ +#define MC_CMD_FC_OUT_GET_ASSERT_GLOBAL_FLAGS_OFST 0 +#define MC_CMD_FC_OUT_GET_ASSERT_STATE_LBN 8 +#define MC_CMD_FC_OUT_GET_ASSERT_STATE_WIDTH 8 +/* enum: No crash data available */ +#define MC_CMD_FC_GET_ASSERT_FLAGS_STATE_CLEAR 0x0 +/* enum: New crash data available */ +#define MC_CMD_FC_GET_ASSERT_FLAGS_STATE_NEW 0x1 +/* enum: Crash data has been sent */ +#define MC_CMD_FC_GET_ASSERT_FLAGS_STATE_NOTIFIED 0x2 +#define MC_CMD_FC_OUT_GET_ASSERT_TYPE_LBN 0 +#define MC_CMD_FC_OUT_GET_ASSERT_TYPE_WIDTH 8 +/* enum: No crash has been recorded. */ +#define MC_CMD_FC_GET_ASSERT_FLAGS_TYPE_NONE 0x0 +/* enum: Crash due to exception. */ +#define MC_CMD_FC_GET_ASSERT_FLAGS_TYPE_EXCEPTION 0x1 +/* enum: Crash due to assertion. */ +#define MC_CMD_FC_GET_ASSERT_FLAGS_TYPE_ASSERTION 0x2 +/* Failing PC value */ +#define MC_CMD_FC_OUT_GET_ASSERT_SAVED_PC_OFFS_OFST 4 +/* Saved GP regs */ +#define MC_CMD_FC_OUT_GET_ASSERT_GP_REGS_OFFS_OFST 8 +#define MC_CMD_FC_OUT_GET_ASSERT_GP_REGS_OFFS_LEN 4 +#define MC_CMD_FC_OUT_GET_ASSERT_GP_REGS_OFFS_NUM 31 +/* Exception Type */ +#define MC_CMD_FC_OUT_GET_ASSERT_EXCEPTION_TYPE_OFFS_OFST 132 +/* Instruction at which exception occurred */ +#define MC_CMD_FC_OUT_GET_ASSERT_EXCEPTION_PC_ADDR_OFFS_OFST 136 +/* BAD Address that triggered address-based exception */ +#define MC_CMD_FC_OUT_GET_ASSERT_EXCEPTION_BAD_ADDR_OFFS_OFST 140 + +/* MC_CMD_FC_OUT_FPGA_BUILD msgresponse */ +#define MC_CMD_FC_OUT_FPGA_BUILD_LEN 32 +#define MC_CMD_FC_OUT_FPGA_BUILD_COMPONENT_INFO_OFST 0 +#define MC_CMD_FC_OUT_FPGA_BUILD_IS_APPLICATION_LBN 31 +#define MC_CMD_FC_OUT_FPGA_BUILD_IS_APPLICATION_WIDTH 1 +#define MC_CMD_FC_OUT_FPGA_BUILD_IS_LICENSED_LBN 30 +#define MC_CMD_FC_OUT_FPGA_BUILD_IS_LICENSED_WIDTH 1 +#define MC_CMD_FC_OUT_FPGA_BUILD_COMPONENT_ID_LBN 16 +#define MC_CMD_FC_OUT_FPGA_BUILD_COMPONENT_ID_WIDTH 14 +#define MC_CMD_FC_OUT_FPGA_BUILD_VERSION_MAJOR_LBN 12 +#define MC_CMD_FC_OUT_FPGA_BUILD_VERSION_MAJOR_WIDTH 4 +#define MC_CMD_FC_OUT_FPGA_BUILD_VERSION_MINOR_LBN 4 +#define MC_CMD_FC_OUT_FPGA_BUILD_VERSION_MINOR_WIDTH 8 +#define MC_CMD_FC_OUT_FPGA_BUILD_BUILD_NUM_LBN 0 +#define MC_CMD_FC_OUT_FPGA_BUILD_BUILD_NUM_WIDTH 4 +/* Build timestamp (seconds since epoch) */ +#define MC_CMD_FC_OUT_FPGA_BUILD_TIMESTAMP_OFST 4 +#define MC_CMD_FC_OUT_FPGA_BUILD_PARAMETERS_OFST 8 +#define MC_CMD_FC_OUT_FPGA_BUILD_FPGA_TYPE_LBN 0 +#define MC_CMD_FC_OUT_FPGA_BUILD_FPGA_TYPE_WIDTH 8 +#define MC_CMD_FC_FPGA_TYPE_A7 0xa7 /* enum */ +#define MC_CMD_FC_FPGA_TYPE_A5 0xa5 /* enum */ +#define MC_CMD_FC_OUT_FPGA_BUILD_RESERVED1_LBN 8 +#define MC_CMD_FC_OUT_FPGA_BUILD_RESERVED1_WIDTH 10 +#define MC_CMD_FC_OUT_FPGA_BUILD_PTP_ENABLED_LBN 18 +#define MC_CMD_FC_OUT_FPGA_BUILD_PTP_ENABLED_WIDTH 1 +#define MC_CMD_FC_OUT_FPGA_BUILD_SODIMM1_RLDRAM_DEF_LBN 19 +#define MC_CMD_FC_OUT_FPGA_BUILD_SODIMM1_RLDRAM_DEF_WIDTH 1 +#define MC_CMD_FC_OUT_FPGA_BUILD_SODIMM2_RLDRAM_DEF_LBN 20 +#define MC_CMD_FC_OUT_FPGA_BUILD_SODIMM2_RLDRAM_DEF_WIDTH 1 +#define MC_CMD_FC_OUT_FPGA_BUILD_SODIMM3_RLDRAM_DEF_LBN 21 +#define MC_CMD_FC_OUT_FPGA_BUILD_SODIMM3_RLDRAM_DEF_WIDTH 1 +#define MC_CMD_FC_OUT_FPGA_BUILD_SODIMM4_RLDRAM_DEF_LBN 22 +#define MC_CMD_FC_OUT_FPGA_BUILD_SODIMM4_RLDRAM_DEF_WIDTH 1 +#define MC_CMD_FC_OUT_FPGA_BUILD_SODIMM_T0_DDR3_DEF_LBN 23 +#define MC_CMD_FC_OUT_FPGA_BUILD_SODIMM_T0_DDR3_DEF_WIDTH 1 +#define MC_CMD_FC_OUT_FPGA_BUILD_SODIMM_T1_DDR3_DEF_LBN 24 +#define MC_CMD_FC_OUT_FPGA_BUILD_SODIMM_T1_DDR3_DEF_WIDTH 1 +#define MC_CMD_FC_OUT_FPGA_BUILD_SODIMM_B0_DDR3_DEF_LBN 25 +#define MC_CMD_FC_OUT_FPGA_BUILD_SODIMM_B0_DDR3_DEF_WIDTH 1 +#define MC_CMD_FC_OUT_FPGA_BUILD_SODIMM_B1_DDR3_DEF_LBN 26 +#define MC_CMD_FC_OUT_FPGA_BUILD_SODIMM_B1_DDR3_DEF_WIDTH 1 +#define MC_CMD_FC_OUT_FPGA_BUILD_DDR3_ECC_ENABLED_LBN 27 +#define MC_CMD_FC_OUT_FPGA_BUILD_DDR3_ECC_ENABLED_WIDTH 1 +#define MC_CMD_FC_OUT_FPGA_BUILD_SODIMM_T1_QDR_DEF_LBN 28 +#define MC_CMD_FC_OUT_FPGA_BUILD_SODIMM_T1_QDR_DEF_WIDTH 1 +#define MC_CMD_FC_OUT_FPGA_BUILD_RESERVED2_LBN 29 +#define MC_CMD_FC_OUT_FPGA_BUILD_RESERVED2_WIDTH 2 +#define MC_CMD_FC_OUT_FPGA_BUILD_CRC_APPEND_LBN 31 +#define MC_CMD_FC_OUT_FPGA_BUILD_CRC_APPEND_WIDTH 1 +#define MC_CMD_FC_OUT_FPGA_BUILD_IDENTIFIER_OFST 12 +#define MC_CMD_FC_OUT_FPGA_BUILD_CHANGESET_LBN 0 +#define MC_CMD_FC_OUT_FPGA_BUILD_CHANGESET_WIDTH 16 +#define MC_CMD_FC_OUT_FPGA_BUILD_BUILD_FLAG_LBN 16 +#define MC_CMD_FC_OUT_FPGA_BUILD_BUILD_FLAG_WIDTH 1 +#define MC_CMD_FC_FPGA_BUILD_FLAG_INTERNAL 0x0 /* enum */ +#define MC_CMD_FC_FPGA_BUILD_FLAG_RELEASE 0x1 /* enum */ +#define MC_CMD_FC_OUT_FPGA_BUILD_RESERVED3_LBN 17 +#define MC_CMD_FC_OUT_FPGA_BUILD_RESERVED3_WIDTH 15 +#define MC_CMD_FC_OUT_FPGA_BUILD_VERSION_HI_OFST 16 +#define MC_CMD_FC_OUT_FPGA_BUILD_DEPLOYMENT_VERSION_MINOR_LBN 0 +#define MC_CMD_FC_OUT_FPGA_BUILD_DEPLOYMENT_VERSION_MINOR_WIDTH 16 +#define MC_CMD_FC_OUT_FPGA_BUILD_DEPLOYMENT_VERSION_MAJOR_LBN 16 +#define MC_CMD_FC_OUT_FPGA_BUILD_DEPLOYMENT_VERSION_MAJOR_WIDTH 16 +#define MC_CMD_FC_OUT_FPGA_BUILD_VERSION_LO_OFST 20 +#define MC_CMD_FC_OUT_FPGA_BUILD_DEPLOYMENT_VERSION_BUILD_LBN 0 +#define MC_CMD_FC_OUT_FPGA_BUILD_DEPLOYMENT_VERSION_BUILD_WIDTH 16 +#define MC_CMD_FC_OUT_FPGA_BUILD_DEPLOYMENT_VERSION_MICRO_LBN 16 +#define MC_CMD_FC_OUT_FPGA_BUILD_DEPLOYMENT_VERSION_MICRO_WIDTH 16 +#define MC_CMD_FC_OUT_FPGA_BUILD_RESERVED4_OFST 16 +#define MC_CMD_FC_OUT_FPGA_BUILD_RESERVED4_LEN 8 +#define MC_CMD_FC_OUT_FPGA_BUILD_RESERVED4_LO_OFST 16 +#define MC_CMD_FC_OUT_FPGA_BUILD_RESERVED4_HI_OFST 20 +#define MC_CMD_FC_OUT_FPGA_BUILD_REVISION_LO_OFST 24 +#define MC_CMD_FC_OUT_FPGA_BUILD_REVISION_HI_OFST 28 +#define MC_CMD_FC_OUT_FPGA_BUILD_REVISION_HIGH_LBN 0 +#define MC_CMD_FC_OUT_FPGA_BUILD_REVISION_HIGH_WIDTH 16 + +/* MC_CMD_FC_OUT_FPGA_BUILD_V2 msgresponse */ +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_LEN 32 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_COMPONENT_INFO_OFST 0 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_IS_APPLICATION_LBN 31 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_IS_APPLICATION_WIDTH 1 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_IS_LICENSED_LBN 30 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_IS_LICENSED_WIDTH 1 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_COMPONENT_ID_LBN 16 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_COMPONENT_ID_WIDTH 14 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_VERSION_MAJOR_LBN 12 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_VERSION_MAJOR_WIDTH 4 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_VERSION_MINOR_LBN 4 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_VERSION_MINOR_WIDTH 8 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_BUILD_NUM_LBN 0 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_BUILD_NUM_WIDTH 4 +/* Build timestamp (seconds since epoch) */ +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_TIMESTAMP_OFST 4 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_PARAMETERS_OFST 8 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_PMA_PASSTHROUGH_LBN 31 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_PMA_PASSTHROUGH_WIDTH 1 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_SODIMM2_QDR_DEF_LBN 29 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_SODIMM2_QDR_DEF_WIDTH 1 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_SODIMM1_QDR_DEF_LBN 28 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_SODIMM1_QDR_DEF_WIDTH 1 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_DDR3_ECC_ENABLED_LBN 27 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_DDR3_ECC_ENABLED_WIDTH 1 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_DISCRETE2_DDR3_DEF_LBN 26 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_DISCRETE2_DDR3_DEF_WIDTH 1 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_DISCRETE1_DDR3_DEF_LBN 25 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_DISCRETE1_DDR3_DEF_WIDTH 1 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_SODIMM2_TO_DDR3_DEF_LBN 24 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_SODIMM2_TO_DDR3_DEF_WIDTH 1 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_SODIMM1_T0_DDR3_DEF_LBN 23 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_SODIMM1_T0_DDR3_DEF_WIDTH 1 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_DISCRETE2_RLDRAM_DEF_LBN 22 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_DISCRETE2_RLDRAM_DEF_WIDTH 1 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_DISCRETE1_RLDRAM_DEF_LBN 21 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_DISCRETE1_RLDRAM_DEF_WIDTH 1 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_SODIMM2_RLDRAM_DEF_LBN 20 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_SODIMM2_RLDRAM_DEF_WIDTH 1 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_SODIMM1_RLDRAM_DEF_LBN 19 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_SODIMM1_RLDRAM_DEF_WIDTH 1 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_NIC0_3_SPEED_LBN 18 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_NIC0_3_SPEED_WIDTH 1 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_NIC0_3_SPEED_10G 0x0 /* enum */ +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_NIC0_3_SPEED_40G 0x1 /* enum */ +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_SFP4_7_SPEED_LBN 17 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_SFP4_7_SPEED_WIDTH 1 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_SFP4_7_SPEED_10G 0x0 /* enum */ +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_SFP4_7_SPEED_40G 0x1 /* enum */ +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_SFP0_3_SPEED_LBN 16 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_SFP0_3_SPEED_WIDTH 1 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_SFP0_3_SPEED_10G 0x0 /* enum */ +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_SFP0_3_SPEED_40G 0x1 /* enum */ +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_SFP7_DEF_LBN 15 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_SFP7_DEF_WIDTH 1 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_SFP6_DEF_LBN 14 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_SFP6_DEF_WIDTH 1 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_SFP5_DEF_LBN 13 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_SFP5_DEF_WIDTH 1 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_SFP4_DEF_LBN 12 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_SFP4_DEF_WIDTH 1 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_SFP3_DEF_LBN 11 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_SFP3_DEF_WIDTH 1 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_SFP2_DEF_LBN 10 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_SFP2_DEF_WIDTH 1 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_SFP1_DEF_LBN 9 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_SFP1_DEF_WIDTH 1 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_SFP0_DEF_LBN 8 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_SFP0_DEF_WIDTH 1 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_NIC3_DEF_LBN 7 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_NIC3_DEF_WIDTH 1 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_NIC2_DEF_LBN 6 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_NIC2_DEF_WIDTH 1 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_NIC1_DEF_LBN 5 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_NIC1_DEF_WIDTH 1 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_NIC0_DEF_LBN 4 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_NIC0_DEF_WIDTH 1 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_FPGA_TYPE_LBN 0 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_FPGA_TYPE_WIDTH 4 +#define MC_CMD_FC_FPGA_V2_TYPE_A3 0x0 /* enum */ +#define MC_CMD_FC_FPGA_V2_TYPE_A4 0x1 /* enum */ +#define MC_CMD_FC_FPGA_V2_TYPE_A5 0x2 /* enum */ +#define MC_CMD_FC_FPGA_V2_TYPE_A7 0x3 /* enum */ +#define MC_CMD_FC_FPGA_V2_TYPE_D3 0x8 /* enum */ +#define MC_CMD_FC_FPGA_V2_TYPE_D4 0x9 /* enum */ +#define MC_CMD_FC_FPGA_V2_TYPE_D5 0xa /* enum */ +#define MC_CMD_FC_FPGA_V2_TYPE_D7 0xb /* enum */ +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_IDENTIFIER_OFST 12 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_CHANGESET_LBN 0 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_CHANGESET_WIDTH 16 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_BUILD_FLAG_LBN 16 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_BUILD_FLAG_WIDTH 1 +/* MC_CMD_FC_FPGA_BUILD_FLAG_INTERNAL 0x0 */ +/* MC_CMD_FC_FPGA_BUILD_FLAG_RELEASE 0x1 */ +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_VERSION_HI_OFST 16 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_DEPLOYMENT_VERSION_MINOR_LBN 0 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_DEPLOYMENT_VERSION_MINOR_WIDTH 16 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_DEPLOYMENT_VERSION_MAJOR_LBN 16 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_DEPLOYMENT_VERSION_MAJOR_WIDTH 16 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_VERSION_LO_OFST 20 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_DEPLOYMENT_VERSION_BUILD_LBN 0 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_DEPLOYMENT_VERSION_BUILD_WIDTH 16 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_DEPLOYMENT_VERSION_MICRO_LBN 16 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_DEPLOYMENT_VERSION_MICRO_WIDTH 16 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_REVISION_LO_OFST 24 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_REVISION_HI_OFST 28 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_REVISION_HIGH_LBN 0 +#define MC_CMD_FC_OUT_FPGA_BUILD_V2_REVISION_HIGH_WIDTH 16 + +/* MC_CMD_FC_OUT_FPGA_SERVICES msgresponse */ +#define MC_CMD_FC_OUT_FPGA_SERVICES_LEN 32 +#define MC_CMD_FC_OUT_FPGA_SERVICES_COMPONENT_INFO_OFST 0 +#define MC_CMD_FC_OUT_FPGA_SERVICES_IS_APPLICATION_LBN 31 +#define MC_CMD_FC_OUT_FPGA_SERVICES_IS_APPLICATION_WIDTH 1 +#define MC_CMD_FC_OUT_FPGA_SERVICES_IS_LICENSED_LBN 30 +#define MC_CMD_FC_OUT_FPGA_SERVICES_IS_LICENSED_WIDTH 1 +#define MC_CMD_FC_OUT_FPGA_SERVICES_COMPONENT_ID_LBN 16 +#define MC_CMD_FC_OUT_FPGA_SERVICES_COMPONENT_ID_WIDTH 14 +#define MC_CMD_FC_OUT_FPGA_SERVICES_VERSION_MAJOR_LBN 12 +#define MC_CMD_FC_OUT_FPGA_SERVICES_VERSION_MAJOR_WIDTH 4 +#define MC_CMD_FC_OUT_FPGA_SERVICES_VERSION_MINOR_LBN 4 +#define MC_CMD_FC_OUT_FPGA_SERVICES_VERSION_MINOR_WIDTH 8 +#define MC_CMD_FC_OUT_FPGA_SERVICES_BUILD_NUM_LBN 0 +#define MC_CMD_FC_OUT_FPGA_SERVICES_BUILD_NUM_WIDTH 4 +/* Build timestamp (seconds since epoch) */ +#define MC_CMD_FC_OUT_FPGA_SERVICES_TIMESTAMP_OFST 4 +#define MC_CMD_FC_OUT_FPGA_SERVICES_PARAMETERS_OFST 8 +#define MC_CMD_FC_OUT_FPGA_SERVICES_FC_FLASH_BOOTED_LBN 8 +#define MC_CMD_FC_OUT_FPGA_SERVICES_FC_FLASH_BOOTED_WIDTH 1 +#define MC_CMD_FC_OUT_FPGA_SERVICES_NIC0_DEF_LBN 27 +#define MC_CMD_FC_OUT_FPGA_SERVICES_NIC0_DEF_WIDTH 1 +#define MC_CMD_FC_OUT_FPGA_SERVICES_NIC1_DEF_LBN 28 +#define MC_CMD_FC_OUT_FPGA_SERVICES_NIC1_DEF_WIDTH 1 +#define MC_CMD_FC_OUT_FPGA_SERVICES_SFP0_DEF_LBN 29 +#define MC_CMD_FC_OUT_FPGA_SERVICES_SFP0_DEF_WIDTH 1 +#define MC_CMD_FC_OUT_FPGA_SERVICES_SFP1_DEF_LBN 30 +#define MC_CMD_FC_OUT_FPGA_SERVICES_SFP1_DEF_WIDTH 1 +#define MC_CMD_FC_OUT_FPGA_SERVICES_RESERVED_LBN 31 +#define MC_CMD_FC_OUT_FPGA_SERVICES_RESERVED_WIDTH 1 +#define MC_CMD_FC_OUT_FPGA_SERVICES_IDENTIFIER_OFST 12 +#define MC_CMD_FC_OUT_FPGA_SERVICES_CHANGESET_LBN 0 +#define MC_CMD_FC_OUT_FPGA_SERVICES_CHANGESET_WIDTH 16 +#define MC_CMD_FC_OUT_FPGA_SERVICES_BUILD_FLAG_LBN 16 +#define MC_CMD_FC_OUT_FPGA_SERVICES_BUILD_FLAG_WIDTH 1 +#define MC_CMD_FC_OUT_FPGA_SERVICES_MEMORY_SIZE_OFST 16 +#define MC_CMD_FC_OUT_FPGA_SERVICES_MEMORY_SIZE_WIDTH_LBN 0 +#define MC_CMD_FC_OUT_FPGA_SERVICES_MEMORY_SIZE_WIDTH_WIDTH 16 +#define MC_CMD_FC_OUT_FPGA_SERVICES_MEMORY_SIZE_COUNT_LBN 16 +#define MC_CMD_FC_OUT_FPGA_SERVICES_MEMORY_SIZE_COUNT_WIDTH 16 +#define MC_CMD_FC_OUT_FPGA_SERVICES_INSTANCE_SIZE_OFST 20 +#define MC_CMD_FC_OUT_FPGA_SERVICES_INSTANCE_SIZE_WIDTH_LBN 0 +#define MC_CMD_FC_OUT_FPGA_SERVICES_INSTANCE_SIZE_WIDTH_WIDTH 16 +#define MC_CMD_FC_OUT_FPGA_SERVICES_INSTANCE_SIZE_COUNT_LBN 16 +#define MC_CMD_FC_OUT_FPGA_SERVICES_INSTANCE_SIZE_COUNT_WIDTH 16 +#define MC_CMD_FC_OUT_FPGA_SERVICES_REVISION_LO_OFST 24 +#define MC_CMD_FC_OUT_FPGA_SERVICES_REVISION_HI_OFST 28 +#define MC_CMD_FC_OUT_FPGA_SERVICES_REVISION_HIGH_LBN 0 +#define MC_CMD_FC_OUT_FPGA_SERVICES_REVISION_HIGH_WIDTH 16 + +/* MC_CMD_FC_OUT_FPGA_SERVICES_V2 msgresponse */ +#define MC_CMD_FC_OUT_FPGA_SERVICES_V2_LEN 32 +#define MC_CMD_FC_OUT_FPGA_SERVICES_V2_COMPONENT_INFO_OFST 0 +#define MC_CMD_FC_OUT_FPGA_SERVICES_V2_IS_APPLICATION_LBN 31 +#define MC_CMD_FC_OUT_FPGA_SERVICES_V2_IS_APPLICATION_WIDTH 1 +#define MC_CMD_FC_OUT_FPGA_SERVICES_V2_IS_LICENSED_LBN 30 +#define MC_CMD_FC_OUT_FPGA_SERVICES_V2_IS_LICENSED_WIDTH 1 +#define MC_CMD_FC_OUT_FPGA_SERVICES_V2_COMPONENT_ID_LBN 16 +#define MC_CMD_FC_OUT_FPGA_SERVICES_V2_COMPONENT_ID_WIDTH 14 +#define MC_CMD_FC_OUT_FPGA_SERVICES_V2_VERSION_MAJOR_LBN 12 +#define MC_CMD_FC_OUT_FPGA_SERVICES_V2_VERSION_MAJOR_WIDTH 4 +#define MC_CMD_FC_OUT_FPGA_SERVICES_V2_VERSION_MINOR_LBN 4 +#define MC_CMD_FC_OUT_FPGA_SERVICES_V2_VERSION_MINOR_WIDTH 8 +#define MC_CMD_FC_OUT_FPGA_SERVICES_V2_BUILD_NUM_LBN 0 +#define MC_CMD_FC_OUT_FPGA_SERVICES_V2_BUILD_NUM_WIDTH 4 +/* Build timestamp (seconds since epoch) */ +#define MC_CMD_FC_OUT_FPGA_SERVICES_V2_TIMESTAMP_OFST 4 +#define MC_CMD_FC_OUT_FPGA_SERVICES_V2_PARAMETERS_OFST 8 +#define MC_CMD_FC_OUT_FPGA_SERVICES_V2_PTP_ENABLED_LBN 0 +#define MC_CMD_FC_OUT_FPGA_SERVICES_V2_PTP_ENABLED_WIDTH 1 +#define MC_CMD_FC_OUT_FPGA_SERVICES_V2_FC_FLASH_BOOTED_LBN 8 +#define MC_CMD_FC_OUT_FPGA_SERVICES_V2_FC_FLASH_BOOTED_WIDTH 1 +#define MC_CMD_FC_OUT_FPGA_SERVICES_V2_IDENTIFIER_OFST 12 +#define MC_CMD_FC_OUT_FPGA_SERVICES_V2_CHANGESET_LBN 0 +#define MC_CMD_FC_OUT_FPGA_SERVICES_V2_CHANGESET_WIDTH 16 +#define MC_CMD_FC_OUT_FPGA_SERVICES_V2_BUILD_FLAG_LBN 16 +#define MC_CMD_FC_OUT_FPGA_SERVICES_V2_BUILD_FLAG_WIDTH 1 +/* MC_CMD_FC_FPGA_BUILD_FLAG_INTERNAL 0x0 */ +/* MC_CMD_FC_FPGA_BUILD_FLAG_RELEASE 0x1 */ +#define MC_CMD_FC_OUT_FPGA_SERVICES_V2_REVISION_LO_OFST 24 +#define MC_CMD_FC_OUT_FPGA_SERVICES_V2_REVISION_HI_OFST 28 +#define MC_CMD_FC_OUT_FPGA_SERVICES_V2_REVISION_HIGH_LBN 0 +#define MC_CMD_FC_OUT_FPGA_SERVICES_V2_REVISION_HIGH_WIDTH 16 + +/* MC_CMD_FC_OUT_BSP_VERSION msgresponse */ +#define MC_CMD_FC_OUT_BSP_VERSION_LEN 4 +/* Qsys system ID */ +#define MC_CMD_FC_OUT_BSP_VERSION_SYSID_OFST 0 +#define MC_CMD_FC_OUT_BSP_VERSION_VERSION_MAJOR_LBN 12 +#define MC_CMD_FC_OUT_BSP_VERSION_VERSION_MAJOR_WIDTH 4 +#define MC_CMD_FC_OUT_BSP_VERSION_VERSION_MINOR_LBN 4 +#define MC_CMD_FC_OUT_BSP_VERSION_VERSION_MINOR_WIDTH 8 +#define MC_CMD_FC_OUT_BSP_VERSION_BUILD_NUM_LBN 0 +#define MC_CMD_FC_OUT_BSP_VERSION_BUILD_NUM_WIDTH 4 + +/* MC_CMD_FC_OUT_READ_MAP_COUNT msgresponse */ +#define MC_CMD_FC_OUT_READ_MAP_COUNT_LEN 4 +/* Number of maps */ +#define MC_CMD_FC_OUT_READ_MAP_COUNT_NUM_MAPS_OFST 0 + +/* MC_CMD_FC_OUT_READ_MAP_INDEX msgresponse */ +#define MC_CMD_FC_OUT_READ_MAP_INDEX_LEN 164 +/* Index of the map */ +#define MC_CMD_FC_OUT_READ_MAP_INDEX_INDEX_OFST 0 +/* Options for the map */ +#define MC_CMD_FC_OUT_READ_MAP_INDEX_OPTIONS_OFST 4 +#define MC_CMD_FC_OUT_READ_MAP_INDEX_ALIGN_8 0x0 /* enum */ +#define MC_CMD_FC_OUT_READ_MAP_INDEX_ALIGN_16 0x1 /* enum */ +#define MC_CMD_FC_OUT_READ_MAP_INDEX_ALIGN_32 0x2 /* enum */ +#define MC_CMD_FC_OUT_READ_MAP_INDEX_ALIGN_64 0x3 /* enum */ +#define MC_CMD_FC_OUT_READ_MAP_INDEX_ALIGN_MASK 0x3 /* enum */ +#define MC_CMD_FC_OUT_READ_MAP_INDEX_PATH_FC 0x4 /* enum */ +#define MC_CMD_FC_OUT_READ_MAP_INDEX_PATH_MEM 0x8 /* enum */ +#define MC_CMD_FC_OUT_READ_MAP_INDEX_PERM_READ 0x10 /* enum */ +#define MC_CMD_FC_OUT_READ_MAP_INDEX_PERM_WRITE 0x20 /* enum */ +#define MC_CMD_FC_OUT_READ_MAP_INDEX_LICENSE_FREE 0x0 /* enum */ +#define MC_CMD_FC_OUT_READ_MAP_INDEX_LICENSE_LICENSED 0x40 /* enum */ +/* Address of start of map */ +#define MC_CMD_FC_OUT_READ_MAP_INDEX_ADDRESS_OFST 8 +#define MC_CMD_FC_OUT_READ_MAP_INDEX_ADDRESS_LEN 8 +#define MC_CMD_FC_OUT_READ_MAP_INDEX_ADDRESS_LO_OFST 8 +#define MC_CMD_FC_OUT_READ_MAP_INDEX_ADDRESS_HI_OFST 12 +/* Length of address map */ +#define MC_CMD_FC_OUT_READ_MAP_INDEX_LEN_OFST 16 +#define MC_CMD_FC_OUT_READ_MAP_INDEX_LEN_LEN 8 +#define MC_CMD_FC_OUT_READ_MAP_INDEX_LEN_LO_OFST 16 +#define MC_CMD_FC_OUT_READ_MAP_INDEX_LEN_HI_OFST 20 +/* Component information field */ +#define MC_CMD_FC_OUT_READ_MAP_INDEX_COMP_INFO_OFST 24 +/* License expiry data for map */ +#define MC_CMD_FC_OUT_READ_MAP_INDEX_LICENSE_DATE_OFST 28 +#define MC_CMD_FC_OUT_READ_MAP_INDEX_LICENSE_DATE_LEN 8 +#define MC_CMD_FC_OUT_READ_MAP_INDEX_LICENSE_DATE_LO_OFST 28 +#define MC_CMD_FC_OUT_READ_MAP_INDEX_LICENSE_DATE_HI_OFST 32 +/* Name of the component */ +#define MC_CMD_FC_OUT_READ_MAP_INDEX_NAME_OFST 36 +#define MC_CMD_FC_OUT_READ_MAP_INDEX_NAME_LEN 1 +#define MC_CMD_FC_OUT_READ_MAP_INDEX_NAME_NUM 128 + +/* MC_CMD_FC_OUT_READ_MAP msgresponse */ +#define MC_CMD_FC_OUT_READ_MAP_LEN 0 + +/* MC_CMD_FC_OUT_CAPABILITIES msgresponse */ +#define MC_CMD_FC_OUT_CAPABILITIES_LEN 8 +/* Number of internal ports */ +#define MC_CMD_FC_OUT_CAPABILITIES_INTERNAL_OFST 0 +/* Number of external ports */ +#define MC_CMD_FC_OUT_CAPABILITIES_EXTERNAL_OFST 4 + +/* MC_CMD_FC_OUT_GLOBAL_FLAGS msgresponse */ +#define MC_CMD_FC_OUT_GLOBAL_FLAGS_LEN 4 +#define MC_CMD_FC_OUT_GLOBAL_FLAGS_FLAGS_OFST 0 + +/* MC_CMD_FC_OUT_IO_REL msgresponse */ +#define MC_CMD_FC_OUT_IO_REL_LEN 0 + +/* MC_CMD_FC_OUT_IO_REL_GET_ADDR msgresponse */ +#define MC_CMD_FC_OUT_IO_REL_GET_ADDR_LEN 8 +#define MC_CMD_FC_OUT_IO_REL_GET_ADDR_ADDR_HI_OFST 0 +#define MC_CMD_FC_OUT_IO_REL_GET_ADDR_ADDR_LO_OFST 4 + +/* MC_CMD_FC_OUT_IO_REL_READ32 msgresponse */ +#define MC_CMD_FC_OUT_IO_REL_READ32_LENMIN 4 +#define MC_CMD_FC_OUT_IO_REL_READ32_LENMAX 252 +#define MC_CMD_FC_OUT_IO_REL_READ32_LEN(num) (0+4*(num)) +#define MC_CMD_FC_OUT_IO_REL_READ32_BUFFER_OFST 0 +#define MC_CMD_FC_OUT_IO_REL_READ32_BUFFER_LEN 4 +#define MC_CMD_FC_OUT_IO_REL_READ32_BUFFER_MINNUM 1 +#define MC_CMD_FC_OUT_IO_REL_READ32_BUFFER_MAXNUM 63 + +/* MC_CMD_FC_OUT_IO_REL_WRITE32 msgresponse */ +#define MC_CMD_FC_OUT_IO_REL_WRITE32_LEN 0 + +/* MC_CMD_FC_OUT_UHLINK_PHY msgresponse */ +#define MC_CMD_FC_OUT_UHLINK_PHY_LEN 48 +#define MC_CMD_FC_OUT_UHLINK_PHY_TRC_TX_SETTINGS_0_OFST 0 +#define MC_CMD_FC_OUT_UHLINK_PHY_TRC_TX_VOD_LBN 0 +#define MC_CMD_FC_OUT_UHLINK_PHY_TRC_TX_VOD_WIDTH 16 +#define MC_CMD_FC_OUT_UHLINK_PHY_TRC_TX_PREEMP_1STPOSTTAP_LBN 16 +#define MC_CMD_FC_OUT_UHLINK_PHY_TRC_TX_PREEMP_1STPOSTTAP_WIDTH 16 +/* Transceiver Transmit settings */ +#define MC_CMD_FC_OUT_UHLINK_PHY_TRC_TX_SETTINGS_1_OFST 4 +#define MC_CMD_FC_OUT_UHLINK_PHY_TRC_TX_PREEMP_PRETAP_LBN 0 +#define MC_CMD_FC_OUT_UHLINK_PHY_TRC_TX_PREEMP_PRETAP_WIDTH 16 +#define MC_CMD_FC_OUT_UHLINK_PHY_TRC_TX_PREEMP_2NDPOSTTAP_LBN 16 +#define MC_CMD_FC_OUT_UHLINK_PHY_TRC_TX_PREEMP_2NDPOSTTAP_WIDTH 16 +/* Transceiver Receive settings */ +#define MC_CMD_FC_OUT_UHLINK_PHY_TRC_RX_SETTINGS_OFST 8 +#define MC_CMD_FC_OUT_UHLINK_PHY_TRC_RX_DC_GAIN_LBN 0 +#define MC_CMD_FC_OUT_UHLINK_PHY_TRC_RX_DC_GAIN_WIDTH 16 +#define MC_CMD_FC_OUT_UHLINK_PHY_TRC_RX_EQ_CONTROL_LBN 16 +#define MC_CMD_FC_OUT_UHLINK_PHY_TRC_RX_EQ_CONTROL_WIDTH 16 +/* Rx eye opening */ +#define MC_CMD_FC_OUT_UHLINK_PHY_RX_EYE_OFST 12 +#define MC_CMD_FC_OUT_UHLINK_PHY_RX_EYE_WIDTH_LBN 0 +#define MC_CMD_FC_OUT_UHLINK_PHY_RX_EYE_WIDTH_WIDTH 16 +#define MC_CMD_FC_OUT_UHLINK_PHY_RX_EYE_HEIGHT_LBN 16 +#define MC_CMD_FC_OUT_UHLINK_PHY_RX_EYE_HEIGHT_WIDTH 16 +/* PCS status word */ +#define MC_CMD_FC_OUT_UHLINK_PHY_PCS_STATUS_OFST 16 +/* Link status word */ +#define MC_CMD_FC_OUT_UHLINK_PHY_LINK_STATE_WORD_OFST 20 +#define MC_CMD_FC_OUT_UHLINK_PHY_LINK_STATE_LBN 0 +#define MC_CMD_FC_OUT_UHLINK_PHY_LINK_STATE_WIDTH 1 +#define MC_CMD_FC_OUT_UHLINK_PHY_LINK_CONFIGURED_LBN 1 +#define MC_CMD_FC_OUT_UHLINK_PHY_LINK_CONFIGURED_WIDTH 1 +/* Current SFp parameters applied */ +#define MC_CMD_FC_OUT_UHLINK_PHY_SFP_PARAMS_OFST 24 +#define MC_CMD_FC_OUT_UHLINK_PHY_SFP_PARAMS_LEN 20 +/* Link speed is 100, 1000, 10000 */ +#define MC_CMD_FC_OUT_UHLINK_PHY_SFP_SPEED_OFST 24 +/* Length of copper cable - zero when not relevant */ +#define MC_CMD_FC_OUT_UHLINK_PHY_SFP_COPPER_LEN_OFST 28 +/* True if a dual speed SFP+ module */ +#define MC_CMD_FC_OUT_UHLINK_PHY_SFP_DUAL_SPEED_OFST 32 +/* True if an SFP Module is present (other fields valid when true) */ +#define MC_CMD_FC_OUT_UHLINK_PHY_SFP_PRESENT_OFST 36 +/* The type of the SFP+ Module */ +#define MC_CMD_FC_OUT_UHLINK_PHY_SFP_TYPE_OFST 40 +/* PHY config flags */ +#define MC_CMD_FC_OUT_UHLINK_PHY_PHY_CFG_OFST 44 +#define MC_CMD_FC_OUT_UHLINK_PHY_PHY_CFG_DFE_LBN 0 +#define MC_CMD_FC_OUT_UHLINK_PHY_PHY_CFG_DFE_WIDTH 1 +#define MC_CMD_FC_OUT_UHLINK_PHY_PHY_CFG_AEQ_LBN 1 +#define MC_CMD_FC_OUT_UHLINK_PHY_PHY_CFG_AEQ_WIDTH 1 +#define MC_CMD_FC_OUT_UHLINK_PHY_PHY_CFG_RX_TUNING_LBN 2 +#define MC_CMD_FC_OUT_UHLINK_PHY_PHY_CFG_RX_TUNING_WIDTH 1 + +/* MC_CMD_FC_OUT_UHLINK_MAC msgresponse */ +#define MC_CMD_FC_OUT_UHLINK_MAC_LEN 20 +/* MAC configuration applied */ +#define MC_CMD_FC_OUT_UHLINK_MAC_CONFIG_OFST 0 +/* MTU size */ +#define MC_CMD_FC_OUT_UHLINK_MAC_MTU_OFST 4 +/* IF Mode status */ +#define MC_CMD_FC_OUT_UHLINK_MAC_IF_STATUS_OFST 8 +/* MAC address configured */ +#define MC_CMD_FC_OUT_UHLINK_MAC_ADDR_OFST 12 +#define MC_CMD_FC_OUT_UHLINK_MAC_ADDR_LEN 8 +#define MC_CMD_FC_OUT_UHLINK_MAC_ADDR_LO_OFST 12 +#define MC_CMD_FC_OUT_UHLINK_MAC_ADDR_HI_OFST 16 + +/* MC_CMD_FC_OUT_UHLINK_RX_EYE msgresponse */ +#define MC_CMD_FC_OUT_UHLINK_RX_EYE_LEN ((((0-1+(32*MC_CMD_FC_UHLINK_RX_EYE_PER_BLOCK))+1))>>3) +/* Rx Eye measurements */ +#define MC_CMD_FC_OUT_UHLINK_RX_EYE_RX_EYE_OFST 0 +#define MC_CMD_FC_OUT_UHLINK_RX_EYE_RX_EYE_LEN 4 +#define MC_CMD_FC_OUT_UHLINK_RX_EYE_RX_EYE_NUM MC_CMD_FC_UHLINK_RX_EYE_PER_BLOCK + +/* MC_CMD_FC_OUT_UHLINK_DUMP_RX_EYE_PLOT msgresponse */ +#define MC_CMD_FC_OUT_UHLINK_DUMP_RX_EYE_PLOT_LEN 0 + +/* MC_CMD_FC_OUT_UHLINK_READ_RX_EYE_PLOT msgresponse */ +#define MC_CMD_FC_OUT_UHLINK_READ_RX_EYE_PLOT_LEN ((((32-1+(64*MC_CMD_FC_UHLINK_RX_EYE_PLOT_ROWS_PER_BLOCK))+1))>>3) +/* Has the eye plot dump completed and data returned is valid? */ +#define MC_CMD_FC_OUT_UHLINK_READ_RX_EYE_PLOT_VALID_OFST 0 +/* Rx Eye binary plot */ +#define MC_CMD_FC_OUT_UHLINK_READ_RX_EYE_PLOT_ROWS_OFST 4 +#define MC_CMD_FC_OUT_UHLINK_READ_RX_EYE_PLOT_ROWS_LEN 8 +#define MC_CMD_FC_OUT_UHLINK_READ_RX_EYE_PLOT_ROWS_LO_OFST 4 +#define MC_CMD_FC_OUT_UHLINK_READ_RX_EYE_PLOT_ROWS_HI_OFST 8 +#define MC_CMD_FC_OUT_UHLINK_READ_RX_EYE_PLOT_ROWS_NUM MC_CMD_FC_UHLINK_RX_EYE_PLOT_ROWS_PER_BLOCK + +/* MC_CMD_FC_OUT_UHLINK_RX_TUNE msgresponse */ +#define MC_CMD_FC_OUT_UHLINK_RX_TUNE_LEN 0 + +/* MC_CMD_FC_OUT_UHLINK_LOOPBACK_SET msgresponse */ +#define MC_CMD_FC_OUT_UHLINK_LOOPBACK_SET_LEN 0 + +/* MC_CMD_FC_OUT_UHLINK_LOOPBACK_GET msgresponse */ +#define MC_CMD_FC_OUT_UHLINK_LOOPBACK_GET_LEN 4 +#define MC_CMD_FC_OUT_UHLINK_LOOPBACK_GET_STATE_OFST 0 + +/* MC_CMD_FC_OUT_UHLINK msgresponse */ +#define MC_CMD_FC_OUT_UHLINK_LEN 0 + +/* MC_CMD_FC_OUT_SET_LINK msgresponse */ +#define MC_CMD_FC_OUT_SET_LINK_LEN 0 + +/* MC_CMD_FC_OUT_LICENSE msgresponse */ +#define MC_CMD_FC_OUT_LICENSE_LEN 12 +/* Count of valid keys */ +#define MC_CMD_FC_OUT_LICENSE_VALID_KEYS_OFST 0 +/* Count of invalid keys */ +#define MC_CMD_FC_OUT_LICENSE_INVALID_KEYS_OFST 4 +/* Count of blacklisted keys */ +#define MC_CMD_FC_OUT_LICENSE_BLACKLISTED_KEYS_OFST 8 + +/* MC_CMD_FC_OUT_STARTUP msgresponse */ +#define MC_CMD_FC_OUT_STARTUP_LEN 4 +/* Capabilities of the FPGA/FC */ +#define MC_CMD_FC_OUT_STARTUP_CAPABILITIES_OFST 0 +#define MC_CMD_FC_OUT_STARTUP_CAN_ACCESS_FLASH_LBN 0 +#define MC_CMD_FC_OUT_STARTUP_CAN_ACCESS_FLASH_WIDTH 1 + +/* MC_CMD_FC_OUT_DMA_READ msgresponse */ +#define MC_CMD_FC_OUT_DMA_READ_LENMIN 1 +#define MC_CMD_FC_OUT_DMA_READ_LENMAX 252 +#define MC_CMD_FC_OUT_DMA_READ_LEN(num) (0+1*(num)) +/* The data read */ +#define MC_CMD_FC_OUT_DMA_READ_DATA_OFST 0 +#define MC_CMD_FC_OUT_DMA_READ_DATA_LEN 1 +#define MC_CMD_FC_OUT_DMA_READ_DATA_MINNUM 1 +#define MC_CMD_FC_OUT_DMA_READ_DATA_MAXNUM 252 + +/* MC_CMD_FC_OUT_TIMED_READ_SET msgresponse */ +#define MC_CMD_FC_OUT_TIMED_READ_SET_LEN 4 +/* Timer handle */ +#define MC_CMD_FC_OUT_TIMED_READ_SET_FC_HANDLE_OFST 0 + +/* MC_CMD_FC_OUT_TIMED_READ_GET msgresponse */ +#define MC_CMD_FC_OUT_TIMED_READ_GET_LEN 52 +/* Host supplied handle (unique) */ +#define MC_CMD_FC_OUT_TIMED_READ_GET_HOST_HANDLE_OFST 0 +/* Address into which to transfer data in host */ +#define MC_CMD_FC_OUT_TIMED_READ_GET_HOST_DMA_ADDRESS_OFST 4 +#define MC_CMD_FC_OUT_TIMED_READ_GET_HOST_DMA_ADDRESS_LEN 8 +#define MC_CMD_FC_OUT_TIMED_READ_GET_HOST_DMA_ADDRESS_LO_OFST 4 +#define MC_CMD_FC_OUT_TIMED_READ_GET_HOST_DMA_ADDRESS_HI_OFST 8 +/* AOE address from which to transfer data */ +#define MC_CMD_FC_OUT_TIMED_READ_GET_AOE_ADDRESS_OFST 12 +#define MC_CMD_FC_OUT_TIMED_READ_GET_AOE_ADDRESS_LEN 8 +#define MC_CMD_FC_OUT_TIMED_READ_GET_AOE_ADDRESS_LO_OFST 12 +#define MC_CMD_FC_OUT_TIMED_READ_GET_AOE_ADDRESS_HI_OFST 16 +/* Length of AOE transfer (total) */ +#define MC_CMD_FC_OUT_TIMED_READ_GET_AOE_LENGTH_OFST 20 +/* Length of host transfer (total) */ +#define MC_CMD_FC_OUT_TIMED_READ_GET_HOST_LENGTH_OFST 24 +/* See FLAGS entry for MC_CMD_FC_IN_TIMED_READ_SET */ +#define MC_CMD_FC_OUT_TIMED_READ_GET_FLAGS_OFST 28 +#define MC_CMD_FC_OUT_TIMED_READ_GET_PERIOD_OFST 32 +/* When active, start read time */ +#define MC_CMD_FC_OUT_TIMED_READ_GET_CLOCK_START_OFST 36 +#define MC_CMD_FC_OUT_TIMED_READ_GET_CLOCK_START_LEN 8 +#define MC_CMD_FC_OUT_TIMED_READ_GET_CLOCK_START_LO_OFST 36 +#define MC_CMD_FC_OUT_TIMED_READ_GET_CLOCK_START_HI_OFST 40 +/* When active, end read time */ +#define MC_CMD_FC_OUT_TIMED_READ_GET_CLOCK_END_OFST 44 +#define MC_CMD_FC_OUT_TIMED_READ_GET_CLOCK_END_LEN 8 +#define MC_CMD_FC_OUT_TIMED_READ_GET_CLOCK_END_LO_OFST 44 +#define MC_CMD_FC_OUT_TIMED_READ_GET_CLOCK_END_HI_OFST 48 + +/* MC_CMD_FC_OUT_LOG_ADDR_RANGE msgresponse */ +#define MC_CMD_FC_OUT_LOG_ADDR_RANGE_LEN 0 + +/* MC_CMD_FC_OUT_LOG msgresponse */ +#define MC_CMD_FC_OUT_LOG_LEN 0 + +/* MC_CMD_FC_OUT_CLOCK_GET_TIME msgresponse */ +#define MC_CMD_FC_OUT_CLOCK_GET_TIME_LEN 24 +#define MC_CMD_FC_OUT_CLOCK_GET_TIME_CLOCK_ID_OFST 0 +#define MC_CMD_FC_OUT_CLOCK_GET_TIME_SECONDS_OFST 4 +#define MC_CMD_FC_OUT_CLOCK_GET_TIME_SECONDS_LEN 8 +#define MC_CMD_FC_OUT_CLOCK_GET_TIME_SECONDS_LO_OFST 4 +#define MC_CMD_FC_OUT_CLOCK_GET_TIME_SECONDS_HI_OFST 8 +#define MC_CMD_FC_OUT_CLOCK_GET_TIME_NANOSECONDS_OFST 12 +#define MC_CMD_FC_OUT_CLOCK_GET_TIME_RANGE_OFST 16 +#define MC_CMD_FC_OUT_CLOCK_GET_TIME_PRECISION_OFST 20 + +/* MC_CMD_FC_OUT_CLOCK_SET_TIME msgresponse */ +#define MC_CMD_FC_OUT_CLOCK_SET_TIME_LEN 0 + +/* MC_CMD_FC_OUT_DDR_SET_SPD msgresponse */ +#define MC_CMD_FC_OUT_DDR_SET_SPD_LEN 0 + +/* MC_CMD_FC_OUT_DDR_SET_INFO msgresponse */ +#define MC_CMD_FC_OUT_DDR_SET_INFO_LEN 0 + +/* MC_CMD_FC_OUT_DDR_GET_STATUS msgresponse */ +#define MC_CMD_FC_OUT_DDR_GET_STATUS_LEN 4 +#define MC_CMD_FC_OUT_DDR_GET_STATUS_FLAGS_OFST 0 +#define MC_CMD_FC_OUT_DDR_GET_STATUS_READY_LBN 0 +#define MC_CMD_FC_OUT_DDR_GET_STATUS_READY_WIDTH 1 +#define MC_CMD_FC_OUT_DDR_GET_STATUS_CALIBRATED_LBN 1 +#define MC_CMD_FC_OUT_DDR_GET_STATUS_CALIBRATED_WIDTH 1 + +/* MC_CMD_FC_OUT_TIMESTAMP_READ_TRANSMIT msgresponse */ +#define MC_CMD_FC_OUT_TIMESTAMP_READ_TRANSMIT_LEN 8 +#define MC_CMD_FC_OUT_TIMESTAMP_READ_TRANSMIT_SECONDS_OFST 0 +#define MC_CMD_FC_OUT_TIMESTAMP_READ_TRANSMIT_NANOSECONDS_OFST 4 + +/* MC_CMD_FC_OUT_TIMESTAMP_READ_SNAPSHOT msgresponse */ +#define MC_CMD_FC_OUT_TIMESTAMP_READ_SNAPSHOT_LENMIN 8 +#define MC_CMD_FC_OUT_TIMESTAMP_READ_SNAPSHOT_LENMAX 248 +#define MC_CMD_FC_OUT_TIMESTAMP_READ_SNAPSHOT_LEN(num) (0+8*(num)) +#define MC_CMD_FC_OUT_TIMESTAMP_READ_SNAPSHOT_SECONDS_OFST 0 +#define MC_CMD_FC_OUT_TIMESTAMP_READ_SNAPSHOT_NANOSECONDS_OFST 4 +#define MC_CMD_FC_OUT_TIMESTAMP_READ_SNAPSHOT_TIMESTAMP_OFST 0 +#define MC_CMD_FC_OUT_TIMESTAMP_READ_SNAPSHOT_TIMESTAMP_LEN 8 +#define MC_CMD_FC_OUT_TIMESTAMP_READ_SNAPSHOT_TIMESTAMP_LO_OFST 0 +#define MC_CMD_FC_OUT_TIMESTAMP_READ_SNAPSHOT_TIMESTAMP_HI_OFST 4 +#define MC_CMD_FC_OUT_TIMESTAMP_READ_SNAPSHOT_TIMESTAMP_MINNUM 0 +#define MC_CMD_FC_OUT_TIMESTAMP_READ_SNAPSHOT_TIMESTAMP_MAXNUM 31 + +/* MC_CMD_FC_OUT_SPI_READ msgresponse */ +#define MC_CMD_FC_OUT_SPI_READ_LENMIN 4 +#define MC_CMD_FC_OUT_SPI_READ_LENMAX 252 +#define MC_CMD_FC_OUT_SPI_READ_LEN(num) (0+4*(num)) +#define MC_CMD_FC_OUT_SPI_READ_BUFFER_OFST 0 +#define MC_CMD_FC_OUT_SPI_READ_BUFFER_LEN 4 +#define MC_CMD_FC_OUT_SPI_READ_BUFFER_MINNUM 1 +#define MC_CMD_FC_OUT_SPI_READ_BUFFER_MAXNUM 63 + +/* MC_CMD_FC_OUT_SPI_WRITE msgresponse */ +#define MC_CMD_FC_OUT_SPI_WRITE_LEN 0 + +/* MC_CMD_FC_OUT_SPI_ERASE msgresponse */ +#define MC_CMD_FC_OUT_SPI_ERASE_LEN 0 + +/* MC_CMD_FC_OUT_DIAG_POWER_NOISE_READ_CONFIG msgresponse */ +#define MC_CMD_FC_OUT_DIAG_POWER_NOISE_READ_CONFIG_LEN 8 +/* The 32-bit value read from the toggle count register */ +#define MC_CMD_FC_OUT_DIAG_POWER_NOISE_READ_CONFIG_TOGGLE_COUNT_OFST 0 +/* The 32-bit value read from the clock enable count register */ +#define MC_CMD_FC_OUT_DIAG_POWER_NOISE_READ_CONFIG_CLKEN_COUNT_OFST 4 + +/* MC_CMD_FC_OUT_DIAG_POWER_NOISE_WRITE_CONFIG msgresponse */ +#define MC_CMD_FC_OUT_DIAG_POWER_NOISE_WRITE_CONFIG_LEN 0 + +/* MC_CMD_FC_OUT_DIAG_DDR_SOAK_START msgresponse */ +#define MC_CMD_FC_OUT_DIAG_DDR_SOAK_START_LEN 0 + +/* MC_CMD_FC_OUT_DIAG_DDR_SOAK_RESULT msgresponse */ +#define MC_CMD_FC_OUT_DIAG_DDR_SOAK_RESULT_LEN 8 +/* DDR soak test status word; bits [4:0] are relevant. */ +#define MC_CMD_FC_OUT_DIAG_DDR_SOAK_RESULT_STATUS_OFST 0 +#define MC_CMD_FC_OUT_DIAG_DDR_SOAK_RESULT_PASSED_LBN 0 +#define MC_CMD_FC_OUT_DIAG_DDR_SOAK_RESULT_PASSED_WIDTH 1 +#define MC_CMD_FC_OUT_DIAG_DDR_SOAK_RESULT_FAILED_LBN 1 +#define MC_CMD_FC_OUT_DIAG_DDR_SOAK_RESULT_FAILED_WIDTH 1 +#define MC_CMD_FC_OUT_DIAG_DDR_SOAK_RESULT_COMPLETED_LBN 2 +#define MC_CMD_FC_OUT_DIAG_DDR_SOAK_RESULT_COMPLETED_WIDTH 1 +#define MC_CMD_FC_OUT_DIAG_DDR_SOAK_RESULT_TIMEOUT_LBN 3 +#define MC_CMD_FC_OUT_DIAG_DDR_SOAK_RESULT_TIMEOUT_WIDTH 1 +#define MC_CMD_FC_OUT_DIAG_DDR_SOAK_RESULT_PNF_LBN 4 +#define MC_CMD_FC_OUT_DIAG_DDR_SOAK_RESULT_PNF_WIDTH 1 +/* DDR soak test error count */ +#define MC_CMD_FC_OUT_DIAG_DDR_SOAK_RESULT_ERR_COUNT_OFST 4 + +/* MC_CMD_FC_OUT_DIAG_DDR_SOAK_STOP msgresponse */ +#define MC_CMD_FC_OUT_DIAG_DDR_SOAK_STOP_LEN 0 + +/* MC_CMD_FC_OUT_DIAG_DDR_SOAK_ERROR msgresponse */ +#define MC_CMD_FC_OUT_DIAG_DDR_SOAK_ERROR_LEN 0 + +/* MC_CMD_FC_OUT_DIAG_DATAPATH_CTRL_SET_MODE msgresponse */ +#define MC_CMD_FC_OUT_DIAG_DATAPATH_CTRL_SET_MODE_LEN 0 + +/* MC_CMD_FC_OUT_DIAG_DATAPATH_CTRL_RAW_CONFIG msgresponse */ +#define MC_CMD_FC_OUT_DIAG_DATAPATH_CTRL_RAW_CONFIG_LEN 0 + + +/***********************************/ +/* MC_CMD_AOE + * AOE operations on MC + */ +#define MC_CMD_AOE 0xa + +/* MC_CMD_AOE_IN msgrequest */ +#define MC_CMD_AOE_IN_LEN 4 +#define MC_CMD_AOE_IN_OP_HDR_OFST 0 +#define MC_CMD_AOE_IN_OP_LBN 0 +#define MC_CMD_AOE_IN_OP_WIDTH 8 +/* enum: FPGA and CPLD information */ +#define MC_CMD_AOE_OP_INFO 0x1 +/* enum: Currents and voltages read from MCP3424s; DEBUG */ +#define MC_CMD_AOE_OP_CURRENTS 0x2 +/* enum: Temperatures at locations around the PCB; DEBUG */ +#define MC_CMD_AOE_OP_TEMPERATURES 0x3 +/* enum: Set CPLD to idle */ +#define MC_CMD_AOE_OP_CPLD_IDLE 0x4 +/* enum: Read from CPLD register */ +#define MC_CMD_AOE_OP_CPLD_READ 0x5 +/* enum: Write to CPLD register */ +#define MC_CMD_AOE_OP_CPLD_WRITE 0x6 +/* enum: Execute CPLD instruction */ +#define MC_CMD_AOE_OP_CPLD_INSTRUCTION 0x7 +/* enum: Reprogram the CPLD on the AOE device */ +#define MC_CMD_AOE_OP_CPLD_REPROGRAM 0x8 +/* enum: AOE power control */ +#define MC_CMD_AOE_OP_POWER 0x9 +/* enum: AOE image loading */ +#define MC_CMD_AOE_OP_LOAD 0xa +/* enum: Fan monitoring */ +#define MC_CMD_AOE_OP_FAN_CONTROL 0xb +/* enum: Fan failures since last reset */ +#define MC_CMD_AOE_OP_FAN_FAILURES 0xc +/* enum: Get generic AOE MAC statistics */ +#define MC_CMD_AOE_OP_MAC_STATS 0xd +/* enum: Retrieve PHY specific information */ +#define MC_CMD_AOE_OP_GET_PHY_MEDIA_INFO 0xe +/* enum: Write a number of JTAG primitive commands, return will give data */ +#define MC_CMD_AOE_OP_JTAG_WRITE 0xf +/* enum: Control access to the FPGA via the Siena JTAG Chain */ +#define MC_CMD_AOE_OP_FPGA_ACCESS 0x10 +/* enum: Set the MTU offset between Siena and AOE MACs */ +#define MC_CMD_AOE_OP_SET_MTU_OFFSET 0x11 +/* enum: How link state is handled */ +#define MC_CMD_AOE_OP_LINK_STATE 0x12 +/* enum: How Siena MAC statistics are reported (deprecated - use + * MC_CMD_AOE_OP_ASIC_STATS) + */ +#define MC_CMD_AOE_OP_SIENA_STATS 0x13 +/* enum: How native ASIC MAC statistics are reported - replaces the deprecated + * command MC_CMD_AOE_OP_SIENA_STATS + */ +#define MC_CMD_AOE_OP_ASIC_STATS 0x13 +/* enum: DDR memory information */ +#define MC_CMD_AOE_OP_DDR 0x14 +/* enum: FC control */ +#define MC_CMD_AOE_OP_FC 0x15 +/* enum: DDR ECC status reads */ +#define MC_CMD_AOE_OP_DDR_ECC_STATUS 0x16 +/* enum: Commands for MC-SPI Master emulation */ +#define MC_CMD_AOE_OP_MC_SPI_MASTER 0x17 +/* enum: Commands for FC boot control */ +#define MC_CMD_AOE_OP_FC_BOOT 0x18 + +/* MC_CMD_AOE_OUT msgresponse */ +#define MC_CMD_AOE_OUT_LEN 0 + +/* MC_CMD_AOE_IN_INFO msgrequest */ +#define MC_CMD_AOE_IN_INFO_LEN 4 +#define MC_CMD_AOE_IN_CMD_OFST 0 + +/* MC_CMD_AOE_IN_CURRENTS msgrequest */ +#define MC_CMD_AOE_IN_CURRENTS_LEN 4 +/* MC_CMD_AOE_IN_CMD_OFST 0 */ + +/* MC_CMD_AOE_IN_TEMPERATURES msgrequest */ +#define MC_CMD_AOE_IN_TEMPERATURES_LEN 4 +/* MC_CMD_AOE_IN_CMD_OFST 0 */ + +/* MC_CMD_AOE_IN_CPLD_IDLE msgrequest */ +#define MC_CMD_AOE_IN_CPLD_IDLE_LEN 4 +/* MC_CMD_AOE_IN_CMD_OFST 0 */ + +/* MC_CMD_AOE_IN_CPLD_READ msgrequest */ +#define MC_CMD_AOE_IN_CPLD_READ_LEN 12 +/* MC_CMD_AOE_IN_CMD_OFST 0 */ +#define MC_CMD_AOE_IN_CPLD_READ_REGISTER_OFST 4 +#define MC_CMD_AOE_IN_CPLD_READ_WIDTH_OFST 8 + +/* MC_CMD_AOE_IN_CPLD_WRITE msgrequest */ +#define MC_CMD_AOE_IN_CPLD_WRITE_LEN 16 +/* MC_CMD_AOE_IN_CMD_OFST 0 */ +#define MC_CMD_AOE_IN_CPLD_WRITE_REGISTER_OFST 4 +#define MC_CMD_AOE_IN_CPLD_WRITE_WIDTH_OFST 8 +#define MC_CMD_AOE_IN_CPLD_WRITE_VALUE_OFST 12 + +/* MC_CMD_AOE_IN_CPLD_INSTRUCTION msgrequest */ +#define MC_CMD_AOE_IN_CPLD_INSTRUCTION_LEN 8 +/* MC_CMD_AOE_IN_CMD_OFST 0 */ +#define MC_CMD_AOE_IN_CPLD_INSTRUCTION_INSTRUCTION_OFST 4 + +/* MC_CMD_AOE_IN_CPLD_REPROGRAM msgrequest */ +#define MC_CMD_AOE_IN_CPLD_REPROGRAM_LEN 8 +/* MC_CMD_AOE_IN_CMD_OFST 0 */ +#define MC_CMD_AOE_IN_CPLD_REPROGRAM_OP_OFST 4 +/* enum: Reprogram CPLD, poll for completion */ +#define MC_CMD_AOE_IN_CPLD_REPROGRAM_REPROGRAM 0x1 +/* enum: Reprogram CPLD, send event on completion */ +#define MC_CMD_AOE_IN_CPLD_REPROGRAM_REPROGRAM_EVENT 0x3 +/* enum: Get status of reprogramming operation */ +#define MC_CMD_AOE_IN_CPLD_REPROGRAM_STATUS 0x4 + +/* MC_CMD_AOE_IN_POWER msgrequest */ +#define MC_CMD_AOE_IN_POWER_LEN 8 +/* MC_CMD_AOE_IN_CMD_OFST 0 */ +/* Turn on or off AOE power */ +#define MC_CMD_AOE_IN_POWER_OP_OFST 4 +/* enum: Turn off FPGA power */ +#define MC_CMD_AOE_IN_POWER_OFF 0x0 +/* enum: Turn on FPGA power */ +#define MC_CMD_AOE_IN_POWER_ON 0x1 +/* enum: Clear peak power measurement */ +#define MC_CMD_AOE_IN_POWER_CLEAR 0x2 +/* enum: Show current power in sensors output */ +#define MC_CMD_AOE_IN_POWER_SHOW_CURRENT 0x3 +/* enum: Show peak power in sensors output */ +#define MC_CMD_AOE_IN_POWER_SHOW_PEAK 0x4 +/* enum: Show current DDR current */ +#define MC_CMD_AOE_IN_POWER_DDR_LAST 0x5 +/* enum: Show peak DDR current */ +#define MC_CMD_AOE_IN_POWER_DDR_PEAK 0x6 +/* enum: Clear peak DDR current */ +#define MC_CMD_AOE_IN_POWER_DDR_CLEAR 0x7 + +/* MC_CMD_AOE_IN_LOAD msgrequest */ +#define MC_CMD_AOE_IN_LOAD_LEN 8 +/* MC_CMD_AOE_IN_CMD_OFST 0 */ +/* Image to be loaded (0 - main or 1 - diagnostic) to load in normal sequence + */ +#define MC_CMD_AOE_IN_LOAD_IMAGE_OFST 4 + +/* MC_CMD_AOE_IN_FAN_CONTROL msgrequest */ +#define MC_CMD_AOE_IN_FAN_CONTROL_LEN 8 +/* MC_CMD_AOE_IN_CMD_OFST 0 */ +/* If non zero report measured fan RPM rather than nominal */ +#define MC_CMD_AOE_IN_FAN_CONTROL_REAL_RPM_OFST 4 + +/* MC_CMD_AOE_IN_FAN_FAILURES msgrequest */ +#define MC_CMD_AOE_IN_FAN_FAILURES_LEN 4 +/* MC_CMD_AOE_IN_CMD_OFST 0 */ + +/* MC_CMD_AOE_IN_MAC_STATS msgrequest */ +#define MC_CMD_AOE_IN_MAC_STATS_LEN 24 +/* MC_CMD_AOE_IN_CMD_OFST 0 */ +/* AOE port */ +#define MC_CMD_AOE_IN_MAC_STATS_PORT_OFST 4 +/* Host memory address for statistics */ +#define MC_CMD_AOE_IN_MAC_STATS_DMA_ADDR_OFST 8 +#define MC_CMD_AOE_IN_MAC_STATS_DMA_ADDR_LEN 8 +#define MC_CMD_AOE_IN_MAC_STATS_DMA_ADDR_LO_OFST 8 +#define MC_CMD_AOE_IN_MAC_STATS_DMA_ADDR_HI_OFST 12 +#define MC_CMD_AOE_IN_MAC_STATS_CMD_OFST 16 +#define MC_CMD_AOE_IN_MAC_STATS_DMA_LBN 0 +#define MC_CMD_AOE_IN_MAC_STATS_DMA_WIDTH 1 +#define MC_CMD_AOE_IN_MAC_STATS_CLEAR_LBN 1 +#define MC_CMD_AOE_IN_MAC_STATS_CLEAR_WIDTH 1 +#define MC_CMD_AOE_IN_MAC_STATS_PERIODIC_CHANGE_LBN 2 +#define MC_CMD_AOE_IN_MAC_STATS_PERIODIC_CHANGE_WIDTH 1 +#define MC_CMD_AOE_IN_MAC_STATS_PERIODIC_ENABLE_LBN 3 +#define MC_CMD_AOE_IN_MAC_STATS_PERIODIC_ENABLE_WIDTH 1 +#define MC_CMD_AOE_IN_MAC_STATS_PERIODIC_CLEAR_LBN 4 +#define MC_CMD_AOE_IN_MAC_STATS_PERIODIC_CLEAR_WIDTH 1 +#define MC_CMD_AOE_IN_MAC_STATS_PERIODIC_NOEVENT_LBN 5 +#define MC_CMD_AOE_IN_MAC_STATS_PERIODIC_NOEVENT_WIDTH 1 +#define MC_CMD_AOE_IN_MAC_STATS_PERIOD_MS_LBN 16 +#define MC_CMD_AOE_IN_MAC_STATS_PERIOD_MS_WIDTH 16 +/* Length of DMA data (optional) */ +#define MC_CMD_AOE_IN_MAC_STATS_DMA_LEN_OFST 20 + +/* MC_CMD_AOE_IN_GET_PHY_MEDIA_INFO msgrequest */ +#define MC_CMD_AOE_IN_GET_PHY_MEDIA_INFO_LEN 12 +/* MC_CMD_AOE_IN_CMD_OFST 0 */ +/* AOE port */ +#define MC_CMD_AOE_IN_GET_PHY_MEDIA_INFO_PORT_OFST 4 +#define MC_CMD_AOE_IN_GET_PHY_MEDIA_INFO_PAGE_OFST 8 + +/* MC_CMD_AOE_IN_JTAG_WRITE msgrequest */ +#define MC_CMD_AOE_IN_JTAG_WRITE_LENMIN 12 +#define MC_CMD_AOE_IN_JTAG_WRITE_LENMAX 252 +#define MC_CMD_AOE_IN_JTAG_WRITE_LEN(num) (8+4*(num)) +/* MC_CMD_AOE_IN_CMD_OFST 0 */ +#define MC_CMD_AOE_IN_JTAG_WRITE_DATALEN_OFST 4 +#define MC_CMD_AOE_IN_JTAG_WRITE_DATA_OFST 8 +#define MC_CMD_AOE_IN_JTAG_WRITE_DATA_LEN 4 +#define MC_CMD_AOE_IN_JTAG_WRITE_DATA_MINNUM 1 +#define MC_CMD_AOE_IN_JTAG_WRITE_DATA_MAXNUM 61 + +/* MC_CMD_AOE_IN_FPGA_ACCESS msgrequest */ +#define MC_CMD_AOE_IN_FPGA_ACCESS_LEN 8 +/* MC_CMD_AOE_IN_CMD_OFST 0 */ +/* Enable or disable access */ +#define MC_CMD_AOE_IN_FPGA_ACCESS_OP_OFST 4 +/* enum: Enable access */ +#define MC_CMD_AOE_IN_FPGA_ACCESS_ENABLE 0x1 +/* enum: Disable access */ +#define MC_CMD_AOE_IN_FPGA_ACCESS_DISABLE 0x2 + +/* MC_CMD_AOE_IN_SET_MTU_OFFSET msgrequest */ +#define MC_CMD_AOE_IN_SET_MTU_OFFSET_LEN 12 +/* MC_CMD_AOE_IN_CMD_OFST 0 */ +/* AOE port - when not ALL_EXTERNAL or ALL_INTERNAL specifies port number */ +#define MC_CMD_AOE_IN_SET_MTU_OFFSET_PORT_OFST 4 +/* enum: Apply to all external ports */ +#define MC_CMD_AOE_IN_SET_MTU_OFFSET_ALL_EXTERNAL 0x8000 +/* enum: Apply to all internal ports */ +#define MC_CMD_AOE_IN_SET_MTU_OFFSET_ALL_INTERNAL 0x4000 +/* The MTU offset to be applied to the external ports */ +#define MC_CMD_AOE_IN_SET_MTU_OFFSET_OFFSET_OFST 8 + +/* MC_CMD_AOE_IN_LINK_STATE msgrequest */ +#define MC_CMD_AOE_IN_LINK_STATE_LEN 8 +/* MC_CMD_AOE_IN_CMD_OFST 0 */ +#define MC_CMD_AOE_IN_LINK_STATE_MODE_OFST 4 +#define MC_CMD_AOE_IN_LINK_STATE_CONFIG_MODE_LBN 0 +#define MC_CMD_AOE_IN_LINK_STATE_CONFIG_MODE_WIDTH 8 +/* enum: AOE and associated external port */ +#define MC_CMD_AOE_IN_LINK_STATE_SIMPLE_SEPARATE 0x0 +/* enum: AOE and OR of all external ports */ +#define MC_CMD_AOE_IN_LINK_STATE_SIMPLE_COMBINED 0x1 +/* enum: Individual ports */ +#define MC_CMD_AOE_IN_LINK_STATE_DIAGNOSTIC 0x2 +/* enum: Configure link state mode on given AOE port */ +#define MC_CMD_AOE_IN_LINK_STATE_CUSTOM 0x3 +#define MC_CMD_AOE_IN_LINK_STATE_OPERATION_LBN 8 +#define MC_CMD_AOE_IN_LINK_STATE_OPERATION_WIDTH 8 +/* enum: No-op */ +#define MC_CMD_AOE_IN_LINK_STATE_OP_NONE 0x0 +/* enum: logical OR of all SFP ports link status */ +#define MC_CMD_AOE_IN_LINK_STATE_OP_OR 0x1 +/* enum: logical AND of all SFP ports link status */ +#define MC_CMD_AOE_IN_LINK_STATE_OP_AND 0x2 +#define MC_CMD_AOE_IN_LINK_STATE_SFP_MASK_LBN 16 +#define MC_CMD_AOE_IN_LINK_STATE_SFP_MASK_WIDTH 16 + +/* MC_CMD_AOE_IN_SIENA_STATS msgrequest */ +#define MC_CMD_AOE_IN_SIENA_STATS_LEN 8 +/* MC_CMD_AOE_IN_CMD_OFST 0 */ +/* How MAC statistics are reported */ +#define MC_CMD_AOE_IN_SIENA_STATS_MODE_OFST 4 +/* enum: Statistics from Siena (default) */ +#define MC_CMD_AOE_IN_SIENA_STATS_STATS_SIENA 0x0 +/* enum: Statistics from AOE external ports */ +#define MC_CMD_AOE_IN_SIENA_STATS_STATS_AOE 0x1 + +/* MC_CMD_AOE_IN_ASIC_STATS msgrequest */ +#define MC_CMD_AOE_IN_ASIC_STATS_LEN 8 +/* MC_CMD_AOE_IN_CMD_OFST 0 */ +/* How MAC statistics are reported */ +#define MC_CMD_AOE_IN_ASIC_STATS_MODE_OFST 4 +/* enum: Statistics from the ASIC (default) */ +#define MC_CMD_AOE_IN_ASIC_STATS_STATS_ASIC 0x0 +/* enum: Statistics from AOE external ports */ +#define MC_CMD_AOE_IN_ASIC_STATS_STATS_AOE 0x1 + +/* MC_CMD_AOE_IN_DDR msgrequest */ +#define MC_CMD_AOE_IN_DDR_LEN 12 +/* MC_CMD_AOE_IN_CMD_OFST 0 */ +#define MC_CMD_AOE_IN_DDR_BANK_OFST 4 +/* Enum values, see field(s): */ +/* MC_CMD_FC/MC_CMD_FC_IN_DDR/MC_CMD_FC_IN_DDR_BANK */ +/* Page index of SPD data */ +#define MC_CMD_AOE_IN_DDR_SPD_PAGE_ID_OFST 8 + +/* MC_CMD_AOE_IN_FC msgrequest */ +#define MC_CMD_AOE_IN_FC_LEN 4 +/* MC_CMD_AOE_IN_CMD_OFST 0 */ + +/* MC_CMD_AOE_IN_DDR_ECC_STATUS msgrequest */ +#define MC_CMD_AOE_IN_DDR_ECC_STATUS_LEN 8 +/* MC_CMD_AOE_IN_CMD_OFST 0 */ +#define MC_CMD_AOE_IN_DDR_ECC_STATUS_BANK_OFST 4 +/* Enum values, see field(s): */ +/* MC_CMD_FC/MC_CMD_FC_IN_DDR/MC_CMD_FC_IN_DDR_BANK */ + +/* MC_CMD_AOE_IN_MC_SPI_MASTER msgrequest */ +#define MC_CMD_AOE_IN_MC_SPI_MASTER_LEN 8 +/* MC_CMD_AOE_IN_CMD_OFST 0 */ +/* Basic commands for MC SPI Master emulation. */ +#define MC_CMD_AOE_IN_MC_SPI_MASTER_OP_OFST 4 +/* enum: MC SPI read */ +#define MC_CMD_AOE_IN_MC_SPI_MASTER_READ 0x0 +/* enum: MC SPI write */ +#define MC_CMD_AOE_IN_MC_SPI_MASTER_WRITE 0x1 + +/* MC_CMD_AOE_IN_MC_SPI_MASTER_READ msgrequest */ +#define MC_CMD_AOE_IN_MC_SPI_MASTER_READ_LEN 12 +/* MC_CMD_AOE_IN_CMD_OFST 0 */ +#define MC_CMD_AOE_IN_MC_SPI_MASTER_READ_OP_OFST 4 +#define MC_CMD_AOE_IN_MC_SPI_MASTER_READ_OFFSET_OFST 8 + +/* MC_CMD_AOE_IN_MC_SPI_MASTER_WRITE msgrequest */ +#define MC_CMD_AOE_IN_MC_SPI_MASTER_WRITE_LEN 16 +/* MC_CMD_AOE_IN_CMD_OFST 0 */ +#define MC_CMD_AOE_IN_MC_SPI_MASTER_WRITE_OP_OFST 4 +#define MC_CMD_AOE_IN_MC_SPI_MASTER_WRITE_OFFSET_OFST 8 +#define MC_CMD_AOE_IN_MC_SPI_MASTER_WRITE_DATA_OFST 12 + +/* MC_CMD_AOE_IN_FC_BOOT msgrequest */ +#define MC_CMD_AOE_IN_FC_BOOT_LEN 8 +/* MC_CMD_AOE_IN_CMD_OFST 0 */ +/* FC boot control flags */ +#define MC_CMD_AOE_IN_FC_BOOT_CONTROL_OFST 4 +#define MC_CMD_AOE_IN_FC_BOOT_CONTROL_BOOT_ENABLE_LBN 0 +#define MC_CMD_AOE_IN_FC_BOOT_CONTROL_BOOT_ENABLE_WIDTH 1 + +/* MC_CMD_AOE_OUT_INFO msgresponse */ +#define MC_CMD_AOE_OUT_INFO_LEN 44 +/* JTAG IDCODE of CPLD */ +#define MC_CMD_AOE_OUT_INFO_CPLD_IDCODE_OFST 0 +/* Version of CPLD */ +#define MC_CMD_AOE_OUT_INFO_CPLD_VERSION_OFST 4 +/* JTAG IDCODE of FPGA */ +#define MC_CMD_AOE_OUT_INFO_FPGA_IDCODE_OFST 8 +/* JTAG USERCODE of FPGA */ +#define MC_CMD_AOE_OUT_INFO_FPGA_VERSION_OFST 12 +/* FPGA type - read from CPLD straps */ +#define MC_CMD_AOE_OUT_INFO_FPGA_TYPE_OFST 16 +#define MC_CMD_AOE_OUT_INFO_FPGA_TYPE_A5_C2 0x1 /* enum */ +#define MC_CMD_AOE_OUT_INFO_FPGA_TYPE_A7_C2 0x2 /* enum */ +/* FPGA state (debug) */ +#define MC_CMD_AOE_OUT_INFO_FPGA_STATE_OFST 20 +/* FPGA image - partition from which loaded */ +#define MC_CMD_AOE_OUT_INFO_FPGA_IMAGE_OFST 24 +/* FC state */ +#define MC_CMD_AOE_OUT_INFO_FC_STATE_OFST 28 +/* enum: Set if watchdog working */ +#define MC_CMD_AOE_OUT_INFO_WATCHDOG 0x1 +/* enum: Set if MC-FC communications working */ +#define MC_CMD_AOE_OUT_INFO_COMMS 0x2 +/* Random pieces of information */ +#define MC_CMD_AOE_OUT_INFO_FLAGS_OFST 32 +/* enum: Power to FPGA supplied by PEG connector, not PCIe bus */ +#define MC_CMD_AOE_OUT_INFO_PEG_POWER 0x1 +/* enum: CPLD apparently good */ +#define MC_CMD_AOE_OUT_INFO_CPLD_GOOD 0x2 +/* enum: FPGA working normally */ +#define MC_CMD_AOE_OUT_INFO_FPGA_GOOD 0x4 +/* enum: FPGA is powered */ +#define MC_CMD_AOE_OUT_INFO_FPGA_POWER 0x8 +/* enum: Board has incompatible SODIMMs fitted */ +#define MC_CMD_AOE_OUT_INFO_BAD_SODIMM 0x10 +/* enum: Board has ByteBlaster connected */ +#define MC_CMD_AOE_OUT_INFO_HAS_BYTEBLASTER 0x20 +/* enum: FPGA Boot flash has an invalid header. */ +#define MC_CMD_AOE_OUT_INFO_FPGA_BAD_BOOT_HDR 0x40 +/* enum: FPGA Application flash is accessible. */ +#define MC_CMD_AOE_OUT_INFO_FPGA_APP_FLASH_GOOD 0x80 +/* Revision of Modena and Sorrento boards. Sorrento can be R1_2 or R1_3. */ +#define MC_CMD_AOE_OUT_INFO_BOARD_REVISION_OFST 36 +#define MC_CMD_AOE_OUT_INFO_UNKNOWN 0x0 /* enum */ +#define MC_CMD_AOE_OUT_INFO_R1_0 0x10 /* enum */ +#define MC_CMD_AOE_OUT_INFO_R1_1 0x11 /* enum */ +#define MC_CMD_AOE_OUT_INFO_R1_2 0x12 /* enum */ +#define MC_CMD_AOE_OUT_INFO_R1_3 0x13 /* enum */ +/* Result of FC booting - not valid while a ByteBlaster is connected. */ +#define MC_CMD_AOE_OUT_INFO_FC_BOOT_RESULT_OFST 40 +/* enum: No error */ +#define MC_CMD_AOE_OUT_INFO_FC_BOOT_FAIL_NO_ERROR 0x0 +/* enum: Bad address set in CPLD */ +#define MC_CMD_AOE_OUT_INFO_FC_BOOT_FAIL_BAD_ADDRESS 0x1 +/* enum: Bad header */ +#define MC_CMD_AOE_OUT_INFO_FC_BOOT_FAIL_BAD_MAGIC 0x2 +/* enum: Bad text section details */ +#define MC_CMD_AOE_OUT_INFO_FC_BOOT_FAIL_BAD_TEXT 0x3 +/* enum: Bad checksum */ +#define MC_CMD_AOE_OUT_INFO_FC_BOOT_FAIL_BAD_CHECKSUM 0x4 +/* enum: Bad BSP */ +#define MC_CMD_AOE_OUT_INFO_FC_BOOT_FAIL_BAD_BSP 0x5 +/* enum: Flash mode is invalid */ +#define MC_CMD_AOE_OUT_INFO_FC_BOOT_FAIL_INVALID_FLASH_MODE 0x6 +/* enum: FC application loaded and execution attempted */ +#define MC_CMD_AOE_OUT_INFO_FC_BOOT_APP_EXECUTE 0x80 +/* enum: FC application Started */ +#define MC_CMD_AOE_OUT_INFO_FC_BOOT_APP_STARTED 0x81 +/* enum: No bootrom in FPGA */ +#define MC_CMD_AOE_OUT_INFO_FC_BOOT_NO_BOOTROM 0xff + +/* MC_CMD_AOE_OUT_CURRENTS msgresponse */ +#define MC_CMD_AOE_OUT_CURRENTS_LEN 68 +/* Set of currents and voltages (mA or mV as appropriate) */ +#define MC_CMD_AOE_OUT_CURRENTS_VALUES_OFST 0 +#define MC_CMD_AOE_OUT_CURRENTS_VALUES_LEN 4 +#define MC_CMD_AOE_OUT_CURRENTS_VALUES_NUM 17 +#define MC_CMD_AOE_OUT_CURRENTS_I_2V5 0x0 /* enum */ +#define MC_CMD_AOE_OUT_CURRENTS_I_1V8 0x1 /* enum */ +#define MC_CMD_AOE_OUT_CURRENTS_I_GXB 0x2 /* enum */ +#define MC_CMD_AOE_OUT_CURRENTS_I_PGM 0x3 /* enum */ +#define MC_CMD_AOE_OUT_CURRENTS_I_XCVR 0x4 /* enum */ +#define MC_CMD_AOE_OUT_CURRENTS_I_1V5 0x5 /* enum */ +#define MC_CMD_AOE_OUT_CURRENTS_V_3V3 0x6 /* enum */ +#define MC_CMD_AOE_OUT_CURRENTS_V_1V5 0x7 /* enum */ +#define MC_CMD_AOE_OUT_CURRENTS_I_IN 0x8 /* enum */ +#define MC_CMD_AOE_OUT_CURRENTS_I_OUT 0x9 /* enum */ +#define MC_CMD_AOE_OUT_CURRENTS_V_IN 0xa /* enum */ +#define MC_CMD_AOE_OUT_CURRENTS_I_OUT_DDR1 0xb /* enum */ +#define MC_CMD_AOE_OUT_CURRENTS_V_OUT_DDR1 0xc /* enum */ +#define MC_CMD_AOE_OUT_CURRENTS_I_OUT_DDR2 0xd /* enum */ +#define MC_CMD_AOE_OUT_CURRENTS_V_OUT_DDR2 0xe /* enum */ +#define MC_CMD_AOE_OUT_CURRENTS_I_OUT_DDR3 0xf /* enum */ +#define MC_CMD_AOE_OUT_CURRENTS_V_OUT_DDR3 0x10 /* enum */ + +/* MC_CMD_AOE_OUT_TEMPERATURES msgresponse */ +#define MC_CMD_AOE_OUT_TEMPERATURES_LEN 40 +/* Set of temperatures */ +#define MC_CMD_AOE_OUT_TEMPERATURES_VALUES_OFST 0 +#define MC_CMD_AOE_OUT_TEMPERATURES_VALUES_LEN 4 +#define MC_CMD_AOE_OUT_TEMPERATURES_VALUES_NUM 10 +/* enum: The first set of enum values are for Modena code. */ +#define MC_CMD_AOE_OUT_TEMPERATURES_MAIN_0 0x0 +#define MC_CMD_AOE_OUT_TEMPERATURES_MAIN_1 0x1 /* enum */ +#define MC_CMD_AOE_OUT_TEMPERATURES_IND_0 0x2 /* enum */ +#define MC_CMD_AOE_OUT_TEMPERATURES_IND_1 0x3 /* enum */ +#define MC_CMD_AOE_OUT_TEMPERATURES_VCCIO1 0x4 /* enum */ +#define MC_CMD_AOE_OUT_TEMPERATURES_VCCIO2 0x5 /* enum */ +#define MC_CMD_AOE_OUT_TEMPERATURES_VCCIO3 0x6 /* enum */ +#define MC_CMD_AOE_OUT_TEMPERATURES_PSU 0x7 /* enum */ +#define MC_CMD_AOE_OUT_TEMPERATURES_FPGA 0x8 /* enum */ +#define MC_CMD_AOE_OUT_TEMPERATURES_SIENA 0x9 /* enum */ +/* enum: The second set of enum values are for Sorrento code. */ +#define MC_CMD_AOE_OUT_TEMPERATURES_SORRENTO_MAIN_0 0x0 +#define MC_CMD_AOE_OUT_TEMPERATURES_SORRENTO_MAIN_1 0x1 /* enum */ +#define MC_CMD_AOE_OUT_TEMPERATURES_SORRENTO_IND_0 0x2 /* enum */ +#define MC_CMD_AOE_OUT_TEMPERATURES_SORRENTO_IND_1 0x3 /* enum */ +#define MC_CMD_AOE_OUT_TEMPERATURES_SORRENTO_SODIMM_0 0x4 /* enum */ +#define MC_CMD_AOE_OUT_TEMPERATURES_SORRENTO_SODIMM_1 0x5 /* enum */ +#define MC_CMD_AOE_OUT_TEMPERATURES_SORRENTO_FPGA 0x6 /* enum */ +#define MC_CMD_AOE_OUT_TEMPERATURES_SORRENTO_PHY0 0x7 /* enum */ +#define MC_CMD_AOE_OUT_TEMPERATURES_SORRENTO_PHY1 0x8 /* enum */ + +/* MC_CMD_AOE_OUT_CPLD_READ msgresponse */ +#define MC_CMD_AOE_OUT_CPLD_READ_LEN 4 +/* The value read from the CPLD */ +#define MC_CMD_AOE_OUT_CPLD_READ_VALUE_OFST 0 + +/* MC_CMD_AOE_OUT_FAN_FAILURES msgresponse */ +#define MC_CMD_AOE_OUT_FAN_FAILURES_LENMIN 4 +#define MC_CMD_AOE_OUT_FAN_FAILURES_LENMAX 252 +#define MC_CMD_AOE_OUT_FAN_FAILURES_LEN(num) (0+4*(num)) +/* Failure counts for each fan */ +#define MC_CMD_AOE_OUT_FAN_FAILURES_COUNT_OFST 0 +#define MC_CMD_AOE_OUT_FAN_FAILURES_COUNT_LEN 4 +#define MC_CMD_AOE_OUT_FAN_FAILURES_COUNT_MINNUM 1 +#define MC_CMD_AOE_OUT_FAN_FAILURES_COUNT_MAXNUM 63 + +/* MC_CMD_AOE_OUT_CPLD_REPROGRAM msgresponse */ +#define MC_CMD_AOE_OUT_CPLD_REPROGRAM_LEN 4 +/* Results of status command (only) */ +#define MC_CMD_AOE_OUT_CPLD_REPROGRAM_STATUS_OFST 0 + +/* MC_CMD_AOE_OUT_POWER_OFF msgresponse */ +#define MC_CMD_AOE_OUT_POWER_OFF_LEN 0 + +/* MC_CMD_AOE_OUT_POWER_ON msgresponse */ +#define MC_CMD_AOE_OUT_POWER_ON_LEN 0 + +/* MC_CMD_AOE_OUT_LOAD msgresponse */ +#define MC_CMD_AOE_OUT_LOAD_LEN 0 + +/* MC_CMD_AOE_OUT_MAC_STATS_DMA msgresponse */ +#define MC_CMD_AOE_OUT_MAC_STATS_DMA_LEN 0 + +/* MC_CMD_AOE_OUT_MAC_STATS_NO_DMA msgresponse: See MC_CMD_MAC_STATS_OUT_NO_DMA + * for details + */ +#define MC_CMD_AOE_OUT_MAC_STATS_NO_DMA_LEN (((MC_CMD_MAC_NSTATS*64))>>3) +#define MC_CMD_AOE_OUT_MAC_STATS_NO_DMA_STATISTICS_OFST 0 +#define MC_CMD_AOE_OUT_MAC_STATS_NO_DMA_STATISTICS_LEN 8 +#define MC_CMD_AOE_OUT_MAC_STATS_NO_DMA_STATISTICS_LO_OFST 0 +#define MC_CMD_AOE_OUT_MAC_STATS_NO_DMA_STATISTICS_HI_OFST 4 +#define MC_CMD_AOE_OUT_MAC_STATS_NO_DMA_STATISTICS_NUM MC_CMD_MAC_NSTATS + +/* MC_CMD_AOE_OUT_GET_PHY_MEDIA_INFO msgresponse */ +#define MC_CMD_AOE_OUT_GET_PHY_MEDIA_INFO_LENMIN 5 +#define MC_CMD_AOE_OUT_GET_PHY_MEDIA_INFO_LENMAX 252 +#define MC_CMD_AOE_OUT_GET_PHY_MEDIA_INFO_LEN(num) (4+1*(num)) +/* in bytes */ +#define MC_CMD_AOE_OUT_GET_PHY_MEDIA_INFO_DATALEN_OFST 0 +#define MC_CMD_AOE_OUT_GET_PHY_MEDIA_INFO_DATA_OFST 4 +#define MC_CMD_AOE_OUT_GET_PHY_MEDIA_INFO_DATA_LEN 1 +#define MC_CMD_AOE_OUT_GET_PHY_MEDIA_INFO_DATA_MINNUM 1 +#define MC_CMD_AOE_OUT_GET_PHY_MEDIA_INFO_DATA_MAXNUM 248 + +/* MC_CMD_AOE_OUT_JTAG_WRITE msgresponse */ +#define MC_CMD_AOE_OUT_JTAG_WRITE_LENMIN 12 +#define MC_CMD_AOE_OUT_JTAG_WRITE_LENMAX 252 +#define MC_CMD_AOE_OUT_JTAG_WRITE_LEN(num) (8+4*(num)) +/* Used to align the in and out data blocks so the MC can re-use the cmd */ +#define MC_CMD_AOE_OUT_JTAG_WRITE_DATALEN_OFST 0 +/* out bytes */ +#define MC_CMD_AOE_OUT_JTAG_WRITE_PAD_OFST 4 +#define MC_CMD_AOE_OUT_JTAG_WRITE_DATA_OFST 8 +#define MC_CMD_AOE_OUT_JTAG_WRITE_DATA_LEN 4 +#define MC_CMD_AOE_OUT_JTAG_WRITE_DATA_MINNUM 1 +#define MC_CMD_AOE_OUT_JTAG_WRITE_DATA_MAXNUM 61 + +/* MC_CMD_AOE_OUT_FPGA_ACCESS msgresponse */ +#define MC_CMD_AOE_OUT_FPGA_ACCESS_LEN 0 + +/* MC_CMD_AOE_OUT_DDR msgresponse */ +#define MC_CMD_AOE_OUT_DDR_LENMIN 17 +#define MC_CMD_AOE_OUT_DDR_LENMAX 252 +#define MC_CMD_AOE_OUT_DDR_LEN(num) (16+1*(num)) +/* Information on the module. */ +#define MC_CMD_AOE_OUT_DDR_FLAGS_OFST 0 +#define MC_CMD_AOE_OUT_DDR_PRESENT_LBN 0 +#define MC_CMD_AOE_OUT_DDR_PRESENT_WIDTH 1 +#define MC_CMD_AOE_OUT_DDR_POWERED_LBN 1 +#define MC_CMD_AOE_OUT_DDR_POWERED_WIDTH 1 +#define MC_CMD_AOE_OUT_DDR_OPERATIONAL_LBN 2 +#define MC_CMD_AOE_OUT_DDR_OPERATIONAL_WIDTH 1 +#define MC_CMD_AOE_OUT_DDR_NOT_REACHABLE_LBN 3 +#define MC_CMD_AOE_OUT_DDR_NOT_REACHABLE_WIDTH 1 +/* Memory size, in MB. */ +#define MC_CMD_AOE_OUT_DDR_CAPACITY_OFST 4 +/* The memory type, as reported from SPD information */ +#define MC_CMD_AOE_OUT_DDR_TYPE_OFST 8 +/* Nominal voltage of the module (as applied) */ +#define MC_CMD_AOE_OUT_DDR_VOLTAGE_OFST 12 +/* SPD data read from the module */ +#define MC_CMD_AOE_OUT_DDR_SPD_OFST 16 +#define MC_CMD_AOE_OUT_DDR_SPD_LEN 1 +#define MC_CMD_AOE_OUT_DDR_SPD_MINNUM 1 +#define MC_CMD_AOE_OUT_DDR_SPD_MAXNUM 236 + +/* MC_CMD_AOE_OUT_SET_MTU_OFFSET msgresponse */ +#define MC_CMD_AOE_OUT_SET_MTU_OFFSET_LEN 0 + +/* MC_CMD_AOE_OUT_LINK_STATE msgresponse */ +#define MC_CMD_AOE_OUT_LINK_STATE_LEN 0 + +/* MC_CMD_AOE_OUT_SIENA_STATS msgresponse */ +#define MC_CMD_AOE_OUT_SIENA_STATS_LEN 0 + +/* MC_CMD_AOE_OUT_ASIC_STATS msgresponse */ +#define MC_CMD_AOE_OUT_ASIC_STATS_LEN 0 + +/* MC_CMD_AOE_OUT_FC msgresponse */ +#define MC_CMD_AOE_OUT_FC_LEN 0 + +/* MC_CMD_AOE_OUT_DDR_ECC_STATUS msgresponse */ +#define MC_CMD_AOE_OUT_DDR_ECC_STATUS_LEN 8 +/* Flags describing status info on the module. */ +#define MC_CMD_AOE_OUT_DDR_ECC_STATUS_FLAGS_OFST 0 +#define MC_CMD_AOE_OUT_DDR_ECC_STATUS_VALID_LBN 0 +#define MC_CMD_AOE_OUT_DDR_ECC_STATUS_VALID_WIDTH 1 +/* DDR ECC status on the module. */ +#define MC_CMD_AOE_OUT_DDR_ECC_STATUS_STATUS_OFST 4 +#define MC_CMD_AOE_OUT_DDR_ECC_STATUS_SBE_LBN 0 +#define MC_CMD_AOE_OUT_DDR_ECC_STATUS_SBE_WIDTH 1 +#define MC_CMD_AOE_OUT_DDR_ECC_STATUS_DBE_LBN 1 +#define MC_CMD_AOE_OUT_DDR_ECC_STATUS_DBE_WIDTH 1 +#define MC_CMD_AOE_OUT_DDR_ECC_STATUS_CORDROP_LBN 2 +#define MC_CMD_AOE_OUT_DDR_ECC_STATUS_CORDROP_WIDTH 1 +#define MC_CMD_AOE_OUT_DDR_ECC_STATUS_SBE_COUNT_LBN 8 +#define MC_CMD_AOE_OUT_DDR_ECC_STATUS_SBE_COUNT_WIDTH 8 +#define MC_CMD_AOE_OUT_DDR_ECC_STATUS_DBE_COUNT_LBN 16 +#define MC_CMD_AOE_OUT_DDR_ECC_STATUS_DBE_COUNT_WIDTH 8 +#define MC_CMD_AOE_OUT_DDR_ECC_STATUS_CORDROP_COUNT_LBN 24 +#define MC_CMD_AOE_OUT_DDR_ECC_STATUS_CORDROP_COUNT_WIDTH 8 + +/* MC_CMD_AOE_OUT_MC_SPI_MASTER_READ msgresponse */ +#define MC_CMD_AOE_OUT_MC_SPI_MASTER_READ_LEN 4 +#define MC_CMD_AOE_OUT_MC_SPI_MASTER_READ_DATA_OFST 0 + +/* MC_CMD_AOE_OUT_MC_SPI_MASTER_WRITE msgresponse */ +#define MC_CMD_AOE_OUT_MC_SPI_MASTER_WRITE_LEN 0 + +/* MC_CMD_AOE_OUT_MC_SPI_MASTER msgresponse */ +#define MC_CMD_AOE_OUT_MC_SPI_MASTER_LEN 0 + +/* MC_CMD_AOE_OUT_FC_BOOT msgresponse */ +#define MC_CMD_AOE_OUT_FC_BOOT_LEN 0 + + +/***********************************/ +/* MC_CMD_PTP + * Perform PTP operation + */ +#define MC_CMD_PTP 0xb +#undef MC_CMD_0xb_PRIVILEGE_CTG + +#define MC_CMD_0xb_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_PTP_IN msgrequest */ +#define MC_CMD_PTP_IN_LEN 1 +/* PTP operation code */ +#define MC_CMD_PTP_IN_OP_OFST 0 +#define MC_CMD_PTP_IN_OP_LEN 1 +/* enum: Enable PTP packet timestamping operation. */ +#define MC_CMD_PTP_OP_ENABLE 0x1 +/* enum: Disable PTP packet timestamping operation. */ +#define MC_CMD_PTP_OP_DISABLE 0x2 +/* enum: Send a PTP packet. */ +#define MC_CMD_PTP_OP_TRANSMIT 0x3 +/* enum: Read the current NIC time. */ +#define MC_CMD_PTP_OP_READ_NIC_TIME 0x4 +/* enum: Get the current PTP status. */ +#define MC_CMD_PTP_OP_STATUS 0x5 +/* enum: Adjust the PTP NIC's time. */ +#define MC_CMD_PTP_OP_ADJUST 0x6 +/* enum: Synchronize host and NIC time. */ +#define MC_CMD_PTP_OP_SYNCHRONIZE 0x7 +/* enum: Basic manufacturing tests. */ +#define MC_CMD_PTP_OP_MANFTEST_BASIC 0x8 +/* enum: Packet based manufacturing tests. */ +#define MC_CMD_PTP_OP_MANFTEST_PACKET 0x9 +/* enum: Reset some of the PTP related statistics */ +#define MC_CMD_PTP_OP_RESET_STATS 0xa +/* enum: Debug operations to MC. */ +#define MC_CMD_PTP_OP_DEBUG 0xb +/* enum: Read an FPGA register */ +#define MC_CMD_PTP_OP_FPGAREAD 0xc +/* enum: Write an FPGA register */ +#define MC_CMD_PTP_OP_FPGAWRITE 0xd +/* enum: Apply an offset to the NIC clock */ +#define MC_CMD_PTP_OP_CLOCK_OFFSET_ADJUST 0xe +/* enum: Change Apply an offset to the NIC clock */ +#define MC_CMD_PTP_OP_CLOCK_FREQ_ADJUST 0xf +/* enum: Set the MC packet filter VLAN tags for received PTP packets */ +#define MC_CMD_PTP_OP_RX_SET_VLAN_FILTER 0x10 +/* enum: Set the MC packet filter UUID for received PTP packets */ +#define MC_CMD_PTP_OP_RX_SET_UUID_FILTER 0x11 +/* enum: Set the MC packet filter Domain for received PTP packets */ +#define MC_CMD_PTP_OP_RX_SET_DOMAIN_FILTER 0x12 +/* enum: Set the clock source */ +#define MC_CMD_PTP_OP_SET_CLK_SRC 0x13 +/* enum: Reset value of Timer Reg. */ +#define MC_CMD_PTP_OP_RST_CLK 0x14 +/* enum: Enable the forwarding of PPS events to the host */ +#define MC_CMD_PTP_OP_PPS_ENABLE 0x15 +/* enum: Get the time format used by this NIC for PTP operations */ +#define MC_CMD_PTP_OP_GET_TIME_FORMAT 0x16 +/* enum: Get the clock attributes. NOTE- extended version of + * MC_CMD_PTP_OP_GET_TIME_FORMAT + */ +#define MC_CMD_PTP_OP_GET_ATTRIBUTES 0x16 +/* enum: Get corrections that should be applied to the various different + * timestamps + */ +#define MC_CMD_PTP_OP_GET_TIMESTAMP_CORRECTIONS 0x17 +/* enum: Subscribe to receive periodic time events indicating the current NIC + * time + */ +#define MC_CMD_PTP_OP_TIME_EVENT_SUBSCRIBE 0x18 +/* enum: Unsubscribe to stop receiving time events */ +#define MC_CMD_PTP_OP_TIME_EVENT_UNSUBSCRIBE 0x19 +/* enum: PPS based manfacturing tests. Requires PPS output to be looped to PPS + * input on the same NIC. + */ +#define MC_CMD_PTP_OP_MANFTEST_PPS 0x1a +/* enum: Set the PTP sync status. Status is used by firmware to report to event + * subscribers. + */ +#define MC_CMD_PTP_OP_SET_SYNC_STATUS 0x1b +/* enum: Above this for future use. */ +#define MC_CMD_PTP_OP_MAX 0x1c + +/* MC_CMD_PTP_IN_ENABLE msgrequest */ +#define MC_CMD_PTP_IN_ENABLE_LEN 16 +#define MC_CMD_PTP_IN_CMD_OFST 0 +#define MC_CMD_PTP_IN_PERIPH_ID_OFST 4 +/* Event queue for PTP events */ +#define MC_CMD_PTP_IN_ENABLE_QUEUE_OFST 8 +/* PTP timestamping mode */ +#define MC_CMD_PTP_IN_ENABLE_MODE_OFST 12 +/* enum: PTP, version 1 */ +#define MC_CMD_PTP_MODE_V1 0x0 +/* enum: PTP, version 1, with VLAN headers - deprecated */ +#define MC_CMD_PTP_MODE_V1_VLAN 0x1 +/* enum: PTP, version 2 */ +#define MC_CMD_PTP_MODE_V2 0x2 +/* enum: PTP, version 2, with VLAN headers - deprecated */ +#define MC_CMD_PTP_MODE_V2_VLAN 0x3 +/* enum: PTP, version 2, with improved UUID filtering */ +#define MC_CMD_PTP_MODE_V2_ENHANCED 0x4 +/* enum: FCoE (seconds and microseconds) */ +#define MC_CMD_PTP_MODE_FCOE 0x5 + +/* MC_CMD_PTP_IN_DISABLE msgrequest */ +#define MC_CMD_PTP_IN_DISABLE_LEN 8 +/* MC_CMD_PTP_IN_CMD_OFST 0 */ +/* MC_CMD_PTP_IN_PERIPH_ID_OFST 4 */ + +/* MC_CMD_PTP_IN_TRANSMIT msgrequest */ +#define MC_CMD_PTP_IN_TRANSMIT_LENMIN 13 +#define MC_CMD_PTP_IN_TRANSMIT_LENMAX 252 +#define MC_CMD_PTP_IN_TRANSMIT_LEN(num) (12+1*(num)) +/* MC_CMD_PTP_IN_CMD_OFST 0 */ +/* MC_CMD_PTP_IN_PERIPH_ID_OFST 4 */ +/* Transmit packet length */ +#define MC_CMD_PTP_IN_TRANSMIT_LENGTH_OFST 8 +/* Transmit packet data */ +#define MC_CMD_PTP_IN_TRANSMIT_PACKET_OFST 12 +#define MC_CMD_PTP_IN_TRANSMIT_PACKET_LEN 1 +#define MC_CMD_PTP_IN_TRANSMIT_PACKET_MINNUM 1 +#define MC_CMD_PTP_IN_TRANSMIT_PACKET_MAXNUM 240 + +/* MC_CMD_PTP_IN_READ_NIC_TIME msgrequest */ +#define MC_CMD_PTP_IN_READ_NIC_TIME_LEN 8 +/* MC_CMD_PTP_IN_CMD_OFST 0 */ +/* MC_CMD_PTP_IN_PERIPH_ID_OFST 4 */ + +/* MC_CMD_PTP_IN_STATUS msgrequest */ +#define MC_CMD_PTP_IN_STATUS_LEN 8 +/* MC_CMD_PTP_IN_CMD_OFST 0 */ +/* MC_CMD_PTP_IN_PERIPH_ID_OFST 4 */ + +/* MC_CMD_PTP_IN_ADJUST msgrequest */ +#define MC_CMD_PTP_IN_ADJUST_LEN 24 +/* MC_CMD_PTP_IN_CMD_OFST 0 */ +/* MC_CMD_PTP_IN_PERIPH_ID_OFST 4 */ +/* Frequency adjustment 40 bit fixed point ns */ +#define MC_CMD_PTP_IN_ADJUST_FREQ_OFST 8 +#define MC_CMD_PTP_IN_ADJUST_FREQ_LEN 8 +#define MC_CMD_PTP_IN_ADJUST_FREQ_LO_OFST 8 +#define MC_CMD_PTP_IN_ADJUST_FREQ_HI_OFST 12 +/* enum: Number of fractional bits in frequency adjustment */ +#define MC_CMD_PTP_IN_ADJUST_BITS 0x28 +/* Time adjustment in seconds */ +#define MC_CMD_PTP_IN_ADJUST_SECONDS_OFST 16 +/* Time adjustment major value */ +#define MC_CMD_PTP_IN_ADJUST_MAJOR_OFST 16 +/* Time adjustment in nanoseconds */ +#define MC_CMD_PTP_IN_ADJUST_NANOSECONDS_OFST 20 +/* Time adjustment minor value */ +#define MC_CMD_PTP_IN_ADJUST_MINOR_OFST 20 + +/* MC_CMD_PTP_IN_SYNCHRONIZE msgrequest */ +#define MC_CMD_PTP_IN_SYNCHRONIZE_LEN 20 +/* MC_CMD_PTP_IN_CMD_OFST 0 */ +/* MC_CMD_PTP_IN_PERIPH_ID_OFST 4 */ +/* Number of time readings to capture */ +#define MC_CMD_PTP_IN_SYNCHRONIZE_NUMTIMESETS_OFST 8 +/* Host address in which to write "synchronization started" indication (64 + * bits) + */ +#define MC_CMD_PTP_IN_SYNCHRONIZE_START_ADDR_OFST 12 +#define MC_CMD_PTP_IN_SYNCHRONIZE_START_ADDR_LEN 8 +#define MC_CMD_PTP_IN_SYNCHRONIZE_START_ADDR_LO_OFST 12 +#define MC_CMD_PTP_IN_SYNCHRONIZE_START_ADDR_HI_OFST 16 + +/* MC_CMD_PTP_IN_MANFTEST_BASIC msgrequest */ +#define MC_CMD_PTP_IN_MANFTEST_BASIC_LEN 8 +/* MC_CMD_PTP_IN_CMD_OFST 0 */ +/* MC_CMD_PTP_IN_PERIPH_ID_OFST 4 */ + +/* MC_CMD_PTP_IN_MANFTEST_PACKET msgrequest */ +#define MC_CMD_PTP_IN_MANFTEST_PACKET_LEN 12 +/* MC_CMD_PTP_IN_CMD_OFST 0 */ +/* MC_CMD_PTP_IN_PERIPH_ID_OFST 4 */ +/* Enable or disable packet testing */ +#define MC_CMD_PTP_IN_MANFTEST_PACKET_TEST_ENABLE_OFST 8 + +/* MC_CMD_PTP_IN_RESET_STATS msgrequest */ +#define MC_CMD_PTP_IN_RESET_STATS_LEN 8 +/* MC_CMD_PTP_IN_CMD_OFST 0 */ +/* Reset PTP statistics */ +/* MC_CMD_PTP_IN_PERIPH_ID_OFST 4 */ + +/* MC_CMD_PTP_IN_DEBUG msgrequest */ +#define MC_CMD_PTP_IN_DEBUG_LEN 12 +/* MC_CMD_PTP_IN_CMD_OFST 0 */ +/* MC_CMD_PTP_IN_PERIPH_ID_OFST 4 */ +/* Debug operations */ +#define MC_CMD_PTP_IN_DEBUG_DEBUG_PARAM_OFST 8 + +/* MC_CMD_PTP_IN_FPGAREAD msgrequest */ +#define MC_CMD_PTP_IN_FPGAREAD_LEN 16 +/* MC_CMD_PTP_IN_CMD_OFST 0 */ +/* MC_CMD_PTP_IN_PERIPH_ID_OFST 4 */ +#define MC_CMD_PTP_IN_FPGAREAD_ADDR_OFST 8 +#define MC_CMD_PTP_IN_FPGAREAD_NUMBYTES_OFST 12 + +/* MC_CMD_PTP_IN_FPGAWRITE msgrequest */ +#define MC_CMD_PTP_IN_FPGAWRITE_LENMIN 13 +#define MC_CMD_PTP_IN_FPGAWRITE_LENMAX 252 +#define MC_CMD_PTP_IN_FPGAWRITE_LEN(num) (12+1*(num)) +/* MC_CMD_PTP_IN_CMD_OFST 0 */ +/* MC_CMD_PTP_IN_PERIPH_ID_OFST 4 */ +#define MC_CMD_PTP_IN_FPGAWRITE_ADDR_OFST 8 +#define MC_CMD_PTP_IN_FPGAWRITE_BUFFER_OFST 12 +#define MC_CMD_PTP_IN_FPGAWRITE_BUFFER_LEN 1 +#define MC_CMD_PTP_IN_FPGAWRITE_BUFFER_MINNUM 1 +#define MC_CMD_PTP_IN_FPGAWRITE_BUFFER_MAXNUM 240 + +/* MC_CMD_PTP_IN_CLOCK_OFFSET_ADJUST msgrequest */ +#define MC_CMD_PTP_IN_CLOCK_OFFSET_ADJUST_LEN 16 +/* MC_CMD_PTP_IN_CMD_OFST 0 */ +/* MC_CMD_PTP_IN_PERIPH_ID_OFST 4 */ +/* Time adjustment in seconds */ +#define MC_CMD_PTP_IN_CLOCK_OFFSET_ADJUST_SECONDS_OFST 8 +/* Time adjustment major value */ +#define MC_CMD_PTP_IN_CLOCK_OFFSET_ADJUST_MAJOR_OFST 8 +/* Time adjustment in nanoseconds */ +#define MC_CMD_PTP_IN_CLOCK_OFFSET_ADJUST_NANOSECONDS_OFST 12 +/* Time adjustment minor value */ +#define MC_CMD_PTP_IN_CLOCK_OFFSET_ADJUST_MINOR_OFST 12 + +/* MC_CMD_PTP_IN_CLOCK_FREQ_ADJUST msgrequest */ +#define MC_CMD_PTP_IN_CLOCK_FREQ_ADJUST_LEN 16 +/* MC_CMD_PTP_IN_CMD_OFST 0 */ +/* MC_CMD_PTP_IN_PERIPH_ID_OFST 4 */ +/* Frequency adjustment 40 bit fixed point ns */ +#define MC_CMD_PTP_IN_CLOCK_FREQ_ADJUST_FREQ_OFST 8 +#define MC_CMD_PTP_IN_CLOCK_FREQ_ADJUST_FREQ_LEN 8 +#define MC_CMD_PTP_IN_CLOCK_FREQ_ADJUST_FREQ_LO_OFST 8 +#define MC_CMD_PTP_IN_CLOCK_FREQ_ADJUST_FREQ_HI_OFST 12 +/* enum: Number of fractional bits in frequency adjustment */ +/* MC_CMD_PTP_IN_ADJUST_BITS 0x28 */ + +/* MC_CMD_PTP_IN_RX_SET_VLAN_FILTER msgrequest */ +#define MC_CMD_PTP_IN_RX_SET_VLAN_FILTER_LEN 24 +/* MC_CMD_PTP_IN_CMD_OFST 0 */ +/* MC_CMD_PTP_IN_PERIPH_ID_OFST 4 */ +/* Number of VLAN tags, 0 if not VLAN */ +#define MC_CMD_PTP_IN_RX_SET_VLAN_FILTER_NUM_VLAN_TAGS_OFST 8 +/* Set of VLAN tags to filter against */ +#define MC_CMD_PTP_IN_RX_SET_VLAN_FILTER_VLAN_TAG_OFST 12 +#define MC_CMD_PTP_IN_RX_SET_VLAN_FILTER_VLAN_TAG_LEN 4 +#define MC_CMD_PTP_IN_RX_SET_VLAN_FILTER_VLAN_TAG_NUM 3 + +/* MC_CMD_PTP_IN_RX_SET_UUID_FILTER msgrequest */ +#define MC_CMD_PTP_IN_RX_SET_UUID_FILTER_LEN 20 +/* MC_CMD_PTP_IN_CMD_OFST 0 */ +/* MC_CMD_PTP_IN_PERIPH_ID_OFST 4 */ +/* 1 to enable UUID filtering, 0 to disable */ +#define MC_CMD_PTP_IN_RX_SET_UUID_FILTER_ENABLE_OFST 8 +/* UUID to filter against */ +#define MC_CMD_PTP_IN_RX_SET_UUID_FILTER_UUID_OFST 12 +#define MC_CMD_PTP_IN_RX_SET_UUID_FILTER_UUID_LEN 8 +#define MC_CMD_PTP_IN_RX_SET_UUID_FILTER_UUID_LO_OFST 12 +#define MC_CMD_PTP_IN_RX_SET_UUID_FILTER_UUID_HI_OFST 16 + +/* MC_CMD_PTP_IN_RX_SET_DOMAIN_FILTER msgrequest */ +#define MC_CMD_PTP_IN_RX_SET_DOMAIN_FILTER_LEN 16 +/* MC_CMD_PTP_IN_CMD_OFST 0 */ +/* MC_CMD_PTP_IN_PERIPH_ID_OFST 4 */ +/* 1 to enable Domain filtering, 0 to disable */ +#define MC_CMD_PTP_IN_RX_SET_DOMAIN_FILTER_ENABLE_OFST 8 +/* Domain number to filter against */ +#define MC_CMD_PTP_IN_RX_SET_DOMAIN_FILTER_DOMAIN_OFST 12 + +/* MC_CMD_PTP_IN_SET_CLK_SRC msgrequest */ +#define MC_CMD_PTP_IN_SET_CLK_SRC_LEN 12 +/* MC_CMD_PTP_IN_CMD_OFST 0 */ +/* MC_CMD_PTP_IN_PERIPH_ID_OFST 4 */ +/* Set the clock source. */ +#define MC_CMD_PTP_IN_SET_CLK_SRC_CLK_OFST 8 +/* enum: Internal. */ +#define MC_CMD_PTP_CLK_SRC_INTERNAL 0x0 +/* enum: External. */ +#define MC_CMD_PTP_CLK_SRC_EXTERNAL 0x1 + +/* MC_CMD_PTP_IN_RST_CLK msgrequest */ +#define MC_CMD_PTP_IN_RST_CLK_LEN 8 +/* MC_CMD_PTP_IN_CMD_OFST 0 */ +/* Reset value of Timer Reg. */ +/* MC_CMD_PTP_IN_PERIPH_ID_OFST 4 */ + +/* MC_CMD_PTP_IN_PPS_ENABLE msgrequest */ +#define MC_CMD_PTP_IN_PPS_ENABLE_LEN 12 +/* MC_CMD_PTP_IN_CMD_OFST 0 */ +/* Enable or disable */ +#define MC_CMD_PTP_IN_PPS_ENABLE_OP_OFST 4 +/* enum: Enable */ +#define MC_CMD_PTP_ENABLE_PPS 0x0 +/* enum: Disable */ +#define MC_CMD_PTP_DISABLE_PPS 0x1 +/* Queue id to send events back */ +#define MC_CMD_PTP_IN_PPS_ENABLE_QUEUE_ID_OFST 8 + +/* MC_CMD_PTP_IN_GET_TIME_FORMAT msgrequest */ +#define MC_CMD_PTP_IN_GET_TIME_FORMAT_LEN 8 +/* MC_CMD_PTP_IN_CMD_OFST 0 */ +/* MC_CMD_PTP_IN_PERIPH_ID_OFST 4 */ + +/* MC_CMD_PTP_IN_GET_ATTRIBUTES msgrequest */ +#define MC_CMD_PTP_IN_GET_ATTRIBUTES_LEN 8 +/* MC_CMD_PTP_IN_CMD_OFST 0 */ +/* MC_CMD_PTP_IN_PERIPH_ID_OFST 4 */ + +/* MC_CMD_PTP_IN_GET_TIMESTAMP_CORRECTIONS msgrequest */ +#define MC_CMD_PTP_IN_GET_TIMESTAMP_CORRECTIONS_LEN 8 +/* MC_CMD_PTP_IN_CMD_OFST 0 */ +/* MC_CMD_PTP_IN_PERIPH_ID_OFST 4 */ + +/* MC_CMD_PTP_IN_TIME_EVENT_SUBSCRIBE msgrequest */ +#define MC_CMD_PTP_IN_TIME_EVENT_SUBSCRIBE_LEN 12 +/* MC_CMD_PTP_IN_CMD_OFST 0 */ +/* MC_CMD_PTP_IN_PERIPH_ID_OFST 4 */ +/* Original field containing queue ID. Now extended to include flags. */ +#define MC_CMD_PTP_IN_TIME_EVENT_SUBSCRIBE_QUEUE_OFST 8 +#define MC_CMD_PTP_IN_TIME_EVENT_SUBSCRIBE_QUEUE_ID_LBN 0 +#define MC_CMD_PTP_IN_TIME_EVENT_SUBSCRIBE_QUEUE_ID_WIDTH 16 +#define MC_CMD_PTP_IN_TIME_EVENT_SUBSCRIBE_REPORT_SYNC_STATUS_LBN 31 +#define MC_CMD_PTP_IN_TIME_EVENT_SUBSCRIBE_REPORT_SYNC_STATUS_WIDTH 1 + +/* MC_CMD_PTP_IN_TIME_EVENT_UNSUBSCRIBE msgrequest */ +#define MC_CMD_PTP_IN_TIME_EVENT_UNSUBSCRIBE_LEN 16 +/* MC_CMD_PTP_IN_CMD_OFST 0 */ +/* MC_CMD_PTP_IN_PERIPH_ID_OFST 4 */ +/* Unsubscribe options */ +#define MC_CMD_PTP_IN_TIME_EVENT_UNSUBSCRIBE_CONTROL_OFST 8 +/* enum: Unsubscribe a single queue */ +#define MC_CMD_PTP_IN_TIME_EVENT_UNSUBSCRIBE_SINGLE 0x0 +/* enum: Unsubscribe all queues */ +#define MC_CMD_PTP_IN_TIME_EVENT_UNSUBSCRIBE_ALL 0x1 +/* Event queue ID */ +#define MC_CMD_PTP_IN_TIME_EVENT_UNSUBSCRIBE_QUEUE_OFST 12 + +/* MC_CMD_PTP_IN_MANFTEST_PPS msgrequest */ +#define MC_CMD_PTP_IN_MANFTEST_PPS_LEN 12 +/* MC_CMD_PTP_IN_CMD_OFST 0 */ +/* MC_CMD_PTP_IN_PERIPH_ID_OFST 4 */ +/* 1 to enable PPS test mode, 0 to disable and return result. */ +#define MC_CMD_PTP_IN_MANFTEST_PPS_TEST_ENABLE_OFST 8 + +/* MC_CMD_PTP_IN_SET_SYNC_STATUS msgrequest */ +#define MC_CMD_PTP_IN_SET_SYNC_STATUS_LEN 24 +/* MC_CMD_PTP_IN_CMD_OFST 0 */ +/* MC_CMD_PTP_IN_PERIPH_ID_OFST 4 */ +/* NIC - Host System Clock Synchronization status */ +#define MC_CMD_PTP_IN_SET_SYNC_STATUS_STATUS_OFST 8 +/* enum: Host System clock and NIC clock are not in sync */ +#define MC_CMD_PTP_IN_SET_SYNC_STATUS_NOT_IN_SYNC 0x0 +/* enum: Host System clock and NIC clock are synchronized */ +#define MC_CMD_PTP_IN_SET_SYNC_STATUS_IN_SYNC 0x1 +/* If synchronized, number of seconds until clocks should be considered to be + * no longer in sync. + */ +#define MC_CMD_PTP_IN_SET_SYNC_STATUS_TIMEOUT_OFST 12 +#define MC_CMD_PTP_IN_SET_SYNC_STATUS_RESERVED0_OFST 16 +#define MC_CMD_PTP_IN_SET_SYNC_STATUS_RESERVED1_OFST 20 + +/* MC_CMD_PTP_OUT msgresponse */ +#define MC_CMD_PTP_OUT_LEN 0 + +/* MC_CMD_PTP_OUT_TRANSMIT msgresponse */ +#define MC_CMD_PTP_OUT_TRANSMIT_LEN 8 +/* Value of seconds timestamp */ +#define MC_CMD_PTP_OUT_TRANSMIT_SECONDS_OFST 0 +/* Timestamp major value */ +#define MC_CMD_PTP_OUT_TRANSMIT_MAJOR_OFST 0 +/* Value of nanoseconds timestamp */ +#define MC_CMD_PTP_OUT_TRANSMIT_NANOSECONDS_OFST 4 +/* Timestamp minor value */ +#define MC_CMD_PTP_OUT_TRANSMIT_MINOR_OFST 4 + +/* MC_CMD_PTP_OUT_TIME_EVENT_SUBSCRIBE msgresponse */ +#define MC_CMD_PTP_OUT_TIME_EVENT_SUBSCRIBE_LEN 0 + +/* MC_CMD_PTP_OUT_TIME_EVENT_UNSUBSCRIBE msgresponse */ +#define MC_CMD_PTP_OUT_TIME_EVENT_UNSUBSCRIBE_LEN 0 + +/* MC_CMD_PTP_OUT_READ_NIC_TIME msgresponse */ +#define MC_CMD_PTP_OUT_READ_NIC_TIME_LEN 8 +/* Value of seconds timestamp */ +#define MC_CMD_PTP_OUT_READ_NIC_TIME_SECONDS_OFST 0 +/* Timestamp major value */ +#define MC_CMD_PTP_OUT_READ_NIC_TIME_MAJOR_OFST 0 +/* Value of nanoseconds timestamp */ +#define MC_CMD_PTP_OUT_READ_NIC_TIME_NANOSECONDS_OFST 4 +/* Timestamp minor value */ +#define MC_CMD_PTP_OUT_READ_NIC_TIME_MINOR_OFST 4 + +/* MC_CMD_PTP_OUT_STATUS msgresponse */ +#define MC_CMD_PTP_OUT_STATUS_LEN 64 +/* Frequency of NIC's hardware clock */ +#define MC_CMD_PTP_OUT_STATUS_CLOCK_FREQ_OFST 0 +/* Number of packets transmitted and timestamped */ +#define MC_CMD_PTP_OUT_STATUS_STATS_TX_OFST 4 +/* Number of packets received and timestamped */ +#define MC_CMD_PTP_OUT_STATUS_STATS_RX_OFST 8 +/* Number of packets timestamped by the FPGA */ +#define MC_CMD_PTP_OUT_STATUS_STATS_TS_OFST 12 +/* Number of packets filter matched */ +#define MC_CMD_PTP_OUT_STATUS_STATS_FM_OFST 16 +/* Number of packets not filter matched */ +#define MC_CMD_PTP_OUT_STATUS_STATS_NFM_OFST 20 +/* Number of PPS overflows (noise on input?) */ +#define MC_CMD_PTP_OUT_STATUS_STATS_PPS_OFLOW_OFST 24 +/* Number of PPS bad periods */ +#define MC_CMD_PTP_OUT_STATUS_STATS_PPS_BAD_OFST 28 +/* Minimum period of PPS pulse in nanoseconds */ +#define MC_CMD_PTP_OUT_STATUS_STATS_PPS_PER_MIN_OFST 32 +/* Maximum period of PPS pulse in nanoseconds */ +#define MC_CMD_PTP_OUT_STATUS_STATS_PPS_PER_MAX_OFST 36 +/* Last period of PPS pulse in nanoseconds */ +#define MC_CMD_PTP_OUT_STATUS_STATS_PPS_PER_LAST_OFST 40 +/* Mean period of PPS pulse in nanoseconds */ +#define MC_CMD_PTP_OUT_STATUS_STATS_PPS_PER_MEAN_OFST 44 +/* Minimum offset of PPS pulse in nanoseconds (signed) */ +#define MC_CMD_PTP_OUT_STATUS_STATS_PPS_OFF_MIN_OFST 48 +/* Maximum offset of PPS pulse in nanoseconds (signed) */ +#define MC_CMD_PTP_OUT_STATUS_STATS_PPS_OFF_MAX_OFST 52 +/* Last offset of PPS pulse in nanoseconds (signed) */ +#define MC_CMD_PTP_OUT_STATUS_STATS_PPS_OFF_LAST_OFST 56 +/* Mean offset of PPS pulse in nanoseconds (signed) */ +#define MC_CMD_PTP_OUT_STATUS_STATS_PPS_OFF_MEAN_OFST 60 + +/* MC_CMD_PTP_OUT_SYNCHRONIZE msgresponse */ +#define MC_CMD_PTP_OUT_SYNCHRONIZE_LENMIN 20 +#define MC_CMD_PTP_OUT_SYNCHRONIZE_LENMAX 240 +#define MC_CMD_PTP_OUT_SYNCHRONIZE_LEN(num) (0+20*(num)) +/* A set of host and NIC times */ +#define MC_CMD_PTP_OUT_SYNCHRONIZE_TIMESET_OFST 0 +#define MC_CMD_PTP_OUT_SYNCHRONIZE_TIMESET_LEN 20 +#define MC_CMD_PTP_OUT_SYNCHRONIZE_TIMESET_MINNUM 1 +#define MC_CMD_PTP_OUT_SYNCHRONIZE_TIMESET_MAXNUM 12 +/* Host time immediately before NIC's hardware clock read */ +#define MC_CMD_PTP_OUT_SYNCHRONIZE_HOSTSTART_OFST 0 +/* Value of seconds timestamp */ +#define MC_CMD_PTP_OUT_SYNCHRONIZE_SECONDS_OFST 4 +/* Timestamp major value */ +#define MC_CMD_PTP_OUT_SYNCHRONIZE_MAJOR_OFST 4 +/* Value of nanoseconds timestamp */ +#define MC_CMD_PTP_OUT_SYNCHRONIZE_NANOSECONDS_OFST 8 +/* Timestamp minor value */ +#define MC_CMD_PTP_OUT_SYNCHRONIZE_MINOR_OFST 8 +/* Host time immediately after NIC's hardware clock read */ +#define MC_CMD_PTP_OUT_SYNCHRONIZE_HOSTEND_OFST 12 +/* Number of nanoseconds waited after reading NIC's hardware clock */ +#define MC_CMD_PTP_OUT_SYNCHRONIZE_WAITNS_OFST 16 + +/* MC_CMD_PTP_OUT_MANFTEST_BASIC msgresponse */ +#define MC_CMD_PTP_OUT_MANFTEST_BASIC_LEN 8 +/* Results of testing */ +#define MC_CMD_PTP_OUT_MANFTEST_BASIC_TEST_RESULT_OFST 0 +/* enum: Successful test */ +#define MC_CMD_PTP_MANF_SUCCESS 0x0 +/* enum: FPGA load failed */ +#define MC_CMD_PTP_MANF_FPGA_LOAD 0x1 +/* enum: FPGA version invalid */ +#define MC_CMD_PTP_MANF_FPGA_VERSION 0x2 +/* enum: FPGA registers incorrect */ +#define MC_CMD_PTP_MANF_FPGA_REGISTERS 0x3 +/* enum: Oscillator possibly not working? */ +#define MC_CMD_PTP_MANF_OSCILLATOR 0x4 +/* enum: Timestamps not increasing */ +#define MC_CMD_PTP_MANF_TIMESTAMPS 0x5 +/* enum: Mismatched packet count */ +#define MC_CMD_PTP_MANF_PACKET_COUNT 0x6 +/* enum: Mismatched packet count (Siena filter and FPGA) */ +#define MC_CMD_PTP_MANF_FILTER_COUNT 0x7 +/* enum: Not enough packets to perform timestamp check */ +#define MC_CMD_PTP_MANF_PACKET_ENOUGH 0x8 +/* enum: Timestamp trigger GPIO not working */ +#define MC_CMD_PTP_MANF_GPIO_TRIGGER 0x9 +/* enum: Insufficient PPS events to perform checks */ +#define MC_CMD_PTP_MANF_PPS_ENOUGH 0xa +/* enum: PPS time event period not sufficiently close to 1s. */ +#define MC_CMD_PTP_MANF_PPS_PERIOD 0xb +/* enum: PPS time event nS reading not sufficiently close to zero. */ +#define MC_CMD_PTP_MANF_PPS_NS 0xc +/* enum: PTP peripheral registers incorrect */ +#define MC_CMD_PTP_MANF_REGISTERS 0xd +/* enum: Failed to read time from PTP peripheral */ +#define MC_CMD_PTP_MANF_CLOCK_READ 0xe +/* Presence of external oscillator */ +#define MC_CMD_PTP_OUT_MANFTEST_BASIC_TEST_EXTOSC_OFST 4 + +/* MC_CMD_PTP_OUT_MANFTEST_PACKET msgresponse */ +#define MC_CMD_PTP_OUT_MANFTEST_PACKET_LEN 12 +/* Results of testing */ +#define MC_CMD_PTP_OUT_MANFTEST_PACKET_TEST_RESULT_OFST 0 +/* Number of packets received by FPGA */ +#define MC_CMD_PTP_OUT_MANFTEST_PACKET_TEST_FPGACOUNT_OFST 4 +/* Number of packets received by Siena filters */ +#define MC_CMD_PTP_OUT_MANFTEST_PACKET_TEST_FILTERCOUNT_OFST 8 + +/* MC_CMD_PTP_OUT_FPGAREAD msgresponse */ +#define MC_CMD_PTP_OUT_FPGAREAD_LENMIN 1 +#define MC_CMD_PTP_OUT_FPGAREAD_LENMAX 252 +#define MC_CMD_PTP_OUT_FPGAREAD_LEN(num) (0+1*(num)) +#define MC_CMD_PTP_OUT_FPGAREAD_BUFFER_OFST 0 +#define MC_CMD_PTP_OUT_FPGAREAD_BUFFER_LEN 1 +#define MC_CMD_PTP_OUT_FPGAREAD_BUFFER_MINNUM 1 +#define MC_CMD_PTP_OUT_FPGAREAD_BUFFER_MAXNUM 252 + +/* MC_CMD_PTP_OUT_GET_TIME_FORMAT msgresponse */ +#define MC_CMD_PTP_OUT_GET_TIME_FORMAT_LEN 4 +/* Time format required/used by for this NIC. Applies to all PTP MCDI + * operations that pass times between the host and firmware. If this operation + * is not supported (older firmware) a format of seconds and nanoseconds should + * be assumed. + */ +#define MC_CMD_PTP_OUT_GET_TIME_FORMAT_FORMAT_OFST 0 +/* enum: Times are in seconds and nanoseconds */ +#define MC_CMD_PTP_OUT_GET_TIME_FORMAT_SECONDS_NANOSECONDS 0x0 +/* enum: Major register has units of 16 second per tick, minor 8 ns per tick */ +#define MC_CMD_PTP_OUT_GET_TIME_FORMAT_16SECONDS_8NANOSECONDS 0x1 +/* enum: Major register has units of seconds, minor 2^-27s per tick */ +#define MC_CMD_PTP_OUT_GET_TIME_FORMAT_SECONDS_27FRACTION 0x2 + +/* MC_CMD_PTP_OUT_GET_ATTRIBUTES msgresponse */ +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_LEN 24 +/* Time format required/used by for this NIC. Applies to all PTP MCDI + * operations that pass times between the host and firmware. If this operation + * is not supported (older firmware) a format of seconds and nanoseconds should + * be assumed. + */ +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_TIME_FORMAT_OFST 0 +/* enum: Times are in seconds and nanoseconds */ +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_SECONDS_NANOSECONDS 0x0 +/* enum: Major register has units of 16 second per tick, minor 8 ns per tick */ +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_16SECONDS_8NANOSECONDS 0x1 +/* enum: Major register has units of seconds, minor 2^-27s per tick */ +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_SECONDS_27FRACTION 0x2 +/* Minimum acceptable value for a corrected synchronization timeset. When + * comparing host and NIC clock times, the MC returns a set of samples that + * contain the host start and end time, the MC time when the host start was + * detected and the time the MC waited between reading the time and detecting + * the host end. The corrected sync window is the difference between the host + * end and start times minus the time that the MC waited for host end. + */ +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_SYNC_WINDOW_MIN_OFST 4 +/* Various PTP capabilities */ +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_CAPABILITIES_OFST 8 +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_REPORT_SYNC_STATUS_LBN 0 +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_REPORT_SYNC_STATUS_WIDTH 1 +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_RX_TSTAMP_OOB_LBN 1 +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_RX_TSTAMP_OOB_WIDTH 1 +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_RESERVED0_OFST 12 +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_RESERVED1_OFST 16 +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_RESERVED2_OFST 20 + +/* MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS msgresponse */ +#define MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS_LEN 16 +/* Uncorrected error on PTP transmit timestamps in NIC clock format */ +#define MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS_TRANSMIT_OFST 0 +/* Uncorrected error on PTP receive timestamps in NIC clock format */ +#define MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS_RECEIVE_OFST 4 +/* Uncorrected error on PPS output in NIC clock format */ +#define MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS_PPS_OUT_OFST 8 +/* Uncorrected error on PPS input in NIC clock format */ +#define MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS_PPS_IN_OFST 12 + +/* MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS_V2 msgresponse */ +#define MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS_V2_LEN 24 +/* Uncorrected error on PTP transmit timestamps in NIC clock format */ +#define MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS_V2_PTP_TX_OFST 0 +/* Uncorrected error on PTP receive timestamps in NIC clock format */ +#define MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS_V2_PTP_RX_OFST 4 +/* Uncorrected error on PPS output in NIC clock format */ +#define MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS_V2_PPS_OUT_OFST 8 +/* Uncorrected error on PPS input in NIC clock format */ +#define MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS_V2_PPS_IN_OFST 12 +/* Uncorrected error on non-PTP transmit timestamps in NIC clock format */ +#define MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS_V2_GENERAL_TX_OFST 16 +/* Uncorrected error on non-PTP receive timestamps in NIC clock format */ +#define MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS_V2_GENERAL_RX_OFST 20 + +/* MC_CMD_PTP_OUT_MANFTEST_PPS msgresponse */ +#define MC_CMD_PTP_OUT_MANFTEST_PPS_LEN 4 +/* Results of testing */ +#define MC_CMD_PTP_OUT_MANFTEST_PPS_TEST_RESULT_OFST 0 +/* Enum values, see field(s): */ +/* MC_CMD_PTP_OUT_MANFTEST_BASIC/TEST_RESULT */ + +/* MC_CMD_PTP_OUT_SET_SYNC_STATUS msgresponse */ +#define MC_CMD_PTP_OUT_SET_SYNC_STATUS_LEN 0 + + +/***********************************/ +/* MC_CMD_CSR_READ32 + * Read 32bit words from the indirect memory map. + */ +#define MC_CMD_CSR_READ32 0xc +#undef MC_CMD_0xc_PRIVILEGE_CTG + +#define MC_CMD_0xc_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_CSR_READ32_IN msgrequest */ +#define MC_CMD_CSR_READ32_IN_LEN 12 +/* Address */ +#define MC_CMD_CSR_READ32_IN_ADDR_OFST 0 +#define MC_CMD_CSR_READ32_IN_STEP_OFST 4 +#define MC_CMD_CSR_READ32_IN_NUMWORDS_OFST 8 + +/* MC_CMD_CSR_READ32_OUT msgresponse */ +#define MC_CMD_CSR_READ32_OUT_LENMIN 4 +#define MC_CMD_CSR_READ32_OUT_LENMAX 252 +#define MC_CMD_CSR_READ32_OUT_LEN(num) (0+4*(num)) +/* The last dword is the status, not a value read */ +#define MC_CMD_CSR_READ32_OUT_BUFFER_OFST 0 +#define MC_CMD_CSR_READ32_OUT_BUFFER_LEN 4 +#define MC_CMD_CSR_READ32_OUT_BUFFER_MINNUM 1 +#define MC_CMD_CSR_READ32_OUT_BUFFER_MAXNUM 63 + + +/***********************************/ +/* MC_CMD_CSR_WRITE32 + * Write 32bit dwords to the indirect memory map. + */ +#define MC_CMD_CSR_WRITE32 0xd +#undef MC_CMD_0xd_PRIVILEGE_CTG + +#define MC_CMD_0xd_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_CSR_WRITE32_IN msgrequest */ +#define MC_CMD_CSR_WRITE32_IN_LENMIN 12 +#define MC_CMD_CSR_WRITE32_IN_LENMAX 252 +#define MC_CMD_CSR_WRITE32_IN_LEN(num) (8+4*(num)) +/* Address */ +#define MC_CMD_CSR_WRITE32_IN_ADDR_OFST 0 +#define MC_CMD_CSR_WRITE32_IN_STEP_OFST 4 +#define MC_CMD_CSR_WRITE32_IN_BUFFER_OFST 8 +#define MC_CMD_CSR_WRITE32_IN_BUFFER_LEN 4 +#define MC_CMD_CSR_WRITE32_IN_BUFFER_MINNUM 1 +#define MC_CMD_CSR_WRITE32_IN_BUFFER_MAXNUM 61 + +/* MC_CMD_CSR_WRITE32_OUT msgresponse */ +#define MC_CMD_CSR_WRITE32_OUT_LEN 4 +#define MC_CMD_CSR_WRITE32_OUT_STATUS_OFST 0 + + +/***********************************/ +/* MC_CMD_HP + * These commands are used for HP related features. They are grouped under one + * MCDI command to avoid creating too many MCDI commands. + */ +#define MC_CMD_HP 0x54 +#undef MC_CMD_0x54_PRIVILEGE_CTG + +#define MC_CMD_0x54_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_HP_IN msgrequest */ +#define MC_CMD_HP_IN_LEN 16 +/* HP OCSD sub-command. When address is not NULL, request activation of OCSD at + * the specified address with the specified interval.When address is NULL, + * INTERVAL is interpreted as a command: 0: stop OCSD / 1: Report OCSD current + * state / 2: (debug) Show temperature reported by one of the supported + * sensors. + */ +#define MC_CMD_HP_IN_SUBCMD_OFST 0 +/* enum: OCSD (Option Card Sensor Data) sub-command. */ +#define MC_CMD_HP_IN_OCSD_SUBCMD 0x0 +/* enum: Last known valid HP sub-command. */ +#define MC_CMD_HP_IN_LAST_SUBCMD 0x0 +/* The address to the array of sensor fields. (Or NULL to use a sub-command.) + */ +#define MC_CMD_HP_IN_OCSD_ADDR_OFST 4 +#define MC_CMD_HP_IN_OCSD_ADDR_LEN 8 +#define MC_CMD_HP_IN_OCSD_ADDR_LO_OFST 4 +#define MC_CMD_HP_IN_OCSD_ADDR_HI_OFST 8 +/* The requested update interval, in seconds. (Or the sub-command if ADDR is + * NULL.) + */ +#define MC_CMD_HP_IN_OCSD_INTERVAL_OFST 12 + +/* MC_CMD_HP_OUT msgresponse */ +#define MC_CMD_HP_OUT_LEN 4 +#define MC_CMD_HP_OUT_OCSD_STATUS_OFST 0 +/* enum: OCSD stopped for this card. */ +#define MC_CMD_HP_OUT_OCSD_STOPPED 0x1 +/* enum: OCSD was successfully started with the address provided. */ +#define MC_CMD_HP_OUT_OCSD_STARTED 0x2 +/* enum: OCSD was already started for this card. */ +#define MC_CMD_HP_OUT_OCSD_ALREADY_STARTED 0x3 + + +/***********************************/ +/* MC_CMD_STACKINFO + * Get stack information. + */ +#define MC_CMD_STACKINFO 0xf +#undef MC_CMD_0xf_PRIVILEGE_CTG + +#define MC_CMD_0xf_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_STACKINFO_IN msgrequest */ +#define MC_CMD_STACKINFO_IN_LEN 0 + +/* MC_CMD_STACKINFO_OUT msgresponse */ +#define MC_CMD_STACKINFO_OUT_LENMIN 12 +#define MC_CMD_STACKINFO_OUT_LENMAX 252 +#define MC_CMD_STACKINFO_OUT_LEN(num) (0+12*(num)) +/* (thread ptr, stack size, free space) for each thread in system */ +#define MC_CMD_STACKINFO_OUT_THREAD_INFO_OFST 0 +#define MC_CMD_STACKINFO_OUT_THREAD_INFO_LEN 12 +#define MC_CMD_STACKINFO_OUT_THREAD_INFO_MINNUM 1 +#define MC_CMD_STACKINFO_OUT_THREAD_INFO_MAXNUM 21 + + +/***********************************/ +/* MC_CMD_MDIO_READ + * MDIO register read. + */ +#define MC_CMD_MDIO_READ 0x10 +#undef MC_CMD_0x10_PRIVILEGE_CTG + +#define MC_CMD_0x10_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_MDIO_READ_IN msgrequest */ +#define MC_CMD_MDIO_READ_IN_LEN 16 +/* Bus number; there are two MDIO buses: one for the internal PHY, and one for + * external devices. + */ +#define MC_CMD_MDIO_READ_IN_BUS_OFST 0 +/* enum: Internal. */ +#define MC_CMD_MDIO_BUS_INTERNAL 0x0 +/* enum: External. */ +#define MC_CMD_MDIO_BUS_EXTERNAL 0x1 +/* Port address */ +#define MC_CMD_MDIO_READ_IN_PRTAD_OFST 4 +/* Device Address or clause 22. */ +#define MC_CMD_MDIO_READ_IN_DEVAD_OFST 8 +/* enum: By default all the MCDI MDIO operations perform clause45 mode. If you + * want to use clause22 then set DEVAD = MC_CMD_MDIO_CLAUSE22. + */ +#define MC_CMD_MDIO_CLAUSE22 0x20 +/* Address */ +#define MC_CMD_MDIO_READ_IN_ADDR_OFST 12 + +/* MC_CMD_MDIO_READ_OUT msgresponse */ +#define MC_CMD_MDIO_READ_OUT_LEN 8 +/* Value */ +#define MC_CMD_MDIO_READ_OUT_VALUE_OFST 0 +/* Status the MDIO commands return the raw status bits from the MDIO block. A + * "good" transaction should have the DONE bit set and all other bits clear. + */ +#define MC_CMD_MDIO_READ_OUT_STATUS_OFST 4 +/* enum: Good. */ +#define MC_CMD_MDIO_STATUS_GOOD 0x8 + + +/***********************************/ +/* MC_CMD_MDIO_WRITE + * MDIO register write. + */ +#define MC_CMD_MDIO_WRITE 0x11 +#undef MC_CMD_0x11_PRIVILEGE_CTG + +#define MC_CMD_0x11_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_MDIO_WRITE_IN msgrequest */ +#define MC_CMD_MDIO_WRITE_IN_LEN 20 +/* Bus number; there are two MDIO buses: one for the internal PHY, and one for + * external devices. + */ +#define MC_CMD_MDIO_WRITE_IN_BUS_OFST 0 +/* enum: Internal. */ +/* MC_CMD_MDIO_BUS_INTERNAL 0x0 */ +/* enum: External. */ +/* MC_CMD_MDIO_BUS_EXTERNAL 0x1 */ +/* Port address */ +#define MC_CMD_MDIO_WRITE_IN_PRTAD_OFST 4 +/* Device Address or clause 22. */ +#define MC_CMD_MDIO_WRITE_IN_DEVAD_OFST 8 +/* enum: By default all the MCDI MDIO operations perform clause45 mode. If you + * want to use clause22 then set DEVAD = MC_CMD_MDIO_CLAUSE22. + */ +/* MC_CMD_MDIO_CLAUSE22 0x20 */ +/* Address */ +#define MC_CMD_MDIO_WRITE_IN_ADDR_OFST 12 +/* Value */ +#define MC_CMD_MDIO_WRITE_IN_VALUE_OFST 16 + +/* MC_CMD_MDIO_WRITE_OUT msgresponse */ +#define MC_CMD_MDIO_WRITE_OUT_LEN 4 +/* Status; the MDIO commands return the raw status bits from the MDIO block. A + * "good" transaction should have the DONE bit set and all other bits clear. + */ +#define MC_CMD_MDIO_WRITE_OUT_STATUS_OFST 0 +/* enum: Good. */ +/* MC_CMD_MDIO_STATUS_GOOD 0x8 */ + + +/***********************************/ +/* MC_CMD_DBI_WRITE + * Write DBI register(s). + */ +#define MC_CMD_DBI_WRITE 0x12 +#undef MC_CMD_0x12_PRIVILEGE_CTG + +#define MC_CMD_0x12_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_DBI_WRITE_IN msgrequest */ +#define MC_CMD_DBI_WRITE_IN_LENMIN 12 +#define MC_CMD_DBI_WRITE_IN_LENMAX 252 +#define MC_CMD_DBI_WRITE_IN_LEN(num) (0+12*(num)) +/* Each write op consists of an address (offset 0), byte enable/VF/CS2 (offset + * 32) and value (offset 64). See MC_CMD_DBIWROP_TYPEDEF. + */ +#define MC_CMD_DBI_WRITE_IN_DBIWROP_OFST 0 +#define MC_CMD_DBI_WRITE_IN_DBIWROP_LEN 12 +#define MC_CMD_DBI_WRITE_IN_DBIWROP_MINNUM 1 +#define MC_CMD_DBI_WRITE_IN_DBIWROP_MAXNUM 21 + +/* MC_CMD_DBI_WRITE_OUT msgresponse */ +#define MC_CMD_DBI_WRITE_OUT_LEN 0 + +/* MC_CMD_DBIWROP_TYPEDEF structuredef */ +#define MC_CMD_DBIWROP_TYPEDEF_LEN 12 +#define MC_CMD_DBIWROP_TYPEDEF_ADDRESS_OFST 0 +#define MC_CMD_DBIWROP_TYPEDEF_ADDRESS_LBN 0 +#define MC_CMD_DBIWROP_TYPEDEF_ADDRESS_WIDTH 32 +#define MC_CMD_DBIWROP_TYPEDEF_PARMS_OFST 4 +#define MC_CMD_DBIWROP_TYPEDEF_VF_NUM_LBN 16 +#define MC_CMD_DBIWROP_TYPEDEF_VF_NUM_WIDTH 16 +#define MC_CMD_DBIWROP_TYPEDEF_VF_ACTIVE_LBN 15 +#define MC_CMD_DBIWROP_TYPEDEF_VF_ACTIVE_WIDTH 1 +#define MC_CMD_DBIWROP_TYPEDEF_CS2_LBN 14 +#define MC_CMD_DBIWROP_TYPEDEF_CS2_WIDTH 1 +#define MC_CMD_DBIWROP_TYPEDEF_PARMS_LBN 32 +#define MC_CMD_DBIWROP_TYPEDEF_PARMS_WIDTH 32 +#define MC_CMD_DBIWROP_TYPEDEF_VALUE_OFST 8 +#define MC_CMD_DBIWROP_TYPEDEF_VALUE_LBN 64 +#define MC_CMD_DBIWROP_TYPEDEF_VALUE_WIDTH 32 + + +/***********************************/ +/* MC_CMD_PORT_READ32 + * Read a 32-bit register from the indirect port register map. The port to + * access is implied by the Shared memory channel used. + */ +#define MC_CMD_PORT_READ32 0x14 + +/* MC_CMD_PORT_READ32_IN msgrequest */ +#define MC_CMD_PORT_READ32_IN_LEN 4 +/* Address */ +#define MC_CMD_PORT_READ32_IN_ADDR_OFST 0 + +/* MC_CMD_PORT_READ32_OUT msgresponse */ +#define MC_CMD_PORT_READ32_OUT_LEN 8 +/* Value */ +#define MC_CMD_PORT_READ32_OUT_VALUE_OFST 0 +/* Status */ +#define MC_CMD_PORT_READ32_OUT_STATUS_OFST 4 + + +/***********************************/ +/* MC_CMD_PORT_WRITE32 + * Write a 32-bit register to the indirect port register map. The port to + * access is implied by the Shared memory channel used. + */ +#define MC_CMD_PORT_WRITE32 0x15 + +/* MC_CMD_PORT_WRITE32_IN msgrequest */ +#define MC_CMD_PORT_WRITE32_IN_LEN 8 +/* Address */ +#define MC_CMD_PORT_WRITE32_IN_ADDR_OFST 0 +/* Value */ +#define MC_CMD_PORT_WRITE32_IN_VALUE_OFST 4 + +/* MC_CMD_PORT_WRITE32_OUT msgresponse */ +#define MC_CMD_PORT_WRITE32_OUT_LEN 4 +/* Status */ +#define MC_CMD_PORT_WRITE32_OUT_STATUS_OFST 0 + + +/***********************************/ +/* MC_CMD_PORT_READ128 + * Read a 128-bit register from the indirect port register map. The port to + * access is implied by the Shared memory channel used. + */ +#define MC_CMD_PORT_READ128 0x16 + +/* MC_CMD_PORT_READ128_IN msgrequest */ +#define MC_CMD_PORT_READ128_IN_LEN 4 +/* Address */ +#define MC_CMD_PORT_READ128_IN_ADDR_OFST 0 + +/* MC_CMD_PORT_READ128_OUT msgresponse */ +#define MC_CMD_PORT_READ128_OUT_LEN 20 +/* Value */ +#define MC_CMD_PORT_READ128_OUT_VALUE_OFST 0 +#define MC_CMD_PORT_READ128_OUT_VALUE_LEN 16 +/* Status */ +#define MC_CMD_PORT_READ128_OUT_STATUS_OFST 16 + + +/***********************************/ +/* MC_CMD_PORT_WRITE128 + * Write a 128-bit register to the indirect port register map. The port to + * access is implied by the Shared memory channel used. + */ +#define MC_CMD_PORT_WRITE128 0x17 + +/* MC_CMD_PORT_WRITE128_IN msgrequest */ +#define MC_CMD_PORT_WRITE128_IN_LEN 20 +/* Address */ +#define MC_CMD_PORT_WRITE128_IN_ADDR_OFST 0 +/* Value */ +#define MC_CMD_PORT_WRITE128_IN_VALUE_OFST 4 +#define MC_CMD_PORT_WRITE128_IN_VALUE_LEN 16 + +/* MC_CMD_PORT_WRITE128_OUT msgresponse */ +#define MC_CMD_PORT_WRITE128_OUT_LEN 4 +/* Status */ +#define MC_CMD_PORT_WRITE128_OUT_STATUS_OFST 0 + +/* MC_CMD_CAPABILITIES structuredef */ +#define MC_CMD_CAPABILITIES_LEN 4 +/* Small buf table. */ +#define MC_CMD_CAPABILITIES_SMALL_BUF_TBL_LBN 0 +#define MC_CMD_CAPABILITIES_SMALL_BUF_TBL_WIDTH 1 +/* Turbo mode (for Maranello). */ +#define MC_CMD_CAPABILITIES_TURBO_LBN 1 +#define MC_CMD_CAPABILITIES_TURBO_WIDTH 1 +/* Turbo mode active (for Maranello). */ +#define MC_CMD_CAPABILITIES_TURBO_ACTIVE_LBN 2 +#define MC_CMD_CAPABILITIES_TURBO_ACTIVE_WIDTH 1 +/* PTP offload. */ +#define MC_CMD_CAPABILITIES_PTP_LBN 3 +#define MC_CMD_CAPABILITIES_PTP_WIDTH 1 +/* AOE mode. */ +#define MC_CMD_CAPABILITIES_AOE_LBN 4 +#define MC_CMD_CAPABILITIES_AOE_WIDTH 1 +/* AOE mode active. */ +#define MC_CMD_CAPABILITIES_AOE_ACTIVE_LBN 5 +#define MC_CMD_CAPABILITIES_AOE_ACTIVE_WIDTH 1 +/* AOE mode active. */ +#define MC_CMD_CAPABILITIES_FC_ACTIVE_LBN 6 +#define MC_CMD_CAPABILITIES_FC_ACTIVE_WIDTH 1 +#define MC_CMD_CAPABILITIES_RESERVED_LBN 7 +#define MC_CMD_CAPABILITIES_RESERVED_WIDTH 25 + + +/***********************************/ +/* MC_CMD_GET_BOARD_CFG + * Returns the MC firmware configuration structure. + */ +#define MC_CMD_GET_BOARD_CFG 0x18 +#undef MC_CMD_0x18_PRIVILEGE_CTG + +#define MC_CMD_0x18_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_GET_BOARD_CFG_IN msgrequest */ +#define MC_CMD_GET_BOARD_CFG_IN_LEN 0 + +/* MC_CMD_GET_BOARD_CFG_OUT msgresponse */ +#define MC_CMD_GET_BOARD_CFG_OUT_LENMIN 96 +#define MC_CMD_GET_BOARD_CFG_OUT_LENMAX 136 +#define MC_CMD_GET_BOARD_CFG_OUT_LEN(num) (72+2*(num)) +#define MC_CMD_GET_BOARD_CFG_OUT_BOARD_TYPE_OFST 0 +#define MC_CMD_GET_BOARD_CFG_OUT_BOARD_NAME_OFST 4 +#define MC_CMD_GET_BOARD_CFG_OUT_BOARD_NAME_LEN 32 +/* See MC_CMD_CAPABILITIES */ +#define MC_CMD_GET_BOARD_CFG_OUT_CAPABILITIES_PORT0_OFST 36 +/* See MC_CMD_CAPABILITIES */ +#define MC_CMD_GET_BOARD_CFG_OUT_CAPABILITIES_PORT1_OFST 40 +#define MC_CMD_GET_BOARD_CFG_OUT_MAC_ADDR_BASE_PORT0_OFST 44 +#define MC_CMD_GET_BOARD_CFG_OUT_MAC_ADDR_BASE_PORT0_LEN 6 +#define MC_CMD_GET_BOARD_CFG_OUT_MAC_ADDR_BASE_PORT1_OFST 50 +#define MC_CMD_GET_BOARD_CFG_OUT_MAC_ADDR_BASE_PORT1_LEN 6 +#define MC_CMD_GET_BOARD_CFG_OUT_MAC_COUNT_PORT0_OFST 56 +#define MC_CMD_GET_BOARD_CFG_OUT_MAC_COUNT_PORT1_OFST 60 +#define MC_CMD_GET_BOARD_CFG_OUT_MAC_STRIDE_PORT0_OFST 64 +#define MC_CMD_GET_BOARD_CFG_OUT_MAC_STRIDE_PORT1_OFST 68 +/* This field contains a 16-bit value for each of the types of NVRAM area. The + * values are defined in the firmware/mc/platform/.c file for a specific board + * type, but otherwise have no meaning to the MC; they are used by the driver + * to manage selection of appropriate firmware updates. + */ +#define MC_CMD_GET_BOARD_CFG_OUT_FW_SUBTYPE_LIST_OFST 72 +#define MC_CMD_GET_BOARD_CFG_OUT_FW_SUBTYPE_LIST_LEN 2 +#define MC_CMD_GET_BOARD_CFG_OUT_FW_SUBTYPE_LIST_MINNUM 12 +#define MC_CMD_GET_BOARD_CFG_OUT_FW_SUBTYPE_LIST_MAXNUM 32 + + +/***********************************/ +/* MC_CMD_DBI_READX + * Read DBI register(s) -- extended functionality + */ +#define MC_CMD_DBI_READX 0x19 +#undef MC_CMD_0x19_PRIVILEGE_CTG + +#define MC_CMD_0x19_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_DBI_READX_IN msgrequest */ +#define MC_CMD_DBI_READX_IN_LENMIN 8 +#define MC_CMD_DBI_READX_IN_LENMAX 248 +#define MC_CMD_DBI_READX_IN_LEN(num) (0+8*(num)) +/* Each Read op consists of an address (offset 0), VF/CS2) */ +#define MC_CMD_DBI_READX_IN_DBIRDOP_OFST 0 +#define MC_CMD_DBI_READX_IN_DBIRDOP_LEN 8 +#define MC_CMD_DBI_READX_IN_DBIRDOP_LO_OFST 0 +#define MC_CMD_DBI_READX_IN_DBIRDOP_HI_OFST 4 +#define MC_CMD_DBI_READX_IN_DBIRDOP_MINNUM 1 +#define MC_CMD_DBI_READX_IN_DBIRDOP_MAXNUM 31 + +/* MC_CMD_DBI_READX_OUT msgresponse */ +#define MC_CMD_DBI_READX_OUT_LENMIN 4 +#define MC_CMD_DBI_READX_OUT_LENMAX 252 +#define MC_CMD_DBI_READX_OUT_LEN(num) (0+4*(num)) +/* Value */ +#define MC_CMD_DBI_READX_OUT_VALUE_OFST 0 +#define MC_CMD_DBI_READX_OUT_VALUE_LEN 4 +#define MC_CMD_DBI_READX_OUT_VALUE_MINNUM 1 +#define MC_CMD_DBI_READX_OUT_VALUE_MAXNUM 63 + +/* MC_CMD_DBIRDOP_TYPEDEF structuredef */ +#define MC_CMD_DBIRDOP_TYPEDEF_LEN 8 +#define MC_CMD_DBIRDOP_TYPEDEF_ADDRESS_OFST 0 +#define MC_CMD_DBIRDOP_TYPEDEF_ADDRESS_LBN 0 +#define MC_CMD_DBIRDOP_TYPEDEF_ADDRESS_WIDTH 32 +#define MC_CMD_DBIRDOP_TYPEDEF_PARMS_OFST 4 +#define MC_CMD_DBIRDOP_TYPEDEF_VF_NUM_LBN 16 +#define MC_CMD_DBIRDOP_TYPEDEF_VF_NUM_WIDTH 16 +#define MC_CMD_DBIRDOP_TYPEDEF_VF_ACTIVE_LBN 15 +#define MC_CMD_DBIRDOP_TYPEDEF_VF_ACTIVE_WIDTH 1 +#define MC_CMD_DBIRDOP_TYPEDEF_CS2_LBN 14 +#define MC_CMD_DBIRDOP_TYPEDEF_CS2_WIDTH 1 +#define MC_CMD_DBIRDOP_TYPEDEF_PARMS_LBN 32 +#define MC_CMD_DBIRDOP_TYPEDEF_PARMS_WIDTH 32 + + +/***********************************/ +/* MC_CMD_SET_RAND_SEED + * Set the 16byte seed for the MC pseudo-random generator. + */ +#define MC_CMD_SET_RAND_SEED 0x1a +#undef MC_CMD_0x1a_PRIVILEGE_CTG + +#define MC_CMD_0x1a_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_SET_RAND_SEED_IN msgrequest */ +#define MC_CMD_SET_RAND_SEED_IN_LEN 16 +/* Seed value. */ +#define MC_CMD_SET_RAND_SEED_IN_SEED_OFST 0 +#define MC_CMD_SET_RAND_SEED_IN_SEED_LEN 16 + +/* MC_CMD_SET_RAND_SEED_OUT msgresponse */ +#define MC_CMD_SET_RAND_SEED_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_LTSSM_HIST + * Retrieve the history of the LTSSM, if the build supports it. + */ +#define MC_CMD_LTSSM_HIST 0x1b + +/* MC_CMD_LTSSM_HIST_IN msgrequest */ +#define MC_CMD_LTSSM_HIST_IN_LEN 0 + +/* MC_CMD_LTSSM_HIST_OUT msgresponse */ +#define MC_CMD_LTSSM_HIST_OUT_LENMIN 0 +#define MC_CMD_LTSSM_HIST_OUT_LENMAX 252 +#define MC_CMD_LTSSM_HIST_OUT_LEN(num) (0+4*(num)) +/* variable number of LTSSM values, as bytes. The history is read-to-clear. */ +#define MC_CMD_LTSSM_HIST_OUT_DATA_OFST 0 +#define MC_CMD_LTSSM_HIST_OUT_DATA_LEN 4 +#define MC_CMD_LTSSM_HIST_OUT_DATA_MINNUM 0 +#define MC_CMD_LTSSM_HIST_OUT_DATA_MAXNUM 63 + + +/***********************************/ +/* MC_CMD_DRV_ATTACH + * Inform MCPU that this port is managed on the host (i.e. driver active). For + * Huntington, also request the preferred datapath firmware to use if possible + * (it may not be possible for this request to be fulfilled; the driver must + * issue a subsequent MC_CMD_GET_CAPABILITIES command to determine which + * features are actually available). The FIRMWARE_ID field is ignored by older + * platforms. + */ +#define MC_CMD_DRV_ATTACH 0x1c +#undef MC_CMD_0x1c_PRIVILEGE_CTG + +#define MC_CMD_0x1c_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_DRV_ATTACH_IN msgrequest */ +#define MC_CMD_DRV_ATTACH_IN_LEN 12 +/* new state to set if UPDATE=1 */ +#define MC_CMD_DRV_ATTACH_IN_NEW_STATE_OFST 0 +#define MC_CMD_DRV_ATTACH_LBN 0 +#define MC_CMD_DRV_ATTACH_WIDTH 1 +#define MC_CMD_DRV_PREBOOT_LBN 1 +#define MC_CMD_DRV_PREBOOT_WIDTH 1 +/* 1 to set new state, or 0 to just report the existing state */ +#define MC_CMD_DRV_ATTACH_IN_UPDATE_OFST 4 +/* preferred datapath firmware (for Huntington; ignored for Siena) */ +#define MC_CMD_DRV_ATTACH_IN_FIRMWARE_ID_OFST 8 +/* enum: Prefer to use full featured firmware */ +#define MC_CMD_FW_FULL_FEATURED 0x0 +/* enum: Prefer to use firmware with fewer features but lower latency */ +#define MC_CMD_FW_LOW_LATENCY 0x1 +/* enum: Prefer to use firmware for SolarCapture packed stream mode */ +#define MC_CMD_FW_PACKED_STREAM 0x2 +/* enum: Prefer to use firmware with fewer features and simpler TX event + * batching but higher TX packet rate + */ +#define MC_CMD_FW_HIGH_TX_RATE 0x3 +/* enum: Reserved value */ +#define MC_CMD_FW_PACKED_STREAM_HASH_MODE_1 0x4 +/* enum: Prefer to use firmware with additional "rules engine" filtering + * support + */ +#define MC_CMD_FW_RULES_ENGINE 0x5 +/* enum: Only this option is allowed for non-admin functions */ +#define MC_CMD_FW_DONT_CARE 0xffffffff + +/* MC_CMD_DRV_ATTACH_OUT msgresponse */ +#define MC_CMD_DRV_ATTACH_OUT_LEN 4 +/* previous or existing state, see the bitmask at NEW_STATE */ +#define MC_CMD_DRV_ATTACH_OUT_OLD_STATE_OFST 0 + +/* MC_CMD_DRV_ATTACH_EXT_OUT msgresponse */ +#define MC_CMD_DRV_ATTACH_EXT_OUT_LEN 8 +/* previous or existing state, see the bitmask at NEW_STATE */ +#define MC_CMD_DRV_ATTACH_EXT_OUT_OLD_STATE_OFST 0 +/* Flags associated with this function */ +#define MC_CMD_DRV_ATTACH_EXT_OUT_FUNC_FLAGS_OFST 4 +/* enum: Labels the lowest-numbered function visible to the OS */ +#define MC_CMD_DRV_ATTACH_EXT_OUT_FLAG_PRIMARY 0x0 +/* enum: The function can control the link state of the physical port it is + * bound to. + */ +#define MC_CMD_DRV_ATTACH_EXT_OUT_FLAG_LINKCTRL 0x1 +/* enum: The function can perform privileged operations */ +#define MC_CMD_DRV_ATTACH_EXT_OUT_FLAG_TRUSTED 0x2 +/* enum: The function does not have an active port associated with it. The port + * refers to the Sorrento external FPGA port. + */ +#define MC_CMD_DRV_ATTACH_EXT_OUT_FLAG_NO_ACTIVE_PORT 0x3 + + +/***********************************/ +/* MC_CMD_SHMUART + * Route UART output to circular buffer in shared memory instead. + */ +#define MC_CMD_SHMUART 0x1f + +/* MC_CMD_SHMUART_IN msgrequest */ +#define MC_CMD_SHMUART_IN_LEN 4 +/* ??? */ +#define MC_CMD_SHMUART_IN_FLAG_OFST 0 + +/* MC_CMD_SHMUART_OUT msgresponse */ +#define MC_CMD_SHMUART_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_PORT_RESET + * Generic per-port reset. There is no equivalent for per-board reset. Locks + * required: None; Return code: 0, ETIME. NOTE: This command is deprecated - + * use MC_CMD_ENTITY_RESET instead. + */ +#define MC_CMD_PORT_RESET 0x20 +#undef MC_CMD_0x20_PRIVILEGE_CTG + +#define MC_CMD_0x20_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_PORT_RESET_IN msgrequest */ +#define MC_CMD_PORT_RESET_IN_LEN 0 + +/* MC_CMD_PORT_RESET_OUT msgresponse */ +#define MC_CMD_PORT_RESET_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_ENTITY_RESET + * Generic per-resource reset. There is no equivalent for per-board reset. + * Locks required: None; Return code: 0, ETIME. NOTE: This command is an + * extended version of the deprecated MC_CMD_PORT_RESET with added fields. + */ +#define MC_CMD_ENTITY_RESET 0x20 +/* MC_CMD_0x20_PRIVILEGE_CTG SRIOV_CTG_GENERAL */ + +/* MC_CMD_ENTITY_RESET_IN msgrequest */ +#define MC_CMD_ENTITY_RESET_IN_LEN 4 +/* Optional flags field. Omitting this will perform a "legacy" reset action + * (TBD). + */ +#define MC_CMD_ENTITY_RESET_IN_FLAG_OFST 0 +#define MC_CMD_ENTITY_RESET_IN_FUNCTION_RESOURCE_RESET_LBN 0 +#define MC_CMD_ENTITY_RESET_IN_FUNCTION_RESOURCE_RESET_WIDTH 1 + +/* MC_CMD_ENTITY_RESET_OUT msgresponse */ +#define MC_CMD_ENTITY_RESET_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_PCIE_CREDITS + * Read instantaneous and minimum flow control thresholds. + */ +#define MC_CMD_PCIE_CREDITS 0x21 + +/* MC_CMD_PCIE_CREDITS_IN msgrequest */ +#define MC_CMD_PCIE_CREDITS_IN_LEN 8 +/* poll period. 0 is disabled */ +#define MC_CMD_PCIE_CREDITS_IN_POLL_PERIOD_OFST 0 +/* wipe statistics */ +#define MC_CMD_PCIE_CREDITS_IN_WIPE_OFST 4 + +/* MC_CMD_PCIE_CREDITS_OUT msgresponse */ +#define MC_CMD_PCIE_CREDITS_OUT_LEN 16 +#define MC_CMD_PCIE_CREDITS_OUT_CURRENT_P_HDR_OFST 0 +#define MC_CMD_PCIE_CREDITS_OUT_CURRENT_P_HDR_LEN 2 +#define MC_CMD_PCIE_CREDITS_OUT_CURRENT_P_DATA_OFST 2 +#define MC_CMD_PCIE_CREDITS_OUT_CURRENT_P_DATA_LEN 2 +#define MC_CMD_PCIE_CREDITS_OUT_CURRENT_NP_HDR_OFST 4 +#define MC_CMD_PCIE_CREDITS_OUT_CURRENT_NP_HDR_LEN 2 +#define MC_CMD_PCIE_CREDITS_OUT_CURRENT_NP_DATA_OFST 6 +#define MC_CMD_PCIE_CREDITS_OUT_CURRENT_NP_DATA_LEN 2 +#define MC_CMD_PCIE_CREDITS_OUT_MINIMUM_P_HDR_OFST 8 +#define MC_CMD_PCIE_CREDITS_OUT_MINIMUM_P_HDR_LEN 2 +#define MC_CMD_PCIE_CREDITS_OUT_MINIMUM_P_DATA_OFST 10 +#define MC_CMD_PCIE_CREDITS_OUT_MINIMUM_P_DATA_LEN 2 +#define MC_CMD_PCIE_CREDITS_OUT_MINIMUM_NP_HDR_OFST 12 +#define MC_CMD_PCIE_CREDITS_OUT_MINIMUM_NP_HDR_LEN 2 +#define MC_CMD_PCIE_CREDITS_OUT_MINIMUM_NP_DATA_OFST 14 +#define MC_CMD_PCIE_CREDITS_OUT_MINIMUM_NP_DATA_LEN 2 + + +/***********************************/ +/* MC_CMD_RXD_MONITOR + * Get histogram of RX queue fill level. + */ +#define MC_CMD_RXD_MONITOR 0x22 + +/* MC_CMD_RXD_MONITOR_IN msgrequest */ +#define MC_CMD_RXD_MONITOR_IN_LEN 12 +#define MC_CMD_RXD_MONITOR_IN_QID_OFST 0 +#define MC_CMD_RXD_MONITOR_IN_POLL_PERIOD_OFST 4 +#define MC_CMD_RXD_MONITOR_IN_WIPE_OFST 8 + +/* MC_CMD_RXD_MONITOR_OUT msgresponse */ +#define MC_CMD_RXD_MONITOR_OUT_LEN 80 +#define MC_CMD_RXD_MONITOR_OUT_QID_OFST 0 +#define MC_CMD_RXD_MONITOR_OUT_RING_FILL_OFST 4 +#define MC_CMD_RXD_MONITOR_OUT_CACHE_FILL_OFST 8 +#define MC_CMD_RXD_MONITOR_OUT_RING_LT_1_OFST 12 +#define MC_CMD_RXD_MONITOR_OUT_RING_LT_2_OFST 16 +#define MC_CMD_RXD_MONITOR_OUT_RING_LT_4_OFST 20 +#define MC_CMD_RXD_MONITOR_OUT_RING_LT_8_OFST 24 +#define MC_CMD_RXD_MONITOR_OUT_RING_LT_16_OFST 28 +#define MC_CMD_RXD_MONITOR_OUT_RING_LT_32_OFST 32 +#define MC_CMD_RXD_MONITOR_OUT_RING_LT_64_OFST 36 +#define MC_CMD_RXD_MONITOR_OUT_RING_LT_128_OFST 40 +#define MC_CMD_RXD_MONITOR_OUT_RING_LT_256_OFST 44 +#define MC_CMD_RXD_MONITOR_OUT_RING_GE_256_OFST 48 +#define MC_CMD_RXD_MONITOR_OUT_CACHE_LT_1_OFST 52 +#define MC_CMD_RXD_MONITOR_OUT_CACHE_LT_2_OFST 56 +#define MC_CMD_RXD_MONITOR_OUT_CACHE_LT_4_OFST 60 +#define MC_CMD_RXD_MONITOR_OUT_CACHE_LT_8_OFST 64 +#define MC_CMD_RXD_MONITOR_OUT_CACHE_LT_16_OFST 68 +#define MC_CMD_RXD_MONITOR_OUT_CACHE_LT_32_OFST 72 +#define MC_CMD_RXD_MONITOR_OUT_CACHE_GE_32_OFST 76 + + +/***********************************/ +/* MC_CMD_PUTS + * Copy the given ASCII string out onto UART and/or out of the network port. + */ +#define MC_CMD_PUTS 0x23 +#undef MC_CMD_0x23_PRIVILEGE_CTG + +#define MC_CMD_0x23_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_PUTS_IN msgrequest */ +#define MC_CMD_PUTS_IN_LENMIN 13 +#define MC_CMD_PUTS_IN_LENMAX 252 +#define MC_CMD_PUTS_IN_LEN(num) (12+1*(num)) +#define MC_CMD_PUTS_IN_DEST_OFST 0 +#define MC_CMD_PUTS_IN_UART_LBN 0 +#define MC_CMD_PUTS_IN_UART_WIDTH 1 +#define MC_CMD_PUTS_IN_PORT_LBN 1 +#define MC_CMD_PUTS_IN_PORT_WIDTH 1 +#define MC_CMD_PUTS_IN_DHOST_OFST 4 +#define MC_CMD_PUTS_IN_DHOST_LEN 6 +#define MC_CMD_PUTS_IN_STRING_OFST 12 +#define MC_CMD_PUTS_IN_STRING_LEN 1 +#define MC_CMD_PUTS_IN_STRING_MINNUM 1 +#define MC_CMD_PUTS_IN_STRING_MAXNUM 240 + +/* MC_CMD_PUTS_OUT msgresponse */ +#define MC_CMD_PUTS_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_GET_PHY_CFG + * Report PHY configuration. This guarantees to succeed even if the PHY is in a + * 'zombie' state. Locks required: None + */ +#define MC_CMD_GET_PHY_CFG 0x24 +#undef MC_CMD_0x24_PRIVILEGE_CTG + +#define MC_CMD_0x24_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_GET_PHY_CFG_IN msgrequest */ +#define MC_CMD_GET_PHY_CFG_IN_LEN 0 + +/* MC_CMD_GET_PHY_CFG_OUT msgresponse */ +#define MC_CMD_GET_PHY_CFG_OUT_LEN 72 +/* flags */ +#define MC_CMD_GET_PHY_CFG_OUT_FLAGS_OFST 0 +#define MC_CMD_GET_PHY_CFG_OUT_PRESENT_LBN 0 +#define MC_CMD_GET_PHY_CFG_OUT_PRESENT_WIDTH 1 +#define MC_CMD_GET_PHY_CFG_OUT_BIST_CABLE_SHORT_LBN 1 +#define MC_CMD_GET_PHY_CFG_OUT_BIST_CABLE_SHORT_WIDTH 1 +#define MC_CMD_GET_PHY_CFG_OUT_BIST_CABLE_LONG_LBN 2 +#define MC_CMD_GET_PHY_CFG_OUT_BIST_CABLE_LONG_WIDTH 1 +#define MC_CMD_GET_PHY_CFG_OUT_LOWPOWER_LBN 3 +#define MC_CMD_GET_PHY_CFG_OUT_LOWPOWER_WIDTH 1 +#define MC_CMD_GET_PHY_CFG_OUT_POWEROFF_LBN 4 +#define MC_CMD_GET_PHY_CFG_OUT_POWEROFF_WIDTH 1 +#define MC_CMD_GET_PHY_CFG_OUT_TXDIS_LBN 5 +#define MC_CMD_GET_PHY_CFG_OUT_TXDIS_WIDTH 1 +#define MC_CMD_GET_PHY_CFG_OUT_BIST_LBN 6 +#define MC_CMD_GET_PHY_CFG_OUT_BIST_WIDTH 1 +/* ?? */ +#define MC_CMD_GET_PHY_CFG_OUT_TYPE_OFST 4 +/* Bitmask of supported capabilities */ +#define MC_CMD_GET_PHY_CFG_OUT_SUPPORTED_CAP_OFST 8 +#define MC_CMD_PHY_CAP_10HDX_LBN 1 +#define MC_CMD_PHY_CAP_10HDX_WIDTH 1 +#define MC_CMD_PHY_CAP_10FDX_LBN 2 +#define MC_CMD_PHY_CAP_10FDX_WIDTH 1 +#define MC_CMD_PHY_CAP_100HDX_LBN 3 +#define MC_CMD_PHY_CAP_100HDX_WIDTH 1 +#define MC_CMD_PHY_CAP_100FDX_LBN 4 +#define MC_CMD_PHY_CAP_100FDX_WIDTH 1 +#define MC_CMD_PHY_CAP_1000HDX_LBN 5 +#define MC_CMD_PHY_CAP_1000HDX_WIDTH 1 +#define MC_CMD_PHY_CAP_1000FDX_LBN 6 +#define MC_CMD_PHY_CAP_1000FDX_WIDTH 1 +#define MC_CMD_PHY_CAP_10000FDX_LBN 7 +#define MC_CMD_PHY_CAP_10000FDX_WIDTH 1 +#define MC_CMD_PHY_CAP_PAUSE_LBN 8 +#define MC_CMD_PHY_CAP_PAUSE_WIDTH 1 +#define MC_CMD_PHY_CAP_ASYM_LBN 9 +#define MC_CMD_PHY_CAP_ASYM_WIDTH 1 +#define MC_CMD_PHY_CAP_AN_LBN 10 +#define MC_CMD_PHY_CAP_AN_WIDTH 1 +#define MC_CMD_PHY_CAP_40000FDX_LBN 11 +#define MC_CMD_PHY_CAP_40000FDX_WIDTH 1 +#define MC_CMD_PHY_CAP_DDM_LBN 12 +#define MC_CMD_PHY_CAP_DDM_WIDTH 1 +/* ?? */ +#define MC_CMD_GET_PHY_CFG_OUT_CHANNEL_OFST 12 +/* ?? */ +#define MC_CMD_GET_PHY_CFG_OUT_PRT_OFST 16 +/* ?? */ +#define MC_CMD_GET_PHY_CFG_OUT_STATS_MASK_OFST 20 +/* ?? */ +#define MC_CMD_GET_PHY_CFG_OUT_NAME_OFST 24 +#define MC_CMD_GET_PHY_CFG_OUT_NAME_LEN 20 +/* ?? */ +#define MC_CMD_GET_PHY_CFG_OUT_MEDIA_TYPE_OFST 44 +/* enum: Xaui. */ +#define MC_CMD_MEDIA_XAUI 0x1 +/* enum: CX4. */ +#define MC_CMD_MEDIA_CX4 0x2 +/* enum: KX4. */ +#define MC_CMD_MEDIA_KX4 0x3 +/* enum: XFP Far. */ +#define MC_CMD_MEDIA_XFP 0x4 +/* enum: SFP+. */ +#define MC_CMD_MEDIA_SFP_PLUS 0x5 +/* enum: 10GBaseT. */ +#define MC_CMD_MEDIA_BASE_T 0x6 +/* enum: QSFP+. */ +#define MC_CMD_MEDIA_QSFP_PLUS 0x7 +#define MC_CMD_GET_PHY_CFG_OUT_MMD_MASK_OFST 48 +/* enum: Native clause 22 */ +#define MC_CMD_MMD_CLAUSE22 0x0 +#define MC_CMD_MMD_CLAUSE45_PMAPMD 0x1 /* enum */ +#define MC_CMD_MMD_CLAUSE45_WIS 0x2 /* enum */ +#define MC_CMD_MMD_CLAUSE45_PCS 0x3 /* enum */ +#define MC_CMD_MMD_CLAUSE45_PHYXS 0x4 /* enum */ +#define MC_CMD_MMD_CLAUSE45_DTEXS 0x5 /* enum */ +#define MC_CMD_MMD_CLAUSE45_TC 0x6 /* enum */ +#define MC_CMD_MMD_CLAUSE45_AN 0x7 /* enum */ +/* enum: Clause22 proxied over clause45 by PHY. */ +#define MC_CMD_MMD_CLAUSE45_C22EXT 0x1d +#define MC_CMD_MMD_CLAUSE45_VEND1 0x1e /* enum */ +#define MC_CMD_MMD_CLAUSE45_VEND2 0x1f /* enum */ +#define MC_CMD_GET_PHY_CFG_OUT_REVISION_OFST 52 +#define MC_CMD_GET_PHY_CFG_OUT_REVISION_LEN 20 + + +/***********************************/ +/* MC_CMD_START_BIST + * Start a BIST test on the PHY. Locks required: PHY_LOCK if doing a PHY BIST + * Return code: 0, EINVAL, EACCES (if PHY_LOCK is not held) + */ +#define MC_CMD_START_BIST 0x25 +#undef MC_CMD_0x25_PRIVILEGE_CTG + +#define MC_CMD_0x25_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_START_BIST_IN msgrequest */ +#define MC_CMD_START_BIST_IN_LEN 4 +/* Type of test. */ +#define MC_CMD_START_BIST_IN_TYPE_OFST 0 +/* enum: Run the PHY's short cable BIST. */ +#define MC_CMD_PHY_BIST_CABLE_SHORT 0x1 +/* enum: Run the PHY's long cable BIST. */ +#define MC_CMD_PHY_BIST_CABLE_LONG 0x2 +/* enum: Run BIST on the currently selected BPX Serdes (XAUI or XFI) . */ +#define MC_CMD_BPX_SERDES_BIST 0x3 +/* enum: Run the MC loopback tests. */ +#define MC_CMD_MC_LOOPBACK_BIST 0x4 +/* enum: Run the PHY's standard BIST. */ +#define MC_CMD_PHY_BIST 0x5 +/* enum: Run MC RAM test. */ +#define MC_CMD_MC_MEM_BIST 0x6 +/* enum: Run Port RAM test. */ +#define MC_CMD_PORT_MEM_BIST 0x7 +/* enum: Run register test. */ +#define MC_CMD_REG_BIST 0x8 + +/* MC_CMD_START_BIST_OUT msgresponse */ +#define MC_CMD_START_BIST_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_POLL_BIST + * Poll for BIST completion. Returns a single status code, and optionally some + * PHY specific bist output. The driver should only consume the BIST output + * after validating OUTLEN and MC_CMD_GET_PHY_CFG.TYPE. If a driver can't + * successfully parse the BIST output, it should still respect the pass/Fail in + * OUT.RESULT. Locks required: PHY_LOCK if doing a PHY BIST. Return code: 0, + * EACCES (if PHY_LOCK is not held). + */ +#define MC_CMD_POLL_BIST 0x26 +#undef MC_CMD_0x26_PRIVILEGE_CTG + +#define MC_CMD_0x26_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_POLL_BIST_IN msgrequest */ +#define MC_CMD_POLL_BIST_IN_LEN 0 + +/* MC_CMD_POLL_BIST_OUT msgresponse */ +#define MC_CMD_POLL_BIST_OUT_LEN 8 +/* result */ +#define MC_CMD_POLL_BIST_OUT_RESULT_OFST 0 +/* enum: Running. */ +#define MC_CMD_POLL_BIST_RUNNING 0x1 +/* enum: Passed. */ +#define MC_CMD_POLL_BIST_PASSED 0x2 +/* enum: Failed. */ +#define MC_CMD_POLL_BIST_FAILED 0x3 +/* enum: Timed-out. */ +#define MC_CMD_POLL_BIST_TIMEOUT 0x4 +#define MC_CMD_POLL_BIST_OUT_PRIVATE_OFST 4 + +/* MC_CMD_POLL_BIST_OUT_SFT9001 msgresponse */ +#define MC_CMD_POLL_BIST_OUT_SFT9001_LEN 36 +/* result */ +/* MC_CMD_POLL_BIST_OUT_RESULT_OFST 0 */ +/* Enum values, see field(s): */ +/* MC_CMD_POLL_BIST_OUT/MC_CMD_POLL_BIST_OUT_RESULT */ +#define MC_CMD_POLL_BIST_OUT_SFT9001_CABLE_LENGTH_A_OFST 4 +#define MC_CMD_POLL_BIST_OUT_SFT9001_CABLE_LENGTH_B_OFST 8 +#define MC_CMD_POLL_BIST_OUT_SFT9001_CABLE_LENGTH_C_OFST 12 +#define MC_CMD_POLL_BIST_OUT_SFT9001_CABLE_LENGTH_D_OFST 16 +/* Status of each channel A */ +#define MC_CMD_POLL_BIST_OUT_SFT9001_CABLE_STATUS_A_OFST 20 +/* enum: Ok. */ +#define MC_CMD_POLL_BIST_SFT9001_PAIR_OK 0x1 +/* enum: Open. */ +#define MC_CMD_POLL_BIST_SFT9001_PAIR_OPEN 0x2 +/* enum: Intra-pair short. */ +#define MC_CMD_POLL_BIST_SFT9001_INTRA_PAIR_SHORT 0x3 +/* enum: Inter-pair short. */ +#define MC_CMD_POLL_BIST_SFT9001_INTER_PAIR_SHORT 0x4 +/* enum: Busy. */ +#define MC_CMD_POLL_BIST_SFT9001_PAIR_BUSY 0x9 +/* Status of each channel B */ +#define MC_CMD_POLL_BIST_OUT_SFT9001_CABLE_STATUS_B_OFST 24 +/* Enum values, see field(s): */ +/* CABLE_STATUS_A */ +/* Status of each channel C */ +#define MC_CMD_POLL_BIST_OUT_SFT9001_CABLE_STATUS_C_OFST 28 +/* Enum values, see field(s): */ +/* CABLE_STATUS_A */ +/* Status of each channel D */ +#define MC_CMD_POLL_BIST_OUT_SFT9001_CABLE_STATUS_D_OFST 32 +/* Enum values, see field(s): */ +/* CABLE_STATUS_A */ + +/* MC_CMD_POLL_BIST_OUT_MRSFP msgresponse */ +#define MC_CMD_POLL_BIST_OUT_MRSFP_LEN 8 +/* result */ +/* MC_CMD_POLL_BIST_OUT_RESULT_OFST 0 */ +/* Enum values, see field(s): */ +/* MC_CMD_POLL_BIST_OUT/MC_CMD_POLL_BIST_OUT_RESULT */ +#define MC_CMD_POLL_BIST_OUT_MRSFP_TEST_OFST 4 +/* enum: Complete. */ +#define MC_CMD_POLL_BIST_MRSFP_TEST_COMPLETE 0x0 +/* enum: Bus switch off I2C write. */ +#define MC_CMD_POLL_BIST_MRSFP_TEST_BUS_SWITCH_OFF_I2C_WRITE 0x1 +/* enum: Bus switch off I2C no access IO exp. */ +#define MC_CMD_POLL_BIST_MRSFP_TEST_BUS_SWITCH_OFF_I2C_NO_ACCESS_IO_EXP 0x2 +/* enum: Bus switch off I2C no access module. */ +#define MC_CMD_POLL_BIST_MRSFP_TEST_BUS_SWITCH_OFF_I2C_NO_ACCESS_MODULE 0x3 +/* enum: IO exp I2C configure. */ +#define MC_CMD_POLL_BIST_MRSFP_TEST_IO_EXP_I2C_CONFIGURE 0x4 +/* enum: Bus switch I2C no cross talk. */ +#define MC_CMD_POLL_BIST_MRSFP_TEST_BUS_SWITCH_I2C_NO_CROSSTALK 0x5 +/* enum: Module presence. */ +#define MC_CMD_POLL_BIST_MRSFP_TEST_MODULE_PRESENCE 0x6 +/* enum: Module ID I2C access. */ +#define MC_CMD_POLL_BIST_MRSFP_TEST_MODULE_ID_I2C_ACCESS 0x7 +/* enum: Module ID sane value. */ +#define MC_CMD_POLL_BIST_MRSFP_TEST_MODULE_ID_SANE_VALUE 0x8 + +/* MC_CMD_POLL_BIST_OUT_MEM msgresponse */ +#define MC_CMD_POLL_BIST_OUT_MEM_LEN 36 +/* result */ +/* MC_CMD_POLL_BIST_OUT_RESULT_OFST 0 */ +/* Enum values, see field(s): */ +/* MC_CMD_POLL_BIST_OUT/MC_CMD_POLL_BIST_OUT_RESULT */ +#define MC_CMD_POLL_BIST_OUT_MEM_TEST_OFST 4 +/* enum: Test has completed. */ +#define MC_CMD_POLL_BIST_MEM_COMPLETE 0x0 +/* enum: RAM test - walk ones. */ +#define MC_CMD_POLL_BIST_MEM_MEM_WALK_ONES 0x1 +/* enum: RAM test - walk zeros. */ +#define MC_CMD_POLL_BIST_MEM_MEM_WALK_ZEROS 0x2 +/* enum: RAM test - walking inversions zeros/ones. */ +#define MC_CMD_POLL_BIST_MEM_MEM_INV_ZERO_ONE 0x3 +/* enum: RAM test - walking inversions checkerboard. */ +#define MC_CMD_POLL_BIST_MEM_MEM_INV_CHKBOARD 0x4 +/* enum: Register test - set / clear individual bits. */ +#define MC_CMD_POLL_BIST_MEM_REG 0x5 +/* enum: ECC error detected. */ +#define MC_CMD_POLL_BIST_MEM_ECC 0x6 +/* Failure address, only valid if result is POLL_BIST_FAILED */ +#define MC_CMD_POLL_BIST_OUT_MEM_ADDR_OFST 8 +/* Bus or address space to which the failure address corresponds */ +#define MC_CMD_POLL_BIST_OUT_MEM_BUS_OFST 12 +/* enum: MC MIPS bus. */ +#define MC_CMD_POLL_BIST_MEM_BUS_MC 0x0 +/* enum: CSR IREG bus. */ +#define MC_CMD_POLL_BIST_MEM_BUS_CSR 0x1 +/* enum: RX0 DPCPU bus. */ +#define MC_CMD_POLL_BIST_MEM_BUS_DPCPU_RX 0x2 +/* enum: TX0 DPCPU bus. */ +#define MC_CMD_POLL_BIST_MEM_BUS_DPCPU_TX0 0x3 +/* enum: TX1 DPCPU bus. */ +#define MC_CMD_POLL_BIST_MEM_BUS_DPCPU_TX1 0x4 +/* enum: RX0 DICPU bus. */ +#define MC_CMD_POLL_BIST_MEM_BUS_DICPU_RX 0x5 +/* enum: TX DICPU bus. */ +#define MC_CMD_POLL_BIST_MEM_BUS_DICPU_TX 0x6 +/* enum: RX1 DPCPU bus. */ +#define MC_CMD_POLL_BIST_MEM_BUS_DPCPU_RX1 0x7 +/* enum: RX1 DICPU bus. */ +#define MC_CMD_POLL_BIST_MEM_BUS_DICPU_RX1 0x8 +/* Pattern written to RAM / register */ +#define MC_CMD_POLL_BIST_OUT_MEM_EXPECT_OFST 16 +/* Actual value read from RAM / register */ +#define MC_CMD_POLL_BIST_OUT_MEM_ACTUAL_OFST 20 +/* ECC error mask */ +#define MC_CMD_POLL_BIST_OUT_MEM_ECC_OFST 24 +/* ECC parity error mask */ +#define MC_CMD_POLL_BIST_OUT_MEM_ECC_PARITY_OFST 28 +/* ECC fatal error mask */ +#define MC_CMD_POLL_BIST_OUT_MEM_ECC_FATAL_OFST 32 + + +/***********************************/ +/* MC_CMD_FLUSH_RX_QUEUES + * Flush receive queue(s). If SRIOV is enabled (via MC_CMD_SRIOV), then RXQ + * flushes should be initiated via this MCDI operation, rather than via + * directly writing FLUSH_CMD. + * + * The flush is completed (either done/fail) asynchronously (after this command + * returns). The driver must still wait for flush done/failure events as usual. + */ +#define MC_CMD_FLUSH_RX_QUEUES 0x27 + +/* MC_CMD_FLUSH_RX_QUEUES_IN msgrequest */ +#define MC_CMD_FLUSH_RX_QUEUES_IN_LENMIN 4 +#define MC_CMD_FLUSH_RX_QUEUES_IN_LENMAX 252 +#define MC_CMD_FLUSH_RX_QUEUES_IN_LEN(num) (0+4*(num)) +#define MC_CMD_FLUSH_RX_QUEUES_IN_QID_OFST_OFST 0 +#define MC_CMD_FLUSH_RX_QUEUES_IN_QID_OFST_LEN 4 +#define MC_CMD_FLUSH_RX_QUEUES_IN_QID_OFST_MINNUM 1 +#define MC_CMD_FLUSH_RX_QUEUES_IN_QID_OFST_MAXNUM 63 + +/* MC_CMD_FLUSH_RX_QUEUES_OUT msgresponse */ +#define MC_CMD_FLUSH_RX_QUEUES_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_GET_LOOPBACK_MODES + * Returns a bitmask of loopback modes available at each speed. + */ +#define MC_CMD_GET_LOOPBACK_MODES 0x28 +#undef MC_CMD_0x28_PRIVILEGE_CTG + +#define MC_CMD_0x28_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_GET_LOOPBACK_MODES_IN msgrequest */ +#define MC_CMD_GET_LOOPBACK_MODES_IN_LEN 0 + +/* MC_CMD_GET_LOOPBACK_MODES_OUT msgresponse */ +#define MC_CMD_GET_LOOPBACK_MODES_OUT_LEN 40 +/* Supported loopbacks. */ +#define MC_CMD_GET_LOOPBACK_MODES_OUT_100M_OFST 0 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_100M_LEN 8 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_100M_LO_OFST 0 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_100M_HI_OFST 4 +/* enum: None. */ +#define MC_CMD_LOOPBACK_NONE 0x0 +/* enum: Data. */ +#define MC_CMD_LOOPBACK_DATA 0x1 +/* enum: GMAC. */ +#define MC_CMD_LOOPBACK_GMAC 0x2 +/* enum: XGMII. */ +#define MC_CMD_LOOPBACK_XGMII 0x3 +/* enum: XGXS. */ +#define MC_CMD_LOOPBACK_XGXS 0x4 +/* enum: XAUI. */ +#define MC_CMD_LOOPBACK_XAUI 0x5 +/* enum: GMII. */ +#define MC_CMD_LOOPBACK_GMII 0x6 +/* enum: SGMII. */ +#define MC_CMD_LOOPBACK_SGMII 0x7 +/* enum: XGBR. */ +#define MC_CMD_LOOPBACK_XGBR 0x8 +/* enum: XFI. */ +#define MC_CMD_LOOPBACK_XFI 0x9 +/* enum: XAUI Far. */ +#define MC_CMD_LOOPBACK_XAUI_FAR 0xa +/* enum: GMII Far. */ +#define MC_CMD_LOOPBACK_GMII_FAR 0xb +/* enum: SGMII Far. */ +#define MC_CMD_LOOPBACK_SGMII_FAR 0xc +/* enum: XFI Far. */ +#define MC_CMD_LOOPBACK_XFI_FAR 0xd +/* enum: GPhy. */ +#define MC_CMD_LOOPBACK_GPHY 0xe +/* enum: PhyXS. */ +#define MC_CMD_LOOPBACK_PHYXS 0xf +/* enum: PCS. */ +#define MC_CMD_LOOPBACK_PCS 0x10 +/* enum: PMA-PMD. */ +#define MC_CMD_LOOPBACK_PMAPMD 0x11 +/* enum: Cross-Port. */ +#define MC_CMD_LOOPBACK_XPORT 0x12 +/* enum: XGMII-Wireside. */ +#define MC_CMD_LOOPBACK_XGMII_WS 0x13 +/* enum: XAUI Wireside. */ +#define MC_CMD_LOOPBACK_XAUI_WS 0x14 +/* enum: XAUI Wireside Far. */ +#define MC_CMD_LOOPBACK_XAUI_WS_FAR 0x15 +/* enum: XAUI Wireside near. */ +#define MC_CMD_LOOPBACK_XAUI_WS_NEAR 0x16 +/* enum: GMII Wireside. */ +#define MC_CMD_LOOPBACK_GMII_WS 0x17 +/* enum: XFI Wireside. */ +#define MC_CMD_LOOPBACK_XFI_WS 0x18 +/* enum: XFI Wireside Far. */ +#define MC_CMD_LOOPBACK_XFI_WS_FAR 0x19 +/* enum: PhyXS Wireside. */ +#define MC_CMD_LOOPBACK_PHYXS_WS 0x1a +/* enum: PMA lanes MAC-Serdes. */ +#define MC_CMD_LOOPBACK_PMA_INT 0x1b +/* enum: KR Serdes Parallel (Encoder). */ +#define MC_CMD_LOOPBACK_SD_NEAR 0x1c +/* enum: KR Serdes Serial. */ +#define MC_CMD_LOOPBACK_SD_FAR 0x1d +/* enum: PMA lanes MAC-Serdes Wireside. */ +#define MC_CMD_LOOPBACK_PMA_INT_WS 0x1e +/* enum: KR Serdes Parallel Wireside (Full PCS). */ +#define MC_CMD_LOOPBACK_SD_FEP2_WS 0x1f +/* enum: KR Serdes Parallel Wireside (Sym Aligner to TX). */ +#define MC_CMD_LOOPBACK_SD_FEP1_5_WS 0x20 +/* enum: KR Serdes Parallel Wireside (Deserializer to Serializer). */ +#define MC_CMD_LOOPBACK_SD_FEP_WS 0x21 +/* enum: KR Serdes Serial Wireside. */ +#define MC_CMD_LOOPBACK_SD_FES_WS 0x22 +/* enum: Near side of AOE Siena side port */ +#define MC_CMD_LOOPBACK_AOE_INT_NEAR 0x23 +/* enum: Medford Wireside datapath loopback */ +#define MC_CMD_LOOPBACK_DATA_WS 0x24 +/* enum: Force link up without setting up any physical loopback (snapper use + * only) + */ +#define MC_CMD_LOOPBACK_FORCE_EXT_LINK 0x25 +/* Supported loopbacks. */ +#define MC_CMD_GET_LOOPBACK_MODES_OUT_1G_OFST 8 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_1G_LEN 8 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_1G_LO_OFST 8 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_1G_HI_OFST 12 +/* Enum values, see field(s): */ +/* 100M */ +/* Supported loopbacks. */ +#define MC_CMD_GET_LOOPBACK_MODES_OUT_10G_OFST 16 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_10G_LEN 8 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_10G_LO_OFST 16 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_10G_HI_OFST 20 +/* Enum values, see field(s): */ +/* 100M */ +/* Supported loopbacks. */ +#define MC_CMD_GET_LOOPBACK_MODES_OUT_SUGGESTED_OFST 24 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_SUGGESTED_LEN 8 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_SUGGESTED_LO_OFST 24 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_SUGGESTED_HI_OFST 28 +/* Enum values, see field(s): */ +/* 100M */ +/* Supported loopbacks. */ +#define MC_CMD_GET_LOOPBACK_MODES_OUT_40G_OFST 32 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_40G_LEN 8 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_40G_LO_OFST 32 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_40G_HI_OFST 36 +/* Enum values, see field(s): */ +/* 100M */ + + +/***********************************/ +/* MC_CMD_GET_LINK + * Read the unified MAC/PHY link state. Locks required: None Return code: 0, + * ETIME. + */ +#define MC_CMD_GET_LINK 0x29 +#undef MC_CMD_0x29_PRIVILEGE_CTG + +#define MC_CMD_0x29_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_GET_LINK_IN msgrequest */ +#define MC_CMD_GET_LINK_IN_LEN 0 + +/* MC_CMD_GET_LINK_OUT msgresponse */ +#define MC_CMD_GET_LINK_OUT_LEN 28 +/* near-side advertised capabilities */ +#define MC_CMD_GET_LINK_OUT_CAP_OFST 0 +/* link-partner advertised capabilities */ +#define MC_CMD_GET_LINK_OUT_LP_CAP_OFST 4 +/* Autonegotiated speed in mbit/s. The link may still be down even if this + * reads non-zero. + */ +#define MC_CMD_GET_LINK_OUT_LINK_SPEED_OFST 8 +/* Current loopback setting. */ +#define MC_CMD_GET_LINK_OUT_LOOPBACK_MODE_OFST 12 +/* Enum values, see field(s): */ +/* MC_CMD_GET_LOOPBACK_MODES/MC_CMD_GET_LOOPBACK_MODES_OUT/100M */ +#define MC_CMD_GET_LINK_OUT_FLAGS_OFST 16 +#define MC_CMD_GET_LINK_OUT_LINK_UP_LBN 0 +#define MC_CMD_GET_LINK_OUT_LINK_UP_WIDTH 1 +#define MC_CMD_GET_LINK_OUT_FULL_DUPLEX_LBN 1 +#define MC_CMD_GET_LINK_OUT_FULL_DUPLEX_WIDTH 1 +#define MC_CMD_GET_LINK_OUT_BPX_LINK_LBN 2 +#define MC_CMD_GET_LINK_OUT_BPX_LINK_WIDTH 1 +#define MC_CMD_GET_LINK_OUT_PHY_LINK_LBN 3 +#define MC_CMD_GET_LINK_OUT_PHY_LINK_WIDTH 1 +#define MC_CMD_GET_LINK_OUT_LINK_FAULT_RX_LBN 6 +#define MC_CMD_GET_LINK_OUT_LINK_FAULT_RX_WIDTH 1 +#define MC_CMD_GET_LINK_OUT_LINK_FAULT_TX_LBN 7 +#define MC_CMD_GET_LINK_OUT_LINK_FAULT_TX_WIDTH 1 +/* This returns the negotiated flow control value. */ +#define MC_CMD_GET_LINK_OUT_FCNTL_OFST 20 +/* Enum values, see field(s): */ +/* MC_CMD_SET_MAC/MC_CMD_SET_MAC_IN/FCNTL */ +#define MC_CMD_GET_LINK_OUT_MAC_FAULT_OFST 24 +#define MC_CMD_MAC_FAULT_XGMII_LOCAL_LBN 0 +#define MC_CMD_MAC_FAULT_XGMII_LOCAL_WIDTH 1 +#define MC_CMD_MAC_FAULT_XGMII_REMOTE_LBN 1 +#define MC_CMD_MAC_FAULT_XGMII_REMOTE_WIDTH 1 +#define MC_CMD_MAC_FAULT_SGMII_REMOTE_LBN 2 +#define MC_CMD_MAC_FAULT_SGMII_REMOTE_WIDTH 1 +#define MC_CMD_MAC_FAULT_PENDING_RECONFIG_LBN 3 +#define MC_CMD_MAC_FAULT_PENDING_RECONFIG_WIDTH 1 + + +/***********************************/ +/* MC_CMD_SET_LINK + * Write the unified MAC/PHY link configuration. Locks required: None. Return + * code: 0, EINVAL, ETIME + */ +#define MC_CMD_SET_LINK 0x2a +#undef MC_CMD_0x2a_PRIVILEGE_CTG + +#define MC_CMD_0x2a_PRIVILEGE_CTG SRIOV_CTG_LINK + +/* MC_CMD_SET_LINK_IN msgrequest */ +#define MC_CMD_SET_LINK_IN_LEN 16 +/* ??? */ +#define MC_CMD_SET_LINK_IN_CAP_OFST 0 +/* Flags */ +#define MC_CMD_SET_LINK_IN_FLAGS_OFST 4 +#define MC_CMD_SET_LINK_IN_LOWPOWER_LBN 0 +#define MC_CMD_SET_LINK_IN_LOWPOWER_WIDTH 1 +#define MC_CMD_SET_LINK_IN_POWEROFF_LBN 1 +#define MC_CMD_SET_LINK_IN_POWEROFF_WIDTH 1 +#define MC_CMD_SET_LINK_IN_TXDIS_LBN 2 +#define MC_CMD_SET_LINK_IN_TXDIS_WIDTH 1 +/* Loopback mode. */ +#define MC_CMD_SET_LINK_IN_LOOPBACK_MODE_OFST 8 +/* Enum values, see field(s): */ +/* MC_CMD_GET_LOOPBACK_MODES/MC_CMD_GET_LOOPBACK_MODES_OUT/100M */ +/* A loopback speed of "0" is supported, and means (choose any available + * speed). + */ +#define MC_CMD_SET_LINK_IN_LOOPBACK_SPEED_OFST 12 + +/* MC_CMD_SET_LINK_OUT msgresponse */ +#define MC_CMD_SET_LINK_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_SET_ID_LED + * Set identification LED state. Locks required: None. Return code: 0, EINVAL + */ +#define MC_CMD_SET_ID_LED 0x2b +#undef MC_CMD_0x2b_PRIVILEGE_CTG + +#define MC_CMD_0x2b_PRIVILEGE_CTG SRIOV_CTG_LINK + +/* MC_CMD_SET_ID_LED_IN msgrequest */ +#define MC_CMD_SET_ID_LED_IN_LEN 4 +/* Set LED state. */ +#define MC_CMD_SET_ID_LED_IN_STATE_OFST 0 +#define MC_CMD_LED_OFF 0x0 /* enum */ +#define MC_CMD_LED_ON 0x1 /* enum */ +#define MC_CMD_LED_DEFAULT 0x2 /* enum */ + +/* MC_CMD_SET_ID_LED_OUT msgresponse */ +#define MC_CMD_SET_ID_LED_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_SET_MAC + * Set MAC configuration. Locks required: None. Return code: 0, EINVAL + */ +#define MC_CMD_SET_MAC 0x2c +#undef MC_CMD_0x2c_PRIVILEGE_CTG + +#define MC_CMD_0x2c_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_SET_MAC_IN msgrequest */ +#define MC_CMD_SET_MAC_IN_LEN 28 +/* The MTU is the MTU programmed directly into the XMAC/GMAC (inclusive of + * EtherII, VLAN, bug16011 padding). + */ +#define MC_CMD_SET_MAC_IN_MTU_OFST 0 +#define MC_CMD_SET_MAC_IN_DRAIN_OFST 4 +#define MC_CMD_SET_MAC_IN_ADDR_OFST 8 +#define MC_CMD_SET_MAC_IN_ADDR_LEN 8 +#define MC_CMD_SET_MAC_IN_ADDR_LO_OFST 8 +#define MC_CMD_SET_MAC_IN_ADDR_HI_OFST 12 +#define MC_CMD_SET_MAC_IN_REJECT_OFST 16 +#define MC_CMD_SET_MAC_IN_REJECT_UNCST_LBN 0 +#define MC_CMD_SET_MAC_IN_REJECT_UNCST_WIDTH 1 +#define MC_CMD_SET_MAC_IN_REJECT_BRDCST_LBN 1 +#define MC_CMD_SET_MAC_IN_REJECT_BRDCST_WIDTH 1 +#define MC_CMD_SET_MAC_IN_FCNTL_OFST 20 +/* enum: Flow control is off. */ +#define MC_CMD_FCNTL_OFF 0x0 +/* enum: Respond to flow control. */ +#define MC_CMD_FCNTL_RESPOND 0x1 +/* enum: Respond to and Issue flow control. */ +#define MC_CMD_FCNTL_BIDIR 0x2 +/* enum: Auto neg flow control. */ +#define MC_CMD_FCNTL_AUTO 0x3 +/* enum: Priority flow control (eftest builds only). */ +#define MC_CMD_FCNTL_QBB 0x4 +/* enum: Issue flow control. */ +#define MC_CMD_FCNTL_GENERATE 0x5 +#define MC_CMD_SET_MAC_IN_FLAGS_OFST 24 +#define MC_CMD_SET_MAC_IN_FLAG_INCLUDE_FCS_LBN 0 +#define MC_CMD_SET_MAC_IN_FLAG_INCLUDE_FCS_WIDTH 1 + +/* MC_CMD_SET_MAC_EXT_IN msgrequest */ +#define MC_CMD_SET_MAC_EXT_IN_LEN 32 +/* The MTU is the MTU programmed directly into the XMAC/GMAC (inclusive of + * EtherII, VLAN, bug16011 padding). + */ +#define MC_CMD_SET_MAC_EXT_IN_MTU_OFST 0 +#define MC_CMD_SET_MAC_EXT_IN_DRAIN_OFST 4 +#define MC_CMD_SET_MAC_EXT_IN_ADDR_OFST 8 +#define MC_CMD_SET_MAC_EXT_IN_ADDR_LEN 8 +#define MC_CMD_SET_MAC_EXT_IN_ADDR_LO_OFST 8 +#define MC_CMD_SET_MAC_EXT_IN_ADDR_HI_OFST 12 +#define MC_CMD_SET_MAC_EXT_IN_REJECT_OFST 16 +#define MC_CMD_SET_MAC_EXT_IN_REJECT_UNCST_LBN 0 +#define MC_CMD_SET_MAC_EXT_IN_REJECT_UNCST_WIDTH 1 +#define MC_CMD_SET_MAC_EXT_IN_REJECT_BRDCST_LBN 1 +#define MC_CMD_SET_MAC_EXT_IN_REJECT_BRDCST_WIDTH 1 +#define MC_CMD_SET_MAC_EXT_IN_FCNTL_OFST 20 +/* enum: Flow control is off. */ +/* MC_CMD_FCNTL_OFF 0x0 */ +/* enum: Respond to flow control. */ +/* MC_CMD_FCNTL_RESPOND 0x1 */ +/* enum: Respond to and Issue flow control. */ +/* MC_CMD_FCNTL_BIDIR 0x2 */ +/* enum: Auto neg flow control. */ +/* MC_CMD_FCNTL_AUTO 0x3 */ +/* enum: Priority flow control (eftest builds only). */ +/* MC_CMD_FCNTL_QBB 0x4 */ +/* enum: Issue flow control. */ +/* MC_CMD_FCNTL_GENERATE 0x5 */ +#define MC_CMD_SET_MAC_EXT_IN_FLAGS_OFST 24 +#define MC_CMD_SET_MAC_EXT_IN_FLAG_INCLUDE_FCS_LBN 0 +#define MC_CMD_SET_MAC_EXT_IN_FLAG_INCLUDE_FCS_WIDTH 1 +/* Select which parameters to configure. A parameter will only be modified if + * the corresponding control flag is set. If SET_MAC_ENHANCED is not set in + * capabilities then this field is ignored (and all flags are assumed to be + * set). + */ +#define MC_CMD_SET_MAC_EXT_IN_CONTROL_OFST 28 +#define MC_CMD_SET_MAC_EXT_IN_CFG_MTU_LBN 0 +#define MC_CMD_SET_MAC_EXT_IN_CFG_MTU_WIDTH 1 +#define MC_CMD_SET_MAC_EXT_IN_CFG_DRAIN_LBN 1 +#define MC_CMD_SET_MAC_EXT_IN_CFG_DRAIN_WIDTH 1 +#define MC_CMD_SET_MAC_EXT_IN_CFG_REJECT_LBN 2 +#define MC_CMD_SET_MAC_EXT_IN_CFG_REJECT_WIDTH 1 +#define MC_CMD_SET_MAC_EXT_IN_CFG_FCNTL_LBN 3 +#define MC_CMD_SET_MAC_EXT_IN_CFG_FCNTL_WIDTH 1 +#define MC_CMD_SET_MAC_EXT_IN_CFG_FCS_LBN 4 +#define MC_CMD_SET_MAC_EXT_IN_CFG_FCS_WIDTH 1 + +/* MC_CMD_SET_MAC_OUT msgresponse */ +#define MC_CMD_SET_MAC_OUT_LEN 0 + +/* MC_CMD_SET_MAC_V2_OUT msgresponse */ +#define MC_CMD_SET_MAC_V2_OUT_LEN 4 +/* MTU as configured after processing the request. See comment at + * MC_CMD_SET_MAC_IN/MTU. To query MTU without doing any changes, set CONTROL + * to 0. + */ +#define MC_CMD_SET_MAC_V2_OUT_MTU_OFST 0 + + +/***********************************/ +/* MC_CMD_PHY_STATS + * Get generic PHY statistics. This call returns the statistics for a generic + * PHY in a sparse array (indexed by the enumerate). Each value is represented + * by a 32bit number. If the DMA_ADDR is 0, then no DMA is performed, and the + * statistics may be read from the message response. If DMA_ADDR != 0, then the + * statistics are dmad to that (page-aligned location). Locks required: None. + * Returns: 0, ETIME + */ +#define MC_CMD_PHY_STATS 0x2d +#undef MC_CMD_0x2d_PRIVILEGE_CTG + +#define MC_CMD_0x2d_PRIVILEGE_CTG SRIOV_CTG_LINK + +/* MC_CMD_PHY_STATS_IN msgrequest */ +#define MC_CMD_PHY_STATS_IN_LEN 8 +/* ??? */ +#define MC_CMD_PHY_STATS_IN_DMA_ADDR_OFST 0 +#define MC_CMD_PHY_STATS_IN_DMA_ADDR_LEN 8 +#define MC_CMD_PHY_STATS_IN_DMA_ADDR_LO_OFST 0 +#define MC_CMD_PHY_STATS_IN_DMA_ADDR_HI_OFST 4 + +/* MC_CMD_PHY_STATS_OUT_DMA msgresponse */ +#define MC_CMD_PHY_STATS_OUT_DMA_LEN 0 + +/* MC_CMD_PHY_STATS_OUT_NO_DMA msgresponse */ +#define MC_CMD_PHY_STATS_OUT_NO_DMA_LEN (((MC_CMD_PHY_NSTATS*32))>>3) +#define MC_CMD_PHY_STATS_OUT_NO_DMA_STATISTICS_OFST 0 +#define MC_CMD_PHY_STATS_OUT_NO_DMA_STATISTICS_LEN 4 +#define MC_CMD_PHY_STATS_OUT_NO_DMA_STATISTICS_NUM MC_CMD_PHY_NSTATS +/* enum: OUI. */ +#define MC_CMD_OUI 0x0 +/* enum: PMA-PMD Link Up. */ +#define MC_CMD_PMA_PMD_LINK_UP 0x1 +/* enum: PMA-PMD RX Fault. */ +#define MC_CMD_PMA_PMD_RX_FAULT 0x2 +/* enum: PMA-PMD TX Fault. */ +#define MC_CMD_PMA_PMD_TX_FAULT 0x3 +/* enum: PMA-PMD Signal */ +#define MC_CMD_PMA_PMD_SIGNAL 0x4 +/* enum: PMA-PMD SNR A. */ +#define MC_CMD_PMA_PMD_SNR_A 0x5 +/* enum: PMA-PMD SNR B. */ +#define MC_CMD_PMA_PMD_SNR_B 0x6 +/* enum: PMA-PMD SNR C. */ +#define MC_CMD_PMA_PMD_SNR_C 0x7 +/* enum: PMA-PMD SNR D. */ +#define MC_CMD_PMA_PMD_SNR_D 0x8 +/* enum: PCS Link Up. */ +#define MC_CMD_PCS_LINK_UP 0x9 +/* enum: PCS RX Fault. */ +#define MC_CMD_PCS_RX_FAULT 0xa +/* enum: PCS TX Fault. */ +#define MC_CMD_PCS_TX_FAULT 0xb +/* enum: PCS BER. */ +#define MC_CMD_PCS_BER 0xc +/* enum: PCS Block Errors. */ +#define MC_CMD_PCS_BLOCK_ERRORS 0xd +/* enum: PhyXS Link Up. */ +#define MC_CMD_PHYXS_LINK_UP 0xe +/* enum: PhyXS RX Fault. */ +#define MC_CMD_PHYXS_RX_FAULT 0xf +/* enum: PhyXS TX Fault. */ +#define MC_CMD_PHYXS_TX_FAULT 0x10 +/* enum: PhyXS Align. */ +#define MC_CMD_PHYXS_ALIGN 0x11 +/* enum: PhyXS Sync. */ +#define MC_CMD_PHYXS_SYNC 0x12 +/* enum: AN link-up. */ +#define MC_CMD_AN_LINK_UP 0x13 +/* enum: AN Complete. */ +#define MC_CMD_AN_COMPLETE 0x14 +/* enum: AN 10GBaseT Status. */ +#define MC_CMD_AN_10GBT_STATUS 0x15 +/* enum: Clause 22 Link-Up. */ +#define MC_CMD_CL22_LINK_UP 0x16 +/* enum: (Last entry) */ +#define MC_CMD_PHY_NSTATS 0x17 + + +/***********************************/ +/* MC_CMD_MAC_STATS + * Get generic MAC statistics. This call returns unified statistics maintained + * by the MC as it switches between the GMAC and XMAC. The MC will write out + * all supported stats. The driver should zero initialise the buffer to + * guarantee consistent results. If the DMA_ADDR is 0, then no DMA is + * performed, and the statistics may be read from the message response. If + * DMA_ADDR != 0, then the statistics are dmad to that (page-aligned location). + * Locks required: None. The PERIODIC_CLEAR option is not used and now has no + * effect. Returns: 0, ETIME + */ +#define MC_CMD_MAC_STATS 0x2e +#undef MC_CMD_0x2e_PRIVILEGE_CTG + +#define MC_CMD_0x2e_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_MAC_STATS_IN msgrequest */ +#define MC_CMD_MAC_STATS_IN_LEN 20 +/* ??? */ +#define MC_CMD_MAC_STATS_IN_DMA_ADDR_OFST 0 +#define MC_CMD_MAC_STATS_IN_DMA_ADDR_LEN 8 +#define MC_CMD_MAC_STATS_IN_DMA_ADDR_LO_OFST 0 +#define MC_CMD_MAC_STATS_IN_DMA_ADDR_HI_OFST 4 +#define MC_CMD_MAC_STATS_IN_CMD_OFST 8 +#define MC_CMD_MAC_STATS_IN_DMA_LBN 0 +#define MC_CMD_MAC_STATS_IN_DMA_WIDTH 1 +#define MC_CMD_MAC_STATS_IN_CLEAR_LBN 1 +#define MC_CMD_MAC_STATS_IN_CLEAR_WIDTH 1 +#define MC_CMD_MAC_STATS_IN_PERIODIC_CHANGE_LBN 2 +#define MC_CMD_MAC_STATS_IN_PERIODIC_CHANGE_WIDTH 1 +#define MC_CMD_MAC_STATS_IN_PERIODIC_ENABLE_LBN 3 +#define MC_CMD_MAC_STATS_IN_PERIODIC_ENABLE_WIDTH 1 +#define MC_CMD_MAC_STATS_IN_PERIODIC_CLEAR_LBN 4 +#define MC_CMD_MAC_STATS_IN_PERIODIC_CLEAR_WIDTH 1 +#define MC_CMD_MAC_STATS_IN_PERIODIC_NOEVENT_LBN 5 +#define MC_CMD_MAC_STATS_IN_PERIODIC_NOEVENT_WIDTH 1 +#define MC_CMD_MAC_STATS_IN_PERIOD_MS_LBN 16 +#define MC_CMD_MAC_STATS_IN_PERIOD_MS_WIDTH 16 +#define MC_CMD_MAC_STATS_IN_DMA_LEN_OFST 12 +/* port id so vadapter stats can be provided */ +#define MC_CMD_MAC_STATS_IN_PORT_ID_OFST 16 + +/* MC_CMD_MAC_STATS_OUT_DMA msgresponse */ +#define MC_CMD_MAC_STATS_OUT_DMA_LEN 0 + +/* MC_CMD_MAC_STATS_OUT_NO_DMA msgresponse */ +#define MC_CMD_MAC_STATS_OUT_NO_DMA_LEN (((MC_CMD_MAC_NSTATS*64))>>3) +#define MC_CMD_MAC_STATS_OUT_NO_DMA_STATISTICS_OFST 0 +#define MC_CMD_MAC_STATS_OUT_NO_DMA_STATISTICS_LEN 8 +#define MC_CMD_MAC_STATS_OUT_NO_DMA_STATISTICS_LO_OFST 0 +#define MC_CMD_MAC_STATS_OUT_NO_DMA_STATISTICS_HI_OFST 4 +#define MC_CMD_MAC_STATS_OUT_NO_DMA_STATISTICS_NUM MC_CMD_MAC_NSTATS +#define MC_CMD_MAC_GENERATION_START 0x0 /* enum */ +#define MC_CMD_MAC_DMABUF_START 0x1 /* enum */ +#define MC_CMD_MAC_TX_PKTS 0x1 /* enum */ +#define MC_CMD_MAC_TX_PAUSE_PKTS 0x2 /* enum */ +#define MC_CMD_MAC_TX_CONTROL_PKTS 0x3 /* enum */ +#define MC_CMD_MAC_TX_UNICAST_PKTS 0x4 /* enum */ +#define MC_CMD_MAC_TX_MULTICAST_PKTS 0x5 /* enum */ +#define MC_CMD_MAC_TX_BROADCAST_PKTS 0x6 /* enum */ +#define MC_CMD_MAC_TX_BYTES 0x7 /* enum */ +#define MC_CMD_MAC_TX_BAD_BYTES 0x8 /* enum */ +#define MC_CMD_MAC_TX_LT64_PKTS 0x9 /* enum */ +#define MC_CMD_MAC_TX_64_PKTS 0xa /* enum */ +#define MC_CMD_MAC_TX_65_TO_127_PKTS 0xb /* enum */ +#define MC_CMD_MAC_TX_128_TO_255_PKTS 0xc /* enum */ +#define MC_CMD_MAC_TX_256_TO_511_PKTS 0xd /* enum */ +#define MC_CMD_MAC_TX_512_TO_1023_PKTS 0xe /* enum */ +#define MC_CMD_MAC_TX_1024_TO_15XX_PKTS 0xf /* enum */ +#define MC_CMD_MAC_TX_15XX_TO_JUMBO_PKTS 0x10 /* enum */ +#define MC_CMD_MAC_TX_GTJUMBO_PKTS 0x11 /* enum */ +#define MC_CMD_MAC_TX_BAD_FCS_PKTS 0x12 /* enum */ +#define MC_CMD_MAC_TX_SINGLE_COLLISION_PKTS 0x13 /* enum */ +#define MC_CMD_MAC_TX_MULTIPLE_COLLISION_PKTS 0x14 /* enum */ +#define MC_CMD_MAC_TX_EXCESSIVE_COLLISION_PKTS 0x15 /* enum */ +#define MC_CMD_MAC_TX_LATE_COLLISION_PKTS 0x16 /* enum */ +#define MC_CMD_MAC_TX_DEFERRED_PKTS 0x17 /* enum */ +#define MC_CMD_MAC_TX_EXCESSIVE_DEFERRED_PKTS 0x18 /* enum */ +#define MC_CMD_MAC_TX_NON_TCPUDP_PKTS 0x19 /* enum */ +#define MC_CMD_MAC_TX_MAC_SRC_ERR_PKTS 0x1a /* enum */ +#define MC_CMD_MAC_TX_IP_SRC_ERR_PKTS 0x1b /* enum */ +#define MC_CMD_MAC_RX_PKTS 0x1c /* enum */ +#define MC_CMD_MAC_RX_PAUSE_PKTS 0x1d /* enum */ +#define MC_CMD_MAC_RX_GOOD_PKTS 0x1e /* enum */ +#define MC_CMD_MAC_RX_CONTROL_PKTS 0x1f /* enum */ +#define MC_CMD_MAC_RX_UNICAST_PKTS 0x20 /* enum */ +#define MC_CMD_MAC_RX_MULTICAST_PKTS 0x21 /* enum */ +#define MC_CMD_MAC_RX_BROADCAST_PKTS 0x22 /* enum */ +#define MC_CMD_MAC_RX_BYTES 0x23 /* enum */ +#define MC_CMD_MAC_RX_BAD_BYTES 0x24 /* enum */ +#define MC_CMD_MAC_RX_64_PKTS 0x25 /* enum */ +#define MC_CMD_MAC_RX_65_TO_127_PKTS 0x26 /* enum */ +#define MC_CMD_MAC_RX_128_TO_255_PKTS 0x27 /* enum */ +#define MC_CMD_MAC_RX_256_TO_511_PKTS 0x28 /* enum */ +#define MC_CMD_MAC_RX_512_TO_1023_PKTS 0x29 /* enum */ +#define MC_CMD_MAC_RX_1024_TO_15XX_PKTS 0x2a /* enum */ +#define MC_CMD_MAC_RX_15XX_TO_JUMBO_PKTS 0x2b /* enum */ +#define MC_CMD_MAC_RX_GTJUMBO_PKTS 0x2c /* enum */ +#define MC_CMD_MAC_RX_UNDERSIZE_PKTS 0x2d /* enum */ +#define MC_CMD_MAC_RX_BAD_FCS_PKTS 0x2e /* enum */ +#define MC_CMD_MAC_RX_OVERFLOW_PKTS 0x2f /* enum */ +#define MC_CMD_MAC_RX_FALSE_CARRIER_PKTS 0x30 /* enum */ +#define MC_CMD_MAC_RX_SYMBOL_ERROR_PKTS 0x31 /* enum */ +#define MC_CMD_MAC_RX_ALIGN_ERROR_PKTS 0x32 /* enum */ +#define MC_CMD_MAC_RX_LENGTH_ERROR_PKTS 0x33 /* enum */ +#define MC_CMD_MAC_RX_INTERNAL_ERROR_PKTS 0x34 /* enum */ +#define MC_CMD_MAC_RX_JABBER_PKTS 0x35 /* enum */ +#define MC_CMD_MAC_RX_NODESC_DROPS 0x36 /* enum */ +#define MC_CMD_MAC_RX_LANES01_CHAR_ERR 0x37 /* enum */ +#define MC_CMD_MAC_RX_LANES23_CHAR_ERR 0x38 /* enum */ +#define MC_CMD_MAC_RX_LANES01_DISP_ERR 0x39 /* enum */ +#define MC_CMD_MAC_RX_LANES23_DISP_ERR 0x3a /* enum */ +#define MC_CMD_MAC_RX_MATCH_FAULT 0x3b /* enum */ +/* enum: PM trunc_bb_overflow counter. Valid for EF10 with PM_AND_RXDP_COUNTERS + * capability only. + */ +#define MC_CMD_MAC_PM_TRUNC_BB_OVERFLOW 0x3c +/* enum: PM discard_bb_overflow counter. Valid for EF10 with + * PM_AND_RXDP_COUNTERS capability only. + */ +#define MC_CMD_MAC_PM_DISCARD_BB_OVERFLOW 0x3d +/* enum: PM trunc_vfifo_full counter. Valid for EF10 with PM_AND_RXDP_COUNTERS + * capability only. + */ +#define MC_CMD_MAC_PM_TRUNC_VFIFO_FULL 0x3e +/* enum: PM discard_vfifo_full counter. Valid for EF10 with + * PM_AND_RXDP_COUNTERS capability only. + */ +#define MC_CMD_MAC_PM_DISCARD_VFIFO_FULL 0x3f +/* enum: PM trunc_qbb counter. Valid for EF10 with PM_AND_RXDP_COUNTERS + * capability only. + */ +#define MC_CMD_MAC_PM_TRUNC_QBB 0x40 +/* enum: PM discard_qbb counter. Valid for EF10 with PM_AND_RXDP_COUNTERS + * capability only. + */ +#define MC_CMD_MAC_PM_DISCARD_QBB 0x41 +/* enum: PM discard_mapping counter. Valid for EF10 with PM_AND_RXDP_COUNTERS + * capability only. + */ +#define MC_CMD_MAC_PM_DISCARD_MAPPING 0x42 +/* enum: RXDP counter: Number of packets dropped due to the queue being + * disabled. Valid for EF10 with PM_AND_RXDP_COUNTERS capability only. + */ +#define MC_CMD_MAC_RXDP_Q_DISABLED_PKTS 0x43 +/* enum: RXDP counter: Number of packets dropped by the DICPU. Valid for EF10 + * with PM_AND_RXDP_COUNTERS capability only. + */ +#define MC_CMD_MAC_RXDP_DI_DROPPED_PKTS 0x45 +/* enum: RXDP counter: Number of non-host packets. Valid for EF10 with + * PM_AND_RXDP_COUNTERS capability only. + */ +#define MC_CMD_MAC_RXDP_STREAMING_PKTS 0x46 +/* enum: RXDP counter: Number of times an hlb descriptor fetch was performed. + * Valid for EF10 with PM_AND_RXDP_COUNTERS capability only. + */ +#define MC_CMD_MAC_RXDP_HLB_FETCH_CONDITIONS 0x47 +/* enum: RXDP counter: Number of times the DPCPU waited for an existing + * descriptor fetch. Valid for EF10 with PM_AND_RXDP_COUNTERS capability only. + */ +#define MC_CMD_MAC_RXDP_HLB_WAIT_CONDITIONS 0x48 +#define MC_CMD_MAC_VADAPTER_RX_DMABUF_START 0x4c /* enum */ +#define MC_CMD_MAC_VADAPTER_RX_UNICAST_PACKETS 0x4c /* enum */ +#define MC_CMD_MAC_VADAPTER_RX_UNICAST_BYTES 0x4d /* enum */ +#define MC_CMD_MAC_VADAPTER_RX_MULTICAST_PACKETS 0x4e /* enum */ +#define MC_CMD_MAC_VADAPTER_RX_MULTICAST_BYTES 0x4f /* enum */ +#define MC_CMD_MAC_VADAPTER_RX_BROADCAST_PACKETS 0x50 /* enum */ +#define MC_CMD_MAC_VADAPTER_RX_BROADCAST_BYTES 0x51 /* enum */ +#define MC_CMD_MAC_VADAPTER_RX_BAD_PACKETS 0x52 /* enum */ +#define MC_CMD_MAC_VADAPTER_RX_BAD_BYTES 0x53 /* enum */ +#define MC_CMD_MAC_VADAPTER_RX_OVERFLOW 0x54 /* enum */ +#define MC_CMD_MAC_VADAPTER_TX_DMABUF_START 0x57 /* enum */ +#define MC_CMD_MAC_VADAPTER_TX_UNICAST_PACKETS 0x57 /* enum */ +#define MC_CMD_MAC_VADAPTER_TX_UNICAST_BYTES 0x58 /* enum */ +#define MC_CMD_MAC_VADAPTER_TX_MULTICAST_PACKETS 0x59 /* enum */ +#define MC_CMD_MAC_VADAPTER_TX_MULTICAST_BYTES 0x5a /* enum */ +#define MC_CMD_MAC_VADAPTER_TX_BROADCAST_PACKETS 0x5b /* enum */ +#define MC_CMD_MAC_VADAPTER_TX_BROADCAST_BYTES 0x5c /* enum */ +#define MC_CMD_MAC_VADAPTER_TX_BAD_PACKETS 0x5d /* enum */ +#define MC_CMD_MAC_VADAPTER_TX_BAD_BYTES 0x5e /* enum */ +#define MC_CMD_MAC_VADAPTER_TX_OVERFLOW 0x5f /* enum */ +/* enum: Start of GMAC stats buffer space, for Siena only. */ +#define MC_CMD_GMAC_DMABUF_START 0x40 +/* enum: End of GMAC stats buffer space, for Siena only. */ +#define MC_CMD_GMAC_DMABUF_END 0x5f +#define MC_CMD_MAC_GENERATION_END 0x60 /* enum */ +#define MC_CMD_MAC_NSTATS 0x61 /* enum */ + + +/***********************************/ +/* MC_CMD_SRIOV + * to be documented + */ +#define MC_CMD_SRIOV 0x30 + +/* MC_CMD_SRIOV_IN msgrequest */ +#define MC_CMD_SRIOV_IN_LEN 12 +#define MC_CMD_SRIOV_IN_ENABLE_OFST 0 +#define MC_CMD_SRIOV_IN_VI_BASE_OFST 4 +#define MC_CMD_SRIOV_IN_VF_COUNT_OFST 8 + +/* MC_CMD_SRIOV_OUT msgresponse */ +#define MC_CMD_SRIOV_OUT_LEN 8 +#define MC_CMD_SRIOV_OUT_VI_SCALE_OFST 0 +#define MC_CMD_SRIOV_OUT_VF_TOTAL_OFST 4 + +/* MC_CMD_MEMCPY_RECORD_TYPEDEF structuredef */ +#define MC_CMD_MEMCPY_RECORD_TYPEDEF_LEN 32 +/* this is only used for the first record */ +#define MC_CMD_MEMCPY_RECORD_TYPEDEF_NUM_RECORDS_OFST 0 +#define MC_CMD_MEMCPY_RECORD_TYPEDEF_NUM_RECORDS_LBN 0 +#define MC_CMD_MEMCPY_RECORD_TYPEDEF_NUM_RECORDS_WIDTH 32 +#define MC_CMD_MEMCPY_RECORD_TYPEDEF_TO_RID_OFST 4 +#define MC_CMD_MEMCPY_RECORD_TYPEDEF_TO_RID_LBN 32 +#define MC_CMD_MEMCPY_RECORD_TYPEDEF_TO_RID_WIDTH 32 +#define MC_CMD_MEMCPY_RECORD_TYPEDEF_TO_ADDR_OFST 8 +#define MC_CMD_MEMCPY_RECORD_TYPEDEF_TO_ADDR_LEN 8 +#define MC_CMD_MEMCPY_RECORD_TYPEDEF_TO_ADDR_LO_OFST 8 +#define MC_CMD_MEMCPY_RECORD_TYPEDEF_TO_ADDR_HI_OFST 12 +#define MC_CMD_MEMCPY_RECORD_TYPEDEF_TO_ADDR_LBN 64 +#define MC_CMD_MEMCPY_RECORD_TYPEDEF_TO_ADDR_WIDTH 64 +#define MC_CMD_MEMCPY_RECORD_TYPEDEF_FROM_RID_OFST 16 +#define MC_CMD_MEMCPY_RECORD_TYPEDEF_RID_INLINE 0x100 /* enum */ +#define MC_CMD_MEMCPY_RECORD_TYPEDEF_FROM_RID_LBN 128 +#define MC_CMD_MEMCPY_RECORD_TYPEDEF_FROM_RID_WIDTH 32 +#define MC_CMD_MEMCPY_RECORD_TYPEDEF_FROM_ADDR_OFST 20 +#define MC_CMD_MEMCPY_RECORD_TYPEDEF_FROM_ADDR_LEN 8 +#define MC_CMD_MEMCPY_RECORD_TYPEDEF_FROM_ADDR_LO_OFST 20 +#define MC_CMD_MEMCPY_RECORD_TYPEDEF_FROM_ADDR_HI_OFST 24 +#define MC_CMD_MEMCPY_RECORD_TYPEDEF_FROM_ADDR_LBN 160 +#define MC_CMD_MEMCPY_RECORD_TYPEDEF_FROM_ADDR_WIDTH 64 +#define MC_CMD_MEMCPY_RECORD_TYPEDEF_LENGTH_OFST 28 +#define MC_CMD_MEMCPY_RECORD_TYPEDEF_LENGTH_LBN 224 +#define MC_CMD_MEMCPY_RECORD_TYPEDEF_LENGTH_WIDTH 32 + + +/***********************************/ +/* MC_CMD_MEMCPY + * DMA write data into (Rid,Addr), either by dma reading (Rid,Addr), or by data + * embedded directly in the command. + * + * A common pattern is for a client to use generation counts to signal a dma + * update of a datastructure. To facilitate this, this MCDI operation can + * contain multiple requests which are executed in strict order. Requests take + * the form of duplicating the entire MCDI request continuously (including the + * requests record, which is ignored in all but the first structure) + * + * The source data can either come from a DMA from the host, or it can be + * embedded within the request directly, thereby eliminating a DMA read. To + * indicate this, the client sets FROM_RID=%RID_INLINE, ADDR_HI=0, and + * ADDR_LO=offset, and inserts the data at %offset from the start of the + * payload. It's the callers responsibility to ensure that the embedded data + * doesn't overlap the records. + * + * Returns: 0, EINVAL (invalid RID) + */ +#define MC_CMD_MEMCPY 0x31 + +/* MC_CMD_MEMCPY_IN msgrequest */ +#define MC_CMD_MEMCPY_IN_LENMIN 32 +#define MC_CMD_MEMCPY_IN_LENMAX 224 +#define MC_CMD_MEMCPY_IN_LEN(num) (0+32*(num)) +/* see MC_CMD_MEMCPY_RECORD_TYPEDEF */ +#define MC_CMD_MEMCPY_IN_RECORD_OFST 0 +#define MC_CMD_MEMCPY_IN_RECORD_LEN 32 +#define MC_CMD_MEMCPY_IN_RECORD_MINNUM 1 +#define MC_CMD_MEMCPY_IN_RECORD_MAXNUM 7 + +/* MC_CMD_MEMCPY_OUT msgresponse */ +#define MC_CMD_MEMCPY_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_WOL_FILTER_SET + * Set a WoL filter. + */ +#define MC_CMD_WOL_FILTER_SET 0x32 +#undef MC_CMD_0x32_PRIVILEGE_CTG + +#define MC_CMD_0x32_PRIVILEGE_CTG SRIOV_CTG_LINK + +/* MC_CMD_WOL_FILTER_SET_IN msgrequest */ +#define MC_CMD_WOL_FILTER_SET_IN_LEN 192 +#define MC_CMD_WOL_FILTER_SET_IN_FILTER_MODE_OFST 0 +#define MC_CMD_FILTER_MODE_SIMPLE 0x0 /* enum */ +#define MC_CMD_FILTER_MODE_STRUCTURED 0xffffffff /* enum */ +/* A type value of 1 is unused. */ +#define MC_CMD_WOL_FILTER_SET_IN_WOL_TYPE_OFST 4 +/* enum: Magic */ +#define MC_CMD_WOL_TYPE_MAGIC 0x0 +/* enum: MS Windows Magic */ +#define MC_CMD_WOL_TYPE_WIN_MAGIC 0x2 +/* enum: IPv4 Syn */ +#define MC_CMD_WOL_TYPE_IPV4_SYN 0x3 +/* enum: IPv6 Syn */ +#define MC_CMD_WOL_TYPE_IPV6_SYN 0x4 +/* enum: Bitmap */ +#define MC_CMD_WOL_TYPE_BITMAP 0x5 +/* enum: Link */ +#define MC_CMD_WOL_TYPE_LINK 0x6 +/* enum: (Above this for future use) */ +#define MC_CMD_WOL_TYPE_MAX 0x7 +#define MC_CMD_WOL_FILTER_SET_IN_DATA_OFST 8 +#define MC_CMD_WOL_FILTER_SET_IN_DATA_LEN 4 +#define MC_CMD_WOL_FILTER_SET_IN_DATA_NUM 46 + +/* MC_CMD_WOL_FILTER_SET_IN_MAGIC msgrequest */ +#define MC_CMD_WOL_FILTER_SET_IN_MAGIC_LEN 16 +/* MC_CMD_WOL_FILTER_SET_IN_FILTER_MODE_OFST 0 */ +/* MC_CMD_WOL_FILTER_SET_IN_WOL_TYPE_OFST 4 */ +#define MC_CMD_WOL_FILTER_SET_IN_MAGIC_MAC_OFST 8 +#define MC_CMD_WOL_FILTER_SET_IN_MAGIC_MAC_LEN 8 +#define MC_CMD_WOL_FILTER_SET_IN_MAGIC_MAC_LO_OFST 8 +#define MC_CMD_WOL_FILTER_SET_IN_MAGIC_MAC_HI_OFST 12 + +/* MC_CMD_WOL_FILTER_SET_IN_IPV4_SYN msgrequest */ +#define MC_CMD_WOL_FILTER_SET_IN_IPV4_SYN_LEN 20 +/* MC_CMD_WOL_FILTER_SET_IN_FILTER_MODE_OFST 0 */ +/* MC_CMD_WOL_FILTER_SET_IN_WOL_TYPE_OFST 4 */ +#define MC_CMD_WOL_FILTER_SET_IN_IPV4_SYN_SRC_IP_OFST 8 +#define MC_CMD_WOL_FILTER_SET_IN_IPV4_SYN_DST_IP_OFST 12 +#define MC_CMD_WOL_FILTER_SET_IN_IPV4_SYN_SRC_PORT_OFST 16 +#define MC_CMD_WOL_FILTER_SET_IN_IPV4_SYN_SRC_PORT_LEN 2 +#define MC_CMD_WOL_FILTER_SET_IN_IPV4_SYN_DST_PORT_OFST 18 +#define MC_CMD_WOL_FILTER_SET_IN_IPV4_SYN_DST_PORT_LEN 2 + +/* MC_CMD_WOL_FILTER_SET_IN_IPV6_SYN msgrequest */ +#define MC_CMD_WOL_FILTER_SET_IN_IPV6_SYN_LEN 44 +/* MC_CMD_WOL_FILTER_SET_IN_FILTER_MODE_OFST 0 */ +/* MC_CMD_WOL_FILTER_SET_IN_WOL_TYPE_OFST 4 */ +#define MC_CMD_WOL_FILTER_SET_IN_IPV6_SYN_SRC_IP_OFST 8 +#define MC_CMD_WOL_FILTER_SET_IN_IPV6_SYN_SRC_IP_LEN 16 +#define MC_CMD_WOL_FILTER_SET_IN_IPV6_SYN_DST_IP_OFST 24 +#define MC_CMD_WOL_FILTER_SET_IN_IPV6_SYN_DST_IP_LEN 16 +#define MC_CMD_WOL_FILTER_SET_IN_IPV6_SYN_SRC_PORT_OFST 40 +#define MC_CMD_WOL_FILTER_SET_IN_IPV6_SYN_SRC_PORT_LEN 2 +#define MC_CMD_WOL_FILTER_SET_IN_IPV6_SYN_DST_PORT_OFST 42 +#define MC_CMD_WOL_FILTER_SET_IN_IPV6_SYN_DST_PORT_LEN 2 + +/* MC_CMD_WOL_FILTER_SET_IN_BITMAP msgrequest */ +#define MC_CMD_WOL_FILTER_SET_IN_BITMAP_LEN 187 +/* MC_CMD_WOL_FILTER_SET_IN_FILTER_MODE_OFST 0 */ +/* MC_CMD_WOL_FILTER_SET_IN_WOL_TYPE_OFST 4 */ +#define MC_CMD_WOL_FILTER_SET_IN_BITMAP_MASK_OFST 8 +#define MC_CMD_WOL_FILTER_SET_IN_BITMAP_MASK_LEN 48 +#define MC_CMD_WOL_FILTER_SET_IN_BITMAP_BITMAP_OFST 56 +#define MC_CMD_WOL_FILTER_SET_IN_BITMAP_BITMAP_LEN 128 +#define MC_CMD_WOL_FILTER_SET_IN_BITMAP_LEN_OFST 184 +#define MC_CMD_WOL_FILTER_SET_IN_BITMAP_LEN_LEN 1 +#define MC_CMD_WOL_FILTER_SET_IN_BITMAP_LAYER3_OFST 185 +#define MC_CMD_WOL_FILTER_SET_IN_BITMAP_LAYER3_LEN 1 +#define MC_CMD_WOL_FILTER_SET_IN_BITMAP_LAYER4_OFST 186 +#define MC_CMD_WOL_FILTER_SET_IN_BITMAP_LAYER4_LEN 1 + +/* MC_CMD_WOL_FILTER_SET_IN_LINK msgrequest */ +#define MC_CMD_WOL_FILTER_SET_IN_LINK_LEN 12 +/* MC_CMD_WOL_FILTER_SET_IN_FILTER_MODE_OFST 0 */ +/* MC_CMD_WOL_FILTER_SET_IN_WOL_TYPE_OFST 4 */ +#define MC_CMD_WOL_FILTER_SET_IN_LINK_MASK_OFST 8 +#define MC_CMD_WOL_FILTER_SET_IN_LINK_UP_LBN 0 +#define MC_CMD_WOL_FILTER_SET_IN_LINK_UP_WIDTH 1 +#define MC_CMD_WOL_FILTER_SET_IN_LINK_DOWN_LBN 1 +#define MC_CMD_WOL_FILTER_SET_IN_LINK_DOWN_WIDTH 1 + +/* MC_CMD_WOL_FILTER_SET_OUT msgresponse */ +#define MC_CMD_WOL_FILTER_SET_OUT_LEN 4 +#define MC_CMD_WOL_FILTER_SET_OUT_FILTER_ID_OFST 0 + + +/***********************************/ +/* MC_CMD_WOL_FILTER_REMOVE + * Remove a WoL filter. Locks required: None. Returns: 0, EINVAL, ENOSYS + */ +#define MC_CMD_WOL_FILTER_REMOVE 0x33 +#undef MC_CMD_0x33_PRIVILEGE_CTG + +#define MC_CMD_0x33_PRIVILEGE_CTG SRIOV_CTG_LINK + +/* MC_CMD_WOL_FILTER_REMOVE_IN msgrequest */ +#define MC_CMD_WOL_FILTER_REMOVE_IN_LEN 4 +#define MC_CMD_WOL_FILTER_REMOVE_IN_FILTER_ID_OFST 0 + +/* MC_CMD_WOL_FILTER_REMOVE_OUT msgresponse */ +#define MC_CMD_WOL_FILTER_REMOVE_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_WOL_FILTER_RESET + * Reset (i.e. remove all) WoL filters. Locks required: None. Returns: 0, + * ENOSYS + */ +#define MC_CMD_WOL_FILTER_RESET 0x34 +#undef MC_CMD_0x34_PRIVILEGE_CTG + +#define MC_CMD_0x34_PRIVILEGE_CTG SRIOV_CTG_LINK + +/* MC_CMD_WOL_FILTER_RESET_IN msgrequest */ +#define MC_CMD_WOL_FILTER_RESET_IN_LEN 4 +#define MC_CMD_WOL_FILTER_RESET_IN_MASK_OFST 0 +#define MC_CMD_WOL_FILTER_RESET_IN_WAKE_FILTERS 0x1 /* enum */ +#define MC_CMD_WOL_FILTER_RESET_IN_LIGHTSOUT_OFFLOADS 0x2 /* enum */ + +/* MC_CMD_WOL_FILTER_RESET_OUT msgresponse */ +#define MC_CMD_WOL_FILTER_RESET_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_SET_MCAST_HASH + * Set the MCAST hash value without otherwise reconfiguring the MAC + */ +#define MC_CMD_SET_MCAST_HASH 0x35 + +/* MC_CMD_SET_MCAST_HASH_IN msgrequest */ +#define MC_CMD_SET_MCAST_HASH_IN_LEN 32 +#define MC_CMD_SET_MCAST_HASH_IN_HASH0_OFST 0 +#define MC_CMD_SET_MCAST_HASH_IN_HASH0_LEN 16 +#define MC_CMD_SET_MCAST_HASH_IN_HASH1_OFST 16 +#define MC_CMD_SET_MCAST_HASH_IN_HASH1_LEN 16 + +/* MC_CMD_SET_MCAST_HASH_OUT msgresponse */ +#define MC_CMD_SET_MCAST_HASH_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_NVRAM_TYPES + * Return bitfield indicating available types of virtual NVRAM partitions. + * Locks required: none. Returns: 0 + */ +#define MC_CMD_NVRAM_TYPES 0x36 +#undef MC_CMD_0x36_PRIVILEGE_CTG + +#define MC_CMD_0x36_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_NVRAM_TYPES_IN msgrequest */ +#define MC_CMD_NVRAM_TYPES_IN_LEN 0 + +/* MC_CMD_NVRAM_TYPES_OUT msgresponse */ +#define MC_CMD_NVRAM_TYPES_OUT_LEN 4 +/* Bit mask of supported types. */ +#define MC_CMD_NVRAM_TYPES_OUT_TYPES_OFST 0 +/* enum: Disabled callisto. */ +#define MC_CMD_NVRAM_TYPE_DISABLED_CALLISTO 0x0 +/* enum: MC firmware. */ +#define MC_CMD_NVRAM_TYPE_MC_FW 0x1 +/* enum: MC backup firmware. */ +#define MC_CMD_NVRAM_TYPE_MC_FW_BACKUP 0x2 +/* enum: Static configuration Port0. */ +#define MC_CMD_NVRAM_TYPE_STATIC_CFG_PORT0 0x3 +/* enum: Static configuration Port1. */ +#define MC_CMD_NVRAM_TYPE_STATIC_CFG_PORT1 0x4 +/* enum: Dynamic configuration Port0. */ +#define MC_CMD_NVRAM_TYPE_DYNAMIC_CFG_PORT0 0x5 +/* enum: Dynamic configuration Port1. */ +#define MC_CMD_NVRAM_TYPE_DYNAMIC_CFG_PORT1 0x6 +/* enum: Expansion Rom. */ +#define MC_CMD_NVRAM_TYPE_EXP_ROM 0x7 +/* enum: Expansion Rom Configuration Port0. */ +#define MC_CMD_NVRAM_TYPE_EXP_ROM_CFG_PORT0 0x8 +/* enum: Expansion Rom Configuration Port1. */ +#define MC_CMD_NVRAM_TYPE_EXP_ROM_CFG_PORT1 0x9 +/* enum: Phy Configuration Port0. */ +#define MC_CMD_NVRAM_TYPE_PHY_PORT0 0xa +/* enum: Phy Configuration Port1. */ +#define MC_CMD_NVRAM_TYPE_PHY_PORT1 0xb +/* enum: Log. */ +#define MC_CMD_NVRAM_TYPE_LOG 0xc +/* enum: FPGA image. */ +#define MC_CMD_NVRAM_TYPE_FPGA 0xd +/* enum: FPGA backup image */ +#define MC_CMD_NVRAM_TYPE_FPGA_BACKUP 0xe +/* enum: FC firmware. */ +#define MC_CMD_NVRAM_TYPE_FC_FW 0xf +/* enum: FC backup firmware. */ +#define MC_CMD_NVRAM_TYPE_FC_FW_BACKUP 0x10 +/* enum: CPLD image. */ +#define MC_CMD_NVRAM_TYPE_CPLD 0x11 +/* enum: Licensing information. */ +#define MC_CMD_NVRAM_TYPE_LICENSE 0x12 +/* enum: FC Log. */ +#define MC_CMD_NVRAM_TYPE_FC_LOG 0x13 +/* enum: Additional flash on FPGA. */ +#define MC_CMD_NVRAM_TYPE_FC_EXTRA 0x14 + + +/***********************************/ +/* MC_CMD_NVRAM_INFO + * Read info about a virtual NVRAM partition. Locks required: none. Returns: 0, + * EINVAL (bad type). + */ +#define MC_CMD_NVRAM_INFO 0x37 +#undef MC_CMD_0x37_PRIVILEGE_CTG + +#define MC_CMD_0x37_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_NVRAM_INFO_IN msgrequest */ +#define MC_CMD_NVRAM_INFO_IN_LEN 4 +#define MC_CMD_NVRAM_INFO_IN_TYPE_OFST 0 +/* Enum values, see field(s): */ +/* MC_CMD_NVRAM_TYPES/MC_CMD_NVRAM_TYPES_OUT/TYPES */ + +/* MC_CMD_NVRAM_INFO_OUT msgresponse */ +#define MC_CMD_NVRAM_INFO_OUT_LEN 24 +#define MC_CMD_NVRAM_INFO_OUT_TYPE_OFST 0 +/* Enum values, see field(s): */ +/* MC_CMD_NVRAM_TYPES/MC_CMD_NVRAM_TYPES_OUT/TYPES */ +#define MC_CMD_NVRAM_INFO_OUT_SIZE_OFST 4 +#define MC_CMD_NVRAM_INFO_OUT_ERASESIZE_OFST 8 +#define MC_CMD_NVRAM_INFO_OUT_FLAGS_OFST 12 +#define MC_CMD_NVRAM_INFO_OUT_PROTECTED_LBN 0 +#define MC_CMD_NVRAM_INFO_OUT_PROTECTED_WIDTH 1 +#define MC_CMD_NVRAM_INFO_OUT_TLV_LBN 1 +#define MC_CMD_NVRAM_INFO_OUT_TLV_WIDTH 1 +#define MC_CMD_NVRAM_INFO_OUT_READ_ONLY_LBN 5 +#define MC_CMD_NVRAM_INFO_OUT_READ_ONLY_WIDTH 1 +#define MC_CMD_NVRAM_INFO_OUT_CMAC_LBN 6 +#define MC_CMD_NVRAM_INFO_OUT_CMAC_WIDTH 1 +#define MC_CMD_NVRAM_INFO_OUT_A_B_LBN 7 +#define MC_CMD_NVRAM_INFO_OUT_A_B_WIDTH 1 +#define MC_CMD_NVRAM_INFO_OUT_PHYSDEV_OFST 16 +#define MC_CMD_NVRAM_INFO_OUT_PHYSADDR_OFST 20 + +/* MC_CMD_NVRAM_INFO_V2_OUT msgresponse */ +#define MC_CMD_NVRAM_INFO_V2_OUT_LEN 28 +#define MC_CMD_NVRAM_INFO_V2_OUT_TYPE_OFST 0 +/* Enum values, see field(s): */ +/* MC_CMD_NVRAM_TYPES/MC_CMD_NVRAM_TYPES_OUT/TYPES */ +#define MC_CMD_NVRAM_INFO_V2_OUT_SIZE_OFST 4 +#define MC_CMD_NVRAM_INFO_V2_OUT_ERASESIZE_OFST 8 +#define MC_CMD_NVRAM_INFO_V2_OUT_FLAGS_OFST 12 +#define MC_CMD_NVRAM_INFO_V2_OUT_PROTECTED_LBN 0 +#define MC_CMD_NVRAM_INFO_V2_OUT_PROTECTED_WIDTH 1 +#define MC_CMD_NVRAM_INFO_V2_OUT_TLV_LBN 1 +#define MC_CMD_NVRAM_INFO_V2_OUT_TLV_WIDTH 1 +#define MC_CMD_NVRAM_INFO_V2_OUT_READ_ONLY_LBN 5 +#define MC_CMD_NVRAM_INFO_V2_OUT_READ_ONLY_WIDTH 1 +#define MC_CMD_NVRAM_INFO_V2_OUT_A_B_LBN 7 +#define MC_CMD_NVRAM_INFO_V2_OUT_A_B_WIDTH 1 +#define MC_CMD_NVRAM_INFO_V2_OUT_PHYSDEV_OFST 16 +#define MC_CMD_NVRAM_INFO_V2_OUT_PHYSADDR_OFST 20 +/* Writes must be multiples of this size. Added to support the MUM on Sorrento. + */ +#define MC_CMD_NVRAM_INFO_V2_OUT_WRITESIZE_OFST 24 + + +/***********************************/ +/* MC_CMD_NVRAM_UPDATE_START + * Start a group of update operations on a virtual NVRAM partition. Locks + * required: PHY_LOCK if type==*PHY*. Returns: 0, EINVAL (bad type), EACCES (if + * PHY_LOCK required and not held). + */ +#define MC_CMD_NVRAM_UPDATE_START 0x38 +#undef MC_CMD_0x38_PRIVILEGE_CTG + +#define MC_CMD_0x38_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_NVRAM_UPDATE_START_IN msgrequest: Legacy NVRAM_UPDATE_START request. + * Use NVRAM_UPDATE_START_V2_IN in new code + */ +#define MC_CMD_NVRAM_UPDATE_START_IN_LEN 4 +#define MC_CMD_NVRAM_UPDATE_START_IN_TYPE_OFST 0 +/* Enum values, see field(s): */ +/* MC_CMD_NVRAM_TYPES/MC_CMD_NVRAM_TYPES_OUT/TYPES */ + +/* MC_CMD_NVRAM_UPDATE_START_V2_IN msgrequest: Extended NVRAM_UPDATE_START + * request with additional flags indicating version of command in use. See + * MC_CMD_NVRAM_UPDATE_FINISH_V2_OUT for details of extended functionality. Use + * paired up with NVRAM_UPDATE_FINISH_V2_IN. + */ +#define MC_CMD_NVRAM_UPDATE_START_V2_IN_LEN 8 +#define MC_CMD_NVRAM_UPDATE_START_V2_IN_TYPE_OFST 0 +/* Enum values, see field(s): */ +/* MC_CMD_NVRAM_TYPES/MC_CMD_NVRAM_TYPES_OUT/TYPES */ +#define MC_CMD_NVRAM_UPDATE_START_V2_IN_FLAGS_OFST 4 +#define MC_CMD_NVRAM_UPDATE_START_V2_IN_FLAG_REPORT_VERIFY_RESULT_LBN 0 +#define MC_CMD_NVRAM_UPDATE_START_V2_IN_FLAG_REPORT_VERIFY_RESULT_WIDTH 1 + +/* MC_CMD_NVRAM_UPDATE_START_OUT msgresponse */ +#define MC_CMD_NVRAM_UPDATE_START_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_NVRAM_READ + * Read data from a virtual NVRAM partition. Locks required: PHY_LOCK if + * type==*PHY*. Returns: 0, EINVAL (bad type/offset/length), EACCES (if + * PHY_LOCK required and not held) + */ +#define MC_CMD_NVRAM_READ 0x39 +#undef MC_CMD_0x39_PRIVILEGE_CTG + +#define MC_CMD_0x39_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_NVRAM_READ_IN msgrequest */ +#define MC_CMD_NVRAM_READ_IN_LEN 12 +#define MC_CMD_NVRAM_READ_IN_TYPE_OFST 0 +/* Enum values, see field(s): */ +/* MC_CMD_NVRAM_TYPES/MC_CMD_NVRAM_TYPES_OUT/TYPES */ +#define MC_CMD_NVRAM_READ_IN_OFFSET_OFST 4 +/* amount to read in bytes */ +#define MC_CMD_NVRAM_READ_IN_LENGTH_OFST 8 + +/* MC_CMD_NVRAM_READ_IN_V2 msgrequest */ +#define MC_CMD_NVRAM_READ_IN_V2_LEN 16 +#define MC_CMD_NVRAM_READ_IN_V2_TYPE_OFST 0 +/* Enum values, see field(s): */ +/* MC_CMD_NVRAM_TYPES/MC_CMD_NVRAM_TYPES_OUT/TYPES */ +#define MC_CMD_NVRAM_READ_IN_V2_OFFSET_OFST 4 +/* amount to read in bytes */ +#define MC_CMD_NVRAM_READ_IN_V2_LENGTH_OFST 8 +/* Optional control info. If a partition is stored with an A/B versioning + * scheme (i.e. in more than one physical partition in NVRAM) the host can set + * this to control which underlying physical partition is used to read data + * from. This allows it to perform a read-modify-write-verify with the write + * lock continuously held by calling NVRAM_UPDATE_START, reading the old + * contents using MODE=TARGET_CURRENT, overwriting the old partition and then + * verifying by reading with MODE=TARGET_BACKUP. + */ +#define MC_CMD_NVRAM_READ_IN_V2_MODE_OFST 12 +/* enum: Same as omitting MODE: caller sees data in current partition unless it + * holds the write lock in which case it sees data in the partition it is + * updating. + */ +#define MC_CMD_NVRAM_READ_IN_V2_DEFAULT 0x0 +/* enum: Read from the current partition of an A/B pair, even if holding the + * write lock. + */ +#define MC_CMD_NVRAM_READ_IN_V2_TARGET_CURRENT 0x1 +/* enum: Read from the non-current (i.e. to be updated) partition of an A/B + * pair + */ +#define MC_CMD_NVRAM_READ_IN_V2_TARGET_BACKUP 0x2 + +/* MC_CMD_NVRAM_READ_OUT msgresponse */ +#define MC_CMD_NVRAM_READ_OUT_LENMIN 1 +#define MC_CMD_NVRAM_READ_OUT_LENMAX 252 +#define MC_CMD_NVRAM_READ_OUT_LEN(num) (0+1*(num)) +#define MC_CMD_NVRAM_READ_OUT_READ_BUFFER_OFST 0 +#define MC_CMD_NVRAM_READ_OUT_READ_BUFFER_LEN 1 +#define MC_CMD_NVRAM_READ_OUT_READ_BUFFER_MINNUM 1 +#define MC_CMD_NVRAM_READ_OUT_READ_BUFFER_MAXNUM 252 + + +/***********************************/ +/* MC_CMD_NVRAM_WRITE + * Write data to a virtual NVRAM partition. Locks required: PHY_LOCK if + * type==*PHY*. Returns: 0, EINVAL (bad type/offset/length), EACCES (if + * PHY_LOCK required and not held) + */ +#define MC_CMD_NVRAM_WRITE 0x3a +#undef MC_CMD_0x3a_PRIVILEGE_CTG + +#define MC_CMD_0x3a_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_NVRAM_WRITE_IN msgrequest */ +#define MC_CMD_NVRAM_WRITE_IN_LENMIN 13 +#define MC_CMD_NVRAM_WRITE_IN_LENMAX 252 +#define MC_CMD_NVRAM_WRITE_IN_LEN(num) (12+1*(num)) +#define MC_CMD_NVRAM_WRITE_IN_TYPE_OFST 0 +/* Enum values, see field(s): */ +/* MC_CMD_NVRAM_TYPES/MC_CMD_NVRAM_TYPES_OUT/TYPES */ +#define MC_CMD_NVRAM_WRITE_IN_OFFSET_OFST 4 +#define MC_CMD_NVRAM_WRITE_IN_LENGTH_OFST 8 +#define MC_CMD_NVRAM_WRITE_IN_WRITE_BUFFER_OFST 12 +#define MC_CMD_NVRAM_WRITE_IN_WRITE_BUFFER_LEN 1 +#define MC_CMD_NVRAM_WRITE_IN_WRITE_BUFFER_MINNUM 1 +#define MC_CMD_NVRAM_WRITE_IN_WRITE_BUFFER_MAXNUM 240 + +/* MC_CMD_NVRAM_WRITE_OUT msgresponse */ +#define MC_CMD_NVRAM_WRITE_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_NVRAM_ERASE + * Erase sector(s) from a virtual NVRAM partition. Locks required: PHY_LOCK if + * type==*PHY*. Returns: 0, EINVAL (bad type/offset/length), EACCES (if + * PHY_LOCK required and not held) + */ +#define MC_CMD_NVRAM_ERASE 0x3b +#undef MC_CMD_0x3b_PRIVILEGE_CTG + +#define MC_CMD_0x3b_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_NVRAM_ERASE_IN msgrequest */ +#define MC_CMD_NVRAM_ERASE_IN_LEN 12 +#define MC_CMD_NVRAM_ERASE_IN_TYPE_OFST 0 +/* Enum values, see field(s): */ +/* MC_CMD_NVRAM_TYPES/MC_CMD_NVRAM_TYPES_OUT/TYPES */ +#define MC_CMD_NVRAM_ERASE_IN_OFFSET_OFST 4 +#define MC_CMD_NVRAM_ERASE_IN_LENGTH_OFST 8 + +/* MC_CMD_NVRAM_ERASE_OUT msgresponse */ +#define MC_CMD_NVRAM_ERASE_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_NVRAM_UPDATE_FINISH + * Finish a group of update operations on a virtual NVRAM partition. Locks + * required: PHY_LOCK if type==*PHY*. Returns: 0, EINVAL (bad + * type/offset/length), EACCES (if PHY_LOCK required and not held) + */ +#define MC_CMD_NVRAM_UPDATE_FINISH 0x3c +#undef MC_CMD_0x3c_PRIVILEGE_CTG + +#define MC_CMD_0x3c_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_NVRAM_UPDATE_FINISH_IN msgrequest: Legacy NVRAM_UPDATE_FINISH + * request. Use NVRAM_UPDATE_FINISH_V2_IN in new code + */ +#define MC_CMD_NVRAM_UPDATE_FINISH_IN_LEN 8 +#define MC_CMD_NVRAM_UPDATE_FINISH_IN_TYPE_OFST 0 +/* Enum values, see field(s): */ +/* MC_CMD_NVRAM_TYPES/MC_CMD_NVRAM_TYPES_OUT/TYPES */ +#define MC_CMD_NVRAM_UPDATE_FINISH_IN_REBOOT_OFST 4 + +/* MC_CMD_NVRAM_UPDATE_FINISH_V2_IN msgrequest: Extended NVRAM_UPDATE_FINISH + * request with additional flags indicating version of NVRAM_UPDATE commands in + * use. See MC_CMD_NVRAM_UPDATE_FINISH_V2_OUT for details of extended + * functionality. Use paired up with NVRAM_UPDATE_START_V2_IN. + */ +#define MC_CMD_NVRAM_UPDATE_FINISH_V2_IN_LEN 12 +#define MC_CMD_NVRAM_UPDATE_FINISH_V2_IN_TYPE_OFST 0 +/* Enum values, see field(s): */ +/* MC_CMD_NVRAM_TYPES/MC_CMD_NVRAM_TYPES_OUT/TYPES */ +#define MC_CMD_NVRAM_UPDATE_FINISH_V2_IN_REBOOT_OFST 4 +#define MC_CMD_NVRAM_UPDATE_FINISH_V2_IN_FLAGS_OFST 8 +#define MC_CMD_NVRAM_UPDATE_FINISH_V2_IN_FLAG_REPORT_VERIFY_RESULT_LBN 0 +#define MC_CMD_NVRAM_UPDATE_FINISH_V2_IN_FLAG_REPORT_VERIFY_RESULT_WIDTH 1 + +/* MC_CMD_NVRAM_UPDATE_FINISH_OUT msgresponse: Legacy NVRAM_UPDATE_FINISH + * response. Use NVRAM_UPDATE_FINISH_V2_OUT in new code + */ +#define MC_CMD_NVRAM_UPDATE_FINISH_OUT_LEN 0 + +/* MC_CMD_NVRAM_UPDATE_FINISH_V2_OUT msgresponse: + * + * Extended NVRAM_UPDATE_FINISH response that communicates the result of secure + * firmware validation where applicable back to the host. + * + * Medford only: For signed firmware images, such as those for medford, the MC + * firmware verifies the signature before marking the firmware image as valid. + * This process takes a few seconds to complete. So is likely to take more than + * the MCDI timeout. Hence signature verification is initiated when + * MC_CMD_NVRAM_UPDATE_FINISH_V2_IN is received by the firmware, however, the + * MCDI command is run in a background MCDI processing thread. This response + * payload includes the results of the signature verification. Note that the + * per-partition nvram lock in firmware is only released after the verification + * has completed. + */ +#define MC_CMD_NVRAM_UPDATE_FINISH_V2_OUT_LEN 4 +/* Result of nvram update completion processing */ +#define MC_CMD_NVRAM_UPDATE_FINISH_V2_OUT_RESULT_CODE_OFST 0 +/* enum: Invalid return code; only non-zero values are defined. Defined as + * unknown for backwards compatibility with NVRAM_UPDATE_FINISH_OUT. + */ +#define MC_CMD_NVRAM_VERIFY_RC_UNKNOWN 0x0 +/* enum: Verify succeeded without any errors. */ +#define MC_CMD_NVRAM_VERIFY_RC_SUCCESS 0x1 +/* enum: CMS format verification failed due to an internal error. */ +#define MC_CMD_NVRAM_VERIFY_RC_CMS_CHECK_FAILED 0x2 +/* enum: Invalid CMS format in image metadata. */ +#define MC_CMD_NVRAM_VERIFY_RC_INVALID_CMS_FORMAT 0x3 +/* enum: Message digest verification failed due to an internal error. */ +#define MC_CMD_NVRAM_VERIFY_RC_MESSAGE_DIGEST_CHECK_FAILED 0x4 +/* enum: Error in message digest calculated over the reflash-header, payload + * and reflash-trailer. + */ +#define MC_CMD_NVRAM_VERIFY_RC_BAD_MESSAGE_DIGEST 0x5 +/* enum: Signature verification failed due to an internal error. */ +#define MC_CMD_NVRAM_VERIFY_RC_SIGNATURE_CHECK_FAILED 0x6 +/* enum: There are no valid signatures in the image. */ +#define MC_CMD_NVRAM_VERIFY_RC_NO_VALID_SIGNATURES 0x7 +/* enum: Trusted approvers verification failed due to an internal error. */ +#define MC_CMD_NVRAM_VERIFY_RC_TRUSTED_APPROVERS_CHECK_FAILED 0x8 +/* enum: The Trusted approver's list is empty. */ +#define MC_CMD_NVRAM_VERIFY_RC_NO_TRUSTED_APPROVERS 0x9 +/* enum: Signature chain verification failed due to an internal error. */ +#define MC_CMD_NVRAM_VERIFY_RC_SIGNATURE_CHAIN_CHECK_FAILED 0xa +/* enum: The signers of the signatures in the image are not listed in the + * Trusted approver's list. + */ +#define MC_CMD_NVRAM_VERIFY_RC_NO_SIGNATURE_MATCH 0xb +/* enum: The image contains a test-signed certificate, but the adapter accepts + * only production signed images. + */ +#define MC_CMD_NVRAM_VERIFY_RC_REJECT_TEST_SIGNED 0xc + + +/***********************************/ +/* MC_CMD_REBOOT + * Reboot the MC. + * + * The AFTER_ASSERTION flag is intended to be used when the driver notices an + * assertion failure (at which point it is expected to perform a complete tear + * down and reinitialise), to allow both ports to reset the MC once in an + * atomic fashion. + * + * Production mc firmwares are generally compiled with REBOOT_ON_ASSERT=1, + * which means that they will automatically reboot out of the assertion + * handler, so this is in practise an optional operation. It is still + * recommended that drivers execute this to support custom firmwares with + * REBOOT_ON_ASSERT=0. + * + * Locks required: NONE Returns: Nothing. You get back a response with ERR=1, + * DATALEN=0 + */ +#define MC_CMD_REBOOT 0x3d +#undef MC_CMD_0x3d_PRIVILEGE_CTG + +#define MC_CMD_0x3d_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_REBOOT_IN msgrequest */ +#define MC_CMD_REBOOT_IN_LEN 4 +#define MC_CMD_REBOOT_IN_FLAGS_OFST 0 +#define MC_CMD_REBOOT_FLAGS_AFTER_ASSERTION 0x1 /* enum */ + +/* MC_CMD_REBOOT_OUT msgresponse */ +#define MC_CMD_REBOOT_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_SCHEDINFO + * Request scheduler info. Locks required: NONE. Returns: An array of + * (timeslice,maximum overrun), one for each thread, in ascending order of + * thread address. + */ +#define MC_CMD_SCHEDINFO 0x3e +#undef MC_CMD_0x3e_PRIVILEGE_CTG + +#define MC_CMD_0x3e_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_SCHEDINFO_IN msgrequest */ +#define MC_CMD_SCHEDINFO_IN_LEN 0 + +/* MC_CMD_SCHEDINFO_OUT msgresponse */ +#define MC_CMD_SCHEDINFO_OUT_LENMIN 4 +#define MC_CMD_SCHEDINFO_OUT_LENMAX 252 +#define MC_CMD_SCHEDINFO_OUT_LEN(num) (0+4*(num)) +#define MC_CMD_SCHEDINFO_OUT_DATA_OFST 0 +#define MC_CMD_SCHEDINFO_OUT_DATA_LEN 4 +#define MC_CMD_SCHEDINFO_OUT_DATA_MINNUM 1 +#define MC_CMD_SCHEDINFO_OUT_DATA_MAXNUM 63 + + +/***********************************/ +/* MC_CMD_REBOOT_MODE + * Set the mode for the next MC reboot. Locks required: NONE. Sets the reboot + * mode to the specified value. Returns the old mode. + */ +#define MC_CMD_REBOOT_MODE 0x3f +#undef MC_CMD_0x3f_PRIVILEGE_CTG + +#define MC_CMD_0x3f_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_REBOOT_MODE_IN msgrequest */ +#define MC_CMD_REBOOT_MODE_IN_LEN 4 +#define MC_CMD_REBOOT_MODE_IN_VALUE_OFST 0 +/* enum: Normal. */ +#define MC_CMD_REBOOT_MODE_NORMAL 0x0 +/* enum: Power-on Reset. */ +#define MC_CMD_REBOOT_MODE_POR 0x2 +/* enum: Snapper. */ +#define MC_CMD_REBOOT_MODE_SNAPPER 0x3 +/* enum: snapper fake POR */ +#define MC_CMD_REBOOT_MODE_SNAPPER_POR 0x4 +#define MC_CMD_REBOOT_MODE_IN_FAKE_LBN 7 +#define MC_CMD_REBOOT_MODE_IN_FAKE_WIDTH 1 + +/* MC_CMD_REBOOT_MODE_OUT msgresponse */ +#define MC_CMD_REBOOT_MODE_OUT_LEN 4 +#define MC_CMD_REBOOT_MODE_OUT_VALUE_OFST 0 + + +/***********************************/ +/* MC_CMD_SENSOR_INFO + * Returns information about every available sensor. + * + * Each sensor has a single (16bit) value, and a corresponding state. The + * mapping between value and state is nominally determined by the MC, but may + * be implemented using up to 2 ranges per sensor. + * + * This call returns a mask (32bit) of the sensors that are supported by this + * platform, then an array of sensor information structures, in order of sensor + * type (but without gaps for unimplemented sensors). Each structure defines + * the ranges for the corresponding sensor. An unused range is indicated by + * equal limit values. If one range is used, a value outside that range results + * in STATE_FATAL. If two ranges are used, a value outside the second range + * results in STATE_FATAL while a value outside the first and inside the second + * range results in STATE_WARNING. + * + * Sensor masks and sensor information arrays are organised into pages. For + * backward compatibility, older host software can only use sensors in page 0. + * Bit 32 in the sensor mask was previously unused, and is no reserved for use + * as the next page flag. + * + * If the request does not contain a PAGE value then firmware will only return + * page 0 of sensor information, with bit 31 in the sensor mask cleared. + * + * If the request contains a PAGE value then firmware responds with the sensor + * mask and sensor information array for that page of sensors. In this case bit + * 31 in the mask is set if another page exists. + * + * Locks required: None Returns: 0 + */ +#define MC_CMD_SENSOR_INFO 0x41 +#undef MC_CMD_0x41_PRIVILEGE_CTG + +#define MC_CMD_0x41_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_SENSOR_INFO_IN msgrequest */ +#define MC_CMD_SENSOR_INFO_IN_LEN 0 + +/* MC_CMD_SENSOR_INFO_EXT_IN msgrequest */ +#define MC_CMD_SENSOR_INFO_EXT_IN_LEN 4 +/* Which page of sensors to report. + * + * Page 0 contains sensors 0 to 30 (sensor 31 is the next page bit). + * + * Page 1 contains sensors 32 to 62 (sensor 63 is the next page bit). etc. + */ +#define MC_CMD_SENSOR_INFO_EXT_IN_PAGE_OFST 0 + +/* MC_CMD_SENSOR_INFO_OUT msgresponse */ +#define MC_CMD_SENSOR_INFO_OUT_LENMIN 4 +#define MC_CMD_SENSOR_INFO_OUT_LENMAX 252 +#define MC_CMD_SENSOR_INFO_OUT_LEN(num) (4+8*(num)) +#define MC_CMD_SENSOR_INFO_OUT_MASK_OFST 0 +/* enum: Controller temperature: degC */ +#define MC_CMD_SENSOR_CONTROLLER_TEMP 0x0 +/* enum: Phy common temperature: degC */ +#define MC_CMD_SENSOR_PHY_COMMON_TEMP 0x1 +/* enum: Controller cooling: bool */ +#define MC_CMD_SENSOR_CONTROLLER_COOLING 0x2 +/* enum: Phy 0 temperature: degC */ +#define MC_CMD_SENSOR_PHY0_TEMP 0x3 +/* enum: Phy 0 cooling: bool */ +#define MC_CMD_SENSOR_PHY0_COOLING 0x4 +/* enum: Phy 1 temperature: degC */ +#define MC_CMD_SENSOR_PHY1_TEMP 0x5 +/* enum: Phy 1 cooling: bool */ +#define MC_CMD_SENSOR_PHY1_COOLING 0x6 +/* enum: 1.0v power: mV */ +#define MC_CMD_SENSOR_IN_1V0 0x7 +/* enum: 1.2v power: mV */ +#define MC_CMD_SENSOR_IN_1V2 0x8 +/* enum: 1.8v power: mV */ +#define MC_CMD_SENSOR_IN_1V8 0x9 +/* enum: 2.5v power: mV */ +#define MC_CMD_SENSOR_IN_2V5 0xa +/* enum: 3.3v power: mV */ +#define MC_CMD_SENSOR_IN_3V3 0xb +/* enum: 12v power: mV */ +#define MC_CMD_SENSOR_IN_12V0 0xc +/* enum: 1.2v analogue power: mV */ +#define MC_CMD_SENSOR_IN_1V2A 0xd +/* enum: reference voltage: mV */ +#define MC_CMD_SENSOR_IN_VREF 0xe +/* enum: AOE FPGA power: mV */ +#define MC_CMD_SENSOR_OUT_VAOE 0xf +/* enum: AOE FPGA temperature: degC */ +#define MC_CMD_SENSOR_AOE_TEMP 0x10 +/* enum: AOE FPGA PSU temperature: degC */ +#define MC_CMD_SENSOR_PSU_AOE_TEMP 0x11 +/* enum: AOE PSU temperature: degC */ +#define MC_CMD_SENSOR_PSU_TEMP 0x12 +/* enum: Fan 0 speed: RPM */ +#define MC_CMD_SENSOR_FAN_0 0x13 +/* enum: Fan 1 speed: RPM */ +#define MC_CMD_SENSOR_FAN_1 0x14 +/* enum: Fan 2 speed: RPM */ +#define MC_CMD_SENSOR_FAN_2 0x15 +/* enum: Fan 3 speed: RPM */ +#define MC_CMD_SENSOR_FAN_3 0x16 +/* enum: Fan 4 speed: RPM */ +#define MC_CMD_SENSOR_FAN_4 0x17 +/* enum: AOE FPGA input power: mV */ +#define MC_CMD_SENSOR_IN_VAOE 0x18 +/* enum: AOE FPGA current: mA */ +#define MC_CMD_SENSOR_OUT_IAOE 0x19 +/* enum: AOE FPGA input current: mA */ +#define MC_CMD_SENSOR_IN_IAOE 0x1a +/* enum: NIC power consumption: W */ +#define MC_CMD_SENSOR_NIC_POWER 0x1b +/* enum: 0.9v power voltage: mV */ +#define MC_CMD_SENSOR_IN_0V9 0x1c +/* enum: 0.9v power current: mA */ +#define MC_CMD_SENSOR_IN_I0V9 0x1d +/* enum: 1.2v power current: mA */ +#define MC_CMD_SENSOR_IN_I1V2 0x1e +/* enum: Not a sensor: reserved for the next page flag */ +#define MC_CMD_SENSOR_PAGE0_NEXT 0x1f +/* enum: 0.9v power voltage (at ADC): mV */ +#define MC_CMD_SENSOR_IN_0V9_ADC 0x20 +/* enum: Controller temperature 2: degC */ +#define MC_CMD_SENSOR_CONTROLLER_2_TEMP 0x21 +/* enum: Voltage regulator internal temperature: degC */ +#define MC_CMD_SENSOR_VREG_INTERNAL_TEMP 0x22 +/* enum: 0.9V voltage regulator temperature: degC */ +#define MC_CMD_SENSOR_VREG_0V9_TEMP 0x23 +/* enum: 1.2V voltage regulator temperature: degC */ +#define MC_CMD_SENSOR_VREG_1V2_TEMP 0x24 +/* enum: controller internal temperature sensor voltage (internal ADC): mV */ +#define MC_CMD_SENSOR_CONTROLLER_VPTAT 0x25 +/* enum: controller internal temperature (internal ADC): degC */ +#define MC_CMD_SENSOR_CONTROLLER_INTERNAL_TEMP 0x26 +/* enum: controller internal temperature sensor voltage (external ADC): mV */ +#define MC_CMD_SENSOR_CONTROLLER_VPTAT_EXTADC 0x27 +/* enum: controller internal temperature (external ADC): degC */ +#define MC_CMD_SENSOR_CONTROLLER_INTERNAL_TEMP_EXTADC 0x28 +/* enum: ambient temperature: degC */ +#define MC_CMD_SENSOR_AMBIENT_TEMP 0x29 +/* enum: air flow: bool */ +#define MC_CMD_SENSOR_AIRFLOW 0x2a +/* enum: voltage between VSS08D and VSS08D at CSR: mV */ +#define MC_CMD_SENSOR_VDD08D_VSS08D_CSR 0x2b +/* enum: voltage between VSS08D and VSS08D at CSR (external ADC): mV */ +#define MC_CMD_SENSOR_VDD08D_VSS08D_CSR_EXTADC 0x2c +/* enum: Hotpoint temperature: degC */ +#define MC_CMD_SENSOR_HOTPOINT_TEMP 0x2d +/* enum: Port 0 PHY power switch over-current: bool */ +#define MC_CMD_SENSOR_PHY_POWER_PORT0 0x2e +/* enum: Port 1 PHY power switch over-current: bool */ +#define MC_CMD_SENSOR_PHY_POWER_PORT1 0x2f +/* enum: Mop-up microcontroller reference voltage (millivolts) */ +#define MC_CMD_SENSOR_MUM_VCC 0x30 +/* enum: 0.9v power phase A voltage: mV */ +#define MC_CMD_SENSOR_IN_0V9_A 0x31 +/* enum: 0.9v power phase A current: mA */ +#define MC_CMD_SENSOR_IN_I0V9_A 0x32 +/* enum: 0.9V voltage regulator phase A temperature: degC */ +#define MC_CMD_SENSOR_VREG_0V9_A_TEMP 0x33 +/* enum: 0.9v power phase B voltage: mV */ +#define MC_CMD_SENSOR_IN_0V9_B 0x34 +/* enum: 0.9v power phase B current: mA */ +#define MC_CMD_SENSOR_IN_I0V9_B 0x35 +/* enum: 0.9V voltage regulator phase B temperature: degC */ +#define MC_CMD_SENSOR_VREG_0V9_B_TEMP 0x36 +/* enum: CCOM AVREG 1v2 supply (interval ADC): mV */ +#define MC_CMD_SENSOR_CCOM_AVREG_1V2_SUPPLY 0x37 +/* enum: CCOM AVREG 1v2 supply (external ADC): mV */ +#define MC_CMD_SENSOR_CCOM_AVREG_1V2_SUPPLY_EXTADC 0x38 +/* enum: CCOM AVREG 1v8 supply (interval ADC): mV */ +#define MC_CMD_SENSOR_CCOM_AVREG_1V8_SUPPLY 0x39 +/* enum: CCOM AVREG 1v8 supply (external ADC): mV */ +#define MC_CMD_SENSOR_CCOM_AVREG_1V8_SUPPLY_EXTADC 0x3a +/* enum: CCOM RTS temperature: degC */ +#define MC_CMD_SENSOR_CONTROLLER_RTS 0x3b +/* enum: Not a sensor: reserved for the next page flag */ +#define MC_CMD_SENSOR_PAGE1_NEXT 0x3f +/* enum: controller internal temperature sensor voltage on master core + * (internal ADC): mV + */ +#define MC_CMD_SENSOR_CONTROLLER_MASTER_VPTAT 0x40 +/* enum: controller internal temperature on master core (internal ADC): degC */ +#define MC_CMD_SENSOR_CONTROLLER_MASTER_INTERNAL_TEMP 0x41 +/* enum: controller internal temperature sensor voltage on master core + * (external ADC): mV + */ +#define MC_CMD_SENSOR_CONTROLLER_MASTER_VPTAT_EXTADC 0x42 +/* enum: controller internal temperature on master core (external ADC): degC */ +#define MC_CMD_SENSOR_CONTROLLER_MASTER_INTERNAL_TEMP_EXTADC 0x43 +/* enum: controller internal temperature on slave core sensor voltage (internal + * ADC): mV + */ +#define MC_CMD_SENSOR_CONTROLLER_SLAVE_VPTAT 0x44 +/* enum: controller internal temperature on slave core (internal ADC): degC */ +#define MC_CMD_SENSOR_CONTROLLER_SLAVE_INTERNAL_TEMP 0x45 +/* enum: controller internal temperature on slave core sensor voltage (external + * ADC): mV + */ +#define MC_CMD_SENSOR_CONTROLLER_SLAVE_VPTAT_EXTADC 0x46 +/* enum: controller internal temperature on slave core (external ADC): degC */ +#define MC_CMD_SENSOR_CONTROLLER_SLAVE_INTERNAL_TEMP_EXTADC 0x47 +/* enum: Voltage supplied to the SODIMMs from their power supply: mV */ +#define MC_CMD_SENSOR_SODIMM_VOUT 0x49 +/* enum: Temperature of SODIMM 0 (if installed): degC */ +#define MC_CMD_SENSOR_SODIMM_0_TEMP 0x4a +/* enum: Temperature of SODIMM 1 (if installed): degC */ +#define MC_CMD_SENSOR_SODIMM_1_TEMP 0x4b +/* enum: Voltage supplied to the QSFP #0 from their power supply: mV */ +#define MC_CMD_SENSOR_PHY0_VCC 0x4c +/* enum: Voltage supplied to the QSFP #1 from their power supply: mV */ +#define MC_CMD_SENSOR_PHY1_VCC 0x4d +/* enum: Controller die temperature (TDIODE): degC */ +#define MC_CMD_SENSOR_CONTROLLER_TDIODE_TEMP 0x4e +/* enum: Board temperature (front): degC */ +#define MC_CMD_SENSOR_BOARD_FRONT_TEMP 0x4f +/* enum: Board temperature (back): degC */ +#define MC_CMD_SENSOR_BOARD_BACK_TEMP 0x50 +/* MC_CMD_SENSOR_INFO_ENTRY_TYPEDEF */ +#define MC_CMD_SENSOR_ENTRY_OFST 4 +#define MC_CMD_SENSOR_ENTRY_LEN 8 +#define MC_CMD_SENSOR_ENTRY_LO_OFST 4 +#define MC_CMD_SENSOR_ENTRY_HI_OFST 8 +#define MC_CMD_SENSOR_ENTRY_MINNUM 0 +#define MC_CMD_SENSOR_ENTRY_MAXNUM 31 + +/* MC_CMD_SENSOR_INFO_EXT_OUT msgresponse */ +#define MC_CMD_SENSOR_INFO_EXT_OUT_LENMIN 4 +#define MC_CMD_SENSOR_INFO_EXT_OUT_LENMAX 252 +#define MC_CMD_SENSOR_INFO_EXT_OUT_LEN(num) (4+8*(num)) +#define MC_CMD_SENSOR_INFO_EXT_OUT_MASK_OFST 0 +/* Enum values, see field(s): */ +/* MC_CMD_SENSOR_INFO_OUT */ +#define MC_CMD_SENSOR_INFO_EXT_OUT_NEXT_PAGE_LBN 31 +#define MC_CMD_SENSOR_INFO_EXT_OUT_NEXT_PAGE_WIDTH 1 +/* MC_CMD_SENSOR_INFO_ENTRY_TYPEDEF */ +/* MC_CMD_SENSOR_ENTRY_OFST 4 */ +/* MC_CMD_SENSOR_ENTRY_LEN 8 */ +/* MC_CMD_SENSOR_ENTRY_LO_OFST 4 */ +/* MC_CMD_SENSOR_ENTRY_HI_OFST 8 */ +/* MC_CMD_SENSOR_ENTRY_MINNUM 0 */ +/* MC_CMD_SENSOR_ENTRY_MAXNUM 31 */ + +/* MC_CMD_SENSOR_INFO_ENTRY_TYPEDEF structuredef */ +#define MC_CMD_SENSOR_INFO_ENTRY_TYPEDEF_LEN 8 +#define MC_CMD_SENSOR_INFO_ENTRY_TYPEDEF_MIN1_OFST 0 +#define MC_CMD_SENSOR_INFO_ENTRY_TYPEDEF_MIN1_LEN 2 +#define MC_CMD_SENSOR_INFO_ENTRY_TYPEDEF_MIN1_LBN 0 +#define MC_CMD_SENSOR_INFO_ENTRY_TYPEDEF_MIN1_WIDTH 16 +#define MC_CMD_SENSOR_INFO_ENTRY_TYPEDEF_MAX1_OFST 2 +#define MC_CMD_SENSOR_INFO_ENTRY_TYPEDEF_MAX1_LEN 2 +#define MC_CMD_SENSOR_INFO_ENTRY_TYPEDEF_MAX1_LBN 16 +#define MC_CMD_SENSOR_INFO_ENTRY_TYPEDEF_MAX1_WIDTH 16 +#define MC_CMD_SENSOR_INFO_ENTRY_TYPEDEF_MIN2_OFST 4 +#define MC_CMD_SENSOR_INFO_ENTRY_TYPEDEF_MIN2_LEN 2 +#define MC_CMD_SENSOR_INFO_ENTRY_TYPEDEF_MIN2_LBN 32 +#define MC_CMD_SENSOR_INFO_ENTRY_TYPEDEF_MIN2_WIDTH 16 +#define MC_CMD_SENSOR_INFO_ENTRY_TYPEDEF_MAX2_OFST 6 +#define MC_CMD_SENSOR_INFO_ENTRY_TYPEDEF_MAX2_LEN 2 +#define MC_CMD_SENSOR_INFO_ENTRY_TYPEDEF_MAX2_LBN 48 +#define MC_CMD_SENSOR_INFO_ENTRY_TYPEDEF_MAX2_WIDTH 16 + + +/***********************************/ +/* MC_CMD_READ_SENSORS + * Returns the current reading from each sensor. DMAs an array of sensor + * readings, in order of sensor type (but without gaps for unimplemented + * sensors), into host memory. Each array element is a + * MC_CMD_SENSOR_VALUE_ENTRY_TYPEDEF dword. + * + * If the request does not contain the LENGTH field then only sensors 0 to 30 + * are reported, to avoid DMA buffer overflow in older host software. If the + * sensor reading require more space than the LENGTH allows, then return + * EINVAL. + * + * The MC will send a SENSOREVT event every time any sensor changes state. The + * driver is responsible for ensuring that it doesn't miss any events. The + * board will function normally if all sensors are in STATE_OK or + * STATE_WARNING. Otherwise the board should not be expected to function. + */ +#define MC_CMD_READ_SENSORS 0x42 +#undef MC_CMD_0x42_PRIVILEGE_CTG + +#define MC_CMD_0x42_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_READ_SENSORS_IN msgrequest */ +#define MC_CMD_READ_SENSORS_IN_LEN 8 +/* DMA address of host buffer for sensor readings (must be 4Kbyte aligned). */ +#define MC_CMD_READ_SENSORS_IN_DMA_ADDR_OFST 0 +#define MC_CMD_READ_SENSORS_IN_DMA_ADDR_LEN 8 +#define MC_CMD_READ_SENSORS_IN_DMA_ADDR_LO_OFST 0 +#define MC_CMD_READ_SENSORS_IN_DMA_ADDR_HI_OFST 4 + +/* MC_CMD_READ_SENSORS_EXT_IN msgrequest */ +#define MC_CMD_READ_SENSORS_EXT_IN_LEN 12 +/* DMA address of host buffer for sensor readings (must be 4Kbyte aligned). */ +#define MC_CMD_READ_SENSORS_EXT_IN_DMA_ADDR_OFST 0 +#define MC_CMD_READ_SENSORS_EXT_IN_DMA_ADDR_LEN 8 +#define MC_CMD_READ_SENSORS_EXT_IN_DMA_ADDR_LO_OFST 0 +#define MC_CMD_READ_SENSORS_EXT_IN_DMA_ADDR_HI_OFST 4 +/* Size in bytes of host buffer. */ +#define MC_CMD_READ_SENSORS_EXT_IN_LENGTH_OFST 8 + +/* MC_CMD_READ_SENSORS_OUT msgresponse */ +#define MC_CMD_READ_SENSORS_OUT_LEN 0 + +/* MC_CMD_READ_SENSORS_EXT_OUT msgresponse */ +#define MC_CMD_READ_SENSORS_EXT_OUT_LEN 0 + +/* MC_CMD_SENSOR_VALUE_ENTRY_TYPEDEF structuredef */ +#define MC_CMD_SENSOR_VALUE_ENTRY_TYPEDEF_LEN 4 +#define MC_CMD_SENSOR_VALUE_ENTRY_TYPEDEF_VALUE_OFST 0 +#define MC_CMD_SENSOR_VALUE_ENTRY_TYPEDEF_VALUE_LEN 2 +#define MC_CMD_SENSOR_VALUE_ENTRY_TYPEDEF_VALUE_LBN 0 +#define MC_CMD_SENSOR_VALUE_ENTRY_TYPEDEF_VALUE_WIDTH 16 +#define MC_CMD_SENSOR_VALUE_ENTRY_TYPEDEF_STATE_OFST 2 +#define MC_CMD_SENSOR_VALUE_ENTRY_TYPEDEF_STATE_LEN 1 +/* enum: Ok. */ +#define MC_CMD_SENSOR_STATE_OK 0x0 +/* enum: Breached warning threshold. */ +#define MC_CMD_SENSOR_STATE_WARNING 0x1 +/* enum: Breached fatal threshold. */ +#define MC_CMD_SENSOR_STATE_FATAL 0x2 +/* enum: Fault with sensor. */ +#define MC_CMD_SENSOR_STATE_BROKEN 0x3 +/* enum: Sensor is working but does not currently have a reading. */ +#define MC_CMD_SENSOR_STATE_NO_READING 0x4 +/* enum: Sensor initialisation failed. */ +#define MC_CMD_SENSOR_STATE_INIT_FAILED 0x5 +#define MC_CMD_SENSOR_VALUE_ENTRY_TYPEDEF_STATE_LBN 16 +#define MC_CMD_SENSOR_VALUE_ENTRY_TYPEDEF_STATE_WIDTH 8 +#define MC_CMD_SENSOR_VALUE_ENTRY_TYPEDEF_TYPE_OFST 3 +#define MC_CMD_SENSOR_VALUE_ENTRY_TYPEDEF_TYPE_LEN 1 +/* Enum values, see field(s): */ +/* MC_CMD_SENSOR_INFO/MC_CMD_SENSOR_INFO_OUT/MASK */ +#define MC_CMD_SENSOR_VALUE_ENTRY_TYPEDEF_TYPE_LBN 24 +#define MC_CMD_SENSOR_VALUE_ENTRY_TYPEDEF_TYPE_WIDTH 8 + + +/***********************************/ +/* MC_CMD_GET_PHY_STATE + * Report current state of PHY. A 'zombie' PHY is a PHY that has failed to boot + * (e.g. due to missing or corrupted firmware). Locks required: None. Return + * code: 0 + */ +#define MC_CMD_GET_PHY_STATE 0x43 +#undef MC_CMD_0x43_PRIVILEGE_CTG + +#define MC_CMD_0x43_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_GET_PHY_STATE_IN msgrequest */ +#define MC_CMD_GET_PHY_STATE_IN_LEN 0 + +/* MC_CMD_GET_PHY_STATE_OUT msgresponse */ +#define MC_CMD_GET_PHY_STATE_OUT_LEN 4 +#define MC_CMD_GET_PHY_STATE_OUT_STATE_OFST 0 +/* enum: Ok. */ +#define MC_CMD_PHY_STATE_OK 0x1 +/* enum: Faulty. */ +#define MC_CMD_PHY_STATE_ZOMBIE 0x2 + + +/***********************************/ +/* MC_CMD_SETUP_8021QBB + * 802.1Qbb control. 8 Tx queues that map to priorities 0 - 7. Use all 1s to + * disable 802.Qbb for a given priority. + */ +#define MC_CMD_SETUP_8021QBB 0x44 + +/* MC_CMD_SETUP_8021QBB_IN msgrequest */ +#define MC_CMD_SETUP_8021QBB_IN_LEN 32 +#define MC_CMD_SETUP_8021QBB_IN_TXQS_OFST 0 +#define MC_CMD_SETUP_8021QBB_IN_TXQS_LEN 32 + +/* MC_CMD_SETUP_8021QBB_OUT msgresponse */ +#define MC_CMD_SETUP_8021QBB_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_WOL_FILTER_GET + * Retrieve ID of any WoL filters. Locks required: None. Returns: 0, ENOSYS + */ +#define MC_CMD_WOL_FILTER_GET 0x45 +#undef MC_CMD_0x45_PRIVILEGE_CTG + +#define MC_CMD_0x45_PRIVILEGE_CTG SRIOV_CTG_LINK + +/* MC_CMD_WOL_FILTER_GET_IN msgrequest */ +#define MC_CMD_WOL_FILTER_GET_IN_LEN 0 + +/* MC_CMD_WOL_FILTER_GET_OUT msgresponse */ +#define MC_CMD_WOL_FILTER_GET_OUT_LEN 4 +#define MC_CMD_WOL_FILTER_GET_OUT_FILTER_ID_OFST 0 + + +/***********************************/ +/* MC_CMD_ADD_LIGHTSOUT_OFFLOAD + * Add a protocol offload to NIC for lights-out state. Locks required: None. + * Returns: 0, ENOSYS + */ +#define MC_CMD_ADD_LIGHTSOUT_OFFLOAD 0x46 +#undef MC_CMD_0x46_PRIVILEGE_CTG + +#define MC_CMD_0x46_PRIVILEGE_CTG SRIOV_CTG_LINK + +/* MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN msgrequest */ +#define MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_LENMIN 8 +#define MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_LENMAX 252 +#define MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_LEN(num) (4+4*(num)) +#define MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_PROTOCOL_OFST 0 +#define MC_CMD_LIGHTSOUT_OFFLOAD_PROTOCOL_ARP 0x1 /* enum */ +#define MC_CMD_LIGHTSOUT_OFFLOAD_PROTOCOL_NS 0x2 /* enum */ +#define MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_DATA_OFST 4 +#define MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_DATA_LEN 4 +#define MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_DATA_MINNUM 1 +#define MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_DATA_MAXNUM 62 + +/* MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_ARP msgrequest */ +#define MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_ARP_LEN 14 +/* MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_PROTOCOL_OFST 0 */ +#define MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_ARP_MAC_OFST 4 +#define MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_ARP_MAC_LEN 6 +#define MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_ARP_IP_OFST 10 + +/* MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_NS msgrequest */ +#define MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_NS_LEN 42 +/* MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_PROTOCOL_OFST 0 */ +#define MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_NS_MAC_OFST 4 +#define MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_NS_MAC_LEN 6 +#define MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_NS_SNIPV6_OFST 10 +#define MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_NS_SNIPV6_LEN 16 +#define MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_NS_IPV6_OFST 26 +#define MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_NS_IPV6_LEN 16 + +/* MC_CMD_ADD_LIGHTSOUT_OFFLOAD_OUT msgresponse */ +#define MC_CMD_ADD_LIGHTSOUT_OFFLOAD_OUT_LEN 4 +#define MC_CMD_ADD_LIGHTSOUT_OFFLOAD_OUT_FILTER_ID_OFST 0 + + +/***********************************/ +/* MC_CMD_REMOVE_LIGHTSOUT_OFFLOAD + * Remove a protocol offload from NIC for lights-out state. Locks required: + * None. Returns: 0, ENOSYS + */ +#define MC_CMD_REMOVE_LIGHTSOUT_OFFLOAD 0x47 +#undef MC_CMD_0x47_PRIVILEGE_CTG + +#define MC_CMD_0x47_PRIVILEGE_CTG SRIOV_CTG_LINK + +/* MC_CMD_REMOVE_LIGHTSOUT_OFFLOAD_IN msgrequest */ +#define MC_CMD_REMOVE_LIGHTSOUT_OFFLOAD_IN_LEN 8 +#define MC_CMD_REMOVE_LIGHTSOUT_OFFLOAD_IN_PROTOCOL_OFST 0 +#define MC_CMD_REMOVE_LIGHTSOUT_OFFLOAD_IN_FILTER_ID_OFST 4 + +/* MC_CMD_REMOVE_LIGHTSOUT_OFFLOAD_OUT msgresponse */ +#define MC_CMD_REMOVE_LIGHTSOUT_OFFLOAD_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_MAC_RESET_RESTORE + * Restore MAC after block reset. Locks required: None. Returns: 0. + */ +#define MC_CMD_MAC_RESET_RESTORE 0x48 + +/* MC_CMD_MAC_RESET_RESTORE_IN msgrequest */ +#define MC_CMD_MAC_RESET_RESTORE_IN_LEN 0 + +/* MC_CMD_MAC_RESET_RESTORE_OUT msgresponse */ +#define MC_CMD_MAC_RESET_RESTORE_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_TESTASSERT + * Deliberately trigger an assert-detonation in the firmware for testing + * purposes (i.e. to allow tests that the driver copes gracefully). Locks + * required: None Returns: 0 + */ +#define MC_CMD_TESTASSERT 0x49 +#undef MC_CMD_0x49_PRIVILEGE_CTG + +#define MC_CMD_0x49_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_TESTASSERT_IN msgrequest */ +#define MC_CMD_TESTASSERT_IN_LEN 0 + +/* MC_CMD_TESTASSERT_OUT msgresponse */ +#define MC_CMD_TESTASSERT_OUT_LEN 0 + +/* MC_CMD_TESTASSERT_V2_IN msgrequest */ +#define MC_CMD_TESTASSERT_V2_IN_LEN 4 +/* How to provoke the assertion */ +#define MC_CMD_TESTASSERT_V2_IN_TYPE_OFST 0 +/* enum: Assert using the FAIL_ASSERTION_WITH_USEFUL_VALUES macro. Unless + * you're testing firmware, this is what you want. + */ +#define MC_CMD_TESTASSERT_V2_IN_FAIL_ASSERTION_WITH_USEFUL_VALUES 0x0 +/* enum: Assert using assert(0); */ +#define MC_CMD_TESTASSERT_V2_IN_ASSERT_FALSE 0x1 +/* enum: Deliberately trigger a watchdog */ +#define MC_CMD_TESTASSERT_V2_IN_WATCHDOG 0x2 +/* enum: Deliberately trigger a trap by loading from an invalid address */ +#define MC_CMD_TESTASSERT_V2_IN_LOAD_TRAP 0x3 +/* enum: Deliberately trigger a trap by storing to an invalid address */ +#define MC_CMD_TESTASSERT_V2_IN_STORE_TRAP 0x4 +/* enum: Jump to an invalid address */ +#define MC_CMD_TESTASSERT_V2_IN_JUMP_TRAP 0x5 + +/* MC_CMD_TESTASSERT_V2_OUT msgresponse */ +#define MC_CMD_TESTASSERT_V2_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_WORKAROUND + * Enable/Disable a given workaround. The mcfw will return EINVAL if it doesn't + * understand the given workaround number - which should not be treated as a + * hard error by client code. This op does not imply any semantics about each + * workaround, that's between the driver and the mcfw on a per-workaround + * basis. Locks required: None. Returns: 0, EINVAL . + */ +#define MC_CMD_WORKAROUND 0x4a +#undef MC_CMD_0x4a_PRIVILEGE_CTG + +#define MC_CMD_0x4a_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_WORKAROUND_IN msgrequest */ +#define MC_CMD_WORKAROUND_IN_LEN 8 +/* The enums here must correspond with those in MC_CMD_GET_WORKAROUND. */ +#define MC_CMD_WORKAROUND_IN_TYPE_OFST 0 +/* enum: Bug 17230 work around. */ +#define MC_CMD_WORKAROUND_BUG17230 0x1 +/* enum: Bug 35388 work around (unsafe EVQ writes). */ +#define MC_CMD_WORKAROUND_BUG35388 0x2 +/* enum: Bug35017 workaround (A64 tables must be identity map) */ +#define MC_CMD_WORKAROUND_BUG35017 0x3 +/* enum: Bug 41750 present (MC_CMD_TRIGGER_INTERRUPT won't work) */ +#define MC_CMD_WORKAROUND_BUG41750 0x4 +/* enum: Bug 42008 present (Interrupts can overtake associated events). Caution + * - before adding code that queries this workaround, remember that there's + * released Monza firmware that doesn't understand MC_CMD_WORKAROUND_BUG42008, + * and will hence (incorrectly) report that the bug doesn't exist. + */ +#define MC_CMD_WORKAROUND_BUG42008 0x5 +/* enum: Bug 26807 features present in firmware (multicast filter chaining) + * This feature cannot be turned on/off while there are any filters already + * present. The behaviour in such case depends on the acting client's privilege + * level. If the client has the admin privilege, then all functions that have + * filters installed will be FLRed and the FLR_DONE flag will be set. Otherwise + * the command will fail with MC_CMD_ERR_FILTERS_PRESENT. + */ +#define MC_CMD_WORKAROUND_BUG26807 0x6 +/* enum: Bug 61265 work around (broken EVQ TMR writes). */ +#define MC_CMD_WORKAROUND_BUG61265 0x7 +/* 0 = disable the workaround indicated by TYPE; any non-zero value = enable + * the workaround + */ +#define MC_CMD_WORKAROUND_IN_ENABLED_OFST 4 + +/* MC_CMD_WORKAROUND_OUT msgresponse */ +#define MC_CMD_WORKAROUND_OUT_LEN 0 + +/* MC_CMD_WORKAROUND_EXT_OUT msgresponse: This response format will be used + * when (TYPE == MC_CMD_WORKAROUND_BUG26807) + */ +#define MC_CMD_WORKAROUND_EXT_OUT_LEN 4 +#define MC_CMD_WORKAROUND_EXT_OUT_FLAGS_OFST 0 +#define MC_CMD_WORKAROUND_EXT_OUT_FLR_DONE_LBN 0 +#define MC_CMD_WORKAROUND_EXT_OUT_FLR_DONE_WIDTH 1 + + +/***********************************/ +/* MC_CMD_GET_PHY_MEDIA_INFO + * Read media-specific data from PHY (e.g. SFP/SFP+ module ID information for + * SFP+ PHYs). The 'media type' can be found via GET_PHY_CFG + * (GET_PHY_CFG_OUT_MEDIA_TYPE); the valid 'page number' input values, and the + * output data, are interpreted on a per-type basis. For SFP+: PAGE=0 or 1 + * returns a 128-byte block read from module I2C address 0xA0 offset 0 or 0x80. + * Anything else: currently undefined. Locks required: None. Return code: 0. + */ +#define MC_CMD_GET_PHY_MEDIA_INFO 0x4b +#undef MC_CMD_0x4b_PRIVILEGE_CTG + +#define MC_CMD_0x4b_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_GET_PHY_MEDIA_INFO_IN msgrequest */ +#define MC_CMD_GET_PHY_MEDIA_INFO_IN_LEN 4 +#define MC_CMD_GET_PHY_MEDIA_INFO_IN_PAGE_OFST 0 + +/* MC_CMD_GET_PHY_MEDIA_INFO_OUT msgresponse */ +#define MC_CMD_GET_PHY_MEDIA_INFO_OUT_LENMIN 5 +#define MC_CMD_GET_PHY_MEDIA_INFO_OUT_LENMAX 252 +#define MC_CMD_GET_PHY_MEDIA_INFO_OUT_LEN(num) (4+1*(num)) +/* in bytes */ +#define MC_CMD_GET_PHY_MEDIA_INFO_OUT_DATALEN_OFST 0 +#define MC_CMD_GET_PHY_MEDIA_INFO_OUT_DATA_OFST 4 +#define MC_CMD_GET_PHY_MEDIA_INFO_OUT_DATA_LEN 1 +#define MC_CMD_GET_PHY_MEDIA_INFO_OUT_DATA_MINNUM 1 +#define MC_CMD_GET_PHY_MEDIA_INFO_OUT_DATA_MAXNUM 248 + + +/***********************************/ +/* MC_CMD_NVRAM_TEST + * Test a particular NVRAM partition for valid contents (where "valid" depends + * on the type of partition). + */ +#define MC_CMD_NVRAM_TEST 0x4c +#undef MC_CMD_0x4c_PRIVILEGE_CTG + +#define MC_CMD_0x4c_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_NVRAM_TEST_IN msgrequest */ +#define MC_CMD_NVRAM_TEST_IN_LEN 4 +#define MC_CMD_NVRAM_TEST_IN_TYPE_OFST 0 +/* Enum values, see field(s): */ +/* MC_CMD_NVRAM_TYPES/MC_CMD_NVRAM_TYPES_OUT/TYPES */ + +/* MC_CMD_NVRAM_TEST_OUT msgresponse */ +#define MC_CMD_NVRAM_TEST_OUT_LEN 4 +#define MC_CMD_NVRAM_TEST_OUT_RESULT_OFST 0 +/* enum: Passed. */ +#define MC_CMD_NVRAM_TEST_PASS 0x0 +/* enum: Failed. */ +#define MC_CMD_NVRAM_TEST_FAIL 0x1 +/* enum: Not supported. */ +#define MC_CMD_NVRAM_TEST_NOTSUPP 0x2 + + +/***********************************/ +/* MC_CMD_MRSFP_TWEAK + * Read status and/or set parameters for the 'mrsfp' driver in mr_rusty builds. + * I2C I/O expander bits are always read; if equaliser parameters are supplied, + * they are configured first. Locks required: None. Return code: 0, EINVAL. + */ +#define MC_CMD_MRSFP_TWEAK 0x4d + +/* MC_CMD_MRSFP_TWEAK_IN_EQ_CONFIG msgrequest */ +#define MC_CMD_MRSFP_TWEAK_IN_EQ_CONFIG_LEN 16 +/* 0-6 low->high de-emph. */ +#define MC_CMD_MRSFP_TWEAK_IN_EQ_CONFIG_TXEQ_LEVEL_OFST 0 +/* 0-8 low->high ref.V */ +#define MC_CMD_MRSFP_TWEAK_IN_EQ_CONFIG_TXEQ_DT_CFG_OFST 4 +/* 0-8 0-8 low->high boost */ +#define MC_CMD_MRSFP_TWEAK_IN_EQ_CONFIG_RXEQ_BOOST_OFST 8 +/* 0-8 low->high ref.V */ +#define MC_CMD_MRSFP_TWEAK_IN_EQ_CONFIG_RXEQ_DT_CFG_OFST 12 + +/* MC_CMD_MRSFP_TWEAK_IN_READ_ONLY msgrequest */ +#define MC_CMD_MRSFP_TWEAK_IN_READ_ONLY_LEN 0 + +/* MC_CMD_MRSFP_TWEAK_OUT msgresponse */ +#define MC_CMD_MRSFP_TWEAK_OUT_LEN 12 +/* input bits */ +#define MC_CMD_MRSFP_TWEAK_OUT_IOEXP_INPUTS_OFST 0 +/* output bits */ +#define MC_CMD_MRSFP_TWEAK_OUT_IOEXP_OUTPUTS_OFST 4 +/* direction */ +#define MC_CMD_MRSFP_TWEAK_OUT_IOEXP_DIRECTION_OFST 8 +/* enum: Out. */ +#define MC_CMD_MRSFP_TWEAK_OUT_IOEXP_DIRECTION_OUT 0x0 +/* enum: In. */ +#define MC_CMD_MRSFP_TWEAK_OUT_IOEXP_DIRECTION_IN 0x1 + + +/***********************************/ +/* MC_CMD_SENSOR_SET_LIMS + * Adjusts the sensor limits. This is a warranty-voiding operation. Returns: + * ENOENT if the sensor specified does not exist, EINVAL if the limits are out + * of range. + */ +#define MC_CMD_SENSOR_SET_LIMS 0x4e +#undef MC_CMD_0x4e_PRIVILEGE_CTG + +#define MC_CMD_0x4e_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_SENSOR_SET_LIMS_IN msgrequest */ +#define MC_CMD_SENSOR_SET_LIMS_IN_LEN 20 +#define MC_CMD_SENSOR_SET_LIMS_IN_SENSOR_OFST 0 +/* Enum values, see field(s): */ +/* MC_CMD_SENSOR_INFO/MC_CMD_SENSOR_INFO_OUT/MASK */ +/* interpretation is is sensor-specific. */ +#define MC_CMD_SENSOR_SET_LIMS_IN_LOW0_OFST 4 +/* interpretation is is sensor-specific. */ +#define MC_CMD_SENSOR_SET_LIMS_IN_HI0_OFST 8 +/* interpretation is is sensor-specific. */ +#define MC_CMD_SENSOR_SET_LIMS_IN_LOW1_OFST 12 +/* interpretation is is sensor-specific. */ +#define MC_CMD_SENSOR_SET_LIMS_IN_HI1_OFST 16 + +/* MC_CMD_SENSOR_SET_LIMS_OUT msgresponse */ +#define MC_CMD_SENSOR_SET_LIMS_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_GET_RESOURCE_LIMITS + */ +#define MC_CMD_GET_RESOURCE_LIMITS 0x4f + +/* MC_CMD_GET_RESOURCE_LIMITS_IN msgrequest */ +#define MC_CMD_GET_RESOURCE_LIMITS_IN_LEN 0 + +/* MC_CMD_GET_RESOURCE_LIMITS_OUT msgresponse */ +#define MC_CMD_GET_RESOURCE_LIMITS_OUT_LEN 16 +#define MC_CMD_GET_RESOURCE_LIMITS_OUT_BUFTBL_OFST 0 +#define MC_CMD_GET_RESOURCE_LIMITS_OUT_EVQ_OFST 4 +#define MC_CMD_GET_RESOURCE_LIMITS_OUT_RXQ_OFST 8 +#define MC_CMD_GET_RESOURCE_LIMITS_OUT_TXQ_OFST 12 + + +/***********************************/ +/* MC_CMD_NVRAM_PARTITIONS + * Reads the list of available virtual NVRAM partition types. Locks required: + * none. Returns: 0, EINVAL (bad type). + */ +#define MC_CMD_NVRAM_PARTITIONS 0x51 +#undef MC_CMD_0x51_PRIVILEGE_CTG + +#define MC_CMD_0x51_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_NVRAM_PARTITIONS_IN msgrequest */ +#define MC_CMD_NVRAM_PARTITIONS_IN_LEN 0 + +/* MC_CMD_NVRAM_PARTITIONS_OUT msgresponse */ +#define MC_CMD_NVRAM_PARTITIONS_OUT_LENMIN 4 +#define MC_CMD_NVRAM_PARTITIONS_OUT_LENMAX 252 +#define MC_CMD_NVRAM_PARTITIONS_OUT_LEN(num) (4+4*(num)) +/* total number of partitions */ +#define MC_CMD_NVRAM_PARTITIONS_OUT_NUM_PARTITIONS_OFST 0 +/* type ID code for each of NUM_PARTITIONS partitions */ +#define MC_CMD_NVRAM_PARTITIONS_OUT_TYPE_ID_OFST 4 +#define MC_CMD_NVRAM_PARTITIONS_OUT_TYPE_ID_LEN 4 +#define MC_CMD_NVRAM_PARTITIONS_OUT_TYPE_ID_MINNUM 0 +#define MC_CMD_NVRAM_PARTITIONS_OUT_TYPE_ID_MAXNUM 62 + + +/***********************************/ +/* MC_CMD_NVRAM_METADATA + * Reads soft metadata for a virtual NVRAM partition type. Locks required: + * none. Returns: 0, EINVAL (bad type). + */ +#define MC_CMD_NVRAM_METADATA 0x52 +#undef MC_CMD_0x52_PRIVILEGE_CTG + +#define MC_CMD_0x52_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_NVRAM_METADATA_IN msgrequest */ +#define MC_CMD_NVRAM_METADATA_IN_LEN 4 +/* Partition type ID code */ +#define MC_CMD_NVRAM_METADATA_IN_TYPE_OFST 0 + +/* MC_CMD_NVRAM_METADATA_OUT msgresponse */ +#define MC_CMD_NVRAM_METADATA_OUT_LENMIN 20 +#define MC_CMD_NVRAM_METADATA_OUT_LENMAX 252 +#define MC_CMD_NVRAM_METADATA_OUT_LEN(num) (20+1*(num)) +/* Partition type ID code */ +#define MC_CMD_NVRAM_METADATA_OUT_TYPE_OFST 0 +#define MC_CMD_NVRAM_METADATA_OUT_FLAGS_OFST 4 +#define MC_CMD_NVRAM_METADATA_OUT_SUBTYPE_VALID_LBN 0 +#define MC_CMD_NVRAM_METADATA_OUT_SUBTYPE_VALID_WIDTH 1 +#define MC_CMD_NVRAM_METADATA_OUT_VERSION_VALID_LBN 1 +#define MC_CMD_NVRAM_METADATA_OUT_VERSION_VALID_WIDTH 1 +#define MC_CMD_NVRAM_METADATA_OUT_DESCRIPTION_VALID_LBN 2 +#define MC_CMD_NVRAM_METADATA_OUT_DESCRIPTION_VALID_WIDTH 1 +/* Subtype ID code for content of this partition */ +#define MC_CMD_NVRAM_METADATA_OUT_SUBTYPE_OFST 8 +/* 1st component of W.X.Y.Z version number for content of this partition */ +#define MC_CMD_NVRAM_METADATA_OUT_VERSION_W_OFST 12 +#define MC_CMD_NVRAM_METADATA_OUT_VERSION_W_LEN 2 +/* 2nd component of W.X.Y.Z version number for content of this partition */ +#define MC_CMD_NVRAM_METADATA_OUT_VERSION_X_OFST 14 +#define MC_CMD_NVRAM_METADATA_OUT_VERSION_X_LEN 2 +/* 3rd component of W.X.Y.Z version number for content of this partition */ +#define MC_CMD_NVRAM_METADATA_OUT_VERSION_Y_OFST 16 +#define MC_CMD_NVRAM_METADATA_OUT_VERSION_Y_LEN 2 +/* 4th component of W.X.Y.Z version number for content of this partition */ +#define MC_CMD_NVRAM_METADATA_OUT_VERSION_Z_OFST 18 +#define MC_CMD_NVRAM_METADATA_OUT_VERSION_Z_LEN 2 +/* Zero-terminated string describing the content of this partition */ +#define MC_CMD_NVRAM_METADATA_OUT_DESCRIPTION_OFST 20 +#define MC_CMD_NVRAM_METADATA_OUT_DESCRIPTION_LEN 1 +#define MC_CMD_NVRAM_METADATA_OUT_DESCRIPTION_MINNUM 0 +#define MC_CMD_NVRAM_METADATA_OUT_DESCRIPTION_MAXNUM 232 + + +/***********************************/ +/* MC_CMD_GET_MAC_ADDRESSES + * Returns the base MAC, count and stride for the requesting function + */ +#define MC_CMD_GET_MAC_ADDRESSES 0x55 +#undef MC_CMD_0x55_PRIVILEGE_CTG + +#define MC_CMD_0x55_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_GET_MAC_ADDRESSES_IN msgrequest */ +#define MC_CMD_GET_MAC_ADDRESSES_IN_LEN 0 + +/* MC_CMD_GET_MAC_ADDRESSES_OUT msgresponse */ +#define MC_CMD_GET_MAC_ADDRESSES_OUT_LEN 16 +/* Base MAC address */ +#define MC_CMD_GET_MAC_ADDRESSES_OUT_MAC_ADDR_BASE_OFST 0 +#define MC_CMD_GET_MAC_ADDRESSES_OUT_MAC_ADDR_BASE_LEN 6 +/* Padding */ +#define MC_CMD_GET_MAC_ADDRESSES_OUT_RESERVED_OFST 6 +#define MC_CMD_GET_MAC_ADDRESSES_OUT_RESERVED_LEN 2 +/* Number of allocated MAC addresses */ +#define MC_CMD_GET_MAC_ADDRESSES_OUT_MAC_COUNT_OFST 8 +/* Spacing of allocated MAC addresses */ +#define MC_CMD_GET_MAC_ADDRESSES_OUT_MAC_STRIDE_OFST 12 + + +/***********************************/ +/* MC_CMD_CLP + * Perform a CLP related operation + */ +#define MC_CMD_CLP 0x56 +#undef MC_CMD_0x56_PRIVILEGE_CTG + +#define MC_CMD_0x56_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_CLP_IN msgrequest */ +#define MC_CMD_CLP_IN_LEN 4 +/* Sub operation */ +#define MC_CMD_CLP_IN_OP_OFST 0 +/* enum: Return to factory default settings */ +#define MC_CMD_CLP_OP_DEFAULT 0x1 +/* enum: Set MAC address */ +#define MC_CMD_CLP_OP_SET_MAC 0x2 +/* enum: Get MAC address */ +#define MC_CMD_CLP_OP_GET_MAC 0x3 +/* enum: Set UEFI/GPXE boot mode */ +#define MC_CMD_CLP_OP_SET_BOOT 0x4 +/* enum: Get UEFI/GPXE boot mode */ +#define MC_CMD_CLP_OP_GET_BOOT 0x5 + +/* MC_CMD_CLP_OUT msgresponse */ +#define MC_CMD_CLP_OUT_LEN 0 + +/* MC_CMD_CLP_IN_DEFAULT msgrequest */ +#define MC_CMD_CLP_IN_DEFAULT_LEN 4 +/* MC_CMD_CLP_IN_OP_OFST 0 */ + +/* MC_CMD_CLP_OUT_DEFAULT msgresponse */ +#define MC_CMD_CLP_OUT_DEFAULT_LEN 0 + +/* MC_CMD_CLP_IN_SET_MAC msgrequest */ +#define MC_CMD_CLP_IN_SET_MAC_LEN 12 +/* MC_CMD_CLP_IN_OP_OFST 0 */ +/* MAC address assigned to port */ +#define MC_CMD_CLP_IN_SET_MAC_ADDR_OFST 4 +#define MC_CMD_CLP_IN_SET_MAC_ADDR_LEN 6 +/* Padding */ +#define MC_CMD_CLP_IN_SET_MAC_RESERVED_OFST 10 +#define MC_CMD_CLP_IN_SET_MAC_RESERVED_LEN 2 + +/* MC_CMD_CLP_OUT_SET_MAC msgresponse */ +#define MC_CMD_CLP_OUT_SET_MAC_LEN 0 + +/* MC_CMD_CLP_IN_GET_MAC msgrequest */ +#define MC_CMD_CLP_IN_GET_MAC_LEN 4 +/* MC_CMD_CLP_IN_OP_OFST 0 */ + +/* MC_CMD_CLP_OUT_GET_MAC msgresponse */ +#define MC_CMD_CLP_OUT_GET_MAC_LEN 8 +/* MAC address assigned to port */ +#define MC_CMD_CLP_OUT_GET_MAC_ADDR_OFST 0 +#define MC_CMD_CLP_OUT_GET_MAC_ADDR_LEN 6 +/* Padding */ +#define MC_CMD_CLP_OUT_GET_MAC_RESERVED_OFST 6 +#define MC_CMD_CLP_OUT_GET_MAC_RESERVED_LEN 2 + +/* MC_CMD_CLP_IN_SET_BOOT msgrequest */ +#define MC_CMD_CLP_IN_SET_BOOT_LEN 5 +/* MC_CMD_CLP_IN_OP_OFST 0 */ +/* Boot flag */ +#define MC_CMD_CLP_IN_SET_BOOT_FLAG_OFST 4 +#define MC_CMD_CLP_IN_SET_BOOT_FLAG_LEN 1 + +/* MC_CMD_CLP_OUT_SET_BOOT msgresponse */ +#define MC_CMD_CLP_OUT_SET_BOOT_LEN 0 + +/* MC_CMD_CLP_IN_GET_BOOT msgrequest */ +#define MC_CMD_CLP_IN_GET_BOOT_LEN 4 +/* MC_CMD_CLP_IN_OP_OFST 0 */ + +/* MC_CMD_CLP_OUT_GET_BOOT msgresponse */ +#define MC_CMD_CLP_OUT_GET_BOOT_LEN 4 +/* Boot flag */ +#define MC_CMD_CLP_OUT_GET_BOOT_FLAG_OFST 0 +#define MC_CMD_CLP_OUT_GET_BOOT_FLAG_LEN 1 +/* Padding */ +#define MC_CMD_CLP_OUT_GET_BOOT_RESERVED_OFST 1 +#define MC_CMD_CLP_OUT_GET_BOOT_RESERVED_LEN 3 + + +/***********************************/ +/* MC_CMD_MUM + * Perform a MUM operation + */ +#define MC_CMD_MUM 0x57 +#undef MC_CMD_0x57_PRIVILEGE_CTG + +#define MC_CMD_0x57_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_MUM_IN msgrequest */ +#define MC_CMD_MUM_IN_LEN 4 +#define MC_CMD_MUM_IN_OP_HDR_OFST 0 +#define MC_CMD_MUM_IN_OP_LBN 0 +#define MC_CMD_MUM_IN_OP_WIDTH 8 +/* enum: NULL MCDI command to MUM */ +#define MC_CMD_MUM_OP_NULL 0x1 +/* enum: Get MUM version */ +#define MC_CMD_MUM_OP_GET_VERSION 0x2 +/* enum: Issue raw I2C command to MUM */ +#define MC_CMD_MUM_OP_RAW_CMD 0x3 +/* enum: Read from registers on devices connected to MUM. */ +#define MC_CMD_MUM_OP_READ 0x4 +/* enum: Write to registers on devices connected to MUM. */ +#define MC_CMD_MUM_OP_WRITE 0x5 +/* enum: Control UART logging. */ +#define MC_CMD_MUM_OP_LOG 0x6 +/* enum: Operations on MUM GPIO lines */ +#define MC_CMD_MUM_OP_GPIO 0x7 +/* enum: Get sensor readings from MUM */ +#define MC_CMD_MUM_OP_READ_SENSORS 0x8 +/* enum: Initiate clock programming on the MUM */ +#define MC_CMD_MUM_OP_PROGRAM_CLOCKS 0x9 +/* enum: Initiate FPGA load from flash on the MUM */ +#define MC_CMD_MUM_OP_FPGA_LOAD 0xa +/* enum: Request sensor reading from MUM ADC resulting from earlier request via + * MUM ATB + */ +#define MC_CMD_MUM_OP_READ_ATB_SENSOR 0xb +/* enum: Send commands relating to the QSFP ports via the MUM for PHY + * operations + */ +#define MC_CMD_MUM_OP_QSFP 0xc +/* enum: Request discrete and SODIMM DDR info (type, size, speed grade, voltage + * level) from MUM + */ +#define MC_CMD_MUM_OP_READ_DDR_INFO 0xd + +/* MC_CMD_MUM_IN_NULL msgrequest */ +#define MC_CMD_MUM_IN_NULL_LEN 4 +/* MUM cmd header */ +#define MC_CMD_MUM_IN_CMD_OFST 0 + +/* MC_CMD_MUM_IN_GET_VERSION msgrequest */ +#define MC_CMD_MUM_IN_GET_VERSION_LEN 4 +/* MUM cmd header */ +/* MC_CMD_MUM_IN_CMD_OFST 0 */ + +/* MC_CMD_MUM_IN_READ msgrequest */ +#define MC_CMD_MUM_IN_READ_LEN 16 +/* MUM cmd header */ +/* MC_CMD_MUM_IN_CMD_OFST 0 */ +/* ID of (device connected to MUM) to read from registers of */ +#define MC_CMD_MUM_IN_READ_DEVICE_OFST 4 +/* enum: Hittite HMC1035 clock generator on Sorrento board */ +#define MC_CMD_MUM_DEV_HITTITE 0x1 +/* enum: Hittite HMC1035 clock generator for NIC-side on Sorrento board */ +#define MC_CMD_MUM_DEV_HITTITE_NIC 0x2 +/* 32-bit address to read from */ +#define MC_CMD_MUM_IN_READ_ADDR_OFST 8 +/* Number of words to read. */ +#define MC_CMD_MUM_IN_READ_NUMWORDS_OFST 12 + +/* MC_CMD_MUM_IN_WRITE msgrequest */ +#define MC_CMD_MUM_IN_WRITE_LENMIN 16 +#define MC_CMD_MUM_IN_WRITE_LENMAX 252 +#define MC_CMD_MUM_IN_WRITE_LEN(num) (12+4*(num)) +/* MUM cmd header */ +/* MC_CMD_MUM_IN_CMD_OFST 0 */ +/* ID of (device connected to MUM) to write to registers of */ +#define MC_CMD_MUM_IN_WRITE_DEVICE_OFST 4 +/* enum: Hittite HMC1035 clock generator on Sorrento board */ +/* MC_CMD_MUM_DEV_HITTITE 0x1 */ +/* 32-bit address to write to */ +#define MC_CMD_MUM_IN_WRITE_ADDR_OFST 8 +/* Words to write */ +#define MC_CMD_MUM_IN_WRITE_BUFFER_OFST 12 +#define MC_CMD_MUM_IN_WRITE_BUFFER_LEN 4 +#define MC_CMD_MUM_IN_WRITE_BUFFER_MINNUM 1 +#define MC_CMD_MUM_IN_WRITE_BUFFER_MAXNUM 60 + +/* MC_CMD_MUM_IN_RAW_CMD msgrequest */ +#define MC_CMD_MUM_IN_RAW_CMD_LENMIN 17 +#define MC_CMD_MUM_IN_RAW_CMD_LENMAX 252 +#define MC_CMD_MUM_IN_RAW_CMD_LEN(num) (16+1*(num)) +/* MUM cmd header */ +/* MC_CMD_MUM_IN_CMD_OFST 0 */ +/* MUM I2C cmd code */ +#define MC_CMD_MUM_IN_RAW_CMD_CMD_CODE_OFST 4 +/* Number of bytes to write */ +#define MC_CMD_MUM_IN_RAW_CMD_NUM_WRITE_OFST 8 +/* Number of bytes to read */ +#define MC_CMD_MUM_IN_RAW_CMD_NUM_READ_OFST 12 +/* Bytes to write */ +#define MC_CMD_MUM_IN_RAW_CMD_WRITE_DATA_OFST 16 +#define MC_CMD_MUM_IN_RAW_CMD_WRITE_DATA_LEN 1 +#define MC_CMD_MUM_IN_RAW_CMD_WRITE_DATA_MINNUM 1 +#define MC_CMD_MUM_IN_RAW_CMD_WRITE_DATA_MAXNUM 236 + +/* MC_CMD_MUM_IN_LOG msgrequest */ +#define MC_CMD_MUM_IN_LOG_LEN 8 +/* MUM cmd header */ +/* MC_CMD_MUM_IN_CMD_OFST 0 */ +#define MC_CMD_MUM_IN_LOG_OP_OFST 4 +#define MC_CMD_MUM_IN_LOG_OP_UART 0x1 /* enum */ + +/* MC_CMD_MUM_IN_LOG_OP_UART msgrequest */ +#define MC_CMD_MUM_IN_LOG_OP_UART_LEN 12 +/* MC_CMD_MUM_IN_CMD_OFST 0 */ +/* MC_CMD_MUM_IN_LOG_OP_OFST 4 */ +/* Enable/disable debug output to UART */ +#define MC_CMD_MUM_IN_LOG_OP_UART_ENABLE_OFST 8 + +/* MC_CMD_MUM_IN_GPIO msgrequest */ +#define MC_CMD_MUM_IN_GPIO_LEN 8 +/* MUM cmd header */ +/* MC_CMD_MUM_IN_CMD_OFST 0 */ +#define MC_CMD_MUM_IN_GPIO_HDR_OFST 4 +#define MC_CMD_MUM_IN_GPIO_OPCODE_LBN 0 +#define MC_CMD_MUM_IN_GPIO_OPCODE_WIDTH 8 +#define MC_CMD_MUM_IN_GPIO_IN_READ 0x0 /* enum */ +#define MC_CMD_MUM_IN_GPIO_OUT_WRITE 0x1 /* enum */ +#define MC_CMD_MUM_IN_GPIO_OUT_READ 0x2 /* enum */ +#define MC_CMD_MUM_IN_GPIO_OUT_ENABLE_WRITE 0x3 /* enum */ +#define MC_CMD_MUM_IN_GPIO_OUT_ENABLE_READ 0x4 /* enum */ +#define MC_CMD_MUM_IN_GPIO_OP 0x5 /* enum */ + +/* MC_CMD_MUM_IN_GPIO_IN_READ msgrequest */ +#define MC_CMD_MUM_IN_GPIO_IN_READ_LEN 8 +/* MC_CMD_MUM_IN_CMD_OFST 0 */ +#define MC_CMD_MUM_IN_GPIO_IN_READ_HDR_OFST 4 + +/* MC_CMD_MUM_IN_GPIO_OUT_WRITE msgrequest */ +#define MC_CMD_MUM_IN_GPIO_OUT_WRITE_LEN 16 +/* MC_CMD_MUM_IN_CMD_OFST 0 */ +#define MC_CMD_MUM_IN_GPIO_OUT_WRITE_HDR_OFST 4 +/* The first 32-bit word to be written to the GPIO OUT register. */ +#define MC_CMD_MUM_IN_GPIO_OUT_WRITE_GPIOMASK1_OFST 8 +/* The second 32-bit word to be written to the GPIO OUT register. */ +#define MC_CMD_MUM_IN_GPIO_OUT_WRITE_GPIOMASK2_OFST 12 + +/* MC_CMD_MUM_IN_GPIO_OUT_READ msgrequest */ +#define MC_CMD_MUM_IN_GPIO_OUT_READ_LEN 8 +/* MC_CMD_MUM_IN_CMD_OFST 0 */ +#define MC_CMD_MUM_IN_GPIO_OUT_READ_HDR_OFST 4 + +/* MC_CMD_MUM_IN_GPIO_OUT_ENABLE_WRITE msgrequest */ +#define MC_CMD_MUM_IN_GPIO_OUT_ENABLE_WRITE_LEN 16 +/* MC_CMD_MUM_IN_CMD_OFST 0 */ +#define MC_CMD_MUM_IN_GPIO_OUT_ENABLE_WRITE_HDR_OFST 4 +/* The first 32-bit word to be written to the GPIO OUT ENABLE register. */ +#define MC_CMD_MUM_IN_GPIO_OUT_ENABLE_WRITE_GPIOMASK1_OFST 8 +/* The second 32-bit word to be written to the GPIO OUT ENABLE register. */ +#define MC_CMD_MUM_IN_GPIO_OUT_ENABLE_WRITE_GPIOMASK2_OFST 12 + +/* MC_CMD_MUM_IN_GPIO_OUT_ENABLE_READ msgrequest */ +#define MC_CMD_MUM_IN_GPIO_OUT_ENABLE_READ_LEN 8 +/* MC_CMD_MUM_IN_CMD_OFST 0 */ +#define MC_CMD_MUM_IN_GPIO_OUT_ENABLE_READ_HDR_OFST 4 + +/* MC_CMD_MUM_IN_GPIO_OP msgrequest */ +#define MC_CMD_MUM_IN_GPIO_OP_LEN 8 +/* MC_CMD_MUM_IN_CMD_OFST 0 */ +#define MC_CMD_MUM_IN_GPIO_OP_HDR_OFST 4 +#define MC_CMD_MUM_IN_GPIO_OP_BITWISE_OP_LBN 8 +#define MC_CMD_MUM_IN_GPIO_OP_BITWISE_OP_WIDTH 8 +#define MC_CMD_MUM_IN_GPIO_OP_OUT_READ 0x0 /* enum */ +#define MC_CMD_MUM_IN_GPIO_OP_OUT_WRITE 0x1 /* enum */ +#define MC_CMD_MUM_IN_GPIO_OP_OUT_CONFIG 0x2 /* enum */ +#define MC_CMD_MUM_IN_GPIO_OP_OUT_ENABLE 0x3 /* enum */ +#define MC_CMD_MUM_IN_GPIO_OP_GPIO_NUMBER_LBN 16 +#define MC_CMD_MUM_IN_GPIO_OP_GPIO_NUMBER_WIDTH 8 + +/* MC_CMD_MUM_IN_GPIO_OP_OUT_READ msgrequest */ +#define MC_CMD_MUM_IN_GPIO_OP_OUT_READ_LEN 8 +/* MC_CMD_MUM_IN_CMD_OFST 0 */ +#define MC_CMD_MUM_IN_GPIO_OP_OUT_READ_HDR_OFST 4 + +/* MC_CMD_MUM_IN_GPIO_OP_OUT_WRITE msgrequest */ +#define MC_CMD_MUM_IN_GPIO_OP_OUT_WRITE_LEN 8 +/* MC_CMD_MUM_IN_CMD_OFST 0 */ +#define MC_CMD_MUM_IN_GPIO_OP_OUT_WRITE_HDR_OFST 4 +#define MC_CMD_MUM_IN_GPIO_OP_OUT_WRITE_WRITEBIT_LBN 24 +#define MC_CMD_MUM_IN_GPIO_OP_OUT_WRITE_WRITEBIT_WIDTH 8 + +/* MC_CMD_MUM_IN_GPIO_OP_OUT_CONFIG msgrequest */ +#define MC_CMD_MUM_IN_GPIO_OP_OUT_CONFIG_LEN 8 +/* MC_CMD_MUM_IN_CMD_OFST 0 */ +#define MC_CMD_MUM_IN_GPIO_OP_OUT_CONFIG_HDR_OFST 4 +#define MC_CMD_MUM_IN_GPIO_OP_OUT_CONFIG_CFG_LBN 24 +#define MC_CMD_MUM_IN_GPIO_OP_OUT_CONFIG_CFG_WIDTH 8 + +/* MC_CMD_MUM_IN_GPIO_OP_OUT_ENABLE msgrequest */ +#define MC_CMD_MUM_IN_GPIO_OP_OUT_ENABLE_LEN 8 +/* MC_CMD_MUM_IN_CMD_OFST 0 */ +#define MC_CMD_MUM_IN_GPIO_OP_OUT_ENABLE_HDR_OFST 4 +#define MC_CMD_MUM_IN_GPIO_OP_OUT_ENABLE_ENABLEBIT_LBN 24 +#define MC_CMD_MUM_IN_GPIO_OP_OUT_ENABLE_ENABLEBIT_WIDTH 8 + +/* MC_CMD_MUM_IN_READ_SENSORS msgrequest */ +#define MC_CMD_MUM_IN_READ_SENSORS_LEN 8 +/* MUM cmd header */ +/* MC_CMD_MUM_IN_CMD_OFST 0 */ +#define MC_CMD_MUM_IN_READ_SENSORS_PARAMS_OFST 4 +#define MC_CMD_MUM_IN_READ_SENSORS_SENSOR_ID_LBN 0 +#define MC_CMD_MUM_IN_READ_SENSORS_SENSOR_ID_WIDTH 8 +#define MC_CMD_MUM_IN_READ_SENSORS_NUM_SENSORS_LBN 8 +#define MC_CMD_MUM_IN_READ_SENSORS_NUM_SENSORS_WIDTH 8 + +/* MC_CMD_MUM_IN_PROGRAM_CLOCKS msgrequest */ +#define MC_CMD_MUM_IN_PROGRAM_CLOCKS_LEN 12 +/* MUM cmd header */ +/* MC_CMD_MUM_IN_CMD_OFST 0 */ +/* Bit-mask of clocks to be programmed */ +#define MC_CMD_MUM_IN_PROGRAM_CLOCKS_MASK_OFST 4 +#define MC_CMD_MUM_CLOCK_ID_FPGA 0x0 /* enum */ +#define MC_CMD_MUM_CLOCK_ID_DDR 0x1 /* enum */ +#define MC_CMD_MUM_CLOCK_ID_NIC 0x2 /* enum */ +/* Control flags for clock programming */ +#define MC_CMD_MUM_IN_PROGRAM_CLOCKS_FLAGS_OFST 8 +#define MC_CMD_MUM_IN_PROGRAM_CLOCKS_OVERCLOCK_110_LBN 0 +#define MC_CMD_MUM_IN_PROGRAM_CLOCKS_OVERCLOCK_110_WIDTH 1 +#define MC_CMD_MUM_IN_PROGRAM_CLOCKS_CLOCK_NIC_FROM_FPGA_LBN 1 +#define MC_CMD_MUM_IN_PROGRAM_CLOCKS_CLOCK_NIC_FROM_FPGA_WIDTH 1 +#define MC_CMD_MUM_IN_PROGRAM_CLOCKS_CLOCK_REF_FROM_XO_LBN 2 +#define MC_CMD_MUM_IN_PROGRAM_CLOCKS_CLOCK_REF_FROM_XO_WIDTH 1 + +/* MC_CMD_MUM_IN_FPGA_LOAD msgrequest */ +#define MC_CMD_MUM_IN_FPGA_LOAD_LEN 8 +/* MUM cmd header */ +/* MC_CMD_MUM_IN_CMD_OFST 0 */ +/* Enable/Disable FPGA config from flash */ +#define MC_CMD_MUM_IN_FPGA_LOAD_ENABLE_OFST 4 + +/* MC_CMD_MUM_IN_READ_ATB_SENSOR msgrequest */ +#define MC_CMD_MUM_IN_READ_ATB_SENSOR_LEN 4 +/* MUM cmd header */ +/* MC_CMD_MUM_IN_CMD_OFST 0 */ + +/* MC_CMD_MUM_IN_QSFP msgrequest */ +#define MC_CMD_MUM_IN_QSFP_LEN 12 +/* MUM cmd header */ +/* MC_CMD_MUM_IN_CMD_OFST 0 */ +#define MC_CMD_MUM_IN_QSFP_HDR_OFST 4 +#define MC_CMD_MUM_IN_QSFP_OPCODE_LBN 0 +#define MC_CMD_MUM_IN_QSFP_OPCODE_WIDTH 4 +#define MC_CMD_MUM_IN_QSFP_INIT 0x0 /* enum */ +#define MC_CMD_MUM_IN_QSFP_RECONFIGURE 0x1 /* enum */ +#define MC_CMD_MUM_IN_QSFP_GET_SUPPORTED_CAP 0x2 /* enum */ +#define MC_CMD_MUM_IN_QSFP_GET_MEDIA_INFO 0x3 /* enum */ +#define MC_CMD_MUM_IN_QSFP_FILL_STATS 0x4 /* enum */ +#define MC_CMD_MUM_IN_QSFP_POLL_BIST 0x5 /* enum */ +#define MC_CMD_MUM_IN_QSFP_IDX_OFST 8 + +/* MC_CMD_MUM_IN_QSFP_INIT msgrequest */ +#define MC_CMD_MUM_IN_QSFP_INIT_LEN 16 +/* MC_CMD_MUM_IN_CMD_OFST 0 */ +#define MC_CMD_MUM_IN_QSFP_INIT_HDR_OFST 4 +#define MC_CMD_MUM_IN_QSFP_INIT_IDX_OFST 8 +#define MC_CMD_MUM_IN_QSFP_INIT_CAGE_OFST 12 + +/* MC_CMD_MUM_IN_QSFP_RECONFIGURE msgrequest */ +#define MC_CMD_MUM_IN_QSFP_RECONFIGURE_LEN 24 +/* MC_CMD_MUM_IN_CMD_OFST 0 */ +#define MC_CMD_MUM_IN_QSFP_RECONFIGURE_HDR_OFST 4 +#define MC_CMD_MUM_IN_QSFP_RECONFIGURE_IDX_OFST 8 +#define MC_CMD_MUM_IN_QSFP_RECONFIGURE_TX_DISABLE_OFST 12 +#define MC_CMD_MUM_IN_QSFP_RECONFIGURE_PORT_LANES_OFST 16 +#define MC_CMD_MUM_IN_QSFP_RECONFIGURE_PORT_LINK_SPEED_OFST 20 + +/* MC_CMD_MUM_IN_QSFP_GET_SUPPORTED_CAP msgrequest */ +#define MC_CMD_MUM_IN_QSFP_GET_SUPPORTED_CAP_LEN 12 +/* MC_CMD_MUM_IN_CMD_OFST 0 */ +#define MC_CMD_MUM_IN_QSFP_GET_SUPPORTED_CAP_HDR_OFST 4 +#define MC_CMD_MUM_IN_QSFP_GET_SUPPORTED_CAP_IDX_OFST 8 + +/* MC_CMD_MUM_IN_QSFP_GET_MEDIA_INFO msgrequest */ +#define MC_CMD_MUM_IN_QSFP_GET_MEDIA_INFO_LEN 16 +/* MC_CMD_MUM_IN_CMD_OFST 0 */ +#define MC_CMD_MUM_IN_QSFP_GET_MEDIA_INFO_HDR_OFST 4 +#define MC_CMD_MUM_IN_QSFP_GET_MEDIA_INFO_IDX_OFST 8 +#define MC_CMD_MUM_IN_QSFP_GET_MEDIA_INFO_PAGE_OFST 12 + +/* MC_CMD_MUM_IN_QSFP_FILL_STATS msgrequest */ +#define MC_CMD_MUM_IN_QSFP_FILL_STATS_LEN 12 +/* MC_CMD_MUM_IN_CMD_OFST 0 */ +#define MC_CMD_MUM_IN_QSFP_FILL_STATS_HDR_OFST 4 +#define MC_CMD_MUM_IN_QSFP_FILL_STATS_IDX_OFST 8 + +/* MC_CMD_MUM_IN_QSFP_POLL_BIST msgrequest */ +#define MC_CMD_MUM_IN_QSFP_POLL_BIST_LEN 12 +/* MC_CMD_MUM_IN_CMD_OFST 0 */ +#define MC_CMD_MUM_IN_QSFP_POLL_BIST_HDR_OFST 4 +#define MC_CMD_MUM_IN_QSFP_POLL_BIST_IDX_OFST 8 + +/* MC_CMD_MUM_IN_READ_DDR_INFO msgrequest */ +#define MC_CMD_MUM_IN_READ_DDR_INFO_LEN 4 +/* MUM cmd header */ +/* MC_CMD_MUM_IN_CMD_OFST 0 */ + +/* MC_CMD_MUM_OUT msgresponse */ +#define MC_CMD_MUM_OUT_LEN 0 + +/* MC_CMD_MUM_OUT_NULL msgresponse */ +#define MC_CMD_MUM_OUT_NULL_LEN 0 + +/* MC_CMD_MUM_OUT_GET_VERSION msgresponse */ +#define MC_CMD_MUM_OUT_GET_VERSION_LEN 12 +#define MC_CMD_MUM_OUT_GET_VERSION_FIRMWARE_OFST 0 +#define MC_CMD_MUM_OUT_GET_VERSION_VERSION_OFST 4 +#define MC_CMD_MUM_OUT_GET_VERSION_VERSION_LEN 8 +#define MC_CMD_MUM_OUT_GET_VERSION_VERSION_LO_OFST 4 +#define MC_CMD_MUM_OUT_GET_VERSION_VERSION_HI_OFST 8 + +/* MC_CMD_MUM_OUT_RAW_CMD msgresponse */ +#define MC_CMD_MUM_OUT_RAW_CMD_LENMIN 1 +#define MC_CMD_MUM_OUT_RAW_CMD_LENMAX 252 +#define MC_CMD_MUM_OUT_RAW_CMD_LEN(num) (0+1*(num)) +/* returned data */ +#define MC_CMD_MUM_OUT_RAW_CMD_DATA_OFST 0 +#define MC_CMD_MUM_OUT_RAW_CMD_DATA_LEN 1 +#define MC_CMD_MUM_OUT_RAW_CMD_DATA_MINNUM 1 +#define MC_CMD_MUM_OUT_RAW_CMD_DATA_MAXNUM 252 + +/* MC_CMD_MUM_OUT_READ msgresponse */ +#define MC_CMD_MUM_OUT_READ_LENMIN 4 +#define MC_CMD_MUM_OUT_READ_LENMAX 252 +#define MC_CMD_MUM_OUT_READ_LEN(num) (0+4*(num)) +#define MC_CMD_MUM_OUT_READ_BUFFER_OFST 0 +#define MC_CMD_MUM_OUT_READ_BUFFER_LEN 4 +#define MC_CMD_MUM_OUT_READ_BUFFER_MINNUM 1 +#define MC_CMD_MUM_OUT_READ_BUFFER_MAXNUM 63 + +/* MC_CMD_MUM_OUT_WRITE msgresponse */ +#define MC_CMD_MUM_OUT_WRITE_LEN 0 + +/* MC_CMD_MUM_OUT_LOG msgresponse */ +#define MC_CMD_MUM_OUT_LOG_LEN 0 + +/* MC_CMD_MUM_OUT_LOG_OP_UART msgresponse */ +#define MC_CMD_MUM_OUT_LOG_OP_UART_LEN 0 + +/* MC_CMD_MUM_OUT_GPIO_IN_READ msgresponse */ +#define MC_CMD_MUM_OUT_GPIO_IN_READ_LEN 8 +/* The first 32-bit word read from the GPIO IN register. */ +#define MC_CMD_MUM_OUT_GPIO_IN_READ_GPIOMASK1_OFST 0 +/* The second 32-bit word read from the GPIO IN register. */ +#define MC_CMD_MUM_OUT_GPIO_IN_READ_GPIOMASK2_OFST 4 + +/* MC_CMD_MUM_OUT_GPIO_OUT_WRITE msgresponse */ +#define MC_CMD_MUM_OUT_GPIO_OUT_WRITE_LEN 0 + +/* MC_CMD_MUM_OUT_GPIO_OUT_READ msgresponse */ +#define MC_CMD_MUM_OUT_GPIO_OUT_READ_LEN 8 +/* The first 32-bit word read from the GPIO OUT register. */ +#define MC_CMD_MUM_OUT_GPIO_OUT_READ_GPIOMASK1_OFST 0 +/* The second 32-bit word read from the GPIO OUT register. */ +#define MC_CMD_MUM_OUT_GPIO_OUT_READ_GPIOMASK2_OFST 4 + +/* MC_CMD_MUM_OUT_GPIO_OUT_ENABLE_WRITE msgresponse */ +#define MC_CMD_MUM_OUT_GPIO_OUT_ENABLE_WRITE_LEN 0 + +/* MC_CMD_MUM_OUT_GPIO_OUT_ENABLE_READ msgresponse */ +#define MC_CMD_MUM_OUT_GPIO_OUT_ENABLE_READ_LEN 8 +#define MC_CMD_MUM_OUT_GPIO_OUT_ENABLE_READ_GPIOMASK1_OFST 0 +#define MC_CMD_MUM_OUT_GPIO_OUT_ENABLE_READ_GPIOMASK2_OFST 4 + +/* MC_CMD_MUM_OUT_GPIO_OP_OUT_READ msgresponse */ +#define MC_CMD_MUM_OUT_GPIO_OP_OUT_READ_LEN 4 +#define MC_CMD_MUM_OUT_GPIO_OP_OUT_READ_BIT_READ_OFST 0 + +/* MC_CMD_MUM_OUT_GPIO_OP_OUT_WRITE msgresponse */ +#define MC_CMD_MUM_OUT_GPIO_OP_OUT_WRITE_LEN 0 + +/* MC_CMD_MUM_OUT_GPIO_OP_OUT_CONFIG msgresponse */ +#define MC_CMD_MUM_OUT_GPIO_OP_OUT_CONFIG_LEN 0 + +/* MC_CMD_MUM_OUT_GPIO_OP_OUT_ENABLE msgresponse */ +#define MC_CMD_MUM_OUT_GPIO_OP_OUT_ENABLE_LEN 0 + +/* MC_CMD_MUM_OUT_READ_SENSORS msgresponse */ +#define MC_CMD_MUM_OUT_READ_SENSORS_LENMIN 4 +#define MC_CMD_MUM_OUT_READ_SENSORS_LENMAX 252 +#define MC_CMD_MUM_OUT_READ_SENSORS_LEN(num) (0+4*(num)) +#define MC_CMD_MUM_OUT_READ_SENSORS_DATA_OFST 0 +#define MC_CMD_MUM_OUT_READ_SENSORS_DATA_LEN 4 +#define MC_CMD_MUM_OUT_READ_SENSORS_DATA_MINNUM 1 +#define MC_CMD_MUM_OUT_READ_SENSORS_DATA_MAXNUM 63 +#define MC_CMD_MUM_OUT_READ_SENSORS_READING_LBN 0 +#define MC_CMD_MUM_OUT_READ_SENSORS_READING_WIDTH 16 +#define MC_CMD_MUM_OUT_READ_SENSORS_STATE_LBN 16 +#define MC_CMD_MUM_OUT_READ_SENSORS_STATE_WIDTH 8 +#define MC_CMD_MUM_OUT_READ_SENSORS_TYPE_LBN 24 +#define MC_CMD_MUM_OUT_READ_SENSORS_TYPE_WIDTH 8 + +/* MC_CMD_MUM_OUT_PROGRAM_CLOCKS msgresponse */ +#define MC_CMD_MUM_OUT_PROGRAM_CLOCKS_LEN 4 +#define MC_CMD_MUM_OUT_PROGRAM_CLOCKS_OK_MASK_OFST 0 + +/* MC_CMD_MUM_OUT_FPGA_LOAD msgresponse */ +#define MC_CMD_MUM_OUT_FPGA_LOAD_LEN 0 + +/* MC_CMD_MUM_OUT_READ_ATB_SENSOR msgresponse */ +#define MC_CMD_MUM_OUT_READ_ATB_SENSOR_LEN 4 +#define MC_CMD_MUM_OUT_READ_ATB_SENSOR_RESULT_OFST 0 + +/* MC_CMD_MUM_OUT_QSFP_INIT msgresponse */ +#define MC_CMD_MUM_OUT_QSFP_INIT_LEN 0 + +/* MC_CMD_MUM_OUT_QSFP_RECONFIGURE msgresponse */ +#define MC_CMD_MUM_OUT_QSFP_RECONFIGURE_LEN 8 +#define MC_CMD_MUM_OUT_QSFP_RECONFIGURE_PORT_PHY_LP_CAP_OFST 0 +#define MC_CMD_MUM_OUT_QSFP_RECONFIGURE_PORT_PHY_FLAGS_OFST 4 +#define MC_CMD_MUM_OUT_QSFP_RECONFIGURE_PORT_PHY_READY_LBN 0 +#define MC_CMD_MUM_OUT_QSFP_RECONFIGURE_PORT_PHY_READY_WIDTH 1 +#define MC_CMD_MUM_OUT_QSFP_RECONFIGURE_PORT_PHY_LINK_UP_LBN 1 +#define MC_CMD_MUM_OUT_QSFP_RECONFIGURE_PORT_PHY_LINK_UP_WIDTH 1 + +/* MC_CMD_MUM_OUT_QSFP_GET_SUPPORTED_CAP msgresponse */ +#define MC_CMD_MUM_OUT_QSFP_GET_SUPPORTED_CAP_LEN 4 +#define MC_CMD_MUM_OUT_QSFP_GET_SUPPORTED_CAP_PORT_PHY_LP_CAP_OFST 0 + +/* MC_CMD_MUM_OUT_QSFP_GET_MEDIA_INFO msgresponse */ +#define MC_CMD_MUM_OUT_QSFP_GET_MEDIA_INFO_LENMIN 5 +#define MC_CMD_MUM_OUT_QSFP_GET_MEDIA_INFO_LENMAX 252 +#define MC_CMD_MUM_OUT_QSFP_GET_MEDIA_INFO_LEN(num) (4+1*(num)) +/* in bytes */ +#define MC_CMD_MUM_OUT_QSFP_GET_MEDIA_INFO_DATALEN_OFST 0 +#define MC_CMD_MUM_OUT_QSFP_GET_MEDIA_INFO_DATA_OFST 4 +#define MC_CMD_MUM_OUT_QSFP_GET_MEDIA_INFO_DATA_LEN 1 +#define MC_CMD_MUM_OUT_QSFP_GET_MEDIA_INFO_DATA_MINNUM 1 +#define MC_CMD_MUM_OUT_QSFP_GET_MEDIA_INFO_DATA_MAXNUM 248 + +/* MC_CMD_MUM_OUT_QSFP_FILL_STATS msgresponse */ +#define MC_CMD_MUM_OUT_QSFP_FILL_STATS_LEN 8 +#define MC_CMD_MUM_OUT_QSFP_FILL_STATS_PORT_PHY_STATS_PMA_PMD_LINK_UP_OFST 0 +#define MC_CMD_MUM_OUT_QSFP_FILL_STATS_PORT_PHY_STATS_PCS_LINK_UP_OFST 4 + +/* MC_CMD_MUM_OUT_QSFP_POLL_BIST msgresponse */ +#define MC_CMD_MUM_OUT_QSFP_POLL_BIST_LEN 4 +#define MC_CMD_MUM_OUT_QSFP_POLL_BIST_TEST_OFST 0 + +/* MC_CMD_MUM_OUT_READ_DDR_INFO msgresponse */ +#define MC_CMD_MUM_OUT_READ_DDR_INFO_LENMIN 24 +#define MC_CMD_MUM_OUT_READ_DDR_INFO_LENMAX 248 +#define MC_CMD_MUM_OUT_READ_DDR_INFO_LEN(num) (8+8*(num)) +/* Discrete (soldered) DDR resistor strap info */ +#define MC_CMD_MUM_OUT_READ_DDR_INFO_DISCRETE_DDR_INFO_OFST 0 +#define MC_CMD_MUM_OUT_READ_DDR_INFO_VRATIO_LBN 0 +#define MC_CMD_MUM_OUT_READ_DDR_INFO_VRATIO_WIDTH 16 +#define MC_CMD_MUM_OUT_READ_DDR_INFO_RESERVED1_LBN 16 +#define MC_CMD_MUM_OUT_READ_DDR_INFO_RESERVED1_WIDTH 16 +/* Number of SODIMM info records */ +#define MC_CMD_MUM_OUT_READ_DDR_INFO_NUM_RECORDS_OFST 4 +/* Array of SODIMM info records */ +#define MC_CMD_MUM_OUT_READ_DDR_INFO_SODIMM_INFO_RECORD_OFST 8 +#define MC_CMD_MUM_OUT_READ_DDR_INFO_SODIMM_INFO_RECORD_LEN 8 +#define MC_CMD_MUM_OUT_READ_DDR_INFO_SODIMM_INFO_RECORD_LO_OFST 8 +#define MC_CMD_MUM_OUT_READ_DDR_INFO_SODIMM_INFO_RECORD_HI_OFST 12 +#define MC_CMD_MUM_OUT_READ_DDR_INFO_SODIMM_INFO_RECORD_MINNUM 2 +#define MC_CMD_MUM_OUT_READ_DDR_INFO_SODIMM_INFO_RECORD_MAXNUM 30 +#define MC_CMD_MUM_OUT_READ_DDR_INFO_BANK_ID_LBN 0 +#define MC_CMD_MUM_OUT_READ_DDR_INFO_BANK_ID_WIDTH 8 +/* enum: SODIMM bank 1 (Top SODIMM for Sorrento) */ +#define MC_CMD_MUM_OUT_READ_DDR_INFO_BANK1 0x0 +/* enum: SODIMM bank 2 (Bottom SODDIMM for Sorrento) */ +#define MC_CMD_MUM_OUT_READ_DDR_INFO_BANK2 0x1 +/* enum: Total number of SODIMM banks */ +#define MC_CMD_MUM_OUT_READ_DDR_INFO_NUM_BANKS 0x2 +#define MC_CMD_MUM_OUT_READ_DDR_INFO_TYPE_LBN 8 +#define MC_CMD_MUM_OUT_READ_DDR_INFO_TYPE_WIDTH 8 +#define MC_CMD_MUM_OUT_READ_DDR_INFO_RANK_LBN 16 +#define MC_CMD_MUM_OUT_READ_DDR_INFO_RANK_WIDTH 4 +#define MC_CMD_MUM_OUT_READ_DDR_INFO_VOLTAGE_LBN 20 +#define MC_CMD_MUM_OUT_READ_DDR_INFO_VOLTAGE_WIDTH 4 +#define MC_CMD_MUM_OUT_READ_DDR_INFO_NOT_POWERED 0x0 /* enum */ +#define MC_CMD_MUM_OUT_READ_DDR_INFO_1V25 0x1 /* enum */ +#define MC_CMD_MUM_OUT_READ_DDR_INFO_1V35 0x2 /* enum */ +#define MC_CMD_MUM_OUT_READ_DDR_INFO_1V5 0x3 /* enum */ +/* enum: Values 5-15 are reserved for future usage */ +#define MC_CMD_MUM_OUT_READ_DDR_INFO_1V8 0x4 +#define MC_CMD_MUM_OUT_READ_DDR_INFO_SIZE_LBN 24 +#define MC_CMD_MUM_OUT_READ_DDR_INFO_SIZE_WIDTH 8 +#define MC_CMD_MUM_OUT_READ_DDR_INFO_SPEED_LBN 32 +#define MC_CMD_MUM_OUT_READ_DDR_INFO_SPEED_WIDTH 16 +#define MC_CMD_MUM_OUT_READ_DDR_INFO_STATE_LBN 48 +#define MC_CMD_MUM_OUT_READ_DDR_INFO_STATE_WIDTH 4 +/* enum: No module present */ +#define MC_CMD_MUM_OUT_READ_DDR_INFO_ABSENT 0x0 +/* enum: Module present supported and powered on */ +#define MC_CMD_MUM_OUT_READ_DDR_INFO_PRESENT_POWERED 0x1 +/* enum: Module present but bad type */ +#define MC_CMD_MUM_OUT_READ_DDR_INFO_PRESENT_BAD_TYPE 0x2 +/* enum: Module present but incompatible voltage */ +#define MC_CMD_MUM_OUT_READ_DDR_INFO_PRESENT_BAD_VOLTAGE 0x3 +/* enum: Module present but unknown SPD */ +#define MC_CMD_MUM_OUT_READ_DDR_INFO_PRESENT_BAD_SPD 0x4 +/* enum: Module present but slot cannot support it */ +#define MC_CMD_MUM_OUT_READ_DDR_INFO_PRESENT_BAD_SLOT 0x5 +/* enum: Modules may or may not be present, but cannot establish contact by I2C + */ +#define MC_CMD_MUM_OUT_READ_DDR_INFO_NOT_REACHABLE 0x6 +#define MC_CMD_MUM_OUT_READ_DDR_INFO_RESERVED2_LBN 52 +#define MC_CMD_MUM_OUT_READ_DDR_INFO_RESERVED2_WIDTH 12 + +/* MC_CMD_RESOURCE_SPECIFIER enum */ +/* enum: Any */ +#define MC_CMD_RESOURCE_INSTANCE_ANY 0xffffffff +/* enum: None */ +#define MC_CMD_RESOURCE_INSTANCE_NONE 0xfffffffe + +/* EVB_PORT_ID structuredef */ +#define EVB_PORT_ID_LEN 4 +#define EVB_PORT_ID_PORT_ID_OFST 0 +/* enum: An invalid port handle. */ +#define EVB_PORT_ID_NULL 0x0 +/* enum: The port assigned to this function.. */ +#define EVB_PORT_ID_ASSIGNED 0x1000000 +/* enum: External network port 0 */ +#define EVB_PORT_ID_MAC0 0x2000000 +/* enum: External network port 1 */ +#define EVB_PORT_ID_MAC1 0x2000001 +/* enum: External network port 2 */ +#define EVB_PORT_ID_MAC2 0x2000002 +/* enum: External network port 3 */ +#define EVB_PORT_ID_MAC3 0x2000003 +#define EVB_PORT_ID_PORT_ID_LBN 0 +#define EVB_PORT_ID_PORT_ID_WIDTH 32 + +/* EVB_VLAN_TAG structuredef */ +#define EVB_VLAN_TAG_LEN 2 +/* The VLAN tag value */ +#define EVB_VLAN_TAG_VLAN_ID_LBN 0 +#define EVB_VLAN_TAG_VLAN_ID_WIDTH 12 +#define EVB_VLAN_TAG_MODE_LBN 12 +#define EVB_VLAN_TAG_MODE_WIDTH 4 +/* enum: Insert the VLAN. */ +#define EVB_VLAN_TAG_INSERT 0x0 +/* enum: Replace the VLAN if already present. */ +#define EVB_VLAN_TAG_REPLACE 0x1 + +/* BUFTBL_ENTRY structuredef */ +#define BUFTBL_ENTRY_LEN 12 +/* the owner ID */ +#define BUFTBL_ENTRY_OID_OFST 0 +#define BUFTBL_ENTRY_OID_LEN 2 +#define BUFTBL_ENTRY_OID_LBN 0 +#define BUFTBL_ENTRY_OID_WIDTH 16 +/* the page parameter as one of ESE_DZ_SMC_PAGE_SIZE_ */ +#define BUFTBL_ENTRY_PGSZ_OFST 2 +#define BUFTBL_ENTRY_PGSZ_LEN 2 +#define BUFTBL_ENTRY_PGSZ_LBN 16 +#define BUFTBL_ENTRY_PGSZ_WIDTH 16 +/* the raw 64-bit address field from the SMC, not adjusted for page size */ +#define BUFTBL_ENTRY_RAWADDR_OFST 4 +#define BUFTBL_ENTRY_RAWADDR_LEN 8 +#define BUFTBL_ENTRY_RAWADDR_LO_OFST 4 +#define BUFTBL_ENTRY_RAWADDR_HI_OFST 8 +#define BUFTBL_ENTRY_RAWADDR_LBN 32 +#define BUFTBL_ENTRY_RAWADDR_WIDTH 64 + +/* NVRAM_PARTITION_TYPE structuredef */ +#define NVRAM_PARTITION_TYPE_LEN 2 +#define NVRAM_PARTITION_TYPE_ID_OFST 0 +#define NVRAM_PARTITION_TYPE_ID_LEN 2 +/* enum: Primary MC firmware partition */ +#define NVRAM_PARTITION_TYPE_MC_FIRMWARE 0x100 +/* enum: Secondary MC firmware partition */ +#define NVRAM_PARTITION_TYPE_MC_FIRMWARE_BACKUP 0x200 +/* enum: Expansion ROM partition */ +#define NVRAM_PARTITION_TYPE_EXPANSION_ROM 0x300 +/* enum: Static configuration TLV partition */ +#define NVRAM_PARTITION_TYPE_STATIC_CONFIG 0x400 +/* enum: Dynamic configuration TLV partition */ +#define NVRAM_PARTITION_TYPE_DYNAMIC_CONFIG 0x500 +/* enum: Expansion ROM configuration data for port 0 */ +#define NVRAM_PARTITION_TYPE_EXPROM_CONFIG_PORT0 0x600 +/* enum: Synonym for EXPROM_CONFIG_PORT0 as used in pmap files */ +#define NVRAM_PARTITION_TYPE_EXPROM_CONFIG 0x600 +/* enum: Expansion ROM configuration data for port 1 */ +#define NVRAM_PARTITION_TYPE_EXPROM_CONFIG_PORT1 0x601 +/* enum: Expansion ROM configuration data for port 2 */ +#define NVRAM_PARTITION_TYPE_EXPROM_CONFIG_PORT2 0x602 +/* enum: Expansion ROM configuration data for port 3 */ +#define NVRAM_PARTITION_TYPE_EXPROM_CONFIG_PORT3 0x603 +/* enum: Non-volatile log output partition */ +#define NVRAM_PARTITION_TYPE_LOG 0x700 +/* enum: Non-volatile log output of second core on dual-core device */ +#define NVRAM_PARTITION_TYPE_LOG_SLAVE 0x701 +/* enum: Device state dump output partition */ +#define NVRAM_PARTITION_TYPE_DUMP 0x800 +/* enum: Application license key storage partition */ +#define NVRAM_PARTITION_TYPE_LICENSE 0x900 +/* enum: Start of range used for PHY partitions (low 8 bits are the PHY ID) */ +#define NVRAM_PARTITION_TYPE_PHY_MIN 0xa00 +/* enum: End of range used for PHY partitions (low 8 bits are the PHY ID) */ +#define NVRAM_PARTITION_TYPE_PHY_MAX 0xaff +/* enum: Primary FPGA partition */ +#define NVRAM_PARTITION_TYPE_FPGA 0xb00 +/* enum: Secondary FPGA partition */ +#define NVRAM_PARTITION_TYPE_FPGA_BACKUP 0xb01 +/* enum: FC firmware partition */ +#define NVRAM_PARTITION_TYPE_FC_FIRMWARE 0xb02 +/* enum: FC License partition */ +#define NVRAM_PARTITION_TYPE_FC_LICENSE 0xb03 +/* enum: Non-volatile log output partition for FC */ +#define NVRAM_PARTITION_TYPE_FC_LOG 0xb04 +/* enum: MUM firmware partition */ +#define NVRAM_PARTITION_TYPE_MUM_FIRMWARE 0xc00 +/* enum: MUM Non-volatile log output partition. */ +#define NVRAM_PARTITION_TYPE_MUM_LOG 0xc01 +/* enum: MUM Application table partition. */ +#define NVRAM_PARTITION_TYPE_MUM_APPTABLE 0xc02 +/* enum: MUM boot rom partition. */ +#define NVRAM_PARTITION_TYPE_MUM_BOOT_ROM 0xc03 +/* enum: MUM production signatures & calibration rom partition. */ +#define NVRAM_PARTITION_TYPE_MUM_PROD_ROM 0xc04 +/* enum: MUM user signatures & calibration rom partition. */ +#define NVRAM_PARTITION_TYPE_MUM_USER_ROM 0xc05 +/* enum: MUM fuses and lockbits partition. */ +#define NVRAM_PARTITION_TYPE_MUM_FUSELOCK 0xc06 +/* enum: UEFI expansion ROM if separate from PXE */ +#define NVRAM_PARTITION_TYPE_EXPANSION_UEFI 0xd00 +/* enum: Spare partition 0 */ +#define NVRAM_PARTITION_TYPE_SPARE_0 0x1000 +/* enum: Used for XIP code of shmbooted images */ +#define NVRAM_PARTITION_TYPE_XIP_SCRATCH 0x1100 +/* enum: Spare partition 2 */ +#define NVRAM_PARTITION_TYPE_SPARE_2 0x1200 +/* enum: Manufacturing partition. Used during manufacture to pass information + * between XJTAG and Manftest. + */ +#define NVRAM_PARTITION_TYPE_MANUFACTURING 0x1300 +/* enum: Spare partition 4 */ +#define NVRAM_PARTITION_TYPE_SPARE_4 0x1400 +/* enum: Spare partition 5 */ +#define NVRAM_PARTITION_TYPE_SPARE_5 0x1500 +/* enum: Partition for reporting MC status. See mc_flash_layout.h + * medford_mc_status_hdr_t for layout on Medford. + */ +#define NVRAM_PARTITION_TYPE_STATUS 0x1600 +/* enum: Start of reserved value range (firmware may use for any purpose) */ +#define NVRAM_PARTITION_TYPE_RESERVED_VALUES_MIN 0xff00 +/* enum: End of reserved value range (firmware may use for any purpose) */ +#define NVRAM_PARTITION_TYPE_RESERVED_VALUES_MAX 0xfffd +/* enum: Recovery partition map (provided if real map is missing or corrupt) */ +#define NVRAM_PARTITION_TYPE_RECOVERY_MAP 0xfffe +/* enum: Partition map (real map as stored in flash) */ +#define NVRAM_PARTITION_TYPE_PARTITION_MAP 0xffff +#define NVRAM_PARTITION_TYPE_ID_LBN 0 +#define NVRAM_PARTITION_TYPE_ID_WIDTH 16 + +/* LICENSED_APP_ID structuredef */ +#define LICENSED_APP_ID_LEN 4 +#define LICENSED_APP_ID_ID_OFST 0 +/* enum: OpenOnload */ +#define LICENSED_APP_ID_ONLOAD 0x1 +/* enum: PTP timestamping */ +#define LICENSED_APP_ID_PTP 0x2 +/* enum: SolarCapture Pro */ +#define LICENSED_APP_ID_SOLARCAPTURE_PRO 0x4 +/* enum: SolarSecure filter engine */ +#define LICENSED_APP_ID_SOLARSECURE 0x8 +/* enum: Performance monitor */ +#define LICENSED_APP_ID_PERF_MONITOR 0x10 +/* enum: SolarCapture Live */ +#define LICENSED_APP_ID_SOLARCAPTURE_LIVE 0x20 +/* enum: Capture SolarSystem */ +#define LICENSED_APP_ID_CAPTURE_SOLARSYSTEM 0x40 +/* enum: Network Access Control */ +#define LICENSED_APP_ID_NETWORK_ACCESS_CONTROL 0x80 +/* enum: TCP Direct */ +#define LICENSED_APP_ID_TCP_DIRECT 0x100 +/* enum: Low Latency */ +#define LICENSED_APP_ID_LOW_LATENCY 0x200 +/* enum: SolarCapture Tap */ +#define LICENSED_APP_ID_SOLARCAPTURE_TAP 0x400 +/* enum: Capture SolarSystem 40G */ +#define LICENSED_APP_ID_CAPTURE_SOLARSYSTEM_40G 0x800 +/* enum: Capture SolarSystem 1G */ +#define LICENSED_APP_ID_CAPTURE_SOLARSYSTEM_1G 0x1000 +#define LICENSED_APP_ID_ID_LBN 0 +#define LICENSED_APP_ID_ID_WIDTH 32 + +/* LICENSED_FEATURES structuredef */ +#define LICENSED_FEATURES_LEN 8 +/* Bitmask of licensed firmware features */ +#define LICENSED_FEATURES_MASK_OFST 0 +#define LICENSED_FEATURES_MASK_LEN 8 +#define LICENSED_FEATURES_MASK_LO_OFST 0 +#define LICENSED_FEATURES_MASK_HI_OFST 4 +#define LICENSED_FEATURES_RX_CUT_THROUGH_LBN 0 +#define LICENSED_FEATURES_RX_CUT_THROUGH_WIDTH 1 +#define LICENSED_FEATURES_PIO_LBN 1 +#define LICENSED_FEATURES_PIO_WIDTH 1 +#define LICENSED_FEATURES_EVQ_TIMER_LBN 2 +#define LICENSED_FEATURES_EVQ_TIMER_WIDTH 1 +#define LICENSED_FEATURES_CLOCK_LBN 3 +#define LICENSED_FEATURES_CLOCK_WIDTH 1 +#define LICENSED_FEATURES_RX_TIMESTAMPS_LBN 4 +#define LICENSED_FEATURES_RX_TIMESTAMPS_WIDTH 1 +#define LICENSED_FEATURES_TX_TIMESTAMPS_LBN 5 +#define LICENSED_FEATURES_TX_TIMESTAMPS_WIDTH 1 +#define LICENSED_FEATURES_RX_SNIFF_LBN 6 +#define LICENSED_FEATURES_RX_SNIFF_WIDTH 1 +#define LICENSED_FEATURES_TX_SNIFF_LBN 7 +#define LICENSED_FEATURES_TX_SNIFF_WIDTH 1 +#define LICENSED_FEATURES_PROXY_FILTER_OPS_LBN 8 +#define LICENSED_FEATURES_PROXY_FILTER_OPS_WIDTH 1 +#define LICENSED_FEATURES_EVENT_CUT_THROUGH_LBN 9 +#define LICENSED_FEATURES_EVENT_CUT_THROUGH_WIDTH 1 +#define LICENSED_FEATURES_MASK_LBN 0 +#define LICENSED_FEATURES_MASK_WIDTH 64 + +/* LICENSED_V3_APPS structuredef */ +#define LICENSED_V3_APPS_LEN 8 +/* Bitmask of licensed applications */ +#define LICENSED_V3_APPS_MASK_OFST 0 +#define LICENSED_V3_APPS_MASK_LEN 8 +#define LICENSED_V3_APPS_MASK_LO_OFST 0 +#define LICENSED_V3_APPS_MASK_HI_OFST 4 +#define LICENSED_V3_APPS_ONLOAD_LBN 0 +#define LICENSED_V3_APPS_ONLOAD_WIDTH 1 +#define LICENSED_V3_APPS_PTP_LBN 1 +#define LICENSED_V3_APPS_PTP_WIDTH 1 +#define LICENSED_V3_APPS_SOLARCAPTURE_PRO_LBN 2 +#define LICENSED_V3_APPS_SOLARCAPTURE_PRO_WIDTH 1 +#define LICENSED_V3_APPS_SOLARSECURE_LBN 3 +#define LICENSED_V3_APPS_SOLARSECURE_WIDTH 1 +#define LICENSED_V3_APPS_PERF_MONITOR_LBN 4 +#define LICENSED_V3_APPS_PERF_MONITOR_WIDTH 1 +#define LICENSED_V3_APPS_SOLARCAPTURE_LIVE_LBN 5 +#define LICENSED_V3_APPS_SOLARCAPTURE_LIVE_WIDTH 1 +#define LICENSED_V3_APPS_CAPTURE_SOLARSYSTEM_LBN 6 +#define LICENSED_V3_APPS_CAPTURE_SOLARSYSTEM_WIDTH 1 +#define LICENSED_V3_APPS_NETWORK_ACCESS_CONTROL_LBN 7 +#define LICENSED_V3_APPS_NETWORK_ACCESS_CONTROL_WIDTH 1 +#define LICENSED_V3_APPS_TCP_DIRECT_LBN 8 +#define LICENSED_V3_APPS_TCP_DIRECT_WIDTH 1 +#define LICENSED_V3_APPS_LOW_LATENCY_LBN 9 +#define LICENSED_V3_APPS_LOW_LATENCY_WIDTH 1 +#define LICENSED_V3_APPS_SOLARCAPTURE_TAP_LBN 10 +#define LICENSED_V3_APPS_SOLARCAPTURE_TAP_WIDTH 1 +#define LICENSED_V3_APPS_CAPTURE_SOLARSYSTEM_40G_LBN 11 +#define LICENSED_V3_APPS_CAPTURE_SOLARSYSTEM_40G_WIDTH 1 +#define LICENSED_V3_APPS_CAPTURE_SOLARSYSTEM_1G_LBN 12 +#define LICENSED_V3_APPS_CAPTURE_SOLARSYSTEM_1G_WIDTH 1 +#define LICENSED_V3_APPS_MASK_LBN 0 +#define LICENSED_V3_APPS_MASK_WIDTH 64 + +/* LICENSED_V3_FEATURES structuredef */ +#define LICENSED_V3_FEATURES_LEN 8 +/* Bitmask of licensed firmware features */ +#define LICENSED_V3_FEATURES_MASK_OFST 0 +#define LICENSED_V3_FEATURES_MASK_LEN 8 +#define LICENSED_V3_FEATURES_MASK_LO_OFST 0 +#define LICENSED_V3_FEATURES_MASK_HI_OFST 4 +#define LICENSED_V3_FEATURES_RX_CUT_THROUGH_LBN 0 +#define LICENSED_V3_FEATURES_RX_CUT_THROUGH_WIDTH 1 +#define LICENSED_V3_FEATURES_PIO_LBN 1 +#define LICENSED_V3_FEATURES_PIO_WIDTH 1 +#define LICENSED_V3_FEATURES_EVQ_TIMER_LBN 2 +#define LICENSED_V3_FEATURES_EVQ_TIMER_WIDTH 1 +#define LICENSED_V3_FEATURES_CLOCK_LBN 3 +#define LICENSED_V3_FEATURES_CLOCK_WIDTH 1 +#define LICENSED_V3_FEATURES_RX_TIMESTAMPS_LBN 4 +#define LICENSED_V3_FEATURES_RX_TIMESTAMPS_WIDTH 1 +#define LICENSED_V3_FEATURES_TX_TIMESTAMPS_LBN 5 +#define LICENSED_V3_FEATURES_TX_TIMESTAMPS_WIDTH 1 +#define LICENSED_V3_FEATURES_RX_SNIFF_LBN 6 +#define LICENSED_V3_FEATURES_RX_SNIFF_WIDTH 1 +#define LICENSED_V3_FEATURES_TX_SNIFF_LBN 7 +#define LICENSED_V3_FEATURES_TX_SNIFF_WIDTH 1 +#define LICENSED_V3_FEATURES_PROXY_FILTER_OPS_LBN 8 +#define LICENSED_V3_FEATURES_PROXY_FILTER_OPS_WIDTH 1 +#define LICENSED_V3_FEATURES_EVENT_CUT_THROUGH_LBN 9 +#define LICENSED_V3_FEATURES_EVENT_CUT_THROUGH_WIDTH 1 +#define LICENSED_V3_FEATURES_MASK_LBN 0 +#define LICENSED_V3_FEATURES_MASK_WIDTH 64 + +/* TX_TIMESTAMP_EVENT structuredef */ +#define TX_TIMESTAMP_EVENT_LEN 6 +/* lower 16 bits of timestamp data */ +#define TX_TIMESTAMP_EVENT_TSTAMP_DATA_LO_OFST 0 +#define TX_TIMESTAMP_EVENT_TSTAMP_DATA_LO_LEN 2 +#define TX_TIMESTAMP_EVENT_TSTAMP_DATA_LO_LBN 0 +#define TX_TIMESTAMP_EVENT_TSTAMP_DATA_LO_WIDTH 16 +/* Type of TX event, ordinary TX completion, low or high part of TX timestamp + */ +#define TX_TIMESTAMP_EVENT_TX_EV_TYPE_OFST 3 +#define TX_TIMESTAMP_EVENT_TX_EV_TYPE_LEN 1 +/* enum: This is a TX completion event, not a timestamp */ +#define TX_TIMESTAMP_EVENT_TX_EV_COMPLETION 0x0 +/* enum: This is the low part of a TX timestamp event */ +#define TX_TIMESTAMP_EVENT_TX_EV_TSTAMP_LO 0x51 +/* enum: This is the high part of a TX timestamp event */ +#define TX_TIMESTAMP_EVENT_TX_EV_TSTAMP_HI 0x52 +#define TX_TIMESTAMP_EVENT_TX_EV_TYPE_LBN 24 +#define TX_TIMESTAMP_EVENT_TX_EV_TYPE_WIDTH 8 +/* upper 16 bits of timestamp data */ +#define TX_TIMESTAMP_EVENT_TSTAMP_DATA_HI_OFST 4 +#define TX_TIMESTAMP_EVENT_TSTAMP_DATA_HI_LEN 2 +#define TX_TIMESTAMP_EVENT_TSTAMP_DATA_HI_LBN 32 +#define TX_TIMESTAMP_EVENT_TSTAMP_DATA_HI_WIDTH 16 + +/* RSS_MODE structuredef */ +#define RSS_MODE_LEN 1 +/* The RSS mode for a particular packet type is a value from 0 - 15 which can + * be considered as 4 bits selecting which fields are included in the hash. (A + * value 0 effectively disables RSS spreading for the packet type.) The YAML + * generation tools require this structure to be a whole number of bytes wide, + * but only 4 bits are relevant. + */ +#define RSS_MODE_HASH_SELECTOR_OFST 0 +#define RSS_MODE_HASH_SELECTOR_LEN 1 +#define RSS_MODE_HASH_SRC_ADDR_LBN 0 +#define RSS_MODE_HASH_SRC_ADDR_WIDTH 1 +#define RSS_MODE_HASH_DST_ADDR_LBN 1 +#define RSS_MODE_HASH_DST_ADDR_WIDTH 1 +#define RSS_MODE_HASH_SRC_PORT_LBN 2 +#define RSS_MODE_HASH_SRC_PORT_WIDTH 1 +#define RSS_MODE_HASH_DST_PORT_LBN 3 +#define RSS_MODE_HASH_DST_PORT_WIDTH 1 +#define RSS_MODE_HASH_SELECTOR_LBN 0 +#define RSS_MODE_HASH_SELECTOR_WIDTH 8 + +/* CTPIO_STATS_MAP structuredef */ +#define CTPIO_STATS_MAP_LEN 4 +/* The (function relative) VI number */ +#define CTPIO_STATS_MAP_VI_OFST 0 +#define CTPIO_STATS_MAP_VI_LEN 2 +#define CTPIO_STATS_MAP_VI_LBN 0 +#define CTPIO_STATS_MAP_VI_WIDTH 16 +/* The target bucket for the VI */ +#define CTPIO_STATS_MAP_BUCKET_OFST 2 +#define CTPIO_STATS_MAP_BUCKET_LEN 2 +#define CTPIO_STATS_MAP_BUCKET_LBN 16 +#define CTPIO_STATS_MAP_BUCKET_WIDTH 16 + + +/***********************************/ +/* MC_CMD_READ_REGS + * Get a dump of the MCPU registers + */ +#define MC_CMD_READ_REGS 0x50 +#undef MC_CMD_0x50_PRIVILEGE_CTG + +#define MC_CMD_0x50_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_READ_REGS_IN msgrequest */ +#define MC_CMD_READ_REGS_IN_LEN 0 + +/* MC_CMD_READ_REGS_OUT msgresponse */ +#define MC_CMD_READ_REGS_OUT_LEN 308 +/* Whether the corresponding register entry contains a valid value */ +#define MC_CMD_READ_REGS_OUT_MASK_OFST 0 +#define MC_CMD_READ_REGS_OUT_MASK_LEN 16 +/* Same order as MIPS GDB (r0-r31, sr, lo, hi, bad, cause, 32 x float, fsr, + * fir, fp) + */ +#define MC_CMD_READ_REGS_OUT_REGS_OFST 16 +#define MC_CMD_READ_REGS_OUT_REGS_LEN 4 +#define MC_CMD_READ_REGS_OUT_REGS_NUM 73 + + +/***********************************/ +/* MC_CMD_INIT_EVQ + * Set up an event queue according to the supplied parameters. The IN arguments + * end with an address for each 4k of host memory required to back the EVQ. + */ +#define MC_CMD_INIT_EVQ 0x80 +#undef MC_CMD_0x80_PRIVILEGE_CTG + +#define MC_CMD_0x80_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_INIT_EVQ_IN msgrequest */ +#define MC_CMD_INIT_EVQ_IN_LENMIN 44 +#define MC_CMD_INIT_EVQ_IN_LENMAX 548 +#define MC_CMD_INIT_EVQ_IN_LEN(num) (36+8*(num)) +/* Size, in entries */ +#define MC_CMD_INIT_EVQ_IN_SIZE_OFST 0 +/* Desired instance. Must be set to a specific instance, which is a function + * local queue index. + */ +#define MC_CMD_INIT_EVQ_IN_INSTANCE_OFST 4 +/* The initial timer value. The load value is ignored if the timer mode is DIS. + */ +#define MC_CMD_INIT_EVQ_IN_TMR_LOAD_OFST 8 +/* The reload value is ignored in one-shot modes */ +#define MC_CMD_INIT_EVQ_IN_TMR_RELOAD_OFST 12 +/* tbd */ +#define MC_CMD_INIT_EVQ_IN_FLAGS_OFST 16 +#define MC_CMD_INIT_EVQ_IN_FLAG_INTERRUPTING_LBN 0 +#define MC_CMD_INIT_EVQ_IN_FLAG_INTERRUPTING_WIDTH 1 +#define MC_CMD_INIT_EVQ_IN_FLAG_RPTR_DOS_LBN 1 +#define MC_CMD_INIT_EVQ_IN_FLAG_RPTR_DOS_WIDTH 1 +#define MC_CMD_INIT_EVQ_IN_FLAG_INT_ARMD_LBN 2 +#define MC_CMD_INIT_EVQ_IN_FLAG_INT_ARMD_WIDTH 1 +#define MC_CMD_INIT_EVQ_IN_FLAG_CUT_THRU_LBN 3 +#define MC_CMD_INIT_EVQ_IN_FLAG_CUT_THRU_WIDTH 1 +#define MC_CMD_INIT_EVQ_IN_FLAG_RX_MERGE_LBN 4 +#define MC_CMD_INIT_EVQ_IN_FLAG_RX_MERGE_WIDTH 1 +#define MC_CMD_INIT_EVQ_IN_FLAG_TX_MERGE_LBN 5 +#define MC_CMD_INIT_EVQ_IN_FLAG_TX_MERGE_WIDTH 1 +#define MC_CMD_INIT_EVQ_IN_FLAG_USE_TIMER_LBN 6 +#define MC_CMD_INIT_EVQ_IN_FLAG_USE_TIMER_WIDTH 1 +#define MC_CMD_INIT_EVQ_IN_TMR_MODE_OFST 20 +/* enum: Disabled */ +#define MC_CMD_INIT_EVQ_IN_TMR_MODE_DIS 0x0 +/* enum: Immediate */ +#define MC_CMD_INIT_EVQ_IN_TMR_IMMED_START 0x1 +/* enum: Triggered */ +#define MC_CMD_INIT_EVQ_IN_TMR_TRIG_START 0x2 +/* enum: Hold-off */ +#define MC_CMD_INIT_EVQ_IN_TMR_INT_HLDOFF 0x3 +/* Target EVQ for wakeups if in wakeup mode. */ +#define MC_CMD_INIT_EVQ_IN_TARGET_EVQ_OFST 24 +/* Target interrupt if in interrupting mode (note union with target EVQ). Use + * MC_CMD_RESOURCE_INSTANCE_ANY unless a specific one required for test + * purposes. + */ +#define MC_CMD_INIT_EVQ_IN_IRQ_NUM_OFST 24 +/* Event Counter Mode. */ +#define MC_CMD_INIT_EVQ_IN_COUNT_MODE_OFST 28 +/* enum: Disabled */ +#define MC_CMD_INIT_EVQ_IN_COUNT_MODE_DIS 0x0 +/* enum: Disabled */ +#define MC_CMD_INIT_EVQ_IN_COUNT_MODE_RX 0x1 +/* enum: Disabled */ +#define MC_CMD_INIT_EVQ_IN_COUNT_MODE_TX 0x2 +/* enum: Disabled */ +#define MC_CMD_INIT_EVQ_IN_COUNT_MODE_RXTX 0x3 +/* Event queue packet count threshold. */ +#define MC_CMD_INIT_EVQ_IN_COUNT_THRSHLD_OFST 32 +/* 64-bit address of 4k of 4k-aligned host memory buffer */ +#define MC_CMD_INIT_EVQ_IN_DMA_ADDR_OFST 36 +#define MC_CMD_INIT_EVQ_IN_DMA_ADDR_LEN 8 +#define MC_CMD_INIT_EVQ_IN_DMA_ADDR_LO_OFST 36 +#define MC_CMD_INIT_EVQ_IN_DMA_ADDR_HI_OFST 40 +#define MC_CMD_INIT_EVQ_IN_DMA_ADDR_MINNUM 1 +#define MC_CMD_INIT_EVQ_IN_DMA_ADDR_MAXNUM 64 + +/* MC_CMD_INIT_EVQ_OUT msgresponse */ +#define MC_CMD_INIT_EVQ_OUT_LEN 4 +/* Only valid if INTRFLAG was true */ +#define MC_CMD_INIT_EVQ_OUT_IRQ_OFST 0 + +/* MC_CMD_INIT_EVQ_V2_IN msgrequest */ +#define MC_CMD_INIT_EVQ_V2_IN_LENMIN 44 +#define MC_CMD_INIT_EVQ_V2_IN_LENMAX 548 +#define MC_CMD_INIT_EVQ_V2_IN_LEN(num) (36+8*(num)) +/* Size, in entries */ +#define MC_CMD_INIT_EVQ_V2_IN_SIZE_OFST 0 +/* Desired instance. Must be set to a specific instance, which is a function + * local queue index. + */ +#define MC_CMD_INIT_EVQ_V2_IN_INSTANCE_OFST 4 +/* The initial timer value. The load value is ignored if the timer mode is DIS. + */ +#define MC_CMD_INIT_EVQ_V2_IN_TMR_LOAD_OFST 8 +/* The reload value is ignored in one-shot modes */ +#define MC_CMD_INIT_EVQ_V2_IN_TMR_RELOAD_OFST 12 +/* tbd */ +#define MC_CMD_INIT_EVQ_V2_IN_FLAGS_OFST 16 +#define MC_CMD_INIT_EVQ_V2_IN_FLAG_INTERRUPTING_LBN 0 +#define MC_CMD_INIT_EVQ_V2_IN_FLAG_INTERRUPTING_WIDTH 1 +#define MC_CMD_INIT_EVQ_V2_IN_FLAG_RPTR_DOS_LBN 1 +#define MC_CMD_INIT_EVQ_V2_IN_FLAG_RPTR_DOS_WIDTH 1 +#define MC_CMD_INIT_EVQ_V2_IN_FLAG_INT_ARMD_LBN 2 +#define MC_CMD_INIT_EVQ_V2_IN_FLAG_INT_ARMD_WIDTH 1 +#define MC_CMD_INIT_EVQ_V2_IN_FLAG_CUT_THRU_LBN 3 +#define MC_CMD_INIT_EVQ_V2_IN_FLAG_CUT_THRU_WIDTH 1 +#define MC_CMD_INIT_EVQ_V2_IN_FLAG_RX_MERGE_LBN 4 +#define MC_CMD_INIT_EVQ_V2_IN_FLAG_RX_MERGE_WIDTH 1 +#define MC_CMD_INIT_EVQ_V2_IN_FLAG_TX_MERGE_LBN 5 +#define MC_CMD_INIT_EVQ_V2_IN_FLAG_TX_MERGE_WIDTH 1 +#define MC_CMD_INIT_EVQ_V2_IN_FLAG_USE_TIMER_LBN 6 +#define MC_CMD_INIT_EVQ_V2_IN_FLAG_USE_TIMER_WIDTH 1 +#define MC_CMD_INIT_EVQ_V2_IN_FLAG_TYPE_LBN 7 +#define MC_CMD_INIT_EVQ_V2_IN_FLAG_TYPE_WIDTH 4 +/* enum: All initialisation flags specified by host. */ +#define MC_CMD_INIT_EVQ_V2_IN_FLAG_TYPE_MANUAL 0x0 +/* enum: MEDFORD only. Certain initialisation flags specified by host may be + * over-ridden by firmware based on licenses and firmware variant in order to + * provide the lowest latency achievable. See + * MC_CMD_INIT_EVQ_V2/MC_CMD_INIT_EVQ_V2_OUT/FLAGS for list of affected flags. + */ +#define MC_CMD_INIT_EVQ_V2_IN_FLAG_TYPE_LOW_LATENCY 0x1 +/* enum: MEDFORD only. Certain initialisation flags specified by host may be + * over-ridden by firmware based on licenses and firmware variant in order to + * provide the best throughput achievable. See + * MC_CMD_INIT_EVQ_V2/MC_CMD_INIT_EVQ_V2_OUT/FLAGS for list of affected flags. + */ +#define MC_CMD_INIT_EVQ_V2_IN_FLAG_TYPE_THROUGHPUT 0x2 +/* enum: MEDFORD only. Certain initialisation flags may be over-ridden by + * firmware based on licenses and firmware variant. See + * MC_CMD_INIT_EVQ_V2/MC_CMD_INIT_EVQ_V2_OUT/FLAGS for list of affected flags. + */ +#define MC_CMD_INIT_EVQ_V2_IN_FLAG_TYPE_AUTO 0x3 +#define MC_CMD_INIT_EVQ_V2_IN_TMR_MODE_OFST 20 +/* enum: Disabled */ +#define MC_CMD_INIT_EVQ_V2_IN_TMR_MODE_DIS 0x0 +/* enum: Immediate */ +#define MC_CMD_INIT_EVQ_V2_IN_TMR_IMMED_START 0x1 +/* enum: Triggered */ +#define MC_CMD_INIT_EVQ_V2_IN_TMR_TRIG_START 0x2 +/* enum: Hold-off */ +#define MC_CMD_INIT_EVQ_V2_IN_TMR_INT_HLDOFF 0x3 +/* Target EVQ for wakeups if in wakeup mode. */ +#define MC_CMD_INIT_EVQ_V2_IN_TARGET_EVQ_OFST 24 +/* Target interrupt if in interrupting mode (note union with target EVQ). Use + * MC_CMD_RESOURCE_INSTANCE_ANY unless a specific one required for test + * purposes. + */ +#define MC_CMD_INIT_EVQ_V2_IN_IRQ_NUM_OFST 24 +/* Event Counter Mode. */ +#define MC_CMD_INIT_EVQ_V2_IN_COUNT_MODE_OFST 28 +/* enum: Disabled */ +#define MC_CMD_INIT_EVQ_V2_IN_COUNT_MODE_DIS 0x0 +/* enum: Disabled */ +#define MC_CMD_INIT_EVQ_V2_IN_COUNT_MODE_RX 0x1 +/* enum: Disabled */ +#define MC_CMD_INIT_EVQ_V2_IN_COUNT_MODE_TX 0x2 +/* enum: Disabled */ +#define MC_CMD_INIT_EVQ_V2_IN_COUNT_MODE_RXTX 0x3 +/* Event queue packet count threshold. */ +#define MC_CMD_INIT_EVQ_V2_IN_COUNT_THRSHLD_OFST 32 +/* 64-bit address of 4k of 4k-aligned host memory buffer */ +#define MC_CMD_INIT_EVQ_V2_IN_DMA_ADDR_OFST 36 +#define MC_CMD_INIT_EVQ_V2_IN_DMA_ADDR_LEN 8 +#define MC_CMD_INIT_EVQ_V2_IN_DMA_ADDR_LO_OFST 36 +#define MC_CMD_INIT_EVQ_V2_IN_DMA_ADDR_HI_OFST 40 +#define MC_CMD_INIT_EVQ_V2_IN_DMA_ADDR_MINNUM 1 +#define MC_CMD_INIT_EVQ_V2_IN_DMA_ADDR_MAXNUM 64 + +/* MC_CMD_INIT_EVQ_V2_OUT msgresponse */ +#define MC_CMD_INIT_EVQ_V2_OUT_LEN 8 +/* Only valid if INTRFLAG was true */ +#define MC_CMD_INIT_EVQ_V2_OUT_IRQ_OFST 0 +/* Actual configuration applied on the card */ +#define MC_CMD_INIT_EVQ_V2_OUT_FLAGS_OFST 4 +#define MC_CMD_INIT_EVQ_V2_OUT_FLAG_CUT_THRU_LBN 0 +#define MC_CMD_INIT_EVQ_V2_OUT_FLAG_CUT_THRU_WIDTH 1 +#define MC_CMD_INIT_EVQ_V2_OUT_FLAG_RX_MERGE_LBN 1 +#define MC_CMD_INIT_EVQ_V2_OUT_FLAG_RX_MERGE_WIDTH 1 +#define MC_CMD_INIT_EVQ_V2_OUT_FLAG_TX_MERGE_LBN 2 +#define MC_CMD_INIT_EVQ_V2_OUT_FLAG_TX_MERGE_WIDTH 1 +#define MC_CMD_INIT_EVQ_V2_OUT_FLAG_RXQ_FORCE_EV_MERGING_LBN 3 +#define MC_CMD_INIT_EVQ_V2_OUT_FLAG_RXQ_FORCE_EV_MERGING_WIDTH 1 + +/* QUEUE_CRC_MODE structuredef */ +#define QUEUE_CRC_MODE_LEN 1 +#define QUEUE_CRC_MODE_MODE_LBN 0 +#define QUEUE_CRC_MODE_MODE_WIDTH 4 +/* enum: No CRC. */ +#define QUEUE_CRC_MODE_NONE 0x0 +/* enum: CRC Fiber channel over ethernet. */ +#define QUEUE_CRC_MODE_FCOE 0x1 +/* enum: CRC (digest) iSCSI header only. */ +#define QUEUE_CRC_MODE_ISCSI_HDR 0x2 +/* enum: CRC (digest) iSCSI header and payload. */ +#define QUEUE_CRC_MODE_ISCSI 0x3 +/* enum: CRC Fiber channel over IP over ethernet. */ +#define QUEUE_CRC_MODE_FCOIPOE 0x4 +/* enum: CRC MPA. */ +#define QUEUE_CRC_MODE_MPA 0x5 +#define QUEUE_CRC_MODE_SPARE_LBN 4 +#define QUEUE_CRC_MODE_SPARE_WIDTH 4 + + +/***********************************/ +/* MC_CMD_INIT_RXQ + * set up a receive queue according to the supplied parameters. The IN + * arguments end with an address for each 4k of host memory required to back + * the RXQ. + */ +#define MC_CMD_INIT_RXQ 0x81 +#undef MC_CMD_0x81_PRIVILEGE_CTG + +#define MC_CMD_0x81_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_INIT_RXQ_IN msgrequest: Legacy RXQ_INIT request. Use extended version + * in new code. + */ +#define MC_CMD_INIT_RXQ_IN_LENMIN 36 +#define MC_CMD_INIT_RXQ_IN_LENMAX 252 +#define MC_CMD_INIT_RXQ_IN_LEN(num) (28+8*(num)) +/* Size, in entries */ +#define MC_CMD_INIT_RXQ_IN_SIZE_OFST 0 +/* The EVQ to send events to. This is an index originally specified to INIT_EVQ + */ +#define MC_CMD_INIT_RXQ_IN_TARGET_EVQ_OFST 4 +/* The value to put in the event data. Check hardware spec. for valid range. */ +#define MC_CMD_INIT_RXQ_IN_LABEL_OFST 8 +/* Desired instance. Must be set to a specific instance, which is a function + * local queue index. + */ +#define MC_CMD_INIT_RXQ_IN_INSTANCE_OFST 12 +/* There will be more flags here. */ +#define MC_CMD_INIT_RXQ_IN_FLAGS_OFST 16 +#define MC_CMD_INIT_RXQ_IN_FLAG_BUFF_MODE_LBN 0 +#define MC_CMD_INIT_RXQ_IN_FLAG_BUFF_MODE_WIDTH 1 +#define MC_CMD_INIT_RXQ_IN_FLAG_HDR_SPLIT_LBN 1 +#define MC_CMD_INIT_RXQ_IN_FLAG_HDR_SPLIT_WIDTH 1 +#define MC_CMD_INIT_RXQ_IN_FLAG_TIMESTAMP_LBN 2 +#define MC_CMD_INIT_RXQ_IN_FLAG_TIMESTAMP_WIDTH 1 +#define MC_CMD_INIT_RXQ_IN_CRC_MODE_LBN 3 +#define MC_CMD_INIT_RXQ_IN_CRC_MODE_WIDTH 4 +#define MC_CMD_INIT_RXQ_IN_FLAG_CHAIN_LBN 7 +#define MC_CMD_INIT_RXQ_IN_FLAG_CHAIN_WIDTH 1 +#define MC_CMD_INIT_RXQ_IN_FLAG_PREFIX_LBN 8 +#define MC_CMD_INIT_RXQ_IN_FLAG_PREFIX_WIDTH 1 +#define MC_CMD_INIT_RXQ_IN_FLAG_DISABLE_SCATTER_LBN 9 +#define MC_CMD_INIT_RXQ_IN_FLAG_DISABLE_SCATTER_WIDTH 1 +#define MC_CMD_INIT_RXQ_IN_UNUSED_LBN 10 +#define MC_CMD_INIT_RXQ_IN_UNUSED_WIDTH 1 +/* Owner ID to use if in buffer mode (zero if physical) */ +#define MC_CMD_INIT_RXQ_IN_OWNER_ID_OFST 20 +/* The port ID associated with the v-adaptor which should contain this DMAQ. */ +#define MC_CMD_INIT_RXQ_IN_PORT_ID_OFST 24 +/* 64-bit address of 4k of 4k-aligned host memory buffer */ +#define MC_CMD_INIT_RXQ_IN_DMA_ADDR_OFST 28 +#define MC_CMD_INIT_RXQ_IN_DMA_ADDR_LEN 8 +#define MC_CMD_INIT_RXQ_IN_DMA_ADDR_LO_OFST 28 +#define MC_CMD_INIT_RXQ_IN_DMA_ADDR_HI_OFST 32 +#define MC_CMD_INIT_RXQ_IN_DMA_ADDR_MINNUM 1 +#define MC_CMD_INIT_RXQ_IN_DMA_ADDR_MAXNUM 28 + +/* MC_CMD_INIT_RXQ_EXT_IN msgrequest: Extended RXQ_INIT with additional mode + * flags + */ +#define MC_CMD_INIT_RXQ_EXT_IN_LEN 544 +/* Size, in entries */ +#define MC_CMD_INIT_RXQ_EXT_IN_SIZE_OFST 0 +/* The EVQ to send events to. This is an index originally specified to INIT_EVQ + */ +#define MC_CMD_INIT_RXQ_EXT_IN_TARGET_EVQ_OFST 4 +/* The value to put in the event data. Check hardware spec. for valid range. */ +#define MC_CMD_INIT_RXQ_EXT_IN_LABEL_OFST 8 +/* Desired instance. Must be set to a specific instance, which is a function + * local queue index. + */ +#define MC_CMD_INIT_RXQ_EXT_IN_INSTANCE_OFST 12 +/* There will be more flags here. */ +#define MC_CMD_INIT_RXQ_EXT_IN_FLAGS_OFST 16 +#define MC_CMD_INIT_RXQ_EXT_IN_FLAG_BUFF_MODE_LBN 0 +#define MC_CMD_INIT_RXQ_EXT_IN_FLAG_BUFF_MODE_WIDTH 1 +#define MC_CMD_INIT_RXQ_EXT_IN_FLAG_HDR_SPLIT_LBN 1 +#define MC_CMD_INIT_RXQ_EXT_IN_FLAG_HDR_SPLIT_WIDTH 1 +#define MC_CMD_INIT_RXQ_EXT_IN_FLAG_TIMESTAMP_LBN 2 +#define MC_CMD_INIT_RXQ_EXT_IN_FLAG_TIMESTAMP_WIDTH 1 +#define MC_CMD_INIT_RXQ_EXT_IN_CRC_MODE_LBN 3 +#define MC_CMD_INIT_RXQ_EXT_IN_CRC_MODE_WIDTH 4 +#define MC_CMD_INIT_RXQ_EXT_IN_FLAG_CHAIN_LBN 7 +#define MC_CMD_INIT_RXQ_EXT_IN_FLAG_CHAIN_WIDTH 1 +#define MC_CMD_INIT_RXQ_EXT_IN_FLAG_PREFIX_LBN 8 +#define MC_CMD_INIT_RXQ_EXT_IN_FLAG_PREFIX_WIDTH 1 +#define MC_CMD_INIT_RXQ_EXT_IN_FLAG_DISABLE_SCATTER_LBN 9 +#define MC_CMD_INIT_RXQ_EXT_IN_FLAG_DISABLE_SCATTER_WIDTH 1 +#define MC_CMD_INIT_RXQ_EXT_IN_DMA_MODE_LBN 10 +#define MC_CMD_INIT_RXQ_EXT_IN_DMA_MODE_WIDTH 4 +/* enum: One packet per descriptor (for normal networking) */ +#define MC_CMD_INIT_RXQ_EXT_IN_SINGLE_PACKET 0x0 +/* enum: Pack multiple packets into large descriptors (for SolarCapture) */ +#define MC_CMD_INIT_RXQ_EXT_IN_PACKED_STREAM 0x1 +#define MC_CMD_INIT_RXQ_EXT_IN_FLAG_SNAPSHOT_MODE_LBN 14 +#define MC_CMD_INIT_RXQ_EXT_IN_FLAG_SNAPSHOT_MODE_WIDTH 1 +#define MC_CMD_INIT_RXQ_EXT_IN_PACKED_STREAM_BUFF_SIZE_LBN 15 +#define MC_CMD_INIT_RXQ_EXT_IN_PACKED_STREAM_BUFF_SIZE_WIDTH 3 +#define MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_1M 0x0 /* enum */ +#define MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_512K 0x1 /* enum */ +#define MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_256K 0x2 /* enum */ +#define MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_128K 0x3 /* enum */ +#define MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_64K 0x4 /* enum */ +#define MC_CMD_INIT_RXQ_EXT_IN_FLAG_WANT_OUTER_CLASSES_LBN 18 +#define MC_CMD_INIT_RXQ_EXT_IN_FLAG_WANT_OUTER_CLASSES_WIDTH 1 +#define MC_CMD_INIT_RXQ_EXT_IN_FLAG_FORCE_EV_MERGING_LBN 19 +#define MC_CMD_INIT_RXQ_EXT_IN_FLAG_FORCE_EV_MERGING_WIDTH 1 +/* Owner ID to use if in buffer mode (zero if physical) */ +#define MC_CMD_INIT_RXQ_EXT_IN_OWNER_ID_OFST 20 +/* The port ID associated with the v-adaptor which should contain this DMAQ. */ +#define MC_CMD_INIT_RXQ_EXT_IN_PORT_ID_OFST 24 +/* 64-bit address of 4k of 4k-aligned host memory buffer */ +#define MC_CMD_INIT_RXQ_EXT_IN_DMA_ADDR_OFST 28 +#define MC_CMD_INIT_RXQ_EXT_IN_DMA_ADDR_LEN 8 +#define MC_CMD_INIT_RXQ_EXT_IN_DMA_ADDR_LO_OFST 28 +#define MC_CMD_INIT_RXQ_EXT_IN_DMA_ADDR_HI_OFST 32 +#define MC_CMD_INIT_RXQ_EXT_IN_DMA_ADDR_NUM 64 +/* Maximum length of packet to receive, if SNAPSHOT_MODE flag is set */ +#define MC_CMD_INIT_RXQ_EXT_IN_SNAPSHOT_LENGTH_OFST 540 + +/* MC_CMD_INIT_RXQ_OUT msgresponse */ +#define MC_CMD_INIT_RXQ_OUT_LEN 0 + +/* MC_CMD_INIT_RXQ_EXT_OUT msgresponse */ +#define MC_CMD_INIT_RXQ_EXT_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_INIT_TXQ + */ +#define MC_CMD_INIT_TXQ 0x82 +#undef MC_CMD_0x82_PRIVILEGE_CTG + +#define MC_CMD_0x82_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_INIT_TXQ_IN msgrequest: Legacy INIT_TXQ request. Use extended version + * in new code. + */ +#define MC_CMD_INIT_TXQ_IN_LENMIN 36 +#define MC_CMD_INIT_TXQ_IN_LENMAX 252 +#define MC_CMD_INIT_TXQ_IN_LEN(num) (28+8*(num)) +/* Size, in entries */ +#define MC_CMD_INIT_TXQ_IN_SIZE_OFST 0 +/* The EVQ to send events to. This is an index originally specified to + * INIT_EVQ. + */ +#define MC_CMD_INIT_TXQ_IN_TARGET_EVQ_OFST 4 +/* The value to put in the event data. Check hardware spec. for valid range. */ +#define MC_CMD_INIT_TXQ_IN_LABEL_OFST 8 +/* Desired instance. Must be set to a specific instance, which is a function + * local queue index. + */ +#define MC_CMD_INIT_TXQ_IN_INSTANCE_OFST 12 +/* There will be more flags here. */ +#define MC_CMD_INIT_TXQ_IN_FLAGS_OFST 16 +#define MC_CMD_INIT_TXQ_IN_FLAG_BUFF_MODE_LBN 0 +#define MC_CMD_INIT_TXQ_IN_FLAG_BUFF_MODE_WIDTH 1 +#define MC_CMD_INIT_TXQ_IN_FLAG_IP_CSUM_DIS_LBN 1 +#define MC_CMD_INIT_TXQ_IN_FLAG_IP_CSUM_DIS_WIDTH 1 +#define MC_CMD_INIT_TXQ_IN_FLAG_TCP_CSUM_DIS_LBN 2 +#define MC_CMD_INIT_TXQ_IN_FLAG_TCP_CSUM_DIS_WIDTH 1 +#define MC_CMD_INIT_TXQ_IN_FLAG_TCP_UDP_ONLY_LBN 3 +#define MC_CMD_INIT_TXQ_IN_FLAG_TCP_UDP_ONLY_WIDTH 1 +#define MC_CMD_INIT_TXQ_IN_CRC_MODE_LBN 4 +#define MC_CMD_INIT_TXQ_IN_CRC_MODE_WIDTH 4 +#define MC_CMD_INIT_TXQ_IN_FLAG_TIMESTAMP_LBN 8 +#define MC_CMD_INIT_TXQ_IN_FLAG_TIMESTAMP_WIDTH 1 +#define MC_CMD_INIT_TXQ_IN_FLAG_PACER_BYPASS_LBN 9 +#define MC_CMD_INIT_TXQ_IN_FLAG_PACER_BYPASS_WIDTH 1 +#define MC_CMD_INIT_TXQ_IN_FLAG_INNER_IP_CSUM_EN_LBN 10 +#define MC_CMD_INIT_TXQ_IN_FLAG_INNER_IP_CSUM_EN_WIDTH 1 +#define MC_CMD_INIT_TXQ_IN_FLAG_INNER_TCP_CSUM_EN_LBN 11 +#define MC_CMD_INIT_TXQ_IN_FLAG_INNER_TCP_CSUM_EN_WIDTH 1 +/* Owner ID to use if in buffer mode (zero if physical) */ +#define MC_CMD_INIT_TXQ_IN_OWNER_ID_OFST 20 +/* The port ID associated with the v-adaptor which should contain this DMAQ. */ +#define MC_CMD_INIT_TXQ_IN_PORT_ID_OFST 24 +/* 64-bit address of 4k of 4k-aligned host memory buffer */ +#define MC_CMD_INIT_TXQ_IN_DMA_ADDR_OFST 28 +#define MC_CMD_INIT_TXQ_IN_DMA_ADDR_LEN 8 +#define MC_CMD_INIT_TXQ_IN_DMA_ADDR_LO_OFST 28 +#define MC_CMD_INIT_TXQ_IN_DMA_ADDR_HI_OFST 32 +#define MC_CMD_INIT_TXQ_IN_DMA_ADDR_MINNUM 1 +#define MC_CMD_INIT_TXQ_IN_DMA_ADDR_MAXNUM 28 + +/* MC_CMD_INIT_TXQ_EXT_IN msgrequest: Extended INIT_TXQ with additional mode + * flags + */ +#define MC_CMD_INIT_TXQ_EXT_IN_LEN 544 +/* Size, in entries */ +#define MC_CMD_INIT_TXQ_EXT_IN_SIZE_OFST 0 +/* The EVQ to send events to. This is an index originally specified to + * INIT_EVQ. + */ +#define MC_CMD_INIT_TXQ_EXT_IN_TARGET_EVQ_OFST 4 +/* The value to put in the event data. Check hardware spec. for valid range. */ +#define MC_CMD_INIT_TXQ_EXT_IN_LABEL_OFST 8 +/* Desired instance. Must be set to a specific instance, which is a function + * local queue index. + */ +#define MC_CMD_INIT_TXQ_EXT_IN_INSTANCE_OFST 12 +/* There will be more flags here. */ +#define MC_CMD_INIT_TXQ_EXT_IN_FLAGS_OFST 16 +#define MC_CMD_INIT_TXQ_EXT_IN_FLAG_BUFF_MODE_LBN 0 +#define MC_CMD_INIT_TXQ_EXT_IN_FLAG_BUFF_MODE_WIDTH 1 +#define MC_CMD_INIT_TXQ_EXT_IN_FLAG_IP_CSUM_DIS_LBN 1 +#define MC_CMD_INIT_TXQ_EXT_IN_FLAG_IP_CSUM_DIS_WIDTH 1 +#define MC_CMD_INIT_TXQ_EXT_IN_FLAG_TCP_CSUM_DIS_LBN 2 +#define MC_CMD_INIT_TXQ_EXT_IN_FLAG_TCP_CSUM_DIS_WIDTH 1 +#define MC_CMD_INIT_TXQ_EXT_IN_FLAG_TCP_UDP_ONLY_LBN 3 +#define MC_CMD_INIT_TXQ_EXT_IN_FLAG_TCP_UDP_ONLY_WIDTH 1 +#define MC_CMD_INIT_TXQ_EXT_IN_CRC_MODE_LBN 4 +#define MC_CMD_INIT_TXQ_EXT_IN_CRC_MODE_WIDTH 4 +#define MC_CMD_INIT_TXQ_EXT_IN_FLAG_TIMESTAMP_LBN 8 +#define MC_CMD_INIT_TXQ_EXT_IN_FLAG_TIMESTAMP_WIDTH 1 +#define MC_CMD_INIT_TXQ_EXT_IN_FLAG_PACER_BYPASS_LBN 9 +#define MC_CMD_INIT_TXQ_EXT_IN_FLAG_PACER_BYPASS_WIDTH 1 +#define MC_CMD_INIT_TXQ_EXT_IN_FLAG_INNER_IP_CSUM_EN_LBN 10 +#define MC_CMD_INIT_TXQ_EXT_IN_FLAG_INNER_IP_CSUM_EN_WIDTH 1 +#define MC_CMD_INIT_TXQ_EXT_IN_FLAG_INNER_TCP_CSUM_EN_LBN 11 +#define MC_CMD_INIT_TXQ_EXT_IN_FLAG_INNER_TCP_CSUM_EN_WIDTH 1 +#define MC_CMD_INIT_TXQ_EXT_IN_FLAG_TSOV2_EN_LBN 12 +#define MC_CMD_INIT_TXQ_EXT_IN_FLAG_TSOV2_EN_WIDTH 1 +#define MC_CMD_INIT_TXQ_EXT_IN_FLAG_CTPIO_LBN 13 +#define MC_CMD_INIT_TXQ_EXT_IN_FLAG_CTPIO_WIDTH 1 +/* Owner ID to use if in buffer mode (zero if physical) */ +#define MC_CMD_INIT_TXQ_EXT_IN_OWNER_ID_OFST 20 +/* The port ID associated with the v-adaptor which should contain this DMAQ. */ +#define MC_CMD_INIT_TXQ_EXT_IN_PORT_ID_OFST 24 +/* 64-bit address of 4k of 4k-aligned host memory buffer */ +#define MC_CMD_INIT_TXQ_EXT_IN_DMA_ADDR_OFST 28 +#define MC_CMD_INIT_TXQ_EXT_IN_DMA_ADDR_LEN 8 +#define MC_CMD_INIT_TXQ_EXT_IN_DMA_ADDR_LO_OFST 28 +#define MC_CMD_INIT_TXQ_EXT_IN_DMA_ADDR_HI_OFST 32 +#define MC_CMD_INIT_TXQ_EXT_IN_DMA_ADDR_MINNUM 1 +#define MC_CMD_INIT_TXQ_EXT_IN_DMA_ADDR_MAXNUM 64 +/* Flags related to Qbb flow control mode. */ +#define MC_CMD_INIT_TXQ_EXT_IN_QBB_FLAGS_OFST 540 +#define MC_CMD_INIT_TXQ_EXT_IN_QBB_ENABLE_LBN 0 +#define MC_CMD_INIT_TXQ_EXT_IN_QBB_ENABLE_WIDTH 1 +#define MC_CMD_INIT_TXQ_EXT_IN_QBB_PRIORITY_LBN 1 +#define MC_CMD_INIT_TXQ_EXT_IN_QBB_PRIORITY_WIDTH 3 + +/* MC_CMD_INIT_TXQ_OUT msgresponse */ +#define MC_CMD_INIT_TXQ_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_FINI_EVQ + * Teardown an EVQ. + * + * All DMAQs or EVQs that point to the EVQ to tear down must be torn down first + * or the operation will fail with EBUSY + */ +#define MC_CMD_FINI_EVQ 0x83 +#undef MC_CMD_0x83_PRIVILEGE_CTG + +#define MC_CMD_0x83_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_FINI_EVQ_IN msgrequest */ +#define MC_CMD_FINI_EVQ_IN_LEN 4 +/* Instance of EVQ to destroy. Should be the same instance as that previously + * passed to INIT_EVQ + */ +#define MC_CMD_FINI_EVQ_IN_INSTANCE_OFST 0 + +/* MC_CMD_FINI_EVQ_OUT msgresponse */ +#define MC_CMD_FINI_EVQ_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_FINI_RXQ + * Teardown a RXQ. + */ +#define MC_CMD_FINI_RXQ 0x84 +#undef MC_CMD_0x84_PRIVILEGE_CTG + +#define MC_CMD_0x84_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_FINI_RXQ_IN msgrequest */ +#define MC_CMD_FINI_RXQ_IN_LEN 4 +/* Instance of RXQ to destroy */ +#define MC_CMD_FINI_RXQ_IN_INSTANCE_OFST 0 + +/* MC_CMD_FINI_RXQ_OUT msgresponse */ +#define MC_CMD_FINI_RXQ_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_FINI_TXQ + * Teardown a TXQ. + */ +#define MC_CMD_FINI_TXQ 0x85 +#undef MC_CMD_0x85_PRIVILEGE_CTG + +#define MC_CMD_0x85_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_FINI_TXQ_IN msgrequest */ +#define MC_CMD_FINI_TXQ_IN_LEN 4 +/* Instance of TXQ to destroy */ +#define MC_CMD_FINI_TXQ_IN_INSTANCE_OFST 0 + +/* MC_CMD_FINI_TXQ_OUT msgresponse */ +#define MC_CMD_FINI_TXQ_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_DRIVER_EVENT + * Generate an event on an EVQ belonging to the function issuing the command. + */ +#define MC_CMD_DRIVER_EVENT 0x86 +#undef MC_CMD_0x86_PRIVILEGE_CTG + +#define MC_CMD_0x86_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_DRIVER_EVENT_IN msgrequest */ +#define MC_CMD_DRIVER_EVENT_IN_LEN 12 +/* Handle of target EVQ */ +#define MC_CMD_DRIVER_EVENT_IN_EVQ_OFST 0 +/* Bits 0 - 63 of event */ +#define MC_CMD_DRIVER_EVENT_IN_DATA_OFST 4 +#define MC_CMD_DRIVER_EVENT_IN_DATA_LEN 8 +#define MC_CMD_DRIVER_EVENT_IN_DATA_LO_OFST 4 +#define MC_CMD_DRIVER_EVENT_IN_DATA_HI_OFST 8 + +/* MC_CMD_DRIVER_EVENT_OUT msgresponse */ +#define MC_CMD_DRIVER_EVENT_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_PROXY_CMD + * Execute an arbitrary MCDI command on behalf of a different function, subject + * to security restrictions. The command to be proxied follows immediately + * afterward in the host buffer (or on the UART). This command supercedes + * MC_CMD_SET_FUNC, which remains available for Siena but now deprecated. + */ +#define MC_CMD_PROXY_CMD 0x5b +#undef MC_CMD_0x5b_PRIVILEGE_CTG + +#define MC_CMD_0x5b_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_PROXY_CMD_IN msgrequest */ +#define MC_CMD_PROXY_CMD_IN_LEN 4 +/* The handle of the target function. */ +#define MC_CMD_PROXY_CMD_IN_TARGET_OFST 0 +#define MC_CMD_PROXY_CMD_IN_TARGET_PF_LBN 0 +#define MC_CMD_PROXY_CMD_IN_TARGET_PF_WIDTH 16 +#define MC_CMD_PROXY_CMD_IN_TARGET_VF_LBN 16 +#define MC_CMD_PROXY_CMD_IN_TARGET_VF_WIDTH 16 +#define MC_CMD_PROXY_CMD_IN_VF_NULL 0xffff /* enum */ + +/* MC_CMD_PROXY_CMD_OUT msgresponse */ +#define MC_CMD_PROXY_CMD_OUT_LEN 0 + +/* MC_PROXY_STATUS_BUFFER structuredef: Host memory status buffer used to + * manage proxied requests + */ +#define MC_PROXY_STATUS_BUFFER_LEN 16 +/* Handle allocated by the firmware for this proxy transaction */ +#define MC_PROXY_STATUS_BUFFER_HANDLE_OFST 0 +/* enum: An invalid handle. */ +#define MC_PROXY_STATUS_BUFFER_HANDLE_INVALID 0x0 +#define MC_PROXY_STATUS_BUFFER_HANDLE_LBN 0 +#define MC_PROXY_STATUS_BUFFER_HANDLE_WIDTH 32 +/* The requesting physical function number */ +#define MC_PROXY_STATUS_BUFFER_PF_OFST 4 +#define MC_PROXY_STATUS_BUFFER_PF_LEN 2 +#define MC_PROXY_STATUS_BUFFER_PF_LBN 32 +#define MC_PROXY_STATUS_BUFFER_PF_WIDTH 16 +/* The requesting virtual function number. Set to VF_NULL if the target is a + * PF. + */ +#define MC_PROXY_STATUS_BUFFER_VF_OFST 6 +#define MC_PROXY_STATUS_BUFFER_VF_LEN 2 +#define MC_PROXY_STATUS_BUFFER_VF_LBN 48 +#define MC_PROXY_STATUS_BUFFER_VF_WIDTH 16 +/* The target function RID. */ +#define MC_PROXY_STATUS_BUFFER_RID_OFST 8 +#define MC_PROXY_STATUS_BUFFER_RID_LEN 2 +#define MC_PROXY_STATUS_BUFFER_RID_LBN 64 +#define MC_PROXY_STATUS_BUFFER_RID_WIDTH 16 +/* The status of the proxy as described in MC_CMD_PROXY_COMPLETE. */ +#define MC_PROXY_STATUS_BUFFER_STATUS_OFST 10 +#define MC_PROXY_STATUS_BUFFER_STATUS_LEN 2 +#define MC_PROXY_STATUS_BUFFER_STATUS_LBN 80 +#define MC_PROXY_STATUS_BUFFER_STATUS_WIDTH 16 +/* If a request is authorized rather than carried out by the host, this is the + * elevated privilege mask granted to the requesting function. + */ +#define MC_PROXY_STATUS_BUFFER_GRANTED_PRIVILEGES_OFST 12 +#define MC_PROXY_STATUS_BUFFER_GRANTED_PRIVILEGES_LBN 96 +#define MC_PROXY_STATUS_BUFFER_GRANTED_PRIVILEGES_WIDTH 32 + + +/***********************************/ +/* MC_CMD_PROXY_CONFIGURE + * Enable/disable authorization of MCDI requests from unprivileged functions by + * a designated admin function + */ +#define MC_CMD_PROXY_CONFIGURE 0x58 +#undef MC_CMD_0x58_PRIVILEGE_CTG + +#define MC_CMD_0x58_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_PROXY_CONFIGURE_IN msgrequest */ +#define MC_CMD_PROXY_CONFIGURE_IN_LEN 108 +#define MC_CMD_PROXY_CONFIGURE_IN_FLAGS_OFST 0 +#define MC_CMD_PROXY_CONFIGURE_IN_ENABLE_LBN 0 +#define MC_CMD_PROXY_CONFIGURE_IN_ENABLE_WIDTH 1 +/* Host provides a contiguous memory buffer that contains at least NUM_BLOCKS + * of blocks, each of the size REQUEST_BLOCK_SIZE. + */ +#define MC_CMD_PROXY_CONFIGURE_IN_STATUS_BUFF_ADDR_OFST 4 +#define MC_CMD_PROXY_CONFIGURE_IN_STATUS_BUFF_ADDR_LEN 8 +#define MC_CMD_PROXY_CONFIGURE_IN_STATUS_BUFF_ADDR_LO_OFST 4 +#define MC_CMD_PROXY_CONFIGURE_IN_STATUS_BUFF_ADDR_HI_OFST 8 +/* Must be a power of 2 */ +#define MC_CMD_PROXY_CONFIGURE_IN_STATUS_BLOCK_SIZE_OFST 12 +/* Host provides a contiguous memory buffer that contains at least NUM_BLOCKS + * of blocks, each of the size REPLY_BLOCK_SIZE. + */ +#define MC_CMD_PROXY_CONFIGURE_IN_REQUEST_BUFF_ADDR_OFST 16 +#define MC_CMD_PROXY_CONFIGURE_IN_REQUEST_BUFF_ADDR_LEN 8 +#define MC_CMD_PROXY_CONFIGURE_IN_REQUEST_BUFF_ADDR_LO_OFST 16 +#define MC_CMD_PROXY_CONFIGURE_IN_REQUEST_BUFF_ADDR_HI_OFST 20 +/* Must be a power of 2 */ +#define MC_CMD_PROXY_CONFIGURE_IN_REQUEST_BLOCK_SIZE_OFST 24 +/* Host provides a contiguous memory buffer that contains at least NUM_BLOCKS + * of blocks, each of the size STATUS_BLOCK_SIZE. This buffer is only needed if + * host intends to complete proxied operations by using MC_CMD_PROXY_CMD. + */ +#define MC_CMD_PROXY_CONFIGURE_IN_REPLY_BUFF_ADDR_OFST 28 +#define MC_CMD_PROXY_CONFIGURE_IN_REPLY_BUFF_ADDR_LEN 8 +#define MC_CMD_PROXY_CONFIGURE_IN_REPLY_BUFF_ADDR_LO_OFST 28 +#define MC_CMD_PROXY_CONFIGURE_IN_REPLY_BUFF_ADDR_HI_OFST 32 +/* Must be a power of 2, or zero if this buffer is not provided */ +#define MC_CMD_PROXY_CONFIGURE_IN_REPLY_BLOCK_SIZE_OFST 36 +/* Applies to all three buffers */ +#define MC_CMD_PROXY_CONFIGURE_IN_NUM_BLOCKS_OFST 40 +/* A bit mask defining which MCDI operations may be proxied */ +#define MC_CMD_PROXY_CONFIGURE_IN_ALLOWED_MCDI_MASK_OFST 44 +#define MC_CMD_PROXY_CONFIGURE_IN_ALLOWED_MCDI_MASK_LEN 64 + +/* MC_CMD_PROXY_CONFIGURE_EXT_IN msgrequest */ +#define MC_CMD_PROXY_CONFIGURE_EXT_IN_LEN 112 +#define MC_CMD_PROXY_CONFIGURE_EXT_IN_FLAGS_OFST 0 +#define MC_CMD_PROXY_CONFIGURE_EXT_IN_ENABLE_LBN 0 +#define MC_CMD_PROXY_CONFIGURE_EXT_IN_ENABLE_WIDTH 1 +/* Host provides a contiguous memory buffer that contains at least NUM_BLOCKS + * of blocks, each of the size REQUEST_BLOCK_SIZE. + */ +#define MC_CMD_PROXY_CONFIGURE_EXT_IN_STATUS_BUFF_ADDR_OFST 4 +#define MC_CMD_PROXY_CONFIGURE_EXT_IN_STATUS_BUFF_ADDR_LEN 8 +#define MC_CMD_PROXY_CONFIGURE_EXT_IN_STATUS_BUFF_ADDR_LO_OFST 4 +#define MC_CMD_PROXY_CONFIGURE_EXT_IN_STATUS_BUFF_ADDR_HI_OFST 8 +/* Must be a power of 2 */ +#define MC_CMD_PROXY_CONFIGURE_EXT_IN_STATUS_BLOCK_SIZE_OFST 12 +/* Host provides a contiguous memory buffer that contains at least NUM_BLOCKS + * of blocks, each of the size REPLY_BLOCK_SIZE. + */ +#define MC_CMD_PROXY_CONFIGURE_EXT_IN_REQUEST_BUFF_ADDR_OFST 16 +#define MC_CMD_PROXY_CONFIGURE_EXT_IN_REQUEST_BUFF_ADDR_LEN 8 +#define MC_CMD_PROXY_CONFIGURE_EXT_IN_REQUEST_BUFF_ADDR_LO_OFST 16 +#define MC_CMD_PROXY_CONFIGURE_EXT_IN_REQUEST_BUFF_ADDR_HI_OFST 20 +/* Must be a power of 2 */ +#define MC_CMD_PROXY_CONFIGURE_EXT_IN_REQUEST_BLOCK_SIZE_OFST 24 +/* Host provides a contiguous memory buffer that contains at least NUM_BLOCKS + * of blocks, each of the size STATUS_BLOCK_SIZE. This buffer is only needed if + * host intends to complete proxied operations by using MC_CMD_PROXY_CMD. + */ +#define MC_CMD_PROXY_CONFIGURE_EXT_IN_REPLY_BUFF_ADDR_OFST 28 +#define MC_CMD_PROXY_CONFIGURE_EXT_IN_REPLY_BUFF_ADDR_LEN 8 +#define MC_CMD_PROXY_CONFIGURE_EXT_IN_REPLY_BUFF_ADDR_LO_OFST 28 +#define MC_CMD_PROXY_CONFIGURE_EXT_IN_REPLY_BUFF_ADDR_HI_OFST 32 +/* Must be a power of 2, or zero if this buffer is not provided */ +#define MC_CMD_PROXY_CONFIGURE_EXT_IN_REPLY_BLOCK_SIZE_OFST 36 +/* Applies to all three buffers */ +#define MC_CMD_PROXY_CONFIGURE_EXT_IN_NUM_BLOCKS_OFST 40 +/* A bit mask defining which MCDI operations may be proxied */ +#define MC_CMD_PROXY_CONFIGURE_EXT_IN_ALLOWED_MCDI_MASK_OFST 44 +#define MC_CMD_PROXY_CONFIGURE_EXT_IN_ALLOWED_MCDI_MASK_LEN 64 +#define MC_CMD_PROXY_CONFIGURE_EXT_IN_RESERVED_OFST 108 + +/* MC_CMD_PROXY_CONFIGURE_OUT msgresponse */ +#define MC_CMD_PROXY_CONFIGURE_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_PROXY_COMPLETE + * Tells FW that a requested proxy operation has either been completed (by + * using MC_CMD_PROXY_CMD) or authorized/declined. May only be sent by the + * function that enabled proxying/authorization (by using + * MC_CMD_PROXY_CONFIGURE). + */ +#define MC_CMD_PROXY_COMPLETE 0x5f +#undef MC_CMD_0x5f_PRIVILEGE_CTG + +#define MC_CMD_0x5f_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_PROXY_COMPLETE_IN msgrequest */ +#define MC_CMD_PROXY_COMPLETE_IN_LEN 12 +#define MC_CMD_PROXY_COMPLETE_IN_BLOCK_INDEX_OFST 0 +#define MC_CMD_PROXY_COMPLETE_IN_STATUS_OFST 4 +/* enum: The operation has been completed by using MC_CMD_PROXY_CMD, the reply + * is stored in the REPLY_BUFF. + */ +#define MC_CMD_PROXY_COMPLETE_IN_COMPLETE 0x0 +/* enum: The operation has been authorized. The originating function may now + * try again. + */ +#define MC_CMD_PROXY_COMPLETE_IN_AUTHORIZED 0x1 +/* enum: The operation has been declined. */ +#define MC_CMD_PROXY_COMPLETE_IN_DECLINED 0x2 +/* enum: The authorization failed because the relevant application did not + * respond in time. + */ +#define MC_CMD_PROXY_COMPLETE_IN_TIMEDOUT 0x3 +#define MC_CMD_PROXY_COMPLETE_IN_HANDLE_OFST 8 + +/* MC_CMD_PROXY_COMPLETE_OUT msgresponse */ +#define MC_CMD_PROXY_COMPLETE_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_ALLOC_BUFTBL_CHUNK + * Allocate a set of buffer table entries using the specified owner ID. This + * operation allocates the required buffer table entries (and fails if it + * cannot do so). The buffer table entries will initially be zeroed. + */ +#define MC_CMD_ALLOC_BUFTBL_CHUNK 0x87 +#undef MC_CMD_0x87_PRIVILEGE_CTG + +#define MC_CMD_0x87_PRIVILEGE_CTG SRIOV_CTG_ONLOAD + +/* MC_CMD_ALLOC_BUFTBL_CHUNK_IN msgrequest */ +#define MC_CMD_ALLOC_BUFTBL_CHUNK_IN_LEN 8 +/* Owner ID to use */ +#define MC_CMD_ALLOC_BUFTBL_CHUNK_IN_OWNER_OFST 0 +/* Size of buffer table pages to use, in bytes (note that only a few values are + * legal on any specific hardware). + */ +#define MC_CMD_ALLOC_BUFTBL_CHUNK_IN_PAGE_SIZE_OFST 4 + +/* MC_CMD_ALLOC_BUFTBL_CHUNK_OUT msgresponse */ +#define MC_CMD_ALLOC_BUFTBL_CHUNK_OUT_LEN 12 +#define MC_CMD_ALLOC_BUFTBL_CHUNK_OUT_HANDLE_OFST 0 +#define MC_CMD_ALLOC_BUFTBL_CHUNK_OUT_NUMENTRIES_OFST 4 +/* Buffer table IDs for use in DMA descriptors. */ +#define MC_CMD_ALLOC_BUFTBL_CHUNK_OUT_ID_OFST 8 + + +/***********************************/ +/* MC_CMD_PROGRAM_BUFTBL_ENTRIES + * Reprogram a set of buffer table entries in the specified chunk. + */ +#define MC_CMD_PROGRAM_BUFTBL_ENTRIES 0x88 +#undef MC_CMD_0x88_PRIVILEGE_CTG + +#define MC_CMD_0x88_PRIVILEGE_CTG SRIOV_CTG_ONLOAD + +/* MC_CMD_PROGRAM_BUFTBL_ENTRIES_IN msgrequest */ +#define MC_CMD_PROGRAM_BUFTBL_ENTRIES_IN_LENMIN 20 +#define MC_CMD_PROGRAM_BUFTBL_ENTRIES_IN_LENMAX 268 +#define MC_CMD_PROGRAM_BUFTBL_ENTRIES_IN_LEN(num) (12+8*(num)) +#define MC_CMD_PROGRAM_BUFTBL_ENTRIES_IN_HANDLE_OFST 0 +/* ID */ +#define MC_CMD_PROGRAM_BUFTBL_ENTRIES_IN_FIRSTID_OFST 4 +/* Num entries */ +#define MC_CMD_PROGRAM_BUFTBL_ENTRIES_IN_NUMENTRIES_OFST 8 +/* Buffer table entry address */ +#define MC_CMD_PROGRAM_BUFTBL_ENTRIES_IN_ENTRY_OFST 12 +#define MC_CMD_PROGRAM_BUFTBL_ENTRIES_IN_ENTRY_LEN 8 +#define MC_CMD_PROGRAM_BUFTBL_ENTRIES_IN_ENTRY_LO_OFST 12 +#define MC_CMD_PROGRAM_BUFTBL_ENTRIES_IN_ENTRY_HI_OFST 16 +#define MC_CMD_PROGRAM_BUFTBL_ENTRIES_IN_ENTRY_MINNUM 1 +#define MC_CMD_PROGRAM_BUFTBL_ENTRIES_IN_ENTRY_MAXNUM 32 + +/* MC_CMD_PROGRAM_BUFTBL_ENTRIES_OUT msgresponse */ +#define MC_CMD_PROGRAM_BUFTBL_ENTRIES_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_FREE_BUFTBL_CHUNK + */ +#define MC_CMD_FREE_BUFTBL_CHUNK 0x89 +#undef MC_CMD_0x89_PRIVILEGE_CTG + +#define MC_CMD_0x89_PRIVILEGE_CTG SRIOV_CTG_ONLOAD + +/* MC_CMD_FREE_BUFTBL_CHUNK_IN msgrequest */ +#define MC_CMD_FREE_BUFTBL_CHUNK_IN_LEN 4 +#define MC_CMD_FREE_BUFTBL_CHUNK_IN_HANDLE_OFST 0 + +/* MC_CMD_FREE_BUFTBL_CHUNK_OUT msgresponse */ +#define MC_CMD_FREE_BUFTBL_CHUNK_OUT_LEN 0 + +/* PORT_CONFIG_ENTRY structuredef */ +#define PORT_CONFIG_ENTRY_LEN 16 +/* External port number (label) */ +#define PORT_CONFIG_ENTRY_EXT_NUMBER_OFST 0 +#define PORT_CONFIG_ENTRY_EXT_NUMBER_LEN 1 +#define PORT_CONFIG_ENTRY_EXT_NUMBER_LBN 0 +#define PORT_CONFIG_ENTRY_EXT_NUMBER_WIDTH 8 +/* Port core location */ +#define PORT_CONFIG_ENTRY_CORE_OFST 1 +#define PORT_CONFIG_ENTRY_CORE_LEN 1 +#define PORT_CONFIG_ENTRY_STANDALONE 0x0 /* enum */ +#define PORT_CONFIG_ENTRY_MASTER 0x1 /* enum */ +#define PORT_CONFIG_ENTRY_SLAVE 0x2 /* enum */ +#define PORT_CONFIG_ENTRY_CORE_LBN 8 +#define PORT_CONFIG_ENTRY_CORE_WIDTH 8 +/* Internal number (HW resource) relative to the core */ +#define PORT_CONFIG_ENTRY_INT_NUMBER_OFST 2 +#define PORT_CONFIG_ENTRY_INT_NUMBER_LEN 1 +#define PORT_CONFIG_ENTRY_INT_NUMBER_LBN 16 +#define PORT_CONFIG_ENTRY_INT_NUMBER_WIDTH 8 +/* Reserved */ +#define PORT_CONFIG_ENTRY_RSVD_OFST 3 +#define PORT_CONFIG_ENTRY_RSVD_LEN 1 +#define PORT_CONFIG_ENTRY_RSVD_LBN 24 +#define PORT_CONFIG_ENTRY_RSVD_WIDTH 8 +/* Bitmask of KR lanes used by the port */ +#define PORT_CONFIG_ENTRY_LANES_OFST 4 +#define PORT_CONFIG_ENTRY_LANES_LBN 32 +#define PORT_CONFIG_ENTRY_LANES_WIDTH 32 +/* Port capabilities (MC_CMD_PHY_CAP_*) */ +#define PORT_CONFIG_ENTRY_SUPPORTED_CAPS_OFST 8 +#define PORT_CONFIG_ENTRY_SUPPORTED_CAPS_LBN 64 +#define PORT_CONFIG_ENTRY_SUPPORTED_CAPS_WIDTH 32 +/* Reserved (align to 16 bytes) */ +#define PORT_CONFIG_ENTRY_RSVD2_OFST 12 +#define PORT_CONFIG_ENTRY_RSVD2_LBN 96 +#define PORT_CONFIG_ENTRY_RSVD2_WIDTH 32 + + +/***********************************/ +/* MC_CMD_FILTER_OP + * Multiplexed MCDI call for filter operations + */ +#define MC_CMD_FILTER_OP 0x8a +#undef MC_CMD_0x8a_PRIVILEGE_CTG + +#define MC_CMD_0x8a_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_FILTER_OP_IN msgrequest */ +#define MC_CMD_FILTER_OP_IN_LEN 108 +/* identifies the type of operation requested */ +#define MC_CMD_FILTER_OP_IN_OP_OFST 0 +/* enum: single-recipient filter insert */ +#define MC_CMD_FILTER_OP_IN_OP_INSERT 0x0 +/* enum: single-recipient filter remove */ +#define MC_CMD_FILTER_OP_IN_OP_REMOVE 0x1 +/* enum: multi-recipient filter subscribe */ +#define MC_CMD_FILTER_OP_IN_OP_SUBSCRIBE 0x2 +/* enum: multi-recipient filter unsubscribe */ +#define MC_CMD_FILTER_OP_IN_OP_UNSUBSCRIBE 0x3 +/* enum: replace one recipient with another (warning - the filter handle may + * change) + */ +#define MC_CMD_FILTER_OP_IN_OP_REPLACE 0x4 +/* filter handle (for remove / unsubscribe operations) */ +#define MC_CMD_FILTER_OP_IN_HANDLE_OFST 4 +#define MC_CMD_FILTER_OP_IN_HANDLE_LEN 8 +#define MC_CMD_FILTER_OP_IN_HANDLE_LO_OFST 4 +#define MC_CMD_FILTER_OP_IN_HANDLE_HI_OFST 8 +/* The port ID associated with the v-adaptor which should contain this filter. + */ +#define MC_CMD_FILTER_OP_IN_PORT_ID_OFST 12 +/* fields to include in match criteria */ +#define MC_CMD_FILTER_OP_IN_MATCH_FIELDS_OFST 16 +#define MC_CMD_FILTER_OP_IN_MATCH_SRC_IP_LBN 0 +#define MC_CMD_FILTER_OP_IN_MATCH_SRC_IP_WIDTH 1 +#define MC_CMD_FILTER_OP_IN_MATCH_DST_IP_LBN 1 +#define MC_CMD_FILTER_OP_IN_MATCH_DST_IP_WIDTH 1 +#define MC_CMD_FILTER_OP_IN_MATCH_SRC_MAC_LBN 2 +#define MC_CMD_FILTER_OP_IN_MATCH_SRC_MAC_WIDTH 1 +#define MC_CMD_FILTER_OP_IN_MATCH_SRC_PORT_LBN 3 +#define MC_CMD_FILTER_OP_IN_MATCH_SRC_PORT_WIDTH 1 +#define MC_CMD_FILTER_OP_IN_MATCH_DST_MAC_LBN 4 +#define MC_CMD_FILTER_OP_IN_MATCH_DST_MAC_WIDTH 1 +#define MC_CMD_FILTER_OP_IN_MATCH_DST_PORT_LBN 5 +#define MC_CMD_FILTER_OP_IN_MATCH_DST_PORT_WIDTH 1 +#define MC_CMD_FILTER_OP_IN_MATCH_ETHER_TYPE_LBN 6 +#define MC_CMD_FILTER_OP_IN_MATCH_ETHER_TYPE_WIDTH 1 +#define MC_CMD_FILTER_OP_IN_MATCH_INNER_VLAN_LBN 7 +#define MC_CMD_FILTER_OP_IN_MATCH_INNER_VLAN_WIDTH 1 +#define MC_CMD_FILTER_OP_IN_MATCH_OUTER_VLAN_LBN 8 +#define MC_CMD_FILTER_OP_IN_MATCH_OUTER_VLAN_WIDTH 1 +#define MC_CMD_FILTER_OP_IN_MATCH_IP_PROTO_LBN 9 +#define MC_CMD_FILTER_OP_IN_MATCH_IP_PROTO_WIDTH 1 +#define MC_CMD_FILTER_OP_IN_MATCH_FWDEF0_LBN 10 +#define MC_CMD_FILTER_OP_IN_MATCH_FWDEF0_WIDTH 1 +#define MC_CMD_FILTER_OP_IN_MATCH_FWDEF1_LBN 11 +#define MC_CMD_FILTER_OP_IN_MATCH_FWDEF1_WIDTH 1 +#define MC_CMD_FILTER_OP_IN_MATCH_UNKNOWN_MCAST_DST_LBN 30 +#define MC_CMD_FILTER_OP_IN_MATCH_UNKNOWN_MCAST_DST_WIDTH 1 +#define MC_CMD_FILTER_OP_IN_MATCH_UNKNOWN_UCAST_DST_LBN 31 +#define MC_CMD_FILTER_OP_IN_MATCH_UNKNOWN_UCAST_DST_WIDTH 1 +/* receive destination */ +#define MC_CMD_FILTER_OP_IN_RX_DEST_OFST 20 +/* enum: drop packets */ +#define MC_CMD_FILTER_OP_IN_RX_DEST_DROP 0x0 +/* enum: receive to host */ +#define MC_CMD_FILTER_OP_IN_RX_DEST_HOST 0x1 +/* enum: receive to MC */ +#define MC_CMD_FILTER_OP_IN_RX_DEST_MC 0x2 +/* enum: loop back to TXDP 0 */ +#define MC_CMD_FILTER_OP_IN_RX_DEST_TX0 0x3 +/* enum: loop back to TXDP 1 */ +#define MC_CMD_FILTER_OP_IN_RX_DEST_TX1 0x4 +/* receive queue handle (for multiple queue modes, this is the base queue) */ +#define MC_CMD_FILTER_OP_IN_RX_QUEUE_OFST 24 +/* receive mode */ +#define MC_CMD_FILTER_OP_IN_RX_MODE_OFST 28 +/* enum: receive to just the specified queue */ +#define MC_CMD_FILTER_OP_IN_RX_MODE_SIMPLE 0x0 +/* enum: receive to multiple queues using RSS context */ +#define MC_CMD_FILTER_OP_IN_RX_MODE_RSS 0x1 +/* enum: receive to multiple queues using .1p mapping */ +#define MC_CMD_FILTER_OP_IN_RX_MODE_DOT1P_MAPPING 0x2 +/* enum: install a filter entry that will never match; for test purposes only + */ +#define MC_CMD_FILTER_OP_IN_RX_MODE_TEST_NEVER_MATCH 0x80000000 +/* RSS context (for RX_MODE_RSS) or .1p mapping handle (for + * RX_MODE_DOT1P_MAPPING), as returned by MC_CMD_RSS_CONTEXT_ALLOC or + * MC_CMD_DOT1P_MAPPING_ALLOC. + */ +#define MC_CMD_FILTER_OP_IN_RX_CONTEXT_OFST 32 +/* transmit domain (reserved; set to 0) */ +#define MC_CMD_FILTER_OP_IN_TX_DOMAIN_OFST 36 +/* transmit destination (either set the MAC and/or PM bits for explicit + * control, or set this field to TX_DEST_DEFAULT for sensible default + * behaviour) + */ +#define MC_CMD_FILTER_OP_IN_TX_DEST_OFST 40 +/* enum: request default behaviour (based on filter type) */ +#define MC_CMD_FILTER_OP_IN_TX_DEST_DEFAULT 0xffffffff +#define MC_CMD_FILTER_OP_IN_TX_DEST_MAC_LBN 0 +#define MC_CMD_FILTER_OP_IN_TX_DEST_MAC_WIDTH 1 +#define MC_CMD_FILTER_OP_IN_TX_DEST_PM_LBN 1 +#define MC_CMD_FILTER_OP_IN_TX_DEST_PM_WIDTH 1 +/* source MAC address to match (as bytes in network order) */ +#define MC_CMD_FILTER_OP_IN_SRC_MAC_OFST 44 +#define MC_CMD_FILTER_OP_IN_SRC_MAC_LEN 6 +/* source port to match (as bytes in network order) */ +#define MC_CMD_FILTER_OP_IN_SRC_PORT_OFST 50 +#define MC_CMD_FILTER_OP_IN_SRC_PORT_LEN 2 +/* destination MAC address to match (as bytes in network order) */ +#define MC_CMD_FILTER_OP_IN_DST_MAC_OFST 52 +#define MC_CMD_FILTER_OP_IN_DST_MAC_LEN 6 +/* destination port to match (as bytes in network order) */ +#define MC_CMD_FILTER_OP_IN_DST_PORT_OFST 58 +#define MC_CMD_FILTER_OP_IN_DST_PORT_LEN 2 +/* Ethernet type to match (as bytes in network order) */ +#define MC_CMD_FILTER_OP_IN_ETHER_TYPE_OFST 60 +#define MC_CMD_FILTER_OP_IN_ETHER_TYPE_LEN 2 +/* Inner VLAN tag to match (as bytes in network order) */ +#define MC_CMD_FILTER_OP_IN_INNER_VLAN_OFST 62 +#define MC_CMD_FILTER_OP_IN_INNER_VLAN_LEN 2 +/* Outer VLAN tag to match (as bytes in network order) */ +#define MC_CMD_FILTER_OP_IN_OUTER_VLAN_OFST 64 +#define MC_CMD_FILTER_OP_IN_OUTER_VLAN_LEN 2 +/* IP protocol to match (in low byte; set high byte to 0) */ +#define MC_CMD_FILTER_OP_IN_IP_PROTO_OFST 66 +#define MC_CMD_FILTER_OP_IN_IP_PROTO_LEN 2 +/* Firmware defined register 0 to match (reserved; set to 0) */ +#define MC_CMD_FILTER_OP_IN_FWDEF0_OFST 68 +/* Firmware defined register 1 to match (reserved; set to 0) */ +#define MC_CMD_FILTER_OP_IN_FWDEF1_OFST 72 +/* source IP address to match (as bytes in network order; set last 12 bytes to + * 0 for IPv4 address) + */ +#define MC_CMD_FILTER_OP_IN_SRC_IP_OFST 76 +#define MC_CMD_FILTER_OP_IN_SRC_IP_LEN 16 +/* destination IP address to match (as bytes in network order; set last 12 + * bytes to 0 for IPv4 address) + */ +#define MC_CMD_FILTER_OP_IN_DST_IP_OFST 92 +#define MC_CMD_FILTER_OP_IN_DST_IP_LEN 16 + +/* MC_CMD_FILTER_OP_EXT_IN msgrequest: Extension to MC_CMD_FILTER_OP_IN to + * include handling of VXLAN/NVGRE encapsulated frame filtering (which is + * supported on Medford only). + */ +#define MC_CMD_FILTER_OP_EXT_IN_LEN 172 +/* identifies the type of operation requested */ +#define MC_CMD_FILTER_OP_EXT_IN_OP_OFST 0 +/* Enum values, see field(s): */ +/* MC_CMD_FILTER_OP_IN/OP */ +/* filter handle (for remove / unsubscribe operations) */ +#define MC_CMD_FILTER_OP_EXT_IN_HANDLE_OFST 4 +#define MC_CMD_FILTER_OP_EXT_IN_HANDLE_LEN 8 +#define MC_CMD_FILTER_OP_EXT_IN_HANDLE_LO_OFST 4 +#define MC_CMD_FILTER_OP_EXT_IN_HANDLE_HI_OFST 8 +/* The port ID associated with the v-adaptor which should contain this filter. + */ +#define MC_CMD_FILTER_OP_EXT_IN_PORT_ID_OFST 12 +/* fields to include in match criteria */ +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_FIELDS_OFST 16 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_SRC_IP_LBN 0 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_SRC_IP_WIDTH 1 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_DST_IP_LBN 1 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_DST_IP_WIDTH 1 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_SRC_MAC_LBN 2 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_SRC_MAC_WIDTH 1 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_SRC_PORT_LBN 3 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_SRC_PORT_WIDTH 1 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_DST_MAC_LBN 4 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_DST_MAC_WIDTH 1 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_DST_PORT_LBN 5 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_DST_PORT_WIDTH 1 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_ETHER_TYPE_LBN 6 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_ETHER_TYPE_WIDTH 1 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_INNER_VLAN_LBN 7 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_INNER_VLAN_WIDTH 1 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_OUTER_VLAN_LBN 8 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_OUTER_VLAN_WIDTH 1 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_IP_PROTO_LBN 9 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_IP_PROTO_WIDTH 1 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_FWDEF0_LBN 10 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_FWDEF0_WIDTH 1 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_VNI_OR_VSID_LBN 11 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_VNI_OR_VSID_WIDTH 1 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_SRC_IP_LBN 12 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_SRC_IP_WIDTH 1 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_DST_IP_LBN 13 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_DST_IP_WIDTH 1 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_SRC_MAC_LBN 14 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_SRC_MAC_WIDTH 1 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_SRC_PORT_LBN 15 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_SRC_PORT_WIDTH 1 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_DST_MAC_LBN 16 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_DST_MAC_WIDTH 1 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_DST_PORT_LBN 17 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_DST_PORT_WIDTH 1 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_ETHER_TYPE_LBN 18 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_ETHER_TYPE_WIDTH 1 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_INNER_VLAN_LBN 19 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_INNER_VLAN_WIDTH 1 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_OUTER_VLAN_LBN 20 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_OUTER_VLAN_WIDTH 1 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_IP_PROTO_LBN 21 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_IP_PROTO_WIDTH 1 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_FWDEF0_LBN 22 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_FWDEF0_WIDTH 1 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_FWDEF1_LBN 23 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_FWDEF1_WIDTH 1 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_UNKNOWN_MCAST_DST_LBN 24 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_UNKNOWN_MCAST_DST_WIDTH 1 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_UNKNOWN_UCAST_DST_LBN 25 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_UNKNOWN_UCAST_DST_WIDTH 1 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_UNKNOWN_MCAST_DST_LBN 30 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_UNKNOWN_MCAST_DST_WIDTH 1 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_UNKNOWN_UCAST_DST_LBN 31 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_UNKNOWN_UCAST_DST_WIDTH 1 +/* receive destination */ +#define MC_CMD_FILTER_OP_EXT_IN_RX_DEST_OFST 20 +/* enum: drop packets */ +#define MC_CMD_FILTER_OP_EXT_IN_RX_DEST_DROP 0x0 +/* enum: receive to host */ +#define MC_CMD_FILTER_OP_EXT_IN_RX_DEST_HOST 0x1 +/* enum: receive to MC */ +#define MC_CMD_FILTER_OP_EXT_IN_RX_DEST_MC 0x2 +/* enum: loop back to TXDP 0 */ +#define MC_CMD_FILTER_OP_EXT_IN_RX_DEST_TX0 0x3 +/* enum: loop back to TXDP 1 */ +#define MC_CMD_FILTER_OP_EXT_IN_RX_DEST_TX1 0x4 +/* receive queue handle (for multiple queue modes, this is the base queue) */ +#define MC_CMD_FILTER_OP_EXT_IN_RX_QUEUE_OFST 24 +/* receive mode */ +#define MC_CMD_FILTER_OP_EXT_IN_RX_MODE_OFST 28 +/* enum: receive to just the specified queue */ +#define MC_CMD_FILTER_OP_EXT_IN_RX_MODE_SIMPLE 0x0 +/* enum: receive to multiple queues using RSS context */ +#define MC_CMD_FILTER_OP_EXT_IN_RX_MODE_RSS 0x1 +/* enum: receive to multiple queues using .1p mapping */ +#define MC_CMD_FILTER_OP_EXT_IN_RX_MODE_DOT1P_MAPPING 0x2 +/* enum: install a filter entry that will never match; for test purposes only + */ +#define MC_CMD_FILTER_OP_EXT_IN_RX_MODE_TEST_NEVER_MATCH 0x80000000 +/* RSS context (for RX_MODE_RSS) or .1p mapping handle (for + * RX_MODE_DOT1P_MAPPING), as returned by MC_CMD_RSS_CONTEXT_ALLOC or + * MC_CMD_DOT1P_MAPPING_ALLOC. + */ +#define MC_CMD_FILTER_OP_EXT_IN_RX_CONTEXT_OFST 32 +/* transmit domain (reserved; set to 0) */ +#define MC_CMD_FILTER_OP_EXT_IN_TX_DOMAIN_OFST 36 +/* transmit destination (either set the MAC and/or PM bits for explicit + * control, or set this field to TX_DEST_DEFAULT for sensible default + * behaviour) + */ +#define MC_CMD_FILTER_OP_EXT_IN_TX_DEST_OFST 40 +/* enum: request default behaviour (based on filter type) */ +#define MC_CMD_FILTER_OP_EXT_IN_TX_DEST_DEFAULT 0xffffffff +#define MC_CMD_FILTER_OP_EXT_IN_TX_DEST_MAC_LBN 0 +#define MC_CMD_FILTER_OP_EXT_IN_TX_DEST_MAC_WIDTH 1 +#define MC_CMD_FILTER_OP_EXT_IN_TX_DEST_PM_LBN 1 +#define MC_CMD_FILTER_OP_EXT_IN_TX_DEST_PM_WIDTH 1 +/* source MAC address to match (as bytes in network order) */ +#define MC_CMD_FILTER_OP_EXT_IN_SRC_MAC_OFST 44 +#define MC_CMD_FILTER_OP_EXT_IN_SRC_MAC_LEN 6 +/* source port to match (as bytes in network order) */ +#define MC_CMD_FILTER_OP_EXT_IN_SRC_PORT_OFST 50 +#define MC_CMD_FILTER_OP_EXT_IN_SRC_PORT_LEN 2 +/* destination MAC address to match (as bytes in network order) */ +#define MC_CMD_FILTER_OP_EXT_IN_DST_MAC_OFST 52 +#define MC_CMD_FILTER_OP_EXT_IN_DST_MAC_LEN 6 +/* destination port to match (as bytes in network order) */ +#define MC_CMD_FILTER_OP_EXT_IN_DST_PORT_OFST 58 +#define MC_CMD_FILTER_OP_EXT_IN_DST_PORT_LEN 2 +/* Ethernet type to match (as bytes in network order) */ +#define MC_CMD_FILTER_OP_EXT_IN_ETHER_TYPE_OFST 60 +#define MC_CMD_FILTER_OP_EXT_IN_ETHER_TYPE_LEN 2 +/* Inner VLAN tag to match (as bytes in network order) */ +#define MC_CMD_FILTER_OP_EXT_IN_INNER_VLAN_OFST 62 +#define MC_CMD_FILTER_OP_EXT_IN_INNER_VLAN_LEN 2 +/* Outer VLAN tag to match (as bytes in network order) */ +#define MC_CMD_FILTER_OP_EXT_IN_OUTER_VLAN_OFST 64 +#define MC_CMD_FILTER_OP_EXT_IN_OUTER_VLAN_LEN 2 +/* IP protocol to match (in low byte; set high byte to 0) */ +#define MC_CMD_FILTER_OP_EXT_IN_IP_PROTO_OFST 66 +#define MC_CMD_FILTER_OP_EXT_IN_IP_PROTO_LEN 2 +/* Firmware defined register 0 to match (reserved; set to 0) */ +#define MC_CMD_FILTER_OP_EXT_IN_FWDEF0_OFST 68 +/* VNI (for VXLAN/Geneve, when IP protocol is UDP) or VSID (for NVGRE, when IP + * protocol is GRE) to match (as bytes in network order; set last byte to 0 for + * VXLAN/NVGRE, or 1 for Geneve) + */ +#define MC_CMD_FILTER_OP_EXT_IN_VNI_OR_VSID_OFST 72 +#define MC_CMD_FILTER_OP_EXT_IN_VNI_VALUE_LBN 0 +#define MC_CMD_FILTER_OP_EXT_IN_VNI_VALUE_WIDTH 24 +#define MC_CMD_FILTER_OP_EXT_IN_VNI_TYPE_LBN 24 +#define MC_CMD_FILTER_OP_EXT_IN_VNI_TYPE_WIDTH 8 +/* enum: Match VXLAN traffic with this VNI */ +#define MC_CMD_FILTER_OP_EXT_IN_VNI_TYPE_VXLAN 0x0 +/* enum: Match Geneve traffic with this VNI */ +#define MC_CMD_FILTER_OP_EXT_IN_VNI_TYPE_GENEVE 0x1 +/* enum: Reserved for experimental development use */ +#define MC_CMD_FILTER_OP_EXT_IN_VNI_TYPE_EXPERIMENTAL 0xfe +#define MC_CMD_FILTER_OP_EXT_IN_VSID_VALUE_LBN 0 +#define MC_CMD_FILTER_OP_EXT_IN_VSID_VALUE_WIDTH 24 +#define MC_CMD_FILTER_OP_EXT_IN_VSID_TYPE_LBN 24 +#define MC_CMD_FILTER_OP_EXT_IN_VSID_TYPE_WIDTH 8 +/* enum: Match NVGRE traffic with this VSID */ +#define MC_CMD_FILTER_OP_EXT_IN_VSID_TYPE_NVGRE 0x0 +/* source IP address to match (as bytes in network order; set last 12 bytes to + * 0 for IPv4 address) + */ +#define MC_CMD_FILTER_OP_EXT_IN_SRC_IP_OFST 76 +#define MC_CMD_FILTER_OP_EXT_IN_SRC_IP_LEN 16 +/* destination IP address to match (as bytes in network order; set last 12 + * bytes to 0 for IPv4 address) + */ +#define MC_CMD_FILTER_OP_EXT_IN_DST_IP_OFST 92 +#define MC_CMD_FILTER_OP_EXT_IN_DST_IP_LEN 16 +/* VXLAN/NVGRE inner frame source MAC address to match (as bytes in network + * order) + */ +#define MC_CMD_FILTER_OP_EXT_IN_IFRM_SRC_MAC_OFST 108 +#define MC_CMD_FILTER_OP_EXT_IN_IFRM_SRC_MAC_LEN 6 +/* VXLAN/NVGRE inner frame source port to match (as bytes in network order) */ +#define MC_CMD_FILTER_OP_EXT_IN_IFRM_SRC_PORT_OFST 114 +#define MC_CMD_FILTER_OP_EXT_IN_IFRM_SRC_PORT_LEN 2 +/* VXLAN/NVGRE inner frame destination MAC address to match (as bytes in + * network order) + */ +#define MC_CMD_FILTER_OP_EXT_IN_IFRM_DST_MAC_OFST 116 +#define MC_CMD_FILTER_OP_EXT_IN_IFRM_DST_MAC_LEN 6 +/* VXLAN/NVGRE inner frame destination port to match (as bytes in network + * order) + */ +#define MC_CMD_FILTER_OP_EXT_IN_IFRM_DST_PORT_OFST 122 +#define MC_CMD_FILTER_OP_EXT_IN_IFRM_DST_PORT_LEN 2 +/* VXLAN/NVGRE inner frame Ethernet type to match (as bytes in network order) + */ +#define MC_CMD_FILTER_OP_EXT_IN_IFRM_ETHER_TYPE_OFST 124 +#define MC_CMD_FILTER_OP_EXT_IN_IFRM_ETHER_TYPE_LEN 2 +/* VXLAN/NVGRE inner frame Inner VLAN tag to match (as bytes in network order) + */ +#define MC_CMD_FILTER_OP_EXT_IN_IFRM_INNER_VLAN_OFST 126 +#define MC_CMD_FILTER_OP_EXT_IN_IFRM_INNER_VLAN_LEN 2 +/* VXLAN/NVGRE inner frame Outer VLAN tag to match (as bytes in network order) + */ +#define MC_CMD_FILTER_OP_EXT_IN_IFRM_OUTER_VLAN_OFST 128 +#define MC_CMD_FILTER_OP_EXT_IN_IFRM_OUTER_VLAN_LEN 2 +/* VXLAN/NVGRE inner frame IP protocol to match (in low byte; set high byte to + * 0) + */ +#define MC_CMD_FILTER_OP_EXT_IN_IFRM_IP_PROTO_OFST 130 +#define MC_CMD_FILTER_OP_EXT_IN_IFRM_IP_PROTO_LEN 2 +/* VXLAN/NVGRE inner frame Firmware defined register 0 to match (reserved; set + * to 0) + */ +#define MC_CMD_FILTER_OP_EXT_IN_IFRM_FWDEF0_OFST 132 +/* VXLAN/NVGRE inner frame Firmware defined register 1 to match (reserved; set + * to 0) + */ +#define MC_CMD_FILTER_OP_EXT_IN_IFRM_FWDEF1_OFST 136 +/* VXLAN/NVGRE inner frame source IP address to match (as bytes in network + * order; set last 12 bytes to 0 for IPv4 address) + */ +#define MC_CMD_FILTER_OP_EXT_IN_IFRM_SRC_IP_OFST 140 +#define MC_CMD_FILTER_OP_EXT_IN_IFRM_SRC_IP_LEN 16 +/* VXLAN/NVGRE inner frame destination IP address to match (as bytes in network + * order; set last 12 bytes to 0 for IPv4 address) + */ +#define MC_CMD_FILTER_OP_EXT_IN_IFRM_DST_IP_OFST 156 +#define MC_CMD_FILTER_OP_EXT_IN_IFRM_DST_IP_LEN 16 + +/* MC_CMD_FILTER_OP_OUT msgresponse */ +#define MC_CMD_FILTER_OP_OUT_LEN 12 +/* identifies the type of operation requested */ +#define MC_CMD_FILTER_OP_OUT_OP_OFST 0 +/* Enum values, see field(s): */ +/* MC_CMD_FILTER_OP_IN/OP */ +/* Returned filter handle (for insert / subscribe operations). Note that these + * handles should be considered opaque to the host, although a value of + * 0xFFFFFFFF_FFFFFFFF is guaranteed never to be a valid handle. + */ +#define MC_CMD_FILTER_OP_OUT_HANDLE_OFST 4 +#define MC_CMD_FILTER_OP_OUT_HANDLE_LEN 8 +#define MC_CMD_FILTER_OP_OUT_HANDLE_LO_OFST 4 +#define MC_CMD_FILTER_OP_OUT_HANDLE_HI_OFST 8 +/* enum: guaranteed invalid filter handle (low 32 bits) */ +#define MC_CMD_FILTER_OP_OUT_HANDLE_LO_INVALID 0xffffffff +/* enum: guaranteed invalid filter handle (high 32 bits) */ +#define MC_CMD_FILTER_OP_OUT_HANDLE_HI_INVALID 0xffffffff + +/* MC_CMD_FILTER_OP_EXT_OUT msgresponse */ +#define MC_CMD_FILTER_OP_EXT_OUT_LEN 12 +/* identifies the type of operation requested */ +#define MC_CMD_FILTER_OP_EXT_OUT_OP_OFST 0 +/* Enum values, see field(s): */ +/* MC_CMD_FILTER_OP_EXT_IN/OP */ +/* Returned filter handle (for insert / subscribe operations). Note that these + * handles should be considered opaque to the host, although a value of + * 0xFFFFFFFF_FFFFFFFF is guaranteed never to be a valid handle. + */ +#define MC_CMD_FILTER_OP_EXT_OUT_HANDLE_OFST 4 +#define MC_CMD_FILTER_OP_EXT_OUT_HANDLE_LEN 8 +#define MC_CMD_FILTER_OP_EXT_OUT_HANDLE_LO_OFST 4 +#define MC_CMD_FILTER_OP_EXT_OUT_HANDLE_HI_OFST 8 +/* Enum values, see field(s): */ +/* MC_CMD_FILTER_OP_OUT/HANDLE */ + + +/***********************************/ +/* MC_CMD_GET_PARSER_DISP_INFO + * Get information related to the parser-dispatcher subsystem + */ +#define MC_CMD_GET_PARSER_DISP_INFO 0xe4 +#undef MC_CMD_0xe4_PRIVILEGE_CTG + +#define MC_CMD_0xe4_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_GET_PARSER_DISP_INFO_IN msgrequest */ +#define MC_CMD_GET_PARSER_DISP_INFO_IN_LEN 4 +/* identifies the type of operation requested */ +#define MC_CMD_GET_PARSER_DISP_INFO_IN_OP_OFST 0 +/* enum: read the list of supported RX filter matches */ +#define MC_CMD_GET_PARSER_DISP_INFO_IN_OP_GET_SUPPORTED_RX_MATCHES 0x1 +/* enum: read flags indicating restrictions on filter insertion for the calling + * client + */ +#define MC_CMD_GET_PARSER_DISP_INFO_IN_OP_GET_RESTRICTIONS 0x2 +/* enum: read properties relating to security rules (Medford-only; for use by + * SolarSecure apps, not directly by drivers. See SF-114946-SW.) + */ +#define MC_CMD_GET_PARSER_DISP_INFO_IN_OP_GET_SECURITY_RULE_INFO 0x3 +/* enum: read the list of supported RX filter matches for VXLAN/NVGRE + * encapsulated frames, which follow a different match sequence to normal + * frames (Medford only) + */ +#define MC_CMD_GET_PARSER_DISP_INFO_IN_OP_GET_SUPPORTED_ENCAP_RX_MATCHES 0x4 + +/* MC_CMD_GET_PARSER_DISP_INFO_OUT msgresponse */ +#define MC_CMD_GET_PARSER_DISP_INFO_OUT_LENMIN 8 +#define MC_CMD_GET_PARSER_DISP_INFO_OUT_LENMAX 252 +#define MC_CMD_GET_PARSER_DISP_INFO_OUT_LEN(num) (8+4*(num)) +/* identifies the type of operation requested */ +#define MC_CMD_GET_PARSER_DISP_INFO_OUT_OP_OFST 0 +/* Enum values, see field(s): */ +/* MC_CMD_GET_PARSER_DISP_INFO_IN/OP */ +/* number of supported match types */ +#define MC_CMD_GET_PARSER_DISP_INFO_OUT_NUM_SUPPORTED_MATCHES_OFST 4 +/* array of supported match types (valid MATCH_FIELDS values for + * MC_CMD_FILTER_OP) sorted in decreasing priority order + */ +#define MC_CMD_GET_PARSER_DISP_INFO_OUT_SUPPORTED_MATCHES_OFST 8 +#define MC_CMD_GET_PARSER_DISP_INFO_OUT_SUPPORTED_MATCHES_LEN 4 +#define MC_CMD_GET_PARSER_DISP_INFO_OUT_SUPPORTED_MATCHES_MINNUM 0 +#define MC_CMD_GET_PARSER_DISP_INFO_OUT_SUPPORTED_MATCHES_MAXNUM 61 + +/* MC_CMD_GET_PARSER_DISP_RESTRICTIONS_OUT msgresponse */ +#define MC_CMD_GET_PARSER_DISP_RESTRICTIONS_OUT_LEN 8 +/* identifies the type of operation requested */ +#define MC_CMD_GET_PARSER_DISP_RESTRICTIONS_OUT_OP_OFST 0 +/* Enum values, see field(s): */ +/* MC_CMD_GET_PARSER_DISP_INFO_IN/OP */ +/* bitfield of filter insertion restrictions */ +#define MC_CMD_GET_PARSER_DISP_RESTRICTIONS_OUT_RESTRICTION_FLAGS_OFST 4 +#define MC_CMD_GET_PARSER_DISP_RESTRICTIONS_OUT_DST_IP_MCAST_ONLY_LBN 0 +#define MC_CMD_GET_PARSER_DISP_RESTRICTIONS_OUT_DST_IP_MCAST_ONLY_WIDTH 1 + +/* MC_CMD_GET_PARSER_DISP_SECURITY_RULE_INFO_OUT msgresponse: + * GET_PARSER_DISP_INFO response format for OP_GET_SECURITY_RULE_INFO. + * (Medford-only; for use by SolarSecure apps, not directly by drivers. See + * SF-114946-SW.) NOTE - this message definition is provisional. It has not yet + * been used in any released code and may change during development. This note + * will be removed once it is regarded as stable. + */ +#define MC_CMD_GET_PARSER_DISP_SECURITY_RULE_INFO_OUT_LEN 36 +/* identifies the type of operation requested */ +#define MC_CMD_GET_PARSER_DISP_SECURITY_RULE_INFO_OUT_OP_OFST 0 +/* Enum values, see field(s): */ +/* MC_CMD_GET_PARSER_DISP_INFO_IN/OP */ +/* a version number representing the set of rule lookups that are implemented + * by the currently running firmware + */ +#define MC_CMD_GET_PARSER_DISP_SECURITY_RULE_INFO_OUT_RULES_VERSION_OFST 4 +/* enum: implements lookup sequences described in SF-114946-SW draft C */ +#define MC_CMD_GET_PARSER_DISP_SECURITY_RULE_INFO_OUT_RULES_VERSION_SF_114946_SW_C 0x0 +/* the number of nodes in the subnet map */ +#define MC_CMD_GET_PARSER_DISP_SECURITY_RULE_INFO_OUT_SUBNET_MAP_NUM_NODES_OFST 8 +/* the number of entries in one subnet map node */ +#define MC_CMD_GET_PARSER_DISP_SECURITY_RULE_INFO_OUT_SUBNET_MAP_NUM_ENTRIES_PER_NODE_OFST 12 +/* minimum valid value for a subnet ID in a subnet map leaf */ +#define MC_CMD_GET_PARSER_DISP_SECURITY_RULE_INFO_OUT_SUBNET_ID_MIN_OFST 16 +/* maximum valid value for a subnet ID in a subnet map leaf */ +#define MC_CMD_GET_PARSER_DISP_SECURITY_RULE_INFO_OUT_SUBNET_ID_MAX_OFST 20 +/* the number of entries in the local and remote port range maps */ +#define MC_CMD_GET_PARSER_DISP_SECURITY_RULE_INFO_OUT_PORTRANGE_TREE_NUM_ENTRIES_OFST 24 +/* minimum valid value for a portrange ID in a port range map leaf */ +#define MC_CMD_GET_PARSER_DISP_SECURITY_RULE_INFO_OUT_PORTRANGE_ID_MIN_OFST 28 +/* maximum valid value for a portrange ID in a port range map leaf */ +#define MC_CMD_GET_PARSER_DISP_SECURITY_RULE_INFO_OUT_PORTRANGE_ID_MAX_OFST 32 + + +/***********************************/ +/* MC_CMD_PARSER_DISP_RW + * Direct read/write of parser-dispatcher state (DICPUs and LUE) for debugging. + * Please note that this interface is only of use to debug tools which have + * knowledge of firmware and hardware data structures; nothing here is intended + * for use by normal driver code. + */ +#define MC_CMD_PARSER_DISP_RW 0xe5 +#undef MC_CMD_0xe5_PRIVILEGE_CTG + +#define MC_CMD_0xe5_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_PARSER_DISP_RW_IN msgrequest */ +#define MC_CMD_PARSER_DISP_RW_IN_LEN 32 +/* identifies the target of the operation */ +#define MC_CMD_PARSER_DISP_RW_IN_TARGET_OFST 0 +/* enum: RX dispatcher CPU */ +#define MC_CMD_PARSER_DISP_RW_IN_RX_DICPU 0x0 +/* enum: TX dispatcher CPU */ +#define MC_CMD_PARSER_DISP_RW_IN_TX_DICPU 0x1 +/* enum: Lookup engine (with original metadata format) */ +#define MC_CMD_PARSER_DISP_RW_IN_LUE 0x2 +/* enum: Lookup engine (with requested metadata format) */ +#define MC_CMD_PARSER_DISP_RW_IN_LUE_VERSIONED_METADATA 0x3 +/* enum: RX0 dispatcher CPU (alias for RX_DICPU; Medford has 2 RX DICPUs) */ +#define MC_CMD_PARSER_DISP_RW_IN_RX0_DICPU 0x0 +/* enum: RX1 dispatcher CPU (only valid for Medford) */ +#define MC_CMD_PARSER_DISP_RW_IN_RX1_DICPU 0x4 +/* enum: Miscellaneous other state (only valid for Medford) */ +#define MC_CMD_PARSER_DISP_RW_IN_MISC_STATE 0x5 +/* identifies the type of operation requested */ +#define MC_CMD_PARSER_DISP_RW_IN_OP_OFST 4 +/* enum: read a word of DICPU DMEM or a LUE entry */ +#define MC_CMD_PARSER_DISP_RW_IN_READ 0x0 +/* enum: write a word of DICPU DMEM or a LUE entry */ +#define MC_CMD_PARSER_DISP_RW_IN_WRITE 0x1 +/* enum: read-modify-write a word of DICPU DMEM (not valid for LUE) */ +#define MC_CMD_PARSER_DISP_RW_IN_RMW 0x2 +/* data memory address (DICPU targets) or LUE index (LUE targets) */ +#define MC_CMD_PARSER_DISP_RW_IN_ADDRESS_OFST 8 +/* selector (for MISC_STATE target) */ +#define MC_CMD_PARSER_DISP_RW_IN_SELECTOR_OFST 8 +/* enum: Port to datapath mapping */ +#define MC_CMD_PARSER_DISP_RW_IN_PORT_DP_MAPPING 0x1 +/* value to write (for DMEM writes) */ +#define MC_CMD_PARSER_DISP_RW_IN_DMEM_WRITE_VALUE_OFST 12 +/* XOR value (for DMEM read-modify-writes: new = (old & mask) ^ value) */ +#define MC_CMD_PARSER_DISP_RW_IN_DMEM_RMW_XOR_VALUE_OFST 12 +/* AND mask (for DMEM read-modify-writes: new = (old & mask) ^ value) */ +#define MC_CMD_PARSER_DISP_RW_IN_DMEM_RMW_AND_MASK_OFST 16 +/* metadata format (for LUE reads using LUE_VERSIONED_METADATA) */ +#define MC_CMD_PARSER_DISP_RW_IN_LUE_READ_METADATA_VERSION_OFST 12 +/* value to write (for LUE writes) */ +#define MC_CMD_PARSER_DISP_RW_IN_LUE_WRITE_VALUE_OFST 12 +#define MC_CMD_PARSER_DISP_RW_IN_LUE_WRITE_VALUE_LEN 20 + +/* MC_CMD_PARSER_DISP_RW_OUT msgresponse */ +#define MC_CMD_PARSER_DISP_RW_OUT_LEN 52 +/* value read (for DMEM reads) */ +#define MC_CMD_PARSER_DISP_RW_OUT_DMEM_READ_VALUE_OFST 0 +/* value read (for LUE reads) */ +#define MC_CMD_PARSER_DISP_RW_OUT_LUE_READ_VALUE_OFST 0 +#define MC_CMD_PARSER_DISP_RW_OUT_LUE_READ_VALUE_LEN 20 +/* up to 8 32-bit words of additional soft state from the LUE manager (the + * exact content is firmware-dependent and intended only for debug use) + */ +#define MC_CMD_PARSER_DISP_RW_OUT_LUE_MGR_STATE_OFST 20 +#define MC_CMD_PARSER_DISP_RW_OUT_LUE_MGR_STATE_LEN 32 +/* datapath(s) used for each port (for MISC_STATE PORT_DP_MAPPING selector) */ +#define MC_CMD_PARSER_DISP_RW_OUT_PORT_DP_MAPPING_OFST 0 +#define MC_CMD_PARSER_DISP_RW_OUT_PORT_DP_MAPPING_LEN 4 +#define MC_CMD_PARSER_DISP_RW_OUT_PORT_DP_MAPPING_NUM 4 +#define MC_CMD_PARSER_DISP_RW_OUT_DP0 0x1 /* enum */ +#define MC_CMD_PARSER_DISP_RW_OUT_DP1 0x2 /* enum */ + + +/***********************************/ +/* MC_CMD_GET_PF_COUNT + * Get number of PFs on the device. + */ +#define MC_CMD_GET_PF_COUNT 0xb6 +#undef MC_CMD_0xb6_PRIVILEGE_CTG + +#define MC_CMD_0xb6_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_GET_PF_COUNT_IN msgrequest */ +#define MC_CMD_GET_PF_COUNT_IN_LEN 0 + +/* MC_CMD_GET_PF_COUNT_OUT msgresponse */ +#define MC_CMD_GET_PF_COUNT_OUT_LEN 1 +/* Identifies the number of PFs on the device. */ +#define MC_CMD_GET_PF_COUNT_OUT_PF_COUNT_OFST 0 +#define MC_CMD_GET_PF_COUNT_OUT_PF_COUNT_LEN 1 + + +/***********************************/ +/* MC_CMD_SET_PF_COUNT + * Set number of PFs on the device. + */ +#define MC_CMD_SET_PF_COUNT 0xb7 + +/* MC_CMD_SET_PF_COUNT_IN msgrequest */ +#define MC_CMD_SET_PF_COUNT_IN_LEN 4 +/* New number of PFs on the device. */ +#define MC_CMD_SET_PF_COUNT_IN_PF_COUNT_OFST 0 + +/* MC_CMD_SET_PF_COUNT_OUT msgresponse */ +#define MC_CMD_SET_PF_COUNT_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_GET_PORT_ASSIGNMENT + * Get port assignment for current PCI function. + */ +#define MC_CMD_GET_PORT_ASSIGNMENT 0xb8 +#undef MC_CMD_0xb8_PRIVILEGE_CTG + +#define MC_CMD_0xb8_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_GET_PORT_ASSIGNMENT_IN msgrequest */ +#define MC_CMD_GET_PORT_ASSIGNMENT_IN_LEN 0 + +/* MC_CMD_GET_PORT_ASSIGNMENT_OUT msgresponse */ +#define MC_CMD_GET_PORT_ASSIGNMENT_OUT_LEN 4 +/* Identifies the port assignment for this function. */ +#define MC_CMD_GET_PORT_ASSIGNMENT_OUT_PORT_OFST 0 + + +/***********************************/ +/* MC_CMD_SET_PORT_ASSIGNMENT + * Set port assignment for current PCI function. + */ +#define MC_CMD_SET_PORT_ASSIGNMENT 0xb9 +#undef MC_CMD_0xb9_PRIVILEGE_CTG + +#define MC_CMD_0xb9_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_SET_PORT_ASSIGNMENT_IN msgrequest */ +#define MC_CMD_SET_PORT_ASSIGNMENT_IN_LEN 4 +/* Identifies the port assignment for this function. */ +#define MC_CMD_SET_PORT_ASSIGNMENT_IN_PORT_OFST 0 + +/* MC_CMD_SET_PORT_ASSIGNMENT_OUT msgresponse */ +#define MC_CMD_SET_PORT_ASSIGNMENT_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_ALLOC_VIS + * Allocate VIs for current PCI function. + */ +#define MC_CMD_ALLOC_VIS 0x8b +#undef MC_CMD_0x8b_PRIVILEGE_CTG + +#define MC_CMD_0x8b_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_ALLOC_VIS_IN msgrequest */ +#define MC_CMD_ALLOC_VIS_IN_LEN 8 +/* The minimum number of VIs that is acceptable */ +#define MC_CMD_ALLOC_VIS_IN_MIN_VI_COUNT_OFST 0 +/* The maximum number of VIs that would be useful */ +#define MC_CMD_ALLOC_VIS_IN_MAX_VI_COUNT_OFST 4 + +/* MC_CMD_ALLOC_VIS_OUT msgresponse: Huntington-compatible VI_ALLOC request. + * Use extended version in new code. + */ +#define MC_CMD_ALLOC_VIS_OUT_LEN 8 +/* The number of VIs allocated on this function */ +#define MC_CMD_ALLOC_VIS_OUT_VI_COUNT_OFST 0 +/* The base absolute VI number allocated to this function. Required to + * correctly interpret wakeup events. + */ +#define MC_CMD_ALLOC_VIS_OUT_VI_BASE_OFST 4 + +/* MC_CMD_ALLOC_VIS_EXT_OUT msgresponse */ +#define MC_CMD_ALLOC_VIS_EXT_OUT_LEN 12 +/* The number of VIs allocated on this function */ +#define MC_CMD_ALLOC_VIS_EXT_OUT_VI_COUNT_OFST 0 +/* The base absolute VI number allocated to this function. Required to + * correctly interpret wakeup events. + */ +#define MC_CMD_ALLOC_VIS_EXT_OUT_VI_BASE_OFST 4 +/* Function's port vi_shift value (always 0 on Huntington) */ +#define MC_CMD_ALLOC_VIS_EXT_OUT_VI_SHIFT_OFST 8 + + +/***********************************/ +/* MC_CMD_FREE_VIS + * Free VIs for current PCI function. Any linked PIO buffers will be unlinked, + * but not freed. + */ +#define MC_CMD_FREE_VIS 0x8c +#undef MC_CMD_0x8c_PRIVILEGE_CTG + +#define MC_CMD_0x8c_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_FREE_VIS_IN msgrequest */ +#define MC_CMD_FREE_VIS_IN_LEN 0 + +/* MC_CMD_FREE_VIS_OUT msgresponse */ +#define MC_CMD_FREE_VIS_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_GET_SRIOV_CFG + * Get SRIOV config for this PF. + */ +#define MC_CMD_GET_SRIOV_CFG 0xba +#undef MC_CMD_0xba_PRIVILEGE_CTG + +#define MC_CMD_0xba_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_GET_SRIOV_CFG_IN msgrequest */ +#define MC_CMD_GET_SRIOV_CFG_IN_LEN 0 + +/* MC_CMD_GET_SRIOV_CFG_OUT msgresponse */ +#define MC_CMD_GET_SRIOV_CFG_OUT_LEN 20 +/* Number of VFs currently enabled. */ +#define MC_CMD_GET_SRIOV_CFG_OUT_VF_CURRENT_OFST 0 +/* Max number of VFs before sriov stride and offset may need to be changed. */ +#define MC_CMD_GET_SRIOV_CFG_OUT_VF_MAX_OFST 4 +#define MC_CMD_GET_SRIOV_CFG_OUT_FLAGS_OFST 8 +#define MC_CMD_GET_SRIOV_CFG_OUT_VF_ENABLED_LBN 0 +#define MC_CMD_GET_SRIOV_CFG_OUT_VF_ENABLED_WIDTH 1 +/* RID offset of first VF from PF. */ +#define MC_CMD_GET_SRIOV_CFG_OUT_VF_OFFSET_OFST 12 +/* RID offset of each subsequent VF from the previous. */ +#define MC_CMD_GET_SRIOV_CFG_OUT_VF_STRIDE_OFST 16 + + +/***********************************/ +/* MC_CMD_SET_SRIOV_CFG + * Set SRIOV config for this PF. + */ +#define MC_CMD_SET_SRIOV_CFG 0xbb +#undef MC_CMD_0xbb_PRIVILEGE_CTG + +#define MC_CMD_0xbb_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_SET_SRIOV_CFG_IN msgrequest */ +#define MC_CMD_SET_SRIOV_CFG_IN_LEN 20 +/* Number of VFs currently enabled. */ +#define MC_CMD_SET_SRIOV_CFG_IN_VF_CURRENT_OFST 0 +/* Max number of VFs before sriov stride and offset may need to be changed. */ +#define MC_CMD_SET_SRIOV_CFG_IN_VF_MAX_OFST 4 +#define MC_CMD_SET_SRIOV_CFG_IN_FLAGS_OFST 8 +#define MC_CMD_SET_SRIOV_CFG_IN_VF_ENABLED_LBN 0 +#define MC_CMD_SET_SRIOV_CFG_IN_VF_ENABLED_WIDTH 1 +/* RID offset of first VF from PF, or 0 for no change, or + * MC_CMD_RESOURCE_INSTANCE_ANY to allow the system to allocate an offset. + */ +#define MC_CMD_SET_SRIOV_CFG_IN_VF_OFFSET_OFST 12 +/* RID offset of each subsequent VF from the previous, 0 for no change, or + * MC_CMD_RESOURCE_INSTANCE_ANY to allow the system to allocate a stride. + */ +#define MC_CMD_SET_SRIOV_CFG_IN_VF_STRIDE_OFST 16 + +/* MC_CMD_SET_SRIOV_CFG_OUT msgresponse */ +#define MC_CMD_SET_SRIOV_CFG_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_GET_VI_ALLOC_INFO + * Get information about number of VI's and base VI number allocated to this + * function. + */ +#define MC_CMD_GET_VI_ALLOC_INFO 0x8d +#undef MC_CMD_0x8d_PRIVILEGE_CTG + +#define MC_CMD_0x8d_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_GET_VI_ALLOC_INFO_IN msgrequest */ +#define MC_CMD_GET_VI_ALLOC_INFO_IN_LEN 0 + +/* MC_CMD_GET_VI_ALLOC_INFO_OUT msgresponse */ +#define MC_CMD_GET_VI_ALLOC_INFO_OUT_LEN 12 +/* The number of VIs allocated on this function */ +#define MC_CMD_GET_VI_ALLOC_INFO_OUT_VI_COUNT_OFST 0 +/* The base absolute VI number allocated to this function. Required to + * correctly interpret wakeup events. + */ +#define MC_CMD_GET_VI_ALLOC_INFO_OUT_VI_BASE_OFST 4 +/* Function's port vi_shift value (always 0 on Huntington) */ +#define MC_CMD_GET_VI_ALLOC_INFO_OUT_VI_SHIFT_OFST 8 + + +/***********************************/ +/* MC_CMD_DUMP_VI_STATE + * For CmdClient use. Dump pertinent information on a specific absolute VI. + */ +#define MC_CMD_DUMP_VI_STATE 0x8e +#undef MC_CMD_0x8e_PRIVILEGE_CTG + +#define MC_CMD_0x8e_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_DUMP_VI_STATE_IN msgrequest */ +#define MC_CMD_DUMP_VI_STATE_IN_LEN 4 +/* The VI number to query. */ +#define MC_CMD_DUMP_VI_STATE_IN_VI_NUMBER_OFST 0 + +/* MC_CMD_DUMP_VI_STATE_OUT msgresponse */ +#define MC_CMD_DUMP_VI_STATE_OUT_LEN 96 +/* The PF part of the function owning this VI. */ +#define MC_CMD_DUMP_VI_STATE_OUT_OWNER_PF_OFST 0 +#define MC_CMD_DUMP_VI_STATE_OUT_OWNER_PF_LEN 2 +/* The VF part of the function owning this VI. */ +#define MC_CMD_DUMP_VI_STATE_OUT_OWNER_VF_OFST 2 +#define MC_CMD_DUMP_VI_STATE_OUT_OWNER_VF_LEN 2 +/* Base of VIs allocated to this function. */ +#define MC_CMD_DUMP_VI_STATE_OUT_FUNC_VI_BASE_OFST 4 +#define MC_CMD_DUMP_VI_STATE_OUT_FUNC_VI_BASE_LEN 2 +/* Count of VIs allocated to the owner function. */ +#define MC_CMD_DUMP_VI_STATE_OUT_FUNC_VI_COUNT_OFST 6 +#define MC_CMD_DUMP_VI_STATE_OUT_FUNC_VI_COUNT_LEN 2 +/* Base interrupt vector allocated to this function. */ +#define MC_CMD_DUMP_VI_STATE_OUT_FUNC_VECTOR_BASE_OFST 8 +#define MC_CMD_DUMP_VI_STATE_OUT_FUNC_VECTOR_BASE_LEN 2 +/* Number of interrupt vectors allocated to this function. */ +#define MC_CMD_DUMP_VI_STATE_OUT_FUNC_VECTOR_COUNT_OFST 10 +#define MC_CMD_DUMP_VI_STATE_OUT_FUNC_VECTOR_COUNT_LEN 2 +/* Raw evq ptr table data. */ +#define MC_CMD_DUMP_VI_STATE_OUT_VI_EVQ_PTR_RAW_OFST 12 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_EVQ_PTR_RAW_LEN 8 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_EVQ_PTR_RAW_LO_OFST 12 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_EVQ_PTR_RAW_HI_OFST 16 +/* Raw evq timer table data. */ +#define MC_CMD_DUMP_VI_STATE_OUT_VI_EV_TIMER_RAW_OFST 20 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_EV_TIMER_RAW_LEN 8 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_EV_TIMER_RAW_LO_OFST 20 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_EV_TIMER_RAW_HI_OFST 24 +/* Combined metadata field. */ +#define MC_CMD_DUMP_VI_STATE_OUT_VI_EV_META_OFST 28 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_EV_META_BUFS_BASE_LBN 0 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_EV_META_BUFS_BASE_WIDTH 16 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_EV_META_BUFS_NPAGES_LBN 16 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_EV_META_BUFS_NPAGES_WIDTH 8 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_EV_META_WKUP_REF_LBN 24 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_EV_META_WKUP_REF_WIDTH 8 +/* TXDPCPU raw table data for queue. */ +#define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_RAW_TBL_0_OFST 32 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_RAW_TBL_0_LEN 8 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_RAW_TBL_0_LO_OFST 32 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_RAW_TBL_0_HI_OFST 36 +/* TXDPCPU raw table data for queue. */ +#define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_RAW_TBL_1_OFST 40 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_RAW_TBL_1_LEN 8 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_RAW_TBL_1_LO_OFST 40 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_RAW_TBL_1_HI_OFST 44 +/* TXDPCPU raw table data for queue. */ +#define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_RAW_TBL_2_OFST 48 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_RAW_TBL_2_LEN 8 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_RAW_TBL_2_LO_OFST 48 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_RAW_TBL_2_HI_OFST 52 +/* Combined metadata field. */ +#define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_META_OFST 56 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_META_LEN 8 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_META_LO_OFST 56 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_META_HI_OFST 60 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_META_BUFS_BASE_LBN 0 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_META_BUFS_BASE_WIDTH 16 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_META_BUFS_NPAGES_LBN 16 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_META_BUFS_NPAGES_WIDTH 8 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_META_QSTATE_LBN 24 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_META_QSTATE_WIDTH 8 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_META_WAITCOUNT_LBN 32 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_META_WAITCOUNT_WIDTH 8 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_PADDING_LBN 40 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_PADDING_WIDTH 24 +/* RXDPCPU raw table data for queue. */ +#define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_RAW_TBL_0_OFST 64 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_RAW_TBL_0_LEN 8 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_RAW_TBL_0_LO_OFST 64 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_RAW_TBL_0_HI_OFST 68 +/* RXDPCPU raw table data for queue. */ +#define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_RAW_TBL_1_OFST 72 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_RAW_TBL_1_LEN 8 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_RAW_TBL_1_LO_OFST 72 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_RAW_TBL_1_HI_OFST 76 +/* Reserved, currently 0. */ +#define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_RAW_TBL_2_OFST 80 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_RAW_TBL_2_LEN 8 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_RAW_TBL_2_LO_OFST 80 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_RAW_TBL_2_HI_OFST 84 +/* Combined metadata field. */ +#define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_META_OFST 88 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_META_LEN 8 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_META_LO_OFST 88 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_META_HI_OFST 92 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_META_BUFS_BASE_LBN 0 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_META_BUFS_BASE_WIDTH 16 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_META_BUFS_NPAGES_LBN 16 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_META_BUFS_NPAGES_WIDTH 8 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_META_QSTATE_LBN 24 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_META_QSTATE_WIDTH 8 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_META_WAITCOUNT_LBN 32 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_META_WAITCOUNT_WIDTH 8 + + +/***********************************/ +/* MC_CMD_ALLOC_PIOBUF + * Allocate a push I/O buffer for later use with a tx queue. + */ +#define MC_CMD_ALLOC_PIOBUF 0x8f +#undef MC_CMD_0x8f_PRIVILEGE_CTG + +#define MC_CMD_0x8f_PRIVILEGE_CTG SRIOV_CTG_ONLOAD + +/* MC_CMD_ALLOC_PIOBUF_IN msgrequest */ +#define MC_CMD_ALLOC_PIOBUF_IN_LEN 0 + +/* MC_CMD_ALLOC_PIOBUF_OUT msgresponse */ +#define MC_CMD_ALLOC_PIOBUF_OUT_LEN 4 +/* Handle for allocated push I/O buffer. */ +#define MC_CMD_ALLOC_PIOBUF_OUT_PIOBUF_HANDLE_OFST 0 + + +/***********************************/ +/* MC_CMD_FREE_PIOBUF + * Free a push I/O buffer. + */ +#define MC_CMD_FREE_PIOBUF 0x90 +#undef MC_CMD_0x90_PRIVILEGE_CTG + +#define MC_CMD_0x90_PRIVILEGE_CTG SRIOV_CTG_ONLOAD + +/* MC_CMD_FREE_PIOBUF_IN msgrequest */ +#define MC_CMD_FREE_PIOBUF_IN_LEN 4 +/* Handle for allocated push I/O buffer. */ +#define MC_CMD_FREE_PIOBUF_IN_PIOBUF_HANDLE_OFST 0 + +/* MC_CMD_FREE_PIOBUF_OUT msgresponse */ +#define MC_CMD_FREE_PIOBUF_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_GET_VI_TLP_PROCESSING + * Get TLP steering and ordering information for a VI. + */ +#define MC_CMD_GET_VI_TLP_PROCESSING 0xb0 +#undef MC_CMD_0xb0_PRIVILEGE_CTG + +#define MC_CMD_0xb0_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_GET_VI_TLP_PROCESSING_IN msgrequest */ +#define MC_CMD_GET_VI_TLP_PROCESSING_IN_LEN 4 +/* VI number to get information for. */ +#define MC_CMD_GET_VI_TLP_PROCESSING_IN_INSTANCE_OFST 0 + +/* MC_CMD_GET_VI_TLP_PROCESSING_OUT msgresponse */ +#define MC_CMD_GET_VI_TLP_PROCESSING_OUT_LEN 4 +/* Transaction processing steering hint 1 for use with the Rx Queue. */ +#define MC_CMD_GET_VI_TLP_PROCESSING_OUT_TPH_TAG1_RX_OFST 0 +#define MC_CMD_GET_VI_TLP_PROCESSING_OUT_TPH_TAG1_RX_LEN 1 +/* Transaction processing steering hint 2 for use with the Ev Queue. */ +#define MC_CMD_GET_VI_TLP_PROCESSING_OUT_TPH_TAG2_EV_OFST 1 +#define MC_CMD_GET_VI_TLP_PROCESSING_OUT_TPH_TAG2_EV_LEN 1 +/* Use Relaxed ordering model for TLPs on this VI. */ +#define MC_CMD_GET_VI_TLP_PROCESSING_OUT_RELAXED_ORDERING_LBN 16 +#define MC_CMD_GET_VI_TLP_PROCESSING_OUT_RELAXED_ORDERING_WIDTH 1 +/* Use ID based ordering for TLPs on this VI. */ +#define MC_CMD_GET_VI_TLP_PROCESSING_OUT_ID_BASED_ORDERING_LBN 17 +#define MC_CMD_GET_VI_TLP_PROCESSING_OUT_ID_BASED_ORDERING_WIDTH 1 +/* Set no snoop bit for TLPs on this VI. */ +#define MC_CMD_GET_VI_TLP_PROCESSING_OUT_NO_SNOOP_LBN 18 +#define MC_CMD_GET_VI_TLP_PROCESSING_OUT_NO_SNOOP_WIDTH 1 +/* Enable TPH for TLPs on this VI. */ +#define MC_CMD_GET_VI_TLP_PROCESSING_OUT_TPH_ON_LBN 19 +#define MC_CMD_GET_VI_TLP_PROCESSING_OUT_TPH_ON_WIDTH 1 +#define MC_CMD_GET_VI_TLP_PROCESSING_OUT_DATA_OFST 0 + + +/***********************************/ +/* MC_CMD_SET_VI_TLP_PROCESSING + * Set TLP steering and ordering information for a VI. + */ +#define MC_CMD_SET_VI_TLP_PROCESSING 0xb1 +#undef MC_CMD_0xb1_PRIVILEGE_CTG + +#define MC_CMD_0xb1_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_SET_VI_TLP_PROCESSING_IN msgrequest */ +#define MC_CMD_SET_VI_TLP_PROCESSING_IN_LEN 8 +/* VI number to set information for. */ +#define MC_CMD_SET_VI_TLP_PROCESSING_IN_INSTANCE_OFST 0 +/* Transaction processing steering hint 1 for use with the Rx Queue. */ +#define MC_CMD_SET_VI_TLP_PROCESSING_IN_TPH_TAG1_RX_OFST 4 +#define MC_CMD_SET_VI_TLP_PROCESSING_IN_TPH_TAG1_RX_LEN 1 +/* Transaction processing steering hint 2 for use with the Ev Queue. */ +#define MC_CMD_SET_VI_TLP_PROCESSING_IN_TPH_TAG2_EV_OFST 5 +#define MC_CMD_SET_VI_TLP_PROCESSING_IN_TPH_TAG2_EV_LEN 1 +/* Use Relaxed ordering model for TLPs on this VI. */ +#define MC_CMD_SET_VI_TLP_PROCESSING_IN_RELAXED_ORDERING_LBN 48 +#define MC_CMD_SET_VI_TLP_PROCESSING_IN_RELAXED_ORDERING_WIDTH 1 +/* Use ID based ordering for TLPs on this VI. */ +#define MC_CMD_SET_VI_TLP_PROCESSING_IN_ID_BASED_ORDERING_LBN 49 +#define MC_CMD_SET_VI_TLP_PROCESSING_IN_ID_BASED_ORDERING_WIDTH 1 +/* Set the no snoop bit for TLPs on this VI. */ +#define MC_CMD_SET_VI_TLP_PROCESSING_IN_NO_SNOOP_LBN 50 +#define MC_CMD_SET_VI_TLP_PROCESSING_IN_NO_SNOOP_WIDTH 1 +/* Enable TPH for TLPs on this VI. */ +#define MC_CMD_SET_VI_TLP_PROCESSING_IN_TPH_ON_LBN 51 +#define MC_CMD_SET_VI_TLP_PROCESSING_IN_TPH_ON_WIDTH 1 +#define MC_CMD_SET_VI_TLP_PROCESSING_IN_DATA_OFST 4 + +/* MC_CMD_SET_VI_TLP_PROCESSING_OUT msgresponse */ +#define MC_CMD_SET_VI_TLP_PROCESSING_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_GET_TLP_PROCESSING_GLOBALS + * Get global PCIe steering and transaction processing configuration. + */ +#define MC_CMD_GET_TLP_PROCESSING_GLOBALS 0xbc +#undef MC_CMD_0xbc_PRIVILEGE_CTG + +#define MC_CMD_0xbc_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_GET_TLP_PROCESSING_GLOBALS_IN msgrequest */ +#define MC_CMD_GET_TLP_PROCESSING_GLOBALS_IN_LEN 4 +#define MC_CMD_GET_TLP_PROCESSING_GLOBALS_IN_TLP_GLOBAL_CATEGORY_OFST 0 +/* enum: MISC. */ +#define MC_CMD_GET_TLP_PROCESSING_GLOBALS_IN_TLP_GLOBAL_CATEGORY_MISC 0x0 +/* enum: IDO. */ +#define MC_CMD_GET_TLP_PROCESSING_GLOBALS_IN_TLP_GLOBAL_CATEGORY_IDO 0x1 +/* enum: RO. */ +#define MC_CMD_GET_TLP_PROCESSING_GLOBALS_IN_TLP_GLOBAL_CATEGORY_RO 0x2 +/* enum: TPH Type. */ +#define MC_CMD_GET_TLP_PROCESSING_GLOBALS_IN_TLP_GLOBAL_CATEGORY_TPH_TYPE 0x3 + +/* MC_CMD_GET_TLP_PROCESSING_GLOBALS_OUT msgresponse */ +#define MC_CMD_GET_TLP_PROCESSING_GLOBALS_OUT_LEN 8 +#define MC_CMD_GET_TLP_PROCESSING_GLOBALS_OUT_TLP_GLOBAL_CATEGORY_OFST 0 +/* Enum values, see field(s): */ +/* MC_CMD_GET_TLP_PROCESSING_GLOBALS_IN/TLP_GLOBAL_CATEGORY */ +/* Amalgamated TLP info word. */ +#define MC_CMD_GET_TLP_PROCESSING_GLOBALS_OUT_TLP_INFO_WORD_OFST 4 +#define MC_CMD_GET_TLP_PROCESSING_GLOBALS_OUT_TLP_INFO_MISC_WTAG_EN_LBN 0 +#define MC_CMD_GET_TLP_PROCESSING_GLOBALS_OUT_TLP_INFO_MISC_WTAG_EN_WIDTH 1 +#define MC_CMD_GET_TLP_PROCESSING_GLOBALS_OUT_TLP_INFO_MISC_SPARE_LBN 1 +#define MC_CMD_GET_TLP_PROCESSING_GLOBALS_OUT_TLP_INFO_MISC_SPARE_WIDTH 31 +#define MC_CMD_GET_TLP_PROCESSING_GLOBALS_OUT_TLP_INFO_IDO_DL_EN_LBN 0 +#define MC_CMD_GET_TLP_PROCESSING_GLOBALS_OUT_TLP_INFO_IDO_DL_EN_WIDTH 1 +#define MC_CMD_GET_TLP_PROCESSING_GLOBALS_OUT_TLP_INFO_IDO_TX_EN_LBN 1 +#define MC_CMD_GET_TLP_PROCESSING_GLOBALS_OUT_TLP_INFO_IDO_TX_EN_WIDTH 1 +#define MC_CMD_GET_TLP_PROCESSING_GLOBALS_OUT_TLP_INFO_IDO_EV_EN_LBN 2 +#define MC_CMD_GET_TLP_PROCESSING_GLOBALS_OUT_TLP_INFO_IDO_EV_EN_WIDTH 1 +#define MC_CMD_GET_TLP_PROCESSING_GLOBALS_OUT_TLP_INFO_IDO_RX_EN_LBN 3 +#define MC_CMD_GET_TLP_PROCESSING_GLOBALS_OUT_TLP_INFO_IDO_RX_EN_WIDTH 1 +#define MC_CMD_GET_TLP_PROCESSING_GLOBALS_OUT_TLP_INFO_IDO_SPARE_LBN 4 +#define MC_CMD_GET_TLP_PROCESSING_GLOBALS_OUT_TLP_INFO_IDO_SPARE_WIDTH 28 +#define MC_CMD_GET_TLP_PROCESSING_GLOBALS_OUT_TLP_INFO_RO_RXDMA_EN_LBN 0 +#define MC_CMD_GET_TLP_PROCESSING_GLOBALS_OUT_TLP_INFO_RO_RXDMA_EN_WIDTH 1 +#define MC_CMD_GET_TLP_PROCESSING_GLOBALS_OUT_TLP_INFO_RO_TXDMA_EN_LBN 1 +#define MC_CMD_GET_TLP_PROCESSING_GLOBALS_OUT_TLP_INFO_RO_TXDMA_EN_WIDTH 1 +#define MC_CMD_GET_TLP_PROCESSING_GLOBALS_OUT_TLP_INFO_RO_DL_EN_LBN 2 +#define MC_CMD_GET_TLP_PROCESSING_GLOBALS_OUT_TLP_INFO_RO_DL_EN_WIDTH 1 +#define MC_CMD_GET_TLP_PROCESSING_GLOBALS_OUT_TLP_INFO_RO_SPARE_LBN 3 +#define MC_CMD_GET_TLP_PROCESSING_GLOBALS_OUT_TLP_INFO_RO_SPARE_WIDTH 29 +#define MC_CMD_GET_TLP_PROCESSING_GLOBALS_OUT_TLP_INFO_TPH_TYPE_MSIX_LBN 0 +#define MC_CMD_GET_TLP_PROCESSING_GLOBALS_OUT_TLP_INFO_TPH_TYPE_MSIX_WIDTH 2 +#define MC_CMD_GET_TLP_PROCESSING_GLOBALS_OUT_TLP_INFO_TPH_TYPE_DL_LBN 2 +#define MC_CMD_GET_TLP_PROCESSING_GLOBALS_OUT_TLP_INFO_TPH_TYPE_DL_WIDTH 2 +#define MC_CMD_GET_TLP_PROCESSING_GLOBALS_OUT_TLP_INFO_TPH_TYPE_TX_LBN 4 +#define MC_CMD_GET_TLP_PROCESSING_GLOBALS_OUT_TLP_INFO_TPH_TYPE_TX_WIDTH 2 +#define MC_CMD_GET_TLP_PROCESSING_GLOBALS_OUT_TLP_INFO_TPH_TYPE_EV_LBN 6 +#define MC_CMD_GET_TLP_PROCESSING_GLOBALS_OUT_TLP_INFO_TPH_TYPE_EV_WIDTH 2 +#define MC_CMD_GET_TLP_PROCESSING_GLOBALS_OUT_TLP_INFO_TPH_TYPE_RX_LBN 8 +#define MC_CMD_GET_TLP_PROCESSING_GLOBALS_OUT_TLP_INFO_TPH_TYPE_RX_WIDTH 2 +#define MC_CMD_GET_TLP_PROCESSING_GLOBALS_OUT_TLP_INFO_TLP_TYPE_SPARE_LBN 9 +#define MC_CMD_GET_TLP_PROCESSING_GLOBALS_OUT_TLP_INFO_TLP_TYPE_SPARE_WIDTH 23 + + +/***********************************/ +/* MC_CMD_SET_TLP_PROCESSING_GLOBALS + * Set global PCIe steering and transaction processing configuration. + */ +#define MC_CMD_SET_TLP_PROCESSING_GLOBALS 0xbd +#undef MC_CMD_0xbd_PRIVILEGE_CTG + +#define MC_CMD_0xbd_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_SET_TLP_PROCESSING_GLOBALS_IN msgrequest */ +#define MC_CMD_SET_TLP_PROCESSING_GLOBALS_IN_LEN 8 +#define MC_CMD_SET_TLP_PROCESSING_GLOBALS_IN_TLP_GLOBAL_CATEGORY_OFST 0 +/* Enum values, see field(s): */ +/* MC_CMD_GET_TLP_PROCESSING_GLOBALS/MC_CMD_GET_TLP_PROCESSING_GLOBALS_IN/TLP_GLOBAL_CATEGORY */ +/* Amalgamated TLP info word. */ +#define MC_CMD_SET_TLP_PROCESSING_GLOBALS_IN_TLP_INFO_WORD_OFST 4 +#define MC_CMD_SET_TLP_PROCESSING_GLOBALS_IN_TLP_INFO_MISC_WTAG_EN_LBN 0 +#define MC_CMD_SET_TLP_PROCESSING_GLOBALS_IN_TLP_INFO_MISC_WTAG_EN_WIDTH 1 +#define MC_CMD_SET_TLP_PROCESSING_GLOBALS_IN_TLP_INFO_IDO_DL_EN_LBN 0 +#define MC_CMD_SET_TLP_PROCESSING_GLOBALS_IN_TLP_INFO_IDO_DL_EN_WIDTH 1 +#define MC_CMD_SET_TLP_PROCESSING_GLOBALS_IN_TLP_INFO_IDO_TX_EN_LBN 1 +#define MC_CMD_SET_TLP_PROCESSING_GLOBALS_IN_TLP_INFO_IDO_TX_EN_WIDTH 1 +#define MC_CMD_SET_TLP_PROCESSING_GLOBALS_IN_TLP_INFO_IDO_EV_EN_LBN 2 +#define MC_CMD_SET_TLP_PROCESSING_GLOBALS_IN_TLP_INFO_IDO_EV_EN_WIDTH 1 +#define MC_CMD_SET_TLP_PROCESSING_GLOBALS_IN_TLP_INFO_IDO_RX_EN_LBN 3 +#define MC_CMD_SET_TLP_PROCESSING_GLOBALS_IN_TLP_INFO_IDO_RX_EN_WIDTH 1 +#define MC_CMD_SET_TLP_PROCESSING_GLOBALS_IN_TLP_INFO_RO_RXDMA_EN_LBN 0 +#define MC_CMD_SET_TLP_PROCESSING_GLOBALS_IN_TLP_INFO_RO_RXDMA_EN_WIDTH 1 +#define MC_CMD_SET_TLP_PROCESSING_GLOBALS_IN_TLP_INFO_RO_TXDMA_EN_LBN 1 +#define MC_CMD_SET_TLP_PROCESSING_GLOBALS_IN_TLP_INFO_RO_TXDMA_EN_WIDTH 1 +#define MC_CMD_SET_TLP_PROCESSING_GLOBALS_IN_TLP_INFO_RO_DL_EN_LBN 2 +#define MC_CMD_SET_TLP_PROCESSING_GLOBALS_IN_TLP_INFO_RO_DL_EN_WIDTH 1 +#define MC_CMD_SET_TLP_PROCESSING_GLOBALS_IN_TLP_INFO_TPH_TYPE_MSIX_LBN 0 +#define MC_CMD_SET_TLP_PROCESSING_GLOBALS_IN_TLP_INFO_TPH_TYPE_MSIX_WIDTH 2 +#define MC_CMD_SET_TLP_PROCESSING_GLOBALS_IN_TLP_INFO_TPH_TYPE_DL_LBN 2 +#define MC_CMD_SET_TLP_PROCESSING_GLOBALS_IN_TLP_INFO_TPH_TYPE_DL_WIDTH 2 +#define MC_CMD_SET_TLP_PROCESSING_GLOBALS_IN_TLP_INFO_TPH_TYPE_TX_LBN 4 +#define MC_CMD_SET_TLP_PROCESSING_GLOBALS_IN_TLP_INFO_TPH_TYPE_TX_WIDTH 2 +#define MC_CMD_SET_TLP_PROCESSING_GLOBALS_IN_TLP_INFO_TPH_TYPE_EV_LBN 6 +#define MC_CMD_SET_TLP_PROCESSING_GLOBALS_IN_TLP_INFO_TPH_TYPE_EV_WIDTH 2 +#define MC_CMD_SET_TLP_PROCESSING_GLOBALS_IN_TLP_INFO_TPH_TYPE_RX_LBN 8 +#define MC_CMD_SET_TLP_PROCESSING_GLOBALS_IN_TLP_INFO_TPH_TYPE_RX_WIDTH 2 +#define MC_CMD_SET_TLP_PROCESSING_GLOBALS_IN_TLP_INFO_SPARE_LBN 10 +#define MC_CMD_SET_TLP_PROCESSING_GLOBALS_IN_TLP_INFO_SPARE_WIDTH 22 + +/* MC_CMD_SET_TLP_PROCESSING_GLOBALS_OUT msgresponse */ +#define MC_CMD_SET_TLP_PROCESSING_GLOBALS_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_SATELLITE_DOWNLOAD + * Download a new set of images to the satellite CPUs from the host. + */ +#define MC_CMD_SATELLITE_DOWNLOAD 0x91 +#undef MC_CMD_0x91_PRIVILEGE_CTG + +#define MC_CMD_0x91_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_SATELLITE_DOWNLOAD_IN msgrequest: The reset requirements for the CPUs + * are subtle, and so downloads must proceed in a number of phases. + * + * 1) PHASE_RESET with a target of TARGET_ALL and chunk ID/length of 0. + * + * 2) PHASE_IMEMS for each of the IMEM targets (target IDs 0-11). Each download + * may consist of multiple chunks. The final chunk (with CHUNK_ID_LAST) should + * be a checksum (a simple 32-bit sum) of the transferred data. An individual + * download may be aborted using CHUNK_ID_ABORT. + * + * 3) PHASE_VECTORS for each of the vector table targets (target IDs 12-15), + * similar to PHASE_IMEMS. + * + * 4) PHASE_READY with a target of TARGET_ALL and chunk ID/length of 0. + * + * After any error (a requested abort is not considered to be an error) the + * sequence must be restarted from PHASE_RESET. + */ +#define MC_CMD_SATELLITE_DOWNLOAD_IN_LENMIN 20 +#define MC_CMD_SATELLITE_DOWNLOAD_IN_LENMAX 252 +#define MC_CMD_SATELLITE_DOWNLOAD_IN_LEN(num) (16+4*(num)) +/* Download phase. (Note: the IDLE phase is used internally and is never valid + * in a command from the host.) + */ +#define MC_CMD_SATELLITE_DOWNLOAD_IN_PHASE_OFST 0 +#define MC_CMD_SATELLITE_DOWNLOAD_IN_PHASE_IDLE 0x0 /* enum */ +#define MC_CMD_SATELLITE_DOWNLOAD_IN_PHASE_RESET 0x1 /* enum */ +#define MC_CMD_SATELLITE_DOWNLOAD_IN_PHASE_IMEMS 0x2 /* enum */ +#define MC_CMD_SATELLITE_DOWNLOAD_IN_PHASE_VECTORS 0x3 /* enum */ +#define MC_CMD_SATELLITE_DOWNLOAD_IN_PHASE_READY 0x4 /* enum */ +/* Target for download. (These match the blob numbers defined in + * mc_flash_layout.h.) + */ +#define MC_CMD_SATELLITE_DOWNLOAD_IN_TARGET_OFST 4 +/* enum: Valid in phase 2 (PHASE_IMEMS) only */ +#define MC_CMD_SATELLITE_DOWNLOAD_IN_TARGET_TXDI_TEXT 0x0 +/* enum: Valid in phase 2 (PHASE_IMEMS) only */ +#define MC_CMD_SATELLITE_DOWNLOAD_IN_TARGET_RXDI_TEXT 0x1 +/* enum: Valid in phase 2 (PHASE_IMEMS) only */ +#define MC_CMD_SATELLITE_DOWNLOAD_IN_TARGET_TXDP_TEXT 0x2 +/* enum: Valid in phase 2 (PHASE_IMEMS) only */ +#define MC_CMD_SATELLITE_DOWNLOAD_IN_TARGET_RXDP_TEXT 0x3 +/* enum: Valid in phase 2 (PHASE_IMEMS) only */ +#define MC_CMD_SATELLITE_DOWNLOAD_IN_TARGET_RXHRSL_HR_LUT 0x4 +/* enum: Valid in phase 2 (PHASE_IMEMS) only */ +#define MC_CMD_SATELLITE_DOWNLOAD_IN_TARGET_RXHRSL_HR_LUT_CFG 0x5 +/* enum: Valid in phase 2 (PHASE_IMEMS) only */ +#define MC_CMD_SATELLITE_DOWNLOAD_IN_TARGET_TXHRSL_HR_LUT 0x6 +/* enum: Valid in phase 2 (PHASE_IMEMS) only */ +#define MC_CMD_SATELLITE_DOWNLOAD_IN_TARGET_TXHRSL_HR_LUT_CFG 0x7 +/* enum: Valid in phase 2 (PHASE_IMEMS) only */ +#define MC_CMD_SATELLITE_DOWNLOAD_IN_TARGET_RXHRSL_HR_PGM 0x8 +/* enum: Valid in phase 2 (PHASE_IMEMS) only */ +#define MC_CMD_SATELLITE_DOWNLOAD_IN_TARGET_RXHRSL_SL_PGM 0x9 +/* enum: Valid in phase 2 (PHASE_IMEMS) only */ +#define MC_CMD_SATELLITE_DOWNLOAD_IN_TARGET_TXHRSL_HR_PGM 0xa +/* enum: Valid in phase 2 (PHASE_IMEMS) only */ +#define MC_CMD_SATELLITE_DOWNLOAD_IN_TARGET_TXHRSL_SL_PGM 0xb +/* enum: Valid in phase 3 (PHASE_VECTORS) only */ +#define MC_CMD_SATELLITE_DOWNLOAD_IN_TARGET_RXDI_VTBL0 0xc +/* enum: Valid in phase 3 (PHASE_VECTORS) only */ +#define MC_CMD_SATELLITE_DOWNLOAD_IN_TARGET_TXDI_VTBL0 0xd +/* enum: Valid in phase 3 (PHASE_VECTORS) only */ +#define MC_CMD_SATELLITE_DOWNLOAD_IN_TARGET_RXDI_VTBL1 0xe +/* enum: Valid in phase 3 (PHASE_VECTORS) only */ +#define MC_CMD_SATELLITE_DOWNLOAD_IN_TARGET_TXDI_VTBL1 0xf +/* enum: Valid in phases 1 (PHASE_RESET) and 4 (PHASE_READY) only */ +#define MC_CMD_SATELLITE_DOWNLOAD_IN_TARGET_ALL 0xffffffff +/* Chunk ID, or CHUNK_ID_LAST or CHUNK_ID_ABORT */ +#define MC_CMD_SATELLITE_DOWNLOAD_IN_CHUNK_ID_OFST 8 +/* enum: Last chunk, containing checksum rather than data */ +#define MC_CMD_SATELLITE_DOWNLOAD_IN_CHUNK_ID_LAST 0xffffffff +/* enum: Abort download of this item */ +#define MC_CMD_SATELLITE_DOWNLOAD_IN_CHUNK_ID_ABORT 0xfffffffe +/* Length of this chunk in bytes */ +#define MC_CMD_SATELLITE_DOWNLOAD_IN_CHUNK_LEN_OFST 12 +/* Data for this chunk */ +#define MC_CMD_SATELLITE_DOWNLOAD_IN_CHUNK_DATA_OFST 16 +#define MC_CMD_SATELLITE_DOWNLOAD_IN_CHUNK_DATA_LEN 4 +#define MC_CMD_SATELLITE_DOWNLOAD_IN_CHUNK_DATA_MINNUM 1 +#define MC_CMD_SATELLITE_DOWNLOAD_IN_CHUNK_DATA_MAXNUM 59 + +/* MC_CMD_SATELLITE_DOWNLOAD_OUT msgresponse */ +#define MC_CMD_SATELLITE_DOWNLOAD_OUT_LEN 8 +/* Same as MC_CMD_ERR field, but included as 0 in success cases */ +#define MC_CMD_SATELLITE_DOWNLOAD_OUT_RESULT_OFST 0 +/* Extra status information */ +#define MC_CMD_SATELLITE_DOWNLOAD_OUT_INFO_OFST 4 +/* enum: Code download OK, completed. */ +#define MC_CMD_SATELLITE_DOWNLOAD_OUT_OK_COMPLETE 0x0 +/* enum: Code download aborted as requested. */ +#define MC_CMD_SATELLITE_DOWNLOAD_OUT_OK_ABORTED 0x1 +/* enum: Code download OK so far, send next chunk. */ +#define MC_CMD_SATELLITE_DOWNLOAD_OUT_OK_NEXT_CHUNK 0x2 +/* enum: Download phases out of sequence */ +#define MC_CMD_SATELLITE_DOWNLOAD_OUT_ERR_BAD_PHASE 0x100 +/* enum: Bad target for this phase */ +#define MC_CMD_SATELLITE_DOWNLOAD_OUT_ERR_BAD_TARGET 0x101 +/* enum: Chunk ID out of sequence */ +#define MC_CMD_SATELLITE_DOWNLOAD_OUT_ERR_BAD_CHUNK_ID 0x200 +/* enum: Chunk length zero or too large */ +#define MC_CMD_SATELLITE_DOWNLOAD_OUT_ERR_BAD_CHUNK_LEN 0x201 +/* enum: Checksum was incorrect */ +#define MC_CMD_SATELLITE_DOWNLOAD_OUT_ERR_BAD_CHECKSUM 0x300 + + +/***********************************/ +/* MC_CMD_GET_CAPABILITIES + * Get device capabilities. + * + * This is supplementary to the MC_CMD_GET_BOARD_CFG command, and intended to + * reference inherent device capabilities as opposed to current NVRAM config. + */ +#define MC_CMD_GET_CAPABILITIES 0xbe +#undef MC_CMD_0xbe_PRIVILEGE_CTG + +#define MC_CMD_0xbe_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_GET_CAPABILITIES_IN msgrequest */ +#define MC_CMD_GET_CAPABILITIES_IN_LEN 0 + +/* MC_CMD_GET_CAPABILITIES_OUT msgresponse */ +#define MC_CMD_GET_CAPABILITIES_OUT_LEN 20 +/* First word of flags. */ +#define MC_CMD_GET_CAPABILITIES_OUT_FLAGS1_OFST 0 +#define MC_CMD_GET_CAPABILITIES_OUT_VPORT_RECONFIGURE_LBN 3 +#define MC_CMD_GET_CAPABILITIES_OUT_VPORT_RECONFIGURE_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_OUT_TX_STRIPING_LBN 4 +#define MC_CMD_GET_CAPABILITIES_OUT_TX_STRIPING_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_OUT_VADAPTOR_QUERY_LBN 5 +#define MC_CMD_GET_CAPABILITIES_OUT_VADAPTOR_QUERY_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_OUT_EVB_PORT_VLAN_RESTRICT_LBN 6 +#define MC_CMD_GET_CAPABILITIES_OUT_EVB_PORT_VLAN_RESTRICT_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_OUT_DRV_ATTACH_PREBOOT_LBN 7 +#define MC_CMD_GET_CAPABILITIES_OUT_DRV_ATTACH_PREBOOT_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_OUT_RX_FORCE_EVENT_MERGING_LBN 8 +#define MC_CMD_GET_CAPABILITIES_OUT_RX_FORCE_EVENT_MERGING_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_OUT_SET_MAC_ENHANCED_LBN 9 +#define MC_CMD_GET_CAPABILITIES_OUT_SET_MAC_ENHANCED_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_OUT_UNKNOWN_UCAST_DST_FILTER_ALWAYS_MULTI_RECIPIENT_LBN 10 +#define MC_CMD_GET_CAPABILITIES_OUT_UNKNOWN_UCAST_DST_FILTER_ALWAYS_MULTI_RECIPIENT_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_OUT_VADAPTOR_PERMIT_SET_MAC_WHEN_FILTERS_INSTALLED_LBN 11 +#define MC_CMD_GET_CAPABILITIES_OUT_VADAPTOR_PERMIT_SET_MAC_WHEN_FILTERS_INSTALLED_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_OUT_TX_MAC_SECURITY_FILTERING_LBN 12 +#define MC_CMD_GET_CAPABILITIES_OUT_TX_MAC_SECURITY_FILTERING_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_OUT_ADDITIONAL_RSS_MODES_LBN 13 +#define MC_CMD_GET_CAPABILITIES_OUT_ADDITIONAL_RSS_MODES_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_OUT_QBB_LBN 14 +#define MC_CMD_GET_CAPABILITIES_OUT_QBB_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_OUT_RX_PACKED_STREAM_VAR_BUFFERS_LBN 15 +#define MC_CMD_GET_CAPABILITIES_OUT_RX_PACKED_STREAM_VAR_BUFFERS_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_OUT_RX_RSS_LIMITED_LBN 16 +#define MC_CMD_GET_CAPABILITIES_OUT_RX_RSS_LIMITED_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_OUT_RX_PACKED_STREAM_LBN 17 +#define MC_CMD_GET_CAPABILITIES_OUT_RX_PACKED_STREAM_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_OUT_RX_INCLUDE_FCS_LBN 18 +#define MC_CMD_GET_CAPABILITIES_OUT_RX_INCLUDE_FCS_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_OUT_TX_VLAN_INSERTION_LBN 19 +#define MC_CMD_GET_CAPABILITIES_OUT_TX_VLAN_INSERTION_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_OUT_RX_VLAN_STRIPPING_LBN 20 +#define MC_CMD_GET_CAPABILITIES_OUT_RX_VLAN_STRIPPING_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_OUT_TX_TSO_LBN 21 +#define MC_CMD_GET_CAPABILITIES_OUT_TX_TSO_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_OUT_RX_PREFIX_LEN_0_LBN 22 +#define MC_CMD_GET_CAPABILITIES_OUT_RX_PREFIX_LEN_0_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_OUT_RX_PREFIX_LEN_14_LBN 23 +#define MC_CMD_GET_CAPABILITIES_OUT_RX_PREFIX_LEN_14_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_OUT_RX_TIMESTAMP_LBN 24 +#define MC_CMD_GET_CAPABILITIES_OUT_RX_TIMESTAMP_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_OUT_RX_BATCHING_LBN 25 +#define MC_CMD_GET_CAPABILITIES_OUT_RX_BATCHING_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_OUT_MCAST_FILTER_CHAINING_LBN 26 +#define MC_CMD_GET_CAPABILITIES_OUT_MCAST_FILTER_CHAINING_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_OUT_PM_AND_RXDP_COUNTERS_LBN 27 +#define MC_CMD_GET_CAPABILITIES_OUT_PM_AND_RXDP_COUNTERS_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_OUT_RX_DISABLE_SCATTER_LBN 28 +#define MC_CMD_GET_CAPABILITIES_OUT_RX_DISABLE_SCATTER_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_OUT_TX_MCAST_UDP_LOOPBACK_LBN 29 +#define MC_CMD_GET_CAPABILITIES_OUT_TX_MCAST_UDP_LOOPBACK_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_OUT_EVB_LBN 30 +#define MC_CMD_GET_CAPABILITIES_OUT_EVB_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_OUT_VXLAN_NVGRE_LBN 31 +#define MC_CMD_GET_CAPABILITIES_OUT_VXLAN_NVGRE_WIDTH 1 +/* RxDPCPU firmware id. */ +#define MC_CMD_GET_CAPABILITIES_OUT_RX_DPCPU_FW_ID_OFST 4 +#define MC_CMD_GET_CAPABILITIES_OUT_RX_DPCPU_FW_ID_LEN 2 +/* enum: Standard RXDP firmware */ +#define MC_CMD_GET_CAPABILITIES_OUT_RXDP 0x0 +/* enum: Low latency RXDP firmware */ +#define MC_CMD_GET_CAPABILITIES_OUT_RXDP_LOW_LATENCY 0x1 +/* enum: Packed stream RXDP firmware */ +#define MC_CMD_GET_CAPABILITIES_OUT_RXDP_PACKED_STREAM 0x2 +/* enum: BIST RXDP firmware */ +#define MC_CMD_GET_CAPABILITIES_OUT_RXDP_BIST 0x10a +/* enum: RXDP Test firmware image 1 */ +#define MC_CMD_GET_CAPABILITIES_OUT_RXDP_TEST_FW_TO_MC_CUT_THROUGH 0x101 +/* enum: RXDP Test firmware image 2 */ +#define MC_CMD_GET_CAPABILITIES_OUT_RXDP_TEST_FW_TO_MC_STORE_FORWARD 0x102 +/* enum: RXDP Test firmware image 3 */ +#define MC_CMD_GET_CAPABILITIES_OUT_RXDP_TEST_FW_TO_MC_STORE_FORWARD_FIRST 0x103 +/* enum: RXDP Test firmware image 4 */ +#define MC_CMD_GET_CAPABILITIES_OUT_RXDP_TEST_EVERY_EVENT_BATCHABLE 0x104 +/* enum: RXDP Test firmware image 5 */ +#define MC_CMD_GET_CAPABILITIES_OUT_RXDP_TEST_BACKPRESSURE 0x105 +/* enum: RXDP Test firmware image 6 */ +#define MC_CMD_GET_CAPABILITIES_OUT_RXDP_TEST_FW_PACKET_EDITS 0x106 +/* enum: RXDP Test firmware image 7 */ +#define MC_CMD_GET_CAPABILITIES_OUT_RXDP_TEST_FW_RX_HDR_SPLIT 0x107 +/* enum: RXDP Test firmware image 8 */ +#define MC_CMD_GET_CAPABILITIES_OUT_RXDP_TEST_FW_DISABLE_DL 0x108 +/* enum: RXDP Test firmware image 9 */ +#define MC_CMD_GET_CAPABILITIES_OUT_RXDP_TEST_FW_DOORBELL_DELAY 0x10b +/* TxDPCPU firmware id. */ +#define MC_CMD_GET_CAPABILITIES_OUT_TX_DPCPU_FW_ID_OFST 6 +#define MC_CMD_GET_CAPABILITIES_OUT_TX_DPCPU_FW_ID_LEN 2 +/* enum: Standard TXDP firmware */ +#define MC_CMD_GET_CAPABILITIES_OUT_TXDP 0x0 +/* enum: Low latency TXDP firmware */ +#define MC_CMD_GET_CAPABILITIES_OUT_TXDP_LOW_LATENCY 0x1 +/* enum: High packet rate TXDP firmware */ +#define MC_CMD_GET_CAPABILITIES_OUT_TXDP_HIGH_PACKET_RATE 0x3 +/* enum: BIST TXDP firmware */ +#define MC_CMD_GET_CAPABILITIES_OUT_TXDP_BIST 0x12d +/* enum: TXDP Test firmware image 1 */ +#define MC_CMD_GET_CAPABILITIES_OUT_TXDP_TEST_FW_TSO_EDIT 0x101 +/* enum: TXDP Test firmware image 2 */ +#define MC_CMD_GET_CAPABILITIES_OUT_TXDP_TEST_FW_PACKET_EDITS 0x102 +/* enum: TXDP CSR bus test firmware */ +#define MC_CMD_GET_CAPABILITIES_OUT_TXDP_TEST_FW_CSR 0x103 +#define MC_CMD_GET_CAPABILITIES_OUT_RXPD_FW_VERSION_OFST 8 +#define MC_CMD_GET_CAPABILITIES_OUT_RXPD_FW_VERSION_LEN 2 +#define MC_CMD_GET_CAPABILITIES_OUT_RXPD_FW_VERSION_REV_LBN 0 +#define MC_CMD_GET_CAPABILITIES_OUT_RXPD_FW_VERSION_REV_WIDTH 12 +#define MC_CMD_GET_CAPABILITIES_OUT_RXPD_FW_VERSION_TYPE_LBN 12 +#define MC_CMD_GET_CAPABILITIES_OUT_RXPD_FW_VERSION_TYPE_WIDTH 4 +/* enum: reserved value - do not use (may indicate alternative interpretation + * of REV field in future) + */ +#define MC_CMD_GET_CAPABILITIES_OUT_RXPD_FW_TYPE_RESERVED 0x0 +/* enum: Trivial RX PD firmware for early Huntington development (Huntington + * development only) + */ +#define MC_CMD_GET_CAPABILITIES_OUT_RXPD_FW_TYPE_FIRST_PKT 0x1 +/* enum: RX PD firmware with approximately Siena-compatible behaviour + * (Huntington development only) + */ +#define MC_CMD_GET_CAPABILITIES_OUT_RXPD_FW_TYPE_SIENA_COMPAT 0x2 +/* enum: Full featured RX PD production firmware */ +#define MC_CMD_GET_CAPABILITIES_OUT_RXPD_FW_TYPE_FULL_FEATURED 0x3 +/* enum: (deprecated original name for the FULL_FEATURED variant) */ +#define MC_CMD_GET_CAPABILITIES_OUT_RXPD_FW_TYPE_VSWITCH 0x3 +/* enum: siena_compat variant RX PD firmware using PM rather than MAC + * (Huntington development only) + */ +#define MC_CMD_GET_CAPABILITIES_OUT_RXPD_FW_TYPE_SIENA_COMPAT_PM 0x4 +/* enum: Low latency RX PD production firmware */ +#define MC_CMD_GET_CAPABILITIES_OUT_RXPD_FW_TYPE_LOW_LATENCY 0x5 +/* enum: Packed stream RX PD production firmware */ +#define MC_CMD_GET_CAPABILITIES_OUT_RXPD_FW_TYPE_PACKED_STREAM 0x6 +/* enum: RX PD firmware handling layer 2 only for high packet rate performance + * tests (Medford development only) + */ +#define MC_CMD_GET_CAPABILITIES_OUT_RXPD_FW_TYPE_LAYER2_PERF 0x7 +/* enum: Rules engine RX PD production firmware */ +#define MC_CMD_GET_CAPABILITIES_OUT_RXPD_FW_TYPE_RULES_ENGINE 0x8 +/* enum: RX PD firmware for GUE parsing prototype (Medford development only) */ +#define MC_CMD_GET_CAPABILITIES_OUT_RXPD_FW_TYPE_TESTFW_GUE_PROTOTYPE 0xe +/* enum: RX PD firmware parsing but not filtering network overlay tunnel + * encapsulations (Medford development only) + */ +#define MC_CMD_GET_CAPABILITIES_OUT_RXPD_FW_TYPE_TESTFW_ENCAP_PARSING_ONLY 0xf +#define MC_CMD_GET_CAPABILITIES_OUT_TXPD_FW_VERSION_OFST 10 +#define MC_CMD_GET_CAPABILITIES_OUT_TXPD_FW_VERSION_LEN 2 +#define MC_CMD_GET_CAPABILITIES_OUT_TXPD_FW_VERSION_REV_LBN 0 +#define MC_CMD_GET_CAPABILITIES_OUT_TXPD_FW_VERSION_REV_WIDTH 12 +#define MC_CMD_GET_CAPABILITIES_OUT_TXPD_FW_VERSION_TYPE_LBN 12 +#define MC_CMD_GET_CAPABILITIES_OUT_TXPD_FW_VERSION_TYPE_WIDTH 4 +/* enum: reserved value - do not use (may indicate alternative interpretation + * of REV field in future) + */ +#define MC_CMD_GET_CAPABILITIES_OUT_TXPD_FW_TYPE_RESERVED 0x0 +/* enum: Trivial TX PD firmware for early Huntington development (Huntington + * development only) + */ +#define MC_CMD_GET_CAPABILITIES_OUT_TXPD_FW_TYPE_FIRST_PKT 0x1 +/* enum: TX PD firmware with approximately Siena-compatible behaviour + * (Huntington development only) + */ +#define MC_CMD_GET_CAPABILITIES_OUT_TXPD_FW_TYPE_SIENA_COMPAT 0x2 +/* enum: Full featured TX PD production firmware */ +#define MC_CMD_GET_CAPABILITIES_OUT_TXPD_FW_TYPE_FULL_FEATURED 0x3 +/* enum: (deprecated original name for the FULL_FEATURED variant) */ +#define MC_CMD_GET_CAPABILITIES_OUT_TXPD_FW_TYPE_VSWITCH 0x3 +/* enum: siena_compat variant TX PD firmware using PM rather than MAC + * (Huntington development only) + */ +#define MC_CMD_GET_CAPABILITIES_OUT_TXPD_FW_TYPE_SIENA_COMPAT_PM 0x4 +#define MC_CMD_GET_CAPABILITIES_OUT_TXPD_FW_TYPE_LOW_LATENCY 0x5 /* enum */ +/* enum: TX PD firmware handling layer 2 only for high packet rate performance + * tests (Medford development only) + */ +#define MC_CMD_GET_CAPABILITIES_OUT_TXPD_FW_TYPE_LAYER2_PERF 0x7 +/* enum: Rules engine TX PD production firmware */ +#define MC_CMD_GET_CAPABILITIES_OUT_TXPD_FW_TYPE_RULES_ENGINE 0x8 +/* enum: RX PD firmware for GUE parsing prototype (Medford development only) */ +#define MC_CMD_GET_CAPABILITIES_OUT_TXPD_FW_TYPE_TESTFW_GUE_PROTOTYPE 0xe +/* Hardware capabilities of NIC */ +#define MC_CMD_GET_CAPABILITIES_OUT_HW_CAPABILITIES_OFST 12 +/* Licensed capabilities */ +#define MC_CMD_GET_CAPABILITIES_OUT_LICENSE_CAPABILITIES_OFST 16 + +/* MC_CMD_GET_CAPABILITIES_V2_IN msgrequest */ +#define MC_CMD_GET_CAPABILITIES_V2_IN_LEN 0 + +/* MC_CMD_GET_CAPABILITIES_V2_OUT msgresponse */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_LEN 72 +/* First word of flags. */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_FLAGS1_OFST 0 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_VPORT_RECONFIGURE_LBN 3 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_VPORT_RECONFIGURE_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_TX_STRIPING_LBN 4 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_TX_STRIPING_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_VADAPTOR_QUERY_LBN 5 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_VADAPTOR_QUERY_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_EVB_PORT_VLAN_RESTRICT_LBN 6 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_EVB_PORT_VLAN_RESTRICT_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_DRV_ATTACH_PREBOOT_LBN 7 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_DRV_ATTACH_PREBOOT_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RX_FORCE_EVENT_MERGING_LBN 8 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RX_FORCE_EVENT_MERGING_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_SET_MAC_ENHANCED_LBN 9 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_SET_MAC_ENHANCED_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_UNKNOWN_UCAST_DST_FILTER_ALWAYS_MULTI_RECIPIENT_LBN 10 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_UNKNOWN_UCAST_DST_FILTER_ALWAYS_MULTI_RECIPIENT_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_VADAPTOR_PERMIT_SET_MAC_WHEN_FILTERS_INSTALLED_LBN 11 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_VADAPTOR_PERMIT_SET_MAC_WHEN_FILTERS_INSTALLED_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_TX_MAC_SECURITY_FILTERING_LBN 12 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_TX_MAC_SECURITY_FILTERING_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_ADDITIONAL_RSS_MODES_LBN 13 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_ADDITIONAL_RSS_MODES_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_QBB_LBN 14 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_QBB_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RX_PACKED_STREAM_VAR_BUFFERS_LBN 15 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RX_PACKED_STREAM_VAR_BUFFERS_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RX_RSS_LIMITED_LBN 16 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RX_RSS_LIMITED_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RX_PACKED_STREAM_LBN 17 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RX_PACKED_STREAM_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RX_INCLUDE_FCS_LBN 18 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RX_INCLUDE_FCS_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_TX_VLAN_INSERTION_LBN 19 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_TX_VLAN_INSERTION_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RX_VLAN_STRIPPING_LBN 20 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RX_VLAN_STRIPPING_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_TX_TSO_LBN 21 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_TX_TSO_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RX_PREFIX_LEN_0_LBN 22 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RX_PREFIX_LEN_0_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RX_PREFIX_LEN_14_LBN 23 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RX_PREFIX_LEN_14_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RX_TIMESTAMP_LBN 24 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RX_TIMESTAMP_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RX_BATCHING_LBN 25 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RX_BATCHING_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_MCAST_FILTER_CHAINING_LBN 26 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_MCAST_FILTER_CHAINING_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_PM_AND_RXDP_COUNTERS_LBN 27 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_PM_AND_RXDP_COUNTERS_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RX_DISABLE_SCATTER_LBN 28 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RX_DISABLE_SCATTER_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_TX_MCAST_UDP_LOOPBACK_LBN 29 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_TX_MCAST_UDP_LOOPBACK_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_EVB_LBN 30 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_EVB_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_VXLAN_NVGRE_LBN 31 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_VXLAN_NVGRE_WIDTH 1 +/* RxDPCPU firmware id. */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RX_DPCPU_FW_ID_OFST 4 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RX_DPCPU_FW_ID_LEN 2 +/* enum: Standard RXDP firmware */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RXDP 0x0 +/* enum: Low latency RXDP firmware */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RXDP_LOW_LATENCY 0x1 +/* enum: Packed stream RXDP firmware */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RXDP_PACKED_STREAM 0x2 +/* enum: BIST RXDP firmware */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RXDP_BIST 0x10a +/* enum: RXDP Test firmware image 1 */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RXDP_TEST_FW_TO_MC_CUT_THROUGH 0x101 +/* enum: RXDP Test firmware image 2 */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RXDP_TEST_FW_TO_MC_STORE_FORWARD 0x102 +/* enum: RXDP Test firmware image 3 */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RXDP_TEST_FW_TO_MC_STORE_FORWARD_FIRST 0x103 +/* enum: RXDP Test firmware image 4 */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RXDP_TEST_EVERY_EVENT_BATCHABLE 0x104 +/* enum: RXDP Test firmware image 5 */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RXDP_TEST_BACKPRESSURE 0x105 +/* enum: RXDP Test firmware image 6 */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RXDP_TEST_FW_PACKET_EDITS 0x106 +/* enum: RXDP Test firmware image 7 */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RXDP_TEST_FW_RX_HDR_SPLIT 0x107 +/* enum: RXDP Test firmware image 8 */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RXDP_TEST_FW_DISABLE_DL 0x108 +/* enum: RXDP Test firmware image 9 */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RXDP_TEST_FW_DOORBELL_DELAY 0x10b +/* TxDPCPU firmware id. */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_TX_DPCPU_FW_ID_OFST 6 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_TX_DPCPU_FW_ID_LEN 2 +/* enum: Standard TXDP firmware */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_TXDP 0x0 +/* enum: Low latency TXDP firmware */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_TXDP_LOW_LATENCY 0x1 +/* enum: High packet rate TXDP firmware */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_TXDP_HIGH_PACKET_RATE 0x3 +/* enum: BIST TXDP firmware */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_TXDP_BIST 0x12d +/* enum: TXDP Test firmware image 1 */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_TXDP_TEST_FW_TSO_EDIT 0x101 +/* enum: TXDP Test firmware image 2 */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_TXDP_TEST_FW_PACKET_EDITS 0x102 +/* enum: TXDP CSR bus test firmware */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_TXDP_TEST_FW_CSR 0x103 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RXPD_FW_VERSION_OFST 8 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RXPD_FW_VERSION_LEN 2 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RXPD_FW_VERSION_REV_LBN 0 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RXPD_FW_VERSION_REV_WIDTH 12 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RXPD_FW_VERSION_TYPE_LBN 12 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RXPD_FW_VERSION_TYPE_WIDTH 4 +/* enum: reserved value - do not use (may indicate alternative interpretation + * of REV field in future) + */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RXPD_FW_TYPE_RESERVED 0x0 +/* enum: Trivial RX PD firmware for early Huntington development (Huntington + * development only) + */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RXPD_FW_TYPE_FIRST_PKT 0x1 +/* enum: RX PD firmware with approximately Siena-compatible behaviour + * (Huntington development only) + */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RXPD_FW_TYPE_SIENA_COMPAT 0x2 +/* enum: Full featured RX PD production firmware */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RXPD_FW_TYPE_FULL_FEATURED 0x3 +/* enum: (deprecated original name for the FULL_FEATURED variant) */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RXPD_FW_TYPE_VSWITCH 0x3 +/* enum: siena_compat variant RX PD firmware using PM rather than MAC + * (Huntington development only) + */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RXPD_FW_TYPE_SIENA_COMPAT_PM 0x4 +/* enum: Low latency RX PD production firmware */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RXPD_FW_TYPE_LOW_LATENCY 0x5 +/* enum: Packed stream RX PD production firmware */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RXPD_FW_TYPE_PACKED_STREAM 0x6 +/* enum: RX PD firmware handling layer 2 only for high packet rate performance + * tests (Medford development only) + */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RXPD_FW_TYPE_LAYER2_PERF 0x7 +/* enum: Rules engine RX PD production firmware */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RXPD_FW_TYPE_RULES_ENGINE 0x8 +/* enum: RX PD firmware for GUE parsing prototype (Medford development only) */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RXPD_FW_TYPE_TESTFW_GUE_PROTOTYPE 0xe +/* enum: RX PD firmware parsing but not filtering network overlay tunnel + * encapsulations (Medford development only) + */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RXPD_FW_TYPE_TESTFW_ENCAP_PARSING_ONLY 0xf +#define MC_CMD_GET_CAPABILITIES_V2_OUT_TXPD_FW_VERSION_OFST 10 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_TXPD_FW_VERSION_LEN 2 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_TXPD_FW_VERSION_REV_LBN 0 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_TXPD_FW_VERSION_REV_WIDTH 12 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_TXPD_FW_VERSION_TYPE_LBN 12 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_TXPD_FW_VERSION_TYPE_WIDTH 4 +/* enum: reserved value - do not use (may indicate alternative interpretation + * of REV field in future) + */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_TXPD_FW_TYPE_RESERVED 0x0 +/* enum: Trivial TX PD firmware for early Huntington development (Huntington + * development only) + */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_TXPD_FW_TYPE_FIRST_PKT 0x1 +/* enum: TX PD firmware with approximately Siena-compatible behaviour + * (Huntington development only) + */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_TXPD_FW_TYPE_SIENA_COMPAT 0x2 +/* enum: Full featured TX PD production firmware */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_TXPD_FW_TYPE_FULL_FEATURED 0x3 +/* enum: (deprecated original name for the FULL_FEATURED variant) */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_TXPD_FW_TYPE_VSWITCH 0x3 +/* enum: siena_compat variant TX PD firmware using PM rather than MAC + * (Huntington development only) + */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_TXPD_FW_TYPE_SIENA_COMPAT_PM 0x4 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_TXPD_FW_TYPE_LOW_LATENCY 0x5 /* enum */ +/* enum: TX PD firmware handling layer 2 only for high packet rate performance + * tests (Medford development only) + */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_TXPD_FW_TYPE_LAYER2_PERF 0x7 +/* enum: Rules engine TX PD production firmware */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_TXPD_FW_TYPE_RULES_ENGINE 0x8 +/* enum: RX PD firmware for GUE parsing prototype (Medford development only) */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_TXPD_FW_TYPE_TESTFW_GUE_PROTOTYPE 0xe +/* Hardware capabilities of NIC */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_HW_CAPABILITIES_OFST 12 +/* Licensed capabilities */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_LICENSE_CAPABILITIES_OFST 16 +/* Second word of flags. Not present on older firmware (check the length). */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_FLAGS2_OFST 20 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_TX_TSO_V2_LBN 0 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_TX_TSO_V2_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_TX_TSO_V2_ENCAP_LBN 1 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_TX_TSO_V2_ENCAP_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_EVQ_TIMER_CTRL_LBN 2 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_EVQ_TIMER_CTRL_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_EVENT_CUT_THROUGH_LBN 3 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_EVENT_CUT_THROUGH_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RX_CUT_THROUGH_LBN 4 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RX_CUT_THROUGH_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_TX_VFIFO_ULL_MODE_LBN 5 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_TX_VFIFO_ULL_MODE_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_MAC_STATS_40G_TX_SIZE_BINS_LBN 6 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_MAC_STATS_40G_TX_SIZE_BINS_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_INIT_EVQ_V2_LBN 7 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_INIT_EVQ_V2_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_TX_MAC_TIMESTAMPING_LBN 8 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_TX_MAC_TIMESTAMPING_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_TX_TIMESTAMP_LBN 9 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_TX_TIMESTAMP_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RX_SNIFF_LBN 10 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RX_SNIFF_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_TX_SNIFF_LBN 11 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_TX_SNIFF_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_NVRAM_UPDATE_REPORT_VERIFY_RESULT_LBN 12 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_NVRAM_UPDATE_REPORT_VERIFY_RESULT_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_MCDI_BACKGROUND_LBN 13 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_MCDI_BACKGROUND_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_MCDI_DB_RETURN_LBN 14 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_MCDI_DB_RETURN_WIDTH 1 +/* Number of FATSOv2 contexts per datapath supported by this NIC. Not present + * on older firmware (check the length). + */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_TX_TSO_V2_N_CONTEXTS_OFST 24 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_TX_TSO_V2_N_CONTEXTS_LEN 2 +/* One byte per PF containing the number of the external port assigned to this + * PF, indexed by PF number. Special values indicate that a PF is either not + * present or not assigned. + */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_PFS_TO_PORTS_ASSIGNMENT_OFST 26 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_PFS_TO_PORTS_ASSIGNMENT_LEN 1 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_PFS_TO_PORTS_ASSIGNMENT_NUM 16 +/* enum: The caller is not permitted to access information on this PF. */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_ACCESS_NOT_PERMITTED 0xff +/* enum: PF does not exist. */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_PF_NOT_PRESENT 0xfe +/* enum: PF does exist but is not assigned to any external port. */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_PF_NOT_ASSIGNED 0xfd +/* enum: This value indicates that PF is assigned, but it cannot be expressed + * in this field. It is intended for a possible future situation where a more + * complex scheme of PFs to ports mapping is being used. The future driver + * should look for a new field supporting the new scheme. The current/old + * driver should treat this value as PF_NOT_ASSIGNED. + */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_INCOMPATIBLE_ASSIGNMENT 0xfc +/* One byte per PF containing the number of its VFs, indexed by PF number. A + * special value indicates that a PF is not present. + */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_NUM_VFS_PER_PF_OFST 42 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_NUM_VFS_PER_PF_LEN 1 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_NUM_VFS_PER_PF_NUM 16 +/* enum: The caller is not permitted to access information on this PF. */ +/* MC_CMD_GET_CAPABILITIES_V2_OUT_ACCESS_NOT_PERMITTED 0xff */ +/* enum: PF does not exist. */ +/* MC_CMD_GET_CAPABILITIES_V2_OUT_PF_NOT_PRESENT 0xfe */ +/* Number of VIs available for each external port */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_NUM_VIS_PER_PORT_OFST 58 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_NUM_VIS_PER_PORT_LEN 2 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_NUM_VIS_PER_PORT_NUM 4 +/* Size of RX descriptor cache expressed as binary logarithm The actual size + * equals (2 ^ RX_DESC_CACHE_SIZE) + */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RX_DESC_CACHE_SIZE_OFST 66 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_RX_DESC_CACHE_SIZE_LEN 1 +/* Size of TX descriptor cache expressed as binary logarithm The actual size + * equals (2 ^ TX_DESC_CACHE_SIZE) + */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_TX_DESC_CACHE_SIZE_OFST 67 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_TX_DESC_CACHE_SIZE_LEN 1 +/* Total number of available PIO buffers */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_NUM_PIO_BUFFS_OFST 68 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_NUM_PIO_BUFFS_LEN 2 +/* Size of a single PIO buffer */ +#define MC_CMD_GET_CAPABILITIES_V2_OUT_SIZE_PIO_BUFF_OFST 70 +#define MC_CMD_GET_CAPABILITIES_V2_OUT_SIZE_PIO_BUFF_LEN 2 + +/* MC_CMD_GET_CAPABILITIES_V3_OUT msgresponse */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_LEN 76 +/* First word of flags. */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_FLAGS1_OFST 0 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_VPORT_RECONFIGURE_LBN 3 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_VPORT_RECONFIGURE_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_TX_STRIPING_LBN 4 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_TX_STRIPING_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_VADAPTOR_QUERY_LBN 5 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_VADAPTOR_QUERY_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_EVB_PORT_VLAN_RESTRICT_LBN 6 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_EVB_PORT_VLAN_RESTRICT_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_DRV_ATTACH_PREBOOT_LBN 7 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_DRV_ATTACH_PREBOOT_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RX_FORCE_EVENT_MERGING_LBN 8 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RX_FORCE_EVENT_MERGING_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_SET_MAC_ENHANCED_LBN 9 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_SET_MAC_ENHANCED_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_UNKNOWN_UCAST_DST_FILTER_ALWAYS_MULTI_RECIPIENT_LBN 10 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_UNKNOWN_UCAST_DST_FILTER_ALWAYS_MULTI_RECIPIENT_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_VADAPTOR_PERMIT_SET_MAC_WHEN_FILTERS_INSTALLED_LBN 11 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_VADAPTOR_PERMIT_SET_MAC_WHEN_FILTERS_INSTALLED_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_TX_MAC_SECURITY_FILTERING_LBN 12 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_TX_MAC_SECURITY_FILTERING_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_ADDITIONAL_RSS_MODES_LBN 13 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_ADDITIONAL_RSS_MODES_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_QBB_LBN 14 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_QBB_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RX_PACKED_STREAM_VAR_BUFFERS_LBN 15 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RX_PACKED_STREAM_VAR_BUFFERS_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RX_RSS_LIMITED_LBN 16 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RX_RSS_LIMITED_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RX_PACKED_STREAM_LBN 17 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RX_PACKED_STREAM_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RX_INCLUDE_FCS_LBN 18 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RX_INCLUDE_FCS_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_TX_VLAN_INSERTION_LBN 19 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_TX_VLAN_INSERTION_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RX_VLAN_STRIPPING_LBN 20 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RX_VLAN_STRIPPING_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_TX_TSO_LBN 21 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_TX_TSO_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RX_PREFIX_LEN_0_LBN 22 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RX_PREFIX_LEN_0_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RX_PREFIX_LEN_14_LBN 23 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RX_PREFIX_LEN_14_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RX_TIMESTAMP_LBN 24 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RX_TIMESTAMP_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RX_BATCHING_LBN 25 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RX_BATCHING_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_MCAST_FILTER_CHAINING_LBN 26 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_MCAST_FILTER_CHAINING_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_PM_AND_RXDP_COUNTERS_LBN 27 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_PM_AND_RXDP_COUNTERS_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RX_DISABLE_SCATTER_LBN 28 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RX_DISABLE_SCATTER_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_TX_MCAST_UDP_LOOPBACK_LBN 29 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_TX_MCAST_UDP_LOOPBACK_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_EVB_LBN 30 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_EVB_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_VXLAN_NVGRE_LBN 31 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_VXLAN_NVGRE_WIDTH 1 +/* RxDPCPU firmware id. */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RX_DPCPU_FW_ID_OFST 4 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RX_DPCPU_FW_ID_LEN 2 +/* enum: Standard RXDP firmware */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RXDP 0x0 +/* enum: Low latency RXDP firmware */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RXDP_LOW_LATENCY 0x1 +/* enum: Packed stream RXDP firmware */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RXDP_PACKED_STREAM 0x2 +/* enum: BIST RXDP firmware */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RXDP_BIST 0x10a +/* enum: RXDP Test firmware image 1 */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RXDP_TEST_FW_TO_MC_CUT_THROUGH 0x101 +/* enum: RXDP Test firmware image 2 */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RXDP_TEST_FW_TO_MC_STORE_FORWARD 0x102 +/* enum: RXDP Test firmware image 3 */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RXDP_TEST_FW_TO_MC_STORE_FORWARD_FIRST 0x103 +/* enum: RXDP Test firmware image 4 */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RXDP_TEST_EVERY_EVENT_BATCHABLE 0x104 +/* enum: RXDP Test firmware image 5 */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RXDP_TEST_BACKPRESSURE 0x105 +/* enum: RXDP Test firmware image 6 */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RXDP_TEST_FW_PACKET_EDITS 0x106 +/* enum: RXDP Test firmware image 7 */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RXDP_TEST_FW_RX_HDR_SPLIT 0x107 +/* enum: RXDP Test firmware image 8 */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RXDP_TEST_FW_DISABLE_DL 0x108 +/* enum: RXDP Test firmware image 9 */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RXDP_TEST_FW_DOORBELL_DELAY 0x10b +/* TxDPCPU firmware id. */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_TX_DPCPU_FW_ID_OFST 6 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_TX_DPCPU_FW_ID_LEN 2 +/* enum: Standard TXDP firmware */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_TXDP 0x0 +/* enum: Low latency TXDP firmware */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_TXDP_LOW_LATENCY 0x1 +/* enum: High packet rate TXDP firmware */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_TXDP_HIGH_PACKET_RATE 0x3 +/* enum: BIST TXDP firmware */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_TXDP_BIST 0x12d +/* enum: TXDP Test firmware image 1 */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_TXDP_TEST_FW_TSO_EDIT 0x101 +/* enum: TXDP Test firmware image 2 */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_TXDP_TEST_FW_PACKET_EDITS 0x102 +/* enum: TXDP CSR bus test firmware */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_TXDP_TEST_FW_CSR 0x103 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RXPD_FW_VERSION_OFST 8 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RXPD_FW_VERSION_LEN 2 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RXPD_FW_VERSION_REV_LBN 0 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RXPD_FW_VERSION_REV_WIDTH 12 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RXPD_FW_VERSION_TYPE_LBN 12 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RXPD_FW_VERSION_TYPE_WIDTH 4 +/* enum: reserved value - do not use (may indicate alternative interpretation + * of REV field in future) + */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RXPD_FW_TYPE_RESERVED 0x0 +/* enum: Trivial RX PD firmware for early Huntington development (Huntington + * development only) + */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RXPD_FW_TYPE_FIRST_PKT 0x1 +/* enum: RX PD firmware with approximately Siena-compatible behaviour + * (Huntington development only) + */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RXPD_FW_TYPE_SIENA_COMPAT 0x2 +/* enum: Full featured RX PD production firmware */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RXPD_FW_TYPE_FULL_FEATURED 0x3 +/* enum: (deprecated original name for the FULL_FEATURED variant) */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RXPD_FW_TYPE_VSWITCH 0x3 +/* enum: siena_compat variant RX PD firmware using PM rather than MAC + * (Huntington development only) + */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RXPD_FW_TYPE_SIENA_COMPAT_PM 0x4 +/* enum: Low latency RX PD production firmware */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RXPD_FW_TYPE_LOW_LATENCY 0x5 +/* enum: Packed stream RX PD production firmware */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RXPD_FW_TYPE_PACKED_STREAM 0x6 +/* enum: RX PD firmware handling layer 2 only for high packet rate performance + * tests (Medford development only) + */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RXPD_FW_TYPE_LAYER2_PERF 0x7 +/* enum: Rules engine RX PD production firmware */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RXPD_FW_TYPE_RULES_ENGINE 0x8 +/* enum: RX PD firmware for GUE parsing prototype (Medford development only) */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RXPD_FW_TYPE_TESTFW_GUE_PROTOTYPE 0xe +/* enum: RX PD firmware parsing but not filtering network overlay tunnel + * encapsulations (Medford development only) + */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RXPD_FW_TYPE_TESTFW_ENCAP_PARSING_ONLY 0xf +#define MC_CMD_GET_CAPABILITIES_V3_OUT_TXPD_FW_VERSION_OFST 10 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_TXPD_FW_VERSION_LEN 2 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_TXPD_FW_VERSION_REV_LBN 0 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_TXPD_FW_VERSION_REV_WIDTH 12 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_TXPD_FW_VERSION_TYPE_LBN 12 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_TXPD_FW_VERSION_TYPE_WIDTH 4 +/* enum: reserved value - do not use (may indicate alternative interpretation + * of REV field in future) + */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_TXPD_FW_TYPE_RESERVED 0x0 +/* enum: Trivial TX PD firmware for early Huntington development (Huntington + * development only) + */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_TXPD_FW_TYPE_FIRST_PKT 0x1 +/* enum: TX PD firmware with approximately Siena-compatible behaviour + * (Huntington development only) + */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_TXPD_FW_TYPE_SIENA_COMPAT 0x2 +/* enum: Full featured TX PD production firmware */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_TXPD_FW_TYPE_FULL_FEATURED 0x3 +/* enum: (deprecated original name for the FULL_FEATURED variant) */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_TXPD_FW_TYPE_VSWITCH 0x3 +/* enum: siena_compat variant TX PD firmware using PM rather than MAC + * (Huntington development only) + */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_TXPD_FW_TYPE_SIENA_COMPAT_PM 0x4 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_TXPD_FW_TYPE_LOW_LATENCY 0x5 /* enum */ +/* enum: TX PD firmware handling layer 2 only for high packet rate performance + * tests (Medford development only) + */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_TXPD_FW_TYPE_LAYER2_PERF 0x7 +/* enum: Rules engine TX PD production firmware */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_TXPD_FW_TYPE_RULES_ENGINE 0x8 +/* enum: RX PD firmware for GUE parsing prototype (Medford development only) */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_TXPD_FW_TYPE_TESTFW_GUE_PROTOTYPE 0xe +/* Hardware capabilities of NIC */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_HW_CAPABILITIES_OFST 12 +/* Licensed capabilities */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_LICENSE_CAPABILITIES_OFST 16 +/* Second word of flags. Not present on older firmware (check the length). */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_FLAGS2_OFST 20 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_TX_TSO_V2_LBN 0 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_TX_TSO_V2_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_TX_TSO_V2_ENCAP_LBN 1 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_TX_TSO_V2_ENCAP_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_EVQ_TIMER_CTRL_LBN 2 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_EVQ_TIMER_CTRL_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_EVENT_CUT_THROUGH_LBN 3 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_EVENT_CUT_THROUGH_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RX_CUT_THROUGH_LBN 4 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RX_CUT_THROUGH_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_TX_VFIFO_ULL_MODE_LBN 5 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_TX_VFIFO_ULL_MODE_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_MAC_STATS_40G_TX_SIZE_BINS_LBN 6 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_MAC_STATS_40G_TX_SIZE_BINS_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_INIT_EVQ_V2_LBN 7 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_INIT_EVQ_V2_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_TX_MAC_TIMESTAMPING_LBN 8 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_TX_MAC_TIMESTAMPING_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_TX_TIMESTAMP_LBN 9 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_TX_TIMESTAMP_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RX_SNIFF_LBN 10 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RX_SNIFF_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_TX_SNIFF_LBN 11 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_TX_SNIFF_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_NVRAM_UPDATE_REPORT_VERIFY_RESULT_LBN 12 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_NVRAM_UPDATE_REPORT_VERIFY_RESULT_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_MCDI_BACKGROUND_LBN 13 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_MCDI_BACKGROUND_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_MCDI_DB_RETURN_LBN 14 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_MCDI_DB_RETURN_WIDTH 1 +/* Number of FATSOv2 contexts per datapath supported by this NIC. Not present + * on older firmware (check the length). + */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_TX_TSO_V2_N_CONTEXTS_OFST 24 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_TX_TSO_V2_N_CONTEXTS_LEN 2 +/* One byte per PF containing the number of the external port assigned to this + * PF, indexed by PF number. Special values indicate that a PF is either not + * present or not assigned. + */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_PFS_TO_PORTS_ASSIGNMENT_OFST 26 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_PFS_TO_PORTS_ASSIGNMENT_LEN 1 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_PFS_TO_PORTS_ASSIGNMENT_NUM 16 +/* enum: The caller is not permitted to access information on this PF. */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_ACCESS_NOT_PERMITTED 0xff +/* enum: PF does not exist. */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_PF_NOT_PRESENT 0xfe +/* enum: PF does exist but is not assigned to any external port. */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_PF_NOT_ASSIGNED 0xfd +/* enum: This value indicates that PF is assigned, but it cannot be expressed + * in this field. It is intended for a possible future situation where a more + * complex scheme of PFs to ports mapping is being used. The future driver + * should look for a new field supporting the new scheme. The current/old + * driver should treat this value as PF_NOT_ASSIGNED. + */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_INCOMPATIBLE_ASSIGNMENT 0xfc +/* One byte per PF containing the number of its VFs, indexed by PF number. A + * special value indicates that a PF is not present. + */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_NUM_VFS_PER_PF_OFST 42 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_NUM_VFS_PER_PF_LEN 1 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_NUM_VFS_PER_PF_NUM 16 +/* enum: The caller is not permitted to access information on this PF. */ +/* MC_CMD_GET_CAPABILITIES_V3_OUT_ACCESS_NOT_PERMITTED 0xff */ +/* enum: PF does not exist. */ +/* MC_CMD_GET_CAPABILITIES_V3_OUT_PF_NOT_PRESENT 0xfe */ +/* Number of VIs available for each external port */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_NUM_VIS_PER_PORT_OFST 58 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_NUM_VIS_PER_PORT_LEN 2 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_NUM_VIS_PER_PORT_NUM 4 +/* Size of RX descriptor cache expressed as binary logarithm The actual size + * equals (2 ^ RX_DESC_CACHE_SIZE) + */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RX_DESC_CACHE_SIZE_OFST 66 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_RX_DESC_CACHE_SIZE_LEN 1 +/* Size of TX descriptor cache expressed as binary logarithm The actual size + * equals (2 ^ TX_DESC_CACHE_SIZE) + */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_TX_DESC_CACHE_SIZE_OFST 67 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_TX_DESC_CACHE_SIZE_LEN 1 +/* Total number of available PIO buffers */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_NUM_PIO_BUFFS_OFST 68 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_NUM_PIO_BUFFS_LEN 2 +/* Size of a single PIO buffer */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_SIZE_PIO_BUFF_OFST 70 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_SIZE_PIO_BUFF_LEN 2 +/* On chips later than Medford the amount of address space assigned to each VI + * is configurable. This is a global setting that the driver must query to + * discover the VI to address mapping. Cut-through PIO (CTPIO) is not available + * with 8k VI windows. + */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_VI_WINDOW_MODE_OFST 72 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_VI_WINDOW_MODE_LEN 1 +/* enum: Each VI occupies 8k as on Huntington and Medford. PIO is at offset 4k. + * CTPIO is not mapped. + */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_VI_WINDOW_MODE_8K 0x0 +/* enum: Each VI occupies 16k. PIO is at offset 4k. CTPIO is at offset 12k. */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_VI_WINDOW_MODE_16K 0x1 +/* enum: Each VI occupies 64k. PIO is at offset 4k. CTPIO is at offset 12k. */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_VI_WINDOW_MODE_64K 0x2 +/* Number of vFIFOs per adapter that can be used for VFIFO Stuffing + * (SF-115995-SW) in the present configuration of firmware and port mode. + */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_VFIFO_STUFFING_NUM_VFIFOS_OFST 73 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_VFIFO_STUFFING_NUM_VFIFOS_LEN 1 +/* Number of buffers per adapter that can be used for VFIFO Stuffing + * (SF-115995-SW) in the present configuration of firmware and port mode. + */ +#define MC_CMD_GET_CAPABILITIES_V3_OUT_VFIFO_STUFFING_NUM_CP_BUFFERS_OFST 74 +#define MC_CMD_GET_CAPABILITIES_V3_OUT_VFIFO_STUFFING_NUM_CP_BUFFERS_LEN 2 + + +/***********************************/ +/* MC_CMD_V2_EXTN + * Encapsulation for a v2 extended command + */ +#define MC_CMD_V2_EXTN 0x7f + +/* MC_CMD_V2_EXTN_IN msgrequest */ +#define MC_CMD_V2_EXTN_IN_LEN 4 +/* the extended command number */ +#define MC_CMD_V2_EXTN_IN_EXTENDED_CMD_LBN 0 +#define MC_CMD_V2_EXTN_IN_EXTENDED_CMD_WIDTH 15 +#define MC_CMD_V2_EXTN_IN_UNUSED_LBN 15 +#define MC_CMD_V2_EXTN_IN_UNUSED_WIDTH 1 +/* the actual length of the encapsulated command (which is not in the v1 + * header) + */ +#define MC_CMD_V2_EXTN_IN_ACTUAL_LEN_LBN 16 +#define MC_CMD_V2_EXTN_IN_ACTUAL_LEN_WIDTH 10 +#define MC_CMD_V2_EXTN_IN_UNUSED2_LBN 26 +#define MC_CMD_V2_EXTN_IN_UNUSED2_WIDTH 6 + + +/***********************************/ +/* MC_CMD_TCM_BUCKET_ALLOC + * Allocate a pacer bucket (for qau rp or a snapper test) + */ +#define MC_CMD_TCM_BUCKET_ALLOC 0xb2 +#undef MC_CMD_0xb2_PRIVILEGE_CTG + +#define MC_CMD_0xb2_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_TCM_BUCKET_ALLOC_IN msgrequest */ +#define MC_CMD_TCM_BUCKET_ALLOC_IN_LEN 0 + +/* MC_CMD_TCM_BUCKET_ALLOC_OUT msgresponse */ +#define MC_CMD_TCM_BUCKET_ALLOC_OUT_LEN 4 +/* the bucket id */ +#define MC_CMD_TCM_BUCKET_ALLOC_OUT_BUCKET_OFST 0 + + +/***********************************/ +/* MC_CMD_TCM_BUCKET_FREE + * Free a pacer bucket + */ +#define MC_CMD_TCM_BUCKET_FREE 0xb3 +#undef MC_CMD_0xb3_PRIVILEGE_CTG + +#define MC_CMD_0xb3_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_TCM_BUCKET_FREE_IN msgrequest */ +#define MC_CMD_TCM_BUCKET_FREE_IN_LEN 4 +/* the bucket id */ +#define MC_CMD_TCM_BUCKET_FREE_IN_BUCKET_OFST 0 + +/* MC_CMD_TCM_BUCKET_FREE_OUT msgresponse */ +#define MC_CMD_TCM_BUCKET_FREE_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_TCM_BUCKET_INIT + * Initialise pacer bucket with a given rate + */ +#define MC_CMD_TCM_BUCKET_INIT 0xb4 +#undef MC_CMD_0xb4_PRIVILEGE_CTG + +#define MC_CMD_0xb4_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_TCM_BUCKET_INIT_IN msgrequest */ +#define MC_CMD_TCM_BUCKET_INIT_IN_LEN 8 +/* the bucket id */ +#define MC_CMD_TCM_BUCKET_INIT_IN_BUCKET_OFST 0 +/* the rate in mbps */ +#define MC_CMD_TCM_BUCKET_INIT_IN_RATE_OFST 4 + +/* MC_CMD_TCM_BUCKET_INIT_EXT_IN msgrequest */ +#define MC_CMD_TCM_BUCKET_INIT_EXT_IN_LEN 12 +/* the bucket id */ +#define MC_CMD_TCM_BUCKET_INIT_EXT_IN_BUCKET_OFST 0 +/* the rate in mbps */ +#define MC_CMD_TCM_BUCKET_INIT_EXT_IN_RATE_OFST 4 +/* the desired maximum fill level */ +#define MC_CMD_TCM_BUCKET_INIT_EXT_IN_MAX_FILL_OFST 8 + +/* MC_CMD_TCM_BUCKET_INIT_OUT msgresponse */ +#define MC_CMD_TCM_BUCKET_INIT_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_TCM_TXQ_INIT + * Initialise txq in pacer with given options or set options + */ +#define MC_CMD_TCM_TXQ_INIT 0xb5 +#undef MC_CMD_0xb5_PRIVILEGE_CTG + +#define MC_CMD_0xb5_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_TCM_TXQ_INIT_IN msgrequest */ +#define MC_CMD_TCM_TXQ_INIT_IN_LEN 28 +/* the txq id */ +#define MC_CMD_TCM_TXQ_INIT_IN_QID_OFST 0 +/* the static priority associated with the txq */ +#define MC_CMD_TCM_TXQ_INIT_IN_LABEL_OFST 4 +/* bitmask of the priority queues this txq is inserted into when inserted. */ +#define MC_CMD_TCM_TXQ_INIT_IN_PQ_FLAGS_OFST 8 +#define MC_CMD_TCM_TXQ_INIT_IN_PQ_FLAG_GUARANTEED_LBN 0 +#define MC_CMD_TCM_TXQ_INIT_IN_PQ_FLAG_GUARANTEED_WIDTH 1 +#define MC_CMD_TCM_TXQ_INIT_IN_PQ_FLAG_NORMAL_LBN 1 +#define MC_CMD_TCM_TXQ_INIT_IN_PQ_FLAG_NORMAL_WIDTH 1 +#define MC_CMD_TCM_TXQ_INIT_IN_PQ_FLAG_LOW_LBN 2 +#define MC_CMD_TCM_TXQ_INIT_IN_PQ_FLAG_LOW_WIDTH 1 +/* the reaction point (RP) bucket */ +#define MC_CMD_TCM_TXQ_INIT_IN_RP_BKT_OFST 12 +/* an already reserved bucket (typically set to bucket associated with outer + * vswitch) + */ +#define MC_CMD_TCM_TXQ_INIT_IN_MAX_BKT1_OFST 16 +/* an already reserved bucket (typically set to bucket associated with inner + * vswitch) + */ +#define MC_CMD_TCM_TXQ_INIT_IN_MAX_BKT2_OFST 20 +/* the min bucket (typically for ETS/minimum bandwidth) */ +#define MC_CMD_TCM_TXQ_INIT_IN_MIN_BKT_OFST 24 + +/* MC_CMD_TCM_TXQ_INIT_EXT_IN msgrequest */ +#define MC_CMD_TCM_TXQ_INIT_EXT_IN_LEN 32 +/* the txq id */ +#define MC_CMD_TCM_TXQ_INIT_EXT_IN_QID_OFST 0 +/* the static priority associated with the txq */ +#define MC_CMD_TCM_TXQ_INIT_EXT_IN_LABEL_NORMAL_OFST 4 +/* bitmask of the priority queues this txq is inserted into when inserted. */ +#define MC_CMD_TCM_TXQ_INIT_EXT_IN_PQ_FLAGS_OFST 8 +#define MC_CMD_TCM_TXQ_INIT_EXT_IN_PQ_FLAG_GUARANTEED_LBN 0 +#define MC_CMD_TCM_TXQ_INIT_EXT_IN_PQ_FLAG_GUARANTEED_WIDTH 1 +#define MC_CMD_TCM_TXQ_INIT_EXT_IN_PQ_FLAG_NORMAL_LBN 1 +#define MC_CMD_TCM_TXQ_INIT_EXT_IN_PQ_FLAG_NORMAL_WIDTH 1 +#define MC_CMD_TCM_TXQ_INIT_EXT_IN_PQ_FLAG_LOW_LBN 2 +#define MC_CMD_TCM_TXQ_INIT_EXT_IN_PQ_FLAG_LOW_WIDTH 1 +/* the reaction point (RP) bucket */ +#define MC_CMD_TCM_TXQ_INIT_EXT_IN_RP_BKT_OFST 12 +/* an already reserved bucket (typically set to bucket associated with outer + * vswitch) + */ +#define MC_CMD_TCM_TXQ_INIT_EXT_IN_MAX_BKT1_OFST 16 +/* an already reserved bucket (typically set to bucket associated with inner + * vswitch) + */ +#define MC_CMD_TCM_TXQ_INIT_EXT_IN_MAX_BKT2_OFST 20 +/* the min bucket (typically for ETS/minimum bandwidth) */ +#define MC_CMD_TCM_TXQ_INIT_EXT_IN_MIN_BKT_OFST 24 +/* the static priority associated with the txq */ +#define MC_CMD_TCM_TXQ_INIT_EXT_IN_LABEL_GUARANTEED_OFST 28 + +/* MC_CMD_TCM_TXQ_INIT_OUT msgresponse */ +#define MC_CMD_TCM_TXQ_INIT_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_LINK_PIOBUF + * Link a push I/O buffer to a TxQ + */ +#define MC_CMD_LINK_PIOBUF 0x92 +#undef MC_CMD_0x92_PRIVILEGE_CTG + +#define MC_CMD_0x92_PRIVILEGE_CTG SRIOV_CTG_ONLOAD + +/* MC_CMD_LINK_PIOBUF_IN msgrequest */ +#define MC_CMD_LINK_PIOBUF_IN_LEN 8 +/* Handle for allocated push I/O buffer. */ +#define MC_CMD_LINK_PIOBUF_IN_PIOBUF_HANDLE_OFST 0 +/* Function Local Instance (VI) number. */ +#define MC_CMD_LINK_PIOBUF_IN_TXQ_INSTANCE_OFST 4 + +/* MC_CMD_LINK_PIOBUF_OUT msgresponse */ +#define MC_CMD_LINK_PIOBUF_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_UNLINK_PIOBUF + * Unlink a push I/O buffer from a TxQ + */ +#define MC_CMD_UNLINK_PIOBUF 0x93 +#undef MC_CMD_0x93_PRIVILEGE_CTG + +#define MC_CMD_0x93_PRIVILEGE_CTG SRIOV_CTG_ONLOAD + +/* MC_CMD_UNLINK_PIOBUF_IN msgrequest */ +#define MC_CMD_UNLINK_PIOBUF_IN_LEN 4 +/* Function Local Instance (VI) number. */ +#define MC_CMD_UNLINK_PIOBUF_IN_TXQ_INSTANCE_OFST 0 + +/* MC_CMD_UNLINK_PIOBUF_OUT msgresponse */ +#define MC_CMD_UNLINK_PIOBUF_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_VSWITCH_ALLOC + * allocate and initialise a v-switch. + */ +#define MC_CMD_VSWITCH_ALLOC 0x94 +#undef MC_CMD_0x94_PRIVILEGE_CTG + +#define MC_CMD_0x94_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_VSWITCH_ALLOC_IN msgrequest */ +#define MC_CMD_VSWITCH_ALLOC_IN_LEN 16 +/* The port to connect to the v-switch's upstream port. */ +#define MC_CMD_VSWITCH_ALLOC_IN_UPSTREAM_PORT_ID_OFST 0 +/* The type of v-switch to create. */ +#define MC_CMD_VSWITCH_ALLOC_IN_TYPE_OFST 4 +/* enum: VLAN */ +#define MC_CMD_VSWITCH_ALLOC_IN_VSWITCH_TYPE_VLAN 0x1 +/* enum: VEB */ +#define MC_CMD_VSWITCH_ALLOC_IN_VSWITCH_TYPE_VEB 0x2 +/* enum: VEPA (obsolete) */ +#define MC_CMD_VSWITCH_ALLOC_IN_VSWITCH_TYPE_VEPA 0x3 +/* enum: MUX */ +#define MC_CMD_VSWITCH_ALLOC_IN_VSWITCH_TYPE_MUX 0x4 +/* enum: Snapper specific; semantics TBD */ +#define MC_CMD_VSWITCH_ALLOC_IN_VSWITCH_TYPE_TEST 0x5 +/* Flags controlling v-port creation */ +#define MC_CMD_VSWITCH_ALLOC_IN_FLAGS_OFST 8 +#define MC_CMD_VSWITCH_ALLOC_IN_FLAG_AUTO_PORT_LBN 0 +#define MC_CMD_VSWITCH_ALLOC_IN_FLAG_AUTO_PORT_WIDTH 1 +/* The number of VLAN tags to allow for attached v-ports. For VLAN aggregators, + * this must be one or greated, and the attached v-ports must have exactly this + * number of tags. For other v-switch types, this must be zero of greater, and + * is an upper limit on the number of VLAN tags for attached v-ports. An error + * will be returned if existing configuration means we can't support attached + * v-ports with this number of tags. + */ +#define MC_CMD_VSWITCH_ALLOC_IN_NUM_VLAN_TAGS_OFST 12 + +/* MC_CMD_VSWITCH_ALLOC_OUT msgresponse */ +#define MC_CMD_VSWITCH_ALLOC_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_VSWITCH_FREE + * de-allocate a v-switch. + */ +#define MC_CMD_VSWITCH_FREE 0x95 +#undef MC_CMD_0x95_PRIVILEGE_CTG + +#define MC_CMD_0x95_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_VSWITCH_FREE_IN msgrequest */ +#define MC_CMD_VSWITCH_FREE_IN_LEN 4 +/* The port to which the v-switch is connected. */ +#define MC_CMD_VSWITCH_FREE_IN_UPSTREAM_PORT_ID_OFST 0 + +/* MC_CMD_VSWITCH_FREE_OUT msgresponse */ +#define MC_CMD_VSWITCH_FREE_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_VSWITCH_QUERY + * read some config of v-switch. For now this command is an empty placeholder. + * It may be used to check if a v-switch is connected to a given EVB port (if + * not, then the command returns ENOENT). + */ +#define MC_CMD_VSWITCH_QUERY 0x63 +#undef MC_CMD_0x63_PRIVILEGE_CTG + +#define MC_CMD_0x63_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_VSWITCH_QUERY_IN msgrequest */ +#define MC_CMD_VSWITCH_QUERY_IN_LEN 4 +/* The port to which the v-switch is connected. */ +#define MC_CMD_VSWITCH_QUERY_IN_UPSTREAM_PORT_ID_OFST 0 + +/* MC_CMD_VSWITCH_QUERY_OUT msgresponse */ +#define MC_CMD_VSWITCH_QUERY_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_VPORT_ALLOC + * allocate a v-port. + */ +#define MC_CMD_VPORT_ALLOC 0x96 +#undef MC_CMD_0x96_PRIVILEGE_CTG + +#define MC_CMD_0x96_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_VPORT_ALLOC_IN msgrequest */ +#define MC_CMD_VPORT_ALLOC_IN_LEN 20 +/* The port to which the v-switch is connected. */ +#define MC_CMD_VPORT_ALLOC_IN_UPSTREAM_PORT_ID_OFST 0 +/* The type of the new v-port. */ +#define MC_CMD_VPORT_ALLOC_IN_TYPE_OFST 4 +/* enum: VLAN (obsolete) */ +#define MC_CMD_VPORT_ALLOC_IN_VPORT_TYPE_VLAN 0x1 +/* enum: VEB (obsolete) */ +#define MC_CMD_VPORT_ALLOC_IN_VPORT_TYPE_VEB 0x2 +/* enum: VEPA (obsolete) */ +#define MC_CMD_VPORT_ALLOC_IN_VPORT_TYPE_VEPA 0x3 +/* enum: A normal v-port receives packets which match a specified MAC and/or + * VLAN. + */ +#define MC_CMD_VPORT_ALLOC_IN_VPORT_TYPE_NORMAL 0x4 +/* enum: An expansion v-port packets traffic which don't match any other + * v-port. + */ +#define MC_CMD_VPORT_ALLOC_IN_VPORT_TYPE_EXPANSION 0x5 +/* enum: An test v-port receives packets which match any filters installed by + * its downstream components. + */ +#define MC_CMD_VPORT_ALLOC_IN_VPORT_TYPE_TEST 0x6 +/* Flags controlling v-port creation */ +#define MC_CMD_VPORT_ALLOC_IN_FLAGS_OFST 8 +#define MC_CMD_VPORT_ALLOC_IN_FLAG_AUTO_PORT_LBN 0 +#define MC_CMD_VPORT_ALLOC_IN_FLAG_AUTO_PORT_WIDTH 1 +#define MC_CMD_VPORT_ALLOC_IN_FLAG_VLAN_RESTRICT_LBN 1 +#define MC_CMD_VPORT_ALLOC_IN_FLAG_VLAN_RESTRICT_WIDTH 1 +/* The number of VLAN tags to insert/remove. An error will be returned if + * incompatible with the number of VLAN tags specified for the upstream + * v-switch. + */ +#define MC_CMD_VPORT_ALLOC_IN_NUM_VLAN_TAGS_OFST 12 +/* The actual VLAN tags to insert/remove */ +#define MC_CMD_VPORT_ALLOC_IN_VLAN_TAGS_OFST 16 +#define MC_CMD_VPORT_ALLOC_IN_VLAN_TAG_0_LBN 0 +#define MC_CMD_VPORT_ALLOC_IN_VLAN_TAG_0_WIDTH 16 +#define MC_CMD_VPORT_ALLOC_IN_VLAN_TAG_1_LBN 16 +#define MC_CMD_VPORT_ALLOC_IN_VLAN_TAG_1_WIDTH 16 + +/* MC_CMD_VPORT_ALLOC_OUT msgresponse */ +#define MC_CMD_VPORT_ALLOC_OUT_LEN 4 +/* The handle of the new v-port */ +#define MC_CMD_VPORT_ALLOC_OUT_VPORT_ID_OFST 0 + + +/***********************************/ +/* MC_CMD_VPORT_FREE + * de-allocate a v-port. + */ +#define MC_CMD_VPORT_FREE 0x97 +#undef MC_CMD_0x97_PRIVILEGE_CTG + +#define MC_CMD_0x97_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_VPORT_FREE_IN msgrequest */ +#define MC_CMD_VPORT_FREE_IN_LEN 4 +/* The handle of the v-port */ +#define MC_CMD_VPORT_FREE_IN_VPORT_ID_OFST 0 + +/* MC_CMD_VPORT_FREE_OUT msgresponse */ +#define MC_CMD_VPORT_FREE_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_VADAPTOR_ALLOC + * allocate a v-adaptor. + */ +#define MC_CMD_VADAPTOR_ALLOC 0x98 +#undef MC_CMD_0x98_PRIVILEGE_CTG + +#define MC_CMD_0x98_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_VADAPTOR_ALLOC_IN msgrequest */ +#define MC_CMD_VADAPTOR_ALLOC_IN_LEN 30 +/* The port to connect to the v-adaptor's port. */ +#define MC_CMD_VADAPTOR_ALLOC_IN_UPSTREAM_PORT_ID_OFST 0 +/* Flags controlling v-adaptor creation */ +#define MC_CMD_VADAPTOR_ALLOC_IN_FLAGS_OFST 8 +#define MC_CMD_VADAPTOR_ALLOC_IN_FLAG_AUTO_VADAPTOR_LBN 0 +#define MC_CMD_VADAPTOR_ALLOC_IN_FLAG_AUTO_VADAPTOR_WIDTH 1 +#define MC_CMD_VADAPTOR_ALLOC_IN_FLAG_PERMIT_SET_MAC_WHEN_FILTERS_INSTALLED_LBN 1 +#define MC_CMD_VADAPTOR_ALLOC_IN_FLAG_PERMIT_SET_MAC_WHEN_FILTERS_INSTALLED_WIDTH 1 +/* The number of VLAN tags to strip on receive */ +#define MC_CMD_VADAPTOR_ALLOC_IN_NUM_VLANS_OFST 12 +/* The number of VLAN tags to transparently insert/remove. */ +#define MC_CMD_VADAPTOR_ALLOC_IN_NUM_VLAN_TAGS_OFST 16 +/* The actual VLAN tags to insert/remove */ +#define MC_CMD_VADAPTOR_ALLOC_IN_VLAN_TAGS_OFST 20 +#define MC_CMD_VADAPTOR_ALLOC_IN_VLAN_TAG_0_LBN 0 +#define MC_CMD_VADAPTOR_ALLOC_IN_VLAN_TAG_0_WIDTH 16 +#define MC_CMD_VADAPTOR_ALLOC_IN_VLAN_TAG_1_LBN 16 +#define MC_CMD_VADAPTOR_ALLOC_IN_VLAN_TAG_1_WIDTH 16 +/* The MAC address to assign to this v-adaptor */ +#define MC_CMD_VADAPTOR_ALLOC_IN_MACADDR_OFST 24 +#define MC_CMD_VADAPTOR_ALLOC_IN_MACADDR_LEN 6 +/* enum: Derive the MAC address from the upstream port */ +#define MC_CMD_VADAPTOR_ALLOC_IN_AUTO_MAC 0x0 + +/* MC_CMD_VADAPTOR_ALLOC_OUT msgresponse */ +#define MC_CMD_VADAPTOR_ALLOC_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_VADAPTOR_FREE + * de-allocate a v-adaptor. + */ +#define MC_CMD_VADAPTOR_FREE 0x99 +#undef MC_CMD_0x99_PRIVILEGE_CTG + +#define MC_CMD_0x99_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_VADAPTOR_FREE_IN msgrequest */ +#define MC_CMD_VADAPTOR_FREE_IN_LEN 4 +/* The port to which the v-adaptor is connected. */ +#define MC_CMD_VADAPTOR_FREE_IN_UPSTREAM_PORT_ID_OFST 0 + +/* MC_CMD_VADAPTOR_FREE_OUT msgresponse */ +#define MC_CMD_VADAPTOR_FREE_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_VADAPTOR_SET_MAC + * assign a new MAC address to a v-adaptor. + */ +#define MC_CMD_VADAPTOR_SET_MAC 0x5d +#undef MC_CMD_0x5d_PRIVILEGE_CTG + +#define MC_CMD_0x5d_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_VADAPTOR_SET_MAC_IN msgrequest */ +#define MC_CMD_VADAPTOR_SET_MAC_IN_LEN 10 +/* The port to which the v-adaptor is connected. */ +#define MC_CMD_VADAPTOR_SET_MAC_IN_UPSTREAM_PORT_ID_OFST 0 +/* The new MAC address to assign to this v-adaptor */ +#define MC_CMD_VADAPTOR_SET_MAC_IN_MACADDR_OFST 4 +#define MC_CMD_VADAPTOR_SET_MAC_IN_MACADDR_LEN 6 + +/* MC_CMD_VADAPTOR_SET_MAC_OUT msgresponse */ +#define MC_CMD_VADAPTOR_SET_MAC_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_VADAPTOR_GET_MAC + * read the MAC address assigned to a v-adaptor. + */ +#define MC_CMD_VADAPTOR_GET_MAC 0x5e +#undef MC_CMD_0x5e_PRIVILEGE_CTG + +#define MC_CMD_0x5e_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_VADAPTOR_GET_MAC_IN msgrequest */ +#define MC_CMD_VADAPTOR_GET_MAC_IN_LEN 4 +/* The port to which the v-adaptor is connected. */ +#define MC_CMD_VADAPTOR_GET_MAC_IN_UPSTREAM_PORT_ID_OFST 0 + +/* MC_CMD_VADAPTOR_GET_MAC_OUT msgresponse */ +#define MC_CMD_VADAPTOR_GET_MAC_OUT_LEN 6 +/* The MAC address assigned to this v-adaptor */ +#define MC_CMD_VADAPTOR_GET_MAC_OUT_MACADDR_OFST 0 +#define MC_CMD_VADAPTOR_GET_MAC_OUT_MACADDR_LEN 6 + + +/***********************************/ +/* MC_CMD_VADAPTOR_QUERY + * read some config of v-adaptor. + */ +#define MC_CMD_VADAPTOR_QUERY 0x61 +#undef MC_CMD_0x61_PRIVILEGE_CTG + +#define MC_CMD_0x61_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_VADAPTOR_QUERY_IN msgrequest */ +#define MC_CMD_VADAPTOR_QUERY_IN_LEN 4 +/* The port to which the v-adaptor is connected. */ +#define MC_CMD_VADAPTOR_QUERY_IN_UPSTREAM_PORT_ID_OFST 0 + +/* MC_CMD_VADAPTOR_QUERY_OUT msgresponse */ +#define MC_CMD_VADAPTOR_QUERY_OUT_LEN 12 +/* The EVB port flags as defined at MC_CMD_VPORT_ALLOC. */ +#define MC_CMD_VADAPTOR_QUERY_OUT_PORT_FLAGS_OFST 0 +/* The v-adaptor flags as defined at MC_CMD_VADAPTOR_ALLOC. */ +#define MC_CMD_VADAPTOR_QUERY_OUT_VADAPTOR_FLAGS_OFST 4 +/* The number of VLAN tags that may still be added */ +#define MC_CMD_VADAPTOR_QUERY_OUT_NUM_AVAILABLE_VLAN_TAGS_OFST 8 + + +/***********************************/ +/* MC_CMD_EVB_PORT_ASSIGN + * assign a port to a PCI function. + */ +#define MC_CMD_EVB_PORT_ASSIGN 0x9a +#undef MC_CMD_0x9a_PRIVILEGE_CTG + +#define MC_CMD_0x9a_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_EVB_PORT_ASSIGN_IN msgrequest */ +#define MC_CMD_EVB_PORT_ASSIGN_IN_LEN 8 +/* The port to assign. */ +#define MC_CMD_EVB_PORT_ASSIGN_IN_PORT_ID_OFST 0 +/* The target function to modify. */ +#define MC_CMD_EVB_PORT_ASSIGN_IN_FUNCTION_OFST 4 +#define MC_CMD_EVB_PORT_ASSIGN_IN_PF_LBN 0 +#define MC_CMD_EVB_PORT_ASSIGN_IN_PF_WIDTH 16 +#define MC_CMD_EVB_PORT_ASSIGN_IN_VF_LBN 16 +#define MC_CMD_EVB_PORT_ASSIGN_IN_VF_WIDTH 16 + +/* MC_CMD_EVB_PORT_ASSIGN_OUT msgresponse */ +#define MC_CMD_EVB_PORT_ASSIGN_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_RDWR_A64_REGIONS + * Assign the 64 bit region addresses. + */ +#define MC_CMD_RDWR_A64_REGIONS 0x9b +#undef MC_CMD_0x9b_PRIVILEGE_CTG + +#define MC_CMD_0x9b_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_RDWR_A64_REGIONS_IN msgrequest */ +#define MC_CMD_RDWR_A64_REGIONS_IN_LEN 17 +#define MC_CMD_RDWR_A64_REGIONS_IN_REGION0_OFST 0 +#define MC_CMD_RDWR_A64_REGIONS_IN_REGION1_OFST 4 +#define MC_CMD_RDWR_A64_REGIONS_IN_REGION2_OFST 8 +#define MC_CMD_RDWR_A64_REGIONS_IN_REGION3_OFST 12 +/* Write enable bits 0-3, set to write, clear to read. */ +#define MC_CMD_RDWR_A64_REGIONS_IN_WRITE_MASK_LBN 128 +#define MC_CMD_RDWR_A64_REGIONS_IN_WRITE_MASK_WIDTH 4 +#define MC_CMD_RDWR_A64_REGIONS_IN_WRITE_MASK_BYTE_OFST 16 +#define MC_CMD_RDWR_A64_REGIONS_IN_WRITE_MASK_BYTE_LEN 1 + +/* MC_CMD_RDWR_A64_REGIONS_OUT msgresponse: This data always included + * regardless of state of write bits in the request. + */ +#define MC_CMD_RDWR_A64_REGIONS_OUT_LEN 16 +#define MC_CMD_RDWR_A64_REGIONS_OUT_REGION0_OFST 0 +#define MC_CMD_RDWR_A64_REGIONS_OUT_REGION1_OFST 4 +#define MC_CMD_RDWR_A64_REGIONS_OUT_REGION2_OFST 8 +#define MC_CMD_RDWR_A64_REGIONS_OUT_REGION3_OFST 12 + + +/***********************************/ +/* MC_CMD_ONLOAD_STACK_ALLOC + * Allocate an Onload stack ID. + */ +#define MC_CMD_ONLOAD_STACK_ALLOC 0x9c +#undef MC_CMD_0x9c_PRIVILEGE_CTG + +#define MC_CMD_0x9c_PRIVILEGE_CTG SRIOV_CTG_ONLOAD + +/* MC_CMD_ONLOAD_STACK_ALLOC_IN msgrequest */ +#define MC_CMD_ONLOAD_STACK_ALLOC_IN_LEN 4 +/* The handle of the owning upstream port */ +#define MC_CMD_ONLOAD_STACK_ALLOC_IN_UPSTREAM_PORT_ID_OFST 0 + +/* MC_CMD_ONLOAD_STACK_ALLOC_OUT msgresponse */ +#define MC_CMD_ONLOAD_STACK_ALLOC_OUT_LEN 4 +/* The handle of the new Onload stack */ +#define MC_CMD_ONLOAD_STACK_ALLOC_OUT_ONLOAD_STACK_ID_OFST 0 + + +/***********************************/ +/* MC_CMD_ONLOAD_STACK_FREE + * Free an Onload stack ID. + */ +#define MC_CMD_ONLOAD_STACK_FREE 0x9d +#undef MC_CMD_0x9d_PRIVILEGE_CTG + +#define MC_CMD_0x9d_PRIVILEGE_CTG SRIOV_CTG_ONLOAD + +/* MC_CMD_ONLOAD_STACK_FREE_IN msgrequest */ +#define MC_CMD_ONLOAD_STACK_FREE_IN_LEN 4 +/* The handle of the Onload stack */ +#define MC_CMD_ONLOAD_STACK_FREE_IN_ONLOAD_STACK_ID_OFST 0 + +/* MC_CMD_ONLOAD_STACK_FREE_OUT msgresponse */ +#define MC_CMD_ONLOAD_STACK_FREE_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_RSS_CONTEXT_ALLOC + * Allocate an RSS context. + */ +#define MC_CMD_RSS_CONTEXT_ALLOC 0x9e +#undef MC_CMD_0x9e_PRIVILEGE_CTG + +#define MC_CMD_0x9e_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_RSS_CONTEXT_ALLOC_IN msgrequest */ +#define MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN 12 +/* The handle of the owning upstream port */ +#define MC_CMD_RSS_CONTEXT_ALLOC_IN_UPSTREAM_PORT_ID_OFST 0 +/* The type of context to allocate */ +#define MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_OFST 4 +/* enum: Allocate a context for exclusive use. The key and indirection table + * must be explicitly configured. + */ +#define MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_EXCLUSIVE 0x0 +/* enum: Allocate a context for shared use; this will spread across a range of + * queues, but the key and indirection table are pre-configured and may not be + * changed. For this mode, NUM_QUEUES must 2, 4, 8, 16, 32 or 64. + */ +#define MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_SHARED 0x1 +/* Number of queues spanned by this context, in the range 1-64; valid offsets + * in the indirection table will be in the range 0 to NUM_QUEUES-1. + */ +#define MC_CMD_RSS_CONTEXT_ALLOC_IN_NUM_QUEUES_OFST 8 + +/* MC_CMD_RSS_CONTEXT_ALLOC_OUT msgresponse */ +#define MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN 4 +/* The handle of the new RSS context. This should be considered opaque to the + * host, although a value of 0xFFFFFFFF is guaranteed never to be a valid + * handle. + */ +#define MC_CMD_RSS_CONTEXT_ALLOC_OUT_RSS_CONTEXT_ID_OFST 0 +/* enum: guaranteed invalid RSS context handle value */ +#define MC_CMD_RSS_CONTEXT_ALLOC_OUT_RSS_CONTEXT_ID_INVALID 0xffffffff + + +/***********************************/ +/* MC_CMD_RSS_CONTEXT_FREE + * Free an RSS context. + */ +#define MC_CMD_RSS_CONTEXT_FREE 0x9f +#undef MC_CMD_0x9f_PRIVILEGE_CTG + +#define MC_CMD_0x9f_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_RSS_CONTEXT_FREE_IN msgrequest */ +#define MC_CMD_RSS_CONTEXT_FREE_IN_LEN 4 +/* The handle of the RSS context */ +#define MC_CMD_RSS_CONTEXT_FREE_IN_RSS_CONTEXT_ID_OFST 0 + +/* MC_CMD_RSS_CONTEXT_FREE_OUT msgresponse */ +#define MC_CMD_RSS_CONTEXT_FREE_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_RSS_CONTEXT_SET_KEY + * Set the Toeplitz hash key for an RSS context. + */ +#define MC_CMD_RSS_CONTEXT_SET_KEY 0xa0 +#undef MC_CMD_0xa0_PRIVILEGE_CTG + +#define MC_CMD_0xa0_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_RSS_CONTEXT_SET_KEY_IN msgrequest */ +#define MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN 44 +/* The handle of the RSS context */ +#define MC_CMD_RSS_CONTEXT_SET_KEY_IN_RSS_CONTEXT_ID_OFST 0 +/* The 40-byte Toeplitz hash key (TBD endianness issues?) */ +#define MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_OFST 4 +#define MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN 40 + +/* MC_CMD_RSS_CONTEXT_SET_KEY_OUT msgresponse */ +#define MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_RSS_CONTEXT_GET_KEY + * Get the Toeplitz hash key for an RSS context. + */ +#define MC_CMD_RSS_CONTEXT_GET_KEY 0xa1 +#undef MC_CMD_0xa1_PRIVILEGE_CTG + +#define MC_CMD_0xa1_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_RSS_CONTEXT_GET_KEY_IN msgrequest */ +#define MC_CMD_RSS_CONTEXT_GET_KEY_IN_LEN 4 +/* The handle of the RSS context */ +#define MC_CMD_RSS_CONTEXT_GET_KEY_IN_RSS_CONTEXT_ID_OFST 0 + +/* MC_CMD_RSS_CONTEXT_GET_KEY_OUT msgresponse */ +#define MC_CMD_RSS_CONTEXT_GET_KEY_OUT_LEN 44 +/* The 40-byte Toeplitz hash key (TBD endianness issues?) */ +#define MC_CMD_RSS_CONTEXT_GET_KEY_OUT_TOEPLITZ_KEY_OFST 4 +#define MC_CMD_RSS_CONTEXT_GET_KEY_OUT_TOEPLITZ_KEY_LEN 40 + + +/***********************************/ +/* MC_CMD_RSS_CONTEXT_SET_TABLE + * Set the indirection table for an RSS context. + */ +#define MC_CMD_RSS_CONTEXT_SET_TABLE 0xa2 +#undef MC_CMD_0xa2_PRIVILEGE_CTG + +#define MC_CMD_0xa2_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_RSS_CONTEXT_SET_TABLE_IN msgrequest */ +#define MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN 132 +/* The handle of the RSS context */ +#define MC_CMD_RSS_CONTEXT_SET_TABLE_IN_RSS_CONTEXT_ID_OFST 0 +/* The 128-byte indirection table (1 byte per entry) */ +#define MC_CMD_RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE_OFST 4 +#define MC_CMD_RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE_LEN 128 + +/* MC_CMD_RSS_CONTEXT_SET_TABLE_OUT msgresponse */ +#define MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_RSS_CONTEXT_GET_TABLE + * Get the indirection table for an RSS context. + */ +#define MC_CMD_RSS_CONTEXT_GET_TABLE 0xa3 +#undef MC_CMD_0xa3_PRIVILEGE_CTG + +#define MC_CMD_0xa3_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_RSS_CONTEXT_GET_TABLE_IN msgrequest */ +#define MC_CMD_RSS_CONTEXT_GET_TABLE_IN_LEN 4 +/* The handle of the RSS context */ +#define MC_CMD_RSS_CONTEXT_GET_TABLE_IN_RSS_CONTEXT_ID_OFST 0 + +/* MC_CMD_RSS_CONTEXT_GET_TABLE_OUT msgresponse */ +#define MC_CMD_RSS_CONTEXT_GET_TABLE_OUT_LEN 132 +/* The 128-byte indirection table (1 byte per entry) */ +#define MC_CMD_RSS_CONTEXT_GET_TABLE_OUT_INDIRECTION_TABLE_OFST 4 +#define MC_CMD_RSS_CONTEXT_GET_TABLE_OUT_INDIRECTION_TABLE_LEN 128 + + +/***********************************/ +/* MC_CMD_RSS_CONTEXT_SET_FLAGS + * Set various control flags for an RSS context. + */ +#define MC_CMD_RSS_CONTEXT_SET_FLAGS 0xe1 +#undef MC_CMD_0xe1_PRIVILEGE_CTG + +#define MC_CMD_0xe1_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_RSS_CONTEXT_SET_FLAGS_IN msgrequest */ +#define MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN 8 +/* The handle of the RSS context */ +#define MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_RSS_CONTEXT_ID_OFST 0 +/* Hash control flags. The _EN bits are always supported, but new modes are + * available when ADDITIONAL_RSS_MODES is reported by MC_CMD_GET_CAPABILITIES: + * in this case, the MODE fields may be set to non-zero values, and will take + * effect regardless of the settings of the _EN flags. See the RSS_MODE + * structure for the meaning of the mode bits. Drivers must check the + * capability before trying to set any _MODE fields, as older firmware will + * reject any attempt to set the FLAGS field to a value > 0xff with EINVAL. In + * the case where all the _MODE flags are zero, the _EN flags take effect, + * providing backward compatibility for existing drivers. (Setting all _MODE + * *and* all _EN flags to zero is valid, to disable RSS spreading for that + * particular packet type.) + */ +#define MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_FLAGS_OFST 4 +#define MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV4_EN_LBN 0 +#define MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV4_EN_WIDTH 1 +#define MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV4_EN_LBN 1 +#define MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV4_EN_WIDTH 1 +#define MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV6_EN_LBN 2 +#define MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV6_EN_WIDTH 1 +#define MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV6_EN_LBN 3 +#define MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV6_EN_WIDTH 1 +#define MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_RESERVED_LBN 4 +#define MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_RESERVED_WIDTH 4 +#define MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV4_RSS_MODE_LBN 8 +#define MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV4_RSS_MODE_WIDTH 4 +#define MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_UDP_IPV4_RSS_MODE_LBN 12 +#define MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_UDP_IPV4_RSS_MODE_WIDTH 4 +#define MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV4_RSS_MODE_LBN 16 +#define MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV4_RSS_MODE_WIDTH 4 +#define MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV6_RSS_MODE_LBN 20 +#define MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV6_RSS_MODE_WIDTH 4 +#define MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_UDP_IPV6_RSS_MODE_LBN 24 +#define MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_UDP_IPV6_RSS_MODE_WIDTH 4 +#define MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV6_RSS_MODE_LBN 28 +#define MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV6_RSS_MODE_WIDTH 4 + +/* MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT msgresponse */ +#define MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_RSS_CONTEXT_GET_FLAGS + * Get various control flags for an RSS context. + */ +#define MC_CMD_RSS_CONTEXT_GET_FLAGS 0xe2 +#undef MC_CMD_0xe2_PRIVILEGE_CTG + +#define MC_CMD_0xe2_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_RSS_CONTEXT_GET_FLAGS_IN msgrequest */ +#define MC_CMD_RSS_CONTEXT_GET_FLAGS_IN_LEN 4 +/* The handle of the RSS context */ +#define MC_CMD_RSS_CONTEXT_GET_FLAGS_IN_RSS_CONTEXT_ID_OFST 0 + +/* MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT msgresponse */ +#define MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_LEN 8 +/* Hash control flags. If all _MODE bits are zero (which will always be true + * for older firmware which does not report the ADDITIONAL_RSS_MODES + * capability), the _EN bits report the state. If any _MODE bits are non-zero + * (which will only be true when the firmware reports ADDITIONAL_RSS_MODES) + * then the _EN bits should be disregarded, although the _MODE flags are + * guaranteed to be consistent with the _EN flags for a freshly-allocated RSS + * context and in the case where the _EN flags were used in the SET. This + * provides backward compatibility: old drivers will not be attempting to + * derive any meaning from the _MODE bits (and can never set them to any value + * not representable by the _EN bits); new drivers can always determine the + * mode by looking only at the _MODE bits; the value returned by a GET can + * always be used for a SET regardless of old/new driver vs. old/new firmware. + */ +#define MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_FLAGS_OFST 4 +#define MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TOEPLITZ_IPV4_EN_LBN 0 +#define MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TOEPLITZ_IPV4_EN_WIDTH 1 +#define MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TOEPLITZ_TCPV4_EN_LBN 1 +#define MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TOEPLITZ_TCPV4_EN_WIDTH 1 +#define MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TOEPLITZ_IPV6_EN_LBN 2 +#define MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TOEPLITZ_IPV6_EN_WIDTH 1 +#define MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TOEPLITZ_TCPV6_EN_LBN 3 +#define MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TOEPLITZ_TCPV6_EN_WIDTH 1 +#define MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_RESERVED_LBN 4 +#define MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_RESERVED_WIDTH 4 +#define MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TCP_IPV4_RSS_MODE_LBN 8 +#define MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TCP_IPV4_RSS_MODE_WIDTH 4 +#define MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_UDP_IPV4_RSS_MODE_LBN 12 +#define MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_UDP_IPV4_RSS_MODE_WIDTH 4 +#define MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_OTHER_IPV4_RSS_MODE_LBN 16 +#define MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_OTHER_IPV4_RSS_MODE_WIDTH 4 +#define MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TCP_IPV6_RSS_MODE_LBN 20 +#define MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TCP_IPV6_RSS_MODE_WIDTH 4 +#define MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_UDP_IPV6_RSS_MODE_LBN 24 +#define MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_UDP_IPV6_RSS_MODE_WIDTH 4 +#define MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_OTHER_IPV6_RSS_MODE_LBN 28 +#define MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_OTHER_IPV6_RSS_MODE_WIDTH 4 + + +/***********************************/ +/* MC_CMD_DOT1P_MAPPING_ALLOC + * Allocate a .1p mapping. + */ +#define MC_CMD_DOT1P_MAPPING_ALLOC 0xa4 +#undef MC_CMD_0xa4_PRIVILEGE_CTG + +#define MC_CMD_0xa4_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_DOT1P_MAPPING_ALLOC_IN msgrequest */ +#define MC_CMD_DOT1P_MAPPING_ALLOC_IN_LEN 8 +/* The handle of the owning upstream port */ +#define MC_CMD_DOT1P_MAPPING_ALLOC_IN_UPSTREAM_PORT_ID_OFST 0 +/* Number of queues spanned by this mapping, in the range 1-64; valid fixed + * offsets in the mapping table will be in the range 0 to NUM_QUEUES-1, and + * referenced RSS contexts must span no more than this number. + */ +#define MC_CMD_DOT1P_MAPPING_ALLOC_IN_NUM_QUEUES_OFST 4 + +/* MC_CMD_DOT1P_MAPPING_ALLOC_OUT msgresponse */ +#define MC_CMD_DOT1P_MAPPING_ALLOC_OUT_LEN 4 +/* The handle of the new .1p mapping. This should be considered opaque to the + * host, although a value of 0xFFFFFFFF is guaranteed never to be a valid + * handle. + */ +#define MC_CMD_DOT1P_MAPPING_ALLOC_OUT_DOT1P_MAPPING_ID_OFST 0 +/* enum: guaranteed invalid .1p mapping handle value */ +#define MC_CMD_DOT1P_MAPPING_ALLOC_OUT_DOT1P_MAPPING_ID_INVALID 0xffffffff + + +/***********************************/ +/* MC_CMD_DOT1P_MAPPING_FREE + * Free a .1p mapping. + */ +#define MC_CMD_DOT1P_MAPPING_FREE 0xa5 +#undef MC_CMD_0xa5_PRIVILEGE_CTG + +#define MC_CMD_0xa5_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_DOT1P_MAPPING_FREE_IN msgrequest */ +#define MC_CMD_DOT1P_MAPPING_FREE_IN_LEN 4 +/* The handle of the .1p mapping */ +#define MC_CMD_DOT1P_MAPPING_FREE_IN_DOT1P_MAPPING_ID_OFST 0 + +/* MC_CMD_DOT1P_MAPPING_FREE_OUT msgresponse */ +#define MC_CMD_DOT1P_MAPPING_FREE_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_DOT1P_MAPPING_SET_TABLE + * Set the mapping table for a .1p mapping. + */ +#define MC_CMD_DOT1P_MAPPING_SET_TABLE 0xa6 +#undef MC_CMD_0xa6_PRIVILEGE_CTG + +#define MC_CMD_0xa6_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_DOT1P_MAPPING_SET_TABLE_IN msgrequest */ +#define MC_CMD_DOT1P_MAPPING_SET_TABLE_IN_LEN 36 +/* The handle of the .1p mapping */ +#define MC_CMD_DOT1P_MAPPING_SET_TABLE_IN_DOT1P_MAPPING_ID_OFST 0 +/* Per-priority mappings (1 32-bit word per entry - an offset or RSS context + * handle) + */ +#define MC_CMD_DOT1P_MAPPING_SET_TABLE_IN_MAPPING_TABLE_OFST 4 +#define MC_CMD_DOT1P_MAPPING_SET_TABLE_IN_MAPPING_TABLE_LEN 32 + +/* MC_CMD_DOT1P_MAPPING_SET_TABLE_OUT msgresponse */ +#define MC_CMD_DOT1P_MAPPING_SET_TABLE_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_DOT1P_MAPPING_GET_TABLE + * Get the mapping table for a .1p mapping. + */ +#define MC_CMD_DOT1P_MAPPING_GET_TABLE 0xa7 +#undef MC_CMD_0xa7_PRIVILEGE_CTG + +#define MC_CMD_0xa7_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_DOT1P_MAPPING_GET_TABLE_IN msgrequest */ +#define MC_CMD_DOT1P_MAPPING_GET_TABLE_IN_LEN 4 +/* The handle of the .1p mapping */ +#define MC_CMD_DOT1P_MAPPING_GET_TABLE_IN_DOT1P_MAPPING_ID_OFST 0 + +/* MC_CMD_DOT1P_MAPPING_GET_TABLE_OUT msgresponse */ +#define MC_CMD_DOT1P_MAPPING_GET_TABLE_OUT_LEN 36 +/* Per-priority mappings (1 32-bit word per entry - an offset or RSS context + * handle) + */ +#define MC_CMD_DOT1P_MAPPING_GET_TABLE_OUT_MAPPING_TABLE_OFST 4 +#define MC_CMD_DOT1P_MAPPING_GET_TABLE_OUT_MAPPING_TABLE_LEN 32 + + +/***********************************/ +/* MC_CMD_GET_VECTOR_CFG + * Get Interrupt Vector config for this PF. + */ +#define MC_CMD_GET_VECTOR_CFG 0xbf +#undef MC_CMD_0xbf_PRIVILEGE_CTG + +#define MC_CMD_0xbf_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_GET_VECTOR_CFG_IN msgrequest */ +#define MC_CMD_GET_VECTOR_CFG_IN_LEN 0 + +/* MC_CMD_GET_VECTOR_CFG_OUT msgresponse */ +#define MC_CMD_GET_VECTOR_CFG_OUT_LEN 12 +/* Base absolute interrupt vector number. */ +#define MC_CMD_GET_VECTOR_CFG_OUT_VEC_BASE_OFST 0 +/* Number of interrupt vectors allocate to this PF. */ +#define MC_CMD_GET_VECTOR_CFG_OUT_VECS_PER_PF_OFST 4 +/* Number of interrupt vectors to allocate per VF. */ +#define MC_CMD_GET_VECTOR_CFG_OUT_VECS_PER_VF_OFST 8 + + +/***********************************/ +/* MC_CMD_SET_VECTOR_CFG + * Set Interrupt Vector config for this PF. + */ +#define MC_CMD_SET_VECTOR_CFG 0xc0 +#undef MC_CMD_0xc0_PRIVILEGE_CTG + +#define MC_CMD_0xc0_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_SET_VECTOR_CFG_IN msgrequest */ +#define MC_CMD_SET_VECTOR_CFG_IN_LEN 12 +/* Base absolute interrupt vector number, or MC_CMD_RESOURCE_INSTANCE_ANY to + * let the system find a suitable base. + */ +#define MC_CMD_SET_VECTOR_CFG_IN_VEC_BASE_OFST 0 +/* Number of interrupt vectors allocate to this PF. */ +#define MC_CMD_SET_VECTOR_CFG_IN_VECS_PER_PF_OFST 4 +/* Number of interrupt vectors to allocate per VF. */ +#define MC_CMD_SET_VECTOR_CFG_IN_VECS_PER_VF_OFST 8 + +/* MC_CMD_SET_VECTOR_CFG_OUT msgresponse */ +#define MC_CMD_SET_VECTOR_CFG_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_VPORT_ADD_MAC_ADDRESS + * Add a MAC address to a v-port + */ +#define MC_CMD_VPORT_ADD_MAC_ADDRESS 0xa8 +#undef MC_CMD_0xa8_PRIVILEGE_CTG + +#define MC_CMD_0xa8_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_VPORT_ADD_MAC_ADDRESS_IN msgrequest */ +#define MC_CMD_VPORT_ADD_MAC_ADDRESS_IN_LEN 10 +/* The handle of the v-port */ +#define MC_CMD_VPORT_ADD_MAC_ADDRESS_IN_VPORT_ID_OFST 0 +/* MAC address to add */ +#define MC_CMD_VPORT_ADD_MAC_ADDRESS_IN_MACADDR_OFST 4 +#define MC_CMD_VPORT_ADD_MAC_ADDRESS_IN_MACADDR_LEN 6 + +/* MC_CMD_VPORT_ADD_MAC_ADDRESS_OUT msgresponse */ +#define MC_CMD_VPORT_ADD_MAC_ADDRESS_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_VPORT_DEL_MAC_ADDRESS + * Delete a MAC address from a v-port + */ +#define MC_CMD_VPORT_DEL_MAC_ADDRESS 0xa9 +#undef MC_CMD_0xa9_PRIVILEGE_CTG + +#define MC_CMD_0xa9_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_VPORT_DEL_MAC_ADDRESS_IN msgrequest */ +#define MC_CMD_VPORT_DEL_MAC_ADDRESS_IN_LEN 10 +/* The handle of the v-port */ +#define MC_CMD_VPORT_DEL_MAC_ADDRESS_IN_VPORT_ID_OFST 0 +/* MAC address to add */ +#define MC_CMD_VPORT_DEL_MAC_ADDRESS_IN_MACADDR_OFST 4 +#define MC_CMD_VPORT_DEL_MAC_ADDRESS_IN_MACADDR_LEN 6 + +/* MC_CMD_VPORT_DEL_MAC_ADDRESS_OUT msgresponse */ +#define MC_CMD_VPORT_DEL_MAC_ADDRESS_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_VPORT_GET_MAC_ADDRESSES + * Delete a MAC address from a v-port + */ +#define MC_CMD_VPORT_GET_MAC_ADDRESSES 0xaa +#undef MC_CMD_0xaa_PRIVILEGE_CTG + +#define MC_CMD_0xaa_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_VPORT_GET_MAC_ADDRESSES_IN msgrequest */ +#define MC_CMD_VPORT_GET_MAC_ADDRESSES_IN_LEN 4 +/* The handle of the v-port */ +#define MC_CMD_VPORT_GET_MAC_ADDRESSES_IN_VPORT_ID_OFST 0 + +/* MC_CMD_VPORT_GET_MAC_ADDRESSES_OUT msgresponse */ +#define MC_CMD_VPORT_GET_MAC_ADDRESSES_OUT_LENMIN 4 +#define MC_CMD_VPORT_GET_MAC_ADDRESSES_OUT_LENMAX 250 +#define MC_CMD_VPORT_GET_MAC_ADDRESSES_OUT_LEN(num) (4+6*(num)) +/* The number of MAC addresses returned */ +#define MC_CMD_VPORT_GET_MAC_ADDRESSES_OUT_MACADDR_COUNT_OFST 0 +/* Array of MAC addresses */ +#define MC_CMD_VPORT_GET_MAC_ADDRESSES_OUT_MACADDR_OFST 4 +#define MC_CMD_VPORT_GET_MAC_ADDRESSES_OUT_MACADDR_LEN 6 +#define MC_CMD_VPORT_GET_MAC_ADDRESSES_OUT_MACADDR_MINNUM 0 +#define MC_CMD_VPORT_GET_MAC_ADDRESSES_OUT_MACADDR_MAXNUM 41 + + +/***********************************/ +/* MC_CMD_VPORT_RECONFIGURE + * Replace VLAN tags and/or MAC addresses of an existing v-port. If the v-port + * has already been passed to another function (v-port's user), then that + * function will be reset before applying the changes. + */ +#define MC_CMD_VPORT_RECONFIGURE 0xeb +#undef MC_CMD_0xeb_PRIVILEGE_CTG + +#define MC_CMD_0xeb_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_VPORT_RECONFIGURE_IN msgrequest */ +#define MC_CMD_VPORT_RECONFIGURE_IN_LEN 44 +/* The handle of the v-port */ +#define MC_CMD_VPORT_RECONFIGURE_IN_VPORT_ID_OFST 0 +/* Flags requesting what should be changed. */ +#define MC_CMD_VPORT_RECONFIGURE_IN_FLAGS_OFST 4 +#define MC_CMD_VPORT_RECONFIGURE_IN_REPLACE_VLAN_TAGS_LBN 0 +#define MC_CMD_VPORT_RECONFIGURE_IN_REPLACE_VLAN_TAGS_WIDTH 1 +#define MC_CMD_VPORT_RECONFIGURE_IN_REPLACE_MACADDRS_LBN 1 +#define MC_CMD_VPORT_RECONFIGURE_IN_REPLACE_MACADDRS_WIDTH 1 +/* The number of VLAN tags to insert/remove. An error will be returned if + * incompatible with the number of VLAN tags specified for the upstream + * v-switch. + */ +#define MC_CMD_VPORT_RECONFIGURE_IN_NUM_VLAN_TAGS_OFST 8 +/* The actual VLAN tags to insert/remove */ +#define MC_CMD_VPORT_RECONFIGURE_IN_VLAN_TAGS_OFST 12 +#define MC_CMD_VPORT_RECONFIGURE_IN_VLAN_TAG_0_LBN 0 +#define MC_CMD_VPORT_RECONFIGURE_IN_VLAN_TAG_0_WIDTH 16 +#define MC_CMD_VPORT_RECONFIGURE_IN_VLAN_TAG_1_LBN 16 +#define MC_CMD_VPORT_RECONFIGURE_IN_VLAN_TAG_1_WIDTH 16 +/* The number of MAC addresses to add */ +#define MC_CMD_VPORT_RECONFIGURE_IN_NUM_MACADDRS_OFST 16 +/* MAC addresses to add */ +#define MC_CMD_VPORT_RECONFIGURE_IN_MACADDRS_OFST 20 +#define MC_CMD_VPORT_RECONFIGURE_IN_MACADDRS_LEN 6 +#define MC_CMD_VPORT_RECONFIGURE_IN_MACADDRS_NUM 4 + +/* MC_CMD_VPORT_RECONFIGURE_OUT msgresponse */ +#define MC_CMD_VPORT_RECONFIGURE_OUT_LEN 4 +#define MC_CMD_VPORT_RECONFIGURE_OUT_FLAGS_OFST 0 +#define MC_CMD_VPORT_RECONFIGURE_OUT_RESET_DONE_LBN 0 +#define MC_CMD_VPORT_RECONFIGURE_OUT_RESET_DONE_WIDTH 1 + + +/***********************************/ +/* MC_CMD_EVB_PORT_QUERY + * read some config of v-port. + */ +#define MC_CMD_EVB_PORT_QUERY 0x62 +#undef MC_CMD_0x62_PRIVILEGE_CTG + +#define MC_CMD_0x62_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_EVB_PORT_QUERY_IN msgrequest */ +#define MC_CMD_EVB_PORT_QUERY_IN_LEN 4 +/* The handle of the v-port */ +#define MC_CMD_EVB_PORT_QUERY_IN_PORT_ID_OFST 0 + +/* MC_CMD_EVB_PORT_QUERY_OUT msgresponse */ +#define MC_CMD_EVB_PORT_QUERY_OUT_LEN 8 +/* The EVB port flags as defined at MC_CMD_VPORT_ALLOC. */ +#define MC_CMD_EVB_PORT_QUERY_OUT_PORT_FLAGS_OFST 0 +/* The number of VLAN tags that may be used on a v-adaptor connected to this + * EVB port. + */ +#define MC_CMD_EVB_PORT_QUERY_OUT_NUM_AVAILABLE_VLAN_TAGS_OFST 4 + + +/***********************************/ +/* MC_CMD_DUMP_BUFTBL_ENTRIES + * Dump buffer table entries, mainly for command client debug use. Dumps + * absolute entries, and does not use chunk handles. All entries must be in + * range, and used for q page mapping, Although the latter restriction may be + * lifted in future. + */ +#define MC_CMD_DUMP_BUFTBL_ENTRIES 0xab +#undef MC_CMD_0xab_PRIVILEGE_CTG + +#define MC_CMD_0xab_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_DUMP_BUFTBL_ENTRIES_IN msgrequest */ +#define MC_CMD_DUMP_BUFTBL_ENTRIES_IN_LEN 8 +/* Index of the first buffer table entry. */ +#define MC_CMD_DUMP_BUFTBL_ENTRIES_IN_FIRSTID_OFST 0 +/* Number of buffer table entries to dump. */ +#define MC_CMD_DUMP_BUFTBL_ENTRIES_IN_NUMENTRIES_OFST 4 + +/* MC_CMD_DUMP_BUFTBL_ENTRIES_OUT msgresponse */ +#define MC_CMD_DUMP_BUFTBL_ENTRIES_OUT_LENMIN 12 +#define MC_CMD_DUMP_BUFTBL_ENTRIES_OUT_LENMAX 252 +#define MC_CMD_DUMP_BUFTBL_ENTRIES_OUT_LEN(num) (0+12*(num)) +/* Raw buffer table entries, layed out as BUFTBL_ENTRY. */ +#define MC_CMD_DUMP_BUFTBL_ENTRIES_OUT_ENTRY_OFST 0 +#define MC_CMD_DUMP_BUFTBL_ENTRIES_OUT_ENTRY_LEN 12 +#define MC_CMD_DUMP_BUFTBL_ENTRIES_OUT_ENTRY_MINNUM 1 +#define MC_CMD_DUMP_BUFTBL_ENTRIES_OUT_ENTRY_MAXNUM 21 + + +/***********************************/ +/* MC_CMD_SET_RXDP_CONFIG + * Set global RXDP configuration settings + */ +#define MC_CMD_SET_RXDP_CONFIG 0xc1 +#undef MC_CMD_0xc1_PRIVILEGE_CTG + +#define MC_CMD_0xc1_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_SET_RXDP_CONFIG_IN msgrequest */ +#define MC_CMD_SET_RXDP_CONFIG_IN_LEN 4 +#define MC_CMD_SET_RXDP_CONFIG_IN_DATA_OFST 0 +#define MC_CMD_SET_RXDP_CONFIG_IN_PAD_HOST_DMA_LBN 0 +#define MC_CMD_SET_RXDP_CONFIG_IN_PAD_HOST_DMA_WIDTH 1 +#define MC_CMD_SET_RXDP_CONFIG_IN_PAD_HOST_LEN_LBN 1 +#define MC_CMD_SET_RXDP_CONFIG_IN_PAD_HOST_LEN_WIDTH 2 +/* enum: pad to 64 bytes */ +#define MC_CMD_SET_RXDP_CONFIG_IN_PAD_HOST_64 0x0 +/* enum: pad to 128 bytes (Medford only) */ +#define MC_CMD_SET_RXDP_CONFIG_IN_PAD_HOST_128 0x1 +/* enum: pad to 256 bytes (Medford only) */ +#define MC_CMD_SET_RXDP_CONFIG_IN_PAD_HOST_256 0x2 + +/* MC_CMD_SET_RXDP_CONFIG_OUT msgresponse */ +#define MC_CMD_SET_RXDP_CONFIG_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_GET_RXDP_CONFIG + * Get global RXDP configuration settings + */ +#define MC_CMD_GET_RXDP_CONFIG 0xc2 +#undef MC_CMD_0xc2_PRIVILEGE_CTG + +#define MC_CMD_0xc2_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_GET_RXDP_CONFIG_IN msgrequest */ +#define MC_CMD_GET_RXDP_CONFIG_IN_LEN 0 + +/* MC_CMD_GET_RXDP_CONFIG_OUT msgresponse */ +#define MC_CMD_GET_RXDP_CONFIG_OUT_LEN 4 +#define MC_CMD_GET_RXDP_CONFIG_OUT_DATA_OFST 0 +#define MC_CMD_GET_RXDP_CONFIG_OUT_PAD_HOST_DMA_LBN 0 +#define MC_CMD_GET_RXDP_CONFIG_OUT_PAD_HOST_DMA_WIDTH 1 +#define MC_CMD_GET_RXDP_CONFIG_OUT_PAD_HOST_LEN_LBN 1 +#define MC_CMD_GET_RXDP_CONFIG_OUT_PAD_HOST_LEN_WIDTH 2 +/* Enum values, see field(s): */ +/* MC_CMD_SET_RXDP_CONFIG/MC_CMD_SET_RXDP_CONFIG_IN/PAD_HOST_LEN */ + + +/***********************************/ +/* MC_CMD_GET_CLOCK + * Return the system and PDCPU clock frequencies. + */ +#define MC_CMD_GET_CLOCK 0xac +#undef MC_CMD_0xac_PRIVILEGE_CTG + +#define MC_CMD_0xac_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_GET_CLOCK_IN msgrequest */ +#define MC_CMD_GET_CLOCK_IN_LEN 0 + +/* MC_CMD_GET_CLOCK_OUT msgresponse */ +#define MC_CMD_GET_CLOCK_OUT_LEN 8 +/* System frequency, MHz */ +#define MC_CMD_GET_CLOCK_OUT_SYS_FREQ_OFST 0 +/* DPCPU frequency, MHz */ +#define MC_CMD_GET_CLOCK_OUT_DPCPU_FREQ_OFST 4 + + +/***********************************/ +/* MC_CMD_SET_CLOCK + * Control the system and DPCPU clock frequencies. Changes are lost reboot. + */ +#define MC_CMD_SET_CLOCK 0xad +#undef MC_CMD_0xad_PRIVILEGE_CTG + +#define MC_CMD_0xad_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_SET_CLOCK_IN msgrequest */ +#define MC_CMD_SET_CLOCK_IN_LEN 28 +/* Requested frequency in MHz for system clock domain */ +#define MC_CMD_SET_CLOCK_IN_SYS_FREQ_OFST 0 +/* enum: Leave the system clock domain frequency unchanged */ +#define MC_CMD_SET_CLOCK_IN_SYS_DOMAIN_DONT_CHANGE 0x0 +/* Requested frequency in MHz for inter-core clock domain */ +#define MC_CMD_SET_CLOCK_IN_ICORE_FREQ_OFST 4 +/* enum: Leave the inter-core clock domain frequency unchanged */ +#define MC_CMD_SET_CLOCK_IN_ICORE_DOMAIN_DONT_CHANGE 0x0 +/* Requested frequency in MHz for DPCPU clock domain */ +#define MC_CMD_SET_CLOCK_IN_DPCPU_FREQ_OFST 8 +/* enum: Leave the DPCPU clock domain frequency unchanged */ +#define MC_CMD_SET_CLOCK_IN_DPCPU_DOMAIN_DONT_CHANGE 0x0 +/* Requested frequency in MHz for PCS clock domain */ +#define MC_CMD_SET_CLOCK_IN_PCS_FREQ_OFST 12 +/* enum: Leave the PCS clock domain frequency unchanged */ +#define MC_CMD_SET_CLOCK_IN_PCS_DOMAIN_DONT_CHANGE 0x0 +/* Requested frequency in MHz for MC clock domain */ +#define MC_CMD_SET_CLOCK_IN_MC_FREQ_OFST 16 +/* enum: Leave the MC clock domain frequency unchanged */ +#define MC_CMD_SET_CLOCK_IN_MC_DOMAIN_DONT_CHANGE 0x0 +/* Requested frequency in MHz for rmon clock domain */ +#define MC_CMD_SET_CLOCK_IN_RMON_FREQ_OFST 20 +/* enum: Leave the rmon clock domain frequency unchanged */ +#define MC_CMD_SET_CLOCK_IN_RMON_DOMAIN_DONT_CHANGE 0x0 +/* Requested frequency in MHz for vswitch clock domain */ +#define MC_CMD_SET_CLOCK_IN_VSWITCH_FREQ_OFST 24 +/* enum: Leave the vswitch clock domain frequency unchanged */ +#define MC_CMD_SET_CLOCK_IN_VSWITCH_DOMAIN_DONT_CHANGE 0x0 + +/* MC_CMD_SET_CLOCK_OUT msgresponse */ +#define MC_CMD_SET_CLOCK_OUT_LEN 28 +/* Resulting system frequency in MHz */ +#define MC_CMD_SET_CLOCK_OUT_SYS_FREQ_OFST 0 +/* enum: The system clock domain doesn't exist */ +#define MC_CMD_SET_CLOCK_OUT_SYS_DOMAIN_UNSUPPORTED 0x0 +/* Resulting inter-core frequency in MHz */ +#define MC_CMD_SET_CLOCK_OUT_ICORE_FREQ_OFST 4 +/* enum: The inter-core clock domain doesn't exist / isn't used */ +#define MC_CMD_SET_CLOCK_OUT_ICORE_DOMAIN_UNSUPPORTED 0x0 +/* Resulting DPCPU frequency in MHz */ +#define MC_CMD_SET_CLOCK_OUT_DPCPU_FREQ_OFST 8 +/* enum: The dpcpu clock domain doesn't exist */ +#define MC_CMD_SET_CLOCK_OUT_DPCPU_DOMAIN_UNSUPPORTED 0x0 +/* Resulting PCS frequency in MHz */ +#define MC_CMD_SET_CLOCK_OUT_PCS_FREQ_OFST 12 +/* enum: The PCS clock domain doesn't exist / isn't controlled */ +#define MC_CMD_SET_CLOCK_OUT_PCS_DOMAIN_UNSUPPORTED 0x0 +/* Resulting MC frequency in MHz */ +#define MC_CMD_SET_CLOCK_OUT_MC_FREQ_OFST 16 +/* enum: The MC clock domain doesn't exist / isn't controlled */ +#define MC_CMD_SET_CLOCK_OUT_MC_DOMAIN_UNSUPPORTED 0x0 +/* Resulting rmon frequency in MHz */ +#define MC_CMD_SET_CLOCK_OUT_RMON_FREQ_OFST 20 +/* enum: The rmon clock domain doesn't exist / isn't controlled */ +#define MC_CMD_SET_CLOCK_OUT_RMON_DOMAIN_UNSUPPORTED 0x0 +/* Resulting vswitch frequency in MHz */ +#define MC_CMD_SET_CLOCK_OUT_VSWITCH_FREQ_OFST 24 +/* enum: The vswitch clock domain doesn't exist / isn't controlled */ +#define MC_CMD_SET_CLOCK_OUT_VSWITCH_DOMAIN_UNSUPPORTED 0x0 + + +/***********************************/ +/* MC_CMD_DPCPU_RPC + * Send an arbitrary DPCPU message. + */ +#define MC_CMD_DPCPU_RPC 0xae +#undef MC_CMD_0xae_PRIVILEGE_CTG + +#define MC_CMD_0xae_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_DPCPU_RPC_IN msgrequest */ +#define MC_CMD_DPCPU_RPC_IN_LEN 36 +#define MC_CMD_DPCPU_RPC_IN_CPU_OFST 0 +/* enum: RxDPCPU0 */ +#define MC_CMD_DPCPU_RPC_IN_DPCPU_RX0 0x0 +/* enum: TxDPCPU0 */ +#define MC_CMD_DPCPU_RPC_IN_DPCPU_TX0 0x1 +/* enum: TxDPCPU1 */ +#define MC_CMD_DPCPU_RPC_IN_DPCPU_TX1 0x2 +/* enum: RxDPCPU1 (Medford only) */ +#define MC_CMD_DPCPU_RPC_IN_DPCPU_RX1 0x3 +/* enum: RxDPCPU (will be for the calling function; for now, just an alias of + * DPCPU_RX0) + */ +#define MC_CMD_DPCPU_RPC_IN_DPCPU_RX 0x80 +/* enum: TxDPCPU (will be for the calling function; for now, just an alias of + * DPCPU_TX0) + */ +#define MC_CMD_DPCPU_RPC_IN_DPCPU_TX 0x81 +/* First 8 bits [39:32] of DATA are consumed by MC-DPCPU protocol and must be + * initialised to zero + */ +#define MC_CMD_DPCPU_RPC_IN_DATA_OFST 4 +#define MC_CMD_DPCPU_RPC_IN_DATA_LEN 32 +#define MC_CMD_DPCPU_RPC_IN_HDR_CMD_CMDNUM_LBN 8 +#define MC_CMD_DPCPU_RPC_IN_HDR_CMD_CMDNUM_WIDTH 8 +#define MC_CMD_DPCPU_RPC_IN_CMDNUM_TXDPCPU_READ 0x6 /* enum */ +#define MC_CMD_DPCPU_RPC_IN_CMDNUM_TXDPCPU_WRITE 0x7 /* enum */ +#define MC_CMD_DPCPU_RPC_IN_CMDNUM_TXDPCPU_SELF_TEST 0xc /* enum */ +#define MC_CMD_DPCPU_RPC_IN_CMDNUM_TXDPCPU_CSR_ACCESS 0xe /* enum */ +#define MC_CMD_DPCPU_RPC_IN_CMDNUM_RXDPCPU_READ 0x46 /* enum */ +#define MC_CMD_DPCPU_RPC_IN_CMDNUM_RXDPCPU_WRITE 0x47 /* enum */ +#define MC_CMD_DPCPU_RPC_IN_CMDNUM_RXDPCPU_SELF_TEST 0x4a /* enum */ +#define MC_CMD_DPCPU_RPC_IN_CMDNUM_RXDPCPU_CSR_ACCESS 0x4c /* enum */ +#define MC_CMD_DPCPU_RPC_IN_CMDNUM_RXDPCPU_SET_MC_REPLAY_CNTXT 0x4d /* enum */ +#define MC_CMD_DPCPU_RPC_IN_HDR_CMD_REQ_OBJID_LBN 16 +#define MC_CMD_DPCPU_RPC_IN_HDR_CMD_REQ_OBJID_WIDTH 16 +#define MC_CMD_DPCPU_RPC_IN_HDR_CMD_REQ_ADDR_LBN 16 +#define MC_CMD_DPCPU_RPC_IN_HDR_CMD_REQ_ADDR_WIDTH 16 +#define MC_CMD_DPCPU_RPC_IN_HDR_CMD_REQ_COUNT_LBN 48 +#define MC_CMD_DPCPU_RPC_IN_HDR_CMD_REQ_COUNT_WIDTH 16 +#define MC_CMD_DPCPU_RPC_IN_CSR_ACCESS_INFO_LBN 16 +#define MC_CMD_DPCPU_RPC_IN_CSR_ACCESS_INFO_WIDTH 240 +#define MC_CMD_DPCPU_RPC_IN_CSR_ACCESS_CMD_LBN 16 +#define MC_CMD_DPCPU_RPC_IN_CSR_ACCESS_CMD_WIDTH 16 +#define MC_CMD_DPCPU_RPC_IN_CSR_ACCESS_CMD_STOP_RETURN_RESULT 0x0 /* enum */ +#define MC_CMD_DPCPU_RPC_IN_CSR_ACCESS_CMD_START_READ 0x1 /* enum */ +#define MC_CMD_DPCPU_RPC_IN_CSR_ACCESS_CMD_START_WRITE 0x2 /* enum */ +#define MC_CMD_DPCPU_RPC_IN_CSR_ACCESS_CMD_START_WRITE_READ 0x3 /* enum */ +#define MC_CMD_DPCPU_RPC_IN_CSR_ACCESS_CMD_START_PIPELINED_READ 0x4 /* enum */ +#define MC_CMD_DPCPU_RPC_IN_CSR_ACCESS_START_DELAY_LBN 48 +#define MC_CMD_DPCPU_RPC_IN_CSR_ACCESS_START_DELAY_WIDTH 16 +#define MC_CMD_DPCPU_RPC_IN_CSR_ACCESS_RPT_COUNT_LBN 64 +#define MC_CMD_DPCPU_RPC_IN_CSR_ACCESS_RPT_COUNT_WIDTH 16 +#define MC_CMD_DPCPU_RPC_IN_CSR_ACCESS_GAP_DELAY_LBN 80 +#define MC_CMD_DPCPU_RPC_IN_CSR_ACCESS_GAP_DELAY_WIDTH 16 +#define MC_CMD_DPCPU_RPC_IN_MC_REPLAY_MODE_LBN 16 +#define MC_CMD_DPCPU_RPC_IN_MC_REPLAY_MODE_WIDTH 16 +#define MC_CMD_DPCPU_RPC_IN_MC_REPLAY_MODE_CUT_THROUGH 0x1 /* enum */ +#define MC_CMD_DPCPU_RPC_IN_MC_REPLAY_MODE_STORE_FORWARD 0x2 /* enum */ +#define MC_CMD_DPCPU_RPC_IN_MC_REPLAY_MODE_STORE_FORWARD_FIRST 0x3 /* enum */ +#define MC_CMD_DPCPU_RPC_IN_MC_REPLAY_CNTXT_LBN 64 +#define MC_CMD_DPCPU_RPC_IN_MC_REPLAY_CNTXT_WIDTH 16 +#define MC_CMD_DPCPU_RPC_IN_WDATA_OFST 12 +#define MC_CMD_DPCPU_RPC_IN_WDATA_LEN 24 +/* Register data to write. Only valid in write/write-read. */ +#define MC_CMD_DPCPU_RPC_IN_CSR_ACCESS_DATA_OFST 16 +/* Register address. */ +#define MC_CMD_DPCPU_RPC_IN_CSR_ACCESS_ADDRESS_OFST 20 + +/* MC_CMD_DPCPU_RPC_OUT msgresponse */ +#define MC_CMD_DPCPU_RPC_OUT_LEN 36 +#define MC_CMD_DPCPU_RPC_OUT_RC_OFST 0 +/* DATA */ +#define MC_CMD_DPCPU_RPC_OUT_DATA_OFST 4 +#define MC_CMD_DPCPU_RPC_OUT_DATA_LEN 32 +#define MC_CMD_DPCPU_RPC_OUT_HDR_CMD_RESP_ERRCODE_LBN 32 +#define MC_CMD_DPCPU_RPC_OUT_HDR_CMD_RESP_ERRCODE_WIDTH 16 +#define MC_CMD_DPCPU_RPC_OUT_CSR_ACCESS_READ_COUNT_LBN 48 +#define MC_CMD_DPCPU_RPC_OUT_CSR_ACCESS_READ_COUNT_WIDTH 16 +#define MC_CMD_DPCPU_RPC_OUT_RDATA_OFST 12 +#define MC_CMD_DPCPU_RPC_OUT_RDATA_LEN 24 +#define MC_CMD_DPCPU_RPC_OUT_CSR_ACCESS_READ_VAL_1_OFST 12 +#define MC_CMD_DPCPU_RPC_OUT_CSR_ACCESS_READ_VAL_2_OFST 16 +#define MC_CMD_DPCPU_RPC_OUT_CSR_ACCESS_READ_VAL_3_OFST 20 +#define MC_CMD_DPCPU_RPC_OUT_CSR_ACCESS_READ_VAL_4_OFST 24 + + +/***********************************/ +/* MC_CMD_TRIGGER_INTERRUPT + * Trigger an interrupt by prodding the BIU. + */ +#define MC_CMD_TRIGGER_INTERRUPT 0xe3 +#undef MC_CMD_0xe3_PRIVILEGE_CTG + +#define MC_CMD_0xe3_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_TRIGGER_INTERRUPT_IN msgrequest */ +#define MC_CMD_TRIGGER_INTERRUPT_IN_LEN 4 +/* Interrupt level relative to base for function. */ +#define MC_CMD_TRIGGER_INTERRUPT_IN_INTR_LEVEL_OFST 0 + +/* MC_CMD_TRIGGER_INTERRUPT_OUT msgresponse */ +#define MC_CMD_TRIGGER_INTERRUPT_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_SHMBOOT_OP + * Special operations to support (for now) shmboot. + */ +#define MC_CMD_SHMBOOT_OP 0xe6 +#undef MC_CMD_0xe6_PRIVILEGE_CTG + +#define MC_CMD_0xe6_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_SHMBOOT_OP_IN msgrequest */ +#define MC_CMD_SHMBOOT_OP_IN_LEN 4 +/* Identifies the operation to perform */ +#define MC_CMD_SHMBOOT_OP_IN_SHMBOOT_OP_OFST 0 +/* enum: Copy slave_data section to the slave core. (Greenport only) */ +#define MC_CMD_SHMBOOT_OP_IN_PUSH_SLAVE_DATA 0x0 + +/* MC_CMD_SHMBOOT_OP_OUT msgresponse */ +#define MC_CMD_SHMBOOT_OP_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_CAP_BLK_READ + * Read multiple 64bit words from capture block memory + */ +#define MC_CMD_CAP_BLK_READ 0xe7 +#undef MC_CMD_0xe7_PRIVILEGE_CTG + +#define MC_CMD_0xe7_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_CAP_BLK_READ_IN msgrequest */ +#define MC_CMD_CAP_BLK_READ_IN_LEN 12 +#define MC_CMD_CAP_BLK_READ_IN_CAP_REG_OFST 0 +#define MC_CMD_CAP_BLK_READ_IN_ADDR_OFST 4 +#define MC_CMD_CAP_BLK_READ_IN_COUNT_OFST 8 + +/* MC_CMD_CAP_BLK_READ_OUT msgresponse */ +#define MC_CMD_CAP_BLK_READ_OUT_LENMIN 8 +#define MC_CMD_CAP_BLK_READ_OUT_LENMAX 248 +#define MC_CMD_CAP_BLK_READ_OUT_LEN(num) (0+8*(num)) +#define MC_CMD_CAP_BLK_READ_OUT_BUFFER_OFST 0 +#define MC_CMD_CAP_BLK_READ_OUT_BUFFER_LEN 8 +#define MC_CMD_CAP_BLK_READ_OUT_BUFFER_LO_OFST 0 +#define MC_CMD_CAP_BLK_READ_OUT_BUFFER_HI_OFST 4 +#define MC_CMD_CAP_BLK_READ_OUT_BUFFER_MINNUM 1 +#define MC_CMD_CAP_BLK_READ_OUT_BUFFER_MAXNUM 31 + + +/***********************************/ +/* MC_CMD_DUMP_DO + * Take a dump of the DUT state + */ +#define MC_CMD_DUMP_DO 0xe8 +#undef MC_CMD_0xe8_PRIVILEGE_CTG + +#define MC_CMD_0xe8_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_DUMP_DO_IN msgrequest */ +#define MC_CMD_DUMP_DO_IN_LEN 52 +#define MC_CMD_DUMP_DO_IN_PADDING_OFST 0 +#define MC_CMD_DUMP_DO_IN_DUMPSPEC_SRC_OFST 4 +#define MC_CMD_DUMP_DO_IN_DUMPSPEC_SRC_CUSTOM 0x0 /* enum */ +#define MC_CMD_DUMP_DO_IN_DUMPSPEC_SRC_DEFAULT 0x1 /* enum */ +#define MC_CMD_DUMP_DO_IN_DUMPSPEC_SRC_CUSTOM_TYPE_OFST 8 +#define MC_CMD_DUMP_DO_IN_DUMP_LOCATION_NVRAM 0x1 /* enum */ +#define MC_CMD_DUMP_DO_IN_DUMP_LOCATION_HOST_MEMORY 0x2 /* enum */ +#define MC_CMD_DUMP_DO_IN_DUMP_LOCATION_HOST_MEMORY_MLI 0x3 /* enum */ +#define MC_CMD_DUMP_DO_IN_DUMP_LOCATION_UART 0x4 /* enum */ +#define MC_CMD_DUMP_DO_IN_DUMPSPEC_SRC_CUSTOM_NVRAM_PARTITION_TYPE_ID_OFST 12 +#define MC_CMD_DUMP_DO_IN_DUMPSPEC_SRC_CUSTOM_NVRAM_OFFSET_OFST 16 +#define MC_CMD_DUMP_DO_IN_DUMPSPEC_SRC_CUSTOM_HOST_MEMORY_ADDR_LO_OFST 12 +#define MC_CMD_DUMP_DO_IN_DUMPSPEC_SRC_CUSTOM_HOST_MEMORY_ADDR_HI_OFST 16 +#define MC_CMD_DUMP_DO_IN_DUMPSPEC_SRC_CUSTOM_HOST_MEMORY_MLI_ROOT_ADDR_LO_OFST 12 +#define MC_CMD_DUMP_DO_IN_HOST_MEMORY_MLI_PAGE_SIZE 0x1000 /* enum */ +#define MC_CMD_DUMP_DO_IN_DUMPSPEC_SRC_CUSTOM_HOST_MEMORY_MLI_ROOT_ADDR_HI_OFST 16 +#define MC_CMD_DUMP_DO_IN_DUMPSPEC_SRC_CUSTOM_HOST_MEMORY_MLI_DEPTH_OFST 20 +#define MC_CMD_DUMP_DO_IN_HOST_MEMORY_MLI_MAX_DEPTH 0x2 /* enum */ +#define MC_CMD_DUMP_DO_IN_DUMPSPEC_SRC_CUSTOM_UART_PORT_OFST 12 +/* enum: The uart port this command was received over (if using a uart + * transport) + */ +#define MC_CMD_DUMP_DO_IN_UART_PORT_SRC 0xff +#define MC_CMD_DUMP_DO_IN_DUMPSPEC_SRC_CUSTOM_SIZE_OFST 24 +#define MC_CMD_DUMP_DO_IN_DUMPFILE_DST_OFST 28 +#define MC_CMD_DUMP_DO_IN_DUMPFILE_DST_CUSTOM 0x0 /* enum */ +#define MC_CMD_DUMP_DO_IN_DUMPFILE_DST_NVRAM_DUMP_PARTITION 0x1 /* enum */ +#define MC_CMD_DUMP_DO_IN_DUMPFILE_DST_CUSTOM_TYPE_OFST 32 +/* Enum values, see field(s): */ +/* MC_CMD_DUMP_DO_IN/DUMPSPEC_SRC_CUSTOM_TYPE */ +#define MC_CMD_DUMP_DO_IN_DUMPFILE_DST_CUSTOM_NVRAM_PARTITION_TYPE_ID_OFST 36 +#define MC_CMD_DUMP_DO_IN_DUMPFILE_DST_CUSTOM_NVRAM_OFFSET_OFST 40 +#define MC_CMD_DUMP_DO_IN_DUMPFILE_DST_CUSTOM_HOST_MEMORY_ADDR_LO_OFST 36 +#define MC_CMD_DUMP_DO_IN_DUMPFILE_DST_CUSTOM_HOST_MEMORY_ADDR_HI_OFST 40 +#define MC_CMD_DUMP_DO_IN_DUMPFILE_DST_CUSTOM_HOST_MEMORY_MLI_ROOT_ADDR_LO_OFST 36 +#define MC_CMD_DUMP_DO_IN_DUMPFILE_DST_CUSTOM_HOST_MEMORY_MLI_ROOT_ADDR_HI_OFST 40 +#define MC_CMD_DUMP_DO_IN_DUMPFILE_DST_CUSTOM_HOST_MEMORY_MLI_DEPTH_OFST 44 +#define MC_CMD_DUMP_DO_IN_DUMPFILE_DST_CUSTOM_UART_PORT_OFST 36 +#define MC_CMD_DUMP_DO_IN_DUMPFILE_DST_CUSTOM_SIZE_OFST 48 + +/* MC_CMD_DUMP_DO_OUT msgresponse */ +#define MC_CMD_DUMP_DO_OUT_LEN 4 +#define MC_CMD_DUMP_DO_OUT_DUMPFILE_SIZE_OFST 0 + + +/***********************************/ +/* MC_CMD_DUMP_CONFIGURE_UNSOLICITED + * Configure unsolicited dumps + */ +#define MC_CMD_DUMP_CONFIGURE_UNSOLICITED 0xe9 +#undef MC_CMD_0xe9_PRIVILEGE_CTG + +#define MC_CMD_0xe9_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN msgrequest */ +#define MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_LEN 52 +#define MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_ENABLE_OFST 0 +#define MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPSPEC_SRC_OFST 4 +/* Enum values, see field(s): */ +/* MC_CMD_DUMP_DO/MC_CMD_DUMP_DO_IN/DUMPSPEC_SRC */ +#define MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPSPEC_SRC_CUSTOM_TYPE_OFST 8 +/* Enum values, see field(s): */ +/* MC_CMD_DUMP_DO/MC_CMD_DUMP_DO_IN/DUMPSPEC_SRC_CUSTOM_TYPE */ +#define MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPSPEC_SRC_CUSTOM_NVRAM_PARTITION_TYPE_ID_OFST 12 +#define MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPSPEC_SRC_CUSTOM_NVRAM_OFFSET_OFST 16 +#define MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPSPEC_SRC_CUSTOM_HOST_MEMORY_ADDR_LO_OFST 12 +#define MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPSPEC_SRC_CUSTOM_HOST_MEMORY_ADDR_HI_OFST 16 +#define MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPSPEC_SRC_CUSTOM_HOST_MEMORY_MLI_ROOT_ADDR_LO_OFST 12 +#define MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPSPEC_SRC_CUSTOM_HOST_MEMORY_MLI_ROOT_ADDR_HI_OFST 16 +#define MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPSPEC_SRC_CUSTOM_HOST_MEMORY_MLI_DEPTH_OFST 20 +#define MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPSPEC_SRC_CUSTOM_UART_PORT_OFST 12 +#define MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPSPEC_SRC_CUSTOM_SIZE_OFST 24 +#define MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPFILE_DST_OFST 28 +/* Enum values, see field(s): */ +/* MC_CMD_DUMP_DO/MC_CMD_DUMP_DO_IN/DUMPFILE_DST */ +#define MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPFILE_DST_CUSTOM_TYPE_OFST 32 +/* Enum values, see field(s): */ +/* MC_CMD_DUMP_DO/MC_CMD_DUMP_DO_IN/DUMPSPEC_SRC_CUSTOM_TYPE */ +#define MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPFILE_DST_CUSTOM_NVRAM_PARTITION_TYPE_ID_OFST 36 +#define MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPFILE_DST_CUSTOM_NVRAM_OFFSET_OFST 40 +#define MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPFILE_DST_CUSTOM_HOST_MEMORY_ADDR_LO_OFST 36 +#define MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPFILE_DST_CUSTOM_HOST_MEMORY_ADDR_HI_OFST 40 +#define MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPFILE_DST_CUSTOM_HOST_MEMORY_MLI_ROOT_ADDR_LO_OFST 36 +#define MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPFILE_DST_CUSTOM_HOST_MEMORY_MLI_ROOT_ADDR_HI_OFST 40 +#define MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPFILE_DST_CUSTOM_HOST_MEMORY_MLI_DEPTH_OFST 44 +#define MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPFILE_DST_CUSTOM_UART_PORT_OFST 36 +#define MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPFILE_DST_CUSTOM_SIZE_OFST 48 + + +/***********************************/ +/* MC_CMD_SET_PSU + * Adjusts power supply parameters. This is a warranty-voiding operation. + * Returns: ENOENT if the parameter or rail specified does not exist, EINVAL if + * the parameter is out of range. + */ +#define MC_CMD_SET_PSU 0xea +#undef MC_CMD_0xea_PRIVILEGE_CTG + +#define MC_CMD_0xea_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_SET_PSU_IN msgrequest */ +#define MC_CMD_SET_PSU_IN_LEN 12 +#define MC_CMD_SET_PSU_IN_PARAM_OFST 0 +#define MC_CMD_SET_PSU_IN_PARAM_SUPPLY_VOLTAGE 0x0 /* enum */ +#define MC_CMD_SET_PSU_IN_RAIL_OFST 4 +#define MC_CMD_SET_PSU_IN_RAIL_0V9 0x0 /* enum */ +#define MC_CMD_SET_PSU_IN_RAIL_1V2 0x1 /* enum */ +/* desired value, eg voltage in mV */ +#define MC_CMD_SET_PSU_IN_VALUE_OFST 8 + +/* MC_CMD_SET_PSU_OUT msgresponse */ +#define MC_CMD_SET_PSU_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_GET_FUNCTION_INFO + * Get function information. PF and VF number. + */ +#define MC_CMD_GET_FUNCTION_INFO 0xec +#undef MC_CMD_0xec_PRIVILEGE_CTG + +#define MC_CMD_0xec_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_GET_FUNCTION_INFO_IN msgrequest */ +#define MC_CMD_GET_FUNCTION_INFO_IN_LEN 0 + +/* MC_CMD_GET_FUNCTION_INFO_OUT msgresponse */ +#define MC_CMD_GET_FUNCTION_INFO_OUT_LEN 8 +#define MC_CMD_GET_FUNCTION_INFO_OUT_PF_OFST 0 +#define MC_CMD_GET_FUNCTION_INFO_OUT_VF_OFST 4 + + +/***********************************/ +/* MC_CMD_ENABLE_OFFLINE_BIST + * Enters offline BIST mode. All queues are torn down, chip enters quiescent + * mode, calling function gets exclusive MCDI ownership. The only way out is + * reboot. + */ +#define MC_CMD_ENABLE_OFFLINE_BIST 0xed +#undef MC_CMD_0xed_PRIVILEGE_CTG + +#define MC_CMD_0xed_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_ENABLE_OFFLINE_BIST_IN msgrequest */ +#define MC_CMD_ENABLE_OFFLINE_BIST_IN_LEN 0 + +/* MC_CMD_ENABLE_OFFLINE_BIST_OUT msgresponse */ +#define MC_CMD_ENABLE_OFFLINE_BIST_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_UART_SEND_DATA + * Send checksummed[sic] block of data over the uart. Response is a placeholder + * should we wish to make this reliable; currently requests are fire-and- + * forget. + */ +#define MC_CMD_UART_SEND_DATA 0xee +#undef MC_CMD_0xee_PRIVILEGE_CTG + +#define MC_CMD_0xee_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_UART_SEND_DATA_OUT msgrequest */ +#define MC_CMD_UART_SEND_DATA_OUT_LENMIN 16 +#define MC_CMD_UART_SEND_DATA_OUT_LENMAX 252 +#define MC_CMD_UART_SEND_DATA_OUT_LEN(num) (16+1*(num)) +/* CRC32 over OFFSET, LENGTH, RESERVED, DATA */ +#define MC_CMD_UART_SEND_DATA_OUT_CHECKSUM_OFST 0 +/* Offset at which to write the data */ +#define MC_CMD_UART_SEND_DATA_OUT_OFFSET_OFST 4 +/* Length of data */ +#define MC_CMD_UART_SEND_DATA_OUT_LENGTH_OFST 8 +/* Reserved for future use */ +#define MC_CMD_UART_SEND_DATA_OUT_RESERVED_OFST 12 +#define MC_CMD_UART_SEND_DATA_OUT_DATA_OFST 16 +#define MC_CMD_UART_SEND_DATA_OUT_DATA_LEN 1 +#define MC_CMD_UART_SEND_DATA_OUT_DATA_MINNUM 0 +#define MC_CMD_UART_SEND_DATA_OUT_DATA_MAXNUM 236 + +/* MC_CMD_UART_SEND_DATA_IN msgresponse */ +#define MC_CMD_UART_SEND_DATA_IN_LEN 0 + + +/***********************************/ +/* MC_CMD_UART_RECV_DATA + * Request checksummed[sic] block of data over the uart. Only a placeholder, + * subject to change and not currently implemented. + */ +#define MC_CMD_UART_RECV_DATA 0xef +#undef MC_CMD_0xef_PRIVILEGE_CTG + +#define MC_CMD_0xef_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_UART_RECV_DATA_OUT msgrequest */ +#define MC_CMD_UART_RECV_DATA_OUT_LEN 16 +/* CRC32 over OFFSET, LENGTH, RESERVED */ +#define MC_CMD_UART_RECV_DATA_OUT_CHECKSUM_OFST 0 +/* Offset from which to read the data */ +#define MC_CMD_UART_RECV_DATA_OUT_OFFSET_OFST 4 +/* Length of data */ +#define MC_CMD_UART_RECV_DATA_OUT_LENGTH_OFST 8 +/* Reserved for future use */ +#define MC_CMD_UART_RECV_DATA_OUT_RESERVED_OFST 12 + +/* MC_CMD_UART_RECV_DATA_IN msgresponse */ +#define MC_CMD_UART_RECV_DATA_IN_LENMIN 16 +#define MC_CMD_UART_RECV_DATA_IN_LENMAX 252 +#define MC_CMD_UART_RECV_DATA_IN_LEN(num) (16+1*(num)) +/* CRC32 over RESERVED1, RESERVED2, RESERVED3, DATA */ +#define MC_CMD_UART_RECV_DATA_IN_CHECKSUM_OFST 0 +/* Offset at which to write the data */ +#define MC_CMD_UART_RECV_DATA_IN_RESERVED1_OFST 4 +/* Length of data */ +#define MC_CMD_UART_RECV_DATA_IN_RESERVED2_OFST 8 +/* Reserved for future use */ +#define MC_CMD_UART_RECV_DATA_IN_RESERVED3_OFST 12 +#define MC_CMD_UART_RECV_DATA_IN_DATA_OFST 16 +#define MC_CMD_UART_RECV_DATA_IN_DATA_LEN 1 +#define MC_CMD_UART_RECV_DATA_IN_DATA_MINNUM 0 +#define MC_CMD_UART_RECV_DATA_IN_DATA_MAXNUM 236 + + +/***********************************/ +/* MC_CMD_READ_FUSES + * Read data programmed into the device One-Time-Programmable (OTP) Fuses + */ +#define MC_CMD_READ_FUSES 0xf0 +#undef MC_CMD_0xf0_PRIVILEGE_CTG + +#define MC_CMD_0xf0_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_READ_FUSES_IN msgrequest */ +#define MC_CMD_READ_FUSES_IN_LEN 8 +/* Offset in OTP to read */ +#define MC_CMD_READ_FUSES_IN_OFFSET_OFST 0 +/* Length of data to read in bytes */ +#define MC_CMD_READ_FUSES_IN_LENGTH_OFST 4 + +/* MC_CMD_READ_FUSES_OUT msgresponse */ +#define MC_CMD_READ_FUSES_OUT_LENMIN 4 +#define MC_CMD_READ_FUSES_OUT_LENMAX 252 +#define MC_CMD_READ_FUSES_OUT_LEN(num) (4+1*(num)) +/* Length of returned OTP data in bytes */ +#define MC_CMD_READ_FUSES_OUT_LENGTH_OFST 0 +/* Returned data */ +#define MC_CMD_READ_FUSES_OUT_DATA_OFST 4 +#define MC_CMD_READ_FUSES_OUT_DATA_LEN 1 +#define MC_CMD_READ_FUSES_OUT_DATA_MINNUM 0 +#define MC_CMD_READ_FUSES_OUT_DATA_MAXNUM 248 + + +/***********************************/ +/* MC_CMD_KR_TUNE + * Get or set KR Serdes RXEQ and TX Driver settings + */ +#define MC_CMD_KR_TUNE 0xf1 +#undef MC_CMD_0xf1_PRIVILEGE_CTG + +#define MC_CMD_0xf1_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_KR_TUNE_IN msgrequest */ +#define MC_CMD_KR_TUNE_IN_LENMIN 4 +#define MC_CMD_KR_TUNE_IN_LENMAX 252 +#define MC_CMD_KR_TUNE_IN_LEN(num) (4+4*(num)) +/* Requested operation */ +#define MC_CMD_KR_TUNE_IN_KR_TUNE_OP_OFST 0 +#define MC_CMD_KR_TUNE_IN_KR_TUNE_OP_LEN 1 +/* enum: Get current RXEQ settings */ +#define MC_CMD_KR_TUNE_IN_RXEQ_GET 0x0 +/* enum: Override RXEQ settings */ +#define MC_CMD_KR_TUNE_IN_RXEQ_SET 0x1 +/* enum: Get current TX Driver settings */ +#define MC_CMD_KR_TUNE_IN_TXEQ_GET 0x2 +/* enum: Override TX Driver settings */ +#define MC_CMD_KR_TUNE_IN_TXEQ_SET 0x3 +/* enum: Force KR Serdes reset / recalibration */ +#define MC_CMD_KR_TUNE_IN_RECAL 0x4 +/* enum: Start KR Serdes Eye diagram plot on a given lane. Lane must have valid + * signal. + */ +#define MC_CMD_KR_TUNE_IN_START_EYE_PLOT 0x5 +/* enum: Poll KR Serdes Eye diagram plot. Returns one row of BER data. The + * caller should call this command repeatedly after starting eye plot, until no + * more data is returned. + */ +#define MC_CMD_KR_TUNE_IN_POLL_EYE_PLOT 0x6 +/* enum: Read Figure Of Merit (eye quality, higher is better). */ +#define MC_CMD_KR_TUNE_IN_READ_FOM 0x7 +/* Align the arguments to 32 bits */ +#define MC_CMD_KR_TUNE_IN_KR_TUNE_RSVD_OFST 1 +#define MC_CMD_KR_TUNE_IN_KR_TUNE_RSVD_LEN 3 +/* Arguments specific to the operation */ +#define MC_CMD_KR_TUNE_IN_KR_TUNE_ARGS_OFST 4 +#define MC_CMD_KR_TUNE_IN_KR_TUNE_ARGS_LEN 4 +#define MC_CMD_KR_TUNE_IN_KR_TUNE_ARGS_MINNUM 0 +#define MC_CMD_KR_TUNE_IN_KR_TUNE_ARGS_MAXNUM 62 + +/* MC_CMD_KR_TUNE_OUT msgresponse */ +#define MC_CMD_KR_TUNE_OUT_LEN 0 + +/* MC_CMD_KR_TUNE_RXEQ_GET_IN msgrequest */ +#define MC_CMD_KR_TUNE_RXEQ_GET_IN_LEN 4 +/* Requested operation */ +#define MC_CMD_KR_TUNE_RXEQ_GET_IN_KR_TUNE_OP_OFST 0 +#define MC_CMD_KR_TUNE_RXEQ_GET_IN_KR_TUNE_OP_LEN 1 +/* Align the arguments to 32 bits */ +#define MC_CMD_KR_TUNE_RXEQ_GET_IN_KR_TUNE_RSVD_OFST 1 +#define MC_CMD_KR_TUNE_RXEQ_GET_IN_KR_TUNE_RSVD_LEN 3 + +/* MC_CMD_KR_TUNE_RXEQ_GET_OUT msgresponse */ +#define MC_CMD_KR_TUNE_RXEQ_GET_OUT_LENMIN 4 +#define MC_CMD_KR_TUNE_RXEQ_GET_OUT_LENMAX 252 +#define MC_CMD_KR_TUNE_RXEQ_GET_OUT_LEN(num) (0+4*(num)) +/* RXEQ Parameter */ +#define MC_CMD_KR_TUNE_RXEQ_GET_OUT_PARAM_OFST 0 +#define MC_CMD_KR_TUNE_RXEQ_GET_OUT_PARAM_LEN 4 +#define MC_CMD_KR_TUNE_RXEQ_GET_OUT_PARAM_MINNUM 1 +#define MC_CMD_KR_TUNE_RXEQ_GET_OUT_PARAM_MAXNUM 63 +#define MC_CMD_KR_TUNE_RXEQ_GET_OUT_PARAM_ID_LBN 0 +#define MC_CMD_KR_TUNE_RXEQ_GET_OUT_PARAM_ID_WIDTH 8 +/* enum: Attenuation (0-15, Huntington) */ +#define MC_CMD_KR_TUNE_RXEQ_GET_OUT_ATT 0x0 +/* enum: CTLE Boost (0-15, Huntington) */ +#define MC_CMD_KR_TUNE_RXEQ_GET_OUT_BOOST 0x1 +/* enum: Edge DFE Tap1 (Huntington - 0 - max negative, 64 - zero, 127 - max + * positive, Medford - 0-31) + */ +#define MC_CMD_KR_TUNE_RXEQ_GET_OUT_EDFE_TAP1 0x2 +/* enum: Edge DFE Tap2 (Huntington - 0 - max negative, 32 - zero, 63 - max + * positive, Medford - 0-31) + */ +#define MC_CMD_KR_TUNE_RXEQ_GET_OUT_EDFE_TAP2 0x3 +/* enum: Edge DFE Tap3 (Huntington - 0 - max negative, 32 - zero, 63 - max + * positive, Medford - 0-16) + */ +#define MC_CMD_KR_TUNE_RXEQ_GET_OUT_EDFE_TAP3 0x4 +/* enum: Edge DFE Tap4 (Huntington - 0 - max negative, 32 - zero, 63 - max + * positive, Medford - 0-16) + */ +#define MC_CMD_KR_TUNE_RXEQ_GET_OUT_EDFE_TAP4 0x5 +/* enum: Edge DFE Tap5 (Huntington - 0 - max negative, 32 - zero, 63 - max + * positive, Medford - 0-16) + */ +#define MC_CMD_KR_TUNE_RXEQ_GET_OUT_EDFE_TAP5 0x6 +/* enum: Edge DFE DLEV (0-128 for Medford) */ +#define MC_CMD_KR_TUNE_RXEQ_GET_OUT_EDFE_DLEV 0x7 +/* enum: Variable Gain Amplifier (0-15, Medford) */ +#define MC_CMD_KR_TUNE_RXEQ_GET_OUT_VGA 0x8 +/* enum: CTLE EQ Capacitor (0-15, Medford) */ +#define MC_CMD_KR_TUNE_RXEQ_GET_OUT_CTLE_EQC 0x9 +/* enum: CTLE EQ Resistor (0-7, Medford) */ +#define MC_CMD_KR_TUNE_RXEQ_GET_OUT_CTLE_EQRES 0xa +#define MC_CMD_KR_TUNE_RXEQ_GET_OUT_PARAM_LANE_LBN 8 +#define MC_CMD_KR_TUNE_RXEQ_GET_OUT_PARAM_LANE_WIDTH 3 +#define MC_CMD_KR_TUNE_RXEQ_GET_OUT_LANE_0 0x0 /* enum */ +#define MC_CMD_KR_TUNE_RXEQ_GET_OUT_LANE_1 0x1 /* enum */ +#define MC_CMD_KR_TUNE_RXEQ_GET_OUT_LANE_2 0x2 /* enum */ +#define MC_CMD_KR_TUNE_RXEQ_GET_OUT_LANE_3 0x3 /* enum */ +#define MC_CMD_KR_TUNE_RXEQ_GET_OUT_LANE_ALL 0x4 /* enum */ +#define MC_CMD_KR_TUNE_RXEQ_GET_OUT_PARAM_AUTOCAL_LBN 11 +#define MC_CMD_KR_TUNE_RXEQ_GET_OUT_PARAM_AUTOCAL_WIDTH 1 +#define MC_CMD_KR_TUNE_RXEQ_GET_OUT_RESERVED_LBN 12 +#define MC_CMD_KR_TUNE_RXEQ_GET_OUT_RESERVED_WIDTH 4 +#define MC_CMD_KR_TUNE_RXEQ_GET_OUT_PARAM_INITIAL_LBN 16 +#define MC_CMD_KR_TUNE_RXEQ_GET_OUT_PARAM_INITIAL_WIDTH 8 +#define MC_CMD_KR_TUNE_RXEQ_GET_OUT_PARAM_CURRENT_LBN 24 +#define MC_CMD_KR_TUNE_RXEQ_GET_OUT_PARAM_CURRENT_WIDTH 8 + +/* MC_CMD_KR_TUNE_RXEQ_SET_IN msgrequest */ +#define MC_CMD_KR_TUNE_RXEQ_SET_IN_LENMIN 8 +#define MC_CMD_KR_TUNE_RXEQ_SET_IN_LENMAX 252 +#define MC_CMD_KR_TUNE_RXEQ_SET_IN_LEN(num) (4+4*(num)) +/* Requested operation */ +#define MC_CMD_KR_TUNE_RXEQ_SET_IN_KR_TUNE_OP_OFST 0 +#define MC_CMD_KR_TUNE_RXEQ_SET_IN_KR_TUNE_OP_LEN 1 +/* Align the arguments to 32 bits */ +#define MC_CMD_KR_TUNE_RXEQ_SET_IN_KR_TUNE_RSVD_OFST 1 +#define MC_CMD_KR_TUNE_RXEQ_SET_IN_KR_TUNE_RSVD_LEN 3 +/* RXEQ Parameter */ +#define MC_CMD_KR_TUNE_RXEQ_SET_IN_PARAM_OFST 4 +#define MC_CMD_KR_TUNE_RXEQ_SET_IN_PARAM_LEN 4 +#define MC_CMD_KR_TUNE_RXEQ_SET_IN_PARAM_MINNUM 1 +#define MC_CMD_KR_TUNE_RXEQ_SET_IN_PARAM_MAXNUM 62 +#define MC_CMD_KR_TUNE_RXEQ_SET_IN_PARAM_ID_LBN 0 +#define MC_CMD_KR_TUNE_RXEQ_SET_IN_PARAM_ID_WIDTH 8 +/* Enum values, see field(s): */ +/* MC_CMD_KR_TUNE_RXEQ_GET_OUT/PARAM_ID */ +#define MC_CMD_KR_TUNE_RXEQ_SET_IN_PARAM_LANE_LBN 8 +#define MC_CMD_KR_TUNE_RXEQ_SET_IN_PARAM_LANE_WIDTH 3 +/* Enum values, see field(s): */ +/* MC_CMD_KR_TUNE_RXEQ_GET_OUT/PARAM_LANE */ +#define MC_CMD_KR_TUNE_RXEQ_SET_IN_PARAM_AUTOCAL_LBN 11 +#define MC_CMD_KR_TUNE_RXEQ_SET_IN_PARAM_AUTOCAL_WIDTH 1 +#define MC_CMD_KR_TUNE_RXEQ_SET_IN_RESERVED_LBN 12 +#define MC_CMD_KR_TUNE_RXEQ_SET_IN_RESERVED_WIDTH 4 +#define MC_CMD_KR_TUNE_RXEQ_SET_IN_PARAM_INITIAL_LBN 16 +#define MC_CMD_KR_TUNE_RXEQ_SET_IN_PARAM_INITIAL_WIDTH 8 +#define MC_CMD_KR_TUNE_RXEQ_SET_IN_RESERVED2_LBN 24 +#define MC_CMD_KR_TUNE_RXEQ_SET_IN_RESERVED2_WIDTH 8 + +/* MC_CMD_KR_TUNE_RXEQ_SET_OUT msgresponse */ +#define MC_CMD_KR_TUNE_RXEQ_SET_OUT_LEN 0 + +/* MC_CMD_KR_TUNE_TXEQ_GET_IN msgrequest */ +#define MC_CMD_KR_TUNE_TXEQ_GET_IN_LEN 4 +/* Requested operation */ +#define MC_CMD_KR_TUNE_TXEQ_GET_IN_KR_TUNE_OP_OFST 0 +#define MC_CMD_KR_TUNE_TXEQ_GET_IN_KR_TUNE_OP_LEN 1 +/* Align the arguments to 32 bits */ +#define MC_CMD_KR_TUNE_TXEQ_GET_IN_KR_TUNE_RSVD_OFST 1 +#define MC_CMD_KR_TUNE_TXEQ_GET_IN_KR_TUNE_RSVD_LEN 3 + +/* MC_CMD_KR_TUNE_TXEQ_GET_OUT msgresponse */ +#define MC_CMD_KR_TUNE_TXEQ_GET_OUT_LENMIN 4 +#define MC_CMD_KR_TUNE_TXEQ_GET_OUT_LENMAX 252 +#define MC_CMD_KR_TUNE_TXEQ_GET_OUT_LEN(num) (0+4*(num)) +/* TXEQ Parameter */ +#define MC_CMD_KR_TUNE_TXEQ_GET_OUT_PARAM_OFST 0 +#define MC_CMD_KR_TUNE_TXEQ_GET_OUT_PARAM_LEN 4 +#define MC_CMD_KR_TUNE_TXEQ_GET_OUT_PARAM_MINNUM 1 +#define MC_CMD_KR_TUNE_TXEQ_GET_OUT_PARAM_MAXNUM 63 +#define MC_CMD_KR_TUNE_TXEQ_GET_OUT_PARAM_ID_LBN 0 +#define MC_CMD_KR_TUNE_TXEQ_GET_OUT_PARAM_ID_WIDTH 8 +/* enum: TX Amplitude (Huntington, Medford) */ +#define MC_CMD_KR_TUNE_TXEQ_GET_OUT_TX_LEV 0x0 +/* enum: De-Emphasis Tap1 Magnitude (0-7) (Huntington) */ +#define MC_CMD_KR_TUNE_TXEQ_GET_OUT_TX_MODE 0x1 +/* enum: De-Emphasis Tap1 Fine */ +#define MC_CMD_KR_TUNE_TXEQ_GET_OUT_TX_DTLEV 0x2 +/* enum: De-Emphasis Tap2 Magnitude (0-6) (Huntington) */ +#define MC_CMD_KR_TUNE_TXEQ_GET_OUT_TX_D2 0x3 +/* enum: De-Emphasis Tap2 Fine (Huntington) */ +#define MC_CMD_KR_TUNE_TXEQ_GET_OUT_TX_D2TLEV 0x4 +/* enum: Pre-Emphasis Magnitude (Huntington) */ +#define MC_CMD_KR_TUNE_TXEQ_GET_OUT_TX_E 0x5 +/* enum: Pre-Emphasis Fine (Huntington) */ +#define MC_CMD_KR_TUNE_TXEQ_GET_OUT_TX_ETLEV 0x6 +/* enum: TX Slew Rate Coarse control (Huntington) */ +#define MC_CMD_KR_TUNE_TXEQ_GET_OUT_TX_PREDRV_DLY 0x7 +/* enum: TX Slew Rate Fine control (Huntington) */ +#define MC_CMD_KR_TUNE_TXEQ_GET_OUT_TX_SR_SET 0x8 +/* enum: TX Termination Impedance control (Huntington) */ +#define MC_CMD_KR_TUNE_TXEQ_GET_OUT_TX_RT_SET 0x9 +/* enum: TX Amplitude Fine control (Medford) */ +#define MC_CMD_KR_TUNE_TXEQ_GET_OUT_TX_LEV_FINE 0xa +/* enum: Pre-shoot Tap (Medford) */ +#define MC_CMD_KR_TUNE_TXEQ_GET_OUT_TAP_ADV 0xb +/* enum: De-emphasis Tap (Medford) */ +#define MC_CMD_KR_TUNE_TXEQ_GET_OUT_TAP_DLY 0xc +#define MC_CMD_KR_TUNE_TXEQ_GET_OUT_PARAM_LANE_LBN 8 +#define MC_CMD_KR_TUNE_TXEQ_GET_OUT_PARAM_LANE_WIDTH 3 +#define MC_CMD_KR_TUNE_TXEQ_GET_OUT_LANE_0 0x0 /* enum */ +#define MC_CMD_KR_TUNE_TXEQ_GET_OUT_LANE_1 0x1 /* enum */ +#define MC_CMD_KR_TUNE_TXEQ_GET_OUT_LANE_2 0x2 /* enum */ +#define MC_CMD_KR_TUNE_TXEQ_GET_OUT_LANE_3 0x3 /* enum */ +#define MC_CMD_KR_TUNE_TXEQ_GET_OUT_LANE_ALL 0x4 /* enum */ +#define MC_CMD_KR_TUNE_TXEQ_GET_OUT_RESERVED_LBN 11 +#define MC_CMD_KR_TUNE_TXEQ_GET_OUT_RESERVED_WIDTH 5 +#define MC_CMD_KR_TUNE_TXEQ_GET_OUT_PARAM_INITIAL_LBN 16 +#define MC_CMD_KR_TUNE_TXEQ_GET_OUT_PARAM_INITIAL_WIDTH 8 +#define MC_CMD_KR_TUNE_TXEQ_GET_OUT_RESERVED2_LBN 24 +#define MC_CMD_KR_TUNE_TXEQ_GET_OUT_RESERVED2_WIDTH 8 + +/* MC_CMD_KR_TUNE_TXEQ_SET_IN msgrequest */ +#define MC_CMD_KR_TUNE_TXEQ_SET_IN_LENMIN 8 +#define MC_CMD_KR_TUNE_TXEQ_SET_IN_LENMAX 252 +#define MC_CMD_KR_TUNE_TXEQ_SET_IN_LEN(num) (4+4*(num)) +/* Requested operation */ +#define MC_CMD_KR_TUNE_TXEQ_SET_IN_KR_TUNE_OP_OFST 0 +#define MC_CMD_KR_TUNE_TXEQ_SET_IN_KR_TUNE_OP_LEN 1 +/* Align the arguments to 32 bits */ +#define MC_CMD_KR_TUNE_TXEQ_SET_IN_KR_TUNE_RSVD_OFST 1 +#define MC_CMD_KR_TUNE_TXEQ_SET_IN_KR_TUNE_RSVD_LEN 3 +/* TXEQ Parameter */ +#define MC_CMD_KR_TUNE_TXEQ_SET_IN_PARAM_OFST 4 +#define MC_CMD_KR_TUNE_TXEQ_SET_IN_PARAM_LEN 4 +#define MC_CMD_KR_TUNE_TXEQ_SET_IN_PARAM_MINNUM 1 +#define MC_CMD_KR_TUNE_TXEQ_SET_IN_PARAM_MAXNUM 62 +#define MC_CMD_KR_TUNE_TXEQ_SET_IN_PARAM_ID_LBN 0 +#define MC_CMD_KR_TUNE_TXEQ_SET_IN_PARAM_ID_WIDTH 8 +/* Enum values, see field(s): */ +/* MC_CMD_KR_TUNE_TXEQ_GET_OUT/PARAM_ID */ +#define MC_CMD_KR_TUNE_TXEQ_SET_IN_PARAM_LANE_LBN 8 +#define MC_CMD_KR_TUNE_TXEQ_SET_IN_PARAM_LANE_WIDTH 3 +/* Enum values, see field(s): */ +/* MC_CMD_KR_TUNE_TXEQ_GET_OUT/PARAM_LANE */ +#define MC_CMD_KR_TUNE_TXEQ_SET_IN_RESERVED_LBN 11 +#define MC_CMD_KR_TUNE_TXEQ_SET_IN_RESERVED_WIDTH 5 +#define MC_CMD_KR_TUNE_TXEQ_SET_IN_PARAM_INITIAL_LBN 16 +#define MC_CMD_KR_TUNE_TXEQ_SET_IN_PARAM_INITIAL_WIDTH 8 +#define MC_CMD_KR_TUNE_TXEQ_SET_IN_RESERVED2_LBN 24 +#define MC_CMD_KR_TUNE_TXEQ_SET_IN_RESERVED2_WIDTH 8 + +/* MC_CMD_KR_TUNE_TXEQ_SET_OUT msgresponse */ +#define MC_CMD_KR_TUNE_TXEQ_SET_OUT_LEN 0 + +/* MC_CMD_KR_TUNE_RECAL_IN msgrequest */ +#define MC_CMD_KR_TUNE_RECAL_IN_LEN 4 +/* Requested operation */ +#define MC_CMD_KR_TUNE_RECAL_IN_KR_TUNE_OP_OFST 0 +#define MC_CMD_KR_TUNE_RECAL_IN_KR_TUNE_OP_LEN 1 +/* Align the arguments to 32 bits */ +#define MC_CMD_KR_TUNE_RECAL_IN_KR_TUNE_RSVD_OFST 1 +#define MC_CMD_KR_TUNE_RECAL_IN_KR_TUNE_RSVD_LEN 3 + +/* MC_CMD_KR_TUNE_RECAL_OUT msgresponse */ +#define MC_CMD_KR_TUNE_RECAL_OUT_LEN 0 + +/* MC_CMD_KR_TUNE_START_EYE_PLOT_IN msgrequest */ +#define MC_CMD_KR_TUNE_START_EYE_PLOT_IN_LEN 8 +/* Requested operation */ +#define MC_CMD_KR_TUNE_START_EYE_PLOT_IN_KR_TUNE_OP_OFST 0 +#define MC_CMD_KR_TUNE_START_EYE_PLOT_IN_KR_TUNE_OP_LEN 1 +/* Align the arguments to 32 bits */ +#define MC_CMD_KR_TUNE_START_EYE_PLOT_IN_KR_TUNE_RSVD_OFST 1 +#define MC_CMD_KR_TUNE_START_EYE_PLOT_IN_KR_TUNE_RSVD_LEN 3 +#define MC_CMD_KR_TUNE_START_EYE_PLOT_IN_LANE_OFST 4 + +/* MC_CMD_KR_TUNE_START_EYE_PLOT_OUT msgresponse */ +#define MC_CMD_KR_TUNE_START_EYE_PLOT_OUT_LEN 0 + +/* MC_CMD_KR_TUNE_POLL_EYE_PLOT_IN msgrequest */ +#define MC_CMD_KR_TUNE_POLL_EYE_PLOT_IN_LEN 4 +/* Requested operation */ +#define MC_CMD_KR_TUNE_POLL_EYE_PLOT_IN_KR_TUNE_OP_OFST 0 +#define MC_CMD_KR_TUNE_POLL_EYE_PLOT_IN_KR_TUNE_OP_LEN 1 +/* Align the arguments to 32 bits */ +#define MC_CMD_KR_TUNE_POLL_EYE_PLOT_IN_KR_TUNE_RSVD_OFST 1 +#define MC_CMD_KR_TUNE_POLL_EYE_PLOT_IN_KR_TUNE_RSVD_LEN 3 + +/* MC_CMD_KR_TUNE_POLL_EYE_PLOT_OUT msgresponse */ +#define MC_CMD_KR_TUNE_POLL_EYE_PLOT_OUT_LENMIN 0 +#define MC_CMD_KR_TUNE_POLL_EYE_PLOT_OUT_LENMAX 252 +#define MC_CMD_KR_TUNE_POLL_EYE_PLOT_OUT_LEN(num) (0+2*(num)) +#define MC_CMD_KR_TUNE_POLL_EYE_PLOT_OUT_SAMPLES_OFST 0 +#define MC_CMD_KR_TUNE_POLL_EYE_PLOT_OUT_SAMPLES_LEN 2 +#define MC_CMD_KR_TUNE_POLL_EYE_PLOT_OUT_SAMPLES_MINNUM 0 +#define MC_CMD_KR_TUNE_POLL_EYE_PLOT_OUT_SAMPLES_MAXNUM 126 + +/* MC_CMD_KR_TUNE_READ_FOM_IN msgrequest */ +#define MC_CMD_KR_TUNE_READ_FOM_IN_LEN 8 +/* Requested operation */ +#define MC_CMD_KR_TUNE_READ_FOM_IN_KR_TUNE_OP_OFST 0 +#define MC_CMD_KR_TUNE_READ_FOM_IN_KR_TUNE_OP_LEN 1 +/* Align the arguments to 32 bits */ +#define MC_CMD_KR_TUNE_READ_FOM_IN_KR_TUNE_RSVD_OFST 1 +#define MC_CMD_KR_TUNE_READ_FOM_IN_KR_TUNE_RSVD_LEN 3 +#define MC_CMD_KR_TUNE_READ_FOM_IN_LANE_OFST 4 + +/* MC_CMD_KR_TUNE_READ_FOM_OUT msgresponse */ +#define MC_CMD_KR_TUNE_READ_FOM_OUT_LEN 4 +#define MC_CMD_KR_TUNE_READ_FOM_OUT_FOM_OFST 0 + + +/***********************************/ +/* MC_CMD_PCIE_TUNE + * Get or set PCIE Serdes RXEQ and TX Driver settings + */ +#define MC_CMD_PCIE_TUNE 0xf2 +#undef MC_CMD_0xf2_PRIVILEGE_CTG + +#define MC_CMD_0xf2_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_PCIE_TUNE_IN msgrequest */ +#define MC_CMD_PCIE_TUNE_IN_LENMIN 4 +#define MC_CMD_PCIE_TUNE_IN_LENMAX 252 +#define MC_CMD_PCIE_TUNE_IN_LEN(num) (4+4*(num)) +/* Requested operation */ +#define MC_CMD_PCIE_TUNE_IN_PCIE_TUNE_OP_OFST 0 +#define MC_CMD_PCIE_TUNE_IN_PCIE_TUNE_OP_LEN 1 +/* enum: Get current RXEQ settings */ +#define MC_CMD_PCIE_TUNE_IN_RXEQ_GET 0x0 +/* enum: Override RXEQ settings */ +#define MC_CMD_PCIE_TUNE_IN_RXEQ_SET 0x1 +/* enum: Get current TX Driver settings */ +#define MC_CMD_PCIE_TUNE_IN_TXEQ_GET 0x2 +/* enum: Override TX Driver settings */ +#define MC_CMD_PCIE_TUNE_IN_TXEQ_SET 0x3 +/* enum: Start PCIe Serdes Eye diagram plot on a given lane. */ +#define MC_CMD_PCIE_TUNE_IN_START_EYE_PLOT 0x5 +/* enum: Poll PCIe Serdes Eye diagram plot. Returns one row of BER data. The + * caller should call this command repeatedly after starting eye plot, until no + * more data is returned. + */ +#define MC_CMD_PCIE_TUNE_IN_POLL_EYE_PLOT 0x6 +/* enum: Enable the SERDES BIST and set it to generate a 200MHz square wave */ +#define MC_CMD_PCIE_TUNE_IN_BIST_SQUARE_WAVE 0x7 +/* Align the arguments to 32 bits */ +#define MC_CMD_PCIE_TUNE_IN_PCIE_TUNE_RSVD_OFST 1 +#define MC_CMD_PCIE_TUNE_IN_PCIE_TUNE_RSVD_LEN 3 +/* Arguments specific to the operation */ +#define MC_CMD_PCIE_TUNE_IN_PCIE_TUNE_ARGS_OFST 4 +#define MC_CMD_PCIE_TUNE_IN_PCIE_TUNE_ARGS_LEN 4 +#define MC_CMD_PCIE_TUNE_IN_PCIE_TUNE_ARGS_MINNUM 0 +#define MC_CMD_PCIE_TUNE_IN_PCIE_TUNE_ARGS_MAXNUM 62 + +/* MC_CMD_PCIE_TUNE_OUT msgresponse */ +#define MC_CMD_PCIE_TUNE_OUT_LEN 0 + +/* MC_CMD_PCIE_TUNE_RXEQ_GET_IN msgrequest */ +#define MC_CMD_PCIE_TUNE_RXEQ_GET_IN_LEN 4 +/* Requested operation */ +#define MC_CMD_PCIE_TUNE_RXEQ_GET_IN_PCIE_TUNE_OP_OFST 0 +#define MC_CMD_PCIE_TUNE_RXEQ_GET_IN_PCIE_TUNE_OP_LEN 1 +/* Align the arguments to 32 bits */ +#define MC_CMD_PCIE_TUNE_RXEQ_GET_IN_PCIE_TUNE_RSVD_OFST 1 +#define MC_CMD_PCIE_TUNE_RXEQ_GET_IN_PCIE_TUNE_RSVD_LEN 3 + +/* MC_CMD_PCIE_TUNE_RXEQ_GET_OUT msgresponse */ +#define MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_LENMIN 4 +#define MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_LENMAX 252 +#define MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_LEN(num) (0+4*(num)) +/* RXEQ Parameter */ +#define MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_PARAM_OFST 0 +#define MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_PARAM_LEN 4 +#define MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_PARAM_MINNUM 1 +#define MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_PARAM_MAXNUM 63 +#define MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_PARAM_ID_LBN 0 +#define MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_PARAM_ID_WIDTH 8 +/* enum: Attenuation (0-15) */ +#define MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_ATT 0x0 +/* enum: CTLE Boost (0-15) */ +#define MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_BOOST 0x1 +/* enum: DFE Tap1 (0 - max negative, 64 - zero, 127 - max positive) */ +#define MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_DFE_TAP1 0x2 +/* enum: DFE Tap2 (0 - max negative, 32 - zero, 63 - max positive) */ +#define MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_DFE_TAP2 0x3 +/* enum: DFE Tap3 (0 - max negative, 32 - zero, 63 - max positive) */ +#define MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_DFE_TAP3 0x4 +/* enum: DFE Tap4 (0 - max negative, 32 - zero, 63 - max positive) */ +#define MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_DFE_TAP4 0x5 +/* enum: DFE Tap5 (0 - max negative, 32 - zero, 63 - max positive) */ +#define MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_DFE_TAP5 0x6 +/* enum: DFE DLev */ +#define MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_DFE_DLEV 0x7 +/* enum: Figure of Merit */ +#define MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_FOM 0x8 +/* enum: CTLE EQ Capacitor (HF Gain) */ +#define MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_CTLE_EQC 0x9 +/* enum: CTLE EQ Resistor (DC Gain) */ +#define MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_CTLE_EQRES 0xa +#define MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_PARAM_LANE_LBN 8 +#define MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_PARAM_LANE_WIDTH 5 +#define MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_LANE_0 0x0 /* enum */ +#define MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_LANE_1 0x1 /* enum */ +#define MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_LANE_2 0x2 /* enum */ +#define MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_LANE_3 0x3 /* enum */ +#define MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_LANE_4 0x4 /* enum */ +#define MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_LANE_5 0x5 /* enum */ +#define MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_LANE_6 0x6 /* enum */ +#define MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_LANE_7 0x7 /* enum */ +#define MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_LANE_8 0x8 /* enum */ +#define MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_LANE_9 0x9 /* enum */ +#define MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_LANE_10 0xa /* enum */ +#define MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_LANE_11 0xb /* enum */ +#define MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_LANE_12 0xc /* enum */ +#define MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_LANE_13 0xd /* enum */ +#define MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_LANE_14 0xe /* enum */ +#define MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_LANE_15 0xf /* enum */ +#define MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_LANE_ALL 0x10 /* enum */ +#define MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_PARAM_AUTOCAL_LBN 13 +#define MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_PARAM_AUTOCAL_WIDTH 1 +#define MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_RESERVED_LBN 14 +#define MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_RESERVED_WIDTH 10 +#define MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_PARAM_CURRENT_LBN 24 +#define MC_CMD_PCIE_TUNE_RXEQ_GET_OUT_PARAM_CURRENT_WIDTH 8 + +/* MC_CMD_PCIE_TUNE_RXEQ_SET_IN msgrequest */ +#define MC_CMD_PCIE_TUNE_RXEQ_SET_IN_LENMIN 8 +#define MC_CMD_PCIE_TUNE_RXEQ_SET_IN_LENMAX 252 +#define MC_CMD_PCIE_TUNE_RXEQ_SET_IN_LEN(num) (4+4*(num)) +/* Requested operation */ +#define MC_CMD_PCIE_TUNE_RXEQ_SET_IN_PCIE_TUNE_OP_OFST 0 +#define MC_CMD_PCIE_TUNE_RXEQ_SET_IN_PCIE_TUNE_OP_LEN 1 +/* Align the arguments to 32 bits */ +#define MC_CMD_PCIE_TUNE_RXEQ_SET_IN_PCIE_TUNE_RSVD_OFST 1 +#define MC_CMD_PCIE_TUNE_RXEQ_SET_IN_PCIE_TUNE_RSVD_LEN 3 +/* RXEQ Parameter */ +#define MC_CMD_PCIE_TUNE_RXEQ_SET_IN_PARAM_OFST 4 +#define MC_CMD_PCIE_TUNE_RXEQ_SET_IN_PARAM_LEN 4 +#define MC_CMD_PCIE_TUNE_RXEQ_SET_IN_PARAM_MINNUM 1 +#define MC_CMD_PCIE_TUNE_RXEQ_SET_IN_PARAM_MAXNUM 62 +#define MC_CMD_PCIE_TUNE_RXEQ_SET_IN_PARAM_ID_LBN 0 +#define MC_CMD_PCIE_TUNE_RXEQ_SET_IN_PARAM_ID_WIDTH 8 +/* Enum values, see field(s): */ +/* MC_CMD_PCIE_TUNE_RXEQ_GET_OUT/PARAM_ID */ +#define MC_CMD_PCIE_TUNE_RXEQ_SET_IN_PARAM_LANE_LBN 8 +#define MC_CMD_PCIE_TUNE_RXEQ_SET_IN_PARAM_LANE_WIDTH 5 +/* Enum values, see field(s): */ +/* MC_CMD_PCIE_TUNE_RXEQ_GET_OUT/PARAM_LANE */ +#define MC_CMD_PCIE_TUNE_RXEQ_SET_IN_PARAM_AUTOCAL_LBN 13 +#define MC_CMD_PCIE_TUNE_RXEQ_SET_IN_PARAM_AUTOCAL_WIDTH 1 +#define MC_CMD_PCIE_TUNE_RXEQ_SET_IN_RESERVED_LBN 14 +#define MC_CMD_PCIE_TUNE_RXEQ_SET_IN_RESERVED_WIDTH 2 +#define MC_CMD_PCIE_TUNE_RXEQ_SET_IN_PARAM_INITIAL_LBN 16 +#define MC_CMD_PCIE_TUNE_RXEQ_SET_IN_PARAM_INITIAL_WIDTH 8 +#define MC_CMD_PCIE_TUNE_RXEQ_SET_IN_RESERVED2_LBN 24 +#define MC_CMD_PCIE_TUNE_RXEQ_SET_IN_RESERVED2_WIDTH 8 + +/* MC_CMD_PCIE_TUNE_RXEQ_SET_OUT msgresponse */ +#define MC_CMD_PCIE_TUNE_RXEQ_SET_OUT_LEN 0 + +/* MC_CMD_PCIE_TUNE_TXEQ_GET_IN msgrequest */ +#define MC_CMD_PCIE_TUNE_TXEQ_GET_IN_LEN 4 +/* Requested operation */ +#define MC_CMD_PCIE_TUNE_TXEQ_GET_IN_PCIE_TUNE_OP_OFST 0 +#define MC_CMD_PCIE_TUNE_TXEQ_GET_IN_PCIE_TUNE_OP_LEN 1 +/* Align the arguments to 32 bits */ +#define MC_CMD_PCIE_TUNE_TXEQ_GET_IN_PCIE_TUNE_RSVD_OFST 1 +#define MC_CMD_PCIE_TUNE_TXEQ_GET_IN_PCIE_TUNE_RSVD_LEN 3 + +/* MC_CMD_PCIE_TUNE_TXEQ_GET_OUT msgresponse */ +#define MC_CMD_PCIE_TUNE_TXEQ_GET_OUT_LENMIN 4 +#define MC_CMD_PCIE_TUNE_TXEQ_GET_OUT_LENMAX 252 +#define MC_CMD_PCIE_TUNE_TXEQ_GET_OUT_LEN(num) (0+4*(num)) +/* RXEQ Parameter */ +#define MC_CMD_PCIE_TUNE_TXEQ_GET_OUT_PARAM_OFST 0 +#define MC_CMD_PCIE_TUNE_TXEQ_GET_OUT_PARAM_LEN 4 +#define MC_CMD_PCIE_TUNE_TXEQ_GET_OUT_PARAM_MINNUM 1 +#define MC_CMD_PCIE_TUNE_TXEQ_GET_OUT_PARAM_MAXNUM 63 +#define MC_CMD_PCIE_TUNE_TXEQ_GET_OUT_PARAM_ID_LBN 0 +#define MC_CMD_PCIE_TUNE_TXEQ_GET_OUT_PARAM_ID_WIDTH 8 +/* enum: TxMargin (PIPE) */ +#define MC_CMD_PCIE_TUNE_TXEQ_GET_OUT_TXMARGIN 0x0 +/* enum: TxSwing (PIPE) */ +#define MC_CMD_PCIE_TUNE_TXEQ_GET_OUT_TXSWING 0x1 +/* enum: De-emphasis coefficient C(-1) (PIPE) */ +#define MC_CMD_PCIE_TUNE_TXEQ_GET_OUT_CM1 0x2 +/* enum: De-emphasis coefficient C(0) (PIPE) */ +#define MC_CMD_PCIE_TUNE_TXEQ_GET_OUT_C0 0x3 +/* enum: De-emphasis coefficient C(+1) (PIPE) */ +#define MC_CMD_PCIE_TUNE_TXEQ_GET_OUT_CP1 0x4 +#define MC_CMD_PCIE_TUNE_TXEQ_GET_OUT_PARAM_LANE_LBN 8 +#define MC_CMD_PCIE_TUNE_TXEQ_GET_OUT_PARAM_LANE_WIDTH 4 +/* Enum values, see field(s): */ +/* MC_CMD_PCIE_TUNE_RXEQ_GET_OUT/PARAM_LANE */ +#define MC_CMD_PCIE_TUNE_TXEQ_GET_OUT_RESERVED_LBN 12 +#define MC_CMD_PCIE_TUNE_TXEQ_GET_OUT_RESERVED_WIDTH 12 +#define MC_CMD_PCIE_TUNE_TXEQ_GET_OUT_PARAM_CURRENT_LBN 24 +#define MC_CMD_PCIE_TUNE_TXEQ_GET_OUT_PARAM_CURRENT_WIDTH 8 + +/* MC_CMD_PCIE_TUNE_START_EYE_PLOT_IN msgrequest */ +#define MC_CMD_PCIE_TUNE_START_EYE_PLOT_IN_LEN 8 +/* Requested operation */ +#define MC_CMD_PCIE_TUNE_START_EYE_PLOT_IN_PCIE_TUNE_OP_OFST 0 +#define MC_CMD_PCIE_TUNE_START_EYE_PLOT_IN_PCIE_TUNE_OP_LEN 1 +/* Align the arguments to 32 bits */ +#define MC_CMD_PCIE_TUNE_START_EYE_PLOT_IN_PCIE_TUNE_RSVD_OFST 1 +#define MC_CMD_PCIE_TUNE_START_EYE_PLOT_IN_PCIE_TUNE_RSVD_LEN 3 +#define MC_CMD_PCIE_TUNE_START_EYE_PLOT_IN_LANE_OFST 4 + +/* MC_CMD_PCIE_TUNE_START_EYE_PLOT_OUT msgresponse */ +#define MC_CMD_PCIE_TUNE_START_EYE_PLOT_OUT_LEN 0 + +/* MC_CMD_PCIE_TUNE_POLL_EYE_PLOT_IN msgrequest */ +#define MC_CMD_PCIE_TUNE_POLL_EYE_PLOT_IN_LEN 4 +/* Requested operation */ +#define MC_CMD_PCIE_TUNE_POLL_EYE_PLOT_IN_PCIE_TUNE_OP_OFST 0 +#define MC_CMD_PCIE_TUNE_POLL_EYE_PLOT_IN_PCIE_TUNE_OP_LEN 1 +/* Align the arguments to 32 bits */ +#define MC_CMD_PCIE_TUNE_POLL_EYE_PLOT_IN_PCIE_TUNE_RSVD_OFST 1 +#define MC_CMD_PCIE_TUNE_POLL_EYE_PLOT_IN_PCIE_TUNE_RSVD_LEN 3 + +/* MC_CMD_PCIE_TUNE_POLL_EYE_PLOT_OUT msgresponse */ +#define MC_CMD_PCIE_TUNE_POLL_EYE_PLOT_OUT_LENMIN 0 +#define MC_CMD_PCIE_TUNE_POLL_EYE_PLOT_OUT_LENMAX 252 +#define MC_CMD_PCIE_TUNE_POLL_EYE_PLOT_OUT_LEN(num) (0+2*(num)) +#define MC_CMD_PCIE_TUNE_POLL_EYE_PLOT_OUT_SAMPLES_OFST 0 +#define MC_CMD_PCIE_TUNE_POLL_EYE_PLOT_OUT_SAMPLES_LEN 2 +#define MC_CMD_PCIE_TUNE_POLL_EYE_PLOT_OUT_SAMPLES_MINNUM 0 +#define MC_CMD_PCIE_TUNE_POLL_EYE_PLOT_OUT_SAMPLES_MAXNUM 126 + +/* MC_CMD_PCIE_TUNE_BIST_SQUARE_WAVE_IN msgrequest */ +#define MC_CMD_PCIE_TUNE_BIST_SQUARE_WAVE_IN_LEN 0 + +/* MC_CMD_PCIE_TUNE_BIST_SQUARE_WAVE_OUT msgrequest */ +#define MC_CMD_PCIE_TUNE_BIST_SQUARE_WAVE_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_LICENSING + * Operations on the NVRAM_PARTITION_TYPE_LICENSE application license partition + * - not used for V3 licensing + */ +#define MC_CMD_LICENSING 0xf3 +#undef MC_CMD_0xf3_PRIVILEGE_CTG + +#define MC_CMD_0xf3_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_LICENSING_IN msgrequest */ +#define MC_CMD_LICENSING_IN_LEN 4 +/* identifies the type of operation requested */ +#define MC_CMD_LICENSING_IN_OP_OFST 0 +/* enum: re-read and apply licenses after a license key partition update; note + * that this operation returns a zero-length response + */ +#define MC_CMD_LICENSING_IN_OP_UPDATE_LICENSE 0x0 +/* enum: report counts of installed licenses */ +#define MC_CMD_LICENSING_IN_OP_GET_KEY_STATS 0x1 + +/* MC_CMD_LICENSING_OUT msgresponse */ +#define MC_CMD_LICENSING_OUT_LEN 28 +/* count of application keys which are valid */ +#define MC_CMD_LICENSING_OUT_VALID_APP_KEYS_OFST 0 +/* sum of UNVERIFIABLE_APP_KEYS + WRONG_NODE_APP_KEYS (for compatibility with + * MC_CMD_FC_OP_LICENSE) + */ +#define MC_CMD_LICENSING_OUT_INVALID_APP_KEYS_OFST 4 +/* count of application keys which are invalid due to being blacklisted */ +#define MC_CMD_LICENSING_OUT_BLACKLISTED_APP_KEYS_OFST 8 +/* count of application keys which are invalid due to being unverifiable */ +#define MC_CMD_LICENSING_OUT_UNVERIFIABLE_APP_KEYS_OFST 12 +/* count of application keys which are invalid due to being for the wrong node + */ +#define MC_CMD_LICENSING_OUT_WRONG_NODE_APP_KEYS_OFST 16 +/* licensing state (for diagnostics; the exact meaning of the bits in this + * field are private to the firmware) + */ +#define MC_CMD_LICENSING_OUT_LICENSING_STATE_OFST 20 +/* licensing subsystem self-test report (for manftest) */ +#define MC_CMD_LICENSING_OUT_LICENSING_SELF_TEST_OFST 24 +/* enum: licensing subsystem self-test failed */ +#define MC_CMD_LICENSING_OUT_SELF_TEST_FAIL 0x0 +/* enum: licensing subsystem self-test passed */ +#define MC_CMD_LICENSING_OUT_SELF_TEST_PASS 0x1 + + +/***********************************/ +/* MC_CMD_LICENSING_V3 + * Operations on the NVRAM_PARTITION_TYPE_LICENSE application license partition + * - V3 licensing (Medford) + */ +#define MC_CMD_LICENSING_V3 0xd0 +#undef MC_CMD_0xd0_PRIVILEGE_CTG + +#define MC_CMD_0xd0_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_LICENSING_V3_IN msgrequest */ +#define MC_CMD_LICENSING_V3_IN_LEN 4 +/* identifies the type of operation requested */ +#define MC_CMD_LICENSING_V3_IN_OP_OFST 0 +/* enum: re-read and apply licenses after a license key partition update; note + * that this operation returns a zero-length response + */ +#define MC_CMD_LICENSING_V3_IN_OP_UPDATE_LICENSE 0x0 +/* enum: report counts of installed licenses Returns EAGAIN if license + * processing (updating) has been started but not yet completed. + */ +#define MC_CMD_LICENSING_V3_IN_OP_REPORT_LICENSE 0x1 + +/* MC_CMD_LICENSING_V3_OUT msgresponse */ +#define MC_CMD_LICENSING_V3_OUT_LEN 88 +/* count of keys which are valid */ +#define MC_CMD_LICENSING_V3_OUT_VALID_KEYS_OFST 0 +/* sum of UNVERIFIABLE_KEYS + WRONG_NODE_KEYS (for compatibility with + * MC_CMD_FC_OP_LICENSE) + */ +#define MC_CMD_LICENSING_V3_OUT_INVALID_KEYS_OFST 4 +/* count of keys which are invalid due to being unverifiable */ +#define MC_CMD_LICENSING_V3_OUT_UNVERIFIABLE_KEYS_OFST 8 +/* count of keys which are invalid due to being for the wrong node */ +#define MC_CMD_LICENSING_V3_OUT_WRONG_NODE_KEYS_OFST 12 +/* licensing state (for diagnostics; the exact meaning of the bits in this + * field are private to the firmware) + */ +#define MC_CMD_LICENSING_V3_OUT_LICENSING_STATE_OFST 16 +/* licensing subsystem self-test report (for manftest) */ +#define MC_CMD_LICENSING_V3_OUT_LICENSING_SELF_TEST_OFST 20 +/* enum: licensing subsystem self-test failed */ +#define MC_CMD_LICENSING_V3_OUT_SELF_TEST_FAIL 0x0 +/* enum: licensing subsystem self-test passed */ +#define MC_CMD_LICENSING_V3_OUT_SELF_TEST_PASS 0x1 +/* bitmask of licensed applications */ +#define MC_CMD_LICENSING_V3_OUT_LICENSED_APPS_OFST 24 +#define MC_CMD_LICENSING_V3_OUT_LICENSED_APPS_LEN 8 +#define MC_CMD_LICENSING_V3_OUT_LICENSED_APPS_LO_OFST 24 +#define MC_CMD_LICENSING_V3_OUT_LICENSED_APPS_HI_OFST 28 +/* reserved for future use */ +#define MC_CMD_LICENSING_V3_OUT_RESERVED_0_OFST 32 +#define MC_CMD_LICENSING_V3_OUT_RESERVED_0_LEN 24 +/* bitmask of licensed features */ +#define MC_CMD_LICENSING_V3_OUT_LICENSED_FEATURES_OFST 56 +#define MC_CMD_LICENSING_V3_OUT_LICENSED_FEATURES_LEN 8 +#define MC_CMD_LICENSING_V3_OUT_LICENSED_FEATURES_LO_OFST 56 +#define MC_CMD_LICENSING_V3_OUT_LICENSED_FEATURES_HI_OFST 60 +/* reserved for future use */ +#define MC_CMD_LICENSING_V3_OUT_RESERVED_1_OFST 64 +#define MC_CMD_LICENSING_V3_OUT_RESERVED_1_LEN 24 + + +/***********************************/ +/* MC_CMD_LICENSING_GET_ID_V3 + * Get ID and type from the NVRAM_PARTITION_TYPE_LICENSE application license + * partition - V3 licensing (Medford) + */ +#define MC_CMD_LICENSING_GET_ID_V3 0xd1 +#undef MC_CMD_0xd1_PRIVILEGE_CTG + +#define MC_CMD_0xd1_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_LICENSING_GET_ID_V3_IN msgrequest */ +#define MC_CMD_LICENSING_GET_ID_V3_IN_LEN 0 + +/* MC_CMD_LICENSING_GET_ID_V3_OUT msgresponse */ +#define MC_CMD_LICENSING_GET_ID_V3_OUT_LENMIN 8 +#define MC_CMD_LICENSING_GET_ID_V3_OUT_LENMAX 252 +#define MC_CMD_LICENSING_GET_ID_V3_OUT_LEN(num) (8+1*(num)) +/* type of license (eg 3) */ +#define MC_CMD_LICENSING_GET_ID_V3_OUT_LICENSE_TYPE_OFST 0 +/* length of the license ID (in bytes) */ +#define MC_CMD_LICENSING_GET_ID_V3_OUT_LICENSE_ID_LENGTH_OFST 4 +/* the unique license ID of the adapter */ +#define MC_CMD_LICENSING_GET_ID_V3_OUT_LICENSE_ID_OFST 8 +#define MC_CMD_LICENSING_GET_ID_V3_OUT_LICENSE_ID_LEN 1 +#define MC_CMD_LICENSING_GET_ID_V3_OUT_LICENSE_ID_MINNUM 0 +#define MC_CMD_LICENSING_GET_ID_V3_OUT_LICENSE_ID_MAXNUM 244 + + +/***********************************/ +/* MC_CMD_MC2MC_PROXY + * Execute an arbitrary MCDI command on the slave MC of a dual-core device. + * This will fail on a single-core system. + */ +#define MC_CMD_MC2MC_PROXY 0xf4 +#undef MC_CMD_0xf4_PRIVILEGE_CTG + +#define MC_CMD_0xf4_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_MC2MC_PROXY_IN msgrequest */ +#define MC_CMD_MC2MC_PROXY_IN_LEN 0 + +/* MC_CMD_MC2MC_PROXY_OUT msgresponse */ +#define MC_CMD_MC2MC_PROXY_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_GET_LICENSED_APP_STATE + * Query the state of an individual licensed application. (Note that the actual + * state may be invalidated by the MC_CMD_LICENSING OP_UPDATE_LICENSE operation + * or a reboot of the MC.) Not used for V3 licensing + */ +#define MC_CMD_GET_LICENSED_APP_STATE 0xf5 +#undef MC_CMD_0xf5_PRIVILEGE_CTG + +#define MC_CMD_0xf5_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_GET_LICENSED_APP_STATE_IN msgrequest */ +#define MC_CMD_GET_LICENSED_APP_STATE_IN_LEN 4 +/* application ID to query (LICENSED_APP_ID_xxx) */ +#define MC_CMD_GET_LICENSED_APP_STATE_IN_APP_ID_OFST 0 + +/* MC_CMD_GET_LICENSED_APP_STATE_OUT msgresponse */ +#define MC_CMD_GET_LICENSED_APP_STATE_OUT_LEN 4 +/* state of this application */ +#define MC_CMD_GET_LICENSED_APP_STATE_OUT_STATE_OFST 0 +/* enum: no (or invalid) license is present for the application */ +#define MC_CMD_GET_LICENSED_APP_STATE_OUT_NOT_LICENSED 0x0 +/* enum: a valid license is present for the application */ +#define MC_CMD_GET_LICENSED_APP_STATE_OUT_LICENSED 0x1 + + +/***********************************/ +/* MC_CMD_GET_LICENSED_V3_APP_STATE + * Query the state of an individual licensed application. (Note that the actual + * state may be invalidated by the MC_CMD_LICENSING_V3 OP_UPDATE_LICENSE + * operation or a reboot of the MC.) Used for V3 licensing (Medford) + */ +#define MC_CMD_GET_LICENSED_V3_APP_STATE 0xd2 +#undef MC_CMD_0xd2_PRIVILEGE_CTG + +#define MC_CMD_0xd2_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_GET_LICENSED_V3_APP_STATE_IN msgrequest */ +#define MC_CMD_GET_LICENSED_V3_APP_STATE_IN_LEN 8 +/* application ID to query (LICENSED_V3_APPS_xxx) expressed as a single bit + * mask + */ +#define MC_CMD_GET_LICENSED_V3_APP_STATE_IN_APP_ID_OFST 0 +#define MC_CMD_GET_LICENSED_V3_APP_STATE_IN_APP_ID_LEN 8 +#define MC_CMD_GET_LICENSED_V3_APP_STATE_IN_APP_ID_LO_OFST 0 +#define MC_CMD_GET_LICENSED_V3_APP_STATE_IN_APP_ID_HI_OFST 4 + +/* MC_CMD_GET_LICENSED_V3_APP_STATE_OUT msgresponse */ +#define MC_CMD_GET_LICENSED_V3_APP_STATE_OUT_LEN 4 +/* state of this application */ +#define MC_CMD_GET_LICENSED_V3_APP_STATE_OUT_STATE_OFST 0 +/* enum: no (or invalid) license is present for the application */ +#define MC_CMD_GET_LICENSED_V3_APP_STATE_OUT_NOT_LICENSED 0x0 +/* enum: a valid license is present for the application */ +#define MC_CMD_GET_LICENSED_V3_APP_STATE_OUT_LICENSED 0x1 + + +/***********************************/ +/* MC_CMD_GET_LICENSED_V3_FEATURE_STATES + * Query the state of an one or more licensed features. (Note that the actual + * state may be invalidated by the MC_CMD_LICENSING_V3 OP_UPDATE_LICENSE + * operation or a reboot of the MC.) Used for V3 licensing (Medford) + */ +#define MC_CMD_GET_LICENSED_V3_FEATURE_STATES 0xd3 +#undef MC_CMD_0xd3_PRIVILEGE_CTG + +#define MC_CMD_0xd3_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_GET_LICENSED_V3_FEATURE_STATES_IN msgrequest */ +#define MC_CMD_GET_LICENSED_V3_FEATURE_STATES_IN_LEN 8 +/* features to query (LICENSED_V3_FEATURES_xxx) expressed as a mask with one or + * more bits set + */ +#define MC_CMD_GET_LICENSED_V3_FEATURE_STATES_IN_FEATURES_OFST 0 +#define MC_CMD_GET_LICENSED_V3_FEATURE_STATES_IN_FEATURES_LEN 8 +#define MC_CMD_GET_LICENSED_V3_FEATURE_STATES_IN_FEATURES_LO_OFST 0 +#define MC_CMD_GET_LICENSED_V3_FEATURE_STATES_IN_FEATURES_HI_OFST 4 + +/* MC_CMD_GET_LICENSED_V3_FEATURE_STATES_OUT msgresponse */ +#define MC_CMD_GET_LICENSED_V3_FEATURE_STATES_OUT_LEN 8 +/* states of these features - bit set for licensed, clear for not licensed */ +#define MC_CMD_GET_LICENSED_V3_FEATURE_STATES_OUT_STATES_OFST 0 +#define MC_CMD_GET_LICENSED_V3_FEATURE_STATES_OUT_STATES_LEN 8 +#define MC_CMD_GET_LICENSED_V3_FEATURE_STATES_OUT_STATES_LO_OFST 0 +#define MC_CMD_GET_LICENSED_V3_FEATURE_STATES_OUT_STATES_HI_OFST 4 + + +/***********************************/ +/* MC_CMD_LICENSED_APP_OP + * Perform an action for an individual licensed application - not used for V3 + * licensing. + */ +#define MC_CMD_LICENSED_APP_OP 0xf6 +#undef MC_CMD_0xf6_PRIVILEGE_CTG + +#define MC_CMD_0xf6_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_LICENSED_APP_OP_IN msgrequest */ +#define MC_CMD_LICENSED_APP_OP_IN_LENMIN 8 +#define MC_CMD_LICENSED_APP_OP_IN_LENMAX 252 +#define MC_CMD_LICENSED_APP_OP_IN_LEN(num) (8+4*(num)) +/* application ID */ +#define MC_CMD_LICENSED_APP_OP_IN_APP_ID_OFST 0 +/* the type of operation requested */ +#define MC_CMD_LICENSED_APP_OP_IN_OP_OFST 4 +/* enum: validate application */ +#define MC_CMD_LICENSED_APP_OP_IN_OP_VALIDATE 0x0 +/* enum: mask application */ +#define MC_CMD_LICENSED_APP_OP_IN_OP_MASK 0x1 +/* arguments specific to this particular operation */ +#define MC_CMD_LICENSED_APP_OP_IN_ARGS_OFST 8 +#define MC_CMD_LICENSED_APP_OP_IN_ARGS_LEN 4 +#define MC_CMD_LICENSED_APP_OP_IN_ARGS_MINNUM 0 +#define MC_CMD_LICENSED_APP_OP_IN_ARGS_MAXNUM 61 + +/* MC_CMD_LICENSED_APP_OP_OUT msgresponse */ +#define MC_CMD_LICENSED_APP_OP_OUT_LENMIN 0 +#define MC_CMD_LICENSED_APP_OP_OUT_LENMAX 252 +#define MC_CMD_LICENSED_APP_OP_OUT_LEN(num) (0+4*(num)) +/* result specific to this particular operation */ +#define MC_CMD_LICENSED_APP_OP_OUT_RESULT_OFST 0 +#define MC_CMD_LICENSED_APP_OP_OUT_RESULT_LEN 4 +#define MC_CMD_LICENSED_APP_OP_OUT_RESULT_MINNUM 0 +#define MC_CMD_LICENSED_APP_OP_OUT_RESULT_MAXNUM 63 + +/* MC_CMD_LICENSED_APP_OP_VALIDATE_IN msgrequest */ +#define MC_CMD_LICENSED_APP_OP_VALIDATE_IN_LEN 72 +/* application ID */ +#define MC_CMD_LICENSED_APP_OP_VALIDATE_IN_APP_ID_OFST 0 +/* the type of operation requested */ +#define MC_CMD_LICENSED_APP_OP_VALIDATE_IN_OP_OFST 4 +/* validation challenge */ +#define MC_CMD_LICENSED_APP_OP_VALIDATE_IN_CHALLENGE_OFST 8 +#define MC_CMD_LICENSED_APP_OP_VALIDATE_IN_CHALLENGE_LEN 64 + +/* MC_CMD_LICENSED_APP_OP_VALIDATE_OUT msgresponse */ +#define MC_CMD_LICENSED_APP_OP_VALIDATE_OUT_LEN 68 +/* feature expiry (time_t) */ +#define MC_CMD_LICENSED_APP_OP_VALIDATE_OUT_EXPIRY_OFST 0 +/* validation response */ +#define MC_CMD_LICENSED_APP_OP_VALIDATE_OUT_RESPONSE_OFST 4 +#define MC_CMD_LICENSED_APP_OP_VALIDATE_OUT_RESPONSE_LEN 64 + +/* MC_CMD_LICENSED_APP_OP_MASK_IN msgrequest */ +#define MC_CMD_LICENSED_APP_OP_MASK_IN_LEN 12 +/* application ID */ +#define MC_CMD_LICENSED_APP_OP_MASK_IN_APP_ID_OFST 0 +/* the type of operation requested */ +#define MC_CMD_LICENSED_APP_OP_MASK_IN_OP_OFST 4 +/* flag */ +#define MC_CMD_LICENSED_APP_OP_MASK_IN_FLAG_OFST 8 + +/* MC_CMD_LICENSED_APP_OP_MASK_OUT msgresponse */ +#define MC_CMD_LICENSED_APP_OP_MASK_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_LICENSED_V3_VALIDATE_APP + * Perform validation for an individual licensed application - V3 licensing + * (Medford) + */ +#define MC_CMD_LICENSED_V3_VALIDATE_APP 0xd4 +#undef MC_CMD_0xd4_PRIVILEGE_CTG + +#define MC_CMD_0xd4_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_LICENSED_V3_VALIDATE_APP_IN msgrequest */ +#define MC_CMD_LICENSED_V3_VALIDATE_APP_IN_LEN 56 +/* challenge for validation (384 bits) */ +#define MC_CMD_LICENSED_V3_VALIDATE_APP_IN_CHALLENGE_OFST 0 +#define MC_CMD_LICENSED_V3_VALIDATE_APP_IN_CHALLENGE_LEN 48 +/* application ID expressed as a single bit mask */ +#define MC_CMD_LICENSED_V3_VALIDATE_APP_IN_APP_ID_OFST 48 +#define MC_CMD_LICENSED_V3_VALIDATE_APP_IN_APP_ID_LEN 8 +#define MC_CMD_LICENSED_V3_VALIDATE_APP_IN_APP_ID_LO_OFST 48 +#define MC_CMD_LICENSED_V3_VALIDATE_APP_IN_APP_ID_HI_OFST 52 + +/* MC_CMD_LICENSED_V3_VALIDATE_APP_OUT msgresponse */ +#define MC_CMD_LICENSED_V3_VALIDATE_APP_OUT_LEN 116 +/* validation response to challenge in the form of ECDSA signature consisting + * of two 384-bit integers, r and s, in big-endian order. The signature signs a + * SHA-384 digest of a message constructed from the concatenation of the input + * message and the remaining fields of this output message, e.g. challenge[48 + * bytes] ... expiry_time[4 bytes] ... + */ +#define MC_CMD_LICENSED_V3_VALIDATE_APP_OUT_RESPONSE_OFST 0 +#define MC_CMD_LICENSED_V3_VALIDATE_APP_OUT_RESPONSE_LEN 96 +/* application expiry time */ +#define MC_CMD_LICENSED_V3_VALIDATE_APP_OUT_EXPIRY_TIME_OFST 96 +/* application expiry units */ +#define MC_CMD_LICENSED_V3_VALIDATE_APP_OUT_EXPIRY_UNITS_OFST 100 +/* enum: expiry units are accounting units */ +#define MC_CMD_LICENSED_V3_VALIDATE_APP_OUT_EXPIRY_UNIT_ACC 0x0 +/* enum: expiry units are calendar days */ +#define MC_CMD_LICENSED_V3_VALIDATE_APP_OUT_EXPIRY_UNIT_DAYS 0x1 +/* base MAC address of the NIC stored in NVRAM (note that this is a constant + * value for a given NIC regardless which function is calling, effectively this + * is PF0 base MAC address) + */ +#define MC_CMD_LICENSED_V3_VALIDATE_APP_OUT_BASE_MACADDR_OFST 104 +#define MC_CMD_LICENSED_V3_VALIDATE_APP_OUT_BASE_MACADDR_LEN 6 +/* MAC address of v-adaptor associated with the client. If no such v-adapator + * exists, then the field is filled with 0xFF. + */ +#define MC_CMD_LICENSED_V3_VALIDATE_APP_OUT_VADAPTOR_MACADDR_OFST 110 +#define MC_CMD_LICENSED_V3_VALIDATE_APP_OUT_VADAPTOR_MACADDR_LEN 6 + + +/***********************************/ +/* MC_CMD_LICENSED_V3_MASK_FEATURES + * Mask features - V3 licensing (Medford) + */ +#define MC_CMD_LICENSED_V3_MASK_FEATURES 0xd5 +#undef MC_CMD_0xd5_PRIVILEGE_CTG + +#define MC_CMD_0xd5_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_LICENSED_V3_MASK_FEATURES_IN msgrequest */ +#define MC_CMD_LICENSED_V3_MASK_FEATURES_IN_LEN 12 +/* mask to be applied to features to be changed */ +#define MC_CMD_LICENSED_V3_MASK_FEATURES_IN_MASK_OFST 0 +#define MC_CMD_LICENSED_V3_MASK_FEATURES_IN_MASK_LEN 8 +#define MC_CMD_LICENSED_V3_MASK_FEATURES_IN_MASK_LO_OFST 0 +#define MC_CMD_LICENSED_V3_MASK_FEATURES_IN_MASK_HI_OFST 4 +/* whether to turn on or turn off the masked features */ +#define MC_CMD_LICENSED_V3_MASK_FEATURES_IN_FLAG_OFST 8 +/* enum: turn the features off */ +#define MC_CMD_LICENSED_V3_MASK_FEATURES_IN_OFF 0x0 +/* enum: turn the features back on */ +#define MC_CMD_LICENSED_V3_MASK_FEATURES_IN_ON 0x1 + +/* MC_CMD_LICENSED_V3_MASK_FEATURES_OUT msgresponse */ +#define MC_CMD_LICENSED_V3_MASK_FEATURES_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_LICENSING_V3_TEMPORARY + * Perform operations to support installation of a single temporary license in + * the adapter, in addition to those found in the licensing partition. See + * SF-116124-SW for an overview of how this could be used. The license is + * stored in MC persistent data and so will survive a MC reboot, but will be + * erased when the adapter is power cycled + */ +#define MC_CMD_LICENSING_V3_TEMPORARY 0xd6 +#undef MC_CMD_0xd6_PRIVILEGE_CTG + +#define MC_CMD_0xd6_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_LICENSING_V3_TEMPORARY_IN msgrequest */ +#define MC_CMD_LICENSING_V3_TEMPORARY_IN_LEN 4 +/* operation code */ +#define MC_CMD_LICENSING_V3_TEMPORARY_IN_OP_OFST 0 +/* enum: install a new license, overwriting any existing temporary license. + * This is an asynchronous operation owing to the time taken to validate an + * ECDSA license + */ +#define MC_CMD_LICENSING_V3_TEMPORARY_SET 0x0 +/* enum: clear the license immediately rather than waiting for the next power + * cycle + */ +#define MC_CMD_LICENSING_V3_TEMPORARY_CLEAR 0x1 +/* enum: get the status of the asynchronous MC_CMD_LICENSING_V3_TEMPORARY_SET + * operation + */ +#define MC_CMD_LICENSING_V3_TEMPORARY_STATUS 0x2 + +/* MC_CMD_LICENSING_V3_TEMPORARY_IN_SET msgrequest */ +#define MC_CMD_LICENSING_V3_TEMPORARY_IN_SET_LEN 164 +#define MC_CMD_LICENSING_V3_TEMPORARY_IN_SET_OP_OFST 0 +/* ECDSA license and signature */ +#define MC_CMD_LICENSING_V3_TEMPORARY_IN_SET_LICENSE_OFST 4 +#define MC_CMD_LICENSING_V3_TEMPORARY_IN_SET_LICENSE_LEN 160 + +/* MC_CMD_LICENSING_V3_TEMPORARY_IN_CLEAR msgrequest */ +#define MC_CMD_LICENSING_V3_TEMPORARY_IN_CLEAR_LEN 4 +#define MC_CMD_LICENSING_V3_TEMPORARY_IN_CLEAR_OP_OFST 0 + +/* MC_CMD_LICENSING_V3_TEMPORARY_IN_STATUS msgrequest */ +#define MC_CMD_LICENSING_V3_TEMPORARY_IN_STATUS_LEN 4 +#define MC_CMD_LICENSING_V3_TEMPORARY_IN_STATUS_OP_OFST 0 + +/* MC_CMD_LICENSING_V3_TEMPORARY_OUT_STATUS msgresponse */ +#define MC_CMD_LICENSING_V3_TEMPORARY_OUT_STATUS_LEN 12 +/* status code */ +#define MC_CMD_LICENSING_V3_TEMPORARY_OUT_STATUS_STATUS_OFST 0 +/* enum: finished validating and installing license */ +#define MC_CMD_LICENSING_V3_TEMPORARY_STATUS_OK 0x0 +/* enum: license validation and installation in progress */ +#define MC_CMD_LICENSING_V3_TEMPORARY_STATUS_IN_PROGRESS 0x1 +/* enum: licensing error. More specific error messages are not provided to + * avoid exposing details of the licensing system to the client + */ +#define MC_CMD_LICENSING_V3_TEMPORARY_STATUS_ERROR 0x2 +/* bitmask of licensed features */ +#define MC_CMD_LICENSING_V3_TEMPORARY_OUT_STATUS_LICENSED_FEATURES_OFST 4 +#define MC_CMD_LICENSING_V3_TEMPORARY_OUT_STATUS_LICENSED_FEATURES_LEN 8 +#define MC_CMD_LICENSING_V3_TEMPORARY_OUT_STATUS_LICENSED_FEATURES_LO_OFST 4 +#define MC_CMD_LICENSING_V3_TEMPORARY_OUT_STATUS_LICENSED_FEATURES_HI_OFST 8 + + +/***********************************/ +/* MC_CMD_SET_PORT_SNIFF_CONFIG + * Configure RX port sniffing for the physical port associated with the calling + * function. Only a privileged function may change the port sniffing + * configuration. A copy of all traffic delivered to the host (non-promiscuous + * mode) or all traffic arriving at the port (promiscuous mode) may be + * delivered to a specific queue, or a set of queues with RSS. + */ +#define MC_CMD_SET_PORT_SNIFF_CONFIG 0xf7 +#undef MC_CMD_0xf7_PRIVILEGE_CTG + +#define MC_CMD_0xf7_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_SET_PORT_SNIFF_CONFIG_IN msgrequest */ +#define MC_CMD_SET_PORT_SNIFF_CONFIG_IN_LEN 16 +/* configuration flags */ +#define MC_CMD_SET_PORT_SNIFF_CONFIG_IN_FLAGS_OFST 0 +#define MC_CMD_SET_PORT_SNIFF_CONFIG_IN_ENABLE_LBN 0 +#define MC_CMD_SET_PORT_SNIFF_CONFIG_IN_ENABLE_WIDTH 1 +#define MC_CMD_SET_PORT_SNIFF_CONFIG_IN_PROMISCUOUS_LBN 1 +#define MC_CMD_SET_PORT_SNIFF_CONFIG_IN_PROMISCUOUS_WIDTH 1 +/* receive queue handle (for RSS mode, this is the base queue) */ +#define MC_CMD_SET_PORT_SNIFF_CONFIG_IN_RX_QUEUE_OFST 4 +/* receive mode */ +#define MC_CMD_SET_PORT_SNIFF_CONFIG_IN_RX_MODE_OFST 8 +/* enum: receive to just the specified queue */ +#define MC_CMD_SET_PORT_SNIFF_CONFIG_IN_RX_MODE_SIMPLE 0x0 +/* enum: receive to multiple queues using RSS context */ +#define MC_CMD_SET_PORT_SNIFF_CONFIG_IN_RX_MODE_RSS 0x1 +/* RSS context (for RX_MODE_RSS) as returned by MC_CMD_RSS_CONTEXT_ALLOC. Note + * that these handles should be considered opaque to the host, although a value + * of 0xFFFFFFFF is guaranteed never to be a valid handle. + */ +#define MC_CMD_SET_PORT_SNIFF_CONFIG_IN_RX_CONTEXT_OFST 12 + +/* MC_CMD_SET_PORT_SNIFF_CONFIG_OUT msgresponse */ +#define MC_CMD_SET_PORT_SNIFF_CONFIG_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_GET_PORT_SNIFF_CONFIG + * Obtain the current RX port sniffing configuration for the physical port + * associated with the calling function. Only a privileged function may read + * the configuration. + */ +#define MC_CMD_GET_PORT_SNIFF_CONFIG 0xf8 +#undef MC_CMD_0xf8_PRIVILEGE_CTG + +#define MC_CMD_0xf8_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_GET_PORT_SNIFF_CONFIG_IN msgrequest */ +#define MC_CMD_GET_PORT_SNIFF_CONFIG_IN_LEN 0 + +/* MC_CMD_GET_PORT_SNIFF_CONFIG_OUT msgresponse */ +#define MC_CMD_GET_PORT_SNIFF_CONFIG_OUT_LEN 16 +/* configuration flags */ +#define MC_CMD_GET_PORT_SNIFF_CONFIG_OUT_FLAGS_OFST 0 +#define MC_CMD_GET_PORT_SNIFF_CONFIG_OUT_ENABLE_LBN 0 +#define MC_CMD_GET_PORT_SNIFF_CONFIG_OUT_ENABLE_WIDTH 1 +#define MC_CMD_GET_PORT_SNIFF_CONFIG_OUT_PROMISCUOUS_LBN 1 +#define MC_CMD_GET_PORT_SNIFF_CONFIG_OUT_PROMISCUOUS_WIDTH 1 +/* receiving queue handle (for RSS mode, this is the base queue) */ +#define MC_CMD_GET_PORT_SNIFF_CONFIG_OUT_RX_QUEUE_OFST 4 +/* receive mode */ +#define MC_CMD_GET_PORT_SNIFF_CONFIG_OUT_RX_MODE_OFST 8 +/* enum: receiving to just the specified queue */ +#define MC_CMD_GET_PORT_SNIFF_CONFIG_OUT_RX_MODE_SIMPLE 0x0 +/* enum: receiving to multiple queues using RSS context */ +#define MC_CMD_GET_PORT_SNIFF_CONFIG_OUT_RX_MODE_RSS 0x1 +/* RSS context (for RX_MODE_RSS) */ +#define MC_CMD_GET_PORT_SNIFF_CONFIG_OUT_RX_CONTEXT_OFST 12 + + +/***********************************/ +/* MC_CMD_SET_PARSER_DISP_CONFIG + * Change configuration related to the parser-dispatcher subsystem. + */ +#define MC_CMD_SET_PARSER_DISP_CONFIG 0xf9 +#undef MC_CMD_0xf9_PRIVILEGE_CTG + +#define MC_CMD_0xf9_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_SET_PARSER_DISP_CONFIG_IN msgrequest */ +#define MC_CMD_SET_PARSER_DISP_CONFIG_IN_LENMIN 12 +#define MC_CMD_SET_PARSER_DISP_CONFIG_IN_LENMAX 252 +#define MC_CMD_SET_PARSER_DISP_CONFIG_IN_LEN(num) (8+4*(num)) +/* the type of configuration setting to change */ +#define MC_CMD_SET_PARSER_DISP_CONFIG_IN_TYPE_OFST 0 +/* enum: Per-TXQ enable for multicast UDP destination lookup for possible + * internal loopback. (ENTITY is a queue handle, VALUE is a single boolean.) + */ +#define MC_CMD_SET_PARSER_DISP_CONFIG_IN_TXQ_MCAST_UDP_DST_LOOKUP_EN 0x0 +/* enum: Per-v-adaptor enable for suppression of self-transmissions on the + * internal loopback path. (ENTITY is an EVB_PORT_ID, VALUE is a single + * boolean.) + */ +#define MC_CMD_SET_PARSER_DISP_CONFIG_IN_VADAPTOR_SUPPRESS_SELF_TX 0x1 +/* handle for the entity to update: queue handle, EVB port ID, etc. depending + * on the type of configuration setting being changed + */ +#define MC_CMD_SET_PARSER_DISP_CONFIG_IN_ENTITY_OFST 4 +/* new value: the details depend on the type of configuration setting being + * changed + */ +#define MC_CMD_SET_PARSER_DISP_CONFIG_IN_VALUE_OFST 8 +#define MC_CMD_SET_PARSER_DISP_CONFIG_IN_VALUE_LEN 4 +#define MC_CMD_SET_PARSER_DISP_CONFIG_IN_VALUE_MINNUM 1 +#define MC_CMD_SET_PARSER_DISP_CONFIG_IN_VALUE_MAXNUM 61 + +/* MC_CMD_SET_PARSER_DISP_CONFIG_OUT msgresponse */ +#define MC_CMD_SET_PARSER_DISP_CONFIG_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_GET_PARSER_DISP_CONFIG + * Read configuration related to the parser-dispatcher subsystem. + */ +#define MC_CMD_GET_PARSER_DISP_CONFIG 0xfa +#undef MC_CMD_0xfa_PRIVILEGE_CTG + +#define MC_CMD_0xfa_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_GET_PARSER_DISP_CONFIG_IN msgrequest */ +#define MC_CMD_GET_PARSER_DISP_CONFIG_IN_LEN 8 +/* the type of configuration setting to read */ +#define MC_CMD_GET_PARSER_DISP_CONFIG_IN_TYPE_OFST 0 +/* Enum values, see field(s): */ +/* MC_CMD_SET_PARSER_DISP_CONFIG/MC_CMD_SET_PARSER_DISP_CONFIG_IN/TYPE */ +/* handle for the entity to query: queue handle, EVB port ID, etc. depending on + * the type of configuration setting being read + */ +#define MC_CMD_GET_PARSER_DISP_CONFIG_IN_ENTITY_OFST 4 + +/* MC_CMD_GET_PARSER_DISP_CONFIG_OUT msgresponse */ +#define MC_CMD_GET_PARSER_DISP_CONFIG_OUT_LENMIN 4 +#define MC_CMD_GET_PARSER_DISP_CONFIG_OUT_LENMAX 252 +#define MC_CMD_GET_PARSER_DISP_CONFIG_OUT_LEN(num) (0+4*(num)) +/* current value: the details depend on the type of configuration setting being + * read + */ +#define MC_CMD_GET_PARSER_DISP_CONFIG_OUT_VALUE_OFST 0 +#define MC_CMD_GET_PARSER_DISP_CONFIG_OUT_VALUE_LEN 4 +#define MC_CMD_GET_PARSER_DISP_CONFIG_OUT_VALUE_MINNUM 1 +#define MC_CMD_GET_PARSER_DISP_CONFIG_OUT_VALUE_MAXNUM 63 + + +/***********************************/ +/* MC_CMD_SET_TX_PORT_SNIFF_CONFIG + * Configure TX port sniffing for the physical port associated with the calling + * function. Only a privileged function may change the port sniffing + * configuration. A copy of all traffic transmitted through the port may be + * delivered to a specific queue, or a set of queues with RSS. Note that these + * packets are delivered with transmit timestamps in the packet prefix, not + * receive timestamps, so it is likely that the queue(s) will need to be + * dedicated as TX sniff receivers. + */ +#define MC_CMD_SET_TX_PORT_SNIFF_CONFIG 0xfb +#undef MC_CMD_0xfb_PRIVILEGE_CTG + +#define MC_CMD_0xfb_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_SET_TX_PORT_SNIFF_CONFIG_IN msgrequest */ +#define MC_CMD_SET_TX_PORT_SNIFF_CONFIG_IN_LEN 16 +/* configuration flags */ +#define MC_CMD_SET_TX_PORT_SNIFF_CONFIG_IN_FLAGS_OFST 0 +#define MC_CMD_SET_TX_PORT_SNIFF_CONFIG_IN_ENABLE_LBN 0 +#define MC_CMD_SET_TX_PORT_SNIFF_CONFIG_IN_ENABLE_WIDTH 1 +/* receive queue handle (for RSS mode, this is the base queue) */ +#define MC_CMD_SET_TX_PORT_SNIFF_CONFIG_IN_RX_QUEUE_OFST 4 +/* receive mode */ +#define MC_CMD_SET_TX_PORT_SNIFF_CONFIG_IN_RX_MODE_OFST 8 +/* enum: receive to just the specified queue */ +#define MC_CMD_SET_TX_PORT_SNIFF_CONFIG_IN_RX_MODE_SIMPLE 0x0 +/* enum: receive to multiple queues using RSS context */ +#define MC_CMD_SET_TX_PORT_SNIFF_CONFIG_IN_RX_MODE_RSS 0x1 +/* RSS context (for RX_MODE_RSS) as returned by MC_CMD_RSS_CONTEXT_ALLOC. Note + * that these handles should be considered opaque to the host, although a value + * of 0xFFFFFFFF is guaranteed never to be a valid handle. + */ +#define MC_CMD_SET_TX_PORT_SNIFF_CONFIG_IN_RX_CONTEXT_OFST 12 + +/* MC_CMD_SET_TX_PORT_SNIFF_CONFIG_OUT msgresponse */ +#define MC_CMD_SET_TX_PORT_SNIFF_CONFIG_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_GET_TX_PORT_SNIFF_CONFIG + * Obtain the current TX port sniffing configuration for the physical port + * associated with the calling function. Only a privileged function may read + * the configuration. + */ +#define MC_CMD_GET_TX_PORT_SNIFF_CONFIG 0xfc +#undef MC_CMD_0xfc_PRIVILEGE_CTG + +#define MC_CMD_0xfc_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_GET_TX_PORT_SNIFF_CONFIG_IN msgrequest */ +#define MC_CMD_GET_TX_PORT_SNIFF_CONFIG_IN_LEN 0 + +/* MC_CMD_GET_TX_PORT_SNIFF_CONFIG_OUT msgresponse */ +#define MC_CMD_GET_TX_PORT_SNIFF_CONFIG_OUT_LEN 16 +/* configuration flags */ +#define MC_CMD_GET_TX_PORT_SNIFF_CONFIG_OUT_FLAGS_OFST 0 +#define MC_CMD_GET_TX_PORT_SNIFF_CONFIG_OUT_ENABLE_LBN 0 +#define MC_CMD_GET_TX_PORT_SNIFF_CONFIG_OUT_ENABLE_WIDTH 1 +/* receiving queue handle (for RSS mode, this is the base queue) */ +#define MC_CMD_GET_TX_PORT_SNIFF_CONFIG_OUT_RX_QUEUE_OFST 4 +/* receive mode */ +#define MC_CMD_GET_TX_PORT_SNIFF_CONFIG_OUT_RX_MODE_OFST 8 +/* enum: receiving to just the specified queue */ +#define MC_CMD_GET_TX_PORT_SNIFF_CONFIG_OUT_RX_MODE_SIMPLE 0x0 +/* enum: receiving to multiple queues using RSS context */ +#define MC_CMD_GET_TX_PORT_SNIFF_CONFIG_OUT_RX_MODE_RSS 0x1 +/* RSS context (for RX_MODE_RSS) */ +#define MC_CMD_GET_TX_PORT_SNIFF_CONFIG_OUT_RX_CONTEXT_OFST 12 + + +/***********************************/ +/* MC_CMD_RMON_STATS_RX_ERRORS + * Per queue rx error stats. + */ +#define MC_CMD_RMON_STATS_RX_ERRORS 0xfe +#undef MC_CMD_0xfe_PRIVILEGE_CTG + +#define MC_CMD_0xfe_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_RMON_STATS_RX_ERRORS_IN msgrequest */ +#define MC_CMD_RMON_STATS_RX_ERRORS_IN_LEN 8 +/* The rx queue to get stats for. */ +#define MC_CMD_RMON_STATS_RX_ERRORS_IN_RX_QUEUE_OFST 0 +#define MC_CMD_RMON_STATS_RX_ERRORS_IN_FLAGS_OFST 4 +#define MC_CMD_RMON_STATS_RX_ERRORS_IN_RST_LBN 0 +#define MC_CMD_RMON_STATS_RX_ERRORS_IN_RST_WIDTH 1 + +/* MC_CMD_RMON_STATS_RX_ERRORS_OUT msgresponse */ +#define MC_CMD_RMON_STATS_RX_ERRORS_OUT_LEN 16 +#define MC_CMD_RMON_STATS_RX_ERRORS_OUT_CRC_ERRORS_OFST 0 +#define MC_CMD_RMON_STATS_RX_ERRORS_OUT_TRUNC_ERRORS_OFST 4 +#define MC_CMD_RMON_STATS_RX_ERRORS_OUT_RX_NO_DESC_DROPS_OFST 8 +#define MC_CMD_RMON_STATS_RX_ERRORS_OUT_RX_ABORT_OFST 12 + + +/***********************************/ +/* MC_CMD_GET_PCIE_RESOURCE_INFO + * Find out about available PCIE resources + */ +#define MC_CMD_GET_PCIE_RESOURCE_INFO 0xfd + +/* MC_CMD_GET_PCIE_RESOURCE_INFO_IN msgrequest */ +#define MC_CMD_GET_PCIE_RESOURCE_INFO_IN_LEN 0 + +/* MC_CMD_GET_PCIE_RESOURCE_INFO_OUT msgresponse */ +#define MC_CMD_GET_PCIE_RESOURCE_INFO_OUT_LEN 28 +/* The maximum number of PFs the device can expose */ +#define MC_CMD_GET_PCIE_RESOURCE_INFO_OUT_MAX_PFS_OFST 0 +/* The maximum number of VFs the device can expose in total */ +#define MC_CMD_GET_PCIE_RESOURCE_INFO_OUT_MAX_VFS_OFST 4 +/* The maximum number of MSI-X vectors the device can provide in total */ +#define MC_CMD_GET_PCIE_RESOURCE_INFO_OUT_MAX_VECTORS_OFST 8 +/* the number of MSI-X vectors the device will allocate by default to each PF + */ +#define MC_CMD_GET_PCIE_RESOURCE_INFO_OUT_DEFAULT_PF_VECTORS_OFST 12 +/* the number of MSI-X vectors the device will allocate by default to each VF + */ +#define MC_CMD_GET_PCIE_RESOURCE_INFO_OUT_DEFAULT_VF_VECTORS_OFST 16 +/* the maximum number of MSI-X vectors the device can allocate to any one PF */ +#define MC_CMD_GET_PCIE_RESOURCE_INFO_OUT_MAX_PF_VECTORS_OFST 20 +/* the maximum number of MSI-X vectors the device can allocate to any one VF */ +#define MC_CMD_GET_PCIE_RESOURCE_INFO_OUT_MAX_VF_VECTORS_OFST 24 + + +/***********************************/ +/* MC_CMD_GET_PORT_MODES + * Find out about available port modes + */ +#define MC_CMD_GET_PORT_MODES 0xff +#undef MC_CMD_0xff_PRIVILEGE_CTG + +#define MC_CMD_0xff_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_GET_PORT_MODES_IN msgrequest */ +#define MC_CMD_GET_PORT_MODES_IN_LEN 0 + +/* MC_CMD_GET_PORT_MODES_OUT msgresponse */ +#define MC_CMD_GET_PORT_MODES_OUT_LEN 12 +/* Bitmask of port modes available on the board (indexed by TLV_PORT_MODE_*) */ +#define MC_CMD_GET_PORT_MODES_OUT_MODES_OFST 0 +/* Default (canonical) board mode */ +#define MC_CMD_GET_PORT_MODES_OUT_DEFAULT_MODE_OFST 4 +/* Current board mode */ +#define MC_CMD_GET_PORT_MODES_OUT_CURRENT_MODE_OFST 8 + + +/***********************************/ +/* MC_CMD_READ_ATB + * Sample voltages on the ATB + */ +#define MC_CMD_READ_ATB 0x100 +#undef MC_CMD_0x100_PRIVILEGE_CTG + +#define MC_CMD_0x100_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_READ_ATB_IN msgrequest */ +#define MC_CMD_READ_ATB_IN_LEN 16 +#define MC_CMD_READ_ATB_IN_SIGNAL_BUS_OFST 0 +#define MC_CMD_READ_ATB_IN_BUS_CCOM 0x0 /* enum */ +#define MC_CMD_READ_ATB_IN_BUS_CKR 0x1 /* enum */ +#define MC_CMD_READ_ATB_IN_BUS_CPCIE 0x8 /* enum */ +#define MC_CMD_READ_ATB_IN_SIGNAL_EN_BITNO_OFST 4 +#define MC_CMD_READ_ATB_IN_SIGNAL_SEL_OFST 8 +#define MC_CMD_READ_ATB_IN_SETTLING_TIME_US_OFST 12 + +/* MC_CMD_READ_ATB_OUT msgresponse */ +#define MC_CMD_READ_ATB_OUT_LEN 4 +#define MC_CMD_READ_ATB_OUT_SAMPLE_MV_OFST 0 + + +/***********************************/ +/* MC_CMD_GET_WORKAROUNDS + * Read the list of all implemented and all currently enabled workarounds. The + * enums here must correspond with those in MC_CMD_WORKAROUND. + */ +#define MC_CMD_GET_WORKAROUNDS 0x59 +#undef MC_CMD_0x59_PRIVILEGE_CTG + +#define MC_CMD_0x59_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_GET_WORKAROUNDS_OUT msgresponse */ +#define MC_CMD_GET_WORKAROUNDS_OUT_LEN 8 +/* Each workaround is represented by a single bit according to the enums below. + */ +#define MC_CMD_GET_WORKAROUNDS_OUT_IMPLEMENTED_OFST 0 +#define MC_CMD_GET_WORKAROUNDS_OUT_ENABLED_OFST 4 +/* enum: Bug 17230 work around. */ +#define MC_CMD_GET_WORKAROUNDS_OUT_BUG17230 0x2 +/* enum: Bug 35388 work around (unsafe EVQ writes). */ +#define MC_CMD_GET_WORKAROUNDS_OUT_BUG35388 0x4 +/* enum: Bug35017 workaround (A64 tables must be identity map) */ +#define MC_CMD_GET_WORKAROUNDS_OUT_BUG35017 0x8 +/* enum: Bug 41750 present (MC_CMD_TRIGGER_INTERRUPT won't work) */ +#define MC_CMD_GET_WORKAROUNDS_OUT_BUG41750 0x10 +/* enum: Bug 42008 present (Interrupts can overtake associated events). Caution + * - before adding code that queries this workaround, remember that there's + * released Monza firmware that doesn't understand MC_CMD_WORKAROUND_BUG42008, + * and will hence (incorrectly) report that the bug doesn't exist. + */ +#define MC_CMD_GET_WORKAROUNDS_OUT_BUG42008 0x20 +/* enum: Bug 26807 features present in firmware (multicast filter chaining) */ +#define MC_CMD_GET_WORKAROUNDS_OUT_BUG26807 0x40 +/* enum: Bug 61265 work around (broken EVQ TMR writes). */ +#define MC_CMD_GET_WORKAROUNDS_OUT_BUG61265 0x80 + + +/***********************************/ +/* MC_CMD_PRIVILEGE_MASK + * Read/set privileges of an arbitrary PCIe function + */ +#define MC_CMD_PRIVILEGE_MASK 0x5a +#undef MC_CMD_0x5a_PRIVILEGE_CTG + +#define MC_CMD_0x5a_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_PRIVILEGE_MASK_IN msgrequest */ +#define MC_CMD_PRIVILEGE_MASK_IN_LEN 8 +/* The target function to have its mask read or set e.g. PF 0 = 0xFFFF0000, VF + * 1,3 = 0x00030001 + */ +#define MC_CMD_PRIVILEGE_MASK_IN_FUNCTION_OFST 0 +#define MC_CMD_PRIVILEGE_MASK_IN_FUNCTION_PF_LBN 0 +#define MC_CMD_PRIVILEGE_MASK_IN_FUNCTION_PF_WIDTH 16 +#define MC_CMD_PRIVILEGE_MASK_IN_FUNCTION_VF_LBN 16 +#define MC_CMD_PRIVILEGE_MASK_IN_FUNCTION_VF_WIDTH 16 +#define MC_CMD_PRIVILEGE_MASK_IN_VF_NULL 0xffff /* enum */ +/* New privilege mask to be set. The mask will only be changed if the MSB is + * set to 1. + */ +#define MC_CMD_PRIVILEGE_MASK_IN_NEW_MASK_OFST 4 +#define MC_CMD_PRIVILEGE_MASK_IN_GRP_ADMIN 0x1 /* enum */ +#define MC_CMD_PRIVILEGE_MASK_IN_GRP_LINK 0x2 /* enum */ +#define MC_CMD_PRIVILEGE_MASK_IN_GRP_ONLOAD 0x4 /* enum */ +#define MC_CMD_PRIVILEGE_MASK_IN_GRP_PTP 0x8 /* enum */ +#define MC_CMD_PRIVILEGE_MASK_IN_GRP_INSECURE_FILTERS 0x10 /* enum */ +/* enum: Deprecated. Equivalent to MAC_SPOOFING_TX combined with CHANGE_MAC. */ +#define MC_CMD_PRIVILEGE_MASK_IN_GRP_MAC_SPOOFING 0x20 +#define MC_CMD_PRIVILEGE_MASK_IN_GRP_UNICAST 0x40 /* enum */ +#define MC_CMD_PRIVILEGE_MASK_IN_GRP_MULTICAST 0x80 /* enum */ +#define MC_CMD_PRIVILEGE_MASK_IN_GRP_BROADCAST 0x100 /* enum */ +#define MC_CMD_PRIVILEGE_MASK_IN_GRP_ALL_MULTICAST 0x200 /* enum */ +#define MC_CMD_PRIVILEGE_MASK_IN_GRP_PROMISCUOUS 0x400 /* enum */ +/* enum: Allows to set the TX packets' source MAC address to any arbitrary MAC + * adress. + */ +#define MC_CMD_PRIVILEGE_MASK_IN_GRP_MAC_SPOOFING_TX 0x800 +/* enum: Privilege that allows a Function to change the MAC address configured + * in its associated vAdapter/vPort. + */ +#define MC_CMD_PRIVILEGE_MASK_IN_GRP_CHANGE_MAC 0x1000 +/* enum: Privilege that allows a Function to install filters that specify VLANs + * that are not in the permit list for the associated vPort. This privilege is + * primarily to support ESX where vPorts are created that restrict traffic to + * only a set of permitted VLANs. See the vPort flag FLAG_VLAN_RESTRICT. + */ +#define MC_CMD_PRIVILEGE_MASK_IN_GRP_UNRESTRICTED_VLAN 0x2000 +/* enum: Set this bit to indicate that a new privilege mask is to be set, + * otherwise the command will only read the existing mask. + */ +#define MC_CMD_PRIVILEGE_MASK_IN_DO_CHANGE 0x80000000 + +/* MC_CMD_PRIVILEGE_MASK_OUT msgresponse */ +#define MC_CMD_PRIVILEGE_MASK_OUT_LEN 4 +/* For an admin function, always all the privileges are reported. */ +#define MC_CMD_PRIVILEGE_MASK_OUT_OLD_MASK_OFST 0 + + +/***********************************/ +/* MC_CMD_LINK_STATE_MODE + * Read/set link state mode of a VF + */ +#define MC_CMD_LINK_STATE_MODE 0x5c +#undef MC_CMD_0x5c_PRIVILEGE_CTG + +#define MC_CMD_0x5c_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_LINK_STATE_MODE_IN msgrequest */ +#define MC_CMD_LINK_STATE_MODE_IN_LEN 8 +/* The target function to have its link state mode read or set, must be a VF + * e.g. VF 1,3 = 0x00030001 + */ +#define MC_CMD_LINK_STATE_MODE_IN_FUNCTION_OFST 0 +#define MC_CMD_LINK_STATE_MODE_IN_FUNCTION_PF_LBN 0 +#define MC_CMD_LINK_STATE_MODE_IN_FUNCTION_PF_WIDTH 16 +#define MC_CMD_LINK_STATE_MODE_IN_FUNCTION_VF_LBN 16 +#define MC_CMD_LINK_STATE_MODE_IN_FUNCTION_VF_WIDTH 16 +/* New link state mode to be set */ +#define MC_CMD_LINK_STATE_MODE_IN_NEW_MODE_OFST 4 +#define MC_CMD_LINK_STATE_MODE_IN_LINK_STATE_AUTO 0x0 /* enum */ +#define MC_CMD_LINK_STATE_MODE_IN_LINK_STATE_UP 0x1 /* enum */ +#define MC_CMD_LINK_STATE_MODE_IN_LINK_STATE_DOWN 0x2 /* enum */ +/* enum: Use this value to just read the existing setting without modifying it. + */ +#define MC_CMD_LINK_STATE_MODE_IN_DO_NOT_CHANGE 0xffffffff + +/* MC_CMD_LINK_STATE_MODE_OUT msgresponse */ +#define MC_CMD_LINK_STATE_MODE_OUT_LEN 4 +#define MC_CMD_LINK_STATE_MODE_OUT_OLD_MODE_OFST 0 + + +/***********************************/ +/* MC_CMD_GET_SNAPSHOT_LENGTH + * Obtain the curent range of allowable values for the SNAPSHOT_LENGTH + * parameter to MC_CMD_INIT_RXQ. + */ +#define MC_CMD_GET_SNAPSHOT_LENGTH 0x101 +#undef MC_CMD_0x101_PRIVILEGE_CTG + +#define MC_CMD_0x101_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_GET_SNAPSHOT_LENGTH_IN msgrequest */ +#define MC_CMD_GET_SNAPSHOT_LENGTH_IN_LEN 0 + +/* MC_CMD_GET_SNAPSHOT_LENGTH_OUT msgresponse */ +#define MC_CMD_GET_SNAPSHOT_LENGTH_OUT_LEN 8 +/* Minimum acceptable snapshot length. */ +#define MC_CMD_GET_SNAPSHOT_LENGTH_OUT_RX_SNAPLEN_MIN_OFST 0 +/* Maximum acceptable snapshot length. */ +#define MC_CMD_GET_SNAPSHOT_LENGTH_OUT_RX_SNAPLEN_MAX_OFST 4 + + +/***********************************/ +/* MC_CMD_FUSE_DIAGS + * Additional fuse diagnostics + */ +#define MC_CMD_FUSE_DIAGS 0x102 +#undef MC_CMD_0x102_PRIVILEGE_CTG + +#define MC_CMD_0x102_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_FUSE_DIAGS_IN msgrequest */ +#define MC_CMD_FUSE_DIAGS_IN_LEN 0 + +/* MC_CMD_FUSE_DIAGS_OUT msgresponse */ +#define MC_CMD_FUSE_DIAGS_OUT_LEN 48 +/* Total number of mismatched bits between pairs in area 0 */ +#define MC_CMD_FUSE_DIAGS_OUT_AREA0_MISMATCH_BITS_OFST 0 +/* Total number of unexpectedly clear (set in B but not A) bits in area 0 */ +#define MC_CMD_FUSE_DIAGS_OUT_AREA0_PAIR_A_BAD_BITS_OFST 4 +/* Total number of unexpectedly clear (set in A but not B) bits in area 0 */ +#define MC_CMD_FUSE_DIAGS_OUT_AREA0_PAIR_B_BAD_BITS_OFST 8 +/* Checksum of data after logical OR of pairs in area 0 */ +#define MC_CMD_FUSE_DIAGS_OUT_AREA0_CHECKSUM_OFST 12 +/* Total number of mismatched bits between pairs in area 1 */ +#define MC_CMD_FUSE_DIAGS_OUT_AREA1_MISMATCH_BITS_OFST 16 +/* Total number of unexpectedly clear (set in B but not A) bits in area 1 */ +#define MC_CMD_FUSE_DIAGS_OUT_AREA1_PAIR_A_BAD_BITS_OFST 20 +/* Total number of unexpectedly clear (set in A but not B) bits in area 1 */ +#define MC_CMD_FUSE_DIAGS_OUT_AREA1_PAIR_B_BAD_BITS_OFST 24 +/* Checksum of data after logical OR of pairs in area 1 */ +#define MC_CMD_FUSE_DIAGS_OUT_AREA1_CHECKSUM_OFST 28 +/* Total number of mismatched bits between pairs in area 2 */ +#define MC_CMD_FUSE_DIAGS_OUT_AREA2_MISMATCH_BITS_OFST 32 +/* Total number of unexpectedly clear (set in B but not A) bits in area 2 */ +#define MC_CMD_FUSE_DIAGS_OUT_AREA2_PAIR_A_BAD_BITS_OFST 36 +/* Total number of unexpectedly clear (set in A but not B) bits in area 2 */ +#define MC_CMD_FUSE_DIAGS_OUT_AREA2_PAIR_B_BAD_BITS_OFST 40 +/* Checksum of data after logical OR of pairs in area 2 */ +#define MC_CMD_FUSE_DIAGS_OUT_AREA2_CHECKSUM_OFST 44 + + +/***********************************/ +/* MC_CMD_PRIVILEGE_MODIFY + * Modify the privileges of a set of PCIe functions. Note that this operation + * only effects non-admin functions unless the admin privilege itself is + * included in one of the masks provided. + */ +#define MC_CMD_PRIVILEGE_MODIFY 0x60 +#undef MC_CMD_0x60_PRIVILEGE_CTG + +#define MC_CMD_0x60_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_PRIVILEGE_MODIFY_IN msgrequest */ +#define MC_CMD_PRIVILEGE_MODIFY_IN_LEN 16 +/* The groups of functions to have their privilege masks modified. */ +#define MC_CMD_PRIVILEGE_MODIFY_IN_FN_GROUP_OFST 0 +#define MC_CMD_PRIVILEGE_MODIFY_IN_NONE 0x0 /* enum */ +#define MC_CMD_PRIVILEGE_MODIFY_IN_ALL 0x1 /* enum */ +#define MC_CMD_PRIVILEGE_MODIFY_IN_PFS_ONLY 0x2 /* enum */ +#define MC_CMD_PRIVILEGE_MODIFY_IN_VFS_ONLY 0x3 /* enum */ +#define MC_CMD_PRIVILEGE_MODIFY_IN_VFS_OF_PF 0x4 /* enum */ +#define MC_CMD_PRIVILEGE_MODIFY_IN_ONE 0x5 /* enum */ +/* For VFS_OF_PF specify the PF, for ONE specify the target function */ +#define MC_CMD_PRIVILEGE_MODIFY_IN_FUNCTION_OFST 4 +#define MC_CMD_PRIVILEGE_MODIFY_IN_FUNCTION_PF_LBN 0 +#define MC_CMD_PRIVILEGE_MODIFY_IN_FUNCTION_PF_WIDTH 16 +#define MC_CMD_PRIVILEGE_MODIFY_IN_FUNCTION_VF_LBN 16 +#define MC_CMD_PRIVILEGE_MODIFY_IN_FUNCTION_VF_WIDTH 16 +/* Privileges to be added to the target functions. For privilege definitions + * refer to the command MC_CMD_PRIVILEGE_MASK + */ +#define MC_CMD_PRIVILEGE_MODIFY_IN_ADD_MASK_OFST 8 +/* Privileges to be removed from the target functions. For privilege + * definitions refer to the command MC_CMD_PRIVILEGE_MASK + */ +#define MC_CMD_PRIVILEGE_MODIFY_IN_REMOVE_MASK_OFST 12 + +/* MC_CMD_PRIVILEGE_MODIFY_OUT msgresponse */ +#define MC_CMD_PRIVILEGE_MODIFY_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_XPM_READ_BYTES + * Read XPM memory + */ +#define MC_CMD_XPM_READ_BYTES 0x103 +#undef MC_CMD_0x103_PRIVILEGE_CTG + +#define MC_CMD_0x103_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_XPM_READ_BYTES_IN msgrequest */ +#define MC_CMD_XPM_READ_BYTES_IN_LEN 8 +/* Start address (byte) */ +#define MC_CMD_XPM_READ_BYTES_IN_ADDR_OFST 0 +/* Count (bytes) */ +#define MC_CMD_XPM_READ_BYTES_IN_COUNT_OFST 4 + +/* MC_CMD_XPM_READ_BYTES_OUT msgresponse */ +#define MC_CMD_XPM_READ_BYTES_OUT_LENMIN 0 +#define MC_CMD_XPM_READ_BYTES_OUT_LENMAX 252 +#define MC_CMD_XPM_READ_BYTES_OUT_LEN(num) (0+1*(num)) +/* Data */ +#define MC_CMD_XPM_READ_BYTES_OUT_DATA_OFST 0 +#define MC_CMD_XPM_READ_BYTES_OUT_DATA_LEN 1 +#define MC_CMD_XPM_READ_BYTES_OUT_DATA_MINNUM 0 +#define MC_CMD_XPM_READ_BYTES_OUT_DATA_MAXNUM 252 + + +/***********************************/ +/* MC_CMD_XPM_WRITE_BYTES + * Write XPM memory + */ +#define MC_CMD_XPM_WRITE_BYTES 0x104 +#undef MC_CMD_0x104_PRIVILEGE_CTG + +#define MC_CMD_0x104_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_XPM_WRITE_BYTES_IN msgrequest */ +#define MC_CMD_XPM_WRITE_BYTES_IN_LENMIN 8 +#define MC_CMD_XPM_WRITE_BYTES_IN_LENMAX 252 +#define MC_CMD_XPM_WRITE_BYTES_IN_LEN(num) (8+1*(num)) +/* Start address (byte) */ +#define MC_CMD_XPM_WRITE_BYTES_IN_ADDR_OFST 0 +/* Count (bytes) */ +#define MC_CMD_XPM_WRITE_BYTES_IN_COUNT_OFST 4 +/* Data */ +#define MC_CMD_XPM_WRITE_BYTES_IN_DATA_OFST 8 +#define MC_CMD_XPM_WRITE_BYTES_IN_DATA_LEN 1 +#define MC_CMD_XPM_WRITE_BYTES_IN_DATA_MINNUM 0 +#define MC_CMD_XPM_WRITE_BYTES_IN_DATA_MAXNUM 244 + +/* MC_CMD_XPM_WRITE_BYTES_OUT msgresponse */ +#define MC_CMD_XPM_WRITE_BYTES_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_XPM_READ_SECTOR + * Read XPM sector + */ +#define MC_CMD_XPM_READ_SECTOR 0x105 +#undef MC_CMD_0x105_PRIVILEGE_CTG + +#define MC_CMD_0x105_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_XPM_READ_SECTOR_IN msgrequest */ +#define MC_CMD_XPM_READ_SECTOR_IN_LEN 8 +/* Sector index */ +#define MC_CMD_XPM_READ_SECTOR_IN_INDEX_OFST 0 +/* Sector size */ +#define MC_CMD_XPM_READ_SECTOR_IN_SIZE_OFST 4 + +/* MC_CMD_XPM_READ_SECTOR_OUT msgresponse */ +#define MC_CMD_XPM_READ_SECTOR_OUT_LENMIN 4 +#define MC_CMD_XPM_READ_SECTOR_OUT_LENMAX 36 +#define MC_CMD_XPM_READ_SECTOR_OUT_LEN(num) (4+1*(num)) +/* Sector type */ +#define MC_CMD_XPM_READ_SECTOR_OUT_TYPE_OFST 0 +#define MC_CMD_XPM_READ_SECTOR_OUT_BLANK 0x0 /* enum */ +#define MC_CMD_XPM_READ_SECTOR_OUT_CRYPTO_KEY_128 0x1 /* enum */ +#define MC_CMD_XPM_READ_SECTOR_OUT_CRYPTO_KEY_256 0x2 /* enum */ +#define MC_CMD_XPM_READ_SECTOR_OUT_INVALID 0xff /* enum */ +/* Sector data */ +#define MC_CMD_XPM_READ_SECTOR_OUT_DATA_OFST 4 +#define MC_CMD_XPM_READ_SECTOR_OUT_DATA_LEN 1 +#define MC_CMD_XPM_READ_SECTOR_OUT_DATA_MINNUM 0 +#define MC_CMD_XPM_READ_SECTOR_OUT_DATA_MAXNUM 32 + + +/***********************************/ +/* MC_CMD_XPM_WRITE_SECTOR + * Write XPM sector + */ +#define MC_CMD_XPM_WRITE_SECTOR 0x106 +#undef MC_CMD_0x106_PRIVILEGE_CTG + +#define MC_CMD_0x106_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_XPM_WRITE_SECTOR_IN msgrequest */ +#define MC_CMD_XPM_WRITE_SECTOR_IN_LENMIN 12 +#define MC_CMD_XPM_WRITE_SECTOR_IN_LENMAX 44 +#define MC_CMD_XPM_WRITE_SECTOR_IN_LEN(num) (12+1*(num)) +/* If writing fails due to an uncorrectable error, try up to RETRIES following + * sectors (or until no more space available). If 0, only one write attempt is + * made. Note that uncorrectable errors are unlikely, thanks to XPM self-repair + * mechanism. + */ +#define MC_CMD_XPM_WRITE_SECTOR_IN_RETRIES_OFST 0 +#define MC_CMD_XPM_WRITE_SECTOR_IN_RETRIES_LEN 1 +#define MC_CMD_XPM_WRITE_SECTOR_IN_RESERVED_OFST 1 +#define MC_CMD_XPM_WRITE_SECTOR_IN_RESERVED_LEN 3 +/* Sector type */ +#define MC_CMD_XPM_WRITE_SECTOR_IN_TYPE_OFST 4 +/* Enum values, see field(s): */ +/* MC_CMD_XPM_READ_SECTOR/MC_CMD_XPM_READ_SECTOR_OUT/TYPE */ +/* Sector size */ +#define MC_CMD_XPM_WRITE_SECTOR_IN_SIZE_OFST 8 +/* Sector data */ +#define MC_CMD_XPM_WRITE_SECTOR_IN_DATA_OFST 12 +#define MC_CMD_XPM_WRITE_SECTOR_IN_DATA_LEN 1 +#define MC_CMD_XPM_WRITE_SECTOR_IN_DATA_MINNUM 0 +#define MC_CMD_XPM_WRITE_SECTOR_IN_DATA_MAXNUM 32 + +/* MC_CMD_XPM_WRITE_SECTOR_OUT msgresponse */ +#define MC_CMD_XPM_WRITE_SECTOR_OUT_LEN 4 +/* New sector index */ +#define MC_CMD_XPM_WRITE_SECTOR_OUT_INDEX_OFST 0 + + +/***********************************/ +/* MC_CMD_XPM_INVALIDATE_SECTOR + * Invalidate XPM sector + */ +#define MC_CMD_XPM_INVALIDATE_SECTOR 0x107 +#undef MC_CMD_0x107_PRIVILEGE_CTG + +#define MC_CMD_0x107_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_XPM_INVALIDATE_SECTOR_IN msgrequest */ +#define MC_CMD_XPM_INVALIDATE_SECTOR_IN_LEN 4 +/* Sector index */ +#define MC_CMD_XPM_INVALIDATE_SECTOR_IN_INDEX_OFST 0 + +/* MC_CMD_XPM_INVALIDATE_SECTOR_OUT msgresponse */ +#define MC_CMD_XPM_INVALIDATE_SECTOR_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_XPM_BLANK_CHECK + * Blank-check XPM memory and report bad locations + */ +#define MC_CMD_XPM_BLANK_CHECK 0x108 +#undef MC_CMD_0x108_PRIVILEGE_CTG + +#define MC_CMD_0x108_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_XPM_BLANK_CHECK_IN msgrequest */ +#define MC_CMD_XPM_BLANK_CHECK_IN_LEN 8 +/* Start address (byte) */ +#define MC_CMD_XPM_BLANK_CHECK_IN_ADDR_OFST 0 +/* Count (bytes) */ +#define MC_CMD_XPM_BLANK_CHECK_IN_COUNT_OFST 4 + +/* MC_CMD_XPM_BLANK_CHECK_OUT msgresponse */ +#define MC_CMD_XPM_BLANK_CHECK_OUT_LENMIN 4 +#define MC_CMD_XPM_BLANK_CHECK_OUT_LENMAX 252 +#define MC_CMD_XPM_BLANK_CHECK_OUT_LEN(num) (4+2*(num)) +/* Total number of bad (non-blank) locations */ +#define MC_CMD_XPM_BLANK_CHECK_OUT_BAD_COUNT_OFST 0 +/* Addresses of bad locations (may be less than BAD_COUNT, if all cannot fit + * into MCDI response) + */ +#define MC_CMD_XPM_BLANK_CHECK_OUT_BAD_ADDR_OFST 4 +#define MC_CMD_XPM_BLANK_CHECK_OUT_BAD_ADDR_LEN 2 +#define MC_CMD_XPM_BLANK_CHECK_OUT_BAD_ADDR_MINNUM 0 +#define MC_CMD_XPM_BLANK_CHECK_OUT_BAD_ADDR_MAXNUM 124 + + +/***********************************/ +/* MC_CMD_XPM_REPAIR + * Blank-check and repair XPM memory + */ +#define MC_CMD_XPM_REPAIR 0x109 +#undef MC_CMD_0x109_PRIVILEGE_CTG + +#define MC_CMD_0x109_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_XPM_REPAIR_IN msgrequest */ +#define MC_CMD_XPM_REPAIR_IN_LEN 8 +/* Start address (byte) */ +#define MC_CMD_XPM_REPAIR_IN_ADDR_OFST 0 +/* Count (bytes) */ +#define MC_CMD_XPM_REPAIR_IN_COUNT_OFST 4 + +/* MC_CMD_XPM_REPAIR_OUT msgresponse */ +#define MC_CMD_XPM_REPAIR_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_XPM_DECODER_TEST + * Test XPM memory address decoders for gross manufacturing defects. Can only + * be performed on an unprogrammed part. + */ +#define MC_CMD_XPM_DECODER_TEST 0x10a +#undef MC_CMD_0x10a_PRIVILEGE_CTG + +#define MC_CMD_0x10a_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_XPM_DECODER_TEST_IN msgrequest */ +#define MC_CMD_XPM_DECODER_TEST_IN_LEN 0 + +/* MC_CMD_XPM_DECODER_TEST_OUT msgresponse */ +#define MC_CMD_XPM_DECODER_TEST_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_XPM_WRITE_TEST + * XPM memory write test. Test XPM write logic for gross manufacturing defects + * by writing to a dedicated test row. There are 16 locations in the test row + * and the test can only be performed on locations that have not been + * previously used (i.e. can be run at most 16 times). The test will pick the + * first available location to use, or fail with ENOSPC if none left. + */ +#define MC_CMD_XPM_WRITE_TEST 0x10b +#undef MC_CMD_0x10b_PRIVILEGE_CTG + +#define MC_CMD_0x10b_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_XPM_WRITE_TEST_IN msgrequest */ +#define MC_CMD_XPM_WRITE_TEST_IN_LEN 0 + +/* MC_CMD_XPM_WRITE_TEST_OUT msgresponse */ +#define MC_CMD_XPM_WRITE_TEST_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_EXEC_SIGNED + * Check the CMAC of the contents of IMEM and DMEM against the value supplied + * and if correct begin execution from the start of IMEM. The caller supplies a + * key ID, the length of IMEM and DMEM to validate and the expected CMAC. CMAC + * computation runs from the start of IMEM, and from the start of DMEM + 16k, + * to match flash booting. The command will respond with EINVAL if the CMAC + * does match, otherwise it will respond with success before it jumps to IMEM. + */ +#define MC_CMD_EXEC_SIGNED 0x10c +#undef MC_CMD_0x10c_PRIVILEGE_CTG + +#define MC_CMD_0x10c_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_EXEC_SIGNED_IN msgrequest */ +#define MC_CMD_EXEC_SIGNED_IN_LEN 28 +/* the length of code to include in the CMAC */ +#define MC_CMD_EXEC_SIGNED_IN_CODELEN_OFST 0 +/* the length of date to include in the CMAC */ +#define MC_CMD_EXEC_SIGNED_IN_DATALEN_OFST 4 +/* the XPM sector containing the key to use */ +#define MC_CMD_EXEC_SIGNED_IN_KEYSECTOR_OFST 8 +/* the expected CMAC value */ +#define MC_CMD_EXEC_SIGNED_IN_CMAC_OFST 12 +#define MC_CMD_EXEC_SIGNED_IN_CMAC_LEN 16 + +/* MC_CMD_EXEC_SIGNED_OUT msgresponse */ +#define MC_CMD_EXEC_SIGNED_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_PREPARE_SIGNED + * Prepare to upload a signed image. This will scrub the specified length of + * the data region, which must be at least as large as the DATALEN supplied to + * MC_CMD_EXEC_SIGNED. + */ +#define MC_CMD_PREPARE_SIGNED 0x10d +#undef MC_CMD_0x10d_PRIVILEGE_CTG + +#define MC_CMD_0x10d_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_PREPARE_SIGNED_IN msgrequest */ +#define MC_CMD_PREPARE_SIGNED_IN_LEN 4 +/* the length of data area to clear */ +#define MC_CMD_PREPARE_SIGNED_IN_DATALEN_OFST 0 + +/* MC_CMD_PREPARE_SIGNED_OUT msgresponse */ +#define MC_CMD_PREPARE_SIGNED_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_SET_SECURITY_RULE + * Set blacklist and/or whitelist action for a particular match criteria. + * (Medford-only; for use by SolarSecure apps, not directly by drivers. See + * SF-114946-SW.) NOTE - this message definition is provisional. It has not yet + * been used in any released code and may change during development. This note + * will be removed once it is regarded as stable. + */ +#define MC_CMD_SET_SECURITY_RULE 0x10f +#undef MC_CMD_0x10f_PRIVILEGE_CTG + +#define MC_CMD_0x10f_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_SET_SECURITY_RULE_IN msgrequest */ +#define MC_CMD_SET_SECURITY_RULE_IN_LEN 92 +/* fields to include in match criteria */ +#define MC_CMD_SET_SECURITY_RULE_IN_MATCH_FIELDS_OFST 0 +#define MC_CMD_SET_SECURITY_RULE_IN_MATCH_REMOTE_IP_LBN 0 +#define MC_CMD_SET_SECURITY_RULE_IN_MATCH_REMOTE_IP_WIDTH 1 +#define MC_CMD_SET_SECURITY_RULE_IN_MATCH_LOCAL_IP_LBN 1 +#define MC_CMD_SET_SECURITY_RULE_IN_MATCH_LOCAL_IP_WIDTH 1 +#define MC_CMD_SET_SECURITY_RULE_IN_MATCH_REMOTE_MAC_LBN 2 +#define MC_CMD_SET_SECURITY_RULE_IN_MATCH_REMOTE_MAC_WIDTH 1 +#define MC_CMD_SET_SECURITY_RULE_IN_MATCH_REMOTE_PORT_LBN 3 +#define MC_CMD_SET_SECURITY_RULE_IN_MATCH_REMOTE_PORT_WIDTH 1 +#define MC_CMD_SET_SECURITY_RULE_IN_MATCH_LOCAL_MAC_LBN 4 +#define MC_CMD_SET_SECURITY_RULE_IN_MATCH_LOCAL_MAC_WIDTH 1 +#define MC_CMD_SET_SECURITY_RULE_IN_MATCH_LOCAL_PORT_LBN 5 +#define MC_CMD_SET_SECURITY_RULE_IN_MATCH_LOCAL_PORT_WIDTH 1 +#define MC_CMD_SET_SECURITY_RULE_IN_MATCH_ETHER_TYPE_LBN 6 +#define MC_CMD_SET_SECURITY_RULE_IN_MATCH_ETHER_TYPE_WIDTH 1 +#define MC_CMD_SET_SECURITY_RULE_IN_MATCH_INNER_VLAN_LBN 7 +#define MC_CMD_SET_SECURITY_RULE_IN_MATCH_INNER_VLAN_WIDTH 1 +#define MC_CMD_SET_SECURITY_RULE_IN_MATCH_OUTER_VLAN_LBN 8 +#define MC_CMD_SET_SECURITY_RULE_IN_MATCH_OUTER_VLAN_WIDTH 1 +#define MC_CMD_SET_SECURITY_RULE_IN_MATCH_IP_PROTO_LBN 9 +#define MC_CMD_SET_SECURITY_RULE_IN_MATCH_IP_PROTO_WIDTH 1 +#define MC_CMD_SET_SECURITY_RULE_IN_MATCH_PHYSICAL_PORT_LBN 10 +#define MC_CMD_SET_SECURITY_RULE_IN_MATCH_PHYSICAL_PORT_WIDTH 1 +#define MC_CMD_SET_SECURITY_RULE_IN_MATCH_RESERVED_LBN 11 +#define MC_CMD_SET_SECURITY_RULE_IN_MATCH_RESERVED_WIDTH 1 +#define MC_CMD_SET_SECURITY_RULE_IN_MATCH_REMOTE_SUBNET_ID_LBN 12 +#define MC_CMD_SET_SECURITY_RULE_IN_MATCH_REMOTE_SUBNET_ID_WIDTH 1 +#define MC_CMD_SET_SECURITY_RULE_IN_MATCH_REMOTE_PORTRANGE_ID_LBN 13 +#define MC_CMD_SET_SECURITY_RULE_IN_MATCH_REMOTE_PORTRANGE_ID_WIDTH 1 +#define MC_CMD_SET_SECURITY_RULE_IN_MATCH_LOCAL_PORTRANGE_ID_LBN 14 +#define MC_CMD_SET_SECURITY_RULE_IN_MATCH_LOCAL_PORTRANGE_ID_WIDTH 1 +/* remote MAC address to match (as bytes in network order) */ +#define MC_CMD_SET_SECURITY_RULE_IN_REMOTE_MAC_OFST 4 +#define MC_CMD_SET_SECURITY_RULE_IN_REMOTE_MAC_LEN 6 +/* remote port to match (as bytes in network order) */ +#define MC_CMD_SET_SECURITY_RULE_IN_REMOTE_PORT_OFST 10 +#define MC_CMD_SET_SECURITY_RULE_IN_REMOTE_PORT_LEN 2 +/* local MAC address to match (as bytes in network order) */ +#define MC_CMD_SET_SECURITY_RULE_IN_LOCAL_MAC_OFST 12 +#define MC_CMD_SET_SECURITY_RULE_IN_LOCAL_MAC_LEN 6 +/* local port to match (as bytes in network order) */ +#define MC_CMD_SET_SECURITY_RULE_IN_LOCAL_PORT_OFST 18 +#define MC_CMD_SET_SECURITY_RULE_IN_LOCAL_PORT_LEN 2 +/* Ethernet type to match (as bytes in network order) */ +#define MC_CMD_SET_SECURITY_RULE_IN_ETHER_TYPE_OFST 20 +#define MC_CMD_SET_SECURITY_RULE_IN_ETHER_TYPE_LEN 2 +/* Inner VLAN tag to match (as bytes in network order) */ +#define MC_CMD_SET_SECURITY_RULE_IN_INNER_VLAN_OFST 22 +#define MC_CMD_SET_SECURITY_RULE_IN_INNER_VLAN_LEN 2 +/* Outer VLAN tag to match (as bytes in network order) */ +#define MC_CMD_SET_SECURITY_RULE_IN_OUTER_VLAN_OFST 24 +#define MC_CMD_SET_SECURITY_RULE_IN_OUTER_VLAN_LEN 2 +/* IP protocol to match (in low byte; set high byte to 0) */ +#define MC_CMD_SET_SECURITY_RULE_IN_IP_PROTO_OFST 26 +#define MC_CMD_SET_SECURITY_RULE_IN_IP_PROTO_LEN 2 +/* Physical port to match (as little-endian 32-bit value) */ +#define MC_CMD_SET_SECURITY_RULE_IN_PHYSICAL_PORT_OFST 28 +/* Reserved; set to 0 */ +#define MC_CMD_SET_SECURITY_RULE_IN_RESERVED_OFST 32 +/* remote IP address to match (as bytes in network order; set last 12 bytes to + * 0 for IPv4 address) + */ +#define MC_CMD_SET_SECURITY_RULE_IN_REMOTE_IP_OFST 36 +#define MC_CMD_SET_SECURITY_RULE_IN_REMOTE_IP_LEN 16 +/* local IP address to match (as bytes in network order; set last 12 bytes to 0 + * for IPv4 address) + */ +#define MC_CMD_SET_SECURITY_RULE_IN_LOCAL_IP_OFST 52 +#define MC_CMD_SET_SECURITY_RULE_IN_LOCAL_IP_LEN 16 +/* remote subnet ID to match (as little-endian 32-bit value); note that remote + * subnets are matched by mapping the remote IP address to a "subnet ID" via a + * data structure which must already have been configured using + * MC_CMD_SUBNET_MAP_SET_NODE appropriately + */ +#define MC_CMD_SET_SECURITY_RULE_IN_REMOTE_SUBNET_ID_OFST 68 +/* remote portrange ID to match (as little-endian 32-bit value); note that + * remote port ranges are matched by mapping the remote port to a "portrange + * ID" via a data structure which must already have been configured using + * MC_CMD_REMOTE_PORTRANGE_MAP_SET_TREE + */ +#define MC_CMD_SET_SECURITY_RULE_IN_REMOTE_PORTRANGE_ID_OFST 72 +/* local portrange ID to match (as little-endian 32-bit value); note that local + * port ranges are matched by mapping the local port to a "portrange ID" via a + * data structure which must already have been configured using + * MC_CMD_LOCAL_PORTRANGE_MAP_SET_TREE + */ +#define MC_CMD_SET_SECURITY_RULE_IN_LOCAL_PORTRANGE_ID_OFST 76 +/* set the action for transmitted packets matching this rule */ +#define MC_CMD_SET_SECURITY_RULE_IN_TX_ACTION_OFST 80 +/* enum: make no decision */ +#define MC_CMD_SET_SECURITY_RULE_IN_TX_ACTION_NONE 0x0 +/* enum: decide to accept the packet */ +#define MC_CMD_SET_SECURITY_RULE_IN_TX_ACTION_WHITELIST 0x1 +/* enum: decide to drop the packet */ +#define MC_CMD_SET_SECURITY_RULE_IN_TX_ACTION_BLACKLIST 0x2 +/* enum: do not change the current TX action */ +#define MC_CMD_SET_SECURITY_RULE_IN_TX_ACTION_UNCHANGED 0xffffffff +/* set the action for received packets matching this rule */ +#define MC_CMD_SET_SECURITY_RULE_IN_RX_ACTION_OFST 84 +/* enum: make no decision */ +#define MC_CMD_SET_SECURITY_RULE_IN_RX_ACTION_NONE 0x0 +/* enum: decide to accept the packet */ +#define MC_CMD_SET_SECURITY_RULE_IN_RX_ACTION_WHITELIST 0x1 +/* enum: decide to drop the packet */ +#define MC_CMD_SET_SECURITY_RULE_IN_RX_ACTION_BLACKLIST 0x2 +/* enum: do not change the current RX action */ +#define MC_CMD_SET_SECURITY_RULE_IN_RX_ACTION_UNCHANGED 0xffffffff +/* counter ID to associate with this rule; IDs are allocated using + * MC_CMD_SECURITY_RULE_COUNTER_ALLOC + */ +#define MC_CMD_SET_SECURITY_RULE_IN_COUNTER_ID_OFST 88 +/* enum: special value for the null counter ID */ +#define MC_CMD_SET_SECURITY_RULE_IN_COUNTER_ID_NONE 0x0 + +/* MC_CMD_SET_SECURITY_RULE_OUT msgresponse */ +#define MC_CMD_SET_SECURITY_RULE_OUT_LEN 28 +/* new reference count for uses of counter ID */ +#define MC_CMD_SET_SECURITY_RULE_OUT_COUNTER_REFCNT_OFST 0 +/* constructed match bits for this rule (as a tracing aid only) */ +#define MC_CMD_SET_SECURITY_RULE_OUT_LUE_MATCH_BITS_OFST 4 +#define MC_CMD_SET_SECURITY_RULE_OUT_LUE_MATCH_BITS_LEN 12 +/* constructed discriminator bits for this rule (as a tracing aid only) */ +#define MC_CMD_SET_SECURITY_RULE_OUT_LUE_DISCRIMINATOR_OFST 16 +/* base location for probes for this rule (as a tracing aid only) */ +#define MC_CMD_SET_SECURITY_RULE_OUT_LUE_PROBE_BASE_OFST 20 +/* step for probes for this rule (as a tracing aid only) */ +#define MC_CMD_SET_SECURITY_RULE_OUT_LUE_PROBE_STEP_OFST 24 + + +/***********************************/ +/* MC_CMD_RESET_SECURITY_RULES + * Reset all blacklist and whitelist actions for a particular physical port, or + * all ports. (Medford-only; for use by SolarSecure apps, not directly by + * drivers. See SF-114946-SW.) NOTE - this message definition is provisional. + * It has not yet been used in any released code and may change during + * development. This note will be removed once it is regarded as stable. + */ +#define MC_CMD_RESET_SECURITY_RULES 0x110 +#undef MC_CMD_0x110_PRIVILEGE_CTG + +#define MC_CMD_0x110_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_RESET_SECURITY_RULES_IN msgrequest */ +#define MC_CMD_RESET_SECURITY_RULES_IN_LEN 4 +/* index of physical port to reset (or ALL_PHYSICAL_PORTS to reset all) */ +#define MC_CMD_RESET_SECURITY_RULES_IN_PHYSICAL_PORT_OFST 0 +/* enum: special value to reset all physical ports */ +#define MC_CMD_RESET_SECURITY_RULES_IN_ALL_PHYSICAL_PORTS 0xffffffff + +/* MC_CMD_RESET_SECURITY_RULES_OUT msgresponse */ +#define MC_CMD_RESET_SECURITY_RULES_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_GET_SECURITY_RULESET_VERSION + * Return a large hash value representing a "version" of the complete set of + * currently active blacklist / whitelist rules and associated data structures. + * (Medford-only; for use by SolarSecure apps, not directly by drivers. See + * SF-114946-SW.) NOTE - this message definition is provisional. It has not yet + * been used in any released code and may change during development. This note + * will be removed once it is regarded as stable. + */ +#define MC_CMD_GET_SECURITY_RULESET_VERSION 0x111 +#undef MC_CMD_0x111_PRIVILEGE_CTG + +#define MC_CMD_0x111_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_GET_SECURITY_RULESET_VERSION_IN msgrequest */ +#define MC_CMD_GET_SECURITY_RULESET_VERSION_IN_LEN 0 + +/* MC_CMD_GET_SECURITY_RULESET_VERSION_OUT msgresponse */ +#define MC_CMD_GET_SECURITY_RULESET_VERSION_OUT_LENMIN 1 +#define MC_CMD_GET_SECURITY_RULESET_VERSION_OUT_LENMAX 252 +#define MC_CMD_GET_SECURITY_RULESET_VERSION_OUT_LEN(num) (0+1*(num)) +/* Opaque hash value; length may vary depending on the hash scheme used */ +#define MC_CMD_GET_SECURITY_RULESET_VERSION_OUT_VERSION_OFST 0 +#define MC_CMD_GET_SECURITY_RULESET_VERSION_OUT_VERSION_LEN 1 +#define MC_CMD_GET_SECURITY_RULESET_VERSION_OUT_VERSION_MINNUM 1 +#define MC_CMD_GET_SECURITY_RULESET_VERSION_OUT_VERSION_MAXNUM 252 + + +/***********************************/ +/* MC_CMD_SECURITY_RULE_COUNTER_ALLOC + * Allocate counters for use with blacklist / whitelist rules. (Medford-only; + * for use by SolarSecure apps, not directly by drivers. See SF-114946-SW.) + * NOTE - this message definition is provisional. It has not yet been used in + * any released code and may change during development. This note will be + * removed once it is regarded as stable. + */ +#define MC_CMD_SECURITY_RULE_COUNTER_ALLOC 0x112 +#undef MC_CMD_0x112_PRIVILEGE_CTG + +#define MC_CMD_0x112_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_SECURITY_RULE_COUNTER_ALLOC_IN msgrequest */ +#define MC_CMD_SECURITY_RULE_COUNTER_ALLOC_IN_LEN 4 +/* the number of new counter IDs to request */ +#define MC_CMD_SECURITY_RULE_COUNTER_ALLOC_IN_NUM_COUNTERS_OFST 0 + +/* MC_CMD_SECURITY_RULE_COUNTER_ALLOC_OUT msgresponse */ +#define MC_CMD_SECURITY_RULE_COUNTER_ALLOC_OUT_LENMIN 4 +#define MC_CMD_SECURITY_RULE_COUNTER_ALLOC_OUT_LENMAX 252 +#define MC_CMD_SECURITY_RULE_COUNTER_ALLOC_OUT_LEN(num) (4+4*(num)) +/* the number of new counter IDs allocated (may be less than the number + * requested if resources are unavailable) + */ +#define MC_CMD_SECURITY_RULE_COUNTER_ALLOC_OUT_NUM_COUNTERS_OFST 0 +/* new counter ID(s) */ +#define MC_CMD_SECURITY_RULE_COUNTER_ALLOC_OUT_COUNTER_ID_OFST 4 +#define MC_CMD_SECURITY_RULE_COUNTER_ALLOC_OUT_COUNTER_ID_LEN 4 +#define MC_CMD_SECURITY_RULE_COUNTER_ALLOC_OUT_COUNTER_ID_MINNUM 0 +#define MC_CMD_SECURITY_RULE_COUNTER_ALLOC_OUT_COUNTER_ID_MAXNUM 62 + + +/***********************************/ +/* MC_CMD_SECURITY_RULE_COUNTER_FREE + * Allocate counters for use with blacklist / whitelist rules. (Medford-only; + * for use by SolarSecure apps, not directly by drivers. See SF-114946-SW.) + * NOTE - this message definition is provisional. It has not yet been used in + * any released code and may change during development. This note will be + * removed once it is regarded as stable. + */ +#define MC_CMD_SECURITY_RULE_COUNTER_FREE 0x113 +#undef MC_CMD_0x113_PRIVILEGE_CTG + +#define MC_CMD_0x113_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_SECURITY_RULE_COUNTER_FREE_IN msgrequest */ +#define MC_CMD_SECURITY_RULE_COUNTER_FREE_IN_LENMIN 4 +#define MC_CMD_SECURITY_RULE_COUNTER_FREE_IN_LENMAX 252 +#define MC_CMD_SECURITY_RULE_COUNTER_FREE_IN_LEN(num) (4+4*(num)) +/* the number of counter IDs to free */ +#define MC_CMD_SECURITY_RULE_COUNTER_FREE_IN_NUM_COUNTERS_OFST 0 +/* the counter ID(s) to free */ +#define MC_CMD_SECURITY_RULE_COUNTER_FREE_IN_COUNTER_ID_OFST 4 +#define MC_CMD_SECURITY_RULE_COUNTER_FREE_IN_COUNTER_ID_LEN 4 +#define MC_CMD_SECURITY_RULE_COUNTER_FREE_IN_COUNTER_ID_MINNUM 0 +#define MC_CMD_SECURITY_RULE_COUNTER_FREE_IN_COUNTER_ID_MAXNUM 62 + +/* MC_CMD_SECURITY_RULE_COUNTER_FREE_OUT msgresponse */ +#define MC_CMD_SECURITY_RULE_COUNTER_FREE_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_SUBNET_MAP_SET_NODE + * Atomically update a trie node in the map of subnets to subnet IDs. The + * constants in the descriptions of the fields of this message may be retrieved + * by the GET_SECURITY_RULE_INFO op of MC_CMD_GET_PARSER_DISP_INFO. (Medford- + * only; for use by SolarSecure apps, not directly by drivers. See + * SF-114946-SW.) NOTE - this message definition is provisional. It has not yet + * been used in any released code and may change during development. This note + * will be removed once it is regarded as stable. + */ +#define MC_CMD_SUBNET_MAP_SET_NODE 0x114 +#undef MC_CMD_0x114_PRIVILEGE_CTG + +#define MC_CMD_0x114_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_SUBNET_MAP_SET_NODE_IN msgrequest */ +#define MC_CMD_SUBNET_MAP_SET_NODE_IN_LENMIN 6 +#define MC_CMD_SUBNET_MAP_SET_NODE_IN_LENMAX 252 +#define MC_CMD_SUBNET_MAP_SET_NODE_IN_LEN(num) (4+2*(num)) +/* node to update in the range 0 .. SUBNET_MAP_NUM_NODES-1 */ +#define MC_CMD_SUBNET_MAP_SET_NODE_IN_NODE_ID_OFST 0 +/* SUBNET_MAP_NUM_ENTRIES_PER_NODE new entries; each entry is either a pointer + * to the next node, expressed as an offset in the trie memory (i.e. node ID + * multiplied by SUBNET_MAP_NUM_ENTRIES_PER_NODE), or a leaf value in the range + * SUBNET_ID_MIN .. SUBNET_ID_MAX + */ +#define MC_CMD_SUBNET_MAP_SET_NODE_IN_ENTRY_OFST 4 +#define MC_CMD_SUBNET_MAP_SET_NODE_IN_ENTRY_LEN 2 +#define MC_CMD_SUBNET_MAP_SET_NODE_IN_ENTRY_MINNUM 1 +#define MC_CMD_SUBNET_MAP_SET_NODE_IN_ENTRY_MAXNUM 124 + +/* MC_CMD_SUBNET_MAP_SET_NODE_OUT msgresponse */ +#define MC_CMD_SUBNET_MAP_SET_NODE_OUT_LEN 0 + +/* PORTRANGE_TREE_ENTRY structuredef */ +#define PORTRANGE_TREE_ENTRY_LEN 4 +/* key for branch nodes (<= key takes left branch, > key takes right branch), + * or magic value for leaf nodes + */ +#define PORTRANGE_TREE_ENTRY_BRANCH_KEY_OFST 0 +#define PORTRANGE_TREE_ENTRY_BRANCH_KEY_LEN 2 +#define PORTRANGE_TREE_ENTRY_LEAF_NODE_KEY 0xffff /* enum */ +#define PORTRANGE_TREE_ENTRY_BRANCH_KEY_LBN 0 +#define PORTRANGE_TREE_ENTRY_BRANCH_KEY_WIDTH 16 +/* final portrange ID for leaf nodes (don't care for branch nodes) */ +#define PORTRANGE_TREE_ENTRY_LEAF_PORTRANGE_ID_OFST 2 +#define PORTRANGE_TREE_ENTRY_LEAF_PORTRANGE_ID_LEN 2 +#define PORTRANGE_TREE_ENTRY_LEAF_PORTRANGE_ID_LBN 16 +#define PORTRANGE_TREE_ENTRY_LEAF_PORTRANGE_ID_WIDTH 16 + + +/***********************************/ +/* MC_CMD_REMOTE_PORTRANGE_MAP_SET_TREE + * Atomically update the entire tree mapping remote port ranges to portrange + * IDs. The constants in the descriptions of the fields of this message may be + * retrieved by the GET_SECURITY_RULE_INFO op of MC_CMD_GET_PARSER_DISP_INFO. + * (Medford-only; for use by SolarSecure apps, not directly by drivers. See + * SF-114946-SW.) NOTE - this message definition is provisional. It has not yet + * been used in any released code and may change during development. This note + * will be removed once it is regarded as stable. + */ +#define MC_CMD_REMOTE_PORTRANGE_MAP_SET_TREE 0x115 +#undef MC_CMD_0x115_PRIVILEGE_CTG + +#define MC_CMD_0x115_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_REMOTE_PORTRANGE_MAP_SET_TREE_IN msgrequest */ +#define MC_CMD_REMOTE_PORTRANGE_MAP_SET_TREE_IN_LENMIN 4 +#define MC_CMD_REMOTE_PORTRANGE_MAP_SET_TREE_IN_LENMAX 252 +#define MC_CMD_REMOTE_PORTRANGE_MAP_SET_TREE_IN_LEN(num) (0+4*(num)) +/* PORTRANGE_TREE_NUM_ENTRIES new entries, each laid out as a + * PORTRANGE_TREE_ENTRY + */ +#define MC_CMD_REMOTE_PORTRANGE_MAP_SET_TREE_IN_ENTRIES_OFST 0 +#define MC_CMD_REMOTE_PORTRANGE_MAP_SET_TREE_IN_ENTRIES_LEN 4 +#define MC_CMD_REMOTE_PORTRANGE_MAP_SET_TREE_IN_ENTRIES_MINNUM 1 +#define MC_CMD_REMOTE_PORTRANGE_MAP_SET_TREE_IN_ENTRIES_MAXNUM 63 + +/* MC_CMD_REMOTE_PORTRANGE_MAP_SET_TREE_OUT msgresponse */ +#define MC_CMD_REMOTE_PORTRANGE_MAP_SET_TREE_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_LOCAL_PORTRANGE_MAP_SET_TREE + * Atomically update the entire tree mapping remote port ranges to portrange + * IDs. The constants in the descriptions of the fields of this message may be + * retrieved by the GET_SECURITY_RULE_INFO op of MC_CMD_GET_PARSER_DISP_INFO. + * (Medford-only; for use by SolarSecure apps, not directly by drivers. See + * SF-114946-SW.) NOTE - this message definition is provisional. It has not yet + * been used in any released code and may change during development. This note + * will be removed once it is regarded as stable. + */ +#define MC_CMD_LOCAL_PORTRANGE_MAP_SET_TREE 0x116 +#undef MC_CMD_0x116_PRIVILEGE_CTG + +#define MC_CMD_0x116_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_LOCAL_PORTRANGE_MAP_SET_TREE_IN msgrequest */ +#define MC_CMD_LOCAL_PORTRANGE_MAP_SET_TREE_IN_LENMIN 4 +#define MC_CMD_LOCAL_PORTRANGE_MAP_SET_TREE_IN_LENMAX 252 +#define MC_CMD_LOCAL_PORTRANGE_MAP_SET_TREE_IN_LEN(num) (0+4*(num)) +/* PORTRANGE_TREE_NUM_ENTRIES new entries, each laid out as a + * PORTRANGE_TREE_ENTRY + */ +#define MC_CMD_LOCAL_PORTRANGE_MAP_SET_TREE_IN_ENTRIES_OFST 0 +#define MC_CMD_LOCAL_PORTRANGE_MAP_SET_TREE_IN_ENTRIES_LEN 4 +#define MC_CMD_LOCAL_PORTRANGE_MAP_SET_TREE_IN_ENTRIES_MINNUM 1 +#define MC_CMD_LOCAL_PORTRANGE_MAP_SET_TREE_IN_ENTRIES_MAXNUM 63 + +/* MC_CMD_LOCAL_PORTRANGE_MAP_SET_TREE_OUT msgresponse */ +#define MC_CMD_LOCAL_PORTRANGE_MAP_SET_TREE_OUT_LEN 0 + +/* TUNNEL_ENCAP_UDP_PORT_ENTRY structuredef */ +#define TUNNEL_ENCAP_UDP_PORT_ENTRY_LEN 4 +/* UDP port (the standard ports are named below but any port may be used) */ +#define TUNNEL_ENCAP_UDP_PORT_ENTRY_UDP_PORT_OFST 0 +#define TUNNEL_ENCAP_UDP_PORT_ENTRY_UDP_PORT_LEN 2 +/* enum: the IANA allocated UDP port for VXLAN */ +#define TUNNEL_ENCAP_UDP_PORT_ENTRY_IANA_VXLAN_UDP_PORT 0x12b5 +/* enum: the IANA allocated UDP port for Geneve */ +#define TUNNEL_ENCAP_UDP_PORT_ENTRY_IANA_GENEVE_UDP_PORT 0x17c1 +#define TUNNEL_ENCAP_UDP_PORT_ENTRY_UDP_PORT_LBN 0 +#define TUNNEL_ENCAP_UDP_PORT_ENTRY_UDP_PORT_WIDTH 16 +/* tunnel encapsulation protocol (only those named below are supported) */ +#define TUNNEL_ENCAP_UDP_PORT_ENTRY_PROTOCOL_OFST 2 +#define TUNNEL_ENCAP_UDP_PORT_ENTRY_PROTOCOL_LEN 2 +/* enum: This port will be used for VXLAN on both IPv4 and IPv6 */ +#define TUNNEL_ENCAP_UDP_PORT_ENTRY_VXLAN 0x0 +/* enum: This port will be used for Geneve on both IPv4 and IPv6 */ +#define TUNNEL_ENCAP_UDP_PORT_ENTRY_GENEVE 0x1 +#define TUNNEL_ENCAP_UDP_PORT_ENTRY_PROTOCOL_LBN 16 +#define TUNNEL_ENCAP_UDP_PORT_ENTRY_PROTOCOL_WIDTH 16 + + +/***********************************/ +/* MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS + * Configure UDP ports for tunnel encapsulation hardware acceleration. The + * parser-dispatcher will attempt to parse traffic on these ports as tunnel + * encapsulation PDUs and filter them using the tunnel encapsulation filter + * chain rather than the standard filter chain. Note that this command can + * cause all functions to see a reset. (Available on Medford only.) + */ +#define MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS 0x117 +#undef MC_CMD_0x117_PRIVILEGE_CTG + +#define MC_CMD_0x117_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_IN msgrequest */ +#define MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_IN_LENMIN 4 +#define MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_IN_LENMAX 68 +#define MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_IN_LEN(num) (4+4*(num)) +/* Flags */ +#define MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_IN_FLAGS_OFST 0 +#define MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_IN_FLAGS_LEN 2 +#define MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_IN_UNLOADING_LBN 0 +#define MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_IN_UNLOADING_WIDTH 1 +/* The number of entries in the ENTRIES array */ +#define MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_IN_NUM_ENTRIES_OFST 2 +#define MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_IN_NUM_ENTRIES_LEN 2 +/* Entries defining the UDP port to protocol mapping, each laid out as a + * TUNNEL_ENCAP_UDP_PORT_ENTRY + */ +#define MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_IN_ENTRIES_OFST 4 +#define MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_IN_ENTRIES_LEN 4 +#define MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_IN_ENTRIES_MINNUM 0 +#define MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_IN_ENTRIES_MAXNUM 16 + +/* MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_OUT msgresponse */ +#define MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_OUT_LEN 2 +/* Flags */ +#define MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_OUT_FLAGS_OFST 0 +#define MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_OUT_FLAGS_LEN 2 +#define MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_OUT_RESETTING_LBN 0 +#define MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_OUT_RESETTING_WIDTH 1 + + +/***********************************/ +/* MC_CMD_RX_BALANCING + * Configure a port upconverter to distribute the packets on both RX engines. + * Packets are distributed based on a table with the destination vFIFO. The + * index of the table is a hash of source and destination of IPV4 and VLAN + * priority. + */ +#define MC_CMD_RX_BALANCING 0x118 +#undef MC_CMD_0x118_PRIVILEGE_CTG + +#define MC_CMD_0x118_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_RX_BALANCING_IN msgrequest */ +#define MC_CMD_RX_BALANCING_IN_LEN 16 +/* The RX port whose upconverter table will be modified */ +#define MC_CMD_RX_BALANCING_IN_PORT_OFST 0 +/* The VLAN priority associated to the table index and vFIFO */ +#define MC_CMD_RX_BALANCING_IN_PRIORITY_OFST 4 +/* The resulting bit of SRC^DST for indexing the table */ +#define MC_CMD_RX_BALANCING_IN_SRC_DST_OFST 8 +/* The RX engine to which the vFIFO in the table entry will point to */ +#define MC_CMD_RX_BALANCING_IN_ENG_OFST 12 + +/* MC_CMD_RX_BALANCING_OUT msgresponse */ +#define MC_CMD_RX_BALANCING_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_TSA_BIND + * TSAN - TSAC binding communication protocol. Refer to SF-115479-TC for more + * info in respect to the binding protocol. This MCDI command is only available + * over a TLS secure connection between the TSAN and TSAC, and is not available + * to host software. Note- The messages definitions that do comprise this MCDI + * command deemed as provisional. This MCDI command has not yet been used in + * any released code and may change during development. This note will be + * removed once it is regarded as stable. + */ +#define MC_CMD_TSA_BIND 0x119 +#undef MC_CMD_0x119_PRIVILEGE_CTG + +#define MC_CMD_0x119_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_TSA_BIND_IN msgrequest: Protocol operation code */ +#define MC_CMD_TSA_BIND_IN_LEN 4 +#define MC_CMD_TSA_BIND_IN_OP_OFST 0 +/* enum: Retrieve the TSAN ID from a TSAN. TSAN ID is a unique identifier for + * the network adapter. More specifically, TSAN ID equals the MAC address of + * the network adapter. TSAN ID is used as part of the TSAN authentication + * protocol. Refer to SF-114946-SW for more information. + */ +#define MC_CMD_TSA_BIND_OP_GET_ID 0x1 +/* enum: Get a binding ticket from the TSAN. The binding ticket is used as part + * of the binding procedure to authorize the binding of an adapter to a TSAID. + * Refer to SF-114946-SW for more information. + */ +#define MC_CMD_TSA_BIND_OP_GET_TICKET 0x2 +/* enum: Opcode associated with the propagation of a private key that TSAN uses + * as part of post-binding authentication procedure. More specifically, TSAN + * uses this key for a signing operation. TSAC uses the counterpart public key + * to verify the signature. Note - The post-binding authentication occurs when + * the TSAN-TSAC connection terminates and TSAN tries to reconnect. Refer to + * SF-114946-SW for more information. + */ +#define MC_CMD_TSA_BIND_OP_SET_KEY 0x3 +/* enum: Request an unbinding operation. Note- TSAN clears the binding ticket + * from the Nvram section. + */ +#define MC_CMD_TSA_BIND_OP_UNBIND 0x4 + +/* MC_CMD_TSA_BIND_IN_GET_ID msgrequest */ +#define MC_CMD_TSA_BIND_IN_GET_ID_LEN 20 +/* The operation requested. */ +#define MC_CMD_TSA_BIND_IN_GET_ID_OP_OFST 0 +/* Cryptographic nonce that TSAC generates and sends to TSAN. TSAC generates + * the nonce every time as part of the TSAN post-binding authentication + * procedure when the TSAN-TSAC connection terminates and TSAN does need to re- + * connect to the TSAC. Refer to SF-114946-SW for more information. + */ +#define MC_CMD_TSA_BIND_IN_GET_ID_NONCE_OFST 4 +#define MC_CMD_TSA_BIND_IN_GET_ID_NONCE_LEN 16 + +/* MC_CMD_TSA_BIND_IN_GET_TICKET msgrequest */ +#define MC_CMD_TSA_BIND_IN_GET_TICKET_LEN 4 +/* The operation requested. */ +#define MC_CMD_TSA_BIND_IN_GET_TICKET_OP_OFST 0 + +/* MC_CMD_TSA_BIND_IN_SET_KEY msgrequest */ +#define MC_CMD_TSA_BIND_IN_SET_KEY_LENMIN 5 +#define MC_CMD_TSA_BIND_IN_SET_KEY_LENMAX 252 +#define MC_CMD_TSA_BIND_IN_SET_KEY_LEN(num) (4+1*(num)) +/* The operation requested. */ +#define MC_CMD_TSA_BIND_IN_SET_KEY_OP_OFST 0 +/* This data blob contains the private key generated by the TSAC. TSAN uses + * this key for a signing operation. Note- This private key is used in + * conjunction with the post-binding TSAN authentication procedure that occurs + * when the TSAN-TSAC connection terminates and TSAN tries to reconnect. Refer + * to SF-114946-SW for more information. + */ +#define MC_CMD_TSA_BIND_IN_SET_KEY_DATKEY_OFST 4 +#define MC_CMD_TSA_BIND_IN_SET_KEY_DATKEY_LEN 1 +#define MC_CMD_TSA_BIND_IN_SET_KEY_DATKEY_MINNUM 1 +#define MC_CMD_TSA_BIND_IN_SET_KEY_DATKEY_MAXNUM 248 + +/* MC_CMD_TSA_BIND_IN_UNBIND msgrequest: Asks for the un-binding procedure */ +#define MC_CMD_TSA_BIND_IN_UNBIND_LEN 10 +/* The operation requested. */ +#define MC_CMD_TSA_BIND_IN_UNBIND_OP_OFST 0 +/* TSAN unique identifier for the network adapter */ +#define MC_CMD_TSA_BIND_IN_UNBIND_TSANID_OFST 4 +#define MC_CMD_TSA_BIND_IN_UNBIND_TSANID_LEN 6 + +/* MC_CMD_TSA_BIND_OUT_GET_ID msgresponse */ +#define MC_CMD_TSA_BIND_OUT_GET_ID_LENMIN 15 +#define MC_CMD_TSA_BIND_OUT_GET_ID_LENMAX 252 +#define MC_CMD_TSA_BIND_OUT_GET_ID_LEN(num) (14+1*(num)) +/* The operation completion code. */ +#define MC_CMD_TSA_BIND_OUT_GET_ID_OP_OFST 0 +/* Rules engine type. Note- The rules engine type allows TSAC to further + * identify the connected endpoint (e.g. TSAN, NIC Emulator) type and take the + * proper action accordingly. As an example, TSAC uses the rules engine type to + * select the SF key that differs in the case of TSAN vs. NIC Emulator. + */ +#define MC_CMD_TSA_BIND_OUT_GET_ID_RULE_ENGINE_OFST 4 +/* enum: Hardware rules engine. */ +#define MC_CMD_TSA_BIND_OUT_GET_ID_RULE_ENGINE_TSAN 0x1 +/* enum: Nic emulator rules engine. */ +#define MC_CMD_TSA_BIND_OUT_GET_ID_RULE_ENGINE_NEMU 0x2 +/* enum: SSFE. */ +#define MC_CMD_TSA_BIND_OUT_GET_ID_RULE_ENGINE_SSFE 0x3 +/* TSAN unique identifier for the network adapter */ +#define MC_CMD_TSA_BIND_OUT_GET_ID_TSANID_OFST 8 +#define MC_CMD_TSA_BIND_OUT_GET_ID_TSANID_LEN 6 +/* The signature data blob. The signature is computed against the message + * formed by TSAN ID concatenated with the NONCE value. Refer to SF-115479-TC + * for more information also in respect to the private keys that are used to + * sign the message based on TSAN pre/post-binding authentication procedure. + */ +#define MC_CMD_TSA_BIND_OUT_GET_ID_SIG_OFST 14 +#define MC_CMD_TSA_BIND_OUT_GET_ID_SIG_LEN 1 +#define MC_CMD_TSA_BIND_OUT_GET_ID_SIG_MINNUM 1 +#define MC_CMD_TSA_BIND_OUT_GET_ID_SIG_MAXNUM 238 + +/* MC_CMD_TSA_BIND_OUT_GET_TICKET msgresponse */ +#define MC_CMD_TSA_BIND_OUT_GET_TICKET_LENMIN 5 +#define MC_CMD_TSA_BIND_OUT_GET_TICKET_LENMAX 252 +#define MC_CMD_TSA_BIND_OUT_GET_TICKET_LEN(num) (4+1*(num)) +/* The operation completion code. */ +#define MC_CMD_TSA_BIND_OUT_GET_TICKET_OP_OFST 0 +/* The ticket represents the data blob construct that TSAN sends to TSAC as + * part of the binding protocol. From the TSAN perspective the ticket is an + * opaque construct. For more info refer to SF-115479-TC. + */ +#define MC_CMD_TSA_BIND_OUT_GET_TICKET_TICKET_OFST 4 +#define MC_CMD_TSA_BIND_OUT_GET_TICKET_TICKET_LEN 1 +#define MC_CMD_TSA_BIND_OUT_GET_TICKET_TICKET_MINNUM 1 +#define MC_CMD_TSA_BIND_OUT_GET_TICKET_TICKET_MAXNUM 248 + +/* MC_CMD_TSA_BIND_OUT_SET_KEY msgresponse */ +#define MC_CMD_TSA_BIND_OUT_SET_KEY_LEN 4 +/* The operation completion code. */ +#define MC_CMD_TSA_BIND_OUT_SET_KEY_OP_OFST 0 + +/* MC_CMD_TSA_BIND_OUT_UNBIND msgresponse */ +#define MC_CMD_TSA_BIND_OUT_UNBIND_LEN 8 +/* Same as MC_CMD_ERR field, but included as 0 in success cases */ +#define MC_CMD_TSA_BIND_OUT_UNBIND_RESULT_OFST 0 +/* Extra status information */ +#define MC_CMD_TSA_BIND_OUT_UNBIND_INFO_OFST 4 +/* enum: Unbind successful. */ +#define MC_CMD_TSA_BIND_OUT_UNBIND_OK_UNBOUND 0x0 +/* enum: TSANID mismatch */ +#define MC_CMD_TSA_BIND_OUT_UNBIND_ERR_BAD_TSANID 0x1 +/* enum: Unable to remove the binding ticket from persistent storage. */ +#define MC_CMD_TSA_BIND_OUT_UNBIND_ERR_REMOVE_TICKET 0x2 +/* enum: TSAN is not bound to a binding ticket. */ +#define MC_CMD_TSA_BIND_OUT_UNBIND_ERR_NOT_BOUND 0x3 + + +/***********************************/ +/* MC_CMD_MANAGE_SECURITY_RULESET_CACHE + * Manage the persistent NVRAM cache of security rules created with + * MC_CMD_SET_SECURITY_RULE. Note that the cache is not automatically updated + * as rules are added or removed; the active ruleset must be explicitly + * committed to the cache. The cache may also be explicitly invalidated, + * without affecting the currently active ruleset. When the cache is valid, it + * will be loaded at power on or MC reboot, instead of the default ruleset. + * Rollback of the currently active ruleset to the cached version (when it is + * valid) is also supported. (Medford-only; for use by SolarSecure apps, not + * directly by drivers. See SF-114946-SW.) NOTE - this message definition is + * provisional. It has not yet been used in any released code and may change + * during development. This note will be removed once it is regarded as stable. + */ +#define MC_CMD_MANAGE_SECURITY_RULESET_CACHE 0x11a +#undef MC_CMD_0x11a_PRIVILEGE_CTG + +#define MC_CMD_0x11a_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_MANAGE_SECURITY_RULESET_CACHE_IN msgrequest */ +#define MC_CMD_MANAGE_SECURITY_RULESET_CACHE_IN_LEN 4 +/* the operation to perform */ +#define MC_CMD_MANAGE_SECURITY_RULESET_CACHE_IN_OP_OFST 0 +/* enum: reports the ruleset version that is cached in persistent storage but + * performs no other action + */ +#define MC_CMD_MANAGE_SECURITY_RULESET_CACHE_IN_OP_GET_CACHED_VERSION 0x0 +/* enum: rolls back the active state to the cached version. (May fail with + * ENOENT if there is no valid cached version.) + */ +#define MC_CMD_MANAGE_SECURITY_RULESET_CACHE_IN_OP_ROLLBACK 0x1 +/* enum: commits the active state to the persistent cache */ +#define MC_CMD_MANAGE_SECURITY_RULESET_CACHE_IN_OP_COMMIT 0x2 +/* enum: invalidates the persistent cache without affecting the active state */ +#define MC_CMD_MANAGE_SECURITY_RULESET_CACHE_IN_OP_INVALIDATE 0x3 + +/* MC_CMD_MANAGE_SECURITY_RULESET_CACHE_OUT msgresponse */ +#define MC_CMD_MANAGE_SECURITY_RULESET_CACHE_OUT_LENMIN 5 +#define MC_CMD_MANAGE_SECURITY_RULESET_CACHE_OUT_LENMAX 252 +#define MC_CMD_MANAGE_SECURITY_RULESET_CACHE_OUT_LEN(num) (4+1*(num)) +/* indicates whether the persistent cache is valid (after completion of the + * requested operation in the case of rollback, commit, or invalidate) + */ +#define MC_CMD_MANAGE_SECURITY_RULESET_CACHE_OUT_STATE_OFST 0 +/* enum: persistent cache is invalid (the VERSION field will be empty in this + * case) + */ +#define MC_CMD_MANAGE_SECURITY_RULESET_CACHE_OUT_STATE_INVALID 0x0 +/* enum: persistent cache is valid */ +#define MC_CMD_MANAGE_SECURITY_RULESET_CACHE_OUT_STATE_VALID 0x1 +/* cached ruleset version (after completion of the requested operation, in the + * case of rollback, commit, or invalidate) as an opaque hash value in the same + * form as MC_CMD_GET_SECURITY_RULESET_VERSION_OUT_VERSION + */ +#define MC_CMD_MANAGE_SECURITY_RULESET_CACHE_OUT_VERSION_OFST 4 +#define MC_CMD_MANAGE_SECURITY_RULESET_CACHE_OUT_VERSION_LEN 1 +#define MC_CMD_MANAGE_SECURITY_RULESET_CACHE_OUT_VERSION_MINNUM 1 +#define MC_CMD_MANAGE_SECURITY_RULESET_CACHE_OUT_VERSION_MAXNUM 248 + + +/***********************************/ +/* MC_CMD_NVRAM_PRIVATE_APPEND + * Append a single TLV to the MC_USAGE_TLV partition. Returns MC_CMD_ERR_EEXIST + * if the tag is already present. + */ +#define MC_CMD_NVRAM_PRIVATE_APPEND 0x11c +#undef MC_CMD_0x11c_PRIVILEGE_CTG + +#define MC_CMD_0x11c_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_NVRAM_PRIVATE_APPEND_IN msgrequest */ +#define MC_CMD_NVRAM_PRIVATE_APPEND_IN_LENMIN 9 +#define MC_CMD_NVRAM_PRIVATE_APPEND_IN_LENMAX 252 +#define MC_CMD_NVRAM_PRIVATE_APPEND_IN_LEN(num) (8+1*(num)) +/* The tag to be appended */ +#define MC_CMD_NVRAM_PRIVATE_APPEND_IN_TAG_OFST 0 +/* The length of the data */ +#define MC_CMD_NVRAM_PRIVATE_APPEND_IN_LENGTH_OFST 4 +/* The data to be contained in the TLV structure */ +#define MC_CMD_NVRAM_PRIVATE_APPEND_IN_DATA_BUFFER_OFST 8 +#define MC_CMD_NVRAM_PRIVATE_APPEND_IN_DATA_BUFFER_LEN 1 +#define MC_CMD_NVRAM_PRIVATE_APPEND_IN_DATA_BUFFER_MINNUM 1 +#define MC_CMD_NVRAM_PRIVATE_APPEND_IN_DATA_BUFFER_MAXNUM 244 + +/* MC_CMD_NVRAM_PRIVATE_APPEND_OUT msgresponse */ +#define MC_CMD_NVRAM_PRIVATE_APPEND_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_XPM_VERIFY_CONTENTS + * Verify that the contents of the XPM memory is correct (Medford only). This + * is used during manufacture to check that the XPM memory has been programmed + * correctly at ATE. + */ +#define MC_CMD_XPM_VERIFY_CONTENTS 0x11b +#undef MC_CMD_0x11b_PRIVILEGE_CTG + +#define MC_CMD_0x11b_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_XPM_VERIFY_CONTENTS_IN msgrequest */ +#define MC_CMD_XPM_VERIFY_CONTENTS_IN_LEN 4 +/* Data type to be checked */ +#define MC_CMD_XPM_VERIFY_CONTENTS_IN_DATA_TYPE_OFST 0 + +/* MC_CMD_XPM_VERIFY_CONTENTS_OUT msgresponse */ +#define MC_CMD_XPM_VERIFY_CONTENTS_OUT_LENMIN 12 +#define MC_CMD_XPM_VERIFY_CONTENTS_OUT_LENMAX 252 +#define MC_CMD_XPM_VERIFY_CONTENTS_OUT_LEN(num) (12+1*(num)) +/* Number of sectors found (test builds only) */ +#define MC_CMD_XPM_VERIFY_CONTENTS_OUT_NUM_SECTORS_OFST 0 +/* Number of bytes found (test builds only) */ +#define MC_CMD_XPM_VERIFY_CONTENTS_OUT_NUM_BYTES_OFST 4 +/* Length of signature */ +#define MC_CMD_XPM_VERIFY_CONTENTS_OUT_SIG_LENGTH_OFST 8 +/* Signature */ +#define MC_CMD_XPM_VERIFY_CONTENTS_OUT_SIGNATURE_OFST 12 +#define MC_CMD_XPM_VERIFY_CONTENTS_OUT_SIGNATURE_LEN 1 +#define MC_CMD_XPM_VERIFY_CONTENTS_OUT_SIGNATURE_MINNUM 0 +#define MC_CMD_XPM_VERIFY_CONTENTS_OUT_SIGNATURE_MAXNUM 240 + + +/***********************************/ +/* MC_CMD_SET_EVQ_TMR + * Update the timer load, timer reload and timer mode values for a given EVQ. + * The requested timer values (in TMR_LOAD_REQ_NS and TMR_RELOAD_REQ_NS) will + * be rounded up to the granularity supported by the hardware, then truncated + * to the range supported by the hardware. The resulting value after the + * rounding and truncation will be returned to the caller (in TMR_LOAD_ACT_NS + * and TMR_RELOAD_ACT_NS). + */ +#define MC_CMD_SET_EVQ_TMR 0x120 +#undef MC_CMD_0x120_PRIVILEGE_CTG + +#define MC_CMD_0x120_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_SET_EVQ_TMR_IN msgrequest */ +#define MC_CMD_SET_EVQ_TMR_IN_LEN 16 +/* Function-relative queue instance */ +#define MC_CMD_SET_EVQ_TMR_IN_INSTANCE_OFST 0 +/* Requested value for timer load (in nanoseconds) */ +#define MC_CMD_SET_EVQ_TMR_IN_TMR_LOAD_REQ_NS_OFST 4 +/* Requested value for timer reload (in nanoseconds) */ +#define MC_CMD_SET_EVQ_TMR_IN_TMR_RELOAD_REQ_NS_OFST 8 +/* Timer mode. Meanings as per EVQ_TMR_REG.TC_TIMER_VAL */ +#define MC_CMD_SET_EVQ_TMR_IN_TMR_MODE_OFST 12 +#define MC_CMD_SET_EVQ_TMR_IN_TIMER_MODE_DIS 0x0 /* enum */ +#define MC_CMD_SET_EVQ_TMR_IN_TIMER_MODE_IMMED_START 0x1 /* enum */ +#define MC_CMD_SET_EVQ_TMR_IN_TIMER_MODE_TRIG_START 0x2 /* enum */ +#define MC_CMD_SET_EVQ_TMR_IN_TIMER_MODE_INT_HLDOFF 0x3 /* enum */ + +/* MC_CMD_SET_EVQ_TMR_OUT msgresponse */ +#define MC_CMD_SET_EVQ_TMR_OUT_LEN 8 +/* Actual value for timer load (in nanoseconds) */ +#define MC_CMD_SET_EVQ_TMR_OUT_TMR_LOAD_ACT_NS_OFST 0 +/* Actual value for timer reload (in nanoseconds) */ +#define MC_CMD_SET_EVQ_TMR_OUT_TMR_RELOAD_ACT_NS_OFST 4 + + +/***********************************/ +/* MC_CMD_GET_EVQ_TMR_PROPERTIES + * Query properties about the event queue timers. + */ +#define MC_CMD_GET_EVQ_TMR_PROPERTIES 0x122 +#undef MC_CMD_0x122_PRIVILEGE_CTG + +#define MC_CMD_0x122_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_GET_EVQ_TMR_PROPERTIES_IN msgrequest */ +#define MC_CMD_GET_EVQ_TMR_PROPERTIES_IN_LEN 0 + +/* MC_CMD_GET_EVQ_TMR_PROPERTIES_OUT msgresponse */ +#define MC_CMD_GET_EVQ_TMR_PROPERTIES_OUT_LEN 36 +/* Reserved for future use. */ +#define MC_CMD_GET_EVQ_TMR_PROPERTIES_OUT_FLAGS_OFST 0 +/* For timers updated via writes to EVQ_TMR_REG, this is the time interval (in + * nanoseconds) for each increment of the timer load/reload count. The + * requested duration of a timer is this value multiplied by the timer + * load/reload count. + */ +#define MC_CMD_GET_EVQ_TMR_PROPERTIES_OUT_TMR_REG_NS_PER_COUNT_OFST 4 +/* For timers updated via writes to EVQ_TMR_REG, this is the maximum value + * allowed for timer load/reload counts. + */ +#define MC_CMD_GET_EVQ_TMR_PROPERTIES_OUT_TMR_REG_MAX_COUNT_OFST 8 +/* For timers updated via writes to EVQ_TMR_REG, timer load/reload counts not a + * multiple of this step size will be rounded in an implementation defined + * manner. + */ +#define MC_CMD_GET_EVQ_TMR_PROPERTIES_OUT_TMR_REG_STEP_OFST 12 +/* Maximum timer duration (in nanoseconds) for timers updated via MCDI. Only + * meaningful if MC_CMD_SET_EVQ_TMR is implemented. + */ +#define MC_CMD_GET_EVQ_TMR_PROPERTIES_OUT_MCDI_TMR_MAX_NS_OFST 16 +/* Timer durations requested via MCDI that are not a multiple of this step size + * will be rounded up. Only meaningful if MC_CMD_SET_EVQ_TMR is implemented. + */ +#define MC_CMD_GET_EVQ_TMR_PROPERTIES_OUT_MCDI_TMR_STEP_NS_OFST 20 +/* For timers updated using the bug35388 workaround, this is the time interval + * (in nanoseconds) for each increment of the timer load/reload count. The + * requested duration of a timer is this value multiplied by the timer + * load/reload count. This field is only meaningful if the bug35388 workaround + * is enabled. + */ +#define MC_CMD_GET_EVQ_TMR_PROPERTIES_OUT_BUG35388_TMR_NS_PER_COUNT_OFST 24 +/* For timers updated using the bug35388 workaround, this is the maximum value + * allowed for timer load/reload counts. This field is only meaningful if the + * bug35388 workaround is enabled. + */ +#define MC_CMD_GET_EVQ_TMR_PROPERTIES_OUT_BUG35388_TMR_MAX_COUNT_OFST 28 +/* For timers updated using the bug35388 workaround, timer load/reload counts + * not a multiple of this step size will be rounded in an implementation + * defined manner. This field is only meaningful if the bug35388 workaround is + * enabled. + */ +#define MC_CMD_GET_EVQ_TMR_PROPERTIES_OUT_BUG35388_TMR_STEP_OFST 32 + + +/***********************************/ +/* MC_CMD_ALLOCATE_TX_VFIFO_CP + * When we use the TX_vFIFO_ULL mode, we can allocate common pools using the + * non used switch buffers. + */ +#define MC_CMD_ALLOCATE_TX_VFIFO_CP 0x11d +#undef MC_CMD_0x11d_PRIVILEGE_CTG + +#define MC_CMD_0x11d_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_ALLOCATE_TX_VFIFO_CP_IN msgrequest */ +#define MC_CMD_ALLOCATE_TX_VFIFO_CP_IN_LEN 20 +/* Desired instance. Must be set to a specific instance, which is a function + * local queue index. + */ +#define MC_CMD_ALLOCATE_TX_VFIFO_CP_IN_INSTANCE_OFST 0 +/* Will the common pool be used as TX_vFIFO_ULL (1) */ +#define MC_CMD_ALLOCATE_TX_VFIFO_CP_IN_MODE_OFST 4 +#define MC_CMD_ALLOCATE_TX_VFIFO_CP_IN_ENABLED 0x1 /* enum */ +/* enum: Using this interface without TX_vFIFO_ULL is not supported for now */ +#define MC_CMD_ALLOCATE_TX_VFIFO_CP_IN_DISABLED 0x0 +/* Number of buffers to reserve for the common pool */ +#define MC_CMD_ALLOCATE_TX_VFIFO_CP_IN_SIZE_OFST 8 +/* TX datapath to which the Common Pool is connected to. */ +#define MC_CMD_ALLOCATE_TX_VFIFO_CP_IN_INGRESS_OFST 12 +/* enum: Extracts information from function */ +#define MC_CMD_ALLOCATE_TX_VFIFO_CP_IN_USE_FUNCTION_VALUE -0x1 +/* Network port or RX Engine to which the common pool connects. */ +#define MC_CMD_ALLOCATE_TX_VFIFO_CP_IN_EGRESS_OFST 16 +/* enum: Extracts information from function */ +/* MC_CMD_ALLOCATE_TX_VFIFO_CP_IN_USE_FUNCTION_VALUE -0x1 */ +#define MC_CMD_ALLOCATE_TX_VFIFO_CP_IN_PORT0 0x0 /* enum */ +#define MC_CMD_ALLOCATE_TX_VFIFO_CP_IN_PORT1 0x1 /* enum */ +#define MC_CMD_ALLOCATE_TX_VFIFO_CP_IN_PORT2 0x2 /* enum */ +#define MC_CMD_ALLOCATE_TX_VFIFO_CP_IN_PORT3 0x3 /* enum */ +/* enum: To enable Switch loopback with Rx engine 0 */ +#define MC_CMD_ALLOCATE_TX_VFIFO_CP_IN_RX_ENGINE0 0x4 +/* enum: To enable Switch loopback with Rx engine 1 */ +#define MC_CMD_ALLOCATE_TX_VFIFO_CP_IN_RX_ENGINE1 0x5 + +/* MC_CMD_ALLOCATE_TX_VFIFO_CP_OUT msgresponse */ +#define MC_CMD_ALLOCATE_TX_VFIFO_CP_OUT_LEN 4 +/* ID of the common pool allocated */ +#define MC_CMD_ALLOCATE_TX_VFIFO_CP_OUT_CP_ID_OFST 0 + + +/***********************************/ +/* MC_CMD_ALLOCATE_TX_VFIFO_VFIFO + * When we use the TX_vFIFO_ULL mode, we can allocate vFIFOs using the + * previously allocated common pools. + */ +#define MC_CMD_ALLOCATE_TX_VFIFO_VFIFO 0x11e +#undef MC_CMD_0x11e_PRIVILEGE_CTG + +#define MC_CMD_0x11e_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_ALLOCATE_TX_VFIFO_VFIFO_IN msgrequest */ +#define MC_CMD_ALLOCATE_TX_VFIFO_VFIFO_IN_LEN 20 +/* Common pool previously allocated to which the new vFIFO will be associated + */ +#define MC_CMD_ALLOCATE_TX_VFIFO_VFIFO_IN_CP_OFST 0 +/* Port or RX engine to associate the vFIFO egress */ +#define MC_CMD_ALLOCATE_TX_VFIFO_VFIFO_IN_EGRESS_OFST 4 +/* enum: Extracts information from common pool */ +#define MC_CMD_ALLOCATE_TX_VFIFO_VFIFO_IN_USE_CP_VALUE -0x1 +#define MC_CMD_ALLOCATE_TX_VFIFO_VFIFO_IN_PORT0 0x0 /* enum */ +#define MC_CMD_ALLOCATE_TX_VFIFO_VFIFO_IN_PORT1 0x1 /* enum */ +#define MC_CMD_ALLOCATE_TX_VFIFO_VFIFO_IN_PORT2 0x2 /* enum */ +#define MC_CMD_ALLOCATE_TX_VFIFO_VFIFO_IN_PORT3 0x3 /* enum */ +/* enum: To enable Switch loopback with Rx engine 0 */ +#define MC_CMD_ALLOCATE_TX_VFIFO_VFIFO_IN_RX_ENGINE0 0x4 +/* enum: To enable Switch loopback with Rx engine 1 */ +#define MC_CMD_ALLOCATE_TX_VFIFO_VFIFO_IN_RX_ENGINE1 0x5 +/* Minimum number of buffers that the pool must have */ +#define MC_CMD_ALLOCATE_TX_VFIFO_VFIFO_IN_SIZE_OFST 8 +/* enum: Do not check the space available */ +#define MC_CMD_ALLOCATE_TX_VFIFO_VFIFO_IN_NO_MINIMUM 0x0 +/* Will the vFIFO be used as TX_vFIFO_ULL */ +#define MC_CMD_ALLOCATE_TX_VFIFO_VFIFO_IN_MODE_OFST 12 +/* Network priority of the vFIFO,if applicable */ +#define MC_CMD_ALLOCATE_TX_VFIFO_VFIFO_IN_PRIORITY_OFST 16 +/* enum: Search for the lowest unused priority */ +#define MC_CMD_ALLOCATE_TX_VFIFO_VFIFO_IN_LOWEST_AVAILABLE -0x1 + +/* MC_CMD_ALLOCATE_TX_VFIFO_VFIFO_OUT msgresponse */ +#define MC_CMD_ALLOCATE_TX_VFIFO_VFIFO_OUT_LEN 8 +/* Short vFIFO ID */ +#define MC_CMD_ALLOCATE_TX_VFIFO_VFIFO_OUT_VID_OFST 0 +/* Network priority of the vFIFO */ +#define MC_CMD_ALLOCATE_TX_VFIFO_VFIFO_OUT_PRIORITY_OFST 4 + + +/***********************************/ +/* MC_CMD_TEARDOWN_TX_VFIFO_VF + * This interface clears the configuration of the given vFIFO and leaves it + * ready to be re-used. + */ +#define MC_CMD_TEARDOWN_TX_VFIFO_VF 0x11f +#undef MC_CMD_0x11f_PRIVILEGE_CTG + +#define MC_CMD_0x11f_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_TEARDOWN_TX_VFIFO_VF_IN msgrequest */ +#define MC_CMD_TEARDOWN_TX_VFIFO_VF_IN_LEN 4 +/* Short vFIFO ID */ +#define MC_CMD_TEARDOWN_TX_VFIFO_VF_IN_VFIFO_OFST 0 + +/* MC_CMD_TEARDOWN_TX_VFIFO_VF_OUT msgresponse */ +#define MC_CMD_TEARDOWN_TX_VFIFO_VF_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_DEALLOCATE_TX_VFIFO_CP + * This interface clears the configuration of the given common pool and leaves + * it ready to be re-used. + */ +#define MC_CMD_DEALLOCATE_TX_VFIFO_CP 0x121 +#undef MC_CMD_0x121_PRIVILEGE_CTG + +#define MC_CMD_0x121_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_DEALLOCATE_TX_VFIFO_CP_IN msgrequest */ +#define MC_CMD_DEALLOCATE_TX_VFIFO_CP_IN_LEN 4 +/* Common pool ID given when pool allocated */ +#define MC_CMD_DEALLOCATE_TX_VFIFO_CP_IN_POOL_ID_OFST 0 + +/* MC_CMD_DEALLOCATE_TX_VFIFO_CP_OUT msgresponse */ +#define MC_CMD_DEALLOCATE_TX_VFIFO_CP_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_REKEY + * This request causes the NIC to generate a new per-NIC key and program it + * into the write-once memory. During the process all flash partitions that are + * protected with a CMAC are verified with the old per-NIC key and then signed + * with the new per-NIC key. If the NIC has already reached its rekey limit the + * REKEY op will return MC_CMD_ERR_ERANGE. The REKEY op may block until + * completion or it may return 0 and continue processing, therefore the caller + * must poll at least once to confirm that the rekeying has completed. The POLL + * operation returns MC_CMD_ERR_EBUSY if the rekey process is still running + * otherwise it will return the result of the last completed rekey operation, + * or 0 if there has not been a previous rekey. + */ +#define MC_CMD_REKEY 0x123 +#undef MC_CMD_0x123_PRIVILEGE_CTG + +#define MC_CMD_0x123_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_REKEY_IN msgrequest */ +#define MC_CMD_REKEY_IN_LEN 4 +/* the type of operation requested */ +#define MC_CMD_REKEY_IN_OP_OFST 0 +/* enum: Start the rekeying operation */ +#define MC_CMD_REKEY_IN_OP_REKEY 0x0 +/* enum: Poll for completion of the rekeying operation */ +#define MC_CMD_REKEY_IN_OP_POLL 0x1 + +/* MC_CMD_REKEY_OUT msgresponse */ +#define MC_CMD_REKEY_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_SWITCH_GET_UNASSIGNED_BUFFERS + * This interface allows the host to find out how many common pool buffers are + * not yet assigned. + */ +#define MC_CMD_SWITCH_GET_UNASSIGNED_BUFFERS 0x124 +#undef MC_CMD_0x124_PRIVILEGE_CTG + +#define MC_CMD_0x124_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_SWITCH_GET_UNASSIGNED_BUFFERS_IN msgrequest */ +#define MC_CMD_SWITCH_GET_UNASSIGNED_BUFFERS_IN_LEN 0 + +/* MC_CMD_SWITCH_GET_UNASSIGNED_BUFFERS_OUT msgresponse */ +#define MC_CMD_SWITCH_GET_UNASSIGNED_BUFFERS_OUT_LEN 8 +/* Available buffers for the ENG to NET vFIFOs. */ +#define MC_CMD_SWITCH_GET_UNASSIGNED_BUFFERS_OUT_NET_OFST 0 +/* Available buffers for the ENG to ENG and NET to ENG vFIFOs. */ +#define MC_CMD_SWITCH_GET_UNASSIGNED_BUFFERS_OUT_ENG_OFST 4 + + +/***********************************/ +/* MC_CMD_SET_SECURITY_FUSES + * Change the security level of the adapter by setting bits in the write-once + * memory. The firmware maps each flag in the message to a set of one or more + * hardware-defined or software-defined bits and sets these bits in the write- + * once memory. For Medford the hardware-defined bits are defined in + * SF-112079-PS 5.3, the software-defined bits are defined in xpm.h. Returns 0 + * if all of the required bits were set and returns MC_CMD_ERR_EIO if any of + * the required bits were not set. + */ +#define MC_CMD_SET_SECURITY_FUSES 0x126 +#undef MC_CMD_0x126_PRIVILEGE_CTG + +#define MC_CMD_0x126_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_SET_SECURITY_FUSES_IN msgrequest */ +#define MC_CMD_SET_SECURITY_FUSES_IN_LEN 4 +/* Flags specifying what type of security features are being set */ +#define MC_CMD_SET_SECURITY_FUSES_IN_FLAGS_OFST 0 +#define MC_CMD_SET_SECURITY_FUSES_IN_SECURE_BOOT_LBN 0 +#define MC_CMD_SET_SECURITY_FUSES_IN_SECURE_BOOT_WIDTH 1 +#define MC_CMD_SET_SECURITY_FUSES_IN_REJECT_TEST_SIGNED_LBN 1 +#define MC_CMD_SET_SECURITY_FUSES_IN_REJECT_TEST_SIGNED_WIDTH 1 + +/* MC_CMD_SET_SECURITY_FUSES_OUT msgresponse */ +#define MC_CMD_SET_SECURITY_FUSES_OUT_LEN 0 + +#endif /* _SIENA_MC_DRIVER_PCOL_H */ +/*! \cidoxg_end */ diff --git a/dpdk/drivers/net/sfc/base/efx_regs_pci.h b/dpdk/drivers/net/sfc/base/efx_regs_pci.h new file mode 100644 index 00000000..f90f9565 --- /dev/null +++ b/dpdk/drivers/net/sfc/base/efx_regs_pci.h @@ -0,0 +1,2356 @@ +/* + * Copyright (c) 2007-2016 Solarflare Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are + * those of the authors and should not be interpreted as representing official + * policies, either expressed or implied, of the FreeBSD Project. + */ + +#ifndef _SYS_EFX_REGS_PCI_H +#define _SYS_EFX_REGS_PCI_H + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * PC_VEND_ID_REG(16bit): + * Vendor ID register + */ + +#define PCR_AZ_VEND_ID_REG 0x00000000 +/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */ + +#define PCRF_AZ_VEND_ID_LBN 0 +#define PCRF_AZ_VEND_ID_WIDTH 16 + + +/* + * PC_DEV_ID_REG(16bit): + * Device ID register + */ + +#define PCR_AZ_DEV_ID_REG 0x00000002 +/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */ + +#define PCRF_AZ_DEV_ID_LBN 0 +#define PCRF_AZ_DEV_ID_WIDTH 16 + + +/* + * PC_CMD_REG(16bit): + * Command register + */ + +#define PCR_AZ_CMD_REG 0x00000004 +/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */ + +#define PCRF_AZ_INTX_DIS_LBN 10 +#define PCRF_AZ_INTX_DIS_WIDTH 1 +#define PCRF_AZ_FB2B_EN_LBN 9 +#define PCRF_AZ_FB2B_EN_WIDTH 1 +#define PCRF_AZ_SERR_EN_LBN 8 +#define PCRF_AZ_SERR_EN_WIDTH 1 +#define PCRF_AZ_IDSEL_CTL_LBN 7 +#define PCRF_AZ_IDSEL_CTL_WIDTH 1 +#define PCRF_AZ_PERR_EN_LBN 6 +#define PCRF_AZ_PERR_EN_WIDTH 1 +#define PCRF_AZ_VGA_PAL_SNP_LBN 5 +#define PCRF_AZ_VGA_PAL_SNP_WIDTH 1 +#define PCRF_AZ_MWI_EN_LBN 4 +#define PCRF_AZ_MWI_EN_WIDTH 1 +#define PCRF_AZ_SPEC_CYC_LBN 3 +#define PCRF_AZ_SPEC_CYC_WIDTH 1 +#define PCRF_AZ_MST_EN_LBN 2 +#define PCRF_AZ_MST_EN_WIDTH 1 +#define PCRF_AZ_MEM_EN_LBN 1 +#define PCRF_AZ_MEM_EN_WIDTH 1 +#define PCRF_AZ_IO_EN_LBN 0 +#define PCRF_AZ_IO_EN_WIDTH 1 + + +/* + * PC_STAT_REG(16bit): + * Status register + */ + +#define PCR_AZ_STAT_REG 0x00000006 +/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */ + +#define PCRF_AZ_DET_PERR_LBN 15 +#define PCRF_AZ_DET_PERR_WIDTH 1 +#define PCRF_AZ_SIG_SERR_LBN 14 +#define PCRF_AZ_SIG_SERR_WIDTH 1 +#define PCRF_AZ_GOT_MABRT_LBN 13 +#define PCRF_AZ_GOT_MABRT_WIDTH 1 +#define PCRF_AZ_GOT_TABRT_LBN 12 +#define PCRF_AZ_GOT_TABRT_WIDTH 1 +#define PCRF_AZ_SIG_TABRT_LBN 11 +#define PCRF_AZ_SIG_TABRT_WIDTH 1 +#define PCRF_AZ_DEVSEL_TIM_LBN 9 +#define PCRF_AZ_DEVSEL_TIM_WIDTH 2 +#define PCRF_AZ_MDAT_PERR_LBN 8 +#define PCRF_AZ_MDAT_PERR_WIDTH 1 +#define PCRF_AZ_FB2B_CAP_LBN 7 +#define PCRF_AZ_FB2B_CAP_WIDTH 1 +#define PCRF_AZ_66MHZ_CAP_LBN 5 +#define PCRF_AZ_66MHZ_CAP_WIDTH 1 +#define PCRF_AZ_CAP_LIST_LBN 4 +#define PCRF_AZ_CAP_LIST_WIDTH 1 +#define PCRF_AZ_INTX_STAT_LBN 3 +#define PCRF_AZ_INTX_STAT_WIDTH 1 + + +/* + * PC_REV_ID_REG(8bit): + * Class code & revision ID register + */ + +#define PCR_AZ_REV_ID_REG 0x00000008 +/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */ + +#define PCRF_AZ_REV_ID_LBN 0 +#define PCRF_AZ_REV_ID_WIDTH 8 + + +/* + * PC_CC_REG(24bit): + * Class code register + */ + +#define PCR_AZ_CC_REG 0x00000009 +/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */ + +#define PCRF_AZ_BASE_CC_LBN 16 +#define PCRF_AZ_BASE_CC_WIDTH 8 +#define PCRF_AZ_SUB_CC_LBN 8 +#define PCRF_AZ_SUB_CC_WIDTH 8 +#define PCRF_AZ_PROG_IF_LBN 0 +#define PCRF_AZ_PROG_IF_WIDTH 8 + + +/* + * PC_CACHE_LSIZE_REG(8bit): + * Cache line size + */ + +#define PCR_AZ_CACHE_LSIZE_REG 0x0000000c +/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */ + +#define PCRF_AZ_CACHE_LSIZE_LBN 0 +#define PCRF_AZ_CACHE_LSIZE_WIDTH 8 + + +/* + * PC_MST_LAT_REG(8bit): + * Master latency timer register + */ + +#define PCR_AZ_MST_LAT_REG 0x0000000d +/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */ + +#define PCRF_AZ_MST_LAT_LBN 0 +#define PCRF_AZ_MST_LAT_WIDTH 8 + + +/* + * PC_HDR_TYPE_REG(8bit): + * Header type register + */ + +#define PCR_AZ_HDR_TYPE_REG 0x0000000e +/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */ + +#define PCRF_AZ_MULT_FUNC_LBN 7 +#define PCRF_AZ_MULT_FUNC_WIDTH 1 +#define PCRF_AZ_TYPE_LBN 0 +#define PCRF_AZ_TYPE_WIDTH 7 + + +/* + * PC_BIST_REG(8bit): + * BIST register + */ + +#define PCR_AZ_BIST_REG 0x0000000f +/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */ + +#define PCRF_AZ_BIST_LBN 0 +#define PCRF_AZ_BIST_WIDTH 8 + + +/* + * PC_BAR0_REG(32bit): + * Primary function base address register 0 + */ + +#define PCR_AZ_BAR0_REG 0x00000010 +/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */ + +#define PCRF_AZ_BAR0_LBN 4 +#define PCRF_AZ_BAR0_WIDTH 28 +#define PCRF_AZ_BAR0_PREF_LBN 3 +#define PCRF_AZ_BAR0_PREF_WIDTH 1 +#define PCRF_AZ_BAR0_TYPE_LBN 1 +#define PCRF_AZ_BAR0_TYPE_WIDTH 2 +#define PCRF_AZ_BAR0_IOM_LBN 0 +#define PCRF_AZ_BAR0_IOM_WIDTH 1 + + +/* + * PC_BAR1_REG(32bit): + * Primary function base address register 1, BAR1 is not implemented so read only. + */ + +#define PCR_DZ_BAR1_REG 0x00000014 +/* hunta0=pci_f0_config */ + +#define PCRF_DZ_BAR1_LBN 0 +#define PCRF_DZ_BAR1_WIDTH 32 + + +/* + * PC_BAR2_LO_REG(32bit): + * Primary function base address register 2 low bits + */ + +#define PCR_AZ_BAR2_LO_REG 0x00000018 +/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */ + +#define PCRF_AZ_BAR2_LO_LBN 4 +#define PCRF_AZ_BAR2_LO_WIDTH 28 +#define PCRF_AZ_BAR2_PREF_LBN 3 +#define PCRF_AZ_BAR2_PREF_WIDTH 1 +#define PCRF_AZ_BAR2_TYPE_LBN 1 +#define PCRF_AZ_BAR2_TYPE_WIDTH 2 +#define PCRF_AZ_BAR2_IOM_LBN 0 +#define PCRF_AZ_BAR2_IOM_WIDTH 1 + + +/* + * PC_BAR2_HI_REG(32bit): + * Primary function base address register 2 high bits + */ + +#define PCR_AZ_BAR2_HI_REG 0x0000001c +/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */ + +#define PCRF_AZ_BAR2_HI_LBN 0 +#define PCRF_AZ_BAR2_HI_WIDTH 32 + + +/* + * PC_BAR4_LO_REG(32bit): + * Primary function base address register 2 low bits + */ + +#define PCR_CZ_BAR4_LO_REG 0x00000020 +/* sienaa0,hunta0=pci_f0_config */ + +#define PCRF_CZ_BAR4_LO_LBN 4 +#define PCRF_CZ_BAR4_LO_WIDTH 28 +#define PCRF_CZ_BAR4_PREF_LBN 3 +#define PCRF_CZ_BAR4_PREF_WIDTH 1 +#define PCRF_CZ_BAR4_TYPE_LBN 1 +#define PCRF_CZ_BAR4_TYPE_WIDTH 2 +#define PCRF_CZ_BAR4_IOM_LBN 0 +#define PCRF_CZ_BAR4_IOM_WIDTH 1 + + +/* + * PC_BAR4_HI_REG(32bit): + * Primary function base address register 2 high bits + */ + +#define PCR_CZ_BAR4_HI_REG 0x00000024 +/* sienaa0,hunta0=pci_f0_config */ + +#define PCRF_CZ_BAR4_HI_LBN 0 +#define PCRF_CZ_BAR4_HI_WIDTH 32 + + +/* + * PC_SS_VEND_ID_REG(16bit): + * Sub-system vendor ID register + */ + +#define PCR_AZ_SS_VEND_ID_REG 0x0000002c +/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */ + +#define PCRF_AZ_SS_VEND_ID_LBN 0 +#define PCRF_AZ_SS_VEND_ID_WIDTH 16 + + +/* + * PC_SS_ID_REG(16bit): + * Sub-system ID register + */ + +#define PCR_AZ_SS_ID_REG 0x0000002e +/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */ + +#define PCRF_AZ_SS_ID_LBN 0 +#define PCRF_AZ_SS_ID_WIDTH 16 + + +/* + * PC_EXPROM_BAR_REG(32bit): + * Expansion ROM base address register + */ + +#define PCR_AZ_EXPROM_BAR_REG 0x00000030 +/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */ + +#define PCRF_AZ_EXPROM_BAR_LBN 11 +#define PCRF_AZ_EXPROM_BAR_WIDTH 21 +#define PCRF_AB_EXPROM_MIN_SIZE_LBN 2 +#define PCRF_AB_EXPROM_MIN_SIZE_WIDTH 9 +#define PCRF_CZ_EXPROM_MIN_SIZE_LBN 1 +#define PCRF_CZ_EXPROM_MIN_SIZE_WIDTH 10 +#define PCRF_AB_EXPROM_FEATURE_ENABLE_LBN 1 +#define PCRF_AB_EXPROM_FEATURE_ENABLE_WIDTH 1 +#define PCRF_AZ_EXPROM_EN_LBN 0 +#define PCRF_AZ_EXPROM_EN_WIDTH 1 + + +/* + * PC_CAP_PTR_REG(8bit): + * Capability pointer register + */ + +#define PCR_AZ_CAP_PTR_REG 0x00000034 +/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */ + +#define PCRF_AZ_CAP_PTR_LBN 0 +#define PCRF_AZ_CAP_PTR_WIDTH 8 + + +/* + * PC_INT_LINE_REG(8bit): + * Interrupt line register + */ + +#define PCR_AZ_INT_LINE_REG 0x0000003c +/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */ + +#define PCRF_AZ_INT_LINE_LBN 0 +#define PCRF_AZ_INT_LINE_WIDTH 8 + + +/* + * PC_INT_PIN_REG(8bit): + * Interrupt pin register + */ + +#define PCR_AZ_INT_PIN_REG 0x0000003d +/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */ + +#define PCRF_AZ_INT_PIN_LBN 0 +#define PCRF_AZ_INT_PIN_WIDTH 8 +#define PCFE_DZ_INTPIN_INTD 4 +#define PCFE_DZ_INTPIN_INTC 3 +#define PCFE_DZ_INTPIN_INTB 2 +#define PCFE_DZ_INTPIN_INTA 1 + + +/* + * PC_PM_CAP_ID_REG(8bit): + * Power management capability ID + */ + +#define PCR_AZ_PM_CAP_ID_REG 0x00000040 +/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */ + +#define PCRF_AZ_PM_CAP_ID_LBN 0 +#define PCRF_AZ_PM_CAP_ID_WIDTH 8 + + +/* + * PC_PM_NXT_PTR_REG(8bit): + * Power management next item pointer + */ + +#define PCR_AZ_PM_NXT_PTR_REG 0x00000041 +/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */ + +#define PCRF_AZ_PM_NXT_PTR_LBN 0 +#define PCRF_AZ_PM_NXT_PTR_WIDTH 8 + + +/* + * PC_PM_CAP_REG(16bit): + * Power management capabilities register + */ + +#define PCR_AZ_PM_CAP_REG 0x00000042 +/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */ + +#define PCRF_AZ_PM_PME_SUPT_LBN 11 +#define PCRF_AZ_PM_PME_SUPT_WIDTH 5 +#define PCRF_AZ_PM_D2_SUPT_LBN 10 +#define PCRF_AZ_PM_D2_SUPT_WIDTH 1 +#define PCRF_AZ_PM_D1_SUPT_LBN 9 +#define PCRF_AZ_PM_D1_SUPT_WIDTH 1 +#define PCRF_AZ_PM_AUX_CURR_LBN 6 +#define PCRF_AZ_PM_AUX_CURR_WIDTH 3 +#define PCRF_AZ_PM_DSI_LBN 5 +#define PCRF_AZ_PM_DSI_WIDTH 1 +#define PCRF_AZ_PM_PME_CLK_LBN 3 +#define PCRF_AZ_PM_PME_CLK_WIDTH 1 +#define PCRF_AZ_PM_PME_VER_LBN 0 +#define PCRF_AZ_PM_PME_VER_WIDTH 3 + + +/* + * PC_PM_CS_REG(16bit): + * Power management control & status register + */ + +#define PCR_AZ_PM_CS_REG 0x00000044 +/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */ + +#define PCRF_AZ_PM_PME_STAT_LBN 15 +#define PCRF_AZ_PM_PME_STAT_WIDTH 1 +#define PCRF_AZ_PM_DAT_SCALE_LBN 13 +#define PCRF_AZ_PM_DAT_SCALE_WIDTH 2 +#define PCRF_AZ_PM_DAT_SEL_LBN 9 +#define PCRF_AZ_PM_DAT_SEL_WIDTH 4 +#define PCRF_AZ_PM_PME_EN_LBN 8 +#define PCRF_AZ_PM_PME_EN_WIDTH 1 +#define PCRF_CZ_NO_SOFT_RESET_LBN 3 +#define PCRF_CZ_NO_SOFT_RESET_WIDTH 1 +#define PCRF_AZ_PM_PWR_ST_LBN 0 +#define PCRF_AZ_PM_PWR_ST_WIDTH 2 + + +/* + * PC_MSI_CAP_ID_REG(8bit): + * MSI capability ID + */ + +#define PCR_AZ_MSI_CAP_ID_REG 0x00000050 +/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */ + +#define PCRF_AZ_MSI_CAP_ID_LBN 0 +#define PCRF_AZ_MSI_CAP_ID_WIDTH 8 + + +/* + * PC_MSI_NXT_PTR_REG(8bit): + * MSI next item pointer + */ + +#define PCR_AZ_MSI_NXT_PTR_REG 0x00000051 +/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */ + +#define PCRF_AZ_MSI_NXT_PTR_LBN 0 +#define PCRF_AZ_MSI_NXT_PTR_WIDTH 8 + + +/* + * PC_MSI_CTL_REG(16bit): + * MSI control register + */ + +#define PCR_AZ_MSI_CTL_REG 0x00000052 +/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */ + +#define PCRF_AZ_MSI_64_EN_LBN 7 +#define PCRF_AZ_MSI_64_EN_WIDTH 1 +#define PCRF_AZ_MSI_MULT_MSG_EN_LBN 4 +#define PCRF_AZ_MSI_MULT_MSG_EN_WIDTH 3 +#define PCRF_AZ_MSI_MULT_MSG_CAP_LBN 1 +#define PCRF_AZ_MSI_MULT_MSG_CAP_WIDTH 3 +#define PCRF_AZ_MSI_EN_LBN 0 +#define PCRF_AZ_MSI_EN_WIDTH 1 + + +/* + * PC_MSI_ADR_LO_REG(32bit): + * MSI low 32 bits address register + */ + +#define PCR_AZ_MSI_ADR_LO_REG 0x00000054 +/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */ + +#define PCRF_AZ_MSI_ADR_LO_LBN 2 +#define PCRF_AZ_MSI_ADR_LO_WIDTH 30 + + +/* + * PC_MSI_ADR_HI_REG(32bit): + * MSI high 32 bits address register + */ + +#define PCR_AZ_MSI_ADR_HI_REG 0x00000058 +/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */ + +#define PCRF_AZ_MSI_ADR_HI_LBN 0 +#define PCRF_AZ_MSI_ADR_HI_WIDTH 32 + + +/* + * PC_MSI_DAT_REG(16bit): + * MSI data register + */ + +#define PCR_AZ_MSI_DAT_REG 0x0000005c +/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */ + +#define PCRF_AZ_MSI_DAT_LBN 0 +#define PCRF_AZ_MSI_DAT_WIDTH 16 + + +/* + * PC_PCIE_CAP_LIST_REG(16bit): + * PCIe capability list register + */ + +#define PCR_AB_PCIE_CAP_LIST_REG 0x00000060 +/* falcona0,falconb0=pci_f0_config */ + +#define PCR_CZ_PCIE_CAP_LIST_REG 0x00000070 +/* sienaa0,hunta0=pci_f0_config */ + +#define PCRF_AZ_PCIE_NXT_PTR_LBN 8 +#define PCRF_AZ_PCIE_NXT_PTR_WIDTH 8 +#define PCRF_AZ_PCIE_CAP_ID_LBN 0 +#define PCRF_AZ_PCIE_CAP_ID_WIDTH 8 + + +/* + * PC_PCIE_CAP_REG(16bit): + * PCIe capability register + */ + +#define PCR_AB_PCIE_CAP_REG 0x00000062 +/* falcona0,falconb0=pci_f0_config */ + +#define PCR_CZ_PCIE_CAP_REG 0x00000072 +/* sienaa0,hunta0=pci_f0_config */ + +#define PCRF_AZ_PCIE_INT_MSG_NUM_LBN 9 +#define PCRF_AZ_PCIE_INT_MSG_NUM_WIDTH 5 +#define PCRF_AZ_PCIE_SLOT_IMP_LBN 8 +#define PCRF_AZ_PCIE_SLOT_IMP_WIDTH 1 +#define PCRF_AZ_PCIE_DEV_PORT_TYPE_LBN 4 +#define PCRF_AZ_PCIE_DEV_PORT_TYPE_WIDTH 4 +#define PCRF_AZ_PCIE_CAP_VER_LBN 0 +#define PCRF_AZ_PCIE_CAP_VER_WIDTH 4 + + +/* + * PC_DEV_CAP_REG(32bit): + * PCIe device capabilities register + */ + +#define PCR_AB_DEV_CAP_REG 0x00000064 +/* falcona0,falconb0=pci_f0_config */ + +#define PCR_CZ_DEV_CAP_REG 0x00000074 +/* sienaa0=pci_f0_config,hunta0=pci_f0_config */ + +#define PCRF_CZ_CAP_FN_LEVEL_RESET_LBN 28 +#define PCRF_CZ_CAP_FN_LEVEL_RESET_WIDTH 1 +#define PCRF_AZ_CAP_SLOT_PWR_SCL_LBN 26 +#define PCRF_AZ_CAP_SLOT_PWR_SCL_WIDTH 2 +#define PCRF_AZ_CAP_SLOT_PWR_VAL_LBN 18 +#define PCRF_AZ_CAP_SLOT_PWR_VAL_WIDTH 8 +#define PCRF_CZ_ROLE_BASE_ERR_REPORTING_LBN 15 +#define PCRF_CZ_ROLE_BASE_ERR_REPORTING_WIDTH 1 +#define PCRF_AB_PWR_IND_LBN 14 +#define PCRF_AB_PWR_IND_WIDTH 1 +#define PCRF_AB_ATTN_IND_LBN 13 +#define PCRF_AB_ATTN_IND_WIDTH 1 +#define PCRF_AB_ATTN_BUTTON_LBN 12 +#define PCRF_AB_ATTN_BUTTON_WIDTH 1 +#define PCRF_AZ_ENDPT_L1_LAT_LBN 9 +#define PCRF_AZ_ENDPT_L1_LAT_WIDTH 3 +#define PCRF_AZ_ENDPT_L0_LAT_LBN 6 +#define PCRF_AZ_ENDPT_L0_LAT_WIDTH 3 +#define PCRF_AZ_TAG_FIELD_LBN 5 +#define PCRF_AZ_TAG_FIELD_WIDTH 1 +#define PCRF_AZ_PHAN_FUNC_LBN 3 +#define PCRF_AZ_PHAN_FUNC_WIDTH 2 +#define PCRF_AZ_MAX_PAYL_SIZE_SUPT_LBN 0 +#define PCRF_AZ_MAX_PAYL_SIZE_SUPT_WIDTH 3 + + +/* + * PC_DEV_CTL_REG(16bit): + * PCIe device control register + */ + +#define PCR_AB_DEV_CTL_REG 0x00000068 +/* falcona0,falconb0=pci_f0_config */ + +#define PCR_CZ_DEV_CTL_REG 0x00000078 +/* sienaa0,hunta0=pci_f0_config */ + +#define PCRF_CZ_FN_LEVEL_RESET_LBN 15 +#define PCRF_CZ_FN_LEVEL_RESET_WIDTH 1 +#define PCRF_AZ_MAX_RD_REQ_SIZE_LBN 12 +#define PCRF_AZ_MAX_RD_REQ_SIZE_WIDTH 3 +#define PCFE_AZ_MAX_RD_REQ_SIZE_4096 5 +#define PCFE_AZ_MAX_RD_REQ_SIZE_2048 4 +#define PCFE_AZ_MAX_RD_REQ_SIZE_1024 3 +#define PCFE_AZ_MAX_RD_REQ_SIZE_512 2 +#define PCFE_AZ_MAX_RD_REQ_SIZE_256 1 +#define PCFE_AZ_MAX_RD_REQ_SIZE_128 0 +#define PCRF_AZ_EN_NO_SNOOP_LBN 11 +#define PCRF_AZ_EN_NO_SNOOP_WIDTH 1 +#define PCRF_AZ_AUX_PWR_PM_EN_LBN 10 +#define PCRF_AZ_AUX_PWR_PM_EN_WIDTH 1 +#define PCRF_AZ_PHAN_FUNC_EN_LBN 9 +#define PCRF_AZ_PHAN_FUNC_EN_WIDTH 1 +#define PCRF_AB_DEV_CAP_REG_RSVD0_LBN 8 +#define PCRF_AB_DEV_CAP_REG_RSVD0_WIDTH 1 +#define PCRF_CZ_EXTENDED_TAG_EN_LBN 8 +#define PCRF_CZ_EXTENDED_TAG_EN_WIDTH 1 +#define PCRF_AZ_MAX_PAYL_SIZE_LBN 5 +#define PCRF_AZ_MAX_PAYL_SIZE_WIDTH 3 +#define PCFE_AZ_MAX_PAYL_SIZE_4096 5 +#define PCFE_AZ_MAX_PAYL_SIZE_2048 4 +#define PCFE_AZ_MAX_PAYL_SIZE_1024 3 +#define PCFE_AZ_MAX_PAYL_SIZE_512 2 +#define PCFE_AZ_MAX_PAYL_SIZE_256 1 +#define PCFE_AZ_MAX_PAYL_SIZE_128 0 +#define PCRF_AZ_EN_RELAX_ORDER_LBN 4 +#define PCRF_AZ_EN_RELAX_ORDER_WIDTH 1 +#define PCRF_AZ_UNSUP_REQ_RPT_EN_LBN 3 +#define PCRF_AZ_UNSUP_REQ_RPT_EN_WIDTH 1 +#define PCRF_AZ_FATAL_ERR_RPT_EN_LBN 2 +#define PCRF_AZ_FATAL_ERR_RPT_EN_WIDTH 1 +#define PCRF_AZ_NONFATAL_ERR_RPT_EN_LBN 1 +#define PCRF_AZ_NONFATAL_ERR_RPT_EN_WIDTH 1 +#define PCRF_AZ_CORR_ERR_RPT_EN_LBN 0 +#define PCRF_AZ_CORR_ERR_RPT_EN_WIDTH 1 + + +/* + * PC_DEV_STAT_REG(16bit): + * PCIe device status register + */ + +#define PCR_AB_DEV_STAT_REG 0x0000006a +/* falcona0,falconb0=pci_f0_config */ + +#define PCR_CZ_DEV_STAT_REG 0x0000007a +/* sienaa0,hunta0=pci_f0_config */ + +#define PCRF_AZ_TRNS_PEND_LBN 5 +#define PCRF_AZ_TRNS_PEND_WIDTH 1 +#define PCRF_AZ_AUX_PWR_DET_LBN 4 +#define PCRF_AZ_AUX_PWR_DET_WIDTH 1 +#define PCRF_AZ_UNSUP_REQ_DET_LBN 3 +#define PCRF_AZ_UNSUP_REQ_DET_WIDTH 1 +#define PCRF_AZ_FATAL_ERR_DET_LBN 2 +#define PCRF_AZ_FATAL_ERR_DET_WIDTH 1 +#define PCRF_AZ_NONFATAL_ERR_DET_LBN 1 +#define PCRF_AZ_NONFATAL_ERR_DET_WIDTH 1 +#define PCRF_AZ_CORR_ERR_DET_LBN 0 +#define PCRF_AZ_CORR_ERR_DET_WIDTH 1 + + +/* + * PC_LNK_CAP_REG(32bit): + * PCIe link capabilities register + */ + +#define PCR_AB_LNK_CAP_REG 0x0000006c +/* falcona0,falconb0=pci_f0_config */ + +#define PCR_CZ_LNK_CAP_REG 0x0000007c +/* sienaa0,hunta0=pci_f0_config */ + +#define PCRF_AZ_PORT_NUM_LBN 24 +#define PCRF_AZ_PORT_NUM_WIDTH 8 +#define PCRF_DZ_ASPM_OPTIONALITY_CAP_LBN 22 +#define PCRF_DZ_ASPM_OPTIONALITY_CAP_WIDTH 1 +#define PCRF_CZ_LINK_BWDITH_NOTIF_CAP_LBN 21 +#define PCRF_CZ_LINK_BWDITH_NOTIF_CAP_WIDTH 1 +#define PCRF_CZ_DATA_LINK_ACTIVE_RPT_CAP_LBN 20 +#define PCRF_CZ_DATA_LINK_ACTIVE_RPT_CAP_WIDTH 1 +#define PCRF_CZ_SURPISE_DOWN_RPT_CAP_LBN 19 +#define PCRF_CZ_SURPISE_DOWN_RPT_CAP_WIDTH 1 +#define PCRF_CZ_CLOCK_PWR_MNGMNT_CAP_LBN 18 +#define PCRF_CZ_CLOCK_PWR_MNGMNT_CAP_WIDTH 1 +#define PCRF_AZ_DEF_L1_EXIT_LAT_LBN 15 +#define PCRF_AZ_DEF_L1_EXIT_LAT_WIDTH 3 +#define PCRF_AZ_DEF_L0_EXIT_LATPORT_NUM_LBN 12 +#define PCRF_AZ_DEF_L0_EXIT_LATPORT_NUM_WIDTH 3 +#define PCRF_AZ_AS_LNK_PM_SUPT_LBN 10 +#define PCRF_AZ_AS_LNK_PM_SUPT_WIDTH 2 +#define PCRF_AZ_MAX_LNK_WIDTH_LBN 4 +#define PCRF_AZ_MAX_LNK_WIDTH_WIDTH 6 +#define PCRF_AZ_MAX_LNK_SP_LBN 0 +#define PCRF_AZ_MAX_LNK_SP_WIDTH 4 + + +/* + * PC_LNK_CTL_REG(16bit): + * PCIe link control register + */ + +#define PCR_AB_LNK_CTL_REG 0x00000070 +/* falcona0,falconb0=pci_f0_config */ + +#define PCR_CZ_LNK_CTL_REG 0x00000080 +/* sienaa0,hunta0=pci_f0_config */ + +#define PCRF_AZ_EXT_SYNC_LBN 7 +#define PCRF_AZ_EXT_SYNC_WIDTH 1 +#define PCRF_AZ_COMM_CLK_CFG_LBN 6 +#define PCRF_AZ_COMM_CLK_CFG_WIDTH 1 +#define PCRF_AB_LNK_CTL_REG_RSVD0_LBN 5 +#define PCRF_AB_LNK_CTL_REG_RSVD0_WIDTH 1 +#define PCRF_CZ_LNK_RETRAIN_LBN 5 +#define PCRF_CZ_LNK_RETRAIN_WIDTH 1 +#define PCRF_AZ_LNK_DIS_LBN 4 +#define PCRF_AZ_LNK_DIS_WIDTH 1 +#define PCRF_AZ_RD_COM_BDRY_LBN 3 +#define PCRF_AZ_RD_COM_BDRY_WIDTH 1 +#define PCRF_AZ_ACT_ST_LNK_PM_CTL_LBN 0 +#define PCRF_AZ_ACT_ST_LNK_PM_CTL_WIDTH 2 + + +/* + * PC_LNK_STAT_REG(16bit): + * PCIe link status register + */ + +#define PCR_AB_LNK_STAT_REG 0x00000072 +/* falcona0,falconb0=pci_f0_config */ + +#define PCR_CZ_LNK_STAT_REG 0x00000082 +/* sienaa0,hunta0=pci_f0_config */ + +#define PCRF_AZ_SLOT_CLK_CFG_LBN 12 +#define PCRF_AZ_SLOT_CLK_CFG_WIDTH 1 +#define PCRF_AZ_LNK_TRAIN_LBN 11 +#define PCRF_AZ_LNK_TRAIN_WIDTH 1 +#define PCRF_AB_TRAIN_ERR_LBN 10 +#define PCRF_AB_TRAIN_ERR_WIDTH 1 +#define PCRF_AZ_LNK_WIDTH_LBN 4 +#define PCRF_AZ_LNK_WIDTH_WIDTH 6 +#define PCRF_AZ_LNK_SP_LBN 0 +#define PCRF_AZ_LNK_SP_WIDTH 4 + + +/* + * PC_SLOT_CAP_REG(32bit): + * PCIe slot capabilities register + */ + +#define PCR_AB_SLOT_CAP_REG 0x00000074 +/* falcona0,falconb0=pci_f0_config */ + +#define PCRF_AB_SLOT_NUM_LBN 19 +#define PCRF_AB_SLOT_NUM_WIDTH 13 +#define PCRF_AB_SLOT_PWR_LIM_SCL_LBN 15 +#define PCRF_AB_SLOT_PWR_LIM_SCL_WIDTH 2 +#define PCRF_AB_SLOT_PWR_LIM_VAL_LBN 7 +#define PCRF_AB_SLOT_PWR_LIM_VAL_WIDTH 8 +#define PCRF_AB_SLOT_HP_CAP_LBN 6 +#define PCRF_AB_SLOT_HP_CAP_WIDTH 1 +#define PCRF_AB_SLOT_HP_SURP_LBN 5 +#define PCRF_AB_SLOT_HP_SURP_WIDTH 1 +#define PCRF_AB_SLOT_PWR_IND_PRST_LBN 4 +#define PCRF_AB_SLOT_PWR_IND_PRST_WIDTH 1 +#define PCRF_AB_SLOT_ATTN_IND_PRST_LBN 3 +#define PCRF_AB_SLOT_ATTN_IND_PRST_WIDTH 1 +#define PCRF_AB_SLOT_MRL_SENS_PRST_LBN 2 +#define PCRF_AB_SLOT_MRL_SENS_PRST_WIDTH 1 +#define PCRF_AB_SLOT_PWR_CTL_PRST_LBN 1 +#define PCRF_AB_SLOT_PWR_CTL_PRST_WIDTH 1 +#define PCRF_AB_SLOT_ATTN_BUT_PRST_LBN 0 +#define PCRF_AB_SLOT_ATTN_BUT_PRST_WIDTH 1 + + +/* + * PC_SLOT_CTL_REG(16bit): + * PCIe slot control register + */ + +#define PCR_AB_SLOT_CTL_REG 0x00000078 +/* falcona0,falconb0=pci_f0_config */ + +#define PCRF_AB_SLOT_PWR_CTLR_CTL_LBN 10 +#define PCRF_AB_SLOT_PWR_CTLR_CTL_WIDTH 1 +#define PCRF_AB_SLOT_PWR_IND_CTL_LBN 8 +#define PCRF_AB_SLOT_PWR_IND_CTL_WIDTH 2 +#define PCRF_AB_SLOT_ATT_IND_CTL_LBN 6 +#define PCRF_AB_SLOT_ATT_IND_CTL_WIDTH 2 +#define PCRF_AB_SLOT_HP_INT_EN_LBN 5 +#define PCRF_AB_SLOT_HP_INT_EN_WIDTH 1 +#define PCRF_AB_SLOT_CMD_COMP_INT_EN_LBN 4 +#define PCRF_AB_SLOT_CMD_COMP_INT_EN_WIDTH 1 +#define PCRF_AB_SLOT_PRES_DET_CHG_EN_LBN 3 +#define PCRF_AB_SLOT_PRES_DET_CHG_EN_WIDTH 1 +#define PCRF_AB_SLOT_MRL_SENS_CHG_EN_LBN 2 +#define PCRF_AB_SLOT_MRL_SENS_CHG_EN_WIDTH 1 +#define PCRF_AB_SLOT_PWR_FLTDET_EN_LBN 1 +#define PCRF_AB_SLOT_PWR_FLTDET_EN_WIDTH 1 +#define PCRF_AB_SLOT_ATTN_BUT_EN_LBN 0 +#define PCRF_AB_SLOT_ATTN_BUT_EN_WIDTH 1 + + +/* + * PC_SLOT_STAT_REG(16bit): + * PCIe slot status register + */ + +#define PCR_AB_SLOT_STAT_REG 0x0000007a +/* falcona0,falconb0=pci_f0_config */ + +#define PCRF_AB_PRES_DET_ST_LBN 6 +#define PCRF_AB_PRES_DET_ST_WIDTH 1 +#define PCRF_AB_MRL_SENS_ST_LBN 5 +#define PCRF_AB_MRL_SENS_ST_WIDTH 1 +#define PCRF_AB_SLOT_PWR_IND_LBN 4 +#define PCRF_AB_SLOT_PWR_IND_WIDTH 1 +#define PCRF_AB_SLOT_ATTN_IND_LBN 3 +#define PCRF_AB_SLOT_ATTN_IND_WIDTH 1 +#define PCRF_AB_SLOT_MRL_SENS_LBN 2 +#define PCRF_AB_SLOT_MRL_SENS_WIDTH 1 +#define PCRF_AB_PWR_FLTDET_LBN 1 +#define PCRF_AB_PWR_FLTDET_WIDTH 1 +#define PCRF_AB_ATTN_BUTDET_LBN 0 +#define PCRF_AB_ATTN_BUTDET_WIDTH 1 + + +/* + * PC_MSIX_CAP_ID_REG(8bit): + * MSIX Capability ID + */ + +#define PCR_BB_MSIX_CAP_ID_REG 0x00000090 +/* falconb0=pci_f0_config */ + +#define PCR_CZ_MSIX_CAP_ID_REG 0x000000b0 +/* sienaa0,hunta0=pci_f0_config */ + +#define PCRF_BZ_MSIX_CAP_ID_LBN 0 +#define PCRF_BZ_MSIX_CAP_ID_WIDTH 8 + + +/* + * PC_MSIX_NXT_PTR_REG(8bit): + * MSIX Capability Next Capability Ptr + */ + +#define PCR_BB_MSIX_NXT_PTR_REG 0x00000091 +/* falconb0=pci_f0_config */ + +#define PCR_CZ_MSIX_NXT_PTR_REG 0x000000b1 +/* sienaa0,hunta0=pci_f0_config */ + +#define PCRF_BZ_MSIX_NXT_PTR_LBN 0 +#define PCRF_BZ_MSIX_NXT_PTR_WIDTH 8 + + +/* + * PC_MSIX_CTL_REG(16bit): + * MSIX control register + */ + +#define PCR_BB_MSIX_CTL_REG 0x00000092 +/* falconb0=pci_f0_config */ + +#define PCR_CZ_MSIX_CTL_REG 0x000000b2 +/* sienaa0,hunta0=pci_f0_config */ + +#define PCRF_BZ_MSIX_EN_LBN 15 +#define PCRF_BZ_MSIX_EN_WIDTH 1 +#define PCRF_BZ_MSIX_FUNC_MASK_LBN 14 +#define PCRF_BZ_MSIX_FUNC_MASK_WIDTH 1 +#define PCRF_BZ_MSIX_TBL_SIZE_LBN 0 +#define PCRF_BZ_MSIX_TBL_SIZE_WIDTH 11 + + +/* + * PC_MSIX_TBL_BASE_REG(32bit): + * MSIX Capability Vector Table Base + */ + +#define PCR_BB_MSIX_TBL_BASE_REG 0x00000094 +/* falconb0=pci_f0_config */ + +#define PCR_CZ_MSIX_TBL_BASE_REG 0x000000b4 +/* sienaa0,hunta0=pci_f0_config */ + +#define PCRF_BZ_MSIX_TBL_OFF_LBN 3 +#define PCRF_BZ_MSIX_TBL_OFF_WIDTH 29 +#define PCRF_BZ_MSIX_TBL_BIR_LBN 0 +#define PCRF_BZ_MSIX_TBL_BIR_WIDTH 3 + + +/* + * PC_DEV_CAP2_REG(32bit): + * PCIe Device Capabilities 2 + */ + +#define PCR_CZ_DEV_CAP2_REG 0x00000094 +/* sienaa0=pci_f0_config,hunta0=pci_f0_config */ + +#define PCRF_DZ_OBFF_SUPPORTED_LBN 18 +#define PCRF_DZ_OBFF_SUPPORTED_WIDTH 2 +#define PCRF_DZ_TPH_CMPL_SUPPORTED_LBN 12 +#define PCRF_DZ_TPH_CMPL_SUPPORTED_WIDTH 2 +#define PCRF_DZ_LTR_M_SUPPORTED_LBN 11 +#define PCRF_DZ_LTR_M_SUPPORTED_WIDTH 1 +#define PCRF_CC_CMPL_TIMEOUT_DIS_LBN 4 +#define PCRF_CC_CMPL_TIMEOUT_DIS_WIDTH 1 +#define PCRF_DZ_CMPL_TIMEOUT_DIS_SUPPORTED_LBN 4 +#define PCRF_DZ_CMPL_TIMEOUT_DIS_SUPPORTED_WIDTH 1 +#define PCRF_CZ_CMPL_TIMEOUT_LBN 0 +#define PCRF_CZ_CMPL_TIMEOUT_WIDTH 4 +#define PCFE_CZ_CMPL_TIMEOUT_17000_TO_6400MS 14 +#define PCFE_CZ_CMPL_TIMEOUT_4000_TO_1300MS 13 +#define PCFE_CZ_CMPL_TIMEOUT_1000_TO_3500MS 10 +#define PCFE_CZ_CMPL_TIMEOUT_260_TO_900MS 9 +#define PCFE_CZ_CMPL_TIMEOUT_65_TO_210MS 6 +#define PCFE_CZ_CMPL_TIMEOUT_16_TO_55MS 5 +#define PCFE_CZ_CMPL_TIMEOUT_1_TO_10MS 2 +#define PCFE_CZ_CMPL_TIMEOUT_50_TO_100US 1 +#define PCFE_CZ_CMPL_TIMEOUT_DEFAULT 0 + + +/* + * PC_DEV_CTL2_REG(16bit): + * PCIe Device Control 2 + */ + +#define PCR_CZ_DEV_CTL2_REG 0x00000098 +/* sienaa0,hunta0=pci_f0_config */ + +#define PCRF_DZ_OBFF_ENABLE_LBN 13 +#define PCRF_DZ_OBFF_ENABLE_WIDTH 2 +#define PCRF_DZ_LTR_ENABLE_LBN 10 +#define PCRF_DZ_LTR_ENABLE_WIDTH 1 +#define PCRF_DZ_IDO_COMPLETION_ENABLE_LBN 9 +#define PCRF_DZ_IDO_COMPLETION_ENABLE_WIDTH 1 +#define PCRF_DZ_IDO_REQUEST_ENABLE_LBN 8 +#define PCRF_DZ_IDO_REQUEST_ENABLE_WIDTH 1 +#define PCRF_CZ_CMPL_TIMEOUT_DIS_CTL_LBN 4 +#define PCRF_CZ_CMPL_TIMEOUT_DIS_CTL_WIDTH 1 +#define PCRF_CZ_CMPL_TIMEOUT_CTL_LBN 0 +#define PCRF_CZ_CMPL_TIMEOUT_CTL_WIDTH 4 + + +/* + * PC_MSIX_PBA_BASE_REG(32bit): + * MSIX Capability PBA Base + */ + +#define PCR_BB_MSIX_PBA_BASE_REG 0x00000098 +/* falconb0=pci_f0_config */ + +#define PCR_CZ_MSIX_PBA_BASE_REG 0x000000b8 +/* sienaa0,hunta0=pci_f0_config */ + +#define PCRF_BZ_MSIX_PBA_OFF_LBN 3 +#define PCRF_BZ_MSIX_PBA_OFF_WIDTH 29 +#define PCRF_BZ_MSIX_PBA_BIR_LBN 0 +#define PCRF_BZ_MSIX_PBA_BIR_WIDTH 3 + + +/* + * PC_LNK_CAP2_REG(32bit): + * PCIe Link Capability 2 + */ + +#define PCR_DZ_LNK_CAP2_REG 0x0000009c +/* hunta0=pci_f0_config */ + +#define PCRF_DZ_LNK_SPEED_SUP_LBN 1 +#define PCRF_DZ_LNK_SPEED_SUP_WIDTH 7 + + +/* + * PC_LNK_CTL2_REG(16bit): + * PCIe Link Control 2 + */ + +#define PCR_CZ_LNK_CTL2_REG 0x000000a0 +/* sienaa0,hunta0=pci_f0_config */ + +#define PCRF_CZ_POLLING_DEEMPH_LVL_LBN 12 +#define PCRF_CZ_POLLING_DEEMPH_LVL_WIDTH 1 +#define PCRF_CZ_COMPLIANCE_SOS_CTL_LBN 11 +#define PCRF_CZ_COMPLIANCE_SOS_CTL_WIDTH 1 +#define PCRF_CZ_ENTER_MODIFIED_COMPLIANCE_CTL_LBN 10 +#define PCRF_CZ_ENTER_MODIFIED_COMPLIANCE_CTL_WIDTH 1 +#define PCRF_CZ_TRANSMIT_MARGIN_LBN 7 +#define PCRF_CZ_TRANSMIT_MARGIN_WIDTH 3 +#define PCRF_CZ_SELECT_DEEMPH_LBN 6 +#define PCRF_CZ_SELECT_DEEMPH_WIDTH 1 +#define PCRF_CZ_HW_AUTONOMOUS_SPEED_DIS_LBN 5 +#define PCRF_CZ_HW_AUTONOMOUS_SPEED_DIS_WIDTH 1 +#define PCRF_CZ_ENTER_COMPLIANCE_CTL_LBN 4 +#define PCRF_CZ_ENTER_COMPLIANCE_CTL_WIDTH 1 +#define PCRF_CZ_TGT_LNK_SPEED_CTL_LBN 0 +#define PCRF_CZ_TGT_LNK_SPEED_CTL_WIDTH 4 +#define PCFE_DZ_LCTL2_TGT_SPEED_GEN3 3 +#define PCFE_DZ_LCTL2_TGT_SPEED_GEN2 2 +#define PCFE_DZ_LCTL2_TGT_SPEED_GEN1 1 + + +/* + * PC_LNK_STAT2_REG(16bit): + * PCIe Link Status 2 + */ + +#define PCR_CZ_LNK_STAT2_REG 0x000000a2 +/* sienaa0,hunta0=pci_f0_config */ + +#define PCRF_CZ_CURRENT_DEEMPH_LBN 0 +#define PCRF_CZ_CURRENT_DEEMPH_WIDTH 1 + + +/* + * PC_VPD_CAP_ID_REG(8bit): + * VPD data register + */ + +#define PCR_AB_VPD_CAP_ID_REG 0x000000b0 +/* falcona0,falconb0=pci_f0_config */ + +#define PCRF_AB_VPD_CAP_ID_LBN 0 +#define PCRF_AB_VPD_CAP_ID_WIDTH 8 + + +/* + * PC_VPD_NXT_PTR_REG(8bit): + * VPD next item pointer + */ + +#define PCR_AB_VPD_NXT_PTR_REG 0x000000b1 +/* falcona0,falconb0=pci_f0_config */ + +#define PCRF_AB_VPD_NXT_PTR_LBN 0 +#define PCRF_AB_VPD_NXT_PTR_WIDTH 8 + + +/* + * PC_VPD_ADDR_REG(16bit): + * VPD address register + */ + +#define PCR_AB_VPD_ADDR_REG 0x000000b2 +/* falcona0,falconb0=pci_f0_config */ + +#define PCRF_AB_VPD_FLAG_LBN 15 +#define PCRF_AB_VPD_FLAG_WIDTH 1 +#define PCRF_AB_VPD_ADDR_LBN 0 +#define PCRF_AB_VPD_ADDR_WIDTH 15 + + +/* + * PC_VPD_CAP_DATA_REG(32bit): + * documentation to be written for sum_PC_VPD_CAP_DATA_REG + */ + +#define PCR_AB_VPD_CAP_DATA_REG 0x000000b4 +/* falcona0,falconb0=pci_f0_config */ + +#define PCR_CZ_VPD_CAP_DATA_REG 0x000000d4 +/* sienaa0,hunta0=pci_f0_config */ + +#define PCRF_AZ_VPD_DATA_LBN 0 +#define PCRF_AZ_VPD_DATA_WIDTH 32 + + +/* + * PC_VPD_CAP_CTL_REG(8bit): + * VPD control and capabilities register + */ + +#define PCR_CZ_VPD_CAP_CTL_REG 0x000000d0 +/* sienaa0,hunta0=pci_f0_config */ + +#define PCRF_CZ_VPD_FLAG_LBN 31 +#define PCRF_CZ_VPD_FLAG_WIDTH 1 +#define PCRF_CZ_VPD_ADDR_LBN 16 +#define PCRF_CZ_VPD_ADDR_WIDTH 15 +#define PCRF_CZ_VPD_NXT_PTR_LBN 8 +#define PCRF_CZ_VPD_NXT_PTR_WIDTH 8 +#define PCRF_CZ_VPD_CAP_ID_LBN 0 +#define PCRF_CZ_VPD_CAP_ID_WIDTH 8 + + +/* + * PC_AER_CAP_HDR_REG(32bit): + * AER capability header register + */ + +#define PCR_AZ_AER_CAP_HDR_REG 0x00000100 +/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */ + +#define PCRF_AZ_AERCAPHDR_NXT_PTR_LBN 20 +#define PCRF_AZ_AERCAPHDR_NXT_PTR_WIDTH 12 +#define PCRF_AZ_AERCAPHDR_VER_LBN 16 +#define PCRF_AZ_AERCAPHDR_VER_WIDTH 4 +#define PCRF_AZ_AERCAPHDR_ID_LBN 0 +#define PCRF_AZ_AERCAPHDR_ID_WIDTH 16 + + +/* + * PC_AER_UNCORR_ERR_STAT_REG(32bit): + * AER Uncorrectable error status register + */ + +#define PCR_AZ_AER_UNCORR_ERR_STAT_REG 0x00000104 +/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */ + +#define PCRF_AZ_UNSUPT_REQ_ERR_STAT_LBN 20 +#define PCRF_AZ_UNSUPT_REQ_ERR_STAT_WIDTH 1 +#define PCRF_AZ_ECRC_ERR_STAT_LBN 19 +#define PCRF_AZ_ECRC_ERR_STAT_WIDTH 1 +#define PCRF_AZ_MALF_TLP_STAT_LBN 18 +#define PCRF_AZ_MALF_TLP_STAT_WIDTH 1 +#define PCRF_AZ_RX_OVF_STAT_LBN 17 +#define PCRF_AZ_RX_OVF_STAT_WIDTH 1 +#define PCRF_AZ_UNEXP_COMP_STAT_LBN 16 +#define PCRF_AZ_UNEXP_COMP_STAT_WIDTH 1 +#define PCRF_AZ_COMP_ABRT_STAT_LBN 15 +#define PCRF_AZ_COMP_ABRT_STAT_WIDTH 1 +#define PCRF_AZ_COMP_TIMEOUT_STAT_LBN 14 +#define PCRF_AZ_COMP_TIMEOUT_STAT_WIDTH 1 +#define PCRF_AZ_FC_PROTO_ERR_STAT_LBN 13 +#define PCRF_AZ_FC_PROTO_ERR_STAT_WIDTH 1 +#define PCRF_AZ_PSON_TLP_STAT_LBN 12 +#define PCRF_AZ_PSON_TLP_STAT_WIDTH 1 +#define PCRF_AZ_DL_PROTO_ERR_STAT_LBN 4 +#define PCRF_AZ_DL_PROTO_ERR_STAT_WIDTH 1 +#define PCRF_AB_TRAIN_ERR_STAT_LBN 0 +#define PCRF_AB_TRAIN_ERR_STAT_WIDTH 1 + + +/* + * PC_AER_UNCORR_ERR_MASK_REG(32bit): + * AER Uncorrectable error mask register + */ + +#define PCR_AZ_AER_UNCORR_ERR_MASK_REG 0x00000108 +/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */ + +#define PCRF_DZ_ATOMIC_OP_EGR_BLOCKED_MASK_LBN 24 +#define PCRF_DZ_ATOMIC_OP_EGR_BLOCKED_MASK_WIDTH 1 +#define PCRF_DZ_UNCORR_INT_ERR_MASK_LBN 22 +#define PCRF_DZ_UNCORR_INT_ERR_MASK_WIDTH 1 +#define PCRF_AZ_UNSUPT_REQ_ERR_MASK_LBN 20 +#define PCRF_AZ_UNSUPT_REQ_ERR_MASK_WIDTH 1 +#define PCRF_AZ_ECRC_ERR_MASK_LBN 19 +#define PCRF_AZ_ECRC_ERR_MASK_WIDTH 1 +#define PCRF_AZ_MALF_TLP_MASK_LBN 18 +#define PCRF_AZ_MALF_TLP_MASK_WIDTH 1 +#define PCRF_AZ_RX_OVF_MASK_LBN 17 +#define PCRF_AZ_RX_OVF_MASK_WIDTH 1 +#define PCRF_AZ_UNEXP_COMP_MASK_LBN 16 +#define PCRF_AZ_UNEXP_COMP_MASK_WIDTH 1 +#define PCRF_AZ_COMP_ABRT_MASK_LBN 15 +#define PCRF_AZ_COMP_ABRT_MASK_WIDTH 1 +#define PCRF_AZ_COMP_TIMEOUT_MASK_LBN 14 +#define PCRF_AZ_COMP_TIMEOUT_MASK_WIDTH 1 +#define PCRF_AZ_FC_PROTO_ERR_MASK_LBN 13 +#define PCRF_AZ_FC_PROTO_ERR_MASK_WIDTH 1 +#define PCRF_AZ_PSON_TLP_MASK_LBN 12 +#define PCRF_AZ_PSON_TLP_MASK_WIDTH 1 +#define PCRF_AZ_DL_PROTO_ERR_MASK_LBN 4 +#define PCRF_AZ_DL_PROTO_ERR_MASK_WIDTH 1 +#define PCRF_AB_TRAIN_ERR_MASK_LBN 0 +#define PCRF_AB_TRAIN_ERR_MASK_WIDTH 1 + + +/* + * PC_AER_UNCORR_ERR_SEV_REG(32bit): + * AER Uncorrectable error severity register + */ + +#define PCR_AZ_AER_UNCORR_ERR_SEV_REG 0x0000010c +/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */ + +#define PCRF_AZ_UNSUPT_REQ_ERR_SEV_LBN 20 +#define PCRF_AZ_UNSUPT_REQ_ERR_SEV_WIDTH 1 +#define PCRF_AZ_ECRC_ERR_SEV_LBN 19 +#define PCRF_AZ_ECRC_ERR_SEV_WIDTH 1 +#define PCRF_AZ_MALF_TLP_SEV_LBN 18 +#define PCRF_AZ_MALF_TLP_SEV_WIDTH 1 +#define PCRF_AZ_RX_OVF_SEV_LBN 17 +#define PCRF_AZ_RX_OVF_SEV_WIDTH 1 +#define PCRF_AZ_UNEXP_COMP_SEV_LBN 16 +#define PCRF_AZ_UNEXP_COMP_SEV_WIDTH 1 +#define PCRF_AZ_COMP_ABRT_SEV_LBN 15 +#define PCRF_AZ_COMP_ABRT_SEV_WIDTH 1 +#define PCRF_AZ_COMP_TIMEOUT_SEV_LBN 14 +#define PCRF_AZ_COMP_TIMEOUT_SEV_WIDTH 1 +#define PCRF_AZ_FC_PROTO_ERR_SEV_LBN 13 +#define PCRF_AZ_FC_PROTO_ERR_SEV_WIDTH 1 +#define PCRF_AZ_PSON_TLP_SEV_LBN 12 +#define PCRF_AZ_PSON_TLP_SEV_WIDTH 1 +#define PCRF_AZ_DL_PROTO_ERR_SEV_LBN 4 +#define PCRF_AZ_DL_PROTO_ERR_SEV_WIDTH 1 +#define PCRF_AB_TRAIN_ERR_SEV_LBN 0 +#define PCRF_AB_TRAIN_ERR_SEV_WIDTH 1 + + +/* + * PC_AER_CORR_ERR_STAT_REG(32bit): + * AER Correctable error status register + */ + +#define PCR_AZ_AER_CORR_ERR_STAT_REG 0x00000110 +/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */ + +#define PCRF_CZ_ADVSY_NON_FATAL_STAT_LBN 13 +#define PCRF_CZ_ADVSY_NON_FATAL_STAT_WIDTH 1 +#define PCRF_AZ_RPLY_TMR_TOUT_STAT_LBN 12 +#define PCRF_AZ_RPLY_TMR_TOUT_STAT_WIDTH 1 +#define PCRF_AZ_RPLAY_NUM_RO_STAT_LBN 8 +#define PCRF_AZ_RPLAY_NUM_RO_STAT_WIDTH 1 +#define PCRF_AZ_BAD_DLLP_STAT_LBN 7 +#define PCRF_AZ_BAD_DLLP_STAT_WIDTH 1 +#define PCRF_AZ_BAD_TLP_STAT_LBN 6 +#define PCRF_AZ_BAD_TLP_STAT_WIDTH 1 +#define PCRF_AZ_RX_ERR_STAT_LBN 0 +#define PCRF_AZ_RX_ERR_STAT_WIDTH 1 + + +/* + * PC_AER_CORR_ERR_MASK_REG(32bit): + * AER Correctable error status register + */ + +#define PCR_AZ_AER_CORR_ERR_MASK_REG 0x00000114 +/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */ + +#define PCRF_CZ_ADVSY_NON_FATAL_MASK_LBN 13 +#define PCRF_CZ_ADVSY_NON_FATAL_MASK_WIDTH 1 +#define PCRF_AZ_RPLY_TMR_TOUT_MASK_LBN 12 +#define PCRF_AZ_RPLY_TMR_TOUT_MASK_WIDTH 1 +#define PCRF_AZ_RPLAY_NUM_RO_MASK_LBN 8 +#define PCRF_AZ_RPLAY_NUM_RO_MASK_WIDTH 1 +#define PCRF_AZ_BAD_DLLP_MASK_LBN 7 +#define PCRF_AZ_BAD_DLLP_MASK_WIDTH 1 +#define PCRF_AZ_BAD_TLP_MASK_LBN 6 +#define PCRF_AZ_BAD_TLP_MASK_WIDTH 1 +#define PCRF_AZ_RX_ERR_MASK_LBN 0 +#define PCRF_AZ_RX_ERR_MASK_WIDTH 1 + + +/* + * PC_AER_CAP_CTL_REG(32bit): + * AER capability and control register + */ + +#define PCR_AZ_AER_CAP_CTL_REG 0x00000118 +/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */ + +#define PCRF_AZ_ECRC_CHK_EN_LBN 8 +#define PCRF_AZ_ECRC_CHK_EN_WIDTH 1 +#define PCRF_AZ_ECRC_CHK_CAP_LBN 7 +#define PCRF_AZ_ECRC_CHK_CAP_WIDTH 1 +#define PCRF_AZ_ECRC_GEN_EN_LBN 6 +#define PCRF_AZ_ECRC_GEN_EN_WIDTH 1 +#define PCRF_AZ_ECRC_GEN_CAP_LBN 5 +#define PCRF_AZ_ECRC_GEN_CAP_WIDTH 1 +#define PCRF_AZ_1ST_ERR_PTR_LBN 0 +#define PCRF_AZ_1ST_ERR_PTR_WIDTH 5 + + +/* + * PC_AER_HDR_LOG_REG(128bit): + * AER Header log register + */ + +#define PCR_AZ_AER_HDR_LOG_REG 0x0000011c +/* falcona0,falconb0,sienaa0,hunta0=pci_f0_config */ + +#define PCRF_AZ_HDR_LOG_LBN 0 +#define PCRF_AZ_HDR_LOG_WIDTH 128 + + +/* + * PC_DEVSN_CAP_HDR_REG(32bit): + * Device serial number capability header register + */ + +#define PCR_CZ_DEVSN_CAP_HDR_REG 0x00000140 +/* sienaa0,hunta0=pci_f0_config */ + +#define PCRF_CZ_DEVSNCAPHDR_NXT_PTR_LBN 20 +#define PCRF_CZ_DEVSNCAPHDR_NXT_PTR_WIDTH 12 +#define PCRF_CZ_DEVSNCAPHDR_VER_LBN 16 +#define PCRF_CZ_DEVSNCAPHDR_VER_WIDTH 4 +#define PCRF_CZ_DEVSNCAPHDR_ID_LBN 0 +#define PCRF_CZ_DEVSNCAPHDR_ID_WIDTH 16 + + +/* + * PC_DEVSN_DWORD0_REG(32bit): + * Device serial number DWORD0 + */ + +#define PCR_CZ_DEVSN_DWORD0_REG 0x00000144 +/* sienaa0,hunta0=pci_f0_config */ + +#define PCRF_CZ_DEVSN_DWORD0_LBN 0 +#define PCRF_CZ_DEVSN_DWORD0_WIDTH 32 + + +/* + * PC_DEVSN_DWORD1_REG(32bit): + * Device serial number DWORD0 + */ + +#define PCR_CZ_DEVSN_DWORD1_REG 0x00000148 +/* sienaa0,hunta0=pci_f0_config */ + +#define PCRF_CZ_DEVSN_DWORD1_LBN 0 +#define PCRF_CZ_DEVSN_DWORD1_WIDTH 32 + + +/* + * PC_ARI_CAP_HDR_REG(32bit): + * ARI capability header register + */ + +#define PCR_CZ_ARI_CAP_HDR_REG 0x00000150 +/* sienaa0,hunta0=pci_f0_config */ + +#define PCRF_CZ_ARICAPHDR_NXT_PTR_LBN 20 +#define PCRF_CZ_ARICAPHDR_NXT_PTR_WIDTH 12 +#define PCRF_CZ_ARICAPHDR_VER_LBN 16 +#define PCRF_CZ_ARICAPHDR_VER_WIDTH 4 +#define PCRF_CZ_ARICAPHDR_ID_LBN 0 +#define PCRF_CZ_ARICAPHDR_ID_WIDTH 16 + + +/* + * PC_ARI_CAP_REG(16bit): + * ARI Capabilities + */ + +#define PCR_CZ_ARI_CAP_REG 0x00000154 +/* sienaa0,hunta0=pci_f0_config */ + +#define PCRF_CZ_ARI_NXT_FN_NUM_LBN 8 +#define PCRF_CZ_ARI_NXT_FN_NUM_WIDTH 8 +#define PCRF_CZ_ARI_ACS_FNGRP_CAP_LBN 1 +#define PCRF_CZ_ARI_ACS_FNGRP_CAP_WIDTH 1 +#define PCRF_CZ_ARI_MFVC_FNGRP_CAP_LBN 0 +#define PCRF_CZ_ARI_MFVC_FNGRP_CAP_WIDTH 1 + + +/* + * PC_ARI_CTL_REG(16bit): + * ARI Control + */ + +#define PCR_CZ_ARI_CTL_REG 0x00000156 +/* sienaa0,hunta0=pci_f0_config */ + +#define PCRF_CZ_ARI_FN_GRP_LBN 4 +#define PCRF_CZ_ARI_FN_GRP_WIDTH 3 +#define PCRF_CZ_ARI_ACS_FNGRP_EN_LBN 1 +#define PCRF_CZ_ARI_ACS_FNGRP_EN_WIDTH 1 +#define PCRF_CZ_ARI_MFVC_FNGRP_EN_LBN 0 +#define PCRF_CZ_ARI_MFVC_FNGRP_EN_WIDTH 1 + + +/* + * PC_SEC_PCIE_CAP_REG(32bit): + * Secondary PCIE Capability Register + */ + +#define PCR_DZ_SEC_PCIE_CAP_REG 0x00000160 +/* hunta0=pci_f0_config */ + +#define PCRF_DZ_SEC_NXT_PTR_LBN 20 +#define PCRF_DZ_SEC_NXT_PTR_WIDTH 12 +#define PCRF_DZ_SEC_VERSION_LBN 16 +#define PCRF_DZ_SEC_VERSION_WIDTH 4 +#define PCRF_DZ_SEC_EXT_CAP_ID_LBN 0 +#define PCRF_DZ_SEC_EXT_CAP_ID_WIDTH 16 + + +/* + * PC_SRIOV_CAP_HDR_REG(32bit): + * SRIOV capability header register + */ + +#define PCR_CC_SRIOV_CAP_HDR_REG 0x00000160 +/* sienaa0=pci_f0_config */ + +#define PCR_DZ_SRIOV_CAP_HDR_REG 0x00000180 +/* hunta0=pci_f0_config */ + +#define PCRF_CZ_SRIOVCAPHDR_NXT_PTR_LBN 20 +#define PCRF_CZ_SRIOVCAPHDR_NXT_PTR_WIDTH 12 +#define PCRF_CZ_SRIOVCAPHDR_VER_LBN 16 +#define PCRF_CZ_SRIOVCAPHDR_VER_WIDTH 4 +#define PCRF_CZ_SRIOVCAPHDR_ID_LBN 0 +#define PCRF_CZ_SRIOVCAPHDR_ID_WIDTH 16 + + +/* + * PC_SRIOV_CAP_REG(32bit): + * SRIOV Capabilities + */ + +#define PCR_CC_SRIOV_CAP_REG 0x00000164 +/* sienaa0=pci_f0_config */ + +#define PCR_DZ_SRIOV_CAP_REG 0x00000184 +/* hunta0=pci_f0_config */ + +#define PCRF_CZ_VF_MIGR_INT_MSG_NUM_LBN 21 +#define PCRF_CZ_VF_MIGR_INT_MSG_NUM_WIDTH 11 +#define PCRF_DZ_VF_ARI_CAP_PRESV_LBN 1 +#define PCRF_DZ_VF_ARI_CAP_PRESV_WIDTH 1 +#define PCRF_CZ_VF_MIGR_CAP_LBN 0 +#define PCRF_CZ_VF_MIGR_CAP_WIDTH 1 + + +/* + * PC_LINK_CONTROL3_REG(32bit): + * Link Control 3. + */ + +#define PCR_DZ_LINK_CONTROL3_REG 0x00000164 +/* hunta0=pci_f0_config */ + +#define PCRF_DZ_LINK_EQ_INT_EN_LBN 1 +#define PCRF_DZ_LINK_EQ_INT_EN_WIDTH 1 +#define PCRF_DZ_PERFORM_EQL_LBN 0 +#define PCRF_DZ_PERFORM_EQL_WIDTH 1 + + +/* + * PC_LANE_ERROR_STAT_REG(32bit): + * Lane Error Status Register. + */ + +#define PCR_DZ_LANE_ERROR_STAT_REG 0x00000168 +/* hunta0=pci_f0_config */ + +#define PCRF_DZ_LANE_STATUS_LBN 0 +#define PCRF_DZ_LANE_STATUS_WIDTH 8 + + +/* + * PC_SRIOV_CTL_REG(16bit): + * SRIOV Control + */ + +#define PCR_CC_SRIOV_CTL_REG 0x00000168 +/* sienaa0=pci_f0_config */ + +#define PCR_DZ_SRIOV_CTL_REG 0x00000188 +/* hunta0=pci_f0_config */ + +#define PCRF_CZ_VF_ARI_CAP_HRCHY_LBN 4 +#define PCRF_CZ_VF_ARI_CAP_HRCHY_WIDTH 1 +#define PCRF_CZ_VF_MSE_LBN 3 +#define PCRF_CZ_VF_MSE_WIDTH 1 +#define PCRF_CZ_VF_MIGR_INT_EN_LBN 2 +#define PCRF_CZ_VF_MIGR_INT_EN_WIDTH 1 +#define PCRF_CZ_VF_MIGR_EN_LBN 1 +#define PCRF_CZ_VF_MIGR_EN_WIDTH 1 +#define PCRF_CZ_VF_EN_LBN 0 +#define PCRF_CZ_VF_EN_WIDTH 1 + + +/* + * PC_SRIOV_STAT_REG(16bit): + * SRIOV Status + */ + +#define PCR_CC_SRIOV_STAT_REG 0x0000016a +/* sienaa0=pci_f0_config */ + +#define PCR_DZ_SRIOV_STAT_REG 0x0000018a +/* hunta0=pci_f0_config */ + +#define PCRF_CZ_VF_MIGR_STAT_LBN 0 +#define PCRF_CZ_VF_MIGR_STAT_WIDTH 1 + + +/* + * PC_LANE01_EQU_CONTROL_REG(32bit): + * Lanes 0,1 Equalization Control Register. + */ + +#define PCR_DZ_LANE01_EQU_CONTROL_REG 0x0000016c +/* hunta0=pci_f0_config */ + +#define PCRF_DZ_LANE1_EQ_CTRL_LBN 16 +#define PCRF_DZ_LANE1_EQ_CTRL_WIDTH 16 +#define PCRF_DZ_LANE0_EQ_CTRL_LBN 0 +#define PCRF_DZ_LANE0_EQ_CTRL_WIDTH 16 + + +/* + * PC_SRIOV_INITIALVFS_REG(16bit): + * SRIOV Initial VFs + */ + +#define PCR_CC_SRIOV_INITIALVFS_REG 0x0000016c +/* sienaa0=pci_f0_config */ + +#define PCR_DZ_SRIOV_INITIALVFS_REG 0x0000018c +/* hunta0=pci_f0_config */ + +#define PCRF_CZ_VF_INITIALVFS_LBN 0 +#define PCRF_CZ_VF_INITIALVFS_WIDTH 16 + + +/* + * PC_SRIOV_TOTALVFS_REG(10bit): + * SRIOV Total VFs + */ + +#define PCR_CC_SRIOV_TOTALVFS_REG 0x0000016e +/* sienaa0=pci_f0_config */ + +#define PCR_DZ_SRIOV_TOTALVFS_REG 0x0000018e +/* hunta0=pci_f0_config */ + +#define PCRF_CZ_VF_TOTALVFS_LBN 0 +#define PCRF_CZ_VF_TOTALVFS_WIDTH 16 + + +/* + * PC_SRIOV_NUMVFS_REG(16bit): + * SRIOV Number of VFs + */ + +#define PCR_CC_SRIOV_NUMVFS_REG 0x00000170 +/* sienaa0=pci_f0_config */ + +#define PCR_DZ_SRIOV_NUMVFS_REG 0x00000190 +/* hunta0=pci_f0_config */ + +#define PCRF_CZ_VF_NUMVFS_LBN 0 +#define PCRF_CZ_VF_NUMVFS_WIDTH 16 + + +/* + * PC_LANE23_EQU_CONTROL_REG(32bit): + * Lanes 2,3 Equalization Control Register. + */ + +#define PCR_DZ_LANE23_EQU_CONTROL_REG 0x00000170 +/* hunta0=pci_f0_config */ + +#define PCRF_DZ_LANE3_EQ_CTRL_LBN 16 +#define PCRF_DZ_LANE3_EQ_CTRL_WIDTH 16 +#define PCRF_DZ_LANE2_EQ_CTRL_LBN 0 +#define PCRF_DZ_LANE2_EQ_CTRL_WIDTH 16 + + +/* + * PC_SRIOV_FN_DPND_LNK_REG(16bit): + * SRIOV Function dependency link + */ + +#define PCR_CC_SRIOV_FN_DPND_LNK_REG 0x00000172 +/* sienaa0=pci_f0_config */ + +#define PCR_DZ_SRIOV_FN_DPND_LNK_REG 0x00000192 +/* hunta0=pci_f0_config */ + +#define PCRF_CZ_SRIOV_FN_DPND_LNK_LBN 0 +#define PCRF_CZ_SRIOV_FN_DPND_LNK_WIDTH 8 + + +/* + * PC_SRIOV_1STVF_OFFSET_REG(16bit): + * SRIOV First VF Offset + */ + +#define PCR_CC_SRIOV_1STVF_OFFSET_REG 0x00000174 +/* sienaa0=pci_f0_config */ + +#define PCR_DZ_SRIOV_1STVF_OFFSET_REG 0x00000194 +/* hunta0=pci_f0_config */ + +#define PCRF_CZ_VF_1STVF_OFFSET_LBN 0 +#define PCRF_CZ_VF_1STVF_OFFSET_WIDTH 16 + + +/* + * PC_LANE45_EQU_CONTROL_REG(32bit): + * Lanes 4,5 Equalization Control Register. + */ + +#define PCR_DZ_LANE45_EQU_CONTROL_REG 0x00000174 +/* hunta0=pci_f0_config */ + +#define PCRF_DZ_LANE5_EQ_CTRL_LBN 16 +#define PCRF_DZ_LANE5_EQ_CTRL_WIDTH 16 +#define PCRF_DZ_LANE4_EQ_CTRL_LBN 0 +#define PCRF_DZ_LANE4_EQ_CTRL_WIDTH 16 + + +/* + * PC_SRIOV_VFSTRIDE_REG(16bit): + * SRIOV VF Stride + */ + +#define PCR_CC_SRIOV_VFSTRIDE_REG 0x00000176 +/* sienaa0=pci_f0_config */ + +#define PCR_DZ_SRIOV_VFSTRIDE_REG 0x00000196 +/* hunta0=pci_f0_config */ + +#define PCRF_CZ_VF_VFSTRIDE_LBN 0 +#define PCRF_CZ_VF_VFSTRIDE_WIDTH 16 + + +/* + * PC_LANE67_EQU_CONTROL_REG(32bit): + * Lanes 6,7 Equalization Control Register. + */ + +#define PCR_DZ_LANE67_EQU_CONTROL_REG 0x00000178 +/* hunta0=pci_f0_config */ + +#define PCRF_DZ_LANE7_EQ_CTRL_LBN 16 +#define PCRF_DZ_LANE7_EQ_CTRL_WIDTH 16 +#define PCRF_DZ_LANE6_EQ_CTRL_LBN 0 +#define PCRF_DZ_LANE6_EQ_CTRL_WIDTH 16 + + +/* + * PC_SRIOV_DEVID_REG(16bit): + * SRIOV VF Device ID + */ + +#define PCR_CC_SRIOV_DEVID_REG 0x0000017a +/* sienaa0=pci_f0_config */ + +#define PCR_DZ_SRIOV_DEVID_REG 0x0000019a +/* hunta0=pci_f0_config */ + +#define PCRF_CZ_VF_DEVID_LBN 0 +#define PCRF_CZ_VF_DEVID_WIDTH 16 + + +/* + * PC_SRIOV_SUP_PAGESZ_REG(16bit): + * SRIOV Supported Page Sizes + */ + +#define PCR_CC_SRIOV_SUP_PAGESZ_REG 0x0000017c +/* sienaa0=pci_f0_config */ + +#define PCR_DZ_SRIOV_SUP_PAGESZ_REG 0x0000019c +/* hunta0=pci_f0_config */ + +#define PCRF_CZ_VF_SUP_PAGESZ_LBN 0 +#define PCRF_CZ_VF_SUP_PAGESZ_WIDTH 16 + + +/* + * PC_SRIOV_SYS_PAGESZ_REG(32bit): + * SRIOV System Page Size + */ + +#define PCR_CC_SRIOV_SYS_PAGESZ_REG 0x00000180 +/* sienaa0=pci_f0_config */ + +#define PCR_DZ_SRIOV_SYS_PAGESZ_REG 0x000001a0 +/* hunta0=pci_f0_config */ + +#define PCRF_CZ_VF_SYS_PAGESZ_LBN 0 +#define PCRF_CZ_VF_SYS_PAGESZ_WIDTH 16 + + +/* + * PC_SRIOV_BAR0_REG(32bit): + * SRIOV VF Bar0 + */ + +#define PCR_CC_SRIOV_BAR0_REG 0x00000184 +/* sienaa0=pci_f0_config */ + +#define PCR_DZ_SRIOV_BAR0_REG 0x000001a4 +/* hunta0=pci_f0_config */ + +#define PCRF_CC_VF_BAR_ADDRESS_LBN 0 +#define PCRF_CC_VF_BAR_ADDRESS_WIDTH 32 +#define PCRF_DZ_VF_BAR0_ADDRESS_LBN 4 +#define PCRF_DZ_VF_BAR0_ADDRESS_WIDTH 28 +#define PCRF_DZ_VF_BAR0_PREF_LBN 3 +#define PCRF_DZ_VF_BAR0_PREF_WIDTH 1 +#define PCRF_DZ_VF_BAR0_TYPE_LBN 1 +#define PCRF_DZ_VF_BAR0_TYPE_WIDTH 2 +#define PCRF_DZ_VF_BAR0_IOM_LBN 0 +#define PCRF_DZ_VF_BAR0_IOM_WIDTH 1 + + +/* + * PC_SRIOV_BAR1_REG(32bit): + * SRIOV Bar1 + */ + +#define PCR_CC_SRIOV_BAR1_REG 0x00000188 +/* sienaa0=pci_f0_config */ + +#define PCR_DZ_SRIOV_BAR1_REG 0x000001a8 +/* hunta0=pci_f0_config */ + +/* defined as PCRF_CC_VF_BAR_ADDRESS_LBN 0; */ +/* defined as PCRF_CC_VF_BAR_ADDRESS_WIDTH 32 */ +#define PCRF_DZ_VF_BAR1_ADDRESS_LBN 0 +#define PCRF_DZ_VF_BAR1_ADDRESS_WIDTH 32 + + +/* + * PC_SRIOV_BAR2_REG(32bit): + * SRIOV Bar2 + */ + +#define PCR_CC_SRIOV_BAR2_REG 0x0000018c +/* sienaa0=pci_f0_config */ + +#define PCR_DZ_SRIOV_BAR2_REG 0x000001ac +/* hunta0=pci_f0_config */ + +/* defined as PCRF_CC_VF_BAR_ADDRESS_LBN 0; */ +/* defined as PCRF_CC_VF_BAR_ADDRESS_WIDTH 32 */ +#define PCRF_DZ_VF_BAR2_ADDRESS_LBN 4 +#define PCRF_DZ_VF_BAR2_ADDRESS_WIDTH 28 +#define PCRF_DZ_VF_BAR2_PREF_LBN 3 +#define PCRF_DZ_VF_BAR2_PREF_WIDTH 1 +#define PCRF_DZ_VF_BAR2_TYPE_LBN 1 +#define PCRF_DZ_VF_BAR2_TYPE_WIDTH 2 +#define PCRF_DZ_VF_BAR2_IOM_LBN 0 +#define PCRF_DZ_VF_BAR2_IOM_WIDTH 1 + + +/* + * PC_SRIOV_BAR3_REG(32bit): + * SRIOV Bar3 + */ + +#define PCR_CC_SRIOV_BAR3_REG 0x00000190 +/* sienaa0=pci_f0_config */ + +#define PCR_DZ_SRIOV_BAR3_REG 0x000001b0 +/* hunta0=pci_f0_config */ + +/* defined as PCRF_CC_VF_BAR_ADDRESS_LBN 0; */ +/* defined as PCRF_CC_VF_BAR_ADDRESS_WIDTH 32 */ +#define PCRF_DZ_VF_BAR3_ADDRESS_LBN 0 +#define PCRF_DZ_VF_BAR3_ADDRESS_WIDTH 32 + + +/* + * PC_SRIOV_BAR4_REG(32bit): + * SRIOV Bar4 + */ + +#define PCR_CC_SRIOV_BAR4_REG 0x00000194 +/* sienaa0=pci_f0_config */ + +#define PCR_DZ_SRIOV_BAR4_REG 0x000001b4 +/* hunta0=pci_f0_config */ + +/* defined as PCRF_CC_VF_BAR_ADDRESS_LBN 0; */ +/* defined as PCRF_CC_VF_BAR_ADDRESS_WIDTH 32 */ +#define PCRF_DZ_VF_BAR4_ADDRESS_LBN 0 +#define PCRF_DZ_VF_BAR4_ADDRESS_WIDTH 32 + + +/* + * PC_SRIOV_BAR5_REG(32bit): + * SRIOV Bar5 + */ + +#define PCR_CC_SRIOV_BAR5_REG 0x00000198 +/* sienaa0=pci_f0_config */ + +#define PCR_DZ_SRIOV_BAR5_REG 0x000001b8 +/* hunta0=pci_f0_config */ + +/* defined as PCRF_CC_VF_BAR_ADDRESS_LBN 0; */ +/* defined as PCRF_CC_VF_BAR_ADDRESS_WIDTH 32 */ +#define PCRF_DZ_VF_BAR5_ADDRESS_LBN 0 +#define PCRF_DZ_VF_BAR5_ADDRESS_WIDTH 32 + + +/* + * PC_SRIOV_RSVD_REG(16bit): + * Reserved register + */ + +#define PCR_DZ_SRIOV_RSVD_REG 0x00000198 +/* hunta0=pci_f0_config */ + +#define PCRF_DZ_VF_RSVD_LBN 0 +#define PCRF_DZ_VF_RSVD_WIDTH 16 + + +/* + * PC_SRIOV_MIBR_SARRAY_OFFSET_REG(32bit): + * SRIOV VF Migration State Array Offset + */ + +#define PCR_CC_SRIOV_MIBR_SARRAY_OFFSET_REG 0x0000019c +/* sienaa0=pci_f0_config */ + +#define PCR_DZ_SRIOV_MIBR_SARRAY_OFFSET_REG 0x000001bc +/* hunta0=pci_f0_config */ + +#define PCRF_CZ_VF_MIGR_OFFSET_LBN 3 +#define PCRF_CZ_VF_MIGR_OFFSET_WIDTH 29 +#define PCRF_CZ_VF_MIGR_BIR_LBN 0 +#define PCRF_CZ_VF_MIGR_BIR_WIDTH 3 + + +/* + * PC_TPH_CAP_HDR_REG(32bit): + * TPH Capability Header Register + */ + +#define PCR_DZ_TPH_CAP_HDR_REG 0x000001c0 +/* hunta0=pci_f0_config */ + +#define PCRF_DZ_TPH_NXT_PTR_LBN 20 +#define PCRF_DZ_TPH_NXT_PTR_WIDTH 12 +#define PCRF_DZ_TPH_VERSION_LBN 16 +#define PCRF_DZ_TPH_VERSION_WIDTH 4 +#define PCRF_DZ_TPH_EXT_CAP_ID_LBN 0 +#define PCRF_DZ_TPH_EXT_CAP_ID_WIDTH 16 + + +/* + * PC_TPH_REQ_CAP_REG(32bit): + * TPH Requester Capability Register + */ + +#define PCR_DZ_TPH_REQ_CAP_REG 0x000001c4 +/* hunta0=pci_f0_config */ + +#define PCRF_DZ_ST_TBLE_SIZE_LBN 16 +#define PCRF_DZ_ST_TBLE_SIZE_WIDTH 11 +#define PCRF_DZ_ST_TBLE_LOC_LBN 9 +#define PCRF_DZ_ST_TBLE_LOC_WIDTH 2 +#define PCRF_DZ_EXT_TPH_MODE_SUP_LBN 8 +#define PCRF_DZ_EXT_TPH_MODE_SUP_WIDTH 1 +#define PCRF_DZ_TPH_DEV_MODE_SUP_LBN 2 +#define PCRF_DZ_TPH_DEV_MODE_SUP_WIDTH 1 +#define PCRF_DZ_TPH_INT_MODE_SUP_LBN 1 +#define PCRF_DZ_TPH_INT_MODE_SUP_WIDTH 1 +#define PCRF_DZ_TPH_NOST_MODE_SUP_LBN 0 +#define PCRF_DZ_TPH_NOST_MODE_SUP_WIDTH 1 + + +/* + * PC_TPH_REQ_CTL_REG(32bit): + * TPH Requester Control Register + */ + +#define PCR_DZ_TPH_REQ_CTL_REG 0x000001c8 +/* hunta0=pci_f0_config */ + +#define PCRF_DZ_TPH_REQ_ENABLE_LBN 8 +#define PCRF_DZ_TPH_REQ_ENABLE_WIDTH 2 +#define PCRF_DZ_TPH_ST_MODE_LBN 0 +#define PCRF_DZ_TPH_ST_MODE_WIDTH 3 + + +/* + * PC_LTR_CAP_HDR_REG(32bit): + * Latency Tolerance Reporting Cap Header Reg + */ + +#define PCR_DZ_LTR_CAP_HDR_REG 0x00000290 +/* hunta0=pci_f0_config */ + +#define PCRF_DZ_LTR_NXT_PTR_LBN 20 +#define PCRF_DZ_LTR_NXT_PTR_WIDTH 12 +#define PCRF_DZ_LTR_VERSION_LBN 16 +#define PCRF_DZ_LTR_VERSION_WIDTH 4 +#define PCRF_DZ_LTR_EXT_CAP_ID_LBN 0 +#define PCRF_DZ_LTR_EXT_CAP_ID_WIDTH 16 + + +/* + * PC_LTR_MAX_SNOOP_REG(32bit): + * LTR Maximum Snoop/No Snoop Register + */ + +#define PCR_DZ_LTR_MAX_SNOOP_REG 0x00000294 +/* hunta0=pci_f0_config */ + +#define PCRF_DZ_LTR_MAX_NOSNOOP_SCALE_LBN 26 +#define PCRF_DZ_LTR_MAX_NOSNOOP_SCALE_WIDTH 3 +#define PCRF_DZ_LTR_MAX_NOSNOOP_LAT_LBN 16 +#define PCRF_DZ_LTR_MAX_NOSNOOP_LAT_WIDTH 10 +#define PCRF_DZ_LTR_MAX_SNOOP_SCALE_LBN 10 +#define PCRF_DZ_LTR_MAX_SNOOP_SCALE_WIDTH 3 +#define PCRF_DZ_LTR_MAX_SNOOP_LAT_LBN 0 +#define PCRF_DZ_LTR_MAX_SNOOP_LAT_WIDTH 10 + + +/* + * PC_ACK_LAT_TMR_REG(32bit): + * ACK latency timer & replay timer register + */ + +#define PCR_AC_ACK_LAT_TMR_REG 0x00000700 +/* falcona0,falconb0,sienaa0=pci_f0_config */ + +#define PCRF_AC_RT_LBN 16 +#define PCRF_AC_RT_WIDTH 16 +#define PCRF_AC_ALT_LBN 0 +#define PCRF_AC_ALT_WIDTH 16 + + +/* + * PC_OTHER_MSG_REG(32bit): + * Other message register + */ + +#define PCR_AC_OTHER_MSG_REG 0x00000704 +/* falcona0,falconb0,sienaa0=pci_f0_config */ + +#define PCRF_AC_OM_CRPT3_LBN 24 +#define PCRF_AC_OM_CRPT3_WIDTH 8 +#define PCRF_AC_OM_CRPT2_LBN 16 +#define PCRF_AC_OM_CRPT2_WIDTH 8 +#define PCRF_AC_OM_CRPT1_LBN 8 +#define PCRF_AC_OM_CRPT1_WIDTH 8 +#define PCRF_AC_OM_CRPT0_LBN 0 +#define PCRF_AC_OM_CRPT0_WIDTH 8 + + +/* + * PC_FORCE_LNK_REG(24bit): + * Port force link register + */ + +#define PCR_AC_FORCE_LNK_REG 0x00000708 +/* falcona0,falconb0,sienaa0=pci_f0_config */ + +#define PCRF_AC_LFS_LBN 16 +#define PCRF_AC_LFS_WIDTH 6 +#define PCRF_AC_FL_LBN 15 +#define PCRF_AC_FL_WIDTH 1 +#define PCRF_AC_LN_LBN 0 +#define PCRF_AC_LN_WIDTH 8 + + +/* + * PC_ACK_FREQ_REG(32bit): + * ACK frequency register + */ + +#define PCR_AC_ACK_FREQ_REG 0x0000070c +/* falcona0,falconb0,sienaa0=pci_f0_config */ + +#define PCRF_CC_ALLOW_L1_WITHOUT_L0S_LBN 30 +#define PCRF_CC_ALLOW_L1_WITHOUT_L0S_WIDTH 1 +#define PCRF_AC_L1_ENTR_LAT_LBN 27 +#define PCRF_AC_L1_ENTR_LAT_WIDTH 3 +#define PCRF_AC_L0_ENTR_LAT_LBN 24 +#define PCRF_AC_L0_ENTR_LAT_WIDTH 3 +#define PCRF_CC_COMM_NFTS_LBN 16 +#define PCRF_CC_COMM_NFTS_WIDTH 8 +#define PCRF_AB_ACK_FREQ_REG_RSVD0_LBN 16 +#define PCRF_AB_ACK_FREQ_REG_RSVD0_WIDTH 3 +#define PCRF_AC_MAX_FTS_LBN 8 +#define PCRF_AC_MAX_FTS_WIDTH 8 +#define PCRF_AC_ACK_FREQ_LBN 0 +#define PCRF_AC_ACK_FREQ_WIDTH 8 + + +/* + * PC_PORT_LNK_CTL_REG(32bit): + * Port link control register + */ + +#define PCR_AC_PORT_LNK_CTL_REG 0x00000710 +/* falcona0,falconb0,sienaa0=pci_f0_config */ + +#define PCRF_AB_LRE_LBN 27 +#define PCRF_AB_LRE_WIDTH 1 +#define PCRF_AB_ESYNC_LBN 26 +#define PCRF_AB_ESYNC_WIDTH 1 +#define PCRF_AB_CRPT_LBN 25 +#define PCRF_AB_CRPT_WIDTH 1 +#define PCRF_AB_XB_LBN 24 +#define PCRF_AB_XB_WIDTH 1 +#define PCRF_AC_LC_LBN 16 +#define PCRF_AC_LC_WIDTH 6 +#define PCRF_AC_LDR_LBN 8 +#define PCRF_AC_LDR_WIDTH 4 +#define PCRF_AC_FLM_LBN 7 +#define PCRF_AC_FLM_WIDTH 1 +#define PCRF_AC_LKD_LBN 6 +#define PCRF_AC_LKD_WIDTH 1 +#define PCRF_AC_DLE_LBN 5 +#define PCRF_AC_DLE_WIDTH 1 +#define PCRF_AB_PORT_LNK_CTL_REG_RSVD0_LBN 4 +#define PCRF_AB_PORT_LNK_CTL_REG_RSVD0_WIDTH 1 +#define PCRF_AC_RA_LBN 3 +#define PCRF_AC_RA_WIDTH 1 +#define PCRF_AC_LE_LBN 2 +#define PCRF_AC_LE_WIDTH 1 +#define PCRF_AC_SD_LBN 1 +#define PCRF_AC_SD_WIDTH 1 +#define PCRF_AC_OMR_LBN 0 +#define PCRF_AC_OMR_WIDTH 1 + + +/* + * PC_LN_SKEW_REG(32bit): + * Lane skew register + */ + +#define PCR_AC_LN_SKEW_REG 0x00000714 +/* falcona0,falconb0,sienaa0=pci_f0_config */ + +#define PCRF_AC_DIS_LBN 31 +#define PCRF_AC_DIS_WIDTH 1 +#define PCRF_AB_RST_LBN 30 +#define PCRF_AB_RST_WIDTH 1 +#define PCRF_AC_AD_LBN 25 +#define PCRF_AC_AD_WIDTH 1 +#define PCRF_AC_FCD_LBN 24 +#define PCRF_AC_FCD_WIDTH 1 +#define PCRF_AC_LS2_LBN 16 +#define PCRF_AC_LS2_WIDTH 8 +#define PCRF_AC_LS1_LBN 8 +#define PCRF_AC_LS1_WIDTH 8 +#define PCRF_AC_LS0_LBN 0 +#define PCRF_AC_LS0_WIDTH 8 + + +/* + * PC_SYM_NUM_REG(16bit): + * Symbol number register + */ + +#define PCR_AC_SYM_NUM_REG 0x00000718 +/* falcona0,falconb0,sienaa0=pci_f0_config */ + +#define PCRF_CC_MAX_FUNCTIONS_LBN 29 +#define PCRF_CC_MAX_FUNCTIONS_WIDTH 3 +#define PCRF_CC_FC_WATCHDOG_TMR_LBN 24 +#define PCRF_CC_FC_WATCHDOG_TMR_WIDTH 5 +#define PCRF_CC_ACK_NAK_TMR_MOD_LBN 19 +#define PCRF_CC_ACK_NAK_TMR_MOD_WIDTH 5 +#define PCRF_CC_REPLAY_TMR_MOD_LBN 14 +#define PCRF_CC_REPLAY_TMR_MOD_WIDTH 5 +#define PCRF_AB_ES_LBN 12 +#define PCRF_AB_ES_WIDTH 3 +#define PCRF_AB_SYM_NUM_REG_RSVD0_LBN 11 +#define PCRF_AB_SYM_NUM_REG_RSVD0_WIDTH 1 +#define PCRF_CC_NUM_SKP_SYMS_LBN 8 +#define PCRF_CC_NUM_SKP_SYMS_WIDTH 3 +#define PCRF_AB_TS2_LBN 4 +#define PCRF_AB_TS2_WIDTH 4 +#define PCRF_AC_TS1_LBN 0 +#define PCRF_AC_TS1_WIDTH 4 + + +/* + * PC_SYM_TMR_FLT_MSK_REG(16bit): + * Symbol timer and Filter Mask Register + */ + +#define PCR_CC_SYM_TMR_FLT_MSK_REG 0x0000071c +/* sienaa0=pci_f0_config */ + +#define PCRF_CC_DEFAULT_FLT_MSK1_LBN 16 +#define PCRF_CC_DEFAULT_FLT_MSK1_WIDTH 16 +#define PCRF_CC_FC_WDOG_TMR_DIS_LBN 15 +#define PCRF_CC_FC_WDOG_TMR_DIS_WIDTH 1 +#define PCRF_CC_SI1_LBN 8 +#define PCRF_CC_SI1_WIDTH 3 +#define PCRF_CC_SKIP_INT_VAL_LBN 0 +#define PCRF_CC_SKIP_INT_VAL_WIDTH 11 +#define PCRF_CC_SI0_LBN 0 +#define PCRF_CC_SI0_WIDTH 8 + + +/* + * PC_SYM_TMR_REG(16bit): + * Symbol timer register + */ + +#define PCR_AB_SYM_TMR_REG 0x0000071c +/* falcona0,falconb0=pci_f0_config */ + +#define PCRF_AB_ET_LBN 11 +#define PCRF_AB_ET_WIDTH 4 +#define PCRF_AB_SI1_LBN 8 +#define PCRF_AB_SI1_WIDTH 3 +#define PCRF_AB_SI0_LBN 0 +#define PCRF_AB_SI0_WIDTH 8 + + +/* + * PC_FLT_MSK_REG(32bit): + * Filter Mask Register 2 + */ + +#define PCR_CC_FLT_MSK_REG 0x00000720 +/* sienaa0=pci_f0_config */ + +#define PCRF_CC_DEFAULT_FLT_MSK2_LBN 0 +#define PCRF_CC_DEFAULT_FLT_MSK2_WIDTH 32 + + +/* + * PC_PHY_STAT_REG(32bit): + * PHY status register + */ + +#define PCR_AB_PHY_STAT_REG 0x00000720 +/* falcona0,falconb0=pci_f0_config */ + +#define PCR_CC_PHY_STAT_REG 0x00000810 +/* sienaa0=pci_f0_config */ + +#define PCRF_AC_SSL_LBN 3 +#define PCRF_AC_SSL_WIDTH 1 +#define PCRF_AC_SSR_LBN 2 +#define PCRF_AC_SSR_WIDTH 1 +#define PCRF_AC_SSCL_LBN 1 +#define PCRF_AC_SSCL_WIDTH 1 +#define PCRF_AC_SSCD_LBN 0 +#define PCRF_AC_SSCD_WIDTH 1 + + +/* + * PC_PHY_CTL_REG(32bit): + * PHY control register + */ + +#define PCR_AB_PHY_CTL_REG 0x00000724 +/* falcona0,falconb0=pci_f0_config */ + +#define PCR_CC_PHY_CTL_REG 0x00000814 +/* sienaa0=pci_f0_config */ + +#define PCRF_AC_BD_LBN 31 +#define PCRF_AC_BD_WIDTH 1 +#define PCRF_AC_CDS_LBN 30 +#define PCRF_AC_CDS_WIDTH 1 +#define PCRF_AC_DWRAP_LB_LBN 29 +#define PCRF_AC_DWRAP_LB_WIDTH 1 +#define PCRF_AC_EBD_LBN 28 +#define PCRF_AC_EBD_WIDTH 1 +#define PCRF_AC_SNR_LBN 27 +#define PCRF_AC_SNR_WIDTH 1 +#define PCRF_AC_RX_NOT_DET_LBN 2 +#define PCRF_AC_RX_NOT_DET_WIDTH 1 +#define PCRF_AC_FORCE_LOS_VAL_LBN 1 +#define PCRF_AC_FORCE_LOS_VAL_WIDTH 1 +#define PCRF_AC_FORCE_LOS_EN_LBN 0 +#define PCRF_AC_FORCE_LOS_EN_WIDTH 1 + + +/* + * PC_DEBUG0_REG(32bit): + * Debug register 0 + */ + +#define PCR_AC_DEBUG0_REG 0x00000728 +/* falcona0,falconb0,sienaa0=pci_f0_config */ + +#define PCRF_AC_CDI03_LBN 24 +#define PCRF_AC_CDI03_WIDTH 8 +#define PCRF_AC_CDI0_LBN 0 +#define PCRF_AC_CDI0_WIDTH 32 +#define PCRF_AC_CDI02_LBN 16 +#define PCRF_AC_CDI02_WIDTH 8 +#define PCRF_AC_CDI01_LBN 8 +#define PCRF_AC_CDI01_WIDTH 8 +#define PCRF_AC_CDI00_LBN 0 +#define PCRF_AC_CDI00_WIDTH 8 + + +/* + * PC_DEBUG1_REG(32bit): + * Debug register 1 + */ + +#define PCR_AC_DEBUG1_REG 0x0000072c +/* falcona0,falconb0,sienaa0=pci_f0_config */ + +#define PCRF_AC_CDI13_LBN 24 +#define PCRF_AC_CDI13_WIDTH 8 +#define PCRF_AC_CDI1_LBN 0 +#define PCRF_AC_CDI1_WIDTH 32 +#define PCRF_AC_CDI12_LBN 16 +#define PCRF_AC_CDI12_WIDTH 8 +#define PCRF_AC_CDI11_LBN 8 +#define PCRF_AC_CDI11_WIDTH 8 +#define PCRF_AC_CDI10_LBN 0 +#define PCRF_AC_CDI10_WIDTH 8 + + +/* + * PC_XPFCC_STAT_REG(24bit): + * documentation to be written for sum_PC_XPFCC_STAT_REG + */ + +#define PCR_AC_XPFCC_STAT_REG 0x00000730 +/* falcona0,falconb0,sienaa0=pci_f0_config */ + +#define PCRF_AC_XPDC_LBN 12 +#define PCRF_AC_XPDC_WIDTH 8 +#define PCRF_AC_XPHC_LBN 0 +#define PCRF_AC_XPHC_WIDTH 12 + + +/* + * PC_XNPFCC_STAT_REG(24bit): + * documentation to be written for sum_PC_XNPFCC_STAT_REG + */ + +#define PCR_AC_XNPFCC_STAT_REG 0x00000734 +/* falcona0,falconb0,sienaa0=pci_f0_config */ + +#define PCRF_AC_XNPDC_LBN 12 +#define PCRF_AC_XNPDC_WIDTH 8 +#define PCRF_AC_XNPHC_LBN 0 +#define PCRF_AC_XNPHC_WIDTH 12 + + +/* + * PC_XCFCC_STAT_REG(24bit): + * documentation to be written for sum_PC_XCFCC_STAT_REG + */ + +#define PCR_AC_XCFCC_STAT_REG 0x00000738 +/* falcona0,falconb0,sienaa0=pci_f0_config */ + +#define PCRF_AC_XCDC_LBN 12 +#define PCRF_AC_XCDC_WIDTH 8 +#define PCRF_AC_XCHC_LBN 0 +#define PCRF_AC_XCHC_WIDTH 12 + + +/* + * PC_Q_STAT_REG(8bit): + * documentation to be written for sum_PC_Q_STAT_REG + */ + +#define PCR_AC_Q_STAT_REG 0x0000073c +/* falcona0,falconb0,sienaa0=pci_f0_config */ + +#define PCRF_AC_RQNE_LBN 2 +#define PCRF_AC_RQNE_WIDTH 1 +#define PCRF_AC_XRNE_LBN 1 +#define PCRF_AC_XRNE_WIDTH 1 +#define PCRF_AC_RCNR_LBN 0 +#define PCRF_AC_RCNR_WIDTH 1 + + +/* + * PC_VC_XMIT_ARB1_REG(32bit): + * VC Transmit Arbitration Register 1 + */ + +#define PCR_CC_VC_XMIT_ARB1_REG 0x00000740 +/* sienaa0=pci_f0_config */ + + + +/* + * PC_VC_XMIT_ARB2_REG(32bit): + * VC Transmit Arbitration Register 2 + */ + +#define PCR_CC_VC_XMIT_ARB2_REG 0x00000744 +/* sienaa0=pci_f0_config */ + + + +/* + * PC_VC0_P_RQ_CTL_REG(32bit): + * VC0 Posted Receive Queue Control + */ + +#define PCR_CC_VC0_P_RQ_CTL_REG 0x00000748 +/* sienaa0=pci_f0_config */ + + + +/* + * PC_VC0_NP_RQ_CTL_REG(32bit): + * VC0 Non-Posted Receive Queue Control + */ + +#define PCR_CC_VC0_NP_RQ_CTL_REG 0x0000074c +/* sienaa0=pci_f0_config */ + + + +/* + * PC_VC0_C_RQ_CTL_REG(32bit): + * VC0 Completion Receive Queue Control + */ + +#define PCR_CC_VC0_C_RQ_CTL_REG 0x00000750 +/* sienaa0=pci_f0_config */ + + + +/* + * PC_GEN2_REG(32bit): + * Gen2 Register + */ + +#define PCR_CC_GEN2_REG 0x0000080c +/* sienaa0=pci_f0_config */ + +#define PCRF_CC_SET_DE_EMPHASIS_LBN 20 +#define PCRF_CC_SET_DE_EMPHASIS_WIDTH 1 +#define PCRF_CC_CFG_TX_COMPLIANCE_LBN 19 +#define PCRF_CC_CFG_TX_COMPLIANCE_WIDTH 1 +#define PCRF_CC_CFG_TX_SWING_LBN 18 +#define PCRF_CC_CFG_TX_SWING_WIDTH 1 +#define PCRF_CC_DIR_SPEED_CHANGE_LBN 17 +#define PCRF_CC_DIR_SPEED_CHANGE_WIDTH 1 +#define PCRF_CC_LANE_ENABLE_LBN 8 +#define PCRF_CC_LANE_ENABLE_WIDTH 9 +#define PCRF_CC_NUM_FTS_LBN 0 +#define PCRF_CC_NUM_FTS_WIDTH 8 + + +#ifdef __cplusplus +} +#endif + +#endif /* _SYS_EFX_REGS_PCI_H */ diff --git a/dpdk/drivers/net/sfc/base/efx_rx.c b/dpdk/drivers/net/sfc/base/efx_rx.c new file mode 100644 index 00000000..785365d3 --- /dev/null +++ b/dpdk/drivers/net/sfc/base/efx_rx.c @@ -0,0 +1,1419 @@ +/* + * Copyright (c) 2007-2016 Solarflare Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are + * those of the authors and should not be interpreted as representing official + * policies, either expressed or implied, of the FreeBSD Project. + */ + +#include "efx.h" +#include "efx_impl.h" + + +#if EFSYS_OPT_SIENA + +static __checkReturn efx_rc_t +siena_rx_init( + __in efx_nic_t *enp); + +static void +siena_rx_fini( + __in efx_nic_t *enp); + +#if EFSYS_OPT_RX_SCATTER +static __checkReturn efx_rc_t +siena_rx_scatter_enable( + __in efx_nic_t *enp, + __in unsigned int buf_size); +#endif /* EFSYS_OPT_RX_SCATTER */ + +#if EFSYS_OPT_RX_SCALE +static __checkReturn efx_rc_t +siena_rx_scale_mode_set( + __in efx_nic_t *enp, + __in uint32_t rss_context, + __in efx_rx_hash_alg_t alg, + __in efx_rx_hash_type_t type, + __in boolean_t insert); + +static __checkReturn efx_rc_t +siena_rx_scale_key_set( + __in efx_nic_t *enp, + __in uint32_t rss_context, + __in_ecount(n) uint8_t *key, + __in size_t n); + +static __checkReturn efx_rc_t +siena_rx_scale_tbl_set( + __in efx_nic_t *enp, + __in uint32_t rss_context, + __in_ecount(n) unsigned int *table, + __in size_t n); + +static __checkReturn uint32_t +siena_rx_prefix_hash( + __in efx_nic_t *enp, + __in efx_rx_hash_alg_t func, + __in uint8_t *buffer); + +#endif /* EFSYS_OPT_RX_SCALE */ + +static __checkReturn efx_rc_t +siena_rx_prefix_pktlen( + __in efx_nic_t *enp, + __in uint8_t *buffer, + __out uint16_t *lengthp); + +static void +siena_rx_qpost( + __in efx_rxq_t *erp, + __in_ecount(n) efsys_dma_addr_t *addrp, + __in size_t size, + __in unsigned int n, + __in unsigned int completed, + __in unsigned int added); + +static void +siena_rx_qpush( + __in efx_rxq_t *erp, + __in unsigned int added, + __inout unsigned int *pushedp); + +#if EFSYS_OPT_RX_PACKED_STREAM +static void +siena_rx_qps_update_credits( + __in efx_rxq_t *erp); + +static __checkReturn uint8_t * +siena_rx_qps_packet_info( + __in efx_rxq_t *erp, + __in uint8_t *buffer, + __in uint32_t buffer_length, + __in uint32_t current_offset, + __out uint16_t *lengthp, + __out uint32_t *next_offsetp, + __out uint32_t *timestamp); +#endif + +static __checkReturn efx_rc_t +siena_rx_qflush( + __in efx_rxq_t *erp); + +static void +siena_rx_qenable( + __in efx_rxq_t *erp); + +static __checkReturn efx_rc_t +siena_rx_qcreate( + __in efx_nic_t *enp, + __in unsigned int index, + __in unsigned int label, + __in efx_rxq_type_t type, + __in efsys_mem_t *esmp, + __in size_t n, + __in uint32_t id, + __in efx_evq_t *eep, + __in efx_rxq_t *erp); + +static void +siena_rx_qdestroy( + __in efx_rxq_t *erp); + +#endif /* EFSYS_OPT_SIENA */ + + +#if EFSYS_OPT_SIENA +static const efx_rx_ops_t __efx_rx_siena_ops = { + siena_rx_init, /* erxo_init */ + siena_rx_fini, /* erxo_fini */ +#if EFSYS_OPT_RX_SCATTER + siena_rx_scatter_enable, /* erxo_scatter_enable */ +#endif +#if EFSYS_OPT_RX_SCALE + NULL, /* erxo_scale_context_alloc */ + NULL, /* erxo_scale_context_free */ + siena_rx_scale_mode_set, /* erxo_scale_mode_set */ + siena_rx_scale_key_set, /* erxo_scale_key_set */ + siena_rx_scale_tbl_set, /* erxo_scale_tbl_set */ + siena_rx_prefix_hash, /* erxo_prefix_hash */ +#endif + siena_rx_prefix_pktlen, /* erxo_prefix_pktlen */ + siena_rx_qpost, /* erxo_qpost */ + siena_rx_qpush, /* erxo_qpush */ +#if EFSYS_OPT_RX_PACKED_STREAM + siena_rx_qps_update_credits, /* erxo_qps_update_credits */ + siena_rx_qps_packet_info, /* erxo_qps_packet_info */ +#endif + siena_rx_qflush, /* erxo_qflush */ + siena_rx_qenable, /* erxo_qenable */ + siena_rx_qcreate, /* erxo_qcreate */ + siena_rx_qdestroy, /* erxo_qdestroy */ +}; +#endif /* EFSYS_OPT_SIENA */ + +#if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD +static const efx_rx_ops_t __efx_rx_ef10_ops = { + ef10_rx_init, /* erxo_init */ + ef10_rx_fini, /* erxo_fini */ +#if EFSYS_OPT_RX_SCATTER + ef10_rx_scatter_enable, /* erxo_scatter_enable */ +#endif +#if EFSYS_OPT_RX_SCALE + ef10_rx_scale_context_alloc, /* erxo_scale_context_alloc */ + ef10_rx_scale_context_free, /* erxo_scale_context_free */ + ef10_rx_scale_mode_set, /* erxo_scale_mode_set */ + ef10_rx_scale_key_set, /* erxo_scale_key_set */ + ef10_rx_scale_tbl_set, /* erxo_scale_tbl_set */ + ef10_rx_prefix_hash, /* erxo_prefix_hash */ +#endif + ef10_rx_prefix_pktlen, /* erxo_prefix_pktlen */ + ef10_rx_qpost, /* erxo_qpost */ + ef10_rx_qpush, /* erxo_qpush */ +#if EFSYS_OPT_RX_PACKED_STREAM + ef10_rx_qps_update_credits, /* erxo_qps_update_credits */ + ef10_rx_qps_packet_info, /* erxo_qps_packet_info */ +#endif + ef10_rx_qflush, /* erxo_qflush */ + ef10_rx_qenable, /* erxo_qenable */ + ef10_rx_qcreate, /* erxo_qcreate */ + ef10_rx_qdestroy, /* erxo_qdestroy */ +}; +#endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */ + + + __checkReturn efx_rc_t +efx_rx_init( + __inout efx_nic_t *enp) +{ + const efx_rx_ops_t *erxop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); + + if (!(enp->en_mod_flags & EFX_MOD_EV)) { + rc = EINVAL; + goto fail1; + } + + if (enp->en_mod_flags & EFX_MOD_RX) { + rc = EINVAL; + goto fail2; + } + + switch (enp->en_family) { +#if EFSYS_OPT_SIENA + case EFX_FAMILY_SIENA: + erxop = &__efx_rx_siena_ops; + break; +#endif /* EFSYS_OPT_SIENA */ + +#if EFSYS_OPT_HUNTINGTON + case EFX_FAMILY_HUNTINGTON: + erxop = &__efx_rx_ef10_ops; + break; +#endif /* EFSYS_OPT_HUNTINGTON */ + +#if EFSYS_OPT_MEDFORD + case EFX_FAMILY_MEDFORD: + erxop = &__efx_rx_ef10_ops; + break; +#endif /* EFSYS_OPT_MEDFORD */ + + default: + EFSYS_ASSERT(0); + rc = ENOTSUP; + goto fail3; + } + + if ((rc = erxop->erxo_init(enp)) != 0) + goto fail4; + + enp->en_erxop = erxop; + enp->en_mod_flags |= EFX_MOD_RX; + return (0); + +fail4: + EFSYS_PROBE(fail4); +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + enp->en_erxop = NULL; + enp->en_mod_flags &= ~EFX_MOD_RX; + return (rc); +} + + void +efx_rx_fini( + __in efx_nic_t *enp) +{ + const efx_rx_ops_t *erxop = enp->en_erxop; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); + EFSYS_ASSERT3U(enp->en_rx_qcount, ==, 0); + + erxop->erxo_fini(enp); + + enp->en_erxop = NULL; + enp->en_mod_flags &= ~EFX_MOD_RX; +} + +#if EFSYS_OPT_RX_SCATTER + __checkReturn efx_rc_t +efx_rx_scatter_enable( + __in efx_nic_t *enp, + __in unsigned int buf_size) +{ + const efx_rx_ops_t *erxop = enp->en_erxop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); + + if ((rc = erxop->erxo_scatter_enable(enp, buf_size)) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + return (rc); +} +#endif /* EFSYS_OPT_RX_SCATTER */ + +#if EFSYS_OPT_RX_SCALE + __checkReturn efx_rc_t +efx_rx_hash_default_support_get( + __in efx_nic_t *enp, + __out efx_rx_hash_support_t *supportp) +{ + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); + + if (supportp == NULL) { + rc = EINVAL; + goto fail1; + } + + /* + * Report the hashing support the client gets by default if it + * does not allocate an RSS context itself. + */ + *supportp = enp->en_hash_support; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_rx_scale_default_support_get( + __in efx_nic_t *enp, + __out efx_rx_scale_context_type_t *typep) +{ + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); + + if (typep == NULL) { + rc = EINVAL; + goto fail1; + } + + /* + * Report the RSS support the client gets by default if it + * does not allocate an RSS context itself. + */ + *typep = enp->en_rss_context_type; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} +#endif /* EFSYS_OPT_RX_SCALE */ + +#if EFSYS_OPT_RX_SCALE + __checkReturn efx_rc_t +efx_rx_scale_context_alloc( + __in efx_nic_t *enp, + __in efx_rx_scale_context_type_t type, + __in uint32_t num_queues, + __out uint32_t *rss_contextp) +{ + const efx_rx_ops_t *erxop = enp->en_erxop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); + + if (erxop->erxo_scale_context_alloc == NULL) { + rc = ENOTSUP; + goto fail1; + } + if ((rc = erxop->erxo_scale_context_alloc(enp, type, + num_queues, rss_contextp)) != 0) { + goto fail2; + } + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + return (rc); +} +#endif /* EFSYS_OPT_RX_SCALE */ + +#if EFSYS_OPT_RX_SCALE + __checkReturn efx_rc_t +efx_rx_scale_context_free( + __in efx_nic_t *enp, + __in uint32_t rss_context) +{ + const efx_rx_ops_t *erxop = enp->en_erxop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); + + if (erxop->erxo_scale_context_free == NULL) { + rc = ENOTSUP; + goto fail1; + } + if ((rc = erxop->erxo_scale_context_free(enp, rss_context)) != 0) + goto fail2; + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + return (rc); +} +#endif /* EFSYS_OPT_RX_SCALE */ + +#if EFSYS_OPT_RX_SCALE + __checkReturn efx_rc_t +efx_rx_scale_mode_set( + __in efx_nic_t *enp, + __in uint32_t rss_context, + __in efx_rx_hash_alg_t alg, + __in efx_rx_hash_type_t type, + __in boolean_t insert) +{ + const efx_rx_ops_t *erxop = enp->en_erxop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); + + if (erxop->erxo_scale_mode_set != NULL) { + if ((rc = erxop->erxo_scale_mode_set(enp, rss_context, alg, + type, insert)) != 0) + goto fail1; + } + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + return (rc); +} +#endif /* EFSYS_OPT_RX_SCALE */ + +#if EFSYS_OPT_RX_SCALE + __checkReturn efx_rc_t +efx_rx_scale_key_set( + __in efx_nic_t *enp, + __in uint32_t rss_context, + __in_ecount(n) uint8_t *key, + __in size_t n) +{ + const efx_rx_ops_t *erxop = enp->en_erxop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); + + if ((rc = erxop->erxo_scale_key_set(enp, rss_context, key, n)) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} +#endif /* EFSYS_OPT_RX_SCALE */ + +#if EFSYS_OPT_RX_SCALE + __checkReturn efx_rc_t +efx_rx_scale_tbl_set( + __in efx_nic_t *enp, + __in uint32_t rss_context, + __in_ecount(n) unsigned int *table, + __in size_t n) +{ + const efx_rx_ops_t *erxop = enp->en_erxop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); + + if ((rc = erxop->erxo_scale_tbl_set(enp, rss_context, table, n)) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} +#endif /* EFSYS_OPT_RX_SCALE */ + + void +efx_rx_qpost( + __in efx_rxq_t *erp, + __in_ecount(n) efsys_dma_addr_t *addrp, + __in size_t size, + __in unsigned int n, + __in unsigned int completed, + __in unsigned int added) +{ + efx_nic_t *enp = erp->er_enp; + const efx_rx_ops_t *erxop = enp->en_erxop; + + EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); + + erxop->erxo_qpost(erp, addrp, size, n, completed, added); +} + +#if EFSYS_OPT_RX_PACKED_STREAM + + void +efx_rx_qps_update_credits( + __in efx_rxq_t *erp) +{ + efx_nic_t *enp = erp->er_enp; + const efx_rx_ops_t *erxop = enp->en_erxop; + + EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); + + erxop->erxo_qps_update_credits(erp); +} + + __checkReturn uint8_t * +efx_rx_qps_packet_info( + __in efx_rxq_t *erp, + __in uint8_t *buffer, + __in uint32_t buffer_length, + __in uint32_t current_offset, + __out uint16_t *lengthp, + __out uint32_t *next_offsetp, + __out uint32_t *timestamp) +{ + efx_nic_t *enp = erp->er_enp; + const efx_rx_ops_t *erxop = enp->en_erxop; + + return (erxop->erxo_qps_packet_info(erp, buffer, + buffer_length, current_offset, lengthp, + next_offsetp, timestamp)); +} + +#endif /* EFSYS_OPT_RX_PACKED_STREAM */ + + void +efx_rx_qpush( + __in efx_rxq_t *erp, + __in unsigned int added, + __inout unsigned int *pushedp) +{ + efx_nic_t *enp = erp->er_enp; + const efx_rx_ops_t *erxop = enp->en_erxop; + + EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); + + erxop->erxo_qpush(erp, added, pushedp); +} + + __checkReturn efx_rc_t +efx_rx_qflush( + __in efx_rxq_t *erp) +{ + efx_nic_t *enp = erp->er_enp; + const efx_rx_ops_t *erxop = enp->en_erxop; + efx_rc_t rc; + + EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); + + if ((rc = erxop->erxo_qflush(erp)) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + void +efx_rx_qenable( + __in efx_rxq_t *erp) +{ + efx_nic_t *enp = erp->er_enp; + const efx_rx_ops_t *erxop = enp->en_erxop; + + EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); + + erxop->erxo_qenable(erp); +} + + __checkReturn efx_rc_t +efx_rx_qcreate( + __in efx_nic_t *enp, + __in unsigned int index, + __in unsigned int label, + __in efx_rxq_type_t type, + __in efsys_mem_t *esmp, + __in size_t n, + __in uint32_t id, + __in efx_evq_t *eep, + __deref_out efx_rxq_t **erpp) +{ + const efx_rx_ops_t *erxop = enp->en_erxop; + efx_rxq_t *erp; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); + + /* Allocate an RXQ object */ + EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_rxq_t), erp); + + if (erp == NULL) { + rc = ENOMEM; + goto fail1; + } + + erp->er_magic = EFX_RXQ_MAGIC; + erp->er_enp = enp; + erp->er_index = index; + erp->er_mask = n - 1; + erp->er_esmp = esmp; + + if ((rc = erxop->erxo_qcreate(enp, index, label, type, esmp, n, id, + eep, erp)) != 0) + goto fail2; + + enp->en_rx_qcount++; + *erpp = erp; + + return (0); + +fail2: + EFSYS_PROBE(fail2); + + EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + void +efx_rx_qdestroy( + __in efx_rxq_t *erp) +{ + efx_nic_t *enp = erp->er_enp; + const efx_rx_ops_t *erxop = enp->en_erxop; + + EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); + + erxop->erxo_qdestroy(erp); +} + + __checkReturn efx_rc_t +efx_pseudo_hdr_pkt_length_get( + __in efx_rxq_t *erp, + __in uint8_t *buffer, + __out uint16_t *lengthp) +{ + efx_nic_t *enp = erp->er_enp; + const efx_rx_ops_t *erxop = enp->en_erxop; + + EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); + + return (erxop->erxo_prefix_pktlen(enp, buffer, lengthp)); +} + +#if EFSYS_OPT_RX_SCALE + __checkReturn uint32_t +efx_pseudo_hdr_hash_get( + __in efx_rxq_t *erp, + __in efx_rx_hash_alg_t func, + __in uint8_t *buffer) +{ + efx_nic_t *enp = erp->er_enp; + const efx_rx_ops_t *erxop = enp->en_erxop; + + EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); + + EFSYS_ASSERT3U(enp->en_hash_support, ==, EFX_RX_HASH_AVAILABLE); + return (erxop->erxo_prefix_hash(enp, func, buffer)); +} +#endif /* EFSYS_OPT_RX_SCALE */ + +#if EFSYS_OPT_SIENA + +static __checkReturn efx_rc_t +siena_rx_init( + __in efx_nic_t *enp) +{ + efx_oword_t oword; + unsigned int index; + + EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword); + + EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_DESC_PUSH_EN, 0); + EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0); + EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0); + EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0); + EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, 0); + EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, 0x3000 / 32); + EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword); + + /* Zero the RSS table */ + for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; + index++) { + EFX_ZERO_OWORD(oword); + EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL, + index, &oword, B_TRUE); + } + +#if EFSYS_OPT_RX_SCALE + /* The RSS key and indirection table are writable. */ + enp->en_rss_context_type = EFX_RX_SCALE_EXCLUSIVE; + + /* Hardware can insert RX hash with/without RSS */ + enp->en_hash_support = EFX_RX_HASH_AVAILABLE; +#endif /* EFSYS_OPT_RX_SCALE */ + + return (0); +} + +#if EFSYS_OPT_RX_SCATTER +static __checkReturn efx_rc_t +siena_rx_scatter_enable( + __in efx_nic_t *enp, + __in unsigned int buf_size) +{ + unsigned int nbuf32; + efx_oword_t oword; + efx_rc_t rc; + + nbuf32 = buf_size / 32; + if ((nbuf32 == 0) || + (nbuf32 >= (1 << FRF_BZ_RX_USR_BUF_SIZE_WIDTH)) || + ((buf_size % 32) != 0)) { + rc = EINVAL; + goto fail1; + } + + if (enp->en_rx_qcount > 0) { + rc = EBUSY; + goto fail2; + } + + /* Set scatter buffer size */ + EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword); + EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, nbuf32); + EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword); + + /* Enable scatter for packets not matching a filter */ + EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword); + EFX_SET_OWORD_FIELD(oword, FRF_BZ_SCATTER_ENBL_NO_MATCH_Q, 1); + EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword); + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} +#endif /* EFSYS_OPT_RX_SCATTER */ + + +#define EFX_RX_LFSR_HASH(_enp, _insert) \ + do { \ + efx_oword_t oword; \ + \ + EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword); \ + EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0); \ + EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0); \ + EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0); \ + EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, \ + (_insert) ? 1 : 0); \ + EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword); \ + \ + if ((_enp)->en_family == EFX_FAMILY_SIENA) { \ + EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, \ + &oword); \ + EFX_SET_OWORD_FIELD(oword, \ + FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 0); \ + EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, \ + &oword); \ + } \ + \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#define EFX_RX_TOEPLITZ_IPV4_HASH(_enp, _insert, _ip, _tcp) \ + do { \ + efx_oword_t oword; \ + \ + EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword); \ + EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 1); \ + EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, \ + (_ip) ? 1 : 0); \ + EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, \ + (_tcp) ? 0 : 1); \ + EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, \ + (_insert) ? 1 : 0); \ + EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword); \ + \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#define EFX_RX_TOEPLITZ_IPV6_HASH(_enp, _ip, _tcp, _rc) \ + do { \ + efx_oword_t oword; \ + \ + EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \ + EFX_SET_OWORD_FIELD(oword, \ + FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 1); \ + EFX_SET_OWORD_FIELD(oword, \ + FRF_CZ_RX_RSS_IPV6_IP_THASH_ENABLE, (_ip) ? 1 : 0); \ + EFX_SET_OWORD_FIELD(oword, \ + FRF_CZ_RX_RSS_IPV6_TCP_SUPPRESS, (_tcp) ? 0 : 1); \ + EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \ + \ + (_rc) = 0; \ + \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + + +#if EFSYS_OPT_RX_SCALE + +static __checkReturn efx_rc_t +siena_rx_scale_mode_set( + __in efx_nic_t *enp, + __in uint32_t rss_context, + __in efx_rx_hash_alg_t alg, + __in efx_rx_hash_type_t type, + __in boolean_t insert) +{ + efx_rc_t rc; + + if (rss_context != EFX_RSS_CONTEXT_DEFAULT) { + rc = EINVAL; + goto fail1; + } + + switch (alg) { + case EFX_RX_HASHALG_LFSR: + EFX_RX_LFSR_HASH(enp, insert); + break; + + case EFX_RX_HASHALG_TOEPLITZ: + EFX_RX_TOEPLITZ_IPV4_HASH(enp, insert, + type & EFX_RX_HASH_IPV4, + type & EFX_RX_HASH_TCPIPV4); + + EFX_RX_TOEPLITZ_IPV6_HASH(enp, + type & EFX_RX_HASH_IPV6, + type & EFX_RX_HASH_TCPIPV6, + rc); + if (rc != 0) + goto fail2; + + break; + + default: + rc = EINVAL; + goto fail3; + } + + return (0); + +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + EFX_RX_LFSR_HASH(enp, B_FALSE); + + return (rc); +} +#endif + +#if EFSYS_OPT_RX_SCALE +static __checkReturn efx_rc_t +siena_rx_scale_key_set( + __in efx_nic_t *enp, + __in uint32_t rss_context, + __in_ecount(n) uint8_t *key, + __in size_t n) +{ + efx_oword_t oword; + unsigned int byte; + unsigned int offset; + efx_rc_t rc; + + if (rss_context != EFX_RSS_CONTEXT_DEFAULT) { + rc = EINVAL; + goto fail1; + } + + byte = 0; + + /* Write Toeplitz IPv4 hash key */ + EFX_ZERO_OWORD(oword); + for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8; + offset > 0 && byte < n; + --offset) + oword.eo_u8[offset - 1] = key[byte++]; + + EFX_BAR_WRITEO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword); + + byte = 0; + + /* Verify Toeplitz IPv4 hash key */ + EFX_BAR_READO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword); + for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8; + offset > 0 && byte < n; + --offset) { + if (oword.eo_u8[offset - 1] != key[byte++]) { + rc = EFAULT; + goto fail2; + } + } + + if ((enp->en_features & EFX_FEATURE_IPV6) == 0) + goto done; + + byte = 0; + + /* Write Toeplitz IPv6 hash key 3 */ + EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword); + for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN + + FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8; + offset > 0 && byte < n; + --offset) + oword.eo_u8[offset - 1] = key[byte++]; + + EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword); + + /* Write Toeplitz IPv6 hash key 2 */ + EFX_ZERO_OWORD(oword); + for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN + + FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8; + offset > 0 && byte < n; + --offset) + oword.eo_u8[offset - 1] = key[byte++]; + + EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword); + + /* Write Toeplitz IPv6 hash key 1 */ + EFX_ZERO_OWORD(oword); + for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN + + FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8; + offset > 0 && byte < n; + --offset) + oword.eo_u8[offset - 1] = key[byte++]; + + EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword); + + byte = 0; + + /* Verify Toeplitz IPv6 hash key 3 */ + EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword); + for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN + + FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8; + offset > 0 && byte < n; + --offset) { + if (oword.eo_u8[offset - 1] != key[byte++]) { + rc = EFAULT; + goto fail3; + } + } + + /* Verify Toeplitz IPv6 hash key 2 */ + EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword); + for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN + + FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8; + offset > 0 && byte < n; + --offset) { + if (oword.eo_u8[offset - 1] != key[byte++]) { + rc = EFAULT; + goto fail4; + } + } + + /* Verify Toeplitz IPv6 hash key 1 */ + EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword); + for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN + + FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8; + offset > 0 && byte < n; + --offset) { + if (oword.eo_u8[offset - 1] != key[byte++]) { + rc = EFAULT; + goto fail5; + } + } + +done: + return (0); + +fail5: + EFSYS_PROBE(fail5); +fail4: + EFSYS_PROBE(fail4); +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} +#endif + +#if EFSYS_OPT_RX_SCALE +static __checkReturn efx_rc_t +siena_rx_scale_tbl_set( + __in efx_nic_t *enp, + __in uint32_t rss_context, + __in_ecount(n) unsigned int *table, + __in size_t n) +{ + efx_oword_t oword; + int index; + efx_rc_t rc; + + EFX_STATIC_ASSERT(EFX_RSS_TBL_SIZE == FR_BZ_RX_INDIRECTION_TBL_ROWS); + EFX_STATIC_ASSERT(EFX_MAXRSS == (1 << FRF_BZ_IT_QUEUE_WIDTH)); + + if (rss_context != EFX_RSS_CONTEXT_DEFAULT) { + rc = EINVAL; + goto fail1; + } + + if (n > FR_BZ_RX_INDIRECTION_TBL_ROWS) { + rc = EINVAL; + goto fail2; + } + + for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; index++) { + uint32_t byte; + + /* Calculate the entry to place in the table */ + byte = (n > 0) ? (uint32_t)table[index % n] : 0; + + EFSYS_PROBE2(table, int, index, uint32_t, byte); + + EFX_POPULATE_OWORD_1(oword, FRF_BZ_IT_QUEUE, byte); + + /* Write the table */ + EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL, + index, &oword, B_TRUE); + } + + for (index = FR_BZ_RX_INDIRECTION_TBL_ROWS - 1; index >= 0; --index) { + uint32_t byte; + + /* Determine if we're starting a new batch */ + byte = (n > 0) ? (uint32_t)table[index % n] : 0; + + /* Read the table */ + EFX_BAR_TBL_READO(enp, FR_BZ_RX_INDIRECTION_TBL, + index, &oword, B_TRUE); + + /* Verify the entry */ + if (EFX_OWORD_FIELD(oword, FRF_BZ_IT_QUEUE) != byte) { + rc = EFAULT; + goto fail3; + } + } + + return (0); + +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} +#endif + +/* + * Falcon/Siena pseudo-header + * -------------------------- + * + * Receive packets are prefixed by an optional 16 byte pseudo-header. + * The pseudo-header is a byte array of one of the forms: + * + * 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 + * xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.TT.TT.TT.TT + * xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.LL.LL + * + * where: + * TT.TT.TT.TT Toeplitz hash (32-bit big-endian) + * LL.LL LFSR hash (16-bit big-endian) + */ + +#if EFSYS_OPT_RX_SCALE +static __checkReturn uint32_t +siena_rx_prefix_hash( + __in efx_nic_t *enp, + __in efx_rx_hash_alg_t func, + __in uint8_t *buffer) +{ + _NOTE(ARGUNUSED(enp)) + + switch (func) { + case EFX_RX_HASHALG_TOEPLITZ: + return ((buffer[12] << 24) | + (buffer[13] << 16) | + (buffer[14] << 8) | + buffer[15]); + + case EFX_RX_HASHALG_LFSR: + return ((buffer[14] << 8) | buffer[15]); + + default: + EFSYS_ASSERT(0); + return (0); + } +} +#endif /* EFSYS_OPT_RX_SCALE */ + +static __checkReturn efx_rc_t +siena_rx_prefix_pktlen( + __in efx_nic_t *enp, + __in uint8_t *buffer, + __out uint16_t *lengthp) +{ + _NOTE(ARGUNUSED(enp, buffer, lengthp)) + + /* Not supported by Falcon/Siena hardware */ + EFSYS_ASSERT(0); + return (ENOTSUP); +} + + +static void +siena_rx_qpost( + __in efx_rxq_t *erp, + __in_ecount(n) efsys_dma_addr_t *addrp, + __in size_t size, + __in unsigned int n, + __in unsigned int completed, + __in unsigned int added) +{ + efx_qword_t qword; + unsigned int i; + unsigned int offset; + unsigned int id; + + /* The client driver must not overfill the queue */ + EFSYS_ASSERT3U(added - completed + n, <=, + EFX_RXQ_LIMIT(erp->er_mask + 1)); + + id = added & (erp->er_mask); + for (i = 0; i < n; i++) { + EFSYS_PROBE4(rx_post, unsigned int, erp->er_index, + unsigned int, id, efsys_dma_addr_t, addrp[i], + size_t, size); + + EFX_POPULATE_QWORD_3(qword, + FSF_AZ_RX_KER_BUF_SIZE, (uint32_t)(size), + FSF_AZ_RX_KER_BUF_ADDR_DW0, + (uint32_t)(addrp[i] & 0xffffffff), + FSF_AZ_RX_KER_BUF_ADDR_DW1, + (uint32_t)(addrp[i] >> 32)); + + offset = id * sizeof (efx_qword_t); + EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword); + + id = (id + 1) & (erp->er_mask); + } +} + +static void +siena_rx_qpush( + __in efx_rxq_t *erp, + __in unsigned int added, + __inout unsigned int *pushedp) +{ + efx_nic_t *enp = erp->er_enp; + unsigned int pushed = *pushedp; + uint32_t wptr; + efx_oword_t oword; + efx_dword_t dword; + + /* All descriptors are pushed */ + *pushedp = added; + + /* Push the populated descriptors out */ + wptr = added & erp->er_mask; + + EFX_POPULATE_OWORD_1(oword, FRF_AZ_RX_DESC_WPTR, wptr); + + /* Only write the third DWORD */ + EFX_POPULATE_DWORD_1(dword, + EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3)); + + /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */ + EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1, + wptr, pushed & erp->er_mask); + EFSYS_PIO_WRITE_BARRIER(); + EFX_BAR_TBL_WRITED3(enp, FR_BZ_RX_DESC_UPD_REGP0, + erp->er_index, &dword, B_FALSE); +} + +#if EFSYS_OPT_RX_PACKED_STREAM +static void +siena_rx_qps_update_credits( + __in efx_rxq_t *erp) +{ + /* Not supported by Siena hardware */ + EFSYS_ASSERT(0); +} + +static uint8_t * +siena_rx_qps_packet_info( + __in efx_rxq_t *erp, + __in uint8_t *buffer, + __in uint32_t buffer_length, + __in uint32_t current_offset, + __out uint16_t *lengthp, + __out uint32_t *next_offsetp, + __out uint32_t *timestamp) +{ + /* Not supported by Siena hardware */ + EFSYS_ASSERT(0); + + return (NULL); +} +#endif /* EFSYS_OPT_RX_PACKED_STREAM */ + +static __checkReturn efx_rc_t +siena_rx_qflush( + __in efx_rxq_t *erp) +{ + efx_nic_t *enp = erp->er_enp; + efx_oword_t oword; + uint32_t label; + + label = erp->er_index; + + /* Flush the queue */ + EFX_POPULATE_OWORD_2(oword, FRF_AZ_RX_FLUSH_DESCQ_CMD, 1, + FRF_AZ_RX_FLUSH_DESCQ, label); + EFX_BAR_WRITEO(enp, FR_AZ_RX_FLUSH_DESCQ_REG, &oword); + + return (0); +} + +static void +siena_rx_qenable( + __in efx_rxq_t *erp) +{ + efx_nic_t *enp = erp->er_enp; + efx_oword_t oword; + + EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); + + EFX_BAR_TBL_READO(enp, FR_AZ_RX_DESC_PTR_TBL, + erp->er_index, &oword, B_TRUE); + + EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DC_HW_RPTR, 0); + EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_HW_RPTR, 0); + EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_EN, 1); + + EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL, + erp->er_index, &oword, B_TRUE); +} + +static __checkReturn efx_rc_t +siena_rx_qcreate( + __in efx_nic_t *enp, + __in unsigned int index, + __in unsigned int label, + __in efx_rxq_type_t type, + __in efsys_mem_t *esmp, + __in size_t n, + __in uint32_t id, + __in efx_evq_t *eep, + __in efx_rxq_t *erp) +{ + efx_nic_cfg_t *encp = &(enp->en_nic_cfg); + efx_oword_t oword; + uint32_t size; + boolean_t jumbo; + efx_rc_t rc; + + _NOTE(ARGUNUSED(esmp)) + + EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS == + (1 << FRF_AZ_RX_DESCQ_LABEL_WIDTH)); + EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS); + EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit); + + EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MAXNDESCS)); + EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MINNDESCS)); + + if (!ISP2(n) || (n < EFX_RXQ_MINNDESCS) || (n > EFX_RXQ_MAXNDESCS)) { + rc = EINVAL; + goto fail1; + } + if (index >= encp->enc_rxq_limit) { + rc = EINVAL; + goto fail2; + } + for (size = 0; (1 << size) <= (EFX_RXQ_MAXNDESCS / EFX_RXQ_MINNDESCS); + size++) + if ((1 << size) == (int)(n / EFX_RXQ_MINNDESCS)) + break; + if (id + (1 << size) >= encp->enc_buftbl_limit) { + rc = EINVAL; + goto fail3; + } + + switch (type) { + case EFX_RXQ_TYPE_DEFAULT: + jumbo = B_FALSE; + break; + +#if EFSYS_OPT_RX_SCATTER + case EFX_RXQ_TYPE_SCATTER: + if (enp->en_family < EFX_FAMILY_SIENA) { + rc = EINVAL; + goto fail4; + } + jumbo = B_TRUE; + break; +#endif /* EFSYS_OPT_RX_SCATTER */ + + default: + rc = EINVAL; + goto fail4; + } + + /* Set up the new descriptor queue */ + EFX_POPULATE_OWORD_7(oword, + FRF_AZ_RX_DESCQ_BUF_BASE_ID, id, + FRF_AZ_RX_DESCQ_EVQ_ID, eep->ee_index, + FRF_AZ_RX_DESCQ_OWNER_ID, 0, + FRF_AZ_RX_DESCQ_LABEL, label, + FRF_AZ_RX_DESCQ_SIZE, size, + FRF_AZ_RX_DESCQ_TYPE, 0, + FRF_AZ_RX_DESCQ_JUMBO, jumbo); + + EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL, + erp->er_index, &oword, B_TRUE); + + return (0); + +fail4: + EFSYS_PROBE(fail4); +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +static void +siena_rx_qdestroy( + __in efx_rxq_t *erp) +{ + efx_nic_t *enp = erp->er_enp; + efx_oword_t oword; + + EFSYS_ASSERT(enp->en_rx_qcount != 0); + --enp->en_rx_qcount; + + /* Purge descriptor queue */ + EFX_ZERO_OWORD(oword); + + EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL, + erp->er_index, &oword, B_TRUE); + + /* Free the RXQ object */ + EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp); +} + +static void +siena_rx_fini( + __in efx_nic_t *enp) +{ + _NOTE(ARGUNUSED(enp)) +} + +#endif /* EFSYS_OPT_SIENA */ diff --git a/dpdk/drivers/net/sfc/base/efx_sram.c b/dpdk/drivers/net/sfc/base/efx_sram.c new file mode 100644 index 00000000..5f4edea7 --- /dev/null +++ b/dpdk/drivers/net/sfc/base/efx_sram.c @@ -0,0 +1,331 @@ +/* + * Copyright (c) 2007-2016 Solarflare Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are + * those of the authors and should not be interpreted as representing official + * policies, either expressed or implied, of the FreeBSD Project. + */ + +#include "efx.h" +#include "efx_impl.h" + + __checkReturn efx_rc_t +efx_sram_buf_tbl_set( + __in efx_nic_t *enp, + __in uint32_t id, + __in efsys_mem_t *esmp, + __in size_t n) +{ + efx_qword_t qword; + uint32_t start = id; + uint32_t stop = start + n; + efsys_dma_addr_t addr; + efx_oword_t oword; + unsigned int count; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); + +#if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD + if (enp->en_family == EFX_FAMILY_HUNTINGTON || + enp->en_family == EFX_FAMILY_MEDFORD) { + /* + * FIXME: the efx_sram_buf_tbl_*() functionality needs to be + * pulled inside the Falcon/Siena queue create/destroy code, + * and then the original functions can be removed (see bug30834 + * comment #1). But, for now, we just ensure that they are + * no-ops for EF10, to allow bringing up existing drivers + * without modification. + */ + + return (0); + } +#endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */ + + if (stop >= EFX_BUF_TBL_SIZE) { + rc = EFBIG; + goto fail1; + } + + /* Add the entries into the buffer table */ + addr = EFSYS_MEM_ADDR(esmp); + for (id = start; id != stop; id++) { + EFX_POPULATE_QWORD_5(qword, + FRF_AZ_IP_DAT_BUF_SIZE, 0, FRF_AZ_BUF_ADR_REGION, 0, + FRF_AZ_BUF_ADR_FBUF_DW0, + (uint32_t)((addr >> 12) & 0xffffffff), + FRF_AZ_BUF_ADR_FBUF_DW1, + (uint32_t)((addr >> 12) >> 32), + FRF_AZ_BUF_OWNER_ID_FBUF, 0); + + EFX_BAR_TBL_WRITEQ(enp, FR_AZ_BUF_FULL_TBL, + id, &qword); + + addr += EFX_BUF_SIZE; + } + + EFSYS_PROBE2(buf, uint32_t, start, uint32_t, stop - 1); + + /* Flush the write buffer */ + EFX_POPULATE_OWORD_2(oword, FRF_AZ_BUF_UPD_CMD, 1, + FRF_AZ_BUF_CLR_CMD, 0); + EFX_BAR_WRITEO(enp, FR_AZ_BUF_TBL_UPD_REG, &oword); + + /* Poll for the last entry being written to the buffer table */ + EFSYS_ASSERT3U(id, ==, stop); + addr -= EFX_BUF_SIZE; + + count = 0; + do { + EFSYS_PROBE1(wait, unsigned int, count); + + /* Spin for 1 ms */ + EFSYS_SPIN(1000); + + EFX_BAR_TBL_READQ(enp, FR_AZ_BUF_FULL_TBL, + id - 1, &qword); + + if (EFX_QWORD_FIELD(qword, FRF_AZ_BUF_ADR_FBUF_DW0) == + (uint32_t)((addr >> 12) & 0xffffffff) && + EFX_QWORD_FIELD(qword, FRF_AZ_BUF_ADR_FBUF_DW1) == + (uint32_t)((addr >> 12) >> 32)) + goto verify; + + } while (++count < 100); + + rc = ETIMEDOUT; + goto fail2; + +verify: + /* Verify the rest of the entries in the buffer table */ + while (--id != start) { + addr -= EFX_BUF_SIZE; + + /* Read the buffer table entry */ + EFX_BAR_TBL_READQ(enp, FR_AZ_BUF_FULL_TBL, + id - 1, &qword); + + if (EFX_QWORD_FIELD(qword, FRF_AZ_BUF_ADR_FBUF_DW0) != + (uint32_t)((addr >> 12) & 0xffffffff) || + EFX_QWORD_FIELD(qword, FRF_AZ_BUF_ADR_FBUF_DW1) != + (uint32_t)((addr >> 12) >> 32)) { + rc = EFAULT; + goto fail3; + } + } + + return (0); + +fail3: + EFSYS_PROBE(fail3); + + id = stop; + +fail2: + EFSYS_PROBE(fail2); + + EFX_POPULATE_OWORD_4(oword, FRF_AZ_BUF_UPD_CMD, 0, + FRF_AZ_BUF_CLR_CMD, 1, FRF_AZ_BUF_CLR_END_ID, id - 1, + FRF_AZ_BUF_CLR_START_ID, start); + EFX_BAR_WRITEO(enp, FR_AZ_BUF_TBL_UPD_REG, &oword); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + void +efx_sram_buf_tbl_clear( + __in efx_nic_t *enp, + __in uint32_t id, + __in size_t n) +{ + efx_oword_t oword; + uint32_t start = id; + uint32_t stop = start + n; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); + +#if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD + if (enp->en_family == EFX_FAMILY_HUNTINGTON || + enp->en_family == EFX_FAMILY_MEDFORD) { + /* + * FIXME: the efx_sram_buf_tbl_*() functionality needs to be + * pulled inside the Falcon/Siena queue create/destroy code, + * and then the original functions can be removed (see bug30834 + * comment #1). But, for now, we just ensure that they are + * no-ops for EF10, to allow bringing up existing drivers + * without modification. + */ + + return; + } +#endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */ + + EFSYS_ASSERT3U(stop, <, EFX_BUF_TBL_SIZE); + + EFSYS_PROBE2(buf, uint32_t, start, uint32_t, stop - 1); + + EFX_POPULATE_OWORD_4(oword, FRF_AZ_BUF_UPD_CMD, 0, + FRF_AZ_BUF_CLR_CMD, 1, FRF_AZ_BUF_CLR_END_ID, stop - 1, + FRF_AZ_BUF_CLR_START_ID, start); + EFX_BAR_WRITEO(enp, FR_AZ_BUF_TBL_UPD_REG, &oword); +} + + +#if EFSYS_OPT_DIAG + +static void +efx_sram_byte_increment_set( + __in size_t row, + __in boolean_t negate, + __out efx_qword_t *eqp) +{ + size_t offset = row * FR_AZ_SRM_DBG_REG_STEP; + unsigned int index; + + _NOTE(ARGUNUSED(negate)) + + for (index = 0; index < sizeof (efx_qword_t); index++) + eqp->eq_u8[index] = offset + index; +} + +static void +efx_sram_all_the_same_set( + __in size_t row, + __in boolean_t negate, + __out efx_qword_t *eqp) +{ + _NOTE(ARGUNUSED(row)) + + if (negate) + EFX_SET_QWORD(*eqp); + else + EFX_ZERO_QWORD(*eqp); +} + +static void +efx_sram_bit_alternate_set( + __in size_t row, + __in boolean_t negate, + __out efx_qword_t *eqp) +{ + _NOTE(ARGUNUSED(row)) + + EFX_POPULATE_QWORD_2(*eqp, + EFX_DWORD_0, (negate) ? 0x55555555 : 0xaaaaaaaa, + EFX_DWORD_1, (negate) ? 0x55555555 : 0xaaaaaaaa); +} + +static void +efx_sram_byte_alternate_set( + __in size_t row, + __in boolean_t negate, + __out efx_qword_t *eqp) +{ + _NOTE(ARGUNUSED(row)) + + EFX_POPULATE_QWORD_2(*eqp, + EFX_DWORD_0, (negate) ? 0x00ff00ff : 0xff00ff00, + EFX_DWORD_1, (negate) ? 0x00ff00ff : 0xff00ff00); +} + +static void +efx_sram_byte_changing_set( + __in size_t row, + __in boolean_t negate, + __out efx_qword_t *eqp) +{ + size_t offset = row * FR_AZ_SRM_DBG_REG_STEP; + unsigned int index; + + for (index = 0; index < sizeof (efx_qword_t); index++) { + uint8_t byte; + + if (offset / 256 == 0) + byte = (uint8_t)((offset % 257) % 256); + else + byte = (uint8_t)(~((offset - 8) % 257) % 256); + + eqp->eq_u8[index] = (negate) ? ~byte : byte; + } +} + +static void +efx_sram_bit_sweep_set( + __in size_t row, + __in boolean_t negate, + __out efx_qword_t *eqp) +{ + size_t offset = row * FR_AZ_SRM_DBG_REG_STEP; + + if (negate) { + EFX_SET_QWORD(*eqp); + EFX_CLEAR_QWORD_BIT(*eqp, (offset / sizeof (efx_qword_t)) % 64); + } else { + EFX_ZERO_QWORD(*eqp); + EFX_SET_QWORD_BIT(*eqp, (offset / sizeof (efx_qword_t)) % 64); + } +} + +efx_sram_pattern_fn_t __efx_sram_pattern_fns[] = { + efx_sram_byte_increment_set, + efx_sram_all_the_same_set, + efx_sram_bit_alternate_set, + efx_sram_byte_alternate_set, + efx_sram_byte_changing_set, + efx_sram_bit_sweep_set +}; + + __checkReturn efx_rc_t +efx_sram_test( + __in efx_nic_t *enp, + __in efx_pattern_type_t type) +{ + efx_sram_pattern_fn_t func; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); + + EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_RX)); + EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_TX)); + EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_EV)); + + /* SRAM testing is only available on Siena. */ + if (enp->en_family != EFX_FAMILY_SIENA) + return (0); + + /* Select pattern generator */ + EFSYS_ASSERT3U(type, <, EFX_PATTERN_NTYPES); + func = __efx_sram_pattern_fns[type]; + + return (siena_sram_test(enp, func)); +} + +#endif /* EFSYS_OPT_DIAG */ diff --git a/dpdk/drivers/net/sfc/base/efx_tx.c b/dpdk/drivers/net/sfc/base/efx_tx.c new file mode 100644 index 00000000..ceb29206 --- /dev/null +++ b/dpdk/drivers/net/sfc/base/efx_tx.c @@ -0,0 +1,1097 @@ +/* + * Copyright (c) 2007-2016 Solarflare Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are + * those of the authors and should not be interpreted as representing official + * policies, either expressed or implied, of the FreeBSD Project. + */ + +#include "efx.h" +#include "efx_impl.h" + +#if EFSYS_OPT_QSTATS +#define EFX_TX_QSTAT_INCR(_etp, _stat) \ + do { \ + (_etp)->et_stat[_stat]++; \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) +#else +#define EFX_TX_QSTAT_INCR(_etp, _stat) +#endif + +#if EFSYS_OPT_SIENA + +static __checkReturn efx_rc_t +siena_tx_init( + __in efx_nic_t *enp); + +static void +siena_tx_fini( + __in efx_nic_t *enp); + +static __checkReturn efx_rc_t +siena_tx_qcreate( + __in efx_nic_t *enp, + __in unsigned int index, + __in unsigned int label, + __in efsys_mem_t *esmp, + __in size_t n, + __in uint32_t id, + __in uint16_t flags, + __in efx_evq_t *eep, + __in efx_txq_t *etp, + __out unsigned int *addedp); + +static void +siena_tx_qdestroy( + __in efx_txq_t *etp); + +static __checkReturn efx_rc_t +siena_tx_qpost( + __in efx_txq_t *etp, + __in_ecount(n) efx_buffer_t *eb, + __in unsigned int n, + __in unsigned int completed, + __inout unsigned int *addedp); + +static void +siena_tx_qpush( + __in efx_txq_t *etp, + __in unsigned int added, + __in unsigned int pushed); + +static __checkReturn efx_rc_t +siena_tx_qpace( + __in efx_txq_t *etp, + __in unsigned int ns); + +static __checkReturn efx_rc_t +siena_tx_qflush( + __in efx_txq_t *etp); + +static void +siena_tx_qenable( + __in efx_txq_t *etp); + + __checkReturn efx_rc_t +siena_tx_qdesc_post( + __in efx_txq_t *etp, + __in_ecount(n) efx_desc_t *ed, + __in unsigned int n, + __in unsigned int completed, + __inout unsigned int *addedp); + + void +siena_tx_qdesc_dma_create( + __in efx_txq_t *etp, + __in efsys_dma_addr_t addr, + __in size_t size, + __in boolean_t eop, + __out efx_desc_t *edp); + +#if EFSYS_OPT_QSTATS +static void +siena_tx_qstats_update( + __in efx_txq_t *etp, + __inout_ecount(TX_NQSTATS) efsys_stat_t *stat); +#endif + +#endif /* EFSYS_OPT_SIENA */ + + +#if EFSYS_OPT_SIENA +static const efx_tx_ops_t __efx_tx_siena_ops = { + siena_tx_init, /* etxo_init */ + siena_tx_fini, /* etxo_fini */ + siena_tx_qcreate, /* etxo_qcreate */ + siena_tx_qdestroy, /* etxo_qdestroy */ + siena_tx_qpost, /* etxo_qpost */ + siena_tx_qpush, /* etxo_qpush */ + siena_tx_qpace, /* etxo_qpace */ + siena_tx_qflush, /* etxo_qflush */ + siena_tx_qenable, /* etxo_qenable */ + NULL, /* etxo_qpio_enable */ + NULL, /* etxo_qpio_disable */ + NULL, /* etxo_qpio_write */ + NULL, /* etxo_qpio_post */ + siena_tx_qdesc_post, /* etxo_qdesc_post */ + siena_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ + NULL, /* etxo_qdesc_tso_create */ + NULL, /* etxo_qdesc_tso2_create */ + NULL, /* etxo_qdesc_vlantci_create */ +#if EFSYS_OPT_QSTATS + siena_tx_qstats_update, /* etxo_qstats_update */ +#endif +}; +#endif /* EFSYS_OPT_SIENA */ + +#if EFSYS_OPT_HUNTINGTON +static const efx_tx_ops_t __efx_tx_hunt_ops = { + ef10_tx_init, /* etxo_init */ + ef10_tx_fini, /* etxo_fini */ + ef10_tx_qcreate, /* etxo_qcreate */ + ef10_tx_qdestroy, /* etxo_qdestroy */ + ef10_tx_qpost, /* etxo_qpost */ + ef10_tx_qpush, /* etxo_qpush */ + ef10_tx_qpace, /* etxo_qpace */ + ef10_tx_qflush, /* etxo_qflush */ + ef10_tx_qenable, /* etxo_qenable */ + ef10_tx_qpio_enable, /* etxo_qpio_enable */ + ef10_tx_qpio_disable, /* etxo_qpio_disable */ + ef10_tx_qpio_write, /* etxo_qpio_write */ + ef10_tx_qpio_post, /* etxo_qpio_post */ + ef10_tx_qdesc_post, /* etxo_qdesc_post */ + ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ + ef10_tx_qdesc_tso_create, /* etxo_qdesc_tso_create */ + ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */ + ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */ +#if EFSYS_OPT_QSTATS + ef10_tx_qstats_update, /* etxo_qstats_update */ +#endif +}; +#endif /* EFSYS_OPT_HUNTINGTON */ + +#if EFSYS_OPT_MEDFORD +static const efx_tx_ops_t __efx_tx_medford_ops = { + ef10_tx_init, /* etxo_init */ + ef10_tx_fini, /* etxo_fini */ + ef10_tx_qcreate, /* etxo_qcreate */ + ef10_tx_qdestroy, /* etxo_qdestroy */ + ef10_tx_qpost, /* etxo_qpost */ + ef10_tx_qpush, /* etxo_qpush */ + ef10_tx_qpace, /* etxo_qpace */ + ef10_tx_qflush, /* etxo_qflush */ + ef10_tx_qenable, /* etxo_qenable */ + ef10_tx_qpio_enable, /* etxo_qpio_enable */ + ef10_tx_qpio_disable, /* etxo_qpio_disable */ + ef10_tx_qpio_write, /* etxo_qpio_write */ + ef10_tx_qpio_post, /* etxo_qpio_post */ + ef10_tx_qdesc_post, /* etxo_qdesc_post */ + ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ + NULL, /* etxo_qdesc_tso_create */ + ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */ + ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */ +#if EFSYS_OPT_QSTATS + ef10_tx_qstats_update, /* etxo_qstats_update */ +#endif +}; +#endif /* EFSYS_OPT_MEDFORD */ + + __checkReturn efx_rc_t +efx_tx_init( + __in efx_nic_t *enp) +{ + const efx_tx_ops_t *etxop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); + + if (!(enp->en_mod_flags & EFX_MOD_EV)) { + rc = EINVAL; + goto fail1; + } + + if (enp->en_mod_flags & EFX_MOD_TX) { + rc = EINVAL; + goto fail2; + } + + switch (enp->en_family) { +#if EFSYS_OPT_SIENA + case EFX_FAMILY_SIENA: + etxop = &__efx_tx_siena_ops; + break; +#endif /* EFSYS_OPT_SIENA */ + +#if EFSYS_OPT_HUNTINGTON + case EFX_FAMILY_HUNTINGTON: + etxop = &__efx_tx_hunt_ops; + break; +#endif /* EFSYS_OPT_HUNTINGTON */ + +#if EFSYS_OPT_MEDFORD + case EFX_FAMILY_MEDFORD: + etxop = &__efx_tx_medford_ops; + break; +#endif /* EFSYS_OPT_MEDFORD */ + + default: + EFSYS_ASSERT(0); + rc = ENOTSUP; + goto fail3; + } + + EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0); + + if ((rc = etxop->etxo_init(enp)) != 0) + goto fail4; + + enp->en_etxop = etxop; + enp->en_mod_flags |= EFX_MOD_TX; + return (0); + +fail4: + EFSYS_PROBE(fail4); +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + enp->en_etxop = NULL; + enp->en_mod_flags &= ~EFX_MOD_TX; + return (rc); +} + + void +efx_tx_fini( + __in efx_nic_t *enp) +{ + const efx_tx_ops_t *etxop = enp->en_etxop; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX); + EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0); + + etxop->etxo_fini(enp); + + enp->en_etxop = NULL; + enp->en_mod_flags &= ~EFX_MOD_TX; +} + + __checkReturn efx_rc_t +efx_tx_qcreate( + __in efx_nic_t *enp, + __in unsigned int index, + __in unsigned int label, + __in efsys_mem_t *esmp, + __in size_t n, + __in uint32_t id, + __in uint16_t flags, + __in efx_evq_t *eep, + __deref_out efx_txq_t **etpp, + __out unsigned int *addedp) +{ + const efx_tx_ops_t *etxop = enp->en_etxop; + efx_nic_cfg_t *encp = &(enp->en_nic_cfg); + efx_txq_t *etp; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX); + + EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <, encp->enc_txq_limit); + + /* Allocate an TXQ object */ + EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp); + + if (etp == NULL) { + rc = ENOMEM; + goto fail1; + } + + etp->et_magic = EFX_TXQ_MAGIC; + etp->et_enp = enp; + etp->et_index = index; + etp->et_mask = n - 1; + etp->et_esmp = esmp; + + /* Initial descriptor index may be modified by etxo_qcreate */ + *addedp = 0; + + if ((rc = etxop->etxo_qcreate(enp, index, label, esmp, + n, id, flags, eep, etp, addedp)) != 0) + goto fail2; + + enp->en_tx_qcount++; + *etpp = etp; + + return (0); + +fail2: + EFSYS_PROBE(fail2); + EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + return (rc); +} + + void +efx_tx_qdestroy( + __in efx_txq_t *etp) +{ + efx_nic_t *enp = etp->et_enp; + const efx_tx_ops_t *etxop = enp->en_etxop; + + EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); + + EFSYS_ASSERT(enp->en_tx_qcount != 0); + --enp->en_tx_qcount; + + etxop->etxo_qdestroy(etp); + + /* Free the TXQ object */ + EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp); +} + + __checkReturn efx_rc_t +efx_tx_qpost( + __in efx_txq_t *etp, + __in_ecount(n) efx_buffer_t *eb, + __in unsigned int n, + __in unsigned int completed, + __inout unsigned int *addedp) +{ + efx_nic_t *enp = etp->et_enp; + const efx_tx_ops_t *etxop = enp->en_etxop; + efx_rc_t rc; + + EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); + + if ((rc = etxop->etxo_qpost(etp, eb, + n, completed, addedp)) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + return (rc); +} + + void +efx_tx_qpush( + __in efx_txq_t *etp, + __in unsigned int added, + __in unsigned int pushed) +{ + efx_nic_t *enp = etp->et_enp; + const efx_tx_ops_t *etxop = enp->en_etxop; + + EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); + + etxop->etxo_qpush(etp, added, pushed); +} + + __checkReturn efx_rc_t +efx_tx_qpace( + __in efx_txq_t *etp, + __in unsigned int ns) +{ + efx_nic_t *enp = etp->et_enp; + const efx_tx_ops_t *etxop = enp->en_etxop; + efx_rc_t rc; + + EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); + + if ((rc = etxop->etxo_qpace(etp, ns)) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + return (rc); +} + + __checkReturn efx_rc_t +efx_tx_qflush( + __in efx_txq_t *etp) +{ + efx_nic_t *enp = etp->et_enp; + const efx_tx_ops_t *etxop = enp->en_etxop; + efx_rc_t rc; + + EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); + + if ((rc = etxop->etxo_qflush(etp)) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + return (rc); +} + + void +efx_tx_qenable( + __in efx_txq_t *etp) +{ + efx_nic_t *enp = etp->et_enp; + const efx_tx_ops_t *etxop = enp->en_etxop; + + EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); + + etxop->etxo_qenable(etp); +} + + __checkReturn efx_rc_t +efx_tx_qpio_enable( + __in efx_txq_t *etp) +{ + efx_nic_t *enp = etp->et_enp; + const efx_tx_ops_t *etxop = enp->en_etxop; + efx_rc_t rc; + + EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); + + if (~enp->en_features & EFX_FEATURE_PIO_BUFFERS) { + rc = ENOTSUP; + goto fail1; + } + if (etxop->etxo_qpio_enable == NULL) { + rc = ENOTSUP; + goto fail2; + } + if ((rc = etxop->etxo_qpio_enable(etp)) != 0) + goto fail3; + + return (0); + +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + return (rc); +} + + void +efx_tx_qpio_disable( + __in efx_txq_t *etp) +{ + efx_nic_t *enp = etp->et_enp; + const efx_tx_ops_t *etxop = enp->en_etxop; + + EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); + + if (etxop->etxo_qpio_disable != NULL) + etxop->etxo_qpio_disable(etp); +} + + __checkReturn efx_rc_t +efx_tx_qpio_write( + __in efx_txq_t *etp, + __in_ecount(buf_length) uint8_t *buffer, + __in size_t buf_length, + __in size_t pio_buf_offset) +{ + efx_nic_t *enp = etp->et_enp; + const efx_tx_ops_t *etxop = enp->en_etxop; + efx_rc_t rc; + + EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); + + if (etxop->etxo_qpio_write != NULL) { + if ((rc = etxop->etxo_qpio_write(etp, buffer, buf_length, + pio_buf_offset)) != 0) + goto fail1; + return (0); + } + + return (ENOTSUP); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + return (rc); +} + + __checkReturn efx_rc_t +efx_tx_qpio_post( + __in efx_txq_t *etp, + __in size_t pkt_length, + __in unsigned int completed, + __inout unsigned int *addedp) +{ + efx_nic_t *enp = etp->et_enp; + const efx_tx_ops_t *etxop = enp->en_etxop; + efx_rc_t rc; + + EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); + + if (etxop->etxo_qpio_post != NULL) { + if ((rc = etxop->etxo_qpio_post(etp, pkt_length, completed, + addedp)) != 0) + goto fail1; + return (0); + } + + return (ENOTSUP); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + return (rc); +} + + __checkReturn efx_rc_t +efx_tx_qdesc_post( + __in efx_txq_t *etp, + __in_ecount(n) efx_desc_t *ed, + __in unsigned int n, + __in unsigned int completed, + __inout unsigned int *addedp) +{ + efx_nic_t *enp = etp->et_enp; + const efx_tx_ops_t *etxop = enp->en_etxop; + efx_rc_t rc; + + EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); + + if ((rc = etxop->etxo_qdesc_post(etp, ed, + n, completed, addedp)) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + return (rc); +} + + void +efx_tx_qdesc_dma_create( + __in efx_txq_t *etp, + __in efsys_dma_addr_t addr, + __in size_t size, + __in boolean_t eop, + __out efx_desc_t *edp) +{ + efx_nic_t *enp = etp->et_enp; + const efx_tx_ops_t *etxop = enp->en_etxop; + + EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); + EFSYS_ASSERT(etxop->etxo_qdesc_dma_create != NULL); + + etxop->etxo_qdesc_dma_create(etp, addr, size, eop, edp); +} + + void +efx_tx_qdesc_tso_create( + __in efx_txq_t *etp, + __in uint16_t ipv4_id, + __in uint32_t tcp_seq, + __in uint8_t tcp_flags, + __out efx_desc_t *edp) +{ + efx_nic_t *enp = etp->et_enp; + const efx_tx_ops_t *etxop = enp->en_etxop; + + EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); + EFSYS_ASSERT(etxop->etxo_qdesc_tso_create != NULL); + + etxop->etxo_qdesc_tso_create(etp, ipv4_id, tcp_seq, tcp_flags, edp); +} + + void +efx_tx_qdesc_tso2_create( + __in efx_txq_t *etp, + __in uint16_t ipv4_id, + __in uint32_t tcp_seq, + __in uint16_t mss, + __out_ecount(count) efx_desc_t *edp, + __in int count) +{ + efx_nic_t *enp = etp->et_enp; + const efx_tx_ops_t *etxop = enp->en_etxop; + + EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); + EFSYS_ASSERT(etxop->etxo_qdesc_tso2_create != NULL); + + etxop->etxo_qdesc_tso2_create(etp, ipv4_id, tcp_seq, mss, edp, count); +} + + void +efx_tx_qdesc_vlantci_create( + __in efx_txq_t *etp, + __in uint16_t tci, + __out efx_desc_t *edp) +{ + efx_nic_t *enp = etp->et_enp; + const efx_tx_ops_t *etxop = enp->en_etxop; + + EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); + EFSYS_ASSERT(etxop->etxo_qdesc_vlantci_create != NULL); + + etxop->etxo_qdesc_vlantci_create(etp, tci, edp); +} + + +#if EFSYS_OPT_QSTATS + void +efx_tx_qstats_update( + __in efx_txq_t *etp, + __inout_ecount(TX_NQSTATS) efsys_stat_t *stat) +{ + efx_nic_t *enp = etp->et_enp; + const efx_tx_ops_t *etxop = enp->en_etxop; + + EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); + + etxop->etxo_qstats_update(etp, stat); +} +#endif + + +#if EFSYS_OPT_SIENA + +static __checkReturn efx_rc_t +siena_tx_init( + __in efx_nic_t *enp) +{ + efx_oword_t oword; + + /* + * Disable the timer-based TX DMA backoff and allow TX DMA to be + * controlled by the RX FIFO fill level (although always allow a + * minimal trickle). + */ + EFX_BAR_READO(enp, FR_AZ_TX_RESERVED_REG, &oword); + EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER, 0xfe); + EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER_EN, 1); + EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_ONE_PKT_PER_Q, 1); + EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PUSH_EN, 0); + EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DIS_NON_IP_EV, 1); + EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_THRESHOLD, 2); + EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff); + + /* + * Filter all packets less than 14 bytes to avoid parsing + * errors. + */ + EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1); + EFX_BAR_WRITEO(enp, FR_AZ_TX_RESERVED_REG, &oword); + + /* + * Do not set TX_NO_EOP_DISC_EN, since it limits packets to 16 + * descriptors (which is bad). + */ + EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword); + EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_NO_EOP_DISC_EN, 0); + EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword); + + return (0); +} + +#define EFX_TX_DESC(_etp, _addr, _size, _eop, _added) \ + do { \ + unsigned int id; \ + size_t offset; \ + efx_qword_t qword; \ + \ + id = (_added)++ & (_etp)->et_mask; \ + offset = id * sizeof (efx_qword_t); \ + \ + EFSYS_PROBE5(tx_post, unsigned int, (_etp)->et_index, \ + unsigned int, id, efsys_dma_addr_t, (_addr), \ + size_t, (_size), boolean_t, (_eop)); \ + \ + EFX_POPULATE_QWORD_4(qword, \ + FSF_AZ_TX_KER_CONT, (_eop) ? 0 : 1, \ + FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)(_size), \ + FSF_AZ_TX_KER_BUF_ADDR_DW0, \ + (uint32_t)((_addr) & 0xffffffff), \ + FSF_AZ_TX_KER_BUF_ADDR_DW1, \ + (uint32_t)((_addr) >> 32)); \ + EFSYS_MEM_WRITEQ((_etp)->et_esmp, offset, &qword); \ + \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +static __checkReturn efx_rc_t +siena_tx_qpost( + __in efx_txq_t *etp, + __in_ecount(n) efx_buffer_t *eb, + __in unsigned int n, + __in unsigned int completed, + __inout unsigned int *addedp) +{ + unsigned int added = *addedp; + unsigned int i; + int rc = ENOSPC; + + if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1)) + goto fail1; + + for (i = 0; i < n; i++) { + efx_buffer_t *ebp = &eb[i]; + efsys_dma_addr_t start = ebp->eb_addr; + size_t size = ebp->eb_size; + efsys_dma_addr_t end = start + size; + + /* + * Fragments must not span 4k boundaries. + * Here it is a stricter requirement than the maximum length. + */ + EFSYS_ASSERT(P2ROUNDUP(start + 1, + etp->et_enp->en_nic_cfg.enc_tx_dma_desc_boundary) >= end); + + EFX_TX_DESC(etp, start, size, ebp->eb_eop, added); + } + + EFX_TX_QSTAT_INCR(etp, TX_POST); + + *addedp = added; + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +static void +siena_tx_qpush( + __in efx_txq_t *etp, + __in unsigned int added, + __in unsigned int pushed) +{ + efx_nic_t *enp = etp->et_enp; + uint32_t wptr; + efx_dword_t dword; + efx_oword_t oword; + + /* Push the populated descriptors out */ + wptr = added & etp->et_mask; + + EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DESC_WPTR, wptr); + + /* Only write the third DWORD */ + EFX_POPULATE_DWORD_1(dword, + EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3)); + + /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */ + EFX_DMA_SYNC_QUEUE_FOR_DEVICE(etp->et_esmp, etp->et_mask + 1, + wptr, pushed & etp->et_mask); + EFSYS_PIO_WRITE_BARRIER(); + EFX_BAR_TBL_WRITED3(enp, FR_BZ_TX_DESC_UPD_REGP0, + etp->et_index, &dword, B_FALSE); +} + +#define EFX_MAX_PACE_VALUE 20 + +static __checkReturn efx_rc_t +siena_tx_qpace( + __in efx_txq_t *etp, + __in unsigned int ns) +{ + efx_nic_t *enp = etp->et_enp; + efx_nic_cfg_t *encp = &(enp->en_nic_cfg); + efx_oword_t oword; + unsigned int pace_val; + unsigned int timer_period; + efx_rc_t rc; + + if (ns == 0) { + pace_val = 0; + } else { + /* + * The pace_val to write into the table is s.t + * ns <= timer_period * (2 ^ pace_val) + */ + timer_period = 104 / encp->enc_clk_mult; + for (pace_val = 1; pace_val <= EFX_MAX_PACE_VALUE; pace_val++) { + if ((timer_period << pace_val) >= ns) + break; + } + } + if (pace_val > EFX_MAX_PACE_VALUE) { + rc = EINVAL; + goto fail1; + } + + /* Update the pacing table */ + EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_PACE, pace_val); + EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_PACE_TBL, etp->et_index, + &oword, B_TRUE); + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +static __checkReturn efx_rc_t +siena_tx_qflush( + __in efx_txq_t *etp) +{ + efx_nic_t *enp = etp->et_enp; + efx_oword_t oword; + uint32_t label; + + efx_tx_qpace(etp, 0); + + label = etp->et_index; + + /* Flush the queue */ + EFX_POPULATE_OWORD_2(oword, FRF_AZ_TX_FLUSH_DESCQ_CMD, 1, + FRF_AZ_TX_FLUSH_DESCQ, label); + EFX_BAR_WRITEO(enp, FR_AZ_TX_FLUSH_DESCQ_REG, &oword); + + return (0); +} + +static void +siena_tx_qenable( + __in efx_txq_t *etp) +{ + efx_nic_t *enp = etp->et_enp; + efx_oword_t oword; + + EFX_BAR_TBL_READO(enp, FR_AZ_TX_DESC_PTR_TBL, + etp->et_index, &oword, B_TRUE); + + EFSYS_PROBE5(tx_descq_ptr, unsigned int, etp->et_index, + uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_3), + uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_2), + uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_1), + uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_0)); + + EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DC_HW_RPTR, 0); + EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_HW_RPTR, 0); + EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_EN, 1); + + EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, + etp->et_index, &oword, B_TRUE); +} + +static __checkReturn efx_rc_t +siena_tx_qcreate( + __in efx_nic_t *enp, + __in unsigned int index, + __in unsigned int label, + __in efsys_mem_t *esmp, + __in size_t n, + __in uint32_t id, + __in uint16_t flags, + __in efx_evq_t *eep, + __in efx_txq_t *etp, + __out unsigned int *addedp) +{ + efx_nic_cfg_t *encp = &(enp->en_nic_cfg); + efx_oword_t oword; + uint32_t size; + efx_rc_t rc; + + _NOTE(ARGUNUSED(esmp)) + + EFX_STATIC_ASSERT(EFX_EV_TX_NLABELS == + (1 << FRF_AZ_TX_DESCQ_LABEL_WIDTH)); + EFSYS_ASSERT3U(label, <, EFX_EV_TX_NLABELS); + + EFSYS_ASSERT(ISP2(encp->enc_txq_max_ndescs)); + EFX_STATIC_ASSERT(ISP2(EFX_TXQ_MINNDESCS)); + + if (!ISP2(n) || (n < EFX_TXQ_MINNDESCS) || (n > EFX_EVQ_MAXNEVS)) { + rc = EINVAL; + goto fail1; + } + if (index >= encp->enc_txq_limit) { + rc = EINVAL; + goto fail2; + } + for (size = 0; + (1 << size) <= (int)(encp->enc_txq_max_ndescs / EFX_TXQ_MINNDESCS); + size++) + if ((1 << size) == (int)(n / EFX_TXQ_MINNDESCS)) + break; + if (id + (1 << size) >= encp->enc_buftbl_limit) { + rc = EINVAL; + goto fail3; + } + + /* Set up the new descriptor queue */ + *addedp = 0; + + EFX_POPULATE_OWORD_6(oword, + FRF_AZ_TX_DESCQ_BUF_BASE_ID, id, + FRF_AZ_TX_DESCQ_EVQ_ID, eep->ee_index, + FRF_AZ_TX_DESCQ_OWNER_ID, 0, + FRF_AZ_TX_DESCQ_LABEL, label, + FRF_AZ_TX_DESCQ_SIZE, size, + FRF_AZ_TX_DESCQ_TYPE, 0); + + EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_NON_IP_DROP_DIS, 1); + EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_IP_CHKSM_DIS, + (flags & EFX_TXQ_CKSUM_IPV4) ? 0 : 1); + EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_TCP_CHKSM_DIS, + (flags & EFX_TXQ_CKSUM_TCPUDP) ? 0 : 1); + + EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, + etp->et_index, &oword, B_TRUE); + + return (0); + +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +siena_tx_qdesc_post( + __in efx_txq_t *etp, + __in_ecount(n) efx_desc_t *ed, + __in unsigned int n, + __in unsigned int completed, + __inout unsigned int *addedp) +{ + unsigned int added = *addedp; + unsigned int i; + efx_rc_t rc; + + if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1)) { + rc = ENOSPC; + goto fail1; + } + + for (i = 0; i < n; i++) { + efx_desc_t *edp = &ed[i]; + unsigned int id; + size_t offset; + + id = added++ & etp->et_mask; + offset = id * sizeof (efx_desc_t); + + EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &edp->ed_eq); + } + + EFSYS_PROBE3(tx_desc_post, unsigned int, etp->et_index, + unsigned int, added, unsigned int, n); + + EFX_TX_QSTAT_INCR(etp, TX_POST); + + *addedp = added; + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + return (rc); +} + + void +siena_tx_qdesc_dma_create( + __in efx_txq_t *etp, + __in efsys_dma_addr_t addr, + __in size_t size, + __in boolean_t eop, + __out efx_desc_t *edp) +{ + /* + * Fragments must not span 4k boundaries. + * Here it is a stricter requirement than the maximum length. + */ + EFSYS_ASSERT(P2ROUNDUP(addr + 1, + etp->et_enp->en_nic_cfg.enc_tx_dma_desc_boundary) >= addr + size); + + EFSYS_PROBE4(tx_desc_dma_create, unsigned int, etp->et_index, + efsys_dma_addr_t, addr, + size_t, size, boolean_t, eop); + + EFX_POPULATE_QWORD_4(edp->ed_eq, + FSF_AZ_TX_KER_CONT, eop ? 0 : 1, + FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)size, + FSF_AZ_TX_KER_BUF_ADDR_DW0, + (uint32_t)(addr & 0xffffffff), + FSF_AZ_TX_KER_BUF_ADDR_DW1, + (uint32_t)(addr >> 32)); +} + +#endif /* EFSYS_OPT_SIENA */ + +#if EFSYS_OPT_QSTATS +#if EFSYS_OPT_NAMES +/* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 2866874ecd7a363b */ +static const char * const __efx_tx_qstat_name[] = { + "post", + "post_pio", +}; +/* END MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock */ + + const char * +efx_tx_qstat_name( + __in efx_nic_t *enp, + __in unsigned int id) +{ + _NOTE(ARGUNUSED(enp)) + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(id, <, TX_NQSTATS); + + return (__efx_tx_qstat_name[id]); +} +#endif /* EFSYS_OPT_NAMES */ +#endif /* EFSYS_OPT_QSTATS */ + +#if EFSYS_OPT_SIENA + +#if EFSYS_OPT_QSTATS +static void +siena_tx_qstats_update( + __in efx_txq_t *etp, + __inout_ecount(TX_NQSTATS) efsys_stat_t *stat) +{ + unsigned int id; + + for (id = 0; id < TX_NQSTATS; id++) { + efsys_stat_t *essp = &stat[id]; + + EFSYS_STAT_INCR(essp, etp->et_stat[id]); + etp->et_stat[id] = 0; + } +} +#endif /* EFSYS_OPT_QSTATS */ + +static void +siena_tx_qdestroy( + __in efx_txq_t *etp) +{ + efx_nic_t *enp = etp->et_enp; + efx_oword_t oword; + + /* Purge descriptor queue */ + EFX_ZERO_OWORD(oword); + + EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, + etp->et_index, &oword, B_TRUE); +} + +static void +siena_tx_fini( + __in efx_nic_t *enp) +{ + _NOTE(ARGUNUSED(enp)) +} + +#endif /* EFSYS_OPT_SIENA */ diff --git a/dpdk/drivers/net/sfc/base/efx_types.h b/dpdk/drivers/net/sfc/base/efx_types.h new file mode 100644 index 00000000..b8ee14a6 --- /dev/null +++ b/dpdk/drivers/net/sfc/base/efx_types.h @@ -0,0 +1,1647 @@ +/* + * Copyright (c) 2007-2016 Solarflare Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are + * those of the authors and should not be interpreted as representing official + * policies, either expressed or implied, of the FreeBSD Project. + * + * Ackowledgement to Fen Systems Ltd. + */ + +#ifndef _SYS_EFX_TYPES_H +#define _SYS_EFX_TYPES_H + +#include "efsys.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * Bitfield access + * + * Solarflare NICs make extensive use of bitfields up to 128 bits + * wide. Since there is no native 128-bit datatype on most systems, + * and since 64-bit datatypes are inefficient on 32-bit systems and + * vice versa, we wrap accesses in a way that uses the most efficient + * datatype. + * + * The NICs are PCI devices and therefore little-endian. Since most + * of the quantities that we deal with are DMAed to/from host memory, + * we define our datatypes (efx_oword_t, efx_qword_t and efx_dword_t) + * to be little-endian. + * + * In the less common case of using PIO for individual register + * writes, we construct the little-endian datatype in host memory and + * then use non-swapping register access primitives, rather than + * constructing a native-endian datatype and relying on implicit + * byte-swapping. (We use a similar strategy for register reads.) + */ + +/* + * NOTE: Field definitions here and elsewhere are done in terms of a lowest + * bit number (LBN) and a width. + */ + +#define EFX_DUMMY_FIELD_LBN 0 +#define EFX_DUMMY_FIELD_WIDTH 0 + +#define EFX_BYTE_0_LBN 0 +#define EFX_BYTE_0_WIDTH 8 + +#define EFX_BYTE_1_LBN 8 +#define EFX_BYTE_1_WIDTH 8 + +#define EFX_BYTE_2_LBN 16 +#define EFX_BYTE_2_WIDTH 8 + +#define EFX_BYTE_3_LBN 24 +#define EFX_BYTE_3_WIDTH 8 + +#define EFX_BYTE_4_LBN 32 +#define EFX_BYTE_4_WIDTH 8 + +#define EFX_BYTE_5_LBN 40 +#define EFX_BYTE_5_WIDTH 8 + +#define EFX_BYTE_6_LBN 48 +#define EFX_BYTE_6_WIDTH 8 + +#define EFX_BYTE_7_LBN 56 +#define EFX_BYTE_7_WIDTH 8 + +#define EFX_WORD_0_LBN 0 +#define EFX_WORD_0_WIDTH 16 + +#define EFX_WORD_1_LBN 16 +#define EFX_WORD_1_WIDTH 16 + +#define EFX_WORD_2_LBN 32 +#define EFX_WORD_2_WIDTH 16 + +#define EFX_WORD_3_LBN 48 +#define EFX_WORD_3_WIDTH 16 + +#define EFX_DWORD_0_LBN 0 +#define EFX_DWORD_0_WIDTH 32 + +#define EFX_DWORD_1_LBN 32 +#define EFX_DWORD_1_WIDTH 32 + +#define EFX_DWORD_2_LBN 64 +#define EFX_DWORD_2_WIDTH 32 + +#define EFX_DWORD_3_LBN 96 +#define EFX_DWORD_3_WIDTH 32 + +/* There are intentionally no EFX_QWORD_0 or EFX_QWORD_1 field definitions + * here as the implementaion of EFX_QWORD_FIELD and EFX_OWORD_FIELD do not + * support field widths larger than 32 bits. + */ + +/* Specified attribute (i.e. LBN ow WIDTH) of the specified field */ +#define EFX_VAL(_field, _attribute) \ + _field ## _ ## _attribute + +/* Lowest bit number of the specified field */ +#define EFX_LOW_BIT(_field) \ + EFX_VAL(_field, LBN) + +/* Width of the specified field */ +#define EFX_WIDTH(_field) \ + EFX_VAL(_field, WIDTH) + +/* Highest bit number of the specified field */ +#define EFX_HIGH_BIT(_field) \ + (EFX_LOW_BIT(_field) + EFX_WIDTH(_field) - 1) + +/* + * 64-bit mask equal in width to the specified field. + * + * For example, a field with width 5 would have a mask of 0x000000000000001f. + */ +#define EFX_MASK64(_field) \ + ((EFX_WIDTH(_field) == 64) ? ~((uint64_t)0) : \ + (((((uint64_t)1) << EFX_WIDTH(_field))) - 1)) +/* + * 32-bit mask equal in width to the specified field. + * + * For example, a field with width 5 would have a mask of 0x0000001f. + */ +#define EFX_MASK32(_field) \ + ((EFX_WIDTH(_field) == 32) ? ~((uint32_t)0) : \ + (((((uint32_t)1) << EFX_WIDTH(_field))) - 1)) + +/* + * 16-bit mask equal in width to the specified field. + * + * For example, a field with width 5 would have a mask of 0x001f. + */ +#define EFX_MASK16(_field) \ + ((EFX_WIDTH(_field) == 16) ? 0xffffu : \ + (uint16_t)((1 << EFX_WIDTH(_field)) - 1)) + +/* + * 8-bit mask equal in width to the specified field. + * + * For example, a field with width 5 would have a mask of 0x1f. + */ +#define EFX_MASK8(_field) \ + ((uint8_t)((1 << EFX_WIDTH(_field)) - 1)) + +#pragma pack(1) + +/* + * A byte (i.e. 8-bit) datatype + */ +typedef union efx_byte_u { + uint8_t eb_u8[1]; +} efx_byte_t; + +/* + * A word (i.e. 16-bit) datatype + * + * This datatype is defined to be little-endian. + */ +typedef union efx_word_u { + efx_byte_t ew_byte[2]; + uint16_t ew_u16[1]; + uint8_t ew_u8[2]; +} efx_word_t; + +/* + * A doubleword (i.e. 32-bit) datatype + * + * This datatype is defined to be little-endian. + */ +typedef union efx_dword_u { + efx_byte_t ed_byte[4]; + efx_word_t ed_word[2]; + uint32_t ed_u32[1]; + uint16_t ed_u16[2]; + uint8_t ed_u8[4]; +} efx_dword_t; + +/* + * A quadword (i.e. 64-bit) datatype + * + * This datatype is defined to be little-endian. + */ +typedef union efx_qword_u { + efx_byte_t eq_byte[8]; + efx_word_t eq_word[4]; + efx_dword_t eq_dword[2]; +#if EFSYS_HAS_UINT64 + uint64_t eq_u64[1]; +#endif + uint32_t eq_u32[2]; + uint16_t eq_u16[4]; + uint8_t eq_u8[8]; +} efx_qword_t; + +/* + * An octword (i.e. 128-bit) datatype + * + * This datatype is defined to be little-endian. + */ +typedef union efx_oword_u { + efx_byte_t eo_byte[16]; + efx_word_t eo_word[8]; + efx_dword_t eo_dword[4]; + efx_qword_t eo_qword[2]; +#if EFSYS_HAS_SSE2_M128 + __m128i eo_u128[1]; +#endif +#if EFSYS_HAS_UINT64 + uint64_t eo_u64[2]; +#endif + uint32_t eo_u32[4]; + uint16_t eo_u16[8]; + uint8_t eo_u8[16]; +} efx_oword_t; + +#pragma pack() + +#define __SWAP16(_x) \ + ((((_x) & 0xff) << 8) | \ + (((_x) >> 8) & 0xff)) + +#define __SWAP32(_x) \ + ((__SWAP16((_x) & 0xffff) << 16) | \ + __SWAP16(((_x) >> 16) & 0xffff)) + +#define __SWAP64(_x) \ + ((__SWAP32((_x) & 0xffffffff) << 32) | \ + __SWAP32(((_x) >> 32) & 0xffffffff)) + +#define __NOSWAP16(_x) (_x) +#define __NOSWAP32(_x) (_x) +#define __NOSWAP64(_x) (_x) + +#if EFSYS_IS_BIG_ENDIAN + +#define __CPU_TO_LE_16(_x) ((uint16_t)__SWAP16(_x)) +#define __LE_TO_CPU_16(_x) ((uint16_t)__SWAP16(_x)) +#define __CPU_TO_BE_16(_x) ((uint16_t)__NOSWAP16(_x)) +#define __BE_TO_CPU_16(_x) ((uint16_t)__NOSWAP16(_x)) + +#define __CPU_TO_LE_32(_x) ((uint32_t)__SWAP32(_x)) +#define __LE_TO_CPU_32(_x) ((uint32_t)__SWAP32(_x)) +#define __CPU_TO_BE_32(_x) ((uint32_t)__NOSWAP32(_x)) +#define __BE_TO_CPU_32(_x) ((uint32_t)__NOSWAP32(_x)) + +#define __CPU_TO_LE_64(_x) ((uint64_t)__SWAP64(_x)) +#define __LE_TO_CPU_64(_x) ((uint64_t)__SWAP64(_x)) +#define __CPU_TO_BE_64(_x) ((uint64_t)__NOSWAP64(_x)) +#define __BE_TO_CPU_64(_x) ((uint64_t)__NOSWAP64(_x)) + +#elif EFSYS_IS_LITTLE_ENDIAN + +#define __CPU_TO_LE_16(_x) ((uint16_t)__NOSWAP16(_x)) +#define __LE_TO_CPU_16(_x) ((uint16_t)__NOSWAP16(_x)) +#define __CPU_TO_BE_16(_x) ((uint16_t)__SWAP16(_x)) +#define __BE_TO_CPU_16(_x) ((uint16_t)__SWAP16(_x)) + +#define __CPU_TO_LE_32(_x) ((uint32_t)__NOSWAP32(_x)) +#define __LE_TO_CPU_32(_x) ((uint32_t)__NOSWAP32(_x)) +#define __CPU_TO_BE_32(_x) ((uint32_t)__SWAP32(_x)) +#define __BE_TO_CPU_32(_x) ((uint32_t)__SWAP32(_x)) + +#define __CPU_TO_LE_64(_x) ((uint64_t)__NOSWAP64(_x)) +#define __LE_TO_CPU_64(_x) ((uint64_t)__NOSWAP64(_x)) +#define __CPU_TO_BE_64(_x) ((uint64_t)__SWAP64(_x)) +#define __BE_TO_CPU_64(_x) ((uint64_t)__SWAP64(_x)) + +#else + +#error "Neither of EFSYS_IS_{BIG,LITTLE}_ENDIAN is set" + +#endif + +#define __NATIVE_8(_x) (uint8_t)(_x) + +/* Format string for printing an efx_byte_t */ +#define EFX_BYTE_FMT "0x%02x" + +/* Format string for printing an efx_word_t */ +#define EFX_WORD_FMT "0x%04x" + +/* Format string for printing an efx_dword_t */ +#define EFX_DWORD_FMT "0x%08x" + +/* Format string for printing an efx_qword_t */ +#define EFX_QWORD_FMT "0x%08x:%08x" + +/* Format string for printing an efx_oword_t */ +#define EFX_OWORD_FMT "0x%08x:%08x:%08x:%08x" + +/* Parameters for printing an efx_byte_t */ +#define EFX_BYTE_VAL(_byte) \ + ((unsigned int)__NATIVE_8((_byte).eb_u8[0])) + +/* Parameters for printing an efx_word_t */ +#define EFX_WORD_VAL(_word) \ + ((unsigned int)__LE_TO_CPU_16((_word).ew_u16[0])) + +/* Parameters for printing an efx_dword_t */ +#define EFX_DWORD_VAL(_dword) \ + ((unsigned int)__LE_TO_CPU_32((_dword).ed_u32[0])) + +/* Parameters for printing an efx_qword_t */ +#define EFX_QWORD_VAL(_qword) \ + ((unsigned int)__LE_TO_CPU_32((_qword).eq_u32[1])), \ + ((unsigned int)__LE_TO_CPU_32((_qword).eq_u32[0])) + +/* Parameters for printing an efx_oword_t */ +#define EFX_OWORD_VAL(_oword) \ + ((unsigned int)__LE_TO_CPU_32((_oword).eo_u32[3])), \ + ((unsigned int)__LE_TO_CPU_32((_oword).eo_u32[2])), \ + ((unsigned int)__LE_TO_CPU_32((_oword).eo_u32[1])), \ + ((unsigned int)__LE_TO_CPU_32((_oword).eo_u32[0])) + +/* + * Stop lint complaining about some shifts. + */ +#ifdef __lint +extern int fix_lint; +#define FIX_LINT(_x) (_x + fix_lint) +#else +#define FIX_LINT(_x) (_x) +#endif + +/* + * Extract bit field portion [low,high) from the native-endian element + * which contains bits [min,max). + * + * For example, suppose "element" represents the high 32 bits of a + * 64-bit value, and we wish to extract the bits belonging to the bit + * field occupying bits 28-45 of this 64-bit value. + * + * Then EFX_EXTRACT(_element, 32, 63, 28, 45) would give + * + * (_element) << 4 + * + * The result will contain the relevant bits filled in in the range + * [0,high-low), with garbage in bits [high-low+1,...). + */ +#define EFX_EXTRACT_NATIVE(_element, _min, _max, _low, _high) \ + ((FIX_LINT(_low > _max) || FIX_LINT(_high < _min)) ? \ + 0U : \ + ((_low > _min) ? \ + ((_element) >> (_low - _min)) : \ + ((_element) << (_min - _low)))) + +/* + * Extract bit field portion [low,high) from the 64-bit little-endian + * element which contains bits [min,max) + */ +#define EFX_EXTRACT64(_element, _min, _max, _low, _high) \ + EFX_EXTRACT_NATIVE(__LE_TO_CPU_64(_element), _min, _max, _low, _high) + +/* + * Extract bit field portion [low,high) from the 32-bit little-endian + * element which contains bits [min,max) + */ +#define EFX_EXTRACT32(_element, _min, _max, _low, _high) \ + EFX_EXTRACT_NATIVE(__LE_TO_CPU_32(_element), _min, _max, _low, _high) + +/* + * Extract bit field portion [low,high) from the 16-bit little-endian + * element which contains bits [min,max) + */ +#define EFX_EXTRACT16(_element, _min, _max, _low, _high) \ + EFX_EXTRACT_NATIVE(__LE_TO_CPU_16(_element), _min, _max, _low, _high) + +/* + * Extract bit field portion [low,high) from the 8-bit + * element which contains bits [min,max) + */ +#define EFX_EXTRACT8(_element, _min, _max, _low, _high) \ + EFX_EXTRACT_NATIVE(__NATIVE_8(_element), _min, _max, _low, _high) + +#define EFX_EXTRACT_OWORD64(_oword, _low, _high) \ + (EFX_EXTRACT64((_oword).eo_u64[0], FIX_LINT(0), FIX_LINT(63), \ + _low, _high) | \ + EFX_EXTRACT64((_oword).eo_u64[1], FIX_LINT(64), FIX_LINT(127), \ + _low, _high)) + +#define EFX_EXTRACT_OWORD32(_oword, _low, _high) \ + (EFX_EXTRACT32((_oword).eo_u32[0], FIX_LINT(0), FIX_LINT(31), \ + _low, _high) | \ + EFX_EXTRACT32((_oword).eo_u32[1], FIX_LINT(32), FIX_LINT(63), \ + _low, _high) | \ + EFX_EXTRACT32((_oword).eo_u32[2], FIX_LINT(64), FIX_LINT(95), \ + _low, _high) | \ + EFX_EXTRACT32((_oword).eo_u32[3], FIX_LINT(96), FIX_LINT(127), \ + _low, _high)) + +#define EFX_EXTRACT_QWORD64(_qword, _low, _high) \ + (EFX_EXTRACT64((_qword).eq_u64[0], FIX_LINT(0), FIX_LINT(63), \ + _low, _high)) + +#define EFX_EXTRACT_QWORD32(_qword, _low, _high) \ + (EFX_EXTRACT32((_qword).eq_u32[0], FIX_LINT(0), FIX_LINT(31), \ + _low, _high) | \ + EFX_EXTRACT32((_qword).eq_u32[1], FIX_LINT(32), FIX_LINT(63), \ + _low, _high)) + +#define EFX_EXTRACT_DWORD(_dword, _low, _high) \ + (EFX_EXTRACT32((_dword).ed_u32[0], FIX_LINT(0), FIX_LINT(31), \ + _low, _high)) + +#define EFX_EXTRACT_WORD(_word, _low, _high) \ + (EFX_EXTRACT16((_word).ew_u16[0], FIX_LINT(0), FIX_LINT(15), \ + _low, _high)) + +#define EFX_EXTRACT_BYTE(_byte, _low, _high) \ + (EFX_EXTRACT8((_byte).eb_u8[0], FIX_LINT(0), FIX_LINT(7), \ + _low, _high)) + + +#define EFX_OWORD_FIELD64(_oword, _field) \ + ((uint32_t)EFX_EXTRACT_OWORD64(_oword, EFX_LOW_BIT(_field), \ + EFX_HIGH_BIT(_field)) & EFX_MASK32(_field)) + +#define EFX_OWORD_FIELD32(_oword, _field) \ + (EFX_EXTRACT_OWORD32(_oword, EFX_LOW_BIT(_field), \ + EFX_HIGH_BIT(_field)) & EFX_MASK32(_field)) + +#define EFX_QWORD_FIELD64(_qword, _field) \ + ((uint32_t)EFX_EXTRACT_QWORD64(_qword, EFX_LOW_BIT(_field), \ + EFX_HIGH_BIT(_field)) & EFX_MASK32(_field)) + +#define EFX_QWORD_FIELD32(_qword, _field) \ + (EFX_EXTRACT_QWORD32(_qword, EFX_LOW_BIT(_field), \ + EFX_HIGH_BIT(_field)) & EFX_MASK32(_field)) + +#define EFX_DWORD_FIELD(_dword, _field) \ + (EFX_EXTRACT_DWORD(_dword, EFX_LOW_BIT(_field), \ + EFX_HIGH_BIT(_field)) & EFX_MASK32(_field)) + +#define EFX_WORD_FIELD(_word, _field) \ + (EFX_EXTRACT_WORD(_word, EFX_LOW_BIT(_field), \ + EFX_HIGH_BIT(_field)) & EFX_MASK16(_field)) + +#define EFX_BYTE_FIELD(_byte, _field) \ + (EFX_EXTRACT_BYTE(_byte, EFX_LOW_BIT(_field), \ + EFX_HIGH_BIT(_field)) & EFX_MASK8(_field)) + + +#define EFX_OWORD_IS_EQUAL64(_oword_a, _oword_b) \ + ((_oword_a).eo_u64[0] == (_oword_b).eo_u64[0] && \ + (_oword_a).eo_u64[1] == (_oword_b).eo_u64[1]) + +#define EFX_OWORD_IS_EQUAL32(_oword_a, _oword_b) \ + ((_oword_a).eo_u32[0] == (_oword_b).eo_u32[0] && \ + (_oword_a).eo_u32[1] == (_oword_b).eo_u32[1] && \ + (_oword_a).eo_u32[2] == (_oword_b).eo_u32[2] && \ + (_oword_a).eo_u32[3] == (_oword_b).eo_u32[3]) + +#define EFX_QWORD_IS_EQUAL64(_qword_a, _qword_b) \ + ((_qword_a).eq_u64[0] == (_qword_b).eq_u64[0]) + +#define EFX_QWORD_IS_EQUAL32(_qword_a, _qword_b) \ + ((_qword_a).eq_u32[0] == (_qword_b).eq_u32[0] && \ + (_qword_a).eq_u32[1] == (_qword_b).eq_u32[1]) + +#define EFX_DWORD_IS_EQUAL(_dword_a, _dword_b) \ + ((_dword_a).ed_u32[0] == (_dword_b).ed_u32[0]) + +#define EFX_WORD_IS_EQUAL(_word_a, _word_b) \ + ((_word_a).ew_u16[0] == (_word_b).ew_u16[0]) + +#define EFX_BYTE_IS_EQUAL(_byte_a, _byte_b) \ + ((_byte_a).eb_u8[0] == (_byte_b).eb_u8[0]) + + +#define EFX_OWORD_IS_ZERO64(_oword) \ + (((_oword).eo_u64[0] | \ + (_oword).eo_u64[1]) == 0) + +#define EFX_OWORD_IS_ZERO32(_oword) \ + (((_oword).eo_u32[0] | \ + (_oword).eo_u32[1] | \ + (_oword).eo_u32[2] | \ + (_oword).eo_u32[3]) == 0) + +#define EFX_QWORD_IS_ZERO64(_qword) \ + (((_qword).eq_u64[0]) == 0) + +#define EFX_QWORD_IS_ZERO32(_qword) \ + (((_qword).eq_u32[0] | \ + (_qword).eq_u32[1]) == 0) + +#define EFX_DWORD_IS_ZERO(_dword) \ + (((_dword).ed_u32[0]) == 0) + +#define EFX_WORD_IS_ZERO(_word) \ + (((_word).ew_u16[0]) == 0) + +#define EFX_BYTE_IS_ZERO(_byte) \ + (((_byte).eb_u8[0]) == 0) + + +#define EFX_OWORD_IS_SET64(_oword) \ + (((_oword).eo_u64[0] & \ + (_oword).eo_u64[1]) == ~((uint64_t)0)) + +#define EFX_OWORD_IS_SET32(_oword) \ + (((_oword).eo_u32[0] & \ + (_oword).eo_u32[1] & \ + (_oword).eo_u32[2] & \ + (_oword).eo_u32[3]) == ~((uint32_t)0)) + +#define EFX_QWORD_IS_SET64(_qword) \ + (((_qword).eq_u64[0]) == ~((uint64_t)0)) + +#define EFX_QWORD_IS_SET32(_qword) \ + (((_qword).eq_u32[0] & \ + (_qword).eq_u32[1]) == ~((uint32_t)0)) + +#define EFX_DWORD_IS_SET(_dword) \ + ((_dword).ed_u32[0] == ~((uint32_t)0)) + +#define EFX_WORD_IS_SET(_word) \ + ((_word).ew_u16[0] == ~((uint16_t)0)) + +#define EFX_BYTE_IS_SET(_byte) \ + ((_byte).eb_u8[0] == ~((uint8_t)0)) + +/* + * Construct bit field portion + * + * Creates the portion of the bit field [low,high) that lies within + * the range [min,max). + */ + +#define EFX_INSERT_NATIVE64(_min, _max, _low, _high, _value) \ + (((_low > _max) || (_high < _min)) ? \ + 0U : \ + ((_low > _min) ? \ + (((uint64_t)(_value)) << (_low - _min)) : \ + (((uint64_t)(_value)) >> (_min - _low)))) + +#define EFX_INSERT_NATIVE32(_min, _max, _low, _high, _value) \ + (((_low > _max) || (_high < _min)) ? \ + 0U : \ + ((_low > _min) ? \ + (((uint32_t)(_value)) << (_low - _min)) : \ + (((uint32_t)(_value)) >> (_min - _low)))) + +#define EFX_INSERT_NATIVE16(_min, _max, _low, _high, _value) \ + (((_low > _max) || (_high < _min)) ? \ + 0U : \ + (uint16_t)((_low > _min) ? \ + ((_value) << (_low - _min)) : \ + ((_value) >> (_min - _low)))) + +#define EFX_INSERT_NATIVE8(_min, _max, _low, _high, _value) \ + (((_low > _max) || (_high < _min)) ? \ + 0U : \ + (uint8_t)((_low > _min) ? \ + ((_value) << (_low - _min)) : \ + ((_value) >> (_min - _low)))) + +/* + * Construct bit field portion + * + * Creates the portion of the named bit field that lies within the + * range [min,max). + */ +#define EFX_INSERT_FIELD_NATIVE64(_min, _max, _field, _value) \ + EFX_INSERT_NATIVE64(_min, _max, EFX_LOW_BIT(_field), \ + EFX_HIGH_BIT(_field), _value) + +#define EFX_INSERT_FIELD_NATIVE32(_min, _max, _field, _value) \ + EFX_INSERT_NATIVE32(_min, _max, EFX_LOW_BIT(_field), \ + EFX_HIGH_BIT(_field), _value) + +#define EFX_INSERT_FIELD_NATIVE16(_min, _max, _field, _value) \ + EFX_INSERT_NATIVE16(_min, _max, EFX_LOW_BIT(_field), \ + EFX_HIGH_BIT(_field), _value) + +#define EFX_INSERT_FIELD_NATIVE8(_min, _max, _field, _value) \ + EFX_INSERT_NATIVE8(_min, _max, EFX_LOW_BIT(_field), \ + EFX_HIGH_BIT(_field), _value) + +/* + * Construct bit field + * + * Creates the portion of the named bit fields that lie within the + * range [min,max). + */ +#define EFX_INSERT_FIELDS64(_min, _max, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5, _field6, _value6, \ + _field7, _value7, _field8, _value8, _field9, _value9, \ + _field10, _value10) \ + __CPU_TO_LE_64( \ + EFX_INSERT_FIELD_NATIVE64(_min, _max, _field1, _value1) | \ + EFX_INSERT_FIELD_NATIVE64(_min, _max, _field2, _value2) | \ + EFX_INSERT_FIELD_NATIVE64(_min, _max, _field3, _value3) | \ + EFX_INSERT_FIELD_NATIVE64(_min, _max, _field4, _value4) | \ + EFX_INSERT_FIELD_NATIVE64(_min, _max, _field5, _value5) | \ + EFX_INSERT_FIELD_NATIVE64(_min, _max, _field6, _value6) | \ + EFX_INSERT_FIELD_NATIVE64(_min, _max, _field7, _value7) | \ + EFX_INSERT_FIELD_NATIVE64(_min, _max, _field8, _value8) | \ + EFX_INSERT_FIELD_NATIVE64(_min, _max, _field9, _value9) | \ + EFX_INSERT_FIELD_NATIVE64(_min, _max, _field10, _value10)) + +#define EFX_INSERT_FIELDS32(_min, _max, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5, _field6, _value6, \ + _field7, _value7, _field8, _value8, _field9, _value9, \ + _field10, _value10) \ + __CPU_TO_LE_32( \ + EFX_INSERT_FIELD_NATIVE32(_min, _max, _field1, _value1) | \ + EFX_INSERT_FIELD_NATIVE32(_min, _max, _field2, _value2) | \ + EFX_INSERT_FIELD_NATIVE32(_min, _max, _field3, _value3) | \ + EFX_INSERT_FIELD_NATIVE32(_min, _max, _field4, _value4) | \ + EFX_INSERT_FIELD_NATIVE32(_min, _max, _field5, _value5) | \ + EFX_INSERT_FIELD_NATIVE32(_min, _max, _field6, _value6) | \ + EFX_INSERT_FIELD_NATIVE32(_min, _max, _field7, _value7) | \ + EFX_INSERT_FIELD_NATIVE32(_min, _max, _field8, _value8) | \ + EFX_INSERT_FIELD_NATIVE32(_min, _max, _field9, _value9) | \ + EFX_INSERT_FIELD_NATIVE32(_min, _max, _field10, _value10)) + +#define EFX_INSERT_FIELDS16(_min, _max, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5, _field6, _value6, \ + _field7, _value7, _field8, _value8, _field9, _value9, \ + _field10, _value10) \ + __CPU_TO_LE_16( \ + EFX_INSERT_FIELD_NATIVE16(_min, _max, _field1, _value1) | \ + EFX_INSERT_FIELD_NATIVE16(_min, _max, _field2, _value2) | \ + EFX_INSERT_FIELD_NATIVE16(_min, _max, _field3, _value3) | \ + EFX_INSERT_FIELD_NATIVE16(_min, _max, _field4, _value4) | \ + EFX_INSERT_FIELD_NATIVE16(_min, _max, _field5, _value5) | \ + EFX_INSERT_FIELD_NATIVE16(_min, _max, _field6, _value6) | \ + EFX_INSERT_FIELD_NATIVE16(_min, _max, _field7, _value7) | \ + EFX_INSERT_FIELD_NATIVE16(_min, _max, _field8, _value8) | \ + EFX_INSERT_FIELD_NATIVE16(_min, _max, _field9, _value9) | \ + EFX_INSERT_FIELD_NATIVE16(_min, _max, _field10, _value10)) + +#define EFX_INSERT_FIELDS8(_min, _max, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5, _field6, _value6, \ + _field7, _value7, _field8, _value8, _field9, _value9, \ + _field10, _value10) \ + __NATIVE_8( \ + EFX_INSERT_FIELD_NATIVE8(_min, _max, _field1, _value1) | \ + EFX_INSERT_FIELD_NATIVE8(_min, _max, _field2, _value2) | \ + EFX_INSERT_FIELD_NATIVE8(_min, _max, _field3, _value3) | \ + EFX_INSERT_FIELD_NATIVE8(_min, _max, _field4, _value4) | \ + EFX_INSERT_FIELD_NATIVE8(_min, _max, _field5, _value5) | \ + EFX_INSERT_FIELD_NATIVE8(_min, _max, _field6, _value6) | \ + EFX_INSERT_FIELD_NATIVE8(_min, _max, _field7, _value7) | \ + EFX_INSERT_FIELD_NATIVE8(_min, _max, _field8, _value8) | \ + EFX_INSERT_FIELD_NATIVE8(_min, _max, _field9, _value9) | \ + EFX_INSERT_FIELD_NATIVE8(_min, _max, _field10, _value10)) + +#define EFX_POPULATE_OWORD64(_oword, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5, _field6, _value6, \ + _field7, _value7, _field8, _value8, _field9, _value9, \ + _field10, _value10) \ + do { \ + _NOTE(CONSTANTCONDITION) \ + (_oword).eo_u64[0] = EFX_INSERT_FIELDS64(0, 63, \ + _field1, _value1, _field2, _value2, \ + _field3, _value3, _field4, _value4, \ + _field5, _value5, _field6, _value6, \ + _field7, _value7, _field8, _value8, \ + _field9, _value9, _field10, _value10); \ + _NOTE(CONSTANTCONDITION) \ + (_oword).eo_u64[1] = EFX_INSERT_FIELDS64(64, 127, \ + _field1, _value1, _field2, _value2, \ + _field3, _value3, _field4, _value4, \ + _field5, _value5, _field6, _value6, \ + _field7, _value7, _field8, _value8, \ + _field9, _value9, _field10, _value10); \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#define EFX_POPULATE_OWORD32(_oword, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5, _field6, _value6, \ + _field7, _value7, _field8, _value8, _field9, _value9, \ + _field10, _value10) \ + do { \ + _NOTE(CONSTANTCONDITION) \ + (_oword).eo_u32[0] = EFX_INSERT_FIELDS32(0, 31, \ + _field1, _value1, _field2, _value2, \ + _field3, _value3, _field4, _value4, \ + _field5, _value5, _field6, _value6, \ + _field7, _value7, _field8, _value8, \ + _field9, _value9, _field10, _value10); \ + _NOTE(CONSTANTCONDITION) \ + (_oword).eo_u32[1] = EFX_INSERT_FIELDS32(32, 63, \ + _field1, _value1, _field2, _value2, \ + _field3, _value3, _field4, _value4, \ + _field5, _value5, _field6, _value6, \ + _field7, _value7, _field8, _value8, \ + _field9, _value9, _field10, _value10); \ + _NOTE(CONSTANTCONDITION) \ + (_oword).eo_u32[2] = EFX_INSERT_FIELDS32(64, 95, \ + _field1, _value1, _field2, _value2, \ + _field3, _value3, _field4, _value4, \ + _field5, _value5, _field6, _value6, \ + _field7, _value7, _field8, _value8, \ + _field9, _value9, _field10, _value10); \ + _NOTE(CONSTANTCONDITION) \ + (_oword).eo_u32[3] = EFX_INSERT_FIELDS32(96, 127, \ + _field1, _value1, _field2, _value2, \ + _field3, _value3, _field4, _value4, \ + _field5, _value5, _field6, _value6, \ + _field7, _value7, _field8, _value8, \ + _field9, _value9, _field10, _value10); \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#define EFX_POPULATE_QWORD64(_qword, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5, _field6, _value6, \ + _field7, _value7, _field8, _value8, _field9, _value9, \ + _field10, _value10) \ + do { \ + _NOTE(CONSTANTCONDITION) \ + (_qword).eq_u64[0] = EFX_INSERT_FIELDS64(0, 63, \ + _field1, _value1, _field2, _value2, \ + _field3, _value3, _field4, _value4, \ + _field5, _value5, _field6, _value6, \ + _field7, _value7, _field8, _value8, \ + _field9, _value9, _field10, _value10); \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#define EFX_POPULATE_QWORD32(_qword, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5, _field6, _value6, \ + _field7, _value7, _field8, _value8, _field9, _value9, \ + _field10, _value10) \ + do { \ + _NOTE(CONSTANTCONDITION) \ + (_qword).eq_u32[0] = EFX_INSERT_FIELDS32(0, 31, \ + _field1, _value1, _field2, _value2, \ + _field3, _value3, _field4, _value4, \ + _field5, _value5, _field6, _value6, \ + _field7, _value7, _field8, _value8, \ + _field9, _value9, _field10, _value10); \ + _NOTE(CONSTANTCONDITION) \ + (_qword).eq_u32[1] = EFX_INSERT_FIELDS32(32, 63, \ + _field1, _value1, _field2, _value2, \ + _field3, _value3, _field4, _value4, \ + _field5, _value5, _field6, _value6, \ + _field7, _value7, _field8, _value8, \ + _field9, _value9, _field10, _value10); \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#define EFX_POPULATE_DWORD(_dword, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5, _field6, _value6, \ + _field7, _value7, _field8, _value8, _field9, _value9, \ + _field10, _value10) \ + do { \ + _NOTE(CONSTANTCONDITION) \ + (_dword).ed_u32[0] = EFX_INSERT_FIELDS32(0, 31, \ + _field1, _value1, _field2, _value2, \ + _field3, _value3, _field4, _value4, \ + _field5, _value5, _field6, _value6, \ + _field7, _value7, _field8, _value8, \ + _field9, _value9, _field10, _value10); \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#define EFX_POPULATE_WORD(_word, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5, _field6, _value6, \ + _field7, _value7, _field8, _value8, _field9, _value9, \ + _field10, _value10) \ + do { \ + _NOTE(CONSTANTCONDITION) \ + (_word).ew_u16[0] = EFX_INSERT_FIELDS16(0, 15, \ + _field1, _value1, _field2, _value2, \ + _field3, _value3, _field4, _value4, \ + _field5, _value5, _field6, _value6, \ + _field7, _value7, _field8, _value8, \ + _field9, _value9, _field10, _value10); \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#define EFX_POPULATE_BYTE(_byte, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5, _field6, _value6, \ + _field7, _value7, _field8, _value8, _field9, _value9, \ + _field10, _value10) \ + do { \ + _NOTE(CONSTANTCONDITION) \ + (_byte).eb_u8[0] = EFX_INSERT_FIELDS8(0, 7, \ + _field1, _value1, _field2, _value2, \ + _field3, _value3, _field4, _value4, \ + _field5, _value5, _field6, _value6, \ + _field7, _value7, _field8, _value8, \ + _field9, _value9, _field10, _value10); \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +/* Populate an octword field with various numbers of arguments */ +#define EFX_POPULATE_OWORD_10 EFX_POPULATE_OWORD + +#define EFX_POPULATE_OWORD_9(_oword, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5, _field6, _value6, \ + _field7, _value7, _field8, _value8, _field9, _value9) \ + EFX_POPULATE_OWORD_10(_oword, EFX_DUMMY_FIELD, 0, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5, _field6, _value6, \ + _field7, _value7, _field8, _value8, _field9, _value9) + +#define EFX_POPULATE_OWORD_8(_oword, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5, _field6, _value6, \ + _field7, _value7, _field8, _value8) \ + EFX_POPULATE_OWORD_9(_oword, EFX_DUMMY_FIELD, 0, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5, _field6, _value6, \ + _field7, _value7, _field8, _value8) + +#define EFX_POPULATE_OWORD_7(_oword, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5, _field6, _value6, \ + _field7, _value7) \ + EFX_POPULATE_OWORD_8(_oword, EFX_DUMMY_FIELD, 0, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5, _field6, _value6, \ + _field7, _value7) + +#define EFX_POPULATE_OWORD_6(_oword, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5, _field6, _value6) \ + EFX_POPULATE_OWORD_7(_oword, EFX_DUMMY_FIELD, 0, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5, _field6, _value6) + +#define EFX_POPULATE_OWORD_5(_oword, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5) \ + EFX_POPULATE_OWORD_6(_oword, EFX_DUMMY_FIELD, 0, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5) + +#define EFX_POPULATE_OWORD_4(_oword, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4) \ + EFX_POPULATE_OWORD_5(_oword, EFX_DUMMY_FIELD, 0, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4) + +#define EFX_POPULATE_OWORD_3(_oword, \ + _field1, _value1, _field2, _value2, _field3, _value3) \ + EFX_POPULATE_OWORD_4(_oword, EFX_DUMMY_FIELD, 0, \ + _field1, _value1, _field2, _value2, _field3, _value3) + +#define EFX_POPULATE_OWORD_2(_oword, \ + _field1, _value1, _field2, _value2) \ + EFX_POPULATE_OWORD_3(_oword, EFX_DUMMY_FIELD, 0, \ + _field1, _value1, _field2, _value2) + +#define EFX_POPULATE_OWORD_1(_oword, \ + _field1, _value1) \ + EFX_POPULATE_OWORD_2(_oword, EFX_DUMMY_FIELD, 0, \ + _field1, _value1) + +#define EFX_ZERO_OWORD(_oword) \ + EFX_POPULATE_OWORD_1(_oword, EFX_DUMMY_FIELD, 0) + +#define EFX_SET_OWORD(_oword) \ + EFX_POPULATE_OWORD_4(_oword, \ + EFX_DWORD_0, 0xffffffff, EFX_DWORD_1, 0xffffffff, \ + EFX_DWORD_2, 0xffffffff, EFX_DWORD_3, 0xffffffff) + +/* Populate a quadword field with various numbers of arguments */ +#define EFX_POPULATE_QWORD_10 EFX_POPULATE_QWORD + +#define EFX_POPULATE_QWORD_9(_qword, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5, _field6, _value6, \ + _field7, _value7, _field8, _value8, _field9, _value9) \ + EFX_POPULATE_QWORD_10(_qword, EFX_DUMMY_FIELD, 0, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5, _field6, _value6, \ + _field7, _value7, _field8, _value8, _field9, _value9) + +#define EFX_POPULATE_QWORD_8(_qword, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5, _field6, _value6, \ + _field7, _value7, _field8, _value8) \ + EFX_POPULATE_QWORD_9(_qword, EFX_DUMMY_FIELD, 0, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5, _field6, _value6, \ + _field7, _value7, _field8, _value8) + +#define EFX_POPULATE_QWORD_7(_qword, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5, _field6, _value6, \ + _field7, _value7) \ + EFX_POPULATE_QWORD_8(_qword, EFX_DUMMY_FIELD, 0, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5, _field6, _value6, \ + _field7, _value7) + +#define EFX_POPULATE_QWORD_6(_qword, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5, _field6, _value6) \ + EFX_POPULATE_QWORD_7(_qword, EFX_DUMMY_FIELD, 0, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5, _field6, _value6) + +#define EFX_POPULATE_QWORD_5(_qword, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5) \ + EFX_POPULATE_QWORD_6(_qword, EFX_DUMMY_FIELD, 0, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5) + +#define EFX_POPULATE_QWORD_4(_qword, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4) \ + EFX_POPULATE_QWORD_5(_qword, EFX_DUMMY_FIELD, 0, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4) + +#define EFX_POPULATE_QWORD_3(_qword, \ + _field1, _value1, _field2, _value2, _field3, _value3) \ + EFX_POPULATE_QWORD_4(_qword, EFX_DUMMY_FIELD, 0, \ + _field1, _value1, _field2, _value2, _field3, _value3) + +#define EFX_POPULATE_QWORD_2(_qword, \ + _field1, _value1, _field2, _value2) \ + EFX_POPULATE_QWORD_3(_qword, EFX_DUMMY_FIELD, 0, \ + _field1, _value1, _field2, _value2) + +#define EFX_POPULATE_QWORD_1(_qword, \ + _field1, _value1) \ + EFX_POPULATE_QWORD_2(_qword, EFX_DUMMY_FIELD, 0, \ + _field1, _value1) + +#define EFX_ZERO_QWORD(_qword) \ + EFX_POPULATE_QWORD_1(_qword, EFX_DUMMY_FIELD, 0) + +#define EFX_SET_QWORD(_qword) \ + EFX_POPULATE_QWORD_2(_qword, \ + EFX_DWORD_0, 0xffffffff, EFX_DWORD_1, 0xffffffff) + +/* Populate a dword field with various numbers of arguments */ +#define EFX_POPULATE_DWORD_10 EFX_POPULATE_DWORD + +#define EFX_POPULATE_DWORD_9(_dword, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5, _field6, _value6, \ + _field7, _value7, _field8, _value8, _field9, _value9) \ + EFX_POPULATE_DWORD_10(_dword, EFX_DUMMY_FIELD, 0, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5, _field6, _value6, \ + _field7, _value7, _field8, _value8, _field9, _value9) + +#define EFX_POPULATE_DWORD_8(_dword, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5, _field6, _value6, \ + _field7, _value7, _field8, _value8) \ + EFX_POPULATE_DWORD_9(_dword, EFX_DUMMY_FIELD, 0, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5, _field6, _value6, \ + _field7, _value7, _field8, _value8) + +#define EFX_POPULATE_DWORD_7(_dword, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5, _field6, _value6, \ + _field7, _value7) \ + EFX_POPULATE_DWORD_8(_dword, EFX_DUMMY_FIELD, 0, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5, _field6, _value6, \ + _field7, _value7) + +#define EFX_POPULATE_DWORD_6(_dword, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5, _field6, _value6) \ + EFX_POPULATE_DWORD_7(_dword, EFX_DUMMY_FIELD, 0, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5, _field6, _value6) + +#define EFX_POPULATE_DWORD_5(_dword, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5) \ + EFX_POPULATE_DWORD_6(_dword, EFX_DUMMY_FIELD, 0, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5) + +#define EFX_POPULATE_DWORD_4(_dword, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4) \ + EFX_POPULATE_DWORD_5(_dword, EFX_DUMMY_FIELD, 0, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4) + +#define EFX_POPULATE_DWORD_3(_dword, \ + _field1, _value1, _field2, _value2, _field3, _value3) \ + EFX_POPULATE_DWORD_4(_dword, EFX_DUMMY_FIELD, 0, \ + _field1, _value1, _field2, _value2, _field3, _value3) + +#define EFX_POPULATE_DWORD_2(_dword, \ + _field1, _value1, _field2, _value2) \ + EFX_POPULATE_DWORD_3(_dword, EFX_DUMMY_FIELD, 0, \ + _field1, _value1, _field2, _value2) + +#define EFX_POPULATE_DWORD_1(_dword, \ + _field1, _value1) \ + EFX_POPULATE_DWORD_2(_dword, EFX_DUMMY_FIELD, 0, \ + _field1, _value1) + +#define EFX_ZERO_DWORD(_dword) \ + EFX_POPULATE_DWORD_1(_dword, EFX_DUMMY_FIELD, 0) + +#define EFX_SET_DWORD(_dword) \ + EFX_POPULATE_DWORD_1(_dword, \ + EFX_DWORD_0, 0xffffffff) + +/* Populate a word field with various numbers of arguments */ +#define EFX_POPULATE_WORD_10 EFX_POPULATE_WORD + +#define EFX_POPULATE_WORD_9(_word, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5, _field6, _value6, \ + _field7, _value7, _field8, _value8, _field9, _value9) \ + EFX_POPULATE_WORD_10(_word, EFX_DUMMY_FIELD, 0, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5, _field6, _value6, \ + _field7, _value7, _field8, _value8, _field9, _value9) + +#define EFX_POPULATE_WORD_8(_word, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5, _field6, _value6, \ + _field7, _value7, _field8, _value8) \ + EFX_POPULATE_WORD_9(_word, EFX_DUMMY_FIELD, 0, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5, _field6, _value6, \ + _field7, _value7, _field8, _value8) + +#define EFX_POPULATE_WORD_7(_word, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5, _field6, _value6, \ + _field7, _value7) \ + EFX_POPULATE_WORD_8(_word, EFX_DUMMY_FIELD, 0, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5, _field6, _value6, \ + _field7, _value7) + +#define EFX_POPULATE_WORD_6(_word, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5, _field6, _value6) \ + EFX_POPULATE_WORD_7(_word, EFX_DUMMY_FIELD, 0, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5, _field6, _value6) + +#define EFX_POPULATE_WORD_5(_word, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5) \ + EFX_POPULATE_WORD_6(_word, EFX_DUMMY_FIELD, 0, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5) + +#define EFX_POPULATE_WORD_4(_word, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4) \ + EFX_POPULATE_WORD_5(_word, EFX_DUMMY_FIELD, 0, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4) + +#define EFX_POPULATE_WORD_3(_word, \ + _field1, _value1, _field2, _value2, _field3, _value3) \ + EFX_POPULATE_WORD_4(_word, EFX_DUMMY_FIELD, 0, \ + _field1, _value1, _field2, _value2, _field3, _value3) + +#define EFX_POPULATE_WORD_2(_word, \ + _field1, _value1, _field2, _value2) \ + EFX_POPULATE_WORD_3(_word, EFX_DUMMY_FIELD, 0, \ + _field1, _value1, _field2, _value2) + +#define EFX_POPULATE_WORD_1(_word, \ + _field1, _value1) \ + EFX_POPULATE_WORD_2(_word, EFX_DUMMY_FIELD, 0, \ + _field1, _value1) + +#define EFX_ZERO_WORD(_word) \ + EFX_POPULATE_WORD_1(_word, EFX_DUMMY_FIELD, 0) + +#define EFX_SET_WORD(_word) \ + EFX_POPULATE_WORD_1(_word, \ + EFX_WORD_0, 0xffff) + +/* Populate a byte field with various numbers of arguments */ +#define EFX_POPULATE_BYTE_10 EFX_POPULATE_BYTE + +#define EFX_POPULATE_BYTE_9(_byte, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5, _field6, _value6, \ + _field7, _value7, _field8, _value8, _field9, _value9) \ + EFX_POPULATE_BYTE_10(_byte, EFX_DUMMY_FIELD, 0, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5, _field6, _value6, \ + _field7, _value7, _field8, _value8, _field9, _value9) + +#define EFX_POPULATE_BYTE_8(_byte, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5, _field6, _value6, \ + _field7, _value7, _field8, _value8) \ + EFX_POPULATE_BYTE_9(_byte, EFX_DUMMY_FIELD, 0, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5, _field6, _value6, \ + _field7, _value7, _field8, _value8) + +#define EFX_POPULATE_BYTE_7(_byte, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5, _field6, _value6, \ + _field7, _value7) \ + EFX_POPULATE_BYTE_8(_byte, EFX_DUMMY_FIELD, 0, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5, _field6, _value6, \ + _field7, _value7) + +#define EFX_POPULATE_BYTE_6(_byte, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5, _field6, _value6) \ + EFX_POPULATE_BYTE_7(_byte, EFX_DUMMY_FIELD, 0, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5, _field6, _value6) + +#define EFX_POPULATE_BYTE_5(_byte, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5) \ + EFX_POPULATE_BYTE_6(_byte, EFX_DUMMY_FIELD, 0, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4, _field5, _value5) + +#define EFX_POPULATE_BYTE_4(_byte, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4) \ + EFX_POPULATE_BYTE_5(_byte, EFX_DUMMY_FIELD, 0, \ + _field1, _value1, _field2, _value2, _field3, _value3, \ + _field4, _value4) + +#define EFX_POPULATE_BYTE_3(_byte, \ + _field1, _value1, _field2, _value2, _field3, _value3) \ + EFX_POPULATE_BYTE_4(_byte, EFX_DUMMY_FIELD, 0, \ + _field1, _value1, _field2, _value2, _field3, _value3) + +#define EFX_POPULATE_BYTE_2(_byte, \ + _field1, _value1, _field2, _value2) \ + EFX_POPULATE_BYTE_3(_byte, EFX_DUMMY_FIELD, 0, \ + _field1, _value1, _field2, _value2) + +#define EFX_POPULATE_BYTE_1(_byte, \ + _field1, _value1) \ + EFX_POPULATE_BYTE_2(_byte, EFX_DUMMY_FIELD, 0, \ + _field1, _value1) + +#define EFX_ZERO_BYTE(_byte) \ + EFX_POPULATE_BYTE_1(_byte, EFX_DUMMY_FIELD, 0) + +#define EFX_SET_BYTE(_byte) \ + EFX_POPULATE_BYTE_1(_byte, \ + EFX_BYTE_0, 0xff) + +/* + * Modify a named field within an already-populated structure. Used + * for read-modify-write operations. + */ + +#define EFX_INSERT_FIELD64(_min, _max, _field, _value) \ + __CPU_TO_LE_64(EFX_INSERT_FIELD_NATIVE64(_min, _max, _field, _value)) + +#define EFX_INSERT_FIELD32(_min, _max, _field, _value) \ + __CPU_TO_LE_32(EFX_INSERT_FIELD_NATIVE32(_min, _max, _field, _value)) + +#define EFX_INSERT_FIELD16(_min, _max, _field, _value) \ + __CPU_TO_LE_16(EFX_INSERT_FIELD_NATIVE16(_min, _max, _field, _value)) + +#define EFX_INSERT_FIELD8(_min, _max, _field, _value) \ + __NATIVE_8(EFX_INSERT_FIELD_NATIVE8(_min, _max, _field, _value)) + +#define EFX_INPLACE_MASK64(_min, _max, _field) \ + EFX_INSERT_FIELD64(_min, _max, _field, EFX_MASK64(_field)) + +#define EFX_INPLACE_MASK32(_min, _max, _field) \ + EFX_INSERT_FIELD32(_min, _max, _field, EFX_MASK32(_field)) + +#define EFX_INPLACE_MASK16(_min, _max, _field) \ + EFX_INSERT_FIELD16(_min, _max, _field, EFX_MASK16(_field)) + +#define EFX_INPLACE_MASK8(_min, _max, _field) \ + EFX_INSERT_FIELD8(_min, _max, _field, EFX_MASK8(_field)) + +#define EFX_SET_OWORD_FIELD64(_oword, _field, _value) \ + do { \ + _NOTE(CONSTANTCONDITION) \ + (_oword).eo_u64[0] = (((_oword).eo_u64[0] & \ + ~EFX_INPLACE_MASK64(0, 63, _field)) | \ + EFX_INSERT_FIELD64(0, 63, _field, _value)); \ + _NOTE(CONSTANTCONDITION) \ + (_oword).eo_u64[1] = (((_oword).eo_u64[1] & \ + ~EFX_INPLACE_MASK64(64, 127, _field)) | \ + EFX_INSERT_FIELD64(64, 127, _field, _value)); \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#define EFX_SET_OWORD_FIELD32(_oword, _field, _value) \ + do { \ + _NOTE(CONSTANTCONDITION) \ + (_oword).eo_u32[0] = (((_oword).eo_u32[0] & \ + ~EFX_INPLACE_MASK32(0, 31, _field)) | \ + EFX_INSERT_FIELD32(0, 31, _field, _value)); \ + _NOTE(CONSTANTCONDITION) \ + (_oword).eo_u32[1] = (((_oword).eo_u32[1] & \ + ~EFX_INPLACE_MASK32(32, 63, _field)) | \ + EFX_INSERT_FIELD32(32, 63, _field, _value)); \ + _NOTE(CONSTANTCONDITION) \ + (_oword).eo_u32[2] = (((_oword).eo_u32[2] & \ + ~EFX_INPLACE_MASK32(64, 95, _field)) | \ + EFX_INSERT_FIELD32(64, 95, _field, _value)); \ + _NOTE(CONSTANTCONDITION) \ + (_oword).eo_u32[3] = (((_oword).eo_u32[3] & \ + ~EFX_INPLACE_MASK32(96, 127, _field)) | \ + EFX_INSERT_FIELD32(96, 127, _field, _value)); \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#define EFX_SET_QWORD_FIELD64(_qword, _field, _value) \ + do { \ + _NOTE(CONSTANTCONDITION) \ + (_qword).eq_u64[0] = (((_qword).eq_u64[0] & \ + ~EFX_INPLACE_MASK64(0, 63, _field)) | \ + EFX_INSERT_FIELD64(0, 63, _field, _value)); \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#define EFX_SET_QWORD_FIELD32(_qword, _field, _value) \ + do { \ + _NOTE(CONSTANTCONDITION) \ + (_qword).eq_u32[0] = (((_qword).eq_u32[0] & \ + ~EFX_INPLACE_MASK32(0, 31, _field)) | \ + EFX_INSERT_FIELD32(0, 31, _field, _value)); \ + _NOTE(CONSTANTCONDITION) \ + (_qword).eq_u32[1] = (((_qword).eq_u32[1] & \ + ~EFX_INPLACE_MASK32(32, 63, _field)) | \ + EFX_INSERT_FIELD32(32, 63, _field, _value)); \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#define EFX_SET_DWORD_FIELD(_dword, _field, _value) \ + do { \ + _NOTE(CONSTANTCONDITION) \ + (_dword).ed_u32[0] = (((_dword).ed_u32[0] & \ + ~EFX_INPLACE_MASK32(0, 31, _field)) | \ + EFX_INSERT_FIELD32(0, 31, _field, _value)); \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#define EFX_SET_WORD_FIELD(_word, _field, _value) \ + do { \ + _NOTE(CONSTANTCONDITION) \ + (_word).ew_u16[0] = (((_word).ew_u16[0] & \ + ~EFX_INPLACE_MASK16(0, 15, _field)) | \ + EFX_INSERT_FIELD16(0, 15, _field, _value)); \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#define EFX_SET_BYTE_FIELD(_byte, _field, _value) \ + do { \ + _NOTE(CONSTANTCONDITION) \ + (_byte).eb_u8[0] = (((_byte).eb_u8[0] & \ + ~EFX_INPLACE_MASK8(0, 7, _field)) | \ + EFX_INSERT_FIELD8(0, 7, _field, _value)); \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +/* + * Set or clear a numbered bit within an octword. + */ + +#define EFX_SHIFT64(_bit, _base) \ + (((_bit) >= (_base) && (_bit) < (_base) + 64) ? \ + ((uint64_t)1 << ((_bit) - (_base))) : \ + 0U) + +#define EFX_SHIFT32(_bit, _base) \ + (((_bit) >= (_base) && (_bit) < (_base) + 32) ? \ + ((uint32_t)1 << ((_bit) - (_base))) : \ + 0U) + +#define EFX_SHIFT16(_bit, _base) \ + (((_bit) >= (_base) && (_bit) < (_base) + 16) ? \ + (uint16_t)(1 << ((_bit) - (_base))) : \ + 0U) + +#define EFX_SHIFT8(_bit, _base) \ + (((_bit) >= (_base) && (_bit) < (_base) + 8) ? \ + (uint8_t)(1 << ((_bit) - (_base))) : \ + 0U) + +#define EFX_SET_OWORD_BIT64(_oword, _bit) \ + do { \ + _NOTE(CONSTANTCONDITION) \ + (_oword).eo_u64[0] |= \ + __CPU_TO_LE_64(EFX_SHIFT64(_bit, FIX_LINT(0))); \ + (_oword).eo_u64[1] |= \ + __CPU_TO_LE_64(EFX_SHIFT64(_bit, FIX_LINT(64))); \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#define EFX_SET_OWORD_BIT32(_oword, _bit) \ + do { \ + _NOTE(CONSTANTCONDITION) \ + (_oword).eo_u32[0] |= \ + __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(0))); \ + (_oword).eo_u32[1] |= \ + __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(32))); \ + (_oword).eo_u32[2] |= \ + __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(64))); \ + (_oword).eo_u32[3] |= \ + __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(96))); \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#define EFX_CLEAR_OWORD_BIT64(_oword, _bit) \ + do { \ + _NOTE(CONSTANTCONDITION) \ + (_oword).eo_u64[0] &= \ + __CPU_TO_LE_64(~EFX_SHIFT64(_bit, FIX_LINT(0))); \ + (_oword).eo_u64[1] &= \ + __CPU_TO_LE_64(~EFX_SHIFT64(_bit, FIX_LINT(64))); \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#define EFX_CLEAR_OWORD_BIT32(_oword, _bit) \ + do { \ + _NOTE(CONSTANTCONDITION) \ + (_oword).eo_u32[0] &= \ + __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(0))); \ + (_oword).eo_u32[1] &= \ + __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(32))); \ + (_oword).eo_u32[2] &= \ + __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(64))); \ + (_oword).eo_u32[3] &= \ + __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(96))); \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#define EFX_TEST_OWORD_BIT64(_oword, _bit) \ + (((_oword).eo_u64[0] & \ + __CPU_TO_LE_64(EFX_SHIFT64(_bit, FIX_LINT(0)))) || \ + ((_oword).eo_u64[1] & \ + __CPU_TO_LE_64(EFX_SHIFT64(_bit, FIX_LINT(64))))) + +#define EFX_TEST_OWORD_BIT32(_oword, _bit) \ + (((_oword).eo_u32[0] & \ + __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(0)))) || \ + ((_oword).eo_u32[1] & \ + __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(32)))) || \ + ((_oword).eo_u32[2] & \ + __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(64)))) || \ + ((_oword).eo_u32[3] & \ + __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(96))))) + + +#define EFX_SET_QWORD_BIT64(_qword, _bit) \ + do { \ + _NOTE(CONSTANTCONDITION) \ + (_qword).eq_u64[0] |= \ + __CPU_TO_LE_64(EFX_SHIFT64(_bit, FIX_LINT(0))); \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#define EFX_SET_QWORD_BIT32(_qword, _bit) \ + do { \ + _NOTE(CONSTANTCONDITION) \ + (_qword).eq_u32[0] |= \ + __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(0))); \ + (_qword).eq_u32[1] |= \ + __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(32))); \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#define EFX_CLEAR_QWORD_BIT64(_qword, _bit) \ + do { \ + _NOTE(CONSTANTCONDITION) \ + (_qword).eq_u64[0] &= \ + __CPU_TO_LE_64(~EFX_SHIFT64(_bit, FIX_LINT(0))); \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#define EFX_CLEAR_QWORD_BIT32(_qword, _bit) \ + do { \ + _NOTE(CONSTANTCONDITION) \ + (_qword).eq_u32[0] &= \ + __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(0))); \ + (_qword).eq_u32[1] &= \ + __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(32))); \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#define EFX_TEST_QWORD_BIT64(_qword, _bit) \ + (((_qword).eq_u64[0] & \ + __CPU_TO_LE_64(EFX_SHIFT64(_bit, FIX_LINT(0)))) != 0) + +#define EFX_TEST_QWORD_BIT32(_qword, _bit) \ + (((_qword).eq_u32[0] & \ + __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(0)))) || \ + ((_qword).eq_u32[1] & \ + __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(32))))) + + +#define EFX_SET_DWORD_BIT(_dword, _bit) \ + do { \ + (_dword).ed_u32[0] |= \ + __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(0))); \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#define EFX_CLEAR_DWORD_BIT(_dword, _bit) \ + do { \ + (_dword).ed_u32[0] &= \ + __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(0))); \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#define EFX_TEST_DWORD_BIT(_dword, _bit) \ + (((_dword).ed_u32[0] & \ + __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(0)))) != 0) + + +#define EFX_SET_WORD_BIT(_word, _bit) \ + do { \ + (_word).ew_u16[0] |= \ + __CPU_TO_LE_16(EFX_SHIFT16(_bit, FIX_LINT(0))); \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#define EFX_CLEAR_WORD_BIT(_word, _bit) \ + do { \ + (_word).ew_u32[0] &= \ + __CPU_TO_LE_16(~EFX_SHIFT16(_bit, FIX_LINT(0))); \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#define EFX_TEST_WORD_BIT(_word, _bit) \ + (((_word).ew_u16[0] & \ + __CPU_TO_LE_16(EFX_SHIFT16(_bit, FIX_LINT(0)))) != 0) + + +#define EFX_SET_BYTE_BIT(_byte, _bit) \ + do { \ + (_byte).eb_u8[0] |= \ + __NATIVE_8(EFX_SHIFT8(_bit, FIX_LINT(0))); \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#define EFX_CLEAR_BYTE_BIT(_byte, _bit) \ + do { \ + (_byte).eb_u8[0] &= \ + __NATIVE_8(~EFX_SHIFT8(_bit, FIX_LINT(0))); \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#define EFX_TEST_BYTE_BIT(_byte, _bit) \ + (((_byte).eb_u8[0] & \ + __NATIVE_8(EFX_SHIFT8(_bit, FIX_LINT(0)))) != 0) + + +#define EFX_OR_OWORD64(_oword1, _oword2) \ + do { \ + (_oword1).eo_u64[0] |= (_oword2).eo_u64[0]; \ + (_oword1).eo_u64[1] |= (_oword2).eo_u64[1]; \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#define EFX_OR_OWORD32(_oword1, _oword2) \ + do { \ + (_oword1).eo_u32[0] |= (_oword2).eo_u32[0]; \ + (_oword1).eo_u32[1] |= (_oword2).eo_u32[1]; \ + (_oword1).eo_u32[2] |= (_oword2).eo_u32[2]; \ + (_oword1).eo_u32[3] |= (_oword2).eo_u32[3]; \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#define EFX_AND_OWORD64(_oword1, _oword2) \ + do { \ + (_oword1).eo_u64[0] &= (_oword2).eo_u64[0]; \ + (_oword1).eo_u64[1] &= (_oword2).eo_u64[1]; \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#define EFX_AND_OWORD32(_oword1, _oword2) \ + do { \ + (_oword1).eo_u32[0] &= (_oword2).eo_u32[0]; \ + (_oword1).eo_u32[1] &= (_oword2).eo_u32[1]; \ + (_oword1).eo_u32[2] &= (_oword2).eo_u32[2]; \ + (_oword1).eo_u32[3] &= (_oword2).eo_u32[3]; \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#define EFX_OR_QWORD64(_qword1, _qword2) \ + do { \ + (_qword1).eq_u64[0] |= (_qword2).eq_u64[0]; \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#define EFX_OR_QWORD32(_qword1, _qword2) \ + do { \ + (_qword1).eq_u32[0] |= (_qword2).eq_u32[0]; \ + (_qword1).eq_u32[1] |= (_qword2).eq_u32[1]; \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#define EFX_AND_QWORD64(_qword1, _qword2) \ + do { \ + (_qword1).eq_u64[0] &= (_qword2).eq_u64[0]; \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#define EFX_AND_QWORD32(_qword1, _qword2) \ + do { \ + (_qword1).eq_u32[0] &= (_qword2).eq_u32[0]; \ + (_qword1).eq_u32[1] &= (_qword2).eq_u32[1]; \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#define EFX_OR_DWORD(_dword1, _dword2) \ + do { \ + (_dword1).ed_u32[0] |= (_dword2).ed_u32[0]; \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#define EFX_AND_DWORD(_dword1, _dword2) \ + do { \ + (_dword1).ed_u32[0] &= (_dword2).ed_u32[0]; \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#define EFX_OR_WORD(_word1, _word2) \ + do { \ + (_word1).ew_u16[0] |= (_word2).ew_u16[0]; \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#define EFX_AND_WORD(_word1, _word2) \ + do { \ + (_word1).ew_u16[0] &= (_word2).ew_u16[0]; \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#define EFX_OR_BYTE(_byte1, _byte2) \ + do { \ + (_byte1).eb_u8[0] |= (_byte2).eb_u8[0]; \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#define EFX_AND_BYTE(_byte1, _byte2) \ + do { \ + (_byte1).eb_u8[0] &= (_byte2).eb_u8[0]; \ + _NOTE(CONSTANTCONDITION) \ + } while (B_FALSE) + +#if EFSYS_USE_UINT64 +#define EFX_OWORD_FIELD EFX_OWORD_FIELD64 +#define EFX_QWORD_FIELD EFX_QWORD_FIELD64 +#define EFX_OWORD_IS_EQUAL EFX_OWORD_IS_EQUAL64 +#define EFX_QWORD_IS_EQUAL EFX_QWORD_IS_EQUAL64 +#define EFX_OWORD_IS_ZERO EFX_OWORD_IS_ZERO64 +#define EFX_QWORD_IS_ZERO EFX_QWORD_IS_ZERO64 +#define EFX_OWORD_IS_SET EFX_OWORD_IS_SET64 +#define EFX_QWORD_IS_SET EFX_QWORD_IS_SET64 +#define EFX_POPULATE_OWORD EFX_POPULATE_OWORD64 +#define EFX_POPULATE_QWORD EFX_POPULATE_QWORD64 +#define EFX_SET_OWORD_FIELD EFX_SET_OWORD_FIELD64 +#define EFX_SET_QWORD_FIELD EFX_SET_QWORD_FIELD64 +#define EFX_SET_OWORD_BIT EFX_SET_OWORD_BIT64 +#define EFX_CLEAR_OWORD_BIT EFX_CLEAR_OWORD_BIT64 +#define EFX_TEST_OWORD_BIT EFX_TEST_OWORD_BIT64 +#define EFX_SET_QWORD_BIT EFX_SET_QWORD_BIT64 +#define EFX_CLEAR_QWORD_BIT EFX_CLEAR_QWORD_BIT64 +#define EFX_TEST_QWORD_BIT EFX_TEST_QWORD_BIT64 +#define EFX_OR_OWORD EFX_OR_OWORD64 +#define EFX_AND_OWORD EFX_AND_OWORD64 +#define EFX_OR_QWORD EFX_OR_QWORD64 +#define EFX_AND_QWORD EFX_AND_QWORD64 +#else +#define EFX_OWORD_FIELD EFX_OWORD_FIELD32 +#define EFX_QWORD_FIELD EFX_QWORD_FIELD32 +#define EFX_OWORD_IS_EQUAL EFX_OWORD_IS_EQUAL32 +#define EFX_QWORD_IS_EQUAL EFX_QWORD_IS_EQUAL32 +#define EFX_OWORD_IS_ZERO EFX_OWORD_IS_ZERO32 +#define EFX_QWORD_IS_ZERO EFX_QWORD_IS_ZERO32 +#define EFX_OWORD_IS_SET EFX_OWORD_IS_SET32 +#define EFX_QWORD_IS_SET EFX_QWORD_IS_SET32 +#define EFX_POPULATE_OWORD EFX_POPULATE_OWORD32 +#define EFX_POPULATE_QWORD EFX_POPULATE_QWORD32 +#define EFX_SET_OWORD_FIELD EFX_SET_OWORD_FIELD32 +#define EFX_SET_QWORD_FIELD EFX_SET_QWORD_FIELD32 +#define EFX_SET_OWORD_BIT EFX_SET_OWORD_BIT32 +#define EFX_CLEAR_OWORD_BIT EFX_CLEAR_OWORD_BIT32 +#define EFX_TEST_OWORD_BIT EFX_TEST_OWORD_BIT32 +#define EFX_SET_QWORD_BIT EFX_SET_QWORD_BIT32 +#define EFX_CLEAR_QWORD_BIT EFX_CLEAR_QWORD_BIT32 +#define EFX_TEST_QWORD_BIT EFX_TEST_QWORD_BIT32 +#define EFX_OR_OWORD EFX_OR_OWORD32 +#define EFX_AND_OWORD EFX_AND_OWORD32 +#define EFX_OR_QWORD EFX_OR_QWORD32 +#define EFX_AND_QWORD EFX_AND_QWORD32 +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* _SYS_EFX_TYPES_H */ diff --git a/dpdk/drivers/net/sfc/base/efx_vpd.c b/dpdk/drivers/net/sfc/base/efx_vpd.c new file mode 100644 index 00000000..1e47df2c --- /dev/null +++ b/dpdk/drivers/net/sfc/base/efx_vpd.c @@ -0,0 +1,1016 @@ +/* + * Copyright (c) 2009-2016 Solarflare Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are + * those of the authors and should not be interpreted as representing official + * policies, either expressed or implied, of the FreeBSD Project. + */ + +#include "efx.h" +#include "efx_impl.h" + +#if EFSYS_OPT_VPD + +#define TAG_TYPE_LBN 7 +#define TAG_TYPE_WIDTH 1 +#define TAG_TYPE_LARGE_ITEM_DECODE 1 +#define TAG_TYPE_SMALL_ITEM_DECODE 0 + +#define TAG_SMALL_ITEM_NAME_LBN 3 +#define TAG_SMALL_ITEM_NAME_WIDTH 4 +#define TAG_SMALL_ITEM_SIZE_LBN 0 +#define TAG_SMALL_ITEM_SIZE_WIDTH 3 + +#define TAG_LARGE_ITEM_NAME_LBN 0 +#define TAG_LARGE_ITEM_NAME_WIDTH 7 + +#define TAG_NAME_END_DECODE 0x0f +#define TAG_NAME_ID_STRING_DECODE 0x02 +#define TAG_NAME_VPD_R_DECODE 0x10 +#define TAG_NAME_VPD_W_DECODE 0x11 + +#if EFSYS_OPT_SIENA + +static const efx_vpd_ops_t __efx_vpd_siena_ops = { + siena_vpd_init, /* evpdo_init */ + siena_vpd_size, /* evpdo_size */ + siena_vpd_read, /* evpdo_read */ + siena_vpd_verify, /* evpdo_verify */ + siena_vpd_reinit, /* evpdo_reinit */ + siena_vpd_get, /* evpdo_get */ + siena_vpd_set, /* evpdo_set */ + siena_vpd_next, /* evpdo_next */ + siena_vpd_write, /* evpdo_write */ + siena_vpd_fini, /* evpdo_fini */ +}; + +#endif /* EFSYS_OPT_SIENA */ + +#if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD + +static const efx_vpd_ops_t __efx_vpd_ef10_ops = { + ef10_vpd_init, /* evpdo_init */ + ef10_vpd_size, /* evpdo_size */ + ef10_vpd_read, /* evpdo_read */ + ef10_vpd_verify, /* evpdo_verify */ + ef10_vpd_reinit, /* evpdo_reinit */ + ef10_vpd_get, /* evpdo_get */ + ef10_vpd_set, /* evpdo_set */ + ef10_vpd_next, /* evpdo_next */ + ef10_vpd_write, /* evpdo_write */ + ef10_vpd_fini, /* evpdo_fini */ +}; + +#endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */ + + __checkReturn efx_rc_t +efx_vpd_init( + __in efx_nic_t *enp) +{ + const efx_vpd_ops_t *evpdop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE); + EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_VPD)); + + switch (enp->en_family) { +#if EFSYS_OPT_SIENA + case EFX_FAMILY_SIENA: + evpdop = &__efx_vpd_siena_ops; + break; +#endif /* EFSYS_OPT_SIENA */ + +#if EFSYS_OPT_HUNTINGTON + case EFX_FAMILY_HUNTINGTON: + evpdop = &__efx_vpd_ef10_ops; + break; +#endif /* EFSYS_OPT_HUNTINGTON */ + +#if EFSYS_OPT_MEDFORD + case EFX_FAMILY_MEDFORD: + evpdop = &__efx_vpd_ef10_ops; + break; +#endif /* EFSYS_OPT_MEDFORD */ + + default: + EFSYS_ASSERT(0); + rc = ENOTSUP; + goto fail1; + } + + if (evpdop->evpdo_init != NULL) { + if ((rc = evpdop->evpdo_init(enp)) != 0) + goto fail2; + } + + enp->en_evpdop = evpdop; + enp->en_mod_flags |= EFX_MOD_VPD; + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_vpd_size( + __in efx_nic_t *enp, + __out size_t *sizep) +{ + const efx_vpd_ops_t *evpdop = enp->en_evpdop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_VPD); + + if ((rc = evpdop->evpdo_size(enp, sizep)) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_vpd_read( + __in efx_nic_t *enp, + __out_bcount(size) caddr_t data, + __in size_t size) +{ + const efx_vpd_ops_t *evpdop = enp->en_evpdop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_VPD); + + if ((rc = evpdop->evpdo_read(enp, data, size)) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_vpd_verify( + __in efx_nic_t *enp, + __in_bcount(size) caddr_t data, + __in size_t size) +{ + const efx_vpd_ops_t *evpdop = enp->en_evpdop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_VPD); + + if ((rc = evpdop->evpdo_verify(enp, data, size)) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_vpd_reinit( + __in efx_nic_t *enp, + __in_bcount(size) caddr_t data, + __in size_t size) +{ + const efx_vpd_ops_t *evpdop = enp->en_evpdop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_VPD); + + if (evpdop->evpdo_reinit == NULL) { + rc = ENOTSUP; + goto fail1; + } + + if ((rc = evpdop->evpdo_reinit(enp, data, size)) != 0) + goto fail2; + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_vpd_get( + __in efx_nic_t *enp, + __in_bcount(size) caddr_t data, + __in size_t size, + __inout efx_vpd_value_t *evvp) +{ + const efx_vpd_ops_t *evpdop = enp->en_evpdop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_VPD); + + if ((rc = evpdop->evpdo_get(enp, data, size, evvp)) != 0) { + if (rc == ENOENT) + return (rc); + + goto fail1; + } + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_vpd_set( + __in efx_nic_t *enp, + __inout_bcount(size) caddr_t data, + __in size_t size, + __in efx_vpd_value_t *evvp) +{ + const efx_vpd_ops_t *evpdop = enp->en_evpdop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_VPD); + + if ((rc = evpdop->evpdo_set(enp, data, size, evvp)) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_vpd_next( + __in efx_nic_t *enp, + __inout_bcount(size) caddr_t data, + __in size_t size, + __out efx_vpd_value_t *evvp, + __inout unsigned int *contp) +{ + const efx_vpd_ops_t *evpdop = enp->en_evpdop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_VPD); + + if ((rc = evpdop->evpdo_next(enp, data, size, evvp, contp)) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_vpd_write( + __in efx_nic_t *enp, + __in_bcount(size) caddr_t data, + __in size_t size) +{ + const efx_vpd_ops_t *evpdop = enp->en_evpdop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_VPD); + + if ((rc = evpdop->evpdo_write(enp, data, size)) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +static __checkReturn efx_rc_t +efx_vpd_next_tag( + __in caddr_t data, + __in size_t size, + __inout unsigned int *offsetp, + __out efx_vpd_tag_t *tagp, + __out uint16_t *lengthp) +{ + efx_byte_t byte; + efx_word_t word; + uint8_t name; + uint16_t length; + size_t headlen; + efx_rc_t rc; + + if (*offsetp >= size) { + rc = EFAULT; + goto fail1; + } + + EFX_POPULATE_BYTE_1(byte, EFX_BYTE_0, data[*offsetp]); + + switch (EFX_BYTE_FIELD(byte, TAG_TYPE)) { + case TAG_TYPE_SMALL_ITEM_DECODE: + headlen = 1; + + name = EFX_BYTE_FIELD(byte, TAG_SMALL_ITEM_NAME); + length = (uint16_t)EFX_BYTE_FIELD(byte, TAG_SMALL_ITEM_SIZE); + + break; + + case TAG_TYPE_LARGE_ITEM_DECODE: + headlen = 3; + + if (*offsetp + headlen > size) { + rc = EFAULT; + goto fail2; + } + + name = EFX_BYTE_FIELD(byte, TAG_LARGE_ITEM_NAME); + EFX_POPULATE_WORD_2(word, + EFX_BYTE_0, data[*offsetp + 1], + EFX_BYTE_1, data[*offsetp + 2]); + length = EFX_WORD_FIELD(word, EFX_WORD_0); + + break; + + default: + rc = EFAULT; + goto fail2; + } + + if (*offsetp + headlen + length > size) { + rc = EFAULT; + goto fail3; + } + + EFX_STATIC_ASSERT(TAG_NAME_END_DECODE == EFX_VPD_END); + EFX_STATIC_ASSERT(TAG_NAME_ID_STRING_DECODE == EFX_VPD_ID); + EFX_STATIC_ASSERT(TAG_NAME_VPD_R_DECODE == EFX_VPD_RO); + EFX_STATIC_ASSERT(TAG_NAME_VPD_W_DECODE == EFX_VPD_RW); + if (name != EFX_VPD_END && name != EFX_VPD_ID && + name != EFX_VPD_RO) { + rc = EFAULT; + goto fail4; + } + + *tagp = name; + *lengthp = length; + *offsetp += headlen; + + return (0); + +fail4: + EFSYS_PROBE(fail4); +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +static __checkReturn efx_rc_t +efx_vpd_next_keyword( + __in_bcount(size) caddr_t tag, + __in size_t size, + __in unsigned int pos, + __out efx_vpd_keyword_t *keywordp, + __out uint8_t *lengthp) +{ + efx_vpd_keyword_t keyword; + uint8_t length; + efx_rc_t rc; + + if (pos + 3U > size) { + rc = EFAULT; + goto fail1; + } + + keyword = EFX_VPD_KEYWORD(tag[pos], tag[pos + 1]); + length = tag[pos + 2]; + + if (length == 0 || pos + 3U + length > size) { + rc = EFAULT; + goto fail2; + } + + *keywordp = keyword; + *lengthp = length; + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_vpd_hunk_length( + __in_bcount(size) caddr_t data, + __in size_t size, + __out size_t *lengthp) +{ + efx_vpd_tag_t tag; + unsigned int offset; + uint16_t taglen; + efx_rc_t rc; + + offset = 0; + _NOTE(CONSTANTCONDITION) + while (1) { + if ((rc = efx_vpd_next_tag(data, size, &offset, + &tag, &taglen)) != 0) + goto fail1; + offset += taglen; + if (tag == EFX_VPD_END) + break; + } + + *lengthp = offset; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_vpd_hunk_verify( + __in_bcount(size) caddr_t data, + __in size_t size, + __out_opt boolean_t *cksummedp) +{ + efx_vpd_tag_t tag; + efx_vpd_keyword_t keyword; + unsigned int offset; + unsigned int pos; + unsigned int i; + uint16_t taglen; + uint8_t keylen; + uint8_t cksum; + boolean_t cksummed = B_FALSE; + efx_rc_t rc; + + /* + * Parse every tag,keyword in the existing VPD. If the csum is present, + * the assert it is correct, and is the final keyword in the RO block. + */ + offset = 0; + _NOTE(CONSTANTCONDITION) + while (1) { + if ((rc = efx_vpd_next_tag(data, size, &offset, + &tag, &taglen)) != 0) + goto fail1; + if (tag == EFX_VPD_END) + break; + else if (tag == EFX_VPD_ID) + goto done; + + for (pos = 0; pos != taglen; pos += 3 + keylen) { + /* RV keyword must be the last in the block */ + if (cksummed) { + rc = EFAULT; + goto fail2; + } + + if ((rc = efx_vpd_next_keyword(data + offset, + taglen, pos, &keyword, &keylen)) != 0) + goto fail3; + + if (keyword == EFX_VPD_KEYWORD('R', 'V')) { + cksum = 0; + for (i = 0; i < offset + pos + 4; i++) + cksum += data[i]; + + if (cksum != 0) { + rc = EFAULT; + goto fail4; + } + + cksummed = B_TRUE; + } + } + + done: + offset += taglen; + } + + if (!cksummed) { + rc = EFAULT; + goto fail5; + } + + if (cksummedp != NULL) + *cksummedp = cksummed; + + return (0); + +fail5: + EFSYS_PROBE(fail5); +fail4: + EFSYS_PROBE(fail4); +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +static uint8_t __efx_vpd_blank_pid[] = { + /* Large resource type ID length 1 */ + 0x82, 0x01, 0x00, + /* Product name ' ' */ + 0x32, +}; + +static uint8_t __efx_vpd_blank_r[] = { + /* Large resource type VPD-R length 4 */ + 0x90, 0x04, 0x00, + /* RV keyword length 1 */ + 'R', 'V', 0x01, + /* RV payload checksum */ + 0x00, +}; + + __checkReturn efx_rc_t +efx_vpd_hunk_reinit( + __in_bcount(size) caddr_t data, + __in size_t size, + __in boolean_t wantpid) +{ + unsigned int offset = 0; + unsigned int pos; + efx_byte_t byte; + uint8_t cksum; + efx_rc_t rc; + + if (size < 0x100) { + rc = ENOSPC; + goto fail1; + } + + if (wantpid) { + memcpy(data + offset, __efx_vpd_blank_pid, + sizeof (__efx_vpd_blank_pid)); + offset += sizeof (__efx_vpd_blank_pid); + } + + memcpy(data + offset, __efx_vpd_blank_r, sizeof (__efx_vpd_blank_r)); + offset += sizeof (__efx_vpd_blank_r); + + /* Update checksum */ + cksum = 0; + for (pos = 0; pos < offset; pos++) + cksum += data[pos]; + data[offset - 1] -= cksum; + + /* Append trailing tag */ + EFX_POPULATE_BYTE_3(byte, + TAG_TYPE, TAG_TYPE_SMALL_ITEM_DECODE, + TAG_SMALL_ITEM_NAME, TAG_NAME_END_DECODE, + TAG_SMALL_ITEM_SIZE, 0); + data[offset] = EFX_BYTE_FIELD(byte, EFX_BYTE_0); + offset++; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_vpd_hunk_next( + __in_bcount(size) caddr_t data, + __in size_t size, + __out efx_vpd_tag_t *tagp, + __out efx_vpd_keyword_t *keywordp, + __out_opt unsigned int *payloadp, + __out_opt uint8_t *paylenp, + __inout unsigned int *contp) +{ + efx_vpd_tag_t tag; + efx_vpd_keyword_t keyword = 0; + unsigned int offset; + unsigned int pos; + unsigned int index; + uint16_t taglen; + uint8_t keylen; + uint8_t paylen; + efx_rc_t rc; + + offset = index = 0; + _NOTE(CONSTANTCONDITION) + while (1) { + if ((rc = efx_vpd_next_tag(data, size, &offset, + &tag, &taglen)) != 0) + goto fail1; + + if (tag == EFX_VPD_END) { + keyword = 0; + paylen = 0; + index = 0; + break; + } + + if (tag == EFX_VPD_ID) { + if (index++ == *contp) { + EFSYS_ASSERT3U(taglen, <, 0x100); + keyword = 0; + paylen = (uint8_t)MIN(taglen, 0xff); + + goto done; + } + } else { + for (pos = 0; pos != taglen; pos += 3 + keylen) { + if ((rc = efx_vpd_next_keyword(data + offset, + taglen, pos, &keyword, &keylen)) != 0) + goto fail2; + + if (index++ == *contp) { + offset += pos + 3; + paylen = keylen; + + goto done; + } + } + } + + offset += taglen; + } + +done: + *tagp = tag; + *keywordp = keyword; + if (payloadp != NULL) + *payloadp = offset; + if (paylenp != NULL) + *paylenp = paylen; + + *contp = index; + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_vpd_hunk_get( + __in_bcount(size) caddr_t data, + __in size_t size, + __in efx_vpd_tag_t tag, + __in efx_vpd_keyword_t keyword, + __out unsigned int *payloadp, + __out uint8_t *paylenp) +{ + efx_vpd_tag_t itag; + efx_vpd_keyword_t ikeyword; + unsigned int offset; + unsigned int pos; + uint16_t taglen; + uint8_t keylen; + efx_rc_t rc; + + offset = 0; + _NOTE(CONSTANTCONDITION) + while (1) { + if ((rc = efx_vpd_next_tag(data, size, &offset, + &itag, &taglen)) != 0) + goto fail1; + if (itag == EFX_VPD_END) + break; + + if (itag == tag) { + if (itag == EFX_VPD_ID) { + EFSYS_ASSERT3U(taglen, <, 0x100); + + *paylenp = (uint8_t)MIN(taglen, 0xff); + *payloadp = offset; + return (0); + } + + for (pos = 0; pos != taglen; pos += 3 + keylen) { + if ((rc = efx_vpd_next_keyword(data + offset, + taglen, pos, &ikeyword, &keylen)) != 0) + goto fail2; + + if (ikeyword == keyword) { + *paylenp = keylen; + *payloadp = offset + pos + 3; + return (0); + } + } + } + + offset += taglen; + } + + /* Not an error */ + return (ENOENT); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_vpd_hunk_set( + __in_bcount(size) caddr_t data, + __in size_t size, + __in efx_vpd_value_t *evvp) +{ + efx_word_t word; + efx_vpd_tag_t tag; + efx_vpd_keyword_t keyword; + unsigned int offset; + unsigned int pos; + unsigned int taghead; + unsigned int source; + unsigned int dest; + unsigned int i; + uint16_t taglen; + uint8_t keylen; + uint8_t cksum; + size_t used; + efx_rc_t rc; + + switch (evvp->evv_tag) { + case EFX_VPD_ID: + if (evvp->evv_keyword != 0) { + rc = EINVAL; + goto fail1; + } + + /* Can't delete the ID keyword */ + if (evvp->evv_length == 0) { + rc = EINVAL; + goto fail1; + } + break; + + case EFX_VPD_RO: + if (evvp->evv_keyword == EFX_VPD_KEYWORD('R', 'V')) { + rc = EINVAL; + goto fail1; + } + break; + + default: + rc = EINVAL; + goto fail1; + } + + /* Determine total size of all current tags */ + if ((rc = efx_vpd_hunk_length(data, size, &used)) != 0) + goto fail2; + + offset = 0; + _NOTE(CONSTANTCONDITION) + while (1) { + taghead = offset; + if ((rc = efx_vpd_next_tag(data, size, &offset, + &tag, &taglen)) != 0) + goto fail3; + if (tag == EFX_VPD_END) + break; + else if (tag != evvp->evv_tag) { + offset += taglen; + continue; + } + + /* We only support modifying large resource tags */ + if (offset - taghead != 3) { + rc = EINVAL; + goto fail4; + } + + /* + * Work out the offset of the byte immediately after the + * old (=source) and new (=dest) new keyword/tag + */ + pos = 0; + if (tag == EFX_VPD_ID) { + source = offset + taglen; + dest = offset + evvp->evv_length; + goto check_space; + } + + EFSYS_ASSERT3U(tag, ==, EFX_VPD_RO); + source = dest = 0; + for (pos = 0; pos != taglen; pos += 3 + keylen) { + if ((rc = efx_vpd_next_keyword(data + offset, + taglen, pos, &keyword, &keylen)) != 0) + goto fail5; + + if (keyword == evvp->evv_keyword && + evvp->evv_length == 0) { + /* Deleting this keyword */ + source = offset + pos + 3 + keylen; + dest = offset + pos; + break; + + } else if (keyword == evvp->evv_keyword) { + /* Adjusting this keyword */ + source = offset + pos + 3 + keylen; + dest = offset + pos + 3 + evvp->evv_length; + break; + + } else if (keyword == EFX_VPD_KEYWORD('R', 'V')) { + /* The RV keyword must be at the end */ + EFSYS_ASSERT3U(pos + 3 + keylen, ==, taglen); + + /* + * The keyword doesn't already exist. If the + * user deleting a non-existant keyword then + * this is a no-op. + */ + if (evvp->evv_length == 0) + return (0); + + /* Insert this keyword before the RV keyword */ + source = offset + pos; + dest = offset + pos + 3 + evvp->evv_length; + break; + } + } + + check_space: + if (used + dest > size + source) { + rc = ENOSPC; + goto fail6; + } + + /* Move trailing data */ + (void) memmove(data + dest, data + source, used - source); + + /* Copy contents */ + memcpy(data + dest - evvp->evv_length, evvp->evv_value, + evvp->evv_length); + + /* Insert new keyword header if required */ + if (tag != EFX_VPD_ID && evvp->evv_length > 0) { + EFX_POPULATE_WORD_1(word, EFX_WORD_0, + evvp->evv_keyword); + data[offset + pos + 0] = + EFX_WORD_FIELD(word, EFX_BYTE_0); + data[offset + pos + 1] = + EFX_WORD_FIELD(word, EFX_BYTE_1); + data[offset + pos + 2] = evvp->evv_length; + } + + /* Modify tag length (large resource type) */ + taglen += (dest - source); + EFX_POPULATE_WORD_1(word, EFX_WORD_0, taglen); + data[offset - 2] = EFX_WORD_FIELD(word, EFX_BYTE_0); + data[offset - 1] = EFX_WORD_FIELD(word, EFX_BYTE_1); + + goto checksum; + } + + /* Unable to find the matching tag */ + rc = ENOENT; + goto fail7; + +checksum: + /* Find the RV tag, and update the checksum */ + offset = 0; + _NOTE(CONSTANTCONDITION) + while (1) { + if ((rc = efx_vpd_next_tag(data, size, &offset, + &tag, &taglen)) != 0) + goto fail8; + if (tag == EFX_VPD_END) + break; + if (tag == EFX_VPD_RO) { + for (pos = 0; pos != taglen; pos += 3 + keylen) { + if ((rc = efx_vpd_next_keyword(data + offset, + taglen, pos, &keyword, &keylen)) != 0) + goto fail9; + + if (keyword == EFX_VPD_KEYWORD('R', 'V')) { + cksum = 0; + for (i = 0; i < offset + pos + 3; i++) + cksum += data[i]; + data[i] = -cksum; + break; + } + } + } + + offset += taglen; + } + + /* Zero out the unused portion */ + (void) memset(data + offset + taglen, 0xff, size - offset - taglen); + + return (0); + +fail9: + EFSYS_PROBE(fail9); +fail8: + EFSYS_PROBE(fail8); +fail7: + EFSYS_PROBE(fail7); +fail6: + EFSYS_PROBE(fail6); +fail5: + EFSYS_PROBE(fail5); +fail4: + EFSYS_PROBE(fail4); +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + void +efx_vpd_fini( + __in efx_nic_t *enp) +{ + const efx_vpd_ops_t *evpdop = enp->en_evpdop; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_VPD); + + if (evpdop->evpdo_fini != NULL) + evpdop->evpdo_fini(enp); + + enp->en_evpdop = NULL; + enp->en_mod_flags &= ~EFX_MOD_VPD; +} + +#endif /* EFSYS_OPT_VPD */ diff --git a/dpdk/drivers/net/sfc/base/hunt_impl.h b/dpdk/drivers/net/sfc/base/hunt_impl.h new file mode 100644 index 00000000..0e0c870f --- /dev/null +++ b/dpdk/drivers/net/sfc/base/hunt_impl.h @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2012-2016 Solarflare Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are + * those of the authors and should not be interpreted as representing official + * policies, either expressed or implied, of the FreeBSD Project. + */ + +#ifndef _SYS_HUNT_IMPL_H +#define _SYS_HUNT_IMPL_H + +#include "efx.h" +#include "efx_regs.h" +#include "efx_regs_ef10.h" +#include "efx_mcdi.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* Missing register definitions */ +#ifndef ER_DZ_TX_PIOBUF_OFST +#define ER_DZ_TX_PIOBUF_OFST 0x00001000 +#endif +#ifndef ER_DZ_TX_PIOBUF_STEP +#define ER_DZ_TX_PIOBUF_STEP 8192 +#endif +#ifndef ER_DZ_TX_PIOBUF_ROWS +#define ER_DZ_TX_PIOBUF_ROWS 2048 +#endif + +#ifndef ER_DZ_TX_PIOBUF_SIZE +#define ER_DZ_TX_PIOBUF_SIZE 2048 +#endif + +#define HUNT_PIOBUF_NBUFS (16) +#define HUNT_PIOBUF_SIZE (ER_DZ_TX_PIOBUF_SIZE) + +#define HUNT_MIN_PIO_ALLOC_SIZE (HUNT_PIOBUF_SIZE / 32) + + +/* NIC */ + +extern __checkReturn efx_rc_t +hunt_board_cfg( + __in efx_nic_t *enp); + +#ifdef __cplusplus +} +#endif + +#endif /* _SYS_HUNT_IMPL_H */ diff --git a/dpdk/drivers/net/sfc/base/hunt_nic.c b/dpdk/drivers/net/sfc/base/hunt_nic.c new file mode 100644 index 00000000..19fb7cfb --- /dev/null +++ b/dpdk/drivers/net/sfc/base/hunt_nic.c @@ -0,0 +1,409 @@ +/* + * Copyright (c) 2012-2016 Solarflare Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are + * those of the authors and should not be interpreted as representing official + * policies, either expressed or implied, of the FreeBSD Project. + */ + +#include "efx.h" +#include "efx_impl.h" +#if EFSYS_OPT_MON_MCDI +#include "mcdi_mon.h" +#endif + +#if EFSYS_OPT_HUNTINGTON + +#include "ef10_tlv_layout.h" + +static __checkReturn efx_rc_t +hunt_nic_get_required_pcie_bandwidth( + __in efx_nic_t *enp, + __out uint32_t *bandwidth_mbpsp) +{ + uint32_t port_modes; + uint32_t max_port_mode; + uint32_t bandwidth; + efx_rc_t rc; + + /* + * On Huntington, the firmware may not give us the current port mode, so + * we need to go by the set of available port modes and assume the most + * capable mode is in use. + */ + + if ((rc = efx_mcdi_get_port_modes(enp, &port_modes, NULL)) != 0) { + /* No port mode info available */ + bandwidth = 0; + goto out; + } + + if (port_modes & (1 << TLV_PORT_MODE_40G_40G)) { + /* + * This needs the full PCIe bandwidth (and could use + * more) - roughly 64 Gbit/s for 8 lanes of Gen3. + */ + if ((rc = efx_nic_calculate_pcie_link_bandwidth(8, + EFX_PCIE_LINK_SPEED_GEN3, &bandwidth)) != 0) + goto fail1; + } else { + if (port_modes & (1 << TLV_PORT_MODE_40G)) { + max_port_mode = TLV_PORT_MODE_40G; + } else if (port_modes & (1 << TLV_PORT_MODE_10G_10G_10G_10G)) { + max_port_mode = TLV_PORT_MODE_10G_10G_10G_10G; + } else { + /* Assume two 10G ports */ + max_port_mode = TLV_PORT_MODE_10G_10G; + } + + if ((rc = ef10_nic_get_port_mode_bandwidth(max_port_mode, + &bandwidth)) != 0) + goto fail2; + } + +out: + *bandwidth_mbpsp = bandwidth; + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +hunt_board_cfg( + __in efx_nic_t *enp) +{ + efx_mcdi_iface_t *emip = &(enp->en_mcdi.em_emip); + efx_nic_cfg_t *encp = &(enp->en_nic_cfg); + uint8_t mac_addr[6] = { 0 }; + uint32_t board_type = 0; + ef10_link_state_t els; + efx_port_t *epp = &(enp->en_port); + uint32_t port; + uint32_t pf; + uint32_t vf; + uint32_t mask; + uint32_t flags; + uint32_t sysclk, dpcpu_clk; + uint32_t base, nvec; + uint32_t bandwidth; + efx_rc_t rc; + + if ((rc = efx_mcdi_get_port_assignment(enp, &port)) != 0) + goto fail1; + + /* + * NOTE: The MCDI protocol numbers ports from zero. + * The common code MCDI interface numbers ports from one. + */ + emip->emi_port = port + 1; + + if ((rc = ef10_external_port_mapping(enp, port, + &encp->enc_external_port)) != 0) + goto fail2; + + /* + * Get PCIe function number from firmware (used for + * per-function privilege and dynamic config info). + * - PCIe PF: pf = PF number, vf = 0xffff. + * - PCIe VF: pf = parent PF, vf = VF number. + */ + if ((rc = efx_mcdi_get_function_info(enp, &pf, &vf)) != 0) + goto fail3; + + encp->enc_pf = pf; + encp->enc_vf = vf; + + /* MAC address for this function */ + if (EFX_PCI_FUNCTION_IS_PF(encp)) { + rc = efx_mcdi_get_mac_address_pf(enp, mac_addr); + if ((rc == 0) && (mac_addr[0] & 0x02)) { + /* + * If the static config does not include a global MAC + * address pool then the board may return a locally + * administered MAC address (this should only happen on + * incorrectly programmed boards). + */ + rc = EINVAL; + } + } else { + rc = efx_mcdi_get_mac_address_vf(enp, mac_addr); + } + if (rc != 0) + goto fail4; + + EFX_MAC_ADDR_COPY(encp->enc_mac_addr, mac_addr); + + /* Board configuration */ + rc = efx_mcdi_get_board_cfg(enp, &board_type, NULL, NULL); + if (rc != 0) { + /* Unprivileged functions may not be able to read board cfg */ + if (rc == EACCES) + board_type = 0; + else + goto fail5; + } + + encp->enc_board_type = board_type; + encp->enc_clk_mult = 1; /* not used for Huntington */ + + /* Fill out fields in enp->en_port and enp->en_nic_cfg from MCDI */ + if ((rc = efx_mcdi_get_phy_cfg(enp)) != 0) + goto fail6; + + /* Obtain the default PHY advertised capabilities */ + if ((rc = ef10_phy_get_link(enp, &els)) != 0) + goto fail7; + epp->ep_default_adv_cap_mask = els.els_adv_cap_mask; + epp->ep_adv_cap_mask = els.els_adv_cap_mask; + + /* + * Enable firmware workarounds for hardware errata. + * Expected responses are: + * - 0 (zero): + * Success: workaround enabled or disabled as requested. + * - MC_CMD_ERR_ENOSYS (reported as ENOTSUP): + * Firmware does not support the MC_CMD_WORKAROUND request. + * (assume that the workaround is not supported). + * - MC_CMD_ERR_ENOENT (reported as ENOENT): + * Firmware does not support the requested workaround. + * - MC_CMD_ERR_EPERM (reported as EACCES): + * Unprivileged function cannot enable/disable workarounds. + * + * See efx_mcdi_request_errcode() for MCDI error translations. + */ + + /* + * If the bug35388 workaround is enabled, then use an indirect access + * method to avoid unsafe EVQ writes. + */ + rc = efx_mcdi_set_workaround(enp, MC_CMD_WORKAROUND_BUG35388, B_TRUE, + NULL); + if ((rc == 0) || (rc == EACCES)) + encp->enc_bug35388_workaround = B_TRUE; + else if ((rc == ENOTSUP) || (rc == ENOENT)) + encp->enc_bug35388_workaround = B_FALSE; + else + goto fail8; + + /* + * If the bug41750 workaround is enabled, then do not test interrupts, + * as the test will fail (seen with Greenport controllers). + */ + rc = efx_mcdi_set_workaround(enp, MC_CMD_WORKAROUND_BUG41750, B_TRUE, + NULL); + if (rc == 0) { + encp->enc_bug41750_workaround = B_TRUE; + } else if (rc == EACCES) { + /* Assume a controller with 40G ports needs the workaround. */ + if (epp->ep_default_adv_cap_mask & EFX_PHY_CAP_40000FDX) + encp->enc_bug41750_workaround = B_TRUE; + else + encp->enc_bug41750_workaround = B_FALSE; + } else if ((rc == ENOTSUP) || (rc == ENOENT)) { + encp->enc_bug41750_workaround = B_FALSE; + } else { + goto fail9; + } + if (EFX_PCI_FUNCTION_IS_VF(encp)) { + /* Interrupt testing does not work for VFs. See bug50084. */ + encp->enc_bug41750_workaround = B_TRUE; + } + + /* + * If the bug26807 workaround is enabled, then firmware has enabled + * support for chained multicast filters. Firmware will reset (FLR) + * functions which have filters in the hardware filter table when the + * workaround is enabled/disabled. + * + * We must recheck if the workaround is enabled after inserting the + * first hardware filter, in case it has been changed since this check. + */ + rc = efx_mcdi_set_workaround(enp, MC_CMD_WORKAROUND_BUG26807, + B_TRUE, &flags); + if (rc == 0) { + encp->enc_bug26807_workaround = B_TRUE; + if (flags & (1 << MC_CMD_WORKAROUND_EXT_OUT_FLR_DONE_LBN)) { + /* + * Other functions had installed filters before the + * workaround was enabled, and they have been reset + * by firmware. + */ + EFSYS_PROBE(bug26807_workaround_flr_done); + /* FIXME: bump MC warm boot count ? */ + } + } else if (rc == EACCES) { + /* + * Unprivileged functions cannot enable the workaround in older + * firmware. + */ + encp->enc_bug26807_workaround = B_FALSE; + } else if ((rc == ENOTSUP) || (rc == ENOENT)) { + encp->enc_bug26807_workaround = B_FALSE; + } else { + goto fail10; + } + + /* Get clock frequencies (in MHz). */ + if ((rc = efx_mcdi_get_clock(enp, &sysclk, &dpcpu_clk)) != 0) + goto fail11; + + /* + * The Huntington timer quantum is 1536 sysclk cycles, documented for + * the EV_TMR_VAL field of EV_TIMER_TBL. Scale for MHz and ns units. + */ + encp->enc_evq_timer_quantum_ns = 1536000UL / sysclk; /* 1536 cycles */ + if (encp->enc_bug35388_workaround) { + encp->enc_evq_timer_max_us = (encp->enc_evq_timer_quantum_ns << + ERF_DD_EVQ_IND_TIMER_VAL_WIDTH) / 1000; + } else { + encp->enc_evq_timer_max_us = (encp->enc_evq_timer_quantum_ns << + FRF_CZ_TC_TIMER_VAL_WIDTH) / 1000; + } + + encp->enc_bug61265_workaround = B_FALSE; /* Medford only */ + + /* Check capabilities of running datapath firmware */ + if ((rc = ef10_get_datapath_caps(enp)) != 0) + goto fail12; + + /* Alignment for receive packet DMA buffers */ + encp->enc_rx_buf_align_start = 1; + encp->enc_rx_buf_align_end = 64; /* RX DMA end padding */ + + /* Alignment for WPTR updates */ + encp->enc_rx_push_align = EF10_RX_WPTR_ALIGN; + + /* + * Maximum number of exclusive RSS contexts which can be allocated. The + * hardware supports 64, but 6 are reserved for shared contexts. They + * are a global resource so not all may be available. + */ + encp->enc_rx_scale_max_exclusive_contexts = 58; + + encp->enc_tx_dma_desc_size_max = EFX_MASK32(ESF_DZ_RX_KER_BYTE_CNT); + /* No boundary crossing limits */ + encp->enc_tx_dma_desc_boundary = 0; + + /* + * Set resource limits for MC_CMD_ALLOC_VIS. Note that we cannot use + * MC_CMD_GET_RESOURCE_LIMITS here as that reports the available + * resources (allocated to this PCIe function), which is zero until + * after we have allocated VIs. + */ + encp->enc_evq_limit = 1024; + encp->enc_rxq_limit = EFX_RXQ_LIMIT_TARGET; + encp->enc_txq_limit = EFX_TXQ_LIMIT_TARGET; + + /* + * The workaround for bug35388 uses the top bit of transmit queue + * descriptor writes, preventing the use of 4096 descriptor TXQs. + */ + encp->enc_txq_max_ndescs = encp->enc_bug35388_workaround ? 2048 : 4096; + + encp->enc_buftbl_limit = 0xFFFFFFFF; + + encp->enc_piobuf_limit = HUNT_PIOBUF_NBUFS; + encp->enc_piobuf_size = HUNT_PIOBUF_SIZE; + encp->enc_piobuf_min_alloc_size = HUNT_MIN_PIO_ALLOC_SIZE; + + /* + * Get the current privilege mask. Note that this may be modified + * dynamically, so this value is informational only. DO NOT use + * the privilege mask to check for sufficient privileges, as that + * can result in time-of-check/time-of-use bugs. + */ + if ((rc = ef10_get_privilege_mask(enp, &mask)) != 0) + goto fail13; + encp->enc_privilege_mask = mask; + + /* Get interrupt vector limits */ + if ((rc = efx_mcdi_get_vector_cfg(enp, &base, &nvec, NULL)) != 0) { + if (EFX_PCI_FUNCTION_IS_PF(encp)) + goto fail14; + + /* Ignore error (cannot query vector limits from a VF). */ + base = 0; + nvec = 1024; + } + encp->enc_intr_vec_base = base; + encp->enc_intr_limit = nvec; + + /* + * Maximum number of bytes into the frame the TCP header can start for + * firmware assisted TSO to work. + */ + encp->enc_tx_tso_tcp_header_offset_limit = EF10_TCP_HEADER_OFFSET_LIMIT; + + if ((rc = hunt_nic_get_required_pcie_bandwidth(enp, &bandwidth)) != 0) + goto fail15; + encp->enc_required_pcie_bandwidth_mbps = bandwidth; + + /* All Huntington devices have a PCIe Gen3, 8 lane connector */ + encp->enc_max_pcie_link_gen = EFX_PCIE_LINK_SPEED_GEN3; + + return (0); + +fail15: + EFSYS_PROBE(fail15); +fail14: + EFSYS_PROBE(fail14); +fail13: + EFSYS_PROBE(fail13); +fail12: + EFSYS_PROBE(fail12); +fail11: + EFSYS_PROBE(fail11); +fail10: + EFSYS_PROBE(fail10); +fail9: + EFSYS_PROBE(fail9); +fail8: + EFSYS_PROBE(fail8); +fail7: + EFSYS_PROBE(fail7); +fail6: + EFSYS_PROBE(fail6); +fail5: + EFSYS_PROBE(fail5); +fail4: + EFSYS_PROBE(fail4); +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + +#endif /* EFSYS_OPT_HUNTINGTON */ diff --git a/dpdk/drivers/net/sfc/base/mcdi_mon.c b/dpdk/drivers/net/sfc/base/mcdi_mon.c new file mode 100644 index 00000000..c5360c31 --- /dev/null +++ b/dpdk/drivers/net/sfc/base/mcdi_mon.c @@ -0,0 +1,565 @@ +/* + * Copyright (c) 2009-2016 Solarflare Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are + * those of the authors and should not be interpreted as representing official + * policies, either expressed or implied, of the FreeBSD Project. + */ + +#include "efx.h" +#include "efx_impl.h" + +#if EFSYS_OPT_MON_MCDI + +#if EFSYS_OPT_MON_STATS + +#define MCDI_MON_NEXT_PAGE ((uint16_t)0xfffe) +#define MCDI_MON_INVALID_SENSOR ((uint16_t)0xfffd) +#define MCDI_MON_PAGE_SIZE 0x20 + +/* Bitmasks of valid port(s) for each sensor */ +#define MCDI_MON_PORT_NONE (0x00) +#define MCDI_MON_PORT_P1 (0x01) +#define MCDI_MON_PORT_P2 (0x02) +#define MCDI_MON_PORT_P3 (0x04) +#define MCDI_MON_PORT_P4 (0x08) +#define MCDI_MON_PORT_Px (0xFFFF) + +/* Get port mask from one-based MCDI port number */ +#define MCDI_MON_PORT_MASK(_emip) (1U << ((_emip)->emi_port - 1)) + +/* Entry for MCDI sensor in sensor map */ +#define STAT(portmask, stat) \ + { (MCDI_MON_PORT_##portmask), (EFX_MON_STAT_##stat) } + +/* Entry for sensor next page flag in sensor map */ +#define STAT_NEXT_PAGE() \ + { MCDI_MON_PORT_NONE, MCDI_MON_NEXT_PAGE } + +/* Placeholder for gaps in the array */ +#define STAT_NO_SENSOR() \ + { MCDI_MON_PORT_NONE, MCDI_MON_INVALID_SENSOR } + +/* Map from MC sensors to monitor statistics */ +static const struct mcdi_sensor_map_s { + uint16_t msm_port_mask; + uint16_t msm_stat; +} mcdi_sensor_map[] = { + /* Sensor page 0 MC_CMD_SENSOR_xxx */ + STAT(Px, INT_TEMP), /* 0x00 CONTROLLER_TEMP */ + STAT(Px, EXT_TEMP), /* 0x01 PHY_COMMON_TEMP */ + STAT(Px, INT_COOLING), /* 0x02 CONTROLLER_COOLING */ + STAT(P1, EXT_TEMP), /* 0x03 PHY0_TEMP */ + STAT(P1, EXT_COOLING), /* 0x04 PHY0_COOLING */ + STAT(P2, EXT_TEMP), /* 0x05 PHY1_TEMP */ + STAT(P2, EXT_COOLING), /* 0x06 PHY1_COOLING */ + STAT(Px, 1V), /* 0x07 IN_1V0 */ + STAT(Px, 1_2V), /* 0x08 IN_1V2 */ + STAT(Px, 1_8V), /* 0x09 IN_1V8 */ + STAT(Px, 2_5V), /* 0x0a IN_2V5 */ + STAT(Px, 3_3V), /* 0x0b IN_3V3 */ + STAT(Px, 12V), /* 0x0c IN_12V0 */ + STAT(Px, 1_2VA), /* 0x0d IN_1V2A */ + STAT(Px, VREF), /* 0x0e IN_VREF */ + STAT(Px, VAOE), /* 0x0f OUT_VAOE */ + STAT(Px, AOE_TEMP), /* 0x10 AOE_TEMP */ + STAT(Px, PSU_AOE_TEMP), /* 0x11 PSU_AOE_TEMP */ + STAT(Px, PSU_TEMP), /* 0x12 PSU_TEMP */ + STAT(Px, FAN0), /* 0x13 FAN_0 */ + STAT(Px, FAN1), /* 0x14 FAN_1 */ + STAT(Px, FAN2), /* 0x15 FAN_2 */ + STAT(Px, FAN3), /* 0x16 FAN_3 */ + STAT(Px, FAN4), /* 0x17 FAN_4 */ + STAT(Px, VAOE_IN), /* 0x18 IN_VAOE */ + STAT(Px, IAOE), /* 0x19 OUT_IAOE */ + STAT(Px, IAOE_IN), /* 0x1a IN_IAOE */ + STAT(Px, NIC_POWER), /* 0x1b NIC_POWER */ + STAT(Px, 0_9V), /* 0x1c IN_0V9 */ + STAT(Px, I0_9V), /* 0x1d IN_I0V9 */ + STAT(Px, I1_2V), /* 0x1e IN_I1V2 */ + STAT_NEXT_PAGE(), /* 0x1f Next page flag (not a sensor) */ + + /* Sensor page 1 MC_CMD_SENSOR_xxx */ + STAT(Px, 0_9V_ADC), /* 0x20 IN_0V9_ADC */ + STAT(Px, INT_TEMP2), /* 0x21 CONTROLLER_2_TEMP */ + STAT(Px, VREG_TEMP), /* 0x22 VREG_INTERNAL_TEMP */ + STAT(Px, VREG_0_9V_TEMP), /* 0x23 VREG_0V9_TEMP */ + STAT(Px, VREG_1_2V_TEMP), /* 0x24 VREG_1V2_TEMP */ + STAT(Px, INT_VPTAT), /* 0x25 CTRLR. VPTAT */ + STAT(Px, INT_ADC_TEMP), /* 0x26 CTRLR. INTERNAL_TEMP */ + STAT(Px, EXT_VPTAT), /* 0x27 CTRLR. VPTAT_EXTADC */ + STAT(Px, EXT_ADC_TEMP), /* 0x28 CTRLR. INTERNAL_TEMP_EXTADC */ + STAT(Px, AMBIENT_TEMP), /* 0x29 AMBIENT_TEMP */ + STAT(Px, AIRFLOW), /* 0x2a AIRFLOW */ + STAT(Px, VDD08D_VSS08D_CSR), /* 0x2b VDD08D_VSS08D_CSR */ + STAT(Px, VDD08D_VSS08D_CSR_EXTADC), /* 0x2c VDD08D_VSS08D_CSR_EXTADC */ + STAT(Px, HOTPOINT_TEMP), /* 0x2d HOTPOINT_TEMP */ + STAT(P1, PHY_POWER_SWITCH_PORT0), /* 0x2e PHY_POWER_SWITCH_PORT0 */ + STAT(P2, PHY_POWER_SWITCH_PORT1), /* 0x2f PHY_POWER_SWITCH_PORT1 */ + STAT(Px, MUM_VCC), /* 0x30 MUM_VCC */ + STAT(Px, 0V9_A), /* 0x31 0V9_A */ + STAT(Px, I0V9_A), /* 0x32 I0V9_A */ + STAT(Px, 0V9_A_TEMP), /* 0x33 0V9_A_TEMP */ + STAT(Px, 0V9_B), /* 0x34 0V9_B */ + STAT(Px, I0V9_B), /* 0x35 I0V9_B */ + STAT(Px, 0V9_B_TEMP), /* 0x36 0V9_B_TEMP */ + STAT(Px, CCOM_AVREG_1V2_SUPPLY), /* 0x37 CCOM_AVREG_1V2_SUPPLY */ + STAT(Px, CCOM_AVREG_1V2_SUPPLY_EXT_ADC), + /* 0x38 CCOM_AVREG_1V2_SUPPLY_EXT_ADC */ + STAT(Px, CCOM_AVREG_1V8_SUPPLY), /* 0x39 CCOM_AVREG_1V8_SUPPLY */ + STAT(Px, CCOM_AVREG_1V8_SUPPLY_EXT_ADC), + /* 0x3a CCOM_AVREG_1V8_SUPPLY_EXT_ADC */ + STAT_NO_SENSOR(), /* 0x3b (no sensor) */ + STAT_NO_SENSOR(), /* 0x3c (no sensor) */ + STAT_NO_SENSOR(), /* 0x3d (no sensor) */ + STAT_NO_SENSOR(), /* 0x3e (no sensor) */ + STAT_NEXT_PAGE(), /* 0x3f Next page flag (not a sensor) */ + + /* Sensor page 2 MC_CMD_SENSOR_xxx */ + STAT(Px, CONTROLLER_MASTER_VPTAT), /* 0x40 MASTER_VPTAT */ + STAT(Px, CONTROLLER_MASTER_INTERNAL_TEMP), /* 0x41 MASTER_INT_TEMP */ + STAT(Px, CONTROLLER_MASTER_VPTAT_EXT_ADC), /* 0x42 MAST_VPTAT_EXT_ADC */ + STAT(Px, CONTROLLER_MASTER_INTERNAL_TEMP_EXT_ADC), + /* 0x43 MASTER_INTERNAL_TEMP_EXT_ADC */ + STAT(Px, CONTROLLER_SLAVE_VPTAT), /* 0x44 SLAVE_VPTAT */ + STAT(Px, CONTROLLER_SLAVE_INTERNAL_TEMP), /* 0x45 SLAVE_INTERNAL_TEMP */ + STAT(Px, CONTROLLER_SLAVE_VPTAT_EXT_ADC), /* 0x46 SLAVE_VPTAT_EXT_ADC */ + STAT(Px, CONTROLLER_SLAVE_INTERNAL_TEMP_EXT_ADC), + /* 0x47 SLAVE_INTERNAL_TEMP_EXT_ADC */ + STAT_NO_SENSOR(), /* 0x48 (no sensor) */ + STAT(Px, SODIMM_VOUT), /* 0x49 SODIMM_VOUT */ + STAT(Px, SODIMM_0_TEMP), /* 0x4a SODIMM_0_TEMP */ + STAT(Px, SODIMM_1_TEMP), /* 0x4b SODIMM_1_TEMP */ + STAT(Px, PHY0_VCC), /* 0x4c PHY0_VCC */ + STAT(Px, PHY1_VCC), /* 0x4d PHY1_VCC */ + STAT(Px, CONTROLLER_TDIODE_TEMP), /* 0x4e CONTROLLER_TDIODE_TEMP */ + STAT(Px, BOARD_FRONT_TEMP), /* 0x4f BOARD_FRONT_TEMP */ + STAT(Px, BOARD_BACK_TEMP), /* 0x50 BOARD_BACK_TEMP */ +}; + +#define MCDI_STATIC_SENSOR_ASSERT(_field) \ + EFX_STATIC_ASSERT(MC_CMD_SENSOR_STATE_ ## _field \ + == EFX_MON_STAT_STATE_ ## _field) + +static void +mcdi_mon_decode_stats( + __in efx_nic_t *enp, + __in_bcount(sensor_mask_size) uint32_t *sensor_mask, + __in size_t sensor_mask_size, + __in_opt efsys_mem_t *esmp, + __out_bcount_opt(sensor_mask_size) uint32_t *stat_maskp, + __inout_ecount_opt(EFX_MON_NSTATS) efx_mon_stat_value_t *stat) +{ + efx_mcdi_iface_t *emip = &(enp->en_mcdi.em_emip); + uint16_t port_mask; + uint16_t sensor; + size_t sensor_max; + uint32_t stat_mask[(EFX_ARRAY_SIZE(mcdi_sensor_map) + 31) / 32]; + uint32_t idx = 0; + uint32_t page = 0; + + /* Assert the MC_CMD_SENSOR and EFX_MON_STATE namespaces agree */ + MCDI_STATIC_SENSOR_ASSERT(OK); + MCDI_STATIC_SENSOR_ASSERT(WARNING); + MCDI_STATIC_SENSOR_ASSERT(FATAL); + MCDI_STATIC_SENSOR_ASSERT(BROKEN); + MCDI_STATIC_SENSOR_ASSERT(NO_READING); + + EFX_STATIC_ASSERT(sizeof (stat_mask[0]) * 8 == + EFX_MON_MASK_ELEMENT_SIZE); + sensor_max = + MIN((8 * sensor_mask_size), EFX_ARRAY_SIZE(mcdi_sensor_map)); + + EFSYS_ASSERT(emip->emi_port > 0); /* MCDI port number is one-based */ + port_mask = MCDI_MON_PORT_MASK(emip); + + memset(stat_mask, 0, sizeof (stat_mask)); + + /* + * The MCDI sensor readings in the DMA buffer are a packed array of + * MC_CMD_SENSOR_VALUE_ENTRY structures, which only includes entries for + * supported sensors (bit set in sensor_mask). The sensor_mask and + * sensor readings do not include entries for the per-page NEXT_PAGE + * flag. + * + * sensor_mask may legitimately contain MCDI sensors that the driver + * does not understand. + */ + for (sensor = 0; sensor < sensor_max; ++sensor) { + efx_mon_stat_t id = mcdi_sensor_map[sensor].msm_stat; + + if ((sensor % MCDI_MON_PAGE_SIZE) == MC_CMD_SENSOR_PAGE0_NEXT) { + EFSYS_ASSERT3U(id, ==, MCDI_MON_NEXT_PAGE); + page++; + continue; + } + if (~(sensor_mask[page]) & (1U << sensor)) + continue; + idx++; + + if ((port_mask & mcdi_sensor_map[sensor].msm_port_mask) == 0) + continue; + EFSYS_ASSERT(id < EFX_MON_NSTATS); + + /* + * stat_mask is a bitmask indexed by EFX_MON_* monitor statistic + * identifiers from efx_mon_stat_t (without NEXT_PAGE bits). + * + * If there is an entry in the MCDI sensor to monitor statistic + * map then the sensor reading is used for the value of the + * monitor statistic. + */ + stat_mask[id / EFX_MON_MASK_ELEMENT_SIZE] |= + (1U << (id % EFX_MON_MASK_ELEMENT_SIZE)); + + if (stat != NULL && esmp != NULL && !EFSYS_MEM_IS_NULL(esmp)) { + efx_dword_t dword; + + /* Get MCDI sensor reading from DMA buffer */ + EFSYS_MEM_READD(esmp, 4 * (idx - 1), &dword); + + /* Update EFX monitor stat from MCDI sensor reading */ + stat[id].emsv_value = (uint16_t)EFX_DWORD_FIELD(dword, + MC_CMD_SENSOR_VALUE_ENTRY_TYPEDEF_VALUE); + + stat[id].emsv_state = (uint16_t)EFX_DWORD_FIELD(dword, + MC_CMD_SENSOR_VALUE_ENTRY_TYPEDEF_STATE); + } + } + + if (stat_maskp != NULL) { + memcpy(stat_maskp, stat_mask, sizeof (stat_mask)); + } +} + + __checkReturn efx_rc_t +mcdi_mon_ev( + __in efx_nic_t *enp, + __in efx_qword_t *eqp, + __out efx_mon_stat_t *idp, + __out efx_mon_stat_value_t *valuep) +{ + efx_mcdi_iface_t *emip = &(enp->en_mcdi.em_emip); + efx_nic_cfg_t *encp = &(enp->en_nic_cfg); + uint16_t port_mask; + uint16_t sensor; + uint16_t state; + uint16_t value; + efx_mon_stat_t id; + efx_rc_t rc; + + EFSYS_ASSERT(emip->emi_port > 0); /* MCDI port number is one-based */ + port_mask = MCDI_MON_PORT_MASK(emip); + + sensor = (uint16_t)MCDI_EV_FIELD(eqp, SENSOREVT_MONITOR); + state = (uint16_t)MCDI_EV_FIELD(eqp, SENSOREVT_STATE); + value = (uint16_t)MCDI_EV_FIELD(eqp, SENSOREVT_VALUE); + + /* Hardware must support this MCDI sensor */ + EFSYS_ASSERT3U(sensor, <, (8 * encp->enc_mcdi_sensor_mask_size)); + EFSYS_ASSERT((sensor % MCDI_MON_PAGE_SIZE) != MC_CMD_SENSOR_PAGE0_NEXT); + EFSYS_ASSERT(encp->enc_mcdi_sensor_maskp != NULL); + EFSYS_ASSERT((encp->enc_mcdi_sensor_maskp[sensor / MCDI_MON_PAGE_SIZE] & + (1U << (sensor % MCDI_MON_PAGE_SIZE))) != 0); + + /* But we don't have to understand it */ + if (sensor >= EFX_ARRAY_SIZE(mcdi_sensor_map)) { + rc = ENOTSUP; + goto fail1; + } + id = mcdi_sensor_map[sensor].msm_stat; + if ((port_mask & mcdi_sensor_map[sensor].msm_port_mask) == 0) + return (ENODEV); + EFSYS_ASSERT(id < EFX_MON_NSTATS); + + *idp = id; + valuep->emsv_value = value; + valuep->emsv_state = state; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + +static __checkReturn efx_rc_t +efx_mcdi_read_sensors( + __in efx_nic_t *enp, + __in efsys_mem_t *esmp, + __in uint32_t size) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_READ_SENSORS_EXT_IN_LEN, + MC_CMD_READ_SENSORS_EXT_OUT_LEN)]; + uint32_t addr_lo, addr_hi; + + req.emr_cmd = MC_CMD_READ_SENSORS; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_READ_SENSORS_EXT_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_READ_SENSORS_EXT_OUT_LEN; + + addr_lo = (uint32_t)(EFSYS_MEM_ADDR(esmp) & 0xffffffff); + addr_hi = (uint32_t)(EFSYS_MEM_ADDR(esmp) >> 32); + + MCDI_IN_SET_DWORD(req, READ_SENSORS_EXT_IN_DMA_ADDR_LO, addr_lo); + MCDI_IN_SET_DWORD(req, READ_SENSORS_EXT_IN_DMA_ADDR_HI, addr_hi); + MCDI_IN_SET_DWORD(req, READ_SENSORS_EXT_IN_LENGTH, size); + + efx_mcdi_execute(enp, &req); + + return (req.emr_rc); +} + +static __checkReturn efx_rc_t +efx_mcdi_sensor_info_npages( + __in efx_nic_t *enp, + __out uint32_t *npagesp) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_SENSOR_INFO_EXT_IN_LEN, + MC_CMD_SENSOR_INFO_OUT_LENMAX)]; + int page; + efx_rc_t rc; + + EFSYS_ASSERT(npagesp != NULL); + + page = 0; + do { + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_SENSOR_INFO; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_SENSOR_INFO_EXT_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_SENSOR_INFO_OUT_LENMAX; + + MCDI_IN_SET_DWORD(req, SENSOR_INFO_EXT_IN_PAGE, page++); + + efx_mcdi_execute_quiet(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + } while (MCDI_OUT_DWORD(req, SENSOR_INFO_OUT_MASK) & + (1U << MC_CMD_SENSOR_PAGE0_NEXT)); + + *npagesp = page; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +static __checkReturn efx_rc_t +efx_mcdi_sensor_info( + __in efx_nic_t *enp, + __out_ecount(npages) uint32_t *sensor_maskp, + __in size_t npages) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_SENSOR_INFO_EXT_IN_LEN, + MC_CMD_SENSOR_INFO_OUT_LENMAX)]; + uint32_t page; + efx_rc_t rc; + + EFSYS_ASSERT(sensor_maskp != NULL); + + for (page = 0; page < npages; page++) { + uint32_t mask; + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_SENSOR_INFO; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_SENSOR_INFO_EXT_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_SENSOR_INFO_OUT_LENMAX; + + MCDI_IN_SET_DWORD(req, SENSOR_INFO_EXT_IN_PAGE, page); + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + mask = MCDI_OUT_DWORD(req, SENSOR_INFO_OUT_MASK); + + if ((page != (npages - 1)) && + ((mask & (1U << MC_CMD_SENSOR_PAGE0_NEXT)) == 0)) { + rc = EINVAL; + goto fail2; + } + sensor_maskp[page] = mask; + } + + if (sensor_maskp[npages - 1] & (1U << MC_CMD_SENSOR_PAGE0_NEXT)) { + rc = EINVAL; + goto fail3; + } + + return (0); + +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +mcdi_mon_stats_update( + __in efx_nic_t *enp, + __in efsys_mem_t *esmp, + __inout_ecount(EFX_MON_NSTATS) efx_mon_stat_value_t *values) +{ + efx_nic_cfg_t *encp = &(enp->en_nic_cfg); + uint32_t size = encp->enc_mon_stat_dma_buf_size; + efx_rc_t rc; + + if ((rc = efx_mcdi_read_sensors(enp, esmp, size)) != 0) + goto fail1; + + EFSYS_DMA_SYNC_FOR_KERNEL(esmp, 0, size); + + mcdi_mon_decode_stats(enp, + encp->enc_mcdi_sensor_maskp, + encp->enc_mcdi_sensor_mask_size, + esmp, NULL, values); + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +mcdi_mon_cfg_build( + __in efx_nic_t *enp) +{ + efx_nic_cfg_t *encp = &(enp->en_nic_cfg); + uint32_t npages; + efx_rc_t rc; + + switch (enp->en_family) { +#if EFSYS_OPT_SIENA + case EFX_FAMILY_SIENA: + encp->enc_mon_type = EFX_MON_SFC90X0; + break; +#endif +#if EFSYS_OPT_HUNTINGTON + case EFX_FAMILY_HUNTINGTON: + encp->enc_mon_type = EFX_MON_SFC91X0; + break; +#endif +#if EFSYS_OPT_MEDFORD + case EFX_FAMILY_MEDFORD: + encp->enc_mon_type = EFX_MON_SFC92X0; + break; +#endif + default: + rc = EINVAL; + goto fail1; + } + + /* Get mc sensor mask size */ + npages = 0; + if ((rc = efx_mcdi_sensor_info_npages(enp, &npages)) != 0) + goto fail2; + + encp->enc_mon_stat_dma_buf_size = npages * EFX_MON_STATS_PAGE_SIZE; + encp->enc_mcdi_sensor_mask_size = npages * sizeof (uint32_t); + + /* Allocate mc sensor mask */ + EFSYS_KMEM_ALLOC(enp->en_esip, + encp->enc_mcdi_sensor_mask_size, + encp->enc_mcdi_sensor_maskp); + + if (encp->enc_mcdi_sensor_maskp == NULL) { + rc = ENOMEM; + goto fail3; + } + + /* Read mc sensor mask */ + if ((rc = efx_mcdi_sensor_info(enp, + encp->enc_mcdi_sensor_maskp, + npages)) != 0) + goto fail4; + + /* Build monitor statistics mask */ + mcdi_mon_decode_stats(enp, + encp->enc_mcdi_sensor_maskp, + encp->enc_mcdi_sensor_mask_size, + NULL, encp->enc_mon_stat_mask, NULL); + + return (0); + +fail4: + EFSYS_PROBE(fail4); + EFSYS_KMEM_FREE(enp->en_esip, + encp->enc_mcdi_sensor_mask_size, + encp->enc_mcdi_sensor_maskp); + +fail3: + EFSYS_PROBE(fail3); + +fail2: + EFSYS_PROBE(fail2); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + void +mcdi_mon_cfg_free( + __in efx_nic_t *enp) +{ + efx_nic_cfg_t *encp = &(enp->en_nic_cfg); + + if (encp->enc_mcdi_sensor_maskp != NULL) { + EFSYS_KMEM_FREE(enp->en_esip, + encp->enc_mcdi_sensor_mask_size, + encp->enc_mcdi_sensor_maskp); + } +} + + +#endif /* EFSYS_OPT_MON_STATS */ + +#endif /* EFSYS_OPT_MON_MCDI */ diff --git a/dpdk/drivers/net/sfc/base/mcdi_mon.h b/dpdk/drivers/net/sfc/base/mcdi_mon.h new file mode 100644 index 00000000..e07b5280 --- /dev/null +++ b/dpdk/drivers/net/sfc/base/mcdi_mon.h @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2009-2016 Solarflare Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are + * those of the authors and should not be interpreted as representing official + * policies, either expressed or implied, of the FreeBSD Project. + */ + +#ifndef _SYS_MCDI_MON_H +#define _SYS_MCDI_MON_H + +#include "efx.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#if EFSYS_OPT_MON_MCDI + +#if EFSYS_OPT_MON_STATS + + __checkReturn efx_rc_t +mcdi_mon_cfg_build( + __in efx_nic_t *enp); + + void +mcdi_mon_cfg_free( + __in efx_nic_t *enp); + + +extern __checkReturn efx_rc_t +mcdi_mon_ev( + __in efx_nic_t *enp, + __in efx_qword_t *eqp, + __out efx_mon_stat_t *idp, + __out efx_mon_stat_value_t *valuep); + +extern __checkReturn efx_rc_t +mcdi_mon_stats_update( + __in efx_nic_t *enp, + __in efsys_mem_t *esmp, + __inout_ecount(EFX_MON_NSTATS) efx_mon_stat_value_t *values); + +#endif /* EFSYS_OPT_MON_STATS */ + +#endif /* EFSYS_OPT_MON_MCDI */ + +#ifdef __cplusplus +} +#endif + +#endif /* _SYS_MCDI_MON_H */ diff --git a/dpdk/drivers/net/sfc/base/medford_impl.h b/dpdk/drivers/net/sfc/base/medford_impl.h new file mode 100644 index 00000000..de2f5cf0 --- /dev/null +++ b/dpdk/drivers/net/sfc/base/medford_impl.h @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2015-2016 Solarflare Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are + * those of the authors and should not be interpreted as representing official + * policies, either expressed or implied, of the FreeBSD Project. + */ + +#ifndef _SYS_MEDFORD_IMPL_H +#define _SYS_MEDFORD_IMPL_H + +#ifdef __cplusplus +extern "C" { +#endif + +/* Alignment requirement for value written to RX WPTR: + * the WPTR must be aligned to an 8 descriptor boundary + * + * FIXME: Is this the same on Medford as Huntington? + */ +#define MEDFORD_RX_WPTR_ALIGN 8 + + + +#ifndef ER_EZ_TX_PIOBUF_SIZE +#define ER_EZ_TX_PIOBUF_SIZE 4096 +#endif + + +#define MEDFORD_PIOBUF_NBUFS (16) +#define MEDFORD_PIOBUF_SIZE (ER_EZ_TX_PIOBUF_SIZE) + +#define MEDFORD_MIN_PIO_ALLOC_SIZE (MEDFORD_PIOBUF_SIZE / 32) + + +extern __checkReturn efx_rc_t +medford_board_cfg( + __in efx_nic_t *enp); + + +#ifdef __cplusplus +} +#endif + +#endif /* _SYS_MEDFORD_IMPL_H */ diff --git a/dpdk/drivers/net/sfc/base/medford_nic.c b/dpdk/drivers/net/sfc/base/medford_nic.c new file mode 100644 index 00000000..d361d654 --- /dev/null +++ b/dpdk/drivers/net/sfc/base/medford_nic.c @@ -0,0 +1,409 @@ +/* + * Copyright (c) 2015-2016 Solarflare Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are + * those of the authors and should not be interpreted as representing official + * policies, either expressed or implied, of the FreeBSD Project. + */ + +#include "efx.h" +#include "efx_impl.h" + + +#if EFSYS_OPT_MEDFORD + +static __checkReturn efx_rc_t +efx_mcdi_get_rxdp_config( + __in efx_nic_t *enp, + __out uint32_t *end_paddingp) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_GET_RXDP_CONFIG_IN_LEN, + MC_CMD_GET_RXDP_CONFIG_OUT_LEN)]; + uint32_t end_padding; + efx_rc_t rc; + + memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_GET_RXDP_CONFIG; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_GET_RXDP_CONFIG_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_GET_RXDP_CONFIG_OUT_LEN; + + efx_mcdi_execute(enp, &req); + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + if (MCDI_OUT_DWORD_FIELD(req, GET_RXDP_CONFIG_OUT_DATA, + GET_RXDP_CONFIG_OUT_PAD_HOST_DMA) == 0) { + /* RX DMA end padding is disabled */ + end_padding = 0; + } else { + switch (MCDI_OUT_DWORD_FIELD(req, GET_RXDP_CONFIG_OUT_DATA, + GET_RXDP_CONFIG_OUT_PAD_HOST_LEN)) { + case MC_CMD_SET_RXDP_CONFIG_IN_PAD_HOST_64: + end_padding = 64; + break; + case MC_CMD_SET_RXDP_CONFIG_IN_PAD_HOST_128: + end_padding = 128; + break; + case MC_CMD_SET_RXDP_CONFIG_IN_PAD_HOST_256: + end_padding = 256; + break; + default: + rc = ENOTSUP; + goto fail2; + } + } + + *end_paddingp = end_padding; + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +static __checkReturn efx_rc_t +medford_nic_get_required_pcie_bandwidth( + __in efx_nic_t *enp, + __out uint32_t *bandwidth_mbpsp) +{ + uint32_t port_modes; + uint32_t current_mode; + uint32_t bandwidth; + efx_rc_t rc; + + if ((rc = efx_mcdi_get_port_modes(enp, &port_modes, + ¤t_mode)) != 0) { + /* No port mode info available. */ + bandwidth = 0; + goto out; + } + + if ((rc = ef10_nic_get_port_mode_bandwidth(current_mode, + &bandwidth)) != 0) + goto fail1; + +out: + *bandwidth_mbpsp = bandwidth; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +medford_board_cfg( + __in efx_nic_t *enp) +{ + efx_mcdi_iface_t *emip = &(enp->en_mcdi.em_emip); + efx_nic_cfg_t *encp = &(enp->en_nic_cfg); + uint8_t mac_addr[6] = { 0 }; + uint32_t board_type = 0; + ef10_link_state_t els; + efx_port_t *epp = &(enp->en_port); + uint32_t port; + uint32_t pf; + uint32_t vf; + uint32_t mask; + uint32_t sysclk, dpcpu_clk; + uint32_t base, nvec; + uint32_t end_padding; + uint32_t bandwidth; + efx_rc_t rc; + + /* + * FIXME: Likely to be incomplete and incorrect. + * Parts of this should be shared with Huntington. + */ + + if ((rc = efx_mcdi_get_port_assignment(enp, &port)) != 0) + goto fail1; + + /* + * NOTE: The MCDI protocol numbers ports from zero. + * The common code MCDI interface numbers ports from one. + */ + emip->emi_port = port + 1; + + if ((rc = ef10_external_port_mapping(enp, port, + &encp->enc_external_port)) != 0) + goto fail2; + + /* + * Get PCIe function number from firmware (used for + * per-function privilege and dynamic config info). + * - PCIe PF: pf = PF number, vf = 0xffff. + * - PCIe VF: pf = parent PF, vf = VF number. + */ + if ((rc = efx_mcdi_get_function_info(enp, &pf, &vf)) != 0) + goto fail3; + + encp->enc_pf = pf; + encp->enc_vf = vf; + + /* MAC address for this function */ + if (EFX_PCI_FUNCTION_IS_PF(encp)) { + rc = efx_mcdi_get_mac_address_pf(enp, mac_addr); +#if EFSYS_OPT_ALLOW_UNCONFIGURED_NIC + /* Disable static config checking for Medford NICs, ONLY + * for manufacturing test and setup at the factory, to + * allow the static config to be installed. + */ +#else /* EFSYS_OPT_ALLOW_UNCONFIGURED_NIC */ + if ((rc == 0) && (mac_addr[0] & 0x02)) { + /* + * If the static config does not include a global MAC + * address pool then the board may return a locally + * administered MAC address (this should only happen on + * incorrectly programmed boards). + */ + rc = EINVAL; + } +#endif /* EFSYS_OPT_ALLOW_UNCONFIGURED_NIC */ + } else { + rc = efx_mcdi_get_mac_address_vf(enp, mac_addr); + } + if (rc != 0) + goto fail4; + + EFX_MAC_ADDR_COPY(encp->enc_mac_addr, mac_addr); + + /* Board configuration */ + rc = efx_mcdi_get_board_cfg(enp, &board_type, NULL, NULL); + if (rc != 0) { + /* Unprivileged functions may not be able to read board cfg */ + if (rc == EACCES) + board_type = 0; + else + goto fail5; + } + + encp->enc_board_type = board_type; + encp->enc_clk_mult = 1; /* not used for Medford */ + + /* Fill out fields in enp->en_port and enp->en_nic_cfg from MCDI */ + if ((rc = efx_mcdi_get_phy_cfg(enp)) != 0) + goto fail6; + + /* Obtain the default PHY advertised capabilities */ + if ((rc = ef10_phy_get_link(enp, &els)) != 0) + goto fail7; + epp->ep_default_adv_cap_mask = els.els_adv_cap_mask; + epp->ep_adv_cap_mask = els.els_adv_cap_mask; + + /* + * Enable firmware workarounds for hardware errata. + * Expected responses are: + * - 0 (zero): + * Success: workaround enabled or disabled as requested. + * - MC_CMD_ERR_ENOSYS (reported as ENOTSUP): + * Firmware does not support the MC_CMD_WORKAROUND request. + * (assume that the workaround is not supported). + * - MC_CMD_ERR_ENOENT (reported as ENOENT): + * Firmware does not support the requested workaround. + * - MC_CMD_ERR_EPERM (reported as EACCES): + * Unprivileged function cannot enable/disable workarounds. + * + * See efx_mcdi_request_errcode() for MCDI error translations. + */ + + + if (EFX_PCI_FUNCTION_IS_VF(encp)) { + /* + * Interrupt testing does not work for VFs. See bug50084. + * FIXME: Does this still apply to Medford? + */ + encp->enc_bug41750_workaround = B_TRUE; + } + + /* Chained multicast is always enabled on Medford */ + encp->enc_bug26807_workaround = B_TRUE; + + /* + * If the bug61265 workaround is enabled, then interrupt holdoff timers + * cannot be controlled by timer table writes, so MCDI must be used + * (timer table writes can still be used for wakeup timers). + */ + rc = efx_mcdi_set_workaround(enp, MC_CMD_WORKAROUND_BUG61265, B_TRUE, + NULL); + if ((rc == 0) || (rc == EACCES)) + encp->enc_bug61265_workaround = B_TRUE; + else if ((rc == ENOTSUP) || (rc == ENOENT)) + encp->enc_bug61265_workaround = B_FALSE; + else + goto fail8; + + /* Get clock frequencies (in MHz). */ + if ((rc = efx_mcdi_get_clock(enp, &sysclk, &dpcpu_clk)) != 0) + goto fail9; + + /* + * The Medford timer quantum is 1536 dpcpu_clk cycles, documented for + * the EV_TMR_VAL field of EV_TIMER_TBL. Scale for MHz and ns units. + */ + encp->enc_evq_timer_quantum_ns = 1536000UL / dpcpu_clk; /* 1536 cycles */ + encp->enc_evq_timer_max_us = (encp->enc_evq_timer_quantum_ns << + FRF_CZ_TC_TIMER_VAL_WIDTH) / 1000; + + /* Check capabilities of running datapath firmware */ + if ((rc = ef10_get_datapath_caps(enp)) != 0) + goto fail10; + + /* Alignment for receive packet DMA buffers */ + encp->enc_rx_buf_align_start = 1; + + /* Get the RX DMA end padding alignment configuration */ + if ((rc = efx_mcdi_get_rxdp_config(enp, &end_padding)) != 0) { + if (rc != EACCES) + goto fail11; + + /* Assume largest tail padding size supported by hardware */ + end_padding = 256; + } + encp->enc_rx_buf_align_end = end_padding; + + /* Alignment for WPTR updates */ + encp->enc_rx_push_align = EF10_RX_WPTR_ALIGN; + + /* + * Maximum number of exclusive RSS contexts which can be allocated. The + * hardware supports 64, but 6 are reserved for shared contexts. They + * are a global resource so not all may be available. + */ + encp->enc_rx_scale_max_exclusive_contexts = 58; + + encp->enc_tx_dma_desc_size_max = EFX_MASK32(ESF_DZ_RX_KER_BYTE_CNT); + /* No boundary crossing limits */ + encp->enc_tx_dma_desc_boundary = 0; + + /* + * Set resource limits for MC_CMD_ALLOC_VIS. Note that we cannot use + * MC_CMD_GET_RESOURCE_LIMITS here as that reports the available + * resources (allocated to this PCIe function), which is zero until + * after we have allocated VIs. + */ + encp->enc_evq_limit = 1024; + encp->enc_rxq_limit = EFX_RXQ_LIMIT_TARGET; + encp->enc_txq_limit = EFX_TXQ_LIMIT_TARGET; + + /* + * The maximum supported transmit queue size is 2048. TXQs with 4096 + * descriptors are not supported as the top bit is used for vfifo + * stuffing. + */ + encp->enc_txq_max_ndescs = 2048; + + encp->enc_buftbl_limit = 0xFFFFFFFF; + + encp->enc_piobuf_limit = MEDFORD_PIOBUF_NBUFS; + encp->enc_piobuf_size = MEDFORD_PIOBUF_SIZE; + encp->enc_piobuf_min_alloc_size = MEDFORD_MIN_PIO_ALLOC_SIZE; + + /* + * Get the current privilege mask. Note that this may be modified + * dynamically, so this value is informational only. DO NOT use + * the privilege mask to check for sufficient privileges, as that + * can result in time-of-check/time-of-use bugs. + */ + if ((rc = ef10_get_privilege_mask(enp, &mask)) != 0) + goto fail12; + encp->enc_privilege_mask = mask; + + /* Get interrupt vector limits */ + if ((rc = efx_mcdi_get_vector_cfg(enp, &base, &nvec, NULL)) != 0) { + if (EFX_PCI_FUNCTION_IS_PF(encp)) + goto fail13; + + /* Ignore error (cannot query vector limits from a VF). */ + base = 0; + nvec = 1024; + } + encp->enc_intr_vec_base = base; + encp->enc_intr_limit = nvec; + + /* + * Maximum number of bytes into the frame the TCP header can start for + * firmware assisted TSO to work. + */ + encp->enc_tx_tso_tcp_header_offset_limit = EF10_TCP_HEADER_OFFSET_LIMIT; + + /* + * Medford stores a single global copy of VPD, not per-PF as on + * Huntington. + */ + encp->enc_vpd_is_global = B_TRUE; + + rc = medford_nic_get_required_pcie_bandwidth(enp, &bandwidth); + if (rc != 0) + goto fail14; + encp->enc_required_pcie_bandwidth_mbps = bandwidth; + encp->enc_max_pcie_link_gen = EFX_PCIE_LINK_SPEED_GEN3; + + return (0); + +fail14: + EFSYS_PROBE(fail14); +fail13: + EFSYS_PROBE(fail13); +fail12: + EFSYS_PROBE(fail12); +fail11: + EFSYS_PROBE(fail11); +fail10: + EFSYS_PROBE(fail10); +fail9: + EFSYS_PROBE(fail9); +fail8: + EFSYS_PROBE(fail8); +fail7: + EFSYS_PROBE(fail7); +fail6: + EFSYS_PROBE(fail6); +fail5: + EFSYS_PROBE(fail5); +fail4: + EFSYS_PROBE(fail4); +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +#endif /* EFSYS_OPT_MEDFORD */ diff --git a/dpdk/drivers/net/sfc/base/siena_flash.h b/dpdk/drivers/net/sfc/base/siena_flash.h new file mode 100644 index 00000000..e2700554 --- /dev/null +++ b/dpdk/drivers/net/sfc/base/siena_flash.h @@ -0,0 +1,215 @@ +/* + * Copyright (c) 2007-2016 Solarflare Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are + * those of the authors and should not be interpreted as representing official + * policies, either expressed or implied, of the FreeBSD Project. + */ + +#ifndef _SYS_SIENA_FLASH_H +#define _SYS_SIENA_FLASH_H + +#pragma pack(1) + +/* Fixed locations near the start of flash (which may be in the internal PHY + * firmware header) point to the boot header. + * + * - parsed by MC boot ROM and firmware + * - reserved (but not parsed) by PHY firmware + * - opaque to driver + */ + +#define SIENA_MC_BOOT_PHY_FW_HDR_LEN (0x20) + +#define SIENA_MC_BOOT_PTR_LOCATION (0x18) /* First thing we try to boot */ +#define SIENA_MC_BOOT_ALT_PTR_LOCATION (0x1c) /* Alternative if that fails */ + +#define SIENA_MC_BOOT_HDR_LEN (0x200) + +#define SIENA_MC_BOOT_MAGIC (0x51E4A001) +#define SIENA_MC_BOOT_VERSION (1) + + +/*Structures supporting an arbitrary number of binary blobs in the flash image + intended to house code and tables for the satellite cpus*/ +/*thanks to random.org for:*/ +#define BLOBS_HEADER_MAGIC (0xBDA3BBD4) +#define BLOB_HEADER_MAGIC (0xA1478A91) + +typedef struct blobs_hdr_s { /* GENERATED BY scripts/genfwdef */ + efx_dword_t magic; + efx_dword_t no_of_blobs; +} blobs_hdr_t; + +typedef struct blob_hdr_s { /* GENERATED BY scripts/genfwdef */ + efx_dword_t magic; + efx_dword_t cpu_type; + efx_dword_t build_variant; + efx_dword_t offset; + efx_dword_t length; + efx_dword_t checksum; +} blob_hdr_t; + +#define BLOB_CPU_TYPE_TXDI_TEXT (0) +#define BLOB_CPU_TYPE_RXDI_TEXT (1) +#define BLOB_CPU_TYPE_TXDP_TEXT (2) +#define BLOB_CPU_TYPE_RXDP_TEXT (3) +#define BLOB_CPU_TYPE_RXHRSL_HR_LUT (4) +#define BLOB_CPU_TYPE_RXHRSL_HR_LUT_CFG (5) +#define BLOB_CPU_TYPE_TXHRSL_HR_LUT (6) +#define BLOB_CPU_TYPE_TXHRSL_HR_LUT_CFG (7) +#define BLOB_CPU_TYPE_RXHRSL_HR_PGM (8) +#define BLOB_CPU_TYPE_RXHRSL_SL_PGM (9) +#define BLOB_CPU_TYPE_TXHRSL_HR_PGM (10) +#define BLOB_CPU_TYPE_TXHRSL_SL_PGM (11) +#define BLOB_CPU_TYPE_RXDI_VTBL0 (12) +#define BLOB_CPU_TYPE_TXDI_VTBL0 (13) +#define BLOB_CPU_TYPE_RXDI_VTBL1 (14) +#define BLOB_CPU_TYPE_TXDI_VTBL1 (15) +#define BLOB_CPU_TYPE_DUMPSPEC (32) +#define BLOB_CPU_TYPE_MC_XIP (33) + +#define BLOB_CPU_TYPE_INVALID (31) + +/* + * The upper four bits of the CPU type field specify the compression + * algorithm used for this blob. + */ +#define BLOB_COMPRESSION_MASK (0xf0000000) +#define BLOB_CPU_TYPE_MASK (0x0fffffff) + +#define BLOB_COMPRESSION_NONE (0x00000000) /* Stored as is */ +#define BLOB_COMPRESSION_LZ (0x10000000) /* see lib/lzdecoder.c */ + +typedef struct siena_mc_boot_hdr_s { /* GENERATED BY scripts/genfwdef */ + efx_dword_t magic; /* = SIENA_MC_BOOT_MAGIC */ + efx_word_t hdr_version; /* this structure definition is version 1 */ + efx_byte_t board_type; + efx_byte_t firmware_version_a; + efx_byte_t firmware_version_b; + efx_byte_t firmware_version_c; + efx_word_t checksum; /* of whole header area + firmware image */ + efx_word_t firmware_version_d; + efx_byte_t mcfw_subtype; + efx_byte_t generation; /* Valid for medford, SBZ for earlier chips */ + efx_dword_t firmware_text_offset; /* offset to firmware .text */ + efx_dword_t firmware_text_size; /* length of firmware .text, in bytes */ + efx_dword_t firmware_data_offset; /* offset to firmware .data */ + efx_dword_t firmware_data_size; /* length of firmware .data, in bytes */ + efx_byte_t spi_rate; /* SPI rate for reading image, 0 is BootROM default */ + efx_byte_t spi_phase_adj; /* SPI SDO/SCL phase adjustment, 0 is default (no adj) */ + efx_word_t xpm_sector; /* The sector that contains the key, or 0xffff if unsigned (medford) SBZ (earlier) */ + efx_dword_t reserved_c[7]; /* (set to 0) */ +} siena_mc_boot_hdr_t; + +#define SIENA_MC_BOOT_HDR_PADDING \ + (SIENA_MC_BOOT_HDR_LEN - sizeof(siena_mc_boot_hdr_t)) + +#define SIENA_MC_STATIC_CONFIG_MAGIC (0xBDCF5555) +#define SIENA_MC_STATIC_CONFIG_VERSION (0) + +typedef struct siena_mc_static_config_hdr_s { /* GENERATED BY scripts/genfwdef */ + efx_dword_t magic; /* = SIENA_MC_STATIC_CONFIG_MAGIC */ + efx_word_t length; /* of header area (i.e. not including VPD) */ + efx_byte_t version; + efx_byte_t csum; /* over header area (i.e. not including VPD) */ + efx_dword_t static_vpd_offset; + efx_dword_t static_vpd_length; + efx_dword_t capabilities; + efx_byte_t mac_addr_base[6]; + efx_byte_t green_mode_cal; /* Green mode calibration result */ + efx_byte_t green_mode_valid; /* Whether cal holds a valid value */ + efx_word_t mac_addr_count; + efx_word_t mac_addr_stride; + efx_word_t calibrated_vref; /* Vref as measured during production */ + efx_word_t adc_vref; /* Vref as read by ADC */ + efx_dword_t reserved2[1]; /* (write as zero) */ + efx_dword_t num_dbi_items; + struct { + efx_word_t addr; + efx_word_t byte_enables; + efx_dword_t value; + } dbi[]; +} siena_mc_static_config_hdr_t; + +/* This prefixes a valid XIP partition */ +#define XIP_PARTITION_MAGIC (0x51DEC0DE) + +#define SIENA_MC_DYNAMIC_CONFIG_MAGIC (0xBDCFDDDD) +#define SIENA_MC_DYNAMIC_CONFIG_VERSION (0) + +typedef struct siena_mc_fw_version_s { /* GENERATED BY scripts/genfwdef */ + efx_dword_t fw_subtype; + efx_word_t version_w; + efx_word_t version_x; + efx_word_t version_y; + efx_word_t version_z; +} siena_mc_fw_version_t; + +typedef struct siena_mc_dynamic_config_hdr_s { /* GENERATED BY scripts/genfwdef */ + efx_dword_t magic; /* = SIENA_MC_DYNAMIC_CONFIG_MAGIC */ + efx_word_t length; /* of header area (i.e. not including VPD) */ + efx_byte_t version; + efx_byte_t csum; /* over header area (i.e. not including VPD) */ + efx_dword_t dynamic_vpd_offset; + efx_dword_t dynamic_vpd_length; + efx_dword_t num_fw_version_items; + siena_mc_fw_version_t fw_version[]; +} siena_mc_dynamic_config_hdr_t; + +#define SIENA_MC_EXPROM_SINGLE_MAGIC (0xAA55) /* little-endian uint16_t */ + +#define SIENA_MC_EXPROM_COMBO_MAGIC (0xB0070102) /* little-endian uint32_t */ +#define SIENA_MC_EXPROM_COMBO_V2_MAGIC (0xB0070103) /* little-endian uint32_t */ + +typedef struct siena_mc_combo_rom_hdr_s { /* GENERATED BY scripts/genfwdef */ + efx_dword_t magic; /* = SIENA_MC_EXPROM_COMBO_MAGIC or SIENA_MC_EXPROM_COMBO_V2_MAGIC */ + union { + struct { + efx_dword_t len1; /* length of first image */ + efx_dword_t len2; /* length of second image */ + efx_dword_t off1; /* offset of first byte to edit to combine images */ + efx_dword_t off2; /* offset of second byte to edit to combine images */ + efx_word_t infoblk0_off;/* infoblk offset */ + efx_word_t infoblk1_off;/* infoblk offset */ + efx_byte_t infoblk_len;/* length of space reserved for one infoblk structure */ + efx_byte_t reserved[7];/* (set to 0) */ + } v1; + struct { + efx_dword_t len1; /* length of first image */ + efx_dword_t len2; /* length of second image */ + efx_dword_t off1; /* offset of first byte to edit to combine images */ + efx_dword_t off2; /* offset of second byte to edit to combine images */ + efx_word_t infoblk_off;/* infoblk start offset */ + efx_word_t infoblk_count;/* infoblk count */ + efx_byte_t infoblk_len;/* length of space reserved for one infoblk structure */ + efx_byte_t reserved[7];/* (set to 0) */ + } v2; + } data; +} siena_mc_combo_rom_hdr_t; + +#pragma pack() + +#endif /* _SYS_SIENA_FLASH_H */ diff --git a/dpdk/drivers/net/sfc/base/siena_impl.h b/dpdk/drivers/net/sfc/base/siena_impl.h new file mode 100644 index 00000000..ea6de983 --- /dev/null +++ b/dpdk/drivers/net/sfc/base/siena_impl.h @@ -0,0 +1,431 @@ +/* + * Copyright (c) 2009-2016 Solarflare Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are + * those of the authors and should not be interpreted as representing official + * policies, either expressed or implied, of the FreeBSD Project. + */ + +#ifndef _SYS_SIENA_IMPL_H +#define _SYS_SIENA_IMPL_H + +#include "efx.h" +#include "efx_regs.h" +#include "efx_mcdi.h" +#include "siena_flash.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define SIENA_NVRAM_CHUNK 0x80 + +extern __checkReturn efx_rc_t +siena_nic_probe( + __in efx_nic_t *enp); + +extern __checkReturn efx_rc_t +siena_nic_reset( + __in efx_nic_t *enp); + +extern __checkReturn efx_rc_t +siena_nic_init( + __in efx_nic_t *enp); + +#if EFSYS_OPT_DIAG + +extern __checkReturn efx_rc_t +siena_nic_register_test( + __in efx_nic_t *enp); + +#endif /* EFSYS_OPT_DIAG */ + +extern void +siena_nic_fini( + __in efx_nic_t *enp); + +extern void +siena_nic_unprobe( + __in efx_nic_t *enp); + +#define SIENA_SRAM_ROWS 0x12000 + +extern void +siena_sram_init( + __in efx_nic_t *enp); + +#if EFSYS_OPT_DIAG + +extern __checkReturn efx_rc_t +siena_sram_test( + __in efx_nic_t *enp, + __in efx_sram_pattern_fn_t func); + +#endif /* EFSYS_OPT_DIAG */ + +#if EFSYS_OPT_MCDI + +extern __checkReturn efx_rc_t +siena_mcdi_init( + __in efx_nic_t *enp, + __in const efx_mcdi_transport_t *mtp); + +extern void +siena_mcdi_send_request( + __in efx_nic_t *enp, + __in_bcount(hdr_len) void *hdrp, + __in size_t hdr_len, + __in_bcount(sdu_len) void *sdup, + __in size_t sdu_len); + +extern __checkReturn boolean_t +siena_mcdi_poll_response( + __in efx_nic_t *enp); + +extern void +siena_mcdi_read_response( + __in efx_nic_t *enp, + __out_bcount(length) void *bufferp, + __in size_t offset, + __in size_t length); + +extern efx_rc_t +siena_mcdi_poll_reboot( + __in efx_nic_t *enp); + +extern void +siena_mcdi_fini( + __in efx_nic_t *enp); + +extern __checkReturn efx_rc_t +siena_mcdi_feature_supported( + __in efx_nic_t *enp, + __in efx_mcdi_feature_id_t id, + __out boolean_t *supportedp); + +extern void +siena_mcdi_get_timeout( + __in efx_nic_t *enp, + __in efx_mcdi_req_t *emrp, + __out uint32_t *timeoutp); + +#endif /* EFSYS_OPT_MCDI */ + +#if EFSYS_OPT_NVRAM || EFSYS_OPT_VPD + +extern __checkReturn efx_rc_t +siena_nvram_partn_lock( + __in efx_nic_t *enp, + __in uint32_t partn); + +extern __checkReturn efx_rc_t +siena_nvram_partn_unlock( + __in efx_nic_t *enp, + __in uint32_t partn); + +extern __checkReturn efx_rc_t +siena_nvram_get_dynamic_cfg( + __in efx_nic_t *enp, + __in uint32_t partn, + __in boolean_t vpd, + __out siena_mc_dynamic_config_hdr_t **dcfgp, + __out size_t *sizep); + +#endif /* EFSYS_OPT_VPD || EFSYS_OPT_NVRAM */ + +#if EFSYS_OPT_NVRAM + +#if EFSYS_OPT_DIAG + +extern __checkReturn efx_rc_t +siena_nvram_test( + __in efx_nic_t *enp); + +#endif /* EFSYS_OPT_DIAG */ + +extern __checkReturn efx_rc_t +siena_nvram_get_subtype( + __in efx_nic_t *enp, + __in uint32_t partn, + __out uint32_t *subtypep); + +extern __checkReturn efx_rc_t +siena_nvram_type_to_partn( + __in efx_nic_t *enp, + __in efx_nvram_type_t type, + __out uint32_t *partnp); + +extern __checkReturn efx_rc_t +siena_nvram_partn_size( + __in efx_nic_t *enp, + __in uint32_t partn, + __out size_t *sizep); + +extern __checkReturn efx_rc_t +siena_nvram_partn_rw_start( + __in efx_nic_t *enp, + __in uint32_t partn, + __out size_t *chunk_sizep); + +extern __checkReturn efx_rc_t +siena_nvram_partn_read( + __in efx_nic_t *enp, + __in uint32_t partn, + __in unsigned int offset, + __out_bcount(size) caddr_t data, + __in size_t size); + +extern __checkReturn efx_rc_t +siena_nvram_partn_erase( + __in efx_nic_t *enp, + __in uint32_t partn, + __in unsigned int offset, + __in size_t size); + +extern __checkReturn efx_rc_t +siena_nvram_partn_write( + __in efx_nic_t *enp, + __in uint32_t partn, + __in unsigned int offset, + __out_bcount(size) caddr_t data, + __in size_t size); + +extern __checkReturn efx_rc_t +siena_nvram_partn_rw_finish( + __in efx_nic_t *enp, + __in uint32_t partn); + +extern __checkReturn efx_rc_t +siena_nvram_partn_get_version( + __in efx_nic_t *enp, + __in uint32_t partn, + __out uint32_t *subtypep, + __out_ecount(4) uint16_t version[4]); + +extern __checkReturn efx_rc_t +siena_nvram_partn_set_version( + __in efx_nic_t *enp, + __in uint32_t partn, + __in_ecount(4) uint16_t version[4]); + +#endif /* EFSYS_OPT_NVRAM */ + +#if EFSYS_OPT_VPD + +extern __checkReturn efx_rc_t +siena_vpd_init( + __in efx_nic_t *enp); + +extern __checkReturn efx_rc_t +siena_vpd_size( + __in efx_nic_t *enp, + __out size_t *sizep); + +extern __checkReturn efx_rc_t +siena_vpd_read( + __in efx_nic_t *enp, + __out_bcount(size) caddr_t data, + __in size_t size); + +extern __checkReturn efx_rc_t +siena_vpd_verify( + __in efx_nic_t *enp, + __in_bcount(size) caddr_t data, + __in size_t size); + +extern __checkReturn efx_rc_t +siena_vpd_reinit( + __in efx_nic_t *enp, + __in_bcount(size) caddr_t data, + __in size_t size); + +extern __checkReturn efx_rc_t +siena_vpd_get( + __in efx_nic_t *enp, + __in_bcount(size) caddr_t data, + __in size_t size, + __inout efx_vpd_value_t *evvp); + +extern __checkReturn efx_rc_t +siena_vpd_set( + __in efx_nic_t *enp, + __in_bcount(size) caddr_t data, + __in size_t size, + __in efx_vpd_value_t *evvp); + +extern __checkReturn efx_rc_t +siena_vpd_next( + __in efx_nic_t *enp, + __in_bcount(size) caddr_t data, + __in size_t size, + __out efx_vpd_value_t *evvp, + __inout unsigned int *contp); + +extern __checkReturn efx_rc_t +siena_vpd_write( + __in efx_nic_t *enp, + __in_bcount(size) caddr_t data, + __in size_t size); + +extern void +siena_vpd_fini( + __in efx_nic_t *enp); + +#endif /* EFSYS_OPT_VPD */ + +typedef struct siena_link_state_s { + uint32_t sls_adv_cap_mask; + uint32_t sls_lp_cap_mask; + unsigned int sls_fcntl; + efx_link_mode_t sls_link_mode; +#if EFSYS_OPT_LOOPBACK + efx_loopback_type_t sls_loopback; +#endif + boolean_t sls_mac_up; +} siena_link_state_t; + +extern void +siena_phy_link_ev( + __in efx_nic_t *enp, + __in efx_qword_t *eqp, + __out efx_link_mode_t *link_modep); + +extern __checkReturn efx_rc_t +siena_phy_get_link( + __in efx_nic_t *enp, + __out siena_link_state_t *slsp); + +extern __checkReturn efx_rc_t +siena_phy_power( + __in efx_nic_t *enp, + __in boolean_t on); + +extern __checkReturn efx_rc_t +siena_phy_reconfigure( + __in efx_nic_t *enp); + +extern __checkReturn efx_rc_t +siena_phy_verify( + __in efx_nic_t *enp); + +extern __checkReturn efx_rc_t +siena_phy_oui_get( + __in efx_nic_t *enp, + __out uint32_t *ouip); + +#if EFSYS_OPT_PHY_STATS + +extern void +siena_phy_decode_stats( + __in efx_nic_t *enp, + __in uint32_t vmask, + __in_opt efsys_mem_t *esmp, + __out_opt uint64_t *smaskp, + __inout_ecount_opt(EFX_PHY_NSTATS) uint32_t *stat); + +extern __checkReturn efx_rc_t +siena_phy_stats_update( + __in efx_nic_t *enp, + __in efsys_mem_t *esmp, + __inout_ecount(EFX_PHY_NSTATS) uint32_t *stat); + +#endif /* EFSYS_OPT_PHY_STATS */ + +#if EFSYS_OPT_BIST + +extern __checkReturn efx_rc_t +siena_phy_bist_start( + __in efx_nic_t *enp, + __in efx_bist_type_t type); + +extern __checkReturn efx_rc_t +siena_phy_bist_poll( + __in efx_nic_t *enp, + __in efx_bist_type_t type, + __out efx_bist_result_t *resultp, + __out_opt __drv_when(count > 0, __notnull) + uint32_t *value_maskp, + __out_ecount_opt(count) __drv_when(count > 0, __notnull) + unsigned long *valuesp, + __in size_t count); + +extern void +siena_phy_bist_stop( + __in efx_nic_t *enp, + __in efx_bist_type_t type); + +#endif /* EFSYS_OPT_BIST */ + +extern __checkReturn efx_rc_t +siena_mac_poll( + __in efx_nic_t *enp, + __out efx_link_mode_t *link_modep); + +extern __checkReturn efx_rc_t +siena_mac_up( + __in efx_nic_t *enp, + __out boolean_t *mac_upp); + +extern __checkReturn efx_rc_t +siena_mac_reconfigure( + __in efx_nic_t *enp); + +extern __checkReturn efx_rc_t +siena_mac_pdu_get( + __in efx_nic_t *enp, + __out size_t *pdu); + +#if EFSYS_OPT_LOOPBACK + +extern __checkReturn efx_rc_t +siena_mac_loopback_set( + __in efx_nic_t *enp, + __in efx_link_mode_t link_mode, + __in efx_loopback_type_t loopback_type); + +#endif /* EFSYS_OPT_LOOPBACK */ + +#if EFSYS_OPT_MAC_STATS + +extern __checkReturn efx_rc_t +siena_mac_stats_get_mask( + __in efx_nic_t *enp, + __inout_bcount(mask_size) uint32_t *maskp, + __in size_t mask_size); + +extern __checkReturn efx_rc_t +siena_mac_stats_update( + __in efx_nic_t *enp, + __in efsys_mem_t *esmp, + __inout_ecount(EFX_MAC_NSTATS) efsys_stat_t *stat, + __inout_opt uint32_t *generationp); + +#endif /* EFSYS_OPT_MAC_STATS */ + +#ifdef __cplusplus +} +#endif + +#endif /* _SYS_SIENA_IMPL_H */ diff --git a/dpdk/drivers/net/sfc/base/siena_mac.c b/dpdk/drivers/net/sfc/base/siena_mac.c new file mode 100644 index 00000000..29bbff8a --- /dev/null +++ b/dpdk/drivers/net/sfc/base/siena_mac.c @@ -0,0 +1,476 @@ +/* + * Copyright (c) 2009-2016 Solarflare Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are + * those of the authors and should not be interpreted as representing official + * policies, either expressed or implied, of the FreeBSD Project. + */ + +#include "efx.h" +#include "efx_impl.h" + +#if EFSYS_OPT_SIENA + + __checkReturn efx_rc_t +siena_mac_poll( + __in efx_nic_t *enp, + __out efx_link_mode_t *link_modep) +{ + efx_port_t *epp = &(enp->en_port); + siena_link_state_t sls; + efx_rc_t rc; + + if ((rc = siena_phy_get_link(enp, &sls)) != 0) + goto fail1; + + epp->ep_adv_cap_mask = sls.sls_adv_cap_mask; + epp->ep_fcntl = sls.sls_fcntl; + + *link_modep = sls.sls_link_mode; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + *link_modep = EFX_LINK_UNKNOWN; + + return (rc); +} + + __checkReturn efx_rc_t +siena_mac_up( + __in efx_nic_t *enp, + __out boolean_t *mac_upp) +{ + siena_link_state_t sls; + efx_rc_t rc; + + /* + * Because Siena doesn't *require* polling, we can't rely on + * siena_mac_poll() being executed to populate epp->ep_mac_up. + */ + if ((rc = siena_phy_get_link(enp, &sls)) != 0) + goto fail1; + + *mac_upp = sls.sls_mac_up; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +siena_mac_reconfigure( + __in efx_nic_t *enp) +{ + efx_port_t *epp = &(enp->en_port); + efx_oword_t multicast_hash[2]; + efx_mcdi_req_t req; + uint8_t payload[MAX(MAX(MC_CMD_SET_MAC_IN_LEN, + MC_CMD_SET_MAC_OUT_LEN), + MAX(MC_CMD_SET_MCAST_HASH_IN_LEN, + MC_CMD_SET_MCAST_HASH_OUT_LEN))]; + unsigned int fcntl; + efx_rc_t rc; + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_SET_MAC; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_SET_MAC_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_SET_MAC_OUT_LEN; + + MCDI_IN_SET_DWORD(req, SET_MAC_IN_MTU, epp->ep_mac_pdu); + MCDI_IN_SET_DWORD(req, SET_MAC_IN_DRAIN, epp->ep_mac_drain ? 1 : 0); + EFX_MAC_ADDR_COPY(MCDI_IN2(req, uint8_t, SET_MAC_IN_ADDR), + epp->ep_mac_addr); + MCDI_IN_POPULATE_DWORD_2(req, SET_MAC_IN_REJECT, + SET_MAC_IN_REJECT_UNCST, !epp->ep_all_unicst, + SET_MAC_IN_REJECT_BRDCST, !epp->ep_brdcst); + + if (epp->ep_fcntl_autoneg) + /* efx_fcntl_set() has already set the phy capabilities */ + fcntl = MC_CMD_FCNTL_AUTO; + else if (epp->ep_fcntl & EFX_FCNTL_RESPOND) + fcntl = (epp->ep_fcntl & EFX_FCNTL_GENERATE) + ? MC_CMD_FCNTL_BIDIR + : MC_CMD_FCNTL_RESPOND; + else + fcntl = MC_CMD_FCNTL_OFF; + + MCDI_IN_SET_DWORD(req, SET_MAC_IN_FCNTL, fcntl); + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + /* Push multicast hash */ + + if (epp->ep_all_mulcst) { + /* A hash matching all multicast is all 1s */ + EFX_SET_OWORD(multicast_hash[0]); + EFX_SET_OWORD(multicast_hash[1]); + } else if (epp->ep_mulcst) { + /* Use the hash set by the multicast list */ + multicast_hash[0] = epp->ep_multicst_hash[0]; + multicast_hash[1] = epp->ep_multicst_hash[1]; + } else { + /* A hash matching no traffic is simply 0 */ + EFX_ZERO_OWORD(multicast_hash[0]); + EFX_ZERO_OWORD(multicast_hash[1]); + } + + /* + * Broadcast packets go through the multicast hash filter. + * The IEEE 802.3 CRC32 of the broadcast address is 0xbe2612ff + * so we always add bit 0xff to the mask (bit 0x7f in the + * second octword). + */ + if (epp->ep_brdcst) { + /* + * NOTE: due to constant folding, some of this evaluates + * to null expressions, giving E_EXPR_NULL_EFFECT during + * lint on Illumos. No good way to fix this without + * explicit coding the individual word/bit setting. + * So just suppress lint for this one line. + */ + /* LINTED */ + EFX_SET_OWORD_BIT(multicast_hash[1], 0x7f); + } + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_SET_MCAST_HASH; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_SET_MCAST_HASH_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_SET_MCAST_HASH_OUT_LEN; + + memcpy(MCDI_IN2(req, uint8_t, SET_MCAST_HASH_IN_HASH0), + multicast_hash, sizeof (multicast_hash)); + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail2; + } + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +#if EFSYS_OPT_LOOPBACK + + __checkReturn efx_rc_t +siena_mac_loopback_set( + __in efx_nic_t *enp, + __in efx_link_mode_t link_mode, + __in efx_loopback_type_t loopback_type) +{ + efx_port_t *epp = &(enp->en_port); + const efx_phy_ops_t *epop = epp->ep_epop; + efx_loopback_type_t old_loopback_type; + efx_link_mode_t old_loopback_link_mode; + efx_rc_t rc; + + /* The PHY object handles this on Siena */ + old_loopback_type = epp->ep_loopback_type; + old_loopback_link_mode = epp->ep_loopback_link_mode; + epp->ep_loopback_type = loopback_type; + epp->ep_loopback_link_mode = link_mode; + + if ((rc = epop->epo_reconfigure(enp)) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + epp->ep_loopback_type = old_loopback_type; + epp->ep_loopback_link_mode = old_loopback_link_mode; + + return (rc); +} + +#endif /* EFSYS_OPT_LOOPBACK */ + +#if EFSYS_OPT_MAC_STATS + + __checkReturn efx_rc_t +siena_mac_stats_get_mask( + __in efx_nic_t *enp, + __inout_bcount(mask_size) uint32_t *maskp, + __in size_t mask_size) +{ + const struct efx_mac_stats_range siena_stats[] = { + { EFX_MAC_RX_OCTETS, EFX_MAC_RX_GE_15XX_PKTS }, + /* EFX_MAC_RX_ERRORS is not supported */ + { EFX_MAC_RX_FCS_ERRORS, EFX_MAC_TX_EX_DEF_PKTS }, + }; + efx_rc_t rc; + + _NOTE(ARGUNUSED(enp)) + + if ((rc = efx_mac_stats_mask_add_ranges(maskp, mask_size, + siena_stats, EFX_ARRAY_SIZE(siena_stats))) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +#define SIENA_MAC_STAT_READ(_esmp, _field, _eqp) \ + EFSYS_MEM_READQ((_esmp), (_field) * sizeof (efx_qword_t), _eqp) + + __checkReturn efx_rc_t +siena_mac_stats_update( + __in efx_nic_t *enp, + __in efsys_mem_t *esmp, + __inout_ecount(EFX_MAC_NSTATS) efsys_stat_t *stat, + __inout_opt uint32_t *generationp) +{ + efx_qword_t value; + efx_qword_t generation_start; + efx_qword_t generation_end; + + _NOTE(ARGUNUSED(enp)) + + /* Read END first so we don't race with the MC */ + EFSYS_DMA_SYNC_FOR_KERNEL(esmp, 0, EFX_MAC_STATS_SIZE); + SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_GENERATION_END, + &generation_end); + EFSYS_MEM_READ_BARRIER(); + + /* TX */ + SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_PKTS]), &value); + SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_CONTROL_PKTS, &value); + EFSYS_STAT_SUBR_QWORD(&(stat[EFX_MAC_TX_PKTS]), &value); + + SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_PAUSE_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_PAUSE_PKTS]), &value); + + SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_UNICAST_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_UNICST_PKTS]), &value); + + SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_MULTICAST_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_MULTICST_PKTS]), &value); + + SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_BROADCAST_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_BRDCST_PKTS]), &value); + + SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_BYTES, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_OCTETS]), &value); + + SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_LT64_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_LE_64_PKTS]), &value); + SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_64_PKTS, &value); + EFSYS_STAT_INCR_QWORD(&(stat[EFX_MAC_TX_LE_64_PKTS]), &value); + + SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_65_TO_127_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_65_TO_127_PKTS]), &value); + + SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_128_TO_255_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_128_TO_255_PKTS]), &value); + + SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_256_TO_511_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_256_TO_511_PKTS]), &value); + + SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_512_TO_1023_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_512_TO_1023_PKTS]), &value); + + SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_1024_TO_15XX_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_1024_TO_15XX_PKTS]), &value); + + SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_15XX_TO_JUMBO_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_GE_15XX_PKTS]), &value); + SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_GTJUMBO_PKTS, &value); + EFSYS_STAT_INCR_QWORD(&(stat[EFX_MAC_TX_GE_15XX_PKTS]), &value); + + SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_BAD_FCS_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_ERRORS]), &value); + + SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_SINGLE_COLLISION_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_SGL_COL_PKTS]), &value); + + SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_MULTIPLE_COLLISION_PKTS, + &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_MULT_COL_PKTS]), &value); + + SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_EXCESSIVE_COLLISION_PKTS, + &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_EX_COL_PKTS]), &value); + + SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_LATE_COLLISION_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_LATE_COL_PKTS]), &value); + + SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_DEFERRED_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_DEF_PKTS]), &value); + + SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_TX_EXCESSIVE_DEFERRED_PKTS, + &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_TX_EX_DEF_PKTS]), &value); + + /* RX */ + SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_BYTES, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_OCTETS]), &value); + + SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_PKTS]), &value); + + SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_UNICAST_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_UNICST_PKTS]), &value); + + SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_MULTICAST_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_MULTICST_PKTS]), &value); + + SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_BROADCAST_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_BRDCST_PKTS]), &value); + + SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_PAUSE_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_PAUSE_PKTS]), &value); + + SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_UNDERSIZE_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_LE_64_PKTS]), &value); + SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_64_PKTS, &value); + EFSYS_STAT_INCR_QWORD(&(stat[EFX_MAC_RX_LE_64_PKTS]), &value); + + SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_65_TO_127_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_65_TO_127_PKTS]), &value); + + SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_128_TO_255_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_128_TO_255_PKTS]), &value); + + SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_256_TO_511_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_256_TO_511_PKTS]), &value); + + SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_512_TO_1023_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_512_TO_1023_PKTS]), &value); + + SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_1024_TO_15XX_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_1024_TO_15XX_PKTS]), &value); + + SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_15XX_TO_JUMBO_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_GE_15XX_PKTS]), &value); + SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_GTJUMBO_PKTS, &value); + EFSYS_STAT_INCR_QWORD(&(stat[EFX_MAC_RX_GE_15XX_PKTS]), &value); + + SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_BAD_FCS_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_FCS_ERRORS]), &value); + + SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_OVERFLOW_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_DROP_EVENTS]), &value); + + SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_FALSE_CARRIER_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_FALSE_CARRIER_ERRORS]), &value); + + SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_SYMBOL_ERROR_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_SYMBOL_ERRORS]), &value); + + SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_ALIGN_ERROR_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_ALIGN_ERRORS]), &value); + + SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_INTERNAL_ERROR_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_INTERNAL_ERRORS]), &value); + + SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_JABBER_PKTS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_JABBER_PKTS]), &value); + + SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_LANES01_CHAR_ERR, &value); + EFSYS_STAT_SET_DWORD(&(stat[EFX_MAC_RX_LANE0_CHAR_ERR]), + &(value.eq_dword[0])); + EFSYS_STAT_SET_DWORD(&(stat[EFX_MAC_RX_LANE1_CHAR_ERR]), + &(value.eq_dword[1])); + + SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_LANES23_CHAR_ERR, &value); + EFSYS_STAT_SET_DWORD(&(stat[EFX_MAC_RX_LANE2_CHAR_ERR]), + &(value.eq_dword[0])); + EFSYS_STAT_SET_DWORD(&(stat[EFX_MAC_RX_LANE3_CHAR_ERR]), + &(value.eq_dword[1])); + + SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_LANES01_DISP_ERR, &value); + EFSYS_STAT_SET_DWORD(&(stat[EFX_MAC_RX_LANE0_DISP_ERR]), + &(value.eq_dword[0])); + EFSYS_STAT_SET_DWORD(&(stat[EFX_MAC_RX_LANE1_DISP_ERR]), + &(value.eq_dword[1])); + + SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_LANES23_DISP_ERR, &value); + EFSYS_STAT_SET_DWORD(&(stat[EFX_MAC_RX_LANE2_DISP_ERR]), + &(value.eq_dword[0])); + EFSYS_STAT_SET_DWORD(&(stat[EFX_MAC_RX_LANE3_DISP_ERR]), + &(value.eq_dword[1])); + + SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_MATCH_FAULT, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_MATCH_FAULT]), &value); + + SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_RX_NODESC_DROPS, &value); + EFSYS_STAT_SET_QWORD(&(stat[EFX_MAC_RX_NODESC_DROP_CNT]), &value); + + EFSYS_DMA_SYNC_FOR_KERNEL(esmp, 0, EFX_MAC_STATS_SIZE); + EFSYS_MEM_READ_BARRIER(); + SIENA_MAC_STAT_READ(esmp, MC_CMD_MAC_GENERATION_START, + &generation_start); + + /* Check that we didn't read the stats in the middle of a DMA */ + /* Not a good enough check ? */ + if (memcmp(&generation_start, &generation_end, + sizeof (generation_start))) + return (EAGAIN); + + if (generationp) + *generationp = EFX_QWORD_FIELD(generation_start, EFX_DWORD_0); + + return (0); +} + +#endif /* EFSYS_OPT_MAC_STATS */ + + __checkReturn efx_rc_t +siena_mac_pdu_get( + __in efx_nic_t *enp, + __out size_t *pdu) +{ + return (ENOTSUP); +} + +#endif /* EFSYS_OPT_SIENA */ diff --git a/dpdk/drivers/net/sfc/base/siena_mcdi.c b/dpdk/drivers/net/sfc/base/siena_mcdi.c new file mode 100644 index 00000000..63c29fcb --- /dev/null +++ b/dpdk/drivers/net/sfc/base/siena_mcdi.c @@ -0,0 +1,263 @@ +/* + * Copyright (c) 2012-2016 Solarflare Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are + * those of the authors and should not be interpreted as representing official + * policies, either expressed or implied, of the FreeBSD Project. + */ + +#include "efx.h" +#include "efx_impl.h" + +#if EFSYS_OPT_SIENA && EFSYS_OPT_MCDI + +#define SIENA_MCDI_PDU(_emip) \ + (((emip)->emi_port == 1) \ + ? MC_SMEM_P0_PDU_OFST >> 2 \ + : MC_SMEM_P1_PDU_OFST >> 2) + +#define SIENA_MCDI_DOORBELL(_emip) \ + (((emip)->emi_port == 1) \ + ? MC_SMEM_P0_DOORBELL_OFST >> 2 \ + : MC_SMEM_P1_DOORBELL_OFST >> 2) + +#define SIENA_MCDI_STATUS(_emip) \ + (((emip)->emi_port == 1) \ + ? MC_SMEM_P0_STATUS_OFST >> 2 \ + : MC_SMEM_P1_STATUS_OFST >> 2) + + + void +siena_mcdi_send_request( + __in efx_nic_t *enp, + __in_bcount(hdr_len) void *hdrp, + __in size_t hdr_len, + __in_bcount(sdu_len) void *sdup, + __in size_t sdu_len) +{ + efx_mcdi_iface_t *emip = &(enp->en_mcdi.em_emip); + efx_dword_t dword; + unsigned int pdur; + unsigned int dbr; + unsigned int pos; + + EFSYS_ASSERT(enp->en_family == EFX_FAMILY_SIENA); + + EFSYS_ASSERT(emip->emi_port == 1 || emip->emi_port == 2); + pdur = SIENA_MCDI_PDU(emip); + dbr = SIENA_MCDI_DOORBELL(emip); + + /* Write the header */ + EFSYS_ASSERT3U(hdr_len, ==, sizeof (efx_dword_t)); + dword = *(efx_dword_t *)hdrp; + EFX_BAR_TBL_WRITED(enp, FR_CZ_MC_TREG_SMEM, pdur, &dword, B_TRUE); + + /* Write the payload */ + for (pos = 0; pos < sdu_len; pos += sizeof (efx_dword_t)) { + dword = *(efx_dword_t *)((uint8_t *)sdup + pos); + EFX_BAR_TBL_WRITED(enp, FR_CZ_MC_TREG_SMEM, + pdur + 1 + (pos >> 2), &dword, B_FALSE); + } + + /* Ring the doorbell */ + EFX_POPULATE_DWORD_1(dword, EFX_DWORD_0, 0xd004be11); + EFX_BAR_TBL_WRITED(enp, FR_CZ_MC_TREG_SMEM, dbr, &dword, B_FALSE); +} + + efx_rc_t +siena_mcdi_poll_reboot( + __in efx_nic_t *enp) +{ +#if 1 + /* + * XXX Bug 25922, bug 26099: This function is not being used + * properly. Until its callers are fixed, it should always + * return 0. + */ + _NOTE(ARGUNUSED(enp)) + return (0); +#else + efx_mcdi_iface_t *emip = &(enp->en_mcdi.em_emip); + unsigned int rebootr; + efx_dword_t dword; + uint32_t value; + + EFSYS_ASSERT(enp->en_family == EFX_FAMILY_SIENA); + EFSYS_ASSERT(emip->emi_port == 1 || emip->emi_port == 2); + rebootr = SIENA_MCDI_STATUS(emip); + + EFX_BAR_TBL_READD(enp, FR_CZ_MC_TREG_SMEM, rebootr, &dword, B_FALSE); + value = EFX_DWORD_FIELD(dword, EFX_DWORD_0); + + if (value == 0) + return (0); + + EFX_ZERO_DWORD(dword); + EFX_BAR_TBL_WRITED(enp, FR_CZ_MC_TREG_SMEM, rebootr, &dword, B_FALSE); + + if (value == MC_STATUS_DWORD_ASSERT) + return (EINTR); + else + return (EIO); +#endif +} + +extern __checkReturn boolean_t +siena_mcdi_poll_response( + __in efx_nic_t *enp) +{ + efx_mcdi_iface_t *emip = &(enp->en_mcdi.em_emip); + efx_dword_t hdr; + unsigned int pdur; + + EFSYS_ASSERT(emip->emi_port == 1 || emip->emi_port == 2); + pdur = SIENA_MCDI_PDU(emip); + + EFX_BAR_TBL_READD(enp, FR_CZ_MC_TREG_SMEM, pdur, &hdr, B_FALSE); + return (EFX_DWORD_FIELD(hdr, MCDI_HEADER_RESPONSE) ? B_TRUE : B_FALSE); +} + + void +siena_mcdi_read_response( + __in efx_nic_t *enp, + __out_bcount(length) void *bufferp, + __in size_t offset, + __in size_t length) +{ + efx_mcdi_iface_t *emip = &(enp->en_mcdi.em_emip); + unsigned int pdur; + unsigned int pos; + efx_dword_t data; + + EFSYS_ASSERT(emip->emi_port == 1 || emip->emi_port == 2); + pdur = SIENA_MCDI_PDU(emip); + + for (pos = 0; pos < length; pos += sizeof (efx_dword_t)) { + EFX_BAR_TBL_READD(enp, FR_CZ_MC_TREG_SMEM, + pdur + ((offset + pos) >> 2), &data, B_FALSE); + memcpy((uint8_t *)bufferp + pos, &data, + MIN(sizeof (data), length - pos)); + } +} + + __checkReturn efx_rc_t +siena_mcdi_init( + __in efx_nic_t *enp, + __in const efx_mcdi_transport_t *mtp) +{ + efx_mcdi_iface_t *emip = &(enp->en_mcdi.em_emip); + efx_oword_t oword; + unsigned int portnum; + efx_rc_t rc; + + _NOTE(ARGUNUSED(mtp)) + + EFSYS_ASSERT(enp->en_family == EFX_FAMILY_SIENA); + + /* Determine the port number to use for MCDI */ + EFX_BAR_READO(enp, FR_AZ_CS_DEBUG_REG, &oword); + portnum = EFX_OWORD_FIELD(oword, FRF_CZ_CS_PORT_NUM); + + if (portnum == 0) { + /* Presumably booted from ROM; only MCDI port 1 will work */ + emip->emi_port = 1; + } else if (portnum <= 2) { + emip->emi_port = portnum; + } else { + rc = EINVAL; + goto fail1; + } + + /* Siena BootROM and firmware only support MCDIv1 */ + emip->emi_max_version = 1; + + /* + * Wipe the atomic reboot status so subsequent MCDI requests succeed. + * BOOT_STATUS is preserved so eno_nic_probe() can boot out of the + * assertion handler. + */ + (void) siena_mcdi_poll_reboot(enp); + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + void +siena_mcdi_fini( + __in efx_nic_t *enp) +{ + _NOTE(ARGUNUSED(enp)) +} + + __checkReturn efx_rc_t +siena_mcdi_feature_supported( + __in efx_nic_t *enp, + __in efx_mcdi_feature_id_t id, + __out boolean_t *supportedp) +{ + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_family, ==, EFX_FAMILY_SIENA); + + switch (id) { + case EFX_MCDI_FEATURE_FW_UPDATE: + case EFX_MCDI_FEATURE_LINK_CONTROL: + case EFX_MCDI_FEATURE_MACADDR_CHANGE: + case EFX_MCDI_FEATURE_MAC_SPOOFING: + *supportedp = B_TRUE; + break; + default: + rc = ENOTSUP; + goto fail1; + } + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +/* Default timeout for MCDI command processing. */ +#define SIENA_MCDI_CMD_TIMEOUT_US (10 * 1000 * 1000) + + void +siena_mcdi_get_timeout( + __in efx_nic_t *enp, + __in efx_mcdi_req_t *emrp, + __out uint32_t *timeoutp) +{ + _NOTE(ARGUNUSED(enp, emrp)) + + *timeoutp = SIENA_MCDI_CMD_TIMEOUT_US; +} + + +#endif /* EFSYS_OPT_SIENA && EFSYS_OPT_MCDI */ diff --git a/dpdk/drivers/net/sfc/base/siena_nic.c b/dpdk/drivers/net/sfc/base/siena_nic.c new file mode 100644 index 00000000..fcc8f151 --- /dev/null +++ b/dpdk/drivers/net/sfc/base/siena_nic.c @@ -0,0 +1,588 @@ +/* + * Copyright (c) 2009-2016 Solarflare Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are + * those of the authors and should not be interpreted as representing official + * policies, either expressed or implied, of the FreeBSD Project. + */ + +#include "efx.h" +#include "efx_impl.h" +#include "mcdi_mon.h" + +#if EFSYS_OPT_SIENA + +#if EFSYS_OPT_VPD || EFSYS_OPT_NVRAM + +static __checkReturn efx_rc_t +siena_nic_get_partn_mask( + __in efx_nic_t *enp, + __out unsigned int *maskp) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_NVRAM_TYPES_IN_LEN, + MC_CMD_NVRAM_TYPES_OUT_LEN)]; + efx_rc_t rc; + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_NVRAM_TYPES; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_NVRAM_TYPES_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_NVRAM_TYPES_OUT_LEN; + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + if (req.emr_out_length_used < MC_CMD_NVRAM_TYPES_OUT_LEN) { + rc = EMSGSIZE; + goto fail2; + } + + *maskp = MCDI_OUT_DWORD(req, NVRAM_TYPES_OUT_TYPES); + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +#endif /* EFSYS_OPT_VPD || EFSYS_OPT_NVRAM */ + +static __checkReturn efx_rc_t +siena_board_cfg( + __in efx_nic_t *enp) +{ + efx_nic_cfg_t *encp = &(enp->en_nic_cfg); + uint8_t mac_addr[6]; + efx_dword_t capabilities; + uint32_t board_type; + uint32_t nevq, nrxq, ntxq; + efx_rc_t rc; + + /* External port identifier using one-based port numbering */ + encp->enc_external_port = (uint8_t)enp->en_mcdi.em_emip.emi_port; + + /* Board configuration */ + if ((rc = efx_mcdi_get_board_cfg(enp, &board_type, + &capabilities, mac_addr)) != 0) + goto fail1; + + EFX_MAC_ADDR_COPY(encp->enc_mac_addr, mac_addr); + + encp->enc_board_type = board_type; + + /* + * There is no possibility to determine the number of PFs on Siena + * by issuing MCDI request, and it is not an easy task to find the + * value based on the board type, so 'enc_hw_pf_count' is set to 1 + */ + encp->enc_hw_pf_count = 1; + + /* Additional capabilities */ + encp->enc_clk_mult = 1; + if (EFX_DWORD_FIELD(capabilities, MC_CMD_CAPABILITIES_TURBO)) { + enp->en_features |= EFX_FEATURE_TURBO; + + if (EFX_DWORD_FIELD(capabilities, + MC_CMD_CAPABILITIES_TURBO_ACTIVE)) { + encp->enc_clk_mult = 2; + } + } + + encp->enc_evq_timer_quantum_ns = + EFX_EVQ_SIENA_TIMER_QUANTUM_NS / encp->enc_clk_mult; + encp->enc_evq_timer_max_us = (encp->enc_evq_timer_quantum_ns << + FRF_CZ_TC_TIMER_VAL_WIDTH) / 1000; + + /* When hash header insertion is enabled, Siena inserts 16 bytes */ + encp->enc_rx_prefix_size = 16; + + /* Alignment for receive packet DMA buffers */ + encp->enc_rx_buf_align_start = 1; + encp->enc_rx_buf_align_end = 1; + + /* Alignment for WPTR updates */ + encp->enc_rx_push_align = 1; + + /* There is one RSS context per function */ + encp->enc_rx_scale_max_exclusive_contexts = 1; + + encp->enc_tx_dma_desc_size_max = EFX_MASK32(FSF_AZ_TX_KER_BYTE_COUNT); + /* Fragments must not span 4k boundaries. */ + encp->enc_tx_dma_desc_boundary = 4096; + + /* Resource limits */ + rc = efx_mcdi_get_resource_limits(enp, &nevq, &nrxq, &ntxq); + if (rc != 0) { + if (rc != ENOTSUP) + goto fail2; + + nevq = 1024; + nrxq = EFX_RXQ_LIMIT_TARGET; + ntxq = EFX_TXQ_LIMIT_TARGET; + } + encp->enc_evq_limit = nevq; + encp->enc_rxq_limit = MIN(EFX_RXQ_LIMIT_TARGET, nrxq); + encp->enc_txq_limit = MIN(EFX_TXQ_LIMIT_TARGET, ntxq); + + encp->enc_txq_max_ndescs = 4096; + + encp->enc_buftbl_limit = SIENA_SRAM_ROWS - + (encp->enc_txq_limit * EFX_TXQ_DC_NDESCS(EFX_TXQ_DC_SIZE)) - + (encp->enc_rxq_limit * EFX_RXQ_DC_NDESCS(EFX_RXQ_DC_SIZE)); + + encp->enc_hw_tx_insert_vlan_enabled = B_FALSE; + encp->enc_fw_assisted_tso_enabled = B_FALSE; + encp->enc_fw_assisted_tso_v2_enabled = B_FALSE; + encp->enc_fw_assisted_tso_v2_n_contexts = 0; + encp->enc_allow_set_mac_with_installed_filters = B_TRUE; + encp->enc_rx_packed_stream_supported = B_FALSE; + encp->enc_rx_var_packed_stream_supported = B_FALSE; + + /* Siena supports two 10G ports, and 8 lanes of PCIe Gen2 */ + encp->enc_required_pcie_bandwidth_mbps = 2 * 10000; + encp->enc_max_pcie_link_gen = EFX_PCIE_LINK_SPEED_GEN2; + + encp->enc_fw_verified_nvram_update_required = B_FALSE; + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +static __checkReturn efx_rc_t +siena_phy_cfg( + __in efx_nic_t *enp) +{ + efx_nic_cfg_t *encp = &(enp->en_nic_cfg); + efx_rc_t rc; + + /* Fill out fields in enp->en_port and enp->en_nic_cfg from MCDI */ + if ((rc = efx_mcdi_get_phy_cfg(enp)) != 0) + goto fail1; + +#if EFSYS_OPT_PHY_STATS + /* Convert the MCDI statistic mask into the EFX_PHY_STAT mask */ + siena_phy_decode_stats(enp, encp->enc_mcdi_phy_stat_mask, + NULL, &encp->enc_phy_stat_mask, NULL); +#endif /* EFSYS_OPT_PHY_STATS */ + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +siena_nic_probe( + __in efx_nic_t *enp) +{ + efx_port_t *epp = &(enp->en_port); + efx_nic_cfg_t *encp = &(enp->en_nic_cfg); + siena_link_state_t sls; + unsigned int mask; + efx_oword_t oword; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_family, ==, EFX_FAMILY_SIENA); + + /* Test BIU */ + if ((rc = efx_nic_biu_test(enp)) != 0) + goto fail1; + + /* Clear the region register */ + EFX_POPULATE_OWORD_4(oword, + FRF_AZ_ADR_REGION0, 0, + FRF_AZ_ADR_REGION1, (1 << 16), + FRF_AZ_ADR_REGION2, (2 << 16), + FRF_AZ_ADR_REGION3, (3 << 16)); + EFX_BAR_WRITEO(enp, FR_AZ_ADR_REGION_REG, &oword); + + /* Read clear any assertion state */ + if ((rc = efx_mcdi_read_assertion(enp)) != 0) + goto fail2; + + /* Exit the assertion handler */ + if ((rc = efx_mcdi_exit_assertion_handler(enp)) != 0) + goto fail3; + + /* Wrestle control from the BMC */ + if ((rc = efx_mcdi_drv_attach(enp, B_TRUE)) != 0) + goto fail4; + + if ((rc = siena_board_cfg(enp)) != 0) + goto fail5; + + if ((rc = siena_phy_cfg(enp)) != 0) + goto fail6; + + /* Obtain the default PHY advertised capabilities */ + if ((rc = siena_nic_reset(enp)) != 0) + goto fail7; + if ((rc = siena_phy_get_link(enp, &sls)) != 0) + goto fail8; + epp->ep_default_adv_cap_mask = sls.sls_adv_cap_mask; + epp->ep_adv_cap_mask = sls.sls_adv_cap_mask; + +#if EFSYS_OPT_VPD || EFSYS_OPT_NVRAM + if ((rc = siena_nic_get_partn_mask(enp, &mask)) != 0) + goto fail9; + enp->en_u.siena.enu_partn_mask = mask; +#endif + +#if EFSYS_OPT_MAC_STATS + /* Wipe the MAC statistics */ + if ((rc = efx_mcdi_mac_stats_clear(enp)) != 0) + goto fail10; +#endif + +#if EFSYS_OPT_LOOPBACK + if ((rc = efx_mcdi_get_loopback_modes(enp)) != 0) + goto fail11; +#endif + +#if EFSYS_OPT_MON_STATS + if ((rc = mcdi_mon_cfg_build(enp)) != 0) + goto fail12; +#endif + + encp->enc_features = enp->en_features; + + return (0); + +#if EFSYS_OPT_MON_STATS +fail12: + EFSYS_PROBE(fail12); +#endif +#if EFSYS_OPT_LOOPBACK +fail11: + EFSYS_PROBE(fail11); +#endif +#if EFSYS_OPT_MAC_STATS +fail10: + EFSYS_PROBE(fail10); +#endif +#if EFSYS_OPT_VPD || EFSYS_OPT_NVRAM +fail9: + EFSYS_PROBE(fail9); +#endif +fail8: + EFSYS_PROBE(fail8); +fail7: + EFSYS_PROBE(fail7); +fail6: + EFSYS_PROBE(fail6); +fail5: + EFSYS_PROBE(fail5); +fail4: + EFSYS_PROBE(fail4); +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +siena_nic_reset( + __in efx_nic_t *enp) +{ + efx_mcdi_req_t req; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_family, ==, EFX_FAMILY_SIENA); + + /* siena_nic_reset() is called to recover from BADASSERT failures. */ + if ((rc = efx_mcdi_read_assertion(enp)) != 0) + goto fail1; + if ((rc = efx_mcdi_exit_assertion_handler(enp)) != 0) + goto fail2; + + /* + * Bug24908: ENTITY_RESET_IN_LEN is non zero but zero may be supplied + * for backwards compatibility with PORT_RESET_IN_LEN. + */ + EFX_STATIC_ASSERT(MC_CMD_ENTITY_RESET_OUT_LEN == 0); + + req.emr_cmd = MC_CMD_ENTITY_RESET; + req.emr_in_buf = NULL; + req.emr_in_length = 0; + req.emr_out_buf = NULL; + req.emr_out_length = 0; + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail3; + } + + return (0); + +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (0); +} + +static void +siena_nic_rx_cfg( + __in efx_nic_t *enp) +{ + efx_oword_t oword; + + /* + * RX_INGR_EN is always enabled on Siena, because we rely on + * the RX parser to be resiliant to missing SOP/EOP. + */ + EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword); + EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_INGR_EN, 1); + EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword); + + /* Disable parsing of additional 802.1Q in Q packets */ + EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword); + EFX_SET_OWORD_FIELD(oword, FRF_CZ_RX_FILTER_ALL_VLAN_ETHERTYPES, 0); + EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword); +} + +static void +siena_nic_usrev_dis( + __in efx_nic_t *enp) +{ + efx_oword_t oword; + + EFX_POPULATE_OWORD_1(oword, FRF_CZ_USREV_DIS, 1); + EFX_BAR_WRITEO(enp, FR_CZ_USR_EV_CFG, &oword); +} + + __checkReturn efx_rc_t +siena_nic_init( + __in efx_nic_t *enp) +{ + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_family, ==, EFX_FAMILY_SIENA); + + /* Enable reporting of some events (e.g. link change) */ + if ((rc = efx_mcdi_log_ctrl(enp)) != 0) + goto fail1; + + siena_sram_init(enp); + + /* Configure Siena's RX block */ + siena_nic_rx_cfg(enp); + + /* Disable USR_EVents for now */ + siena_nic_usrev_dis(enp); + + /* bug17057: Ensure set_link is called */ + if ((rc = siena_phy_reconfigure(enp)) != 0) + goto fail2; + + enp->en_nic_cfg.enc_mcdi_max_payload_length = MCDI_CTL_SDU_LEN_MAX_V1; + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + void +siena_nic_fini( + __in efx_nic_t *enp) +{ + _NOTE(ARGUNUSED(enp)) +} + + void +siena_nic_unprobe( + __in efx_nic_t *enp) +{ +#if EFSYS_OPT_MON_STATS + mcdi_mon_cfg_free(enp); +#endif /* EFSYS_OPT_MON_STATS */ + (void) efx_mcdi_drv_attach(enp, B_FALSE); +} + +#if EFSYS_OPT_DIAG + +static efx_register_set_t __siena_registers[] = { + { FR_AZ_ADR_REGION_REG_OFST, 0, 1 }, + { FR_CZ_USR_EV_CFG_OFST, 0, 1 }, + { FR_AZ_RX_CFG_REG_OFST, 0, 1 }, + { FR_AZ_TX_CFG_REG_OFST, 0, 1 }, + { FR_AZ_TX_RESERVED_REG_OFST, 0, 1 }, + { FR_AZ_SRM_TX_DC_CFG_REG_OFST, 0, 1 }, + { FR_AZ_RX_DC_CFG_REG_OFST, 0, 1 }, + { FR_AZ_RX_DC_PF_WM_REG_OFST, 0, 1 }, + { FR_AZ_DP_CTRL_REG_OFST, 0, 1 }, + { FR_BZ_RX_RSS_TKEY_REG_OFST, 0, 1}, + { FR_CZ_RX_RSS_IPV6_REG1_OFST, 0, 1}, + { FR_CZ_RX_RSS_IPV6_REG2_OFST, 0, 1}, + { FR_CZ_RX_RSS_IPV6_REG3_OFST, 0, 1} +}; + +static const uint32_t __siena_register_masks[] = { + 0x0003FFFF, 0x0003FFFF, 0x0003FFFF, 0x0003FFFF, + 0x000103FF, 0x00000000, 0x00000000, 0x00000000, + 0xFFFFFFFE, 0xFFFFFFFF, 0x0003FFFF, 0x00000000, + 0x7FFF0037, 0xFFFF8000, 0xFFFFFFFF, 0x03FFFFFF, + 0xFFFEFE80, 0x1FFFFFFF, 0x020000FE, 0x007FFFFF, + 0x001FFFFF, 0x00000000, 0x00000000, 0x00000000, + 0x00000003, 0x00000000, 0x00000000, 0x00000000, + 0x000003FF, 0x00000000, 0x00000000, 0x00000000, + 0x00000FFF, 0x00000000, 0x00000000, 0x00000000, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0x00000007, 0x00000000 +}; + +static efx_register_set_t __siena_tables[] = { + { FR_AZ_RX_FILTER_TBL0_OFST, FR_AZ_RX_FILTER_TBL0_STEP, + FR_AZ_RX_FILTER_TBL0_ROWS }, + { FR_CZ_RX_MAC_FILTER_TBL0_OFST, FR_CZ_RX_MAC_FILTER_TBL0_STEP, + FR_CZ_RX_MAC_FILTER_TBL0_ROWS }, + { FR_AZ_RX_DESC_PTR_TBL_OFST, + FR_AZ_RX_DESC_PTR_TBL_STEP, FR_CZ_RX_DESC_PTR_TBL_ROWS }, + { FR_AZ_TX_DESC_PTR_TBL_OFST, + FR_AZ_TX_DESC_PTR_TBL_STEP, FR_CZ_TX_DESC_PTR_TBL_ROWS }, + { FR_AZ_TIMER_TBL_OFST, FR_AZ_TIMER_TBL_STEP, FR_CZ_TIMER_TBL_ROWS }, + { FR_CZ_TX_FILTER_TBL0_OFST, + FR_CZ_TX_FILTER_TBL0_STEP, FR_CZ_TX_FILTER_TBL0_ROWS }, + { FR_CZ_TX_MAC_FILTER_TBL0_OFST, + FR_CZ_TX_MAC_FILTER_TBL0_STEP, FR_CZ_TX_MAC_FILTER_TBL0_ROWS } +}; + +static const uint32_t __siena_table_masks[] = { + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x000003FF, + 0xFFFF0FFF, 0xFFFFFFFF, 0x00000E7F, 0x00000000, + 0xFFFFFFFE, 0x0FFFFFFF, 0x01800000, 0x00000000, + 0xFFFFFFFE, 0x0FFFFFFF, 0x0C000000, 0x00000000, + 0x3FFFFFFF, 0x00000000, 0x00000000, 0x00000000, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x000013FF, + 0xFFFF07FF, 0xFFFFFFFF, 0x0000007F, 0x00000000, +}; + + __checkReturn efx_rc_t +siena_nic_register_test( + __in efx_nic_t *enp) +{ + efx_register_set_t *rsp; + const uint32_t *dwordp; + unsigned int nitems; + unsigned int count; + efx_rc_t rc; + + /* Fill out the register mask entries */ + EFX_STATIC_ASSERT(EFX_ARRAY_SIZE(__siena_register_masks) + == EFX_ARRAY_SIZE(__siena_registers) * 4); + + nitems = EFX_ARRAY_SIZE(__siena_registers); + dwordp = __siena_register_masks; + for (count = 0; count < nitems; ++count) { + rsp = __siena_registers + count; + rsp->mask.eo_u32[0] = *dwordp++; + rsp->mask.eo_u32[1] = *dwordp++; + rsp->mask.eo_u32[2] = *dwordp++; + rsp->mask.eo_u32[3] = *dwordp++; + } + + /* Fill out the register table entries */ + EFX_STATIC_ASSERT(EFX_ARRAY_SIZE(__siena_table_masks) + == EFX_ARRAY_SIZE(__siena_tables) * 4); + + nitems = EFX_ARRAY_SIZE(__siena_tables); + dwordp = __siena_table_masks; + for (count = 0; count < nitems; ++count) { + rsp = __siena_tables + count; + rsp->mask.eo_u32[0] = *dwordp++; + rsp->mask.eo_u32[1] = *dwordp++; + rsp->mask.eo_u32[2] = *dwordp++; + rsp->mask.eo_u32[3] = *dwordp++; + } + + if ((rc = efx_nic_test_registers(enp, __siena_registers, + EFX_ARRAY_SIZE(__siena_registers))) != 0) + goto fail1; + + if ((rc = efx_nic_test_tables(enp, __siena_tables, + EFX_PATTERN_BYTE_ALTERNATE, + EFX_ARRAY_SIZE(__siena_tables))) != 0) + goto fail2; + + if ((rc = efx_nic_test_tables(enp, __siena_tables, + EFX_PATTERN_BYTE_CHANGING, + EFX_ARRAY_SIZE(__siena_tables))) != 0) + goto fail3; + + if ((rc = efx_nic_test_tables(enp, __siena_tables, + EFX_PATTERN_BIT_SWEEP, EFX_ARRAY_SIZE(__siena_tables))) != 0) + goto fail4; + + return (0); + +fail4: + EFSYS_PROBE(fail4); +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +#endif /* EFSYS_OPT_DIAG */ + +#endif /* EFSYS_OPT_SIENA */ diff --git a/dpdk/drivers/net/sfc/base/siena_nvram.c b/dpdk/drivers/net/sfc/base/siena_nvram.c new file mode 100644 index 00000000..af4cf172 --- /dev/null +++ b/dpdk/drivers/net/sfc/base/siena_nvram.c @@ -0,0 +1,734 @@ +/* + * Copyright (c) 2009-2016 Solarflare Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are + * those of the authors and should not be interpreted as representing official + * policies, either expressed or implied, of the FreeBSD Project. + */ + +#include "efx.h" +#include "efx_impl.h" + +#if EFSYS_OPT_SIENA + +#if EFSYS_OPT_VPD || EFSYS_OPT_NVRAM + + __checkReturn efx_rc_t +siena_nvram_partn_size( + __in efx_nic_t *enp, + __in uint32_t partn, + __out size_t *sizep) +{ + efx_rc_t rc; + + if ((1 << partn) & ~enp->en_u.siena.enu_partn_mask) { + rc = ENOTSUP; + goto fail1; + } + + if ((rc = efx_mcdi_nvram_info(enp, partn, sizep, + NULL, NULL, NULL)) != 0) { + goto fail2; + } + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +siena_nvram_partn_lock( + __in efx_nic_t *enp, + __in uint32_t partn) +{ + efx_rc_t rc; + + if ((rc = efx_mcdi_nvram_update_start(enp, partn)) != 0) { + goto fail1; + } + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +siena_nvram_partn_read( + __in efx_nic_t *enp, + __in uint32_t partn, + __in unsigned int offset, + __out_bcount(size) caddr_t data, + __in size_t size) +{ + size_t chunk; + efx_rc_t rc; + + while (size > 0) { + chunk = MIN(size, SIENA_NVRAM_CHUNK); + + if ((rc = efx_mcdi_nvram_read(enp, partn, offset, data, chunk, + MC_CMD_NVRAM_READ_IN_V2_DEFAULT)) != 0) { + goto fail1; + } + + size -= chunk; + data += chunk; + offset += chunk; + } + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +siena_nvram_partn_erase( + __in efx_nic_t *enp, + __in uint32_t partn, + __in unsigned int offset, + __in size_t size) +{ + efx_rc_t rc; + + if ((rc = efx_mcdi_nvram_erase(enp, partn, offset, size)) != 0) { + goto fail1; + } + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +siena_nvram_partn_write( + __in efx_nic_t *enp, + __in uint32_t partn, + __in unsigned int offset, + __out_bcount(size) caddr_t data, + __in size_t size) +{ + size_t chunk; + efx_rc_t rc; + + while (size > 0) { + chunk = MIN(size, SIENA_NVRAM_CHUNK); + + if ((rc = efx_mcdi_nvram_write(enp, partn, offset, + data, chunk)) != 0) { + goto fail1; + } + + size -= chunk; + data += chunk; + offset += chunk; + } + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +siena_nvram_partn_unlock( + __in efx_nic_t *enp, + __in uint32_t partn) +{ + boolean_t reboot; + efx_rc_t rc; + + /* + * Reboot into the new image only for PHYs. The driver has to + * explicitly cope with an MC reboot after a firmware update. + */ + reboot = (partn == MC_CMD_NVRAM_TYPE_PHY_PORT0 || + partn == MC_CMD_NVRAM_TYPE_PHY_PORT1 || + partn == MC_CMD_NVRAM_TYPE_DISABLED_CALLISTO); + + rc = efx_mcdi_nvram_update_finish(enp, partn, reboot, NULL); + if (rc != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +#endif /* EFSYS_OPT_VPD || EFSYS_OPT_NVRAM */ + +#if EFSYS_OPT_NVRAM + +typedef struct siena_parttbl_entry_s { + unsigned int partn; + unsigned int port; + efx_nvram_type_t nvtype; +} siena_parttbl_entry_t; + +static siena_parttbl_entry_t siena_parttbl[] = { + {MC_CMD_NVRAM_TYPE_DISABLED_CALLISTO, 1, EFX_NVRAM_NULLPHY}, + {MC_CMD_NVRAM_TYPE_DISABLED_CALLISTO, 2, EFX_NVRAM_NULLPHY}, + {MC_CMD_NVRAM_TYPE_MC_FW, 1, EFX_NVRAM_MC_FIRMWARE}, + {MC_CMD_NVRAM_TYPE_MC_FW, 2, EFX_NVRAM_MC_FIRMWARE}, + {MC_CMD_NVRAM_TYPE_MC_FW_BACKUP, 1, EFX_NVRAM_MC_GOLDEN}, + {MC_CMD_NVRAM_TYPE_MC_FW_BACKUP, 2, EFX_NVRAM_MC_GOLDEN}, + {MC_CMD_NVRAM_TYPE_EXP_ROM, 1, EFX_NVRAM_BOOTROM}, + {MC_CMD_NVRAM_TYPE_EXP_ROM, 2, EFX_NVRAM_BOOTROM}, + {MC_CMD_NVRAM_TYPE_EXP_ROM_CFG_PORT0, 1, EFX_NVRAM_BOOTROM_CFG}, + {MC_CMD_NVRAM_TYPE_EXP_ROM_CFG_PORT1, 2, EFX_NVRAM_BOOTROM_CFG}, + {MC_CMD_NVRAM_TYPE_PHY_PORT0, 1, EFX_NVRAM_PHY}, + {MC_CMD_NVRAM_TYPE_PHY_PORT1, 2, EFX_NVRAM_PHY}, + {MC_CMD_NVRAM_TYPE_FPGA, 1, EFX_NVRAM_FPGA}, + {MC_CMD_NVRAM_TYPE_FPGA, 2, EFX_NVRAM_FPGA}, + {MC_CMD_NVRAM_TYPE_FPGA_BACKUP, 1, EFX_NVRAM_FPGA_BACKUP}, + {MC_CMD_NVRAM_TYPE_FPGA_BACKUP, 2, EFX_NVRAM_FPGA_BACKUP}, + {MC_CMD_NVRAM_TYPE_FC_FW, 1, EFX_NVRAM_FCFW}, + {MC_CMD_NVRAM_TYPE_FC_FW, 2, EFX_NVRAM_FCFW}, + {MC_CMD_NVRAM_TYPE_CPLD, 1, EFX_NVRAM_CPLD}, + {MC_CMD_NVRAM_TYPE_CPLD, 2, EFX_NVRAM_CPLD}, + {MC_CMD_NVRAM_TYPE_LICENSE, 1, EFX_NVRAM_LICENSE}, + {MC_CMD_NVRAM_TYPE_LICENSE, 2, EFX_NVRAM_LICENSE} +}; + + __checkReturn efx_rc_t +siena_nvram_type_to_partn( + __in efx_nic_t *enp, + __in efx_nvram_type_t type, + __out uint32_t *partnp) +{ + efx_mcdi_iface_t *emip = &(enp->en_mcdi.em_emip); + unsigned int i; + + EFSYS_ASSERT3U(type, <, EFX_NVRAM_NTYPES); + EFSYS_ASSERT(partnp != NULL); + + for (i = 0; i < EFX_ARRAY_SIZE(siena_parttbl); i++) { + siena_parttbl_entry_t *entry = &siena_parttbl[i]; + + if (entry->port == emip->emi_port && entry->nvtype == type) { + *partnp = entry->partn; + return (0); + } + } + + return (ENOTSUP); +} + + +#if EFSYS_OPT_DIAG + + __checkReturn efx_rc_t +siena_nvram_test( + __in efx_nic_t *enp) +{ + efx_mcdi_iface_t *emip = &(enp->en_mcdi.em_emip); + siena_parttbl_entry_t *entry; + unsigned int i; + efx_rc_t rc; + + /* + * Iterate over the list of supported partition types + * applicable to *this* port + */ + for (i = 0; i < EFX_ARRAY_SIZE(siena_parttbl); i++) { + entry = &siena_parttbl[i]; + + if (entry->port != emip->emi_port || + !(enp->en_u.siena.enu_partn_mask & (1 << entry->partn))) + continue; + + if ((rc = efx_mcdi_nvram_test(enp, entry->partn)) != 0) { + goto fail1; + } + } + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +#endif /* EFSYS_OPT_DIAG */ + + +#define SIENA_DYNAMIC_CFG_SIZE(_nitems) \ + (sizeof (siena_mc_dynamic_config_hdr_t) + ((_nitems) * \ + sizeof (((siena_mc_dynamic_config_hdr_t *)NULL)->fw_version[0]))) + + __checkReturn efx_rc_t +siena_nvram_get_dynamic_cfg( + __in efx_nic_t *enp, + __in uint32_t partn, + __in boolean_t vpd, + __out siena_mc_dynamic_config_hdr_t **dcfgp, + __out size_t *sizep) +{ + siena_mc_dynamic_config_hdr_t *dcfg = NULL; + size_t size; + uint8_t cksum; + unsigned int vpd_offset; + unsigned int vpd_length; + unsigned int hdr_length; + unsigned int nversions; + unsigned int pos; + unsigned int region; + efx_rc_t rc; + + EFSYS_ASSERT(partn == MC_CMD_NVRAM_TYPE_DYNAMIC_CFG_PORT0 || + partn == MC_CMD_NVRAM_TYPE_DYNAMIC_CFG_PORT1); + + /* + * Allocate sufficient memory for the entire dynamiccfg area, even + * if we're not actually going to read in the VPD. + */ + if ((rc = siena_nvram_partn_size(enp, partn, &size)) != 0) + goto fail1; + + EFSYS_KMEM_ALLOC(enp->en_esip, size, dcfg); + if (dcfg == NULL) { + rc = ENOMEM; + goto fail2; + } + + if ((rc = siena_nvram_partn_read(enp, partn, 0, + (caddr_t)dcfg, SIENA_NVRAM_CHUNK)) != 0) + goto fail3; + + /* Verify the magic */ + if (EFX_DWORD_FIELD(dcfg->magic, EFX_DWORD_0) + != SIENA_MC_DYNAMIC_CONFIG_MAGIC) + goto invalid1; + + /* All future versions of the structure must be backwards compatible */ + EFX_STATIC_ASSERT(SIENA_MC_DYNAMIC_CONFIG_VERSION == 0); + + hdr_length = EFX_WORD_FIELD(dcfg->length, EFX_WORD_0); + nversions = EFX_DWORD_FIELD(dcfg->num_fw_version_items, EFX_DWORD_0); + vpd_offset = EFX_DWORD_FIELD(dcfg->dynamic_vpd_offset, EFX_DWORD_0); + vpd_length = EFX_DWORD_FIELD(dcfg->dynamic_vpd_length, EFX_DWORD_0); + + /* Verify the hdr doesn't overflow the partn size */ + if (hdr_length > size || vpd_offset > size || vpd_length > size || + vpd_length + vpd_offset > size) + goto invalid2; + + /* Verify the header has room for all it's versions */ + if (hdr_length < SIENA_DYNAMIC_CFG_SIZE(0) || + hdr_length < SIENA_DYNAMIC_CFG_SIZE(nversions)) + goto invalid3; + + /* + * Read the remaining portion of the dcfg, either including + * the whole of VPD (there is no vpd length in this structure, + * so we have to parse each tag), or just the dcfg header itself + */ + region = vpd ? vpd_offset + vpd_length : hdr_length; + if (region > SIENA_NVRAM_CHUNK) { + if ((rc = siena_nvram_partn_read(enp, partn, SIENA_NVRAM_CHUNK, + (caddr_t)dcfg + SIENA_NVRAM_CHUNK, + region - SIENA_NVRAM_CHUNK)) != 0) + goto fail4; + } + + /* Verify checksum */ + cksum = 0; + for (pos = 0; pos < hdr_length; pos++) + cksum += ((uint8_t *)dcfg)[pos]; + if (cksum != 0) + goto invalid4; + + goto done; + +invalid4: + EFSYS_PROBE(invalid4); +invalid3: + EFSYS_PROBE(invalid3); +invalid2: + EFSYS_PROBE(invalid2); +invalid1: + EFSYS_PROBE(invalid1); + + /* + * Construct a new "null" dcfg, with an empty version vector, + * and an empty VPD chunk trailing. This has the neat side effect + * of testing the exception paths in the write path. + */ + EFX_POPULATE_DWORD_1(dcfg->magic, + EFX_DWORD_0, SIENA_MC_DYNAMIC_CONFIG_MAGIC); + EFX_POPULATE_WORD_1(dcfg->length, EFX_WORD_0, sizeof (*dcfg)); + EFX_POPULATE_BYTE_1(dcfg->version, EFX_BYTE_0, + SIENA_MC_DYNAMIC_CONFIG_VERSION); + EFX_POPULATE_DWORD_1(dcfg->dynamic_vpd_offset, + EFX_DWORD_0, sizeof (*dcfg)); + EFX_POPULATE_DWORD_1(dcfg->dynamic_vpd_length, EFX_DWORD_0, 0); + EFX_POPULATE_DWORD_1(dcfg->num_fw_version_items, EFX_DWORD_0, 0); + +done: + *dcfgp = dcfg; + *sizep = size; + + return (0); + +fail4: + EFSYS_PROBE(fail4); +fail3: + EFSYS_PROBE(fail3); + + EFSYS_KMEM_FREE(enp->en_esip, size, dcfg); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +siena_nvram_get_subtype( + __in efx_nic_t *enp, + __in uint32_t partn, + __out uint32_t *subtypep) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_GET_BOARD_CFG_IN_LEN, + MC_CMD_GET_BOARD_CFG_OUT_LENMAX)]; + efx_word_t *fw_list; + efx_rc_t rc; + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_GET_BOARD_CFG; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_GET_BOARD_CFG_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_GET_BOARD_CFG_OUT_LENMAX; + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + if (req.emr_out_length_used < MC_CMD_GET_BOARD_CFG_OUT_LENMIN) { + rc = EMSGSIZE; + goto fail2; + } + + if (req.emr_out_length_used < + MC_CMD_GET_BOARD_CFG_OUT_FW_SUBTYPE_LIST_OFST + + (partn + 1) * sizeof (efx_word_t)) { + rc = ENOENT; + goto fail3; + } + + fw_list = MCDI_OUT2(req, efx_word_t, + GET_BOARD_CFG_OUT_FW_SUBTYPE_LIST); + *subtypep = EFX_WORD_FIELD(fw_list[partn], EFX_WORD_0); + + return (0); + +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +siena_nvram_partn_get_version( + __in efx_nic_t *enp, + __in uint32_t partn, + __out uint32_t *subtypep, + __out_ecount(4) uint16_t version[4]) +{ + siena_mc_dynamic_config_hdr_t *dcfg; + siena_parttbl_entry_t *entry; + uint32_t dcfg_partn; + unsigned int i; + efx_rc_t rc; + + if ((1 << partn) & ~enp->en_u.siena.enu_partn_mask) { + rc = ENOTSUP; + goto fail1; + } + + if ((rc = siena_nvram_get_subtype(enp, partn, subtypep)) != 0) + goto fail2; + + /* + * Some partitions are accessible from both ports (for instance BOOTROM) + * Find the highest version reported by all dcfg structures on ports + * that have access to this partition. + */ + version[0] = version[1] = version[2] = version[3] = 0; + for (i = 0; i < EFX_ARRAY_SIZE(siena_parttbl); i++) { + siena_mc_fw_version_t *verp; + unsigned int nitems; + uint16_t temp[4]; + size_t length; + + entry = &siena_parttbl[i]; + if (entry->partn != partn) + continue; + + dcfg_partn = (entry->port == 1) + ? MC_CMD_NVRAM_TYPE_DYNAMIC_CFG_PORT0 + : MC_CMD_NVRAM_TYPE_DYNAMIC_CFG_PORT1; + /* + * Ingore missing partitions on port 2, assuming they're due + * to to running on a single port part. + */ + if ((1 << dcfg_partn) & ~enp->en_u.siena.enu_partn_mask) { + if (entry->port == 2) + continue; + } + + if ((rc = siena_nvram_get_dynamic_cfg(enp, dcfg_partn, + B_FALSE, &dcfg, &length)) != 0) + goto fail3; + + nitems = EFX_DWORD_FIELD(dcfg->num_fw_version_items, + EFX_DWORD_0); + if (nitems < entry->partn) + goto done; + + verp = &dcfg->fw_version[partn]; + temp[0] = EFX_WORD_FIELD(verp->version_w, EFX_WORD_0); + temp[1] = EFX_WORD_FIELD(verp->version_x, EFX_WORD_0); + temp[2] = EFX_WORD_FIELD(verp->version_y, EFX_WORD_0); + temp[3] = EFX_WORD_FIELD(verp->version_z, EFX_WORD_0); + if (memcmp(version, temp, sizeof (temp)) < 0) + memcpy(version, temp, sizeof (temp)); + +done: + EFSYS_KMEM_FREE(enp->en_esip, length, dcfg); + } + + return (0); + +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +siena_nvram_partn_rw_start( + __in efx_nic_t *enp, + __in uint32_t partn, + __out size_t *chunk_sizep) +{ + efx_rc_t rc; + + if ((rc = siena_nvram_partn_lock(enp, partn)) != 0) + goto fail1; + + if (chunk_sizep != NULL) + *chunk_sizep = SIENA_NVRAM_CHUNK; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +siena_nvram_partn_rw_finish( + __in efx_nic_t *enp, + __in uint32_t partn) +{ + efx_rc_t rc; + + if ((rc = siena_nvram_partn_unlock(enp, partn)) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +siena_nvram_partn_set_version( + __in efx_nic_t *enp, + __in uint32_t partn, + __in_ecount(4) uint16_t version[4]) +{ + efx_mcdi_iface_t *emip = &(enp->en_mcdi.em_emip); + siena_mc_dynamic_config_hdr_t *dcfg = NULL; + siena_mc_fw_version_t *fwverp; + uint32_t dcfg_partn; + size_t dcfg_size; + unsigned int hdr_length; + unsigned int vpd_length; + unsigned int vpd_offset; + unsigned int nitems; + unsigned int required_hdr_length; + unsigned int pos; + uint8_t cksum; + uint32_t subtype; + size_t length; + efx_rc_t rc; + + dcfg_partn = (emip->emi_port == 1) + ? MC_CMD_NVRAM_TYPE_DYNAMIC_CFG_PORT0 + : MC_CMD_NVRAM_TYPE_DYNAMIC_CFG_PORT1; + + if ((rc = siena_nvram_partn_size(enp, dcfg_partn, &dcfg_size)) != 0) + goto fail1; + + if ((rc = siena_nvram_partn_lock(enp, dcfg_partn)) != 0) + goto fail2; + + if ((rc = siena_nvram_get_dynamic_cfg(enp, dcfg_partn, + B_TRUE, &dcfg, &length)) != 0) + goto fail3; + + hdr_length = EFX_WORD_FIELD(dcfg->length, EFX_WORD_0); + nitems = EFX_DWORD_FIELD(dcfg->num_fw_version_items, EFX_DWORD_0); + vpd_length = EFX_DWORD_FIELD(dcfg->dynamic_vpd_length, EFX_DWORD_0); + vpd_offset = EFX_DWORD_FIELD(dcfg->dynamic_vpd_offset, EFX_DWORD_0); + + /* + * NOTE: This function will blatt any fields trailing the version + * vector, or the VPD chunk. + */ + required_hdr_length = SIENA_DYNAMIC_CFG_SIZE(partn + 1); + if (required_hdr_length + vpd_length > length) { + rc = ENOSPC; + goto fail4; + } + + if (vpd_offset < required_hdr_length) { + (void) memmove((caddr_t)dcfg + required_hdr_length, + (caddr_t)dcfg + vpd_offset, vpd_length); + vpd_offset = required_hdr_length; + EFX_POPULATE_DWORD_1(dcfg->dynamic_vpd_offset, + EFX_DWORD_0, vpd_offset); + } + + if (hdr_length < required_hdr_length) { + (void) memset((caddr_t)dcfg + hdr_length, 0, + required_hdr_length - hdr_length); + hdr_length = required_hdr_length; + EFX_POPULATE_WORD_1(dcfg->length, + EFX_WORD_0, hdr_length); + } + + /* Get the subtype to insert into the fw_subtype array */ + if ((rc = siena_nvram_get_subtype(enp, partn, &subtype)) != 0) + goto fail5; + + /* Fill out the new version */ + fwverp = &dcfg->fw_version[partn]; + EFX_POPULATE_DWORD_1(fwverp->fw_subtype, EFX_DWORD_0, subtype); + EFX_POPULATE_WORD_1(fwverp->version_w, EFX_WORD_0, version[0]); + EFX_POPULATE_WORD_1(fwverp->version_x, EFX_WORD_0, version[1]); + EFX_POPULATE_WORD_1(fwverp->version_y, EFX_WORD_0, version[2]); + EFX_POPULATE_WORD_1(fwverp->version_z, EFX_WORD_0, version[3]); + + /* Update the version count */ + if (nitems < partn + 1) { + nitems = partn + 1; + EFX_POPULATE_DWORD_1(dcfg->num_fw_version_items, + EFX_DWORD_0, nitems); + } + + /* Update the checksum */ + cksum = 0; + for (pos = 0; pos < hdr_length; pos++) + cksum += ((uint8_t *)dcfg)[pos]; + dcfg->csum.eb_u8[0] -= cksum; + + /* Erase and write the new partition */ + if ((rc = siena_nvram_partn_erase(enp, dcfg_partn, 0, dcfg_size)) != 0) + goto fail6; + + /* Write out the new structure to nvram */ + if ((rc = siena_nvram_partn_write(enp, dcfg_partn, 0, + (caddr_t)dcfg, vpd_offset + vpd_length)) != 0) + goto fail7; + + EFSYS_KMEM_FREE(enp->en_esip, length, dcfg); + + siena_nvram_partn_unlock(enp, dcfg_partn); + + return (0); + +fail7: + EFSYS_PROBE(fail7); +fail6: + EFSYS_PROBE(fail6); +fail5: + EFSYS_PROBE(fail5); +fail4: + EFSYS_PROBE(fail4); + + EFSYS_KMEM_FREE(enp->en_esip, length, dcfg); +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +#endif /* EFSYS_OPT_NVRAM */ + +#endif /* EFSYS_OPT_SIENA */ diff --git a/dpdk/drivers/net/sfc/base/siena_phy.c b/dpdk/drivers/net/sfc/base/siena_phy.c new file mode 100644 index 00000000..b90ccabc --- /dev/null +++ b/dpdk/drivers/net/sfc/base/siena_phy.c @@ -0,0 +1,797 @@ +/* + * Copyright (c) 2009-2016 Solarflare Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are + * those of the authors and should not be interpreted as representing official + * policies, either expressed or implied, of the FreeBSD Project. + */ + +#include "efx.h" +#include "efx_impl.h" + +#if EFSYS_OPT_SIENA + +static void +siena_phy_decode_cap( + __in uint32_t mcdi_cap, + __out uint32_t *maskp) +{ + uint32_t mask; + + mask = 0; + if (mcdi_cap & (1 << MC_CMD_PHY_CAP_10HDX_LBN)) + mask |= (1 << EFX_PHY_CAP_10HDX); + if (mcdi_cap & (1 << MC_CMD_PHY_CAP_10FDX_LBN)) + mask |= (1 << EFX_PHY_CAP_10FDX); + if (mcdi_cap & (1 << MC_CMD_PHY_CAP_100HDX_LBN)) + mask |= (1 << EFX_PHY_CAP_100HDX); + if (mcdi_cap & (1 << MC_CMD_PHY_CAP_100FDX_LBN)) + mask |= (1 << EFX_PHY_CAP_100FDX); + if (mcdi_cap & (1 << MC_CMD_PHY_CAP_1000HDX_LBN)) + mask |= (1 << EFX_PHY_CAP_1000HDX); + if (mcdi_cap & (1 << MC_CMD_PHY_CAP_1000FDX_LBN)) + mask |= (1 << EFX_PHY_CAP_1000FDX); + if (mcdi_cap & (1 << MC_CMD_PHY_CAP_10000FDX_LBN)) + mask |= (1 << EFX_PHY_CAP_10000FDX); + if (mcdi_cap & (1 << MC_CMD_PHY_CAP_PAUSE_LBN)) + mask |= (1 << EFX_PHY_CAP_PAUSE); + if (mcdi_cap & (1 << MC_CMD_PHY_CAP_ASYM_LBN)) + mask |= (1 << EFX_PHY_CAP_ASYM); + if (mcdi_cap & (1 << MC_CMD_PHY_CAP_AN_LBN)) + mask |= (1 << EFX_PHY_CAP_AN); + + *maskp = mask; +} + +static void +siena_phy_decode_link_mode( + __in efx_nic_t *enp, + __in uint32_t link_flags, + __in unsigned int speed, + __in unsigned int fcntl, + __out efx_link_mode_t *link_modep, + __out unsigned int *fcntlp) +{ + boolean_t fd = !!(link_flags & + (1 << MC_CMD_GET_LINK_OUT_FULL_DUPLEX_LBN)); + boolean_t up = !!(link_flags & + (1 << MC_CMD_GET_LINK_OUT_LINK_UP_LBN)); + + _NOTE(ARGUNUSED(enp)) + + if (!up) + *link_modep = EFX_LINK_DOWN; + else if (speed == 10000 && fd) + *link_modep = EFX_LINK_10000FDX; + else if (speed == 1000) + *link_modep = fd ? EFX_LINK_1000FDX : EFX_LINK_1000HDX; + else if (speed == 100) + *link_modep = fd ? EFX_LINK_100FDX : EFX_LINK_100HDX; + else if (speed == 10) + *link_modep = fd ? EFX_LINK_10FDX : EFX_LINK_10HDX; + else + *link_modep = EFX_LINK_UNKNOWN; + + if (fcntl == MC_CMD_FCNTL_OFF) + *fcntlp = 0; + else if (fcntl == MC_CMD_FCNTL_RESPOND) + *fcntlp = EFX_FCNTL_RESPOND; + else if (fcntl == MC_CMD_FCNTL_BIDIR) + *fcntlp = EFX_FCNTL_RESPOND | EFX_FCNTL_GENERATE; + else { + EFSYS_PROBE1(mc_pcol_error, int, fcntl); + *fcntlp = 0; + } +} + + void +siena_phy_link_ev( + __in efx_nic_t *enp, + __in efx_qword_t *eqp, + __out efx_link_mode_t *link_modep) +{ + efx_port_t *epp = &(enp->en_port); + unsigned int link_flags; + unsigned int speed; + unsigned int fcntl; + efx_link_mode_t link_mode; + uint32_t lp_cap_mask; + + /* + * Convert the LINKCHANGE speed enumeration into mbit/s, in the + * same way as GET_LINK encodes the speed + */ + switch (MCDI_EV_FIELD(eqp, LINKCHANGE_SPEED)) { + case MCDI_EVENT_LINKCHANGE_SPEED_100M: + speed = 100; + break; + case MCDI_EVENT_LINKCHANGE_SPEED_1G: + speed = 1000; + break; + case MCDI_EVENT_LINKCHANGE_SPEED_10G: + speed = 10000; + break; + default: + speed = 0; + break; + } + + link_flags = MCDI_EV_FIELD(eqp, LINKCHANGE_LINK_FLAGS); + siena_phy_decode_link_mode(enp, link_flags, speed, + MCDI_EV_FIELD(eqp, LINKCHANGE_FCNTL), + &link_mode, &fcntl); + siena_phy_decode_cap(MCDI_EV_FIELD(eqp, LINKCHANGE_LP_CAP), + &lp_cap_mask); + + /* + * It's safe to update ep_lp_cap_mask without the driver's port lock + * because presumably any concurrently running efx_port_poll() is + * only going to arrive at the same value. + * + * ep_fcntl has two meanings. It's either the link common fcntl + * (if the PHY supports AN), or it's the forced link state. If + * the former, it's safe to update the value for the same reason as + * for ep_lp_cap_mask. If the latter, then just ignore the value, + * because we can race with efx_mac_fcntl_set(). + */ + epp->ep_lp_cap_mask = lp_cap_mask; + if (epp->ep_phy_cap_mask & (1 << EFX_PHY_CAP_AN)) + epp->ep_fcntl = fcntl; + + *link_modep = link_mode; +} + + __checkReturn efx_rc_t +siena_phy_power( + __in efx_nic_t *enp, + __in boolean_t power) +{ + efx_rc_t rc; + + if (!power) + return (0); + + /* Check if the PHY is a zombie */ + if ((rc = siena_phy_verify(enp)) != 0) + goto fail1; + + enp->en_reset_flags |= EFX_RESET_PHY; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +siena_phy_get_link( + __in efx_nic_t *enp, + __out siena_link_state_t *slsp) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_GET_LINK_IN_LEN, + MC_CMD_GET_LINK_OUT_LEN)]; + efx_rc_t rc; + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_GET_LINK; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_GET_LINK_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_GET_LINK_OUT_LEN; + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + if (req.emr_out_length_used < MC_CMD_GET_LINK_OUT_LEN) { + rc = EMSGSIZE; + goto fail2; + } + + siena_phy_decode_cap(MCDI_OUT_DWORD(req, GET_LINK_OUT_CAP), + &slsp->sls_adv_cap_mask); + siena_phy_decode_cap(MCDI_OUT_DWORD(req, GET_LINK_OUT_LP_CAP), + &slsp->sls_lp_cap_mask); + + siena_phy_decode_link_mode(enp, MCDI_OUT_DWORD(req, GET_LINK_OUT_FLAGS), + MCDI_OUT_DWORD(req, GET_LINK_OUT_LINK_SPEED), + MCDI_OUT_DWORD(req, GET_LINK_OUT_FCNTL), + &slsp->sls_link_mode, &slsp->sls_fcntl); + +#if EFSYS_OPT_LOOPBACK + /* Assert the MC_CMD_LOOPBACK and EFX_LOOPBACK namespace agree */ + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_NONE == EFX_LOOPBACK_OFF); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_DATA == EFX_LOOPBACK_DATA); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_GMAC == EFX_LOOPBACK_GMAC); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_XGMII == EFX_LOOPBACK_XGMII); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_XGXS == EFX_LOOPBACK_XGXS); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_XAUI == EFX_LOOPBACK_XAUI); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_GMII == EFX_LOOPBACK_GMII); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_SGMII == EFX_LOOPBACK_SGMII); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_XGBR == EFX_LOOPBACK_XGBR); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_XFI == EFX_LOOPBACK_XFI); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_XAUI_FAR == EFX_LOOPBACK_XAUI_FAR); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_GMII_FAR == EFX_LOOPBACK_GMII_FAR); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_SGMII_FAR == EFX_LOOPBACK_SGMII_FAR); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_XFI_FAR == EFX_LOOPBACK_XFI_FAR); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_GPHY == EFX_LOOPBACK_GPHY); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_PHYXS == EFX_LOOPBACK_PHY_XS); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_PCS == EFX_LOOPBACK_PCS); + EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_PMAPMD == EFX_LOOPBACK_PMA_PMD); + + slsp->sls_loopback = MCDI_OUT_DWORD(req, GET_LINK_OUT_LOOPBACK_MODE); +#endif /* EFSYS_OPT_LOOPBACK */ + + slsp->sls_mac_up = MCDI_OUT_DWORD(req, GET_LINK_OUT_MAC_FAULT) == 0; + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +siena_phy_reconfigure( + __in efx_nic_t *enp) +{ + efx_port_t *epp = &(enp->en_port); + efx_mcdi_req_t req; + uint8_t payload[MAX(MAX(MC_CMD_SET_ID_LED_IN_LEN, + MC_CMD_SET_ID_LED_OUT_LEN), + MAX(MC_CMD_SET_LINK_IN_LEN, + MC_CMD_SET_LINK_OUT_LEN))]; + uint32_t cap_mask; + unsigned int led_mode; + unsigned int speed; + efx_rc_t rc; + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_SET_LINK; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_SET_LINK_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_SET_LINK_OUT_LEN; + + cap_mask = epp->ep_adv_cap_mask; + MCDI_IN_POPULATE_DWORD_10(req, SET_LINK_IN_CAP, + PHY_CAP_10HDX, (cap_mask >> EFX_PHY_CAP_10HDX) & 0x1, + PHY_CAP_10FDX, (cap_mask >> EFX_PHY_CAP_10FDX) & 0x1, + PHY_CAP_100HDX, (cap_mask >> EFX_PHY_CAP_100HDX) & 0x1, + PHY_CAP_100FDX, (cap_mask >> EFX_PHY_CAP_100FDX) & 0x1, + PHY_CAP_1000HDX, (cap_mask >> EFX_PHY_CAP_1000HDX) & 0x1, + PHY_CAP_1000FDX, (cap_mask >> EFX_PHY_CAP_1000FDX) & 0x1, + PHY_CAP_10000FDX, (cap_mask >> EFX_PHY_CAP_10000FDX) & 0x1, + PHY_CAP_PAUSE, (cap_mask >> EFX_PHY_CAP_PAUSE) & 0x1, + PHY_CAP_ASYM, (cap_mask >> EFX_PHY_CAP_ASYM) & 0x1, + PHY_CAP_AN, (cap_mask >> EFX_PHY_CAP_AN) & 0x1); + +#if EFSYS_OPT_LOOPBACK + MCDI_IN_SET_DWORD(req, SET_LINK_IN_LOOPBACK_MODE, + epp->ep_loopback_type); + switch (epp->ep_loopback_link_mode) { + case EFX_LINK_100FDX: + speed = 100; + break; + case EFX_LINK_1000FDX: + speed = 1000; + break; + case EFX_LINK_10000FDX: + speed = 10000; + break; + default: + speed = 0; + } +#else + MCDI_IN_SET_DWORD(req, SET_LINK_IN_LOOPBACK_MODE, MC_CMD_LOOPBACK_NONE); + speed = 0; +#endif /* EFSYS_OPT_LOOPBACK */ + MCDI_IN_SET_DWORD(req, SET_LINK_IN_LOOPBACK_SPEED, speed); + +#if EFSYS_OPT_PHY_FLAGS + MCDI_IN_SET_DWORD(req, SET_LINK_IN_FLAGS, epp->ep_phy_flags); +#else + MCDI_IN_SET_DWORD(req, SET_LINK_IN_FLAGS, 0); +#endif /* EFSYS_OPT_PHY_FLAGS */ + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + /* And set the blink mode */ + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_SET_ID_LED; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_SET_ID_LED_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_SET_ID_LED_OUT_LEN; + +#if EFSYS_OPT_PHY_LED_CONTROL + switch (epp->ep_phy_led_mode) { + case EFX_PHY_LED_DEFAULT: + led_mode = MC_CMD_LED_DEFAULT; + break; + case EFX_PHY_LED_OFF: + led_mode = MC_CMD_LED_OFF; + break; + case EFX_PHY_LED_ON: + led_mode = MC_CMD_LED_ON; + break; + default: + EFSYS_ASSERT(0); + led_mode = MC_CMD_LED_DEFAULT; + } + + MCDI_IN_SET_DWORD(req, SET_ID_LED_IN_STATE, led_mode); +#else + MCDI_IN_SET_DWORD(req, SET_ID_LED_IN_STATE, MC_CMD_LED_DEFAULT); +#endif /* EFSYS_OPT_PHY_LED_CONTROL */ + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail2; + } + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +siena_phy_verify( + __in efx_nic_t *enp) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_GET_PHY_STATE_IN_LEN, + MC_CMD_GET_PHY_STATE_OUT_LEN)]; + uint32_t state; + efx_rc_t rc; + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_GET_PHY_STATE; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_GET_PHY_STATE_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_GET_PHY_STATE_OUT_LEN; + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + if (req.emr_out_length_used < MC_CMD_GET_PHY_STATE_OUT_LEN) { + rc = EMSGSIZE; + goto fail2; + } + + state = MCDI_OUT_DWORD(req, GET_PHY_STATE_OUT_STATE); + if (state != MC_CMD_PHY_STATE_OK) { + if (state != MC_CMD_PHY_STATE_ZOMBIE) + EFSYS_PROBE1(mc_pcol_error, int, state); + rc = ENOTACTIVE; + goto fail3; + } + + return (0); + +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +siena_phy_oui_get( + __in efx_nic_t *enp, + __out uint32_t *ouip) +{ + _NOTE(ARGUNUSED(enp, ouip)) + + return (ENOTSUP); +} + +#if EFSYS_OPT_PHY_STATS + +#define SIENA_SIMPLE_STAT_SET(_vmask, _esmp, _smask, _stat, \ + _mc_record, _efx_record) \ + if ((_vmask) & (1ULL << (_mc_record))) { \ + (_smask) |= (1ULL << (_efx_record)); \ + if ((_stat) != NULL && !EFSYS_MEM_IS_NULL(_esmp)) { \ + efx_dword_t dword; \ + EFSYS_MEM_READD(_esmp, (_mc_record) * 4, &dword);\ + (_stat)[_efx_record] = \ + EFX_DWORD_FIELD(dword, EFX_DWORD_0); \ + } \ + } + +#define SIENA_SIMPLE_STAT_SET2(_vmask, _esmp, _smask, _stat, _record) \ + SIENA_SIMPLE_STAT_SET(_vmask, _esmp, _smask, _stat, \ + MC_CMD_ ## _record, \ + EFX_PHY_STAT_ ## _record) + + void +siena_phy_decode_stats( + __in efx_nic_t *enp, + __in uint32_t vmask, + __in_opt efsys_mem_t *esmp, + __out_opt uint64_t *smaskp, + __inout_ecount_opt(EFX_PHY_NSTATS) uint32_t *stat) +{ + uint64_t smask = 0; + + _NOTE(ARGUNUSED(enp)) + + SIENA_SIMPLE_STAT_SET2(vmask, esmp, smask, stat, OUI); + SIENA_SIMPLE_STAT_SET2(vmask, esmp, smask, stat, PMA_PMD_LINK_UP); + SIENA_SIMPLE_STAT_SET2(vmask, esmp, smask, stat, PMA_PMD_RX_FAULT); + SIENA_SIMPLE_STAT_SET2(vmask, esmp, smask, stat, PMA_PMD_TX_FAULT); + + if (vmask & (1 << MC_CMD_PMA_PMD_SIGNAL)) { + smask |= ((1ULL << EFX_PHY_STAT_PMA_PMD_SIGNAL_A) | + (1ULL << EFX_PHY_STAT_PMA_PMD_SIGNAL_B) | + (1ULL << EFX_PHY_STAT_PMA_PMD_SIGNAL_C) | + (1ULL << EFX_PHY_STAT_PMA_PMD_SIGNAL_D)); + if (stat != NULL && esmp != NULL && !EFSYS_MEM_IS_NULL(esmp)) { + efx_dword_t dword; + uint32_t sig; + EFSYS_MEM_READD(esmp, 4 * MC_CMD_PMA_PMD_SIGNAL, + &dword); + sig = EFX_DWORD_FIELD(dword, EFX_DWORD_0); + stat[EFX_PHY_STAT_PMA_PMD_SIGNAL_A] = (sig >> 1) & 1; + stat[EFX_PHY_STAT_PMA_PMD_SIGNAL_B] = (sig >> 2) & 1; + stat[EFX_PHY_STAT_PMA_PMD_SIGNAL_C] = (sig >> 3) & 1; + stat[EFX_PHY_STAT_PMA_PMD_SIGNAL_D] = (sig >> 4) & 1; + } + } + + SIENA_SIMPLE_STAT_SET(vmask, esmp, smask, stat, MC_CMD_PMA_PMD_SNR_A, + EFX_PHY_STAT_SNR_A); + SIENA_SIMPLE_STAT_SET(vmask, esmp, smask, stat, MC_CMD_PMA_PMD_SNR_B, + EFX_PHY_STAT_SNR_B); + SIENA_SIMPLE_STAT_SET(vmask, esmp, smask, stat, MC_CMD_PMA_PMD_SNR_C, + EFX_PHY_STAT_SNR_C); + SIENA_SIMPLE_STAT_SET(vmask, esmp, smask, stat, MC_CMD_PMA_PMD_SNR_D, + EFX_PHY_STAT_SNR_D); + + SIENA_SIMPLE_STAT_SET2(vmask, esmp, smask, stat, PCS_LINK_UP); + SIENA_SIMPLE_STAT_SET2(vmask, esmp, smask, stat, PCS_RX_FAULT); + SIENA_SIMPLE_STAT_SET2(vmask, esmp, smask, stat, PCS_TX_FAULT); + SIENA_SIMPLE_STAT_SET2(vmask, esmp, smask, stat, PCS_BER); + SIENA_SIMPLE_STAT_SET2(vmask, esmp, smask, stat, PCS_BLOCK_ERRORS); + + SIENA_SIMPLE_STAT_SET(vmask, esmp, smask, stat, MC_CMD_PHYXS_LINK_UP, + EFX_PHY_STAT_PHY_XS_LINK_UP); + SIENA_SIMPLE_STAT_SET(vmask, esmp, smask, stat, MC_CMD_PHYXS_RX_FAULT, + EFX_PHY_STAT_PHY_XS_RX_FAULT); + SIENA_SIMPLE_STAT_SET(vmask, esmp, smask, stat, MC_CMD_PHYXS_TX_FAULT, + EFX_PHY_STAT_PHY_XS_TX_FAULT); + SIENA_SIMPLE_STAT_SET(vmask, esmp, smask, stat, MC_CMD_PHYXS_ALIGN, + EFX_PHY_STAT_PHY_XS_ALIGN); + + if (vmask & (1 << MC_CMD_PHYXS_SYNC)) { + smask |= ((1 << EFX_PHY_STAT_PHY_XS_SYNC_A) | + (1 << EFX_PHY_STAT_PHY_XS_SYNC_B) | + (1 << EFX_PHY_STAT_PHY_XS_SYNC_C) | + (1 << EFX_PHY_STAT_PHY_XS_SYNC_D)); + if (stat != NULL && !EFSYS_MEM_IS_NULL(esmp)) { + efx_dword_t dword; + uint32_t sync; + EFSYS_MEM_READD(esmp, 4 * MC_CMD_PHYXS_SYNC, &dword); + sync = EFX_DWORD_FIELD(dword, EFX_DWORD_0); + stat[EFX_PHY_STAT_PHY_XS_SYNC_A] = (sync >> 0) & 1; + stat[EFX_PHY_STAT_PHY_XS_SYNC_B] = (sync >> 1) & 1; + stat[EFX_PHY_STAT_PHY_XS_SYNC_C] = (sync >> 2) & 1; + stat[EFX_PHY_STAT_PHY_XS_SYNC_D] = (sync >> 3) & 1; + } + } + + SIENA_SIMPLE_STAT_SET2(vmask, esmp, smask, stat, AN_LINK_UP); + SIENA_SIMPLE_STAT_SET2(vmask, esmp, smask, stat, AN_COMPLETE); + + SIENA_SIMPLE_STAT_SET(vmask, esmp, smask, stat, MC_CMD_CL22_LINK_UP, + EFX_PHY_STAT_CL22EXT_LINK_UP); + + if (smaskp != NULL) + *smaskp = smask; +} + + __checkReturn efx_rc_t +siena_phy_stats_update( + __in efx_nic_t *enp, + __in efsys_mem_t *esmp, + __inout_ecount(EFX_PHY_NSTATS) uint32_t *stat) +{ + efx_nic_cfg_t *encp = &(enp->en_nic_cfg); + uint32_t vmask = encp->enc_mcdi_phy_stat_mask; + uint64_t smask; + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_PHY_STATS_IN_LEN, + MC_CMD_PHY_STATS_OUT_DMA_LEN)]; + efx_rc_t rc; + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_PHY_STATS; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_PHY_STATS_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_PHY_STATS_OUT_DMA_LEN; + + MCDI_IN_SET_DWORD(req, PHY_STATS_IN_DMA_ADDR_LO, + EFSYS_MEM_ADDR(esmp) & 0xffffffff); + MCDI_IN_SET_DWORD(req, PHY_STATS_IN_DMA_ADDR_HI, + EFSYS_MEM_ADDR(esmp) >> 32); + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + EFSYS_ASSERT3U(req.emr_out_length, ==, MC_CMD_PHY_STATS_OUT_DMA_LEN); + + siena_phy_decode_stats(enp, vmask, esmp, &smask, stat); + EFSYS_ASSERT(smask == encp->enc_phy_stat_mask); + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (0); +} + +#endif /* EFSYS_OPT_PHY_STATS */ + +#if EFSYS_OPT_BIST + + __checkReturn efx_rc_t +siena_phy_bist_start( + __in efx_nic_t *enp, + __in efx_bist_type_t type) +{ + efx_rc_t rc; + + if ((rc = efx_mcdi_bist_start(enp, type)) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + +static __checkReturn unsigned long +siena_phy_sft9001_bist_status( + __in uint16_t code) +{ + switch (code) { + case MC_CMD_POLL_BIST_SFT9001_PAIR_BUSY: + return (EFX_PHY_CABLE_STATUS_BUSY); + case MC_CMD_POLL_BIST_SFT9001_INTER_PAIR_SHORT: + return (EFX_PHY_CABLE_STATUS_INTERPAIRSHORT); + case MC_CMD_POLL_BIST_SFT9001_INTRA_PAIR_SHORT: + return (EFX_PHY_CABLE_STATUS_INTRAPAIRSHORT); + case MC_CMD_POLL_BIST_SFT9001_PAIR_OPEN: + return (EFX_PHY_CABLE_STATUS_OPEN); + case MC_CMD_POLL_BIST_SFT9001_PAIR_OK: + return (EFX_PHY_CABLE_STATUS_OK); + default: + return (EFX_PHY_CABLE_STATUS_INVALID); + } +} + + __checkReturn efx_rc_t +siena_phy_bist_poll( + __in efx_nic_t *enp, + __in efx_bist_type_t type, + __out efx_bist_result_t *resultp, + __out_opt __drv_when(count > 0, __notnull) + uint32_t *value_maskp, + __out_ecount_opt(count) __drv_when(count > 0, __notnull) + unsigned long *valuesp, + __in size_t count) +{ + efx_nic_cfg_t *encp = &(enp->en_nic_cfg); + uint8_t payload[MAX(MC_CMD_POLL_BIST_IN_LEN, + MCDI_CTL_SDU_LEN_MAX)]; + uint32_t value_mask = 0; + efx_mcdi_req_t req; + uint32_t result; + efx_rc_t rc; + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_POLL_BIST; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_POLL_BIST_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MCDI_CTL_SDU_LEN_MAX; + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail1; + } + + if (req.emr_out_length_used < MC_CMD_POLL_BIST_OUT_RESULT_OFST + 4) { + rc = EMSGSIZE; + goto fail2; + } + + if (count > 0) + (void) memset(valuesp, '\0', count * sizeof (unsigned long)); + + result = MCDI_OUT_DWORD(req, POLL_BIST_OUT_RESULT); + + /* Extract PHY specific results */ + if (result == MC_CMD_POLL_BIST_PASSED && + encp->enc_phy_type == EFX_PHY_SFT9001B && + req.emr_out_length_used >= MC_CMD_POLL_BIST_OUT_SFT9001_LEN && + (type == EFX_BIST_TYPE_PHY_CABLE_SHORT || + type == EFX_BIST_TYPE_PHY_CABLE_LONG)) { + uint16_t word; + + if (count > EFX_BIST_PHY_CABLE_LENGTH_A) { + if (valuesp != NULL) + valuesp[EFX_BIST_PHY_CABLE_LENGTH_A] = + MCDI_OUT_DWORD(req, + POLL_BIST_OUT_SFT9001_CABLE_LENGTH_A); + value_mask |= (1 << EFX_BIST_PHY_CABLE_LENGTH_A); + } + + if (count > EFX_BIST_PHY_CABLE_LENGTH_B) { + if (valuesp != NULL) + valuesp[EFX_BIST_PHY_CABLE_LENGTH_B] = + MCDI_OUT_DWORD(req, + POLL_BIST_OUT_SFT9001_CABLE_LENGTH_B); + value_mask |= (1 << EFX_BIST_PHY_CABLE_LENGTH_B); + } + + if (count > EFX_BIST_PHY_CABLE_LENGTH_C) { + if (valuesp != NULL) + valuesp[EFX_BIST_PHY_CABLE_LENGTH_C] = + MCDI_OUT_DWORD(req, + POLL_BIST_OUT_SFT9001_CABLE_LENGTH_C); + value_mask |= (1 << EFX_BIST_PHY_CABLE_LENGTH_C); + } + + if (count > EFX_BIST_PHY_CABLE_LENGTH_D) { + if (valuesp != NULL) + valuesp[EFX_BIST_PHY_CABLE_LENGTH_D] = + MCDI_OUT_DWORD(req, + POLL_BIST_OUT_SFT9001_CABLE_LENGTH_D); + value_mask |= (1 << EFX_BIST_PHY_CABLE_LENGTH_D); + } + + if (count > EFX_BIST_PHY_CABLE_STATUS_A) { + if (valuesp != NULL) { + word = MCDI_OUT_WORD(req, + POLL_BIST_OUT_SFT9001_CABLE_STATUS_A); + valuesp[EFX_BIST_PHY_CABLE_STATUS_A] = + siena_phy_sft9001_bist_status(word); + } + value_mask |= (1 << EFX_BIST_PHY_CABLE_STATUS_A); + } + + if (count > EFX_BIST_PHY_CABLE_STATUS_B) { + if (valuesp != NULL) { + word = MCDI_OUT_WORD(req, + POLL_BIST_OUT_SFT9001_CABLE_STATUS_B); + valuesp[EFX_BIST_PHY_CABLE_STATUS_B] = + siena_phy_sft9001_bist_status(word); + } + value_mask |= (1 << EFX_BIST_PHY_CABLE_STATUS_B); + } + + if (count > EFX_BIST_PHY_CABLE_STATUS_C) { + if (valuesp != NULL) { + word = MCDI_OUT_WORD(req, + POLL_BIST_OUT_SFT9001_CABLE_STATUS_C); + valuesp[EFX_BIST_PHY_CABLE_STATUS_C] = + siena_phy_sft9001_bist_status(word); + } + value_mask |= (1 << EFX_BIST_PHY_CABLE_STATUS_C); + } + + if (count > EFX_BIST_PHY_CABLE_STATUS_D) { + if (valuesp != NULL) { + word = MCDI_OUT_WORD(req, + POLL_BIST_OUT_SFT9001_CABLE_STATUS_D); + valuesp[EFX_BIST_PHY_CABLE_STATUS_D] = + siena_phy_sft9001_bist_status(word); + } + value_mask |= (1 << EFX_BIST_PHY_CABLE_STATUS_D); + } + + } else if (result == MC_CMD_POLL_BIST_FAILED && + encp->enc_phy_type == EFX_PHY_QLX111V && + req.emr_out_length >= MC_CMD_POLL_BIST_OUT_MRSFP_LEN && + count > EFX_BIST_FAULT_CODE) { + if (valuesp != NULL) + valuesp[EFX_BIST_FAULT_CODE] = + MCDI_OUT_DWORD(req, POLL_BIST_OUT_MRSFP_TEST); + value_mask |= 1 << EFX_BIST_FAULT_CODE; + } + + if (value_maskp != NULL) + *value_maskp = value_mask; + + EFSYS_ASSERT(resultp != NULL); + if (result == MC_CMD_POLL_BIST_RUNNING) + *resultp = EFX_BIST_RESULT_RUNNING; + else if (result == MC_CMD_POLL_BIST_PASSED) + *resultp = EFX_BIST_RESULT_PASSED; + else + *resultp = EFX_BIST_RESULT_FAILED; + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + void +siena_phy_bist_stop( + __in efx_nic_t *enp, + __in efx_bist_type_t type) +{ + /* There is no way to stop BIST on Siena */ + _NOTE(ARGUNUSED(enp, type)) +} + +#endif /* EFSYS_OPT_BIST */ + +#endif /* EFSYS_OPT_SIENA */ diff --git a/dpdk/drivers/net/sfc/base/siena_sram.c b/dpdk/drivers/net/sfc/base/siena_sram.c new file mode 100644 index 00000000..572c2e9a --- /dev/null +++ b/dpdk/drivers/net/sfc/base/siena_sram.c @@ -0,0 +1,178 @@ +/* + * Copyright (c) 2009-2016 Solarflare Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are + * those of the authors and should not be interpreted as representing official + * policies, either expressed or implied, of the FreeBSD Project. + */ + +#include "efx.h" +#include "efx_impl.h" + +#if EFSYS_OPT_SIENA + + void +siena_sram_init( + __in efx_nic_t *enp) +{ + efx_nic_cfg_t *encp = &(enp->en_nic_cfg); + efx_oword_t oword; + uint32_t rx_base, tx_base; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT(enp->en_family == EFX_FAMILY_SIENA); + + rx_base = encp->enc_buftbl_limit; + tx_base = rx_base + (encp->enc_rxq_limit * + EFX_RXQ_DC_NDESCS(EFX_RXQ_DC_SIZE)); + + /* Initialize the transmit descriptor cache */ + EFX_POPULATE_OWORD_1(oword, FRF_AZ_SRM_TX_DC_BASE_ADR, tx_base); + EFX_BAR_WRITEO(enp, FR_AZ_SRM_TX_DC_CFG_REG, &oword); + + EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DC_SIZE, EFX_TXQ_DC_SIZE); + EFX_BAR_WRITEO(enp, FR_AZ_TX_DC_CFG_REG, &oword); + + /* Initialize the receive descriptor cache */ + EFX_POPULATE_OWORD_1(oword, FRF_AZ_SRM_RX_DC_BASE_ADR, rx_base); + EFX_BAR_WRITEO(enp, FR_AZ_SRM_RX_DC_CFG_REG, &oword); + + EFX_POPULATE_OWORD_1(oword, FRF_AZ_RX_DC_SIZE, EFX_RXQ_DC_SIZE); + EFX_BAR_WRITEO(enp, FR_AZ_RX_DC_CFG_REG, &oword); + + /* Set receive descriptor pre-fetch low water mark */ + EFX_POPULATE_OWORD_1(oword, FRF_AZ_RX_DC_PF_LWM, 56); + EFX_BAR_WRITEO(enp, FR_AZ_RX_DC_PF_WM_REG, &oword); + + /* Set the event queue to use for SRAM updates */ + EFX_POPULATE_OWORD_1(oword, FRF_AZ_SRM_UPD_EVQ_ID, 0); + EFX_BAR_WRITEO(enp, FR_AZ_SRM_UPD_EVQ_REG, &oword); +} + +#if EFSYS_OPT_DIAG + + __checkReturn efx_rc_t +siena_sram_test( + __in efx_nic_t *enp, + __in efx_sram_pattern_fn_t func) +{ + efx_oword_t oword; + efx_qword_t qword; + efx_qword_t verify; + size_t rows; + unsigned int wptr; + unsigned int rptr; + efx_rc_t rc; + + EFSYS_ASSERT(enp->en_family == EFX_FAMILY_SIENA); + + /* Reconfigure into HALF buffer table mode */ + EFX_POPULATE_OWORD_1(oword, FRF_AZ_BUF_TBL_MODE, 0); + EFX_BAR_WRITEO(enp, FR_AZ_BUF_TBL_CFG_REG, &oword); + + /* + * Move the descriptor caches up to the top of SRAM, and test + * all of SRAM below them. We only miss out one row here. + */ + rows = SIENA_SRAM_ROWS - 1; + EFX_POPULATE_OWORD_1(oword, FRF_AZ_SRM_RX_DC_BASE_ADR, rows); + EFX_BAR_WRITEO(enp, FR_AZ_SRM_RX_DC_CFG_REG, &oword); + + EFX_POPULATE_OWORD_1(oword, FRF_AZ_SRM_TX_DC_BASE_ADR, rows + 1); + EFX_BAR_WRITEO(enp, FR_AZ_SRM_TX_DC_CFG_REG, &oword); + + /* + * Write the pattern through BUF_HALF_TBL. Write + * in 64 entry batches, waiting 1us in between each batch + * to guarantee not to overflow the SRAM fifo + */ + for (wptr = 0, rptr = 0; wptr < rows; ++wptr) { + func(wptr, B_FALSE, &qword); + EFX_BAR_TBL_WRITEQ(enp, FR_AZ_BUF_HALF_TBL, wptr, &qword); + + if ((wptr - rptr) < 64 && wptr < rows - 1) + continue; + + EFSYS_SPIN(1); + + for (; rptr <= wptr; ++rptr) { + func(rptr, B_FALSE, &qword); + EFX_BAR_TBL_READQ(enp, FR_AZ_BUF_HALF_TBL, rptr, + &verify); + + if (!EFX_QWORD_IS_EQUAL(verify, qword)) { + rc = EFAULT; + goto fail1; + } + } + } + + /* And do the same negated */ + for (wptr = 0, rptr = 0; wptr < rows; ++wptr) { + func(wptr, B_TRUE, &qword); + EFX_BAR_TBL_WRITEQ(enp, FR_AZ_BUF_HALF_TBL, wptr, &qword); + + if ((wptr - rptr) < 64 && wptr < rows - 1) + continue; + + EFSYS_SPIN(1); + + for (; rptr <= wptr; ++rptr) { + func(rptr, B_TRUE, &qword); + EFX_BAR_TBL_READQ(enp, FR_AZ_BUF_HALF_TBL, rptr, + &verify); + + if (!EFX_QWORD_IS_EQUAL(verify, qword)) { + rc = EFAULT; + goto fail2; + } + } + } + + /* Restore back to FULL buffer table mode */ + EFX_POPULATE_OWORD_1(oword, FRF_AZ_BUF_TBL_MODE, 1); + EFX_BAR_WRITEO(enp, FR_AZ_BUF_TBL_CFG_REG, &oword); + + /* + * We don't need to reconfigure SRAM again because the API + * requires efx_nic_fini() to be called after an sram test. + */ + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + /* Restore back to FULL buffer table mode */ + EFX_POPULATE_OWORD_1(oword, FRF_AZ_BUF_TBL_MODE, 1); + EFX_BAR_WRITEO(enp, FR_AZ_BUF_TBL_CFG_REG, &oword); + + return (rc); +} + +#endif /* EFSYS_OPT_DIAG */ + +#endif /* EFSYS_OPT_SIENA */ diff --git a/dpdk/drivers/net/sfc/base/siena_vpd.c b/dpdk/drivers/net/sfc/base/siena_vpd.c new file mode 100644 index 00000000..4fb2e426 --- /dev/null +++ b/dpdk/drivers/net/sfc/base/siena_vpd.c @@ -0,0 +1,618 @@ +/* + * Copyright (c) 2009-2016 Solarflare Communications Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * The views and conclusions contained in the software and documentation are + * those of the authors and should not be interpreted as representing official + * policies, either expressed or implied, of the FreeBSD Project. + */ + +#include "efx.h" +#include "efx_impl.h" + +#if EFSYS_OPT_VPD + +#if EFSYS_OPT_SIENA + +static __checkReturn efx_rc_t +siena_vpd_get_static( + __in efx_nic_t *enp, + __in uint32_t partn, + __deref_out_bcount_opt(*sizep) caddr_t *svpdp, + __out size_t *sizep) +{ + siena_mc_static_config_hdr_t *scfg; + caddr_t svpd; + size_t size; + uint8_t cksum; + unsigned int vpd_offset; + unsigned int vpd_length; + unsigned int hdr_length; + unsigned int pos; + unsigned int region; + efx_rc_t rc; + + EFSYS_ASSERT(partn == MC_CMD_NVRAM_TYPE_STATIC_CFG_PORT0 || + partn == MC_CMD_NVRAM_TYPE_STATIC_CFG_PORT1); + + /* Allocate sufficient memory for the entire static cfg area */ + if ((rc = siena_nvram_partn_size(enp, partn, &size)) != 0) + goto fail1; + + EFSYS_KMEM_ALLOC(enp->en_esip, size, scfg); + if (scfg == NULL) { + rc = ENOMEM; + goto fail2; + } + + if ((rc = siena_nvram_partn_read(enp, partn, 0, + (caddr_t)scfg, SIENA_NVRAM_CHUNK)) != 0) + goto fail3; + + /* Verify the magic number */ + if (EFX_DWORD_FIELD(scfg->magic, EFX_DWORD_0) != + SIENA_MC_STATIC_CONFIG_MAGIC) { + rc = EINVAL; + goto fail4; + } + + /* All future versions of the structure must be backwards compatible */ + EFX_STATIC_ASSERT(SIENA_MC_STATIC_CONFIG_VERSION == 0); + + hdr_length = EFX_WORD_FIELD(scfg->length, EFX_WORD_0); + vpd_offset = EFX_DWORD_FIELD(scfg->static_vpd_offset, EFX_DWORD_0); + vpd_length = EFX_DWORD_FIELD(scfg->static_vpd_length, EFX_DWORD_0); + + /* Verify the hdr doesn't overflow the sector size */ + if (hdr_length > size || vpd_offset > size || vpd_length > size || + vpd_length + vpd_offset > size) { + rc = EINVAL; + goto fail5; + } + + /* Read the remainder of scfg + static vpd */ + region = vpd_offset + vpd_length; + if (region > SIENA_NVRAM_CHUNK) { + if ((rc = siena_nvram_partn_read(enp, partn, SIENA_NVRAM_CHUNK, + (caddr_t)scfg + SIENA_NVRAM_CHUNK, + region - SIENA_NVRAM_CHUNK)) != 0) + goto fail6; + } + + /* Verify checksum */ + cksum = 0; + for (pos = 0; pos < hdr_length; pos++) + cksum += ((uint8_t *)scfg)[pos]; + if (cksum != 0) { + rc = EINVAL; + goto fail7; + } + + if (vpd_length == 0) + svpd = NULL; + else { + /* Copy the vpd data out */ + EFSYS_KMEM_ALLOC(enp->en_esip, vpd_length, svpd); + if (svpd == NULL) { + rc = ENOMEM; + goto fail8; + } + memcpy(svpd, (caddr_t)scfg + vpd_offset, vpd_length); + } + + EFSYS_KMEM_FREE(enp->en_esip, size, scfg); + + *svpdp = svpd; + *sizep = vpd_length; + + return (0); + +fail8: + EFSYS_PROBE(fail8); +fail7: + EFSYS_PROBE(fail7); +fail6: + EFSYS_PROBE(fail6); +fail5: + EFSYS_PROBE(fail5); +fail4: + EFSYS_PROBE(fail4); +fail3: + EFSYS_PROBE(fail3); + + EFSYS_KMEM_FREE(enp->en_esip, size, scfg); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +siena_vpd_init( + __in efx_nic_t *enp) +{ + efx_mcdi_iface_t *emip = &(enp->en_mcdi.em_emip); + caddr_t svpd = NULL; + unsigned int partn; + size_t size = 0; + efx_rc_t rc; + + EFSYS_ASSERT(enp->en_family == EFX_FAMILY_SIENA); + + partn = (emip->emi_port == 1) + ? MC_CMD_NVRAM_TYPE_STATIC_CFG_PORT0 + : MC_CMD_NVRAM_TYPE_STATIC_CFG_PORT1; + + /* + * We need the static VPD sector to present a unified static+dynamic + * VPD, that is, basically on every read, write, verify cycle. Since + * it should *never* change we can just cache it here. + */ + if ((rc = siena_vpd_get_static(enp, partn, &svpd, &size)) != 0) + goto fail1; + + if (svpd != NULL && size > 0) { + if ((rc = efx_vpd_hunk_verify(svpd, size, NULL)) != 0) + goto fail2; + } + + enp->en_u.siena.enu_svpd = svpd; + enp->en_u.siena.enu_svpd_length = size; + + return (0); + +fail2: + EFSYS_PROBE(fail2); + + EFSYS_KMEM_FREE(enp->en_esip, size, svpd); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +siena_vpd_size( + __in efx_nic_t *enp, + __out size_t *sizep) +{ + efx_mcdi_iface_t *emip = &(enp->en_mcdi.em_emip); + uint32_t partn; + efx_rc_t rc; + + EFSYS_ASSERT(enp->en_family == EFX_FAMILY_SIENA); + + /* + * This function returns the total size the user should allocate + * for all VPD operations. We've already cached the static vpd, + * so we just need to return an upper bound on the dynamic vpd. + * Since the dynamic_config structure can change under our feet, + * (as version numbers are inserted), just be safe and return the + * total size of the dynamic_config *sector* + */ + partn = (emip->emi_port == 1) + ? MC_CMD_NVRAM_TYPE_DYNAMIC_CFG_PORT0 + : MC_CMD_NVRAM_TYPE_DYNAMIC_CFG_PORT1; + + if ((rc = siena_nvram_partn_size(enp, partn, sizep)) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +siena_vpd_read( + __in efx_nic_t *enp, + __out_bcount(size) caddr_t data, + __in size_t size) +{ + efx_mcdi_iface_t *emip = &(enp->en_mcdi.em_emip); + siena_mc_dynamic_config_hdr_t *dcfg = NULL; + unsigned int vpd_length; + unsigned int vpd_offset; + unsigned int dcfg_partn; + size_t dcfg_size; + efx_rc_t rc; + + EFSYS_ASSERT(enp->en_family == EFX_FAMILY_SIENA); + + dcfg_partn = (emip->emi_port == 1) + ? MC_CMD_NVRAM_TYPE_DYNAMIC_CFG_PORT0 + : MC_CMD_NVRAM_TYPE_DYNAMIC_CFG_PORT1; + + if ((rc = siena_nvram_get_dynamic_cfg(enp, dcfg_partn, + B_TRUE, &dcfg, &dcfg_size)) != 0) + goto fail1; + + vpd_length = EFX_DWORD_FIELD(dcfg->dynamic_vpd_length, EFX_DWORD_0); + vpd_offset = EFX_DWORD_FIELD(dcfg->dynamic_vpd_offset, EFX_DWORD_0); + + if (vpd_length > size) { + rc = EFAULT; /* Invalid dcfg: header bigger than sector */ + goto fail2; + } + + EFSYS_ASSERT3U(vpd_length, <=, size); + memcpy(data, (caddr_t)dcfg + vpd_offset, vpd_length); + + /* Pad data with all-1s, consistent with update operations */ + memset(data + vpd_length, 0xff, size - vpd_length); + + EFSYS_KMEM_FREE(enp->en_esip, dcfg_size, dcfg); + + return (0); + +fail2: + EFSYS_PROBE(fail2); + + EFSYS_KMEM_FREE(enp->en_esip, dcfg_size, dcfg); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +siena_vpd_verify( + __in efx_nic_t *enp, + __in_bcount(size) caddr_t data, + __in size_t size) +{ + efx_vpd_tag_t stag; + efx_vpd_tag_t dtag; + efx_vpd_keyword_t skey; + efx_vpd_keyword_t dkey; + unsigned int scont; + unsigned int dcont; + + efx_rc_t rc; + + EFSYS_ASSERT(enp->en_family == EFX_FAMILY_SIENA); + + /* + * Strictly you could take the view that dynamic vpd is optional. + * Instead, to conform more closely to the read/verify/reinit() + * paradigm, we require dynamic vpd. siena_vpd_reinit() will + * reinitialize it as required. + */ + if ((rc = efx_vpd_hunk_verify(data, size, NULL)) != 0) + goto fail1; + + /* + * Verify that there is no duplication between the static and + * dynamic cfg sectors. + */ + if (enp->en_u.siena.enu_svpd_length == 0) + goto done; + + dcont = 0; + _NOTE(CONSTANTCONDITION) + while (1) { + if ((rc = efx_vpd_hunk_next(data, size, &dtag, + &dkey, NULL, NULL, &dcont)) != 0) + goto fail2; + if (dcont == 0) + break; + + /* + * Skip the RV keyword. It should be present in both the static + * and dynamic cfg sectors. + */ + if (dtag == EFX_VPD_RO && dkey == EFX_VPD_KEYWORD('R', 'V')) + continue; + + scont = 0; + _NOTE(CONSTANTCONDITION) + while (1) { + if ((rc = efx_vpd_hunk_next( + enp->en_u.siena.enu_svpd, + enp->en_u.siena.enu_svpd_length, &stag, &skey, + NULL, NULL, &scont)) != 0) + goto fail3; + if (scont == 0) + break; + + if (stag == dtag && skey == dkey) { + rc = EEXIST; + goto fail4; + } + } + } + +done: + return (0); + +fail4: + EFSYS_PROBE(fail4); +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +siena_vpd_reinit( + __in efx_nic_t *enp, + __in_bcount(size) caddr_t data, + __in size_t size) +{ + boolean_t wantpid; + efx_rc_t rc; + + /* + * Only create a PID if the dynamic cfg doesn't have one + */ + if (enp->en_u.siena.enu_svpd_length == 0) + wantpid = B_TRUE; + else { + unsigned int offset; + uint8_t length; + + rc = efx_vpd_hunk_get(enp->en_u.siena.enu_svpd, + enp->en_u.siena.enu_svpd_length, + EFX_VPD_ID, 0, &offset, &length); + if (rc == 0) + wantpid = B_FALSE; + else if (rc == ENOENT) + wantpid = B_TRUE; + else + goto fail1; + } + + if ((rc = efx_vpd_hunk_reinit(data, size, wantpid)) != 0) + goto fail2; + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +siena_vpd_get( + __in efx_nic_t *enp, + __in_bcount(size) caddr_t data, + __in size_t size, + __inout efx_vpd_value_t *evvp) +{ + unsigned int offset; + uint8_t length; + efx_rc_t rc; + + EFSYS_ASSERT(enp->en_family == EFX_FAMILY_SIENA); + + /* Attempt to satisfy the request from svpd first */ + if (enp->en_u.siena.enu_svpd_length > 0) { + if ((rc = efx_vpd_hunk_get(enp->en_u.siena.enu_svpd, + enp->en_u.siena.enu_svpd_length, evvp->evv_tag, + evvp->evv_keyword, &offset, &length)) == 0) { + evvp->evv_length = length; + memcpy(evvp->evv_value, + enp->en_u.siena.enu_svpd + offset, length); + return (0); + } else if (rc != ENOENT) + goto fail1; + } + + /* And then from the provided data buffer */ + if ((rc = efx_vpd_hunk_get(data, size, evvp->evv_tag, + evvp->evv_keyword, &offset, &length)) != 0) { + if (rc == ENOENT) + return (rc); + + goto fail2; + } + + evvp->evv_length = length; + memcpy(evvp->evv_value, data + offset, length); + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +siena_vpd_set( + __in efx_nic_t *enp, + __in_bcount(size) caddr_t data, + __in size_t size, + __in efx_vpd_value_t *evvp) +{ + efx_rc_t rc; + + EFSYS_ASSERT(enp->en_family == EFX_FAMILY_SIENA); + + /* If the provided (tag,keyword) exists in svpd, then it is readonly */ + if (enp->en_u.siena.enu_svpd_length > 0) { + unsigned int offset; + uint8_t length; + + if ((rc = efx_vpd_hunk_get(enp->en_u.siena.enu_svpd, + enp->en_u.siena.enu_svpd_length, evvp->evv_tag, + evvp->evv_keyword, &offset, &length)) == 0) { + rc = EACCES; + goto fail1; + } + } + + if ((rc = efx_vpd_hunk_set(data, size, evvp)) != 0) + goto fail2; + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +siena_vpd_next( + __in efx_nic_t *enp, + __in_bcount(size) caddr_t data, + __in size_t size, + __out efx_vpd_value_t *evvp, + __inout unsigned int *contp) +{ + _NOTE(ARGUNUSED(enp, data, size, evvp, contp)) + + return (ENOTSUP); +} + + __checkReturn efx_rc_t +siena_vpd_write( + __in efx_nic_t *enp, + __in_bcount(size) caddr_t data, + __in size_t size) +{ + efx_mcdi_iface_t *emip = &(enp->en_mcdi.em_emip); + siena_mc_dynamic_config_hdr_t *dcfg = NULL; + unsigned int vpd_offset; + unsigned int dcfg_partn; + unsigned int hdr_length; + unsigned int pos; + uint8_t cksum; + size_t partn_size, dcfg_size; + size_t vpd_length; + efx_rc_t rc; + + EFSYS_ASSERT(enp->en_family == EFX_FAMILY_SIENA); + + /* Determine total length of all tags */ + if ((rc = efx_vpd_hunk_length(data, size, &vpd_length)) != 0) + goto fail1; + + /* Lock dynamic config sector for write, and read structure only */ + dcfg_partn = (emip->emi_port == 1) + ? MC_CMD_NVRAM_TYPE_DYNAMIC_CFG_PORT0 + : MC_CMD_NVRAM_TYPE_DYNAMIC_CFG_PORT1; + + if ((rc = siena_nvram_partn_size(enp, dcfg_partn, &partn_size)) != 0) + goto fail2; + + if ((rc = siena_nvram_partn_lock(enp, dcfg_partn)) != 0) + goto fail3; + + if ((rc = siena_nvram_get_dynamic_cfg(enp, dcfg_partn, + B_FALSE, &dcfg, &dcfg_size)) != 0) + goto fail4; + + hdr_length = EFX_WORD_FIELD(dcfg->length, EFX_WORD_0); + + /* Allocated memory should have room for the new VPD */ + if (hdr_length + vpd_length > dcfg_size) { + rc = ENOSPC; + goto fail5; + } + + /* Copy in new vpd and update header */ + vpd_offset = dcfg_size - vpd_length; + EFX_POPULATE_DWORD_1(dcfg->dynamic_vpd_offset, EFX_DWORD_0, vpd_offset); + memcpy((caddr_t)dcfg + vpd_offset, data, vpd_length); + EFX_POPULATE_DWORD_1(dcfg->dynamic_vpd_length, EFX_DWORD_0, vpd_length); + + /* Update the checksum */ + cksum = 0; + for (pos = 0; pos < hdr_length; pos++) + cksum += ((uint8_t *)dcfg)[pos]; + dcfg->csum.eb_u8[0] -= cksum; + + /* Erase and write the new sector */ + if ((rc = siena_nvram_partn_erase(enp, dcfg_partn, 0, partn_size)) != 0) + goto fail6; + + /* Write out the new structure to nvram */ + if ((rc = siena_nvram_partn_write(enp, dcfg_partn, 0, (caddr_t)dcfg, + vpd_offset + vpd_length)) != 0) + goto fail7; + + EFSYS_KMEM_FREE(enp->en_esip, dcfg_size, dcfg); + + siena_nvram_partn_unlock(enp, dcfg_partn); + + return (0); + +fail7: + EFSYS_PROBE(fail7); +fail6: + EFSYS_PROBE(fail6); +fail5: + EFSYS_PROBE(fail5); + + EFSYS_KMEM_FREE(enp->en_esip, dcfg_size, dcfg); +fail4: + EFSYS_PROBE(fail4); + + siena_nvram_partn_unlock(enp, dcfg_partn); +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + void +siena_vpd_fini( + __in efx_nic_t *enp) +{ + EFSYS_ASSERT(enp->en_family == EFX_FAMILY_SIENA); + + if (enp->en_u.siena.enu_svpd_length > 0) { + EFSYS_KMEM_FREE(enp->en_esip, enp->en_u.siena.enu_svpd_length, + enp->en_u.siena.enu_svpd); + + enp->en_u.siena.enu_svpd = NULL; + enp->en_u.siena.enu_svpd_length = 0; + } +} + +#endif /* EFSYS_OPT_SIENA */ + +#endif /* EFSYS_OPT_VPD */ diff --git a/dpdk/drivers/net/sfc/efsys.h b/dpdk/drivers/net/sfc/efsys.h new file mode 100644 index 00000000..f428b624 --- /dev/null +++ b/dpdk/drivers/net/sfc/efsys.h @@ -0,0 +1,780 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2016-2017 Solarflare Communications Inc. + * All rights reserved. + * + * This software was jointly developed between OKTET Labs (under contract + * for Solarflare) and Solarflare Communications, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _SFC_COMMON_EFSYS_H +#define _SFC_COMMON_EFSYS_H + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sfc_debug.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define EFSYS_HAS_UINT64 1 +#define EFSYS_USE_UINT64 1 +#define EFSYS_HAS_SSE2_M128 1 + +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN +#define EFSYS_IS_BIG_ENDIAN 1 +#define EFSYS_IS_LITTLE_ENDIAN 0 +#elif RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN +#define EFSYS_IS_BIG_ENDIAN 0 +#define EFSYS_IS_LITTLE_ENDIAN 1 +#else +#error "Cannot determine system endianness" +#endif +#include "efx_types.h" + + +#ifndef _NOTE +#define _NOTE(s) +#endif + +typedef bool boolean_t; + +#ifndef B_FALSE +#define B_FALSE false +#endif +#ifndef B_TRUE +#define B_TRUE true +#endif + +/* + * RTE_MAX() and RTE_MIN() cannot be used since braced-group within + * expression allowed only inside a function, but MAX() is used as + * a number of elements in array. + */ +#ifndef MAX +#define MAX(v1, v2) ((v1) > (v2) ? (v1) : (v2)) +#endif +#ifndef MIN +#define MIN(v1, v2) ((v1) < (v2) ? (v1) : (v2)) +#endif + +/* There are macros for alignment in DPDK, but we need to make a proper + * correspondence here, if we want to re-use them at all + */ +#ifndef IS_P2ALIGNED +#define IS_P2ALIGNED(v, a) ((((uintptr_t)(v)) & ((uintptr_t)(a) - 1)) == 0) +#endif + +#ifndef P2ROUNDUP +#define P2ROUNDUP(x, align) (-(-(x) & -(align))) +#endif + +#ifndef P2ALIGN +#define P2ALIGN(_x, _a) ((_x) & -(_a)) +#endif + +#ifndef IS2P +#define ISP2(x) rte_is_power_of_2(x) +#endif + +#define ENOTACTIVE ENOTCONN + +static inline void +prefetch_read_many(const volatile void *addr) +{ + rte_prefetch0(addr); +} + +static inline void +prefetch_read_once(const volatile void *addr) +{ + rte_prefetch_non_temporal(addr); +} + +/* Modifiers used for Windows builds */ +#define __in +#define __in_opt +#define __in_ecount(_n) +#define __in_ecount_opt(_n) +#define __in_bcount(_n) +#define __in_bcount_opt(_n) + +#define __out +#define __out_opt +#define __out_ecount(_n) +#define __out_ecount_opt(_n) +#define __out_bcount(_n) +#define __out_bcount_opt(_n) + +#define __deref_out + +#define __inout +#define __inout_opt +#define __inout_ecount(_n) +#define __inout_ecount_opt(_n) +#define __inout_bcount(_n) +#define __inout_bcount_opt(_n) +#define __inout_bcount_full_opt(_n) + +#define __deref_out_bcount_opt(n) + +#define __checkReturn +#define __success(_x) + +#define __drv_when(_p, _c) + +/* Code inclusion options */ + + +#define EFSYS_OPT_NAMES 1 + +/* Disable SFN5xxx/SFN6xxx since it requires specific support in the PMD */ +#define EFSYS_OPT_SIENA 0 +/* Enable SFN7xxx support */ +#define EFSYS_OPT_HUNTINGTON 1 +/* Enable SFN8xxx support */ +#define EFSYS_OPT_MEDFORD 1 +#ifdef RTE_LIBRTE_SFC_EFX_DEBUG +#define EFSYS_OPT_CHECK_REG 1 +#else +#define EFSYS_OPT_CHECK_REG 0 +#endif + +/* MCDI is required for SFN7xxx and SFN8xx */ +#define EFSYS_OPT_MCDI 1 +#define EFSYS_OPT_MCDI_LOGGING 1 +#define EFSYS_OPT_MCDI_PROXY_AUTH 1 + +#define EFSYS_OPT_MAC_STATS 1 + +#define EFSYS_OPT_LOOPBACK 0 + +#define EFSYS_OPT_MON_MCDI 0 +#define EFSYS_OPT_MON_STATS 0 + +#define EFSYS_OPT_PHY_STATS 0 +#define EFSYS_OPT_BIST 0 +#define EFSYS_OPT_PHY_LED_CONTROL 0 +#define EFSYS_OPT_PHY_FLAGS 0 + +#define EFSYS_OPT_VPD 0 +#define EFSYS_OPT_NVRAM 0 +#define EFSYS_OPT_BOOTCFG 0 + +#define EFSYS_OPT_DIAG 0 +#define EFSYS_OPT_RX_SCALE 1 +#define EFSYS_OPT_QSTATS 0 +/* Filters support is required for SFN7xxx and SFN8xx */ +#define EFSYS_OPT_FILTER 1 +#define EFSYS_OPT_RX_SCATTER 0 + +#define EFSYS_OPT_EV_PREFETCH 0 + +#define EFSYS_OPT_DECODE_INTR_FATAL 0 + +#define EFSYS_OPT_LICENSING 0 + +#define EFSYS_OPT_ALLOW_UNCONFIGURED_NIC 0 + +#define EFSYS_OPT_RX_PACKED_STREAM 0 + +/* ID */ + +typedef struct __efsys_identifier_s efsys_identifier_t; + + +#define EFSYS_PROBE(_name) \ + do { } while (0) + +#define EFSYS_PROBE1(_name, _type1, _arg1) \ + do { } while (0) + +#define EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2) \ + do { } while (0) + +#define EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2, \ + _type3, _arg3) \ + do { } while (0) + +#define EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ + _type3, _arg3, _type4, _arg4) \ + do { } while (0) + +#define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ + _type3, _arg3, _type4, _arg4, _type5, _arg5) \ + do { } while (0) + +#define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ + _type3, _arg3, _type4, _arg4, _type5, _arg5, \ + _type6, _arg6) \ + do { } while (0) + +#define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ + _type3, _arg3, _type4, _arg4, _type5, _arg5, \ + _type6, _arg6, _type7, _arg7) \ + do { } while (0) + + +/* DMA */ + +typedef rte_iova_t efsys_dma_addr_t; + +typedef struct efsys_mem_s { + const struct rte_memzone *esm_mz; + /* + * Ideally it should have volatile qualifier to denote that + * the memory may be updated by someone else. However, it adds + * qualifier discard warnings when the pointer or its derivative + * is passed to memset() or rte_mov16(). + * So, skip the qualifier here, but make sure that it is added + * below in access macros. + */ + void *esm_base; + efsys_dma_addr_t esm_addr; +} efsys_mem_t; + + +#define EFSYS_MEM_ZERO(_esmp, _size) \ + do { \ + (void)memset((void *)(_esmp)->esm_base, 0, (_size)); \ + \ + _NOTE(CONSTANTCONDITION); \ + } while (B_FALSE) + +#define EFSYS_MEM_READD(_esmp, _offset, _edp) \ + do { \ + volatile uint8_t *_base = (_esmp)->esm_base; \ + volatile uint32_t *_addr; \ + \ + _NOTE(CONSTANTCONDITION); \ + SFC_ASSERT(IS_P2ALIGNED(_offset, sizeof(efx_dword_t))); \ + \ + _addr = (volatile uint32_t *)(_base + (_offset)); \ + (_edp)->ed_u32[0] = _addr[0]; \ + \ + EFSYS_PROBE2(mem_readl, unsigned int, (_offset), \ + uint32_t, (_edp)->ed_u32[0]); \ + \ + _NOTE(CONSTANTCONDITION); \ + } while (B_FALSE) + +#define EFSYS_MEM_READQ(_esmp, _offset, _eqp) \ + do { \ + volatile uint8_t *_base = (_esmp)->esm_base; \ + volatile uint64_t *_addr; \ + \ + _NOTE(CONSTANTCONDITION); \ + SFC_ASSERT(IS_P2ALIGNED(_offset, sizeof(efx_qword_t))); \ + \ + _addr = (volatile uint64_t *)(_base + (_offset)); \ + (_eqp)->eq_u64[0] = _addr[0]; \ + \ + EFSYS_PROBE3(mem_readq, unsigned int, (_offset), \ + uint32_t, (_eqp)->eq_u32[1], \ + uint32_t, (_eqp)->eq_u32[0]); \ + \ + _NOTE(CONSTANTCONDITION); \ + } while (B_FALSE) + +#define EFSYS_MEM_READO(_esmp, _offset, _eop) \ + do { \ + volatile uint8_t *_base = (_esmp)->esm_base; \ + volatile __m128i *_addr; \ + \ + _NOTE(CONSTANTCONDITION); \ + SFC_ASSERT(IS_P2ALIGNED(_offset, sizeof(efx_oword_t))); \ + \ + _addr = (volatile __m128i *)(_base + (_offset)); \ + (_eop)->eo_u128[0] = _addr[0]; \ + \ + EFSYS_PROBE5(mem_reado, unsigned int, (_offset), \ + uint32_t, (_eop)->eo_u32[3], \ + uint32_t, (_eop)->eo_u32[2], \ + uint32_t, (_eop)->eo_u32[1], \ + uint32_t, (_eop)->eo_u32[0]); \ + \ + _NOTE(CONSTANTCONDITION); \ + } while (B_FALSE) + + +#define EFSYS_MEM_WRITED(_esmp, _offset, _edp) \ + do { \ + volatile uint8_t *_base = (_esmp)->esm_base; \ + volatile uint32_t *_addr; \ + \ + _NOTE(CONSTANTCONDITION); \ + SFC_ASSERT(IS_P2ALIGNED(_offset, sizeof(efx_dword_t))); \ + \ + EFSYS_PROBE2(mem_writed, unsigned int, (_offset), \ + uint32_t, (_edp)->ed_u32[0]); \ + \ + _addr = (volatile uint32_t *)(_base + (_offset)); \ + _addr[0] = (_edp)->ed_u32[0]; \ + \ + _NOTE(CONSTANTCONDITION); \ + } while (B_FALSE) + +#define EFSYS_MEM_WRITEQ(_esmp, _offset, _eqp) \ + do { \ + volatile uint8_t *_base = (_esmp)->esm_base; \ + volatile uint64_t *_addr; \ + \ + _NOTE(CONSTANTCONDITION); \ + SFC_ASSERT(IS_P2ALIGNED(_offset, sizeof(efx_qword_t))); \ + \ + EFSYS_PROBE3(mem_writeq, unsigned int, (_offset), \ + uint32_t, (_eqp)->eq_u32[1], \ + uint32_t, (_eqp)->eq_u32[0]); \ + \ + _addr = (volatile uint64_t *)(_base + (_offset)); \ + _addr[0] = (_eqp)->eq_u64[0]; \ + \ + _NOTE(CONSTANTCONDITION); \ + } while (B_FALSE) + +#define EFSYS_MEM_WRITEO(_esmp, _offset, _eop) \ + do { \ + volatile uint8_t *_base = (_esmp)->esm_base; \ + volatile __m128i *_addr; \ + \ + _NOTE(CONSTANTCONDITION); \ + SFC_ASSERT(IS_P2ALIGNED(_offset, sizeof(efx_oword_t))); \ + \ + \ + EFSYS_PROBE5(mem_writeo, unsigned int, (_offset), \ + uint32_t, (_eop)->eo_u32[3], \ + uint32_t, (_eop)->eo_u32[2], \ + uint32_t, (_eop)->eo_u32[1], \ + uint32_t, (_eop)->eo_u32[0]); \ + \ + _addr = (volatile __m128i *)(_base + (_offset)); \ + _addr[0] = (_eop)->eo_u128[0]; \ + \ + _NOTE(CONSTANTCONDITION); \ + } while (B_FALSE) + + +#define EFSYS_MEM_ADDR(_esmp) \ + ((_esmp)->esm_addr) + +#define EFSYS_MEM_IS_NULL(_esmp) \ + ((_esmp)->esm_base == NULL) + +#define EFSYS_MEM_PREFETCH(_esmp, _offset) \ + do { \ + volatile uint8_t *_base = (_esmp)->esm_base; \ + \ + rte_prefetch0(_base + (_offset)); \ + } while (0) + + +/* BAR */ + +typedef struct efsys_bar_s { + rte_spinlock_t esb_lock; + int esb_rid; + struct rte_pci_device *esb_dev; + /* + * Ideally it should have volatile qualifier to denote that + * the memory may be updated by someone else. However, it adds + * qualifier discard warnings when the pointer or its derivative + * is passed to memset() or rte_mov16(). + * So, skip the qualifier here, but make sure that it is added + * below in access macros. + */ + void *esb_base; +} efsys_bar_t; + +#define SFC_BAR_LOCK_INIT(_esbp, _ifname) \ + do { \ + rte_spinlock_init(&(_esbp)->esb_lock); \ + _NOTE(CONSTANTCONDITION); \ + } while (B_FALSE) +#define SFC_BAR_LOCK_DESTROY(_esbp) ((void)0) +#define SFC_BAR_LOCK(_esbp) rte_spinlock_lock(&(_esbp)->esb_lock) +#define SFC_BAR_UNLOCK(_esbp) rte_spinlock_unlock(&(_esbp)->esb_lock) + +#define EFSYS_BAR_READD(_esbp, _offset, _edp, _lock) \ + do { \ + volatile uint8_t *_base = (_esbp)->esb_base; \ + volatile uint32_t *_addr; \ + \ + _NOTE(CONSTANTCONDITION); \ + SFC_ASSERT(IS_P2ALIGNED(_offset, sizeof(efx_dword_t))); \ + _NOTE(CONSTANTCONDITION); \ + if (_lock) \ + SFC_BAR_LOCK(_esbp); \ + \ + _addr = (volatile uint32_t *)(_base + (_offset)); \ + rte_rmb(); \ + (_edp)->ed_u32[0] = rte_read32_relaxed(_addr); \ + \ + EFSYS_PROBE2(bar_readd, unsigned int, (_offset), \ + uint32_t, (_edp)->ed_u32[0]); \ + \ + _NOTE(CONSTANTCONDITION); \ + if (_lock) \ + SFC_BAR_UNLOCK(_esbp); \ + _NOTE(CONSTANTCONDITION); \ + } while (B_FALSE) + +#define EFSYS_BAR_READQ(_esbp, _offset, _eqp) \ + do { \ + volatile uint8_t *_base = (_esbp)->esb_base; \ + volatile uint64_t *_addr; \ + \ + _NOTE(CONSTANTCONDITION); \ + SFC_ASSERT(IS_P2ALIGNED(_offset, sizeof(efx_qword_t))); \ + \ + SFC_BAR_LOCK(_esbp); \ + \ + _addr = (volatile uint64_t *)(_base + (_offset)); \ + rte_rmb(); \ + (_eqp)->eq_u64[0] = rte_read64_relaxed(_addr); \ + \ + EFSYS_PROBE3(bar_readq, unsigned int, (_offset), \ + uint32_t, (_eqp)->eq_u32[1], \ + uint32_t, (_eqp)->eq_u32[0]); \ + \ + SFC_BAR_UNLOCK(_esbp); \ + _NOTE(CONSTANTCONDITION); \ + } while (B_FALSE) + +#define EFSYS_BAR_READO(_esbp, _offset, _eop, _lock) \ + do { \ + volatile uint8_t *_base = (_esbp)->esb_base; \ + volatile __m128i *_addr; \ + \ + _NOTE(CONSTANTCONDITION); \ + SFC_ASSERT(IS_P2ALIGNED(_offset, sizeof(efx_oword_t))); \ + \ + _NOTE(CONSTANTCONDITION); \ + if (_lock) \ + SFC_BAR_LOCK(_esbp); \ + \ + _addr = (volatile __m128i *)(_base + (_offset)); \ + rte_rmb(); \ + /* There is no rte_read128_relaxed() yet */ \ + (_eop)->eo_u128[0] = _addr[0]; \ + \ + EFSYS_PROBE5(bar_reado, unsigned int, (_offset), \ + uint32_t, (_eop)->eo_u32[3], \ + uint32_t, (_eop)->eo_u32[2], \ + uint32_t, (_eop)->eo_u32[1], \ + uint32_t, (_eop)->eo_u32[0]); \ + \ + _NOTE(CONSTANTCONDITION); \ + if (_lock) \ + SFC_BAR_UNLOCK(_esbp); \ + _NOTE(CONSTANTCONDITION); \ + } while (B_FALSE) + + +#define EFSYS_BAR_WRITED(_esbp, _offset, _edp, _lock) \ + do { \ + volatile uint8_t *_base = (_esbp)->esb_base; \ + volatile uint32_t *_addr; \ + \ + _NOTE(CONSTANTCONDITION); \ + SFC_ASSERT(IS_P2ALIGNED(_offset, sizeof(efx_dword_t))); \ + \ + _NOTE(CONSTANTCONDITION); \ + if (_lock) \ + SFC_BAR_LOCK(_esbp); \ + \ + EFSYS_PROBE2(bar_writed, unsigned int, (_offset), \ + uint32_t, (_edp)->ed_u32[0]); \ + \ + _addr = (volatile uint32_t *)(_base + (_offset)); \ + rte_write32_relaxed((_edp)->ed_u32[0], _addr); \ + rte_wmb(); \ + \ + _NOTE(CONSTANTCONDITION); \ + if (_lock) \ + SFC_BAR_UNLOCK(_esbp); \ + _NOTE(CONSTANTCONDITION); \ + } while (B_FALSE) + +#define EFSYS_BAR_WRITEQ(_esbp, _offset, _eqp) \ + do { \ + volatile uint8_t *_base = (_esbp)->esb_base; \ + volatile uint64_t *_addr; \ + \ + _NOTE(CONSTANTCONDITION); \ + SFC_ASSERT(IS_P2ALIGNED(_offset, sizeof(efx_qword_t))); \ + \ + SFC_BAR_LOCK(_esbp); \ + \ + EFSYS_PROBE3(bar_writeq, unsigned int, (_offset), \ + uint32_t, (_eqp)->eq_u32[1], \ + uint32_t, (_eqp)->eq_u32[0]); \ + \ + _addr = (volatile uint64_t *)(_base + (_offset)); \ + rte_write64_relaxed((_eqp)->eq_u64[0], _addr); \ + rte_wmb(); \ + \ + SFC_BAR_UNLOCK(_esbp); \ + _NOTE(CONSTANTCONDITION); \ + } while (B_FALSE) + +/* + * Guarantees 64bit aligned 64bit writes to write combined BAR mapping + * (required by PIO hardware). + * + * Neither VFIO, nor UIO, nor NIC UIO (on FreeBSD) support + * write-combined memory mapped to user-land, so just abort if used. + */ +#define EFSYS_BAR_WC_WRITEQ(_esbp, _offset, _eqp) \ + do { \ + rte_panic("Write-combined BAR access not supported"); \ + } while (B_FALSE) + +#define EFSYS_BAR_WRITEO(_esbp, _offset, _eop, _lock) \ + do { \ + volatile uint8_t *_base = (_esbp)->esb_base; \ + volatile __m128i *_addr; \ + \ + _NOTE(CONSTANTCONDITION); \ + SFC_ASSERT(IS_P2ALIGNED(_offset, sizeof(efx_oword_t))); \ + \ + _NOTE(CONSTANTCONDITION); \ + if (_lock) \ + SFC_BAR_LOCK(_esbp); \ + \ + EFSYS_PROBE5(bar_writeo, unsigned int, (_offset), \ + uint32_t, (_eop)->eo_u32[3], \ + uint32_t, (_eop)->eo_u32[2], \ + uint32_t, (_eop)->eo_u32[1], \ + uint32_t, (_eop)->eo_u32[0]); \ + \ + _addr = (volatile __m128i *)(_base + (_offset)); \ + /* There is no rte_write128_relaxed() yet */ \ + _addr[0] = (_eop)->eo_u128[0]; \ + rte_wmb(); \ + \ + _NOTE(CONSTANTCONDITION); \ + if (_lock) \ + SFC_BAR_UNLOCK(_esbp); \ + _NOTE(CONSTANTCONDITION); \ + } while (B_FALSE) + +/* Use the standard octo-word write for doorbell writes */ +#define EFSYS_BAR_DOORBELL_WRITEO(_esbp, _offset, _eop) \ + do { \ + EFSYS_BAR_WRITEO((_esbp), (_offset), (_eop), B_FALSE); \ + _NOTE(CONSTANTCONDITION); \ + } while (B_FALSE) + +/* SPIN */ + +#define EFSYS_SPIN(_us) \ + do { \ + rte_delay_us(_us); \ + _NOTE(CONSTANTCONDITION); \ + } while (B_FALSE) + +#define EFSYS_SLEEP EFSYS_SPIN + +/* BARRIERS */ + +#define EFSYS_MEM_READ_BARRIER() rte_rmb() +#define EFSYS_PIO_WRITE_BARRIER() rte_io_wmb() + +/* DMA SYNC */ + +/* + * DPDK does not provide any DMA syncing API, and no PMD drivers + * have any traces of explicit DMA syncing. + * DMA mapping is assumed to be coherent. + */ + +#define EFSYS_DMA_SYNC_FOR_KERNEL(_esmp, _offset, _size) ((void)0) + +/* Just avoid store and compiler (impliciltly) reordering */ +#define EFSYS_DMA_SYNC_FOR_DEVICE(_esmp, _offset, _size) rte_wmb() + +/* TIMESTAMP */ + +typedef uint64_t efsys_timestamp_t; + +#define EFSYS_TIMESTAMP(_usp) \ + do { \ + *(_usp) = rte_get_timer_cycles() * 1000000 / \ + rte_get_timer_hz(); \ + _NOTE(CONSTANTCONDITION); \ + } while (B_FALSE) + +/* KMEM */ + +#define EFSYS_KMEM_ALLOC(_esip, _size, _p) \ + do { \ + (_esip) = (_esip); \ + (_p) = rte_zmalloc("sfc", (_size), 0); \ + _NOTE(CONSTANTCONDITION); \ + } while (B_FALSE) + +#define EFSYS_KMEM_FREE(_esip, _size, _p) \ + do { \ + (void)(_esip); \ + (void)(_size); \ + rte_free((_p)); \ + _NOTE(CONSTANTCONDITION); \ + } while (B_FALSE) + +/* LOCK */ + +typedef rte_spinlock_t efsys_lock_t; + +#define SFC_EFSYS_LOCK_INIT(_eslp, _ifname, _label) \ + rte_spinlock_init((_eslp)) +#define SFC_EFSYS_LOCK_DESTROY(_eslp) ((void)0) +#define SFC_EFSYS_LOCK(_eslp) \ + rte_spinlock_lock((_eslp)) +#define SFC_EFSYS_UNLOCK(_eslp) \ + rte_spinlock_unlock((_eslp)) +#define SFC_EFSYS_LOCK_ASSERT_OWNED(_eslp) \ + SFC_ASSERT(rte_spinlock_is_locked((_eslp))) + +typedef int efsys_lock_state_t; + +#define EFSYS_LOCK_MAGIC 0x000010c4 + +#define EFSYS_LOCK(_lockp, _state) \ + do { \ + SFC_EFSYS_LOCK(_lockp); \ + (_state) = EFSYS_LOCK_MAGIC; \ + _NOTE(CONSTANTCONDITION); \ + } while (B_FALSE) + +#define EFSYS_UNLOCK(_lockp, _state) \ + do { \ + SFC_ASSERT((_state) == EFSYS_LOCK_MAGIC); \ + SFC_EFSYS_UNLOCK(_lockp); \ + _NOTE(CONSTANTCONDITION); \ + } while (B_FALSE) + +/* STAT */ + +typedef uint64_t efsys_stat_t; + +#define EFSYS_STAT_INCR(_knp, _delta) \ + do { \ + *(_knp) += (_delta); \ + _NOTE(CONSTANTCONDITION); \ + } while (B_FALSE) + +#define EFSYS_STAT_DECR(_knp, _delta) \ + do { \ + *(_knp) -= (_delta); \ + _NOTE(CONSTANTCONDITION); \ + } while (B_FALSE) + +#define EFSYS_STAT_SET(_knp, _val) \ + do { \ + *(_knp) = (_val); \ + _NOTE(CONSTANTCONDITION); \ + } while (B_FALSE) + +#define EFSYS_STAT_SET_QWORD(_knp, _valp) \ + do { \ + *(_knp) = rte_le_to_cpu_64((_valp)->eq_u64[0]); \ + _NOTE(CONSTANTCONDITION); \ + } while (B_FALSE) + +#define EFSYS_STAT_SET_DWORD(_knp, _valp) \ + do { \ + *(_knp) = rte_le_to_cpu_32((_valp)->ed_u32[0]); \ + _NOTE(CONSTANTCONDITION); \ + } while (B_FALSE) + +#define EFSYS_STAT_INCR_QWORD(_knp, _valp) \ + do { \ + *(_knp) += rte_le_to_cpu_64((_valp)->eq_u64[0]); \ + _NOTE(CONSTANTCONDITION); \ + } while (B_FALSE) + +#define EFSYS_STAT_SUBR_QWORD(_knp, _valp) \ + do { \ + *(_knp) -= rte_le_to_cpu_64((_valp)->eq_u64[0]); \ + _NOTE(CONSTANTCONDITION); \ + } while (B_FALSE) + +/* ERR */ + +#if EFSYS_OPT_DECODE_INTR_FATAL +#define EFSYS_ERR(_esip, _code, _dword0, _dword1) \ + do { \ + (void)(_esip); \ + RTE_LOG(ERR, PMD, "FATAL ERROR #%u (0x%08x%08x)\n", \ + (_code), (_dword0), (_dword1)); \ + _NOTE(CONSTANTCONDITION); \ + } while (B_FALSE) +#endif + +/* ASSERT */ + +/* RTE_VERIFY from DPDK treats expressions with % operator incorrectly, + * so we re-implement it here + */ +#ifdef RTE_LIBRTE_SFC_EFX_DEBUG +#define EFSYS_ASSERT(_exp) \ + do { \ + if (unlikely(!(_exp))) \ + rte_panic("line %d\tassert \"%s\" failed\n", \ + __LINE__, (#_exp)); \ + } while (0) +#else +#define EFSYS_ASSERT(_exp) (void)(_exp) +#endif + +#define EFSYS_ASSERT3(_x, _op, _y, _t) EFSYS_ASSERT((_t)(_x) _op (_t)(_y)) + +#define EFSYS_ASSERT3U(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, uint64_t) +#define EFSYS_ASSERT3S(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, int64_t) +#define EFSYS_ASSERT3P(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, uintptr_t) + +/* ROTATE */ + +#define EFSYS_HAS_ROTL_DWORD 0 + +#ifdef __cplusplus +} +#endif + +#endif /* _SFC_COMMON_EFSYS_H */ diff --git a/dpdk/drivers/net/sfc/rte_pmd_sfc_efx_version.map b/dpdk/drivers/net/sfc/rte_pmd_sfc_efx_version.map new file mode 100644 index 00000000..31eca32e --- /dev/null +++ b/dpdk/drivers/net/sfc/rte_pmd_sfc_efx_version.map @@ -0,0 +1,4 @@ +DPDK_17.02 { + + local: *; +}; diff --git a/dpdk/drivers/net/sfc/sfc.c b/dpdk/drivers/net/sfc/sfc.c new file mode 100644 index 00000000..49d7e937 --- /dev/null +++ b/dpdk/drivers/net/sfc/sfc.c @@ -0,0 +1,754 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2016-2017 Solarflare Communications Inc. + * All rights reserved. + * + * This software was jointly developed between OKTET Labs (under contract + * for Solarflare) and Solarflare Communications, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* sysconf() */ +#include + +#include + +#include "efx.h" + +#include "sfc.h" +#include "sfc_log.h" +#include "sfc_ev.h" +#include "sfc_rx.h" +#include "sfc_tx.h" + + +int +sfc_dma_alloc(const struct sfc_adapter *sa, const char *name, uint16_t id, + size_t len, int socket_id, efsys_mem_t *esmp) +{ + const struct rte_memzone *mz; + + sfc_log_init(sa, "name=%s id=%u len=%lu socket_id=%d", + name, id, len, socket_id); + + mz = rte_eth_dma_zone_reserve(sa->eth_dev, name, id, len, + sysconf(_SC_PAGESIZE), socket_id); + if (mz == NULL) { + sfc_err(sa, "cannot reserve DMA zone for %s:%u %#x@%d: %s", + name, (unsigned int)id, (unsigned int)len, socket_id, + rte_strerror(rte_errno)); + return ENOMEM; + } + + esmp->esm_addr = mz->iova; + if (esmp->esm_addr == RTE_BAD_IOVA) { + (void)rte_memzone_free(mz); + return EFAULT; + } + + esmp->esm_mz = mz; + esmp->esm_base = mz->addr; + + return 0; +} + +void +sfc_dma_free(const struct sfc_adapter *sa, efsys_mem_t *esmp) +{ + int rc; + + sfc_log_init(sa, "name=%s", esmp->esm_mz->name); + + rc = rte_memzone_free(esmp->esm_mz); + if (rc != 0) + sfc_err(sa, "rte_memzone_free(() failed: %d", rc); + + memset(esmp, 0, sizeof(*esmp)); +} + +static uint32_t +sfc_phy_cap_from_link_speeds(uint32_t speeds) +{ + uint32_t phy_caps = 0; + + if (~speeds & ETH_LINK_SPEED_FIXED) { + phy_caps |= (1 << EFX_PHY_CAP_AN); + /* + * If no speeds are specified in the mask, any supported + * may be negotiated + */ + if (speeds == ETH_LINK_SPEED_AUTONEG) + phy_caps |= + (1 << EFX_PHY_CAP_1000FDX) | + (1 << EFX_PHY_CAP_10000FDX) | + (1 << EFX_PHY_CAP_40000FDX); + } + if (speeds & ETH_LINK_SPEED_1G) + phy_caps |= (1 << EFX_PHY_CAP_1000FDX); + if (speeds & ETH_LINK_SPEED_10G) + phy_caps |= (1 << EFX_PHY_CAP_10000FDX); + if (speeds & ETH_LINK_SPEED_40G) + phy_caps |= (1 << EFX_PHY_CAP_40000FDX); + + return phy_caps; +} + +/* + * Check requested device level configuration. + * Receive and transmit configuration is checked in corresponding + * modules. + */ +static int +sfc_check_conf(struct sfc_adapter *sa) +{ + const struct rte_eth_conf *conf = &sa->eth_dev->data->dev_conf; + int rc = 0; + + sa->port.phy_adv_cap = + sfc_phy_cap_from_link_speeds(conf->link_speeds) & + sa->port.phy_adv_cap_mask; + if ((sa->port.phy_adv_cap & ~(1 << EFX_PHY_CAP_AN)) == 0) { + sfc_err(sa, "No link speeds from mask %#x are supported", + conf->link_speeds); + rc = EINVAL; + } + + if (conf->lpbk_mode != 0) { + sfc_err(sa, "Loopback not supported"); + rc = EINVAL; + } + + if (conf->dcb_capability_en != 0) { + sfc_err(sa, "Priority-based flow control not supported"); + rc = EINVAL; + } + + if (conf->fdir_conf.mode != RTE_FDIR_MODE_NONE) { + sfc_err(sa, "Flow Director not supported"); + rc = EINVAL; + } + + if ((conf->intr_conf.lsc != 0) && + (sa->intr.type != EFX_INTR_LINE) && + (sa->intr.type != EFX_INTR_MESSAGE)) { + sfc_err(sa, "Link status change interrupt not supported"); + rc = EINVAL; + } + + if (conf->intr_conf.rxq != 0) { + sfc_err(sa, "Receive queue interrupt not supported"); + rc = EINVAL; + } + + return rc; +} + +/* + * Find out maximum number of receive and transmit queues which could be + * advertised. + * + * NIC is kept initialized on success to allow other modules acquire + * defaults and capabilities. + */ +static int +sfc_estimate_resource_limits(struct sfc_adapter *sa) +{ + const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic); + efx_drv_limits_t limits; + int rc; + uint32_t evq_allocated; + uint32_t rxq_allocated; + uint32_t txq_allocated; + + memset(&limits, 0, sizeof(limits)); + + /* Request at least one Rx and Tx queue */ + limits.edl_min_rxq_count = 1; + limits.edl_min_txq_count = 1; + /* Management event queue plus event queue for each Tx and Rx queue */ + limits.edl_min_evq_count = + 1 + limits.edl_min_rxq_count + limits.edl_min_txq_count; + + /* Divide by number of functions to guarantee that all functions + * will get promised resources + */ + /* FIXME Divide by number of functions (not 2) below */ + limits.edl_max_evq_count = encp->enc_evq_limit / 2; + SFC_ASSERT(limits.edl_max_evq_count >= limits.edl_min_rxq_count); + + /* Split equally between receive and transmit */ + limits.edl_max_rxq_count = + MIN(encp->enc_rxq_limit, (limits.edl_max_evq_count - 1) / 2); + SFC_ASSERT(limits.edl_max_rxq_count >= limits.edl_min_rxq_count); + + limits.edl_max_txq_count = + MIN(encp->enc_txq_limit, + limits.edl_max_evq_count - 1 - limits.edl_max_rxq_count); + + if (sa->tso) + limits.edl_max_txq_count = + MIN(limits.edl_max_txq_count, + encp->enc_fw_assisted_tso_v2_n_contexts / + encp->enc_hw_pf_count); + + SFC_ASSERT(limits.edl_max_txq_count >= limits.edl_min_rxq_count); + + /* Configure the minimum required resources needed for the + * driver to operate, and the maximum desired resources that the + * driver is capable of using. + */ + efx_nic_set_drv_limits(sa->nic, &limits); + + sfc_log_init(sa, "init nic"); + rc = efx_nic_init(sa->nic); + if (rc != 0) + goto fail_nic_init; + + /* Find resource dimensions assigned by firmware to this function */ + rc = efx_nic_get_vi_pool(sa->nic, &evq_allocated, &rxq_allocated, + &txq_allocated); + if (rc != 0) + goto fail_get_vi_pool; + + /* It still may allocate more than maximum, ensure limit */ + evq_allocated = MIN(evq_allocated, limits.edl_max_evq_count); + rxq_allocated = MIN(rxq_allocated, limits.edl_max_rxq_count); + txq_allocated = MIN(txq_allocated, limits.edl_max_txq_count); + + /* Subtract management EVQ not used for traffic */ + SFC_ASSERT(evq_allocated > 0); + evq_allocated--; + + /* Right now we use separate EVQ for Rx and Tx */ + sa->rxq_max = MIN(rxq_allocated, evq_allocated / 2); + sa->txq_max = MIN(txq_allocated, evq_allocated - sa->rxq_max); + + /* Keep NIC initialized */ + return 0; + +fail_get_vi_pool: +fail_nic_init: + efx_nic_fini(sa->nic); + return rc; +} + +static int +sfc_set_drv_limits(struct sfc_adapter *sa) +{ + const struct rte_eth_dev_data *data = sa->eth_dev->data; + efx_drv_limits_t lim; + + memset(&lim, 0, sizeof(lim)); + + /* Limits are strict since take into account initial estimation */ + lim.edl_min_evq_count = lim.edl_max_evq_count = + 1 + data->nb_rx_queues + data->nb_tx_queues; + lim.edl_min_rxq_count = lim.edl_max_rxq_count = data->nb_rx_queues; + lim.edl_min_txq_count = lim.edl_max_txq_count = data->nb_tx_queues; + + return efx_nic_set_drv_limits(sa->nic, &lim); +} + +int +sfc_start(struct sfc_adapter *sa) +{ + int rc; + + sfc_log_init(sa, "entry"); + + SFC_ASSERT(sfc_adapter_is_locked(sa)); + + switch (sa->state) { + case SFC_ADAPTER_CONFIGURED: + break; + case SFC_ADAPTER_STARTED: + sfc_info(sa, "already started"); + return 0; + default: + rc = EINVAL; + goto fail_bad_state; + } + + sa->state = SFC_ADAPTER_STARTING; + + sfc_log_init(sa, "set resource limits"); + rc = sfc_set_drv_limits(sa); + if (rc != 0) + goto fail_set_drv_limits; + + sfc_log_init(sa, "init nic"); + rc = efx_nic_init(sa->nic); + if (rc != 0) + goto fail_nic_init; + + rc = sfc_intr_start(sa); + if (rc != 0) + goto fail_intr_start; + + rc = sfc_ev_start(sa); + if (rc != 0) + goto fail_ev_start; + + rc = sfc_port_start(sa); + if (rc != 0) + goto fail_port_start; + + rc = sfc_rx_start(sa); + if (rc != 0) + goto fail_rx_start; + + rc = sfc_tx_start(sa); + if (rc != 0) + goto fail_tx_start; + + rc = sfc_flow_start(sa); + if (rc != 0) + goto fail_flows_insert; + + sa->state = SFC_ADAPTER_STARTED; + sfc_log_init(sa, "done"); + return 0; + +fail_flows_insert: + sfc_tx_stop(sa); + +fail_tx_start: + sfc_rx_stop(sa); + +fail_rx_start: + sfc_port_stop(sa); + +fail_port_start: + sfc_ev_stop(sa); + +fail_ev_start: + sfc_intr_stop(sa); + +fail_intr_start: + efx_nic_fini(sa->nic); + +fail_nic_init: +fail_set_drv_limits: + sa->state = SFC_ADAPTER_CONFIGURED; +fail_bad_state: + sfc_log_init(sa, "failed %d", rc); + return rc; +} + +void +sfc_stop(struct sfc_adapter *sa) +{ + sfc_log_init(sa, "entry"); + + SFC_ASSERT(sfc_adapter_is_locked(sa)); + + switch (sa->state) { + case SFC_ADAPTER_STARTED: + break; + case SFC_ADAPTER_CONFIGURED: + sfc_info(sa, "already stopped"); + return; + default: + sfc_err(sa, "stop in unexpected state %u", sa->state); + SFC_ASSERT(B_FALSE); + return; + } + + sa->state = SFC_ADAPTER_STOPPING; + + sfc_flow_stop(sa); + sfc_tx_stop(sa); + sfc_rx_stop(sa); + sfc_port_stop(sa); + sfc_ev_stop(sa); + sfc_intr_stop(sa); + efx_nic_fini(sa->nic); + + sa->state = SFC_ADAPTER_CONFIGURED; + sfc_log_init(sa, "done"); +} + +int +sfc_configure(struct sfc_adapter *sa) +{ + int rc; + + sfc_log_init(sa, "entry"); + + SFC_ASSERT(sfc_adapter_is_locked(sa)); + + SFC_ASSERT(sa->state == SFC_ADAPTER_INITIALIZED || + sa->state == SFC_ADAPTER_CONFIGURED); + sa->state = SFC_ADAPTER_CONFIGURING; + + rc = sfc_check_conf(sa); + if (rc != 0) + goto fail_check_conf; + + rc = sfc_intr_configure(sa); + if (rc != 0) + goto fail_intr_configure; + + rc = sfc_port_configure(sa); + if (rc != 0) + goto fail_port_configure; + + rc = sfc_rx_configure(sa); + if (rc != 0) + goto fail_rx_configure; + + rc = sfc_tx_configure(sa); + if (rc != 0) + goto fail_tx_configure; + + sa->state = SFC_ADAPTER_CONFIGURED; + sfc_log_init(sa, "done"); + return 0; + +fail_tx_configure: + sfc_rx_close(sa); + +fail_rx_configure: + sfc_port_close(sa); + +fail_port_configure: + sfc_intr_close(sa); + +fail_intr_configure: +fail_check_conf: + sa->state = SFC_ADAPTER_INITIALIZED; + sfc_log_init(sa, "failed %d", rc); + return rc; +} + +void +sfc_close(struct sfc_adapter *sa) +{ + sfc_log_init(sa, "entry"); + + SFC_ASSERT(sfc_adapter_is_locked(sa)); + + SFC_ASSERT(sa->state == SFC_ADAPTER_CONFIGURED); + sa->state = SFC_ADAPTER_CLOSING; + + sfc_tx_close(sa); + sfc_rx_close(sa); + sfc_port_close(sa); + sfc_intr_close(sa); + + sa->state = SFC_ADAPTER_INITIALIZED; + sfc_log_init(sa, "done"); +} + +static int +sfc_mem_bar_init(struct sfc_adapter *sa) +{ + struct rte_eth_dev *eth_dev = sa->eth_dev; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); + efsys_bar_t *ebp = &sa->mem_bar; + unsigned int i; + struct rte_mem_resource *res; + + for (i = 0; i < RTE_DIM(pci_dev->mem_resource); i++) { + res = &pci_dev->mem_resource[i]; + if ((res->len != 0) && (res->phys_addr != 0)) { + /* Found first memory BAR */ + SFC_BAR_LOCK_INIT(ebp, eth_dev->data->name); + ebp->esb_rid = i; + ebp->esb_dev = pci_dev; + ebp->esb_base = res->addr; + return 0; + } + } + + return EFAULT; +} + +static void +sfc_mem_bar_fini(struct sfc_adapter *sa) +{ + efsys_bar_t *ebp = &sa->mem_bar; + + SFC_BAR_LOCK_DESTROY(ebp); + memset(ebp, 0, sizeof(*ebp)); +} + +#if EFSYS_OPT_RX_SCALE +/* + * A fixed RSS key which has a property of being symmetric + * (symmetrical flows are distributed to the same CPU) + * and also known to give a uniform distribution + * (a good distribution of traffic between different CPUs) + */ +static const uint8_t default_rss_key[EFX_RSS_KEY_SIZE] = { + 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, + 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, + 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, + 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, + 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, +}; +#endif + +#if EFSYS_OPT_RX_SCALE +static int +sfc_set_rss_defaults(struct sfc_adapter *sa) +{ + int rc; + + rc = efx_intr_init(sa->nic, sa->intr.type, NULL); + if (rc != 0) + goto fail_intr_init; + + rc = efx_ev_init(sa->nic); + if (rc != 0) + goto fail_ev_init; + + rc = efx_rx_init(sa->nic); + if (rc != 0) + goto fail_rx_init; + + rc = efx_rx_scale_default_support_get(sa->nic, &sa->rss_support); + if (rc != 0) + goto fail_scale_support_get; + + rc = efx_rx_hash_default_support_get(sa->nic, &sa->hash_support); + if (rc != 0) + goto fail_hash_support_get; + + efx_rx_fini(sa->nic); + efx_ev_fini(sa->nic); + efx_intr_fini(sa->nic); + + sa->rss_hash_types = sfc_rte_to_efx_hash_type(SFC_RSS_OFFLOADS); + + rte_memcpy(sa->rss_key, default_rss_key, sizeof(sa->rss_key)); + + return 0; + +fail_hash_support_get: +fail_scale_support_get: +fail_rx_init: + efx_ev_fini(sa->nic); + +fail_ev_init: + efx_intr_fini(sa->nic); + +fail_intr_init: + return rc; +} +#else +static int +sfc_set_rss_defaults(__rte_unused struct sfc_adapter *sa) +{ + return 0; +} +#endif + +int +sfc_attach(struct sfc_adapter *sa) +{ + const efx_nic_cfg_t *encp; + efx_nic_t *enp = sa->nic; + int rc; + + sfc_log_init(sa, "entry"); + + SFC_ASSERT(sfc_adapter_is_locked(sa)); + + efx_mcdi_new_epoch(enp); + + sfc_log_init(sa, "reset nic"); + rc = efx_nic_reset(enp); + if (rc != 0) + goto fail_nic_reset; + + encp = efx_nic_cfg_get(sa->nic); + + if (sa->dp_tx->features & SFC_DP_TX_FEAT_TSO) { + sa->tso = encp->enc_fw_assisted_tso_v2_enabled; + if (!sa->tso) + sfc_warn(sa, + "TSO support isn't available on this adapter"); + } + + sfc_log_init(sa, "estimate resource limits"); + rc = sfc_estimate_resource_limits(sa); + if (rc != 0) + goto fail_estimate_rsrc_limits; + + sa->txq_max_entries = encp->enc_txq_max_ndescs; + SFC_ASSERT(rte_is_power_of_2(sa->txq_max_entries)); + + rc = sfc_intr_attach(sa); + if (rc != 0) + goto fail_intr_attach; + + rc = sfc_ev_attach(sa); + if (rc != 0) + goto fail_ev_attach; + + rc = sfc_port_attach(sa); + if (rc != 0) + goto fail_port_attach; + + rc = sfc_set_rss_defaults(sa); + if (rc != 0) + goto fail_set_rss_defaults; + + rc = sfc_filter_attach(sa); + if (rc != 0) + goto fail_filter_attach; + + sfc_log_init(sa, "fini nic"); + efx_nic_fini(enp); + + sfc_flow_init(sa); + + sa->state = SFC_ADAPTER_INITIALIZED; + + sfc_log_init(sa, "done"); + return 0; + +fail_filter_attach: +fail_set_rss_defaults: + sfc_port_detach(sa); + +fail_port_attach: + sfc_ev_detach(sa); + +fail_ev_attach: + sfc_intr_detach(sa); + +fail_intr_attach: + efx_nic_fini(sa->nic); + +fail_estimate_rsrc_limits: +fail_nic_reset: + + sfc_log_init(sa, "failed %d", rc); + return rc; +} + +void +sfc_detach(struct sfc_adapter *sa) +{ + sfc_log_init(sa, "entry"); + + SFC_ASSERT(sfc_adapter_is_locked(sa)); + + sfc_flow_fini(sa); + + sfc_filter_detach(sa); + sfc_port_detach(sa); + sfc_ev_detach(sa); + sfc_intr_detach(sa); + + sa->state = SFC_ADAPTER_UNINITIALIZED; +} + +int +sfc_probe(struct sfc_adapter *sa) +{ + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(sa->eth_dev); + efx_nic_t *enp; + int rc; + + sfc_log_init(sa, "entry"); + + SFC_ASSERT(sfc_adapter_is_locked(sa)); + + sa->socket_id = rte_socket_id(); + + sfc_log_init(sa, "init mem bar"); + rc = sfc_mem_bar_init(sa); + if (rc != 0) + goto fail_mem_bar_init; + + sfc_log_init(sa, "get family"); + rc = efx_family(pci_dev->id.vendor_id, pci_dev->id.device_id, + &sa->family); + if (rc != 0) + goto fail_family; + sfc_log_init(sa, "family is %u", sa->family); + + sfc_log_init(sa, "create nic"); + rte_spinlock_init(&sa->nic_lock); + rc = efx_nic_create(sa->family, (efsys_identifier_t *)sa, + &sa->mem_bar, &sa->nic_lock, &enp); + if (rc != 0) + goto fail_nic_create; + sa->nic = enp; + + rc = sfc_mcdi_init(sa); + if (rc != 0) + goto fail_mcdi_init; + + sfc_log_init(sa, "probe nic"); + rc = efx_nic_probe(enp); + if (rc != 0) + goto fail_nic_probe; + + sfc_log_init(sa, "done"); + return 0; + +fail_nic_probe: + sfc_mcdi_fini(sa); + +fail_mcdi_init: + sfc_log_init(sa, "destroy nic"); + sa->nic = NULL; + efx_nic_destroy(enp); + +fail_nic_create: +fail_family: + sfc_mem_bar_fini(sa); + +fail_mem_bar_init: + sfc_log_init(sa, "failed %d", rc); + return rc; +} + +void +sfc_unprobe(struct sfc_adapter *sa) +{ + efx_nic_t *enp = sa->nic; + + sfc_log_init(sa, "entry"); + + SFC_ASSERT(sfc_adapter_is_locked(sa)); + + sfc_log_init(sa, "unprobe nic"); + efx_nic_unprobe(enp); + + sfc_mcdi_fini(sa); + + sfc_log_init(sa, "destroy nic"); + sa->nic = NULL; + efx_nic_destroy(enp); + + sfc_mem_bar_fini(sa); + + sfc_flow_fini(sa); + sa->state = SFC_ADAPTER_UNINITIALIZED; +} diff --git a/dpdk/drivers/net/sfc/sfc.h b/dpdk/drivers/net/sfc/sfc.h new file mode 100644 index 00000000..ef980a40 --- /dev/null +++ b/dpdk/drivers/net/sfc/sfc.h @@ -0,0 +1,362 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2016-2017 Solarflare Communications Inc. + * All rights reserved. + * + * This software was jointly developed between OKTET Labs (under contract + * for Solarflare) and Solarflare Communications, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _SFC_H +#define _SFC_H + +#include + +#include +#include +#include +#include +#include + +#include "efx.h" + +#include "sfc_filter.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#if EFSYS_OPT_RX_SCALE +/** RSS hash offloads mask */ +#define SFC_RSS_OFFLOADS (ETH_RSS_IP | ETH_RSS_TCP) +#endif + +/* + * +---------------+ + * | UNINITIALIZED |<-----------+ + * +---------------+ | + * |.eth_dev_init |.eth_dev_uninit + * V | + * +---------------+------------+ + * | INITIALIZED | + * +---------------+<-----------<---------------+ + * |.dev_configure | | + * V |failed | + * +---------------+------------+ | + * | CONFIGURING | | + * +---------------+----+ | + * |success | | + * | | +---------------+ + * | | | CLOSING | + * | | +---------------+ + * | | ^ + * V |.dev_configure | + * +---------------+----+ |.dev_close + * | CONFIGURED |----------------------------+ + * +---------------+<-----------+ + * |.dev_start | + * V | + * +---------------+ | + * | STARTING |------------^ + * +---------------+ failed | + * |success | + * | +---------------+ + * | | STOPPING | + * | +---------------+ + * | ^ + * V |.dev_stop + * +---------------+------------+ + * | STARTED | + * +---------------+ + */ +enum sfc_adapter_state { + SFC_ADAPTER_UNINITIALIZED = 0, + SFC_ADAPTER_INITIALIZED, + SFC_ADAPTER_CONFIGURING, + SFC_ADAPTER_CONFIGURED, + SFC_ADAPTER_CLOSING, + SFC_ADAPTER_STARTING, + SFC_ADAPTER_STARTED, + SFC_ADAPTER_STOPPING, + + SFC_ADAPTER_NSTATES +}; + +enum sfc_dev_filter_mode { + SFC_DEV_FILTER_MODE_PROMISC = 0, + SFC_DEV_FILTER_MODE_ALLMULTI, + + SFC_DEV_FILTER_NMODES +}; + +enum sfc_mcdi_state { + SFC_MCDI_UNINITIALIZED = 0, + SFC_MCDI_INITIALIZED, + SFC_MCDI_BUSY, + SFC_MCDI_COMPLETED, + + SFC_MCDI_NSTATES +}; + +struct sfc_mcdi { + rte_spinlock_t lock; + efsys_mem_t mem; + enum sfc_mcdi_state state; + efx_mcdi_transport_t transport; + bool logging; + uint32_t proxy_handle; + efx_rc_t proxy_result; +}; + +struct sfc_intr { + efx_intr_type_t type; + rte_intr_callback_fn handler; + boolean_t lsc_intr; +}; + +struct sfc_rxq_info; +struct sfc_txq_info; +struct sfc_dp_rx; + +struct sfc_port { + unsigned int lsc_seq; + + uint32_t phy_adv_cap_mask; + uint32_t phy_adv_cap; + + unsigned int flow_ctrl; + boolean_t flow_ctrl_autoneg; + size_t pdu; + + /* + * Flow API isolated mode overrides promisc and allmulti settings; + * they won't be applied if isolated mode is active + */ + boolean_t isolated; + boolean_t promisc; + boolean_t allmulti; + + struct ether_addr default_mac_addr; + + unsigned int max_mcast_addrs; + unsigned int nb_mcast_addrs; + uint8_t *mcast_addrs; + + rte_spinlock_t mac_stats_lock; + uint64_t *mac_stats_buf; + unsigned int mac_stats_nb_supported; + efsys_mem_t mac_stats_dma_mem; + boolean_t mac_stats_reset_pending; + uint16_t mac_stats_update_period_ms; + uint32_t mac_stats_update_generation; + boolean_t mac_stats_periodic_dma_supported; + uint64_t mac_stats_last_request_timestamp; + + uint32_t mac_stats_mask[EFX_MAC_STATS_MASK_NPAGES]; +}; + +/* Adapter private data */ +struct sfc_adapter { + /* + * PMD setup and configuration is not thread safe. Since it is not + * performance sensitive, it is better to guarantee thread-safety + * and add device level lock. Adapter control operations which + * change its state should acquire the lock. + */ + rte_spinlock_t lock; + enum sfc_adapter_state state; + struct rte_pci_addr pci_addr; + uint16_t port_id; + struct rte_eth_dev *eth_dev; + struct rte_kvargs *kvargs; + bool debug_init; + int socket_id; + efsys_bar_t mem_bar; + efx_family_t family; + efx_nic_t *nic; + rte_spinlock_t nic_lock; + + struct sfc_mcdi mcdi; + struct sfc_intr intr; + struct sfc_port port; + struct sfc_filter filter; + + unsigned int rxq_max; + unsigned int txq_max; + + unsigned int txq_max_entries; + + uint32_t evq_flags; + unsigned int evq_count; + + unsigned int mgmt_evq_index; + /* + * The lock is used to serialise management event queue polling + * which can be done from different context. Also the lock + * guarantees that mgmt_evq_running is preserved while the lock + * is held. It is used to serialise polling and start/stop + * operations. + * + * Locks which may be held when the lock is acquired: + * - adapter lock, when: + * - device start/stop to change mgmt_evq_running + * - any control operations in client side MCDI proxy handling to + * poll management event queue waiting for proxy response + * - MCDI lock, when: + * - any control operations in client side MCDI proxy handling to + * poll management event queue waiting for proxy response + * + * Locks which are acquired with the lock held: + * - nic_lock, when: + * - MC event processing on management event queue polling + * (e.g. MC REBOOT or BADASSERT events) + */ + rte_spinlock_t mgmt_evq_lock; + bool mgmt_evq_running; + struct sfc_evq *mgmt_evq; + + unsigned int rxq_count; + struct sfc_rxq_info *rxq_info; + + unsigned int txq_count; + struct sfc_txq_info *txq_info; + + boolean_t tso; + + unsigned int rss_channels; + +#if EFSYS_OPT_RX_SCALE + efx_rx_scale_context_type_t rss_support; + efx_rx_hash_support_t hash_support; + efx_rx_hash_type_t rss_hash_types; + unsigned int rss_tbl[EFX_RSS_TBL_SIZE]; + uint8_t rss_key[EFX_RSS_KEY_SIZE]; +#endif + + /* + * Shared memory copy of the Rx datapath name to be used by + * the secondary process to find Rx datapath to be used. + */ + char *dp_rx_name; + const struct sfc_dp_rx *dp_rx; + + /* + * Shared memory copy of the Tx datapath name to be used by + * the secondary process to find Rx datapath to be used. + */ + char *dp_tx_name; + const struct sfc_dp_tx *dp_tx; +}; + +/* + * Add wrapper functions to acquire/release lock to be able to remove or + * change the lock in one place. + */ + +static inline void +sfc_adapter_lock_init(struct sfc_adapter *sa) +{ + rte_spinlock_init(&sa->lock); +} + +static inline int +sfc_adapter_is_locked(struct sfc_adapter *sa) +{ + return rte_spinlock_is_locked(&sa->lock); +} + +static inline void +sfc_adapter_lock(struct sfc_adapter *sa) +{ + rte_spinlock_lock(&sa->lock); +} + +static inline int +sfc_adapter_trylock(struct sfc_adapter *sa) +{ + return rte_spinlock_trylock(&sa->lock); +} + +static inline void +sfc_adapter_unlock(struct sfc_adapter *sa) +{ + rte_spinlock_unlock(&sa->lock); +} + +static inline void +sfc_adapter_lock_fini(__rte_unused struct sfc_adapter *sa) +{ + /* Just for symmetry of the API */ +} + +/** Get the number of milliseconds since boot from the default timer */ +static inline uint64_t +sfc_get_system_msecs(void) +{ + return rte_get_timer_cycles() * MS_PER_S / rte_get_timer_hz(); +} + +int sfc_dma_alloc(const struct sfc_adapter *sa, const char *name, uint16_t id, + size_t len, int socket_id, efsys_mem_t *esmp); +void sfc_dma_free(const struct sfc_adapter *sa, efsys_mem_t *esmp); + +int sfc_probe(struct sfc_adapter *sa); +void sfc_unprobe(struct sfc_adapter *sa); +int sfc_attach(struct sfc_adapter *sa); +void sfc_detach(struct sfc_adapter *sa); +int sfc_start(struct sfc_adapter *sa); +void sfc_stop(struct sfc_adapter *sa); + +int sfc_mcdi_init(struct sfc_adapter *sa); +void sfc_mcdi_fini(struct sfc_adapter *sa); + +int sfc_configure(struct sfc_adapter *sa); +void sfc_close(struct sfc_adapter *sa); + +int sfc_intr_attach(struct sfc_adapter *sa); +void sfc_intr_detach(struct sfc_adapter *sa); +int sfc_intr_configure(struct sfc_adapter *sa); +void sfc_intr_close(struct sfc_adapter *sa); +int sfc_intr_start(struct sfc_adapter *sa); +void sfc_intr_stop(struct sfc_adapter *sa); + +int sfc_port_attach(struct sfc_adapter *sa); +void sfc_port_detach(struct sfc_adapter *sa); +int sfc_port_configure(struct sfc_adapter *sa); +void sfc_port_close(struct sfc_adapter *sa); +int sfc_port_start(struct sfc_adapter *sa); +void sfc_port_stop(struct sfc_adapter *sa); +void sfc_port_link_mode_to_info(efx_link_mode_t link_mode, + struct rte_eth_link *link_info); +int sfc_port_update_mac_stats(struct sfc_adapter *sa); +int sfc_port_reset_mac_stats(struct sfc_adapter *sa); +int sfc_set_rx_mode(struct sfc_adapter *sa); + + +#ifdef __cplusplus +} +#endif + +#endif /* _SFC_H */ diff --git a/dpdk/drivers/net/sfc/sfc_debug.h b/dpdk/drivers/net/sfc/sfc_debug.h new file mode 100644 index 00000000..92eba9c3 --- /dev/null +++ b/dpdk/drivers/net/sfc/sfc_debug.h @@ -0,0 +1,58 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2016-2017 Solarflare Communications Inc. + * All rights reserved. + * + * This software was jointly developed between OKTET Labs (under contract + * for Solarflare) and Solarflare Communications, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _SFC_DEBUG_H_ +#define _SFC_DEBUG_H_ + +#include + +#ifdef RTE_LIBRTE_SFC_EFX_DEBUG +/* Avoid dependency from RTE_LOG_LEVEL to be able to enable debug check + * in the driver only. + */ +#define SFC_ASSERT(exp) RTE_VERIFY(exp) +#else +/* If the driver debug is not enabled, follow DPDK debug/non-debug */ +#define SFC_ASSERT(exp) RTE_ASSERT(exp) +#endif + +/* Log PMD message, automatically add prefix and \n */ +#define sfc_panic(sa, fmt, args...) \ + do { \ + const struct sfc_adapter *_sa = (sa); \ + \ + rte_panic("sfc " PCI_PRI_FMT " #%" PRIu8 ": " fmt "\n", \ + _sa->pci_addr.domain, _sa->pci_addr.bus, \ + _sa->pci_addr.devid, _sa->pci_addr.function, \ + _sa->port_id, ##args); \ + } while (0) + +#endif /* _SFC_DEBUG_H_ */ diff --git a/dpdk/drivers/net/sfc/sfc_dp.c b/dpdk/drivers/net/sfc/sfc_dp.c new file mode 100644 index 00000000..860aa921 --- /dev/null +++ b/dpdk/drivers/net/sfc/sfc_dp.c @@ -0,0 +1,100 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2017 Solarflare Communications Inc. + * All rights reserved. + * + * This software was jointly developed between OKTET Labs (under contract + * for Solarflare) and Solarflare Communications, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include + +#include + +#include "sfc_dp.h" + +void +sfc_dp_queue_init(struct sfc_dp_queue *dpq, uint16_t port_id, uint16_t queue_id, + const struct rte_pci_addr *pci_addr) +{ + dpq->port_id = port_id; + dpq->queue_id = queue_id; + dpq->pci_addr = *pci_addr; +} + +struct sfc_dp * +sfc_dp_find_by_name(struct sfc_dp_list *head, enum sfc_dp_type type, + const char *name) +{ + struct sfc_dp *entry; + + TAILQ_FOREACH(entry, head, links) { + if (entry->type != type) + continue; + + if (strcmp(entry->name, name) == 0) + return entry; + } + + return NULL; +} + +struct sfc_dp * +sfc_dp_find_by_caps(struct sfc_dp_list *head, enum sfc_dp_type type, + unsigned int avail_caps) +{ + struct sfc_dp *entry; + + TAILQ_FOREACH(entry, head, links) { + if (entry->type != type) + continue; + + /* Take the first matching */ + if (sfc_dp_match_hw_fw_caps(entry, avail_caps)) + return entry; + } + + return NULL; +} + +int +sfc_dp_register(struct sfc_dp_list *head, struct sfc_dp *entry) +{ + if (sfc_dp_find_by_name(head, entry->type, entry->name) != NULL) { + rte_log(RTE_LOG_ERR, RTE_LOGTYPE_PMD, + "sfc %s dapapath '%s' already registered\n", + entry->type == SFC_DP_RX ? "Rx" : + entry->type == SFC_DP_TX ? "Tx" : + "unknown", + entry->name); + return EEXIST; + } + + TAILQ_INSERT_TAIL(head, entry, links); + + return 0; +} diff --git a/dpdk/drivers/net/sfc/sfc_dp.h b/dpdk/drivers/net/sfc/sfc_dp.h new file mode 100644 index 00000000..eff0aa87 --- /dev/null +++ b/dpdk/drivers/net/sfc/sfc_dp.h @@ -0,0 +1,125 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2017 Solarflare Communications Inc. + * All rights reserved. + * + * This software was jointly developed between OKTET Labs (under contract + * for Solarflare) and Solarflare Communications, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _SFC_DP_H +#define _SFC_DP_H + +#include +#include + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#define SFC_DIV_ROUND_UP(a, b) \ + __extension__ ({ \ + typeof(a) _a = (a); \ + typeof(b) _b = (b); \ + \ + (_a + (_b - 1)) / _b; \ + }) + +/** + * Datapath exception handler to be provided by the control path. + */ +typedef void (sfc_dp_exception_t)(void *ctrl); + +enum sfc_dp_type { + SFC_DP_RX = 0, /**< Receive datapath */ + SFC_DP_TX, /**< Transmit datapath */ +}; + + +/** Datapath queue run-time information */ +struct sfc_dp_queue { + uint16_t port_id; + uint16_t queue_id; + struct rte_pci_addr pci_addr; +}; + +void sfc_dp_queue_init(struct sfc_dp_queue *dpq, + uint16_t port_id, uint16_t queue_id, + const struct rte_pci_addr *pci_addr); + +/* + * Helper macro to define datapath logging macros and have uniform + * logging. + */ +#define SFC_DP_LOG(dp_name, level, dpq, ...) \ + do { \ + const struct sfc_dp_queue *_dpq = (dpq); \ + const struct rte_pci_addr *_addr = &(_dpq)->pci_addr; \ + \ + RTE_LOG(level, PMD, \ + RTE_FMT("%s " PCI_PRI_FMT \ + " #%" PRIu16 ".%" PRIu16 ": " \ + RTE_FMT_HEAD(__VA_ARGS__,) "\n", \ + dp_name, \ + _addr->domain, _addr->bus, \ + _addr->devid, _addr->function, \ + _dpq->port_id, _dpq->queue_id, \ + RTE_FMT_TAIL(__VA_ARGS__,))); \ + } while (0) + + +/** Datapath definition */ +struct sfc_dp { + TAILQ_ENTRY(sfc_dp) links; + const char *name; + enum sfc_dp_type type; + /* Mask of required hardware/firmware capabilities */ + unsigned int hw_fw_caps; +#define SFC_DP_HW_FW_CAP_EF10 0x1 +}; + +/** List of datapath variants */ +TAILQ_HEAD(sfc_dp_list, sfc_dp); + +/* Check if available HW/FW capabilities are sufficient for the datapath */ +static inline bool +sfc_dp_match_hw_fw_caps(const struct sfc_dp *dp, unsigned int avail_caps) +{ + return (dp->hw_fw_caps & avail_caps) == dp->hw_fw_caps; +} + +struct sfc_dp *sfc_dp_find_by_name(struct sfc_dp_list *head, + enum sfc_dp_type type, const char *name); +struct sfc_dp *sfc_dp_find_by_caps(struct sfc_dp_list *head, + enum sfc_dp_type type, + unsigned int avail_caps); +int sfc_dp_register(struct sfc_dp_list *head, struct sfc_dp *entry); + +#ifdef __cplusplus +} +#endif +#endif /* _SFC_DP_H */ diff --git a/dpdk/drivers/net/sfc/sfc_dp_rx.h b/dpdk/drivers/net/sfc/sfc_dp_rx.h new file mode 100644 index 00000000..3f6a604b --- /dev/null +++ b/dpdk/drivers/net/sfc/sfc_dp_rx.h @@ -0,0 +1,203 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2017 Solarflare Communications Inc. + * All rights reserved. + * + * This software was jointly developed between OKTET Labs (under contract + * for Solarflare) and Solarflare Communications, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _SFC_DP_RX_H +#define _SFC_DP_RX_H + +#include +#include + +#include "sfc_dp.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Generic receive queue information used on data path. + * It must be kept as small as it is possible since it is built into + * the structure used on datapath. + */ +struct sfc_dp_rxq { + struct sfc_dp_queue dpq; +}; + +/** + * Datapath receive queue creation information. + * + * The structure is used just to pass information from control path to + * datapath. It could be just function arguments, but it would be hardly + * readable. + */ +struct sfc_dp_rx_qcreate_info { + /** Memory pool to allocate Rx buffer from */ + struct rte_mempool *refill_mb_pool; + /** Minimum number of unused Rx descriptors to do refill */ + unsigned int refill_threshold; + /** + * Usable mbuf data space in accordance with alignment and + * padding requirements imposed by HW. + */ + unsigned int buf_size; + + /** + * Maximum number of Rx descriptors completed in one Rx event. + * Just for sanity checks if datapath would like to do. + */ + unsigned int batch_max; + + /** Pseudo-header size */ + unsigned int prefix_size; + + /** Receive queue flags initializer */ + unsigned int flags; +#define SFC_RXQ_FLAG_RSS_HASH 0x1 + + /** Rx queue size */ + unsigned int rxq_entries; + /** DMA-mapped Rx descriptors ring */ + void *rxq_hw_ring; + + /** Associated event queue size */ + unsigned int evq_entries; + /** Hardware event ring */ + void *evq_hw_ring; + + /** The queue index in hardware (required to push right doorbell) */ + unsigned int hw_index; + /** + * Virtual address of the memory-mapped BAR to push Rx refill + * doorbell + */ + volatile void *mem_bar; +}; + +/** + * Allocate and initialize datapath receive queue. + * + * @param port_id The port identifier + * @param queue_id The queue identifier + * @param pci_addr PCI function address + * @param socket_id Socket identifier to allocate memory + * @param info Receive queue information + * @param dp_rxqp Location for generic datapath receive queue pointer + * + * @return 0 or positive errno. + */ +typedef int (sfc_dp_rx_qcreate_t)(uint16_t port_id, uint16_t queue_id, + const struct rte_pci_addr *pci_addr, + int socket_id, + const struct sfc_dp_rx_qcreate_info *info, + struct sfc_dp_rxq **dp_rxqp); + +/** + * Free resources allocated for datapath recevie queue. + */ +typedef void (sfc_dp_rx_qdestroy_t)(struct sfc_dp_rxq *dp_rxq); + +/** + * Receive queue start callback. + * + * It handovers EvQ to the datapath. + */ +typedef int (sfc_dp_rx_qstart_t)(struct sfc_dp_rxq *dp_rxq, + unsigned int evq_read_ptr); + +/** + * Receive queue stop function called before flush. + */ +typedef void (sfc_dp_rx_qstop_t)(struct sfc_dp_rxq *dp_rxq, + unsigned int *evq_read_ptr); + +/** + * Receive event handler used during queue flush only. + */ +typedef bool (sfc_dp_rx_qrx_ev_t)(struct sfc_dp_rxq *dp_rxq, unsigned int id); + +/** + * Receive queue purge function called after queue flush. + * + * Should be used to free unused recevie buffers. + */ +typedef void (sfc_dp_rx_qpurge_t)(struct sfc_dp_rxq *dp_rxq); + +/** Get packet types recognized/classified */ +typedef const uint32_t * (sfc_dp_rx_supported_ptypes_get_t)(void); + +/** Get number of pending Rx descriptors */ +typedef unsigned int (sfc_dp_rx_qdesc_npending_t)(struct sfc_dp_rxq *dp_rxq); + +/** Check Rx descriptor status */ +typedef int (sfc_dp_rx_qdesc_status_t)(struct sfc_dp_rxq *dp_rxq, + uint16_t offset); + +/** Receive datapath definition */ +struct sfc_dp_rx { + struct sfc_dp dp; + + unsigned int features; +#define SFC_DP_RX_FEAT_SCATTER 0x1 +#define SFC_DP_RX_FEAT_MULTI_PROCESS 0x2 + sfc_dp_rx_qcreate_t *qcreate; + sfc_dp_rx_qdestroy_t *qdestroy; + sfc_dp_rx_qstart_t *qstart; + sfc_dp_rx_qstop_t *qstop; + sfc_dp_rx_qrx_ev_t *qrx_ev; + sfc_dp_rx_qpurge_t *qpurge; + sfc_dp_rx_supported_ptypes_get_t *supported_ptypes_get; + sfc_dp_rx_qdesc_npending_t *qdesc_npending; + sfc_dp_rx_qdesc_status_t *qdesc_status; + eth_rx_burst_t pkt_burst; +}; + +static inline struct sfc_dp_rx * +sfc_dp_find_rx_by_name(struct sfc_dp_list *head, const char *name) +{ + struct sfc_dp *p = sfc_dp_find_by_name(head, SFC_DP_RX, name); + + return (p == NULL) ? NULL : container_of(p, struct sfc_dp_rx, dp); +} + +static inline struct sfc_dp_rx * +sfc_dp_find_rx_by_caps(struct sfc_dp_list *head, unsigned int avail_caps) +{ + struct sfc_dp *p = sfc_dp_find_by_caps(head, SFC_DP_RX, avail_caps); + + return (p == NULL) ? NULL : container_of(p, struct sfc_dp_rx, dp); +} + +extern struct sfc_dp_rx sfc_efx_rx; +extern struct sfc_dp_rx sfc_ef10_rx; + +#ifdef __cplusplus +} +#endif +#endif /* _SFC_DP_RX_H */ diff --git a/dpdk/drivers/net/sfc/sfc_dp_tx.h b/dpdk/drivers/net/sfc/sfc_dp_tx.h new file mode 100644 index 00000000..94d1b108 --- /dev/null +++ b/dpdk/drivers/net/sfc/sfc_dp_tx.h @@ -0,0 +1,180 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2016 Solarflare Communications Inc. + * All rights reserved. + * + * This software was jointly developed between OKTET Labs (under contract + * for Solarflare) and Solarflare Communications, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _SFC_DP_TX_H +#define _SFC_DP_TX_H + +#include + +#include "sfc_dp.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Generic transmit queue information used on data path. + * It must be kept as small as it is possible since it is built into + * the structure used on datapath. + */ +struct sfc_dp_txq { + struct sfc_dp_queue dpq; +}; + +/** + * Datapath transmit queue creation information. + * + * The structure is used just to pass information from control path to + * datapath. It could be just function arguments, but it would be hardly + * readable. + */ +struct sfc_dp_tx_qcreate_info { + /** Minimum number of unused Tx descriptors to do reap */ + unsigned int free_thresh; + /** Transmit queue configuration flags */ + unsigned int flags; + /** Tx queue size */ + unsigned int txq_entries; + /** Maximum size of data in the DMA descriptor */ + uint16_t dma_desc_size_max; + /** DMA-mapped Tx descriptors ring */ + void *txq_hw_ring; + /** Associated event queue size */ + unsigned int evq_entries; + /** Hardware event ring */ + void *evq_hw_ring; + /** The queue index in hardware (required to push right doorbell) */ + unsigned int hw_index; + /** Virtual address of the memory-mapped BAR to push Tx doorbell */ + volatile void *mem_bar; +}; + +/** + * Allocate and initialize datapath transmit queue. + * + * @param port_id The port identifier + * @param queue_id The queue identifier + * @param pci_addr PCI function address + * @param socket_id Socket identifier to allocate memory + * @param info Tx queue details wrapped in structure + * @param dp_txqp Location for generic datapath transmit queue pointer + * + * @return 0 or positive errno. + */ +typedef int (sfc_dp_tx_qcreate_t)(uint16_t port_id, uint16_t queue_id, + const struct rte_pci_addr *pci_addr, + int socket_id, + const struct sfc_dp_tx_qcreate_info *info, + struct sfc_dp_txq **dp_txqp); + +/** + * Free resources allocated for datapath transmit queue. + */ +typedef void (sfc_dp_tx_qdestroy_t)(struct sfc_dp_txq *dp_txq); + +/** + * Transmit queue start callback. + * + * It handovers EvQ to the datapath. + */ +typedef int (sfc_dp_tx_qstart_t)(struct sfc_dp_txq *dp_txq, + unsigned int evq_read_ptr, + unsigned int txq_desc_index); + +/** + * Transmit queue stop function called before the queue flush. + * + * It returns EvQ to the control path. + */ +typedef void (sfc_dp_tx_qstop_t)(struct sfc_dp_txq *dp_txq, + unsigned int *evq_read_ptr); + +/** + * Transmit event handler used during queue flush only. + */ +typedef bool (sfc_dp_tx_qtx_ev_t)(struct sfc_dp_txq *dp_txq, unsigned int id); + +/** + * Transmit queue function called after the queue flush. + */ +typedef void (sfc_dp_tx_qreap_t)(struct sfc_dp_txq *dp_txq); + +/** + * Check Tx descriptor status + */ +typedef int (sfc_dp_tx_qdesc_status_t)(struct sfc_dp_txq *dp_txq, + uint16_t offset); + +/** Transmit datapath definition */ +struct sfc_dp_tx { + struct sfc_dp dp; + + unsigned int features; +#define SFC_DP_TX_FEAT_VLAN_INSERT 0x1 +#define SFC_DP_TX_FEAT_TSO 0x2 +#define SFC_DP_TX_FEAT_MULTI_SEG 0x4 +#define SFC_DP_TX_FEAT_MULTI_PROCESS 0x8 +#define SFC_DP_TX_FEAT_MULTI_POOL 0x10 +#define SFC_DP_TX_FEAT_REFCNT 0x20 + sfc_dp_tx_qcreate_t *qcreate; + sfc_dp_tx_qdestroy_t *qdestroy; + sfc_dp_tx_qstart_t *qstart; + sfc_dp_tx_qstop_t *qstop; + sfc_dp_tx_qtx_ev_t *qtx_ev; + sfc_dp_tx_qreap_t *qreap; + sfc_dp_tx_qdesc_status_t *qdesc_status; + eth_tx_burst_t pkt_burst; +}; + +static inline struct sfc_dp_tx * +sfc_dp_find_tx_by_name(struct sfc_dp_list *head, const char *name) +{ + struct sfc_dp *p = sfc_dp_find_by_name(head, SFC_DP_TX, name); + + return (p == NULL) ? NULL : container_of(p, struct sfc_dp_tx, dp); +} + +static inline struct sfc_dp_tx * +sfc_dp_find_tx_by_caps(struct sfc_dp_list *head, unsigned int avail_caps) +{ + struct sfc_dp *p = sfc_dp_find_by_caps(head, SFC_DP_TX, avail_caps); + + return (p == NULL) ? NULL : container_of(p, struct sfc_dp_tx, dp); +} + +extern struct sfc_dp_tx sfc_efx_tx; +extern struct sfc_dp_tx sfc_ef10_tx; +extern struct sfc_dp_tx sfc_ef10_simple_tx; + +#ifdef __cplusplus +} +#endif +#endif /* _SFC_DP_TX_H */ diff --git a/dpdk/drivers/net/sfc/sfc_ef10.h b/dpdk/drivers/net/sfc/sfc_ef10.h new file mode 100644 index 00000000..060d8fef --- /dev/null +++ b/dpdk/drivers/net/sfc/sfc_ef10.h @@ -0,0 +1,107 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2017 Solarflare Communications Inc. + * All rights reserved. + * + * This software was jointly developed between OKTET Labs (under contract + * for Solarflare) and Solarflare Communications, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _SFC_EF10_H +#define _SFC_EF10_H + +#ifdef __cplusplus +extern "C" { +#endif + +/* Number of events in one cache line */ +#define SFC_EF10_EV_PER_CACHE_LINE \ + (RTE_CACHE_LINE_SIZE / sizeof(efx_qword_t)) + +#define SFC_EF10_EV_QCLEAR_MASK (~(SFC_EF10_EV_PER_CACHE_LINE - 1)) + +#if defined(SFC_EF10_EV_QCLEAR_USE_EFX) +static inline void +sfc_ef10_ev_qclear_cache_line(void *ptr) +{ + efx_qword_t *entry = ptr; + unsigned int i; + + for (i = 0; i < SFC_EF10_EV_PER_CACHE_LINE; ++i) + EFX_SET_QWORD(entry[i]); +} +#else +/* + * It is possible to do it using AVX2 and AVX512F, but it shows less + * performance. + */ +static inline void +sfc_ef10_ev_qclear_cache_line(void *ptr) +{ + const __m128i val = _mm_set1_epi64x(UINT64_MAX); + __m128i *addr = ptr; + unsigned int i; + + RTE_BUILD_BUG_ON(sizeof(val) > RTE_CACHE_LINE_SIZE); + RTE_BUILD_BUG_ON(RTE_CACHE_LINE_SIZE % sizeof(val) != 0); + + for (i = 0; i < RTE_CACHE_LINE_SIZE / sizeof(val); ++i) + _mm_store_si128(&addr[i], val); +} +#endif + +static inline void +sfc_ef10_ev_qclear(efx_qword_t *hw_ring, unsigned int ptr_mask, + unsigned int old_read_ptr, unsigned int read_ptr) +{ + const unsigned int clear_ptr = read_ptr & SFC_EF10_EV_QCLEAR_MASK; + unsigned int old_clear_ptr = old_read_ptr & SFC_EF10_EV_QCLEAR_MASK; + + while (old_clear_ptr != clear_ptr) { + sfc_ef10_ev_qclear_cache_line( + &hw_ring[old_clear_ptr & ptr_mask]); + old_clear_ptr += SFC_EF10_EV_PER_CACHE_LINE; + } + + /* + * No barriers here. + * Functions which push doorbell should care about correct + * ordering: store instructions which fill in EvQ ring should be + * retired from CPU and DMA sync before doorbell which will allow + * to use these event entries. + */ +} + +static inline bool +sfc_ef10_ev_present(const efx_qword_t ev) +{ + return ~EFX_QWORD_FIELD(ev, EFX_DWORD_0) | + ~EFX_QWORD_FIELD(ev, EFX_DWORD_1); +} + +#ifdef __cplusplus +} +#endif +#endif /* _SFC_EF10_H */ diff --git a/dpdk/drivers/net/sfc/sfc_ef10_rx.c b/dpdk/drivers/net/sfc/sfc_ef10_rx.c new file mode 100644 index 00000000..4c76f747 --- /dev/null +++ b/dpdk/drivers/net/sfc/sfc_ef10_rx.c @@ -0,0 +1,721 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2016 Solarflare Communications Inc. + * All rights reserved. + * + * This software was jointly developed between OKTET Labs (under contract + * for Solarflare) and Solarflare Communications, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* EF10 native datapath implementation */ + +#include + +#include +#include +#include +#include + +#include "efx.h" +#include "efx_types.h" +#include "efx_regs.h" +#include "efx_regs_ef10.h" + +#include "sfc_tweak.h" +#include "sfc_dp_rx.h" +#include "sfc_kvargs.h" +#include "sfc_ef10.h" + +#define sfc_ef10_rx_err(dpq, ...) \ + SFC_DP_LOG(SFC_KVARG_DATAPATH_EF10, ERR, dpq, __VA_ARGS__) + +/** + * Alignment requirement for value written to RX WPTR: + * the WPTR must be aligned to an 8 descriptor boundary. + */ +#define SFC_EF10_RX_WPTR_ALIGN 8 + +/** + * Maximum number of descriptors/buffers in the Rx ring. + * It should guarantee that corresponding event queue never overfill. + * EF10 native datapath uses event queue of the same size as Rx queue. + * Maximum number of events on datapath can be estimated as number of + * Rx queue entries (one event per Rx buffer in the worst case) plus + * Rx error and flush events. + */ +#define SFC_EF10_RXQ_LIMIT(_ndesc) \ + ((_ndesc) - 1 /* head must not step on tail */ - \ + (SFC_EF10_EV_PER_CACHE_LINE - 1) /* max unused EvQ entries */ - \ + 1 /* Rx error */ - 1 /* flush */) + +struct sfc_ef10_rx_sw_desc { + struct rte_mbuf *mbuf; +}; + +struct sfc_ef10_rxq { + /* Used on data path */ + unsigned int flags; +#define SFC_EF10_RXQ_STARTED 0x1 +#define SFC_EF10_RXQ_NOT_RUNNING 0x2 +#define SFC_EF10_RXQ_EXCEPTION 0x4 +#define SFC_EF10_RXQ_RSS_HASH 0x8 + unsigned int ptr_mask; + unsigned int prepared; + unsigned int completed; + unsigned int evq_read_ptr; + efx_qword_t *evq_hw_ring; + struct sfc_ef10_rx_sw_desc *sw_ring; + uint64_t rearm_data; + uint16_t prefix_size; + + /* Used on refill */ + uint16_t buf_size; + unsigned int added; + unsigned int refill_threshold; + struct rte_mempool *refill_mb_pool; + efx_qword_t *rxq_hw_ring; + volatile void *doorbell; + + /* Datapath receive queue anchor */ + struct sfc_dp_rxq dp; +}; + +static inline struct sfc_ef10_rxq * +sfc_ef10_rxq_by_dp_rxq(struct sfc_dp_rxq *dp_rxq) +{ + return container_of(dp_rxq, struct sfc_ef10_rxq, dp); +} + +static void +sfc_ef10_rx_qpush(struct sfc_ef10_rxq *rxq) +{ + efx_dword_t dword; + + /* Hardware has alignment restriction for WPTR */ + RTE_BUILD_BUG_ON(SFC_RX_REFILL_BULK % SFC_EF10_RX_WPTR_ALIGN != 0); + SFC_ASSERT(RTE_ALIGN(rxq->added, SFC_EF10_RX_WPTR_ALIGN) == rxq->added); + + EFX_POPULATE_DWORD_1(dword, ERF_DZ_RX_DESC_WPTR, + rxq->added & rxq->ptr_mask); + + /* DMA sync to device is not required */ + + /* + * rte_write32() has rte_io_wmb() which guarantees that the STORE + * operations (i.e. Rx and event descriptor updates) that precede + * the rte_io_wmb() call are visible to NIC before the STORE + * operations that follow it (i.e. doorbell write). + */ + rte_write32(dword.ed_u32[0], rxq->doorbell); +} + +static void +sfc_ef10_rx_qrefill(struct sfc_ef10_rxq *rxq) +{ + const unsigned int ptr_mask = rxq->ptr_mask; + const uint32_t buf_size = rxq->buf_size; + unsigned int free_space; + unsigned int bulks; + void *objs[SFC_RX_REFILL_BULK]; + unsigned int added = rxq->added; + + free_space = SFC_EF10_RXQ_LIMIT(ptr_mask + 1) - + (added - rxq->completed); + + if (free_space < rxq->refill_threshold) + return; + + bulks = free_space / RTE_DIM(objs); + /* refill_threshold guarantees that bulks is positive */ + SFC_ASSERT(bulks > 0); + + do { + unsigned int id; + unsigned int i; + + if (unlikely(rte_mempool_get_bulk(rxq->refill_mb_pool, objs, + RTE_DIM(objs)) < 0)) { + struct rte_eth_dev_data *dev_data = + rte_eth_devices[rxq->dp.dpq.port_id].data; + + /* + * It is hardly a safe way to increment counter + * from different contexts, but all PMDs do it. + */ + dev_data->rx_mbuf_alloc_failed += RTE_DIM(objs); + /* Return if we have posted nothing yet */ + if (added == rxq->added) + return; + /* Push posted */ + break; + } + + for (i = 0, id = added & ptr_mask; + i < RTE_DIM(objs); + ++i, ++id) { + struct rte_mbuf *m = objs[i]; + struct sfc_ef10_rx_sw_desc *rxd; + rte_iova_t phys_addr; + + SFC_ASSERT((id & ~ptr_mask) == 0); + rxd = &rxq->sw_ring[id]; + rxd->mbuf = m; + + /* + * Avoid writing to mbuf. It is cheaper to do it + * when we receive packet and fill in nearby + * structure members. + */ + + phys_addr = rte_mbuf_data_iova_default(m); + EFX_POPULATE_QWORD_2(rxq->rxq_hw_ring[id], + ESF_DZ_RX_KER_BYTE_CNT, buf_size, + ESF_DZ_RX_KER_BUF_ADDR, phys_addr); + } + + added += RTE_DIM(objs); + } while (--bulks > 0); + + SFC_ASSERT(rxq->added != added); + rxq->added = added; + sfc_ef10_rx_qpush(rxq); +} + +static void +sfc_ef10_rx_prefetch_next(struct sfc_ef10_rxq *rxq, unsigned int next_id) +{ + struct rte_mbuf *next_mbuf; + + /* Prefetch next bunch of software descriptors */ + if ((next_id % (RTE_CACHE_LINE_SIZE / sizeof(rxq->sw_ring[0]))) == 0) + rte_prefetch0(&rxq->sw_ring[next_id]); + + /* + * It looks strange to prefetch depending on previous prefetch + * data, but measurements show that it is really efficient and + * increases packet rate. + */ + next_mbuf = rxq->sw_ring[next_id].mbuf; + if (likely(next_mbuf != NULL)) { + /* Prefetch the next mbuf structure */ + rte_mbuf_prefetch_part1(next_mbuf); + + /* Prefetch pseudo header of the next packet */ + /* data_off is not filled in yet */ + /* Yes, data could be not ready yet, but we hope */ + rte_prefetch0((uint8_t *)next_mbuf->buf_addr + + RTE_PKTMBUF_HEADROOM); + } +} + +static uint16_t +sfc_ef10_rx_prepared(struct sfc_ef10_rxq *rxq, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts) +{ + uint16_t n_rx_pkts = RTE_MIN(nb_pkts, rxq->prepared); + unsigned int completed = rxq->completed; + unsigned int i; + + rxq->prepared -= n_rx_pkts; + rxq->completed = completed + n_rx_pkts; + + for (i = 0; i < n_rx_pkts; ++i, ++completed) + rx_pkts[i] = rxq->sw_ring[completed & rxq->ptr_mask].mbuf; + + return n_rx_pkts; +} + +static void +sfc_ef10_rx_ev_to_offloads(struct sfc_ef10_rxq *rxq, const efx_qword_t rx_ev, + struct rte_mbuf *m) +{ + uint32_t l2_ptype = 0; + uint32_t l3_ptype = 0; + uint32_t l4_ptype = 0; + uint64_t ol_flags = 0; + + if (unlikely(EFX_TEST_QWORD_BIT(rx_ev, ESF_DZ_RX_PARSE_INCOMPLETE_LBN))) + goto done; + + switch (EFX_QWORD_FIELD(rx_ev, ESF_DZ_RX_ETH_TAG_CLASS)) { + case ESE_DZ_ETH_TAG_CLASS_NONE: + l2_ptype = RTE_PTYPE_L2_ETHER; + break; + case ESE_DZ_ETH_TAG_CLASS_VLAN1: + l2_ptype = RTE_PTYPE_L2_ETHER_VLAN; + break; + case ESE_DZ_ETH_TAG_CLASS_VLAN2: + l2_ptype = RTE_PTYPE_L2_ETHER_QINQ; + break; + default: + /* Unexpected Eth tag class */ + SFC_ASSERT(false); + } + + switch (EFX_QWORD_FIELD(rx_ev, ESF_DZ_RX_L3_CLASS)) { + case ESE_DZ_L3_CLASS_IP4_FRAG: + l4_ptype = RTE_PTYPE_L4_FRAG; + /* FALLTHROUGH */ + case ESE_DZ_L3_CLASS_IP4: + l3_ptype = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN; + ol_flags |= PKT_RX_RSS_HASH | + ((EFX_TEST_QWORD_BIT(rx_ev, + ESF_DZ_RX_IPCKSUM_ERR_LBN)) ? + PKT_RX_IP_CKSUM_BAD : PKT_RX_IP_CKSUM_GOOD); + break; + case ESE_DZ_L3_CLASS_IP6_FRAG: + l4_ptype = RTE_PTYPE_L4_FRAG; + /* FALLTHROUGH */ + case ESE_DZ_L3_CLASS_IP6: + l3_ptype = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN; + ol_flags |= PKT_RX_RSS_HASH; + break; + case ESE_DZ_L3_CLASS_ARP: + /* Override Layer 2 packet type */ + l2_ptype = RTE_PTYPE_L2_ETHER_ARP; + break; + default: + /* Unexpected Layer 3 class */ + SFC_ASSERT(false); + } + + switch (EFX_QWORD_FIELD(rx_ev, ESF_DZ_RX_L4_CLASS)) { + case ESE_DZ_L4_CLASS_TCP: + l4_ptype = RTE_PTYPE_L4_TCP; + ol_flags |= + (EFX_TEST_QWORD_BIT(rx_ev, + ESF_DZ_RX_TCPUDP_CKSUM_ERR_LBN)) ? + PKT_RX_L4_CKSUM_BAD : PKT_RX_L4_CKSUM_GOOD; + break; + case ESE_DZ_L4_CLASS_UDP: + l4_ptype = RTE_PTYPE_L4_UDP; + ol_flags |= + (EFX_TEST_QWORD_BIT(rx_ev, + ESF_DZ_RX_TCPUDP_CKSUM_ERR_LBN)) ? + PKT_RX_L4_CKSUM_BAD : PKT_RX_L4_CKSUM_GOOD; + break; + case ESE_DZ_L4_CLASS_UNKNOWN: + break; + default: + /* Unexpected Layer 4 class */ + SFC_ASSERT(false); + } + + /* Remove RSS hash offload flag if RSS is not enabled */ + if (~rxq->flags & SFC_EF10_RXQ_RSS_HASH) + ol_flags &= ~PKT_RX_RSS_HASH; + +done: + m->ol_flags = ol_flags; + m->packet_type = l2_ptype | l3_ptype | l4_ptype; +} + +static uint16_t +sfc_ef10_rx_pseudo_hdr_get_len(const uint8_t *pseudo_hdr) +{ + return rte_le_to_cpu_16(*(const uint16_t *)&pseudo_hdr[8]); +} + +static uint32_t +sfc_ef10_rx_pseudo_hdr_get_hash(const uint8_t *pseudo_hdr) +{ + return rte_le_to_cpu_32(*(const uint32_t *)pseudo_hdr); +} + +static uint16_t +sfc_ef10_rx_process_event(struct sfc_ef10_rxq *rxq, efx_qword_t rx_ev, + struct rte_mbuf **rx_pkts, uint16_t nb_pkts) +{ + const unsigned int ptr_mask = rxq->ptr_mask; + unsigned int completed = rxq->completed; + unsigned int ready; + struct sfc_ef10_rx_sw_desc *rxd; + struct rte_mbuf *m; + struct rte_mbuf *m0; + uint16_t n_rx_pkts; + const uint8_t *pseudo_hdr; + uint16_t pkt_len; + + ready = (EFX_QWORD_FIELD(rx_ev, ESF_DZ_RX_DSC_PTR_LBITS) - completed) & + EFX_MASK32(ESF_DZ_RX_DSC_PTR_LBITS); + SFC_ASSERT(ready > 0); + + if (rx_ev.eq_u64[0] & + rte_cpu_to_le_64((1ull << ESF_DZ_RX_ECC_ERR_LBN) | + (1ull << ESF_DZ_RX_ECRC_ERR_LBN))) { + SFC_ASSERT(rxq->prepared == 0); + rxq->completed += ready; + while (ready-- > 0) { + rxd = &rxq->sw_ring[completed++ & ptr_mask]; + rte_mempool_put(rxq->refill_mb_pool, rxd->mbuf); + } + return 0; + } + + n_rx_pkts = RTE_MIN(ready, nb_pkts); + rxq->prepared = ready - n_rx_pkts; + rxq->completed += n_rx_pkts; + + rxd = &rxq->sw_ring[completed++ & ptr_mask]; + + sfc_ef10_rx_prefetch_next(rxq, completed & ptr_mask); + + m = rxd->mbuf; + + *rx_pkts++ = m; + + RTE_BUILD_BUG_ON(sizeof(m->rearm_data[0]) != sizeof(rxq->rearm_data)); + m->rearm_data[0] = rxq->rearm_data; + + /* Classify packet based on Rx event */ + sfc_ef10_rx_ev_to_offloads(rxq, rx_ev, m); + + /* data_off already moved past pseudo header */ + pseudo_hdr = (uint8_t *)m->buf_addr + RTE_PKTMBUF_HEADROOM; + + /* + * Always get RSS hash from pseudo header to avoid + * condition/branching. If it is valid or not depends on + * PKT_RX_RSS_HASH in m->ol_flags. + */ + m->hash.rss = sfc_ef10_rx_pseudo_hdr_get_hash(pseudo_hdr); + + if (ready == 1) + pkt_len = EFX_QWORD_FIELD(rx_ev, ESF_DZ_RX_BYTES) - + rxq->prefix_size; + else + pkt_len = sfc_ef10_rx_pseudo_hdr_get_len(pseudo_hdr); + SFC_ASSERT(pkt_len > 0); + rte_pktmbuf_data_len(m) = pkt_len; + rte_pktmbuf_pkt_len(m) = pkt_len; + + SFC_ASSERT(m->next == NULL); + + /* Remember mbuf to copy offload flags and packet type from */ + m0 = m; + for (--ready; ready > 0; --ready) { + rxd = &rxq->sw_ring[completed++ & ptr_mask]; + + sfc_ef10_rx_prefetch_next(rxq, completed & ptr_mask); + + m = rxd->mbuf; + + if (ready > rxq->prepared) + *rx_pkts++ = m; + + RTE_BUILD_BUG_ON(sizeof(m->rearm_data[0]) != + sizeof(rxq->rearm_data)); + m->rearm_data[0] = rxq->rearm_data; + + /* Event-dependent information is the same */ + m->ol_flags = m0->ol_flags; + m->packet_type = m0->packet_type; + + /* data_off already moved past pseudo header */ + pseudo_hdr = (uint8_t *)m->buf_addr + RTE_PKTMBUF_HEADROOM; + + /* + * Always get RSS hash from pseudo header to avoid + * condition/branching. If it is valid or not depends on + * PKT_RX_RSS_HASH in m->ol_flags. + */ + m->hash.rss = sfc_ef10_rx_pseudo_hdr_get_hash(pseudo_hdr); + + pkt_len = sfc_ef10_rx_pseudo_hdr_get_len(pseudo_hdr); + SFC_ASSERT(pkt_len > 0); + rte_pktmbuf_data_len(m) = pkt_len; + rte_pktmbuf_pkt_len(m) = pkt_len; + + SFC_ASSERT(m->next == NULL); + } + + return n_rx_pkts; +} + +static bool +sfc_ef10_rx_get_event(struct sfc_ef10_rxq *rxq, efx_qword_t *rx_ev) +{ + *rx_ev = rxq->evq_hw_ring[rxq->evq_read_ptr & rxq->ptr_mask]; + + if (!sfc_ef10_ev_present(*rx_ev)) + return false; + + if (unlikely(EFX_QWORD_FIELD(*rx_ev, FSF_AZ_EV_CODE) != + FSE_AZ_EV_CODE_RX_EV)) { + /* + * Do not move read_ptr to keep the event for exception + * handling by the control path. + */ + rxq->flags |= SFC_EF10_RXQ_EXCEPTION; + sfc_ef10_rx_err(&rxq->dp.dpq, + "RxQ exception at EvQ read ptr %#x", + rxq->evq_read_ptr); + return false; + } + + rxq->evq_read_ptr++; + return true; +} + +static uint16_t +sfc_ef10_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) +{ + struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(rx_queue); + unsigned int evq_old_read_ptr; + uint16_t n_rx_pkts; + efx_qword_t rx_ev; + + if (unlikely(rxq->flags & + (SFC_EF10_RXQ_NOT_RUNNING | SFC_EF10_RXQ_EXCEPTION))) + return 0; + + n_rx_pkts = sfc_ef10_rx_prepared(rxq, rx_pkts, nb_pkts); + + evq_old_read_ptr = rxq->evq_read_ptr; + while (n_rx_pkts != nb_pkts && sfc_ef10_rx_get_event(rxq, &rx_ev)) { + /* + * DROP_EVENT is an internal to the NIC, software should + * never see it and, therefore, may ignore it. + */ + + n_rx_pkts += sfc_ef10_rx_process_event(rxq, rx_ev, + rx_pkts + n_rx_pkts, + nb_pkts - n_rx_pkts); + } + + sfc_ef10_ev_qclear(rxq->evq_hw_ring, rxq->ptr_mask, evq_old_read_ptr, + rxq->evq_read_ptr); + + /* It is not a problem if we refill in the case of exception */ + sfc_ef10_rx_qrefill(rxq); + + return n_rx_pkts; +} + +static const uint32_t * +sfc_ef10_supported_ptypes_get(void) +{ + static const uint32_t ef10_native_ptypes[] = { + RTE_PTYPE_L2_ETHER, + RTE_PTYPE_L2_ETHER_ARP, + RTE_PTYPE_L2_ETHER_VLAN, + RTE_PTYPE_L2_ETHER_QINQ, + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN, + RTE_PTYPE_L3_IPV6_EXT_UNKNOWN, + RTE_PTYPE_L4_FRAG, + RTE_PTYPE_L4_TCP, + RTE_PTYPE_L4_UDP, + RTE_PTYPE_UNKNOWN + }; + + return ef10_native_ptypes; +} + +static sfc_dp_rx_qdesc_npending_t sfc_ef10_rx_qdesc_npending; +static unsigned int +sfc_ef10_rx_qdesc_npending(__rte_unused struct sfc_dp_rxq *dp_rxq) +{ + /* + * Correct implementation requires EvQ polling and events + * processing (keeping all ready mbufs in prepared). + */ + return -ENOTSUP; +} + +static sfc_dp_rx_qdesc_status_t sfc_ef10_rx_qdesc_status; +static int +sfc_ef10_rx_qdesc_status(__rte_unused struct sfc_dp_rxq *dp_rxq, + __rte_unused uint16_t offset) +{ + return -ENOTSUP; +} + + +static uint64_t +sfc_ef10_mk_mbuf_rearm_data(uint16_t port_id, uint16_t prefix_size) +{ + struct rte_mbuf m; + + memset(&m, 0, sizeof(m)); + + rte_mbuf_refcnt_set(&m, 1); + m.data_off = RTE_PKTMBUF_HEADROOM + prefix_size; + m.nb_segs = 1; + m.port = port_id; + + /* rearm_data covers structure members filled in above */ + rte_compiler_barrier(); + RTE_BUILD_BUG_ON(sizeof(m.rearm_data[0]) != sizeof(uint64_t)); + return m.rearm_data[0]; +} + +static sfc_dp_rx_qcreate_t sfc_ef10_rx_qcreate; +static int +sfc_ef10_rx_qcreate(uint16_t port_id, uint16_t queue_id, + const struct rte_pci_addr *pci_addr, int socket_id, + const struct sfc_dp_rx_qcreate_info *info, + struct sfc_dp_rxq **dp_rxqp) +{ + struct sfc_ef10_rxq *rxq; + int rc; + + rc = EINVAL; + if (info->rxq_entries != info->evq_entries) + goto fail_rxq_args; + + rc = ENOMEM; + rxq = rte_zmalloc_socket("sfc-ef10-rxq", sizeof(*rxq), + RTE_CACHE_LINE_SIZE, socket_id); + if (rxq == NULL) + goto fail_rxq_alloc; + + sfc_dp_queue_init(&rxq->dp.dpq, port_id, queue_id, pci_addr); + + rc = ENOMEM; + rxq->sw_ring = rte_calloc_socket("sfc-ef10-rxq-sw_ring", + info->rxq_entries, + sizeof(*rxq->sw_ring), + RTE_CACHE_LINE_SIZE, socket_id); + if (rxq->sw_ring == NULL) + goto fail_desc_alloc; + + rxq->flags |= SFC_EF10_RXQ_NOT_RUNNING; + if (info->flags & SFC_RXQ_FLAG_RSS_HASH) + rxq->flags |= SFC_EF10_RXQ_RSS_HASH; + rxq->ptr_mask = info->rxq_entries - 1; + rxq->evq_hw_ring = info->evq_hw_ring; + rxq->refill_threshold = info->refill_threshold; + rxq->rearm_data = + sfc_ef10_mk_mbuf_rearm_data(port_id, info->prefix_size); + rxq->prefix_size = info->prefix_size; + rxq->buf_size = info->buf_size; + rxq->refill_mb_pool = info->refill_mb_pool; + rxq->rxq_hw_ring = info->rxq_hw_ring; + rxq->doorbell = (volatile uint8_t *)info->mem_bar + + ER_DZ_RX_DESC_UPD_REG_OFST + + info->hw_index * ER_DZ_RX_DESC_UPD_REG_STEP; + + *dp_rxqp = &rxq->dp; + return 0; + +fail_desc_alloc: + rte_free(rxq); + +fail_rxq_alloc: +fail_rxq_args: + return rc; +} + +static sfc_dp_rx_qdestroy_t sfc_ef10_rx_qdestroy; +static void +sfc_ef10_rx_qdestroy(struct sfc_dp_rxq *dp_rxq) +{ + struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq); + + rte_free(rxq->sw_ring); + rte_free(rxq); +} + +static sfc_dp_rx_qstart_t sfc_ef10_rx_qstart; +static int +sfc_ef10_rx_qstart(struct sfc_dp_rxq *dp_rxq, unsigned int evq_read_ptr) +{ + struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq); + + rxq->prepared = 0; + rxq->completed = rxq->added = 0; + + sfc_ef10_rx_qrefill(rxq); + + rxq->evq_read_ptr = evq_read_ptr; + + rxq->flags |= SFC_EF10_RXQ_STARTED; + rxq->flags &= ~(SFC_EF10_RXQ_NOT_RUNNING | SFC_EF10_RXQ_EXCEPTION); + + return 0; +} + +static sfc_dp_rx_qstop_t sfc_ef10_rx_qstop; +static void +sfc_ef10_rx_qstop(struct sfc_dp_rxq *dp_rxq, unsigned int *evq_read_ptr) +{ + struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq); + + rxq->flags |= SFC_EF10_RXQ_NOT_RUNNING; + + *evq_read_ptr = rxq->evq_read_ptr; +} + +static sfc_dp_rx_qrx_ev_t sfc_ef10_rx_qrx_ev; +static bool +sfc_ef10_rx_qrx_ev(struct sfc_dp_rxq *dp_rxq, __rte_unused unsigned int id) +{ + __rte_unused struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq); + + SFC_ASSERT(rxq->flags & SFC_EF10_RXQ_NOT_RUNNING); + + /* + * It is safe to ignore Rx event since we free all mbufs on + * queue purge anyway. + */ + + return false; +} + +static sfc_dp_rx_qpurge_t sfc_ef10_rx_qpurge; +static void +sfc_ef10_rx_qpurge(struct sfc_dp_rxq *dp_rxq) +{ + struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq); + unsigned int i; + struct sfc_ef10_rx_sw_desc *rxd; + + for (i = rxq->completed; i != rxq->added; ++i) { + rxd = &rxq->sw_ring[i & rxq->ptr_mask]; + rte_mempool_put(rxq->refill_mb_pool, rxd->mbuf); + rxd->mbuf = NULL; + } + + rxq->flags &= ~SFC_EF10_RXQ_STARTED; +} + +struct sfc_dp_rx sfc_ef10_rx = { + .dp = { + .name = SFC_KVARG_DATAPATH_EF10, + .type = SFC_DP_RX, + .hw_fw_caps = SFC_DP_HW_FW_CAP_EF10, + }, + .features = SFC_DP_RX_FEAT_MULTI_PROCESS, + .qcreate = sfc_ef10_rx_qcreate, + .qdestroy = sfc_ef10_rx_qdestroy, + .qstart = sfc_ef10_rx_qstart, + .qstop = sfc_ef10_rx_qstop, + .qrx_ev = sfc_ef10_rx_qrx_ev, + .qpurge = sfc_ef10_rx_qpurge, + .supported_ptypes_get = sfc_ef10_supported_ptypes_get, + .qdesc_npending = sfc_ef10_rx_qdesc_npending, + .qdesc_status = sfc_ef10_rx_qdesc_status, + .pkt_burst = sfc_ef10_recv_pkts, +}; diff --git a/dpdk/drivers/net/sfc/sfc_ef10_tx.c b/dpdk/drivers/net/sfc/sfc_ef10_tx.c new file mode 100644 index 00000000..0454e79a --- /dev/null +++ b/dpdk/drivers/net/sfc/sfc_ef10_tx.c @@ -0,0 +1,655 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2016 Solarflare Communications Inc. + * All rights reserved. + * + * This software was jointly developed between OKTET Labs (under contract + * for Solarflare) and Solarflare Communications, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +#include +#include + +#include "efx.h" +#include "efx_types.h" +#include "efx_regs.h" +#include "efx_regs_ef10.h" + +#include "sfc_dp_tx.h" +#include "sfc_tweak.h" +#include "sfc_kvargs.h" +#include "sfc_ef10.h" + +#define sfc_ef10_tx_err(dpq, ...) \ + SFC_DP_LOG(SFC_KVARG_DATAPATH_EF10, ERR, dpq, __VA_ARGS__) + +/** Maximum length of the DMA descriptor data */ +#define SFC_EF10_TX_DMA_DESC_LEN_MAX \ + ((1u << ESF_DZ_TX_KER_BYTE_CNT_WIDTH) - 1) + +/** + * Maximum number of descriptors/buffers in the Tx ring. + * It should guarantee that corresponding event queue never overfill. + * EF10 native datapath uses event queue of the same size as Tx queue. + * Maximum number of events on datapath can be estimated as number of + * Tx queue entries (one event per Tx buffer in the worst case) plus + * Tx error and flush events. + */ +#define SFC_EF10_TXQ_LIMIT(_ndesc) \ + ((_ndesc) - 1 /* head must not step on tail */ - \ + (SFC_EF10_EV_PER_CACHE_LINE - 1) /* max unused EvQ entries */ - \ + 1 /* Rx error */ - 1 /* flush */) + +struct sfc_ef10_tx_sw_desc { + struct rte_mbuf *mbuf; +}; + +struct sfc_ef10_txq { + unsigned int flags; +#define SFC_EF10_TXQ_STARTED 0x1 +#define SFC_EF10_TXQ_NOT_RUNNING 0x2 +#define SFC_EF10_TXQ_EXCEPTION 0x4 + + unsigned int ptr_mask; + unsigned int added; + unsigned int completed; + unsigned int free_thresh; + unsigned int evq_read_ptr; + struct sfc_ef10_tx_sw_desc *sw_ring; + efx_qword_t *txq_hw_ring; + volatile void *doorbell; + efx_qword_t *evq_hw_ring; + + /* Datapath transmit queue anchor */ + struct sfc_dp_txq dp; +}; + +static inline struct sfc_ef10_txq * +sfc_ef10_txq_by_dp_txq(struct sfc_dp_txq *dp_txq) +{ + return container_of(dp_txq, struct sfc_ef10_txq, dp); +} + +static bool +sfc_ef10_tx_get_event(struct sfc_ef10_txq *txq, efx_qword_t *tx_ev) +{ + volatile efx_qword_t *evq_hw_ring = txq->evq_hw_ring; + + /* + * Exception flag is set when reap is done. + * It is never done twice per packet burst get and absence of + * the flag is checked on burst get entry. + */ + SFC_ASSERT((txq->flags & SFC_EF10_TXQ_EXCEPTION) == 0); + + *tx_ev = evq_hw_ring[txq->evq_read_ptr & txq->ptr_mask]; + + if (!sfc_ef10_ev_present(*tx_ev)) + return false; + + if (unlikely(EFX_QWORD_FIELD(*tx_ev, FSF_AZ_EV_CODE) != + FSE_AZ_EV_CODE_TX_EV)) { + /* + * Do not move read_ptr to keep the event for exception + * handling by the control path. + */ + txq->flags |= SFC_EF10_TXQ_EXCEPTION; + sfc_ef10_tx_err(&txq->dp.dpq, + "TxQ exception at EvQ read ptr %#x", + txq->evq_read_ptr); + return false; + } + + txq->evq_read_ptr++; + return true; +} + +static unsigned int +sfc_ef10_tx_process_events(struct sfc_ef10_txq *txq) +{ + const unsigned int curr_done = txq->completed - 1; + unsigned int anew_done = curr_done; + efx_qword_t tx_ev; + + while (sfc_ef10_tx_get_event(txq, &tx_ev)) { + /* + * DROP_EVENT is an internal to the NIC, software should + * never see it and, therefore, may ignore it. + */ + + /* Update the latest done descriptor */ + anew_done = EFX_QWORD_FIELD(tx_ev, ESF_DZ_TX_DESCR_INDX); + } + return (anew_done - curr_done) & txq->ptr_mask; +} + +static void +sfc_ef10_tx_reap(struct sfc_ef10_txq *txq) +{ + const unsigned int old_read_ptr = txq->evq_read_ptr; + const unsigned int ptr_mask = txq->ptr_mask; + unsigned int completed = txq->completed; + unsigned int pending = completed; + + pending += sfc_ef10_tx_process_events(txq); + + if (pending != completed) { + struct rte_mbuf *bulk[SFC_TX_REAP_BULK_SIZE]; + unsigned int nb = 0; + + do { + struct sfc_ef10_tx_sw_desc *txd; + struct rte_mbuf *m; + + txd = &txq->sw_ring[completed & ptr_mask]; + if (txd->mbuf == NULL) + continue; + + m = rte_pktmbuf_prefree_seg(txd->mbuf); + txd->mbuf = NULL; + if (m == NULL) + continue; + + if ((nb == RTE_DIM(bulk)) || + ((nb != 0) && (m->pool != bulk[0]->pool))) { + rte_mempool_put_bulk(bulk[0]->pool, + (void *)bulk, nb); + nb = 0; + } + + bulk[nb++] = m; + } while (++completed != pending); + + if (nb != 0) + rte_mempool_put_bulk(bulk[0]->pool, (void *)bulk, nb); + + txq->completed = completed; + } + + sfc_ef10_ev_qclear(txq->evq_hw_ring, ptr_mask, old_read_ptr, + txq->evq_read_ptr); +} + +static void +sfc_ef10_tx_qdesc_dma_create(rte_iova_t addr, uint16_t size, bool eop, + efx_qword_t *edp) +{ + EFX_POPULATE_QWORD_4(*edp, + ESF_DZ_TX_KER_TYPE, 0, + ESF_DZ_TX_KER_CONT, !eop, + ESF_DZ_TX_KER_BYTE_CNT, size, + ESF_DZ_TX_KER_BUF_ADDR, addr); +} + +static inline void +sfc_ef10_tx_qpush(struct sfc_ef10_txq *txq, unsigned int added, + unsigned int pushed) +{ + efx_qword_t desc; + efx_oword_t oword; + + /* + * This improves performance by pushing a TX descriptor at the same + * time as the doorbell. The descriptor must be added to the TXQ, + * so that can be used if the hardware decides not to use the pushed + * descriptor. + */ + desc.eq_u64[0] = txq->txq_hw_ring[pushed & txq->ptr_mask].eq_u64[0]; + EFX_POPULATE_OWORD_3(oword, + ERF_DZ_TX_DESC_WPTR, added & txq->ptr_mask, + ERF_DZ_TX_DESC_HWORD, EFX_QWORD_FIELD(desc, EFX_DWORD_1), + ERF_DZ_TX_DESC_LWORD, EFX_QWORD_FIELD(desc, EFX_DWORD_0)); + + /* DMA sync to device is not required */ + + /* + * rte_io_wmb() which guarantees that the STORE operations + * (i.e. Tx and event descriptor updates) that precede + * the rte_io_wmb() call are visible to NIC before the STORE + * operations that follow it (i.e. doorbell write). + */ + rte_io_wmb(); + + *(volatile __m128i *)txq->doorbell = oword.eo_u128[0]; +} + +static unsigned int +sfc_ef10_tx_pkt_descs_max(const struct rte_mbuf *m) +{ + unsigned int extra_descs_per_seg; + unsigned int extra_descs_per_pkt; + + /* + * VLAN offload is not supported yet, so no extra descriptors + * are required for VLAN option descriptor. + */ + +/** Maximum length of the mbuf segment data */ +#define SFC_MBUF_SEG_LEN_MAX UINT16_MAX + RTE_BUILD_BUG_ON(sizeof(m->data_len) != 2); + + /* + * Each segment is already counted once below. So, calculate + * how many extra DMA descriptors may be required per segment in + * the worst case because of maximum DMA descriptor length limit. + * If maximum segment length is less or equal to maximum DMA + * descriptor length, no extra DMA descriptors are required. + */ + extra_descs_per_seg = + (SFC_MBUF_SEG_LEN_MAX - 1) / SFC_EF10_TX_DMA_DESC_LEN_MAX; + +/** Maximum length of the packet */ +#define SFC_MBUF_PKT_LEN_MAX UINT32_MAX + RTE_BUILD_BUG_ON(sizeof(m->pkt_len) != 4); + + /* + * One more limitation on maximum number of extra DMA descriptors + * comes from slicing entire packet because of DMA descriptor length + * limit taking into account that there is at least one segment + * which is already counted below (so division of the maximum + * packet length minus one with round down). + * TSO is not supported yet, so packet length is limited by + * maximum PDU size. + */ + extra_descs_per_pkt = + (RTE_MIN((unsigned int)EFX_MAC_PDU_MAX, + SFC_MBUF_PKT_LEN_MAX) - 1) / + SFC_EF10_TX_DMA_DESC_LEN_MAX; + + return m->nb_segs + RTE_MIN(m->nb_segs * extra_descs_per_seg, + extra_descs_per_pkt); +} + +static uint16_t +sfc_ef10_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) +{ + struct sfc_ef10_txq * const txq = sfc_ef10_txq_by_dp_txq(tx_queue); + unsigned int ptr_mask; + unsigned int added; + unsigned int dma_desc_space; + bool reap_done; + struct rte_mbuf **pktp; + struct rte_mbuf **pktp_end; + + if (unlikely(txq->flags & + (SFC_EF10_TXQ_NOT_RUNNING | SFC_EF10_TXQ_EXCEPTION))) + return 0; + + ptr_mask = txq->ptr_mask; + added = txq->added; + dma_desc_space = SFC_EF10_TXQ_LIMIT(ptr_mask + 1) - + (added - txq->completed); + + reap_done = (dma_desc_space < txq->free_thresh); + if (reap_done) { + sfc_ef10_tx_reap(txq); + dma_desc_space = SFC_EF10_TXQ_LIMIT(ptr_mask + 1) - + (added - txq->completed); + } + + for (pktp = &tx_pkts[0], pktp_end = &tx_pkts[nb_pkts]; + pktp != pktp_end; + ++pktp) { + struct rte_mbuf *m_seg = *pktp; + unsigned int pkt_start = added; + uint32_t pkt_len; + + if (likely(pktp + 1 != pktp_end)) + rte_mbuf_prefetch_part1(pktp[1]); + + if (sfc_ef10_tx_pkt_descs_max(m_seg) > dma_desc_space) { + if (reap_done) + break; + + /* Push already prepared descriptors before polling */ + if (added != txq->added) { + sfc_ef10_tx_qpush(txq, added, txq->added); + txq->added = added; + } + + sfc_ef10_tx_reap(txq); + reap_done = true; + dma_desc_space = SFC_EF10_TXQ_LIMIT(ptr_mask + 1) - + (added - txq->completed); + if (sfc_ef10_tx_pkt_descs_max(m_seg) > dma_desc_space) + break; + } + + pkt_len = m_seg->pkt_len; + do { + rte_iova_t seg_addr = rte_mbuf_data_iova(m_seg); + unsigned int seg_len = rte_pktmbuf_data_len(m_seg); + unsigned int id = added & ptr_mask; + + SFC_ASSERT(seg_len <= SFC_EF10_TX_DMA_DESC_LEN_MAX); + + pkt_len -= seg_len; + + sfc_ef10_tx_qdesc_dma_create(seg_addr, + seg_len, (pkt_len == 0), + &txq->txq_hw_ring[id]); + + /* + * rte_pktmbuf_free() is commonly used in DPDK for + * recycling packets - the function checks every + * segment's reference counter and returns the + * buffer to its pool whenever possible; + * nevertheless, freeing mbuf segments one by one + * may entail some performance decline; + * from this point, sfc_efx_tx_reap() does the same job + * on its own and frees buffers in bulks (all mbufs + * within a bulk belong to the same pool); + * from this perspective, individual segment pointers + * must be associated with the corresponding SW + * descriptors independently so that only one loop + * is sufficient on reap to inspect all the buffers + */ + txq->sw_ring[id].mbuf = m_seg; + + ++added; + + } while ((m_seg = m_seg->next) != 0); + + dma_desc_space -= (added - pkt_start); + } + + if (likely(added != txq->added)) { + sfc_ef10_tx_qpush(txq, added, txq->added); + txq->added = added; + } + +#if SFC_TX_XMIT_PKTS_REAP_AT_LEAST_ONCE + if (!reap_done) + sfc_ef10_tx_reap(txq); +#endif + + return pktp - &tx_pkts[0]; +} + +static void +sfc_ef10_simple_tx_reap(struct sfc_ef10_txq *txq) +{ + const unsigned int old_read_ptr = txq->evq_read_ptr; + const unsigned int ptr_mask = txq->ptr_mask; + unsigned int completed = txq->completed; + unsigned int pending = completed; + + pending += sfc_ef10_tx_process_events(txq); + + if (pending != completed) { + struct rte_mbuf *bulk[SFC_TX_REAP_BULK_SIZE]; + unsigned int nb = 0; + + do { + struct sfc_ef10_tx_sw_desc *txd; + + txd = &txq->sw_ring[completed & ptr_mask]; + + if (nb == RTE_DIM(bulk)) { + rte_mempool_put_bulk(bulk[0]->pool, + (void *)bulk, nb); + nb = 0; + } + + bulk[nb++] = txd->mbuf; + } while (++completed != pending); + + rte_mempool_put_bulk(bulk[0]->pool, (void *)bulk, nb); + + txq->completed = completed; + } + + sfc_ef10_ev_qclear(txq->evq_hw_ring, ptr_mask, old_read_ptr, + txq->evq_read_ptr); +} + + +static uint16_t +sfc_ef10_simple_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts) +{ + struct sfc_ef10_txq * const txq = sfc_ef10_txq_by_dp_txq(tx_queue); + unsigned int ptr_mask; + unsigned int added; + unsigned int dma_desc_space; + bool reap_done; + struct rte_mbuf **pktp; + struct rte_mbuf **pktp_end; + + if (unlikely(txq->flags & + (SFC_EF10_TXQ_NOT_RUNNING | SFC_EF10_TXQ_EXCEPTION))) + return 0; + + ptr_mask = txq->ptr_mask; + added = txq->added; + dma_desc_space = SFC_EF10_TXQ_LIMIT(ptr_mask + 1) - + (added - txq->completed); + + reap_done = (dma_desc_space < RTE_MAX(txq->free_thresh, nb_pkts)); + if (reap_done) { + sfc_ef10_simple_tx_reap(txq); + dma_desc_space = SFC_EF10_TXQ_LIMIT(ptr_mask + 1) - + (added - txq->completed); + } + + pktp_end = &tx_pkts[MIN(nb_pkts, dma_desc_space)]; + for (pktp = &tx_pkts[0]; pktp != pktp_end; ++pktp) { + struct rte_mbuf *pkt = *pktp; + unsigned int id = added & ptr_mask; + + SFC_ASSERT(rte_pktmbuf_data_len(pkt) <= + SFC_EF10_TX_DMA_DESC_LEN_MAX); + + sfc_ef10_tx_qdesc_dma_create(rte_mbuf_data_iova(pkt), + rte_pktmbuf_data_len(pkt), + true, &txq->txq_hw_ring[id]); + + txq->sw_ring[id].mbuf = pkt; + + ++added; + } + + if (likely(added != txq->added)) { + sfc_ef10_tx_qpush(txq, added, txq->added); + txq->added = added; + } + +#if SFC_TX_XMIT_PKTS_REAP_AT_LEAST_ONCE + if (!reap_done) + sfc_ef10_simple_tx_reap(txq); +#endif + + return pktp - &tx_pkts[0]; +} + + +static sfc_dp_tx_qcreate_t sfc_ef10_tx_qcreate; +static int +sfc_ef10_tx_qcreate(uint16_t port_id, uint16_t queue_id, + const struct rte_pci_addr *pci_addr, int socket_id, + const struct sfc_dp_tx_qcreate_info *info, + struct sfc_dp_txq **dp_txqp) +{ + struct sfc_ef10_txq *txq; + int rc; + + rc = EINVAL; + if (info->txq_entries != info->evq_entries) + goto fail_bad_args; + + rc = ENOMEM; + txq = rte_zmalloc_socket("sfc-ef10-txq", sizeof(*txq), + RTE_CACHE_LINE_SIZE, socket_id); + if (txq == NULL) + goto fail_txq_alloc; + + sfc_dp_queue_init(&txq->dp.dpq, port_id, queue_id, pci_addr); + + rc = ENOMEM; + txq->sw_ring = rte_calloc_socket("sfc-ef10-txq-sw_ring", + info->txq_entries, + sizeof(*txq->sw_ring), + RTE_CACHE_LINE_SIZE, socket_id); + if (txq->sw_ring == NULL) + goto fail_sw_ring_alloc; + + txq->flags = SFC_EF10_TXQ_NOT_RUNNING; + txq->ptr_mask = info->txq_entries - 1; + txq->free_thresh = info->free_thresh; + txq->txq_hw_ring = info->txq_hw_ring; + txq->doorbell = (volatile uint8_t *)info->mem_bar + + ER_DZ_TX_DESC_UPD_REG_OFST + + info->hw_index * ER_DZ_TX_DESC_UPD_REG_STEP; + txq->evq_hw_ring = info->evq_hw_ring; + + *dp_txqp = &txq->dp; + return 0; + +fail_sw_ring_alloc: + rte_free(txq); + +fail_txq_alloc: +fail_bad_args: + return rc; +} + +static sfc_dp_tx_qdestroy_t sfc_ef10_tx_qdestroy; +static void +sfc_ef10_tx_qdestroy(struct sfc_dp_txq *dp_txq) +{ + struct sfc_ef10_txq *txq = sfc_ef10_txq_by_dp_txq(dp_txq); + + rte_free(txq->sw_ring); + rte_free(txq); +} + +static sfc_dp_tx_qstart_t sfc_ef10_tx_qstart; +static int +sfc_ef10_tx_qstart(struct sfc_dp_txq *dp_txq, unsigned int evq_read_ptr, + unsigned int txq_desc_index) +{ + struct sfc_ef10_txq *txq = sfc_ef10_txq_by_dp_txq(dp_txq); + + txq->evq_read_ptr = evq_read_ptr; + txq->added = txq->completed = txq_desc_index; + + txq->flags |= SFC_EF10_TXQ_STARTED; + txq->flags &= ~(SFC_EF10_TXQ_NOT_RUNNING | SFC_EF10_TXQ_EXCEPTION); + + return 0; +} + +static sfc_dp_tx_qstop_t sfc_ef10_tx_qstop; +static void +sfc_ef10_tx_qstop(struct sfc_dp_txq *dp_txq, unsigned int *evq_read_ptr) +{ + struct sfc_ef10_txq *txq = sfc_ef10_txq_by_dp_txq(dp_txq); + + txq->flags |= SFC_EF10_TXQ_NOT_RUNNING; + + *evq_read_ptr = txq->evq_read_ptr; +} + +static sfc_dp_tx_qtx_ev_t sfc_ef10_tx_qtx_ev; +static bool +sfc_ef10_tx_qtx_ev(struct sfc_dp_txq *dp_txq, __rte_unused unsigned int id) +{ + __rte_unused struct sfc_ef10_txq *txq = sfc_ef10_txq_by_dp_txq(dp_txq); + + SFC_ASSERT(txq->flags & SFC_EF10_TXQ_NOT_RUNNING); + + /* + * It is safe to ignore Tx event since we reap all mbufs on + * queue purge anyway. + */ + + return false; +} + +static sfc_dp_tx_qreap_t sfc_ef10_tx_qreap; +static void +sfc_ef10_tx_qreap(struct sfc_dp_txq *dp_txq) +{ + struct sfc_ef10_txq *txq = sfc_ef10_txq_by_dp_txq(dp_txq); + unsigned int completed; + + for (completed = txq->completed; completed != txq->added; ++completed) { + struct sfc_ef10_tx_sw_desc *txd; + + txd = &txq->sw_ring[completed & txq->ptr_mask]; + if (txd->mbuf != NULL) { + rte_pktmbuf_free_seg(txd->mbuf); + txd->mbuf = NULL; + } + } + + txq->flags &= ~SFC_EF10_TXQ_STARTED; +} + +static sfc_dp_tx_qdesc_status_t sfc_ef10_tx_qdesc_status; +static int +sfc_ef10_tx_qdesc_status(__rte_unused struct sfc_dp_txq *dp_txq, + __rte_unused uint16_t offset) +{ + return -ENOTSUP; +} + +struct sfc_dp_tx sfc_ef10_tx = { + .dp = { + .name = SFC_KVARG_DATAPATH_EF10, + .type = SFC_DP_TX, + .hw_fw_caps = SFC_DP_HW_FW_CAP_EF10, + }, + .features = SFC_DP_TX_FEAT_MULTI_SEG | + SFC_DP_TX_FEAT_MULTI_POOL | + SFC_DP_TX_FEAT_REFCNT | + SFC_DP_TX_FEAT_MULTI_PROCESS, + .qcreate = sfc_ef10_tx_qcreate, + .qdestroy = sfc_ef10_tx_qdestroy, + .qstart = sfc_ef10_tx_qstart, + .qtx_ev = sfc_ef10_tx_qtx_ev, + .qstop = sfc_ef10_tx_qstop, + .qreap = sfc_ef10_tx_qreap, + .qdesc_status = sfc_ef10_tx_qdesc_status, + .pkt_burst = sfc_ef10_xmit_pkts, +}; + +struct sfc_dp_tx sfc_ef10_simple_tx = { + .dp = { + .name = SFC_KVARG_DATAPATH_EF10_SIMPLE, + .type = SFC_DP_TX, + }, + .features = SFC_DP_TX_FEAT_MULTI_PROCESS, + .qcreate = sfc_ef10_tx_qcreate, + .qdestroy = sfc_ef10_tx_qdestroy, + .qstart = sfc_ef10_tx_qstart, + .qtx_ev = sfc_ef10_tx_qtx_ev, + .qstop = sfc_ef10_tx_qstop, + .qreap = sfc_ef10_tx_qreap, + .qdesc_status = sfc_ef10_tx_qdesc_status, + .pkt_burst = sfc_ef10_simple_xmit_pkts, +}; diff --git a/dpdk/drivers/net/sfc/sfc_ethdev.c b/dpdk/drivers/net/sfc/sfc_ethdev.c new file mode 100644 index 00000000..fabcc329 --- /dev/null +++ b/dpdk/drivers/net/sfc/sfc_ethdev.c @@ -0,0 +1,1954 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2016-2017 Solarflare Communications Inc. + * All rights reserved. + * + * This software was jointly developed between OKTET Labs (under contract + * for Solarflare) and Solarflare Communications, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include + +#include "efx.h" + +#include "sfc.h" +#include "sfc_debug.h" +#include "sfc_log.h" +#include "sfc_kvargs.h" +#include "sfc_ev.h" +#include "sfc_rx.h" +#include "sfc_tx.h" +#include "sfc_flow.h" +#include "sfc_dp.h" +#include "sfc_dp_rx.h" + +static struct sfc_dp_list sfc_dp_head = + TAILQ_HEAD_INITIALIZER(sfc_dp_head); + +static int +sfc_fw_version_get(struct rte_eth_dev *dev, char *fw_version, size_t fw_size) +{ + struct sfc_adapter *sa = dev->data->dev_private; + efx_nic_fw_info_t enfi; + int ret; + int rc; + + /* + * Return value of the callback is likely supposed to be + * equal to or greater than 0, nevertheless, if an error + * occurs, it will be desirable to pass it to the caller + */ + if ((fw_version == NULL) || (fw_size == 0)) + return -EINVAL; + + rc = efx_nic_get_fw_version(sa->nic, &enfi); + if (rc != 0) + return -rc; + + ret = snprintf(fw_version, fw_size, + "%" PRIu16 ".%" PRIu16 ".%" PRIu16 ".%" PRIu16, + enfi.enfi_mc_fw_version[0], enfi.enfi_mc_fw_version[1], + enfi.enfi_mc_fw_version[2], enfi.enfi_mc_fw_version[3]); + if (ret < 0) + return ret; + + if (enfi.enfi_dpcpu_fw_ids_valid) { + size_t dpcpu_fw_ids_offset = MIN(fw_size - 1, (size_t)ret); + int ret_extra; + + ret_extra = snprintf(fw_version + dpcpu_fw_ids_offset, + fw_size - dpcpu_fw_ids_offset, + " rx%" PRIx16 " tx%" PRIx16, + enfi.enfi_rx_dpcpu_fw_id, + enfi.enfi_tx_dpcpu_fw_id); + if (ret_extra < 0) + return ret_extra; + + ret += ret_extra; + } + + if (fw_size < (size_t)(++ret)) + return ret; + else + return 0; +} + +static void +sfc_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) +{ + struct sfc_adapter *sa = dev->data->dev_private; + const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic); + + sfc_log_init(sa, "entry"); + + dev_info->pci_dev = RTE_ETH_DEV_TO_PCI(dev); + dev_info->max_rx_pktlen = EFX_MAC_PDU_MAX; + + /* Autonegotiation may be disabled */ + dev_info->speed_capa = ETH_LINK_SPEED_FIXED; + if (sa->port.phy_adv_cap_mask & EFX_PHY_CAP_1000FDX) + dev_info->speed_capa |= ETH_LINK_SPEED_1G; + if (sa->port.phy_adv_cap_mask & EFX_PHY_CAP_10000FDX) + dev_info->speed_capa |= ETH_LINK_SPEED_10G; + if (sa->port.phy_adv_cap_mask & EFX_PHY_CAP_40000FDX) + dev_info->speed_capa |= ETH_LINK_SPEED_40G; + + dev_info->max_rx_queues = sa->rxq_max; + dev_info->max_tx_queues = sa->txq_max; + + /* By default packets are dropped if no descriptors are available */ + dev_info->default_rxconf.rx_drop_en = 1; + + dev_info->rx_offload_capa = + DEV_RX_OFFLOAD_IPV4_CKSUM | + DEV_RX_OFFLOAD_UDP_CKSUM | + DEV_RX_OFFLOAD_TCP_CKSUM; + + dev_info->tx_offload_capa = + DEV_TX_OFFLOAD_IPV4_CKSUM | + DEV_TX_OFFLOAD_UDP_CKSUM | + DEV_TX_OFFLOAD_TCP_CKSUM; + + dev_info->default_txconf.txq_flags = ETH_TXQ_FLAGS_NOXSUMSCTP; + if ((~sa->dp_tx->features & SFC_DP_TX_FEAT_VLAN_INSERT) || + !encp->enc_hw_tx_insert_vlan_enabled) + dev_info->default_txconf.txq_flags |= ETH_TXQ_FLAGS_NOVLANOFFL; + else + dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_VLAN_INSERT; + + if (~sa->dp_tx->features & SFC_DP_TX_FEAT_MULTI_SEG) + dev_info->default_txconf.txq_flags |= ETH_TXQ_FLAGS_NOMULTSEGS; + + if (~sa->dp_tx->features & SFC_DP_TX_FEAT_MULTI_POOL) + dev_info->default_txconf.txq_flags |= ETH_TXQ_FLAGS_NOMULTMEMP; + + if (~sa->dp_tx->features & SFC_DP_TX_FEAT_REFCNT) + dev_info->default_txconf.txq_flags |= ETH_TXQ_FLAGS_NOREFCOUNT; + +#if EFSYS_OPT_RX_SCALE + if (sa->rss_support != EFX_RX_SCALE_UNAVAILABLE) { + dev_info->reta_size = EFX_RSS_TBL_SIZE; + dev_info->hash_key_size = EFX_RSS_KEY_SIZE; + dev_info->flow_type_rss_offloads = SFC_RSS_OFFLOADS; + } +#endif + + if (sa->tso) + dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_TCP_TSO; + + dev_info->rx_desc_lim.nb_max = EFX_RXQ_MAXNDESCS; + dev_info->rx_desc_lim.nb_min = EFX_RXQ_MINNDESCS; + /* The RXQ hardware requires that the descriptor count is a power + * of 2, but rx_desc_lim cannot properly describe that constraint. + */ + dev_info->rx_desc_lim.nb_align = EFX_RXQ_MINNDESCS; + + dev_info->tx_desc_lim.nb_max = sa->txq_max_entries; + dev_info->tx_desc_lim.nb_min = EFX_TXQ_MINNDESCS; + /* + * The TXQ hardware requires that the descriptor count is a power + * of 2, but tx_desc_lim cannot properly describe that constraint + */ + dev_info->tx_desc_lim.nb_align = EFX_TXQ_MINNDESCS; +} + +static const uint32_t * +sfc_dev_supported_ptypes_get(struct rte_eth_dev *dev) +{ + struct sfc_adapter *sa = dev->data->dev_private; + + return sa->dp_rx->supported_ptypes_get(); +} + +static int +sfc_dev_configure(struct rte_eth_dev *dev) +{ + struct rte_eth_dev_data *dev_data = dev->data; + struct sfc_adapter *sa = dev_data->dev_private; + int rc; + + sfc_log_init(sa, "entry n_rxq=%u n_txq=%u", + dev_data->nb_rx_queues, dev_data->nb_tx_queues); + + sfc_adapter_lock(sa); + switch (sa->state) { + case SFC_ADAPTER_CONFIGURED: + /* FALLTHROUGH */ + case SFC_ADAPTER_INITIALIZED: + rc = sfc_configure(sa); + break; + default: + sfc_err(sa, "unexpected adapter state %u to configure", + sa->state); + rc = EINVAL; + break; + } + sfc_adapter_unlock(sa); + + sfc_log_init(sa, "done %d", rc); + SFC_ASSERT(rc >= 0); + return -rc; +} + +static int +sfc_dev_start(struct rte_eth_dev *dev) +{ + struct sfc_adapter *sa = dev->data->dev_private; + int rc; + + sfc_log_init(sa, "entry"); + + sfc_adapter_lock(sa); + rc = sfc_start(sa); + sfc_adapter_unlock(sa); + + sfc_log_init(sa, "done %d", rc); + SFC_ASSERT(rc >= 0); + return -rc; +} + +static int +sfc_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete) +{ + struct sfc_adapter *sa = dev->data->dev_private; + struct rte_eth_link *dev_link = &dev->data->dev_link; + struct rte_eth_link old_link; + struct rte_eth_link current_link; + + sfc_log_init(sa, "entry"); + +retry: + EFX_STATIC_ASSERT(sizeof(*dev_link) == sizeof(rte_atomic64_t)); + *(int64_t *)&old_link = rte_atomic64_read((rte_atomic64_t *)dev_link); + + if (sa->state != SFC_ADAPTER_STARTED) { + sfc_port_link_mode_to_info(EFX_LINK_UNKNOWN, ¤t_link); + if (!rte_atomic64_cmpset((volatile uint64_t *)dev_link, + *(uint64_t *)&old_link, + *(uint64_t *)¤t_link)) + goto retry; + } else if (wait_to_complete) { + efx_link_mode_t link_mode; + + if (efx_port_poll(sa->nic, &link_mode) != 0) + link_mode = EFX_LINK_UNKNOWN; + sfc_port_link_mode_to_info(link_mode, ¤t_link); + + if (!rte_atomic64_cmpset((volatile uint64_t *)dev_link, + *(uint64_t *)&old_link, + *(uint64_t *)¤t_link)) + goto retry; + } else { + sfc_ev_mgmt_qpoll(sa); + *(int64_t *)¤t_link = + rte_atomic64_read((rte_atomic64_t *)dev_link); + } + + if (old_link.link_status != current_link.link_status) + sfc_info(sa, "Link status is %s", + current_link.link_status ? "UP" : "DOWN"); + + return old_link.link_status == current_link.link_status ? 0 : -1; +} + +static void +sfc_dev_stop(struct rte_eth_dev *dev) +{ + struct sfc_adapter *sa = dev->data->dev_private; + + sfc_log_init(sa, "entry"); + + sfc_adapter_lock(sa); + sfc_stop(sa); + sfc_adapter_unlock(sa); + + sfc_log_init(sa, "done"); +} + +static int +sfc_dev_set_link_up(struct rte_eth_dev *dev) +{ + struct sfc_adapter *sa = dev->data->dev_private; + int rc; + + sfc_log_init(sa, "entry"); + + sfc_adapter_lock(sa); + rc = sfc_start(sa); + sfc_adapter_unlock(sa); + + SFC_ASSERT(rc >= 0); + return -rc; +} + +static int +sfc_dev_set_link_down(struct rte_eth_dev *dev) +{ + struct sfc_adapter *sa = dev->data->dev_private; + + sfc_log_init(sa, "entry"); + + sfc_adapter_lock(sa); + sfc_stop(sa); + sfc_adapter_unlock(sa); + + return 0; +} + +static void +sfc_dev_close(struct rte_eth_dev *dev) +{ + struct sfc_adapter *sa = dev->data->dev_private; + + sfc_log_init(sa, "entry"); + + sfc_adapter_lock(sa); + switch (sa->state) { + case SFC_ADAPTER_STARTED: + sfc_stop(sa); + SFC_ASSERT(sa->state == SFC_ADAPTER_CONFIGURED); + /* FALLTHROUGH */ + case SFC_ADAPTER_CONFIGURED: + sfc_close(sa); + SFC_ASSERT(sa->state == SFC_ADAPTER_INITIALIZED); + /* FALLTHROUGH */ + case SFC_ADAPTER_INITIALIZED: + break; + default: + sfc_err(sa, "unexpected adapter state %u on close", sa->state); + break; + } + sfc_adapter_unlock(sa); + + sfc_log_init(sa, "done"); +} + +static void +sfc_dev_filter_set(struct rte_eth_dev *dev, enum sfc_dev_filter_mode mode, + boolean_t enabled) +{ + struct sfc_port *port; + boolean_t *toggle; + struct sfc_adapter *sa = dev->data->dev_private; + boolean_t allmulti = (mode == SFC_DEV_FILTER_MODE_ALLMULTI); + const char *desc = (allmulti) ? "all-multi" : "promiscuous"; + + sfc_adapter_lock(sa); + + port = &sa->port; + toggle = (allmulti) ? (&port->allmulti) : (&port->promisc); + + if (*toggle != enabled) { + *toggle = enabled; + + if (port->isolated) { + sfc_warn(sa, "isolated mode is active on the port"); + sfc_warn(sa, "the change is to be applied on the next " + "start provided that isolated mode is " + "disabled prior the next start"); + } else if ((sa->state == SFC_ADAPTER_STARTED) && + (sfc_set_rx_mode(sa) != 0)) { + *toggle = !(enabled); + sfc_warn(sa, "Failed to %s %s mode", + ((enabled) ? "enable" : "disable"), desc); + } + } + + sfc_adapter_unlock(sa); +} + +static void +sfc_dev_promisc_enable(struct rte_eth_dev *dev) +{ + sfc_dev_filter_set(dev, SFC_DEV_FILTER_MODE_PROMISC, B_TRUE); +} + +static void +sfc_dev_promisc_disable(struct rte_eth_dev *dev) +{ + sfc_dev_filter_set(dev, SFC_DEV_FILTER_MODE_PROMISC, B_FALSE); +} + +static void +sfc_dev_allmulti_enable(struct rte_eth_dev *dev) +{ + sfc_dev_filter_set(dev, SFC_DEV_FILTER_MODE_ALLMULTI, B_TRUE); +} + +static void +sfc_dev_allmulti_disable(struct rte_eth_dev *dev) +{ + sfc_dev_filter_set(dev, SFC_DEV_FILTER_MODE_ALLMULTI, B_FALSE); +} + +static int +sfc_rx_queue_setup(struct rte_eth_dev *dev, uint16_t rx_queue_id, + uint16_t nb_rx_desc, unsigned int socket_id, + const struct rte_eth_rxconf *rx_conf, + struct rte_mempool *mb_pool) +{ + struct sfc_adapter *sa = dev->data->dev_private; + int rc; + + sfc_log_init(sa, "RxQ=%u nb_rx_desc=%u socket_id=%u", + rx_queue_id, nb_rx_desc, socket_id); + + sfc_adapter_lock(sa); + + rc = sfc_rx_qinit(sa, rx_queue_id, nb_rx_desc, socket_id, + rx_conf, mb_pool); + if (rc != 0) + goto fail_rx_qinit; + + dev->data->rx_queues[rx_queue_id] = sa->rxq_info[rx_queue_id].rxq->dp; + + sfc_adapter_unlock(sa); + + return 0; + +fail_rx_qinit: + sfc_adapter_unlock(sa); + SFC_ASSERT(rc > 0); + return -rc; +} + +static void +sfc_rx_queue_release(void *queue) +{ + struct sfc_dp_rxq *dp_rxq = queue; + struct sfc_rxq *rxq; + struct sfc_adapter *sa; + unsigned int sw_index; + + if (dp_rxq == NULL) + return; + + rxq = sfc_rxq_by_dp_rxq(dp_rxq); + sa = rxq->evq->sa; + sfc_adapter_lock(sa); + + sw_index = sfc_rxq_sw_index(rxq); + + sfc_log_init(sa, "RxQ=%u", sw_index); + + sa->eth_dev->data->rx_queues[sw_index] = NULL; + + sfc_rx_qfini(sa, sw_index); + + sfc_adapter_unlock(sa); +} + +static int +sfc_tx_queue_setup(struct rte_eth_dev *dev, uint16_t tx_queue_id, + uint16_t nb_tx_desc, unsigned int socket_id, + const struct rte_eth_txconf *tx_conf) +{ + struct sfc_adapter *sa = dev->data->dev_private; + int rc; + + sfc_log_init(sa, "TxQ = %u, nb_tx_desc = %u, socket_id = %u", + tx_queue_id, nb_tx_desc, socket_id); + + sfc_adapter_lock(sa); + + rc = sfc_tx_qinit(sa, tx_queue_id, nb_tx_desc, socket_id, tx_conf); + if (rc != 0) + goto fail_tx_qinit; + + dev->data->tx_queues[tx_queue_id] = sa->txq_info[tx_queue_id].txq->dp; + + sfc_adapter_unlock(sa); + return 0; + +fail_tx_qinit: + sfc_adapter_unlock(sa); + SFC_ASSERT(rc > 0); + return -rc; +} + +static void +sfc_tx_queue_release(void *queue) +{ + struct sfc_dp_txq *dp_txq = queue; + struct sfc_txq *txq; + unsigned int sw_index; + struct sfc_adapter *sa; + + if (dp_txq == NULL) + return; + + txq = sfc_txq_by_dp_txq(dp_txq); + sw_index = sfc_txq_sw_index(txq); + + SFC_ASSERT(txq->evq != NULL); + sa = txq->evq->sa; + + sfc_log_init(sa, "TxQ = %u", sw_index); + + sfc_adapter_lock(sa); + + SFC_ASSERT(sw_index < sa->eth_dev->data->nb_tx_queues); + sa->eth_dev->data->tx_queues[sw_index] = NULL; + + sfc_tx_qfini(sa, sw_index); + + sfc_adapter_unlock(sa); +} + +static int +sfc_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) +{ + struct sfc_adapter *sa = dev->data->dev_private; + struct sfc_port *port = &sa->port; + uint64_t *mac_stats; + int ret; + + rte_spinlock_lock(&port->mac_stats_lock); + + ret = sfc_port_update_mac_stats(sa); + if (ret != 0) + goto unlock; + + mac_stats = port->mac_stats_buf; + + if (EFX_MAC_STAT_SUPPORTED(port->mac_stats_mask, + EFX_MAC_VADAPTER_RX_UNICAST_PACKETS)) { + stats->ipackets = + mac_stats[EFX_MAC_VADAPTER_RX_UNICAST_PACKETS] + + mac_stats[EFX_MAC_VADAPTER_RX_MULTICAST_PACKETS] + + mac_stats[EFX_MAC_VADAPTER_RX_BROADCAST_PACKETS]; + stats->opackets = + mac_stats[EFX_MAC_VADAPTER_TX_UNICAST_PACKETS] + + mac_stats[EFX_MAC_VADAPTER_TX_MULTICAST_PACKETS] + + mac_stats[EFX_MAC_VADAPTER_TX_BROADCAST_PACKETS]; + stats->ibytes = + mac_stats[EFX_MAC_VADAPTER_RX_UNICAST_BYTES] + + mac_stats[EFX_MAC_VADAPTER_RX_MULTICAST_BYTES] + + mac_stats[EFX_MAC_VADAPTER_RX_BROADCAST_BYTES]; + stats->obytes = + mac_stats[EFX_MAC_VADAPTER_TX_UNICAST_BYTES] + + mac_stats[EFX_MAC_VADAPTER_TX_MULTICAST_BYTES] + + mac_stats[EFX_MAC_VADAPTER_TX_BROADCAST_BYTES]; + stats->imissed = mac_stats[EFX_MAC_VADAPTER_RX_OVERFLOW]; + stats->ierrors = mac_stats[EFX_MAC_VADAPTER_RX_BAD_PACKETS]; + stats->oerrors = mac_stats[EFX_MAC_VADAPTER_TX_BAD_PACKETS]; + } else { + stats->ipackets = mac_stats[EFX_MAC_RX_PKTS]; + stats->opackets = mac_stats[EFX_MAC_TX_PKTS]; + stats->ibytes = mac_stats[EFX_MAC_RX_OCTETS]; + stats->obytes = mac_stats[EFX_MAC_TX_OCTETS]; + /* + * Take into account stats which are whenever supported + * on EF10. If some stat is not supported by current + * firmware variant or HW revision, it is guaranteed + * to be zero in mac_stats. + */ + stats->imissed = + mac_stats[EFX_MAC_RX_NODESC_DROP_CNT] + + mac_stats[EFX_MAC_PM_TRUNC_BB_OVERFLOW] + + mac_stats[EFX_MAC_PM_DISCARD_BB_OVERFLOW] + + mac_stats[EFX_MAC_PM_TRUNC_VFIFO_FULL] + + mac_stats[EFX_MAC_PM_DISCARD_VFIFO_FULL] + + mac_stats[EFX_MAC_PM_TRUNC_QBB] + + mac_stats[EFX_MAC_PM_DISCARD_QBB] + + mac_stats[EFX_MAC_PM_DISCARD_MAPPING] + + mac_stats[EFX_MAC_RXDP_Q_DISABLED_PKTS] + + mac_stats[EFX_MAC_RXDP_DI_DROPPED_PKTS]; + stats->ierrors = + mac_stats[EFX_MAC_RX_FCS_ERRORS] + + mac_stats[EFX_MAC_RX_ALIGN_ERRORS] + + mac_stats[EFX_MAC_RX_JABBER_PKTS]; + /* no oerrors counters supported on EF10 */ + } + +unlock: + rte_spinlock_unlock(&port->mac_stats_lock); + SFC_ASSERT(ret >= 0); + return -ret; +} + +static void +sfc_stats_reset(struct rte_eth_dev *dev) +{ + struct sfc_adapter *sa = dev->data->dev_private; + struct sfc_port *port = &sa->port; + int rc; + + if (sa->state != SFC_ADAPTER_STARTED) { + /* + * The operation cannot be done if port is not started; it + * will be scheduled to be done during the next port start + */ + port->mac_stats_reset_pending = B_TRUE; + return; + } + + rc = sfc_port_reset_mac_stats(sa); + if (rc != 0) + sfc_err(sa, "failed to reset statistics (rc = %d)", rc); +} + +static int +sfc_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, + unsigned int xstats_count) +{ + struct sfc_adapter *sa = dev->data->dev_private; + struct sfc_port *port = &sa->port; + uint64_t *mac_stats; + int rc; + unsigned int i; + int nstats = 0; + + rte_spinlock_lock(&port->mac_stats_lock); + + rc = sfc_port_update_mac_stats(sa); + if (rc != 0) { + SFC_ASSERT(rc > 0); + nstats = -rc; + goto unlock; + } + + mac_stats = port->mac_stats_buf; + + for (i = 0; i < EFX_MAC_NSTATS; ++i) { + if (EFX_MAC_STAT_SUPPORTED(port->mac_stats_mask, i)) { + if (xstats != NULL && nstats < (int)xstats_count) { + xstats[nstats].id = nstats; + xstats[nstats].value = mac_stats[i]; + } + nstats++; + } + } + +unlock: + rte_spinlock_unlock(&port->mac_stats_lock); + + return nstats; +} + +static int +sfc_xstats_get_names(struct rte_eth_dev *dev, + struct rte_eth_xstat_name *xstats_names, + unsigned int xstats_count) +{ + struct sfc_adapter *sa = dev->data->dev_private; + struct sfc_port *port = &sa->port; + unsigned int i; + unsigned int nstats = 0; + + for (i = 0; i < EFX_MAC_NSTATS; ++i) { + if (EFX_MAC_STAT_SUPPORTED(port->mac_stats_mask, i)) { + if (xstats_names != NULL && nstats < xstats_count) + strncpy(xstats_names[nstats].name, + efx_mac_stat_name(sa->nic, i), + sizeof(xstats_names[0].name)); + nstats++; + } + } + + return nstats; +} + +static int +sfc_xstats_get_by_id(struct rte_eth_dev *dev, const uint64_t *ids, + uint64_t *values, unsigned int n) +{ + struct sfc_adapter *sa = dev->data->dev_private; + struct sfc_port *port = &sa->port; + uint64_t *mac_stats; + unsigned int nb_supported = 0; + unsigned int nb_written = 0; + unsigned int i; + int ret; + int rc; + + if (unlikely(values == NULL) || + unlikely((ids == NULL) && (n < port->mac_stats_nb_supported))) + return port->mac_stats_nb_supported; + + rte_spinlock_lock(&port->mac_stats_lock); + + rc = sfc_port_update_mac_stats(sa); + if (rc != 0) { + SFC_ASSERT(rc > 0); + ret = -rc; + goto unlock; + } + + mac_stats = port->mac_stats_buf; + + for (i = 0; (i < EFX_MAC_NSTATS) && (nb_written < n); ++i) { + if (!EFX_MAC_STAT_SUPPORTED(port->mac_stats_mask, i)) + continue; + + if ((ids == NULL) || (ids[nb_written] == nb_supported)) + values[nb_written++] = mac_stats[i]; + + ++nb_supported; + } + + ret = nb_written; + +unlock: + rte_spinlock_unlock(&port->mac_stats_lock); + + return ret; +} + +static int +sfc_xstats_get_names_by_id(struct rte_eth_dev *dev, + struct rte_eth_xstat_name *xstats_names, + const uint64_t *ids, unsigned int size) +{ + struct sfc_adapter *sa = dev->data->dev_private; + struct sfc_port *port = &sa->port; + unsigned int nb_supported = 0; + unsigned int nb_written = 0; + unsigned int i; + + if (unlikely(xstats_names == NULL) || + unlikely((ids == NULL) && (size < port->mac_stats_nb_supported))) + return port->mac_stats_nb_supported; + + for (i = 0; (i < EFX_MAC_NSTATS) && (nb_written < size); ++i) { + if (!EFX_MAC_STAT_SUPPORTED(port->mac_stats_mask, i)) + continue; + + if ((ids == NULL) || (ids[nb_written] == nb_supported)) { + char *name = xstats_names[nb_written++].name; + + strncpy(name, efx_mac_stat_name(sa->nic, i), + sizeof(xstats_names[0].name)); + name[sizeof(xstats_names[0].name) - 1] = '\0'; + } + + ++nb_supported; + } + + return nb_written; +} + +static int +sfc_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) +{ + struct sfc_adapter *sa = dev->data->dev_private; + unsigned int wanted_fc, link_fc; + + memset(fc_conf, 0, sizeof(*fc_conf)); + + sfc_adapter_lock(sa); + + if (sa->state == SFC_ADAPTER_STARTED) + efx_mac_fcntl_get(sa->nic, &wanted_fc, &link_fc); + else + link_fc = sa->port.flow_ctrl; + + switch (link_fc) { + case 0: + fc_conf->mode = RTE_FC_NONE; + break; + case EFX_FCNTL_RESPOND: + fc_conf->mode = RTE_FC_RX_PAUSE; + break; + case EFX_FCNTL_GENERATE: + fc_conf->mode = RTE_FC_TX_PAUSE; + break; + case (EFX_FCNTL_RESPOND | EFX_FCNTL_GENERATE): + fc_conf->mode = RTE_FC_FULL; + break; + default: + sfc_err(sa, "%s: unexpected flow control value %#x", + __func__, link_fc); + } + + fc_conf->autoneg = sa->port.flow_ctrl_autoneg; + + sfc_adapter_unlock(sa); + + return 0; +} + +static int +sfc_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) +{ + struct sfc_adapter *sa = dev->data->dev_private; + struct sfc_port *port = &sa->port; + unsigned int fcntl; + int rc; + + if (fc_conf->high_water != 0 || fc_conf->low_water != 0 || + fc_conf->pause_time != 0 || fc_conf->send_xon != 0 || + fc_conf->mac_ctrl_frame_fwd != 0) { + sfc_err(sa, "unsupported flow control settings specified"); + rc = EINVAL; + goto fail_inval; + } + + switch (fc_conf->mode) { + case RTE_FC_NONE: + fcntl = 0; + break; + case RTE_FC_RX_PAUSE: + fcntl = EFX_FCNTL_RESPOND; + break; + case RTE_FC_TX_PAUSE: + fcntl = EFX_FCNTL_GENERATE; + break; + case RTE_FC_FULL: + fcntl = EFX_FCNTL_RESPOND | EFX_FCNTL_GENERATE; + break; + default: + rc = EINVAL; + goto fail_inval; + } + + sfc_adapter_lock(sa); + + if (sa->state == SFC_ADAPTER_STARTED) { + rc = efx_mac_fcntl_set(sa->nic, fcntl, fc_conf->autoneg); + if (rc != 0) + goto fail_mac_fcntl_set; + } + + port->flow_ctrl = fcntl; + port->flow_ctrl_autoneg = fc_conf->autoneg; + + sfc_adapter_unlock(sa); + + return 0; + +fail_mac_fcntl_set: + sfc_adapter_unlock(sa); +fail_inval: + SFC_ASSERT(rc > 0); + return -rc; +} + +static int +sfc_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu) +{ + struct sfc_adapter *sa = dev->data->dev_private; + size_t pdu = EFX_MAC_PDU(mtu); + size_t old_pdu; + int rc; + + sfc_log_init(sa, "mtu=%u", mtu); + + rc = EINVAL; + if (pdu < EFX_MAC_PDU_MIN) { + sfc_err(sa, "too small MTU %u (PDU size %u less than min %u)", + (unsigned int)mtu, (unsigned int)pdu, + EFX_MAC_PDU_MIN); + goto fail_inval; + } + if (pdu > EFX_MAC_PDU_MAX) { + sfc_err(sa, "too big MTU %u (PDU size %u greater than max %u)", + (unsigned int)mtu, (unsigned int)pdu, + EFX_MAC_PDU_MAX); + goto fail_inval; + } + + sfc_adapter_lock(sa); + + if (pdu != sa->port.pdu) { + if (sa->state == SFC_ADAPTER_STARTED) { + sfc_stop(sa); + + old_pdu = sa->port.pdu; + sa->port.pdu = pdu; + rc = sfc_start(sa); + if (rc != 0) + goto fail_start; + } else { + sa->port.pdu = pdu; + } + } + + /* + * The driver does not use it, but other PMDs update jumbo_frame + * flag and max_rx_pkt_len when MTU is set. + */ + dev->data->dev_conf.rxmode.jumbo_frame = (mtu > ETHER_MAX_LEN); + dev->data->dev_conf.rxmode.max_rx_pkt_len = sa->port.pdu; + + sfc_adapter_unlock(sa); + + sfc_log_init(sa, "done"); + return 0; + +fail_start: + sa->port.pdu = old_pdu; + if (sfc_start(sa) != 0) + sfc_err(sa, "cannot start with neither new (%u) nor old (%u) " + "PDU max size - port is stopped", + (unsigned int)pdu, (unsigned int)old_pdu); + sfc_adapter_unlock(sa); + +fail_inval: + sfc_log_init(sa, "failed %d", rc); + SFC_ASSERT(rc > 0); + return -rc; +} +static void +sfc_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr) +{ + struct sfc_adapter *sa = dev->data->dev_private; + const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic); + struct sfc_port *port = &sa->port; + int rc; + + sfc_adapter_lock(sa); + + /* + * Copy the address to the device private data so that + * it could be recalled in the case of adapter restart. + */ + ether_addr_copy(mac_addr, &port->default_mac_addr); + + if (port->isolated) { + sfc_err(sa, "isolated mode is active on the port"); + sfc_err(sa, "will not set MAC address"); + goto unlock; + } + + if (sa->state != SFC_ADAPTER_STARTED) { + sfc_info(sa, "the port is not started"); + sfc_info(sa, "the new MAC address will be set on port start"); + + goto unlock; + } + + if (encp->enc_allow_set_mac_with_installed_filters) { + rc = efx_mac_addr_set(sa->nic, mac_addr->addr_bytes); + if (rc != 0) { + sfc_err(sa, "cannot set MAC address (rc = %u)", rc); + goto unlock; + } + + /* + * Changing the MAC address by means of MCDI request + * has no effect on received traffic, therefore + * we also need to update unicast filters + */ + rc = sfc_set_rx_mode(sa); + if (rc != 0) + sfc_err(sa, "cannot set filter (rc = %u)", rc); + } else { + sfc_warn(sa, "cannot set MAC address with filters installed"); + sfc_warn(sa, "adapter will be restarted to pick the new MAC"); + sfc_warn(sa, "(some traffic may be dropped)"); + + /* + * Since setting MAC address with filters installed is not + * allowed on the adapter, the new MAC address will be set + * by means of adapter restart. sfc_start() shall retrieve + * the new address from the device private data and set it. + */ + sfc_stop(sa); + rc = sfc_start(sa); + if (rc != 0) + sfc_err(sa, "cannot restart adapter (rc = %u)", rc); + } + +unlock: + /* + * In the case of failure sa->port->default_mac_addr does not + * need rollback since no error code is returned, and the upper + * API will anyway update the external MAC address storage. + * To be consistent with that new value it is better to keep + * the device private value the same. + */ + sfc_adapter_unlock(sa); +} + + +static int +sfc_set_mc_addr_list(struct rte_eth_dev *dev, struct ether_addr *mc_addr_set, + uint32_t nb_mc_addr) +{ + struct sfc_adapter *sa = dev->data->dev_private; + struct sfc_port *port = &sa->port; + uint8_t *mc_addrs = port->mcast_addrs; + int rc; + unsigned int i; + + if (port->isolated) { + sfc_err(sa, "isolated mode is active on the port"); + sfc_err(sa, "will not set multicast address list"); + return -ENOTSUP; + } + + if (mc_addrs == NULL) + return -ENOBUFS; + + if (nb_mc_addr > port->max_mcast_addrs) { + sfc_err(sa, "too many multicast addresses: %u > %u", + nb_mc_addr, port->max_mcast_addrs); + return -EINVAL; + } + + for (i = 0; i < nb_mc_addr; ++i) { + rte_memcpy(mc_addrs, mc_addr_set[i].addr_bytes, + EFX_MAC_ADDR_LEN); + mc_addrs += EFX_MAC_ADDR_LEN; + } + + port->nb_mcast_addrs = nb_mc_addr; + + if (sa->state != SFC_ADAPTER_STARTED) + return 0; + + rc = efx_mac_multicast_list_set(sa->nic, port->mcast_addrs, + port->nb_mcast_addrs); + if (rc != 0) + sfc_err(sa, "cannot set multicast address list (rc = %u)", rc); + + SFC_ASSERT(rc > 0); + return -rc; +} + +/* + * The function is used by the secondary process as well. It must not + * use any process-local pointers from the adapter data. + */ +static void +sfc_rx_queue_info_get(struct rte_eth_dev *dev, uint16_t rx_queue_id, + struct rte_eth_rxq_info *qinfo) +{ + struct sfc_adapter *sa = dev->data->dev_private; + struct sfc_rxq_info *rxq_info; + struct sfc_rxq *rxq; + + sfc_adapter_lock(sa); + + SFC_ASSERT(rx_queue_id < sa->rxq_count); + + rxq_info = &sa->rxq_info[rx_queue_id]; + rxq = rxq_info->rxq; + SFC_ASSERT(rxq != NULL); + + qinfo->mp = rxq->refill_mb_pool; + qinfo->conf.rx_free_thresh = rxq->refill_threshold; + qinfo->conf.rx_drop_en = 1; + qinfo->conf.rx_deferred_start = rxq_info->deferred_start; + qinfo->scattered_rx = (rxq_info->type == EFX_RXQ_TYPE_SCATTER); + qinfo->nb_desc = rxq_info->entries; + + sfc_adapter_unlock(sa); +} + +/* + * The function is used by the secondary process as well. It must not + * use any process-local pointers from the adapter data. + */ +static void +sfc_tx_queue_info_get(struct rte_eth_dev *dev, uint16_t tx_queue_id, + struct rte_eth_txq_info *qinfo) +{ + struct sfc_adapter *sa = dev->data->dev_private; + struct sfc_txq_info *txq_info; + + sfc_adapter_lock(sa); + + SFC_ASSERT(tx_queue_id < sa->txq_count); + + txq_info = &sa->txq_info[tx_queue_id]; + SFC_ASSERT(txq_info->txq != NULL); + + memset(qinfo, 0, sizeof(*qinfo)); + + qinfo->conf.txq_flags = txq_info->txq->flags; + qinfo->conf.tx_free_thresh = txq_info->txq->free_thresh; + qinfo->conf.tx_deferred_start = txq_info->deferred_start; + qinfo->nb_desc = txq_info->entries; + + sfc_adapter_unlock(sa); +} + +static uint32_t +sfc_rx_queue_count(struct rte_eth_dev *dev, uint16_t rx_queue_id) +{ + struct sfc_adapter *sa = dev->data->dev_private; + + sfc_log_init(sa, "RxQ=%u", rx_queue_id); + + return sfc_rx_qdesc_npending(sa, rx_queue_id); +} + +static int +sfc_rx_descriptor_done(void *queue, uint16_t offset) +{ + struct sfc_dp_rxq *dp_rxq = queue; + + return sfc_rx_qdesc_done(dp_rxq, offset); +} + +static int +sfc_rx_descriptor_status(void *queue, uint16_t offset) +{ + struct sfc_dp_rxq *dp_rxq = queue; + struct sfc_rxq *rxq = sfc_rxq_by_dp_rxq(dp_rxq); + + return rxq->evq->sa->dp_rx->qdesc_status(dp_rxq, offset); +} + +static int +sfc_tx_descriptor_status(void *queue, uint16_t offset) +{ + struct sfc_dp_txq *dp_txq = queue; + struct sfc_txq *txq = sfc_txq_by_dp_txq(dp_txq); + + return txq->evq->sa->dp_tx->qdesc_status(dp_txq, offset); +} + +static int +sfc_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id) +{ + struct sfc_adapter *sa = dev->data->dev_private; + int rc; + + sfc_log_init(sa, "RxQ=%u", rx_queue_id); + + sfc_adapter_lock(sa); + + rc = EINVAL; + if (sa->state != SFC_ADAPTER_STARTED) + goto fail_not_started; + + rc = sfc_rx_qstart(sa, rx_queue_id); + if (rc != 0) + goto fail_rx_qstart; + + sa->rxq_info[rx_queue_id].deferred_started = B_TRUE; + + sfc_adapter_unlock(sa); + + return 0; + +fail_rx_qstart: +fail_not_started: + sfc_adapter_unlock(sa); + SFC_ASSERT(rc > 0); + return -rc; +} + +static int +sfc_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id) +{ + struct sfc_adapter *sa = dev->data->dev_private; + + sfc_log_init(sa, "RxQ=%u", rx_queue_id); + + sfc_adapter_lock(sa); + sfc_rx_qstop(sa, rx_queue_id); + + sa->rxq_info[rx_queue_id].deferred_started = B_FALSE; + + sfc_adapter_unlock(sa); + + return 0; +} + +static int +sfc_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id) +{ + struct sfc_adapter *sa = dev->data->dev_private; + int rc; + + sfc_log_init(sa, "TxQ = %u", tx_queue_id); + + sfc_adapter_lock(sa); + + rc = EINVAL; + if (sa->state != SFC_ADAPTER_STARTED) + goto fail_not_started; + + rc = sfc_tx_qstart(sa, tx_queue_id); + if (rc != 0) + goto fail_tx_qstart; + + sa->txq_info[tx_queue_id].deferred_started = B_TRUE; + + sfc_adapter_unlock(sa); + return 0; + +fail_tx_qstart: + +fail_not_started: + sfc_adapter_unlock(sa); + SFC_ASSERT(rc > 0); + return -rc; +} + +static int +sfc_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id) +{ + struct sfc_adapter *sa = dev->data->dev_private; + + sfc_log_init(sa, "TxQ = %u", tx_queue_id); + + sfc_adapter_lock(sa); + + sfc_tx_qstop(sa, tx_queue_id); + + sa->txq_info[tx_queue_id].deferred_started = B_FALSE; + + sfc_adapter_unlock(sa); + return 0; +} + +#if EFSYS_OPT_RX_SCALE +static int +sfc_dev_rss_hash_conf_get(struct rte_eth_dev *dev, + struct rte_eth_rss_conf *rss_conf) +{ + struct sfc_adapter *sa = dev->data->dev_private; + struct sfc_port *port = &sa->port; + + if ((sa->rss_support != EFX_RX_SCALE_EXCLUSIVE) || port->isolated) + return -ENOTSUP; + + if (sa->rss_channels == 0) + return -EINVAL; + + sfc_adapter_lock(sa); + + /* + * Mapping of hash configuration between RTE and EFX is not one-to-one, + * hence, conversion is done here to derive a correct set of ETH_RSS + * flags which corresponds to the active EFX configuration stored + * locally in 'sfc_adapter' and kept up-to-date + */ + rss_conf->rss_hf = sfc_efx_to_rte_hash_type(sa->rss_hash_types); + rss_conf->rss_key_len = EFX_RSS_KEY_SIZE; + if (rss_conf->rss_key != NULL) + rte_memcpy(rss_conf->rss_key, sa->rss_key, EFX_RSS_KEY_SIZE); + + sfc_adapter_unlock(sa); + + return 0; +} + +static int +sfc_dev_rss_hash_update(struct rte_eth_dev *dev, + struct rte_eth_rss_conf *rss_conf) +{ + struct sfc_adapter *sa = dev->data->dev_private; + struct sfc_port *port = &sa->port; + unsigned int efx_hash_types; + int rc = 0; + + if (port->isolated) + return -ENOTSUP; + + if (sa->rss_support != EFX_RX_SCALE_EXCLUSIVE) { + sfc_err(sa, "RSS is not available"); + return -ENOTSUP; + } + + if (sa->rss_channels == 0) { + sfc_err(sa, "RSS is not configured"); + return -EINVAL; + } + + if ((rss_conf->rss_key != NULL) && + (rss_conf->rss_key_len != sizeof(sa->rss_key))) { + sfc_err(sa, "RSS key size is wrong (should be %lu)", + sizeof(sa->rss_key)); + return -EINVAL; + } + + if ((rss_conf->rss_hf & ~SFC_RSS_OFFLOADS) != 0) { + sfc_err(sa, "unsupported hash functions requested"); + return -EINVAL; + } + + sfc_adapter_lock(sa); + + efx_hash_types = sfc_rte_to_efx_hash_type(rss_conf->rss_hf); + + rc = efx_rx_scale_mode_set(sa->nic, EFX_RSS_CONTEXT_DEFAULT, + EFX_RX_HASHALG_TOEPLITZ, + efx_hash_types, B_TRUE); + if (rc != 0) + goto fail_scale_mode_set; + + if (rss_conf->rss_key != NULL) { + if (sa->state == SFC_ADAPTER_STARTED) { + rc = efx_rx_scale_key_set(sa->nic, + EFX_RSS_CONTEXT_DEFAULT, + rss_conf->rss_key, + sizeof(sa->rss_key)); + if (rc != 0) + goto fail_scale_key_set; + } + + rte_memcpy(sa->rss_key, rss_conf->rss_key, sizeof(sa->rss_key)); + } + + sa->rss_hash_types = efx_hash_types; + + sfc_adapter_unlock(sa); + + return 0; + +fail_scale_key_set: + if (efx_rx_scale_mode_set(sa->nic, EFX_RSS_CONTEXT_DEFAULT, + EFX_RX_HASHALG_TOEPLITZ, + sa->rss_hash_types, B_TRUE) != 0) + sfc_err(sa, "failed to restore RSS mode"); + +fail_scale_mode_set: + sfc_adapter_unlock(sa); + return -rc; +} + +static int +sfc_dev_rss_reta_query(struct rte_eth_dev *dev, + struct rte_eth_rss_reta_entry64 *reta_conf, + uint16_t reta_size) +{ + struct sfc_adapter *sa = dev->data->dev_private; + struct sfc_port *port = &sa->port; + int entry; + + if ((sa->rss_support != EFX_RX_SCALE_EXCLUSIVE) || port->isolated) + return -ENOTSUP; + + if (sa->rss_channels == 0) + return -EINVAL; + + if (reta_size != EFX_RSS_TBL_SIZE) + return -EINVAL; + + sfc_adapter_lock(sa); + + for (entry = 0; entry < reta_size; entry++) { + int grp = entry / RTE_RETA_GROUP_SIZE; + int grp_idx = entry % RTE_RETA_GROUP_SIZE; + + if ((reta_conf[grp].mask >> grp_idx) & 1) + reta_conf[grp].reta[grp_idx] = sa->rss_tbl[entry]; + } + + sfc_adapter_unlock(sa); + + return 0; +} + +static int +sfc_dev_rss_reta_update(struct rte_eth_dev *dev, + struct rte_eth_rss_reta_entry64 *reta_conf, + uint16_t reta_size) +{ + struct sfc_adapter *sa = dev->data->dev_private; + struct sfc_port *port = &sa->port; + unsigned int *rss_tbl_new; + uint16_t entry; + int rc = 0; + + + if (port->isolated) + return -ENOTSUP; + + if (sa->rss_support != EFX_RX_SCALE_EXCLUSIVE) { + sfc_err(sa, "RSS is not available"); + return -ENOTSUP; + } + + if (sa->rss_channels == 0) { + sfc_err(sa, "RSS is not configured"); + return -EINVAL; + } + + if (reta_size != EFX_RSS_TBL_SIZE) { + sfc_err(sa, "RETA size is wrong (should be %u)", + EFX_RSS_TBL_SIZE); + return -EINVAL; + } + + rss_tbl_new = rte_zmalloc("rss_tbl_new", sizeof(sa->rss_tbl), 0); + if (rss_tbl_new == NULL) + return -ENOMEM; + + sfc_adapter_lock(sa); + + rte_memcpy(rss_tbl_new, sa->rss_tbl, sizeof(sa->rss_tbl)); + + for (entry = 0; entry < reta_size; entry++) { + int grp_idx = entry % RTE_RETA_GROUP_SIZE; + struct rte_eth_rss_reta_entry64 *grp; + + grp = &reta_conf[entry / RTE_RETA_GROUP_SIZE]; + + if (grp->mask & (1ull << grp_idx)) { + if (grp->reta[grp_idx] >= sa->rss_channels) { + rc = EINVAL; + goto bad_reta_entry; + } + rss_tbl_new[entry] = grp->reta[grp_idx]; + } + } + + if (sa->state == SFC_ADAPTER_STARTED) { + rc = efx_rx_scale_tbl_set(sa->nic, EFX_RSS_CONTEXT_DEFAULT, + rss_tbl_new, EFX_RSS_TBL_SIZE); + if (rc != 0) + goto fail_scale_tbl_set; + } + + rte_memcpy(sa->rss_tbl, rss_tbl_new, sizeof(sa->rss_tbl)); + +fail_scale_tbl_set: +bad_reta_entry: + sfc_adapter_unlock(sa); + + rte_free(rss_tbl_new); + + SFC_ASSERT(rc >= 0); + return -rc; +} +#endif + +static int +sfc_dev_filter_ctrl(struct rte_eth_dev *dev, enum rte_filter_type filter_type, + enum rte_filter_op filter_op, + void *arg) +{ + struct sfc_adapter *sa = dev->data->dev_private; + int rc = ENOTSUP; + + sfc_log_init(sa, "entry"); + + switch (filter_type) { + case RTE_ETH_FILTER_NONE: + sfc_err(sa, "Global filters configuration not supported"); + break; + case RTE_ETH_FILTER_MACVLAN: + sfc_err(sa, "MACVLAN filters not supported"); + break; + case RTE_ETH_FILTER_ETHERTYPE: + sfc_err(sa, "EtherType filters not supported"); + break; + case RTE_ETH_FILTER_FLEXIBLE: + sfc_err(sa, "Flexible filters not supported"); + break; + case RTE_ETH_FILTER_SYN: + sfc_err(sa, "SYN filters not supported"); + break; + case RTE_ETH_FILTER_NTUPLE: + sfc_err(sa, "NTUPLE filters not supported"); + break; + case RTE_ETH_FILTER_TUNNEL: + sfc_err(sa, "Tunnel filters not supported"); + break; + case RTE_ETH_FILTER_FDIR: + sfc_err(sa, "Flow Director filters not supported"); + break; + case RTE_ETH_FILTER_HASH: + sfc_err(sa, "Hash filters not supported"); + break; + case RTE_ETH_FILTER_GENERIC: + if (filter_op != RTE_ETH_FILTER_GET) { + rc = EINVAL; + } else { + *(const void **)arg = &sfc_flow_ops; + rc = 0; + } + break; + default: + sfc_err(sa, "Unknown filter type %u", filter_type); + break; + } + + sfc_log_init(sa, "exit: %d", -rc); + SFC_ASSERT(rc >= 0); + return -rc; +} + +static const struct eth_dev_ops sfc_eth_dev_ops = { + .dev_configure = sfc_dev_configure, + .dev_start = sfc_dev_start, + .dev_stop = sfc_dev_stop, + .dev_set_link_up = sfc_dev_set_link_up, + .dev_set_link_down = sfc_dev_set_link_down, + .dev_close = sfc_dev_close, + .promiscuous_enable = sfc_dev_promisc_enable, + .promiscuous_disable = sfc_dev_promisc_disable, + .allmulticast_enable = sfc_dev_allmulti_enable, + .allmulticast_disable = sfc_dev_allmulti_disable, + .link_update = sfc_dev_link_update, + .stats_get = sfc_stats_get, + .stats_reset = sfc_stats_reset, + .xstats_get = sfc_xstats_get, + .xstats_reset = sfc_stats_reset, + .xstats_get_names = sfc_xstats_get_names, + .dev_infos_get = sfc_dev_infos_get, + .dev_supported_ptypes_get = sfc_dev_supported_ptypes_get, + .mtu_set = sfc_dev_set_mtu, + .rx_queue_start = sfc_rx_queue_start, + .rx_queue_stop = sfc_rx_queue_stop, + .tx_queue_start = sfc_tx_queue_start, + .tx_queue_stop = sfc_tx_queue_stop, + .rx_queue_setup = sfc_rx_queue_setup, + .rx_queue_release = sfc_rx_queue_release, + .rx_queue_count = sfc_rx_queue_count, + .rx_descriptor_done = sfc_rx_descriptor_done, + .rx_descriptor_status = sfc_rx_descriptor_status, + .tx_descriptor_status = sfc_tx_descriptor_status, + .tx_queue_setup = sfc_tx_queue_setup, + .tx_queue_release = sfc_tx_queue_release, + .flow_ctrl_get = sfc_flow_ctrl_get, + .flow_ctrl_set = sfc_flow_ctrl_set, + .mac_addr_set = sfc_mac_addr_set, +#if EFSYS_OPT_RX_SCALE + .reta_update = sfc_dev_rss_reta_update, + .reta_query = sfc_dev_rss_reta_query, + .rss_hash_update = sfc_dev_rss_hash_update, + .rss_hash_conf_get = sfc_dev_rss_hash_conf_get, +#endif + .filter_ctrl = sfc_dev_filter_ctrl, + .set_mc_addr_list = sfc_set_mc_addr_list, + .rxq_info_get = sfc_rx_queue_info_get, + .txq_info_get = sfc_tx_queue_info_get, + .fw_version_get = sfc_fw_version_get, + .xstats_get_by_id = sfc_xstats_get_by_id, + .xstats_get_names_by_id = sfc_xstats_get_names_by_id, +}; + +/** + * Duplicate a string in potentially shared memory required for + * multi-process support. + * + * strdup() allocates from process-local heap/memory. + */ +static char * +sfc_strdup(const char *str) +{ + size_t size; + char *copy; + + if (str == NULL) + return NULL; + + size = strlen(str) + 1; + copy = rte_malloc(__func__, size, 0); + if (copy != NULL) + rte_memcpy(copy, str, size); + + return copy; +} + +static int +sfc_eth_dev_set_ops(struct rte_eth_dev *dev) +{ + struct sfc_adapter *sa = dev->data->dev_private; + unsigned int avail_caps = 0; + const char *rx_name = NULL; + const char *tx_name = NULL; + int rc; + + switch (sa->family) { + case EFX_FAMILY_HUNTINGTON: + case EFX_FAMILY_MEDFORD: + avail_caps |= SFC_DP_HW_FW_CAP_EF10; + break; + default: + break; + } + + rc = sfc_kvargs_process(sa, SFC_KVARG_RX_DATAPATH, + sfc_kvarg_string_handler, &rx_name); + if (rc != 0) + goto fail_kvarg_rx_datapath; + + if (rx_name != NULL) { + sa->dp_rx = sfc_dp_find_rx_by_name(&sfc_dp_head, rx_name); + if (sa->dp_rx == NULL) { + sfc_err(sa, "Rx datapath %s not found", rx_name); + rc = ENOENT; + goto fail_dp_rx; + } + if (!sfc_dp_match_hw_fw_caps(&sa->dp_rx->dp, avail_caps)) { + sfc_err(sa, + "Insufficient Hw/FW capabilities to use Rx datapath %s", + rx_name); + rc = EINVAL; + goto fail_dp_rx_caps; + } + } else { + sa->dp_rx = sfc_dp_find_rx_by_caps(&sfc_dp_head, avail_caps); + if (sa->dp_rx == NULL) { + sfc_err(sa, "Rx datapath by caps %#x not found", + avail_caps); + rc = ENOENT; + goto fail_dp_rx; + } + } + + sa->dp_rx_name = sfc_strdup(sa->dp_rx->dp.name); + if (sa->dp_rx_name == NULL) { + rc = ENOMEM; + goto fail_dp_rx_name; + } + + sfc_info(sa, "use %s Rx datapath", sa->dp_rx_name); + + dev->rx_pkt_burst = sa->dp_rx->pkt_burst; + + rc = sfc_kvargs_process(sa, SFC_KVARG_TX_DATAPATH, + sfc_kvarg_string_handler, &tx_name); + if (rc != 0) + goto fail_kvarg_tx_datapath; + + if (tx_name != NULL) { + sa->dp_tx = sfc_dp_find_tx_by_name(&sfc_dp_head, tx_name); + if (sa->dp_tx == NULL) { + sfc_err(sa, "Tx datapath %s not found", tx_name); + rc = ENOENT; + goto fail_dp_tx; + } + if (!sfc_dp_match_hw_fw_caps(&sa->dp_tx->dp, avail_caps)) { + sfc_err(sa, + "Insufficient Hw/FW capabilities to use Tx datapath %s", + tx_name); + rc = EINVAL; + goto fail_dp_tx_caps; + } + } else { + sa->dp_tx = sfc_dp_find_tx_by_caps(&sfc_dp_head, avail_caps); + if (sa->dp_tx == NULL) { + sfc_err(sa, "Tx datapath by caps %#x not found", + avail_caps); + rc = ENOENT; + goto fail_dp_tx; + } + } + + sa->dp_tx_name = sfc_strdup(sa->dp_tx->dp.name); + if (sa->dp_tx_name == NULL) { + rc = ENOMEM; + goto fail_dp_tx_name; + } + + sfc_info(sa, "use %s Tx datapath", sa->dp_tx_name); + + dev->tx_pkt_burst = sa->dp_tx->pkt_burst; + + dev->dev_ops = &sfc_eth_dev_ops; + + return 0; + +fail_dp_tx_name: +fail_dp_tx_caps: + sa->dp_tx = NULL; + +fail_dp_tx: +fail_kvarg_tx_datapath: + rte_free(sa->dp_rx_name); + sa->dp_rx_name = NULL; + +fail_dp_rx_name: +fail_dp_rx_caps: + sa->dp_rx = NULL; + +fail_dp_rx: +fail_kvarg_rx_datapath: + return rc; +} + +static void +sfc_eth_dev_clear_ops(struct rte_eth_dev *dev) +{ + struct sfc_adapter *sa = dev->data->dev_private; + + dev->dev_ops = NULL; + dev->rx_pkt_burst = NULL; + dev->tx_pkt_burst = NULL; + + rte_free(sa->dp_tx_name); + sa->dp_tx_name = NULL; + sa->dp_tx = NULL; + + rte_free(sa->dp_rx_name); + sa->dp_rx_name = NULL; + sa->dp_rx = NULL; +} + +static const struct eth_dev_ops sfc_eth_dev_secondary_ops = { + .rxq_info_get = sfc_rx_queue_info_get, + .txq_info_get = sfc_tx_queue_info_get, +}; + +static int +sfc_eth_dev_secondary_set_ops(struct rte_eth_dev *dev) +{ + /* + * Device private data has really many process-local pointers. + * Below code should be extremely careful to use data located + * in shared memory only. + */ + struct sfc_adapter *sa = dev->data->dev_private; + const struct sfc_dp_rx *dp_rx; + const struct sfc_dp_tx *dp_tx; + int rc; + + dp_rx = sfc_dp_find_rx_by_name(&sfc_dp_head, sa->dp_rx_name); + if (dp_rx == NULL) { + sfc_err(sa, "cannot find %s Rx datapath", sa->dp_tx_name); + rc = ENOENT; + goto fail_dp_rx; + } + if (~dp_rx->features & SFC_DP_RX_FEAT_MULTI_PROCESS) { + sfc_err(sa, "%s Rx datapath does not support multi-process", + sa->dp_tx_name); + rc = EINVAL; + goto fail_dp_rx_multi_process; + } + + dp_tx = sfc_dp_find_tx_by_name(&sfc_dp_head, sa->dp_tx_name); + if (dp_tx == NULL) { + sfc_err(sa, "cannot find %s Tx datapath", sa->dp_tx_name); + rc = ENOENT; + goto fail_dp_tx; + } + if (~dp_tx->features & SFC_DP_TX_FEAT_MULTI_PROCESS) { + sfc_err(sa, "%s Tx datapath does not support multi-process", + sa->dp_tx_name); + rc = EINVAL; + goto fail_dp_tx_multi_process; + } + + dev->rx_pkt_burst = dp_rx->pkt_burst; + dev->tx_pkt_burst = dp_tx->pkt_burst; + dev->dev_ops = &sfc_eth_dev_secondary_ops; + + return 0; + +fail_dp_tx_multi_process: +fail_dp_tx: +fail_dp_rx_multi_process: +fail_dp_rx: + return rc; +} + +static void +sfc_eth_dev_secondary_clear_ops(struct rte_eth_dev *dev) +{ + dev->dev_ops = NULL; + dev->tx_pkt_burst = NULL; + dev->rx_pkt_burst = NULL; +} + +static void +sfc_register_dp(void) +{ + /* Register once */ + if (TAILQ_EMPTY(&sfc_dp_head)) { + /* Prefer EF10 datapath */ + sfc_dp_register(&sfc_dp_head, &sfc_ef10_rx.dp); + sfc_dp_register(&sfc_dp_head, &sfc_efx_rx.dp); + + sfc_dp_register(&sfc_dp_head, &sfc_ef10_tx.dp); + sfc_dp_register(&sfc_dp_head, &sfc_efx_tx.dp); + sfc_dp_register(&sfc_dp_head, &sfc_ef10_simple_tx.dp); + } +} + +static int +sfc_eth_dev_init(struct rte_eth_dev *dev) +{ + struct sfc_adapter *sa = dev->data->dev_private; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + int rc; + const efx_nic_cfg_t *encp; + const struct ether_addr *from; + + sfc_register_dp(); + + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return -sfc_eth_dev_secondary_set_ops(dev); + + /* Required for logging */ + sa->pci_addr = pci_dev->addr; + sa->port_id = dev->data->port_id; + + sa->eth_dev = dev; + + /* Copy PCI device info to the dev->data */ + rte_eth_copy_pci_info(dev, pci_dev); + + rc = sfc_kvargs_parse(sa); + if (rc != 0) + goto fail_kvargs_parse; + + rc = sfc_kvargs_process(sa, SFC_KVARG_DEBUG_INIT, + sfc_kvarg_bool_handler, &sa->debug_init); + if (rc != 0) + goto fail_kvarg_debug_init; + + sfc_log_init(sa, "entry"); + + dev->data->mac_addrs = rte_zmalloc("sfc", ETHER_ADDR_LEN, 0); + if (dev->data->mac_addrs == NULL) { + rc = ENOMEM; + goto fail_mac_addrs; + } + + sfc_adapter_lock_init(sa); + sfc_adapter_lock(sa); + + sfc_log_init(sa, "probing"); + rc = sfc_probe(sa); + if (rc != 0) + goto fail_probe; + + sfc_log_init(sa, "set device ops"); + rc = sfc_eth_dev_set_ops(dev); + if (rc != 0) + goto fail_set_ops; + + sfc_log_init(sa, "attaching"); + rc = sfc_attach(sa); + if (rc != 0) + goto fail_attach; + + encp = efx_nic_cfg_get(sa->nic); + + /* + * The arguments are really reverse order in comparison to + * Linux kernel. Copy from NIC config to Ethernet device data. + */ + from = (const struct ether_addr *)(encp->enc_mac_addr); + ether_addr_copy(from, &dev->data->mac_addrs[0]); + + sfc_adapter_unlock(sa); + + sfc_log_init(sa, "done"); + return 0; + +fail_attach: + sfc_eth_dev_clear_ops(dev); + +fail_set_ops: + sfc_unprobe(sa); + +fail_probe: + sfc_adapter_unlock(sa); + sfc_adapter_lock_fini(sa); + rte_free(dev->data->mac_addrs); + dev->data->mac_addrs = NULL; + +fail_mac_addrs: +fail_kvarg_debug_init: + sfc_kvargs_cleanup(sa); + +fail_kvargs_parse: + sfc_log_init(sa, "failed %d", rc); + SFC_ASSERT(rc > 0); + return -rc; +} + +static int +sfc_eth_dev_uninit(struct rte_eth_dev *dev) +{ + struct sfc_adapter *sa; + + if (rte_eal_process_type() != RTE_PROC_PRIMARY) { + sfc_eth_dev_secondary_clear_ops(dev); + return 0; + } + + sa = dev->data->dev_private; + sfc_log_init(sa, "entry"); + + sfc_adapter_lock(sa); + + sfc_eth_dev_clear_ops(dev); + + sfc_detach(sa); + sfc_unprobe(sa); + + rte_free(dev->data->mac_addrs); + dev->data->mac_addrs = NULL; + + sfc_kvargs_cleanup(sa); + + sfc_adapter_unlock(sa); + sfc_adapter_lock_fini(sa); + + sfc_log_init(sa, "done"); + + /* Required for logging, so cleanup last */ + sa->eth_dev = NULL; + return 0; +} + +static const struct rte_pci_id pci_id_sfc_efx_map[] = { + { RTE_PCI_DEVICE(EFX_PCI_VENID_SFC, EFX_PCI_DEVID_FARMINGDALE) }, + { RTE_PCI_DEVICE(EFX_PCI_VENID_SFC, EFX_PCI_DEVID_FARMINGDALE_VF) }, + { RTE_PCI_DEVICE(EFX_PCI_VENID_SFC, EFX_PCI_DEVID_GREENPORT) }, + { RTE_PCI_DEVICE(EFX_PCI_VENID_SFC, EFX_PCI_DEVID_GREENPORT_VF) }, + { RTE_PCI_DEVICE(EFX_PCI_VENID_SFC, EFX_PCI_DEVID_MEDFORD) }, + { RTE_PCI_DEVICE(EFX_PCI_VENID_SFC, EFX_PCI_DEVID_MEDFORD_VF) }, + { .vendor_id = 0 /* sentinel */ } +}; + +static int sfc_eth_dev_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, + struct rte_pci_device *pci_dev) +{ + return rte_eth_dev_pci_generic_probe(pci_dev, + sizeof(struct sfc_adapter), sfc_eth_dev_init); +} + +static int sfc_eth_dev_pci_remove(struct rte_pci_device *pci_dev) +{ + return rte_eth_dev_pci_generic_remove(pci_dev, sfc_eth_dev_uninit); +} + +static struct rte_pci_driver sfc_efx_pmd = { + .id_table = pci_id_sfc_efx_map, + .drv_flags = + RTE_PCI_DRV_INTR_LSC | + RTE_PCI_DRV_NEED_MAPPING, + .probe = sfc_eth_dev_pci_probe, + .remove = sfc_eth_dev_pci_remove, +}; + +RTE_PMD_REGISTER_PCI(net_sfc_efx, sfc_efx_pmd); +RTE_PMD_REGISTER_PCI_TABLE(net_sfc_efx, pci_id_sfc_efx_map); +RTE_PMD_REGISTER_KMOD_DEP(net_sfc_efx, "* igb_uio | uio_pci_generic | vfio-pci"); +RTE_PMD_REGISTER_PARAM_STRING(net_sfc_efx, + SFC_KVARG_RX_DATAPATH "=" SFC_KVARG_VALUES_RX_DATAPATH " " + SFC_KVARG_TX_DATAPATH "=" SFC_KVARG_VALUES_TX_DATAPATH " " + SFC_KVARG_PERF_PROFILE "=" SFC_KVARG_VALUES_PERF_PROFILE " " + SFC_KVARG_STATS_UPDATE_PERIOD_MS "= " + SFC_KVARG_MCDI_LOGGING "=" SFC_KVARG_VALUES_BOOL " " + SFC_KVARG_DEBUG_INIT "=" SFC_KVARG_VALUES_BOOL); diff --git a/dpdk/drivers/net/sfc/sfc_ev.c b/dpdk/drivers/net/sfc/sfc_ev.c new file mode 100644 index 00000000..5fbebbf1 --- /dev/null +++ b/dpdk/drivers/net/sfc/sfc_ev.c @@ -0,0 +1,925 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2016-2017 Solarflare Communications Inc. + * All rights reserved. + * + * This software was jointly developed between OKTET Labs (under contract + * for Solarflare) and Solarflare Communications, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include + +#include "efx.h" + +#include "sfc.h" +#include "sfc_debug.h" +#include "sfc_log.h" +#include "sfc_ev.h" +#include "sfc_rx.h" +#include "sfc_tx.h" +#include "sfc_kvargs.h" + + +/* Initial delay when waiting for event queue init complete event */ +#define SFC_EVQ_INIT_BACKOFF_START_US (1) +/* Maximum delay between event queue polling attempts */ +#define SFC_EVQ_INIT_BACKOFF_MAX_US (10 * 1000) +/* Event queue init approx timeout */ +#define SFC_EVQ_INIT_TIMEOUT_US (2 * US_PER_S) + +/* Management event queue polling period in microseconds */ +#define SFC_MGMT_EV_QPOLL_PERIOD_US (US_PER_S) + +static const char * +sfc_evq_type2str(enum sfc_evq_type type) +{ + switch (type) { + case SFC_EVQ_TYPE_MGMT: + return "mgmt-evq"; + case SFC_EVQ_TYPE_RX: + return "rx-evq"; + case SFC_EVQ_TYPE_TX: + return "tx-evq"; + default: + SFC_ASSERT(B_FALSE); + return NULL; + } +} + +static boolean_t +sfc_ev_initialized(void *arg) +{ + struct sfc_evq *evq = arg; + + /* Init done events may be duplicated on SFN7xxx (SFC bug 31631) */ + SFC_ASSERT(evq->init_state == SFC_EVQ_STARTING || + evq->init_state == SFC_EVQ_STARTED); + + evq->init_state = SFC_EVQ_STARTED; + + return B_FALSE; +} + +static boolean_t +sfc_ev_nop_rx(void *arg, uint32_t label, uint32_t id, + uint32_t size, uint16_t flags) +{ + struct sfc_evq *evq = arg; + + sfc_err(evq->sa, + "EVQ %u unexpected Rx event label=%u id=%#x size=%u flags=%#x", + evq->evq_index, label, id, size, flags); + return B_TRUE; +} + +static boolean_t +sfc_ev_efx_rx(void *arg, __rte_unused uint32_t label, uint32_t id, + uint32_t size, uint16_t flags) +{ + struct sfc_evq *evq = arg; + struct sfc_efx_rxq *rxq; + unsigned int stop; + unsigned int pending_id; + unsigned int delta; + unsigned int i; + struct sfc_efx_rx_sw_desc *rxd; + + if (unlikely(evq->exception)) + goto done; + + rxq = sfc_efx_rxq_by_dp_rxq(evq->dp_rxq); + + SFC_ASSERT(rxq != NULL); + SFC_ASSERT(rxq->evq == evq); + SFC_ASSERT(rxq->flags & SFC_EFX_RXQ_FLAG_STARTED); + + stop = (id + 1) & rxq->ptr_mask; + pending_id = rxq->pending & rxq->ptr_mask; + delta = (stop >= pending_id) ? (stop - pending_id) : + (rxq->ptr_mask + 1 - pending_id + stop); + + if (delta == 0) { + /* + * Rx event with no new descriptors done and zero length + * is used to abort scattered packet when there is no room + * for the tail. + */ + if (unlikely(size != 0)) { + evq->exception = B_TRUE; + sfc_err(evq->sa, + "EVQ %u RxQ %u invalid RX abort " + "(id=%#x size=%u flags=%#x); needs restart", + evq->evq_index, rxq->dp.dpq.queue_id, + id, size, flags); + goto done; + } + + /* Add discard flag to the first fragment */ + rxq->sw_desc[pending_id].flags |= EFX_DISCARD; + /* Remove continue flag from the last fragment */ + rxq->sw_desc[id].flags &= ~EFX_PKT_CONT; + } else if (unlikely(delta > rxq->batch_max)) { + evq->exception = B_TRUE; + + sfc_err(evq->sa, + "EVQ %u RxQ %u completion out of order " + "(id=%#x delta=%u flags=%#x); needs restart", + evq->evq_index, rxq->dp.dpq.queue_id, + id, delta, flags); + + goto done; + } + + for (i = pending_id; i != stop; i = (i + 1) & rxq->ptr_mask) { + rxd = &rxq->sw_desc[i]; + + rxd->flags = flags; + + SFC_ASSERT(size < (1 << 16)); + rxd->size = (uint16_t)size; + } + + rxq->pending += delta; + +done: + return B_FALSE; +} + +static boolean_t +sfc_ev_dp_rx(void *arg, __rte_unused uint32_t label, uint32_t id, + __rte_unused uint32_t size, __rte_unused uint16_t flags) +{ + struct sfc_evq *evq = arg; + struct sfc_dp_rxq *dp_rxq; + + dp_rxq = evq->dp_rxq; + SFC_ASSERT(dp_rxq != NULL); + + SFC_ASSERT(evq->sa->dp_rx->qrx_ev != NULL); + return evq->sa->dp_rx->qrx_ev(dp_rxq, id); +} + +static boolean_t +sfc_ev_nop_tx(void *arg, uint32_t label, uint32_t id) +{ + struct sfc_evq *evq = arg; + + sfc_err(evq->sa, "EVQ %u unexpected Tx event label=%u id=%#x", + evq->evq_index, label, id); + return B_TRUE; +} + +static boolean_t +sfc_ev_tx(void *arg, __rte_unused uint32_t label, uint32_t id) +{ + struct sfc_evq *evq = arg; + struct sfc_dp_txq *dp_txq; + struct sfc_efx_txq *txq; + unsigned int stop; + unsigned int delta; + + dp_txq = evq->dp_txq; + SFC_ASSERT(dp_txq != NULL); + + txq = sfc_efx_txq_by_dp_txq(dp_txq); + SFC_ASSERT(txq->evq == evq); + + if (unlikely((txq->flags & SFC_EFX_TXQ_FLAG_STARTED) == 0)) + goto done; + + stop = (id + 1) & txq->ptr_mask; + id = txq->pending & txq->ptr_mask; + + delta = (stop >= id) ? (stop - id) : (txq->ptr_mask + 1 - id + stop); + + txq->pending += delta; + +done: + return B_FALSE; +} + +static boolean_t +sfc_ev_dp_tx(void *arg, __rte_unused uint32_t label, uint32_t id) +{ + struct sfc_evq *evq = arg; + struct sfc_dp_txq *dp_txq; + + dp_txq = evq->dp_txq; + SFC_ASSERT(dp_txq != NULL); + + SFC_ASSERT(evq->sa->dp_tx->qtx_ev != NULL); + return evq->sa->dp_tx->qtx_ev(dp_txq, id); +} + +static boolean_t +sfc_ev_exception(void *arg, uint32_t code, __rte_unused uint32_t data) +{ + struct sfc_evq *evq = arg; + + if (code == EFX_EXCEPTION_UNKNOWN_SENSOREVT) + return B_FALSE; + + evq->exception = B_TRUE; + sfc_warn(evq->sa, + "hardware exception %s (code=%u, data=%#x) on EVQ %u;" + " needs recovery", + (code == EFX_EXCEPTION_RX_RECOVERY) ? "RX_RECOVERY" : + (code == EFX_EXCEPTION_RX_DSC_ERROR) ? "RX_DSC_ERROR" : + (code == EFX_EXCEPTION_TX_DSC_ERROR) ? "TX_DSC_ERROR" : + (code == EFX_EXCEPTION_FWALERT_SRAM) ? "FWALERT_SRAM" : + (code == EFX_EXCEPTION_UNKNOWN_FWALERT) ? "UNKNOWN_FWALERT" : + (code == EFX_EXCEPTION_RX_ERROR) ? "RX_ERROR" : + (code == EFX_EXCEPTION_TX_ERROR) ? "TX_ERROR" : + (code == EFX_EXCEPTION_EV_ERROR) ? "EV_ERROR" : + "UNKNOWN", + code, data, evq->evq_index); + + return B_TRUE; +} + +static boolean_t +sfc_ev_nop_rxq_flush_done(void *arg, uint32_t rxq_hw_index) +{ + struct sfc_evq *evq = arg; + + sfc_err(evq->sa, "EVQ %u unexpected RxQ %u flush done", + evq->evq_index, rxq_hw_index); + return B_TRUE; +} + +static boolean_t +sfc_ev_rxq_flush_done(void *arg, __rte_unused uint32_t rxq_hw_index) +{ + struct sfc_evq *evq = arg; + struct sfc_dp_rxq *dp_rxq; + struct sfc_rxq *rxq; + + dp_rxq = evq->dp_rxq; + SFC_ASSERT(dp_rxq != NULL); + + rxq = sfc_rxq_by_dp_rxq(dp_rxq); + SFC_ASSERT(rxq != NULL); + SFC_ASSERT(rxq->hw_index == rxq_hw_index); + SFC_ASSERT(rxq->evq == evq); + sfc_rx_qflush_done(rxq); + + return B_FALSE; +} + +static boolean_t +sfc_ev_nop_rxq_flush_failed(void *arg, uint32_t rxq_hw_index) +{ + struct sfc_evq *evq = arg; + + sfc_err(evq->sa, "EVQ %u unexpected RxQ %u flush failed", + evq->evq_index, rxq_hw_index); + return B_TRUE; +} + +static boolean_t +sfc_ev_rxq_flush_failed(void *arg, __rte_unused uint32_t rxq_hw_index) +{ + struct sfc_evq *evq = arg; + struct sfc_dp_rxq *dp_rxq; + struct sfc_rxq *rxq; + + dp_rxq = evq->dp_rxq; + SFC_ASSERT(dp_rxq != NULL); + + rxq = sfc_rxq_by_dp_rxq(dp_rxq); + SFC_ASSERT(rxq != NULL); + SFC_ASSERT(rxq->hw_index == rxq_hw_index); + SFC_ASSERT(rxq->evq == evq); + sfc_rx_qflush_failed(rxq); + + return B_FALSE; +} + +static boolean_t +sfc_ev_nop_txq_flush_done(void *arg, uint32_t txq_hw_index) +{ + struct sfc_evq *evq = arg; + + sfc_err(evq->sa, "EVQ %u unexpected TxQ %u flush done", + evq->evq_index, txq_hw_index); + return B_TRUE; +} + +static boolean_t +sfc_ev_txq_flush_done(void *arg, __rte_unused uint32_t txq_hw_index) +{ + struct sfc_evq *evq = arg; + struct sfc_dp_txq *dp_txq; + struct sfc_txq *txq; + + dp_txq = evq->dp_txq; + SFC_ASSERT(dp_txq != NULL); + + txq = sfc_txq_by_dp_txq(dp_txq); + SFC_ASSERT(txq != NULL); + SFC_ASSERT(txq->hw_index == txq_hw_index); + SFC_ASSERT(txq->evq == evq); + sfc_tx_qflush_done(txq); + + return B_FALSE; +} + +static boolean_t +sfc_ev_software(void *arg, uint16_t magic) +{ + struct sfc_evq *evq = arg; + + sfc_err(evq->sa, "EVQ %u unexpected software event magic=%#.4x", + evq->evq_index, magic); + return B_TRUE; +} + +static boolean_t +sfc_ev_sram(void *arg, uint32_t code) +{ + struct sfc_evq *evq = arg; + + sfc_err(evq->sa, "EVQ %u unexpected SRAM event code=%u", + evq->evq_index, code); + return B_TRUE; +} + +static boolean_t +sfc_ev_wake_up(void *arg, uint32_t index) +{ + struct sfc_evq *evq = arg; + + sfc_err(evq->sa, "EVQ %u unexpected wake up event index=%u", + evq->evq_index, index); + return B_TRUE; +} + +static boolean_t +sfc_ev_timer(void *arg, uint32_t index) +{ + struct sfc_evq *evq = arg; + + sfc_err(evq->sa, "EVQ %u unexpected timer event index=%u", + evq->evq_index, index); + return B_TRUE; +} + +static boolean_t +sfc_ev_nop_link_change(void *arg, __rte_unused efx_link_mode_t link_mode) +{ + struct sfc_evq *evq = arg; + + sfc_err(evq->sa, "EVQ %u unexpected link change event", + evq->evq_index); + return B_TRUE; +} + +static boolean_t +sfc_ev_link_change(void *arg, efx_link_mode_t link_mode) +{ + struct sfc_evq *evq = arg; + struct sfc_adapter *sa = evq->sa; + struct rte_eth_link *dev_link = &sa->eth_dev->data->dev_link; + struct rte_eth_link new_link; + uint64_t new_link_u64; + uint64_t old_link_u64; + + EFX_STATIC_ASSERT(sizeof(*dev_link) == sizeof(rte_atomic64_t)); + + sfc_port_link_mode_to_info(link_mode, &new_link); + + new_link_u64 = *(uint64_t *)&new_link; + do { + old_link_u64 = rte_atomic64_read((rte_atomic64_t *)dev_link); + if (old_link_u64 == new_link_u64) + break; + + if (rte_atomic64_cmpset((volatile uint64_t *)dev_link, + old_link_u64, new_link_u64)) { + evq->sa->port.lsc_seq++; + break; + } + } while (B_TRUE); + + return B_FALSE; +} + +static const efx_ev_callbacks_t sfc_ev_callbacks = { + .eec_initialized = sfc_ev_initialized, + .eec_rx = sfc_ev_nop_rx, + .eec_tx = sfc_ev_nop_tx, + .eec_exception = sfc_ev_exception, + .eec_rxq_flush_done = sfc_ev_nop_rxq_flush_done, + .eec_rxq_flush_failed = sfc_ev_nop_rxq_flush_failed, + .eec_txq_flush_done = sfc_ev_nop_txq_flush_done, + .eec_software = sfc_ev_software, + .eec_sram = sfc_ev_sram, + .eec_wake_up = sfc_ev_wake_up, + .eec_timer = sfc_ev_timer, + .eec_link_change = sfc_ev_link_change, +}; + +static const efx_ev_callbacks_t sfc_ev_callbacks_efx_rx = { + .eec_initialized = sfc_ev_initialized, + .eec_rx = sfc_ev_efx_rx, + .eec_tx = sfc_ev_nop_tx, + .eec_exception = sfc_ev_exception, + .eec_rxq_flush_done = sfc_ev_rxq_flush_done, + .eec_rxq_flush_failed = sfc_ev_rxq_flush_failed, + .eec_txq_flush_done = sfc_ev_nop_txq_flush_done, + .eec_software = sfc_ev_software, + .eec_sram = sfc_ev_sram, + .eec_wake_up = sfc_ev_wake_up, + .eec_timer = sfc_ev_timer, + .eec_link_change = sfc_ev_nop_link_change, +}; + +static const efx_ev_callbacks_t sfc_ev_callbacks_dp_rx = { + .eec_initialized = sfc_ev_initialized, + .eec_rx = sfc_ev_dp_rx, + .eec_tx = sfc_ev_nop_tx, + .eec_exception = sfc_ev_exception, + .eec_rxq_flush_done = sfc_ev_rxq_flush_done, + .eec_rxq_flush_failed = sfc_ev_rxq_flush_failed, + .eec_txq_flush_done = sfc_ev_nop_txq_flush_done, + .eec_software = sfc_ev_software, + .eec_sram = sfc_ev_sram, + .eec_wake_up = sfc_ev_wake_up, + .eec_timer = sfc_ev_timer, + .eec_link_change = sfc_ev_nop_link_change, +}; + +static const efx_ev_callbacks_t sfc_ev_callbacks_efx_tx = { + .eec_initialized = sfc_ev_initialized, + .eec_rx = sfc_ev_nop_rx, + .eec_tx = sfc_ev_tx, + .eec_exception = sfc_ev_exception, + .eec_rxq_flush_done = sfc_ev_nop_rxq_flush_done, + .eec_rxq_flush_failed = sfc_ev_nop_rxq_flush_failed, + .eec_txq_flush_done = sfc_ev_txq_flush_done, + .eec_software = sfc_ev_software, + .eec_sram = sfc_ev_sram, + .eec_wake_up = sfc_ev_wake_up, + .eec_timer = sfc_ev_timer, + .eec_link_change = sfc_ev_nop_link_change, +}; + +static const efx_ev_callbacks_t sfc_ev_callbacks_dp_tx = { + .eec_initialized = sfc_ev_initialized, + .eec_rx = sfc_ev_nop_rx, + .eec_tx = sfc_ev_dp_tx, + .eec_exception = sfc_ev_exception, + .eec_rxq_flush_done = sfc_ev_nop_rxq_flush_done, + .eec_rxq_flush_failed = sfc_ev_nop_rxq_flush_failed, + .eec_txq_flush_done = sfc_ev_txq_flush_done, + .eec_software = sfc_ev_software, + .eec_sram = sfc_ev_sram, + .eec_wake_up = sfc_ev_wake_up, + .eec_timer = sfc_ev_timer, + .eec_link_change = sfc_ev_nop_link_change, +}; + + +void +sfc_ev_qpoll(struct sfc_evq *evq) +{ + SFC_ASSERT(evq->init_state == SFC_EVQ_STARTED || + evq->init_state == SFC_EVQ_STARTING); + + /* Synchronize the DMA memory for reading not required */ + + efx_ev_qpoll(evq->common, &evq->read_ptr, evq->callbacks, evq); + + if (unlikely(evq->exception) && sfc_adapter_trylock(evq->sa)) { + struct sfc_adapter *sa = evq->sa; + int rc; + + if (evq->dp_rxq != NULL) { + unsigned int rxq_sw_index; + + rxq_sw_index = evq->dp_rxq->dpq.queue_id; + + sfc_warn(sa, + "restart RxQ %u because of exception on its EvQ %u", + rxq_sw_index, evq->evq_index); + + sfc_rx_qstop(sa, rxq_sw_index); + rc = sfc_rx_qstart(sa, rxq_sw_index); + if (rc != 0) + sfc_err(sa, "cannot restart RxQ %u", + rxq_sw_index); + } + + if (evq->dp_txq != NULL) { + unsigned int txq_sw_index; + + txq_sw_index = evq->dp_txq->dpq.queue_id; + + sfc_warn(sa, + "restart TxQ %u because of exception on its EvQ %u", + txq_sw_index, evq->evq_index); + + sfc_tx_qstop(sa, txq_sw_index); + rc = sfc_tx_qstart(sa, txq_sw_index); + if (rc != 0) + sfc_err(sa, "cannot restart TxQ %u", + txq_sw_index); + } + + if (evq->exception) + sfc_panic(sa, "unrecoverable exception on EvQ %u", + evq->evq_index); + + sfc_adapter_unlock(sa); + } + + /* Poll-mode driver does not re-prime the event queue for interrupts */ +} + +void +sfc_ev_mgmt_qpoll(struct sfc_adapter *sa) +{ + if (rte_spinlock_trylock(&sa->mgmt_evq_lock)) { + if (sa->mgmt_evq_running) + sfc_ev_qpoll(sa->mgmt_evq); + + rte_spinlock_unlock(&sa->mgmt_evq_lock); + } +} + +int +sfc_ev_qprime(struct sfc_evq *evq) +{ + SFC_ASSERT(evq->init_state == SFC_EVQ_STARTED); + return efx_ev_qprime(evq->common, evq->read_ptr); +} + +/* Event queue HW index allocation scheme is described in sfc_ev.h. */ +int +sfc_ev_qstart(struct sfc_evq *evq, unsigned int hw_index) +{ + struct sfc_adapter *sa = evq->sa; + efsys_mem_t *esmp; + uint32_t evq_flags = sa->evq_flags; + unsigned int total_delay_us; + unsigned int delay_us; + int rc; + + sfc_log_init(sa, "hw_index=%u", hw_index); + + esmp = &evq->mem; + + evq->evq_index = hw_index; + + /* Clear all events */ + (void)memset((void *)esmp->esm_base, 0xff, EFX_EVQ_SIZE(evq->entries)); + + if (sa->intr.lsc_intr && hw_index == sa->mgmt_evq_index) + evq_flags |= EFX_EVQ_FLAGS_NOTIFY_INTERRUPT; + else + evq_flags |= EFX_EVQ_FLAGS_NOTIFY_DISABLED; + + /* Create the common code event queue */ + rc = efx_ev_qcreate(sa->nic, hw_index, esmp, evq->entries, + 0 /* unused on EF10 */, 0, evq_flags, + &evq->common); + if (rc != 0) + goto fail_ev_qcreate; + + SFC_ASSERT(evq->dp_rxq == NULL || evq->dp_txq == NULL); + if (evq->dp_rxq != 0) { + if (strcmp(sa->dp_rx->dp.name, SFC_KVARG_DATAPATH_EFX) == 0) + evq->callbacks = &sfc_ev_callbacks_efx_rx; + else + evq->callbacks = &sfc_ev_callbacks_dp_rx; + } else if (evq->dp_txq != 0) { + if (strcmp(sa->dp_tx->dp.name, SFC_KVARG_DATAPATH_EFX) == 0) + evq->callbacks = &sfc_ev_callbacks_efx_tx; + else + evq->callbacks = &sfc_ev_callbacks_dp_tx; + } else { + evq->callbacks = &sfc_ev_callbacks; + } + + evq->init_state = SFC_EVQ_STARTING; + + /* Wait for the initialization event */ + total_delay_us = 0; + delay_us = SFC_EVQ_INIT_BACKOFF_START_US; + do { + (void)sfc_ev_qpoll(evq); + + /* Check to see if the initialization complete indication + * posted by the hardware. + */ + if (evq->init_state == SFC_EVQ_STARTED) + goto done; + + /* Give event queue some time to init */ + rte_delay_us(delay_us); + + total_delay_us += delay_us; + + /* Exponential backoff */ + delay_us *= 2; + if (delay_us > SFC_EVQ_INIT_BACKOFF_MAX_US) + delay_us = SFC_EVQ_INIT_BACKOFF_MAX_US; + + } while (total_delay_us < SFC_EVQ_INIT_TIMEOUT_US); + + rc = ETIMEDOUT; + goto fail_timedout; + +done: + return 0; + +fail_timedout: + evq->init_state = SFC_EVQ_INITIALIZED; + efx_ev_qdestroy(evq->common); + +fail_ev_qcreate: + sfc_log_init(sa, "failed %d", rc); + return rc; +} + +void +sfc_ev_qstop(struct sfc_evq *evq) +{ + if (evq == NULL) + return; + + sfc_log_init(evq->sa, "hw_index=%u", evq->evq_index); + + if (evq->init_state != SFC_EVQ_STARTED) + return; + + evq->init_state = SFC_EVQ_INITIALIZED; + evq->callbacks = NULL; + evq->read_ptr = 0; + evq->exception = B_FALSE; + + efx_ev_qdestroy(evq->common); + + evq->evq_index = 0; +} + +static void +sfc_ev_mgmt_periodic_qpoll(void *arg) +{ + struct sfc_adapter *sa = arg; + int rc; + + sfc_ev_mgmt_qpoll(sa); + + rc = rte_eal_alarm_set(SFC_MGMT_EV_QPOLL_PERIOD_US, + sfc_ev_mgmt_periodic_qpoll, sa); + if (rc == -ENOTSUP) { + sfc_warn(sa, "alarms are not supported"); + sfc_warn(sa, "management EVQ must be polled indirectly using no-wait link status update"); + } else if (rc != 0) { + sfc_err(sa, + "cannot rearm management EVQ polling alarm (rc=%d)", + rc); + } +} + +static void +sfc_ev_mgmt_periodic_qpoll_start(struct sfc_adapter *sa) +{ + sfc_ev_mgmt_periodic_qpoll(sa); +} + +static void +sfc_ev_mgmt_periodic_qpoll_stop(struct sfc_adapter *sa) +{ + rte_eal_alarm_cancel(sfc_ev_mgmt_periodic_qpoll, sa); +} + +int +sfc_ev_start(struct sfc_adapter *sa) +{ + int rc; + + sfc_log_init(sa, "entry"); + + rc = efx_ev_init(sa->nic); + if (rc != 0) + goto fail_ev_init; + + /* Start management EVQ used for global events */ + + /* + * Management event queue start polls the queue, but it cannot + * interfere with other polling contexts since mgmt_evq_running + * is false yet. + */ + rc = sfc_ev_qstart(sa->mgmt_evq, sa->mgmt_evq_index); + if (rc != 0) + goto fail_mgmt_evq_start; + + rte_spinlock_lock(&sa->mgmt_evq_lock); + sa->mgmt_evq_running = true; + rte_spinlock_unlock(&sa->mgmt_evq_lock); + + if (sa->intr.lsc_intr) { + rc = sfc_ev_qprime(sa->mgmt_evq); + if (rc != 0) + goto fail_mgmt_evq_prime; + } + + /* + * Start management EVQ polling. If interrupts are disabled + * (not used), it is required to process link status change + * and other device level events to avoid unrecoverable + * error because the event queue overflow. + */ + sfc_ev_mgmt_periodic_qpoll_start(sa); + + /* + * Rx/Tx event queues are started/stopped when corresponding + * Rx/Tx queue is started/stopped. + */ + + return 0; + +fail_mgmt_evq_prime: + sfc_ev_qstop(sa->mgmt_evq); + +fail_mgmt_evq_start: + efx_ev_fini(sa->nic); + +fail_ev_init: + sfc_log_init(sa, "failed %d", rc); + return rc; +} + +void +sfc_ev_stop(struct sfc_adapter *sa) +{ + sfc_log_init(sa, "entry"); + + sfc_ev_mgmt_periodic_qpoll_stop(sa); + + rte_spinlock_lock(&sa->mgmt_evq_lock); + sa->mgmt_evq_running = false; + rte_spinlock_unlock(&sa->mgmt_evq_lock); + + sfc_ev_qstop(sa->mgmt_evq); + + efx_ev_fini(sa->nic); +} + +int +sfc_ev_qinit(struct sfc_adapter *sa, + enum sfc_evq_type type, unsigned int type_index, + unsigned int entries, int socket_id, struct sfc_evq **evqp) +{ + struct sfc_evq *evq; + int rc; + + sfc_log_init(sa, "type=%s type_index=%u", + sfc_evq_type2str(type), type_index); + + SFC_ASSERT(rte_is_power_of_2(entries)); + + rc = ENOMEM; + evq = rte_zmalloc_socket("sfc-evq", sizeof(*evq), RTE_CACHE_LINE_SIZE, + socket_id); + if (evq == NULL) + goto fail_evq_alloc; + + evq->sa = sa; + evq->type = type; + evq->entries = entries; + + /* Allocate DMA space */ + rc = sfc_dma_alloc(sa, sfc_evq_type2str(type), type_index, + EFX_EVQ_SIZE(evq->entries), socket_id, &evq->mem); + if (rc != 0) + goto fail_dma_alloc; + + evq->init_state = SFC_EVQ_INITIALIZED; + + sa->evq_count++; + + *evqp = evq; + + return 0; + +fail_dma_alloc: + rte_free(evq); + +fail_evq_alloc: + + sfc_log_init(sa, "failed %d", rc); + return rc; +} + +void +sfc_ev_qfini(struct sfc_evq *evq) +{ + struct sfc_adapter *sa = evq->sa; + + SFC_ASSERT(evq->init_state == SFC_EVQ_INITIALIZED); + + sfc_dma_free(sa, &evq->mem); + + rte_free(evq); + + SFC_ASSERT(sa->evq_count > 0); + sa->evq_count--; +} + +static int +sfc_kvarg_perf_profile_handler(__rte_unused const char *key, + const char *value_str, void *opaque) +{ + uint64_t *value = opaque; + + if (strcasecmp(value_str, SFC_KVARG_PERF_PROFILE_THROUGHPUT) == 0) + *value = EFX_EVQ_FLAGS_TYPE_THROUGHPUT; + else if (strcasecmp(value_str, SFC_KVARG_PERF_PROFILE_LOW_LATENCY) == 0) + *value = EFX_EVQ_FLAGS_TYPE_LOW_LATENCY; + else if (strcasecmp(value_str, SFC_KVARG_PERF_PROFILE_AUTO) == 0) + *value = EFX_EVQ_FLAGS_TYPE_AUTO; + else + return -EINVAL; + + return 0; +} + +int +sfc_ev_attach(struct sfc_adapter *sa) +{ + int rc; + + sfc_log_init(sa, "entry"); + + sa->evq_flags = EFX_EVQ_FLAGS_TYPE_THROUGHPUT; + rc = sfc_kvargs_process(sa, SFC_KVARG_PERF_PROFILE, + sfc_kvarg_perf_profile_handler, + &sa->evq_flags); + if (rc != 0) { + sfc_err(sa, "invalid %s parameter value", + SFC_KVARG_PERF_PROFILE); + goto fail_kvarg_perf_profile; + } + + sa->mgmt_evq_index = 0; + rte_spinlock_init(&sa->mgmt_evq_lock); + + rc = sfc_ev_qinit(sa, SFC_EVQ_TYPE_MGMT, 0, SFC_MGMT_EVQ_ENTRIES, + sa->socket_id, &sa->mgmt_evq); + if (rc != 0) + goto fail_mgmt_evq_init; + + /* + * Rx/Tx event queues are created/destroyed when corresponding + * Rx/Tx queue is created/destroyed. + */ + + return 0; + +fail_mgmt_evq_init: + +fail_kvarg_perf_profile: + sfc_log_init(sa, "failed %d", rc); + return rc; +} + +void +sfc_ev_detach(struct sfc_adapter *sa) +{ + sfc_log_init(sa, "entry"); + + sfc_ev_qfini(sa->mgmt_evq); + + if (sa->evq_count != 0) + sfc_err(sa, "%u EvQs are not destroyed before detach", + sa->evq_count); +} diff --git a/dpdk/drivers/net/sfc/sfc_ev.h b/dpdk/drivers/net/sfc/sfc_ev.h new file mode 100644 index 00000000..065defe0 --- /dev/null +++ b/dpdk/drivers/net/sfc/sfc_ev.h @@ -0,0 +1,129 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2016-2017 Solarflare Communications Inc. + * All rights reserved. + * + * This software was jointly developed between OKTET Labs (under contract + * for Solarflare) and Solarflare Communications, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _SFC_EV_H_ +#define _SFC_EV_H_ + +#include + +#include "efx.h" + +#include "sfc.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* Number of entries in the management event queue */ +#define SFC_MGMT_EVQ_ENTRIES (EFX_EVQ_MINNEVS) + +struct sfc_adapter; +struct sfc_dp_rxq; +struct sfc_dp_txq; + +enum sfc_evq_state { + SFC_EVQ_UNINITIALIZED = 0, + SFC_EVQ_INITIALIZED, + SFC_EVQ_STARTING, + SFC_EVQ_STARTED, + + SFC_EVQ_NSTATES +}; + +enum sfc_evq_type { + SFC_EVQ_TYPE_MGMT = 0, + SFC_EVQ_TYPE_RX, + SFC_EVQ_TYPE_TX, + + SFC_EVQ_NTYPES +}; + +struct sfc_evq { + /* Used on datapath */ + efx_evq_t *common; + const efx_ev_callbacks_t *callbacks; + unsigned int read_ptr; + boolean_t exception; + efsys_mem_t mem; + struct sfc_dp_rxq *dp_rxq; + struct sfc_dp_txq *dp_txq; + + /* Not used on datapath */ + struct sfc_adapter *sa; + unsigned int evq_index; + enum sfc_evq_state init_state; + enum sfc_evq_type type; + unsigned int entries; +}; + +/* + * Functions below define event queue to transmit/receive queue and vice + * versa mapping. + * Own event queue is allocated for management, each Rx and each Tx queue. + * Zero event queue is used for management events. + * Rx event queues from 1 to RxQ number follow management event queue. + * Tx event queues follow Rx event queues. + */ + +static inline unsigned int +sfc_evq_index_by_rxq_sw_index(__rte_unused struct sfc_adapter *sa, + unsigned int rxq_sw_index) +{ + return 1 + rxq_sw_index; +} + +static inline unsigned int +sfc_evq_index_by_txq_sw_index(struct sfc_adapter *sa, unsigned int txq_sw_index) +{ + return 1 + sa->eth_dev->data->nb_rx_queues + txq_sw_index; +} + +int sfc_ev_attach(struct sfc_adapter *sa); +void sfc_ev_detach(struct sfc_adapter *sa); +int sfc_ev_start(struct sfc_adapter *sa); +void sfc_ev_stop(struct sfc_adapter *sa); + +int sfc_ev_qinit(struct sfc_adapter *sa, + enum sfc_evq_type type, unsigned int type_index, + unsigned int entries, int socket_id, struct sfc_evq **evqp); +void sfc_ev_qfini(struct sfc_evq *evq); +int sfc_ev_qstart(struct sfc_evq *evq, unsigned int hw_index); +void sfc_ev_qstop(struct sfc_evq *evq); + +int sfc_ev_qprime(struct sfc_evq *evq); +void sfc_ev_qpoll(struct sfc_evq *evq); + +void sfc_ev_mgmt_qpoll(struct sfc_adapter *sa); + +#ifdef __cplusplus +} +#endif +#endif /* _SFC_EV_H_ */ diff --git a/dpdk/drivers/net/sfc/sfc_filter.c b/dpdk/drivers/net/sfc/sfc_filter.c new file mode 100644 index 00000000..58b74de7 --- /dev/null +++ b/dpdk/drivers/net/sfc/sfc_filter.c @@ -0,0 +1,137 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2017 Solarflare Communications Inc. + * All rights reserved. + * + * This software was jointly developed between OKTET Labs (under contract + * for Solarflare) and Solarflare Communications, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +#include "efx.h" + +#include "sfc.h" +#include "sfc_log.h" + +boolean_t +sfc_filter_is_match_supported(struct sfc_adapter *sa, uint32_t match) +{ + struct sfc_filter *filter = &sa->filter; + size_t i; + + for (i = 0; i < filter->supported_match_num; ++i) { + if (match == filter->supported_match[i]) + return B_TRUE; + } + + return B_FALSE; +} + +static int +sfc_filter_cache_match_supported(struct sfc_adapter *sa) +{ + struct sfc_filter *filter = &sa->filter; + size_t num = filter->supported_match_num; + uint32_t *buf = filter->supported_match; + unsigned int retry; + int rc; + + /* Just a guess of possibly sufficient entries */ + if (num == 0) + num = 16; + + for (retry = 0; retry < 2; ++retry) { + if (num != filter->supported_match_num) { + rc = ENOMEM; + buf = rte_realloc(buf, num * sizeof(*buf), 0); + if (buf == NULL) + goto fail_realloc; + } + + rc = efx_filter_supported_filters(sa->nic, buf, num, &num); + if (rc == 0) { + filter->supported_match_num = num; + filter->supported_match = buf; + + return 0; + } else if (rc != ENOSPC) { + goto fail_efx_filter_supported_filters; + } + } + + SFC_ASSERT(rc == ENOSPC); + +fail_efx_filter_supported_filters: +fail_realloc: + /* Original pointer is not freed by rte_realloc() on failure */ + rte_free(buf); + filter->supported_match = NULL; + filter->supported_match_num = 0; + return rc; +} + +int +sfc_filter_attach(struct sfc_adapter *sa) +{ + int rc; + + sfc_log_init(sa, "entry"); + + rc = efx_filter_init(sa->nic); + if (rc != 0) + goto fail_filter_init; + + rc = sfc_filter_cache_match_supported(sa); + if (rc != 0) + goto fail_cache_match_supported; + + efx_filter_fini(sa->nic); + + sfc_log_init(sa, "done"); + + return 0; + +fail_cache_match_supported: + efx_filter_fini(sa->nic); + +fail_filter_init: + sfc_log_init(sa, "failed %d", rc); + return rc; +} + +void +sfc_filter_detach(struct sfc_adapter *sa) +{ + struct sfc_filter *filter = &sa->filter; + + sfc_log_init(sa, "entry"); + + rte_free(filter->supported_match); + filter->supported_match = NULL; + filter->supported_match_num = 0; + + sfc_log_init(sa, "done"); +} diff --git a/dpdk/drivers/net/sfc/sfc_filter.h b/dpdk/drivers/net/sfc/sfc_filter.h new file mode 100644 index 00000000..d884f37d --- /dev/null +++ b/dpdk/drivers/net/sfc/sfc_filter.h @@ -0,0 +1,62 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2017 Solarflare Communications Inc. + * All rights reserved. + * + * This software was jointly developed between OKTET Labs (under contract + * for Solarflare) and Solarflare Communications, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _SFC_FILTER_H +#define _SFC_FILTER_H + +#include "efx.h" + +#include "sfc_flow.h" + +#ifdef __cplusplus +extern "C" { +#endif + +struct sfc_filter { + /** Number of elements in match_supported array */ + size_t supported_match_num; + /** Driver cache of supported filter match masks */ + uint32_t *supported_match; + /** List of flow rules */ + struct sfc_flow_list flow_list; +}; + +struct sfc_adapter; + +int sfc_filter_attach(struct sfc_adapter *sa); +void sfc_filter_detach(struct sfc_adapter *sa); + +boolean_t sfc_filter_is_match_supported(struct sfc_adapter *sa, uint32_t match); + +#ifdef __cplusplus +} +#endif +#endif /* _SFC_FILTER_H */ diff --git a/dpdk/drivers/net/sfc/sfc_flow.c b/dpdk/drivers/net/sfc/sfc_flow.c new file mode 100644 index 00000000..e770b98e --- /dev/null +++ b/dpdk/drivers/net/sfc/sfc_flow.c @@ -0,0 +1,1376 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2017 Solarflare Communications Inc. + * All rights reserved. + * + * This software was jointly developed between OKTET Labs (under contract + * for Solarflare) and Solarflare Communications, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "efx.h" + +#include "sfc.h" +#include "sfc_rx.h" +#include "sfc_filter.h" +#include "sfc_flow.h" +#include "sfc_log.h" + +/* + * At now flow API is implemented in such a manner that each + * flow rule is converted to a hardware filter. + * All elements of flow rule (attributes, pattern items, actions) + * correspond to one or more fields in the efx_filter_spec_s structure + * that is responsible for the hardware filter. + */ + +enum sfc_flow_item_layers { + SFC_FLOW_ITEM_ANY_LAYER, + SFC_FLOW_ITEM_START_LAYER, + SFC_FLOW_ITEM_L2, + SFC_FLOW_ITEM_L3, + SFC_FLOW_ITEM_L4, +}; + +typedef int (sfc_flow_item_parse)(const struct rte_flow_item *item, + efx_filter_spec_t *spec, + struct rte_flow_error *error); + +struct sfc_flow_item { + enum rte_flow_item_type type; /* Type of item */ + enum sfc_flow_item_layers layer; /* Layer of item */ + enum sfc_flow_item_layers prev_layer; /* Previous layer of item */ + sfc_flow_item_parse *parse; /* Parsing function */ +}; + +static sfc_flow_item_parse sfc_flow_parse_void; +static sfc_flow_item_parse sfc_flow_parse_eth; +static sfc_flow_item_parse sfc_flow_parse_vlan; +static sfc_flow_item_parse sfc_flow_parse_ipv4; +static sfc_flow_item_parse sfc_flow_parse_ipv6; +static sfc_flow_item_parse sfc_flow_parse_tcp; +static sfc_flow_item_parse sfc_flow_parse_udp; + +static boolean_t +sfc_flow_is_zero(const uint8_t *buf, unsigned int size) +{ + uint8_t sum = 0; + unsigned int i; + + for (i = 0; i < size; i++) + sum |= buf[i]; + + return (sum == 0) ? B_TRUE : B_FALSE; +} + +/* + * Validate item and prepare structures spec and mask for parsing + */ +static int +sfc_flow_parse_init(const struct rte_flow_item *item, + const void **spec_ptr, + const void **mask_ptr, + const void *supp_mask, + const void *def_mask, + unsigned int size, + struct rte_flow_error *error) +{ + const uint8_t *spec; + const uint8_t *mask; + const uint8_t *last; + uint8_t match; + uint8_t supp; + unsigned int i; + + if (item == NULL) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, NULL, + "NULL item"); + return -rte_errno; + } + + if ((item->last != NULL || item->mask != NULL) && item->spec == NULL) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, item, + "Mask or last is set without spec"); + return -rte_errno; + } + + /* + * If "mask" is not set, default mask is used, + * but if default mask is NULL, "mask" should be set + */ + if (item->mask == NULL) { + if (def_mask == NULL) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, NULL, + "Mask should be specified"); + return -rte_errno; + } + + mask = (const uint8_t *)def_mask; + } else { + mask = (const uint8_t *)item->mask; + } + + spec = (const uint8_t *)item->spec; + last = (const uint8_t *)item->last; + + if (spec == NULL) + goto exit; + + /* + * If field values in "last" are either 0 or equal to the corresponding + * values in "spec" then they are ignored + */ + if (last != NULL && + !sfc_flow_is_zero(last, size) && + memcmp(last, spec, size) != 0) { + rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ITEM, item, + "Ranging is not supported"); + return -rte_errno; + } + + if (supp_mask == NULL) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "Supported mask for item should be specified"); + return -rte_errno; + } + + /* Check that mask and spec not asks for more match than supp_mask */ + for (i = 0; i < size; i++) { + match = spec[i] | mask[i]; + supp = ((const uint8_t *)supp_mask)[i]; + + if ((match | supp) != supp) { + rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ITEM, item, + "Item's field is not supported"); + return -rte_errno; + } + } + +exit: + *spec_ptr = spec; + *mask_ptr = mask; + return 0; +} + +/* + * Protocol parsers. + * Masking is not supported, so masks in items should be either + * full or empty (zeroed) and set only for supported fields which + * are specified in the supp_mask. + */ + +static int +sfc_flow_parse_void(__rte_unused const struct rte_flow_item *item, + __rte_unused efx_filter_spec_t *efx_spec, + __rte_unused struct rte_flow_error *error) +{ + return 0; +} + +/** + * Convert Ethernet item to EFX filter specification. + * + * @param item[in] + * Item specification. Only source and destination addresses and + * Ethernet type fields are supported. In addition to full and + * empty masks of destination address, individual/group mask is + * also supported. If the mask is NULL, default mask will be used. + * Ranging is not supported. + * @param efx_spec[in, out] + * EFX filter specification to update. + * @param[out] error + * Perform verbose error reporting if not NULL. + */ +static int +sfc_flow_parse_eth(const struct rte_flow_item *item, + efx_filter_spec_t *efx_spec, + struct rte_flow_error *error) +{ + int rc; + const struct rte_flow_item_eth *spec = NULL; + const struct rte_flow_item_eth *mask = NULL; + const struct rte_flow_item_eth supp_mask = { + .dst.addr_bytes = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .src.addr_bytes = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .type = 0xffff, + }; + const uint8_t ig_mask[EFX_MAC_ADDR_LEN] = { + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00 + }; + + rc = sfc_flow_parse_init(item, + (const void **)&spec, + (const void **)&mask, + &supp_mask, + &rte_flow_item_eth_mask, + sizeof(struct rte_flow_item_eth), + error); + if (rc != 0) + return rc; + + /* If "spec" is not set, could be any Ethernet */ + if (spec == NULL) + return 0; + + if (is_same_ether_addr(&mask->dst, &supp_mask.dst)) { + efx_spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_MAC; + rte_memcpy(efx_spec->efs_loc_mac, spec->dst.addr_bytes, + EFX_MAC_ADDR_LEN); + } else if (memcmp(mask->dst.addr_bytes, ig_mask, + EFX_MAC_ADDR_LEN) == 0) { + if (is_unicast_ether_addr(&spec->dst)) + efx_spec->efs_match_flags |= + EFX_FILTER_MATCH_UNKNOWN_UCAST_DST; + else + efx_spec->efs_match_flags |= + EFX_FILTER_MATCH_UNKNOWN_MCAST_DST; + } else if (!is_zero_ether_addr(&mask->dst)) { + goto fail_bad_mask; + } + + if (is_same_ether_addr(&mask->src, &supp_mask.src)) { + efx_spec->efs_match_flags |= EFX_FILTER_MATCH_REM_MAC; + rte_memcpy(efx_spec->efs_rem_mac, spec->src.addr_bytes, + EFX_MAC_ADDR_LEN); + } else if (!is_zero_ether_addr(&mask->src)) { + goto fail_bad_mask; + } + + /* + * Ether type is in big-endian byte order in item and + * in little-endian in efx_spec, so byte swap is used + */ + if (mask->type == supp_mask.type) { + efx_spec->efs_match_flags |= EFX_FILTER_MATCH_ETHER_TYPE; + efx_spec->efs_ether_type = rte_bswap16(spec->type); + } else if (mask->type != 0) { + goto fail_bad_mask; + } + + return 0; + +fail_bad_mask: + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, item, + "Bad mask in the ETH pattern item"); + return -rte_errno; +} + +/** + * Convert VLAN item to EFX filter specification. + * + * @param item[in] + * Item specification. Only VID field is supported. + * The mask can not be NULL. Ranging is not supported. + * @param efx_spec[in, out] + * EFX filter specification to update. + * @param[out] error + * Perform verbose error reporting if not NULL. + */ +static int +sfc_flow_parse_vlan(const struct rte_flow_item *item, + efx_filter_spec_t *efx_spec, + struct rte_flow_error *error) +{ + int rc; + uint16_t vid; + const struct rte_flow_item_vlan *spec = NULL; + const struct rte_flow_item_vlan *mask = NULL; + const struct rte_flow_item_vlan supp_mask = { + .tci = rte_cpu_to_be_16(ETH_VLAN_ID_MAX), + }; + + rc = sfc_flow_parse_init(item, + (const void **)&spec, + (const void **)&mask, + &supp_mask, + NULL, + sizeof(struct rte_flow_item_vlan), + error); + if (rc != 0) + return rc; + + /* + * VID is in big-endian byte order in item and + * in little-endian in efx_spec, so byte swap is used. + * If two VLAN items are included, the first matches + * the outer tag and the next matches the inner tag. + */ + if (mask->tci == supp_mask.tci) { + vid = rte_bswap16(spec->tci); + + if (!(efx_spec->efs_match_flags & + EFX_FILTER_MATCH_OUTER_VID)) { + efx_spec->efs_match_flags |= EFX_FILTER_MATCH_OUTER_VID; + efx_spec->efs_outer_vid = vid; + } else if (!(efx_spec->efs_match_flags & + EFX_FILTER_MATCH_INNER_VID)) { + efx_spec->efs_match_flags |= EFX_FILTER_MATCH_INNER_VID; + efx_spec->efs_inner_vid = vid; + } else { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, item, + "More than two VLAN items"); + return -rte_errno; + } + } else { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, item, + "VLAN ID in TCI match is required"); + return -rte_errno; + } + + return 0; +} + +/** + * Convert IPv4 item to EFX filter specification. + * + * @param item[in] + * Item specification. Only source and destination addresses and + * protocol fields are supported. If the mask is NULL, default + * mask will be used. Ranging is not supported. + * @param efx_spec[in, out] + * EFX filter specification to update. + * @param[out] error + * Perform verbose error reporting if not NULL. + */ +static int +sfc_flow_parse_ipv4(const struct rte_flow_item *item, + efx_filter_spec_t *efx_spec, + struct rte_flow_error *error) +{ + int rc; + const struct rte_flow_item_ipv4 *spec = NULL; + const struct rte_flow_item_ipv4 *mask = NULL; + const uint16_t ether_type_ipv4 = rte_cpu_to_le_16(EFX_ETHER_TYPE_IPV4); + const struct rte_flow_item_ipv4 supp_mask = { + .hdr = { + .src_addr = 0xffffffff, + .dst_addr = 0xffffffff, + .next_proto_id = 0xff, + } + }; + + rc = sfc_flow_parse_init(item, + (const void **)&spec, + (const void **)&mask, + &supp_mask, + &rte_flow_item_ipv4_mask, + sizeof(struct rte_flow_item_ipv4), + error); + if (rc != 0) + return rc; + + /* + * Filtering by IPv4 source and destination addresses requires + * the appropriate ETHER_TYPE in hardware filters + */ + if (!(efx_spec->efs_match_flags & EFX_FILTER_MATCH_ETHER_TYPE)) { + efx_spec->efs_match_flags |= EFX_FILTER_MATCH_ETHER_TYPE; + efx_spec->efs_ether_type = ether_type_ipv4; + } else if (efx_spec->efs_ether_type != ether_type_ipv4) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, item, + "Ethertype in pattern with IPV4 item should be appropriate"); + return -rte_errno; + } + + if (spec == NULL) + return 0; + + /* + * IPv4 addresses are in big-endian byte order in item and in + * efx_spec + */ + if (mask->hdr.src_addr == supp_mask.hdr.src_addr) { + efx_spec->efs_match_flags |= EFX_FILTER_MATCH_REM_HOST; + efx_spec->efs_rem_host.eo_u32[0] = spec->hdr.src_addr; + } else if (mask->hdr.src_addr != 0) { + goto fail_bad_mask; + } + + if (mask->hdr.dst_addr == supp_mask.hdr.dst_addr) { + efx_spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_HOST; + efx_spec->efs_loc_host.eo_u32[0] = spec->hdr.dst_addr; + } else if (mask->hdr.dst_addr != 0) { + goto fail_bad_mask; + } + + if (mask->hdr.next_proto_id == supp_mask.hdr.next_proto_id) { + efx_spec->efs_match_flags |= EFX_FILTER_MATCH_IP_PROTO; + efx_spec->efs_ip_proto = spec->hdr.next_proto_id; + } else if (mask->hdr.next_proto_id != 0) { + goto fail_bad_mask; + } + + return 0; + +fail_bad_mask: + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, item, + "Bad mask in the IPV4 pattern item"); + return -rte_errno; +} + +/** + * Convert IPv6 item to EFX filter specification. + * + * @param item[in] + * Item specification. Only source and destination addresses and + * next header fields are supported. If the mask is NULL, default + * mask will be used. Ranging is not supported. + * @param efx_spec[in, out] + * EFX filter specification to update. + * @param[out] error + * Perform verbose error reporting if not NULL. + */ +static int +sfc_flow_parse_ipv6(const struct rte_flow_item *item, + efx_filter_spec_t *efx_spec, + struct rte_flow_error *error) +{ + int rc; + const struct rte_flow_item_ipv6 *spec = NULL; + const struct rte_flow_item_ipv6 *mask = NULL; + const uint16_t ether_type_ipv6 = rte_cpu_to_le_16(EFX_ETHER_TYPE_IPV6); + const struct rte_flow_item_ipv6 supp_mask = { + .hdr = { + .src_addr = { 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff }, + .dst_addr = { 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff }, + .proto = 0xff, + } + }; + + rc = sfc_flow_parse_init(item, + (const void **)&spec, + (const void **)&mask, + &supp_mask, + &rte_flow_item_ipv6_mask, + sizeof(struct rte_flow_item_ipv6), + error); + if (rc != 0) + return rc; + + /* + * Filtering by IPv6 source and destination addresses requires + * the appropriate ETHER_TYPE in hardware filters + */ + if (!(efx_spec->efs_match_flags & EFX_FILTER_MATCH_ETHER_TYPE)) { + efx_spec->efs_match_flags |= EFX_FILTER_MATCH_ETHER_TYPE; + efx_spec->efs_ether_type = ether_type_ipv6; + } else if (efx_spec->efs_ether_type != ether_type_ipv6) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, item, + "Ethertype in pattern with IPV6 item should be appropriate"); + return -rte_errno; + } + + if (spec == NULL) + return 0; + + /* + * IPv6 addresses are in big-endian byte order in item and in + * efx_spec + */ + if (memcmp(mask->hdr.src_addr, supp_mask.hdr.src_addr, + sizeof(mask->hdr.src_addr)) == 0) { + efx_spec->efs_match_flags |= EFX_FILTER_MATCH_REM_HOST; + + RTE_BUILD_BUG_ON(sizeof(efx_spec->efs_rem_host) != + sizeof(spec->hdr.src_addr)); + rte_memcpy(&efx_spec->efs_rem_host, spec->hdr.src_addr, + sizeof(efx_spec->efs_rem_host)); + } else if (!sfc_flow_is_zero(mask->hdr.src_addr, + sizeof(mask->hdr.src_addr))) { + goto fail_bad_mask; + } + + if (memcmp(mask->hdr.dst_addr, supp_mask.hdr.dst_addr, + sizeof(mask->hdr.dst_addr)) == 0) { + efx_spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_HOST; + + RTE_BUILD_BUG_ON(sizeof(efx_spec->efs_loc_host) != + sizeof(spec->hdr.dst_addr)); + rte_memcpy(&efx_spec->efs_loc_host, spec->hdr.dst_addr, + sizeof(efx_spec->efs_loc_host)); + } else if (!sfc_flow_is_zero(mask->hdr.dst_addr, + sizeof(mask->hdr.dst_addr))) { + goto fail_bad_mask; + } + + if (mask->hdr.proto == supp_mask.hdr.proto) { + efx_spec->efs_match_flags |= EFX_FILTER_MATCH_IP_PROTO; + efx_spec->efs_ip_proto = spec->hdr.proto; + } else if (mask->hdr.proto != 0) { + goto fail_bad_mask; + } + + return 0; + +fail_bad_mask: + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, item, + "Bad mask in the IPV6 pattern item"); + return -rte_errno; +} + +/** + * Convert TCP item to EFX filter specification. + * + * @param item[in] + * Item specification. Only source and destination ports fields + * are supported. If the mask is NULL, default mask will be used. + * Ranging is not supported. + * @param efx_spec[in, out] + * EFX filter specification to update. + * @param[out] error + * Perform verbose error reporting if not NULL. + */ +static int +sfc_flow_parse_tcp(const struct rte_flow_item *item, + efx_filter_spec_t *efx_spec, + struct rte_flow_error *error) +{ + int rc; + const struct rte_flow_item_tcp *spec = NULL; + const struct rte_flow_item_tcp *mask = NULL; + const struct rte_flow_item_tcp supp_mask = { + .hdr = { + .src_port = 0xffff, + .dst_port = 0xffff, + } + }; + + rc = sfc_flow_parse_init(item, + (const void **)&spec, + (const void **)&mask, + &supp_mask, + &rte_flow_item_tcp_mask, + sizeof(struct rte_flow_item_tcp), + error); + if (rc != 0) + return rc; + + /* + * Filtering by TCP source and destination ports requires + * the appropriate IP_PROTO in hardware filters + */ + if (!(efx_spec->efs_match_flags & EFX_FILTER_MATCH_IP_PROTO)) { + efx_spec->efs_match_flags |= EFX_FILTER_MATCH_IP_PROTO; + efx_spec->efs_ip_proto = EFX_IPPROTO_TCP; + } else if (efx_spec->efs_ip_proto != EFX_IPPROTO_TCP) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, item, + "IP proto in pattern with TCP item should be appropriate"); + return -rte_errno; + } + + if (spec == NULL) + return 0; + + /* + * Source and destination ports are in big-endian byte order in item and + * in little-endian in efx_spec, so byte swap is used + */ + if (mask->hdr.src_port == supp_mask.hdr.src_port) { + efx_spec->efs_match_flags |= EFX_FILTER_MATCH_REM_PORT; + efx_spec->efs_rem_port = rte_bswap16(spec->hdr.src_port); + } else if (mask->hdr.src_port != 0) { + goto fail_bad_mask; + } + + if (mask->hdr.dst_port == supp_mask.hdr.dst_port) { + efx_spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_PORT; + efx_spec->efs_loc_port = rte_bswap16(spec->hdr.dst_port); + } else if (mask->hdr.dst_port != 0) { + goto fail_bad_mask; + } + + return 0; + +fail_bad_mask: + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, item, + "Bad mask in the TCP pattern item"); + return -rte_errno; +} + +/** + * Convert UDP item to EFX filter specification. + * + * @param item[in] + * Item specification. Only source and destination ports fields + * are supported. If the mask is NULL, default mask will be used. + * Ranging is not supported. + * @param efx_spec[in, out] + * EFX filter specification to update. + * @param[out] error + * Perform verbose error reporting if not NULL. + */ +static int +sfc_flow_parse_udp(const struct rte_flow_item *item, + efx_filter_spec_t *efx_spec, + struct rte_flow_error *error) +{ + int rc; + const struct rte_flow_item_udp *spec = NULL; + const struct rte_flow_item_udp *mask = NULL; + const struct rte_flow_item_udp supp_mask = { + .hdr = { + .src_port = 0xffff, + .dst_port = 0xffff, + } + }; + + rc = sfc_flow_parse_init(item, + (const void **)&spec, + (const void **)&mask, + &supp_mask, + &rte_flow_item_udp_mask, + sizeof(struct rte_flow_item_udp), + error); + if (rc != 0) + return rc; + + /* + * Filtering by UDP source and destination ports requires + * the appropriate IP_PROTO in hardware filters + */ + if (!(efx_spec->efs_match_flags & EFX_FILTER_MATCH_IP_PROTO)) { + efx_spec->efs_match_flags |= EFX_FILTER_MATCH_IP_PROTO; + efx_spec->efs_ip_proto = EFX_IPPROTO_UDP; + } else if (efx_spec->efs_ip_proto != EFX_IPPROTO_UDP) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, item, + "IP proto in pattern with UDP item should be appropriate"); + return -rte_errno; + } + + if (spec == NULL) + return 0; + + /* + * Source and destination ports are in big-endian byte order in item and + * in little-endian in efx_spec, so byte swap is used + */ + if (mask->hdr.src_port == supp_mask.hdr.src_port) { + efx_spec->efs_match_flags |= EFX_FILTER_MATCH_REM_PORT; + efx_spec->efs_rem_port = rte_bswap16(spec->hdr.src_port); + } else if (mask->hdr.src_port != 0) { + goto fail_bad_mask; + } + + if (mask->hdr.dst_port == supp_mask.hdr.dst_port) { + efx_spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_PORT; + efx_spec->efs_loc_port = rte_bswap16(spec->hdr.dst_port); + } else if (mask->hdr.dst_port != 0) { + goto fail_bad_mask; + } + + return 0; + +fail_bad_mask: + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, item, + "Bad mask in the UDP pattern item"); + return -rte_errno; +} + +static const struct sfc_flow_item sfc_flow_items[] = { + { + .type = RTE_FLOW_ITEM_TYPE_VOID, + .prev_layer = SFC_FLOW_ITEM_ANY_LAYER, + .layer = SFC_FLOW_ITEM_ANY_LAYER, + .parse = sfc_flow_parse_void, + }, + { + .type = RTE_FLOW_ITEM_TYPE_ETH, + .prev_layer = SFC_FLOW_ITEM_START_LAYER, + .layer = SFC_FLOW_ITEM_L2, + .parse = sfc_flow_parse_eth, + }, + { + .type = RTE_FLOW_ITEM_TYPE_VLAN, + .prev_layer = SFC_FLOW_ITEM_L2, + .layer = SFC_FLOW_ITEM_L2, + .parse = sfc_flow_parse_vlan, + }, + { + .type = RTE_FLOW_ITEM_TYPE_IPV4, + .prev_layer = SFC_FLOW_ITEM_L2, + .layer = SFC_FLOW_ITEM_L3, + .parse = sfc_flow_parse_ipv4, + }, + { + .type = RTE_FLOW_ITEM_TYPE_IPV6, + .prev_layer = SFC_FLOW_ITEM_L2, + .layer = SFC_FLOW_ITEM_L3, + .parse = sfc_flow_parse_ipv6, + }, + { + .type = RTE_FLOW_ITEM_TYPE_TCP, + .prev_layer = SFC_FLOW_ITEM_L3, + .layer = SFC_FLOW_ITEM_L4, + .parse = sfc_flow_parse_tcp, + }, + { + .type = RTE_FLOW_ITEM_TYPE_UDP, + .prev_layer = SFC_FLOW_ITEM_L3, + .layer = SFC_FLOW_ITEM_L4, + .parse = sfc_flow_parse_udp, + }, +}; + +/* + * Protocol-independent flow API support + */ +static int +sfc_flow_parse_attr(const struct rte_flow_attr *attr, + struct rte_flow *flow, + struct rte_flow_error *error) +{ + if (attr == NULL) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR, NULL, + "NULL attribute"); + return -rte_errno; + } + if (attr->group != 0) { + rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ATTR_GROUP, attr, + "Groups are not supported"); + return -rte_errno; + } + if (attr->priority != 0) { + rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, attr, + "Priorities are not supported"); + return -rte_errno; + } + if (attr->egress != 0) { + rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, attr, + "Egress is not supported"); + return -rte_errno; + } + if (attr->ingress == 0) { + rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, attr, + "Only ingress is supported"); + return -rte_errno; + } + + flow->spec.efs_flags |= EFX_FILTER_FLAG_RX; + flow->spec.efs_rss_context = EFX_RSS_CONTEXT_DEFAULT; + + return 0; +} + +/* Get item from array sfc_flow_items */ +static const struct sfc_flow_item * +sfc_flow_get_item(enum rte_flow_item_type type) +{ + unsigned int i; + + for (i = 0; i < RTE_DIM(sfc_flow_items); i++) + if (sfc_flow_items[i].type == type) + return &sfc_flow_items[i]; + + return NULL; +} + +static int +sfc_flow_parse_pattern(const struct rte_flow_item pattern[], + struct rte_flow *flow, + struct rte_flow_error *error) +{ + int rc; + unsigned int prev_layer = SFC_FLOW_ITEM_ANY_LAYER; + const struct sfc_flow_item *item; + + if (pattern == NULL) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM_NUM, NULL, + "NULL pattern"); + return -rte_errno; + } + + for (; pattern->type != RTE_FLOW_ITEM_TYPE_END; pattern++) { + item = sfc_flow_get_item(pattern->type); + if (item == NULL) { + rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ITEM, pattern, + "Unsupported pattern item"); + return -rte_errno; + } + + /* + * Omitting one or several protocol layers at the beginning + * of pattern is supported + */ + if (item->prev_layer != SFC_FLOW_ITEM_ANY_LAYER && + prev_layer != SFC_FLOW_ITEM_ANY_LAYER && + item->prev_layer != prev_layer) { + rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ITEM, pattern, + "Unexpected sequence of pattern items"); + return -rte_errno; + } + + rc = item->parse(pattern, &flow->spec, error); + if (rc != 0) + return rc; + + if (item->layer != SFC_FLOW_ITEM_ANY_LAYER) + prev_layer = item->layer; + } + + return 0; +} + +static int +sfc_flow_parse_queue(struct sfc_adapter *sa, + const struct rte_flow_action_queue *queue, + struct rte_flow *flow) +{ + struct sfc_rxq *rxq; + + if (queue->index >= sa->rxq_count) + return -EINVAL; + + rxq = sa->rxq_info[queue->index].rxq; + flow->spec.efs_dmaq_id = (uint16_t)rxq->hw_index; + + return 0; +} + +#if EFSYS_OPT_RX_SCALE +static int +sfc_flow_parse_rss(struct sfc_adapter *sa, + const struct rte_flow_action_rss *rss, + struct rte_flow *flow) +{ + unsigned int rxq_sw_index; + struct sfc_rxq *rxq; + unsigned int rxq_hw_index_min; + unsigned int rxq_hw_index_max; + const struct rte_eth_rss_conf *rss_conf = rss->rss_conf; + uint64_t rss_hf; + uint8_t *rss_key = NULL; + struct sfc_flow_rss *sfc_rss_conf = &flow->rss_conf; + unsigned int i; + + if (rss->num == 0) + return -EINVAL; + + rxq_sw_index = sa->rxq_count - 1; + rxq = sa->rxq_info[rxq_sw_index].rxq; + rxq_hw_index_min = rxq->hw_index; + rxq_hw_index_max = 0; + + for (i = 0; i < rss->num; ++i) { + rxq_sw_index = rss->queue[i]; + + if (rxq_sw_index >= sa->rxq_count) + return -EINVAL; + + rxq = sa->rxq_info[rxq_sw_index].rxq; + + if (rxq->hw_index < rxq_hw_index_min) + rxq_hw_index_min = rxq->hw_index; + + if (rxq->hw_index > rxq_hw_index_max) + rxq_hw_index_max = rxq->hw_index; + } + + rss_hf = (rss_conf != NULL) ? rss_conf->rss_hf : SFC_RSS_OFFLOADS; + if ((rss_hf & ~SFC_RSS_OFFLOADS) != 0) + return -EINVAL; + + if (rss_conf != NULL) { + if (rss_conf->rss_key_len != sizeof(sa->rss_key)) + return -EINVAL; + + rss_key = rss_conf->rss_key; + } else { + rss_key = sa->rss_key; + } + + flow->rss = B_TRUE; + + sfc_rss_conf->rxq_hw_index_min = rxq_hw_index_min; + sfc_rss_conf->rxq_hw_index_max = rxq_hw_index_max; + sfc_rss_conf->rss_hash_types = sfc_rte_to_efx_hash_type(rss_hf); + rte_memcpy(sfc_rss_conf->rss_key, rss_key, sizeof(sa->rss_key)); + + for (i = 0; i < RTE_DIM(sfc_rss_conf->rss_tbl); ++i) { + unsigned int rxq_sw_index = rss->queue[i % rss->num]; + struct sfc_rxq *rxq = sa->rxq_info[rxq_sw_index].rxq; + + sfc_rss_conf->rss_tbl[i] = rxq->hw_index - rxq_hw_index_min; + } + + return 0; +} +#endif /* EFSYS_OPT_RX_SCALE */ + +static int +sfc_flow_filter_insert(struct sfc_adapter *sa, + struct rte_flow *flow) +{ + efx_filter_spec_t *spec = &flow->spec; + +#if EFSYS_OPT_RX_SCALE + struct sfc_flow_rss *rss = &flow->rss_conf; + int rc = 0; + + if (flow->rss) { + unsigned int rss_spread = MIN(rss->rxq_hw_index_max - + rss->rxq_hw_index_min + 1, + EFX_MAXRSS); + + rc = efx_rx_scale_context_alloc(sa->nic, + EFX_RX_SCALE_EXCLUSIVE, + rss_spread, + &spec->efs_rss_context); + if (rc != 0) + goto fail_scale_context_alloc; + + rc = efx_rx_scale_mode_set(sa->nic, spec->efs_rss_context, + EFX_RX_HASHALG_TOEPLITZ, + rss->rss_hash_types, B_TRUE); + if (rc != 0) + goto fail_scale_mode_set; + + rc = efx_rx_scale_key_set(sa->nic, spec->efs_rss_context, + rss->rss_key, + sizeof(sa->rss_key)); + if (rc != 0) + goto fail_scale_key_set; + + spec->efs_dmaq_id = rss->rxq_hw_index_min; + spec->efs_flags |= EFX_FILTER_FLAG_RX_RSS; + } + + rc = efx_filter_insert(sa->nic, spec); + if (rc != 0) + goto fail_filter_insert; + + if (flow->rss) { + /* + * Scale table is set after filter insertion because + * the table entries are relative to the base RxQ ID + * and the latter is submitted to the HW by means of + * inserting a filter, so by the time of the request + * the HW knows all the information needed to verify + * the table entries, and the operation will succeed + */ + rc = efx_rx_scale_tbl_set(sa->nic, spec->efs_rss_context, + rss->rss_tbl, RTE_DIM(rss->rss_tbl)); + if (rc != 0) + goto fail_scale_tbl_set; + } + + return 0; + +fail_scale_tbl_set: + efx_filter_remove(sa->nic, spec); + +fail_filter_insert: +fail_scale_key_set: +fail_scale_mode_set: + if (flow->rss) + efx_rx_scale_context_free(sa->nic, spec->efs_rss_context); + +fail_scale_context_alloc: + return rc; +#else /* !EFSYS_OPT_RX_SCALE */ + return efx_filter_insert(sa->nic, spec); +#endif /* EFSYS_OPT_RX_SCALE */ +} + +static int +sfc_flow_filter_remove(struct sfc_adapter *sa, + struct rte_flow *flow) +{ + efx_filter_spec_t *spec = &flow->spec; + int rc = 0; + + rc = efx_filter_remove(sa->nic, spec); + if (rc != 0) + return rc; + +#if EFSYS_OPT_RX_SCALE + if (flow->rss) + rc = efx_rx_scale_context_free(sa->nic, spec->efs_rss_context); +#endif /* EFSYS_OPT_RX_SCALE */ + + return rc; +} + +static int +sfc_flow_parse_actions(struct sfc_adapter *sa, + const struct rte_flow_action actions[], + struct rte_flow *flow, + struct rte_flow_error *error) +{ + int rc; + boolean_t is_specified = B_FALSE; + + if (actions == NULL) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION_NUM, NULL, + "NULL actions"); + return -rte_errno; + } + + for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { + switch (actions->type) { + case RTE_FLOW_ACTION_TYPE_VOID: + break; + + case RTE_FLOW_ACTION_TYPE_QUEUE: + rc = sfc_flow_parse_queue(sa, actions->conf, flow); + if (rc != 0) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, actions, + "Bad QUEUE action"); + return -rte_errno; + } + + is_specified = B_TRUE; + break; + +#if EFSYS_OPT_RX_SCALE + case RTE_FLOW_ACTION_TYPE_RSS: + rc = sfc_flow_parse_rss(sa, actions->conf, flow); + if (rc != 0) { + rte_flow_error_set(error, rc, + RTE_FLOW_ERROR_TYPE_ACTION, actions, + "Bad RSS action"); + return -rte_errno; + } + + is_specified = B_TRUE; + break; +#endif /* EFSYS_OPT_RX_SCALE */ + + default: + rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, actions, + "Action is not supported"); + return -rte_errno; + } + } + + if (!is_specified) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION_NUM, actions, + "Action is unspecified"); + return -rte_errno; + } + + return 0; +} + +static int +sfc_flow_parse(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow *flow, + struct rte_flow_error *error) +{ + struct sfc_adapter *sa = dev->data->dev_private; + int rc; + + rc = sfc_flow_parse_attr(attr, flow, error); + if (rc != 0) + goto fail_bad_value; + + rc = sfc_flow_parse_pattern(pattern, flow, error); + if (rc != 0) + goto fail_bad_value; + + rc = sfc_flow_parse_actions(sa, actions, flow, error); + if (rc != 0) + goto fail_bad_value; + + if (!sfc_filter_is_match_supported(sa, flow->spec.efs_match_flags)) { + rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "Flow rule pattern is not supported"); + return -rte_errno; + } + +fail_bad_value: + return rc; +} + +static int +sfc_flow_validate(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error) +{ + struct rte_flow flow; + + memset(&flow, 0, sizeof(flow)); + + return sfc_flow_parse(dev, attr, pattern, actions, &flow, error); +} + +static struct rte_flow * +sfc_flow_create(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error) +{ + struct sfc_adapter *sa = dev->data->dev_private; + struct rte_flow *flow = NULL; + int rc; + + flow = rte_zmalloc("sfc_rte_flow", sizeof(*flow), 0); + if (flow == NULL) { + rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "Failed to allocate memory"); + goto fail_no_mem; + } + + rc = sfc_flow_parse(dev, attr, pattern, actions, flow, error); + if (rc != 0) + goto fail_bad_value; + + TAILQ_INSERT_TAIL(&sa->filter.flow_list, flow, entries); + + sfc_adapter_lock(sa); + + if (sa->state == SFC_ADAPTER_STARTED) { + rc = sfc_flow_filter_insert(sa, flow); + if (rc != 0) { + rte_flow_error_set(error, rc, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "Failed to insert filter"); + goto fail_filter_insert; + } + } + + sfc_adapter_unlock(sa); + + return flow; + +fail_filter_insert: + TAILQ_REMOVE(&sa->filter.flow_list, flow, entries); + +fail_bad_value: + rte_free(flow); + sfc_adapter_unlock(sa); + +fail_no_mem: + return NULL; +} + +static int +sfc_flow_remove(struct sfc_adapter *sa, + struct rte_flow *flow, + struct rte_flow_error *error) +{ + int rc = 0; + + SFC_ASSERT(sfc_adapter_is_locked(sa)); + + if (sa->state == SFC_ADAPTER_STARTED) { + rc = sfc_flow_filter_remove(sa, flow); + if (rc != 0) + rte_flow_error_set(error, rc, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "Failed to destroy flow rule"); + } + + TAILQ_REMOVE(&sa->filter.flow_list, flow, entries); + rte_free(flow); + + return rc; +} + +static int +sfc_flow_destroy(struct rte_eth_dev *dev, + struct rte_flow *flow, + struct rte_flow_error *error) +{ + struct sfc_adapter *sa = dev->data->dev_private; + struct rte_flow *flow_ptr; + int rc = EINVAL; + + sfc_adapter_lock(sa); + + TAILQ_FOREACH(flow_ptr, &sa->filter.flow_list, entries) { + if (flow_ptr == flow) + rc = 0; + } + if (rc != 0) { + rte_flow_error_set(error, rc, + RTE_FLOW_ERROR_TYPE_HANDLE, NULL, + "Failed to find flow rule to destroy"); + goto fail_bad_value; + } + + rc = sfc_flow_remove(sa, flow, error); + +fail_bad_value: + sfc_adapter_unlock(sa); + + return -rc; +} + +static int +sfc_flow_flush(struct rte_eth_dev *dev, + struct rte_flow_error *error) +{ + struct sfc_adapter *sa = dev->data->dev_private; + struct rte_flow *flow; + int rc = 0; + int ret = 0; + + sfc_adapter_lock(sa); + + while ((flow = TAILQ_FIRST(&sa->filter.flow_list)) != NULL) { + rc = sfc_flow_remove(sa, flow, error); + if (rc != 0) + ret = rc; + } + + sfc_adapter_unlock(sa); + + return -ret; +} + +static int +sfc_flow_isolate(struct rte_eth_dev *dev, int enable, + struct rte_flow_error *error) +{ + struct sfc_adapter *sa = dev->data->dev_private; + struct sfc_port *port = &sa->port; + int ret = 0; + + sfc_adapter_lock(sa); + if (sa->state != SFC_ADAPTER_INITIALIZED) { + rte_flow_error_set(error, EBUSY, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "please close the port first"); + ret = -rte_errno; + } else { + port->isolated = (enable) ? B_TRUE : B_FALSE; + } + sfc_adapter_unlock(sa); + + return ret; +} + +const struct rte_flow_ops sfc_flow_ops = { + .validate = sfc_flow_validate, + .create = sfc_flow_create, + .destroy = sfc_flow_destroy, + .flush = sfc_flow_flush, + .query = NULL, + .isolate = sfc_flow_isolate, +}; + +void +sfc_flow_init(struct sfc_adapter *sa) +{ + SFC_ASSERT(sfc_adapter_is_locked(sa)); + + TAILQ_INIT(&sa->filter.flow_list); +} + +void +sfc_flow_fini(struct sfc_adapter *sa) +{ + struct rte_flow *flow; + + SFC_ASSERT(sfc_adapter_is_locked(sa)); + + while ((flow = TAILQ_FIRST(&sa->filter.flow_list)) != NULL) { + TAILQ_REMOVE(&sa->filter.flow_list, flow, entries); + rte_free(flow); + } +} + +void +sfc_flow_stop(struct sfc_adapter *sa) +{ + struct rte_flow *flow; + + SFC_ASSERT(sfc_adapter_is_locked(sa)); + + TAILQ_FOREACH(flow, &sa->filter.flow_list, entries) + sfc_flow_filter_remove(sa, flow); +} + +int +sfc_flow_start(struct sfc_adapter *sa) +{ + struct rte_flow *flow; + int rc = 0; + + sfc_log_init(sa, "entry"); + + SFC_ASSERT(sfc_adapter_is_locked(sa)); + + TAILQ_FOREACH(flow, &sa->filter.flow_list, entries) { + rc = sfc_flow_filter_insert(sa, flow); + if (rc != 0) + goto fail_bad_flow; + } + + sfc_log_init(sa, "done"); + +fail_bad_flow: + return rc; +} diff --git a/dpdk/drivers/net/sfc/sfc_flow.h b/dpdk/drivers/net/sfc/sfc_flow.h new file mode 100644 index 00000000..aa740d7d --- /dev/null +++ b/dpdk/drivers/net/sfc/sfc_flow.h @@ -0,0 +1,79 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2017 Solarflare Communications Inc. + * All rights reserved. + * + * This software was jointly developed between OKTET Labs (under contract + * for Solarflare) and Solarflare Communications, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _SFC_FLOW_H +#define _SFC_FLOW_H + +#include +#include + +#include "efx.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#if EFSYS_OPT_RX_SCALE +/* RSS configuration storage */ +struct sfc_flow_rss { + unsigned int rxq_hw_index_min; + unsigned int rxq_hw_index_max; + unsigned int rss_hash_types; + uint8_t rss_key[EFX_RSS_KEY_SIZE]; + unsigned int rss_tbl[EFX_RSS_TBL_SIZE]; +}; +#endif /* EFSYS_OPT_RX_SCALE */ + +/* PMD-specific definition of the opaque type from rte_flow.h */ +struct rte_flow { + efx_filter_spec_t spec; /* filter specification */ +#if EFSYS_OPT_RX_SCALE + boolean_t rss; /* RSS toggle */ + struct sfc_flow_rss rss_conf; /* RSS configuration */ +#endif /* EFSYS_OPT_RX_SCALE */ + TAILQ_ENTRY(rte_flow) entries; /* flow list entries */ +}; + +TAILQ_HEAD(sfc_flow_list, rte_flow); + +extern const struct rte_flow_ops sfc_flow_ops; + +struct sfc_adapter; + +void sfc_flow_init(struct sfc_adapter *sa); +void sfc_flow_fini(struct sfc_adapter *sa); +int sfc_flow_start(struct sfc_adapter *sa); +void sfc_flow_stop(struct sfc_adapter *sa); + +#ifdef __cplusplus +} +#endif +#endif /* _SFC_FLOW_H */ diff --git a/dpdk/drivers/net/sfc/sfc_intr.c b/dpdk/drivers/net/sfc/sfc_intr.c new file mode 100644 index 00000000..de65f8c0 --- /dev/null +++ b/dpdk/drivers/net/sfc/sfc_intr.c @@ -0,0 +1,345 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2016-2017 Solarflare Communications Inc. + * All rights reserved. + * + * This software was jointly developed between OKTET Labs (under contract + * for Solarflare) and Solarflare Communications, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * At the momemt of writing DPDK v16.07 has notion of two types of + * interrupts: LSC (link status change) and RXQ (receive indication). + * It allows to register interrupt callback for entire device which is + * not intended to be used for receive indication (i.e. link status + * change indication only). The handler has no information which HW + * interrupt has triggered it, so we don't know which event queue should + * be polled/reprimed (except qmask in the case of legacy line interrupt). + */ + +#include +#include + +#include "efx.h" + +#include "sfc.h" +#include "sfc_log.h" +#include "sfc_ev.h" + +static void +sfc_intr_handle_mgmt_evq(struct sfc_adapter *sa) +{ + struct sfc_evq *evq; + + rte_spinlock_lock(&sa->mgmt_evq_lock); + + evq = sa->mgmt_evq; + + if (!sa->mgmt_evq_running) { + sfc_log_init(sa, "interrupt on not running management EVQ %u", + evq->evq_index); + } else { + sfc_ev_qpoll(evq); + + if (sfc_ev_qprime(evq) != 0) + sfc_err(sa, "cannot prime EVQ %u", evq->evq_index); + } + + rte_spinlock_unlock(&sa->mgmt_evq_lock); +} + +static void +sfc_intr_line_handler(void *cb_arg) +{ + struct sfc_adapter *sa = (struct sfc_adapter *)cb_arg; + efx_nic_t *enp = sa->nic; + boolean_t fatal; + uint32_t qmask; + unsigned int lsc_seq = sa->port.lsc_seq; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(sa->eth_dev); + + sfc_log_init(sa, "entry"); + + if (sa->state != SFC_ADAPTER_STARTED && + sa->state != SFC_ADAPTER_STARTING && + sa->state != SFC_ADAPTER_STOPPING) { + sfc_log_init(sa, + "interrupt on stopped adapter, don't reenable"); + goto exit; + } + + efx_intr_status_line(enp, &fatal, &qmask); + if (fatal) { + (void)efx_intr_disable(enp); + (void)efx_intr_fatal(enp); + sfc_err(sa, "fatal, interrupts disabled"); + goto exit; + } + + if (qmask & (1 << sa->mgmt_evq_index)) + sfc_intr_handle_mgmt_evq(sa); + + if (rte_intr_enable(&pci_dev->intr_handle) != 0) + sfc_err(sa, "cannot reenable interrupts"); + + sfc_log_init(sa, "done"); + +exit: + if (lsc_seq != sa->port.lsc_seq) { + sfc_info(sa, "link status change event: link %s", + sa->eth_dev->data->dev_link.link_status ? + "UP" : "DOWN"); + _rte_eth_dev_callback_process(sa->eth_dev, + RTE_ETH_EVENT_INTR_LSC, + NULL, NULL); + } +} + +static void +sfc_intr_message_handler(void *cb_arg) +{ + struct sfc_adapter *sa = (struct sfc_adapter *)cb_arg; + efx_nic_t *enp = sa->nic; + boolean_t fatal; + unsigned int lsc_seq = sa->port.lsc_seq; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(sa->eth_dev); + + sfc_log_init(sa, "entry"); + + if (sa->state != SFC_ADAPTER_STARTED && + sa->state != SFC_ADAPTER_STARTING && + sa->state != SFC_ADAPTER_STOPPING) { + sfc_log_init(sa, "adapter not-started, don't reenable"); + goto exit; + } + + efx_intr_status_message(enp, sa->mgmt_evq_index, &fatal); + if (fatal) { + (void)efx_intr_disable(enp); + (void)efx_intr_fatal(enp); + sfc_err(sa, "fatal, interrupts disabled"); + goto exit; + } + + sfc_intr_handle_mgmt_evq(sa); + + if (rte_intr_enable(&pci_dev->intr_handle) != 0) + sfc_err(sa, "cannot reenable interrupts"); + + sfc_log_init(sa, "done"); + +exit: + if (lsc_seq != sa->port.lsc_seq) { + sfc_info(sa, "link status change event"); + _rte_eth_dev_callback_process(sa->eth_dev, + RTE_ETH_EVENT_INTR_LSC, + NULL, NULL); + } +} + +int +sfc_intr_start(struct sfc_adapter *sa) +{ + struct sfc_intr *intr = &sa->intr; + struct rte_intr_handle *intr_handle; + struct rte_pci_device *pci_dev; + int rc; + + sfc_log_init(sa, "entry"); + + /* + * The EFX common code event queue module depends on the interrupt + * module. Ensure that the interrupt module is always initialized + * (even if interrupts are not used). Status memory is required + * for Siena only and may be NULL for EF10. + */ + sfc_log_init(sa, "efx_intr_init"); + rc = efx_intr_init(sa->nic, intr->type, NULL); + if (rc != 0) + goto fail_intr_init; + + pci_dev = RTE_ETH_DEV_TO_PCI(sa->eth_dev); + intr_handle = &pci_dev->intr_handle; + + if (intr->handler != NULL) { + sfc_log_init(sa, "rte_intr_callback_register"); + rc = rte_intr_callback_register(intr_handle, intr->handler, + (void *)sa); + if (rc != 0) { + sfc_err(sa, + "cannot register interrupt handler (rc=%d)", + rc); + /* + * Convert error code from negative returned by RTE API + * to positive used in the driver. + */ + rc = -rc; + goto fail_rte_intr_cb_reg; + } + + sfc_log_init(sa, "rte_intr_enable"); + rc = rte_intr_enable(intr_handle); + if (rc != 0) { + sfc_err(sa, "cannot enable interrupts (rc=%d)", rc); + /* + * Convert error code from negative returned by RTE API + * to positive used in the driver. + */ + rc = -rc; + goto fail_rte_intr_enable; + } + + sfc_log_init(sa, "efx_intr_enable"); + efx_intr_enable(sa->nic); + } + + sfc_log_init(sa, "done type=%u max_intr=%d nb_efd=%u vec=%p", + intr_handle->type, intr_handle->max_intr, + intr_handle->nb_efd, intr_handle->intr_vec); + return 0; + +fail_rte_intr_enable: + rte_intr_callback_unregister(intr_handle, intr->handler, (void *)sa); + +fail_rte_intr_cb_reg: + efx_intr_fini(sa->nic); + +fail_intr_init: + sfc_log_init(sa, "failed %d", rc); + return rc; +} + +void +sfc_intr_stop(struct sfc_adapter *sa) +{ + struct sfc_intr *intr = &sa->intr; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(sa->eth_dev); + + sfc_log_init(sa, "entry"); + + if (intr->handler != NULL) { + struct rte_intr_handle *intr_handle; + int rc; + + efx_intr_disable(sa->nic); + + intr_handle = &pci_dev->intr_handle; + if (rte_intr_disable(intr_handle) != 0) + sfc_err(sa, "cannot disable interrupts"); + + while ((rc = rte_intr_callback_unregister(intr_handle, + intr->handler, (void *)sa)) == -EAGAIN) + ; + if (rc != 1) + sfc_err(sa, + "cannot unregister interrupt handler %d", + rc); + } + + efx_intr_fini(sa->nic); + + sfc_log_init(sa, "done"); +} + +int +sfc_intr_configure(struct sfc_adapter *sa) +{ + struct sfc_intr *intr = &sa->intr; + + sfc_log_init(sa, "entry"); + + intr->handler = NULL; + intr->lsc_intr = (sa->eth_dev->data->dev_conf.intr_conf.lsc != 0); + if (!intr->lsc_intr) { + sfc_info(sa, "LSC tracking using interrupts is disabled"); + goto done; + } + + switch (intr->type) { + case EFX_INTR_MESSAGE: + intr->handler = sfc_intr_message_handler; + break; + case EFX_INTR_LINE: + intr->handler = sfc_intr_line_handler; + break; + case EFX_INTR_INVALID: + sfc_warn(sa, "interrupts are not supported"); + break; + default: + sfc_panic(sa, "unexpected EFX interrupt type %u\n", intr->type); + break; + } + +done: + sfc_log_init(sa, "done"); + return 0; +} + +void +sfc_intr_close(struct sfc_adapter *sa) +{ + sfc_log_init(sa, "entry"); + + sfc_log_init(sa, "done"); +} + +int +sfc_intr_attach(struct sfc_adapter *sa) +{ + struct sfc_intr *intr = &sa->intr; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(sa->eth_dev); + + sfc_log_init(sa, "entry"); + + switch (pci_dev->intr_handle.type) { +#ifdef RTE_EXEC_ENV_LINUXAPP + case RTE_INTR_HANDLE_UIO_INTX: + case RTE_INTR_HANDLE_VFIO_LEGACY: + intr->type = EFX_INTR_LINE; + break; + case RTE_INTR_HANDLE_UIO: + case RTE_INTR_HANDLE_VFIO_MSI: + case RTE_INTR_HANDLE_VFIO_MSIX: + intr->type = EFX_INTR_MESSAGE; + break; +#endif + default: + intr->type = EFX_INTR_INVALID; + break; + } + + sfc_log_init(sa, "done"); + return 0; +} + +void +sfc_intr_detach(struct sfc_adapter *sa) +{ + sfc_log_init(sa, "entry"); + + sa->intr.type = EFX_INTR_INVALID; + + sfc_log_init(sa, "done"); +} diff --git a/dpdk/drivers/net/sfc/sfc_kvargs.c b/dpdk/drivers/net/sfc/sfc_kvargs.c new file mode 100644 index 00000000..7bcd5951 --- /dev/null +++ b/dpdk/drivers/net/sfc/sfc_kvargs.c @@ -0,0 +1,145 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2016-2017 Solarflare Communications Inc. + * All rights reserved. + * + * This software was jointly developed between OKTET Labs (under contract + * for Solarflare) and Solarflare Communications, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include + +#include +#include + +#include "sfc.h" +#include "sfc_kvargs.h" + +int +sfc_kvargs_parse(struct sfc_adapter *sa) +{ + struct rte_eth_dev *eth_dev = (sa)->eth_dev; + struct rte_devargs *devargs = eth_dev->device->devargs; + const char **params = (const char *[]){ + SFC_KVARG_STATS_UPDATE_PERIOD_MS, + SFC_KVARG_DEBUG_INIT, + SFC_KVARG_MCDI_LOGGING, + SFC_KVARG_PERF_PROFILE, + SFC_KVARG_RX_DATAPATH, + SFC_KVARG_TX_DATAPATH, + NULL, + }; + + if (devargs == NULL) + return 0; + + sa->kvargs = rte_kvargs_parse(devargs->args, params); + if (sa->kvargs == NULL) + return EINVAL; + + return 0; +} + +void +sfc_kvargs_cleanup(struct sfc_adapter *sa) +{ + rte_kvargs_free(sa->kvargs); +} + +static int +sfc_kvarg_match_value(const char *value, const char * const *values, + unsigned int n_values) +{ + unsigned int i; + + for (i = 0; i < n_values; ++i) + if (strcasecmp(value, values[i]) == 0) + return 1; + + return 0; +} + +int +sfc_kvargs_process(struct sfc_adapter *sa, const char *key_match, + arg_handler_t handler, void *opaque_arg) +{ + if (sa->kvargs == NULL) + return 0; + + return -rte_kvargs_process(sa->kvargs, key_match, handler, opaque_arg); +} + +int +sfc_kvarg_bool_handler(__rte_unused const char *key, + const char *value_str, void *opaque) +{ + const char * const true_strs[] = { + "1", "y", "yes", "on", "true" + }; + const char * const false_strs[] = { + "0", "n", "no", "off", "false" + }; + bool *value = opaque; + + if (sfc_kvarg_match_value(value_str, true_strs, + RTE_DIM(true_strs))) + *value = true; + else if (sfc_kvarg_match_value(value_str, false_strs, + RTE_DIM(false_strs))) + *value = false; + else + return -EINVAL; + + return 0; +} + +int +sfc_kvarg_long_handler(__rte_unused const char *key, + const char *value_str, void *opaque) +{ + long value; + char *endptr; + + if (!value_str || !opaque) + return -EINVAL; + + value = strtol(value_str, &endptr, 0); + if (endptr == value_str) + return -EINVAL; + + *(long *)opaque = value; + + return 0; +} + +int +sfc_kvarg_string_handler(__rte_unused const char *key, + const char *value_str, void *opaque) +{ + *(const char **)opaque = value_str; + + return 0; +} diff --git a/dpdk/drivers/net/sfc/sfc_kvargs.h b/dpdk/drivers/net/sfc/sfc_kvargs.h new file mode 100644 index 00000000..d9c3b1da --- /dev/null +++ b/dpdk/drivers/net/sfc/sfc_kvargs.h @@ -0,0 +1,93 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2016-2017 Solarflare Communications Inc. + * All rights reserved. + * + * This software was jointly developed between OKTET Labs (under contract + * for Solarflare) and Solarflare Communications, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _SFC_KVARGS_H +#define _SFC_KVARGS_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#define SFC_KVARG_VALUES_BOOL "[1|y|yes|on|0|n|no|off]" + +#define SFC_KVARG_DEBUG_INIT "debug_init" + +#define SFC_KVARG_MCDI_LOGGING "mcdi_logging" + +#define SFC_KVARG_PERF_PROFILE "perf_profile" + +#define SFC_KVARG_PERF_PROFILE_AUTO "auto" +#define SFC_KVARG_PERF_PROFILE_THROUGHPUT "throughput" +#define SFC_KVARG_PERF_PROFILE_LOW_LATENCY "low-latency" +#define SFC_KVARG_VALUES_PERF_PROFILE \ + "[" SFC_KVARG_PERF_PROFILE_AUTO "|" \ + SFC_KVARG_PERF_PROFILE_THROUGHPUT "|" \ + SFC_KVARG_PERF_PROFILE_LOW_LATENCY "]" + +#define SFC_KVARG_STATS_UPDATE_PERIOD_MS "stats_update_period_ms" + +#define SFC_KVARG_DATAPATH_EFX "efx" +#define SFC_KVARG_DATAPATH_EF10 "ef10" +#define SFC_KVARG_DATAPATH_EF10_SIMPLE "ef10_simple" + +#define SFC_KVARG_RX_DATAPATH "rx_datapath" +#define SFC_KVARG_VALUES_RX_DATAPATH \ + "[" SFC_KVARG_DATAPATH_EFX "|" \ + SFC_KVARG_DATAPATH_EF10 "]" + +#define SFC_KVARG_TX_DATAPATH "tx_datapath" +#define SFC_KVARG_VALUES_TX_DATAPATH \ + "[" SFC_KVARG_DATAPATH_EFX "|" \ + SFC_KVARG_DATAPATH_EF10 "|" \ + SFC_KVARG_DATAPATH_EF10_SIMPLE "]" + +struct sfc_adapter; + +int sfc_kvargs_parse(struct sfc_adapter *sa); +void sfc_kvargs_cleanup(struct sfc_adapter *sa); + +int sfc_kvargs_process(struct sfc_adapter *sa, const char *key_match, + arg_handler_t handler, void *opaque_arg); + +int sfc_kvarg_bool_handler(const char *key, const char *value_str, + void *opaque); +int sfc_kvarg_long_handler(const char *key, const char *value_str, + void *opaque); +int sfc_kvarg_string_handler(const char *key, const char *value_str, + void *opaque); + +#ifdef __cplusplus +} +#endif + +#endif /* _SFC_KVARGS_H */ diff --git a/dpdk/drivers/net/sfc/sfc_log.h b/dpdk/drivers/net/sfc/sfc_log.h new file mode 100644 index 00000000..b1a9df44 --- /dev/null +++ b/dpdk/drivers/net/sfc/sfc_log.h @@ -0,0 +1,75 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2016-2017 Solarflare Communications Inc. + * All rights reserved. + * + * This software was jointly developed between OKTET Labs (under contract + * for Solarflare) and Solarflare Communications, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _SFC_LOG_H_ +#define _SFC_LOG_H_ + +/* Log PMD message, automatically add prefix and \n */ +#define SFC_LOG(sa, level, ...) \ + do { \ + const struct sfc_adapter *__sa = (sa); \ + \ + RTE_LOG(level, PMD, \ + RTE_FMT("sfc_efx " PCI_PRI_FMT " #%" PRIu8 ": " \ + RTE_FMT_HEAD(__VA_ARGS__,) "\n", \ + __sa->pci_addr.domain, \ + __sa->pci_addr.bus, \ + __sa->pci_addr.devid, \ + __sa->pci_addr.function, \ + __sa->port_id, \ + RTE_FMT_TAIL(__VA_ARGS__,))); \ + } while (0) + +#define sfc_err(sa, ...) \ + SFC_LOG(sa, ERR, __VA_ARGS__) + +#define sfc_warn(sa, ...) \ + SFC_LOG(sa, WARNING, __VA_ARGS__) + +#define sfc_notice(sa, ...) \ + SFC_LOG(sa, NOTICE, __VA_ARGS__) + +#define sfc_info(sa, ...) \ + SFC_LOG(sa, INFO, __VA_ARGS__) + +#define sfc_log_init(sa, ...) \ + do { \ + const struct sfc_adapter *_sa = (sa); \ + \ + if (_sa->debug_init) \ + SFC_LOG(_sa, INFO, \ + RTE_FMT("%s(): " \ + RTE_FMT_HEAD(__VA_ARGS__,), \ + __func__, \ + RTE_FMT_TAIL(__VA_ARGS__,))); \ + } while (0) + +#endif /* _SFC_LOG_H_ */ diff --git a/dpdk/drivers/net/sfc/sfc_mcdi.c b/dpdk/drivers/net/sfc/sfc_mcdi.c new file mode 100644 index 00000000..0faad3ed --- /dev/null +++ b/dpdk/drivers/net/sfc/sfc_mcdi.c @@ -0,0 +1,331 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2016-2017 Solarflare Communications Inc. + * All rights reserved. + * + * This software was jointly developed between OKTET Labs (under contract + * for Solarflare) and Solarflare Communications, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +#include "efx.h" +#include "efx_mcdi.h" +#include "efx_regs_mcdi.h" + +#include "sfc.h" +#include "sfc_log.h" +#include "sfc_kvargs.h" +#include "sfc_ev.h" + +#define SFC_MCDI_POLL_INTERVAL_MIN_US 10 /* 10us in 1us units */ +#define SFC_MCDI_POLL_INTERVAL_MAX_US (US_PER_S / 10) /* 100ms in 1us units */ +#define SFC_MCDI_WATCHDOG_INTERVAL_US (10 * US_PER_S) /* 10s in 1us units */ + +static void +sfc_mcdi_timeout(struct sfc_adapter *sa) +{ + sfc_warn(sa, "MC TIMEOUT"); + + sfc_panic(sa, "MCDI timeout handling is not implemented\n"); +} + +static inline boolean_t +sfc_mcdi_proxy_event_available(struct sfc_adapter *sa) +{ + struct sfc_mcdi *mcdi = &sa->mcdi; + + mcdi->proxy_handle = 0; + mcdi->proxy_result = ETIMEDOUT; + sfc_ev_mgmt_qpoll(sa); + if (mcdi->proxy_result != ETIMEDOUT) + return B_TRUE; + + return B_FALSE; +} + +static void +sfc_mcdi_poll(struct sfc_adapter *sa, boolean_t proxy) +{ + efx_nic_t *enp; + unsigned int delay_total; + unsigned int delay_us; + boolean_t aborted __rte_unused; + + delay_total = 0; + delay_us = SFC_MCDI_POLL_INTERVAL_MIN_US; + enp = sa->nic; + + do { + boolean_t poll_completed; + + poll_completed = (proxy) ? sfc_mcdi_proxy_event_available(sa) : + efx_mcdi_request_poll(enp); + if (poll_completed) + return; + + if (delay_total > SFC_MCDI_WATCHDOG_INTERVAL_US) { + if (!proxy) { + aborted = efx_mcdi_request_abort(enp); + SFC_ASSERT(aborted); + sfc_mcdi_timeout(sa); + } + + return; + } + + rte_delay_us(delay_us); + + delay_total += delay_us; + + /* Exponentially back off the poll frequency */ + RTE_BUILD_BUG_ON(SFC_MCDI_POLL_INTERVAL_MAX_US > UINT_MAX / 2); + delay_us *= 2; + if (delay_us > SFC_MCDI_POLL_INTERVAL_MAX_US) + delay_us = SFC_MCDI_POLL_INTERVAL_MAX_US; + + } while (1); +} + +static void +sfc_mcdi_execute(void *arg, efx_mcdi_req_t *emrp) +{ + struct sfc_adapter *sa = (struct sfc_adapter *)arg; + struct sfc_mcdi *mcdi = &sa->mcdi; + uint32_t proxy_handle; + + rte_spinlock_lock(&mcdi->lock); + + SFC_ASSERT(mcdi->state == SFC_MCDI_INITIALIZED); + + efx_mcdi_request_start(sa->nic, emrp, B_FALSE); + sfc_mcdi_poll(sa, B_FALSE); + + if (efx_mcdi_get_proxy_handle(sa->nic, emrp, &proxy_handle) == 0) { + /* + * Authorization is required for the MCDI request; + * wait for an MCDI proxy response event to bring + * a non-zero proxy handle (should be the same as + * the value obtained above) and operation status + */ + sfc_mcdi_poll(sa, B_TRUE); + + if ((mcdi->proxy_handle != 0) && + (mcdi->proxy_handle != proxy_handle)) { + sfc_err(sa, "Unexpected MCDI proxy event"); + emrp->emr_rc = EFAULT; + } else if (mcdi->proxy_result == 0) { + /* + * Authorization succeeded; re-issue the original + * request and poll for an ordinary MCDI response + */ + efx_mcdi_request_start(sa->nic, emrp, B_FALSE); + sfc_mcdi_poll(sa, B_FALSE); + } else { + emrp->emr_rc = mcdi->proxy_result; + sfc_err(sa, "MCDI proxy authorization failed " + "(handle=%08x, result=%d)", + proxy_handle, mcdi->proxy_result); + } + } + + rte_spinlock_unlock(&mcdi->lock); +} + +static void +sfc_mcdi_ev_cpl(void *arg) +{ + struct sfc_adapter *sa = (struct sfc_adapter *)arg; + struct sfc_mcdi *mcdi __rte_unused; + + mcdi = &sa->mcdi; + SFC_ASSERT(mcdi->state == SFC_MCDI_INITIALIZED); + + /* MCDI is polled, completions are not expected */ + SFC_ASSERT(0); +} + +static void +sfc_mcdi_exception(void *arg, efx_mcdi_exception_t eme) +{ + struct sfc_adapter *sa = (struct sfc_adapter *)arg; + + sfc_warn(sa, "MC %s", + (eme == EFX_MCDI_EXCEPTION_MC_REBOOT) ? "REBOOT" : + (eme == EFX_MCDI_EXCEPTION_MC_BADASSERT) ? "BADASSERT" : "UNKNOWN"); + + sfc_panic(sa, "MCDI exceptions handling is not implemented\n"); +} + +#define SFC_MCDI_LOG_BUF_SIZE 128 + +static size_t +sfc_mcdi_do_log(const struct sfc_adapter *sa, + char *buffer, void *data, size_t data_size, + size_t pfxsize, size_t position) +{ + uint32_t *words = data; + /* Space separator plus 2 characters per byte */ + const size_t word_str_space = 1 + 2 * sizeof(*words); + size_t i; + + for (i = 0; i < data_size; i += sizeof(*words)) { + if (position + word_str_space >= + SFC_MCDI_LOG_BUF_SIZE) { + /* Flush at SFC_MCDI_LOG_BUF_SIZE with backslash + * at the end which is required by netlogdecode. + */ + buffer[position] = '\0'; + sfc_info(sa, "%s \\", buffer); + /* Preserve prefix for the next log message */ + position = pfxsize; + } + position += snprintf(buffer + position, + SFC_MCDI_LOG_BUF_SIZE - position, + " %08x", *words); + words++; + } + return position; +} + +static void +sfc_mcdi_logger(void *arg, efx_log_msg_t type, + void *header, size_t header_size, + void *data, size_t data_size) +{ + struct sfc_adapter *sa = (struct sfc_adapter *)arg; + char buffer[SFC_MCDI_LOG_BUF_SIZE]; + size_t pfxsize; + size_t start; + + if (!sa->mcdi.logging) + return; + + /* The format including prefix added by sfc_info() is the format + * consumed by the Solarflare netlogdecode tool. + */ + pfxsize = snprintf(buffer, sizeof(buffer), "MCDI RPC %s:", + type == EFX_LOG_MCDI_REQUEST ? "REQ" : + type == EFX_LOG_MCDI_RESPONSE ? "RESP" : "???"); + start = sfc_mcdi_do_log(sa, buffer, header, header_size, + pfxsize, pfxsize); + start = sfc_mcdi_do_log(sa, buffer, data, data_size, pfxsize, start); + if (start != pfxsize) { + buffer[start] = '\0'; + sfc_info(sa, "%s", buffer); + } +} + +static void +sfc_mcdi_ev_proxy_response(void *arg, uint32_t handle, efx_rc_t result) +{ + struct sfc_adapter *sa = (struct sfc_adapter *)arg; + struct sfc_mcdi *mcdi = &sa->mcdi; + + mcdi->proxy_handle = handle; + mcdi->proxy_result = result; +} + +int +sfc_mcdi_init(struct sfc_adapter *sa) +{ + struct sfc_mcdi *mcdi; + size_t max_msg_size; + efx_mcdi_transport_t *emtp; + int rc; + + sfc_log_init(sa, "entry"); + + mcdi = &sa->mcdi; + + SFC_ASSERT(mcdi->state == SFC_MCDI_UNINITIALIZED); + + rte_spinlock_init(&mcdi->lock); + + mcdi->state = SFC_MCDI_INITIALIZED; + + max_msg_size = sizeof(uint32_t) + MCDI_CTL_SDU_LEN_MAX_V2; + rc = sfc_dma_alloc(sa, "mcdi", 0, max_msg_size, sa->socket_id, + &mcdi->mem); + if (rc != 0) + goto fail_dma_alloc; + + /* Convert negative error to positive used in the driver */ + rc = sfc_kvargs_process(sa, SFC_KVARG_MCDI_LOGGING, + sfc_kvarg_bool_handler, &mcdi->logging); + if (rc != 0) + goto fail_kvargs_process; + + emtp = &mcdi->transport; + emtp->emt_context = sa; + emtp->emt_dma_mem = &mcdi->mem; + emtp->emt_execute = sfc_mcdi_execute; + emtp->emt_ev_cpl = sfc_mcdi_ev_cpl; + emtp->emt_exception = sfc_mcdi_exception; + emtp->emt_logger = sfc_mcdi_logger; + emtp->emt_ev_proxy_response = sfc_mcdi_ev_proxy_response; + + sfc_log_init(sa, "init MCDI"); + rc = efx_mcdi_init(sa->nic, emtp); + if (rc != 0) + goto fail_mcdi_init; + + return 0; + +fail_mcdi_init: + memset(emtp, 0, sizeof(*emtp)); + +fail_kvargs_process: + sfc_dma_free(sa, &mcdi->mem); + +fail_dma_alloc: + mcdi->state = SFC_MCDI_UNINITIALIZED; + return rc; +} + +void +sfc_mcdi_fini(struct sfc_adapter *sa) +{ + struct sfc_mcdi *mcdi; + efx_mcdi_transport_t *emtp; + + sfc_log_init(sa, "entry"); + + mcdi = &sa->mcdi; + emtp = &mcdi->transport; + + rte_spinlock_lock(&mcdi->lock); + + SFC_ASSERT(mcdi->state == SFC_MCDI_INITIALIZED); + mcdi->state = SFC_MCDI_UNINITIALIZED; + + sfc_log_init(sa, "fini MCDI"); + efx_mcdi_fini(sa->nic); + memset(emtp, 0, sizeof(*emtp)); + + rte_spinlock_unlock(&mcdi->lock); + + sfc_dma_free(sa, &mcdi->mem); +} diff --git a/dpdk/drivers/net/sfc/sfc_port.c b/dpdk/drivers/net/sfc/sfc_port.c new file mode 100644 index 00000000..5254394c --- /dev/null +++ b/dpdk/drivers/net/sfc/sfc_port.c @@ -0,0 +1,509 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2016-2017 Solarflare Communications Inc. + * All rights reserved. + * + * This software was jointly developed between OKTET Labs (under contract + * for Solarflare) and Solarflare Communications, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "efx.h" + +#include "sfc.h" +#include "sfc_log.h" +#include "sfc_kvargs.h" + +/** Default MAC statistics update period is 1 second */ +#define SFC_MAC_STATS_UPDATE_PERIOD_MS_DEF MS_PER_S + +/** The number of microseconds to sleep on attempt to get statistics update */ +#define SFC_MAC_STATS_UPDATE_RETRY_INTERVAL_US 10 + +/** The number of attempts to await arrival of freshly generated statistics */ +#define SFC_MAC_STATS_UPDATE_NB_ATTEMPTS 50 + +/** + * Update MAC statistics in the buffer. + * + * @param sa Adapter + * + * @return Status code + * @retval 0 Success + * @retval EAGAIN Try again + * @retval ENOMEM Memory allocation failure + */ +int +sfc_port_update_mac_stats(struct sfc_adapter *sa) +{ + struct sfc_port *port = &sa->port; + efsys_mem_t *esmp = &port->mac_stats_dma_mem; + uint32_t *genp = NULL; + uint32_t gen_old; + unsigned int nb_attempts = 0; + int rc; + + SFC_ASSERT(rte_spinlock_is_locked(&port->mac_stats_lock)); + + if (sa->state != SFC_ADAPTER_STARTED) + return EINVAL; + + /* + * If periodic statistics DMA'ing is off or if not supported, + * make a manual request and keep an eye on timer if need be + */ + if (!port->mac_stats_periodic_dma_supported || + (port->mac_stats_update_period_ms == 0)) { + if (port->mac_stats_update_period_ms != 0) { + uint64_t timestamp = sfc_get_system_msecs(); + + if ((timestamp - + port->mac_stats_last_request_timestamp) < + port->mac_stats_update_period_ms) + return 0; + + port->mac_stats_last_request_timestamp = timestamp; + } + + rc = efx_mac_stats_upload(sa->nic, esmp); + if (rc != 0) + return rc; + + genp = &port->mac_stats_update_generation; + gen_old = *genp; + } + + do { + if (nb_attempts > 0) + rte_delay_us(SFC_MAC_STATS_UPDATE_RETRY_INTERVAL_US); + + rc = efx_mac_stats_update(sa->nic, esmp, + port->mac_stats_buf, genp); + if (rc != 0) + return rc; + + } while ((genp != NULL) && (*genp == gen_old) && + (++nb_attempts < SFC_MAC_STATS_UPDATE_NB_ATTEMPTS)); + + return 0; +} + +int +sfc_port_reset_mac_stats(struct sfc_adapter *sa) +{ + struct sfc_port *port = &sa->port; + int rc; + + rte_spinlock_lock(&port->mac_stats_lock); + rc = efx_mac_stats_clear(sa->nic); + rte_spinlock_unlock(&port->mac_stats_lock); + + return rc; +} + +static int +sfc_port_init_dev_link(struct sfc_adapter *sa) +{ + struct rte_eth_link *dev_link = &sa->eth_dev->data->dev_link; + int rc; + efx_link_mode_t link_mode; + struct rte_eth_link current_link; + + rc = efx_port_poll(sa->nic, &link_mode); + if (rc != 0) + return rc; + + sfc_port_link_mode_to_info(link_mode, ¤t_link); + + EFX_STATIC_ASSERT(sizeof(*dev_link) == sizeof(rte_atomic64_t)); + rte_atomic64_set((rte_atomic64_t *)dev_link, + *(uint64_t *)¤t_link); + + return 0; +} + +int +sfc_port_start(struct sfc_adapter *sa) +{ + struct sfc_port *port = &sa->port; + int rc; + uint32_t phy_adv_cap; + const uint32_t phy_pause_caps = + ((1u << EFX_PHY_CAP_PAUSE) | (1u << EFX_PHY_CAP_ASYM)); + unsigned int i; + + sfc_log_init(sa, "entry"); + + sfc_log_init(sa, "init filters"); + rc = efx_filter_init(sa->nic); + if (rc != 0) + goto fail_filter_init; + + sfc_log_init(sa, "init port"); + rc = efx_port_init(sa->nic); + if (rc != 0) + goto fail_port_init; + + sfc_log_init(sa, "set flow control to %#x autoneg=%u", + port->flow_ctrl, port->flow_ctrl_autoneg); + rc = efx_mac_fcntl_set(sa->nic, port->flow_ctrl, + port->flow_ctrl_autoneg); + if (rc != 0) + goto fail_mac_fcntl_set; + + /* Preserve pause capabilities set by above efx_mac_fcntl_set() */ + efx_phy_adv_cap_get(sa->nic, EFX_PHY_CAP_CURRENT, &phy_adv_cap); + SFC_ASSERT((port->phy_adv_cap & phy_pause_caps) == 0); + phy_adv_cap = port->phy_adv_cap | (phy_adv_cap & phy_pause_caps); + + sfc_log_init(sa, "set phy adv caps to %#x", phy_adv_cap); + rc = efx_phy_adv_cap_set(sa->nic, phy_adv_cap); + if (rc != 0) + goto fail_phy_adv_cap_set; + + sfc_log_init(sa, "set MAC PDU %u", (unsigned int)port->pdu); + rc = efx_mac_pdu_set(sa->nic, port->pdu); + if (rc != 0) + goto fail_mac_pdu_set; + + if (!port->isolated) { + struct ether_addr *addr = &port->default_mac_addr; + + sfc_log_init(sa, "set MAC address"); + rc = efx_mac_addr_set(sa->nic, addr->addr_bytes); + if (rc != 0) + goto fail_mac_addr_set; + + sfc_log_init(sa, "set MAC filters"); + port->promisc = (sa->eth_dev->data->promiscuous != 0) ? + B_TRUE : B_FALSE; + port->allmulti = (sa->eth_dev->data->all_multicast != 0) ? + B_TRUE : B_FALSE; + rc = sfc_set_rx_mode(sa); + if (rc != 0) + goto fail_mac_filter_set; + + sfc_log_init(sa, "set multicast address list"); + rc = efx_mac_multicast_list_set(sa->nic, port->mcast_addrs, + port->nb_mcast_addrs); + if (rc != 0) + goto fail_mcast_address_list_set; + } + + if (port->mac_stats_reset_pending) { + rc = sfc_port_reset_mac_stats(sa); + if (rc != 0) + sfc_err(sa, "statistics reset failed (requested " + "before the port was started)"); + + port->mac_stats_reset_pending = B_FALSE; + } + + efx_mac_stats_get_mask(sa->nic, port->mac_stats_mask, + sizeof(port->mac_stats_mask)); + + for (i = 0, port->mac_stats_nb_supported = 0; i < EFX_MAC_NSTATS; ++i) + if (EFX_MAC_STAT_SUPPORTED(port->mac_stats_mask, i)) + port->mac_stats_nb_supported++; + + port->mac_stats_update_generation = 0; + + if (port->mac_stats_update_period_ms != 0) { + /* + * Update MAC stats using periodic DMA; + * any positive update interval different from + * 1000 ms can be set only on SFN8xxx provided + * that FW version is 6.2.1.1033 or higher + */ + sfc_log_init(sa, "request MAC stats DMA'ing"); + rc = efx_mac_stats_periodic(sa->nic, &port->mac_stats_dma_mem, + port->mac_stats_update_period_ms, + B_FALSE); + if (rc == 0) { + port->mac_stats_periodic_dma_supported = B_TRUE; + } else if (rc == EOPNOTSUPP) { + port->mac_stats_periodic_dma_supported = B_FALSE; + port->mac_stats_last_request_timestamp = 0; + } else { + goto fail_mac_stats_periodic; + } + } + + if ((port->mac_stats_update_period_ms != 0) && + port->mac_stats_periodic_dma_supported) { + /* + * Request an explicit MAC stats upload immediately to + * preclude bogus figures readback if the user decides + * to read stats before periodic DMA is really started + */ + rc = efx_mac_stats_upload(sa->nic, &port->mac_stats_dma_mem); + if (rc != 0) + goto fail_mac_stats_upload; + } + + sfc_log_init(sa, "disable MAC drain"); + rc = efx_mac_drain(sa->nic, B_FALSE); + if (rc != 0) + goto fail_mac_drain; + + /* Synchronize link status knowledge */ + rc = sfc_port_init_dev_link(sa); + if (rc != 0) + goto fail_port_init_dev_link; + + sfc_log_init(sa, "done"); + return 0; + +fail_port_init_dev_link: + (void)efx_mac_drain(sa->nic, B_TRUE); + +fail_mac_drain: +fail_mac_stats_upload: + (void)efx_mac_stats_periodic(sa->nic, &port->mac_stats_dma_mem, + 0, B_FALSE); + +fail_mac_stats_periodic: +fail_mcast_address_list_set: +fail_mac_filter_set: +fail_mac_addr_set: +fail_mac_pdu_set: +fail_phy_adv_cap_set: +fail_mac_fcntl_set: + efx_port_fini(sa->nic); + +fail_port_init: + efx_filter_fini(sa->nic); + +fail_filter_init: + sfc_log_init(sa, "failed %d", rc); + return rc; +} + +void +sfc_port_stop(struct sfc_adapter *sa) +{ + sfc_log_init(sa, "entry"); + + efx_mac_drain(sa->nic, B_TRUE); + + (void)efx_mac_stats_periodic(sa->nic, &sa->port.mac_stats_dma_mem, + 0, B_FALSE); + + efx_port_fini(sa->nic); + efx_filter_fini(sa->nic); + + sfc_log_init(sa, "done"); +} + +int +sfc_port_configure(struct sfc_adapter *sa) +{ + const struct rte_eth_dev_data *dev_data = sa->eth_dev->data; + struct sfc_port *port = &sa->port; + + sfc_log_init(sa, "entry"); + + if (dev_data->dev_conf.rxmode.jumbo_frame) + port->pdu = dev_data->dev_conf.rxmode.max_rx_pkt_len; + else + port->pdu = EFX_MAC_PDU(dev_data->mtu); + + return 0; +} + +void +sfc_port_close(struct sfc_adapter *sa) +{ + sfc_log_init(sa, "entry"); +} + +int +sfc_port_attach(struct sfc_adapter *sa) +{ + struct sfc_port *port = &sa->port; + const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic); + const struct ether_addr *from; + long kvarg_stats_update_period_ms; + int rc; + + sfc_log_init(sa, "entry"); + + efx_phy_adv_cap_get(sa->nic, EFX_PHY_CAP_PERM, &port->phy_adv_cap_mask); + + /* Enable flow control by default */ + port->flow_ctrl = EFX_FCNTL_RESPOND | EFX_FCNTL_GENERATE; + port->flow_ctrl_autoneg = B_TRUE; + + RTE_BUILD_BUG_ON(sizeof(encp->enc_mac_addr) != sizeof(*from)); + from = (const struct ether_addr *)(encp->enc_mac_addr); + ether_addr_copy(from, &port->default_mac_addr); + + port->max_mcast_addrs = EFX_MAC_MULTICAST_LIST_MAX; + port->nb_mcast_addrs = 0; + port->mcast_addrs = rte_calloc_socket("mcast_addr_list_buf", + port->max_mcast_addrs, + EFX_MAC_ADDR_LEN, 0, + sa->socket_id); + if (port->mcast_addrs == NULL) { + rc = ENOMEM; + goto fail_mcast_addr_list_buf_alloc; + } + + rte_spinlock_init(&port->mac_stats_lock); + + rc = ENOMEM; + port->mac_stats_buf = rte_calloc_socket("mac_stats_buf", EFX_MAC_NSTATS, + sizeof(uint64_t), 0, + sa->socket_id); + if (port->mac_stats_buf == NULL) + goto fail_mac_stats_buf_alloc; + + rc = sfc_dma_alloc(sa, "mac_stats", 0, EFX_MAC_STATS_SIZE, + sa->socket_id, &port->mac_stats_dma_mem); + if (rc != 0) + goto fail_mac_stats_dma_alloc; + + port->mac_stats_reset_pending = B_FALSE; + + kvarg_stats_update_period_ms = SFC_MAC_STATS_UPDATE_PERIOD_MS_DEF; + + rc = sfc_kvargs_process(sa, SFC_KVARG_STATS_UPDATE_PERIOD_MS, + sfc_kvarg_long_handler, + &kvarg_stats_update_period_ms); + if ((rc == 0) && + ((kvarg_stats_update_period_ms < 0) || + (kvarg_stats_update_period_ms > UINT16_MAX))) { + sfc_err(sa, "wrong '" SFC_KVARG_STATS_UPDATE_PERIOD_MS "' " + "was set (%ld);", kvarg_stats_update_period_ms); + sfc_err(sa, "it must not be less than 0 " + "or greater than %" PRIu16, UINT16_MAX); + rc = EINVAL; + goto fail_kvarg_stats_update_period_ms; + } else if (rc != 0) { + goto fail_kvarg_stats_update_period_ms; + } + + port->mac_stats_update_period_ms = kvarg_stats_update_period_ms; + + sfc_log_init(sa, "done"); + return 0; + +fail_kvarg_stats_update_period_ms: + sfc_dma_free(sa, &port->mac_stats_dma_mem); + +fail_mac_stats_dma_alloc: + rte_free(port->mac_stats_buf); + +fail_mac_stats_buf_alloc: + rte_free(port->mcast_addrs); + +fail_mcast_addr_list_buf_alloc: + sfc_log_init(sa, "failed %d", rc); + return rc; +} + +void +sfc_port_detach(struct sfc_adapter *sa) +{ + struct sfc_port *port = &sa->port; + + sfc_log_init(sa, "entry"); + + sfc_dma_free(sa, &port->mac_stats_dma_mem); + rte_free(port->mac_stats_buf); + + rte_free(port->mcast_addrs); + + sfc_log_init(sa, "done"); +} + +int +sfc_set_rx_mode(struct sfc_adapter *sa) +{ + struct sfc_port *port = &sa->port; + int rc; + + rc = efx_mac_filter_set(sa->nic, port->promisc, B_TRUE, + port->promisc || port->allmulti, B_TRUE); + + return rc; +} + +void +sfc_port_link_mode_to_info(efx_link_mode_t link_mode, + struct rte_eth_link *link_info) +{ + SFC_ASSERT(link_mode < EFX_LINK_NMODES); + + memset(link_info, 0, sizeof(*link_info)); + if ((link_mode == EFX_LINK_DOWN) || (link_mode == EFX_LINK_UNKNOWN)) + link_info->link_status = ETH_LINK_DOWN; + else + link_info->link_status = ETH_LINK_UP; + + switch (link_mode) { + case EFX_LINK_10HDX: + link_info->link_speed = ETH_SPEED_NUM_10M; + link_info->link_duplex = ETH_LINK_HALF_DUPLEX; + break; + case EFX_LINK_10FDX: + link_info->link_speed = ETH_SPEED_NUM_10M; + link_info->link_duplex = ETH_LINK_FULL_DUPLEX; + break; + case EFX_LINK_100HDX: + link_info->link_speed = ETH_SPEED_NUM_100M; + link_info->link_duplex = ETH_LINK_HALF_DUPLEX; + break; + case EFX_LINK_100FDX: + link_info->link_speed = ETH_SPEED_NUM_100M; + link_info->link_duplex = ETH_LINK_FULL_DUPLEX; + break; + case EFX_LINK_1000HDX: + link_info->link_speed = ETH_SPEED_NUM_1G; + link_info->link_duplex = ETH_LINK_HALF_DUPLEX; + break; + case EFX_LINK_1000FDX: + link_info->link_speed = ETH_SPEED_NUM_1G; + link_info->link_duplex = ETH_LINK_FULL_DUPLEX; + break; + case EFX_LINK_10000FDX: + link_info->link_speed = ETH_SPEED_NUM_10G; + link_info->link_duplex = ETH_LINK_FULL_DUPLEX; + break; + case EFX_LINK_40000FDX: + link_info->link_speed = ETH_SPEED_NUM_40G; + link_info->link_duplex = ETH_LINK_FULL_DUPLEX; + break; + default: + SFC_ASSERT(B_FALSE); + /* FALLTHROUGH */ + case EFX_LINK_UNKNOWN: + case EFX_LINK_DOWN: + link_info->link_speed = ETH_SPEED_NUM_NONE; + link_info->link_duplex = 0; + break; + } + + link_info->link_autoneg = ETH_LINK_AUTONEG; +} diff --git a/dpdk/drivers/net/sfc/sfc_rx.c b/dpdk/drivers/net/sfc/sfc_rx.c new file mode 100644 index 00000000..7816393b --- /dev/null +++ b/dpdk/drivers/net/sfc/sfc_rx.c @@ -0,0 +1,1384 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2016-2017 Solarflare Communications Inc. + * All rights reserved. + * + * This software was jointly developed between OKTET Labs (under contract + * for Solarflare) and Solarflare Communications, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +#include "efx.h" + +#include "sfc.h" +#include "sfc_debug.h" +#include "sfc_log.h" +#include "sfc_ev.h" +#include "sfc_rx.h" +#include "sfc_kvargs.h" +#include "sfc_tweak.h" + +/* + * Maximum number of Rx queue flush attempt in the case of failure or + * flush timeout + */ +#define SFC_RX_QFLUSH_ATTEMPTS (3) + +/* + * Time to wait between event queue polling attempts when waiting for Rx + * queue flush done or failed events. + */ +#define SFC_RX_QFLUSH_POLL_WAIT_MS (1) + +/* + * Maximum number of event queue polling attempts when waiting for Rx queue + * flush done or failed events. It defines Rx queue flush attempt timeout + * together with SFC_RX_QFLUSH_POLL_WAIT_MS. + */ +#define SFC_RX_QFLUSH_POLL_ATTEMPTS (2000) + +void +sfc_rx_qflush_done(struct sfc_rxq *rxq) +{ + rxq->state |= SFC_RXQ_FLUSHED; + rxq->state &= ~SFC_RXQ_FLUSHING; +} + +void +sfc_rx_qflush_failed(struct sfc_rxq *rxq) +{ + rxq->state |= SFC_RXQ_FLUSH_FAILED; + rxq->state &= ~SFC_RXQ_FLUSHING; +} + +static void +sfc_efx_rx_qrefill(struct sfc_efx_rxq *rxq) +{ + unsigned int free_space; + unsigned int bulks; + void *objs[SFC_RX_REFILL_BULK]; + efsys_dma_addr_t addr[RTE_DIM(objs)]; + unsigned int added = rxq->added; + unsigned int id; + unsigned int i; + struct sfc_efx_rx_sw_desc *rxd; + struct rte_mbuf *m; + uint16_t port_id = rxq->dp.dpq.port_id; + + free_space = EFX_RXQ_LIMIT(rxq->ptr_mask + 1) - + (added - rxq->completed); + + if (free_space < rxq->refill_threshold) + return; + + bulks = free_space / RTE_DIM(objs); + /* refill_threshold guarantees that bulks is positive */ + SFC_ASSERT(bulks > 0); + + id = added & rxq->ptr_mask; + do { + if (unlikely(rte_mempool_get_bulk(rxq->refill_mb_pool, objs, + RTE_DIM(objs)) < 0)) { + /* + * It is hardly a safe way to increment counter + * from different contexts, but all PMDs do it. + */ + rxq->evq->sa->eth_dev->data->rx_mbuf_alloc_failed += + RTE_DIM(objs); + /* Return if we have posted nothing yet */ + if (added == rxq->added) + return; + /* Push posted */ + break; + } + + for (i = 0; i < RTE_DIM(objs); + ++i, id = (id + 1) & rxq->ptr_mask) { + m = objs[i]; + + rxd = &rxq->sw_desc[id]; + rxd->mbuf = m; + + SFC_ASSERT(rte_mbuf_refcnt_read(m) == 1); + m->data_off = RTE_PKTMBUF_HEADROOM; + SFC_ASSERT(m->next == NULL); + SFC_ASSERT(m->nb_segs == 1); + m->port = port_id; + + addr[i] = rte_pktmbuf_iova(m); + } + + efx_rx_qpost(rxq->common, addr, rxq->buf_size, + RTE_DIM(objs), rxq->completed, added); + added += RTE_DIM(objs); + } while (--bulks > 0); + + SFC_ASSERT(added != rxq->added); + rxq->added = added; + efx_rx_qpush(rxq->common, added, &rxq->pushed); +} + +static uint64_t +sfc_efx_rx_desc_flags_to_offload_flags(const unsigned int desc_flags) +{ + uint64_t mbuf_flags = 0; + + switch (desc_flags & (EFX_PKT_IPV4 | EFX_CKSUM_IPV4)) { + case (EFX_PKT_IPV4 | EFX_CKSUM_IPV4): + mbuf_flags |= PKT_RX_IP_CKSUM_GOOD; + break; + case EFX_PKT_IPV4: + mbuf_flags |= PKT_RX_IP_CKSUM_BAD; + break; + default: + RTE_BUILD_BUG_ON(PKT_RX_IP_CKSUM_UNKNOWN != 0); + SFC_ASSERT((mbuf_flags & PKT_RX_IP_CKSUM_MASK) == + PKT_RX_IP_CKSUM_UNKNOWN); + break; + } + + switch ((desc_flags & + (EFX_PKT_TCP | EFX_PKT_UDP | EFX_CKSUM_TCPUDP))) { + case (EFX_PKT_TCP | EFX_CKSUM_TCPUDP): + case (EFX_PKT_UDP | EFX_CKSUM_TCPUDP): + mbuf_flags |= PKT_RX_L4_CKSUM_GOOD; + break; + case EFX_PKT_TCP: + case EFX_PKT_UDP: + mbuf_flags |= PKT_RX_L4_CKSUM_BAD; + break; + default: + RTE_BUILD_BUG_ON(PKT_RX_L4_CKSUM_UNKNOWN != 0); + SFC_ASSERT((mbuf_flags & PKT_RX_L4_CKSUM_MASK) == + PKT_RX_L4_CKSUM_UNKNOWN); + break; + } + + return mbuf_flags; +} + +static uint32_t +sfc_efx_rx_desc_flags_to_packet_type(const unsigned int desc_flags) +{ + return RTE_PTYPE_L2_ETHER | + ((desc_flags & EFX_PKT_IPV4) ? + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN : 0) | + ((desc_flags & EFX_PKT_IPV6) ? + RTE_PTYPE_L3_IPV6_EXT_UNKNOWN : 0) | + ((desc_flags & EFX_PKT_TCP) ? RTE_PTYPE_L4_TCP : 0) | + ((desc_flags & EFX_PKT_UDP) ? RTE_PTYPE_L4_UDP : 0); +} + +static const uint32_t * +sfc_efx_supported_ptypes_get(void) +{ + static const uint32_t ptypes[] = { + RTE_PTYPE_L2_ETHER, + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN, + RTE_PTYPE_L3_IPV6_EXT_UNKNOWN, + RTE_PTYPE_L4_TCP, + RTE_PTYPE_L4_UDP, + RTE_PTYPE_UNKNOWN + }; + + return ptypes; +} + +#if EFSYS_OPT_RX_SCALE +static void +sfc_efx_rx_set_rss_hash(struct sfc_efx_rxq *rxq, unsigned int flags, + struct rte_mbuf *m) +{ + uint8_t *mbuf_data; + + + if ((rxq->flags & SFC_EFX_RXQ_FLAG_RSS_HASH) == 0) + return; + + mbuf_data = rte_pktmbuf_mtod(m, uint8_t *); + + if (flags & (EFX_PKT_IPV4 | EFX_PKT_IPV6)) { + m->hash.rss = efx_pseudo_hdr_hash_get(rxq->common, + EFX_RX_HASHALG_TOEPLITZ, + mbuf_data); + + m->ol_flags |= PKT_RX_RSS_HASH; + } +} +#else +static void +sfc_efx_rx_set_rss_hash(__rte_unused struct sfc_efx_rxq *rxq, + __rte_unused unsigned int flags, + __rte_unused struct rte_mbuf *m) +{ +} +#endif + +static uint16_t +sfc_efx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) +{ + struct sfc_dp_rxq *dp_rxq = rx_queue; + struct sfc_efx_rxq *rxq = sfc_efx_rxq_by_dp_rxq(dp_rxq); + unsigned int completed; + unsigned int prefix_size = rxq->prefix_size; + unsigned int done_pkts = 0; + boolean_t discard_next = B_FALSE; + struct rte_mbuf *scatter_pkt = NULL; + + if (unlikely((rxq->flags & SFC_EFX_RXQ_FLAG_RUNNING) == 0)) + return 0; + + sfc_ev_qpoll(rxq->evq); + + completed = rxq->completed; + while (completed != rxq->pending && done_pkts < nb_pkts) { + unsigned int id; + struct sfc_efx_rx_sw_desc *rxd; + struct rte_mbuf *m; + unsigned int seg_len; + unsigned int desc_flags; + + id = completed++ & rxq->ptr_mask; + rxd = &rxq->sw_desc[id]; + m = rxd->mbuf; + desc_flags = rxd->flags; + + if (discard_next) + goto discard; + + if (desc_flags & (EFX_ADDR_MISMATCH | EFX_DISCARD)) + goto discard; + + if (desc_flags & EFX_PKT_PREFIX_LEN) { + uint16_t tmp_size; + int rc __rte_unused; + + rc = efx_pseudo_hdr_pkt_length_get(rxq->common, + rte_pktmbuf_mtod(m, uint8_t *), &tmp_size); + SFC_ASSERT(rc == 0); + seg_len = tmp_size; + } else { + seg_len = rxd->size - prefix_size; + } + + rte_pktmbuf_data_len(m) = seg_len; + rte_pktmbuf_pkt_len(m) = seg_len; + + if (scatter_pkt != NULL) { + if (rte_pktmbuf_chain(scatter_pkt, m) != 0) { + rte_pktmbuf_free(scatter_pkt); + goto discard; + } + /* The packet to deliver */ + m = scatter_pkt; + } + + if (desc_flags & EFX_PKT_CONT) { + /* The packet is scattered, more fragments to come */ + scatter_pkt = m; + /* Further fragments have no prefix */ + prefix_size = 0; + continue; + } + + /* Scattered packet is done */ + scatter_pkt = NULL; + /* The first fragment of the packet has prefix */ + prefix_size = rxq->prefix_size; + + m->ol_flags = + sfc_efx_rx_desc_flags_to_offload_flags(desc_flags); + m->packet_type = + sfc_efx_rx_desc_flags_to_packet_type(desc_flags); + + /* + * Extract RSS hash from the packet prefix and + * set the corresponding field (if needed and possible) + */ + sfc_efx_rx_set_rss_hash(rxq, desc_flags, m); + + m->data_off += prefix_size; + + *rx_pkts++ = m; + done_pkts++; + continue; + +discard: + discard_next = ((desc_flags & EFX_PKT_CONT) != 0); + rte_mempool_put(rxq->refill_mb_pool, m); + rxd->mbuf = NULL; + } + + /* pending is only moved when entire packet is received */ + SFC_ASSERT(scatter_pkt == NULL); + + rxq->completed = completed; + + sfc_efx_rx_qrefill(rxq); + + return done_pkts; +} + +static sfc_dp_rx_qdesc_npending_t sfc_efx_rx_qdesc_npending; +static unsigned int +sfc_efx_rx_qdesc_npending(struct sfc_dp_rxq *dp_rxq) +{ + struct sfc_efx_rxq *rxq = sfc_efx_rxq_by_dp_rxq(dp_rxq); + + if ((rxq->flags & SFC_EFX_RXQ_FLAG_RUNNING) == 0) + return 0; + + sfc_ev_qpoll(rxq->evq); + + return rxq->pending - rxq->completed; +} + +static sfc_dp_rx_qdesc_status_t sfc_efx_rx_qdesc_status; +static int +sfc_efx_rx_qdesc_status(struct sfc_dp_rxq *dp_rxq, uint16_t offset) +{ + struct sfc_efx_rxq *rxq = sfc_efx_rxq_by_dp_rxq(dp_rxq); + + if (unlikely(offset > rxq->ptr_mask)) + return -EINVAL; + + /* + * Poll EvQ to derive up-to-date 'rxq->pending' figure; + * it is required for the queue to be running, but the + * check is omitted because API design assumes that it + * is the duty of the caller to satisfy all conditions + */ + SFC_ASSERT((rxq->flags & SFC_EFX_RXQ_FLAG_RUNNING) == + SFC_EFX_RXQ_FLAG_RUNNING); + sfc_ev_qpoll(rxq->evq); + + /* + * There is a handful of reserved entries in the ring, + * but an explicit check whether the offset points to + * a reserved entry is neglected since the two checks + * below rely on the figures which take the HW limits + * into account and thus if an entry is reserved, the + * checks will fail and UNAVAIL code will be returned + */ + + if (offset < (rxq->pending - rxq->completed)) + return RTE_ETH_RX_DESC_DONE; + + if (offset < (rxq->added - rxq->completed)) + return RTE_ETH_RX_DESC_AVAIL; + + return RTE_ETH_RX_DESC_UNAVAIL; +} + +struct sfc_rxq * +sfc_rxq_by_dp_rxq(const struct sfc_dp_rxq *dp_rxq) +{ + const struct sfc_dp_queue *dpq = &dp_rxq->dpq; + struct rte_eth_dev *eth_dev; + struct sfc_adapter *sa; + struct sfc_rxq *rxq; + + SFC_ASSERT(rte_eth_dev_is_valid_port(dpq->port_id)); + eth_dev = &rte_eth_devices[dpq->port_id]; + + sa = eth_dev->data->dev_private; + + SFC_ASSERT(dpq->queue_id < sa->rxq_count); + rxq = sa->rxq_info[dpq->queue_id].rxq; + + SFC_ASSERT(rxq != NULL); + return rxq; +} + +static sfc_dp_rx_qcreate_t sfc_efx_rx_qcreate; +static int +sfc_efx_rx_qcreate(uint16_t port_id, uint16_t queue_id, + const struct rte_pci_addr *pci_addr, int socket_id, + const struct sfc_dp_rx_qcreate_info *info, + struct sfc_dp_rxq **dp_rxqp) +{ + struct sfc_efx_rxq *rxq; + int rc; + + rc = ENOMEM; + rxq = rte_zmalloc_socket("sfc-efx-rxq", sizeof(*rxq), + RTE_CACHE_LINE_SIZE, socket_id); + if (rxq == NULL) + goto fail_rxq_alloc; + + sfc_dp_queue_init(&rxq->dp.dpq, port_id, queue_id, pci_addr); + + rc = ENOMEM; + rxq->sw_desc = rte_calloc_socket("sfc-efx-rxq-sw_desc", + info->rxq_entries, + sizeof(*rxq->sw_desc), + RTE_CACHE_LINE_SIZE, socket_id); + if (rxq->sw_desc == NULL) + goto fail_desc_alloc; + + /* efx datapath is bound to efx control path */ + rxq->evq = sfc_rxq_by_dp_rxq(&rxq->dp)->evq; + if (info->flags & SFC_RXQ_FLAG_RSS_HASH) + rxq->flags |= SFC_EFX_RXQ_FLAG_RSS_HASH; + rxq->ptr_mask = info->rxq_entries - 1; + rxq->batch_max = info->batch_max; + rxq->prefix_size = info->prefix_size; + rxq->refill_threshold = info->refill_threshold; + rxq->buf_size = info->buf_size; + rxq->refill_mb_pool = info->refill_mb_pool; + + *dp_rxqp = &rxq->dp; + return 0; + +fail_desc_alloc: + rte_free(rxq); + +fail_rxq_alloc: + return rc; +} + +static sfc_dp_rx_qdestroy_t sfc_efx_rx_qdestroy; +static void +sfc_efx_rx_qdestroy(struct sfc_dp_rxq *dp_rxq) +{ + struct sfc_efx_rxq *rxq = sfc_efx_rxq_by_dp_rxq(dp_rxq); + + rte_free(rxq->sw_desc); + rte_free(rxq); +} + +static sfc_dp_rx_qstart_t sfc_efx_rx_qstart; +static int +sfc_efx_rx_qstart(struct sfc_dp_rxq *dp_rxq, + __rte_unused unsigned int evq_read_ptr) +{ + /* libefx-based datapath is specific to libefx-based PMD */ + struct sfc_efx_rxq *rxq = sfc_efx_rxq_by_dp_rxq(dp_rxq); + struct sfc_rxq *crxq = sfc_rxq_by_dp_rxq(dp_rxq); + + rxq->common = crxq->common; + + rxq->pending = rxq->completed = rxq->added = rxq->pushed = 0; + + sfc_efx_rx_qrefill(rxq); + + rxq->flags |= (SFC_EFX_RXQ_FLAG_STARTED | SFC_EFX_RXQ_FLAG_RUNNING); + + return 0; +} + +static sfc_dp_rx_qstop_t sfc_efx_rx_qstop; +static void +sfc_efx_rx_qstop(struct sfc_dp_rxq *dp_rxq, + __rte_unused unsigned int *evq_read_ptr) +{ + struct sfc_efx_rxq *rxq = sfc_efx_rxq_by_dp_rxq(dp_rxq); + + rxq->flags &= ~SFC_EFX_RXQ_FLAG_RUNNING; + + /* libefx-based datapath is bound to libefx-based PMD and uses + * event queue structure directly. So, there is no necessity to + * return EvQ read pointer. + */ +} + +static sfc_dp_rx_qpurge_t sfc_efx_rx_qpurge; +static void +sfc_efx_rx_qpurge(struct sfc_dp_rxq *dp_rxq) +{ + struct sfc_efx_rxq *rxq = sfc_efx_rxq_by_dp_rxq(dp_rxq); + unsigned int i; + struct sfc_efx_rx_sw_desc *rxd; + + for (i = rxq->completed; i != rxq->added; ++i) { + rxd = &rxq->sw_desc[i & rxq->ptr_mask]; + rte_mempool_put(rxq->refill_mb_pool, rxd->mbuf); + rxd->mbuf = NULL; + /* Packed stream relies on 0 in inactive SW desc. + * Rx queue stop is not performance critical, so + * there is no harm to do it always. + */ + rxd->flags = 0; + rxd->size = 0; + } + + rxq->flags &= ~SFC_EFX_RXQ_FLAG_STARTED; +} + +struct sfc_dp_rx sfc_efx_rx = { + .dp = { + .name = SFC_KVARG_DATAPATH_EFX, + .type = SFC_DP_RX, + .hw_fw_caps = 0, + }, + .features = SFC_DP_RX_FEAT_SCATTER, + .qcreate = sfc_efx_rx_qcreate, + .qdestroy = sfc_efx_rx_qdestroy, + .qstart = sfc_efx_rx_qstart, + .qstop = sfc_efx_rx_qstop, + .qpurge = sfc_efx_rx_qpurge, + .supported_ptypes_get = sfc_efx_supported_ptypes_get, + .qdesc_npending = sfc_efx_rx_qdesc_npending, + .qdesc_status = sfc_efx_rx_qdesc_status, + .pkt_burst = sfc_efx_recv_pkts, +}; + +unsigned int +sfc_rx_qdesc_npending(struct sfc_adapter *sa, unsigned int sw_index) +{ + struct sfc_rxq *rxq; + + SFC_ASSERT(sw_index < sa->rxq_count); + rxq = sa->rxq_info[sw_index].rxq; + + if (rxq == NULL || (rxq->state & SFC_RXQ_STARTED) == 0) + return 0; + + return sa->dp_rx->qdesc_npending(rxq->dp); +} + +int +sfc_rx_qdesc_done(struct sfc_dp_rxq *dp_rxq, unsigned int offset) +{ + struct sfc_rxq *rxq = sfc_rxq_by_dp_rxq(dp_rxq); + + return offset < rxq->evq->sa->dp_rx->qdesc_npending(dp_rxq); +} + +static void +sfc_rx_qflush(struct sfc_adapter *sa, unsigned int sw_index) +{ + struct sfc_rxq *rxq; + unsigned int retry_count; + unsigned int wait_count; + int rc; + + rxq = sa->rxq_info[sw_index].rxq; + SFC_ASSERT(rxq->state & SFC_RXQ_STARTED); + + /* + * Retry Rx queue flushing in the case of flush failed or + * timeout. In the worst case it can delay for 6 seconds. + */ + for (retry_count = 0; + ((rxq->state & SFC_RXQ_FLUSHED) == 0) && + (retry_count < SFC_RX_QFLUSH_ATTEMPTS); + ++retry_count) { + rc = efx_rx_qflush(rxq->common); + if (rc != 0) { + rxq->state |= (rc == EALREADY) ? + SFC_RXQ_FLUSHED : SFC_RXQ_FLUSH_FAILED; + break; + } + rxq->state &= ~SFC_RXQ_FLUSH_FAILED; + rxq->state |= SFC_RXQ_FLUSHING; + + /* + * Wait for Rx queue flush done or failed event at least + * SFC_RX_QFLUSH_POLL_WAIT_MS milliseconds and not more + * than 2 seconds (SFC_RX_QFLUSH_POLL_WAIT_MS multiplied + * by SFC_RX_QFLUSH_POLL_ATTEMPTS). + */ + wait_count = 0; + do { + rte_delay_ms(SFC_RX_QFLUSH_POLL_WAIT_MS); + sfc_ev_qpoll(rxq->evq); + } while ((rxq->state & SFC_RXQ_FLUSHING) && + (wait_count++ < SFC_RX_QFLUSH_POLL_ATTEMPTS)); + + if (rxq->state & SFC_RXQ_FLUSHING) + sfc_err(sa, "RxQ %u flush timed out", sw_index); + + if (rxq->state & SFC_RXQ_FLUSH_FAILED) + sfc_err(sa, "RxQ %u flush failed", sw_index); + + if (rxq->state & SFC_RXQ_FLUSHED) + sfc_info(sa, "RxQ %u flushed", sw_index); + } + + sa->dp_rx->qpurge(rxq->dp); +} + +static int +sfc_rx_default_rxq_set_filter(struct sfc_adapter *sa, struct sfc_rxq *rxq) +{ + boolean_t rss = (sa->rss_channels > 0) ? B_TRUE : B_FALSE; + struct sfc_port *port = &sa->port; + int rc; + + /* + * If promiscuous or all-multicast mode has been requested, setting + * filter for the default Rx queue might fail, in particular, while + * running over PCI function which is not a member of corresponding + * privilege groups; if this occurs, few iterations will be made to + * repeat this step without promiscuous and all-multicast flags set + */ +retry: + rc = efx_mac_filter_default_rxq_set(sa->nic, rxq->common, rss); + if (rc == 0) + return 0; + else if (rc != EOPNOTSUPP) + return rc; + + if (port->promisc) { + sfc_warn(sa, "promiscuous mode has been requested, " + "but the HW rejects it"); + sfc_warn(sa, "promiscuous mode will be disabled"); + + port->promisc = B_FALSE; + rc = sfc_set_rx_mode(sa); + if (rc != 0) + return rc; + + goto retry; + } + + if (port->allmulti) { + sfc_warn(sa, "all-multicast mode has been requested, " + "but the HW rejects it"); + sfc_warn(sa, "all-multicast mode will be disabled"); + + port->allmulti = B_FALSE; + rc = sfc_set_rx_mode(sa); + if (rc != 0) + return rc; + + goto retry; + } + + return rc; +} + +int +sfc_rx_qstart(struct sfc_adapter *sa, unsigned int sw_index) +{ + struct sfc_port *port = &sa->port; + struct sfc_rxq_info *rxq_info; + struct sfc_rxq *rxq; + struct sfc_evq *evq; + int rc; + + sfc_log_init(sa, "sw_index=%u", sw_index); + + SFC_ASSERT(sw_index < sa->rxq_count); + + rxq_info = &sa->rxq_info[sw_index]; + rxq = rxq_info->rxq; + SFC_ASSERT(rxq->state == SFC_RXQ_INITIALIZED); + + evq = rxq->evq; + + rc = sfc_ev_qstart(evq, sfc_evq_index_by_rxq_sw_index(sa, sw_index)); + if (rc != 0) + goto fail_ev_qstart; + + rc = efx_rx_qcreate(sa->nic, rxq->hw_index, 0, rxq_info->type, + &rxq->mem, rxq_info->entries, + 0 /* not used on EF10 */, evq->common, + &rxq->common); + if (rc != 0) + goto fail_rx_qcreate; + + efx_rx_qenable(rxq->common); + + rc = sa->dp_rx->qstart(rxq->dp, evq->read_ptr); + if (rc != 0) + goto fail_dp_qstart; + + rxq->state |= SFC_RXQ_STARTED; + + if ((sw_index == 0) && !port->isolated) { + rc = sfc_rx_default_rxq_set_filter(sa, rxq); + if (rc != 0) + goto fail_mac_filter_default_rxq_set; + } + + /* It seems to be used by DPDK for debug purposes only ('rte_ether') */ + sa->eth_dev->data->rx_queue_state[sw_index] = + RTE_ETH_QUEUE_STATE_STARTED; + + return 0; + +fail_mac_filter_default_rxq_set: + sa->dp_rx->qstop(rxq->dp, &rxq->evq->read_ptr); + +fail_dp_qstart: + sfc_rx_qflush(sa, sw_index); + +fail_rx_qcreate: + sfc_ev_qstop(evq); + +fail_ev_qstart: + return rc; +} + +void +sfc_rx_qstop(struct sfc_adapter *sa, unsigned int sw_index) +{ + struct sfc_rxq_info *rxq_info; + struct sfc_rxq *rxq; + + sfc_log_init(sa, "sw_index=%u", sw_index); + + SFC_ASSERT(sw_index < sa->rxq_count); + + rxq_info = &sa->rxq_info[sw_index]; + rxq = rxq_info->rxq; + + if (rxq->state == SFC_RXQ_INITIALIZED) + return; + SFC_ASSERT(rxq->state & SFC_RXQ_STARTED); + + /* It seems to be used by DPDK for debug purposes only ('rte_ether') */ + sa->eth_dev->data->rx_queue_state[sw_index] = + RTE_ETH_QUEUE_STATE_STOPPED; + + sa->dp_rx->qstop(rxq->dp, &rxq->evq->read_ptr); + + if (sw_index == 0) + efx_mac_filter_default_rxq_clear(sa->nic); + + sfc_rx_qflush(sa, sw_index); + + rxq->state = SFC_RXQ_INITIALIZED; + + efx_rx_qdestroy(rxq->common); + + sfc_ev_qstop(rxq->evq); +} + +static int +sfc_rx_qcheck_conf(struct sfc_adapter *sa, uint16_t nb_rx_desc, + const struct rte_eth_rxconf *rx_conf) +{ + const uint16_t rx_free_thresh_max = EFX_RXQ_LIMIT(nb_rx_desc); + int rc = 0; + + if (rx_conf->rx_thresh.pthresh != 0 || + rx_conf->rx_thresh.hthresh != 0 || + rx_conf->rx_thresh.wthresh != 0) { + sfc_warn(sa, + "RxQ prefetch/host/writeback thresholds are not supported"); + } + + if (rx_conf->rx_free_thresh > rx_free_thresh_max) { + sfc_err(sa, + "RxQ free threshold too large: %u vs maximum %u", + rx_conf->rx_free_thresh, rx_free_thresh_max); + rc = EINVAL; + } + + if (rx_conf->rx_drop_en == 0) { + sfc_err(sa, "RxQ drop disable is not supported"); + rc = EINVAL; + } + + return rc; +} + +static unsigned int +sfc_rx_mbuf_data_alignment(struct rte_mempool *mb_pool) +{ + uint32_t data_off; + uint32_t order; + + /* The mbuf object itself is always cache line aligned */ + order = rte_bsf32(RTE_CACHE_LINE_SIZE); + + /* Data offset from mbuf object start */ + data_off = sizeof(struct rte_mbuf) + rte_pktmbuf_priv_size(mb_pool) + + RTE_PKTMBUF_HEADROOM; + + order = MIN(order, rte_bsf32(data_off)); + + return 1u << (order - 1); +} + +static uint16_t +sfc_rx_mb_pool_buf_size(struct sfc_adapter *sa, struct rte_mempool *mb_pool) +{ + const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic); + const uint32_t nic_align_start = MAX(1, encp->enc_rx_buf_align_start); + const uint32_t nic_align_end = MAX(1, encp->enc_rx_buf_align_end); + uint16_t buf_size; + unsigned int buf_aligned; + unsigned int start_alignment; + unsigned int end_padding_alignment; + + /* Below it is assumed that both alignments are power of 2 */ + SFC_ASSERT(rte_is_power_of_2(nic_align_start)); + SFC_ASSERT(rte_is_power_of_2(nic_align_end)); + + /* + * mbuf is always cache line aligned, double-check + * that it meets rx buffer start alignment requirements. + */ + + /* Start from mbuf pool data room size */ + buf_size = rte_pktmbuf_data_room_size(mb_pool); + + /* Remove headroom */ + if (buf_size <= RTE_PKTMBUF_HEADROOM) { + sfc_err(sa, + "RxQ mbuf pool %s object data room size %u is smaller than headroom %u", + mb_pool->name, buf_size, RTE_PKTMBUF_HEADROOM); + return 0; + } + buf_size -= RTE_PKTMBUF_HEADROOM; + + /* Calculate guaranteed data start alignment */ + buf_aligned = sfc_rx_mbuf_data_alignment(mb_pool); + + /* Reserve space for start alignment */ + if (buf_aligned < nic_align_start) { + start_alignment = nic_align_start - buf_aligned; + if (buf_size <= start_alignment) { + sfc_err(sa, + "RxQ mbuf pool %s object data room size %u is insufficient for headroom %u and buffer start alignment %u required by NIC", + mb_pool->name, + rte_pktmbuf_data_room_size(mb_pool), + RTE_PKTMBUF_HEADROOM, start_alignment); + return 0; + } + buf_aligned = nic_align_start; + buf_size -= start_alignment; + } else { + start_alignment = 0; + } + + /* Make sure that end padding does not write beyond the buffer */ + if (buf_aligned < nic_align_end) { + /* + * Estimate space which can be lost. If guarnteed buffer + * size is odd, lost space is (nic_align_end - 1). More + * accurate formula is below. + */ + end_padding_alignment = nic_align_end - + MIN(buf_aligned, 1u << (rte_bsf32(buf_size) - 1)); + if (buf_size <= end_padding_alignment) { + sfc_err(sa, + "RxQ mbuf pool %s object data room size %u is insufficient for headroom %u, buffer start alignment %u and end padding alignment %u required by NIC", + mb_pool->name, + rte_pktmbuf_data_room_size(mb_pool), + RTE_PKTMBUF_HEADROOM, start_alignment, + end_padding_alignment); + return 0; + } + buf_size -= end_padding_alignment; + } else { + /* + * Start is aligned the same or better than end, + * just align length. + */ + buf_size = P2ALIGN(buf_size, nic_align_end); + } + + return buf_size; +} + +int +sfc_rx_qinit(struct sfc_adapter *sa, unsigned int sw_index, + uint16_t nb_rx_desc, unsigned int socket_id, + const struct rte_eth_rxconf *rx_conf, + struct rte_mempool *mb_pool) +{ + const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic); + int rc; + uint16_t buf_size; + struct sfc_rxq_info *rxq_info; + struct sfc_evq *evq; + struct sfc_rxq *rxq; + struct sfc_dp_rx_qcreate_info info; + + rc = sfc_rx_qcheck_conf(sa, nb_rx_desc, rx_conf); + if (rc != 0) + goto fail_bad_conf; + + buf_size = sfc_rx_mb_pool_buf_size(sa, mb_pool); + if (buf_size == 0) { + sfc_err(sa, "RxQ %u mbuf pool object size is too small", + sw_index); + rc = EINVAL; + goto fail_bad_conf; + } + + if ((buf_size < sa->port.pdu + encp->enc_rx_prefix_size) && + !sa->eth_dev->data->dev_conf.rxmode.enable_scatter) { + sfc_err(sa, "Rx scatter is disabled and RxQ %u mbuf pool " + "object size is too small", sw_index); + sfc_err(sa, "RxQ %u calculated Rx buffer size is %u vs " + "PDU size %u plus Rx prefix %u bytes", + sw_index, buf_size, (unsigned int)sa->port.pdu, + encp->enc_rx_prefix_size); + rc = EINVAL; + goto fail_bad_conf; + } + + SFC_ASSERT(sw_index < sa->rxq_count); + rxq_info = &sa->rxq_info[sw_index]; + + SFC_ASSERT(nb_rx_desc <= rxq_info->max_entries); + rxq_info->entries = nb_rx_desc; + rxq_info->type = + sa->eth_dev->data->dev_conf.rxmode.enable_scatter ? + EFX_RXQ_TYPE_SCATTER : EFX_RXQ_TYPE_DEFAULT; + + rc = sfc_ev_qinit(sa, SFC_EVQ_TYPE_RX, sw_index, + rxq_info->entries, socket_id, &evq); + if (rc != 0) + goto fail_ev_qinit; + + rc = ENOMEM; + rxq = rte_zmalloc_socket("sfc-rxq", sizeof(*rxq), RTE_CACHE_LINE_SIZE, + socket_id); + if (rxq == NULL) + goto fail_rxq_alloc; + + rxq_info->rxq = rxq; + + rxq->evq = evq; + rxq->hw_index = sw_index; + rxq->refill_threshold = + RTE_MAX(rx_conf->rx_free_thresh, SFC_RX_REFILL_BULK); + rxq->refill_mb_pool = mb_pool; + + rc = sfc_dma_alloc(sa, "rxq", sw_index, EFX_RXQ_SIZE(rxq_info->entries), + socket_id, &rxq->mem); + if (rc != 0) + goto fail_dma_alloc; + + memset(&info, 0, sizeof(info)); + info.refill_mb_pool = rxq->refill_mb_pool; + info.refill_threshold = rxq->refill_threshold; + info.buf_size = buf_size; + info.batch_max = encp->enc_rx_batch_max; + info.prefix_size = encp->enc_rx_prefix_size; + +#if EFSYS_OPT_RX_SCALE + if (sa->hash_support == EFX_RX_HASH_AVAILABLE && sa->rss_channels > 0) + info.flags |= SFC_RXQ_FLAG_RSS_HASH; +#endif + + info.rxq_entries = rxq_info->entries; + info.rxq_hw_ring = rxq->mem.esm_base; + info.evq_entries = rxq_info->entries; + info.evq_hw_ring = evq->mem.esm_base; + info.hw_index = rxq->hw_index; + info.mem_bar = sa->mem_bar.esb_base; + + rc = sa->dp_rx->qcreate(sa->eth_dev->data->port_id, sw_index, + &RTE_ETH_DEV_TO_PCI(sa->eth_dev)->addr, + socket_id, &info, &rxq->dp); + if (rc != 0) + goto fail_dp_rx_qcreate; + + evq->dp_rxq = rxq->dp; + + rxq->state = SFC_RXQ_INITIALIZED; + + rxq_info->deferred_start = (rx_conf->rx_deferred_start != 0); + + return 0; + +fail_dp_rx_qcreate: + sfc_dma_free(sa, &rxq->mem); + +fail_dma_alloc: + rxq_info->rxq = NULL; + rte_free(rxq); + +fail_rxq_alloc: + sfc_ev_qfini(evq); + +fail_ev_qinit: + rxq_info->entries = 0; + +fail_bad_conf: + sfc_log_init(sa, "failed %d", rc); + return rc; +} + +void +sfc_rx_qfini(struct sfc_adapter *sa, unsigned int sw_index) +{ + struct sfc_rxq_info *rxq_info; + struct sfc_rxq *rxq; + + SFC_ASSERT(sw_index < sa->rxq_count); + + rxq_info = &sa->rxq_info[sw_index]; + + rxq = rxq_info->rxq; + SFC_ASSERT(rxq->state == SFC_RXQ_INITIALIZED); + + sa->dp_rx->qdestroy(rxq->dp); + rxq->dp = NULL; + + rxq_info->rxq = NULL; + rxq_info->entries = 0; + + sfc_dma_free(sa, &rxq->mem); + + sfc_ev_qfini(rxq->evq); + rxq->evq = NULL; + + rte_free(rxq); +} + +#if EFSYS_OPT_RX_SCALE +efx_rx_hash_type_t +sfc_rte_to_efx_hash_type(uint64_t rss_hf) +{ + efx_rx_hash_type_t efx_hash_types = 0; + + if ((rss_hf & (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | + ETH_RSS_NONFRAG_IPV4_OTHER)) != 0) + efx_hash_types |= EFX_RX_HASH_IPV4; + + if ((rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) != 0) + efx_hash_types |= EFX_RX_HASH_TCPIPV4; + + if ((rss_hf & (ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | + ETH_RSS_NONFRAG_IPV6_OTHER | ETH_RSS_IPV6_EX)) != 0) + efx_hash_types |= EFX_RX_HASH_IPV6; + + if ((rss_hf & (ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_IPV6_TCP_EX)) != 0) + efx_hash_types |= EFX_RX_HASH_TCPIPV6; + + return efx_hash_types; +} + +uint64_t +sfc_efx_to_rte_hash_type(efx_rx_hash_type_t efx_hash_types) +{ + uint64_t rss_hf = 0; + + if ((efx_hash_types & EFX_RX_HASH_IPV4) != 0) + rss_hf |= (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | + ETH_RSS_NONFRAG_IPV4_OTHER); + + if ((efx_hash_types & EFX_RX_HASH_TCPIPV4) != 0) + rss_hf |= ETH_RSS_NONFRAG_IPV4_TCP; + + if ((efx_hash_types & EFX_RX_HASH_IPV6) != 0) + rss_hf |= (ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | + ETH_RSS_NONFRAG_IPV6_OTHER | ETH_RSS_IPV6_EX); + + if ((efx_hash_types & EFX_RX_HASH_TCPIPV6) != 0) + rss_hf |= (ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_IPV6_TCP_EX); + + return rss_hf; +} +#endif + +#if EFSYS_OPT_RX_SCALE +static int +sfc_rx_rss_config(struct sfc_adapter *sa) +{ + int rc = 0; + + if (sa->rss_channels > 0) { + rc = efx_rx_scale_mode_set(sa->nic, EFX_RSS_CONTEXT_DEFAULT, + EFX_RX_HASHALG_TOEPLITZ, + sa->rss_hash_types, B_TRUE); + if (rc != 0) + goto finish; + + rc = efx_rx_scale_key_set(sa->nic, EFX_RSS_CONTEXT_DEFAULT, + sa->rss_key, + sizeof(sa->rss_key)); + if (rc != 0) + goto finish; + + rc = efx_rx_scale_tbl_set(sa->nic, EFX_RSS_CONTEXT_DEFAULT, + sa->rss_tbl, RTE_DIM(sa->rss_tbl)); + } + +finish: + return rc; +} +#else +static int +sfc_rx_rss_config(__rte_unused struct sfc_adapter *sa) +{ + return 0; +} +#endif + +int +sfc_rx_start(struct sfc_adapter *sa) +{ + unsigned int sw_index; + int rc; + + sfc_log_init(sa, "rxq_count=%u", sa->rxq_count); + + rc = efx_rx_init(sa->nic); + if (rc != 0) + goto fail_rx_init; + + rc = sfc_rx_rss_config(sa); + if (rc != 0) + goto fail_rss_config; + + for (sw_index = 0; sw_index < sa->rxq_count; ++sw_index) { + if ((!sa->rxq_info[sw_index].deferred_start || + sa->rxq_info[sw_index].deferred_started)) { + rc = sfc_rx_qstart(sa, sw_index); + if (rc != 0) + goto fail_rx_qstart; + } + } + + return 0; + +fail_rx_qstart: + while (sw_index-- > 0) + sfc_rx_qstop(sa, sw_index); + +fail_rss_config: + efx_rx_fini(sa->nic); + +fail_rx_init: + sfc_log_init(sa, "failed %d", rc); + return rc; +} + +void +sfc_rx_stop(struct sfc_adapter *sa) +{ + unsigned int sw_index; + + sfc_log_init(sa, "rxq_count=%u", sa->rxq_count); + + sw_index = sa->rxq_count; + while (sw_index-- > 0) { + if (sa->rxq_info[sw_index].rxq != NULL) + sfc_rx_qstop(sa, sw_index); + } + + efx_rx_fini(sa->nic); +} + +static int +sfc_rx_qinit_info(struct sfc_adapter *sa, unsigned int sw_index) +{ + struct sfc_rxq_info *rxq_info = &sa->rxq_info[sw_index]; + unsigned int max_entries; + + max_entries = EFX_RXQ_MAXNDESCS; + SFC_ASSERT(rte_is_power_of_2(max_entries)); + + rxq_info->max_entries = max_entries; + + return 0; +} + +static int +sfc_rx_check_mode(struct sfc_adapter *sa, struct rte_eth_rxmode *rxmode) +{ + int rc = 0; + + switch (rxmode->mq_mode) { + case ETH_MQ_RX_NONE: + /* No special checks are required */ + break; +#if EFSYS_OPT_RX_SCALE + case ETH_MQ_RX_RSS: + if (sa->rss_support == EFX_RX_SCALE_UNAVAILABLE) { + sfc_err(sa, "RSS is not available"); + rc = EINVAL; + } + break; +#endif + default: + sfc_err(sa, "Rx multi-queue mode %u not supported", + rxmode->mq_mode); + rc = EINVAL; + } + + if (rxmode->header_split) { + sfc_err(sa, "Header split on Rx not supported"); + rc = EINVAL; + } + + if (rxmode->hw_vlan_filter) { + sfc_err(sa, "HW VLAN filtering not supported"); + rc = EINVAL; + } + + if (rxmode->hw_vlan_strip) { + sfc_err(sa, "HW VLAN stripping not supported"); + rc = EINVAL; + } + + if (rxmode->hw_vlan_extend) { + sfc_err(sa, + "Q-in-Q HW VLAN stripping not supported"); + rc = EINVAL; + } + + if (!rxmode->hw_strip_crc) { + sfc_warn(sa, + "FCS stripping control not supported - always stripped"); + rxmode->hw_strip_crc = 1; + } + + if (rxmode->enable_scatter && + (~sa->dp_rx->features & SFC_DP_RX_FEAT_SCATTER)) { + sfc_err(sa, "Rx scatter not supported by %s datapath", + sa->dp_rx->dp.name); + rc = EINVAL; + } + + if (rxmode->enable_lro) { + sfc_err(sa, "LRO not supported"); + rc = EINVAL; + } + + return rc; +} + +/** + * Destroy excess queues that are no longer needed after reconfiguration + * or complete close. + */ +static void +sfc_rx_fini_queues(struct sfc_adapter *sa, unsigned int nb_rx_queues) +{ + int sw_index; + + SFC_ASSERT(nb_rx_queues <= sa->rxq_count); + + sw_index = sa->rxq_count; + while (--sw_index >= (int)nb_rx_queues) { + if (sa->rxq_info[sw_index].rxq != NULL) + sfc_rx_qfini(sa, sw_index); + } + + sa->rxq_count = nb_rx_queues; +} + +/** + * Initialize Rx subsystem. + * + * Called at device (re)configuration stage when number of receive queues is + * specified together with other device level receive configuration. + * + * It should be used to allocate NUMA-unaware resources. + */ +int +sfc_rx_configure(struct sfc_adapter *sa) +{ + struct rte_eth_conf *dev_conf = &sa->eth_dev->data->dev_conf; + const unsigned int nb_rx_queues = sa->eth_dev->data->nb_rx_queues; + int rc; + + sfc_log_init(sa, "nb_rx_queues=%u (old %u)", + nb_rx_queues, sa->rxq_count); + + rc = sfc_rx_check_mode(sa, &dev_conf->rxmode); + if (rc != 0) + goto fail_check_mode; + + if (nb_rx_queues == sa->rxq_count) + goto done; + + if (sa->rxq_info == NULL) { + rc = ENOMEM; + sa->rxq_info = rte_calloc_socket("sfc-rxqs", nb_rx_queues, + sizeof(sa->rxq_info[0]), 0, + sa->socket_id); + if (sa->rxq_info == NULL) + goto fail_rxqs_alloc; + } else { + struct sfc_rxq_info *new_rxq_info; + + if (nb_rx_queues < sa->rxq_count) + sfc_rx_fini_queues(sa, nb_rx_queues); + + rc = ENOMEM; + new_rxq_info = + rte_realloc(sa->rxq_info, + nb_rx_queues * sizeof(sa->rxq_info[0]), 0); + if (new_rxq_info == NULL && nb_rx_queues > 0) + goto fail_rxqs_realloc; + + sa->rxq_info = new_rxq_info; + if (nb_rx_queues > sa->rxq_count) + memset(&sa->rxq_info[sa->rxq_count], 0, + (nb_rx_queues - sa->rxq_count) * + sizeof(sa->rxq_info[0])); + } + + while (sa->rxq_count < nb_rx_queues) { + rc = sfc_rx_qinit_info(sa, sa->rxq_count); + if (rc != 0) + goto fail_rx_qinit_info; + + sa->rxq_count++; + } + +#if EFSYS_OPT_RX_SCALE + sa->rss_channels = (dev_conf->rxmode.mq_mode == ETH_MQ_RX_RSS) ? + MIN(sa->rxq_count, EFX_MAXRSS) : 0; + + if (sa->rss_channels > 0) { + unsigned int sw_index; + + for (sw_index = 0; sw_index < EFX_RSS_TBL_SIZE; ++sw_index) + sa->rss_tbl[sw_index] = sw_index % sa->rss_channels; + } +#endif + +done: + return 0; + +fail_rx_qinit_info: +fail_rxqs_realloc: +fail_rxqs_alloc: + sfc_rx_close(sa); + +fail_check_mode: + sfc_log_init(sa, "failed %d", rc); + return rc; +} + +/** + * Shutdown Rx subsystem. + * + * Called at device close stage, for example, before device shutdown. + */ +void +sfc_rx_close(struct sfc_adapter *sa) +{ + sfc_rx_fini_queues(sa, 0); + + sa->rss_channels = 0; + + rte_free(sa->rxq_info); + sa->rxq_info = NULL; +} diff --git a/dpdk/drivers/net/sfc/sfc_rx.h b/dpdk/drivers/net/sfc/sfc_rx.h new file mode 100644 index 00000000..9e6282ea --- /dev/null +++ b/dpdk/drivers/net/sfc/sfc_rx.h @@ -0,0 +1,180 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2016-2017 Solarflare Communications Inc. + * All rights reserved. + * + * This software was jointly developed between OKTET Labs (under contract + * for Solarflare) and Solarflare Communications, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _SFC_RX_H +#define _SFC_RX_H + +#include +#include +#include + +#include "efx.h" + +#include "sfc_dp_rx.h" + +#ifdef __cplusplus +extern "C" { +#endif + +struct sfc_adapter; +struct sfc_evq; + +/** + * Software Rx descriptor information associated with hardware Rx + * descriptor. + */ +struct sfc_efx_rx_sw_desc { + struct rte_mbuf *mbuf; + unsigned int flags; + unsigned int size; +}; + +/** Receive queue state bits */ +enum sfc_rxq_state_bit { + SFC_RXQ_INITIALIZED_BIT = 0, +#define SFC_RXQ_INITIALIZED (1 << SFC_RXQ_INITIALIZED_BIT) + SFC_RXQ_STARTED_BIT, +#define SFC_RXQ_STARTED (1 << SFC_RXQ_STARTED_BIT) + SFC_RXQ_FLUSHING_BIT, +#define SFC_RXQ_FLUSHING (1 << SFC_RXQ_FLUSHING_BIT) + SFC_RXQ_FLUSHED_BIT, +#define SFC_RXQ_FLUSHED (1 << SFC_RXQ_FLUSHED_BIT) + SFC_RXQ_FLUSH_FAILED_BIT, +#define SFC_RXQ_FLUSH_FAILED (1 << SFC_RXQ_FLUSH_FAILED_BIT) +}; + +/** + * Receive queue control information. + * Allocated on the socket specified on the queue setup. + */ +struct sfc_rxq { + struct sfc_evq *evq; + efx_rxq_t *common; + efsys_mem_t mem; + unsigned int hw_index; + unsigned int refill_threshold; + struct rte_mempool *refill_mb_pool; + struct sfc_dp_rxq *dp; + unsigned int state; +}; + +static inline unsigned int +sfc_rxq_sw_index_by_hw_index(unsigned int hw_index) +{ + return hw_index; +} + +static inline unsigned int +sfc_rxq_sw_index(const struct sfc_rxq *rxq) +{ + return sfc_rxq_sw_index_by_hw_index(rxq->hw_index); +} + +struct sfc_rxq *sfc_rxq_by_dp_rxq(const struct sfc_dp_rxq *dp_rxq); + +/** + * Receive queue information used on libefx-based data path. + * Allocated on the socket specified on the queue setup. + */ +struct sfc_efx_rxq { + /* Used on data path */ + struct sfc_evq *evq; + unsigned int flags; +#define SFC_EFX_RXQ_FLAG_STARTED 0x1 +#define SFC_EFX_RXQ_FLAG_RUNNING 0x2 +#define SFC_EFX_RXQ_FLAG_RSS_HASH 0x4 + unsigned int ptr_mask; + unsigned int pending; + unsigned int completed; + uint16_t batch_max; + uint16_t prefix_size; + struct sfc_efx_rx_sw_desc *sw_desc; + + /* Used on refill */ + unsigned int added; + unsigned int pushed; + unsigned int refill_threshold; + uint16_t buf_size; + struct rte_mempool *refill_mb_pool; + efx_rxq_t *common; + + /* Datapath receive queue anchor */ + struct sfc_dp_rxq dp; +}; + +static inline struct sfc_efx_rxq * +sfc_efx_rxq_by_dp_rxq(struct sfc_dp_rxq *dp_rxq) +{ + return container_of(dp_rxq, struct sfc_efx_rxq, dp); +} + +/** + * Receive queue information used during setup/release only. + * Allocated on the same socket as adapter data. + */ +struct sfc_rxq_info { + unsigned int max_entries; + unsigned int entries; + efx_rxq_type_t type; + struct sfc_rxq *rxq; + boolean_t deferred_start; + boolean_t deferred_started; +}; + +int sfc_rx_configure(struct sfc_adapter *sa); +void sfc_rx_close(struct sfc_adapter *sa); +int sfc_rx_start(struct sfc_adapter *sa); +void sfc_rx_stop(struct sfc_adapter *sa); + +int sfc_rx_qinit(struct sfc_adapter *sa, unsigned int rx_queue_id, + uint16_t nb_rx_desc, unsigned int socket_id, + const struct rte_eth_rxconf *rx_conf, + struct rte_mempool *mb_pool); +void sfc_rx_qfini(struct sfc_adapter *sa, unsigned int sw_index); +int sfc_rx_qstart(struct sfc_adapter *sa, unsigned int sw_index); +void sfc_rx_qstop(struct sfc_adapter *sa, unsigned int sw_index); + +void sfc_rx_qflush_done(struct sfc_rxq *rxq); +void sfc_rx_qflush_failed(struct sfc_rxq *rxq); + +unsigned int sfc_rx_qdesc_npending(struct sfc_adapter *sa, + unsigned int sw_index); +int sfc_rx_qdesc_done(struct sfc_dp_rxq *dp_rxq, unsigned int offset); + +#if EFSYS_OPT_RX_SCALE +efx_rx_hash_type_t sfc_rte_to_efx_hash_type(uint64_t rss_hf); +uint64_t sfc_efx_to_rte_hash_type(efx_rx_hash_type_t efx_hash_types); +#endif + +#ifdef __cplusplus +} +#endif +#endif /* _SFC_RX_H */ diff --git a/dpdk/drivers/net/sfc/sfc_tso.c b/dpdk/drivers/net/sfc/sfc_tso.c new file mode 100644 index 00000000..2e7b595b --- /dev/null +++ b/dpdk/drivers/net/sfc/sfc_tso.c @@ -0,0 +1,201 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2016-2017 Solarflare Communications Inc. + * All rights reserved. + * + * This software was jointly developed between OKTET Labs (under contract + * for Solarflare) and Solarflare Communications, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include + +#include "sfc.h" +#include "sfc_debug.h" +#include "sfc_tx.h" +#include "sfc_ev.h" + +/** Standard TSO header length */ +#define SFC_TSOH_STD_LEN 256 + +/** The number of TSO option descriptors that precede the packet descriptors */ +#define SFC_TSO_OPDESCS_IDX_SHIFT 2 + +int +sfc_efx_tso_alloc_tsoh_objs(struct sfc_efx_tx_sw_desc *sw_ring, + unsigned int txq_entries, unsigned int socket_id) +{ + unsigned int i; + + for (i = 0; i < txq_entries; ++i) { + sw_ring[i].tsoh = rte_malloc_socket("sfc-efx-txq-tsoh-obj", + SFC_TSOH_STD_LEN, + RTE_CACHE_LINE_SIZE, + socket_id); + if (sw_ring[i].tsoh == NULL) + goto fail_alloc_tsoh_objs; + } + + return 0; + +fail_alloc_tsoh_objs: + while (i > 0) + rte_free(sw_ring[--i].tsoh); + + return ENOMEM; +} + +void +sfc_efx_tso_free_tsoh_objs(struct sfc_efx_tx_sw_desc *sw_ring, + unsigned int txq_entries) +{ + unsigned int i; + + for (i = 0; i < txq_entries; ++i) { + rte_free(sw_ring[i].tsoh); + sw_ring[i].tsoh = NULL; + } +} + +static void +sfc_efx_tso_prepare_header(struct sfc_efx_txq *txq, struct rte_mbuf **in_seg, + size_t *in_off, unsigned int idx, size_t bytes_left) +{ + struct rte_mbuf *m = *in_seg; + size_t bytes_to_copy = 0; + uint8_t *tsoh = txq->sw_ring[idx & txq->ptr_mask].tsoh; + + do { + bytes_to_copy = MIN(bytes_left, m->data_len); + + rte_memcpy(tsoh, rte_pktmbuf_mtod(m, uint8_t *), + bytes_to_copy); + + bytes_left -= bytes_to_copy; + tsoh += bytes_to_copy; + + if (bytes_left > 0) { + m = m->next; + SFC_ASSERT(m != NULL); + } + } while (bytes_left > 0); + + if (bytes_to_copy == m->data_len) { + *in_seg = m->next; + *in_off = 0; + } else { + *in_seg = m; + *in_off = bytes_to_copy; + } +} + +int +sfc_efx_tso_do(struct sfc_efx_txq *txq, unsigned int idx, + struct rte_mbuf **in_seg, size_t *in_off, efx_desc_t **pend, + unsigned int *pkt_descs, size_t *pkt_len) +{ + uint8_t *tsoh; + const struct tcp_hdr *th; + efsys_dma_addr_t header_paddr; + uint16_t packet_id; + uint32_t sent_seq; + struct rte_mbuf *m = *in_seg; + size_t nh_off = m->l2_len; /* IP header offset */ + size_t tcph_off = m->l2_len + m->l3_len; /* TCP header offset */ + size_t header_len = m->l2_len + m->l3_len + m->l4_len; + const efx_nic_cfg_t *encp = efx_nic_cfg_get(txq->evq->sa->nic); + + idx += SFC_TSO_OPDESCS_IDX_SHIFT; + + /* Packets which have too big headers should be discarded */ + if (unlikely(header_len > SFC_TSOH_STD_LEN)) + return EMSGSIZE; + + /* + * The TCP header must start at most 208 bytes into the frame. + * If it starts later than this then the NIC won't realise + * it's a TCP packet and TSO edits won't be applied + */ + if (unlikely(tcph_off > encp->enc_tx_tso_tcp_header_offset_limit)) + return EMSGSIZE; + + header_paddr = rte_pktmbuf_iova(m); + + /* + * Sometimes headers may be split across multiple mbufs. In such cases + * we need to glue those pieces and store them in some temporary place. + * Also, packet headers must be contiguous in memory, so that + * they can be referred to with a single DMA descriptor. EF10 has no + * limitations on address boundaries crossing by DMA descriptor data. + */ + if (m->data_len < header_len) { + sfc_efx_tso_prepare_header(txq, in_seg, in_off, idx, + header_len); + tsoh = txq->sw_ring[idx & txq->ptr_mask].tsoh; + + header_paddr = rte_malloc_virt2iova((void *)tsoh); + } else { + if (m->data_len == header_len) { + *in_off = 0; + *in_seg = m->next; + } else { + *in_off = header_len; + } + + tsoh = rte_pktmbuf_mtod(m, uint8_t *); + } + + /* Handle IP header */ + if (m->ol_flags & PKT_TX_IPV4) { + const struct ipv4_hdr *iphe4; + + iphe4 = (const struct ipv4_hdr *)(tsoh + nh_off); + rte_memcpy(&packet_id, &iphe4->packet_id, sizeof(uint16_t)); + packet_id = rte_be_to_cpu_16(packet_id); + } else if (m->ol_flags & PKT_TX_IPV6) { + packet_id = 0; + } else { + return EINVAL; + } + + /* Handle TCP header */ + th = (const struct tcp_hdr *)(tsoh + tcph_off); + + rte_memcpy(&sent_seq, &th->sent_seq, sizeof(uint32_t)); + sent_seq = rte_be_to_cpu_32(sent_seq); + + efx_tx_qdesc_tso2_create(txq->common, packet_id, sent_seq, m->tso_segsz, + *pend, EFX_TX_FATSOV2_OPT_NDESCS); + + *pend += EFX_TX_FATSOV2_OPT_NDESCS; + *pkt_descs += EFX_TX_FATSOV2_OPT_NDESCS; + + efx_tx_qdesc_dma_create(txq->common, header_paddr, header_len, + B_FALSE, (*pend)++); + (*pkt_descs)++; + *pkt_len -= header_len; + + return 0; +} diff --git a/dpdk/drivers/net/sfc/sfc_tweak.h b/dpdk/drivers/net/sfc/sfc_tweak.h new file mode 100644 index 00000000..fd2f75c3 --- /dev/null +++ b/dpdk/drivers/net/sfc/sfc_tweak.h @@ -0,0 +1,59 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2016-2017 Solarflare Communications Inc. + * All rights reserved. + * + * This software was jointly developed between OKTET Labs (under contract + * for Solarflare) and Solarflare Communications, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _SFC_TWEAK_H_ +#define _SFC_TWEAK_H_ + +/* + * The header is intended to collect defines/constants which could be + * tweaked to improve the PMD performance characteristics depending on + * the usecase or requirements (CPU load, packet rate, latency). + */ + +/** + * Number of Rx descriptors in the bulk submitted on Rx ring refill. + */ +#define SFC_RX_REFILL_BULK (RTE_CACHE_LINE_SIZE / sizeof(efx_qword_t)) + +/** + * Make the transmit path reap at least one time per a burst; + * this improves cache locality because the same mbufs may be used to send + * subsequent bursts in certain cases because of well-timed reap + */ +#define SFC_TX_XMIT_PKTS_REAP_AT_LEAST_ONCE 0 + +/** Default free threshold follows recommendations from DPDK documentation */ +#define SFC_TX_DEFAULT_FREE_THRESH 32 + +/** Number of mbufs to be freed in bulk in a single call */ +#define SFC_TX_REAP_BULK_SIZE 32 + +#endif /* _SFC_TWEAK_H_ */ diff --git a/dpdk/drivers/net/sfc/sfc_tx.c b/dpdk/drivers/net/sfc/sfc_tx.c new file mode 100644 index 00000000..d1320f46 --- /dev/null +++ b/dpdk/drivers/net/sfc/sfc_tx.c @@ -0,0 +1,1050 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2016-2017 Solarflare Communications Inc. + * All rights reserved. + * + * This software was jointly developed between OKTET Labs (under contract + * for Solarflare) and Solarflare Communications, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "sfc.h" +#include "sfc_debug.h" +#include "sfc_log.h" +#include "sfc_ev.h" +#include "sfc_tx.h" +#include "sfc_tweak.h" +#include "sfc_kvargs.h" + +/* + * Maximum number of TX queue flush attempts in case of + * failure or flush timeout + */ +#define SFC_TX_QFLUSH_ATTEMPTS (3) + +/* + * Time to wait between event queue polling attempts when waiting for TX + * queue flush done or flush failed events + */ +#define SFC_TX_QFLUSH_POLL_WAIT_MS (1) + +/* + * Maximum number of event queue polling attempts when waiting for TX queue + * flush done or flush failed events; it defines TX queue flush attempt timeout + * together with SFC_TX_QFLUSH_POLL_WAIT_MS + */ +#define SFC_TX_QFLUSH_POLL_ATTEMPTS (2000) + +static int +sfc_tx_qcheck_conf(struct sfc_adapter *sa, uint16_t nb_tx_desc, + const struct rte_eth_txconf *tx_conf) +{ + unsigned int flags = tx_conf->txq_flags; + const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic); + int rc = 0; + + if (tx_conf->tx_rs_thresh != 0) { + sfc_err(sa, "RS bit in transmit descriptor is not supported"); + rc = EINVAL; + } + + if (tx_conf->tx_free_thresh > EFX_TXQ_LIMIT(nb_tx_desc)) { + sfc_err(sa, + "TxQ free threshold too large: %u vs maximum %u", + tx_conf->tx_free_thresh, EFX_TXQ_LIMIT(nb_tx_desc)); + rc = EINVAL; + } + + if (tx_conf->tx_thresh.pthresh != 0 || + tx_conf->tx_thresh.hthresh != 0 || + tx_conf->tx_thresh.wthresh != 0) { + sfc_warn(sa, + "prefetch/host/writeback thresholds are not supported"); + } + + if (((flags & ETH_TXQ_FLAGS_NOMULTSEGS) == 0) && + (~sa->dp_tx->features & SFC_DP_TX_FEAT_MULTI_SEG)) { + sfc_err(sa, "Multi-segment is not supported by %s datapath", + sa->dp_tx->dp.name); + rc = EINVAL; + } + + if (((flags & ETH_TXQ_FLAGS_NOMULTMEMP) == 0) && + (~sa->dp_tx->features & SFC_DP_TX_FEAT_MULTI_POOL)) { + sfc_err(sa, "multi-mempool is not supported by %s datapath", + sa->dp_tx->dp.name); + rc = EINVAL; + } + + if (((flags & ETH_TXQ_FLAGS_NOREFCOUNT) == 0) && + (~sa->dp_tx->features & SFC_DP_TX_FEAT_REFCNT)) { + sfc_err(sa, + "mbuf reference counters are neglected by %s datapath", + sa->dp_tx->dp.name); + rc = EINVAL; + } + + if ((flags & ETH_TXQ_FLAGS_NOVLANOFFL) == 0) { + if (!encp->enc_hw_tx_insert_vlan_enabled) { + sfc_err(sa, "VLAN offload is not supported"); + rc = EINVAL; + } else if (~sa->dp_tx->features & SFC_DP_TX_FEAT_VLAN_INSERT) { + sfc_err(sa, + "VLAN offload is not supported by %s datapath", + sa->dp_tx->dp.name); + rc = EINVAL; + } + } + + if ((flags & ETH_TXQ_FLAGS_NOXSUMSCTP) == 0) { + sfc_err(sa, "SCTP offload is not supported"); + rc = EINVAL; + } + + /* We either perform both TCP and UDP offload, or no offload at all */ + if (((flags & ETH_TXQ_FLAGS_NOXSUMTCP) == 0) != + ((flags & ETH_TXQ_FLAGS_NOXSUMUDP) == 0)) { + sfc_err(sa, "TCP and UDP offloads can't be set independently"); + rc = EINVAL; + } + + return rc; +} + +void +sfc_tx_qflush_done(struct sfc_txq *txq) +{ + txq->state |= SFC_TXQ_FLUSHED; + txq->state &= ~SFC_TXQ_FLUSHING; +} + +int +sfc_tx_qinit(struct sfc_adapter *sa, unsigned int sw_index, + uint16_t nb_tx_desc, unsigned int socket_id, + const struct rte_eth_txconf *tx_conf) +{ + const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic); + struct sfc_txq_info *txq_info; + struct sfc_evq *evq; + struct sfc_txq *txq; + int rc = 0; + struct sfc_dp_tx_qcreate_info info; + + sfc_log_init(sa, "TxQ = %u", sw_index); + + rc = sfc_tx_qcheck_conf(sa, nb_tx_desc, tx_conf); + if (rc != 0) + goto fail_bad_conf; + + SFC_ASSERT(sw_index < sa->txq_count); + txq_info = &sa->txq_info[sw_index]; + + SFC_ASSERT(nb_tx_desc <= sa->txq_max_entries); + txq_info->entries = nb_tx_desc; + + rc = sfc_ev_qinit(sa, SFC_EVQ_TYPE_TX, sw_index, + txq_info->entries, socket_id, &evq); + if (rc != 0) + goto fail_ev_qinit; + + rc = ENOMEM; + txq = rte_zmalloc_socket("sfc-txq", sizeof(*txq), 0, socket_id); + if (txq == NULL) + goto fail_txq_alloc; + + txq_info->txq = txq; + + txq->hw_index = sw_index; + txq->evq = evq; + txq->free_thresh = + (tx_conf->tx_free_thresh) ? tx_conf->tx_free_thresh : + SFC_TX_DEFAULT_FREE_THRESH; + txq->flags = tx_conf->txq_flags; + + rc = sfc_dma_alloc(sa, "txq", sw_index, EFX_TXQ_SIZE(txq_info->entries), + socket_id, &txq->mem); + if (rc != 0) + goto fail_dma_alloc; + + memset(&info, 0, sizeof(info)); + info.free_thresh = txq->free_thresh; + info.flags = tx_conf->txq_flags; + info.txq_entries = txq_info->entries; + info.dma_desc_size_max = encp->enc_tx_dma_desc_size_max; + info.txq_hw_ring = txq->mem.esm_base; + info.evq_entries = txq_info->entries; + info.evq_hw_ring = evq->mem.esm_base; + info.hw_index = txq->hw_index; + info.mem_bar = sa->mem_bar.esb_base; + + rc = sa->dp_tx->qcreate(sa->eth_dev->data->port_id, sw_index, + &RTE_ETH_DEV_TO_PCI(sa->eth_dev)->addr, + socket_id, &info, &txq->dp); + if (rc != 0) + goto fail_dp_tx_qinit; + + evq->dp_txq = txq->dp; + + txq->state = SFC_TXQ_INITIALIZED; + + txq_info->deferred_start = (tx_conf->tx_deferred_start != 0); + + return 0; + +fail_dp_tx_qinit: + sfc_dma_free(sa, &txq->mem); + +fail_dma_alloc: + txq_info->txq = NULL; + rte_free(txq); + +fail_txq_alloc: + sfc_ev_qfini(evq); + +fail_ev_qinit: + txq_info->entries = 0; + +fail_bad_conf: + sfc_log_init(sa, "failed (TxQ = %u, rc = %d)", sw_index, rc); + return rc; +} + +void +sfc_tx_qfini(struct sfc_adapter *sa, unsigned int sw_index) +{ + struct sfc_txq_info *txq_info; + struct sfc_txq *txq; + + sfc_log_init(sa, "TxQ = %u", sw_index); + + SFC_ASSERT(sw_index < sa->txq_count); + txq_info = &sa->txq_info[sw_index]; + + txq = txq_info->txq; + SFC_ASSERT(txq != NULL); + SFC_ASSERT(txq->state == SFC_TXQ_INITIALIZED); + + sa->dp_tx->qdestroy(txq->dp); + txq->dp = NULL; + + txq_info->txq = NULL; + txq_info->entries = 0; + + sfc_dma_free(sa, &txq->mem); + + sfc_ev_qfini(txq->evq); + txq->evq = NULL; + + rte_free(txq); +} + +static int +sfc_tx_qinit_info(struct sfc_adapter *sa, unsigned int sw_index) +{ + sfc_log_init(sa, "TxQ = %u", sw_index); + + return 0; +} + +static int +sfc_tx_check_mode(struct sfc_adapter *sa, const struct rte_eth_txmode *txmode) +{ + int rc = 0; + + switch (txmode->mq_mode) { + case ETH_MQ_TX_NONE: + break; + default: + sfc_err(sa, "Tx multi-queue mode %u not supported", + txmode->mq_mode); + rc = EINVAL; + } + + /* + * These features are claimed to be i40e-specific, + * but it does make sense to double-check their absence + */ + if (txmode->hw_vlan_reject_tagged) { + sfc_err(sa, "Rejecting tagged packets not supported"); + rc = EINVAL; + } + + if (txmode->hw_vlan_reject_untagged) { + sfc_err(sa, "Rejecting untagged packets not supported"); + rc = EINVAL; + } + + if (txmode->hw_vlan_insert_pvid) { + sfc_err(sa, "Port-based VLAN insertion not supported"); + rc = EINVAL; + } + + return rc; +} + +/** + * Destroy excess queues that are no longer needed after reconfiguration + * or complete close. + */ +static void +sfc_tx_fini_queues(struct sfc_adapter *sa, unsigned int nb_tx_queues) +{ + int sw_index; + + SFC_ASSERT(nb_tx_queues <= sa->txq_count); + + sw_index = sa->txq_count; + while (--sw_index >= (int)nb_tx_queues) { + if (sa->txq_info[sw_index].txq != NULL) + sfc_tx_qfini(sa, sw_index); + } + + sa->txq_count = nb_tx_queues; +} + +int +sfc_tx_configure(struct sfc_adapter *sa) +{ + const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic); + const struct rte_eth_conf *dev_conf = &sa->eth_dev->data->dev_conf; + const unsigned int nb_tx_queues = sa->eth_dev->data->nb_tx_queues; + int rc = 0; + + sfc_log_init(sa, "nb_tx_queues=%u (old %u)", + nb_tx_queues, sa->txq_count); + + /* + * The datapath implementation assumes absence of boundary + * limits on Tx DMA descriptors. Addition of these checks on + * datapath would simply make the datapath slower. + */ + if (encp->enc_tx_dma_desc_boundary != 0) { + rc = ENOTSUP; + goto fail_tx_dma_desc_boundary; + } + + rc = sfc_tx_check_mode(sa, &dev_conf->txmode); + if (rc != 0) + goto fail_check_mode; + + if (nb_tx_queues == sa->txq_count) + goto done; + + if (sa->txq_info == NULL) { + sa->txq_info = rte_calloc_socket("sfc-txqs", nb_tx_queues, + sizeof(sa->txq_info[0]), 0, + sa->socket_id); + if (sa->txq_info == NULL) + goto fail_txqs_alloc; + } else { + struct sfc_txq_info *new_txq_info; + + if (nb_tx_queues < sa->txq_count) + sfc_tx_fini_queues(sa, nb_tx_queues); + + new_txq_info = + rte_realloc(sa->txq_info, + nb_tx_queues * sizeof(sa->txq_info[0]), 0); + if (new_txq_info == NULL && nb_tx_queues > 0) + goto fail_txqs_realloc; + + sa->txq_info = new_txq_info; + if (nb_tx_queues > sa->txq_count) + memset(&sa->txq_info[sa->txq_count], 0, + (nb_tx_queues - sa->txq_count) * + sizeof(sa->txq_info[0])); + } + + while (sa->txq_count < nb_tx_queues) { + rc = sfc_tx_qinit_info(sa, sa->txq_count); + if (rc != 0) + goto fail_tx_qinit_info; + + sa->txq_count++; + } + +done: + return 0; + +fail_tx_qinit_info: +fail_txqs_realloc: +fail_txqs_alloc: + sfc_tx_close(sa); + +fail_check_mode: +fail_tx_dma_desc_boundary: + sfc_log_init(sa, "failed (rc = %d)", rc); + return rc; +} + +void +sfc_tx_close(struct sfc_adapter *sa) +{ + sfc_tx_fini_queues(sa, 0); + + rte_free(sa->txq_info); + sa->txq_info = NULL; +} + +int +sfc_tx_qstart(struct sfc_adapter *sa, unsigned int sw_index) +{ + struct rte_eth_dev_data *dev_data; + struct sfc_txq_info *txq_info; + struct sfc_txq *txq; + struct sfc_evq *evq; + uint16_t flags; + unsigned int desc_index; + int rc = 0; + + sfc_log_init(sa, "TxQ = %u", sw_index); + + SFC_ASSERT(sw_index < sa->txq_count); + txq_info = &sa->txq_info[sw_index]; + + txq = txq_info->txq; + + SFC_ASSERT(txq->state == SFC_TXQ_INITIALIZED); + + evq = txq->evq; + + rc = sfc_ev_qstart(evq, sfc_evq_index_by_txq_sw_index(sa, sw_index)); + if (rc != 0) + goto fail_ev_qstart; + + /* + * It seems that DPDK has no controls regarding IPv4 offloads, + * hence, we always enable it here + */ + if ((txq->flags & ETH_TXQ_FLAGS_NOXSUMTCP) || + (txq->flags & ETH_TXQ_FLAGS_NOXSUMUDP)) { + flags = EFX_TXQ_CKSUM_IPV4; + } else { + flags = EFX_TXQ_CKSUM_IPV4 | EFX_TXQ_CKSUM_TCPUDP; + + if (sa->tso) + flags |= EFX_TXQ_FATSOV2; + } + + rc = efx_tx_qcreate(sa->nic, sw_index, 0, &txq->mem, + txq_info->entries, 0 /* not used on EF10 */, + flags, evq->common, + &txq->common, &desc_index); + if (rc != 0) { + if (sa->tso && (rc == ENOSPC)) + sfc_err(sa, "ran out of TSO contexts"); + + goto fail_tx_qcreate; + } + + efx_tx_qenable(txq->common); + + txq->state |= SFC_TXQ_STARTED; + + rc = sa->dp_tx->qstart(txq->dp, evq->read_ptr, desc_index); + if (rc != 0) + goto fail_dp_qstart; + + /* + * It seems to be used by DPDK for debug purposes only ('rte_ether') + */ + dev_data = sa->eth_dev->data; + dev_data->tx_queue_state[sw_index] = RTE_ETH_QUEUE_STATE_STARTED; + + return 0; + +fail_dp_qstart: + txq->state = SFC_TXQ_INITIALIZED; + efx_tx_qdestroy(txq->common); + +fail_tx_qcreate: + sfc_ev_qstop(evq); + +fail_ev_qstart: + return rc; +} + +void +sfc_tx_qstop(struct sfc_adapter *sa, unsigned int sw_index) +{ + struct rte_eth_dev_data *dev_data; + struct sfc_txq_info *txq_info; + struct sfc_txq *txq; + unsigned int retry_count; + unsigned int wait_count; + int rc; + + sfc_log_init(sa, "TxQ = %u", sw_index); + + SFC_ASSERT(sw_index < sa->txq_count); + txq_info = &sa->txq_info[sw_index]; + + txq = txq_info->txq; + + if (txq->state == SFC_TXQ_INITIALIZED) + return; + + SFC_ASSERT(txq->state & SFC_TXQ_STARTED); + + sa->dp_tx->qstop(txq->dp, &txq->evq->read_ptr); + + /* + * Retry TX queue flushing in case of flush failed or + * timeout; in the worst case it can delay for 6 seconds + */ + for (retry_count = 0; + ((txq->state & SFC_TXQ_FLUSHED) == 0) && + (retry_count < SFC_TX_QFLUSH_ATTEMPTS); + ++retry_count) { + rc = efx_tx_qflush(txq->common); + if (rc != 0) { + txq->state |= (rc == EALREADY) ? + SFC_TXQ_FLUSHED : SFC_TXQ_FLUSH_FAILED; + break; + } + + /* + * Wait for TX queue flush done or flush failed event at least + * SFC_TX_QFLUSH_POLL_WAIT_MS milliseconds and not more + * than 2 seconds (SFC_TX_QFLUSH_POLL_WAIT_MS multiplied + * by SFC_TX_QFLUSH_POLL_ATTEMPTS) + */ + wait_count = 0; + do { + rte_delay_ms(SFC_TX_QFLUSH_POLL_WAIT_MS); + sfc_ev_qpoll(txq->evq); + } while ((txq->state & SFC_TXQ_FLUSHING) && + wait_count++ < SFC_TX_QFLUSH_POLL_ATTEMPTS); + + if (txq->state & SFC_TXQ_FLUSHING) + sfc_err(sa, "TxQ %u flush timed out", sw_index); + + if (txq->state & SFC_TXQ_FLUSHED) + sfc_info(sa, "TxQ %u flushed", sw_index); + } + + sa->dp_tx->qreap(txq->dp); + + txq->state = SFC_TXQ_INITIALIZED; + + efx_tx_qdestroy(txq->common); + + sfc_ev_qstop(txq->evq); + + /* + * It seems to be used by DPDK for debug purposes only ('rte_ether') + */ + dev_data = sa->eth_dev->data; + dev_data->tx_queue_state[sw_index] = RTE_ETH_QUEUE_STATE_STOPPED; +} + +int +sfc_tx_start(struct sfc_adapter *sa) +{ + unsigned int sw_index; + int rc = 0; + + sfc_log_init(sa, "txq_count = %u", sa->txq_count); + + if (sa->tso) { + if (!efx_nic_cfg_get(sa->nic)->enc_fw_assisted_tso_v2_enabled) { + sfc_warn(sa, "TSO support was unable to be restored"); + sa->tso = B_FALSE; + } + } + + rc = efx_tx_init(sa->nic); + if (rc != 0) + goto fail_efx_tx_init; + + for (sw_index = 0; sw_index < sa->txq_count; ++sw_index) { + if (!(sa->txq_info[sw_index].deferred_start) || + sa->txq_info[sw_index].deferred_started) { + rc = sfc_tx_qstart(sa, sw_index); + if (rc != 0) + goto fail_tx_qstart; + } + } + + return 0; + +fail_tx_qstart: + while (sw_index-- > 0) + sfc_tx_qstop(sa, sw_index); + + efx_tx_fini(sa->nic); + +fail_efx_tx_init: + sfc_log_init(sa, "failed (rc = %d)", rc); + return rc; +} + +void +sfc_tx_stop(struct sfc_adapter *sa) +{ + unsigned int sw_index; + + sfc_log_init(sa, "txq_count = %u", sa->txq_count); + + sw_index = sa->txq_count; + while (sw_index-- > 0) { + if (sa->txq_info[sw_index].txq != NULL) + sfc_tx_qstop(sa, sw_index); + } + + efx_tx_fini(sa->nic); +} + +static void +sfc_efx_tx_reap(struct sfc_efx_txq *txq) +{ + unsigned int completed; + + sfc_ev_qpoll(txq->evq); + + for (completed = txq->completed; + completed != txq->pending; completed++) { + struct sfc_efx_tx_sw_desc *txd; + + txd = &txq->sw_ring[completed & txq->ptr_mask]; + + if (txd->mbuf != NULL) { + rte_pktmbuf_free(txd->mbuf); + txd->mbuf = NULL; + } + } + + txq->completed = completed; +} + +/* + * The function is used to insert or update VLAN tag; + * the firmware has state of the firmware tag to insert per TxQ + * (controlled by option descriptors), hence, if the tag of the + * packet to be sent is different from one remembered by the firmware, + * the function will update it + */ +static unsigned int +sfc_efx_tx_maybe_insert_tag(struct sfc_efx_txq *txq, struct rte_mbuf *m, + efx_desc_t **pend) +{ + uint16_t this_tag = ((m->ol_flags & PKT_TX_VLAN_PKT) ? + m->vlan_tci : 0); + + if (this_tag == txq->hw_vlan_tci) + return 0; + + /* + * The expression inside SFC_ASSERT() is not desired to be checked in + * a non-debug build because it might be too expensive on the data path + */ + SFC_ASSERT(efx_nic_cfg_get(txq->evq->sa->nic)->enc_hw_tx_insert_vlan_enabled); + + efx_tx_qdesc_vlantci_create(txq->common, rte_cpu_to_be_16(this_tag), + *pend); + (*pend)++; + txq->hw_vlan_tci = this_tag; + + return 1; +} + +static uint16_t +sfc_efx_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) +{ + struct sfc_dp_txq *dp_txq = (struct sfc_dp_txq *)tx_queue; + struct sfc_efx_txq *txq = sfc_efx_txq_by_dp_txq(dp_txq); + unsigned int added = txq->added; + unsigned int pushed = added; + unsigned int pkts_sent = 0; + efx_desc_t *pend = &txq->pend_desc[0]; + const unsigned int hard_max_fill = EFX_TXQ_LIMIT(txq->ptr_mask + 1); + const unsigned int soft_max_fill = hard_max_fill - txq->free_thresh; + unsigned int fill_level = added - txq->completed; + boolean_t reap_done; + int rc __rte_unused; + struct rte_mbuf **pktp; + + if (unlikely((txq->flags & SFC_EFX_TXQ_FLAG_RUNNING) == 0)) + goto done; + + /* + * If insufficient space for a single packet is present, + * we should reap; otherwise, we shouldn't do that all the time + * to avoid latency increase + */ + reap_done = (fill_level > soft_max_fill); + + if (reap_done) { + sfc_efx_tx_reap(txq); + /* + * Recalculate fill level since 'txq->completed' + * might have changed on reap + */ + fill_level = added - txq->completed; + } + + for (pkts_sent = 0, pktp = &tx_pkts[0]; + (pkts_sent < nb_pkts) && (fill_level <= soft_max_fill); + pkts_sent++, pktp++) { + struct rte_mbuf *m_seg = *pktp; + size_t pkt_len = m_seg->pkt_len; + unsigned int pkt_descs = 0; + size_t in_off = 0; + + /* + * Here VLAN TCI is expected to be zero in case if no + * DEV_TX_VLAN_OFFLOAD capability is advertised; + * if the calling app ignores the absence of + * DEV_TX_VLAN_OFFLOAD and pushes VLAN TCI, then + * TX_ERROR will occur + */ + pkt_descs += sfc_efx_tx_maybe_insert_tag(txq, m_seg, &pend); + + if (m_seg->ol_flags & PKT_TX_TCP_SEG) { + /* + * We expect correct 'pkt->l[2, 3, 4]_len' values + * to be set correctly by the caller + */ + if (sfc_efx_tso_do(txq, added, &m_seg, &in_off, &pend, + &pkt_descs, &pkt_len) != 0) { + /* We may have reached this place for + * one of the following reasons: + * + * 1) Packet header length is greater + * than SFC_TSOH_STD_LEN + * 2) TCP header starts at more then + * 208 bytes into the frame + * + * We will deceive RTE saying that we have sent + * the packet, but we will actually drop it. + * Hence, we should revert 'pend' to the + * previous state (in case we have added + * VLAN descriptor) and start processing + * another one packet. But the original + * mbuf shouldn't be orphaned + */ + pend -= pkt_descs; + + rte_pktmbuf_free(*pktp); + + continue; + } + + /* + * We've only added 2 FATSOv2 option descriptors + * and 1 descriptor for the linearized packet header. + * The outstanding work will be done in the same manner + * as for the usual non-TSO path + */ + } + + for (; m_seg != NULL; m_seg = m_seg->next) { + efsys_dma_addr_t next_frag; + size_t seg_len; + + seg_len = m_seg->data_len; + next_frag = rte_mbuf_data_iova(m_seg); + + /* + * If we've started TSO transaction few steps earlier, + * we'll skip packet header using an offset in the + * current segment (which has been set to the + * first one containing payload) + */ + seg_len -= in_off; + next_frag += in_off; + in_off = 0; + + do { + efsys_dma_addr_t frag_addr = next_frag; + size_t frag_len; + + /* + * It is assumed here that there is no + * limitation on address boundary + * crossing by DMA descriptor. + */ + frag_len = MIN(seg_len, txq->dma_desc_size_max); + next_frag += frag_len; + seg_len -= frag_len; + pkt_len -= frag_len; + + efx_tx_qdesc_dma_create(txq->common, + frag_addr, frag_len, + (pkt_len == 0), + pend++); + + pkt_descs++; + } while (seg_len != 0); + } + + added += pkt_descs; + + fill_level += pkt_descs; + if (unlikely(fill_level > hard_max_fill)) { + /* + * Our estimation for maximum number of descriptors + * required to send a packet seems to be wrong. + * Try to reap (if we haven't yet). + */ + if (!reap_done) { + sfc_efx_tx_reap(txq); + reap_done = B_TRUE; + fill_level = added - txq->completed; + if (fill_level > hard_max_fill) { + pend -= pkt_descs; + break; + } + } else { + pend -= pkt_descs; + break; + } + } + + /* Assign mbuf to the last used desc */ + txq->sw_ring[(added - 1) & txq->ptr_mask].mbuf = *pktp; + } + + if (likely(pkts_sent > 0)) { + rc = efx_tx_qdesc_post(txq->common, txq->pend_desc, + pend - &txq->pend_desc[0], + txq->completed, &txq->added); + SFC_ASSERT(rc == 0); + + if (likely(pushed != txq->added)) + efx_tx_qpush(txq->common, txq->added, pushed); + } + +#if SFC_TX_XMIT_PKTS_REAP_AT_LEAST_ONCE + if (!reap_done) + sfc_efx_tx_reap(txq); +#endif + +done: + return pkts_sent; +} + +struct sfc_txq * +sfc_txq_by_dp_txq(const struct sfc_dp_txq *dp_txq) +{ + const struct sfc_dp_queue *dpq = &dp_txq->dpq; + struct rte_eth_dev *eth_dev; + struct sfc_adapter *sa; + struct sfc_txq *txq; + + SFC_ASSERT(rte_eth_dev_is_valid_port(dpq->port_id)); + eth_dev = &rte_eth_devices[dpq->port_id]; + + sa = eth_dev->data->dev_private; + + SFC_ASSERT(dpq->queue_id < sa->txq_count); + txq = sa->txq_info[dpq->queue_id].txq; + + SFC_ASSERT(txq != NULL); + return txq; +} + +static sfc_dp_tx_qcreate_t sfc_efx_tx_qcreate; +static int +sfc_efx_tx_qcreate(uint16_t port_id, uint16_t queue_id, + const struct rte_pci_addr *pci_addr, + int socket_id, + const struct sfc_dp_tx_qcreate_info *info, + struct sfc_dp_txq **dp_txqp) +{ + struct sfc_efx_txq *txq; + struct sfc_txq *ctrl_txq; + int rc; + + rc = ENOMEM; + txq = rte_zmalloc_socket("sfc-efx-txq", sizeof(*txq), + RTE_CACHE_LINE_SIZE, socket_id); + if (txq == NULL) + goto fail_txq_alloc; + + sfc_dp_queue_init(&txq->dp.dpq, port_id, queue_id, pci_addr); + + rc = ENOMEM; + txq->pend_desc = rte_calloc_socket("sfc-efx-txq-pend-desc", + EFX_TXQ_LIMIT(info->txq_entries), + sizeof(*txq->pend_desc), 0, + socket_id); + if (txq->pend_desc == NULL) + goto fail_pend_desc_alloc; + + rc = ENOMEM; + txq->sw_ring = rte_calloc_socket("sfc-efx-txq-sw_ring", + info->txq_entries, + sizeof(*txq->sw_ring), + RTE_CACHE_LINE_SIZE, socket_id); + if (txq->sw_ring == NULL) + goto fail_sw_ring_alloc; + + ctrl_txq = sfc_txq_by_dp_txq(&txq->dp); + if (ctrl_txq->evq->sa->tso) { + rc = sfc_efx_tso_alloc_tsoh_objs(txq->sw_ring, + info->txq_entries, socket_id); + if (rc != 0) + goto fail_alloc_tsoh_objs; + } + + txq->evq = ctrl_txq->evq; + txq->ptr_mask = info->txq_entries - 1; + txq->free_thresh = info->free_thresh; + txq->dma_desc_size_max = info->dma_desc_size_max; + + *dp_txqp = &txq->dp; + return 0; + +fail_alloc_tsoh_objs: + rte_free(txq->sw_ring); + +fail_sw_ring_alloc: + rte_free(txq->pend_desc); + +fail_pend_desc_alloc: + rte_free(txq); + +fail_txq_alloc: + return rc; +} + +static sfc_dp_tx_qdestroy_t sfc_efx_tx_qdestroy; +static void +sfc_efx_tx_qdestroy(struct sfc_dp_txq *dp_txq) +{ + struct sfc_efx_txq *txq = sfc_efx_txq_by_dp_txq(dp_txq); + + sfc_efx_tso_free_tsoh_objs(txq->sw_ring, txq->ptr_mask + 1); + rte_free(txq->sw_ring); + rte_free(txq->pend_desc); + rte_free(txq); +} + +static sfc_dp_tx_qstart_t sfc_efx_tx_qstart; +static int +sfc_efx_tx_qstart(struct sfc_dp_txq *dp_txq, + __rte_unused unsigned int evq_read_ptr, + unsigned int txq_desc_index) +{ + /* libefx-based datapath is specific to libefx-based PMD */ + struct sfc_efx_txq *txq = sfc_efx_txq_by_dp_txq(dp_txq); + struct sfc_txq *ctrl_txq = sfc_txq_by_dp_txq(dp_txq); + + txq->common = ctrl_txq->common; + + txq->pending = txq->completed = txq->added = txq_desc_index; + txq->hw_vlan_tci = 0; + + txq->flags |= (SFC_EFX_TXQ_FLAG_STARTED | SFC_EFX_TXQ_FLAG_RUNNING); + + return 0; +} + +static sfc_dp_tx_qstop_t sfc_efx_tx_qstop; +static void +sfc_efx_tx_qstop(struct sfc_dp_txq *dp_txq, + __rte_unused unsigned int *evq_read_ptr) +{ + struct sfc_efx_txq *txq = sfc_efx_txq_by_dp_txq(dp_txq); + + txq->flags &= ~SFC_EFX_TXQ_FLAG_RUNNING; +} + +static sfc_dp_tx_qreap_t sfc_efx_tx_qreap; +static void +sfc_efx_tx_qreap(struct sfc_dp_txq *dp_txq) +{ + struct sfc_efx_txq *txq = sfc_efx_txq_by_dp_txq(dp_txq); + unsigned int txds; + + sfc_efx_tx_reap(txq); + + for (txds = 0; txds <= txq->ptr_mask; txds++) { + if (txq->sw_ring[txds].mbuf != NULL) { + rte_pktmbuf_free(txq->sw_ring[txds].mbuf); + txq->sw_ring[txds].mbuf = NULL; + } + } + + txq->flags &= ~SFC_EFX_TXQ_FLAG_STARTED; +} + +static sfc_dp_tx_qdesc_status_t sfc_efx_tx_qdesc_status; +static int +sfc_efx_tx_qdesc_status(struct sfc_dp_txq *dp_txq, uint16_t offset) +{ + struct sfc_efx_txq *txq = sfc_efx_txq_by_dp_txq(dp_txq); + + if (unlikely(offset > txq->ptr_mask)) + return -EINVAL; + + if (unlikely(offset >= EFX_TXQ_LIMIT(txq->ptr_mask + 1))) + return RTE_ETH_TX_DESC_UNAVAIL; + + /* + * Poll EvQ to derive up-to-date 'txq->pending' figure; + * it is required for the queue to be running, but the + * check is omitted because API design assumes that it + * is the duty of the caller to satisfy all conditions + */ + SFC_ASSERT((txq->flags & SFC_EFX_TXQ_FLAG_RUNNING) == + SFC_EFX_TXQ_FLAG_RUNNING); + sfc_ev_qpoll(txq->evq); + + /* + * Ring tail is 'txq->pending', and although descriptors + * between 'txq->completed' and 'txq->pending' are still + * in use by the driver, they should be reported as DONE + */ + if (unlikely(offset < (txq->added - txq->pending))) + return RTE_ETH_TX_DESC_FULL; + + /* + * There is no separate return value for unused descriptors; + * the latter will be reported as DONE because genuine DONE + * descriptors will be freed anyway in SW on the next burst + */ + return RTE_ETH_TX_DESC_DONE; +} + +struct sfc_dp_tx sfc_efx_tx = { + .dp = { + .name = SFC_KVARG_DATAPATH_EFX, + .type = SFC_DP_TX, + .hw_fw_caps = 0, + }, + .features = SFC_DP_TX_FEAT_VLAN_INSERT | + SFC_DP_TX_FEAT_TSO | + SFC_DP_TX_FEAT_MULTI_POOL | + SFC_DP_TX_FEAT_REFCNT | + SFC_DP_TX_FEAT_MULTI_SEG, + .qcreate = sfc_efx_tx_qcreate, + .qdestroy = sfc_efx_tx_qdestroy, + .qstart = sfc_efx_tx_qstart, + .qstop = sfc_efx_tx_qstop, + .qreap = sfc_efx_tx_qreap, + .qdesc_status = sfc_efx_tx_qdesc_status, + .pkt_burst = sfc_efx_xmit_pkts, +}; diff --git a/dpdk/drivers/net/sfc/sfc_tx.h b/dpdk/drivers/net/sfc/sfc_tx.h new file mode 100644 index 00000000..0c1c7083 --- /dev/null +++ b/dpdk/drivers/net/sfc/sfc_tx.h @@ -0,0 +1,166 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2016-2017 Solarflare Communications Inc. + * All rights reserved. + * + * This software was jointly developed between OKTET Labs (under contract + * for Solarflare) and Solarflare Communications, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _SFC_TX_H +#define _SFC_TX_H + +#include +#include + +#include "efx.h" + +#include "sfc_dp_tx.h" + +#ifdef __cplusplus +extern "C" { +#endif + +struct sfc_adapter; +struct sfc_evq; + +/** + * Software Tx descriptor information associated with hardware Tx + * descriptor. + */ +struct sfc_efx_tx_sw_desc { + struct rte_mbuf *mbuf; + uint8_t *tsoh; /* Buffer to store TSO header */ +}; + +enum sfc_txq_state_bit { + SFC_TXQ_INITIALIZED_BIT = 0, +#define SFC_TXQ_INITIALIZED (1 << SFC_TXQ_INITIALIZED_BIT) + SFC_TXQ_STARTED_BIT, +#define SFC_TXQ_STARTED (1 << SFC_TXQ_STARTED_BIT) + SFC_TXQ_FLUSHING_BIT, +#define SFC_TXQ_FLUSHING (1 << SFC_TXQ_FLUSHING_BIT) + SFC_TXQ_FLUSHED_BIT, +#define SFC_TXQ_FLUSHED (1 << SFC_TXQ_FLUSHED_BIT) + SFC_TXQ_FLUSH_FAILED_BIT, +#define SFC_TXQ_FLUSH_FAILED (1 << SFC_TXQ_FLUSH_FAILED_BIT) +}; + +/** + * Transmit queue control information. Not used on datapath. + * Allocated on the socket specified on the queue setup. + */ +struct sfc_txq { + unsigned int state; + unsigned int hw_index; + struct sfc_evq *evq; + efsys_mem_t mem; + struct sfc_dp_txq *dp; + efx_txq_t *common; + unsigned int free_thresh; + unsigned int flags; +}; + +static inline unsigned int +sfc_txq_sw_index_by_hw_index(unsigned int hw_index) +{ + return hw_index; +} + +static inline unsigned int +sfc_txq_sw_index(const struct sfc_txq *txq) +{ + return sfc_txq_sw_index_by_hw_index(txq->hw_index); +} + +struct sfc_txq *sfc_txq_by_dp_txq(const struct sfc_dp_txq *dp_txq); + +/** + * Transmit queue information used on libefx-based data path. + * Allocated on the socket specified on the queue setup. + */ +struct sfc_efx_txq { + struct sfc_evq *evq; + struct sfc_efx_tx_sw_desc *sw_ring; + unsigned int ptr_mask; + efx_desc_t *pend_desc; + efx_txq_t *common; + unsigned int added; + unsigned int pending; + unsigned int completed; + unsigned int free_thresh; + uint16_t hw_vlan_tci; + uint16_t dma_desc_size_max; + + unsigned int hw_index; + unsigned int flags; +#define SFC_EFX_TXQ_FLAG_STARTED 0x1 +#define SFC_EFX_TXQ_FLAG_RUNNING 0x2 + + /* Datapath transmit queue anchor */ + struct sfc_dp_txq dp; +}; + +static inline struct sfc_efx_txq * +sfc_efx_txq_by_dp_txq(struct sfc_dp_txq *dp_txq) +{ + return container_of(dp_txq, struct sfc_efx_txq, dp); +} + +struct sfc_txq_info { + unsigned int entries; + struct sfc_txq *txq; + boolean_t deferred_start; + boolean_t deferred_started; +}; + +int sfc_tx_configure(struct sfc_adapter *sa); +void sfc_tx_close(struct sfc_adapter *sa); + +int sfc_tx_qinit(struct sfc_adapter *sa, unsigned int sw_index, + uint16_t nb_tx_desc, unsigned int socket_id, + const struct rte_eth_txconf *tx_conf); +void sfc_tx_qfini(struct sfc_adapter *sa, unsigned int sw_index); + +void sfc_tx_qflush_done(struct sfc_txq *txq); +int sfc_tx_qstart(struct sfc_adapter *sa, unsigned int sw_index); +void sfc_tx_qstop(struct sfc_adapter *sa, unsigned int sw_index); +int sfc_tx_start(struct sfc_adapter *sa); +void sfc_tx_stop(struct sfc_adapter *sa); + +/* From 'sfc_tso.c' */ +int sfc_efx_tso_alloc_tsoh_objs(struct sfc_efx_tx_sw_desc *sw_ring, + unsigned int txq_entries, + unsigned int socket_id); +void sfc_efx_tso_free_tsoh_objs(struct sfc_efx_tx_sw_desc *sw_ring, + unsigned int txq_entries); +int sfc_efx_tso_do(struct sfc_efx_txq *txq, unsigned int idx, + struct rte_mbuf **in_seg, size_t *in_off, efx_desc_t **pend, + unsigned int *pkt_descs, size_t *pkt_len); + +#ifdef __cplusplus +} +#endif +#endif /* _SFC_TX_H */ diff --git a/dpdk/drivers/net/softnic/Makefile b/dpdk/drivers/net/softnic/Makefile new file mode 100644 index 00000000..09ed62ea --- /dev/null +++ b/dpdk/drivers/net/softnic/Makefile @@ -0,0 +1,60 @@ +# BSD LICENSE +# +# Copyright(c) 2017 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +include $(RTE_SDK)/mk/rte.vars.mk + +# +# library name +# +LIB = librte_pmd_softnic.a + +CFLAGS += -O3 +CFLAGS += $(WERROR_FLAGS) +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs -lrte_sched +LDLIBS += -lrte_bus_vdev + +EXPORT_MAP := rte_pmd_eth_softnic_version.map + +LIBABIVER := 1 + +# +# all source are stored in SRCS-y +# +SRCS-$(CONFIG_RTE_LIBRTE_PMD_SOFTNIC) += rte_eth_softnic.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_SOFTNIC) += rte_eth_softnic_tm.c + +# +# Export include files +# +SYMLINK-y-include += rte_eth_softnic.h + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/net/softnic/rte_eth_softnic.c b/dpdk/drivers/net/softnic/rte_eth_softnic.c new file mode 100644 index 00000000..c8f918df --- /dev/null +++ b/dpdk/drivers/net/softnic/rte_eth_softnic.c @@ -0,0 +1,851 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "rte_eth_softnic.h" +#include "rte_eth_softnic_internals.h" + +#define DEV_HARD(p) \ + (&rte_eth_devices[p->hard.port_id]) + +#define PMD_PARAM_SOFT_TM "soft_tm" +#define PMD_PARAM_SOFT_TM_RATE "soft_tm_rate" +#define PMD_PARAM_SOFT_TM_NB_QUEUES "soft_tm_nb_queues" +#define PMD_PARAM_SOFT_TM_QSIZE0 "soft_tm_qsize0" +#define PMD_PARAM_SOFT_TM_QSIZE1 "soft_tm_qsize1" +#define PMD_PARAM_SOFT_TM_QSIZE2 "soft_tm_qsize2" +#define PMD_PARAM_SOFT_TM_QSIZE3 "soft_tm_qsize3" +#define PMD_PARAM_SOFT_TM_ENQ_BSZ "soft_tm_enq_bsz" +#define PMD_PARAM_SOFT_TM_DEQ_BSZ "soft_tm_deq_bsz" + +#define PMD_PARAM_HARD_NAME "hard_name" +#define PMD_PARAM_HARD_TX_QUEUE_ID "hard_tx_queue_id" + +static const char *pmd_valid_args[] = { + PMD_PARAM_SOFT_TM, + PMD_PARAM_SOFT_TM_RATE, + PMD_PARAM_SOFT_TM_NB_QUEUES, + PMD_PARAM_SOFT_TM_QSIZE0, + PMD_PARAM_SOFT_TM_QSIZE1, + PMD_PARAM_SOFT_TM_QSIZE2, + PMD_PARAM_SOFT_TM_QSIZE3, + PMD_PARAM_SOFT_TM_ENQ_BSZ, + PMD_PARAM_SOFT_TM_DEQ_BSZ, + PMD_PARAM_HARD_NAME, + PMD_PARAM_HARD_TX_QUEUE_ID, + NULL +}; + +static const struct rte_eth_dev_info pmd_dev_info = { + .min_rx_bufsize = 0, + .max_rx_pktlen = UINT32_MAX, + .max_rx_queues = UINT16_MAX, + .max_tx_queues = UINT16_MAX, + .rx_desc_lim = { + .nb_max = UINT16_MAX, + .nb_min = 0, + .nb_align = 1, + }, + .tx_desc_lim = { + .nb_max = UINT16_MAX, + .nb_min = 0, + .nb_align = 1, + }, +}; + +static void +pmd_dev_infos_get(struct rte_eth_dev *dev __rte_unused, + struct rte_eth_dev_info *dev_info) +{ + memcpy(dev_info, &pmd_dev_info, sizeof(*dev_info)); +} + +static int +pmd_dev_configure(struct rte_eth_dev *dev) +{ + struct pmd_internals *p = dev->data->dev_private; + struct rte_eth_dev *hard_dev = DEV_HARD(p); + + if (dev->data->nb_rx_queues > hard_dev->data->nb_rx_queues) + return -1; + + if (p->params.hard.tx_queue_id >= hard_dev->data->nb_tx_queues) + return -1; + + return 0; +} + +static int +pmd_rx_queue_setup(struct rte_eth_dev *dev, + uint16_t rx_queue_id, + uint16_t nb_rx_desc __rte_unused, + unsigned int socket_id, + const struct rte_eth_rxconf *rx_conf __rte_unused, + struct rte_mempool *mb_pool __rte_unused) +{ + struct pmd_internals *p = dev->data->dev_private; + + if (p->params.soft.intrusive == 0) { + struct pmd_rx_queue *rxq; + + rxq = rte_zmalloc_socket(p->params.soft.name, + sizeof(struct pmd_rx_queue), 0, socket_id); + if (rxq == NULL) + return -ENOMEM; + + rxq->hard.port_id = p->hard.port_id; + rxq->hard.rx_queue_id = rx_queue_id; + dev->data->rx_queues[rx_queue_id] = rxq; + } else { + struct rte_eth_dev *hard_dev = DEV_HARD(p); + void *rxq = hard_dev->data->rx_queues[rx_queue_id]; + + if (rxq == NULL) + return -1; + + dev->data->rx_queues[rx_queue_id] = rxq; + } + return 0; +} + +static int +pmd_tx_queue_setup(struct rte_eth_dev *dev, + uint16_t tx_queue_id, + uint16_t nb_tx_desc, + unsigned int socket_id, + const struct rte_eth_txconf *tx_conf __rte_unused) +{ + uint32_t size = RTE_ETH_NAME_MAX_LEN + strlen("_txq") + 4; + char name[size]; + struct rte_ring *r; + + snprintf(name, sizeof(name), "%s_txq%04x", + dev->data->name, tx_queue_id); + r = rte_ring_create(name, nb_tx_desc, socket_id, + RING_F_SP_ENQ | RING_F_SC_DEQ); + if (r == NULL) + return -1; + + dev->data->tx_queues[tx_queue_id] = r; + return 0; +} + +static int +pmd_dev_start(struct rte_eth_dev *dev) +{ + struct pmd_internals *p = dev->data->dev_private; + + if (tm_used(dev)) { + int status = tm_start(p); + + if (status) + return status; + } + + dev->data->dev_link.link_status = ETH_LINK_UP; + + if (p->params.soft.intrusive) { + struct rte_eth_dev *hard_dev = DEV_HARD(p); + + /* The hard_dev->rx_pkt_burst should be stable by now */ + dev->rx_pkt_burst = hard_dev->rx_pkt_burst; + } + + return 0; +} + +static void +pmd_dev_stop(struct rte_eth_dev *dev) +{ + struct pmd_internals *p = dev->data->dev_private; + + dev->data->dev_link.link_status = ETH_LINK_DOWN; + + if (tm_used(dev)) + tm_stop(p); +} + +static void +pmd_dev_close(struct rte_eth_dev *dev) +{ + uint32_t i; + + /* TX queues */ + for (i = 0; i < dev->data->nb_tx_queues; i++) + rte_ring_free((struct rte_ring *)dev->data->tx_queues[i]); +} + +static int +pmd_link_update(struct rte_eth_dev *dev __rte_unused, + int wait_to_complete __rte_unused) +{ + return 0; +} + +static int +pmd_tm_ops_get(struct rte_eth_dev *dev, void *arg) +{ + *(const struct rte_tm_ops **)arg = + (tm_enabled(dev)) ? &pmd_tm_ops : NULL; + + return 0; +} + +static const struct eth_dev_ops pmd_ops = { + .dev_configure = pmd_dev_configure, + .dev_start = pmd_dev_start, + .dev_stop = pmd_dev_stop, + .dev_close = pmd_dev_close, + .link_update = pmd_link_update, + .dev_infos_get = pmd_dev_infos_get, + .rx_queue_setup = pmd_rx_queue_setup, + .tx_queue_setup = pmd_tx_queue_setup, + .tm_ops_get = pmd_tm_ops_get, +}; + +static uint16_t +pmd_rx_pkt_burst(void *rxq, + struct rte_mbuf **rx_pkts, + uint16_t nb_pkts) +{ + struct pmd_rx_queue *rx_queue = rxq; + + return rte_eth_rx_burst(rx_queue->hard.port_id, + rx_queue->hard.rx_queue_id, + rx_pkts, + nb_pkts); +} + +static uint16_t +pmd_tx_pkt_burst(void *txq, + struct rte_mbuf **tx_pkts, + uint16_t nb_pkts) +{ + return (uint16_t)rte_ring_enqueue_burst(txq, + (void **)tx_pkts, + nb_pkts, + NULL); +} + +static __rte_always_inline int +run_default(struct rte_eth_dev *dev) +{ + struct pmd_internals *p = dev->data->dev_private; + + /* Persistent context: Read Only (update not required) */ + struct rte_mbuf **pkts = p->soft.def.pkts; + uint16_t nb_tx_queues = dev->data->nb_tx_queues; + + /* Persistent context: Read - Write (update required) */ + uint32_t txq_pos = p->soft.def.txq_pos; + uint32_t pkts_len = p->soft.def.pkts_len; + uint32_t flush_count = p->soft.def.flush_count; + + /* Not part of the persistent context */ + uint32_t pos; + uint16_t i; + + /* Soft device TXQ read, Hard device TXQ write */ + for (i = 0; i < nb_tx_queues; i++) { + struct rte_ring *txq = dev->data->tx_queues[txq_pos]; + + /* Read soft device TXQ burst to packet enqueue buffer */ + pkts_len += rte_ring_sc_dequeue_burst(txq, + (void **)&pkts[pkts_len], + DEFAULT_BURST_SIZE, + NULL); + + /* Increment soft device TXQ */ + txq_pos++; + if (txq_pos >= nb_tx_queues) + txq_pos = 0; + + /* Hard device TXQ write when complete burst is available */ + if (pkts_len >= DEFAULT_BURST_SIZE) { + for (pos = 0; pos < pkts_len; ) + pos += rte_eth_tx_burst(p->hard.port_id, + p->params.hard.tx_queue_id, + &pkts[pos], + (uint16_t)(pkts_len - pos)); + + pkts_len = 0; + flush_count = 0; + break; + } + } + + if (flush_count >= FLUSH_COUNT_THRESHOLD) { + for (pos = 0; pos < pkts_len; ) + pos += rte_eth_tx_burst(p->hard.port_id, + p->params.hard.tx_queue_id, + &pkts[pos], + (uint16_t)(pkts_len - pos)); + + pkts_len = 0; + flush_count = 0; + } + + p->soft.def.txq_pos = txq_pos; + p->soft.def.pkts_len = pkts_len; + p->soft.def.flush_count = flush_count + 1; + + return 0; +} + +static __rte_always_inline int +run_tm(struct rte_eth_dev *dev) +{ + struct pmd_internals *p = dev->data->dev_private; + + /* Persistent context: Read Only (update not required) */ + struct rte_sched_port *sched = p->soft.tm.sched; + struct rte_mbuf **pkts_enq = p->soft.tm.pkts_enq; + struct rte_mbuf **pkts_deq = p->soft.tm.pkts_deq; + uint32_t enq_bsz = p->params.soft.tm.enq_bsz; + uint32_t deq_bsz = p->params.soft.tm.deq_bsz; + uint16_t nb_tx_queues = dev->data->nb_tx_queues; + + /* Persistent context: Read - Write (update required) */ + uint32_t txq_pos = p->soft.tm.txq_pos; + uint32_t pkts_enq_len = p->soft.tm.pkts_enq_len; + uint32_t flush_count = p->soft.tm.flush_count; + + /* Not part of the persistent context */ + uint32_t pkts_deq_len, pos; + uint16_t i; + + /* Soft device TXQ read, TM enqueue */ + for (i = 0; i < nb_tx_queues; i++) { + struct rte_ring *txq = dev->data->tx_queues[txq_pos]; + + /* Read TXQ burst to packet enqueue buffer */ + pkts_enq_len += rte_ring_sc_dequeue_burst(txq, + (void **)&pkts_enq[pkts_enq_len], + enq_bsz, + NULL); + + /* Increment TXQ */ + txq_pos++; + if (txq_pos >= nb_tx_queues) + txq_pos = 0; + + /* TM enqueue when complete burst is available */ + if (pkts_enq_len >= enq_bsz) { + rte_sched_port_enqueue(sched, pkts_enq, pkts_enq_len); + + pkts_enq_len = 0; + flush_count = 0; + break; + } + } + + if (flush_count >= FLUSH_COUNT_THRESHOLD) { + if (pkts_enq_len) + rte_sched_port_enqueue(sched, pkts_enq, pkts_enq_len); + + pkts_enq_len = 0; + flush_count = 0; + } + + p->soft.tm.txq_pos = txq_pos; + p->soft.tm.pkts_enq_len = pkts_enq_len; + p->soft.tm.flush_count = flush_count + 1; + + /* TM dequeue, Hard device TXQ write */ + pkts_deq_len = rte_sched_port_dequeue(sched, pkts_deq, deq_bsz); + + for (pos = 0; pos < pkts_deq_len; ) + pos += rte_eth_tx_burst(p->hard.port_id, + p->params.hard.tx_queue_id, + &pkts_deq[pos], + (uint16_t)(pkts_deq_len - pos)); + + return 0; +} + +int +rte_pmd_softnic_run(uint16_t port_id) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + +#ifdef RTE_LIBRTE_ETHDEV_DEBUG + RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, 0); +#endif + + return (tm_used(dev)) ? run_tm(dev) : run_default(dev); +} + +static struct ether_addr eth_addr = { .addr_bytes = {0} }; + +static uint32_t +eth_dev_speed_max_mbps(uint32_t speed_capa) +{ + uint32_t rate_mbps[32] = { + ETH_SPEED_NUM_NONE, + ETH_SPEED_NUM_10M, + ETH_SPEED_NUM_10M, + ETH_SPEED_NUM_100M, + ETH_SPEED_NUM_100M, + ETH_SPEED_NUM_1G, + ETH_SPEED_NUM_2_5G, + ETH_SPEED_NUM_5G, + ETH_SPEED_NUM_10G, + ETH_SPEED_NUM_20G, + ETH_SPEED_NUM_25G, + ETH_SPEED_NUM_40G, + ETH_SPEED_NUM_50G, + ETH_SPEED_NUM_56G, + ETH_SPEED_NUM_100G, + }; + + uint32_t pos = (speed_capa) ? (31 - __builtin_clz(speed_capa)) : 0; + return rate_mbps[pos]; +} + +static int +default_init(struct pmd_internals *p, + struct pmd_params *params, + int numa_node) +{ + p->soft.def.pkts = rte_zmalloc_socket(params->soft.name, + 2 * DEFAULT_BURST_SIZE * sizeof(struct rte_mbuf *), + 0, + numa_node); + + if (p->soft.def.pkts == NULL) + return -ENOMEM; + + return 0; +} + +static void +default_free(struct pmd_internals *p) +{ + rte_free(p->soft.def.pkts); +} + +static void * +pmd_init(struct pmd_params *params, int numa_node) +{ + struct pmd_internals *p; + int status; + + p = rte_zmalloc_socket(params->soft.name, + sizeof(struct pmd_internals), + 0, + numa_node); + if (p == NULL) + return NULL; + + memcpy(&p->params, params, sizeof(p->params)); + rte_eth_dev_get_port_by_name(params->hard.name, &p->hard.port_id); + + /* Default */ + status = default_init(p, params, numa_node); + if (status) { + free(p->params.hard.name); + rte_free(p); + return NULL; + } + + /* Traffic Management (TM)*/ + if (params->soft.flags & PMD_FEATURE_TM) { + status = tm_init(p, params, numa_node); + if (status) { + default_free(p); + free(p->params.hard.name); + rte_free(p); + return NULL; + } + } + + return p; +} + +static void +pmd_free(struct pmd_internals *p) +{ + if (p->params.soft.flags & PMD_FEATURE_TM) + tm_free(p); + + default_free(p); + + free(p->params.hard.name); + rte_free(p); +} + +static int +pmd_ethdev_register(struct rte_vdev_device *vdev, + struct pmd_params *params, + void *dev_private) +{ + struct rte_eth_dev_info hard_info; + struct rte_eth_dev *soft_dev; + uint32_t hard_speed; + int numa_node; + uint16_t hard_port_id; + + rte_eth_dev_get_port_by_name(params->hard.name, &hard_port_id); + rte_eth_dev_info_get(hard_port_id, &hard_info); + hard_speed = eth_dev_speed_max_mbps(hard_info.speed_capa); + numa_node = rte_eth_dev_socket_id(hard_port_id); + + /* Ethdev entry allocation */ + soft_dev = rte_eth_dev_allocate(params->soft.name); + if (!soft_dev) + return -ENOMEM; + + /* dev */ + soft_dev->rx_pkt_burst = (params->soft.intrusive) ? + NULL : /* set up later */ + pmd_rx_pkt_burst; + soft_dev->tx_pkt_burst = pmd_tx_pkt_burst; + soft_dev->tx_pkt_prepare = NULL; + soft_dev->dev_ops = &pmd_ops; + soft_dev->device = &vdev->device; + + /* dev->data */ + soft_dev->data->dev_private = dev_private; + soft_dev->data->dev_link.link_speed = hard_speed; + soft_dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX; + soft_dev->data->dev_link.link_autoneg = ETH_LINK_AUTONEG; + soft_dev->data->dev_link.link_status = ETH_LINK_DOWN; + soft_dev->data->mac_addrs = ð_addr; + soft_dev->data->promiscuous = 1; + soft_dev->data->kdrv = RTE_KDRV_NONE; + soft_dev->data->numa_node = numa_node; + + return 0; +} + +static int +get_string(const char *key __rte_unused, const char *value, void *extra_args) +{ + if (!value || !extra_args) + return -EINVAL; + + *(char **)extra_args = strdup(value); + + if (!*(char **)extra_args) + return -ENOMEM; + + return 0; +} + +static int +get_uint32(const char *key __rte_unused, const char *value, void *extra_args) +{ + if (!value || !extra_args) + return -EINVAL; + + *(uint32_t *)extra_args = strtoull(value, NULL, 0); + + return 0; +} + +static int +pmd_parse_args(struct pmd_params *p, const char *name, const char *params) +{ + struct rte_kvargs *kvlist; + int i, ret; + + kvlist = rte_kvargs_parse(params, pmd_valid_args); + if (kvlist == NULL) + return -EINVAL; + + /* Set default values */ + memset(p, 0, sizeof(*p)); + p->soft.name = name; + p->soft.intrusive = INTRUSIVE; + p->soft.tm.rate = 0; + p->soft.tm.nb_queues = SOFTNIC_SOFT_TM_NB_QUEUES; + for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++) + p->soft.tm.qsize[i] = SOFTNIC_SOFT_TM_QUEUE_SIZE; + p->soft.tm.enq_bsz = SOFTNIC_SOFT_TM_ENQ_BSZ; + p->soft.tm.deq_bsz = SOFTNIC_SOFT_TM_DEQ_BSZ; + p->hard.tx_queue_id = SOFTNIC_HARD_TX_QUEUE_ID; + + /* SOFT: TM (optional) */ + if (rte_kvargs_count(kvlist, PMD_PARAM_SOFT_TM) == 1) { + char *s; + + ret = rte_kvargs_process(kvlist, PMD_PARAM_SOFT_TM, + &get_string, &s); + if (ret < 0) + goto out_free; + + if (strcmp(s, "on") == 0) + p->soft.flags |= PMD_FEATURE_TM; + else if (strcmp(s, "off") == 0) + p->soft.flags &= ~PMD_FEATURE_TM; + else + ret = -EINVAL; + + free(s); + if (ret) + goto out_free; + } + + /* SOFT: TM rate (measured in bytes/second) (optional) */ + if (rte_kvargs_count(kvlist, PMD_PARAM_SOFT_TM_RATE) == 1) { + ret = rte_kvargs_process(kvlist, PMD_PARAM_SOFT_TM_RATE, + &get_uint32, &p->soft.tm.rate); + if (ret < 0) + goto out_free; + + p->soft.flags |= PMD_FEATURE_TM; + } + + /* SOFT: TM number of queues (optional) */ + if (rte_kvargs_count(kvlist, PMD_PARAM_SOFT_TM_NB_QUEUES) == 1) { + ret = rte_kvargs_process(kvlist, PMD_PARAM_SOFT_TM_NB_QUEUES, + &get_uint32, &p->soft.tm.nb_queues); + if (ret < 0) + goto out_free; + + p->soft.flags |= PMD_FEATURE_TM; + } + + /* SOFT: TM queue size 0 .. 3 (optional) */ + if (rte_kvargs_count(kvlist, PMD_PARAM_SOFT_TM_QSIZE0) == 1) { + uint32_t qsize; + + ret = rte_kvargs_process(kvlist, PMD_PARAM_SOFT_TM_QSIZE0, + &get_uint32, &qsize); + if (ret < 0) + goto out_free; + + p->soft.tm.qsize[0] = (uint16_t)qsize; + p->soft.flags |= PMD_FEATURE_TM; + } + + if (rte_kvargs_count(kvlist, PMD_PARAM_SOFT_TM_QSIZE1) == 1) { + uint32_t qsize; + + ret = rte_kvargs_process(kvlist, PMD_PARAM_SOFT_TM_QSIZE1, + &get_uint32, &qsize); + if (ret < 0) + goto out_free; + + p->soft.tm.qsize[1] = (uint16_t)qsize; + p->soft.flags |= PMD_FEATURE_TM; + } + + if (rte_kvargs_count(kvlist, PMD_PARAM_SOFT_TM_QSIZE2) == 1) { + uint32_t qsize; + + ret = rte_kvargs_process(kvlist, PMD_PARAM_SOFT_TM_QSIZE2, + &get_uint32, &qsize); + if (ret < 0) + goto out_free; + + p->soft.tm.qsize[2] = (uint16_t)qsize; + p->soft.flags |= PMD_FEATURE_TM; + } + + if (rte_kvargs_count(kvlist, PMD_PARAM_SOFT_TM_QSIZE3) == 1) { + uint32_t qsize; + + ret = rte_kvargs_process(kvlist, PMD_PARAM_SOFT_TM_QSIZE3, + &get_uint32, &qsize); + if (ret < 0) + goto out_free; + + p->soft.tm.qsize[3] = (uint16_t)qsize; + p->soft.flags |= PMD_FEATURE_TM; + } + + /* SOFT: TM enqueue burst size (optional) */ + if (rte_kvargs_count(kvlist, PMD_PARAM_SOFT_TM_ENQ_BSZ) == 1) { + ret = rte_kvargs_process(kvlist, PMD_PARAM_SOFT_TM_ENQ_BSZ, + &get_uint32, &p->soft.tm.enq_bsz); + if (ret < 0) + goto out_free; + + p->soft.flags |= PMD_FEATURE_TM; + } + + /* SOFT: TM dequeue burst size (optional) */ + if (rte_kvargs_count(kvlist, PMD_PARAM_SOFT_TM_DEQ_BSZ) == 1) { + ret = rte_kvargs_process(kvlist, PMD_PARAM_SOFT_TM_DEQ_BSZ, + &get_uint32, &p->soft.tm.deq_bsz); + if (ret < 0) + goto out_free; + + p->soft.flags |= PMD_FEATURE_TM; + } + + /* HARD: name (mandatory) */ + if (rte_kvargs_count(kvlist, PMD_PARAM_HARD_NAME) == 1) { + ret = rte_kvargs_process(kvlist, PMD_PARAM_HARD_NAME, + &get_string, &p->hard.name); + if (ret < 0) + goto out_free; + } else { + ret = -EINVAL; + goto out_free; + } + + /* HARD: tx_queue_id (optional) */ + if (rte_kvargs_count(kvlist, PMD_PARAM_HARD_TX_QUEUE_ID) == 1) { + ret = rte_kvargs_process(kvlist, PMD_PARAM_HARD_TX_QUEUE_ID, + &get_uint32, &p->hard.tx_queue_id); + if (ret < 0) + goto out_free; + } + +out_free: + rte_kvargs_free(kvlist); + return ret; +} + +static int +pmd_probe(struct rte_vdev_device *vdev) +{ + struct pmd_params p; + const char *params; + int status; + + struct rte_eth_dev_info hard_info; + uint32_t hard_speed; + uint16_t hard_port_id; + int numa_node; + void *dev_private; + + RTE_LOG(INFO, PMD, + "Probing device \"%s\"\n", + rte_vdev_device_name(vdev)); + + /* Parse input arguments */ + params = rte_vdev_device_args(vdev); + if (!params) + return -EINVAL; + + status = pmd_parse_args(&p, rte_vdev_device_name(vdev), params); + if (status) + return status; + + /* Check input arguments */ + if (rte_eth_dev_get_port_by_name(p.hard.name, &hard_port_id)) + return -EINVAL; + + rte_eth_dev_info_get(hard_port_id, &hard_info); + hard_speed = eth_dev_speed_max_mbps(hard_info.speed_capa); + numa_node = rte_eth_dev_socket_id(hard_port_id); + + if (p.hard.tx_queue_id >= hard_info.max_tx_queues) + return -EINVAL; + + if (p.soft.flags & PMD_FEATURE_TM) { + status = tm_params_check(&p, hard_speed); + + if (status) + return status; + } + + /* Allocate and initialize soft ethdev private data */ + dev_private = pmd_init(&p, numa_node); + if (dev_private == NULL) + return -ENOMEM; + + /* Register soft ethdev */ + RTE_LOG(INFO, PMD, + "Creating soft ethdev \"%s\" for hard ethdev \"%s\"\n", + p.soft.name, p.hard.name); + + status = pmd_ethdev_register(vdev, &p, dev_private); + if (status) { + pmd_free(dev_private); + return status; + } + + return 0; +} + +static int +pmd_remove(struct rte_vdev_device *vdev) +{ + struct rte_eth_dev *dev = NULL; + struct pmd_internals *p; + + if (!vdev) + return -EINVAL; + + RTE_LOG(INFO, PMD, "Removing device \"%s\"\n", + rte_vdev_device_name(vdev)); + + /* Find the ethdev entry */ + dev = rte_eth_dev_allocated(rte_vdev_device_name(vdev)); + if (dev == NULL) + return -ENODEV; + p = dev->data->dev_private; + + /* Free device data structures*/ + pmd_free(p); + rte_free(dev->data); + rte_eth_dev_release_port(dev); + + return 0; +} + +static struct rte_vdev_driver pmd_softnic_drv = { + .probe = pmd_probe, + .remove = pmd_remove, +}; + +RTE_PMD_REGISTER_VDEV(net_softnic, pmd_softnic_drv); +RTE_PMD_REGISTER_PARAM_STRING(net_softnic, + PMD_PARAM_SOFT_TM "=on|off " + PMD_PARAM_SOFT_TM_RATE "= " + PMD_PARAM_SOFT_TM_NB_QUEUES "= " + PMD_PARAM_SOFT_TM_QSIZE0 "= " + PMD_PARAM_SOFT_TM_QSIZE1 "= " + PMD_PARAM_SOFT_TM_QSIZE2 "= " + PMD_PARAM_SOFT_TM_QSIZE3 "= " + PMD_PARAM_SOFT_TM_ENQ_BSZ "= " + PMD_PARAM_SOFT_TM_DEQ_BSZ "= " + PMD_PARAM_HARD_NAME "= " + PMD_PARAM_HARD_TX_QUEUE_ID "="); diff --git a/dpdk/drivers/net/softnic/rte_eth_softnic.h b/dpdk/drivers/net/softnic/rte_eth_softnic.h new file mode 100644 index 00000000..b49e5829 --- /dev/null +++ b/dpdk/drivers/net/softnic/rte_eth_softnic.h @@ -0,0 +1,83 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __INCLUDE_RTE_ETH_SOFTNIC_H__ +#define __INCLUDE_RTE_ETH_SOFTNIC_H__ + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef SOFTNIC_SOFT_TM_NB_QUEUES +#define SOFTNIC_SOFT_TM_NB_QUEUES 65536 +#endif + +#ifndef SOFTNIC_SOFT_TM_QUEUE_SIZE +#define SOFTNIC_SOFT_TM_QUEUE_SIZE 64 +#endif + +#ifndef SOFTNIC_SOFT_TM_ENQ_BSZ +#define SOFTNIC_SOFT_TM_ENQ_BSZ 32 +#endif + +#ifndef SOFTNIC_SOFT_TM_DEQ_BSZ +#define SOFTNIC_SOFT_TM_DEQ_BSZ 24 +#endif + +#ifndef SOFTNIC_HARD_TX_QUEUE_ID +#define SOFTNIC_HARD_TX_QUEUE_ID 0 +#endif + +/** + * Run the traffic management function on the softnic device + * + * This function read the packets from the softnic input queues, insert into + * QoS scheduler queues based on mbuf sched field value and transmit the + * scheduled packets out through the hard device interface. + * + * @param portid + * port id of the soft device. + * @return + * zero. + */ + +int +rte_pmd_softnic_run(uint16_t port_id); + +#ifdef __cplusplus +} +#endif + +#endif /* __INCLUDE_RTE_ETH_SOFTNIC_H__ */ diff --git a/dpdk/drivers/net/softnic/rte_eth_softnic_internals.h b/dpdk/drivers/net/softnic/rte_eth_softnic_internals.h new file mode 100644 index 00000000..1f758069 --- /dev/null +++ b/dpdk/drivers/net/softnic/rte_eth_softnic_internals.h @@ -0,0 +1,291 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __INCLUDE_RTE_ETH_SOFTNIC_INTERNALS_H__ +#define __INCLUDE_RTE_ETH_SOFTNIC_INTERNALS_H__ + +#include + +#include +#include +#include +#include + +#include "rte_eth_softnic.h" + +/** + * PMD Parameters + */ + +enum pmd_feature { + PMD_FEATURE_TM = 1, /**< Traffic Management (TM) */ +}; + +#ifndef INTRUSIVE +#define INTRUSIVE 0 +#endif + +struct pmd_params { + /** Parameters for the soft device (to be created) */ + struct { + const char *name; /**< Name */ + uint32_t flags; /**< Flags */ + + /** 0 = Access hard device though API only (potentially slower, + * but safer); + * 1 = Access hard device private data structures is allowed + * (potentially faster). + */ + int intrusive; + + /** Traffic Management (TM) */ + struct { + uint32_t rate; /**< Rate (bytes/second) */ + uint32_t nb_queues; /**< Number of queues */ + uint16_t qsize[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE]; + /**< Queue size per traffic class */ + uint32_t enq_bsz; /**< Enqueue burst size */ + uint32_t deq_bsz; /**< Dequeue burst size */ + } tm; + } soft; + + /** Parameters for the hard device (existing) */ + struct { + char *name; /**< Name */ + uint16_t tx_queue_id; /**< TX queue ID */ + } hard; +}; + +/** + * Default Internals + */ + +#ifndef DEFAULT_BURST_SIZE +#define DEFAULT_BURST_SIZE 32 +#endif + +#ifndef FLUSH_COUNT_THRESHOLD +#define FLUSH_COUNT_THRESHOLD (1 << 17) +#endif + +struct default_internals { + struct rte_mbuf **pkts; + uint32_t pkts_len; + uint32_t txq_pos; + uint32_t flush_count; +}; + +/** + * Traffic Management (TM) Internals + */ + +#ifndef TM_MAX_SUBPORTS +#define TM_MAX_SUBPORTS 8 +#endif + +#ifndef TM_MAX_PIPES_PER_SUBPORT +#define TM_MAX_PIPES_PER_SUBPORT 4096 +#endif + +struct tm_params { + struct rte_sched_port_params port_params; + + struct rte_sched_subport_params subport_params[TM_MAX_SUBPORTS]; + + struct rte_sched_pipe_params + pipe_profiles[RTE_SCHED_PIPE_PROFILES_PER_PORT]; + uint32_t n_pipe_profiles; + uint32_t pipe_to_profile[TM_MAX_SUBPORTS * TM_MAX_PIPES_PER_SUBPORT]; +}; + +/* TM Levels */ +enum tm_node_level { + TM_NODE_LEVEL_PORT = 0, + TM_NODE_LEVEL_SUBPORT, + TM_NODE_LEVEL_PIPE, + TM_NODE_LEVEL_TC, + TM_NODE_LEVEL_QUEUE, + TM_NODE_LEVEL_MAX, +}; + +/* TM Shaper Profile */ +struct tm_shaper_profile { + TAILQ_ENTRY(tm_shaper_profile) node; + uint32_t shaper_profile_id; + uint32_t n_users; + struct rte_tm_shaper_params params; +}; + +TAILQ_HEAD(tm_shaper_profile_list, tm_shaper_profile); + +/* TM Shared Shaper */ +struct tm_shared_shaper { + TAILQ_ENTRY(tm_shared_shaper) node; + uint32_t shared_shaper_id; + uint32_t n_users; + uint32_t shaper_profile_id; +}; + +TAILQ_HEAD(tm_shared_shaper_list, tm_shared_shaper); + +/* TM WRED Profile */ +struct tm_wred_profile { + TAILQ_ENTRY(tm_wred_profile) node; + uint32_t wred_profile_id; + uint32_t n_users; + struct rte_tm_wred_params params; +}; + +TAILQ_HEAD(tm_wred_profile_list, tm_wred_profile); + +/* TM Node */ +struct tm_node { + TAILQ_ENTRY(tm_node) node; + uint32_t node_id; + uint32_t parent_node_id; + uint32_t priority; + uint32_t weight; + uint32_t level; + struct tm_node *parent_node; + struct tm_shaper_profile *shaper_profile; + struct tm_wred_profile *wred_profile; + struct rte_tm_node_params params; + struct rte_tm_node_stats stats; + uint32_t n_children; +}; + +TAILQ_HEAD(tm_node_list, tm_node); + +/* TM Hierarchy Specification */ +struct tm_hierarchy { + struct tm_shaper_profile_list shaper_profiles; + struct tm_shared_shaper_list shared_shapers; + struct tm_wred_profile_list wred_profiles; + struct tm_node_list nodes; + + uint32_t n_shaper_profiles; + uint32_t n_shared_shapers; + uint32_t n_wred_profiles; + uint32_t n_nodes; + + uint32_t n_tm_nodes[TM_NODE_LEVEL_MAX]; +}; + +struct tm_internals { + /** Hierarchy specification + * + * -Hierarchy is unfrozen at init and when port is stopped. + * -Hierarchy is frozen on successful hierarchy commit. + * -Run-time hierarchy changes are not allowed, therefore it makes + * sense to keep the hierarchy frozen after the port is started. + */ + struct tm_hierarchy h; + int hierarchy_frozen; + + /** Blueprints */ + struct tm_params params; + + /** Run-time */ + struct rte_sched_port *sched; + struct rte_mbuf **pkts_enq; + struct rte_mbuf **pkts_deq; + uint32_t pkts_enq_len; + uint32_t txq_pos; + uint32_t flush_count; +}; + +/** + * PMD Internals + */ +struct pmd_internals { + /** Params */ + struct pmd_params params; + + /** Soft device */ + struct { + struct default_internals def; /**< Default */ + struct tm_internals tm; /**< Traffic Management */ + } soft; + + /** Hard device */ + struct { + uint16_t port_id; + } hard; +}; + +struct pmd_rx_queue { + /** Hard device */ + struct { + uint16_t port_id; + uint16_t rx_queue_id; + } hard; +}; + +/** + * Traffic Management (TM) Operation + */ +extern const struct rte_tm_ops pmd_tm_ops; + +int +tm_params_check(struct pmd_params *params, uint32_t hard_rate); + +int +tm_init(struct pmd_internals *p, struct pmd_params *params, int numa_node); + +void +tm_free(struct pmd_internals *p); + +int +tm_start(struct pmd_internals *p); + +void +tm_stop(struct pmd_internals *p); + +static inline int +tm_enabled(struct rte_eth_dev *dev) +{ + struct pmd_internals *p = dev->data->dev_private; + + return (p->params.soft.flags & PMD_FEATURE_TM); +} + +static inline int +tm_used(struct rte_eth_dev *dev) +{ + struct pmd_internals *p = dev->data->dev_private; + + return (p->params.soft.flags & PMD_FEATURE_TM) && + p->soft.tm.h.n_tm_nodes[TM_NODE_LEVEL_PORT]; +} + +#endif /* __INCLUDE_RTE_ETH_SOFTNIC_INTERNALS_H__ */ diff --git a/dpdk/drivers/net/softnic/rte_eth_softnic_tm.c b/dpdk/drivers/net/softnic/rte_eth_softnic_tm.c new file mode 100644 index 00000000..a4599006 --- /dev/null +++ b/dpdk/drivers/net/softnic/rte_eth_softnic_tm.c @@ -0,0 +1,3452 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include + +#include + +#include "rte_eth_softnic_internals.h" +#include "rte_eth_softnic.h" + +#define BYTES_IN_MBPS (1000 * 1000 / 8) +#define SUBPORT_TC_PERIOD 10 +#define PIPE_TC_PERIOD 40 + +int +tm_params_check(struct pmd_params *params, uint32_t hard_rate) +{ + uint64_t hard_rate_bytes_per_sec = (uint64_t)hard_rate * BYTES_IN_MBPS; + uint32_t i; + + /* rate */ + if (params->soft.tm.rate) { + if (params->soft.tm.rate > hard_rate_bytes_per_sec) + return -EINVAL; + } else { + params->soft.tm.rate = + (hard_rate_bytes_per_sec > UINT32_MAX) ? + UINT32_MAX : hard_rate_bytes_per_sec; + } + + /* nb_queues */ + if (params->soft.tm.nb_queues == 0) + return -EINVAL; + + if (params->soft.tm.nb_queues < RTE_SCHED_QUEUES_PER_PIPE) + params->soft.tm.nb_queues = RTE_SCHED_QUEUES_PER_PIPE; + + params->soft.tm.nb_queues = + rte_align32pow2(params->soft.tm.nb_queues); + + /* qsize */ + for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++) { + if (params->soft.tm.qsize[i] == 0) + return -EINVAL; + + params->soft.tm.qsize[i] = + rte_align32pow2(params->soft.tm.qsize[i]); + } + + /* enq_bsz, deq_bsz */ + if (params->soft.tm.enq_bsz == 0 || + params->soft.tm.deq_bsz == 0 || + params->soft.tm.deq_bsz >= params->soft.tm.enq_bsz) + return -EINVAL; + + return 0; +} + +static void +tm_hierarchy_init(struct pmd_internals *p) +{ + memset(&p->soft.tm.h, 0, sizeof(p->soft.tm.h)); + + /* Initialize shaper profile list */ + TAILQ_INIT(&p->soft.tm.h.shaper_profiles); + + /* Initialize shared shaper list */ + TAILQ_INIT(&p->soft.tm.h.shared_shapers); + + /* Initialize wred profile list */ + TAILQ_INIT(&p->soft.tm.h.wred_profiles); + + /* Initialize TM node list */ + TAILQ_INIT(&p->soft.tm.h.nodes); +} + +static void +tm_hierarchy_uninit(struct pmd_internals *p) +{ + /* Remove all nodes*/ + for ( ; ; ) { + struct tm_node *tm_node; + + tm_node = TAILQ_FIRST(&p->soft.tm.h.nodes); + if (tm_node == NULL) + break; + + TAILQ_REMOVE(&p->soft.tm.h.nodes, tm_node, node); + free(tm_node); + } + + /* Remove all WRED profiles */ + for ( ; ; ) { + struct tm_wred_profile *wred_profile; + + wred_profile = TAILQ_FIRST(&p->soft.tm.h.wred_profiles); + if (wred_profile == NULL) + break; + + TAILQ_REMOVE(&p->soft.tm.h.wred_profiles, wred_profile, node); + free(wred_profile); + } + + /* Remove all shared shapers */ + for ( ; ; ) { + struct tm_shared_shaper *shared_shaper; + + shared_shaper = TAILQ_FIRST(&p->soft.tm.h.shared_shapers); + if (shared_shaper == NULL) + break; + + TAILQ_REMOVE(&p->soft.tm.h.shared_shapers, shared_shaper, node); + free(shared_shaper); + } + + /* Remove all shaper profiles */ + for ( ; ; ) { + struct tm_shaper_profile *shaper_profile; + + shaper_profile = TAILQ_FIRST(&p->soft.tm.h.shaper_profiles); + if (shaper_profile == NULL) + break; + + TAILQ_REMOVE(&p->soft.tm.h.shaper_profiles, + shaper_profile, node); + free(shaper_profile); + } + + memset(&p->soft.tm.h, 0, sizeof(p->soft.tm.h)); +} + +int +tm_init(struct pmd_internals *p, + struct pmd_params *params, + int numa_node) +{ + uint32_t enq_bsz = params->soft.tm.enq_bsz; + uint32_t deq_bsz = params->soft.tm.deq_bsz; + + p->soft.tm.pkts_enq = rte_zmalloc_socket(params->soft.name, + 2 * enq_bsz * sizeof(struct rte_mbuf *), + 0, + numa_node); + + if (p->soft.tm.pkts_enq == NULL) + return -ENOMEM; + + p->soft.tm.pkts_deq = rte_zmalloc_socket(params->soft.name, + deq_bsz * sizeof(struct rte_mbuf *), + 0, + numa_node); + + if (p->soft.tm.pkts_deq == NULL) { + rte_free(p->soft.tm.pkts_enq); + return -ENOMEM; + } + + tm_hierarchy_init(p); + + return 0; +} + +void +tm_free(struct pmd_internals *p) +{ + tm_hierarchy_uninit(p); + rte_free(p->soft.tm.pkts_enq); + rte_free(p->soft.tm.pkts_deq); +} + +int +tm_start(struct pmd_internals *p) +{ + struct tm_params *t = &p->soft.tm.params; + uint32_t n_subports, subport_id; + int status; + + /* Is hierarchy frozen? */ + if (p->soft.tm.hierarchy_frozen == 0) + return -1; + + /* Port */ + p->soft.tm.sched = rte_sched_port_config(&t->port_params); + if (p->soft.tm.sched == NULL) + return -1; + + /* Subport */ + n_subports = t->port_params.n_subports_per_port; + for (subport_id = 0; subport_id < n_subports; subport_id++) { + uint32_t n_pipes_per_subport = + t->port_params.n_pipes_per_subport; + uint32_t pipe_id; + + status = rte_sched_subport_config(p->soft.tm.sched, + subport_id, + &t->subport_params[subport_id]); + if (status) { + rte_sched_port_free(p->soft.tm.sched); + return -1; + } + + /* Pipe */ + n_pipes_per_subport = t->port_params.n_pipes_per_subport; + for (pipe_id = 0; pipe_id < n_pipes_per_subport; pipe_id++) { + int pos = subport_id * TM_MAX_PIPES_PER_SUBPORT + + pipe_id; + int profile_id = t->pipe_to_profile[pos]; + + if (profile_id < 0) + continue; + + status = rte_sched_pipe_config(p->soft.tm.sched, + subport_id, + pipe_id, + profile_id); + if (status) { + rte_sched_port_free(p->soft.tm.sched); + return -1; + } + } + } + + return 0; +} + +void +tm_stop(struct pmd_internals *p) +{ + if (p->soft.tm.sched) + rte_sched_port_free(p->soft.tm.sched); + + /* Unfreeze hierarchy */ + p->soft.tm.hierarchy_frozen = 0; +} + +static struct tm_shaper_profile * +tm_shaper_profile_search(struct rte_eth_dev *dev, uint32_t shaper_profile_id) +{ + struct pmd_internals *p = dev->data->dev_private; + struct tm_shaper_profile_list *spl = &p->soft.tm.h.shaper_profiles; + struct tm_shaper_profile *sp; + + TAILQ_FOREACH(sp, spl, node) + if (shaper_profile_id == sp->shaper_profile_id) + return sp; + + return NULL; +} + +static struct tm_shared_shaper * +tm_shared_shaper_search(struct rte_eth_dev *dev, uint32_t shared_shaper_id) +{ + struct pmd_internals *p = dev->data->dev_private; + struct tm_shared_shaper_list *ssl = &p->soft.tm.h.shared_shapers; + struct tm_shared_shaper *ss; + + TAILQ_FOREACH(ss, ssl, node) + if (shared_shaper_id == ss->shared_shaper_id) + return ss; + + return NULL; +} + +static struct tm_wred_profile * +tm_wred_profile_search(struct rte_eth_dev *dev, uint32_t wred_profile_id) +{ + struct pmd_internals *p = dev->data->dev_private; + struct tm_wred_profile_list *wpl = &p->soft.tm.h.wred_profiles; + struct tm_wred_profile *wp; + + TAILQ_FOREACH(wp, wpl, node) + if (wred_profile_id == wp->wred_profile_id) + return wp; + + return NULL; +} + +static struct tm_node * +tm_node_search(struct rte_eth_dev *dev, uint32_t node_id) +{ + struct pmd_internals *p = dev->data->dev_private; + struct tm_node_list *nl = &p->soft.tm.h.nodes; + struct tm_node *n; + + TAILQ_FOREACH(n, nl, node) + if (n->node_id == node_id) + return n; + + return NULL; +} + +static struct tm_node * +tm_root_node_present(struct rte_eth_dev *dev) +{ + struct pmd_internals *p = dev->data->dev_private; + struct tm_node_list *nl = &p->soft.tm.h.nodes; + struct tm_node *n; + + TAILQ_FOREACH(n, nl, node) + if (n->parent_node_id == RTE_TM_NODE_ID_NULL) + return n; + + return NULL; +} + +static uint32_t +tm_node_subport_id(struct rte_eth_dev *dev, struct tm_node *subport_node) +{ + struct pmd_internals *p = dev->data->dev_private; + struct tm_node_list *nl = &p->soft.tm.h.nodes; + struct tm_node *ns; + uint32_t subport_id; + + subport_id = 0; + TAILQ_FOREACH(ns, nl, node) { + if (ns->level != TM_NODE_LEVEL_SUBPORT) + continue; + + if (ns->node_id == subport_node->node_id) + return subport_id; + + subport_id++; + } + + return UINT32_MAX; +} + +static uint32_t +tm_node_pipe_id(struct rte_eth_dev *dev, struct tm_node *pipe_node) +{ + struct pmd_internals *p = dev->data->dev_private; + struct tm_node_list *nl = &p->soft.tm.h.nodes; + struct tm_node *np; + uint32_t pipe_id; + + pipe_id = 0; + TAILQ_FOREACH(np, nl, node) { + if (np->level != TM_NODE_LEVEL_PIPE || + np->parent_node_id != pipe_node->parent_node_id) + continue; + + if (np->node_id == pipe_node->node_id) + return pipe_id; + + pipe_id++; + } + + return UINT32_MAX; +} + +static uint32_t +tm_node_tc_id(struct rte_eth_dev *dev __rte_unused, struct tm_node *tc_node) +{ + return tc_node->priority; +} + +static uint32_t +tm_node_queue_id(struct rte_eth_dev *dev, struct tm_node *queue_node) +{ + struct pmd_internals *p = dev->data->dev_private; + struct tm_node_list *nl = &p->soft.tm.h.nodes; + struct tm_node *nq; + uint32_t queue_id; + + queue_id = 0; + TAILQ_FOREACH(nq, nl, node) { + if (nq->level != TM_NODE_LEVEL_QUEUE || + nq->parent_node_id != queue_node->parent_node_id) + continue; + + if (nq->node_id == queue_node->node_id) + return queue_id; + + queue_id++; + } + + return UINT32_MAX; +} + +static uint32_t +tm_level_get_max_nodes(struct rte_eth_dev *dev, enum tm_node_level level) +{ + struct pmd_internals *p = dev->data->dev_private; + uint32_t n_queues_max = p->params.soft.tm.nb_queues; + uint32_t n_tc_max = n_queues_max / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS; + uint32_t n_pipes_max = n_tc_max / RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; + uint32_t n_subports_max = n_pipes_max; + uint32_t n_root_max = 1; + + switch (level) { + case TM_NODE_LEVEL_PORT: + return n_root_max; + case TM_NODE_LEVEL_SUBPORT: + return n_subports_max; + case TM_NODE_LEVEL_PIPE: + return n_pipes_max; + case TM_NODE_LEVEL_TC: + return n_tc_max; + case TM_NODE_LEVEL_QUEUE: + default: + return n_queues_max; + } +} + +/* Traffic manager node type get */ +static int +pmd_tm_node_type_get(struct rte_eth_dev *dev, + uint32_t node_id, + int *is_leaf, + struct rte_tm_error *error) +{ + struct pmd_internals *p = dev->data->dev_private; + + if (is_leaf == NULL) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_UNSPECIFIED, + NULL, + rte_strerror(EINVAL)); + + if (node_id == RTE_TM_NODE_ID_NULL || + (tm_node_search(dev, node_id) == NULL)) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_NODE_ID, + NULL, + rte_strerror(EINVAL)); + + *is_leaf = node_id < p->params.soft.tm.nb_queues; + + return 0; +} + +#ifdef RTE_SCHED_RED +#define WRED_SUPPORTED 1 +#else +#define WRED_SUPPORTED 0 +#endif + +#define STATS_MASK_DEFAULT \ + (RTE_TM_STATS_N_PKTS | \ + RTE_TM_STATS_N_BYTES | \ + RTE_TM_STATS_N_PKTS_GREEN_DROPPED | \ + RTE_TM_STATS_N_BYTES_GREEN_DROPPED) + +#define STATS_MASK_QUEUE \ + (STATS_MASK_DEFAULT | \ + RTE_TM_STATS_N_PKTS_QUEUED) + +static const struct rte_tm_capabilities tm_cap = { + .n_nodes_max = UINT32_MAX, + .n_levels_max = TM_NODE_LEVEL_MAX, + + .non_leaf_nodes_identical = 0, + .leaf_nodes_identical = 1, + + .shaper_n_max = UINT32_MAX, + .shaper_private_n_max = UINT32_MAX, + .shaper_private_dual_rate_n_max = 0, + .shaper_private_rate_min = 1, + .shaper_private_rate_max = UINT32_MAX, + + .shaper_shared_n_max = UINT32_MAX, + .shaper_shared_n_nodes_per_shaper_max = UINT32_MAX, + .shaper_shared_n_shapers_per_node_max = 1, + .shaper_shared_dual_rate_n_max = 0, + .shaper_shared_rate_min = 1, + .shaper_shared_rate_max = UINT32_MAX, + + .shaper_pkt_length_adjust_min = RTE_TM_ETH_FRAMING_OVERHEAD_FCS, + .shaper_pkt_length_adjust_max = RTE_TM_ETH_FRAMING_OVERHEAD_FCS, + + .sched_n_children_max = UINT32_MAX, + .sched_sp_n_priorities_max = RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE, + .sched_wfq_n_children_per_group_max = UINT32_MAX, + .sched_wfq_n_groups_max = 1, + .sched_wfq_weight_max = UINT32_MAX, + + .cman_head_drop_supported = 0, + .cman_wred_context_n_max = 0, + .cman_wred_context_private_n_max = 0, + .cman_wred_context_shared_n_max = 0, + .cman_wred_context_shared_n_nodes_per_context_max = 0, + .cman_wred_context_shared_n_contexts_per_node_max = 0, + + .mark_vlan_dei_supported = {0, 0, 0}, + .mark_ip_ecn_tcp_supported = {0, 0, 0}, + .mark_ip_ecn_sctp_supported = {0, 0, 0}, + .mark_ip_dscp_supported = {0, 0, 0}, + + .dynamic_update_mask = 0, + + .stats_mask = STATS_MASK_QUEUE, +}; + +/* Traffic manager capabilities get */ +static int +pmd_tm_capabilities_get(struct rte_eth_dev *dev __rte_unused, + struct rte_tm_capabilities *cap, + struct rte_tm_error *error) +{ + if (cap == NULL) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_CAPABILITIES, + NULL, + rte_strerror(EINVAL)); + + memcpy(cap, &tm_cap, sizeof(*cap)); + + cap->n_nodes_max = tm_level_get_max_nodes(dev, TM_NODE_LEVEL_PORT) + + tm_level_get_max_nodes(dev, TM_NODE_LEVEL_SUBPORT) + + tm_level_get_max_nodes(dev, TM_NODE_LEVEL_PIPE) + + tm_level_get_max_nodes(dev, TM_NODE_LEVEL_TC) + + tm_level_get_max_nodes(dev, TM_NODE_LEVEL_QUEUE); + + cap->shaper_private_n_max = + tm_level_get_max_nodes(dev, TM_NODE_LEVEL_PORT) + + tm_level_get_max_nodes(dev, TM_NODE_LEVEL_SUBPORT) + + tm_level_get_max_nodes(dev, TM_NODE_LEVEL_PIPE) + + tm_level_get_max_nodes(dev, TM_NODE_LEVEL_TC); + + cap->shaper_shared_n_max = RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE * + tm_level_get_max_nodes(dev, TM_NODE_LEVEL_SUBPORT); + + cap->shaper_n_max = cap->shaper_private_n_max + + cap->shaper_shared_n_max; + + cap->shaper_shared_n_nodes_per_shaper_max = + tm_level_get_max_nodes(dev, TM_NODE_LEVEL_PIPE); + + cap->sched_n_children_max = RTE_MAX( + tm_level_get_max_nodes(dev, TM_NODE_LEVEL_PIPE), + (uint32_t)RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE); + + cap->sched_wfq_n_children_per_group_max = cap->sched_n_children_max; + + if (WRED_SUPPORTED) + cap->cman_wred_context_private_n_max = + tm_level_get_max_nodes(dev, TM_NODE_LEVEL_QUEUE); + + cap->cman_wred_context_n_max = cap->cman_wred_context_private_n_max + + cap->cman_wred_context_shared_n_max; + + return 0; +} + +static const struct rte_tm_level_capabilities tm_level_cap[] = { + [TM_NODE_LEVEL_PORT] = { + .n_nodes_max = 1, + .n_nodes_nonleaf_max = 1, + .n_nodes_leaf_max = 0, + .non_leaf_nodes_identical = 1, + .leaf_nodes_identical = 0, + + {.nonleaf = { + .shaper_private_supported = 1, + .shaper_private_dual_rate_supported = 0, + .shaper_private_rate_min = 1, + .shaper_private_rate_max = UINT32_MAX, + .shaper_shared_n_max = 0, + + .sched_n_children_max = UINT32_MAX, + .sched_sp_n_priorities_max = 1, + .sched_wfq_n_children_per_group_max = UINT32_MAX, + .sched_wfq_n_groups_max = 1, + .sched_wfq_weight_max = 1, + + .stats_mask = STATS_MASK_DEFAULT, + } }, + }, + + [TM_NODE_LEVEL_SUBPORT] = { + .n_nodes_max = UINT32_MAX, + .n_nodes_nonleaf_max = UINT32_MAX, + .n_nodes_leaf_max = 0, + .non_leaf_nodes_identical = 1, + .leaf_nodes_identical = 0, + + {.nonleaf = { + .shaper_private_supported = 1, + .shaper_private_dual_rate_supported = 0, + .shaper_private_rate_min = 1, + .shaper_private_rate_max = UINT32_MAX, + .shaper_shared_n_max = 0, + + .sched_n_children_max = UINT32_MAX, + .sched_sp_n_priorities_max = 1, + .sched_wfq_n_children_per_group_max = UINT32_MAX, + .sched_wfq_n_groups_max = 1, +#ifdef RTE_SCHED_SUBPORT_TC_OV + .sched_wfq_weight_max = UINT32_MAX, +#else + .sched_wfq_weight_max = 1, +#endif + .stats_mask = STATS_MASK_DEFAULT, + } }, + }, + + [TM_NODE_LEVEL_PIPE] = { + .n_nodes_max = UINT32_MAX, + .n_nodes_nonleaf_max = UINT32_MAX, + .n_nodes_leaf_max = 0, + .non_leaf_nodes_identical = 1, + .leaf_nodes_identical = 0, + + {.nonleaf = { + .shaper_private_supported = 1, + .shaper_private_dual_rate_supported = 0, + .shaper_private_rate_min = 1, + .shaper_private_rate_max = UINT32_MAX, + .shaper_shared_n_max = 0, + + .sched_n_children_max = + RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE, + .sched_sp_n_priorities_max = + RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE, + .sched_wfq_n_children_per_group_max = 1, + .sched_wfq_n_groups_max = 0, + .sched_wfq_weight_max = 1, + + .stats_mask = STATS_MASK_DEFAULT, + } }, + }, + + [TM_NODE_LEVEL_TC] = { + .n_nodes_max = UINT32_MAX, + .n_nodes_nonleaf_max = UINT32_MAX, + .n_nodes_leaf_max = 0, + .non_leaf_nodes_identical = 1, + .leaf_nodes_identical = 0, + + {.nonleaf = { + .shaper_private_supported = 1, + .shaper_private_dual_rate_supported = 0, + .shaper_private_rate_min = 1, + .shaper_private_rate_max = UINT32_MAX, + .shaper_shared_n_max = 1, + + .sched_n_children_max = + RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS, + .sched_sp_n_priorities_max = 1, + .sched_wfq_n_children_per_group_max = + RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS, + .sched_wfq_n_groups_max = 1, + .sched_wfq_weight_max = UINT32_MAX, + + .stats_mask = STATS_MASK_DEFAULT, + } }, + }, + + [TM_NODE_LEVEL_QUEUE] = { + .n_nodes_max = UINT32_MAX, + .n_nodes_nonleaf_max = 0, + .n_nodes_leaf_max = UINT32_MAX, + .non_leaf_nodes_identical = 0, + .leaf_nodes_identical = 1, + + {.leaf = { + .shaper_private_supported = 0, + .shaper_private_dual_rate_supported = 0, + .shaper_private_rate_min = 0, + .shaper_private_rate_max = 0, + .shaper_shared_n_max = 0, + + .cman_head_drop_supported = 0, + .cman_wred_context_private_supported = WRED_SUPPORTED, + .cman_wred_context_shared_n_max = 0, + + .stats_mask = STATS_MASK_QUEUE, + } }, + }, +}; + +/* Traffic manager level capabilities get */ +static int +pmd_tm_level_capabilities_get(struct rte_eth_dev *dev __rte_unused, + uint32_t level_id, + struct rte_tm_level_capabilities *cap, + struct rte_tm_error *error) +{ + if (cap == NULL) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_CAPABILITIES, + NULL, + rte_strerror(EINVAL)); + + if (level_id >= TM_NODE_LEVEL_MAX) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_LEVEL_ID, + NULL, + rte_strerror(EINVAL)); + + memcpy(cap, &tm_level_cap[level_id], sizeof(*cap)); + + switch (level_id) { + case TM_NODE_LEVEL_PORT: + cap->nonleaf.sched_n_children_max = + tm_level_get_max_nodes(dev, + TM_NODE_LEVEL_SUBPORT); + cap->nonleaf.sched_wfq_n_children_per_group_max = + cap->nonleaf.sched_n_children_max; + break; + + case TM_NODE_LEVEL_SUBPORT: + cap->n_nodes_max = tm_level_get_max_nodes(dev, + TM_NODE_LEVEL_SUBPORT); + cap->n_nodes_nonleaf_max = cap->n_nodes_max; + cap->nonleaf.sched_n_children_max = + tm_level_get_max_nodes(dev, + TM_NODE_LEVEL_PIPE); + cap->nonleaf.sched_wfq_n_children_per_group_max = + cap->nonleaf.sched_n_children_max; + break; + + case TM_NODE_LEVEL_PIPE: + cap->n_nodes_max = tm_level_get_max_nodes(dev, + TM_NODE_LEVEL_PIPE); + cap->n_nodes_nonleaf_max = cap->n_nodes_max; + break; + + case TM_NODE_LEVEL_TC: + cap->n_nodes_max = tm_level_get_max_nodes(dev, + TM_NODE_LEVEL_TC); + cap->n_nodes_nonleaf_max = cap->n_nodes_max; + break; + + case TM_NODE_LEVEL_QUEUE: + default: + cap->n_nodes_max = tm_level_get_max_nodes(dev, + TM_NODE_LEVEL_QUEUE); + cap->n_nodes_leaf_max = cap->n_nodes_max; + break; + } + + return 0; +} + +static const struct rte_tm_node_capabilities tm_node_cap[] = { + [TM_NODE_LEVEL_PORT] = { + .shaper_private_supported = 1, + .shaper_private_dual_rate_supported = 0, + .shaper_private_rate_min = 1, + .shaper_private_rate_max = UINT32_MAX, + .shaper_shared_n_max = 0, + + {.nonleaf = { + .sched_n_children_max = UINT32_MAX, + .sched_sp_n_priorities_max = 1, + .sched_wfq_n_children_per_group_max = UINT32_MAX, + .sched_wfq_n_groups_max = 1, + .sched_wfq_weight_max = 1, + } }, + + .stats_mask = STATS_MASK_DEFAULT, + }, + + [TM_NODE_LEVEL_SUBPORT] = { + .shaper_private_supported = 1, + .shaper_private_dual_rate_supported = 0, + .shaper_private_rate_min = 1, + .shaper_private_rate_max = UINT32_MAX, + .shaper_shared_n_max = 0, + + {.nonleaf = { + .sched_n_children_max = UINT32_MAX, + .sched_sp_n_priorities_max = 1, + .sched_wfq_n_children_per_group_max = UINT32_MAX, + .sched_wfq_n_groups_max = 1, + .sched_wfq_weight_max = UINT32_MAX, + } }, + + .stats_mask = STATS_MASK_DEFAULT, + }, + + [TM_NODE_LEVEL_PIPE] = { + .shaper_private_supported = 1, + .shaper_private_dual_rate_supported = 0, + .shaper_private_rate_min = 1, + .shaper_private_rate_max = UINT32_MAX, + .shaper_shared_n_max = 0, + + {.nonleaf = { + .sched_n_children_max = + RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE, + .sched_sp_n_priorities_max = + RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE, + .sched_wfq_n_children_per_group_max = 1, + .sched_wfq_n_groups_max = 0, + .sched_wfq_weight_max = 1, + } }, + + .stats_mask = STATS_MASK_DEFAULT, + }, + + [TM_NODE_LEVEL_TC] = { + .shaper_private_supported = 1, + .shaper_private_dual_rate_supported = 0, + .shaper_private_rate_min = 1, + .shaper_private_rate_max = UINT32_MAX, + .shaper_shared_n_max = 1, + + {.nonleaf = { + .sched_n_children_max = + RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS, + .sched_sp_n_priorities_max = 1, + .sched_wfq_n_children_per_group_max = + RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS, + .sched_wfq_n_groups_max = 1, + .sched_wfq_weight_max = UINT32_MAX, + } }, + + .stats_mask = STATS_MASK_DEFAULT, + }, + + [TM_NODE_LEVEL_QUEUE] = { + .shaper_private_supported = 0, + .shaper_private_dual_rate_supported = 0, + .shaper_private_rate_min = 0, + .shaper_private_rate_max = 0, + .shaper_shared_n_max = 0, + + + {.leaf = { + .cman_head_drop_supported = 0, + .cman_wred_context_private_supported = WRED_SUPPORTED, + .cman_wred_context_shared_n_max = 0, + } }, + + .stats_mask = STATS_MASK_QUEUE, + }, +}; + +/* Traffic manager node capabilities get */ +static int +pmd_tm_node_capabilities_get(struct rte_eth_dev *dev __rte_unused, + uint32_t node_id, + struct rte_tm_node_capabilities *cap, + struct rte_tm_error *error) +{ + struct tm_node *tm_node; + + if (cap == NULL) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_CAPABILITIES, + NULL, + rte_strerror(EINVAL)); + + tm_node = tm_node_search(dev, node_id); + if (tm_node == NULL) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_NODE_ID, + NULL, + rte_strerror(EINVAL)); + + memcpy(cap, &tm_node_cap[tm_node->level], sizeof(*cap)); + + switch (tm_node->level) { + case TM_NODE_LEVEL_PORT: + cap->nonleaf.sched_n_children_max = + tm_level_get_max_nodes(dev, + TM_NODE_LEVEL_SUBPORT); + cap->nonleaf.sched_wfq_n_children_per_group_max = + cap->nonleaf.sched_n_children_max; + break; + + case TM_NODE_LEVEL_SUBPORT: + cap->nonleaf.sched_n_children_max = + tm_level_get_max_nodes(dev, + TM_NODE_LEVEL_PIPE); + cap->nonleaf.sched_wfq_n_children_per_group_max = + cap->nonleaf.sched_n_children_max; + break; + + case TM_NODE_LEVEL_PIPE: + case TM_NODE_LEVEL_TC: + case TM_NODE_LEVEL_QUEUE: + default: + break; + } + + return 0; +} + +static int +shaper_profile_check(struct rte_eth_dev *dev, + uint32_t shaper_profile_id, + struct rte_tm_shaper_params *profile, + struct rte_tm_error *error) +{ + struct tm_shaper_profile *sp; + + /* Shaper profile ID must not be NONE. */ + if (shaper_profile_id == RTE_TM_SHAPER_PROFILE_ID_NONE) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_SHAPER_PROFILE_ID, + NULL, + rte_strerror(EINVAL)); + + /* Shaper profile must not exist. */ + sp = tm_shaper_profile_search(dev, shaper_profile_id); + if (sp) + return -rte_tm_error_set(error, + EEXIST, + RTE_TM_ERROR_TYPE_SHAPER_PROFILE_ID, + NULL, + rte_strerror(EEXIST)); + + /* Profile must not be NULL. */ + if (profile == NULL) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_SHAPER_PROFILE, + NULL, + rte_strerror(EINVAL)); + + /* Peak rate: non-zero, 32-bit */ + if (profile->peak.rate == 0 || + profile->peak.rate >= UINT32_MAX) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_SHAPER_PROFILE_PEAK_RATE, + NULL, + rte_strerror(EINVAL)); + + /* Peak size: non-zero, 32-bit */ + if (profile->peak.size == 0 || + profile->peak.size >= UINT32_MAX) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_SHAPER_PROFILE_PEAK_SIZE, + NULL, + rte_strerror(EINVAL)); + + /* Dual-rate profiles are not supported. */ + if (profile->committed.rate != 0) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_SHAPER_PROFILE_COMMITTED_RATE, + NULL, + rte_strerror(EINVAL)); + + /* Packet length adjust: 24 bytes */ + if (profile->pkt_length_adjust != RTE_TM_ETH_FRAMING_OVERHEAD_FCS) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_SHAPER_PROFILE_PKT_ADJUST_LEN, + NULL, + rte_strerror(EINVAL)); + + return 0; +} + +/* Traffic manager shaper profile add */ +static int +pmd_tm_shaper_profile_add(struct rte_eth_dev *dev, + uint32_t shaper_profile_id, + struct rte_tm_shaper_params *profile, + struct rte_tm_error *error) +{ + struct pmd_internals *p = dev->data->dev_private; + struct tm_shaper_profile_list *spl = &p->soft.tm.h.shaper_profiles; + struct tm_shaper_profile *sp; + int status; + + /* Check input params */ + status = shaper_profile_check(dev, shaper_profile_id, profile, error); + if (status) + return status; + + /* Memory allocation */ + sp = calloc(1, sizeof(struct tm_shaper_profile)); + if (sp == NULL) + return -rte_tm_error_set(error, + ENOMEM, + RTE_TM_ERROR_TYPE_UNSPECIFIED, + NULL, + rte_strerror(ENOMEM)); + + /* Fill in */ + sp->shaper_profile_id = shaper_profile_id; + memcpy(&sp->params, profile, sizeof(sp->params)); + + /* Add to list */ + TAILQ_INSERT_TAIL(spl, sp, node); + p->soft.tm.h.n_shaper_profiles++; + + return 0; +} + +/* Traffic manager shaper profile delete */ +static int +pmd_tm_shaper_profile_delete(struct rte_eth_dev *dev, + uint32_t shaper_profile_id, + struct rte_tm_error *error) +{ + struct pmd_internals *p = dev->data->dev_private; + struct tm_shaper_profile *sp; + + /* Check existing */ + sp = tm_shaper_profile_search(dev, shaper_profile_id); + if (sp == NULL) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_SHAPER_PROFILE_ID, + NULL, + rte_strerror(EINVAL)); + + /* Check unused */ + if (sp->n_users) + return -rte_tm_error_set(error, + EBUSY, + RTE_TM_ERROR_TYPE_SHAPER_PROFILE_ID, + NULL, + rte_strerror(EBUSY)); + + /* Remove from list */ + TAILQ_REMOVE(&p->soft.tm.h.shaper_profiles, sp, node); + p->soft.tm.h.n_shaper_profiles--; + free(sp); + + return 0; +} + +static struct tm_node * +tm_shared_shaper_get_tc(struct rte_eth_dev *dev, + struct tm_shared_shaper *ss) +{ + struct pmd_internals *p = dev->data->dev_private; + struct tm_node_list *nl = &p->soft.tm.h.nodes; + struct tm_node *n; + + /* Subport: each TC uses shared shaper */ + TAILQ_FOREACH(n, nl, node) { + if (n->level != TM_NODE_LEVEL_TC || + n->params.n_shared_shapers == 0 || + n->params.shared_shaper_id[0] != ss->shared_shaper_id) + continue; + + return n; + } + + return NULL; +} + +static int +update_subport_tc_rate(struct rte_eth_dev *dev, + struct tm_node *nt, + struct tm_shared_shaper *ss, + struct tm_shaper_profile *sp_new) +{ + struct pmd_internals *p = dev->data->dev_private; + uint32_t tc_id = tm_node_tc_id(dev, nt); + + struct tm_node *np = nt->parent_node; + + struct tm_node *ns = np->parent_node; + uint32_t subport_id = tm_node_subport_id(dev, ns); + + struct rte_sched_subport_params subport_params; + + struct tm_shaper_profile *sp_old = tm_shaper_profile_search(dev, + ss->shaper_profile_id); + + /* Derive new subport configuration. */ + memcpy(&subport_params, + &p->soft.tm.params.subport_params[subport_id], + sizeof(subport_params)); + subport_params.tc_rate[tc_id] = sp_new->params.peak.rate; + + /* Update the subport configuration. */ + if (rte_sched_subport_config(p->soft.tm.sched, + subport_id, &subport_params)) + return -1; + + /* Commit changes. */ + sp_old->n_users--; + + ss->shaper_profile_id = sp_new->shaper_profile_id; + sp_new->n_users++; + + memcpy(&p->soft.tm.params.subport_params[subport_id], + &subport_params, + sizeof(subport_params)); + + return 0; +} + +/* Traffic manager shared shaper add/update */ +static int +pmd_tm_shared_shaper_add_update(struct rte_eth_dev *dev, + uint32_t shared_shaper_id, + uint32_t shaper_profile_id, + struct rte_tm_error *error) +{ + struct pmd_internals *p = dev->data->dev_private; + struct tm_shared_shaper *ss; + struct tm_shaper_profile *sp; + struct tm_node *nt; + + /* Shaper profile must be valid. */ + sp = tm_shaper_profile_search(dev, shaper_profile_id); + if (sp == NULL) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_SHAPER_PROFILE_ID, + NULL, + rte_strerror(EINVAL)); + + /** + * Add new shared shaper + */ + ss = tm_shared_shaper_search(dev, shared_shaper_id); + if (ss == NULL) { + struct tm_shared_shaper_list *ssl = + &p->soft.tm.h.shared_shapers; + + /* Hierarchy must not be frozen */ + if (p->soft.tm.hierarchy_frozen) + return -rte_tm_error_set(error, + EBUSY, + RTE_TM_ERROR_TYPE_UNSPECIFIED, + NULL, + rte_strerror(EBUSY)); + + /* Memory allocation */ + ss = calloc(1, sizeof(struct tm_shared_shaper)); + if (ss == NULL) + return -rte_tm_error_set(error, + ENOMEM, + RTE_TM_ERROR_TYPE_UNSPECIFIED, + NULL, + rte_strerror(ENOMEM)); + + /* Fill in */ + ss->shared_shaper_id = shared_shaper_id; + ss->shaper_profile_id = shaper_profile_id; + + /* Add to list */ + TAILQ_INSERT_TAIL(ssl, ss, node); + p->soft.tm.h.n_shared_shapers++; + + return 0; + } + + /** + * Update existing shared shaper + */ + /* Hierarchy must be frozen (run-time update) */ + if (p->soft.tm.hierarchy_frozen == 0) + return -rte_tm_error_set(error, + EBUSY, + RTE_TM_ERROR_TYPE_UNSPECIFIED, + NULL, + rte_strerror(EBUSY)); + + + /* Propagate change. */ + nt = tm_shared_shaper_get_tc(dev, ss); + if (update_subport_tc_rate(dev, nt, ss, sp)) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_UNSPECIFIED, + NULL, + rte_strerror(EINVAL)); + + return 0; +} + +/* Traffic manager shared shaper delete */ +static int +pmd_tm_shared_shaper_delete(struct rte_eth_dev *dev, + uint32_t shared_shaper_id, + struct rte_tm_error *error) +{ + struct pmd_internals *p = dev->data->dev_private; + struct tm_shared_shaper *ss; + + /* Check existing */ + ss = tm_shared_shaper_search(dev, shared_shaper_id); + if (ss == NULL) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_SHARED_SHAPER_ID, + NULL, + rte_strerror(EINVAL)); + + /* Check unused */ + if (ss->n_users) + return -rte_tm_error_set(error, + EBUSY, + RTE_TM_ERROR_TYPE_SHARED_SHAPER_ID, + NULL, + rte_strerror(EBUSY)); + + /* Remove from list */ + TAILQ_REMOVE(&p->soft.tm.h.shared_shapers, ss, node); + p->soft.tm.h.n_shared_shapers--; + free(ss); + + return 0; +} + +static int +wred_profile_check(struct rte_eth_dev *dev, + uint32_t wred_profile_id, + struct rte_tm_wred_params *profile, + struct rte_tm_error *error) +{ + struct tm_wred_profile *wp; + enum rte_tm_color color; + + /* WRED profile ID must not be NONE. */ + if (wred_profile_id == RTE_TM_WRED_PROFILE_ID_NONE) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_WRED_PROFILE_ID, + NULL, + rte_strerror(EINVAL)); + + /* WRED profile must not exist. */ + wp = tm_wred_profile_search(dev, wred_profile_id); + if (wp) + return -rte_tm_error_set(error, + EEXIST, + RTE_TM_ERROR_TYPE_WRED_PROFILE_ID, + NULL, + rte_strerror(EEXIST)); + + /* Profile must not be NULL. */ + if (profile == NULL) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_WRED_PROFILE, + NULL, + rte_strerror(EINVAL)); + + /* min_th <= max_th, max_th > 0 */ + for (color = RTE_TM_GREEN; color < RTE_TM_COLORS; color++) { + uint16_t min_th = profile->red_params[color].min_th; + uint16_t max_th = profile->red_params[color].max_th; + + if (min_th > max_th || max_th == 0) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_WRED_PROFILE, + NULL, + rte_strerror(EINVAL)); + } + + return 0; +} + +/* Traffic manager WRED profile add */ +static int +pmd_tm_wred_profile_add(struct rte_eth_dev *dev, + uint32_t wred_profile_id, + struct rte_tm_wred_params *profile, + struct rte_tm_error *error) +{ + struct pmd_internals *p = dev->data->dev_private; + struct tm_wred_profile_list *wpl = &p->soft.tm.h.wred_profiles; + struct tm_wred_profile *wp; + int status; + + /* Check input params */ + status = wred_profile_check(dev, wred_profile_id, profile, error); + if (status) + return status; + + /* Memory allocation */ + wp = calloc(1, sizeof(struct tm_wred_profile)); + if (wp == NULL) + return -rte_tm_error_set(error, + ENOMEM, + RTE_TM_ERROR_TYPE_UNSPECIFIED, + NULL, + rte_strerror(ENOMEM)); + + /* Fill in */ + wp->wred_profile_id = wred_profile_id; + memcpy(&wp->params, profile, sizeof(wp->params)); + + /* Add to list */ + TAILQ_INSERT_TAIL(wpl, wp, node); + p->soft.tm.h.n_wred_profiles++; + + return 0; +} + +/* Traffic manager WRED profile delete */ +static int +pmd_tm_wred_profile_delete(struct rte_eth_dev *dev, + uint32_t wred_profile_id, + struct rte_tm_error *error) +{ + struct pmd_internals *p = dev->data->dev_private; + struct tm_wred_profile *wp; + + /* Check existing */ + wp = tm_wred_profile_search(dev, wred_profile_id); + if (wp == NULL) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_WRED_PROFILE_ID, + NULL, + rte_strerror(EINVAL)); + + /* Check unused */ + if (wp->n_users) + return -rte_tm_error_set(error, + EBUSY, + RTE_TM_ERROR_TYPE_WRED_PROFILE_ID, + NULL, + rte_strerror(EBUSY)); + + /* Remove from list */ + TAILQ_REMOVE(&p->soft.tm.h.wred_profiles, wp, node); + p->soft.tm.h.n_wred_profiles--; + free(wp); + + return 0; +} + +static int +node_add_check_port(struct rte_eth_dev *dev, + uint32_t node_id, + uint32_t parent_node_id __rte_unused, + uint32_t priority, + uint32_t weight, + uint32_t level_id __rte_unused, + struct rte_tm_node_params *params, + struct rte_tm_error *error) +{ + struct pmd_internals *p = dev->data->dev_private; + struct tm_shaper_profile *sp = tm_shaper_profile_search(dev, + params->shaper_profile_id); + + /* node type: non-leaf */ + if (node_id < p->params.soft.tm.nb_queues) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_NODE_ID, + NULL, + rte_strerror(EINVAL)); + + /* Priority must be 0 */ + if (priority != 0) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_NODE_PRIORITY, + NULL, + rte_strerror(EINVAL)); + + /* Weight must be 1 */ + if (weight != 1) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_NODE_WEIGHT, + NULL, + rte_strerror(EINVAL)); + + /* Shaper must be valid. + * Shaper profile peak rate must fit the configured port rate. + */ + if (params->shaper_profile_id == RTE_TM_SHAPER_PROFILE_ID_NONE || + sp == NULL || + sp->params.peak.rate > p->params.soft.tm.rate) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_NODE_PARAMS_SHAPER_PROFILE_ID, + NULL, + rte_strerror(EINVAL)); + + /* No shared shapers */ + if (params->n_shared_shapers != 0) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_NODE_PARAMS_N_SHARED_SHAPERS, + NULL, + rte_strerror(EINVAL)); + + /* Number of SP priorities must be 1 */ + if (params->nonleaf.n_sp_priorities != 1) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_NODE_PARAMS_N_SP_PRIORITIES, + NULL, + rte_strerror(EINVAL)); + + /* Stats */ + if (params->stats_mask & ~STATS_MASK_DEFAULT) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_NODE_PARAMS_STATS, + NULL, + rte_strerror(EINVAL)); + + return 0; +} + +static int +node_add_check_subport(struct rte_eth_dev *dev, + uint32_t node_id, + uint32_t parent_node_id __rte_unused, + uint32_t priority, + uint32_t weight, + uint32_t level_id __rte_unused, + struct rte_tm_node_params *params, + struct rte_tm_error *error) +{ + struct pmd_internals *p = dev->data->dev_private; + + /* node type: non-leaf */ + if (node_id < p->params.soft.tm.nb_queues) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_NODE_ID, + NULL, + rte_strerror(EINVAL)); + + /* Priority must be 0 */ + if (priority != 0) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_NODE_PRIORITY, + NULL, + rte_strerror(EINVAL)); + + /* Weight must be 1 */ + if (weight != 1) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_NODE_WEIGHT, + NULL, + rte_strerror(EINVAL)); + + /* Shaper must be valid */ + if (params->shaper_profile_id == RTE_TM_SHAPER_PROFILE_ID_NONE || + (!tm_shaper_profile_search(dev, params->shaper_profile_id))) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_NODE_PARAMS_SHAPER_PROFILE_ID, + NULL, + rte_strerror(EINVAL)); + + /* No shared shapers */ + if (params->n_shared_shapers != 0) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_NODE_PARAMS_N_SHARED_SHAPERS, + NULL, + rte_strerror(EINVAL)); + + /* Number of SP priorities must be 1 */ + if (params->nonleaf.n_sp_priorities != 1) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_NODE_PARAMS_N_SP_PRIORITIES, + NULL, + rte_strerror(EINVAL)); + + /* Stats */ + if (params->stats_mask & ~STATS_MASK_DEFAULT) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_NODE_PARAMS_STATS, + NULL, + rte_strerror(EINVAL)); + + return 0; +} + +static int +node_add_check_pipe(struct rte_eth_dev *dev, + uint32_t node_id, + uint32_t parent_node_id __rte_unused, + uint32_t priority, + uint32_t weight __rte_unused, + uint32_t level_id __rte_unused, + struct rte_tm_node_params *params, + struct rte_tm_error *error) +{ + struct pmd_internals *p = dev->data->dev_private; + + /* node type: non-leaf */ + if (node_id < p->params.soft.tm.nb_queues) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_NODE_ID, + NULL, + rte_strerror(EINVAL)); + + /* Priority must be 0 */ + if (priority != 0) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_NODE_PRIORITY, + NULL, + rte_strerror(EINVAL)); + + /* Shaper must be valid */ + if (params->shaper_profile_id == RTE_TM_SHAPER_PROFILE_ID_NONE || + (!tm_shaper_profile_search(dev, params->shaper_profile_id))) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_NODE_PARAMS_SHAPER_PROFILE_ID, + NULL, + rte_strerror(EINVAL)); + + /* No shared shapers */ + if (params->n_shared_shapers != 0) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_NODE_PARAMS_N_SHARED_SHAPERS, + NULL, + rte_strerror(EINVAL)); + + /* Number of SP priorities must be 4 */ + if (params->nonleaf.n_sp_priorities != + RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_NODE_PARAMS_N_SP_PRIORITIES, + NULL, + rte_strerror(EINVAL)); + + /* WFQ mode must be byte mode */ + if (params->nonleaf.wfq_weight_mode != NULL && + params->nonleaf.wfq_weight_mode[0] != 0 && + params->nonleaf.wfq_weight_mode[1] != 0 && + params->nonleaf.wfq_weight_mode[2] != 0 && + params->nonleaf.wfq_weight_mode[3] != 0) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_NODE_PARAMS_WFQ_WEIGHT_MODE, + NULL, + rte_strerror(EINVAL)); + + /* Stats */ + if (params->stats_mask & ~STATS_MASK_DEFAULT) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_NODE_PARAMS_STATS, + NULL, + rte_strerror(EINVAL)); + + return 0; +} + +static int +node_add_check_tc(struct rte_eth_dev *dev, + uint32_t node_id, + uint32_t parent_node_id __rte_unused, + uint32_t priority __rte_unused, + uint32_t weight, + uint32_t level_id __rte_unused, + struct rte_tm_node_params *params, + struct rte_tm_error *error) +{ + struct pmd_internals *p = dev->data->dev_private; + + /* node type: non-leaf */ + if (node_id < p->params.soft.tm.nb_queues) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_NODE_ID, + NULL, + rte_strerror(EINVAL)); + + /* Weight must be 1 */ + if (weight != 1) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_NODE_WEIGHT, + NULL, + rte_strerror(EINVAL)); + + /* Shaper must be valid */ + if (params->shaper_profile_id == RTE_TM_SHAPER_PROFILE_ID_NONE || + (!tm_shaper_profile_search(dev, params->shaper_profile_id))) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_NODE_PARAMS_SHAPER_PROFILE_ID, + NULL, + rte_strerror(EINVAL)); + + /* Single valid shared shaper */ + if (params->n_shared_shapers > 1) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_NODE_PARAMS_N_SHARED_SHAPERS, + NULL, + rte_strerror(EINVAL)); + + if (params->n_shared_shapers == 1 && + (params->shared_shaper_id == NULL || + (!tm_shared_shaper_search(dev, params->shared_shaper_id[0])))) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_NODE_PARAMS_SHARED_SHAPER_ID, + NULL, + rte_strerror(EINVAL)); + + /* Number of priorities must be 1 */ + if (params->nonleaf.n_sp_priorities != 1) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_NODE_PARAMS_N_SP_PRIORITIES, + NULL, + rte_strerror(EINVAL)); + + /* Stats */ + if (params->stats_mask & ~STATS_MASK_DEFAULT) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_NODE_PARAMS_STATS, + NULL, + rte_strerror(EINVAL)); + + return 0; +} + +static int +node_add_check_queue(struct rte_eth_dev *dev, + uint32_t node_id, + uint32_t parent_node_id __rte_unused, + uint32_t priority, + uint32_t weight __rte_unused, + uint32_t level_id __rte_unused, + struct rte_tm_node_params *params, + struct rte_tm_error *error) +{ + struct pmd_internals *p = dev->data->dev_private; + + /* node type: leaf */ + if (node_id >= p->params.soft.tm.nb_queues) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_NODE_ID, + NULL, + rte_strerror(EINVAL)); + + /* Priority must be 0 */ + if (priority != 0) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_NODE_PRIORITY, + NULL, + rte_strerror(EINVAL)); + + /* No shaper */ + if (params->shaper_profile_id != RTE_TM_SHAPER_PROFILE_ID_NONE) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_NODE_PARAMS_SHAPER_PROFILE_ID, + NULL, + rte_strerror(EINVAL)); + + /* No shared shapers */ + if (params->n_shared_shapers != 0) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_NODE_PARAMS_N_SHARED_SHAPERS, + NULL, + rte_strerror(EINVAL)); + + /* Congestion management must not be head drop */ + if (params->leaf.cman == RTE_TM_CMAN_HEAD_DROP) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_NODE_PARAMS_CMAN, + NULL, + rte_strerror(EINVAL)); + + /* Congestion management set to WRED */ + if (params->leaf.cman == RTE_TM_CMAN_WRED) { + uint32_t wred_profile_id = params->leaf.wred.wred_profile_id; + struct tm_wred_profile *wp = tm_wred_profile_search(dev, + wred_profile_id); + + /* WRED profile (for private WRED context) must be valid */ + if (wred_profile_id == RTE_TM_WRED_PROFILE_ID_NONE || + wp == NULL) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_NODE_PARAMS_WRED_PROFILE_ID, + NULL, + rte_strerror(EINVAL)); + + /* No shared WRED contexts */ + if (params->leaf.wred.n_shared_wred_contexts != 0) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_NODE_PARAMS_N_SHARED_WRED_CONTEXTS, + NULL, + rte_strerror(EINVAL)); + } + + /* Stats */ + if (params->stats_mask & ~STATS_MASK_QUEUE) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_NODE_PARAMS_STATS, + NULL, + rte_strerror(EINVAL)); + + return 0; +} + +static int +node_add_check(struct rte_eth_dev *dev, + uint32_t node_id, + uint32_t parent_node_id, + uint32_t priority, + uint32_t weight, + uint32_t level_id, + struct rte_tm_node_params *params, + struct rte_tm_error *error) +{ + struct tm_node *pn; + uint32_t level; + int status; + + /* node_id, parent_node_id: + * -node_id must not be RTE_TM_NODE_ID_NULL + * -node_id must not be in use + * -root node add (parent_node_id is RTE_TM_NODE_ID_NULL): + * -root node must not exist + * -non-root node add (parent_node_id is not RTE_TM_NODE_ID_NULL): + * -parent_node_id must be valid + */ + if (node_id == RTE_TM_NODE_ID_NULL) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_NODE_ID, + NULL, + rte_strerror(EINVAL)); + + if (tm_node_search(dev, node_id)) + return -rte_tm_error_set(error, + EEXIST, + RTE_TM_ERROR_TYPE_NODE_ID, + NULL, + rte_strerror(EEXIST)); + + if (parent_node_id == RTE_TM_NODE_ID_NULL) { + pn = NULL; + if (tm_root_node_present(dev)) + return -rte_tm_error_set(error, + EEXIST, + RTE_TM_ERROR_TYPE_NODE_PARENT_NODE_ID, + NULL, + rte_strerror(EEXIST)); + } else { + pn = tm_node_search(dev, parent_node_id); + if (pn == NULL) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_NODE_PARENT_NODE_ID, + NULL, + rte_strerror(EINVAL)); + } + + /* priority: must be 0 .. 3 */ + if (priority >= RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_NODE_PRIORITY, + NULL, + rte_strerror(EINVAL)); + + /* weight: must be 1 .. 255 */ + if (weight == 0 || weight >= UINT8_MAX) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_NODE_WEIGHT, + NULL, + rte_strerror(EINVAL)); + + /* level_id: if valid, then + * -root node add (parent_node_id is RTE_TM_NODE_ID_NULL): + * -level_id must be zero + * -non-root node add (parent_node_id is not RTE_TM_NODE_ID_NULL): + * -level_id must be parent level ID plus one + */ + level = (pn == NULL) ? 0 : pn->level + 1; + if (level_id != RTE_TM_NODE_LEVEL_ID_ANY && level_id != level) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_LEVEL_ID, + NULL, + rte_strerror(EINVAL)); + + /* params: must not be NULL */ + if (params == NULL) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_NODE_PARAMS, + NULL, + rte_strerror(EINVAL)); + + /* params: per level checks */ + switch (level) { + case TM_NODE_LEVEL_PORT: + status = node_add_check_port(dev, node_id, + parent_node_id, priority, weight, level_id, + params, error); + if (status) + return status; + break; + + case TM_NODE_LEVEL_SUBPORT: + status = node_add_check_subport(dev, node_id, + parent_node_id, priority, weight, level_id, + params, error); + if (status) + return status; + break; + + case TM_NODE_LEVEL_PIPE: + status = node_add_check_pipe(dev, node_id, + parent_node_id, priority, weight, level_id, + params, error); + if (status) + return status; + break; + + case TM_NODE_LEVEL_TC: + status = node_add_check_tc(dev, node_id, + parent_node_id, priority, weight, level_id, + params, error); + if (status) + return status; + break; + + case TM_NODE_LEVEL_QUEUE: + status = node_add_check_queue(dev, node_id, + parent_node_id, priority, weight, level_id, + params, error); + if (status) + return status; + break; + + default: + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_LEVEL_ID, + NULL, + rte_strerror(EINVAL)); + } + + return 0; +} + +/* Traffic manager node add */ +static int +pmd_tm_node_add(struct rte_eth_dev *dev, + uint32_t node_id, + uint32_t parent_node_id, + uint32_t priority, + uint32_t weight, + uint32_t level_id, + struct rte_tm_node_params *params, + struct rte_tm_error *error) +{ + struct pmd_internals *p = dev->data->dev_private; + struct tm_node_list *nl = &p->soft.tm.h.nodes; + struct tm_node *n; + uint32_t i; + int status; + + /* Checks */ + if (p->soft.tm.hierarchy_frozen) + return -rte_tm_error_set(error, + EBUSY, + RTE_TM_ERROR_TYPE_UNSPECIFIED, + NULL, + rte_strerror(EBUSY)); + + status = node_add_check(dev, node_id, parent_node_id, priority, weight, + level_id, params, error); + if (status) + return status; + + /* Memory allocation */ + n = calloc(1, sizeof(struct tm_node)); + if (n == NULL) + return -rte_tm_error_set(error, + ENOMEM, + RTE_TM_ERROR_TYPE_UNSPECIFIED, + NULL, + rte_strerror(ENOMEM)); + + /* Fill in */ + n->node_id = node_id; + n->parent_node_id = parent_node_id; + n->priority = priority; + n->weight = weight; + + if (parent_node_id != RTE_TM_NODE_ID_NULL) { + n->parent_node = tm_node_search(dev, parent_node_id); + n->level = n->parent_node->level + 1; + } + + if (params->shaper_profile_id != RTE_TM_SHAPER_PROFILE_ID_NONE) + n->shaper_profile = tm_shaper_profile_search(dev, + params->shaper_profile_id); + + if (n->level == TM_NODE_LEVEL_QUEUE && + params->leaf.cman == RTE_TM_CMAN_WRED) + n->wred_profile = tm_wred_profile_search(dev, + params->leaf.wred.wred_profile_id); + + memcpy(&n->params, params, sizeof(n->params)); + + /* Add to list */ + TAILQ_INSERT_TAIL(nl, n, node); + p->soft.tm.h.n_nodes++; + + /* Update dependencies */ + if (n->parent_node) + n->parent_node->n_children++; + + if (n->shaper_profile) + n->shaper_profile->n_users++; + + for (i = 0; i < params->n_shared_shapers; i++) { + struct tm_shared_shaper *ss; + + ss = tm_shared_shaper_search(dev, params->shared_shaper_id[i]); + ss->n_users++; + } + + if (n->wred_profile) + n->wred_profile->n_users++; + + p->soft.tm.h.n_tm_nodes[n->level]++; + + return 0; +} + +/* Traffic manager node delete */ +static int +pmd_tm_node_delete(struct rte_eth_dev *dev, + uint32_t node_id, + struct rte_tm_error *error) +{ + struct pmd_internals *p = dev->data->dev_private; + struct tm_node *n; + uint32_t i; + + /* Check hierarchy changes are currently allowed */ + if (p->soft.tm.hierarchy_frozen) + return -rte_tm_error_set(error, + EBUSY, + RTE_TM_ERROR_TYPE_UNSPECIFIED, + NULL, + rte_strerror(EBUSY)); + + /* Check existing */ + n = tm_node_search(dev, node_id); + if (n == NULL) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_NODE_ID, + NULL, + rte_strerror(EINVAL)); + + /* Check unused */ + if (n->n_children) + return -rte_tm_error_set(error, + EBUSY, + RTE_TM_ERROR_TYPE_NODE_ID, + NULL, + rte_strerror(EBUSY)); + + /* Update dependencies */ + p->soft.tm.h.n_tm_nodes[n->level]--; + + if (n->wred_profile) + n->wred_profile->n_users--; + + for (i = 0; i < n->params.n_shared_shapers; i++) { + struct tm_shared_shaper *ss; + + ss = tm_shared_shaper_search(dev, + n->params.shared_shaper_id[i]); + ss->n_users--; + } + + if (n->shaper_profile) + n->shaper_profile->n_users--; + + if (n->parent_node) + n->parent_node->n_children--; + + /* Remove from list */ + TAILQ_REMOVE(&p->soft.tm.h.nodes, n, node); + p->soft.tm.h.n_nodes--; + free(n); + + return 0; +} + + +static void +pipe_profile_build(struct rte_eth_dev *dev, + struct tm_node *np, + struct rte_sched_pipe_params *pp) +{ + struct pmd_internals *p = dev->data->dev_private; + struct tm_hierarchy *h = &p->soft.tm.h; + struct tm_node_list *nl = &h->nodes; + struct tm_node *nt, *nq; + + memset(pp, 0, sizeof(*pp)); + + /* Pipe */ + pp->tb_rate = np->shaper_profile->params.peak.rate; + pp->tb_size = np->shaper_profile->params.peak.size; + + /* Traffic Class (TC) */ + pp->tc_period = PIPE_TC_PERIOD; + +#ifdef RTE_SCHED_SUBPORT_TC_OV + pp->tc_ov_weight = np->weight; +#endif + + TAILQ_FOREACH(nt, nl, node) { + uint32_t queue_id = 0; + + if (nt->level != TM_NODE_LEVEL_TC || + nt->parent_node_id != np->node_id) + continue; + + pp->tc_rate[nt->priority] = + nt->shaper_profile->params.peak.rate; + + /* Queue */ + TAILQ_FOREACH(nq, nl, node) { + uint32_t pipe_queue_id; + + if (nq->level != TM_NODE_LEVEL_QUEUE || + nq->parent_node_id != nt->node_id) + continue; + + pipe_queue_id = nt->priority * + RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS + queue_id; + pp->wrr_weights[pipe_queue_id] = nq->weight; + + queue_id++; + } + } +} + +static int +pipe_profile_free_exists(struct rte_eth_dev *dev, + uint32_t *pipe_profile_id) +{ + struct pmd_internals *p = dev->data->dev_private; + struct tm_params *t = &p->soft.tm.params; + + if (t->n_pipe_profiles < RTE_SCHED_PIPE_PROFILES_PER_PORT) { + *pipe_profile_id = t->n_pipe_profiles; + return 1; + } + + return 0; +} + +static int +pipe_profile_exists(struct rte_eth_dev *dev, + struct rte_sched_pipe_params *pp, + uint32_t *pipe_profile_id) +{ + struct pmd_internals *p = dev->data->dev_private; + struct tm_params *t = &p->soft.tm.params; + uint32_t i; + + for (i = 0; i < t->n_pipe_profiles; i++) + if (memcmp(&t->pipe_profiles[i], pp, sizeof(*pp)) == 0) { + if (pipe_profile_id) + *pipe_profile_id = i; + return 1; + } + + return 0; +} + +static void +pipe_profile_install(struct rte_eth_dev *dev, + struct rte_sched_pipe_params *pp, + uint32_t pipe_profile_id) +{ + struct pmd_internals *p = dev->data->dev_private; + struct tm_params *t = &p->soft.tm.params; + + memcpy(&t->pipe_profiles[pipe_profile_id], pp, sizeof(*pp)); + t->n_pipe_profiles++; +} + +static void +pipe_profile_mark(struct rte_eth_dev *dev, + uint32_t subport_id, + uint32_t pipe_id, + uint32_t pipe_profile_id) +{ + struct pmd_internals *p = dev->data->dev_private; + struct tm_hierarchy *h = &p->soft.tm.h; + struct tm_params *t = &p->soft.tm.params; + uint32_t n_pipes_per_subport, pos; + + n_pipes_per_subport = h->n_tm_nodes[TM_NODE_LEVEL_PIPE] / + h->n_tm_nodes[TM_NODE_LEVEL_SUBPORT]; + pos = subport_id * n_pipes_per_subport + pipe_id; + + t->pipe_to_profile[pos] = pipe_profile_id; +} + +static struct rte_sched_pipe_params * +pipe_profile_get(struct rte_eth_dev *dev, struct tm_node *np) +{ + struct pmd_internals *p = dev->data->dev_private; + struct tm_hierarchy *h = &p->soft.tm.h; + struct tm_params *t = &p->soft.tm.params; + uint32_t n_pipes_per_subport = h->n_tm_nodes[TM_NODE_LEVEL_PIPE] / + h->n_tm_nodes[TM_NODE_LEVEL_SUBPORT]; + + uint32_t subport_id = tm_node_subport_id(dev, np->parent_node); + uint32_t pipe_id = tm_node_pipe_id(dev, np); + + uint32_t pos = subport_id * n_pipes_per_subport + pipe_id; + uint32_t pipe_profile_id = t->pipe_to_profile[pos]; + + return &t->pipe_profiles[pipe_profile_id]; +} + +static int +pipe_profiles_generate(struct rte_eth_dev *dev) +{ + struct pmd_internals *p = dev->data->dev_private; + struct tm_hierarchy *h = &p->soft.tm.h; + struct tm_node_list *nl = &h->nodes; + struct tm_node *ns, *np; + uint32_t subport_id; + + /* Objective: Fill in the following fields in struct tm_params: + * - pipe_profiles + * - n_pipe_profiles + * - pipe_to_profile + */ + + subport_id = 0; + TAILQ_FOREACH(ns, nl, node) { + uint32_t pipe_id; + + if (ns->level != TM_NODE_LEVEL_SUBPORT) + continue; + + pipe_id = 0; + TAILQ_FOREACH(np, nl, node) { + struct rte_sched_pipe_params pp; + uint32_t pos; + + if (np->level != TM_NODE_LEVEL_PIPE || + np->parent_node_id != ns->node_id) + continue; + + pipe_profile_build(dev, np, &pp); + + if (!pipe_profile_exists(dev, &pp, &pos)) { + if (!pipe_profile_free_exists(dev, &pos)) + return -1; + + pipe_profile_install(dev, &pp, pos); + } + + pipe_profile_mark(dev, subport_id, pipe_id, pos); + + pipe_id++; + } + + subport_id++; + } + + return 0; +} + +static struct tm_wred_profile * +tm_tc_wred_profile_get(struct rte_eth_dev *dev, uint32_t tc_id) +{ + struct pmd_internals *p = dev->data->dev_private; + struct tm_hierarchy *h = &p->soft.tm.h; + struct tm_node_list *nl = &h->nodes; + struct tm_node *nq; + + TAILQ_FOREACH(nq, nl, node) { + if (nq->level != TM_NODE_LEVEL_QUEUE || + nq->parent_node->priority != tc_id) + continue; + + return nq->wred_profile; + } + + return NULL; +} + +#ifdef RTE_SCHED_RED + +static void +wred_profiles_set(struct rte_eth_dev *dev) +{ + struct pmd_internals *p = dev->data->dev_private; + struct rte_sched_port_params *pp = &p->soft.tm.params.port_params; + uint32_t tc_id; + enum rte_tm_color color; + + for (tc_id = 0; tc_id < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; tc_id++) + for (color = RTE_TM_GREEN; color < RTE_TM_COLORS; color++) { + struct rte_red_params *dst = + &pp->red_params[tc_id][color]; + struct tm_wred_profile *src_wp = + tm_tc_wred_profile_get(dev, tc_id); + struct rte_tm_red_params *src = + &src_wp->params.red_params[color]; + + memcpy(dst, src, sizeof(*dst)); + } +} + +#else + +#define wred_profiles_set(dev) + +#endif + +static struct tm_shared_shaper * +tm_tc_shared_shaper_get(struct rte_eth_dev *dev, struct tm_node *tc_node) +{ + return (tc_node->params.n_shared_shapers) ? + tm_shared_shaper_search(dev, + tc_node->params.shared_shaper_id[0]) : + NULL; +} + +static struct tm_shared_shaper * +tm_subport_tc_shared_shaper_get(struct rte_eth_dev *dev, + struct tm_node *subport_node, + uint32_t tc_id) +{ + struct pmd_internals *p = dev->data->dev_private; + struct tm_node_list *nl = &p->soft.tm.h.nodes; + struct tm_node *n; + + TAILQ_FOREACH(n, nl, node) { + if (n->level != TM_NODE_LEVEL_TC || + n->parent_node->parent_node_id != + subport_node->node_id || + n->priority != tc_id) + continue; + + return tm_tc_shared_shaper_get(dev, n); + } + + return NULL; +} + +static int +hierarchy_commit_check(struct rte_eth_dev *dev, struct rte_tm_error *error) +{ + struct pmd_internals *p = dev->data->dev_private; + struct tm_hierarchy *h = &p->soft.tm.h; + struct tm_node_list *nl = &h->nodes; + struct tm_shared_shaper_list *ssl = &h->shared_shapers; + struct tm_wred_profile_list *wpl = &h->wred_profiles; + struct tm_node *nr = tm_root_node_present(dev), *ns, *np, *nt, *nq; + struct tm_shared_shaper *ss; + + uint32_t n_pipes_per_subport; + + /* Root node exists. */ + if (nr == NULL) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_LEVEL_ID, + NULL, + rte_strerror(EINVAL)); + + /* There is at least one subport, max is not exceeded. */ + if (nr->n_children == 0 || nr->n_children > TM_MAX_SUBPORTS) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_LEVEL_ID, + NULL, + rte_strerror(EINVAL)); + + /* There is at least one pipe. */ + if (h->n_tm_nodes[TM_NODE_LEVEL_PIPE] == 0) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_LEVEL_ID, + NULL, + rte_strerror(EINVAL)); + + /* Number of pipes is the same for all subports. Maximum number of pipes + * per subport is not exceeded. + */ + n_pipes_per_subport = h->n_tm_nodes[TM_NODE_LEVEL_PIPE] / + h->n_tm_nodes[TM_NODE_LEVEL_SUBPORT]; + + if (n_pipes_per_subport > TM_MAX_PIPES_PER_SUBPORT) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_UNSPECIFIED, + NULL, + rte_strerror(EINVAL)); + + TAILQ_FOREACH(ns, nl, node) { + if (ns->level != TM_NODE_LEVEL_SUBPORT) + continue; + + if (ns->n_children != n_pipes_per_subport) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_UNSPECIFIED, + NULL, + rte_strerror(EINVAL)); + } + + /* Each pipe has exactly 4 TCs, with exactly one TC for each priority */ + TAILQ_FOREACH(np, nl, node) { + uint32_t mask = 0, mask_expected = + RTE_LEN2MASK(RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE, + uint32_t); + + if (np->level != TM_NODE_LEVEL_PIPE) + continue; + + if (np->n_children != RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_UNSPECIFIED, + NULL, + rte_strerror(EINVAL)); + + TAILQ_FOREACH(nt, nl, node) { + if (nt->level != TM_NODE_LEVEL_TC || + nt->parent_node_id != np->node_id) + continue; + + mask |= 1 << nt->priority; + } + + if (mask != mask_expected) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_UNSPECIFIED, + NULL, + rte_strerror(EINVAL)); + } + + /* Each TC has exactly 4 packet queues. */ + TAILQ_FOREACH(nt, nl, node) { + if (nt->level != TM_NODE_LEVEL_TC) + continue; + + if (nt->n_children != RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_UNSPECIFIED, + NULL, + rte_strerror(EINVAL)); + } + + /** + * Shared shapers: + * -For each TC #i, all pipes in the same subport use the same + * shared shaper (or no shared shaper) for their TC#i. + * -Each shared shaper needs to have at least one user. All its + * users have to be TC nodes with the same priority and the same + * subport. + */ + TAILQ_FOREACH(ns, nl, node) { + struct tm_shared_shaper *s[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE]; + uint32_t id; + + if (ns->level != TM_NODE_LEVEL_SUBPORT) + continue; + + for (id = 0; id < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; id++) + s[id] = tm_subport_tc_shared_shaper_get(dev, ns, id); + + TAILQ_FOREACH(nt, nl, node) { + struct tm_shared_shaper *subport_ss, *tc_ss; + + if (nt->level != TM_NODE_LEVEL_TC || + nt->parent_node->parent_node_id != + ns->node_id) + continue; + + subport_ss = s[nt->priority]; + tc_ss = tm_tc_shared_shaper_get(dev, nt); + + if (subport_ss == NULL && tc_ss == NULL) + continue; + + if ((subport_ss == NULL && tc_ss != NULL) || + (subport_ss != NULL && tc_ss == NULL) || + subport_ss->shared_shaper_id != + tc_ss->shared_shaper_id) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_UNSPECIFIED, + NULL, + rte_strerror(EINVAL)); + } + } + + TAILQ_FOREACH(ss, ssl, node) { + struct tm_node *nt_any = tm_shared_shaper_get_tc(dev, ss); + uint32_t n_users = 0; + + if (nt_any != NULL) + TAILQ_FOREACH(nt, nl, node) { + if (nt->level != TM_NODE_LEVEL_TC || + nt->priority != nt_any->priority || + nt->parent_node->parent_node_id != + nt_any->parent_node->parent_node_id) + continue; + + n_users++; + } + + if (ss->n_users == 0 || ss->n_users != n_users) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_UNSPECIFIED, + NULL, + rte_strerror(EINVAL)); + } + + /* Not too many pipe profiles. */ + if (pipe_profiles_generate(dev)) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_UNSPECIFIED, + NULL, + rte_strerror(EINVAL)); + + /** + * WRED (when used, i.e. at least one WRED profile defined): + * -Each WRED profile must have at least one user. + * -All leaf nodes must have their private WRED context enabled. + * -For each TC #i, all leaf nodes must use the same WRED profile + * for their private WRED context. + */ + if (h->n_wred_profiles) { + struct tm_wred_profile *wp; + struct tm_wred_profile *w[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE]; + uint32_t id; + + TAILQ_FOREACH(wp, wpl, node) + if (wp->n_users == 0) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_UNSPECIFIED, + NULL, + rte_strerror(EINVAL)); + + for (id = 0; id < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; id++) { + w[id] = tm_tc_wred_profile_get(dev, id); + + if (w[id] == NULL) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_UNSPECIFIED, + NULL, + rte_strerror(EINVAL)); + } + + TAILQ_FOREACH(nq, nl, node) { + uint32_t id; + + if (nq->level != TM_NODE_LEVEL_QUEUE) + continue; + + id = nq->parent_node->priority; + + if (nq->wred_profile == NULL || + nq->wred_profile->wred_profile_id != + w[id]->wred_profile_id) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_UNSPECIFIED, + NULL, + rte_strerror(EINVAL)); + } + } + + return 0; +} + +static void +hierarchy_blueprints_create(struct rte_eth_dev *dev) +{ + struct pmd_internals *p = dev->data->dev_private; + struct tm_params *t = &p->soft.tm.params; + struct tm_hierarchy *h = &p->soft.tm.h; + + struct tm_node_list *nl = &h->nodes; + struct tm_node *root = tm_root_node_present(dev), *n; + + uint32_t subport_id; + + t->port_params = (struct rte_sched_port_params) { + .name = dev->data->name, + .socket = dev->data->numa_node, + .rate = root->shaper_profile->params.peak.rate, + .mtu = dev->data->mtu, + .frame_overhead = + root->shaper_profile->params.pkt_length_adjust, + .n_subports_per_port = root->n_children, + .n_pipes_per_subport = h->n_tm_nodes[TM_NODE_LEVEL_PIPE] / + h->n_tm_nodes[TM_NODE_LEVEL_SUBPORT], + .qsize = {p->params.soft.tm.qsize[0], + p->params.soft.tm.qsize[1], + p->params.soft.tm.qsize[2], + p->params.soft.tm.qsize[3], + }, + .pipe_profiles = t->pipe_profiles, + .n_pipe_profiles = t->n_pipe_profiles, + }; + + wred_profiles_set(dev); + + subport_id = 0; + TAILQ_FOREACH(n, nl, node) { + uint64_t tc_rate[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE]; + uint32_t i; + + if (n->level != TM_NODE_LEVEL_SUBPORT) + continue; + + for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++) { + struct tm_shared_shaper *ss; + struct tm_shaper_profile *sp; + + ss = tm_subport_tc_shared_shaper_get(dev, n, i); + sp = (ss) ? tm_shaper_profile_search(dev, + ss->shaper_profile_id) : + n->shaper_profile; + tc_rate[i] = sp->params.peak.rate; + } + + t->subport_params[subport_id] = + (struct rte_sched_subport_params) { + .tb_rate = n->shaper_profile->params.peak.rate, + .tb_size = n->shaper_profile->params.peak.size, + + .tc_rate = {tc_rate[0], + tc_rate[1], + tc_rate[2], + tc_rate[3], + }, + .tc_period = SUBPORT_TC_PERIOD, + }; + + subport_id++; + } +} + +/* Traffic manager hierarchy commit */ +static int +pmd_tm_hierarchy_commit(struct rte_eth_dev *dev, + int clear_on_fail, + struct rte_tm_error *error) +{ + struct pmd_internals *p = dev->data->dev_private; + int status; + + /* Checks */ + if (p->soft.tm.hierarchy_frozen) + return -rte_tm_error_set(error, + EBUSY, + RTE_TM_ERROR_TYPE_UNSPECIFIED, + NULL, + rte_strerror(EBUSY)); + + status = hierarchy_commit_check(dev, error); + if (status) { + if (clear_on_fail) { + tm_hierarchy_uninit(p); + tm_hierarchy_init(p); + } + + return status; + } + + /* Create blueprints */ + hierarchy_blueprints_create(dev); + + /* Freeze hierarchy */ + p->soft.tm.hierarchy_frozen = 1; + + return 0; +} + +#ifdef RTE_SCHED_SUBPORT_TC_OV + +static int +update_pipe_weight(struct rte_eth_dev *dev, struct tm_node *np, uint32_t weight) +{ + struct pmd_internals *p = dev->data->dev_private; + uint32_t pipe_id = tm_node_pipe_id(dev, np); + + struct tm_node *ns = np->parent_node; + uint32_t subport_id = tm_node_subport_id(dev, ns); + + struct rte_sched_pipe_params *profile0 = pipe_profile_get(dev, np); + struct rte_sched_pipe_params profile1; + uint32_t pipe_profile_id; + + /* Derive new pipe profile. */ + memcpy(&profile1, profile0, sizeof(profile1)); + profile1.tc_ov_weight = (uint8_t)weight; + + /* Since implementation does not allow adding more pipe profiles after + * port configuration, the pipe configuration can be successfully + * updated only if the new profile is also part of the existing set of + * pipe profiles. + */ + if (pipe_profile_exists(dev, &profile1, &pipe_profile_id) == 0) + return -1; + + /* Update the pipe profile used by the current pipe. */ + if (rte_sched_pipe_config(p->soft.tm.sched, subport_id, pipe_id, + (int32_t)pipe_profile_id)) + return -1; + + /* Commit changes. */ + pipe_profile_mark(dev, subport_id, pipe_id, pipe_profile_id); + np->weight = weight; + + return 0; +} + +#endif + +static int +update_queue_weight(struct rte_eth_dev *dev, + struct tm_node *nq, uint32_t weight) +{ + struct pmd_internals *p = dev->data->dev_private; + uint32_t queue_id = tm_node_queue_id(dev, nq); + + struct tm_node *nt = nq->parent_node; + uint32_t tc_id = tm_node_tc_id(dev, nt); + + struct tm_node *np = nt->parent_node; + uint32_t pipe_id = tm_node_pipe_id(dev, np); + + struct tm_node *ns = np->parent_node; + uint32_t subport_id = tm_node_subport_id(dev, ns); + + uint32_t pipe_queue_id = + tc_id * RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS + queue_id; + + struct rte_sched_pipe_params *profile0 = pipe_profile_get(dev, np); + struct rte_sched_pipe_params profile1; + uint32_t pipe_profile_id; + + /* Derive new pipe profile. */ + memcpy(&profile1, profile0, sizeof(profile1)); + profile1.wrr_weights[pipe_queue_id] = (uint8_t)weight; + + /* Since implementation does not allow adding more pipe profiles after + * port configuration, the pipe configuration can be successfully + * updated only if the new profile is also part of the existing set + * of pipe profiles. + */ + if (pipe_profile_exists(dev, &profile1, &pipe_profile_id) == 0) + return -1; + + /* Update the pipe profile used by the current pipe. */ + if (rte_sched_pipe_config(p->soft.tm.sched, subport_id, pipe_id, + (int32_t)pipe_profile_id)) + return -1; + + /* Commit changes. */ + pipe_profile_mark(dev, subport_id, pipe_id, pipe_profile_id); + nq->weight = weight; + + return 0; +} + +/* Traffic manager node parent update */ +static int +pmd_tm_node_parent_update(struct rte_eth_dev *dev, + uint32_t node_id, + uint32_t parent_node_id, + uint32_t priority, + uint32_t weight, + struct rte_tm_error *error) +{ + struct tm_node *n; + + /* Port must be started and TM used. */ + if (dev->data->dev_started == 0 && (tm_used(dev) == 0)) + return -rte_tm_error_set(error, + EBUSY, + RTE_TM_ERROR_TYPE_UNSPECIFIED, + NULL, + rte_strerror(EBUSY)); + + /* Node must be valid */ + n = tm_node_search(dev, node_id); + if (n == NULL) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_NODE_ID, + NULL, + rte_strerror(EINVAL)); + + /* Parent node must be the same */ + if (n->parent_node_id != parent_node_id) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_NODE_PARENT_NODE_ID, + NULL, + rte_strerror(EINVAL)); + + /* Priority must be the same */ + if (n->priority != priority) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_NODE_PRIORITY, + NULL, + rte_strerror(EINVAL)); + + /* weight: must be 1 .. 255 */ + if (weight == 0 || weight >= UINT8_MAX) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_NODE_WEIGHT, + NULL, + rte_strerror(EINVAL)); + + switch (n->level) { + case TM_NODE_LEVEL_PORT: + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_NODE_WEIGHT, + NULL, + rte_strerror(EINVAL)); + /* fall-through */ + case TM_NODE_LEVEL_SUBPORT: + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_NODE_WEIGHT, + NULL, + rte_strerror(EINVAL)); + /* fall-through */ + case TM_NODE_LEVEL_PIPE: +#ifdef RTE_SCHED_SUBPORT_TC_OV + if (update_pipe_weight(dev, n, weight)) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_UNSPECIFIED, + NULL, + rte_strerror(EINVAL)); + return 0; +#else + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_NODE_WEIGHT, + NULL, + rte_strerror(EINVAL)); +#endif + /* fall-through */ + case TM_NODE_LEVEL_TC: + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_NODE_WEIGHT, + NULL, + rte_strerror(EINVAL)); + /* fall-through */ + case TM_NODE_LEVEL_QUEUE: + /* fall-through */ + default: + if (update_queue_weight(dev, n, weight)) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_UNSPECIFIED, + NULL, + rte_strerror(EINVAL)); + return 0; + } +} + +static int +update_subport_rate(struct rte_eth_dev *dev, + struct tm_node *ns, + struct tm_shaper_profile *sp) +{ + struct pmd_internals *p = dev->data->dev_private; + uint32_t subport_id = tm_node_subport_id(dev, ns); + + struct rte_sched_subport_params subport_params; + + /* Derive new subport configuration. */ + memcpy(&subport_params, + &p->soft.tm.params.subport_params[subport_id], + sizeof(subport_params)); + subport_params.tb_rate = sp->params.peak.rate; + subport_params.tb_size = sp->params.peak.size; + + /* Update the subport configuration. */ + if (rte_sched_subport_config(p->soft.tm.sched, subport_id, + &subport_params)) + return -1; + + /* Commit changes. */ + ns->shaper_profile->n_users--; + + ns->shaper_profile = sp; + ns->params.shaper_profile_id = sp->shaper_profile_id; + sp->n_users++; + + memcpy(&p->soft.tm.params.subport_params[subport_id], + &subport_params, + sizeof(subport_params)); + + return 0; +} + +static int +update_pipe_rate(struct rte_eth_dev *dev, + struct tm_node *np, + struct tm_shaper_profile *sp) +{ + struct pmd_internals *p = dev->data->dev_private; + uint32_t pipe_id = tm_node_pipe_id(dev, np); + + struct tm_node *ns = np->parent_node; + uint32_t subport_id = tm_node_subport_id(dev, ns); + + struct rte_sched_pipe_params *profile0 = pipe_profile_get(dev, np); + struct rte_sched_pipe_params profile1; + uint32_t pipe_profile_id; + + /* Derive new pipe profile. */ + memcpy(&profile1, profile0, sizeof(profile1)); + profile1.tb_rate = sp->params.peak.rate; + profile1.tb_size = sp->params.peak.size; + + /* Since implementation does not allow adding more pipe profiles after + * port configuration, the pipe configuration can be successfully + * updated only if the new profile is also part of the existing set of + * pipe profiles. + */ + if (pipe_profile_exists(dev, &profile1, &pipe_profile_id) == 0) + return -1; + + /* Update the pipe profile used by the current pipe. */ + if (rte_sched_pipe_config(p->soft.tm.sched, subport_id, pipe_id, + (int32_t)pipe_profile_id)) + return -1; + + /* Commit changes. */ + pipe_profile_mark(dev, subport_id, pipe_id, pipe_profile_id); + np->shaper_profile->n_users--; + np->shaper_profile = sp; + np->params.shaper_profile_id = sp->shaper_profile_id; + sp->n_users++; + + return 0; +} + +static int +update_tc_rate(struct rte_eth_dev *dev, + struct tm_node *nt, + struct tm_shaper_profile *sp) +{ + struct pmd_internals *p = dev->data->dev_private; + uint32_t tc_id = tm_node_tc_id(dev, nt); + + struct tm_node *np = nt->parent_node; + uint32_t pipe_id = tm_node_pipe_id(dev, np); + + struct tm_node *ns = np->parent_node; + uint32_t subport_id = tm_node_subport_id(dev, ns); + + struct rte_sched_pipe_params *profile0 = pipe_profile_get(dev, np); + struct rte_sched_pipe_params profile1; + uint32_t pipe_profile_id; + + /* Derive new pipe profile. */ + memcpy(&profile1, profile0, sizeof(profile1)); + profile1.tc_rate[tc_id] = sp->params.peak.rate; + + /* Since implementation does not allow adding more pipe profiles after + * port configuration, the pipe configuration can be successfully + * updated only if the new profile is also part of the existing set of + * pipe profiles. + */ + if (pipe_profile_exists(dev, &profile1, &pipe_profile_id) == 0) + return -1; + + /* Update the pipe profile used by the current pipe. */ + if (rte_sched_pipe_config(p->soft.tm.sched, subport_id, pipe_id, + (int32_t)pipe_profile_id)) + return -1; + + /* Commit changes. */ + pipe_profile_mark(dev, subport_id, pipe_id, pipe_profile_id); + nt->shaper_profile->n_users--; + nt->shaper_profile = sp; + nt->params.shaper_profile_id = sp->shaper_profile_id; + sp->n_users++; + + return 0; +} + +/* Traffic manager node shaper update */ +static int +pmd_tm_node_shaper_update(struct rte_eth_dev *dev, + uint32_t node_id, + uint32_t shaper_profile_id, + struct rte_tm_error *error) +{ + struct tm_node *n; + struct tm_shaper_profile *sp; + + /* Port must be started and TM used. */ + if (dev->data->dev_started == 0 && (tm_used(dev) == 0)) + return -rte_tm_error_set(error, + EBUSY, + RTE_TM_ERROR_TYPE_UNSPECIFIED, + NULL, + rte_strerror(EBUSY)); + + /* Node must be valid */ + n = tm_node_search(dev, node_id); + if (n == NULL) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_NODE_ID, + NULL, + rte_strerror(EINVAL)); + + /* Shaper profile must be valid. */ + sp = tm_shaper_profile_search(dev, shaper_profile_id); + if (sp == NULL) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_SHAPER_PROFILE, + NULL, + rte_strerror(EINVAL)); + + switch (n->level) { + case TM_NODE_LEVEL_PORT: + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_UNSPECIFIED, + NULL, + rte_strerror(EINVAL)); + /* fall-through */ + case TM_NODE_LEVEL_SUBPORT: + if (update_subport_rate(dev, n, sp)) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_UNSPECIFIED, + NULL, + rte_strerror(EINVAL)); + return 0; + /* fall-through */ + case TM_NODE_LEVEL_PIPE: + if (update_pipe_rate(dev, n, sp)) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_UNSPECIFIED, + NULL, + rte_strerror(EINVAL)); + return 0; + /* fall-through */ + case TM_NODE_LEVEL_TC: + if (update_tc_rate(dev, n, sp)) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_UNSPECIFIED, + NULL, + rte_strerror(EINVAL)); + return 0; + /* fall-through */ + case TM_NODE_LEVEL_QUEUE: + /* fall-through */ + default: + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_UNSPECIFIED, + NULL, + rte_strerror(EINVAL)); + } +} + +static inline uint32_t +tm_port_queue_id(struct rte_eth_dev *dev, + uint32_t port_subport_id, + uint32_t subport_pipe_id, + uint32_t pipe_tc_id, + uint32_t tc_queue_id) +{ + struct pmd_internals *p = dev->data->dev_private; + struct tm_hierarchy *h = &p->soft.tm.h; + uint32_t n_pipes_per_subport = h->n_tm_nodes[TM_NODE_LEVEL_PIPE] / + h->n_tm_nodes[TM_NODE_LEVEL_SUBPORT]; + + uint32_t port_pipe_id = + port_subport_id * n_pipes_per_subport + subport_pipe_id; + uint32_t port_tc_id = + port_pipe_id * RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE + pipe_tc_id; + uint32_t port_queue_id = + port_tc_id * RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS + tc_queue_id; + + return port_queue_id; +} + +static int +read_port_stats(struct rte_eth_dev *dev, + struct tm_node *nr, + struct rte_tm_node_stats *stats, + uint64_t *stats_mask, + int clear) +{ + struct pmd_internals *p = dev->data->dev_private; + struct tm_hierarchy *h = &p->soft.tm.h; + uint32_t n_subports_per_port = h->n_tm_nodes[TM_NODE_LEVEL_SUBPORT]; + uint32_t subport_id; + + for (subport_id = 0; subport_id < n_subports_per_port; subport_id++) { + struct rte_sched_subport_stats s; + uint32_t tc_ov, id; + + /* Stats read */ + int status = rte_sched_subport_read_stats( + p->soft.tm.sched, + subport_id, + &s, + &tc_ov); + if (status) + return status; + + /* Stats accumulate */ + for (id = 0; id < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; id++) { + nr->stats.n_pkts += + s.n_pkts_tc[id] - s.n_pkts_tc_dropped[id]; + nr->stats.n_bytes += + s.n_bytes_tc[id] - s.n_bytes_tc_dropped[id]; + nr->stats.leaf.n_pkts_dropped[RTE_TM_GREEN] += + s.n_pkts_tc_dropped[id]; + nr->stats.leaf.n_bytes_dropped[RTE_TM_GREEN] += + s.n_bytes_tc_dropped[id]; + } + } + + /* Stats copy */ + if (stats) + memcpy(stats, &nr->stats, sizeof(*stats)); + + if (stats_mask) + *stats_mask = STATS_MASK_DEFAULT; + + /* Stats clear */ + if (clear) + memset(&nr->stats, 0, sizeof(nr->stats)); + + return 0; +} + +static int +read_subport_stats(struct rte_eth_dev *dev, + struct tm_node *ns, + struct rte_tm_node_stats *stats, + uint64_t *stats_mask, + int clear) +{ + struct pmd_internals *p = dev->data->dev_private; + uint32_t subport_id = tm_node_subport_id(dev, ns); + struct rte_sched_subport_stats s; + uint32_t tc_ov, tc_id; + + /* Stats read */ + int status = rte_sched_subport_read_stats( + p->soft.tm.sched, + subport_id, + &s, + &tc_ov); + if (status) + return status; + + /* Stats accumulate */ + for (tc_id = 0; tc_id < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; tc_id++) { + ns->stats.n_pkts += + s.n_pkts_tc[tc_id] - s.n_pkts_tc_dropped[tc_id]; + ns->stats.n_bytes += + s.n_bytes_tc[tc_id] - s.n_bytes_tc_dropped[tc_id]; + ns->stats.leaf.n_pkts_dropped[RTE_TM_GREEN] += + s.n_pkts_tc_dropped[tc_id]; + ns->stats.leaf.n_bytes_dropped[RTE_TM_GREEN] += + s.n_bytes_tc_dropped[tc_id]; + } + + /* Stats copy */ + if (stats) + memcpy(stats, &ns->stats, sizeof(*stats)); + + if (stats_mask) + *stats_mask = STATS_MASK_DEFAULT; + + /* Stats clear */ + if (clear) + memset(&ns->stats, 0, sizeof(ns->stats)); + + return 0; +} + +static int +read_pipe_stats(struct rte_eth_dev *dev, + struct tm_node *np, + struct rte_tm_node_stats *stats, + uint64_t *stats_mask, + int clear) +{ + struct pmd_internals *p = dev->data->dev_private; + + uint32_t pipe_id = tm_node_pipe_id(dev, np); + + struct tm_node *ns = np->parent_node; + uint32_t subport_id = tm_node_subport_id(dev, ns); + + uint32_t i; + + /* Stats read */ + for (i = 0; i < RTE_SCHED_QUEUES_PER_PIPE; i++) { + struct rte_sched_queue_stats s; + uint16_t qlen; + + uint32_t qid = tm_port_queue_id(dev, + subport_id, + pipe_id, + i / RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS, + i % RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS); + + int status = rte_sched_queue_read_stats( + p->soft.tm.sched, + qid, + &s, + &qlen); + if (status) + return status; + + /* Stats accumulate */ + np->stats.n_pkts += s.n_pkts - s.n_pkts_dropped; + np->stats.n_bytes += s.n_bytes - s.n_bytes_dropped; + np->stats.leaf.n_pkts_dropped[RTE_TM_GREEN] += s.n_pkts_dropped; + np->stats.leaf.n_bytes_dropped[RTE_TM_GREEN] += + s.n_bytes_dropped; + np->stats.leaf.n_pkts_queued = qlen; + } + + /* Stats copy */ + if (stats) + memcpy(stats, &np->stats, sizeof(*stats)); + + if (stats_mask) + *stats_mask = STATS_MASK_DEFAULT; + + /* Stats clear */ + if (clear) + memset(&np->stats, 0, sizeof(np->stats)); + + return 0; +} + +static int +read_tc_stats(struct rte_eth_dev *dev, + struct tm_node *nt, + struct rte_tm_node_stats *stats, + uint64_t *stats_mask, + int clear) +{ + struct pmd_internals *p = dev->data->dev_private; + + uint32_t tc_id = tm_node_tc_id(dev, nt); + + struct tm_node *np = nt->parent_node; + uint32_t pipe_id = tm_node_pipe_id(dev, np); + + struct tm_node *ns = np->parent_node; + uint32_t subport_id = tm_node_subport_id(dev, ns); + + uint32_t i; + + /* Stats read */ + for (i = 0; i < RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS; i++) { + struct rte_sched_queue_stats s; + uint16_t qlen; + + uint32_t qid = tm_port_queue_id(dev, + subport_id, + pipe_id, + tc_id, + i); + + int status = rte_sched_queue_read_stats( + p->soft.tm.sched, + qid, + &s, + &qlen); + if (status) + return status; + + /* Stats accumulate */ + nt->stats.n_pkts += s.n_pkts - s.n_pkts_dropped; + nt->stats.n_bytes += s.n_bytes - s.n_bytes_dropped; + nt->stats.leaf.n_pkts_dropped[RTE_TM_GREEN] += s.n_pkts_dropped; + nt->stats.leaf.n_bytes_dropped[RTE_TM_GREEN] += + s.n_bytes_dropped; + nt->stats.leaf.n_pkts_queued = qlen; + } + + /* Stats copy */ + if (stats) + memcpy(stats, &nt->stats, sizeof(*stats)); + + if (stats_mask) + *stats_mask = STATS_MASK_DEFAULT; + + /* Stats clear */ + if (clear) + memset(&nt->stats, 0, sizeof(nt->stats)); + + return 0; +} + +static int +read_queue_stats(struct rte_eth_dev *dev, + struct tm_node *nq, + struct rte_tm_node_stats *stats, + uint64_t *stats_mask, + int clear) +{ + struct pmd_internals *p = dev->data->dev_private; + struct rte_sched_queue_stats s; + uint16_t qlen; + + uint32_t queue_id = tm_node_queue_id(dev, nq); + + struct tm_node *nt = nq->parent_node; + uint32_t tc_id = tm_node_tc_id(dev, nt); + + struct tm_node *np = nt->parent_node; + uint32_t pipe_id = tm_node_pipe_id(dev, np); + + struct tm_node *ns = np->parent_node; + uint32_t subport_id = tm_node_subport_id(dev, ns); + + /* Stats read */ + uint32_t qid = tm_port_queue_id(dev, + subport_id, + pipe_id, + tc_id, + queue_id); + + int status = rte_sched_queue_read_stats( + p->soft.tm.sched, + qid, + &s, + &qlen); + if (status) + return status; + + /* Stats accumulate */ + nq->stats.n_pkts += s.n_pkts - s.n_pkts_dropped; + nq->stats.n_bytes += s.n_bytes - s.n_bytes_dropped; + nq->stats.leaf.n_pkts_dropped[RTE_TM_GREEN] += s.n_pkts_dropped; + nq->stats.leaf.n_bytes_dropped[RTE_TM_GREEN] += + s.n_bytes_dropped; + nq->stats.leaf.n_pkts_queued = qlen; + + /* Stats copy */ + if (stats) + memcpy(stats, &nq->stats, sizeof(*stats)); + + if (stats_mask) + *stats_mask = STATS_MASK_QUEUE; + + /* Stats clear */ + if (clear) + memset(&nq->stats, 0, sizeof(nq->stats)); + + return 0; +} + +/* Traffic manager read stats counters for specific node */ +static int +pmd_tm_node_stats_read(struct rte_eth_dev *dev, + uint32_t node_id, + struct rte_tm_node_stats *stats, + uint64_t *stats_mask, + int clear, + struct rte_tm_error *error) +{ + struct tm_node *n; + + /* Port must be started and TM used. */ + if (dev->data->dev_started == 0 && (tm_used(dev) == 0)) + return -rte_tm_error_set(error, + EBUSY, + RTE_TM_ERROR_TYPE_UNSPECIFIED, + NULL, + rte_strerror(EBUSY)); + + /* Node must be valid */ + n = tm_node_search(dev, node_id); + if (n == NULL) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_NODE_ID, + NULL, + rte_strerror(EINVAL)); + + switch (n->level) { + case TM_NODE_LEVEL_PORT: + if (read_port_stats(dev, n, stats, stats_mask, clear)) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_UNSPECIFIED, + NULL, + rte_strerror(EINVAL)); + return 0; + + case TM_NODE_LEVEL_SUBPORT: + if (read_subport_stats(dev, n, stats, stats_mask, clear)) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_UNSPECIFIED, + NULL, + rte_strerror(EINVAL)); + return 0; + + case TM_NODE_LEVEL_PIPE: + if (read_pipe_stats(dev, n, stats, stats_mask, clear)) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_UNSPECIFIED, + NULL, + rte_strerror(EINVAL)); + return 0; + + case TM_NODE_LEVEL_TC: + if (read_tc_stats(dev, n, stats, stats_mask, clear)) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_UNSPECIFIED, + NULL, + rte_strerror(EINVAL)); + return 0; + + case TM_NODE_LEVEL_QUEUE: + default: + if (read_queue_stats(dev, n, stats, stats_mask, clear)) + return -rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_UNSPECIFIED, + NULL, + rte_strerror(EINVAL)); + return 0; + } +} + +const struct rte_tm_ops pmd_tm_ops = { + .node_type_get = pmd_tm_node_type_get, + .capabilities_get = pmd_tm_capabilities_get, + .level_capabilities_get = pmd_tm_level_capabilities_get, + .node_capabilities_get = pmd_tm_node_capabilities_get, + + .wred_profile_add = pmd_tm_wred_profile_add, + .wred_profile_delete = pmd_tm_wred_profile_delete, + .shared_wred_context_add_update = NULL, + .shared_wred_context_delete = NULL, + + .shaper_profile_add = pmd_tm_shaper_profile_add, + .shaper_profile_delete = pmd_tm_shaper_profile_delete, + .shared_shaper_add_update = pmd_tm_shared_shaper_add_update, + .shared_shaper_delete = pmd_tm_shared_shaper_delete, + + .node_add = pmd_tm_node_add, + .node_delete = pmd_tm_node_delete, + .node_suspend = NULL, + .node_resume = NULL, + .hierarchy_commit = pmd_tm_hierarchy_commit, + + .node_parent_update = pmd_tm_node_parent_update, + .node_shaper_update = pmd_tm_node_shaper_update, + .node_shared_shaper_update = NULL, + .node_stats_update = NULL, + .node_wfq_weight_mode_update = NULL, + .node_cman_update = NULL, + .node_wred_context_update = NULL, + .node_shared_wred_context_update = NULL, + + .node_stats_read = pmd_tm_node_stats_read, +}; diff --git a/dpdk/drivers/net/softnic/rte_pmd_eth_softnic_version.map b/dpdk/drivers/net/softnic/rte_pmd_eth_softnic_version.map new file mode 100644 index 00000000..fb2cb68c --- /dev/null +++ b/dpdk/drivers/net/softnic/rte_pmd_eth_softnic_version.map @@ -0,0 +1,7 @@ +DPDK_17.11 { + global: + + rte_pmd_softnic_run; + + local: *; +}; diff --git a/dpdk/drivers/net/szedata2/Makefile b/dpdk/drivers/net/szedata2/Makefile index 4a7b14c9..0ebd3ec5 100644 --- a/dpdk/drivers/net/szedata2/Makefile +++ b/dpdk/drivers/net/szedata2/Makefile @@ -39,6 +39,9 @@ LIB = librte_pmd_szedata2.a CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) LDLIBS += -lsze2 +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs +LDLIBS += -lrte_bus_pci EXPORT_MAP := rte_pmd_szedata2_version.map @@ -48,17 +51,11 @@ LIBABIVER := 1 # all source are stored in SRCS-y # SRCS-$(CONFIG_RTE_LIBRTE_PMD_SZEDATA2) += rte_eth_szedata2.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_SZEDATA2) += szedata2_iobuf.c # # Export include files # SYMLINK-y-include += -# this lib depends upon: -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_SZEDATA2) += lib/librte_eal -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_SZEDATA2) += lib/librte_mbuf -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_SZEDATA2) += lib/librte_mempool -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_SZEDATA2) += lib/librte_ether -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_SZEDATA2) += lib/librte_kvargs - include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/net/szedata2/rte_eth_szedata2.c b/dpdk/drivers/net/szedata2/rte_eth_szedata2.c index 483d7894..45aebed3 100644 --- a/dpdk/drivers/net/szedata2/rte_eth_szedata2.c +++ b/dpdk/drivers/net/szedata2/rte_eth_szedata2.c @@ -45,6 +45,7 @@ #include #include +#include #include #include #include @@ -52,6 +53,7 @@ #include #include "rte_eth_szedata2.h" +#include "szedata2_iobuf.h" #define RTE_ETH_SZEDATA2_MAX_RX_QUEUES 32 #define RTE_ETH_SZEDATA2_MAX_TX_QUEUES 32 @@ -62,15 +64,14 @@ */ #define RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED 8 -#define RTE_SZEDATA2_DRIVER_NAME rte_szedata2_pmd -#define RTE_SZEDATA2_PCI_DRIVER_NAME "rte_szedata2_pmd" +#define RTE_SZEDATA2_DRIVER_NAME net_szedata2 #define SZEDATA2_DEV_PATH_FMT "/dev/szedataII%u" struct szedata2_rx_queue { struct szedata *sze; uint8_t rx_channel; - uint8_t in_port; + uint16_t in_port; struct rte_mempool *mb_pool; volatile uint64_t rx_pkts; volatile uint64_t rx_bytes; @@ -91,6 +92,7 @@ struct pmd_internals { uint16_t max_rx_queues; uint16_t max_tx_queues; char sze_dev[PATH_MAX]; + struct rte_mem_resource *pci_rsc; }; static struct ether_addr eth_addr = { @@ -680,7 +682,7 @@ eth_szedata2_tx(void *queue, uint32_t hwpkt_len; uint32_t unlock_size; uint32_t rem_len; - uint8_t mbuf_segs; + uint16_t mbuf_segs; uint16_t pkt_left = nb_pkts; if (sze_q->sze == NULL || nb_pkts == 0) @@ -1030,6 +1032,7 @@ eth_dev_info(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) { struct pmd_internals *internals = dev->data->dev_private; + dev_info->pci_dev = RTE_ETH_DEV_TO_PCI(dev); dev_info->if_index = 0; dev_info->max_mac_addrs = 1; dev_info->max_rx_pktlen = (uint32_t)-1; @@ -1039,7 +1042,7 @@ eth_dev_info(struct rte_eth_dev *dev, dev_info->speed_capa = ETH_LINK_SPEED_100G; } -static void +static int eth_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) { @@ -1074,6 +1077,8 @@ eth_stats_get(struct rte_eth_dev *dev, stats->ibytes = rx_total_bytes; stats->obytes = tx_total_bytes; stats->oerrors = tx_err_total; + + return 0; } static void @@ -1137,6 +1142,33 @@ eth_dev_close(struct rte_eth_dev *dev) dev->data->nb_tx_queues = 0; } +/** + * Function takes value from first IBUF status register. + * Values in IBUF and OBUF should be same. + * + * @param internals + * Pointer to device private structure. + * @return + * Link speed constant. + */ +static inline enum szedata2_link_speed +get_link_speed(const struct pmd_internals *internals) +{ + const volatile struct szedata2_ibuf *ibuf = + ibuf_ptr_by_index(internals->pci_rsc, 0); + uint32_t speed = (szedata2_read32(&ibuf->ibuf_st) & 0x70) >> 4; + switch (speed) { + case 0x03: + return SZEDATA2_LINK_SPEED_10G; + case 0x04: + return SZEDATA2_LINK_SPEED_40G; + case 0x05: + return SZEDATA2_LINK_SPEED_100G; + default: + return SZEDATA2_LINK_SPEED_DEFAULT; + } +} + static int eth_link_update(struct rte_eth_dev *dev, int wait_to_complete __rte_unused) @@ -1144,11 +1176,13 @@ eth_link_update(struct rte_eth_dev *dev, struct rte_eth_link link; struct rte_eth_link *link_ptr = &link; struct rte_eth_link *dev_link = &dev->data->dev_link; - volatile struct szedata2_cgmii_ibuf *ibuf = SZEDATA2_PCI_RESOURCE_PTR( - dev, SZEDATA2_CGMII_IBUF_BASE_OFF, - volatile struct szedata2_cgmii_ibuf *); + struct pmd_internals *internals = (struct pmd_internals *) + dev->data->dev_private; + const volatile struct szedata2_ibuf *ibuf; + uint32_t i; + bool link_is_up = false; - switch (cgmii_link_speed(ibuf)) { + switch (get_link_speed(internals)) { case SZEDATA2_LINK_SPEED_10G: link.link_speed = ETH_SPEED_NUM_10G; break; @@ -1166,10 +1200,21 @@ eth_link_update(struct rte_eth_dev *dev, /* szedata2 uses only full duplex */ link.link_duplex = ETH_LINK_FULL_DUPLEX; - link.link_status = (cgmii_ibuf_is_enabled(ibuf) && - cgmii_ibuf_is_link_up(ibuf)) ? ETH_LINK_UP : ETH_LINK_DOWN; + for (i = 0; i < szedata2_ibuf_count; i++) { + ibuf = ibuf_ptr_by_index(internals->pci_rsc, i); + /* + * Link is considered up if at least one ibuf is enabled + * and up. + */ + if (ibuf_is_enabled(ibuf) && ibuf_is_link_up(ibuf)) { + link_is_up = true; + break; + } + } - link.link_autoneg = ETH_LINK_SPEED_FIXED; + link.link_status = (link_is_up) ? ETH_LINK_UP : ETH_LINK_DOWN; + + link.link_autoneg = ETH_LINK_FIXED; rte_atomic64_cmpset((uint64_t *)dev_link, *(uint64_t *)dev_link, *(uint64_t *)link_ptr); @@ -1180,30 +1225,28 @@ eth_link_update(struct rte_eth_dev *dev, static int eth_dev_set_link_up(struct rte_eth_dev *dev) { - volatile struct szedata2_cgmii_ibuf *ibuf = SZEDATA2_PCI_RESOURCE_PTR( - dev, SZEDATA2_CGMII_IBUF_BASE_OFF, - volatile struct szedata2_cgmii_ibuf *); - volatile struct szedata2_cgmii_obuf *obuf = SZEDATA2_PCI_RESOURCE_PTR( - dev, SZEDATA2_CGMII_OBUF_BASE_OFF, - volatile struct szedata2_cgmii_obuf *); + struct pmd_internals *internals = (struct pmd_internals *) + dev->data->dev_private; + uint32_t i; - cgmii_ibuf_enable(ibuf); - cgmii_obuf_enable(obuf); + for (i = 0; i < szedata2_ibuf_count; i++) + ibuf_enable(ibuf_ptr_by_index(internals->pci_rsc, i)); + for (i = 0; i < szedata2_obuf_count; i++) + obuf_enable(obuf_ptr_by_index(internals->pci_rsc, i)); return 0; } static int eth_dev_set_link_down(struct rte_eth_dev *dev) { - volatile struct szedata2_cgmii_ibuf *ibuf = SZEDATA2_PCI_RESOURCE_PTR( - dev, SZEDATA2_CGMII_IBUF_BASE_OFF, - volatile struct szedata2_cgmii_ibuf *); - volatile struct szedata2_cgmii_obuf *obuf = SZEDATA2_PCI_RESOURCE_PTR( - dev, SZEDATA2_CGMII_OBUF_BASE_OFF, - volatile struct szedata2_cgmii_obuf *); + struct pmd_internals *internals = (struct pmd_internals *) + dev->data->dev_private; + uint32_t i; - cgmii_ibuf_disable(ibuf); - cgmii_obuf_disable(obuf); + for (i = 0; i < szedata2_ibuf_count; i++) + ibuf_disable(ibuf_ptr_by_index(internals->pci_rsc, i)); + for (i = 0; i < szedata2_obuf_count; i++) + obuf_disable(obuf_ptr_by_index(internals->pci_rsc, i)); return 0; } @@ -1281,37 +1324,53 @@ eth_mac_addr_set(struct rte_eth_dev *dev __rte_unused, static void eth_promiscuous_enable(struct rte_eth_dev *dev) { - volatile struct szedata2_cgmii_ibuf *ibuf = SZEDATA2_PCI_RESOURCE_PTR( - dev, SZEDATA2_CGMII_IBUF_BASE_OFF, - volatile struct szedata2_cgmii_ibuf *); - cgmii_ibuf_mac_mode_write(ibuf, SZEDATA2_MAC_CHMODE_PROMISC); + struct pmd_internals *internals = (struct pmd_internals *) + dev->data->dev_private; + uint32_t i; + + for (i = 0; i < szedata2_ibuf_count; i++) { + ibuf_mac_mode_write(ibuf_ptr_by_index(internals->pci_rsc, i), + SZEDATA2_MAC_CHMODE_PROMISC); + } } static void eth_promiscuous_disable(struct rte_eth_dev *dev) { - volatile struct szedata2_cgmii_ibuf *ibuf = SZEDATA2_PCI_RESOURCE_PTR( - dev, SZEDATA2_CGMII_IBUF_BASE_OFF, - volatile struct szedata2_cgmii_ibuf *); - cgmii_ibuf_mac_mode_write(ibuf, SZEDATA2_MAC_CHMODE_ONLY_VALID); + struct pmd_internals *internals = (struct pmd_internals *) + dev->data->dev_private; + uint32_t i; + + for (i = 0; i < szedata2_ibuf_count; i++) { + ibuf_mac_mode_write(ibuf_ptr_by_index(internals->pci_rsc, i), + SZEDATA2_MAC_CHMODE_ONLY_VALID); + } } static void eth_allmulticast_enable(struct rte_eth_dev *dev) { - volatile struct szedata2_cgmii_ibuf *ibuf = SZEDATA2_PCI_RESOURCE_PTR( - dev, SZEDATA2_CGMII_IBUF_BASE_OFF, - volatile struct szedata2_cgmii_ibuf *); - cgmii_ibuf_mac_mode_write(ibuf, SZEDATA2_MAC_CHMODE_ALL_MULTICAST); + struct pmd_internals *internals = (struct pmd_internals *) + dev->data->dev_private; + uint32_t i; + + for (i = 0; i < szedata2_ibuf_count; i++) { + ibuf_mac_mode_write(ibuf_ptr_by_index(internals->pci_rsc, i), + SZEDATA2_MAC_CHMODE_ALL_MULTICAST); + } } static void eth_allmulticast_disable(struct rte_eth_dev *dev) { - volatile struct szedata2_cgmii_ibuf *ibuf = SZEDATA2_PCI_RESOURCE_PTR( - dev, SZEDATA2_CGMII_IBUF_BASE_OFF, - volatile struct szedata2_cgmii_ibuf *); - cgmii_ibuf_mac_mode_write(ibuf, SZEDATA2_MAC_CHMODE_ONLY_VALID); + struct pmd_internals *internals = (struct pmd_internals *) + dev->data->dev_private; + uint32_t i; + + for (i = 0; i < szedata2_ibuf_count; i++) { + ibuf_mac_mode_write(ibuf_ptr_by_index(internals->pci_rsc, i), + SZEDATA2_MAC_CHMODE_ONLY_VALID); + } } static const struct eth_dev_ops ops = { @@ -1349,7 +1408,7 @@ static const struct eth_dev_ops ops = { * -1 on error */ static int -get_szedata2_index(struct rte_eth_dev *dev, uint32_t *index) +get_szedata2_index(const struct rte_pci_addr *pcislot_addr, uint32_t *index) { DIR *dir; struct dirent *entry; @@ -1357,7 +1416,6 @@ get_szedata2_index(struct rte_eth_dev *dev, uint32_t *index) uint32_t tmp_index; FILE *fd; char pcislot_path[PATH_MAX]; - struct rte_pci_addr pcislot_addr = dev->pci_dev->addr; uint32_t domain; uint32_t bus; uint32_t devid; @@ -1392,10 +1450,10 @@ get_szedata2_index(struct rte_eth_dev *dev, uint32_t *index) if (ret != 4) continue; - if (pcislot_addr.domain == domain && - pcislot_addr.bus == bus && - pcislot_addr.devid == devid && - pcislot_addr.function == function) { + if (pcislot_addr->domain == domain && + pcislot_addr->bus == bus && + pcislot_addr->devid == devid && + pcislot_addr->function == function) { *index = tmp_index; closedir(dir); return 0; @@ -1415,9 +1473,10 @@ rte_szedata2_eth_dev_init(struct rte_eth_dev *dev) struct szedata *szedata_temp; int ret; uint32_t szedata2_index; - struct rte_pci_addr *pci_addr = &dev->pci_dev->addr; - struct rte_pci_resource *pci_rsc = - &dev->pci_dev->mem_resource[PCI_RESOURCE_NUMBER]; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + struct rte_pci_addr *pci_addr = &pci_dev->addr; + struct rte_mem_resource *pci_rsc = + &pci_dev->mem_resource[PCI_RESOURCE_NUMBER]; char rsc_filename[PATH_MAX]; void *pci_resource_ptr = NULL; int fd; @@ -1427,7 +1486,7 @@ rte_szedata2_eth_dev_init(struct rte_eth_dev *dev) pci_addr->function); /* Get index of szedata2 device file and create path to device file */ - ret = get_szedata2_index(dev, &szedata2_index); + ret = get_szedata2_index(pci_addr, &szedata2_index); if (ret != 0) { RTE_LOG(ERR, PMD, "Failed to get szedata2 device index!\n"); return -ENODEV; @@ -1471,17 +1530,17 @@ rte_szedata2_eth_dev_init(struct rte_eth_dev *dev) /* Set function callbacks for Ethernet API */ dev->dev_ops = &ops; - rte_eth_copy_pci_info(dev, dev->pci_dev); + rte_eth_copy_pci_info(dev, pci_dev); - /* mmap pci resource0 file to rte_pci_resource structure */ - if (dev->pci_dev->mem_resource[PCI_RESOURCE_NUMBER].phys_addr == + /* mmap pci resource0 file to rte_mem_resource structure */ + if (pci_dev->mem_resource[PCI_RESOURCE_NUMBER].phys_addr == 0) { RTE_LOG(ERR, PMD, "Missing resource%u file\n", PCI_RESOURCE_NUMBER); return -EINVAL; } snprintf(rsc_filename, PATH_MAX, - "%s/" PCI_PRI_FMT "/resource%u", pci_get_sysfs_path(), + "%s/" PCI_PRI_FMT "/resource%u", rte_pci_get_sysfs_path(), pci_addr->domain, pci_addr->bus, pci_addr->devid, pci_addr->function, PCI_RESOURCE_NUMBER); fd = open(rsc_filename, O_RDWR); @@ -1491,16 +1550,16 @@ rte_szedata2_eth_dev_init(struct rte_eth_dev *dev) } pci_resource_ptr = mmap(0, - dev->pci_dev->mem_resource[PCI_RESOURCE_NUMBER].len, + pci_dev->mem_resource[PCI_RESOURCE_NUMBER].len, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); close(fd); - if (pci_resource_ptr == NULL) { + if (pci_resource_ptr == MAP_FAILED) { RTE_LOG(ERR, PMD, "Could not mmap file %s (fd = %d)\n", rsc_filename, fd); return -EINVAL; } - dev->pci_dev->mem_resource[PCI_RESOURCE_NUMBER].addr = - pci_resource_ptr; + pci_dev->mem_resource[PCI_RESOURCE_NUMBER].addr = pci_resource_ptr; + internals->pci_rsc = pci_rsc; RTE_LOG(DEBUG, PMD, "resource%u phys_addr = 0x%llx len = %llu " "virt addr = %llx\n", PCI_RESOURCE_NUMBER, @@ -1516,8 +1575,8 @@ rte_szedata2_eth_dev_init(struct rte_eth_dev *dev) RTE_CACHE_LINE_SIZE); if (data->mac_addrs == NULL) { RTE_LOG(ERR, PMD, "Could not alloc space for MAC address!\n"); - munmap(dev->pci_dev->mem_resource[PCI_RESOURCE_NUMBER].addr, - dev->pci_dev->mem_resource[PCI_RESOURCE_NUMBER].len); + munmap(pci_dev->mem_resource[PCI_RESOURCE_NUMBER].addr, + pci_dev->mem_resource[PCI_RESOURCE_NUMBER].len); return -EINVAL; } @@ -1537,12 +1596,13 @@ rte_szedata2_eth_dev_init(struct rte_eth_dev *dev) static int rte_szedata2_eth_dev_uninit(struct rte_eth_dev *dev) { - struct rte_pci_addr *pci_addr = &dev->pci_dev->addr; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + struct rte_pci_addr *pci_addr = &pci_dev->addr; rte_free(dev->data->mac_addrs); dev->data->mac_addrs = NULL; - munmap(dev->pci_dev->mem_resource[PCI_RESOURCE_NUMBER].addr, - dev->pci_dev->mem_resource[PCI_RESOURCE_NUMBER].len); + munmap(pci_dev->mem_resource[PCI_RESOURCE_NUMBER].addr, + pci_dev->mem_resource[PCI_RESOURCE_NUMBER].len); RTE_LOG(INFO, PMD, "szedata2 device (" PCI_PRI_FMT ") successfully uninitialized\n", @@ -1570,35 +1630,26 @@ static const struct rte_pci_id rte_szedata2_pci_id_table[] = { } }; -static struct eth_driver szedata2_eth_driver = { - .pci_drv = { - .name = RTE_SZEDATA2_PCI_DRIVER_NAME, - .id_table = rte_szedata2_pci_id_table, - }, - .eth_dev_init = rte_szedata2_eth_dev_init, - .eth_dev_uninit = rte_szedata2_eth_dev_uninit, - .dev_private_size = sizeof(struct pmd_internals), -}; - -static int -rte_szedata2_init(const char *name __rte_unused, - const char *args __rte_unused) +static int szedata2_eth_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, + struct rte_pci_device *pci_dev) { - rte_eth_driver_register(&szedata2_eth_driver); - return 0; + return rte_eth_dev_pci_generic_probe(pci_dev, + sizeof(struct pmd_internals), rte_szedata2_eth_dev_init); } -static int -rte_szedata2_uninit(const char *name __rte_unused) +static int szedata2_eth_pci_remove(struct rte_pci_device *pci_dev) { - return 0; + return rte_eth_dev_pci_generic_remove(pci_dev, + rte_szedata2_eth_dev_uninit); } -static struct rte_driver rte_szedata2_driver = { - .type = PMD_PDEV, - .init = rte_szedata2_init, - .uninit = rte_szedata2_uninit, +static struct rte_pci_driver szedata2_eth_driver = { + .id_table = rte_szedata2_pci_id_table, + .probe = szedata2_eth_pci_probe, + .remove = szedata2_eth_pci_remove, }; -PMD_REGISTER_DRIVER(rte_szedata2_driver, RTE_SZEDATA2_DRIVER_NAME); -DRIVER_REGISTER_PCI_TABLE(RTE_SZEDATA2_DRIVER_NAME, rte_szedata2_pci_id_table); +RTE_PMD_REGISTER_PCI(RTE_SZEDATA2_DRIVER_NAME, szedata2_eth_driver); +RTE_PMD_REGISTER_PCI_TABLE(RTE_SZEDATA2_DRIVER_NAME, rte_szedata2_pci_id_table); +RTE_PMD_REGISTER_KMOD_DEP(RTE_SZEDATA2_DRIVER_NAME, + "* combo6core & combov3 & szedata2 & szedata2_cv3"); diff --git a/dpdk/drivers/net/szedata2/rte_eth_szedata2.h b/dpdk/drivers/net/szedata2/rte_eth_szedata2.h index 522cf47f..f25d4c59 100644 --- a/dpdk/drivers/net/szedata2/rte_eth_szedata2.h +++ b/dpdk/drivers/net/szedata2/rte_eth_szedata2.h @@ -34,9 +34,11 @@ #ifndef RTE_PMD_SZEDATA2_H_ #define RTE_PMD_SZEDATA2_H_ -#include +#include -#include +#include + +#include /* PCI Vendor ID */ #define PCI_VENDOR_ID_NETCOPE 0x1b26 @@ -58,7 +60,7 @@ * Round 'what' to the nearest larger (or equal) multiple of '8' * (szedata2 packet is aligned to 8 bytes) */ -#define RTE_SZE2_ALIGN8(what) (((what) + ((8) - 1)) & (~((8) - 1))) +#define RTE_SZE2_ALIGN8(what) RTE_ALIGN(what, 8) /*! main handle structure */ struct szedata { @@ -113,350 +115,4 @@ struct szedata { int numa_node; }; -/* - * @return Byte from PCI resource at offset "offset". - */ -static inline uint8_t -pci_resource_read8(struct rte_eth_dev *dev, uint32_t offset) -{ - return *((uint8_t *)((uint8_t *) - dev->pci_dev->mem_resource[PCI_RESOURCE_NUMBER].addr + - offset)); -} - -/* - * @return Two bytes from PCI resource starting at offset "offset". - */ -static inline uint16_t -pci_resource_read16(struct rte_eth_dev *dev, uint32_t offset) -{ - return rte_le_to_cpu_16(*((uint16_t *)((uint8_t *) - dev->pci_dev->mem_resource[PCI_RESOURCE_NUMBER].addr + - offset))); -} - -/* - * @return Four bytes from PCI resource starting at offset "offset". - */ -static inline uint32_t -pci_resource_read32(struct rte_eth_dev *dev, uint32_t offset) -{ - return rte_le_to_cpu_32(*((uint32_t *)((uint8_t *) - dev->pci_dev->mem_resource[PCI_RESOURCE_NUMBER].addr + - offset))); -} - -/* - * @return Eight bytes from PCI resource starting at offset "offset". - */ -static inline uint64_t -pci_resource_read64(struct rte_eth_dev *dev, uint32_t offset) -{ - return rte_le_to_cpu_64(*((uint64_t *)((uint8_t *) - dev->pci_dev->mem_resource[PCI_RESOURCE_NUMBER].addr + - offset))); -} - -/* - * Write one byte to PCI resource address space at offset "offset". - */ -static inline void -pci_resource_write8(struct rte_eth_dev *dev, uint32_t offset, uint8_t val) -{ - *((uint8_t *)((uint8_t *) - dev->pci_dev->mem_resource[PCI_RESOURCE_NUMBER].addr + - offset)) = val; -} - -/* - * Write two bytes to PCI resource address space at offset "offset". - */ -static inline void -pci_resource_write16(struct rte_eth_dev *dev, uint32_t offset, uint16_t val) -{ - *((uint16_t *)((uint8_t *) - dev->pci_dev->mem_resource[PCI_RESOURCE_NUMBER].addr + - offset)) = rte_cpu_to_le_16(val); -} - -/* - * Write four bytes to PCI resource address space at offset "offset". - */ -static inline void -pci_resource_write32(struct rte_eth_dev *dev, uint32_t offset, uint32_t val) -{ - *((uint32_t *)((uint8_t *) - dev->pci_dev->mem_resource[PCI_RESOURCE_NUMBER].addr + - offset)) = rte_cpu_to_le_32(val); -} - -/* - * Write eight bytes to PCI resource address space at offset "offset". - */ -static inline void -pci_resource_write64(struct rte_eth_dev *dev, uint32_t offset, uint64_t val) -{ - *((uint64_t *)((uint8_t *) - dev->pci_dev->mem_resource[PCI_RESOURCE_NUMBER].addr + - offset)) = rte_cpu_to_le_64(val); -} - -#define SZEDATA2_PCI_RESOURCE_PTR(dev, offset, type) \ - ((type)((uint8_t *) \ - ((dev)->pci_dev->mem_resource[PCI_RESOURCE_NUMBER].addr) \ - + (offset))) - -enum szedata2_link_speed { - SZEDATA2_LINK_SPEED_DEFAULT = 0, - SZEDATA2_LINK_SPEED_10G, - SZEDATA2_LINK_SPEED_40G, - SZEDATA2_LINK_SPEED_100G, -}; - -enum szedata2_mac_check_mode { - SZEDATA2_MAC_CHMODE_PROMISC = 0x0, - SZEDATA2_MAC_CHMODE_ONLY_VALID = 0x1, - SZEDATA2_MAC_CHMODE_ALL_BROADCAST = 0x2, - SZEDATA2_MAC_CHMODE_ALL_MULTICAST = 0x3, -}; - -/* - * Structure describes CGMII IBUF address space - */ -struct szedata2_cgmii_ibuf { - /** Total Received Frames Counter low part */ - uint32_t trfcl; - /** Correct Frames Counter low part */ - uint32_t cfcl; - /** Discarded Frames Counter low part */ - uint32_t dfcl; - /** Counter of frames discarded due to buffer overflow low part */ - uint32_t bodfcl; - /** Total Received Frames Counter high part */ - uint32_t trfch; - /** Correct Frames Counter high part */ - uint32_t cfch; - /** Discarded Frames Counter high part */ - uint32_t dfch; - /** Counter of frames discarded due to buffer overflow high part */ - uint32_t bodfch; - /** IBUF enable register */ - uint32_t ibuf_en; - /** Error mask register */ - uint32_t err_mask; - /** IBUF status register */ - uint32_t ibuf_st; - /** IBUF command register */ - uint32_t ibuf_cmd; - /** Minimum frame length allowed */ - uint32_t mfla; - /** Frame MTU */ - uint32_t mtu; - /** MAC address check mode */ - uint32_t mac_chmode; - /** Octets Received OK Counter low part */ - uint32_t orocl; - /** Octets Received OK Counter high part */ - uint32_t oroch; -} __rte_packed; - -/* Offset of CGMII IBUF memory for MAC addresses */ -#define SZEDATA2_CGMII_IBUF_MAC_MEM_OFF 0x80 - -/* - * @return - * true if IBUF is enabled - * false if IBUF is disabled - */ -static inline bool -cgmii_ibuf_is_enabled(volatile struct szedata2_cgmii_ibuf *ibuf) -{ - return ((rte_le_to_cpu_32(ibuf->ibuf_en) & 0x1) != 0) ? true : false; -} - -/* - * Enables IBUF. - */ -static inline void -cgmii_ibuf_enable(volatile struct szedata2_cgmii_ibuf *ibuf) -{ - ibuf->ibuf_en = - rte_cpu_to_le_32(rte_le_to_cpu_32(ibuf->ibuf_en) | 0x1); -} - -/* - * Disables IBUF. - */ -static inline void -cgmii_ibuf_disable(volatile struct szedata2_cgmii_ibuf *ibuf) -{ - ibuf->ibuf_en = - rte_cpu_to_le_32(rte_le_to_cpu_32(ibuf->ibuf_en) & ~0x1); -} - -/* - * @return - * true if ibuf link is up - * false if ibuf link is down - */ -static inline bool -cgmii_ibuf_is_link_up(volatile struct szedata2_cgmii_ibuf *ibuf) -{ - return ((rte_le_to_cpu_32(ibuf->ibuf_st) & 0x80) != 0) ? true : false; -} - -/* - * @return - * MAC address check mode - */ -static inline enum szedata2_mac_check_mode -cgmii_ibuf_mac_mode_read(volatile struct szedata2_cgmii_ibuf *ibuf) -{ - switch (rte_le_to_cpu_32(ibuf->mac_chmode) & 0x3) { - case 0x0: - return SZEDATA2_MAC_CHMODE_PROMISC; - case 0x1: - return SZEDATA2_MAC_CHMODE_ONLY_VALID; - case 0x2: - return SZEDATA2_MAC_CHMODE_ALL_BROADCAST; - case 0x3: - return SZEDATA2_MAC_CHMODE_ALL_MULTICAST; - default: - return SZEDATA2_MAC_CHMODE_PROMISC; - } -} - -/* - * Writes "mode" in MAC address check mode register. - */ -static inline void -cgmii_ibuf_mac_mode_write(volatile struct szedata2_cgmii_ibuf *ibuf, - enum szedata2_mac_check_mode mode) -{ - ibuf->mac_chmode = rte_cpu_to_le_32( - (rte_le_to_cpu_32(ibuf->mac_chmode) & ~0x3) | mode); -} - -/* - * Structure describes CGMII OBUF address space - */ -struct szedata2_cgmii_obuf { - /** Total Sent Frames Counter low part */ - uint32_t tsfcl; - /** Octets Sent Counter low part */ - uint32_t oscl; - /** Total Discarded Frames Counter low part */ - uint32_t tdfcl; - /** reserved */ - uint32_t reserved1; - /** Total Sent Frames Counter high part */ - uint32_t tsfch; - /** Octets Sent Counter high part */ - uint32_t osch; - /** Total Discarded Frames Counter high part */ - uint32_t tdfch; - /** reserved */ - uint32_t reserved2; - /** OBUF enable register */ - uint32_t obuf_en; - /** reserved */ - uint64_t reserved3; - /** OBUF control register */ - uint32_t ctrl; - /** OBUF status register */ - uint32_t obuf_st; -} __rte_packed; - -/* - * @return - * true if OBUF is enabled - * false if OBUF is disabled - */ -static inline bool -cgmii_obuf_is_enabled(volatile struct szedata2_cgmii_obuf *obuf) -{ - return ((rte_le_to_cpu_32(obuf->obuf_en) & 0x1) != 0) ? true : false; -} - -/* - * Enables OBUF. - */ -static inline void -cgmii_obuf_enable(volatile struct szedata2_cgmii_obuf *obuf) -{ - obuf->obuf_en = - rte_cpu_to_le_32(rte_le_to_cpu_32(obuf->obuf_en) | 0x1); -} - -/* - * Disables OBUF. - */ -static inline void -cgmii_obuf_disable(volatile struct szedata2_cgmii_obuf *obuf) -{ - obuf->obuf_en = - rte_cpu_to_le_32(rte_le_to_cpu_32(obuf->obuf_en) & ~0x1); -} - -/* - * Function takes value from IBUF status register. Values in IBUF and OBUF - * should be same. - * - * @return Link speed constant. - */ -static inline enum szedata2_link_speed -cgmii_link_speed(volatile struct szedata2_cgmii_ibuf *ibuf) -{ - uint32_t speed = (rte_le_to_cpu_32(ibuf->ibuf_st) & 0x70) >> 4; - switch (speed) { - case 0x03: - return SZEDATA2_LINK_SPEED_10G; - case 0x04: - return SZEDATA2_LINK_SPEED_40G; - case 0x05: - return SZEDATA2_LINK_SPEED_100G; - default: - return SZEDATA2_LINK_SPEED_DEFAULT; - } -} - -/* - * IBUFs and OBUFs can generally be located at different offsets in different - * firmwares. - * This part defines base offsets of IBUFs and OBUFs through various firmwares. - * Currently one firmware type is supported. - * Type of firmware is set through configuration option - * CONFIG_RTE_LIBRTE_PMD_SZEDATA_AS. - * Possible values are: - * 0 - for firmwares: - * NIC_100G1_LR4 - * HANIC_100G1_LR4 - * HANIC_100G1_SR10 - */ -#if !defined(RTE_LIBRTE_PMD_SZEDATA2_AS) -#error "RTE_LIBRTE_PMD_SZEDATA2_AS has to be defined" -#elif RTE_LIBRTE_PMD_SZEDATA2_AS == 0 - -/* - * CGMII IBUF offset from the beginning of PCI resource address space. - */ -#define SZEDATA2_CGMII_IBUF_BASE_OFF 0x8000 -/* - * Size of CGMII IBUF. - */ -#define SZEDATA2_CGMII_IBUF_SIZE 0x200 - -/* - * GCMII OBUF offset from the beginning of PCI resource address space. - */ -#define SZEDATA2_CGMII_OBUF_BASE_OFF 0x9000 -/* - * Size of CGMII OBUF. - */ -#define SZEDATA2_CGMII_OBUF_SIZE 0x100 - -#else -#error "RTE_LIBRTE_PMD_SZEDATA2_AS has wrong value, see comments in config file" -#endif - -#endif +#endif /* RTE_PMD_SZEDATA2_H_ */ diff --git a/dpdk/drivers/net/szedata2/szedata2_iobuf.c b/dpdk/drivers/net/szedata2/szedata2_iobuf.c new file mode 100644 index 00000000..3b9a71fe --- /dev/null +++ b/dpdk/drivers/net/szedata2/szedata2_iobuf.c @@ -0,0 +1,203 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2017 CESNET + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of CESNET nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +#include + +#include "szedata2_iobuf.h" + +/* + * IBUFs and OBUFs can generally be located at different offsets in different + * firmwares (modes). + * This part defines base offsets of IBUFs and OBUFs for various cards + * and firmwares (modes). + * Type of firmware (mode) is set through configuration option + * CONFIG_RTE_LIBRTE_PMD_SZEDATA2_AS. + * Possible values are: + * 0 - for cards (modes): + * NFB-100G1 (100G1) + * + * 1 - for cards (modes): + * NFB-100G2Q (100G1) + * + * 2 - for cards (modes): + * NFB-40G2 (40G2) + * NFB-100G2C (100G2) + * NFB-100G2Q (40G2) + * + * 3 - for cards (modes): + * NFB-40G2 (10G8) + * NFB-100G2Q (10G8) + * + * 4 - for cards (modes): + * NFB-100G1 (10G10) + * + * 5 - for experimental firmwares and future use + */ +#if !defined(RTE_LIBRTE_PMD_SZEDATA2_AS) +#error "RTE_LIBRTE_PMD_SZEDATA2_AS has to be defined" +#elif RTE_LIBRTE_PMD_SZEDATA2_AS == 0 + +/* + * Cards (modes): + * NFB-100G1 (100G1) + */ + +const uint32_t szedata2_ibuf_base_table[] = { + 0x8000 +}; +const uint32_t szedata2_obuf_base_table[] = { + 0x9000 +}; + +#elif RTE_LIBRTE_PMD_SZEDATA2_AS == 1 + +/* + * Cards (modes): + * NFB-100G2Q (100G1) + */ + +const uint32_t szedata2_ibuf_base_table[] = { + 0x8800 +}; +const uint32_t szedata2_obuf_base_table[] = { + 0x9800 +}; + +#elif RTE_LIBRTE_PMD_SZEDATA2_AS == 2 + +/* + * Cards (modes): + * NFB-40G2 (40G2) + * NFB-100G2C (100G2) + * NFB-100G2Q (40G2) + */ + +const uint32_t szedata2_ibuf_base_table[] = { + 0x8000, + 0x8800 +}; +const uint32_t szedata2_obuf_base_table[] = { + 0x9000, + 0x9800 +}; + +#elif RTE_LIBRTE_PMD_SZEDATA2_AS == 3 + +/* + * Cards (modes): + * NFB-40G2 (10G8) + * NFB-100G2Q (10G8) + */ + +const uint32_t szedata2_ibuf_base_table[] = { + 0x8000, + 0x8200, + 0x8400, + 0x8600, + 0x8800, + 0x8A00, + 0x8C00, + 0x8E00 +}; +const uint32_t szedata2_obuf_base_table[] = { + 0x9000, + 0x9200, + 0x9400, + 0x9600, + 0x9800, + 0x9A00, + 0x9C00, + 0x9E00 +}; + +#elif RTE_LIBRTE_PMD_SZEDATA2_AS == 4 + +/* + * Cards (modes): + * NFB-100G1 (10G10) + */ + +const uint32_t szedata2_ibuf_base_table[] = { + 0x8000, + 0x8200, + 0x8400, + 0x8600, + 0x8800, + 0x8A00, + 0x8C00, + 0x8E00, + 0x9000, + 0x9200 +}; +const uint32_t szedata2_obuf_base_table[] = { + 0xA000, + 0xA200, + 0xA400, + 0xA600, + 0xA800, + 0xAA00, + 0xAC00, + 0xAE00, + 0xB000, + 0xB200 +}; + +#elif RTE_LIBRTE_PMD_SZEDATA2_AS == 5 + +/* + * Future use and experimental firmwares. + */ + +const uint32_t szedata2_ibuf_base_table[] = { + 0x8000, + 0x8200, + 0x8400, + 0x8600, + 0x8800 +}; +const uint32_t szedata2_obuf_base_table[] = { + 0x9000, + 0x9200, + 0x9400, + 0x9600, + 0x9800 +}; + +#else +#error "RTE_LIBRTE_PMD_SZEDATA2_AS has wrong value, see comments in config file" +#endif + +const uint32_t szedata2_ibuf_count = RTE_DIM(szedata2_ibuf_base_table); +const uint32_t szedata2_obuf_count = RTE_DIM(szedata2_obuf_base_table); diff --git a/dpdk/drivers/net/szedata2/szedata2_iobuf.h b/dpdk/drivers/net/szedata2/szedata2_iobuf.h new file mode 100644 index 00000000..f1ccb3b2 --- /dev/null +++ b/dpdk/drivers/net/szedata2/szedata2_iobuf.h @@ -0,0 +1,356 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2017 CESNET + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of CESNET nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _SZEDATA2_IOBUF_H_ +#define _SZEDATA2_IOBUF_H_ + +#include +#include + +#include +#include +#include + +/* IBUF offsets from the beginning of the PCI resource address space. */ +extern const uint32_t szedata2_ibuf_base_table[]; +extern const uint32_t szedata2_ibuf_count; + +/* OBUF offsets from the beginning of the PCI resource address space. */ +extern const uint32_t szedata2_obuf_base_table[]; +extern const uint32_t szedata2_obuf_count; + +enum szedata2_link_speed { + SZEDATA2_LINK_SPEED_DEFAULT = 0, + SZEDATA2_LINK_SPEED_10G, + SZEDATA2_LINK_SPEED_40G, + SZEDATA2_LINK_SPEED_100G, +}; + +enum szedata2_mac_check_mode { + SZEDATA2_MAC_CHMODE_PROMISC = 0x0, + SZEDATA2_MAC_CHMODE_ONLY_VALID = 0x1, + SZEDATA2_MAC_CHMODE_ALL_BROADCAST = 0x2, + SZEDATA2_MAC_CHMODE_ALL_MULTICAST = 0x3, +}; + +/** + * Macro takes pointer to pci resource structure (rsc) + * and returns pointer to mapped resource memory at + * specified offset (offset) typecast to the type (type). + */ +#define SZEDATA2_PCI_RESOURCE_PTR(rsc, offset, type) \ + ((type)(((uint8_t *)(rsc)->addr) + (offset))) + +/** + * Maximum possible number of MAC addresses (limited by IBUF status + * register value MAC_COUNT which has 5 bits). + */ +#define SZEDATA2_IBUF_MAX_MAC_COUNT 32 + +/** + * Structure describes IBUF address space. + */ +struct szedata2_ibuf { + /** Total Received Frames Counter low part */ + uint32_t trfcl; /**< 0x00 */ + /** Correct Frames Counter low part */ + uint32_t cfcl; /**< 0x04 */ + /** Discarded Frames Counter low part */ + uint32_t dfcl; /**< 0x08 */ + /** Counter of frames discarded due to buffer overflow low part */ + uint32_t bodfcl; /**< 0x0C */ + /** Total Received Frames Counter high part */ + uint32_t trfch; /**< 0x10 */ + /** Correct Frames Counter high part */ + uint32_t cfch; /**< 0x14 */ + /** Discarded Frames Counter high part */ + uint32_t dfch; /**< 0x18 */ + /** Counter of frames discarded due to buffer overflow high part */ + uint32_t bodfch; /**< 0x1C */ + /** IBUF enable register */ + uint32_t ibuf_en; /**< 0x20 */ + /** Error mask register */ + uint32_t err_mask; /**< 0x24 */ + /** IBUF status register */ + uint32_t ibuf_st; /**< 0x28 */ + /** IBUF command register */ + uint32_t ibuf_cmd; /**< 0x2C */ + /** Minimum frame length allowed */ + uint32_t mfla; /**< 0x30 */ + /** Frame MTU */ + uint32_t mtu; /**< 0x34 */ + /** MAC address check mode */ + uint32_t mac_chmode; /**< 0x38 */ + /** Octets Received OK Counter low part */ + uint32_t orocl; /**< 0x3C */ + /** Octets Received OK Counter high part */ + uint32_t oroch; /**< 0x40 */ + /** reserved */ + uint8_t reserved[60]; /**< 0x4C */ + /** IBUF memory for MAC addresses */ + uint32_t mac_mem[2 * SZEDATA2_IBUF_MAX_MAC_COUNT]; /**< 0x80 */ +} __rte_packed; + +/** + * Structure describes OBUF address space. + */ +struct szedata2_obuf { + /** Total Sent Frames Counter low part */ + uint32_t tsfcl; /**< 0x00 */ + /** Octets Sent Counter low part */ + uint32_t oscl; /**< 0x04 */ + /** Total Discarded Frames Counter low part */ + uint32_t tdfcl; /**< 0x08 */ + /** reserved */ + uint32_t reserved1; /**< 0x0C */ + /** Total Sent Frames Counter high part */ + uint32_t tsfch; /**< 0x10 */ + /** Octets Sent Counter high part */ + uint32_t osch; /**< 0x14 */ + /** Total Discarded Frames Counter high part */ + uint32_t tdfch; /**< 0x18 */ + /** reserved */ + uint32_t reserved2; /**< 0x1C */ + /** OBUF enable register */ + uint32_t obuf_en; /**< 0x20 */ + /** reserved */ + uint64_t reserved3; /**< 0x24 */ + /** OBUF control register */ + uint32_t ctrl; /**< 0x2C */ + /** OBUF status register */ + uint32_t obuf_st; /**< 0x30 */ +} __rte_packed; + +/** + * Wrapper for reading 4 bytes from device memory in correct endianness. + * + * @param addr + * Address for reading. + * @return + * 4 B value. + */ +static inline uint32_t +szedata2_read32(const volatile void *addr) +{ + return rte_le_to_cpu_32(rte_read32(addr)); +} + +/** + * Wrapper for writing 4 bytes to device memory in correct endianness. + * + * @param value + * Value to write. + * @param addr + * Address for writing. + */ +static inline void +szedata2_write32(uint32_t value, volatile void *addr) +{ + rte_write32(rte_cpu_to_le_32(value), addr); +} + +/** + * Get pointer to IBUF structure according to specified index. + * + * @param rsc + * Pointer to base address of memory resource. + * @param index + * Index of IBUF. + * @return + * Pointer to IBUF structure. + */ +static inline struct szedata2_ibuf * +ibuf_ptr_by_index(struct rte_mem_resource *rsc, uint32_t index) +{ + if (index >= szedata2_ibuf_count) + index = szedata2_ibuf_count - 1; + return SZEDATA2_PCI_RESOURCE_PTR(rsc, szedata2_ibuf_base_table[index], + struct szedata2_ibuf *); +} + +/** + * Get pointer to OBUF structure according to specified idnex. + * + * @param rsc + * Pointer to base address of memory resource. + * @param index + * Index of OBUF. + * @return + * Pointer to OBUF structure. + */ +static inline struct szedata2_obuf * +obuf_ptr_by_index(struct rte_mem_resource *rsc, uint32_t index) +{ + if (index >= szedata2_obuf_count) + index = szedata2_obuf_count - 1; + return SZEDATA2_PCI_RESOURCE_PTR(rsc, szedata2_obuf_base_table[index], + struct szedata2_obuf *); +} + +/** + * Checks if IBUF is enabled. + * + * @param ibuf + * Pointer to IBUF structure. + * @return + * true if IBUF is enabled. + * false if IBUF is disabled. + */ +static inline bool +ibuf_is_enabled(const volatile struct szedata2_ibuf *ibuf) +{ + return ((szedata2_read32(&ibuf->ibuf_en) & 0x1) != 0) ? true : false; +} + +/** + * Enables IBUF. + * + * @param ibuf + * Pointer to IBUF structure. + */ +static inline void +ibuf_enable(volatile struct szedata2_ibuf *ibuf) +{ + szedata2_write32(szedata2_read32(&ibuf->ibuf_en) | 0x1, &ibuf->ibuf_en); +} + +/** + * Disables IBUF. + * + * @param ibuf + * Pointer to IBUF structure. + */ +static inline void +ibuf_disable(volatile struct szedata2_ibuf *ibuf) +{ + szedata2_write32(szedata2_read32(&ibuf->ibuf_en) & ~0x1, + &ibuf->ibuf_en); +} + +/** + * Checks if link is up. + * + * @param ibuf + * Pointer to IBUF structure. + * @return + * true if ibuf link is up. + * false if ibuf link is down. + */ +static inline bool +ibuf_is_link_up(const volatile struct szedata2_ibuf *ibuf) +{ + return ((szedata2_read32(&ibuf->ibuf_st) & 0x80) != 0) ? true : false; +} + +/** + * Get current MAC address check mode from IBUF. + * + * @param ibuf + * Pointer to IBUF structure. + * @return + * MAC address check mode constant. + */ +static inline enum szedata2_mac_check_mode +ibuf_mac_mode_read(const volatile struct szedata2_ibuf *ibuf) +{ + switch (szedata2_read32(&ibuf->mac_chmode) & 0x3) { + case 0x0: + return SZEDATA2_MAC_CHMODE_PROMISC; + case 0x1: + return SZEDATA2_MAC_CHMODE_ONLY_VALID; + case 0x2: + return SZEDATA2_MAC_CHMODE_ALL_BROADCAST; + case 0x3: + return SZEDATA2_MAC_CHMODE_ALL_MULTICAST; + default: + return SZEDATA2_MAC_CHMODE_PROMISC; + } +} + +/** + * Writes mode in MAC address check mode register in IBUF. + * + * @param ibuf + * Pointer to IBUF structure. + * @param mode + * MAC address check mode to set. + */ +static inline void +ibuf_mac_mode_write(volatile struct szedata2_ibuf *ibuf, + enum szedata2_mac_check_mode mode) +{ + szedata2_write32((szedata2_read32(&ibuf->mac_chmode) & ~0x3) | mode, + &ibuf->mac_chmode); +} + +/** + * Checks if obuf is enabled. + * + * @param obuf + * Pointer to OBUF structure. + * @return + * true if OBUF is enabled. + * false if OBUF is disabled. + */ +static inline bool +obuf_is_enabled(const volatile struct szedata2_obuf *obuf) +{ + return ((szedata2_read32(&obuf->obuf_en) & 0x1) != 0) ? true : false; +} + +/** + * Enables OBUF. + * + * @param obuf + * Pointer to OBUF structure. + */ +static inline void +obuf_enable(volatile struct szedata2_obuf *obuf) +{ + szedata2_write32(szedata2_read32(&obuf->obuf_en) | 0x1, &obuf->obuf_en); +} + +/** + * Disables OBUF. + * + * @param obuf + * Pointer to OBUF structure. + */ +static inline void +obuf_disable(volatile struct szedata2_obuf *obuf) +{ + szedata2_write32(szedata2_read32(&obuf->obuf_en) & ~0x1, + &obuf->obuf_en); +} + +#endif /* _SZEDATA2_IOBUF_H_ */ diff --git a/dpdk/drivers/net/tap/Makefile b/dpdk/drivers/net/tap/Makefile new file mode 100644 index 00000000..405b49e4 --- /dev/null +++ b/dpdk/drivers/net/tap/Makefile @@ -0,0 +1,96 @@ +# BSD LICENSE +# +# Copyright(c) 2016 Intel Corporation. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +include $(RTE_SDK)/mk/rte.vars.mk + +# +# library name +# +LIB = librte_pmd_tap.a + +EXPORT_MAP := rte_pmd_tap_version.map + +LIBABIVER := 1 + +CFLAGS += -O3 +CFLAGS += -I$(SRCDIR) +CFLAGS += -I. +CFLAGS += $(WERROR_FLAGS) +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs -lrte_hash +LDLIBS += -lrte_bus_vdev + +# +# all source are stored in SRCS-y +# +SRCS-$(CONFIG_RTE_LIBRTE_PMD_TAP) += rte_eth_tap.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_TAP) += tap_flow.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_TAP) += tap_netlink.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_TAP) += tap_tcmsgs.c + +include $(RTE_SDK)/mk/rte.lib.mk + +# Generate and clean-up tap_autoconf.h. + +export CC CFLAGS CPPFLAGS EXTRA_CFLAGS EXTRA_CPPFLAGS +export AUTO_CONFIG_CFLAGS = -Wno-error + +ifndef V +AUTOCONF_OUTPUT := >/dev/null +endif + +tap_autoconf.h.new: FORCE + +tap_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh + $Q $(RM) -f -- '$@' + $Q sh -- '$<' '$@' \ + HAVE_TC_FLOWER \ + linux/pkt_cls.h \ + enum TCA_FLOWER_UNSPEC \ + $(AUTOCONF_OUTPUT) + $Q sh -- '$<' '$@' \ + HAVE_TC_VLAN_ID \ + linux/pkt_cls.h \ + enum TCA_FLOWER_KEY_VLAN_PRIO \ + $(AUTOCONF_OUTPUT) + +# Create tap_autoconf.h or update it in case it differs from the new one. + +tap_autoconf.h: tap_autoconf.h.new + $Q [ -f '$@' ] && \ + cmp '$<' '$@' $(AUTOCONF_OUTPUT) || \ + mv '$<' '$@' + +$(SRCS-$(CONFIG_RTE_LIBRTE_PMD_TAP):.c=.o): tap_autoconf.h + +clean_tap: FORCE + $Q rm -f -- tap_autoconf.h tap_autoconf.h.new + +clean: clean_tap diff --git a/dpdk/drivers/net/tap/rte_eth_tap.c b/dpdk/drivers/net/tap/rte_eth_tap.c new file mode 100644 index 00000000..b8187f9b --- /dev/null +++ b/dpdk/drivers/net/tap/rte_eth_tap.c @@ -0,0 +1,1615 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +/* Linux based path to the TUN device */ +#define TUN_TAP_DEV_PATH "/dev/net/tun" +#define DEFAULT_TAP_NAME "dtap" + +#define ETH_TAP_IFACE_ARG "iface" +#define ETH_TAP_SPEED_ARG "speed" +#define ETH_TAP_REMOTE_ARG "remote" +#define ETH_TAP_MAC_ARG "mac" +#define ETH_TAP_MAC_FIXED "fixed" + +static struct rte_vdev_driver pmd_tap_drv; + +static const char *valid_arguments[] = { + ETH_TAP_IFACE_ARG, + ETH_TAP_SPEED_ARG, + ETH_TAP_REMOTE_ARG, + ETH_TAP_MAC_ARG, + NULL +}; + +static int tap_unit; + +static volatile uint32_t tap_trigger; /* Rx trigger */ + +static struct rte_eth_link pmd_link = { + .link_speed = ETH_SPEED_NUM_10G, + .link_duplex = ETH_LINK_FULL_DUPLEX, + .link_status = ETH_LINK_DOWN, + .link_autoneg = ETH_LINK_AUTONEG +}; + +static void +tap_trigger_cb(int sig __rte_unused) +{ + /* Valid trigger values are nonzero */ + tap_trigger = (tap_trigger + 1) | 0x80000000; +} + +/* Specifies on what netdevices the ioctl should be applied */ +enum ioctl_mode { + LOCAL_AND_REMOTE, + LOCAL_ONLY, + REMOTE_ONLY, +}; + +static int tap_intr_handle_set(struct rte_eth_dev *dev, int set); + +/* Tun/Tap allocation routine + * + * name is the number of the interface to use, unless NULL to take the host + * supplied name. + */ +static int +tun_alloc(struct pmd_internals *pmd) +{ + struct ifreq ifr; +#ifdef IFF_MULTI_QUEUE + unsigned int features; +#endif + int fd; + + memset(&ifr, 0, sizeof(struct ifreq)); + + /* + * Do not set IFF_NO_PI as packet information header will be needed + * to check if a received packet has been truncated. + */ + ifr.ifr_flags = IFF_TAP; + snprintf(ifr.ifr_name, IFNAMSIZ, "%s", pmd->name); + + RTE_LOG(DEBUG, PMD, "ifr_name '%s'\n", ifr.ifr_name); + + fd = open(TUN_TAP_DEV_PATH, O_RDWR); + if (fd < 0) { + RTE_LOG(ERR, PMD, "Unable to create TAP interface\n"); + goto error; + } + +#ifdef IFF_MULTI_QUEUE + /* Grab the TUN features to verify we can work multi-queue */ + if (ioctl(fd, TUNGETFEATURES, &features) < 0) { + RTE_LOG(ERR, PMD, "TAP unable to get TUN/TAP features\n"); + goto error; + } + RTE_LOG(DEBUG, PMD, " TAP Features %08x\n", features); + + if (features & IFF_MULTI_QUEUE) { + RTE_LOG(DEBUG, PMD, " Multi-queue support for %d queues\n", + RTE_PMD_TAP_MAX_QUEUES); + ifr.ifr_flags |= IFF_MULTI_QUEUE; + } else +#endif + { + ifr.ifr_flags |= IFF_ONE_QUEUE; + RTE_LOG(DEBUG, PMD, " Single queue only support\n"); + } + + /* Set the TUN/TAP configuration and set the name if needed */ + if (ioctl(fd, TUNSETIFF, (void *)&ifr) < 0) { + RTE_LOG(WARNING, PMD, + "Unable to set TUNSETIFF for %s\n", + ifr.ifr_name); + perror("TUNSETIFF"); + goto error; + } + + /* Always set the file descriptor to non-blocking */ + if (fcntl(fd, F_SETFL, O_NONBLOCK) < 0) { + RTE_LOG(WARNING, PMD, + "Unable to set %s to nonblocking\n", + ifr.ifr_name); + perror("F_SETFL, NONBLOCK"); + goto error; + } + + /* Set up trigger to optimize empty Rx bursts */ + errno = 0; + do { + struct sigaction sa; + int flags = fcntl(fd, F_GETFL); + + if (flags == -1 || sigaction(SIGIO, NULL, &sa) == -1) + break; + if (sa.sa_handler != tap_trigger_cb) { + /* + * Make sure SIGIO is not already taken. This is done + * as late as possible to leave the application a + * chance to set up its own signal handler first. + */ + if (sa.sa_handler != SIG_IGN && + sa.sa_handler != SIG_DFL) { + errno = EBUSY; + break; + } + sa = (struct sigaction){ + .sa_flags = SA_RESTART, + .sa_handler = tap_trigger_cb, + }; + if (sigaction(SIGIO, &sa, NULL) == -1) + break; + } + /* Enable SIGIO on file descriptor */ + fcntl(fd, F_SETFL, flags | O_ASYNC); + fcntl(fd, F_SETOWN, getpid()); + } while (0); + if (errno) { + /* Disable trigger globally in case of error */ + tap_trigger = 0; + RTE_LOG(WARNING, PMD, "Rx trigger disabled: %s\n", + strerror(errno)); + } + + return fd; + +error: + if (fd > 0) + close(fd); + return -1; +} + +static void +tap_verify_csum(struct rte_mbuf *mbuf) +{ + uint32_t l2 = mbuf->packet_type & RTE_PTYPE_L2_MASK; + uint32_t l3 = mbuf->packet_type & RTE_PTYPE_L3_MASK; + uint32_t l4 = mbuf->packet_type & RTE_PTYPE_L4_MASK; + unsigned int l2_len = sizeof(struct ether_hdr); + unsigned int l3_len; + uint16_t cksum = 0; + void *l3_hdr; + void *l4_hdr; + + if (l2 == RTE_PTYPE_L2_ETHER_VLAN) + l2_len += 4; + else if (l2 == RTE_PTYPE_L2_ETHER_QINQ) + l2_len += 8; + /* Don't verify checksum for packets with discontinuous L2 header */ + if (unlikely(l2_len + sizeof(struct ipv4_hdr) > + rte_pktmbuf_data_len(mbuf))) + return; + l3_hdr = rte_pktmbuf_mtod_offset(mbuf, void *, l2_len); + if (l3 == RTE_PTYPE_L3_IPV4 || l3 == RTE_PTYPE_L3_IPV4_EXT) { + struct ipv4_hdr *iph = l3_hdr; + + /* ihl contains the number of 4-byte words in the header */ + l3_len = 4 * (iph->version_ihl & 0xf); + if (unlikely(l2_len + l3_len > rte_pktmbuf_data_len(mbuf))) + return; + + cksum = ~rte_raw_cksum(iph, l3_len); + mbuf->ol_flags |= cksum ? + PKT_RX_IP_CKSUM_BAD : + PKT_RX_IP_CKSUM_GOOD; + } else if (l3 == RTE_PTYPE_L3_IPV6) { + l3_len = sizeof(struct ipv6_hdr); + } else { + /* IPv6 extensions are not supported */ + return; + } + if (l4 == RTE_PTYPE_L4_UDP || l4 == RTE_PTYPE_L4_TCP) { + l4_hdr = rte_pktmbuf_mtod_offset(mbuf, void *, l2_len + l3_len); + /* Don't verify checksum for multi-segment packets. */ + if (mbuf->nb_segs > 1) + return; + if (l3 == RTE_PTYPE_L3_IPV4) + cksum = ~rte_ipv4_udptcp_cksum(l3_hdr, l4_hdr); + else if (l3 == RTE_PTYPE_L3_IPV6) + cksum = ~rte_ipv6_udptcp_cksum(l3_hdr, l4_hdr); + mbuf->ol_flags |= cksum ? + PKT_RX_L4_CKSUM_BAD : + PKT_RX_L4_CKSUM_GOOD; + } +} + +/* Callback to handle the rx burst of packets to the correct interface and + * file descriptor(s) in a multi-queue setup. + */ +static uint16_t +pmd_rx_burst(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) +{ + struct rx_queue *rxq = queue; + uint16_t num_rx; + unsigned long num_rx_bytes = 0; + uint32_t trigger = tap_trigger; + + if (trigger == rxq->trigger_seen) + return 0; + if (trigger) + rxq->trigger_seen = trigger; + rte_compiler_barrier(); + for (num_rx = 0; num_rx < nb_pkts; ) { + struct rte_mbuf *mbuf = rxq->pool; + struct rte_mbuf *seg = NULL; + struct rte_mbuf *new_tail = NULL; + uint16_t data_off = rte_pktmbuf_headroom(mbuf); + int len; + + len = readv(rxq->fd, *rxq->iovecs, + 1 + (rxq->rxmode->enable_scatter ? + rxq->nb_rx_desc : 1)); + if (len < (int)sizeof(struct tun_pi)) + break; + + /* Packet couldn't fit in the provided mbuf */ + if (unlikely(rxq->pi.flags & TUN_PKT_STRIP)) { + rxq->stats.ierrors++; + continue; + } + + len -= sizeof(struct tun_pi); + + mbuf->pkt_len = len; + mbuf->port = rxq->in_port; + while (1) { + struct rte_mbuf *buf = rte_pktmbuf_alloc(rxq->mp); + + if (unlikely(!buf)) { + rxq->stats.rx_nombuf++; + /* No new buf has been allocated: do nothing */ + if (!new_tail || !seg) + goto end; + + seg->next = NULL; + rte_pktmbuf_free(mbuf); + + goto end; + } + seg = seg ? seg->next : mbuf; + if (rxq->pool == mbuf) + rxq->pool = buf; + if (new_tail) + new_tail->next = buf; + new_tail = buf; + new_tail->next = seg->next; + + /* iovecs[0] is reserved for packet info (pi) */ + (*rxq->iovecs)[mbuf->nb_segs].iov_len = + buf->buf_len - data_off; + (*rxq->iovecs)[mbuf->nb_segs].iov_base = + (char *)buf->buf_addr + data_off; + + seg->data_len = RTE_MIN(seg->buf_len - data_off, len); + seg->data_off = data_off; + + len -= seg->data_len; + if (len <= 0) + break; + mbuf->nb_segs++; + /* First segment has headroom, not the others */ + data_off = 0; + } + seg->next = NULL; + mbuf->packet_type = rte_net_get_ptype(mbuf, NULL, + RTE_PTYPE_ALL_MASK); + if (rxq->rxmode->hw_ip_checksum) + tap_verify_csum(mbuf); + + /* account for the receive frame */ + bufs[num_rx++] = mbuf; + num_rx_bytes += mbuf->pkt_len; + } +end: + rxq->stats.ipackets += num_rx; + rxq->stats.ibytes += num_rx_bytes; + + return num_rx; +} + +static void +tap_tx_offload(char *packet, uint64_t ol_flags, unsigned int l2_len, + unsigned int l3_len) +{ + void *l3_hdr = packet + l2_len; + + if (ol_flags & (PKT_TX_IP_CKSUM | PKT_TX_IPV4)) { + struct ipv4_hdr *iph = l3_hdr; + uint16_t cksum; + + iph->hdr_checksum = 0; + cksum = rte_raw_cksum(iph, l3_len); + iph->hdr_checksum = (cksum == 0xffff) ? cksum : ~cksum; + } + if (ol_flags & PKT_TX_L4_MASK) { + uint16_t l4_len; + uint32_t cksum; + uint16_t *l4_cksum; + void *l4_hdr; + + l4_hdr = packet + l2_len + l3_len; + if ((ol_flags & PKT_TX_L4_MASK) == PKT_TX_UDP_CKSUM) + l4_cksum = &((struct udp_hdr *)l4_hdr)->dgram_cksum; + else if ((ol_flags & PKT_TX_L4_MASK) == PKT_TX_TCP_CKSUM) + l4_cksum = &((struct tcp_hdr *)l4_hdr)->cksum; + else + return; + *l4_cksum = 0; + if (ol_flags & PKT_TX_IPV4) { + struct ipv4_hdr *iph = l3_hdr; + + l4_len = rte_be_to_cpu_16(iph->total_length) - l3_len; + cksum = rte_ipv4_phdr_cksum(l3_hdr, 0); + } else { + struct ipv6_hdr *ip6h = l3_hdr; + + /* payload_len does not include ext headers */ + l4_len = rte_be_to_cpu_16(ip6h->payload_len) - + l3_len + sizeof(struct ipv6_hdr); + cksum = rte_ipv6_phdr_cksum(l3_hdr, 0); + } + cksum += rte_raw_cksum(l4_hdr, l4_len); + cksum = ((cksum & 0xffff0000) >> 16) + (cksum & 0xffff); + cksum = (~cksum) & 0xffff; + if (cksum == 0) + cksum = 0xffff; + *l4_cksum = cksum; + } +} + +/* Callback to handle sending packets from the tap interface + */ +static uint16_t +pmd_tx_burst(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) +{ + struct tx_queue *txq = queue; + uint16_t num_tx = 0; + unsigned long num_tx_bytes = 0; + uint32_t max_size; + int i; + + if (unlikely(nb_pkts == 0)) + return 0; + + max_size = *txq->mtu + (ETHER_HDR_LEN + ETHER_CRC_LEN + 4); + for (i = 0; i < nb_pkts; i++) { + struct rte_mbuf *mbuf = bufs[num_tx]; + struct iovec iovecs[mbuf->nb_segs + 1]; + struct tun_pi pi = { .flags = 0 }; + struct rte_mbuf *seg = mbuf; + char m_copy[mbuf->data_len]; + int n; + int j; + + /* stats.errs will be incremented */ + if (rte_pktmbuf_pkt_len(mbuf) > max_size) + break; + + iovecs[0].iov_base = π + iovecs[0].iov_len = sizeof(pi); + for (j = 1; j <= mbuf->nb_segs; j++) { + iovecs[j].iov_len = rte_pktmbuf_data_len(seg); + iovecs[j].iov_base = + rte_pktmbuf_mtod(seg, void *); + seg = seg->next; + } + if (mbuf->ol_flags & (PKT_TX_IP_CKSUM | PKT_TX_IPV4) || + (mbuf->ol_flags & PKT_TX_L4_MASK) == PKT_TX_UDP_CKSUM || + (mbuf->ol_flags & PKT_TX_L4_MASK) == PKT_TX_TCP_CKSUM) { + /* Support only packets with all data in the same seg */ + if (mbuf->nb_segs > 1) + break; + /* To change checksums, work on a copy of data. */ + rte_memcpy(m_copy, rte_pktmbuf_mtod(mbuf, void *), + rte_pktmbuf_data_len(mbuf)); + tap_tx_offload(m_copy, mbuf->ol_flags, + mbuf->l2_len, mbuf->l3_len); + iovecs[1].iov_base = m_copy; + } + /* copy the tx frame data */ + n = writev(txq->fd, iovecs, mbuf->nb_segs + 1); + if (n <= 0) + break; + + num_tx++; + num_tx_bytes += mbuf->pkt_len; + rte_pktmbuf_free(mbuf); + } + + txq->stats.opackets += num_tx; + txq->stats.errs += nb_pkts - num_tx; + txq->stats.obytes += num_tx_bytes; + + return num_tx; +} + +static const char * +tap_ioctl_req2str(unsigned long request) +{ + switch (request) { + case SIOCSIFFLAGS: + return "SIOCSIFFLAGS"; + case SIOCGIFFLAGS: + return "SIOCGIFFLAGS"; + case SIOCGIFHWADDR: + return "SIOCGIFHWADDR"; + case SIOCSIFHWADDR: + return "SIOCSIFHWADDR"; + case SIOCSIFMTU: + return "SIOCSIFMTU"; + } + return "UNKNOWN"; +} + +static int +tap_ioctl(struct pmd_internals *pmd, unsigned long request, + struct ifreq *ifr, int set, enum ioctl_mode mode) +{ + short req_flags = ifr->ifr_flags; + int remote = pmd->remote_if_index && + (mode == REMOTE_ONLY || mode == LOCAL_AND_REMOTE); + + if (!pmd->remote_if_index && mode == REMOTE_ONLY) + return 0; + /* + * If there is a remote netdevice, apply ioctl on it, then apply it on + * the tap netdevice. + */ +apply: + if (remote) + snprintf(ifr->ifr_name, IFNAMSIZ, "%s", pmd->remote_iface); + else if (mode == LOCAL_ONLY || mode == LOCAL_AND_REMOTE) + snprintf(ifr->ifr_name, IFNAMSIZ, "%s", pmd->name); + switch (request) { + case SIOCSIFFLAGS: + /* fetch current flags to leave other flags untouched */ + if (ioctl(pmd->ioctl_sock, SIOCGIFFLAGS, ifr) < 0) + goto error; + if (set) + ifr->ifr_flags |= req_flags; + else + ifr->ifr_flags &= ~req_flags; + break; + case SIOCGIFFLAGS: + case SIOCGIFHWADDR: + case SIOCSIFHWADDR: + case SIOCSIFMTU: + break; + default: + RTE_ASSERT(!"unsupported request type: must not happen"); + } + if (ioctl(pmd->ioctl_sock, request, ifr) < 0) + goto error; + if (remote-- && mode == LOCAL_AND_REMOTE) + goto apply; + return 0; + +error: + RTE_LOG(DEBUG, PMD, "%s: %s(%s) failed: %s(%d)\n", ifr->ifr_name, + __func__, tap_ioctl_req2str(request), strerror(errno), errno); + return -errno; +} + +static int +tap_link_set_down(struct rte_eth_dev *dev) +{ + struct pmd_internals *pmd = dev->data->dev_private; + struct ifreq ifr = { .ifr_flags = IFF_UP }; + + dev->data->dev_link.link_status = ETH_LINK_DOWN; + return tap_ioctl(pmd, SIOCSIFFLAGS, &ifr, 0, LOCAL_ONLY); +} + +static int +tap_link_set_up(struct rte_eth_dev *dev) +{ + struct pmd_internals *pmd = dev->data->dev_private; + struct ifreq ifr = { .ifr_flags = IFF_UP }; + + dev->data->dev_link.link_status = ETH_LINK_UP; + return tap_ioctl(pmd, SIOCSIFFLAGS, &ifr, 1, LOCAL_AND_REMOTE); +} + +static int +tap_dev_start(struct rte_eth_dev *dev) +{ + int err; + + err = tap_intr_handle_set(dev, 1); + if (err) + return err; + return tap_link_set_up(dev); +} + +/* This function gets called when the current port gets stopped. + */ +static void +tap_dev_stop(struct rte_eth_dev *dev) +{ + tap_intr_handle_set(dev, 0); + tap_link_set_down(dev); +} + +static int +tap_dev_configure(struct rte_eth_dev *dev) +{ + if (dev->data->nb_rx_queues > RTE_PMD_TAP_MAX_QUEUES) { + RTE_LOG(ERR, PMD, + "%s: number of rx queues %d exceeds max num of queues %d\n", + dev->device->name, + dev->data->nb_rx_queues, + RTE_PMD_TAP_MAX_QUEUES); + return -1; + } + if (dev->data->nb_tx_queues > RTE_PMD_TAP_MAX_QUEUES) { + RTE_LOG(ERR, PMD, + "%s: number of tx queues %d exceeds max num of queues %d\n", + dev->device->name, + dev->data->nb_tx_queues, + RTE_PMD_TAP_MAX_QUEUES); + return -1; + } + + RTE_LOG(INFO, PMD, "%s: %p: TX configured queues number: %u\n", + dev->device->name, (void *)dev, dev->data->nb_tx_queues); + + RTE_LOG(INFO, PMD, "%s: %p: RX configured queues number: %u\n", + dev->device->name, (void *)dev, dev->data->nb_rx_queues); + + return 0; +} + +static uint32_t +tap_dev_speed_capa(void) +{ + uint32_t speed = pmd_link.link_speed; + uint32_t capa = 0; + + if (speed >= ETH_SPEED_NUM_10M) + capa |= ETH_LINK_SPEED_10M; + if (speed >= ETH_SPEED_NUM_100M) + capa |= ETH_LINK_SPEED_100M; + if (speed >= ETH_SPEED_NUM_1G) + capa |= ETH_LINK_SPEED_1G; + if (speed >= ETH_SPEED_NUM_5G) + capa |= ETH_LINK_SPEED_2_5G; + if (speed >= ETH_SPEED_NUM_5G) + capa |= ETH_LINK_SPEED_5G; + if (speed >= ETH_SPEED_NUM_10G) + capa |= ETH_LINK_SPEED_10G; + if (speed >= ETH_SPEED_NUM_20G) + capa |= ETH_LINK_SPEED_20G; + if (speed >= ETH_SPEED_NUM_25G) + capa |= ETH_LINK_SPEED_25G; + if (speed >= ETH_SPEED_NUM_40G) + capa |= ETH_LINK_SPEED_40G; + if (speed >= ETH_SPEED_NUM_50G) + capa |= ETH_LINK_SPEED_50G; + if (speed >= ETH_SPEED_NUM_56G) + capa |= ETH_LINK_SPEED_56G; + if (speed >= ETH_SPEED_NUM_100G) + capa |= ETH_LINK_SPEED_100G; + + return capa; +} + +static void +tap_dev_info(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) +{ + struct pmd_internals *internals = dev->data->dev_private; + + dev_info->if_index = internals->if_index; + dev_info->max_mac_addrs = 1; + dev_info->max_rx_pktlen = (uint32_t)ETHER_MAX_VLAN_FRAME_LEN; + dev_info->max_rx_queues = RTE_PMD_TAP_MAX_QUEUES; + dev_info->max_tx_queues = RTE_PMD_TAP_MAX_QUEUES; + dev_info->min_rx_bufsize = 0; + dev_info->pci_dev = NULL; + dev_info->speed_capa = tap_dev_speed_capa(); + dev_info->rx_offload_capa = (DEV_RX_OFFLOAD_IPV4_CKSUM | + DEV_RX_OFFLOAD_UDP_CKSUM | + DEV_RX_OFFLOAD_TCP_CKSUM); + dev_info->tx_offload_capa = + (DEV_TX_OFFLOAD_IPV4_CKSUM | + DEV_TX_OFFLOAD_UDP_CKSUM | + DEV_TX_OFFLOAD_TCP_CKSUM); +} + +static int +tap_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *tap_stats) +{ + unsigned int i, imax; + unsigned long rx_total = 0, tx_total = 0, tx_err_total = 0; + unsigned long rx_bytes_total = 0, tx_bytes_total = 0; + unsigned long rx_nombuf = 0, ierrors = 0; + const struct pmd_internals *pmd = dev->data->dev_private; + + /* rx queue statistics */ + imax = (dev->data->nb_rx_queues < RTE_ETHDEV_QUEUE_STAT_CNTRS) ? + dev->data->nb_rx_queues : RTE_ETHDEV_QUEUE_STAT_CNTRS; + for (i = 0; i < imax; i++) { + tap_stats->q_ipackets[i] = pmd->rxq[i].stats.ipackets; + tap_stats->q_ibytes[i] = pmd->rxq[i].stats.ibytes; + rx_total += tap_stats->q_ipackets[i]; + rx_bytes_total += tap_stats->q_ibytes[i]; + rx_nombuf += pmd->rxq[i].stats.rx_nombuf; + ierrors += pmd->rxq[i].stats.ierrors; + } + + /* tx queue statistics */ + imax = (dev->data->nb_tx_queues < RTE_ETHDEV_QUEUE_STAT_CNTRS) ? + dev->data->nb_tx_queues : RTE_ETHDEV_QUEUE_STAT_CNTRS; + + for (i = 0; i < imax; i++) { + tap_stats->q_opackets[i] = pmd->txq[i].stats.opackets; + tap_stats->q_errors[i] = pmd->txq[i].stats.errs; + tap_stats->q_obytes[i] = pmd->txq[i].stats.obytes; + tx_total += tap_stats->q_opackets[i]; + tx_err_total += tap_stats->q_errors[i]; + tx_bytes_total += tap_stats->q_obytes[i]; + } + + tap_stats->ipackets = rx_total; + tap_stats->ibytes = rx_bytes_total; + tap_stats->ierrors = ierrors; + tap_stats->rx_nombuf = rx_nombuf; + tap_stats->opackets = tx_total; + tap_stats->oerrors = tx_err_total; + tap_stats->obytes = tx_bytes_total; + return 0; +} + +static void +tap_stats_reset(struct rte_eth_dev *dev) +{ + int i; + struct pmd_internals *pmd = dev->data->dev_private; + + for (i = 0; i < RTE_PMD_TAP_MAX_QUEUES; i++) { + pmd->rxq[i].stats.ipackets = 0; + pmd->rxq[i].stats.ibytes = 0; + pmd->rxq[i].stats.ierrors = 0; + pmd->rxq[i].stats.rx_nombuf = 0; + + pmd->txq[i].stats.opackets = 0; + pmd->txq[i].stats.errs = 0; + pmd->txq[i].stats.obytes = 0; + } +} + +static void +tap_dev_close(struct rte_eth_dev *dev) +{ + int i; + struct pmd_internals *internals = dev->data->dev_private; + + tap_link_set_down(dev); + tap_flow_flush(dev, NULL); + tap_flow_implicit_flush(internals, NULL); + + for (i = 0; i < RTE_PMD_TAP_MAX_QUEUES; i++) { + if (internals->rxq[i].fd != -1) { + close(internals->rxq[i].fd); + internals->rxq[i].fd = -1; + } + if (internals->txq[i].fd != -1) { + close(internals->txq[i].fd); + internals->txq[i].fd = -1; + } + } + + if (internals->remote_if_index) { + /* Restore initial remote state */ + ioctl(internals->ioctl_sock, SIOCSIFFLAGS, + &internals->remote_initial_flags); + } +} + +static void +tap_rx_queue_release(void *queue) +{ + struct rx_queue *rxq = queue; + + if (rxq && (rxq->fd > 0)) { + close(rxq->fd); + rxq->fd = -1; + rte_pktmbuf_free(rxq->pool); + rte_free(rxq->iovecs); + rxq->pool = NULL; + rxq->iovecs = NULL; + } +} + +static void +tap_tx_queue_release(void *queue) +{ + struct tx_queue *txq = queue; + + if (txq && (txq->fd > 0)) { + close(txq->fd); + txq->fd = -1; + } +} + +static int +tap_link_update(struct rte_eth_dev *dev, int wait_to_complete __rte_unused) +{ + struct rte_eth_link *dev_link = &dev->data->dev_link; + struct pmd_internals *pmd = dev->data->dev_private; + struct ifreq ifr = { .ifr_flags = 0 }; + + if (pmd->remote_if_index) { + tap_ioctl(pmd, SIOCGIFFLAGS, &ifr, 0, REMOTE_ONLY); + if (!(ifr.ifr_flags & IFF_UP) || + !(ifr.ifr_flags & IFF_RUNNING)) { + dev_link->link_status = ETH_LINK_DOWN; + return 0; + } + } + tap_ioctl(pmd, SIOCGIFFLAGS, &ifr, 0, LOCAL_ONLY); + dev_link->link_status = + ((ifr.ifr_flags & IFF_UP) && (ifr.ifr_flags & IFF_RUNNING) ? + ETH_LINK_UP : + ETH_LINK_DOWN); + return 0; +} + +static void +tap_promisc_enable(struct rte_eth_dev *dev) +{ + struct pmd_internals *pmd = dev->data->dev_private; + struct ifreq ifr = { .ifr_flags = IFF_PROMISC }; + + dev->data->promiscuous = 1; + tap_ioctl(pmd, SIOCSIFFLAGS, &ifr, 1, LOCAL_AND_REMOTE); + if (pmd->remote_if_index && !pmd->flow_isolate) + tap_flow_implicit_create(pmd, TAP_REMOTE_PROMISC); +} + +static void +tap_promisc_disable(struct rte_eth_dev *dev) +{ + struct pmd_internals *pmd = dev->data->dev_private; + struct ifreq ifr = { .ifr_flags = IFF_PROMISC }; + + dev->data->promiscuous = 0; + tap_ioctl(pmd, SIOCSIFFLAGS, &ifr, 0, LOCAL_AND_REMOTE); + if (pmd->remote_if_index && !pmd->flow_isolate) + tap_flow_implicit_destroy(pmd, TAP_REMOTE_PROMISC); +} + +static void +tap_allmulti_enable(struct rte_eth_dev *dev) +{ + struct pmd_internals *pmd = dev->data->dev_private; + struct ifreq ifr = { .ifr_flags = IFF_ALLMULTI }; + + dev->data->all_multicast = 1; + tap_ioctl(pmd, SIOCSIFFLAGS, &ifr, 1, LOCAL_AND_REMOTE); + if (pmd->remote_if_index && !pmd->flow_isolate) + tap_flow_implicit_create(pmd, TAP_REMOTE_ALLMULTI); +} + +static void +tap_allmulti_disable(struct rte_eth_dev *dev) +{ + struct pmd_internals *pmd = dev->data->dev_private; + struct ifreq ifr = { .ifr_flags = IFF_ALLMULTI }; + + dev->data->all_multicast = 0; + tap_ioctl(pmd, SIOCSIFFLAGS, &ifr, 0, LOCAL_AND_REMOTE); + if (pmd->remote_if_index && !pmd->flow_isolate) + tap_flow_implicit_destroy(pmd, TAP_REMOTE_ALLMULTI); +} + +static void +tap_mac_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr) +{ + struct pmd_internals *pmd = dev->data->dev_private; + enum ioctl_mode mode = LOCAL_ONLY; + struct ifreq ifr; + + if (is_zero_ether_addr(mac_addr)) { + RTE_LOG(ERR, PMD, "%s: can't set an empty MAC address\n", + dev->device->name); + return; + } + /* Check the actual current MAC address on the tap netdevice */ + if (tap_ioctl(pmd, SIOCGIFHWADDR, &ifr, 0, LOCAL_ONLY) < 0) + return; + if (is_same_ether_addr((struct ether_addr *)&ifr.ifr_hwaddr.sa_data, + mac_addr)) + return; + /* Check the current MAC address on the remote */ + if (tap_ioctl(pmd, SIOCGIFHWADDR, &ifr, 0, REMOTE_ONLY) < 0) + return; + if (!is_same_ether_addr((struct ether_addr *)&ifr.ifr_hwaddr.sa_data, + mac_addr)) + mode = LOCAL_AND_REMOTE; + ifr.ifr_hwaddr.sa_family = AF_LOCAL; + rte_memcpy(ifr.ifr_hwaddr.sa_data, mac_addr, ETHER_ADDR_LEN); + if (tap_ioctl(pmd, SIOCSIFHWADDR, &ifr, 1, mode) < 0) + return; + rte_memcpy(&pmd->eth_addr, mac_addr, ETHER_ADDR_LEN); + if (pmd->remote_if_index && !pmd->flow_isolate) { + /* Replace MAC redirection rule after a MAC change */ + if (tap_flow_implicit_destroy(pmd, TAP_REMOTE_LOCAL_MAC) < 0) { + RTE_LOG(ERR, PMD, + "%s: Couldn't delete MAC redirection rule\n", + dev->device->name); + return; + } + if (tap_flow_implicit_create(pmd, TAP_REMOTE_LOCAL_MAC) < 0) + RTE_LOG(ERR, PMD, + "%s: Couldn't add MAC redirection rule\n", + dev->device->name); + } +} + +static int +tap_setup_queue(struct rte_eth_dev *dev, + struct pmd_internals *internals, + uint16_t qid, + int is_rx) +{ + int *fd; + int *other_fd; + const char *dir; + struct pmd_internals *pmd = dev->data->dev_private; + struct rx_queue *rx = &internals->rxq[qid]; + struct tx_queue *tx = &internals->txq[qid]; + + if (is_rx) { + fd = &rx->fd; + other_fd = &tx->fd; + dir = "rx"; + } else { + fd = &tx->fd; + other_fd = &rx->fd; + dir = "tx"; + } + if (*fd != -1) { + /* fd for this queue already exists */ + RTE_LOG(DEBUG, PMD, "%s: fd %d for %s queue qid %d exists\n", + pmd->name, *fd, dir, qid); + } else if (*other_fd != -1) { + /* Only other_fd exists. dup it */ + *fd = dup(*other_fd); + if (*fd < 0) { + *fd = -1; + RTE_LOG(ERR, PMD, "%s: dup() failed.\n", + pmd->name); + return -1; + } + RTE_LOG(DEBUG, PMD, "%s: dup fd %d for %s queue qid %d (%d)\n", + pmd->name, *other_fd, dir, qid, *fd); + } else { + /* Both RX and TX fds do not exist (equal -1). Create fd */ + *fd = tun_alloc(pmd); + if (*fd < 0) { + *fd = -1; /* restore original value */ + RTE_LOG(ERR, PMD, "%s: tun_alloc() failed.\n", + pmd->name); + return -1; + } + RTE_LOG(DEBUG, PMD, "%s: add %s queue for qid %d fd %d\n", + pmd->name, dir, qid, *fd); + } + + tx->mtu = &dev->data->mtu; + rx->rxmode = &dev->data->dev_conf.rxmode; + + return *fd; +} + +static int +tap_rx_queue_setup(struct rte_eth_dev *dev, + uint16_t rx_queue_id, + uint16_t nb_rx_desc, + unsigned int socket_id, + const struct rte_eth_rxconf *rx_conf __rte_unused, + struct rte_mempool *mp) +{ + struct pmd_internals *internals = dev->data->dev_private; + struct rx_queue *rxq = &internals->rxq[rx_queue_id]; + struct rte_mbuf **tmp = &rxq->pool; + long iov_max = sysconf(_SC_IOV_MAX); + uint16_t nb_desc = RTE_MIN(nb_rx_desc, iov_max - 1); + struct iovec (*iovecs)[nb_desc + 1]; + int data_off = RTE_PKTMBUF_HEADROOM; + int ret = 0; + int fd; + int i; + + if (rx_queue_id >= dev->data->nb_rx_queues || !mp) { + RTE_LOG(WARNING, PMD, + "nb_rx_queues %d too small or mempool NULL\n", + dev->data->nb_rx_queues); + return -1; + } + + rxq->mp = mp; + rxq->trigger_seen = 1; /* force initial burst */ + rxq->in_port = dev->data->port_id; + rxq->nb_rx_desc = nb_desc; + iovecs = rte_zmalloc_socket(dev->device->name, sizeof(*iovecs), 0, + socket_id); + if (!iovecs) { + RTE_LOG(WARNING, PMD, + "%s: Couldn't allocate %d RX descriptors\n", + dev->device->name, nb_desc); + return -ENOMEM; + } + rxq->iovecs = iovecs; + + dev->data->rx_queues[rx_queue_id] = rxq; + fd = tap_setup_queue(dev, internals, rx_queue_id, 1); + if (fd == -1) { + ret = fd; + goto error; + } + + (*rxq->iovecs)[0].iov_len = sizeof(struct tun_pi); + (*rxq->iovecs)[0].iov_base = &rxq->pi; + + for (i = 1; i <= nb_desc; i++) { + *tmp = rte_pktmbuf_alloc(rxq->mp); + if (!*tmp) { + RTE_LOG(WARNING, PMD, + "%s: couldn't allocate memory for queue %d\n", + dev->device->name, rx_queue_id); + ret = -ENOMEM; + goto error; + } + (*rxq->iovecs)[i].iov_len = (*tmp)->buf_len - data_off; + (*rxq->iovecs)[i].iov_base = + (char *)(*tmp)->buf_addr + data_off; + data_off = 0; + tmp = &(*tmp)->next; + } + + RTE_LOG(DEBUG, PMD, " RX TAP device name %s, qid %d on fd %d\n", + internals->name, rx_queue_id, internals->rxq[rx_queue_id].fd); + + return 0; + +error: + rte_pktmbuf_free(rxq->pool); + rxq->pool = NULL; + rte_free(rxq->iovecs); + rxq->iovecs = NULL; + return ret; +} + +static int +tap_tx_queue_setup(struct rte_eth_dev *dev, + uint16_t tx_queue_id, + uint16_t nb_tx_desc __rte_unused, + unsigned int socket_id __rte_unused, + const struct rte_eth_txconf *tx_conf __rte_unused) +{ + struct pmd_internals *internals = dev->data->dev_private; + int ret; + + if (tx_queue_id >= dev->data->nb_tx_queues) + return -1; + + dev->data->tx_queues[tx_queue_id] = &internals->txq[tx_queue_id]; + ret = tap_setup_queue(dev, internals, tx_queue_id, 0); + if (ret == -1) + return -1; + + RTE_LOG(DEBUG, PMD, " TX TAP device name %s, qid %d on fd %d\n", + internals->name, tx_queue_id, internals->txq[tx_queue_id].fd); + + return 0; +} + +static int +tap_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) +{ + struct pmd_internals *pmd = dev->data->dev_private; + struct ifreq ifr = { .ifr_mtu = mtu }; + int err = 0; + + err = tap_ioctl(pmd, SIOCSIFMTU, &ifr, 1, LOCAL_AND_REMOTE); + if (!err) + dev->data->mtu = mtu; + + return err; +} + +static int +tap_set_mc_addr_list(struct rte_eth_dev *dev __rte_unused, + struct ether_addr *mc_addr_set __rte_unused, + uint32_t nb_mc_addr __rte_unused) +{ + /* + * Nothing to do actually: the tap has no filtering whatsoever, every + * packet is received. + */ + return 0; +} + +static int +tap_nl_msg_handler(struct nlmsghdr *nh, void *arg) +{ + struct rte_eth_dev *dev = arg; + struct pmd_internals *pmd = dev->data->dev_private; + struct ifinfomsg *info = NLMSG_DATA(nh); + + if (nh->nlmsg_type != RTM_NEWLINK || + (info->ifi_index != pmd->if_index && + info->ifi_index != pmd->remote_if_index)) + return 0; + return tap_link_update(dev, 0); +} + +static void +tap_dev_intr_handler(void *cb_arg) +{ + struct rte_eth_dev *dev = cb_arg; + struct pmd_internals *pmd = dev->data->dev_private; + + nl_recv(pmd->intr_handle.fd, tap_nl_msg_handler, dev); +} + +static int +tap_intr_handle_set(struct rte_eth_dev *dev, int set) +{ + struct pmd_internals *pmd = dev->data->dev_private; + + /* In any case, disable interrupt if the conf is no longer there. */ + if (!dev->data->dev_conf.intr_conf.lsc) { + if (pmd->intr_handle.fd != -1) { + nl_final(pmd->intr_handle.fd); + rte_intr_callback_unregister(&pmd->intr_handle, + tap_dev_intr_handler, dev); + } + return 0; + } + if (set) { + pmd->intr_handle.fd = nl_init(RTMGRP_LINK); + if (unlikely(pmd->intr_handle.fd == -1)) + return -EBADF; + return rte_intr_callback_register( + &pmd->intr_handle, tap_dev_intr_handler, dev); + } + nl_final(pmd->intr_handle.fd); + return rte_intr_callback_unregister(&pmd->intr_handle, + tap_dev_intr_handler, dev); +} + +static const uint32_t* +tap_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused) +{ + static const uint32_t ptypes[] = { + RTE_PTYPE_INNER_L2_ETHER, + RTE_PTYPE_INNER_L2_ETHER_VLAN, + RTE_PTYPE_INNER_L2_ETHER_QINQ, + RTE_PTYPE_INNER_L3_IPV4, + RTE_PTYPE_INNER_L3_IPV4_EXT, + RTE_PTYPE_INNER_L3_IPV6, + RTE_PTYPE_INNER_L3_IPV6_EXT, + RTE_PTYPE_INNER_L4_FRAG, + RTE_PTYPE_INNER_L4_UDP, + RTE_PTYPE_INNER_L4_TCP, + RTE_PTYPE_INNER_L4_SCTP, + RTE_PTYPE_L2_ETHER, + RTE_PTYPE_L2_ETHER_VLAN, + RTE_PTYPE_L2_ETHER_QINQ, + RTE_PTYPE_L3_IPV4, + RTE_PTYPE_L3_IPV4_EXT, + RTE_PTYPE_L3_IPV6_EXT, + RTE_PTYPE_L3_IPV6, + RTE_PTYPE_L4_FRAG, + RTE_PTYPE_L4_UDP, + RTE_PTYPE_L4_TCP, + RTE_PTYPE_L4_SCTP, + }; + + return ptypes; +} + +static int +tap_flow_ctrl_get(struct rte_eth_dev *dev __rte_unused, + struct rte_eth_fc_conf *fc_conf) +{ + fc_conf->mode = RTE_FC_NONE; + return 0; +} + +static int +tap_flow_ctrl_set(struct rte_eth_dev *dev __rte_unused, + struct rte_eth_fc_conf *fc_conf) +{ + if (fc_conf->mode != RTE_FC_NONE) + return -ENOTSUP; + return 0; +} + +static const struct eth_dev_ops ops = { + .dev_start = tap_dev_start, + .dev_stop = tap_dev_stop, + .dev_close = tap_dev_close, + .dev_configure = tap_dev_configure, + .dev_infos_get = tap_dev_info, + .rx_queue_setup = tap_rx_queue_setup, + .tx_queue_setup = tap_tx_queue_setup, + .rx_queue_release = tap_rx_queue_release, + .tx_queue_release = tap_tx_queue_release, + .flow_ctrl_get = tap_flow_ctrl_get, + .flow_ctrl_set = tap_flow_ctrl_set, + .link_update = tap_link_update, + .dev_set_link_up = tap_link_set_up, + .dev_set_link_down = tap_link_set_down, + .promiscuous_enable = tap_promisc_enable, + .promiscuous_disable = tap_promisc_disable, + .allmulticast_enable = tap_allmulti_enable, + .allmulticast_disable = tap_allmulti_disable, + .mac_addr_set = tap_mac_set, + .mtu_set = tap_mtu_set, + .set_mc_addr_list = tap_set_mc_addr_list, + .stats_get = tap_stats_get, + .stats_reset = tap_stats_reset, + .dev_supported_ptypes_get = tap_dev_supported_ptypes_get, + .filter_ctrl = tap_dev_filter_ctrl, +}; + +static int +eth_dev_tap_create(struct rte_vdev_device *vdev, char *tap_name, + char *remote_iface, int fixed_mac_type) +{ + int numa_node = rte_socket_id(); + struct rte_eth_dev *dev; + struct pmd_internals *pmd; + struct rte_eth_dev_data *data; + struct ifreq ifr; + int i; + + RTE_LOG(DEBUG, PMD, " TAP device on numa %u\n", rte_socket_id()); + + data = rte_zmalloc_socket(tap_name, sizeof(*data), 0, numa_node); + if (!data) { + RTE_LOG(ERR, PMD, "TAP Failed to allocate data\n"); + goto error_exit_nodev; + } + + dev = rte_eth_vdev_allocate(vdev, sizeof(*pmd)); + if (!dev) { + RTE_LOG(ERR, PMD, "TAP Unable to allocate device struct\n"); + goto error_exit_nodev; + } + + pmd = dev->data->dev_private; + pmd->dev = dev; + snprintf(pmd->name, sizeof(pmd->name), "%s", tap_name); + + pmd->ioctl_sock = socket(AF_INET, SOCK_DGRAM, 0); + if (pmd->ioctl_sock == -1) { + RTE_LOG(ERR, PMD, + "TAP Unable to get a socket for management: %s\n", + strerror(errno)); + goto error_exit; + } + + /* Setup some default values */ + rte_memcpy(data, dev->data, sizeof(*data)); + data->dev_private = pmd; + data->dev_flags = RTE_ETH_DEV_INTR_LSC; + data->numa_node = numa_node; + + data->dev_link = pmd_link; + data->mac_addrs = &pmd->eth_addr; + /* Set the number of RX and TX queues */ + data->nb_rx_queues = 0; + data->nb_tx_queues = 0; + + dev->data = data; + dev->dev_ops = &ops; + dev->rx_pkt_burst = pmd_rx_burst; + dev->tx_pkt_burst = pmd_tx_burst; + + pmd->intr_handle.type = RTE_INTR_HANDLE_EXT; + pmd->intr_handle.fd = -1; + + /* Presetup the fds to -1 as being not valid */ + for (i = 0; i < RTE_PMD_TAP_MAX_QUEUES; i++) { + pmd->rxq[i].fd = -1; + pmd->txq[i].fd = -1; + } + + if (fixed_mac_type) { + /* fixed mac = 00:64:74:61:70: */ + static int iface_idx; + char mac[ETHER_ADDR_LEN] = "\0dtap"; + + mac[ETHER_ADDR_LEN - 1] = iface_idx++; + rte_memcpy(&pmd->eth_addr, mac, ETHER_ADDR_LEN); + } else { + eth_random_addr((uint8_t *)&pmd->eth_addr); + } + + /* Immediately create the netdevice (this will create the 1st queue). */ + /* rx queue */ + if (tap_setup_queue(dev, pmd, 0, 1) == -1) + goto error_exit; + /* tx queue */ + if (tap_setup_queue(dev, pmd, 0, 0) == -1) + goto error_exit; + + ifr.ifr_mtu = dev->data->mtu; + if (tap_ioctl(pmd, SIOCSIFMTU, &ifr, 1, LOCAL_AND_REMOTE) < 0) + goto error_exit; + + memset(&ifr, 0, sizeof(struct ifreq)); + ifr.ifr_hwaddr.sa_family = AF_LOCAL; + rte_memcpy(ifr.ifr_hwaddr.sa_data, &pmd->eth_addr, ETHER_ADDR_LEN); + if (tap_ioctl(pmd, SIOCSIFHWADDR, &ifr, 0, LOCAL_ONLY) < 0) + goto error_exit; + + /* + * Set up everything related to rte_flow: + * - netlink socket + * - tap / remote if_index + * - mandatory QDISCs + * - rte_flow actual/implicit lists + * - implicit rules + */ + pmd->nlsk_fd = nl_init(0); + if (pmd->nlsk_fd == -1) { + RTE_LOG(WARNING, PMD, "%s: failed to create netlink socket.\n", + pmd->name); + goto disable_rte_flow; + } + pmd->if_index = if_nametoindex(pmd->name); + if (!pmd->if_index) { + RTE_LOG(ERR, PMD, "%s: failed to get if_index.\n", pmd->name); + goto disable_rte_flow; + } + if (qdisc_create_multiq(pmd->nlsk_fd, pmd->if_index) < 0) { + RTE_LOG(ERR, PMD, "%s: failed to create multiq qdisc.\n", + pmd->name); + goto disable_rte_flow; + } + if (qdisc_create_ingress(pmd->nlsk_fd, pmd->if_index) < 0) { + RTE_LOG(ERR, PMD, "%s: failed to create ingress qdisc.\n", + pmd->name); + goto disable_rte_flow; + } + LIST_INIT(&pmd->flows); + + if (strlen(remote_iface)) { + pmd->remote_if_index = if_nametoindex(remote_iface); + if (!pmd->remote_if_index) { + RTE_LOG(ERR, PMD, "%s: failed to get %s if_index.\n", + pmd->name, remote_iface); + goto error_remote; + } + snprintf(pmd->remote_iface, RTE_ETH_NAME_MAX_LEN, + "%s", remote_iface); + + /* Save state of remote device */ + tap_ioctl(pmd, SIOCGIFFLAGS, &pmd->remote_initial_flags, 0, REMOTE_ONLY); + + /* Replicate remote MAC address */ + if (tap_ioctl(pmd, SIOCGIFHWADDR, &ifr, 0, REMOTE_ONLY) < 0) { + RTE_LOG(ERR, PMD, "%s: failed to get %s MAC address.\n", + pmd->name, pmd->remote_iface); + goto error_remote; + } + rte_memcpy(&pmd->eth_addr, ifr.ifr_hwaddr.sa_data, + ETHER_ADDR_LEN); + /* The desired MAC is already in ifreq after SIOCGIFHWADDR. */ + if (tap_ioctl(pmd, SIOCSIFHWADDR, &ifr, 0, LOCAL_ONLY) < 0) { + RTE_LOG(ERR, PMD, "%s: failed to get %s MAC address.\n", + pmd->name, remote_iface); + goto error_remote; + } + + /* + * Flush usually returns negative value because it tries to + * delete every QDISC (and on a running device, one QDISC at + * least is needed). Ignore negative return value. + */ + qdisc_flush(pmd->nlsk_fd, pmd->remote_if_index); + if (qdisc_create_ingress(pmd->nlsk_fd, + pmd->remote_if_index) < 0) { + RTE_LOG(ERR, PMD, "%s: failed to create ingress qdisc.\n", + pmd->remote_iface); + goto error_remote; + } + LIST_INIT(&pmd->implicit_flows); + if (tap_flow_implicit_create(pmd, TAP_REMOTE_TX) < 0 || + tap_flow_implicit_create(pmd, TAP_REMOTE_LOCAL_MAC) < 0 || + tap_flow_implicit_create(pmd, TAP_REMOTE_BROADCAST) < 0 || + tap_flow_implicit_create(pmd, TAP_REMOTE_BROADCASTV6) < 0) { + RTE_LOG(ERR, PMD, + "%s: failed to create implicit rules.\n", + pmd->name); + goto error_remote; + } + } + + return 0; + +disable_rte_flow: + RTE_LOG(ERR, PMD, " Disabling rte flow support: %s(%d)\n", + strerror(errno), errno); + if (strlen(remote_iface)) { + RTE_LOG(ERR, PMD, "Remote feature requires flow support.\n"); + goto error_exit; + } + return 0; + +error_remote: + RTE_LOG(ERR, PMD, " Can't set up remote feature: %s(%d)\n", + strerror(errno), errno); + tap_flow_implicit_flush(pmd, NULL); + +error_exit: + if (pmd->ioctl_sock > 0) + close(pmd->ioctl_sock); + rte_eth_dev_release_port(dev); + +error_exit_nodev: + RTE_LOG(ERR, PMD, "TAP Unable to initialize %s\n", + rte_vdev_device_name(vdev)); + + rte_free(data); + return -EINVAL; +} + +static int +set_interface_name(const char *key __rte_unused, + const char *value, + void *extra_args) +{ + char *name = (char *)extra_args; + + if (value) + snprintf(name, RTE_ETH_NAME_MAX_LEN - 1, "%s", value); + else + snprintf(name, RTE_ETH_NAME_MAX_LEN - 1, "%s%d", + DEFAULT_TAP_NAME, (tap_unit - 1)); + + return 0; +} + +static int +set_interface_speed(const char *key __rte_unused, + const char *value, + void *extra_args) +{ + *(int *)extra_args = (value) ? atoi(value) : ETH_SPEED_NUM_10G; + + return 0; +} + +static int +set_remote_iface(const char *key __rte_unused, + const char *value, + void *extra_args) +{ + char *name = (char *)extra_args; + + if (value) + snprintf(name, RTE_ETH_NAME_MAX_LEN, "%s", value); + + return 0; +} + +static int +set_mac_type(const char *key __rte_unused, + const char *value, + void *extra_args) +{ + if (value && + !strncasecmp(ETH_TAP_MAC_FIXED, value, strlen(ETH_TAP_MAC_FIXED))) + *(int *)extra_args = 1; + return 0; +} + +/* Open a TAP interface device. + */ +static int +rte_pmd_tap_probe(struct rte_vdev_device *dev) +{ + const char *name, *params; + int ret; + struct rte_kvargs *kvlist = NULL; + int speed; + char tap_name[RTE_ETH_NAME_MAX_LEN]; + char remote_iface[RTE_ETH_NAME_MAX_LEN]; + int fixed_mac_type = 0; + + name = rte_vdev_device_name(dev); + params = rte_vdev_device_args(dev); + + speed = ETH_SPEED_NUM_10G; + snprintf(tap_name, sizeof(tap_name), "%s%d", + DEFAULT_TAP_NAME, tap_unit++); + memset(remote_iface, 0, RTE_ETH_NAME_MAX_LEN); + + if (params && (params[0] != '\0')) { + RTE_LOG(DEBUG, PMD, "parameters (%s)\n", params); + + kvlist = rte_kvargs_parse(params, valid_arguments); + if (kvlist) { + if (rte_kvargs_count(kvlist, ETH_TAP_SPEED_ARG) == 1) { + ret = rte_kvargs_process(kvlist, + ETH_TAP_SPEED_ARG, + &set_interface_speed, + &speed); + if (ret == -1) + goto leave; + } + + if (rte_kvargs_count(kvlist, ETH_TAP_IFACE_ARG) == 1) { + ret = rte_kvargs_process(kvlist, + ETH_TAP_IFACE_ARG, + &set_interface_name, + tap_name); + if (ret == -1) + goto leave; + } + + if (rte_kvargs_count(kvlist, ETH_TAP_REMOTE_ARG) == 1) { + ret = rte_kvargs_process(kvlist, + ETH_TAP_REMOTE_ARG, + &set_remote_iface, + remote_iface); + if (ret == -1) + goto leave; + } + + if (rte_kvargs_count(kvlist, ETH_TAP_MAC_ARG) == 1) { + ret = rte_kvargs_process(kvlist, + ETH_TAP_MAC_ARG, + &set_mac_type, + &fixed_mac_type); + if (ret == -1) + goto leave; + } + } + } + pmd_link.link_speed = speed; + + RTE_LOG(NOTICE, PMD, "Initializing pmd_tap for %s as %s\n", + name, tap_name); + + ret = eth_dev_tap_create(dev, tap_name, remote_iface, fixed_mac_type); + +leave: + if (ret == -1) { + RTE_LOG(ERR, PMD, "Failed to create pmd for %s as %s\n", + name, tap_name); + tap_unit--; /* Restore the unit number */ + } + rte_kvargs_free(kvlist); + + return ret; +} + +/* detach a TAP device. + */ +static int +rte_pmd_tap_remove(struct rte_vdev_device *dev) +{ + struct rte_eth_dev *eth_dev = NULL; + struct pmd_internals *internals; + int i; + + RTE_LOG(DEBUG, PMD, "Closing TUN/TAP Ethernet device on numa %u\n", + rte_socket_id()); + + /* find the ethdev entry */ + eth_dev = rte_eth_dev_allocated(rte_vdev_device_name(dev)); + if (!eth_dev) + return 0; + + internals = eth_dev->data->dev_private; + if (internals->nlsk_fd) { + tap_flow_flush(eth_dev, NULL); + tap_flow_implicit_flush(internals, NULL); + nl_final(internals->nlsk_fd); + } + for (i = 0; i < RTE_PMD_TAP_MAX_QUEUES; i++) { + if (internals->rxq[i].fd != -1) { + close(internals->rxq[i].fd); + internals->rxq[i].fd = -1; + } + if (internals->txq[i].fd != -1) { + close(internals->txq[i].fd); + internals->txq[i].fd = -1; + } + } + + close(internals->ioctl_sock); + rte_free(eth_dev->data->dev_private); + rte_free(eth_dev->data); + + rte_eth_dev_release_port(eth_dev); + + return 0; +} + +static struct rte_vdev_driver pmd_tap_drv = { + .probe = rte_pmd_tap_probe, + .remove = rte_pmd_tap_remove, +}; +RTE_PMD_REGISTER_VDEV(net_tap, pmd_tap_drv); +RTE_PMD_REGISTER_ALIAS(net_tap, eth_tap); +RTE_PMD_REGISTER_PARAM_STRING(net_tap, + ETH_TAP_IFACE_ARG "= " + ETH_TAP_SPEED_ARG "= " + ETH_TAP_MAC_ARG "=" ETH_TAP_MAC_FIXED " " + ETH_TAP_REMOTE_ARG "="); diff --git a/dpdk/drivers/net/tap/rte_eth_tap.h b/dpdk/drivers/net/tap/rte_eth_tap.h new file mode 100644 index 00000000..829f32f3 --- /dev/null +++ b/dpdk/drivers/net/tap/rte_eth_tap.h @@ -0,0 +1,101 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of 6WIND S.A. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_ETH_TAP_H_ +#define _RTE_ETH_TAP_H_ + +#include +#include +#include +#include + +#include + +#include +#include + +#ifdef IFF_MULTI_QUEUE +#define RTE_PMD_TAP_MAX_QUEUES 16 +#else +#define RTE_PMD_TAP_MAX_QUEUES 1 +#endif + +struct pkt_stats { + uint64_t opackets; /* Number of output packets */ + uint64_t ipackets; /* Number of input packets */ + uint64_t obytes; /* Number of bytes on output */ + uint64_t ibytes; /* Number of bytes on input */ + uint64_t errs; /* Number of TX error packets */ + uint64_t ierrors; /* Number of RX error packets */ + uint64_t rx_nombuf; /* Nb of RX mbuf alloc failures */ +}; + +struct rx_queue { + struct rte_mempool *mp; /* Mempool for RX packets */ + uint32_t trigger_seen; /* Last seen Rx trigger value */ + uint16_t in_port; /* Port ID */ + int fd; + struct pkt_stats stats; /* Stats for this RX queue */ + uint16_t nb_rx_desc; /* max number of mbufs available */ + struct rte_eth_rxmode *rxmode; /* RX features */ + struct rte_mbuf *pool; /* mbufs pool for this queue */ + struct iovec (*iovecs)[]; /* descriptors for this queue */ + struct tun_pi pi; /* packet info for iovecs */ +}; + +struct tx_queue { + int fd; + uint16_t *mtu; /* Pointer to MTU from dev_data */ + struct pkt_stats stats; /* Stats for this TX queue */ +}; + +struct pmd_internals { + struct rte_eth_dev *dev; /* Ethernet device. */ + char remote_iface[RTE_ETH_NAME_MAX_LEN]; /* Remote netdevice name */ + char name[RTE_ETH_NAME_MAX_LEN]; /* Internal Tap device name */ + struct ether_addr eth_addr; /* Mac address of the device port */ + struct ifreq remote_initial_flags; /* Remote netdevice flags on init */ + int remote_if_index; /* remote netdevice IF_INDEX */ + int if_index; /* IF_INDEX for the port */ + int ioctl_sock; /* socket for ioctl calls */ + int nlsk_fd; /* Netlink socket fd */ + int flow_isolate; /* 1 if flow isolation is enabled */ + LIST_HEAD(tap_flows, rte_flow) flows; /* rte_flow rules */ + /* implicit rte_flow rules set when a remote device is active */ + LIST_HEAD(tap_implicit_flows, rte_flow) implicit_flows; + struct rx_queue rxq[RTE_PMD_TAP_MAX_QUEUES]; /* List of RX queues */ + struct tx_queue txq[RTE_PMD_TAP_MAX_QUEUES]; /* List of TX queues */ + struct rte_intr_handle intr_handle; /* LSC interrupt handle. */ +}; + +#endif /* _RTE_ETH_TAP_H_ */ diff --git a/dpdk/drivers/net/tap/rte_pmd_tap_version.map b/dpdk/drivers/net/tap/rte_pmd_tap_version.map new file mode 100644 index 00000000..31eca32e --- /dev/null +++ b/dpdk/drivers/net/tap/rte_pmd_tap_version.map @@ -0,0 +1,4 @@ +DPDK_17.02 { + + local: *; +}; diff --git a/dpdk/drivers/net/tap/tap_flow.c b/dpdk/drivers/net/tap/tap_flow.c new file mode 100644 index 00000000..ffc0b85b --- /dev/null +++ b/dpdk/drivers/net/tap/tap_flow.c @@ -0,0 +1,1668 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of 6WIND S.A. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#ifndef HAVE_TC_FLOWER +/* + * For kernels < 4.2, this enum is not defined. Runtime checks will be made to + * avoid sending TC messages the kernel cannot understand. + */ +enum { + TCA_FLOWER_UNSPEC, + TCA_FLOWER_CLASSID, + TCA_FLOWER_INDEV, + TCA_FLOWER_ACT, + TCA_FLOWER_KEY_ETH_DST, /* ETH_ALEN */ + TCA_FLOWER_KEY_ETH_DST_MASK, /* ETH_ALEN */ + TCA_FLOWER_KEY_ETH_SRC, /* ETH_ALEN */ + TCA_FLOWER_KEY_ETH_SRC_MASK, /* ETH_ALEN */ + TCA_FLOWER_KEY_ETH_TYPE, /* be16 */ + TCA_FLOWER_KEY_IP_PROTO, /* u8 */ + TCA_FLOWER_KEY_IPV4_SRC, /* be32 */ + TCA_FLOWER_KEY_IPV4_SRC_MASK, /* be32 */ + TCA_FLOWER_KEY_IPV4_DST, /* be32 */ + TCA_FLOWER_KEY_IPV4_DST_MASK, /* be32 */ + TCA_FLOWER_KEY_IPV6_SRC, /* struct in6_addr */ + TCA_FLOWER_KEY_IPV6_SRC_MASK, /* struct in6_addr */ + TCA_FLOWER_KEY_IPV6_DST, /* struct in6_addr */ + TCA_FLOWER_KEY_IPV6_DST_MASK, /* struct in6_addr */ + TCA_FLOWER_KEY_TCP_SRC, /* be16 */ + TCA_FLOWER_KEY_TCP_DST, /* be16 */ + TCA_FLOWER_KEY_UDP_SRC, /* be16 */ + TCA_FLOWER_KEY_UDP_DST, /* be16 */ +}; +#endif +#ifndef HAVE_TC_VLAN_ID +enum { + /* TCA_FLOWER_FLAGS, */ + TCA_FLOWER_KEY_VLAN_ID = TCA_FLOWER_KEY_UDP_DST + 2, /* be16 */ + TCA_FLOWER_KEY_VLAN_PRIO, /* u8 */ + TCA_FLOWER_KEY_VLAN_ETH_TYPE, /* be16 */ +}; +#endif + +#define ISOLATE_HANDLE 1 + +struct rte_flow { + LIST_ENTRY(rte_flow) next; /* Pointer to the next rte_flow structure */ + struct rte_flow *remote_flow; /* associated remote flow */ + struct nlmsg msg; +}; + +struct convert_data { + uint16_t eth_type; + uint16_t ip_proto; + uint8_t vlan; + struct rte_flow *flow; +}; + +struct remote_rule { + struct rte_flow_attr attr; + struct rte_flow_item items[2]; + struct rte_flow_action actions[2]; + int mirred; +}; + +static int tap_flow_create_eth(const struct rte_flow_item *item, void *data); +static int tap_flow_create_vlan(const struct rte_flow_item *item, void *data); +static int tap_flow_create_ipv4(const struct rte_flow_item *item, void *data); +static int tap_flow_create_ipv6(const struct rte_flow_item *item, void *data); +static int tap_flow_create_udp(const struct rte_flow_item *item, void *data); +static int tap_flow_create_tcp(const struct rte_flow_item *item, void *data); +static int +tap_flow_validate(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item items[], + const struct rte_flow_action actions[], + struct rte_flow_error *error); + +static struct rte_flow * +tap_flow_create(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item items[], + const struct rte_flow_action actions[], + struct rte_flow_error *error); + +static int +tap_flow_destroy(struct rte_eth_dev *dev, + struct rte_flow *flow, + struct rte_flow_error *error); + +static int +tap_flow_isolate(struct rte_eth_dev *dev, + int set, + struct rte_flow_error *error); + +static const struct rte_flow_ops tap_flow_ops = { + .validate = tap_flow_validate, + .create = tap_flow_create, + .destroy = tap_flow_destroy, + .flush = tap_flow_flush, + .isolate = tap_flow_isolate, +}; + +/* Static initializer for items. */ +#define ITEMS(...) \ + (const enum rte_flow_item_type []){ \ + __VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \ + } + +/* Structure to generate a simple graph of layers supported by the NIC. */ +struct tap_flow_items { + /* Bit-mask corresponding to what is supported for this item. */ + const void *mask; + const unsigned int mask_sz; /* Bit-mask size in bytes. */ + /* + * Bit-mask corresponding to the default mask, if none is provided + * along with the item. + */ + const void *default_mask; + /** + * Conversion function from rte_flow to netlink attributes. + * + * @param item + * rte_flow item to convert. + * @param data + * Internal structure to store the conversion. + * + * @return + * 0 on success, negative value otherwise. + */ + int (*convert)(const struct rte_flow_item *item, void *data); + /** List of possible following items. */ + const enum rte_flow_item_type *const items; +}; + +/* Graph of supported items and associated actions. */ +static const struct tap_flow_items tap_flow_items[] = { + [RTE_FLOW_ITEM_TYPE_END] = { + .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH), + }, + [RTE_FLOW_ITEM_TYPE_ETH] = { + .items = ITEMS( + RTE_FLOW_ITEM_TYPE_VLAN, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_IPV6), + .mask = &(const struct rte_flow_item_eth){ + .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", + .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", + .type = -1, + }, + .mask_sz = sizeof(struct rte_flow_item_eth), + .default_mask = &rte_flow_item_eth_mask, + .convert = tap_flow_create_eth, + }, + [RTE_FLOW_ITEM_TYPE_VLAN] = { + .items = ITEMS(RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_IPV6), + .mask = &(const struct rte_flow_item_vlan){ + .tpid = -1, + /* DEI matching is not supported */ +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + .tci = 0xffef, +#else + .tci = 0xefff, +#endif + }, + .mask_sz = sizeof(struct rte_flow_item_vlan), + .default_mask = &rte_flow_item_vlan_mask, + .convert = tap_flow_create_vlan, + }, + [RTE_FLOW_ITEM_TYPE_IPV4] = { + .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_TCP), + .mask = &(const struct rte_flow_item_ipv4){ + .hdr = { + .src_addr = -1, + .dst_addr = -1, + .next_proto_id = -1, + }, + }, + .mask_sz = sizeof(struct rte_flow_item_ipv4), + .default_mask = &rte_flow_item_ipv4_mask, + .convert = tap_flow_create_ipv4, + }, + [RTE_FLOW_ITEM_TYPE_IPV6] = { + .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_TCP), + .mask = &(const struct rte_flow_item_ipv6){ + .hdr = { + .src_addr = { + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff", + }, + .dst_addr = { + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff", + }, + .proto = -1, + }, + }, + .mask_sz = sizeof(struct rte_flow_item_ipv6), + .default_mask = &rte_flow_item_ipv6_mask, + .convert = tap_flow_create_ipv6, + }, + [RTE_FLOW_ITEM_TYPE_UDP] = { + .mask = &(const struct rte_flow_item_udp){ + .hdr = { + .src_port = -1, + .dst_port = -1, + }, + }, + .mask_sz = sizeof(struct rte_flow_item_udp), + .default_mask = &rte_flow_item_udp_mask, + .convert = tap_flow_create_udp, + }, + [RTE_FLOW_ITEM_TYPE_TCP] = { + .mask = &(const struct rte_flow_item_tcp){ + .hdr = { + .src_port = -1, + .dst_port = -1, + }, + }, + .mask_sz = sizeof(struct rte_flow_item_tcp), + .default_mask = &rte_flow_item_tcp_mask, + .convert = tap_flow_create_tcp, + }, +}; + +/* + * TC rules, by growing priority + * + * Remote netdevice Tap netdevice + * +-------------+-------------+ +-------------+-------------+ + * | Ingress | Egress | | Ingress | Egress | + * |-------------|-------------| |-------------|-------------| + * | | \ / | | | REMOTE TX | prio 1 + * | | \ / | | | \ / | prio 2 + * | EXPLICIT | \ / | | EXPLICIT | \ / | . + * | | \ / | | | \ / | . + * | RULES | X | | RULES | X | . + * | . | / \ | | . | / \ | . + * | . | / \ | | . | / \ | . + * | . | / \ | | . | / \ | . + * | . | / \ | | . | / \ | . + * + * .... .... .... .... + * + * | . | \ / | | . | \ / | . + * | . | \ / | | . | \ / | . + * | | \ / | | | \ / | + * | LOCAL_MAC | \ / | | \ / | \ / | last prio - 5 + * | PROMISC | X | | \ / | X | last prio - 4 + * | ALLMULTI | / \ | | X | / \ | last prio - 3 + * | BROADCAST | / \ | | / \ | / \ | last prio - 2 + * | BROADCASTV6 | / \ | | / \ | / \ | last prio - 1 + * | xx | / \ | | ISOLATE | / \ | last prio + * +-------------+-------------+ +-------------+-------------+ + * + * The implicit flow rules are stored in a list in with mandatorily the last two + * being the ISOLATE and REMOTE_TX rules. e.g.: + * + * LOCAL_MAC -> BROADCAST -> BROADCASTV6 -> REMOTE_TX -> ISOLATE -> NULL + * + * That enables tap_flow_isolate() to remove implicit rules by popping the list + * head and remove it as long as it applies on the remote netdevice. The + * implicit rule for TX redirection is not removed, as isolate concerns only + * incoming traffic. + */ + +static struct remote_rule implicit_rte_flows[TAP_REMOTE_MAX_IDX] = { + [TAP_REMOTE_LOCAL_MAC] = { + .attr = { + .group = MAX_GROUP, + .priority = PRIORITY_MASK - TAP_REMOTE_LOCAL_MAC, + .ingress = 1, + }, + .items[0] = { + .type = RTE_FLOW_ITEM_TYPE_ETH, + .mask = &(const struct rte_flow_item_eth){ + .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", + }, + }, + .items[1] = { + .type = RTE_FLOW_ITEM_TYPE_END, + }, + .mirred = TCA_EGRESS_REDIR, + }, + [TAP_REMOTE_BROADCAST] = { + .attr = { + .group = MAX_GROUP, + .priority = PRIORITY_MASK - TAP_REMOTE_BROADCAST, + .ingress = 1, + }, + .items[0] = { + .type = RTE_FLOW_ITEM_TYPE_ETH, + .mask = &(const struct rte_flow_item_eth){ + .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", + }, + .spec = &(const struct rte_flow_item_eth){ + .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", + }, + }, + .items[1] = { + .type = RTE_FLOW_ITEM_TYPE_END, + }, + .mirred = TCA_EGRESS_MIRROR, + }, + [TAP_REMOTE_BROADCASTV6] = { + .attr = { + .group = MAX_GROUP, + .priority = PRIORITY_MASK - TAP_REMOTE_BROADCASTV6, + .ingress = 1, + }, + .items[0] = { + .type = RTE_FLOW_ITEM_TYPE_ETH, + .mask = &(const struct rte_flow_item_eth){ + .dst.addr_bytes = "\x33\x33\x00\x00\x00\x00", + }, + .spec = &(const struct rte_flow_item_eth){ + .dst.addr_bytes = "\x33\x33\x00\x00\x00\x00", + }, + }, + .items[1] = { + .type = RTE_FLOW_ITEM_TYPE_END, + }, + .mirred = TCA_EGRESS_MIRROR, + }, + [TAP_REMOTE_PROMISC] = { + .attr = { + .group = MAX_GROUP, + .priority = PRIORITY_MASK - TAP_REMOTE_PROMISC, + .ingress = 1, + }, + .items[0] = { + .type = RTE_FLOW_ITEM_TYPE_VOID, + }, + .items[1] = { + .type = RTE_FLOW_ITEM_TYPE_END, + }, + .mirred = TCA_EGRESS_MIRROR, + }, + [TAP_REMOTE_ALLMULTI] = { + .attr = { + .group = MAX_GROUP, + .priority = PRIORITY_MASK - TAP_REMOTE_ALLMULTI, + .ingress = 1, + }, + .items[0] = { + .type = RTE_FLOW_ITEM_TYPE_ETH, + .mask = &(const struct rte_flow_item_eth){ + .dst.addr_bytes = "\x01\x00\x00\x00\x00\x00", + }, + .spec = &(const struct rte_flow_item_eth){ + .dst.addr_bytes = "\x01\x00\x00\x00\x00\x00", + }, + }, + .items[1] = { + .type = RTE_FLOW_ITEM_TYPE_END, + }, + .mirred = TCA_EGRESS_MIRROR, + }, + [TAP_REMOTE_TX] = { + .attr = { + .group = 0, + .priority = TAP_REMOTE_TX, + .egress = 1, + }, + .items[0] = { + .type = RTE_FLOW_ITEM_TYPE_VOID, + }, + .items[1] = { + .type = RTE_FLOW_ITEM_TYPE_END, + }, + .mirred = TCA_EGRESS_MIRROR, + }, + [TAP_ISOLATE] = { + .attr = { + .group = MAX_GROUP, + .priority = PRIORITY_MASK - TAP_ISOLATE, + .ingress = 1, + }, + .items[0] = { + .type = RTE_FLOW_ITEM_TYPE_VOID, + }, + .items[1] = { + .type = RTE_FLOW_ITEM_TYPE_END, + }, + }, +}; + +/** + * Make as much checks as possible on an Ethernet item, and if a flow is + * provided, fill it appropriately with Ethernet info. + * + * @param[in] item + * Item specification. + * @param[in, out] data + * Additional data structure to tell next layers we've been here. + * + * @return + * 0 if checks are alright, -1 otherwise. + */ +static int +tap_flow_create_eth(const struct rte_flow_item *item, void *data) +{ + struct convert_data *info = (struct convert_data *)data; + const struct rte_flow_item_eth *spec = item->spec; + const struct rte_flow_item_eth *mask = item->mask; + struct rte_flow *flow = info->flow; + struct nlmsg *msg; + + /* use default mask if none provided */ + if (!mask) + mask = tap_flow_items[RTE_FLOW_ITEM_TYPE_ETH].default_mask; + /* TC does not support eth_type masking. Only accept if exact match. */ + if (mask->type && mask->type != 0xffff) + return -1; + if (!spec) + return 0; + /* store eth_type for consistency if ipv4/6 pattern item comes next */ + if (spec->type & mask->type) + info->eth_type = spec->type; + if (!flow) + return 0; + msg = &flow->msg; + if (!is_zero_ether_addr(&spec->dst)) { + nlattr_add(&msg->nh, TCA_FLOWER_KEY_ETH_DST, ETHER_ADDR_LEN, + &spec->dst.addr_bytes); + nlattr_add(&msg->nh, + TCA_FLOWER_KEY_ETH_DST_MASK, ETHER_ADDR_LEN, + &mask->dst.addr_bytes); + } + if (!is_zero_ether_addr(&mask->src)) { + nlattr_add(&msg->nh, TCA_FLOWER_KEY_ETH_SRC, ETHER_ADDR_LEN, + &spec->src.addr_bytes); + nlattr_add(&msg->nh, + TCA_FLOWER_KEY_ETH_SRC_MASK, ETHER_ADDR_LEN, + &mask->src.addr_bytes); + } + return 0; +} + +/** + * Make as much checks as possible on a VLAN item, and if a flow is provided, + * fill it appropriately with VLAN info. + * + * @param[in] item + * Item specification. + * @param[in, out] data + * Additional data structure to tell next layers we've been here. + * + * @return + * 0 if checks are alright, -1 otherwise. + */ +static int +tap_flow_create_vlan(const struct rte_flow_item *item, void *data) +{ + struct convert_data *info = (struct convert_data *)data; + const struct rte_flow_item_vlan *spec = item->spec; + const struct rte_flow_item_vlan *mask = item->mask; + struct rte_flow *flow = info->flow; + struct nlmsg *msg; + + /* use default mask if none provided */ + if (!mask) + mask = tap_flow_items[RTE_FLOW_ITEM_TYPE_VLAN].default_mask; + /* TC does not support tpid masking. Only accept if exact match. */ + if (mask->tpid && mask->tpid != 0xffff) + return -1; + /* Double-tagging not supported. */ + if (spec && mask->tpid && spec->tpid != htons(ETH_P_8021Q)) + return -1; + info->vlan = 1; + if (!flow) + return 0; + msg = &flow->msg; + msg->t.tcm_info = TC_H_MAKE(msg->t.tcm_info, htons(ETH_P_8021Q)); +#define VLAN_PRIO(tci) ((tci) >> 13) +#define VLAN_ID(tci) ((tci) & 0xfff) + if (!spec) + return 0; + if (spec->tci) { + uint16_t tci = ntohs(spec->tci) & mask->tci; + uint16_t prio = VLAN_PRIO(tci); + uint8_t vid = VLAN_ID(tci); + + if (prio) + nlattr_add8(&msg->nh, TCA_FLOWER_KEY_VLAN_PRIO, prio); + if (vid) + nlattr_add16(&msg->nh, TCA_FLOWER_KEY_VLAN_ID, vid); + } + return 0; +} + +/** + * Make as much checks as possible on an IPv4 item, and if a flow is provided, + * fill it appropriately with IPv4 info. + * + * @param[in] item + * Item specification. + * @param[in, out] data + * Additional data structure to tell next layers we've been here. + * + * @return + * 0 if checks are alright, -1 otherwise. + */ +static int +tap_flow_create_ipv4(const struct rte_flow_item *item, void *data) +{ + struct convert_data *info = (struct convert_data *)data; + const struct rte_flow_item_ipv4 *spec = item->spec; + const struct rte_flow_item_ipv4 *mask = item->mask; + struct rte_flow *flow = info->flow; + struct nlmsg *msg; + + /* use default mask if none provided */ + if (!mask) + mask = tap_flow_items[RTE_FLOW_ITEM_TYPE_IPV4].default_mask; + /* check that previous eth type is compatible with ipv4 */ + if (info->eth_type && info->eth_type != htons(ETH_P_IP)) + return -1; + /* store ip_proto for consistency if udp/tcp pattern item comes next */ + if (spec) + info->ip_proto = spec->hdr.next_proto_id; + if (!flow) + return 0; + msg = &flow->msg; + if (!info->eth_type) + info->eth_type = htons(ETH_P_IP); + if (!spec) + return 0; + if (spec->hdr.dst_addr) { + nlattr_add32(&msg->nh, TCA_FLOWER_KEY_IPV4_DST, + spec->hdr.dst_addr); + nlattr_add32(&msg->nh, TCA_FLOWER_KEY_IPV4_DST_MASK, + mask->hdr.dst_addr); + } + if (spec->hdr.src_addr) { + nlattr_add32(&msg->nh, TCA_FLOWER_KEY_IPV4_SRC, + spec->hdr.src_addr); + nlattr_add32(&msg->nh, TCA_FLOWER_KEY_IPV4_SRC_MASK, + mask->hdr.src_addr); + } + if (spec->hdr.next_proto_id) + nlattr_add8(&msg->nh, TCA_FLOWER_KEY_IP_PROTO, + spec->hdr.next_proto_id); + return 0; +} + +/** + * Make as much checks as possible on an IPv6 item, and if a flow is provided, + * fill it appropriately with IPv6 info. + * + * @param[in] item + * Item specification. + * @param[in, out] data + * Additional data structure to tell next layers we've been here. + * + * @return + * 0 if checks are alright, -1 otherwise. + */ +static int +tap_flow_create_ipv6(const struct rte_flow_item *item, void *data) +{ + struct convert_data *info = (struct convert_data *)data; + const struct rte_flow_item_ipv6 *spec = item->spec; + const struct rte_flow_item_ipv6 *mask = item->mask; + struct rte_flow *flow = info->flow; + uint8_t empty_addr[16] = { 0 }; + struct nlmsg *msg; + + /* use default mask if none provided */ + if (!mask) + mask = tap_flow_items[RTE_FLOW_ITEM_TYPE_IPV6].default_mask; + /* check that previous eth type is compatible with ipv6 */ + if (info->eth_type && info->eth_type != htons(ETH_P_IPV6)) + return -1; + /* store ip_proto for consistency if udp/tcp pattern item comes next */ + if (spec) + info->ip_proto = spec->hdr.proto; + if (!flow) + return 0; + msg = &flow->msg; + if (!info->eth_type) + info->eth_type = htons(ETH_P_IPV6); + if (!spec) + return 0; + if (memcmp(spec->hdr.dst_addr, empty_addr, 16)) { + nlattr_add(&msg->nh, TCA_FLOWER_KEY_IPV6_DST, + sizeof(spec->hdr.dst_addr), &spec->hdr.dst_addr); + nlattr_add(&msg->nh, TCA_FLOWER_KEY_IPV6_DST_MASK, + sizeof(mask->hdr.dst_addr), &mask->hdr.dst_addr); + } + if (memcmp(spec->hdr.src_addr, empty_addr, 16)) { + nlattr_add(&msg->nh, TCA_FLOWER_KEY_IPV6_SRC, + sizeof(spec->hdr.src_addr), &spec->hdr.src_addr); + nlattr_add(&msg->nh, TCA_FLOWER_KEY_IPV6_SRC_MASK, + sizeof(mask->hdr.src_addr), &mask->hdr.src_addr); + } + if (spec->hdr.proto) + nlattr_add8(&msg->nh, TCA_FLOWER_KEY_IP_PROTO, spec->hdr.proto); + return 0; +} + +/** + * Make as much checks as possible on a UDP item, and if a flow is provided, + * fill it appropriately with UDP info. + * + * @param[in] item + * Item specification. + * @param[in, out] data + * Additional data structure to tell next layers we've been here. + * + * @return + * 0 if checks are alright, -1 otherwise. + */ +static int +tap_flow_create_udp(const struct rte_flow_item *item, void *data) +{ + struct convert_data *info = (struct convert_data *)data; + const struct rte_flow_item_udp *spec = item->spec; + const struct rte_flow_item_udp *mask = item->mask; + struct rte_flow *flow = info->flow; + struct nlmsg *msg; + + /* use default mask if none provided */ + if (!mask) + mask = tap_flow_items[RTE_FLOW_ITEM_TYPE_UDP].default_mask; + /* check that previous ip_proto is compatible with udp */ + if (info->ip_proto && info->ip_proto != IPPROTO_UDP) + return -1; + /* TC does not support UDP port masking. Only accept if exact match. */ + if ((mask->hdr.src_port && mask->hdr.src_port != 0xffff) || + (mask->hdr.dst_port && mask->hdr.dst_port != 0xffff)) + return -1; + if (!flow) + return 0; + msg = &flow->msg; + nlattr_add8(&msg->nh, TCA_FLOWER_KEY_IP_PROTO, IPPROTO_UDP); + if (!spec) + return 0; + if (spec->hdr.dst_port & mask->hdr.dst_port) + nlattr_add16(&msg->nh, TCA_FLOWER_KEY_UDP_DST, + spec->hdr.dst_port); + if (spec->hdr.src_port & mask->hdr.src_port) + nlattr_add16(&msg->nh, TCA_FLOWER_KEY_UDP_SRC, + spec->hdr.src_port); + return 0; +} + +/** + * Make as much checks as possible on a TCP item, and if a flow is provided, + * fill it appropriately with TCP info. + * + * @param[in] item + * Item specification. + * @param[in, out] data + * Additional data structure to tell next layers we've been here. + * + * @return + * 0 if checks are alright, -1 otherwise. + */ +static int +tap_flow_create_tcp(const struct rte_flow_item *item, void *data) +{ + struct convert_data *info = (struct convert_data *)data; + const struct rte_flow_item_tcp *spec = item->spec; + const struct rte_flow_item_tcp *mask = item->mask; + struct rte_flow *flow = info->flow; + struct nlmsg *msg; + + /* use default mask if none provided */ + if (!mask) + mask = tap_flow_items[RTE_FLOW_ITEM_TYPE_TCP].default_mask; + /* check that previous ip_proto is compatible with tcp */ + if (info->ip_proto && info->ip_proto != IPPROTO_TCP) + return -1; + /* TC does not support TCP port masking. Only accept if exact match. */ + if ((mask->hdr.src_port && mask->hdr.src_port != 0xffff) || + (mask->hdr.dst_port && mask->hdr.dst_port != 0xffff)) + return -1; + if (!flow) + return 0; + msg = &flow->msg; + nlattr_add8(&msg->nh, TCA_FLOWER_KEY_IP_PROTO, IPPROTO_TCP); + if (!spec) + return 0; + if (spec->hdr.dst_port & mask->hdr.dst_port) + nlattr_add16(&msg->nh, TCA_FLOWER_KEY_TCP_DST, + spec->hdr.dst_port); + if (spec->hdr.src_port & mask->hdr.src_port) + nlattr_add16(&msg->nh, TCA_FLOWER_KEY_TCP_SRC, + spec->hdr.src_port); + return 0; +} + +/** + * Check support for a given item. + * + * @param[in] item + * Item specification. + * @param size + * Bit-Mask size in bytes. + * @param[in] supported_mask + * Bit-mask covering supported fields to compare with spec, last and mask in + * \item. + * @param[in] default_mask + * Bit-mask default mask if none is provided in \item. + * + * @return + * 0 on success. + */ +static int +tap_flow_item_validate(const struct rte_flow_item *item, + unsigned int size, + const uint8_t *supported_mask, + const uint8_t *default_mask) +{ + int ret = 0; + + /* An empty layer is allowed, as long as all fields are NULL */ + if (!item->spec && (item->mask || item->last)) + return -1; + /* Is the item spec compatible with what the NIC supports? */ + if (item->spec && !item->mask) { + unsigned int i; + const uint8_t *spec = item->spec; + + for (i = 0; i < size; ++i) + if ((spec[i] | supported_mask[i]) != supported_mask[i]) + return -1; + /* Is the default mask compatible with what the NIC supports? */ + for (i = 0; i < size; i++) + if ((default_mask[i] | supported_mask[i]) != + supported_mask[i]) + return -1; + } + /* Is the item last compatible with what the NIC supports? */ + if (item->last && !item->mask) { + unsigned int i; + const uint8_t *spec = item->last; + + for (i = 0; i < size; ++i) + if ((spec[i] | supported_mask[i]) != supported_mask[i]) + return -1; + } + /* Is the item mask compatible with what the NIC supports? */ + if (item->mask) { + unsigned int i; + const uint8_t *spec = item->mask; + + for (i = 0; i < size; ++i) + if ((spec[i] | supported_mask[i]) != supported_mask[i]) + return -1; + } + /** + * Once masked, Are item spec and item last equal? + * TC does not support range so anything else is invalid. + */ + if (item->spec && item->last) { + uint8_t spec[size]; + uint8_t last[size]; + const uint8_t *apply = default_mask; + unsigned int i; + + if (item->mask) + apply = item->mask; + for (i = 0; i < size; ++i) { + spec[i] = ((const uint8_t *)item->spec)[i] & apply[i]; + last[i] = ((const uint8_t *)item->last)[i] & apply[i]; + } + ret = memcmp(spec, last, size); + } + return ret; +} + +/** + * Transform a DROP/PASSTHRU action item in the provided flow for TC. + * + * @param[in, out] flow + * Flow to be filled. + * @param[in] action + * Appropriate action to be set in the TCA_GACT_PARMS structure. + * + * @return + * 0 if checks are alright, -1 otherwise. + */ +static int +add_action_gact(struct rte_flow *flow, int action) +{ + struct nlmsg *msg = &flow->msg; + size_t act_index = 1; + struct tc_gact p = { + .action = action + }; + + if (nlattr_nested_start(msg, TCA_FLOWER_ACT) < 0) + return -1; + if (nlattr_nested_start(msg, act_index++) < 0) + return -1; + nlattr_add(&msg->nh, TCA_ACT_KIND, sizeof("gact"), "gact"); + if (nlattr_nested_start(msg, TCA_ACT_OPTIONS) < 0) + return -1; + nlattr_add(&msg->nh, TCA_GACT_PARMS, sizeof(p), &p); + nlattr_nested_finish(msg); /* nested TCA_ACT_OPTIONS */ + nlattr_nested_finish(msg); /* nested act_index */ + nlattr_nested_finish(msg); /* nested TCA_FLOWER_ACT */ + return 0; +} + +/** + * Transform a MIRRED action item in the provided flow for TC. + * + * @param[in, out] flow + * Flow to be filled. + * @param[in] ifindex + * Netdevice ifindex, where to mirror/redirect packet to. + * @param[in] action_type + * Either TCA_EGRESS_REDIR for redirection or TCA_EGRESS_MIRROR for mirroring. + * + * @return + * 0 if checks are alright, -1 otherwise. + */ +static int +add_action_mirred(struct rte_flow *flow, uint16_t ifindex, uint16_t action_type) +{ + struct nlmsg *msg = &flow->msg; + size_t act_index = 1; + struct tc_mirred p = { + .eaction = action_type, + .ifindex = ifindex, + }; + + if (nlattr_nested_start(msg, TCA_FLOWER_ACT) < 0) + return -1; + if (nlattr_nested_start(msg, act_index++) < 0) + return -1; + nlattr_add(&msg->nh, TCA_ACT_KIND, sizeof("mirred"), "mirred"); + if (nlattr_nested_start(msg, TCA_ACT_OPTIONS) < 0) + return -1; + if (action_type == TCA_EGRESS_MIRROR) + p.action = TC_ACT_PIPE; + else /* REDIRECT */ + p.action = TC_ACT_STOLEN; + nlattr_add(&msg->nh, TCA_MIRRED_PARMS, sizeof(p), &p); + nlattr_nested_finish(msg); /* nested TCA_ACT_OPTIONS */ + nlattr_nested_finish(msg); /* nested act_index */ + nlattr_nested_finish(msg); /* nested TCA_FLOWER_ACT */ + return 0; +} + +/** + * Transform a QUEUE action item in the provided flow for TC. + * + * @param[in, out] flow + * Flow to be filled. + * @param[in] queue + * Queue id to use. + * + * @return + * 0 if checks are alright, -1 otherwise. + */ +static int +add_action_skbedit(struct rte_flow *flow, uint16_t queue) +{ + struct nlmsg *msg = &flow->msg; + size_t act_index = 1; + struct tc_skbedit p = { + .action = TC_ACT_PIPE + }; + + if (nlattr_nested_start(msg, TCA_FLOWER_ACT) < 0) + return -1; + if (nlattr_nested_start(msg, act_index++) < 0) + return -1; + nlattr_add(&msg->nh, TCA_ACT_KIND, sizeof("skbedit"), "skbedit"); + if (nlattr_nested_start(msg, TCA_ACT_OPTIONS) < 0) + return -1; + nlattr_add(&msg->nh, TCA_SKBEDIT_PARMS, sizeof(p), &p); + nlattr_add16(&msg->nh, TCA_SKBEDIT_QUEUE_MAPPING, queue); + nlattr_nested_finish(msg); /* nested TCA_ACT_OPTIONS */ + nlattr_nested_finish(msg); /* nested act_index */ + nlattr_nested_finish(msg); /* nested TCA_FLOWER_ACT */ + return 0; +} + +/** + * Validate a flow supported by TC. + * If flow param is not NULL, then also fill the netlink message inside. + * + * @param pmd + * Pointer to private structure. + * @param[in] attr + * Flow rule attributes. + * @param[in] pattern + * Pattern specification (list terminated by the END pattern item). + * @param[in] actions + * Associated actions (list terminated by the END action). + * @param[out] error + * Perform verbose error reporting if not NULL. + * @param[in, out] flow + * Flow structure to update. + * @param[in] mirred + * If set to TCA_EGRESS_REDIR, provided actions will be replaced with a + * redirection to the tap netdevice, and the TC rule will be configured + * on the remote netdevice in pmd. + * If set to TCA_EGRESS_MIRROR, provided actions will be replaced with a + * mirroring to the tap netdevice, and the TC rule will be configured + * on the remote netdevice in pmd. Matching packets will thus be duplicated. + * If set to 0, the standard behavior is to be used: set correct actions for + * the TC rule, and apply it on the tap netdevice. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +priv_flow_process(struct pmd_internals *pmd, + const struct rte_flow_attr *attr, + const struct rte_flow_item items[], + const struct rte_flow_action actions[], + struct rte_flow_error *error, + struct rte_flow *flow, + int mirred) +{ + const struct tap_flow_items *cur_item = tap_flow_items; + struct convert_data data = { + .eth_type = 0, + .ip_proto = 0, + .flow = flow, + }; + int action = 0; /* Only one action authorized for now */ + + if (attr->group > MAX_GROUP) { + rte_flow_error_set( + error, EINVAL, RTE_FLOW_ERROR_TYPE_ATTR_GROUP, + NULL, "group value too big: cannot exceed 15"); + return -rte_errno; + } + if (attr->priority > MAX_PRIORITY) { + rte_flow_error_set( + error, EINVAL, RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, + NULL, "priority value too big"); + return -rte_errno; + } else if (flow) { + uint16_t group = attr->group << GROUP_SHIFT; + uint16_t prio = group | (attr->priority + PRIORITY_OFFSET); + flow->msg.t.tcm_info = TC_H_MAKE(prio << 16, + flow->msg.t.tcm_info); + } + if (flow) { + if (mirred) { + /* + * If attr->ingress, the rule applies on remote ingress + * to match incoming packets + * If attr->egress, the rule applies on tap ingress (as + * seen from the kernel) to deal with packets going out + * from the DPDK app. + */ + flow->msg.t.tcm_parent = TC_H_MAKE(TC_H_INGRESS, 0); + } else { + /* Standard rule on tap egress (kernel standpoint). */ + flow->msg.t.tcm_parent = + TC_H_MAKE(MULTIQ_MAJOR_HANDLE, 0); + } + /* use flower filter type */ + nlattr_add(&flow->msg.nh, TCA_KIND, sizeof("flower"), "flower"); + if (nlattr_nested_start(&flow->msg, TCA_OPTIONS) < 0) + goto exit_item_not_supported; + } + for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) { + const struct tap_flow_items *token = NULL; + unsigned int i; + int err = 0; + + if (items->type == RTE_FLOW_ITEM_TYPE_VOID) + continue; + for (i = 0; + cur_item->items && + cur_item->items[i] != RTE_FLOW_ITEM_TYPE_END; + ++i) { + if (cur_item->items[i] == items->type) { + token = &tap_flow_items[items->type]; + break; + } + } + if (!token) + goto exit_item_not_supported; + cur_item = token; + err = tap_flow_item_validate( + items, cur_item->mask_sz, + (const uint8_t *)cur_item->mask, + (const uint8_t *)cur_item->default_mask); + if (err) + goto exit_item_not_supported; + if (flow && cur_item->convert) { + err = cur_item->convert(items, &data); + if (err) + goto exit_item_not_supported; + } + } + if (flow) { + if (data.vlan) { + nlattr_add16(&flow->msg.nh, TCA_FLOWER_KEY_ETH_TYPE, + htons(ETH_P_8021Q)); + nlattr_add16(&flow->msg.nh, + TCA_FLOWER_KEY_VLAN_ETH_TYPE, + data.eth_type ? + data.eth_type : htons(ETH_P_ALL)); + } else if (data.eth_type) { + nlattr_add16(&flow->msg.nh, TCA_FLOWER_KEY_ETH_TYPE, + data.eth_type); + } + } + if (mirred && flow) { + uint16_t if_index = pmd->if_index; + + /* + * If attr->egress && mirred, then this is a special + * case where the rule must be applied on the tap, to + * redirect packets coming from the DPDK App, out + * through the remote netdevice. + */ + if (attr->egress) + if_index = pmd->remote_if_index; + if (add_action_mirred(flow, if_index, mirred) < 0) + goto exit_action_not_supported; + else + goto end; + } + for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) { + int err = 0; + + if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) { + continue; + } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) { + if (action) + goto exit_action_not_supported; + action = 1; + if (flow) + err = add_action_gact(flow, TC_ACT_SHOT); + } else if (actions->type == RTE_FLOW_ACTION_TYPE_PASSTHRU) { + if (action) + goto exit_action_not_supported; + action = 1; + if (flow) + err = add_action_gact(flow, TC_ACT_UNSPEC); + } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) { + const struct rte_flow_action_queue *queue = + (const struct rte_flow_action_queue *) + actions->conf; + + if (action) + goto exit_action_not_supported; + action = 1; + if (!queue || + (queue->index > pmd->dev->data->nb_rx_queues - 1)) + goto exit_action_not_supported; + if (flow) + err = add_action_skbedit(flow, queue->index); + } else if (actions->type == RTE_FLOW_ACTION_TYPE_RSS) { + /* Fake RSS support. */ + const struct rte_flow_action_rss *rss = + (const struct rte_flow_action_rss *) + actions->conf; + + if (action) + goto exit_action_not_supported; + action = 1; + if (!rss || rss->num < 1 || + (rss->queue[0] > pmd->dev->data->nb_rx_queues - 1)) + goto exit_action_not_supported; + if (flow) + err = add_action_skbedit(flow, rss->queue[0]); + } else { + goto exit_action_not_supported; + } + if (err) + goto exit_action_not_supported; + } +end: + if (flow) + nlattr_nested_finish(&flow->msg); /* nested TCA_OPTIONS */ + return 0; +exit_item_not_supported: + rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, + items, "item not supported"); + return -rte_errno; +exit_action_not_supported: + rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, + actions, "action not supported"); + return -rte_errno; +} + + + +/** + * Validate a flow. + * + * @see rte_flow_validate() + * @see rte_flow_ops + */ +static int +tap_flow_validate(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item items[], + const struct rte_flow_action actions[], + struct rte_flow_error *error) +{ + struct pmd_internals *pmd = dev->data->dev_private; + + return priv_flow_process(pmd, attr, items, actions, error, NULL, 0); +} + +/** + * Set a unique handle in a flow. + * + * The kernel supports TC rules with equal priority, as long as they use the + * same matching fields (e.g.: dst mac and ipv4) with different values (and + * full mask to ensure no collision is possible). + * In those rules, the handle (uint32_t) is the part that would identify + * specifically each rule. + * + * On 32-bit architectures, the handle can simply be the flow's pointer address. + * On 64-bit architectures, we rely on jhash(flow) to find a (sufficiently) + * unique handle. + * + * @param[in, out] flow + * The flow that needs its handle set. + */ +static void +tap_flow_set_handle(struct rte_flow *flow) +{ + uint32_t handle = 0; + + if (sizeof(flow) > 4) + handle = rte_jhash(&flow, sizeof(flow), 1); + else + handle = (uintptr_t)flow; + /* must be at least 1 to avoid letting the kernel choose one for us */ + if (!handle) + handle = 1; + flow->msg.t.tcm_handle = handle; +} + +/** + * Create a flow. + * + * @see rte_flow_create() + * @see rte_flow_ops + */ +static struct rte_flow * +tap_flow_create(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item items[], + const struct rte_flow_action actions[], + struct rte_flow_error *error) +{ + struct pmd_internals *pmd = dev->data->dev_private; + struct rte_flow *remote_flow = NULL; + struct rte_flow *flow = NULL; + struct nlmsg *msg = NULL; + int err; + + if (!pmd->if_index) { + rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE, + NULL, + "can't create rule, ifindex not found"); + goto fail; + } + /* + * No rules configured through standard rte_flow should be set on the + * priorities used by implicit rules. + */ + if ((attr->group == MAX_GROUP) && + attr->priority > (MAX_PRIORITY - TAP_REMOTE_MAX_IDX)) { + rte_flow_error_set( + error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, + NULL, "priority value too big"); + goto fail; + } + flow = rte_malloc(__func__, sizeof(struct rte_flow), 0); + if (!flow) { + rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, + NULL, "cannot allocate memory for rte_flow"); + goto fail; + } + msg = &flow->msg; + tc_init_msg(msg, pmd->if_index, RTM_NEWTFILTER, + NLM_F_REQUEST | NLM_F_ACK | NLM_F_EXCL | NLM_F_CREATE); + msg->t.tcm_info = TC_H_MAKE(0, htons(ETH_P_ALL)); + tap_flow_set_handle(flow); + if (priv_flow_process(pmd, attr, items, actions, error, flow, 0)) + goto fail; + err = nl_send(pmd->nlsk_fd, &msg->nh); + if (err < 0) { + rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE, + NULL, "couldn't send request to kernel"); + goto fail; + } + err = nl_recv_ack(pmd->nlsk_fd); + if (err < 0) { + RTE_LOG(ERR, PMD, + "Kernel refused TC filter rule creation (%d): %s\n", + errno, strerror(errno)); + rte_flow_error_set(error, EEXIST, RTE_FLOW_ERROR_TYPE_HANDLE, + NULL, + "overlapping rules or Kernel too old for flower support"); + goto fail; + } + LIST_INSERT_HEAD(&pmd->flows, flow, next); + /** + * If a remote device is configured, a TC rule with identical items for + * matching must be set on that device, with a single action: redirect + * to the local pmd->if_index. + */ + if (pmd->remote_if_index) { + remote_flow = rte_malloc(__func__, sizeof(struct rte_flow), 0); + if (!remote_flow) { + rte_flow_error_set( + error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, NULL, + "cannot allocate memory for rte_flow"); + goto fail; + } + msg = &remote_flow->msg; + /* set the rule if_index for the remote netdevice */ + tc_init_msg( + msg, pmd->remote_if_index, RTM_NEWTFILTER, + NLM_F_REQUEST | NLM_F_ACK | NLM_F_EXCL | NLM_F_CREATE); + msg->t.tcm_info = TC_H_MAKE(0, htons(ETH_P_ALL)); + tap_flow_set_handle(remote_flow); + if (priv_flow_process(pmd, attr, items, NULL, + error, remote_flow, TCA_EGRESS_REDIR)) { + rte_flow_error_set( + error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, + NULL, "rte flow rule validation failed"); + goto fail; + } + err = nl_send(pmd->nlsk_fd, &msg->nh); + if (err < 0) { + rte_flow_error_set( + error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, + NULL, "Failure sending nl request"); + goto fail; + } + err = nl_recv_ack(pmd->nlsk_fd); + if (err < 0) { + RTE_LOG(ERR, PMD, + "Kernel refused TC filter rule creation (%d): %s\n", + errno, strerror(errno)); + rte_flow_error_set( + error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, + NULL, + "overlapping rules or Kernel too old for flower support"); + goto fail; + } + flow->remote_flow = remote_flow; + } + return flow; +fail: + if (remote_flow) + rte_free(remote_flow); + if (flow) + rte_free(flow); + return NULL; +} + +/** + * Destroy a flow using pointer to pmd_internal. + * + * @param[in, out] pmd + * Pointer to private structure. + * @param[in] flow + * Pointer to the flow to destroy. + * @param[in, out] error + * Pointer to the flow error handler + * + * @return 0 if the flow could be destroyed, -1 otherwise. + */ +static int +tap_flow_destroy_pmd(struct pmd_internals *pmd, + struct rte_flow *flow, + struct rte_flow_error *error) +{ + struct rte_flow *remote_flow = flow->remote_flow; + int ret = 0; + + LIST_REMOVE(flow, next); + flow->msg.nh.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK; + flow->msg.nh.nlmsg_type = RTM_DELTFILTER; + + ret = nl_send(pmd->nlsk_fd, &flow->msg.nh); + if (ret < 0) { + rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE, + NULL, "couldn't send request to kernel"); + goto end; + } + ret = nl_recv_ack(pmd->nlsk_fd); + /* If errno is ENOENT, the rule is already no longer in the kernel. */ + if (ret < 0 && errno == ENOENT) + ret = 0; + if (ret < 0) { + RTE_LOG(ERR, PMD, + "Kernel refused TC filter rule deletion (%d): %s\n", + errno, strerror(errno)); + rte_flow_error_set( + error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE, NULL, + "couldn't receive kernel ack to our request"); + goto end; + } + if (remote_flow) { + remote_flow->msg.nh.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK; + remote_flow->msg.nh.nlmsg_type = RTM_DELTFILTER; + + ret = nl_send(pmd->nlsk_fd, &remote_flow->msg.nh); + if (ret < 0) { + rte_flow_error_set( + error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, + NULL, "Failure sending nl request"); + goto end; + } + ret = nl_recv_ack(pmd->nlsk_fd); + if (ret < 0 && errno == ENOENT) + ret = 0; + if (ret < 0) { + RTE_LOG(ERR, PMD, + "Kernel refused TC filter rule deletion (%d): %s\n", + errno, strerror(errno)); + rte_flow_error_set( + error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, + NULL, "Failure trying to receive nl ack"); + goto end; + } + } +end: + if (remote_flow) + rte_free(remote_flow); + rte_free(flow); + return ret; +} + +/** + * Destroy a flow. + * + * @see rte_flow_destroy() + * @see rte_flow_ops + */ +static int +tap_flow_destroy(struct rte_eth_dev *dev, + struct rte_flow *flow, + struct rte_flow_error *error) +{ + struct pmd_internals *pmd = dev->data->dev_private; + + return tap_flow_destroy_pmd(pmd, flow, error); +} + +/** + * Enable/disable flow isolation. + * + * @see rte_flow_isolate() + * @see rte_flow_ops + */ +static int +tap_flow_isolate(struct rte_eth_dev *dev, + int set, + struct rte_flow_error *error __rte_unused) +{ + struct pmd_internals *pmd = dev->data->dev_private; + + if (set) + pmd->flow_isolate = 1; + else + pmd->flow_isolate = 0; + /* + * If netdevice is there, setup appropriate flow rules immediately. + * Otherwise it will be set when bringing up the netdevice (tun_alloc). + */ + if (!pmd->rxq[0].fd) + return 0; + if (set) { + struct rte_flow *flow; + + while (1) { + flow = LIST_FIRST(&pmd->implicit_flows); + if (!flow) + break; + /* + * Remove all implicit rules on the remote. + * Keep the local rule to redirect packets on TX. + * Keep also the last implicit local rule: ISOLATE. + */ + if (flow->msg.t.tcm_ifindex == pmd->if_index) + break; + if (tap_flow_destroy_pmd(pmd, flow, NULL) < 0) + goto error; + } + /* Switch the TC rule according to pmd->flow_isolate */ + if (tap_flow_implicit_create(pmd, TAP_ISOLATE) == -1) + goto error; + } else { + /* Switch the TC rule according to pmd->flow_isolate */ + if (tap_flow_implicit_create(pmd, TAP_ISOLATE) == -1) + goto error; + if (!pmd->remote_if_index) + return 0; + if (tap_flow_implicit_create(pmd, TAP_REMOTE_TX) < 0) + goto error; + if (tap_flow_implicit_create(pmd, TAP_REMOTE_LOCAL_MAC) < 0) + goto error; + if (tap_flow_implicit_create(pmd, TAP_REMOTE_BROADCAST) < 0) + goto error; + if (tap_flow_implicit_create(pmd, TAP_REMOTE_BROADCASTV6) < 0) + goto error; + if (dev->data->promiscuous && + tap_flow_implicit_create(pmd, TAP_REMOTE_PROMISC) < 0) + goto error; + if (dev->data->all_multicast && + tap_flow_implicit_create(pmd, TAP_REMOTE_ALLMULTI) < 0) + goto error; + } + return 0; +error: + pmd->flow_isolate = 0; + return rte_flow_error_set( + error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "TC rule creation failed"); +} + +/** + * Destroy all flows. + * + * @see rte_flow_flush() + * @see rte_flow_ops + */ +int +tap_flow_flush(struct rte_eth_dev *dev, struct rte_flow_error *error) +{ + struct pmd_internals *pmd = dev->data->dev_private; + struct rte_flow *flow; + + while (!LIST_EMPTY(&pmd->flows)) { + flow = LIST_FIRST(&pmd->flows); + if (tap_flow_destroy(dev, flow, error) < 0) + return -1; + } + return 0; +} + +/** + * Add an implicit flow rule on the remote device to make sure traffic gets to + * the tap netdevice from there. + * + * @param pmd + * Pointer to private structure. + * @param[in] idx + * The idx in the implicit_rte_flows array specifying which rule to apply. + * + * @return -1 if the rule couldn't be applied, 0 otherwise. + */ +int tap_flow_implicit_create(struct pmd_internals *pmd, + enum implicit_rule_index idx) +{ + uint16_t flags = NLM_F_REQUEST | NLM_F_ACK | NLM_F_EXCL | NLM_F_CREATE; + struct rte_flow_action *actions = implicit_rte_flows[idx].actions; + struct rte_flow_action isolate_actions[2] = { + [1] = { + .type = RTE_FLOW_ACTION_TYPE_END, + }, + }; + struct rte_flow_item *items = implicit_rte_flows[idx].items; + struct rte_flow_attr *attr = &implicit_rte_flows[idx].attr; + struct rte_flow_item_eth eth_local = { .type = 0 }; + uint16_t if_index = pmd->remote_if_index; + struct rte_flow *remote_flow = NULL; + struct nlmsg *msg = NULL; + int err = 0; + struct rte_flow_item items_local[2] = { + [0] = { + .type = items[0].type, + .spec = ð_local, + .mask = items[0].mask, + }, + [1] = { + .type = items[1].type, + } + }; + + remote_flow = rte_malloc(__func__, sizeof(struct rte_flow), 0); + if (!remote_flow) { + RTE_LOG(ERR, PMD, "Cannot allocate memory for rte_flow\n"); + goto fail; + } + msg = &remote_flow->msg; + if (idx == TAP_REMOTE_TX) { + if_index = pmd->if_index; + } else if (idx == TAP_ISOLATE) { + if_index = pmd->if_index; + /* Don't be exclusive for this rule, it can be changed later. */ + flags = NLM_F_REQUEST | NLM_F_ACK | NLM_F_CREATE; + isolate_actions[0].type = pmd->flow_isolate ? + RTE_FLOW_ACTION_TYPE_DROP : + RTE_FLOW_ACTION_TYPE_PASSTHRU; + actions = isolate_actions; + } else if (idx == TAP_REMOTE_LOCAL_MAC) { + /* + * eth addr couldn't be set in implicit_rte_flows[] as it is not + * known at compile time. + */ + memcpy(ð_local.dst, &pmd->eth_addr, sizeof(pmd->eth_addr)); + items = items_local; + } + tc_init_msg(msg, if_index, RTM_NEWTFILTER, flags); + msg->t.tcm_info = TC_H_MAKE(0, htons(ETH_P_ALL)); + /* + * The ISOLATE rule is always present and must have a static handle, as + * the action is changed whether the feature is enabled (DROP) or + * disabled (PASSTHRU). + */ + if (idx == TAP_ISOLATE) + remote_flow->msg.t.tcm_handle = ISOLATE_HANDLE; + else + tap_flow_set_handle(remote_flow); + if (priv_flow_process(pmd, attr, items, actions, NULL, + remote_flow, implicit_rte_flows[idx].mirred)) { + RTE_LOG(ERR, PMD, "rte flow rule validation failed\n"); + goto fail; + } + err = nl_send(pmd->nlsk_fd, &msg->nh); + if (err < 0) { + RTE_LOG(ERR, PMD, "Failure sending nl request\n"); + goto fail; + } + err = nl_recv_ack(pmd->nlsk_fd); + if (err < 0) { + RTE_LOG(ERR, PMD, + "Kernel refused TC filter rule creation (%d): %s\n", + errno, strerror(errno)); + goto fail; + } + LIST_INSERT_HEAD(&pmd->implicit_flows, remote_flow, next); + return 0; +fail: + if (remote_flow) + rte_free(remote_flow); + return -1; +} + +/** + * Remove specific implicit flow rule on the remote device. + * + * @param[in, out] pmd + * Pointer to private structure. + * @param[in] idx + * The idx in the implicit_rte_flows array specifying which rule to remove. + * + * @return -1 if one of the implicit rules couldn't be created, 0 otherwise. + */ +int tap_flow_implicit_destroy(struct pmd_internals *pmd, + enum implicit_rule_index idx) +{ + struct rte_flow *remote_flow; + int cur_prio = -1; + int idx_prio = implicit_rte_flows[idx].attr.priority + PRIORITY_OFFSET; + + for (remote_flow = LIST_FIRST(&pmd->implicit_flows); + remote_flow; + remote_flow = LIST_NEXT(remote_flow, next)) { + cur_prio = (remote_flow->msg.t.tcm_info >> 16) & PRIORITY_MASK; + if (cur_prio != idx_prio) + continue; + return tap_flow_destroy_pmd(pmd, remote_flow, NULL); + } + return 0; +} + +/** + * Destroy all implicit flows. + * + * @see rte_flow_flush() + */ +int +tap_flow_implicit_flush(struct pmd_internals *pmd, struct rte_flow_error *error) +{ + struct rte_flow *remote_flow; + + while (!LIST_EMPTY(&pmd->implicit_flows)) { + remote_flow = LIST_FIRST(&pmd->implicit_flows); + if (tap_flow_destroy_pmd(pmd, remote_flow, error) < 0) + return -1; + } + return 0; +} + +/** + * Manage filter operations. + * + * @param dev + * Pointer to Ethernet device structure. + * @param filter_type + * Filter type. + * @param filter_op + * Operation to perform. + * @param arg + * Pointer to operation-specific structure. + * + * @return + * 0 on success, negative errno value on failure. + */ +int +tap_dev_filter_ctrl(struct rte_eth_dev *dev, + enum rte_filter_type filter_type, + enum rte_filter_op filter_op, + void *arg) +{ + switch (filter_type) { + case RTE_ETH_FILTER_GENERIC: + if (filter_op != RTE_ETH_FILTER_GET) + return -EINVAL; + *(const void **)arg = &tap_flow_ops; + return 0; + default: + RTE_LOG(ERR, PMD, "%p: filter type (%d) not supported\n", + (void *)dev, filter_type); + } + return -EINVAL; +} + diff --git a/dpdk/drivers/net/tap/tap_flow.h b/dpdk/drivers/net/tap/tap_flow.h new file mode 100644 index 00000000..9e332b03 --- /dev/null +++ b/dpdk/drivers/net/tap/tap_flow.h @@ -0,0 +1,83 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of 6WIND S.A. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _TAP_FLOW_H_ +#define _TAP_FLOW_H_ + +#include +#include +#include + +/** + * In TC, priority 0 means we require the kernel to allocate one for us. + * In rte_flow, however, we want the priority 0 to be the most important one. + * Use an offset to have the most important priority being 1 in TC. + */ +#define PRIORITY_OFFSET 1 +#define PRIORITY_MASK (0xfff) +#define MAX_PRIORITY (PRIORITY_MASK - PRIORITY_OFFSET) +#define GROUP_MASK (0xf) +#define GROUP_SHIFT 12 +#define MAX_GROUP GROUP_MASK + +/** + * These index are actually in reversed order: their priority is processed + * by subtracting their value to the lowest priority (PRIORITY_MASK). + * Thus the first one will have the lowest priority in the end + * (but biggest value). + */ +enum implicit_rule_index { + TAP_REMOTE_TX, + TAP_ISOLATE, + TAP_REMOTE_BROADCASTV6, + TAP_REMOTE_BROADCAST, + TAP_REMOTE_ALLMULTI, + TAP_REMOTE_PROMISC, + TAP_REMOTE_LOCAL_MAC, + TAP_REMOTE_MAX_IDX, +}; + +int tap_dev_filter_ctrl(struct rte_eth_dev *dev, + enum rte_filter_type filter_type, + enum rte_filter_op filter_op, + void *arg); +int tap_flow_flush(struct rte_eth_dev *dev, struct rte_flow_error *error); + +int tap_flow_implicit_create(struct pmd_internals *pmd, + enum implicit_rule_index idx); +int tap_flow_implicit_destroy(struct pmd_internals *pmd, + enum implicit_rule_index idx); +int tap_flow_implicit_flush(struct pmd_internals *pmd, + struct rte_flow_error *error); + +#endif /* _TAP_FLOW_H_ */ diff --git a/dpdk/drivers/net/tap/tap_netlink.c b/dpdk/drivers/net/tap/tap_netlink.c new file mode 100644 index 00000000..ee92e2e7 --- /dev/null +++ b/dpdk/drivers/net/tap/tap_netlink.c @@ -0,0 +1,367 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of 6WIND S.A. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +/* Must be quite large to support dumping a huge list of QDISC or filters. */ +#define BUF_SIZE (32 * 1024) /* Size of the buffer to receive kernel messages */ +#define SNDBUF_SIZE 32768 /* Send buffer size for the netlink socket */ +#define RCVBUF_SIZE 32768 /* Receive buffer size for the netlink socket */ + +struct nested_tail { + struct rtattr *tail; + struct nested_tail *prev; +}; + +/** + * Initialize a netlink socket for communicating with the kernel. + * + * @param nl_groups + * Set it to a netlink group value (e.g. RTMGRP_LINK) to receive messages for + * specific netlink multicast groups. Otherwise, no subscription will be made. + * + * @return + * netlink socket file descriptor on success, -1 otherwise. + */ +int +nl_init(uint32_t nl_groups) +{ + int fd, sndbuf_size = SNDBUF_SIZE, rcvbuf_size = RCVBUF_SIZE; + struct sockaddr_nl local = { + .nl_family = AF_NETLINK, + .nl_groups = nl_groups, + }; + + fd = socket(AF_NETLINK, SOCK_RAW | SOCK_CLOEXEC, NETLINK_ROUTE); + if (fd < 0) { + RTE_LOG(ERR, PMD, "Unable to create a netlink socket\n"); + return -1; + } + if (setsockopt(fd, SOL_SOCKET, SO_SNDBUF, &sndbuf_size, sizeof(int))) { + RTE_LOG(ERR, PMD, "Unable to set socket buffer send size\n"); + return -1; + } + if (setsockopt(fd, SOL_SOCKET, SO_RCVBUF, &rcvbuf_size, sizeof(int))) { + RTE_LOG(ERR, PMD, "Unable to set socket buffer receive size\n"); + return -1; + } + if (bind(fd, (struct sockaddr *)&local, sizeof(local)) < 0) { + RTE_LOG(ERR, PMD, "Unable to bind to the netlink socket\n"); + return -1; + } + return fd; +} + +/** + * Clean up a netlink socket once all communicating with the kernel is finished. + * + * @param[in] nlsk_fd + * The netlink socket file descriptor used for communication. + * + * @return + * 0 on success, -1 otherwise. + */ +int +nl_final(int nlsk_fd) +{ + if (close(nlsk_fd)) { + RTE_LOG(ERR, PMD, "Failed to close netlink socket: %s (%d)\n", + strerror(errno), errno); + return -1; + } + return 0; +} + +/** + * Send a message to the kernel on the netlink socket. + * + * @param[in] nlsk_fd + * The netlink socket file descriptor used for communication. + * @param[in] nh + * The netlink message send to the kernel. + * + * @return + * the number of sent bytes on success, -1 otherwise. + */ +int +nl_send(int nlsk_fd, struct nlmsghdr *nh) +{ + /* man 7 netlink EXAMPLE */ + struct sockaddr_nl sa = { + .nl_family = AF_NETLINK, + }; + struct iovec iov = { + .iov_base = nh, + .iov_len = nh->nlmsg_len, + }; + struct msghdr msg = { + .msg_name = &sa, + .msg_namelen = sizeof(sa), + .msg_iov = &iov, + .msg_iovlen = 1, + }; + int send_bytes; + + nh->nlmsg_pid = 0; /* communication with the kernel uses pid 0 */ + nh->nlmsg_seq = (uint32_t)rte_rand(); + send_bytes = sendmsg(nlsk_fd, &msg, 0); + if (send_bytes < 0) { + RTE_LOG(ERR, PMD, "Failed to send netlink message: %s (%d)\n", + strerror(errno), errno); + return -1; + } + return send_bytes; +} + +/** + * Check that the kernel sends an appropriate ACK in response to an nl_send(). + * + * @param[in] nlsk_fd + * The netlink socket file descriptor used for communication. + * + * @return + * 0 on success, -1 otherwise with errno set. + */ +int +nl_recv_ack(int nlsk_fd) +{ + return nl_recv(nlsk_fd, NULL, NULL); +} + +/** + * Receive a message from the kernel on the netlink socket, following an + * nl_send(). + * + * @param[in] nlsk_fd + * The netlink socket file descriptor used for communication. + * @param[in] cb + * The callback function to call for each netlink message received. + * @param[in, out] arg + * Custom arguments for the callback. + * + * @return + * 0 on success, -1 otherwise with errno set. + */ +int +nl_recv(int nlsk_fd, int (*cb)(struct nlmsghdr *, void *arg), void *arg) +{ + /* man 7 netlink EXAMPLE */ + struct sockaddr_nl sa; + char buf[BUF_SIZE]; + struct iovec iov = { + .iov_base = buf, + .iov_len = sizeof(buf), + }; + struct msghdr msg = { + .msg_name = &sa, + .msg_namelen = sizeof(sa), + .msg_iov = &iov, + /* One message at a time */ + .msg_iovlen = 1, + }; + int multipart = 0; + int ret = 0; + + do { + struct nlmsghdr *nh; + int recv_bytes = 0; + + recv_bytes = recvmsg(nlsk_fd, &msg, 0); + if (recv_bytes < 0) + return -1; + for (nh = (struct nlmsghdr *)buf; + NLMSG_OK(nh, (unsigned int)recv_bytes); + nh = NLMSG_NEXT(nh, recv_bytes)) { + if (nh->nlmsg_type == NLMSG_ERROR) { + struct nlmsgerr *err_data = NLMSG_DATA(nh); + + if (err_data->error < 0) { + errno = -err_data->error; + return -1; + } + /* Ack message. */ + return 0; + } + /* Multi-part msgs and their trailing DONE message. */ + if (nh->nlmsg_flags & NLM_F_MULTI) { + if (nh->nlmsg_type == NLMSG_DONE) + return 0; + multipart = 1; + } + if (cb) + ret = cb(nh, arg); + } + } while (multipart); + return ret; +} + +/** + * Append a netlink attribute to a message. + * + * @param[in, out] nh + * The netlink message to parse, received from the kernel. + * @param[in] type + * The type of attribute to append. + * @param[in] data_len + * The length of the data to append. + * @param[in] data + * The data to append. + */ +void +nlattr_add(struct nlmsghdr *nh, unsigned short type, + unsigned int data_len, const void *data) +{ + /* see man 3 rtnetlink */ + struct rtattr *rta; + + rta = (struct rtattr *)NLMSG_TAIL(nh); + rta->rta_len = RTA_LENGTH(data_len); + rta->rta_type = type; + memcpy(RTA_DATA(rta), data, data_len); + nh->nlmsg_len = NLMSG_ALIGN(nh->nlmsg_len) + RTA_ALIGN(rta->rta_len); +} + +/** + * Append a uint8_t netlink attribute to a message. + * + * @param[in, out] nh + * The netlink message to parse, received from the kernel. + * @param[in] type + * The type of attribute to append. + * @param[in] data + * The data to append. + */ +void +nlattr_add8(struct nlmsghdr *nh, unsigned short type, uint8_t data) +{ + nlattr_add(nh, type, sizeof(uint8_t), &data); +} + +/** + * Append a uint16_t netlink attribute to a message. + * + * @param[in, out] nh + * The netlink message to parse, received from the kernel. + * @param[in] type + * The type of attribute to append. + * @param[in] data + * The data to append. + */ +void +nlattr_add16(struct nlmsghdr *nh, unsigned short type, uint16_t data) +{ + nlattr_add(nh, type, sizeof(uint16_t), &data); +} + +/** + * Append a uint16_t netlink attribute to a message. + * + * @param[in, out] nh + * The netlink message to parse, received from the kernel. + * @param[in] type + * The type of attribute to append. + * @param[in] data + * The data to append. + */ +void +nlattr_add32(struct nlmsghdr *nh, unsigned short type, uint32_t data) +{ + nlattr_add(nh, type, sizeof(uint32_t), &data); +} + +/** + * Start a nested netlink attribute. + * It must be followed later by a call to nlattr_nested_finish(). + * + * @param[in, out] msg + * The netlink message where to edit the nested_tails metadata. + * @param[in] type + * The nested attribute type to append. + * + * @return + * -1 if adding a nested netlink attribute failed, 0 otherwise. + */ +int +nlattr_nested_start(struct nlmsg *msg, uint16_t type) +{ + struct nested_tail *tail; + + tail = rte_zmalloc(NULL, sizeof(struct nested_tail), 0); + if (!tail) { + RTE_LOG(ERR, PMD, + "Couldn't allocate memory for nested netlink" + " attribute\n"); + return -1; + } + + tail->tail = (struct rtattr *)NLMSG_TAIL(&msg->nh); + + nlattr_add(&msg->nh, type, 0, NULL); + + tail->prev = msg->nested_tails; + + msg->nested_tails = tail; + + return 0; +} + +/** + * End a nested netlink attribute. + * It follows a call to nlattr_nested_start(). + * In effect, it will modify the nested attribute length to include every bytes + * from the nested attribute start, up to here. + * + * @param[in, out] msg + * The netlink message where to edit the nested_tails metadata. + */ +void +nlattr_nested_finish(struct nlmsg *msg) +{ + struct nested_tail *tail = msg->nested_tails; + + tail->tail->rta_len = (char *)NLMSG_TAIL(&msg->nh) - (char *)tail->tail; + + if (tail->prev) + msg->nested_tails = tail->prev; + + rte_free(tail); +} diff --git a/dpdk/drivers/net/tap/tap_netlink.h b/dpdk/drivers/net/tap/tap_netlink.h new file mode 100644 index 00000000..98e13902 --- /dev/null +++ b/dpdk/drivers/net/tap/tap_netlink.h @@ -0,0 +1,69 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of 6WIND S.A. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _TAP_NETLINK_H_ +#define _TAP_NETLINK_H_ + +#include +#include +#include +#include +#include + +#include + +#define NLMSG_BUF 512 + +struct nlmsg { + struct nlmsghdr nh; + struct tcmsg t; + char buf[NLMSG_BUF]; + struct nested_tail *nested_tails; +}; + +#define NLMSG_TAIL(nlh) (void *)((char *)(nlh) + NLMSG_ALIGN((nlh)->nlmsg_len)) + +int nl_init(uint32_t nl_groups); +int nl_final(int nlsk_fd); +int nl_send(int nlsk_fd, struct nlmsghdr *nh); +int nl_recv(int nlsk_fd, int (*callback)(struct nlmsghdr *, void *), void *arg); +int nl_recv_ack(int nlsk_fd); +void nlattr_add(struct nlmsghdr *nh, unsigned short type, + unsigned int data_len, const void *data); +void nlattr_add8(struct nlmsghdr *nh, unsigned short type, uint8_t data); +void nlattr_add16(struct nlmsghdr *nh, unsigned short type, uint16_t data); +void nlattr_add32(struct nlmsghdr *nh, unsigned short type, uint32_t data); +int nlattr_nested_start(struct nlmsg *msg, uint16_t type); +void nlattr_nested_finish(struct nlmsg *msg); + +#endif /* _TAP_NETLINK_H_ */ diff --git a/dpdk/drivers/net/tap/tap_tcmsgs.c b/dpdk/drivers/net/tap/tap_tcmsgs.c new file mode 100644 index 00000000..d74ac805 --- /dev/null +++ b/dpdk/drivers/net/tap/tap_tcmsgs.c @@ -0,0 +1,323 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of 6WIND S.A. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include + +#include +#include + +struct qdisc { + uint32_t handle; + uint32_t parent; +}; + +struct list_args { + int nlsk_fd; + uint16_t ifindex; + void *custom_arg; +}; + +struct qdisc_custom_arg { + uint32_t handle; + uint32_t parent; + uint8_t exists; +}; + +/** + * Initialize a netlink message with a TC header. + * + * @param[in, out] msg + * The netlink message to fill. + * @param[in] ifindex + * The netdevice ifindex where the rule will be applied. + * @param[in] type + * The type of TC message to create (RTM_NEWTFILTER, RTM_NEWQDISC, etc.). + * @param[in] flags + * Overrides the default netlink flags for this msg with those specified. + */ +void +tc_init_msg(struct nlmsg *msg, uint16_t ifindex, uint16_t type, uint16_t flags) +{ + struct nlmsghdr *n = &msg->nh; + + n->nlmsg_len = NLMSG_LENGTH(sizeof(struct tcmsg)); + n->nlmsg_type = type; + if (flags) + n->nlmsg_flags = flags; + else + n->nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK; + msg->t.tcm_family = AF_UNSPEC; + msg->t.tcm_ifindex = ifindex; +} + +/** + * Delete a specific QDISC identified by its iface, and it's handle and parent. + * + * @param[in] nlsk_fd + * The netlink socket file descriptor used for communication. + * @param[in] ifindex + * The netdevice ifindex on whom the deletion will happen. + * @param[in] qinfo + * Additional info to identify the QDISC (handle and parent). + * + * @return + * 0 on success, -1 otherwise with errno set. + */ +static int +qdisc_del(int nlsk_fd, uint16_t ifindex, struct qdisc *qinfo) +{ + struct nlmsg msg; + int fd = 0; + + tc_init_msg(&msg, ifindex, RTM_DELQDISC, 0); + msg.t.tcm_handle = qinfo->handle; + msg.t.tcm_parent = qinfo->parent; + /* if no netlink socket is provided, create one */ + if (!nlsk_fd) { + fd = nl_init(0); + if (fd < 0) { + RTE_LOG(ERR, PMD, + "Could not delete QDISC: null netlink socket\n"); + return -1; + } + } else { + fd = nlsk_fd; + } + if (nl_send(fd, &msg.nh) < 0) + goto error; + if (nl_recv_ack(fd) < 0) + goto error; + if (!nlsk_fd) + return nl_final(fd); + return 0; +error: + if (!nlsk_fd) + nl_final(fd); + return -1; +} + +/** + * Add the multiqueue QDISC with MULTIQ_MAJOR_HANDLE handle. + * + * @param[in] nlsk_fd + * The netlink socket file descriptor used for communication. + * @param[in] ifindex + * The netdevice ifindex where to add the multiqueue QDISC. + * + * @return + * 0 on success, -1 otherwise with errno set. + */ +int +qdisc_add_multiq(int nlsk_fd, uint16_t ifindex) +{ + struct tc_multiq_qopt opt; + struct nlmsg msg; + + tc_init_msg(&msg, ifindex, RTM_NEWQDISC, + NLM_F_REQUEST | NLM_F_ACK | NLM_F_EXCL | NLM_F_CREATE); + msg.t.tcm_handle = TC_H_MAKE(MULTIQ_MAJOR_HANDLE, 0); + msg.t.tcm_parent = TC_H_ROOT; + nlattr_add(&msg.nh, TCA_KIND, sizeof("multiq"), "multiq"); + nlattr_add(&msg.nh, TCA_OPTIONS, sizeof(opt), &opt); + if (nl_send(nlsk_fd, &msg.nh) < 0) + return -1; + if (nl_recv_ack(nlsk_fd) < 0) + return -1; + return 0; +} + +/** + * Add the ingress QDISC with default ffff: handle. + * + * @param[in] nlsk_fd + * The netlink socket file descriptor used for communication. + * @param[in] ifindex + * The netdevice ifindex where the QDISC will be added. + * + * @return + * 0 on success, -1 otherwise with errno set. + */ +int +qdisc_add_ingress(int nlsk_fd, uint16_t ifindex) +{ + struct nlmsg msg; + + tc_init_msg(&msg, ifindex, RTM_NEWQDISC, + NLM_F_REQUEST | NLM_F_ACK | NLM_F_EXCL | NLM_F_CREATE); + msg.t.tcm_handle = TC_H_MAKE(TC_H_INGRESS, 0); + msg.t.tcm_parent = TC_H_INGRESS; + nlattr_add(&msg.nh, TCA_KIND, sizeof("ingress"), "ingress"); + if (nl_send(nlsk_fd, &msg.nh) < 0) + return -1; + if (nl_recv_ack(nlsk_fd) < 0) + return -1; + return 0; +} + +/** + * Callback function to delete a QDISC. + * + * @param[in] nh + * The netlink message to parse, received from the kernel. + * @param[in] arg + * Custom arguments for the callback. + * + * @return + * 0 on success, -1 otherwise with errno set. + */ +static int +qdisc_del_cb(struct nlmsghdr *nh, void *arg) +{ + struct tcmsg *t = NLMSG_DATA(nh); + struct list_args *args = arg; + + struct qdisc qinfo = { + .handle = t->tcm_handle, + .parent = t->tcm_parent, + }; + + /* filter out other ifaces' qdiscs */ + if (args->ifindex != (unsigned int)t->tcm_ifindex) + return 0; + /* + * Use another nlsk_fd (0) to avoid tampering with the current list + * iteration. + */ + return qdisc_del(0, args->ifindex, &qinfo); +} + +/** + * Iterate over all QDISC, and call the callback() function for each. + * + * @param[in] nlsk_fd + * The netlink socket file descriptor used for communication. + * @param[in] ifindex + * The netdevice ifindex where to find QDISCs. + * @param[in] callback + * The function to call for each QDISC. + * @param[in, out] arg + * The arguments to provide the callback function with. + * + * @return + * 0 on success, -1 otherwise with errno set. + */ +static int +qdisc_iterate(int nlsk_fd, uint16_t ifindex, + int (*callback)(struct nlmsghdr *, void *), void *arg) +{ + struct nlmsg msg; + struct list_args args = { + .nlsk_fd = nlsk_fd, + .ifindex = ifindex, + .custom_arg = arg, + }; + + tc_init_msg(&msg, ifindex, RTM_GETQDISC, NLM_F_REQUEST | NLM_F_DUMP); + if (nl_send(nlsk_fd, &msg.nh) < 0) + return -1; + if (nl_recv(nlsk_fd, callback, &args) < 0) + return -1; + return 0; +} + +/** + * Delete all QDISCs for a given netdevice. + * + * @param[in] nlsk_fd + * The netlink socket file descriptor used for communication. + * @param[in] ifindex + * The netdevice ifindex where to find QDISCs. + * + * @return + * 0 on success, -1 otherwise with errno set. + */ +int +qdisc_flush(int nlsk_fd, uint16_t ifindex) +{ + return qdisc_iterate(nlsk_fd, ifindex, qdisc_del_cb, NULL); +} + +/** + * Create the multiqueue QDISC, only if it does not exist already. + * + * @param[in] nlsk_fd + * The netlink socket file descriptor used for communication. + * @param[in] ifindex + * The netdevice ifindex where to add the multiqueue QDISC. + * + * @return + * 0 if the qdisc exists or if has been successfully added. + * Return -1 otherwise. + */ +int +qdisc_create_multiq(int nlsk_fd, uint16_t ifindex) +{ + int err = 0; + + err = qdisc_add_multiq(nlsk_fd, ifindex); + if (err < 0 && errno != -EEXIST) { + RTE_LOG(ERR, PMD, "Could not add multiq qdisc (%d): %s\n", + errno, strerror(errno)); + return -1; + } + return 0; +} + +/** + * Create the ingress QDISC, only if it does not exist already. + * + * @param[in] nlsk_fd + * The netlink socket file descriptor used for communication. + * @param[in] ifindex + * The netdevice ifindex where to add the ingress QDISC. + * + * @return + * 0 if the qdisc exists or if has been successfully added. + * Return -1 otherwise. + */ +int +qdisc_create_ingress(int nlsk_fd, uint16_t ifindex) +{ + int err = 0; + + err = qdisc_add_ingress(nlsk_fd, ifindex); + if (err < 0 && errno != -EEXIST) { + RTE_LOG(ERR, PMD, "Could not add ingress qdisc (%d): %s\n", + errno, strerror(errno)); + return -1; + } + return 0; +} diff --git a/dpdk/drivers/net/tap/tap_tcmsgs.h b/dpdk/drivers/net/tap/tap_tcmsgs.h new file mode 100644 index 00000000..78959577 --- /dev/null +++ b/dpdk/drivers/net/tap/tap_tcmsgs.h @@ -0,0 +1,61 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of 6WIND S.A. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _TAP_TCMSGS_H_ +#define _TAP_TCMSGS_H_ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#define MULTIQ_MAJOR_HANDLE (1 << 16) + +void tc_init_msg(struct nlmsg *msg, uint16_t ifindex, uint16_t type, + uint16_t flags); +int qdisc_list(int nlsk_fd, uint16_t ifindex); +int qdisc_flush(int nlsk_fd, uint16_t ifindex); +int qdisc_create_ingress(int nlsk_fd, uint16_t ifindex); +int qdisc_create_multiq(int nlsk_fd, uint16_t ifindex); +int qdisc_add_ingress(int nlsk_fd, uint16_t ifindex); +int qdisc_add_multiq(int nlsk_fd, uint16_t ifindex); +int filter_list_ingress(int nlsk_fd, uint16_t ifindex); + +#endif /* _TAP_TCMSGS_H_ */ diff --git a/dpdk/drivers/net/thunderx/Makefile b/dpdk/drivers/net/thunderx/Makefile index 8ea6b454..e50e1ad8 100644 --- a/dpdk/drivers/net/thunderx/Makefile +++ b/dpdk/drivers/net/thunderx/Makefile @@ -1,6 +1,6 @@ # BSD LICENSE # -# Copyright(c) 2016 Cavium Networks. All rights reserved. +# Copyright(c) 2016 Cavium, Inc. All rights reserved. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -13,7 +13,7 @@ # notice, this list of conditions and the following disclaimer in # the documentation and/or other materials provided with the # distribution. -# * Neither the name of Cavium Networks nor the names of its +# * Neither the name of Cavium, Inc nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # @@ -40,12 +40,15 @@ LIB = librte_pmd_thunderx_nicvf.a CFLAGS += $(WERROR_FLAGS) LDLIBS += -lm +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs +LDLIBS += -lrte_bus_pci EXPORT_MAP := rte_pmd_thunderx_nicvf_version.map LIBABIVER := 1 -OBJS_BASE_DRIVER=$(patsubst %.c,%.o,$(notdir $(wildcard $(SRCDIR)/base/*.c))) +OBJS_BASE_DRIVER=$(sort $(patsubst %.c,%.o,$(notdir $(wildcard $(SRCDIR)/base/*.c)))) $(foreach obj, $(OBJS_BASE_DRIVER), $(eval CFLAGS_$(obj)+=$(CFLAGS_BASE_DRIVER))) VPATH += $(SRCDIR)/base @@ -57,14 +60,12 @@ SRCS-$(CONFIG_RTE_LIBRTE_THUNDERX_NICVF_PMD) += nicvf_rxtx.c SRCS-$(CONFIG_RTE_LIBRTE_THUNDERX_NICVF_PMD) += nicvf_hw.c SRCS-$(CONFIG_RTE_LIBRTE_THUNDERX_NICVF_PMD) += nicvf_mbox.c SRCS-$(CONFIG_RTE_LIBRTE_THUNDERX_NICVF_PMD) += nicvf_ethdev.c +SRCS-$(CONFIG_RTE_LIBRTE_THUNDERX_NICVF_PMD) += nicvf_bsvf.c +SRCS-$(CONFIG_RTE_LIBRTE_THUNDERX_NICVF_PMD) += nicvf_svf.c ifeq ($(CONFIG_RTE_TOOLCHAIN_GCC),y) CFLAGS_nicvf_rxtx.o += -fno-prefetch-loop-arrays endif CFLAGS_nicvf_rxtx.o += -Ofast -# this lib depends upon: -DEPDIRS-$(CONFIG_RTE_LIBRTE_THUNDERX_NICVF_PMD) += lib/librte_eal lib/librte_ether -DEPDIRS-$(CONFIG_RTE_LIBRTE_THUNDERX_NICVF_PMD) += lib/librte_mempool lib/librte_mbuf - include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/net/thunderx/base/nicvf_bsvf.c b/dpdk/drivers/net/thunderx/base/nicvf_bsvf.c new file mode 100644 index 00000000..a4fc04cb --- /dev/null +++ b/dpdk/drivers/net/thunderx/base/nicvf_bsvf.c @@ -0,0 +1,72 @@ +/* + * BSD LICENSE + * + * Copyright (C) Cavium, Inc. 2016. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include + +#include "nicvf_bsvf.h" +#include "nicvf_plat.h" + +static STAILQ_HEAD(, svf_entry) head = STAILQ_HEAD_INITIALIZER(head); + +void +nicvf_bsvf_push(struct svf_entry *entry) +{ + assert(entry != NULL); + assert(entry->vf != NULL); + + STAILQ_INSERT_TAIL(&head, entry, next); +} + +struct svf_entry * +nicvf_bsvf_pop(void) +{ + struct svf_entry *entry; + + assert(!STAILQ_EMPTY(&head)); + + entry = STAILQ_FIRST(&head); + + assert(entry != NULL); + assert(entry->vf != NULL); + + STAILQ_REMOVE_HEAD(&head, next); + + return entry; +} + +int +nicvf_bsvf_empty(void) +{ + return STAILQ_EMPTY(&head); +} diff --git a/dpdk/drivers/net/thunderx/base/nicvf_bsvf.h b/dpdk/drivers/net/thunderx/base/nicvf_bsvf.h new file mode 100644 index 00000000..2d9448aa --- /dev/null +++ b/dpdk/drivers/net/thunderx/base/nicvf_bsvf.h @@ -0,0 +1,76 @@ +/* + * BSD LICENSE + * + * Copyright (C) Cavium, Inc. 2016. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __THUNDERX_NICVF_BSVF_H__ +#define __THUNDERX_NICVF_BSVF_H__ + +#include + +struct nicvf; + +/** + * The base queue structure to hold secondary qsets. + */ +struct svf_entry { + STAILQ_ENTRY(svf_entry) next; /**< Next element's pointer */ + struct nicvf *vf; /**< Holder of a secondary qset */ +}; + +/** + * Enqueue new entry to secondary qsets. + * + * @param entry + * Entry to be enqueued. + */ +void +nicvf_bsvf_push(struct svf_entry *entry); + +/** + * Dequeue an entry from secondary qsets. + * + * @return + * Dequeued entry. + */ +struct svf_entry * +nicvf_bsvf_pop(void); + +/** + * Check if the queue of secondary qsets is empty. + * + * @return + * 0 on non-empty + * otherwise empty + */ +int +nicvf_bsvf_empty(void); + +#endif /* __THUNDERX_NICVF_BSVF_H__ */ diff --git a/dpdk/drivers/net/thunderx/base/nicvf_hw.c b/dpdk/drivers/net/thunderx/base/nicvf_hw.c index 001b0edd..dc0af1ca 100644 --- a/dpdk/drivers/net/thunderx/base/nicvf_hw.c +++ b/dpdk/drivers/net/thunderx/base/nicvf_hw.c @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2016. + * Copyright (C) Cavium, Inc. 2016. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * @@ -140,8 +140,15 @@ nicvf_base_init(struct nicvf *nic) if (nic->subsystem_device_id == 0) return NICVF_ERR_BASE_INIT; - if (nicvf_hw_version(nic) == NICVF_PASS2) - nic->hwcap |= NICVF_CAP_TUNNEL_PARSING; + if (nicvf_hw_version(nic) == PCI_SUB_DEVICE_ID_CN88XX_PASS2_NICVF) + nic->hwcap |= NICVF_CAP_TUNNEL_PARSING | NICVF_CAP_CQE_RX2; + + if (nicvf_hw_version(nic) == PCI_SUB_DEVICE_ID_CN81XX_NICVF) + nic->hwcap |= NICVF_CAP_TUNNEL_PARSING | NICVF_CAP_CQE_RX2; + + if (nicvf_hw_version(nic) == PCI_SUB_DEVICE_ID_CN83XX_NICVF) + nic->hwcap |= NICVF_CAP_TUNNEL_PARSING | NICVF_CAP_CQE_RX2 | + NICVF_CAP_DISABLE_APAD; return NICVF_OK; } @@ -441,7 +448,8 @@ nicvf_qsize_regbit(uint32_t len, uint32_t len_shift) { int val; - val = ((uint32_t)log2(len) - len_shift); + val = nicvf_log2_u32(len) - len_shift; + assert(val >= NICVF_QSIZE_MIN_VAL); assert(val <= NICVF_QSIZE_MAX_VAL); return val; @@ -494,14 +502,14 @@ nicvf_qsize_rbdr_roundup(uint32_t val) } int -nicvf_qset_rbdr_precharge(struct nicvf *nic, uint16_t ridx, - rbdr_pool_get_handler handler, - void *opaque, uint32_t max_buffs) +nicvf_qset_rbdr_precharge(void *dev, struct nicvf *nic, + uint16_t ridx, rbdr_pool_get_handler handler, + uint32_t max_buffs) { struct rbdr_entry_t *desc, *desc0; struct nicvf_rbdr *rbdr = nic->rbdr; uint32_t count; - nicvf_phys_addr_t phy; + nicvf_iova_addr_t phy; assert(rbdr != NULL); desc = rbdr->desc; @@ -511,7 +519,7 @@ nicvf_qset_rbdr_precharge(struct nicvf *nic, uint16_t ridx, if (count >= max_buffs) break; desc0 = desc + count; - phy = handler(opaque); + phy = handler(dev, nic); if (phy) { desc0->full_addr = phy; count++; @@ -578,6 +586,7 @@ nicvf_qset_sq_config(struct nicvf *nic, uint16_t qidx, struct nicvf_txq *txq) nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_BASE, qidx, txq->phys); /* Enable send queue & set queue size */ + sq_cfg.cq_limit = 0; sq_cfg.ena = 1; sq_cfg.reset = 0; sq_cfg.ldwb = 0; @@ -721,6 +730,24 @@ nicvf_vlan_hw_strip(struct nicvf *nic, bool enable) nicvf_reg_write(nic, NIC_VNIC_RQ_GEN_CFG, val); } +void +nicvf_apad_config(struct nicvf *nic, bool enable) +{ + uint64_t val; + + /* APAD always enabled in this device */ + if (!(nic->hwcap & NICVF_CAP_DISABLE_APAD)) + return; + + val = nicvf_reg_read(nic, NIC_VNIC_RQ_GEN_CFG); + if (enable) + val &= ~(1ULL << NICVF_QS_RQ_DIS_APAD_SHIFT); + else + val |= (1ULL << NICVF_QS_RQ_DIS_APAD_SHIFT); + + nicvf_reg_write(nic, NIC_VNIC_RQ_GEN_CFG, val); +} + void nicvf_rss_set_key(struct nicvf *nic, uint8_t *key) { @@ -776,7 +803,7 @@ nicvf_rss_reta_update(struct nicvf *nic, uint8_t *tbl, uint32_t max_count) return NICVF_ERR_RSS_GET_SZ; assert(rss->rss_size > 0); - rss->hash_bits = (uint8_t)log2(rss->rss_size); + rss->hash_bits = (uint8_t)nicvf_log2_u32(rss->rss_size); for (idx = 0; idx < rss->rss_size && idx < max_count; idx++) rss->ind_tbl[idx] = tbl[idx]; @@ -797,7 +824,8 @@ nicvf_rss_reta_query(struct nicvf *nic, uint8_t *tbl, uint32_t max_count) return NICVF_ERR_RSS_GET_SZ; assert(rss->rss_size > 0); - rss->hash_bits = (uint8_t)log2(rss->rss_size); + rss->hash_bits = (uint8_t)nicvf_log2_u32(rss->rss_size); + for (idx = 0; idx < rss->rss_size && idx < max_count; idx++) tbl[idx] = rss->ind_tbl[idx]; diff --git a/dpdk/drivers/net/thunderx/base/nicvf_hw.h b/dpdk/drivers/net/thunderx/base/nicvf_hw.h index 9db1d30c..698aa487 100644 --- a/dpdk/drivers/net/thunderx/base/nicvf_hw.h +++ b/dpdk/drivers/net/thunderx/base/nicvf_hw.h @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2016. + * Copyright (C) Cavium, Inc. 2016. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * @@ -37,11 +37,13 @@ #include "nicvf_hw_defs.h" -#define PCI_VENDOR_ID_CAVIUM 0x177D -#define PCI_DEVICE_ID_THUNDERX_PASS1_NICVF 0x0011 -#define PCI_DEVICE_ID_THUNDERX_PASS2_NICVF 0xA034 -#define PCI_SUB_DEVICE_ID_THUNDERX_PASS1_NICVF 0xA11E -#define PCI_SUB_DEVICE_ID_THUNDERX_PASS2_NICVF 0xA134 +#define PCI_VENDOR_ID_CAVIUM 0x177D +#define PCI_DEVICE_ID_THUNDERX_CN88XX_PASS1_NICVF 0x0011 +#define PCI_DEVICE_ID_THUNDERX_NICVF 0xA034 +#define PCI_SUB_DEVICE_ID_CN88XX_PASS1_NICVF 0xA11E +#define PCI_SUB_DEVICE_ID_CN88XX_PASS2_NICVF 0xA134 +#define PCI_SUB_DEVICE_ID_CN81XX_NICVF 0xA234 +#define PCI_SUB_DEVICE_ID_CN83XX_NICVF 0xA334 #define NICVF_ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0])) @@ -50,10 +52,11 @@ #define NICVF_GET_TX_STATS(reg) \ nicvf_reg_read(nic, NIC_VNIC_TX_STAT_0_4 | (reg << 3)) -#define NICVF_PASS1 (PCI_SUB_DEVICE_ID_THUNDERX_PASS1_NICVF) -#define NICVF_PASS2 (PCI_SUB_DEVICE_ID_THUNDERX_PASS2_NICVF) - -#define NICVF_CAP_TUNNEL_PARSING (1ULL << 0) +#define NICVF_CAP_TUNNEL_PARSING (1ULL << 0) +/* Additional word in Rx descriptor to hold optional tunneling extension info */ +#define NICVF_CAP_CQE_RX2 (1ULL << 1) +/* The device capable of setting NIC_CQE_RX_S[APAD] == 0 */ +#define NICVF_CAP_DISABLE_APAD (1ULL << 2) enum nicvf_tns_mode { NIC_TNS_BYPASS_MODE, @@ -85,7 +88,7 @@ enum nicvf_err_e { NICVF_ERR_RSS_GET_SZ, /* -8171 */ }; -typedef nicvf_phys_addr_t (*rbdr_pool_get_handler)(void *opaque); +typedef nicvf_iova_addr_t (*rbdr_pool_get_handler)(void *dev, void *opaque); struct nicvf_hw_rx_qstats { uint64_t q_rx_bytes; @@ -194,8 +197,8 @@ int nicvf_qset_reclaim(struct nicvf *nic); int nicvf_qset_rbdr_config(struct nicvf *nic, uint16_t qidx); int nicvf_qset_rbdr_reclaim(struct nicvf *nic, uint16_t qidx); -int nicvf_qset_rbdr_precharge(struct nicvf *nic, uint16_t ridx, - rbdr_pool_get_handler handler, void *opaque, +int nicvf_qset_rbdr_precharge(void *dev, struct nicvf *nic, + uint16_t ridx, rbdr_pool_get_handler handler, uint32_t max_buffs); int nicvf_qset_rbdr_active(struct nicvf *nic, uint16_t qidx); @@ -217,6 +220,8 @@ uint32_t nicvf_qsize_sq_roundup(uint32_t val); void nicvf_vlan_hw_strip(struct nicvf *nic, bool enable); +void nicvf_apad_config(struct nicvf *nic, bool enable); + int nicvf_rss_config(struct nicvf *nic, uint32_t qcnt, uint64_t cfg); int nicvf_rss_term(struct nicvf *nic); diff --git a/dpdk/drivers/net/thunderx/base/nicvf_hw_defs.h b/dpdk/drivers/net/thunderx/base/nicvf_hw_defs.h index 2f2b2259..e7e092b6 100644 --- a/dpdk/drivers/net/thunderx/base/nicvf_hw_defs.h +++ b/dpdk/drivers/net/thunderx/base/nicvf_hw_defs.h @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2016. + * Copyright (C) Cavium, Inc. 2016. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * @@ -36,6 +36,8 @@ #include #include +#include "nicvf_plat.h" + /* Virtual function register offsets */ #define NIC_VF_CFG (0x000020) @@ -105,6 +107,8 @@ #define NICVF_INTR_MBOX_SHIFT 22 #define NICVF_INTR_QS_ERR_SHIFT 23 +#define NICVF_QS_RQ_DIS_APAD_SHIFT 22 + #define NICVF_INTR_CQ_MASK (0xFF << NICVF_INTR_CQ_SHIFT) #define NICVF_INTR_SQ_MASK (0xFF << NICVF_INTR_SQ_SHIFT) #define NICVF_INTR_RBDR_MASK (0x03 << NICVF_INTR_RBDR_SHIFT) @@ -207,12 +211,9 @@ #define NICVF_CQE_RX2_RBPTR_WORD (7) #define NICVF_STATIC_ASSERT(s) _Static_assert(s, #s) +#define assert_primary(nic) assert((nic)->sqs_mode == 0) -typedef uint64_t nicvf_phys_addr_t; - -#ifndef __BYTE_ORDER__ -#error __BYTE_ORDER__ not defined -#endif +typedef uint64_t nicvf_iova_addr_t; /* vNIC HW Enumerations */ @@ -556,7 +557,7 @@ enum nic_stat_vnic_tx_e { typedef union { uint64_t u64; struct { -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +#if NICVF_BYTE_ORDER == NICVF_BIG_ENDIAN uint64_t cqe_type:4; uint64_t stdn_fault:1; uint64_t rsvd0:1; @@ -601,7 +602,7 @@ typedef union { typedef union { uint64_t u64; struct { -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +#if NICVF_BYTE_ORDER == NICVF_BIG_ENDIAN uint64_t pkt_len:16; uint64_t l2_ptr:8; uint64_t l3_ptr:8; @@ -626,7 +627,7 @@ typedef union { typedef union { uint64_t u64; struct { -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +#if NICVF_BYTE_ORDER == NICVF_BIG_ENDIAN uint64_t rss_tag:32; uint64_t vlan_tci:16; uint64_t vlan_ptr:8; @@ -643,7 +644,7 @@ typedef union { typedef union { uint64_t u64; struct { -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +#if NICVF_BYTE_ORDER == NICVF_BIG_ENDIAN uint16_t rb3_sz; uint16_t rb2_sz; uint16_t rb1_sz; @@ -660,7 +661,7 @@ typedef union { typedef union { uint64_t u64; struct { -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +#if NICVF_BYTE_ORDER == NICVF_BIG_ENDIAN uint16_t rb7_sz; uint16_t rb6_sz; uint16_t rb5_sz; @@ -677,7 +678,7 @@ typedef union { typedef union { uint64_t u64; struct { -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +#if NICVF_BYTE_ORDER == NICVF_BIG_ENDIAN uint16_t rb11_sz; uint16_t rb10_sz; uint16_t rb9_sz; @@ -694,7 +695,7 @@ typedef union { typedef union { uint64_t u64; struct { -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +#if NICVF_BYTE_ORDER == NICVF_BIG_ENDIAN uint64_t vlan_found:1; uint64_t vlan_stripped:1; uint64_t vlan2_found:1; @@ -739,7 +740,7 @@ struct cqe_rx_t { }; struct cqe_rx_tcp_err_t { -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +#if NICVF_BYTE_ORDER == NICVF_BIG_ENDIAN uint64_t cqe_type:4; /* W0 */ uint64_t rsvd0:60; @@ -761,7 +762,7 @@ struct cqe_rx_tcp_err_t { }; struct cqe_rx_tcp_t { -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +#if NICVF_BYTE_ORDER == NICVF_BIG_ENDIAN uint64_t cqe_type:4; /* W0 */ uint64_t rsvd0:52; uint64_t cq_tcp_status:8; @@ -783,7 +784,7 @@ struct cqe_rx_tcp_t { }; struct cqe_send_t { -#if defined(__BIG_ENDIAN_BITFIELD) +#if NICVF_BYTE_ORDER == NICVF_BIG_ENDIAN uint64_t cqe_type:4; /* W0 */ uint64_t rsvd0:4; uint64_t sqe_ptr:16; @@ -795,7 +796,7 @@ struct cqe_send_t { uint64_t send_status:8; uint64_t ptp_timestamp:64; /* W1 */ -#elif defined(__LITTLE_ENDIAN_BITFIELD) +#elif NICVF_BYTE_ORDER == NICVF_LITTLE_ENDIAN uint64_t send_status:8; uint64_t rsvd3:8; uint64_t sq_idx:3; @@ -811,7 +812,7 @@ struct cqe_send_t { }; struct cq_entry_type_t { -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +#if NICVF_BYTE_ORDER == NICVF_BIG_ENDIAN uint64_t cqe_type:4; uint64_t __pad:60; #else @@ -832,14 +833,14 @@ union cq_entry_t { NICVF_STATIC_ASSERT(sizeof(union cq_entry_t) == 512); struct rbdr_entry_t { -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +#if NICVF_BYTE_ORDER == NICVF_BIG_ENDIAN union { struct { uint64_t rsvd0:15; uint64_t buf_addr:42; uint64_t cache_align:7; }; - nicvf_phys_addr_t full_addr; + nicvf_iova_addr_t full_addr; }; #else union { @@ -848,7 +849,7 @@ struct rbdr_entry_t { uint64_t buf_addr:42; uint64_t rsvd0:15; }; - nicvf_phys_addr_t full_addr; + nicvf_iova_addr_t full_addr; }; #endif }; @@ -857,7 +858,7 @@ NICVF_STATIC_ASSERT(sizeof(struct rbdr_entry_t) == sizeof(uint64_t)); /* TCP reassembly context */ struct rbe_tcp_cnxt_t { -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +#if NICVF_BYTE_ORDER == NICVF_BIG_ENDIAN uint64_t tcp_pkt_cnt:12; uint64_t rsvd1:4; uint64_t align_hdr_bytes:4; @@ -896,7 +897,7 @@ struct rx_hdr_t { }; struct sq_crc_subdesc { -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +#if NICVF_BYTE_ORDER == NICVF_BIG_ENDIAN uint64_t rsvd1:32; uint64_t crc_ival:32; uint64_t subdesc_type:4; @@ -918,7 +919,7 @@ struct sq_crc_subdesc { }; struct sq_gather_subdesc { -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +#if NICVF_BYTE_ORDER == NICVF_BIG_ENDIAN uint64_t subdesc_type:4; /* W0 */ uint64_t ld_type:2; uint64_t rsvd0:42; @@ -939,7 +940,7 @@ struct sq_gather_subdesc { /* SQ immediate subdescriptor */ struct sq_imm_subdesc { -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +#if NICVF_BYTE_ORDER == NICVF_BIG_ENDIAN uint64_t subdesc_type:4; /* W0 */ uint64_t rsvd0:46; uint64_t len:14; @@ -955,7 +956,7 @@ struct sq_imm_subdesc { }; struct sq_mem_subdesc { -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +#if NICVF_BYTE_ORDER == NICVF_BIG_ENDIAN uint64_t subdesc_type:4; /* W0 */ uint64_t mem_alg:4; uint64_t mem_dsz:2; @@ -979,7 +980,7 @@ struct sq_mem_subdesc { }; struct sq_hdr_subdesc { -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +#if NICVF_BYTE_ORDER == NICVF_BIG_ENDIAN uint64_t subdesc_type:4; uint64_t tso:1; uint64_t post_cqe:1; /* Post CQE on no error also */ @@ -1042,7 +1043,7 @@ NICVF_STATIC_ASSERT(sizeof(union sq_entry_t) == 16); /* Queue config register formats */ struct rq_cfg { union { struct { -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +#if NICVF_BYTE_ORDER == NICVF_BIG_ENDIAN uint64_t reserved_2_63:62; uint64_t ena:1; uint64_t reserved_0:1; @@ -1056,7 +1057,7 @@ struct rq_cfg { union { struct { }; }; struct cq_cfg { union { struct { -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +#if NICVF_BYTE_ORDER == NICVF_BIG_ENDIAN uint64_t reserved_43_63:21; uint64_t ena:1; uint64_t reset:1; @@ -1082,8 +1083,9 @@ struct cq_cfg { union { struct { }; }; struct sq_cfg { union { struct { -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - uint64_t reserved_20_63:44; +#if NICVF_BYTE_ORDER == NICVF_BIG_ENDIAN + uint64_t reserved_32_63:32; + uint64_t cq_limit:8; uint64_t ena:1; uint64_t reserved_18_18:1; uint64_t reset:1; @@ -1101,14 +1103,15 @@ struct sq_cfg { union { struct { uint64_t reset:1; uint64_t reserved_18_18:1; uint64_t ena:1; - uint64_t reserved_20_63:44; + uint64_t cq_limit:8; + uint64_t reserved_32_63:32; #endif }; uint64_t value; }; }; struct rbdr_cfg { union { struct { -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +#if NICVF_BYTE_ORDER == NICVF_BIG_ENDIAN uint64_t reserved_45_63:19; uint64_t ena:1; uint64_t reset:1; @@ -1136,7 +1139,7 @@ struct rbdr_cfg { union { struct { }; }; struct pf_qs_cfg { union { struct { -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +#if NICVF_BYTE_ORDER == NICVF_BIG_ENDIAN uint64_t reserved_32_63:32; uint64_t ena:1; uint64_t reserved_27_30:4; @@ -1166,7 +1169,7 @@ struct pf_qs_cfg { union { struct { }; }; struct pf_rq_cfg { union { struct { -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +#if NICVF_BYTE_ORDER == NICVF_BIG_ENDIAN uint64_t reserved1:1; uint64_t reserved0:34; uint64_t strip_pre_l2:1; @@ -1194,7 +1197,7 @@ struct pf_rq_cfg { union { struct { }; }; struct pf_rq_drop_cfg { union { struct { -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +#if NICVF_BYTE_ORDER == NICVF_BIG_ENDIAN uint64_t rbdr_red:1; uint64_t cq_red:1; uint64_t reserved3:14; diff --git a/dpdk/drivers/net/thunderx/base/nicvf_mbox.c b/dpdk/drivers/net/thunderx/base/nicvf_mbox.c index 9c5cd834..e26319bb 100644 --- a/dpdk/drivers/net/thunderx/base/nicvf_mbox.c +++ b/dpdk/drivers/net/thunderx/base/nicvf_mbox.c @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2016. + * Copyright (C) Cavium, Inc. 2016. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * @@ -183,6 +183,26 @@ nicvf_handle_mbx_intr(struct nicvf *nic) nic->speed = mbx.link_status.speed; nic->pf_acked = true; break; + case NIC_MBOX_MSG_ALLOC_SQS: + assert_primary(nic); + if (mbx.sqs_alloc.qs_count != nic->sqs_count) { + nicvf_log_error("Received %" PRIu8 "/%" PRIu8 + " secondary qsets", + mbx.sqs_alloc.qs_count, + nic->sqs_count); + abort(); + } + for (i = 0; i < mbx.sqs_alloc.qs_count; i++) { + if (mbx.sqs_alloc.svf[i] != nic->snicvf[i]->vf_id) { + nicvf_log_error("Received secondary qset[%zu] " + "ID %" PRIu8 " expected %" + PRIu8, i, mbx.sqs_alloc.svf[i], + nic->snicvf[i]->vf_id); + abort(); + } + } + nic->pf_acked = true; + break; default: nicvf_log_error("Invalid message from PF, msg_id=0x%hhx %s", mbx.msg.msg, nicvf_mbox_msg_str(mbx.msg.msg)); @@ -308,16 +328,38 @@ nicvf_mbox_qset_config(struct nicvf *nic, struct pf_qs_cfg *qs_cfg) { struct nic_mbx mbx = { .msg = { 0 } }; -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +#if NICVF_BYTE_ORDER == NICVF_BIG_ENDIAN qs_cfg->be = 1; #endif /* Send a mailbox msg to PF to config Qset */ mbx.msg.msg = NIC_MBOX_MSG_QS_CFG; mbx.qs.num = nic->vf_id; + mbx.qs.sqs_count = nic->sqs_count; mbx.qs.cfg = qs_cfg->value; return nicvf_mbox_send_msg_to_pf(nic, &mbx); } +int +nicvf_mbox_request_sqs(struct nicvf *nic) +{ + struct nic_mbx mbx = { .msg = { 0 } }; + size_t i; + + assert_primary(nic); + assert(nic->sqs_count > 0); + assert(nic->sqs_count <= MAX_SQS_PER_VF); + + mbx.sqs_alloc.msg = NIC_MBOX_MSG_ALLOC_SQS; + mbx.sqs_alloc.spec = 1; + mbx.sqs_alloc.qs_count = nic->sqs_count; + + /* Set no of Rx/Tx queues in each of the SQsets */ + for (i = 0; i < nic->sqs_count; i++) + mbx.sqs_alloc.svf[i] = nic->snicvf[i]->vf_id; + + return nicvf_mbox_send_msg_to_pf(nic, &mbx); +} + int nicvf_mbox_rq_drop_config(struct nicvf *nic, uint16_t qidx, bool enable) { diff --git a/dpdk/drivers/net/thunderx/base/nicvf_mbox.h b/dpdk/drivers/net/thunderx/base/nicvf_mbox.h index 7c0c6a97..6724ae1c 100644 --- a/dpdk/drivers/net/thunderx/base/nicvf_mbox.h +++ b/dpdk/drivers/net/thunderx/base/nicvf_mbox.h @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2016. + * Copyright (C) Cavium, Inc. 2016. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * @@ -36,6 +36,7 @@ #include #include "nicvf_plat.h" +#include "../nicvf_struct.h" /* PF <--> VF Mailbox communication * Two 64bit registers are shared between PF and VF for each VF @@ -150,11 +151,20 @@ struct rss_cfg_msg { /* Physical interface link status */ struct bgx_link_status { uint8_t msg; + uint8_t mac_type; uint8_t link_up; uint8_t duplex; uint32_t speed; }; +/* Allocate additional SQS to VF */ +struct sqs_alloc { + uint8_t msg; + uint8_t spec; + uint8_t qs_count; + uint8_t svf[MAX_SQS_PER_VF]; +}; + /* Set interface in loopback mode */ struct set_loopback { uint8_t msg; @@ -201,6 +211,7 @@ union { struct rss_sz_msg rss_size; struct rss_cfg_msg rss_cfg; struct bgx_link_status link_status; + struct sqs_alloc sqs_alloc; struct set_loopback lbk; struct reset_stat_cfg reset_stat; }; @@ -211,6 +222,7 @@ NICVF_STATIC_ASSERT(sizeof(struct nic_mbx) <= 16); int nicvf_handle_mbx_intr(struct nicvf *nic); int nicvf_mbox_check_pf_ready(struct nicvf *nic); int nicvf_mbox_qset_config(struct nicvf *nic, struct pf_qs_cfg *qs_cfg); +int nicvf_mbox_request_sqs(struct nicvf *nic); int nicvf_mbox_rq_config(struct nicvf *nic, uint16_t qidx, struct pf_rq_cfg *pf_rq_cfg); int nicvf_mbox_sq_config(struct nicvf *nic, uint16_t qidx); diff --git a/dpdk/drivers/net/thunderx/base/nicvf_plat.h b/dpdk/drivers/net/thunderx/base/nicvf_plat.h index 83c1844d..f821c56f 100644 --- a/dpdk/drivers/net/thunderx/base/nicvf_plat.h +++ b/dpdk/drivers/net/thunderx/base/nicvf_plat.h @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2016. + * Copyright (C) Cavium, Inc. 2016. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * @@ -59,41 +59,30 @@ /* utils */ #include #define nicvf_min(x, y) RTE_MIN(x, y) +#define nicvf_log2_u32(x) rte_log2_u32(x) /* byte order */ #include #define nicvf_cpu_to_be_64(x) rte_cpu_to_be_64(x) #define nicvf_be_to_cpu_64(x) rte_be_to_cpu_64(x) +#define NICVF_BYTE_ORDER RTE_BYTE_ORDER +#define NICVF_BIG_ENDIAN RTE_BIG_ENDIAN +#define NICVF_LITTLE_ENDIAN RTE_LITTLE_ENDIAN + /* Constants */ #include #define NICVF_MAC_ADDR_SIZE ETHER_ADDR_LEN +#include +#define nicvf_addr_write(addr, val) rte_write64_relaxed((val), (void *)(addr)) +#define nicvf_addr_read(addr) rte_read64_relaxed((void *)(addr)) + /* ARM64 specific functions */ #if defined(RTE_ARCH_ARM64) #define nicvf_prefetch_store_keep(_ptr) ({\ - asm volatile("prfm pstl1keep, %a0\n" : : "p" (_ptr)); }) + asm volatile("prfm pstl1keep, [%x0]\n" : : "r" (_ptr)); }) -static inline void __attribute__((always_inline)) -nicvf_addr_write(uintptr_t addr, uint64_t val) -{ - asm volatile( - "str %x[val], [%x[addr]]" - : - : [val] "r" (val), [addr] "r" (addr)); -} - -static inline uint64_t __attribute__((always_inline)) -nicvf_addr_read(uintptr_t addr) -{ - uint64_t val; - - asm volatile( - "ldr %x[val], [%x[addr]]" - : [val] "=r" (val) - : [addr] "r" (addr)); - return val; -} #define NICVF_LOAD_PAIR(reg1, reg2, addr) ({ \ asm volatile( \ @@ -106,18 +95,6 @@ nicvf_addr_read(uintptr_t addr) #define nicvf_prefetch_store_keep(_ptr) do {} while (0) -static inline void __attribute__((always_inline)) -nicvf_addr_write(uintptr_t addr, uint64_t val) -{ - *(volatile uint64_t *)addr = val; -} - -static inline uint64_t __attribute__((always_inline)) -nicvf_addr_read(uintptr_t addr) -{ - return *(volatile uint64_t *)addr; -} - #define NICVF_LOAD_PAIR(reg1, reg2, addr) \ do { \ reg1 = nicvf_addr_read((uintptr_t)addr); \ diff --git a/dpdk/drivers/net/thunderx/nicvf_ethdev.c b/dpdk/drivers/net/thunderx/nicvf_ethdev.c index 4f875c02..c62371cb 100644 --- a/dpdk/drivers/net/thunderx/nicvf_ethdev.c +++ b/dpdk/drivers/net/thunderx/nicvf_ethdev.c @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2016. + * Copyright (C) Cavium, Inc. 2016. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * @@ -41,7 +41,6 @@ #include #include #include -#include #include #include @@ -54,6 +53,7 @@ #include #include #include +#include #include #include #include @@ -61,15 +61,20 @@ #include #include #include +#include #include #include "base/nicvf_plat.h" #include "nicvf_ethdev.h" #include "nicvf_rxtx.h" +#include "nicvf_svf.h" #include "nicvf_logs.h" static void nicvf_dev_stop(struct rte_eth_dev *dev); +static void nicvf_dev_stop_cleanup(struct rte_eth_dev *dev, bool cleanup); +static void nicvf_vf_stop(struct rte_eth_dev *dev, struct nicvf *nic, + bool cleanup); static inline int nicvf_atomic_write_link_status(struct rte_eth_dev *dev, @@ -95,53 +100,76 @@ nicvf_set_eth_link_status(struct nicvf *nic, struct rte_eth_link *link) else if (nic->duplex == NICVF_FULL_DUPLEX) link->link_duplex = ETH_LINK_FULL_DUPLEX; link->link_speed = nic->speed; - link->link_autoneg = ETH_LINK_SPEED_AUTONEG; + link->link_autoneg = ETH_LINK_AUTONEG; } static void nicvf_interrupt(void *arg) { - struct nicvf *nic = arg; + struct rte_eth_dev *dev = arg; + struct nicvf *nic = nicvf_pmd_priv(dev); if (nicvf_reg_poll_interrupts(nic) == NIC_MBOX_MSG_BGX_LINK_CHANGE) { - if (nic->eth_dev->data->dev_conf.intr_conf.lsc) - nicvf_set_eth_link_status(nic, - &nic->eth_dev->data->dev_link); - _rte_eth_dev_callback_process(nic->eth_dev, - RTE_ETH_EVENT_INTR_LSC); + if (dev->data->dev_conf.intr_conf.lsc) + nicvf_set_eth_link_status(nic, &dev->data->dev_link); + _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, + NULL, NULL); } rte_eal_alarm_set(NICVF_INTR_POLL_INTERVAL_MS * 1000, - nicvf_interrupt, nic); + nicvf_interrupt, dev); +} + +static void +nicvf_vf_interrupt(void *arg) +{ + struct nicvf *nic = arg; + + nicvf_reg_poll_interrupts(nic); + + rte_eal_alarm_set(NICVF_INTR_POLL_INTERVAL_MS * 1000, + nicvf_vf_interrupt, nic); } static int -nicvf_periodic_alarm_start(struct nicvf *nic) +nicvf_periodic_alarm_start(void (fn)(void *), void *arg) { - return rte_eal_alarm_set(NICVF_INTR_POLL_INTERVAL_MS * 1000, - nicvf_interrupt, nic); + return rte_eal_alarm_set(NICVF_INTR_POLL_INTERVAL_MS * 1000, fn, arg); } static int -nicvf_periodic_alarm_stop(struct nicvf *nic) +nicvf_periodic_alarm_stop(void (fn)(void *), void *arg) { - return rte_eal_alarm_cancel(nicvf_interrupt, nic); + return rte_eal_alarm_cancel(fn, arg); } /* * Return 0 means link status changed, -1 means not changed */ static int -nicvf_dev_link_update(struct rte_eth_dev *dev, - int wait_to_complete __rte_unused) +nicvf_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete) { +#define CHECK_INTERVAL 100 /* 100ms */ +#define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ struct rte_eth_link link; struct nicvf *nic = nicvf_pmd_priv(dev); + int i; PMD_INIT_FUNC_TRACE(); - memset(&link, 0, sizeof(link)); - nicvf_set_eth_link_status(nic, &link); + if (wait_to_complete) { + /* rte_eth_link_get() might need to wait up to 9 seconds */ + for (i = 0; i < MAX_CHECK_TIME; i++) { + memset(&link, 0, sizeof(link)); + nicvf_set_eth_link_status(nic, &link); + if (link.link_status) + break; + rte_delay_ms(CHECK_INTERVAL); + } + } else { + memset(&link, 0, sizeof(link)); + nicvf_set_eth_link_status(nic, &link); + } return nicvf_atomic_write_link_status(dev, &link); } @@ -150,6 +178,7 @@ nicvf_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu) { struct nicvf *nic = nicvf_pmd_priv(dev); uint32_t buffsz, frame_size = mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; + size_t i; PMD_INIT_FUNC_TRACE(); @@ -185,6 +214,10 @@ nicvf_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu) /* Update max frame size */ dev->data->dev_conf.rxmode.max_rx_pkt_len = (uint32_t)frame_size; nic->mtu = mtu; + + for (i = 0; i < nic->sqs_count; i++) + nic->snicvf[i]->mtu = mtu; + return 0; } @@ -210,7 +243,7 @@ nicvf_dev_get_regs(struct rte_eth_dev *dev, struct rte_dev_reg_info *regs) return -ENOTSUP; } -static void +static int nicvf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) { uint16_t qidx; @@ -218,10 +251,16 @@ nicvf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) struct nicvf_hw_tx_qstats tx_qstats; struct nicvf_hw_stats port_stats; struct nicvf *nic = nicvf_pmd_priv(dev); + uint16_t rx_start, rx_end; + uint16_t tx_start, tx_end; + size_t i; + + /* RX queue indices for the first VF */ + nicvf_rx_range(dev, nic, &rx_start, &rx_end); /* Reading per RX ring stats */ - for (qidx = 0; qidx < dev->data->nb_rx_queues; qidx++) { - if (qidx == RTE_ETHDEV_QUEUE_STAT_CNTRS) + for (qidx = rx_start; qidx <= rx_end; qidx++) { + if (qidx >= RTE_ETHDEV_QUEUE_STAT_CNTRS) break; nicvf_hw_get_rx_qstats(nic, &rx_qstats, qidx); @@ -229,9 +268,12 @@ nicvf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) stats->q_ipackets[qidx] = rx_qstats.q_rx_packets; } + /* TX queue indices for the first VF */ + nicvf_tx_range(dev, nic, &tx_start, &tx_end); + /* Reading per TX ring stats */ - for (qidx = 0; qidx < dev->data->nb_tx_queues; qidx++) { - if (qidx == RTE_ETHDEV_QUEUE_STAT_CNTRS) + for (qidx = tx_start; qidx <= tx_end; qidx++) { + if (qidx >= RTE_ETHDEV_QUEUE_STAT_CNTRS) break; nicvf_hw_get_tx_qstats(nic, &tx_qstats, qidx); @@ -239,6 +281,40 @@ nicvf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) stats->q_opackets[qidx] = tx_qstats.q_tx_packets; } + for (i = 0; i < nic->sqs_count; i++) { + struct nicvf *snic = nic->snicvf[i]; + + if (snic == NULL) + break; + + /* RX queue indices for a secondary VF */ + nicvf_rx_range(dev, snic, &rx_start, &rx_end); + + /* Reading per RX ring stats */ + for (qidx = rx_start; qidx <= rx_end; qidx++) { + if (qidx >= RTE_ETHDEV_QUEUE_STAT_CNTRS) + break; + + nicvf_hw_get_rx_qstats(snic, &rx_qstats, + qidx % MAX_RCV_QUEUES_PER_QS); + stats->q_ibytes[qidx] = rx_qstats.q_rx_bytes; + stats->q_ipackets[qidx] = rx_qstats.q_rx_packets; + } + + /* TX queue indices for a secondary VF */ + nicvf_tx_range(dev, snic, &tx_start, &tx_end); + /* Reading per TX ring stats */ + for (qidx = tx_start; qidx <= tx_end; qidx++) { + if (qidx >= RTE_ETHDEV_QUEUE_STAT_CNTRS) + break; + + nicvf_hw_get_tx_qstats(snic, &tx_qstats, + qidx % MAX_SND_QUEUES_PER_QS); + stats->q_obytes[qidx] = tx_qstats.q_tx_bytes; + stats->q_opackets[qidx] = tx_qstats.q_tx_packets; + } + } + nicvf_hw_get_stats(nic, &port_stats); stats->ibytes = port_stats.rx_bytes; stats->ipackets = port_stats.rx_ucast_frames; @@ -257,6 +333,8 @@ nicvf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) stats->opackets += port_stats.tx_bcast_frames_ok; stats->opackets += port_stats.tx_mcast_frames_ok; stats->oerrors = port_stats.tx_drops; + + return 0; } static const uint32_t * @@ -265,7 +343,7 @@ nicvf_dev_supported_ptypes_get(struct rte_eth_dev *dev) size_t copied; static uint32_t ptypes[32]; struct nicvf *nic = nicvf_pmd_priv(dev); - static const uint32_t ptypes_pass1[] = { + static const uint32_t ptypes_common[] = { RTE_PTYPE_L3_IPV4, RTE_PTYPE_L3_IPV4_EXT, RTE_PTYPE_L3_IPV6, @@ -274,7 +352,7 @@ nicvf_dev_supported_ptypes_get(struct rte_eth_dev *dev) RTE_PTYPE_L4_UDP, RTE_PTYPE_L4_FRAG, }; - static const uint32_t ptypes_pass2[] = { + static const uint32_t ptypes_tunnel[] = { RTE_PTYPE_TUNNEL_GRE, RTE_PTYPE_TUNNEL_GENEVE, RTE_PTYPE_TUNNEL_VXLAN, @@ -282,12 +360,12 @@ nicvf_dev_supported_ptypes_get(struct rte_eth_dev *dev) }; static const uint32_t ptypes_end = RTE_PTYPE_UNKNOWN; - copied = sizeof(ptypes_pass1); - memcpy(ptypes, ptypes_pass1, copied); - if (nicvf_hw_version(nic) == NICVF_PASS2) { - memcpy((char *)ptypes + copied, ptypes_pass2, - sizeof(ptypes_pass2)); - copied += sizeof(ptypes_pass2); + copied = sizeof(ptypes_common); + memcpy(ptypes, ptypes_common, copied); + if (nicvf_hw_cap(nic) & NICVF_CAP_TUNNEL_PARSING) { + memcpy((char *)ptypes + copied, ptypes_tunnel, + sizeof(ptypes_tunnel)); + copied += sizeof(ptypes_tunnel); } memcpy((char *)ptypes + copied, &ptypes_end, sizeof(ptypes_end)); @@ -304,13 +382,36 @@ nicvf_dev_stats_reset(struct rte_eth_dev *dev) int i; uint16_t rxqs = 0, txqs = 0; struct nicvf *nic = nicvf_pmd_priv(dev); + uint16_t rx_start, rx_end; + uint16_t tx_start, tx_end; - for (i = 0; i < dev->data->nb_rx_queues; i++) + /* Reset all primary nic counters */ + nicvf_rx_range(dev, nic, &rx_start, &rx_end); + for (i = rx_start; i <= rx_end; i++) rxqs |= (0x3 << (i * 2)); - for (i = 0; i < dev->data->nb_tx_queues; i++) + + nicvf_tx_range(dev, nic, &tx_start, &tx_end); + for (i = tx_start; i <= tx_end; i++) txqs |= (0x3 << (i * 2)); nicvf_mbox_reset_stat_counters(nic, 0x3FFF, 0x1F, rxqs, txqs); + + /* Reset secondary nic queue counters */ + for (i = 0; i < nic->sqs_count; i++) { + struct nicvf *snic = nic->snicvf[i]; + if (snic == NULL) + break; + + nicvf_rx_range(dev, snic, &rx_start, &rx_end); + for (i = rx_start; i <= rx_end; i++) + rxqs |= (0x3 << ((i % MAX_CMP_QUEUES_PER_QS) * 2)); + + nicvf_tx_range(dev, snic, &tx_start, &tx_end); + for (i = tx_start; i <= tx_end; i++) + txqs |= (0x3 << ((i % MAX_SND_QUEUES_PER_QS) * 2)); + + nicvf_mbox_reset_stat_counters(snic, 0, 0, rxqs, txqs); + } } /* Promiscuous mode enabled by default in LMAC to VF 1:1 map configuration */ @@ -488,14 +589,15 @@ nicvf_dev_rss_hash_update(struct rte_eth_dev *dev, } static int -nicvf_qset_cq_alloc(struct nicvf *nic, struct nicvf_rxq *rxq, uint16_t qidx, - uint32_t desc_cnt) +nicvf_qset_cq_alloc(struct rte_eth_dev *dev, struct nicvf *nic, + struct nicvf_rxq *rxq, uint16_t qidx, uint32_t desc_cnt) { const struct rte_memzone *rz; uint32_t ring_size = CMP_QUEUE_SZ_MAX * sizeof(union cq_entry_t); - rz = rte_eth_dma_zone_reserve(nic->eth_dev, "cq_ring", qidx, ring_size, - NICVF_CQ_BASE_ALIGN_BYTES, nic->node); + rz = rte_eth_dma_zone_reserve(dev, "cq_ring", + nicvf_netdev_qidx(nic, qidx), ring_size, + NICVF_CQ_BASE_ALIGN_BYTES, nic->node); if (rz == NULL) { PMD_INIT_LOG(ERR, "Failed to allocate mem for cq hw ring"); return -ENOMEM; @@ -503,7 +605,7 @@ nicvf_qset_cq_alloc(struct nicvf *nic, struct nicvf_rxq *rxq, uint16_t qidx, memset(rz->addr, 0, ring_size); - rxq->phys = rz->phys_addr; + rxq->phys = rz->iova; rxq->desc = rz->addr; rxq->qlen_mask = desc_cnt - 1; @@ -511,14 +613,15 @@ nicvf_qset_cq_alloc(struct nicvf *nic, struct nicvf_rxq *rxq, uint16_t qidx, } static int -nicvf_qset_sq_alloc(struct nicvf *nic, struct nicvf_txq *sq, uint16_t qidx, - uint32_t desc_cnt) +nicvf_qset_sq_alloc(struct rte_eth_dev *dev, struct nicvf *nic, + struct nicvf_txq *sq, uint16_t qidx, uint32_t desc_cnt) { const struct rte_memzone *rz; uint32_t ring_size = SND_QUEUE_SZ_MAX * sizeof(union sq_entry_t); - rz = rte_eth_dma_zone_reserve(nic->eth_dev, "sq", qidx, ring_size, - NICVF_SQ_BASE_ALIGN_BYTES, nic->node); + rz = rte_eth_dma_zone_reserve(dev, "sq", + nicvf_netdev_qidx(nic, qidx), ring_size, + NICVF_SQ_BASE_ALIGN_BYTES, nic->node); if (rz == NULL) { PMD_INIT_LOG(ERR, "Failed allocate mem for sq hw ring"); return -ENOMEM; @@ -526,7 +629,7 @@ nicvf_qset_sq_alloc(struct nicvf *nic, struct nicvf_txq *sq, uint16_t qidx, memset(rz->addr, 0, ring_size); - sq->phys = rz->phys_addr; + sq->phys = rz->iova; sq->desc = rz->addr; sq->qlen_mask = desc_cnt - 1; @@ -534,7 +637,8 @@ nicvf_qset_sq_alloc(struct nicvf *nic, struct nicvf_txq *sq, uint16_t qidx, } static int -nicvf_qset_rbdr_alloc(struct nicvf *nic, uint32_t desc_cnt, uint32_t buffsz) +nicvf_qset_rbdr_alloc(struct rte_eth_dev *dev, struct nicvf *nic, + uint32_t desc_cnt, uint32_t buffsz) { struct nicvf_rbdr *rbdr; const struct rte_memzone *rz; @@ -549,8 +653,9 @@ nicvf_qset_rbdr_alloc(struct nicvf *nic, uint32_t desc_cnt, uint32_t buffsz) } ring_size = sizeof(struct rbdr_entry_t) * RBDR_QUEUE_SZ_MAX; - rz = rte_eth_dma_zone_reserve(nic->eth_dev, "rbdr", 0, ring_size, - NICVF_RBDR_BASE_ALIGN_BYTES, nic->node); + rz = rte_eth_dma_zone_reserve(dev, "rbdr", + nicvf_netdev_qidx(nic, 0), ring_size, + NICVF_RBDR_BASE_ALIGN_BYTES, nic->node); if (rz == NULL) { PMD_INIT_LOG(ERR, "Failed to allocate mem for rbdr desc ring"); return -ENOMEM; @@ -558,7 +663,7 @@ nicvf_qset_rbdr_alloc(struct nicvf *nic, uint32_t desc_cnt, uint32_t buffsz) memset(rz->addr, 0, ring_size); - rbdr->phys = rz->phys_addr; + rbdr->phys = rz->iova; rbdr->tail = 0; rbdr->next_tail = 0; rbdr->desc = rz->addr; @@ -574,14 +679,19 @@ nicvf_qset_rbdr_alloc(struct nicvf *nic, uint32_t desc_cnt, uint32_t buffsz) } static void -nicvf_rbdr_release_mbuf(struct nicvf *nic, nicvf_phys_addr_t phy) +nicvf_rbdr_release_mbuf(struct rte_eth_dev *dev, struct nicvf *nic, + nicvf_iova_addr_t phy) { uint16_t qidx; void *obj; struct nicvf_rxq *rxq; + uint16_t rx_start, rx_end; - for (qidx = 0; qidx < nic->eth_dev->data->nb_rx_queues; qidx++) { - rxq = nic->eth_dev->data->rx_queues[qidx]; + /* Get queue ranges for this VF */ + nicvf_rx_range(dev, nic, &rx_start, &rx_end); + + for (qidx = rx_start; qidx <= rx_end; qidx++) { + rxq = dev->data->rx_queues[qidx]; if (rxq->precharge_cnt) { obj = (void *)nicvf_mbuff_phy2virt(phy, rxq->mbuf_phys_off); @@ -593,7 +703,7 @@ nicvf_rbdr_release_mbuf(struct nicvf *nic, nicvf_phys_addr_t phy) } static inline void -nicvf_rbdr_release_mbufs(struct nicvf *nic) +nicvf_rbdr_release_mbufs(struct rte_eth_dev *dev, struct nicvf *nic) { uint32_t qlen_mask, head; struct rbdr_entry_t *entry; @@ -603,7 +713,7 @@ nicvf_rbdr_release_mbufs(struct nicvf *nic) head = rbdr->head; while (head != rbdr->tail) { entry = rbdr->desc + head; - nicvf_rbdr_release_mbuf(nic, entry->full_addr); + nicvf_rbdr_release_mbuf(dev, nic, entry->full_addr); head++; head = head & qlen_mask; } @@ -638,48 +748,60 @@ nicvf_tx_queue_reset(struct nicvf_txq *txq) } static inline int -nicvf_start_tx_queue(struct rte_eth_dev *dev, uint16_t qidx) +nicvf_vf_start_tx_queue(struct rte_eth_dev *dev, struct nicvf *nic, + uint16_t qidx) { struct nicvf_txq *txq; int ret; - if (dev->data->tx_queue_state[qidx] == RTE_ETH_QUEUE_STATE_STARTED) + assert(qidx < MAX_SND_QUEUES_PER_QS); + + if (dev->data->tx_queue_state[nicvf_netdev_qidx(nic, qidx)] == + RTE_ETH_QUEUE_STATE_STARTED) return 0; - txq = dev->data->tx_queues[qidx]; + txq = dev->data->tx_queues[nicvf_netdev_qidx(nic, qidx)]; txq->pool = NULL; - ret = nicvf_qset_sq_config(nicvf_pmd_priv(dev), qidx, txq); + ret = nicvf_qset_sq_config(nic, qidx, txq); if (ret) { - PMD_INIT_LOG(ERR, "Failed to configure sq %d %d", qidx, ret); + PMD_INIT_LOG(ERR, "Failed to configure sq VF%d %d %d", + nic->vf_id, qidx, ret); goto config_sq_error; } - dev->data->tx_queue_state[qidx] = RTE_ETH_QUEUE_STATE_STARTED; + dev->data->tx_queue_state[nicvf_netdev_qidx(nic, qidx)] = + RTE_ETH_QUEUE_STATE_STARTED; return ret; config_sq_error: - nicvf_qset_sq_reclaim(nicvf_pmd_priv(dev), qidx); + nicvf_qset_sq_reclaim(nic, qidx); return ret; } static inline int -nicvf_stop_tx_queue(struct rte_eth_dev *dev, uint16_t qidx) +nicvf_vf_stop_tx_queue(struct rte_eth_dev *dev, struct nicvf *nic, + uint16_t qidx) { struct nicvf_txq *txq; int ret; - if (dev->data->tx_queue_state[qidx] == RTE_ETH_QUEUE_STATE_STOPPED) + assert(qidx < MAX_SND_QUEUES_PER_QS); + + if (dev->data->tx_queue_state[nicvf_netdev_qidx(nic, qidx)] == + RTE_ETH_QUEUE_STATE_STOPPED) return 0; - ret = nicvf_qset_sq_reclaim(nicvf_pmd_priv(dev), qidx); + ret = nicvf_qset_sq_reclaim(nic, qidx); if (ret) - PMD_INIT_LOG(ERR, "Failed to reclaim sq %d %d", qidx, ret); + PMD_INIT_LOG(ERR, "Failed to reclaim sq VF%d %d %d", + nic->vf_id, qidx, ret); - txq = dev->data->tx_queues[qidx]; + txq = dev->data->tx_queues[nicvf_netdev_qidx(nic, qidx)]; nicvf_tx_queue_release_mbufs(txq); nicvf_tx_queue_reset(txq); - dev->data->tx_queue_state[qidx] = RTE_ETH_QUEUE_STATE_STOPPED; + dev->data->tx_queue_state[nicvf_netdev_qidx(nic, qidx)] = + RTE_ETH_QUEUE_STATE_STOPPED; return ret; } @@ -691,7 +813,7 @@ nicvf_configure_cpi(struct rte_eth_dev *dev) int ret; /* Count started rx queues */ - for (qidx = qcnt = 0; qidx < nic->eth_dev->data->nb_rx_queues; qidx++) + for (qidx = qcnt = 0; qidx < dev->data->nb_rx_queues; qidx++) if (dev->data->rx_queue_state[qidx] == RTE_ETH_QUEUE_STATE_STARTED) qcnt++; @@ -715,14 +837,13 @@ nicvf_configure_rss(struct rte_eth_dev *dev) dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf); PMD_DRV_LOG(INFO, "mode=%d rx_queues=%d loopback=%d rsshf=0x%" PRIx64, dev->data->dev_conf.rxmode.mq_mode, - nic->eth_dev->data->nb_rx_queues, - nic->eth_dev->data->dev_conf.lpbk_mode, rsshf); + dev->data->nb_rx_queues, + dev->data->dev_conf.lpbk_mode, rsshf); if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_NONE) ret = nicvf_rss_term(nic); else if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_RSS) - ret = nicvf_rss_config(nic, - nic->eth_dev->data->nb_rx_queues, rsshf); + ret = nicvf_rss_config(nic, dev->data->nb_rx_queues, rsshf); if (ret) PMD_INIT_LOG(ERR, "Failed to configure RSS %d", ret); @@ -827,6 +948,11 @@ nicvf_dev_tx_queue_setup(struct rte_eth_dev *dev, uint16_t qidx, PMD_INIT_FUNC_TRACE(); + if (qidx >= MAX_SND_QUEUES_PER_QS) + nic = nic->snicvf[qidx / MAX_SND_QUEUES_PER_QS - 1]; + + qidx = qidx % MAX_SND_QUEUES_PER_QS; + /* Socket id check */ if (socket_id != (unsigned int)SOCKET_ID_ANY && socket_id != nic->node) PMD_DRV_LOG(WARNING, "socket_id expected %d, configured %d", @@ -861,18 +987,20 @@ nicvf_dev_tx_queue_setup(struct rte_eth_dev *dev, uint16_t qidx, } /* Free memory prior to re-allocation if needed. */ - if (dev->data->tx_queues[qidx] != NULL) { + if (dev->data->tx_queues[nicvf_netdev_qidx(nic, qidx)] != NULL) { PMD_TX_LOG(DEBUG, "Freeing memory prior to re-allocation %d", - qidx); - nicvf_dev_tx_queue_release(dev->data->tx_queues[qidx]); - dev->data->tx_queues[qidx] = NULL; + nicvf_netdev_qidx(nic, qidx)); + nicvf_dev_tx_queue_release( + dev->data->tx_queues[nicvf_netdev_qidx(nic, qidx)]); + dev->data->tx_queues[nicvf_netdev_qidx(nic, qidx)] = NULL; } /* Allocating tx queue data structure */ txq = rte_zmalloc_socket("ethdev TX queue", sizeof(struct nicvf_txq), RTE_CACHE_LINE_SIZE, nic->node); if (txq == NULL) { - PMD_INIT_LOG(ERR, "Failed to allocate txq=%d", qidx); + PMD_INIT_LOG(ERR, "Failed to allocate txq=%d", + nicvf_netdev_qidx(nic, qidx)); return -ENOMEM; } @@ -906,7 +1034,7 @@ nicvf_dev_tx_queue_setup(struct rte_eth_dev *dev, uint16_t qidx, return -ENOMEM; } - if (nicvf_qset_sq_alloc(nic, txq, qidx, nb_desc)) { + if (nicvf_qset_sq_alloc(dev, nic, txq, qidx, nb_desc)) { PMD_INIT_LOG(ERR, "Failed to allocate mem for sq %d", qidx); nicvf_dev_tx_queue_release(txq); return -ENOMEM; @@ -915,26 +1043,28 @@ nicvf_dev_tx_queue_setup(struct rte_eth_dev *dev, uint16_t qidx, nicvf_tx_queue_reset(txq); PMD_TX_LOG(DEBUG, "[%d] txq=%p nb_desc=%d desc=%p phys=0x%" PRIx64, - qidx, txq, nb_desc, txq->desc, txq->phys); + nicvf_netdev_qidx(nic, qidx), txq, nb_desc, txq->desc, + txq->phys); - dev->data->tx_queues[qidx] = txq; - dev->data->tx_queue_state[qidx] = RTE_ETH_QUEUE_STATE_STOPPED; + dev->data->tx_queues[nicvf_netdev_qidx(nic, qidx)] = txq; + dev->data->tx_queue_state[nicvf_netdev_qidx(nic, qidx)] = + RTE_ETH_QUEUE_STATE_STOPPED; return 0; } static inline void -nicvf_rx_queue_release_mbufs(struct nicvf_rxq *rxq) +nicvf_rx_queue_release_mbufs(struct rte_eth_dev *dev, struct nicvf_rxq *rxq) { uint32_t rxq_cnt; uint32_t nb_pkts, released_pkts = 0; uint32_t refill_cnt = 0; - struct rte_eth_dev *dev = rxq->nic->eth_dev; struct rte_mbuf *rx_pkts[NICVF_MAX_RX_FREE_THRESH]; if (dev->rx_pkt_burst == NULL) return; - while ((rxq_cnt = nicvf_dev_rx_queue_count(dev, rxq->queue_id))) { + while ((rxq_cnt = nicvf_dev_rx_queue_count(dev, + nicvf_netdev_qidx(rxq->nic, rxq->queue_id)))) { nb_pkts = dev->rx_pkt_burst(rxq, rx_pkts, NICVF_MAX_RX_FREE_THRESH); PMD_DRV_LOG(INFO, "nb_pkts=%d rxq_cnt=%d", nb_pkts, rxq_cnt); @@ -944,7 +1074,10 @@ nicvf_rx_queue_release_mbufs(struct nicvf_rxq *rxq) } } - refill_cnt += nicvf_dev_rbdr_refill(dev, rxq->queue_id); + + refill_cnt += nicvf_dev_rbdr_refill(dev, + nicvf_netdev_qidx(rxq->nic, rxq->queue_id)); + PMD_DRV_LOG(INFO, "free_cnt=%d refill_cnt=%d", released_pkts, refill_cnt); } @@ -958,31 +1091,37 @@ nicvf_rx_queue_reset(struct nicvf_rxq *rxq) } static inline int -nicvf_start_rx_queue(struct rte_eth_dev *dev, uint16_t qidx) +nicvf_vf_start_rx_queue(struct rte_eth_dev *dev, struct nicvf *nic, + uint16_t qidx) { - struct nicvf *nic = nicvf_pmd_priv(dev); struct nicvf_rxq *rxq; int ret; - if (dev->data->rx_queue_state[qidx] == RTE_ETH_QUEUE_STATE_STARTED) + assert(qidx < MAX_RCV_QUEUES_PER_QS); + + if (dev->data->rx_queue_state[nicvf_netdev_qidx(nic, qidx)] == + RTE_ETH_QUEUE_STATE_STARTED) return 0; /* Update rbdr pointer to all rxq */ - rxq = dev->data->rx_queues[qidx]; + rxq = dev->data->rx_queues[nicvf_netdev_qidx(nic, qidx)]; rxq->shared_rbdr = nic->rbdr; ret = nicvf_qset_rq_config(nic, qidx, rxq); if (ret) { - PMD_INIT_LOG(ERR, "Failed to configure rq %d %d", qidx, ret); + PMD_INIT_LOG(ERR, "Failed to configure rq VF%d %d %d", + nic->vf_id, qidx, ret); goto config_rq_error; } ret = nicvf_qset_cq_config(nic, qidx, rxq); if (ret) { - PMD_INIT_LOG(ERR, "Failed to configure cq %d %d", qidx, ret); + PMD_INIT_LOG(ERR, "Failed to configure cq VF%d %d %d", + nic->vf_id, qidx, ret); goto config_cq_error; } - dev->data->rx_queue_state[qidx] = RTE_ETH_QUEUE_STATE_STARTED; + dev->data->rx_queue_state[nicvf_netdev_qidx(nic, qidx)] = + RTE_ETH_QUEUE_STATE_STARTED; return 0; config_cq_error: @@ -993,50 +1132,57 @@ config_rq_error: } static inline int -nicvf_stop_rx_queue(struct rte_eth_dev *dev, uint16_t qidx) +nicvf_vf_stop_rx_queue(struct rte_eth_dev *dev, struct nicvf *nic, + uint16_t qidx) { - struct nicvf *nic = nicvf_pmd_priv(dev); struct nicvf_rxq *rxq; int ret, other_error; - if (dev->data->rx_queue_state[qidx] == RTE_ETH_QUEUE_STATE_STOPPED) + if (dev->data->rx_queue_state[nicvf_netdev_qidx(nic, qidx)] == + RTE_ETH_QUEUE_STATE_STOPPED) return 0; ret = nicvf_qset_rq_reclaim(nic, qidx); if (ret) - PMD_INIT_LOG(ERR, "Failed to reclaim rq %d %d", qidx, ret); + PMD_INIT_LOG(ERR, "Failed to reclaim rq VF%d %d %d", + nic->vf_id, qidx, ret); other_error = ret; - rxq = dev->data->rx_queues[qidx]; - nicvf_rx_queue_release_mbufs(rxq); + rxq = dev->data->rx_queues[nicvf_netdev_qidx(nic, qidx)]; + nicvf_rx_queue_release_mbufs(dev, rxq); nicvf_rx_queue_reset(rxq); ret = nicvf_qset_cq_reclaim(nic, qidx); if (ret) - PMD_INIT_LOG(ERR, "Failed to reclaim cq %d %d", qidx, ret); + PMD_INIT_LOG(ERR, "Failed to reclaim cq VF%d %d %d", + nic->vf_id, qidx, ret); other_error |= ret; - dev->data->rx_queue_state[qidx] = RTE_ETH_QUEUE_STATE_STOPPED; + dev->data->rx_queue_state[nicvf_netdev_qidx(nic, qidx)] = + RTE_ETH_QUEUE_STATE_STOPPED; return other_error; } static void nicvf_dev_rx_queue_release(void *rx_queue) { - struct nicvf_rxq *rxq = rx_queue; - PMD_INIT_FUNC_TRACE(); - if (rxq) - rte_free(rxq); + rte_free(rx_queue); } static int nicvf_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t qidx) { + struct nicvf *nic = nicvf_pmd_priv(dev); int ret; - ret = nicvf_start_rx_queue(dev, qidx); + if (qidx >= MAX_RCV_QUEUES_PER_QS) + nic = nic->snicvf[(qidx / MAX_RCV_QUEUES_PER_QS - 1)]; + + qidx = qidx % MAX_RCV_QUEUES_PER_QS; + + ret = nicvf_vf_start_rx_queue(dev, nic, qidx); if (ret) return ret; @@ -1051,8 +1197,14 @@ static int nicvf_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t qidx) { int ret; + struct nicvf *nic = nicvf_pmd_priv(dev); - ret = nicvf_stop_rx_queue(dev, qidx); + if (qidx >= MAX_SND_QUEUES_PER_QS) + nic = nic->snicvf[(qidx / MAX_SND_QUEUES_PER_QS - 1)]; + + qidx = qidx % MAX_RCV_QUEUES_PER_QS; + + ret = nicvf_vf_stop_rx_queue(dev, nic, qidx); ret |= nicvf_configure_cpi(dev); ret |= nicvf_configure_rss_reta(dev); return ret; @@ -1061,13 +1213,52 @@ nicvf_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t qidx) static int nicvf_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t qidx) { - return nicvf_start_tx_queue(dev, qidx); + struct nicvf *nic = nicvf_pmd_priv(dev); + + if (qidx >= MAX_SND_QUEUES_PER_QS) + nic = nic->snicvf[(qidx / MAX_SND_QUEUES_PER_QS - 1)]; + + qidx = qidx % MAX_SND_QUEUES_PER_QS; + + return nicvf_vf_start_tx_queue(dev, nic, qidx); } static int nicvf_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t qidx) { - return nicvf_stop_tx_queue(dev, qidx); + struct nicvf *nic = nicvf_pmd_priv(dev); + + if (qidx >= MAX_SND_QUEUES_PER_QS) + nic = nic->snicvf[(qidx / MAX_SND_QUEUES_PER_QS - 1)]; + + qidx = qidx % MAX_SND_QUEUES_PER_QS; + + return nicvf_vf_stop_tx_queue(dev, nic, qidx); +} + +static inline void +nicvf_rxq_mbuf_setup(struct nicvf_rxq *rxq) +{ + uintptr_t p; + struct rte_mbuf mb_def; + + RTE_BUILD_BUG_ON(sizeof(union mbuf_initializer) != 8); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_off) % 8 != 0); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, refcnt) - + offsetof(struct rte_mbuf, data_off) != 2); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, nb_segs) - + offsetof(struct rte_mbuf, data_off) != 4); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, port) - + offsetof(struct rte_mbuf, data_off) != 6); + mb_def.nb_segs = 1; + mb_def.data_off = RTE_PKTMBUF_HEADROOM; + mb_def.port = rxq->port_id; + rte_mbuf_refcnt_set(&mb_def, 1); + + /* Prevent compiler reordering: rearm_data covers previous fields */ + rte_compiler_barrier(); + p = (uintptr_t)&mb_def.rearm_data; + rxq->mbuf_initializer.value = *(uint64_t *)p; } static int @@ -1082,14 +1273,25 @@ nicvf_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t qidx, PMD_INIT_FUNC_TRACE(); + if (qidx >= MAX_RCV_QUEUES_PER_QS) + nic = nic->snicvf[qidx / MAX_RCV_QUEUES_PER_QS - 1]; + + qidx = qidx % MAX_RCV_QUEUES_PER_QS; + /* Socket id check */ if (socket_id != (unsigned int)SOCKET_ID_ANY && socket_id != nic->node) PMD_DRV_LOG(WARNING, "socket_id expected %d, configured %d", socket_id, nic->node); - /* Mempool memory should be contiguous */ + /* Mempool memory must be contiguous, so must be one memory segment*/ if (mp->nb_mem_chunks != 1) { - PMD_INIT_LOG(ERR, "Non contiguous mempool, check huge page sz"); + PMD_INIT_LOG(ERR, "Non-contiguous mempool, add more huge pages"); + return -EINVAL; + } + + /* Mempool memory must be physically contiguous */ + if (mp->flags & MEMPOOL_F_NO_PHYS_CONTIG) { + PMD_INIT_LOG(ERR, "Mempool memory must be physically contiguous"); return -EINVAL; } @@ -1118,18 +1320,20 @@ nicvf_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t qidx, } /* Free memory prior to re-allocation if needed */ - if (dev->data->rx_queues[qidx] != NULL) { + if (dev->data->rx_queues[nicvf_netdev_qidx(nic, qidx)] != NULL) { PMD_RX_LOG(DEBUG, "Freeing memory prior to re-allocation %d", - qidx); - nicvf_dev_rx_queue_release(dev->data->rx_queues[qidx]); - dev->data->rx_queues[qidx] = NULL; + nicvf_netdev_qidx(nic, qidx)); + nicvf_dev_rx_queue_release( + dev->data->rx_queues[nicvf_netdev_qidx(nic, qidx)]); + dev->data->rx_queues[nicvf_netdev_qidx(nic, qidx)] = NULL; } /* Allocate rxq memory */ rxq = rte_zmalloc_socket("ethdev rx queue", sizeof(struct nicvf_rxq), RTE_CACHE_LINE_SIZE, nic->node); if (rxq == NULL) { - PMD_INIT_LOG(ERR, "Failed to allocate rxq=%d", qidx); + PMD_INIT_LOG(ERR, "Failed to allocate rxq=%d", + nicvf_netdev_qidx(nic, qidx)); return -ENOMEM; } @@ -1142,10 +1346,16 @@ nicvf_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t qidx, rxq->cq_status = nicvf_qset_base(nic, qidx) + NIC_QSET_CQ_0_7_STATUS; rxq->cq_door = nicvf_qset_base(nic, qidx) + NIC_QSET_CQ_0_7_DOOR; rxq->precharge_cnt = 0; - rxq->rbptr_offset = NICVF_CQE_RBPTR_WORD; + + if (nicvf_hw_cap(nic) & NICVF_CAP_CQE_RX2) + rxq->rbptr_offset = NICVF_CQE_RX2_RBPTR_WORD; + else + rxq->rbptr_offset = NICVF_CQE_RBPTR_WORD; + + nicvf_rxq_mbuf_setup(rxq); /* Alloc completion queue */ - if (nicvf_qset_cq_alloc(nic, rxq, rxq->queue_id, nb_desc)) { + if (nicvf_qset_cq_alloc(dev, nic, rxq, rxq->queue_id, nb_desc)) { PMD_INIT_LOG(ERR, "failed to allocate cq %u", rxq->queue_id); nicvf_dev_rx_queue_release(rxq); return -ENOMEM; @@ -1154,11 +1364,12 @@ nicvf_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t qidx, nicvf_rx_queue_reset(rxq); PMD_RX_LOG(DEBUG, "[%d] rxq=%p pool=%s nb_desc=(%d/%d) phy=%" PRIx64, - qidx, rxq, mp->name, nb_desc, + nicvf_netdev_qidx(nic, qidx), rxq, mp->name, nb_desc, rte_mempool_avail_count(mp), rxq->phys); - dev->data->rx_queues[qidx] = rxq; - dev->data->rx_queue_state[qidx] = RTE_ETH_QUEUE_STATE_STOPPED; + dev->data->rx_queues[nicvf_netdev_qidx(nic, qidx)] = rxq; + dev->data->rx_queue_state[nicvf_netdev_qidx(nic, qidx)] = + RTE_ETH_QUEUE_STATE_STOPPED; return 0; } @@ -1166,15 +1377,27 @@ static void nicvf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) { struct nicvf *nic = nicvf_pmd_priv(dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); PMD_INIT_FUNC_TRACE(); + dev_info->pci_dev = RTE_ETH_DEV_TO_PCI(dev); + + /* Autonegotiation may be disabled */ + dev_info->speed_capa = ETH_LINK_SPEED_FIXED; + dev_info->speed_capa |= ETH_LINK_SPEED_10M | ETH_LINK_SPEED_100M | + ETH_LINK_SPEED_1G | ETH_LINK_SPEED_10G; + if (nicvf_hw_version(nic) != PCI_SUB_DEVICE_ID_CN81XX_NICVF) + dev_info->speed_capa |= ETH_LINK_SPEED_40G; + dev_info->min_rx_bufsize = ETHER_MIN_MTU; dev_info->max_rx_pktlen = NIC_HW_MAX_FRS; - dev_info->max_rx_queues = (uint16_t)MAX_RCV_QUEUES_PER_QS; - dev_info->max_tx_queues = (uint16_t)MAX_SND_QUEUES_PER_QS; + dev_info->max_rx_queues = + (uint16_t)MAX_RCV_QUEUES_PER_QS * (MAX_SQS_PER_VF + 1); + dev_info->max_tx_queues = + (uint16_t)MAX_SND_QUEUES_PER_QS * (MAX_SQS_PER_VF + 1); dev_info->max_mac_addrs = 1; - dev_info->max_vfs = dev->pci_dev->max_vfs; + dev_info->max_vfs = pci_dev->max_vfs; dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP; dev_info->tx_offload_capa = @@ -1206,16 +1429,21 @@ nicvf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) }; } -static nicvf_phys_addr_t -rbdr_rte_mempool_get(void *opaque) +static nicvf_iova_addr_t +rbdr_rte_mempool_get(void *dev, void *opaque) { uint16_t qidx; uintptr_t mbuf; struct nicvf_rxq *rxq; - struct nicvf *nic = nicvf_pmd_priv((struct rte_eth_dev *)opaque); + struct rte_eth_dev *eth_dev = (struct rte_eth_dev *)dev; + struct nicvf *nic = (struct nicvf *)opaque; + uint16_t rx_start, rx_end; - for (qidx = 0; qidx < nic->eth_dev->data->nb_rx_queues; qidx++) { - rxq = nic->eth_dev->data->rx_queues[qidx]; + /* Get queue ranges for this VF */ + nicvf_rx_range(eth_dev, nic, &rx_start, &rx_end); + + for (qidx = rx_start; qidx <= rx_end; qidx++) { + rxq = eth_dev->data->rx_queues[qidx]; /* Maintain equal buffer count across all pools */ if (rxq->precharge_cnt >= rxq->qlen_mask) continue; @@ -1228,25 +1456,25 @@ rbdr_rte_mempool_get(void *opaque) } static int -nicvf_dev_start(struct rte_eth_dev *dev) +nicvf_vf_start(struct rte_eth_dev *dev, struct nicvf *nic, uint32_t rbdrsz) { int ret; - uint16_t qidx; - uint32_t buffsz = 0, rbdrsz = 0; + uint16_t qidx, data_off; uint32_t total_rxq_desc, nb_rbdr_desc, exp_buffs; uint64_t mbuf_phys_off = 0; struct nicvf_rxq *rxq; - struct rte_pktmbuf_pool_private *mbp_priv; struct rte_mbuf *mbuf; - struct nicvf *nic = nicvf_pmd_priv(dev); - struct rte_eth_rxmode *rx_conf = &dev->data->dev_conf.rxmode; - uint16_t mtu; + uint16_t rx_start, rx_end; + uint16_t tx_start, tx_end; PMD_INIT_FUNC_TRACE(); /* Userspace process exited without proper shutdown in last run */ if (nicvf_qset_rbdr_active(nic, 0)) - nicvf_dev_stop(dev); + nicvf_vf_stop(dev, nic, false); + + /* Get queue ranges for this VF */ + nicvf_rx_range(dev, nic, &rx_start, &rx_end); /* * Thunderx nicvf PMD can support more than one pool per port only when @@ -1261,55 +1489,47 @@ nicvf_dev_start(struct rte_eth_dev *dev) * */ - /* Validate RBDR buff size */ - for (qidx = 0; qidx < nic->eth_dev->data->nb_rx_queues; qidx++) { - rxq = dev->data->rx_queues[qidx]; - mbp_priv = rte_mempool_get_priv(rxq->pool); - buffsz = mbp_priv->mbuf_data_room_size - RTE_PKTMBUF_HEADROOM; - if (buffsz % 128) { - PMD_INIT_LOG(ERR, "rxbuf size must be multiply of 128"); - return -EINVAL; - } - if (rbdrsz == 0) - rbdrsz = buffsz; - if (rbdrsz != buffsz) { - PMD_INIT_LOG(ERR, "buffsz not same, qid=%d (%d/%d)", - qidx, rbdrsz, buffsz); - return -EINVAL; - } - } - /* Validate mempool attributes */ - for (qidx = 0; qidx < nic->eth_dev->data->nb_rx_queues; qidx++) { + for (qidx = rx_start; qidx <= rx_end; qidx++) { rxq = dev->data->rx_queues[qidx]; rxq->mbuf_phys_off = nicvf_mempool_phy_offset(rxq->pool); mbuf = rte_pktmbuf_alloc(rxq->pool); if (mbuf == NULL) { - PMD_INIT_LOG(ERR, "Failed allocate mbuf qid=%d pool=%s", - qidx, rxq->pool->name); + PMD_INIT_LOG(ERR, "Failed allocate mbuf VF%d qid=%d " + "pool=%s", + nic->vf_id, qidx, rxq->pool->name); return -ENOMEM; } - rxq->mbuf_phys_off -= nicvf_mbuff_meta_length(mbuf); - rxq->mbuf_phys_off -= RTE_PKTMBUF_HEADROOM; + data_off = nicvf_mbuff_meta_length(mbuf); + data_off += RTE_PKTMBUF_HEADROOM; rte_pktmbuf_free(mbuf); + if (data_off % RTE_CACHE_LINE_SIZE) { + PMD_INIT_LOG(ERR, "%s: unaligned data_off=%d delta=%d", + rxq->pool->name, data_off, + data_off % RTE_CACHE_LINE_SIZE); + return -EINVAL; + } + rxq->mbuf_phys_off -= data_off; + if (mbuf_phys_off == 0) mbuf_phys_off = rxq->mbuf_phys_off; if (mbuf_phys_off != rxq->mbuf_phys_off) { - PMD_INIT_LOG(ERR, "pool params not same,%s %" PRIx64, - rxq->pool->name, mbuf_phys_off); + PMD_INIT_LOG(ERR, "pool params not same,%s VF%d %" + PRIx64, rxq->pool->name, nic->vf_id, + mbuf_phys_off); return -EINVAL; } } /* Check the level of buffers in the pool */ total_rxq_desc = 0; - for (qidx = 0; qidx < nic->eth_dev->data->nb_rx_queues; qidx++) { + for (qidx = rx_start; qidx <= rx_end; qidx++) { rxq = dev->data->rx_queues[qidx]; /* Count total numbers of rxq descs */ total_rxq_desc += rxq->qlen_mask + 1; exp_buffs = RTE_MEMPOOL_CACHE_MAX_SIZE + rxq->rx_free_thresh; - exp_buffs *= nic->eth_dev->data->nb_rx_queues; + exp_buffs *= dev->data->nb_rx_queues; if (rte_mempool_avail_count(rxq->pool) < exp_buffs) { PMD_INIT_LOG(ERR, "Buff shortage in pool=%s (%d/%d)", rxq->pool->name, @@ -1322,82 +1542,171 @@ nicvf_dev_start(struct rte_eth_dev *dev) /* Check RBDR desc overflow */ ret = nicvf_qsize_rbdr_roundup(total_rxq_desc); if (ret == 0) { - PMD_INIT_LOG(ERR, "Reached RBDR desc limit, reduce nr desc"); + PMD_INIT_LOG(ERR, "Reached RBDR desc limit, reduce nr desc " + "VF%d", nic->vf_id); return -ENOMEM; } /* Enable qset */ ret = nicvf_qset_config(nic); if (ret) { - PMD_INIT_LOG(ERR, "Failed to enable qset %d", ret); + PMD_INIT_LOG(ERR, "Failed to enable qset %d VF%d", ret, + nic->vf_id); return ret; } /* Allocate RBDR and RBDR ring desc */ nb_rbdr_desc = nicvf_qsize_rbdr_roundup(total_rxq_desc); - ret = nicvf_qset_rbdr_alloc(nic, nb_rbdr_desc, rbdrsz); + ret = nicvf_qset_rbdr_alloc(dev, nic, nb_rbdr_desc, rbdrsz); if (ret) { - PMD_INIT_LOG(ERR, "Failed to allocate memory for rbdr alloc"); + PMD_INIT_LOG(ERR, "Failed to allocate memory for rbdr alloc " + "VF%d", nic->vf_id); goto qset_reclaim; } /* Enable and configure RBDR registers */ ret = nicvf_qset_rbdr_config(nic, 0); if (ret) { - PMD_INIT_LOG(ERR, "Failed to configure rbdr %d", ret); + PMD_INIT_LOG(ERR, "Failed to configure rbdr %d VF%d", ret, + nic->vf_id); goto qset_rbdr_free; } /* Fill rte_mempool buffers in RBDR pool and precharge it */ - ret = nicvf_qset_rbdr_precharge(nic, 0, rbdr_rte_mempool_get, - dev, total_rxq_desc); + ret = nicvf_qset_rbdr_precharge(dev, nic, 0, rbdr_rte_mempool_get, + total_rxq_desc); if (ret) { - PMD_INIT_LOG(ERR, "Failed to fill rbdr %d", ret); + PMD_INIT_LOG(ERR, "Failed to fill rbdr %d VF%d", ret, + nic->vf_id); goto qset_rbdr_reclaim; } - PMD_DRV_LOG(INFO, "Filled %d out of %d entries in RBDR", - nic->rbdr->tail, nb_rbdr_desc); - - /* Configure RX queues */ - for (qidx = 0; qidx < nic->eth_dev->data->nb_rx_queues; qidx++) { - ret = nicvf_start_rx_queue(dev, qidx); - if (ret) - goto start_rxq_error; - } + PMD_DRV_LOG(INFO, "Filled %d out of %d entries in RBDR VF%d", + nic->rbdr->tail, nb_rbdr_desc, nic->vf_id); /* Configure VLAN Strip */ nicvf_vlan_hw_strip(nic, dev->data->dev_conf.rxmode.hw_vlan_strip); + /* Based on the packet type(IPv4 or IPv6), the nicvf HW aligns L3 data + * to the 64bit memory address. + * The alignment creates a hole in mbuf(between the end of headroom and + * packet data start). The new revision of the HW provides an option to + * disable the L3 alignment feature and make mbuf layout looks + * more like other NICs. For better application compatibility, disabling + * l3 alignment feature on the hardware revisions it supports + */ + nicvf_apad_config(nic, false); + + /* Get queue ranges for this VF */ + nicvf_tx_range(dev, nic, &tx_start, &tx_end); + /* Configure TX queues */ - for (qidx = 0; qidx < nic->eth_dev->data->nb_tx_queues; qidx++) { - ret = nicvf_start_tx_queue(dev, qidx); + for (qidx = tx_start; qidx <= tx_end; qidx++) { + ret = nicvf_vf_start_tx_queue(dev, nic, + qidx % MAX_SND_QUEUES_PER_QS); if (ret) goto start_txq_error; } - /* Configure CPI algorithm */ - ret = nicvf_configure_cpi(dev); - if (ret) - goto start_txq_error; + /* Configure RX queues */ + for (qidx = rx_start; qidx <= rx_end; qidx++) { + ret = nicvf_vf_start_rx_queue(dev, nic, + qidx % MAX_RCV_QUEUES_PER_QS); + if (ret) + goto start_rxq_error; + } - /* Configure RSS */ - ret = nicvf_configure_rss(dev); - if (ret) - goto qset_rss_error; + if (!nic->sqs_mode) { + /* Configure CPI algorithm */ + ret = nicvf_configure_cpi(dev); + if (ret) + goto start_txq_error; + + ret = nicvf_mbox_get_rss_size(nic); + if (ret) { + PMD_INIT_LOG(ERR, "Failed to get rss table size"); + goto qset_rss_error; + } + + /* Configure RSS */ + ret = nicvf_configure_rss(dev); + if (ret) + goto qset_rss_error; + } + + /* Done; Let PF make the BGX's RX and TX switches to ON position */ + nicvf_mbox_cfg_done(nic); + return 0; + +qset_rss_error: + nicvf_rss_term(nic); +start_rxq_error: + for (qidx = rx_start; qidx <= rx_end; qidx++) + nicvf_vf_stop_rx_queue(dev, nic, qidx % MAX_RCV_QUEUES_PER_QS); +start_txq_error: + for (qidx = tx_start; qidx <= tx_end; qidx++) + nicvf_vf_stop_tx_queue(dev, nic, qidx % MAX_SND_QUEUES_PER_QS); +qset_rbdr_reclaim: + nicvf_qset_rbdr_reclaim(nic, 0); + nicvf_rbdr_release_mbufs(dev, nic); +qset_rbdr_free: + if (nic->rbdr) { + rte_free(nic->rbdr); + nic->rbdr = NULL; + } +qset_reclaim: + nicvf_qset_reclaim(nic); + return ret; +} + +static int +nicvf_dev_start(struct rte_eth_dev *dev) +{ + uint16_t qidx; + int ret; + size_t i; + struct nicvf *nic = nicvf_pmd_priv(dev); + struct rte_eth_rxmode *rx_conf = &dev->data->dev_conf.rxmode; + uint16_t mtu; + uint32_t buffsz = 0, rbdrsz = 0; + struct rte_pktmbuf_pool_private *mbp_priv; + struct nicvf_rxq *rxq; + + PMD_INIT_FUNC_TRACE(); + + /* This function must be called for a primary device */ + assert_primary(nic); + + /* Validate RBDR buff size */ + for (qidx = 0; qidx < dev->data->nb_rx_queues; qidx++) { + rxq = dev->data->rx_queues[qidx]; + mbp_priv = rte_mempool_get_priv(rxq->pool); + buffsz = mbp_priv->mbuf_data_room_size - RTE_PKTMBUF_HEADROOM; + if (buffsz % 128) { + PMD_INIT_LOG(ERR, "rxbuf size must be multiply of 128"); + return -EINVAL; + } + if (rbdrsz == 0) + rbdrsz = buffsz; + if (rbdrsz != buffsz) { + PMD_INIT_LOG(ERR, "buffsz not same, qidx=%d (%d/%d)", + qidx, rbdrsz, buffsz); + return -EINVAL; + } + } /* Configure loopback */ ret = nicvf_loopback_config(nic, dev->data->dev_conf.lpbk_mode); if (ret) { PMD_INIT_LOG(ERR, "Failed to configure loopback %d", ret); - goto qset_rss_error; + return ret; } /* Reset all statistics counters attached to this port */ ret = nicvf_mbox_reset_stat_counters(nic, 0x3FFF, 0x1F, 0xFFFF, 0xFFFF); if (ret) { PMD_INIT_LOG(ERR, "Failed to reset stat counters %d", ret); - goto qset_rss_error; + return ret; } /* Setup scatter mode if needed by jumbo */ @@ -1418,62 +1727,94 @@ nicvf_dev_start(struct rte_eth_dev *dev) return -EBUSY; } + ret = nicvf_vf_start(dev, nic, rbdrsz); + if (ret != 0) + return ret; + + for (i = 0; i < nic->sqs_count; i++) { + assert(nic->snicvf[i]); + + ret = nicvf_vf_start(dev, nic->snicvf[i], rbdrsz); + if (ret != 0) + return ret; + } + /* Configure callbacks based on scatter mode */ nicvf_set_tx_function(dev); nicvf_set_rx_function(dev); - /* Done; Let PF make the BGX's RX and TX switches to ON position */ - nicvf_mbox_cfg_done(nic); return 0; - -qset_rss_error: - nicvf_rss_term(nic); -start_txq_error: - for (qidx = 0; qidx < nic->eth_dev->data->nb_tx_queues; qidx++) - nicvf_stop_tx_queue(dev, qidx); -start_rxq_error: - for (qidx = 0; qidx < nic->eth_dev->data->nb_rx_queues; qidx++) - nicvf_stop_rx_queue(dev, qidx); -qset_rbdr_reclaim: - nicvf_qset_rbdr_reclaim(nic, 0); - nicvf_rbdr_release_mbufs(nic); -qset_rbdr_free: - if (nic->rbdr) { - rte_free(nic->rbdr); - nic->rbdr = NULL; - } -qset_reclaim: - nicvf_qset_reclaim(nic); - return ret; } static void -nicvf_dev_stop(struct rte_eth_dev *dev) +nicvf_dev_stop_cleanup(struct rte_eth_dev *dev, bool cleanup) { + size_t i; int ret; - uint16_t qidx; struct nicvf *nic = nicvf_pmd_priv(dev); PMD_INIT_FUNC_TRACE(); - /* Let PF make the BGX's RX and TX switches to OFF position */ - nicvf_mbox_shutdown(nic); + /* Teardown secondary vf first */ + for (i = 0; i < nic->sqs_count; i++) { + if (!nic->snicvf[i]) + continue; + + nicvf_vf_stop(dev, nic->snicvf[i], cleanup); + } + + /* Stop the primary VF now */ + nicvf_vf_stop(dev, nic, cleanup); /* Disable loopback */ ret = nicvf_loopback_config(nic, 0); if (ret) PMD_INIT_LOG(ERR, "Failed to disable loopback %d", ret); + /* Reclaim CPI configuration */ + ret = nicvf_mbox_config_cpi(nic, 0); + if (ret) + PMD_INIT_LOG(ERR, "Failed to reclaim CPI config %d", ret); +} + +static void +nicvf_dev_stop(struct rte_eth_dev *dev) +{ + PMD_INIT_FUNC_TRACE(); + + nicvf_dev_stop_cleanup(dev, false); +} + +static void +nicvf_vf_stop(struct rte_eth_dev *dev, struct nicvf *nic, bool cleanup) +{ + int ret; + uint16_t qidx; + uint16_t tx_start, tx_end; + uint16_t rx_start, rx_end; + + PMD_INIT_FUNC_TRACE(); + + if (cleanup) { + /* Let PF make the BGX's RX and TX switches to OFF position */ + nicvf_mbox_shutdown(nic); + } + /* Disable VLAN Strip */ nicvf_vlan_hw_strip(nic, 0); - /* Reclaim sq */ - for (qidx = 0; qidx < dev->data->nb_tx_queues; qidx++) - nicvf_stop_tx_queue(dev, qidx); + /* Get queue ranges for this VF */ + nicvf_tx_range(dev, nic, &tx_start, &tx_end); + + for (qidx = tx_start; qidx <= tx_end; qidx++) + nicvf_vf_stop_tx_queue(dev, nic, qidx % MAX_SND_QUEUES_PER_QS); + + /* Get queue ranges for this VF */ + nicvf_rx_range(dev, nic, &rx_start, &rx_end); /* Reclaim rq */ - for (qidx = 0; qidx < dev->data->nb_rx_queues; qidx++) - nicvf_stop_rx_queue(dev, qidx); + for (qidx = rx_start; qidx <= rx_end; qidx++) + nicvf_vf_stop_rx_queue(dev, nic, qidx % MAX_RCV_QUEUES_PER_QS); /* Reclaim RBDR */ ret = nicvf_qset_rbdr_reclaim(nic, 0); @@ -1482,17 +1823,10 @@ nicvf_dev_stop(struct rte_eth_dev *dev) /* Move all charged buffers in RBDR back to pool */ if (nic->rbdr != NULL) - nicvf_rbdr_release_mbufs(nic); - - /* Reclaim CPI configuration */ - if (!nic->sqs_mode) { - ret = nicvf_mbox_config_cpi(nic, 0); - if (ret) - PMD_INIT_LOG(ERR, "Failed to reclaim CPI config"); - } + nicvf_rbdr_release_mbufs(dev, nic); /* Disable qset */ - ret = nicvf_qset_config(nic); + ret = nicvf_qset_reclaim(nic); if (ret) PMD_INIT_LOG(ERR, "Failed to disable qset %d", ret); @@ -1509,21 +1843,54 @@ nicvf_dev_stop(struct rte_eth_dev *dev) static void nicvf_dev_close(struct rte_eth_dev *dev) { + size_t i; struct nicvf *nic = nicvf_pmd_priv(dev); PMD_INIT_FUNC_TRACE(); - nicvf_dev_stop(dev); - nicvf_periodic_alarm_stop(nic); + nicvf_dev_stop_cleanup(dev, true); + nicvf_periodic_alarm_stop(nicvf_interrupt, dev); + + for (i = 0; i < nic->sqs_count; i++) { + if (!nic->snicvf[i]) + continue; + + nicvf_periodic_alarm_stop(nicvf_vf_interrupt, nic->snicvf[i]); + } +} + +static int +nicvf_request_sqs(struct nicvf *nic) +{ + size_t i; + + assert_primary(nic); + assert(nic->sqs_count > 0); + assert(nic->sqs_count <= MAX_SQS_PER_VF); + + /* Set no of Rx/Tx queues in each of the SQsets */ + for (i = 0; i < nic->sqs_count; i++) { + if (nicvf_svf_empty()) + rte_panic("Cannot assign sufficient number of " + "secondary queues to primary VF%" PRIu8 "\n", + nic->vf_id); + + nic->snicvf[i] = nicvf_svf_pop(); + nic->snicvf[i]->sqs_id = i; + } + + return nicvf_mbox_request_sqs(nic); } static int nicvf_dev_configure(struct rte_eth_dev *dev) { - struct rte_eth_conf *conf = &dev->data->dev_conf; + struct rte_eth_dev_data *data = dev->data; + struct rte_eth_conf *conf = &data->dev_conf; struct rte_eth_rxmode *rxmode = &conf->rxmode; struct rte_eth_txmode *txmode = &conf->txmode; struct nicvf *nic = nicvf_pmd_priv(dev); + uint8_t cqcount; PMD_INIT_FUNC_TRACE(); @@ -1588,6 +1955,26 @@ nicvf_dev_configure(struct rte_eth_dev *dev) return -EINVAL; } + assert_primary(nic); + NICVF_STATIC_ASSERT(MAX_RCV_QUEUES_PER_QS == MAX_SND_QUEUES_PER_QS); + cqcount = RTE_MAX(data->nb_tx_queues, data->nb_rx_queues); + if (cqcount > MAX_RCV_QUEUES_PER_QS) { + nic->sqs_count = RTE_ALIGN_CEIL(cqcount, MAX_RCV_QUEUES_PER_QS); + nic->sqs_count = (nic->sqs_count / MAX_RCV_QUEUES_PER_QS) - 1; + } else { + nic->sqs_count = 0; + } + + assert(nic->sqs_count <= MAX_SQS_PER_VF); + + if (nic->sqs_count > 0) { + if (nicvf_request_sqs(nic)) { + rte_panic("Cannot assign sufficient number of " + "secondary queues to PORT%d VF%" PRIu8 "\n", + dev->data->port_id, nic->vf_id); + } + } + PMD_INIT_LOG(DEBUG, "Configured ethdev port%d hwcap=0x%" PRIx64, dev->data->port_id, nicvf_hw_cap(nic)); @@ -1636,20 +2023,25 @@ nicvf_eth_dev_init(struct rte_eth_dev *eth_dev) /* For secondary processes, the primary has done all the work */ if (rte_eal_process_type() != RTE_PROC_PRIMARY) { - /* Setup callbacks for secondary process */ - nicvf_set_tx_function(eth_dev); - nicvf_set_rx_function(eth_dev); - return 0; + if (nic) { + /* Setup callbacks for secondary process */ + nicvf_set_tx_function(eth_dev); + nicvf_set_rx_function(eth_dev); + return 0; + } else { + /* If nic == NULL than it is secondary function + * so ethdev need to be released by caller */ + return ENOTSUP; + } } - pci_dev = eth_dev->pci_dev; + pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); rte_eth_copy_pci_info(eth_dev, pci_dev); nic->device_id = pci_dev->id.device_id; nic->vendor_id = pci_dev->id.vendor_id; nic->subsystem_device_id = pci_dev->id.subsystem_device_id; nic->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id; - nic->eth_dev = eth_dev; PMD_INIT_LOG(DEBUG, "nicvf: device (%x:%x) %u:%u:%u:%u", pci_dev->id.vendor_id, pci_dev->id.device_id, @@ -1665,7 +2057,7 @@ nicvf_eth_dev_init(struct rte_eth_dev *eth_dev) nicvf_disable_all_interrupts(nic); - ret = nicvf_periodic_alarm_start(nic); + ret = nicvf_periodic_alarm_start(nicvf_interrupt, eth_dev); if (ret) { PMD_INIT_LOG(ERR, "Failed to start period alarm"); goto fail; @@ -1685,11 +2077,28 @@ nicvf_eth_dev_init(struct rte_eth_dev *eth_dev) ); } + ret = nicvf_base_init(nic); + if (ret) { + PMD_INIT_LOG(ERR, "Failed to execute nicvf_base_init"); + goto malloc_fail; + } + if (nic->sqs_mode) { - PMD_INIT_LOG(INFO, "Unsupported SQS VF detected, Detaching..."); - /* Detach port by returning Positive error number */ - ret = ENOTSUP; - goto alarm_fail; + /* Push nic to stack of secondary vfs */ + nicvf_svf_push(nic); + + /* Steal nic pointer from the device for further reuse */ + eth_dev->data->dev_private = NULL; + + nicvf_periodic_alarm_stop(nicvf_interrupt, eth_dev); + ret = nicvf_periodic_alarm_start(nicvf_vf_interrupt, nic); + if (ret) { + PMD_INIT_LOG(ERR, "Failed to start period alarm"); + goto fail; + } + + /* Detach port by returning positive error number */ + return ENOTSUP; } eth_dev->data->mac_addrs = rte_zmalloc("mac_addr", ETHER_ADDR_LEN, 0); @@ -1710,18 +2119,6 @@ nicvf_eth_dev_init(struct rte_eth_dev *eth_dev) goto malloc_fail; } - ret = nicvf_base_init(nic); - if (ret) { - PMD_INIT_LOG(ERR, "Failed to execute nicvf_base_init"); - goto malloc_fail; - } - - ret = nicvf_mbox_get_rss_size(nic); - if (ret) { - PMD_INIT_LOG(ERR, "Failed to get rss table size"); - goto malloc_fail; - } - PMD_INIT_LOG(INFO, "Port %d (%x:%x) mac=%02x:%02x:%02x:%02x:%02x:%02x", eth_dev->data->port_id, nic->vendor_id, nic->device_id, nic->mac_addr[0], nic->mac_addr[1], nic->mac_addr[2], @@ -1732,7 +2129,7 @@ nicvf_eth_dev_init(struct rte_eth_dev *eth_dev) malloc_fail: rte_free(eth_dev->data->mac_addrs); alarm_fail: - nicvf_periodic_alarm_stop(nic); + nicvf_periodic_alarm_stop(nicvf_interrupt, eth_dev); fail: return ret; } @@ -1741,47 +2138,56 @@ static const struct rte_pci_id pci_id_nicvf_map[] = { { .class_id = RTE_CLASS_ANY_ID, .vendor_id = PCI_VENDOR_ID_CAVIUM, - .device_id = PCI_DEVICE_ID_THUNDERX_PASS1_NICVF, + .device_id = PCI_DEVICE_ID_THUNDERX_CN88XX_PASS1_NICVF, .subsystem_vendor_id = PCI_VENDOR_ID_CAVIUM, - .subsystem_device_id = PCI_SUB_DEVICE_ID_THUNDERX_PASS1_NICVF, + .subsystem_device_id = PCI_SUB_DEVICE_ID_CN88XX_PASS1_NICVF, }, { .class_id = RTE_CLASS_ANY_ID, .vendor_id = PCI_VENDOR_ID_CAVIUM, - .device_id = PCI_DEVICE_ID_THUNDERX_PASS2_NICVF, + .device_id = PCI_DEVICE_ID_THUNDERX_NICVF, .subsystem_vendor_id = PCI_VENDOR_ID_CAVIUM, - .subsystem_device_id = PCI_SUB_DEVICE_ID_THUNDERX_PASS2_NICVF, + .subsystem_device_id = PCI_SUB_DEVICE_ID_CN88XX_PASS2_NICVF, + }, + { + .class_id = RTE_CLASS_ANY_ID, + .vendor_id = PCI_VENDOR_ID_CAVIUM, + .device_id = PCI_DEVICE_ID_THUNDERX_NICVF, + .subsystem_vendor_id = PCI_VENDOR_ID_CAVIUM, + .subsystem_device_id = PCI_SUB_DEVICE_ID_CN81XX_NICVF, + }, + { + .class_id = RTE_CLASS_ANY_ID, + .vendor_id = PCI_VENDOR_ID_CAVIUM, + .device_id = PCI_DEVICE_ID_THUNDERX_NICVF, + .subsystem_vendor_id = PCI_VENDOR_ID_CAVIUM, + .subsystem_device_id = PCI_SUB_DEVICE_ID_CN83XX_NICVF, }, { .vendor_id = 0, }, }; -static struct eth_driver rte_nicvf_pmd = { - .pci_drv = { - .name = "rte_nicvf_pmd", - .id_table = pci_id_nicvf_map, - .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC, - }, - .eth_dev_init = nicvf_eth_dev_init, - .dev_private_size = sizeof(struct nicvf), -}; - -static int -rte_nicvf_pmd_init(const char *name __rte_unused, const char *para __rte_unused) +static int nicvf_eth_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, + struct rte_pci_device *pci_dev) { - PMD_INIT_FUNC_TRACE(); - PMD_INIT_LOG(INFO, "librte_pmd_thunderx nicvf version %s", - THUNDERX_NICVF_PMD_VERSION); - - rte_eth_driver_register(&rte_nicvf_pmd); - return 0; + return rte_eth_dev_pci_generic_probe(pci_dev, sizeof(struct nicvf), + nicvf_eth_dev_init); } -static struct rte_driver rte_nicvf_driver = { - .type = PMD_PDEV, - .init = rte_nicvf_pmd_init, +static int nicvf_eth_pci_remove(struct rte_pci_device *pci_dev) +{ + return rte_eth_dev_pci_generic_remove(pci_dev, NULL); +} + +static struct rte_pci_driver rte_nicvf_pmd = { + .id_table = pci_id_nicvf_map, + .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_KEEP_MAPPED_RES | + RTE_PCI_DRV_INTR_LSC, + .probe = nicvf_eth_pci_probe, + .remove = nicvf_eth_pci_remove, }; -PMD_REGISTER_DRIVER(rte_nicvf_driver, thunderx_nicvf); -DRIVER_REGISTER_PCI_TABLE(thunderx_nicvf, pci_id_nicvf_map); +RTE_PMD_REGISTER_PCI(net_thunderx, rte_nicvf_pmd); +RTE_PMD_REGISTER_PCI_TABLE(net_thunderx, pci_id_nicvf_map); +RTE_PMD_REGISTER_KMOD_DEP(net_thunderx, "* igb_uio | uio_pci_generic | vfio-pci"); diff --git a/dpdk/drivers/net/thunderx/nicvf_ethdev.h b/dpdk/drivers/net/thunderx/nicvf_ethdev.h index 34447e05..71bc3cf2 100644 --- a/dpdk/drivers/net/thunderx/nicvf_ethdev.h +++ b/dpdk/drivers/net/thunderx/nicvf_ethdev.h @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2016. + * Copyright (C) Cavium, Inc. 2016. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * @@ -35,7 +35,7 @@ #include -#define THUNDERX_NICVF_PMD_VERSION "1.0" +#define THUNDERX_NICVF_PMD_VERSION "2.0" #define THUNDERX_REG_BYTES 8 #define NICVF_INTR_POLL_INTERVAL_MS 50 @@ -78,7 +78,7 @@ nicvf_mempool_phy_offset(struct rte_mempool *mp) hdr = STAILQ_FIRST(&mp->mem_list); assert(hdr != NULL); - return (uint64_t)((uintptr_t)hdr->addr - hdr->phys_addr); + return (uint64_t)((uintptr_t)hdr->addr - hdr->iova); } static inline uint16_t @@ -87,13 +87,24 @@ nicvf_mbuff_meta_length(struct rte_mbuf *mbuf) return (uint16_t)((uintptr_t)mbuf->buf_addr - (uintptr_t)mbuf); } +static inline uint16_t +nicvf_netdev_qidx(struct nicvf *nic, uint8_t local_qidx) +{ + uint16_t global_qidx = local_qidx; + + if (nic->sqs_mode) + global_qidx += ((nic->sqs_id + 1) * MAX_CMP_QUEUES_PER_QS); + + return global_qidx; +} + /* * Simple phy2virt functions assuming mbufs are in a single huge page * V = P + offset * P = V - offset */ static inline uintptr_t -nicvf_mbuff_phy2virt(phys_addr_t phy, uint64_t mbuf_phys_off) +nicvf_mbuff_phy2virt(rte_iova_t phy, uint64_t mbuf_phys_off) { return (uintptr_t)(phy + mbuf_phys_off); } @@ -101,7 +112,35 @@ nicvf_mbuff_phy2virt(phys_addr_t phy, uint64_t mbuf_phys_off) static inline uintptr_t nicvf_mbuff_virt2phy(uintptr_t virt, uint64_t mbuf_phys_off) { - return (phys_addr_t)(virt - mbuf_phys_off); + return (rte_iova_t)(virt - mbuf_phys_off); +} + +static inline void +nicvf_tx_range(struct rte_eth_dev *dev, struct nicvf *nic, uint16_t *tx_start, + uint16_t *tx_end) +{ + uint16_t tmp; + + *tx_start = RTE_ALIGN_FLOOR(nicvf_netdev_qidx(nic, 0), + MAX_SND_QUEUES_PER_QS); + tmp = RTE_ALIGN_CEIL(nicvf_netdev_qidx(nic, 0) + 1, + MAX_SND_QUEUES_PER_QS) - 1; + *tx_end = dev->data->nb_tx_queues ? + RTE_MIN(tmp, dev->data->nb_tx_queues - 1) : 0; +} + +static inline void +nicvf_rx_range(struct rte_eth_dev *dev, struct nicvf *nic, uint16_t *rx_start, + uint16_t *rx_end) +{ + uint16_t tmp; + + *rx_start = RTE_ALIGN_FLOOR(nicvf_netdev_qidx(nic, 0), + MAX_RCV_QUEUES_PER_QS); + tmp = RTE_ALIGN_CEIL(nicvf_netdev_qidx(nic, 0) + 1, + MAX_RCV_QUEUES_PER_QS) - 1; + *rx_end = dev->data->nb_rx_queues ? + RTE_MIN(tmp, dev->data->nb_rx_queues - 1) : 0; } #endif /* __THUNDERX_NICVF_ETHDEV_H__ */ diff --git a/dpdk/drivers/net/thunderx/nicvf_logs.h b/dpdk/drivers/net/thunderx/nicvf_logs.h index 0667d468..a76d1987 100644 --- a/dpdk/drivers/net/thunderx/nicvf_logs.h +++ b/dpdk/drivers/net/thunderx/nicvf_logs.h @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2016. + * Copyright (C) Cavium, Inc. 2016. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * diff --git a/dpdk/drivers/net/thunderx/nicvf_rxtx.c b/dpdk/drivers/net/thunderx/nicvf_rxtx.c index e15c7303..9d69cf4e 100644 --- a/dpdk/drivers/net/thunderx/nicvf_rxtx.c +++ b/dpdk/drivers/net/thunderx/nicvf_rxtx.c @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2016. + * Copyright (C) Cavium, Inc. 2016. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * @@ -252,7 +252,7 @@ nicvf_xmit_pkts_multiseg(void *tx_queue, struct rte_mbuf **tx_pkts, /* Inform HW to xmit the packets */ nicvf_addr_write(sq->sq_door, used_desc); - return nb_pkts; + return i; } static const uint32_t ptype_table[16][16] __rte_cache_aligned = { @@ -368,7 +368,8 @@ nicvf_fill_rbdr(struct nicvf_rxq *rxq, int to_fill) void *obj_p[NICVF_MAX_RX_FREE_THRESH] __rte_cache_aligned; if (unlikely(rte_mempool_get_bulk(rxq->pool, obj_p, to_fill) < 0)) { - rxq->nic->eth_dev->data->rx_mbuf_alloc_failed += to_fill; + rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed += + to_fill; return 0; } @@ -429,9 +430,9 @@ nicvf_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) union cq_entry_t *desc = rxq->desc; const uint64_t cqe_mask = rxq->qlen_mask; uint64_t rb0_ptr, mbuf_phys_off = rxq->mbuf_phys_off; + const uint64_t mbuf_init = rxq->mbuf_initializer.value; uint32_t cqe_head = rxq->head & cqe_mask; int32_t available_space = rxq->available_space; - uint8_t port_id = rxq->port_id; const uint8_t rbptr_offset = rxq->rbptr_offset; to_process = nicvf_rx_pkts_to_process(rxq, nb_pkts, available_space); @@ -447,17 +448,12 @@ nicvf_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) rb0_ptr = *((uint64_t *)cqe_rx + rbptr_offset); pkt = (struct rte_mbuf *)nicvf_mbuff_phy2virt (rb0_ptr - cqe_rx_w1.align_pad, mbuf_phys_off); - pkt->ol_flags = 0; - pkt->port = port_id; pkt->data_len = cqe_rx_w3.rb0_sz; - pkt->data_off = RTE_PKTMBUF_HEADROOM + cqe_rx_w1.align_pad; - pkt->nb_segs = 1; pkt->pkt_len = cqe_rx_w3.rb0_sz; pkt->packet_type = nicvf_rx_classify_pkt(cqe_rx_w0); - + nicvf_mbuff_init_update(pkt, mbuf_init, cqe_rx_w1.align_pad); nicvf_rx_offload(cqe_rx_w0, cqe_rx_w2, pkt); - rte_mbuf_refcnt_set(pkt, 1); rx_pkts[i] = pkt; cqe_head = (cqe_head + 1) & cqe_mask; nicvf_prefetch_store_keep(pkt); @@ -468,11 +464,10 @@ nicvf_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) rxq->head = cqe_head; nicvf_addr_write(rxq->cq_door, to_process); rxq->recv_buffers += to_process; - if (rxq->recv_buffers > rxq->rx_free_thresh) { - rxq->recv_buffers -= nicvf_fill_rbdr(rxq, - rxq->rx_free_thresh); - NICVF_RX_ASSERT(rxq->recv_buffers >= 0); - } + } + if (rxq->recv_buffers > rxq->rx_free_thresh) { + rxq->recv_buffers -= nicvf_fill_rbdr(rxq, rxq->rx_free_thresh); + NICVF_RX_ASSERT(rxq->recv_buffers >= 0); } return to_process; @@ -480,8 +475,9 @@ nicvf_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) static inline uint16_t __hot nicvf_process_cq_mseg_entry(struct cqe_rx_t *cqe_rx, - uint64_t mbuf_phys_off, uint8_t port_id, - struct rte_mbuf **rx_pkt, uint8_t rbptr_offset) + uint64_t mbuf_phys_off, + struct rte_mbuf **rx_pkt, uint8_t rbptr_offset, + uint64_t mbuf_init) { struct rte_mbuf *pkt, *seg, *prev; cqe_rx_word0_t cqe_rx_w0; @@ -500,12 +496,10 @@ nicvf_process_cq_mseg_entry(struct cqe_rx_t *cqe_rx, (rb_ptr[0] - cqe_rx_w1.align_pad, mbuf_phys_off); pkt->ol_flags = 0; - pkt->port = port_id; - pkt->data_off = RTE_PKTMBUF_HEADROOM + cqe_rx_w1.align_pad; - pkt->nb_segs = nb_segs; pkt->pkt_len = cqe_rx_w1.pkt_len; pkt->data_len = rb_sz[nicvf_frag_num(0)]; - rte_mbuf_refcnt_set(pkt, 1); + nicvf_mbuff_init_mseg_update( + pkt, mbuf_init, cqe_rx_w1.align_pad, nb_segs); pkt->packet_type = nicvf_rx_classify_pkt(cqe_rx_w0); nicvf_rx_offload(cqe_rx_w0, cqe_rx_w2, pkt); @@ -517,9 +511,7 @@ nicvf_process_cq_mseg_entry(struct cqe_rx_t *cqe_rx, prev->next = seg; seg->data_len = rb_sz[nicvf_frag_num(seg_idx)]; - seg->port = port_id; - seg->data_off = RTE_PKTMBUF_HEADROOM; - rte_mbuf_refcnt_set(seg, 1); + nicvf_mbuff_init_update(seg, mbuf_init, 0); prev = seg; } @@ -540,7 +532,7 @@ nicvf_recv_pkts_multiseg(void *rx_queue, struct rte_mbuf **rx_pkts, uint32_t i, to_process, cqe_head, buffers_consumed = 0; int32_t available_space = rxq->available_space; uint16_t nb_segs; - const uint8_t port_id = rxq->port_id; + const uint64_t mbuf_init = rxq->mbuf_initializer.value; const uint8_t rbptr_offset = rxq->rbptr_offset; cqe_head = rxq->head & cqe_mask; @@ -551,7 +543,7 @@ nicvf_recv_pkts_multiseg(void *rx_queue, struct rte_mbuf **rx_pkts, cq_entry = &desc[cqe_head]; cqe_rx = (struct cqe_rx_t *)cq_entry; nb_segs = nicvf_process_cq_mseg_entry(cqe_rx, mbuf_phys_off, - port_id, rx_pkts + i, rbptr_offset); + rx_pkts + i, rbptr_offset, mbuf_init); buffers_consumed += nb_segs; cqe_head = (cqe_head + 1) & cqe_mask; nicvf_prefetch_store_keep(rx_pkts[i]); @@ -562,11 +554,10 @@ nicvf_recv_pkts_multiseg(void *rx_queue, struct rte_mbuf **rx_pkts, rxq->head = cqe_head; nicvf_addr_write(rxq->cq_door, to_process); rxq->recv_buffers += buffers_consumed; - if (rxq->recv_buffers > rxq->rx_free_thresh) { - rxq->recv_buffers -= - nicvf_fill_rbdr(rxq, rxq->rx_free_thresh); - NICVF_RX_ASSERT(rxq->recv_buffers >= 0); - } + } + if (rxq->recv_buffers > rxq->rx_free_thresh) { + rxq->recv_buffers -= nicvf_fill_rbdr(rxq, rxq->rx_free_thresh); + NICVF_RX_ASSERT(rxq->recv_buffers >= 0); } return to_process; diff --git a/dpdk/drivers/net/thunderx/nicvf_rxtx.h b/dpdk/drivers/net/thunderx/nicvf_rxtx.h index 9dad8a5a..a3ccce29 100644 --- a/dpdk/drivers/net/thunderx/nicvf_rxtx.h +++ b/dpdk/drivers/net/thunderx/nicvf_rxtx.h @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2016. + * Copyright (C) Cavium, Inc. 2016. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * @@ -60,7 +60,7 @@ fill_sq_desc_gather(union sq_entry_t *entry, struct rte_mbuf *pkt) sqe.gather.subdesc_type = SQ_DESC_TYPE_GATHER; sqe.gather.ld_type = NIC_SEND_LD_TYPE_E_LDT; sqe.gather.size = pkt->data_len; - sqe.gather.addr = rte_mbuf_data_dma_addr(pkt); + sqe.gather.addr = rte_mbuf_data_iova(pkt); entry->buff[0] = sqe.buff[0]; entry->buff[1] = sqe.buff[1]; @@ -80,10 +80,37 @@ fill_sq_desc_gather(union sq_entry_t *entry, struct rte_mbuf *pkt) entry->buff[0] = (uint64_t)SQ_DESC_TYPE_GATHER << 60 | (uint64_t)NIC_SEND_LD_TYPE_E_LDT << 58 | pkt->data_len; - entry->buff[1] = rte_mbuf_data_dma_addr(pkt); + entry->buff[1] = rte_mbuf_data_iova(pkt); } #endif +static inline void +nicvf_mbuff_init_update(struct rte_mbuf *pkt, const uint64_t mbuf_init, + uint16_t apad) +{ + union mbuf_initializer init = {.value = mbuf_init}; +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + init.fields.data_off += apad; +#else + init.value += apad; +#endif + *(uint64_t *)(&pkt->rearm_data) = init.value; +} + +static inline void +nicvf_mbuff_init_mseg_update(struct rte_mbuf *pkt, const uint64_t mbuf_init, + uint16_t apad, uint16_t nb_segs) +{ + union mbuf_initializer init = {.value = mbuf_init}; +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + init.fields.data_off += apad; +#else + init.value += apad; +#endif + init.fields.nb_segs = nb_segs; + *(uint64_t *)(&pkt->rearm_data) = init.value; +} + uint32_t nicvf_dev_rx_queue_count(struct rte_eth_dev *dev, uint16_t queue_idx); uint32_t nicvf_dev_rbdr_refill(struct rte_eth_dev *dev, uint16_t queue_idx); diff --git a/dpdk/drivers/net/thunderx/nicvf_struct.h b/dpdk/drivers/net/thunderx/nicvf_struct.h index c52545d9..0f8208ef 100644 --- a/dpdk/drivers/net/thunderx/nicvf_struct.h +++ b/dpdk/drivers/net/thunderx/nicvf_struct.h @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2016. + * Copyright (C) Cavium, Inc. 2016. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * @@ -43,10 +43,10 @@ #include struct nicvf_rbdr { - uint64_t rbdr_status; - uint64_t rbdr_door; + uintptr_t rbdr_status; + uintptr_t rbdr_door; struct rbdr_entry_t *desc; - nicvf_phys_addr_t phys; + nicvf_iova_addr_t phys; uint32_t buffsz; uint32_t tail; uint32_t next_tail; @@ -56,10 +56,10 @@ struct nicvf_rbdr { struct nicvf_txq { union sq_entry_t *desc; - nicvf_phys_addr_t phys; + nicvf_iova_addr_t phys; struct rte_mbuf **txbuffs; - uint64_t sq_head; - uint64_t sq_door; + uintptr_t sq_head; + uintptr_t sq_door; struct rte_mempool *pool; struct nicvf *nic; void (*pool_free)(struct nicvf_txq *sq); @@ -72,11 +72,22 @@ struct nicvf_txq { uint16_t tx_free_thresh; } __rte_cache_aligned; +union mbuf_initializer { + struct { + uint16_t data_off; + uint16_t refcnt; + uint16_t nb_segs; + uint16_t port; + } fields; + uint64_t value; +}; + struct nicvf_rxq { uint64_t mbuf_phys_off; - uint64_t cq_status; - uint64_t cq_door; - nicvf_phys_addr_t phys; + uintptr_t cq_status; + uintptr_t cq_door; + union mbuf_initializer mbuf_initializer; + nicvf_iova_addr_t phys; union cq_entry_t *desc; struct nicvf_rbdr *shared_rbdr; struct nicvf *nic; @@ -89,7 +100,7 @@ struct nicvf_rxq { uint16_t queue_id; uint16_t precharge_cnt; uint8_t rx_drop_en; - uint8_t port_id; + uint16_t port_id; uint8_t rbptr_offset; } __rte_cache_aligned; @@ -113,12 +124,16 @@ struct nicvf { uint16_t subsystem_vendor_id; struct nicvf_rbdr *rbdr; struct nicvf_rss_reta_info rss_info; - struct rte_eth_dev *eth_dev; struct rte_intr_handle intr_handle; uint8_t cpi_alg; uint16_t mtu; bool vlan_filter_en; uint8_t mac_addr[ETHER_ADDR_LEN]; + /* secondary queue set support */ + uint8_t sqs_id; + uint8_t sqs_count; +#define MAX_SQS_PER_VF 11 + struct nicvf *snicvf[MAX_SQS_PER_VF]; } __rte_cache_aligned; #endif /* _THUNDERX_NICVF_STRUCT_H */ diff --git a/dpdk/drivers/net/thunderx/nicvf_svf.c b/dpdk/drivers/net/thunderx/nicvf_svf.c new file mode 100644 index 00000000..a7cc28a7 --- /dev/null +++ b/dpdk/drivers/net/thunderx/nicvf_svf.c @@ -0,0 +1,78 @@ +/* + * BSD LICENSE + * + * Copyright (C) Cavium, Inc. 2016. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include + +#include +#include + +#include "base/nicvf_bsvf.h" + +#include "nicvf_svf.h" + +void +nicvf_svf_push(struct nicvf *vf) +{ + struct svf_entry *entry = NULL; + + assert(vf != NULL); + + entry = rte_zmalloc("nicvf", sizeof(*entry), RTE_CACHE_LINE_SIZE); + if (entry == NULL) + rte_panic("Cannoc allocate memory for svf_entry\n"); + + entry->vf = vf; + + nicvf_bsvf_push(entry); +} + +struct nicvf * +nicvf_svf_pop(void) +{ + struct nicvf *vf; + struct svf_entry *entry; + + entry = nicvf_bsvf_pop(); + + vf = entry->vf; + + rte_free(entry); + + return vf; +} + +int +nicvf_svf_empty(void) +{ + return nicvf_bsvf_empty(); +} diff --git a/dpdk/drivers/net/thunderx/nicvf_svf.h b/dpdk/drivers/net/thunderx/nicvf_svf.h new file mode 100644 index 00000000..e58ab9d1 --- /dev/null +++ b/dpdk/drivers/net/thunderx/nicvf_svf.h @@ -0,0 +1,66 @@ +/* + * BSD LICENSE + * + * Copyright (C) Cavium, Inc. 2016. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __THUNDERX_NICVF_SVF_H__ +#define __THUNDERX_NICVF_SVF_H__ + +struct nicvf; + +/** + * Enqueue new VF to secondary qsets. + * + * @param entry + * Entry to be enqueued. + */ +void +nicvf_svf_push(struct nicvf *vf); + +/** + * Dequeue a VF from secondary qsets. + * + * @return + * Dequeued entry. + */ +struct nicvf * +nicvf_svf_pop(void); + +/** + * Check if the queue of secondary qsets is empty. + * + * @return + * 0 on non-empty + * otherwise empty + */ +int +nicvf_svf_empty(void); + +#endif /* __THUNDERX_NICVF_SVF_H__ */ diff --git a/dpdk/drivers/net/vhost/Makefile b/dpdk/drivers/net/vhost/Makefile index 050c5aa5..c411745b 100644 --- a/dpdk/drivers/net/vhost/Makefile +++ b/dpdk/drivers/net/vhost/Makefile @@ -37,13 +37,16 @@ include $(RTE_SDK)/mk/rte.vars.mk LIB = librte_pmd_vhost.a LDLIBS += -lpthread +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs -lrte_vhost +LDLIBS += -lrte_bus_vdev CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) EXPORT_MAP := rte_pmd_vhost_version.map -LIBABIVER := 1 +LIBABIVER := 2 # # all source are stored in SRCS-y @@ -55,12 +58,4 @@ SRCS-$(CONFIG_RTE_LIBRTE_PMD_VHOST) += rte_eth_vhost.c # SYMLINK-y-include += rte_eth_vhost.h -# this lib depends upon: -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_VHOST) += lib/librte_eal -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_VHOST) += lib/librte_mbuf -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_VHOST) += lib/librte_mempool -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_VHOST) += lib/librte_ether -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_VHOST) += lib/librte_kvargs -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_VHOST) += lib/librte_vhost - include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/net/vhost/rte_eth_vhost.c b/dpdk/drivers/net/vhost/rte_eth_vhost.c index 7539cd49..2536ee4a 100644 --- a/dpdk/drivers/net/vhost/rte_eth_vhost.c +++ b/dpdk/drivers/net/vhost/rte_eth_vhost.c @@ -33,31 +33,34 @@ #include #include #include -#ifdef RTE_LIBRTE_VHOST_NUMA -#include -#endif #include #include +#include #include #include -#include +#include #include -#include +#include #include #include "rte_eth_vhost.h" +enum {VIRTIO_RXQ, VIRTIO_TXQ, VIRTIO_QNUM}; + #define ETH_VHOST_IFACE_ARG "iface" #define ETH_VHOST_QUEUES_ARG "queues" #define ETH_VHOST_CLIENT_ARG "client" - -static const char *drivername = "VHOST PMD"; +#define ETH_VHOST_DEQUEUE_ZERO_COPY "dequeue-zero-copy" +#define ETH_VHOST_IOMMU_SUPPORT "iommu-support" +#define VHOST_MAX_PKT_BURST 32 static const char *valid_arguments[] = { ETH_VHOST_IFACE_ARG, ETH_VHOST_QUEUES_ARG, ETH_VHOST_CLIENT_ARG, + ETH_VHOST_DEQUEUE_ZERO_COPY, + ETH_VHOST_IOMMU_SUPPORT, NULL }; @@ -72,28 +75,49 @@ static struct ether_addr base_eth_addr = { } }; +enum vhost_xstats_pkts { + VHOST_UNDERSIZE_PKT = 0, + VHOST_64_PKT, + VHOST_65_TO_127_PKT, + VHOST_128_TO_255_PKT, + VHOST_256_TO_511_PKT, + VHOST_512_TO_1023_PKT, + VHOST_1024_TO_1522_PKT, + VHOST_1523_TO_MAX_PKT, + VHOST_BROADCAST_PKT, + VHOST_MULTICAST_PKT, + VHOST_UNICAST_PKT, + VHOST_ERRORS_PKT, + VHOST_ERRORS_FRAGMENTED, + VHOST_ERRORS_JABBER, + VHOST_UNKNOWN_PROTOCOL, + VHOST_XSTATS_MAX, +}; + +struct vhost_stats { + uint64_t pkts; + uint64_t bytes; + uint64_t missed_pkts; + uint64_t xstats[VHOST_XSTATS_MAX]; +}; + struct vhost_queue { int vid; rte_atomic32_t allow_queuing; rte_atomic32_t while_queuing; struct pmd_internal *internal; struct rte_mempool *mb_pool; - uint8_t port; + uint16_t port; uint16_t virtqueue_id; - uint64_t rx_pkts; - uint64_t tx_pkts; - uint64_t missed_pkts; - uint64_t rx_bytes; - uint64_t tx_bytes; + struct vhost_stats stats; }; struct pmd_internal { + rte_atomic32_t dev_attached; char *dev_name; char *iface_name; uint16_t max_queues; - uint64_t flags; - - volatile uint16_t once; + rte_atomic32_t started; }; struct internal_list { @@ -107,9 +131,6 @@ static struct internal_list_head internal_list = static pthread_mutex_t internal_list_lock = PTHREAD_MUTEX_INITIALIZER; -static rte_atomic16_t nb_started_ports; -static pthread_t session_th; - static struct rte_eth_link pmd_link = { .link_speed = 10000, .link_duplex = ETH_LINK_FULL_DUPLEX, @@ -127,11 +148,250 @@ struct rte_vhost_vring_state { static struct rte_vhost_vring_state *vring_states[RTE_MAX_ETHPORTS]; +#define VHOST_XSTATS_NAME_SIZE 64 + +struct vhost_xstats_name_off { + char name[VHOST_XSTATS_NAME_SIZE]; + uint64_t offset; +}; + +/* [rx]_is prepended to the name string here */ +static const struct vhost_xstats_name_off vhost_rxport_stat_strings[] = { + {"good_packets", + offsetof(struct vhost_queue, stats.pkts)}, + {"total_bytes", + offsetof(struct vhost_queue, stats.bytes)}, + {"missed_pkts", + offsetof(struct vhost_queue, stats.missed_pkts)}, + {"broadcast_packets", + offsetof(struct vhost_queue, stats.xstats[VHOST_BROADCAST_PKT])}, + {"multicast_packets", + offsetof(struct vhost_queue, stats.xstats[VHOST_MULTICAST_PKT])}, + {"unicast_packets", + offsetof(struct vhost_queue, stats.xstats[VHOST_UNICAST_PKT])}, + {"undersize_packets", + offsetof(struct vhost_queue, stats.xstats[VHOST_UNDERSIZE_PKT])}, + {"size_64_packets", + offsetof(struct vhost_queue, stats.xstats[VHOST_64_PKT])}, + {"size_65_to_127_packets", + offsetof(struct vhost_queue, stats.xstats[VHOST_65_TO_127_PKT])}, + {"size_128_to_255_packets", + offsetof(struct vhost_queue, stats.xstats[VHOST_128_TO_255_PKT])}, + {"size_256_to_511_packets", + offsetof(struct vhost_queue, stats.xstats[VHOST_256_TO_511_PKT])}, + {"size_512_to_1023_packets", + offsetof(struct vhost_queue, stats.xstats[VHOST_512_TO_1023_PKT])}, + {"size_1024_to_1522_packets", + offsetof(struct vhost_queue, stats.xstats[VHOST_1024_TO_1522_PKT])}, + {"size_1523_to_max_packets", + offsetof(struct vhost_queue, stats.xstats[VHOST_1523_TO_MAX_PKT])}, + {"errors_with_bad_CRC", + offsetof(struct vhost_queue, stats.xstats[VHOST_ERRORS_PKT])}, + {"fragmented_errors", + offsetof(struct vhost_queue, stats.xstats[VHOST_ERRORS_FRAGMENTED])}, + {"jabber_errors", + offsetof(struct vhost_queue, stats.xstats[VHOST_ERRORS_JABBER])}, + {"unknown_protos_packets", + offsetof(struct vhost_queue, stats.xstats[VHOST_UNKNOWN_PROTOCOL])}, +}; + +/* [tx]_ is prepended to the name string here */ +static const struct vhost_xstats_name_off vhost_txport_stat_strings[] = { + {"good_packets", + offsetof(struct vhost_queue, stats.pkts)}, + {"total_bytes", + offsetof(struct vhost_queue, stats.bytes)}, + {"missed_pkts", + offsetof(struct vhost_queue, stats.missed_pkts)}, + {"broadcast_packets", + offsetof(struct vhost_queue, stats.xstats[VHOST_BROADCAST_PKT])}, + {"multicast_packets", + offsetof(struct vhost_queue, stats.xstats[VHOST_MULTICAST_PKT])}, + {"unicast_packets", + offsetof(struct vhost_queue, stats.xstats[VHOST_UNICAST_PKT])}, + {"undersize_packets", + offsetof(struct vhost_queue, stats.xstats[VHOST_UNDERSIZE_PKT])}, + {"size_64_packets", + offsetof(struct vhost_queue, stats.xstats[VHOST_64_PKT])}, + {"size_65_to_127_packets", + offsetof(struct vhost_queue, stats.xstats[VHOST_65_TO_127_PKT])}, + {"size_128_to_255_packets", + offsetof(struct vhost_queue, stats.xstats[VHOST_128_TO_255_PKT])}, + {"size_256_to_511_packets", + offsetof(struct vhost_queue, stats.xstats[VHOST_256_TO_511_PKT])}, + {"size_512_to_1023_packets", + offsetof(struct vhost_queue, stats.xstats[VHOST_512_TO_1023_PKT])}, + {"size_1024_to_1522_packets", + offsetof(struct vhost_queue, stats.xstats[VHOST_1024_TO_1522_PKT])}, + {"size_1523_to_max_packets", + offsetof(struct vhost_queue, stats.xstats[VHOST_1523_TO_MAX_PKT])}, + {"errors_with_bad_CRC", + offsetof(struct vhost_queue, stats.xstats[VHOST_ERRORS_PKT])}, +}; + +#define VHOST_NB_XSTATS_RXPORT (sizeof(vhost_rxport_stat_strings) / \ + sizeof(vhost_rxport_stat_strings[0])) + +#define VHOST_NB_XSTATS_TXPORT (sizeof(vhost_txport_stat_strings) / \ + sizeof(vhost_txport_stat_strings[0])) + +static void +vhost_dev_xstats_reset(struct rte_eth_dev *dev) +{ + struct vhost_queue *vq = NULL; + unsigned int i = 0; + + for (i = 0; i < dev->data->nb_rx_queues; i++) { + vq = dev->data->rx_queues[i]; + if (!vq) + continue; + memset(&vq->stats, 0, sizeof(vq->stats)); + } + for (i = 0; i < dev->data->nb_tx_queues; i++) { + vq = dev->data->tx_queues[i]; + if (!vq) + continue; + memset(&vq->stats, 0, sizeof(vq->stats)); + } +} + +static int +vhost_dev_xstats_get_names(struct rte_eth_dev *dev __rte_unused, + struct rte_eth_xstat_name *xstats_names, + unsigned int limit __rte_unused) +{ + unsigned int t = 0; + int count = 0; + int nstats = VHOST_NB_XSTATS_RXPORT + VHOST_NB_XSTATS_TXPORT; + + if (!xstats_names) + return nstats; + for (t = 0; t < VHOST_NB_XSTATS_RXPORT; t++) { + snprintf(xstats_names[count].name, + sizeof(xstats_names[count].name), + "rx_%s", vhost_rxport_stat_strings[t].name); + count++; + } + for (t = 0; t < VHOST_NB_XSTATS_TXPORT; t++) { + snprintf(xstats_names[count].name, + sizeof(xstats_names[count].name), + "tx_%s", vhost_txport_stat_strings[t].name); + count++; + } + return count; +} + +static int +vhost_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, + unsigned int n) +{ + unsigned int i; + unsigned int t; + unsigned int count = 0; + struct vhost_queue *vq = NULL; + unsigned int nxstats = VHOST_NB_XSTATS_RXPORT + VHOST_NB_XSTATS_TXPORT; + + if (n < nxstats) + return nxstats; + + for (i = 0; i < dev->data->nb_rx_queues; i++) { + vq = dev->data->rx_queues[i]; + if (!vq) + continue; + vq->stats.xstats[VHOST_UNICAST_PKT] = vq->stats.pkts + - (vq->stats.xstats[VHOST_BROADCAST_PKT] + + vq->stats.xstats[VHOST_MULTICAST_PKT]); + } + for (i = 0; i < dev->data->nb_tx_queues; i++) { + vq = dev->data->tx_queues[i]; + if (!vq) + continue; + vq->stats.xstats[VHOST_UNICAST_PKT] = vq->stats.pkts + + vq->stats.missed_pkts + - (vq->stats.xstats[VHOST_BROADCAST_PKT] + + vq->stats.xstats[VHOST_MULTICAST_PKT]); + } + for (t = 0; t < VHOST_NB_XSTATS_RXPORT; t++) { + xstats[count].value = 0; + for (i = 0; i < dev->data->nb_rx_queues; i++) { + vq = dev->data->rx_queues[i]; + if (!vq) + continue; + xstats[count].value += + *(uint64_t *)(((char *)vq) + + vhost_rxport_stat_strings[t].offset); + } + xstats[count].id = count; + count++; + } + for (t = 0; t < VHOST_NB_XSTATS_TXPORT; t++) { + xstats[count].value = 0; + for (i = 0; i < dev->data->nb_tx_queues; i++) { + vq = dev->data->tx_queues[i]; + if (!vq) + continue; + xstats[count].value += + *(uint64_t *)(((char *)vq) + + vhost_txport_stat_strings[t].offset); + } + xstats[count].id = count; + count++; + } + return count; +} + +static inline void +vhost_count_multicast_broadcast(struct vhost_queue *vq, + struct rte_mbuf *mbuf) +{ + struct ether_addr *ea = NULL; + struct vhost_stats *pstats = &vq->stats; + + ea = rte_pktmbuf_mtod(mbuf, struct ether_addr *); + if (is_multicast_ether_addr(ea)) { + if (is_broadcast_ether_addr(ea)) + pstats->xstats[VHOST_BROADCAST_PKT]++; + else + pstats->xstats[VHOST_MULTICAST_PKT]++; + } +} + +static void +vhost_update_packet_xstats(struct vhost_queue *vq, + struct rte_mbuf **bufs, + uint16_t count) +{ + uint32_t pkt_len = 0; + uint64_t i = 0; + uint64_t index; + struct vhost_stats *pstats = &vq->stats; + + for (i = 0; i < count ; i++) { + pkt_len = bufs[i]->pkt_len; + if (pkt_len == 64) { + pstats->xstats[VHOST_64_PKT]++; + } else if (pkt_len > 64 && pkt_len < 1024) { + index = (sizeof(pkt_len) * 8) + - __builtin_clz(pkt_len) - 5; + pstats->xstats[index]++; + } else { + if (pkt_len < 64) + pstats->xstats[VHOST_UNDERSIZE_PKT]++; + else if (pkt_len <= 1522) + pstats->xstats[VHOST_1024_TO_1522_PKT]++; + else if (pkt_len > 1522) + pstats->xstats[VHOST_1523_TO_MAX_PKT]++; + } + vhost_count_multicast_broadcast(vq, bufs[i]); + } +} + static uint16_t eth_vhost_rx(void *q, struct rte_mbuf **bufs, uint16_t nb_bufs) { struct vhost_queue *r = q; uint16_t i, nb_rx = 0; + uint16_t nb_receive = nb_bufs; if (unlikely(rte_atomic32_read(&r->allow_queuing) == 0)) return 0; @@ -142,16 +402,30 @@ eth_vhost_rx(void *q, struct rte_mbuf **bufs, uint16_t nb_bufs) goto out; /* Dequeue packets from guest TX queue */ - nb_rx = rte_vhost_dequeue_burst(r->vid, - r->virtqueue_id, r->mb_pool, bufs, nb_bufs); + while (nb_receive) { + uint16_t nb_pkts; + uint16_t num = (uint16_t)RTE_MIN(nb_receive, + VHOST_MAX_PKT_BURST); - r->rx_pkts += nb_rx; + nb_pkts = rte_vhost_dequeue_burst(r->vid, r->virtqueue_id, + r->mb_pool, &bufs[nb_rx], + num); + + nb_rx += nb_pkts; + nb_receive -= nb_pkts; + if (nb_pkts < num) + break; + } + + r->stats.pkts += nb_rx; for (i = 0; likely(i < nb_rx); i++) { bufs[i]->port = r->port; - r->rx_bytes += bufs[i]->pkt_len; + r->stats.bytes += bufs[i]->pkt_len; } + vhost_update_packet_xstats(r, bufs, nb_rx); + out: rte_atomic32_set(&r->while_queuing, 0); @@ -163,6 +437,7 @@ eth_vhost_tx(void *q, struct rte_mbuf **bufs, uint16_t nb_bufs) { struct vhost_queue *r = q; uint16_t i, nb_tx = 0; + uint16_t nb_send = nb_bufs; if (unlikely(rte_atomic32_read(&r->allow_queuing) == 0)) return 0; @@ -173,14 +448,34 @@ eth_vhost_tx(void *q, struct rte_mbuf **bufs, uint16_t nb_bufs) goto out; /* Enqueue packets to guest RX queue */ - nb_tx = rte_vhost_enqueue_burst(r->vid, - r->virtqueue_id, bufs, nb_bufs); + while (nb_send) { + uint16_t nb_pkts; + uint16_t num = (uint16_t)RTE_MIN(nb_send, + VHOST_MAX_PKT_BURST); - r->tx_pkts += nb_tx; - r->missed_pkts += nb_bufs - nb_tx; + nb_pkts = rte_vhost_enqueue_burst(r->vid, r->virtqueue_id, + &bufs[nb_tx], num); + + nb_tx += nb_pkts; + nb_send -= nb_pkts; + if (nb_pkts < num) + break; + } + + r->stats.pkts += nb_tx; + r->stats.missed_pkts += nb_bufs - nb_tx; for (i = 0; likely(i < nb_tx); i++) - r->tx_bytes += bufs[i]->pkt_len; + r->stats.bytes += bufs[i]->pkt_len; + + vhost_update_packet_xstats(r, bufs, nb_tx); + + /* According to RFC2863 page42 section ifHCOutMulticastPkts and + * ifHCOutBroadcastPkts, the counters "multicast" and "broadcast" + * are increased when packets are not transmitted successfully. + */ + for (i = nb_tx; i < nb_bufs; i++) + vhost_count_multicast_broadcast(r, bufs[i]); for (i = 0; likely(i < nb_tx); i++) rte_pktmbuf_free(bufs[i]); @@ -224,6 +519,38 @@ find_internal_resource(char *ifname) return list; } +static void +update_queuing_status(struct rte_eth_dev *dev) +{ + struct pmd_internal *internal = dev->data->dev_private; + struct vhost_queue *vq; + unsigned int i; + int allow_queuing = 1; + + if (rte_atomic32_read(&internal->started) == 0 || + rte_atomic32_read(&internal->dev_attached) == 0) + allow_queuing = 0; + + /* Wait until rx/tx_pkt_burst stops accessing vhost device */ + for (i = 0; i < dev->data->nb_rx_queues; i++) { + vq = dev->data->rx_queues[i]; + if (vq == NULL) + continue; + rte_atomic32_set(&vq->allow_queuing, allow_queuing); + while (rte_atomic32_read(&vq->while_queuing)) + rte_pause(); + } + + for (i = 0; i < dev->data->nb_tx_queues; i++) { + vq = dev->data->tx_queues[i]; + if (vq == NULL) + continue; + rte_atomic32_set(&vq->allow_queuing, allow_queuing); + while (rte_atomic32_read(&vq->while_queuing)) + rte_pause(); + } +} + static int new_device(int vid) { @@ -270,27 +597,20 @@ new_device(int vid) vq->port = eth_dev->data->port_id; } - for (i = 0; i < rte_vhost_get_queue_num(vid) * VIRTIO_QNUM; i++) + for (i = 0; i < rte_vhost_get_vring_num(vid); i++) rte_vhost_enable_guest_notification(vid, i, 0); + rte_vhost_get_mtu(vid, ð_dev->data->mtu); + eth_dev->data->dev_link.link_status = ETH_LINK_UP; - for (i = 0; i < eth_dev->data->nb_rx_queues; i++) { - vq = eth_dev->data->rx_queues[i]; - if (vq == NULL) - continue; - rte_atomic32_set(&vq->allow_queuing, 1); - } - for (i = 0; i < eth_dev->data->nb_tx_queues; i++) { - vq = eth_dev->data->tx_queues[i]; - if (vq == NULL) - continue; - rte_atomic32_set(&vq->allow_queuing, 1); - } + rte_atomic32_set(&internal->dev_attached, 1); + update_queuing_status(eth_dev); RTE_LOG(INFO, PMD, "New connection established\n"); - _rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_INTR_LSC); + _rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_INTR_LSC, + NULL, NULL); return 0; } @@ -299,6 +619,7 @@ static void destroy_device(int vid) { struct rte_eth_dev *eth_dev; + struct pmd_internal *internal; struct vhost_queue *vq; struct internal_list *list; char ifname[PATH_MAX]; @@ -312,24 +633,10 @@ destroy_device(int vid) return; } eth_dev = list->eth_dev; + internal = eth_dev->data->dev_private; - /* Wait until rx/tx_pkt_burst stops accessing vhost device */ - for (i = 0; i < eth_dev->data->nb_rx_queues; i++) { - vq = eth_dev->data->rx_queues[i]; - if (vq == NULL) - continue; - rte_atomic32_set(&vq->allow_queuing, 0); - while (rte_atomic32_read(&vq->while_queuing)) - rte_pause(); - } - for (i = 0; i < eth_dev->data->nb_tx_queues; i++) { - vq = eth_dev->data->tx_queues[i]; - if (vq == NULL) - continue; - rte_atomic32_set(&vq->allow_queuing, 0); - while (rte_atomic32_read(&vq->while_queuing)) - rte_pause(); - } + rte_atomic32_set(&internal->dev_attached, 0); + update_queuing_status(eth_dev); eth_dev->data->dev_link.link_status = ETH_LINK_DOWN; @@ -357,7 +664,8 @@ destroy_device(int vid) RTE_LOG(INFO, PMD, "Connection closed\n"); - _rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_INTR_LSC); + _rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_INTR_LSC, + NULL, NULL); } static int @@ -386,13 +694,20 @@ vring_state_changed(int vid, uint16_t vring, int enable) RTE_LOG(INFO, PMD, "vring%u is %s\n", vring, enable ? "enabled" : "disabled"); - _rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_QUEUE_STATE); + _rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_QUEUE_STATE, + NULL, NULL); return 0; } +static struct vhost_device_ops vhost_ops = { + .new_device = new_device, + .destroy_device = destroy_device, + .vring_state_changed = vring_state_changed, +}; + int -rte_eth_vhost_get_queue_event(uint8_t port_id, +rte_eth_vhost_get_queue_event(uint16_t port_id, struct rte_eth_vhost_queue_event *event) { struct rte_vhost_vring_state *state; @@ -428,69 +743,44 @@ rte_eth_vhost_get_queue_event(uint8_t port_id, return -1; } -static void * -vhost_driver_session(void *param __rte_unused) +int +rte_eth_vhost_get_vid_from_port_id(uint16_t port_id) { - static struct virtio_net_device_ops vhost_ops; + struct internal_list *list; + struct rte_eth_dev *eth_dev; + struct vhost_queue *vq; + int vid = -1; - /* set vhost arguments */ - vhost_ops.new_device = new_device; - vhost_ops.destroy_device = destroy_device; - vhost_ops.vring_state_changed = vring_state_changed; - if (rte_vhost_driver_callback_register(&vhost_ops) < 0) - RTE_LOG(ERR, PMD, "Can't register callbacks\n"); + if (!rte_eth_dev_is_valid_port(port_id)) + return -1; - /* start event handling */ - rte_vhost_driver_session_start(); + pthread_mutex_lock(&internal_list_lock); - return NULL; -} + TAILQ_FOREACH(list, &internal_list, next) { + eth_dev = list->eth_dev; + if (eth_dev->data->port_id == port_id) { + vq = eth_dev->data->rx_queues[0]; + if (vq) { + vid = vq->vid; + } + break; + } + } -static int -vhost_driver_session_start(void) -{ - int ret; + pthread_mutex_unlock(&internal_list_lock); - ret = pthread_create(&session_th, - NULL, vhost_driver_session, NULL); - if (ret) - RTE_LOG(ERR, PMD, "Can't create a thread\n"); - - return ret; -} - -static void -vhost_driver_session_stop(void) -{ - int ret; - - ret = pthread_cancel(session_th); - if (ret) - RTE_LOG(ERR, PMD, "Can't cancel the thread\n"); - - ret = pthread_join(session_th, NULL); - if (ret) - RTE_LOG(ERR, PMD, "Can't join the thread\n"); + return vid; } static int eth_dev_start(struct rte_eth_dev *dev) { struct pmd_internal *internal = dev->data->dev_private; - int ret = 0; - if (rte_atomic16_cmpset(&internal->once, 0, 1)) { - ret = rte_vhost_driver_register(internal->iface_name, - internal->flags); - if (ret) - return ret; - } + rte_atomic32_set(&internal->started, 1); + update_queuing_status(dev); - /* We need only one message handling thread */ - if (rte_atomic16_add_return(&nb_started_ports, 1) == 1) - ret = vhost_driver_session_start(); - - return ret; + return 0; } static void @@ -498,11 +788,45 @@ eth_dev_stop(struct rte_eth_dev *dev) { struct pmd_internal *internal = dev->data->dev_private; - if (rte_atomic16_cmpset(&internal->once, 1, 0)) - rte_vhost_driver_unregister(internal->iface_name); + rte_atomic32_set(&internal->started, 0); + update_queuing_status(dev); +} - if (rte_atomic16_sub_return(&nb_started_ports, 1) == 0) - vhost_driver_session_stop(); +static void +eth_dev_close(struct rte_eth_dev *dev) +{ + struct pmd_internal *internal; + struct internal_list *list; + unsigned int i; + + internal = dev->data->dev_private; + if (!internal) + return; + + eth_dev_stop(dev); + + rte_vhost_driver_unregister(internal->iface_name); + + list = find_internal_resource(internal->iface_name); + if (!list) + return; + + pthread_mutex_lock(&internal_list_lock); + TAILQ_REMOVE(&internal_list, list, next); + pthread_mutex_unlock(&internal_list_lock); + rte_free(list); + + for (i = 0; i < dev->data->nb_rx_queues; i++) + rte_free(dev->data->rx_queues[i]); + for (i = 0; i < dev->data->nb_tx_queues; i++) + rte_free(dev->data->tx_queues[i]); + + rte_free(dev->data->mac_addrs); + free(internal->dev_name); + free(internal->iface_name); + rte_free(internal); + + dev->data->dev_private = NULL; } static int @@ -561,7 +885,6 @@ eth_dev_info(struct rte_eth_dev *dev, return; } - dev_info->driver_name = drivername; dev_info->max_mac_addrs = 1; dev_info->max_rx_pktlen = (uint32_t)-1; dev_info->max_rx_queues = internal->max_queues; @@ -569,7 +892,7 @@ eth_dev_info(struct rte_eth_dev *dev, dev_info->min_rx_bufsize = 0; } -static void +static int eth_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) { unsigned i; @@ -582,10 +905,10 @@ eth_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) if (dev->data->rx_queues[i] == NULL) continue; vq = dev->data->rx_queues[i]; - stats->q_ipackets[i] = vq->rx_pkts; + stats->q_ipackets[i] = vq->stats.pkts; rx_total += stats->q_ipackets[i]; - stats->q_ibytes[i] = vq->rx_bytes; + stats->q_ibytes[i] = vq->stats.bytes; rx_total_bytes += stats->q_ibytes[i]; } @@ -594,11 +917,11 @@ eth_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) if (dev->data->tx_queues[i] == NULL) continue; vq = dev->data->tx_queues[i]; - stats->q_opackets[i] = vq->tx_pkts; - tx_missed_total += vq->missed_pkts; + stats->q_opackets[i] = vq->stats.pkts; + tx_missed_total += vq->stats.missed_pkts; tx_total += stats->q_opackets[i]; - stats->q_obytes[i] = vq->tx_bytes; + stats->q_obytes[i] = vq->stats.bytes; tx_total_bytes += stats->q_obytes[i]; } @@ -607,6 +930,8 @@ eth_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) stats->oerrors = tx_missed_total; stats->ibytes = rx_total_bytes; stats->obytes = tx_total_bytes; + + return 0; } static void @@ -619,16 +944,16 @@ eth_stats_reset(struct rte_eth_dev *dev) if (dev->data->rx_queues[i] == NULL) continue; vq = dev->data->rx_queues[i]; - vq->rx_pkts = 0; - vq->rx_bytes = 0; + vq->stats.pkts = 0; + vq->stats.bytes = 0; } for (i = 0; i < dev->data->nb_tx_queues; i++) { if (dev->data->tx_queues[i] == NULL) continue; vq = dev->data->tx_queues[i]; - vq->tx_pkts = 0; - vq->tx_bytes = 0; - vq->missed_pkts = 0; + vq->stats.pkts = 0; + vq->stats.bytes = 0; + vq->stats.missed_pkts = 0; } } @@ -638,6 +963,16 @@ eth_queue_release(void *q) rte_free(q); } +static int +eth_tx_done_cleanup(void *txq __rte_unused, uint32_t free_cnt __rte_unused) +{ + /* + * vHost does not hang onto mbuf. eth_vhost_tx() copies packet data + * and releases mbuf, so nothing to cleanup. + */ + return 0; +} + static int eth_link_update(struct rte_eth_dev *dev __rte_unused, int wait_to_complete __rte_unused) @@ -645,49 +980,45 @@ eth_link_update(struct rte_eth_dev *dev __rte_unused, return 0; } -/** - * Disable features in feature_mask. Returns 0 on success. - */ -int -rte_eth_vhost_feature_disable(uint64_t feature_mask) +static uint32_t +eth_rx_queue_count(struct rte_eth_dev *dev, uint16_t rx_queue_id) { - return rte_vhost_feature_disable(feature_mask); -} + struct vhost_queue *vq; -/** - * Enable features in feature_mask. Returns 0 on success. - */ -int -rte_eth_vhost_feature_enable(uint64_t feature_mask) -{ - return rte_vhost_feature_enable(feature_mask); -} + vq = dev->data->rx_queues[rx_queue_id]; + if (vq == NULL) + return 0; -/* Returns currently supported vhost features */ -uint64_t -rte_eth_vhost_feature_get(void) -{ - return rte_vhost_feature_get(); + return rte_vhost_rx_queue_count(vq->vid, vq->virtqueue_id); } static const struct eth_dev_ops ops = { .dev_start = eth_dev_start, .dev_stop = eth_dev_stop, + .dev_close = eth_dev_close, .dev_configure = eth_dev_configure, .dev_infos_get = eth_dev_info, .rx_queue_setup = eth_rx_queue_setup, .tx_queue_setup = eth_tx_queue_setup, .rx_queue_release = eth_queue_release, .tx_queue_release = eth_queue_release, + .tx_done_cleanup = eth_tx_done_cleanup, + .rx_queue_count = eth_rx_queue_count, .link_update = eth_link_update, .stats_get = eth_stats_get, .stats_reset = eth_stats_reset, + .xstats_reset = vhost_dev_xstats_reset, + .xstats_get = vhost_dev_xstats_get, + .xstats_get_names = vhost_dev_xstats_get_names, }; +static struct rte_vdev_driver pmd_vhost_drv; + static int -eth_dev_vhost_create(const char *name, char *iface_name, int16_t queues, - const unsigned numa_node, uint64_t flags) +eth_dev_vhost_create(struct rte_vdev_device *dev, char *iface_name, + int16_t queues, const unsigned int numa_node, uint64_t flags) { + const char *name = rte_vdev_device_name(dev); struct rte_eth_dev_data *data = NULL; struct pmd_internal *internal = NULL; struct rte_eth_dev *eth_dev = NULL; @@ -698,23 +1029,19 @@ eth_dev_vhost_create(const char *name, char *iface_name, int16_t queues, RTE_LOG(INFO, PMD, "Creating VHOST-USER backend on numa socket %u\n", numa_node); - /* now do all data allocation - for eth_dev structure, dummy pci driver - * and internal (private) data + /* now do all data allocation - for eth_dev structure and internal + * (private) data */ data = rte_zmalloc_socket(name, sizeof(*data), 0, numa_node); if (data == NULL) goto error; - internal = rte_zmalloc_socket(name, sizeof(*internal), 0, numa_node); - if (internal == NULL) - goto error; - list = rte_zmalloc_socket(name, sizeof(*list), 0, numa_node); if (list == NULL) goto error; /* reserve an ethdev entry */ - eth_dev = rte_eth_dev_allocate(name, RTE_ETH_DEV_VIRTUAL); + eth_dev = rte_eth_vdev_allocate(dev, sizeof(*internal)); if (eth_dev == NULL) goto error; @@ -729,21 +1056,18 @@ eth_dev_vhost_create(const char *name, char *iface_name, int16_t queues, if (vring_state == NULL) goto error; - TAILQ_INIT(ð_dev->link_intr_cbs); - /* now put it all together * - store queue data in internal, - * - store numa_node info in ethdev data * - point eth_dev_data to internals * - and point eth_dev structure to new eth_dev_data structure */ + internal = eth_dev->data->dev_private; internal->dev_name = strdup(name); if (internal->dev_name == NULL) goto error; internal->iface_name = strdup(iface_name); if (internal->iface_name == NULL) goto error; - internal->flags = flags; list->eth_dev = eth_dev; pthread_mutex_lock(&internal_list_lock); @@ -753,36 +1077,46 @@ eth_dev_vhost_create(const char *name, char *iface_name, int16_t queues, rte_spinlock_init(&vring_state->lock); vring_states[eth_dev->data->port_id] = vring_state; - data->dev_private = internal; - data->port_id = eth_dev->data->port_id; - memmove(data->name, eth_dev->data->name, sizeof(data->name)); + /* We'll replace the 'data' originally allocated by eth_dev. So the + * vhost PMD resources won't be shared between multi processes. + */ + rte_memcpy(data, eth_dev->data, sizeof(*data)); + eth_dev->data = data; + data->nb_rx_queues = queues; data->nb_tx_queues = queues; internal->max_queues = queues; data->dev_link = pmd_link; data->mac_addrs = eth_addr; + data->dev_flags = RTE_ETH_DEV_INTR_LSC; - /* We'll replace the 'data' originally allocated by eth_dev. So the - * vhost PMD resources won't be shared between multi processes. - */ - eth_dev->data = data; eth_dev->dev_ops = &ops; - eth_dev->driver = NULL; - data->dev_flags = - RTE_ETH_DEV_DETACHABLE | RTE_ETH_DEV_INTR_LSC; - data->kdrv = RTE_KDRV_NONE; - data->drv_name = internal->dev_name; - data->numa_node = numa_node; /* finally assign rx and tx ops */ eth_dev->rx_pkt_burst = eth_vhost_rx; eth_dev->tx_pkt_burst = eth_vhost_tx; + if (rte_vhost_driver_register(iface_name, flags)) + goto error; + + if (rte_vhost_driver_callback_register(iface_name, &vhost_ops) < 0) { + RTE_LOG(ERR, PMD, "Can't register callbacks\n"); + goto error; + } + + if (rte_vhost_driver_start(iface_name) < 0) { + RTE_LOG(ERR, PMD, "Failed to start driver for %s\n", + iface_name); + goto error; + } + return data->port_id; error: - if (internal) + if (internal) { + free(internal->iface_name); free(internal->dev_name); + } rte_free(vring_state); rte_free(eth_addr); if (eth_dev) @@ -823,7 +1157,7 @@ open_int(const char *key __rte_unused, const char *value, void *extra_args) } static int -rte_pmd_vhost_devinit(const char *name, const char *params) +rte_pmd_vhost_probe(struct rte_vdev_device *dev) { struct rte_kvargs *kvlist = NULL; int ret = 0; @@ -831,10 +1165,13 @@ rte_pmd_vhost_devinit(const char *name, const char *params) uint16_t queues; uint64_t flags = 0; int client_mode = 0; + int dequeue_zero_copy = 0; + int iommu_support = 0; - RTE_LOG(INFO, PMD, "Initializing pmd_vhost for %s\n", name); + RTE_LOG(INFO, PMD, "Initializing pmd_vhost for %s\n", + rte_vdev_device_name(dev)); - kvlist = rte_kvargs_parse(params, valid_arguments); + kvlist = rte_kvargs_parse(rte_vdev_device_args(dev), valid_arguments); if (kvlist == NULL) return -1; @@ -867,7 +1204,31 @@ rte_pmd_vhost_devinit(const char *name, const char *params) flags |= RTE_VHOST_USER_CLIENT; } - eth_dev_vhost_create(name, iface_name, queues, rte_socket_id(), flags); + if (rte_kvargs_count(kvlist, ETH_VHOST_DEQUEUE_ZERO_COPY) == 1) { + ret = rte_kvargs_process(kvlist, ETH_VHOST_DEQUEUE_ZERO_COPY, + &open_int, &dequeue_zero_copy); + if (ret < 0) + goto out_free; + + if (dequeue_zero_copy) + flags |= RTE_VHOST_USER_DEQUEUE_ZERO_COPY; + } + + if (rte_kvargs_count(kvlist, ETH_VHOST_IOMMU_SUPPORT) == 1) { + ret = rte_kvargs_process(kvlist, ETH_VHOST_IOMMU_SUPPORT, + &open_int, &iommu_support); + if (ret < 0) + goto out_free; + + if (iommu_support) + flags |= RTE_VHOST_USER_IOMMU_SUPPORT; + } + + if (dev->device.numa_node == SOCKET_ID_ANY) + dev->device.numa_node = rte_socket_id(); + + eth_dev_vhost_create(dev, iface_name, queues, dev->device.numa_node, + flags); out_free: rte_kvargs_free(kvlist); @@ -875,13 +1236,12 @@ out_free: } static int -rte_pmd_vhost_devuninit(const char *name) +rte_pmd_vhost_remove(struct rte_vdev_device *dev) { + const char *name; struct rte_eth_dev *eth_dev = NULL; - struct pmd_internal *internal; - struct internal_list *list; - unsigned int i; + name = rte_vdev_device_name(dev); RTE_LOG(INFO, PMD, "Un-Initializing pmd_vhost for %s\n", name); /* find an ethdev entry */ @@ -889,48 +1249,25 @@ rte_pmd_vhost_devuninit(const char *name) if (eth_dev == NULL) return -ENODEV; - internal = eth_dev->data->dev_private; - if (internal == NULL) - return -ENODEV; - - list = find_internal_resource(internal->iface_name); - if (list == NULL) - return -ENODEV; - - pthread_mutex_lock(&internal_list_lock); - TAILQ_REMOVE(&internal_list, list, next); - pthread_mutex_unlock(&internal_list_lock); - rte_free(list); - - eth_dev_stop(eth_dev); + eth_dev_close(eth_dev); rte_free(vring_states[eth_dev->data->port_id]); vring_states[eth_dev->data->port_id] = NULL; - free(internal->dev_name); - free(internal->iface_name); - - for (i = 0; i < eth_dev->data->nb_rx_queues; i++) - rte_free(eth_dev->data->rx_queues[i]); - for (i = 0; i < eth_dev->data->nb_tx_queues; i++) - rte_free(eth_dev->data->tx_queues[i]); - - rte_free(eth_dev->data->mac_addrs); rte_free(eth_dev->data); - rte_free(internal); rte_eth_dev_release_port(eth_dev); return 0; } -static struct rte_driver pmd_vhost_drv = { - .type = PMD_VDEV, - .init = rte_pmd_vhost_devinit, - .uninit = rte_pmd_vhost_devuninit, +static struct rte_vdev_driver pmd_vhost_drv = { + .probe = rte_pmd_vhost_probe, + .remove = rte_pmd_vhost_remove, }; -PMD_REGISTER_DRIVER(pmd_vhost_drv, eth_vhost); -DRIVER_REGISTER_PARAM_STRING(eth_vhost, +RTE_PMD_REGISTER_VDEV(net_vhost, pmd_vhost_drv); +RTE_PMD_REGISTER_ALIAS(net_vhost, eth_vhost); +RTE_PMD_REGISTER_PARAM_STRING(net_vhost, "iface= " "queues="); diff --git a/dpdk/drivers/net/vhost/rte_eth_vhost.h b/dpdk/drivers/net/vhost/rte_eth_vhost.h index ff5d877b..948f3c81 100644 --- a/dpdk/drivers/net/vhost/rte_eth_vhost.h +++ b/dpdk/drivers/net/vhost/rte_eth_vhost.h @@ -41,37 +41,7 @@ extern "C" { #include #include -#include - -/** - * Disable features in feature_mask. - * - * @param feature_mask - * Vhost features defined in "linux/virtio_net.h". - * @return - * - On success, zero. - * - On failure, a negative value. - */ -int rte_eth_vhost_feature_disable(uint64_t feature_mask); - -/** - * Enable features in feature_mask. - * - * @param feature_mask - * Vhost features defined in "linux/virtio_net.h". - * @return - * - On success, zero. - * - On failure, a negative value. - */ -int rte_eth_vhost_feature_enable(uint64_t feature_mask); - -/** - * Returns currently supported vhost features. - * - * @return - * Vhost features defined in "linux/virtio_net.h". - */ -uint64_t rte_eth_vhost_feature_get(void); +#include /* * Event description. @@ -99,9 +69,18 @@ struct rte_eth_vhost_queue_event { * - On success, zero. * - On failure, a negative value. */ -int rte_eth_vhost_get_queue_event(uint8_t port_id, +int rte_eth_vhost_get_queue_event(uint16_t port_id, struct rte_eth_vhost_queue_event *event); +/** + * Get the 'vid' value associated with the specified port. + * + * @return + * - On success, the 'vid' associated with 'port_id'. + * - On failure, a negative value. + */ +int rte_eth_vhost_get_vid_from_port_id(uint16_t port_id); + #ifdef __cplusplus } #endif diff --git a/dpdk/drivers/net/vhost/rte_pmd_vhost_version.map b/dpdk/drivers/net/vhost/rte_pmd_vhost_version.map index 65bf3a8c..695db857 100644 --- a/dpdk/drivers/net/vhost/rte_pmd_vhost_version.map +++ b/dpdk/drivers/net/vhost/rte_pmd_vhost_version.map @@ -1,10 +1,13 @@ DPDK_16.04 { global: - rte_eth_vhost_feature_disable; - rte_eth_vhost_feature_enable; - rte_eth_vhost_feature_get; rte_eth_vhost_get_queue_event; local: *; }; + +DPDK_16.11 { + global: + + rte_eth_vhost_get_vid_from_port_id; +}; diff --git a/dpdk/drivers/net/virtio/Makefile b/dpdk/drivers/net/virtio/Makefile index 3020b688..f2b5d1c3 100644 --- a/dpdk/drivers/net/virtio/Makefile +++ b/dpdk/drivers/net/virtio/Makefile @@ -38,6 +38,12 @@ LIB = librte_pmd_virtio.a CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs +LDLIBS += -lrte_bus_pci +ifeq ($(CONFIG_RTE_VIRTIO_USER),y) +LDLIBS += -lrte_bus_vdev +endif EXPORT_MAP := rte_pmd_virtio_version.map @@ -50,21 +56,20 @@ SRCS-$(CONFIG_RTE_LIBRTE_VIRTIO_PMD) += virtqueue.c SRCS-$(CONFIG_RTE_LIBRTE_VIRTIO_PMD) += virtio_pci.c SRCS-$(CONFIG_RTE_LIBRTE_VIRTIO_PMD) += virtio_rxtx.c SRCS-$(CONFIG_RTE_LIBRTE_VIRTIO_PMD) += virtio_ethdev.c - -ifeq ($(findstring RTE_MACHINE_CPUFLAG_SSSE3,$(CFLAGS)),RTE_MACHINE_CPUFLAG_SSSE3) SRCS-$(CONFIG_RTE_LIBRTE_VIRTIO_PMD) += virtio_rxtx_simple.c + +ifeq ($(CONFIG_RTE_ARCH_X86),y) +SRCS-$(CONFIG_RTE_LIBRTE_VIRTIO_PMD) += virtio_rxtx_simple_sse.c +else ifneq ($(filter y,$(CONFIG_RTE_ARCH_ARM) $(CONFIG_RTE_ARCH_ARM64)),) +SRCS-$(CONFIG_RTE_LIBRTE_VIRTIO_PMD) += virtio_rxtx_simple_neon.c endif ifeq ($(CONFIG_RTE_VIRTIO_USER),y) SRCS-$(CONFIG_RTE_LIBRTE_VIRTIO_PMD) += virtio_user/vhost_user.c +SRCS-$(CONFIG_RTE_LIBRTE_VIRTIO_PMD) += virtio_user/vhost_kernel.c +SRCS-$(CONFIG_RTE_LIBRTE_VIRTIO_PMD) += virtio_user/vhost_kernel_tap.c SRCS-$(CONFIG_RTE_LIBRTE_VIRTIO_PMD) += virtio_user/virtio_user_dev.c SRCS-$(CONFIG_RTE_LIBRTE_VIRTIO_PMD) += virtio_user_ethdev.c endif -# this lib depends upon: -DEPDIRS-$(CONFIG_RTE_LIBRTE_VIRTIO_PMD) += lib/librte_eal lib/librte_ether -DEPDIRS-$(CONFIG_RTE_LIBRTE_VIRTIO_PMD) += lib/librte_mempool lib/librte_mbuf -DEPDIRS-$(CONFIG_RTE_LIBRTE_VIRTIO_PMD) += lib/librte_net -DEPDIRS-$(CONFIG_RTE_LIBRTE_VIRTIO_PMD) += lib/librte_kvargs - include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/net/virtio/virtio_ethdev.c b/dpdk/drivers/net/virtio/virtio_ethdev.c index 86cf8a38..4da1ba32 100644 --- a/dpdk/drivers/net/virtio/virtio_ethdev.c +++ b/dpdk/drivers/net/virtio/virtio_ethdev.c @@ -38,6 +38,7 @@ #include #include +#include #include #include #include @@ -45,9 +46,11 @@ #include #include #include +#include #include #include #include +#include #include #include @@ -70,12 +73,13 @@ static void virtio_dev_allmulticast_disable(struct rte_eth_dev *dev); static void virtio_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info); static int virtio_dev_link_update(struct rte_eth_dev *dev, - __rte_unused int wait_to_complete); + int wait_to_complete); +static int virtio_dev_vlan_offload_set(struct rte_eth_dev *dev, int mask); static void virtio_set_hwaddr(struct virtio_hw *hw); static void virtio_get_hwaddr(struct virtio_hw *hw); -static void virtio_dev_stats_get(struct rte_eth_dev *dev, +static int virtio_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats); static int virtio_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, unsigned n); @@ -86,24 +90,28 @@ static void virtio_dev_stats_reset(struct rte_eth_dev *dev); static void virtio_dev_free_mbufs(struct rte_eth_dev *dev); static int virtio_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on); -static void virtio_mac_addr_add(struct rte_eth_dev *dev, +static int virtio_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac_addr, - uint32_t index, uint32_t vmdq __rte_unused); + uint32_t index, uint32_t vmdq); static void virtio_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index); static void virtio_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr); +static int virtio_intr_enable(struct rte_eth_dev *dev); +static int virtio_intr_disable(struct rte_eth_dev *dev); + static int virtio_dev_queue_stats_mapping_set( - __rte_unused struct rte_eth_dev *eth_dev, - __rte_unused uint16_t queue_id, - __rte_unused uint8_t stat_idx, - __rte_unused uint8_t is_rx); + struct rte_eth_dev *eth_dev, + uint16_t queue_id, + uint8_t stat_idx, + uint8_t is_rx); /* * The set of PCI devices this driver supports */ static const struct rte_pci_id pci_id_virtio_map[] = { - { RTE_PCI_DEVICE(VIRTIO_PCI_VENDORID, VIRTIO_PCI_DEVICEID_MIN) }, + { RTE_PCI_DEVICE(VIRTIO_PCI_VENDORID, VIRTIO_PCI_LEGACY_DEVICEID_NET) }, + { RTE_PCI_DEVICE(VIRTIO_PCI_VENDORID, VIRTIO_PCI_MODERN_DEVICEID_NET) }, { .vendor_id = 0, /* sentinel */ }, }; @@ -151,6 +159,8 @@ static const struct rte_virtio_xstats_name_off rte_virtio_txq_stat_strings[] = { #define VIRTIO_NB_TXQ_XSTATS (sizeof(rte_virtio_txq_stat_strings) / \ sizeof(rte_virtio_txq_stat_strings[0])) +struct virtio_hw_internal virtio_hw_internal[RTE_MAX_ETHPORTS]; + static int virtio_send_command(struct virtnet_ctl *cvq, struct virtio_pmd_ctrl *ctrl, int *dlen, int pkt_num) @@ -158,7 +168,7 @@ virtio_send_command(struct virtnet_ctl *cvq, struct virtio_pmd_ctrl *ctrl, uint32_t head, i; int k, sum = 0; virtio_net_ctrl_ack status = ~0; - struct virtio_pmd_ctrl result; + struct virtio_pmd_ctrl *result; struct virtqueue *vq; ctrl->status = status; @@ -249,10 +259,9 @@ virtio_send_command(struct virtnet_ctl *cvq, struct virtio_pmd_ctrl *ctrl, PMD_INIT_LOG(DEBUG, "vq->vq_free_cnt=%d\nvq->vq_desc_head_idx=%d", vq->vq_free_cnt, vq->vq_desc_head_idx); - memcpy(&result, cvq->virtio_net_hdr_mz->addr, - sizeof(struct virtio_pmd_ctrl)); + result = cvq->virtio_net_hdr_mz->addr; - return result.status; + return result->status; } static int @@ -279,28 +288,54 @@ virtio_set_multiple_queues(struct rte_eth_dev *dev, uint16_t nb_queues) return 0; } -void -virtio_dev_queue_release(struct virtqueue *vq) +static void +virtio_dev_queue_release(void *queue __rte_unused) { - struct virtio_hw *hw; - - if (vq) { - hw = vq->hw; - if (vq->configured) - hw->vtpci_ops->del_queue(hw, vq); - - rte_free(vq->sw_ring); - rte_free(vq); - } + /* do nothing */ } -int virtio_dev_queue_setup(struct rte_eth_dev *dev, - int queue_type, - uint16_t queue_idx, - uint16_t vtpci_queue_idx, - uint16_t nb_desc, - unsigned int socket_id, - void **pvq) +static uint16_t +virtio_get_nr_vq(struct virtio_hw *hw) +{ + uint16_t nr_vq = hw->max_queue_pairs * 2; + + if (vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_VQ)) + nr_vq += 1; + + return nr_vq; +} + +static void +virtio_init_vring(struct virtqueue *vq) +{ + int size = vq->vq_nentries; + struct vring *vr = &vq->vq_ring; + uint8_t *ring_mem = vq->vq_ring_virt_mem; + + PMD_INIT_FUNC_TRACE(); + + /* + * Reinitialise since virtio port might have been stopped and restarted + */ + memset(ring_mem, 0, vq->vq_ring_size); + vring_init(vr, size, ring_mem, VIRTIO_PCI_VRING_ALIGN); + vq->vq_used_cons_idx = 0; + vq->vq_desc_head_idx = 0; + vq->vq_avail_idx = 0; + vq->vq_desc_tail_idx = (uint16_t)(vq->vq_nentries - 1); + vq->vq_free_cnt = vq->vq_nentries; + memset(vq->vq_descx, 0, sizeof(struct vq_desc_extra) * vq->vq_nentries); + + vring_desc_init(vr->desc, size); + + /* + * Disable device(host) interrupting guest + */ + virtqueue_disable_intr(vq); +} + +static int +virtio_init_queue(struct rte_eth_dev *dev, uint16_t vtpci_queue_idx) { char vq_name[VIRTQUEUE_MAX_NAME_SZ]; char vq_hdr_name[VIRTQUEUE_MAX_NAME_SZ]; @@ -311,9 +346,9 @@ int virtio_dev_queue_setup(struct rte_eth_dev *dev, struct virtnet_tx *txvq = NULL; struct virtnet_ctl *cvq = NULL; struct virtqueue *vq; - const char *queue_names[] = {"rvq", "txq", "cvq"}; - size_t sz_vq, sz_q = 0, sz_hdr_mz = 0; + size_t sz_hdr_mz = 0; void *sw_ring = NULL; + int queue_type = virtio_get_queue_type(hw, vtpci_queue_idx); int ret; PMD_INIT_LOG(DEBUG, "setting up queue: %u", vtpci_queue_idx); @@ -322,8 +357,8 @@ int virtio_dev_queue_setup(struct rte_eth_dev *dev, * Read the virtqueue size from the Queue Size field * Always power of 2 and if 0 virtqueue does not exist */ - vq_size = hw->vtpci_ops->get_queue_num(hw, vtpci_queue_idx); - PMD_INIT_LOG(DEBUG, "vq_size: %u nb_desc:%u", vq_size, nb_desc); + vq_size = VTPCI_OPS(hw)->get_queue_num(hw, vtpci_queue_idx); + PMD_INIT_LOG(DEBUG, "vq_size: %u", vq_size); if (vq_size == 0) { PMD_INIT_LOG(ERR, "virtqueue does not exist"); return -EINVAL; @@ -334,40 +369,35 @@ int virtio_dev_queue_setup(struct rte_eth_dev *dev, return -EINVAL; } - snprintf(vq_name, sizeof(vq_name), "port%d_%s%d", - dev->data->port_id, queue_names[queue_type], queue_idx); + snprintf(vq_name, sizeof(vq_name), "port%d_vq%d", + dev->data->port_id, vtpci_queue_idx); - sz_vq = RTE_ALIGN_CEIL(sizeof(*vq) + + size = RTE_ALIGN_CEIL(sizeof(*vq) + vq_size * sizeof(struct vq_desc_extra), RTE_CACHE_LINE_SIZE); - if (queue_type == VTNET_RQ) { - sz_q = sz_vq + sizeof(*rxvq); - } else if (queue_type == VTNET_TQ) { - sz_q = sz_vq + sizeof(*txvq); + if (queue_type == VTNET_TQ) { /* * For each xmit packet, allocate a virtio_net_hdr * and indirect ring elements */ sz_hdr_mz = vq_size * sizeof(struct virtio_tx_region); } else if (queue_type == VTNET_CQ) { - sz_q = sz_vq + sizeof(*cvq); /* Allocate a page for control vq command, data and status */ sz_hdr_mz = PAGE_SIZE; } - vq = rte_zmalloc_socket(vq_name, sz_q, RTE_CACHE_LINE_SIZE, socket_id); + vq = rte_zmalloc_socket(vq_name, size, RTE_CACHE_LINE_SIZE, + SOCKET_ID_ANY); if (vq == NULL) { PMD_INIT_LOG(ERR, "can not allocate vq"); return -ENOMEM; } + hw->vqs[vtpci_queue_idx] = vq; + vq->hw = hw; vq->vq_queue_index = vtpci_queue_idx; vq->vq_nentries = vq_size; - if (nb_desc == 0 || nb_desc > vq_size) - nb_desc = vq_size; - vq->vq_free_cnt = nb_desc; - /* * Reserve a memzone for vring elements */ @@ -376,7 +406,8 @@ int virtio_dev_queue_setup(struct rte_eth_dev *dev, PMD_INIT_LOG(DEBUG, "vring_size: %d, rounded_vring_size: %d", size, vq->vq_ring_size); - mz = rte_memzone_reserve_aligned(vq_name, vq->vq_ring_size, socket_id, + mz = rte_memzone_reserve_aligned(vq_name, vq->vq_ring_size, + SOCKET_ID_ANY, 0, VIRTIO_PCI_VRING_ALIGN); if (mz == NULL) { if (rte_errno == EEXIST) @@ -387,21 +418,22 @@ int virtio_dev_queue_setup(struct rte_eth_dev *dev, } } - memset(mz->addr, 0, sizeof(mz->len)); + memset(mz->addr, 0, mz->len); - vq->vq_ring_mem = mz->phys_addr; + vq->vq_ring_mem = mz->iova; vq->vq_ring_virt_mem = mz->addr; PMD_INIT_LOG(DEBUG, "vq->vq_ring_mem: 0x%" PRIx64, - (uint64_t)mz->phys_addr); + (uint64_t)mz->iova); PMD_INIT_LOG(DEBUG, "vq->vq_ring_virt_mem: 0x%" PRIx64, (uint64_t)(uintptr_t)mz->addr); + virtio_init_vring(vq); + if (sz_hdr_mz) { - snprintf(vq_hdr_name, sizeof(vq_hdr_name), "port%d_%s%d_hdr", - dev->data->port_id, queue_names[queue_type], - queue_idx); + snprintf(vq_hdr_name, sizeof(vq_hdr_name), "port%d_vq%d_hdr", + dev->data->port_id, vtpci_queue_idx); hdr_mz = rte_memzone_reserve_aligned(vq_hdr_name, sz_hdr_mz, - socket_id, 0, + SOCKET_ID_ANY, 0, RTE_CACHE_LINE_SIZE); if (hdr_mz == NULL) { if (rte_errno == EEXIST) @@ -418,7 +450,7 @@ int virtio_dev_queue_setup(struct rte_eth_dev *dev, sizeof(vq->sw_ring[0]); sw_ring = rte_zmalloc_socket("sw_ring", sz_sw, - RTE_CACHE_LINE_SIZE, socket_id); + RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); if (!sw_ring) { PMD_INIT_LOG(ERR, "can not allocate RX soft ring"); ret = -ENOMEM; @@ -426,38 +458,34 @@ int virtio_dev_queue_setup(struct rte_eth_dev *dev, } vq->sw_ring = sw_ring; - rxvq = (struct virtnet_rx *)RTE_PTR_ADD(vq, sz_vq); + rxvq = &vq->rxq; rxvq->vq = vq; rxvq->port_id = dev->data->port_id; - rxvq->queue_id = queue_idx; rxvq->mz = mz; - *pvq = rxvq; } else if (queue_type == VTNET_TQ) { - txvq = (struct virtnet_tx *)RTE_PTR_ADD(vq, sz_vq); + txvq = &vq->txq; txvq->vq = vq; txvq->port_id = dev->data->port_id; - txvq->queue_id = queue_idx; txvq->mz = mz; txvq->virtio_net_hdr_mz = hdr_mz; - txvq->virtio_net_hdr_mem = hdr_mz->phys_addr; - - *pvq = txvq; + txvq->virtio_net_hdr_mem = hdr_mz->iova; } else if (queue_type == VTNET_CQ) { - cvq = (struct virtnet_ctl *)RTE_PTR_ADD(vq, sz_vq); + cvq = &vq->cq; cvq->vq = vq; cvq->mz = mz; cvq->virtio_net_hdr_mz = hdr_mz; - cvq->virtio_net_hdr_mem = hdr_mz->phys_addr; + cvq->virtio_net_hdr_mem = hdr_mz->iova; memset(cvq->virtio_net_hdr_mz->addr, 0, PAGE_SIZE); - *pvq = cvq; + + hw->cvq = cvq; } - /* For virtio_user case (that is when dev->pci_dev is NULL), we use + /* For virtio_user case (that is when hw->dev is NULL), we use * virtual address. And we need properly set _offset_, please see * VIRTIO_MBUF_DATA_DMA_ADDR in virtqueue.h for more information. */ - if (dev->pci_dev) - vq->offset = offsetof(struct rte_mbuf, buf_physaddr); + if (!hw->virtio_user_dev) + vq->offset = offsetof(struct rte_mbuf, buf_iova); else { vq->vq_ring_mem = (uintptr_t)mz->addr; vq->offset = offsetof(struct rte_mbuf, buf_addr); @@ -488,13 +516,11 @@ int virtio_dev_queue_setup(struct rte_eth_dev *dev, } } - if (hw->vtpci_ops->setup_queue(hw, vq) < 0) { + if (VTPCI_OPS(hw)->setup_queue(hw, vq) < 0) { PMD_INIT_LOG(ERR, "setup_queue failed"); - virtio_dev_queue_release(vq); return -EINVAL; } - vq->configured = 1; return 0; fail_q_alloc: @@ -506,56 +532,93 @@ fail_q_alloc: return ret; } -static int -virtio_dev_cq_queue_setup(struct rte_eth_dev *dev, uint16_t vtpci_queue_idx, - uint32_t socket_id) +static void +virtio_free_queues(struct virtio_hw *hw) { - struct virtnet_ctl *cvq; - int ret; - struct virtio_hw *hw = dev->data->dev_private; + uint16_t nr_vq = virtio_get_nr_vq(hw); + struct virtqueue *vq; + int queue_type; + uint16_t i; - PMD_INIT_FUNC_TRACE(); - ret = virtio_dev_queue_setup(dev, VTNET_CQ, VTNET_SQ_CQ_QUEUE_IDX, - vtpci_queue_idx, 0, socket_id, (void **)&cvq); - if (ret < 0) { - PMD_INIT_LOG(ERR, "control vq initialization failed"); - return ret; + if (hw->vqs == NULL) + return; + + for (i = 0; i < nr_vq; i++) { + vq = hw->vqs[i]; + if (!vq) + continue; + + queue_type = virtio_get_queue_type(hw, i); + if (queue_type == VTNET_RQ) { + rte_free(vq->sw_ring); + rte_memzone_free(vq->rxq.mz); + } else if (queue_type == VTNET_TQ) { + rte_memzone_free(vq->txq.mz); + rte_memzone_free(vq->txq.virtio_net_hdr_mz); + } else { + rte_memzone_free(vq->cq.mz); + rte_memzone_free(vq->cq.virtio_net_hdr_mz); + } + + rte_free(vq); + hw->vqs[i] = NULL; + } + + rte_free(hw->vqs); + hw->vqs = NULL; +} + +static int +virtio_alloc_queues(struct rte_eth_dev *dev) +{ + struct virtio_hw *hw = dev->data->dev_private; + uint16_t nr_vq = virtio_get_nr_vq(hw); + uint16_t i; + int ret; + + hw->vqs = rte_zmalloc(NULL, sizeof(struct virtqueue *) * nr_vq, 0); + if (!hw->vqs) { + PMD_INIT_LOG(ERR, "failed to allocate vqs"); + return -ENOMEM; + } + + for (i = 0; i < nr_vq; i++) { + ret = virtio_init_queue(dev, i); + if (ret < 0) { + virtio_free_queues(hw); + return ret; + } } - hw->cvq = cvq; return 0; } -static void -virtio_free_queues(struct rte_eth_dev *dev) -{ - unsigned int i; - - for (i = 0; i < dev->data->nb_rx_queues; i++) - virtio_dev_rx_queue_release(dev->data->rx_queues[i]); - - dev->data->nb_rx_queues = 0; - - for (i = 0; i < dev->data->nb_tx_queues; i++) - virtio_dev_tx_queue_release(dev->data->tx_queues[i]); - - dev->data->nb_tx_queues = 0; -} +static void virtio_queues_unbind_intr(struct rte_eth_dev *dev); static void virtio_dev_close(struct rte_eth_dev *dev) { struct virtio_hw *hw = dev->data->dev_private; + struct rte_intr_conf *intr_conf = &dev->data->dev_conf.intr_conf; PMD_INIT_LOG(DEBUG, "virtio_dev_close"); /* reset the NIC */ if (dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) - vtpci_irq_config(hw, VIRTIO_MSI_NO_VECTOR); + VTPCI_OPS(hw)->set_config_irq(hw, VIRTIO_MSI_NO_VECTOR); + if (intr_conf->rxq) + virtio_queues_unbind_intr(dev); + + if (intr_conf->lsc || intr_conf->rxq) { + virtio_intr_disable(dev); + rte_intr_efd_disable(dev->intr_handle); + rte_free(dev->intr_handle->intr_vec); + dev->intr_handle->intr_vec = NULL; + } + vtpci_reset(hw); - hw->started = 0; virtio_dev_free_mbufs(dev); - virtio_free_queues(dev); + virtio_free_queues(hw); } static void @@ -567,7 +630,7 @@ virtio_dev_promiscuous_enable(struct rte_eth_dev *dev) int ret; if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) { - PMD_INIT_LOG(INFO, "host does not support rx control\n"); + PMD_INIT_LOG(INFO, "host does not support rx control"); return; } @@ -590,7 +653,7 @@ virtio_dev_promiscuous_disable(struct rte_eth_dev *dev) int ret; if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) { - PMD_INIT_LOG(INFO, "host does not support rx control\n"); + PMD_INIT_LOG(INFO, "host does not support rx control"); return; } @@ -613,7 +676,7 @@ virtio_dev_allmulticast_enable(struct rte_eth_dev *dev) int ret; if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) { - PMD_INIT_LOG(INFO, "host does not support rx control\n"); + PMD_INIT_LOG(INFO, "host does not support rx control"); return; } @@ -636,7 +699,7 @@ virtio_dev_allmulticast_disable(struct rte_eth_dev *dev) int ret; if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) { - PMD_INIT_LOG(INFO, "host does not support rx control\n"); + PMD_INIT_LOG(INFO, "host does not support rx control"); return; } @@ -650,6 +713,46 @@ virtio_dev_allmulticast_disable(struct rte_eth_dev *dev) PMD_INIT_LOG(ERR, "Failed to disable allmulticast"); } +#define VLAN_TAG_LEN 4 /* 802.3ac tag (not DMA'd) */ +static int +virtio_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) +{ + struct virtio_hw *hw = dev->data->dev_private; + uint32_t ether_hdr_len = ETHER_HDR_LEN + VLAN_TAG_LEN + + hw->vtnet_hdr_size; + uint32_t frame_size = mtu + ether_hdr_len; + uint32_t max_frame_size = hw->max_mtu + ether_hdr_len; + + max_frame_size = RTE_MIN(max_frame_size, VIRTIO_MAX_RX_PKTLEN); + + if (mtu < ETHER_MIN_MTU || frame_size > max_frame_size) { + PMD_INIT_LOG(ERR, "MTU should be between %d and %d", + ETHER_MIN_MTU, max_frame_size - ether_hdr_len); + return -EINVAL; + } + return 0; +} + +static int +virtio_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) +{ + struct virtnet_rx *rxvq = dev->data->rx_queues[queue_id]; + struct virtqueue *vq = rxvq->vq; + + virtqueue_enable_intr(vq); + return 0; +} + +static int +virtio_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id) +{ + struct virtnet_rx *rxvq = dev->data->rx_queues[queue_id]; + struct virtqueue *vq = rxvq->vq; + + virtqueue_disable_intr(vq); + return 0; +} + /* * dev_ops for virtio, bare necessities for basic operation */ @@ -662,7 +765,7 @@ static const struct eth_dev_ops virtio_eth_dev_ops = { .promiscuous_disable = virtio_dev_promiscuous_disable, .allmulticast_enable = virtio_dev_allmulticast_enable, .allmulticast_disable = virtio_dev_allmulticast_disable, - + .mtu_set = virtio_mtu_set, .dev_infos_get = virtio_dev_info_get, .stats_get = virtio_dev_stats_get, .xstats_get = virtio_dev_xstats_get, @@ -670,10 +773,14 @@ static const struct eth_dev_ops virtio_eth_dev_ops = { .stats_reset = virtio_dev_stats_reset, .xstats_reset = virtio_dev_stats_reset, .link_update = virtio_dev_link_update, + .vlan_offload_set = virtio_dev_vlan_offload_set, .rx_queue_setup = virtio_dev_rx_queue_setup, - .rx_queue_release = virtio_dev_rx_queue_release, + .rx_queue_intr_enable = virtio_dev_rx_queue_intr_enable, + .rx_queue_intr_disable = virtio_dev_rx_queue_intr_disable, + .rx_queue_release = virtio_dev_queue_release, + .rx_descriptor_done = virtio_dev_rx_queue_done, .tx_queue_setup = virtio_dev_tx_queue_setup, - .tx_queue_release = virtio_dev_tx_queue_release, + .tx_queue_release = virtio_dev_queue_release, /* collect stats per queue */ .queue_stats_mapping_set = virtio_dev_queue_stats_mapping_set, .vlan_filter_set = virtio_vlan_filter_set, @@ -775,7 +882,7 @@ static int virtio_dev_xstats_get_names(struct rte_eth_dev *dev, /* Note: limit checked in rte_eth_xstats_names() */ for (i = 0; i < dev->data->nb_rx_queues; i++) { - struct virtqueue *rxvq = dev->data->rx_queues[i]; + struct virtnet_rx *rxvq = dev->data->rx_queues[i]; if (rxvq == NULL) continue; for (t = 0; t < VIRTIO_NB_RXQ_XSTATS; t++) { @@ -788,7 +895,7 @@ static int virtio_dev_xstats_get_names(struct rte_eth_dev *dev, } for (i = 0; i < dev->data->nb_tx_queues; i++) { - struct virtqueue *txvq = dev->data->tx_queues[i]; + struct virtnet_tx *txvq = dev->data->tx_queues[i]; if (txvq == NULL) continue; for (t = 0; t < VIRTIO_NB_TXQ_XSTATS; t++) { @@ -828,6 +935,7 @@ virtio_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, for (t = 0; t < VIRTIO_NB_RXQ_XSTATS; t++) { xstats[count].value = *(uint64_t *)(((char *)rxvq) + rte_virtio_rxq_stat_strings[t].offset); + xstats[count].id = count; count++; } } @@ -843,6 +951,7 @@ virtio_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, for (t = 0; t < VIRTIO_NB_TXQ_XSTATS; t++) { xstats[count].value = *(uint64_t *)(((char *)txvq) + rte_virtio_txq_stat_strings[t].offset); + xstats[count].id = count; count++; } } @@ -850,10 +959,12 @@ virtio_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, return count; } -static void +static int virtio_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) { virtio_update_stats(dev, stats); + + return 0; } static void @@ -911,7 +1022,7 @@ virtio_get_hwaddr(struct virtio_hw *hw) } } -static void +static int virtio_mac_table_set(struct virtio_hw *hw, const struct virtio_net_ctrl_mac *uc, const struct virtio_net_ctrl_mac *mc) @@ -921,7 +1032,7 @@ virtio_mac_table_set(struct virtio_hw *hw, if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_MAC_ADDR)) { PMD_DRV_LOG(INFO, "host does not support mac table"); - return; + return -1; } ctrl.hdr.class = VIRTIO_NET_CTRL_MAC; @@ -936,9 +1047,10 @@ virtio_mac_table_set(struct virtio_hw *hw, err = virtio_send_command(hw->cvq, &ctrl, len, 2); if (err != 0) PMD_DRV_LOG(NOTICE, "mac table set failed: %d", err); + return err; } -static void +static int virtio_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac_addr, uint32_t index, uint32_t vmdq __rte_unused) { @@ -949,7 +1061,7 @@ virtio_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac_addr, if (index >= VIRTIO_MAX_MAC_ADDRS) { PMD_DRV_LOG(ERR, "mac address index %u out of range", index); - return; + return -EINVAL; } uc = alloca(VIRTIO_MAX_MAC_ADDRS * ETHER_ADDR_LEN + sizeof(uc->entries)); @@ -966,7 +1078,7 @@ virtio_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac_addr, memcpy(&tbl->macs[tbl->entries++], addr, ETHER_ADDR_LEN); } - virtio_mac_table_set(hw, uc, mc); + return virtio_mac_table_set(hw, uc, mc); } static void @@ -1040,24 +1152,64 @@ virtio_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) } static int -virtio_negotiate_features(struct virtio_hw *hw) +virtio_intr_enable(struct rte_eth_dev *dev) +{ + struct virtio_hw *hw = dev->data->dev_private; + + if (rte_intr_enable(dev->intr_handle) < 0) + return -1; + + if (!hw->virtio_user_dev) + hw->use_msix = vtpci_msix_detect(RTE_ETH_DEV_TO_PCI(dev)); + + return 0; +} + +static int +virtio_intr_disable(struct rte_eth_dev *dev) +{ + struct virtio_hw *hw = dev->data->dev_private; + + if (rte_intr_disable(dev->intr_handle) < 0) + return -1; + + if (!hw->virtio_user_dev) + hw->use_msix = vtpci_msix_detect(RTE_ETH_DEV_TO_PCI(dev)); + + return 0; +} + +static int +virtio_negotiate_features(struct virtio_hw *hw, uint64_t req_features) { uint64_t host_features; /* Prepare guest_features: feature that driver wants to support */ - hw->guest_features = VIRTIO_PMD_GUEST_FEATURES; PMD_INIT_LOG(DEBUG, "guest_features before negotiate = %" PRIx64, - hw->guest_features); + req_features); /* Read device(host) feature bits */ - host_features = hw->vtpci_ops->get_features(hw); + host_features = VTPCI_OPS(hw)->get_features(hw); PMD_INIT_LOG(DEBUG, "host_features before negotiate = %" PRIx64, host_features); + /* If supported, ensure MTU value is valid before acknowledging it. */ + if (host_features & req_features & (1ULL << VIRTIO_NET_F_MTU)) { + struct virtio_net_config config; + + vtpci_read_dev_config(hw, + offsetof(struct virtio_net_config, mtu), + &config.mtu, sizeof(config.mtu)); + + if (config.mtu < ETHER_MIN_MTU) + req_features &= ~(1ULL << VIRTIO_NET_F_MTU); + } + /* * Negotiate features: Subset of device feature bits are written back * guest feature bits. */ + hw->guest_features = req_features; hw->guest_features = vtpci_negotiate_features(hw, host_features); PMD_INIT_LOG(DEBUG, "features after negotiate = %" PRIx64, hw->guest_features); @@ -1076,6 +1228,8 @@ virtio_negotiate_features(struct virtio_hw *hw) } } + hw->req_guest_features = req_features; + return 0; } @@ -1083,9 +1237,8 @@ virtio_negotiate_features(struct virtio_hw *hw) * Process Virtio Config changed interrupt and call the callback * if link state changed. */ -static void -virtio_interrupt_handler(__rte_unused struct rte_intr_handle *handle, - void *param) +void +virtio_interrupt_handler(void *param) { struct rte_eth_dev *dev = param; struct virtio_hw *hw = dev->data->dev_private; @@ -1095,87 +1248,176 @@ virtio_interrupt_handler(__rte_unused struct rte_intr_handle *handle, isr = vtpci_isr(hw); PMD_DRV_LOG(INFO, "interrupt status = %#x", isr); - if (rte_intr_enable(&dev->pci_dev->intr_handle) < 0) + if (virtio_intr_enable(dev) < 0) PMD_DRV_LOG(ERR, "interrupt enable failed"); if (isr & VIRTIO_PCI_ISR_CONFIG) { if (virtio_dev_link_update(dev, 0) == 0) _rte_eth_dev_callback_process(dev, - RTE_ETH_EVENT_INTR_LSC); + RTE_ETH_EVENT_INTR_LSC, + NULL, NULL); } } +/* set rx and tx handlers according to what is supported */ static void -rx_func_get(struct rte_eth_dev *eth_dev) +set_rxtx_funcs(struct rte_eth_dev *eth_dev) { struct virtio_hw *hw = eth_dev->data->dev_private; - if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) + + if (hw->use_simple_rx) { + PMD_INIT_LOG(INFO, "virtio: using simple Rx path on port %u", + eth_dev->data->port_id); + eth_dev->rx_pkt_burst = virtio_recv_pkts_vec; + } else if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) { + PMD_INIT_LOG(INFO, + "virtio: using mergeable buffer Rx path on port %u", + eth_dev->data->port_id); eth_dev->rx_pkt_burst = &virtio_recv_mergeable_pkts; - else + } else { + PMD_INIT_LOG(INFO, "virtio: using standard Rx path on port %u", + eth_dev->data->port_id); eth_dev->rx_pkt_burst = &virtio_recv_pkts; + } + + if (hw->use_simple_tx) { + PMD_INIT_LOG(INFO, "virtio: using simple Tx path on port %u", + eth_dev->data->port_id); + eth_dev->tx_pkt_burst = virtio_xmit_pkts_simple; + } else { + PMD_INIT_LOG(INFO, "virtio: using standard Tx path on port %u", + eth_dev->data->port_id); + eth_dev->tx_pkt_burst = virtio_xmit_pkts; + } } -/* - * This function is based on probe() function in virtio_pci.c - * It returns 0 on success. +/* Only support 1:1 queue/interrupt mapping so far. + * TODO: support n:1 queue/interrupt mapping when there are limited number of + * interrupt vectors (data->dev_private; + + PMD_INIT_LOG(INFO, "queue/interrupt binding"); + for (i = 0; i < dev->data->nb_rx_queues; ++i) { + dev->intr_handle->intr_vec[i] = i + 1; + if (VTPCI_OPS(hw)->set_queue_irq(hw, hw->vqs[i * 2], i + 1) == + VIRTIO_MSI_NO_VECTOR) { + PMD_DRV_LOG(ERR, "failed to set queue vector"); + return -EBUSY; + } + } + + return 0; +} + +static void +virtio_queues_unbind_intr(struct rte_eth_dev *dev) +{ + uint32_t i; + struct virtio_hw *hw = dev->data->dev_private; + + PMD_INIT_LOG(INFO, "queue/interrupt unbinding"); + for (i = 0; i < dev->data->nb_rx_queues; ++i) + VTPCI_OPS(hw)->set_queue_irq(hw, + hw->vqs[i * VTNET_CQ], + VIRTIO_MSI_NO_VECTOR); +} + +static int +virtio_configure_intr(struct rte_eth_dev *dev) +{ + struct virtio_hw *hw = dev->data->dev_private; + + if (!rte_intr_cap_multiple(dev->intr_handle)) { + PMD_INIT_LOG(ERR, "Multiple intr vector not supported"); + return -ENOTSUP; + } + + if (rte_intr_efd_enable(dev->intr_handle, dev->data->nb_rx_queues)) { + PMD_INIT_LOG(ERR, "Fail to create eventfd"); + return -1; + } + + if (!dev->intr_handle->intr_vec) { + dev->intr_handle->intr_vec = + rte_zmalloc("intr_vec", + hw->max_queue_pairs * sizeof(int), 0); + if (!dev->intr_handle->intr_vec) { + PMD_INIT_LOG(ERR, "Failed to allocate %u rxq vectors", + hw->max_queue_pairs); + return -ENOMEM; + } + } + + /* Re-register callback to update max_intr */ + rte_intr_callback_unregister(dev->intr_handle, + virtio_interrupt_handler, + dev); + rte_intr_callback_register(dev->intr_handle, + virtio_interrupt_handler, + dev); + + /* DO NOT try to remove this! This function will enable msix, or QEMU + * will encounter SIGSEGV when DRIVER_OK is sent. + * And for legacy devices, this should be done before queue/vec binding + * to change the config size from 20 to 24, or VIRTIO_MSI_QUEUE_VECTOR + * (22) will be ignored. + */ + if (virtio_intr_enable(dev) < 0) { + PMD_DRV_LOG(ERR, "interrupt enable failed"); + return -1; + } + + if (virtio_queues_bind_intr(dev) < 0) { + PMD_INIT_LOG(ERR, "Failed to bind queue/interrupt"); + return -1; + } + + return 0; +} + +/* reset device and renegotiate features if needed */ +static int +virtio_init_device(struct rte_eth_dev *eth_dev, uint64_t req_features) { struct virtio_hw *hw = eth_dev->data->dev_private; struct virtio_net_config *config; struct virtio_net_config local_config; - struct rte_pci_device *pci_dev; - uint32_t dev_flags = RTE_ETH_DEV_DETACHABLE; + struct rte_pci_device *pci_dev = NULL; int ret; - RTE_BUILD_BUG_ON(RTE_PKTMBUF_HEADROOM < sizeof(struct virtio_net_hdr_mrg_rxbuf)); - - eth_dev->dev_ops = &virtio_eth_dev_ops; - eth_dev->tx_pkt_burst = &virtio_xmit_pkts; - - if (rte_eal_process_type() == RTE_PROC_SECONDARY) { - rx_func_get(eth_dev); - return 0; - } - - /* Allocate memory for storing MAC addresses */ - eth_dev->data->mac_addrs = rte_zmalloc("virtio", VIRTIO_MAX_MAC_ADDRS * ETHER_ADDR_LEN, 0); - if (eth_dev->data->mac_addrs == NULL) { - PMD_INIT_LOG(ERR, - "Failed to allocate %d bytes needed to store MAC addresses", - VIRTIO_MAX_MAC_ADDRS * ETHER_ADDR_LEN); - return -ENOMEM; - } - - pci_dev = eth_dev->pci_dev; - - if (pci_dev) { - ret = vtpci_init(pci_dev, hw, &dev_flags); - if (ret) - return ret; - } - /* Reset the device although not necessary at startup */ vtpci_reset(hw); + if (hw->vqs) { + virtio_dev_free_mbufs(eth_dev); + virtio_free_queues(hw); + } + /* Tell the host we've noticed this device. */ vtpci_set_status(hw, VIRTIO_CONFIG_STATUS_ACK); /* Tell the host we've known how to drive the device. */ vtpci_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER); - if (virtio_negotiate_features(hw) < 0) + if (virtio_negotiate_features(hw, req_features) < 0) return -1; - /* If host does not support status then disable LSC */ - if (!vtpci_with_feature(hw, VIRTIO_NET_F_STATUS)) - dev_flags &= ~RTE_ETH_DEV_INTR_LSC; + if (!hw->virtio_user_dev) { + pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); + rte_eth_copy_pci_info(eth_dev, pci_dev); + } - rte_eth_copy_pci_info(eth_dev, pci_dev); - eth_dev->data->dev_flags = dev_flags; - - rx_func_get(eth_dev); + /* If host does not support both status and MSI-X then disable LSC */ + if (vtpci_with_feature(hw, VIRTIO_NET_F_STATUS) && + hw->use_msix != VIRTIO_MSIX_NONE) + eth_dev->data->dev_flags |= RTE_ETH_DEV_INTR_LSC; + else + eth_dev->data->dev_flags &= ~RTE_ETH_DEV_INTR_LSC; /* Setting up rx_header size for the device */ if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF) || @@ -1221,16 +1463,33 @@ eth_virtio_dev_init(struct rte_eth_dev *eth_dev) config->max_virtqueue_pairs = 1; } - hw->max_rx_queues = - (VIRTIO_MAX_RX_QUEUES < config->max_virtqueue_pairs) ? - VIRTIO_MAX_RX_QUEUES : config->max_virtqueue_pairs; - hw->max_tx_queues = - (VIRTIO_MAX_TX_QUEUES < config->max_virtqueue_pairs) ? - VIRTIO_MAX_TX_QUEUES : config->max_virtqueue_pairs; + hw->max_queue_pairs = config->max_virtqueue_pairs; - virtio_dev_cq_queue_setup(eth_dev, - config->max_virtqueue_pairs * 2, - SOCKET_ID_ANY); + if (vtpci_with_feature(hw, VIRTIO_NET_F_MTU)) { + vtpci_read_dev_config(hw, + offsetof(struct virtio_net_config, mtu), + &config->mtu, + sizeof(config->mtu)); + + /* + * MTU value has already been checked at negotiation + * time, but check again in case it has changed since + * then, which should not happen. + */ + if (config->mtu < ETHER_MIN_MTU) { + PMD_INIT_LOG(ERR, "invalid max MTU value (%u)", + config->mtu); + return -1; + } + + hw->max_mtu = config->mtu; + /* Set initial MTU to maximum one supported by vhost */ + eth_dev->data->mtu = config->mtu; + + } else { + hw->max_mtu = VIRTIO_MAX_RX_PKTLEN - ETHER_HDR_LEN - + VLAN_TAG_LEN - hw->vtnet_hdr_size; + } PMD_INIT_LOG(DEBUG, "config->max_virtqueue_pairs=%d", config->max_virtqueue_pairs); @@ -1241,94 +1500,203 @@ eth_virtio_dev_init(struct rte_eth_dev *eth_dev) config->mac[2], config->mac[3], config->mac[4], config->mac[5]); } else { - hw->max_rx_queues = 1; - hw->max_tx_queues = 1; + PMD_INIT_LOG(DEBUG, "config->max_virtqueue_pairs=1"); + hw->max_queue_pairs = 1; } - PMD_INIT_LOG(DEBUG, "hw->max_rx_queues=%d hw->max_tx_queues=%d", - hw->max_rx_queues, hw->max_tx_queues); + ret = virtio_alloc_queues(eth_dev); + if (ret < 0) + return ret; + + if (eth_dev->data->dev_conf.intr_conf.rxq) { + if (virtio_configure_intr(eth_dev) < 0) { + PMD_INIT_LOG(ERR, "failed to configure interrupt"); + return -1; + } + } + + vtpci_reinit_complete(hw); + if (pci_dev) PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x", eth_dev->data->port_id, pci_dev->id.vendor_id, pci_dev->id.device_id); - /* Setup interrupt callback */ - if (eth_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) - rte_intr_callback_register(&pci_dev->intr_handle, - virtio_interrupt_handler, eth_dev); + return 0; +} - virtio_dev_cq_start(eth_dev); +/* + * Remap the PCI device again (IO port map for legacy device and + * memory map for modern device), so that the secondary process + * could have the PCI initiated correctly. + */ +static int +virtio_remap_pci(struct rte_pci_device *pci_dev, struct virtio_hw *hw) +{ + if (hw->modern) { + /* + * We don't have to re-parse the PCI config space, since + * rte_pci_map_device() makes sure the mapped address + * in secondary process would equal to the one mapped in + * the primary process: error will be returned if that + * requirement is not met. + * + * That said, we could simply reuse all cap pointers + * (such as dev_cfg, common_cfg, etc.) parsed from the + * primary process, which is stored in shared memory. + */ + if (rte_pci_map_device(pci_dev)) { + PMD_INIT_LOG(DEBUG, "failed to map pci device!"); + return -1; + } + } else { + if (rte_pci_ioport_map(pci_dev, 0, VTPCI_IO(hw)) < 0) + return -1; + } return 0; } +static void +virtio_set_vtpci_ops(struct virtio_hw *hw) +{ +#ifdef RTE_VIRTIO_USER + if (hw->virtio_user_dev) + VTPCI_OPS(hw) = &virtio_user_ops; + else +#endif + if (hw->modern) + VTPCI_OPS(hw) = &modern_ops; + else + VTPCI_OPS(hw) = &legacy_ops; +} + +/* + * This function is based on probe() function in virtio_pci.c + * It returns 0 on success. + */ +int +eth_virtio_dev_init(struct rte_eth_dev *eth_dev) +{ + struct virtio_hw *hw = eth_dev->data->dev_private; + int ret; + + RTE_BUILD_BUG_ON(RTE_PKTMBUF_HEADROOM < sizeof(struct virtio_net_hdr_mrg_rxbuf)); + + eth_dev->dev_ops = &virtio_eth_dev_ops; + + if (rte_eal_process_type() == RTE_PROC_SECONDARY) { + if (!hw->virtio_user_dev) { + ret = virtio_remap_pci(RTE_ETH_DEV_TO_PCI(eth_dev), hw); + if (ret) + return ret; + } + + virtio_set_vtpci_ops(hw); + set_rxtx_funcs(eth_dev); + + return 0; + } + + /* Allocate memory for storing MAC addresses */ + eth_dev->data->mac_addrs = rte_zmalloc("virtio", VIRTIO_MAX_MAC_ADDRS * ETHER_ADDR_LEN, 0); + if (eth_dev->data->mac_addrs == NULL) { + PMD_INIT_LOG(ERR, + "Failed to allocate %d bytes needed to store MAC addresses", + VIRTIO_MAX_MAC_ADDRS * ETHER_ADDR_LEN); + return -ENOMEM; + } + + hw->port_id = eth_dev->data->port_id; + /* For virtio_user case the hw->virtio_user_dev is populated by + * virtio_user_eth_dev_alloc() before eth_virtio_dev_init() is called. + */ + if (!hw->virtio_user_dev) { + ret = vtpci_init(RTE_ETH_DEV_TO_PCI(eth_dev), hw); + if (ret) + goto out; + } + + /* reset device and negotiate default features */ + ret = virtio_init_device(eth_dev, VIRTIO_PMD_DEFAULT_GUEST_FEATURES); + if (ret < 0) + goto out; + + /* Setup interrupt callback */ + if (eth_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) + rte_intr_callback_register(eth_dev->intr_handle, + virtio_interrupt_handler, eth_dev); + + return 0; + +out: + rte_free(eth_dev->data->mac_addrs); + return ret; +} + static int eth_virtio_dev_uninit(struct rte_eth_dev *eth_dev) { - struct rte_pci_device *pci_dev; - struct virtio_hw *hw = eth_dev->data->dev_private; - PMD_INIT_FUNC_TRACE(); if (rte_eal_process_type() == RTE_PROC_SECONDARY) return -EPERM; - if (hw->started == 1) { - virtio_dev_stop(eth_dev); - virtio_dev_close(eth_dev); - } - pci_dev = eth_dev->pci_dev; + virtio_dev_stop(eth_dev); + virtio_dev_close(eth_dev); eth_dev->dev_ops = NULL; eth_dev->tx_pkt_burst = NULL; eth_dev->rx_pkt_burst = NULL; - if (hw->cvq) - virtio_dev_queue_release(hw->cvq->vq); - rte_free(eth_dev->data->mac_addrs); eth_dev->data->mac_addrs = NULL; /* reset interrupt callback */ if (eth_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) - rte_intr_callback_unregister(&pci_dev->intr_handle, + rte_intr_callback_unregister(eth_dev->intr_handle, virtio_interrupt_handler, eth_dev); - rte_eal_pci_unmap_device(pci_dev); + if (eth_dev->device) + rte_pci_unmap_device(RTE_ETH_DEV_TO_PCI(eth_dev)); PMD_INIT_LOG(DEBUG, "dev_uninit completed"); return 0; } -static struct eth_driver rte_virtio_pmd = { - .pci_drv = { - .name = "rte_virtio_pmd", - .id_table = pci_id_virtio_map, - .drv_flags = RTE_PCI_DRV_DETACHABLE, +static int eth_virtio_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, + struct rte_pci_device *pci_dev) +{ + return rte_eth_dev_pci_generic_probe(pci_dev, sizeof(struct virtio_hw), + eth_virtio_dev_init); +} + +static int eth_virtio_pci_remove(struct rte_pci_device *pci_dev) +{ + return rte_eth_dev_pci_generic_remove(pci_dev, eth_virtio_dev_uninit); +} + +static struct rte_pci_driver rte_virtio_pmd = { + .driver = { + .name = "net_virtio", }, - .eth_dev_init = eth_virtio_dev_init, - .eth_dev_uninit = eth_virtio_dev_uninit, - .dev_private_size = sizeof(struct virtio_hw), + .id_table = pci_id_virtio_map, + .drv_flags = 0, + .probe = eth_virtio_pci_probe, + .remove = eth_virtio_pci_remove, }; -/* - * Driver initialization routine. - * Invoked once at EAL init time. - * Register itself as the [Poll Mode] Driver of PCI virtio devices. - * Returns 0 on success. - */ -static int -rte_virtio_pmd_init(const char *name __rte_unused, - const char *param __rte_unused) +RTE_INIT(rte_virtio_pmd_init); +static void +rte_virtio_pmd_init(void) { if (rte_eal_iopl_init() != 0) { PMD_INIT_LOG(ERR, "IOPL call failed - cannot use virtio PMD"); - return -1; + return; } - rte_eth_driver_register(&rte_virtio_pmd); - return 0; + rte_pci_register(&rte_virtio_pmd); } /* @@ -1340,29 +1708,90 @@ virtio_dev_configure(struct rte_eth_dev *dev) { const struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode; struct virtio_hw *hw = dev->data->dev_private; + uint64_t req_features; + int ret; PMD_INIT_LOG(DEBUG, "configure"); + req_features = VIRTIO_PMD_DEFAULT_GUEST_FEATURES; - if (rxmode->hw_ip_checksum) { - PMD_DRV_LOG(ERR, "HW IP checksum not supported"); - return -EINVAL; + if (dev->data->dev_conf.intr_conf.rxq) { + ret = virtio_init_device(dev, hw->req_guest_features); + if (ret < 0) + return ret; } + /* The name hw_ip_checksum is a bit confusing since it can be + * set by the application to request L3 and/or L4 checksums. In + * case of virtio, only L4 checksum is supported. + */ + if (rxmode->hw_ip_checksum) + req_features |= (1ULL << VIRTIO_NET_F_GUEST_CSUM); + + if (rxmode->enable_lro) + req_features |= + (1ULL << VIRTIO_NET_F_GUEST_TSO4) | + (1ULL << VIRTIO_NET_F_GUEST_TSO6); + + /* if request features changed, reinit the device */ + if (req_features != hw->req_guest_features) { + ret = virtio_init_device(dev, req_features); + if (ret < 0) + return ret; + } + + if (rxmode->hw_ip_checksum && + !vtpci_with_feature(hw, VIRTIO_NET_F_GUEST_CSUM)) { + PMD_DRV_LOG(ERR, + "rx checksum not available on this host"); + return -ENOTSUP; + } + + if (rxmode->enable_lro && + (!vtpci_with_feature(hw, VIRTIO_NET_F_GUEST_TSO4) || + !vtpci_with_feature(hw, VIRTIO_NET_F_GUEST_TSO6))) { + PMD_DRV_LOG(ERR, + "Large Receive Offload not available on this host"); + return -ENOTSUP; + } + + /* start control queue */ + if (vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_VQ)) + virtio_dev_cq_start(dev); + hw->vlan_strip = rxmode->hw_vlan_strip; if (rxmode->hw_vlan_filter && !vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_VLAN)) { - PMD_DRV_LOG(NOTICE, + PMD_DRV_LOG(ERR, "vlan filtering not available on this host"); return -ENOTSUP; } if (dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) - if (vtpci_irq_config(hw, 0) == VIRTIO_MSI_NO_VECTOR) { + /* Enable vector (0) for Link State Intrerrupt */ + if (VTPCI_OPS(hw)->set_config_irq(hw, 0) == + VIRTIO_MSI_NO_VECTOR) { PMD_DRV_LOG(ERR, "failed to set config vector"); return -EBUSY; } + hw->use_simple_rx = 1; + hw->use_simple_tx = 1; + +#if defined RTE_ARCH_ARM64 || defined RTE_ARCH_ARM + if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_NEON)) { + hw->use_simple_rx = 0; + hw->use_simple_tx = 0; + } +#endif + if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) { + hw->use_simple_rx = 0; + hw->use_simple_tx = 0; + } + + if (rxmode->hw_ip_checksum) + hw->use_simple_rx = 0; + return 0; } @@ -1371,9 +1800,22 @@ static int virtio_dev_start(struct rte_eth_dev *dev) { uint16_t nb_queues, i; - struct virtio_hw *hw = dev->data->dev_private; struct virtnet_rx *rxvq; struct virtnet_tx *txvq __rte_unused; + struct virtio_hw *hw = dev->data->dev_private; + int ret; + + /* Finish the initialization of the queues */ + for (i = 0; i < dev->data->nb_rx_queues; i++) { + ret = virtio_dev_rx_queue_setup_finish(dev, i); + if (ret < 0) + return ret; + } + for (i = 0; i < dev->data->nb_tx_queues; i++) { + ret = virtio_dev_tx_queue_setup_finish(dev, i); + if (ret < 0) + return ret; + } /* check if lsc interrupt feature is enabled */ if (dev->data->dev_conf.intr_conf.lsc) { @@ -1381,43 +1823,46 @@ virtio_dev_start(struct rte_eth_dev *dev) PMD_DRV_LOG(ERR, "link status not supported by host"); return -ENOTSUP; } + } - if (rte_intr_enable(&dev->pci_dev->intr_handle) < 0) { + /* Enable uio/vfio intr/eventfd mapping: althrough we already did that + * in device configure, but it could be unmapped when device is + * stopped. + */ + if (dev->data->dev_conf.intr_conf.lsc || + dev->data->dev_conf.intr_conf.rxq) { + virtio_intr_disable(dev); + + if (virtio_intr_enable(dev) < 0) { PMD_DRV_LOG(ERR, "interrupt enable failed"); return -EIO; } } - /* Initialize Link state */ - virtio_dev_link_update(dev, 0); - - /* On restart after stop do not touch queues */ - if (hw->started) - return 0; - - /* Do final configuration before rx/tx engine starts */ - virtio_dev_rxtx_start(dev); - vtpci_reinit_complete(hw); - - hw->started = 1; - /*Notify the backend *Otherwise the tap backend might already stop its queue due to fullness. *vhost backend will have no chance to be waked up */ - nb_queues = dev->data->nb_rx_queues; - if (nb_queues > 1) { + nb_queues = RTE_MAX(dev->data->nb_rx_queues, dev->data->nb_tx_queues); + if (hw->max_queue_pairs > 1) { if (virtio_set_multiple_queues(dev, nb_queues) != 0) return -EINVAL; } PMD_INIT_LOG(DEBUG, "nb_queues=%d", nb_queues); - for (i = 0; i < nb_queues; i++) { + for (i = 0; i < dev->data->nb_rx_queues; i++) { rxvq = dev->data->rx_queues[i]; + /* Flush the old packets */ + virtqueue_rxvq_flush(rxvq->vq); virtqueue_notify(rxvq->vq); } + for (i = 0; i < dev->data->nb_tx_queues; i++) { + txvq = dev->data->tx_queues[i]; + virtqueue_notify(txvq->vq); + } + PMD_INIT_LOG(DEBUG, "Notified backend at initialization"); for (i = 0; i < dev->data->nb_rx_queues; i++) { @@ -1430,6 +1875,12 @@ virtio_dev_start(struct rte_eth_dev *dev) VIRTQUEUE_DUMP(txvq->vq); } + set_rxtx_funcs(dev); + hw->started = 1; + + /* Initialize Link state */ + virtio_dev_link_update(dev, 0); + return 0; } @@ -1441,6 +1892,9 @@ static void virtio_dev_free_mbufs(struct rte_eth_dev *dev) for (i = 0; i < dev->data->nb_rx_queues; i++) { struct virtnet_rx *rxvq = dev->data->rx_queues[i]; + if (rxvq == NULL || rxvq->vq == NULL) + continue; + PMD_INIT_LOG(DEBUG, "Before freeing rxq[%d] used and unused buf", i); VIRTQUEUE_DUMP(rxvq->vq); @@ -1460,6 +1914,9 @@ static void virtio_dev_free_mbufs(struct rte_eth_dev *dev) for (i = 0; i < dev->data->nb_tx_queues; i++) { struct virtnet_tx *txvq = dev->data->tx_queues[i]; + if (txvq == NULL || txvq->vq == NULL) + continue; + PMD_INIT_LOG(DEBUG, "Before freeing txq[%d] used and unused bufs", i); @@ -1484,13 +1941,16 @@ static void virtio_dev_free_mbufs(struct rte_eth_dev *dev) static void virtio_dev_stop(struct rte_eth_dev *dev) { + struct virtio_hw *hw = dev->data->dev_private; struct rte_eth_link link; + struct rte_intr_conf *intr_conf = &dev->data->dev_conf.intr_conf; PMD_INIT_LOG(DEBUG, "stop"); - if (dev->data->dev_conf.intr_conf.lsc) - rte_intr_disable(&dev->pci_dev->intr_handle); + if (intr_conf->lsc || intr_conf->rxq) + virtio_intr_disable(dev); + hw->started = 0; memset(&link, 0, sizeof(link)); virtio_dev_atomic_write_link_status(dev, &link); } @@ -1507,7 +1967,9 @@ virtio_dev_link_update(struct rte_eth_dev *dev, __rte_unused int wait_to_complet link.link_duplex = ETH_LINK_FULL_DUPLEX; link.link_speed = SPEED_10G; - if (vtpci_with_feature(hw, VIRTIO_NET_F_STATUS)) { + if (hw->started == 0) { + link.link_status = ETH_LINK_DOWN; + } else if (vtpci_with_feature(hw, VIRTIO_NET_F_STATUS)) { PMD_INIT_LOG(DEBUG, "Get link status from hw"); vtpci_read_dev_config(hw, offsetof(struct virtio_net_config, status), @@ -1529,23 +1991,71 @@ virtio_dev_link_update(struct rte_eth_dev *dev, __rte_unused int wait_to_complet return (old.link_status == link.link_status) ? -1 : 0; } +static int +virtio_dev_vlan_offload_set(struct rte_eth_dev *dev, int mask) +{ + const struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode; + struct virtio_hw *hw = dev->data->dev_private; + + if (mask & ETH_VLAN_FILTER_MASK) { + if (rxmode->hw_vlan_filter && + !vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_VLAN)) { + + PMD_DRV_LOG(NOTICE, + "vlan filtering not available on this host"); + + return -ENOTSUP; + } + } + + if (mask & ETH_VLAN_STRIP_MASK) + hw->vlan_strip = rxmode->hw_vlan_strip; + + return 0; +} + static void virtio_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) { + uint64_t tso_mask, host_features; struct virtio_hw *hw = dev->data->dev_private; - if (dev->pci_dev) - dev_info->driver_name = dev->driver->pci_drv.name; - else - dev_info->driver_name = "virtio_user PMD"; - dev_info->max_rx_queues = (uint16_t)hw->max_rx_queues; - dev_info->max_tx_queues = (uint16_t)hw->max_tx_queues; + dev_info->speed_capa = ETH_LINK_SPEED_10G; /* fake value */ + + dev_info->pci_dev = dev->device ? RTE_ETH_DEV_TO_PCI(dev) : NULL; + dev_info->max_rx_queues = + RTE_MIN(hw->max_queue_pairs, VIRTIO_MAX_RX_QUEUES); + dev_info->max_tx_queues = + RTE_MIN(hw->max_queue_pairs, VIRTIO_MAX_TX_QUEUES); dev_info->min_rx_bufsize = VIRTIO_MIN_RX_BUFSIZE; dev_info->max_rx_pktlen = VIRTIO_MAX_RX_PKTLEN; dev_info->max_mac_addrs = VIRTIO_MAX_MAC_ADDRS; dev_info->default_txconf = (struct rte_eth_txconf) { .txq_flags = ETH_TXQ_FLAGS_NOOFFLOADS }; + + host_features = VTPCI_OPS(hw)->get_features(hw); + dev_info->rx_offload_capa = 0; + if (host_features & (1ULL << VIRTIO_NET_F_GUEST_CSUM)) { + dev_info->rx_offload_capa |= + DEV_RX_OFFLOAD_TCP_CKSUM | + DEV_RX_OFFLOAD_UDP_CKSUM; + } + tso_mask = (1ULL << VIRTIO_NET_F_GUEST_TSO4) | + (1ULL << VIRTIO_NET_F_GUEST_TSO6); + if ((host_features & tso_mask) == tso_mask) + dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_TCP_LRO; + + dev_info->tx_offload_capa = 0; + if (hw->guest_features & (1ULL << VIRTIO_NET_F_CSUM)) { + dev_info->tx_offload_capa |= + DEV_TX_OFFLOAD_UDP_CKSUM | + DEV_TX_OFFLOAD_TCP_CKSUM; + } + tso_mask = (1ULL << VIRTIO_NET_F_HOST_TSO4) | + (1ULL << VIRTIO_NET_F_HOST_TSO6); + if ((hw->guest_features & tso_mask) == tso_mask) + dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_TCP_TSO; } /* @@ -1559,10 +2069,6 @@ __rte_unused uint8_t is_rx) return 0; } -static struct rte_driver rte_virtio_driver = { - .type = PMD_PDEV, - .init = rte_virtio_pmd_init, -}; - -PMD_REGISTER_DRIVER(rte_virtio_driver, virtio_net); -DRIVER_REGISTER_PCI_TABLE(virtio_net, pci_id_virtio_map); +RTE_PMD_EXPORT_NAME(net_virtio, __COUNTER__); +RTE_PMD_REGISTER_PCI_TABLE(net_virtio, pci_id_virtio_map); +RTE_PMD_REGISTER_KMOD_DEP(net_virtio, "* igb_uio | uio_pci_generic | vfio-pci"); diff --git a/dpdk/drivers/net/virtio/virtio_ethdev.h b/dpdk/drivers/net/virtio/virtio_ethdev.h index 2ecec6eb..2039bc54 100644 --- a/dpdk/drivers/net/virtio/virtio_ethdev.h +++ b/dpdk/drivers/net/virtio/virtio_ethdev.h @@ -47,14 +47,14 @@ #define PAGE_SIZE 4096 #endif -#define VIRTIO_MAX_RX_QUEUES 128 -#define VIRTIO_MAX_TX_QUEUES 128 +#define VIRTIO_MAX_RX_QUEUES 128U +#define VIRTIO_MAX_TX_QUEUES 128U #define VIRTIO_MAX_MAC_ADDRS 64 #define VIRTIO_MIN_RX_BUFSIZE 64 -#define VIRTIO_MAX_RX_PKTLEN 9728 +#define VIRTIO_MAX_RX_PKTLEN 9728U /* Features desired/implemented by this driver. */ -#define VIRTIO_PMD_GUEST_FEATURES \ +#define VIRTIO_PMD_DEFAULT_GUEST_FEATURES \ (1u << VIRTIO_NET_F_MAC | \ 1u << VIRTIO_NET_F_STATUS | \ 1u << VIRTIO_NET_F_MQ | \ @@ -62,9 +62,20 @@ 1u << VIRTIO_NET_F_CTRL_VQ | \ 1u << VIRTIO_NET_F_CTRL_RX | \ 1u << VIRTIO_NET_F_CTRL_VLAN | \ + 1u << VIRTIO_NET_F_CSUM | \ + 1u << VIRTIO_NET_F_HOST_TSO4 | \ + 1u << VIRTIO_NET_F_HOST_TSO6 | \ 1u << VIRTIO_NET_F_MRG_RXBUF | \ - 1ULL << VIRTIO_F_VERSION_1) + 1u << VIRTIO_NET_F_MTU | \ + 1u << VIRTIO_RING_F_INDIRECT_DESC | \ + 1ULL << VIRTIO_F_VERSION_1 | \ + 1ULL << VIRTIO_F_IOMMU_PLATFORM) +#define VIRTIO_PMD_SUPPORTED_GUEST_FEATURES \ + (VIRTIO_PMD_DEFAULT_GUEST_FEATURES | \ + 1u << VIRTIO_NET_F_GUEST_CSUM | \ + 1u << VIRTIO_NET_F_GUEST_TSO4 | \ + 1u << VIRTIO_NET_F_GUEST_TSO6) /* * CQ function prototype */ @@ -73,30 +84,23 @@ void virtio_dev_cq_start(struct rte_eth_dev *dev); /* * RX/TX function prototypes */ -void virtio_dev_rxtx_start(struct rte_eth_dev *dev); -int virtio_dev_queue_setup(struct rte_eth_dev *dev, - int queue_type, - uint16_t queue_idx, - uint16_t vtpci_queue_idx, - uint16_t nb_desc, - unsigned int socket_id, - void **pvq); - -void virtio_dev_queue_release(struct virtqueue *vq); +int virtio_dev_rx_queue_done(void *rxq, uint16_t offset); int virtio_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t rx_queue_id, uint16_t nb_rx_desc, unsigned int socket_id, const struct rte_eth_rxconf *rx_conf, struct rte_mempool *mb_pool); -void virtio_dev_rx_queue_release(void *rxq); +int virtio_dev_rx_queue_setup_finish(struct rte_eth_dev *dev, + uint16_t rx_queue_id); int virtio_dev_tx_queue_setup(struct rte_eth_dev *dev, uint16_t tx_queue_id, uint16_t nb_tx_desc, unsigned int socket_id, const struct rte_eth_txconf *tx_conf); -void virtio_dev_tx_queue_release(void *txq); +int virtio_dev_tx_queue_setup_finish(struct rte_eth_dev *dev, + uint16_t tx_queue_id); uint16_t virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts); @@ -115,13 +119,6 @@ uint16_t virtio_xmit_pkts_simple(void *tx_queue, struct rte_mbuf **tx_pkts, int eth_virtio_dev_init(struct rte_eth_dev *eth_dev); -/* - * The VIRTIO_NET_F_GUEST_TSO[46] features permit the host to send us - * frames larger than 1514 bytes. We do not yet support software LRO - * via tcp_lro_rx(). - */ -#define VTNET_LRO_FEATURES (VIRTIO_NET_F_GUEST_TSO4 | \ - VIRTIO_NET_F_GUEST_TSO6 | VIRTIO_NET_F_GUEST_ECN) - +void virtio_interrupt_handler(void *param); #endif /* _VIRTIO_ETHDEV_H_ */ diff --git a/dpdk/drivers/net/virtio/virtio_pci.c b/dpdk/drivers/net/virtio/virtio_pci.c index f1a7ca7e..9574498f 100644 --- a/dpdk/drivers/net/virtio/virtio_pci.c +++ b/dpdk/drivers/net/virtio/virtio_pci.c @@ -37,6 +37,9 @@ #include #endif +#include +#include + #include "virtio_pci.h" #include "virtio_logs.h" #include "virtqueue.h" @@ -48,12 +51,14 @@ */ #define PCI_CAPABILITY_LIST 0x34 #define PCI_CAP_ID_VNDR 0x09 +#define PCI_CAP_ID_MSIX 0x11 /* * The remaining space is defined by each driver as the per-driver * configuration space. */ -#define VIRTIO_PCI_CONFIG(hw) (((hw)->use_msix) ? 24 : 20) +#define VIRTIO_PCI_CONFIG(hw) \ + (((hw)->use_msix == VIRTIO_MSIX_ENABLED) ? 24 : 20) static inline int check_vq_phys_addr_ok(struct virtqueue *vq) @@ -92,17 +97,17 @@ legacy_read_dev_config(struct virtio_hw *hw, size_t offset, while (length > 0) { if (length >= 4) { size = 4; - rte_eal_pci_ioport_read(&hw->io, dst, size, + rte_pci_ioport_read(VTPCI_IO(hw), dst, size, VIRTIO_PCI_CONFIG(hw) + offset); *(uint32_t *)dst = rte_be_to_cpu_32(*(uint32_t *)dst); } else if (length >= 2) { size = 2; - rte_eal_pci_ioport_read(&hw->io, dst, size, + rte_pci_ioport_read(VTPCI_IO(hw), dst, size, VIRTIO_PCI_CONFIG(hw) + offset); *(uint16_t *)dst = rte_be_to_cpu_16(*(uint16_t *)dst); } else { size = 1; - rte_eal_pci_ioport_read(&hw->io, dst, size, + rte_pci_ioport_read(VTPCI_IO(hw), dst, size, VIRTIO_PCI_CONFIG(hw) + offset); } @@ -111,8 +116,8 @@ legacy_read_dev_config(struct virtio_hw *hw, size_t offset, length -= size; } #else - rte_eal_pci_ioport_read(&hw->io, dst, length, - VIRTIO_PCI_CONFIG(hw) + offset); + rte_pci_ioport_read(VTPCI_IO(hw), dst, length, + VIRTIO_PCI_CONFIG(hw) + offset); #endif } @@ -131,16 +136,16 @@ legacy_write_dev_config(struct virtio_hw *hw, size_t offset, if (length >= 4) { size = 4; tmp.u32 = rte_cpu_to_be_32(*(const uint32_t *)src); - rte_eal_pci_ioport_write(&hw->io, &tmp.u32, size, + rte_pci_ioport_write(VTPCI_IO(hw), &tmp.u32, size, VIRTIO_PCI_CONFIG(hw) + offset); } else if (length >= 2) { size = 2; tmp.u16 = rte_cpu_to_be_16(*(const uint16_t *)src); - rte_eal_pci_ioport_write(&hw->io, &tmp.u16, size, + rte_pci_ioport_write(VTPCI_IO(hw), &tmp.u16, size, VIRTIO_PCI_CONFIG(hw) + offset); } else { size = 1; - rte_eal_pci_ioport_write(&hw->io, src, size, + rte_pci_ioport_write(VTPCI_IO(hw), src, size, VIRTIO_PCI_CONFIG(hw) + offset); } @@ -149,8 +154,8 @@ legacy_write_dev_config(struct virtio_hw *hw, size_t offset, length -= size; } #else - rte_eal_pci_ioport_write(&hw->io, src, length, - VIRTIO_PCI_CONFIG(hw) + offset); + rte_pci_ioport_write(VTPCI_IO(hw), src, length, + VIRTIO_PCI_CONFIG(hw) + offset); #endif } @@ -159,7 +164,7 @@ legacy_get_features(struct virtio_hw *hw) { uint32_t dst; - rte_eal_pci_ioport_read(&hw->io, &dst, 4, VIRTIO_PCI_HOST_FEATURES); + rte_pci_ioport_read(VTPCI_IO(hw), &dst, 4, VIRTIO_PCI_HOST_FEATURES); return dst; } @@ -171,8 +176,8 @@ legacy_set_features(struct virtio_hw *hw, uint64_t features) "only 32 bit features are allowed for legacy virtio!"); return; } - rte_eal_pci_ioport_write(&hw->io, &features, 4, - VIRTIO_PCI_GUEST_FEATURES); + rte_pci_ioport_write(VTPCI_IO(hw), &features, 4, + VIRTIO_PCI_GUEST_FEATURES); } static uint8_t @@ -180,14 +185,14 @@ legacy_get_status(struct virtio_hw *hw) { uint8_t dst; - rte_eal_pci_ioport_read(&hw->io, &dst, 1, VIRTIO_PCI_STATUS); + rte_pci_ioport_read(VTPCI_IO(hw), &dst, 1, VIRTIO_PCI_STATUS); return dst; } static void legacy_set_status(struct virtio_hw *hw, uint8_t status) { - rte_eal_pci_ioport_write(&hw->io, &status, 1, VIRTIO_PCI_STATUS); + rte_pci_ioport_write(VTPCI_IO(hw), &status, 1, VIRTIO_PCI_STATUS); } static void @@ -201,7 +206,7 @@ legacy_get_isr(struct virtio_hw *hw) { uint8_t dst; - rte_eal_pci_ioport_read(&hw->io, &dst, 1, VIRTIO_PCI_ISR); + rte_pci_ioport_read(VTPCI_IO(hw), &dst, 1, VIRTIO_PCI_ISR); return dst; } @@ -211,8 +216,20 @@ legacy_set_config_irq(struct virtio_hw *hw, uint16_t vec) { uint16_t dst; - rte_eal_pci_ioport_write(&hw->io, &vec, 2, VIRTIO_MSI_CONFIG_VECTOR); - rte_eal_pci_ioport_read(&hw->io, &dst, 2, VIRTIO_MSI_CONFIG_VECTOR); + rte_pci_ioport_write(VTPCI_IO(hw), &vec, 2, VIRTIO_MSI_CONFIG_VECTOR); + rte_pci_ioport_read(VTPCI_IO(hw), &dst, 2, VIRTIO_MSI_CONFIG_VECTOR); + return dst; +} + +static uint16_t +legacy_set_queue_irq(struct virtio_hw *hw, struct virtqueue *vq, uint16_t vec) +{ + uint16_t dst; + + rte_pci_ioport_write(VTPCI_IO(hw), &vq->vq_queue_index, 2, + VIRTIO_PCI_QUEUE_SEL); + rte_pci_ioport_write(VTPCI_IO(hw), &vec, 2, VIRTIO_MSI_QUEUE_VECTOR); + rte_pci_ioport_read(VTPCI_IO(hw), &dst, 2, VIRTIO_MSI_QUEUE_VECTOR); return dst; } @@ -221,8 +238,8 @@ legacy_get_queue_num(struct virtio_hw *hw, uint16_t queue_id) { uint16_t dst; - rte_eal_pci_ioport_write(&hw->io, &queue_id, 2, VIRTIO_PCI_QUEUE_SEL); - rte_eal_pci_ioport_read(&hw->io, &dst, 2, VIRTIO_PCI_QUEUE_NUM); + rte_pci_ioport_write(VTPCI_IO(hw), &queue_id, 2, VIRTIO_PCI_QUEUE_SEL); + rte_pci_ioport_read(VTPCI_IO(hw), &dst, 2, VIRTIO_PCI_QUEUE_NUM); return dst; } @@ -234,10 +251,10 @@ legacy_setup_queue(struct virtio_hw *hw, struct virtqueue *vq) if (!check_vq_phys_addr_ok(vq)) return -1; - rte_eal_pci_ioport_write(&hw->io, &vq->vq_queue_index, 2, - VIRTIO_PCI_QUEUE_SEL); + rte_pci_ioport_write(VTPCI_IO(hw), &vq->vq_queue_index, 2, + VIRTIO_PCI_QUEUE_SEL); src = vq->vq_ring_mem >> VIRTIO_PCI_QUEUE_ADDR_SHIFT; - rte_eal_pci_ioport_write(&hw->io, &src, 4, VIRTIO_PCI_QUEUE_PFN); + rte_pci_ioport_write(VTPCI_IO(hw), &src, 4, VIRTIO_PCI_QUEUE_PFN); return 0; } @@ -247,60 +264,19 @@ legacy_del_queue(struct virtio_hw *hw, struct virtqueue *vq) { uint32_t src = 0; - rte_eal_pci_ioport_write(&hw->io, &vq->vq_queue_index, 2, - VIRTIO_PCI_QUEUE_SEL); - rte_eal_pci_ioport_write(&hw->io, &src, 4, VIRTIO_PCI_QUEUE_PFN); + rte_pci_ioport_write(VTPCI_IO(hw), &vq->vq_queue_index, 2, + VIRTIO_PCI_QUEUE_SEL); + rte_pci_ioport_write(VTPCI_IO(hw), &src, 4, VIRTIO_PCI_QUEUE_PFN); } static void legacy_notify_queue(struct virtio_hw *hw, struct virtqueue *vq) { - rte_eal_pci_ioport_write(&hw->io, &vq->vq_queue_index, 2, - VIRTIO_PCI_QUEUE_NOTIFY); + rte_pci_ioport_write(VTPCI_IO(hw), &vq->vq_queue_index, 2, + VIRTIO_PCI_QUEUE_NOTIFY); } -#ifdef RTE_EXEC_ENV_LINUXAPP -static int -legacy_virtio_has_msix(const struct rte_pci_addr *loc) -{ - DIR *d; - char dirname[PATH_MAX]; - - snprintf(dirname, sizeof(dirname), - "%s/" PCI_PRI_FMT "/msi_irqs", pci_get_sysfs_path(), - loc->domain, loc->bus, loc->devid, loc->function); - - d = opendir(dirname); - if (d) - closedir(d); - - return d != NULL; -} -#else -static int -legacy_virtio_has_msix(const struct rte_pci_addr *loc __rte_unused) -{ - /* nic_uio does not enable interrupts, return 0 (false). */ - return 0; -} -#endif - -static int -legacy_virtio_resource_init(struct rte_pci_device *pci_dev, - struct virtio_hw *hw, uint32_t *dev_flags) -{ - if (rte_eal_pci_ioport_map(pci_dev, 0, &hw->io) < 0) - return -1; - - if (pci_dev->intr_handle.type != RTE_INTR_HANDLE_UNKNOWN) - *dev_flags |= RTE_ETH_DEV_INTR_LSC; - else - *dev_flags &= ~RTE_ETH_DEV_INTR_LSC; - - return 0; -} - -static const struct virtio_pci_ops legacy_ops = { +const struct virtio_pci_ops legacy_ops = { .read_dev_cfg = legacy_read_dev_config, .write_dev_cfg = legacy_write_dev_config, .reset = legacy_reset, @@ -310,54 +286,18 @@ static const struct virtio_pci_ops legacy_ops = { .set_features = legacy_set_features, .get_isr = legacy_get_isr, .set_config_irq = legacy_set_config_irq, + .set_queue_irq = legacy_set_queue_irq, .get_queue_num = legacy_get_queue_num, .setup_queue = legacy_setup_queue, .del_queue = legacy_del_queue, .notify_queue = legacy_notify_queue, }; - -static inline uint8_t -io_read8(uint8_t *addr) -{ - return *(volatile uint8_t *)addr; -} - -static inline void -io_write8(uint8_t val, uint8_t *addr) -{ - *(volatile uint8_t *)addr = val; -} - -static inline uint16_t -io_read16(uint16_t *addr) -{ - return *(volatile uint16_t *)addr; -} - -static inline void -io_write16(uint16_t val, uint16_t *addr) -{ - *(volatile uint16_t *)addr = val; -} - -static inline uint32_t -io_read32(uint32_t *addr) -{ - return *(volatile uint32_t *)addr; -} - -static inline void -io_write32(uint32_t val, uint32_t *addr) -{ - *(volatile uint32_t *)addr = val; -} - static inline void io_write64_twopart(uint64_t val, uint32_t *lo, uint32_t *hi) { - io_write32(val & ((1ULL << 32) - 1), lo); - io_write32(val >> 32, hi); + rte_write32(val & ((1ULL << 32) - 1), lo); + rte_write32(val >> 32, hi); } static void @@ -369,13 +309,13 @@ modern_read_dev_config(struct virtio_hw *hw, size_t offset, uint8_t old_gen, new_gen; do { - old_gen = io_read8(&hw->common_cfg->config_generation); + old_gen = rte_read8(&hw->common_cfg->config_generation); p = dst; for (i = 0; i < length; i++) - *p++ = io_read8((uint8_t *)hw->dev_cfg + offset + i); + *p++ = rte_read8((uint8_t *)hw->dev_cfg + offset + i); - new_gen = io_read8(&hw->common_cfg->config_generation); + new_gen = rte_read8(&hw->common_cfg->config_generation); } while (old_gen != new_gen); } @@ -387,7 +327,7 @@ modern_write_dev_config(struct virtio_hw *hw, size_t offset, const uint8_t *p = src; for (i = 0; i < length; i++) - io_write8(*p++, (uint8_t *)hw->dev_cfg + offset + i); + rte_write8((*p++), (((uint8_t *)hw->dev_cfg) + offset + i)); } static uint64_t @@ -395,11 +335,11 @@ modern_get_features(struct virtio_hw *hw) { uint32_t features_lo, features_hi; - io_write32(0, &hw->common_cfg->device_feature_select); - features_lo = io_read32(&hw->common_cfg->device_feature); + rte_write32(0, &hw->common_cfg->device_feature_select); + features_lo = rte_read32(&hw->common_cfg->device_feature); - io_write32(1, &hw->common_cfg->device_feature_select); - features_hi = io_read32(&hw->common_cfg->device_feature); + rte_write32(1, &hw->common_cfg->device_feature_select); + features_hi = rte_read32(&hw->common_cfg->device_feature); return ((uint64_t)features_hi << 32) | features_lo; } @@ -407,25 +347,25 @@ modern_get_features(struct virtio_hw *hw) static void modern_set_features(struct virtio_hw *hw, uint64_t features) { - io_write32(0, &hw->common_cfg->guest_feature_select); - io_write32(features & ((1ULL << 32) - 1), - &hw->common_cfg->guest_feature); + rte_write32(0, &hw->common_cfg->guest_feature_select); + rte_write32(features & ((1ULL << 32) - 1), + &hw->common_cfg->guest_feature); - io_write32(1, &hw->common_cfg->guest_feature_select); - io_write32(features >> 32, - &hw->common_cfg->guest_feature); + rte_write32(1, &hw->common_cfg->guest_feature_select); + rte_write32(features >> 32, + &hw->common_cfg->guest_feature); } static uint8_t modern_get_status(struct virtio_hw *hw) { - return io_read8(&hw->common_cfg->device_status); + return rte_read8(&hw->common_cfg->device_status); } static void modern_set_status(struct virtio_hw *hw, uint8_t status) { - io_write8(status, &hw->common_cfg->device_status); + rte_write8(status, &hw->common_cfg->device_status); } static void @@ -438,21 +378,29 @@ modern_reset(struct virtio_hw *hw) static uint8_t modern_get_isr(struct virtio_hw *hw) { - return io_read8(hw->isr); + return rte_read8(hw->isr); } static uint16_t modern_set_config_irq(struct virtio_hw *hw, uint16_t vec) { - io_write16(vec, &hw->common_cfg->msix_config); - return io_read16(&hw->common_cfg->msix_config); + rte_write16(vec, &hw->common_cfg->msix_config); + return rte_read16(&hw->common_cfg->msix_config); +} + +static uint16_t +modern_set_queue_irq(struct virtio_hw *hw, struct virtqueue *vq, uint16_t vec) +{ + rte_write16(vq->vq_queue_index, &hw->common_cfg->queue_select); + rte_write16(vec, &hw->common_cfg->queue_msix_vector); + return rte_read16(&hw->common_cfg->queue_msix_vector); } static uint16_t modern_get_queue_num(struct virtio_hw *hw, uint16_t queue_id) { - io_write16(queue_id, &hw->common_cfg->queue_select); - return io_read16(&hw->common_cfg->queue_size); + rte_write16(queue_id, &hw->common_cfg->queue_select); + return rte_read16(&hw->common_cfg->queue_size); } static int @@ -470,7 +418,7 @@ modern_setup_queue(struct virtio_hw *hw, struct virtqueue *vq) ring[vq->vq_nentries]), VIRTIO_PCI_VRING_ALIGN); - io_write16(vq->vq_queue_index, &hw->common_cfg->queue_select); + rte_write16(vq->vq_queue_index, &hw->common_cfg->queue_select); io_write64_twopart(desc_addr, &hw->common_cfg->queue_desc_lo, &hw->common_cfg->queue_desc_hi); @@ -479,11 +427,11 @@ modern_setup_queue(struct virtio_hw *hw, struct virtqueue *vq) io_write64_twopart(used_addr, &hw->common_cfg->queue_used_lo, &hw->common_cfg->queue_used_hi); - notify_off = io_read16(&hw->common_cfg->queue_notify_off); + notify_off = rte_read16(&hw->common_cfg->queue_notify_off); vq->notify_addr = (void *)((uint8_t *)hw->notify_base + notify_off * hw->notify_off_multiplier); - io_write16(1, &hw->common_cfg->queue_enable); + rte_write16(1, &hw->common_cfg->queue_enable); PMD_INIT_LOG(DEBUG, "queue %u addresses:", vq->vq_queue_index); PMD_INIT_LOG(DEBUG, "\t desc_addr: %" PRIx64, desc_addr); @@ -498,7 +446,7 @@ modern_setup_queue(struct virtio_hw *hw, struct virtqueue *vq) static void modern_del_queue(struct virtio_hw *hw, struct virtqueue *vq) { - io_write16(vq->vq_queue_index, &hw->common_cfg->queue_select); + rte_write16(vq->vq_queue_index, &hw->common_cfg->queue_select); io_write64_twopart(0, &hw->common_cfg->queue_desc_lo, &hw->common_cfg->queue_desc_hi); @@ -507,16 +455,16 @@ modern_del_queue(struct virtio_hw *hw, struct virtqueue *vq) io_write64_twopart(0, &hw->common_cfg->queue_used_lo, &hw->common_cfg->queue_used_hi); - io_write16(0, &hw->common_cfg->queue_enable); + rte_write16(0, &hw->common_cfg->queue_enable); } static void modern_notify_queue(struct virtio_hw *hw __rte_unused, struct virtqueue *vq) { - io_write16(1, vq->notify_addr); + rte_write16(vq->vq_queue_index, vq->notify_addr); } -static const struct virtio_pci_ops modern_ops = { +const struct virtio_pci_ops modern_ops = { .read_dev_cfg = modern_read_dev_config, .write_dev_cfg = modern_write_dev_config, .reset = modern_reset, @@ -526,6 +474,7 @@ static const struct virtio_pci_ops modern_ops = { .set_features = modern_set_features, .get_isr = modern_get_isr, .set_config_irq = modern_set_config_irq, + .set_queue_irq = modern_set_queue_irq, .get_queue_num = modern_get_queue_num, .setup_queue = modern_setup_queue, .del_queue = modern_del_queue, @@ -537,14 +486,14 @@ void vtpci_read_dev_config(struct virtio_hw *hw, size_t offset, void *dst, int length) { - hw->vtpci_ops->read_dev_cfg(hw, offset, dst, length); + VTPCI_OPS(hw)->read_dev_cfg(hw, offset, dst, length); } void vtpci_write_dev_config(struct virtio_hw *hw, size_t offset, const void *src, int length) { - hw->vtpci_ops->write_dev_cfg(hw, offset, src, length); + VTPCI_OPS(hw)->write_dev_cfg(hw, offset, src, length); } uint64_t @@ -557,7 +506,7 @@ vtpci_negotiate_features(struct virtio_hw *hw, uint64_t host_features) * host all support. */ features = host_features & hw->guest_features; - hw->vtpci_ops->set_features(hw, features); + VTPCI_OPS(hw)->set_features(hw, features); return features; } @@ -565,9 +514,9 @@ vtpci_negotiate_features(struct virtio_hw *hw, uint64_t host_features) void vtpci_reset(struct virtio_hw *hw) { - hw->vtpci_ops->set_status(hw, VIRTIO_CONFIG_STATUS_RESET); + VTPCI_OPS(hw)->set_status(hw, VIRTIO_CONFIG_STATUS_RESET); /* flush status write */ - hw->vtpci_ops->get_status(hw); + VTPCI_OPS(hw)->get_status(hw); } void @@ -580,29 +529,21 @@ void vtpci_set_status(struct virtio_hw *hw, uint8_t status) { if (status != VIRTIO_CONFIG_STATUS_RESET) - status |= hw->vtpci_ops->get_status(hw); + status |= VTPCI_OPS(hw)->get_status(hw); - hw->vtpci_ops->set_status(hw, status); + VTPCI_OPS(hw)->set_status(hw, status); } uint8_t vtpci_get_status(struct virtio_hw *hw) { - return hw->vtpci_ops->get_status(hw); + return VTPCI_OPS(hw)->get_status(hw); } uint8_t vtpci_isr(struct virtio_hw *hw) { - return hw->vtpci_ops->get_isr(hw); -} - - -/* Enable one vector (0) for Link State Intrerrupt */ -uint16_t -vtpci_irq_config(struct virtio_hw *hw, uint16_t vec) -{ - return hw->vtpci_ops->set_config_irq(hw, vec); + return VTPCI_OPS(hw)->get_isr(hw); } static void * @@ -613,7 +554,7 @@ get_cfg_addr(struct rte_pci_device *dev, struct virtio_pci_cap *cap) uint32_t offset = cap->offset; uint8_t *base; - if (bar > 5) { + if (bar >= PCI_MAX_RESOURCE) { PMD_INIT_LOG(ERR, "invalid bar: %u", bar); return NULL; } @@ -640,6 +581,8 @@ get_cfg_addr(struct rte_pci_device *dev, struct virtio_pci_cap *cap) return base + offset; } +#define PCI_MSIX_ENABLE 0x8000 + static int virtio_read_caps(struct rte_pci_device *dev, struct virtio_hw *hw) { @@ -647,25 +590,39 @@ virtio_read_caps(struct rte_pci_device *dev, struct virtio_hw *hw) struct virtio_pci_cap cap; int ret; - if (rte_eal_pci_map_device(dev)) { + if (rte_pci_map_device(dev)) { PMD_INIT_LOG(DEBUG, "failed to map pci device!"); return -1; } - ret = rte_eal_pci_read_config(dev, &pos, 1, PCI_CAPABILITY_LIST); + ret = rte_pci_read_config(dev, &pos, 1, PCI_CAPABILITY_LIST); if (ret < 0) { PMD_INIT_LOG(DEBUG, "failed to read pci capability list"); return -1; } while (pos) { - ret = rte_eal_pci_read_config(dev, &cap, sizeof(cap), pos); + ret = rte_pci_read_config(dev, &cap, sizeof(cap), pos); if (ret < 0) { PMD_INIT_LOG(ERR, "failed to read pci cap at pos: %x", pos); break; } + if (cap.cap_vndr == PCI_CAP_ID_MSIX) { + /* Transitional devices would also have this capability, + * that's why we also check if msix is enabled. + * 1st byte is cap ID; 2nd byte is the position of next + * cap; next two bytes are the flags. + */ + uint16_t flags = ((uint16_t *)&cap)[1]; + + if (flags & PCI_MSIX_ENABLE) + hw->use_msix = VIRTIO_MSIX_ENABLED; + else + hw->use_msix = VIRTIO_MSIX_DISABLED; + } + if (cap.cap_vndr != PCI_CAP_ID_VNDR) { PMD_INIT_LOG(DEBUG, "[%2x] skipping non VNDR cap id: %02x", @@ -682,8 +639,8 @@ virtio_read_caps(struct rte_pci_device *dev, struct virtio_hw *hw) hw->common_cfg = get_cfg_addr(dev, &cap); break; case VIRTIO_PCI_CAP_NOTIFY_CFG: - rte_eal_pci_read_config(dev, &hw->notify_off_multiplier, - 4, pos + sizeof(cap)); + rte_pci_read_config(dev, &hw->notify_off_multiplier, + 4, pos + sizeof(cap)); hw->notify_base = get_cfg_addr(dev, &cap); break; case VIRTIO_PCI_CAP_DEVICE_CFG: @@ -724,11 +681,8 @@ next: * Return 0 on success. */ int -vtpci_init(struct rte_pci_device *dev, struct virtio_hw *hw, - uint32_t *dev_flags) +vtpci_init(struct rte_pci_device *dev, struct virtio_hw *hw) { - hw->dev = dev; - /* * Try if we can succeed reading virtio pci caps, which exists * only on modern pci device. If failed, we fallback to legacy @@ -736,17 +690,17 @@ vtpci_init(struct rte_pci_device *dev, struct virtio_hw *hw, */ if (virtio_read_caps(dev, hw) == 0) { PMD_INIT_LOG(INFO, "modern virtio pci detected."); - hw->vtpci_ops = &modern_ops; - hw->modern = 1; - *dev_flags |= RTE_ETH_DEV_INTR_LSC; + virtio_hw_internal[hw->port_id].vtpci_ops = &modern_ops; + hw->modern = 1; return 0; } PMD_INIT_LOG(INFO, "trying with legacy virtio pci."); - if (legacy_virtio_resource_init(dev, hw, dev_flags) < 0) { + if (rte_pci_ioport_map(dev, 0, VTPCI_IO(hw)) < 0) { if (dev->kdrv == RTE_KDRV_UNKNOWN && - (!dev->devargs || - dev->devargs->type != RTE_DEVTYPE_WHITELISTED_PCI)) { + (!dev->device.devargs || + dev->device.devargs->bus != + rte_bus_find_by_name("pci"))) { PMD_INIT_LOG(INFO, "skip kernel managed virtio device."); return 1; @@ -754,9 +708,44 @@ vtpci_init(struct rte_pci_device *dev, struct virtio_hw *hw, return -1; } - hw->vtpci_ops = &legacy_ops; - hw->use_msix = legacy_virtio_has_msix(&dev->addr); + virtio_hw_internal[hw->port_id].vtpci_ops = &legacy_ops; hw->modern = 0; return 0; } + +enum virtio_msix_status +vtpci_msix_detect(struct rte_pci_device *dev) +{ + uint8_t pos; + struct virtio_pci_cap cap; + int ret; + + ret = rte_pci_read_config(dev, &pos, 1, PCI_CAPABILITY_LIST); + if (ret < 0) { + PMD_INIT_LOG(DEBUG, "failed to read pci capability list"); + return VIRTIO_MSIX_NONE; + } + + while (pos) { + ret = rte_pci_read_config(dev, &cap, sizeof(cap), pos); + if (ret < 0) { + PMD_INIT_LOG(ERR, + "failed to read pci cap at pos: %x", pos); + break; + } + + if (cap.cap_vndr == PCI_CAP_ID_MSIX) { + uint16_t flags = ((uint16_t *)&cap)[1]; + + if (flags & PCI_MSIX_ENABLE) + return VIRTIO_MSIX_ENABLED; + else + return VIRTIO_MSIX_DISABLED; + } + + pos = cap.cap_next; + } + + return VIRTIO_MSIX_NONE; +} diff --git a/dpdk/drivers/net/virtio/virtio_pci.h b/dpdk/drivers/net/virtio/virtio_pci.h index dd7693fe..3c5ce66c 100644 --- a/dpdk/drivers/net/virtio/virtio_pci.h +++ b/dpdk/drivers/net/virtio/virtio_pci.h @@ -37,6 +37,7 @@ #include #include +#include #include struct virtqueue; @@ -44,8 +45,8 @@ struct virtnet_ctl; /* VirtIO PCI vendor/device ID. */ #define VIRTIO_PCI_VENDORID 0x1AF4 -#define VIRTIO_PCI_DEVICEID_MIN 0x1000 -#define VIRTIO_PCI_DEVICEID_MAX 0x103F +#define VIRTIO_PCI_LEGACY_DEVICEID_NET 0x1000 +#define VIRTIO_PCI_MODERN_DEVICEID_NET 0x1041 /* VirtIO ABI version, this must match exactly. */ #define VIRTIO_PCI_ABI_VERSION 0 @@ -106,6 +107,7 @@ struct virtnet_ctl; /* The feature bitmap for virtio net */ #define VIRTIO_NET_F_CSUM 0 /* Host handles pkts w/ partial csum */ #define VIRTIO_NET_F_GUEST_CSUM 1 /* Guest handles pkts w/ partial csum */ +#define VIRTIO_NET_F_MTU 3 /* Initial MTU advice. */ #define VIRTIO_NET_F_MAC 5 /* Host has given MAC address. */ #define VIRTIO_NET_F_GUEST_TSO4 7 /* Guest can handle TSOv4 in. */ #define VIRTIO_NET_F_GUEST_TSO6 8 /* Guest can handle TSOv6 in. */ @@ -138,6 +140,7 @@ struct virtnet_ctl; #define VIRTIO_RING_F_INDIRECT_DESC 28 #define VIRTIO_F_VERSION_1 32 +#define VIRTIO_F_IOMMU_PLATFORM 33 /* * Some VirtIO feature bits (currently bits 28 through 31) are @@ -145,7 +148,7 @@ struct virtnet_ctl; * rest are per-device feature bits. */ #define VIRTIO_TRANSPORT_F_START 28 -#define VIRTIO_TRANSPORT_F_END 32 +#define VIRTIO_TRANSPORT_F_END 34 /* The Guest publishes the used index for which it expects an interrupt * at the end of the avail ring. Host should ignore the avail->flags field. */ @@ -159,7 +162,8 @@ struct virtnet_ctl; /* * Maximum number of virtqueues per device. */ -#define VIRTIO_MAX_VIRTQUEUES 8 +#define VIRTIO_MAX_VIRTQUEUE_PAIRS 8 +#define VIRTIO_MAX_VIRTQUEUES (VIRTIO_MAX_VIRTQUEUE_PAIRS * 2 + 1) /* Common configuration */ #define VIRTIO_PCI_CAP_COMMON_CFG 1 @@ -234,6 +238,9 @@ struct virtio_pci_ops { uint16_t (*set_config_irq)(struct virtio_hw *hw, uint16_t vec); + uint16_t (*set_queue_irq)(struct virtio_hw *hw, struct virtqueue *vq, + uint16_t vec); + uint16_t (*get_queue_num)(struct virtio_hw *hw, uint16_t queue_id); int (*setup_queue)(struct virtio_hw *hw, struct virtqueue *vq); void (*del_queue)(struct virtio_hw *hw, struct virtqueue *vq); @@ -244,26 +251,46 @@ struct virtio_net_config; struct virtio_hw { struct virtnet_ctl *cvq; - struct rte_pci_ioport io; + uint64_t req_guest_features; uint64_t guest_features; - uint32_t max_tx_queues; - uint32_t max_rx_queues; + uint32_t max_queue_pairs; + uint16_t started; + uint16_t max_mtu; uint16_t vtnet_hdr_size; uint8_t vlan_strip; uint8_t use_msix; - uint8_t started; uint8_t modern; + uint8_t use_simple_rx; + uint8_t use_simple_tx; + uint16_t port_id; uint8_t mac_addr[ETHER_ADDR_LEN]; uint32_t notify_off_multiplier; uint8_t *isr; uint16_t *notify_base; - struct rte_pci_device *dev; struct virtio_pci_common_cfg *common_cfg; struct virtio_net_config *dev_cfg; - const struct virtio_pci_ops *vtpci_ops; void *virtio_user_dev; + + struct virtqueue **vqs; }; + +/* + * While virtio_hw is stored in shared memory, this structure stores + * some infos that may vary in the multiple process model locally. + * For example, the vtpci_ops pointer. + */ +struct virtio_hw_internal { + const struct virtio_pci_ops *vtpci_ops; + struct rte_pci_ioport io; +}; + +#define VTPCI_OPS(hw) (virtio_hw_internal[(hw)->port_id].vtpci_ops) +#define VTPCI_IO(hw) (&virtio_hw_internal[(hw)->port_id].io) + +extern struct virtio_hw_internal virtio_hw_internal[RTE_MAX_ETHPORTS]; + + /* * This structure is just a reference to read * net device specific config space; it just a chodu structure @@ -275,6 +302,7 @@ struct virtio_net_config { /* See VIRTIO_NET_F_STATUS and VIRTIO_NET_S_* above */ uint16_t status; uint16_t max_virtqueue_pairs; + uint16_t mtu; } __attribute__((packed)); /* @@ -286,6 +314,12 @@ struct virtio_net_config { /* The alignment to use between consumer and producer parts of vring. */ #define VIRTIO_PCI_VRING_ALIGN 4096 +enum virtio_msix_status { + VIRTIO_MSIX_NONE = 0, + VIRTIO_MSIX_DISABLED = 1, + VIRTIO_MSIX_ENABLED = 2 +}; + static inline int vtpci_with_feature(struct virtio_hw *hw, uint64_t bit) { @@ -295,8 +329,7 @@ vtpci_with_feature(struct virtio_hw *hw, uint64_t bit) /* * Function declaration from virtio_pci.c */ -int vtpci_init(struct rte_pci_device *, struct virtio_hw *, - uint32_t *dev_flags); +int vtpci_init(struct rte_pci_device *dev, struct virtio_hw *hw); void vtpci_reset(struct virtio_hw *); void vtpci_reinit_complete(struct virtio_hw *); @@ -312,6 +345,10 @@ void vtpci_read_dev_config(struct virtio_hw *, size_t, void *, int); uint8_t vtpci_isr(struct virtio_hw *); -uint16_t vtpci_irq_config(struct virtio_hw *, uint16_t); +enum virtio_msix_status vtpci_msix_detect(struct rte_pci_device *dev); + +extern const struct virtio_pci_ops legacy_ops; +extern const struct virtio_pci_ops modern_ops; +extern const struct virtio_pci_ops virtio_user_ops; #endif /* _VIRTIO_PCI_H_ */ diff --git a/dpdk/drivers/net/virtio/virtio_rxtx.c b/dpdk/drivers/net/virtio/virtio_rxtx.c index 724517e2..390c137c 100644 --- a/dpdk/drivers/net/virtio/virtio_rxtx.c +++ b/dpdk/drivers/net/virtio/virtio_rxtx.c @@ -39,7 +39,6 @@ #include #include -#include #include #include #include @@ -50,6 +49,10 @@ #include #include #include +#include +#include +#include +#include #include "virtio_logs.h" #include "virtio_ethdev.h" @@ -67,11 +70,16 @@ #define VIRTIO_SIMPLE_FLAGS ((uint32_t)ETH_TXQ_FLAGS_NOMULTSEGS | \ ETH_TXQ_FLAGS_NOOFFLOADS) -#ifdef RTE_MACHINE_CPUFLAG_SSSE3 -static int use_simple_rxtx; -#endif +int +virtio_dev_rx_queue_done(void *rxq, uint16_t offset) +{ + struct virtnet_rx *rxvq = rxq; + struct virtqueue *vq = rxvq->vq; -static void + return VIRTQUEUE_NUSED(vq) >= offset; +} + +void vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx) { struct vring_desc *dp, *dp_tail; @@ -123,7 +131,7 @@ virtqueue_dequeue_burst_rx(struct virtqueue *vq, struct rte_mbuf **rx_pkts, cookie = (struct rte_mbuf *)vq->vq_descx[desc_idx].cookie; if (unlikely(cookie == NULL)) { - PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u\n", + PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u", vq->vq_used_cons_idx); break; } @@ -208,18 +216,76 @@ virtqueue_enqueue_recv_refill(struct virtqueue *vq, struct rte_mbuf *cookie) return 0; } +/* When doing TSO, the IP length is not included in the pseudo header + * checksum of the packet given to the PMD, but for virtio it is + * expected. + */ +static void +virtio_tso_fix_cksum(struct rte_mbuf *m) +{ + /* common case: header is not fragmented */ + if (likely(rte_pktmbuf_data_len(m) >= m->l2_len + m->l3_len + + m->l4_len)) { + struct ipv4_hdr *iph; + struct ipv6_hdr *ip6h; + struct tcp_hdr *th; + uint16_t prev_cksum, new_cksum, ip_len, ip_paylen; + uint32_t tmp; + + iph = rte_pktmbuf_mtod_offset(m, struct ipv4_hdr *, m->l2_len); + th = RTE_PTR_ADD(iph, m->l3_len); + if ((iph->version_ihl >> 4) == 4) { + iph->hdr_checksum = 0; + iph->hdr_checksum = rte_ipv4_cksum(iph); + ip_len = iph->total_length; + ip_paylen = rte_cpu_to_be_16(rte_be_to_cpu_16(ip_len) - + m->l3_len); + } else { + ip6h = (struct ipv6_hdr *)iph; + ip_paylen = ip6h->payload_len; + } + + /* calculate the new phdr checksum not including ip_paylen */ + prev_cksum = th->cksum; + tmp = prev_cksum; + tmp += ip_paylen; + tmp = (tmp & 0xffff) + (tmp >> 16); + new_cksum = tmp; + + /* replace it in the packet */ + th->cksum = new_cksum; + } +} + +static inline int +tx_offload_enabled(struct virtio_hw *hw) +{ + return vtpci_with_feature(hw, VIRTIO_NET_F_CSUM) || + vtpci_with_feature(hw, VIRTIO_NET_F_HOST_TSO4) || + vtpci_with_feature(hw, VIRTIO_NET_F_HOST_TSO6); +} + +/* avoid write operation when necessary, to lessen cache issues */ +#define ASSIGN_UNLESS_EQUAL(var, val) do { \ + if ((var) != (val)) \ + (var) = (val); \ +} while (0) + static inline void virtqueue_enqueue_xmit(struct virtnet_tx *txvq, struct rte_mbuf *cookie, uint16_t needed, int use_indirect, int can_push) { + struct virtio_tx_region *txr = txvq->virtio_net_hdr_mz->addr; struct vq_desc_extra *dxp; struct virtqueue *vq = txvq->vq; struct vring_desc *start_dp; uint16_t seg_num = cookie->nb_segs; uint16_t head_idx, idx; uint16_t head_size = vq->hw->vtnet_hdr_size; - unsigned long offs; + struct virtio_net_hdr *hdr; + int offload; + offload = tx_offload_enabled(vq->hw); head_idx = vq->vq_desc_head_idx; idx = head_idx; dxp = &vq->vq_descx[idx]; @@ -229,10 +295,22 @@ virtqueue_enqueue_xmit(struct virtnet_tx *txvq, struct rte_mbuf *cookie, start_dp = vq->vq_ring.desc; if (can_push) { - /* put on zero'd transmit header (no offloads) */ - void *hdr = rte_pktmbuf_prepend(cookie, head_size); - - memset(hdr, 0, head_size); + /* prepend cannot fail, checked by caller */ + hdr = (struct virtio_net_hdr *) + rte_pktmbuf_prepend(cookie, head_size); + /* rte_pktmbuf_prepend() counts the hdr size to the pkt length, + * which is wrong. Below subtract restores correct pkt size. + */ + cookie->pkt_len -= head_size; + /* if offload disabled, it is not zeroed below, do it now */ + if (offload == 0) { + ASSIGN_UNLESS_EQUAL(hdr->csum_start, 0); + ASSIGN_UNLESS_EQUAL(hdr->csum_offset, 0); + ASSIGN_UNLESS_EQUAL(hdr->flags, 0); + ASSIGN_UNLESS_EQUAL(hdr->gso_type, 0); + ASSIGN_UNLESS_EQUAL(hdr->gso_size, 0); + ASSIGN_UNLESS_EQUAL(hdr->hdr_len, 0); + } } else if (use_indirect) { /* setup tx ring slot to point to indirect * descriptor list stored in reserved region. @@ -240,14 +318,11 @@ virtqueue_enqueue_xmit(struct virtnet_tx *txvq, struct rte_mbuf *cookie, * the first slot in indirect ring is already preset * to point to the header in reserved region */ - struct virtio_tx_region *txr = txvq->virtio_net_hdr_mz->addr; - - offs = idx * sizeof(struct virtio_tx_region) - + offsetof(struct virtio_tx_region, tx_indir); - - start_dp[idx].addr = txvq->virtio_net_hdr_mem + offs; + start_dp[idx].addr = txvq->virtio_net_hdr_mem + + RTE_PTR_DIFF(&txr[idx].tx_indir, txr); start_dp[idx].len = (seg_num + 1) * sizeof(struct vring_desc); start_dp[idx].flags = VRING_DESC_F_INDIRECT; + hdr = (struct virtio_net_hdr *)&txr[idx].tx_hdr; /* loop below will fill in rest of the indirect elements */ start_dp = txr[idx].tx_indir; @@ -256,15 +331,59 @@ virtqueue_enqueue_xmit(struct virtnet_tx *txvq, struct rte_mbuf *cookie, /* setup first tx ring slot to point to header * stored in reserved region. */ - offs = idx * sizeof(struct virtio_tx_region) - + offsetof(struct virtio_tx_region, tx_hdr); - - start_dp[idx].addr = txvq->virtio_net_hdr_mem + offs; + start_dp[idx].addr = txvq->virtio_net_hdr_mem + + RTE_PTR_DIFF(&txr[idx].tx_hdr, txr); start_dp[idx].len = vq->hw->vtnet_hdr_size; start_dp[idx].flags = VRING_DESC_F_NEXT; + hdr = (struct virtio_net_hdr *)&txr[idx].tx_hdr; + idx = start_dp[idx].next; } + /* Checksum Offload / TSO */ + if (offload) { + if (cookie->ol_flags & PKT_TX_TCP_SEG) + cookie->ol_flags |= PKT_TX_TCP_CKSUM; + + switch (cookie->ol_flags & PKT_TX_L4_MASK) { + case PKT_TX_UDP_CKSUM: + hdr->csum_start = cookie->l2_len + cookie->l3_len; + hdr->csum_offset = offsetof(struct udp_hdr, + dgram_cksum); + hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM; + break; + + case PKT_TX_TCP_CKSUM: + hdr->csum_start = cookie->l2_len + cookie->l3_len; + hdr->csum_offset = offsetof(struct tcp_hdr, cksum); + hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM; + break; + + default: + ASSIGN_UNLESS_EQUAL(hdr->csum_start, 0); + ASSIGN_UNLESS_EQUAL(hdr->csum_offset, 0); + ASSIGN_UNLESS_EQUAL(hdr->flags, 0); + break; + } + + /* TCP Segmentation Offload */ + if (cookie->ol_flags & PKT_TX_TCP_SEG) { + virtio_tso_fix_cksum(cookie); + hdr->gso_type = (cookie->ol_flags & PKT_TX_IPV6) ? + VIRTIO_NET_HDR_GSO_TCPV6 : + VIRTIO_NET_HDR_GSO_TCPV4; + hdr->gso_size = cookie->tso_segsz; + hdr->hdr_len = + cookie->l2_len + + cookie->l3_len + + cookie->l4_len; + } else { + ASSIGN_UNLESS_EQUAL(hdr->gso_type, 0); + ASSIGN_UNLESS_EQUAL(hdr->gso_size, 0); + ASSIGN_UNLESS_EQUAL(hdr->hdr_len, 0); + } + } + do { start_dp[idx].addr = VIRTIO_MBUF_DATA_DMA_ADDR(cookie, vq); start_dp[idx].len = cookie->data_len; @@ -282,207 +401,106 @@ virtqueue_enqueue_xmit(struct virtnet_tx *txvq, struct rte_mbuf *cookie, vq_update_avail_ring(vq, head_idx); } -static void -virtio_dev_vring_start(struct virtqueue *vq) -{ - int size = vq->vq_nentries; - struct vring *vr = &vq->vq_ring; - uint8_t *ring_mem = vq->vq_ring_virt_mem; - - PMD_INIT_FUNC_TRACE(); - - /* - * Reinitialise since virtio port might have been stopped and restarted - */ - memset(vq->vq_ring_virt_mem, 0, vq->vq_ring_size); - vring_init(vr, size, ring_mem, VIRTIO_PCI_VRING_ALIGN); - vq->vq_used_cons_idx = 0; - vq->vq_desc_head_idx = 0; - vq->vq_avail_idx = 0; - vq->vq_desc_tail_idx = (uint16_t)(vq->vq_nentries - 1); - vq->vq_free_cnt = vq->vq_nentries; - memset(vq->vq_descx, 0, sizeof(struct vq_desc_extra) * vq->vq_nentries); - - vring_desc_init(vr->desc, size); - - /* - * Disable device(host) interrupting guest - */ - virtqueue_disable_intr(vq); -} - void virtio_dev_cq_start(struct rte_eth_dev *dev) { struct virtio_hw *hw = dev->data->dev_private; if (hw->cvq && hw->cvq->vq) { - virtio_dev_vring_start(hw->cvq->vq); VIRTQUEUE_DUMP((struct virtqueue *)hw->cvq->vq); } } -void -virtio_dev_rxtx_start(struct rte_eth_dev *dev) -{ - /* - * Start receive and transmit vrings - * - Setup vring structure for all queues - * - Initialize descriptor for the rx vring - * - Allocate blank mbufs for the each rx descriptor - * - */ - uint16_t i; - uint16_t desc_idx; - - PMD_INIT_FUNC_TRACE(); - - /* Start rx vring. */ - for (i = 0; i < dev->data->nb_rx_queues; i++) { - struct virtnet_rx *rxvq = dev->data->rx_queues[i]; - struct virtqueue *vq = rxvq->vq; - int error, nbufs; - struct rte_mbuf *m; - - virtio_dev_vring_start(vq); - if (rxvq->mpool == NULL) { - rte_exit(EXIT_FAILURE, - "Cannot allocate mbufs for rx virtqueue"); - } - - /* Allocate blank mbufs for the each rx descriptor */ - nbufs = 0; - error = ENOSPC; - -#ifdef RTE_MACHINE_CPUFLAG_SSSE3 - if (use_simple_rxtx) { - for (desc_idx = 0; desc_idx < vq->vq_nentries; - desc_idx++) { - vq->vq_ring.avail->ring[desc_idx] = desc_idx; - vq->vq_ring.desc[desc_idx].flags = - VRING_DESC_F_WRITE; - } - } -#endif - memset(&rxvq->fake_mbuf, 0, sizeof(rxvq->fake_mbuf)); - for (desc_idx = 0; desc_idx < RTE_PMD_VIRTIO_RX_MAX_BURST; - desc_idx++) { - vq->sw_ring[vq->vq_nentries + desc_idx] = - &rxvq->fake_mbuf; - } - - while (!virtqueue_full(vq)) { - m = rte_mbuf_raw_alloc(rxvq->mpool); - if (m == NULL) - break; - - /****************************************** - * Enqueue allocated buffers * - *******************************************/ -#ifdef RTE_MACHINE_CPUFLAG_SSSE3 - if (use_simple_rxtx) - error = virtqueue_enqueue_recv_refill_simple(vq, m); - else -#endif - error = virtqueue_enqueue_recv_refill(vq, m); - if (error) { - rte_pktmbuf_free(m); - break; - } - nbufs++; - } - - vq_update_avail_idx(vq); - - PMD_INIT_LOG(DEBUG, "Allocated %d bufs", nbufs); - - VIRTQUEUE_DUMP(vq); - } - - /* Start tx vring. */ - for (i = 0; i < dev->data->nb_tx_queues; i++) { - struct virtnet_tx *txvq = dev->data->tx_queues[i]; - struct virtqueue *vq = txvq->vq; - - virtio_dev_vring_start(vq); -#ifdef RTE_MACHINE_CPUFLAG_SSSE3 - if (use_simple_rxtx) { - uint16_t mid_idx = vq->vq_nentries >> 1; - - for (desc_idx = 0; desc_idx < mid_idx; desc_idx++) { - vq->vq_ring.avail->ring[desc_idx] = - desc_idx + mid_idx; - vq->vq_ring.desc[desc_idx + mid_idx].next = - desc_idx; - vq->vq_ring.desc[desc_idx + mid_idx].addr = - txvq->virtio_net_hdr_mem + - offsetof(struct virtio_tx_region, tx_hdr); - vq->vq_ring.desc[desc_idx + mid_idx].len = - vq->hw->vtnet_hdr_size; - vq->vq_ring.desc[desc_idx + mid_idx].flags = - VRING_DESC_F_NEXT; - vq->vq_ring.desc[desc_idx].flags = 0; - } - for (desc_idx = mid_idx; desc_idx < vq->vq_nentries; - desc_idx++) - vq->vq_ring.avail->ring[desc_idx] = desc_idx; - } -#endif - VIRTQUEUE_DUMP(vq); - } -} - int virtio_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, uint16_t nb_desc, - unsigned int socket_id, + unsigned int socket_id __rte_unused, __rte_unused const struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp) { uint16_t vtpci_queue_idx = 2 * queue_idx + VTNET_SQ_RQ_QUEUE_IDX; + struct virtio_hw *hw = dev->data->dev_private; + struct virtqueue *vq = hw->vqs[vtpci_queue_idx]; struct virtnet_rx *rxvq; - int ret; PMD_INIT_FUNC_TRACE(); - ret = virtio_dev_queue_setup(dev, VTNET_RQ, queue_idx, vtpci_queue_idx, - nb_desc, socket_id, (void **)&rxvq); - if (ret < 0) { - PMD_INIT_LOG(ERR, "rvq initialization failed"); - return ret; - } - /* Create mempool for rx mbuf allocation */ + if (nb_desc == 0 || nb_desc > vq->vq_nentries) + nb_desc = vq->vq_nentries; + vq->vq_free_cnt = RTE_MIN(vq->vq_free_cnt, nb_desc); + + rxvq = &vq->rxq; + rxvq->queue_id = queue_idx; rxvq->mpool = mp; - + if (rxvq->mpool == NULL) { + rte_exit(EXIT_FAILURE, + "Cannot allocate mbufs for rx virtqueue"); + } dev->data->rx_queues[queue_idx] = rxvq; -#ifdef RTE_MACHINE_CPUFLAG_SSSE3 - virtio_rxq_vec_setup(rxvq); -#endif - return 0; } -void -virtio_dev_rx_queue_release(void *rxq) +int +virtio_dev_rx_queue_setup_finish(struct rte_eth_dev *dev, uint16_t queue_idx) { - struct virtnet_rx *rxvq = rxq; - struct virtqueue *vq; - const struct rte_memzone *mz; + uint16_t vtpci_queue_idx = 2 * queue_idx + VTNET_SQ_RQ_QUEUE_IDX; + struct virtio_hw *hw = dev->data->dev_private; + struct virtqueue *vq = hw->vqs[vtpci_queue_idx]; + struct virtnet_rx *rxvq = &vq->rxq; + struct rte_mbuf *m; + uint16_t desc_idx; + int error, nbufs; - if (rxvq == NULL) - return; + PMD_INIT_FUNC_TRACE(); - /* - * rxvq is freed when vq is freed, and as mz should be freed after the - * del_queue, so we reserve the mz pointer first. - */ - vq = rxvq->vq; - mz = rxvq->mz; + /* Allocate blank mbufs for the each rx descriptor */ + nbufs = 0; - virtio_dev_queue_release(vq); - rte_memzone_free(mz); + if (hw->use_simple_rx) { + for (desc_idx = 0; desc_idx < vq->vq_nentries; + desc_idx++) { + vq->vq_ring.avail->ring[desc_idx] = desc_idx; + vq->vq_ring.desc[desc_idx].flags = + VRING_DESC_F_WRITE; + } + } + + memset(&rxvq->fake_mbuf, 0, sizeof(rxvq->fake_mbuf)); + for (desc_idx = 0; desc_idx < RTE_PMD_VIRTIO_RX_MAX_BURST; + desc_idx++) { + vq->sw_ring[vq->vq_nentries + desc_idx] = + &rxvq->fake_mbuf; + } + + while (!virtqueue_full(vq)) { + m = rte_mbuf_raw_alloc(rxvq->mpool); + if (m == NULL) + break; + + /* Enqueue allocated buffers */ + if (hw->use_simple_rx) + error = virtqueue_enqueue_recv_refill_simple(vq, m); + else + error = virtqueue_enqueue_recv_refill(vq, m); + + if (error) { + rte_pktmbuf_free(m); + break; + } + nbufs++; + } + + vq_update_avail_idx(vq); + + PMD_INIT_LOG(DEBUG, "Allocated %d bufs", nbufs); + + virtio_rxq_vec_setup(rxvq); + + VIRTQUEUE_DUMP(vq); + + return 0; } /* @@ -496,45 +514,27 @@ int virtio_dev_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, uint16_t nb_desc, - unsigned int socket_id, + unsigned int socket_id __rte_unused, const struct rte_eth_txconf *tx_conf) { uint8_t vtpci_queue_idx = 2 * queue_idx + VTNET_SQ_TQ_QUEUE_IDX; - -#ifdef RTE_MACHINE_CPUFLAG_SSSE3 struct virtio_hw *hw = dev->data->dev_private; -#endif + struct virtqueue *vq = hw->vqs[vtpci_queue_idx]; struct virtnet_tx *txvq; - struct virtqueue *vq; uint16_t tx_free_thresh; - int ret; PMD_INIT_FUNC_TRACE(); - if ((tx_conf->txq_flags & ETH_TXQ_FLAGS_NOXSUMS) - != ETH_TXQ_FLAGS_NOXSUMS) { - PMD_INIT_LOG(ERR, "TX checksum offload not supported\n"); - return -EINVAL; - } + /* cannot use simple rxtx funcs with multisegs or offloads */ + if ((tx_conf->txq_flags & VIRTIO_SIMPLE_FLAGS) != VIRTIO_SIMPLE_FLAGS) + hw->use_simple_tx = 0; -#ifdef RTE_MACHINE_CPUFLAG_SSSE3 - /* Use simple rx/tx func if single segment and no offloads */ - if ((tx_conf->txq_flags & VIRTIO_SIMPLE_FLAGS) == VIRTIO_SIMPLE_FLAGS && - !vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) { - PMD_INIT_LOG(INFO, "Using simple rx/tx path"); - dev->tx_pkt_burst = virtio_xmit_pkts_simple; - dev->rx_pkt_burst = virtio_recv_pkts_vec; - use_simple_rxtx = 1; - } -#endif + if (nb_desc == 0 || nb_desc > vq->vq_nentries) + nb_desc = vq->vq_nentries; + vq->vq_free_cnt = RTE_MIN(vq->vq_free_cnt, nb_desc); - ret = virtio_dev_queue_setup(dev, VTNET_TQ, queue_idx, vtpci_queue_idx, - nb_desc, socket_id, (void **)&txvq); - if (ret < 0) { - PMD_INIT_LOG(ERR, "tvq initialization failed"); - return ret; - } - vq = txvq->vq; + txvq = &vq->txq; + txvq->queue_id = queue_idx; tx_free_thresh = tx_conf->tx_free_thresh; if (tx_free_thresh == 0) @@ -556,28 +556,42 @@ virtio_dev_tx_queue_setup(struct rte_eth_dev *dev, return 0; } -void -virtio_dev_tx_queue_release(void *txq) +int +virtio_dev_tx_queue_setup_finish(struct rte_eth_dev *dev, + uint16_t queue_idx) { - struct virtnet_tx *txvq = txq; - struct virtqueue *vq; - const struct rte_memzone *mz; - const struct rte_memzone *hdr_mz; + uint8_t vtpci_queue_idx = 2 * queue_idx + VTNET_SQ_TQ_QUEUE_IDX; + struct virtio_hw *hw = dev->data->dev_private; + struct virtqueue *vq = hw->vqs[vtpci_queue_idx]; + uint16_t mid_idx = vq->vq_nentries >> 1; + struct virtnet_tx *txvq = &vq->txq; + uint16_t desc_idx; - if (txvq == NULL) - return; + PMD_INIT_FUNC_TRACE(); - /* - * txvq is freed when vq is freed, and as mz should be freed after the - * del_queue, so we reserve the mz pointer first. - */ - vq = txvq->vq; - mz = txvq->mz; - hdr_mz = txvq->virtio_net_hdr_mz; + if (hw->use_simple_tx) { + for (desc_idx = 0; desc_idx < mid_idx; desc_idx++) { + vq->vq_ring.avail->ring[desc_idx] = + desc_idx + mid_idx; + vq->vq_ring.desc[desc_idx + mid_idx].next = + desc_idx; + vq->vq_ring.desc[desc_idx + mid_idx].addr = + txvq->virtio_net_hdr_mem + + offsetof(struct virtio_tx_region, tx_hdr); + vq->vq_ring.desc[desc_idx + mid_idx].len = + vq->hw->vtnet_hdr_size; + vq->vq_ring.desc[desc_idx + mid_idx].flags = + VRING_DESC_F_NEXT; + vq->vq_ring.desc[desc_idx].flags = 0; + } + for (desc_idx = mid_idx; desc_idx < vq->vq_nentries; + desc_idx++) + vq->vq_ring.avail->ring[desc_idx] = desc_idx; + } - virtio_dev_queue_release(vq); - rte_memzone_free(mz); - rte_memzone_free(hdr_mz); + VIRTQUEUE_DUMP(vq); + + return 0; } static void @@ -627,6 +641,86 @@ virtio_update_packet_stats(struct virtnet_stats *stats, struct rte_mbuf *mbuf) } } +/* Optionally fill offload information in structure */ +static int +virtio_rx_offload(struct rte_mbuf *m, struct virtio_net_hdr *hdr) +{ + struct rte_net_hdr_lens hdr_lens; + uint32_t hdrlen, ptype; + int l4_supported = 0; + + /* nothing to do */ + if (hdr->flags == 0 && hdr->gso_type == VIRTIO_NET_HDR_GSO_NONE) + return 0; + + m->ol_flags |= PKT_RX_IP_CKSUM_UNKNOWN; + + ptype = rte_net_get_ptype(m, &hdr_lens, RTE_PTYPE_ALL_MASK); + m->packet_type = ptype; + if ((ptype & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_TCP || + (ptype & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_UDP || + (ptype & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_SCTP) + l4_supported = 1; + + if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) { + hdrlen = hdr_lens.l2_len + hdr_lens.l3_len + hdr_lens.l4_len; + if (hdr->csum_start <= hdrlen && l4_supported) { + m->ol_flags |= PKT_RX_L4_CKSUM_NONE; + } else { + /* Unknown proto or tunnel, do sw cksum. We can assume + * the cksum field is in the first segment since the + * buffers we provided to the host are large enough. + * In case of SCTP, this will be wrong since it's a CRC + * but there's nothing we can do. + */ + uint16_t csum = 0, off; + + rte_raw_cksum_mbuf(m, hdr->csum_start, + rte_pktmbuf_pkt_len(m) - hdr->csum_start, + &csum); + if (likely(csum != 0xffff)) + csum = ~csum; + off = hdr->csum_offset + hdr->csum_start; + if (rte_pktmbuf_data_len(m) >= off + 1) + *rte_pktmbuf_mtod_offset(m, uint16_t *, + off) = csum; + } + } else if (hdr->flags & VIRTIO_NET_HDR_F_DATA_VALID && l4_supported) { + m->ol_flags |= PKT_RX_L4_CKSUM_GOOD; + } + + /* GSO request, save required information in mbuf */ + if (hdr->gso_type != VIRTIO_NET_HDR_GSO_NONE) { + /* Check unsupported modes */ + if ((hdr->gso_type & VIRTIO_NET_HDR_GSO_ECN) || + (hdr->gso_size == 0)) { + return -EINVAL; + } + + /* Update mss lengthes in mbuf */ + m->tso_segsz = hdr->gso_size; + switch (hdr->gso_type & ~VIRTIO_NET_HDR_GSO_ECN) { + case VIRTIO_NET_HDR_GSO_TCPV4: + case VIRTIO_NET_HDR_GSO_TCPV6: + m->ol_flags |= PKT_RX_LRO | \ + PKT_RX_L4_CKSUM_NONE; + break; + default: + return -EINVAL; + } + } + + return 0; +} + +static inline int +rx_offload_enabled(struct virtio_hw *hw) +{ + return vtpci_with_feature(hw, VIRTIO_NET_F_GUEST_CSUM) || + vtpci_with_feature(hw, VIRTIO_NET_F_GUEST_TSO4) || + vtpci_with_feature(hw, VIRTIO_NET_F_GUEST_TSO6); +} + #define VIRTIO_MBUF_BURST_SZ 64 #define DESC_PER_CACHELINE (RTE_CACHE_LINE_SIZE / sizeof(struct vring_desc)) uint16_t @@ -634,7 +728,7 @@ virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) { struct virtnet_rx *rxvq = rx_queue; struct virtqueue *vq = rxvq->vq; - struct virtio_hw *hw; + struct virtio_hw *hw = vq->hw; struct rte_mbuf *rxm, *new_mbuf; uint16_t nb_used, num, nb_rx; uint32_t len[VIRTIO_MBUF_BURST_SZ]; @@ -642,23 +736,29 @@ virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) int error; uint32_t i, nb_enqueued; uint32_t hdr_size; + int offload; + struct virtio_net_hdr *hdr; + + nb_rx = 0; + if (unlikely(hw->started == 0)) + return nb_rx; nb_used = VIRTQUEUE_NUSED(vq); virtio_rmb(); - num = (uint16_t)(likely(nb_used <= nb_pkts) ? nb_used : nb_pkts); - num = (uint16_t)(likely(num <= VIRTIO_MBUF_BURST_SZ) ? num : VIRTIO_MBUF_BURST_SZ); + num = likely(nb_used <= nb_pkts) ? nb_used : nb_pkts; + if (unlikely(num > VIRTIO_MBUF_BURST_SZ)) + num = VIRTIO_MBUF_BURST_SZ; if (likely(num > DESC_PER_CACHELINE)) num = num - ((vq->vq_used_cons_idx + num) % DESC_PER_CACHELINE); num = virtqueue_dequeue_burst_rx(vq, rcv_pkts, len, num); PMD_RX_LOG(DEBUG, "used:%d dequeue:%d", nb_used, num); - hw = vq->hw; - nb_rx = 0; nb_enqueued = 0; hdr_size = hw->vtnet_hdr_size; + offload = rx_offload_enabled(hw); for (i = 0; i < num ; i++) { rxm = rcv_pkts[i]; @@ -678,19 +778,26 @@ virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) rxm->ol_flags = 0; rxm->vlan_tci = 0; - rxm->nb_segs = 1; - rxm->next = NULL; rxm->pkt_len = (uint32_t)(len[i] - hdr_size); rxm->data_len = (uint16_t)(len[i] - hdr_size); + hdr = (struct virtio_net_hdr *)((char *)rxm->buf_addr + + RTE_PKTMBUF_HEADROOM - hdr_size); + if (hw->vlan_strip) rte_vlan_strip(rxm); + if (offload && virtio_rx_offload(rxm, hdr) < 0) { + virtio_discard_rxbuf(vq, rxm); + rxvq->stats.errors++; + continue; + } + VIRTIO_DUMP_PACKET(rxm, rxm->data_len); rx_pkts[nb_rx++] = rxm; - rxvq->stats.bytes += rx_pkts[nb_rx - 1]->pkt_len; + rxvq->stats.bytes += rxm->pkt_len; virtio_update_packet_stats(&rxvq->stats, rxm); } @@ -733,7 +840,7 @@ virtio_recv_mergeable_pkts(void *rx_queue, { struct virtnet_rx *rxvq = rx_queue; struct virtqueue *vq = rxvq->vq; - struct virtio_hw *hw; + struct virtio_hw *hw = vq->hw; struct rte_mbuf *rxm, *new_mbuf; uint16_t nb_used, num, nb_rx; uint32_t len[VIRTIO_MBUF_BURST_SZ]; @@ -745,6 +852,11 @@ virtio_recv_mergeable_pkts(void *rx_queue, uint16_t extra_idx; uint32_t seg_res; uint32_t hdr_size; + int offload; + + nb_rx = 0; + if (unlikely(hw->started == 0)) + return nb_rx; nb_used = VIRTQUEUE_NUSED(vq); @@ -752,14 +864,13 @@ virtio_recv_mergeable_pkts(void *rx_queue, PMD_RX_LOG(DEBUG, "used:%d", nb_used); - hw = vq->hw; - nb_rx = 0; i = 0; nb_enqueued = 0; seg_num = 0; extra_idx = 0; seg_res = 0; hdr_size = hw->vtnet_hdr_size; + offload = rx_offload_enabled(hw); while (i < nb_used) { struct virtio_net_hdr_mrg_rxbuf *header; @@ -795,7 +906,6 @@ virtio_recv_mergeable_pkts(void *rx_queue, rxm->data_off = RTE_PKTMBUF_HEADROOM; rxm->nb_segs = seg_num; - rxm->next = NULL; rxm->ol_flags = 0; rxm->vlan_tci = 0; rxm->pkt_len = (uint32_t)(len[0] - hdr_size); @@ -805,6 +915,12 @@ virtio_recv_mergeable_pkts(void *rx_queue, rx_pkts[nb_rx] = rxm; prev = rxm; + if (offload && virtio_rx_offload(rxm, &header->hdr) < 0) { + virtio_discard_rxbuf(vq, rxm); + rxvq->stats.errors++; + continue; + } + seg_res = seg_num - 1; while (seg_res != 0) { @@ -834,7 +950,6 @@ virtio_recv_mergeable_pkts(void *rx_queue, rxm = rcv_pkts[extra_idx]; rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size; - rxm->next = NULL; rxm->pkt_len = (uint32_t)(len[extra_idx]); rxm->data_len = (uint16_t)(len[extra_idx]); @@ -898,9 +1013,12 @@ virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) struct virtqueue *vq = txvq->vq; struct virtio_hw *hw = vq->hw; uint16_t hdr_size = hw->vtnet_hdr_size; - uint16_t nb_used, nb_tx; + uint16_t nb_used, nb_tx = 0; int error; + if (unlikely(hw->started == 0)) + return nb_tx; + if (unlikely(nb_pkts < 1)) return nb_pkts; @@ -925,7 +1043,8 @@ virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) } /* optimize ring usage */ - if (vtpci_with_feature(hw, VIRTIO_F_ANY_LAYOUT) && + if ((vtpci_with_feature(hw, VIRTIO_F_ANY_LAYOUT) || + vtpci_with_feature(hw, VIRTIO_F_VERSION_1)) && rte_mbuf_refcnt_read(txm) == 1 && RTE_MBUF_DIRECT(txm) && txm->nb_segs == 1 && diff --git a/dpdk/drivers/net/virtio/virtio_rxtx.h b/dpdk/drivers/net/virtio/virtio_rxtx.h index 058b56a1..54f1e849 100644 --- a/dpdk/drivers/net/virtio/virtio_rxtx.h +++ b/dpdk/drivers/net/virtio/virtio_rxtx.h @@ -54,7 +54,7 @@ struct virtnet_rx { struct rte_mempool *mpool; /**< mempool for mbuf allocation */ uint16_t queue_id; /**< DPDK queue index. */ - uint8_t port_id; /**< Device port identifier. */ + uint16_t port_id; /**< Device port identifier. */ /* Statistics */ struct virtnet_stats stats; @@ -66,10 +66,10 @@ struct virtnet_tx { struct virtqueue *vq; /**< memzone to populate hdr. */ const struct rte_memzone *virtio_net_hdr_mz; - phys_addr_t virtio_net_hdr_mem; /**< hdr for each xmit packet */ + rte_iova_t virtio_net_hdr_mem; /**< hdr for each xmit packet */ uint16_t queue_id; /**< DPDK queue index. */ - uint8_t port_id; /**< Device port identifier. */ + uint16_t port_id; /**< Device port identifier. */ /* Statistics */ struct virtnet_stats stats; @@ -81,15 +81,14 @@ struct virtnet_ctl { struct virtqueue *vq; /**< memzone to populate hdr. */ const struct rte_memzone *virtio_net_hdr_mz; - phys_addr_t virtio_net_hdr_mem; /**< hdr for each xmit packet */ - uint8_t port_id; /**< Device port identifier. */ - const struct rte_memzone *mz; /**< mem zone to populate RX ring. */ + rte_iova_t virtio_net_hdr_mem; /**< hdr for each xmit packet */ + uint16_t port_id; /**< Device port identifier. */ + const struct rte_memzone *mz; /**< mem zone to populate CTL ring. */ }; -#ifdef RTE_MACHINE_CPUFLAG_SSSE3 int virtio_rxq_vec_setup(struct virtnet_rx *rxvq); int virtqueue_enqueue_recv_refill_simple(struct virtqueue *vq, struct rte_mbuf *m); -#endif + #endif /* _VIRTIO_RXTX_H_ */ diff --git a/dpdk/drivers/net/virtio/virtio_rxtx_simple.c b/dpdk/drivers/net/virtio/virtio_rxtx_simple.c index 6517aa80..b5bc1c49 100644 --- a/dpdk/drivers/net/virtio/virtio_rxtx_simple.c +++ b/dpdk/drivers/net/virtio/virtio_rxtx_simple.c @@ -37,11 +37,8 @@ #include #include -#include - #include #include -#include #include #include #include @@ -53,14 +50,7 @@ #include #include -#include "virtio_logs.h" -#include "virtio_ethdev.h" -#include "virtqueue.h" -#include "virtio_rxtx.h" - -#define RTE_VIRTIO_VPMD_RX_BURST 32 -#define RTE_VIRTIO_DESC_PER_LOOP 8 -#define RTE_VIRTIO_VPMD_RX_REARM_THRESH RTE_VIRTIO_VPMD_RX_BURST +#include "virtio_rxtx_simple.h" #ifndef __INTEL_COMPILER #pragma GCC diagnostic ignored "-Wcast-qual" @@ -74,6 +64,8 @@ virtqueue_enqueue_recv_refill_simple(struct virtqueue *vq, struct vring_desc *start_dp; uint16_t desc_idx; + cookie->port = vq->rxq.port_id; + desc_idx = vq->vq_avail_idx & (vq->vq_nentries - 1); dxp = &vq->vq_descx[desc_idx]; dxp->cookie = (void *)cookie; @@ -92,269 +84,23 @@ virtqueue_enqueue_recv_refill_simple(struct virtqueue *vq, return 0; } -static inline void -virtio_rxq_rearm_vec(struct virtnet_rx *rxvq) -{ - int i; - uint16_t desc_idx; - struct rte_mbuf **sw_ring; - struct vring_desc *start_dp; - int ret; - struct virtqueue *vq = rxvq->vq; - - desc_idx = vq->vq_avail_idx & (vq->vq_nentries - 1); - sw_ring = &vq->sw_ring[desc_idx]; - start_dp = &vq->vq_ring.desc[desc_idx]; - - ret = rte_mempool_get_bulk(rxvq->mpool, (void **)sw_ring, - RTE_VIRTIO_VPMD_RX_REARM_THRESH); - if (unlikely(ret)) { - rte_eth_devices[rxvq->port_id].data->rx_mbuf_alloc_failed += - RTE_VIRTIO_VPMD_RX_REARM_THRESH; - return; - } - - for (i = 0; i < RTE_VIRTIO_VPMD_RX_REARM_THRESH; i++) { - uintptr_t p; - - p = (uintptr_t)&sw_ring[i]->rearm_data; - *(uint64_t *)p = rxvq->mbuf_initializer; - - start_dp[i].addr = - VIRTIO_MBUF_ADDR(sw_ring[i], vq) + - RTE_PKTMBUF_HEADROOM - vq->hw->vtnet_hdr_size; - start_dp[i].len = sw_ring[i]->buf_len - - RTE_PKTMBUF_HEADROOM + vq->hw->vtnet_hdr_size; - } - - vq->vq_avail_idx += RTE_VIRTIO_VPMD_RX_REARM_THRESH; - vq->vq_free_cnt -= RTE_VIRTIO_VPMD_RX_REARM_THRESH; - vq_update_avail_idx(vq); -} - -/* virtio vPMD receive routine, only accept(nb_pkts >= RTE_VIRTIO_DESC_PER_LOOP) - * - * This routine is for non-mergeable RX, one desc for each guest buffer. - * This routine is based on the RX ring layout optimization. Each entry in the - * avail ring points to the desc with the same index in the desc ring and this - * will never be changed in the driver. - * - * - nb_pkts < RTE_VIRTIO_DESC_PER_LOOP, just return no packet - */ -uint16_t -virtio_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, - uint16_t nb_pkts) -{ - struct virtnet_rx *rxvq = rx_queue; - struct virtqueue *vq = rxvq->vq; - uint16_t nb_used; - uint16_t desc_idx; - struct vring_used_elem *rused; - struct rte_mbuf **sw_ring; - struct rte_mbuf **sw_ring_end; - uint16_t nb_pkts_received; - __m128i shuf_msk1, shuf_msk2, len_adjust; - - shuf_msk1 = _mm_set_epi8( - 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, /* vlan tci */ - 5, 4, /* dat len */ - 0xFF, 0xFF, 5, 4, /* pkt len */ - 0xFF, 0xFF, 0xFF, 0xFF /* packet type */ - - ); - - shuf_msk2 = _mm_set_epi8( - 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, /* vlan tci */ - 13, 12, /* dat len */ - 0xFF, 0xFF, 13, 12, /* pkt len */ - 0xFF, 0xFF, 0xFF, 0xFF /* packet type */ - ); - - /* Subtract the header length. - * In which case do we need the header length in used->len ? - */ - len_adjust = _mm_set_epi16( - 0, 0, - 0, - (uint16_t)-vq->hw->vtnet_hdr_size, - 0, (uint16_t)-vq->hw->vtnet_hdr_size, - 0, 0); - - if (unlikely(nb_pkts < RTE_VIRTIO_DESC_PER_LOOP)) - return 0; - - nb_used = VIRTQUEUE_NUSED(vq); - - rte_compiler_barrier(); - - if (unlikely(nb_used == 0)) - return 0; - - nb_pkts = RTE_ALIGN_FLOOR(nb_pkts, RTE_VIRTIO_DESC_PER_LOOP); - nb_used = RTE_MIN(nb_used, nb_pkts); - - desc_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries - 1)); - rused = &vq->vq_ring.used->ring[desc_idx]; - sw_ring = &vq->sw_ring[desc_idx]; - sw_ring_end = &vq->sw_ring[vq->vq_nentries]; - - _mm_prefetch((const void *)rused, _MM_HINT_T0); - - if (vq->vq_free_cnt >= RTE_VIRTIO_VPMD_RX_REARM_THRESH) { - virtio_rxq_rearm_vec(rxvq); - if (unlikely(virtqueue_kick_prepare(vq))) - virtqueue_notify(vq); - } - - for (nb_pkts_received = 0; - nb_pkts_received < nb_used;) { - __m128i desc[RTE_VIRTIO_DESC_PER_LOOP / 2]; - __m128i mbp[RTE_VIRTIO_DESC_PER_LOOP / 2]; - __m128i pkt_mb[RTE_VIRTIO_DESC_PER_LOOP]; - - mbp[0] = _mm_loadu_si128((__m128i *)(sw_ring + 0)); - desc[0] = _mm_loadu_si128((__m128i *)(rused + 0)); - _mm_storeu_si128((__m128i *)&rx_pkts[0], mbp[0]); - - mbp[1] = _mm_loadu_si128((__m128i *)(sw_ring + 2)); - desc[1] = _mm_loadu_si128((__m128i *)(rused + 2)); - _mm_storeu_si128((__m128i *)&rx_pkts[2], mbp[1]); - - mbp[2] = _mm_loadu_si128((__m128i *)(sw_ring + 4)); - desc[2] = _mm_loadu_si128((__m128i *)(rused + 4)); - _mm_storeu_si128((__m128i *)&rx_pkts[4], mbp[2]); - - mbp[3] = _mm_loadu_si128((__m128i *)(sw_ring + 6)); - desc[3] = _mm_loadu_si128((__m128i *)(rused + 6)); - _mm_storeu_si128((__m128i *)&rx_pkts[6], mbp[3]); - - pkt_mb[1] = _mm_shuffle_epi8(desc[0], shuf_msk2); - pkt_mb[0] = _mm_shuffle_epi8(desc[0], shuf_msk1); - pkt_mb[1] = _mm_add_epi16(pkt_mb[1], len_adjust); - pkt_mb[0] = _mm_add_epi16(pkt_mb[0], len_adjust); - _mm_storeu_si128((void *)&rx_pkts[1]->rx_descriptor_fields1, - pkt_mb[1]); - _mm_storeu_si128((void *)&rx_pkts[0]->rx_descriptor_fields1, - pkt_mb[0]); - - pkt_mb[3] = _mm_shuffle_epi8(desc[1], shuf_msk2); - pkt_mb[2] = _mm_shuffle_epi8(desc[1], shuf_msk1); - pkt_mb[3] = _mm_add_epi16(pkt_mb[3], len_adjust); - pkt_mb[2] = _mm_add_epi16(pkt_mb[2], len_adjust); - _mm_storeu_si128((void *)&rx_pkts[3]->rx_descriptor_fields1, - pkt_mb[3]); - _mm_storeu_si128((void *)&rx_pkts[2]->rx_descriptor_fields1, - pkt_mb[2]); - - pkt_mb[5] = _mm_shuffle_epi8(desc[2], shuf_msk2); - pkt_mb[4] = _mm_shuffle_epi8(desc[2], shuf_msk1); - pkt_mb[5] = _mm_add_epi16(pkt_mb[5], len_adjust); - pkt_mb[4] = _mm_add_epi16(pkt_mb[4], len_adjust); - _mm_storeu_si128((void *)&rx_pkts[5]->rx_descriptor_fields1, - pkt_mb[5]); - _mm_storeu_si128((void *)&rx_pkts[4]->rx_descriptor_fields1, - pkt_mb[4]); - - pkt_mb[7] = _mm_shuffle_epi8(desc[3], shuf_msk2); - pkt_mb[6] = _mm_shuffle_epi8(desc[3], shuf_msk1); - pkt_mb[7] = _mm_add_epi16(pkt_mb[7], len_adjust); - pkt_mb[6] = _mm_add_epi16(pkt_mb[6], len_adjust); - _mm_storeu_si128((void *)&rx_pkts[7]->rx_descriptor_fields1, - pkt_mb[7]); - _mm_storeu_si128((void *)&rx_pkts[6]->rx_descriptor_fields1, - pkt_mb[6]); - - if (unlikely(nb_used <= RTE_VIRTIO_DESC_PER_LOOP)) { - if (sw_ring + nb_used <= sw_ring_end) - nb_pkts_received += nb_used; - else - nb_pkts_received += sw_ring_end - sw_ring; - break; - } else { - if (unlikely(sw_ring + RTE_VIRTIO_DESC_PER_LOOP >= - sw_ring_end)) { - nb_pkts_received += sw_ring_end - sw_ring; - break; - } else { - nb_pkts_received += RTE_VIRTIO_DESC_PER_LOOP; - - rx_pkts += RTE_VIRTIO_DESC_PER_LOOP; - sw_ring += RTE_VIRTIO_DESC_PER_LOOP; - rused += RTE_VIRTIO_DESC_PER_LOOP; - nb_used -= RTE_VIRTIO_DESC_PER_LOOP; - } - } - } - - vq->vq_used_cons_idx += nb_pkts_received; - vq->vq_free_cnt += nb_pkts_received; - rxvq->stats.packets += nb_pkts_received; - return nb_pkts_received; -} - -#define VIRTIO_TX_FREE_THRESH 32 -#define VIRTIO_TX_MAX_FREE_BUF_SZ 32 -#define VIRTIO_TX_FREE_NR 32 -/* TODO: vq->tx_free_cnt could mean num of free slots so we could avoid shift */ -static inline void -virtio_xmit_cleanup(struct virtqueue *vq) -{ - uint16_t i, desc_idx; - uint32_t nb_free = 0; - struct rte_mbuf *m, *free[VIRTIO_TX_MAX_FREE_BUF_SZ]; - - desc_idx = (uint16_t)(vq->vq_used_cons_idx & - ((vq->vq_nentries >> 1) - 1)); - m = (struct rte_mbuf *)vq->vq_descx[desc_idx++].cookie; - m = __rte_pktmbuf_prefree_seg(m); - if (likely(m != NULL)) { - free[0] = m; - nb_free = 1; - for (i = 1; i < VIRTIO_TX_FREE_NR; i++) { - m = (struct rte_mbuf *)vq->vq_descx[desc_idx++].cookie; - m = __rte_pktmbuf_prefree_seg(m); - if (likely(m != NULL)) { - if (likely(m->pool == free[0]->pool)) - free[nb_free++] = m; - else { - rte_mempool_put_bulk(free[0]->pool, - (void **)free, - RTE_MIN(RTE_DIM(free), - nb_free)); - free[0] = m; - nb_free = 1; - } - } - } - rte_mempool_put_bulk(free[0]->pool, (void **)free, - RTE_MIN(RTE_DIM(free), nb_free)); - } else { - for (i = 1; i < VIRTIO_TX_FREE_NR; i++) { - m = (struct rte_mbuf *)vq->vq_descx[desc_idx++].cookie; - m = __rte_pktmbuf_prefree_seg(m); - if (m != NULL) - rte_mempool_put(m->pool, m); - } - } - - vq->vq_used_cons_idx += VIRTIO_TX_FREE_NR; - vq->vq_free_cnt += (VIRTIO_TX_FREE_NR << 1); -} - uint16_t virtio_xmit_pkts_simple(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { struct virtnet_tx *txvq = tx_queue; struct virtqueue *vq = txvq->vq; + struct virtio_hw *hw = vq->hw; uint16_t nb_used; uint16_t desc_idx; struct vring_desc *start_dp; uint16_t nb_tail, nb_commit; int i; uint16_t desc_idx_max = (vq->vq_nentries >> 1) - 1; + uint16_t nb_tx = 0; + + if (unlikely(hw->started == 0)) + return nb_tx; nb_used = VIRTQUEUE_NUSED(vq); rte_compiler_barrier(); @@ -423,3 +169,13 @@ virtio_rxq_vec_setup(struct virtnet_rx *rxq) return 0; } + +/* Stub for linkage when arch specific implementation is not available */ +uint16_t __attribute__((weak)) +virtio_recv_pkts_vec(void *rx_queue __rte_unused, + struct rte_mbuf **rx_pkts __rte_unused, + uint16_t nb_pkts __rte_unused) +{ + rte_panic("Wrong weak function linked by linker\n"); + return 0; +} diff --git a/dpdk/drivers/net/virtio/virtio_rxtx_simple.h b/dpdk/drivers/net/virtio/virtio_rxtx_simple.h new file mode 100644 index 00000000..f531c542 --- /dev/null +++ b/dpdk/drivers/net/virtio/virtio_rxtx_simple.h @@ -0,0 +1,136 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _VIRTIO_RXTX_SIMPLE_H_ +#define _VIRTIO_RXTX_SIMPLE_H_ + +#include + +#include "virtio_logs.h" +#include "virtio_ethdev.h" +#include "virtqueue.h" +#include "virtio_rxtx.h" + +#define RTE_VIRTIO_VPMD_RX_BURST 32 +#define RTE_VIRTIO_VPMD_RX_REARM_THRESH RTE_VIRTIO_VPMD_RX_BURST + +static inline void +virtio_rxq_rearm_vec(struct virtnet_rx *rxvq) +{ + int i; + uint16_t desc_idx; + struct rte_mbuf **sw_ring; + struct vring_desc *start_dp; + int ret; + struct virtqueue *vq = rxvq->vq; + + desc_idx = vq->vq_avail_idx & (vq->vq_nentries - 1); + sw_ring = &vq->sw_ring[desc_idx]; + start_dp = &vq->vq_ring.desc[desc_idx]; + + ret = rte_mempool_get_bulk(rxvq->mpool, (void **)sw_ring, + RTE_VIRTIO_VPMD_RX_REARM_THRESH); + if (unlikely(ret)) { + rte_eth_devices[rxvq->port_id].data->rx_mbuf_alloc_failed += + RTE_VIRTIO_VPMD_RX_REARM_THRESH; + return; + } + + for (i = 0; i < RTE_VIRTIO_VPMD_RX_REARM_THRESH; i++) { + uintptr_t p; + + p = (uintptr_t)&sw_ring[i]->rearm_data; + *(uint64_t *)p = rxvq->mbuf_initializer; + + start_dp[i].addr = + VIRTIO_MBUF_ADDR(sw_ring[i], vq) + + RTE_PKTMBUF_HEADROOM - vq->hw->vtnet_hdr_size; + start_dp[i].len = sw_ring[i]->buf_len - + RTE_PKTMBUF_HEADROOM + vq->hw->vtnet_hdr_size; + } + + vq->vq_avail_idx += RTE_VIRTIO_VPMD_RX_REARM_THRESH; + vq->vq_free_cnt -= RTE_VIRTIO_VPMD_RX_REARM_THRESH; + vq_update_avail_idx(vq); +} + +#define VIRTIO_TX_FREE_THRESH 32 +#define VIRTIO_TX_MAX_FREE_BUF_SZ 32 +#define VIRTIO_TX_FREE_NR 32 +/* TODO: vq->tx_free_cnt could mean num of free slots so we could avoid shift */ +static inline void +virtio_xmit_cleanup(struct virtqueue *vq) +{ + uint16_t i, desc_idx; + uint32_t nb_free = 0; + struct rte_mbuf *m, *free[VIRTIO_TX_MAX_FREE_BUF_SZ]; + + desc_idx = (uint16_t)(vq->vq_used_cons_idx & + ((vq->vq_nentries >> 1) - 1)); + m = (struct rte_mbuf *)vq->vq_descx[desc_idx++].cookie; + m = rte_pktmbuf_prefree_seg(m); + if (likely(m != NULL)) { + free[0] = m; + nb_free = 1; + for (i = 1; i < VIRTIO_TX_FREE_NR; i++) { + m = (struct rte_mbuf *)vq->vq_descx[desc_idx++].cookie; + m = rte_pktmbuf_prefree_seg(m); + if (likely(m != NULL)) { + if (likely(m->pool == free[0]->pool)) + free[nb_free++] = m; + else { + rte_mempool_put_bulk(free[0]->pool, + (void **)free, + RTE_MIN(RTE_DIM(free), + nb_free)); + free[0] = m; + nb_free = 1; + } + } + } + rte_mempool_put_bulk(free[0]->pool, (void **)free, + RTE_MIN(RTE_DIM(free), nb_free)); + } else { + for (i = 1; i < VIRTIO_TX_FREE_NR; i++) { + m = (struct rte_mbuf *)vq->vq_descx[desc_idx++].cookie; + m = rte_pktmbuf_prefree_seg(m); + if (m != NULL) + rte_mempool_put(m->pool, m); + } + } + + vq->vq_used_cons_idx += VIRTIO_TX_FREE_NR; + vq->vq_free_cnt += (VIRTIO_TX_FREE_NR << 1); +} + +#endif /* _VIRTIO_RXTX_SIMPLE_H_ */ diff --git a/dpdk/drivers/net/virtio/virtio_rxtx_simple_neon.c b/dpdk/drivers/net/virtio/virtio_rxtx_simple_neon.c new file mode 100644 index 00000000..b8b93551 --- /dev/null +++ b/dpdk/drivers/net/virtio/virtio_rxtx_simple_neon.c @@ -0,0 +1,238 @@ +/* + * BSD LICENSE + * + * Copyright (C) Cavium, Inc. 2016 + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "virtio_rxtx_simple.h" + +#define RTE_VIRTIO_VPMD_RX_BURST 32 +#define RTE_VIRTIO_DESC_PER_LOOP 8 +#define RTE_VIRTIO_VPMD_RX_REARM_THRESH RTE_VIRTIO_VPMD_RX_BURST + +/* virtio vPMD receive routine, only accept(nb_pkts >= RTE_VIRTIO_DESC_PER_LOOP) + * + * This routine is for non-mergeable RX, one desc for each guest buffer. + * This routine is based on the RX ring layout optimization. Each entry in the + * avail ring points to the desc with the same index in the desc ring and this + * will never be changed in the driver. + * + * - nb_pkts < RTE_VIRTIO_DESC_PER_LOOP, just return no packet + */ +uint16_t +virtio_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts) +{ + struct virtnet_rx *rxvq = rx_queue; + struct virtqueue *vq = rxvq->vq; + struct virtio_hw *hw = vq->hw; + uint16_t nb_used; + uint16_t desc_idx; + struct vring_used_elem *rused; + struct rte_mbuf **sw_ring; + struct rte_mbuf **sw_ring_end; + uint16_t nb_pkts_received = 0; + + uint8x16_t shuf_msk1 = { + 0xFF, 0xFF, 0xFF, 0xFF, /* packet type */ + 4, 5, 0xFF, 0xFF, /* pkt len */ + 4, 5, /* dat len */ + 0xFF, 0xFF, /* vlan tci */ + 0xFF, 0xFF, 0xFF, 0xFF + }; + + uint8x16_t shuf_msk2 = { + 0xFF, 0xFF, 0xFF, 0xFF, /* packet type */ + 12, 13, 0xFF, 0xFF, /* pkt len */ + 12, 13, /* dat len */ + 0xFF, 0xFF, /* vlan tci */ + 0xFF, 0xFF, 0xFF, 0xFF + }; + + /* Subtract the header length. + * In which case do we need the header length in used->len ? + */ + uint16x8_t len_adjust = { + 0, 0, + (uint16_t)vq->hw->vtnet_hdr_size, 0, + (uint16_t)vq->hw->vtnet_hdr_size, + 0, + 0, 0 + }; + + if (unlikely(hw->started == 0)) + return nb_pkts_received; + + if (unlikely(nb_pkts < RTE_VIRTIO_DESC_PER_LOOP)) + return 0; + + nb_used = VIRTQUEUE_NUSED(vq); + + rte_rmb(); + + if (unlikely(nb_used == 0)) + return 0; + + nb_pkts = RTE_ALIGN_FLOOR(nb_pkts, RTE_VIRTIO_DESC_PER_LOOP); + nb_used = RTE_MIN(nb_used, nb_pkts); + + desc_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries - 1)); + rused = &vq->vq_ring.used->ring[desc_idx]; + sw_ring = &vq->sw_ring[desc_idx]; + sw_ring_end = &vq->sw_ring[vq->vq_nentries]; + + rte_prefetch_non_temporal(rused); + + if (vq->vq_free_cnt >= RTE_VIRTIO_VPMD_RX_REARM_THRESH) { + virtio_rxq_rearm_vec(rxvq); + if (unlikely(virtqueue_kick_prepare(vq))) + virtqueue_notify(vq); + } + + for (nb_pkts_received = 0; + nb_pkts_received < nb_used;) { + uint64x2_t desc[RTE_VIRTIO_DESC_PER_LOOP / 2]; + uint64x2_t mbp[RTE_VIRTIO_DESC_PER_LOOP / 2]; + uint64x2_t pkt_mb[RTE_VIRTIO_DESC_PER_LOOP]; + + mbp[0] = vld1q_u64((uint64_t *)(sw_ring + 0)); + desc[0] = vld1q_u64((uint64_t *)(rused + 0)); + vst1q_u64((uint64_t *)&rx_pkts[0], mbp[0]); + + mbp[1] = vld1q_u64((uint64_t *)(sw_ring + 2)); + desc[1] = vld1q_u64((uint64_t *)(rused + 2)); + vst1q_u64((uint64_t *)&rx_pkts[2], mbp[1]); + + mbp[2] = vld1q_u64((uint64_t *)(sw_ring + 4)); + desc[2] = vld1q_u64((uint64_t *)(rused + 4)); + vst1q_u64((uint64_t *)&rx_pkts[4], mbp[2]); + + mbp[3] = vld1q_u64((uint64_t *)(sw_ring + 6)); + desc[3] = vld1q_u64((uint64_t *)(rused + 6)); + vst1q_u64((uint64_t *)&rx_pkts[6], mbp[3]); + + pkt_mb[1] = vreinterpretq_u64_u8(vqtbl1q_u8( + vreinterpretq_u8_u64(desc[0]), shuf_msk2)); + pkt_mb[0] = vreinterpretq_u64_u8(vqtbl1q_u8( + vreinterpretq_u8_u64(desc[0]), shuf_msk1)); + pkt_mb[1] = vreinterpretq_u64_u16(vsubq_u16( + vreinterpretq_u16_u64(pkt_mb[1]), len_adjust)); + pkt_mb[0] = vreinterpretq_u64_u16(vsubq_u16( + vreinterpretq_u16_u64(pkt_mb[0]), len_adjust)); + vst1q_u64((void *)&rx_pkts[1]->rx_descriptor_fields1, + pkt_mb[1]); + vst1q_u64((void *)&rx_pkts[0]->rx_descriptor_fields1, + pkt_mb[0]); + + pkt_mb[3] = vreinterpretq_u64_u8(vqtbl1q_u8( + vreinterpretq_u8_u64(desc[1]), shuf_msk2)); + pkt_mb[2] = vreinterpretq_u64_u8(vqtbl1q_u8( + vreinterpretq_u8_u64(desc[1]), shuf_msk1)); + pkt_mb[3] = vreinterpretq_u64_u16(vsubq_u16( + vreinterpretq_u16_u64(pkt_mb[3]), len_adjust)); + pkt_mb[2] = vreinterpretq_u64_u16(vsubq_u16( + vreinterpretq_u16_u64(pkt_mb[2]), len_adjust)); + vst1q_u64((void *)&rx_pkts[3]->rx_descriptor_fields1, + pkt_mb[3]); + vst1q_u64((void *)&rx_pkts[2]->rx_descriptor_fields1, + pkt_mb[2]); + + pkt_mb[5] = vreinterpretq_u64_u8(vqtbl1q_u8( + vreinterpretq_u8_u64(desc[2]), shuf_msk2)); + pkt_mb[4] = vreinterpretq_u64_u8(vqtbl1q_u8( + vreinterpretq_u8_u64(desc[2]), shuf_msk1)); + pkt_mb[5] = vreinterpretq_u64_u16(vsubq_u16( + vreinterpretq_u16_u64(pkt_mb[5]), len_adjust)); + pkt_mb[4] = vreinterpretq_u64_u16(vsubq_u16( + vreinterpretq_u16_u64(pkt_mb[4]), len_adjust)); + vst1q_u64((void *)&rx_pkts[5]->rx_descriptor_fields1, + pkt_mb[5]); + vst1q_u64((void *)&rx_pkts[4]->rx_descriptor_fields1, + pkt_mb[4]); + + pkt_mb[7] = vreinterpretq_u64_u8(vqtbl1q_u8( + vreinterpretq_u8_u64(desc[3]), shuf_msk2)); + pkt_mb[6] = vreinterpretq_u64_u8(vqtbl1q_u8( + vreinterpretq_u8_u64(desc[3]), shuf_msk1)); + pkt_mb[7] = vreinterpretq_u64_u16(vsubq_u16( + vreinterpretq_u16_u64(pkt_mb[7]), len_adjust)); + pkt_mb[6] = vreinterpretq_u64_u16(vsubq_u16( + vreinterpretq_u16_u64(pkt_mb[6]), len_adjust)); + vst1q_u64((void *)&rx_pkts[7]->rx_descriptor_fields1, + pkt_mb[7]); + vst1q_u64((void *)&rx_pkts[6]->rx_descriptor_fields1, + pkt_mb[6]); + + if (unlikely(nb_used <= RTE_VIRTIO_DESC_PER_LOOP)) { + if (sw_ring + nb_used <= sw_ring_end) + nb_pkts_received += nb_used; + else + nb_pkts_received += sw_ring_end - sw_ring; + break; + } else { + if (unlikely(sw_ring + RTE_VIRTIO_DESC_PER_LOOP >= + sw_ring_end)) { + nb_pkts_received += sw_ring_end - sw_ring; + break; + } else { + nb_pkts_received += RTE_VIRTIO_DESC_PER_LOOP; + + rx_pkts += RTE_VIRTIO_DESC_PER_LOOP; + sw_ring += RTE_VIRTIO_DESC_PER_LOOP; + rused += RTE_VIRTIO_DESC_PER_LOOP; + nb_used -= RTE_VIRTIO_DESC_PER_LOOP; + } + } + } + + vq->vq_used_cons_idx += nb_pkts_received; + vq->vq_free_cnt += nb_pkts_received; + rxvq->stats.packets += nb_pkts_received; + return nb_pkts_received; +} diff --git a/dpdk/drivers/net/virtio/virtio_rxtx_simple_sse.c b/dpdk/drivers/net/virtio/virtio_rxtx_simple_sse.c new file mode 100644 index 00000000..94f65143 --- /dev/null +++ b/dpdk/drivers/net/virtio/virtio_rxtx_simple_sse.c @@ -0,0 +1,225 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "virtio_rxtx_simple.h" + +#define RTE_VIRTIO_VPMD_RX_BURST 32 +#define RTE_VIRTIO_DESC_PER_LOOP 8 +#define RTE_VIRTIO_VPMD_RX_REARM_THRESH RTE_VIRTIO_VPMD_RX_BURST + +/* virtio vPMD receive routine, only accept(nb_pkts >= RTE_VIRTIO_DESC_PER_LOOP) + * + * This routine is for non-mergeable RX, one desc for each guest buffer. + * This routine is based on the RX ring layout optimization. Each entry in the + * avail ring points to the desc with the same index in the desc ring and this + * will never be changed in the driver. + * + * - nb_pkts < RTE_VIRTIO_DESC_PER_LOOP, just return no packet + */ +uint16_t +virtio_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts) +{ + struct virtnet_rx *rxvq = rx_queue; + struct virtqueue *vq = rxvq->vq; + struct virtio_hw *hw = vq->hw; + uint16_t nb_used; + uint16_t desc_idx; + struct vring_used_elem *rused; + struct rte_mbuf **sw_ring; + struct rte_mbuf **sw_ring_end; + uint16_t nb_pkts_received = 0; + __m128i shuf_msk1, shuf_msk2, len_adjust; + + shuf_msk1 = _mm_set_epi8( + 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, /* vlan tci */ + 5, 4, /* dat len */ + 0xFF, 0xFF, 5, 4, /* pkt len */ + 0xFF, 0xFF, 0xFF, 0xFF /* packet type */ + + ); + + shuf_msk2 = _mm_set_epi8( + 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, /* vlan tci */ + 13, 12, /* dat len */ + 0xFF, 0xFF, 13, 12, /* pkt len */ + 0xFF, 0xFF, 0xFF, 0xFF /* packet type */ + ); + + /* Subtract the header length. + * In which case do we need the header length in used->len ? + */ + len_adjust = _mm_set_epi16( + 0, 0, + 0, + (uint16_t)-vq->hw->vtnet_hdr_size, + 0, (uint16_t)-vq->hw->vtnet_hdr_size, + 0, 0); + + if (unlikely(hw->started == 0)) + return nb_pkts_received; + + if (unlikely(nb_pkts < RTE_VIRTIO_DESC_PER_LOOP)) + return 0; + + nb_used = VIRTQUEUE_NUSED(vq); + + rte_compiler_barrier(); + + if (unlikely(nb_used == 0)) + return 0; + + nb_pkts = RTE_ALIGN_FLOOR(nb_pkts, RTE_VIRTIO_DESC_PER_LOOP); + nb_used = RTE_MIN(nb_used, nb_pkts); + + desc_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries - 1)); + rused = &vq->vq_ring.used->ring[desc_idx]; + sw_ring = &vq->sw_ring[desc_idx]; + sw_ring_end = &vq->sw_ring[vq->vq_nentries]; + + rte_prefetch0(rused); + + if (vq->vq_free_cnt >= RTE_VIRTIO_VPMD_RX_REARM_THRESH) { + virtio_rxq_rearm_vec(rxvq); + if (unlikely(virtqueue_kick_prepare(vq))) + virtqueue_notify(vq); + } + + for (nb_pkts_received = 0; + nb_pkts_received < nb_used;) { + __m128i desc[RTE_VIRTIO_DESC_PER_LOOP / 2]; + __m128i mbp[RTE_VIRTIO_DESC_PER_LOOP / 2]; + __m128i pkt_mb[RTE_VIRTIO_DESC_PER_LOOP]; + + mbp[0] = _mm_loadu_si128((__m128i *)(sw_ring + 0)); + desc[0] = _mm_loadu_si128((__m128i *)(rused + 0)); + _mm_storeu_si128((__m128i *)&rx_pkts[0], mbp[0]); + + mbp[1] = _mm_loadu_si128((__m128i *)(sw_ring + 2)); + desc[1] = _mm_loadu_si128((__m128i *)(rused + 2)); + _mm_storeu_si128((__m128i *)&rx_pkts[2], mbp[1]); + + mbp[2] = _mm_loadu_si128((__m128i *)(sw_ring + 4)); + desc[2] = _mm_loadu_si128((__m128i *)(rused + 4)); + _mm_storeu_si128((__m128i *)&rx_pkts[4], mbp[2]); + + mbp[3] = _mm_loadu_si128((__m128i *)(sw_ring + 6)); + desc[3] = _mm_loadu_si128((__m128i *)(rused + 6)); + _mm_storeu_si128((__m128i *)&rx_pkts[6], mbp[3]); + + pkt_mb[1] = _mm_shuffle_epi8(desc[0], shuf_msk2); + pkt_mb[0] = _mm_shuffle_epi8(desc[0], shuf_msk1); + pkt_mb[1] = _mm_add_epi16(pkt_mb[1], len_adjust); + pkt_mb[0] = _mm_add_epi16(pkt_mb[0], len_adjust); + _mm_storeu_si128((void *)&rx_pkts[1]->rx_descriptor_fields1, + pkt_mb[1]); + _mm_storeu_si128((void *)&rx_pkts[0]->rx_descriptor_fields1, + pkt_mb[0]); + + pkt_mb[3] = _mm_shuffle_epi8(desc[1], shuf_msk2); + pkt_mb[2] = _mm_shuffle_epi8(desc[1], shuf_msk1); + pkt_mb[3] = _mm_add_epi16(pkt_mb[3], len_adjust); + pkt_mb[2] = _mm_add_epi16(pkt_mb[2], len_adjust); + _mm_storeu_si128((void *)&rx_pkts[3]->rx_descriptor_fields1, + pkt_mb[3]); + _mm_storeu_si128((void *)&rx_pkts[2]->rx_descriptor_fields1, + pkt_mb[2]); + + pkt_mb[5] = _mm_shuffle_epi8(desc[2], shuf_msk2); + pkt_mb[4] = _mm_shuffle_epi8(desc[2], shuf_msk1); + pkt_mb[5] = _mm_add_epi16(pkt_mb[5], len_adjust); + pkt_mb[4] = _mm_add_epi16(pkt_mb[4], len_adjust); + _mm_storeu_si128((void *)&rx_pkts[5]->rx_descriptor_fields1, + pkt_mb[5]); + _mm_storeu_si128((void *)&rx_pkts[4]->rx_descriptor_fields1, + pkt_mb[4]); + + pkt_mb[7] = _mm_shuffle_epi8(desc[3], shuf_msk2); + pkt_mb[6] = _mm_shuffle_epi8(desc[3], shuf_msk1); + pkt_mb[7] = _mm_add_epi16(pkt_mb[7], len_adjust); + pkt_mb[6] = _mm_add_epi16(pkt_mb[6], len_adjust); + _mm_storeu_si128((void *)&rx_pkts[7]->rx_descriptor_fields1, + pkt_mb[7]); + _mm_storeu_si128((void *)&rx_pkts[6]->rx_descriptor_fields1, + pkt_mb[6]); + + if (unlikely(nb_used <= RTE_VIRTIO_DESC_PER_LOOP)) { + if (sw_ring + nb_used <= sw_ring_end) + nb_pkts_received += nb_used; + else + nb_pkts_received += sw_ring_end - sw_ring; + break; + } else { + if (unlikely(sw_ring + RTE_VIRTIO_DESC_PER_LOOP >= + sw_ring_end)) { + nb_pkts_received += sw_ring_end - sw_ring; + break; + } else { + nb_pkts_received += RTE_VIRTIO_DESC_PER_LOOP; + + rx_pkts += RTE_VIRTIO_DESC_PER_LOOP; + sw_ring += RTE_VIRTIO_DESC_PER_LOOP; + rused += RTE_VIRTIO_DESC_PER_LOOP; + nb_used -= RTE_VIRTIO_DESC_PER_LOOP; + } + } + } + + vq->vq_used_cons_idx += nb_pkts_received; + vq->vq_free_cnt += nb_pkts_received; + rxvq->stats.packets += nb_pkts_received; + return nb_pkts_received; +} diff --git a/dpdk/drivers/net/virtio/virtio_user/vhost.h b/dpdk/drivers/net/virtio/virtio_user/vhost.h index 7adb55f5..5c983bd4 100644 --- a/dpdk/drivers/net/virtio/virtio_user/vhost.h +++ b/dpdk/drivers/net/virtio/virtio_user/vhost.h @@ -42,8 +42,6 @@ #include "../virtio_logs.h" #include "../virtqueue.h" -#define VHOST_MEMORY_MAX_NREGIONS 8 - struct vhost_vring_state { unsigned int index; unsigned int num; @@ -98,6 +96,8 @@ enum vhost_user_request { VHOST_USER_MAX }; +const char * const vhost_msg_strings[VHOST_USER_MAX]; + struct vhost_memory_region { uint64_t guest_phys_addr; uint64_t memory_size; /* bytes */ @@ -105,42 +105,19 @@ struct vhost_memory_region { uint64_t mmap_offset; }; -struct vhost_memory { - uint32_t nregions; - uint32_t padding; - struct vhost_memory_region regions[VHOST_MEMORY_MAX_NREGIONS]; +struct virtio_user_dev; + +struct virtio_user_backend_ops { + int (*setup)(struct virtio_user_dev *dev); + int (*send_request)(struct virtio_user_dev *dev, + enum vhost_user_request req, + void *arg); + int (*enable_qp)(struct virtio_user_dev *dev, + uint16_t pair_idx, + int enable); }; -struct vhost_user_msg { - enum vhost_user_request request; - -#define VHOST_USER_VERSION_MASK 0x3 -#define VHOST_USER_REPLY_MASK (0x1 << 2) - uint32_t flags; - uint32_t size; /* the following payload size */ - union { -#define VHOST_USER_VRING_IDX_MASK 0xff -#define VHOST_USER_VRING_NOFD_MASK (0x1 << 8) - uint64_t u64; - struct vhost_vring_state state; - struct vhost_vring_addr addr; - struct vhost_memory memory; - } payload; - int fds[VHOST_MEMORY_MAX_NREGIONS]; -} __attribute((packed)); - -#define VHOST_USER_HDR_SIZE offsetof(struct vhost_user_msg, payload.u64) -#define VHOST_USER_PAYLOAD_SIZE \ - (sizeof(struct vhost_user_msg) - VHOST_USER_HDR_SIZE) - -/* The version of the protocol we support */ -#define VHOST_USER_VERSION 0x1 - -#define VHOST_USER_F_PROTOCOL_FEATURES 30 -#define VHOST_USER_MQ (1ULL << VHOST_USER_F_PROTOCOL_FEATURES) - -int vhost_user_sock(int vhostfd, enum vhost_user_request req, void *arg); -int vhost_user_setup(const char *path); -int vhost_user_enable_queue_pair(int vhostfd, uint16_t pair_idx, int enable); +struct virtio_user_backend_ops ops_user; +struct virtio_user_backend_ops ops_kernel; #endif diff --git a/dpdk/drivers/net/virtio/virtio_user/vhost_kernel.c b/dpdk/drivers/net/virtio/virtio_user/vhost_kernel.c new file mode 100644 index 00000000..68d28b13 --- /dev/null +++ b/dpdk/drivers/net/virtio/virtio_user/vhost_kernel.c @@ -0,0 +1,403 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include + +#include +#include + +#include "vhost.h" +#include "virtio_user_dev.h" +#include "vhost_kernel_tap.h" + +struct vhost_memory_kernel { + uint32_t nregions; + uint32_t padding; + struct vhost_memory_region regions[0]; +}; + +/* vhost kernel ioctls */ +#define VHOST_VIRTIO 0xAF +#define VHOST_GET_FEATURES _IOR(VHOST_VIRTIO, 0x00, __u64) +#define VHOST_SET_FEATURES _IOW(VHOST_VIRTIO, 0x00, __u64) +#define VHOST_SET_OWNER _IO(VHOST_VIRTIO, 0x01) +#define VHOST_RESET_OWNER _IO(VHOST_VIRTIO, 0x02) +#define VHOST_SET_MEM_TABLE _IOW(VHOST_VIRTIO, 0x03, struct vhost_memory_kernel) +#define VHOST_SET_LOG_BASE _IOW(VHOST_VIRTIO, 0x04, __u64) +#define VHOST_SET_LOG_FD _IOW(VHOST_VIRTIO, 0x07, int) +#define VHOST_SET_VRING_NUM _IOW(VHOST_VIRTIO, 0x10, struct vhost_vring_state) +#define VHOST_SET_VRING_ADDR _IOW(VHOST_VIRTIO, 0x11, struct vhost_vring_addr) +#define VHOST_SET_VRING_BASE _IOW(VHOST_VIRTIO, 0x12, struct vhost_vring_state) +#define VHOST_GET_VRING_BASE _IOWR(VHOST_VIRTIO, 0x12, struct vhost_vring_state) +#define VHOST_SET_VRING_KICK _IOW(VHOST_VIRTIO, 0x20, struct vhost_vring_file) +#define VHOST_SET_VRING_CALL _IOW(VHOST_VIRTIO, 0x21, struct vhost_vring_file) +#define VHOST_SET_VRING_ERR _IOW(VHOST_VIRTIO, 0x22, struct vhost_vring_file) +#define VHOST_NET_SET_BACKEND _IOW(VHOST_VIRTIO, 0x30, struct vhost_vring_file) + +static uint64_t max_regions = 64; + +static void +get_vhost_kernel_max_regions(void) +{ + int fd; + char buf[20] = {'\0'}; + + fd = open("/sys/module/vhost/parameters/max_mem_regions", O_RDONLY); + if (fd < 0) + return; + + if (read(fd, buf, sizeof(buf) - 1) > 0) + max_regions = strtoull(buf, NULL, 10); + + close(fd); +} + +static uint64_t vhost_req_user_to_kernel[] = { + [VHOST_USER_SET_OWNER] = VHOST_SET_OWNER, + [VHOST_USER_RESET_OWNER] = VHOST_RESET_OWNER, + [VHOST_USER_SET_FEATURES] = VHOST_SET_FEATURES, + [VHOST_USER_GET_FEATURES] = VHOST_GET_FEATURES, + [VHOST_USER_SET_VRING_CALL] = VHOST_SET_VRING_CALL, + [VHOST_USER_SET_VRING_NUM] = VHOST_SET_VRING_NUM, + [VHOST_USER_SET_VRING_BASE] = VHOST_SET_VRING_BASE, + [VHOST_USER_GET_VRING_BASE] = VHOST_GET_VRING_BASE, + [VHOST_USER_SET_VRING_ADDR] = VHOST_SET_VRING_ADDR, + [VHOST_USER_SET_VRING_KICK] = VHOST_SET_VRING_KICK, + [VHOST_USER_SET_MEM_TABLE] = VHOST_SET_MEM_TABLE, +}; + +/* By default, vhost kernel module allows 64 regions, but DPDK allows + * 256 segments. As a relief, below function merges those virtually + * adjacent memsegs into one region. + */ +static struct vhost_memory_kernel * +prepare_vhost_memory_kernel(void) +{ + uint32_t i, j, k = 0; + struct rte_memseg *seg; + struct vhost_memory_region *mr; + struct vhost_memory_kernel *vm; + + vm = malloc(sizeof(struct vhost_memory_kernel) + + max_regions * + sizeof(struct vhost_memory_region)); + if (!vm) + return NULL; + + for (i = 0; i < RTE_MAX_MEMSEG; ++i) { + seg = &rte_eal_get_configuration()->mem_config->memseg[i]; + if (!seg->addr) + break; + + int new_region = 1; + + for (j = 0; j < k; ++j) { + mr = &vm->regions[j]; + + if (mr->userspace_addr + mr->memory_size == + (uint64_t)(uintptr_t)seg->addr) { + mr->memory_size += seg->len; + new_region = 0; + break; + } + + if ((uint64_t)(uintptr_t)seg->addr + seg->len == + mr->userspace_addr) { + mr->guest_phys_addr = + (uint64_t)(uintptr_t)seg->addr; + mr->userspace_addr = + (uint64_t)(uintptr_t)seg->addr; + mr->memory_size += seg->len; + new_region = 0; + break; + } + } + + if (new_region == 0) + continue; + + mr = &vm->regions[k++]; + /* use vaddr here! */ + mr->guest_phys_addr = (uint64_t)(uintptr_t)seg->addr; + mr->userspace_addr = (uint64_t)(uintptr_t)seg->addr; + mr->memory_size = seg->len; + mr->mmap_offset = 0; + + if (k >= max_regions) { + free(vm); + return NULL; + } + } + + vm->nregions = k; + vm->padding = 0; + return vm; +} + +/* with below features, vhost kernel does not need to do the checksum and TSO, + * these info will be passed to virtio_user through virtio net header. + */ +#define VHOST_KERNEL_GUEST_OFFLOADS_MASK \ + ((1ULL << VIRTIO_NET_F_GUEST_CSUM) | \ + (1ULL << VIRTIO_NET_F_GUEST_TSO4) | \ + (1ULL << VIRTIO_NET_F_GUEST_TSO6) | \ + (1ULL << VIRTIO_NET_F_GUEST_ECN) | \ + (1ULL << VIRTIO_NET_F_GUEST_UFO)) + +/* with below features, when flows from virtio_user to vhost kernel + * (1) if flows goes up through the kernel networking stack, it does not need + * to verify checksum, which can save CPU cycles; + * (2) if flows goes through a Linux bridge and outside from an interface + * (kernel driver), checksum and TSO will be done by GSO in kernel or even + * offloaded into real physical device. + */ +#define VHOST_KERNEL_HOST_OFFLOADS_MASK \ + ((1ULL << VIRTIO_NET_F_HOST_TSO4) | \ + (1ULL << VIRTIO_NET_F_HOST_TSO6) | \ + (1ULL << VIRTIO_NET_F_CSUM)) + +static int +tap_supporte_mq(void) +{ + int tapfd; + unsigned int tap_features; + + tapfd = open(PATH_NET_TUN, O_RDWR); + if (tapfd < 0) { + PMD_DRV_LOG(ERR, "fail to open %s: %s", + PATH_NET_TUN, strerror(errno)); + return -1; + } + + if (ioctl(tapfd, TUNGETFEATURES, &tap_features) == -1) { + PMD_DRV_LOG(ERR, "TUNGETFEATURES failed: %s", strerror(errno)); + close(tapfd); + return -1; + } + + close(tapfd); + return tap_features & IFF_MULTI_QUEUE; +} + +static int +vhost_kernel_ioctl(struct virtio_user_dev *dev, + enum vhost_user_request req, + void *arg) +{ + int ret = -1; + unsigned int i; + uint64_t req_kernel; + struct vhost_memory_kernel *vm = NULL; + int vhostfd; + unsigned int queue_sel; + + PMD_DRV_LOG(INFO, "%s", vhost_msg_strings[req]); + + req_kernel = vhost_req_user_to_kernel[req]; + + if (req_kernel == VHOST_SET_MEM_TABLE) { + vm = prepare_vhost_memory_kernel(); + if (!vm) + return -1; + arg = (void *)vm; + } + + if (req_kernel == VHOST_SET_FEATURES) { + /* We don't need memory protection here */ + *(uint64_t *)arg &= ~(1ULL << VIRTIO_F_IOMMU_PLATFORM); + + /* VHOST kernel does not know about below flags */ + *(uint64_t *)arg &= ~VHOST_KERNEL_GUEST_OFFLOADS_MASK; + *(uint64_t *)arg &= ~VHOST_KERNEL_HOST_OFFLOADS_MASK; + + *(uint64_t *)arg &= ~(1ULL << VIRTIO_NET_F_MQ); + } + + switch (req_kernel) { + case VHOST_SET_VRING_NUM: + case VHOST_SET_VRING_ADDR: + case VHOST_SET_VRING_BASE: + case VHOST_GET_VRING_BASE: + case VHOST_SET_VRING_KICK: + case VHOST_SET_VRING_CALL: + queue_sel = *(unsigned int *)arg; + vhostfd = dev->vhostfds[queue_sel / 2]; + *(unsigned int *)arg = queue_sel % 2; + PMD_DRV_LOG(DEBUG, "vhostfd=%d, index=%u", + vhostfd, *(unsigned int *)arg); + break; + default: + vhostfd = -1; + } + if (vhostfd == -1) { + for (i = 0; i < dev->max_queue_pairs; ++i) { + if (dev->vhostfds[i] < 0) + continue; + + ret = ioctl(dev->vhostfds[i], req_kernel, arg); + if (ret < 0) + break; + } + } else { + ret = ioctl(vhostfd, req_kernel, arg); + } + + if (!ret && req_kernel == VHOST_GET_FEATURES) { + /* with tap as the backend, all these features are supported + * but not claimed by vhost-net, so we add them back when + * reporting to upper layer. + */ + *((uint64_t *)arg) |= VHOST_KERNEL_GUEST_OFFLOADS_MASK; + *((uint64_t *)arg) |= VHOST_KERNEL_HOST_OFFLOADS_MASK; + + /* vhost_kernel will not declare this feature, but it does + * support multi-queue. + */ + if (tap_supporte_mq()) + *(uint64_t *)arg |= (1ull << VIRTIO_NET_F_MQ); + } + + if (vm) + free(vm); + + if (ret < 0) + PMD_DRV_LOG(ERR, "%s failed: %s", + vhost_msg_strings[req], strerror(errno)); + + return ret; +} + +/** + * Set up environment to talk with a vhost kernel backend. + * + * @return + * - (-1) if fail to set up; + * - (>=0) if successful. + */ +static int +vhost_kernel_setup(struct virtio_user_dev *dev) +{ + int vhostfd; + uint32_t i; + + get_vhost_kernel_max_regions(); + + for (i = 0; i < dev->max_queue_pairs; ++i) { + vhostfd = open(dev->path, O_RDWR); + if (vhostfd < 0) { + PMD_DRV_LOG(ERR, "fail to open %s, %s", + dev->path, strerror(errno)); + return -1; + } + + dev->vhostfds[i] = vhostfd; + } + + return 0; +} + +static int +vhost_kernel_set_backend(int vhostfd, int tapfd) +{ + struct vhost_vring_file f; + + f.fd = tapfd; + f.index = 0; + if (ioctl(vhostfd, VHOST_NET_SET_BACKEND, &f) < 0) { + PMD_DRV_LOG(ERR, "VHOST_NET_SET_BACKEND fails, %s", + strerror(errno)); + return -1; + } + + f.index = 1; + if (ioctl(vhostfd, VHOST_NET_SET_BACKEND, &f) < 0) { + PMD_DRV_LOG(ERR, "VHOST_NET_SET_BACKEND fails, %s", + strerror(errno)); + return -1; + } + + return 0; +} + +static int +vhost_kernel_enable_queue_pair(struct virtio_user_dev *dev, + uint16_t pair_idx, + int enable) +{ + int hdr_size; + int vhostfd; + int tapfd; + int req_mq = (dev->max_queue_pairs > 1); + + vhostfd = dev->vhostfds[pair_idx]; + + if (!enable) { + if (dev->tapfds[pair_idx] >= 0) { + close(dev->tapfds[pair_idx]); + dev->tapfds[pair_idx] = -1; + } + return vhost_kernel_set_backend(vhostfd, -1); + } else if (dev->tapfds[pair_idx] >= 0) { + return 0; + } + + if ((dev->features & (1ULL << VIRTIO_NET_F_MRG_RXBUF)) || + (dev->features & (1ULL << VIRTIO_F_VERSION_1))) + hdr_size = sizeof(struct virtio_net_hdr_mrg_rxbuf); + else + hdr_size = sizeof(struct virtio_net_hdr); + + tapfd = vhost_kernel_open_tap(&dev->ifname, hdr_size, req_mq); + if (tapfd < 0) { + PMD_DRV_LOG(ERR, "fail to open tap for vhost kernel"); + return -1; + } + + if (vhost_kernel_set_backend(vhostfd, tapfd) < 0) { + PMD_DRV_LOG(ERR, "fail to set backend for vhost kernel"); + close(tapfd); + return -1; + } + + dev->tapfds[pair_idx] = tapfd; + return 0; +} + +struct virtio_user_backend_ops ops_kernel = { + .setup = vhost_kernel_setup, + .send_request = vhost_kernel_ioctl, + .enable_qp = vhost_kernel_enable_queue_pair +}; diff --git a/dpdk/drivers/net/virtio/virtio_user/vhost_kernel_tap.c b/dpdk/drivers/net/virtio/virtio_user/vhost_kernel_tap.c new file mode 100644 index 00000000..689a5cff --- /dev/null +++ b/dpdk/drivers/net/virtio/virtio_user/vhost_kernel_tap.c @@ -0,0 +1,133 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "vhost_kernel_tap.h" +#include "../virtio_logs.h" + +int +vhost_kernel_open_tap(char **p_ifname, int hdr_size, int req_mq) +{ + unsigned int tap_features; + int sndbuf = INT_MAX; + struct ifreq ifr; + int tapfd; + unsigned int offload = + TUN_F_CSUM | + TUN_F_TSO4 | + TUN_F_TSO6 | + TUN_F_TSO_ECN | + TUN_F_UFO; + + /* TODO: + * 1. verify we can get/set vnet_hdr_len, tap_probe_vnet_hdr_len + * 2. get number of memory regions from vhost module parameter + * max_mem_regions, supported in newer version linux kernel + */ + tapfd = open(PATH_NET_TUN, O_RDWR); + if (tapfd < 0) { + PMD_DRV_LOG(ERR, "fail to open %s: %s", + PATH_NET_TUN, strerror(errno)); + return -1; + } + + /* Construct ifr */ + memset(&ifr, 0, sizeof(ifr)); + ifr.ifr_flags = IFF_TAP | IFF_NO_PI; + + if (ioctl(tapfd, TUNGETFEATURES, &tap_features) == -1) { + PMD_DRV_LOG(ERR, "TUNGETFEATURES failed: %s", strerror(errno)); + goto error; + } + if (tap_features & IFF_ONE_QUEUE) + ifr.ifr_flags |= IFF_ONE_QUEUE; + + /* Let tap instead of vhost-net handle vnet header, as the latter does + * not support offloading. And in this case, we should not set feature + * bit VHOST_NET_F_VIRTIO_NET_HDR. + */ + if (tap_features & IFF_VNET_HDR) { + ifr.ifr_flags |= IFF_VNET_HDR; + } else { + PMD_DRV_LOG(ERR, "TAP does not support IFF_VNET_HDR"); + goto error; + } + + if (req_mq) + ifr.ifr_flags |= IFF_MULTI_QUEUE; + + if (*p_ifname) + strncpy(ifr.ifr_name, *p_ifname, IFNAMSIZ - 1); + else + strncpy(ifr.ifr_name, "tap%d", IFNAMSIZ - 1); + if (ioctl(tapfd, TUNSETIFF, (void *)&ifr) == -1) { + PMD_DRV_LOG(ERR, "TUNSETIFF failed: %s", strerror(errno)); + goto error; + } + + fcntl(tapfd, F_SETFL, O_NONBLOCK); + + if (ioctl(tapfd, TUNSETVNETHDRSZ, &hdr_size) < 0) { + PMD_DRV_LOG(ERR, "TUNSETVNETHDRSZ failed: %s", strerror(errno)); + goto error; + } + + if (ioctl(tapfd, TUNSETSNDBUF, &sndbuf) < 0) { + PMD_DRV_LOG(ERR, "TUNSETSNDBUF failed: %s", strerror(errno)); + goto error; + } + + /* TODO: before set the offload capabilities, we'd better (1) check + * negotiated features to see if necessary to offload; (2) query tap + * to see if it supports the offload capabilities. + */ + if (ioctl(tapfd, TUNSETOFFLOAD, offload) != 0) + PMD_DRV_LOG(ERR, "TUNSETOFFLOAD ioctl() failed: %s", + strerror(errno)); + + if (!(*p_ifname)) + *p_ifname = strdup(ifr.ifr_name); + + return tapfd; +error: + close(tapfd); + return -1; +} diff --git a/dpdk/drivers/net/virtio/virtio_user/vhost_kernel_tap.h b/dpdk/drivers/net/virtio/virtio_user/vhost_kernel_tap.h new file mode 100644 index 00000000..eae340cc --- /dev/null +++ b/dpdk/drivers/net/virtio/virtio_user/vhost_kernel_tap.h @@ -0,0 +1,67 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +/* TUN ioctls */ +#define TUNSETIFF _IOW('T', 202, int) +#define TUNGETFEATURES _IOR('T', 207, unsigned int) +#define TUNSETOFFLOAD _IOW('T', 208, unsigned int) +#define TUNGETIFF _IOR('T', 210, unsigned int) +#define TUNSETSNDBUF _IOW('T', 212, int) +#define TUNGETVNETHDRSZ _IOR('T', 215, int) +#define TUNSETVNETHDRSZ _IOW('T', 216, int) +#define TUNSETQUEUE _IOW('T', 217, int) +#define TUNSETVNETLE _IOW('T', 220, int) +#define TUNSETVNETBE _IOW('T', 222, int) + +/* TUNSETIFF ifr flags */ +#define IFF_TAP 0x0002 +#define IFF_NO_PI 0x1000 +#define IFF_ONE_QUEUE 0x2000 +#define IFF_VNET_HDR 0x4000 +#define IFF_MULTI_QUEUE 0x0100 +#define IFF_ATTACH_QUEUE 0x0200 +#define IFF_DETACH_QUEUE 0x0400 + +/* Features for GSO (TUNSETOFFLOAD). */ +#define TUN_F_CSUM 0x01 /* You can hand me unchecksummed packets. */ +#define TUN_F_TSO4 0x02 /* I can handle TSO for IPv4 packets */ +#define TUN_F_TSO6 0x04 /* I can handle TSO for IPv6 packets */ +#define TUN_F_TSO_ECN 0x08 /* I can handle TSO with ECN bits. */ +#define TUN_F_UFO 0x10 /* I can handle UFO packets */ + +/* Constants */ +#define PATH_NET_TUN "/dev/net/tun" + +int vhost_kernel_open_tap(char **p_ifname, int hdr_size, int req_mq); diff --git a/dpdk/drivers/net/virtio/virtio_user/vhost_user.c b/dpdk/drivers/net/virtio/virtio_user/vhost_user.c index 082e8217..97bd8326 100644 --- a/dpdk/drivers/net/virtio/virtio_user/vhost_user.c +++ b/dpdk/drivers/net/virtio/virtio_user/vhost_user.c @@ -41,6 +41,39 @@ #include #include "vhost.h" +#include "virtio_user_dev.h" + +/* The version of the protocol we support */ +#define VHOST_USER_VERSION 0x1 + +#define VHOST_MEMORY_MAX_NREGIONS 8 +struct vhost_memory { + uint32_t nregions; + uint32_t padding; + struct vhost_memory_region regions[VHOST_MEMORY_MAX_NREGIONS]; +}; + +struct vhost_user_msg { + enum vhost_user_request request; + +#define VHOST_USER_VERSION_MASK 0x3 +#define VHOST_USER_REPLY_MASK (0x1 << 2) + uint32_t flags; + uint32_t size; /* the following payload size */ + union { +#define VHOST_USER_VRING_IDX_MASK 0xff +#define VHOST_USER_VRING_NOFD_MASK (0x1 << 8) + uint64_t u64; + struct vhost_vring_state state; + struct vhost_vring_addr addr; + struct vhost_memory memory; + } payload; + int fds[VHOST_MEMORY_MAX_NREGIONS]; +} __attribute((packed)); + +#define VHOST_USER_HDR_SIZE offsetof(struct vhost_user_msg, payload.u64) +#define VHOST_USER_PAYLOAD_SIZE \ + (sizeof(struct vhost_user_msg) - VHOST_USER_HDR_SIZE) static int vhost_user_write(int fd, void *buf, int len, int *fds, int fd_num) @@ -97,6 +130,10 @@ vhost_user_read(int fd, struct vhost_user_msg *msg) } sz_payload = msg->size; + + if ((size_t)sz_payload > sizeof(msg->payload)) + goto fail; + if (sz_payload) { ret = recv(fd, (void *)((char *)msg + sz_hdr), sz_payload, 0); if (ret < sz_payload) { @@ -223,24 +260,25 @@ prepare_vhost_memory_user(struct vhost_user_msg *msg, int fds[]) static struct vhost_user_msg m; -static const char * const vhost_msg_strings[] = { - [VHOST_USER_SET_OWNER] = "VHOST_USER_SET_OWNER", - [VHOST_USER_RESET_OWNER] = "VHOST_USER_RESET_OWNER", - [VHOST_USER_SET_FEATURES] = "VHOST_USER_SET_FEATURES", - [VHOST_USER_GET_FEATURES] = "VHOST_USER_GET_FEATURES", - [VHOST_USER_SET_VRING_CALL] = "VHOST_USER_SET_VRING_CALL", - [VHOST_USER_SET_VRING_NUM] = "VHOST_USER_SET_VRING_NUM", - [VHOST_USER_SET_VRING_BASE] = "VHOST_USER_SET_VRING_BASE", - [VHOST_USER_GET_VRING_BASE] = "VHOST_USER_GET_VRING_BASE", - [VHOST_USER_SET_VRING_ADDR] = "VHOST_USER_SET_VRING_ADDR", - [VHOST_USER_SET_VRING_KICK] = "VHOST_USER_SET_VRING_KICK", - [VHOST_USER_SET_MEM_TABLE] = "VHOST_USER_SET_MEM_TABLE", - [VHOST_USER_SET_VRING_ENABLE] = "VHOST_USER_SET_VRING_ENABLE", - NULL, +const char * const vhost_msg_strings[] = { + [VHOST_USER_SET_OWNER] = "VHOST_SET_OWNER", + [VHOST_USER_RESET_OWNER] = "VHOST_RESET_OWNER", + [VHOST_USER_SET_FEATURES] = "VHOST_SET_FEATURES", + [VHOST_USER_GET_FEATURES] = "VHOST_GET_FEATURES", + [VHOST_USER_SET_VRING_CALL] = "VHOST_SET_VRING_CALL", + [VHOST_USER_SET_VRING_NUM] = "VHOST_SET_VRING_NUM", + [VHOST_USER_SET_VRING_BASE] = "VHOST_SET_VRING_BASE", + [VHOST_USER_GET_VRING_BASE] = "VHOST_GET_VRING_BASE", + [VHOST_USER_SET_VRING_ADDR] = "VHOST_SET_VRING_ADDR", + [VHOST_USER_SET_VRING_KICK] = "VHOST_SET_VRING_KICK", + [VHOST_USER_SET_MEM_TABLE] = "VHOST_SET_MEM_TABLE", + [VHOST_USER_SET_VRING_ENABLE] = "VHOST_SET_VRING_ENABLE", }; -int -vhost_user_sock(int vhostfd, enum vhost_user_request req, void *arg) +static int +vhost_user_sock(struct virtio_user_dev *dev, + enum vhost_user_request req, + void *arg) { struct vhost_user_msg msg; struct vhost_vring_file *file = 0; @@ -248,9 +286,9 @@ vhost_user_sock(int vhostfd, enum vhost_user_request req, void *arg) int fds[VHOST_MEMORY_MAX_NREGIONS]; int fd_num = 0; int i, len; + int vhostfd = dev->vhostfd; RTE_SET_USED(m); - RTE_SET_USED(vhost_msg_strings); PMD_DRV_LOG(INFO, "%s", vhost_msg_strings[req]); @@ -371,15 +409,13 @@ vhost_user_sock(int vhostfd, enum vhost_user_request req, void *arg) /** * Set up environment to talk with a vhost user backend. - * @param path - * - The path to vhost user unix socket file. * * @return - * - (-1) if fail to set up; - * - (>=0) if successful, and it is the fd to vhostfd. + * - (-1) if fail; + * - (0) if succeed. */ -int -vhost_user_setup(const char *path) +static int +vhost_user_setup(struct virtio_user_dev *dev) { int fd; int flag; @@ -397,18 +433,21 @@ vhost_user_setup(const char *path) memset(&un, 0, sizeof(un)); un.sun_family = AF_UNIX; - snprintf(un.sun_path, sizeof(un.sun_path), "%s", path); + snprintf(un.sun_path, sizeof(un.sun_path), "%s", dev->path); if (connect(fd, (struct sockaddr *)&un, sizeof(un)) < 0) { PMD_DRV_LOG(ERR, "connect error, %s", strerror(errno)); close(fd); return -1; } - return fd; + dev->vhostfd = fd; + return 0; } -int -vhost_user_enable_queue_pair(int vhostfd, uint16_t pair_idx, int enable) +static int +vhost_user_enable_queue_pair(struct virtio_user_dev *dev, + uint16_t pair_idx, + int enable) { int i; @@ -418,10 +457,15 @@ vhost_user_enable_queue_pair(int vhostfd, uint16_t pair_idx, int enable) .num = enable, }; - if (vhost_user_sock(vhostfd, - VHOST_USER_SET_VRING_ENABLE, &state)) + if (vhost_user_sock(dev, VHOST_USER_SET_VRING_ENABLE, &state)) return -1; } return 0; } + +struct virtio_user_backend_ops ops_user = { + .setup = vhost_user_setup, + .send_request = vhost_user_sock, + .enable_qp = vhost_user_enable_queue_pair +}; diff --git a/dpdk/drivers/net/virtio/virtio_user/virtio_user_dev.c b/dpdk/drivers/net/virtio/virtio_user/virtio_user_dev.c index e239e0eb..7ce512c7 100644 --- a/dpdk/drivers/net/virtio/virtio_user/virtio_user_dev.c +++ b/dpdk/drivers/net/virtio/virtio_user/virtio_user_dev.c @@ -39,6 +39,8 @@ #include #include #include +#include +#include #include "vhost.h" #include "virtio_user_dev.h" @@ -51,21 +53,11 @@ virtio_user_create_queue(struct virtio_user_dev *dev, uint32_t queue_sel) * firstly because vhost depends on this msg to allocate virtqueue * pair. */ - int callfd; struct vhost_vring_file file; - /* May use invalid flag, but some backend leverages kickfd and callfd as - * criteria to judge if dev is alive. so finally we use real event_fd. - */ - callfd = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK); - if (callfd < 0) { - PMD_DRV_LOG(ERR, "callfd error, %s\n", strerror(errno)); - return -1; - } file.index = queue_sel; - file.fd = callfd; - vhost_user_sock(dev->vhostfd, VHOST_USER_SET_VRING_CALL, &file); - dev->callfds[queue_sel] = callfd; + file.fd = dev->callfds[queue_sel]; + dev->ops->send_request(dev, VHOST_USER_SET_VRING_CALL, &file); return 0; } @@ -73,7 +65,6 @@ virtio_user_create_queue(struct virtio_user_dev *dev, uint32_t queue_sel) static int virtio_user_kick_queue(struct virtio_user_dev *dev, uint32_t queue_sel) { - int kickfd; struct vhost_vring_file file; struct vhost_vring_state state; struct vring *vring = &dev->vrings[queue_sel]; @@ -88,26 +79,21 @@ virtio_user_kick_queue(struct virtio_user_dev *dev, uint32_t queue_sel) state.index = queue_sel; state.num = vring->num; - vhost_user_sock(dev->vhostfd, VHOST_USER_SET_VRING_NUM, &state); + dev->ops->send_request(dev, VHOST_USER_SET_VRING_NUM, &state); + state.index = queue_sel; state.num = 0; /* no reservation */ - vhost_user_sock(dev->vhostfd, VHOST_USER_SET_VRING_BASE, &state); + dev->ops->send_request(dev, VHOST_USER_SET_VRING_BASE, &state); - vhost_user_sock(dev->vhostfd, VHOST_USER_SET_VRING_ADDR, &addr); + dev->ops->send_request(dev, VHOST_USER_SET_VRING_ADDR, &addr); /* Of all per virtqueue MSGs, make sure VHOST_USER_SET_VRING_KICK comes * lastly because vhost depends on this msg to judge if * virtio is ready. */ - kickfd = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK); - if (kickfd < 0) { - PMD_DRV_LOG(ERR, "kickfd error, %s\n", strerror(errno)); - return -1; - } file.index = queue_sel; - file.fd = kickfd; - vhost_user_sock(dev->vhostfd, VHOST_USER_SET_VRING_KICK, &file); - dev->kickfds[queue_sel] = kickfd; + file.fd = dev->kickfds[queue_sel]; + dev->ops->send_request(dev, VHOST_USER_SET_VRING_KICK, &file); return 0; } @@ -146,21 +132,20 @@ virtio_user_start_device(struct virtio_user_dev *dev) if (virtio_user_queue_setup(dev, virtio_user_create_queue) < 0) goto error; - /* Step 1: set features - * Make sure VHOST_USER_F_PROTOCOL_FEATURES is added if mq is enabled, - * and VIRTIO_NET_F_MAC is stripped. - */ + /* Step 1: set features */ features = dev->features; - if (dev->max_queue_pairs > 1) - features |= VHOST_USER_MQ; + /* Strip VIRTIO_NET_F_MAC, as MAC address is handled in vdev init */ features &= ~(1ull << VIRTIO_NET_F_MAC); - ret = vhost_user_sock(dev->vhostfd, VHOST_USER_SET_FEATURES, &features); + /* Strip VIRTIO_NET_F_CTRL_VQ, as devices do not really need to know */ + features &= ~(1ull << VIRTIO_NET_F_CTRL_VQ); + features &= ~(1ull << VIRTIO_NET_F_STATUS); + ret = dev->ops->send_request(dev, VHOST_USER_SET_FEATURES, &features); if (ret < 0) goto error; PMD_DRV_LOG(INFO, "set features: %" PRIx64, features); /* Step 2: share memory regions */ - ret = vhost_user_sock(dev->vhostfd, VHOST_USER_SET_MEM_TABLE, NULL); + ret = dev->ops->send_request(dev, VHOST_USER_SET_MEM_TABLE, NULL); if (ret < 0) goto error; @@ -171,7 +156,7 @@ virtio_user_start_device(struct virtio_user_dev *dev) /* Step 4: enable queues * we enable the 1st queue pair by default. */ - vhost_user_enable_queue_pair(dev->vhostfd, 0, 1); + dev->ops->enable_qp(dev, 0, 1); return 0; error: @@ -181,7 +166,17 @@ error: int virtio_user_stop_device(struct virtio_user_dev *dev) { - return vhost_user_sock(dev->vhostfd, VHOST_USER_RESET_OWNER, NULL); + uint32_t i; + + for (i = 0; i < dev->max_queue_pairs; ++i) + dev->ops->enable_qp(dev, i, 0); + + if (dev->ops->send_request(dev, VHOST_USER_RESET_OWNER, NULL) < 0) { + PMD_DRV_LOG(INFO, "Failed to reset the device\n"); + return -1; + } + + return 0; } static inline void @@ -205,9 +200,150 @@ parse_mac(struct virtio_user_dev *dev, const char *mac) } } +int +is_vhost_user_by_type(const char *path) +{ + struct stat sb; + + if (stat(path, &sb) == -1) + return 0; + + return S_ISSOCK(sb.st_mode); +} + +static int +virtio_user_dev_init_notify(struct virtio_user_dev *dev) +{ + uint32_t i, j; + int callfd; + int kickfd; + + for (i = 0; i < VIRTIO_MAX_VIRTQUEUES; ++i) { + if (i >= dev->max_queue_pairs * 2) { + dev->kickfds[i] = -1; + dev->callfds[i] = -1; + continue; + } + + /* May use invalid flag, but some backend uses kickfd and + * callfd as criteria to judge if dev is alive. so finally we + * use real event_fd. + */ + callfd = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK); + if (callfd < 0) { + PMD_DRV_LOG(ERR, "callfd error, %s", strerror(errno)); + break; + } + kickfd = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK); + if (kickfd < 0) { + PMD_DRV_LOG(ERR, "kickfd error, %s", strerror(errno)); + break; + } + dev->callfds[i] = callfd; + dev->kickfds[i] = kickfd; + } + + if (i < VIRTIO_MAX_VIRTQUEUES) { + for (j = 0; j <= i; ++j) { + close(dev->callfds[j]); + close(dev->kickfds[j]); + } + + return -1; + } + + return 0; +} + +static int +virtio_user_fill_intr_handle(struct virtio_user_dev *dev) +{ + uint32_t i; + struct rte_eth_dev *eth_dev = &rte_eth_devices[dev->port_id]; + + if (!eth_dev->intr_handle) { + eth_dev->intr_handle = malloc(sizeof(*eth_dev->intr_handle)); + if (!eth_dev->intr_handle) { + PMD_DRV_LOG(ERR, "fail to allocate intr_handle"); + return -1; + } + memset(eth_dev->intr_handle, 0, sizeof(*eth_dev->intr_handle)); + } + + for (i = 0; i < dev->max_queue_pairs; ++i) + eth_dev->intr_handle->efds[i] = dev->callfds[i]; + eth_dev->intr_handle->nb_efd = dev->max_queue_pairs; + eth_dev->intr_handle->max_intr = dev->max_queue_pairs + 1; + eth_dev->intr_handle->type = RTE_INTR_HANDLE_VDEV; + /* For virtio vdev, no need to read counter for clean */ + eth_dev->intr_handle->efd_counter_size = 0; + if (dev->vhostfd >= 0) + eth_dev->intr_handle->fd = dev->vhostfd; + + return 0; +} + +static int +virtio_user_dev_setup(struct virtio_user_dev *dev) +{ + uint32_t q; + + dev->vhostfd = -1; + dev->vhostfds = NULL; + dev->tapfds = NULL; + + if (is_vhost_user_by_type(dev->path)) { + dev->ops = &ops_user; + } else { + dev->ops = &ops_kernel; + + dev->vhostfds = malloc(dev->max_queue_pairs * sizeof(int)); + dev->tapfds = malloc(dev->max_queue_pairs * sizeof(int)); + if (!dev->vhostfds || !dev->tapfds) { + PMD_INIT_LOG(ERR, "Failed to malloc"); + return -1; + } + + for (q = 0; q < dev->max_queue_pairs; ++q) { + dev->vhostfds[q] = -1; + dev->tapfds[q] = -1; + } + } + + if (dev->ops->setup(dev) < 0) + return -1; + + if (virtio_user_dev_init_notify(dev) < 0) + return -1; + + if (virtio_user_fill_intr_handle(dev) < 0) + return -1; + + return 0; +} + +/* Use below macro to filter features from vhost backend */ +#define VIRTIO_USER_SUPPORTED_FEATURES \ + (1ULL << VIRTIO_NET_F_MAC | \ + 1ULL << VIRTIO_NET_F_STATUS | \ + 1ULL << VIRTIO_NET_F_MQ | \ + 1ULL << VIRTIO_NET_F_CTRL_MAC_ADDR | \ + 1ULL << VIRTIO_NET_F_CTRL_VQ | \ + 1ULL << VIRTIO_NET_F_CTRL_RX | \ + 1ULL << VIRTIO_NET_F_CTRL_VLAN | \ + 1ULL << VIRTIO_NET_F_CSUM | \ + 1ULL << VIRTIO_NET_F_HOST_TSO4 | \ + 1ULL << VIRTIO_NET_F_HOST_TSO6 | \ + 1ULL << VIRTIO_NET_F_MRG_RXBUF | \ + 1ULL << VIRTIO_RING_F_INDIRECT_DESC | \ + 1ULL << VIRTIO_NET_F_GUEST_CSUM | \ + 1ULL << VIRTIO_NET_F_GUEST_TSO4 | \ + 1ULL << VIRTIO_NET_F_GUEST_TSO6 | \ + 1ULL << VIRTIO_F_VERSION_1) + int virtio_user_dev_init(struct virtio_user_dev *dev, char *path, int queues, - int cq, int queue_size, const char *mac) + int cq, int queue_size, const char *mac, char **ifname) { snprintf(dev->path, PATH_MAX, "%s", path); dev->max_queue_pairs = queues; @@ -215,50 +351,49 @@ virtio_user_dev_init(struct virtio_user_dev *dev, char *path, int queues, dev->queue_size = queue_size; dev->mac_specified = 0; parse_mac(dev, mac); - dev->vhostfd = -1; - dev->vhostfd = vhost_user_setup(dev->path); - if (dev->vhostfd < 0) { + if (*ifname) { + dev->ifname = *ifname; + *ifname = NULL; + } + + if (virtio_user_dev_setup(dev) < 0) { PMD_INIT_LOG(ERR, "backend set up fails"); return -1; } - if (vhost_user_sock(dev->vhostfd, VHOST_USER_SET_OWNER, NULL) < 0) { + if (dev->ops->send_request(dev, VHOST_USER_SET_OWNER, NULL) < 0) { PMD_INIT_LOG(ERR, "set_owner fails: %s", strerror(errno)); return -1; } - if (vhost_user_sock(dev->vhostfd, VHOST_USER_GET_FEATURES, - &dev->features) < 0) { + if (dev->ops->send_request(dev, VHOST_USER_GET_FEATURES, + &dev->device_features) < 0) { PMD_INIT_LOG(ERR, "get_features failed: %s", strerror(errno)); return -1; } if (dev->mac_specified) - dev->features |= (1ull << VIRTIO_NET_F_MAC); + dev->device_features |= (1ull << VIRTIO_NET_F_MAC); - if (!cq) { - dev->features &= ~(1ull << VIRTIO_NET_F_CTRL_VQ); - /* Also disable features depends on VIRTIO_NET_F_CTRL_VQ */ - dev->features &= ~(1ull << VIRTIO_NET_F_CTRL_RX); - dev->features &= ~(1ull << VIRTIO_NET_F_CTRL_VLAN); - dev->features &= ~(1ull << VIRTIO_NET_F_GUEST_ANNOUNCE); - dev->features &= ~(1ull << VIRTIO_NET_F_MQ); - dev->features &= ~(1ull << VIRTIO_NET_F_CTRL_MAC_ADDR); - } else { - /* vhost user backend does not need to know ctrl-q, so - * actually we need add this bit into features. However, - * DPDK vhost-user does send features with this bit, so we - * check it instead of OR it for now. + if (cq) { + /* device does not really need to know anything about CQ, + * so if necessary, we just claim to support CQ */ - if (!(dev->features & (1ull << VIRTIO_NET_F_CTRL_VQ))) - PMD_INIT_LOG(INFO, "vhost does not support ctrl-q"); + dev->device_features |= (1ull << VIRTIO_NET_F_CTRL_VQ); + } else { + dev->device_features &= ~(1ull << VIRTIO_NET_F_CTRL_VQ); + /* Also disable features depends on VIRTIO_NET_F_CTRL_VQ */ + dev->device_features &= ~(1ull << VIRTIO_NET_F_CTRL_RX); + dev->device_features &= ~(1ull << VIRTIO_NET_F_CTRL_VLAN); + dev->device_features &= ~(1ull << VIRTIO_NET_F_GUEST_ANNOUNCE); + dev->device_features &= ~(1ull << VIRTIO_NET_F_MQ); + dev->device_features &= ~(1ull << VIRTIO_NET_F_CTRL_MAC_ADDR); } - if (dev->max_queue_pairs > 1) { - if (!(dev->features & VHOST_USER_MQ)) { - PMD_INIT_LOG(ERR, "MQ not supported by the backend"); - return -1; - } - } + /* The backend will not report this feature, we add it explicitly */ + if (is_vhost_user_by_type(dev->path)) + dev->device_features |= (1ull << VIRTIO_NET_F_STATUS); + + dev->device_features &= VIRTIO_USER_SUPPORTED_FEATURES; return 0; } @@ -268,12 +403,23 @@ virtio_user_dev_uninit(struct virtio_user_dev *dev) { uint32_t i; + virtio_user_stop_device(dev); + for (i = 0; i < dev->max_queue_pairs * 2; ++i) { close(dev->callfds[i]); close(dev->kickfds[i]); } close(dev->vhostfd); + + if (dev->vhostfds) { + for (i = 0; i < dev->max_queue_pairs; ++i) + close(dev->vhostfds[i]); + free(dev->vhostfds); + free(dev->tapfds); + } + + free(dev->ifname); } static uint8_t @@ -289,9 +435,9 @@ virtio_user_handle_mq(struct virtio_user_dev *dev, uint16_t q_pairs) } for (i = 0; i < q_pairs; ++i) - ret |= vhost_user_enable_queue_pair(dev->vhostfd, i, 1); + ret |= dev->ops->enable_qp(dev, i, 1); for (i = q_pairs; i < dev->max_queue_pairs; ++i) - ret |= vhost_user_enable_queue_pair(dev->vhostfd, i, 0); + ret |= dev->ops->enable_qp(dev, i, 0); dev->queue_pairs = q_pairs; diff --git a/dpdk/drivers/net/virtio/virtio_user/virtio_user_dev.h b/dpdk/drivers/net/virtio/virtio_user/virtio_user_dev.h index 33690b5c..8361b6bd 100644 --- a/dpdk/drivers/net/virtio/virtio_user/virtio_user_dev.h +++ b/dpdk/drivers/net/virtio/virtio_user/virtio_user_dev.h @@ -37,26 +37,41 @@ #include #include "../virtio_pci.h" #include "../virtio_ring.h" +#include "vhost.h" struct virtio_user_dev { + /* for vhost_user backend */ int vhostfd; - int callfds[VIRTIO_MAX_VIRTQUEUES * 2 + 1]; - int kickfds[VIRTIO_MAX_VIRTQUEUES * 2 + 1]; + + /* for vhost_kernel backend */ + char *ifname; + int *vhostfds; + int *tapfds; + + /* for both vhost_user and vhost_kernel */ + int callfds[VIRTIO_MAX_VIRTQUEUES]; + int kickfds[VIRTIO_MAX_VIRTQUEUES]; int mac_specified; uint32_t max_queue_pairs; uint32_t queue_pairs; uint32_t queue_size; - uint64_t features; + uint64_t features; /* the negotiated features with driver, + * and will be sync with device + */ + uint64_t device_features; /* supported features by device */ uint8_t status; + uint8_t port_id; uint8_t mac_addr[ETHER_ADDR_LEN]; char path[PATH_MAX]; - struct vring vrings[VIRTIO_MAX_VIRTQUEUES * 2 + 1]; + struct vring vrings[VIRTIO_MAX_VIRTQUEUES]; + struct virtio_user_backend_ops *ops; }; +int is_vhost_user_by_type(const char *path); int virtio_user_start_device(struct virtio_user_dev *dev); int virtio_user_stop_device(struct virtio_user_dev *dev); int virtio_user_dev_init(struct virtio_user_dev *dev, char *path, int queues, - int cq, int queue_size, const char *mac); + int cq, int queue_size, const char *mac, char **ifname); void virtio_user_dev_uninit(struct virtio_user_dev *dev); void virtio_user_handle_cq(struct virtio_user_dev *dev, uint16_t queue_idx); #endif diff --git a/dpdk/drivers/net/virtio/virtio_user_ethdev.c b/dpdk/drivers/net/virtio/virtio_user_ethdev.c index bba74028..7be57ce6 100644 --- a/dpdk/drivers/net/virtio/virtio_user_ethdev.c +++ b/dpdk/drivers/net/virtio/virtio_user_ethdev.c @@ -34,9 +34,14 @@ #include #include #include +#include +#include #include #include +#include +#include +#include #include "virtio_ethdev.h" #include "virtio_logs.h" @@ -48,6 +53,17 @@ #define virtio_user_get_dev(hw) \ ((struct virtio_user_dev *)(hw)->virtio_user_dev) +static void +virtio_user_delayed_handler(void *param) +{ + struct virtio_hw *hw = (struct virtio_hw *)param; + struct rte_eth_dev *dev = &rte_eth_devices[hw->port_id]; + + rte_intr_callback_unregister(dev->intr_handle, + virtio_interrupt_handler, + dev); +} + static void virtio_user_read_dev_config(struct virtio_hw *hw, size_t offset, void *dst, int length) @@ -62,8 +78,41 @@ virtio_user_read_dev_config(struct virtio_hw *hw, size_t offset, return; } - if (offset == offsetof(struct virtio_net_config, status)) + if (offset == offsetof(struct virtio_net_config, status)) { + char buf[128]; + + if (dev->vhostfd >= 0) { + int r; + int flags; + + flags = fcntl(dev->vhostfd, F_GETFL); + if (fcntl(dev->vhostfd, F_SETFL, + flags | O_NONBLOCK) == -1) { + PMD_DRV_LOG(ERR, "error setting O_NONBLOCK flag"); + return; + } + r = recv(dev->vhostfd, buf, 128, MSG_PEEK); + if (r == 0 || (r < 0 && errno != EAGAIN)) { + dev->status &= (~VIRTIO_NET_S_LINK_UP); + PMD_DRV_LOG(ERR, "virtio-user port %u is down", + hw->port_id); + /* Only client mode is available now. Once the + * connection is broken, it can never be up + * again. Besides, this function could be called + * in the process of interrupt handling, + * callback cannot be unregistered here, set an + * alarm to do it. + */ + rte_eal_alarm_set(1, + virtio_user_delayed_handler, + (void *)hw); + } else { + dev->status |= VIRTIO_NET_S_LINK_UP; + } + fcntl(dev->vhostfd, F_SETFL, flags & (~O_NONBLOCK)); + } *(uint16_t *)dst = dev->status; + } if (offset == offsetof(struct virtio_net_config, max_virtqueue_pairs)) *(uint16_t *)dst = dev->max_queue_pairs; @@ -81,10 +130,19 @@ virtio_user_write_dev_config(struct virtio_hw *hw, size_t offset, for (i = 0; i < ETHER_ADDR_LEN; ++i) dev->mac_addr[i] = ((const uint8_t *)src)[i]; else - PMD_DRV_LOG(ERR, "not supported offset=%zu, len=%d\n", + PMD_DRV_LOG(ERR, "not supported offset=%zu, len=%d", offset, length); } +static void +virtio_user_reset(struct virtio_hw *hw) +{ + struct virtio_user_dev *dev = virtio_user_get_dev(hw); + + if (dev->status & VIRTIO_CONFIG_STATUS_DRIVER_OK) + virtio_user_stop_device(dev); +} + static void virtio_user_set_status(struct virtio_hw *hw, uint8_t status) { @@ -92,17 +150,11 @@ virtio_user_set_status(struct virtio_hw *hw, uint8_t status) if (status & VIRTIO_CONFIG_STATUS_DRIVER_OK) virtio_user_start_device(dev); + else if (status == VIRTIO_CONFIG_STATUS_RESET) + virtio_user_reset(hw); dev->status = status; } -static void -virtio_user_reset(struct virtio_hw *hw) -{ - struct virtio_user_dev *dev = virtio_user_get_dev(hw); - - virtio_user_stop_device(dev); -} - static uint8_t virtio_user_get_status(struct virtio_hw *hw) { @@ -116,7 +168,8 @@ virtio_user_get_features(struct virtio_hw *hw) { struct virtio_user_dev *dev = virtio_user_get_dev(hw); - return dev->features; + /* unmask feature bits defined in vhost user protocol */ + return dev->device_features & VIRTIO_PMD_SUPPORTED_GUEST_FEATURES; } static void @@ -124,23 +177,32 @@ virtio_user_set_features(struct virtio_hw *hw, uint64_t features) { struct virtio_user_dev *dev = virtio_user_get_dev(hw); - dev->features = features; + dev->features = features & dev->device_features; } static uint8_t virtio_user_get_isr(struct virtio_hw *hw __rte_unused) { - /* When config interrupt happens, driver calls this function to query - * what kinds of change happen. Interrupt mode not supported for now. + /* rxq interrupts and config interrupt are separated in virtio-user, + * here we only report config change. */ - return 0; + return VIRTIO_PCI_ISR_CONFIG; } static uint16_t virtio_user_set_config_irq(struct virtio_hw *hw __rte_unused, uint16_t vec __rte_unused) { - return VIRTIO_MSI_NO_VECTOR; + return 0; +} + +static uint16_t +virtio_user_set_queue_irq(struct virtio_hw *hw __rte_unused, + struct virtqueue *vq __rte_unused, + uint16_t vec) +{ + /* pretend we have done that */ + return vec; } /* This function is to get the queue size, aka, number of descs, of a specified @@ -207,11 +269,11 @@ virtio_user_notify_queue(struct virtio_hw *hw, struct virtqueue *vq) } if (write(dev->kickfds[vq->vq_queue_index], &buf, sizeof(buf)) < 0) - PMD_DRV_LOG(ERR, "failed to kick backend: %s\n", + PMD_DRV_LOG(ERR, "failed to kick backend: %s", strerror(errno)); } -static const struct virtio_pci_ops virtio_user_ops = { +const struct virtio_pci_ops virtio_user_ops = { .read_dev_cfg = virtio_user_read_dev_config, .write_dev_cfg = virtio_user_write_dev_config, .reset = virtio_user_reset, @@ -221,6 +283,7 @@ static const struct virtio_pci_ops virtio_user_ops = { .set_features = virtio_user_set_features, .get_isr = virtio_user_get_isr, .set_config_irq = virtio_user_set_config_irq, + .set_queue_irq = virtio_user_set_queue_irq, .get_queue_num = virtio_user_get_queue_num, .setup_queue = virtio_user_setup_queue, .del_queue = virtio_user_del_queue, @@ -238,6 +301,8 @@ static const char *valid_args[] = { VIRTIO_USER_ARG_PATH, #define VIRTIO_USER_ARG_QUEUE_SIZE "queue_size" VIRTIO_USER_ARG_QUEUE_SIZE, +#define VIRTIO_USER_ARG_INTERFACE_NAME "iface" + VIRTIO_USER_ARG_INTERFACE_NAME, NULL }; @@ -254,6 +319,9 @@ get_string_arg(const char *key __rte_unused, *(char **)extra_args = strdup(value); + if (!*(char **)extra_args) + return -ENOMEM; + return 0; } @@ -269,28 +337,24 @@ get_integer_arg(const char *key __rte_unused, return 0; } +static struct rte_vdev_driver virtio_user_driver; + static struct rte_eth_dev * -virtio_user_eth_dev_alloc(const char *name) +virtio_user_eth_dev_alloc(struct rte_vdev_device *vdev) { struct rte_eth_dev *eth_dev; struct rte_eth_dev_data *data; struct virtio_hw *hw; struct virtio_user_dev *dev; - eth_dev = rte_eth_dev_allocate(name, RTE_ETH_DEV_VIRTUAL); + eth_dev = rte_eth_vdev_allocate(vdev, sizeof(*hw)); if (!eth_dev) { PMD_INIT_LOG(ERR, "cannot alloc rte_eth_dev"); return NULL; } data = eth_dev->data; - - hw = rte_zmalloc(NULL, sizeof(*hw), 0); - if (!hw) { - PMD_INIT_LOG(ERR, "malloc virtio_hw failed"); - rte_eth_dev_release_port(eth_dev); - return NULL; - } + hw = eth_dev->data->dev_private; dev = rte_zmalloc(NULL, sizeof(*dev), 0); if (!dev) { @@ -300,16 +364,18 @@ virtio_user_eth_dev_alloc(const char *name) return NULL; } - hw->vtpci_ops = &virtio_user_ops; - hw->use_msix = 0; + hw->port_id = data->port_id; + dev->port_id = data->port_id; + virtio_hw_internal[hw->port_id].vtpci_ops = &virtio_user_ops; + /* + * MSIX is required to enable LSC (see virtio_init_device). + * Here just pretend that we support msix. + */ + hw->use_msix = 1; hw->modern = 0; + hw->use_simple_rx = 0; + hw->use_simple_tx = 0; hw->virtio_user_dev = dev; - data->dev_private = hw; - data->numa_node = SOCKET_ID_ANY; - data->kdrv = RTE_KDRV_NONE; - data->dev_flags = RTE_ETH_DEV_DETACHABLE; - eth_dev->pci_dev = NULL; - eth_dev->driver = NULL; return eth_dev; } @@ -325,11 +391,11 @@ virtio_user_eth_dev_free(struct rte_eth_dev *eth_dev) } /* Dev initialization routine. Invoked once for each virtio vdev at - * EAL init time, see rte_eal_dev_init(). + * EAL init time, see rte_bus_probe(). * Returns 0 on success. */ static int -virtio_user_pmd_devinit(const char *name, const char *params) +virtio_user_pmd_probe(struct rte_vdev_device *dev) { struct rte_kvargs *kvlist = NULL; struct rte_eth_dev *eth_dev; @@ -338,16 +404,11 @@ virtio_user_pmd_devinit(const char *name, const char *params) uint64_t cq = VIRTIO_USER_DEF_CQ_EN; uint64_t queue_size = VIRTIO_USER_DEF_Q_SZ; char *path = NULL; + char *ifname = NULL; char *mac_addr = NULL; int ret = -1; - if (!params || params[0] == '\0') { - PMD_INIT_LOG(ERR, "arg %s is mandatory for virtio_user", - VIRTIO_USER_ARG_QUEUE_SIZE); - goto end; - } - - kvlist = rte_kvargs_parse(params, valid_args); + kvlist = rte_kvargs_parse(rte_vdev_device_args(dev), valid_args); if (!kvlist) { PMD_INIT_LOG(ERR, "error when parsing param"); goto end; @@ -361,11 +422,27 @@ virtio_user_pmd_devinit(const char *name, const char *params) goto end; } } else { - PMD_INIT_LOG(ERR, "arg %s is mandatory for virtio_user\n", + PMD_INIT_LOG(ERR, "arg %s is mandatory for virtio_user", VIRTIO_USER_ARG_QUEUE_SIZE); goto end; } + if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_INTERFACE_NAME) == 1) { + if (is_vhost_user_by_type(path)) { + PMD_INIT_LOG(ERR, + "arg %s applies only to vhost-kernel backend", + VIRTIO_USER_ARG_INTERFACE_NAME); + goto end; + } + + if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_INTERFACE_NAME, + &get_string_arg, &ifname) < 0) { + PMD_INIT_LOG(ERR, "error to parse %s", + VIRTIO_USER_ARG_INTERFACE_NAME); + goto end; + } + } + if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_MAC) == 1) { if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_MAC, &get_string_arg, &mac_addr) < 0) { @@ -409,21 +486,34 @@ virtio_user_pmd_devinit(const char *name, const char *params) goto end; } - eth_dev = virtio_user_eth_dev_alloc(name); - if (!eth_dev) { - PMD_INIT_LOG(ERR, "virtio_user fails to alloc device"); + if (queues > VIRTIO_MAX_VIRTQUEUE_PAIRS) { + PMD_INIT_LOG(ERR, "arg %s %" PRIu64 " exceeds the limit %u", + VIRTIO_USER_ARG_QUEUES_NUM, queues, + VIRTIO_MAX_VIRTQUEUE_PAIRS); goto end; } - hw = eth_dev->data->dev_private; - if (virtio_user_dev_init(hw->virtio_user_dev, path, queues, cq, - queue_size, mac_addr) < 0) { - PMD_INIT_LOG(ERR, "virtio_user_dev_init fails"); - virtio_user_eth_dev_free(eth_dev); - goto end; + if (rte_eal_process_type() == RTE_PROC_PRIMARY) { + eth_dev = virtio_user_eth_dev_alloc(dev); + if (!eth_dev) { + PMD_INIT_LOG(ERR, "virtio_user fails to alloc device"); + goto end; + } + + hw = eth_dev->data->dev_private; + if (virtio_user_dev_init(hw->virtio_user_dev, path, queues, cq, + queue_size, mac_addr, &ifname) < 0) { + PMD_INIT_LOG(ERR, "virtio_user_dev_init fails"); + virtio_user_eth_dev_free(eth_dev); + goto end; + } + } else { + eth_dev = rte_eth_dev_attach_secondary(rte_vdev_device_name(dev)); + if (!eth_dev) + goto end; } - /* previously called by rte_eal_pci_probe() for physical dev */ + /* previously called by rte_pci_probe() for physical dev */ if (eth_virtio_dev_init(eth_dev) < 0) { PMD_INIT_LOG(ERR, "eth_virtio_dev_init fails"); virtio_user_eth_dev_free(eth_dev); @@ -438,21 +528,25 @@ end: free(path); if (mac_addr) free(mac_addr); + if (ifname) + free(ifname); return ret; } /** Called by rte_eth_dev_detach() */ static int -virtio_user_pmd_devuninit(const char *name) +virtio_user_pmd_remove(struct rte_vdev_device *vdev) { + const char *name; struct rte_eth_dev *eth_dev; struct virtio_hw *hw; struct virtio_user_dev *dev; - if (!name) + if (!vdev) return -EINVAL; - PMD_DRV_LOG(INFO, "Un-Initializing %s\n", name); + name = rte_vdev_device_name(vdev); + PMD_DRV_LOG(INFO, "Un-Initializing %s", name); eth_dev = rte_eth_dev_allocated(name); if (!eth_dev) return -ENODEV; @@ -465,22 +559,22 @@ virtio_user_pmd_devuninit(const char *name) virtio_user_dev_uninit(dev); rte_free(eth_dev->data->dev_private); - rte_free(eth_dev->data); rte_eth_dev_release_port(eth_dev); return 0; } -static struct rte_driver virtio_user_driver = { - .type = PMD_VDEV, - .init = virtio_user_pmd_devinit, - .uninit = virtio_user_pmd_devuninit, +static struct rte_vdev_driver virtio_user_driver = { + .probe = virtio_user_pmd_probe, + .remove = virtio_user_pmd_remove, }; -PMD_REGISTER_DRIVER(virtio_user_driver, virtio_user); -DRIVER_REGISTER_PARAM_STRING(virtio_user, +RTE_PMD_REGISTER_VDEV(net_virtio_user, virtio_user_driver); +RTE_PMD_REGISTER_ALIAS(net_virtio_user, virtio_user); +RTE_PMD_REGISTER_PARAM_STRING(net_virtio_user, "path= " "mac= " "cq= " "queue_size= " - "queues="); + "queues= " + "iface="); diff --git a/dpdk/drivers/net/virtio/virtqueue.c b/dpdk/drivers/net/virtio/virtqueue.c index 7f60e3ef..7f5e996a 100644 --- a/dpdk/drivers/net/virtio/virtqueue.c +++ b/dpdk/drivers/net/virtio/virtqueue.c @@ -37,17 +37,7 @@ #include "virtqueue.h" #include "virtio_logs.h" #include "virtio_pci.h" - -void -virtqueue_disable_intr(struct virtqueue *vq) -{ - /* - * Set VRING_AVAIL_F_NO_INTERRUPT to hint host - * not to interrupt when it consumes packets - * Note: this is only considered a hint to the host - */ - vq->vq_ring.avail->flags |= VRING_AVAIL_F_NO_INTERRUPT; -} +#include "virtio_rxtx_simple.h" /* * Two types of mbuf to be cleaned: @@ -58,15 +48,78 @@ struct rte_mbuf * virtqueue_detatch_unused(struct virtqueue *vq) { struct rte_mbuf *cookie; - int idx; + struct virtio_hw *hw; + uint16_t start, end; + int type, idx; - if (vq != NULL) - for (idx = 0; idx < vq->vq_nentries; idx++) { + if (vq == NULL) + return NULL; + + hw = vq->hw; + type = virtio_get_queue_type(hw, vq->vq_queue_index); + start = vq->vq_avail_idx & (vq->vq_nentries - 1); + end = (vq->vq_avail_idx + vq->vq_free_cnt) & (vq->vq_nentries - 1); + + for (idx = 0; idx < vq->vq_nentries; idx++) { + if (hw->use_simple_rx && type == VTNET_RQ) { + if (start <= end && idx >= start && idx < end) + continue; + if (start > end && (idx >= start || idx < end)) + continue; + cookie = vq->sw_ring[idx]; + if (cookie != NULL) { + vq->sw_ring[idx] = NULL; + return cookie; + } + } else { cookie = vq->vq_descx[idx].cookie; if (cookie != NULL) { vq->vq_descx[idx].cookie = NULL; return cookie; } } + } + return NULL; } + +/* Flush the elements in the used ring. */ +void +virtqueue_rxvq_flush(struct virtqueue *vq) +{ + struct virtnet_rx *rxq = &vq->rxq; + struct virtio_hw *hw = vq->hw; + struct vring_used_elem *uep; + struct vq_desc_extra *dxp; + uint16_t used_idx, desc_idx; + uint16_t nb_used, i; + + nb_used = VIRTQUEUE_NUSED(vq); + + for (i = 0; i < nb_used; i++) { + used_idx = vq->vq_used_cons_idx & (vq->vq_nentries - 1); + uep = &vq->vq_ring.used->ring[used_idx]; + if (hw->use_simple_rx) { + desc_idx = used_idx; + rte_pktmbuf_free(vq->sw_ring[desc_idx]); + vq->vq_free_cnt++; + } else { + desc_idx = (uint16_t)uep->id; + dxp = &vq->vq_descx[desc_idx]; + if (dxp->cookie != NULL) { + rte_pktmbuf_free(dxp->cookie); + dxp->cookie = NULL; + } + vq_ring_free_chain(vq, desc_idx); + } + vq->vq_used_cons_idx++; + } + + if (hw->use_simple_rx) { + while (vq->vq_free_cnt >= RTE_VIRTIO_VPMD_RX_REARM_THRESH) { + virtio_rxq_rearm_vec(rxq); + if (virtqueue_kick_prepare(vq)) + virtqueue_notify(vq); + } + } +} diff --git a/dpdk/drivers/net/virtio/virtqueue.h b/dpdk/drivers/net/virtio/virtqueue.h index 6737b81d..788d04df 100644 --- a/dpdk/drivers/net/virtio/virtqueue.h +++ b/dpdk/drivers/net/virtio/virtqueue.h @@ -38,12 +38,12 @@ #include #include -#include #include #include "virtio_pci.h" #include "virtio_ring.h" #include "virtio_logs.h" +#include "virtio_rxtx.h" struct rte_mbuf; @@ -70,10 +70,16 @@ struct rte_mbuf; /** * Return the physical address (or virtual address in case of * virtio-user) of mbuf data buffer. + * + * The address is firstly casted to the word size (sizeof(uintptr_t)) + * before casting it to uint64_t. This is to make it work with different + * combination of word size (64 bit and 32 bit) and virtio device + * (virtio-pci and virtio-user). */ -#define VIRTIO_MBUF_ADDR(mb, vq) (*(uint64_t *)((uintptr_t)(mb) + (vq)->offset)) +#define VIRTIO_MBUF_ADDR(mb, vq) \ + ((uint64_t)(*(uintptr_t *)((uintptr_t)(mb) + (vq)->offset))) #else -#define VIRTIO_MBUF_ADDR(mb, vq) ((mb)->buf_physaddr) +#define VIRTIO_MBUF_ADDR(mb, vq) ((mb)->buf_iova) #endif /** @@ -136,8 +142,8 @@ struct virtio_net_ctrl_mac { } __attribute__((__packed__)); #define VIRTIO_NET_CTRL_MAC 1 - #define VIRTIO_NET_CTRL_MAC_TABLE_SET 0 - #define VIRTIO_NET_CTRL_MAC_ADDR_SET 1 +#define VIRTIO_NET_CTRL_MAC_TABLE_SET 0 +#define VIRTIO_NET_CTRL_MAC_ADDR_SET 1 /** * Control VLAN filtering @@ -191,8 +197,14 @@ struct virtqueue { void *vq_ring_virt_mem; /**< linear address of vring*/ unsigned int vq_ring_size; - phys_addr_t vq_ring_mem; /**< physical address of vring, - * or virtual address for virtio_user. */ + union { + struct virtnet_rx rxq; + struct virtnet_tx txq; + struct virtnet_ctl cq; + }; + + rte_iova_t vq_ring_mem; /**< physical address of vring, + * or virtual address for virtio_user. */ /** * Head of the free chain in the descriptor table. If @@ -204,7 +216,6 @@ struct virtqueue { uint16_t vq_queue_index; /**< PCI queue index */ uint16_t offset; /**< relative offset to obtain addr in mbuf */ uint16_t *notify_addr; - int configured; struct rte_mbuf **sw_ring; /**< RX software ring. */ struct vq_desc_extra vq_descx[0]; }; @@ -223,6 +234,7 @@ struct virtqueue { */ struct virtio_net_hdr { #define VIRTIO_NET_HDR_F_NEEDS_CSUM 1 /**< Use csum_start,csum_offset*/ +#define VIRTIO_NET_HDR_F_DATA_VALID 2 /**< Checksum is valid */ uint8_t flags; #define VIRTIO_NET_HDR_GSO_NONE 0 /**< Not a GSO frame */ #define VIRTIO_NET_HDR_GSO_TCPV4 1 /**< GSO frame, IPv4 TCP (TSO) */ @@ -267,7 +279,21 @@ vring_desc_init(struct vring_desc *dp, uint16_t n) /** * Tell the backend not to interrupt us. */ -void virtqueue_disable_intr(struct virtqueue *vq); +static inline void +virtqueue_disable_intr(struct virtqueue *vq) +{ + vq->vq_ring.avail->flags |= VRING_AVAIL_F_NO_INTERRUPT; +} + +/** + * Tell the backend to interrupt us. + */ +static inline void +virtqueue_enable_intr(struct virtqueue *vq) +{ + vq->vq_ring.avail->flags &= (~VRING_AVAIL_F_NO_INTERRUPT); +} + /** * Dump virtqueue internal structures, for debug purpose only. */ @@ -277,14 +303,30 @@ void virtqueue_dump(struct virtqueue *vq); */ struct rte_mbuf *virtqueue_detatch_unused(struct virtqueue *vq); +/* Flush the elements in the used ring. */ +void virtqueue_rxvq_flush(struct virtqueue *vq); + static inline int virtqueue_full(const struct virtqueue *vq) { return vq->vq_free_cnt == 0; } +static inline int +virtio_get_queue_type(struct virtio_hw *hw, uint16_t vtpci_queue_idx) +{ + if (vtpci_queue_idx == hw->max_queue_pairs * 2) + return VTNET_CQ; + else if (vtpci_queue_idx % 2 == 0) + return VTNET_RQ; + else + return VTNET_TQ; +} + #define VIRTQUEUE_NUSED(vq) ((uint16_t)((vq)->vq_ring.used->idx - (vq)->vq_used_cons_idx)) +void vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx); + static inline void vq_update_avail_idx(struct virtqueue *vq) { @@ -323,7 +365,7 @@ virtqueue_notify(struct virtqueue *vq) * For virtio on IA, the notificaiton is through io port operation * which is a serialization instruction itself. */ - vq->hw->vtpci_ops->notify_queue(vq->hw, vq); + VTPCI_OPS(vq->hw)->notify_queue(vq->hw, vq); } #ifdef RTE_LIBRTE_VIRTIO_DEBUG_DUMP diff --git a/dpdk/drivers/net/vmxnet3/Makefile b/dpdk/drivers/net/vmxnet3/Makefile index 23ff1da2..f09de96e 100644 --- a/dpdk/drivers/net/vmxnet3/Makefile +++ b/dpdk/drivers/net/vmxnet3/Makefile @@ -63,6 +63,9 @@ CFLAGS_BASE_DRIVER = -Wno-unused-parameter -Wno-unused-value CFLAGS_BASE_DRIVER += -Wno-strict-aliasing -Wno-format-extra-args endif +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs +LDLIBS += -lrte_bus_pci VPATH += $(SRCDIR)/base @@ -76,9 +79,4 @@ LIBABIVER := 1 SRCS-$(CONFIG_RTE_LIBRTE_VMXNET3_PMD) += vmxnet3_rxtx.c SRCS-$(CONFIG_RTE_LIBRTE_VMXNET3_PMD) += vmxnet3_ethdev.c -# this lib depends upon: -DEPDIRS-$(CONFIG_RTE_LIBRTE_VMXNET3_PMD) += lib/librte_eal lib/librte_ether -DEPDIRS-$(CONFIG_RTE_LIBRTE_VMXNET3_PMD) += lib/librte_mempool lib/librte_mbuf -DEPDIRS-$(CONFIG_RTE_LIBRTE_VMXNET3_PMD) += lib/librte_net - include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/net/vmxnet3/base/vmxnet3_defs.h b/dpdk/drivers/net/vmxnet3/base/vmxnet3_defs.h index 68ae8b6d..bfa9622d 100644 --- a/dpdk/drivers/net/vmxnet3/base/vmxnet3_defs.h +++ b/dpdk/drivers/net/vmxnet3/base/vmxnet3_defs.h @@ -109,6 +109,9 @@ typedef enum { VMXNET3_CMD_STOP_EMULATION, VMXNET3_CMD_LOAD_PLUGIN, VMXNET3_CMD_ACTIVATE_VF, + VMXNET3_CMD_RESERVED3, + VMXNET3_CMD_RESERVED4, + VMXNET3_CMD_REGISTER_MEMREGS, VMXNET3_CMD_FIRST_GET = 0xF00D0000, VMXNET3_CMD_GET_QUEUE_STATUS = VMXNET3_CMD_FIRST_GET, @@ -120,7 +123,9 @@ typedef enum { VMXNET3_CMD_GET_DID_HI, VMXNET3_CMD_GET_DEV_EXTRA_INFO, VMXNET3_CMD_GET_CONF_INTR, - VMXNET3_CMD_GET_ADAPTIVE_RING_INFO + VMXNET3_CMD_GET_ADAPTIVE_RING_INFO, + VMXNET3_CMD_GET_TXDATA_DESC_SIZE, + VMXNET3_CMD_RESERVED5, } Vmxnet3_Cmd; /* Adaptive Ring Info Flags */ @@ -402,12 +407,25 @@ typedef union Vmxnet3_GenericDesc { #define VMXNET3_RING_SIZE_ALIGN 32 #define VMXNET3_RING_SIZE_MASK (VMXNET3_RING_SIZE_ALIGN - 1) +/* Tx Data Ring buffer size must be a multiple of 64 */ +#define VMXNET3_TXDATA_DESC_SIZE_ALIGN 64 +#define VMXNET3_TXDATA_DESC_SIZE_MASK (VMXNET3_TXDATA_DESC_SIZE_ALIGN - 1) + +/* Rx Data Ring buffer size must be a multiple of 64 */ +#define VMXNET3_RXDATA_DESC_SIZE_ALIGN 64 +#define VMXNET3_RXDATA_DESC_SIZE_MASK (VMXNET3_RXDATA_DESC_SIZE_ALIGN - 1) + /* Max ring size */ #define VMXNET3_TX_RING_MAX_SIZE 4096 #define VMXNET3_TC_RING_MAX_SIZE 4096 #define VMXNET3_RX_RING_MAX_SIZE 4096 #define VMXNET3_RC_RING_MAX_SIZE 8192 +#define VMXNET3_TXDATA_DESC_MIN_SIZE 128 +#define VMXNET3_TXDATA_DESC_MAX_SIZE 2048 + +#define VMXNET3_RXDATA_DESC_MAX_SIZE 2048 + /* a list of reasons for queue stop */ #define VMXNET3_ERR_NOEOP 0x80000000 /* cannot find the EOP desc of a pkt */ @@ -507,7 +525,9 @@ struct Vmxnet3_TxQueueConf { __le32 compRingSize; /* # of comp desc */ __le32 ddLen; /* size of driver data */ uint8 intrIdx; - uint8 _pad[7]; + uint8 _pad[1]; + __le16 txDataRingDescSize; + uint8 _pad2[4]; } #include "vmware_pack_end.h" Vmxnet3_TxQueueConf; @@ -518,12 +538,14 @@ struct Vmxnet3_RxQueueConf { __le64 rxRingBasePA[2]; __le64 compRingBasePA; __le64 ddPA; /* driver data */ - __le64 reserved; + __le64 rxDataRingBasePA; __le32 rxRingSize[2]; /* # of rx desc */ __le32 compRingSize; /* # of rx comp desc */ __le32 ddLen; /* size of driver data */ uint8 intrIdx; - uint8 _pad[7]; + uint8 _pad1[1]; + __le16 rxDataRingDescSize; /* size of rx data ring buffer */ + uint8 _pad2[4]; } #include "vmware_pack_end.h" Vmxnet3_RxQueueConf; @@ -693,6 +715,52 @@ struct Vmxnet3_RxQueueDesc { #include "vmware_pack_end.h" Vmxnet3_RxQueueDesc; +typedef +#include "vmware_pack_begin.h" +struct Vmxnet3_SetPolling { + uint8 enablePolling; +} +#include "vmware_pack_end.h" +Vmxnet3_SetPolling; + +typedef +#include "vmware_pack_begin.h" +struct Vmxnet3_MemoryRegion { + __le64 startPA; + __le32 length; + __le16 txQueueBits; /* bit n corresponding to tx queue n */ + __le16 rxQueueBits; /* bit n corresponding to rx queue n */ +} +#include "vmware_pack_end.h" +Vmxnet3_MemoryRegion; + +#define MAX_MEMORY_REGION_PER_QUEUE 16 +#define MAX_MEMORY_REGION_PER_DEVICE 256 + +typedef +#include "vmware_pack_begin.h" +struct Vmxnet3_MemRegs { + __le16 numRegs; + __le16 pad[3]; + Vmxnet3_MemoryRegion memRegs[1]; +} +#include "vmware_pack_end.h" +Vmxnet3_MemRegs; + +/* + * If the command data <= 16 bytes, use the shared memory direcly. + * Otherwise, use the variable length configuration descriptor. + */ +typedef +#include "vmware_pack_begin.h" +union Vmxnet3_CmdInfo { + Vmxnet3_VariableLenConfDesc varConf; + Vmxnet3_SetPolling setPolling; + __le64 data[2]; +} +#include "vmware_pack_end.h" +Vmxnet3_CmdInfo; + typedef #include "vmware_pack_begin.h" struct Vmxnet3_DriverShared { @@ -700,7 +768,14 @@ struct Vmxnet3_DriverShared { __le32 pad; /* make devRead start at 64-bit boundaries */ Vmxnet3_DSDevRead devRead; __le32 ecr; - __le32 reserved[5]; + __le32 reserved; + + union { + __le32 reserved1[4]; + Vmxnet3_CmdInfo cmdInfo; /* only valid in the context of executing the + * relevant command + */ + } cu; } #include "vmware_pack_end.h" Vmxnet3_DriverShared; diff --git a/dpdk/drivers/net/vmxnet3/vmxnet3_ethdev.c b/dpdk/drivers/net/vmxnet3/vmxnet3_ethdev.c index 58742153..93d96499 100644 --- a/dpdk/drivers/net/vmxnet3/vmxnet3_ethdev.c +++ b/dpdk/drivers/net/vmxnet3/vmxnet3_ethdev.c @@ -48,6 +48,7 @@ #include #include #include +#include #include #include #include @@ -56,7 +57,7 @@ #include #include #include -#include +#include #include #include #include @@ -69,6 +70,8 @@ #define PROCESS_SYS_EVENTS 0 +#define VMXNET3_TX_MAX_SEG UINT8_MAX + static int eth_vmxnet3_dev_init(struct rte_eth_dev *eth_dev); static int eth_vmxnet3_dev_uninit(struct rte_eth_dev *eth_dev); static int vmxnet3_dev_configure(struct rte_eth_dev *dev); @@ -80,23 +83,29 @@ static void vmxnet3_dev_promiscuous_enable(struct rte_eth_dev *dev); static void vmxnet3_dev_promiscuous_disable(struct rte_eth_dev *dev); static void vmxnet3_dev_allmulticast_enable(struct rte_eth_dev *dev); static void vmxnet3_dev_allmulticast_disable(struct rte_eth_dev *dev); +static int __vmxnet3_dev_link_update(struct rte_eth_dev *dev, + int wait_to_complete); static int vmxnet3_dev_link_update(struct rte_eth_dev *dev, - int wait_to_complete); -static void vmxnet3_dev_stats_get(struct rte_eth_dev *dev, - struct rte_eth_stats *stats); + int wait_to_complete); +static void vmxnet3_hw_stats_save(struct vmxnet3_hw *hw); +static int vmxnet3_dev_stats_get(struct rte_eth_dev *dev, + struct rte_eth_stats *stats); +static int vmxnet3_dev_xstats_get_names(struct rte_eth_dev *dev, + struct rte_eth_xstat_name *xstats, + unsigned int n); +static int vmxnet3_dev_xstats_get(struct rte_eth_dev *dev, + struct rte_eth_xstat *xstats, unsigned int n); static void vmxnet3_dev_info_get(struct rte_eth_dev *dev, - struct rte_eth_dev_info *dev_info); + struct rte_eth_dev_info *dev_info); static const uint32_t * vmxnet3_dev_supported_ptypes_get(struct rte_eth_dev *dev); static int vmxnet3_dev_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vid, int on); -static void vmxnet3_dev_vlan_offload_set(struct rte_eth_dev *dev, int mask); +static int vmxnet3_dev_vlan_offload_set(struct rte_eth_dev *dev, int mask); static void vmxnet3_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr); +static void vmxnet3_interrupt_handler(void *param); -#if PROCESS_SYS_EVENTS == 1 -static void vmxnet3_process_events(struct vmxnet3_hw *); -#endif /* * The set of PCI devices this driver supports */ @@ -118,7 +127,9 @@ static const struct eth_dev_ops vmxnet3_eth_dev_ops = { .allmulticast_disable = vmxnet3_dev_allmulticast_disable, .link_update = vmxnet3_dev_link_update, .stats_get = vmxnet3_dev_stats_get, - .mac_addr_set = vmxnet3_mac_addr_set, + .xstats_get_names = vmxnet3_dev_xstats_get_names, + .xstats_get = vmxnet3_dev_xstats_get, + .mac_addr_set = vmxnet3_mac_addr_set, .dev_infos_get = vmxnet3_dev_info_get, .dev_supported_ptypes_get = vmxnet3_dev_supported_ptypes_get, .vlan_filter_set = vmxnet3_dev_vlan_filter_set, @@ -129,22 +140,50 @@ static const struct eth_dev_ops vmxnet3_eth_dev_ops = { .tx_queue_release = vmxnet3_dev_tx_queue_release, }; +struct vmxnet3_xstats_name_off { + char name[RTE_ETH_XSTATS_NAME_SIZE]; + unsigned int offset; +}; + +/* tx_qX_ is prepended to the name string here */ +static const struct vmxnet3_xstats_name_off vmxnet3_txq_stat_strings[] = { + {"drop_total", offsetof(struct vmxnet3_txq_stats, drop_total)}, + {"drop_too_many_segs", offsetof(struct vmxnet3_txq_stats, drop_too_many_segs)}, + {"drop_tso", offsetof(struct vmxnet3_txq_stats, drop_tso)}, + {"tx_ring_full", offsetof(struct vmxnet3_txq_stats, tx_ring_full)}, +}; + +/* rx_qX_ is prepended to the name string here */ +static const struct vmxnet3_xstats_name_off vmxnet3_rxq_stat_strings[] = { + {"drop_total", offsetof(struct vmxnet3_rxq_stats, drop_total)}, + {"drop_err", offsetof(struct vmxnet3_rxq_stats, drop_err)}, + {"drop_fcs", offsetof(struct vmxnet3_rxq_stats, drop_fcs)}, + {"rx_buf_alloc_failure", offsetof(struct vmxnet3_rxq_stats, rx_buf_alloc_failure)}, +}; + static const struct rte_memzone * gpa_zone_reserve(struct rte_eth_dev *dev, uint32_t size, - const char *post_string, int socket_id, uint16_t align) + const char *post_string, int socket_id, + uint16_t align, bool reuse) { char z_name[RTE_MEMZONE_NAMESIZE]; const struct rte_memzone *mz; snprintf(z_name, sizeof(z_name), "%s_%d_%s", - dev->driver->pci_drv.name, dev->data->port_id, post_string); + dev->device->driver->name, dev->data->port_id, post_string); mz = rte_memzone_lookup(z_name); + if (!reuse) { + if (mz) + rte_memzone_free(mz); + return rte_memzone_reserve_aligned(z_name, size, socket_id, + 0, align); + } + if (mz) return mz; - return rte_memzone_reserve_aligned(z_name, size, - socket_id, 0, align); + return rte_memzone_reserve_aligned(z_name, size, socket_id, 0, align); } /** @@ -194,7 +233,7 @@ vmxnet3_dev_atomic_write_link_status(struct rte_eth_dev *dev, struct rte_eth_link *src = link; if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst, - *(uint64_t *)src) == 0) + *(uint64_t *)src) == 0) return -1; return 0; @@ -211,8 +250,38 @@ vmxnet3_disable_intr(struct vmxnet3_hw *hw) PMD_INIT_FUNC_TRACE(); hw->shared->devRead.intrConf.intrCtrl |= VMXNET3_IC_DISABLE_ALL; - for (i = 0; i < VMXNET3_MAX_INTRS; i++) - VMXNET3_WRITE_BAR0_REG(hw, VMXNET3_REG_IMR + i * 8, 1); + for (i = 0; i < hw->num_intrs; i++) + VMXNET3_WRITE_BAR0_REG(hw, VMXNET3_REG_IMR + i * 8, 1); +} + +static void +vmxnet3_enable_intr(struct vmxnet3_hw *hw) +{ + int i; + + PMD_INIT_FUNC_TRACE(); + + hw->shared->devRead.intrConf.intrCtrl &= ~VMXNET3_IC_DISABLE_ALL; + for (i = 0; i < hw->num_intrs; i++) + VMXNET3_WRITE_BAR0_REG(hw, VMXNET3_REG_IMR + i * 8, 0); +} + +/* + * Gets tx data ring descriptor size. + */ +static uint16_t +eth_vmxnet3_txdata_get(struct vmxnet3_hw *hw) +{ + uint16 txdata_desc_size; + + VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, + VMXNET3_CMD_GET_TXDATA_DESC_SIZE); + txdata_desc_size = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_CMD); + + return (txdata_desc_size < VMXNET3_TXDATA_DESC_MIN_SIZE || + txdata_desc_size > VMXNET3_TXDATA_DESC_MAX_SIZE || + txdata_desc_size & VMXNET3_TXDATA_DESC_SIZE_MASK) ? + sizeof(struct Vmxnet3_TxDataDesc) : txdata_desc_size; } /* @@ -230,7 +299,8 @@ eth_vmxnet3_dev_init(struct rte_eth_dev *eth_dev) eth_dev->dev_ops = &vmxnet3_eth_dev_ops; eth_dev->rx_pkt_burst = &vmxnet3_recv_pkts; eth_dev->tx_pkt_burst = &vmxnet3_xmit_pkts; - pci_dev = eth_dev->pci_dev; + eth_dev->tx_pkt_prepare = vmxnet3_prep_pkts; + pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); /* * for secondary processes, we don't initialize any further as primary @@ -254,13 +324,26 @@ eth_vmxnet3_dev_init(struct rte_eth_dev *eth_dev) /* Check h/w version compatibility with driver. */ ver = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_VRRS); PMD_INIT_LOG(DEBUG, "Hardware version : %d", ver); - if (ver & 0x1) - VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_VRRS, 1); - else { - PMD_INIT_LOG(ERR, "Incompatible h/w version, should be 0x1"); + + if (ver & (1 << VMXNET3_REV_3)) { + VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_VRRS, + 1 << VMXNET3_REV_3); + hw->version = VMXNET3_REV_3 + 1; + } else if (ver & (1 << VMXNET3_REV_2)) { + VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_VRRS, + 1 << VMXNET3_REV_2); + hw->version = VMXNET3_REV_2 + 1; + } else if (ver & (1 << VMXNET3_REV_1)) { + VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_VRRS, + 1 << VMXNET3_REV_1); + hw->version = VMXNET3_REV_1 + 1; + } else { + PMD_INIT_LOG(ERR, "Incompatible hardware version: %d", ver); return -EIO; } + PMD_INIT_LOG(DEBUG, "Using device version %d\n", hw->version); + /* Check UPT version compatibility with driver. */ ver = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_UVRS); PMD_INIT_LOG(DEBUG, "UPT hardware version : %d", ver); @@ -274,8 +357,8 @@ eth_vmxnet3_dev_init(struct rte_eth_dev *eth_dev) /* Getting MAC Address */ mac_lo = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_MACL); mac_hi = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_MACH); - memcpy(hw->perm_addr , &mac_lo, 4); - memcpy(hw->perm_addr+4, &mac_hi, 2); + memcpy(hw->perm_addr, &mac_lo, 4); + memcpy(hw->perm_addr + 4, &mac_hi, 2); /* Allocate memory for storing MAC addresses */ eth_dev->data->mac_addrs = rte_zmalloc("vmxnet3", ETHER_ADDR_LEN * @@ -300,6 +383,18 @@ eth_vmxnet3_dev_init(struct rte_eth_dev *eth_dev) /* allow untagged pkts */ VMXNET3_SET_VFTABLE_ENTRY(hw->shadow_vfta, 0); + hw->txdata_desc_size = VMXNET3_VERSION_GE_3(hw) ? + eth_vmxnet3_txdata_get(hw) : sizeof(struct Vmxnet3_TxDataDesc); + + hw->rxdata_desc_size = VMXNET3_VERSION_GE_3(hw) ? + VMXNET3_DEF_RXDATA_DESC_SIZE : 0; + RTE_ASSERT((hw->rxdata_desc_size & ~VMXNET3_RXDATA_DESC_SIZE_MASK) == + hw->rxdata_desc_size); + + /* clear shadow stats */ + memset(hw->saved_tx_stats, 0, sizeof(hw->saved_tx_stats)); + memset(hw->saved_rx_stats, 0, sizeof(hw->saved_rx_stats)); + return 0; } @@ -319,6 +414,7 @@ eth_vmxnet3_dev_uninit(struct rte_eth_dev *eth_dev) eth_dev->dev_ops = NULL; eth_dev->rx_pkt_burst = NULL; eth_dev->tx_pkt_burst = NULL; + eth_dev->tx_pkt_prepare = NULL; rte_free(eth_dev->data->mac_addrs); eth_dev->data->mac_addrs = NULL; @@ -326,31 +422,25 @@ eth_vmxnet3_dev_uninit(struct rte_eth_dev *eth_dev) return 0; } -static struct eth_driver rte_vmxnet3_pmd = { - .pci_drv = { - .name = "rte_vmxnet3_pmd", - .id_table = pci_id_vmxnet3_map, - .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_DETACHABLE, - }, - .eth_dev_init = eth_vmxnet3_dev_init, - .eth_dev_uninit = eth_vmxnet3_dev_uninit, - .dev_private_size = sizeof(struct vmxnet3_hw), -}; - -/* - * Driver initialization routine. - * Invoked once at EAL init time. - * Register itself as the [Poll Mode] Driver of Virtual PCI VMXNET3 devices. - */ -static int -rte_vmxnet3_pmd_init(const char *name __rte_unused, const char *param __rte_unused) +static int eth_vmxnet3_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, + struct rte_pci_device *pci_dev) { - PMD_INIT_FUNC_TRACE(); - - rte_eth_driver_register(&rte_vmxnet3_pmd); - return 0; + return rte_eth_dev_pci_generic_probe(pci_dev, + sizeof(struct vmxnet3_hw), eth_vmxnet3_dev_init); } +static int eth_vmxnet3_pci_remove(struct rte_pci_device *pci_dev) +{ + return rte_eth_dev_pci_generic_remove(pci_dev, eth_vmxnet3_dev_uninit); +} + +static struct rte_pci_driver rte_vmxnet3_pmd = { + .id_table = pci_id_vmxnet3_map, + .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC, + .probe = eth_vmxnet3_pci_probe, + .remove = eth_vmxnet3_pci_remove, +}; + static int vmxnet3_dev_configure(struct rte_eth_dev *dev) { @@ -360,9 +450,16 @@ vmxnet3_dev_configure(struct rte_eth_dev *dev) PMD_INIT_FUNC_TRACE(); - if (dev->data->nb_rx_queues > UINT8_MAX || - dev->data->nb_tx_queues > UINT8_MAX) + if (dev->data->nb_tx_queues > VMXNET3_MAX_TX_QUEUES || + dev->data->nb_rx_queues > VMXNET3_MAX_RX_QUEUES) { + PMD_INIT_LOG(ERR, "ERROR: Number of queues not supported"); return -EINVAL; + } + + if (!rte_is_power_of_2(dev->data->nb_rx_queues)) { + PMD_INIT_LOG(ERR, "ERROR: Number of rx queues not power of 2"); + return -EINVAL; + } size = dev->data->nb_rx_queues * sizeof(struct Vmxnet3_TxQueueDesc) + dev->data->nb_tx_queues * sizeof(struct Vmxnet3_RxQueueDesc); @@ -378,7 +475,7 @@ vmxnet3_dev_configure(struct rte_eth_dev *dev) * on current socket */ mz = gpa_zone_reserve(dev, sizeof(struct Vmxnet3_DriverShared), - "shared", rte_socket_id(), 8); + "shared", rte_socket_id(), 8, 1); if (mz == NULL) { PMD_INIT_LOG(ERR, "ERROR: Creating shared zone"); @@ -387,14 +484,18 @@ vmxnet3_dev_configure(struct rte_eth_dev *dev) memset(mz->addr, 0, mz->len); hw->shared = mz->addr; - hw->sharedPA = mz->phys_addr; + hw->sharedPA = mz->iova; /* * Allocate a memzone for Vmxnet3_RxQueueDesc - Vmxnet3_TxQueueDesc - * on current socket + * on current socket. + * + * We cannot reuse this memzone from previous allocation as its size + * depends on the number of tx and rx queues, which could be different + * from one config to another. */ - mz = gpa_zone_reserve(dev, size, "queuedesc", - rte_socket_id(), VMXNET3_QUEUE_DESC_ALIGN); + mz = gpa_zone_reserve(dev, size, "queuedesc", rte_socket_id(), + VMXNET3_QUEUE_DESC_ALIGN, 0); if (mz == NULL) { PMD_INIT_LOG(ERR, "ERROR: Creating queue descriptors zone"); return -ENOMEM; @@ -404,14 +505,14 @@ vmxnet3_dev_configure(struct rte_eth_dev *dev) hw->tqd_start = (Vmxnet3_TxQueueDesc *)mz->addr; hw->rqd_start = (Vmxnet3_RxQueueDesc *)(hw->tqd_start + hw->num_tx_queues); - hw->queueDescPA = mz->phys_addr; + hw->queueDescPA = mz->iova; hw->queue_desc_len = (uint16_t)size; if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_RSS) { - /* Allocate memory structure for UPT1_RSSConf and configure */ - mz = gpa_zone_reserve(dev, sizeof(struct VMXNET3_RSSConf), "rss_conf", - rte_socket_id(), RTE_CACHE_LINE_SIZE); + mz = gpa_zone_reserve(dev, sizeof(struct VMXNET3_RSSConf), + "rss_conf", rte_socket_id(), + RTE_CACHE_LINE_SIZE, 1); if (mz == NULL) { PMD_INIT_LOG(ERR, "ERROR: Creating rss_conf structure zone"); @@ -420,7 +521,7 @@ vmxnet3_dev_configure(struct rte_eth_dev *dev) memset(mz->addr, 0, mz->len); hw->rss_conf = mz->addr; - hw->rss_confPA = mz->phys_addr; + hw->rss_confPA = mz->iova; } return 0; @@ -436,13 +537,99 @@ vmxnet3_write_mac(struct vmxnet3_hw *hw, const uint8_t *addr) addr[0], addr[1], addr[2], addr[3], addr[4], addr[5]); - val = *(const uint32_t *)addr; + memcpy(&val, addr, 4); VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_MACL, val); - val = (addr[5] << 8) | addr[4]; + memcpy(&val, addr + 4, 2); VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_MACH, val); } +static int +vmxnet3_dev_setup_memreg(struct rte_eth_dev *dev) +{ + struct vmxnet3_hw *hw = dev->data->dev_private; + Vmxnet3_DriverShared *shared = hw->shared; + Vmxnet3_CmdInfo *cmdInfo; + struct rte_mempool *mp[VMXNET3_MAX_RX_QUEUES]; + uint8_t index[VMXNET3_MAX_RX_QUEUES + VMXNET3_MAX_TX_QUEUES]; + uint32_t num, i, j, size; + + if (hw->memRegsPA == 0) { + const struct rte_memzone *mz; + + size = sizeof(Vmxnet3_MemRegs) + + (VMXNET3_MAX_RX_QUEUES + VMXNET3_MAX_TX_QUEUES) * + sizeof(Vmxnet3_MemoryRegion); + + mz = gpa_zone_reserve(dev, size, "memRegs", rte_socket_id(), 8, + 1); + if (mz == NULL) { + PMD_INIT_LOG(ERR, "ERROR: Creating memRegs zone"); + return -ENOMEM; + } + memset(mz->addr, 0, mz->len); + hw->memRegs = mz->addr; + hw->memRegsPA = mz->iova; + } + + num = hw->num_rx_queues; + + for (i = 0; i < num; i++) { + vmxnet3_rx_queue_t *rxq = dev->data->rx_queues[i]; + + mp[i] = rxq->mp; + index[i] = 1 << i; + } + + /* + * The same mempool could be used by multiple queues. In such a case, + * remove duplicate mempool entries. Only one entry is kept with + * bitmask indicating queues that are using this mempool. + */ + for (i = 1; i < num; i++) { + for (j = 0; j < i; j++) { + if (mp[i] == mp[j]) { + mp[i] = NULL; + index[j] |= 1 << i; + break; + } + } + } + + j = 0; + for (i = 0; i < num; i++) { + if (mp[i] == NULL) + continue; + + Vmxnet3_MemoryRegion *mr = &hw->memRegs->memRegs[j]; + + mr->startPA = + (uintptr_t)STAILQ_FIRST(&mp[i]->mem_list)->iova; + mr->length = STAILQ_FIRST(&mp[i]->mem_list)->len <= INT32_MAX ? + STAILQ_FIRST(&mp[i]->mem_list)->len : INT32_MAX; + mr->txQueueBits = index[i]; + mr->rxQueueBits = index[i]; + + PMD_INIT_LOG(INFO, + "index: %u startPA: %" PRIu64 " length: %u, " + "rxBits: %x", + j, mr->startPA, mr->length, mr->rxQueueBits); + j++; + } + hw->memRegs->numRegs = j; + PMD_INIT_LOG(INFO, "numRegs: %u", j); + + size = sizeof(Vmxnet3_MemRegs) + + (j - 1) * sizeof(Vmxnet3_MemoryRegion); + + cmdInfo = &shared->cu.cmdInfo; + cmdInfo->varConf.confVer = 1; + cmdInfo->varConf.confLen = size; + cmdInfo->varConf.confPA = hw->memRegsPA; + + return 0; +} + static int vmxnet3_setup_driver_shared(struct rte_eth_dev *dev) { @@ -459,8 +646,7 @@ vmxnet3_setup_driver_shared(struct rte_eth_dev *dev) /* Setting up Guest OS information */ devRead->misc.driverInfo.gos.gosBits = sizeof(void *) == 4 ? - VMXNET3_GOS_BITS_32 : - VMXNET3_GOS_BITS_64; + VMXNET3_GOS_BITS_32 : VMXNET3_GOS_BITS_64; devRead->misc.driverInfo.gos.gosType = VMXNET3_GOS_TYPE_LINUX; devRead->misc.driverInfo.vmxnet3RevSpt = 1; devRead->misc.driverInfo.uptVerSpt = 1; @@ -473,11 +659,11 @@ vmxnet3_setup_driver_shared(struct rte_eth_dev *dev) /* * Set number of interrupts to 1 - * PMD disables all the interrupts but this is MUST to activate device - * It needs at least one interrupt for link events to handle - * So we'll disable it later after device activation if needed + * PMD by default disables all the interrupts but this is MUST + * to activate device. It needs at least one interrupt for + * link events to handle */ - devRead->intrConf.numIntrs = 1; + hw->num_intrs = devRead->intrConf.numIntrs = 1; devRead->intrConf.intrCtrl |= VMXNET3_IC_DISABLE_ALL; for (i = 0; i < hw->num_tx_queues; i++) { @@ -493,6 +679,7 @@ vmxnet3_setup_driver_shared(struct rte_eth_dev *dev) tqd->conf.txRingSize = txq->cmd_ring.size; tqd->conf.compRingSize = txq->comp_ring.size; tqd->conf.dataRingSize = txq->data_ring.size; + tqd->conf.txDataRingDescSize = txq->txdata_desc_size; tqd->conf.intrIdx = txq->comp_ring.intr_idx; tqd->status.stopped = TRUE; tqd->status.error = 0; @@ -511,6 +698,10 @@ vmxnet3_setup_driver_shared(struct rte_eth_dev *dev) rqd->conf.rxRingSize[1] = rxq->cmd_ring[1].size; rqd->conf.compRingSize = rxq->comp_ring.size; rqd->conf.intrIdx = rxq->comp_ring.intr_idx; + if (VMXNET3_VERSION_GE_3(hw)) { + rqd->conf.rxDataRingBasePA = rxq->data_ring.basePA; + rqd->conf.rxDataRingDescSize = rxq->data_desc_size; + } rqd->status.stopped = TRUE; rqd->status.error = 0; memset(&rqd->stats, 0, sizeof(rqd->stats)); @@ -523,6 +714,11 @@ vmxnet3_setup_driver_shared(struct rte_eth_dev *dev) if (dev->data->dev_conf.rxmode.hw_ip_checksum) devRead->misc.uptFeatures |= VMXNET3_F_RXCSUM; + if (dev->data->dev_conf.rxmode.enable_lro) { + devRead->misc.uptFeatures |= VMXNET3_F_LRO; + devRead->misc.maxNumRxSG = 0; + } + if (port_conf.rxmode.mq_mode == ETH_MQ_RX_RSS) { ret = vmxnet3_rss_configure(dev); if (ret != VMXNET3_SUCCESS) @@ -534,10 +730,12 @@ vmxnet3_setup_driver_shared(struct rte_eth_dev *dev) devRead->rssConfDesc.confPA = hw->rss_confPA; } - vmxnet3_dev_vlan_offload_set(dev, - ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK); + ret = vmxnet3_dev_vlan_offload_set(dev, + ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK); + if (ret) + return ret; - vmxnet3_write_mac(hw, hw->perm_addr); + vmxnet3_write_mac(hw, dev->data->mac_addrs->addr_bytes); return VMXNET3_SUCCESS; } @@ -550,15 +748,32 @@ vmxnet3_setup_driver_shared(struct rte_eth_dev *dev) static int vmxnet3_dev_start(struct rte_eth_dev *dev) { - int status, ret; + int ret; struct vmxnet3_hw *hw = dev->data->dev_private; PMD_INIT_FUNC_TRACE(); + /* Save stats before it is reset by CMD_ACTIVATE */ + vmxnet3_hw_stats_save(hw); + ret = vmxnet3_setup_driver_shared(dev); if (ret != VMXNET3_SUCCESS) return ret; + /* check if lsc interrupt feature is enabled */ + if (dev->data->dev_conf.intr_conf.lsc) { + struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(dev->device); + + /* Setup interrupt callback */ + rte_intr_callback_register(&pci_dev->intr_handle, + vmxnet3_interrupt_handler, dev); + + if (rte_intr_enable(&pci_dev->intr_handle) < 0) { + PMD_INIT_LOG(ERR, "interrupt enable failed"); + return -EIO; + } + } + /* Exchange shared data with device */ VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_DSAL, VMXNET3_GET_ADDR_LO(hw->sharedPA)); @@ -567,11 +782,25 @@ vmxnet3_dev_start(struct rte_eth_dev *dev) /* Activate device by register write */ VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_ACTIVATE_DEV); - status = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_CMD); + ret = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_CMD); - if (status != 0) { + if (ret != 0) { PMD_INIT_LOG(ERR, "Device activation: UNSUCCESSFUL"); - return -1; + return -EINVAL; + } + + /* Setup memory region for rx buffers */ + ret = vmxnet3_dev_setup_memreg(dev); + if (ret == 0) { + VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, + VMXNET3_CMD_REGISTER_MEMREGS); + ret = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_CMD); + if (ret != 0) + PMD_INIT_LOG(DEBUG, + "Failed in setup memory region cmd\n"); + ret = 0; + } else { + PMD_INIT_LOG(DEBUG, "Failed to setup memory region\n"); } /* Disable interrupts */ @@ -583,22 +812,29 @@ vmxnet3_dev_start(struct rte_eth_dev *dev) */ ret = vmxnet3_dev_rxtx_init(dev); if (ret != VMXNET3_SUCCESS) { - PMD_INIT_LOG(ERR, "Device receive init: UNSUCCESSFUL"); + PMD_INIT_LOG(ERR, "Device queue init: UNSUCCESSFUL"); return ret; } + hw->adapter_stopped = FALSE; + /* Setting proper Rx Mode and issue Rx Mode Update command */ vmxnet3_dev_set_rxmode(hw, VMXNET3_RXM_UCAST | VMXNET3_RXM_BCAST, 1); - /* - * Don't need to handle events for now - */ -#if PROCESS_SYS_EVENTS == 1 - events = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_ECR); - PMD_INIT_LOG(DEBUG, "Reading events: 0x%X", events); - vmxnet3_process_events(hw); -#endif - return status; + if (dev->data->dev_conf.intr_conf.lsc) { + vmxnet3_enable_intr(hw); + + /* + * Update link state from device since this won't be + * done upon starting with lsc in use. This is done + * only after enabling interrupts to avoid any race + * where the link state could change without an + * interrupt being fired. + */ + __vmxnet3_dev_link_update(dev, 0); + } + + return VMXNET3_SUCCESS; } /* @@ -620,6 +856,15 @@ vmxnet3_dev_stop(struct rte_eth_dev *dev) /* disable interrupts */ vmxnet3_disable_intr(hw); + if (dev->data->dev_conf.intr_conf.lsc) { + struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(dev->device); + + rte_intr_disable(&pci_dev->intr_handle); + + rte_intr_callback_unregister(&pci_dev->intr_handle, + vmxnet3_interrupt_handler, dev); + } + /* quiesce the device first */ VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_QUIESCE_DEV); VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_DSAL, 0); @@ -652,59 +897,206 @@ vmxnet3_dev_close(struct rte_eth_dev *dev) } static void +vmxnet3_hw_tx_stats_get(struct vmxnet3_hw *hw, unsigned int q, + struct UPT1_TxStats *res) +{ +#define VMXNET3_UPDATE_TX_STAT(h, i, f, r) \ + ((r)->f = (h)->tqd_start[(i)].stats.f + \ + (h)->saved_tx_stats[(i)].f) + + VMXNET3_UPDATE_TX_STAT(hw, q, ucastPktsTxOK, res); + VMXNET3_UPDATE_TX_STAT(hw, q, mcastPktsTxOK, res); + VMXNET3_UPDATE_TX_STAT(hw, q, bcastPktsTxOK, res); + VMXNET3_UPDATE_TX_STAT(hw, q, ucastBytesTxOK, res); + VMXNET3_UPDATE_TX_STAT(hw, q, mcastBytesTxOK, res); + VMXNET3_UPDATE_TX_STAT(hw, q, bcastBytesTxOK, res); + VMXNET3_UPDATE_TX_STAT(hw, q, pktsTxError, res); + VMXNET3_UPDATE_TX_STAT(hw, q, pktsTxDiscard, res); + +#undef VMXNET3_UPDATE_TX_STAT +} + +static void +vmxnet3_hw_rx_stats_get(struct vmxnet3_hw *hw, unsigned int q, + struct UPT1_RxStats *res) +{ +#define VMXNET3_UPDATE_RX_STAT(h, i, f, r) \ + ((r)->f = (h)->rqd_start[(i)].stats.f + \ + (h)->saved_rx_stats[(i)].f) + + VMXNET3_UPDATE_RX_STAT(hw, q, ucastPktsRxOK, res); + VMXNET3_UPDATE_RX_STAT(hw, q, mcastPktsRxOK, res); + VMXNET3_UPDATE_RX_STAT(hw, q, bcastPktsRxOK, res); + VMXNET3_UPDATE_RX_STAT(hw, q, ucastBytesRxOK, res); + VMXNET3_UPDATE_RX_STAT(hw, q, mcastBytesRxOK, res); + VMXNET3_UPDATE_RX_STAT(hw, q, bcastBytesRxOK, res); + VMXNET3_UPDATE_RX_STAT(hw, q, pktsRxError, res); + VMXNET3_UPDATE_RX_STAT(hw, q, pktsRxOutOfBuf, res); + +#undef VMXNET3_UPDATE_RX_STATS +} + +static void +vmxnet3_hw_stats_save(struct vmxnet3_hw *hw) +{ + unsigned int i; + + VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_GET_STATS); + + RTE_BUILD_BUG_ON(RTE_ETHDEV_QUEUE_STAT_CNTRS < VMXNET3_MAX_TX_QUEUES); + + for (i = 0; i < hw->num_tx_queues; i++) + vmxnet3_hw_tx_stats_get(hw, i, &hw->saved_tx_stats[i]); + for (i = 0; i < hw->num_rx_queues; i++) + vmxnet3_hw_rx_stats_get(hw, i, &hw->saved_rx_stats[i]); +} + +static int +vmxnet3_dev_xstats_get_names(struct rte_eth_dev *dev, + struct rte_eth_xstat_name *xstats_names, + unsigned int n) +{ + unsigned int i, t, count = 0; + unsigned int nstats = + dev->data->nb_tx_queues * RTE_DIM(vmxnet3_txq_stat_strings) + + dev->data->nb_rx_queues * RTE_DIM(vmxnet3_rxq_stat_strings); + + if (!xstats_names || n < nstats) + return nstats; + + for (i = 0; i < dev->data->nb_rx_queues; i++) { + if (!dev->data->rx_queues[i]) + continue; + + for (t = 0; t < RTE_DIM(vmxnet3_rxq_stat_strings); t++) { + snprintf(xstats_names[count].name, + sizeof(xstats_names[count].name), + "rx_q%u_%s", i, + vmxnet3_rxq_stat_strings[t].name); + count++; + } + } + + for (i = 0; i < dev->data->nb_tx_queues; i++) { + if (!dev->data->tx_queues[i]) + continue; + + for (t = 0; t < RTE_DIM(vmxnet3_txq_stat_strings); t++) { + snprintf(xstats_names[count].name, + sizeof(xstats_names[count].name), + "tx_q%u_%s", i, + vmxnet3_txq_stat_strings[t].name); + count++; + } + } + + return count; +} + +static int +vmxnet3_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, + unsigned int n) +{ + unsigned int i, t, count = 0; + unsigned int nstats = + dev->data->nb_tx_queues * RTE_DIM(vmxnet3_txq_stat_strings) + + dev->data->nb_rx_queues * RTE_DIM(vmxnet3_rxq_stat_strings); + + if (n < nstats) + return nstats; + + for (i = 0; i < dev->data->nb_rx_queues; i++) { + struct vmxnet3_rx_queue *rxq = dev->data->rx_queues[i]; + + if (rxq == NULL) + continue; + + for (t = 0; t < RTE_DIM(vmxnet3_rxq_stat_strings); t++) { + xstats[count].value = *(uint64_t *)(((char *)&rxq->stats) + + vmxnet3_rxq_stat_strings[t].offset); + xstats[count].id = count; + count++; + } + } + + for (i = 0; i < dev->data->nb_tx_queues; i++) { + struct vmxnet3_tx_queue *txq = dev->data->tx_queues[i]; + + if (txq == NULL) + continue; + + for (t = 0; t < RTE_DIM(vmxnet3_txq_stat_strings); t++) { + xstats[count].value = *(uint64_t *)(((char *)&txq->stats) + + vmxnet3_txq_stat_strings[t].offset); + xstats[count].id = count; + count++; + } + } + + return count; +} + +static int vmxnet3_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) { unsigned int i; struct vmxnet3_hw *hw = dev->data->dev_private; + struct UPT1_TxStats txStats; + struct UPT1_RxStats rxStats; VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_GET_STATS); RTE_BUILD_BUG_ON(RTE_ETHDEV_QUEUE_STAT_CNTRS < VMXNET3_MAX_TX_QUEUES); for (i = 0; i < hw->num_tx_queues; i++) { - struct UPT1_TxStats *txStats = &hw->tqd_start[i].stats; + vmxnet3_hw_tx_stats_get(hw, i, &txStats); - stats->q_opackets[i] = txStats->ucastPktsTxOK + - txStats->mcastPktsTxOK + - txStats->bcastPktsTxOK; - stats->q_obytes[i] = txStats->ucastBytesTxOK + - txStats->mcastBytesTxOK + - txStats->bcastBytesTxOK; + stats->q_opackets[i] = txStats.ucastPktsTxOK + + txStats.mcastPktsTxOK + + txStats.bcastPktsTxOK; + + stats->q_obytes[i] = txStats.ucastBytesTxOK + + txStats.mcastBytesTxOK + + txStats.bcastBytesTxOK; stats->opackets += stats->q_opackets[i]; stats->obytes += stats->q_obytes[i]; - stats->oerrors += txStats->pktsTxError + - txStats->pktsTxDiscard; + stats->oerrors += txStats.pktsTxError + txStats.pktsTxDiscard; } RTE_BUILD_BUG_ON(RTE_ETHDEV_QUEUE_STAT_CNTRS < VMXNET3_MAX_RX_QUEUES); for (i = 0; i < hw->num_rx_queues; i++) { - struct UPT1_RxStats *rxStats = &hw->rqd_start[i].stats; + vmxnet3_hw_rx_stats_get(hw, i, &rxStats); - stats->q_ipackets[i] = rxStats->ucastPktsRxOK + - rxStats->mcastPktsRxOK + - rxStats->bcastPktsRxOK; + stats->q_ipackets[i] = rxStats.ucastPktsRxOK + + rxStats.mcastPktsRxOK + + rxStats.bcastPktsRxOK; - stats->q_ibytes[i] = rxStats->ucastBytesRxOK + - rxStats->mcastBytesRxOK + - rxStats->bcastBytesRxOK; + stats->q_ibytes[i] = rxStats.ucastBytesRxOK + + rxStats.mcastBytesRxOK + + rxStats.bcastBytesRxOK; stats->ipackets += stats->q_ipackets[i]; stats->ibytes += stats->q_ibytes[i]; - stats->q_errors[i] = rxStats->pktsRxError; - stats->ierrors += rxStats->pktsRxError; - stats->rx_nombuf += rxStats->pktsRxOutOfBuf; + stats->q_errors[i] = rxStats.pktsRxError; + stats->ierrors += rxStats.pktsRxError; + stats->rx_nombuf += rxStats.pktsRxOutOfBuf; } + + return 0; } static void -vmxnet3_dev_info_get(__attribute__((unused))struct rte_eth_dev *dev, +vmxnet3_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) { + dev_info->pci_dev = RTE_ETH_DEV_TO_PCI(dev); + dev_info->max_rx_queues = VMXNET3_MAX_RX_QUEUES; dev_info->max_tx_queues = VMXNET3_MAX_TX_QUEUES; dev_info->min_rx_bufsize = 1518 + RTE_PKTMBUF_HEADROOM; dev_info->max_rx_pktlen = 16384; /* includes CRC, cf MAXFRS register */ + dev_info->speed_capa = ETH_LINK_SPEED_10G; dev_info->max_mac_addrs = VMXNET3_MAX_MAC_ADDRS; dev_info->default_txconf.txq_flags = ETH_TXQ_FLAGS_NOXSUMSCTP; @@ -720,12 +1112,15 @@ vmxnet3_dev_info_get(__attribute__((unused))struct rte_eth_dev *dev, .nb_max = VMXNET3_TX_RING_MAX_SIZE, .nb_min = VMXNET3_DEF_TX_RING_SIZE, .nb_align = 1, + .nb_seg_max = VMXNET3_TX_MAX_SEG, + .nb_mtu_seg_max = VMXNET3_MAX_TXD_PER_PKT, }; dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP | DEV_RX_OFFLOAD_UDP_CKSUM | - DEV_RX_OFFLOAD_TCP_CKSUM; + DEV_RX_OFFLOAD_TCP_CKSUM | + DEV_RX_OFFLOAD_TCP_LRO; dev_info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT | @@ -753,20 +1148,20 @@ vmxnet3_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr) { struct vmxnet3_hw *hw = dev->data->dev_private; + ether_addr_copy(mac_addr, (struct ether_addr *)(hw->perm_addr)); + ether_addr_copy(mac_addr, &dev->data->mac_addrs[0]); vmxnet3_write_mac(hw, mac_addr->addr_bytes); } /* return 0 means link status changed, -1 means not changed */ static int -vmxnet3_dev_link_update(struct rte_eth_dev *dev, __attribute__((unused)) int wait_to_complete) +__vmxnet3_dev_link_update(struct rte_eth_dev *dev, + __rte_unused int wait_to_complete) { struct vmxnet3_hw *hw = dev->data->dev_private; - struct rte_eth_link old, link; + struct rte_eth_link old = { 0 }, link; uint32_t ret; - if (dev->data->dev_started == 0) - return -1; /* Link status doesn't change for stopped dev */ - memset(&link, 0, sizeof(link)); vmxnet3_dev_atomic_read_link_status(dev, &old); @@ -777,7 +1172,7 @@ vmxnet3_dev_link_update(struct rte_eth_dev *dev, __attribute__((unused)) int wai link.link_status = ETH_LINK_UP; link.link_duplex = ETH_LINK_FULL_DUPLEX; link.link_speed = ETH_SPEED_NUM_10G; - link.link_autoneg = ETH_LINK_SPEED_FIXED; + link.link_autoneg = ETH_LINK_AUTONEG; } vmxnet3_dev_atomic_write_link_status(dev, &link); @@ -785,10 +1180,20 @@ vmxnet3_dev_link_update(struct rte_eth_dev *dev, __attribute__((unused)) int wai return (old.link_status == link.link_status) ? -1 : 0; } +static int +vmxnet3_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete) +{ + /* Link status doesn't change for stopped dev */ + if (dev->data->dev_started == 0) + return -1; + + return __vmxnet3_dev_link_update(dev, wait_to_complete); +} + /* Updating rxmode through Vmxnet3_DriverShared structure in adapter */ static void -vmxnet3_dev_set_rxmode(struct vmxnet3_hw *hw, uint32_t feature, int set) { - +vmxnet3_dev_set_rxmode(struct vmxnet3_hw *hw, uint32_t feature, int set) +{ struct Vmxnet3_RxFilterConf *rxConf = &hw->shared->devRead.rxFilterConf; if (set) @@ -820,7 +1225,10 @@ vmxnet3_dev_promiscuous_disable(struct rte_eth_dev *dev) struct vmxnet3_hw *hw = dev->data->dev_private; uint32_t *vf_table = hw->shared->devRead.rxFilterConf.vfTable; - memcpy(vf_table, hw->shadow_vfta, VMXNET3_VFT_TABLE_SIZE); + if (dev->data->dev_conf.rxmode.hw_vlan_filter) + memcpy(vf_table, hw->shadow_vfta, VMXNET3_VFT_TABLE_SIZE); + else + memset(vf_table, 0xff, VMXNET3_VFT_TABLE_SIZE); vmxnet3_dev_set_rxmode(hw, VMXNET3_RXM_PROMISC, 0); VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_UPDATE_VLAN_FILTERS); @@ -873,7 +1281,7 @@ vmxnet3_dev_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vid, int on) return 0; } -static void +static int vmxnet3_dev_vlan_offload_set(struct rte_eth_dev *dev, int mask) { struct vmxnet3_hw *hw = dev->data->dev_private; @@ -899,18 +1307,18 @@ vmxnet3_dev_vlan_offload_set(struct rte_eth_dev *dev, int mask) VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_UPDATE_VLAN_FILTERS); } + + return 0; } -#if PROCESS_SYS_EVENTS == 1 static void -vmxnet3_process_events(struct vmxnet3_hw *hw) +vmxnet3_process_events(struct rte_eth_dev *dev) { + struct vmxnet3_hw *hw = dev->data->dev_private; uint32_t events = hw->shared->ecr; - if (!events) { - PMD_INIT_LOG(ERR, "No events to process"); + if (!events) return; - } /* * ECR bits when written with 1b are cleared. Hence write @@ -919,20 +1327,25 @@ vmxnet3_process_events(struct vmxnet3_hw *hw) VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_ECR, events); /* Check if link state has changed */ - if (events & VMXNET3_ECR_LINK) - PMD_INIT_LOG(ERR, - "Process events in %s(): VMXNET3_ECR_LINK event", __func__); + if (events & VMXNET3_ECR_LINK) { + PMD_DRV_LOG(DEBUG, "Process events: VMXNET3_ECR_LINK event"); + if (vmxnet3_dev_link_update(dev, 0) == 0) + _rte_eth_dev_callback_process(dev, + RTE_ETH_EVENT_INTR_LSC, + NULL, NULL); + } /* Check if there is an error on xmit/recv queues */ if (events & (VMXNET3_ECR_TQERR | VMXNET3_ECR_RQERR)) { - VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_GET_QUEUE_STATUS); + VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, + VMXNET3_CMD_GET_QUEUE_STATUS); if (hw->tqd_start->status.stopped) - PMD_INIT_LOG(ERR, "tq error 0x%x", - hw->tqd_start->status.error); + PMD_DRV_LOG(ERR, "tq error 0x%x", + hw->tqd_start->status.error); if (hw->rqd_start->status.stopped) - PMD_INIT_LOG(ERR, "rq error 0x%x", + PMD_DRV_LOG(ERR, "rq error 0x%x", hw->rqd_start->status.error); /* Reset the device */ @@ -940,18 +1353,24 @@ vmxnet3_process_events(struct vmxnet3_hw *hw) } if (events & VMXNET3_ECR_DIC) - PMD_INIT_LOG(ERR, "Device implementation change event."); + PMD_DRV_LOG(DEBUG, "Device implementation change event."); if (events & VMXNET3_ECR_DEBUG) - PMD_INIT_LOG(ERR, "Debug event generated by device."); - + PMD_DRV_LOG(DEBUG, "Debug event generated by device."); } -#endif -static struct rte_driver rte_vmxnet3_driver = { - .type = PMD_PDEV, - .init = rte_vmxnet3_pmd_init, -}; +static void +vmxnet3_interrupt_handler(void *param) +{ + struct rte_eth_dev *dev = param; + struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(dev->device); -PMD_REGISTER_DRIVER(rte_vmxnet3_driver, vmxnet3); -DRIVER_REGISTER_PCI_TABLE(vmxnet3, pci_id_vmxnet3_map); + vmxnet3_process_events(dev); + + if (rte_intr_enable(&pci_dev->intr_handle) < 0) + PMD_DRV_LOG(ERR, "interrupt enable failed"); +} + +RTE_PMD_REGISTER_PCI(net_vmxnet3, rte_vmxnet3_pmd); +RTE_PMD_REGISTER_PCI_TABLE(net_vmxnet3, pci_id_vmxnet3_map); +RTE_PMD_REGISTER_KMOD_DEP(net_vmxnet3, "* igb_uio | uio_pci_generic | vfio-pci"); diff --git a/dpdk/drivers/net/vmxnet3/vmxnet3_ethdev.h b/dpdk/drivers/net/vmxnet3/vmxnet3_ethdev.h index 1be833ab..b48058af 100644 --- a/dpdk/drivers/net/vmxnet3/vmxnet3_ethdev.h +++ b/dpdk/drivers/net/vmxnet3/vmxnet3_ethdev.h @@ -34,6 +34,8 @@ #ifndef _VMXNET3_ETHDEV_H_ #define _VMXNET3_ETHDEV_H_ +#include + #define VMXNET3_MAX_MAC_ADDRS 1 /* UPT feature to negotiate */ @@ -62,8 +64,7 @@ ETH_RSS_NONFRAG_IPV6_TCP) /* RSS configuration structure - shared with device through GPA */ -typedef -struct VMXNET3_RSSConf { +typedef struct VMXNET3_RSSConf { uint16_t hashType; uint16_t hashFunc; uint16_t hashKeySize; @@ -76,15 +77,13 @@ struct VMXNET3_RSSConf { uint8_t indTable[VMXNET3_RSS_MAX_IND_TABLE_SIZE]; } VMXNET3_RSSConf; -typedef -struct vmxnet3_mf_table { +typedef struct vmxnet3_mf_table { void *mfTableBase; /* Multicast addresses list */ uint64_t mfTablePA; /* Physical address of the list */ uint16_t num_addrs; /* number of multicast addrs */ } vmxnet3_mf_table_t; struct vmxnet3_hw { - uint8_t *hw_addr0; /* BAR0: PT-Passthrough Regs */ uint8_t *hw_addr1; /* BAR1: VD-Virtual Device Regs */ /* BAR2: MSI-X Regs */ @@ -102,6 +101,13 @@ struct vmxnet3_hw { uint8_t num_rx_queues; uint8_t bufs_per_pkt; + uint8_t version; + + uint16_t txdata_desc_size; /* tx data ring buffer size */ + uint16_t rxdata_desc_size; /* rx data ring buffer size */ + + uint8_t num_intrs; + Vmxnet3_TxQueueDesc *tqd_start; /* start address of all tx queue desc */ Vmxnet3_RxQueueDesc *rqd_start; /* start address of all rx queue desc */ @@ -111,28 +117,38 @@ struct vmxnet3_hw { uint64_t queueDescPA; uint16_t queue_desc_len; - VMXNET3_RSSConf *rss_conf; - uint64_t rss_confPA; - vmxnet3_mf_table_t *mf_table; - uint32_t shadow_vfta[VMXNET3_VFT_SIZE]; + VMXNET3_RSSConf *rss_conf; + uint64_t rss_confPA; + vmxnet3_mf_table_t *mf_table; + uint32_t shadow_vfta[VMXNET3_VFT_SIZE]; + Vmxnet3_MemRegs *memRegs; + uint64_t memRegsPA; #define VMXNET3_VFT_TABLE_SIZE (VMXNET3_VFT_SIZE * sizeof(uint32_t)) + UPT1_TxStats saved_tx_stats[VMXNET3_MAX_TX_QUEUES]; + UPT1_RxStats saved_rx_stats[VMXNET3_MAX_RX_QUEUES]; }; +#define VMXNET3_REV_3 2 /* Vmxnet3 Rev. 3 */ +#define VMXNET3_REV_2 1 /* Vmxnet3 Rev. 2 */ +#define VMXNET3_REV_1 0 /* Vmxnet3 Rev. 1 */ + +#define VMXNET3_VERSION_GE_3(hw) ((hw)->version >= VMXNET3_REV_3 + 1) +#define VMXNET3_VERSION_GE_2(hw) ((hw)->version >= VMXNET3_REV_2 + 1) + #define VMXNET3_GET_ADDR_LO(reg) ((uint32_t)(reg)) #define VMXNET3_GET_ADDR_HI(reg) ((uint32_t)(((uint64_t)(reg)) >> 32)) /* Config space read/writes */ -#define VMXNET3_PCI_REG(reg) (*((volatile uint32_t *)(reg))) +#define VMXNET3_PCI_REG(reg) rte_read32(reg) -static inline uint32_t vmxnet3_read_addr(volatile void *addr) +static inline uint32_t +vmxnet3_read_addr(volatile void *addr) { return VMXNET3_PCI_REG(addr); } -#define VMXNET3_PCI_REG_WRITE(reg, value) do { \ - VMXNET3_PCI_REG((reg)) = (value); \ -} while(0) +#define VMXNET3_PCI_REG_WRITE(reg, value) rte_write32((value), (reg)) #define VMXNET3_PCI_BAR0_REG_ADDR(hw, reg) \ ((volatile uint32_t *)((char *)(hw)->hw_addr0 + (reg))) @@ -148,6 +164,20 @@ static inline uint32_t vmxnet3_read_addr(volatile void *addr) #define VMXNET3_WRITE_BAR1_REG(hw, reg, value) \ VMXNET3_PCI_REG_WRITE(VMXNET3_PCI_BAR1_REG_ADDR((hw), (reg)), (value)) +static inline uint8_t +vmxnet3_get_ring_idx(struct vmxnet3_hw *hw, uint32 rqID) +{ + return (rqID >= hw->num_rx_queues && + rqID < 2 * hw->num_rx_queues) ? 1 : 0; +} + +static inline bool +vmxnet3_rx_data_ring(struct vmxnet3_hw *hw, uint32 rqID) +{ + return (rqID >= 2 * hw->num_rx_queues && + rqID < 3 * hw->num_rx_queues); +} + /* * RX/TX function prototypes */ @@ -158,20 +188,22 @@ void vmxnet3_dev_rx_queue_release(void *rxq); void vmxnet3_dev_tx_queue_release(void *txq); int vmxnet3_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t rx_queue_id, - uint16_t nb_rx_desc, unsigned int socket_id, - const struct rte_eth_rxconf *rx_conf, - struct rte_mempool *mb_pool); + uint16_t nb_rx_desc, unsigned int socket_id, + const struct rte_eth_rxconf *rx_conf, + struct rte_mempool *mb_pool); int vmxnet3_dev_tx_queue_setup(struct rte_eth_dev *dev, uint16_t tx_queue_id, - uint16_t nb_tx_desc, unsigned int socket_id, - const struct rte_eth_txconf *tx_conf); + uint16_t nb_tx_desc, unsigned int socket_id, + const struct rte_eth_txconf *tx_conf); int vmxnet3_dev_rxtx_init(struct rte_eth_dev *dev); int vmxnet3_rss_configure(struct rte_eth_dev *dev); uint16_t vmxnet3_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, - uint16_t nb_pkts); + uint16_t nb_pkts); uint16_t vmxnet3_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, - uint16_t nb_pkts); + uint16_t nb_pkts); +uint16_t vmxnet3_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts); #endif /* _VMXNET3_ETHDEV_H_ */ diff --git a/dpdk/drivers/net/vmxnet3/vmxnet3_ring.h b/dpdk/drivers/net/vmxnet3/vmxnet3_ring.h index 69ff2ded..a6fa93ac 100644 --- a/dpdk/drivers/net/vmxnet3/vmxnet3_ring.h +++ b/dpdk/drivers/net/vmxnet3/vmxnet3_ring.h @@ -42,6 +42,9 @@ #define VMXNET3_DEF_TX_RING_SIZE 512 #define VMXNET3_DEF_RX_RING_SIZE 128 +/* Default rx data ring desc size */ +#define VMXNET3_DEF_RXDATA_DESC_SIZE 128 + #define VMXNET3_SUCCESS 0 #define VMXNET3_FAIL -1 @@ -96,12 +99,12 @@ vmxnet3_cmd_ring_desc_empty(struct vmxnet3_cmd_ring *ring) } typedef struct vmxnet3_comp_ring { - uint32_t size; - uint32_t next2proc; - uint8_t gen; - uint8_t intr_idx; + uint32_t size; + uint32_t next2proc; + uint8_t gen; + uint8_t intr_idx; Vmxnet3_GenericDesc *base; - uint64_t basePA; + uint64_t basePA; } vmxnet3_comp_ring_t; struct vmxnet3_data_ring { @@ -121,13 +124,13 @@ vmxnet3_comp_ring_adv_next2proc(struct vmxnet3_comp_ring *ring) } struct vmxnet3_txq_stats { - uint64_t drop_total; /* # of pkts dropped by the driver, + uint64_t drop_total; /* # of pkts dropped by the driver, * the counters below track droppings due to * different reasons */ - uint64_t drop_too_many_segs; - uint64_t drop_tso; - uint64_t tx_ring_full; + uint64_t drop_too_many_segs; + uint64_t drop_tso; + uint64_t tx_ring_full; }; typedef struct vmxnet3_tx_queue { @@ -138,9 +141,11 @@ typedef struct vmxnet3_tx_queue { uint32_t qid; struct Vmxnet3_TxQueueDesc *shared; struct vmxnet3_txq_stats stats; + const struct rte_memzone *mz; bool stopped; uint16_t queue_id; /**< Device TX queue index. */ - uint8_t port_id; /**< Device port identifier. */ + uint16_t port_id; /**< Device port identifier. */ + uint16_t txdata_desc_size; } vmxnet3_tx_queue_t; struct vmxnet3_rxq_stats { @@ -150,20 +155,31 @@ struct vmxnet3_rxq_stats { uint64_t rx_buf_alloc_failure; }; +struct vmxnet3_rx_data_ring { + uint8_t *base; + uint64_t basePA; + uint32_t size; +}; + typedef struct vmxnet3_rx_queue { struct rte_mempool *mp; struct vmxnet3_hw *hw; struct vmxnet3_cmd_ring cmd_ring[VMXNET3_RX_CMDRING_SIZE]; struct vmxnet3_comp_ring comp_ring; + struct vmxnet3_rx_data_ring data_ring; + uint16_t data_desc_size; uint32_t qid1; uint32_t qid2; + /* rqID in RCD for buffer from data ring */ + uint32_t data_ring_qid; Vmxnet3_RxQueueDesc *shared; - struct rte_mbuf *start_seg; - struct rte_mbuf *last_seg; + struct rte_mbuf *start_seg; + struct rte_mbuf *last_seg; struct vmxnet3_rxq_stats stats; + const struct rte_memzone *mz; bool stopped; uint16_t queue_id; /**< Device RX queue index. */ - uint8_t port_id; /**< Device port identifier. */ + uint16_t port_id; /**< Device port identifier. */ } vmxnet3_rx_queue_t; #endif /* _VMXNET3_RING_H_ */ diff --git a/dpdk/drivers/net/vmxnet3/vmxnet3_rxtx.c b/dpdk/drivers/net/vmxnet3/vmxnet3_rxtx.c index 88df576c..aa396ab2 100644 --- a/dpdk/drivers/net/vmxnet3/vmxnet3_rxtx.c +++ b/dpdk/drivers/net/vmxnet3/vmxnet3_rxtx.c @@ -57,7 +57,6 @@ #include #include #include -#include #include #include #include @@ -70,6 +69,7 @@ #include #include #include +#include #include "base/vmxnet3_defs.h" #include "vmxnet3_ring.h" @@ -77,6 +77,14 @@ #include "vmxnet3_logs.h" #include "vmxnet3_ethdev.h" +#define VMXNET3_TX_OFFLOAD_MASK ( \ + PKT_TX_VLAN_PKT | \ + PKT_TX_L4_MASK | \ + PKT_TX_TCP_SEG) + +#define VMXNET3_TX_OFFLOAD_NOTSUP_MASK \ + (PKT_TX_OFFLOAD_MASK ^ VMXNET3_TX_OFFLOAD_MASK) + static const uint32_t rxprod_reg[2] = {VMXNET3_REG_RXPROD, VMXNET3_REG_RXPROD2}; static int vmxnet3_post_rx_bufs(vmxnet3_rx_queue_t*, uint8_t); @@ -183,7 +191,6 @@ vmxnet3_cmd_ring_release(vmxnet3_cmd_ring_t *ring) ring->buf_info = NULL; } - void vmxnet3_dev_tx_queue_release(void *txq) { @@ -194,6 +201,10 @@ vmxnet3_dev_tx_queue_release(void *txq) vmxnet3_tx_cmd_ring_release_mbufs(&tq->cmd_ring); /* Release the cmd_ring */ vmxnet3_cmd_ring_release(&tq->cmd_ring); + /* Release the memzone */ + rte_memzone_free(tq->mz); + /* Release the queue */ + rte_free(tq); } } @@ -211,6 +222,12 @@ vmxnet3_dev_rx_queue_release(void *rxq) /* Release both the cmd_rings */ for (i = 0; i < VMXNET3_RX_CMDRING_SIZE; i++) vmxnet3_cmd_ring_release(&rq->cmd_ring[i]); + + /* Release the memzone */ + rte_memzone_free(rq->mz); + + /* Release the queue */ + rte_free(rq); } } @@ -237,7 +254,7 @@ vmxnet3_dev_tx_queue_reset(void *txq) size = sizeof(struct Vmxnet3_TxDesc) * ring->size; size += sizeof(struct Vmxnet3_TxCompDesc) * comp_ring->size; - size += sizeof(struct Vmxnet3_TxDataDesc) * data_ring->size; + size += tq->txdata_desc_size * data_ring->size; memset(ring->base, 0, size); } @@ -247,15 +264,15 @@ vmxnet3_dev_rx_queue_reset(void *rxq) { int i; vmxnet3_rx_queue_t *rq = rxq; + struct vmxnet3_hw *hw = rq->hw; struct vmxnet3_cmd_ring *ring0, *ring1; struct vmxnet3_comp_ring *comp_ring; + struct vmxnet3_rx_data_ring *data_ring = &rq->data_ring; int size; - if (rq != NULL) { - /* Release both the cmd_rings mbufs */ - for (i = 0; i < VMXNET3_RX_CMDRING_SIZE; i++) - vmxnet3_rx_cmd_ring_release_mbufs(&rq->cmd_ring[i]); - } + /* Release both the cmd_rings mbufs */ + for (i = 0; i < VMXNET3_RX_CMDRING_SIZE; i++) + vmxnet3_rx_cmd_ring_release_mbufs(&rq->cmd_ring[i]); ring0 = &rq->cmd_ring[0]; ring1 = &rq->cmd_ring[1]; @@ -273,6 +290,8 @@ vmxnet3_dev_rx_queue_reset(void *rxq) size = sizeof(struct Vmxnet3_RxDesc) * (ring0->size + ring1->size); size += sizeof(struct Vmxnet3_RxCompDesc) * comp_ring->size; + if (VMXNET3_VERSION_GE_3(hw) && rq->data_desc_size) + size += rq->data_desc_size * data_ring->size; memset(ring0->base, 0, size); } @@ -351,6 +370,53 @@ vmxnet3_tq_tx_complete(vmxnet3_tx_queue_t *txq) PMD_TX_LOG(DEBUG, "Processed %d tx comps & command descs.", completed); } +uint16_t +vmxnet3_prep_pkts(__rte_unused void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts) +{ + int32_t ret; + uint32_t i; + uint64_t ol_flags; + struct rte_mbuf *m; + + for (i = 0; i != nb_pkts; i++) { + m = tx_pkts[i]; + ol_flags = m->ol_flags; + + /* Non-TSO packet cannot occupy more than + * VMXNET3_MAX_TXD_PER_PKT TX descriptors. + */ + if ((ol_flags & PKT_TX_TCP_SEG) == 0 && + m->nb_segs > VMXNET3_MAX_TXD_PER_PKT) { + rte_errno = -EINVAL; + return i; + } + + /* check that only supported TX offloads are requested. */ + if ((ol_flags & VMXNET3_TX_OFFLOAD_NOTSUP_MASK) != 0 || + (ol_flags & PKT_TX_L4_MASK) == + PKT_TX_SCTP_CKSUM) { + rte_errno = -ENOTSUP; + return i; + } + +#ifdef RTE_LIBRTE_ETHDEV_DEBUG + ret = rte_validate_tx_offload(m); + if (ret != 0) { + rte_errno = ret; + return i; + } +#endif + ret = rte_net_intel_cksum_prepare(m); + if (ret != 0) { + rte_errno = ret; + return i; + } + } + + return i; +} + uint16_t vmxnet3_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) @@ -416,10 +482,14 @@ vmxnet3_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, continue; } - if (txm->nb_segs == 1 && rte_pktmbuf_pkt_len(txm) <= VMXNET3_HDR_COPY_SIZE) { + if (txm->nb_segs == 1 && + rte_pktmbuf_pkt_len(txm) <= txq->txdata_desc_size) { struct Vmxnet3_TxDataDesc *tdd; - tdd = txq->data_ring.base + txq->cmd_ring.next2fill; + tdd = (struct Vmxnet3_TxDataDesc *) + ((uint8 *)txq->data_ring.base + + txq->cmd_ring.next2fill * + txq->txdata_desc_size); copy_size = rte_pktmbuf_pkt_len(txm); rte_memcpy(tdd->data, rte_pktmbuf_mtod(txm, char *), copy_size); } @@ -436,12 +506,16 @@ vmxnet3_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, * maximum size of mbuf segment size. */ gdesc = txq->cmd_ring.base + txq->cmd_ring.next2fill; - if (copy_size) - gdesc->txd.addr = rte_cpu_to_le_64(txq->data_ring.basePA + - txq->cmd_ring.next2fill * - sizeof(struct Vmxnet3_TxDataDesc)); - else - gdesc->txd.addr = rte_mbuf_data_dma_addr(m_seg); + if (copy_size) { + uint64 offset = + (uint64)txq->cmd_ring.next2fill * + txq->txdata_desc_size; + gdesc->txd.addr = + rte_cpu_to_le_64(txq->data_ring.basePA + + offset); + } else { + gdesc->txd.addr = rte_mbuf_data_iova(m_seg); + } gdesc->dword[2] = dw2 | m_seg->data_len; gdesc->dword[3] = 0; @@ -519,23 +593,15 @@ vmxnet3_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, return nb_tx; } -/* - * Allocates mbufs and clusters. Post rx descriptors with buffer details - * so that device can receive packets in those buffers. - * Ring layout: - * Among the two rings, 1st ring contains buffers of type 0 and type1. - * bufs_per_pkt is set such that for non-LRO cases all the buffers required - * by a frame will fit in 1st ring (1st buf of type0 and rest of type1). - * 2nd ring contains buffers of type 1 alone. Second ring mostly be used - * only for LRO. - * - */ -static int -vmxnet3_post_rx_bufs(vmxnet3_rx_queue_t *rxq, uint8_t ring_id) +static inline void +vmxnet3_renew_desc(vmxnet3_rx_queue_t *rxq, uint8_t ring_id, + struct rte_mbuf *mbuf) { - int err = 0; - uint32_t i = 0, val = 0; + uint32_t val; struct vmxnet3_cmd_ring *ring = &rxq->cmd_ring[ring_id]; + struct Vmxnet3_RxDesc *rxd = + (struct Vmxnet3_RxDesc *)(ring->base + ring->next2fill); + vmxnet3_buf_info_t *buf_info = &ring->buf_info[ring->next2fill]; if (ring_id == 0) { /* Usually: One HEAD type buf per packet @@ -550,12 +616,44 @@ vmxnet3_post_rx_bufs(vmxnet3_rx_queue_t *rxq, uint8_t ring_id) val = VMXNET3_RXD_BTYPE_BODY; } - while (vmxnet3_cmd_ring_desc_avail(ring) > 0) { - struct Vmxnet3_RxDesc *rxd; - struct rte_mbuf *mbuf; - vmxnet3_buf_info_t *buf_info = &ring->buf_info[ring->next2fill]; + /* + * Load mbuf pointer into buf_info[ring_size] + * buf_info structure is equivalent to cookie for virtio-virtqueue + */ + buf_info->m = mbuf; + buf_info->len = (uint16_t)(mbuf->buf_len - RTE_PKTMBUF_HEADROOM); + buf_info->bufPA = rte_mbuf_data_iova_default(mbuf); - rxd = (struct Vmxnet3_RxDesc *)(ring->base + ring->next2fill); + /* Load Rx Descriptor with the buffer's GPA */ + rxd->addr = buf_info->bufPA; + + /* After this point rxd->addr MUST not be NULL */ + rxd->btype = val; + rxd->len = buf_info->len; + /* Flip gen bit at the end to change ownership */ + rxd->gen = ring->gen; + + vmxnet3_cmd_ring_adv_next2fill(ring); +} +/* + * Allocates mbufs and clusters. Post rx descriptors with buffer details + * so that device can receive packets in those buffers. + * Ring layout: + * Among the two rings, 1st ring contains buffers of type 0 and type 1. + * bufs_per_pkt is set such that for non-LRO cases all the buffers required + * by a frame will fit in 1st ring (1st buf of type0 and rest of type1). + * 2nd ring contains buffers of type 1 alone. Second ring mostly be used + * only for LRO. + */ +static int +vmxnet3_post_rx_bufs(vmxnet3_rx_queue_t *rxq, uint8_t ring_id) +{ + int err = 0; + uint32_t i = 0; + struct vmxnet3_cmd_ring *ring = &rxq->cmd_ring[ring_id]; + + while (vmxnet3_cmd_ring_desc_avail(ring) > 0) { + struct rte_mbuf *mbuf; /* Allocate blank mbuf for the current Rx Descriptor */ mbuf = rte_mbuf_raw_alloc(rxq->mp); @@ -566,26 +664,7 @@ vmxnet3_post_rx_bufs(vmxnet3_rx_queue_t *rxq, uint8_t ring_id) break; } - /* - * Load mbuf pointer into buf_info[ring_size] - * buf_info structure is equivalent to cookie for virtio-virtqueue - */ - buf_info->m = mbuf; - buf_info->len = (uint16_t)(mbuf->buf_len - - RTE_PKTMBUF_HEADROOM); - buf_info->bufPA = - rte_mbuf_data_dma_addr_default(mbuf); - - /* Load Rx Descriptor with the buffer's GPA */ - rxd->addr = buf_info->bufPA; - - /* After this point rxd->addr MUST not be NULL */ - rxd->btype = val; - rxd->len = buf_info->len; - /* Flip gen bit at the end to change ownership */ - rxd->gen = ring->gen; - - vmxnet3_cmd_ring_adv_next2fill(ring); + vmxnet3_renew_desc(rxq, ring_id, mbuf); i++; } @@ -660,11 +739,20 @@ vmxnet3_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) } while (rcd->gen == rxq->comp_ring.gen) { + struct rte_mbuf *newm; + if (nb_rx >= nb_pkts) break; + newm = rte_mbuf_raw_alloc(rxq->mp); + if (unlikely(newm == NULL)) { + PMD_RX_LOG(ERR, "Error allocating mbuf"); + rxq->stats.rx_buf_alloc_failure++; + break; + } + idx = rcd->rxdIdx; - ring_idx = (uint8_t)((rcd->rqID == rxq->qid1) ? 0 : 1); + ring_idx = vmxnet3_get_ring_idx(hw, rcd->rqID); rxd = (Vmxnet3_RxDesc *)rxq->cmd_ring[ring_idx].base + idx; RTE_SET_USED(rxd); /* used only for assert when enabled */ rbi = rxq->cmd_ring[ring_idx].buf_info + idx; @@ -697,10 +785,15 @@ vmxnet3_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) (int)(rcd - (struct Vmxnet3_RxCompDesc *) rxq->comp_ring.base), rcd->rxdIdx); rte_pktmbuf_free_seg(rxm); + if (rxq->start_seg) { + struct rte_mbuf *start = rxq->start_seg; + + rxq->start_seg = NULL; + rte_pktmbuf_free(start); + } goto rcd_done; } - /* Initialize newly received packet buffer */ rxm->port = rxq->port_id; rxm->nb_segs = 1; @@ -731,6 +824,15 @@ vmxnet3_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) goto rcd_done; } + if (vmxnet3_rx_data_ring(hw, rcd->rqID)) { + uint8_t *rdd = rxq->data_ring.base + + idx * rxq->data_desc_size; + + RTE_ASSERT(VMXNET3_VERSION_GE_3(hw)); + rte_memcpy(rte_pktmbuf_mtod(rxm, char *), + rdd, rcd->len); + } + rxq->start_seg = rxm; vmxnet3_rx_offload(rcd, rxm); } else { @@ -750,7 +852,8 @@ vmxnet3_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) /* Check for hardware stripped VLAN tag */ if (rcd->ts) { - start->ol_flags |= (PKT_RX_VLAN_PKT | PKT_RX_VLAN_STRIPPED); + start->ol_flags |= (PKT_RX_VLAN | + PKT_RX_VLAN_STRIPPED); start->vlan_tci = rte_le_to_cpu_16((uint16_t)rcd->tci); } @@ -760,10 +863,11 @@ vmxnet3_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) rcd_done: rxq->cmd_ring[ring_idx].next2comp = idx; - VMXNET3_INC_RING_IDX_ONLY(rxq->cmd_ring[ring_idx].next2comp, rxq->cmd_ring[ring_idx].size); + VMXNET3_INC_RING_IDX_ONLY(rxq->cmd_ring[ring_idx].next2comp, + rxq->cmd_ring[ring_idx].size); - /* It's time to allocate some new buf and renew descriptors */ - vmxnet3_post_rx_bufs(rxq, ring_idx); + /* It's time to renew descriptors */ + vmxnet3_renew_desc(rxq, ring_idx, newm); if (unlikely(rxq->shared->ctrl.updateRxProd)) { VMXNET3_WRITE_BAR0_REG(hw, rxprod_reg[ring_idx] + (rxq->queue_id * VMXNET3_REG_ALIGN), rxq->cmd_ring[ring_idx].next2fill); @@ -775,46 +879,38 @@ rcd_done: rcd = &rxq->comp_ring.base[rxq->comp_ring.next2proc].rcd; nb_rxd++; if (nb_rxd > rxq->cmd_ring[0].size) { - PMD_RX_LOG(ERR, - "Used up quota of receiving packets," + PMD_RX_LOG(ERR, "Used up quota of receiving packets," " relinquish control."); break; } } + if (unlikely(nb_rxd == 0)) { + uint32_t avail; + for (ring_idx = 0; ring_idx < VMXNET3_RX_CMDRING_SIZE; ring_idx++) { + avail = vmxnet3_cmd_ring_desc_avail(&rxq->cmd_ring[ring_idx]); + if (unlikely(avail > 0)) { + /* try to alloc new buf and renew descriptors */ + vmxnet3_post_rx_bufs(rxq, ring_idx); + } + } + if (unlikely(rxq->shared->ctrl.updateRxProd)) { + for (ring_idx = 0; ring_idx < VMXNET3_RX_CMDRING_SIZE; ring_idx++) { + VMXNET3_WRITE_BAR0_REG(hw, rxprod_reg[ring_idx] + (rxq->queue_id * VMXNET3_REG_ALIGN), + rxq->cmd_ring[ring_idx].next2fill); + } + } + } + return nb_rx; } -/* - * Create memzone for device rings. malloc can't be used as the physical address is - * needed. If the memzone is already created, then this function returns a ptr - * to the old one. - */ -static const struct rte_memzone * -ring_dma_zone_reserve(struct rte_eth_dev *dev, const char *ring_name, - uint16_t queue_id, uint32_t ring_size, int socket_id) -{ - char z_name[RTE_MEMZONE_NAMESIZE]; - const struct rte_memzone *mz; - - snprintf(z_name, sizeof(z_name), "%s_%s_%d_%d", - dev->driver->pci_drv.name, ring_name, - dev->data->port_id, queue_id); - - mz = rte_memzone_lookup(z_name); - if (mz) - return mz; - - return rte_memzone_reserve_aligned(z_name, ring_size, - socket_id, 0, VMXNET3_RING_BA_ALIGN); -} - int vmxnet3_dev_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, uint16_t nb_desc, unsigned int socket_id, - __attribute__((unused)) const struct rte_eth_txconf *tx_conf) + const struct rte_eth_txconf *tx_conf) { struct vmxnet3_hw *hw = dev->data->dev_private; const struct rte_memzone *mz; @@ -832,7 +928,8 @@ vmxnet3_dev_tx_queue_setup(struct rte_eth_dev *dev, return -EINVAL; } - txq = rte_zmalloc("ethdev_tx_queue", sizeof(struct vmxnet3_tx_queue), RTE_CACHE_LINE_SIZE); + txq = rte_zmalloc("ethdev_tx_queue", sizeof(struct vmxnet3_tx_queue), + RTE_CACHE_LINE_SIZE); if (txq == NULL) { PMD_INIT_LOG(ERR, "Can not allocate tx queue structure"); return -ENOMEM; @@ -844,6 +941,7 @@ vmxnet3_dev_tx_queue_setup(struct rte_eth_dev *dev, txq->hw = hw; txq->qid = queue_idx; txq->stopped = TRUE; + txq->txdata_desc_size = hw->txdata_desc_size; ring = &txq->cmd_ring; comp_ring = &txq->comp_ring; @@ -873,18 +971,20 @@ vmxnet3_dev_tx_queue_setup(struct rte_eth_dev *dev, size = sizeof(struct Vmxnet3_TxDesc) * ring->size; size += sizeof(struct Vmxnet3_TxCompDesc) * comp_ring->size; - size += sizeof(struct Vmxnet3_TxDataDesc) * data_ring->size; + size += txq->txdata_desc_size * data_ring->size; - mz = ring_dma_zone_reserve(dev, "txdesc", queue_idx, size, socket_id); + mz = rte_eth_dma_zone_reserve(dev, "txdesc", queue_idx, size, + VMXNET3_RING_BA_ALIGN, socket_id); if (mz == NULL) { PMD_INIT_LOG(ERR, "ERROR: Creating queue descriptors zone"); return -ENOMEM; } + txq->mz = mz; memset(mz->addr, 0, mz->len); /* cmd_ring initialization */ ring->base = mz->addr; - ring->basePA = mz->phys_addr; + ring->basePA = mz->iova; /* comp_ring initialization */ comp_ring->base = ring->base + ring->size; @@ -915,21 +1015,23 @@ vmxnet3_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, uint16_t nb_desc, unsigned int socket_id, - __attribute__((unused)) const struct rte_eth_rxconf *rx_conf, + __rte_unused const struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp) { const struct rte_memzone *mz; struct vmxnet3_rx_queue *rxq; - struct vmxnet3_hw *hw = dev->data->dev_private; + struct vmxnet3_hw *hw = dev->data->dev_private; struct vmxnet3_cmd_ring *ring0, *ring1, *ring; struct vmxnet3_comp_ring *comp_ring; + struct vmxnet3_rx_data_ring *data_ring; int size; uint8_t i; char mem_name[32]; PMD_INIT_FUNC_TRACE(); - rxq = rte_zmalloc("ethdev_rx_queue", sizeof(struct vmxnet3_rx_queue), RTE_CACHE_LINE_SIZE); + rxq = rte_zmalloc("ethdev_rx_queue", sizeof(struct vmxnet3_rx_queue), + RTE_CACHE_LINE_SIZE); if (rxq == NULL) { PMD_INIT_LOG(ERR, "Can not allocate rx queue structure"); return -ENOMEM; @@ -942,11 +1044,14 @@ vmxnet3_dev_rx_queue_setup(struct rte_eth_dev *dev, rxq->hw = hw; rxq->qid1 = queue_idx; rxq->qid2 = queue_idx + hw->num_rx_queues; + rxq->data_ring_qid = queue_idx + 2 * hw->num_rx_queues; + rxq->data_desc_size = hw->rxdata_desc_size; rxq->stopped = TRUE; ring0 = &rxq->cmd_ring[0]; ring1 = &rxq->cmd_ring[1]; comp_ring = &rxq->comp_ring; + data_ring = &rxq->data_ring; /* Rx vmxnet rings length should be between 256-4096 */ if (nb_desc < VMXNET3_DEF_RX_RING_SIZE) { @@ -962,6 +1067,7 @@ vmxnet3_dev_rx_queue_setup(struct rte_eth_dev *dev, } comp_ring->size = ring0->size + ring1->size; + data_ring->size = ring0->size; /* Rx vmxnet rings structure initialization */ ring0->next2fill = 0; @@ -975,17 +1081,21 @@ vmxnet3_dev_rx_queue_setup(struct rte_eth_dev *dev, size = sizeof(struct Vmxnet3_RxDesc) * (ring0->size + ring1->size); size += sizeof(struct Vmxnet3_RxCompDesc) * comp_ring->size; + if (VMXNET3_VERSION_GE_3(hw) && rxq->data_desc_size) + size += rxq->data_desc_size * data_ring->size; - mz = ring_dma_zone_reserve(dev, "rxdesc", queue_idx, size, socket_id); + mz = rte_eth_dma_zone_reserve(dev, "rxdesc", queue_idx, size, + VMXNET3_RING_BA_ALIGN, socket_id); if (mz == NULL) { PMD_INIT_LOG(ERR, "ERROR: Creating queue descriptors zone"); return -ENOMEM; } + rxq->mz = mz; memset(mz->addr, 0, mz->len); /* cmd_ring0 initialization */ ring0->base = mz->addr; - ring0->basePA = mz->phys_addr; + ring0->basePA = mz->iova; /* cmd_ring1 initialization */ ring1->base = ring0->base + ring0->size; @@ -996,6 +1106,14 @@ vmxnet3_dev_rx_queue_setup(struct rte_eth_dev *dev, comp_ring->basePA = ring1->basePA + sizeof(struct Vmxnet3_RxDesc) * ring1->size; + /* data_ring initialization */ + if (VMXNET3_VERSION_GE_3(hw) && rxq->data_desc_size) { + data_ring->base = + (uint8_t *)(comp_ring->base + comp_ring->size); + data_ring->basePA = comp_ring->basePA + + sizeof(struct Vmxnet3_RxCompDesc) * comp_ring->size; + } + /* cmd_ring0-cmd_ring1 buf_info allocation */ for (i = 0; i < VMXNET3_RX_CMDRING_SIZE; i++) { @@ -1003,7 +1121,9 @@ vmxnet3_dev_rx_queue_setup(struct rte_eth_dev *dev, ring->rid = i; snprintf(mem_name, sizeof(mem_name), "rx_ring_%d_buf_info", i); - ring->buf_info = rte_zmalloc(mem_name, ring->size * sizeof(vmxnet3_buf_info_t), RTE_CACHE_LINE_SIZE); + ring->buf_info = rte_zmalloc(mem_name, + ring->size * sizeof(vmxnet3_buf_info_t), + RTE_CACHE_LINE_SIZE); if (ring->buf_info == NULL) { PMD_INIT_LOG(ERR, "ERROR: Creating rx_buf_info structure"); return -ENOMEM; @@ -1037,10 +1157,15 @@ vmxnet3_dev_rxtx_init(struct rte_eth_dev *dev) /* Passing 0 as alloc_num will allocate full ring */ ret = vmxnet3_post_rx_bufs(rxq, j); if (ret <= 0) { - PMD_INIT_LOG(ERR, "ERROR: Posting Rxq: %d buffers ring: %d", i, j); + PMD_INIT_LOG(ERR, + "ERROR: Posting Rxq: %d buffers ring: %d", + i, j); return -ret; } - /* Updating device with the index:next2fill to fill the mbufs for coming packets */ + /* + * Updating device with the index:next2fill to fill the + * mbufs for coming packets. + */ if (unlikely(rxq->shared->ctrl.updateRxProd)) { VMXNET3_WRITE_BAR0_REG(hw, rxprod_reg[j] + (rxq->queue_id * VMXNET3_REG_ALIGN), rxq->cmd_ring[j].next2fill); @@ -1088,7 +1213,7 @@ vmxnet3_rss_configure(struct rte_eth_dev *dev) dev_rss_conf->hashFunc = VMXNET3_RSS_HASH_FUNC_TOEPLITZ; /* loading hashKeySize */ dev_rss_conf->hashKeySize = VMXNET3_RSS_MAX_KEY_SIZE; - /* loading indTableSize : Must not exceed VMXNET3_RSS_MAX_IND_TABLE_SIZE (128)*/ + /* loading indTableSize: Must not exceed VMXNET3_RSS_MAX_IND_TABLE_SIZE (128)*/ dev_rss_conf->indTableSize = (uint16_t)(hw->num_rx_queues * 4); if (port_rss_conf->rss_key == NULL) { @@ -1097,7 +1222,8 @@ vmxnet3_rss_configure(struct rte_eth_dev *dev) } /* loading hashKey */ - memcpy(&dev_rss_conf->hashKey[0], port_rss_conf->rss_key, dev_rss_conf->hashKeySize); + memcpy(&dev_rss_conf->hashKey[0], port_rss_conf->rss_key, + dev_rss_conf->hashKeySize); /* loading indTable */ for (i = 0, j = 0; i < dev_rss_conf->indTableSize; i++, j++) { diff --git a/dpdk/drivers/net/xenvirt/Makefile b/dpdk/drivers/net/xenvirt/Makefile deleted file mode 100644 index 1d05b71b..00000000 --- a/dpdk/drivers/net/xenvirt/Makefile +++ /dev/null @@ -1,63 +0,0 @@ -# BSD LICENSE -# -# Copyright(c) 2010-2014 Intel Corporation. All rights reserved. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in -# the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Intel Corporation nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -include $(RTE_SDK)/mk/rte.vars.mk - -# -# library name -# -LIB = librte_pmd_xenvirt.a - -CFLAGS += -O3 -CFLAGS += $(WERROR_FLAGS) -LDLIBS += -lxenstore - -EXPORT_MAP := rte_eth_xenvirt_version.map - -LIBABIVER := 1 - -# -# all source are stored in SRCS-y -# -SRCS-$(CONFIG_RTE_LIBRTE_PMD_XENVIRT) += rte_eth_xenvirt.c rte_mempool_gntalloc.c rte_xen_lib.c - -# -# Export include files -# -SYMLINK-y-include += rte_eth_xenvirt.h - -# this lib depends upon: -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_XENVIRT) += lib/librte_eal lib/librte_ether -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_XENVIRT) += lib/librte_mempool lib/librte_mbuf -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_XENVIRT) += lib/librte_net -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_XENVIRT) += lib/librte_cmdline - -include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/drivers/net/xenvirt/rte_eth_xenvirt.c b/dpdk/drivers/net/xenvirt/rte_eth_xenvirt.c deleted file mode 100644 index 99f6cc81..00000000 --- a/dpdk/drivers/net/xenvirt/rte_eth_xenvirt.c +++ /dev/null @@ -1,770 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#ifndef PAGE_SIZE -#define PAGE_SIZE sysconf(_SC_PAGE_SIZE) -#endif -#include -#include -#if __XEN_LATEST_INTERFACE_VERSION__ < 0x00040200 -#include -#else -#include -#endif -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "rte_xen_lib.h" -#include "virtqueue.h" -#include "rte_eth_xenvirt.h" - -#define VQ_DESC_NUM 256 -#define VIRTIO_MBUF_BURST_SZ 64 - -/* virtio_idx is increased after new device is created.*/ -static int virtio_idx = 0; - -static const char *drivername = "xen virtio PMD"; - -static struct rte_eth_link pmd_link = { - .link_speed = ETH_SPEED_NUM_10G, - .link_duplex = ETH_LINK_FULL_DUPLEX, - .link_status = ETH_LINK_DOWN, - .link_autoneg = ETH_LINK_SPEED_FIXED -}; - -static void -eth_xenvirt_free_queues(struct rte_eth_dev *dev); - -static uint16_t -eth_xenvirt_rx(void *q, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) -{ - struct virtqueue *rxvq = q; - struct rte_mbuf *rxm, *new_mbuf; - uint16_t nb_used, num; - uint32_t len[VIRTIO_MBUF_BURST_SZ]; - uint32_t i; - struct pmd_internals *pi = rxvq->internals; - - nb_used = VIRTQUEUE_NUSED(rxvq); - - rte_smp_rmb(); - num = (uint16_t)(likely(nb_used <= nb_pkts) ? nb_used : nb_pkts); - num = (uint16_t)(likely(num <= VIRTIO_MBUF_BURST_SZ) ? num : VIRTIO_MBUF_BURST_SZ); - if (unlikely(num == 0)) return 0; - - num = virtqueue_dequeue_burst(rxvq, rx_pkts, len, num); - PMD_RX_LOG(DEBUG, "used:%d dequeue:%d\n", nb_used, num); - for (i = 0; i < num ; i ++) { - rxm = rx_pkts[i]; - PMD_RX_LOG(DEBUG, "packet len:%d\n", len[i]); - rxm->next = NULL; - rxm->data_off = RTE_PKTMBUF_HEADROOM; - rxm->data_len = (uint16_t)(len[i] - sizeof(struct virtio_net_hdr)); - rxm->nb_segs = 1; - rxm->port = pi->port_id; - rxm->pkt_len = (uint32_t)(len[i] - sizeof(struct virtio_net_hdr)); - } - /* allocate new mbuf for the used descriptor */ - while (likely(!virtqueue_full(rxvq))) { - new_mbuf = rte_mbuf_raw_alloc(rxvq->mpool); - if (unlikely(new_mbuf == NULL)) { - break; - } - if (unlikely(virtqueue_enqueue_recv_refill(rxvq, new_mbuf))) { - rte_pktmbuf_free_seg(new_mbuf); - break; - } - } - pi->eth_stats.ipackets += num; - return num; -} - -static uint16_t -eth_xenvirt_tx(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) -{ - struct virtqueue *txvq = tx_queue; - struct rte_mbuf *txm; - uint16_t nb_used, nb_tx, num, i; - int error; - uint32_t len[VIRTIO_MBUF_BURST_SZ]; - struct rte_mbuf *snd_pkts[VIRTIO_MBUF_BURST_SZ]; - struct pmd_internals *pi = txvq->internals; - - nb_tx = 0; - - if (unlikely(nb_pkts == 0)) - return 0; - - PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts); - nb_used = VIRTQUEUE_NUSED(txvq); - - rte_smp_rmb(); - - num = (uint16_t)(likely(nb_used <= VIRTIO_MBUF_BURST_SZ) ? nb_used : VIRTIO_MBUF_BURST_SZ); - num = virtqueue_dequeue_burst(txvq, snd_pkts, len, num); - - for (i = 0; i < num ; i ++) { - /* mergable not supported, one segment only */ - rte_pktmbuf_free_seg(snd_pkts[i]); - } - - while (nb_tx < nb_pkts) { - if (likely(!virtqueue_full(txvq))) { - /* TODO drop tx_pkts if it contains multiple segments */ - txm = tx_pkts[nb_tx]; - error = virtqueue_enqueue_xmit(txvq, txm); - if (unlikely(error)) { - if (error == ENOSPC) - PMD_TX_LOG(ERR, "virtqueue_enqueue Free count = 0\n"); - else if (error == EMSGSIZE) - PMD_TX_LOG(ERR, "virtqueue_enqueue Free count < 1\n"); - else - PMD_TX_LOG(ERR, "virtqueue_enqueue error: %d\n", error); - break; - } - nb_tx++; - } else { - PMD_TX_LOG(ERR, "No free tx descriptors to transmit\n"); - /* virtqueue_notify not needed in our para-virt solution */ - break; - } - } - pi->eth_stats.opackets += nb_tx; - return nb_tx; -} - -static int -eth_dev_configure(struct rte_eth_dev *dev __rte_unused) -{ - RTE_LOG(ERR, PMD, "%s\n", __func__); - return 0; -} - -/* - * Create a shared page between guest and host. - * Host monitors this page if it is cleared on unmap, and then - * do necessary clean up. - */ -static void -gntalloc_vring_flag(int vtidx) -{ - char key_str[PATH_MAX]; - char val_str[PATH_MAX]; - uint32_t gref_tmp; - void *ptr; - - if (grefwatch_from_alloc(&gref_tmp, &ptr)) { - RTE_LOG(ERR, PMD, "grefwatch_from_alloc error\n"); - exit(0); - } - - *(uint8_t *)ptr = MAP_FLAG; - snprintf(val_str, sizeof(val_str), "%u", gref_tmp); - snprintf(key_str, sizeof(key_str), - DPDK_XENSTORE_PATH"%d"VRING_FLAG_STR, vtidx); - xenstore_write(key_str, val_str); -} - -/* - * Notify host this virtio device is started. - * Host could start polling this device. - */ -static void -dev_start_notify(int vtidx) -{ - char key_str[PATH_MAX]; - char val_str[PATH_MAX]; - - RTE_LOG(INFO, PMD, "%s: virtio %d is started\n", __func__, vtidx); - gntalloc_vring_flag(vtidx); - - snprintf(key_str, sizeof(key_str), "%s%s%d", - DPDK_XENSTORE_PATH, EVENT_TYPE_START_STR, - vtidx); - snprintf(val_str, sizeof(val_str), "1"); - xenstore_write(key_str, val_str); -} - -/* - * Notify host this virtio device is stopped. - * Host could stop polling this device. - */ -static void -dev_stop_notify(int vtidx) -{ - RTE_SET_USED(vtidx); -} - - -static int -update_mac_address(struct ether_addr *mac_addrs, int vtidx) -{ - char key_str[PATH_MAX]; - char val_str[PATH_MAX]; - int rv; - - if (mac_addrs == NULL) { - RTE_LOG(ERR, PMD, "%s: NULL pointer mac specified\n", __func__); - return -1; - } - rv = snprintf(key_str, sizeof(key_str), - DPDK_XENSTORE_PATH"%d_ether_addr", vtidx); - if (rv == -1) - return rv; - rv = snprintf(val_str, sizeof(val_str), "%02x:%02x:%02x:%02x:%02x:%02x", - mac_addrs->addr_bytes[0], - mac_addrs->addr_bytes[1], - mac_addrs->addr_bytes[2], - mac_addrs->addr_bytes[3], - mac_addrs->addr_bytes[4], - mac_addrs->addr_bytes[5]); - if (rv == -1) - return rv; - if (xenstore_write(key_str, val_str)) - return rv; - return 0; -} - - -static int -eth_dev_start(struct rte_eth_dev *dev) -{ - struct virtqueue *rxvq = dev->data->rx_queues[0]; - struct virtqueue *txvq = dev->data->tx_queues[0]; - struct rte_mbuf *m; - struct pmd_internals *pi = (struct pmd_internals *)dev->data->dev_private; - int rv; - - dev->data->dev_link.link_status = ETH_LINK_UP; - while (!virtqueue_full(rxvq)) { - m = rte_mbuf_raw_alloc(rxvq->mpool); - if (m == NULL) - break; - /* Enqueue allocated buffers. */ - if (virtqueue_enqueue_recv_refill(rxvq, m)) { - rte_pktmbuf_free_seg(m); - break; - } - } - - rxvq->internals = pi; - txvq->internals = pi; - - rv = update_mac_address(dev->data->mac_addrs, pi->virtio_idx); - if (rv) - return -1; - dev_start_notify(pi->virtio_idx); - - return 0; -} - -static void -eth_dev_stop(struct rte_eth_dev *dev) -{ - struct pmd_internals *pi = (struct pmd_internals *)dev->data->dev_private; - - dev->data->dev_link.link_status = ETH_LINK_DOWN; - dev_stop_notify(pi->virtio_idx); -} - -/* - * Notify host this virtio device is closed. - * Host could do necessary clean up to this device. - */ -static void -eth_dev_close(struct rte_eth_dev *dev) -{ - eth_xenvirt_free_queues(dev); -} - -static void -eth_dev_info(struct rte_eth_dev *dev, - struct rte_eth_dev_info *dev_info) -{ - struct pmd_internals *internals = dev->data->dev_private; - - RTE_SET_USED(internals); - dev_info->driver_name = drivername; - dev_info->max_mac_addrs = 1; - dev_info->max_rx_pktlen = (uint32_t)2048; - dev_info->max_rx_queues = (uint16_t)1; - dev_info->max_tx_queues = (uint16_t)1; - dev_info->min_rx_bufsize = 0; - dev_info->pci_dev = NULL; -} - -static void -eth_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) -{ - struct pmd_internals *internals = dev->data->dev_private; - if(stats) - rte_memcpy(stats, &internals->eth_stats, sizeof(*stats)); -} - -static void -eth_stats_reset(struct rte_eth_dev *dev) -{ - struct pmd_internals *internals = dev->data->dev_private; - /* Reset software totals */ - memset(&internals->eth_stats, 0, sizeof(internals->eth_stats)); -} - -static void -eth_queue_release(void *q) -{ - rte_free(q); -} - -static int -eth_link_update(struct rte_eth_dev *dev __rte_unused, - int wait_to_complete __rte_unused) -{ - return 0; -} - -/* - * Create shared vring between guest and host. - * Memory is allocated through grant alloc driver, so it is not physical continuous. - */ -static void * -gntalloc_vring_create(int queue_type, uint32_t size, int vtidx) -{ - char key_str[PATH_MAX] = {0}; - char val_str[PATH_MAX] = {0}; - void *va = NULL; - int pg_size; - uint32_t pg_num; - uint32_t *gref_arr = NULL; - phys_addr_t *pa_arr = NULL; - uint64_t start_index; - int rv; - - pg_size = getpagesize(); - size = RTE_ALIGN_CEIL(size, pg_size); - pg_num = size / pg_size; - - gref_arr = calloc(pg_num, sizeof(gref_arr[0])); - pa_arr = calloc(pg_num, sizeof(pa_arr[0])); - - if (gref_arr == NULL || pa_arr == NULL) { - RTE_LOG(ERR, PMD, "%s: calloc failed\n", __func__); - goto out; - } - - va = gntalloc(size, gref_arr, &start_index); - if (va == NULL) { - RTE_LOG(ERR, PMD, "%s: gntalloc failed\n", __func__); - goto out; - } - - if (get_phys_map(va, pa_arr, pg_num, pg_size)) - goto out; - - /* write in xenstore gref and pfn for each page of vring */ - if (grant_node_create(pg_num, gref_arr, pa_arr, val_str, sizeof(val_str))) { - gntfree(va, size, start_index); - va = NULL; - goto out; - } - - if (queue_type == VTNET_RQ) - rv = snprintf(key_str, sizeof(key_str), DPDK_XENSTORE_PATH"%d"RXVRING_XENSTORE_STR, vtidx); - else - rv = snprintf(key_str, sizeof(key_str), DPDK_XENSTORE_PATH"%d"TXVRING_XENSTORE_STR, vtidx); - if (rv == -1 || xenstore_write(key_str, val_str) == -1) { - gntfree(va, size, start_index); - va = NULL; - } -out: - free(pa_arr); - free(gref_arr); - - return va; -} - - - -static struct virtqueue * -virtio_queue_setup(struct rte_eth_dev *dev, int queue_type) -{ - struct virtqueue *vq = NULL; - uint16_t vq_size = VQ_DESC_NUM; - int i = 0; - char vq_name[VIRTQUEUE_MAX_NAME_SZ]; - size_t size; - struct vring *vr; - - /* Allocate memory for virtqueue. */ - if (queue_type == VTNET_RQ) { - snprintf(vq_name, sizeof(vq_name), "port%d_rvq", - dev->data->port_id); - vq = rte_zmalloc(vq_name, sizeof(struct virtqueue) + - vq_size * sizeof(struct vq_desc_extra), RTE_CACHE_LINE_SIZE); - if (vq == NULL) { - RTE_LOG(ERR, PMD, "%s: unabled to allocate virtqueue\n", __func__); - return NULL; - } - memcpy(vq->vq_name, vq_name, sizeof(vq->vq_name)); - } else if(queue_type == VTNET_TQ) { - snprintf(vq_name, sizeof(vq_name), "port%d_tvq", - dev->data->port_id); - vq = rte_zmalloc(vq_name, sizeof(struct virtqueue) + - vq_size * sizeof(struct vq_desc_extra), RTE_CACHE_LINE_SIZE); - if (vq == NULL) { - RTE_LOG(ERR, PMD, "%s: unabled to allocate virtqueue\n", __func__); - return NULL; - } - memcpy(vq->vq_name, vq_name, sizeof(vq->vq_name)); - } - - memcpy(vq->vq_name, vq_name, sizeof(vq->vq_name)); - - vq->vq_alignment = VIRTIO_PCI_VRING_ALIGN; - vq->vq_nentries = vq_size; - vq->vq_free_cnt = vq_size; - /* Calcuate vring size according to virtio spec */ - size = vring_size(vq_size, VIRTIO_PCI_VRING_ALIGN); - vq->vq_ring_size = RTE_ALIGN_CEIL(size, VIRTIO_PCI_VRING_ALIGN); - /* Allocate memory for virtio vring through gntalloc driver*/ - vq->vq_ring_virt_mem = gntalloc_vring_create(queue_type, vq->vq_ring_size, - ((struct pmd_internals *)dev->data->dev_private)->virtio_idx); - memset(vq->vq_ring_virt_mem, 0, vq->vq_ring_size); - vr = &vq->vq_ring; - vring_init(vr, vq_size, vq->vq_ring_virt_mem, vq->vq_alignment); - /* - * Locally maintained last consumed index, this idex trails - * vq_ring.used->idx. - */ - vq->vq_used_cons_idx = 0; - vq->vq_desc_head_idx = 0; - vq->vq_free_cnt = vq->vq_nentries; - memset(vq->vq_descx, 0, sizeof(struct vq_desc_extra) * vq->vq_nentries); - - /* Chain all the descriptors in the ring with an END */ - for (i = 0; i < vq_size - 1; i++) - vr->desc[i].next = (uint16_t)(i + 1); - vr->desc[i].next = VQ_RING_DESC_CHAIN_END; - - return vq; -} - -static int -eth_rx_queue_setup(struct rte_eth_dev *dev,uint16_t rx_queue_id, - uint16_t nb_rx_desc __rte_unused, - unsigned int socket_id __rte_unused, - const struct rte_eth_rxconf *rx_conf __rte_unused, - struct rte_mempool *mb_pool) -{ - struct virtqueue *vq; - vq = dev->data->rx_queues[rx_queue_id] = virtio_queue_setup(dev, VTNET_RQ); - vq->mpool = mb_pool; - return 0; -} - -static int -eth_tx_queue_setup(struct rte_eth_dev *dev, uint16_t tx_queue_id, - uint16_t nb_tx_desc __rte_unused, - unsigned int socket_id __rte_unused, - const struct rte_eth_txconf *tx_conf __rte_unused) -{ - dev->data->tx_queues[tx_queue_id] = virtio_queue_setup(dev, VTNET_TQ); - return 0; -} - -static void -eth_xenvirt_free_queues(struct rte_eth_dev *dev) -{ - int i; - - for (i = 0; i < dev->data->nb_rx_queues; i++) { - eth_queue_release(dev->data->rx_queues[i]); - dev->data->rx_queues[i] = NULL; - } - dev->data->nb_rx_queues = 0; - - for (i = 0; i < dev->data->nb_tx_queues; i++) { - eth_queue_release(dev->data->tx_queues[i]); - dev->data->tx_queues[i] = NULL; - } - dev->data->nb_tx_queues = 0; -} - -static const struct eth_dev_ops ops = { - .dev_start = eth_dev_start, - .dev_stop = eth_dev_stop, - .dev_close = eth_dev_close, - .dev_configure = eth_dev_configure, - .dev_infos_get = eth_dev_info, - .rx_queue_setup = eth_rx_queue_setup, - .tx_queue_setup = eth_tx_queue_setup, - .rx_queue_release = eth_queue_release, - .tx_queue_release = eth_queue_release, - .link_update = eth_link_update, - .stats_get = eth_stats_get, - .stats_reset = eth_stats_reset, -}; - - -static int -rte_eth_xenvirt_parse_args(struct xenvirt_dict *dict, - const char *name, const char *params) -{ - int i; - char *pairs[RTE_ETH_XENVIRT_MAX_ARGS]; - int num_of_pairs; - char *pair[2]; - char *args; - int ret = -1; - - if (params == NULL) - return 0; - - args = rte_zmalloc(NULL, strlen(params) + 1, RTE_CACHE_LINE_SIZE); - if (args == NULL) { - RTE_LOG(ERR, PMD, "Couldn't parse %s device \n", name); - return -1; - } - rte_memcpy(args, params, strlen(params)); - - num_of_pairs = rte_strsplit(args, strnlen(args, MAX_ARG_STRLEN), - pairs, - RTE_ETH_XENVIRT_MAX_ARGS , - RTE_ETH_XENVIRT_PAIRS_DELIM); - - for (i = 0; i < num_of_pairs; i++) { - pair[0] = NULL; - pair[1] = NULL; - rte_strsplit(pairs[i], strnlen(pairs[i], MAX_ARG_STRLEN), - pair, 2, - RTE_ETH_XENVIRT_KEY_VALUE_DELIM); - - if (pair[0] == NULL || pair[1] == NULL || pair[0][0] == 0 - || pair[1][0] == 0) { - RTE_LOG(ERR, PMD, - "Couldn't parse %s device," - "wrong key or value \n", name); - goto err; - } - - if (!strncmp(pair[0], RTE_ETH_XENVIRT_MAC_PARAM, - sizeof(RTE_ETH_XENVIRT_MAC_PARAM))) { - if (cmdline_parse_etheraddr(NULL, - pair[1], - &dict->addr, - sizeof(dict->addr)) < 0) { - RTE_LOG(ERR, PMD, - "Invalid %s device ether address\n", - name); - goto err; - } - - dict->addr_valid = 1; - } - } - - ret = 0; -err: - rte_free(args); - return ret; -} - -enum dev_action { - DEV_CREATE, - DEV_ATTACH -}; - - -static int -eth_dev_xenvirt_create(const char *name, const char *params, - const unsigned numa_node, - enum dev_action action) -{ - struct rte_eth_dev_data *data = NULL; - struct pmd_internals *internals = NULL; - struct rte_eth_dev *eth_dev = NULL; - struct xenvirt_dict dict; - - memset(&dict, 0, sizeof(struct xenvirt_dict)); - - RTE_LOG(INFO, PMD, "Creating virtio rings backed ethdev on numa socket %u\n", - numa_node); - RTE_SET_USED(action); - - if (rte_eth_xenvirt_parse_args(&dict, name, params) < 0) { - RTE_LOG(ERR, PMD, "%s: Failed to parse ethdev parameters\n", __func__); - return -1; - } - - /* now do all data allocation - for eth_dev structure, dummy pci driver - * and internal (private) data - */ - data = rte_zmalloc_socket(name, sizeof(*data), 0, numa_node); - if (data == NULL) - goto err; - - internals = rte_zmalloc_socket(name, sizeof(*internals), 0, numa_node); - if (internals == NULL) - goto err; - - /* reserve an ethdev entry */ - eth_dev = rte_eth_dev_allocate(name, RTE_ETH_DEV_VIRTUAL); - if (eth_dev == NULL) - goto err; - - data->dev_private = internals; - data->port_id = eth_dev->data->port_id; - data->nb_rx_queues = (uint16_t)1; - data->nb_tx_queues = (uint16_t)1; - data->dev_link = pmd_link; - data->mac_addrs = rte_zmalloc("xen_virtio", ETHER_ADDR_LEN, 0); - - if(dict.addr_valid) - memcpy(&data->mac_addrs->addr_bytes, &dict.addr, sizeof(struct ether_addr)); - else - eth_random_addr(&data->mac_addrs->addr_bytes[0]); - - eth_dev->data = data; - eth_dev->dev_ops = &ops; - - eth_dev->data->dev_flags = RTE_PCI_DRV_DETACHABLE; - eth_dev->data->kdrv = RTE_KDRV_NONE; - eth_dev->data->drv_name = drivername; - eth_dev->driver = NULL; - eth_dev->data->numa_node = numa_node; - - eth_dev->rx_pkt_burst = eth_xenvirt_rx; - eth_dev->tx_pkt_burst = eth_xenvirt_tx; - - internals->virtio_idx = virtio_idx++; - internals->port_id = eth_dev->data->port_id; - - return 0; - -err: - rte_free(data); - rte_free(internals); - - return -1; -} - - -static int -eth_dev_xenvirt_free(const char *name, const unsigned numa_node) -{ - struct rte_eth_dev *eth_dev = NULL; - - RTE_LOG(DEBUG, PMD, - "Free virtio rings backed ethdev on numa socket %u\n", - numa_node); - - /* find an ethdev entry */ - eth_dev = rte_eth_dev_allocated(name); - if (eth_dev == NULL) - return -1; - - if (eth_dev->data->dev_started == 1) { - eth_dev_stop(eth_dev); - eth_dev_close(eth_dev); - } - - eth_dev->rx_pkt_burst = NULL; - eth_dev->tx_pkt_burst = NULL; - eth_dev->dev_ops = NULL; - - rte_free(eth_dev->data); - rte_free(eth_dev->data->dev_private); - rte_free(eth_dev->data->mac_addrs); - - virtio_idx--; - - return 0; -} - -/*TODO: Support multiple process model */ -static int -rte_pmd_xenvirt_devinit(const char *name, const char *params) -{ - if (virtio_idx == 0) { - if (xenstore_init() != 0) { - RTE_LOG(ERR, PMD, "%s: xenstore init failed\n", __func__); - return -1; - } - if (gntalloc_open() != 0) { - RTE_LOG(ERR, PMD, "%s: grant init failed\n", __func__); - return -1; - } - } - eth_dev_xenvirt_create(name, params, rte_socket_id(), DEV_CREATE); - return 0; -} - -static int -rte_pmd_xenvirt_devuninit(const char *name) -{ - eth_dev_xenvirt_free(name, rte_socket_id()); - - if (virtio_idx == 0) { - if (xenstore_uninit() != 0) - RTE_LOG(ERR, PMD, "%s: xenstore uninit failed\n", __func__); - - gntalloc_close(); - } - return 0; -} - -static struct rte_driver pmd_xenvirt_drv = { - .type = PMD_VDEV, - .init = rte_pmd_xenvirt_devinit, - .uninit = rte_pmd_xenvirt_devuninit, -}; - -PMD_REGISTER_DRIVER(pmd_xenvirt_drv, eth_xenvirt); -DRIVER_REGISTER_PARAM_STRING(eth_xenvirt, - "mac="); diff --git a/dpdk/drivers/net/xenvirt/rte_eth_xenvirt.h b/dpdk/drivers/net/xenvirt/rte_eth_xenvirt.h deleted file mode 100644 index 4995a9b4..00000000 --- a/dpdk/drivers/net/xenvirt/rte_eth_xenvirt.h +++ /dev/null @@ -1,62 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef _RTE_ETH_XENVIRT_H_ -#define _RTE_ETH_XENVIRT_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include - -/** - * Creates mempool for xen virtio PMD. - * This function uses memzone_reserve to allocate memory for meta data, - * and uses grant alloc driver to allocate memory for data area. - * The input parameters are exactly the same as rte_mempool_create. - */ -struct rte_mempool * -rte_mempool_gntalloc_create(const char *name, unsigned elt_num, unsigned elt_size, - unsigned cache_size, unsigned private_data_size, - rte_mempool_ctor_t *mp_init, void *mp_init_arg, - rte_mempool_obj_cb_t *obj_init, void *obj_init_arg, - int socket_id, unsigned flags); - - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/dpdk/drivers/net/xenvirt/rte_eth_xenvirt_version.map b/dpdk/drivers/net/xenvirt/rte_eth_xenvirt_version.map deleted file mode 100644 index dd636f72..00000000 --- a/dpdk/drivers/net/xenvirt/rte_eth_xenvirt_version.map +++ /dev/null @@ -1,7 +0,0 @@ -DPDK_2.0 { - global: - - rte_mempool_gntalloc_create; - - local: *; -}; diff --git a/dpdk/drivers/net/xenvirt/rte_mempool_gntalloc.c b/dpdk/drivers/net/xenvirt/rte_mempool_gntalloc.c deleted file mode 100644 index 73e82f80..00000000 --- a/dpdk/drivers/net/xenvirt/rte_mempool_gntalloc.c +++ /dev/null @@ -1,295 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include "rte_xen_lib.h" -#include "rte_eth_xenvirt.h" - -struct _gntarr { - uint32_t gref; - phys_addr_t pa; - uint64_t index; - void *va; -}; - -struct _mempool_gntalloc_info { - struct rte_mempool *mp; - uint32_t pg_num; - uint32_t *gref_arr; - phys_addr_t *pa_arr; - void *va; - uint32_t mempool_idx; - uint64_t start_index; -}; - - -static rte_atomic32_t global_xenvirt_mempool_idx = RTE_ATOMIC32_INIT(-1); - -static int -compare(const void *p1, const void *p2) -{ - return ((const struct _gntarr *)p1)->pa - ((const struct _gntarr *)p2)->pa; -} - - -static struct _mempool_gntalloc_info -_create_mempool(const char *name, unsigned elt_num, unsigned elt_size, - unsigned cache_size, unsigned private_data_size, - rte_mempool_ctor_t *mp_init, void *mp_init_arg, - rte_mempool_obj_cb_t *obj_init, void *obj_init_arg, - int socket_id, unsigned flags) -{ - struct _mempool_gntalloc_info mgi; - struct rte_mempool *mp = NULL; - struct rte_mempool_objsz objsz; - uint32_t pg_num, rpg_num, pg_shift, pg_sz; - char *va, *orig_va, *uv; /* uv: from which, the pages could be freed */ - ssize_t sz, usz; /* usz: unused size */ - /* - * for each page allocated through xen_gntalloc driver, - * gref_arr:stores grant references, - * pa_arr: stores physical address, - * gnt_arr: stores all meta dat - */ - uint32_t *gref_arr = NULL; - phys_addr_t *pa_arr = NULL; - struct _gntarr *gnt_arr = NULL; - /* start index of the grant referances, used for dealloc*/ - uint64_t start_index; - uint32_t i, j; - int rv = 0; - struct ioctl_gntalloc_dealloc_gref arg; - - mgi.mp = NULL; - va = orig_va = uv = NULL; - pg_num = rpg_num = 0; - sz = 0; - - pg_sz = getpagesize(); - if (rte_is_power_of_2(pg_sz) == 0) { - goto out; - } - pg_shift = rte_bsf32(pg_sz); - - rte_mempool_calc_obj_size(elt_size, flags, &objsz); - sz = rte_mempool_xmem_size(elt_num, objsz.total_size, pg_shift); - pg_num = sz >> pg_shift; - - pa_arr = calloc(pg_num, sizeof(pa_arr[0])); - gref_arr = calloc(pg_num, sizeof(gref_arr[0])); - gnt_arr = calloc(pg_num, sizeof(gnt_arr[0])); - if ((gnt_arr == NULL) || (gref_arr == NULL) || (pa_arr == NULL)) - goto out; - - /* grant index is continuous in ascending order */ - orig_va = gntalloc(sz, gref_arr, &start_index); - if (orig_va == NULL) - goto out; - - get_phys_map(orig_va, pa_arr, pg_num, pg_sz); - for (i = 0; i < pg_num; i++) { - gnt_arr[i].index = start_index + i * pg_sz; - gnt_arr[i].gref = gref_arr[i]; - gnt_arr[i].pa = pa_arr[i]; - gnt_arr[i].va = RTE_PTR_ADD(orig_va, i * pg_sz); - } - qsort(gnt_arr, pg_num, sizeof(struct _gntarr), compare); - - va = get_xen_virtual(sz, pg_sz); - if (va == NULL) { - goto out; - } - - /* - * map one by one, as index isn't continuous now. - * pg_num VMAs, doesn't linux has a limitation on this? - */ - for (i = 0; i < pg_num; i++) { - /* update gref_arr and pa_arr after sort */ - gref_arr[i] = gnt_arr[i].gref; - pa_arr[i] = gnt_arr[i].pa; - gnt_arr[i].va = mmap(va + i * pg_sz, pg_sz, PROT_READ | PROT_WRITE, - MAP_SHARED | MAP_FIXED, gntalloc_fd, gnt_arr[i].index); - if ((gnt_arr[i].va == MAP_FAILED) || (gnt_arr[i].va != (va + i * pg_sz))) { - RTE_LOG(ERR, PMD, "failed to map %d pages\n", i); - goto mmap_failed; - } - } - - /* - * Check that allocated size is big enough to hold elt_num - * objects and a calcualte how many bytes are actually required. - */ - usz = rte_mempool_xmem_usage(va, elt_num, objsz.total_size, pa_arr, pg_num, pg_shift); - if (usz < 0) { - mp = NULL; - i = pg_num; - goto mmap_failed; - } else { - /* unmap unused pages if any */ - uv = RTE_PTR_ADD(va, usz); - if ((usz = va + sz - uv) > 0) { - - RTE_LOG(ERR, PMD, - "%s(%s): unmap unused %zu of %zu " - "mmaped bytes @%p orig:%p\n", - __func__, name, usz, sz, uv, va); - munmap(uv, usz); - i = (sz - usz) / pg_sz; - for (; i < pg_num; i++) { - arg.count = 1; - arg.index = gnt_arr[i].index; - rv = ioctl(gntalloc_fd, IOCTL_GNTALLOC_DEALLOC_GREF, &arg); - if (rv) { - /* shouldn't fail here */ - RTE_LOG(ERR, PMD, "va=%p pa=%"PRIu64"x index=%"PRIu64" %s\n", - gnt_arr[i].va, - gnt_arr[i].pa, - arg.index, strerror(errno)); - rte_panic("gntdealloc failed when freeing pages\n"); - } - } - - rpg_num = (sz - usz) >> pg_shift; - } else - rpg_num = pg_num; - - mp = rte_mempool_xmem_create(name, elt_num, elt_size, - cache_size, private_data_size, - mp_init, mp_init_arg, - obj_init, obj_init_arg, - socket_id, flags, va, pa_arr, rpg_num, pg_shift); - - RTE_ASSERT(elt_num == mp->size); - } - mgi.mp = mp; - mgi.pg_num = rpg_num; - mgi.gref_arr = gref_arr; - mgi.pa_arr = pa_arr; - if (mp) - mgi.mempool_idx = rte_atomic32_add_return(&global_xenvirt_mempool_idx, 1); - mgi.start_index = start_index; - mgi.va = va; - - if (mp == NULL) { - i = pg_num; - goto mmap_failed; - } - -/* - * unmap only, without deallocate grant reference. - * unused pages have already been unmaped, - * unmap twice will fail, but it is safe. - */ -mmap_failed: - for (j = 0; j < i; j++) { - if (gnt_arr[i].va) - munmap(gnt_arr[i].va, pg_sz); - } -out: - free(gnt_arr); - if (orig_va) - munmap(orig_va, sz); - if (mp == NULL) { - free(gref_arr); - free(pa_arr); - - /* some gref has already been de-allocated from the list in the driver, - * so dealloc one by one, and it is safe to deallocate twice - */ - if (orig_va) { - for (i = 0; i < pg_num; i++) { - arg.index = start_index + i * pg_sz; - rv = ioctl(gntalloc_fd, IOCTL_GNTALLOC_DEALLOC_GREF, arg); - } - } - } - return mgi; -} - -struct rte_mempool * -rte_mempool_gntalloc_create(const char *name, unsigned elt_num, unsigned elt_size, - unsigned cache_size, unsigned private_data_size, - rte_mempool_ctor_t *mp_init, void *mp_init_arg, - rte_mempool_obj_cb_t *obj_init, void *obj_init_arg, - int socket_id, unsigned flags) -{ - int rv; - uint32_t i; - struct _mempool_gntalloc_info mgi; - struct ioctl_gntalloc_dealloc_gref arg; - int pg_sz = getpagesize(); - - mgi = _create_mempool(name, elt_num, elt_size, - cache_size, private_data_size, - mp_init, mp_init_arg, - obj_init, obj_init_arg, - socket_id, flags); - if (mgi.mp) { - rv = grant_gntalloc_mbuf_pool(mgi.mp, - mgi.pg_num, - mgi.gref_arr, - mgi.pa_arr, - mgi.mempool_idx); - free(mgi.gref_arr); - free(mgi.pa_arr); - if (rv == 0) - return mgi.mp; - /* - * in _create_mempool, unused pages have already been unmapped, deallocagted - * unmap and dealloc the remained ones here. - */ - munmap(mgi.va, pg_sz * mgi.pg_num); - for (i = 0; i < mgi.pg_num; i++) { - arg.index = mgi.start_index + i * pg_sz; - rv = ioctl(gntalloc_fd, IOCTL_GNTALLOC_DEALLOC_GREF, arg); - } - return NULL; - } - return NULL; - - - -} diff --git a/dpdk/drivers/net/xenvirt/rte_xen_lib.c b/dpdk/drivers/net/xenvirt/rte_xen_lib.c deleted file mode 100644 index 6c9a1d49..00000000 --- a/dpdk/drivers/net/xenvirt/rte_xen_lib.c +++ /dev/null @@ -1,454 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#if __XEN_LATEST_INTERFACE_VERSION__ < 0x00040200 -#include -#else -#include -#endif -#include - -#include -#include -#include - -#include "rte_xen_lib.h" - -/* - * The grant node format in xenstore for vring/mpool is: - * 0_rx_vring_gref = "gref1#, gref2#, gref3#" - * 0_mempool_gref = "gref1#, gref2#, gref3#" - * each gref# is a grant reference for a shared page. - * In each shared page, we store the grant_node_item items. - */ -struct grant_node_item { - uint32_t gref; - uint32_t pfn; -} __attribute__((packed)); - -/* fd for xen_gntalloc driver, used to allocate grant pages*/ -int gntalloc_fd = -1; - -/* xenstore path for local domain, now it is '/local/domain/domid/' */ -static char *dompath = NULL; -/* handle to xenstore read/write operations */ -static struct xs_handle *xs = NULL; -/* flag to indicate if xenstore cleanup is required */ -static bool is_xenstore_cleaned_up; - -/* - * Reserve a virtual address space. - * On success, returns the pointer. On failure, returns NULL. - */ -void * -get_xen_virtual(size_t size, size_t page_sz) -{ - void *addr; - uintptr_t aligned_addr; - - addr = mmap(NULL, size + page_sz, PROT_READ, MAP_SHARED | MAP_ANONYMOUS, -1, 0); - if (addr == MAP_FAILED) { - RTE_LOG(ERR, PMD, "failed get a virtual area\n"); - return NULL; - } - - aligned_addr = RTE_ALIGN_CEIL((uintptr_t)addr, page_sz); - addr = (void *)(aligned_addr); - - return addr; -} - -/* - * Get the physical address for virtual memory starting at va. - */ -int -get_phys_map(void *va, phys_addr_t pa[], uint32_t pg_num, uint32_t pg_sz) -{ - int32_t fd, rc = 0; - uint32_t i, nb; - off_t ofs; - - ofs = (uintptr_t)va / pg_sz * sizeof(*pa); - nb = pg_num * sizeof(*pa); - - if ((fd = open(PAGEMAP_FNAME, O_RDONLY)) < 0 || - (rc = pread(fd, pa, nb, ofs)) < 0 || - (rc -= nb) != 0) { - RTE_LOG(ERR, PMD, "%s: failed read of %u bytes from \'%s\' " - "at offset %lu, error code: %d\n", - __func__, nb, PAGEMAP_FNAME, (unsigned long)ofs, errno); - rc = ENOENT; - } - - close(fd); - for (i = 0; i != pg_num; i++) - pa[i] = (pa[i] & PAGEMAP_PFN_MASK) * pg_sz; - - return rc; -} - -int -gntalloc_open(void) -{ - gntalloc_fd = open(XEN_GNTALLOC_FNAME, O_RDWR); - return (gntalloc_fd != -1) ? 0 : -1; -} - -void -gntalloc_close(void) -{ - if (gntalloc_fd != -1) - close(gntalloc_fd); - gntalloc_fd = -1; -} - -void * -gntalloc(size_t size, uint32_t *gref, uint64_t *start_index) -{ - int page_size = getpagesize(); - uint32_t i, pg_num; - void *va; - int rv; - struct ioctl_gntalloc_alloc_gref *arg; - struct ioctl_gntalloc_dealloc_gref arg_d; - - if (size % page_size) { - RTE_LOG(ERR, PMD, "%s: %zu isn't multiple of page size\n", - __func__, size); - return NULL; - } - - pg_num = size / page_size; - arg = malloc(sizeof(*arg) + (pg_num - 1) * sizeof(uint32_t)); - if (arg == NULL) - return NULL; - arg->domid = DOM0_DOMID; - arg->flags = GNTALLOC_FLAG_WRITABLE; - arg->count = pg_num; - - rv = ioctl(gntalloc_fd, IOCTL_GNTALLOC_ALLOC_GREF, arg); - if (rv) { - RTE_LOG(ERR, PMD, "%s: ioctl error\n", __func__); - free(arg); - return NULL; - } - - va = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, gntalloc_fd, arg->index); - if (va == MAP_FAILED) { - RTE_LOG(ERR, PMD, "%s: mmap failed\n", __func__); - arg_d.count = pg_num; - arg_d.index = arg->index; - ioctl(gntalloc_fd, IOCTL_GNTALLOC_DEALLOC_GREF, arg_d); - free(arg); - return NULL; - } - - if (gref) { - for (i = 0; i < pg_num; i++) { - gref[i] = arg->gref_ids[i]; - } - } - if (start_index) - *start_index = arg->index; - - free(arg); - - return va; -} - -int -grefwatch_from_alloc(uint32_t *gref, void **pptr) -{ - int rv; - void *ptr; - int pg_size = getpagesize(); - struct ioctl_gntalloc_alloc_gref arg = { - .domid = DOM0_DOMID, - .flags = GNTALLOC_FLAG_WRITABLE, - .count = 1 - }; - struct ioctl_gntalloc_dealloc_gref arg_d; - struct ioctl_gntalloc_unmap_notify notify = { - .action = UNMAP_NOTIFY_CLEAR_BYTE - }; - - rv = ioctl(gntalloc_fd, IOCTL_GNTALLOC_ALLOC_GREF, &arg); - if (rv) { - RTE_LOG(ERR, PMD, "%s: ioctl error\n", __func__); - return -1; - } - - ptr = (void *)mmap(NULL, pg_size, PROT_READ|PROT_WRITE, MAP_SHARED, gntalloc_fd, arg.index); - arg_d.index = arg.index; - arg_d.count = 1; - if (ptr == MAP_FAILED) { - RTE_LOG(ERR, PMD, "%s: mmap failed\n", __func__); - ioctl(gntalloc_fd, IOCTL_GNTALLOC_DEALLOC_GREF, &arg_d); - return -1; - } - if (pptr) - *pptr = ptr; - if (gref) - *gref = arg.gref_ids[0]; - - notify.index = arg.index; - rv = ioctl(gntalloc_fd, IOCTL_GNTALLOC_SET_UNMAP_NOTIFY, ¬ify); - if (rv) { - RTE_LOG(ERR, PMD, "%s: unmap notify failed\n", __func__); - munmap(ptr, pg_size); - ioctl(gntalloc_fd, IOCTL_GNTALLOC_DEALLOC_GREF, &arg_d); - return -1; - } - - return 0; -} - -void -gntfree(void *va, size_t sz, uint64_t start_index) -{ - struct ioctl_gntalloc_dealloc_gref arg_d; - - if (va && sz) { - munmap(va, sz); - arg_d.count = sz / getpagesize(); - arg_d.index = start_index; - ioctl(gntalloc_fd, IOCTL_GNTALLOC_DEALLOC_GREF, &arg_d); - } -} - -static int -xenstore_cleanup(void) -{ - char store_path[PATH_MAX] = {0}; - - if (snprintf(store_path, sizeof(store_path), - "%s%s", dompath, DPDK_XENSTORE_NODE) == -1) - return -1; - - if (xs_rm(xs, XBT_NULL, store_path) == false) { - RTE_LOG(ERR, PMD, "%s: failed cleanup node\n", __func__); - return -1; - } - - return 0; -} - -int -xenstore_init(void) -{ - unsigned int len, domid; - char *buf; - char *end; - - xs = xs_domain_open(); - if (xs == NULL) { - RTE_LOG(ERR, PMD,"%s: xs_domain_open failed\n", __func__); - return -1; - } - buf = xs_read(xs, XBT_NULL, "domid", &len); - if (buf == NULL) { - RTE_LOG(ERR, PMD, "%s: failed read domid\n", __func__); - return -1; - } - errno = 0; - domid = strtoul(buf, &end, 0); - if (errno != 0 || end == NULL || end == buf || domid == 0) - return -1; - - RTE_LOG(INFO, PMD, "retrieved dom ID = %d\n", domid); - - dompath = xs_get_domain_path(xs, domid); - if (dompath == NULL) - return -1; - - xs_transaction_start(xs); /* When to stop transaction */ - - if (is_xenstore_cleaned_up == 0) { - if (xenstore_cleanup()) - return -1; - is_xenstore_cleaned_up = 1; - } - - return 0; -} - -int -xenstore_uninit(void) -{ - xs_close(xs); - - if (is_xenstore_cleaned_up == 0) { - if (xenstore_cleanup()) - return -1; - is_xenstore_cleaned_up = 1; - } - free(dompath); - dompath = NULL; - - return 0; -} - -int -xenstore_write(const char *key_str, const char *val_str) -{ - char grant_path[PATH_MAX]; - int rv, len; - - if (xs == NULL) { - RTE_LOG(ERR, PMD, "%s: xenstore init failed\n", __func__); - return -1; - } - rv = snprintf(grant_path, sizeof(grant_path), "%s%s", dompath, key_str); - if (rv == -1) { - RTE_LOG(ERR, PMD, "%s: snprintf %s %s failed\n", - __func__, dompath, key_str); - return -1; - } - len = strnlen(val_str, PATH_MAX); - - if (xs_write(xs, XBT_NULL, grant_path, val_str, len) == false) { - RTE_LOG(ERR, PMD, "%s: xs_write failed\n", __func__); - return -1; - } - - return 0; -} - -int -grant_node_create(uint32_t pg_num, uint32_t *gref_arr, phys_addr_t *pa_arr, char *val_str, size_t str_size) -{ - uint64_t start_index; - int pg_size; - uint32_t pg_shift; - void *ptr = NULL; - uint32_t count, entries_per_pg; - uint32_t i, j = 0, k = 0; - uint32_t *gref_tmp; - int first = 1; - char tmp_str[PATH_MAX] = {0}; - int rv = -1; - - pg_size = getpagesize(); - if (rte_is_power_of_2(pg_size) == 0) { - return -1; - } - pg_shift = rte_bsf32(pg_size); - if (pg_size % sizeof(struct grant_node_item)) { - RTE_LOG(ERR, PMD, "pg_size isn't a multiple of grant node item\n"); - return -1; - } - - entries_per_pg = pg_size / sizeof(struct grant_node_item); - count = (pg_num + entries_per_pg - 1 ) / entries_per_pg; - gref_tmp = malloc(count * sizeof(uint32_t)); - if (gref_tmp == NULL) - return -1; - ptr = gntalloc(pg_size * count, gref_tmp, &start_index); - if (ptr == NULL) { - RTE_LOG(ERR, PMD, "%s: gntalloc error of %d pages\n", __func__, count); - free(gref_tmp); - return -1; - } - - while (j < pg_num) { - if (first) { - rv = snprintf(val_str, str_size, "%u", gref_tmp[k]); - first = 0; - } else { - snprintf(tmp_str, PATH_MAX, "%s", val_str); - rv = snprintf(val_str, str_size, "%s,%u", tmp_str, gref_tmp[k]); - } - k++; - if (rv == -1) - break; - - for (i = 0; i < entries_per_pg && j < pg_num ; i++) { - ((struct grant_node_item *)ptr)->gref = gref_arr[j]; - ((struct grant_node_item *)ptr)->pfn = pa_arr[j] >> pg_shift; - ptr = RTE_PTR_ADD(ptr, sizeof(struct grant_node_item)); - j++; - } - } - if (rv == -1) { - gntfree(ptr, pg_size * count, start_index); - } else - rv = 0; - free(gref_tmp); - return rv; -} - - -int -grant_gntalloc_mbuf_pool(struct rte_mempool *mpool, uint32_t pg_num, uint32_t *gref_arr, phys_addr_t *pa_arr, int mempool_idx) -{ - char key_str[PATH_MAX] = {0}; - char val_str[PATH_MAX] = {0}; - void *mempool_obj_va; - - if (grant_node_create(pg_num, gref_arr, pa_arr, val_str, sizeof(val_str))) { - return -1; - } - - if (snprintf(key_str, sizeof(key_str), - DPDK_XENSTORE_PATH"%d"MEMPOOL_XENSTORE_STR, mempool_idx) == -1) - return -1; - if (xenstore_write(key_str, val_str) == -1) - return -1; - - if (snprintf(key_str, sizeof(key_str), - DPDK_XENSTORE_PATH"%d"MEMPOOL_VA_XENSTORE_STR, mempool_idx) == -1) - return -1; - if (mpool->nb_mem_chunks != 1) { - RTE_LOG(ERR, PMD, - "mempool with more than 1 chunk is not supported\n"); - return -1; - } - mempool_obj_va = STAILQ_FIRST(&mpool->mem_list)->addr; - if (snprintf(val_str, sizeof(val_str), "%"PRIxPTR, - (uintptr_t)mempool_obj_va) == -1) - return -1; - if (xenstore_write(key_str, val_str) == -1) - return -1; - - return 0; -} diff --git a/dpdk/drivers/net/xenvirt/rte_xen_lib.h b/dpdk/drivers/net/xenvirt/rte_xen_lib.h deleted file mode 100644 index d973eacb..00000000 --- a/dpdk/drivers/net/xenvirt/rte_xen_lib.h +++ /dev/null @@ -1,116 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef _RTE_XEN_DUMMY_PMD_H -#define _RTE_XEN_DUMMY_PMD_H - -#include - -#include -#include -#include - -#define PAGEMAP_FNAME "/proc/self/pagemap" -#define XEN_GNTALLOC_FNAME "/dev/xen/gntalloc" -#define DPDK_XENSTORE_PATH "/control/dpdk/" -#define DPDK_XENSTORE_NODE "/control/dpdk" -/*format 0_mempool_gref = "1537,1524,1533" */ -#define MEMPOOL_XENSTORE_STR "_mempool_gref" -/*format 0_mempool_va = 0x80340000 */ -#define MEMPOOL_VA_XENSTORE_STR "_mempool_va" -/*format 0_rx_vring_gref = "1537,1524,1533" */ -#define RXVRING_XENSTORE_STR "_rx_vring_gref" -/*format 0_tx_vring_gref = "1537,1524,1533" */ -#define TXVRING_XENSTORE_STR "_tx_vring_gref" -#define VRING_FLAG_STR "_vring_flag" -/*format: event_type_start_0 = 1*/ -#define EVENT_TYPE_START_STR "event_type_start_" - -#define DOM0_DOMID 0 -/* - * the pfn (page frame number) are bits 0-54 (see pagemap.txt in linux - * Documentation). - */ -#define PAGEMAP_PFN_BITS 54 -#define PAGEMAP_PFN_MASK RTE_LEN2MASK(PAGEMAP_PFN_BITS, phys_addr_t) - -#define MAP_FLAG 0xA5 - -#define RTE_ETH_XENVIRT_PAIRS_DELIM ';' -#define RTE_ETH_XENVIRT_KEY_VALUE_DELIM '=' -#define RTE_ETH_XENVIRT_MAX_ARGS 1 -#define RTE_ETH_XENVIRT_MAC_PARAM "mac" -struct xenvirt_dict { - uint8_t addr_valid; - struct ether_addr addr; -}; - -extern int gntalloc_fd; - -int -gntalloc_open(void); - -void -gntalloc_close(void); - -void * -gntalloc(size_t sz, uint32_t *gref, uint64_t *start_index); - -void -gntfree(void *va, size_t sz, uint64_t start_index); - -int -xenstore_init(void); - -int -xenstore_uninit(void); - -int -xenstore_write(const char *key_str, const char *val_str); - -int -get_phys_map(void *va, phys_addr_t pa[], uint32_t pg_num, uint32_t pg_sz); - -void * -get_xen_virtual(size_t size, size_t page_sz); - -int -grefwatch_from_alloc(uint32_t *gref, void **pptr); - - -int grant_node_create(uint32_t pg_num, uint32_t *gref_arr, phys_addr_t *pa_arr, char *val_str, size_t str_size); - -int -grant_gntalloc_mbuf_pool(struct rte_mempool *mpool, uint32_t pg_num, uint32_t *gref_arr, phys_addr_t *pa_arr, int mempool_idx); - -#endif diff --git a/dpdk/drivers/net/xenvirt/virtio_logs.h b/dpdk/drivers/net/xenvirt/virtio_logs.h deleted file mode 100644 index d6c33f7b..00000000 --- a/dpdk/drivers/net/xenvirt/virtio_logs.h +++ /dev/null @@ -1,70 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef _VIRTIO_LOGS_H_ -#define _VIRTIO_LOGS_H_ - -#include - -#ifdef RTE_LIBRTE_VIRTIO_DEBUG_INIT -#define PMD_INIT_LOG(level, fmt, args...) \ - RTE_LOG(level, PMD, "%s(): " fmt "\n", __func__, ## args) -#define PMD_INIT_FUNC_TRACE() PMD_INIT_LOG(DEBUG, " >>") -#else -#define PMD_INIT_LOG(level, fmt, args...) do { } while(0) -#define PMD_INIT_FUNC_TRACE() do { } while(0) -#endif - -#ifdef RTE_LIBRTE_VIRTIO_DEBUG_RX -#define PMD_RX_LOG(level, fmt, args...) \ - RTE_LOG(level, PMD, "%s() rx: " fmt , __func__, ## args) -#else -#define PMD_RX_LOG(level, fmt, args...) do { } while(0) -#endif - -#ifdef RTE_LIBRTE_VIRTIO_DEBUG_TX -#define PMD_TX_LOG(level, fmt, args...) \ - RTE_LOG(level, PMD, "%s() tx: " fmt , __func__, ## args) -#else -#define PMD_TX_LOG(level, fmt, args...) do { } while(0) -#endif - - -#ifdef RTE_LIBRTE_VIRTIO_DEBUG_DRIVER -#define PMD_DRV_LOG(level, fmt, args...) \ - RTE_LOG(level, PMD, "%s(): " fmt , __func__, ## args) -#else -#define PMD_DRV_LOG(level, fmt, args...) do { } while(0) -#endif - -#endif /* _VIRTIO_LOGS_H_ */ diff --git a/dpdk/drivers/net/xenvirt/virtqueue.h b/dpdk/drivers/net/xenvirt/virtqueue.h deleted file mode 100644 index 350eae3e..00000000 --- a/dpdk/drivers/net/xenvirt/virtqueue.h +++ /dev/null @@ -1,273 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef _VIRTQUEUE_H_ -#define _VIRTQUEUE_H_ - -#include -#include -#include - -#include -#include -#include -#include - -#include "virtio_logs.h" - -struct rte_mbuf; - -/* The alignment to use between consumer and producer parts of vring. */ -#define VIRTIO_PCI_VRING_ALIGN 4096 - -enum { VTNET_RQ = 0, VTNET_TQ = 1, VTNET_CQ = 2 }; - -/** - * The maximum virtqueue size is 2^15. Use that value as the end of - * descriptor chain terminator since it will never be a valid index - * in the descriptor table. This is used to verify we are correctly - * handling vq_free_cnt. - */ -#define VQ_RING_DESC_CHAIN_END 32768 - -#define VIRTQUEUE_MAX_NAME_SZ 32 - -struct pmd_internals { - struct rte_eth_stats eth_stats; - int port_id; - int virtio_idx; -}; - - -struct virtqueue { - char vq_name[VIRTQUEUE_MAX_NAME_SZ]; - struct rte_mempool *mpool; /**< mempool for mbuf allocation */ - uint16_t queue_id; /**< DPDK queue index. */ - uint16_t vq_queue_index; /**< PCI queue index */ - uint8_t port_id; /**< Device port identifier. */ - - void *vq_ring_virt_mem; /**< virtual address of vring*/ - int vq_alignment; - int vq_ring_size; - - struct vring vq_ring; /**< vring keeping desc, used and avail */ - struct pmd_internals *internals; /**< virtio device internal info. */ - uint16_t vq_nentries; /**< vring desc numbers */ - uint16_t vq_desc_head_idx; - uint16_t vq_free_cnt; /**< num of desc available */ - uint16_t vq_used_cons_idx; /**< Last consumed desc in used table, trails vq_ring.used->idx*/ - - struct vq_desc_extra { - void *cookie; - uint16_t ndescs; - } vq_descx[0] __rte_cache_aligned; -}; - - -#ifdef RTE_LIBRTE_XENVIRT_DEBUG_DUMP -#define VIRTQUEUE_DUMP(vq) do { \ - uint16_t used_idx, nused; \ - used_idx = (vq)->vq_ring.used->idx; \ - nused = (uint16_t)(used_idx - (vq)->vq_used_cons_idx); \ - PMD_INIT_LOG(DEBUG, \ - "VQ: %s - size=%d; free=%d; used=%d; desc_head_idx=%d;" \ - " avail.idx=%d; used_cons_idx=%d; used.idx=%d;" \ - " avail.flags=0x%x; used.flags=0x%x\n", \ - (vq)->vq_name, (vq)->vq_nentries, (vq)->vq_free_cnt, nused, \ - (vq)->vq_desc_head_idx, (vq)->vq_ring.avail->idx, \ - (vq)->vq_used_cons_idx, (vq)->vq_ring.used->idx, \ - (vq)->vq_ring.avail->flags, (vq)->vq_ring.used->flags); \ -} while (0) -#else -#define VIRTQUEUE_DUMP(vq) do { } while (0) -#endif - - -/** - * Dump virtqueue internal structures, for debug purpose only. - */ -void virtqueue_dump(struct virtqueue *vq); - -/** - * Get all mbufs to be freed. - */ -struct rte_mbuf * virtqueue_detatch_unused(struct virtqueue *vq); - -static inline int __attribute__((always_inline)) -virtqueue_full(const struct virtqueue *vq) -{ - return vq->vq_free_cnt == 0; -} - -#define VIRTQUEUE_NUSED(vq) ((uint16_t)((vq)->vq_ring.used->idx - (vq)->vq_used_cons_idx)) - -static inline void __attribute__((always_inline)) -vq_ring_update_avail(struct virtqueue *vq, uint16_t desc_idx) -{ - uint16_t avail_idx; - /* - * Place the head of the descriptor chain into the next slot and make - * it usable to the host. The chain is made available now rather than - * deferring to virtqueue_notify() in the hopes that if the host is - * currently running on another CPU, we can keep it processing the new - * descriptor. - */ - avail_idx = (uint16_t)(vq->vq_ring.avail->idx & (vq->vq_nentries - 1)); - vq->vq_ring.avail->ring[avail_idx] = desc_idx; - rte_smp_wmb(); - vq->vq_ring.avail->idx++; -} - -static inline void __attribute__((always_inline)) -vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx) -{ - struct vring_desc *dp; - struct vq_desc_extra *dxp; - - dp = &vq->vq_ring.desc[desc_idx]; - dxp = &vq->vq_descx[desc_idx]; - vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt + dxp->ndescs); - while (dp->flags & VRING_DESC_F_NEXT) { - dp = &vq->vq_ring.desc[dp->next]; - } - dxp->ndescs = 0; - - /* - * We must append the existing free chain, if any, to the end of - * newly freed chain. If the virtqueue was completely used, then - * head would be VQ_RING_DESC_CHAIN_END (ASSERTed above). - */ - dp->next = vq->vq_desc_head_idx; - vq->vq_desc_head_idx = desc_idx; -} - -static inline int __attribute__((always_inline)) -virtqueue_enqueue_recv_refill(struct virtqueue *rxvq, struct rte_mbuf *cookie) -{ - const uint16_t needed = 1; - const uint16_t head_idx = rxvq->vq_desc_head_idx; - struct vring_desc *start_dp = rxvq->vq_ring.desc; - struct vq_desc_extra *dxp; - - if (unlikely(rxvq->vq_free_cnt == 0)) - return -ENOSPC; - if (unlikely(rxvq->vq_free_cnt < needed)) - return -EMSGSIZE; - if (unlikely(head_idx >= rxvq->vq_nentries)) - return -EFAULT; - - dxp = &rxvq->vq_descx[head_idx]; - dxp->cookie = (void *)cookie; - dxp->ndescs = needed; - - start_dp[head_idx].addr = - (uint64_t) ((uintptr_t)cookie->buf_addr + RTE_PKTMBUF_HEADROOM - sizeof(struct virtio_net_hdr)); - start_dp[head_idx].len = cookie->buf_len - RTE_PKTMBUF_HEADROOM + sizeof(struct virtio_net_hdr); - start_dp[head_idx].flags = VRING_DESC_F_WRITE; - rxvq->vq_desc_head_idx = start_dp[head_idx].next; - rxvq->vq_free_cnt = (uint16_t)(rxvq->vq_free_cnt - needed); - vq_ring_update_avail(rxvq, head_idx); - - return 0; -} - -static inline int __attribute__((always_inline)) -virtqueue_enqueue_xmit(struct virtqueue *txvq, struct rte_mbuf *cookie) -{ - - const uint16_t needed = 2; - struct vring_desc *start_dp = txvq->vq_ring.desc; - uint16_t head_idx = txvq->vq_desc_head_idx; - uint16_t idx = head_idx; - struct vq_desc_extra *dxp; - - if (unlikely(txvq->vq_free_cnt == 0)) - return -ENOSPC; - if (unlikely(txvq->vq_free_cnt < needed)) - return -EMSGSIZE; - if (unlikely(head_idx >= txvq->vq_nentries)) - return -EFAULT; - - dxp = &txvq->vq_descx[idx]; - dxp->cookie = (void *)cookie; - dxp->ndescs = needed; - - start_dp = txvq->vq_ring.desc; - start_dp[idx].addr = 0; -/* - * TODO: save one desc here? - */ - start_dp[idx].len = sizeof(struct virtio_net_hdr); - start_dp[idx].flags = VRING_DESC_F_NEXT; - start_dp[idx].addr = (uintptr_t)NULL; - idx = start_dp[idx].next; - start_dp[idx].addr = (uint64_t)rte_pktmbuf_mtod(cookie, uintptr_t); - start_dp[idx].len = cookie->data_len; - start_dp[idx].flags = 0; - idx = start_dp[idx].next; - txvq->vq_desc_head_idx = idx; - txvq->vq_free_cnt = (uint16_t)(txvq->vq_free_cnt - needed); - vq_ring_update_avail(txvq, head_idx); - - return 0; -} - -static inline uint16_t __attribute__((always_inline)) -virtqueue_dequeue_burst(struct virtqueue *vq, struct rte_mbuf **rx_pkts, uint32_t *len, uint16_t num) -{ - struct vring_used_elem *uep; - struct rte_mbuf *cookie; - uint16_t used_idx, desc_idx; - uint16_t i; - /* Caller does the check */ - for (i = 0; i < num ; i ++) { - used_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries - 1)); - uep = &vq->vq_ring.used->ring[used_idx]; - desc_idx = (uint16_t) uep->id; - cookie = (struct rte_mbuf *)vq->vq_descx[desc_idx].cookie; - if (unlikely(cookie == NULL)) { - PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u\n", - vq->vq_used_cons_idx); - RTE_LOG(ERR, PMD, "%s: inconsistent (%u, %u)\n", __func__, used_idx , desc_idx); - break; - } - len[i] = uep->len; - rx_pkts[i] = cookie; - vq->vq_used_cons_idx++; - vq_ring_free_chain(vq, desc_idx); - vq->vq_descx[desc_idx].cookie = NULL; - } - return i; -} - -#endif /* _VIRTQUEUE_H_ */ diff --git a/dpdk/examples/Makefile b/dpdk/examples/Makefile index 18b41b90..9f7974a1 100644 --- a/dpdk/examples/Makefile +++ b/dpdk/examples/Makefile @@ -32,7 +32,7 @@ ifeq ($(RTE_SDK),) $(error "Please define RTE_SDK environment variable") endif -# Default target, can be overriden by command line or environment +# Default target, can be overridden by command line or environment RTE_TARGET ?= x86_64-native-linuxapp-gcc include $(RTE_SDK)/mk/rte.vars.mk @@ -40,18 +40,18 @@ include $(RTE_SDK)/mk/rte.vars.mk DIRS-$(CONFIG_RTE_LIBRTE_PMD_BOND) += bond DIRS-y += cmdline DIRS-$(CONFIG_RTE_LIBRTE_DISTRIBUTOR) += distributor -ifneq ($(ICP_ROOT),) -DIRS-y += dpdk_qat -endif DIRS-y += ethtool DIRS-y += exception_path +DIRS-$(CONFIG_RTE_LIBRTE_EFD) += server_node_efd +DIRS-$(CONFIG_RTE_LIBRTE_FLOW_CLASSIFY) += flow_classify +DIRS-y += flow_filtering DIRS-y += helloworld DIRS-$(CONFIG_RTE_LIBRTE_PIPELINE) += ip_pipeline ifeq ($(CONFIG_RTE_LIBRTE_LPM),y) DIRS-$(CONFIG_RTE_IP_FRAG) += ip_reassembly DIRS-$(CONFIG_RTE_IP_FRAG) += ip_fragmentation endif -ifeq ($(CONFIG_RTE_LIBRTE_ACL)$(CONFIG_RTE_LIBRTE_HASH)$(CONFIG_RTE_LIBRTE_LPM),yyy) +ifeq ($(CONFIG_RTE_LIBRTE_ACL)$(CONFIG_RTE_LIBRTE_HASH)$(CONFIG_RTE_LIBRTE_LPM)$(CONFIG_RTE_LIBRTE_SECURITY),yyyy) DIRS-$(CONFIG_RTE_LIBRTE_CRYPTODEV) += ipsec-secgw endif DIRS-$(CONFIG_RTE_LIBRTE_HASH) += ipv4_multicast @@ -61,7 +61,6 @@ ifneq ($(PQOS_INSTALL_PATH),) DIRS-y += l2fwd-cat endif DIRS-$(CONFIG_RTE_LIBRTE_CRYPTODEV) += l2fwd-crypto -DIRS-$(CONFIG_RTE_LIBRTE_IVSHMEM) += l2fwd-ivshmem DIRS-$(CONFIG_RTE_LIBRTE_JOBSTATS) += l2fwd-jobstats DIRS-y += l2fwd-keepalive DIRS-y += l2fwd-keepalive/ka-agent @@ -78,18 +77,21 @@ DIRS-$(CONFIG_RTE_LIBRTE_LPM) += load_balancer DIRS-y += multi_process DIRS-y += netmap_compat/bridge DIRS-$(CONFIG_RTE_LIBRTE_REORDER) += packet_ordering +ifeq ($(CONFIG_RTE_ARCH_X86_64),y) +DIRS-y += performance-thread +endif DIRS-$(CONFIG_RTE_LIBRTE_IEEE1588) += ptpclient DIRS-$(CONFIG_RTE_LIBRTE_METER) += qos_meter DIRS-$(CONFIG_RTE_LIBRTE_SCHED) += qos_sched DIRS-y += quota_watermark DIRS-$(CONFIG_RTE_ETHDEV_RXTX_CALLBACKS) += rxtx_callbacks +DIRS-y += service_cores DIRS-y += skeleton ifeq ($(CONFIG_RTE_LIBRTE_HASH),y) DIRS-$(CONFIG_RTE_LIBRTE_VHOST) += tep_termination endif DIRS-$(CONFIG_RTE_LIBRTE_TIMER) += timer -DIRS-$(CONFIG_RTE_LIBRTE_VHOST) += vhost -DIRS-$(CONFIG_RTE_LIBRTE_XEN_DOM0) += vhost_xen +DIRS-$(CONFIG_RTE_LIBRTE_VHOST) += vhost vhost_scsi DIRS-y += vmdq DIRS-y += vmdq_dcb ifeq ($(CONFIG_RTE_LIBRTE_POWER), y) @@ -100,4 +102,6 @@ $(info vm_power_manager requires libvirt >= 0.9.3) endif endif +DIRS-y += eventdev_pipeline_sw_pmd + include $(RTE_SDK)/mk/rte.extsubdir.mk diff --git a/dpdk/examples/bond/Makefile b/dpdk/examples/bond/Makefile index 626d79d9..ae4cb6e1 100644 --- a/dpdk/examples/bond/Makefile +++ b/dpdk/examples/bond/Makefile @@ -54,4 +54,8 @@ endif CFLAGS += -O3 +ifeq ($(CONFIG_RTE_BUILD_SHARED_LIB),y) +LDLIBS += -lrte_pmd_bond +endif + include $(RTE_SDK)/mk/rte.extapp.mk diff --git a/dpdk/examples/bond/main.c b/dpdk/examples/bond/main.c index 776fad0a..4f533e2c 100644 --- a/dpdk/examples/bond/main.c +++ b/dpdk/examples/bond/main.c @@ -51,9 +51,7 @@ #include #include #include -#include #include -#include #include #include #include @@ -62,16 +60,12 @@ #include #include #include -#include #include #include #include #include -#include -#include #include #include -#include #include #include #include @@ -145,10 +139,10 @@ addr.addr_bytes[0], addr.addr_bytes[1], addr.addr_bytes[2], \ addr.addr_bytes[3], addr.addr_bytes[4], addr.addr_bytes[5]) -uint8_t slaves[RTE_MAX_ETHPORTS]; -uint8_t slaves_count; +uint16_t slaves[RTE_MAX_ETHPORTS]; +uint16_t slaves_count; -static uint8_t BOND_PORT = 0xff; +static uint16_t BOND_PORT = 0xffff; static struct rte_mempool *mbuf_pool; @@ -161,7 +155,7 @@ static struct rte_eth_conf port_conf = { .hw_ip_checksum = 0, /**< IP checksum offload enabled */ .hw_vlan_filter = 0, /**< VLAN filtering disabled */ .jumbo_frame = 0, /**< Jumbo Frame Support disabled */ - .hw_strip_crc = 0, /**< CRC stripped by hardware */ + .hw_strip_crc = 1, /**< CRC stripped by hardware */ }, .rx_adv_conf = { .rss_conf = { @@ -175,9 +169,11 @@ static struct rte_eth_conf port_conf = { }; static void -slave_port_init(uint8_t portid, struct rte_mempool *mbuf_pool) +slave_port_init(uint16_t portid, struct rte_mempool *mbuf_pool) { int retval; + uint16_t nb_rxd = RTE_RX_DESC_DEFAULT; + uint16_t nb_txd = RTE_TX_DESC_DEFAULT; if (portid >= rte_eth_dev_count()) rte_exit(EXIT_FAILURE, "Invalid port\n"); @@ -187,8 +183,13 @@ slave_port_init(uint8_t portid, struct rte_mempool *mbuf_pool) rte_exit(EXIT_FAILURE, "port %u: configuration failed (res=%d)\n", portid, retval); + retval = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd, &nb_txd); + if (retval != 0) + rte_exit(EXIT_FAILURE, "port %u: rte_eth_dev_adjust_nb_rx_tx_desc " + "failed (res=%d)\n", portid, retval); + /* RX setup */ - retval = rte_eth_rx_queue_setup(portid, 0, RTE_RX_DESC_DEFAULT, + retval = rte_eth_rx_queue_setup(portid, 0, nb_rxd, rte_eth_dev_socket_id(portid), NULL, mbuf_pool); if (retval < 0) @@ -196,7 +197,7 @@ slave_port_init(uint8_t portid, struct rte_mempool *mbuf_pool) portid, retval); /* TX setup */ - retval = rte_eth_tx_queue_setup(portid, 0, RTE_TX_DESC_DEFAULT, + retval = rte_eth_tx_queue_setup(portid, 0, nb_txd, rte_eth_dev_socket_id(portid), NULL); if (retval < 0) @@ -212,7 +213,7 @@ slave_port_init(uint8_t portid, struct rte_mempool *mbuf_pool) struct ether_addr addr; rte_eth_macaddr_get(portid, &addr); - printf("Port %u MAC: ", (unsigned)portid); + printf("Port %u MAC: ", portid); PRINT_MAC(addr); printf("\n"); } @@ -222,22 +223,29 @@ bond_port_init(struct rte_mempool *mbuf_pool) { int retval; uint8_t i; + uint16_t nb_rxd = RTE_RX_DESC_DEFAULT; + uint16_t nb_txd = RTE_TX_DESC_DEFAULT; - retval = rte_eth_bond_create("bond0", BONDING_MODE_ALB, + retval = rte_eth_bond_create("net_bonding0", BONDING_MODE_ALB, 0 /*SOCKET_ID_ANY*/); if (retval < 0) rte_exit(EXIT_FAILURE, "Faled to create bond port\n"); - BOND_PORT = (uint8_t)retval; + BOND_PORT = retval; retval = rte_eth_dev_configure(BOND_PORT, 1, 1, &port_conf); if (retval != 0) rte_exit(EXIT_FAILURE, "port %u: configuration failed (res=%d)\n", BOND_PORT, retval); + retval = rte_eth_dev_adjust_nb_rx_tx_desc(BOND_PORT, &nb_rxd, &nb_txd); + if (retval != 0) + rte_exit(EXIT_FAILURE, "port %u: rte_eth_dev_adjust_nb_rx_tx_desc " + "failed (res=%d)\n", BOND_PORT, retval); + /* RX setup */ - retval = rte_eth_rx_queue_setup(BOND_PORT, 0, RTE_RX_DESC_DEFAULT, + retval = rte_eth_rx_queue_setup(BOND_PORT, 0, nb_rxd, rte_eth_dev_socket_id(BOND_PORT), NULL, mbuf_pool); if (retval < 0) @@ -245,7 +253,7 @@ bond_port_init(struct rte_mempool *mbuf_pool) BOND_PORT, retval); /* TX setup */ - retval = rte_eth_tx_queue_setup(BOND_PORT, 0, RTE_TX_DESC_DEFAULT, + retval = rte_eth_tx_queue_setup(BOND_PORT, 0, nb_txd, rte_eth_dev_socket_id(BOND_PORT), NULL); if (retval < 0) @@ -438,6 +446,11 @@ static void cmd_obj_send_parsed(void *parsed_result, (BOND_IP_3 << 16) | (BOND_IP_4 << 24); created_pkt = rte_pktmbuf_alloc(mbuf_pool); + if (created_pkt == NULL) { + cmdline_printf(cl, "Failed to allocate mbuf\n"); + return; + } + pkt_size = sizeof(struct ether_hdr) + sizeof(struct arp_hdr); created_pkt->data_len = pkt_size; created_pkt->pkt_len = pkt_size; @@ -551,7 +564,7 @@ static void cmd_help_parsed(__attribute__((unused)) void *parsed_result, { cmdline_printf(cl, "ALB - link bonding mode 6 example\n" - "send IP - sends one ARPrequest thru bonding for IP.\n" + "send IP - sends one ARPrequest through bonding for IP.\n" "start - starts listening ARPs.\n" "stop - stops lcore_main.\n" "show - shows some bond info: ex. active slaves etc.\n" @@ -665,10 +678,10 @@ static void cmd_show_parsed(__attribute__((unused)) void *parsed_result, struct cmdline *cl, __attribute__((unused)) void *data) { - uint8_t slaves[16] = {0}; + uint16_t slaves[16] = {0}; uint8_t len = 16; struct ether_addr addr; - uint8_t i = 0; + uint16_t i = 0; while (i < slaves_count) { rte_eth_macaddr_get(i, &addr); diff --git a/dpdk/examples/cmdline/Makefile b/dpdk/examples/cmdline/Makefile index 9ebe4355..5155a6c8 100644 --- a/dpdk/examples/cmdline/Makefile +++ b/dpdk/examples/cmdline/Makefile @@ -33,7 +33,7 @@ ifeq ($(RTE_SDK),) $(error "Please define RTE_SDK environment variable") endif -# Default target, can be overriden by command line or environment +# Default target, can be overridden by command line or environment RTE_TARGET ?= x86_64-native-linuxapp-gcc include $(RTE_SDK)/mk/rte.vars.mk diff --git a/dpdk/examples/cmdline/main.c b/dpdk/examples/cmdline/main.c index c966df03..c6de944e 100644 --- a/dpdk/examples/cmdline/main.c +++ b/dpdk/examples/cmdline/main.c @@ -71,7 +71,6 @@ #include #include -#include #include #include diff --git a/dpdk/examples/distributor/Makefile b/dpdk/examples/distributor/Makefile index 6a5badaa..404993eb 100644 --- a/dpdk/examples/distributor/Makefile +++ b/dpdk/examples/distributor/Makefile @@ -33,7 +33,7 @@ ifeq ($(RTE_SDK),) $(error "Please define RTE_SDK environment variable") endif -# Default target, can be overriden by command line or environment +# Default target, can be overridden by command line or environment RTE_TARGET ?= x86_64-native-linuxapp-gcc include $(RTE_SDK)/mk/rte.vars.mk diff --git a/dpdk/examples/distributor/main.c b/dpdk/examples/distributor/main.c index 537cee1f..61e6e6b9 100644 --- a/dpdk/examples/distributor/main.c +++ b/dpdk/examples/distributor/main.c @@ -1,8 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. + * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -44,34 +43,67 @@ #include #include #include +#include -#define RX_RING_SIZE 256 +#define RX_RING_SIZE 512 #define TX_RING_SIZE 512 #define NUM_MBUFS ((64*1024)-1) -#define MBUF_CACHE_SIZE 250 -#define BURST_SIZE 32 -#define RTE_RING_SZ 1024 +#define MBUF_CACHE_SIZE 128 +#define BURST_SIZE 64 +#define SCHED_RX_RING_SZ 8192 +#define SCHED_TX_RING_SZ 65536 +#define BURST_SIZE_TX 32 #define RTE_LOGTYPE_DISTRAPP RTE_LOGTYPE_USER1 +#define ANSI_COLOR_RED "\x1b[31m" +#define ANSI_COLOR_RESET "\x1b[0m" + /* mask of enabled ports */ static uint32_t enabled_port_mask; volatile uint8_t quit_signal; volatile uint8_t quit_signal_rx; +volatile uint8_t quit_signal_dist; +volatile uint8_t quit_signal_work; static volatile struct app_stats { struct { uint64_t rx_pkts; uint64_t returned_pkts; uint64_t enqueued_pkts; + uint64_t enqdrop_pkts; } rx __rte_cache_aligned; + int pad1 __rte_cache_aligned; + + struct { + uint64_t in_pkts; + uint64_t ret_pkts; + uint64_t sent_pkts; + uint64_t enqdrop_pkts; + } dist __rte_cache_aligned; + int pad2 __rte_cache_aligned; struct { uint64_t dequeue_pkts; uint64_t tx_pkts; + uint64_t enqdrop_pkts; } tx __rte_cache_aligned; + int pad3 __rte_cache_aligned; + + uint64_t worker_pkts[64] __rte_cache_aligned; + + int pad4 __rte_cache_aligned; + + uint64_t worker_bursts[64][8] __rte_cache_aligned; + + int pad5 __rte_cache_aligned; + + uint64_t port_rx_pkts[64] __rte_cache_aligned; + uint64_t port_tx_pkts[64] __rte_cache_aligned; } app_stats; +struct app_stats prev_app_stats; + static const struct rte_eth_conf port_conf_default = { .rxmode = { .mq_mode = ETH_MQ_RX_RSS, @@ -93,17 +125,21 @@ struct output_buffer { struct rte_mbuf *mbufs[BURST_SIZE]; }; +static void print_stats(void); + /* * Initialises a given port using global settings and with the rx buffers * coming from the mbuf_pool passed as parameter */ static inline int -port_init(uint8_t port, struct rte_mempool *mbuf_pool) +port_init(uint16_t port, struct rte_mempool *mbuf_pool) { struct rte_eth_conf port_conf = port_conf_default; const uint16_t rxRings = 1, txRings = rte_lcore_count() - 1; int retval; uint16_t q; + uint16_t nb_rxd = RX_RING_SIZE; + uint16_t nb_txd = TX_RING_SIZE; if (port >= rte_eth_dev_count()) return -1; @@ -112,8 +148,12 @@ port_init(uint8_t port, struct rte_mempool *mbuf_pool) if (retval != 0) return retval; + retval = rte_eth_dev_adjust_nb_rx_tx_desc(port, &nb_rxd, &nb_txd); + if (retval != 0) + return retval; + for (q = 0; q < rxRings; q++) { - retval = rte_eth_rx_queue_setup(port, q, RX_RING_SIZE, + retval = rte_eth_rx_queue_setup(port, q, nb_rxd, rte_eth_dev_socket_id(port), NULL, mbuf_pool); if (retval < 0) @@ -121,7 +161,7 @@ port_init(uint8_t port, struct rte_mempool *mbuf_pool) } for (q = 0; q < txRings; q++) { - retval = rte_eth_tx_queue_setup(port, q, TX_RING_SIZE, + retval = rte_eth_tx_queue_setup(port, q, nb_txd, rte_eth_dev_socket_id(port), NULL); if (retval < 0) @@ -134,13 +174,14 @@ port_init(uint8_t port, struct rte_mempool *mbuf_pool) struct rte_eth_link link; rte_eth_link_get_nowait(port, &link); - if (!link.link_status) { + while (!link.link_status) { + printf("Waiting for Link up on port %"PRIu16"\n", port); sleep(1); rte_eth_link_get_nowait(port, &link); } if (!link.link_status) { - printf("Link down on port %"PRIu8"\n", port); + printf("Link down on port %"PRIu16"\n", port); return 0; } @@ -148,7 +189,7 @@ port_init(uint8_t port, struct rte_mempool *mbuf_pool) rte_eth_macaddr_get(port, &addr); printf("Port %u MAC: %02"PRIx8" %02"PRIx8" %02"PRIx8 " %02"PRIx8" %02"PRIx8" %02"PRIx8"\n", - (unsigned)port, + port, addr.addr_bytes[0], addr.addr_bytes[1], addr.addr_bytes[2], addr.addr_bytes[3], addr.addr_bytes[4], addr.addr_bytes[5]); @@ -161,40 +202,18 @@ port_init(uint8_t port, struct rte_mempool *mbuf_pool) struct lcore_params { unsigned worker_id; struct rte_distributor *d; - struct rte_ring *r; + struct rte_ring *rx_dist_ring; + struct rte_ring *dist_tx_ring; struct rte_mempool *mem_pool; }; -static int -quit_workers(struct rte_distributor *d, struct rte_mempool *p) -{ - const unsigned num_workers = rte_lcore_count() - 2; - unsigned i; - struct rte_mbuf *bufs[num_workers]; - - if (rte_mempool_get_bulk(p, (void *)bufs, num_workers) != 0) { - printf("line %d: Error getting mbufs from pool\n", __LINE__); - return -1; - } - - for (i = 0; i < num_workers; i++) - bufs[i]->hash.rss = i << 1; - - rte_distributor_process(d, bufs, num_workers); - rte_mempool_put_bulk(p, (void *)bufs, num_workers); - - return 0; -} - static int lcore_rx(struct lcore_params *p) { - struct rte_distributor *d = p->d; - struct rte_mempool *mem_pool = p->mem_pool; - struct rte_ring *r = p->r; - const uint8_t nb_ports = rte_eth_dev_count(); + const uint16_t nb_ports = rte_eth_dev_count(); const int socket_id = rte_socket_id(); - uint8_t port; + uint16_t port; + struct rte_mbuf *bufs[BURST_SIZE*2]; for (port = 0; port < nb_ports; port++) { /* skip ports that are not enabled */ @@ -218,7 +237,6 @@ lcore_rx(struct lcore_params *p) port = 0; continue; } - struct rte_mbuf *bufs[BURST_SIZE*2]; const uint16_t nb_rx = rte_eth_rx_burst(port, 0, bufs, BURST_SIZE); if (unlikely(nb_rx == 0)) { @@ -228,9 +246,15 @@ lcore_rx(struct lcore_params *p) } app_stats.rx.rx_pkts += nb_rx; - rte_distributor_process(d, bufs, nb_rx); - const uint16_t nb_ret = rte_distributor_returned_pkts(d, - bufs, BURST_SIZE*2); +/* + * You can run the distributor on the rx core with this code. Returned + * packets are then send straight to the tx core. + */ +#if 0 + rte_distributor_process(d, bufs, nb_rx); + const uint16_t nb_ret = rte_distributor_returned_pktsd, + bufs, BURST_SIZE*2); + app_stats.rx.returned_pkts += nb_ret; if (unlikely(nb_ret == 0)) { if (++port == nb_ports) @@ -238,10 +262,26 @@ lcore_rx(struct lcore_params *p) continue; } - uint16_t sent = rte_ring_enqueue_burst(r, (void *)bufs, nb_ret); + struct rte_ring *tx_ring = p->dist_tx_ring; + uint16_t sent = rte_ring_enqueue_burst(tx_ring, + (void *)bufs, nb_ret, NULL); +#else + uint16_t nb_ret = nb_rx; + /* + * Swap the following two lines if you want the rx traffic + * to go directly to tx, no distribution. + */ + struct rte_ring *out_ring = p->rx_dist_ring; + /* struct rte_ring *out_ring = p->dist_tx_ring; */ + + uint16_t sent = rte_ring_enqueue_burst(out_ring, + (void *)bufs, nb_ret, NULL); +#endif + app_stats.rx.enqueued_pkts += sent; if (unlikely(sent < nb_ret)) { - RTE_LOG(DEBUG, DISTRAPP, + app_stats.rx.enqdrop_pkts += nb_ret - sent; + RTE_LOG_DP(DEBUG, DISTRAPP, "%s:Packet loss due to full ring\n", __func__); while (sent < nb_ret) rte_pktmbuf_free(bufs[sent++]); @@ -249,33 +289,21 @@ lcore_rx(struct lcore_params *p) if (++port == nb_ports) port = 0; } - rte_distributor_process(d, NULL, 0); - /* flush distributor to bring to known state */ - rte_distributor_flush(d); /* set worker & tx threads quit flag */ + printf("\nCore %u exiting rx task.\n", rte_lcore_id()); quit_signal = 1; - /* - * worker threads may hang in get packet as - * distributor process is not running, just make sure workers - * get packets till quit_signal is actually been - * received and they gracefully shutdown - */ - if (quit_workers(d, mem_pool) != 0) - return -1; - /* rx thread should quit at last */ return 0; } static inline void flush_one_port(struct output_buffer *outbuf, uint8_t outp) { - unsigned nb_tx = rte_eth_tx_burst(outp, 0, outbuf->mbufs, - outbuf->count); - app_stats.tx.tx_pkts += nb_tx; + unsigned int nb_tx = rte_eth_tx_burst(outp, 0, + outbuf->mbufs, outbuf->count); + app_stats.tx.tx_pkts += outbuf->count; if (unlikely(nb_tx < outbuf->count)) { - RTE_LOG(DEBUG, DISTRAPP, - "%s:Packet loss with tx_burst\n", __func__); + app_stats.tx.enqdrop_pkts += outbuf->count - nb_tx; do { rte_pktmbuf_free(outbuf->mbufs[nb_tx]); } while (++nb_tx < outbuf->count); @@ -284,9 +312,10 @@ flush_one_port(struct output_buffer *outbuf, uint8_t outp) } static inline void -flush_all_ports(struct output_buffer *tx_buffers, uint8_t nb_ports) +flush_all_ports(struct output_buffer *tx_buffers, uint16_t nb_ports) { - uint8_t outp; + uint16_t outp; + for (outp = 0; outp < nb_ports; outp++) { /* skip ports that are not enabled */ if ((enabled_port_mask & (1 << outp)) == 0) @@ -299,13 +328,65 @@ flush_all_ports(struct output_buffer *tx_buffers, uint8_t nb_ports) } } + + +static int +lcore_distributor(struct lcore_params *p) +{ + struct rte_ring *in_r = p->rx_dist_ring; + struct rte_ring *out_r = p->dist_tx_ring; + struct rte_mbuf *bufs[BURST_SIZE * 4]; + struct rte_distributor *d = p->d; + + printf("\nCore %u acting as distributor core.\n", rte_lcore_id()); + while (!quit_signal_dist) { + const uint16_t nb_rx = rte_ring_dequeue_burst(in_r, + (void *)bufs, BURST_SIZE*1, NULL); + if (nb_rx) { + app_stats.dist.in_pkts += nb_rx; + + /* Distribute the packets */ + rte_distributor_process(d, bufs, nb_rx); + /* Handle Returns */ + const uint16_t nb_ret = + rte_distributor_returned_pkts(d, + bufs, BURST_SIZE*2); + + if (unlikely(nb_ret == 0)) + continue; + app_stats.dist.ret_pkts += nb_ret; + + uint16_t sent = rte_ring_enqueue_burst(out_r, + (void *)bufs, nb_ret, NULL); + app_stats.dist.sent_pkts += sent; + if (unlikely(sent < nb_ret)) { + app_stats.dist.enqdrop_pkts += nb_ret - sent; + RTE_LOG(DEBUG, DISTRAPP, + "%s:Packet loss due to full out ring\n", + __func__); + while (sent < nb_ret) + rte_pktmbuf_free(bufs[sent++]); + } + } + } + printf("\nCore %u exiting distributor task.\n", rte_lcore_id()); + quit_signal_work = 1; + + rte_distributor_flush(d); + /* Unblock any returns so workers can exit */ + rte_distributor_clear_returns(d); + quit_signal_rx = 1; + return 0; +} + + static int lcore_tx(struct rte_ring *in_r) { static struct output_buffer tx_buffers[RTE_MAX_ETHPORTS]; - const uint8_t nb_ports = rte_eth_dev_count(); + const uint16_t nb_ports = rte_eth_dev_count(); const int socket_id = rte_socket_id(); - uint8_t port; + uint16_t port; for (port = 0; port < nb_ports; port++) { /* skip ports that are not enabled */ @@ -327,9 +408,9 @@ lcore_tx(struct rte_ring *in_r) if ((enabled_port_mask & (1 << port)) == 0) continue; - struct rte_mbuf *bufs[BURST_SIZE]; + struct rte_mbuf *bufs[BURST_SIZE_TX]; const uint16_t nb_rx = rte_ring_dequeue_burst(in_r, - (void *)bufs, BURST_SIZE); + (void *)bufs, BURST_SIZE_TX, NULL); app_stats.tx.dequeue_pkts += nb_rx; /* if we get no traffic, flush anything we have */ @@ -358,11 +439,12 @@ lcore_tx(struct rte_ring *in_r) outbuf = &tx_buffers[outp]; outbuf->mbufs[outbuf->count++] = bufs[i]; - if (outbuf->count == BURST_SIZE) + if (outbuf->count == BURST_SIZE_TX) flush_one_port(outbuf, outp); } } } + printf("\nCore %u exiting tx task.\n", rte_lcore_id()); return 0; } @@ -371,32 +453,98 @@ int_handler(int sig_num) { printf("Exiting on signal %d\n", sig_num); /* set quit flag for rx thread to exit */ - quit_signal_rx = 1; + quit_signal_dist = 1; } static void print_stats(void) { struct rte_eth_stats eth_stats; - unsigned i; - - printf("\nRX thread stats:\n"); - printf(" - Received: %"PRIu64"\n", app_stats.rx.rx_pkts); - printf(" - Processed: %"PRIu64"\n", app_stats.rx.returned_pkts); - printf(" - Enqueued: %"PRIu64"\n", app_stats.rx.enqueued_pkts); - - printf("\nTX thread stats:\n"); - printf(" - Dequeued: %"PRIu64"\n", app_stats.tx.dequeue_pkts); - printf(" - Transmitted: %"PRIu64"\n", app_stats.tx.tx_pkts); + unsigned int i, j; + const unsigned int num_workers = rte_lcore_count() - 4; for (i = 0; i < rte_eth_dev_count(); i++) { rte_eth_stats_get(i, ð_stats); - printf("\nPort %u stats:\n", i); - printf(" - Pkts in: %"PRIu64"\n", eth_stats.ipackets); - printf(" - Pkts out: %"PRIu64"\n", eth_stats.opackets); - printf(" - In Errs: %"PRIu64"\n", eth_stats.ierrors); - printf(" - Out Errs: %"PRIu64"\n", eth_stats.oerrors); - printf(" - Mbuf Errs: %"PRIu64"\n", eth_stats.rx_nombuf); + app_stats.port_rx_pkts[i] = eth_stats.ipackets; + app_stats.port_tx_pkts[i] = eth_stats.opackets; + } + + printf("\n\nRX Thread:\n"); + for (i = 0; i < rte_eth_dev_count(); i++) { + printf("Port %u Pktsin : %5.2f\n", i, + (app_stats.port_rx_pkts[i] - + prev_app_stats.port_rx_pkts[i])/1000000.0); + prev_app_stats.port_rx_pkts[i] = app_stats.port_rx_pkts[i]; + } + printf(" - Received: %5.2f\n", + (app_stats.rx.rx_pkts - + prev_app_stats.rx.rx_pkts)/1000000.0); + printf(" - Returned: %5.2f\n", + (app_stats.rx.returned_pkts - + prev_app_stats.rx.returned_pkts)/1000000.0); + printf(" - Enqueued: %5.2f\n", + (app_stats.rx.enqueued_pkts - + prev_app_stats.rx.enqueued_pkts)/1000000.0); + printf(" - Dropped: %s%5.2f%s\n", ANSI_COLOR_RED, + (app_stats.rx.enqdrop_pkts - + prev_app_stats.rx.enqdrop_pkts)/1000000.0, + ANSI_COLOR_RESET); + + printf("Distributor thread:\n"); + printf(" - In: %5.2f\n", + (app_stats.dist.in_pkts - + prev_app_stats.dist.in_pkts)/1000000.0); + printf(" - Returned: %5.2f\n", + (app_stats.dist.ret_pkts - + prev_app_stats.dist.ret_pkts)/1000000.0); + printf(" - Sent: %5.2f\n", + (app_stats.dist.sent_pkts - + prev_app_stats.dist.sent_pkts)/1000000.0); + printf(" - Dropped %s%5.2f%s\n", ANSI_COLOR_RED, + (app_stats.dist.enqdrop_pkts - + prev_app_stats.dist.enqdrop_pkts)/1000000.0, + ANSI_COLOR_RESET); + + printf("TX thread:\n"); + printf(" - Dequeued: %5.2f\n", + (app_stats.tx.dequeue_pkts - + prev_app_stats.tx.dequeue_pkts)/1000000.0); + for (i = 0; i < rte_eth_dev_count(); i++) { + printf("Port %u Pktsout: %5.2f\n", + i, (app_stats.port_tx_pkts[i] - + prev_app_stats.port_tx_pkts[i])/1000000.0); + prev_app_stats.port_tx_pkts[i] = app_stats.port_tx_pkts[i]; + } + printf(" - Transmitted: %5.2f\n", + (app_stats.tx.tx_pkts - + prev_app_stats.tx.tx_pkts)/1000000.0); + printf(" - Dropped: %s%5.2f%s\n", ANSI_COLOR_RED, + (app_stats.tx.enqdrop_pkts - + prev_app_stats.tx.enqdrop_pkts)/1000000.0, + ANSI_COLOR_RESET); + + prev_app_stats.rx.rx_pkts = app_stats.rx.rx_pkts; + prev_app_stats.rx.returned_pkts = app_stats.rx.returned_pkts; + prev_app_stats.rx.enqueued_pkts = app_stats.rx.enqueued_pkts; + prev_app_stats.rx.enqdrop_pkts = app_stats.rx.enqdrop_pkts; + prev_app_stats.dist.in_pkts = app_stats.dist.in_pkts; + prev_app_stats.dist.ret_pkts = app_stats.dist.ret_pkts; + prev_app_stats.dist.sent_pkts = app_stats.dist.sent_pkts; + prev_app_stats.dist.enqdrop_pkts = app_stats.dist.enqdrop_pkts; + prev_app_stats.tx.dequeue_pkts = app_stats.tx.dequeue_pkts; + prev_app_stats.tx.tx_pkts = app_stats.tx.tx_pkts; + prev_app_stats.tx.enqdrop_pkts = app_stats.tx.enqdrop_pkts; + + for (i = 0; i < num_workers; i++) { + printf("Worker %02u Pkts: %5.2f. Bursts(1-8): ", i, + (app_stats.worker_pkts[i] - + prev_app_stats.worker_pkts[i])/1000000.0); + for (j = 0; j < 8; j++) { + printf("%"PRIu64" ", app_stats.worker_bursts[i][j]); + app_stats.worker_bursts[i][j] = 0; + } + printf("\n"); + prev_app_stats.worker_pkts[i] = app_stats.worker_pkts[i]; } } @@ -405,17 +553,36 @@ lcore_worker(struct lcore_params *p) { struct rte_distributor *d = p->d; const unsigned id = p->worker_id; + unsigned int num = 0; + unsigned int i; + /* * for single port, xor_val will be zero so we won't modify the output * port, otherwise we send traffic from 0 to 1, 2 to 3, and vice versa */ const unsigned xor_val = (rte_eth_dev_count() > 1); - struct rte_mbuf *buf = NULL; + struct rte_mbuf *buf[8] __rte_cache_aligned; + + for (i = 0; i < 8; i++) + buf[i] = NULL; + + app_stats.worker_pkts[p->worker_id] = 1; printf("\nCore %u acting as worker core.\n", rte_lcore_id()); - while (!quit_signal) { - buf = rte_distributor_get_pkt(d, id, buf); - buf->port ^= xor_val; + while (!quit_signal_work) { + num = rte_distributor_get_pkt(d, id, buf, buf, num); + /* Do a little bit of work for each packet */ + for (i = 0; i < num; i++) { + uint64_t t = rte_rdtsc()+100; + + while (rte_rdtsc() < t) + rte_pause(); + buf[i]->port ^= xor_val; + } + + app_stats.worker_pkts[p->worker_id] += num; + if (num > 0) + app_stats.worker_bursts[p->worker_id][num-1]++; } return 0; } @@ -487,7 +654,7 @@ parse_args(int argc, char **argv) argv[optind-1] = prgname; - optind = 0; /* reset getopt lib */ + optind = 1; /* reset getopt lib */ return 0; } @@ -497,11 +664,13 @@ main(int argc, char *argv[]) { struct rte_mempool *mbuf_pool; struct rte_distributor *d; - struct rte_ring *output_ring; + struct rte_ring *dist_tx_ring; + struct rte_ring *rx_dist_ring; unsigned lcore_id, worker_id = 0; unsigned nb_ports; - uint8_t portid; - uint8_t nb_ports_available; + uint16_t portid; + uint16_t nb_ports_available; + uint64_t t, freq; /* catch ctrl-c so we can print on exit */ signal(SIGINT, int_handler); @@ -518,10 +687,12 @@ main(int argc, char *argv[]) if (ret < 0) rte_exit(EXIT_FAILURE, "Invalid distributor parameters\n"); - if (rte_lcore_count() < 3) + if (rte_lcore_count() < 5) rte_exit(EXIT_FAILURE, "Error, This application needs at " - "least 3 logical cores to run:\n" - "1 lcore for packet RX and distribution\n" + "least 5 logical cores to run:\n" + "1 lcore for stats (can be core 0)\n" + "1 lcore for packet RX\n" + "1 lcore for distribution\n" "1 lcore for packet TX\n" "and at least 1 lcore for worker threads\n"); @@ -548,10 +719,10 @@ main(int argc, char *argv[]) continue; } /* init port */ - printf("Initializing port %u... done\n", (unsigned) portid); + printf("Initializing port %u... done\n", portid); if (port_init(portid, mbuf_pool) != 0) - rte_exit(EXIT_FAILURE, "Cannot initialize port %"PRIu8"\n", + rte_exit(EXIT_FAILURE, "Cannot initialize port %u\n", portid); } @@ -561,40 +732,82 @@ main(int argc, char *argv[]) } d = rte_distributor_create("PKT_DIST", rte_socket_id(), - rte_lcore_count() - 2); + rte_lcore_count() - 4, + RTE_DIST_ALG_BURST); if (d == NULL) rte_exit(EXIT_FAILURE, "Cannot create distributor\n"); /* - * scheduler ring is read only by the transmitter core, but written to - * by multiple threads + * scheduler ring is read by the transmitter core, and written to + * by scheduler core */ - output_ring = rte_ring_create("Output_ring", RTE_RING_SZ, - rte_socket_id(), RING_F_SC_DEQ); - if (output_ring == NULL) + dist_tx_ring = rte_ring_create("Output_ring", SCHED_TX_RING_SZ, + rte_socket_id(), RING_F_SC_DEQ | RING_F_SP_ENQ); + if (dist_tx_ring == NULL) + rte_exit(EXIT_FAILURE, "Cannot create output ring\n"); + + rx_dist_ring = rte_ring_create("Input_ring", SCHED_RX_RING_SZ, + rte_socket_id(), RING_F_SC_DEQ | RING_F_SP_ENQ); + if (rx_dist_ring == NULL) rte_exit(EXIT_FAILURE, "Cannot create output ring\n"); RTE_LCORE_FOREACH_SLAVE(lcore_id) { - if (worker_id == rte_lcore_count() - 2) - rte_eal_remote_launch((lcore_function_t *)lcore_tx, - output_ring, lcore_id); - else { + if (worker_id == rte_lcore_count() - 3) { + printf("Starting distributor on lcore_id %d\n", + lcore_id); + /* distributor core */ struct lcore_params *p = rte_malloc(NULL, sizeof(*p), 0); if (!p) rte_panic("malloc failure\n"); - *p = (struct lcore_params){worker_id, d, output_ring, mbuf_pool}; + *p = (struct lcore_params){worker_id, d, + rx_dist_ring, dist_tx_ring, mbuf_pool}; + rte_eal_remote_launch( + (lcore_function_t *)lcore_distributor, + p, lcore_id); + } else if (worker_id == rte_lcore_count() - 4) { + printf("Starting tx on worker_id %d, lcore_id %d\n", + worker_id, lcore_id); + /* tx core */ + rte_eal_remote_launch((lcore_function_t *)lcore_tx, + dist_tx_ring, lcore_id); + } else if (worker_id == rte_lcore_count() - 2) { + printf("Starting rx on worker_id %d, lcore_id %d\n", + worker_id, lcore_id); + /* rx core */ + struct lcore_params *p = + rte_malloc(NULL, sizeof(*p), 0); + if (!p) + rte_panic("malloc failure\n"); + *p = (struct lcore_params){worker_id, d, rx_dist_ring, + dist_tx_ring, mbuf_pool}; + rte_eal_remote_launch((lcore_function_t *)lcore_rx, + p, lcore_id); + } else { + printf("Starting worker on worker_id %d, lcore_id %d\n", + worker_id, lcore_id); + struct lcore_params *p = + rte_malloc(NULL, sizeof(*p), 0); + if (!p) + rte_panic("malloc failure\n"); + *p = (struct lcore_params){worker_id, d, rx_dist_ring, + dist_tx_ring, mbuf_pool}; rte_eal_remote_launch((lcore_function_t *)lcore_worker, p, lcore_id); } worker_id++; } - /* call lcore_main on master core only */ - struct lcore_params p = { 0, d, output_ring, mbuf_pool}; - if (lcore_rx(&p) != 0) - return -1; + freq = rte_get_timer_hz(); + t = rte_rdtsc() + freq; + while (!quit_signal_dist) { + if (t < rte_rdtsc()) { + print_stats(); + t = rte_rdtsc() + freq; + } + usleep(1000); + } RTE_LCORE_FOREACH_SLAVE(lcore_id) { if (rte_eal_wait_lcore(lcore_id) < 0) diff --git a/dpdk/examples/dpdk_qat/Makefile b/dpdk/examples/dpdk_qat/Makefile deleted file mode 100644 index 01d61bcf..00000000 --- a/dpdk/examples/dpdk_qat/Makefile +++ /dev/null @@ -1,93 +0,0 @@ -# BSD LICENSE -# -# Copyright(c) 2010-2013 Intel Corporation. All rights reserved. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in -# the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Intel Corporation nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -ifeq ($(RTE_SDK),) -$(error "Please define RTE_SDK environment variable") -endif - -ifeq ($(ICP_ROOT),) -$(error "Please define ICP_ROOT environment variable") -endif - -# Default target, can be overriden by command line or environment -RTE_TARGET ?= x86_64-native-linuxapp-gcc - -include $(RTE_SDK)/mk/rte.vars.mk - -ifneq ($(CONFIG_RTE_EXEC_ENV),"linuxapp") -$(error This application can only operate in a linuxapp environment, \ -please change the definition of the RTE_TARGET environment variable) -endif - -LBITS := $(shell uname -p) -ifeq ($(CROSS_COMPILE),) - ifneq ($(CONFIG_RTE_ARCH),"x86_64") - ifneq ($(LBITS),i686) - $(error The RTE_TARGET chosen is not compatible with this environment \ - (x86_64), for this application. Please change the definition of the \ - RTE_TARGET environment variable, or run the application on a i686 OS) - endif - endif -endif - -# binary name -APP = dpdk_qat - -# all source are stored in SRCS-y -SRCS-y := main.c crypto.c - -CFLAGS += -O3 -CFLAGS += $(WERROR_FLAGS) -CFLAGS += -I$(ICP_ROOT)/quickassist/include \ - -I$(ICP_ROOT)/quickassist/include/lac \ - -I$(ICP_ROOT)/quickassist/lookaside/access_layer/include - -# From CRF 1.2 driver, library was renamed to libicp_qa_al.a -ifneq ($(wildcard $(ICP_ROOT)/build/icp_qa_al.a),) -ICP_LIBRARY_PATH = $(ICP_ROOT)/build/icp_qa_al.a -else -ICP_LIBRARY_PATH = $(ICP_ROOT)/build/libicp_qa_al.a -endif - -LDLIBS += -L$(ICP_ROOT)/build -LDLIBS += $(ICP_LIBRARY_PATH) \ - -lz \ - -losal \ - -ladf_proxy \ - -lcrypto - -# workaround for a gcc bug with noreturn attribute -# http://gcc.gnu.org/bugzilla/show_bug.cgi?id=12603 -ifeq ($(CONFIG_RTE_TOOLCHAIN_GCC),y) -CFLAGS_main.o += -Wno-return-type -endif - -include $(RTE_SDK)/mk/rte.extapp.mk diff --git a/dpdk/examples/dpdk_qat/config_files/coleto/dh895xcc_qa_dev0.conf b/dpdk/examples/dpdk_qat/config_files/coleto/dh895xcc_qa_dev0.conf deleted file mode 100644 index fd139e2f..00000000 --- a/dpdk/examples/dpdk_qat/config_files/coleto/dh895xcc_qa_dev0.conf +++ /dev/null @@ -1,65 +0,0 @@ -[GENERAL] -ServicesEnabled = cy;dc -ConfigVersion = 2 -cyHmacAuthMode = 1 -dcTotalSRAMAvailable = 0 -Firmware_MofPath = dh895xcc/mof_firmware.bin -Firmware_MmpPath = dh895xcc/mmp_firmware.bin -statsGeneral = 1 -statsDc = 1 -statsDh = 1 -statsDrbg = 1 -statsDsa = 1 -statsEcc = 1 -statsKeyGen = 1 -statsLn = 1 -statsPrime = 1 -statsRsa = 1 -statsSym = 1 -SRIOV_Enabled = 0 -ProcDebug = 1 - -[KERNEL] -NumberCyInstances = 0 -NumberDcInstances = 0 - -[SSL] -NumberCyInstances = 8 -NumberDcInstances = 0 -NumProcesses = 1 -LimitDevAccess = 0 - -Cy0Name = "SSL0" -Cy0IsPolled = 1 -Cy0CoreAffinity = 0 - -Cy1Name = "SSL1" -Cy1IsPolled = 1 -Cy1CoreAffinity = 1 - -Cy2Name = "SSL2" -Cy2IsPolled = 1 -Cy2CoreAffinity = 2 - -Cy3Name = "SSL3" -Cy3IsPolled = 1 -Cy3CoreAffinity = 3 - - -Cy4Name = "SSL4" -Cy4IsPolled = 1 -Cy4CoreAffinity = 4 - - -Cy5Name = "SSL5" -Cy5IsPolled = 1 -Cy5CoreAffinity = 5 - -Cy6Name = "SSL6" -Cy6IsPolled = 1 -Cy6CoreAffinity = 6 - - -Cy7Name = "SSL7" -Cy7IsPolled = 1 -Cy7CoreAffinity = 7 diff --git a/dpdk/examples/dpdk_qat/config_files/shumway/dh89xxcc_qa_dev0.conf b/dpdk/examples/dpdk_qat/config_files/shumway/dh89xxcc_qa_dev0.conf deleted file mode 100644 index 9e1c1d11..00000000 --- a/dpdk/examples/dpdk_qat/config_files/shumway/dh89xxcc_qa_dev0.conf +++ /dev/null @@ -1,293 +0,0 @@ -######################################################################### -# -# @par -# BSD LICENSE -# -# Copyright(c) 2010-2014 Intel Corporation. All rights reserved. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in -# the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Intel Corporation nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# ######################################################################### -# ######################################################## -# -# This file is the configuration for a single dh89xxcc_qa -# device. -# -# Each device has up to two accelerators. -# - The client may load balance between these -# accelerators. -# Each accelerator has 8 independent ring banks. -# - The interrupt for each can be directed to a -# specific core. -# Each ring bank as 16 rings (hardware assisted queues). -# -######################################################### -# General Section -############################################## - -[GENERAL] -ServicesEnabled = cy0;cy1 - -# Use version 2 of the config file -ConfigVersion = 2 -# Look Aside Cryptographic Configuration -cyHmacAuthMode = 1 - -# Look Aside Compression Configuration -dcTotalSRAMAvailable = 0 - -# Firmware Location Configuration -Firmware_MofPath = mof_firmware.bin -Firmware_MmpPath = mmp_firmware.bin - -#Default values for number of concurrent requests*/ -CyNumConcurrentSymRequests = 512 -CyNumConcurrentAsymRequests = 64 -DcNumConcurrentRequests = 512 - -#Statistics, valid values: 1,0 -statsGeneral = 1 -statsDc = 1 -statsDh = 1 -statsDrbg = 1 -statsDsa = 1 -statsEcc = 1 -statsKeyGen = 1 -statsLn = 1 -statsPrime = 1 -statsRsa = 1 -statsSym = 1 - -# Enables or disables Single Root Complex IO Virtualization. -# If this is enabled (1) then SRIOV and VT-d need to be enabled in -# BIOS and there can be no Cy or Dc instances created in PF (Dom0). -# If this i disabled (0) then SRIOV and VT-d need to be disabled -# in BIOS and Cy and/or Dc instances can be used in PF (Dom0) -SRIOV_Enabled = 0 - -#Debug feature, if set to 1 it enables additional entries in /proc filesystem -ProcDebug = 1 - -####################################################### -# -# Logical Instances Section -# A logical instance allows each address domain -# (kernel space and individual user space processes) -# to configure rings (i.e. hardware assisted queues) -# to be used by that address domain and to define the -# behavior of that ring. -# -# The address domains are in the following format -# - For kernel address domains -# [KERNEL] -# - For user process address domains -# [xxxxx] -# Where xxxxx may be any ascii value which uniquely identifies -# the user mode process. -# To allow the driver correctly configure the -# logical instances associated with this user process, -# the process must call the icp_sal_userStartMultiProcess(...) -# passing the xxxxx string during process initialisation. -# When the user space process is finished it must call -# icp_sal_userStop(...) to free resources. -# NumProcesses will indicate the maximum number of processes -# that can call icp_sal_userStartMultiProcess on this instance. -# Warning: the resources are preallocated: if NumProcesses -# is too high, the driver will fail to load -# -# Items configurable by a logical instance are: -# - Name of the logical instance -# - The accelerator associated with this logical -# instance -# - The core the instance is affinitized to (optional) -# -# Note: Logical instances may not share the same ring, but -# may share a ring bank. -# -# The format of the logical instances are: -# - For crypto: -# CyName = "xxxx" -# CyAcceleratorNumber = 0-3 -# CyCoreAffinity = 0-7 -# -# - For Data Compression -# DcName = "xxxx" -# DcAcceleratorNumber = 0-1 -# DcCoreAffinity = 0-7 -# -# Where: -# - n is the number of this logical instance starting at 0. -# - xxxx may be any ascii value which identifies the logical instance. -# -# Note: for user space processes, a list of values can be specified for -# the accelerator number and the core affinity: for example -# Cy0AcceleratorNumber = 0,2 -# Cy0CoreAffinity = 0,2,4 -# These comma-separated lists will allow the multiple processes to use -# different accelerators and cores, and will wrap around the numbers -# in the list. In the above example, process 0 will use accelerator 0, -# and process 1 will use accelerator 2 -# -######################################################## - -############################################## -# Kernel Instances Section -############################################## -[KERNEL] -NumberCyInstances = 0 -NumberDcInstances = 0 - -############################################## -# User Process Instance Section -############################################## -[SSL] -NumberCyInstances = 16 -NumberDcInstances = 0 -NumProcesses = 1 -LimitDevAccess = 0 - -# Crypto - User instance #0 -Cy0Name = "SSL0" -Cy0IsPolled = 1 -Cy0AcceleratorNumber = 0 -# List of core affinities -Cy0CoreAffinity = 0 - -# Crypto - User instance #1 -Cy1Name = "SSL1" -Cy1IsPolled = 1 -Cy1AcceleratorNumber = 1 -# List of core affinities -Cy1CoreAffinity = 1 - -# Crypto - User instance #2 -Cy2Name = "SSL2" -Cy2IsPolled = 1 -Cy2AcceleratorNumber = 2 -# List of core affinities -Cy2CoreAffinity = 2 - -# Crypto - User instance #3 -Cy3Name = "SSL3" -Cy3IsPolled = 1 -Cy3AcceleratorNumber = 3 -# List of core affinities -Cy3CoreAffinity = 3 - -# Crypto - User instance #4 -Cy4Name = "SSL4" -Cy4IsPolled = 1 -Cy4AcceleratorNumber = 0 -# List of core affinities -Cy4CoreAffinity = 4 - -# Crypto - User instance #5 -Cy5Name = "SSL5" -Cy5IsPolled = 1 -Cy5AcceleratorNumber = 1 -# List of core affinities -Cy5CoreAffinity = 5 - -# Crypto - User instance #6 -Cy6Name = "SSL6" -Cy6IsPolled = 1 -Cy6AcceleratorNumber = 2 -# List of core affinities -Cy6CoreAffinity = 6 - -# Crypto - User instance #7 -Cy7Name = "SSL7" -Cy7IsPolled = 1 -Cy7AcceleratorNumber = 3 -# List of core affinities -Cy7CoreAffinity = 7 - -# Crypto - User instance #8 -Cy8Name = "SSL8" -Cy8IsPolled = 1 -Cy8AcceleratorNumber = 0 -# List of core affinities -Cy8CoreAffinity = 16 - -# Crypto - User instance #9 -Cy9Name = "SSL9" -Cy9IsPolled = 1 -Cy9AcceleratorNumber = 1 -# List of core affinities -Cy9CoreAffinity = 17 - -# Crypto - User instance #10 -Cy10Name = "SSL10" -Cy10IsPolled = 1 -Cy10AcceleratorNumber = 2 -# List of core affinities -Cy10CoreAffinity = 18 - -# Crypto - User instance #11 -Cy11Name = "SSL11" -Cy11IsPolled = 1 -Cy11AcceleratorNumber = 3 -# List of core affinities -Cy11CoreAffinity = 19 - -# Crypto - User instance #12 -Cy12Name = "SSL12" -Cy12IsPolled = 1 -Cy12AcceleratorNumber = 0 -# List of core affinities -Cy12CoreAffinity = 20 - -# Crypto - User instance #13 -Cy13Name = "SSL13" -Cy13IsPolled = 1 -Cy13AcceleratorNumber = 1 -# List of core affinities -Cy13CoreAffinity = 21 - -# Crypto - User instance #14 -Cy14Name = "SSL14" -Cy14IsPolled = 1 -Cy14AcceleratorNumber = 2 -# List of core affinities -Cy14CoreAffinity = 22 - -# Crypto - User instance #15 -Cy15Name = "SSL15" -Cy15IsPolled = 1 -Cy15AcceleratorNumber = 3 -# List of core affinities -Cy15CoreAffinity = 23 - - - -############################################## -# Wireless Process Instance Section -############################################## -[WIRELESS] -NumberCyInstances = 0 -NumberDcInstances = 0 -NumProcesses = 0 diff --git a/dpdk/examples/dpdk_qat/config_files/shumway/dh89xxcc_qa_dev1.conf b/dpdk/examples/dpdk_qat/config_files/shumway/dh89xxcc_qa_dev1.conf deleted file mode 100644 index 3e8d8b6b..00000000 --- a/dpdk/examples/dpdk_qat/config_files/shumway/dh89xxcc_qa_dev1.conf +++ /dev/null @@ -1,292 +0,0 @@ -######################################################################### -# -# @par -# BSD LICENSE -# -# Copyright(c) 2010-2014 Intel Corporation. All rights reserved. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in -# the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Intel Corporation nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# ######################################################################### -# ######################################################## -# -# This file is the configuration for a single dh89xxcc_qa -# device. -# -# Each device has up to two accelerators. -# - The client may load balance between these -# accelerators. -# Each accelerator has 8 independent ring banks. -# - The interrupt for each can be directed to a -# specific core. -# Each ring bank as 16 rings (hardware assisted queues). -# -######################################################### -# General Section -############################################## - -[GENERAL] -ServicesEnabled = cy0;cy1 - -# Use version 2 of the config file -ConfigVersion = 2 -# Look Aside Cryptographic Configuration -cyHmacAuthMode = 1 - -# Look Aside Compression Configuration -dcTotalSRAMAvailable = 0 - -# Firmware Location Configuration -Firmware_MofPath = mof_firmware.bin -Firmware_MmpPath = mmp_firmware.bin - -#Default values for number of concurrent requests*/ -CyNumConcurrentSymRequests = 512 -CyNumConcurrentAsymRequests = 64 -DcNumConcurrentRequests = 512 - -#Statistics, valid values: 1,0 -statsGeneral = 1 -statsDc = 1 -statsDh = 1 -statsDrbg = 1 -statsDsa = 1 -statsEcc = 1 -statsKeyGen = 1 -statsLn = 1 -statsPrime = 1 -statsRsa = 1 -statsSym = 1 - -# Enables or disables Single Root Complex IO Virtualization. -# If this is enabled (1) then SRIOV and VT-d need to be enabled in -# BIOS and there can be no Cy or Dc instances created in PF (Dom0). -# If this i disabled (0) then SRIOV and VT-d need to be disabled -# in BIOS and Cy and/or Dc instances can be used in PF (Dom0) -SRIOV_Enabled = 0 - -#Debug feature, if set to 1 it enables additional entries in /proc filesystem -ProcDebug = 1 - -####################################################### -# -# Logical Instances Section -# A logical instance allows each address domain -# (kernel space and individual user space processes) -# to configure rings (i.e. hardware assisted queues) -# to be used by that address domain and to define the -# behavior of that ring. -# -# The address domains are in the following format -# - For kernel address domains -# [KERNEL] -# - For user process address domains -# [xxxxx] -# Where xxxxx may be any ascii value which uniquely identifies -# the user mode process. -# To allow the driver correctly configure the -# logical instances associated with this user process, -# the process must call the icp_sal_userStartMultiProcess(...) -# passing the xxxxx string during process initialisation. -# When the user space process is finished it must call -# icp_sal_userStop(...) to free resources. -# NumProcesses will indicate the maximum number of processes -# that can call icp_sal_userStartMultiProcess on this instance. -# Warning: the resources are preallocated: if NumProcesses -# is too high, the driver will fail to load -# -# Items configurable by a logical instance are: -# - Name of the logical instance -# - The accelerator associated with this logical -# instance -# - The core the instance is affinitized to (optional) -# -# Note: Logical instances may not share the same ring, but -# may share a ring bank. -# -# The format of the logical instances are: -# - For crypto: -# CyName = "xxxx" -# CyAcceleratorNumber = 0-3 -# CyCoreAffinity = 0-7 -# -# - For Data Compression -# DcName = "xxxx" -# DcAcceleratorNumber = 0-1 -# DcCoreAffinity = 0-7 -# -# Where: -# - n is the number of this logical instance starting at 0. -# - xxxx may be any ascii value which identifies the logical instance. -# -# Note: for user space processes, a list of values can be specified for -# the accelerator number and the core affinity: for example -# Cy0AcceleratorNumber = 0,2 -# Cy0CoreAffinity = 0,2,4 -# These comma-separated lists will allow the multiple processes to use -# different accelerators and cores, and will wrap around the numbers -# in the list. In the above example, process 0 will use accelerator 0, -# and process 1 will use accelerator 2 -# -######################################################## - -############################################## -# Kernel Instances Section -############################################## -[KERNEL] -NumberCyInstances = 0 -NumberDcInstances = 0 - -############################################## -# User Process Instance Section -############################################## -[SSL] -NumberCyInstances = 16 -NumberDcInstances = 0 -NumProcesses = 1 -LimitDevAccess = 0 - -# Crypto - User instance #0 -Cy0Name = "SSL0" -Cy0IsPolled = 1 -Cy0AcceleratorNumber = 0 -# List of core affinities -Cy0CoreAffinity = 8 - -# Crypto - User instance #1 -Cy1Name = "SSL1" -Cy1IsPolled = 1 -Cy1AcceleratorNumber = 1 -# List of core affinities -Cy1CoreAffinity = 9 - -# Crypto - User instance #2 -Cy2Name = "SSL2" -Cy2IsPolled = 1 -Cy2AcceleratorNumber = 2 -# List of core affinities -Cy2CoreAffinity = 10 - -# Crypto - User instance #3 -Cy3Name = "SSL3" -Cy3IsPolled = 1 -Cy3AcceleratorNumber = 3 -# List of core affinities -Cy3CoreAffinity = 11 - -# Crypto - User instance #4 -Cy4Name = "SSL4" -Cy4IsPolled = 1 -Cy4AcceleratorNumber = 0 -# List of core affinities -Cy4CoreAffinity = 12 - -# Crypto - User instance #5 -Cy5Name = "SSL5" -Cy5IsPolled = 1 -Cy5AcceleratorNumber = 1 -# List of core affinities -Cy5CoreAffinity = 13 - -# Crypto - User instance #6 -Cy6Name = "SSL6" -Cy6IsPolled = 1 -Cy6AcceleratorNumber = 2 -# List of core affinities -Cy6CoreAffinity = 14 - -# Crypto - User instance #7 -Cy7Name = "SSL7" -Cy7IsPolled = 1 -Cy7AcceleratorNumber = 3 -# List of core affinities -Cy7CoreAffinity = 15 - -# Crypto - User instance #8 -Cy8Name = "SSL8" -Cy8IsPolled = 1 -Cy8AcceleratorNumber = 0 -# List of core affinities -Cy8CoreAffinity = 24 - -# Crypto - User instance #9 -Cy9Name = "SSL9" -Cy9IsPolled = 1 -Cy9AcceleratorNumber = 1 -# List of core affinities -Cy9CoreAffinity = 25 - -# Crypto - User instance #10 -Cy10Name = "SSL10" -Cy10IsPolled = 1 -Cy10AcceleratorNumber = 2 -# List of core affinities -Cy10CoreAffinity = 26 - -# Crypto - User instance #11 -Cy11Name = "SSL11" -Cy11IsPolled = 1 -Cy11AcceleratorNumber = 3 -# List of core affinities -Cy11CoreAffinity = 27 - -# Crypto - User instance #12 -Cy12Name = "SSL12" -Cy12IsPolled = 1 -Cy12AcceleratorNumber = 0 -# List of core affinities -Cy12CoreAffinity = 28 - -# Crypto - User instance #13 -Cy13Name = "SSL13" -Cy13IsPolled = 1 -Cy13AcceleratorNumber = 1 -# List of core affinities -Cy13CoreAffinity = 29 - -# Crypto - User instance #14 -Cy14Name = "SSL14" -Cy14IsPolled = 1 -Cy14AcceleratorNumber = 2 -# List of core affinities -Cy14CoreAffinity = 30 - -# Crypto - User instance #15 -Cy15Name = "SSL15" -Cy15IsPolled = 1 -Cy15AcceleratorNumber = 3 -# List of core affinities -Cy15CoreAffinity = 31 - - -############################################## -# Wireless Process Instance Section -############################################## -[WIRELESS] -NumberCyInstances = 0 -NumberDcInstances = 0 -NumProcesses = 0 diff --git a/dpdk/examples/dpdk_qat/config_files/stargo/dh89xxcc_qa_dev0.conf b/dpdk/examples/dpdk_qat/config_files/stargo/dh89xxcc_qa_dev0.conf deleted file mode 100644 index c3a85dea..00000000 --- a/dpdk/examples/dpdk_qat/config_files/stargo/dh89xxcc_qa_dev0.conf +++ /dev/null @@ -1,235 +0,0 @@ -######################################################################### -# -# @par -# BSD LICENSE -# -# Copyright(c) 2010-2014 Intel Corporation. All rights reserved. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in -# the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Intel Corporation nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# ######################################################################### -# ######################################################## -# -# This file is the configuration for a single dh89xxcc_qa -# device. -# -# Each device has up to two accelerators. -# - The client may load balance between these -# accelerators. -# Each accelerator has 8 independent ring banks. -# - The interrupt for each can be directed to a -# specific core. -# Each ring bank as 16 rings (hardware assisted queues). -# -######################################################### -# General Section -############################################## - -[GENERAL] -ServicesEnabled = cy0;cy1 - -# Use version 2 of the config file -ConfigVersion = 2 -# Look Aside Cryptographic Configuration -cyHmacAuthMode = 1 - -# Look Aside Compression Configuration -dcTotalSRAMAvailable = 0 - -# Firmware Location Configuration -Firmware_MofPath = mof_firmware.bin -Firmware_MmpPath = mmp_firmware.bin - -#Default values for number of concurrent requests*/ -CyNumConcurrentSymRequests = 512 -CyNumConcurrentAsymRequests = 64 -DcNumConcurrentRequests = 512 - -#Statistics, valid values: 1,0 -statsGeneral = 1 -statsDc = 1 -statsDh = 1 -statsDrbg = 1 -statsDsa = 1 -statsEcc = 1 -statsKeyGen = 1 -statsLn = 1 -statsPrime = 1 -statsRsa = 1 -statsSym = 1 - -# Enables or disables Single Root Complex IO Virtualization. -# If this is enabled (1) then SRIOV and VT-d need to be enabled in -# BIOS and there can be no Cy or Dc instances created in PF (Dom0). -# If this i disabled (0) then SRIOV and VT-d need to be disabled -# in BIOS and Cy and/or Dc instances can be used in PF (Dom0) -SRIOV_Enabled = 0 - -#Debug feature, if set to 1 it enables additional entries in /proc filesystem -ProcDebug = 1 - -####################################################### -# -# Logical Instances Section -# A logical instance allows each address domain -# (kernel space and individual user space processes) -# to configure rings (i.e. hardware assisted queues) -# to be used by that address domain and to define the -# behavior of that ring. -# -# The address domains are in the following format -# - For kernel address domains -# [KERNEL] -# - For user process address domains -# [xxxxx] -# Where xxxxx may be any ascii value which uniquely identifies -# the user mode process. -# To allow the driver correctly configure the -# logical instances associated with this user process, -# the process must call the icp_sal_userStartMultiProcess(...) -# passing the xxxxx string during process initialisation. -# When the user space process is finished it must call -# icp_sal_userStop(...) to free resources. -# NumProcesses will indicate the maximum number of processes -# that can call icp_sal_userStartMultiProcess on this instance. -# Warning: the resources are preallocated: if NumProcesses -# is too high, the driver will fail to load -# -# Items configurable by a logical instance are: -# - Name of the logical instance -# - The accelerator associated with this logical -# instance -# - The core the instance is affinitized to (optional) -# -# Note: Logical instances may not share the same ring, but -# may share a ring bank. -# -# The format of the logical instances are: -# - For crypto: -# CyName = "xxxx" -# CyAcceleratorNumber = 0-3 -# CyCoreAffinity = 0-7 -# -# - For Data Compression -# DcName = "xxxx" -# DcAcceleratorNumber = 0-1 -# DcCoreAffinity = 0-7 -# -# Where: -# - n is the number of this logical instance starting at 0. -# - xxxx may be any ascii value which identifies the logical instance. -# -# Note: for user space processes, a list of values can be specified for -# the accelerator number and the core affinity: for example -# Cy0AcceleratorNumber = 0,2 -# Cy0CoreAffinity = 0,2,4 -# These comma-separated lists will allow the multiple processes to use -# different accelerators and cores, and will wrap around the numbers -# in the list. In the above example, process 0 will use accelerator 0, -# and process 1 will use accelerator 2 -# -######################################################## - -############################################## -# Kernel Instances Section -############################################## -[KERNEL] -NumberCyInstances = 0 -NumberDcInstances = 0 - -############################################## -# User Process Instance Section -############################################## -[SSL] -NumberCyInstances = 8 -NumberDcInstances = 0 -NumProcesses = 1 -LimitDevAccess = 0 - -# Crypto - User instance #0 -Cy0Name = "SSL0" -Cy0IsPolled = 1 -Cy0AcceleratorNumber = 0 -# List of core affinities -Cy0CoreAffinity = 0 - -# Crypto - User instance #1 -Cy1Name = "SSL1" -Cy1IsPolled = 1 -Cy1AcceleratorNumber = 1 -# List of core affinities -Cy1CoreAffinity = 1 - -# Crypto - User instance #2 -Cy2Name = "SSL2" -Cy2IsPolled = 1 -Cy2AcceleratorNumber = 2 -# List of core affinities -Cy2CoreAffinity = 2 - -# Crypto - User instance #3 -Cy3Name = "SSL3" -Cy3IsPolled = 1 -Cy3AcceleratorNumber = 3 -# List of core affinities -Cy3CoreAffinity = 3 - -# Crypto - User instance #4 -Cy4Name = "SSL4" -Cy4IsPolled = 1 -Cy4AcceleratorNumber = 0 -# List of core affinities -Cy4CoreAffinity = 4 - -# Crypto - User instance #5 -Cy5Name = "SSL5" -Cy5IsPolled = 1 -Cy5AcceleratorNumber = 1 -# List of core affinities -Cy5CoreAffinity = 5 - -# Crypto - User instance #6 -Cy6Name = "SSL6" -Cy6IsPolled = 1 -Cy6AcceleratorNumber = 2 -# List of core affinities -Cy6CoreAffinity = 6 - -# Crypto - User instance #7 -Cy7Name = "SSL7" -Cy7IsPolled = 1 -Cy7AcceleratorNumber = 3 -# List of core affinities -Cy7CoreAffinity = 7 - -############################################## -# Wireless Process Instance Section -############################################## -[WIRELESS] -NumberCyInstances = 0 -NumberDcInstances = 0 -NumProcesses = 0 diff --git a/dpdk/examples/dpdk_qat/crypto.c b/dpdk/examples/dpdk_qat/crypto.c deleted file mode 100644 index 8954bf87..00000000 --- a/dpdk/examples/dpdk_qat/crypto.c +++ /dev/null @@ -1,944 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define CPA_CY_SYM_DP_TMP_WORKAROUND 1 - -#include "cpa.h" -#include "cpa_types.h" -#include "cpa_cy_sym_dp.h" -#include "cpa_cy_common.h" -#include "cpa_cy_im.h" -#include "icp_sal_user.h" -#include "icp_sal_poll.h" - -#include "crypto.h" - -/* CIPHER KEY LENGTHS */ -#define KEY_SIZE_64_IN_BYTES (64 / 8) -#define KEY_SIZE_56_IN_BYTES (56 / 8) -#define KEY_SIZE_128_IN_BYTES (128 / 8) -#define KEY_SIZE_168_IN_BYTES (168 / 8) -#define KEY_SIZE_192_IN_BYTES (192 / 8) -#define KEY_SIZE_256_IN_BYTES (256 / 8) - -/* HMAC AUTH KEY LENGTHS */ -#define AES_XCBC_AUTH_KEY_LENGTH_IN_BYTES (128 / 8) -#define SHA1_AUTH_KEY_LENGTH_IN_BYTES (160 / 8) -#define SHA224_AUTH_KEY_LENGTH_IN_BYTES (224 / 8) -#define SHA256_AUTH_KEY_LENGTH_IN_BYTES (256 / 8) -#define SHA384_AUTH_KEY_LENGTH_IN_BYTES (384 / 8) -#define SHA512_AUTH_KEY_LENGTH_IN_BYTES (512 / 8) -#define MD5_AUTH_KEY_LENGTH_IN_BYTES (128 / 8) -#define KASUMI_AUTH_KEY_LENGTH_IN_BYTES (128 / 8) - -/* HASH DIGEST LENGHTS */ -#define AES_XCBC_DIGEST_LENGTH_IN_BYTES (128 / 8) -#define AES_XCBC_96_DIGEST_LENGTH_IN_BYTES (96 / 8) -#define MD5_DIGEST_LENGTH_IN_BYTES (128 / 8) -#define SHA1_DIGEST_LENGTH_IN_BYTES (160 / 8) -#define SHA1_96_DIGEST_LENGTH_IN_BYTES (96 / 8) -#define SHA224_DIGEST_LENGTH_IN_BYTES (224 / 8) -#define SHA256_DIGEST_LENGTH_IN_BYTES (256 / 8) -#define SHA384_DIGEST_LENGTH_IN_BYTES (384 / 8) -#define SHA512_DIGEST_LENGTH_IN_BYTES (512 / 8) -#define KASUMI_DIGEST_LENGTH_IN_BYTES (32 / 8) - -#define IV_LENGTH_16_BYTES (16) -#define IV_LENGTH_8_BYTES (8) - - -/* - * rte_memzone is used to allocate physically contiguous virtual memory. - * In this application we allocate a single block and divide between variables - * which require a virtual to physical mapping for use by the QAT driver. - * Virt2phys is only performed during initialisation and not on the data-path. - */ - -#define LCORE_MEMZONE_SIZE (1 << 22) - -struct lcore_memzone -{ - const struct rte_memzone *memzone; - void *next_free_address; -}; - -/* - * Size the qa software response queue. - * Note: Head and Tail are 8 bit, therefore, the queue is - * fixed to 256 entries. - */ -#define CRYPTO_SOFTWARE_QUEUE_SIZE 256 - -struct qa_callbackQueue { - uint8_t head; - uint8_t tail; - uint16_t numEntries; - struct rte_mbuf *qaCallbackRing[CRYPTO_SOFTWARE_QUEUE_SIZE]; -}; - -struct qa_core_conf { - CpaCySymDpSessionCtx *encryptSessionHandleTbl[NUM_CRYPTO][NUM_HMAC]; - CpaCySymDpSessionCtx *decryptSessionHandleTbl[NUM_CRYPTO][NUM_HMAC]; - CpaInstanceHandle instanceHandle; - struct qa_callbackQueue callbackQueue; - uint64_t qaOutstandingRequests; - uint64_t numResponseAttempts; - uint8_t kickFreq; - void *pPacketIV; - CpaPhysicalAddr packetIVPhy; - struct lcore_memzone lcoreMemzone; -} __rte_cache_aligned; - -#define MAX_CORES (RTE_MAX_LCORE) - -static struct qa_core_conf qaCoreConf[MAX_CORES]; - -/* - *Create maximum possible key size, - *One for cipher and one for hash - */ -struct glob_keys { - uint8_t cipher_key[32]; - uint8_t hash_key[64]; - uint8_t iv[16]; -}; - -struct glob_keys g_crypto_hash_keys = { - .cipher_key = {0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08, - 0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10, - 0x11,0x12,0x13,0x14,0x15,0x16,0x17,0x18, - 0x19,0x1a,0x1b,0x1c,0x1d,0x1e,0x1f,0x20}, - .hash_key = {0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08, - 0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10, - 0x11,0x12,0x13,0x14,0x15,0x16,0x17,0x18, - 0x19,0x1a,0x1b,0x1c,0x1d,0x1e,0x1f,0x20, - 0x21,0x22,0x23,0x24,0x25,0x26,0x27,0x28, - 0x29,0x2a,0x2b,0x2c,0x2d,0x2e,0x2f,0x30, - 0x31,0x32,0x33,0x34,0x35,0x36,0x37,0x38, - 0x39,0x4a,0x4b,0x4c,0x4d,0x4e,0x4f,0x50}, - .iv = {0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08, - 0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10} -}; - -/* - * Offsets from the start of the packet. - * - */ -#define PACKET_DATA_START_PHYS(p) \ - ((p)->buf_physaddr + (p)->data_off) - -/* - * A fixed offset to where the crypto is to be performed, which is the first - * byte after the Ethernet(14 bytes) and IPv4 headers(20 bytes) - */ -#define CRYPTO_START_OFFSET (14+20) -#define HASH_START_OFFSET (14+20) -#define CIPHER_BLOCK_DEFAULT_SIZE (16) -#define HASH_BLOCK_DEFAULT_SIZE (16) - -/* - * Offset to the opdata from the start of the data portion of packet. - * Assumption: The buffer is physically contiguous. - * +18 takes this to the next cache line. - */ - -#define CRYPTO_OFFSET_TO_OPDATA (ETHER_MAX_LEN+18) - -/* - * Default number of requests to place on the hardware ring before kicking the - * ring pointers. - */ -#define CRYPTO_BURST_TX (16) - -/* - * Only call the qa poll function when the number responses in the software - * queue drops below this number. - */ -#define CRYPTO_QUEUED_RESP_POLL_THRESHOLD (32) - -/* - * Limit the number of polls per call to get_next_response. - */ -#define GET_NEXT_RESPONSE_FREQ (32) - -/* - * Max number of responses to pull from the qa in one poll. - */ -#define CRYPTO_MAX_RESPONSE_QUOTA \ - (CRYPTO_SOFTWARE_QUEUE_SIZE-CRYPTO_QUEUED_RESP_POLL_THRESHOLD-1) - -#if (CRYPTO_QUEUED_RESP_POLL_THRESHOLD + CRYPTO_MAX_RESPONSE_QUOTA >= \ - CRYPTO_SOFTWARE_QUEUE_SIZE) -#error Its possible to overflow the qa response Q with current poll and \ - response quota. -#endif - -static void -crypto_callback(CpaCySymDpOpData *pOpData, - __rte_unused CpaStatus status, - __rte_unused CpaBoolean verifyResult) -{ - uint32_t lcore_id; - lcore_id = rte_lcore_id(); - struct qa_callbackQueue *callbackQ = &(qaCoreConf[lcore_id].callbackQueue); - - /* - * Received a completion from the QA hardware. - * Place the response on the return queue. - */ - callbackQ->qaCallbackRing[callbackQ->head] = pOpData->pCallbackTag; - callbackQ->head++; - callbackQ->numEntries++; - qaCoreConf[lcore_id].qaOutstandingRequests--; -} - -static void -qa_crypto_callback(CpaCySymDpOpData *pOpData, CpaStatus status, - CpaBoolean verifyResult) -{ - crypto_callback(pOpData, status, verifyResult); -} - -/* - * Each allocation from a particular memzone lasts for the life-time of - * the application. No freeing of previous allocations will occur. - */ -static void * -alloc_memzone_region(uint32_t length, uint32_t lcore_id) -{ - char *current_free_addr_ptr = NULL; - struct lcore_memzone *lcore_memzone = &(qaCoreConf[lcore_id].lcoreMemzone); - - current_free_addr_ptr = lcore_memzone->next_free_address; - - if (current_free_addr_ptr + length >= - (char *)lcore_memzone->memzone->addr + lcore_memzone->memzone->len) { - printf("Crypto: No memory available in memzone\n"); - return NULL; - } - lcore_memzone->next_free_address = current_free_addr_ptr + length; - - return (void *)current_free_addr_ptr; -} - -/* - * Virtual to Physical Address translation is only executed during initialization - * and not on the data-path. - */ -static CpaPhysicalAddr -qa_v2p(void *ptr) -{ - const struct rte_memzone *memzone = NULL; - uint32_t lcore_id = 0; - RTE_LCORE_FOREACH(lcore_id) { - memzone = qaCoreConf[lcore_id].lcoreMemzone.memzone; - - if ((char*) ptr >= (char *) memzone->addr && - (char*) ptr < ((char*) memzone->addr + memzone->len)) { - return (CpaPhysicalAddr) - (memzone->phys_addr + ((char *) ptr - (char*) memzone->addr)); - } - } - printf("Crypto: Corresponding physical address not found in memzone\n"); - return (CpaPhysicalAddr) 0; -} - -static CpaStatus -getCoreAffinity(Cpa32U *coreAffinity, const CpaInstanceHandle instanceHandle) -{ - CpaInstanceInfo2 info; - Cpa16U i = 0; - CpaStatus status = CPA_STATUS_SUCCESS; - - memset(&info, 0, sizeof(CpaInstanceInfo2)); - - status = cpaCyInstanceGetInfo2(instanceHandle, &info); - if (CPA_STATUS_SUCCESS != status) { - printf("Crypto: Error getting instance info\n"); - return CPA_STATUS_FAIL; - } - for (i = 0; i < MAX_CORES; i++) { - if (CPA_BITMAP_BIT_TEST(info.coreAffinity, i)) { - *coreAffinity = i; - return CPA_STATUS_SUCCESS; - } - } - return CPA_STATUS_FAIL; -} - -static CpaStatus -get_crypto_instance_on_core(CpaInstanceHandle *pInstanceHandle, - uint32_t lcore_id) -{ - Cpa16U numInstances = 0, i = 0; - CpaStatus status = CPA_STATUS_FAIL; - CpaInstanceHandle *pLocalInstanceHandles = NULL; - Cpa32U coreAffinity = 0; - - status = cpaCyGetNumInstances(&numInstances); - if (CPA_STATUS_SUCCESS != status || numInstances == 0) { - return CPA_STATUS_FAIL; - } - - pLocalInstanceHandles = rte_malloc("pLocalInstanceHandles", - sizeof(CpaInstanceHandle) * numInstances, RTE_CACHE_LINE_SIZE); - - if (NULL == pLocalInstanceHandles) { - return CPA_STATUS_FAIL; - } - status = cpaCyGetInstances(numInstances, pLocalInstanceHandles); - if (CPA_STATUS_SUCCESS != status) { - printf("Crypto: cpaCyGetInstances failed with status: %"PRId32"\n", status); - rte_free((void *) pLocalInstanceHandles); - return CPA_STATUS_FAIL; - } - - for (i = 0; i < numInstances; i++) { - status = getCoreAffinity(&coreAffinity, pLocalInstanceHandles[i]); - if (CPA_STATUS_SUCCESS != status) { - rte_free((void *) pLocalInstanceHandles); - return CPA_STATUS_FAIL; - } - if (coreAffinity == lcore_id) { - printf("Crypto: instance found on core %d\n", i); - *pInstanceHandle = pLocalInstanceHandles[i]; - return CPA_STATUS_SUCCESS; - } - } - /* core affinity not found */ - rte_free((void *) pLocalInstanceHandles); - return CPA_STATUS_FAIL; -} - -static CpaStatus -initCySymSession(const int pkt_cipher_alg, - const int pkt_hash_alg, const CpaCySymHashMode hashMode, - const CpaCySymCipherDirection crypto_direction, - CpaCySymSessionCtx **ppSessionCtx, - const CpaInstanceHandle cyInstanceHandle, - const uint32_t lcore_id) -{ - Cpa32U sessionCtxSizeInBytes = 0; - CpaStatus status = CPA_STATUS_FAIL; - CpaBoolean isCrypto = CPA_TRUE, isHmac = CPA_TRUE; - CpaCySymSessionSetupData sessionSetupData; - - memset(&sessionSetupData, 0, sizeof(CpaCySymSessionSetupData)); - - /* Assumption: key length is set to each algorithm's max length */ - switch (pkt_cipher_alg) { - case NO_CIPHER: - isCrypto = CPA_FALSE; - break; - case CIPHER_DES: - sessionSetupData.cipherSetupData.cipherAlgorithm = - CPA_CY_SYM_CIPHER_DES_ECB; - sessionSetupData.cipherSetupData.cipherKeyLenInBytes = - KEY_SIZE_64_IN_BYTES; - break; - case CIPHER_DES_CBC: - sessionSetupData.cipherSetupData.cipherAlgorithm = - CPA_CY_SYM_CIPHER_DES_CBC; - sessionSetupData.cipherSetupData.cipherKeyLenInBytes = - KEY_SIZE_64_IN_BYTES; - break; - case CIPHER_DES3: - sessionSetupData.cipherSetupData.cipherAlgorithm = - CPA_CY_SYM_CIPHER_3DES_ECB; - sessionSetupData.cipherSetupData.cipherKeyLenInBytes = - KEY_SIZE_192_IN_BYTES; - break; - case CIPHER_DES3_CBC: - sessionSetupData.cipherSetupData.cipherAlgorithm = - CPA_CY_SYM_CIPHER_3DES_CBC; - sessionSetupData.cipherSetupData.cipherKeyLenInBytes = - KEY_SIZE_192_IN_BYTES; - break; - case CIPHER_AES: - sessionSetupData.cipherSetupData.cipherAlgorithm = - CPA_CY_SYM_CIPHER_AES_ECB; - sessionSetupData.cipherSetupData.cipherKeyLenInBytes = - KEY_SIZE_128_IN_BYTES; - break; - case CIPHER_AES_CBC_128: - sessionSetupData.cipherSetupData.cipherAlgorithm = - CPA_CY_SYM_CIPHER_AES_CBC; - sessionSetupData.cipherSetupData.cipherKeyLenInBytes = - KEY_SIZE_128_IN_BYTES; - break; - case CIPHER_KASUMI_F8: - sessionSetupData.cipherSetupData.cipherAlgorithm = - CPA_CY_SYM_CIPHER_KASUMI_F8; - sessionSetupData.cipherSetupData.cipherKeyLenInBytes = - KEY_SIZE_128_IN_BYTES; - break; - default: - printf("Crypto: Undefined Cipher specified\n"); - break; - } - /* Set the cipher direction */ - if (isCrypto) { - sessionSetupData.cipherSetupData.cipherDirection = crypto_direction; - sessionSetupData.cipherSetupData.pCipherKey = - g_crypto_hash_keys.cipher_key; - sessionSetupData.symOperation = CPA_CY_SYM_OP_CIPHER; - } - - /* Setup Hash common fields */ - switch (pkt_hash_alg) { - case NO_HASH: - isHmac = CPA_FALSE; - break; - case HASH_AES_XCBC: - sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_AES_XCBC; - sessionSetupData.hashSetupData.digestResultLenInBytes = - AES_XCBC_DIGEST_LENGTH_IN_BYTES; - break; - case HASH_AES_XCBC_96: - sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_AES_XCBC; - sessionSetupData.hashSetupData.digestResultLenInBytes = - AES_XCBC_96_DIGEST_LENGTH_IN_BYTES; - break; - case HASH_MD5: - sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_MD5; - sessionSetupData.hashSetupData.digestResultLenInBytes = - MD5_DIGEST_LENGTH_IN_BYTES; - break; - case HASH_SHA1: - sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_SHA1; - sessionSetupData.hashSetupData.digestResultLenInBytes = - SHA1_DIGEST_LENGTH_IN_BYTES; - break; - case HASH_SHA1_96: - sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_SHA1; - sessionSetupData.hashSetupData.digestResultLenInBytes = - SHA1_96_DIGEST_LENGTH_IN_BYTES; - break; - case HASH_SHA224: - sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_SHA224; - sessionSetupData.hashSetupData.digestResultLenInBytes = - SHA224_DIGEST_LENGTH_IN_BYTES; - break; - case HASH_SHA256: - sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_SHA256; - sessionSetupData.hashSetupData.digestResultLenInBytes = - SHA256_DIGEST_LENGTH_IN_BYTES; - break; - case HASH_SHA384: - sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_SHA384; - sessionSetupData.hashSetupData.digestResultLenInBytes = - SHA384_DIGEST_LENGTH_IN_BYTES; - break; - case HASH_SHA512: - sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_SHA512; - sessionSetupData.hashSetupData.digestResultLenInBytes = - SHA512_DIGEST_LENGTH_IN_BYTES; - break; - case HASH_KASUMI_F9: - sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_KASUMI_F9; - sessionSetupData.hashSetupData.digestResultLenInBytes = - KASUMI_DIGEST_LENGTH_IN_BYTES; - break; - default: - printf("Crypto: Undefined Hash specified\n"); - break; - } - if (isHmac) { - sessionSetupData.hashSetupData.hashMode = hashMode; - sessionSetupData.symOperation = CPA_CY_SYM_OP_HASH; - /* If using authenticated hash setup key lengths */ - if (CPA_CY_SYM_HASH_MODE_AUTH == hashMode) { - /* Use a common max length key */ - sessionSetupData.hashSetupData.authModeSetupData.authKey = - g_crypto_hash_keys.hash_key; - switch (pkt_hash_alg) { - case HASH_AES_XCBC: - case HASH_AES_XCBC_96: - sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes = - AES_XCBC_AUTH_KEY_LENGTH_IN_BYTES; - break; - case HASH_MD5: - sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes = - SHA1_AUTH_KEY_LENGTH_IN_BYTES; - break; - case HASH_SHA1: - case HASH_SHA1_96: - sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes = - SHA1_AUTH_KEY_LENGTH_IN_BYTES; - break; - case HASH_SHA224: - sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes = - SHA224_AUTH_KEY_LENGTH_IN_BYTES; - break; - case HASH_SHA256: - sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes = - SHA256_AUTH_KEY_LENGTH_IN_BYTES; - break; - case HASH_SHA384: - sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes = - SHA384_AUTH_KEY_LENGTH_IN_BYTES; - break; - case HASH_SHA512: - sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes = - SHA512_AUTH_KEY_LENGTH_IN_BYTES; - break; - case HASH_KASUMI_F9: - sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes = - KASUMI_AUTH_KEY_LENGTH_IN_BYTES; - break; - default: - printf("Crypto: Undefined Hash specified\n"); - return CPA_STATUS_FAIL; - } - } - } - - /* Only high priority supported */ - sessionSetupData.sessionPriority = CPA_CY_PRIORITY_HIGH; - - /* If chaining algorithms */ - if (isCrypto && isHmac) { - sessionSetupData.symOperation = CPA_CY_SYM_OP_ALGORITHM_CHAINING; - /* @assumption Alg Chain order is cipher then hash for encrypt - * and hash then cipher then has for decrypt*/ - if (CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT == crypto_direction) { - sessionSetupData.algChainOrder = - CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH; - } else { - sessionSetupData.algChainOrder = - CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER; - } - } - if (!isCrypto && !isHmac) { - *ppSessionCtx = NULL; - return CPA_STATUS_SUCCESS; - } - - /* Set flags for digest operations */ - sessionSetupData.digestIsAppended = CPA_FALSE; - sessionSetupData.verifyDigest = CPA_TRUE; - - /* Get the session context size based on the crypto and/or hash operations*/ - status = cpaCySymDpSessionCtxGetSize(cyInstanceHandle, &sessionSetupData, - &sessionCtxSizeInBytes); - if (CPA_STATUS_SUCCESS != status) { - printf("Crypto: cpaCySymDpSessionCtxGetSize error, status: %"PRId32"\n", - status); - return CPA_STATUS_FAIL; - } - - *ppSessionCtx = alloc_memzone_region(sessionCtxSizeInBytes, lcore_id); - if (NULL == *ppSessionCtx) { - printf("Crypto: Failed to allocate memory for Session Context\n"); - return CPA_STATUS_FAIL; - } - - status = cpaCySymDpInitSession(cyInstanceHandle, &sessionSetupData, - *ppSessionCtx); - if (CPA_STATUS_SUCCESS != status) { - printf("Crypto: cpaCySymDpInitSession failed with status %"PRId32"\n", status); - return CPA_STATUS_FAIL; - } - return CPA_STATUS_SUCCESS; -} - -static CpaStatus -initSessionDataTables(struct qa_core_conf *qaCoreConf,uint32_t lcore_id) -{ - Cpa32U i = 0, j = 0; - CpaStatus status = CPA_STATUS_FAIL; - for (i = 0; i < NUM_CRYPTO; i++) { - for (j = 0; j < NUM_HMAC; j++) { - if (((i == CIPHER_KASUMI_F8) && (j != NO_HASH) && (j != HASH_KASUMI_F9)) || - ((i != NO_CIPHER) && (i != CIPHER_KASUMI_F8) && (j == HASH_KASUMI_F9))) - continue; - status = initCySymSession(i, j, CPA_CY_SYM_HASH_MODE_AUTH, - CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT, - &qaCoreConf->encryptSessionHandleTbl[i][j], - qaCoreConf->instanceHandle, - lcore_id); - if (CPA_STATUS_SUCCESS != status) { - printf("Crypto: Failed to initialize Encrypt sessions\n"); - return CPA_STATUS_FAIL; - } - status = initCySymSession(i, j, CPA_CY_SYM_HASH_MODE_AUTH, - CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT, - &qaCoreConf->decryptSessionHandleTbl[i][j], - qaCoreConf->instanceHandle, - lcore_id); - if (CPA_STATUS_SUCCESS != status) { - printf("Crypto: Failed to initialize Decrypt sessions\n"); - return CPA_STATUS_FAIL; - } - } - } - return CPA_STATUS_SUCCESS; -} - -int -crypto_init(void) -{ - if (CPA_STATUS_SUCCESS != icp_sal_userStartMultiProcess("SSL",CPA_FALSE)) { - printf("Crypto: Could not start sal for user space\n"); - return CPA_STATUS_FAIL; - } - printf("Crypto: icp_sal_userStartMultiProcess(\"SSL\",CPA_FALSE)\n"); - return 0; -} - -/* - * Per core initialisation - */ -int -per_core_crypto_init(uint32_t lcore_id) -{ - CpaStatus status = CPA_STATUS_FAIL; - char memzone_name[RTE_MEMZONE_NAMESIZE]; - - int socketID = rte_lcore_to_socket_id(lcore_id); - - /* Allocate software ring for response messages. */ - - qaCoreConf[lcore_id].callbackQueue.head = 0; - qaCoreConf[lcore_id].callbackQueue.tail = 0; - qaCoreConf[lcore_id].callbackQueue.numEntries = 0; - qaCoreConf[lcore_id].kickFreq = 0; - qaCoreConf[lcore_id].qaOutstandingRequests = 0; - qaCoreConf[lcore_id].numResponseAttempts = 0; - - /* Initialise and reserve lcore memzone for virt2phys translation */ - snprintf(memzone_name, - RTE_MEMZONE_NAMESIZE, - "lcore_%u", - lcore_id); - - qaCoreConf[lcore_id].lcoreMemzone.memzone = rte_memzone_reserve( - memzone_name, - LCORE_MEMZONE_SIZE, - socketID, - 0); - if (NULL == qaCoreConf[lcore_id].lcoreMemzone.memzone) { - printf("Crypto: Error allocating memzone on lcore %u\n",lcore_id); - return -1; - } - qaCoreConf[lcore_id].lcoreMemzone.next_free_address = - qaCoreConf[lcore_id].lcoreMemzone.memzone->addr; - - qaCoreConf[lcore_id].pPacketIV = alloc_memzone_region(IV_LENGTH_16_BYTES, - lcore_id); - - if (NULL == qaCoreConf[lcore_id].pPacketIV ) { - printf("Crypto: Failed to allocate memory for Initialization Vector\n"); - return -1; - } - - memcpy(qaCoreConf[lcore_id].pPacketIV, &g_crypto_hash_keys.iv, - IV_LENGTH_16_BYTES); - - qaCoreConf[lcore_id].packetIVPhy = qa_v2p(qaCoreConf[lcore_id].pPacketIV); - if (0 == qaCoreConf[lcore_id].packetIVPhy) { - printf("Crypto: Invalid physical address for Initialization Vector\n"); - return -1; - } - - /* - * Obtain the instance handle that is mapped to the current lcore. - * This can fail if an instance is not mapped to a bank which has been - * affinitized to the current lcore. - */ - status = get_crypto_instance_on_core(&(qaCoreConf[lcore_id].instanceHandle), - lcore_id); - if (CPA_STATUS_SUCCESS != status) { - printf("Crypto: get_crypto_instance_on_core failed with status: %"PRId32"\n", - status); - return -1; - } - - status = cpaCySymDpRegCbFunc(qaCoreConf[lcore_id].instanceHandle, - (CpaCySymDpCbFunc) qa_crypto_callback); - if (CPA_STATUS_SUCCESS != status) { - printf("Crypto: cpaCySymDpRegCbFunc failed with status: %"PRId32"\n", status); - return -1; - } - - /* - * Set the address translation callback for virtual to physcial address - * mapping. This will be called by the QAT driver during initialisation only. - */ - status = cpaCySetAddressTranslation(qaCoreConf[lcore_id].instanceHandle, - (CpaVirtualToPhysical) qa_v2p); - if (CPA_STATUS_SUCCESS != status) { - printf("Crypto: cpaCySetAddressTranslation failed with status: %"PRId32"\n", - status); - return -1; - } - - status = initSessionDataTables(&qaCoreConf[lcore_id],lcore_id); - if (CPA_STATUS_SUCCESS != status) { - printf("Crypto: Failed to allocate all session tables."); - return -1; - } - return 0; -} - -static CpaStatus -enqueueOp(CpaCySymDpOpData *opData, uint32_t lcore_id) -{ - - CpaStatus status; - - /* - * Assumption is there is no requirement to do load balancing between - * acceleration units - that is one acceleration unit is tied to a core. - */ - opData->instanceHandle = qaCoreConf[lcore_id].instanceHandle; - - if ((++qaCoreConf[lcore_id].kickFreq) % CRYPTO_BURST_TX == 0) { - status = cpaCySymDpEnqueueOp(opData, CPA_TRUE); - } else { - status = cpaCySymDpEnqueueOp(opData, CPA_FALSE); - } - - qaCoreConf[lcore_id].qaOutstandingRequests++; - - return status; -} - -void -crypto_flush_tx_queue(uint32_t lcore_id) -{ - - cpaCySymDpPerformOpNow(qaCoreConf[lcore_id].instanceHandle); -} - -enum crypto_result -crypto_encrypt(struct rte_mbuf *rte_buff, enum cipher_alg c, enum hash_alg h) -{ - CpaCySymDpOpData *opData = - rte_pktmbuf_mtod_offset(rte_buff, CpaCySymDpOpData *, - CRYPTO_OFFSET_TO_OPDATA); - uint32_t lcore_id; - - if (unlikely(c >= NUM_CRYPTO || h >= NUM_HMAC)) - return CRYPTO_RESULT_FAIL; - - lcore_id = rte_lcore_id(); - - memset(opData, 0, sizeof(CpaCySymDpOpData)); - - opData->srcBuffer = opData->dstBuffer = PACKET_DATA_START_PHYS(rte_buff); - opData->srcBufferLen = opData->dstBufferLen = rte_buff->data_len; - opData->sessionCtx = qaCoreConf[lcore_id].encryptSessionHandleTbl[c][h]; - opData->thisPhys = PACKET_DATA_START_PHYS(rte_buff) - + CRYPTO_OFFSET_TO_OPDATA; - opData->pCallbackTag = rte_buff; - - /* if no crypto or hash operations are specified return fail */ - if (NO_CIPHER == c && NO_HASH == h) - return CRYPTO_RESULT_FAIL; - - if (NO_CIPHER != c) { - opData->pIv = qaCoreConf[lcore_id].pPacketIV; - opData->iv = qaCoreConf[lcore_id].packetIVPhy; - - if (CIPHER_AES_CBC_128 == c) - opData->ivLenInBytes = IV_LENGTH_16_BYTES; - else - opData->ivLenInBytes = IV_LENGTH_8_BYTES; - - opData->cryptoStartSrcOffsetInBytes = CRYPTO_START_OFFSET; - opData->messageLenToCipherInBytes = rte_buff->data_len - - CRYPTO_START_OFFSET; - /* - * Work around for padding, message length has to be a multiple of - * block size. - */ - opData->messageLenToCipherInBytes -= opData->messageLenToCipherInBytes - % CIPHER_BLOCK_DEFAULT_SIZE; - } - - if (NO_HASH != h) { - - opData->hashStartSrcOffsetInBytes = HASH_START_OFFSET; - opData->messageLenToHashInBytes = rte_buff->data_len - - HASH_START_OFFSET; - /* - * Work around for padding, message length has to be a multiple of block - * size. - */ - opData->messageLenToHashInBytes -= opData->messageLenToHashInBytes - % HASH_BLOCK_DEFAULT_SIZE; - - /* - * Assumption: Ok ignore the passed digest pointer and place HMAC at end - * of packet. - */ - opData->digestResult = rte_buff->buf_physaddr + rte_buff->data_len; - } - - if (CPA_STATUS_SUCCESS != enqueueOp(opData, lcore_id)) { - /* - * Failed to place a packet on the hardware queue. - * Most likely because the QA hardware is busy. - */ - return CRYPTO_RESULT_FAIL; - } - return CRYPTO_RESULT_IN_PROGRESS; -} - -enum crypto_result -crypto_decrypt(struct rte_mbuf *rte_buff, enum cipher_alg c, enum hash_alg h) -{ - - CpaCySymDpOpData *opData = rte_pktmbuf_mtod_offset(rte_buff, void *, - CRYPTO_OFFSET_TO_OPDATA); - uint32_t lcore_id; - - if (unlikely(c >= NUM_CRYPTO || h >= NUM_HMAC)) - return CRYPTO_RESULT_FAIL; - - lcore_id = rte_lcore_id(); - - memset(opData, 0, sizeof(CpaCySymDpOpData)); - - opData->dstBuffer = opData->srcBuffer = PACKET_DATA_START_PHYS(rte_buff); - opData->dstBufferLen = opData->srcBufferLen = rte_buff->data_len; - opData->thisPhys = PACKET_DATA_START_PHYS(rte_buff) - + CRYPTO_OFFSET_TO_OPDATA; - opData->sessionCtx = qaCoreConf[lcore_id].decryptSessionHandleTbl[c][h]; - opData->pCallbackTag = rte_buff; - - /* if no crypto or hmac operations are specified return fail */ - if (NO_CIPHER == c && NO_HASH == h) - return CRYPTO_RESULT_FAIL; - - if (NO_CIPHER != c) { - opData->pIv = qaCoreConf[lcore_id].pPacketIV; - opData->iv = qaCoreConf[lcore_id].packetIVPhy; - - if (CIPHER_AES_CBC_128 == c) - opData->ivLenInBytes = IV_LENGTH_16_BYTES; - else - opData->ivLenInBytes = IV_LENGTH_8_BYTES; - - opData->cryptoStartSrcOffsetInBytes = CRYPTO_START_OFFSET; - opData->messageLenToCipherInBytes = rte_buff->data_len - - CRYPTO_START_OFFSET; - - /* - * Work around for padding, message length has to be a multiple of block - * size. - */ - opData->messageLenToCipherInBytes -= opData->messageLenToCipherInBytes - % CIPHER_BLOCK_DEFAULT_SIZE; - } - if (NO_HASH != h) { - opData->hashStartSrcOffsetInBytes = HASH_START_OFFSET; - opData->messageLenToHashInBytes = rte_buff->data_len - - HASH_START_OFFSET; - /* - * Work around for padding, message length has to be a multiple of block - * size. - */ - opData->messageLenToHashInBytes -= opData->messageLenToHashInBytes - % HASH_BLOCK_DEFAULT_SIZE; - opData->digestResult = rte_buff->buf_physaddr + rte_buff->data_len; - } - - if (CPA_STATUS_SUCCESS != enqueueOp(opData, lcore_id)) { - /* - * Failed to place a packet on the hardware queue. - * Most likely because the QA hardware is busy. - */ - return CRYPTO_RESULT_FAIL; - } - return CRYPTO_RESULT_IN_PROGRESS; -} - -void * -crypto_get_next_response(void) -{ - uint32_t lcore_id; - lcore_id = rte_lcore_id(); - struct qa_callbackQueue *callbackQ = &(qaCoreConf[lcore_id].callbackQueue); - void *entry = NULL; - - if (callbackQ->numEntries) { - entry = callbackQ->qaCallbackRing[callbackQ->tail]; - callbackQ->tail++; - callbackQ->numEntries--; - } - - /* If there are no outstanding requests no need to poll, return entry */ - if (qaCoreConf[lcore_id].qaOutstandingRequests == 0) - return entry; - - if (callbackQ->numEntries < CRYPTO_QUEUED_RESP_POLL_THRESHOLD - && qaCoreConf[lcore_id].numResponseAttempts++ - % GET_NEXT_RESPONSE_FREQ == 0) { - /* - * Only poll the hardware when there is less than - * CRYPTO_QUEUED_RESP_POLL_THRESHOLD elements in the software queue - */ - icp_sal_CyPollDpInstance(qaCoreConf[lcore_id].instanceHandle, - CRYPTO_MAX_RESPONSE_QUOTA); - } - return entry; -} diff --git a/dpdk/examples/dpdk_qat/crypto.h b/dpdk/examples/dpdk_qat/crypto.h deleted file mode 100644 index f68b0b65..00000000 --- a/dpdk/examples/dpdk_qat/crypto.h +++ /dev/null @@ -1,90 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef CRYPTO_H_ -#define CRYPTO_H_ - -/* Pass Labels/Values to crypto units */ -enum cipher_alg { - /* Option to not do any cryptography */ - NO_CIPHER, - CIPHER_DES, - CIPHER_DES_CBC, - CIPHER_DES3, - CIPHER_DES3_CBC, - CIPHER_AES, - CIPHER_AES_CBC_128, - CIPHER_KASUMI_F8, - NUM_CRYPTO, -}; - -enum hash_alg { - /* Option to not do any hash */ - NO_HASH, - HASH_MD5, - HASH_SHA1, - HASH_SHA1_96, - HASH_SHA224, - HASH_SHA256, - HASH_SHA384, - HASH_SHA512, - HASH_AES_XCBC, - HASH_AES_XCBC_96, - HASH_KASUMI_F9, - NUM_HMAC, -}; - -/* Return value from crypto_{encrypt/decrypt} */ -enum crypto_result { - /* Packet was successfully put into crypto queue */ - CRYPTO_RESULT_IN_PROGRESS, - /* Cryptography has failed in some way */ - CRYPTO_RESULT_FAIL, -}; - -extern enum crypto_result crypto_encrypt(struct rte_mbuf *pkt, enum cipher_alg c, - enum hash_alg h); -extern enum crypto_result crypto_decrypt(struct rte_mbuf *pkt, enum cipher_alg c, - enum hash_alg h); - -extern int crypto_init(void); - -extern int per_core_crypto_init(uint32_t lcore_id); - -extern void crypto_exit(void); - -extern void *crypto_get_next_response(void); - -extern void crypto_flush_tx_queue(uint32_t lcore_id); - -#endif /* CRYPTO_H_ */ diff --git a/dpdk/examples/dpdk_qat/main.c b/dpdk/examples/dpdk_qat/main.c deleted file mode 100644 index 3c6112d7..00000000 --- a/dpdk/examples/dpdk_qat/main.c +++ /dev/null @@ -1,822 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "crypto.h" - -#define NB_MBUF (32 * 1024) - -#define MAX_PKT_BURST 32 -#define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */ - -#define TX_QUEUE_FLUSH_MASK 0xFFFFFFFF -#define TSC_COUNT_LIMIT 1000 - -#define ACTION_ENCRYPT 1 -#define ACTION_DECRYPT 2 - -/* - * Configurable number of RX/TX ring descriptors - */ -#define RTE_TEST_RX_DESC_DEFAULT 128 -#define RTE_TEST_TX_DESC_DEFAULT 512 -static uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; -static uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; - -/* ethernet addresses of ports */ -static struct ether_addr ports_eth_addr[RTE_MAX_ETHPORTS]; - -/* mask of enabled ports */ -static unsigned enabled_port_mask = 0; -static int promiscuous_on = 1; /**< Ports set in promiscuous mode on by default. */ - -/* list of enabled ports */ -static uint32_t dst_ports[RTE_MAX_ETHPORTS]; - -struct mbuf_table { - uint16_t len; - struct rte_mbuf *m_table[MAX_PKT_BURST]; -}; - -struct lcore_rx_queue { - uint8_t port_id; - uint8_t queue_id; -}; - -#define MAX_RX_QUEUE_PER_LCORE 16 - -#define MAX_LCORE_PARAMS 1024 -struct lcore_params { - uint8_t port_id; - uint8_t queue_id; - uint8_t lcore_id; -}; - -static struct lcore_params lcore_params_array[MAX_LCORE_PARAMS]; -static struct lcore_params lcore_params_array_default[] = { - {0, 0, 2}, - {0, 1, 2}, - {0, 2, 2}, - {1, 0, 2}, - {1, 1, 2}, - {1, 2, 2}, - {2, 0, 2}, - {3, 0, 3}, - {3, 1, 3}, -}; - -static struct lcore_params * lcore_params = lcore_params_array_default; -static uint16_t nb_lcore_params = sizeof(lcore_params_array_default) / - sizeof(lcore_params_array_default[0]); - -static struct rte_eth_conf port_conf = { - .rxmode = { - .mq_mode = ETH_MQ_RX_RSS, - .split_hdr_size = 0, - .header_split = 0, /**< Header Split disabled */ - .hw_ip_checksum = 1, /**< IP checksum offload enabled */ - .hw_vlan_filter = 0, /**< VLAN filtering disabled */ - .jumbo_frame = 0, /**< Jumbo Frame Support disabled */ - .hw_strip_crc = 0, /**< CRC stripped by hardware */ - }, - .rx_adv_conf = { - .rss_conf = { - .rss_key = NULL, - .rss_hf = ETH_RSS_IP, - }, - }, - .txmode = { - .mq_mode = ETH_MQ_TX_NONE, - }, -}; - -static struct rte_mempool * pktmbuf_pool[RTE_MAX_NUMA_NODES]; - -struct lcore_conf { - uint64_t tsc; - uint64_t tsc_count; - uint32_t tx_mask; - uint16_t n_rx_queue; - uint16_t rx_queue_list_pos; - struct lcore_rx_queue rx_queue_list[MAX_RX_QUEUE_PER_LCORE]; - uint16_t tx_queue_id[RTE_MAX_ETHPORTS]; - struct mbuf_table rx_mbuf; - uint32_t rx_mbuf_pos; - uint32_t rx_curr_queue; - struct mbuf_table tx_mbufs[RTE_MAX_ETHPORTS]; -} __rte_cache_aligned; - -static struct lcore_conf lcore_conf[RTE_MAX_LCORE]; - -static inline struct rte_mbuf * -nic_rx_get_packet(struct lcore_conf *qconf) -{ - struct rte_mbuf *pkt; - - if (unlikely(qconf->n_rx_queue == 0)) - return NULL; - - /* Look for the next queue with packets; return if none */ - if (unlikely(qconf->rx_mbuf_pos == qconf->rx_mbuf.len)) { - uint32_t i; - - qconf->rx_mbuf_pos = 0; - for (i = 0; i < qconf->n_rx_queue; i++) { - qconf->rx_mbuf.len = rte_eth_rx_burst( - qconf->rx_queue_list[qconf->rx_curr_queue].port_id, - qconf->rx_queue_list[qconf->rx_curr_queue].queue_id, - qconf->rx_mbuf.m_table, MAX_PKT_BURST); - - qconf->rx_curr_queue++; - if (unlikely(qconf->rx_curr_queue == qconf->n_rx_queue)) - qconf->rx_curr_queue = 0; - if (likely(qconf->rx_mbuf.len > 0)) - break; - } - if (unlikely(i == qconf->n_rx_queue)) - return NULL; - } - - /* Get the next packet from the current queue; if last packet, go to next queue */ - pkt = qconf->rx_mbuf.m_table[qconf->rx_mbuf_pos]; - qconf->rx_mbuf_pos++; - - return pkt; -} - -static inline void -nic_tx_flush_queues(struct lcore_conf *qconf) -{ - uint8_t portid; - - for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++) { - struct rte_mbuf **m_table = NULL; - uint16_t queueid, len; - uint32_t n, i; - - if (likely((qconf->tx_mask & (1 << portid)) == 0)) - continue; - - len = qconf->tx_mbufs[portid].len; - if (likely(len == 0)) - continue; - - queueid = qconf->tx_queue_id[portid]; - m_table = qconf->tx_mbufs[portid].m_table; - - n = rte_eth_tx_burst(portid, queueid, m_table, len); - for (i = n; i < len; i++){ - rte_pktmbuf_free(m_table[i]); - } - - qconf->tx_mbufs[portid].len = 0; - } - - qconf->tx_mask = TX_QUEUE_FLUSH_MASK; -} - -static inline void -nic_tx_send_packet(struct rte_mbuf *pkt, uint8_t port) -{ - struct lcore_conf *qconf; - uint32_t lcoreid; - uint16_t len; - - if (unlikely(pkt == NULL)) { - return; - } - - lcoreid = rte_lcore_id(); - qconf = &lcore_conf[lcoreid]; - - len = qconf->tx_mbufs[port].len; - qconf->tx_mbufs[port].m_table[len] = pkt; - len++; - - /* enough pkts to be sent */ - if (unlikely(len == MAX_PKT_BURST)) { - uint32_t n, i; - uint16_t queueid; - - queueid = qconf->tx_queue_id[port]; - n = rte_eth_tx_burst(port, queueid, qconf->tx_mbufs[port].m_table, MAX_PKT_BURST); - for (i = n; i < MAX_PKT_BURST; i++){ - rte_pktmbuf_free(qconf->tx_mbufs[port].m_table[i]); - } - - qconf->tx_mask &= ~(1 << port); - len = 0; - } - - qconf->tx_mbufs[port].len = len; -} - -/* main processing loop */ -static __attribute__((noreturn)) int -main_loop(__attribute__((unused)) void *dummy) -{ - uint32_t lcoreid; - struct lcore_conf *qconf; - const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / US_PER_S * BURST_TX_DRAIN_US; - - lcoreid = rte_lcore_id(); - qconf = &lcore_conf[lcoreid]; - - printf("Thread %u starting...\n", lcoreid); - - for (;;) { - struct rte_mbuf *pkt; - uint32_t pkt_from_nic_rx = 0; - uint8_t port; - - /* Flush TX queues */ - qconf->tsc_count++; - if (unlikely(qconf->tsc_count == TSC_COUNT_LIMIT)) { - uint64_t tsc, diff_tsc; - - tsc = rte_rdtsc(); - - diff_tsc = tsc - qconf->tsc; - if (unlikely(diff_tsc > drain_tsc)) { - nic_tx_flush_queues(qconf); - crypto_flush_tx_queue(lcoreid); - qconf->tsc = tsc; - } - - qconf->tsc_count = 0; - } - - /* - * Check the Intel QuickAssist queues first - * - ***/ - pkt = (struct rte_mbuf *) crypto_get_next_response(); - if (pkt == NULL) { - pkt = nic_rx_get_packet(qconf); - pkt_from_nic_rx = 1; - } - if (pkt == NULL) - continue; - /* Send packet to either QAT encrypt, QAT decrypt or NIC TX */ - if (pkt_from_nic_rx) { - struct ipv4_hdr *ip = rte_pktmbuf_mtod_offset(pkt, - struct ipv4_hdr *, - sizeof(struct ether_hdr)); - if (ip->src_addr & rte_cpu_to_be_32(ACTION_ENCRYPT)) { - if (CRYPTO_RESULT_FAIL == crypto_encrypt(pkt, - (enum cipher_alg)((ip->src_addr >> 16) & 0xFF), - (enum hash_alg)((ip->src_addr >> 8) & 0xFF))) - rte_pktmbuf_free(pkt); - continue; - } - - if (ip->src_addr & rte_cpu_to_be_32(ACTION_DECRYPT)) { - if(CRYPTO_RESULT_FAIL == crypto_decrypt(pkt, - (enum cipher_alg)((ip->src_addr >> 16) & 0xFF), - (enum hash_alg)((ip->src_addr >> 8) & 0xFF))) - rte_pktmbuf_free(pkt); - continue; - } - } - - port = dst_ports[pkt->port]; - - /* Transmit the packet */ - nic_tx_send_packet(pkt, (uint8_t)port); - } -} - -static inline unsigned -get_port_max_rx_queues(uint8_t port_id) -{ - struct rte_eth_dev_info dev_info; - - rte_eth_dev_info_get(port_id, &dev_info); - return dev_info.max_rx_queues; -} - -static inline unsigned -get_port_max_tx_queues(uint8_t port_id) -{ - struct rte_eth_dev_info dev_info; - - rte_eth_dev_info_get(port_id, &dev_info); - return dev_info.max_tx_queues; -} - -static int -check_lcore_params(void) -{ - uint16_t i; - - for (i = 0; i < nb_lcore_params; ++i) { - if (lcore_params[i].queue_id >= get_port_max_rx_queues(lcore_params[i].port_id)) { - printf("invalid queue number: %hhu\n", lcore_params[i].queue_id); - return -1; - } - if (!rte_lcore_is_enabled(lcore_params[i].lcore_id)) { - printf("error: lcore %hhu is not enabled in lcore mask\n", - lcore_params[i].lcore_id); - return -1; - } - } - return 0; -} - -static int -check_port_config(const unsigned nb_ports) -{ - unsigned portid; - uint16_t i; - - for (i = 0; i < nb_lcore_params; ++i) { - portid = lcore_params[i].port_id; - if ((enabled_port_mask & (1 << portid)) == 0) { - printf("port %u is not enabled in port mask\n", portid); - return -1; - } - if (portid >= nb_ports) { - printf("port %u is not present on the board\n", portid); - return -1; - } - } - return 0; -} - -static uint8_t -get_port_n_rx_queues(const uint8_t port) -{ - int queue = -1; - uint16_t i; - - for (i = 0; i < nb_lcore_params; ++i) { - if (lcore_params[i].port_id == port && lcore_params[i].queue_id > queue) - queue = lcore_params[i].queue_id; - } - return (uint8_t)(++queue); -} - -static int -init_lcore_rx_queues(void) -{ - uint16_t i, nb_rx_queue; - uint8_t lcore; - - for (i = 0; i < nb_lcore_params; ++i) { - lcore = lcore_params[i].lcore_id; - nb_rx_queue = lcore_conf[lcore].n_rx_queue; - if (nb_rx_queue >= MAX_RX_QUEUE_PER_LCORE) { - printf("error: too many queues (%u) for lcore: %u\n", - (unsigned)nb_rx_queue + 1, (unsigned)lcore); - return -1; - } - lcore_conf[lcore].rx_queue_list[nb_rx_queue].port_id = - lcore_params[i].port_id; - lcore_conf[lcore].rx_queue_list[nb_rx_queue].queue_id = - lcore_params[i].queue_id; - lcore_conf[lcore].n_rx_queue++; - } - return 0; -} - -/* display usage */ -static void -print_usage(const char *prgname) -{ - printf ("%s [EAL options] -- -p PORTMASK [--no-promisc]" - " [--config '(port,queue,lcore)[,(port,queue,lcore)]'\n" - " -p PORTMASK: hexadecimal bitmask of ports to configure\n" - " --no-promisc: disable promiscuous mode (default is ON)\n" - " --config '(port,queue,lcore)': rx queues configuration\n", - prgname); -} - -static unsigned -parse_portmask(const char *portmask) -{ - char *end = NULL; - unsigned pm; - - /* parse hexadecimal string */ - pm = strtoul(portmask, &end, 16); - if ((portmask[0] == '\0') || (end == NULL) || (*end != '\0')) - return 0; - - return pm; -} - -static int -parse_config(const char *q_arg) -{ - char s[256]; - const char *p, *p_end = q_arg; - char *end; - enum fieldnames { - FLD_PORT = 0, - FLD_QUEUE, - FLD_LCORE, - _NUM_FLD - }; - unsigned long int_fld[_NUM_FLD]; - char *str_fld[_NUM_FLD]; - int i; - unsigned size; - - nb_lcore_params = 0; - - while ((p = strchr(p_end,'(')) != NULL) { - if (nb_lcore_params >= MAX_LCORE_PARAMS) { - printf("exceeded max number of lcore params: %hu\n", - nb_lcore_params); - return -1; - } - ++p; - if((p_end = strchr(p,')')) == NULL) - return -1; - - size = p_end - p; - if(size >= sizeof(s)) - return -1; - - snprintf(s, sizeof(s), "%.*s", size, p); - if (rte_strsplit(s, sizeof(s), str_fld, _NUM_FLD, ',') != _NUM_FLD) - return -1; - for (i = 0; i < _NUM_FLD; i++) { - errno = 0; - int_fld[i] = strtoul(str_fld[i], &end, 0); - if (errno != 0 || end == str_fld[i] || int_fld[i] > 255) - return -1; - } - lcore_params_array[nb_lcore_params].port_id = (uint8_t)int_fld[FLD_PORT]; - lcore_params_array[nb_lcore_params].queue_id = (uint8_t)int_fld[FLD_QUEUE]; - lcore_params_array[nb_lcore_params].lcore_id = (uint8_t)int_fld[FLD_LCORE]; - ++nb_lcore_params; - } - lcore_params = lcore_params_array; - return 0; -} - -/* Parse the argument given in the command line of the application */ -static int -parse_args(int argc, char **argv) -{ - int opt, ret; - char **argvopt; - int option_index; - char *prgname = argv[0]; - static struct option lgopts[] = { - {"config", 1, 0, 0}, - {"no-promisc", 0, 0, 0}, - {NULL, 0, 0, 0} - }; - - argvopt = argv; - - while ((opt = getopt_long(argc, argvopt, "p:", - lgopts, &option_index)) != EOF) { - - switch (opt) { - /* portmask */ - case 'p': - enabled_port_mask = parse_portmask(optarg); - if (enabled_port_mask == 0) { - printf("invalid portmask\n"); - print_usage(prgname); - return -1; - } - break; - - /* long options */ - case 0: - if (strcmp(lgopts[option_index].name, "config") == 0) { - ret = parse_config(optarg); - if (ret) { - printf("invalid config\n"); - print_usage(prgname); - return -1; - } - } - if (strcmp(lgopts[option_index].name, "no-promisc") == 0) { - printf("Promiscuous mode disabled\n"); - promiscuous_on = 0; - } - break; - default: - print_usage(prgname); - return -1; - } - } - - if (enabled_port_mask == 0) { - printf("portmask not specified\n"); - print_usage(prgname); - return -1; - } - - if (optind >= 0) - argv[optind-1] = prgname; - - ret = optind-1; - optind = 0; /* reset getopt lib */ - return ret; -} - -static void -print_ethaddr(const char *name, const struct ether_addr *eth_addr) -{ - char buf[ETHER_ADDR_FMT_SIZE]; - ether_format_addr(buf, ETHER_ADDR_FMT_SIZE, eth_addr); - printf("%s%s", name, buf); -} - -static int -init_mem(void) -{ - int socketid; - unsigned lcoreid; - char s[64]; - - RTE_LCORE_FOREACH(lcoreid) { - socketid = rte_lcore_to_socket_id(lcoreid); - if (socketid >= RTE_MAX_NUMA_NODES) { - printf("Socket %d of lcore %u is out of range %d\n", - socketid, lcoreid, RTE_MAX_NUMA_NODES); - return -1; - } - if (pktmbuf_pool[socketid] == NULL) { - snprintf(s, sizeof(s), "mbuf_pool_%d", socketid); - pktmbuf_pool[socketid] = - rte_pktmbuf_pool_create(s, NB_MBUF, 32, 0, - RTE_MBUF_DEFAULT_BUF_SIZE, socketid); - if (pktmbuf_pool[socketid] == NULL) { - printf("Cannot init mbuf pool on socket %d\n", socketid); - return -1; - } - printf("Allocated mbuf pool on socket %d\n", socketid); - } - } - return 0; -} - -int -main(int argc, char **argv) -{ - struct lcore_conf *qconf; - struct rte_eth_link link; - int ret; - unsigned nb_ports; - uint16_t queueid; - unsigned lcoreid; - uint32_t nb_tx_queue; - uint8_t portid, nb_rx_queue, queue, socketid, last_port; - unsigned nb_ports_in_mask = 0; - - /* init EAL */ - ret = rte_eal_init(argc, argv); - if (ret < 0) - return -1; - argc -= ret; - argv += ret; - - /* parse application arguments (after the EAL ones) */ - ret = parse_args(argc, argv); - if (ret < 0) - return -1; - - if (check_lcore_params() < 0) - rte_panic("check_lcore_params failed\n"); - - ret = init_lcore_rx_queues(); - if (ret < 0) - return -1; - - ret = init_mem(); - if (ret < 0) - return -1; - - nb_ports = rte_eth_dev_count(); - - if (check_port_config(nb_ports) < 0) - rte_panic("check_port_config failed\n"); - - /* reset dst_ports */ - for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++) - dst_ports[portid] = 0; - last_port = 0; - - /* - * Each logical core is assigned a dedicated TX queue on each port. - */ - for (portid = 0; portid < nb_ports; portid++) { - /* skip ports that are not enabled */ - if ((enabled_port_mask & (1 << portid)) == 0) - continue; - - if (nb_ports_in_mask % 2) { - dst_ports[portid] = last_port; - dst_ports[last_port] = portid; - } - else - last_port = portid; - - nb_ports_in_mask++; - } - if (nb_ports_in_mask % 2) { - printf("Notice: odd number of ports in portmask.\n"); - dst_ports[last_port] = last_port; - } - - /* initialize all ports */ - for (portid = 0; portid < nb_ports; portid++) { - /* skip ports that are not enabled */ - if ((enabled_port_mask & (1 << portid)) == 0) { - printf("\nSkipping disabled port %d\n", portid); - continue; - } - - /* init port */ - printf("Initializing port %d ... ", portid ); - fflush(stdout); - - nb_rx_queue = get_port_n_rx_queues(portid); - if (nb_rx_queue > get_port_max_rx_queues(portid)) - rte_panic("Number of rx queues %d exceeds max number of rx queues %u" - " for port %d\n", nb_rx_queue, get_port_max_rx_queues(portid), - portid); - nb_tx_queue = rte_lcore_count(); - if (nb_tx_queue > get_port_max_tx_queues(portid)) - rte_panic("Number of lcores %u exceeds max number of tx queues %u" - " for port %d\n", nb_tx_queue, get_port_max_tx_queues(portid), - portid); - printf("Creating queues: nb_rxq=%d nb_txq=%u... ", - nb_rx_queue, (unsigned)nb_tx_queue ); - ret = rte_eth_dev_configure(portid, nb_rx_queue, - (uint16_t)nb_tx_queue, &port_conf); - if (ret < 0) - rte_panic("Cannot configure device: err=%d, port=%d\n", - ret, portid); - - rte_eth_macaddr_get(portid, &ports_eth_addr[portid]); - print_ethaddr(" Address:", &ports_eth_addr[portid]); - printf(", "); - - /* init one TX queue per couple (lcore,port) */ - queueid = 0; - RTE_LCORE_FOREACH(lcoreid) { - socketid = (uint8_t)rte_lcore_to_socket_id(lcoreid); - printf("txq=%u,%d,%d ", lcoreid, queueid, socketid); - fflush(stdout); - ret = rte_eth_tx_queue_setup(portid, queueid, nb_txd, - socketid, - NULL); - if (ret < 0) - rte_panic("rte_eth_tx_queue_setup: err=%d, " - "port=%d\n", ret, portid); - - qconf = &lcore_conf[lcoreid]; - qconf->tx_queue_id[portid] = queueid; - queueid++; - } - printf("\n"); - } - - RTE_LCORE_FOREACH(lcoreid) { - qconf = &lcore_conf[lcoreid]; - printf("\nInitializing rx queues on lcore %u ... ", lcoreid ); - fflush(stdout); - /* init RX queues */ - for(queue = 0; queue < qconf->n_rx_queue; ++queue) { - portid = qconf->rx_queue_list[queue].port_id; - queueid = qconf->rx_queue_list[queue].queue_id; - socketid = (uint8_t)rte_lcore_to_socket_id(lcoreid); - printf("rxq=%d,%d,%d ", portid, queueid, socketid); - fflush(stdout); - - ret = rte_eth_rx_queue_setup(portid, queueid, nb_rxd, - socketid, - NULL, - pktmbuf_pool[socketid]); - if (ret < 0) - rte_panic("rte_eth_rx_queue_setup: err=%d," - "port=%d\n", ret, portid); - } - } - - printf("\n"); - - /* start ports */ - for (portid = 0; portid < nb_ports; portid++) { - if ((enabled_port_mask & (1 << portid)) == 0) - continue; - /* Start device */ - ret = rte_eth_dev_start(portid); - if (ret < 0) - rte_panic("rte_eth_dev_start: err=%d, port=%d\n", - ret, portid); - - printf("done: Port %d ", portid); - - /* get link status */ - rte_eth_link_get(portid, &link); - if (link.link_status) - printf(" Link Up - speed %u Mbps - %s\n", - (unsigned) link.link_speed, - (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? - ("full-duplex") : ("half-duplex\n")); - else - printf(" Link Down\n"); - /* - * If enabled, put device in promiscuous mode. - * This allows IO forwarding mode to forward packets - * to itself through 2 cross-connected ports of the - * target machine. - */ - if (promiscuous_on) - rte_eth_promiscuous_enable(portid); - } - printf("Crypto: Initializing Crypto...\n"); - if (crypto_init() != 0) - return -1; - - RTE_LCORE_FOREACH(lcoreid) { - if (per_core_crypto_init(lcoreid) != 0) { - printf("Crypto: Cannot init lcore crypto on lcore %u\n", (unsigned)lcoreid); - return -1; - } - } - printf("Crypto: Initialization complete\n"); - /* launch per-lcore init on every lcore */ - rte_eal_mp_remote_launch(main_loop, NULL, CALL_MASTER); - RTE_LCORE_FOREACH_SLAVE(lcoreid) { - if (rte_eal_wait_lcore(lcoreid) < 0) - return -1; - } - - return 0; -} diff --git a/dpdk/examples/ethtool/Makefile b/dpdk/examples/ethtool/Makefile index 995cd25b..e86d68ac 100644 --- a/dpdk/examples/ethtool/Makefile +++ b/dpdk/examples/ethtool/Makefile @@ -46,4 +46,6 @@ else DIRS-y += lib ethtool-app endif +DEPDIRS-ethtool-app := lib + include $(RTE_SDK)/mk/rte.extsubdir.mk diff --git a/dpdk/examples/ethtool/ethtool-app/Makefile b/dpdk/examples/ethtool/ethtool-app/Makefile index 09c66ad1..96abf53b 100644 --- a/dpdk/examples/ethtool/ethtool-app/Makefile +++ b/dpdk/examples/ethtool/ethtool-app/Makefile @@ -50,5 +50,10 @@ CFLAGS += $(WERROR_FLAGS) LDLIBS += -L$(subst ethtool-app,lib,$(RTE_OUTPUT))/lib LDLIBS += -lrte_ethtool +ifeq ($(CONFIG_RTE_BUILD_SHARED_LIB),y) +ifeq ($(CONFIG_RTE_LIBRTE_IXGBE_PMD),y) +LDLIBS += -lrte_pmd_ixgbe +endif +endif include $(RTE_SDK)/mk/rte.extapp.mk diff --git a/dpdk/examples/ethtool/ethtool-app/ethapp.c b/dpdk/examples/ethtool/ethtool-app/ethapp.c index 38e466c0..35269ea2 100644 --- a/dpdk/examples/ethtool/ethtool-app/ethapp.c +++ b/dpdk/examples/ethtool/ethtool-app/ethapp.c @@ -177,6 +177,7 @@ pcmd_drvinfo_callback(__rte_unused void *ptr_params, int id_port; for (id_port = 0; id_port < rte_eth_dev_count(); id_port++) { + memset(&info, 0, sizeof(info)); if (rte_ethtool_get_drvinfo(id_port, &info)) { printf("Error getting info for port %i\n", id_port); return; @@ -184,6 +185,8 @@ pcmd_drvinfo_callback(__rte_unused void *ptr_params, printf("Port %i driver: %s (ver: %s)\n", id_port, info.driver, info.version ); + printf("firmware-version: %s\n", info.fw_version); + printf("bus-info: %s\n", info.bus_info); } } diff --git a/dpdk/examples/ethtool/ethtool-app/main.c b/dpdk/examples/ethtool/ethtool-app/main.c index 2c655d83..bbab2f6e 100644 --- a/dpdk/examples/ethtool/ethtool-app/main.c +++ b/dpdk/examples/ethtool/ethtool-app/main.c @@ -122,6 +122,8 @@ static void setup_ports(struct app_config *app_cfg, int cnt_ports) struct rte_eth_conf cfg_port; struct rte_eth_dev_info dev_info; char str_name[16]; + uint16_t nb_rxd = PORT_RX_QUEUE_SIZE; + uint16_t nb_txd = PORT_TX_QUEUE_SIZE; memset(&cfg_port, 0, sizeof(cfg_port)); cfg_port.txmode.mq_mode = ETH_MQ_TX_NONE; @@ -154,15 +156,19 @@ static void setup_ports(struct app_config *app_cfg, int cnt_ports) if (rte_eth_dev_configure(idx_port, 1, 1, &cfg_port) < 0) rte_exit(EXIT_FAILURE, "rte_eth_dev_configure failed"); + if (rte_eth_dev_adjust_nb_rx_tx_desc(idx_port, &nb_rxd, + &nb_txd) < 0) + rte_exit(EXIT_FAILURE, + "rte_eth_dev_adjust_nb_rx_tx_desc failed"); if (rte_eth_rx_queue_setup( - idx_port, 0, PORT_RX_QUEUE_SIZE, + idx_port, 0, nb_rxd, rte_eth_dev_socket_id(idx_port), NULL, ptr_port->pkt_pool) < 0) rte_exit(EXIT_FAILURE, "rte_eth_rx_queue_setup failed" ); if (rte_eth_tx_queue_setup( - idx_port, 0, PORT_TX_QUEUE_SIZE, + idx_port, 0, nb_txd, rte_eth_dev_socket_id(idx_port), NULL) < 0) rte_exit(EXIT_FAILURE, "rte_eth_tx_queue_setup failed" @@ -172,7 +178,6 @@ static void setup_ports(struct app_config *app_cfg, int cnt_ports) "%s:%i: rte_eth_dev_start failed", __FILE__, __LINE__ ); - rte_eth_promiscuous_enable(idx_port); rte_eth_macaddr_get(idx_port, &ptr_port->mac_addr); rte_spinlock_init(&ptr_port->lock); } @@ -265,7 +270,7 @@ int main(int argc, char **argv) uint32_t id_core; uint32_t cnt_ports; - /* Init runtime enviornment */ + /* Init runtime environment */ cnt_args_parsed = rte_eal_init(argc, argv); if (cnt_args_parsed < 0) rte_exit(EXIT_FAILURE, "rte_eal_init(): Failed"); diff --git a/dpdk/examples/ethtool/lib/Makefile b/dpdk/examples/ethtool/lib/Makefile index 5b4991e2..cabd82a0 100644 --- a/dpdk/examples/ethtool/lib/Makefile +++ b/dpdk/examples/ethtool/lib/Makefile @@ -54,8 +54,11 @@ SRCS-y := rte_ethtool.c CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) -# internal dependencies -DEPDIRS-y += lib/librte_eal -DEPDIRS-y += lib/librte_ether +ifeq ($(CONFIG_RTE_BUILD_SHARED_LIB),y) +ifeq ($(CONFIG_RTE_LIBRTE_IXGBE_PMD),y) +LDLIBS += -lrte_pmd_ixgbe +endif +endif +LDLIBS += -lrte_eal -lrte_ethdev include $(RTE_SDK)/mk/rte.extlib.mk diff --git a/dpdk/examples/ethtool/lib/rte_ethtool.c b/dpdk/examples/ethtool/lib/rte_ethtool.c index a1f91d45..c70c5478 100644 --- a/dpdk/examples/ethtool/lib/rte_ethtool.c +++ b/dpdk/examples/ethtool/lib/rte_ethtool.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -36,6 +36,10 @@ #include #include #include +#include +#ifdef RTE_LIBRTE_IXGBE_PMD +#include +#endif #include "rte_ethtool.h" #define PKTPOOL_SIZE 512 @@ -43,17 +47,26 @@ int -rte_ethtool_get_drvinfo(uint8_t port_id, struct ethtool_drvinfo *drvinfo) +rte_ethtool_get_drvinfo(uint16_t port_id, struct ethtool_drvinfo *drvinfo) { struct rte_eth_dev_info dev_info; struct rte_dev_reg_info reg_info; int n; + int ret; if (drvinfo == NULL) return -EINVAL; RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); + ret = rte_eth_dev_fw_version_get(port_id, drvinfo->fw_version, + sizeof(drvinfo->fw_version)); + if (ret < 0) + printf("firmware version get error: (%s)\n", strerror(-ret)); + else if (ret > 0) + printf("Insufficient fw version buffer size, " + "the minimum size should be %d\n", ret); + memset(&dev_info, 0, sizeof(dev_info)); rte_eth_dev_info_get(port_id, &dev_info); @@ -61,10 +74,16 @@ rte_ethtool_get_drvinfo(uint8_t port_id, struct ethtool_drvinfo *drvinfo) dev_info.driver_name); snprintf(drvinfo->version, sizeof(drvinfo->version), "%s", rte_version()); - snprintf(drvinfo->bus_info, sizeof(drvinfo->bus_info), - "%04x:%02x:%02x.%x", - dev_info.pci_dev->addr.domain, dev_info.pci_dev->addr.bus, - dev_info.pci_dev->addr.devid, dev_info.pci_dev->addr.function); + /* TODO: replace bus_info by rte_devargs.name */ + if (dev_info.pci_dev) + snprintf(drvinfo->bus_info, sizeof(drvinfo->bus_info), + "%04x:%02x:%02x.%x", + dev_info.pci_dev->addr.domain, + dev_info.pci_dev->addr.bus, + dev_info.pci_dev->addr.devid, + dev_info.pci_dev->addr.function); + else + snprintf(drvinfo->bus_info, sizeof(drvinfo->bus_info), "N/A"); memset(®_info, 0, sizeof(reg_info)); rte_eth_dev_get_reg_info(port_id, ®_info); @@ -87,7 +106,7 @@ rte_ethtool_get_drvinfo(uint8_t port_id, struct ethtool_drvinfo *drvinfo) } int -rte_ethtool_get_regs_len(uint8_t port_id) +rte_ethtool_get_regs_len(uint16_t port_id) { struct rte_dev_reg_info reg_info; int ret; @@ -102,7 +121,7 @@ rte_ethtool_get_regs_len(uint8_t port_id) } int -rte_ethtool_get_regs(uint8_t port_id, struct ethtool_regs *regs, void *data) +rte_ethtool_get_regs(uint16_t port_id, struct ethtool_regs *regs, void *data) { struct rte_dev_reg_info reg_info; int status; @@ -122,7 +141,7 @@ rte_ethtool_get_regs(uint8_t port_id, struct ethtool_regs *regs, void *data) } int -rte_ethtool_get_link(uint8_t port_id) +rte_ethtool_get_link(uint16_t port_id) { struct rte_eth_link link; @@ -132,13 +151,13 @@ rte_ethtool_get_link(uint8_t port_id) } int -rte_ethtool_get_eeprom_len(uint8_t port_id) +rte_ethtool_get_eeprom_len(uint16_t port_id) { return rte_eth_dev_get_eeprom_length(port_id); } int -rte_ethtool_get_eeprom(uint8_t port_id, struct ethtool_eeprom *eeprom, +rte_ethtool_get_eeprom(uint16_t port_id, struct ethtool_eeprom *eeprom, void *words) { struct rte_dev_eeprom_info eeprom_info; @@ -161,7 +180,7 @@ rte_ethtool_get_eeprom(uint8_t port_id, struct ethtool_eeprom *eeprom, } int -rte_ethtool_set_eeprom(uint8_t port_id, struct ethtool_eeprom *eeprom, +rte_ethtool_set_eeprom(uint16_t port_id, struct ethtool_eeprom *eeprom, void *words) { struct rte_dev_eeprom_info eeprom_info; @@ -184,7 +203,7 @@ rte_ethtool_set_eeprom(uint8_t port_id, struct ethtool_eeprom *eeprom, } int -rte_ethtool_get_pauseparam(uint8_t port_id, +rte_ethtool_get_pauseparam(uint16_t port_id, struct ethtool_pauseparam *pause_param) { struct rte_eth_fc_conf fc_conf; @@ -219,7 +238,7 @@ rte_ethtool_get_pauseparam(uint8_t port_id, } int -rte_ethtool_set_pauseparam(uint8_t port_id, +rte_ethtool_set_pauseparam(uint16_t port_id, struct ethtool_pauseparam *pause_param) { struct rte_eth_fc_conf fc_conf; @@ -262,7 +281,7 @@ rte_ethtool_set_pauseparam(uint8_t port_id, } int -rte_ethtool_net_open(uint8_t port_id) +rte_ethtool_net_open(uint16_t port_id) { rte_eth_dev_stop(port_id); @@ -270,7 +289,7 @@ rte_ethtool_net_open(uint8_t port_id) } int -rte_ethtool_net_stop(uint8_t port_id) +rte_ethtool_net_stop(uint16_t port_id) { RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); rte_eth_dev_stop(port_id); @@ -279,7 +298,7 @@ rte_ethtool_net_stop(uint8_t port_id) } int -rte_ethtool_net_get_mac_addr(uint8_t port_id, struct ether_addr *addr) +rte_ethtool_net_get_mac_addr(uint16_t port_id, struct ether_addr *addr) { RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); if (addr == NULL) @@ -290,7 +309,7 @@ rte_ethtool_net_get_mac_addr(uint8_t port_id, struct ether_addr *addr) } int -rte_ethtool_net_set_mac_addr(uint8_t port_id, struct ether_addr *addr) +rte_ethtool_net_set_mac_addr(uint16_t port_id, struct ether_addr *addr) { if (addr == NULL) return -EINVAL; @@ -298,7 +317,7 @@ rte_ethtool_net_set_mac_addr(uint8_t port_id, struct ether_addr *addr) } int -rte_ethtool_net_validate_addr(uint8_t port_id __rte_unused, +rte_ethtool_net_validate_addr(uint16_t port_id __rte_unused, struct ether_addr *addr) { if (addr == NULL) @@ -307,7 +326,7 @@ rte_ethtool_net_validate_addr(uint8_t port_id __rte_unused, } int -rte_ethtool_net_change_mtu(uint8_t port_id, int mtu) +rte_ethtool_net_change_mtu(uint16_t port_id, int mtu) { if (mtu < 0 || mtu > UINT16_MAX) return -EINVAL; @@ -315,7 +334,7 @@ rte_ethtool_net_change_mtu(uint8_t port_id, int mtu) } int -rte_ethtool_net_get_stats64(uint8_t port_id, struct rte_eth_stats *stats) +rte_ethtool_net_get_stats64(uint16_t port_id, struct rte_eth_stats *stats) { if (stats == NULL) return -EINVAL; @@ -323,13 +342,13 @@ rte_ethtool_net_get_stats64(uint8_t port_id, struct rte_eth_stats *stats) } int -rte_ethtool_net_vlan_rx_add_vid(uint8_t port_id, uint16_t vid) +rte_ethtool_net_vlan_rx_add_vid(uint16_t port_id, uint16_t vid) { return rte_eth_dev_vlan_filter(port_id, vid, 1); } int -rte_ethtool_net_vlan_rx_kill_vid(uint8_t port_id, uint16_t vid) +rte_ethtool_net_vlan_rx_kill_vid(uint16_t port_id, uint16_t vid) { return rte_eth_dev_vlan_filter(port_id, vid, 0); } @@ -342,7 +361,7 @@ rte_ethtool_net_vlan_rx_kill_vid(uint8_t port_id, uint16_t vid) * driver can register device-specific implementation */ int -rte_ethtool_net_set_rx_mode(uint8_t port_id) +rte_ethtool_net_set_rx_mode(uint16_t port_id) { uint16_t num_vfs; struct rte_eth_dev_info dev_info; @@ -353,9 +372,12 @@ rte_ethtool_net_set_rx_mode(uint8_t port_id) num_vfs = dev_info.max_vfs; /* Set VF vf_rx_mode, VF unsupport status is discard */ - for (vf = 0; vf < num_vfs; vf++) - rte_eth_dev_set_vf_rxmode(port_id, vf, + for (vf = 0; vf < num_vfs; vf++) { +#ifdef RTE_LIBRTE_IXGBE_PMD + rte_pmd_ixgbe_set_vf_rxmode(port_id, vf, ETH_VMDQ_ACCEPT_UNTAG, 0); +#endif + } /* Enable Rx vlan filter, VF unspport status is discard */ rte_eth_dev_set_vlan_offload(port_id, ETH_VLAN_FILTER_MASK); @@ -365,7 +387,7 @@ rte_ethtool_net_set_rx_mode(uint8_t port_id) int -rte_ethtool_get_ringparam(uint8_t port_id, +rte_ethtool_get_ringparam(uint16_t port_id, struct ethtool_ringparam *ring_param) { struct rte_eth_dev_info dev_info; @@ -397,7 +419,7 @@ rte_ethtool_get_ringparam(uint8_t port_id, int -rte_ethtool_set_ringparam(uint8_t port_id, +rte_ethtool_set_ringparam(uint16_t port_id, struct ethtool_ringparam *ring_param) { struct rte_eth_rxq_info rx_qinfo; diff --git a/dpdk/examples/ethtool/lib/rte_ethtool.h b/dpdk/examples/ethtool/lib/rte_ethtool.h index 2e79d453..1cd8f4d2 100644 --- a/dpdk/examples/ethtool/lib/rte_ethtool.h +++ b/dpdk/examples/ethtool/lib/rte_ethtool.h @@ -79,7 +79,7 @@ extern "C" { * - (0) if successful. * - (-ENODEV) if *port_id* invalid. */ -int rte_ethtool_get_drvinfo(uint8_t port_id, struct ethtool_drvinfo *drvinfo); +int rte_ethtool_get_drvinfo(uint16_t port_id, struct ethtool_drvinfo *drvinfo); /** * Retrieve the Ethernet device register length in bytes. @@ -93,7 +93,7 @@ int rte_ethtool_get_drvinfo(uint8_t port_id, struct ethtool_drvinfo *drvinfo); * - (-ENODEV) if *port_id* invalid. * - others depends on the specific operations implementation. */ -int rte_ethtool_get_regs_len(uint8_t port_id); +int rte_ethtool_get_regs_len(uint16_t port_id); /** * Retrieve the Ethernet device register information according to @@ -111,7 +111,7 @@ int rte_ethtool_get_regs_len(uint8_t port_id); * - (-ENODEV) if *port_id* invalid. * - others depends on the specific operations implementation. */ -int rte_ethtool_get_regs(uint8_t port_id, struct ethtool_regs *regs, +int rte_ethtool_get_regs(uint16_t port_id, struct ethtool_regs *regs, void *data); /** @@ -127,7 +127,7 @@ int rte_ethtool_get_regs(uint8_t port_id, struct ethtool_regs *regs, * - (-EINVAL) if parameters invalid. * - others depends on the specific operations implementation. */ -int rte_ethtool_get_link(uint8_t port_id); +int rte_ethtool_get_link(uint16_t port_id); /** * Retrieve the Ethernet device EEPROM size @@ -141,7 +141,7 @@ int rte_ethtool_get_link(uint8_t port_id); * - (-ENODEV) if *port_id* invalid. * - others depends on the specific operations implementation. */ -int rte_ethtool_get_eeprom_len(uint8_t port_id); +int rte_ethtool_get_eeprom_len(uint16_t port_id); /** * Retrieve EEPROM content based upon eeprom range described in ethtool @@ -159,7 +159,7 @@ int rte_ethtool_get_eeprom_len(uint8_t port_id); * - (-ENODEV) if *port_id* invalid. * - others depends on the specific operations implementation. */ -int rte_ethtool_get_eeprom(uint8_t port_id, struct ethtool_eeprom *eeprom, +int rte_ethtool_get_eeprom(uint16_t port_id, struct ethtool_eeprom *eeprom, void *words); /** @@ -179,7 +179,7 @@ int rte_ethtool_get_eeprom(uint8_t port_id, struct ethtool_eeprom *eeprom, * - (-EINVAL) if parameters invalid. * - others depends on the specific operations implementation. */ -int rte_ethtool_set_eeprom(uint8_t port_id, struct ethtool_eeprom *eeprom, +int rte_ethtool_set_eeprom(uint16_t port_id, struct ethtool_eeprom *eeprom, void *words); /** @@ -199,7 +199,7 @@ int rte_ethtool_set_eeprom(uint8_t port_id, struct ethtool_eeprom *eeprom, * - (-EINVAL) if parameters invalid. * - others depends on the specific operations implementation. */ -int rte_ethtool_get_pauseparam(uint8_t port_id, +int rte_ethtool_get_pauseparam(uint16_t port_id, struct ethtool_pauseparam *pause_param); /** @@ -217,7 +217,7 @@ int rte_ethtool_get_pauseparam(uint8_t port_id, * - (-EINVAL) if parameters invalid. * - others depends on the specific operations implementation. */ -int rte_ethtool_set_pauseparam(uint8_t port_id, +int rte_ethtool_set_pauseparam(uint16_t port_id, struct ethtool_pauseparam *param); /** @@ -231,7 +231,7 @@ int rte_ethtool_set_pauseparam(uint8_t port_id, * - (-ENODEV) if *port_id* invalid. * - others depends on the specific operations implementation. */ -int rte_ethtool_net_open(uint8_t port_id); +int rte_ethtool_net_open(uint16_t port_id); /** * Stop the Ethernet device. @@ -242,7 +242,7 @@ int rte_ethtool_net_open(uint8_t port_id); * - (0) if successful. * - (-ENODEV) if *port_id* invalid. */ -int rte_ethtool_net_stop(uint8_t port_id); +int rte_ethtool_net_stop(uint16_t port_id); /** * Get the Ethernet device MAC address. @@ -255,7 +255,7 @@ int rte_ethtool_net_stop(uint8_t port_id); * - (0) if successful. * - (-ENODEV) if *port_id* invalid. */ -int rte_ethtool_net_get_mac_addr(uint8_t port_id, struct ether_addr *addr); +int rte_ethtool_net_get_mac_addr(uint16_t port_id, struct ether_addr *addr); /** * Setting the Ethernet device MAC address. @@ -271,7 +271,7 @@ int rte_ethtool_net_get_mac_addr(uint8_t port_id, struct ether_addr *addr); * - (-EINVAL) if parameters invalid. * - others depends on the specific operations implementation. */ -int rte_ethtool_net_set_mac_addr(uint8_t port_id, struct ether_addr *addr); +int rte_ethtool_net_set_mac_addr(uint16_t port_id, struct ether_addr *addr); /** * Validate if the provided MAC address is valid unicast address @@ -287,7 +287,7 @@ int rte_ethtool_net_set_mac_addr(uint8_t port_id, struct ether_addr *addr); * - (-EINVAL) if parameters invalid. * - others depends on the specific operations implementation. */ -int rte_ethtool_net_validate_addr(uint8_t port_id, struct ether_addr *addr); +int rte_ethtool_net_validate_addr(uint16_t port_id, struct ether_addr *addr); /** * Setting the Ethernet device maximum Tx unit. @@ -303,7 +303,7 @@ int rte_ethtool_net_validate_addr(uint8_t port_id, struct ether_addr *addr); * - (-EINVAL) if parameters invalid. * - others depends on the specific operations implementation. */ -int rte_ethtool_net_change_mtu(uint8_t port_id, int mtu); +int rte_ethtool_net_change_mtu(uint16_t port_id, int mtu); /** * Retrieve the Ethernet device traffic statistics @@ -319,7 +319,7 @@ int rte_ethtool_net_change_mtu(uint8_t port_id, int mtu); * - (-EINVAL) if parameters invalid. * - others depends on the specific operations implementation. */ -int rte_ethtool_net_get_stats64(uint8_t port_id, struct rte_eth_stats *stats); +int rte_ethtool_net_get_stats64(uint16_t port_id, struct rte_eth_stats *stats); /** * Update the Ethernet device VLAN filter with new vid @@ -334,7 +334,7 @@ int rte_ethtool_net_get_stats64(uint8_t port_id, struct rte_eth_stats *stats); * - (-ENODEV) if *port_id* invalid. * - others depends on the specific operations implementation. */ -int rte_ethtool_net_vlan_rx_add_vid(uint8_t port_id, uint16_t vid); +int rte_ethtool_net_vlan_rx_add_vid(uint16_t port_id, uint16_t vid); /** * Remove VLAN id from Ethernet device. @@ -349,7 +349,7 @@ int rte_ethtool_net_vlan_rx_add_vid(uint8_t port_id, uint16_t vid); * - (-ENODEV) if *port_id* invalid. * - others depends on the specific operations implementation. */ -int rte_ethtool_net_vlan_rx_kill_vid(uint8_t port_id, uint16_t vid); +int rte_ethtool_net_vlan_rx_kill_vid(uint16_t port_id, uint16_t vid); /** * Setting the Ethernet device rx mode. @@ -362,10 +362,10 @@ int rte_ethtool_net_vlan_rx_kill_vid(uint8_t port_id, uint16_t vid); * - (-ENODEV) if *port_id* invalid. * - others depends on the specific operations implementation. */ -int rte_ethtool_net_set_rx_mode(uint8_t port_id); +int rte_ethtool_net_set_rx_mode(uint16_t port_id); /** - * Getting ring paramaters for Ethernet device. + * Getting ring parameters for Ethernet device. * * @param port_id * The port identifier of the Ethernet device. @@ -380,11 +380,11 @@ int rte_ethtool_net_set_rx_mode(uint8_t port_id); * Only the tx_pending and rx_pending fields of struct ethtool_ringparam * are used, and the function only gets parameters for queue 0. */ -int rte_ethtool_get_ringparam(uint8_t port_id, +int rte_ethtool_get_ringparam(uint16_t port_id, struct ethtool_ringparam *ring_param); /** - * Setting ring paramaters for Ethernet device. + * Setting ring parameters for Ethernet device. * * @param port_id * The port identifier of the Ethernet device. @@ -399,7 +399,7 @@ int rte_ethtool_get_ringparam(uint8_t port_id, * Only the tx_pending and rx_pending fields of struct ethtool_ringparam * are used, and the function only sets parameters for queue 0. */ -int rte_ethtool_set_ringparam(uint8_t port_id, +int rte_ethtool_set_ringparam(uint16_t port_id, struct ethtool_ringparam *ring_param); diff --git a/dpdk/examples/eventdev_pipeline_sw_pmd/Makefile b/dpdk/examples/eventdev_pipeline_sw_pmd/Makefile new file mode 100644 index 00000000..de4e22c8 --- /dev/null +++ b/dpdk/examples/eventdev_pipeline_sw_pmd/Makefile @@ -0,0 +1,49 @@ +# BSD LICENSE +# +# Copyright(c) 2016-2017 Intel Corporation. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +ifeq ($(RTE_SDK),) +$(error "Please define RTE_SDK environment variable") +endif + +# Default target, can be overridden by command line or environment +RTE_TARGET ?= x86_64-native-linuxapp-gcc + +include $(RTE_SDK)/mk/rte.vars.mk + +# binary name +APP = eventdev_pipeline_sw_pmd + +# all source are stored in SRCS-y +SRCS-y := main.c + +CFLAGS += -O3 +CFLAGS += $(WERROR_FLAGS) + +include $(RTE_SDK)/mk/rte.extapp.mk diff --git a/dpdk/examples/eventdev_pipeline_sw_pmd/main.c b/dpdk/examples/eventdev_pipeline_sw_pmd/main.c new file mode 100644 index 00000000..5f431d87 --- /dev/null +++ b/dpdk/examples/eventdev_pipeline_sw_pmd/main.c @@ -0,0 +1,1023 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define MAX_NUM_STAGES 8 +#define BATCH_SIZE 16 +#define MAX_NUM_CORE 64 + +struct prod_data { + uint8_t dev_id; + uint8_t port_id; + int32_t qid; + unsigned int num_nic_ports; +} __rte_cache_aligned; + +struct cons_data { + uint8_t dev_id; + uint8_t port_id; +} __rte_cache_aligned; + +static struct prod_data prod_data; +static struct cons_data cons_data; + +struct worker_data { + uint8_t dev_id; + uint8_t port_id; +} __rte_cache_aligned; + +struct fastpath_data { + volatile int done; + uint32_t rx_lock; + uint32_t tx_lock; + uint32_t sched_lock; + uint32_t evdev_service_id; + bool rx_single; + bool tx_single; + bool sched_single; + unsigned int rx_core[MAX_NUM_CORE]; + unsigned int tx_core[MAX_NUM_CORE]; + unsigned int sched_core[MAX_NUM_CORE]; + unsigned int worker_core[MAX_NUM_CORE]; + struct rte_eth_dev_tx_buffer *tx_buf[RTE_MAX_ETHPORTS]; +}; + +static struct fastpath_data *fdata; + +struct config_data { + unsigned int active_cores; + unsigned int num_workers; + int64_t num_packets; + unsigned int num_fids; + int queue_type; + int worker_cycles; + int enable_queue_priorities; + int quiet; + int dump_dev; + int dump_dev_signal; + unsigned int num_stages; + unsigned int worker_cq_depth; + int16_t next_qid[MAX_NUM_STAGES+2]; + int16_t qid[MAX_NUM_STAGES]; +}; + +static struct config_data cdata = { + .num_packets = (1L << 25), /* do ~32M packets */ + .num_fids = 512, + .queue_type = RTE_SCHED_TYPE_ATOMIC, + .next_qid = {-1}, + .qid = {-1}, + .num_stages = 1, + .worker_cq_depth = 16 +}; + +static bool +core_in_use(unsigned int lcore_id) { + return (fdata->rx_core[lcore_id] || fdata->sched_core[lcore_id] || + fdata->tx_core[lcore_id] || fdata->worker_core[lcore_id]); +} + +static void +eth_tx_buffer_retry(struct rte_mbuf **pkts, uint16_t unsent, + void *userdata) +{ + int port_id = (uintptr_t) userdata; + unsigned int _sent = 0; + + do { + /* Note: hard-coded TX queue */ + _sent += rte_eth_tx_burst(port_id, 0, &pkts[_sent], + unsent - _sent); + } while (_sent != unsent); +} + +static int +consumer(void) +{ + const uint64_t freq_khz = rte_get_timer_hz() / 1000; + struct rte_event packets[BATCH_SIZE]; + + static uint64_t received; + static uint64_t last_pkts; + static uint64_t last_time; + static uint64_t start_time; + unsigned int i, j; + uint8_t dev_id = cons_data.dev_id; + uint8_t port_id = cons_data.port_id; + + uint16_t n = rte_event_dequeue_burst(dev_id, port_id, + packets, RTE_DIM(packets), 0); + + if (n == 0) { + for (j = 0; j < rte_eth_dev_count(); j++) + rte_eth_tx_buffer_flush(j, 0, fdata->tx_buf[j]); + return 0; + } + if (start_time == 0) + last_time = start_time = rte_get_timer_cycles(); + + received += n; + for (i = 0; i < n; i++) { + uint8_t outport = packets[i].mbuf->port; + rte_eth_tx_buffer(outport, 0, fdata->tx_buf[outport], + packets[i].mbuf); + } + + /* Print out mpps every 1<22 packets */ + if (!cdata.quiet && received >= last_pkts + (1<<22)) { + const uint64_t now = rte_get_timer_cycles(); + const uint64_t total_ms = (now - start_time) / freq_khz; + const uint64_t delta_ms = (now - last_time) / freq_khz; + uint64_t delta_pkts = received - last_pkts; + + printf("# consumer RX=%"PRIu64", time %"PRIu64 "ms, " + "avg %.3f mpps [current %.3f mpps]\n", + received, + total_ms, + received / (total_ms * 1000.0), + delta_pkts / (delta_ms * 1000.0)); + last_pkts = received; + last_time = now; + } + + cdata.num_packets -= n; + if (cdata.num_packets <= 0) + fdata->done = 1; + + return 0; +} + +static int +producer(void) +{ + static uint8_t eth_port; + struct rte_mbuf *mbufs[BATCH_SIZE+2]; + struct rte_event ev[BATCH_SIZE+2]; + uint32_t i, num_ports = prod_data.num_nic_ports; + int32_t qid = prod_data.qid; + uint8_t dev_id = prod_data.dev_id; + uint8_t port_id = prod_data.port_id; + uint32_t prio_idx = 0; + + const uint16_t nb_rx = rte_eth_rx_burst(eth_port, 0, mbufs, BATCH_SIZE); + if (++eth_port == num_ports) + eth_port = 0; + if (nb_rx == 0) { + rte_pause(); + return 0; + } + + for (i = 0; i < nb_rx; i++) { + ev[i].flow_id = mbufs[i]->hash.rss; + ev[i].op = RTE_EVENT_OP_NEW; + ev[i].sched_type = cdata.queue_type; + ev[i].queue_id = qid; + ev[i].event_type = RTE_EVENT_TYPE_ETHDEV; + ev[i].sub_event_type = 0; + ev[i].priority = RTE_EVENT_DEV_PRIORITY_NORMAL; + ev[i].mbuf = mbufs[i]; + RTE_SET_USED(prio_idx); + } + + const int nb_tx = rte_event_enqueue_burst(dev_id, port_id, ev, nb_rx); + if (nb_tx != nb_rx) { + for (i = nb_tx; i < nb_rx; i++) + rte_pktmbuf_free(mbufs[i]); + } + + return 0; +} + +static inline void +schedule_devices(unsigned int lcore_id) +{ + if (fdata->rx_core[lcore_id] && (fdata->rx_single || + rte_atomic32_cmpset(&(fdata->rx_lock), 0, 1))) { + producer(); + rte_atomic32_clear((rte_atomic32_t *)&(fdata->rx_lock)); + } + + if (fdata->sched_core[lcore_id] && (fdata->sched_single || + rte_atomic32_cmpset(&(fdata->sched_lock), 0, 1))) { + rte_service_run_iter_on_app_lcore(fdata->evdev_service_id, 1); + if (cdata.dump_dev_signal) { + rte_event_dev_dump(0, stdout); + cdata.dump_dev_signal = 0; + } + rte_atomic32_clear((rte_atomic32_t *)&(fdata->sched_lock)); + } + + if (fdata->tx_core[lcore_id] && (fdata->tx_single || + rte_atomic32_cmpset(&(fdata->tx_lock), 0, 1))) { + consumer(); + rte_atomic32_clear((rte_atomic32_t *)&(fdata->tx_lock)); + } +} + +static inline void +work(struct rte_mbuf *m) +{ + struct ether_hdr *eth; + struct ether_addr addr; + + /* change mac addresses on packet (to use mbuf data) */ + /* + * FIXME Swap mac address properly and also handle the + * case for both odd and even number of stages that the + * addresses end up the same at the end of the pipeline + */ + eth = rte_pktmbuf_mtod(m, struct ether_hdr *); + ether_addr_copy(ð->d_addr, &addr); + ether_addr_copy(&addr, ð->d_addr); + + /* do a number of cycles of work per packet */ + volatile uint64_t start_tsc = rte_rdtsc(); + while (rte_rdtsc() < start_tsc + cdata.worker_cycles) + rte_pause(); +} + +static int +worker(void *arg) +{ + struct rte_event events[BATCH_SIZE]; + + struct worker_data *data = (struct worker_data *)arg; + uint8_t dev_id = data->dev_id; + uint8_t port_id = data->port_id; + size_t sent = 0, received = 0; + unsigned int lcore_id = rte_lcore_id(); + + while (!fdata->done) { + uint16_t i; + + schedule_devices(lcore_id); + + if (!fdata->worker_core[lcore_id]) { + rte_pause(); + continue; + } + + const uint16_t nb_rx = rte_event_dequeue_burst(dev_id, port_id, + events, RTE_DIM(events), 0); + + if (nb_rx == 0) { + rte_pause(); + continue; + } + received += nb_rx; + + for (i = 0; i < nb_rx; i++) { + + /* The first worker stage does classification */ + if (events[i].queue_id == cdata.qid[0]) + events[i].flow_id = events[i].mbuf->hash.rss + % cdata.num_fids; + + events[i].queue_id = cdata.next_qid[events[i].queue_id]; + events[i].op = RTE_EVENT_OP_FORWARD; + events[i].sched_type = cdata.queue_type; + + work(events[i].mbuf); + } + uint16_t nb_tx = rte_event_enqueue_burst(dev_id, port_id, + events, nb_rx); + while (nb_tx < nb_rx && !fdata->done) + nb_tx += rte_event_enqueue_burst(dev_id, port_id, + events + nb_tx, + nb_rx - nb_tx); + sent += nb_tx; + } + + if (!cdata.quiet) + printf(" worker %u thread done. RX=%zu TX=%zu\n", + rte_lcore_id(), received, sent); + + return 0; +} + +/* + * Parse the coremask given as argument (hexadecimal string) and fill + * the global configuration (core role and core count) with the parsed + * value. + */ +static int xdigit2val(unsigned char c) +{ + int val; + + if (isdigit(c)) + val = c - '0'; + else if (isupper(c)) + val = c - 'A' + 10; + else + val = c - 'a' + 10; + return val; +} + +static uint64_t +parse_coremask(const char *coremask) +{ + int i, j, idx = 0; + unsigned int count = 0; + char c; + int val; + uint64_t mask = 0; + const int32_t BITS_HEX = 4; + + if (coremask == NULL) + return -1; + /* Remove all blank characters ahead and after . + * Remove 0x/0X if exists. + */ + while (isblank(*coremask)) + coremask++; + if (coremask[0] == '0' && ((coremask[1] == 'x') + || (coremask[1] == 'X'))) + coremask += 2; + i = strlen(coremask); + while ((i > 0) && isblank(coremask[i - 1])) + i--; + if (i == 0) + return -1; + + for (i = i - 1; i >= 0 && idx < MAX_NUM_CORE; i--) { + c = coremask[i]; + if (isxdigit(c) == 0) { + /* invalid characters */ + return -1; + } + val = xdigit2val(c); + for (j = 0; j < BITS_HEX && idx < MAX_NUM_CORE; j++, idx++) { + if ((1 << j) & val) { + mask |= (1UL << idx); + count++; + } + } + } + for (; i >= 0; i--) + if (coremask[i] != '0') + return -1; + if (count == 0) + return -1; + return mask; +} + +static struct option long_options[] = { + {"workers", required_argument, 0, 'w'}, + {"packets", required_argument, 0, 'n'}, + {"atomic-flows", required_argument, 0, 'f'}, + {"num_stages", required_argument, 0, 's'}, + {"rx-mask", required_argument, 0, 'r'}, + {"tx-mask", required_argument, 0, 't'}, + {"sched-mask", required_argument, 0, 'e'}, + {"cq-depth", required_argument, 0, 'c'}, + {"work-cycles", required_argument, 0, 'W'}, + {"queue-priority", no_argument, 0, 'P'}, + {"parallel", no_argument, 0, 'p'}, + {"ordered", no_argument, 0, 'o'}, + {"quiet", no_argument, 0, 'q'}, + {"dump", no_argument, 0, 'D'}, + {0, 0, 0, 0} +}; + +static void +usage(void) +{ + const char *usage_str = + " Usage: eventdev_demo [options]\n" + " Options:\n" + " -n, --packets=N Send N packets (default ~32M), 0 implies no limit\n" + " -f, --atomic-flows=N Use N random flows from 1 to N (default 16)\n" + " -s, --num_stages=N Use N atomic stages (default 1)\n" + " -r, --rx-mask=core mask Run NIC rx on CPUs in core mask\n" + " -w, --worker-mask=core mask Run worker on CPUs in core mask\n" + " -t, --tx-mask=core mask Run NIC tx on CPUs in core mask\n" + " -e --sched-mask=core mask Run scheduler on CPUs in core mask\n" + " -c --cq-depth=N Worker CQ depth (default 16)\n" + " -W --work-cycles=N Worker cycles (default 0)\n" + " -P --queue-priority Enable scheduler queue prioritization\n" + " -o, --ordered Use ordered scheduling\n" + " -p, --parallel Use parallel scheduling\n" + " -q, --quiet Minimize printed output\n" + " -D, --dump Print detailed statistics before exit" + "\n"; + fprintf(stderr, "%s", usage_str); + exit(1); +} + +static void +parse_app_args(int argc, char **argv) +{ + /* Parse cli options*/ + int option_index; + int c; + opterr = 0; + uint64_t rx_lcore_mask = 0; + uint64_t tx_lcore_mask = 0; + uint64_t sched_lcore_mask = 0; + uint64_t worker_lcore_mask = 0; + int i; + + for (;;) { + c = getopt_long(argc, argv, "r:t:e:c:w:n:f:s:poPqDW:", + long_options, &option_index); + if (c == -1) + break; + + int popcnt = 0; + switch (c) { + case 'n': + cdata.num_packets = (int64_t)atol(optarg); + if (cdata.num_packets == 0) + cdata.num_packets = INT64_MAX; + break; + case 'f': + cdata.num_fids = (unsigned int)atoi(optarg); + break; + case 's': + cdata.num_stages = (unsigned int)atoi(optarg); + break; + case 'c': + cdata.worker_cq_depth = (unsigned int)atoi(optarg); + break; + case 'W': + cdata.worker_cycles = (unsigned int)atoi(optarg); + break; + case 'P': + cdata.enable_queue_priorities = 1; + break; + case 'o': + cdata.queue_type = RTE_SCHED_TYPE_ORDERED; + break; + case 'p': + cdata.queue_type = RTE_SCHED_TYPE_PARALLEL; + break; + case 'q': + cdata.quiet = 1; + break; + case 'D': + cdata.dump_dev = 1; + break; + case 'w': + worker_lcore_mask = parse_coremask(optarg); + break; + case 'r': + rx_lcore_mask = parse_coremask(optarg); + popcnt = __builtin_popcountll(rx_lcore_mask); + fdata->rx_single = (popcnt == 1); + break; + case 't': + tx_lcore_mask = parse_coremask(optarg); + popcnt = __builtin_popcountll(tx_lcore_mask); + fdata->tx_single = (popcnt == 1); + break; + case 'e': + sched_lcore_mask = parse_coremask(optarg); + popcnt = __builtin_popcountll(sched_lcore_mask); + fdata->sched_single = (popcnt == 1); + break; + default: + usage(); + } + } + + if (worker_lcore_mask == 0 || rx_lcore_mask == 0 || + sched_lcore_mask == 0 || tx_lcore_mask == 0) { + printf("Core part of pipeline was not assigned any cores. " + "This will stall the pipeline, please check core masks " + "(use -h for details on setting core masks):\n" + "\trx: %"PRIu64"\n\ttx: %"PRIu64"\n\tsched: %"PRIu64 + "\n\tworkers: %"PRIu64"\n", + rx_lcore_mask, tx_lcore_mask, sched_lcore_mask, + worker_lcore_mask); + rte_exit(-1, "Fix core masks\n"); + } + if (cdata.num_stages == 0 || cdata.num_stages > MAX_NUM_STAGES) + usage(); + + for (i = 0; i < MAX_NUM_CORE; i++) { + fdata->rx_core[i] = !!(rx_lcore_mask & (1UL << i)); + fdata->tx_core[i] = !!(tx_lcore_mask & (1UL << i)); + fdata->sched_core[i] = !!(sched_lcore_mask & (1UL << i)); + fdata->worker_core[i] = !!(worker_lcore_mask & (1UL << i)); + + if (fdata->worker_core[i]) + cdata.num_workers++; + if (core_in_use(i)) + cdata.active_cores++; + } +} + +/* + * Initializes a given port using global settings and with the RX buffers + * coming from the mbuf_pool passed as a parameter. + */ +static inline int +port_init(uint8_t port, struct rte_mempool *mbuf_pool) +{ + static const struct rte_eth_conf port_conf_default = { + .rxmode = { + .mq_mode = ETH_MQ_RX_RSS, + .max_rx_pkt_len = ETHER_MAX_LEN + }, + .rx_adv_conf = { + .rss_conf = { + .rss_hf = ETH_RSS_IP | + ETH_RSS_TCP | + ETH_RSS_UDP, + } + } + }; + const uint16_t rx_rings = 1, tx_rings = 1; + const uint16_t rx_ring_size = 512, tx_ring_size = 512; + struct rte_eth_conf port_conf = port_conf_default; + int retval; + uint16_t q; + + if (port >= rte_eth_dev_count()) + return -1; + + /* Configure the Ethernet device. */ + retval = rte_eth_dev_configure(port, rx_rings, tx_rings, &port_conf); + if (retval != 0) + return retval; + + /* Allocate and set up 1 RX queue per Ethernet port. */ + for (q = 0; q < rx_rings; q++) { + retval = rte_eth_rx_queue_setup(port, q, rx_ring_size, + rte_eth_dev_socket_id(port), NULL, mbuf_pool); + if (retval < 0) + return retval; + } + + /* Allocate and set up 1 TX queue per Ethernet port. */ + for (q = 0; q < tx_rings; q++) { + retval = rte_eth_tx_queue_setup(port, q, tx_ring_size, + rte_eth_dev_socket_id(port), NULL); + if (retval < 0) + return retval; + } + + /* Start the Ethernet port. */ + retval = rte_eth_dev_start(port); + if (retval < 0) + return retval; + + /* Display the port MAC address. */ + struct ether_addr addr; + rte_eth_macaddr_get(port, &addr); + printf("Port %u MAC: %02" PRIx8 " %02" PRIx8 " %02" PRIx8 + " %02" PRIx8 " %02" PRIx8 " %02" PRIx8 "\n", + (unsigned int)port, + addr.addr_bytes[0], addr.addr_bytes[1], + addr.addr_bytes[2], addr.addr_bytes[3], + addr.addr_bytes[4], addr.addr_bytes[5]); + + /* Enable RX in promiscuous mode for the Ethernet device. */ + rte_eth_promiscuous_enable(port); + + return 0; +} + +static int +init_ports(unsigned int num_ports) +{ + uint8_t portid; + unsigned int i; + + struct rte_mempool *mp = rte_pktmbuf_pool_create("packet_pool", + /* mbufs */ 16384 * num_ports, + /* cache_size */ 512, + /* priv_size*/ 0, + /* data_room_size */ RTE_MBUF_DEFAULT_BUF_SIZE, + rte_socket_id()); + + for (portid = 0; portid < num_ports; portid++) + if (port_init(portid, mp) != 0) + rte_exit(EXIT_FAILURE, "Cannot init port %"PRIu8 "\n", + portid); + + for (i = 0; i < num_ports; i++) { + void *userdata = (void *)(uintptr_t) i; + fdata->tx_buf[i] = + rte_malloc(NULL, RTE_ETH_TX_BUFFER_SIZE(32), 0); + if (fdata->tx_buf[i] == NULL) + rte_panic("Out of memory\n"); + rte_eth_tx_buffer_init(fdata->tx_buf[i], 32); + rte_eth_tx_buffer_set_err_callback(fdata->tx_buf[i], + eth_tx_buffer_retry, + userdata); + } + + return 0; +} + +struct port_link { + uint8_t queue_id; + uint8_t priority; +}; + +static int +setup_eventdev(struct prod_data *prod_data, + struct cons_data *cons_data, + struct worker_data *worker_data) +{ + const uint8_t dev_id = 0; + /* +1 stages is for a SINGLE_LINK TX stage */ + const uint8_t nb_queues = cdata.num_stages + 1; + /* + 2 is one port for producer and one for consumer */ + const uint8_t nb_ports = cdata.num_workers + 2; + struct rte_event_dev_config config = { + .nb_event_queues = nb_queues, + .nb_event_ports = nb_ports, + .nb_events_limit = 4096, + .nb_event_queue_flows = 1024, + .nb_event_port_dequeue_depth = 128, + .nb_event_port_enqueue_depth = 128, + }; + struct rte_event_port_conf wkr_p_conf = { + .dequeue_depth = cdata.worker_cq_depth, + .enqueue_depth = 64, + .new_event_threshold = 4096, + }; + struct rte_event_queue_conf wkr_q_conf = { + .schedule_type = cdata.queue_type, + .priority = RTE_EVENT_DEV_PRIORITY_NORMAL, + .nb_atomic_flows = 1024, + .nb_atomic_order_sequences = 1024, + }; + struct rte_event_port_conf tx_p_conf = { + .dequeue_depth = 128, + .enqueue_depth = 128, + .new_event_threshold = 4096, + }; + const struct rte_event_queue_conf tx_q_conf = { + .priority = RTE_EVENT_DEV_PRIORITY_HIGHEST, + .event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK, + }; + + struct port_link worker_queues[MAX_NUM_STAGES]; + struct port_link tx_queue; + unsigned int i; + + int ret, ndev = rte_event_dev_count(); + if (ndev < 1) { + printf("%d: No Eventdev Devices Found\n", __LINE__); + return -1; + } + + struct rte_event_dev_info dev_info; + ret = rte_event_dev_info_get(dev_id, &dev_info); + printf("\tEventdev %d: %s\n", dev_id, dev_info.driver_name); + + if (dev_info.max_event_port_dequeue_depth < + config.nb_event_port_dequeue_depth) + config.nb_event_port_dequeue_depth = + dev_info.max_event_port_dequeue_depth; + if (dev_info.max_event_port_enqueue_depth < + config.nb_event_port_enqueue_depth) + config.nb_event_port_enqueue_depth = + dev_info.max_event_port_enqueue_depth; + + ret = rte_event_dev_configure(dev_id, &config); + if (ret < 0) { + printf("%d: Error configuring device\n", __LINE__); + return -1; + } + + /* Q creation - one load balanced per pipeline stage*/ + printf(" Stages:\n"); + for (i = 0; i < cdata.num_stages; i++) { + if (rte_event_queue_setup(dev_id, i, &wkr_q_conf) < 0) { + printf("%d: error creating qid %d\n", __LINE__, i); + return -1; + } + cdata.qid[i] = i; + cdata.next_qid[i] = i+1; + worker_queues[i].queue_id = i; + if (cdata.enable_queue_priorities) { + /* calculate priority stepping for each stage, leaving + * headroom of 1 for the SINGLE_LINK TX below + */ + const uint32_t prio_delta = + (RTE_EVENT_DEV_PRIORITY_LOWEST-1) / nb_queues; + + /* higher priority for queues closer to tx */ + wkr_q_conf.priority = + RTE_EVENT_DEV_PRIORITY_LOWEST - prio_delta * i; + } + + const char *type_str = "Atomic"; + switch (wkr_q_conf.schedule_type) { + case RTE_SCHED_TYPE_ORDERED: + type_str = "Ordered"; + break; + case RTE_SCHED_TYPE_PARALLEL: + type_str = "Parallel"; + break; + } + printf("\tStage %d, Type %s\tPriority = %d\n", i, type_str, + wkr_q_conf.priority); + } + printf("\n"); + + /* final queue for sending to TX core */ + if (rte_event_queue_setup(dev_id, i, &tx_q_conf) < 0) { + printf("%d: error creating qid %d\n", __LINE__, i); + return -1; + } + tx_queue.queue_id = i; + tx_queue.priority = RTE_EVENT_DEV_PRIORITY_HIGHEST; + + if (wkr_p_conf.dequeue_depth > config.nb_event_port_dequeue_depth) + wkr_p_conf.dequeue_depth = config.nb_event_port_dequeue_depth; + if (wkr_p_conf.enqueue_depth > config.nb_event_port_enqueue_depth) + wkr_p_conf.enqueue_depth = config.nb_event_port_enqueue_depth; + + /* set up one port per worker, linking to all stage queues */ + for (i = 0; i < cdata.num_workers; i++) { + struct worker_data *w = &worker_data[i]; + w->dev_id = dev_id; + if (rte_event_port_setup(dev_id, i, &wkr_p_conf) < 0) { + printf("Error setting up port %d\n", i); + return -1; + } + + uint32_t s; + for (s = 0; s < cdata.num_stages; s++) { + if (rte_event_port_link(dev_id, i, + &worker_queues[s].queue_id, + &worker_queues[s].priority, + 1) != 1) { + printf("%d: error creating link for port %d\n", + __LINE__, i); + return -1; + } + } + w->port_id = i; + } + + if (tx_p_conf.dequeue_depth > config.nb_event_port_dequeue_depth) + tx_p_conf.dequeue_depth = config.nb_event_port_dequeue_depth; + if (tx_p_conf.enqueue_depth > config.nb_event_port_enqueue_depth) + tx_p_conf.enqueue_depth = config.nb_event_port_enqueue_depth; + + /* port for consumer, linked to TX queue */ + if (rte_event_port_setup(dev_id, i, &tx_p_conf) < 0) { + printf("Error setting up port %d\n", i); + return -1; + } + if (rte_event_port_link(dev_id, i, &tx_queue.queue_id, + &tx_queue.priority, 1) != 1) { + printf("%d: error creating link for port %d\n", + __LINE__, i); + return -1; + } + /* port for producer, no links */ + struct rte_event_port_conf rx_p_conf = { + .dequeue_depth = 8, + .enqueue_depth = 8, + .new_event_threshold = 1200, + }; + + if (rx_p_conf.dequeue_depth > config.nb_event_port_dequeue_depth) + rx_p_conf.dequeue_depth = config.nb_event_port_dequeue_depth; + if (rx_p_conf.enqueue_depth > config.nb_event_port_enqueue_depth) + rx_p_conf.enqueue_depth = config.nb_event_port_enqueue_depth; + + if (rte_event_port_setup(dev_id, i + 1, &rx_p_conf) < 0) { + printf("Error setting up port %d\n", i); + return -1; + } + + *prod_data = (struct prod_data){.dev_id = dev_id, + .port_id = i + 1, + .qid = cdata.qid[0] }; + *cons_data = (struct cons_data){.dev_id = dev_id, + .port_id = i }; + + ret = rte_event_dev_service_id_get(dev_id, + &fdata->evdev_service_id); + if (ret != -ESRCH && ret != 0) { + printf("Error getting the service ID for sw eventdev\n"); + return -1; + } + rte_service_runstate_set(fdata->evdev_service_id, 1); + rte_service_set_runstate_mapped_check(fdata->evdev_service_id, 0); + if (rte_event_dev_start(dev_id) < 0) { + printf("Error starting eventdev\n"); + return -1; + } + + return dev_id; +} + +static void +signal_handler(int signum) +{ + if (fdata->done) + rte_exit(1, "Exiting on signal %d\n", signum); + if (signum == SIGINT || signum == SIGTERM) { + printf("\n\nSignal %d received, preparing to exit...\n", + signum); + fdata->done = 1; + } + if (signum == SIGTSTP) + rte_event_dev_dump(0, stdout); +} + +static inline uint64_t +port_stat(int dev_id, int32_t p) +{ + char statname[64]; + snprintf(statname, sizeof(statname), "port_%u_rx", p); + return rte_event_dev_xstats_by_name_get(dev_id, statname, NULL); +} + +int +main(int argc, char **argv) +{ + struct worker_data *worker_data; + unsigned int num_ports; + int lcore_id; + int err; + + signal(SIGINT, signal_handler); + signal(SIGTERM, signal_handler); + signal(SIGTSTP, signal_handler); + + err = rte_eal_init(argc, argv); + if (err < 0) + rte_panic("Invalid EAL arguments\n"); + + argc -= err; + argv += err; + + fdata = rte_malloc(NULL, sizeof(struct fastpath_data), 0); + if (fdata == NULL) + rte_panic("Out of memory\n"); + + /* Parse cli options*/ + parse_app_args(argc, argv); + + num_ports = rte_eth_dev_count(); + if (num_ports == 0) + rte_panic("No ethernet ports found\n"); + + const unsigned int cores_needed = cdata.active_cores; + + if (!cdata.quiet) { + printf(" Config:\n"); + printf("\tports: %u\n", num_ports); + printf("\tworkers: %u\n", cdata.num_workers); + printf("\tpackets: %"PRIi64"\n", cdata.num_packets); + printf("\tQueue-prio: %u\n", cdata.enable_queue_priorities); + if (cdata.queue_type == RTE_SCHED_TYPE_ORDERED) + printf("\tqid0 type: ordered\n"); + if (cdata.queue_type == RTE_SCHED_TYPE_ATOMIC) + printf("\tqid0 type: atomic\n"); + printf("\tCores available: %u\n", rte_lcore_count()); + printf("\tCores used: %u\n", cores_needed); + } + + if (rte_lcore_count() < cores_needed) + rte_panic("Too few cores (%d < %d)\n", rte_lcore_count(), + cores_needed); + + const unsigned int ndevs = rte_event_dev_count(); + if (ndevs == 0) + rte_panic("No dev_id devs found. Pasl in a --vdev eventdev.\n"); + if (ndevs > 1) + fprintf(stderr, "Warning: More than one eventdev, using idx 0"); + + worker_data = rte_calloc(0, cdata.num_workers, + sizeof(worker_data[0]), 0); + if (worker_data == NULL) + rte_panic("rte_calloc failed\n"); + + int dev_id = setup_eventdev(&prod_data, &cons_data, worker_data); + if (dev_id < 0) + rte_exit(EXIT_FAILURE, "Error setting up eventdev\n"); + + prod_data.num_nic_ports = num_ports; + init_ports(num_ports); + + int worker_idx = 0; + RTE_LCORE_FOREACH_SLAVE(lcore_id) { + if (lcore_id >= MAX_NUM_CORE) + break; + + if (!fdata->rx_core[lcore_id] && + !fdata->worker_core[lcore_id] && + !fdata->tx_core[lcore_id] && + !fdata->sched_core[lcore_id]) + continue; + + if (fdata->rx_core[lcore_id]) + printf( + "[%s()] lcore %d executing NIC Rx, and using eventdev port %u\n", + __func__, lcore_id, prod_data.port_id); + + if (fdata->tx_core[lcore_id]) + printf( + "[%s()] lcore %d executing NIC Tx, and using eventdev port %u\n", + __func__, lcore_id, cons_data.port_id); + + if (fdata->sched_core[lcore_id]) + printf("[%s()] lcore %d executing scheduler\n", + __func__, lcore_id); + + if (fdata->worker_core[lcore_id]) + printf( + "[%s()] lcore %d executing worker, using eventdev port %u\n", + __func__, lcore_id, + worker_data[worker_idx].port_id); + + err = rte_eal_remote_launch(worker, &worker_data[worker_idx], + lcore_id); + if (err) { + rte_panic("Failed to launch worker on core %d\n", + lcore_id); + continue; + } + if (fdata->worker_core[lcore_id]) + worker_idx++; + } + + lcore_id = rte_lcore_id(); + + if (core_in_use(lcore_id)) + worker(&worker_data[worker_idx++]); + + rte_eal_mp_wait_lcore(); + + if (cdata.dump_dev) + rte_event_dev_dump(dev_id, stdout); + + if (!cdata.quiet && (port_stat(dev_id, worker_data[0].port_id) != + (uint64_t)-ENOTSUP)) { + printf("\nPort Workload distribution:\n"); + uint32_t i; + uint64_t tot_pkts = 0; + uint64_t pkts_per_wkr[RTE_MAX_LCORE] = {0}; + for (i = 0; i < cdata.num_workers; i++) { + pkts_per_wkr[i] = + port_stat(dev_id, worker_data[i].port_id); + tot_pkts += pkts_per_wkr[i]; + } + for (i = 0; i < cdata.num_workers; i++) { + float pc = pkts_per_wkr[i] * 100 / + ((float)tot_pkts); + printf("worker %i :\t%.1f %% (%"PRIu64" pkts)\n", + i, pc, pkts_per_wkr[i]); + } + + } + + return 0; +} diff --git a/dpdk/examples/exception_path/Makefile b/dpdk/examples/exception_path/Makefile index 959914a2..d16f74f6 100644 --- a/dpdk/examples/exception_path/Makefile +++ b/dpdk/examples/exception_path/Makefile @@ -33,17 +33,11 @@ ifeq ($(RTE_SDK),) $(error "Please define RTE_SDK environment variable") endif -# Default target, can be overriden by command line or environment +# Default target, can be overridden by command line or environment RTE_TARGET ?= x86_64-native-linuxapp-gcc include $(RTE_SDK)/mk/rte.vars.mk -ifneq ($(CONFIG_RTE_EXEC_ENV),"linuxapp") -$(info This application can only operate in a linuxapp environment, \ -please change the definition of the RTE_TARGET environment variable) -all: -else - # binary name APP = exception_path @@ -54,5 +48,3 @@ CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) include $(RTE_SDK)/mk/rte.extapp.mk - -endif diff --git a/dpdk/examples/exception_path/main.c b/dpdk/examples/exception_path/main.c index e5eedcc1..f8f5bbdf 100644 --- a/dpdk/examples/exception_path/main.c +++ b/dpdk/examples/exception_path/main.c @@ -42,8 +42,10 @@ #include #include -#include +#include +#ifdef RTE_EXEC_ENV_LINUXAPP #include +#endif #include #include #include @@ -53,7 +55,6 @@ #include #include #include -#include #include #include #include @@ -61,12 +62,9 @@ #include #include #include -#include #include #include #include -#include -#include #include #include #include @@ -115,7 +113,7 @@ static const struct rte_eth_conf port_conf = { .hw_ip_checksum = 0, /* IP checksum offload disabled */ .hw_vlan_filter = 0, /* VLAN filtering disabled */ .jumbo_frame = 0, /* Jumbo Frame Support disabled */ - .hw_strip_crc = 0, /* CRC stripped by hardware */ + .hw_strip_crc = 1, /* CRC stripped by hardware */ }, .txmode = { .mq_mode = ETH_MQ_TX_NONE, @@ -135,7 +133,7 @@ static uint64_t input_cores_mask = 0; static uint64_t output_cores_mask = 0; /* Array storing port_id that is associated with each lcore */ -static uint8_t port_ids[RTE_MAX_LCORE]; +static uint16_t port_ids[RTE_MAX_LCORE]; /* Structure type for recording lcore-specific stats */ struct stats { @@ -183,6 +181,7 @@ signal_handler(int signum) } } +#ifdef RTE_EXEC_ENV_LINUXAPP /* * Create a tap network interface, or use existing one with same name. * If name[0]='\0' then a name is automatically assigned and returned in name. @@ -215,6 +214,29 @@ static int tap_create(char *name) return fd; } +#else +/* + * Find a free tap network interface, or create a new one. + * The name is automatically assigned and returned in name. + */ +static int tap_create(char *name) +{ + int i, fd = -1; + char devname[PATH_MAX]; + + for (i = 0; i < 255; i++) { + snprintf(devname, sizeof(devname), "/dev/tap%d", i); + fd = open(devname, O_RDWR); + if (fd >= 0 || errno != EBUSY) + break; + } + + if (name) + snprintf(name, IFNAMSIZ, "tap%d", i); + + return fd; +} +#endif /* Main processing loop */ static int @@ -336,8 +358,8 @@ static void setup_port_lcore_affinities(void) { unsigned long i; - uint8_t tx_port = 0; - uint8_t rx_port = 0; + uint16_t tx_port = 0; + uint16_t rx_port = 0; /* Setup port_ids[] array, and check masks were ok */ RTE_LCORE_FOREACH(i) { @@ -420,45 +442,54 @@ parse_args(int argc, char **argv) /* Initialise a single port on an Ethernet device */ static void -init_port(uint8_t port) +init_port(uint16_t port) { int ret; + uint16_t nb_rxd = NB_RXD; + uint16_t nb_txd = NB_TXD; /* Initialise device and RX/TX queues */ - PRINT_INFO("Initialising port %u ...", (unsigned)port); + PRINT_INFO("Initialising port %u ...", port); fflush(stdout); ret = rte_eth_dev_configure(port, 1, 1, &port_conf); if (ret < 0) - FATAL_ERROR("Could not configure port%u (%d)", - (unsigned)port, ret); + FATAL_ERROR("Could not configure port%u (%d)", port, ret); - ret = rte_eth_rx_queue_setup(port, 0, NB_RXD, rte_eth_dev_socket_id(port), + ret = rte_eth_dev_adjust_nb_rx_tx_desc(port, &nb_rxd, &nb_txd); + if (ret < 0) + FATAL_ERROR("Could not adjust number of descriptors for port%u (%d)", + port, ret); + + ret = rte_eth_rx_queue_setup(port, 0, nb_rxd, + rte_eth_dev_socket_id(port), NULL, pktmbuf_pool); if (ret < 0) FATAL_ERROR("Could not setup up RX queue for port%u (%d)", - (unsigned)port, ret); + port, ret); - ret = rte_eth_tx_queue_setup(port, 0, NB_TXD, rte_eth_dev_socket_id(port), + ret = rte_eth_tx_queue_setup(port, 0, nb_txd, + rte_eth_dev_socket_id(port), NULL); if (ret < 0) FATAL_ERROR("Could not setup up TX queue for port%u (%d)", - (unsigned)port, ret); + port, ret); ret = rte_eth_dev_start(port); if (ret < 0) - FATAL_ERROR("Could not start port%u (%d)", (unsigned)port, ret); + FATAL_ERROR("Could not start port%u (%d)", port, ret); rte_eth_promiscuous_enable(port); } /* Check the link status of all ports in up to 9s, and print them finally */ static void -check_all_ports_link_status(uint8_t port_num, uint32_t port_mask) +check_all_ports_link_status(uint16_t port_num, uint32_t port_mask) { #define CHECK_INTERVAL 100 /* 100ms */ #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ - uint8_t portid, count, all_ports_up, print_flag = 0; + uint16_t portid; + uint8_t count, all_ports_up, print_flag = 0; struct rte_eth_link link; printf("\nChecking link status"); @@ -473,14 +504,13 @@ check_all_ports_link_status(uint8_t port_num, uint32_t port_mask) /* print link status if flag set */ if (print_flag == 1) { if (link.link_status) - printf("Port %d Link Up - speed %u " - "Mbps - %s\n", (uint8_t)portid, - (unsigned)link.link_speed, + printf( + "Port%d Link Up. Speed %u Mbps - %s\n", + portid, link.link_speed, (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? ("full-duplex") : ("half-duplex\n")); else - printf("Port %d Link Down\n", - (uint8_t)portid); + printf("Port %d Link Down\n", portid); continue; } /* clear all_ports_up flag if any link down */ @@ -513,7 +543,7 @@ main(int argc, char** argv) { int ret; unsigned i,high_port; - uint8_t nb_sys_ports, port; + uint16_t nb_sys_ports, port; /* Associate signal_hanlder function with USR signals */ signal(SIGUSR1, signal_handler); diff --git a/dpdk/examples/flow_classify/Makefile b/dpdk/examples/flow_classify/Makefile new file mode 100644 index 00000000..eecdde14 --- /dev/null +++ b/dpdk/examples/flow_classify/Makefile @@ -0,0 +1,57 @@ +# BSD LICENSE +# +# Copyright(c) 2017 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +ifeq ($(RTE_SDK),) +$(error "Please define RTE_SDK environment variable") +endif + +# Default target, can be overridden by command line or environment +RTE_TARGET ?= x86_64-native-linuxapp-gcc + +include $(RTE_SDK)/mk/rte.vars.mk + +# binary name +APP = flow_classify + + +# all source are stored in SRCS-y +SRCS-y := flow_classify.c + +CFLAGS += -O3 +CFLAGS += $(WERROR_FLAGS) + +# workaround for a gcc bug with noreturn attribute +# http://gcc.gnu.org/bugzilla/show_bug.cgi?id=12603 +ifeq ($(CONFIG_RTE_TOOLCHAIN_GCC),y) +CFLAGS_main.o += -Wno-return-type +endif + +include $(RTE_SDK)/mk/rte.extapp.mk diff --git a/dpdk/examples/flow_classify/flow_classify.c b/dpdk/examples/flow_classify/flow_classify.c new file mode 100644 index 00000000..766f1dd0 --- /dev/null +++ b/dpdk/examples/flow_classify/flow_classify.c @@ -0,0 +1,852 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#define RX_RING_SIZE 128 +#define TX_RING_SIZE 512 + +#define NUM_MBUFS 8191 +#define MBUF_CACHE_SIZE 250 +#define BURST_SIZE 32 + +#define MAX_NUM_CLASSIFY 30 +#define FLOW_CLASSIFY_MAX_RULE_NUM 91 +#define FLOW_CLASSIFY_MAX_PRIORITY 8 +#define FLOW_CLASSIFIER_NAME_SIZE 64 + +#define COMMENT_LEAD_CHAR ('#') +#define OPTION_RULE_IPV4 "rule_ipv4" +#define RTE_LOGTYPE_FLOW_CLASSIFY RTE_LOGTYPE_USER3 +#define flow_classify_log(format, ...) \ + RTE_LOG(ERR, FLOW_CLASSIFY, format, ##__VA_ARGS__) + +#define uint32_t_to_char(ip, a, b, c, d) do {\ + *a = (unsigned char)(ip >> 24 & 0xff);\ + *b = (unsigned char)(ip >> 16 & 0xff);\ + *c = (unsigned char)(ip >> 8 & 0xff);\ + *d = (unsigned char)(ip & 0xff);\ + } while (0) + +enum { + CB_FLD_SRC_ADDR, + CB_FLD_DST_ADDR, + CB_FLD_SRC_PORT, + CB_FLD_SRC_PORT_DLM, + CB_FLD_SRC_PORT_MASK, + CB_FLD_DST_PORT, + CB_FLD_DST_PORT_DLM, + CB_FLD_DST_PORT_MASK, + CB_FLD_PROTO, + CB_FLD_PRIORITY, + CB_FLD_NUM, +}; + +static struct{ + const char *rule_ipv4_name; +} parm_config; +const char cb_port_delim[] = ":"; + +static const struct rte_eth_conf port_conf_default = { + .rxmode = { .max_rx_pkt_len = ETHER_MAX_LEN } +}; + +struct flow_classifier { + struct rte_flow_classifier *cls; + uint32_t table_id[RTE_FLOW_CLASSIFY_TABLE_MAX]; +}; + +struct flow_classifier_acl { + struct flow_classifier cls; +} __rte_cache_aligned; + +/* ACL field definitions for IPv4 5 tuple rule */ + +enum { + PROTO_FIELD_IPV4, + SRC_FIELD_IPV4, + DST_FIELD_IPV4, + SRCP_FIELD_IPV4, + DSTP_FIELD_IPV4, + NUM_FIELDS_IPV4 +}; + +enum { + PROTO_INPUT_IPV4, + SRC_INPUT_IPV4, + DST_INPUT_IPV4, + SRCP_DESTP_INPUT_IPV4 +}; + +static struct rte_acl_field_def ipv4_defs[NUM_FIELDS_IPV4] = { + /* first input field - always one byte long. */ + { + .type = RTE_ACL_FIELD_TYPE_BITMASK, + .size = sizeof(uint8_t), + .field_index = PROTO_FIELD_IPV4, + .input_index = PROTO_INPUT_IPV4, + .offset = sizeof(struct ether_hdr) + + offsetof(struct ipv4_hdr, next_proto_id), + }, + /* next input field (IPv4 source address) - 4 consecutive bytes. */ + { + /* rte_flow uses a bit mask for IPv4 addresses */ + .type = RTE_ACL_FIELD_TYPE_BITMASK, + .size = sizeof(uint32_t), + .field_index = SRC_FIELD_IPV4, + .input_index = SRC_INPUT_IPV4, + .offset = sizeof(struct ether_hdr) + + offsetof(struct ipv4_hdr, src_addr), + }, + /* next input field (IPv4 destination address) - 4 consecutive bytes. */ + { + /* rte_flow uses a bit mask for IPv4 addresses */ + .type = RTE_ACL_FIELD_TYPE_BITMASK, + .size = sizeof(uint32_t), + .field_index = DST_FIELD_IPV4, + .input_index = DST_INPUT_IPV4, + .offset = sizeof(struct ether_hdr) + + offsetof(struct ipv4_hdr, dst_addr), + }, + /* + * Next 2 fields (src & dst ports) form 4 consecutive bytes. + * They share the same input index. + */ + { + /* rte_flow uses a bit mask for protocol ports */ + .type = RTE_ACL_FIELD_TYPE_BITMASK, + .size = sizeof(uint16_t), + .field_index = SRCP_FIELD_IPV4, + .input_index = SRCP_DESTP_INPUT_IPV4, + .offset = sizeof(struct ether_hdr) + + sizeof(struct ipv4_hdr) + + offsetof(struct tcp_hdr, src_port), + }, + { + /* rte_flow uses a bit mask for protocol ports */ + .type = RTE_ACL_FIELD_TYPE_BITMASK, + .size = sizeof(uint16_t), + .field_index = DSTP_FIELD_IPV4, + .input_index = SRCP_DESTP_INPUT_IPV4, + .offset = sizeof(struct ether_hdr) + + sizeof(struct ipv4_hdr) + + offsetof(struct tcp_hdr, dst_port), + }, +}; + +/* flow classify data */ +static int num_classify_rules; +static struct rte_flow_classify_rule *rules[MAX_NUM_CLASSIFY]; +static struct rte_flow_classify_ipv4_5tuple_stats ntuple_stats; +static struct rte_flow_classify_stats classify_stats = { + .stats = (void **)&ntuple_stats +}; + +/* parameters for rte_flow_classify_validate and + * rte_flow_classify_table_entry_add functions + */ + +static struct rte_flow_item eth_item = { RTE_FLOW_ITEM_TYPE_ETH, + 0, 0, 0 }; +static struct rte_flow_item end_item = { RTE_FLOW_ITEM_TYPE_END, + 0, 0, 0 }; + +/* sample actions: + * "actions count / end" + */ +static struct rte_flow_action count_action = { RTE_FLOW_ACTION_TYPE_COUNT, 0}; +static struct rte_flow_action end_action = { RTE_FLOW_ACTION_TYPE_END, 0}; +static struct rte_flow_action actions[2]; + +/* sample attributes */ +static struct rte_flow_attr attr; + +/* flow_classify.c: * Based on DPDK skeleton forwarding example. */ + +/* + * Initializes a given port using global settings and with the RX buffers + * coming from the mbuf_pool passed as a parameter. + */ +static inline int +port_init(uint8_t port, struct rte_mempool *mbuf_pool) +{ + struct rte_eth_conf port_conf = port_conf_default; + struct ether_addr addr; + const uint16_t rx_rings = 1, tx_rings = 1; + int retval; + uint16_t q; + + if (port >= rte_eth_dev_count()) + return -1; + + /* Configure the Ethernet device. */ + retval = rte_eth_dev_configure(port, rx_rings, tx_rings, &port_conf); + if (retval != 0) + return retval; + + /* Allocate and set up 1 RX queue per Ethernet port. */ + for (q = 0; q < rx_rings; q++) { + retval = rte_eth_rx_queue_setup(port, q, RX_RING_SIZE, + rte_eth_dev_socket_id(port), NULL, mbuf_pool); + if (retval < 0) + return retval; + } + + /* Allocate and set up 1 TX queue per Ethernet port. */ + for (q = 0; q < tx_rings; q++) { + retval = rte_eth_tx_queue_setup(port, q, TX_RING_SIZE, + rte_eth_dev_socket_id(port), NULL); + if (retval < 0) + return retval; + } + + /* Start the Ethernet port. */ + retval = rte_eth_dev_start(port); + if (retval < 0) + return retval; + + /* Display the port MAC address. */ + rte_eth_macaddr_get(port, &addr); + printf("Port %u MAC: %02" PRIx8 " %02" PRIx8 " %02" PRIx8 + " %02" PRIx8 " %02" PRIx8 " %02" PRIx8 "\n", + port, + addr.addr_bytes[0], addr.addr_bytes[1], + addr.addr_bytes[2], addr.addr_bytes[3], + addr.addr_bytes[4], addr.addr_bytes[5]); + + /* Enable RX in promiscuous mode for the Ethernet device. */ + rte_eth_promiscuous_enable(port); + + return 0; +} + +/* + * The lcore main. This is the main thread that does the work, reading from + * an input port classifying the packets and writing to an output port. + */ +static __attribute__((noreturn)) void +lcore_main(struct flow_classifier *cls_app) +{ + const uint8_t nb_ports = rte_eth_dev_count(); + uint8_t port; + int ret; + int i = 0; + + ret = rte_flow_classify_table_entry_delete(cls_app->cls, + cls_app->table_id[0], rules[7]); + if (ret) + printf("table_entry_delete failed [7] %d\n\n", ret); + else + printf("table_entry_delete succeeded [7]\n\n"); + + /* + * Check that the port is on the same NUMA node as the polling thread + * for best performance. + */ + for (port = 0; port < nb_ports; port++) + if (rte_eth_dev_socket_id(port) > 0 && + rte_eth_dev_socket_id(port) != (int)rte_socket_id()) { + printf("\n\n"); + printf("WARNING: port %u is on remote NUMA node\n", + port); + printf("to polling thread.\n"); + printf("Performance will not be optimal.\n"); + + printf("\nCore %u forwarding packets. ", + rte_lcore_id()); + printf("[Ctrl+C to quit]\n"); + } + /* Run until the application is quit or killed. */ + for (;;) { + /* + * Receive packets on a port, classify them and forward them + * on the paired port. + * The mapping is 0 -> 1, 1 -> 0, 2 -> 3, 3 -> 2, etc. + */ + for (port = 0; port < nb_ports; port++) { + /* Get burst of RX packets, from first port of pair. */ + struct rte_mbuf *bufs[BURST_SIZE]; + const uint16_t nb_rx = rte_eth_rx_burst(port, 0, + bufs, BURST_SIZE); + + if (unlikely(nb_rx == 0)) + continue; + + for (i = 0; i < MAX_NUM_CLASSIFY; i++) { + if (rules[i]) { + ret = rte_flow_classifier_query( + cls_app->cls, + cls_app->table_id[0], + bufs, nb_rx, rules[i], + &classify_stats); + if (ret) + printf( + "rule [%d] query failed ret [%d]\n\n", + i, ret); + else { + printf( + "rule[%d] count=%"PRIu64"\n", + i, ntuple_stats.counter1); + + printf("proto = %d\n", + ntuple_stats.ipv4_5tuple.proto); + } + } + } + + /* Send burst of TX packets, to second port of pair. */ + const uint16_t nb_tx = rte_eth_tx_burst(port ^ 1, 0, + bufs, nb_rx); + + /* Free any unsent packets. */ + if (unlikely(nb_tx < nb_rx)) { + uint16_t buf; + + for (buf = nb_tx; buf < nb_rx; buf++) + rte_pktmbuf_free(bufs[buf]); + } + } + } +} + +/* + * Parse IPv4 5 tuple rules file, ipv4_rules_file.txt. + * Expected format: + * '/' \ + * '/' \ + * ":" \ + * ":" \ + * '/' \ + * + */ + +static int +get_cb_field(char **in, uint32_t *fd, int base, unsigned long lim, + char dlm) +{ + unsigned long val; + char *end; + + errno = 0; + val = strtoul(*in, &end, base); + if (errno != 0 || end[0] != dlm || val > lim) + return -EINVAL; + *fd = (uint32_t)val; + *in = end + 1; + return 0; +} + +static int +parse_ipv4_net(char *in, uint32_t *addr, uint32_t *mask_len) +{ + uint32_t a, b, c, d, m; + + if (get_cb_field(&in, &a, 0, UINT8_MAX, '.')) + return -EINVAL; + if (get_cb_field(&in, &b, 0, UINT8_MAX, '.')) + return -EINVAL; + if (get_cb_field(&in, &c, 0, UINT8_MAX, '.')) + return -EINVAL; + if (get_cb_field(&in, &d, 0, UINT8_MAX, '/')) + return -EINVAL; + if (get_cb_field(&in, &m, 0, sizeof(uint32_t) * CHAR_BIT, 0)) + return -EINVAL; + + addr[0] = IPv4(a, b, c, d); + mask_len[0] = m; + return 0; +} + +static int +parse_ipv4_5tuple_rule(char *str, struct rte_eth_ntuple_filter *ntuple_filter) +{ + int i, ret; + char *s, *sp, *in[CB_FLD_NUM]; + static const char *dlm = " \t\n"; + int dim = CB_FLD_NUM; + uint32_t temp; + + s = str; + for (i = 0; i != dim; i++, s = NULL) { + in[i] = strtok_r(s, dlm, &sp); + if (in[i] == NULL) + return -EINVAL; + } + + ret = parse_ipv4_net(in[CB_FLD_SRC_ADDR], + &ntuple_filter->src_ip, + &ntuple_filter->src_ip_mask); + if (ret != 0) { + flow_classify_log("failed to read source address/mask: %s\n", + in[CB_FLD_SRC_ADDR]); + return ret; + } + + ret = parse_ipv4_net(in[CB_FLD_DST_ADDR], + &ntuple_filter->dst_ip, + &ntuple_filter->dst_ip_mask); + if (ret != 0) { + flow_classify_log("failed to read source address/mask: %s\n", + in[CB_FLD_DST_ADDR]); + return ret; + } + + if (get_cb_field(&in[CB_FLD_SRC_PORT], &temp, 0, UINT16_MAX, 0)) + return -EINVAL; + ntuple_filter->src_port = (uint16_t)temp; + + if (strncmp(in[CB_FLD_SRC_PORT_DLM], cb_port_delim, + sizeof(cb_port_delim)) != 0) + return -EINVAL; + + if (get_cb_field(&in[CB_FLD_SRC_PORT_MASK], &temp, 0, UINT16_MAX, 0)) + return -EINVAL; + ntuple_filter->src_port_mask = (uint16_t)temp; + + if (get_cb_field(&in[CB_FLD_DST_PORT], &temp, 0, UINT16_MAX, 0)) + return -EINVAL; + ntuple_filter->dst_port = (uint16_t)temp; + + if (strncmp(in[CB_FLD_DST_PORT_DLM], cb_port_delim, + sizeof(cb_port_delim)) != 0) + return -EINVAL; + + if (get_cb_field(&in[CB_FLD_DST_PORT_MASK], &temp, 0, UINT16_MAX, 0)) + return -EINVAL; + ntuple_filter->dst_port_mask = (uint16_t)temp; + + if (get_cb_field(&in[CB_FLD_PROTO], &temp, 0, UINT8_MAX, '/')) + return -EINVAL; + ntuple_filter->proto = (uint8_t)temp; + + if (get_cb_field(&in[CB_FLD_PROTO], &temp, 0, UINT8_MAX, 0)) + return -EINVAL; + ntuple_filter->proto_mask = (uint8_t)temp; + + if (get_cb_field(&in[CB_FLD_PRIORITY], &temp, 0, UINT16_MAX, 0)) + return -EINVAL; + ntuple_filter->priority = (uint16_t)temp; + if (ntuple_filter->priority > FLOW_CLASSIFY_MAX_PRIORITY) + ret = -EINVAL; + + return ret; +} + +/* Bypass comment and empty lines */ +static inline int +is_bypass_line(char *buff) +{ + int i = 0; + + /* comment line */ + if (buff[0] == COMMENT_LEAD_CHAR) + return 1; + /* empty line */ + while (buff[i] != '\0') { + if (!isspace(buff[i])) + return 0; + i++; + } + return 1; +} + +static uint32_t +convert_depth_to_bitmask(uint32_t depth_val) +{ + uint32_t bitmask = 0; + int i, j; + + for (i = depth_val, j = 0; i > 0; i--, j++) + bitmask |= (1 << (31 - j)); + return bitmask; +} + +static int +add_classify_rule(struct rte_eth_ntuple_filter *ntuple_filter, + struct flow_classifier *cls_app) +{ + int ret = -1; + int key_found; + struct rte_flow_error error; + struct rte_flow_item_ipv4 ipv4_spec; + struct rte_flow_item_ipv4 ipv4_mask; + struct rte_flow_item ipv4_udp_item; + struct rte_flow_item ipv4_tcp_item; + struct rte_flow_item ipv4_sctp_item; + struct rte_flow_item_udp udp_spec; + struct rte_flow_item_udp udp_mask; + struct rte_flow_item udp_item; + struct rte_flow_item_tcp tcp_spec; + struct rte_flow_item_tcp tcp_mask; + struct rte_flow_item tcp_item; + struct rte_flow_item_sctp sctp_spec; + struct rte_flow_item_sctp sctp_mask; + struct rte_flow_item sctp_item; + struct rte_flow_item pattern_ipv4_5tuple[4]; + struct rte_flow_classify_rule *rule; + uint8_t ipv4_proto; + + if (num_classify_rules >= MAX_NUM_CLASSIFY) { + printf( + "\nINFO: classify rule capacity %d reached\n", + num_classify_rules); + return ret; + } + + /* set up parameters for validate and add */ + memset(&ipv4_spec, 0, sizeof(ipv4_spec)); + ipv4_spec.hdr.next_proto_id = ntuple_filter->proto; + ipv4_spec.hdr.src_addr = ntuple_filter->src_ip; + ipv4_spec.hdr.dst_addr = ntuple_filter->dst_ip; + ipv4_proto = ipv4_spec.hdr.next_proto_id; + + memset(&ipv4_mask, 0, sizeof(ipv4_mask)); + ipv4_mask.hdr.next_proto_id = ntuple_filter->proto_mask; + ipv4_mask.hdr.src_addr = ntuple_filter->src_ip_mask; + ipv4_mask.hdr.src_addr = + convert_depth_to_bitmask(ipv4_mask.hdr.src_addr); + ipv4_mask.hdr.dst_addr = ntuple_filter->dst_ip_mask; + ipv4_mask.hdr.dst_addr = + convert_depth_to_bitmask(ipv4_mask.hdr.dst_addr); + + switch (ipv4_proto) { + case IPPROTO_UDP: + ipv4_udp_item.type = RTE_FLOW_ITEM_TYPE_IPV4; + ipv4_udp_item.spec = &ipv4_spec; + ipv4_udp_item.mask = &ipv4_mask; + ipv4_udp_item.last = NULL; + + udp_spec.hdr.src_port = ntuple_filter->src_port; + udp_spec.hdr.dst_port = ntuple_filter->dst_port; + udp_spec.hdr.dgram_len = 0; + udp_spec.hdr.dgram_cksum = 0; + + udp_mask.hdr.src_port = ntuple_filter->src_port_mask; + udp_mask.hdr.dst_port = ntuple_filter->dst_port_mask; + udp_mask.hdr.dgram_len = 0; + udp_mask.hdr.dgram_cksum = 0; + + udp_item.type = RTE_FLOW_ITEM_TYPE_UDP; + udp_item.spec = &udp_spec; + udp_item.mask = &udp_mask; + udp_item.last = NULL; + + attr.priority = ntuple_filter->priority; + pattern_ipv4_5tuple[1] = ipv4_udp_item; + pattern_ipv4_5tuple[2] = udp_item; + break; + case IPPROTO_TCP: + ipv4_tcp_item.type = RTE_FLOW_ITEM_TYPE_IPV4; + ipv4_tcp_item.spec = &ipv4_spec; + ipv4_tcp_item.mask = &ipv4_mask; + ipv4_tcp_item.last = NULL; + + memset(&tcp_spec, 0, sizeof(tcp_spec)); + tcp_spec.hdr.src_port = ntuple_filter->src_port; + tcp_spec.hdr.dst_port = ntuple_filter->dst_port; + + memset(&tcp_mask, 0, sizeof(tcp_mask)); + tcp_mask.hdr.src_port = ntuple_filter->src_port_mask; + tcp_mask.hdr.dst_port = ntuple_filter->dst_port_mask; + + tcp_item.type = RTE_FLOW_ITEM_TYPE_TCP; + tcp_item.spec = &tcp_spec; + tcp_item.mask = &tcp_mask; + tcp_item.last = NULL; + + attr.priority = ntuple_filter->priority; + pattern_ipv4_5tuple[1] = ipv4_tcp_item; + pattern_ipv4_5tuple[2] = tcp_item; + break; + case IPPROTO_SCTP: + ipv4_sctp_item.type = RTE_FLOW_ITEM_TYPE_IPV4; + ipv4_sctp_item.spec = &ipv4_spec; + ipv4_sctp_item.mask = &ipv4_mask; + ipv4_sctp_item.last = NULL; + + sctp_spec.hdr.src_port = ntuple_filter->src_port; + sctp_spec.hdr.dst_port = ntuple_filter->dst_port; + sctp_spec.hdr.cksum = 0; + sctp_spec.hdr.tag = 0; + + sctp_mask.hdr.src_port = ntuple_filter->src_port_mask; + sctp_mask.hdr.dst_port = ntuple_filter->dst_port_mask; + sctp_mask.hdr.cksum = 0; + sctp_mask.hdr.tag = 0; + + sctp_item.type = RTE_FLOW_ITEM_TYPE_SCTP; + sctp_item.spec = &sctp_spec; + sctp_item.mask = &sctp_mask; + sctp_item.last = NULL; + + attr.priority = ntuple_filter->priority; + pattern_ipv4_5tuple[1] = ipv4_sctp_item; + pattern_ipv4_5tuple[2] = sctp_item; + break; + default: + return ret; + } + + attr.ingress = 1; + pattern_ipv4_5tuple[0] = eth_item; + pattern_ipv4_5tuple[3] = end_item; + actions[0] = count_action; + actions[1] = end_action; + + rule = rte_flow_classify_table_entry_add( + cls_app->cls, cls_app->table_id[0], &key_found, + &attr, pattern_ipv4_5tuple, actions, &error); + if (rule == NULL) { + printf("table entry add failed ipv4_proto = %u\n", + ipv4_proto); + ret = -1; + return ret; + } + + rules[num_classify_rules] = rule; + num_classify_rules++; + return 0; +} + +static int +add_rules(const char *rule_path, struct flow_classifier *cls_app) +{ + FILE *fh; + char buff[LINE_MAX]; + unsigned int i = 0; + unsigned int total_num = 0; + struct rte_eth_ntuple_filter ntuple_filter; + int ret; + + fh = fopen(rule_path, "rb"); + if (fh == NULL) + rte_exit(EXIT_FAILURE, "%s: fopen %s failed\n", __func__, + rule_path); + + ret = fseek(fh, 0, SEEK_SET); + if (ret) + rte_exit(EXIT_FAILURE, "%s: fseek %d failed\n", __func__, + ret); + + i = 0; + while (fgets(buff, LINE_MAX, fh) != NULL) { + i++; + + if (is_bypass_line(buff)) + continue; + + if (total_num >= FLOW_CLASSIFY_MAX_RULE_NUM - 1) { + printf("\nINFO: classify rule capacity %d reached\n", + total_num); + break; + } + + if (parse_ipv4_5tuple_rule(buff, &ntuple_filter) != 0) + rte_exit(EXIT_FAILURE, + "%s Line %u: parse rules error\n", + rule_path, i); + + if (add_classify_rule(&ntuple_filter, cls_app) != 0) + rte_exit(EXIT_FAILURE, "add rule error\n"); + + total_num++; + } + + fclose(fh); + return 0; +} + +/* display usage */ +static void +print_usage(const char *prgname) +{ + printf("%s usage:\n", prgname); + printf("[EAL options] -- --"OPTION_RULE_IPV4"=FILE: "); + printf("specify the ipv4 rules file.\n"); + printf("Each rule occupies one line in the file.\n"); +} + +/* Parse the argument given in the command line of the application */ +static int +parse_args(int argc, char **argv) +{ + int opt, ret; + char **argvopt; + int option_index; + char *prgname = argv[0]; + static struct option lgopts[] = { + {OPTION_RULE_IPV4, 1, 0, 0}, + {NULL, 0, 0, 0} + }; + + argvopt = argv; + + while ((opt = getopt_long(argc, argvopt, "", + lgopts, &option_index)) != EOF) { + + switch (opt) { + /* long options */ + case 0: + if (!strncmp(lgopts[option_index].name, + OPTION_RULE_IPV4, + sizeof(OPTION_RULE_IPV4))) + parm_config.rule_ipv4_name = optarg; + break; + default: + print_usage(prgname); + return -1; + } + } + + if (optind >= 0) + argv[optind-1] = prgname; + + ret = optind-1; + optind = 1; /* reset getopt lib */ + return ret; +} + +/* + * The main function, which does initialization and calls the lcore_main + * function. + */ +int +main(int argc, char *argv[]) +{ + struct rte_mempool *mbuf_pool; + uint8_t nb_ports; + uint8_t portid; + int ret; + int socket_id; + struct rte_table_acl_params table_acl_params; + struct rte_flow_classify_table_params cls_table_params; + struct flow_classifier *cls_app; + struct rte_flow_classifier_params cls_params; + uint32_t size; + + /* Initialize the Environment Abstraction Layer (EAL). */ + ret = rte_eal_init(argc, argv); + if (ret < 0) + rte_exit(EXIT_FAILURE, "Error with EAL initialization\n"); + + argc -= ret; + argv += ret; + + /* parse application arguments (after the EAL ones) */ + ret = parse_args(argc, argv); + if (ret < 0) + rte_exit(EXIT_FAILURE, "Invalid flow_classify parameters\n"); + + /* Check that there is an even number of ports to send/receive on. */ + nb_ports = rte_eth_dev_count(); + if (nb_ports < 2 || (nb_ports & 1)) + rte_exit(EXIT_FAILURE, "Error: number of ports must be even\n"); + + /* Creates a new mempool in memory to hold the mbufs. */ + mbuf_pool = rte_pktmbuf_pool_create("MBUF_POOL", NUM_MBUFS * nb_ports, + MBUF_CACHE_SIZE, 0, RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id()); + + if (mbuf_pool == NULL) + rte_exit(EXIT_FAILURE, "Cannot create mbuf pool\n"); + + /* Initialize all ports. */ + for (portid = 0; portid < nb_ports; portid++) + if (port_init(portid, mbuf_pool) != 0) + rte_exit(EXIT_FAILURE, "Cannot init port %"PRIu8 "\n", + portid); + + if (rte_lcore_count() > 1) + printf("\nWARNING: Too many lcores enabled. Only 1 used.\n"); + + socket_id = rte_eth_dev_socket_id(0); + + /* Memory allocation */ + size = RTE_CACHE_LINE_ROUNDUP(sizeof(struct flow_classifier_acl)); + cls_app = rte_zmalloc(NULL, size, RTE_CACHE_LINE_SIZE); + if (cls_app == NULL) + rte_exit(EXIT_FAILURE, "Cannot allocate classifier memory\n"); + + cls_params.name = "flow_classifier"; + cls_params.socket_id = socket_id; + cls_params.type = RTE_FLOW_CLASSIFY_TABLE_TYPE_ACL; + + cls_app->cls = rte_flow_classifier_create(&cls_params); + if (cls_app->cls == NULL) { + rte_free(cls_app); + rte_exit(EXIT_FAILURE, "Cannot create classifier\n"); + } + + /* initialise ACL table params */ + table_acl_params.name = "table_acl_ipv4_5tuple"; + table_acl_params.n_rules = FLOW_CLASSIFY_MAX_RULE_NUM; + table_acl_params.n_rule_fields = RTE_DIM(ipv4_defs); + memcpy(table_acl_params.field_format, ipv4_defs, sizeof(ipv4_defs)); + + /* initialise table create params */ + cls_table_params.ops = &rte_table_acl_ops, + cls_table_params.arg_create = &table_acl_params, + + ret = rte_flow_classify_table_create(cls_app->cls, &cls_table_params, + &cls_app->table_id[0]); + if (ret) { + rte_flow_classifier_free(cls_app->cls); + rte_free(cls_app); + rte_exit(EXIT_FAILURE, "Failed to create classifier table\n"); + } + + /* read file of IPv4 5 tuple rules and initialize parameters + * for rte_flow_classify_validate and rte_flow_classify_table_entry_add + * API's. + */ + if (add_rules(parm_config.rule_ipv4_name, cls_app)) { + rte_flow_classifier_free(cls_app->cls); + rte_free(cls_app); + rte_exit(EXIT_FAILURE, "Failed to add rules\n"); + } + + /* Call lcore_main on the master core only. */ + lcore_main(cls_app); + + return 0; +} diff --git a/dpdk/examples/flow_classify/ipv4_rules_file.txt b/dpdk/examples/flow_classify/ipv4_rules_file.txt new file mode 100644 index 00000000..dfa0631f --- /dev/null +++ b/dpdk/examples/flow_classify/ipv4_rules_file.txt @@ -0,0 +1,14 @@ +#file format: +#src_ip/masklen dst_ip/masklen src_port : mask dst_port : mask proto/mask priority +# +2.2.2.3/24 2.2.2.7/24 32 : 0xffff 33 : 0xffff 17/0xff 0 +9.9.9.3/24 9.9.9.7/24 32 : 0xffff 33 : 0xffff 17/0xff 1 +9.9.9.3/24 9.9.9.7/24 32 : 0xffff 33 : 0xffff 6/0xff 2 +9.9.8.3/24 9.9.8.7/24 32 : 0xffff 33 : 0xffff 6/0xff 3 +6.7.8.9/24 2.3.4.5/24 32 : 0x0000 33 : 0x0000 132/0xff 4 +6.7.8.9/32 192.168.0.36/32 10 : 0xffff 11 : 0xffff 6/0xfe 5 +6.7.8.9/24 192.168.0.36/24 10 : 0xffff 11 : 0xffff 6/0xfe 6 +6.7.8.9/16 192.168.0.36/16 10 : 0xffff 11 : 0xffff 6/0xfe 7 +6.7.8.9/8 192.168.0.36/8 10 : 0xffff 11 : 0xffff 6/0xfe 8 +#error rules +#9.8.7.6/8 192.168.0.36/8 10 : 0xffff 11 : 0xffff 6/0xfe 9 \ No newline at end of file diff --git a/dpdk/examples/flow_filtering/Makefile b/dpdk/examples/flow_filtering/Makefile new file mode 100644 index 00000000..70b82fe3 --- /dev/null +++ b/dpdk/examples/flow_filtering/Makefile @@ -0,0 +1,49 @@ +# +# BSD LICENSE +# +# Copyright 2017 Mellanox. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Mellanox nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +ifeq ($(RTE_SDK),) +$(error "Please define RTE_SDK environment variable") +endif + +# Default target, can be overridden by command line or environment +RTE_TARGET ?= x86_64-native-linuxapp-gcc + +include $(RTE_SDK)/mk/rte.vars.mk + +APP = flow + +SRCS-y := main.c + +CFLAGS += -O3 +CFLAGS += $(WERROR_FLAGS) + +include $(RTE_SDK)/mk/rte.extapp.mk diff --git a/dpdk/examples/flow_filtering/flow_blocks.c b/dpdk/examples/flow_filtering/flow_blocks.c new file mode 100644 index 00000000..f92df102 --- /dev/null +++ b/dpdk/examples/flow_filtering/flow_blocks.c @@ -0,0 +1,150 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 Mellanox. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Mellanox nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#define MAX_PATTERN_NUM 4 + +struct rte_flow * +generate_ipv4_flow(uint8_t port_id, uint16_t rx_q, + uint32_t src_ip, uint32_t src_mask, + uint32_t dest_ip, uint32_t dest_mask, + struct rte_flow_error *error); + + +/** + * create a flow rule that sends packets with matching src and dest ip + * to selected queue. + * + * @param port_id + * The selected port. + * @param rx_q + * The selected target queue. + * @param src_ip + * The src ip value to match the input packet. + * @param src_mask + * The mask to apply to the src ip. + * @param dest_ip + * The dest ip value to match the input packet. + * @param dest_mask + * The mask to apply to the dest ip. + * @param[out] error + * Perform verbose error reporting if not NULL. + * + * @return + * A flow if the rule could be created else return NULL. + */ +struct rte_flow * +generate_ipv4_flow(uint8_t port_id, uint16_t rx_q, + uint32_t src_ip, uint32_t src_mask, + uint32_t dest_ip, uint32_t dest_mask, + struct rte_flow_error *error) +{ + struct rte_flow_attr attr; + struct rte_flow_item pattern[MAX_PATTERN_NUM]; + struct rte_flow_action action[MAX_PATTERN_NUM]; + struct rte_flow *flow = NULL; + struct rte_flow_action_queue queue = { .index = rx_q }; + struct rte_flow_item_eth eth_spec; + struct rte_flow_item_eth eth_mask; + struct rte_flow_item_vlan vlan_spec; + struct rte_flow_item_vlan vlan_mask; + struct rte_flow_item_ipv4 ip_spec; + struct rte_flow_item_ipv4 ip_mask; + int res; + + memset(pattern, 0, sizeof(pattern)); + memset(action, 0, sizeof(action)); + + /* + * set the rule attribute. + * in this case only ingress packets will be checked. + */ + memset(&attr, 0, sizeof(struct rte_flow_attr)); + attr.ingress = 1; + + /* + * create the action sequence. + * one action only, move packet to queue + */ + + action[0].type = RTE_FLOW_ACTION_TYPE_QUEUE; + action[0].conf = &queue; + action[1].type = RTE_FLOW_ACTION_TYPE_END; + + /* + * set the first level of the pattern (eth). + * since in this example we just want to get the + * ipv4 we set this level to allow all. + */ + memset(ð_spec, 0, sizeof(struct rte_flow_item_eth)); + memset(ð_mask, 0, sizeof(struct rte_flow_item_eth)); + eth_spec.type = 0; + eth_mask.type = 0; + pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH; + pattern[0].spec = ð_spec; + pattern[0].mask = ð_mask; + + /* + * setting the second level of the pattern (vlan). + * since in this example we just want to get the + * ipv4 we also set this level to allow all. + */ + memset(&vlan_spec, 0, sizeof(struct rte_flow_item_vlan)); + memset(&vlan_mask, 0, sizeof(struct rte_flow_item_vlan)); + pattern[1].type = RTE_FLOW_ITEM_TYPE_VLAN; + pattern[1].spec = &vlan_spec; + pattern[1].mask = &vlan_mask; + + /* + * setting the third level of the pattern (ip). + * in this example this is the level we care about + * so we set it according to the parameters. + */ + memset(&ip_spec, 0, sizeof(struct rte_flow_item_ipv4)); + memset(&ip_mask, 0, sizeof(struct rte_flow_item_ipv4)); + ip_spec.hdr.dst_addr = htonl(dest_ip); + ip_mask.hdr.dst_addr = dest_mask; + ip_spec.hdr.src_addr = htonl(src_ip); + ip_mask.hdr.src_addr = src_mask; + pattern[2].type = RTE_FLOW_ITEM_TYPE_IPV4; + pattern[2].spec = &ip_spec; + pattern[2].mask = &ip_mask; + + /* the final level must be always type end */ + pattern[3].type = RTE_FLOW_ITEM_TYPE_END; + + res = rte_flow_validate(port_id, &attr, pattern, action, error); + if (!res) + flow = rte_flow_create(port_id, &attr, pattern, action, error); + + return flow; +} + diff --git a/dpdk/examples/flow_filtering/main.c b/dpdk/examples/flow_filtering/main.c new file mode 100644 index 00000000..7d739b4a --- /dev/null +++ b/dpdk/examples/flow_filtering/main.c @@ -0,0 +1,244 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 Mellanox. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Mellanox. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static volatile bool force_quit; + +static uint8_t port_id; +static uint16_t nr_queues = 5; +static uint8_t selected_queue = 1; +struct rte_mempool *mbuf_pool; +struct rte_flow *flow; + +#define SRC_IP ((0<<24) + (0<<16) + (0<<8) + 0) /* src ip = 0.0.0.0 */ +#define DEST_IP ((192<<24) + (168<<16) + (1<<8) + 1) /* dest ip = 192.168.1.1 */ +#define FULL_MASK 0xffffffff /* full mask */ +#define EMPTY_MASK 0x0 /* empty mask */ + +#include "flow_blocks.c" + +static inline void +print_ether_addr(const char *what, struct ether_addr *eth_addr) +{ + char buf[ETHER_ADDR_FMT_SIZE]; + ether_format_addr(buf, ETHER_ADDR_FMT_SIZE, eth_addr); + printf("%s%s", what, buf); +} + +static void +main_loop(void) +{ + struct rte_mbuf *mbufs[32]; + struct ether_hdr *eth_hdr; + struct rte_flow_error error; + uint16_t nb_rx; + uint16_t i; + uint16_t j; + + while (!force_quit) { + for (i = 0; i < nr_queues; i++) { + nb_rx = rte_eth_rx_burst(port_id, + i, mbufs, 32); + if (nb_rx) { + for (j = 0; j < nb_rx; j++) { + struct rte_mbuf *m = mbufs[j]; + + eth_hdr = rte_pktmbuf_mtod(m, + struct ether_hdr *); + print_ether_addr("src=", + ð_hdr->s_addr); + print_ether_addr(" - dst=", + ð_hdr->d_addr); + printf(" - queue=0x%x", + (unsigned int)i); + printf("\n"); + + rte_pktmbuf_free(m); + } + } + } + } + + /* closing and releasing resources */ + rte_flow_flush(port_id, &error); + rte_eth_dev_stop(port_id); + rte_eth_dev_close(port_id); +} + +static void +assert_link_status(void) +{ + struct rte_eth_link link; + + memset(&link, 0, sizeof(link)); + rte_eth_link_get(port_id, &link); + if (link.link_status == ETH_LINK_DOWN) + rte_exit(EXIT_FAILURE, ":: error: link is still down\n"); +} + +static void +init_port(void) +{ + int ret; + uint16_t i; + struct rte_eth_conf port_conf = { + .rxmode = { + .split_hdr_size = 0, + /**< Header Split disabled */ + .header_split = 0, + /**< IP checksum offload disabled */ + .hw_ip_checksum = 0, + /**< VLAN filtering disabled */ + .hw_vlan_filter = 0, + /**< Jumbo Frame Support disabled */ + .jumbo_frame = 0, + /**< CRC stripped by hardware */ + .hw_strip_crc = 1, + }, + }; + + printf(":: initializing port: %d\n", port_id); + ret = rte_eth_dev_configure(port_id, + nr_queues, nr_queues, &port_conf); + if (ret < 0) { + rte_exit(EXIT_FAILURE, + ":: cannot configure device: err=%d, port=%u\n", + ret, port_id); + } + + /* only set Rx queues: something we care only so far */ + for (i = 0; i < nr_queues; i++) { + ret = rte_eth_rx_queue_setup(port_id, i, 512, + rte_eth_dev_socket_id(port_id), + NULL, + mbuf_pool); + if (ret < 0) { + rte_exit(EXIT_FAILURE, + ":: Rx queue setup failed: err=%d, port=%u\n", + ret, port_id); + } + } + + rte_eth_promiscuous_enable(port_id); + ret = rte_eth_dev_start(port_id); + if (ret < 0) { + rte_exit(EXIT_FAILURE, + "rte_eth_dev_start:err=%d, port=%u\n", + ret, port_id); + } + + assert_link_status(); + + printf(":: initializing port: %d done\n", port_id); +} + +static void +signal_handler(int signum) +{ + if (signum == SIGINT || signum == SIGTERM) { + printf("\n\nSignal %d received, preparing to exit...\n", + signum); + force_quit = true; + } +} + +int +main(int argc, char **argv) +{ + int ret; + uint8_t nr_ports; + struct rte_flow_error error; + + ret = rte_eal_init(argc, argv); + if (ret < 0) + rte_exit(EXIT_FAILURE, ":: invalid EAL arguments\n"); + + force_quit = false; + signal(SIGINT, signal_handler); + signal(SIGTERM, signal_handler); + + nr_ports = rte_eth_dev_count(); + if (nr_ports == 0) + rte_exit(EXIT_FAILURE, ":: no Ethernet ports found\n"); + port_id = 0; + if (nr_ports != 1) { + printf(":: warn: %d ports detected, but we use only one: port %u\n", + nr_ports, port_id); + } + mbuf_pool = rte_pktmbuf_pool_create("mbuf_pool", 4096, 128, 0, + RTE_MBUF_DEFAULT_BUF_SIZE, + rte_socket_id()); + if (mbuf_pool == NULL) + rte_exit(EXIT_FAILURE, "Cannot init mbuf pool\n"); + + init_port(); + + /* create flow for send packet with */ + flow = generate_ipv4_flow(port_id, selected_queue, + SRC_IP, EMPTY_MASK, + DEST_IP, FULL_MASK, &error); + if (!flow) { + printf("Flow can't be created %d message: %s\n", + error.type, + error.message ? error.message : "(no stated reason)"); + rte_exit(EXIT_FAILURE, "error in creating flow"); + } + + main_loop(); + + return 0; +} diff --git a/dpdk/examples/helloworld/Makefile b/dpdk/examples/helloworld/Makefile index d2cca7a7..c83ec01e 100644 --- a/dpdk/examples/helloworld/Makefile +++ b/dpdk/examples/helloworld/Makefile @@ -33,7 +33,7 @@ ifeq ($(RTE_SDK),) $(error "Please define RTE_SDK environment variable") endif -# Default target, can be overriden by command line or environment +# Default target, can be overridden by command line or environment RTE_TARGET ?= x86_64-native-linuxapp-gcc include $(RTE_SDK)/mk/rte.vars.mk diff --git a/dpdk/examples/helloworld/main.c b/dpdk/examples/helloworld/main.c index 8b7a2de0..916b6ad0 100644 --- a/dpdk/examples/helloworld/main.c +++ b/dpdk/examples/helloworld/main.c @@ -38,7 +38,6 @@ #include #include -#include #include #include #include diff --git a/dpdk/examples/ip_fragmentation/Makefile b/dpdk/examples/ip_fragmentation/Makefile index c321e6a1..4bc01abb 100644 --- a/dpdk/examples/ip_fragmentation/Makefile +++ b/dpdk/examples/ip_fragmentation/Makefile @@ -34,7 +34,7 @@ ifeq ($(RTE_SDK),) $(error "Please define RTE_SDK environment variable") endif -# Default target, can be overriden by command line or environment +# Default target, can be overridden by command line or environment RTE_TARGET ?= x86_64-native-linuxapp-gcc include $(RTE_SDK)/mk/rte.vars.mk diff --git a/dpdk/examples/ip_fragmentation/main.c b/dpdk/examples/ip_fragmentation/main.c index 2f452648..5aefe098 100644 --- a/dpdk/examples/ip_fragmentation/main.c +++ b/dpdk/examples/ip_fragmentation/main.c @@ -48,9 +48,7 @@ #include #include #include -#include #include -#include #include #include #include @@ -59,12 +57,10 @@ #include #include #include -#include #include #include #include #include -#include #include #include #include @@ -156,7 +152,7 @@ struct rx_queue { struct rte_mempool *indirect_pool; struct rte_lpm *lpm; struct rte_lpm6 *lpm6; - uint8_t portid; + uint16_t portid; }; #define MAX_RX_QUEUE_PER_LCORE 16 @@ -169,7 +165,7 @@ struct lcore_queue_conf { } __rte_cache_aligned; struct lcore_queue_conf lcore_queue_conf[RTE_MAX_LCORE]; -static const struct rte_eth_conf port_conf = { +static struct rte_eth_conf port_conf = { .rxmode = { .max_rx_pkt_len = JUMBO_FRAME_MAX_SIZE, .split_hdr_size = 0, @@ -177,7 +173,7 @@ static const struct rte_eth_conf port_conf = { .hw_ip_checksum = 1, /**< IP checksum offload enabled */ .hw_vlan_filter = 0, /**< VLAN filtering disabled */ .jumbo_frame = 1, /**< Jumbo Frame Support enabled */ - .hw_strip_crc = 0, /**< CRC stripped by hardware */ + .hw_strip_crc = 1, /**< CRC stripped by hardware */ }, .txmode = { .mq_mode = ETH_MQ_TX_NONE, @@ -242,7 +238,7 @@ static struct rte_lpm6 *socket_lpm6[RTE_MAX_NUMA_NODES]; /* Send burst of packets on an output interface */ static inline int -send_burst(struct lcore_queue_conf *qconf, uint16_t n, uint8_t port) +send_burst(struct lcore_queue_conf *qconf, uint16_t n, uint16_t port) { struct rte_mbuf **m_table; int ret; @@ -263,11 +259,12 @@ send_burst(struct lcore_queue_conf *qconf, uint16_t n, uint8_t port) static inline void l3fwd_simple_forward(struct rte_mbuf *m, struct lcore_queue_conf *qconf, - uint8_t queueid, uint8_t port_in) + uint8_t queueid, uint16_t port_in) { struct rx_queue *rxq; - uint32_t i, len, next_hop_ipv4; - uint8_t next_hop_ipv6, port_out, ipv6; + uint32_t i, len, next_hop; + uint8_t ipv6; + uint16_t port_out; int32_t len2; ipv6 = 0; @@ -291,9 +288,9 @@ l3fwd_simple_forward(struct rte_mbuf *m, struct lcore_queue_conf *qconf, ip_dst = rte_be_to_cpu_32(ip_hdr->dst_addr); /* Find destination port */ - if (rte_lpm_lookup(rxq->lpm, ip_dst, &next_hop_ipv4) == 0 && - (enabled_port_mask & 1 << next_hop_ipv4) != 0) { - port_out = next_hop_ipv4; + if (rte_lpm_lookup(rxq->lpm, ip_dst, &next_hop) == 0 && + (enabled_port_mask & 1 << next_hop) != 0) { + port_out = next_hop; /* Build transmission burst for new port */ len = qconf->tx_mbufs[port_out].len; @@ -327,9 +324,10 @@ l3fwd_simple_forward(struct rte_mbuf *m, struct lcore_queue_conf *qconf, ip_hdr = rte_pktmbuf_mtod(m, struct ipv6_hdr *); /* Find destination port */ - if (rte_lpm6_lookup(rxq->lpm6, ip_hdr->dst_addr, &next_hop_ipv6) == 0 && - (enabled_port_mask & 1 << next_hop_ipv6) != 0) { - port_out = next_hop_ipv6; + if (rte_lpm6_lookup(rxq->lpm6, ip_hdr->dst_addr, + &next_hop) == 0 && + (enabled_port_mask & 1 << next_hop) != 0) { + port_out = next_hop; /* Build transmission burst for new port */ len = qconf->tx_mbufs[port_out].len; @@ -404,7 +402,7 @@ main_loop(__attribute__((unused)) void *dummy) unsigned lcore_id; uint64_t prev_tsc, diff_tsc, cur_tsc; int i, j, nb_rx; - uint8_t portid; + uint16_t portid; struct lcore_queue_conf *qconf; const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / US_PER_S * BURST_TX_DRAIN_US; @@ -424,7 +422,7 @@ main_loop(__attribute__((unused)) void *dummy) portid = qconf->rx_queue_list[i].portid; RTE_LOG(INFO, IP_FRAG, " -- lcoreid=%u portid=%d\n", lcore_id, - (int) portid); + portid); } while (1) { @@ -587,7 +585,7 @@ parse_args(int argc, char **argv) argv[optind-1] = prgname; ret = optind-1; - optind = 0; /* reset getopt lib */ + optind = 1; /* reset getopt lib */ return ret; } @@ -601,11 +599,12 @@ print_ethaddr(const char *name, struct ether_addr *eth_addr) /* Check the link status of all ports in up to 9s, and print them finally */ static void -check_all_ports_link_status(uint8_t port_num, uint32_t port_mask) +check_all_ports_link_status(uint16_t port_num, uint32_t port_mask) { #define CHECK_INTERVAL 100 /* 100ms */ #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ - uint8_t portid, count, all_ports_up, print_flag = 0; + uint16_t portid; + uint8_t count, all_ports_up, print_flag = 0; struct rte_eth_link link; printf("\nChecking link status"); @@ -620,14 +619,13 @@ check_all_ports_link_status(uint8_t port_num, uint32_t port_mask) /* print link status if flag set */ if (print_flag == 1) { if (link.link_status) - printf("Port %d Link Up - speed %u " - "Mbps - %s\n", (uint8_t)portid, - (unsigned)link.link_speed, + printf( + "Port%d Link Up .Speed %u Mbps - %s\n", + portid, link.link_speed, (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? ("full-duplex") : ("half-duplex\n")); else - printf("Port %d Link Down\n", - (uint8_t)portid); + printf("Port %d Link Down\n", portid); continue; } /* clear all_ports_up flag if any link down */ @@ -654,6 +652,74 @@ check_all_ports_link_status(uint8_t port_num, uint32_t port_mask) } } +/* Check L3 packet type detection capablity of the NIC port */ +static int +check_ptype(int portid) +{ + int i, ret; + int ptype_l3_ipv4 = 0, ptype_l3_ipv6 = 0; + uint32_t ptype_mask = RTE_PTYPE_L3_MASK; + + ret = rte_eth_dev_get_supported_ptypes(portid, ptype_mask, NULL, 0); + if (ret <= 0) + return 0; + + uint32_t ptypes[ret]; + + ret = rte_eth_dev_get_supported_ptypes(portid, ptype_mask, ptypes, ret); + for (i = 0; i < ret; ++i) { + if (ptypes[i] & RTE_PTYPE_L3_IPV4) + ptype_l3_ipv4 = 1; + if (ptypes[i] & RTE_PTYPE_L3_IPV6) + ptype_l3_ipv6 = 1; + } + + if (ptype_l3_ipv4 == 0) + printf("port %d cannot parse RTE_PTYPE_L3_IPV4\n", portid); + + if (ptype_l3_ipv6 == 0) + printf("port %d cannot parse RTE_PTYPE_L3_IPV6\n", portid); + + if (ptype_l3_ipv4 && ptype_l3_ipv6) + return 1; + + return 0; + +} + +/* Parse packet type of a packet by SW */ +static inline void +parse_ptype(struct rte_mbuf *m) +{ + struct ether_hdr *eth_hdr; + uint32_t packet_type = RTE_PTYPE_UNKNOWN; + uint16_t ether_type; + + eth_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *); + ether_type = eth_hdr->ether_type; + if (ether_type == rte_cpu_to_be_16(ETHER_TYPE_IPv4)) + packet_type |= RTE_PTYPE_L3_IPV4_EXT_UNKNOWN; + else if (ether_type == rte_cpu_to_be_16(ETHER_TYPE_IPv6)) + packet_type |= RTE_PTYPE_L3_IPV6_EXT_UNKNOWN; + + m->packet_type = packet_type; +} + +/* callback function to detect packet type for a queue of a port */ +static uint16_t +cb_parse_ptype(uint16_t port __rte_unused, uint16_t queue __rte_unused, + struct rte_mbuf *pkts[], uint16_t nb_pkts, + uint16_t max_pkts __rte_unused, + void *user_param __rte_unused) +{ + uint16_t i; + + for (i = 0; i < nb_pkts; ++i) + parse_ptype(pkts[i]); + + return nb_pkts; +} + static int init_routing_table(void) { @@ -809,7 +875,7 @@ main(int argc, char **argv) uint16_t queueid = 0; unsigned lcore_id = 0, rx_lcore_id = 0; uint32_t n_tx_queue, nb_lcores; - uint8_t portid; + uint16_t portid; /* init EAL */ ret = rte_eal_init(argc, argv); @@ -847,6 +913,11 @@ main(int argc, char **argv) qconf = &lcore_queue_conf[rx_lcore_id]; + /* limit the frame size to the maximum supported by NIC */ + rte_eth_dev_info_get(portid, &dev_info); + port_conf.rxmode.max_rx_pkt_len = RTE_MIN( + dev_info.max_rx_pktlen, port_conf.rxmode.max_rx_pkt_len); + /* get the lcore_id for this port */ while (rte_lcore_is_enabled(rx_lcore_id) == 0 || qconf->n_rx_queue == (unsigned)rx_queue_per_lcore) { @@ -887,6 +958,14 @@ main(int argc, char **argv) ret, portid); } + ret = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd, + &nb_txd); + if (ret < 0) { + printf("\n"); + rte_exit(EXIT_FAILURE, "Cannot adjust number of " + "descriptors: err=%d, port=%d\n", ret, portid); + } + /* init one RX queue */ ret = rte_eth_rx_queue_setup(portid, 0, nb_rxd, socket, NULL, @@ -912,7 +991,6 @@ main(int argc, char **argv) printf("txq=%u,%d ", lcore_id, queueid); fflush(stdout); - rte_eth_dev_info_get(portid, &dev_info); txconf = &dev_info.default_txconf; txconf->txq_flags = 0; ret = rte_eth_tx_queue_setup(portid, queueid, nb_txd, @@ -945,12 +1023,18 @@ main(int argc, char **argv) ret, portid); rte_eth_promiscuous_enable(portid); + + if (check_ptype(portid) == 0) { + rte_eth_add_rx_callback(portid, 0, cb_parse_ptype, NULL); + printf("Add Rx callback function to detect L3 packet type by SW :" + " port = %d\n", portid); + } } if (init_routing_table() < 0) rte_exit(EXIT_FAILURE, "Cannot init routing table\n"); - check_all_ports_link_status((uint8_t)nb_ports, enabled_port_mask); + check_all_ports_link_status(nb_ports, enabled_port_mask); /* launch per-lcore init on every lcore */ rte_eal_mp_remote_launch(main_loop, NULL, CALL_MASTER); diff --git a/dpdk/examples/ip_pipeline/Makefile b/dpdk/examples/ip_pipeline/Makefile index 58271173..12ce0a1d 100644 --- a/dpdk/examples/ip_pipeline/Makefile +++ b/dpdk/examples/ip_pipeline/Makefile @@ -36,8 +36,6 @@ endif # Default target, can be overridden by command line or environment RTE_TARGET ?= x86_64-native-linuxapp-gcc -DIRS-(CONFIG_RTE_LIBRTE_PIPELINE) += pipeline - include $(RTE_SDK)/mk/rte.vars.mk # binary name @@ -45,7 +43,7 @@ APP = ip_pipeline VPATH += $(SRCDIR)/pipeline -INC += $(wildcard *.h) $(wildcard pipeline/*.h) +INC += $(sort $(wildcard *.h)) $(sort $(wildcard pipeline/*.h)) # all source are stored in SRCS-y SRCS-$(CONFIG_RTE_LIBRTE_PIPELINE) := main.c diff --git a/dpdk/examples/ip_pipeline/app.h b/dpdk/examples/ip_pipeline/app.h index 6a6fdd97..94e7a6df 100644 --- a/dpdk/examples/ip_pipeline/app.h +++ b/dpdk/examples/ip_pipeline/app.h @@ -177,19 +177,29 @@ struct app_pktq_tm_params { uint32_t burst_write; }; +struct app_pktq_tap_params { + char *name; + uint32_t parsed; + uint32_t burst_read; + uint32_t burst_write; + uint32_t dropless; + uint64_t n_retries; + uint32_t mempool_id; /* Position in the app->mempool_params */ +}; + struct app_pktq_source_params { char *name; uint32_t parsed; uint32_t mempool_id; /* Position in the app->mempool_params array */ uint32_t burst; - char *file_name; /* Full path of PCAP file to be copied to mbufs */ + const char *file_name; /* Full path of PCAP file to be copied to mbufs */ uint32_t n_bytes_per_pkt; }; struct app_pktq_sink_params { char *name; uint8_t parsed; - char *file_name; /* Full path of PCAP file to be copied to mbufs */ + const char *file_name; /* Full path of PCAP file to be copied to mbufs */ uint32_t n_pkts_to_dump; }; @@ -204,6 +214,7 @@ enum app_pktq_in_type { APP_PKTQ_IN_HWQ, APP_PKTQ_IN_SWQ, APP_PKTQ_IN_TM, + APP_PKTQ_IN_TAP, APP_PKTQ_IN_KNI, APP_PKTQ_IN_SOURCE, }; @@ -217,6 +228,7 @@ enum app_pktq_out_type { APP_PKTQ_OUT_HWQ, APP_PKTQ_OUT_SWQ, APP_PKTQ_OUT_TM, + APP_PKTQ_OUT_TAP, APP_PKTQ_OUT_KNI, APP_PKTQ_OUT_SINK, }; @@ -416,10 +428,6 @@ struct app_eal_params { /* Interrupt mode for VFIO (legacy|msi|msix) */ char *vfio_intr; - /* Support running on Xen dom0 without hugetlbfs */ - uint32_t xen_dom0_present; - int xen_dom0; - uint32_t parsed; }; @@ -441,6 +449,10 @@ struct app_eal_params { #define APP_MAX_PKTQ_TM APP_MAX_LINKS +#ifndef APP_MAX_PKTQ_TAP +#define APP_MAX_PKTQ_TAP APP_MAX_LINKS +#endif + #define APP_MAX_PKTQ_KNI APP_MAX_LINKS #ifndef APP_MAX_PKTQ_SOURCE @@ -475,6 +487,9 @@ struct app_eal_params { #define APP_THREAD_HEADROOM_STATS_COLLECT 1 #endif +#define APP_CORE_MASK_SIZE \ + (RTE_MAX_LCORE / 64 + ((RTE_MAX_LCORE % 64) ? 1 : 0)) + struct app_params { /* Config */ char app_name[APP_APPNAME_SIZE]; @@ -494,6 +509,7 @@ struct app_params { struct app_pktq_hwq_out_params hwq_out_params[APP_MAX_HWQ_OUT]; struct app_pktq_swq_params swq_params[APP_MAX_PKTQ_SWQ]; struct app_pktq_tm_params tm_params[APP_MAX_PKTQ_TM]; + struct app_pktq_tap_params tap_params[APP_MAX_PKTQ_TAP]; struct app_pktq_kni_params kni_params[APP_MAX_PKTQ_KNI]; struct app_pktq_source_params source_params[APP_MAX_PKTQ_SOURCE]; struct app_pktq_sink_params sink_params[APP_MAX_PKTQ_SINK]; @@ -506,6 +522,7 @@ struct app_params { uint32_t n_pktq_hwq_out; uint32_t n_pktq_swq; uint32_t n_pktq_tm; + uint32_t n_pktq_tap; uint32_t n_pktq_kni; uint32_t n_pktq_source; uint32_t n_pktq_sink; @@ -515,11 +532,12 @@ struct app_params { /* Init */ char *eal_argv[1 + APP_EAL_ARGC]; struct cpu_core_map *core_map; - uint64_t core_mask; + uint64_t core_mask[APP_CORE_MASK_SIZE]; struct rte_mempool *mempool[APP_MAX_MEMPOOLS]; struct app_link_data link_data[APP_MAX_LINKS]; struct rte_ring *swq[APP_MAX_PKTQ_SWQ]; struct rte_sched_port *tm[APP_MAX_PKTQ_TM]; + int tap[APP_MAX_PKTQ_TAP]; #ifdef RTE_LIBRTE_KNI struct rte_kni *kni[APP_MAX_PKTQ_KNI]; #endif /* RTE_LIBRTE_KNI */ @@ -785,6 +803,66 @@ app_tm_get_reader(struct app_params *app, return reader; } +static inline uint32_t +app_tap_get_readers(struct app_params *app, struct app_pktq_tap_params *tap) +{ + uint32_t pos = tap - app->tap_params; + uint32_t n_pipelines = RTE_MIN(app->n_pipelines, + RTE_DIM(app->pipeline_params)); + uint32_t n_readers = 0, i; + + for (i = 0; i < n_pipelines; i++) { + struct app_pipeline_params *p = &app->pipeline_params[i]; + uint32_t n_pktq_in = RTE_MIN(p->n_pktq_in, RTE_DIM(p->pktq_in)); + uint32_t j; + + for (j = 0; j < n_pktq_in; j++) { + struct app_pktq_in_params *pktq = &p->pktq_in[j]; + + if ((pktq->type == APP_PKTQ_IN_TAP) && + (pktq->id == pos)) + n_readers++; + } + } + + return n_readers; +} + +static inline struct app_pipeline_params * +app_tap_get_reader(struct app_params *app, + struct app_pktq_tap_params *tap, + uint32_t *pktq_in_id) +{ + struct app_pipeline_params *reader = NULL; + uint32_t pos = tap - app->tap_params; + uint32_t n_pipelines = RTE_MIN(app->n_pipelines, + RTE_DIM(app->pipeline_params)); + uint32_t n_readers = 0, id = 0, i; + + for (i = 0; i < n_pipelines; i++) { + struct app_pipeline_params *p = &app->pipeline_params[i]; + uint32_t n_pktq_in = RTE_MIN(p->n_pktq_in, RTE_DIM(p->pktq_in)); + uint32_t j; + + for (j = 0; j < n_pktq_in; j++) { + struct app_pktq_in_params *pktq = &p->pktq_in[j]; + + if ((pktq->type == APP_PKTQ_IN_TAP) && + (pktq->id == pos)) { + n_readers++; + reader = p; + id = j; + } + } + } + + if (n_readers != 1) + return NULL; + + *pktq_in_id = id; + return reader; +} + static inline uint32_t app_kni_get_readers(struct app_params *app, struct app_pktq_kni_params *kni) { @@ -1042,6 +1120,68 @@ app_tm_get_writer(struct app_params *app, return writer; } +static inline uint32_t +app_tap_get_writers(struct app_params *app, struct app_pktq_tap_params *tap) +{ + uint32_t pos = tap - app->tap_params; + uint32_t n_pipelines = RTE_MIN(app->n_pipelines, + RTE_DIM(app->pipeline_params)); + uint32_t n_writers = 0, i; + + for (i = 0; i < n_pipelines; i++) { + struct app_pipeline_params *p = &app->pipeline_params[i]; + uint32_t n_pktq_out = RTE_MIN(p->n_pktq_out, + RTE_DIM(p->pktq_out)); + uint32_t j; + + for (j = 0; j < n_pktq_out; j++) { + struct app_pktq_out_params *pktq = &p->pktq_out[j]; + + if ((pktq->type == APP_PKTQ_OUT_TAP) && + (pktq->id == pos)) + n_writers++; + } + } + + return n_writers; +} + +static inline struct app_pipeline_params * +app_tap_get_writer(struct app_params *app, + struct app_pktq_tap_params *tap, + uint32_t *pktq_out_id) +{ + struct app_pipeline_params *writer = NULL; + uint32_t pos = tap - app->tap_params; + uint32_t n_pipelines = RTE_MIN(app->n_pipelines, + RTE_DIM(app->pipeline_params)); + uint32_t n_writers = 0, id = 0, i; + + for (i = 0; i < n_pipelines; i++) { + struct app_pipeline_params *p = &app->pipeline_params[i]; + uint32_t n_pktq_out = RTE_MIN(p->n_pktq_out, + RTE_DIM(p->pktq_out)); + uint32_t j; + + for (j = 0; j < n_pktq_out; j++) { + struct app_pktq_out_params *pktq = &p->pktq_out[j]; + + if ((pktq->type == APP_PKTQ_OUT_TAP) && + (pktq->id == pos)) { + n_writers++; + writer = p; + id = j; + } + } + } + + if (n_writers != 1) + return NULL; + + *pktq_out_id = id; + return writer; +} + static inline uint32_t app_kni_get_writers(struct app_params *app, struct app_pktq_kni_params *kni) { @@ -1218,6 +1358,36 @@ app_get_link_for_kni(struct app_params *app, struct app_pktq_kni_params *p_kni) return &app->link_params[link_param_idx]; } +static inline uint32_t +app_core_is_enabled(struct app_params *app, uint32_t lcore_id) +{ + return(app->core_mask[lcore_id / 64] & + (1LLU << (lcore_id % 64))); +} + +static inline void +app_core_enable_in_core_mask(struct app_params *app, int lcore_id) +{ + app->core_mask[lcore_id / 64] |= 1LLU << (lcore_id % 64); + +} + +static inline void +app_core_build_core_mask_string(struct app_params *app, char *mask_buffer) +{ + int i; + + mask_buffer[0] = '\0'; + for (i = (int)RTE_DIM(app->core_mask); i > 0; i--) { + /* For Hex representation of bits in uint64_t */ + char buffer[(64 / 8) * 2 + 1]; + memset(buffer, 0, sizeof(buffer)); + snprintf(buffer, sizeof(buffer), "%016" PRIx64, + app->core_mask[i-1]); + strcat(mask_buffer, buffer); + } +} + void app_pipeline_params_get(struct app_params *app, struct app_pipeline_params *p_in, struct pipeline_params *p_out); diff --git a/dpdk/examples/ip_pipeline/config/diagram-generator.py b/dpdk/examples/ip_pipeline/config/diagram-generator.py index 6b7170b0..17488330 100644 --- a/dpdk/examples/ip_pipeline/config/diagram-generator.py +++ b/dpdk/examples/ip_pipeline/config/diagram-generator.py @@ -36,7 +36,8 @@ # the DPDK ip_pipeline application. # # The input configuration file is translated to an output file in DOT syntax, -# which is then used to create the image file using graphviz (www.graphviz.org). +# which is then used to create the image file using graphviz +# (www.graphviz.org). # from __future__ import print_function @@ -94,6 +95,7 @@ DOT_GRAPH_END = \ # SOURCEx | SOURCEx | SOURCEx | PIPELINEy | SOURCEx # SINKx | SINKx | PIPELINEy | SINKx | SINKx + # # Parse the input configuration file to detect the graph nodes and edges # @@ -321,16 +323,17 @@ def process_config_file(cfgfile): # print('Creating image file "%s" ...' % imgfile) if os.system('which dot > /dev/null'): - print('Error: Unable to locate "dot" executable.' \ - 'Please install the "graphviz" package (www.graphviz.org).') + print('Error: Unable to locate "dot" executable.' + 'Please install the "graphviz" package (www.graphviz.org).') return os.system(dot_cmd) if __name__ == '__main__': - parser = argparse.ArgumentParser(description=\ - 'Create diagram for IP pipeline configuration file.') + parser = argparse.ArgumentParser(description='Create diagram for IP ' + 'pipeline configuration ' + 'file.') parser.add_argument( '-f', diff --git a/dpdk/examples/ip_pipeline/config/network_layers.cfg b/dpdk/examples/ip_pipeline/config/network_layers.cfg index 8054d9fe..397b5d77 100644 --- a/dpdk/examples/ip_pipeline/config/network_layers.cfg +++ b/dpdk/examples/ip_pipeline/config/network_layers.cfg @@ -203,12 +203,16 @@ type = PASS-THROUGH ; Loop-back (UDP place-holder) core = 1 pktq_in = SWQ2 pktq_out = SWQ0 +swap = 282 286 ; IPSRC <-> IPDST +swap = 290 292 ; PORTSRC <-> PORTDST [PIPELINE5] type = PASS-THROUGH ; Loop-back (TCP place-holder) core = 1 pktq_in = SWQ3 pktq_out = SWQ1 +swap = 282 286 ; IPSRC <-> IPDST +swap = 290 292 ; PORTSRC <-> PORTDST [PIPELINE6] type = PASS-THROUGH ; Drop (ICMP place-holder) diff --git a/dpdk/examples/ip_pipeline/config/network_layers.sh b/dpdk/examples/ip_pipeline/config/network_layers.sh index 3b86bebd..449b0069 100644 --- a/dpdk/examples/ip_pipeline/config/network_layers.sh +++ b/dpdk/examples/ip_pipeline/config/network_layers.sh @@ -56,7 +56,7 @@ p 1 route add 100.3.0.0 16 port 3 ether 10.3.0.2 # Prio = 1 (High): [SA = ANY, DA = 10.1.0.1, SP = ANY, DP = 1001, PROTO = UDP] => Allow # Prio = 1 (High): [SA = ANY, DA = 10.2.0.1, SP = ANY, DP = 1002, PROTO = UDP] => Allow # Prio = 1 (High): [SA = ANY, DA = 10.3.0.1, SP = ANY, DP = 1003, PROTO = UDP] => Allow -p 1 firewall add default 1 #SINK0 +p 2 firewall add default 1 #SINK0 p 2 firewall add priority 1 ipv4 0.0.0.0 0 10.0.0.1 32 0 65535 1000 1000 17 0xF port 0 p 2 firewall add priority 1 ipv4 0.0.0.0 0 10.1.0.1 32 0 65535 1001 1001 17 0xF port 0 p 2 firewall add priority 1 ipv4 0.0.0.0 0 10.2.0.1 32 0 65535 1002 1002 17 0xF port 0 @@ -72,8 +72,8 @@ p 2 firewall add priority 1 ipv4 0.0.0.0 0 10.3.0.1 32 0 65535 1003 1003 17 0xF # TCP [SA = 100.2.0.10, DA = 10.2.0.1, SP = 1002, DP = 80] => socket ID = 2 # TCP [SA = 100.3.0.10, DA = 10.3.0.1, SP = 1003, DP = 80] => socket ID = 3 p 3 flow add default 1 #SINK1 -p 3 flow add ipv4 100.0.0.10 10.0.0.1 1000 80 6 port 1 id 0 -p 3 flow add ipv4 100.1.0.10 10.1.0.1 1001 80 6 port 1 id 1 -p 3 flow add ipv4 100.2.0.10 10.2.0.1 1002 80 6 port 1 id 2 -p 3 flow add ipv4 100.3.0.10 10.3.0.1 1003 80 6 port 1 id 3 +p 3 flow add ipv4 100.0.0.10 10.0.0.1 1000 80 6 port 0 id 0 +p 3 flow add ipv4 100.1.0.10 10.1.0.1 1001 80 6 port 0 id 1 +p 3 flow add ipv4 100.2.0.10 10.2.0.1 1002 80 6 port 0 id 2 +p 3 flow add ipv4 100.3.0.10 10.3.0.1 1003 80 6 port 0 id 3 #p 3 flow ls diff --git a/dpdk/examples/ip_pipeline/config/pipeline-to-core-mapping.py b/dpdk/examples/ip_pipeline/config/pipeline-to-core-mapping.py index c2050b82..7a4eaa20 100644 --- a/dpdk/examples/ip_pipeline/config/pipeline-to-core-mapping.py +++ b/dpdk/examples/ip_pipeline/config/pipeline-to-core-mapping.py @@ -39,15 +39,14 @@ # from __future__ import print_function -import sys +from collections import namedtuple +import argparse +import array import errno +import itertools import os import re -import array -import itertools -import re -import argparse -from collections import namedtuple +import sys # default values enable_stage0_traceout = 1 diff --git a/dpdk/examples/ip_pipeline/config/tap.cfg b/dpdk/examples/ip_pipeline/config/tap.cfg new file mode 100644 index 00000000..10d35ebb --- /dev/null +++ b/dpdk/examples/ip_pipeline/config/tap.cfg @@ -0,0 +1,64 @@ +; BSD LICENSE +; +; Copyright(c) 2016 Intel Corporation. All rights reserved. +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions +; are met: +; +; * Redistributions of source code must retain the above copyright +; notice, this list of conditions and the following disclaimer. +; * Redistributions in binary form must reproduce the above copyright +; notice, this list of conditions and the following disclaimer in +; the documentation and/or other materials provided with the +; distribution. +; * Neither the name of Intel Corporation nor the names of its +; contributors may be used to endorse or promote products derived +; from this software without specific prior written permission. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +; ______________ ______________________ +; | | TAP0 | | +; RXQ0.0 --->| |------->|--+ | +; | | TAP1 | | br0 | +; TXQ1.0 <---| |<-------|<-+ | +; | Pass-through | | Linux Kernel | +; | (P1) | | Network Stack | +; | | TAP1 | | +; RXQ1.0 --->| |------->|--+ | +; | | TAP0 | | br0 | +; TXQ0.0 <---| |<-------|<-+ | +; |______________| |______________________| +; +; Configure Linux kernel bridge between TAP0 and TAP1 interfaces: +; [Linux]$ ifconfig TAP0 up +; [Linux]$ ifconfig TAP1 up +; [Linux]$ brctl addbr "br0" +; [Linux]$ brctl addif br0 TAP0 +; [Linux]$ brctl addif br0 TAP1 +; [Linux]$ ifconfig br0 up + +[EAL] +log_level = 0 + +[PIPELINE0] +type = MASTER +core = 0 + +[PIPELINE1] +type = PASS-THROUGH +core = 1 +pktq_in = RXQ0.0 TAP1 RXQ1.0 TAP0 +pktq_out = TAP0 TXQ1.0 TAP1 TXQ0.0 diff --git a/dpdk/examples/ip_pipeline/config_check.c b/dpdk/examples/ip_pipeline/config_check.c index af1b6284..dd9d4d8b 100644 --- a/dpdk/examples/ip_pipeline/config_check.c +++ b/dpdk/examples/ip_pipeline/config_check.c @@ -315,6 +315,36 @@ check_tms(struct app_params *app) } } +static void +check_taps(struct app_params *app) +{ + uint32_t i; + + for (i = 0; i < app->n_pktq_tap; i++) { + struct app_pktq_tap_params *p = &app->tap_params[i]; + uint32_t n_readers = app_tap_get_readers(app, p); + uint32_t n_writers = app_tap_get_writers(app, p); + + APP_CHECK((n_readers != 0), + "%s has no reader\n", p->name); + + APP_CHECK((n_readers == 1), + "%s has more than one reader\n", p->name); + + APP_CHECK((n_writers != 0), + "%s has no writer\n", p->name); + + APP_CHECK((n_writers == 1), + "%s has more than one writer\n", p->name); + + APP_CHECK((p->burst_read > 0), + "%s read burst size is 0\n", p->name); + + APP_CHECK((p->burst_write > 0), + "%s write burst size is 0\n", p->name); + } +} + static void check_knis(struct app_params *app) { uint32_t i; @@ -476,6 +506,7 @@ app_config_check(struct app_params *app) check_txqs(app); check_swqs(app); check_tms(app); + check_taps(app); check_knis(app); check_sources(app); check_sinks(app); diff --git a/dpdk/examples/ip_pipeline/config_parse.c b/dpdk/examples/ip_pipeline/config_parse.c index 8fe81577..3211c6ab 100644 --- a/dpdk/examples/ip_pipeline/config_parse.c +++ b/dpdk/examples/ip_pipeline/config_parse.c @@ -1,4 +1,4 @@ -/*- +/*- * BSD LICENSE * * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. @@ -103,7 +103,7 @@ static const struct app_link_params link_params_default = { .hw_vlan_strip = 0, /* VLAN strip */ .hw_vlan_extend = 0, /* Extended VLAN */ .jumbo_frame = 0, /* Jumbo frame support */ - .hw_strip_crc = 0, /* CRC strip by HW */ + .hw_strip_crc = 1, /* CRC strip by HW */ .enable_scatter = 0, /* Scattered packets RX handler */ .max_rx_pkt_len = 9000, /* Jumbo frame max packet len */ @@ -189,6 +189,15 @@ struct app_pktq_tm_params default_tm_params = { .burst_write = 32, }; +struct app_pktq_tap_params default_tap_params = { + .parsed = 0, + .burst_read = 32, + .burst_write = 32, + .dropless = 0, + .n_retries = 0, + .mempool_id = 0, +}; + struct app_pktq_kni_params default_kni_params = { .parsed = 0, .socket_id = 0, @@ -207,7 +216,7 @@ struct app_pktq_source_params default_source_params = { .parsed = 0, .mempool_id = 0, .burst = 32, - .file_name = NULL, + .file_name = "./config/packets.pcap", .n_bytes_per_pkt = 0, }; @@ -800,21 +809,6 @@ parse_eal(struct app_params *app, continue; } - /* xen_dom0 */ - if (strcmp(entry->name, "xen_dom0") == 0) { - int val; - - PARSE_ERROR_DUPLICATE((p->xen_dom0_present == 0), - section_name, - entry->name); - p->xen_dom0_present = 1; - - val = parser_read_arg_bool(entry->value); - PARSE_ERROR((val >= 0), section_name, entry->name); - p->xen_dom0 = val; - continue; - } - /* unrecognized */ PARSE_ERROR_INVALID(0, section_name, entry->name); } @@ -852,6 +846,9 @@ parse_pipeline_pktq_in(struct app_params *app, type = APP_PKTQ_IN_TM; id = APP_PARAM_ADD(app->tm_params, name); APP_PARAM_ADD_LINK_FOR_TM(app, name); + } else if (validate_name(name, "TAP", 1) == 0) { + type = APP_PKTQ_IN_TAP; + id = APP_PARAM_ADD(app->tap_params, name); } else if (validate_name(name, "KNI", 1) == 0) { type = APP_PKTQ_IN_KNI; id = APP_PARAM_ADD(app->kni_params, name); @@ -901,6 +898,9 @@ parse_pipeline_pktq_out(struct app_params *app, type = APP_PKTQ_OUT_TM; id = APP_PARAM_ADD(app->tm_params, name); APP_PARAM_ADD_LINK_FOR_TM(app, name); + } else if (validate_name(name, "TAP", 1) == 0) { + type = APP_PKTQ_OUT_TAP; + id = APP_PARAM_ADD(app->tap_params, name); } else if (validate_name(name, "KNI", 1) == 0) { type = APP_PKTQ_OUT_KNI; id = APP_PARAM_ADD(app->kni_params, name); @@ -1895,6 +1895,88 @@ parse_tm(struct app_params *app, free(entries); } +static void +parse_tap(struct app_params *app, + const char *section_name, + struct rte_cfgfile *cfg) +{ + struct app_pktq_tap_params *param; + struct rte_cfgfile_entry *entries; + int n_entries, i; + ssize_t param_idx; + + n_entries = rte_cfgfile_section_num_entries(cfg, section_name); + PARSE_ERROR_SECTION_NO_ENTRIES((n_entries > 0), section_name); + + entries = malloc(n_entries * sizeof(struct rte_cfgfile_entry)); + PARSE_ERROR_MALLOC(entries != NULL); + + rte_cfgfile_section_entries(cfg, section_name, entries, n_entries); + + param_idx = APP_PARAM_ADD(app->tap_params, section_name); + param = &app->tap_params[param_idx]; + PARSE_CHECK_DUPLICATE_SECTION(param); + + for (i = 0; i < n_entries; i++) { + struct rte_cfgfile_entry *ent = &entries[i]; + + if (strcmp(ent->name, "burst_read") == 0) { + int status = parser_read_uint32( + ¶m->burst_read, ent->value); + + PARSE_ERROR((status == 0), section_name, + ent->name); + continue; + } + + if (strcmp(ent->name, "burst_write") == 0) { + int status = parser_read_uint32( + ¶m->burst_write, ent->value); + + PARSE_ERROR((status == 0), section_name, + ent->name); + continue; + } + + if (strcmp(ent->name, "dropless") == 0) { + int status = parser_read_arg_bool(ent->value); + + PARSE_ERROR((status != -EINVAL), section_name, + ent->name); + param->dropless = status; + continue; + } + + if (strcmp(ent->name, "n_retries") == 0) { + int status = parser_read_uint64(¶m->n_retries, + ent->value); + + PARSE_ERROR((status == 0), section_name, + ent->name); + continue; + } + + if (strcmp(ent->name, "mempool") == 0) { + int status = validate_name(ent->value, + "MEMPOOL", 1); + ssize_t idx; + + PARSE_ERROR((status == 0), section_name, + ent->name); + + idx = APP_PARAM_ADD(app->mempool_params, ent->value); + param->mempool_id = idx; + + continue; + } + + /* unrecognized */ + PARSE_ERROR_INVALID(0, section_name, ent->name); + } + + free(entries); +} + static void parse_kni(struct app_params *app, const char *section_name, @@ -2286,6 +2368,7 @@ static const struct config_section cfg_file_scheme[] = { {"TXQ", 2, parse_txq}, {"SWQ", 1, parse_swq}, {"TM", 1, parse_tm}, + {"TAP", 1, parse_tap}, {"KNI", 1, parse_kni}, {"SOURCE", 1, parse_source}, {"SINK", 1, parse_sink}, @@ -2425,6 +2508,7 @@ app_config_parse(struct app_params *app, const char *file_name) APP_PARAM_COUNT(app->hwq_out_params, app->n_pktq_hwq_out); APP_PARAM_COUNT(app->swq_params, app->n_pktq_swq); APP_PARAM_COUNT(app->tm_params, app->n_pktq_tm); + APP_PARAM_COUNT(app->tap_params, app->n_pktq_tap); APP_PARAM_COUNT(app->kni_params, app->n_pktq_kni); APP_PARAM_COUNT(app->source_params, app->n_pktq_source); APP_PARAM_COUNT(app->sink_params, app->n_pktq_sink); @@ -2544,10 +2628,6 @@ save_eal_params(struct app_params *app, FILE *f) if (p->vfio_intr) fprintf(f, "%s = %s\n", "vfio_intr", p->vfio_intr); - if (p->xen_dom0_present) - fprintf(f, "%s = %s\n", "xen_dom0", - (p->xen_dom0) ? "yes" : "no"); - fputc('\n', f); } @@ -2788,6 +2868,30 @@ save_tm_params(struct app_params *app, FILE *f) } } +static void +save_tap_params(struct app_params *app, FILE *f) +{ + struct app_pktq_tap_params *p; + size_t i, count; + + count = RTE_DIM(app->tap_params); + for (i = 0; i < count; i++) { + p = &app->tap_params[i]; + if (!APP_PARAM_VALID(p)) + continue; + + fprintf(f, "[%s]\n", p->name); + fprintf(f, "%s = %" PRIu32 "\n", "burst_read", p->burst_read); + fprintf(f, "%s = %" PRIu32 "\n", "burst_write", p->burst_write); + fprintf(f, "%s = %s\n", "dropless", p->dropless ? "yes" : "no"); + fprintf(f, "%s = %" PRIu64 "\n", "n_retries", p->n_retries); + fprintf(f, "%s = %s\n", "mempool", + app->mempool_params[p->mempool_id].name); + + fputc('\n', f); + } +} + static void save_kni_params(struct app_params *app, FILE *f) { @@ -2942,6 +3046,9 @@ save_pipeline_params(struct app_params *app, FILE *f) case APP_PKTQ_IN_TM: name = app->tm_params[pp->id].name; break; + case APP_PKTQ_IN_TAP: + name = app->tap_params[pp->id].name; + break; case APP_PKTQ_IN_KNI: name = app->kni_params[pp->id].name; break; @@ -2979,6 +3086,9 @@ save_pipeline_params(struct app_params *app, FILE *f) case APP_PKTQ_OUT_TM: name = app->tm_params[pp->id].name; break; + case APP_PKTQ_OUT_TAP: + name = app->tap_params[pp->id].name; + break; case APP_PKTQ_OUT_KNI: name = app->kni_params[pp->id].name; break; @@ -3067,6 +3177,7 @@ app_config_save(struct app_params *app, const char *file_name) save_txq_params(app, file); save_swq_params(app, file); save_tm_params(app, file); + save_tap_params(app, file); save_kni_params(app, file); save_source_params(app, file); save_sink_params(app, file); @@ -3083,10 +3194,6 @@ app_config_init(struct app_params *app) memcpy(app, &app_params_default, sizeof(struct app_params)); - /* configure default_source_params */ - default_source_params.file_name = strdup("./config/packets.pcap"); - PARSE_ERROR_MALLOC(default_source_params.file_name != NULL); - for (i = 0; i < RTE_DIM(app->mempool_params); i++) memcpy(&app->mempool_params[i], &mempool_params_default, @@ -3117,6 +3224,11 @@ app_config_init(struct app_params *app) &default_tm_params, sizeof(default_tm_params)); + for (i = 0; i < RTE_DIM(app->tap_params); i++) + memcpy(&app->tap_params[i], + &default_tap_params, + sizeof(default_tap_params)); + for (i = 0; i < RTE_DIM(app->kni_params); i++) memcpy(&app->kni_params[i], &default_kni_params, @@ -3276,7 +3388,7 @@ app_config_args(struct app_params *app, int argc, char **argv) app_print_usage(argv[0]); } - optind = 0; /* reset getopt lib */ + optind = 1; /* reset getopt lib */ /* Check dependencies between args */ if (preproc_params_present && (preproc_present == 0)) diff --git a/dpdk/examples/ip_pipeline/init.c b/dpdk/examples/ip_pipeline/init.c index 891b3274..ffd0fc25 100644 --- a/dpdk/examples/ip_pipeline/init.c +++ b/dpdk/examples/ip_pipeline/init.c @@ -34,6 +34,14 @@ #include #include #include +#include +#ifdef RTE_EXEC_ENV_LINUXAPP +#include +#include +#endif +#include +#include +#include #include #include @@ -41,6 +49,7 @@ #include #include #include +#include #include "app.h" #include "pipeline.h" @@ -61,7 +70,8 @@ static void app_init_core_map(struct app_params *app) { APP_LOG(app, HIGH, "Initializing CPU core map ..."); - app->core_map = cpu_core_map_init(4, 32, 4, 0); + app->core_map = cpu_core_map_init(RTE_MAX_NUMA_NODES, RTE_MAX_LCORE, + 4, 0); if (app->core_map == NULL) rte_panic("Cannot create CPU core map\n"); @@ -70,11 +80,14 @@ app_init_core_map(struct app_params *app) cpu_core_map_print(app->core_map); } +/* Core Mask String in Hex Representation */ +#define APP_CORE_MASK_STRING_SIZE ((64 * APP_CORE_MASK_SIZE) / 8 * 2 + 1) + static void app_init_core_mask(struct app_params *app) { - uint64_t mask = 0; uint32_t i; + char core_mask_str[APP_CORE_MASK_STRING_SIZE]; for (i = 0; i < app->n_pipelines; i++) { struct app_pipeline_params *p = &app->pipeline_params[i]; @@ -88,17 +101,18 @@ app_init_core_mask(struct app_params *app) if (lcore_id < 0) rte_panic("Cannot create CPU core mask\n"); - mask |= 1LLU << lcore_id; + app_core_enable_in_core_mask(app, lcore_id); } - app->core_mask = mask; - APP_LOG(app, HIGH, "CPU core mask = 0x%016" PRIx64, app->core_mask); + app_core_build_core_mask_string(app, core_mask_str); + APP_LOG(app, HIGH, "CPU core mask = 0x%s", core_mask_str); } static void app_init_eal(struct app_params *app) { char buffer[256]; + char core_mask_str[APP_CORE_MASK_STRING_SIZE]; struct app_eal_params *p = &app->eal_params; uint32_t n_args = 0; uint32_t i; @@ -106,7 +120,8 @@ app_init_eal(struct app_params *app) app->eal_argv[n_args++] = strdup(app->app_name); - snprintf(buffer, sizeof(buffer), "-c%" PRIx64, app->core_mask); + app_core_build_core_mask_string(app, core_mask_str); + snprintf(buffer, sizeof(buffer), "-c%s", core_mask_str); app->eal_argv[n_args++] = strdup(buffer); if (p->coremap) { @@ -282,11 +297,6 @@ app_init_eal(struct app_params *app) app->eal_argv[n_args++] = strdup(buffer); } - if ((p->xen_dom0_present) && (p->xen_dom0)) { - snprintf(buffer, sizeof(buffer), "--xen-dom0"); - app->eal_argv[n_args++] = strdup(buffer); - } - snprintf(buffer, sizeof(buffer), "--"); app->eal_argv[n_args++] = strdup(buffer); @@ -316,16 +326,14 @@ app_init_mempool(struct app_params *app) struct app_mempool_params *p = &app->mempool_params[i]; APP_LOG(app, HIGH, "Initializing %s ...", p->name); - app->mempool[i] = rte_mempool_create( - p->name, - p->pool_size, - p->buffer_size, - p->cache_size, - sizeof(struct rte_pktmbuf_pool_private), - rte_pktmbuf_pool_init, NULL, - rte_pktmbuf_init, NULL, - p->cpu_socket_id, - 0); + app->mempool[i] = rte_pktmbuf_pool_create( + p->name, + p->pool_size, + p->cache_size, + 0, /* priv_size */ + p->buffer_size - + sizeof(struct rte_mbuf), /* mbuf data size */ + p->cpu_socket_id); if (app->mempool[i] == NULL) rte_panic("%s init error\n", p->name); @@ -606,29 +614,12 @@ app_link_set_tcp_syn_filter(struct app_params *app, struct app_link_params *cp) } } -static int -app_link_is_virtual(struct app_link_params *p) -{ - uint32_t pmd_id = p->pmd_id; - struct rte_eth_dev *dev = &rte_eth_devices[pmd_id]; - - if (dev->dev_type == RTE_ETH_DEV_VIRTUAL) - return 1; - - return 0; -} - void app_link_up_internal(struct app_params *app, struct app_link_params *cp) { uint32_t i; int status; - if (app_link_is_virtual(cp)) { - cp->state = 1; - return; - } - /* For each link, add filters for IP of current link */ if (cp->ip != 0) { for (i = 0; i < app->n_links; i++) { @@ -722,7 +713,8 @@ app_link_up_internal(struct app_params *app, struct app_link_params *cp) /* PMD link up */ status = rte_eth_dev_set_link_up(cp->pmd_id); - if (status < 0) + /* Do not panic if PMD does not provide link up functionality */ + if (status < 0 && status != -ENOTSUP) rte_panic("%s (%" PRIu32 "): PMD set link up error %" PRId32 "\n", cp->name, cp->pmd_id, status); @@ -736,14 +728,10 @@ app_link_down_internal(struct app_params *app, struct app_link_params *cp) uint32_t i; int status; - if (app_link_is_virtual(cp)) { - cp->state = 0; - return; - } - /* PMD link down */ status = rte_eth_dev_set_link_down(cp->pmd_id); - if (status < 0) + /* Do not panic if PMD does not provide link down functionality */ + if (status < 0 && status != -ENOTSUP) rte_panic("%s (%" PRIu32 "): PMD set link down error %" PRId32 "\n", cp->name, cp->pmd_id, status); @@ -1011,16 +999,30 @@ app_init_link(struct app_params *app) struct app_pktq_hwq_in_params *p_rxq = &app->hwq_in_params[j]; uint32_t rxq_link_id, rxq_queue_id; + uint16_t nb_rxd = p_rxq->size; sscanf(p_rxq->name, "RXQ%" PRIu32 ".%" PRIu32, &rxq_link_id, &rxq_queue_id); if (rxq_link_id != link_id) continue; + status = rte_eth_dev_adjust_nb_rx_tx_desc( + p_link->pmd_id, + &nb_rxd, + NULL); + if (status < 0) + rte_panic("%s (%" PRIu32 "): " + "%s adjust number of Rx descriptors " + "error (%" PRId32 ")\n", + p_link->name, + p_link->pmd_id, + p_rxq->name, + status); + status = rte_eth_rx_queue_setup( p_link->pmd_id, rxq_queue_id, - p_rxq->size, + nb_rxd, app_get_cpu_socket_id(p_link->pmd_id), &p_rxq->conf, app->mempool[p_rxq->mempool_id]); @@ -1038,16 +1040,30 @@ app_init_link(struct app_params *app) struct app_pktq_hwq_out_params *p_txq = &app->hwq_out_params[j]; uint32_t txq_link_id, txq_queue_id; + uint16_t nb_txd = p_txq->size; sscanf(p_txq->name, "TXQ%" PRIu32 ".%" PRIu32, &txq_link_id, &txq_queue_id); if (txq_link_id != link_id) continue; + status = rte_eth_dev_adjust_nb_rx_tx_desc( + p_link->pmd_id, + NULL, + &nb_txd); + if (status < 0) + rte_panic("%s (%" PRIu32 "): " + "%s adjust number of Tx descriptors " + "error (%" PRId32 ")\n", + p_link->name, + p_link->pmd_id, + p_txq->name, + status); + status = rte_eth_tx_queue_setup( p_link->pmd_id, txq_queue_id, - p_txq->size, + nb_txd, app_get_cpu_socket_id(p_link->pmd_id), &p_txq->conf); if (status < 0) @@ -1176,9 +1192,47 @@ app_init_tm(struct app_params *app) } } +#ifndef RTE_EXEC_ENV_LINUXAPP +static void +app_init_tap(struct app_params *app) { + if (app->n_pktq_tap == 0) + return; + + rte_panic("TAP device not supported.\n"); +} +#else +static void +app_init_tap(struct app_params *app) +{ + uint32_t i; + + for (i = 0; i < app->n_pktq_tap; i++) { + struct app_pktq_tap_params *p_tap = &app->tap_params[i]; + struct ifreq ifr; + int fd, status; + + APP_LOG(app, HIGH, "Initializing %s ...", p_tap->name); + + fd = open("/dev/net/tun", O_RDWR | O_NONBLOCK); + if (fd < 0) + rte_panic("Cannot open file /dev/net/tun\n"); + + memset(&ifr, 0, sizeof(ifr)); + ifr.ifr_flags = IFF_TAP | IFF_NO_PI; /* No packet information */ + snprintf(ifr.ifr_name, IFNAMSIZ, "%s", p_tap->name); + + status = ioctl(fd, TUNSETIFF, (void *) &ifr); + if (status < 0) + rte_panic("TAP setup error\n"); + + app->tap[i] = fd; + } +} +#endif + #ifdef RTE_LIBRTE_KNI static int -kni_config_network_interface(uint8_t port_id, uint8_t if_up) { +kni_config_network_interface(uint16_t port_id, uint8_t if_up) { int ret = 0; if (port_id >= rte_eth_dev_count()) @@ -1192,7 +1246,7 @@ kni_config_network_interface(uint8_t port_id, uint8_t if_up) { } static int -kni_change_mtu(uint8_t port_id, unsigned new_mtu) { +kni_change_mtu(uint16_t port_id, unsigned int new_mtu) { int ret; if (port_id >= rte_eth_dev_count()) @@ -1392,6 +1446,24 @@ void app_pipeline_params_get(struct app_params *app, out->burst_size = app->tm_params[in->id].burst_read; break; } +#ifdef RTE_EXEC_ENV_LINUXAPP + case APP_PKTQ_IN_TAP: + { + struct app_pktq_tap_params *tap_params = + &app->tap_params[in->id]; + struct app_mempool_params *mempool_params = + &app->mempool_params[tap_params->mempool_id]; + struct rte_mempool *mempool = + app->mempool[tap_params->mempool_id]; + + out->type = PIPELINE_PORT_IN_FD_READER; + out->params.fd.fd = app->tap[in->id]; + out->params.fd.mtu = mempool_params->buffer_size; + out->params.fd.mempool = mempool; + out->burst_size = app->tap_params[in->id].burst_read; + break; + } +#endif #ifdef RTE_LIBRTE_KNI case APP_PKTQ_IN_KNI: { @@ -1536,6 +1608,19 @@ void app_pipeline_params_get(struct app_params *app, app->tm_params[in->id].burst_write; break; } +#ifdef RTE_EXEC_ENV_LINUXAPP + case APP_PKTQ_OUT_TAP: + { + struct rte_port_fd_writer_params *params = + &out->params.fd; + + out->type = PIPELINE_PORT_OUT_FD_WRITER; + params->fd = app->tap[in->id]; + params->tx_burst_sz = + app->tap_params[in->id].burst_write; + break; + } +#endif #ifdef RTE_LIBRTE_KNI case APP_PKTQ_OUT_KNI: { @@ -1641,7 +1726,7 @@ app_init_pipelines(struct app_params *app) data->ptype = ptype; data->timer_period = (rte_get_tsc_hz() * - params->timer_period) / 100; + params->timer_period) / 1000; } } @@ -1752,6 +1837,7 @@ int app_init(struct app_params *app) app_init_link(app); app_init_swq(app); app_init_tm(app); + app_init_tap(app); app_init_kni(app); app_init_msgq(app); diff --git a/dpdk/examples/ip_pipeline/pipeline/hash_func.h b/dpdk/examples/ip_pipeline/pipeline/hash_func.h index 9db7173f..42128c1f 100644 --- a/dpdk/examples/ip_pipeline/pipeline/hash_func.h +++ b/dpdk/examples/ip_pipeline/pipeline/hash_func.h @@ -34,48 +34,56 @@ #define __INCLUDE_HASH_FUNC_H__ static inline uint64_t -hash_xor_key8(void *key, __rte_unused uint32_t key_size, uint64_t seed) +hash_xor_key8(void *key, void *mask, __rte_unused uint32_t key_size, + uint64_t seed) { uint64_t *k = key; + uint64_t *m = mask; uint64_t xor0; - xor0 = seed ^ k[0]; + xor0 = seed ^ (k[0] & m[0]); return (xor0 >> 32) ^ xor0; } static inline uint64_t -hash_xor_key16(void *key, __rte_unused uint32_t key_size, uint64_t seed) +hash_xor_key16(void *key, void *mask, __rte_unused uint32_t key_size, + uint64_t seed) { uint64_t *k = key; + uint64_t *m = mask; uint64_t xor0; - xor0 = (k[0] ^ seed) ^ k[1]; + xor0 = ((k[0] & m[0]) ^ seed) ^ (k[1] & m[1]); return (xor0 >> 32) ^ xor0; } static inline uint64_t -hash_xor_key24(void *key, __rte_unused uint32_t key_size, uint64_t seed) +hash_xor_key24(void *key, void *mask, __rte_unused uint32_t key_size, + uint64_t seed) { uint64_t *k = key; + uint64_t *m = mask; uint64_t xor0; - xor0 = (k[0] ^ seed) ^ k[1]; + xor0 = ((k[0] & m[0]) ^ seed) ^ (k[1] & m[1]); - xor0 ^= k[2]; + xor0 ^= k[2] & m[2]; return (xor0 >> 32) ^ xor0; } static inline uint64_t -hash_xor_key32(void *key, __rte_unused uint32_t key_size, uint64_t seed) +hash_xor_key32(void *key, void *mask, __rte_unused uint32_t key_size, + uint64_t seed) { uint64_t *k = key; + uint64_t *m = mask; uint64_t xor0, xor1; - xor0 = (k[0] ^ seed) ^ k[1]; - xor1 = k[2] ^ k[3]; + xor0 = ((k[0] & m[0]) ^ seed) ^ (k[1] & m[1]); + xor1 = (k[2] & m[2]) ^ (k[3] & m[3]); xor0 ^= xor1; @@ -83,30 +91,34 @@ hash_xor_key32(void *key, __rte_unused uint32_t key_size, uint64_t seed) } static inline uint64_t -hash_xor_key40(void *key, __rte_unused uint32_t key_size, uint64_t seed) +hash_xor_key40(void *key, void *mask, __rte_unused uint32_t key_size, + uint64_t seed) { uint64_t *k = key; + uint64_t *m = mask; uint64_t xor0, xor1; - xor0 = (k[0] ^ seed) ^ k[1]; - xor1 = k[2] ^ k[3]; + xor0 = ((k[0] & m[0]) ^ seed) ^ (k[1] & m[1]); + xor1 = (k[2] & m[2]) ^ (k[3] & m[3]); xor0 ^= xor1; - xor0 ^= k[4]; + xor0 ^= k[4] & m[4]; return (xor0 >> 32) ^ xor0; } static inline uint64_t -hash_xor_key48(void *key, __rte_unused uint32_t key_size, uint64_t seed) +hash_xor_key48(void *key, void *mask, __rte_unused uint32_t key_size, + uint64_t seed) { uint64_t *k = key; + uint64_t *m = mask; uint64_t xor0, xor1, xor2; - xor0 = (k[0] ^ seed) ^ k[1]; - xor1 = k[2] ^ k[3]; - xor2 = k[4] ^ k[5]; + xor0 = ((k[0] & m[0]) ^ seed) ^ (k[1] & m[1]); + xor1 = (k[2] & m[2]) ^ (k[3] & m[3]); + xor2 = (k[4] & m[4]) ^ (k[5] & m[5]); xor0 ^= xor1; @@ -116,17 +128,19 @@ hash_xor_key48(void *key, __rte_unused uint32_t key_size, uint64_t seed) } static inline uint64_t -hash_xor_key56(void *key, __rte_unused uint32_t key_size, uint64_t seed) +hash_xor_key56(void *key, void *mask, __rte_unused uint32_t key_size, + uint64_t seed) { uint64_t *k = key; + uint64_t *m = mask; uint64_t xor0, xor1, xor2; - xor0 = (k[0] ^ seed) ^ k[1]; - xor1 = k[2] ^ k[3]; - xor2 = k[4] ^ k[5]; + xor0 = ((k[0] & m[0]) ^ seed) ^ (k[1] & m[1]); + xor1 = (k[2] & m[2]) ^ (k[3] & m[3]); + xor2 = (k[4] & m[4]) ^ (k[5] & m[5]); xor0 ^= xor1; - xor2 ^= k[6]; + xor2 ^= k[6] & m[6]; xor0 ^= xor2; @@ -134,15 +148,17 @@ hash_xor_key56(void *key, __rte_unused uint32_t key_size, uint64_t seed) } static inline uint64_t -hash_xor_key64(void *key, __rte_unused uint32_t key_size, uint64_t seed) +hash_xor_key64(void *key, void *mask, __rte_unused uint32_t key_size, + uint64_t seed) { uint64_t *k = key; + uint64_t *m = mask; uint64_t xor0, xor1, xor2, xor3; - xor0 = (k[0] ^ seed) ^ k[1]; - xor1 = k[2] ^ k[3]; - xor2 = k[4] ^ k[5]; - xor3 = k[6] ^ k[7]; + xor0 = ((k[0] & m[0]) ^ seed) ^ (k[1] & m[1]); + xor1 = (k[2] & m[2]) ^ (k[3] & m[3]); + xor2 = (k[4] & m[4]) ^ (k[5] & m[5]); + xor3 = (k[6] & m[6]) ^ (k[7] & m[7]); xor0 ^= xor1; xor2 ^= xor3; @@ -152,31 +168,35 @@ hash_xor_key64(void *key, __rte_unused uint32_t key_size, uint64_t seed) return (xor0 >> 32) ^ xor0; } -#if defined(RTE_ARCH_X86_64) && defined(RTE_MACHINE_CPUFLAG_SSE4_2) +#if defined(RTE_ARCH_X86_64) #include static inline uint64_t -hash_crc_key8(void *key, __rte_unused uint32_t key_size, uint64_t seed) +hash_crc_key8(void *key, void *mask, __rte_unused uint32_t key_size, + uint64_t seed) { uint64_t *k = key; + uint64_t *m = mask; uint64_t crc0; - crc0 = _mm_crc32_u64(seed, k[0]); + crc0 = _mm_crc32_u64(seed, k[0] & m[0]); return crc0; } static inline uint64_t -hash_crc_key16(void *key, __rte_unused uint32_t key_size, uint64_t seed) +hash_crc_key16(void *key, void *mask, __rte_unused uint32_t key_size, + uint64_t seed) { uint64_t *k = key; + uint64_t *m = mask; uint64_t k0, crc0, crc1; - k0 = k[0]; + k0 = k[0] & m[0]; crc0 = _mm_crc32_u64(k0, seed); - crc1 = _mm_crc32_u64(k0 >> 32, k[1]); + crc1 = _mm_crc32_u64(k0 >> 32, k[1] & m[1]); crc0 ^= crc1; @@ -184,16 +204,18 @@ hash_crc_key16(void *key, __rte_unused uint32_t key_size, uint64_t seed) } static inline uint64_t -hash_crc_key24(void *key, __rte_unused uint32_t key_size, uint64_t seed) +hash_crc_key24(void *key, void *mask, __rte_unused uint32_t key_size, + uint64_t seed) { uint64_t *k = key; + uint64_t *m = mask; uint64_t k0, k2, crc0, crc1; - k0 = k[0]; - k2 = k[2]; + k0 = k[0] & m[0]; + k2 = k[2] & m[2]; crc0 = _mm_crc32_u64(k0, seed); - crc1 = _mm_crc32_u64(k0 >> 32, k[1]); + crc1 = _mm_crc32_u64(k0 >> 32, k[1] & m[1]); crc0 = _mm_crc32_u64(crc0, k2); @@ -203,18 +225,20 @@ hash_crc_key24(void *key, __rte_unused uint32_t key_size, uint64_t seed) } static inline uint64_t -hash_crc_key32(void *key, __rte_unused uint32_t key_size, uint64_t seed) +hash_crc_key32(void *key, void *mask, __rte_unused uint32_t key_size, + uint64_t seed) { uint64_t *k = key; + uint64_t *m = mask; uint64_t k0, k2, crc0, crc1, crc2, crc3; - k0 = k[0]; - k2 = k[2]; + k0 = k[0] & m[0]; + k2 = k[2] & m[2]; crc0 = _mm_crc32_u64(k0, seed); - crc1 = _mm_crc32_u64(k0 >> 32, k[1]); + crc1 = _mm_crc32_u64(k0 >> 32, k[1] & m[1]); - crc2 = _mm_crc32_u64(k2, k[3]); + crc2 = _mm_crc32_u64(k2, k[3] & m[3]); crc3 = k2 >> 32; crc0 = _mm_crc32_u64(crc0, crc1); @@ -226,19 +250,21 @@ hash_crc_key32(void *key, __rte_unused uint32_t key_size, uint64_t seed) } static inline uint64_t -hash_crc_key40(void *key, __rte_unused uint32_t key_size, uint64_t seed) +hash_crc_key40(void *key, void *mask, __rte_unused uint32_t key_size, + uint64_t seed) { uint64_t *k = key; + uint64_t *m = mask; uint64_t k0, k2, crc0, crc1, crc2, crc3; - k0 = k[0]; - k2 = k[2]; + k0 = k[0] & m[0]; + k2 = k[2] & m[2]; crc0 = _mm_crc32_u64(k0, seed); - crc1 = _mm_crc32_u64(k0 >> 32, k[1]); + crc1 = _mm_crc32_u64(k0 >> 32, k[1] & m[1]); - crc2 = _mm_crc32_u64(k2, k[3]); - crc3 = _mm_crc32_u64(k2 >> 32, k[4]); + crc2 = _mm_crc32_u64(k2, k[3] & m[3]); + crc3 = _mm_crc32_u64(k2 >> 32, k[4] & m[4]); crc0 = _mm_crc32_u64(crc0, crc1); crc1 = _mm_crc32_u64(crc2, crc3); @@ -249,20 +275,22 @@ hash_crc_key40(void *key, __rte_unused uint32_t key_size, uint64_t seed) } static inline uint64_t -hash_crc_key48(void *key, __rte_unused uint32_t key_size, uint64_t seed) +hash_crc_key48(void *key, void *mask, __rte_unused uint32_t key_size, + uint64_t seed) { uint64_t *k = key; + uint64_t *m = mask; uint64_t k0, k2, k5, crc0, crc1, crc2, crc3; - k0 = k[0]; - k2 = k[2]; - k5 = k[5]; + k0 = k[0] & m[0]; + k2 = k[2] & m[2]; + k5 = k[5] & m[5]; crc0 = _mm_crc32_u64(k0, seed); - crc1 = _mm_crc32_u64(k0 >> 32, k[1]); + crc1 = _mm_crc32_u64(k0 >> 32, k[1] & m[1]); - crc2 = _mm_crc32_u64(k2, k[3]); - crc3 = _mm_crc32_u64(k2 >> 32, k[4]); + crc2 = _mm_crc32_u64(k2, k[3] & m[3]); + crc3 = _mm_crc32_u64(k2 >> 32, k[4] & m[4]); crc0 = _mm_crc32_u64(crc0, (crc1 << 32) ^ crc2); crc1 = _mm_crc32_u64(crc3, k5); @@ -273,22 +301,24 @@ hash_crc_key48(void *key, __rte_unused uint32_t key_size, uint64_t seed) } static inline uint64_t -hash_crc_key56(void *key, __rte_unused uint32_t key_size, uint64_t seed) +hash_crc_key56(void *key, void *mask, __rte_unused uint32_t key_size, + uint64_t seed) { uint64_t *k = key; + uint64_t *m = mask; uint64_t k0, k2, k5, crc0, crc1, crc2, crc3, crc4, crc5; - k0 = k[0]; - k2 = k[2]; - k5 = k[5]; + k0 = k[0] & m[0]; + k2 = k[2] & m[2]; + k5 = k[5] & m[5]; crc0 = _mm_crc32_u64(k0, seed); - crc1 = _mm_crc32_u64(k0 >> 32, k[1]); + crc1 = _mm_crc32_u64(k0 >> 32, k[1] & m[1]); - crc2 = _mm_crc32_u64(k2, k[3]); - crc3 = _mm_crc32_u64(k2 >> 32, k[4]); + crc2 = _mm_crc32_u64(k2, k[3] & m[3]); + crc3 = _mm_crc32_u64(k2 >> 32, k[4] & m[4]); - crc4 = _mm_crc32_u64(k5, k[6]); + crc4 = _mm_crc32_u64(k5, k[6] & m[6]); crc5 = k5 >> 32; crc0 = _mm_crc32_u64(crc0, (crc1 << 32) ^ crc2); @@ -300,23 +330,25 @@ hash_crc_key56(void *key, __rte_unused uint32_t key_size, uint64_t seed) } static inline uint64_t -hash_crc_key64(void *key, __rte_unused uint32_t key_size, uint64_t seed) +hash_crc_key64(void *key, void *mask, __rte_unused uint32_t key_size, + uint64_t seed) { uint64_t *k = key; + uint64_t *m = mask; uint64_t k0, k2, k5, crc0, crc1, crc2, crc3, crc4, crc5; - k0 = k[0]; - k2 = k[2]; - k5 = k[5]; + k0 = k[0] & m[0]; + k2 = k[2] & m[2]; + k5 = k[5] & m[5]; crc0 = _mm_crc32_u64(k0, seed); - crc1 = _mm_crc32_u64(k0 >> 32, k[1]); + crc1 = _mm_crc32_u64(k0 >> 32, k[1] & m[1]); - crc2 = _mm_crc32_u64(k2, k[3]); - crc3 = _mm_crc32_u64(k2 >> 32, k[4]); + crc2 = _mm_crc32_u64(k2, k[3] & m[3]); + crc3 = _mm_crc32_u64(k2 >> 32, k[4] & m[4]); - crc4 = _mm_crc32_u64(k5, k[6]); - crc5 = _mm_crc32_u64(k5 >> 32, k[7]); + crc4 = _mm_crc32_u64(k5, k[6] & m[6]); + crc5 = _mm_crc32_u64(k5 >> 32, k[7] & m[7]); crc0 = _mm_crc32_u64(crc0, (crc1 << 32) ^ crc2); crc1 = _mm_crc32_u64(crc3, (crc4 << 32) ^ crc5); @@ -335,6 +367,8 @@ hash_crc_key64(void *key, __rte_unused uint32_t key_size, uint64_t seed) #define hash_default_key56 hash_crc_key56 #define hash_default_key64 hash_crc_key64 +#elif defined(RTE_ARCH_ARM64) +#include "hash_func_arm64.h" #else #define hash_default_key8 hash_xor_key8 diff --git a/dpdk/examples/ip_pipeline/pipeline/hash_func_arm64.h b/dpdk/examples/ip_pipeline/pipeline/hash_func_arm64.h new file mode 100644 index 00000000..ae6c0f41 --- /dev/null +++ b/dpdk/examples/ip_pipeline/pipeline/hash_func_arm64.h @@ -0,0 +1,261 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Linaro Limited. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef __HASH_FUNC_ARM64_H__ +#define __HASH_FUNC_ARM64_H__ + +#define _CRC32CX(crc, val) \ + __asm__("crc32cx %w[c], %w[c], %x[v]":[c] "+r" (crc):[v] "r" (val)) + +static inline uint64_t +hash_crc_key8(void *key, void *mask, __rte_unused uint32_t key_size, + uint64_t seed) +{ + uint64_t *k = key; + uint64_t *m = mask; + uint32_t crc0; + + crc0 = seed; + _CRC32CX(crc0, k[0] & m[0]); + + return crc0; +} + +static inline uint64_t +hash_crc_key16(void *key, void *mask, __rte_unused uint32_t key_size, + uint64_t seed) +{ + uint64_t *k = key, k0; + uint64_t *m = mask; + uint32_t crc0, crc1; + + k0 = k[0] & m[0]; + + crc0 = k0; + _CRC32CX(crc0, seed); + crc1 = k0 >> 32; + _CRC32CX(crc1, k[1] & m[1]); + + crc0 ^= crc1; + + return crc0; +} + +static inline uint64_t +hash_crc_key24(void *key, void *mask, __rte_unused uint32_t key_size, + uint64_t seed) +{ + uint64_t *k = key, k0, k2; + uint64_t *m = mask; + uint32_t crc0, crc1; + + k0 = k[0] & m[0]; + k2 = k[2] & m[2]; + + crc0 = k0; + _CRC32CX(crc0, seed); + crc1 = k0 >> 32; + _CRC32CX(crc1, k[1] & m[1]); + + _CRC32CX(crc0, k2); + + crc0 ^= crc1; + + return crc0; +} + +static inline uint64_t +hash_crc_key32(void *key, void *mask, __rte_unused uint32_t key_size, + uint64_t seed) +{ + uint64_t *k = key, k0, k2; + uint64_t *m = mask; + uint32_t crc0, crc1, crc2, crc3; + + k0 = k[0] & m[0]; + k2 = k[2] & m[2]; + + crc0 = k0; + _CRC32CX(crc0, seed); + crc1 = k0 >> 32; + _CRC32CX(crc1, k[1] & m[1]); + + crc2 = k2; + _CRC32CX(crc2, k[3] & m[3]); + crc3 = k2 >> 32; + + _CRC32CX(crc0, crc1); + _CRC32CX(crc2, crc3); + + crc0 ^= crc2; + + return crc0; +} + +static inline uint64_t +hash_crc_key40(void *key, void *mask, __rte_unused uint32_t key_size, + uint64_t seed) +{ + uint64_t *k = key, k0, k2; + uint64_t *m = mask; + uint32_t crc0, crc1, crc2, crc3; + + k0 = k[0] & m[0]; + k2 = k[2] & m[2]; + + crc0 = k0; + _CRC32CX(crc0, seed); + crc1 = k0 >> 32; + _CRC32CX(crc1, k[1] & m[1]); + + crc2 = k2; + _CRC32CX(crc2, k[3] & m[3]); + crc3 = k2 >> 32; + _CRC32CX(crc3, k[4] & m[4]); + + _CRC32CX(crc0, crc1); + _CRC32CX(crc2, crc3); + + crc0 ^= crc2; + + return crc0; +} + +static inline uint64_t +hash_crc_key48(void *key, void *mask, __rte_unused uint32_t key_size, + uint64_t seed) +{ + uint64_t *k = key, k0, k2, k5; + uint64_t *m = mask; + uint32_t crc0, crc1, crc2, crc3; + + k0 = k[0] & m[0]; + k2 = k[2] & m[2]; + k5 = k[5] & m[5]; + + crc0 = k0; + _CRC32CX(crc0, seed); + crc1 = k0 >> 32; + _CRC32CX(crc1, k[1] & m[1]); + + crc2 = k2; + _CRC32CX(crc2, k[3] & m[3]); + crc3 = k2 >> 32; + _CRC32CX(crc3, k[4] & m[4]); + + _CRC32CX(crc0, ((uint64_t)crc1 << 32) ^ crc2); + _CRC32CX(crc3, k5); + + crc0 ^= crc3; + + return crc0; +} + +static inline uint64_t +hash_crc_key56(void *key, void *mask, __rte_unused uint32_t key_size, + uint64_t seed) +{ + uint64_t *k = key, k0, k2, k5; + uint64_t *m = mask; + uint32_t crc0, crc1, crc2, crc3, crc4, crc5; + + k0 = k[0] & m[0]; + k2 = k[2] & m[2]; + k5 = k[5] & m[5]; + + crc0 = k0; + _CRC32CX(crc0, seed); + crc1 = k0 >> 32; + _CRC32CX(crc1, k[1] & m[1]); + + crc2 = k2; + _CRC32CX(crc2, k[3] & m[3]); + crc3 = k2 >> 32; + _CRC32CX(crc3, k[4] & m[4]); + + crc4 = k5; + _CRC32CX(crc4, k[6] & m[6]); + crc5 = k5 >> 32; + + _CRC32CX(crc0, ((uint64_t)crc1 << 32) ^ crc2); + _CRC32CX(crc3, ((uint64_t)crc4 << 32) ^ crc5); + + crc0 ^= crc3; + + return crc0; +} + +static inline uint64_t +hash_crc_key64(void *key, void *mask, __rte_unused uint32_t key_size, + uint64_t seed) +{ + uint64_t *k = key, k0, k2, k5; + uint64_t *m = mask; + uint32_t crc0, crc1, crc2, crc3, crc4, crc5; + + k0 = k[0] & m[0]; + k2 = k[2] & m[2]; + k5 = k[5] & m[5]; + + crc0 = k0; + _CRC32CX(crc0, seed); + crc1 = k0 >> 32; + _CRC32CX(crc1, k[1] & m[1]); + + crc2 = k2; + _CRC32CX(crc2, k[3] & m[3]); + crc3 = k2 >> 32; + _CRC32CX(crc3, k[4] & m[4]); + + crc4 = k5; + _CRC32CX(crc4, k[6] & m[6]); + crc5 = k5 >> 32; + _CRC32CX(crc5, k[7] & m[7]); + + _CRC32CX(crc0, ((uint64_t)crc1 << 32) ^ crc2); + _CRC32CX(crc3, ((uint64_t)crc4 << 32) ^ crc5); + + crc0 ^= crc3; + + return crc0; +} + +#define hash_default_key8 hash_crc_key8 +#define hash_default_key16 hash_crc_key16 +#define hash_default_key24 hash_crc_key24 +#define hash_default_key32 hash_crc_key32 +#define hash_default_key40 hash_crc_key40 +#define hash_default_key48 hash_crc_key48 +#define hash_default_key56 hash_crc_key56 +#define hash_default_key64 hash_crc_key64 + +#endif diff --git a/dpdk/examples/ip_pipeline/pipeline/pipeline_common_be.c b/dpdk/examples/ip_pipeline/pipeline/pipeline_common_be.c index 50dcb694..347e72b5 100644 --- a/dpdk/examples/ip_pipeline/pipeline/pipeline_common_be.c +++ b/dpdk/examples/ip_pipeline/pipeline/pipeline_common_be.c @@ -32,7 +32,6 @@ */ #include -#include #include #include "pipeline_common_be.h" diff --git a/dpdk/examples/ip_pipeline/pipeline/pipeline_common_fe.c b/dpdk/examples/ip_pipeline/pipeline/pipeline_common_fe.c index cd1d082a..75211878 100644 --- a/dpdk/examples/ip_pipeline/pipeline/pipeline_common_fe.c +++ b/dpdk/examples/ip_pipeline/pipeline/pipeline_common_fe.c @@ -36,7 +36,6 @@ #include #include -#include #include #include #include @@ -157,6 +156,7 @@ app_pipeline_track_pktq_out_to_link(struct app_params *app, break; } + case APP_PKTQ_OUT_TAP: case APP_PKTQ_OUT_SINK: default: return NULL; diff --git a/dpdk/examples/ip_pipeline/pipeline/pipeline_firewall_be.c b/dpdk/examples/ip_pipeline/pipeline/pipeline_firewall_be.c index b61f3034..2980492b 100644 --- a/dpdk/examples/ip_pipeline/pipeline/pipeline_firewall_be.c +++ b/dpdk/examples/ip_pipeline/pipeline/pipeline_firewall_be.c @@ -161,7 +161,7 @@ static struct rte_acl_field_def field_format_ipv4[] = { .type = RTE_ACL_FIELD_TYPE_RANGE, .size = sizeof(uint16_t), .field_index = 4, - .input_index = 4, + .input_index = 3, .offset = sizeof(struct ether_hdr) + sizeof(struct ipv4_hdr) + offsetof(struct tcp_hdr, dst_port), @@ -221,7 +221,7 @@ static struct rte_acl_field_def field_format_vlan_ipv4[] = { .type = RTE_ACL_FIELD_TYPE_RANGE, .size = sizeof(uint16_t), .field_index = 4, - .input_index = 4, + .input_index = 3, .offset = sizeof(struct ether_hdr) + SIZEOF_VLAN_HDR + sizeof(struct ipv4_hdr) + @@ -282,7 +282,7 @@ static struct rte_acl_field_def field_format_qinq_ipv4[] = { .type = RTE_ACL_FIELD_TYPE_RANGE, .size = sizeof(uint16_t), .field_index = 4, - .input_index = 4, + .input_index = 3, .offset = sizeof(struct ether_hdr) + SIZEOF_QINQ_HEADER + sizeof(struct ipv4_hdr) + diff --git a/dpdk/examples/ip_pipeline/pipeline/pipeline_flow_classification.c b/dpdk/examples/ip_pipeline/pipeline/pipeline_flow_classification.c index 9ef50cc9..70b19381 100644 --- a/dpdk/examples/ip_pipeline/pipeline/pipeline_flow_classification.c +++ b/dpdk/examples/ip_pipeline/pipeline/pipeline_flow_classification.c @@ -88,8 +88,10 @@ app_pipeline_fc_key_convert(struct pipeline_fc_key *key_in, uint32_t *signature) { uint8_t buffer[PIPELINE_FC_FLOW_KEY_MAX_SIZE]; + uint8_t m[PIPELINE_FC_FLOW_KEY_MAX_SIZE]; /* key mask */ void *key_buffer = (key_out) ? key_out : buffer; + memset(m, 0xFF, sizeof(m)); switch (key_in->type) { case FLOW_KEY_QINQ: { @@ -101,7 +103,7 @@ app_pipeline_fc_key_convert(struct pipeline_fc_key *key_in, qinq->cvlan = rte_cpu_to_be_16(key_in->key.qinq.cvlan); if (signature) - *signature = (uint32_t) hash_default_key8(qinq, 8, 0); + *signature = (uint32_t) hash_default_key8(qinq, m, 8, 0); return 0; } @@ -118,7 +120,7 @@ app_pipeline_fc_key_convert(struct pipeline_fc_key *key_in, ipv4->port_dst = rte_cpu_to_be_16(key_in->key.ipv4_5tuple.port_dst); if (signature) - *signature = (uint32_t) hash_default_key16(ipv4, 16, 0); + *signature = (uint32_t) hash_default_key16(ipv4, m, 16, 0); return 0; } @@ -136,7 +138,7 @@ app_pipeline_fc_key_convert(struct pipeline_fc_key *key_in, ipv6->port_dst = rte_cpu_to_be_16(key_in->key.ipv6_5tuple.port_dst); if (signature) - *signature = (uint32_t) hash_default_key64(ipv6, 64, 0); + *signature = (uint32_t) hash_default_key64(ipv6, m, 64, 0); return 0; } @@ -832,12 +834,12 @@ app_pipeline_fc_add_bulk(struct app_params *app, } /* Free resources */ - app_msg_free(app, rsp); for (i = rsp->n_keys; i < n_keys; i++) if (new_flow[i]) rte_free(flow[i]); + app_msg_free(app, rsp); rte_free(flow_rsp); rte_free(flow_req); rte_free(new_flow); diff --git a/dpdk/examples/ip_pipeline/pipeline/pipeline_flow_classification_be.c b/dpdk/examples/ip_pipeline/pipeline/pipeline_flow_classification_be.c index 026f00cd..929d81cb 100644 --- a/dpdk/examples/ip_pipeline/pipeline/pipeline_flow_classification_be.c +++ b/dpdk/examples/ip_pipeline/pipeline/pipeline_flow_classification_be.c @@ -492,40 +492,16 @@ static void *pipeline_fc_init(struct pipeline_params *params, /* Tables */ p->n_tables = 1; { - struct rte_table_hash_key8_ext_params - table_hash_key8_params = { - .n_entries = p_fc->n_flows, - .n_entries_ext = p_fc->n_flows, - .signature_offset = p_fc->hash_offset, - .key_offset = p_fc->key_offset, - .f_hash = hash_func[(p_fc->key_size / 8) - 1], - .key_mask = (p_fc->key_mask_present) ? - p_fc->key_mask : NULL, - .seed = 0, - }; - - struct rte_table_hash_key16_ext_params - table_hash_key16_params = { - .n_entries = p_fc->n_flows, - .n_entries_ext = p_fc->n_flows, - .signature_offset = p_fc->hash_offset, - .key_offset = p_fc->key_offset, - .f_hash = hash_func[(p_fc->key_size / 8) - 1], - .key_mask = (p_fc->key_mask_present) ? - p_fc->key_mask : NULL, - .seed = 0, - }; - - struct rte_table_hash_ext_params - table_hash_params = { + struct rte_table_hash_params table_hash_params = { + .name = p->name, .key_size = p_fc->key_size, + .key_offset = p_fc->key_offset, + .key_mask = (p_fc->key_mask_present) ? + p_fc->key_mask : NULL, .n_keys = p_fc->n_flows, - .n_buckets = p_fc->n_flows / 4, - .n_buckets_ext = p_fc->n_flows / 4, + .n_buckets = rte_align32pow2(p_fc->n_flows / 4), .f_hash = hash_func[(p_fc->key_size / 8) - 1], .seed = 0, - .signature_offset = p_fc->hash_offset, - .key_offset = p_fc->key_offset, }; struct rte_pipeline_table_params table_params = { @@ -542,32 +518,19 @@ static void *pipeline_fc_init(struct pipeline_params *params, switch (p_fc->key_size) { case 8: - if (p_fc->hash_offset != 0) { - table_params.ops = - &rte_table_hash_key8_ext_ops; - } else { - table_params.ops = - &rte_table_hash_key8_ext_dosig_ops; - } - table_params.arg_create = &table_hash_key8_params; + table_params.ops = &rte_table_hash_key8_ext_ops; break; case 16: - if (p_fc->hash_offset != 0) { - table_params.ops = - &rte_table_hash_key16_ext_ops; - } else { - table_params.ops = - &rte_table_hash_key16_ext_dosig_ops; - } - table_params.arg_create = &table_hash_key16_params; + table_params.ops = &rte_table_hash_key16_ext_ops; break; default: table_params.ops = &rte_table_hash_ext_ops; - table_params.arg_create = &table_hash_params; } + table_params.arg_create = &table_hash_params; + status = rte_pipeline_table_create(p->p, &table_params, &p->table_id[0]); diff --git a/dpdk/examples/ip_pipeline/pipeline/pipeline_passthrough.c b/dpdk/examples/ip_pipeline/pipeline/pipeline_passthrough.c index 63ce1472..2c9eb2e3 100644 --- a/dpdk/examples/ip_pipeline/pipeline/pipeline_passthrough.c +++ b/dpdk/examples/ip_pipeline/pipeline/pipeline_passthrough.c @@ -52,7 +52,7 @@ app_pipeline_passthrough_track(struct pipeline_params *p, if (status) return -1; - if (pp.lb_hash_enabled) + if (pp.dma_hash_lb_enabled) return -1; *port_out = port_in / (p->n_ports_in / p->n_ports_out); diff --git a/dpdk/examples/ip_pipeline/pipeline/pipeline_passthrough_be.c b/dpdk/examples/ip_pipeline/pipeline/pipeline_passthrough_be.c index 356f02d0..93eedbe8 100644 --- a/dpdk/examples/ip_pipeline/pipeline/pipeline_passthrough_be.c +++ b/dpdk/examples/ip_pipeline/pipeline/pipeline_passthrough_be.c @@ -31,6 +31,7 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +#include #include #include @@ -45,10 +46,17 @@ #include "parser.h" #include "hash_func.h" +#define SWAP_DIM (PIPELINE_PASSTHROUGH_SWAP_N_FIELDS_MAX * \ + (PIPELINE_PASSTHROUGH_SWAP_FIELD_SIZE_MAX / sizeof(uint64_t))) + struct pipeline_passthrough { struct pipeline p; struct pipeline_passthrough_params params; rte_table_hash_op_hash f_hash; + uint32_t swap_field0_offset[SWAP_DIM]; + uint32_t swap_field1_offset[SWAP_DIM]; + uint64_t swap_field_mask[SWAP_DIM]; + uint32_t swap_n_fields; } __rte_cache_aligned; static pipeline_msg_req_handler handlers[] = { @@ -68,8 +76,8 @@ static pipeline_msg_req_handler handlers[] = { pipeline_msg_req_invalid_handler, }; -static inline __attribute__((always_inline)) void -pkt_work( +static __rte_always_inline void +pkt_work_dma( struct rte_mbuf *pkt, void *arg, uint32_t dma_size, @@ -94,7 +102,7 @@ pkt_work( /* Read (dma_dst), compute (hash), write (hash) */ if (hash_enabled) { - uint32_t hash = p->f_hash(dma_dst, dma_size, 0); + uint32_t hash = p->f_hash(dma_src, dma_mask, dma_size, 0); *dma_hash = hash; if (lb_hash) { @@ -113,8 +121,8 @@ pkt_work( } } -static inline __attribute__((always_inline)) void -pkt4_work( +static __rte_always_inline void +pkt4_work_dma( struct rte_mbuf **pkts, void *arg, uint32_t dma_size, @@ -165,10 +173,10 @@ pkt4_work( /* Read (dma_dst), compute (hash), write (hash) */ if (hash_enabled) { - uint32_t hash0 = p->f_hash(dma_dst0, dma_size, 0); - uint32_t hash1 = p->f_hash(dma_dst1, dma_size, 0); - uint32_t hash2 = p->f_hash(dma_dst2, dma_size, 0); - uint32_t hash3 = p->f_hash(dma_dst3, dma_size, 0); + uint32_t hash0 = p->f_hash(dma_src0, dma_mask, dma_size, 0); + uint32_t hash1 = p->f_hash(dma_src1, dma_mask, dma_size, 0); + uint32_t hash2 = p->f_hash(dma_src2, dma_mask, dma_size, 0); + uint32_t hash3 = p->f_hash(dma_src3, dma_mask, dma_size, 0); *dma_hash0 = hash0; *dma_hash1 = hash1; @@ -209,148 +217,231 @@ pkt4_work( } } -#define PKT_WORK(dma_size, hash_enabled, lb_hash, port_pow2) \ +static __rte_always_inline void +pkt_work_swap( + struct rte_mbuf *pkt, + void *arg) +{ + struct pipeline_passthrough *p = arg; + uint32_t i; + + /* Read(field0, field1), compute(field0, field1), write(field0, field1) */ + for (i = 0; i < p->swap_n_fields; i++) { + uint64_t *field0_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkt, + p->swap_field0_offset[i]); + uint64_t *field1_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkt, + p->swap_field1_offset[i]); + uint64_t mask = p->swap_field_mask[i]; + + uint64_t field0 = *field0_ptr; + uint64_t field1 = *field1_ptr; + + *field0_ptr = (field0 & (~mask)) + (field1 & mask); + *field1_ptr = (field0 & mask) + (field1 & (~mask)); + } +} + +static __rte_always_inline void +pkt4_work_swap( + struct rte_mbuf **pkts, + void *arg) +{ + struct pipeline_passthrough *p = arg; + uint32_t i; + + /* Read(field0, field1), compute(field0, field1), write(field0, field1) */ + for (i = 0; i < p->swap_n_fields; i++) { + uint64_t *pkt0_field0_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkts[0], + p->swap_field0_offset[i]); + uint64_t *pkt1_field0_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkts[1], + p->swap_field0_offset[i]); + uint64_t *pkt2_field0_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkts[2], + p->swap_field0_offset[i]); + uint64_t *pkt3_field0_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkts[3], + p->swap_field0_offset[i]); + + uint64_t *pkt0_field1_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkts[0], + p->swap_field1_offset[i]); + uint64_t *pkt1_field1_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkts[1], + p->swap_field1_offset[i]); + uint64_t *pkt2_field1_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkts[2], + p->swap_field1_offset[i]); + uint64_t *pkt3_field1_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkts[3], + p->swap_field1_offset[i]); + + uint64_t mask = p->swap_field_mask[i]; + + uint64_t pkt0_field0 = *pkt0_field0_ptr; + uint64_t pkt1_field0 = *pkt1_field0_ptr; + uint64_t pkt2_field0 = *pkt2_field0_ptr; + uint64_t pkt3_field0 = *pkt3_field0_ptr; + + uint64_t pkt0_field1 = *pkt0_field1_ptr; + uint64_t pkt1_field1 = *pkt1_field1_ptr; + uint64_t pkt2_field1 = *pkt2_field1_ptr; + uint64_t pkt3_field1 = *pkt3_field1_ptr; + + *pkt0_field0_ptr = (pkt0_field0 & (~mask)) + (pkt0_field1 & mask); + *pkt1_field0_ptr = (pkt1_field0 & (~mask)) + (pkt1_field1 & mask); + *pkt2_field0_ptr = (pkt2_field0 & (~mask)) + (pkt2_field1 & mask); + *pkt3_field0_ptr = (pkt3_field0 & (~mask)) + (pkt3_field1 & mask); + + *pkt0_field1_ptr = (pkt0_field0 & mask) + (pkt0_field1 & (~mask)); + *pkt1_field1_ptr = (pkt1_field0 & mask) + (pkt1_field1 & (~mask)); + *pkt2_field1_ptr = (pkt2_field0 & mask) + (pkt2_field1 & (~mask)); + *pkt3_field1_ptr = (pkt3_field0 & mask) + (pkt3_field1 & (~mask)); + } +} + +#define PKT_WORK_DMA(dma_size, hash_enabled, lb_hash, port_pow2) \ static inline void \ -pkt_work_size##dma_size##_hash##hash_enabled \ +pkt_work_dma_size##dma_size##_hash##hash_enabled \ ##_lb##lb_hash##_pw##port_pow2( \ struct rte_mbuf *pkt, \ void *arg) \ { \ - pkt_work(pkt, arg, dma_size, hash_enabled, lb_hash, port_pow2); \ + pkt_work_dma(pkt, arg, dma_size, hash_enabled, lb_hash, port_pow2); \ } -#define PKT4_WORK(dma_size, hash_enabled, lb_hash, port_pow2) \ +#define PKT4_WORK_DMA(dma_size, hash_enabled, lb_hash, port_pow2) \ static inline void \ -pkt4_work_size##dma_size##_hash##hash_enabled \ +pkt4_work_dma_size##dma_size##_hash##hash_enabled \ ##_lb##lb_hash##_pw##port_pow2( \ struct rte_mbuf **pkts, \ void *arg) \ { \ - pkt4_work(pkts, arg, dma_size, hash_enabled, lb_hash, port_pow2); \ + pkt4_work_dma(pkts, arg, dma_size, hash_enabled, lb_hash, port_pow2); \ } -#define port_in_ah(dma_size, hash_enabled, lb_hash, port_pow2) \ -PKT_WORK(dma_size, hash_enabled, lb_hash, port_pow2) \ -PKT4_WORK(dma_size, hash_enabled, lb_hash, port_pow2) \ -PIPELINE_PORT_IN_AH(port_in_ah_size##dma_size##_hash \ +#define port_in_ah_dma(dma_size, hash_enabled, lb_hash, port_pow2) \ +PKT_WORK_DMA(dma_size, hash_enabled, lb_hash, port_pow2) \ +PKT4_WORK_DMA(dma_size, hash_enabled, lb_hash, port_pow2) \ +PIPELINE_PORT_IN_AH(port_in_ah_dma_size##dma_size##_hash \ ##hash_enabled##_lb##lb_hash##_pw##port_pow2, \ - pkt_work_size##dma_size##_hash##hash_enabled \ + pkt_work_dma_size##dma_size##_hash##hash_enabled \ ##_lb##lb_hash##_pw##port_pow2, \ - pkt4_work_size##dma_size##_hash##hash_enabled \ + pkt4_work_dma_size##dma_size##_hash##hash_enabled \ ##_lb##lb_hash##_pw##port_pow2) #define port_in_ah_lb(dma_size, hash_enabled, lb_hash, port_pow2) \ -PKT_WORK(dma_size, hash_enabled, lb_hash, port_pow2) \ -PKT4_WORK(dma_size, hash_enabled, lb_hash, port_pow2) \ +PKT_WORK_DMA(dma_size, hash_enabled, lb_hash, port_pow2) \ +PKT4_WORK_DMA(dma_size, hash_enabled, lb_hash, port_pow2) \ PIPELINE_PORT_IN_AH_HIJACK_ALL( \ - port_in_ah_size##dma_size##_hash##hash_enabled \ + port_in_ah_lb_size##dma_size##_hash##hash_enabled \ ##_lb##lb_hash##_pw##port_pow2, \ - pkt_work_size##dma_size##_hash##hash_enabled \ + pkt_work_dma_size##dma_size##_hash##hash_enabled \ ##_lb##lb_hash##_pw##port_pow2, \ - pkt4_work_size##dma_size##_hash##hash_enabled \ + pkt4_work_dma_size##dma_size##_hash##hash_enabled \ ##_lb##lb_hash##_pw##port_pow2) -/* Port in AH (dma_size, hash_enabled, lb_hash, port_pow2) */ +PIPELINE_PORT_IN_AH(port_in_ah_swap, pkt_work_swap, pkt4_work_swap) -port_in_ah(8, 0, 0, 0) -port_in_ah(8, 1, 0, 0) + +/* Port in AH DMA(dma_size, hash_enabled, lb_hash, port_pow2) */ + +port_in_ah_dma(8, 0, 0, 0) +port_in_ah_dma(8, 1, 0, 0) port_in_ah_lb(8, 1, 1, 0) port_in_ah_lb(8, 1, 1, 1) -port_in_ah(16, 0, 0, 0) -port_in_ah(16, 1, 0, 0) +port_in_ah_dma(16, 0, 0, 0) +port_in_ah_dma(16, 1, 0, 0) port_in_ah_lb(16, 1, 1, 0) port_in_ah_lb(16, 1, 1, 1) -port_in_ah(24, 0, 0, 0) -port_in_ah(24, 1, 0, 0) +port_in_ah_dma(24, 0, 0, 0) +port_in_ah_dma(24, 1, 0, 0) port_in_ah_lb(24, 1, 1, 0) port_in_ah_lb(24, 1, 1, 1) -port_in_ah(32, 0, 0, 0) -port_in_ah(32, 1, 0, 0) +port_in_ah_dma(32, 0, 0, 0) +port_in_ah_dma(32, 1, 0, 0) port_in_ah_lb(32, 1, 1, 0) port_in_ah_lb(32, 1, 1, 1) -port_in_ah(40, 0, 0, 0) -port_in_ah(40, 1, 0, 0) +port_in_ah_dma(40, 0, 0, 0) +port_in_ah_dma(40, 1, 0, 0) port_in_ah_lb(40, 1, 1, 0) port_in_ah_lb(40, 1, 1, 1) -port_in_ah(48, 0, 0, 0) -port_in_ah(48, 1, 0, 0) +port_in_ah_dma(48, 0, 0, 0) +port_in_ah_dma(48, 1, 0, 0) port_in_ah_lb(48, 1, 1, 0) port_in_ah_lb(48, 1, 1, 1) -port_in_ah(56, 0, 0, 0) -port_in_ah(56, 1, 0, 0) +port_in_ah_dma(56, 0, 0, 0) +port_in_ah_dma(56, 1, 0, 0) port_in_ah_lb(56, 1, 1, 0) port_in_ah_lb(56, 1, 1, 1) -port_in_ah(64, 0, 0, 0) -port_in_ah(64, 1, 0, 0) +port_in_ah_dma(64, 0, 0, 0) +port_in_ah_dma(64, 1, 0, 0) port_in_ah_lb(64, 1, 1, 0) port_in_ah_lb(64, 1, 1, 1) static rte_pipeline_port_in_action_handler get_port_in_ah(struct pipeline_passthrough *p) { - if (p->params.dma_enabled == 0) + if ((p->params.dma_enabled == 0) && + (p->params.swap_enabled == 0)) return NULL; + if (p->params.swap_enabled) + return port_in_ah_swap; + if (p->params.dma_hash_enabled) { - if (p->params.lb_hash_enabled) { + if (p->params.dma_hash_lb_enabled) { if (rte_is_power_of_2(p->p.n_ports_out)) switch (p->params.dma_size) { - case 8: return port_in_ah_size8_hash1_lb1_pw1; - case 16: return port_in_ah_size16_hash1_lb1_pw1; - case 24: return port_in_ah_size24_hash1_lb1_pw1; - case 32: return port_in_ah_size32_hash1_lb1_pw1; - case 40: return port_in_ah_size40_hash1_lb1_pw1; - case 48: return port_in_ah_size48_hash1_lb1_pw1; - case 56: return port_in_ah_size56_hash1_lb1_pw1; - case 64: return port_in_ah_size64_hash1_lb1_pw1; + case 8: return port_in_ah_lb_size8_hash1_lb1_pw1; + case 16: return port_in_ah_lb_size16_hash1_lb1_pw1; + case 24: return port_in_ah_lb_size24_hash1_lb1_pw1; + case 32: return port_in_ah_lb_size32_hash1_lb1_pw1; + case 40: return port_in_ah_lb_size40_hash1_lb1_pw1; + case 48: return port_in_ah_lb_size48_hash1_lb1_pw1; + case 56: return port_in_ah_lb_size56_hash1_lb1_pw1; + case 64: return port_in_ah_lb_size64_hash1_lb1_pw1; default: return NULL; } else switch (p->params.dma_size) { - case 8: return port_in_ah_size8_hash1_lb1_pw0; - case 16: return port_in_ah_size16_hash1_lb1_pw0; - case 24: return port_in_ah_size24_hash1_lb1_pw0; - case 32: return port_in_ah_size32_hash1_lb1_pw0; - case 40: return port_in_ah_size40_hash1_lb1_pw0; - case 48: return port_in_ah_size48_hash1_lb1_pw0; - case 56: return port_in_ah_size56_hash1_lb1_pw0; - case 64: return port_in_ah_size64_hash1_lb1_pw0; + case 8: return port_in_ah_lb_size8_hash1_lb1_pw0; + case 16: return port_in_ah_lb_size16_hash1_lb1_pw0; + case 24: return port_in_ah_lb_size24_hash1_lb1_pw0; + case 32: return port_in_ah_lb_size32_hash1_lb1_pw0; + case 40: return port_in_ah_lb_size40_hash1_lb1_pw0; + case 48: return port_in_ah_lb_size48_hash1_lb1_pw0; + case 56: return port_in_ah_lb_size56_hash1_lb1_pw0; + case 64: return port_in_ah_lb_size64_hash1_lb1_pw0; default: return NULL; } } else switch (p->params.dma_size) { - case 8: return port_in_ah_size8_hash1_lb0_pw0; - case 16: return port_in_ah_size16_hash1_lb0_pw0; - case 24: return port_in_ah_size24_hash1_lb0_pw0; - case 32: return port_in_ah_size32_hash1_lb0_pw0; - case 40: return port_in_ah_size40_hash1_lb0_pw0; - case 48: return port_in_ah_size48_hash1_lb0_pw0; - case 56: return port_in_ah_size56_hash1_lb0_pw0; - case 64: return port_in_ah_size64_hash1_lb0_pw0; + case 8: return port_in_ah_dma_size8_hash1_lb0_pw0; + case 16: return port_in_ah_dma_size16_hash1_lb0_pw0; + case 24: return port_in_ah_dma_size24_hash1_lb0_pw0; + case 32: return port_in_ah_dma_size32_hash1_lb0_pw0; + case 40: return port_in_ah_dma_size40_hash1_lb0_pw0; + case 48: return port_in_ah_dma_size48_hash1_lb0_pw0; + case 56: return port_in_ah_dma_size56_hash1_lb0_pw0; + case 64: return port_in_ah_dma_size64_hash1_lb0_pw0; default: return NULL; } } else switch (p->params.dma_size) { - case 8: return port_in_ah_size8_hash0_lb0_pw0; - case 16: return port_in_ah_size16_hash0_lb0_pw0; - case 24: return port_in_ah_size24_hash0_lb0_pw0; - case 32: return port_in_ah_size32_hash0_lb0_pw0; - case 40: return port_in_ah_size40_hash0_lb0_pw0; - case 48: return port_in_ah_size48_hash0_lb0_pw0; - case 56: return port_in_ah_size56_hash0_lb0_pw0; - case 64: return port_in_ah_size64_hash0_lb0_pw0; + case 8: return port_in_ah_dma_size8_hash0_lb0_pw0; + case 16: return port_in_ah_dma_size16_hash0_lb0_pw0; + case 24: return port_in_ah_dma_size24_hash0_lb0_pw0; + case 32: return port_in_ah_dma_size32_hash0_lb0_pw0; + case 40: return port_in_ah_dma_size40_hash0_lb0_pw0; + case 48: return port_in_ah_dma_size48_hash0_lb0_pw0; + case 56: return port_in_ah_dma_size56_hash0_lb0_pw0; + case 64: return port_in_ah_dma_size64_hash0_lb0_pw0; default: return NULL; } } @@ -362,17 +453,19 @@ pipeline_passthrough_parse_args(struct pipeline_passthrough_params *p, uint32_t dma_dst_offset_present = 0; uint32_t dma_src_offset_present = 0; uint32_t dma_src_mask_present = 0; + char dma_mask_str[PIPELINE_PASSTHROUGH_DMA_SIZE_MAX * 2 + 1]; uint32_t dma_size_present = 0; uint32_t dma_hash_offset_present = 0; - uint32_t lb_present = 0; + uint32_t dma_hash_lb_present = 0; uint32_t i; - char dma_mask_str[PIPELINE_PASSTHROUGH_DMA_SIZE_MAX * 2 + 1]; /* default values */ p->dma_enabled = 0; p->dma_hash_enabled = 0; - p->lb_hash_enabled = 0; + p->dma_hash_lb_enabled = 0; memset(p->dma_src_mask, 0xFF, sizeof(p->dma_src_mask)); + p->swap_enabled = 0; + p->swap_n_fields = 0; for (i = 0; i < params->n_args; i++) { char *arg_name = params->args_name[i]; @@ -485,7 +578,6 @@ pipeline_passthrough_parse_args(struct pipeline_passthrough_params *p, params->name, arg_name, arg_value); p->dma_hash_enabled = 1; - p->dma_enabled = 1; continue; } @@ -493,19 +585,39 @@ pipeline_passthrough_parse_args(struct pipeline_passthrough_params *p, /* load_balance mode */ if (strcmp(arg_name, "lb") == 0) { PIPELINE_PARSE_ERR_DUPLICATE( - lb_present == 0, + dma_hash_lb_present == 0, params->name, arg_name); - lb_present = 1; + dma_hash_lb_present = 1; + + if (strcmp(arg_value, "hash") && + strcmp(arg_value, "HASH")) - if ((strcmp(arg_value, "hash") == 0) || - (strcmp(arg_value, "HASH") == 0)) - p->lb_hash_enabled = 1; - else PIPELINE_PARSE_ERR_INV_VAL(0, params->name, arg_name, arg_value); + p->dma_hash_lb_enabled = 1; + + continue; + } + + /* swap */ + if (strcmp(arg_name, "swap") == 0) { + uint32_t a, b, n_args; + int len; + + n_args = sscanf(arg_value, "%" SCNu32 " %" SCNu32 "%n", + &a, &b, &len); + PIPELINE_PARSE_ERR_INV_VAL(((n_args == 2) && + ((size_t) len == strlen(arg_value))), + params->name, arg_name, arg_value); + + p->swap_field0_offset[p->swap_n_fields] = a; + p->swap_field1_offset[p->swap_n_fields] = b; + p->swap_n_fields++; + p->swap_enabled = 1; + continue; } @@ -514,6 +626,9 @@ pipeline_passthrough_parse_args(struct pipeline_passthrough_params *p, } /* Check correlations between arguments */ + PIPELINE_ARG_CHECK((p->dma_enabled + p->swap_enabled < 2), + "Parse error in section \"%s\": DMA and SWAP actions are both enabled", + params->name); PIPELINE_ARG_CHECK((dma_dst_offset_present == p->dma_enabled), "Parse error in section \"%s\": missing entry " "\"dma_dst_offset\"", params->name); @@ -523,12 +638,12 @@ pipeline_passthrough_parse_args(struct pipeline_passthrough_params *p, PIPELINE_ARG_CHECK((dma_size_present == p->dma_enabled), "Parse error in section \"%s\": missing entry " "\"dma_size\"", params->name); - PIPELINE_ARG_CHECK((dma_hash_offset_present == p->dma_enabled), - "Parse error in section \"%s\": missing entry " - "\"dma_hash_offset\"", params->name); - PIPELINE_ARG_CHECK((p->lb_hash_enabled <= p->dma_hash_enabled), - "Parse error in section \"%s\": missing entry " - "\"dma_hash_offset\"", params->name); + PIPELINE_ARG_CHECK((p->dma_hash_enabled <= p->dma_enabled), + "Parse error in section \"%s\": missing all DMA entries", + params->name); + PIPELINE_ARG_CHECK((p->dma_hash_lb_enabled <= p->dma_hash_enabled), + "Parse error in section \"%s\": missing all DMA hash entries ", + params->name); if (dma_src_mask_present) { uint32_t dma_size = p->dma_size; @@ -547,7 +662,7 @@ pipeline_passthrough_parse_args(struct pipeline_passthrough_params *p, "dma_src_mask", dma_mask_str); } - if (p->lb_hash_enabled) + if (p->dma_hash_lb_enabled) PIPELINE_ARG_CHECK((params->n_ports_out > 1), "Parse error in section \"%s\": entry \"lb\" not " "allowed for single output port pipeline", @@ -562,7 +677,6 @@ pipeline_passthrough_parse_args(struct pipeline_passthrough_params *p, return 0; } - static rte_table_hash_op_hash get_hash_function(struct pipeline_passthrough *p) { @@ -580,6 +694,47 @@ get_hash_function(struct pipeline_passthrough *p) } } +static int +pipeline_passthrough_swap_convert(struct pipeline_passthrough *p) +{ + uint32_t i; + + p->swap_n_fields = 0; + + for (i = 0; i < p->params.swap_n_fields; i++) { + uint32_t offset0 = p->params.swap_field0_offset[i]; + uint32_t offset1 = p->params.swap_field1_offset[i]; + uint32_t size = offset1 - offset0; + uint32_t j; + + /* Check */ + if ((offset0 >= offset1) || + (size > PIPELINE_PASSTHROUGH_SWAP_FIELD_SIZE_MAX) || + (p->swap_n_fields >= SWAP_DIM)) + return -1; + + for (j = 0; j < (size / sizeof(uint64_t)); j++) { + p->swap_field0_offset[p->swap_n_fields] = offset0; + p->swap_field1_offset[p->swap_n_fields] = offset1; + p->swap_field_mask[p->swap_n_fields] = UINT64_MAX; + p->swap_n_fields++; + offset0 += sizeof(uint64_t); + offset1 += sizeof(uint64_t); + } + if (size % sizeof(uint64_t)) { + uint32_t n_bits = (size % sizeof(uint64_t)) * 8; + + p->swap_field0_offset[p->swap_n_fields] = offset0; + p->swap_field1_offset[p->swap_n_fields] = offset1; + p->swap_field_mask[p->swap_n_fields] = + RTE_LEN2MASK(n_bits, uint64_t); + p->swap_n_fields++; + } + } + + return 0; +} + static void* pipeline_passthrough_init(struct pipeline_params *params, __rte_unused void *arg) @@ -609,6 +764,8 @@ pipeline_passthrough_init(struct pipeline_params *params, /* Parse arguments */ if (pipeline_passthrough_parse_args(&p_pt->params, params)) return NULL; + if (pipeline_passthrough_swap_convert(p_pt)) + return NULL; p_pt->f_hash = get_hash_function(p_pt); /* Pipeline */ @@ -712,7 +869,7 @@ pipeline_passthrough_init(struct pipeline_params *params, /* Add entries to tables */ for (i = 0; i < p->n_ports_in; i++) { - uint32_t port_out_id = (p_pt->params.lb_hash_enabled == 0) ? + uint32_t port_out_id = (p_pt->params.dma_hash_lb_enabled == 0) ? (i / (p->n_ports_in / p->n_ports_out)) : 0; diff --git a/dpdk/examples/ip_pipeline/pipeline/pipeline_passthrough_be.h b/dpdk/examples/ip_pipeline/pipeline/pipeline_passthrough_be.h index 9368cec7..decb2684 100644 --- a/dpdk/examples/ip_pipeline/pipeline/pipeline_passthrough_be.h +++ b/dpdk/examples/ip_pipeline/pipeline/pipeline_passthrough_be.h @@ -38,6 +38,14 @@ #define PIPELINE_PASSTHROUGH_DMA_SIZE_MAX 64 +#ifndef PIPELINE_PASSTHROUGH_SWAP_N_FIELDS_MAX +#define PIPELINE_PASSTHROUGH_SWAP_N_FIELDS_MAX 8 +#endif + +#ifndef PIPELINE_PASSTHROUGH_SWAP_FIELD_SIZE_MAX +#define PIPELINE_PASSTHROUGH_SWAP_FIELD_SIZE_MAX 16 +#endif + struct pipeline_passthrough_params { uint32_t dma_enabled; uint32_t dma_dst_offset; @@ -47,7 +55,13 @@ struct pipeline_passthrough_params { uint32_t dma_hash_enabled; uint32_t dma_hash_offset; - uint32_t lb_hash_enabled; + + uint32_t dma_hash_lb_enabled; + + uint32_t swap_enabled; + uint32_t swap_field0_offset[PIPELINE_PASSTHROUGH_SWAP_N_FIELDS_MAX]; + uint32_t swap_field1_offset[PIPELINE_PASSTHROUGH_SWAP_N_FIELDS_MAX]; + uint32_t swap_n_fields; }; int diff --git a/dpdk/examples/ip_pipeline/pipeline/pipeline_routing.c b/dpdk/examples/ip_pipeline/pipeline/pipeline_routing.c index 3aadbf91..3deaff9c 100644 --- a/dpdk/examples/ip_pipeline/pipeline/pipeline_routing.c +++ b/dpdk/examples/ip_pipeline/pipeline/pipeline_routing.c @@ -494,6 +494,26 @@ app_pipeline_routing_add_route(struct app_params *app, /* data */ if (data->port_id >= p->n_ports_out) return -1; + + /* Valid range of VLAN tags 12 bits */ + if (data->flags & PIPELINE_ROUTING_ROUTE_QINQ) + if ((data->l2.qinq.svlan & 0xF000) || + (data->l2.qinq.cvlan & 0xF000)) + return -1; + + /* Max number of MPLS labels supported */ + if (data->flags & PIPELINE_ROUTING_ROUTE_MPLS) { + uint32_t i; + + if (data->l2.mpls.n_labels > + PIPELINE_ROUTING_MPLS_LABELS_MAX) + return -1; + + /* Max MPLS label value 20 bits */ + for (i = 0; i < data->l2.mpls.n_labels; i++) + if (data->l2.mpls.labels[i] & 0xFFF00000) + return -1; + } } break; diff --git a/dpdk/examples/ip_pipeline/pipeline/pipeline_routing_be.c b/dpdk/examples/ip_pipeline/pipeline/pipeline_routing_be.c index 21ac7888..0414f248 100644 --- a/dpdk/examples/ip_pipeline/pipeline/pipeline_routing_be.c +++ b/dpdk/examples/ip_pipeline/pipeline/pipeline_routing_be.c @@ -191,7 +191,7 @@ struct layout { dst->c = src->c; \ } -static inline __attribute__((always_inline)) void +static __rte_always_inline void pkt_work_routing( struct rte_mbuf *pkt, struct rte_pipeline_table_entry *table_entry, @@ -317,7 +317,7 @@ pkt_work_routing( } } -static inline __attribute__((always_inline)) void +static __rte_always_inline void pkt4_work_routing( struct rte_mbuf **pkts, struct rte_pipeline_table_entry **table_entries, @@ -1349,17 +1349,20 @@ pipeline_routing_init(struct pipeline_params *params, /* ARP table configuration */ if (p_rt->params.n_arp_entries) { - struct rte_table_hash_key8_ext_params table_arp_params = { - .n_entries = p_rt->params.n_arp_entries, - .n_entries_ext = p_rt->params.n_arp_entries, + struct rte_table_hash_params table_arp_params = { + .name = p->name, + .key_size = 8, + .key_offset = p_rt->params.arp_key_offset, + .key_mask = NULL, + .n_keys = p_rt->params.n_arp_entries, + .n_buckets = + rte_align32pow2(p_rt->params.n_arp_entries / 4), .f_hash = hash_default_key8, .seed = 0, - .signature_offset = 0, /* Unused */ - .key_offset = p_rt->params.arp_key_offset, }; struct rte_pipeline_table_params table_params = { - .ops = &rte_table_hash_key8_ext_dosig_ops, + .ops = &rte_table_hash_key8_ext_ops, .arg_create = &table_arp_params, .f_action_hit = get_arp_table_ah_hit(p_rt), .f_action_miss = NULL, diff --git a/dpdk/examples/ip_pipeline/pipeline_be.h b/dpdk/examples/ip_pipeline/pipeline_be.h index b562472b..0cfcc809 100644 --- a/dpdk/examples/ip_pipeline/pipeline_be.h +++ b/dpdk/examples/ip_pipeline/pipeline_be.h @@ -39,6 +39,7 @@ #include #include #include +#include #include #ifdef RTE_LIBRTE_KNI #include @@ -52,6 +53,7 @@ enum pipeline_port_in_type { PIPELINE_PORT_IN_RING_READER_IPV4_FRAG, PIPELINE_PORT_IN_RING_READER_IPV6_FRAG, PIPELINE_PORT_IN_SCHED_READER, + PIPELINE_PORT_IN_FD_READER, PIPELINE_PORT_IN_KNI_READER, PIPELINE_PORT_IN_SOURCE, }; @@ -65,6 +67,7 @@ struct pipeline_port_in_params { struct rte_port_ring_reader_ipv4_frag_params ring_ipv4_frag; struct rte_port_ring_reader_ipv6_frag_params ring_ipv6_frag; struct rte_port_sched_reader_params sched; + struct rte_port_fd_reader_params fd; #ifdef RTE_LIBRTE_KNI struct rte_port_kni_reader_params kni; #endif @@ -89,6 +92,8 @@ pipeline_port_in_params_convert(struct pipeline_port_in_params *p) return (void *) &p->params.ring_ipv6_frag; case PIPELINE_PORT_IN_SCHED_READER: return (void *) &p->params.sched; + case PIPELINE_PORT_IN_FD_READER: + return (void *) &p->params.fd; #ifdef RTE_LIBRTE_KNI case PIPELINE_PORT_IN_KNI_READER: return (void *) &p->params.kni; @@ -116,6 +121,8 @@ pipeline_port_in_params_get_ops(struct pipeline_port_in_params *p) return &rte_port_ring_reader_ipv6_frag_ops; case PIPELINE_PORT_IN_SCHED_READER: return &rte_port_sched_reader_ops; + case PIPELINE_PORT_IN_FD_READER: + return &rte_port_fd_reader_ops; #ifdef RTE_LIBRTE_KNI case PIPELINE_PORT_IN_KNI_READER: return &rte_port_kni_reader_ops; @@ -137,6 +144,7 @@ enum pipeline_port_out_type { PIPELINE_PORT_OUT_RING_WRITER_IPV4_RAS, PIPELINE_PORT_OUT_RING_WRITER_IPV6_RAS, PIPELINE_PORT_OUT_SCHED_WRITER, + PIPELINE_PORT_OUT_FD_WRITER, PIPELINE_PORT_OUT_KNI_WRITER, PIPELINE_PORT_OUT_KNI_WRITER_NODROP, PIPELINE_PORT_OUT_SINK, @@ -154,6 +162,7 @@ struct pipeline_port_out_params { struct rte_port_ring_writer_ipv4_ras_params ring_ipv4_ras; struct rte_port_ring_writer_ipv6_ras_params ring_ipv6_ras; struct rte_port_sched_writer_params sched; + struct rte_port_fd_writer_params fd; #ifdef RTE_LIBRTE_KNI struct rte_port_kni_writer_params kni; struct rte_port_kni_writer_nodrop_params kni_nodrop; @@ -184,6 +193,8 @@ pipeline_port_out_params_convert(struct pipeline_port_out_params *p) return (void *) &p->params.ring_ipv6_ras; case PIPELINE_PORT_OUT_SCHED_WRITER: return (void *) &p->params.sched; + case PIPELINE_PORT_OUT_FD_WRITER: + return (void *) &p->params.fd; #ifdef RTE_LIBRTE_KNI case PIPELINE_PORT_OUT_KNI_WRITER: return (void *) &p->params.kni; @@ -219,6 +230,8 @@ pipeline_port_out_params_get_ops(struct pipeline_port_out_params *p) return &rte_port_ring_writer_ipv6_ras_ops; case PIPELINE_PORT_OUT_SCHED_WRITER: return &rte_port_sched_writer_ops; + case PIPELINE_PORT_OUT_FD_WRITER: + return &rte_port_fd_writer_ops; #ifdef RTE_LIBRTE_KNI case PIPELINE_PORT_OUT_KNI_WRITER: return &rte_port_kni_writer_ops; diff --git a/dpdk/examples/ip_pipeline/thread_fe.c b/dpdk/examples/ip_pipeline/thread_fe.c index 6c547ca5..4590c2b5 100644 --- a/dpdk/examples/ip_pipeline/thread_fe.c +++ b/dpdk/examples/ip_pipeline/thread_fe.c @@ -70,8 +70,7 @@ app_pipeline_enable(struct app_params *app, core_id, hyper_th_id); - if ((thread_id < 0) || - ((app->core_mask & (1LLU << thread_id)) == 0)) + if ((thread_id < 0) || !app_core_is_enabled(app, thread_id)) return -1; if (app_pipeline_data(app, pipeline_id) == NULL) @@ -134,8 +133,7 @@ app_pipeline_disable(struct app_params *app, core_id, hyper_th_id); - if ((thread_id < 0) || - ((app->core_mask & (1LLU << thread_id)) == 0)) + if ((thread_id < 0) || !app_core_is_enabled(app, thread_id)) return -1; if (app_pipeline_data(app, pipeline_id) == NULL) @@ -188,8 +186,7 @@ app_thread_headroom(struct app_params *app, core_id, hyper_th_id); - if ((thread_id < 0) || - ((app->core_mask & (1LLU << thread_id)) == 0)) + if ((thread_id < 0) || !app_core_is_enabled(app, thread_id)) return -1; req = app_msg_alloc(app); diff --git a/dpdk/examples/ip_reassembly/Makefile b/dpdk/examples/ip_reassembly/Makefile index d9539a3a..85c64a38 100644 --- a/dpdk/examples/ip_reassembly/Makefile +++ b/dpdk/examples/ip_reassembly/Makefile @@ -34,7 +34,7 @@ ifeq ($(RTE_SDK),) $(error "Please define RTE_SDK environment variable") endif -# Default target, can be overriden by command line or environment +# Default target, can be overridden by command line or environment RTE_TARGET ?= x86_64-native-linuxapp-gcc include $(RTE_SDK)/mk/rte.vars.mk diff --git a/dpdk/examples/ip_reassembly/main.c b/dpdk/examples/ip_reassembly/main.c index ef09a2ed..756f90ef 100644 --- a/dpdk/examples/ip_reassembly/main.c +++ b/dpdk/examples/ip_reassembly/main.c @@ -49,9 +49,7 @@ #include #include #include -#include #include -#include #include #include #include @@ -60,12 +58,10 @@ #include #include #include -#include #include #include #include #include -#include #include #include #include @@ -86,10 +82,10 @@ #define MAX_JUMBO_PKT_LEN 9600 #define BUF_SIZE RTE_MBUF_DEFAULT_DATAROOM -#define MBUF_SIZE \ - (BUF_SIZE + sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM) +#define MBUF_DATA_SIZE RTE_MBUF_DEFAULT_BUF_SIZE #define NB_MBUF 8192 +#define MEMPOOL_CACHE_SIZE 256 /* allow max jumbo frame 9.5 KB */ #define JUMBO_FRAME_MAX_SIZE 0x2600 @@ -168,7 +164,7 @@ struct rx_queue { struct rte_mempool *pool; struct rte_lpm *lpm; struct rte_lpm6 *lpm6; - uint8_t portid; + uint16_t portid; }; struct tx_lcore_stat { @@ -201,7 +197,7 @@ static struct rte_eth_conf port_conf = { .hw_ip_checksum = 1, /**< IP checksum offload enabled */ .hw_vlan_filter = 0, /**< VLAN filtering disabled */ .jumbo_frame = 1, /**< Jumbo Frame Support disabled */ - .hw_strip_crc = 0, /**< CRC stripped by hardware */ + .hw_strip_crc = 1, /**< CRC stripped by hardware */ }, .rx_adv_conf = { .rss_conf = { @@ -279,7 +275,7 @@ static struct rte_lpm6 *socket_lpm6[RTE_MAX_NUMA_NODES]; * send burst of packets on an output interface. */ static inline uint32_t -send_burst(struct lcore_queue_conf *qconf, uint32_t thresh, uint8_t port) +send_burst(struct lcore_queue_conf *qconf, uint32_t thresh, uint16_t port) { uint32_t fill, len, k, n; struct mbuf_table *txmb; @@ -309,7 +305,7 @@ send_burst(struct lcore_queue_conf *qconf, uint32_t thresh, uint8_t port) /* Enqueue a single packet, and send burst if queue is filled */ static inline int -send_single_packet(struct rte_mbuf *m, uint8_t port) +send_single_packet(struct rte_mbuf *m, uint16_t port) { uint32_t fill, lcore_id, len; struct lcore_queue_conf *qconf; @@ -339,7 +335,7 @@ send_single_packet(struct rte_mbuf *m, uint8_t port) } static inline void -reassemble(struct rte_mbuf *m, uint8_t portid, uint32_t queue, +reassemble(struct rte_mbuf *m, uint16_t portid, uint32_t queue, struct lcore_queue_conf *qconf, uint64_t tms) { struct ether_hdr *eth_hdr; @@ -347,8 +343,8 @@ reassemble(struct rte_mbuf *m, uint8_t portid, uint32_t queue, struct rte_ip_frag_death_row *dr; struct rx_queue *rxq; void *d_addr_bytes; - uint32_t next_hop_ipv4; - uint8_t next_hop_ipv6, dst_port; + uint32_t next_hop; + uint16_t dst_port; rxq = &qconf->rx_queue_list[queue]; @@ -391,9 +387,9 @@ reassemble(struct rte_mbuf *m, uint8_t portid, uint32_t queue, ip_dst = rte_be_to_cpu_32(ip_hdr->dst_addr); /* Find destination port */ - if (rte_lpm_lookup(rxq->lpm, ip_dst, &next_hop_ipv4) == 0 && - (enabled_port_mask & 1 << next_hop_ipv4) != 0) { - dst_port = next_hop_ipv4; + if (rte_lpm_lookup(rxq->lpm, ip_dst, &next_hop) == 0 && + (enabled_port_mask & 1 << next_hop) != 0) { + dst_port = next_hop; } eth_hdr->ether_type = rte_be_to_cpu_16(ETHER_TYPE_IPv4); @@ -428,9 +424,10 @@ reassemble(struct rte_mbuf *m, uint8_t portid, uint32_t queue, } /* Find destination port */ - if (rte_lpm6_lookup(rxq->lpm6, ip_hdr->dst_addr, &next_hop_ipv6) == 0 && - (enabled_port_mask & 1 << next_hop_ipv6) != 0) { - dst_port = next_hop_ipv6; + if (rte_lpm6_lookup(rxq->lpm6, ip_hdr->dst_addr, + &next_hop) == 0 && + (enabled_port_mask & 1 << next_hop) != 0) { + dst_port = next_hop; } eth_hdr->ether_type = rte_be_to_cpu_16(ETHER_TYPE_IPv6); @@ -455,7 +452,7 @@ main_loop(__attribute__((unused)) void *dummy) unsigned lcore_id; uint64_t diff_tsc, cur_tsc, prev_tsc; int i, j, nb_rx; - uint8_t portid; + uint16_t portid; struct lcore_queue_conf *qconf; const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / US_PER_S * BURST_TX_DRAIN_US; @@ -474,7 +471,7 @@ main_loop(__attribute__((unused)) void *dummy) for (i = 0; i < qconf->n_rx_queue; i++) { portid = qconf->rx_queue_list[i].portid; - RTE_LOG(INFO, IP_RSMBL, " -- lcoreid=%u portid=%hhu\n", lcore_id, + RTE_LOG(INFO, IP_RSMBL, " -- lcoreid=%u portid=%u\n", lcore_id, portid); } @@ -719,7 +716,7 @@ parse_args(int argc, char **argv) argv[optind-1] = prgname; ret = optind-1; - optind = 0; /* reset getopt lib */ + optind = 1; /* reset getopt lib */ return ret; } @@ -733,11 +730,12 @@ print_ethaddr(const char *name, const struct ether_addr *eth_addr) /* Check the link status of all ports in up to 9s, and print them finally */ static void -check_all_ports_link_status(uint8_t port_num, uint32_t port_mask) +check_all_ports_link_status(uint16_t port_num, uint32_t port_mask) { #define CHECK_INTERVAL 100 /* 100ms */ #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ - uint8_t portid, count, all_ports_up, print_flag = 0; + uint16_t portid; + uint8_t count, all_ports_up, print_flag = 0; struct rte_eth_link link; printf("\nChecking link status"); @@ -752,14 +750,13 @@ check_all_ports_link_status(uint8_t port_num, uint32_t port_mask) /* print link status if flag set */ if (print_flag == 1) { if (link.link_status) - printf("Port %d Link Up - speed %u " - "Mbps - %s\n", (uint8_t)portid, - (unsigned)link.link_speed, + printf( + "Port%d Link Up. Speed %u Mbps - %s\n", + portid, link.link_speed, (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? ("full-duplex") : ("half-duplex\n")); else - printf("Port %d Link Down\n", - (uint8_t)portid); + printf("Port %d Link Down\n", portid); continue; } /* clear all_ports_up flag if any link down */ @@ -904,17 +901,17 @@ setup_queue_tbl(struct rx_queue *rxq, uint32_t lcore, uint32_t queue) nb_mbuf = RTE_MAX(max_flow_num, 2UL * MAX_PKT_BURST) * MAX_FRAG_NUM; nb_mbuf *= (port_conf.rxmode.max_rx_pkt_len + BUF_SIZE - 1) / BUF_SIZE; nb_mbuf *= 2; /* ipv4 and ipv6 */ - nb_mbuf += RTE_TEST_RX_DESC_DEFAULT + RTE_TEST_TX_DESC_DEFAULT; + nb_mbuf += nb_rxd + nb_txd; nb_mbuf = RTE_MAX(nb_mbuf, (uint32_t)NB_MBUF); snprintf(buf, sizeof(buf), "mbuf_pool_%u_%u", lcore, queue); - if ((rxq->pool = rte_mempool_create(buf, nb_mbuf, MBUF_SIZE, 0, - sizeof(struct rte_pktmbuf_pool_private), - rte_pktmbuf_pool_init, NULL, rte_pktmbuf_init, NULL, - socket, MEMPOOL_F_SP_PUT | MEMPOOL_F_SC_GET)) == NULL) { - RTE_LOG(ERR, IP_RSMBL, "mempool_create(%s) failed", buf); + rxq->pool = rte_pktmbuf_pool_create(buf, nb_mbuf, MEMPOOL_CACHE_SIZE, 0, + MBUF_DATA_SIZE, socket); + if (rxq->pool == NULL) { + RTE_LOG(ERR, IP_RSMBL, + "rte_pktmbuf_pool_create(%s) failed", buf); return -1; } @@ -988,7 +985,7 @@ queue_dump_stat(void) qconf = &lcore_queue_conf[lcore]; for (i = 0; i < qconf->n_rx_queue; i++) { - fprintf(stdout, " -- lcoreid=%u portid=%hhu " + fprintf(stdout, " -- lcoreid=%u portid=%u " "frag tbl stat:\n", lcore, qconf->rx_queue_list[i].portid); rte_ip_frag_table_statistics_dump(stdout, @@ -1025,7 +1022,7 @@ main(int argc, char **argv) uint16_t queueid; unsigned lcore_id = 0, rx_lcore_id = 0; uint32_t n_tx_queue, nb_lcores; - uint8_t portid; + uint16_t portid; /* init EAL */ ret = rte_eal_init(argc, argv); @@ -1063,6 +1060,11 @@ main(int argc, char **argv) qconf = &lcore_queue_conf[rx_lcore_id]; + /* limit the frame size to the maximum supported by NIC */ + rte_eth_dev_info_get(portid, &dev_info); + port_conf.rxmode.max_rx_pkt_len = RTE_MIN( + dev_info.max_rx_pktlen, port_conf.rxmode.max_rx_pkt_len); + /* get the lcore_id for this port */ while (rte_lcore_is_enabled(rx_lcore_id) == 0 || qconf->n_rx_queue == (unsigned)rx_queue_per_lcore) { @@ -1083,6 +1085,14 @@ main(int argc, char **argv) rxq->portid = portid; rxq->lpm = socket_lpm[socket]; rxq->lpm6 = socket_lpm6[socket]; + + ret = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd, + &nb_txd); + if (ret < 0) + rte_exit(EXIT_FAILURE, + "Cannot adjust number of descriptors: err=%d, port=%d\n", + ret, portid); + if (setup_queue_tbl(rxq, rx_lcore_id, queueid) < 0) rte_exit(EXIT_FAILURE, "Failed to set up queue table\n"); qconf->n_rx_queue++; @@ -1129,7 +1139,6 @@ main(int argc, char **argv) printf("txq=%u,%d,%d ", lcore_id, queueid, socket); fflush(stdout); - rte_eth_dev_info_get(portid, &dev_info); txconf = &dev_info.default_txconf; txconf->txq_flags = 0; @@ -1166,7 +1175,7 @@ main(int argc, char **argv) if (init_routing_table() < 0) rte_exit(EXIT_FAILURE, "Cannot init routing table\n"); - check_all_ports_link_status((uint8_t)nb_ports, enabled_port_mask); + check_all_ports_link_status(nb_ports, enabled_port_mask); signal(SIGUSR1, signal_handler); signal(SIGTERM, signal_handler); diff --git a/dpdk/examples/ipsec-secgw/Makefile b/dpdk/examples/ipsec-secgw/Makefile index 06b6db1e..9fd33cb7 100644 --- a/dpdk/examples/ipsec-secgw/Makefile +++ b/dpdk/examples/ipsec-secgw/Makefile @@ -38,6 +38,12 @@ RTE_TARGET ?= x86_64-native-linuxapp-gcc include $(RTE_SDK)/mk/rte.vars.mk +ifneq ($(MAKECMDGOALS),clean) +ifneq ($(CONFIG_RTE_LIBRTE_SECURITY),y) +$(error "RTE_LIBRTE_SECURITY is required to build ipsec-secgw") +endif +endif + APP = ipsec-secgw CFLAGS += -O3 -gdwarf-2 @@ -53,6 +59,7 @@ endif # # all source are stored in SRCS-y # +SRCS-y += parser.c SRCS-y += ipsec.c SRCS-y += esp.c SRCS-y += sp4.c diff --git a/dpdk/examples/ipsec-secgw/ep0.cfg b/dpdk/examples/ipsec-secgw/ep0.cfg new file mode 100644 index 00000000..299aa9e0 --- /dev/null +++ b/dpdk/examples/ipsec-secgw/ep0.cfg @@ -0,0 +1,160 @@ +########################################################################### +# IPSEC-SECGW Endpoint sample configuration +# +# The main purpose of this file is to show how to configure two systems +# back-to-back that would forward traffic through an IPsec tunnel. This +# file is the Endpoint 0 configuration. To use this configuration file, +# add the following command-line option: +# +# -f ./ep0.cfg +# +########################################################################### + +#SP IPv4 rules +sp ipv4 out esp protect 5 pri 1 dst 192.168.105.0/24 sport 0:65535 dport 0:65535 +sp ipv4 out esp protect 6 pri 1 dst 192.168.106.0/24 sport 0:65535 dport 0:65535 +sp ipv4 out esp protect 10 pri 1 dst 192.168.175.0/24 sport 0:65535 dport 0:65535 +sp ipv4 out esp protect 11 pri 1 dst 192.168.176.0/24 sport 0:65535 dport 0:65535 +sp ipv4 out esp protect 15 pri 1 dst 192.168.200.0/24 sport 0:65535 dport 0:65535 +sp ipv4 out esp protect 16 pri 1 dst 192.168.201.0/24 sport 0:65535 dport 0:65535 +sp ipv4 out esp protect 25 pri 1 dst 192.168.55.0/24 sport 0:65535 dport 0:65535 +sp ipv4 out esp protect 26 pri 1 dst 192.168.56.0/24 sport 0:65535 dport 0:65535 +sp ipv4 out esp bypass pri 1 dst 192.168.240.0/24 sport 0:65535 dport 0:65535 +sp ipv4 out esp bypass pri 1 dst 192.168.241.0/24 sport 0:65535 dport 0:65535 + +sp ipv4 in esp protect 105 pri 1 dst 192.168.115.0/24 sport 0:65535 dport 0:65535 +sp ipv4 in esp protect 106 pri 1 dst 192.168.116.0/24 sport 0:65535 dport 0:65535 +sp ipv4 in esp protect 110 pri 1 dst 192.168.185.0/24 sport 0:65535 dport 0:65535 +sp ipv4 in esp protect 111 pri 1 dst 192.168.186.0/24 sport 0:65535 dport 0:65535 +sp ipv4 in esp protect 115 pri 1 dst 192.168.210.0/24 sport 0:65535 dport 0:65535 +sp ipv4 in esp protect 116 pri 1 dst 192.168.211.0/24 sport 0:65535 dport 0:65535 +sp ipv4 in esp protect 115 pri 1 dst 192.168.210.0/24 sport 0:65535 dport 0:65535 +sp ipv4 in esp protect 125 pri 1 dst 192.168.65.0/24 sport 0:65535 dport 0:65535 +sp ipv4 in esp protect 125 pri 1 dst 192.168.65.0/24 sport 0:65535 dport 0:65535 +sp ipv4 in esp protect 126 pri 1 dst 192.168.66.0/24 sport 0:65535 dport 0:65535 +sp ipv4 in esp bypass pri 1 dst 192.168.245.0/24 sport 0:65535 dport 0:65535 +sp ipv4 in esp bypass pri 1 dst 192.168.246.0/24 sport 0:65535 dport 0:65535 + +#SP IPv6 rules +sp ipv6 out esp protect 5 pri 1 dst 0000:0000:0000:0000:5555:5555:0000:0000/96 \ +sport 0:65535 dport 0:65535 +sp ipv6 out esp protect 6 pri 1 dst 0000:0000:0000:0000:6666:6666:0000:0000/96 \ +sport 0:65535 dport 0:65535 +sp ipv6 out esp protect 10 pri 1 dst 0000:0000:1111:1111:0000:0000:0000:0000/96 \ +sport 0:65535 dport 0:65535 +sp ipv6 out esp protect 11 pri 1 dst 0000:0000:1111:1111:1111:1111:0000:0000/96 \ +sport 0:65535 dport 0:65535 +sp ipv6 out esp protect 25 pri 1 dst 0000:0000:0000:0000:aaaa:aaaa:0000:0000/96 \ +sport 0:65535 dport 0:65535 +sp ipv6 out esp protect 26 pri 1 dst 0000:0000:0000:0000:bbbb:bbbb:0000:0000/96 \ +sport 0:65535 dport 0:65535 + +sp ipv6 in esp protect 15 pri 1 dst ffff:0000:0000:0000:5555:5555:0000:0000/96 \ +sport 0:65535 dport 0:65535 +sp ipv6 in esp protect 16 pri 1 dst ffff:0000:0000:0000:6666:6666:0000:0000/96 \ +sport 0:65535 dport 0:65535 +sp ipv6 in esp protect 110 pri 1 dst ffff:0000:1111:1111:0000:0000:0000:0000/96 \ +sport 0:65535 dport 0:65535 +sp ipv6 in esp protect 111 pri 1 dst ffff:0000:1111:1111:1111:1111:0000:0000/96 \ +sport 0:65535 dport 0:65535 +sp ipv6 in esp protect 125 pri 1 dst ffff:0000:0000:0000:aaaa:aaaa:0000:0000/96 \ +sport 0:65535 dport 0:65535 +sp ipv6 in esp protect 126 pri 1 dst ffff:0000:0000:0000:bbbb:bbbb:0000:0000/96 \ +sport 0:65535 dport 0:65535 + +#SA rules +sa out 5 cipher_algo aes-128-cbc cipher_key 0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0 \ +auth_algo sha1-hmac auth_key 0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0 \ +mode ipv4-tunnel src 172.16.1.5 dst 172.16.2.5 + +sa out 6 cipher_algo aes-128-cbc cipher_key a0:a0:a0:a0:a0:a0:a0:a0:a0:a0:a0:\ +a0:a0:a0:a0:a0 auth_algo sha1-hmac auth_key a0:a0:a0:a0:a0:a0:a0:a0:a0:a0:a0:\ +a0:a0:a0:a0:a0:a0:a0:a0:a0 mode ipv4-tunnel src 172.16.1.6 dst 172.16.2.6 + +sa out 10 cipher_algo aes-128-cbc cipher_key a1:a1:a1:a1:a1:a1:a1:a1:a1:a1:a1:\ +a1:a1:a1:a1:a1 auth_algo sha1-hmac auth_key a1:a1:a1:a1:a1:a1:a1:a1:a1:a1:a1:\ +a1:a1:a1:a1:a1:a1:a1:a1:a1 mode transport + +sa out 11 cipher_algo aes-128-cbc cipher_key b2:b2:b2:b2:b2:b2:b2:b2:b2:b2:b2:\ +b2:b2:b2:b2:b2 auth_algo sha1-hmac auth_key b2:b2:b2:b2:b2:b2:b2:b2:b2:b2:b2:\ +b2:b2:b2:b2:b2:b2:b2:b2:b2 mode transport + +sa out 15 cipher_algo null auth_algo null mode ipv4-tunnel src 172.16.1.5 \ +dst 172.16.2.5 + +sa out 16 cipher_algo null auth_algo null mode ipv4-tunnel src 172.16.1.6 \ +dst 172.16.2.6 + +sa out 25 cipher_algo aes-128-cbc cipher_key c3:c3:c3:c3:c3:c3:c3:c3:c3:c3:c3:\ +c3:c3:c3:c3:c3 auth_algo sha1-hmac auth_key c3:c3:c3:c3:c3:c3:c3:c3:c3:c3:c3:\ +c3:c3:c3:c3:c3:c3:c3:c3:c3 mode ipv6-tunnel \ +src 1111:1111:1111:1111:1111:1111:1111:5555 \ +dst 2222:2222:2222:2222:2222:2222:2222:5555 + +sa out 26 cipher_algo aes-128-cbc cipher_key 4d:4d:4d:4d:4d:4d:4d:4d:4d:4d:4d:\ +4d:4d:4d:4d:4d auth_algo sha1-hmac auth_key 4d:4d:4d:4d:4d:4d:4d:4d:4d:4d:4d:\ +4d:4d:4d:4d:4d:4d:4d:4d:4d mode ipv6-tunnel \ +src 1111:1111:1111:1111:1111:1111:1111:6666 \ +dst 2222:2222:2222:2222:2222:2222:2222:6666 + +sa in 105 cipher_algo aes-128-cbc cipher_key 0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0 \ +auth_algo sha1-hmac auth_key 0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0 \ +mode ipv4-tunnel src 172.16.2.5 dst 172.16.1.5 + +sa in 106 cipher_algo aes-128-cbc cipher_key a0:a0:a0:a0:a0:a0:a0:a0:a0:a0:a0:\ +a0:a0:a0:a0:a0 auth_algo sha1-hmac auth_key a0:a0:a0:a0:a0:a0:a0:a0:a0:a0:a0:\ +a0:a0:a0:a0:a0:a0:a0:a0:a0 mode ipv4-tunnel src 172.16.2.6 dst 172.16.1.6 + +sa in 110 cipher_algo aes-128-cbc cipher_key a1:a1:a1:a1:a1:a1:a1:a1:a1:a1:a1:\ +a1:a1:a1:a1:a1 auth_algo sha1-hmac auth_key a1:a1:a1:a1:a1:a1:a1:a1:a1:a1:a1:\ +a1:a1:a1:a1:a1:a1:a1:a1:a1 mode transport + +sa in 111 cipher_algo aes-128-cbc cipher_key b2:b2:b2:b2:b2:b2:b2:b2:b2:b2:b2:\ +b2:b2:b2:b2:b2 auth_algo sha1-hmac auth_key b2:b2:b2:b2:b2:b2:b2:b2:b2:b2:b2:\ +b2:b2:b2:b2:b2:b2:b2:b2:b2 mode transport + +sa in 115 cipher_algo null auth_algo null mode ipv4-tunnel src 172.16.2.5 \ +dst 172.16.1.5 + +sa in 116 cipher_algo null auth_algo null mode ipv4-tunnel src 172.16.2.6 dst 172.16.1.6 + +sa in 125 cipher_algo aes-128-cbc cipher_key c3:c3:c3:c3:c3:c3:c3:c3:c3:c3:c3:\ +c3:c3:c3:c3:c3 auth_algo sha1-hmac auth_key c3:c3:c3:c3:c3:c3:c3:c3:c3:c3:c3:\ +c3:c3:c3:c3:c3:c3:c3:c3:c3 mode ipv6-tunnel \ +src 2222:2222:2222:2222:2222:2222:2222:5555 \ +dst 1111:1111:1111:1111:1111:1111:1111:5555 + +sa in 126 cipher_algo aes-128-cbc cipher_key 4d:4d:4d:4d:4d:4d:4d:4d:4d:4d:4d:\ +4d:4d:4d:4d:4d auth_algo sha1-hmac auth_key 4d:4d:4d:4d:4d:4d:4d:4d:4d:4d:4d:\ +4d:4d:4d:4d:4d:4d:4d:4d:4d mode ipv6-tunnel \ +src 2222:2222:2222:2222:2222:2222:2222:6666 \ +dst 1111:1111:1111:1111:1111:1111:1111:6666 + +#Routing rules +rt ipv4 dst 172.16.2.5/32 port 0 +rt ipv4 dst 172.16.2.6/32 port 1 +rt ipv4 dst 192.168.175.0/24 port 0 +rt ipv4 dst 192.168.176.0/24 port 1 +rt ipv4 dst 192.168.240.0/24 port 0 +rt ipv4 dst 192.168.241.0/24 port 1 +rt ipv4 dst 192.168.115.0/24 port 2 +rt ipv4 dst 192.168.116.0/24 port 3 +rt ipv4 dst 192.168.65.0/24 port 2 +rt ipv4 dst 192.168.66.0/24 port 3 +rt ipv4 dst 192.168.185.0/24 port 2 +rt ipv4 dst 192.168.186.0/24 port 3 +rt ipv4 dst 192.168.210.0/24 port 2 +rt ipv4 dst 192.168.211.0/24 port 3 +rt ipv4 dst 192.168.245.0/24 port 2 +rt ipv4 dst 192.168.246.0/24 port 3 + +rt ipv6 dst 2222:2222:2222:2222:2222:2222:2222:5555/116 port 0 +rt ipv6 dst 2222:2222:2222:2222:2222:2222:2222:6666/116 port 1 +rt ipv6 dst 0000:0000:1111:1111:0000:0000:0000:0000/116 port 0 +rt ipv6 dst 0000:0000:1111:1111:1111:1111:0000:0000/116 port 1 +rt ipv6 dst ffff:0000:0000:0000:aaaa:aaaa:0000:0000/116 port 2 +rt ipv6 dst ffff:0000:0000:0000:bbbb:bbbb:0000:0000/116 port 3 +rt ipv6 dst ffff:0000:0000:0000:5555:5555:0000:0000/116 port 2 +rt ipv6 dst ffff:0000:0000:0000:6666:6666:0000:0000/116 port 3 +rt ipv6 dst ffff:0000:1111:1111:0000:0000:0000:0000/116 port 2 +rt ipv6 dst ffff:0000:1111:1111:1111:1111:0000:0000/116 port 3 diff --git a/dpdk/examples/ipsec-secgw/ep1.cfg b/dpdk/examples/ipsec-secgw/ep1.cfg new file mode 100644 index 00000000..3f6ff811 --- /dev/null +++ b/dpdk/examples/ipsec-secgw/ep1.cfg @@ -0,0 +1,160 @@ +########################################################################### +# IPSEC-SECGW Endpoint1 sample configuration +# +# The main purpose of this file is to show how to configure two systems +# back-to-back that would forward traffic through an IPsec tunnel. This +# file is the Endpoint1 configuration. To use this configuration file, +# add the following command-line option: +# +# -f ./ep1.cfg +# +########################################################################### + +#SP IPv4 rules +sp ipv4 in esp protect 5 pri 1 dst 192.168.105.0/24 sport 0:65535 dport 0:65535 +sp ipv4 in esp protect 6 pri 1 dst 192.168.106.0/24 sport 0:65535 dport 0:65535 +sp ipv4 in esp protect 10 pri 1 dst 192.168.175.0/24 sport 0:65535 dport 0:65535 +sp ipv4 in esp protect 11 pri 1 dst 192.168.176.0/24 sport 0:65535 dport 0:65535 +sp ipv4 in esp protect 15 pri 1 dst 192.168.200.0/24 sport 0:65535 dport 0:65535 +sp ipv4 in esp protect 16 pri 1 dst 192.168.201.0/24 sport 0:65535 dport 0:65535 +sp ipv4 in esp protect 25 pri 1 dst 192.168.55.0/24 sport 0:65535 dport 0:65535 +sp ipv4 in esp protect 26 pri 1 dst 192.168.56.0/24 sport 0:65535 dport 0:65535 +sp ipv4 in esp bypass dst 192.168.240.0/24 sport 0:65535 dport 0:65535 +sp ipv4 in esp bypass dst 192.168.241.0/24 sport 0:65535 dport 0:65535 + +sp ipv4 out esp protect 105 pri 1 dst 192.168.115.0/24 sport 0:65535 dport 0:65535 +sp ipv4 out esp protect 106 pri 1 dst 192.168.116.0/24 sport 0:65535 dport 0:65535 +sp ipv4 out esp protect 110 pri 1 dst 192.168.185.0/24 sport 0:65535 dport 0:65535 +sp ipv4 out esp protect 111 pri 1 dst 192.168.186.0/24 sport 0:65535 dport 0:65535 +sp ipv4 out esp protect 115 pri 1 dst 192.168.210.0/24 sport 0:65535 dport 0:65535 +sp ipv4 out esp protect 116 pri 1 dst 192.168.211.0/24 sport 0:65535 dport 0:65535 +sp ipv4 out esp protect 115 pri 1 dst 192.168.210.0/24 sport 0:65535 dport 0:65535 +sp ipv4 out esp protect 125 pri 1 dst 192.168.65.0/24 sport 0:65535 dport 0:65535 +sp ipv4 out esp protect 125 pri 1 dst 192.168.65.0/24 sport 0:65535 dport 0:65535 +sp ipv4 out esp protect 126 pri 1 dst 192.168.66.0/24 sport 0:65535 dport 0:65535 +sp ipv4 out esp bypass pri 1 dst 192.168.245.0/24 sport 0:65535 dport 0:65535 +sp ipv4 out esp bypass pri 1 dst 192.168.246.0/24 sport 0:65535 dport 0:65535 + +#SP IPv6 rules +sp ipv6 in esp protect 5 pri 1 dst 0000:0000:0000:0000:5555:5555:0000:0000/96 \ +sport 0:65535 dport 0:65535 +sp ipv6 in esp protect 6 pri 1 dst 0000:0000:0000:0000:6666:6666:0000:0000/96 \ +sport 0:65535 dport 0:65535 +sp ipv6 in esp protect 10 pri 1 dst 0000:0000:1111:1111:0000:0000:0000:0000/96 \ +sport 0:65535 dport 0:65535 +sp ipv6 in esp protect 11 pri 1 dst 0000:0000:1111:1111:1111:1111:0000:0000/96 \ +sport 0:65535 dport 0:65535 +sp ipv6 in esp protect 25 pri 1 dst 0000:0000:0000:0000:aaaa:aaaa:0000:0000/96 \ +sport 0:65535 dport 0:65535 +sp ipv6 in esp protect 26 pri 1 dst 0000:0000:0000:0000:bbbb:bbbb:0000:0000/96 \ +sport 0:65535 dport 0:65535 + +sp ipv6 out esp protect 15 pri 1 dst ffff:0000:0000:0000:5555:5555:0000:0000/96 \ +sport 0:65535 dport 0:65535 +sp ipv6 out esp protect 16 pri 1 dst ffff:0000:0000:0000:6666:6666:0000:0000/96 \ +sport 0:65535 dport 0:65535 +sp ipv6 out esp protect 110 pri 1 dst ffff:0000:1111:1111:0000:0000:0000:0000/96 \ +sport 0:65535 dport 0:65535 +sp ipv6 out esp protect 111 pri 1 dst ffff:0000:1111:1111:1111:1111:0000:0000/96 \ +sport 0:65535 dport 0:65535 +sp ipv6 out esp protect 125 pri 1 dst ffff:0000:0000:0000:aaaa:aaaa:0000:0000/96 \ +sport 0:65535 dport 0:65535 +sp ipv6 out esp protect 126 pri 1 dst ffff:0000:0000:0000:bbbb:bbbb:0000:0000/96 \ +sport 0:65535 dport 0:65535 + +#SA rules +sa in 5 cipher_algo aes-128-cbc cipher_key 0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0 \ +auth_algo sha1-hmac auth_key 0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0 \ +mode ipv4-tunnel src 172.16.1.5 dst 172.16.2.5 + +sa in 6 cipher_algo aes-128-cbc cipher_key a0:a0:a0:a0:a0:a0:a0:a0:a0:a0:a0:\ +a0:a0:a0:a0:a0 auth_algo sha1-hmac auth_key a0:a0:a0:a0:a0:a0:a0:a0:a0:a0:a0:\ +a0:a0:a0:a0:a0:a0:a0:a0:a0 mode ipv4-tunnel src 172.16.1.6 dst 172.16.2.6 + +sa in 10 cipher_algo aes-128-cbc cipher_key a1:a1:a1:a1:a1:a1:a1:a1:a1:a1:a1:\ +a1:a1:a1:a1:a1 auth_algo sha1-hmac auth_key a1:a1:a1:a1:a1:a1:a1:a1:a1:a1:a1:\ +a1:a1:a1:a1:a1:a1:a1:a1:a1 mode transport + +sa in 11 cipher_algo aes-128-cbc cipher_key b2:b2:b2:b2:b2:b2:b2:b2:b2:b2:b2:\ +b2:b2:b2:b2:b2 auth_algo sha1-hmac auth_key b2:b2:b2:b2:b2:b2:b2:b2:b2:b2:b2:\ +b2:b2:b2:b2:b2:b2:b2:b2:b2 mode transport + +sa in 15 cipher_algo null auth_algo null mode ipv4-tunnel src 172.16.1.5 \ +dst 172.16.2.5 + +sa in 16 cipher_algo null auth_algo null mode ipv4-tunnel src 172.16.1.6 \ +dst 172.16.2.6 + +sa in 25 cipher_algo aes-128-cbc cipher_key c3:c3:c3:c3:c3:c3:c3:c3:c3:c3:c3:\ +c3:c3:c3:c3:c3 auth_algo sha1-hmac auth_key c3:c3:c3:c3:c3:c3:c3:c3:c3:c3:c3:\ +c3:c3:c3:c3:c3:c3:c3:c3:c3 mode ipv6-tunnel \ +src 1111:1111:1111:1111:1111:1111:1111:5555 \ +dst 2222:2222:2222:2222:2222:2222:2222:5555 + +sa in 26 cipher_algo aes-128-cbc cipher_key 4d:4d:4d:4d:4d:4d:4d:4d:4d:4d:4d:\ +4d:4d:4d:4d:4d auth_algo sha1-hmac auth_key 4d:4d:4d:4d:4d:4d:4d:4d:4d:4d:4d:\ +4d:4d:4d:4d:4d:4d:4d:4d:4d mode ipv6-tunnel \ +src 1111:1111:1111:1111:1111:1111:1111:6666 \ +dst 2222:2222:2222:2222:2222:2222:2222:6666 + +sa out 105 cipher_algo aes-128-cbc cipher_key 0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0 \ +auth_algo sha1-hmac auth_key 0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0 \ +mode ipv4-tunnel src 172.16.2.5 dst 172.16.1.5 + +sa out 106 cipher_algo aes-128-cbc cipher_key a0:a0:a0:a0:a0:a0:a0:a0:a0:a0:a0:\ +a0:a0:a0:a0:a0 auth_algo sha1-hmac auth_key a0:a0:a0:a0:a0:a0:a0:a0:a0:a0:a0:\ +a0:a0:a0:a0:a0:a0:a0:a0:a0 mode ipv4-tunnel src 172.16.2.6 dst 172.16.1.6 + +sa out 110 cipher_algo aes-128-cbc cipher_key a1:a1:a1:a1:a1:a1:a1:a1:a1:a1:a1:\ +a1:a1:a1:a1:a1 auth_algo sha1-hmac auth_key a1:a1:a1:a1:a1:a1:a1:a1:a1:a1:a1:\ +a1:a1:a1:a1:a1:a1:a1:a1:a1 mode transport + +sa out 111 cipher_algo aes-128-cbc cipher_key b2:b2:b2:b2:b2:b2:b2:b2:b2:b2:b2:\ +b2:b2:b2:b2:b2 auth_algo sha1-hmac auth_key b2:b2:b2:b2:b2:b2:b2:b2:b2:b2:b2:\ +b2:b2:b2:b2:b2:b2:b2:b2:b2 mode transport + +sa out 115 cipher_algo null auth_algo null mode ipv4-tunnel src 172.16.2.5 \ +dst 172.16.1.5 + +sa out 116 cipher_algo null auth_algo null mode ipv4-tunnel src 172.16.2.6 dst 172.16.1.6 + +sa out 125 cipher_algo aes-128-cbc cipher_key c3:c3:c3:c3:c3:c3:c3:c3:c3:c3:c3:\ +c3:c3:c3:c3:c3 auth_algo sha1-hmac auth_key c3:c3:c3:c3:c3:c3:c3:c3:c3:c3:c3:\ +c3:c3:c3:c3:c3:c3:c3:c3:c3 mode ipv6-tunnel \ +src 2222:2222:2222:2222:2222:2222:2222:5555 \ +dst 1111:1111:1111:1111:1111:1111:1111:5555 + +sa out 126 cipher_algo aes-128-cbc cipher_key 4d:4d:4d:4d:4d:4d:4d:4d:4d:4d:4d:\ +4d:4d:4d:4d:4d auth_algo sha1-hmac auth_key 4d:4d:4d:4d:4d:4d:4d:4d:4d:4d:4d:\ +4d:4d:4d:4d:4d:4d:4d:4d:4d mode ipv6-tunnel \ +src 2222:2222:2222:2222:2222:2222:2222:6666 \ +dst 1111:1111:1111:1111:1111:1111:1111:6666 + +#Routing rules +rt ipv4 dst 172.16.1.5/32 port 0 +rt ipv4 dst 172.16.1.6/32 port 1 +rt ipv4 dst 192.168.185.0/24 port 0 +rt ipv4 dst 192.168.186.0/24 port 1 +rt ipv4 dst 192.168.245.0/24 port 0 +rt ipv4 dst 192.168.246.0/24 port 1 +rt ipv4 dst 192.168.105.0/24 port 2 +rt ipv4 dst 192.168.106.0/24 port 3 +rt ipv4 dst 192.168.55.0/24 port 2 +rt ipv4 dst 192.168.56.0/24 port 3 +rt ipv4 dst 192.168.175.0/24 port 2 +rt ipv4 dst 192.168.176.0/24 port 3 +rt ipv4 dst 192.168.200.0/24 port 2 +rt ipv4 dst 192.168.201.0/24 port 3 +rt ipv4 dst 192.168.240.0/24 port 2 +rt ipv4 dst 192.168.241.0/24 port 3 + +rt ipv6 dst 1111:1111:1111:1111:1111:1111:1111:5555/116 port 0 +rt ipv6 dst 1111:1111:1111:1111:1111:1111:1111:6666/116 port 1 +rt ipv6 dst ffff:0000:1111:1111:0000:0000:0000:0000/116 port 0 +rt ipv6 dst ffff:0000:1111:1111:1111:1111:0000:0000/116 port 1 +rt ipv6 dst 0000:0000:0000:0000:aaaa:aaaa:0000:0000/116 port 2 +rt ipv6 dst 0000:0000:0000:0000:bbbb:bbbb:0000:0000/116 port 3 +rt ipv6 dst 0000:0000:0000:0000:5555:5555:0000:0000/116 port 2 +rt ipv6 dst 0000:0000:0000:0000:6666:6666:0000:0000/116 port 3 +rt ipv6 dst 0000:0000:1111:1111:0000:0000:0000:0000/116 port 2 +rt ipv6 dst 0000:0000:1111:1111:1111:1111:0000:0000/116 port 3 diff --git a/dpdk/examples/ipsec-secgw/esp.c b/dpdk/examples/ipsec-secgw/esp.c index 05caa77a..c3efe52b 100644 --- a/dpdk/examples/ipsec-secgw/esp.c +++ b/dpdk/examples/ipsec-secgw/esp.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -50,21 +50,6 @@ #include "esp.h" #include "ipip.h" -static inline void -random_iv_u64(uint64_t *buf, uint16_t n) -{ - uint32_t left = n & 0x7; - uint32_t i; - - RTE_ASSERT((n & 0x3) == 0); - - for (i = 0; i < (n >> 3); i++) - buf[i] = rte_rand(); - - if (left) - *((uint32_t *)&buf[i]) = (uint32_t)lrand48(); -} - int esp_inbound(struct rte_mbuf *m, struct ipsec_sa *sa, struct rte_crypto_op *cop) @@ -73,8 +58,11 @@ esp_inbound(struct rte_mbuf *m, struct ipsec_sa *sa, struct rte_crypto_sym_op *sym_cop; int32_t payload_len, ip_hdr_len; - RTE_ASSERT(m != NULL); RTE_ASSERT(sa != NULL); + if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) + return 0; + + RTE_ASSERT(m != NULL); RTE_ASSERT(cop != NULL); ip4 = rte_pktmbuf_mtod(m, struct ip *); @@ -93,33 +81,85 @@ esp_inbound(struct rte_mbuf *m, struct ipsec_sa *sa, sizeof(struct esp_hdr) - sa->iv_len - sa->digest_len; if ((payload_len & (sa->block_size - 1)) || (payload_len <= 0)) { - RTE_LOG(DEBUG, IPSEC_ESP, "payload %d not multiple of %u\n", + RTE_LOG_DP(DEBUG, IPSEC_ESP, "payload %d not multiple of %u\n", payload_len, sa->block_size); return -EINVAL; } - sym_cop = (struct rte_crypto_sym_op *)(cop + 1); - + sym_cop = get_sym_cop(cop); sym_cop->m_src = m; - sym_cop->cipher.data.offset = ip_hdr_len + sizeof(struct esp_hdr) + - sa->iv_len; - sym_cop->cipher.data.length = payload_len; - sym_cop->cipher.iv.data = rte_pktmbuf_mtod_offset(m, void*, - ip_hdr_len + sizeof(struct esp_hdr)); - sym_cop->cipher.iv.phys_addr = rte_pktmbuf_mtophys_offset(m, - ip_hdr_len + sizeof(struct esp_hdr)); - sym_cop->cipher.iv.length = sa->iv_len; + if (sa->aead_algo == RTE_CRYPTO_AEAD_AES_GCM) { + sym_cop->aead.data.offset = ip_hdr_len + sizeof(struct esp_hdr) + + sa->iv_len; + sym_cop->aead.data.length = payload_len; - sym_cop->auth.data.offset = ip_hdr_len; - sym_cop->auth.data.length = sizeof(struct esp_hdr) + - sa->iv_len + payload_len; + struct cnt_blk *icb; + uint8_t *aad; + uint8_t *iv = RTE_PTR_ADD(ip4, ip_hdr_len + sizeof(struct esp_hdr)); - sym_cop->auth.digest.data = rte_pktmbuf_mtod_offset(m, void*, - rte_pktmbuf_pkt_len(m) - sa->digest_len); - sym_cop->auth.digest.phys_addr = rte_pktmbuf_mtophys_offset(m, - rte_pktmbuf_pkt_len(m) - sa->digest_len); - sym_cop->auth.digest.length = sa->digest_len; + icb = get_cnt_blk(m); + icb->salt = sa->salt; + memcpy(&icb->iv, iv, 8); + icb->cnt = rte_cpu_to_be_32(1); + + aad = get_aad(m); + memcpy(aad, iv - sizeof(struct esp_hdr), 8); + sym_cop->aead.aad.data = aad; + sym_cop->aead.aad.phys_addr = rte_pktmbuf_iova_offset(m, + aad - rte_pktmbuf_mtod(m, uint8_t *)); + + sym_cop->aead.digest.data = rte_pktmbuf_mtod_offset(m, void*, + rte_pktmbuf_pkt_len(m) - sa->digest_len); + sym_cop->aead.digest.phys_addr = rte_pktmbuf_iova_offset(m, + rte_pktmbuf_pkt_len(m) - sa->digest_len); + } else { + sym_cop->cipher.data.offset = ip_hdr_len + sizeof(struct esp_hdr) + + sa->iv_len; + sym_cop->cipher.data.length = payload_len; + + struct cnt_blk *icb; + uint8_t *iv = RTE_PTR_ADD(ip4, ip_hdr_len + sizeof(struct esp_hdr)); + uint8_t *iv_ptr = rte_crypto_op_ctod_offset(cop, + uint8_t *, IV_OFFSET); + + switch (sa->cipher_algo) { + case RTE_CRYPTO_CIPHER_NULL: + case RTE_CRYPTO_CIPHER_AES_CBC: + /* Copy IV at the end of crypto operation */ + rte_memcpy(iv_ptr, iv, sa->iv_len); + break; + case RTE_CRYPTO_CIPHER_AES_CTR: + icb = get_cnt_blk(m); + icb->salt = sa->salt; + memcpy(&icb->iv, iv, 8); + icb->cnt = rte_cpu_to_be_32(1); + break; + default: + RTE_LOG(ERR, IPSEC_ESP, "unsupported cipher algorithm %u\n", + sa->cipher_algo); + return -EINVAL; + } + + switch (sa->auth_algo) { + case RTE_CRYPTO_AUTH_NULL: + case RTE_CRYPTO_AUTH_SHA1_HMAC: + case RTE_CRYPTO_AUTH_SHA256_HMAC: + sym_cop->auth.data.offset = ip_hdr_len; + sym_cop->auth.data.length = sizeof(struct esp_hdr) + + sa->iv_len + payload_len; + break; + default: + RTE_LOG(ERR, IPSEC_ESP, "unsupported auth algorithm %u\n", + sa->auth_algo); + return -EINVAL; + } + + sym_cop->auth.digest.data = rte_pktmbuf_mtod_offset(m, void*, + rte_pktmbuf_pkt_len(m) - sa->digest_len); + sym_cop->auth.digest.phys_addr = rte_pktmbuf_iova_offset(m, + rte_pktmbuf_pkt_len(m) - sa->digest_len); + } return 0; } @@ -138,29 +178,44 @@ esp_inbound_post(struct rte_mbuf *m, struct ipsec_sa *sa, RTE_ASSERT(sa != NULL); RTE_ASSERT(cop != NULL); + if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) { + if (m->ol_flags & PKT_RX_SEC_OFFLOAD) { + if (m->ol_flags & PKT_RX_SEC_OFFLOAD_FAILED) + cop->status = RTE_CRYPTO_OP_STATUS_ERROR; + else + cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; + } else + cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; + } + if (cop->status != RTE_CRYPTO_OP_STATUS_SUCCESS) { RTE_LOG(ERR, IPSEC_ESP, "failed crypto op\n"); return -1; } - nexthdr = rte_pktmbuf_mtod_offset(m, uint8_t*, - rte_pktmbuf_pkt_len(m) - sa->digest_len - 1); - pad_len = nexthdr - 1; + if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO && + sa->ol_flags & RTE_SECURITY_RX_HW_TRAILER_OFFLOAD) { + nexthdr = &m->inner_esp_next_proto; + } else { + nexthdr = rte_pktmbuf_mtod_offset(m, uint8_t*, + rte_pktmbuf_pkt_len(m) - sa->digest_len - 1); + pad_len = nexthdr - 1; - padding = pad_len - *pad_len; - for (i = 0; i < *pad_len; i++) { - if (padding[i] != i + 1) { - RTE_LOG(ERR, IPSEC_ESP, "invalid padding\n"); + padding = pad_len - *pad_len; + for (i = 0; i < *pad_len; i++) { + if (padding[i] != i + 1) { + RTE_LOG(ERR, IPSEC_ESP, "invalid padding\n"); + return -EINVAL; + } + } + + if (rte_pktmbuf_trim(m, *pad_len + 2 + sa->digest_len)) { + RTE_LOG(ERR, IPSEC_ESP, + "failed to remove pad_len + digest\n"); return -EINVAL; } } - if (rte_pktmbuf_trim(m, *pad_len + 2 + sa->digest_len)) { - RTE_LOG(ERR, IPSEC_ESP, - "failed to remove pad_len + digest\n"); - return -EINVAL; - } - if (unlikely(sa->flags == TRANSPORT)) { ip = rte_pktmbuf_mtod(m, struct ip *); ip4 = (struct ip *)rte_pktmbuf_adj(m, @@ -174,7 +229,8 @@ esp_inbound_post(struct rte_mbuf *m, struct ipsec_sa *sa, /* XXX No option headers supported */ memmove(ip6, ip, sizeof(struct ip6_hdr)); ip6->ip6_nxt = *nexthdr; - ip6->ip6_plen = htons(rte_pktmbuf_data_len(m)); + ip6->ip6_plen = htons(rte_pktmbuf_data_len(m) - + sizeof(struct ip6_hdr)); } } else ipip_inbound(m, sizeof(struct esp_hdr) + sa->iv_len); @@ -189,14 +245,13 @@ esp_outbound(struct rte_mbuf *m, struct ipsec_sa *sa, struct ip *ip4; struct ip6_hdr *ip6; struct esp_hdr *esp = NULL; - uint8_t *padding, *new_ip, nlp; + uint8_t *padding = NULL, *new_ip, nlp; struct rte_crypto_sym_op *sym_cop; int32_t i; uint16_t pad_payload_len, pad_len, ip_hdr_len; RTE_ASSERT(m != NULL); RTE_ASSERT(sa != NULL); - RTE_ASSERT(cop != NULL); ip_hdr_len = 0; @@ -246,12 +301,19 @@ esp_outbound(struct rte_mbuf *m, struct ipsec_sa *sa, return -EINVAL; } - padding = (uint8_t *)rte_pktmbuf_append(m, pad_len + sa->digest_len); - if (unlikely(padding == NULL)) { - RTE_LOG(ERR, IPSEC_ESP, "not enough mbuf trailing space\n"); - return -ENOSPC; + /* Add trailer padding if it is not constructed by HW */ + if (sa->type != RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO || + (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO && + !(sa->ol_flags & RTE_SECURITY_TX_HW_TRAILER_OFFLOAD))) { + padding = (uint8_t *)rte_pktmbuf_append(m, pad_len + + sa->digest_len); + if (unlikely(padding == NULL)) { + RTE_LOG(ERR, IPSEC_ESP, + "not enough mbuf trailing space\n"); + return -ENOSPC; + } + rte_prefetch0(padding); } - rte_prefetch0(padding); switch (sa->flags) { case IP4_TUNNEL: @@ -269,68 +331,157 @@ esp_outbound(struct rte_mbuf *m, struct ipsec_sa *sa, sizeof(struct esp_hdr) + sa->iv_len); memmove(new_ip, ip4, ip_hdr_len); esp = (struct esp_hdr *)(new_ip + ip_hdr_len); + ip4 = (struct ip *)new_ip; if (likely(ip4->ip_v == IPVERSION)) { - ip4 = (struct ip *)new_ip; ip4->ip_p = IPPROTO_ESP; ip4->ip_len = htons(rte_pktmbuf_data_len(m)); } else { ip6 = (struct ip6_hdr *)new_ip; ip6->ip6_nxt = IPPROTO_ESP; - ip6->ip6_plen = htons(rte_pktmbuf_data_len(m)); + ip6->ip6_plen = htons(rte_pktmbuf_data_len(m) - + sizeof(struct ip6_hdr)); } } sa->seq++; esp->spi = rte_cpu_to_be_32(sa->spi); - esp->seq = rte_cpu_to_be_32(sa->seq); + esp->seq = rte_cpu_to_be_32((uint32_t)sa->seq); - if (sa->cipher_algo == RTE_CRYPTO_CIPHER_AES_CBC) - random_iv_u64((uint64_t *)(esp + 1), sa->iv_len); + /* set iv */ + uint64_t *iv = (uint64_t *)(esp + 1); + if (sa->aead_algo == RTE_CRYPTO_AEAD_AES_GCM) { + *iv = rte_cpu_to_be_64(sa->seq); + } else { + switch (sa->cipher_algo) { + case RTE_CRYPTO_CIPHER_NULL: + case RTE_CRYPTO_CIPHER_AES_CBC: + memset(iv, 0, sa->iv_len); + break; + case RTE_CRYPTO_CIPHER_AES_CTR: + *iv = rte_cpu_to_be_64(sa->seq); + break; + default: + RTE_LOG(ERR, IPSEC_ESP, + "unsupported cipher algorithm %u\n", + sa->cipher_algo); + return -EINVAL; + } + } - /* Fill pad_len using default sequential scheme */ - for (i = 0; i < pad_len - 2; i++) - padding[i] = i + 1; - padding[pad_len - 2] = pad_len - 2; - padding[pad_len - 1] = nlp; - - sym_cop = (struct rte_crypto_sym_op *)(cop + 1); + if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) { + if (sa->ol_flags & RTE_SECURITY_TX_HW_TRAILER_OFFLOAD) { + /* Set the inner esp next protocol for HW trailer */ + m->inner_esp_next_proto = nlp; + m->packet_type |= RTE_PTYPE_TUNNEL_ESP; + } else { + padding[pad_len - 2] = pad_len - 2; + padding[pad_len - 1] = nlp; + } + goto done; + } + RTE_ASSERT(cop != NULL); + sym_cop = get_sym_cop(cop); sym_cop->m_src = m; - sym_cop->cipher.data.offset = ip_hdr_len + sizeof(struct esp_hdr) + - sa->iv_len; - sym_cop->cipher.data.length = pad_payload_len; - sym_cop->cipher.iv.data = rte_pktmbuf_mtod_offset(m, uint8_t *, - ip_hdr_len + sizeof(struct esp_hdr)); - sym_cop->cipher.iv.phys_addr = rte_pktmbuf_mtophys_offset(m, - ip_hdr_len + sizeof(struct esp_hdr)); - sym_cop->cipher.iv.length = sa->iv_len; + if (sa->aead_algo == RTE_CRYPTO_AEAD_AES_GCM) { + uint8_t *aad; - sym_cop->auth.data.offset = ip_hdr_len; - sym_cop->auth.data.length = sizeof(struct esp_hdr) + sa->iv_len + - pad_payload_len; + sym_cop->aead.data.offset = ip_hdr_len + + sizeof(struct esp_hdr) + sa->iv_len; + sym_cop->aead.data.length = pad_payload_len; - sym_cop->auth.digest.data = rte_pktmbuf_mtod_offset(m, uint8_t *, + /* Fill pad_len using default sequential scheme */ + for (i = 0; i < pad_len - 2; i++) + padding[i] = i + 1; + padding[pad_len - 2] = pad_len - 2; + padding[pad_len - 1] = nlp; + + struct cnt_blk *icb = get_cnt_blk(m); + icb->salt = sa->salt; + icb->iv = rte_cpu_to_be_64(sa->seq); + icb->cnt = rte_cpu_to_be_32(1); + + aad = get_aad(m); + memcpy(aad, esp, 8); + sym_cop->aead.aad.data = aad; + sym_cop->aead.aad.phys_addr = rte_pktmbuf_iova_offset(m, + aad - rte_pktmbuf_mtod(m, uint8_t *)); + + sym_cop->aead.digest.data = rte_pktmbuf_mtod_offset(m, uint8_t *, rte_pktmbuf_pkt_len(m) - sa->digest_len); - sym_cop->auth.digest.phys_addr = rte_pktmbuf_mtophys_offset(m, + sym_cop->aead.digest.phys_addr = rte_pktmbuf_iova_offset(m, rte_pktmbuf_pkt_len(m) - sa->digest_len); - sym_cop->auth.digest.length = sa->digest_len; + } else { + switch (sa->cipher_algo) { + case RTE_CRYPTO_CIPHER_NULL: + case RTE_CRYPTO_CIPHER_AES_CBC: + sym_cop->cipher.data.offset = ip_hdr_len + + sizeof(struct esp_hdr); + sym_cop->cipher.data.length = pad_payload_len + sa->iv_len; + break; + case RTE_CRYPTO_CIPHER_AES_CTR: + sym_cop->cipher.data.offset = ip_hdr_len + + sizeof(struct esp_hdr) + sa->iv_len; + sym_cop->cipher.data.length = pad_payload_len; + break; + default: + RTE_LOG(ERR, IPSEC_ESP, "unsupported cipher algorithm %u\n", + sa->cipher_algo); + return -EINVAL; + } + /* Fill pad_len using default sequential scheme */ + for (i = 0; i < pad_len - 2; i++) + padding[i] = i + 1; + padding[pad_len - 2] = pad_len - 2; + padding[pad_len - 1] = nlp; + + struct cnt_blk *icb = get_cnt_blk(m); + icb->salt = sa->salt; + icb->iv = rte_cpu_to_be_64(sa->seq); + icb->cnt = rte_cpu_to_be_32(1); + + switch (sa->auth_algo) { + case RTE_CRYPTO_AUTH_NULL: + case RTE_CRYPTO_AUTH_SHA1_HMAC: + case RTE_CRYPTO_AUTH_SHA256_HMAC: + sym_cop->auth.data.offset = ip_hdr_len; + sym_cop->auth.data.length = sizeof(struct esp_hdr) + + sa->iv_len + pad_payload_len; + break; + default: + RTE_LOG(ERR, IPSEC_ESP, "unsupported auth algorithm %u\n", + sa->auth_algo); + return -EINVAL; + } + + sym_cop->auth.digest.data = rte_pktmbuf_mtod_offset(m, uint8_t *, + rte_pktmbuf_pkt_len(m) - sa->digest_len); + sym_cop->auth.digest.phys_addr = rte_pktmbuf_iova_offset(m, + rte_pktmbuf_pkt_len(m) - sa->digest_len); + } + +done: return 0; } int -esp_outbound_post(struct rte_mbuf *m __rte_unused, - struct ipsec_sa *sa __rte_unused, - struct rte_crypto_op *cop) +esp_outbound_post(struct rte_mbuf *m, + struct ipsec_sa *sa, + struct rte_crypto_op *cop) { RTE_ASSERT(m != NULL); RTE_ASSERT(sa != NULL); - RTE_ASSERT(cop != NULL); - if (cop->status != RTE_CRYPTO_OP_STATUS_SUCCESS) { - RTE_LOG(ERR, IPSEC_ESP, "Failed crypto op\n"); - return -1; + if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) { + m->ol_flags |= PKT_TX_SEC_OFFLOAD; + } else { + RTE_ASSERT(cop != NULL); + if (cop->status != RTE_CRYPTO_OP_STATUS_SUCCESS) { + RTE_LOG(ERR, IPSEC_ESP, "Failed crypto op\n"); + return -1; + } } return 0; diff --git a/dpdk/examples/ipsec-secgw/esp.h b/dpdk/examples/ipsec-secgw/esp.h index fa5cc8af..23601e37 100644 --- a/dpdk/examples/ipsec-secgw/esp.h +++ b/dpdk/examples/ipsec-secgw/esp.h @@ -35,16 +35,6 @@ struct mbuf; -/* RFC4303 */ -struct esp_hdr { - uint32_t spi; - uint32_t seq; - /* Payload */ - /* Padding */ - /* Pad Length */ - /* Next Header */ - /* Integrity Check Value - ICV */ -}; int esp_inbound(struct rte_mbuf *m, struct ipsec_sa *sa, diff --git a/dpdk/examples/ipsec-secgw/ipip.h b/dpdk/examples/ipsec-secgw/ipip.h index ff1dccdb..93393d52 100644 --- a/dpdk/examples/ipsec-secgw/ipip.h +++ b/dpdk/examples/ipsec-secgw/ipip.h @@ -72,7 +72,8 @@ ipip_outbound(struct rte_mbuf *m, uint32_t offset, uint32_t is_ipv6, /* Per RFC4301 5.1.2.1 */ outip6->ip6_flow = htonl(IP6_VERSION << 28 | ds_ecn << 20); - outip6->ip6_plen = htons(rte_pktmbuf_data_len(m)); + outip6->ip6_plen = htons(rte_pktmbuf_data_len(m) - + sizeof(struct ip6_hdr)); outip6->ip6_nxt = IPPROTO_ESP; outip6->ip6_hops = IPDEFTTL; diff --git a/dpdk/examples/ipsec-secgw/ipsec-secgw.c b/dpdk/examples/ipsec-secgw/ipsec-secgw.c index 266ae205..b5ec70a1 100644 --- a/dpdk/examples/ipsec-secgw/ipsec-secgw.c +++ b/dpdk/examples/ipsec-secgw/ipsec-secgw.c @@ -57,7 +57,6 @@ #include #include #include -#include #include #include #include @@ -72,6 +71,7 @@ #include #include "ipsec.h" +#include "parser.h" #define RTE_LOGTYPE_IPSEC RTE_LOGTYPE_USER1 @@ -81,6 +81,7 @@ #define NB_MBUF (32000) +#define CDEV_QUEUE_DESC 2048 #define CDEV_MAP_ENTRIES 1024 #define CDEV_MP_NB_OBJS 2048 #define CDEV_MP_CACHE_SZ 64 @@ -88,8 +89,6 @@ #define OPTION_CONFIG "config" #define OPTION_SINGLE_SA "single-sa" -#define OPTION_EP0 "ep0" -#define OPTION_EP1 "ep1" #define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */ @@ -158,18 +157,18 @@ static uint32_t enabled_port_mask; static uint32_t unprotected_port_mask; static int32_t promiscuous_on = 1; static int32_t numa_on = 1; /**< NUMA is enabled by default. */ -static int32_t ep = -1; /**< Endpoint configuration (0 or 1) */ static uint32_t nb_lcores; static uint32_t single_sa; static uint32_t single_sa_idx; +static uint32_t frame_size; struct lcore_rx_queue { - uint8_t port_id; + uint16_t port_id; uint8_t queue_id; } __rte_cache_aligned; struct lcore_params { - uint8_t port_id; + uint16_t port_id; uint8_t queue_id; uint8_t lcore_id; } __rte_cache_aligned; @@ -205,11 +204,9 @@ static struct rte_eth_conf port_conf = { .mq_mode = ETH_MQ_RX_RSS, .max_rx_pkt_len = ETHER_MAX_LEN, .split_hdr_size = 0, - .header_split = 0, /**< Header Split disabled */ - .hw_ip_checksum = 1, /**< IP checksum offload enabled */ - .hw_vlan_filter = 0, /**< VLAN filtering disabled */ - .jumbo_frame = 0, /**< Jumbo Frame Support disabled */ - .hw_strip_crc = 0, /**< CRC stripped by hardware */ + .offloads = DEV_RX_OFFLOAD_CHECKSUM | + DEV_RX_OFFLOAD_CRC_STRIP, + .ignore_offload_bitfield = 1, }, .rx_adv_conf = { .rss_conf = { @@ -291,7 +288,7 @@ prepare_traffic(struct rte_mbuf **pkts, struct ipsec_traffic *t, } static inline void -prepare_tx_pkt(struct rte_mbuf *pkt, uint8_t port) +prepare_tx_pkt(struct rte_mbuf *pkt, uint16_t port) { struct ip *ip; struct ether_hdr *ethhdr; @@ -321,7 +318,7 @@ prepare_tx_pkt(struct rte_mbuf *pkt, uint8_t port) } static inline void -prepare_tx_burst(struct rte_mbuf *pkts[], uint16_t nb_pkts, uint8_t port) +prepare_tx_burst(struct rte_mbuf *pkts[], uint16_t nb_pkts, uint16_t port) { int32_t i; const int32_t prefetch_offset = 2; @@ -337,7 +334,7 @@ prepare_tx_burst(struct rte_mbuf *pkts[], uint16_t nb_pkts, uint8_t port) /* Send burst of packets on an output interface */ static inline int32_t -send_burst(struct lcore_conf *qconf, uint16_t n, uint8_t port) +send_burst(struct lcore_conf *qconf, uint16_t n, uint16_t port) { struct rte_mbuf **m_table; int32_t ret; @@ -360,7 +357,7 @@ send_burst(struct lcore_conf *qconf, uint16_t n, uint8_t port) /* Enqueue a single packet, and send burst if queue is filled */ static inline int32_t -send_single_packet(struct rte_mbuf *m, uint8_t port) +send_single_packet(struct rte_mbuf *m, uint16_t port) { uint32_t lcore_id; uint16_t len; @@ -410,7 +407,8 @@ inbound_sp_sa(struct sp_ctx *sp, struct sa_ctx *sa, struct traffic_type *ip, } /* Only check SPI match for processed IPSec packets */ sa_idx = ip->res[i] & PROTECT_MASK; - if (sa_idx == 0 || !inbound_sa_check(sa, m, sa_idx)) { + if (sa_idx >= IPSEC_SA_MAX_ENTRIES || + !inbound_sa_check(sa, m, sa_idx)) { rte_pktmbuf_free(m); continue; } @@ -475,9 +473,9 @@ outbound_sp(struct sp_ctx *sp, struct traffic_type *ip, for (i = 0; i < ip->num; i++) { m = ip->pkts[i]; sa_idx = ip->res[i] & PROTECT_MASK; - if ((ip->res[i] == 0) || (ip->res[i] & DISCARD)) + if (ip->res[i] & DISCARD) rte_pktmbuf_free(m); - else if (sa_idx != 0) { + else if (sa_idx < IPSEC_SA_MAX_ENTRIES) { ipsec->res[ipsec->num] = sa_idx; ipsec->pkts[ipsec->num++] = m; } else /* BYPASS */ @@ -588,66 +586,139 @@ process_pkts_outbound_nosp(struct ipsec_ctx *ipsec_ctx, traffic->ip6.num = nb_pkts_out; } +static inline int32_t +get_hop_for_offload_pkt(struct rte_mbuf *pkt, int is_ipv6) +{ + struct ipsec_mbuf_metadata *priv; + struct ipsec_sa *sa; + + priv = get_priv(pkt); + + sa = priv->sa; + if (unlikely(sa == NULL)) { + RTE_LOG(ERR, IPSEC, "SA not saved in private data\n"); + goto fail; + } + + if (is_ipv6) + return sa->portid; + + /* else */ + return (sa->portid | RTE_LPM_LOOKUP_SUCCESS); + +fail: + if (is_ipv6) + return -1; + + /* else */ + return 0; +} + static inline void route4_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts) { uint32_t hop[MAX_PKT_BURST * 2]; uint32_t dst_ip[MAX_PKT_BURST * 2]; + int32_t pkt_hop = 0; uint16_t i, offset; + uint16_t lpm_pkts = 0; if (nb_pkts == 0) return; + /* Need to do an LPM lookup for non-inline packets. Inline packets will + * have port ID in the SA + */ + for (i = 0; i < nb_pkts; i++) { - offset = offsetof(struct ip, ip_dst); - dst_ip[i] = *rte_pktmbuf_mtod_offset(pkts[i], - uint32_t *, offset); - dst_ip[i] = rte_be_to_cpu_32(dst_ip[i]); + if (!(pkts[i]->ol_flags & PKT_TX_SEC_OFFLOAD)) { + /* Security offload not enabled. So an LPM lookup is + * required to get the hop + */ + offset = offsetof(struct ip, ip_dst); + dst_ip[lpm_pkts] = *rte_pktmbuf_mtod_offset(pkts[i], + uint32_t *, offset); + dst_ip[lpm_pkts] = rte_be_to_cpu_32(dst_ip[lpm_pkts]); + lpm_pkts++; + } } - rte_lpm_lookup_bulk((struct rte_lpm *)rt_ctx, dst_ip, hop, nb_pkts); + rte_lpm_lookup_bulk((struct rte_lpm *)rt_ctx, dst_ip, hop, lpm_pkts); + + lpm_pkts = 0; for (i = 0; i < nb_pkts; i++) { - if ((hop[i] & RTE_LPM_LOOKUP_SUCCESS) == 0) { + if (pkts[i]->ol_flags & PKT_TX_SEC_OFFLOAD) { + /* Read hop from the SA */ + pkt_hop = get_hop_for_offload_pkt(pkts[i], 0); + } else { + /* Need to use hop returned by lookup */ + pkt_hop = hop[lpm_pkts++]; + } + + if ((pkt_hop & RTE_LPM_LOOKUP_SUCCESS) == 0) { rte_pktmbuf_free(pkts[i]); continue; } - send_single_packet(pkts[i], hop[i] & 0xff); + send_single_packet(pkts[i], pkt_hop & 0xff); } } static inline void route6_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts) { - int16_t hop[MAX_PKT_BURST * 2]; + int32_t hop[MAX_PKT_BURST * 2]; uint8_t dst_ip[MAX_PKT_BURST * 2][16]; uint8_t *ip6_dst; + int32_t pkt_hop = 0; uint16_t i, offset; + uint16_t lpm_pkts = 0; if (nb_pkts == 0) return; + /* Need to do an LPM lookup for non-inline packets. Inline packets will + * have port ID in the SA + */ + for (i = 0; i < nb_pkts; i++) { - offset = offsetof(struct ip6_hdr, ip6_dst); - ip6_dst = rte_pktmbuf_mtod_offset(pkts[i], uint8_t *, offset); - memcpy(&dst_ip[i][0], ip6_dst, 16); + if (!(pkts[i]->ol_flags & PKT_TX_SEC_OFFLOAD)) { + /* Security offload not enabled. So an LPM lookup is + * required to get the hop + */ + offset = offsetof(struct ip6_hdr, ip6_dst); + ip6_dst = rte_pktmbuf_mtod_offset(pkts[i], uint8_t *, + offset); + memcpy(&dst_ip[lpm_pkts][0], ip6_dst, 16); + lpm_pkts++; + } } - rte_lpm6_lookup_bulk_func((struct rte_lpm6 *)rt_ctx, dst_ip, - hop, nb_pkts); + rte_lpm6_lookup_bulk_func((struct rte_lpm6 *)rt_ctx, dst_ip, hop, + lpm_pkts); + + lpm_pkts = 0; for (i = 0; i < nb_pkts; i++) { - if (hop[i] == -1) { + if (pkts[i]->ol_flags & PKT_TX_SEC_OFFLOAD) { + /* Read hop from the SA */ + pkt_hop = get_hop_for_offload_pkt(pkts[i], 1); + } else { + /* Need to use hop returned by lookup */ + pkt_hop = hop[lpm_pkts++]; + } + + if (pkt_hop == -1) { rte_pktmbuf_free(pkts[i]); continue; } - send_single_packet(pkts[i], hop[i] & 0xff); + send_single_packet(pkts[i], pkt_hop & 0xff); } } static inline void process_pkts(struct lcore_conf *qconf, struct rte_mbuf **pkts, - uint8_t nb_pkts, uint8_t portid) + uint8_t nb_pkts, uint16_t portid) { struct ipsec_traffic traffic; @@ -692,7 +763,8 @@ main_loop(__attribute__((unused)) void *dummy) uint32_t lcore_id; uint64_t prev_tsc, diff_tsc, cur_tsc; int32_t i, nb_rx; - uint8_t portid, queueid; + uint16_t portid; + uint8_t queueid; struct lcore_conf *qconf; int32_t socket_id; const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) @@ -711,10 +783,12 @@ main_loop(__attribute__((unused)) void *dummy) qconf->inbound.sp6_ctx = socket_ctx[socket_id].sp_ip6_in; qconf->inbound.sa_ctx = socket_ctx[socket_id].sa_in; qconf->inbound.cdev_map = cdev_map_in; + qconf->inbound.session_pool = socket_ctx[socket_id].session_pool; qconf->outbound.sp4_ctx = socket_ctx[socket_id].sp_ip4_out; qconf->outbound.sp6_ctx = socket_ctx[socket_id].sp_ip6_out; qconf->outbound.sa_ctx = socket_ctx[socket_id].sa_out; qconf->outbound.cdev_map = cdev_map_out; + qconf->outbound.session_pool = socket_ctx[socket_id].session_pool; if (qconf->nb_rx_queue == 0) { RTE_LOG(INFO, IPSEC, "lcore %u has nothing to do\n", lcore_id); @@ -727,7 +801,7 @@ main_loop(__attribute__((unused)) void *dummy) portid = rxql[i].port_id; queueid = rxql[i].queue_id; RTE_LOG(INFO, IPSEC, - " -- lcoreid=%u portid=%hhu rxqueueid=%hhu\n", + " -- lcoreid=%u portid=%u rxqueueid=%hhu\n", lcore_id, portid, queueid); } @@ -758,7 +832,8 @@ main_loop(__attribute__((unused)) void *dummy) static int32_t check_params(void) { - uint8_t lcore, portid, nb_ports; + uint8_t lcore; + uint16_t portid, nb_ports; uint16_t i; int32_t socket_id; @@ -796,7 +871,7 @@ check_params(void) } static uint8_t -get_port_nb_rx_queues(const uint8_t port) +get_port_nb_rx_queues(const uint16_t port) { int32_t queue = -1; uint16_t i; @@ -838,16 +913,17 @@ print_usage(const char *prgname) { printf("%s [EAL options] -- -p PORTMASK -P -u PORTMASK" " --"OPTION_CONFIG" (port,queue,lcore)[,(port,queue,lcore]" - " --single-sa SAIDX --ep0|--ep1\n" + " --single-sa SAIDX -f CONFIG_FILE\n" " -p PORTMASK: hexadecimal bitmask of ports to configure\n" " -P : enable promiscuous mode\n" " -u PORTMASK: hexadecimal bitmask of unprotected ports\n" + " -j FRAMESIZE: jumbo frame maximum size\n" " --"OPTION_CONFIG": (port,queue,lcore): " "rx queues configuration\n" " --single-sa SAIDX: use single SA index for outbound, " "bypassing the SP\n" - " --ep0: Configure as Endpoint 0\n" - " --ep1: Configure as Endpoint 1\n", prgname); + " -f CONFIG_FILE: Configuration file path\n", + prgname); } static int32_t @@ -960,18 +1036,6 @@ parse_args_long_options(struct option *lgopts, int32_t option_index) } } - if (__STRNCMP(optname, OPTION_EP0)) { - printf("endpoint 0\n"); - ep = 0; - ret = 0; - } - - if (__STRNCMP(optname, OPTION_EP1)) { - printf("endpoint 1\n"); - ep = 1; - ret = 0; - } - return ret; } #undef __STRNCMP @@ -986,14 +1050,13 @@ parse_args(int32_t argc, char **argv) static struct option lgopts[] = { {OPTION_CONFIG, 1, 0, 0}, {OPTION_SINGLE_SA, 1, 0, 0}, - {OPTION_EP0, 0, 0, 0}, - {OPTION_EP1, 0, 0, 0}, {NULL, 0, 0, 0} }; + int32_t f_present = 0; argvopt = argv; - while ((opt = getopt_long(argc, argvopt, "p:Pu:", + while ((opt = getopt_long(argc, argvopt, "p:Pu:f:j:", lgopts, &option_index)) != EOF) { switch (opt) { @@ -1017,6 +1080,38 @@ parse_args(int32_t argc, char **argv) return -1; } break; + case 'f': + if (f_present == 1) { + printf("\"-f\" option present more than " + "once!\n"); + print_usage(prgname); + return -1; + } + if (parse_cfg_file(optarg) < 0) { + printf("parsing file \"%s\" failed\n", + optarg); + print_usage(prgname); + return -1; + } + f_present = 1; + break; + case 'j': + { + int32_t size = parse_decimal(optarg); + if (size <= 1518) { + printf("Invalid jumbo frame size\n"); + if (size < 0) { + print_usage(prgname); + return -1; + } + printf("Using default value 9000\n"); + frame_size = 9000; + } else { + frame_size = size; + } + } + printf("Enabled jumbo frames size %u\n", frame_size); + break; case 0: if (parse_args_long_options(lgopts, option_index)) { print_usage(prgname); @@ -1029,11 +1124,16 @@ parse_args(int32_t argc, char **argv) } } + if (f_present == 0) { + printf("Mandatory option \"-f\" not present\n"); + return -1; + } + if (optind >= 0) argv[optind-1] = prgname; ret = optind-1; - optind = 0; /* reset getopt lib */ + optind = 1; /* reset getopt lib */ return ret; } @@ -1047,11 +1147,12 @@ print_ethaddr(const char *name, const struct ether_addr *eth_addr) /* Check the link status of all ports in up to 9s, and print them finally */ static void -check_all_ports_link_status(uint8_t port_num, uint32_t port_mask) +check_all_ports_link_status(uint16_t port_num, uint32_t port_mask) { #define CHECK_INTERVAL 100 /* 100ms */ #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ - uint8_t portid, count, all_ports_up, print_flag = 0; + uint16_t portid; + uint8_t count, all_ports_up, print_flag = 0; struct rte_eth_link link; printf("\nChecking link status"); @@ -1066,14 +1167,13 @@ check_all_ports_link_status(uint8_t port_num, uint32_t port_mask) /* print link status if flag set */ if (print_flag == 1) { if (link.link_status) - printf("Port %d Link Up - speed %u " - "Mbps - %s\n", (uint8_t)portid, - (uint32_t)link.link_speed, + printf( + "Port%d Link Up - speed %u Mbps -%s\n", + portid, link.link_speed, (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? ("full-duplex") : ("half-duplex\n")); else - printf("Port %d Link Down\n", - (uint8_t)portid); + printf("Port %d Link Down\n", portid); continue; } /* clear all_ports_up flag if any link down */ @@ -1105,7 +1205,8 @@ add_mapping(struct rte_hash *map, const char *str, uint16_t cdev_id, uint16_t qp, struct lcore_params *params, struct ipsec_ctx *ipsec_ctx, const struct rte_cryptodev_capabilities *cipher, - const struct rte_cryptodev_capabilities *auth) + const struct rte_cryptodev_capabilities *auth, + const struct rte_cryptodev_capabilities *aead) { int32_t ret = 0; unsigned long i; @@ -1116,6 +1217,8 @@ add_mapping(struct rte_hash *map, const char *str, uint16_t cdev_id, key.cipher_algo = cipher->sym.cipher.algo; if (auth) key.auth_algo = auth->sym.auth.algo; + if (aead) + key.aead_algo = aead->sym.aead.algo; ret = rte_hash_lookup(map, &key); if (ret != -ENOENT) @@ -1184,6 +1287,12 @@ add_cdev_mapping(struct rte_cryptodev_info *dev_info, uint16_t cdev_id, if (i->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC) continue; + if (i->sym.xform_type == RTE_CRYPTO_SYM_XFORM_AEAD) { + ret |= add_mapping(map, str, cdev_id, qp, params, + ipsec_ctx, NULL, NULL, i); + continue; + } + if (i->sym.xform_type != RTE_CRYPTO_SYM_XFORM_CIPHER) continue; @@ -1196,7 +1305,7 @@ add_cdev_mapping(struct rte_cryptodev_info *dev_info, uint16_t cdev_id, continue; ret |= add_mapping(map, str, cdev_id, qp, params, - ipsec_ctx, i, j); + ipsec_ctx, i, j, NULL); } } @@ -1232,6 +1341,13 @@ cryptodevs_init(void) printf("lcore/cryptodev/qp mappings:\n"); + uint32_t max_sess_sz = 0, sess_sz; + for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) { + sess_sz = rte_cryptodev_get_private_session_size(cdev_id); + if (sess_sz > max_sess_sz) + max_sess_sz = sess_sz; + } + idx = 0; /* Start from last cdev id to give HW priority */ for (cdev_id = rte_cryptodev_count() - 1; cdev_id >= 0; cdev_id--) { @@ -1260,17 +1376,39 @@ cryptodevs_init(void) dev_conf.socket_id = rte_cryptodev_socket_id(cdev_id); dev_conf.nb_queue_pairs = qp; - dev_conf.session_mp.nb_objs = CDEV_MP_NB_OBJS; - dev_conf.session_mp.cache_size = CDEV_MP_CACHE_SZ; + + if (!socket_ctx[dev_conf.socket_id].session_pool) { + char mp_name[RTE_MEMPOOL_NAMESIZE]; + struct rte_mempool *sess_mp; + + snprintf(mp_name, RTE_MEMPOOL_NAMESIZE, + "sess_mp_%u", dev_conf.socket_id); + sess_mp = rte_mempool_create(mp_name, + CDEV_MP_NB_OBJS, + max_sess_sz, + CDEV_MP_CACHE_SZ, + 0, NULL, NULL, NULL, + NULL, dev_conf.socket_id, + 0); + if (sess_mp == NULL) + rte_exit(EXIT_FAILURE, + "Cannot create session pool on socket %d\n", + dev_conf.socket_id); + else + printf("Allocated session pool on socket %d\n", + dev_conf.socket_id); + socket_ctx[dev_conf.socket_id].session_pool = sess_mp; + } if (rte_cryptodev_configure(cdev_id, &dev_conf)) - rte_panic("Failed to initialize crypodev %u\n", + rte_panic("Failed to initialize cryptodev %u\n", cdev_id); - qp_conf.nb_descriptors = CDEV_MP_NB_OBJS; + qp_conf.nb_descriptors = CDEV_QUEUE_DESC; for (qp = 0; qp < dev_conf.nb_queue_pairs; qp++) if (rte_cryptodev_queue_pair_setup(cdev_id, qp, - &qp_conf, dev_conf.socket_id)) + &qp_conf, dev_conf.socket_id, + socket_ctx[dev_conf.socket_id].session_pool)) rte_panic("Failed to setup queue %u for " "cdev_id %u\n", 0, cdev_id); @@ -1285,7 +1423,7 @@ cryptodevs_init(void) } static void -port_init(uint8_t portid) +port_init(uint16_t portid) { struct rte_eth_dev_info dev_info; struct rte_eth_txconf *txconf; @@ -1320,12 +1458,27 @@ port_init(uint8_t portid) printf("Creating queues: nb_rx_queue=%d nb_tx_queue=%u...\n", nb_rx_queue, nb_tx_queue); + if (frame_size) { + port_conf.rxmode.max_rx_pkt_len = frame_size; + port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME; + } + + if (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_SECURITY) + port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_SECURITY; + if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_SECURITY) + port_conf.txmode.offloads |= DEV_TX_OFFLOAD_SECURITY; + ret = rte_eth_dev_configure(portid, nb_rx_queue, nb_tx_queue, &port_conf); if (ret < 0) rte_exit(EXIT_FAILURE, "Cannot configure device: " "err=%d, port=%d\n", ret, portid); + ret = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd, &nb_txd); + if (ret < 0) + rte_exit(EXIT_FAILURE, "Cannot adjust number of descriptors: " + "err=%d, port=%d\n", ret, portid); + /* init one TX queue per lcore */ tx_queueid = 0; for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { @@ -1379,11 +1532,14 @@ static void pool_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t nb_mbuf) { char s[64]; + uint32_t buff_size = frame_size ? (frame_size + RTE_PKTMBUF_HEADROOM) : + RTE_MBUF_DEFAULT_BUF_SIZE; + snprintf(s, sizeof(s), "mbuf_pool_%d", socket_id); ctx->mbuf_pool = rte_pktmbuf_pool_create(s, nb_mbuf, MEMPOOL_CACHE_SIZE, ipsec_metadata_size(), - RTE_MBUF_DEFAULT_BUF_SIZE, + buff_size, socket_id); if (ctx->mbuf_pool == NULL) rte_exit(EXIT_FAILURE, "Cannot init mbuf pool on socket %d\n", @@ -1396,8 +1552,9 @@ int32_t main(int32_t argc, char **argv) { int32_t ret; - uint32_t lcore_id, nb_ports; - uint8_t portid, socket_id; + uint32_t lcore_id; + uint8_t socket_id; + uint16_t portid, nb_ports; /* init EAL */ ret = rte_eal_init(argc, argv); @@ -1411,9 +1568,6 @@ main(int32_t argc, char **argv) if (ret < 0) rte_exit(EXIT_FAILURE, "Invalid parameters\n"); - if (ep < 0) - rte_exit(EXIT_FAILURE, "need to choose either EP0 or EP1\n"); - if ((unprotected_port_mask & enabled_port_mask) != unprotected_port_mask) rte_exit(EXIT_FAILURE, "Invalid unprotected portmask 0x%x\n", @@ -1430,7 +1584,7 @@ main(int32_t argc, char **argv) nb_lcores = rte_lcore_count(); - /* Replicate each contex per socket */ + /* Replicate each context per socket */ for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { if (rte_lcore_is_enabled(lcore_id) == 0) continue; @@ -1443,13 +1597,13 @@ main(int32_t argc, char **argv) if (socket_ctx[socket_id].mbuf_pool) continue; - sa_init(&socket_ctx[socket_id], socket_id, ep); + sa_init(&socket_ctx[socket_id], socket_id); - sp4_init(&socket_ctx[socket_id], socket_id, ep); + sp4_init(&socket_ctx[socket_id], socket_id); - sp6_init(&socket_ctx[socket_id], socket_id, ep); + sp6_init(&socket_ctx[socket_id], socket_id); - rt_init(&socket_ctx[socket_id], socket_id, ep); + rt_init(&socket_ctx[socket_id], socket_id); pool_init(&socket_ctx[socket_id], socket_id, NB_MBUF); } @@ -1483,7 +1637,7 @@ main(int32_t argc, char **argv) rte_eth_promiscuous_enable(portid); } - check_all_ports_link_status((uint8_t)nb_ports, enabled_port_mask); + check_all_ports_link_status(nb_ports, enabled_port_mask); /* launch per-lcore init on every lcore */ rte_eal_mp_remote_launch(main_loop, NULL, CALL_MASTER); diff --git a/dpdk/examples/ipsec-secgw/ipsec.c b/dpdk/examples/ipsec-secgw/ipsec.c index 1e87d0df..c850c7b8 100644 --- a/dpdk/examples/ipsec-secgw/ipsec.c +++ b/dpdk/examples/ipsec-secgw/ipsec.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -37,7 +37,9 @@ #include #include #include +#include #include +#include #include #include @@ -45,34 +47,190 @@ #include "esp.h" static inline int -create_session(struct ipsec_ctx *ipsec_ctx __rte_unused, struct ipsec_sa *sa) +create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa) { + struct rte_cryptodev_info cdev_info; unsigned long cdev_id_qp = 0; - int32_t ret; + int32_t ret = 0; struct cdev_key key = { 0 }; key.lcore_id = (uint8_t)rte_lcore_id(); key.cipher_algo = (uint8_t)sa->cipher_algo; key.auth_algo = (uint8_t)sa->auth_algo; + key.aead_algo = (uint8_t)sa->aead_algo; - ret = rte_hash_lookup_data(ipsec_ctx->cdev_map, &key, - (void **)&cdev_id_qp); - if (ret < 0) { - RTE_LOG(ERR, IPSEC, "No cryptodev: core %u, cipher_algo %u, " - "auth_algo %u\n", key.lcore_id, key.cipher_algo, - key.auth_algo); - return -1; + if (sa->type == RTE_SECURITY_ACTION_TYPE_NONE) { + ret = rte_hash_lookup_data(ipsec_ctx->cdev_map, &key, + (void **)&cdev_id_qp); + if (ret < 0) { + RTE_LOG(ERR, IPSEC, + "No cryptodev: core %u, cipher_algo %u, " + "auth_algo %u, aead_algo %u\n", + key.lcore_id, + key.cipher_algo, + key.auth_algo, + key.aead_algo); + return -1; + } } - RTE_LOG(DEBUG, IPSEC, "Create session for SA spi %u on cryptodev " + RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on cryptodev " "%u qp %u\n", sa->spi, ipsec_ctx->tbl[cdev_id_qp].id, ipsec_ctx->tbl[cdev_id_qp].qp); - sa->crypto_session = rte_cryptodev_sym_session_create( - ipsec_ctx->tbl[cdev_id_qp].id, sa->xforms); + if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE) { + struct rte_security_session_conf sess_conf = { + .action_type = sa->type, + .protocol = RTE_SECURITY_PROTOCOL_IPSEC, + {.ipsec = { + .spi = sa->spi, + .salt = sa->salt, + .options = { 0 }, + .direction = sa->direction, + .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP, + .mode = (sa->flags == IP4_TUNNEL || + sa->flags == IP6_TUNNEL) ? + RTE_SECURITY_IPSEC_SA_MODE_TUNNEL : + RTE_SECURITY_IPSEC_SA_MODE_TRANSPORT, + } }, + .crypto_xform = sa->xforms + }; + + if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) { + struct rte_security_ctx *ctx = (struct rte_security_ctx *) + rte_cryptodev_get_sec_ctx( + ipsec_ctx->tbl[cdev_id_qp].id); + + if (sess_conf.ipsec.mode == + RTE_SECURITY_IPSEC_SA_MODE_TUNNEL) { + struct rte_security_ipsec_tunnel_param *tunnel = + &sess_conf.ipsec.tunnel; + if (sa->flags == IP4_TUNNEL) { + tunnel->type = + RTE_SECURITY_IPSEC_TUNNEL_IPV4; + tunnel->ipv4.ttl = IPDEFTTL; + + memcpy((uint8_t *)&tunnel->ipv4.src_ip, + (uint8_t *)&sa->src.ip.ip4, 4); + + memcpy((uint8_t *)&tunnel->ipv4.dst_ip, + (uint8_t *)&sa->dst.ip.ip4, 4); + } + /* TODO support for Transport and IPV6 tunnel */ + } + + sa->sec_session = rte_security_session_create(ctx, + &sess_conf, ipsec_ctx->session_pool); + if (sa->sec_session == NULL) { + RTE_LOG(ERR, IPSEC, + "SEC Session init failed: err: %d\n", ret); + return -1; + } + } else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) { + struct rte_flow_error err; + struct rte_security_ctx *ctx = (struct rte_security_ctx *) + rte_eth_dev_get_sec_ctx( + sa->portid); + const struct rte_security_capability *sec_cap; + + sa->sec_session = rte_security_session_create(ctx, + &sess_conf, ipsec_ctx->session_pool); + if (sa->sec_session == NULL) { + RTE_LOG(ERR, IPSEC, + "SEC Session init failed: err: %d\n", ret); + return -1; + } + + sec_cap = rte_security_capabilities_get(ctx); + + /* iterate until ESP tunnel*/ + while (sec_cap->action != + RTE_SECURITY_ACTION_TYPE_NONE) { + + if (sec_cap->action == sa->type && + sec_cap->protocol == + RTE_SECURITY_PROTOCOL_IPSEC && + sec_cap->ipsec.mode == + RTE_SECURITY_IPSEC_SA_MODE_TUNNEL && + sec_cap->ipsec.direction == sa->direction) + break; + sec_cap++; + } + + if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) { + RTE_LOG(ERR, IPSEC, + "No suitable security capability found\n"); + return -1; + } + + sa->ol_flags = sec_cap->ol_flags; + sa->security_ctx = ctx; + sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH; + + sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4; + sa->pattern[1].mask = &rte_flow_item_ipv4_mask; + if (sa->flags & IP6_TUNNEL) { + sa->pattern[1].spec = &sa->ipv6_spec; + memcpy(sa->ipv6_spec.hdr.dst_addr, + sa->dst.ip.ip6.ip6_b, 16); + memcpy(sa->ipv6_spec.hdr.src_addr, + sa->src.ip.ip6.ip6_b, 16); + } else { + sa->pattern[1].spec = &sa->ipv4_spec; + sa->ipv4_spec.hdr.dst_addr = sa->dst.ip.ip4; + sa->ipv4_spec.hdr.src_addr = sa->src.ip.ip4; + } + + sa->pattern[2].type = RTE_FLOW_ITEM_TYPE_ESP; + sa->pattern[2].spec = &sa->esp_spec; + sa->pattern[2].mask = &rte_flow_item_esp_mask; + sa->esp_spec.hdr.spi = rte_cpu_to_be_32(sa->spi); + + sa->pattern[3].type = RTE_FLOW_ITEM_TYPE_END; + + sa->action[0].type = RTE_FLOW_ACTION_TYPE_SECURITY; + sa->action[0].conf = sa->sec_session; + + sa->action[1].type = RTE_FLOW_ACTION_TYPE_END; + + sa->attr.egress = (sa->direction == + RTE_SECURITY_IPSEC_SA_DIR_EGRESS); + sa->attr.ingress = (sa->direction == + RTE_SECURITY_IPSEC_SA_DIR_INGRESS); + sa->flow = rte_flow_create(sa->portid, + &sa->attr, sa->pattern, sa->action, &err); + if (sa->flow == NULL) { + RTE_LOG(ERR, IPSEC, + "Failed to create ipsec flow msg: %s\n", + err.message); + return -1; + } + } + } else { + sa->crypto_session = rte_cryptodev_sym_session_create( + ipsec_ctx->session_pool); + rte_cryptodev_sym_session_init(ipsec_ctx->tbl[cdev_id_qp].id, + sa->crypto_session, sa->xforms, + ipsec_ctx->session_pool); + + rte_cryptodev_info_get(ipsec_ctx->tbl[cdev_id_qp].id, + &cdev_info); + if (cdev_info.sym.max_nb_sessions_per_qp > 0) { + ret = rte_cryptodev_queue_pair_attach_sym_session( + ipsec_ctx->tbl[cdev_id_qp].id, + ipsec_ctx->tbl[cdev_id_qp].qp, + sa->crypto_session); + if (ret < 0) { + RTE_LOG(ERR, IPSEC, + "Session cannot be attached to qp %u\n", + ipsec_ctx->tbl[cdev_id_qp].qp); + return -1; + } + } + } sa->cdev_id_qp = cdev_id_qp; return 0; @@ -89,7 +247,7 @@ enqueue_cop(struct cdev_qp *cqp, struct rte_crypto_op *cop) ret = rte_cryptodev_enqueue_burst(cqp->id, cqp->qp, cqp->buf, cqp->len); if (ret < cqp->len) { - RTE_LOG(DEBUG, IPSEC, "Cryptodev %u queue %u:" + RTE_LOG_DP(DEBUG, IPSEC, "Cryptodev %u queue %u:" " enqueued %u crypto ops out of %u\n", cqp->id, cqp->qp, ret, cqp->len); @@ -108,7 +266,9 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx, { int32_t ret = 0, i; struct ipsec_mbuf_metadata *priv; + struct rte_crypto_sym_op *sym_cop; struct ipsec_sa *sa; + struct cdev_qp *cqp; for (i = 0; i < nb_pkts; i++) { if (unlikely(sas[i] == NULL)) { @@ -123,23 +283,76 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx, sa = sas[i]; priv->sa = sa; - priv->cop.type = RTE_CRYPTO_OP_TYPE_SYMMETRIC; + switch (sa->type) { + case RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL: + priv->cop.type = RTE_CRYPTO_OP_TYPE_SYMMETRIC; + priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; - rte_prefetch0(&priv->sym_cop); - priv->cop.sym = &priv->sym_cop; + rte_prefetch0(&priv->sym_cop); - if ((unlikely(sa->crypto_session == NULL)) && - create_session(ipsec_ctx, sa)) { - rte_pktmbuf_free(pkts[i]); - continue; - } + if ((unlikely(sa->sec_session == NULL)) && + create_session(ipsec_ctx, sa)) { + rte_pktmbuf_free(pkts[i]); + continue; + } - rte_crypto_op_attach_sym_session(&priv->cop, - sa->crypto_session); + sym_cop = get_sym_cop(&priv->cop); + sym_cop->m_src = pkts[i]; - ret = xform_func(pkts[i], sa, &priv->cop); - if (unlikely(ret)) { - rte_pktmbuf_free(pkts[i]); + rte_security_attach_session(&priv->cop, + sa->sec_session); + break; + case RTE_SECURITY_ACTION_TYPE_NONE: + + priv->cop.type = RTE_CRYPTO_OP_TYPE_SYMMETRIC; + priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; + + rte_prefetch0(&priv->sym_cop); + + if ((unlikely(sa->crypto_session == NULL)) && + create_session(ipsec_ctx, sa)) { + rte_pktmbuf_free(pkts[i]); + continue; + } + + rte_crypto_op_attach_sym_session(&priv->cop, + sa->crypto_session); + + ret = xform_func(pkts[i], sa, &priv->cop); + if (unlikely(ret)) { + rte_pktmbuf_free(pkts[i]); + continue; + } + break; + case RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL: + break; + case RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO: + priv->cop.type = RTE_CRYPTO_OP_TYPE_SYMMETRIC; + priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; + + rte_prefetch0(&priv->sym_cop); + + if ((unlikely(sa->sec_session == NULL)) && + create_session(ipsec_ctx, sa)) { + rte_pktmbuf_free(pkts[i]); + continue; + } + + rte_security_attach_session(&priv->cop, + sa->sec_session); + + ret = xform_func(pkts[i], sa, &priv->cop); + if (unlikely(ret)) { + rte_pktmbuf_free(pkts[i]); + continue; + } + + cqp = &ipsec_ctx->tbl[sa->cdev_id_qp]; + cqp->ol_pkts[cqp->ol_pkts_cnt++] = pkts[i]; + if (sa->ol_flags & RTE_SECURITY_TX_OLOAD_NEED_MDATA) + rte_security_set_pkt_metadata( + sa->security_ctx, + sa->sec_session, pkts[i], NULL); continue; } @@ -150,7 +363,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx, static inline int ipsec_dequeue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx, - struct rte_mbuf *pkts[], uint16_t max_pkts) + struct rte_mbuf *pkts[], uint16_t max_pkts) { int32_t nb_pkts = 0, ret = 0, i, j, nb_cops; struct ipsec_mbuf_metadata *priv; @@ -165,6 +378,19 @@ ipsec_dequeue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx, if (ipsec_ctx->last_qp == ipsec_ctx->nb_qps) ipsec_ctx->last_qp %= ipsec_ctx->nb_qps; + while (cqp->ol_pkts_cnt > 0 && nb_pkts < max_pkts) { + pkt = cqp->ol_pkts[--cqp->ol_pkts_cnt]; + rte_prefetch0(pkt); + priv = get_priv(pkt); + sa = priv->sa; + ret = xform_func(pkt, sa, &priv->cop); + if (unlikely(ret)) { + rte_pktmbuf_free(pkt); + continue; + } + pkts[nb_pkts++] = pkt; + } + if (cqp->in_flight == 0) continue; @@ -182,11 +408,14 @@ ipsec_dequeue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx, RTE_ASSERT(sa != NULL); - ret = xform_func(pkt, sa, cops[j]); - if (unlikely(ret)) - rte_pktmbuf_free(pkt); - else - pkts[nb_pkts++] = pkt; + if (sa->type == RTE_SECURITY_ACTION_TYPE_NONE) { + ret = xform_func(pkt, sa, cops[j]); + if (unlikely(ret)) { + rte_pktmbuf_free(pkt); + continue; + } + } + pkts[nb_pkts++] = pkt; } } diff --git a/dpdk/examples/ipsec-secgw/ipsec.h b/dpdk/examples/ipsec-secgw/ipsec.h index a442a74a..775b316f 100644 --- a/dpdk/examples/ipsec-secgw/ipsec.h +++ b/dpdk/examples/ipsec-secgw/ipsec.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -38,6 +38,8 @@ #include #include +#include +#include #define RTE_LOGTYPE_IPSEC RTE_LOGTYPE_USER1 #define RTE_LOGTYPE_IPSEC_ESP RTE_LOGTYPE_USER2 @@ -48,6 +50,9 @@ #define MAX_DIGEST_SIZE 32 /* Bytes -- 256 bits */ +#define IV_OFFSET (sizeof(struct rte_crypto_op) + \ + sizeof(struct rte_crypto_sym_op)) + #define uint32_t_to_char(ip, a, b, c, d) do {\ *a = (uint8_t)(ip >> 24 & 0xff);\ *b = (uint8_t)(ip >> 16 & 0xff);\ @@ -72,7 +77,6 @@ struct rte_crypto_xform; struct ipsec_xform; -struct rte_cryptodev_session; struct rte_mbuf; struct ipsec_sa; @@ -90,13 +94,20 @@ struct ip_addr { } ip; }; +#define MAX_KEY_SIZE 32 + struct ipsec_sa { uint32_t spi; uint32_t cdev_id_qp; - struct rte_cryptodev_sym_session *crypto_session; - uint32_t seq; + uint64_t seq; + uint32_t salt; + union { + struct rte_cryptodev_sym_session *crypto_session; + struct rte_security_session *sec_session; + }; enum rte_crypto_cipher_algorithm cipher_algo; enum rte_crypto_auth_algorithm auth_algo; + enum rte_crypto_aead_algorithm aead_algo; uint16_t digest_len; uint16_t iv_len; uint16_t block_size; @@ -106,14 +117,41 @@ struct ipsec_sa { #define TRANSPORT (1 << 2) struct ip_addr src; struct ip_addr dst; - struct rte_crypto_sym_xform *xforms; + uint8_t cipher_key[MAX_KEY_SIZE]; + uint16_t cipher_key_len; + uint8_t auth_key[MAX_KEY_SIZE]; + uint16_t auth_key_len; + uint16_t aad_len; + union { + struct rte_crypto_sym_xform *xforms; + struct rte_security_ipsec_xform *sec_xform; + }; + enum rte_security_session_action_type type; + enum rte_security_ipsec_sa_direction direction; + uint16_t portid; + struct rte_security_ctx *security_ctx; + uint32_t ol_flags; + +#define MAX_RTE_FLOW_PATTERN (4) +#define MAX_RTE_FLOW_ACTIONS (2) + struct rte_flow_item pattern[MAX_RTE_FLOW_PATTERN]; + struct rte_flow_action action[MAX_RTE_FLOW_ACTIONS]; + struct rte_flow_attr attr; + union { + struct rte_flow_item_ipv4 ipv4_spec; + struct rte_flow_item_ipv6 ipv6_spec; + }; + struct rte_flow_item_esp esp_spec; + struct rte_flow *flow; + struct rte_security_session_conf sess_conf; } __rte_cache_aligned; struct ipsec_mbuf_metadata { struct ipsec_sa *sa; struct rte_crypto_op cop; struct rte_crypto_sym_op sym_cop; -}; + uint8_t buf[32]; +} __rte_cache_aligned; struct cdev_qp { uint16_t id; @@ -121,6 +159,8 @@ struct cdev_qp { uint16_t in_flight; uint16_t len; struct rte_crypto_op *buf[MAX_PKT_BURST] __rte_aligned(sizeof(void *)); + struct rte_mbuf *ol_pkts[MAX_PKT_BURST] __rte_aligned(sizeof(void *)); + uint16_t ol_pkts_cnt; }; struct ipsec_ctx { @@ -131,12 +171,14 @@ struct ipsec_ctx { uint16_t nb_qps; uint16_t last_qp; struct cdev_qp tbl[MAX_QP_PER_LCORE]; + struct rte_mempool *session_pool; }; struct cdev_key { uint16_t lcore_id; uint8_t cipher_algo; uint8_t auth_algo; + uint8_t aead_algo; }; struct socket_ctx { @@ -149,8 +191,15 @@ struct socket_ctx { struct rt_ctx *rt_ip4; struct rt_ctx *rt_ip6; struct rte_mempool *mbuf_pool; + struct rte_mempool *session_pool; }; +struct cnt_blk { + uint32_t salt; + uint64_t iv; + uint32_t cnt; +} __attribute__((packed)); + uint16_t ipsec_inbound(struct ipsec_ctx *ctx, struct rte_mbuf *pkts[], uint16_t nb_pkts, uint16_t len); @@ -171,6 +220,28 @@ get_priv(struct rte_mbuf *m) return RTE_PTR_ADD(m, sizeof(struct rte_mbuf)); } +static inline void * +get_cnt_blk(struct rte_mbuf *m) +{ + struct ipsec_mbuf_metadata *priv = get_priv(m); + + return &priv->buf[0]; +} + +static inline void * +get_aad(struct rte_mbuf *m) +{ + struct ipsec_mbuf_metadata *priv = get_priv(m); + + return &priv->buf[16]; +} + +static inline void * +get_sym_cop(struct rte_crypto_op *cop) +{ + return (cop + 1); +} + int inbound_sa_check(struct sa_ctx *sa_ctx, struct rte_mbuf *m, uint32_t sa_idx); @@ -183,15 +254,15 @@ outbound_sa_lookup(struct sa_ctx *sa_ctx, uint32_t sa_idx[], struct ipsec_sa *sa[], uint16_t nb_pkts); void -sp4_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t ep); +sp4_init(struct socket_ctx *ctx, int32_t socket_id); void -sp6_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t ep); +sp6_init(struct socket_ctx *ctx, int32_t socket_id); void -sa_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t ep); +sa_init(struct socket_ctx *ctx, int32_t socket_id); void -rt_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t ep); +rt_init(struct socket_ctx *ctx, int32_t socket_id); #endif /* __IPSEC_H__ */ diff --git a/dpdk/examples/ipsec-secgw/parser.c b/dpdk/examples/ipsec-secgw/parser.c new file mode 100644 index 00000000..9d0ea462 --- /dev/null +++ b/dpdk/examples/ipsec-secgw/parser.c @@ -0,0 +1,591 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include +#include + +#include +#include +#include +#include +#include + +#include "ipsec.h" +#include "parser.h" + +#define PARSE_DELIMITER " \f\n\r\t\v" +static int +parse_tokenize_string(char *string, char *tokens[], uint32_t *n_tokens) +{ + uint32_t i; + + if ((string == NULL) || + (tokens == NULL) || + (*n_tokens < 1)) + return -EINVAL; + + for (i = 0; i < *n_tokens; i++) { + tokens[i] = strtok_r(string, PARSE_DELIMITER, &string); + if (tokens[i] == NULL) + break; + } + + if ((i == *n_tokens) && + (NULL != strtok_r(string, PARSE_DELIMITER, &string))) + return -E2BIG; + + *n_tokens = i; + return 0; +} + +#define INADDRSZ 4 +#define IN6ADDRSZ 16 + +/* int + * inet_pton4(src, dst) + * like inet_aton() but without all the hexadecimal and shorthand. + * return: + * 1 if `src' is a valid dotted quad, else 0. + * notice: + * does not touch `dst' unless it's returning 1. + * author: + * Paul Vixie, 1996. + */ +static int +inet_pton4(const char *src, unsigned char *dst) +{ + static const char digits[] = "0123456789"; + int saw_digit, octets, ch; + unsigned char tmp[INADDRSZ], *tp; + + saw_digit = 0; + octets = 0; + *(tp = tmp) = 0; + while ((ch = *src++) != '\0') { + const char *pch; + + pch = strchr(digits, ch); + if (pch != NULL) { + unsigned int new = *tp * 10 + (pch - digits); + + if (new > 255) + return 0; + if (!saw_digit) { + if (++octets > 4) + return 0; + saw_digit = 1; + } + *tp = (unsigned char)new; + } else if (ch == '.' && saw_digit) { + if (octets == 4) + return 0; + *++tp = 0; + saw_digit = 0; + } else + return 0; + } + if (octets < 4) + return 0; + + memcpy(dst, tmp, INADDRSZ); + return 1; +} + +/* int + * inet_pton6(src, dst) + * convert presentation level address to network order binary form. + * return: + * 1 if `src' is a valid [RFC1884 2.2] address, else 0. + * notice: + * (1) does not touch `dst' unless it's returning 1. + * (2) :: in a full address is silently ignored. + * credit: + * inspired by Mark Andrews. + * author: + * Paul Vixie, 1996. + */ +static int +inet_pton6(const char *src, unsigned char *dst) +{ + static const char xdigits_l[] = "0123456789abcdef", + xdigits_u[] = "0123456789ABCDEF"; + unsigned char tmp[IN6ADDRSZ], *tp = 0, *endp = 0, *colonp = 0; + const char *xdigits = 0, *curtok = 0; + int ch = 0, saw_xdigit = 0, count_xdigit = 0; + unsigned int val = 0; + unsigned dbloct_count = 0; + + memset((tp = tmp), '\0', IN6ADDRSZ); + endp = tp + IN6ADDRSZ; + colonp = NULL; + /* Leading :: requires some special handling. */ + if (*src == ':') + if (*++src != ':') + return 0; + curtok = src; + saw_xdigit = count_xdigit = 0; + val = 0; + + while ((ch = *src++) != '\0') { + const char *pch; + + pch = strchr((xdigits = xdigits_l), ch); + if (pch == NULL) + pch = strchr((xdigits = xdigits_u), ch); + if (pch != NULL) { + if (count_xdigit >= 4) + return 0; + val <<= 4; + val |= (pch - xdigits); + if (val > 0xffff) + return 0; + saw_xdigit = 1; + count_xdigit++; + continue; + } + if (ch == ':') { + curtok = src; + if (!saw_xdigit) { + if (colonp) + return 0; + colonp = tp; + continue; + } else if (*src == '\0') { + return 0; + } + if (tp + sizeof(int16_t) > endp) + return 0; + *tp++ = (unsigned char) ((val >> 8) & 0xff); + *tp++ = (unsigned char) (val & 0xff); + saw_xdigit = 0; + count_xdigit = 0; + val = 0; + dbloct_count++; + continue; + } + if (ch == '.' && ((tp + INADDRSZ) <= endp) && + inet_pton4(curtok, tp) > 0) { + tp += INADDRSZ; + saw_xdigit = 0; + dbloct_count += 2; + break; /* '\0' was seen by inet_pton4(). */ + } + return 0; + } + if (saw_xdigit) { + if (tp + sizeof(int16_t) > endp) + return 0; + *tp++ = (unsigned char) ((val >> 8) & 0xff); + *tp++ = (unsigned char) (val & 0xff); + dbloct_count++; + } + if (colonp != NULL) { + /* if we already have 8 double octets, having a colon + * means error */ + if (dbloct_count == 8) + return 0; + + /* + * Since some memmove()'s erroneously fail to handle + * overlapping regions, we'll do the shift by hand. + */ + const int n = tp - colonp; + int i; + + for (i = 1; i <= n; i++) { + endp[-i] = colonp[n - i]; + colonp[n - i] = 0; + } + tp = endp; + } + if (tp != endp) + return 0; + memcpy(dst, tmp, IN6ADDRSZ); + return 1; +} + +int +parse_ipv4_addr(const char *token, struct in_addr *ipv4, uint32_t *mask) +{ + char ip_str[256] = {0}; + char *pch; + + pch = strchr(token, '/'); + if (pch != NULL) { + strncpy(ip_str, token, pch - token); + pch += 1; + if (is_str_num(pch) != 0) + return -EINVAL; + if (mask) + *mask = atoi(pch); + } else { + strncpy(ip_str, token, sizeof(ip_str) - 1); + if (mask) + *mask = 0; + } + + if (strlen(ip_str) >= INET_ADDRSTRLEN) + return -EINVAL; + + if (inet_pton4(ip_str, (unsigned char *)ipv4) != 1) + return -EINVAL; + + return 0; +} + +int +parse_ipv6_addr(const char *token, struct in6_addr *ipv6, uint32_t *mask) +{ + char ip_str[256] = {0}; + char *pch; + + pch = strchr(token, '/'); + if (pch != NULL) { + strncpy(ip_str, token, pch - token); + pch += 1; + if (is_str_num(pch) != 0) + return -EINVAL; + if (mask) + *mask = atoi(pch); + } else { + strncpy(ip_str, token, sizeof(ip_str) - 1); + if (mask) + *mask = 0; + } + + if (strlen(ip_str) >= INET6_ADDRSTRLEN) + return -EINVAL; + + if (inet_pton6(ip_str, (unsigned char *)ipv6) != 1) + return -EINVAL; + + return 0; +} + +int +parse_range(const char *token, uint16_t *low, uint16_t *high) +{ + char ch; + char num_str[20]; + uint32_t pos; + int range_low = -1; + int range_high = -1; + + if (!low || !high) + return -1; + + memset(num_str, 0, 20); + pos = 0; + + while ((ch = *token++) != '\0') { + if (isdigit(ch)) { + if (pos >= 19) + return -1; + num_str[pos++] = ch; + } else if (ch == ':') { + if (range_low != -1) + return -1; + range_low = atoi(num_str); + memset(num_str, 0, 20); + pos = 0; + } + } + + if (strlen(num_str) == 0) + return -1; + + range_high = atoi(num_str); + + *low = (uint16_t)range_low; + *high = (uint16_t)range_high; + + return 0; +} + +/** sp add parse */ +struct cfg_sp_add_cfg_item { + cmdline_fixed_string_t sp_keyword; + cmdline_multi_string_t multi_string; +}; + +static void +cfg_sp_add_cfg_item_parsed(void *parsed_result, + __rte_unused struct cmdline *cl, void *data) +{ + struct cfg_sp_add_cfg_item *params = parsed_result; + char *tokens[32]; + uint32_t n_tokens = RTE_DIM(tokens); + struct parse_status *status = (struct parse_status *)data; + + APP_CHECK((parse_tokenize_string(params->multi_string, tokens, + &n_tokens) == 0), status, "too many arguments"); + + if (status->status < 0) + return; + + if (strcmp(tokens[0], "ipv4") == 0) { + parse_sp4_tokens(tokens, n_tokens, status); + if (status->status < 0) + return; + } else if (strcmp(tokens[0], "ipv6") == 0) { + parse_sp6_tokens(tokens, n_tokens, status); + if (status->status < 0) + return; + } else { + APP_CHECK(0, status, "unrecognizable input %s\n", + tokens[0]); + return; + } +} + +static cmdline_parse_token_string_t cfg_sp_add_sp_str = + TOKEN_STRING_INITIALIZER(struct cfg_sp_add_cfg_item, + sp_keyword, "sp"); + +static cmdline_parse_token_string_t cfg_sp_add_multi_str = + TOKEN_STRING_INITIALIZER(struct cfg_sp_add_cfg_item, multi_string, + TOKEN_STRING_MULTI); + +cmdline_parse_inst_t cfg_sp_add_rule = { + .f = cfg_sp_add_cfg_item_parsed, + .data = NULL, + .help_str = "", + .tokens = { + (void *) &cfg_sp_add_sp_str, + (void *) &cfg_sp_add_multi_str, + NULL, + }, +}; + +/* sa add parse */ +struct cfg_sa_add_cfg_item { + cmdline_fixed_string_t sa_keyword; + cmdline_multi_string_t multi_string; +}; + +static void +cfg_sa_add_cfg_item_parsed(void *parsed_result, + __rte_unused struct cmdline *cl, void *data) +{ + struct cfg_sa_add_cfg_item *params = parsed_result; + char *tokens[32]; + uint32_t n_tokens = RTE_DIM(tokens); + struct parse_status *status = (struct parse_status *)data; + + APP_CHECK(parse_tokenize_string(params->multi_string, tokens, + &n_tokens) == 0, status, "too many arguments\n"); + + parse_sa_tokens(tokens, n_tokens, status); +} + +static cmdline_parse_token_string_t cfg_sa_add_sa_str = + TOKEN_STRING_INITIALIZER(struct cfg_sa_add_cfg_item, + sa_keyword, "sa"); + +static cmdline_parse_token_string_t cfg_sa_add_multi_str = + TOKEN_STRING_INITIALIZER(struct cfg_sa_add_cfg_item, multi_string, + TOKEN_STRING_MULTI); + +cmdline_parse_inst_t cfg_sa_add_rule = { + .f = cfg_sa_add_cfg_item_parsed, + .data = NULL, + .help_str = "", + .tokens = { + (void *) &cfg_sa_add_sa_str, + (void *) &cfg_sa_add_multi_str, + NULL, + }, +}; + +/* rt add parse */ +struct cfg_rt_add_cfg_item { + cmdline_fixed_string_t rt_keyword; + cmdline_multi_string_t multi_string; +}; + +static void +cfg_rt_add_cfg_item_parsed(void *parsed_result, + __rte_unused struct cmdline *cl, void *data) +{ + struct cfg_rt_add_cfg_item *params = parsed_result; + char *tokens[32]; + uint32_t n_tokens = RTE_DIM(tokens); + struct parse_status *status = (struct parse_status *)data; + + APP_CHECK(parse_tokenize_string( + params->multi_string, tokens, &n_tokens) == 0, + status, "too many arguments\n"); + if (status->status < 0) + return; + + parse_rt_tokens(tokens, n_tokens, status); +} + +static cmdline_parse_token_string_t cfg_rt_add_rt_str = + TOKEN_STRING_INITIALIZER(struct cfg_rt_add_cfg_item, + rt_keyword, "rt"); + +static cmdline_parse_token_string_t cfg_rt_add_multi_str = + TOKEN_STRING_INITIALIZER(struct cfg_rt_add_cfg_item, multi_string, + TOKEN_STRING_MULTI); + +cmdline_parse_inst_t cfg_rt_add_rule = { + .f = cfg_rt_add_cfg_item_parsed, + .data = NULL, + .help_str = "", + .tokens = { + (void *) &cfg_rt_add_rt_str, + (void *) &cfg_rt_add_multi_str, + NULL, + }, +}; + +/** set of cfg items */ +cmdline_parse_ctx_t ipsec_ctx[] = { + (cmdline_parse_inst_t *)&cfg_sp_add_rule, + (cmdline_parse_inst_t *)&cfg_sa_add_rule, + (cmdline_parse_inst_t *)&cfg_rt_add_rule, + NULL, +}; + +int +parse_cfg_file(const char *cfg_filename) +{ + struct cmdline *cl = cmdline_stdin_new(ipsec_ctx, ""); + FILE *f = fopen(cfg_filename, "r"); + char str[1024] = {0}, *get_s = NULL; + uint32_t line_num = 0; + struct parse_status status = {0}; + + if (f == NULL) { + rte_panic("Error: invalid file descriptor %s\n", cfg_filename); + goto error_exit; + } + + if (cl == NULL) { + rte_panic("Error: cannot create cmdline instance\n"); + goto error_exit; + } + + cfg_sp_add_rule.data = &status; + cfg_sa_add_rule.data = &status; + cfg_rt_add_rule.data = &status; + + do { + char oneline[1024]; + char *pos; + get_s = fgets(oneline, 1024, f); + + if (!get_s) + break; + + line_num++; + + if (strlen(oneline) > 1022) { + rte_panic("%s:%u: error: " + "the line contains more characters the parser can handle\n", + cfg_filename, line_num); + goto error_exit; + } + + /* process comment char '#' */ + if (oneline[0] == '#') + continue; + + pos = strchr(oneline, '#'); + if (pos != NULL) + *pos = '\0'; + + /* process line concatenator '\' */ + pos = strchr(oneline, 92); + if (pos != NULL) { + if (pos != oneline+strlen(oneline) - 2) { + rte_panic("%s:%u: error: " + "no character should exist after '\\'\n", + cfg_filename, line_num); + goto error_exit; + } + + *pos = '\0'; + + if (strlen(oneline) + strlen(str) > 1022) { + rte_panic("%s:%u: error: " + "the concatenated line contains more characters the parser can handle\n", + cfg_filename, line_num); + goto error_exit; + } + + strncpy(str + strlen(str), oneline, + strlen(oneline)); + + continue; + } + + /* copy the line to str and process */ + if (strlen(oneline) + strlen(str) > 1022) { + rte_panic("%s:%u: error: " + "the line contains more characters the parser can handle\n", + cfg_filename, line_num); + goto error_exit; + } + strncpy(str + strlen(str), oneline, + strlen(oneline)); + + str[strlen(str)] = '\n'; + if (cmdline_parse(cl, str) < 0) { + rte_panic("%s:%u: error: parsing \"%s\" failed\n", + cfg_filename, line_num, str); + goto error_exit; + } + + if (status.status < 0) { + rte_panic("%s:%u: error: %s", cfg_filename, + line_num, status.parse_msg); + goto error_exit; + } + + memset(str, 0, 1024); + } while (1); + + cmdline_stdin_exit(cl); + fclose(f); + + return 0; + +error_exit: + if (cl) + cmdline_stdin_exit(cl); + if (f) + fclose(f); + + return -1; +} diff --git a/dpdk/examples/ipsec-secgw/parser.h b/dpdk/examples/ipsec-secgw/parser.h new file mode 100644 index 00000000..d31ae016 --- /dev/null +++ b/dpdk/examples/ipsec-secgw/parser.h @@ -0,0 +1,116 @@ +/* BSD LICENSE + * + * Copyright(c) 2016 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include + +#ifndef __PARSER_H +#define __PARSER_H + +struct parse_status { + int status; + char parse_msg[256]; +}; + +#define APP_CHECK(exp, status, fmt, ...) \ +do { \ + if (!(exp)) { \ + sprintf(status->parse_msg, fmt "\n", \ + ## __VA_ARGS__); \ + status->status = -1; \ + } else \ + status->status = 0; \ +} while (0) + +#define APP_CHECK_PRESENCE(val, str, status) \ + APP_CHECK(val == 0, status, \ + "item \"%s\" already present", str) + +#define APP_CHECK_TOKEN_EQUAL(tokens, index, ref, status) \ + APP_CHECK(strcmp(tokens[index], ref) == 0, status, \ + "unrecognized input \"%s\": expect \"%s\"\n", \ + tokens[index], ref) + +static inline int +is_str_num(const char *str) +{ + uint32_t i; + + for (i = 0; i < strlen(str); i++) + if (!isdigit(str[i])) + return -1; + + return 0; +} + +#define APP_CHECK_TOKEN_IS_NUM(tokens, index, status) \ + APP_CHECK(is_str_num(tokens[index]) == 0, status, \ + "input \"%s\" is not valid number string", tokens[index]) + + +#define INCREMENT_TOKEN_INDEX(index, max_num, status) \ +do { \ + APP_CHECK(index + 1 < max_num, status, "reaching the end of " \ + "the token array"); \ + index++; \ +} while (0) + +int +parse_ipv4_addr(const char *token, struct in_addr *ipv4, uint32_t *mask); + +int +parse_ipv6_addr(const char *token, struct in6_addr *ipv6, uint32_t *mask); + +int +parse_range(const char *token, uint16_t *low, uint16_t *high); + +void +parse_sp4_tokens(char **tokens, uint32_t n_tokens, + struct parse_status *status); + +void +parse_sp6_tokens(char **tokens, uint32_t n_tokens, + struct parse_status *status); + +void +parse_sa_tokens(char **tokens, uint32_t n_tokens, + struct parse_status *status); + +void +parse_rt_tokens(char **tokens, uint32_t n_tokens, + struct parse_status *status); + +int +parse_cfg_file(const char *cfg_filename); + +#endif diff --git a/dpdk/examples/ipsec-secgw/rt.c b/dpdk/examples/ipsec-secgw/rt.c index fa5f0420..e03c5f0b 100644 --- a/dpdk/examples/ipsec-secgw/rt.c +++ b/dpdk/examples/ipsec-secgw/rt.c @@ -41,6 +41,7 @@ #include #include "ipsec.h" +#include "parser.h" #define RT_IPV4_MAX_RULES 1024 #define RT_IPV6_MAX_RULES 1024 @@ -57,135 +58,106 @@ struct ip6_route { uint8_t if_out; }; -static struct ip4_route rt_ip4_ep0[] = { - /* Outbound */ - /* Tunnels */ - { IPv4(172, 16, 2, 5), 32, 0 }, - { IPv4(172, 16, 2, 6), 32, 1 }, - /* Transport */ - { IPv4(192, 168, 175, 0), 24, 0 }, - { IPv4(192, 168, 176, 0), 24, 1 }, - /* Bypass */ - { IPv4(192, 168, 240, 0), 24, 0 }, - { IPv4(192, 168, 241, 0), 24, 1 }, +struct ip4_route rt_ip4[RT_IPV4_MAX_RULES]; +uint32_t nb_rt_ip4; - /* Inbound */ - /* Tunnels */ - { IPv4(192, 168, 115, 0), 24, 2 }, - { IPv4(192, 168, 116, 0), 24, 3 }, - { IPv4(192, 168, 65, 0), 24, 2 }, - { IPv4(192, 168, 66, 0), 24, 3 }, - /* Transport */ - { IPv4(192, 168, 185, 0), 24, 2 }, - { IPv4(192, 168, 186, 0), 24, 3 }, - /* NULL */ - { IPv4(192, 168, 210, 0), 24, 2 }, - { IPv4(192, 168, 211, 0), 24, 3 }, - /* Bypass */ - { IPv4(192, 168, 245, 0), 24, 2 }, - { IPv4(192, 168, 246, 0), 24, 3 }, -}; - -static struct ip6_route rt_ip6_ep0[] = { - /* Outbound */ - /* Tunnels */ - { { 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, - 0x22, 0x22, 0x22, 0x22, 0x22, 0x55, 0x55 }, 116, 0 }, - { { 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, - 0x22, 0x22, 0x22, 0x22, 0x22, 0x66, 0x66 }, 116, 1 }, - /* Transport */ - { { 0x00, 0x00, 0x00, 0x00, 0x11, 0x11, 0x11, 0x11, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 116, 0 }, - { { 0x00, 0x00, 0x00, 0x00, 0x11, 0x11, 0x11, 0x11, 0x11, - 0x11, 0x11, 0x11, 0x00, 0x00, 0x00, 0x00 }, 116, 1 }, - /* Inbound */ - /* Tunnels */ - { { 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xaa, - 0xaa, 0xaa, 0xaa, 0x00, 0x00, 0x00, 0x00 }, 116, 2 }, - { { 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbb, - 0xbb, 0xbb, 0xbb, 0x00, 0x00, 0x00, 0x00 }, 116, 3 }, - { { 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x55, - 0x55, 0x55, 0x55, 0x00, 0x00, 0x00, 0x00 }, 116, 2 }, - { { 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x66, - 0x66, 0x66, 0x66, 0x00, 0x00, 0x00, 0x00 }, 116, 3 }, - /* Transport */ - { { 0xff, 0xff, 0x00, 0x00, 0x11, 0x11, 0x11, 0x11, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 116, 2 }, - { { 0xff, 0xff, 0x00, 0x00, 0x11, 0x11, 0x11, 0x11, 0x11, - 0x11, 0x11, 0x11, 0x00, 0x00, 0x00, 0x00 }, 116, 3 }, -}; - -static struct ip4_route rt_ip4_ep1[] = { - /* Outbound */ - /* Tunnels */ - { IPv4(172, 16, 1, 5), 32, 0 }, - { IPv4(172, 16, 1, 6), 32, 1 }, - /* Transport */ - { IPv4(192, 168, 185, 0), 24, 0 }, - { IPv4(192, 168, 186, 0), 24, 1 }, - /* Bypass */ - { IPv4(192, 168, 245, 0), 24, 0 }, - { IPv4(192, 168, 246, 0), 24, 1 }, - - /* Inbound */ - /* Tunnels */ - { IPv4(192, 168, 105, 0), 24, 2 }, - { IPv4(192, 168, 106, 0), 24, 3 }, - { IPv4(192, 168, 55, 0), 24, 2 }, - { IPv4(192, 168, 56, 0), 24, 3 }, - /* Transport */ - { IPv4(192, 168, 175, 0), 24, 2 }, - { IPv4(192, 168, 176, 0), 24, 3 }, - /* NULL */ - { IPv4(192, 168, 200, 0), 24, 2 }, - { IPv4(192, 168, 201, 0), 24, 3 }, - /* Bypass */ - { IPv4(192, 168, 240, 0), 24, 2 }, - { IPv4(192, 168, 241, 0), 24, 3 }, -}; - -static struct ip6_route rt_ip6_ep1[] = { - /* Outbound */ - /* Tunnels */ - { { 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, - 0x11, 0x11, 0x11, 0x11, 0x11, 0x55, 0x55 }, 116, 0 }, - { { 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, - 0x11, 0x11, 0x11, 0x11, 0x11, 0x66, 0x66 }, 116, 1 }, - /* Transport */ - { { 0xff, 0xff, 0x00, 0x00, 0x11, 0x11, 0x11, 0x11, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 116, 0 }, - { { 0xff, 0xff, 0x00, 0x00, 0x11, 0x11, 0x11, 0x11, 0x11, - 0x11, 0x11, 0x11, 0x00, 0x00, 0x00, 0x00 }, 116, 1 }, - - /* Inbound */ - /* Tunnels */ - { { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xaa, - 0xaa, 0xaa, 0xaa, 0x00, 0x00, 0x00, 0x00 }, 116, 2 }, - { { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbb, - 0xbb, 0xbb, 0xbb, 0x00, 0x00, 0x00, 0x00 }, 116, 3 }, - { { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x55, - 0x55, 0x55, 0x55, 0x00, 0x00, 0x00, 0x00 }, 116, 2 }, - { { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x66, - 0x66, 0x66, 0x66, 0x00, 0x00, 0x00, 0x00 }, 116, 3 }, - /* Transport */ - { { 0x00, 0x00, 0x00, 0x00, 0x11, 0x11, 0x11, 0x11, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 116, 2 }, - { { 0x00, 0x00, 0x00, 0x00, 0x11, 0x11, 0x11, 0x11, 0x11, - 0x11, 0x11, 0x11, 0x00, 0x00, 0x00, 0x00 }, 116, 3 }, -}; +struct ip6_route rt_ip6[RT_IPV4_MAX_RULES]; +uint32_t nb_rt_ip6; void -rt_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t ep) +parse_rt_tokens(char **tokens, uint32_t n_tokens, + struct parse_status *status) +{ + uint32_t ti; + uint32_t *n_rts = NULL; + struct ip4_route *route_ipv4 = NULL; + struct ip6_route *route_ipv6 = NULL; + + if (strcmp(tokens[0], "ipv4") == 0) { + n_rts = &nb_rt_ip4; + route_ipv4 = &rt_ip4[*n_rts]; + + APP_CHECK(*n_rts <= RT_IPV4_MAX_RULES - 1, status, + "too many rt rules, abort insertion\n"); + if (status->status < 0) + return; + + } else if (strcmp(tokens[0], "ipv6") == 0) { + n_rts = &nb_rt_ip6; + route_ipv6 = &rt_ip6[*n_rts]; + + APP_CHECK(*n_rts <= RT_IPV6_MAX_RULES - 1, status, + "too many rt rules, abort insertion\n"); + if (status->status < 0) + return; + } else { + APP_CHECK(0, status, "unrecognized input \"%s\"", + tokens[0]); + return; + } + + for (ti = 1; ti < n_tokens; ti++) { + if (strcmp(tokens[ti], "dst") == 0) { + INCREMENT_TOKEN_INDEX(ti, n_tokens, status); + if (status->status < 0) + return; + + if (route_ipv4 != NULL) { + struct in_addr ip; + uint32_t depth = 0; + + APP_CHECK(parse_ipv4_addr(tokens[ti], + &ip, &depth) == 0, status, + "unrecognized input \"%s\", " + "expect valid ipv4 addr", + tokens[ti]); + if (status->status < 0) + return; + route_ipv4->ip = rte_bswap32( + (uint32_t)ip.s_addr); + route_ipv4->depth = (uint8_t)depth; + } else { + struct in6_addr ip; + uint32_t depth; + + APP_CHECK(parse_ipv6_addr(tokens[ti], + &ip, &depth) == 0, status, + "unrecognized input \"%s\", " + "expect valid ipv6 address", + tokens[ti]); + if (status->status < 0) + return; + memcpy(route_ipv6->ip, ip.s6_addr, 16); + route_ipv6->depth = (uint8_t)depth; + } + } + + if (strcmp(tokens[ti], "port") == 0) { + INCREMENT_TOKEN_INDEX(ti, n_tokens, status); + if (status->status < 0) + return; + APP_CHECK_TOKEN_IS_NUM(tokens, ti, status); + if (status->status < 0) + return; + if (route_ipv4 != NULL) + route_ipv4->if_out = atoi(tokens[ti]); + else + route_ipv6->if_out = atoi(tokens[ti]); + } + } + + *n_rts = *n_rts + 1; +} + +void +rt_init(struct socket_ctx *ctx, int32_t socket_id) { char name[PATH_MAX]; uint32_t i; int32_t ret; struct rte_lpm *lpm; struct rte_lpm6 *lpm6; - struct ip4_route *rt; - struct ip6_route *rt6; char a, b, c, d; - uint32_t nb_routes, nb_routes6; struct rte_lpm_config conf = { 0 }; struct rte_lpm6_config conf6 = { 0 }; @@ -200,23 +172,12 @@ rt_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t ep) rte_exit(EXIT_FAILURE, "IPv6 Routing Table for socket %u " "already initialized\n", socket_id); + if (nb_rt_ip4 == 0 && nb_rt_ip6 == 0) + RTE_LOG(WARNING, IPSEC, "No Routing rule specified\n"); + printf("Creating IPv4 Routing Table (RT) context with %u max routes\n", RT_IPV4_MAX_RULES); - if (ep == 0) { - rt = rt_ip4_ep0; - nb_routes = RTE_DIM(rt_ip4_ep0); - rt6 = rt_ip6_ep0; - nb_routes6 = RTE_DIM(rt_ip6_ep0); - } else if (ep == 1) { - rt = rt_ip4_ep1; - nb_routes = RTE_DIM(rt_ip4_ep1); - rt6 = rt_ip6_ep1; - nb_routes6 = RTE_DIM(rt_ip6_ep1); - } else - rte_exit(EXIT_FAILURE, "Invalid EP value %u. Only 0 or 1 " - "supported.\n", ep); - /* create the LPM table */ snprintf(name, sizeof(name), "%s_%u", "rt_ip4", socket_id); conf.max_rules = RT_IPV4_MAX_RULES; @@ -227,15 +188,17 @@ rt_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t ep) "on socket %d\n", name, socket_id); /* populate the LPM table */ - for (i = 0; i < nb_routes; i++) { - ret = rte_lpm_add(lpm, rt[i].ip, rt[i].depth, rt[i].if_out); + for (i = 0; i < nb_rt_ip4; i++) { + ret = rte_lpm_add(lpm, rt_ip4[i].ip, rt_ip4[i].depth, + rt_ip4[i].if_out); if (ret < 0) rte_exit(EXIT_FAILURE, "Fail to add entry num %u to %s " "LPM table on socket %d\n", i, name, socket_id); - uint32_t_to_char(rt[i].ip, &a, &b, &c, &d); + uint32_t_to_char(rt_ip4[i].ip, &a, &b, &c, &d); printf("LPM: Adding route %hhu.%hhu.%hhu.%hhu/%hhu (%hhu)\n", - a, b, c, d, rt[i].depth, rt[i].if_out); + a, b, c, d, rt_ip4[i].depth, + rt_ip4[i].if_out); } snprintf(name, sizeof(name), "%s_%u", "rt_ip6", socket_id); @@ -247,24 +210,24 @@ rt_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t ep) "on socket %d\n", name, socket_id); /* populate the LPM table */ - for (i = 0; i < nb_routes6; i++) { - ret = rte_lpm6_add(lpm6, rt6[i].ip, rt6[i].depth, - rt6[i].if_out); + for (i = 0; i < nb_rt_ip6; i++) { + ret = rte_lpm6_add(lpm6, rt_ip6[i].ip, rt_ip6[i].depth, + rt_ip6[i].if_out); if (ret < 0) rte_exit(EXIT_FAILURE, "Fail to add entry num %u to %s " "LPM table on socket %d\n", i, name, socket_id); printf("LPM6: Adding route " " %hx:%hx:%hx:%hx:%hx:%hx:%hx:%hx/%hhx (%hhx)\n", - (uint16_t)((rt6[i].ip[0] << 8) | rt6[i].ip[1]), - (uint16_t)((rt6[i].ip[2] << 8) | rt6[i].ip[3]), - (uint16_t)((rt6[i].ip[4] << 8) | rt6[i].ip[5]), - (uint16_t)((rt6[i].ip[6] << 8) | rt6[i].ip[7]), - (uint16_t)((rt6[i].ip[8] << 8) | rt6[i].ip[9]), - (uint16_t)((rt6[i].ip[10] << 8) | rt6[i].ip[11]), - (uint16_t)((rt6[i].ip[12] << 8) | rt6[i].ip[13]), - (uint16_t)((rt6[i].ip[14] << 8) | rt6[i].ip[15]), - rt6[i].depth, rt6[i].if_out); + (uint16_t)((rt_ip6[i].ip[0] << 8) | rt_ip6[i].ip[1]), + (uint16_t)((rt_ip6[i].ip[2] << 8) | rt_ip6[i].ip[3]), + (uint16_t)((rt_ip6[i].ip[4] << 8) | rt_ip6[i].ip[5]), + (uint16_t)((rt_ip6[i].ip[6] << 8) | rt_ip6[i].ip[7]), + (uint16_t)((rt_ip6[i].ip[8] << 8) | rt_ip6[i].ip[9]), + (uint16_t)((rt_ip6[i].ip[10] << 8) | rt_ip6[i].ip[11]), + (uint16_t)((rt_ip6[i].ip[12] << 8) | rt_ip6[i].ip[13]), + (uint16_t)((rt_ip6[i].ip[14] << 8) | rt_ip6[i].ip[15]), + rt_ip6[i].depth, rt_ip6[i].if_out); } ctx->rt_ip4 = (struct rt_ctx *)lpm; diff --git a/dpdk/examples/ipsec-secgw/sa.c b/dpdk/examples/ipsec-secgw/sa.c index 4439e0f5..b9f4a211 100644 --- a/dpdk/examples/ipsec-secgw/sa.c +++ b/dpdk/examples/ipsec-secgw/sa.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -41,256 +41,678 @@ #include #include +#include #include #include #include #include +#include +#include #include "ipsec.h" #include "esp.h" +#include "parser.h" -/* SAs Outbound */ -const struct ipsec_sa sa_out[] = { - { - .spi = 5, - .src.ip.ip4 = IPv4(172, 16, 1, 5), - .dst.ip.ip4 = IPv4(172, 16, 2, 5), - .cipher_algo = RTE_CRYPTO_CIPHER_AES_CBC, - .auth_algo = RTE_CRYPTO_AUTH_SHA1_HMAC, - .digest_len = 12, - .iv_len = 16, - .block_size = 16, - .flags = IP4_TUNNEL - }, - { - .spi = 6, - .src.ip.ip4 = IPv4(172, 16, 1, 6), - .dst.ip.ip4 = IPv4(172, 16, 2, 6), - .cipher_algo = RTE_CRYPTO_CIPHER_AES_CBC, - .auth_algo = RTE_CRYPTO_AUTH_SHA1_HMAC, - .digest_len = 12, - .iv_len = 16, - .block_size = 16, - .flags = IP4_TUNNEL - }, - { - .spi = 10, - .cipher_algo = RTE_CRYPTO_CIPHER_AES_CBC, - .auth_algo = RTE_CRYPTO_AUTH_SHA1_HMAC, - .digest_len = 12, - .iv_len = 16, - .block_size = 16, - .flags = TRANSPORT - }, - { - .spi = 11, - .cipher_algo = RTE_CRYPTO_CIPHER_AES_CBC, - .auth_algo = RTE_CRYPTO_AUTH_SHA1_HMAC, - .digest_len = 12, - .iv_len = 16, - .block_size = 16, - .flags = TRANSPORT - }, - { - .spi = 15, - .src.ip.ip4 = IPv4(172, 16, 1, 5), - .dst.ip.ip4 = IPv4(172, 16, 2, 5), - .cipher_algo = RTE_CRYPTO_CIPHER_NULL, - .auth_algo = RTE_CRYPTO_AUTH_NULL, - .digest_len = 0, - .iv_len = 0, - .block_size = 4, - .flags = IP4_TUNNEL - }, - { - .spi = 16, - .src.ip.ip4 = IPv4(172, 16, 1, 6), - .dst.ip.ip4 = IPv4(172, 16, 2, 6), - .cipher_algo = RTE_CRYPTO_CIPHER_NULL, - .auth_algo = RTE_CRYPTO_AUTH_NULL, - .digest_len = 0, - .iv_len = 0, - .block_size = 4, - .flags = IP4_TUNNEL - }, - { - .spi = 25, - .src.ip.ip6.ip6_b = { 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, - 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x55, 0x55 }, - .dst.ip.ip6.ip6_b = { 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, - 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x55, 0x55 }, - .cipher_algo = RTE_CRYPTO_CIPHER_AES_CBC, - .auth_algo = RTE_CRYPTO_AUTH_SHA1_HMAC, - .digest_len = 12, - .iv_len = 16, - .block_size = 16, - .flags = IP6_TUNNEL - }, - { - .spi = 26, - .src.ip.ip6.ip6_b = { 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, - 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x66, 0x66 }, - .dst.ip.ip6.ip6_b = { 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, - 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x66, 0x66 }, - .cipher_algo = RTE_CRYPTO_CIPHER_AES_CBC, - .auth_algo = RTE_CRYPTO_AUTH_SHA1_HMAC, - .digest_len = 12, - .iv_len = 16, - .block_size = 16, - .flags = IP6_TUNNEL - }, +#define IPDEFTTL 64 + +struct supported_cipher_algo { + const char *keyword; + enum rte_crypto_cipher_algorithm algo; + uint16_t iv_len; + uint16_t block_size; + uint16_t key_len; }; -/* SAs Inbound */ -const struct ipsec_sa sa_in[] = { - { - .spi = 105, - .src.ip.ip4 = IPv4(172, 16, 2, 5), - .dst.ip.ip4 = IPv4(172, 16, 1, 5), - .cipher_algo = RTE_CRYPTO_CIPHER_AES_CBC, - .auth_algo = RTE_CRYPTO_AUTH_SHA1_HMAC, - .digest_len = 12, - .iv_len = 16, - .block_size = 16, - .flags = IP4_TUNNEL - }, - { - .spi = 106, - .src.ip.ip4 = IPv4(172, 16, 2, 6), - .dst.ip.ip4 = IPv4(172, 16, 1, 6), - .cipher_algo = RTE_CRYPTO_CIPHER_AES_CBC, - .auth_algo = RTE_CRYPTO_AUTH_SHA1_HMAC, - .digest_len = 12, - .iv_len = 16, - .block_size = 16, - .flags = IP4_TUNNEL - }, - { - .spi = 110, - .cipher_algo = RTE_CRYPTO_CIPHER_AES_CBC, - .auth_algo = RTE_CRYPTO_AUTH_SHA1_HMAC, - .digest_len = 12, - .iv_len = 16, - .block_size = 16, - .flags = TRANSPORT - }, - { - .spi = 111, - .cipher_algo = RTE_CRYPTO_CIPHER_AES_CBC, - .auth_algo = RTE_CRYPTO_AUTH_SHA1_HMAC, - .digest_len = 12, - .iv_len = 16, - .block_size = 16, - .flags = TRANSPORT - }, - { - .spi = 115, - .src.ip.ip4 = IPv4(172, 16, 2, 5), - .dst.ip.ip4 = IPv4(172, 16, 1, 5), - .cipher_algo = RTE_CRYPTO_CIPHER_NULL, - .auth_algo = RTE_CRYPTO_AUTH_NULL, - .digest_len = 0, - .iv_len = 0, - .block_size = 4, - .flags = IP4_TUNNEL - }, - { - .spi = 116, - .src.ip.ip4 = IPv4(172, 16, 2, 6), - .dst.ip.ip4 = IPv4(172, 16, 1, 6), - .cipher_algo = RTE_CRYPTO_CIPHER_NULL, - .auth_algo = RTE_CRYPTO_AUTH_NULL, - .digest_len = 0, - .iv_len = 0, - .block_size = 4, - .flags = IP4_TUNNEL - }, - { - .spi = 125, - .src.ip.ip6.ip6_b = { 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, - 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x55, 0x55 }, - .dst.ip.ip6.ip6_b = { 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, - 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x55, 0x55 }, - .cipher_algo = RTE_CRYPTO_CIPHER_AES_CBC, - .auth_algo = RTE_CRYPTO_AUTH_SHA1_HMAC, - .digest_len = 12, - .iv_len = 16, - .block_size = 16, - .flags = IP6_TUNNEL - }, - { - .spi = 126, - .src.ip.ip6.ip6_b = { 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, - 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x66, 0x66 }, - .dst.ip.ip6.ip6_b = { 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, - 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x66, 0x66 }, - .cipher_algo = RTE_CRYPTO_CIPHER_AES_CBC, - .auth_algo = RTE_CRYPTO_AUTH_SHA1_HMAC, - .digest_len = 12, - .iv_len = 16, - .block_size = 16, - .flags = IP6_TUNNEL - }, +struct supported_auth_algo { + const char *keyword; + enum rte_crypto_auth_algorithm algo; + uint16_t digest_len; + uint16_t key_len; + uint8_t key_not_req; }; -static uint8_t cipher_key[256] = "sixteenbytes key"; +struct supported_aead_algo { + const char *keyword; + enum rte_crypto_aead_algorithm algo; + uint16_t iv_len; + uint16_t block_size; + uint16_t digest_len; + uint16_t key_len; + uint8_t aad_len; +}; -/* AES CBC xform */ -const struct rte_crypto_sym_xform aescbc_enc_xf = { - NULL, - RTE_CRYPTO_SYM_XFORM_CIPHER, - {.cipher = { RTE_CRYPTO_CIPHER_OP_ENCRYPT, RTE_CRYPTO_CIPHER_AES_CBC, - .key = { cipher_key, 16 } } + +const struct supported_cipher_algo cipher_algos[] = { + { + .keyword = "null", + .algo = RTE_CRYPTO_CIPHER_NULL, + .iv_len = 0, + .block_size = 4, + .key_len = 0 + }, + { + .keyword = "aes-128-cbc", + .algo = RTE_CRYPTO_CIPHER_AES_CBC, + .iv_len = 16, + .block_size = 16, + .key_len = 16 + }, + { + .keyword = "aes-128-ctr", + .algo = RTE_CRYPTO_CIPHER_AES_CTR, + .iv_len = 8, + .block_size = 16, /* XXX AESNI MB limition, should be 4 */ + .key_len = 20 } }; -const struct rte_crypto_sym_xform aescbc_dec_xf = { - NULL, - RTE_CRYPTO_SYM_XFORM_CIPHER, - {.cipher = { RTE_CRYPTO_CIPHER_OP_DECRYPT, RTE_CRYPTO_CIPHER_AES_CBC, - .key = { cipher_key, 16 } } +const struct supported_auth_algo auth_algos[] = { + { + .keyword = "null", + .algo = RTE_CRYPTO_AUTH_NULL, + .digest_len = 0, + .key_len = 0, + .key_not_req = 1 + }, + { + .keyword = "sha1-hmac", + .algo = RTE_CRYPTO_AUTH_SHA1_HMAC, + .digest_len = 12, + .key_len = 20 + }, + { + .keyword = "sha256-hmac", + .algo = RTE_CRYPTO_AUTH_SHA256_HMAC, + .digest_len = 12, + .key_len = 32 } }; -static uint8_t auth_key[256] = "twentybytes hash key"; - -/* SHA1 HMAC xform */ -const struct rte_crypto_sym_xform sha1hmac_gen_xf = { - NULL, - RTE_CRYPTO_SYM_XFORM_AUTH, - {.auth = { RTE_CRYPTO_AUTH_OP_GENERATE, RTE_CRYPTO_AUTH_SHA1_HMAC, - .key = { auth_key, 20 }, 12, 0 } +const struct supported_aead_algo aead_algos[] = { + { + .keyword = "aes-128-gcm", + .algo = RTE_CRYPTO_AEAD_AES_GCM, + .iv_len = 8, + .block_size = 4, + .key_len = 20, + .digest_len = 16, + .aad_len = 8, } }; -const struct rte_crypto_sym_xform sha1hmac_verify_xf = { - NULL, - RTE_CRYPTO_SYM_XFORM_AUTH, - {.auth = { RTE_CRYPTO_AUTH_OP_VERIFY, RTE_CRYPTO_AUTH_SHA1_HMAC, - .key = { auth_key, 20 }, 12, 0 } - } -}; +struct ipsec_sa sa_out[IPSEC_SA_MAX_ENTRIES]; +uint32_t nb_sa_out; -/* AES CBC xform */ -const struct rte_crypto_sym_xform null_cipher_xf = { - NULL, - RTE_CRYPTO_SYM_XFORM_CIPHER, - {.cipher = { .algo = RTE_CRYPTO_CIPHER_NULL } - } -}; +struct ipsec_sa sa_in[IPSEC_SA_MAX_ENTRIES]; +uint32_t nb_sa_in; -const struct rte_crypto_sym_xform null_auth_xf = { - NULL, - RTE_CRYPTO_SYM_XFORM_AUTH, - {.auth = { .algo = RTE_CRYPTO_AUTH_NULL } +static const struct supported_cipher_algo * +find_match_cipher_algo(const char *cipher_keyword) +{ + size_t i; + + for (i = 0; i < RTE_DIM(cipher_algos); i++) { + const struct supported_cipher_algo *algo = + &cipher_algos[i]; + + if (strcmp(cipher_keyword, algo->keyword) == 0) + return algo; } -}; + + return NULL; +} + +static const struct supported_auth_algo * +find_match_auth_algo(const char *auth_keyword) +{ + size_t i; + + for (i = 0; i < RTE_DIM(auth_algos); i++) { + const struct supported_auth_algo *algo = + &auth_algos[i]; + + if (strcmp(auth_keyword, algo->keyword) == 0) + return algo; + } + + return NULL; +} + +static const struct supported_aead_algo * +find_match_aead_algo(const char *aead_keyword) +{ + size_t i; + + for (i = 0; i < RTE_DIM(aead_algos); i++) { + const struct supported_aead_algo *algo = + &aead_algos[i]; + + if (strcmp(aead_keyword, algo->keyword) == 0) + return algo; + } + + return NULL; +} + +/** parse_key_string + * parse x:x:x:x.... hex number key string into uint8_t *key + * return: + * > 0: number of bytes parsed + * 0: failed + */ +static uint32_t +parse_key_string(const char *key_str, uint8_t *key) +{ + const char *pt_start = key_str, *pt_end = key_str; + uint32_t nb_bytes = 0; + + while (pt_end != NULL) { + char sub_str[3] = {0}; + + pt_end = strchr(pt_start, ':'); + + if (pt_end == NULL) { + if (strlen(pt_start) > 2) + return 0; + strncpy(sub_str, pt_start, 2); + } else { + if (pt_end - pt_start > 2) + return 0; + + strncpy(sub_str, pt_start, pt_end - pt_start); + pt_start = pt_end + 1; + } + + key[nb_bytes++] = strtol(sub_str, NULL, 16); + } + + return nb_bytes; +} + +void +parse_sa_tokens(char **tokens, uint32_t n_tokens, + struct parse_status *status) +{ + struct ipsec_sa *rule = NULL; + uint32_t ti; /*token index*/ + uint32_t *ri /*rule index*/; + uint32_t cipher_algo_p = 0; + uint32_t auth_algo_p = 0; + uint32_t aead_algo_p = 0; + uint32_t src_p = 0; + uint32_t dst_p = 0; + uint32_t mode_p = 0; + uint32_t type_p = 0; + uint32_t portid_p = 0; + + if (strcmp(tokens[0], "in") == 0) { + ri = &nb_sa_in; + + APP_CHECK(*ri <= IPSEC_SA_MAX_ENTRIES - 1, status, + "too many sa rules, abort insertion\n"); + if (status->status < 0) + return; + + rule = &sa_in[*ri]; + } else { + ri = &nb_sa_out; + + APP_CHECK(*ri <= IPSEC_SA_MAX_ENTRIES - 1, status, + "too many sa rules, abort insertion\n"); + if (status->status < 0) + return; + + rule = &sa_out[*ri]; + } + + /* spi number */ + APP_CHECK_TOKEN_IS_NUM(tokens, 1, status); + if (status->status < 0) + return; + if (atoi(tokens[1]) == INVALID_SPI) + return; + rule->spi = atoi(tokens[1]); + + for (ti = 2; ti < n_tokens; ti++) { + if (strcmp(tokens[ti], "mode") == 0) { + APP_CHECK_PRESENCE(mode_p, tokens[ti], status); + if (status->status < 0) + return; + + INCREMENT_TOKEN_INDEX(ti, n_tokens, status); + if (status->status < 0) + return; + + if (strcmp(tokens[ti], "ipv4-tunnel") == 0) + rule->flags = IP4_TUNNEL; + else if (strcmp(tokens[ti], "ipv6-tunnel") == 0) + rule->flags = IP6_TUNNEL; + else if (strcmp(tokens[ti], "transport") == 0) + rule->flags = TRANSPORT; + else { + APP_CHECK(0, status, "unrecognized " + "input \"%s\"", tokens[ti]); + return; + } + + mode_p = 1; + continue; + } + + if (strcmp(tokens[ti], "cipher_algo") == 0) { + const struct supported_cipher_algo *algo; + uint32_t key_len; + + APP_CHECK_PRESENCE(cipher_algo_p, tokens[ti], + status); + if (status->status < 0) + return; + + INCREMENT_TOKEN_INDEX(ti, n_tokens, status); + if (status->status < 0) + return; + + algo = find_match_cipher_algo(tokens[ti]); + + APP_CHECK(algo != NULL, status, "unrecognized " + "input \"%s\"", tokens[ti]); + + rule->cipher_algo = algo->algo; + rule->block_size = algo->block_size; + rule->iv_len = algo->iv_len; + rule->cipher_key_len = algo->key_len; + + /* for NULL algorithm, no cipher key required */ + if (rule->cipher_algo == RTE_CRYPTO_CIPHER_NULL) { + cipher_algo_p = 1; + continue; + } + + INCREMENT_TOKEN_INDEX(ti, n_tokens, status); + if (status->status < 0) + return; + + APP_CHECK(strcmp(tokens[ti], "cipher_key") == 0, + status, "unrecognized input \"%s\", " + "expect \"cipher_key\"", tokens[ti]); + if (status->status < 0) + return; + + INCREMENT_TOKEN_INDEX(ti, n_tokens, status); + if (status->status < 0) + return; + + key_len = parse_key_string(tokens[ti], + rule->cipher_key); + APP_CHECK(key_len == rule->cipher_key_len, status, + "unrecognized input \"%s\"", tokens[ti]); + if (status->status < 0) + return; + + if (algo->algo == RTE_CRYPTO_CIPHER_AES_CBC) + rule->salt = (uint32_t)rte_rand(); + + if (algo->algo == RTE_CRYPTO_CIPHER_AES_CTR) { + key_len -= 4; + rule->cipher_key_len = key_len; + memcpy(&rule->salt, + &rule->cipher_key[key_len], 4); + } + + cipher_algo_p = 1; + continue; + } + + if (strcmp(tokens[ti], "auth_algo") == 0) { + const struct supported_auth_algo *algo; + uint32_t key_len; + + APP_CHECK_PRESENCE(auth_algo_p, tokens[ti], + status); + if (status->status < 0) + return; + + INCREMENT_TOKEN_INDEX(ti, n_tokens, status); + if (status->status < 0) + return; + + algo = find_match_auth_algo(tokens[ti]); + APP_CHECK(algo != NULL, status, "unrecognized " + "input \"%s\"", tokens[ti]); + + rule->auth_algo = algo->algo; + rule->auth_key_len = algo->key_len; + rule->digest_len = algo->digest_len; + + /* NULL algorithm and combined algos do not + * require auth key + */ + if (algo->key_not_req) { + auth_algo_p = 1; + continue; + } + + INCREMENT_TOKEN_INDEX(ti, n_tokens, status); + if (status->status < 0) + return; + + APP_CHECK(strcmp(tokens[ti], "auth_key") == 0, + status, "unrecognized input \"%s\", " + "expect \"auth_key\"", tokens[ti]); + if (status->status < 0) + return; + + INCREMENT_TOKEN_INDEX(ti, n_tokens, status); + if (status->status < 0) + return; + + key_len = parse_key_string(tokens[ti], + rule->auth_key); + APP_CHECK(key_len == rule->auth_key_len, status, + "unrecognized input \"%s\"", tokens[ti]); + if (status->status < 0) + return; + + auth_algo_p = 1; + continue; + } + + if (strcmp(tokens[ti], "aead_algo") == 0) { + const struct supported_aead_algo *algo; + uint32_t key_len; + + APP_CHECK_PRESENCE(aead_algo_p, tokens[ti], + status); + if (status->status < 0) + return; + + INCREMENT_TOKEN_INDEX(ti, n_tokens, status); + if (status->status < 0) + return; + + algo = find_match_aead_algo(tokens[ti]); + + APP_CHECK(algo != NULL, status, "unrecognized " + "input \"%s\"", tokens[ti]); + + rule->aead_algo = algo->algo; + rule->cipher_key_len = algo->key_len; + rule->digest_len = algo->digest_len; + rule->aad_len = algo->aad_len; + rule->block_size = algo->block_size; + rule->iv_len = algo->iv_len; + + INCREMENT_TOKEN_INDEX(ti, n_tokens, status); + if (status->status < 0) + return; + + APP_CHECK(strcmp(tokens[ti], "aead_key") == 0, + status, "unrecognized input \"%s\", " + "expect \"aead_key\"", tokens[ti]); + if (status->status < 0) + return; + + INCREMENT_TOKEN_INDEX(ti, n_tokens, status); + if (status->status < 0) + return; + + key_len = parse_key_string(tokens[ti], + rule->cipher_key); + APP_CHECK(key_len == rule->cipher_key_len, status, + "unrecognized input \"%s\"", tokens[ti]); + if (status->status < 0) + return; + + key_len -= 4; + rule->cipher_key_len = key_len; + memcpy(&rule->salt, + &rule->cipher_key[key_len], 4); + + aead_algo_p = 1; + continue; + } + + if (strcmp(tokens[ti], "src") == 0) { + APP_CHECK_PRESENCE(src_p, tokens[ti], status); + if (status->status < 0) + return; + + INCREMENT_TOKEN_INDEX(ti, n_tokens, status); + if (status->status < 0) + return; + + if (rule->flags == IP4_TUNNEL) { + struct in_addr ip; + + APP_CHECK(parse_ipv4_addr(tokens[ti], + &ip, NULL) == 0, status, + "unrecognized input \"%s\", " + "expect valid ipv4 addr", + tokens[ti]); + if (status->status < 0) + return; + rule->src.ip.ip4 = rte_bswap32( + (uint32_t)ip.s_addr); + } else if (rule->flags == IP6_TUNNEL) { + struct in6_addr ip; + + APP_CHECK(parse_ipv6_addr(tokens[ti], &ip, + NULL) == 0, status, + "unrecognized input \"%s\", " + "expect valid ipv6 addr", + tokens[ti]); + if (status->status < 0) + return; + memcpy(rule->src.ip.ip6.ip6_b, + ip.s6_addr, 16); + } else if (rule->flags == TRANSPORT) { + APP_CHECK(0, status, "unrecognized input " + "\"%s\"", tokens[ti]); + return; + } + + src_p = 1; + continue; + } + + if (strcmp(tokens[ti], "dst") == 0) { + APP_CHECK_PRESENCE(dst_p, tokens[ti], status); + if (status->status < 0) + return; + + INCREMENT_TOKEN_INDEX(ti, n_tokens, status); + if (status->status < 0) + return; + + if (rule->flags == IP4_TUNNEL) { + struct in_addr ip; + + APP_CHECK(parse_ipv4_addr(tokens[ti], + &ip, NULL) == 0, status, + "unrecognized input \"%s\", " + "expect valid ipv4 addr", + tokens[ti]); + if (status->status < 0) + return; + rule->dst.ip.ip4 = rte_bswap32( + (uint32_t)ip.s_addr); + } else if (rule->flags == IP6_TUNNEL) { + struct in6_addr ip; + + APP_CHECK(parse_ipv6_addr(tokens[ti], &ip, + NULL) == 0, status, + "unrecognized input \"%s\", " + "expect valid ipv6 addr", + tokens[ti]); + if (status->status < 0) + return; + memcpy(rule->dst.ip.ip6.ip6_b, ip.s6_addr, 16); + } else if (rule->flags == TRANSPORT) { + APP_CHECK(0, status, "unrecognized " + "input \"%s\"", tokens[ti]); + return; + } + + dst_p = 1; + continue; + } + + if (strcmp(tokens[ti], "type") == 0) { + APP_CHECK_PRESENCE(type_p, tokens[ti], status); + if (status->status < 0) + return; + + INCREMENT_TOKEN_INDEX(ti, n_tokens, status); + if (status->status < 0) + return; + + if (strcmp(tokens[ti], "inline-crypto-offload") == 0) + rule->type = + RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO; + else if (strcmp(tokens[ti], + "inline-protocol-offload") == 0) + rule->type = + RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL; + else if (strcmp(tokens[ti], + "lookaside-protocol-offload") == 0) + rule->type = + RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL; + else if (strcmp(tokens[ti], "no-offload") == 0) + rule->type = RTE_SECURITY_ACTION_TYPE_NONE; + else { + APP_CHECK(0, status, "Invalid input \"%s\"", + tokens[ti]); + return; + } + + type_p = 1; + continue; + } + + if (strcmp(tokens[ti], "port_id") == 0) { + APP_CHECK_PRESENCE(portid_p, tokens[ti], status); + if (status->status < 0) + return; + INCREMENT_TOKEN_INDEX(ti, n_tokens, status); + if (status->status < 0) + return; + rule->portid = atoi(tokens[ti]); + if (status->status < 0) + return; + portid_p = 1; + continue; + } + + /* unrecognizeable input */ + APP_CHECK(0, status, "unrecognized input \"%s\"", + tokens[ti]); + return; + } + + if (aead_algo_p) { + APP_CHECK(cipher_algo_p == 0, status, + "AEAD used, no need for cipher options"); + if (status->status < 0) + return; + + APP_CHECK(auth_algo_p == 0, status, + "AEAD used, no need for auth options"); + if (status->status < 0) + return; + } else { + APP_CHECK(cipher_algo_p == 1, status, "missing cipher or AEAD options"); + if (status->status < 0) + return; + + APP_CHECK(auth_algo_p == 1, status, "missing auth or AEAD options"); + if (status->status < 0) + return; + } + + APP_CHECK(mode_p == 1, status, "missing mode option"); + if (status->status < 0) + return; + + if ((rule->type != RTE_SECURITY_ACTION_TYPE_NONE) && (portid_p == 0)) + printf("Missing portid option, falling back to non-offload\n"); + + if (!type_p || !portid_p) { + rule->type = RTE_SECURITY_ACTION_TYPE_NONE; + rule->portid = -1; + } + + *ri = *ri + 1; +} + +static inline void +print_one_sa_rule(const struct ipsec_sa *sa, int inbound) +{ + uint32_t i; + uint8_t a, b, c, d; + + printf("\tspi_%s(%3u):", inbound?"in":"out", sa->spi); + + for (i = 0; i < RTE_DIM(cipher_algos); i++) { + if (cipher_algos[i].algo == sa->cipher_algo) { + printf("%s ", cipher_algos[i].keyword); + break; + } + } + + for (i = 0; i < RTE_DIM(auth_algos); i++) { + if (auth_algos[i].algo == sa->auth_algo) { + printf("%s ", auth_algos[i].keyword); + break; + } + } + + for (i = 0; i < RTE_DIM(aead_algos); i++) { + if (aead_algos[i].algo == sa->aead_algo) { + printf("%s ", aead_algos[i].keyword); + break; + } + } + + printf("mode:"); + + switch (sa->flags) { + case IP4_TUNNEL: + printf("IP4Tunnel "); + uint32_t_to_char(sa->src.ip.ip4, &a, &b, &c, &d); + printf("%hhu.%hhu.%hhu.%hhu ", d, c, b, a); + uint32_t_to_char(sa->dst.ip.ip4, &a, &b, &c, &d); + printf("%hhu.%hhu.%hhu.%hhu", d, c, b, a); + break; + case IP6_TUNNEL: + printf("IP6Tunnel "); + for (i = 0; i < 16; i++) { + if (i % 2 && i != 15) + printf("%.2x:", sa->src.ip.ip6.ip6_b[i]); + else + printf("%.2x", sa->src.ip.ip6.ip6_b[i]); + } + printf(" "); + for (i = 0; i < 16; i++) { + if (i % 2 && i != 15) + printf("%.2x:", sa->dst.ip.ip6.ip6_b[i]); + else + printf("%.2x", sa->dst.ip.ip6.ip6_b[i]); + } + break; + case TRANSPORT: + printf("Transport"); + break; + } + printf("\n"); +} struct sa_ctx { struct ipsec_sa sa[IPSEC_SA_MAX_ENTRIES]; - struct { - struct rte_crypto_sym_xform a; - struct rte_crypto_sym_xform b; + union { + struct { + struct rte_crypto_sym_xform a; + struct rte_crypto_sym_xform b; + }; } xf[IPSEC_SA_MAX_ENTRIES]; }; @@ -322,12 +744,40 @@ sa_create(const char *name, int32_t socket_id) return sa_ctx; } +static int +check_eth_dev_caps(uint16_t portid, uint32_t inbound) +{ + struct rte_eth_dev_info dev_info; + + rte_eth_dev_info_get(portid, &dev_info); + + if (inbound) { + if ((dev_info.rx_offload_capa & + DEV_RX_OFFLOAD_SECURITY) == 0) { + RTE_LOG(WARNING, PORT, + "hardware RX IPSec offload is not supported\n"); + return -EINVAL; + } + + } else { /* outbound */ + if ((dev_info.tx_offload_capa & + DEV_TX_OFFLOAD_SECURITY) == 0) { + RTE_LOG(WARNING, PORT, + "hardware TX IPSec offload is not supported\n"); + return -EINVAL; + } + } + return 0; +} + + static int sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[], uint32_t nb_entries, uint32_t inbound) { struct ipsec_sa *sa; uint32_t i, idx; + uint16_t iv_length; for (i = 0; i < nb_entries; i++) { idx = SPI2IDX(entries[i].spi); @@ -340,32 +790,110 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[], *sa = entries[i]; sa->seq = 0; + if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL || + sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) { + if (check_eth_dev_caps(sa->portid, inbound)) + return -EINVAL; + } + + sa->direction = (inbound == 1) ? + RTE_SECURITY_IPSEC_SA_DIR_INGRESS : + RTE_SECURITY_IPSEC_SA_DIR_EGRESS; + switch (sa->flags) { case IP4_TUNNEL: sa->src.ip.ip4 = rte_cpu_to_be_32(sa->src.ip.ip4); sa->dst.ip.ip4 = rte_cpu_to_be_32(sa->dst.ip.ip4); } - if (inbound) { - if (sa->cipher_algo == RTE_CRYPTO_CIPHER_NULL) { - sa_ctx->xf[idx].a = null_auth_xf; - sa_ctx->xf[idx].b = null_cipher_xf; - } else { - sa_ctx->xf[idx].a = sha1hmac_verify_xf; - sa_ctx->xf[idx].b = aescbc_dec_xf; + if (sa->aead_algo == RTE_CRYPTO_AEAD_AES_GCM) { + iv_length = 16; + + sa_ctx->xf[idx].a.type = RTE_CRYPTO_SYM_XFORM_AEAD; + sa_ctx->xf[idx].a.aead.algo = sa->aead_algo; + sa_ctx->xf[idx].a.aead.key.data = sa->cipher_key; + sa_ctx->xf[idx].a.aead.key.length = + sa->cipher_key_len; + sa_ctx->xf[idx].a.aead.op = (inbound == 1) ? + RTE_CRYPTO_AEAD_OP_DECRYPT : + RTE_CRYPTO_AEAD_OP_ENCRYPT; + sa_ctx->xf[idx].a.next = NULL; + sa_ctx->xf[idx].a.aead.iv.offset = IV_OFFSET; + sa_ctx->xf[idx].a.aead.iv.length = iv_length; + sa_ctx->xf[idx].a.aead.aad_length = + sa->aad_len; + sa_ctx->xf[idx].a.aead.digest_length = + sa->digest_len; + + sa->xforms = &sa_ctx->xf[idx].a; + + print_one_sa_rule(sa, inbound); + } else { + switch (sa->cipher_algo) { + case RTE_CRYPTO_CIPHER_NULL: + case RTE_CRYPTO_CIPHER_AES_CBC: + iv_length = sa->iv_len; + break; + case RTE_CRYPTO_CIPHER_AES_CTR: + iv_length = 16; + break; + default: + RTE_LOG(ERR, IPSEC_ESP, + "unsupported cipher algorithm %u\n", + sa->cipher_algo); + return -EINVAL; } - } else { /* outbound */ - if (sa->cipher_algo == RTE_CRYPTO_CIPHER_NULL) { - sa_ctx->xf[idx].a = null_cipher_xf; - sa_ctx->xf[idx].b = null_auth_xf; - } else { - sa_ctx->xf[idx].a = aescbc_enc_xf; - sa_ctx->xf[idx].b = sha1hmac_gen_xf; + + if (inbound) { + sa_ctx->xf[idx].b.type = RTE_CRYPTO_SYM_XFORM_CIPHER; + sa_ctx->xf[idx].b.cipher.algo = sa->cipher_algo; + sa_ctx->xf[idx].b.cipher.key.data = sa->cipher_key; + sa_ctx->xf[idx].b.cipher.key.length = + sa->cipher_key_len; + sa_ctx->xf[idx].b.cipher.op = + RTE_CRYPTO_CIPHER_OP_DECRYPT; + sa_ctx->xf[idx].b.next = NULL; + sa_ctx->xf[idx].b.cipher.iv.offset = IV_OFFSET; + sa_ctx->xf[idx].b.cipher.iv.length = iv_length; + + sa_ctx->xf[idx].a.type = RTE_CRYPTO_SYM_XFORM_AUTH; + sa_ctx->xf[idx].a.auth.algo = sa->auth_algo; + sa_ctx->xf[idx].a.auth.key.data = sa->auth_key; + sa_ctx->xf[idx].a.auth.key.length = + sa->auth_key_len; + sa_ctx->xf[idx].a.auth.digest_length = + sa->digest_len; + sa_ctx->xf[idx].a.auth.op = + RTE_CRYPTO_AUTH_OP_VERIFY; + } else { /* outbound */ + sa_ctx->xf[idx].a.type = RTE_CRYPTO_SYM_XFORM_CIPHER; + sa_ctx->xf[idx].a.cipher.algo = sa->cipher_algo; + sa_ctx->xf[idx].a.cipher.key.data = sa->cipher_key; + sa_ctx->xf[idx].a.cipher.key.length = + sa->cipher_key_len; + sa_ctx->xf[idx].a.cipher.op = + RTE_CRYPTO_CIPHER_OP_ENCRYPT; + sa_ctx->xf[idx].a.next = NULL; + sa_ctx->xf[idx].a.cipher.iv.offset = IV_OFFSET; + sa_ctx->xf[idx].a.cipher.iv.length = iv_length; + + sa_ctx->xf[idx].b.type = RTE_CRYPTO_SYM_XFORM_AUTH; + sa_ctx->xf[idx].b.auth.algo = sa->auth_algo; + sa_ctx->xf[idx].b.auth.key.data = sa->auth_key; + sa_ctx->xf[idx].b.auth.key.length = + sa->auth_key_len; + sa_ctx->xf[idx].b.auth.digest_length = + sa->digest_len; + sa_ctx->xf[idx].b.auth.op = + RTE_CRYPTO_AUTH_OP_GENERATE; } + + sa_ctx->xf[idx].a.next = &sa_ctx->xf[idx].b; + sa_ctx->xf[idx].b.next = NULL; + sa->xforms = &sa_ctx->xf[idx].a; + + print_one_sa_rule(sa, inbound); } - sa_ctx->xf[idx].a.next = &sa_ctx->xf[idx].b; - sa_ctx->xf[idx].b.next = NULL; - sa->xforms = &sa_ctx->xf[idx].a; } return 0; @@ -386,10 +914,8 @@ sa_in_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[], } void -sa_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t ep) +sa_init(struct socket_ctx *ctx, int32_t socket_id) { - const struct ipsec_sa *sa_out_entries, *sa_in_entries; - uint32_t nb_out_entries, nb_in_entries; const char *name; if (ctx == NULL) @@ -403,35 +929,30 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t ep) rte_exit(EXIT_FAILURE, "Outbound SA DB for socket %u already " "initialized\n", socket_id); - if (ep == 0) { - sa_out_entries = sa_out; - nb_out_entries = RTE_DIM(sa_out); - sa_in_entries = sa_in; - nb_in_entries = RTE_DIM(sa_in); - } else if (ep == 1) { - sa_out_entries = sa_in; - nb_out_entries = RTE_DIM(sa_in); - sa_in_entries = sa_out; - nb_in_entries = RTE_DIM(sa_out); + if (nb_sa_in > 0) { + name = "sa_in"; + ctx->sa_in = sa_create(name, socket_id); + if (ctx->sa_in == NULL) + rte_exit(EXIT_FAILURE, "Error [%d] creating SA " + "context %s in socket %d\n", rte_errno, + name, socket_id); + + sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in); } else - rte_exit(EXIT_FAILURE, "Invalid EP value %u. " - "Only 0 or 1 supported.\n", ep); + RTE_LOG(WARNING, IPSEC, "No SA Inbound rule specified\n"); - name = "sa_in"; - ctx->sa_in = sa_create(name, socket_id); - if (ctx->sa_in == NULL) - rte_exit(EXIT_FAILURE, "Error [%d] creating SA context %s " - "in socket %d\n", rte_errno, name, socket_id); + if (nb_sa_out > 0) { + name = "sa_out"; + ctx->sa_out = sa_create(name, socket_id); + if (ctx->sa_out == NULL) + rte_exit(EXIT_FAILURE, "Error [%d] creating SA " + "context %s in socket %d\n", rte_errno, + name, socket_id); - name = "sa_out"; - ctx->sa_out = sa_create(name, socket_id); - if (ctx->sa_out == NULL) - rte_exit(EXIT_FAILURE, "Error [%d] creating SA context %s " - "in socket %d\n", rte_errno, name, socket_id); - - sa_in_add_rules(ctx->sa_in, sa_in_entries, nb_in_entries); - - sa_out_add_rules(ctx->sa_out, sa_out_entries, nb_out_entries); + sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out); + } else + RTE_LOG(WARNING, IPSEC, "No SA Outbound rule " + "specified\n"); } int diff --git a/dpdk/examples/ipsec-secgw/sp4.c b/dpdk/examples/ipsec-secgw/sp4.c index 9c4b256b..38c72a92 100644 --- a/dpdk/examples/ipsec-secgw/sp4.c +++ b/dpdk/examples/ipsec-secgw/sp4.c @@ -42,8 +42,9 @@ #include #include "ipsec.h" +#include "parser.h" -#define MAX_ACL_RULE_NUM 1000 +#define MAX_ACL_RULE_NUM 1024 /* * Rule and trace formats definitions. @@ -113,211 +114,306 @@ struct rte_acl_field_def ip4_defs[NUM_FIELDS_IPV4] = { RTE_ACL_RULE_DEF(acl4_rules, RTE_DIM(ip4_defs)); -const struct acl4_rules acl4_rules_out[] = { - { - .data = {.userdata = PROTECT(5), .category_mask = 1, .priority = 1}, - /* destination IPv4 */ - .field[2] = {.value.u32 = IPv4(192, 168, 105, 0), - .mask_range.u32 = 24,}, - /* source port */ - .field[3] = {.value.u16 = 0, .mask_range.u16 = 0xffff,}, - /* destination port */ - .field[4] = {.value.u16 = 0, .mask_range.u16 = 0xffff,} - }, - { - .data = {.userdata = PROTECT(6), .category_mask = 1, .priority = 1}, - /* destination IPv4 */ - .field[2] = {.value.u32 = IPv4(192, 168, 106, 0), - .mask_range.u32 = 24,}, - /* source port */ - .field[3] = {.value.u16 = 0, .mask_range.u16 = 0xffff,}, - /* destination port */ - .field[4] = {.value.u16 = 0, .mask_range.u16 = 0xffff,} - }, - { - .data = {.userdata = PROTECT(10), .category_mask = 1, .priority = 1}, - /* destination IPv4 */ - .field[2] = {.value.u32 = IPv4(192, 168, 175, 0), - .mask_range.u32 = 24,}, - /* source port */ - .field[3] = {.value.u16 = 0, .mask_range.u16 = 0xffff,}, - /* destination port */ - .field[4] = {.value.u16 = 0, .mask_range.u16 = 0xffff,} - }, - { - .data = {.userdata = PROTECT(11), .category_mask = 1, .priority = 1}, - /* destination IPv4 */ - .field[2] = {.value.u32 = IPv4(192, 168, 176, 0), - .mask_range.u32 = 24,}, - /* source port */ - .field[3] = {.value.u16 = 0, .mask_range.u16 = 0xffff,}, - /* destination port */ - .field[4] = {.value.u16 = 0, .mask_range.u16 = 0xffff,} - }, - { - .data = {.userdata = PROTECT(15), .category_mask = 1, .priority = 1}, - /* destination IPv4 */ - .field[2] = {.value.u32 = IPv4(192, 168, 200, 0), - .mask_range.u32 = 24,}, - /* source port */ - .field[3] = {.value.u16 = 0, .mask_range.u16 = 0xffff,}, - /* destination port */ - .field[4] = {.value.u16 = 0, .mask_range.u16 = 0xffff,} - }, - { - .data = {.userdata = PROTECT(16), .category_mask = 1, .priority = 1}, - /* destination IPv4 */ - .field[2] = {.value.u32 = IPv4(192, 168, 201, 0), - .mask_range.u32 = 24,}, - /* source port */ - .field[3] = {.value.u16 = 0, .mask_range.u16 = 0xffff,}, - /* destination port */ - .field[4] = {.value.u16 = 0, .mask_range.u16 = 0xffff,} - }, - { - .data = {.userdata = PROTECT(25), .category_mask = 1, .priority = 1}, - /* destination IPv4 */ - .field[2] = {.value.u32 = IPv4(192, 168, 55, 0), - .mask_range.u32 = 24,}, - /* source port */ - .field[3] = {.value.u16 = 0, .mask_range.u16 = 0xffff,}, - /* destination port */ - .field[4] = {.value.u16 = 0, .mask_range.u16 = 0xffff,} - }, - { - .data = {.userdata = PROTECT(26), .category_mask = 1, .priority = 1}, - /* destination IPv4 */ - .field[2] = {.value.u32 = IPv4(192, 168, 56, 0), - .mask_range.u32 = 24,}, - /* source port */ - .field[3] = {.value.u16 = 0, .mask_range.u16 = 0xffff,}, - /* destination port */ - .field[4] = {.value.u16 = 0, .mask_range.u16 = 0xffff,} - }, - { - .data = {.userdata = BYPASS, .category_mask = 1, .priority = 1}, - /* destination IPv4 */ - .field[2] = {.value.u32 = IPv4(192, 168, 240, 0), - .mask_range.u32 = 24,}, - /* source port */ - .field[3] = {.value.u16 = 0, .mask_range.u16 = 0xffff,}, - /* destination port */ - .field[4] = {.value.u16 = 0, .mask_range.u16 = 0xffff,} - }, - { - .data = {.userdata = BYPASS, .category_mask = 1, .priority = 1}, - /* destination IPv4 */ - .field[2] = {.value.u32 = IPv4(192, 168, 241, 0), - .mask_range.u32 = 24,}, - /* source port */ - .field[3] = {.value.u16 = 0, .mask_range.u16 = 0xffff,}, - /* destination port */ - .field[4] = {.value.u16 = 0, .mask_range.u16 = 0xffff,} - } -}; +struct acl4_rules acl4_rules_out[MAX_ACL_RULE_NUM]; +uint32_t nb_acl4_rules_out; -const struct acl4_rules acl4_rules_in[] = { - { - .data = {.userdata = PROTECT(105), .category_mask = 1, .priority = 1}, - /* destination IPv4 */ - .field[2] = {.value.u32 = IPv4(192, 168, 115, 0), - .mask_range.u32 = 24,}, - /* source port */ - .field[3] = {.value.u16 = 0, .mask_range.u16 = 0xffff,}, - /* destination port */ - .field[4] = {.value.u16 = 0, .mask_range.u16 = 0xffff,} - }, - { - .data = {.userdata = PROTECT(106), .category_mask = 1, .priority = 1}, - /* destination IPv4 */ - .field[2] = {.value.u32 = IPv4(192, 168, 116, 0), - .mask_range.u32 = 24,}, - /* source port */ - .field[3] = {.value.u16 = 0, .mask_range.u16 = 0xffff,}, - /* destination port */ - .field[4] = {.value.u16 = 0, .mask_range.u16 = 0xffff,} - }, - { - .data = {.userdata = PROTECT(110), .category_mask = 1, .priority = 1}, - /* destination IPv4 */ - .field[2] = {.value.u32 = IPv4(192, 168, 185, 0), - .mask_range.u32 = 24,}, - /* source port */ - .field[3] = {.value.u16 = 0, .mask_range.u16 = 0xffff,}, - /* destination port */ - .field[4] = {.value.u16 = 0, .mask_range.u16 = 0xffff,} - }, - { - .data = {.userdata = PROTECT(111), .category_mask = 1, .priority = 1}, - /* destination IPv4 */ - .field[2] = {.value.u32 = IPv4(192, 168, 186, 0), - .mask_range.u32 = 24,}, - /* source port */ - .field[3] = {.value.u16 = 0, .mask_range.u16 = 0xffff,}, - /* destination port */ - .field[4] = {.value.u16 = 0, .mask_range.u16 = 0xffff,} - }, - { - .data = {.userdata = PROTECT(115), .category_mask = 1, .priority = 1}, - /* destination IPv4 */ - .field[2] = {.value.u32 = IPv4(192, 168, 210, 0), - .mask_range.u32 = 24,}, - /* source port */ - .field[3] = {.value.u16 = 0, .mask_range.u16 = 0xffff,}, - /* destination port */ - .field[4] = {.value.u16 = 0, .mask_range.u16 = 0xffff,} - }, - { - .data = {.userdata = PROTECT(116), .category_mask = 1, .priority = 1}, - /* destination IPv4 */ - .field[2] = {.value.u32 = IPv4(192, 168, 211, 0), - .mask_range.u32 = 24,}, - /* source port */ - .field[3] = {.value.u16 = 0, .mask_range.u16 = 0xffff,}, - /* destination port */ - .field[4] = {.value.u16 = 0, .mask_range.u16 = 0xffff,} - }, - { - .data = {.userdata = PROTECT(125), .category_mask = 1, .priority = 1}, - /* destination IPv4 */ - .field[2] = {.value.u32 = IPv4(192, 168, 65, 0), - .mask_range.u32 = 24,}, - /* source port */ - .field[3] = {.value.u16 = 0, .mask_range.u16 = 0xffff,}, - /* destination port */ - .field[4] = {.value.u16 = 0, .mask_range.u16 = 0xffff,} - }, - { - .data = {.userdata = PROTECT(126), .category_mask = 1, .priority = 1}, - /* destination IPv4 */ - .field[2] = {.value.u32 = IPv4(192, 168, 66, 0), - .mask_range.u32 = 24,}, - /* source port */ - .field[3] = {.value.u16 = 0, .mask_range.u16 = 0xffff,}, - /* destination port */ - .field[4] = {.value.u16 = 0, .mask_range.u16 = 0xffff,} - }, - { - .data = {.userdata = BYPASS, .category_mask = 1, .priority = 1}, - /* destination IPv4 */ - .field[2] = {.value.u32 = IPv4(192, 168, 245, 0), - .mask_range.u32 = 24,}, - /* source port */ - .field[3] = {.value.u16 = 0, .mask_range.u16 = 0xffff,}, - /* destination port */ - .field[4] = {.value.u16 = 0, .mask_range.u16 = 0xffff,} - }, - { - .data = {.userdata = BYPASS, .category_mask = 1, .priority = 1}, - /* destination IPv4 */ - .field[2] = {.value.u32 = IPv4(192, 168, 246, 0), - .mask_range.u32 = 24,}, - /* source port */ - .field[3] = {.value.u16 = 0, .mask_range.u16 = 0xffff,}, - /* destination port */ - .field[4] = {.value.u16 = 0, .mask_range.u16 = 0xffff,} +struct acl4_rules acl4_rules_in[MAX_ACL_RULE_NUM]; +uint32_t nb_acl4_rules_in; + +void +parse_sp4_tokens(char **tokens, uint32_t n_tokens, + struct parse_status *status) +{ + struct acl4_rules *rule_ipv4 = NULL; + + uint32_t *ri = NULL; /* rule index */ + uint32_t ti = 0; /* token index */ + + uint32_t esp_p = 0; + uint32_t protect_p = 0; + uint32_t bypass_p = 0; + uint32_t discard_p = 0; + uint32_t pri_p = 0; + uint32_t src_p = 0; + uint32_t dst_p = 0; + uint32_t proto_p = 0; + uint32_t sport_p = 0; + uint32_t dport_p = 0; + + if (strcmp(tokens[1], "in") == 0) { + ri = &nb_acl4_rules_in; + + APP_CHECK(*ri <= MAX_ACL_RULE_NUM - 1, status, + "too many sp rules, abort insertion\n"); + if (status->status < 0) + return; + + rule_ipv4 = &acl4_rules_in[*ri]; + + } else if (strcmp(tokens[1], "out") == 0) { + ri = &nb_acl4_rules_out; + + APP_CHECK(*ri <= MAX_ACL_RULE_NUM - 1, status, + "too many sp rules, abort insertion\n"); + if (status->status < 0) + return; + + rule_ipv4 = &acl4_rules_out[*ri]; + } else { + APP_CHECK(0, status, "unrecognized input \"%s\", expect" + " \"in\" or \"out\"\n", tokens[ti]); + return; } -}; + + rule_ipv4->data.category_mask = 1; + + for (ti = 2; ti < n_tokens; ti++) { + if (strcmp(tokens[ti], "esp") == 0) { + /* currently do nothing */ + APP_CHECK_PRESENCE(esp_p, tokens[ti], status); + if (status->status < 0) + return; + esp_p = 1; + continue; + } + + if (strcmp(tokens[ti], "protect") == 0) { + APP_CHECK_PRESENCE(protect_p, tokens[ti], status); + if (status->status < 0) + return; + APP_CHECK(bypass_p == 0, status, "conflict item " + "between \"%s\" and \"%s\"", tokens[ti], + "bypass"); + if (status->status < 0) + return; + APP_CHECK(discard_p == 0, status, "conflict item " + "between \"%s\" and \"%s\"", tokens[ti], + "discard"); + if (status->status < 0) + return; + INCREMENT_TOKEN_INDEX(ti, n_tokens, status); + if (status->status < 0) + return; + APP_CHECK_TOKEN_IS_NUM(tokens, ti, status); + if (status->status < 0) + return; + + rule_ipv4->data.userdata = + PROTECT(atoi(tokens[ti])); + + protect_p = 1; + continue; + } + + if (strcmp(tokens[ti], "bypass") == 0) { + APP_CHECK_PRESENCE(bypass_p, tokens[ti], status); + if (status->status < 0) + return; + APP_CHECK(protect_p == 0, status, "conflict item " + "between \"%s\" and \"%s\"", tokens[ti], + "protect"); + if (status->status < 0) + return; + APP_CHECK(discard_p == 0, status, "conflict item " + "between \"%s\" and \"%s\"", tokens[ti], + "discard"); + if (status->status < 0) + return; + + rule_ipv4->data.userdata = BYPASS; + + bypass_p = 1; + continue; + } + + if (strcmp(tokens[ti], "discard") == 0) { + APP_CHECK_PRESENCE(discard_p, tokens[ti], status); + if (status->status < 0) + return; + APP_CHECK(protect_p == 0, status, "conflict item " + "between \"%s\" and \"%s\"", tokens[ti], + "protect"); + if (status->status < 0) + return; + APP_CHECK(bypass_p == 0, status, "conflict item " + "between \"%s\" and \"%s\"", tokens[ti], + "discard"); + if (status->status < 0) + return; + + rule_ipv4->data.userdata = DISCARD; + + discard_p = 1; + continue; + } + + if (strcmp(tokens[ti], "pri") == 0) { + APP_CHECK_PRESENCE(pri_p, tokens[ti], status); + if (status->status < 0) + return; + INCREMENT_TOKEN_INDEX(ti, n_tokens, status); + if (status->status < 0) + return; + APP_CHECK_TOKEN_IS_NUM(tokens, ti, status); + if (status->status < 0) + return; + + rule_ipv4->data.priority = atoi(tokens[ti]); + + pri_p = 1; + continue; + } + + if (strcmp(tokens[ti], "src") == 0) { + struct in_addr ip; + uint32_t depth; + + APP_CHECK_PRESENCE(src_p, tokens[ti], status); + if (status->status < 0) + return; + INCREMENT_TOKEN_INDEX(ti, n_tokens, status); + if (status->status < 0) + return; + + APP_CHECK(parse_ipv4_addr(tokens[ti], &ip, + &depth) == 0, status, "unrecognized " + "input \"%s\", expect valid ipv4 addr", + tokens[ti]); + if (status->status < 0) + return; + + rule_ipv4->field[1].value.u32 = + rte_bswap32(ip.s_addr); + rule_ipv4->field[1].mask_range.u32 = + depth; + + src_p = 1; + continue; + } + + if (strcmp(tokens[ti], "dst") == 0) { + struct in_addr ip; + uint32_t depth; + + APP_CHECK_PRESENCE(dst_p, tokens[ti], status); + if (status->status < 0) + return; + INCREMENT_TOKEN_INDEX(ti, n_tokens, status); + if (status->status < 0) + return; + APP_CHECK(parse_ipv4_addr(tokens[ti], &ip, + &depth) == 0, status, "unrecognized " + "input \"%s\", expect valid ipv4 addr", + tokens[ti]); + if (status->status < 0) + return; + + rule_ipv4->field[2].value.u32 = + rte_bswap32(ip.s_addr); + rule_ipv4->field[2].mask_range.u32 = + depth; + + dst_p = 1; + continue; + } + + if (strcmp(tokens[ti], "proto") == 0) { + uint16_t low, high; + + APP_CHECK_PRESENCE(proto_p, tokens[ti], status); + if (status->status < 0) + return; + INCREMENT_TOKEN_INDEX(ti, n_tokens, status); + if (status->status < 0) + return; + + APP_CHECK(parse_range(tokens[ti], &low, &high) + == 0, status, "unrecognized input \"%s\"" + ", expect \"from:to\"", tokens[ti]); + if (status->status < 0) + return; + APP_CHECK(low <= 0xff, status, "proto low " + "over-limit"); + if (status->status < 0) + return; + APP_CHECK(high <= 0xff, status, "proto high " + "over-limit"); + if (status->status < 0) + return; + + rule_ipv4->field[0].value.u8 = (uint8_t)low; + rule_ipv4->field[0].mask_range.u8 = (uint8_t)high; + + proto_p = 1; + continue; + } + + if (strcmp(tokens[ti], "sport") == 0) { + uint16_t port_low, port_high; + + APP_CHECK_PRESENCE(sport_p, tokens[ti], status); + if (status->status < 0) + return; + INCREMENT_TOKEN_INDEX(ti, n_tokens, status); + if (status->status < 0) + return; + + APP_CHECK(parse_range(tokens[ti], &port_low, + &port_high) == 0, status, "unrecognized " + "input \"%s\", expect \"port_from:" + "port_to\"", tokens[ti]); + if (status->status < 0) + return; + + rule_ipv4->field[3].value.u16 = port_low; + rule_ipv4->field[3].mask_range.u16 = port_high; + + sport_p = 1; + continue; + } + + if (strcmp(tokens[ti], "dport") == 0) { + uint16_t port_low, port_high; + + APP_CHECK_PRESENCE(dport_p, tokens[ti], status); + if (status->status < 0) + return; + INCREMENT_TOKEN_INDEX(ti, n_tokens, status); + if (status->status < 0) + return; + + APP_CHECK(parse_range(tokens[ti], &port_low, + &port_high) == 0, status, "unrecognized " + "input \"%s\", expect \"port_from:" + "port_to\"", tokens[ti]); + if (status->status < 0) + return; + + rule_ipv4->field[4].value.u16 = port_low; + rule_ipv4->field[4].mask_range.u16 = port_high; + + dport_p = 1; + continue; + } + + /* unrecognizeable input */ + APP_CHECK(0, status, "unrecognized input \"%s\"", + tokens[ti]); + return; + } + + /* check if argument(s) are missing */ + APP_CHECK(esp_p == 1, status, "missing argument \"esp\""); + if (status->status < 0) + return; + + APP_CHECK(protect_p | bypass_p | discard_p, status, "missing " + "argument \"protect\", \"bypass\", or \"discard\""); + if (status->status < 0) + return; + + *ri = *ri + 1; +} static void print_one_ip4_rule(const struct acl4_rules *rule, int32_t extra) @@ -406,11 +502,9 @@ acl4_init(const char *name, int32_t socketid, const struct acl4_rules *rules, } void -sp4_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t ep) +sp4_init(struct socket_ctx *ctx, int32_t socket_id) { const char *name; - const struct acl4_rules *rules_out, *rules_in; - uint32_t nb_out_rules, nb_in_rules; if (ctx == NULL) rte_exit(EXIT_FAILURE, "NULL context.\n"); @@ -423,25 +517,19 @@ sp4_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t ep) rte_exit(EXIT_FAILURE, "Outbound SP DB for socket %u already " "initialized\n", socket_id); - if (ep == 0) { - rules_out = acl4_rules_out; - nb_out_rules = RTE_DIM(acl4_rules_out); - rules_in = acl4_rules_in; - nb_in_rules = RTE_DIM(acl4_rules_in); - } else if (ep == 1) { - rules_out = acl4_rules_in; - nb_out_rules = RTE_DIM(acl4_rules_in); - rules_in = acl4_rules_out; - nb_in_rules = RTE_DIM(acl4_rules_out); + if (nb_acl4_rules_in > 0) { + name = "sp_ip4_in"; + ctx->sp_ip4_in = (struct sp_ctx *)acl4_init(name, + socket_id, acl4_rules_in, nb_acl4_rules_in); } else - rte_exit(EXIT_FAILURE, "Invalid EP value %u. " - "Only 0 or 1 supported.\n", ep); + RTE_LOG(WARNING, IPSEC, "No IPv4 SP Inbound rule " + "specified\n"); - name = "sp_ip4_in"; - ctx->sp_ip4_in = (struct sp_ctx *)acl4_init(name, socket_id, - rules_in, nb_in_rules); - - name = "sp_ip4_out"; - ctx->sp_ip4_out = (struct sp_ctx *)acl4_init(name, socket_id, - rules_out, nb_out_rules); + if (nb_acl4_rules_out > 0) { + name = "sp_ip4_out"; + ctx->sp_ip4_out = (struct sp_ctx *)acl4_init(name, + socket_id, acl4_rules_out, nb_acl4_rules_out); + } else + RTE_LOG(WARNING, IPSEC, "No IPv4 SP Outbound rule " + "specified\n"); } diff --git a/dpdk/examples/ipsec-secgw/sp6.c b/dpdk/examples/ipsec-secgw/sp6.c index 1dda11a4..62fb492c 100644 --- a/dpdk/examples/ipsec-secgw/sp6.c +++ b/dpdk/examples/ipsec-secgw/sp6.c @@ -42,8 +42,9 @@ #include #include "ipsec.h" +#include "parser.h" -#define MAX_ACL_RULE_NUM 1000 +#define MAX_ACL_RULE_NUM 1024 enum { IP6_PROTO, @@ -144,155 +145,363 @@ struct rte_acl_field_def ip6_defs[IP6_NUM] = { RTE_ACL_RULE_DEF(acl6_rules, RTE_DIM(ip6_defs)); -const struct acl6_rules acl6_rules_out[] = { - { - .data = {.userdata = PROTECT(5), .category_mask = 1, .priority = 1}, - /* destination IPv6 */ - .field[5] = {.value.u32 = 0x0, .mask_range.u32 = 32,}, - .field[6] = {.value.u32 = 0x0, .mask_range.u32 = 32,}, - .field[7] = {.value.u32 = 0x55555555, .mask_range.u32 = 32,}, - .field[8] = {.value.u32 = 0x0, .mask_range.u32 = 0,}, - /* source port */ - .field[9] = {.value.u16 = 0, .mask_range.u16 = 0xffff,}, - /* destination port */ - .field[10] = {.value.u16 = 0, .mask_range.u16 = 0xffff,} - }, - { - .data = {.userdata = PROTECT(6), .category_mask = 1, .priority = 1}, - /* destination IPv6 */ - .field[5] = {.value.u32 = 0x0, .mask_range.u32 = 32,}, - .field[6] = {.value.u32 = 0x0, .mask_range.u32 = 32,}, - .field[7] = {.value.u32 = 0x66666666, .mask_range.u32 = 32,}, - .field[8] = {.value.u32 = 0x0, .mask_range.u32 = 0,}, - /* source port */ - .field[9] = {.value.u16 = 0, .mask_range.u16 = 0xffff,}, - /* destination port */ - .field[10] = {.value.u16 = 0, .mask_range.u16 = 0xffff,} - }, - { - .data = {.userdata = PROTECT(10), .category_mask = 1, .priority = 1}, - /* destination IPv6 */ - .field[5] = {.value.u32 = 0x0, .mask_range.u32 = 32,}, - .field[6] = {.value.u32 = 0x11111111, .mask_range.u32 = 32,}, - .field[7] = {.value.u32 = 0x00000000, .mask_range.u32 = 32,}, - .field[8] = {.value.u32 = 0x0, .mask_range.u32 = 0,}, - /* source port */ - .field[9] = {.value.u16 = 0, .mask_range.u16 = 0xffff,}, - /* destination port */ - .field[10] = {.value.u16 = 0, .mask_range.u16 = 0xffff,} - }, - { - .data = {.userdata = PROTECT(11), .category_mask = 1, .priority = 1}, - /* destination IPv6 */ - .field[5] = {.value.u32 = 0x0, .mask_range.u32 = 32,}, - .field[6] = {.value.u32 = 0x11111111, .mask_range.u32 = 32,}, - .field[7] = {.value.u32 = 0x11111111, .mask_range.u32 = 32,}, - .field[8] = {.value.u32 = 0x0, .mask_range.u32 = 0,}, - /* source port */ - .field[9] = {.value.u16 = 0, .mask_range.u16 = 0xffff,}, - /* destination port */ - .field[10] = {.value.u16 = 0, .mask_range.u16 = 0xffff,} - }, - { - .data = {.userdata = PROTECT(25), .category_mask = 1, .priority = 1}, - /* destination IPv6 */ - .field[5] = {.value.u32 = 0x0, .mask_range.u32 = 32,}, - .field[6] = {.value.u32 = 0x0, .mask_range.u32 = 32,}, - .field[7] = {.value.u32 = 0xaaaaaaaa, .mask_range.u32 = 32,}, - .field[8] = {.value.u32 = 0x0, .mask_range.u32 = 0,}, - /* source port */ - .field[9] = {.value.u16 = 0, .mask_range.u16 = 0xffff,}, - /* destination port */ - .field[10] = {.value.u16 = 0, .mask_range.u16 = 0xffff,} - }, - { - .data = {.userdata = PROTECT(26), .category_mask = 1, .priority = 1}, - /* destination IPv6 */ - .field[5] = {.value.u32 = 0x0, .mask_range.u32 = 32,}, - .field[6] = {.value.u32 = 0x0, .mask_range.u32 = 32,}, - .field[7] = {.value.u32 = 0xbbbbbbbb, .mask_range.u32 = 32,}, - .field[8] = {.value.u32 = 0x0, .mask_range.u32 = 0,}, - /* source port */ - .field[9] = {.value.u16 = 0, .mask_range.u16 = 0xffff,}, - /* destination port */ - .field[10] = {.value.u16 = 0, .mask_range.u16 = 0xffff,} - } -}; +struct acl6_rules acl6_rules_out[MAX_ACL_RULE_NUM]; +uint32_t nb_acl6_rules_out; -const struct acl6_rules acl6_rules_in[] = { - { - .data = {.userdata = PROTECT(15), .category_mask = 1, .priority = 1}, - /* destination IPv6 */ - .field[5] = {.value.u32 = 0xffff0000, .mask_range.u32 = 32,}, - .field[6] = {.value.u32 = 0x0, .mask_range.u32 = 32,}, - .field[7] = {.value.u32 = 0x55555555, .mask_range.u32 = 32,}, - .field[8] = {.value.u32 = 0x0, .mask_range.u32 = 0,}, - /* source port */ - .field[9] = {.value.u16 = 0, .mask_range.u16 = 0xffff,}, - /* destination port */ - .field[10] = {.value.u16 = 0, .mask_range.u16 = 0xffff,} - }, - { - .data = {.userdata = PROTECT(16), .category_mask = 1, .priority = 1}, - /* destination IPv6 */ - .field[5] = {.value.u32 = 0xffff0000, .mask_range.u32 = 32,}, - .field[6] = {.value.u32 = 0x0, .mask_range.u32 = 32,}, - .field[7] = {.value.u32 = 0x66666666, .mask_range.u32 = 32,}, - .field[8] = {.value.u32 = 0x0, .mask_range.u32 = 0,}, - /* source port */ - .field[9] = {.value.u16 = 0, .mask_range.u16 = 0xffff,}, - /* destination port */ - .field[10] = {.value.u16 = 0, .mask_range.u16 = 0xffff,} - }, - { - .data = {.userdata = PROTECT(110), .category_mask = 1, .priority = 1}, - /* destination IPv6 */ - .field[5] = {.value.u32 = 0xffff0000, .mask_range.u32 = 32,}, - .field[6] = {.value.u32 = 0x11111111, .mask_range.u32 = 32,}, - .field[7] = {.value.u32 = 0x00000000, .mask_range.u32 = 32,}, - .field[8] = {.value.u32 = 0x0, .mask_range.u32 = 0,}, - /* source port */ - .field[9] = {.value.u16 = 0, .mask_range.u16 = 0xffff,}, - /* destination port */ - .field[10] = {.value.u16 = 0, .mask_range.u16 = 0xffff,} - }, - { - .data = {.userdata = PROTECT(111), .category_mask = 1, .priority = 1}, - /* destination IPv6 */ - .field[5] = {.value.u32 = 0xffff0000, .mask_range.u32 = 32,}, - .field[6] = {.value.u32 = 0x11111111, .mask_range.u32 = 32,}, - .field[7] = {.value.u32 = 0x11111111, .mask_range.u32 = 32,}, - .field[8] = {.value.u32 = 0x0, .mask_range.u32 = 0,}, - /* source port */ - .field[9] = {.value.u16 = 0, .mask_range.u16 = 0xffff,}, - /* destination port */ - .field[10] = {.value.u16 = 0, .mask_range.u16 = 0xffff,} - }, - { - .data = {.userdata = PROTECT(125), .category_mask = 1, .priority = 1}, - /* destination IPv6 */ - .field[5] = {.value.u32 = 0xffff0000, .mask_range.u32 = 32,}, - .field[6] = {.value.u32 = 0x0, .mask_range.u32 = 32,}, - .field[7] = {.value.u32 = 0xaaaaaaaa, .mask_range.u32 = 32,}, - .field[8] = {.value.u32 = 0x0, .mask_range.u32 = 0,}, - /* source port */ - .field[9] = {.value.u16 = 0, .mask_range.u16 = 0xffff,}, - /* destination port */ - .field[10] = {.value.u16 = 0, .mask_range.u16 = 0xffff,} - }, - { - .data = {.userdata = PROTECT(126), .category_mask = 1, .priority = 1}, - /* destination IPv6 */ - .field[5] = {.value.u32 = 0xffff0000, .mask_range.u32 = 32,}, - .field[6] = {.value.u32 = 0x0, .mask_range.u32 = 32,}, - .field[7] = {.value.u32 = 0xbbbbbbbb, .mask_range.u32 = 32,}, - .field[8] = {.value.u32 = 0x0, .mask_range.u32 = 0,}, - /* source port */ - .field[9] = {.value.u16 = 0, .mask_range.u16 = 0xffff,}, - /* destination port */ - .field[10] = {.value.u16 = 0, .mask_range.u16 = 0xffff,} +struct acl6_rules acl6_rules_in[MAX_ACL_RULE_NUM]; +uint32_t nb_acl6_rules_in; + +void +parse_sp6_tokens(char **tokens, uint32_t n_tokens, + struct parse_status *status) +{ + struct acl6_rules *rule_ipv6 = NULL; + + uint32_t *ri = NULL; /* rule index */ + uint32_t ti = 0; /* token index */ + + uint32_t esp_p = 0; + uint32_t protect_p = 0; + uint32_t bypass_p = 0; + uint32_t discard_p = 0; + uint32_t pri_p = 0; + uint32_t src_p = 0; + uint32_t dst_p = 0; + uint32_t proto_p = 0; + uint32_t sport_p = 0; + uint32_t dport_p = 0; + + if (strcmp(tokens[1], "in") == 0) { + ri = &nb_acl6_rules_in; + + APP_CHECK(*ri <= MAX_ACL_RULE_NUM - 1, status, "too " + "many sp rules, abort insertion\n"); + if (status->status < 0) + return; + + rule_ipv6 = &acl6_rules_in[*ri]; + + } else if (strcmp(tokens[1], "out") == 0) { + ri = &nb_acl6_rules_out; + + APP_CHECK(*ri <= MAX_ACL_RULE_NUM - 1, status, "too " + "many sp rules, abort insertion\n"); + if (status->status < 0) + return; + + rule_ipv6 = &acl6_rules_out[*ri]; + + } else { + APP_CHECK(0, status, "unrecognized input \"%s\", expect" + " \"in\" or \"out\"\n", tokens[ti]); + return; } -}; + + rule_ipv6->data.category_mask = 1; + + + for (ti = 2; ti < n_tokens; ti++) { + if (strcmp(tokens[ti], "esp") == 0) { + /* currently do nothing */ + APP_CHECK_PRESENCE(esp_p, tokens[ti], status); + if (status->status < 0) + return; + esp_p = 1; + continue; + } + + if (strcmp(tokens[ti], "protect") == 0) { + APP_CHECK_PRESENCE(protect_p, tokens[ti], status); + if (status->status < 0) + return; + APP_CHECK(bypass_p == 0, status, "conflict item " + "between \"%s\" and \"%s\"", tokens[ti], + "bypass"); + if (status->status < 0) + return; + APP_CHECK(discard_p == 0, status, "conflict item " + "between \"%s\" and \"%s\"", tokens[ti], + "discard"); + if (status->status < 0) + return; + INCREMENT_TOKEN_INDEX(ti, n_tokens, status); + if (status->status < 0) + return; + APP_CHECK_TOKEN_IS_NUM(tokens, ti, status); + if (status->status < 0) + return; + + rule_ipv6->data.userdata = + PROTECT(atoi(tokens[ti])); + + protect_p = 1; + continue; + } + + if (strcmp(tokens[ti], "bypass") == 0) { + APP_CHECK_PRESENCE(bypass_p, tokens[ti], status); + if (status->status < 0) + return; + APP_CHECK(protect_p == 0, status, "conflict item " + "between \"%s\" and \"%s\"", tokens[ti], + "protect"); + if (status->status < 0) + return; + APP_CHECK(discard_p == 0, status, "conflict item " + "between \"%s\" and \"%s\"", tokens[ti], + "discard"); + if (status->status < 0) + return; + + rule_ipv6->data.userdata = BYPASS; + + bypass_p = 1; + continue; + } + + if (strcmp(tokens[ti], "discard") == 0) { + APP_CHECK_PRESENCE(discard_p, tokens[ti], status); + if (status->status < 0) + return; + APP_CHECK(protect_p == 0, status, "conflict item " + "between \"%s\" and \"%s\"", tokens[ti], + "protect"); + if (status->status < 0) + return; + APP_CHECK(bypass_p == 0, status, "conflict item " + "between \"%s\" and \"%s\"", tokens[ti], + "discard"); + if (status->status < 0) + return; + + rule_ipv6->data.userdata = DISCARD; + + discard_p = 1; + continue; + } + + if (strcmp(tokens[ti], "pri") == 0) { + APP_CHECK_PRESENCE(pri_p, tokens[ti], status); + if (status->status < 0) + return; + INCREMENT_TOKEN_INDEX(ti, n_tokens, status); + if (status->status < 0) + return; + APP_CHECK_TOKEN_IS_NUM(tokens, ti, status); + if (status->status < 0) + return; + + rule_ipv6->data.priority = atoi(tokens[ti]); + + pri_p = 1; + continue; + } + + if (strcmp(tokens[ti], "src") == 0) { + struct in6_addr ip; + uint32_t depth; + + APP_CHECK_PRESENCE(src_p, tokens[ti], status); + if (status->status < 0) + return; + INCREMENT_TOKEN_INDEX(ti, n_tokens, status); + if (status->status < 0) + return; + + APP_CHECK(parse_ipv6_addr(tokens[ti], &ip, + &depth) == 0, status, "unrecognized " + "input \"%s\", expect valid ipv6 " + "addr", tokens[ti]); + if (status->status < 0) + return; + + rule_ipv6->field[1].value.u32 = + (uint32_t)ip.s6_addr[0] << 24 | + (uint32_t)ip.s6_addr[1] << 16 | + (uint32_t)ip.s6_addr[2] << 8 | + (uint32_t)ip.s6_addr[3]; + rule_ipv6->field[1].mask_range.u32 = + (depth > 32) ? 32 : depth; + depth = (depth > 32) ? (depth - 32) : 0; + rule_ipv6->field[2].value.u32 = + (uint32_t)ip.s6_addr[4] << 24 | + (uint32_t)ip.s6_addr[5] << 16 | + (uint32_t)ip.s6_addr[6] << 8 | + (uint32_t)ip.s6_addr[7]; + rule_ipv6->field[2].mask_range.u32 = + (depth > 32) ? 32 : depth; + depth = (depth > 32) ? (depth - 32) : 0; + rule_ipv6->field[3].value.u32 = + (uint32_t)ip.s6_addr[8] << 24 | + (uint32_t)ip.s6_addr[9] << 16 | + (uint32_t)ip.s6_addr[10] << 8 | + (uint32_t)ip.s6_addr[11]; + rule_ipv6->field[3].mask_range.u32 = + (depth > 32) ? 32 : depth; + depth = (depth > 32) ? (depth - 32) : 0; + rule_ipv6->field[4].value.u32 = + (uint32_t)ip.s6_addr[12] << 24 | + (uint32_t)ip.s6_addr[13] << 16 | + (uint32_t)ip.s6_addr[14] << 8 | + (uint32_t)ip.s6_addr[15]; + rule_ipv6->field[4].mask_range.u32 = + (depth > 32) ? 32 : depth; + + src_p = 1; + continue; + } + + if (strcmp(tokens[ti], "dst") == 0) { + struct in6_addr ip; + uint32_t depth; + + APP_CHECK_PRESENCE(dst_p, tokens[ti], status); + if (status->status < 0) + return; + INCREMENT_TOKEN_INDEX(ti, n_tokens, status); + if (status->status < 0) + return; + + APP_CHECK(parse_ipv6_addr(tokens[ti], &ip, + &depth) == 0, status, "unrecognized " + "input \"%s\", expect valid ipv6 " + "addr", tokens[ti]); + if (status->status < 0) + return; + + rule_ipv6->field[5].value.u32 = + (uint32_t)ip.s6_addr[0] << 24 | + (uint32_t)ip.s6_addr[1] << 16 | + (uint32_t)ip.s6_addr[2] << 8 | + (uint32_t)ip.s6_addr[3]; + rule_ipv6->field[5].mask_range.u32 = + (depth > 32) ? 32 : depth; + depth = (depth > 32) ? (depth - 32) : 0; + rule_ipv6->field[6].value.u32 = + (uint32_t)ip.s6_addr[4] << 24 | + (uint32_t)ip.s6_addr[5] << 16 | + (uint32_t)ip.s6_addr[6] << 8 | + (uint32_t)ip.s6_addr[7]; + rule_ipv6->field[6].mask_range.u32 = + (depth > 32) ? 32 : depth; + depth = (depth > 32) ? (depth - 32) : 0; + rule_ipv6->field[7].value.u32 = + (uint32_t)ip.s6_addr[8] << 24 | + (uint32_t)ip.s6_addr[9] << 16 | + (uint32_t)ip.s6_addr[10] << 8 | + (uint32_t)ip.s6_addr[11]; + rule_ipv6->field[7].mask_range.u32 = + (depth > 32) ? 32 : depth; + depth = (depth > 32) ? (depth - 32) : 0; + rule_ipv6->field[8].value.u32 = + (uint32_t)ip.s6_addr[12] << 24 | + (uint32_t)ip.s6_addr[13] << 16 | + (uint32_t)ip.s6_addr[14] << 8 | + (uint32_t)ip.s6_addr[15]; + rule_ipv6->field[8].mask_range.u32 = + (depth > 32) ? 32 : depth; + + dst_p = 1; + continue; + } + + if (strcmp(tokens[ti], "proto") == 0) { + uint16_t low, high; + + APP_CHECK_PRESENCE(proto_p, tokens[ti], status); + if (status->status < 0) + return; + INCREMENT_TOKEN_INDEX(ti, n_tokens, status); + if (status->status < 0) + return; + + APP_CHECK(parse_range(tokens[ti], &low, &high) + == 0, status, "unrecognized input \"%s\"" + ", expect \"from:to\"", tokens[ti]); + if (status->status < 0) + return; + APP_CHECK(low <= 0xff, status, "proto low " + "over-limit"); + if (status->status < 0) + return; + APP_CHECK(high <= 0xff, status, "proto high " + "over-limit"); + if (status->status < 0) + return; + + rule_ipv6->field[0].value.u8 = (uint8_t)low; + rule_ipv6->field[0].mask_range.u8 = (uint8_t)high; + + proto_p = 1; + continue; + } + + if (strcmp(tokens[ti], "sport") == 0) { + uint16_t port_low, port_high; + + APP_CHECK_PRESENCE(sport_p, tokens[ti], status); + if (status->status < 0) + return; + INCREMENT_TOKEN_INDEX(ti, n_tokens, status); + if (status->status < 0) + return; + + APP_CHECK(parse_range(tokens[ti], &port_low, + &port_high) == 0, status, "unrecognized " + "input \"%s\", expect \"port_from:" + "port_to\"", tokens[ti]); + if (status->status < 0) + return; + + rule_ipv6->field[9].value.u16 = port_low; + rule_ipv6->field[9].mask_range.u16 = port_high; + + sport_p = 1; + continue; + } + + if (strcmp(tokens[ti], "dport") == 0) { + uint16_t port_low, port_high; + + APP_CHECK_PRESENCE(dport_p, tokens[ti], status); + if (status->status < 0) + return; + INCREMENT_TOKEN_INDEX(ti, n_tokens, status); + if (status->status < 0) + return; + + APP_CHECK(parse_range(tokens[ti], &port_low, + &port_high) == 0, status, "unrecognized " + "input \"%s\", expect \"port_from:" + "port_to\"", tokens[ti]); + if (status->status < 0) + return; + + rule_ipv6->field[10].value.u16 = port_low; + rule_ipv6->field[10].mask_range.u16 = port_high; + + dport_p = 1; + continue; + } + + /* unrecognizeable input */ + APP_CHECK(0, status, "unrecognized input \"%s\"", + tokens[ti]); + return; + } + + /* check if argument(s) are missing */ + APP_CHECK(esp_p == 1, status, "missing argument \"esp\""); + if (status->status < 0) + return; + + APP_CHECK(protect_p | bypass_p | discard_p, status, "missing " + "argument \"protect\", \"bypass\", or \"discard\""); + if (status->status < 0) + return; + + *ri = *ri + 1; +} static inline void print_one_ip6_rule(const struct acl6_rules *rule, int32_t extra) @@ -407,11 +616,9 @@ acl6_init(const char *name, int32_t socketid, const struct acl6_rules *rules, } void -sp6_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t ep) +sp6_init(struct socket_ctx *ctx, int32_t socket_id) { const char *name; - const struct acl6_rules *rules_out, *rules_in; - uint32_t nb_out_rules, nb_in_rules; if (ctx == NULL) rte_exit(EXIT_FAILURE, "NULL context.\n"); @@ -424,25 +631,19 @@ sp6_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t ep) rte_exit(EXIT_FAILURE, "Outbound IPv6 SP DB for socket %u " "already initialized\n", socket_id); - if (ep == 0) { - rules_out = acl6_rules_out; - nb_out_rules = RTE_DIM(acl6_rules_out); - rules_in = acl6_rules_in; - nb_in_rules = RTE_DIM(acl6_rules_in); - } else if (ep == 1) { - rules_out = acl6_rules_in; - nb_out_rules = RTE_DIM(acl6_rules_in); - rules_in = acl6_rules_out; - nb_in_rules = RTE_DIM(acl6_rules_out); + if (nb_acl6_rules_in > 0) { + name = "sp_ip6_in"; + ctx->sp_ip6_in = (struct sp_ctx *)acl6_init(name, + socket_id, acl6_rules_in, nb_acl6_rules_in); } else - rte_exit(EXIT_FAILURE, "Invalid EP value %u. " - "Only 0 or 1 supported.\n", ep); + RTE_LOG(WARNING, IPSEC, "No IPv6 SP Inbound rule " + "specified\n"); - name = "sp_ip6_in"; - ctx->sp_ip6_in = (struct sp_ctx *)acl6_init(name, socket_id, - rules_in, nb_in_rules); - - name = "sp_ip6_out"; - ctx->sp_ip6_out = (struct sp_ctx *)acl6_init(name, socket_id, - rules_out, nb_out_rules); + if (nb_acl6_rules_out > 0) { + name = "sp_ip6_out"; + ctx->sp_ip6_out = (struct sp_ctx *)acl6_init(name, + socket_id, acl6_rules_out, nb_acl6_rules_out); + } else + RTE_LOG(WARNING, IPSEC, "No IPv6 SP Outbound rule " + "specified\n"); } diff --git a/dpdk/examples/ipv4_multicast/Makefile b/dpdk/examples/ipv4_multicast/Makefile index 44f0a3bb..1f7c53af 100644 --- a/dpdk/examples/ipv4_multicast/Makefile +++ b/dpdk/examples/ipv4_multicast/Makefile @@ -34,7 +34,7 @@ ifeq ($(RTE_SDK),) $(error "Please define RTE_SDK environment variable") endif -# Default target, can be overriden by command line or environment +# Default target, can be overridden by command line or environment RTE_TARGET ?= x86_64-native-linuxapp-gcc include $(RTE_SDK)/mk/rte.vars.mk diff --git a/dpdk/examples/ipv4_multicast/main.c b/dpdk/examples/ipv4_multicast/main.c index f013d927..1c585165 100644 --- a/dpdk/examples/ipv4_multicast/main.c +++ b/dpdk/examples/ipv4_multicast/main.c @@ -47,9 +47,7 @@ #include #include #include -#include #include -#include #include #include #include @@ -58,12 +56,10 @@ #include #include #include -#include #include #include #include #include -#include #include #include #include @@ -118,7 +114,7 @@ static struct ether_addr ports_eth_addr[MAX_PORTS]; /* mask of enabled ports */ static uint32_t enabled_port_mask = 0; -static uint8_t nb_ports = 0; +static uint16_t nb_ports; static int rx_queue_per_lcore = 1; @@ -138,7 +134,7 @@ struct lcore_queue_conf { } __rte_cache_aligned; static struct lcore_queue_conf lcore_queue_conf[RTE_MAX_LCORE]; -static const struct rte_eth_conf port_conf = { +static struct rte_eth_conf port_conf = { .rxmode = { .max_rx_pkt_len = JUMBO_FRAME_MAX_SIZE, .split_hdr_size = 0, @@ -146,7 +142,7 @@ static const struct rte_eth_conf port_conf = { .hw_ip_checksum = 0, /**< IP checksum offload disabled */ .hw_vlan_filter = 0, /**< VLAN filtering disabled */ .jumbo_frame = 1, /**< Jumbo Frame Support enabled */ - .hw_strip_crc = 0, /**< CRC stripped by hardware */ + .hw_strip_crc = 1, /**< CRC stripped by hardware */ }, .txmode = { .mq_mode = ETH_MQ_TX_NONE, @@ -197,7 +193,7 @@ static struct mcast_group_params mcast_group_table[] = { /* Send burst of packets on an output interface */ static void -send_burst(struct lcore_queue_conf *qconf, uint8_t port) +send_burst(struct lcore_queue_conf *qconf, uint16_t port) { struct rte_mbuf **m_table; uint16_t n, queueid; @@ -293,7 +289,7 @@ mcast_out_pkt(struct rte_mbuf *pkt, int use_clone) /* update header's fields */ hdr->pkt_len = (uint16_t)(hdr->data_len + pkt->pkt_len); - hdr->nb_segs = (uint8_t)(pkt->nb_segs + 1); + hdr->nb_segs = pkt->nb_segs + 1; /* copy metadata from source packet*/ hdr->port = pkt->port; @@ -314,7 +310,7 @@ mcast_out_pkt(struct rte_mbuf *pkt, int use_clone) */ static inline void mcast_send_pkt(struct rte_mbuf *pkt, struct ether_addr *dest_addr, - struct lcore_queue_conf *qconf, uint8_t port) + struct lcore_queue_conf *qconf, uint16_t port) { struct ether_hdr *ethdr; uint16_t len; @@ -345,7 +341,7 @@ mcast_forward(struct rte_mbuf *m, struct lcore_queue_conf *qconf) struct ipv4_hdr *iphdr; uint32_t dest_addr, port_mask, port_num, use_clone; int32_t hash; - uint8_t port; + uint16_t port; union { uint64_t as_int; struct ether_addr as_addr; @@ -409,7 +405,7 @@ static inline void send_timeout_burst(struct lcore_queue_conf *qconf) { uint64_t cur_tsc; - uint8_t portid; + uint16_t portid; const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / US_PER_S * BURST_TX_DRAIN_US; cur_tsc = rte_rdtsc(); @@ -430,7 +426,7 @@ main_loop(__rte_unused void *dummy) struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; unsigned lcore_id; int i, j, nb_rx; - uint8_t portid; + uint16_t portid; struct lcore_queue_conf *qconf; lcore_id = rte_lcore_id(); @@ -450,7 +446,7 @@ main_loop(__rte_unused void *dummy) portid = qconf->rx_queue_list[i]; RTE_LOG(INFO, IPv4_MULTICAST, " -- lcoreid=%u portid=%d\n", - lcore_id, (int) portid); + lcore_id, portid); } while (1) { @@ -576,7 +572,7 @@ parse_args(int argc, char **argv) argv[optind-1] = prgname; ret = optind-1; - optind = 0; /* reset getopt lib */ + optind = 1; /* reset getopt lib */ return ret; } @@ -612,11 +608,12 @@ init_mcast_hash(void) /* Check the link status of all ports in up to 9s, and print them finally */ static void -check_all_ports_link_status(uint8_t port_num, uint32_t port_mask) +check_all_ports_link_status(uint16_t port_num, uint32_t port_mask) { #define CHECK_INTERVAL 100 /* 100ms */ #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ - uint8_t portid, count, all_ports_up, print_flag = 0; + uint16_t portid; + uint8_t count, all_ports_up, print_flag = 0; struct rte_eth_link link; printf("\nChecking link status"); @@ -631,14 +628,13 @@ check_all_ports_link_status(uint8_t port_num, uint32_t port_mask) /* print link status if flag set */ if (print_flag == 1) { if (link.link_status) - printf("Port %d Link Up - speed %u " - "Mbps - %s\n", (uint8_t)portid, - (unsigned)link.link_speed, + printf( + "Port%d Link Up. Speed %u Mbps - %s\n", + portid, link.link_speed, (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? ("full-duplex") : ("half-duplex\n")); else - printf("Port %d Link Down\n", - (uint8_t)portid); + printf("Port %d Link Down\n", portid); continue; } /* clear all_ports_up flag if any link down */ @@ -675,7 +671,7 @@ main(int argc, char **argv) uint16_t queueid; unsigned lcore_id = 0, rx_lcore_id = 0; uint32_t n_tx_queue, nb_lcores; - uint8_t portid; + uint16_t portid; /* init EAL */ ret = rte_eal_init(argc, argv); @@ -726,6 +722,11 @@ main(int argc, char **argv) qconf = &lcore_queue_conf[rx_lcore_id]; + /* limit the frame size to the maximum supported by NIC */ + rte_eth_dev_info_get(portid, &dev_info); + port_conf.rxmode.max_rx_pkt_len = RTE_MIN( + dev_info.max_rx_pktlen, port_conf.rxmode.max_rx_pkt_len); + /* get the lcore_id for this port */ while (rte_lcore_is_enabled(rx_lcore_id) == 0 || qconf->n_rx_queue == (unsigned)rx_queue_per_lcore) { @@ -753,6 +754,13 @@ main(int argc, char **argv) rte_exit(EXIT_FAILURE, "Cannot configure device: err=%d, port=%d\n", ret, portid); + ret = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd, + &nb_txd); + if (ret < 0) + rte_exit(EXIT_FAILURE, + "Cannot adjust number of descriptors: err=%d, port=%d\n", + ret, portid); + rte_eth_macaddr_get(portid, &ports_eth_addr[portid]); print_ethaddr(" Address:", &ports_eth_addr[portid]); printf(", "); @@ -778,7 +786,6 @@ main(int argc, char **argv) printf("txq=%u,%hu ", lcore_id, queueid); fflush(stdout); - rte_eth_dev_info_get(portid, &dev_info); txconf = &dev_info.default_txconf; txconf->txq_flags = 0; ret = rte_eth_tx_queue_setup(portid, queueid, nb_txd, diff --git a/dpdk/examples/kni/Makefile b/dpdk/examples/kni/Makefile index 6800dd5c..08a4f0c5 100644 --- a/dpdk/examples/kni/Makefile +++ b/dpdk/examples/kni/Makefile @@ -33,7 +33,7 @@ ifeq ($(RTE_SDK),) $(error "Please define RTE_SDK environment variable") endif -# Default target, can be overriden by command line or environment +# Default target, can be overridden by command line or environment RTE_TARGET ?= x86_64-native-linuxapp-gcc include $(RTE_SDK)/mk/rte.vars.mk diff --git a/dpdk/examples/kni/main.c b/dpdk/examples/kni/main.c index f9fc61e0..3f173854 100644 --- a/dpdk/examples/kni/main.c +++ b/dpdk/examples/kni/main.c @@ -53,7 +53,6 @@ #include #include #include -#include #include #include #include @@ -61,12 +60,10 @@ #include #include #include -#include +#include #include #include #include -#include -#include #include #include #include @@ -112,7 +109,7 @@ * Structure of port parameters */ struct kni_port_params { - uint8_t port_id;/* Port ID */ + uint16_t port_id;/* Port ID */ unsigned lcore_rx; /* lcore ID for RX */ unsigned lcore_tx; /* lcore ID for TX */ uint32_t nb_lcore_k; /* Number of lcores for KNI multi kernel threads */ @@ -131,7 +128,7 @@ static struct rte_eth_conf port_conf = { .hw_ip_checksum = 0, /* IP checksum offload disabled */ .hw_vlan_filter = 0, /* VLAN filtering disabled */ .jumbo_frame = 0, /* Jumbo Frame Support disabled */ - .hw_strip_crc = 0, /* CRC stripped by hardware */ + .hw_strip_crc = 1, /* CRC stripped by hardware */ }, .txmode = { .mq_mode = ETH_MQ_TX_NONE, @@ -164,8 +161,8 @@ struct kni_interface_stats { /* kni device statistics array */ static struct kni_interface_stats kni_stats[RTE_MAX_ETHPORTS]; -static int kni_change_mtu(uint8_t port_id, unsigned new_mtu); -static int kni_config_network_interface(uint8_t port_id, uint8_t if_up); +static int kni_change_mtu(uint16_t port_id, unsigned int new_mtu); +static int kni_config_network_interface(uint16_t port_id, uint8_t if_up); static rte_atomic32_t kni_stop = RTE_ATOMIC32_INIT(0); @@ -173,7 +170,7 @@ static rte_atomic32_t kni_stop = RTE_ATOMIC32_INIT(0); static void print_stats(void) { - uint8_t i; + uint16_t i; printf("\n**KNI example application statistics**\n" "====== ============== ============ ============ ============ ============\n" @@ -240,7 +237,8 @@ kni_burst_free_mbufs(struct rte_mbuf **pkts, unsigned num) static void kni_ingress(struct kni_port_params *p) { - uint8_t i, port_id; + uint8_t i; + uint16_t port_id; unsigned nb_rx, num; uint32_t nb_kni; struct rte_mbuf *pkts_burst[PKT_BURST_SZ]; @@ -276,7 +274,8 @@ kni_ingress(struct kni_port_params *p) static void kni_egress(struct kni_port_params *p) { - uint8_t i, port_id; + uint8_t i; + uint16_t port_id; unsigned nb_tx, num; uint32_t nb_kni; struct rte_mbuf *pkts_burst[PKT_BURST_SZ]; @@ -418,7 +417,7 @@ parse_config(const char *arg) int i, j, nb_token; char *str_fld[_NUM_FLD]; unsigned long int_fld[_NUM_FLD]; - uint8_t port_id, nb_kni_port_params = 0; + uint16_t port_id, nb_kni_port_params = 0; memset(&kni_port_params_array, 0, sizeof(kni_port_params_array)); while (((p = strchr(p0, '(')) != NULL) && @@ -447,7 +446,7 @@ parse_config(const char *arg) } i = 0; - port_id = (uint8_t)int_fld[i++]; + port_id = int_fld[i++]; if (port_id >= RTE_MAX_ETHPORTS) { printf("Port ID %d could not exceed the maximum %d\n", port_id, RTE_MAX_ETHPORTS); @@ -603,9 +602,11 @@ init_kni(void) /* Initialise a single port on an Ethernet device */ static void -init_port(uint8_t port) +init_port(uint16_t port) { int ret; + uint16_t nb_rxd = NB_RXD; + uint16_t nb_txd = NB_TXD; /* Initialise device and RX/TX queues */ RTE_LOG(INFO, APP, "Initialising port %u ...\n", (unsigned)port); @@ -615,13 +616,18 @@ init_port(uint8_t port) rte_exit(EXIT_FAILURE, "Could not configure port%u (%d)\n", (unsigned)port, ret); - ret = rte_eth_rx_queue_setup(port, 0, NB_RXD, + ret = rte_eth_dev_adjust_nb_rx_tx_desc(port, &nb_rxd, &nb_txd); + if (ret < 0) + rte_exit(EXIT_FAILURE, "Could not adjust number of descriptors " + "for port%u (%d)\n", (unsigned)port, ret); + + ret = rte_eth_rx_queue_setup(port, 0, nb_rxd, rte_eth_dev_socket_id(port), NULL, pktmbuf_pool); if (ret < 0) rte_exit(EXIT_FAILURE, "Could not setup up RX queue for " "port%u (%d)\n", (unsigned)port, ret); - ret = rte_eth_tx_queue_setup(port, 0, NB_TXD, + ret = rte_eth_tx_queue_setup(port, 0, nb_txd, rte_eth_dev_socket_id(port), NULL); if (ret < 0) rte_exit(EXIT_FAILURE, "Could not setup up TX queue for " @@ -638,11 +644,12 @@ init_port(uint8_t port) /* Check the link status of all ports in up to 9s, and print them finally */ static void -check_all_ports_link_status(uint8_t port_num, uint32_t port_mask) +check_all_ports_link_status(uint16_t port_num, uint32_t port_mask) { #define CHECK_INTERVAL 100 /* 100ms */ #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ - uint8_t portid, count, all_ports_up, print_flag = 0; + uint16_t portid; + uint8_t count, all_ports_up, print_flag = 0; struct rte_eth_link link; printf("\nChecking link status\n"); @@ -657,14 +664,13 @@ check_all_ports_link_status(uint8_t port_num, uint32_t port_mask) /* print link status if flag set */ if (print_flag == 1) { if (link.link_status) - printf("Port %d Link Up - speed %u " - "Mbps - %s\n", (uint8_t)portid, - (unsigned)link.link_speed, + printf( + "Port%d Link Up - speed %uMbps - %s\n", + portid, link.link_speed, (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? ("full-duplex") : ("half-duplex\n")); else - printf("Port %d Link Down\n", - (uint8_t)portid); + printf("Port %d Link Down\n", portid); continue; } /* clear all_ports_up flag if any link down */ @@ -693,7 +699,7 @@ check_all_ports_link_status(uint8_t port_num, uint32_t port_mask) /* Callback for request of changing MTU */ static int -kni_change_mtu(uint8_t port_id, unsigned new_mtu) +kni_change_mtu(uint16_t port_id, unsigned int new_mtu) { int ret; struct rte_eth_conf conf; @@ -736,7 +742,7 @@ kni_change_mtu(uint8_t port_id, unsigned new_mtu) /* Callback for request of configuring network interface up/down */ static int -kni_config_network_interface(uint8_t port_id, uint8_t if_up) +kni_config_network_interface(uint16_t port_id, uint8_t if_up) { int ret = 0; @@ -761,7 +767,7 @@ kni_config_network_interface(uint8_t port_id, uint8_t if_up) } static int -kni_alloc(uint8_t port_id) +kni_alloc(uint16_t port_id) { uint8_t i; struct rte_kni *kni; @@ -785,7 +791,7 @@ kni_alloc(uint8_t port_id) } else snprintf(conf.name, RTE_KNI_NAMESIZE, "vEth%u", port_id); - conf.group_id = (uint16_t)port_id; + conf.group_id = port_id; conf.mbuf_size = MAX_PACKET_SZ; /* * The first KNI device associated to a port @@ -798,8 +804,11 @@ kni_alloc(uint8_t port_id) memset(&dev_info, 0, sizeof(dev_info)); rte_eth_dev_info_get(port_id, &dev_info); - conf.addr = dev_info.pci_dev->addr; - conf.id = dev_info.pci_dev->id; + + if (dev_info.pci_dev) { + conf.addr = dev_info.pci_dev->addr; + conf.id = dev_info.pci_dev->id; + } memset(&ops, 0, sizeof(ops)); ops.port_id = port_id; @@ -820,7 +829,7 @@ kni_alloc(uint8_t port_id) } static int -kni_free_kni(uint8_t port_id) +kni_free_kni(uint16_t port_id) { uint8_t i; struct kni_port_params **p = kni_port_params_array; @@ -843,7 +852,7 @@ int main(int argc, char** argv) { int ret; - uint8_t nb_sys_ports, port; + uint16_t nb_sys_ports, port; unsigned i; /* Associate signal_hanlder function with USR signals */ @@ -914,9 +923,6 @@ main(int argc, char** argv) continue; kni_free_kni(port); } -#ifdef RTE_LIBRTE_XEN_DOM0 - rte_kni_close(); -#endif for (i = 0; i < RTE_MAX_ETHPORTS; i++) if (kni_port_params_array[i]) { rte_free(kni_port_params_array[i]); diff --git a/dpdk/examples/l2fwd-cat/Makefile b/dpdk/examples/l2fwd-cat/Makefile index ae921ade..a7fe6d68 100644 --- a/dpdk/examples/l2fwd-cat/Makefile +++ b/dpdk/examples/l2fwd-cat/Makefile @@ -40,9 +40,6 @@ endif # Default target, can be overridden by command line or environment RTE_TARGET ?= x86_64-native-linuxapp-gcc -# Location of PQoS library and includes, -PQOS_LIBRARY_PATH = $(PQOS_INSTALL_PATH)/libpqos.a - include $(RTE_SDK)/mk/rte.vars.mk # binary name @@ -65,6 +62,6 @@ CFLAGS += -I$(PQOS_INSTALL_PATH)/../include CFLAGS_cat.o := -D_GNU_SOURCE LDLIBS += -L$(PQOS_INSTALL_PATH) -LDLIBS += $(PQOS_LIBRARY_PATH) +LDLIBS += -lpqos include $(RTE_SDK)/mk/rte.extapp.mk diff --git a/dpdk/examples/l2fwd-cat/cat.c b/dpdk/examples/l2fwd-cat/cat.c index bad39305..689dcb11 100644 --- a/dpdk/examples/l2fwd-cat/cat.c +++ b/dpdk/examples/l2fwd-cat/cat.c @@ -53,7 +53,11 @@ static const struct pqos_cap *m_cap; static const struct pqos_cpuinfo *m_cpu; static const struct pqos_capability *m_cap_l3ca; +#if PQOS_VERSION <= 103 static unsigned m_sockets[PQOS_MAX_SOCKETS]; +#else +static unsigned int *m_sockets; +#endif static unsigned m_sock_count; static struct cat_config m_config[PQOS_MAX_CORES]; static unsigned m_config_count; @@ -271,16 +275,16 @@ parse_l3ca(const char *l3ca) /* scan the separator '@', ','(next) or '\0'(finish) */ l3ca += strcspn(l3ca, "@,"); - if (*l3ca == '@') { - /* explicit assign cpu_set */ - offset = parse_set(l3ca + 1, &cpuset); - if (offset < 0 || CPU_COUNT(&cpuset) == 0) - goto err; - - end = l3ca + 1 + offset; - } else + if (*l3ca != '@') goto err; + /* explicit assign cpu_set */ + offset = parse_set(l3ca + 1, &cpuset); + if (offset < 0 || CPU_COUNT(&cpuset) == 0) + goto err; + + end = l3ca + 1 + offset; + if (*end != ',' && *end != '\0') goto err; @@ -353,9 +357,6 @@ parse_l3ca(const char *l3ca) idx++; } while (*end != '\0' && idx < PQOS_MAX_CORES); - if (m_config_count == 0) - goto err; - return 0; err: @@ -408,7 +409,11 @@ check_cpus(void) goto exit; } +#if PQOS_VERSION <= 103 ret = pqos_l3ca_assoc_get(cpu_id, &cos_id); +#else + ret = pqos_alloc_assoc_get(cpu_id, &cos_id); +#endif if (ret != PQOS_RETVAL_OK) { printf("PQOS: Failed to read COS " "associated to cpu %u.\n", @@ -512,7 +517,11 @@ check_and_select_classes(unsigned cos_id_map[][PQOS_MAX_SOCKETS]) for (j = 0; j < m_cpu->num_cores; j++) { cpu_id = m_cpu->cores[j].lcore; +#if PQOS_VERSION <= 103 ret = pqos_l3ca_assoc_get(cpu_id, &cos_id); +#else + ret = pqos_alloc_assoc_get(cpu_id, &cos_id); +#endif if (ret != PQOS_RETVAL_OK) { printf("PQOS: Failed to read COS associated to " "cpu %u on phy_pkg %u.\n", cpu_id, phy_pkg_id); @@ -598,10 +607,19 @@ configure_cat(unsigned cos_id_map[][PQOS_MAX_SOCKETS]) l3ca.cdp = m_config[i].cdp; if (m_config[i].cdp == 1) { +#if PQOS_VERSION <= 103 l3ca.code_mask = m_config[i].code_mask; l3ca.data_mask = m_config[i].data_mask; +#else + l3ca.u.s.code_mask = m_config[i].code_mask; + l3ca.u.s.data_mask = m_config[i].data_mask; +#endif } else +#if PQOS_VERSION <= 103 l3ca.ways_mask = m_config[i].mask; +#else + l3ca.u.ways_mask = m_config[i].mask; +#endif for (j = 0; j < m_sock_count; j++) { phy_pkg_id = m_sockets[j]; @@ -637,7 +655,11 @@ configure_cat(unsigned cos_id_map[][PQOS_MAX_SOCKETS]) cos_id = cos_id_map[i][phy_pkg_id]; +#if PQOS_VERSION <= 103 ret = pqos_l3ca_assoc_set(cpu_id, cos_id); +#else + ret = pqos_alloc_assoc_set(cpu_id, cos_id); +#endif if (ret != PQOS_RETVAL_OK) { printf("PQOS: Failed to associate COS %u to " "cpu %u\n", cos_id, cpu_id); @@ -686,7 +708,7 @@ parse_args(int argc, char **argv) exit: /* reset getopt lib */ - optind = 0; + optind = 1; /* Restore opterr value */ opterr = oldopterr; @@ -754,24 +776,43 @@ print_cat_config(void) if (tab[n].cdp == 1) { printf("PQOS: COS: %u, cMASK: 0x%llx, " "dMASK: 0x%llx\n", tab[n].class_id, +#if PQOS_VERSION <= 103 (unsigned long long)tab[n].code_mask, (unsigned long long)tab[n].data_mask); +#else + (unsigned long long)tab[n].u.s.code_mask, + (unsigned long long)tab[n].u.s.data_mask); +#endif } else { printf("PQOS: COS: %u, MASK: 0x%llx\n", tab[n].class_id, +#if PQOS_VERSION <= 103 (unsigned long long)tab[n].ways_mask); +#else + (unsigned long long)tab[n].u.ways_mask); +#endif } } } for (i = 0; i < m_sock_count; i++) { +#if PQOS_VERSION <= 103 unsigned lcores[PQOS_MAX_SOCKET_CORES] = {0}; +#else + unsigned int *lcores = NULL; +#endif unsigned lcount = 0; unsigned n = 0; +#if PQOS_VERSION <= 103 ret = pqos_cpu_get_cores(m_cpu, m_sockets[i], PQOS_MAX_SOCKET_CORES, &lcount, &lcores[0]); if (ret != PQOS_RETVAL_OK) { +#else + lcores = pqos_cpu_get_cores(m_cpu, m_sockets[i], + &lcount); + if (lcores == NULL || lcount == 0) { +#endif printf("PQOS: Error retrieving core information!\n"); return; } @@ -780,13 +821,21 @@ print_cat_config(void) for (n = 0; n < lcount; n++) { unsigned class_id = 0; +#if PQOS_VERSION <= 103 ret = pqos_l3ca_assoc_get(lcores[n], &class_id); +#else + ret = pqos_alloc_assoc_get(lcores[n], &class_id); +#endif if (ret == PQOS_RETVAL_OK) printf("PQOS: CPU: %u, COS: %u\n", lcores[n], class_id); else printf("PQOS: CPU: %u, ERROR\n", lcores[n]); } + +#if PQOS_VERSION > 103 + free(lcores); +#endif } } @@ -849,7 +898,12 @@ cat_fini(void) m_cap = NULL; m_cpu = NULL; m_cap_l3ca = NULL; +#if PQOS_VERSION <= 103 memset(m_sockets, 0, sizeof(m_sockets)); +#else + if (m_sockets != NULL) + free(m_sockets); +#endif m_sock_count = 0; memset(m_config, 0, sizeof(m_config)); m_config_count = 0; @@ -875,7 +929,11 @@ cat_exit(void) if (CPU_ISSET(cpu_id, &m_config[i].cpumask) == 0) continue; +#if PQOS_VERSION <= 103 ret = pqos_l3ca_assoc_set(cpu_id, 0); +#else + ret = pqos_alloc_assoc_set(cpu_id, 0); +#endif if (ret != PQOS_RETVAL_OK) { printf("PQOS: Failed to associate COS 0 to " "cpu %u\n", cpu_id); @@ -927,7 +985,9 @@ cat_init(int argc, char **argv) /* PQoS Initialization - Check and initialize CAT capability */ cfg.fd_log = STDOUT_FILENO; cfg.verbose = 0; +#if PQOS_VERSION <= 103 cfg.cdp_cfg = PQOS_REQUIRE_CDP_ANY; +#endif ret = pqos_init(&cfg); if (ret != PQOS_RETVAL_OK) { printf("PQOS: Error initializing PQoS library!\n"); @@ -953,9 +1013,14 @@ cat_init(int argc, char **argv) } /* Get CPU socket information */ +#if PQOS_VERSION <= 103 ret = pqos_cpu_get_sockets(m_cpu, PQOS_MAX_SOCKETS, &m_sock_count, m_sockets); if (ret != PQOS_RETVAL_OK) { +#else + m_sockets = pqos_cpu_get_sockets(m_cpu, &m_sock_count); + if (m_sockets == NULL) { +#endif printf("PQOS: Error retrieving CPU socket information!\n"); ret = -EFAULT; goto err; diff --git a/dpdk/examples/l2fwd-cat/l2fwd-cat.c b/dpdk/examples/l2fwd-cat/l2fwd-cat.c index 8cce33b8..b10ac896 100644 --- a/dpdk/examples/l2fwd-cat/l2fwd-cat.c +++ b/dpdk/examples/l2fwd-cat/l2fwd-cat.c @@ -59,12 +59,14 @@ static const struct rte_eth_conf port_conf_default = { * coming from the mbuf_pool passed as a parameter. */ static inline int -port_init(uint8_t port, struct rte_mempool *mbuf_pool) +port_init(uint16_t port, struct rte_mempool *mbuf_pool) { struct rte_eth_conf port_conf = port_conf_default; const uint16_t rx_rings = 1, tx_rings = 1; int retval; uint16_t q; + uint16_t nb_rxd = RX_RING_SIZE; + uint16_t nb_txd = TX_RING_SIZE; if (port >= rte_eth_dev_count()) return -1; @@ -74,9 +76,13 @@ port_init(uint8_t port, struct rte_mempool *mbuf_pool) if (retval != 0) return retval; + retval = rte_eth_dev_adjust_nb_rx_tx_desc(port, &nb_rxd, &nb_txd); + if (retval != 0) + return retval; + /* Allocate and set up 1 RX queue per Ethernet port. */ for (q = 0; q < rx_rings; q++) { - retval = rte_eth_rx_queue_setup(port, q, RX_RING_SIZE, + retval = rte_eth_rx_queue_setup(port, q, nb_rxd, rte_eth_dev_socket_id(port), NULL, mbuf_pool); if (retval < 0) return retval; @@ -84,7 +90,7 @@ port_init(uint8_t port, struct rte_mempool *mbuf_pool) /* Allocate and set up 1 TX queue per Ethernet port. */ for (q = 0; q < tx_rings; q++) { - retval = rte_eth_tx_queue_setup(port, q, TX_RING_SIZE, + retval = rte_eth_tx_queue_setup(port, q, nb_txd, rte_eth_dev_socket_id(port), NULL); if (retval < 0) return retval; @@ -100,7 +106,7 @@ port_init(uint8_t port, struct rte_mempool *mbuf_pool) rte_eth_macaddr_get(port, &addr); printf("Port %u MAC: %02" PRIx8 " %02" PRIx8 " %02" PRIx8 " %02" PRIx8 " %02" PRIx8 " %02" PRIx8 "\n", - (unsigned)port, + port, addr.addr_bytes[0], addr.addr_bytes[1], addr.addr_bytes[2], addr.addr_bytes[3], addr.addr_bytes[4], addr.addr_bytes[5]); @@ -118,8 +124,8 @@ port_init(uint8_t port, struct rte_mempool *mbuf_pool) static __attribute__((noreturn)) void lcore_main(void) { - const uint8_t nb_ports = rte_eth_dev_count(); - uint8_t port; + const uint16_t nb_ports = rte_eth_dev_count(); + uint16_t port; /* * Check that the port is on the same NUMA node as the polling thread @@ -175,7 +181,7 @@ main(int argc, char *argv[]) { struct rte_mempool *mbuf_pool; unsigned nb_ports; - uint8_t portid; + uint16_t portid; /* Initialize the Environment Abstraction Layer (EAL). */ int ret = rte_eal_init(argc, argv); @@ -211,7 +217,7 @@ main(int argc, char *argv[]) /* Initialize all ports. */ for (portid = 0; portid < nb_ports; portid++) if (port_init(portid, mbuf_pool) != 0) - rte_exit(EXIT_FAILURE, "Cannot init port %"PRIu8 "\n", + rte_exit(EXIT_FAILURE, "Cannot init port %"PRIu16 "\n", portid); if (rte_lcore_count() > 1) diff --git a/dpdk/examples/l2fwd-crypto/main.c b/dpdk/examples/l2fwd-crypto/main.c index 6cfa9168..d4e1682c 100644 --- a/dpdk/examples/l2fwd-crypto/main.c +++ b/dpdk/examples/l2fwd-crypto/main.c @@ -67,12 +67,9 @@ #include #include #include -#include -#include #include #include #include -#include #include enum cdev_type { @@ -87,8 +84,16 @@ enum cdev_type { #define MAX_STR_LEN 32 #define MAX_KEY_SIZE 128 +#define MAX_IV_SIZE 16 +#define MAX_AAD_SIZE 65535 #define MAX_PKT_BURST 32 #define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */ +#define MAX_SESSIONS 32 +#define SESSION_POOL_CACHE_SIZE 0 + +#define MAXIMUM_IV_LENGTH 16 +#define IV_OFFSET (sizeof(struct rte_crypto_op) + \ + sizeof(struct rte_crypto_sym_op)) /* * Configurable number of RX/TX ring descriptors @@ -107,7 +112,7 @@ static uint64_t l2fwd_enabled_port_mask; static uint64_t l2fwd_enabled_crypto_mask; /* list of enabled ports */ -static uint32_t l2fwd_dst_ports[RTE_MAX_ETHPORTS]; +static uint16_t l2fwd_dst_ports[RTE_MAX_ETHPORTS]; struct pkt_buffer { @@ -127,17 +132,20 @@ enum l2fwd_crypto_xform_chain { L2FWD_CRYPTO_CIPHER_HASH, L2FWD_CRYPTO_HASH_CIPHER, L2FWD_CRYPTO_CIPHER_ONLY, - L2FWD_CRYPTO_HASH_ONLY + L2FWD_CRYPTO_HASH_ONLY, + L2FWD_CRYPTO_AEAD }; struct l2fwd_key { uint8_t *data; uint32_t length; - phys_addr_t phys_addr; + rte_iova_t phys_addr; }; -char supported_auth_algo[RTE_CRYPTO_AUTH_LIST_END][MAX_STR_LEN]; -char supported_cipher_algo[RTE_CRYPTO_CIPHER_LIST_END][MAX_STR_LEN]; +struct l2fwd_iv { + uint8_t *data; + uint16_t length; +}; /** l2fwd crypto application command line options */ struct l2fwd_crypto_options { @@ -155,14 +163,26 @@ struct l2fwd_crypto_options { unsigned ckey_param; int ckey_random_size; - struct l2fwd_key iv; - unsigned iv_param; - int iv_random_size; + struct l2fwd_iv cipher_iv; + unsigned int cipher_iv_param; + int cipher_iv_random_size; struct rte_crypto_sym_xform auth_xform; uint8_t akey_param; int akey_random_size; + struct l2fwd_iv auth_iv; + unsigned int auth_iv_param; + int auth_iv_random_size; + + struct rte_crypto_sym_xform aead_xform; + unsigned int aead_key_param; + int aead_key_random_size; + + struct l2fwd_iv aead_iv; + unsigned int aead_iv_param; + int aead_iv_random_size; + struct l2fwd_key aad; unsigned aad_param; int aad_random_size; @@ -171,6 +191,10 @@ struct l2fwd_crypto_options { uint16_t block_size; char string_type[MAX_STR_LEN]; + + uint64_t cryptodev_mask; + + unsigned int mac_updating; }; /** l2fwd crypto lcore params */ @@ -181,27 +205,31 @@ struct l2fwd_crypto_params { unsigned digest_length; unsigned block_size; - struct l2fwd_key iv; + struct l2fwd_iv cipher_iv; + struct l2fwd_iv auth_iv; + struct l2fwd_iv aead_iv; struct l2fwd_key aad; struct rte_cryptodev_sym_session *session; uint8_t do_cipher; uint8_t do_hash; + uint8_t do_aead; uint8_t hash_verify; enum rte_crypto_cipher_algorithm cipher_algo; enum rte_crypto_auth_algorithm auth_algo; + enum rte_crypto_aead_algorithm aead_algo; }; /** lcore configuration */ struct lcore_queue_conf { unsigned nb_rx_ports; - unsigned rx_port_list[MAX_RX_QUEUE_PER_LCORE]; + uint16_t rx_port_list[MAX_RX_QUEUE_PER_LCORE]; unsigned nb_crypto_devs; unsigned cryptodev_list[MAX_RX_QUEUE_PER_LCORE]; - struct op_buffer op_buf[RTE_MAX_ETHPORTS]; + struct op_buffer op_buf[RTE_CRYPTO_MAX_DEVS]; struct pkt_buffer pkt_buf[RTE_MAX_ETHPORTS]; } __rte_cache_aligned; @@ -216,7 +244,7 @@ static const struct rte_eth_conf port_conf = { .hw_ip_checksum = 0, /**< IP checksum offload disabled */ .hw_vlan_filter = 0, /**< VLAN filtering disabled */ .jumbo_frame = 0, /**< Jumbo Frame Support disabled */ - .hw_strip_crc = 0, /**< CRC stripped by hardware */ + .hw_strip_crc = 1, /**< CRC stripped by hardware */ }, .txmode = { .mq_mode = ETH_MQ_TX_NONE, @@ -225,6 +253,7 @@ static const struct rte_eth_conf port_conf = { struct rte_mempool *l2fwd_pktmbuf_pool; struct rte_mempool *l2fwd_crypto_op_pool; +struct rte_mempool *session_pool_socket[RTE_MAX_NUMA_NODES] = { 0 }; /* Per-port statistics struct */ struct l2fwd_port_statistics { @@ -261,7 +290,7 @@ print_stats(void) uint64_t total_packets_dropped, total_packets_tx, total_packets_rx; uint64_t total_packets_enqueued, total_packets_dequeued, total_packets_errors; - unsigned portid; + uint16_t portid; uint64_t cdevid; total_packets_dropped = 0; @@ -300,7 +329,7 @@ print_stats(void) for (cdevid = 0; cdevid < RTE_CRYPTO_MAX_DEVS; cdevid++) { /* skip disabled ports */ - if ((l2fwd_enabled_crypto_mask & (1lu << cdevid)) == 0) + if ((l2fwd_enabled_crypto_mask & (((uint64_t)1) << cdevid)) == 0) continue; printf("\nStatistics for cryptodev %"PRIu64 " -------------------------" @@ -332,39 +361,6 @@ print_stats(void) printf("\n====================================================\n"); } -static void -fill_supported_algorithm_tables(void) -{ - unsigned i; - - for (i = 0; i < RTE_CRYPTO_AUTH_LIST_END; i++) - strcpy(supported_auth_algo[i], "NOT_SUPPORTED"); - - strcpy(supported_auth_algo[RTE_CRYPTO_AUTH_AES_GCM], "AES_GCM"); - strcpy(supported_auth_algo[RTE_CRYPTO_AUTH_MD5_HMAC], "MD5_HMAC"); - strcpy(supported_auth_algo[RTE_CRYPTO_AUTH_NULL], "NULL"); - strcpy(supported_auth_algo[RTE_CRYPTO_AUTH_AES_XCBC_MAC], - "AES_XCBC_MAC"); - strcpy(supported_auth_algo[RTE_CRYPTO_AUTH_SHA1_HMAC], "SHA1_HMAC"); - strcpy(supported_auth_algo[RTE_CRYPTO_AUTH_SHA224_HMAC], "SHA224_HMAC"); - strcpy(supported_auth_algo[RTE_CRYPTO_AUTH_SHA256_HMAC], "SHA256_HMAC"); - strcpy(supported_auth_algo[RTE_CRYPTO_AUTH_SHA384_HMAC], "SHA384_HMAC"); - strcpy(supported_auth_algo[RTE_CRYPTO_AUTH_SHA512_HMAC], "SHA512_HMAC"); - strcpy(supported_auth_algo[RTE_CRYPTO_AUTH_SNOW3G_UIA2], "SNOW3G_UIA2"); - strcpy(supported_auth_algo[RTE_CRYPTO_AUTH_KASUMI_F9], "KASUMI_F9"); - - for (i = 0; i < RTE_CRYPTO_CIPHER_LIST_END; i++) - strcpy(supported_cipher_algo[i], "NOT_SUPPORTED"); - - strcpy(supported_cipher_algo[RTE_CRYPTO_CIPHER_AES_CBC], "AES_CBC"); - strcpy(supported_cipher_algo[RTE_CRYPTO_CIPHER_AES_CTR], "AES_CTR"); - strcpy(supported_cipher_algo[RTE_CRYPTO_CIPHER_AES_GCM], "AES_GCM"); - strcpy(supported_cipher_algo[RTE_CRYPTO_CIPHER_NULL], "NULL"); - strcpy(supported_cipher_algo[RTE_CRYPTO_CIPHER_SNOW3G_UEA2], "SNOW3G_UEA2"); - strcpy(supported_cipher_algo[RTE_CRYPTO_CIPHER_KASUMI_F8], "KASUMI_F8"); -} - - static int l2fwd_crypto_send_burst(struct lcore_queue_conf *qconf, unsigned n, struct l2fwd_crypto_params *cparams) @@ -422,7 +418,8 @@ l2fwd_simple_crypto_enqueue(struct rte_mbuf *m, struct ether_hdr *eth_hdr; struct ipv4_hdr *ip_hdr; - unsigned ipdata_offset, pad_len, data_len; + uint32_t ipdata_offset, data_len; + uint32_t pad_len = 0; char *padding; eth_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *); @@ -445,22 +442,51 @@ l2fwd_simple_crypto_enqueue(struct rte_mbuf *m, if (cparams->do_hash && cparams->hash_verify) data_len -= cparams->digest_length; - pad_len = data_len % cparams->block_size ? cparams->block_size - - (data_len % cparams->block_size) : 0; + if (cparams->do_cipher) { + /* + * Following algorithms are block cipher algorithms, + * and might need padding + */ + switch (cparams->cipher_algo) { + case RTE_CRYPTO_CIPHER_AES_CBC: + case RTE_CRYPTO_CIPHER_AES_ECB: + case RTE_CRYPTO_CIPHER_DES_CBC: + case RTE_CRYPTO_CIPHER_3DES_CBC: + case RTE_CRYPTO_CIPHER_3DES_ECB: + if (data_len % cparams->block_size) + pad_len = cparams->block_size - + (data_len % cparams->block_size); + break; + default: + pad_len = 0; + } - if (pad_len) { - padding = rte_pktmbuf_append(m, pad_len); - if (unlikely(!padding)) - return -1; + if (pad_len) { + padding = rte_pktmbuf_append(m, pad_len); + if (unlikely(!padding)) + return -1; - data_len += pad_len; - memset(padding, 0, pad_len); + data_len += pad_len; + memset(padding, 0, pad_len); + } } /* Set crypto operation data parameters */ rte_crypto_op_attach_sym_session(op, cparams->session); if (cparams->do_hash) { + if (cparams->auth_iv.length) { + uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, + uint8_t *, + IV_OFFSET + + cparams->cipher_iv.length); + /* + * Copy IV at the end of the crypto operation, + * after the cipher IV, if added + */ + rte_memcpy(iv_ptr, cparams->auth_iv.data, + cparams->auth_iv.length); + } if (!cparams->hash_verify) { /* Append space for digest to end of packet */ op->sym->auth.digest.data = (uint8_t *)rte_pktmbuf_append(m, @@ -470,35 +496,32 @@ l2fwd_simple_crypto_enqueue(struct rte_mbuf *m, uint8_t *) + ipdata_offset + data_len; } - op->sym->auth.digest.phys_addr = rte_pktmbuf_mtophys_offset(m, + op->sym->auth.digest.phys_addr = rte_pktmbuf_iova_offset(m, rte_pktmbuf_pkt_len(m) - cparams->digest_length); - op->sym->auth.digest.length = cparams->digest_length; - /* For SNOW3G/KASUMI algorithms, offset/length must be in bits */ + /* For wireless algorithms, offset/length must be in bits */ if (cparams->auth_algo == RTE_CRYPTO_AUTH_SNOW3G_UIA2 || - cparams->auth_algo == RTE_CRYPTO_AUTH_KASUMI_F9) { + cparams->auth_algo == RTE_CRYPTO_AUTH_KASUMI_F9 || + cparams->auth_algo == RTE_CRYPTO_AUTH_ZUC_EIA3) { op->sym->auth.data.offset = ipdata_offset << 3; op->sym->auth.data.length = data_len << 3; } else { op->sym->auth.data.offset = ipdata_offset; op->sym->auth.data.length = data_len; } - - if (cparams->aad.length) { - op->sym->auth.aad.data = cparams->aad.data; - op->sym->auth.aad.phys_addr = cparams->aad.phys_addr; - op->sym->auth.aad.length = cparams->aad.length; - } } if (cparams->do_cipher) { - op->sym->cipher.iv.data = cparams->iv.data; - op->sym->cipher.iv.phys_addr = cparams->iv.phys_addr; - op->sym->cipher.iv.length = cparams->iv.length; + uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, + IV_OFFSET); + /* Copy IV at the end of the crypto operation */ + rte_memcpy(iv_ptr, cparams->cipher_iv.data, + cparams->cipher_iv.length); - /* For SNOW3G algorithms, offset/length must be in bits */ + /* For wireless algorithms, offset/length must be in bits */ if (cparams->cipher_algo == RTE_CRYPTO_CIPHER_SNOW3G_UEA2 || - cparams->cipher_algo == RTE_CRYPTO_CIPHER_KASUMI_F8) { + cparams->cipher_algo == RTE_CRYPTO_CIPHER_KASUMI_F8 || + cparams->cipher_algo == RTE_CRYPTO_CIPHER_ZUC_EEA3) { op->sym->cipher.data.offset = ipdata_offset << 3; op->sym->cipher.data.length = data_len << 3; } else { @@ -507,6 +530,42 @@ l2fwd_simple_crypto_enqueue(struct rte_mbuf *m, } } + if (cparams->do_aead) { + uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, + IV_OFFSET); + /* Copy IV at the end of the crypto operation */ + /* + * If doing AES-CCM, nonce is copied one byte + * after the start of IV field + */ + if (cparams->aead_algo == RTE_CRYPTO_AEAD_AES_CCM) + rte_memcpy(iv_ptr + 1, cparams->aead_iv.data, + cparams->aead_iv.length); + else + rte_memcpy(iv_ptr, cparams->aead_iv.data, + cparams->aead_iv.length); + + op->sym->aead.data.offset = ipdata_offset; + op->sym->aead.data.length = data_len; + + if (!cparams->hash_verify) { + /* Append space for digest to end of packet */ + op->sym->aead.digest.data = (uint8_t *)rte_pktmbuf_append(m, + cparams->digest_length); + } else { + op->sym->aead.digest.data = rte_pktmbuf_mtod(m, + uint8_t *) + ipdata_offset + data_len; + } + + op->sym->aead.digest.phys_addr = rte_pktmbuf_iova_offset(m, + rte_pktmbuf_pkt_len(m) - cparams->digest_length); + + if (cparams->aad.length) { + op->sym->aead.aad.data = cparams->aad.data; + op->sym->aead.aad.phys_addr = cparams->aad.phys_addr; + } + } + op->sym->m_src = m; return l2fwd_crypto_enqueue(op, cparams); @@ -516,7 +575,7 @@ l2fwd_simple_crypto_enqueue(struct rte_mbuf *m, /* Send the burst of packets on an output interface */ static int l2fwd_send_burst(struct lcore_queue_conf *qconf, unsigned n, - uint8_t port) + uint16_t port) { struct rte_mbuf **pkt_buffer; unsigned ret; @@ -537,7 +596,7 @@ l2fwd_send_burst(struct lcore_queue_conf *qconf, unsigned n, /* Enqueue packets for TX and prepare them to be sent */ static int -l2fwd_send_packet(struct rte_mbuf *m, uint8_t port) +l2fwd_send_packet(struct rte_mbuf *m, uint16_t port) { unsigned lcore_id, len; struct lcore_queue_conf *qconf; @@ -560,23 +619,33 @@ l2fwd_send_packet(struct rte_mbuf *m, uint8_t port) } static void -l2fwd_simple_forward(struct rte_mbuf *m, unsigned portid) +l2fwd_mac_updating(struct rte_mbuf *m, uint16_t dest_portid) { struct ether_hdr *eth; void *tmp; - unsigned dst_port; - dst_port = l2fwd_dst_ports[portid]; eth = rte_pktmbuf_mtod(m, struct ether_hdr *); /* 02:00:00:00:00:xx */ tmp = ð->d_addr.addr_bytes[0]; - *((uint64_t *)tmp) = 0x000000000002 + ((uint64_t)dst_port << 40); + *((uint64_t *)tmp) = 0x000000000002 + ((uint64_t)dest_portid << 40); /* src addr */ - ether_addr_copy(&l2fwd_ports_eth_addr[dst_port], ð->s_addr); + ether_addr_copy(&l2fwd_ports_eth_addr[dest_portid], ð->s_addr); +} - l2fwd_send_packet(m, (uint8_t) dst_port); +static void +l2fwd_simple_forward(struct rte_mbuf *m, uint16_t portid, + struct l2fwd_crypto_options *options) +{ + uint16_t dst_port; + + dst_port = l2fwd_dst_ports[portid]; + + if (options->mac_updating) + l2fwd_mac_updating(m, dst_port); + + l2fwd_send_packet(m, dst_port); } /** Generate random key */ @@ -598,12 +667,21 @@ generate_random_key(uint8_t *key, unsigned length) } static struct rte_cryptodev_sym_session * -initialize_crypto_session(struct l2fwd_crypto_options *options, - uint8_t cdev_id) +initialize_crypto_session(struct l2fwd_crypto_options *options, uint8_t cdev_id) { struct rte_crypto_sym_xform *first_xform; + struct rte_cryptodev_sym_session *session; + int retval = rte_cryptodev_socket_id(cdev_id); - if (options->xform_chain == L2FWD_CRYPTO_CIPHER_HASH) { + if (retval < 0) + return NULL; + + uint8_t socket_id = (uint8_t) retval; + struct rte_mempool *sess_mp = session_pool_socket[socket_id]; + + if (options->xform_chain == L2FWD_CRYPTO_AEAD) { + first_xform = &options->aead_xform; + } else if (options->xform_chain == L2FWD_CRYPTO_CIPHER_HASH) { first_xform = &options->cipher_xform; first_xform->next = &options->auth_xform; } else if (options->xform_chain == L2FWD_CRYPTO_HASH_CIPHER) { @@ -615,8 +693,16 @@ initialize_crypto_session(struct l2fwd_crypto_options *options, first_xform = &options->auth_xform; } - /* Setup Cipher Parameters */ - return rte_cryptodev_sym_session_create(cdev_id, first_xform); + session = rte_cryptodev_sym_session_create(sess_mp); + + if (session == NULL) + return NULL; + + if (rte_cryptodev_sym_session_init(cdev_id, session, + first_xform, sess_mp) < 0) + return NULL; + + return session; } static void @@ -631,12 +717,14 @@ l2fwd_main_loop(struct l2fwd_crypto_options *options) unsigned lcore_id = rte_lcore_id(); uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; - unsigned i, j, portid, nb_rx, len; + unsigned int i, j, nb_rx, len; + uint16_t portid; struct lcore_queue_conf *qconf = &lcore_queue_conf[lcore_id]; const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / US_PER_S * BURST_TX_DRAIN_US; struct l2fwd_crypto_params *cparams; struct l2fwd_crypto_params port_cparams[qconf->nb_crypto_devs]; + struct rte_cryptodev_sym_session *session; if (qconf->nb_rx_ports == 0) { RTE_LOG(INFO, L2FWD, "lcore %u has nothing to do\n", lcore_id); @@ -655,8 +743,12 @@ l2fwd_main_loop(struct l2fwd_crypto_options *options) for (i = 0; i < qconf->nb_crypto_devs; i++) { port_cparams[i].do_cipher = 0; port_cparams[i].do_hash = 0; + port_cparams[i].do_aead = 0; switch (options->xform_chain) { + case L2FWD_CRYPTO_AEAD: + port_cparams[i].do_aead = 1; + break; case L2FWD_CRYPTO_CIPHER_HASH: case L2FWD_CRYPTO_HASH_CIPHER: port_cparams[i].do_cipher = 1; @@ -676,43 +768,87 @@ l2fwd_main_loop(struct l2fwd_crypto_options *options) port_cparams[i].block_size = options->block_size; if (port_cparams[i].do_hash) { - port_cparams[i].digest_length = - options->auth_xform.auth.digest_length; - if (options->auth_xform.auth.add_auth_data_length) { - port_cparams[i].aad.data = options->aad.data; - port_cparams[i].aad.length = - options->auth_xform.auth.add_auth_data_length; - port_cparams[i].aad.phys_addr = options->aad.phys_addr; - if (!options->aad_param) - generate_random_key(port_cparams[i].aad.data, - port_cparams[i].aad.length); - - } - + port_cparams[i].auth_iv.data = options->auth_iv.data; + port_cparams[i].auth_iv.length = options->auth_iv.length; + if (!options->auth_iv_param) + generate_random_key(port_cparams[i].auth_iv.data, + port_cparams[i].auth_iv.length); if (options->auth_xform.auth.op == RTE_CRYPTO_AUTH_OP_VERIFY) port_cparams[i].hash_verify = 1; else port_cparams[i].hash_verify = 0; port_cparams[i].auth_algo = options->auth_xform.auth.algo; + port_cparams[i].digest_length = + options->auth_xform.auth.digest_length; + /* Set IV parameters */ + if (options->auth_iv.length) { + options->auth_xform.auth.iv.offset = + IV_OFFSET + options->cipher_iv.length; + options->auth_xform.auth.iv.length = + options->auth_iv.length; + } + } + + if (port_cparams[i].do_aead) { + port_cparams[i].aead_iv.data = options->aead_iv.data; + port_cparams[i].aead_iv.length = options->aead_iv.length; + if (!options->aead_iv_param) + generate_random_key(port_cparams[i].aead_iv.data, + port_cparams[i].aead_iv.length); + port_cparams[i].aead_algo = options->aead_xform.aead.algo; + port_cparams[i].digest_length = + options->aead_xform.aead.digest_length; + if (options->aead_xform.aead.aad_length) { + port_cparams[i].aad.data = options->aad.data; + port_cparams[i].aad.phys_addr = options->aad.phys_addr; + port_cparams[i].aad.length = options->aad.length; + if (!options->aad_param) + generate_random_key(port_cparams[i].aad.data, + port_cparams[i].aad.length); + /* + * If doing AES-CCM, first 18 bytes has to be reserved, + * and actual AAD should start from byte 18 + */ + if (port_cparams[i].aead_algo == RTE_CRYPTO_AEAD_AES_CCM) + memmove(port_cparams[i].aad.data + 18, + port_cparams[i].aad.data, + port_cparams[i].aad.length); + + } else + port_cparams[i].aad.length = 0; + + if (options->aead_xform.aead.op == RTE_CRYPTO_AEAD_OP_DECRYPT) + port_cparams[i].hash_verify = 1; + else + port_cparams[i].hash_verify = 0; + + /* Set IV parameters */ + options->aead_xform.aead.iv.offset = IV_OFFSET; + options->aead_xform.aead.iv.length = options->aead_iv.length; } if (port_cparams[i].do_cipher) { - port_cparams[i].iv.data = options->iv.data; - port_cparams[i].iv.length = options->iv.length; - port_cparams[i].iv.phys_addr = options->iv.phys_addr; - if (!options->iv_param) - generate_random_key(port_cparams[i].iv.data, - port_cparams[i].iv.length); + port_cparams[i].cipher_iv.data = options->cipher_iv.data; + port_cparams[i].cipher_iv.length = options->cipher_iv.length; + if (!options->cipher_iv_param) + generate_random_key(port_cparams[i].cipher_iv.data, + port_cparams[i].cipher_iv.length); port_cparams[i].cipher_algo = options->cipher_xform.cipher.algo; + /* Set IV parameters */ + options->cipher_xform.cipher.iv.offset = IV_OFFSET; + options->cipher_xform.cipher.iv.length = + options->cipher_iv.length; } - port_cparams[i].session = initialize_crypto_session(options, + session = initialize_crypto_session(options, port_cparams[i].dev_id); + if (session == NULL) + rte_exit(EXIT_FAILURE, "Failed to initialize crypto session\n"); + + port_cparams[i].session = session; - if (port_cparams[i].session == NULL) - return; RTE_LOG(INFO, L2FWD, " -- lcoreid=%u cryptoid=%u\n", lcore_id, port_cparams[i].dev_id); } @@ -747,7 +883,7 @@ l2fwd_main_loop(struct l2fwd_crypto_options *options) continue; l2fwd_send_burst(&lcore_queue_conf[lcore_id], qconf->pkt_buf[portid].len, - (uint8_t) portid); + portid); qconf->pkt_buf[portid].len = 0; } @@ -781,7 +917,7 @@ l2fwd_main_loop(struct l2fwd_crypto_options *options) cparams = &port_cparams[i]; - nb_rx = rte_eth_rx_burst((uint8_t) portid, 0, + nb_rx = rte_eth_rx_burst(portid, 0, pkts_burst, MAX_PKT_BURST); port_statistics[portid].rx += nb_rx; @@ -798,7 +934,7 @@ l2fwd_main_loop(struct l2fwd_crypto_options *options) ops_burst, nb_rx) != nb_rx) { for (j = 0; j < nb_rx; j++) - rte_pktmbuf_free(pkts_burst[i]); + rte_pktmbuf_free(pkts_burst[j]); nb_rx = 0; } @@ -826,7 +962,8 @@ l2fwd_main_loop(struct l2fwd_crypto_options *options) m = ops_burst[j]->sym->m_src; rte_crypto_op_free(ops_burst[j]); - l2fwd_simple_forward(m, portid); + l2fwd_simple_forward(m, portid, + options); } } while (nb_rx == MAX_PKT_BURST); } @@ -852,24 +989,41 @@ l2fwd_crypto_usage(const char *prgname) " (0 to disable, 10 default, 86400 maximum)\n" " --cdev_type HW / SW / ANY\n" - " --chain HASH_CIPHER / CIPHER_HASH\n" + " --chain HASH_CIPHER / CIPHER_HASH / CIPHER_ONLY /" + " HASH_ONLY / AEAD\n" " --cipher_algo ALGO\n" " --cipher_op ENCRYPT / DECRYPT\n" " --cipher_key KEY (bytes separated with \":\")\n" " --cipher_key_random_size SIZE: size of cipher key when generated randomly\n" - " --iv IV (bytes separated with \":\")\n" - " --iv_random_size SIZE: size of IV when generated randomly\n" + " --cipher_iv IV (bytes separated with \":\")\n" + " --cipher_iv_random_size SIZE: size of cipher IV when generated randomly\n" " --auth_algo ALGO\n" " --auth_op GENERATE / VERIFY\n" " --auth_key KEY (bytes separated with \":\")\n" " --auth_key_random_size SIZE: size of auth key when generated randomly\n" + " --auth_iv IV (bytes separated with \":\")\n" + " --auth_iv_random_size SIZE: size of auth IV when generated randomly\n" + + " --aead_algo ALGO\n" + " --aead_op ENCRYPT / DECRYPT\n" + " --aead_key KEY (bytes separated with \":\")\n" + " --aead_key_random_size SIZE: size of AEAD key when generated randomly\n" + " --aead_iv IV (bytes separated with \":\")\n" + " --aead_iv_random_size SIZE: size of AEAD IV when generated randomly\n" " --aad AAD (bytes separated with \":\")\n" " --aad_random_size SIZE: size of AAD when generated randomly\n" + " --digest_size SIZE: size of digest to be generated/verified\n" - " --sessionless\n", + " --sessionless\n" + " --cryptodev_mask MASK: hexadecimal bitmask of crypto devices to configure\n" + + " --[no-]mac-updating: Enable or disable MAC addresses updating (enabled by default)\n" + " When enabled:\n" + " - The source MAC address is replaced by the TX port MAC address\n" + " - The destination MAC address is replaced by 02:00:00:00:00:TX_PORT_ID\n", prgname); } @@ -907,6 +1061,9 @@ parse_crypto_opt_chain(struct l2fwd_crypto_options *options, char *optarg) } else if (strcmp("HASH_ONLY", optarg) == 0) { options->xform_chain = L2FWD_CRYPTO_HASH_ONLY; return 0; + } else if (strcmp("AEAD", optarg) == 0) { + options->xform_chain = L2FWD_CRYPTO_AEAD; + return 0; } return -1; @@ -916,17 +1073,14 @@ parse_crypto_opt_chain(struct l2fwd_crypto_options *options, char *optarg) static int parse_cipher_algo(enum rte_crypto_cipher_algorithm *algo, char *optarg) { - unsigned i; - for (i = 0; i < RTE_CRYPTO_CIPHER_LIST_END; i++) { - if (!strcmp(supported_cipher_algo[i], optarg)) { - *algo = (enum rte_crypto_cipher_algorithm)i; - return 0; - } + if (rte_cryptodev_get_cipher_algo_enum(algo, optarg) < 0) { + RTE_LOG(ERR, USER1, "Cipher algorithm specified " + "not supported!\n"); + return -1; } - printf("Cipher algorithm not supported!\n"); - return -1; + return 0; } /** Parse crypto cipher operation command line argument */ @@ -945,15 +1099,16 @@ parse_cipher_op(enum rte_crypto_cipher_operation *op, char *optarg) return -1; } -/** Parse crypto key command line argument */ +/** Parse bytes from command line argument */ static int -parse_key(uint8_t *data, char *input_arg) +parse_bytes(uint8_t *data, char *input_arg, uint16_t max_size) { unsigned byte_count; char *token; + errno = 0; for (byte_count = 0, token = strtok(input_arg, ":"); - (byte_count < MAX_KEY_SIZE) && (token != NULL); + (byte_count < max_size) && (token != NULL); token = strtok(NULL, ":")) { int number = (int)strtol(token, NULL, 16); @@ -992,17 +1147,13 @@ parse_size(int *size, const char *q_arg) static int parse_auth_algo(enum rte_crypto_auth_algorithm *algo, char *optarg) { - unsigned i; - - for (i = 0; i < RTE_CRYPTO_AUTH_LIST_END; i++) { - if (!strcmp(supported_auth_algo[i], optarg)) { - *algo = (enum rte_crypto_auth_algorithm)i; - return 0; - } + if (rte_cryptodev_get_auth_algo_enum(algo, optarg) < 0) { + RTE_LOG(ERR, USER1, "Authentication algorithm specified " + "not supported!\n"); + return -1; } - printf("Authentication algorithm specified not supported!\n"); - return -1; + return 0; } static int @@ -1020,6 +1171,53 @@ parse_auth_op(enum rte_crypto_auth_operation *op, char *optarg) return -1; } +static int +parse_aead_algo(enum rte_crypto_aead_algorithm *algo, char *optarg) +{ + if (rte_cryptodev_get_aead_algo_enum(algo, optarg) < 0) { + RTE_LOG(ERR, USER1, "AEAD algorithm specified " + "not supported!\n"); + return -1; + } + + return 0; +} + +static int +parse_aead_op(enum rte_crypto_aead_operation *op, char *optarg) +{ + if (strcmp("ENCRYPT", optarg) == 0) { + *op = RTE_CRYPTO_AEAD_OP_ENCRYPT; + return 0; + } else if (strcmp("DECRYPT", optarg) == 0) { + *op = RTE_CRYPTO_AEAD_OP_DECRYPT; + return 0; + } + + printf("AEAD operation specified not supported!\n"); + return -1; +} +static int +parse_cryptodev_mask(struct l2fwd_crypto_options *options, + const char *q_arg) +{ + char *end = NULL; + uint64_t pm; + + /* parse hexadecimal string */ + pm = strtoul(q_arg, &end, 16); + if ((pm == '\0') || (end == NULL) || (*end != '\0')) + pm = 0; + + options->cryptodev_mask = pm; + if (options->cryptodev_mask == 0) { + printf("invalid cryptodev_mask specified\n"); + return -1; + } + + return 0; +} + /** Parse long options */ static int l2fwd_crypto_parse_args_long_options(struct l2fwd_crypto_options *options, @@ -1050,7 +1248,8 @@ l2fwd_crypto_parse_args_long_options(struct l2fwd_crypto_options *options, else if (strcmp(lgopts[option_index].name, "cipher_key") == 0) { options->ckey_param = 1; options->cipher_xform.cipher.key.length = - parse_key(options->cipher_xform.cipher.key.data, optarg); + parse_bytes(options->cipher_xform.cipher.key.data, optarg, + MAX_KEY_SIZE); if (options->cipher_xform.cipher.key.length > 0) return 0; else @@ -1060,18 +1259,18 @@ l2fwd_crypto_parse_args_long_options(struct l2fwd_crypto_options *options, else if (strcmp(lgopts[option_index].name, "cipher_key_random_size") == 0) return parse_size(&options->ckey_random_size, optarg); - else if (strcmp(lgopts[option_index].name, "iv") == 0) { - options->iv_param = 1; - options->iv.length = - parse_key(options->iv.data, optarg); - if (options->iv.length > 0) + else if (strcmp(lgopts[option_index].name, "cipher_iv") == 0) { + options->cipher_iv_param = 1; + options->cipher_iv.length = + parse_bytes(options->cipher_iv.data, optarg, MAX_IV_SIZE); + if (options->cipher_iv.length > 0) return 0; else return -1; } - else if (strcmp(lgopts[option_index].name, "iv_random_size") == 0) - return parse_size(&options->iv_random_size, optarg); + else if (strcmp(lgopts[option_index].name, "cipher_iv_random_size") == 0) + return parse_size(&options->cipher_iv_random_size, optarg); /* Authentication options */ else if (strcmp(lgopts[option_index].name, "auth_algo") == 0) { @@ -1086,7 +1285,8 @@ l2fwd_crypto_parse_args_long_options(struct l2fwd_crypto_options *options, else if (strcmp(lgopts[option_index].name, "auth_key") == 0) { options->akey_param = 1; options->auth_xform.auth.key.length = - parse_key(options->auth_xform.auth.key.data, optarg); + parse_bytes(options->auth_xform.auth.key.data, optarg, + MAX_KEY_SIZE); if (options->auth_xform.auth.key.length > 0) return 0; else @@ -1097,10 +1297,61 @@ l2fwd_crypto_parse_args_long_options(struct l2fwd_crypto_options *options, return parse_size(&options->akey_random_size, optarg); } + else if (strcmp(lgopts[option_index].name, "auth_iv") == 0) { + options->auth_iv_param = 1; + options->auth_iv.length = + parse_bytes(options->auth_iv.data, optarg, MAX_IV_SIZE); + if (options->auth_iv.length > 0) + return 0; + else + return -1; + } + + else if (strcmp(lgopts[option_index].name, "auth_iv_random_size") == 0) + return parse_size(&options->auth_iv_random_size, optarg); + + /* AEAD options */ + else if (strcmp(lgopts[option_index].name, "aead_algo") == 0) { + return parse_aead_algo(&options->aead_xform.aead.algo, + optarg); + } + + else if (strcmp(lgopts[option_index].name, "aead_op") == 0) + return parse_aead_op(&options->aead_xform.aead.op, + optarg); + + else if (strcmp(lgopts[option_index].name, "aead_key") == 0) { + options->aead_key_param = 1; + options->aead_xform.aead.key.length = + parse_bytes(options->aead_xform.aead.key.data, optarg, + MAX_KEY_SIZE); + if (options->aead_xform.aead.key.length > 0) + return 0; + else + return -1; + } + + else if (strcmp(lgopts[option_index].name, "aead_key_random_size") == 0) + return parse_size(&options->aead_key_random_size, optarg); + + + else if (strcmp(lgopts[option_index].name, "aead_iv") == 0) { + options->aead_iv_param = 1; + options->aead_iv.length = + parse_bytes(options->aead_iv.data, optarg, MAX_IV_SIZE); + if (options->aead_iv.length > 0) + return 0; + else + return -1; + } + + else if (strcmp(lgopts[option_index].name, "aead_iv_random_size") == 0) + return parse_size(&options->aead_iv_random_size, optarg); + else if (strcmp(lgopts[option_index].name, "aad") == 0) { options->aad_param = 1; options->aad.length = - parse_key(options->aad.data, optarg); + parse_bytes(options->aad.data, optarg, MAX_AAD_SIZE); if (options->aad.length > 0) return 0; else @@ -1120,6 +1371,19 @@ l2fwd_crypto_parse_args_long_options(struct l2fwd_crypto_options *options, return 0; } + else if (strcmp(lgopts[option_index].name, "cryptodev_mask") == 0) + return parse_cryptodev_mask(options, optarg); + + else if (strcmp(lgopts[option_index].name, "mac-updating") == 0) { + options->mac_updating = 1; + return 0; + } + + else if (strcmp(lgopts[option_index].name, "no-mac-updating") == 0) { + options->mac_updating = 0; + return 0; + } + return -1; } @@ -1212,9 +1476,9 @@ l2fwd_crypto_default_options(struct l2fwd_crypto_options *options) options->ckey_param = 0; options->ckey_random_size = -1; options->cipher_xform.cipher.key.length = 0; - options->iv_param = 0; - options->iv_random_size = -1; - options->iv.length = 0; + options->cipher_iv_param = 0; + options->cipher_iv_random_size = -1; + options->cipher_iv.length = 0; options->cipher_xform.cipher.algo = RTE_CRYPTO_CIPHER_AES_CBC; options->cipher_xform.cipher.op = RTE_CRYPTO_CIPHER_OP_ENCRYPT; @@ -1225,15 +1489,36 @@ l2fwd_crypto_default_options(struct l2fwd_crypto_options *options) options->akey_param = 0; options->akey_random_size = -1; options->auth_xform.auth.key.length = 0; - options->aad_param = 0; - options->aad_random_size = -1; - options->aad.length = 0; - options->digest_size = -1; + options->auth_iv_param = 0; + options->auth_iv_random_size = -1; + options->auth_iv.length = 0; options->auth_xform.auth.algo = RTE_CRYPTO_AUTH_SHA1_HMAC; options->auth_xform.auth.op = RTE_CRYPTO_AUTH_OP_GENERATE; + /* AEAD Data */ + options->aead_xform.type = RTE_CRYPTO_SYM_XFORM_AEAD; + options->aead_xform.next = NULL; + options->aead_key_param = 0; + options->aead_key_random_size = -1; + options->aead_xform.aead.key.length = 0; + options->aead_iv_param = 0; + options->aead_iv_random_size = -1; + options->aead_iv.length = 0; + + options->auth_xform.aead.algo = RTE_CRYPTO_AEAD_AES_GCM; + options->auth_xform.aead.op = RTE_CRYPTO_AEAD_OP_ENCRYPT; + + options->aad_param = 0; + options->aad_random_size = -1; + options->aad.length = 0; + + options->digest_size = -1; + options->type = CDEV_TYPE_ANY; + options->cryptodev_mask = UINT64_MAX; + + options->mac_updating = 1; } static void @@ -1241,11 +1526,11 @@ display_cipher_info(struct l2fwd_crypto_options *options) { printf("\n---- Cipher information ---\n"); printf("Algorithm: %s\n", - supported_cipher_algo[options->cipher_xform.cipher.algo]); + rte_crypto_cipher_algorithm_strings[options->cipher_xform.cipher.algo]); rte_hexdump(stdout, "Cipher key:", options->cipher_xform.cipher.key.data, options->cipher_xform.cipher.key.length); - rte_hexdump(stdout, "IV:", options->iv.data, options->iv.length); + rte_hexdump(stdout, "IV:", options->cipher_iv.data, options->cipher_iv.length); } static void @@ -1253,10 +1538,23 @@ display_auth_info(struct l2fwd_crypto_options *options) { printf("\n---- Authentication information ---\n"); printf("Algorithm: %s\n", - supported_auth_algo[options->auth_xform.auth.algo]); + rte_crypto_auth_algorithm_strings[options->auth_xform.auth.algo]); rte_hexdump(stdout, "Auth key:", options->auth_xform.auth.key.data, options->auth_xform.auth.key.length); + rte_hexdump(stdout, "IV:", options->auth_iv.data, options->auth_iv.length); +} + +static void +display_aead_info(struct l2fwd_crypto_options *options) +{ + printf("\n---- AEAD information ---\n"); + printf("Algorithm: %s\n", + rte_crypto_aead_algorithm_strings[options->aead_xform.aead.algo]); + rte_hexdump(stdout, "AEAD key:", + options->aead_xform.aead.key.data, + options->aead_xform.aead.key.length); + rte_hexdump(stdout, "IV:", options->aead_iv.data, options->aead_iv.length); rte_hexdump(stdout, "AAD:", options->aad.data, options->aad.length); } @@ -1265,6 +1563,7 @@ l2fwd_crypto_options_print(struct l2fwd_crypto_options *options) { char string_cipher_op[MAX_STR_LEN]; char string_auth_op[MAX_STR_LEN]; + char string_aead_op[MAX_STR_LEN]; if (options->cipher_xform.cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) strcpy(string_cipher_op, "Encrypt"); @@ -1276,6 +1575,12 @@ l2fwd_crypto_options_print(struct l2fwd_crypto_options *options) else strcpy(string_auth_op, "Auth verify"); + if (options->aead_xform.aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) + strcpy(string_aead_op, "Authenticated encryption"); + else + strcpy(string_aead_op, "Authenticated decryption"); + + printf("Options:-\nn"); printf("portmask: %x\n", options->portmask); printf("ports per lcore: %u\n", options->nb_ports_per_lcore); @@ -1294,14 +1599,21 @@ l2fwd_crypto_options_print(struct l2fwd_crypto_options *options) if (options->akey_param && (options->akey_random_size != -1)) printf("Auth key already parsed, ignoring size of random key\n"); - if (options->iv_param && (options->iv_random_size != -1)) - printf("IV already parsed, ignoring size of random IV\n"); + if (options->cipher_iv_param && (options->cipher_iv_random_size != -1)) + printf("Cipher IV already parsed, ignoring size of random IV\n"); + + if (options->auth_iv_param && (options->auth_iv_random_size != -1)) + printf("Auth IV already parsed, ignoring size of random IV\n"); if (options->aad_param && (options->aad_random_size != -1)) printf("AAD already parsed, ignoring size of random AAD\n"); printf("\nCrypto chain: "); switch (options->xform_chain) { + case L2FWD_CRYPTO_AEAD: + printf("Input --> %s --> Output\n", string_aead_op); + display_aead_info(options); + break; case L2FWD_CRYPTO_CIPHER_HASH: printf("Input --> %s --> %s --> Output\n", string_cipher_op, string_auth_op); @@ -1343,26 +1655,40 @@ l2fwd_crypto_parse_args(struct l2fwd_crypto_options *options, { "cipher_op", required_argument, 0, 0 }, { "cipher_key", required_argument, 0, 0 }, { "cipher_key_random_size", required_argument, 0, 0 }, + { "cipher_iv", required_argument, 0, 0 }, + { "cipher_iv_random_size", required_argument, 0, 0 }, { "auth_algo", required_argument, 0, 0 }, { "auth_op", required_argument, 0, 0 }, { "auth_key", required_argument, 0, 0 }, { "auth_key_random_size", required_argument, 0, 0 }, + { "auth_iv", required_argument, 0, 0 }, + { "auth_iv_random_size", required_argument, 0, 0 }, + + { "aead_algo", required_argument, 0, 0 }, + { "aead_op", required_argument, 0, 0 }, + { "aead_key", required_argument, 0, 0 }, + { "aead_key_random_size", required_argument, 0, 0 }, + { "aead_iv", required_argument, 0, 0 }, + { "aead_iv_random_size", required_argument, 0, 0 }, - { "iv", required_argument, 0, 0 }, - { "iv_random_size", required_argument, 0, 0 }, { "aad", required_argument, 0, 0 }, { "aad_random_size", required_argument, 0, 0 }, + { "digest_size", required_argument, 0, 0 }, { "sessionless", no_argument, 0, 0 }, + { "cryptodev_mask", required_argument, 0, 0}, + + { "mac-updating", no_argument, 0, 0}, + { "no-mac-updating", no_argument, 0, 0}, { NULL, 0, 0, 0 } }; l2fwd_crypto_default_options(options); - while ((opt = getopt_long(argc, argvopt, "p:q:st:", lgopts, + while ((opt = getopt_long(argc, argvopt, "p:q:sT:", lgopts, &option_index)) != EOF) { switch (opt) { /* long options */ @@ -1420,18 +1746,19 @@ l2fwd_crypto_parse_args(struct l2fwd_crypto_options *options, argv[optind-1] = prgname; retval = optind-1; - optind = 0; /* reset getopt lib */ + optind = 1; /* reset getopt lib */ return retval; } /* Check the link status of all ports in up to 9s, and print them finally */ static void -check_all_ports_link_status(uint8_t port_num, uint32_t port_mask) +check_all_ports_link_status(uint16_t port_num, uint32_t port_mask) { #define CHECK_INTERVAL 100 /* 100ms */ #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ - uint8_t portid, count, all_ports_up, print_flag = 0; + uint16_t portid; + uint8_t count, all_ports_up, print_flag = 0; struct rte_eth_link link; printf("\nChecking link status"); @@ -1446,14 +1773,13 @@ check_all_ports_link_status(uint8_t port_num, uint32_t port_mask) /* print link status if flag set */ if (print_flag == 1) { if (link.link_status) - printf("Port %d Link Up - speed %u " - "Mbps - %s\n", (uint8_t)portid, - (unsigned)link.link_speed, + printf( + "Port%d Link Up. Speed %u Mbps - %s\n", + portid, link.link_speed, (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? ("full-duplex") : ("half-duplex\n")); else - printf("Port %d Link Down\n", - (uint8_t)portid); + printf("Port %d Link Down\n", portid); continue; } /* clear all_ports_up flag if any link down */ @@ -1482,7 +1808,8 @@ check_all_ports_link_status(uint8_t port_num, uint32_t port_mask) /* Check if device has to be HW/SW or any */ static int -check_type(struct l2fwd_crypto_options *options, struct rte_cryptodev_info *dev_info) +check_type(const struct l2fwd_crypto_options *options, + const struct rte_cryptodev_info *dev_info) { if (options->type == CDEV_TYPE_HW && (dev_info->feature_flags & RTE_CRYPTODEV_FF_HW_ACCELERATED)) @@ -1496,6 +1823,119 @@ check_type(struct l2fwd_crypto_options *options, struct rte_cryptodev_info *dev_ return -1; } +static const struct rte_cryptodev_capabilities * +check_device_support_cipher_algo(const struct l2fwd_crypto_options *options, + const struct rte_cryptodev_info *dev_info, + uint8_t cdev_id) +{ + unsigned int i = 0; + const struct rte_cryptodev_capabilities *cap = &dev_info->capabilities[0]; + enum rte_crypto_cipher_algorithm cap_cipher_algo; + enum rte_crypto_cipher_algorithm opt_cipher_algo = + options->cipher_xform.cipher.algo; + + while (cap->op != RTE_CRYPTO_OP_TYPE_UNDEFINED) { + cap_cipher_algo = cap->sym.cipher.algo; + if (cap->sym.xform_type == RTE_CRYPTO_SYM_XFORM_CIPHER) { + if (cap_cipher_algo == opt_cipher_algo) { + if (check_type(options, dev_info) == 0) + break; + } + } + cap = &dev_info->capabilities[++i]; + } + + if (cap->op == RTE_CRYPTO_OP_TYPE_UNDEFINED) { + printf("Algorithm %s not supported by cryptodev %u" + " or device not of preferred type (%s)\n", + rte_crypto_cipher_algorithm_strings[opt_cipher_algo], + cdev_id, + options->string_type); + return NULL; + } + + return cap; +} + +static const struct rte_cryptodev_capabilities * +check_device_support_auth_algo(const struct l2fwd_crypto_options *options, + const struct rte_cryptodev_info *dev_info, + uint8_t cdev_id) +{ + unsigned int i = 0; + const struct rte_cryptodev_capabilities *cap = &dev_info->capabilities[0]; + enum rte_crypto_auth_algorithm cap_auth_algo; + enum rte_crypto_auth_algorithm opt_auth_algo = + options->auth_xform.auth.algo; + + while (cap->op != RTE_CRYPTO_OP_TYPE_UNDEFINED) { + cap_auth_algo = cap->sym.auth.algo; + if (cap->sym.xform_type == RTE_CRYPTO_SYM_XFORM_AUTH) { + if (cap_auth_algo == opt_auth_algo) { + if (check_type(options, dev_info) == 0) + break; + } + } + cap = &dev_info->capabilities[++i]; + } + + if (cap->op == RTE_CRYPTO_OP_TYPE_UNDEFINED) { + printf("Algorithm %s not supported by cryptodev %u" + " or device not of preferred type (%s)\n", + rte_crypto_auth_algorithm_strings[opt_auth_algo], + cdev_id, + options->string_type); + return NULL; + } + + return cap; +} + +static const struct rte_cryptodev_capabilities * +check_device_support_aead_algo(const struct l2fwd_crypto_options *options, + const struct rte_cryptodev_info *dev_info, + uint8_t cdev_id) +{ + unsigned int i = 0; + const struct rte_cryptodev_capabilities *cap = &dev_info->capabilities[0]; + enum rte_crypto_aead_algorithm cap_aead_algo; + enum rte_crypto_aead_algorithm opt_aead_algo = + options->aead_xform.aead.algo; + + while (cap->op != RTE_CRYPTO_OP_TYPE_UNDEFINED) { + cap_aead_algo = cap->sym.aead.algo; + if (cap->sym.xform_type == RTE_CRYPTO_SYM_XFORM_AEAD) { + if (cap_aead_algo == opt_aead_algo) { + if (check_type(options, dev_info) == 0) + break; + } + } + cap = &dev_info->capabilities[++i]; + } + + if (cap->op == RTE_CRYPTO_OP_TYPE_UNDEFINED) { + printf("Algorithm %s not supported by cryptodev %u" + " or device not of preferred type (%s)\n", + rte_crypto_aead_algorithm_strings[opt_aead_algo], + cdev_id, + options->string_type); + return NULL; + } + + return cap; +} + +/* Check if the device is enabled by cryptodev_mask */ +static int +check_cryptodev_mask(struct l2fwd_crypto_options *options, + uint8_t cdev_id) +{ + if (options->cryptodev_mask & (1 << cdev_id)) + return 0; + + return -1; +} + static inline int check_supported_size(uint16_t length, uint16_t min, uint16_t max, uint16_t increment) @@ -1518,16 +1958,53 @@ check_supported_size(uint16_t length, uint16_t min, uint16_t max, return -1; } + +static int +check_iv_param(const struct rte_crypto_param_range *iv_range_size, + unsigned int iv_param, int iv_random_size, + uint16_t *iv_length) +{ + /* + * Check if length of provided IV is supported + * by the algorithm chosen. + */ + if (iv_param) { + if (check_supported_size(*iv_length, + iv_range_size->min, + iv_range_size->max, + iv_range_size->increment) + != 0) { + printf("Unsupported IV length\n"); + return -1; + } + /* + * Check if length of IV to be randomly generated + * is supported by the algorithm chosen. + */ + } else if (iv_random_size != -1) { + if (check_supported_size(iv_random_size, + iv_range_size->min, + iv_range_size->max, + iv_range_size->increment) + != 0) { + printf("Unsupported IV length\n"); + return -1; + } + *iv_length = iv_random_size; + /* No size provided, use minimum size. */ + } else + *iv_length = iv_range_size->min; + + return 0; +} + static int initialize_cryptodevs(struct l2fwd_crypto_options *options, unsigned nb_ports, uint8_t *enabled_cdevs) { - unsigned i, cdev_id, cdev_count, enabled_cdev_count = 0; + unsigned int cdev_id, cdev_count, enabled_cdev_count = 0; const struct rte_cryptodev_capabilities *cap; - enum rte_crypto_auth_algorithm cap_auth_algo; - enum rte_crypto_auth_algorithm opt_auth_algo; - enum rte_crypto_cipher_algorithm cap_cipher_algo; - enum rte_crypto_cipher_algorithm opt_cipher_algo; + unsigned int sess_sz, max_sess_sz = 0; int retval; cdev_count = rte_cryptodev_count(); @@ -1536,82 +2013,186 @@ initialize_cryptodevs(struct l2fwd_crypto_options *options, unsigned nb_ports, return -1; } + for (cdev_id = 0; cdev_id < cdev_count; cdev_id++) { + sess_sz = rte_cryptodev_get_private_session_size(cdev_id); + if (sess_sz > max_sess_sz) + max_sess_sz = sess_sz; + } + for (cdev_id = 0; cdev_id < cdev_count && enabled_cdev_count < nb_ports; cdev_id++) { struct rte_cryptodev_qp_conf qp_conf; struct rte_cryptodev_info dev_info; + retval = rte_cryptodev_socket_id(cdev_id); + + if (retval < 0) { + printf("Invalid crypto device id used\n"); + return -1; + } + + uint8_t socket_id = (uint8_t) retval; struct rte_cryptodev_config conf = { .nb_queue_pairs = 1, - .socket_id = SOCKET_ID_ANY, - .session_mp = { - .nb_objs = 2048, - .cache_size = 64 - } + .socket_id = socket_id, }; + if (check_cryptodev_mask(options, (uint8_t)cdev_id)) + continue; + rte_cryptodev_info_get(cdev_id, &dev_info); + if (session_pool_socket[socket_id] == NULL) { + char mp_name[RTE_MEMPOOL_NAMESIZE]; + struct rte_mempool *sess_mp; + + snprintf(mp_name, RTE_MEMPOOL_NAMESIZE, + "sess_mp_%u", socket_id); + + /* + * Create enough objects for session headers and + * device private data + */ + sess_mp = rte_mempool_create(mp_name, + MAX_SESSIONS * 2, + max_sess_sz, + SESSION_POOL_CACHE_SIZE, + 0, NULL, NULL, NULL, + NULL, socket_id, + 0); + + if (sess_mp == NULL) { + printf("Cannot create session pool on socket %d\n", + socket_id); + return -ENOMEM; + } + + printf("Allocated session pool on socket %d\n", socket_id); + session_pool_socket[socket_id] = sess_mp; + } + + /* Set AEAD parameters */ + if (options->xform_chain == L2FWD_CRYPTO_AEAD) { + /* Check if device supports AEAD algo */ + cap = check_device_support_aead_algo(options, &dev_info, + cdev_id); + if (cap == NULL) + continue; + + options->block_size = cap->sym.aead.block_size; + + check_iv_param(&cap->sym.aead.iv_size, + options->aead_iv_param, + options->aead_iv_random_size, + &options->aead_iv.length); + + /* + * Check if length of provided AEAD key is supported + * by the algorithm chosen. + */ + if (options->aead_key_param) { + if (check_supported_size( + options->aead_xform.aead.key.length, + cap->sym.aead.key_size.min, + cap->sym.aead.key_size.max, + cap->sym.aead.key_size.increment) + != 0) { + printf("Unsupported aead key length\n"); + return -1; + } + /* + * Check if length of the aead key to be randomly generated + * is supported by the algorithm chosen. + */ + } else if (options->aead_key_random_size != -1) { + if (check_supported_size(options->aead_key_random_size, + cap->sym.aead.key_size.min, + cap->sym.aead.key_size.max, + cap->sym.aead.key_size.increment) + != 0) { + printf("Unsupported aead key length\n"); + return -1; + } + options->aead_xform.aead.key.length = + options->aead_key_random_size; + /* No size provided, use minimum size. */ + } else + options->aead_xform.aead.key.length = + cap->sym.aead.key_size.min; + + if (!options->aead_key_param) + generate_random_key( + options->aead_xform.aead.key.data, + options->aead_xform.aead.key.length); + + /* + * Check if length of provided AAD is supported + * by the algorithm chosen. + */ + if (options->aad_param) { + if (check_supported_size(options->aad.length, + cap->sym.aead.aad_size.min, + cap->sym.aead.aad_size.max, + cap->sym.aead.aad_size.increment) + != 0) { + printf("Unsupported AAD length\n"); + return -1; + } + /* + * Check if length of AAD to be randomly generated + * is supported by the algorithm chosen. + */ + } else if (options->aad_random_size != -1) { + if (check_supported_size(options->aad_random_size, + cap->sym.aead.aad_size.min, + cap->sym.aead.aad_size.max, + cap->sym.aead.aad_size.increment) + != 0) { + printf("Unsupported AAD length\n"); + return -1; + } + options->aad.length = options->aad_random_size; + /* No size provided, use minimum size. */ + } else + options->aad.length = cap->sym.auth.aad_size.min; + + options->aead_xform.aead.aad_length = + options->aad.length; + + /* Check if digest size is supported by the algorithm. */ + if (options->digest_size != -1) { + if (check_supported_size(options->digest_size, + cap->sym.aead.digest_size.min, + cap->sym.aead.digest_size.max, + cap->sym.aead.digest_size.increment) + != 0) { + printf("Unsupported digest length\n"); + return -1; + } + options->aead_xform.aead.digest_length = + options->digest_size; + /* No size provided, use minimum size. */ + } else + options->aead_xform.aead.digest_length = + cap->sym.aead.digest_size.min; + } + /* Set cipher parameters */ if (options->xform_chain == L2FWD_CRYPTO_CIPHER_HASH || options->xform_chain == L2FWD_CRYPTO_HASH_CIPHER || options->xform_chain == L2FWD_CRYPTO_CIPHER_ONLY) { /* Check if device supports cipher algo */ - i = 0; - opt_cipher_algo = options->cipher_xform.cipher.algo; - cap = &dev_info.capabilities[i]; - while (cap->op != RTE_CRYPTO_OP_TYPE_UNDEFINED) { - cap_cipher_algo = cap->sym.cipher.algo; - if (cap->sym.xform_type == - RTE_CRYPTO_SYM_XFORM_CIPHER) { - if (cap_cipher_algo == opt_cipher_algo) { - if (check_type(options, &dev_info) == 0) - break; - } - } - cap = &dev_info.capabilities[++i]; - } - - if (cap->op == RTE_CRYPTO_OP_TYPE_UNDEFINED) { - printf("Algorithm %s not supported by cryptodev %u" - " or device not of preferred type (%s)\n", - supported_cipher_algo[opt_cipher_algo], - cdev_id, - options->string_type); + cap = check_device_support_cipher_algo(options, &dev_info, + cdev_id); + if (cap == NULL) continue; - } options->block_size = cap->sym.cipher.block_size; - /* - * Check if length of provided IV is supported - * by the algorithm chosen. - */ - if (options->iv_param) { - if (check_supported_size(options->iv.length, - cap->sym.cipher.iv_size.min, - cap->sym.cipher.iv_size.max, - cap->sym.cipher.iv_size.increment) - != 0) { - printf("Unsupported IV length\n"); - return -1; - } - /* - * Check if length of IV to be randomly generated - * is supported by the algorithm chosen. - */ - } else if (options->iv_random_size != -1) { - if (check_supported_size(options->iv_random_size, - cap->sym.cipher.iv_size.min, - cap->sym.cipher.iv_size.max, - cap->sym.cipher.iv_size.increment) - != 0) { - printf("Unsupported IV length\n"); - return -1; - } - options->iv.length = options->iv_random_size; - /* No size provided, use minimum size. */ - } else - options->iv.length = cap->sym.cipher.iv_size.min; + + check_iv_param(&cap->sym.cipher.iv_size, + options->cipher_iv_param, + options->cipher_iv_random_size, + &options->cipher_iv.length); /* * Check if length of provided cipher key is supported @@ -1659,63 +2240,15 @@ initialize_cryptodevs(struct l2fwd_crypto_options *options, unsigned nb_ports, options->xform_chain == L2FWD_CRYPTO_HASH_CIPHER || options->xform_chain == L2FWD_CRYPTO_HASH_ONLY) { /* Check if device supports auth algo */ - i = 0; - opt_auth_algo = options->auth_xform.auth.algo; - cap = &dev_info.capabilities[i]; - while (cap->op != RTE_CRYPTO_OP_TYPE_UNDEFINED) { - cap_auth_algo = cap->sym.auth.algo; - if ((cap->sym.xform_type == RTE_CRYPTO_SYM_XFORM_AUTH) && - (cap_auth_algo == opt_auth_algo) && - (check_type(options, &dev_info) == 0)) { - break; - } - cap = &dev_info.capabilities[++i]; - } - - if (cap->op == RTE_CRYPTO_OP_TYPE_UNDEFINED) { - printf("Algorithm %s not supported by cryptodev %u" - " or device not of preferred type (%s)\n", - supported_auth_algo[opt_auth_algo], - cdev_id, - options->string_type); + cap = check_device_support_auth_algo(options, &dev_info, + cdev_id); + if (cap == NULL) continue; - } - - options->block_size = cap->sym.auth.block_size; - /* - * Check if length of provided AAD is supported - * by the algorithm chosen. - */ - if (options->aad_param) { - if (check_supported_size(options->aad.length, - cap->sym.auth.aad_size.min, - cap->sym.auth.aad_size.max, - cap->sym.auth.aad_size.increment) - != 0) { - printf("Unsupported AAD length\n"); - return -1; - } - /* - * Check if length of AAD to be randomly generated - * is supported by the algorithm chosen. - */ - } else if (options->aad_random_size != -1) { - if (check_supported_size(options->aad_random_size, - cap->sym.auth.aad_size.min, - cap->sym.auth.aad_size.max, - cap->sym.auth.aad_size.increment) - != 0) { - printf("Unsupported AAD length\n"); - return -1; - } - options->aad.length = options->aad_random_size; - /* No size provided, use minimum size. */ - } else - options->aad.length = cap->sym.auth.aad_size.min; - - options->auth_xform.auth.add_auth_data_length = - options->aad.length; + check_iv_param(&cap->sym.auth.iv_size, + options->auth_iv_param, + options->auth_iv_random_size, + &options->auth_iv.length); /* * Check if length of provided auth key is supported * by the algorithm chosen. @@ -1782,7 +2315,7 @@ initialize_cryptodevs(struct l2fwd_crypto_options *options, unsigned nb_ports, qp_conf.nb_descriptors = 2048; retval = rte_cryptodev_queue_pair_setup(cdev_id, 0, &qp_conf, - SOCKET_ID_ANY); + socket_id, session_pool_socket[socket_id]); if (retval < 0) { printf("Failed to setup queue pair %u on cryptodev %u", 0, cdev_id); @@ -1796,7 +2329,7 @@ initialize_cryptodevs(struct l2fwd_crypto_options *options, unsigned nb_ports, return -1; } - l2fwd_enabled_crypto_mask |= (1 << cdev_id); + l2fwd_enabled_crypto_mask |= (((uint64_t)1) << cdev_id); enabled_cdevs[cdev_id] = 1; enabled_cdev_count++; @@ -1808,7 +2341,7 @@ initialize_cryptodevs(struct l2fwd_crypto_options *options, unsigned nb_ports, static int initialize_ports(struct l2fwd_crypto_options *options) { - uint8_t last_portid, portid; + uint16_t last_portid, portid; unsigned enabled_portcount = 0; unsigned nb_ports = rte_eth_dev_count(); @@ -1829,12 +2362,20 @@ initialize_ports(struct l2fwd_crypto_options *options) continue; /* init port */ - printf("Initializing port %u... ", (unsigned) portid); + printf("Initializing port %u... ", portid); fflush(stdout); retval = rte_eth_dev_configure(portid, 1, 1, &port_conf); if (retval < 0) { printf("Cannot configure device: err=%d, port=%u\n", - retval, (unsigned) portid); + retval, portid); + return -1; + } + + retval = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd, + &nb_txd); + if (retval < 0) { + printf("Cannot adjust number of descriptors: err=%d, port=%u\n", + retval, portid); return -1; } @@ -1845,7 +2386,7 @@ initialize_ports(struct l2fwd_crypto_options *options) NULL, l2fwd_pktmbuf_pool); if (retval < 0) { printf("rte_eth_rx_queue_setup:err=%d, port=%u\n", - retval, (unsigned) portid); + retval, portid); return -1; } @@ -1856,7 +2397,7 @@ initialize_ports(struct l2fwd_crypto_options *options) NULL); if (retval < 0) { printf("rte_eth_tx_queue_setup:err=%d, port=%u\n", - retval, (unsigned) portid); + retval, portid); return -1; } @@ -1865,7 +2406,7 @@ initialize_ports(struct l2fwd_crypto_options *options) retval = rte_eth_dev_start(portid); if (retval < 0) { printf("rte_eth_dev_start:err=%d, port=%u\n", - retval, (unsigned) portid); + retval, portid); return -1; } @@ -1874,7 +2415,7 @@ initialize_ports(struct l2fwd_crypto_options *options) rte_eth_macaddr_get(portid, &l2fwd_ports_eth_addr[portid]); printf("Port %u, MAC address: %02X:%02X:%02X:%02X:%02X:%02X\n\n", - (unsigned) portid, + portid, l2fwd_ports_eth_addr[portid].addr_bytes[0], l2fwd_ports_eth_addr[portid].addr_bytes[1], l2fwd_ports_eth_addr[portid].addr_bytes[2], @@ -1917,21 +2458,32 @@ reserve_key_memory(struct l2fwd_crypto_options *options) if (options->cipher_xform.cipher.key.data == NULL) rte_exit(EXIT_FAILURE, "Failed to allocate memory for cipher key"); - options->auth_xform.auth.key.data = rte_malloc("auth key", MAX_KEY_SIZE, 0); if (options->auth_xform.auth.key.data == NULL) rte_exit(EXIT_FAILURE, "Failed to allocate memory for auth key"); - options->iv.data = rte_malloc("iv", MAX_KEY_SIZE, 0); - if (options->iv.data == NULL) - rte_exit(EXIT_FAILURE, "Failed to allocate memory for IV"); - options->iv.phys_addr = rte_malloc_virt2phy(options->iv.data); + options->aead_xform.aead.key.data = rte_malloc("aead key", + MAX_KEY_SIZE, 0); + if (options->aead_xform.aead.key.data == NULL) + rte_exit(EXIT_FAILURE, "Failed to allocate memory for AEAD key"); + + options->cipher_iv.data = rte_malloc("cipher iv", MAX_KEY_SIZE, 0); + if (options->cipher_iv.data == NULL) + rte_exit(EXIT_FAILURE, "Failed to allocate memory for cipher IV"); + + options->auth_iv.data = rte_malloc("auth iv", MAX_KEY_SIZE, 0); + if (options->auth_iv.data == NULL) + rte_exit(EXIT_FAILURE, "Failed to allocate memory for auth IV"); + + options->aead_iv.data = rte_malloc("aead_iv", MAX_KEY_SIZE, 0); + if (options->aead_iv.data == NULL) + rte_exit(EXIT_FAILURE, "Failed to allocate memory for AEAD iv"); options->aad.data = rte_malloc("aad", MAX_KEY_SIZE, 0); if (options->aad.data == NULL) rte_exit(EXIT_FAILURE, "Failed to allocate memory for AAD"); - options->aad.phys_addr = rte_malloc_virt2phy(options->aad.data); + options->aad.phys_addr = rte_malloc_virt2iova(options->aad.data); } int @@ -1940,7 +2492,8 @@ main(int argc, char **argv) struct lcore_queue_conf *qconf; struct l2fwd_crypto_options options; - uint8_t nb_ports, nb_cryptodevs, portid, cdev_id; + uint8_t nb_cryptodevs, cdev_id; + uint16_t nb_ports, portid; unsigned lcore_id, rx_lcore_id; int ret, enabled_cdevcount, enabled_portcount; uint8_t enabled_cdevs[RTE_CRYPTO_MAX_DEVS] = {0}; @@ -1955,14 +2508,14 @@ main(int argc, char **argv) /* reserve memory for Cipher/Auth key and IV */ reserve_key_memory(&options); - /* fill out the supported algorithm tables */ - fill_supported_algorithm_tables(); - /* parse application arguments (after the EAL ones) */ ret = l2fwd_crypto_parse_args(&options, argc, argv); if (ret < 0) rte_exit(EXIT_FAILURE, "Invalid L2FWD-CRYPTO arguments\n"); + printf("MAC updating %s\n", + options.mac_updating ? "enabled" : "disabled"); + /* create the mbuf pool */ l2fwd_pktmbuf_pool = rte_pktmbuf_pool_create("mbuf_pool", NB_MBUF, 512, sizeof(struct rte_crypto_op), @@ -1972,7 +2525,7 @@ main(int argc, char **argv) /* create crypto op pool */ l2fwd_crypto_op_pool = rte_crypto_op_pool_create("crypto_op_pool", - RTE_CRYPTO_OP_TYPE_SYMMETRIC, NB_MBUF, 128, 0, + RTE_CRYPTO_OP_TYPE_SYMMETRIC, NB_MBUF, 128, MAXIMUM_IV_LENGTH, rte_socket_id()); if (l2fwd_crypto_op_pool == NULL) rte_exit(EXIT_FAILURE, "Cannot create crypto op pool\n"); @@ -2017,7 +2570,7 @@ main(int argc, char **argv) qconf->rx_port_list[qconf->nb_rx_ports] = portid; qconf->nb_rx_ports++; - printf("Lcore %u: RX port %u\n", rx_lcore_id, (unsigned)portid); + printf("Lcore %u: RX port %u\n", rx_lcore_id, portid); } /* Enable Crypto devices */ diff --git a/dpdk/examples/l2fwd-ivshmem/Makefile b/dpdk/examples/l2fwd-ivshmem/Makefile deleted file mode 100644 index 5f1d1728..00000000 --- a/dpdk/examples/l2fwd-ivshmem/Makefile +++ /dev/null @@ -1,43 +0,0 @@ -# BSD LICENSE -# -# Copyright(c) 2010-2014 Intel Corporation. All rights reserved. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in -# the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Intel Corporation nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -ifeq ($(RTE_SDK),) -$(error "Please define RTE_SDK environment variable") -endif - -# Default target, can be overriden by command line or environment -RTE_TARGET ?= x86_64-ivshmem-linuxapp-gcc - -include $(RTE_SDK)/mk/rte.vars.mk - -DIRS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += host guest - -include $(RTE_SDK)/mk/rte.extsubdir.mk diff --git a/dpdk/examples/l2fwd-ivshmem/guest/Makefile b/dpdk/examples/l2fwd-ivshmem/guest/Makefile deleted file mode 100644 index 3ca73b43..00000000 --- a/dpdk/examples/l2fwd-ivshmem/guest/Makefile +++ /dev/null @@ -1,50 +0,0 @@ -# BSD LICENSE -# -# Copyright(c) 2010-2014 Intel Corporation. All rights reserved. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in -# the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Intel Corporation nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -ifeq ($(RTE_SDK),) -$(error "Please define RTE_SDK environment variable") -endif - -# Default target, can be overriden by command line or environment -RTE_TARGET ?= x86_64-ivshmem-linuxapp-gcc - -include $(RTE_SDK)/mk/rte.vars.mk - -# binary name -APP = guest - -# all source are stored in SRCS-y -SRCS-y := guest.c - -CFLAGS += -O3 -CFLAGS += $(WERROR_FLAGS) - -include $(RTE_SDK)/mk/rte.extapp.mk diff --git a/dpdk/examples/l2fwd-ivshmem/guest/guest.c b/dpdk/examples/l2fwd-ivshmem/guest/guest.c deleted file mode 100644 index 7c49521b..00000000 --- a/dpdk/examples/l2fwd-ivshmem/guest/guest.c +++ /dev/null @@ -1,452 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "../include/common.h" - -#define MAX_RX_QUEUE_PER_LCORE 16 -#define MAX_TX_QUEUE_PER_PORT 16 -struct lcore_queue_conf { - unsigned n_rx_port; - unsigned rx_port_list[MAX_RX_QUEUE_PER_LCORE]; - struct mbuf_table rx_mbufs[RTE_MAX_ETHPORTS]; - struct vm_port_param * port_param[MAX_RX_QUEUE_PER_LCORE]; -} __rte_cache_aligned; -static struct lcore_queue_conf lcore_queue_conf[RTE_MAX_LCORE]; - -/* Print out statistics on packets dropped */ -static void -print_stats(void) -{ - uint64_t total_packets_dropped, total_packets_tx, total_packets_rx; - unsigned portid; - - total_packets_dropped = 0; - total_packets_tx = 0; - total_packets_rx = 0; - - const char clr[] = { 27, '[', '2', 'J', '\0' }; - const char topLeft[] = { 27, '[', '1', ';', '1', 'H','\0' }; - - /* Clear screen and move to top left */ - printf("%s%s", clr, topLeft); - - printf("\nPort statistics ===================================="); - - for (portid = 0; portid < ctrl->nb_ports; portid++) { - /* skip ports that are not enabled */ - printf("\nStatistics for port %u ------------------------------" - "\nPackets sent: %24"PRIu64 - "\nPackets received: %20"PRIu64 - "\nPackets dropped: %21"PRIu64, - portid, - ctrl->vm_ports[portid].stats.tx, - ctrl->vm_ports[portid].stats.rx, - ctrl->vm_ports[portid].stats.dropped); - - total_packets_dropped += ctrl->vm_ports[portid].stats.dropped; - total_packets_tx += ctrl->vm_ports[portid].stats.tx; - total_packets_rx += ctrl->vm_ports[portid].stats.rx; - } - printf("\nAggregate statistics ===============================" - "\nTotal packets sent: %18"PRIu64 - "\nTotal packets received: %14"PRIu64 - "\nTotal packets dropped: %15"PRIu64, - total_packets_tx, - total_packets_rx, - total_packets_dropped); - printf("\n====================================================\n"); -} - -/* display usage */ -static void -l2fwd_ivshmem_usage(const char *prgname) -{ - printf("%s [EAL options] -- [-q NQ -T PERIOD]\n" - " -q NQ: number of queue (=ports) per lcore (default is 1)\n" - " -T PERIOD: statistics will be refreshed each PERIOD seconds (0 to disable, 10 default, 86400 maximum)\n", - prgname); -} - -static unsigned int -l2fwd_ivshmem_parse_nqueue(const char *q_arg) -{ - char *end = NULL; - unsigned long n; - - /* parse hexadecimal string */ - n = strtoul(q_arg, &end, 10); - if ((q_arg[0] == '\0') || (end == NULL) || (*end != '\0')) - return 0; - if (n == 0) - return 0; - if (n >= MAX_RX_QUEUE_PER_LCORE) - return 0; - - return n; -} - -static int -l2fwd_ivshmem_parse_timer_period(const char *q_arg) -{ - char *end = NULL; - int n; - - /* parse number string */ - n = strtol(q_arg, &end, 10); - if ((q_arg[0] == '\0') || (end == NULL) || (*end != '\0')) - return -1; - if (n >= MAX_TIMER_PERIOD) - return -1; - - return n; -} - -/* Parse the argument given in the command line of the application */ -static int -l2fwd_ivshmem_parse_args(int argc, char **argv) -{ - int opt, ret; - char **argvopt; - int option_index; - char *prgname = argv[0]; - static struct option lgopts[] = { - {NULL, 0, 0, 0} - }; - - argvopt = argv; - - while ((opt = getopt_long(argc, argvopt, "q:p:T:", - lgopts, &option_index)) != EOF) { - - switch (opt) { - - /* nqueue */ - case 'q': - l2fwd_ivshmem_rx_queue_per_lcore = l2fwd_ivshmem_parse_nqueue(optarg); - if (l2fwd_ivshmem_rx_queue_per_lcore == 0) { - printf("invalid queue number\n"); - l2fwd_ivshmem_usage(prgname); - return -1; - } - break; - - /* timer period */ - case 'T': - timer_period = l2fwd_ivshmem_parse_timer_period(optarg) * 1000 * TIMER_MILLISECOND; - if (timer_period < 0) { - printf("invalid timer period\n"); - l2fwd_ivshmem_usage(prgname); - return -1; - } - break; - - /* long options */ - case 0: - l2fwd_ivshmem_usage(prgname); - return -1; - - default: - l2fwd_ivshmem_usage(prgname); - return -1; - } - } - - if (optind >= 0) - argv[optind-1] = prgname; - - ret = optind-1; - optind = 0; /* reset getopt lib */ - return ret; -} - -/* - * this loop is getting packets from RX rings of each port, and puts them - * into TX rings of destination ports. - */ -static void -fwd_loop(void) -{ - - struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; - struct rte_mbuf **m_table; - struct rte_mbuf *m; - struct rte_ring *rx, *tx; - unsigned lcore_id, len; - uint64_t prev_tsc, diff_tsc, cur_tsc, timer_tsc; - unsigned i, j, portid, nb_rx; - struct lcore_queue_conf *qconf; - struct ether_hdr *eth; - void *tmp; - - prev_tsc = 0; - timer_tsc = 0; - - lcore_id = rte_lcore_id(); - qconf = &lcore_queue_conf[lcore_id]; - - if (qconf->n_rx_port == 0) { - RTE_LOG(INFO, L2FWD_IVSHMEM, "lcore %u has nothing to do\n", lcore_id); - return; - } - - RTE_LOG(INFO, L2FWD_IVSHMEM, "entering main loop on lcore %u\n", lcore_id); - - for (i = 0; i < qconf->n_rx_port; i++) { - portid = qconf->rx_port_list[i]; - RTE_LOG(INFO, L2FWD_IVSHMEM, " -- lcoreid=%u portid=%u\n", lcore_id, - portid); - } - - while (ctrl->state == STATE_FWD) { - cur_tsc = rte_rdtsc(); - - diff_tsc = cur_tsc - prev_tsc; - - /* - * Read packet from RX queues and send it to TX queues - */ - for (i = 0; i < qconf->n_rx_port; i++) { - - portid = qconf->rx_port_list[i]; - - len = qconf->rx_mbufs[portid].len; - - rx = ctrl->vm_ports[portid].rx_ring; - tx = ctrl->vm_ports[portid].dst->tx_ring; - - m_table = qconf->rx_mbufs[portid].m_table; - - /* if we have something in the queue, try and transmit it down */ - if (len != 0) { - - /* if we succeed in sending the packets down, mark queue as free */ - if (rte_ring_enqueue_bulk(tx, (void**) m_table, len) == 0) { - ctrl->vm_ports[portid].stats.tx += len; - qconf->rx_mbufs[portid].len = 0; - len = 0; - } - } - - nb_rx = rte_ring_count(rx); - - nb_rx = RTE_MIN(nb_rx, (unsigned) MAX_PKT_BURST); - - if (nb_rx == 0) - continue; - - /* if we can get packets into the m_table */ - if (nb_rx < (RTE_DIM(qconf->rx_mbufs[portid].m_table) - len)) { - - /* this situation cannot exist, so if we fail to dequeue, that - * means something went horribly wrong, hence the failure. */ - if (rte_ring_dequeue_bulk(rx, (void**) pkts_burst, nb_rx) < 0) { - ctrl->state = STATE_FAIL; - return; - } - - ctrl->vm_ports[portid].stats.rx += nb_rx; - - /* put packets into the queue */ - for (j = 0; j < nb_rx; j++) { - m = pkts_burst[j]; - - rte_prefetch0(rte_pktmbuf_mtod(m, void *)); - - m_table[len + j] = m; - - eth = rte_pktmbuf_mtod(m, struct ether_hdr *); - - /* 02:00:00:00:00:xx */ - tmp = ð->d_addr.addr_bytes[0]; - *((uint64_t *)tmp) = 0x000000000002 + ((uint64_t)portid << 40); - - /* src addr */ - ether_addr_copy(&ctrl->vm_ports[portid].dst->ethaddr, - ð->s_addr); - } - qconf->rx_mbufs[portid].len += nb_rx; - - } - - } - - /* if timer is enabled */ - if (timer_period > 0) { - - /* advance the timer */ - timer_tsc += diff_tsc; - - /* if timer has reached its timeout */ - if (unlikely(timer_tsc >= (uint64_t) timer_period)) { - - /* do this only on master core */ - if (lcore_id == rte_get_master_lcore()) { - print_stats(); - /* reset the timer */ - timer_tsc = 0; - } - } - } - - prev_tsc = cur_tsc; - } -} - -static int -l2fwd_ivshmem_launch_one_lcore(__attribute__((unused)) void *dummy) -{ - fwd_loop(); - return 0; -} - -int -main(int argc, char **argv) -{ - struct lcore_queue_conf *qconf; - const struct rte_memzone * mz; - int ret; - uint8_t portid; - unsigned rx_lcore_id, lcore_id; - - /* init EAL */ - ret = rte_eal_init(argc, argv); - if (ret < 0) - rte_exit(EXIT_FAILURE, "Invalid EAL arguments\n"); - argc -= ret; - argv += ret; - - /* parse application arguments (after the EAL ones) */ - ret = l2fwd_ivshmem_parse_args(argc, argv); - if (ret < 0) - rte_exit(EXIT_FAILURE, "Invalid l2fwd-ivshmem arguments\n"); - - /* find control structure */ - mz = rte_memzone_lookup(CTRL_MZ_NAME); - if (mz == NULL) - rte_exit(EXIT_FAILURE, "Cannot find control memzone\n"); - - ctrl = (struct ivshmem_ctrl*) mz->addr; - - /* lock the ctrl so that we don't have conflicts with anything else */ - rte_spinlock_lock(&ctrl->lock); - - if (ctrl->state == STATE_FWD) - rte_exit(EXIT_FAILURE, "Forwarding already started!\n"); - - rx_lcore_id = 0; - qconf = NULL; - - /* Initialize the port/queue configuration of each logical core */ - for (portid = 0; portid < ctrl->nb_ports; portid++) { - - /* get the lcore_id for this port */ - while (rte_lcore_is_enabled(rx_lcore_id) == 0 || - lcore_queue_conf[rx_lcore_id].n_rx_port == - l2fwd_ivshmem_rx_queue_per_lcore) { - rx_lcore_id++; - if (rx_lcore_id >= RTE_MAX_LCORE) - rte_exit(EXIT_FAILURE, "Not enough cores\n"); - } - - if (qconf != &lcore_queue_conf[rx_lcore_id]) - /* Assigned a new logical core in the loop above. */ - qconf = &lcore_queue_conf[rx_lcore_id]; - - qconf->rx_port_list[qconf->n_rx_port] = portid; - qconf->port_param[qconf->n_rx_port] = &ctrl->vm_ports[portid]; - qconf->n_rx_port++; - - printf("Lcore %u: RX port %u\n", rx_lcore_id, (unsigned) portid); - } - - sigsetup(); - - /* indicate that we are ready to forward */ - ctrl->state = STATE_FWD; - - /* unlock */ - rte_spinlock_unlock(&ctrl->lock); - - /* launch per-lcore init on every lcore */ - rte_eal_mp_remote_launch(l2fwd_ivshmem_launch_one_lcore, NULL, CALL_MASTER); - RTE_LCORE_FOREACH_SLAVE(lcore_id) { - if (rte_eal_wait_lcore(lcore_id) < 0) - return -1; - } - - return 0; -} diff --git a/dpdk/examples/l2fwd-ivshmem/host/Makefile b/dpdk/examples/l2fwd-ivshmem/host/Makefile deleted file mode 100644 index f91419e9..00000000 --- a/dpdk/examples/l2fwd-ivshmem/host/Makefile +++ /dev/null @@ -1,50 +0,0 @@ -# BSD LICENSE -# -# Copyright(c) 2010-2014 Intel Corporation. All rights reserved. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in -# the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Intel Corporation nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -ifeq ($(RTE_SDK),) -$(error "Please define RTE_SDK environment variable") -endif - -# Default target, can be overriden by command line or environment -RTE_TARGET ?= x86_64-ivshmem-linuxapp-gcc - -include $(RTE_SDK)/mk/rte.vars.mk - -# binary name -APP = host - -# all source are stored in SRCS-y -SRCS-y := host.c - -CFLAGS += -O3 -CFLAGS += $(WERROR_FLAGS) - -include $(RTE_SDK)/mk/rte.extapp.mk diff --git a/dpdk/examples/l2fwd-ivshmem/host/host.c b/dpdk/examples/l2fwd-ivshmem/host/host.c deleted file mode 100644 index da7b00d9..00000000 --- a/dpdk/examples/l2fwd-ivshmem/host/host.c +++ /dev/null @@ -1,895 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "../include/common.h" - -/* - * Configurable number of RX/TX ring descriptors - */ -#define RTE_TEST_RX_DESC_DEFAULT 128 -#define RTE_TEST_TX_DESC_DEFAULT 512 -static uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; -static uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; - -#define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */ - -/* mask of enabled ports */ -static uint32_t l2fwd_ivshmem_enabled_port_mask = 0; - -static struct ether_addr l2fwd_ivshmem_ports_eth_addr[RTE_MAX_ETHPORTS]; - -#define NB_MBUF 8192 - -#define MAX_RX_QUEUE_PER_LCORE 16 -#define MAX_TX_QUEUE_PER_PORT 16 -struct lcore_queue_conf { - unsigned n_rx_port; - unsigned rx_port_list[MAX_RX_QUEUE_PER_LCORE]; - struct vm_port_param * port_param[MAX_RX_QUEUE_PER_LCORE]; - struct mbuf_table tx_mbufs[RTE_MAX_ETHPORTS]; - struct mbuf_table rx_mbufs[RTE_MAX_ETHPORTS]; -} __rte_cache_aligned; -static struct lcore_queue_conf lcore_queue_conf[RTE_MAX_LCORE]; - -static const struct rte_eth_conf port_conf = { - .rxmode = { - .split_hdr_size = 0, - .header_split = 0, /**< Header Split disabled */ - .hw_ip_checksum = 0, /**< IP checksum offload disabled */ - .hw_vlan_filter = 0, /**< VLAN filtering disabled */ - .jumbo_frame = 0, /**< Jumbo Frame Support disabled */ - .hw_strip_crc = 0, /**< CRC stripped by hardware */ - }, - .txmode = { - .mq_mode = ETH_MQ_TX_NONE, - }, -}; - -#define METADATA_NAME "l2fwd_ivshmem" -#define CMDLINE_OPT_FWD_CONF "fwd-conf" - -#define QEMU_CMD_FMT "/tmp/ivshmem_qemu_cmdline_%s" - -struct port_statistics port_statistics[RTE_MAX_ETHPORTS]; - -struct rte_mempool * l2fwd_ivshmem_pktmbuf_pool = NULL; - -/* Print out statistics on packets dropped */ -static void -print_stats(void) -{ - uint64_t total_packets_dropped, total_packets_tx, total_packets_rx; - uint64_t total_vm_packets_dropped = 0; - uint64_t total_vm_packets_tx, total_vm_packets_rx; - unsigned portid; - - total_packets_dropped = 0; - total_packets_tx = 0; - total_packets_rx = 0; - total_vm_packets_tx = 0; - total_vm_packets_rx = 0; - - const char clr[] = { 27, '[', '2', 'J', '\0' }; - const char topLeft[] = { 27, '[', '1', ';', '1', 'H','\0' }; - - /* Clear screen and move to top left */ - printf("%s%s", clr, topLeft); - - printf("\nPort statistics ===================================="); - - for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++) { - /* skip disabled ports */ - if ((l2fwd_ivshmem_enabled_port_mask & (1 << portid)) == 0) - continue; - printf("\nStatistics for port %u ------------------------------" - "\nPackets sent: %24"PRIu64 - "\nPackets received: %20"PRIu64 - "\nPackets dropped: %21"PRIu64, - portid, - port_statistics[portid].tx, - port_statistics[portid].rx, - port_statistics[portid].dropped); - - total_packets_dropped += port_statistics[portid].dropped; - total_packets_tx += port_statistics[portid].tx; - total_packets_rx += port_statistics[portid].rx; - } - - printf("\nVM statistics ======================================"); - for (portid = 0; portid < ctrl->nb_ports; portid++) { - printf("\nStatistics for port %u ------------------------------" - "\nPackets sent: %24"PRIu64 - "\nPackets received: %20"PRIu64, - portid, - ctrl->vm_ports[portid].stats.tx, - ctrl->vm_ports[portid].stats.rx); - - total_vm_packets_dropped += ctrl->vm_ports[portid].stats.dropped; - total_vm_packets_tx += ctrl->vm_ports[portid].stats.tx; - total_vm_packets_rx += ctrl->vm_ports[portid].stats.rx; - } - printf("\nAggregate statistics ===============================" - "\nTotal packets sent: %18"PRIu64 - "\nTotal packets received: %14"PRIu64 - "\nTotal packets dropped: %15"PRIu64 - "\nTotal VM packets sent: %15"PRIu64 - "\nTotal VM packets received: %11"PRIu64, - total_packets_tx, - total_packets_rx, - total_packets_dropped, - total_vm_packets_tx, - total_vm_packets_rx); - printf("\n====================================================\n"); -} - -static int -print_to_file(const char *cmdline, const char *config_name) -{ - FILE *file; - char path[PATH_MAX]; - - snprintf(path, sizeof(path), QEMU_CMD_FMT, config_name); - file = fopen(path, "w"); - if (file == NULL) { - RTE_LOG(ERR, L2FWD_IVSHMEM, "Could not open '%s' \n", path); - return -1; - } - - RTE_LOG(DEBUG, L2FWD_IVSHMEM, "QEMU command line for config '%s': %s \n", - config_name, cmdline); - - fprintf(file, "%s\n", cmdline); - fclose(file); - return 0; -} - -static int -generate_ivshmem_cmdline(const char *config_name) -{ - char cmdline[PATH_MAX]; - if (rte_ivshmem_metadata_cmdline_generate(cmdline, sizeof(cmdline), - config_name) < 0) - return -1; - - if (print_to_file(cmdline, config_name) < 0) - return -1; - - rte_ivshmem_metadata_dump(stdout, config_name); - return 0; -} - -/* display usage */ -static void -l2fwd_ivshmem_usage(const char *prgname) -{ - printf("%s [EAL options] -- -p PORTMASK [-q NQ -T PERIOD]\n" - " -p PORTMASK: hexadecimal bitmask of ports to configure\n" - " -q NQ: number of queue (=ports) per lcore (default is 1)\n" - " -T PERIOD: statistics will be refreshed each PERIOD seconds " - "(0 to disable, 10 default, 86400 maximum)\n", - prgname); -} - -static unsigned int -l2fwd_ivshmem_parse_nqueue(const char *q_arg) -{ - char *end = NULL; - unsigned long n; - - /* parse hexadecimal string */ - n = strtoul(q_arg, &end, 10); - if ((q_arg[0] == '\0') || (end == NULL) || (*end != '\0')) - return 0; - if (n == 0) - return 0; - if (n >= MAX_RX_QUEUE_PER_LCORE) - return 0; - - return n; -} - -static int -l2fwd_ivshmem_parse_portmask(const char *portmask) -{ - char *end = NULL; - unsigned long pm; - - /* parse hexadecimal string */ - pm = strtoul(portmask, &end, 16); - if ((portmask[0] == '\0') || (end == NULL) || (*end != '\0')) - return -1; - - if (pm == 0) - return -1; - - return pm; -} - -static int -l2fwd_ivshmem_parse_timer_period(const char *q_arg) -{ - char *end = NULL; - int n; - - /* parse number string */ - n = strtol(q_arg, &end, 10); - if ((q_arg[0] == '\0') || (end == NULL) || (*end != '\0')) - return -1; - if (n >= MAX_TIMER_PERIOD) - return -1; - - return n; -} - -/* Parse the argument given in the command line of the application */ -static int -l2fwd_ivshmem_parse_args(int argc, char **argv) -{ - int opt, ret; - char **argvopt; - int option_index; - char *prgname = argv[0]; - static struct option lgopts[] = { - {CMDLINE_OPT_FWD_CONF, 1, 0, 0}, - {NULL, 0, 0, 0} - }; - - argvopt = argv; - - while ((opt = getopt_long(argc, argvopt, "q:p:T:", - lgopts, &option_index)) != EOF) { - - switch (opt) { - /* portmask */ - case 'p': - l2fwd_ivshmem_enabled_port_mask = l2fwd_ivshmem_parse_portmask(optarg); - if (l2fwd_ivshmem_enabled_port_mask == 0) { - printf("invalid portmask\n"); - l2fwd_ivshmem_usage(prgname); - return -1; - } - break; - - /* nqueue */ - case 'q': - l2fwd_ivshmem_rx_queue_per_lcore = l2fwd_ivshmem_parse_nqueue(optarg); - if (l2fwd_ivshmem_rx_queue_per_lcore == 0) { - printf("invalid queue number\n"); - l2fwd_ivshmem_usage(prgname); - return -1; - } - break; - - /* timer period */ - case 'T': - timer_period = l2fwd_ivshmem_parse_timer_period(optarg) * 1000 * TIMER_MILLISECOND; - if (timer_period < 0) { - printf("invalid timer period\n"); - l2fwd_ivshmem_usage(prgname); - return -1; - } - break; - - /* long options */ - case 0: - l2fwd_ivshmem_usage(prgname); - return -1; - - default: - l2fwd_ivshmem_usage(prgname); - return -1; - } - } - - if (optind >= 0) - argv[optind-1] = prgname; - - ret = optind-1; - optind = 0; /* reset getopt lib */ - return ret; -} - -/* Check the link status of all ports in up to 9s, and print them finally */ -static void -check_all_ports_link_status(uint8_t port_num, uint32_t port_mask) -{ -#define CHECK_INTERVAL 100 /* 100ms */ -#define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ - uint8_t portid, count, all_ports_up, print_flag = 0; - struct rte_eth_link link; - - printf("\nChecking link status"); - fflush(stdout); - for (count = 0; count <= MAX_CHECK_TIME; count++) { - all_ports_up = 1; - for (portid = 0; portid < port_num; portid++) { - if ((port_mask & (1 << portid)) == 0) - continue; - memset(&link, 0, sizeof(link)); - rte_eth_link_get_nowait(portid, &link); - /* print link status if flag set */ - if (print_flag == 1) { - if (link.link_status) - printf("Port %d Link Up - speed %u " - "Mbps - %s\n", (uint8_t)portid, - (unsigned)link.link_speed, - (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? - ("full-duplex") : ("half-duplex\n")); - else - printf("Port %d Link Down\n", - (uint8_t)portid); - continue; - } - /* clear all_ports_up flag if any link down */ - if (link.link_status == ETH_LINK_DOWN) { - all_ports_up = 0; - break; - } - } - /* after finally printing all link status, get out */ - if (print_flag == 1) - break; - - if (all_ports_up == 0) { - printf("."); - fflush(stdout); - rte_delay_ms(CHECK_INTERVAL); - } - - /* set the print_flag if all ports up or timeout */ - if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { - print_flag = 1; - printf("done\n"); - } - } -} - -/* Send the burst of packets on an output interface */ -static int -l2fwd_ivshmem_send_burst(struct lcore_queue_conf *qconf, unsigned n, uint8_t port) -{ - struct rte_mbuf **m_table; - unsigned ret; - unsigned queueid =0; - - m_table = (struct rte_mbuf **)qconf->tx_mbufs[port].m_table; - - ret = rte_eth_tx_burst(port, (uint16_t) queueid, m_table, (uint16_t) n); - port_statistics[port].tx += ret; - if (unlikely(ret < n)) { - port_statistics[port].dropped += (n - ret); - do { - rte_pktmbuf_free(m_table[ret]); - } while (++ret < n); - } - - return 0; -} - -/* Enqueue packets for TX and prepare them to be sent on the network */ -static int -l2fwd_ivshmem_send_packet(struct rte_mbuf *m, uint8_t port) -{ - unsigned lcore_id, len; - struct lcore_queue_conf *qconf; - - lcore_id = rte_lcore_id(); - - qconf = &lcore_queue_conf[lcore_id]; - len = qconf->tx_mbufs[port].len; - qconf->tx_mbufs[port].m_table[len] = m; - len++; - - /* enough pkts to be sent */ - if (unlikely(len == MAX_PKT_BURST)) { - l2fwd_ivshmem_send_burst(qconf, MAX_PKT_BURST, port); - len = 0; - } - - qconf->tx_mbufs[port].len = len; - return 0; -} - -static int -l2fwd_ivshmem_receive_burst(struct lcore_queue_conf *qconf, unsigned portid, - unsigned vm_port) -{ - struct rte_mbuf ** m; - struct rte_ring * rx; - unsigned len, pkt_idx; - - m = qconf->rx_mbufs[portid].m_table; - len = qconf->rx_mbufs[portid].len; - rx = qconf->port_param[vm_port]->rx_ring; - - /* if enqueueing failed, ring is probably full, so drop the packets */ - if (rte_ring_enqueue_bulk(rx, (void**) m, len) < 0) { - port_statistics[portid].dropped += len; - - pkt_idx = 0; - do { - rte_pktmbuf_free(m[pkt_idx]); - } while (++pkt_idx < len); - } - else - /* increment rx stats by however many packets we managed to receive */ - port_statistics[portid].rx += len; - - return 0; -} - -/* Enqueue packets for RX and prepare them to be sent to VM */ -static int -l2fwd_ivshmem_receive_packets(struct rte_mbuf ** m, unsigned n, unsigned portid, - unsigned vm_port) -{ - unsigned lcore_id, len, pkt_idx; - struct lcore_queue_conf *qconf; - - lcore_id = rte_lcore_id(); - - qconf = &lcore_queue_conf[lcore_id]; - - len = qconf->rx_mbufs[portid].len; - pkt_idx = 0; - - /* enqueue packets */ - while (pkt_idx < n && len < MAX_PKT_BURST * 2) { - qconf->rx_mbufs[portid].m_table[len++] = m[pkt_idx++]; - } - - /* increment queue len by however many packets we managed to receive */ - qconf->rx_mbufs[portid].len += pkt_idx; - - /* drop the unreceived packets */ - if (unlikely(pkt_idx < n)) { - port_statistics[portid].dropped += n - pkt_idx; - do { - rte_pktmbuf_free(m[pkt_idx]); - } while (++pkt_idx < n); - } - - /* drain the queue halfway through the maximum capacity */ - if (unlikely(qconf->rx_mbufs[portid].len >= MAX_PKT_BURST)) - l2fwd_ivshmem_receive_burst(qconf, portid, vm_port); - - return 0; -} - -/* loop for host forwarding mode. - * the data flow is as follows: - * 1) get packets from TX queue and send it out from a given port - * 2) RX packets from given port and enqueue them on RX ring - * 3) dequeue packets from TX ring and put them on TX queue for a given port - */ -static void -fwd_loop(void) -{ - struct rte_mbuf *pkts_burst[MAX_PKT_BURST * 2]; - struct rte_mbuf *m; - unsigned lcore_id; - uint64_t prev_tsc, diff_tsc, cur_tsc, timer_tsc; - unsigned i, j, portid, nb_rx; - struct lcore_queue_conf *qconf; - struct rte_ring *tx; - const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / US_PER_S * BURST_TX_DRAIN_US; - - prev_tsc = 0; - timer_tsc = 0; - - lcore_id = rte_lcore_id(); - qconf = &lcore_queue_conf[lcore_id]; - - if (qconf->n_rx_port == 0) { - RTE_LOG(INFO, L2FWD_IVSHMEM, "lcore %u has nothing to do\n", lcore_id); - return; - } - - RTE_LOG(INFO, L2FWD_IVSHMEM, "entering main loop on lcore %u\n", lcore_id); - - for (i = 0; i < qconf->n_rx_port; i++) { - - portid = qconf->rx_port_list[i]; - RTE_LOG(INFO, L2FWD_IVSHMEM, " -- lcoreid=%u portid=%u\n", lcore_id, - portid); - } - - while (ctrl->state == STATE_FWD) { - - cur_tsc = rte_rdtsc(); - - /* - * Burst queue drain - */ - diff_tsc = cur_tsc - prev_tsc; - if (unlikely(diff_tsc > drain_tsc)) { - - /* - * TX - */ - for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++) { - if (qconf->tx_mbufs[portid].len == 0) - continue; - l2fwd_ivshmem_send_burst(qconf, - qconf->tx_mbufs[portid].len, - (uint8_t) portid); - qconf->tx_mbufs[portid].len = 0; - } - - /* - * RX - */ - for (i = 0; i < qconf->n_rx_port; i++) { - portid = qconf->rx_port_list[i]; - if (qconf->rx_mbufs[portid].len == 0) - continue; - l2fwd_ivshmem_receive_burst(qconf, portid, i); - qconf->rx_mbufs[portid].len = 0; - } - - /* if timer is enabled */ - if (timer_period > 0) { - - /* advance the timer */ - timer_tsc += diff_tsc; - - /* if timer has reached its timeout */ - if (unlikely(timer_tsc >= (uint64_t) timer_period)) { - - /* do this only on master core */ - if (lcore_id == rte_get_master_lcore()) { - print_stats(); - /* reset the timer */ - timer_tsc = 0; - } - } - } - - prev_tsc = cur_tsc; - } - - /* - * packet RX and forwarding - */ - for (i = 0; i < qconf->n_rx_port; i++) { - - /* RX packets from port and put them on RX ring */ - portid = qconf->rx_port_list[i]; - nb_rx = rte_eth_rx_burst((uint8_t) portid, 0, - pkts_burst, MAX_PKT_BURST); - - if (nb_rx != 0) - l2fwd_ivshmem_receive_packets(pkts_burst, nb_rx, portid, i); - - /* dequeue packets from TX ring and send them to TX queue */ - tx = qconf->port_param[i]->tx_ring; - - nb_rx = rte_ring_count(tx); - - nb_rx = RTE_MIN(nb_rx, (unsigned) MAX_PKT_BURST); - - if (nb_rx == 0) - continue; - - /* should not happen */ - if (unlikely(rte_ring_dequeue_bulk(tx, (void**) pkts_burst, nb_rx) < 0)) { - ctrl->state = STATE_FAIL; - return; - } - - for (j = 0; j < nb_rx; j++) { - m = pkts_burst[j]; - l2fwd_ivshmem_send_packet(m, portid); - } - } - } -} - -static int -l2fwd_ivshmem_launch_one_lcore(__attribute__((unused)) void *dummy) -{ - fwd_loop(); - return 0; -} - -int main(int argc, char **argv) -{ - char name[RTE_RING_NAMESIZE]; - struct rte_ring *r; - struct lcore_queue_conf *qconf; - struct rte_eth_dev_info dev_info; - uint8_t portid, port_nr; - uint8_t nb_ports, nb_ports_available; - uint8_t nb_ports_in_mask; - int ret; - unsigned lcore_id, rx_lcore_id; - - /* init EAL */ - ret = rte_eal_init(argc, argv); - if (ret < 0) - rte_exit(EXIT_FAILURE, "Invalid EAL arguments\n"); - argc -= ret; - argv += ret; - - /* parse application arguments (after the EAL ones) */ - ret = l2fwd_ivshmem_parse_args(argc, argv); - if (ret < 0) - rte_exit(EXIT_FAILURE, "Invalid l2fwd-ivshmem arguments\n"); - - /* create a shared mbuf pool */ - l2fwd_ivshmem_pktmbuf_pool = - rte_pktmbuf_pool_create(MBUF_MP_NAME, NB_MBUF, 32, - 0, RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id()); - if (l2fwd_ivshmem_pktmbuf_pool == NULL) - rte_exit(EXIT_FAILURE, "Cannot init mbuf pool\n"); - - nb_ports = rte_eth_dev_count(); - if (nb_ports == 0) - rte_exit(EXIT_FAILURE, "No Ethernet ports - bye\n"); - - /* - * reserve memzone to communicate with VMs - we cannot use rte_malloc here - * because while it is technically possible, it is a very bad idea to share - * the heap between two primary processes. - */ - ctrl_mz = rte_memzone_reserve(CTRL_MZ_NAME, sizeof(struct ivshmem_ctrl), - SOCKET_ID_ANY, 0); - if (ctrl_mz == NULL) - rte_exit(EXIT_FAILURE, "Cannot reserve control memzone\n"); - ctrl = (struct ivshmem_ctrl*) ctrl_mz->addr; - - memset(ctrl, 0, sizeof(struct ivshmem_ctrl)); - - /* - * Each port is assigned an output port. - */ - nb_ports_in_mask = 0; - for (portid = 0; portid < nb_ports; portid++) { - /* skip ports that are not enabled */ - if ((l2fwd_ivshmem_enabled_port_mask & (1 << portid)) == 0) - continue; - if (portid % 2) { - ctrl->vm_ports[nb_ports_in_mask].dst = &ctrl->vm_ports[nb_ports_in_mask-1]; - ctrl->vm_ports[nb_ports_in_mask-1].dst = &ctrl->vm_ports[nb_ports_in_mask]; - } - - nb_ports_in_mask++; - - rte_eth_dev_info_get(portid, &dev_info); - } - if (nb_ports_in_mask % 2) { - printf("Notice: odd number of ports in portmask.\n"); - ctrl->vm_ports[nb_ports_in_mask-1].dst = - &ctrl->vm_ports[nb_ports_in_mask-1]; - } - - rx_lcore_id = 0; - qconf = NULL; - - printf("Initializing ports configuration...\n"); - - nb_ports_available = nb_ports; - - /* Initialise each port */ - for (portid = 0; portid < nb_ports; portid++) { - - /* skip ports that are not enabled */ - if ((l2fwd_ivshmem_enabled_port_mask & (1 << portid)) == 0) { - printf("Skipping disabled port %u\n", (unsigned) portid); - nb_ports_available--; - continue; - } - - /* init port */ - printf("Initializing port %u... ", (unsigned) portid); - fflush(stdout); - ret = rte_eth_dev_configure(portid, 1, 1, &port_conf); - if (ret < 0) - rte_exit(EXIT_FAILURE, "Cannot configure device: err=%d, port=%u\n", - ret, (unsigned) portid); - - rte_eth_macaddr_get(portid,&l2fwd_ivshmem_ports_eth_addr[portid]); - - /* init one RX queue */ - fflush(stdout); - ret = rte_eth_rx_queue_setup(portid, 0, nb_rxd, - rte_eth_dev_socket_id(portid), - NULL, - l2fwd_ivshmem_pktmbuf_pool); - if (ret < 0) - rte_exit(EXIT_FAILURE, "rte_eth_rx_queue_setup:err=%d, port=%u\n", - ret, (unsigned) portid); - - /* init one TX queue on each port */ - fflush(stdout); - ret = rte_eth_tx_queue_setup(portid, 0, nb_txd, - rte_eth_dev_socket_id(portid), - NULL); - if (ret < 0) - rte_exit(EXIT_FAILURE, "rte_eth_tx_queue_setup:err=%d, port=%u\n", - ret, (unsigned) portid); - - /* Start device */ - ret = rte_eth_dev_start(portid); - if (ret < 0) - rte_exit(EXIT_FAILURE, "rte_eth_dev_start:err=%d, port=%u\n", - ret, (unsigned) portid); - - printf("done: \n"); - - rte_eth_promiscuous_enable(portid); - - printf("Port %u, MAC address: %02X:%02X:%02X:%02X:%02X:%02X\n\n", - (unsigned) portid, - l2fwd_ivshmem_ports_eth_addr[portid].addr_bytes[0], - l2fwd_ivshmem_ports_eth_addr[portid].addr_bytes[1], - l2fwd_ivshmem_ports_eth_addr[portid].addr_bytes[2], - l2fwd_ivshmem_ports_eth_addr[portid].addr_bytes[3], - l2fwd_ivshmem_ports_eth_addr[portid].addr_bytes[4], - l2fwd_ivshmem_ports_eth_addr[portid].addr_bytes[5]); - - /* initialize port stats */ - memset(&port_statistics, 0, sizeof(port_statistics)); - } - - if (!nb_ports_available) { - rte_exit(EXIT_FAILURE, - "All available ports are disabled. Please set portmask.\n"); - } - port_nr = 0; - - /* Initialize the port/queue configuration of each logical core */ - for (portid = 0; portid < nb_ports; portid++) { - if ((l2fwd_ivshmem_enabled_port_mask & (1 << portid)) == 0) - continue; - - /* get the lcore_id for this port */ - while (rte_lcore_is_enabled(rx_lcore_id) == 0 || - lcore_queue_conf[rx_lcore_id].n_rx_port == - l2fwd_ivshmem_rx_queue_per_lcore) { - rx_lcore_id++; - if (rx_lcore_id >= RTE_MAX_LCORE) - rte_exit(EXIT_FAILURE, "Not enough cores\n"); - } - - if (qconf != &lcore_queue_conf[rx_lcore_id]) - /* Assigned a new logical core in the loop above. */ - qconf = &lcore_queue_conf[rx_lcore_id]; - - - rte_eth_macaddr_get(portid, &ctrl->vm_ports[port_nr].ethaddr); - - qconf->rx_port_list[qconf->n_rx_port] = portid; - qconf->port_param[qconf->n_rx_port] = &ctrl->vm_ports[port_nr]; - qconf->n_rx_port++; - port_nr++; - printf("Lcore %u: RX port %u\n", rx_lcore_id, (unsigned) portid); - } - - check_all_ports_link_status(nb_ports_available, l2fwd_ivshmem_enabled_port_mask); - - /* create rings for each VM port (several ports can be on the same VM). - * note that we store the pointers in ctrl - that way, they are the same - * and valid across all VMs because ctrl is also in DPDK memory */ - for (portid = 0; portid < nb_ports_available; portid++) { - - /* RX ring. SP/SC because it's only used by host and a single VM */ - snprintf(name, sizeof(name), "%s%i", RX_RING_PREFIX, portid); - r = rte_ring_create(name, NB_MBUF, - SOCKET_ID_ANY, RING_F_SP_ENQ | RING_F_SC_DEQ); - if (r == NULL) - rte_exit(EXIT_FAILURE, "Cannot create ring %s\n", name); - - ctrl->vm_ports[portid].rx_ring = r; - - /* TX ring. SP/SC because it's only used by host and a single VM */ - snprintf(name, sizeof(name), "%s%i", TX_RING_PREFIX, portid); - r = rte_ring_create(name, NB_MBUF, - SOCKET_ID_ANY, RING_F_SP_ENQ | RING_F_SC_DEQ); - if (r == NULL) - rte_exit(EXIT_FAILURE, "Cannot create ring %s\n", name); - - ctrl->vm_ports[portid].tx_ring = r; - } - - /* create metadata, output cmdline */ - if (rte_ivshmem_metadata_create(METADATA_NAME) < 0) - rte_exit(EXIT_FAILURE, "Cannot create IVSHMEM metadata\n"); - - if (rte_ivshmem_metadata_add_memzone(ctrl_mz, METADATA_NAME)) - rte_exit(EXIT_FAILURE, "Cannot add memzone to IVSHMEM metadata\n"); - - if (rte_ivshmem_metadata_add_mempool(l2fwd_ivshmem_pktmbuf_pool, METADATA_NAME)) - rte_exit(EXIT_FAILURE, "Cannot add mbuf mempool to IVSHMEM metadata\n"); - - for (portid = 0; portid < nb_ports_available; portid++) { - if (rte_ivshmem_metadata_add_ring(ctrl->vm_ports[portid].rx_ring, - METADATA_NAME) < 0) - rte_exit(EXIT_FAILURE, "Cannot add ring %s to IVSHMEM metadata\n", - ctrl->vm_ports[portid].rx_ring->name); - if (rte_ivshmem_metadata_add_ring(ctrl->vm_ports[portid].tx_ring, - METADATA_NAME) < 0) - rte_exit(EXIT_FAILURE, "Cannot add ring %s to IVSHMEM metadata\n", - ctrl->vm_ports[portid].tx_ring->name); - } - generate_ivshmem_cmdline(METADATA_NAME); - - ctrl->nb_ports = nb_ports_available; - - printf("Waiting for VM to initialize...\n"); - - /* wait for VM to initialize */ - while (ctrl->state != STATE_FWD) { - if (ctrl->state == STATE_FAIL) - rte_exit(EXIT_FAILURE, "VM reported failure\n"); - - sleep(1); - } - - printf("Done!\n"); - - sigsetup(); - - /* launch per-lcore init on every lcore */ - rte_eal_mp_remote_launch(l2fwd_ivshmem_launch_one_lcore, NULL, CALL_MASTER); - RTE_LCORE_FOREACH_SLAVE(lcore_id) { - if (rte_eal_wait_lcore(lcore_id) < 0) - return -1; - } - - if (ctrl->state == STATE_FAIL) - rte_exit(EXIT_FAILURE, "VM reported failure\n"); - - return 0; -} diff --git a/dpdk/examples/l2fwd-ivshmem/include/common.h b/dpdk/examples/l2fwd-ivshmem/include/common.h deleted file mode 100644 index 8564d32b..00000000 --- a/dpdk/examples/l2fwd-ivshmem/include/common.h +++ /dev/null @@ -1,111 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef _IVSHMEM_COMMON_H_ -#define _IVSHMEM_COMMON_H_ - -#define RTE_LOGTYPE_L2FWD_IVSHMEM RTE_LOGTYPE_USER1 - -#define CTRL_MZ_NAME "CTRL_MEMZONE" -#define MBUF_MP_NAME "MBUF_MEMPOOL" -#define RX_RING_PREFIX "RX_" -#define TX_RING_PREFIX "TX_" - -/* A tsc-based timer responsible for triggering statistics printout */ -#define TIMER_MILLISECOND 2000000ULL /* around 1ms at 2 Ghz */ -#define MAX_TIMER_PERIOD 86400 /* 1 day max */ -static int64_t timer_period = 10 * TIMER_MILLISECOND * 1000; /* default period is 10 seconds */ - -#define DIM(x)\ - (sizeof(x)/sizeof(x)[0]) - -#define MAX_PKT_BURST 32 - -const struct rte_memzone * ctrl_mz; - -enum l2fwd_state { - STATE_NONE = 0, - STATE_FWD, - STATE_EXIT, - STATE_FAIL -}; - -/* Per-port statistics struct */ -struct port_statistics { - uint64_t tx; - uint64_t rx; - uint64_t dropped; -} __rte_cache_aligned; - -struct mbuf_table { - unsigned len; - struct rte_mbuf *m_table[MAX_PKT_BURST * 2]; /**< allow up to two bursts */ -}; - -struct vm_port_param { - struct rte_ring * rx_ring; /**< receiving ring for current port */ - struct rte_ring * tx_ring; /**< transmitting ring for current port */ - struct vm_port_param * dst; /**< current port's destination port */ - volatile struct port_statistics stats; /**< statistics for current port */ - struct ether_addr ethaddr; /**< Ethernet address of the port */ -}; - -/* control structure, to synchronize host and VM */ -struct ivshmem_ctrl { - rte_spinlock_t lock; - uint8_t nb_ports; /**< total nr of ports */ - volatile enum l2fwd_state state; /**< report state */ - struct vm_port_param vm_ports[RTE_MAX_ETHPORTS]; -}; - -struct ivshmem_ctrl * ctrl; - -static unsigned int l2fwd_ivshmem_rx_queue_per_lcore = 1; - -static void sighandler(int __rte_unused s) -{ - ctrl->state = STATE_EXIT; -} - -static void sigsetup(void) -{ - struct sigaction sigIntHandler; - - sigIntHandler.sa_handler = sighandler; - sigemptyset(&sigIntHandler.sa_mask); - sigIntHandler.sa_flags = 0; - - sigaction(SIGINT, &sigIntHandler, NULL); -} - -#endif /* _IVSHMEM_COMMON_H_ */ diff --git a/dpdk/examples/l2fwd-jobstats/main.c b/dpdk/examples/l2fwd-jobstats/main.c index 614ea604..485370de 100644 --- a/dpdk/examples/l2fwd-jobstats/main.c +++ b/dpdk/examples/l2fwd-jobstats/main.c @@ -38,15 +38,12 @@ #include #include -#include #include #include #include #include #include -#include #include -#include #include #include #include @@ -55,11 +52,9 @@ #include #include #include -#include #include #include #include -#include #include #include #include @@ -68,6 +63,7 @@ #include #include #include +#include #define RTE_LOGTYPE_L2FWD RTE_LOGTYPE_USER1 @@ -127,7 +123,7 @@ static const struct rte_eth_conf port_conf = { .hw_ip_checksum = 0, /**< IP checksum offload disabled */ .hw_vlan_filter = 0, /**< VLAN filtering disabled */ .jumbo_frame = 0, /**< Jumbo Frame Support disabled */ - .hw_strip_crc = 0, /**< CRC stripped by hardware */ + .hw_strip_crc = 1, /**< CRC stripped by hardware */ }, .txmode = { .mq_mode = ETH_MQ_TX_NONE, @@ -178,7 +174,7 @@ show_lcore_stats(unsigned lcore_id) uint64_t busy, busy_min, busy_max; /* Jobs statistics. */ - const uint8_t port_cnt = qconf->n_rx_port; + const uint16_t port_cnt = qconf->n_rx_port; uint64_t jobs_exec_cnt[port_cnt], jobs_period[port_cnt]; uint64_t jobs_exec[port_cnt], jobs_exec_min[port_cnt], jobs_exec_max[port_cnt]; @@ -416,11 +412,11 @@ l2fwd_fwd_job(__rte_unused struct rte_timer *timer, void *arg) struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; struct rte_mbuf *m; - const uint8_t port_idx = (uintptr_t) arg; + const uint16_t port_idx = (uintptr_t) arg; const unsigned lcore_id = rte_lcore_id(); struct lcore_queue_conf *qconf = &lcore_queue_conf[lcore_id]; struct rte_jobstats *job = &qconf->port_fwd_jobs[port_idx]; - const uint8_t portid = qconf->rx_port_list[port_idx]; + const uint16_t portid = qconf->rx_port_list[port_idx]; uint8_t j; uint16_t total_nb_rx; @@ -430,7 +426,7 @@ l2fwd_fwd_job(__rte_unused struct rte_timer *timer, void *arg) /* Call rx burst 2 times. This allow rte_jobstats logic to see if this * function must be called more frequently. */ - total_nb_rx = rte_eth_rx_burst((uint8_t) portid, 0, pkts_burst, + total_nb_rx = rte_eth_rx_burst(portid, 0, pkts_burst, MAX_PKT_BURST); for (j = 0; j < total_nb_rx; j++) { @@ -440,7 +436,7 @@ l2fwd_fwd_job(__rte_unused struct rte_timer *timer, void *arg) } if (total_nb_rx == MAX_PKT_BURST) { - const uint16_t nb_rx = rte_eth_rx_burst((uint8_t) portid, 0, pkts_burst, + const uint16_t nb_rx = rte_eth_rx_burst(portid, 0, pkts_burst, MAX_PKT_BURST); total_nb_rx += nb_rx; @@ -466,7 +462,7 @@ l2fwd_flush_job(__rte_unused struct rte_timer *timer, __rte_unused void *arg) uint64_t now; unsigned lcore_id; struct lcore_queue_conf *qconf; - uint8_t portid; + uint16_t portid; unsigned i; uint32_t sent; struct rte_eth_dev_tx_buffer *buffer; @@ -710,17 +706,18 @@ l2fwd_parse_args(int argc, char **argv) argv[optind-1] = prgname; ret = optind-1; - optind = 0; /* reset getopt lib */ + optind = 1; /* reset getopt lib */ return ret; } /* Check the link status of all ports in up to 9s, and print them finally */ static void -check_all_ports_link_status(uint8_t port_num, uint32_t port_mask) +check_all_ports_link_status(uint16_t port_num, uint32_t port_mask) { #define CHECK_INTERVAL 100 /* 100ms */ #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ - uint8_t portid, count, all_ports_up, print_flag = 0; + uint16_t portid; + uint8_t count, all_ports_up, print_flag = 0; struct rte_eth_link link; printf("\nChecking link status"); @@ -735,14 +732,13 @@ check_all_ports_link_status(uint8_t port_num, uint32_t port_mask) /* print link status if flag set */ if (print_flag == 1) { if (link.link_status) - printf("Port %d Link Up - speed %u " - "Mbps - %s\n", (uint8_t)portid, - (unsigned)link.link_speed, + printf( + "Port%d Link Up. Speed %u Mbps - %s\n", + portid, link.link_speed, (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? ("full-duplex") : ("half-duplex\n")); else - printf("Port %d Link Down\n", - (uint8_t)portid); + printf("Port %d Link Down\n", portid); continue; } /* clear all_ports_up flag if any link down */ @@ -778,9 +774,9 @@ main(int argc, char **argv) unsigned nb_ports_in_mask = 0; int ret; char name[RTE_JOBSTATS_NAMESIZE]; - uint8_t nb_ports; - uint8_t nb_ports_available; - uint8_t portid, last_port; + uint16_t nb_ports; + uint16_t nb_ports_available; + uint16_t portid, last_port; uint8_t i; /* init EAL */ @@ -863,7 +859,7 @@ main(int argc, char **argv) qconf->rx_port_list[qconf->n_rx_port] = portid; qconf->n_rx_port++; - printf("Lcore %u: RX port %u\n", rx_lcore_id, (unsigned) portid); + printf("Lcore %u: RX port %u\n", rx_lcore_id, portid); } nb_ports_available = nb_ports; @@ -872,17 +868,24 @@ main(int argc, char **argv) for (portid = 0; portid < nb_ports; portid++) { /* skip ports that are not enabled */ if ((l2fwd_enabled_port_mask & (1 << portid)) == 0) { - printf("Skipping disabled port %u\n", (unsigned) portid); + printf("Skipping disabled port %u\n", portid); nb_ports_available--; continue; } /* init port */ - printf("Initializing port %u... ", (unsigned) portid); + printf("Initializing port %u... ", portid); fflush(stdout); ret = rte_eth_dev_configure(portid, 1, 1, &port_conf); if (ret < 0) rte_exit(EXIT_FAILURE, "Cannot configure device: err=%d, port=%u\n", - ret, (unsigned) portid); + ret, portid); + + ret = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd, + &nb_txd); + if (ret < 0) + rte_exit(EXIT_FAILURE, + "Cannot adjust number of descriptors: err=%d, port=%u\n", + ret, portid); rte_eth_macaddr_get(portid, &l2fwd_ports_eth_addr[portid]); @@ -894,7 +897,7 @@ main(int argc, char **argv) l2fwd_pktmbuf_pool); if (ret < 0) rte_exit(EXIT_FAILURE, "rte_eth_rx_queue_setup:err=%d, port=%u\n", - ret, (unsigned) portid); + ret, portid); /* init one TX queue on each port */ fflush(stdout); @@ -902,8 +905,9 @@ main(int argc, char **argv) rte_eth_dev_socket_id(portid), NULL); if (ret < 0) - rte_exit(EXIT_FAILURE, "rte_eth_tx_queue_setup:err=%d, port=%u\n", - ret, (unsigned) portid); + rte_exit(EXIT_FAILURE, + "rte_eth_tx_queue_setup:err=%d, port=%u\n", + ret, portid); /* Initialize TX buffers */ tx_buffer[portid] = rte_zmalloc_socket("tx_buffer", @@ -911,7 +915,7 @@ main(int argc, char **argv) rte_eth_dev_socket_id(portid)); if (tx_buffer[portid] == NULL) rte_exit(EXIT_FAILURE, "Cannot allocate buffer for tx on port %u\n", - (unsigned) portid); + portid); rte_eth_tx_buffer_init(tx_buffer[portid], MAX_PKT_BURST); @@ -919,21 +923,22 @@ main(int argc, char **argv) rte_eth_tx_buffer_count_callback, &port_statistics[portid].dropped); if (ret < 0) - rte_exit(EXIT_FAILURE, "Cannot set error callback for " - "tx buffer on port %u\n", (unsigned) portid); + rte_exit(EXIT_FAILURE, + "Cannot set error callback for tx buffer on port %u\n", + portid); /* Start device */ ret = rte_eth_dev_start(portid); if (ret < 0) rte_exit(EXIT_FAILURE, "rte_eth_dev_start:err=%d, port=%u\n", - ret, (unsigned) portid); + ret, portid); printf("done:\n"); rte_eth_promiscuous_enable(portid); printf("Port %u, MAC address: %02X:%02X:%02X:%02X:%02X:%02X\n\n", - (unsigned) portid, + portid, l2fwd_ports_eth_addr[portid].addr_bytes[0], l2fwd_ports_eth_addr[portid].addr_bytes[1], l2fwd_ports_eth_addr[portid].addr_bytes[2], diff --git a/dpdk/examples/l2fwd-keepalive/ka-agent/main.c b/dpdk/examples/l2fwd-keepalive/ka-agent/main.c index be1c7f49..ba0ac352 100644 --- a/dpdk/examples/l2fwd-keepalive/ka-agent/main.c +++ b/dpdk/examples/l2fwd-keepalive/ka-agent/main.c @@ -31,7 +31,6 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#include #include #include #include diff --git a/dpdk/examples/l2fwd-keepalive/main.c b/dpdk/examples/l2fwd-keepalive/main.c index 84a59eb6..358ca5ec 100644 --- a/dpdk/examples/l2fwd-keepalive/main.c +++ b/dpdk/examples/l2fwd-keepalive/main.c @@ -44,15 +44,14 @@ #include #include #include +#include #include #include #include #include #include -#include #include -#include #include #include #include @@ -61,12 +60,10 @@ #include #include #include -#include #include #include #include #include -#include #include #include #include @@ -117,7 +114,7 @@ static const struct rte_eth_conf port_conf = { .hw_ip_checksum = 0, /**< IP checksum offload disabled */ .hw_vlan_filter = 0, /**< VLAN filtering disabled */ .jumbo_frame = 0, /**< Jumbo Frame Support disabled */ - .hw_strip_crc = 0, /**< CRC stripped by hardware */ + .hw_strip_crc = 1, /**< CRC stripped by hardware */ }, .txmode = { .mq_mode = ETH_MQ_TX_NONE, @@ -143,13 +140,22 @@ static int64_t check_period = 5; /* default check cycle is 5ms */ /* Keepalive structure */ struct rte_keepalive *rte_global_keepalive_info; +/* Termination signalling */ +static int terminate_signal_received; + +/* Termination signal handler */ +static void handle_sigterm(__rte_unused int value) +{ + terminate_signal_received = 1; +} + /* Print out statistics on packets dropped */ static void print_stats(__attribute__((unused)) struct rte_timer *ptr_timer, __attribute__((unused)) void *ptr_data) { uint64_t total_packets_dropped, total_packets_tx, total_packets_rx; - unsigned portid; + uint16_t portid; total_packets_dropped = 0; total_packets_tx = 0; @@ -252,7 +258,7 @@ l2fwd_main_loop(void) uint64_t tsc_initial = rte_rdtsc(); uint64_t tsc_lifetime = (rand()&0x07) * rte_get_tsc_hz(); - while (1) { + while (!terminate_signal_received) { /* Keepalive heartbeat */ rte_keepalive_mark_alive(rte_global_keepalive_info); @@ -291,7 +297,7 @@ l2fwd_main_loop(void) for (i = 0; i < qconf->n_rx_port; i++) { portid = qconf->rx_port_list[i]; - nb_rx = rte_eth_rx_burst((uint8_t) portid, 0, + nb_rx = rte_eth_rx_burst(portid, 0, pkts_burst, MAX_PKT_BURST); port_statistics[portid].rx += nb_rx; @@ -465,17 +471,18 @@ l2fwd_parse_args(int argc, char **argv) argv[optind-1] = prgname; ret = optind-1; - optind = 0; /* reset getopt lib */ + optind = 1; /* reset getopt lib */ return ret; } /* Check the link status of all ports in up to 9s, and print them finally */ static void -check_all_ports_link_status(uint8_t port_num, uint32_t port_mask) +check_all_ports_link_status(uint16_t port_num, uint32_t port_mask) { #define CHECK_INTERVAL 100 /* 100ms */ #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ - uint8_t portid, count, all_ports_up, print_flag = 0; + uint16_t portid; + uint8_t count, all_ports_up, print_flag = 0; struct rte_eth_link link; printf("\nChecking link status"); @@ -490,14 +497,13 @@ check_all_ports_link_status(uint8_t port_num, uint32_t port_mask) /* print link status if flag set */ if (print_flag == 1) { if (link.link_status) - printf("Port %d Link Up - speed %u " - "Mbps - %s\n", (uint8_t)portid, - (unsigned)link.link_speed, + printf( + "Port%d Link Up. Speed %u Mbps - %s\n", + portid, link.link_speed, (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? ("full-duplex") : ("half-duplex\n")); else - printf("Port %d Link Down\n", - (uint8_t)portid); + printf("Port %d Link Down\n", portid); continue; } /* clear all_ports_up flag if any link down */ @@ -527,6 +533,8 @@ check_all_ports_link_status(uint8_t port_num, uint32_t port_mask) static void dead_core(__rte_unused void *ptr_data, const int id_core) { + if (terminate_signal_received) + return; printf("Dead core %i - restarting..\n", id_core); if (rte_eal_get_lcore_state(id_core) == FINISHED) { rte_eal_wait_lcore(id_core); @@ -550,11 +558,21 @@ main(int argc, char **argv) struct lcore_queue_conf *qconf; struct rte_eth_dev_info dev_info; int ret; - uint8_t nb_ports; - uint8_t nb_ports_available; - uint8_t portid, last_port; + uint16_t nb_ports; + uint16_t nb_ports_available; + uint16_t portid, last_port; unsigned lcore_id, rx_lcore_id; unsigned nb_ports_in_mask = 0; + struct sigaction signal_handler; + struct rte_keepalive_shm *ka_shm; + + memset(&signal_handler, 0, sizeof(signal_handler)); + terminate_signal_received = 0; + signal_handler.sa_handler = &handle_sigterm; + if (sigaction(SIGINT, &signal_handler, NULL) == -1 || + sigaction(SIGTERM, &signal_handler, NULL) == -1) + rte_exit(EXIT_FAILURE, "SIGNAL\n"); + /* init EAL */ ret = rte_eal_init(argc, argv); @@ -633,7 +651,7 @@ main(int argc, char **argv) qconf->rx_port_list[qconf->n_rx_port] = portid; qconf->n_rx_port++; printf("Lcore %u: RX port %u\n", - rx_lcore_id, (unsigned) portid); + rx_lcore_id, portid); } nb_ports_available = nb_ports; @@ -642,19 +660,25 @@ main(int argc, char **argv) for (portid = 0; portid < nb_ports; portid++) { /* skip ports that are not enabled */ if ((l2fwd_enabled_port_mask & (1 << portid)) == 0) { - printf("Skipping disabled port %u\n", - (unsigned) portid); + printf("Skipping disabled port %u\n", portid); nb_ports_available--; continue; } /* init port */ - printf("Initializing port %u... ", (unsigned) portid); + printf("Initializing port %u... ", portid); fflush(stdout); ret = rte_eth_dev_configure(portid, 1, 1, &port_conf); if (ret < 0) rte_exit(EXIT_FAILURE, "Cannot configure device: err=%d, port=%u\n", - ret, (unsigned) portid); + ret, portid); + + ret = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd, + &nb_txd); + if (ret < 0) + rte_exit(EXIT_FAILURE, + "Cannot adjust number of descriptors: err=%d, port=%u\n", + ret, portid); rte_eth_macaddr_get(portid, &l2fwd_ports_eth_addr[portid]); @@ -667,7 +691,7 @@ main(int argc, char **argv) if (ret < 0) rte_exit(EXIT_FAILURE, "rte_eth_rx_queue_setup:err=%d, port=%u\n", - ret, (unsigned) portid); + ret, portid); /* init one TX queue on each port */ fflush(stdout); @@ -677,7 +701,7 @@ main(int argc, char **argv) if (ret < 0) rte_exit(EXIT_FAILURE, "rte_eth_tx_queue_setup:err=%d, port=%u\n", - ret, (unsigned) portid); + ret, portid); /* Initialize TX buffers */ tx_buffer[portid] = rte_zmalloc_socket("tx_buffer", @@ -685,7 +709,7 @@ main(int argc, char **argv) rte_eth_dev_socket_id(portid)); if (tx_buffer[portid] == NULL) rte_exit(EXIT_FAILURE, "Cannot allocate buffer for tx on port %u\n", - (unsigned) portid); + portid); rte_eth_tx_buffer_init(tx_buffer[portid], MAX_PKT_BURST); @@ -693,21 +717,22 @@ main(int argc, char **argv) rte_eth_tx_buffer_count_callback, &port_statistics[portid].dropped); if (ret < 0) - rte_exit(EXIT_FAILURE, "Cannot set error callback for " - "tx buffer on port %u\n", (unsigned) portid); + rte_exit(EXIT_FAILURE, + "Cannot set error callback for tx buffer on port %u\n", + portid); /* Start device */ ret = rte_eth_dev_start(portid); if (ret < 0) rte_exit(EXIT_FAILURE, "rte_eth_dev_start:err=%d, port=%u\n", - ret, (unsigned) portid); + ret, portid); rte_eth_promiscuous_enable(portid); printf("Port %u, MAC address: " "%02X:%02X:%02X:%02X:%02X:%02X\n\n", - (unsigned) portid, + portid, l2fwd_ports_eth_addr[portid].addr_bytes[0], l2fwd_ports_eth_addr[portid].addr_bytes[1], l2fwd_ports_eth_addr[portid].addr_bytes[2], @@ -731,9 +756,8 @@ main(int argc, char **argv) rte_timer_subsystem_init(); rte_timer_init(&stats_timer); + ka_shm = NULL; if (check_period > 0) { - struct rte_keepalive_shm *ka_shm; - ka_shm = rte_keepalive_shm_create(); if (ka_shm == NULL) rte_exit(EXIT_FAILURE, @@ -783,7 +807,7 @@ main(int argc, char **argv) lcore_id); } } - for (;;) { + while (!terminate_signal_received) { rte_timer_manage(); rte_delay_ms(5); } @@ -793,5 +817,7 @@ main(int argc, char **argv) return -1; } + if (ka_shm != NULL) + rte_keepalive_shm_cleanup(ka_shm); return 0; } diff --git a/dpdk/examples/l2fwd-keepalive/shm.c b/dpdk/examples/l2fwd-keepalive/shm.c index 177aa5b8..fbf5bd79 100644 --- a/dpdk/examples/l2fwd-keepalive/shm.c +++ b/dpdk/examples/l2fwd-keepalive/shm.c @@ -129,3 +129,13 @@ void rte_keepalive_relayed_state(struct rte_keepalive_shm *shm, strerror(errno)); } } + +void rte_keepalive_shm_cleanup(struct rte_keepalive_shm *ka_shm) +{ + if (shm_unlink(RTE_KEEPALIVE_SHM_NAME) == -1 && errno != ENOENT) + printf("Warning: Error unlinking %s (%s)\n", + RTE_KEEPALIVE_SHM_NAME, strerror(errno)); + + if (ka_shm && munmap(ka_shm, sizeof(struct rte_keepalive_shm)) != 0) + printf("Warning: munmap() failed\n"); +} diff --git a/dpdk/examples/l2fwd-keepalive/shm.h b/dpdk/examples/l2fwd-keepalive/shm.h index 25e1b61d..66a60600 100644 --- a/dpdk/examples/l2fwd-keepalive/shm.h +++ b/dpdk/examples/l2fwd-keepalive/shm.h @@ -87,3 +87,12 @@ struct rte_keepalive_shm *rte_keepalive_shm_create(void); void rte_keepalive_relayed_state(struct rte_keepalive_shm *shm, const int id_core, const enum rte_keepalive_state core_state, uint64_t last_alive); + +/** Shutdown cleanup of shared host memory keepalive object. + * @param *shm + * Pointer to SHM keepalive structure. May be NULL. + * + * If *shm is NULL, this function will only attempt to remove the + * shared host memory handle and not unmap the underlying memory. + */ +void rte_keepalive_shm_cleanup(struct rte_keepalive_shm *ka_shm); diff --git a/dpdk/examples/l2fwd/Makefile b/dpdk/examples/l2fwd/Makefile index 78feeeb8..8896ab45 100644 --- a/dpdk/examples/l2fwd/Makefile +++ b/dpdk/examples/l2fwd/Makefile @@ -33,7 +33,7 @@ ifeq ($(RTE_SDK),) $(error "Please define RTE_SDK environment variable") endif -# Default target, can be overriden by command line or environment +# Default target, can be overridden by command line or environment RTE_TARGET ?= x86_64-native-linuxapp-gcc include $(RTE_SDK)/mk/rte.vars.mk diff --git a/dpdk/examples/l2fwd/main.c b/dpdk/examples/l2fwd/main.c index 88979216..e89e2e1b 100644 --- a/dpdk/examples/l2fwd/main.c +++ b/dpdk/examples/l2fwd/main.c @@ -52,9 +52,7 @@ #include #include #include -#include #include -#include #include #include #include @@ -63,17 +61,18 @@ #include #include #include -#include #include #include #include #include -#include #include #include static volatile bool force_quit; +/* MAC updating enabled by default */ +static int mac_updating = 1; + #define RTE_LOGTYPE_L2FWD RTE_LOGTYPE_USER1 #define NB_MBUF 8192 @@ -118,7 +117,7 @@ static const struct rte_eth_conf port_conf = { .hw_ip_checksum = 0, /**< IP checksum offload disabled */ .hw_vlan_filter = 0, /**< VLAN filtering disabled */ .jumbo_frame = 0, /**< Jumbo Frame Support disabled */ - .hw_strip_crc = 0, /**< CRC stripped by hardware */ + .hw_strip_crc = 1, /**< CRC stripped by hardware */ }, .txmode = { .mq_mode = ETH_MQ_TX_NONE, @@ -186,23 +185,32 @@ print_stats(void) } static void -l2fwd_simple_forward(struct rte_mbuf *m, unsigned portid) +l2fwd_mac_updating(struct rte_mbuf *m, unsigned dest_portid) { struct ether_hdr *eth; void *tmp; + + eth = rte_pktmbuf_mtod(m, struct ether_hdr *); + + /* 02:00:00:00:00:xx */ + tmp = ð->d_addr.addr_bytes[0]; + *((uint64_t *)tmp) = 0x000000000002 + ((uint64_t)dest_portid << 40); + + /* src addr */ + ether_addr_copy(&l2fwd_ports_eth_addr[dest_portid], ð->s_addr); +} + +static void +l2fwd_simple_forward(struct rte_mbuf *m, unsigned portid) +{ unsigned dst_port; int sent; struct rte_eth_dev_tx_buffer *buffer; dst_port = l2fwd_dst_ports[portid]; - eth = rte_pktmbuf_mtod(m, struct ether_hdr *); - /* 02:00:00:00:00:xx */ - tmp = ð->d_addr.addr_bytes[0]; - *((uint64_t *)tmp) = 0x000000000002 + ((uint64_t)dst_port << 40); - - /* src addr */ - ether_addr_copy(&l2fwd_ports_eth_addr[dst_port], ð->s_addr); + if (mac_updating) + l2fwd_mac_updating(m, dst_port); buffer = tx_buffer[dst_port]; sent = rte_eth_tx_buffer(dst_port, 0, buffer, m); @@ -294,7 +302,7 @@ l2fwd_main_loop(void) for (i = 0; i < qconf->n_rx_port; i++) { portid = qconf->rx_port_list[i]; - nb_rx = rte_eth_rx_burst((uint8_t) portid, 0, + nb_rx = rte_eth_rx_burst(portid, 0, pkts_burst, MAX_PKT_BURST); port_statistics[portid].rx += nb_rx; @@ -322,7 +330,11 @@ l2fwd_usage(const char *prgname) printf("%s [EAL options] -- -p PORTMASK [-q NQ]\n" " -p PORTMASK: hexadecimal bitmask of ports to configure\n" " -q NQ: number of queue (=ports) per lcore (default is 1)\n" - " -T PERIOD: statistics will be refreshed each PERIOD seconds (0 to disable, 10 default, 86400 maximum)\n", + " -T PERIOD: statistics will be refreshed each PERIOD seconds (0 to disable, 10 default, 86400 maximum)\n" + " --[no-]mac-updating: Enable or disable MAC addresses updating (enabled by default)\n" + " When enabled:\n" + " - The source MAC address is replaced by the TX port MAC address\n" + " - The destination MAC address is replaced by 02:00:00:00:00:TX_PORT_ID\n", prgname); } @@ -377,6 +389,29 @@ l2fwd_parse_timer_period(const char *q_arg) return n; } +static const char short_options[] = + "p:" /* portmask */ + "q:" /* number of queues */ + "T:" /* timer period */ + ; + +#define CMD_LINE_OPT_MAC_UPDATING "mac-updating" +#define CMD_LINE_OPT_NO_MAC_UPDATING "no-mac-updating" + +enum { + /* long options mapped to a short option */ + + /* first long only option value must be >= 256, so that we won't + * conflict with short options */ + CMD_LINE_OPT_MIN_NUM = 256, +}; + +static const struct option lgopts[] = { + { CMD_LINE_OPT_MAC_UPDATING, no_argument, &mac_updating, 1}, + { CMD_LINE_OPT_NO_MAC_UPDATING, no_argument, &mac_updating, 0}, + {NULL, 0, 0, 0} +}; + /* Parse the argument given in the command line of the application */ static int l2fwd_parse_args(int argc, char **argv) @@ -385,13 +420,10 @@ l2fwd_parse_args(int argc, char **argv) char **argvopt; int option_index; char *prgname = argv[0]; - static struct option lgopts[] = { - {NULL, 0, 0, 0} - }; argvopt = argv; - while ((opt = getopt_long(argc, argvopt, "p:q:T:", + while ((opt = getopt_long(argc, argvopt, short_options, lgopts, &option_index)) != EOF) { switch (opt) { @@ -428,8 +460,7 @@ l2fwd_parse_args(int argc, char **argv) /* long options */ case 0: - l2fwd_usage(prgname); - return -1; + break; default: l2fwd_usage(prgname); @@ -441,17 +472,18 @@ l2fwd_parse_args(int argc, char **argv) argv[optind-1] = prgname; ret = optind-1; - optind = 0; /* reset getopt lib */ + optind = 1; /* reset getopt lib */ return ret; } /* Check the link status of all ports in up to 9s, and print them finally */ static void -check_all_ports_link_status(uint8_t port_num, uint32_t port_mask) +check_all_ports_link_status(uint16_t port_num, uint32_t port_mask) { #define CHECK_INTERVAL 100 /* 100ms */ #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ - uint8_t portid, count, all_ports_up, print_flag = 0; + uint16_t portid; + uint8_t count, all_ports_up, print_flag = 0; struct rte_eth_link link; printf("\nChecking link status"); @@ -470,14 +502,13 @@ check_all_ports_link_status(uint8_t port_num, uint32_t port_mask) /* print link status if flag set */ if (print_flag == 1) { if (link.link_status) - printf("Port %d Link Up - speed %u " - "Mbps - %s\n", (uint8_t)portid, - (unsigned)link.link_speed, + printf( + "Port%d Link Up. Speed %u Mbps - %s\n", + portid, link.link_speed, (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? ("full-duplex") : ("half-duplex\n")); else - printf("Port %d Link Down\n", - (uint8_t)portid); + printf("Port %d Link Down\n", portid); continue; } /* clear all_ports_up flag if any link down */ @@ -520,9 +551,9 @@ main(int argc, char **argv) struct lcore_queue_conf *qconf; struct rte_eth_dev_info dev_info; int ret; - uint8_t nb_ports; - uint8_t nb_ports_available; - uint8_t portid, last_port; + uint16_t nb_ports; + uint16_t nb_ports_available; + uint16_t portid, last_port; unsigned lcore_id, rx_lcore_id; unsigned nb_ports_in_mask = 0; @@ -542,6 +573,8 @@ main(int argc, char **argv) if (ret < 0) rte_exit(EXIT_FAILURE, "Invalid L2FWD arguments\n"); + printf("MAC updating %s\n", mac_updating ? "enabled" : "disabled"); + /* convert to number of cycles */ timer_period *= rte_get_timer_hz(); @@ -609,7 +642,7 @@ main(int argc, char **argv) qconf->rx_port_list[qconf->n_rx_port] = portid; qconf->n_rx_port++; - printf("Lcore %u: RX port %u\n", rx_lcore_id, (unsigned) portid); + printf("Lcore %u: RX port %u\n", rx_lcore_id, portid); } nb_ports_available = nb_ports; @@ -618,17 +651,24 @@ main(int argc, char **argv) for (portid = 0; portid < nb_ports; portid++) { /* skip ports that are not enabled */ if ((l2fwd_enabled_port_mask & (1 << portid)) == 0) { - printf("Skipping disabled port %u\n", (unsigned) portid); + printf("Skipping disabled port %u\n", portid); nb_ports_available--; continue; } /* init port */ - printf("Initializing port %u... ", (unsigned) portid); + printf("Initializing port %u... ", portid); fflush(stdout); ret = rte_eth_dev_configure(portid, 1, 1, &port_conf); if (ret < 0) rte_exit(EXIT_FAILURE, "Cannot configure device: err=%d, port=%u\n", - ret, (unsigned) portid); + ret, portid); + + ret = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd, + &nb_txd); + if (ret < 0) + rte_exit(EXIT_FAILURE, + "Cannot adjust number of descriptors: err=%d, port=%u\n", + ret, portid); rte_eth_macaddr_get(portid,&l2fwd_ports_eth_addr[portid]); @@ -640,7 +680,7 @@ main(int argc, char **argv) l2fwd_pktmbuf_pool); if (ret < 0) rte_exit(EXIT_FAILURE, "rte_eth_rx_queue_setup:err=%d, port=%u\n", - ret, (unsigned) portid); + ret, portid); /* init one TX queue on each port */ fflush(stdout); @@ -649,7 +689,7 @@ main(int argc, char **argv) NULL); if (ret < 0) rte_exit(EXIT_FAILURE, "rte_eth_tx_queue_setup:err=%d, port=%u\n", - ret, (unsigned) portid); + ret, portid); /* Initialize TX buffers */ tx_buffer[portid] = rte_zmalloc_socket("tx_buffer", @@ -657,7 +697,7 @@ main(int argc, char **argv) rte_eth_dev_socket_id(portid)); if (tx_buffer[portid] == NULL) rte_exit(EXIT_FAILURE, "Cannot allocate buffer for tx on port %u\n", - (unsigned) portid); + portid); rte_eth_tx_buffer_init(tx_buffer[portid], MAX_PKT_BURST); @@ -665,21 +705,22 @@ main(int argc, char **argv) rte_eth_tx_buffer_count_callback, &port_statistics[portid].dropped); if (ret < 0) - rte_exit(EXIT_FAILURE, "Cannot set error callback for " - "tx buffer on port %u\n", (unsigned) portid); + rte_exit(EXIT_FAILURE, + "Cannot set error callback for tx buffer on port %u\n", + portid); /* Start device */ ret = rte_eth_dev_start(portid); if (ret < 0) rte_exit(EXIT_FAILURE, "rte_eth_dev_start:err=%d, port=%u\n", - ret, (unsigned) portid); + ret, portid); printf("done: \n"); rte_eth_promiscuous_enable(portid); printf("Port %u, MAC address: %02X:%02X:%02X:%02X:%02X:%02X\n\n", - (unsigned) portid, + portid, l2fwd_ports_eth_addr[portid].addr_bytes[0], l2fwd_ports_eth_addr[portid].addr_bytes[1], l2fwd_ports_eth_addr[portid].addr_bytes[2], diff --git a/dpdk/examples/l3fwd-acl/Makefile b/dpdk/examples/l3fwd-acl/Makefile index a3473a83..3cd299f1 100644 --- a/dpdk/examples/l3fwd-acl/Makefile +++ b/dpdk/examples/l3fwd-acl/Makefile @@ -33,7 +33,7 @@ ifeq ($(RTE_SDK),) $(error "Please define RTE_SDK environment variable") endif -# Default target, can be overriden by command line or environment +# Default target, can be overridden by command line or environment RTE_TARGET ?= x86_64-native-linuxapp-gcc include $(RTE_SDK)/mk/rte.vars.mk diff --git a/dpdk/examples/l3fwd-acl/main.c b/dpdk/examples/l3fwd-acl/main.c index 16f6110e..e50b1a1a 100644 --- a/dpdk/examples/l3fwd-acl/main.c +++ b/dpdk/examples/l3fwd-acl/main.c @@ -47,9 +47,7 @@ #include #include #include -#include #include -#include #include #include #include @@ -58,12 +56,10 @@ #include #include #include -#include #include #include #include #include -#include #include #include #include @@ -92,10 +88,10 @@ */ #define NB_MBUF RTE_MAX(\ - (nb_ports * nb_rx_queue*RTE_TEST_RX_DESC_DEFAULT + \ - nb_ports * nb_lcores * MAX_PKT_BURST + \ - nb_ports * n_tx_queue * RTE_TEST_TX_DESC_DEFAULT + \ - nb_lcores * MEMPOOL_CACHE_SIZE), \ + (nb_ports * nb_rx_queue * nb_rxd + \ + nb_ports * nb_lcores * MAX_PKT_BURST + \ + nb_ports * n_tx_queue * nb_txd + \ + nb_lcores * MEMPOOL_CACHE_SIZE), \ (unsigned)8192) #define MAX_PKT_BURST 32 @@ -123,7 +119,7 @@ static int promiscuous_on; /**< Ports set in promiscuous mode off by default. */ static int numa_on = 1; /**< NUMA is enabled by default. */ struct lcore_rx_queue { - uint8_t port_id; + uint16_t port_id; uint8_t queue_id; } __rte_cache_aligned; @@ -133,7 +129,7 @@ struct lcore_rx_queue { #define MAX_LCORE_PARAMS 1024 struct lcore_params { - uint8_t port_id; + uint16_t port_id; uint8_t queue_id; uint8_t lcore_id; } __rte_cache_aligned; @@ -164,7 +160,7 @@ static struct rte_eth_conf port_conf = { .hw_ip_checksum = 1, /**< IP checksum offload enabled */ .hw_vlan_filter = 0, /**< VLAN filtering disabled */ .jumbo_frame = 0, /**< Jumbo Frame Support disabled */ - .hw_strip_crc = 0, /**< CRC stripped by hardware */ + .hw_strip_crc = 1, /**< CRC stripped by hardware */ }, .rx_adv_conf = { .rss_conf = { @@ -186,7 +182,7 @@ static inline int is_valid_ipv4_pkt(struct ipv4_hdr *pkt, uint32_t link_len); #endif static inline void -send_single_packet(struct rte_mbuf *m, uint8_t port); +send_single_packet(struct rte_mbuf *m, uint16_t port); #define MAX_ACL_RULE_NUM 100000 #define DEFAULT_MAX_CATEGORIES 1 @@ -1028,6 +1024,7 @@ add_rules(const char *rule_path, char buff[LINE_MAX]; FILE *fh = fopen(rule_path, "rb"); unsigned int i = 0; + int val; if (fh == NULL) rte_exit(EXIT_FAILURE, "%s: Open %s failed\n", __func__, @@ -1044,7 +1041,11 @@ add_rules(const char *rule_path, rte_exit(EXIT_FAILURE, "Not find any route entries in %s!\n", rule_path); - fseek(fh, 0, SEEK_SET); + val = fseek(fh, 0, SEEK_SET); + if (val < 0) { + rte_exit(EXIT_FAILURE, "%s: File seek operation failed\n", + __func__); + } acl_rules = calloc(acl_num, rule_size); @@ -1299,7 +1300,7 @@ static struct lcore_conf lcore_conf[RTE_MAX_LCORE]; /* Enqueue a single packet, and send burst if queue is filled */ static inline void -send_single_packet(struct rte_mbuf *m, uint8_t port) +send_single_packet(struct rte_mbuf *m, uint16_t port) { uint32_t lcore_id; struct lcore_conf *qconf; @@ -1360,7 +1361,8 @@ main_loop(__attribute__((unused)) void *dummy) unsigned lcore_id; uint64_t prev_tsc, diff_tsc, cur_tsc; int i, nb_rx; - uint8_t portid, queueid; + uint16_t portid; + uint8_t queueid; struct lcore_conf *qconf; int socketid; const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) @@ -1383,7 +1385,7 @@ main_loop(__attribute__((unused)) void *dummy) portid = qconf->rx_queue_list[i].port_id; queueid = qconf->rx_queue_list[i].queue_id; RTE_LOG(INFO, L3FWD, - " -- lcoreid=%u portid=%hhu rxqueueid=%hhu\n", + " -- lcoreid=%u portid=%u rxqueueid=%hhu\n", lcore_id, portid, queueid); } @@ -1502,7 +1504,7 @@ check_port_config(const unsigned nb_ports) } static uint8_t -get_port_n_rx_queues(const uint8_t port) +get_port_n_rx_queues(const uint16_t port) { int queue = -1; uint16_t i; @@ -1777,7 +1779,7 @@ parse_args(int argc, char **argv) argv[optind-1] = prgname; ret = optind-1; - optind = 0; /* reset getopt lib */ + optind = 1; /* reset getopt lib */ return ret; } @@ -1831,11 +1833,12 @@ init_mem(unsigned nb_mbuf) /* Check the link status of all ports in up to 9s, and print them finally */ static void -check_all_ports_link_status(uint8_t port_num, uint32_t port_mask) +check_all_ports_link_status(uint16_t port_num, uint32_t port_mask) { #define CHECK_INTERVAL 100 /* 100ms */ #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ - uint8_t portid, count, all_ports_up, print_flag = 0; + uint16_t portid; + uint8_t count, all_ports_up, print_flag = 0; struct rte_eth_link link; printf("\nChecking link status"); @@ -1850,14 +1853,13 @@ check_all_ports_link_status(uint8_t port_num, uint32_t port_mask) /* print link status if flag set */ if (print_flag == 1) { if (link.link_status) - printf("Port %d Link Up - speed %u " - "Mbps - %s\n", (uint8_t)portid, - (unsigned)link.link_speed, + printf( + "Port%d Link Up. Speed %u Mbps %s\n", + portid, link.link_speed, (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? ("full-duplex") : ("half-duplex\n")); else - printf("Port %d Link Down\n", - (uint8_t)portid); + printf("Port %d Link Down\n", portid); continue; } /* clear all_ports_up flag if any link down */ @@ -1895,7 +1897,8 @@ main(int argc, char **argv) uint16_t queueid; unsigned lcore_id; uint32_t n_tx_queue, nb_lcores; - uint8_t portid, nb_rx_queue, queue, socketid; + uint16_t portid; + uint8_t nb_rx_queue, queue, socketid; /* init EAL */ ret = rte_eal_init(argc, argv); @@ -1952,6 +1955,13 @@ main(int argc, char **argv) "Cannot configure device: err=%d, port=%d\n", ret, portid); + ret = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd, + &nb_txd); + if (ret < 0) + rte_exit(EXIT_FAILURE, + "rte_eth_dev_adjust_nb_rx_tx_desc: err=%d, port=%d\n", + ret, portid); + rte_eth_macaddr_get(portid, &ports_eth_addr[portid]); print_ethaddr(" Address:", &ports_eth_addr[portid]); printf(", "); diff --git a/dpdk/examples/l3fwd-power/Makefile b/dpdk/examples/l3fwd-power/Makefile index 783772a7..9c4f4430 100644 --- a/dpdk/examples/l3fwd-power/Makefile +++ b/dpdk/examples/l3fwd-power/Makefile @@ -33,7 +33,7 @@ ifeq ($(RTE_SDK),) $(error "Please define RTE_SDK environment variable") endif -# Default target, can be overriden by command line or environment +# Default target, can be overridden by command line or environment RTE_TARGET ?= x86_64-native-linuxapp-gcc include $(RTE_SDK)/mk/rte.vars.mk diff --git a/dpdk/examples/l3fwd-power/main.c b/dpdk/examples/l3fwd-power/main.c index f746960e..50c3702f 100644 --- a/dpdk/examples/l3fwd-power/main.c +++ b/dpdk/examples/l3fwd-power/main.c @@ -50,9 +50,7 @@ #include #include #include -#include #include -#include #include #include #include @@ -61,12 +59,10 @@ #include #include #include -#include #include #include #include #include -#include #include #include #include @@ -75,7 +71,6 @@ #include #include #include -#include #include #define RTE_LOGTYPE_L3FWD_POWER RTE_LOGTYPE_USER1 @@ -84,8 +79,6 @@ #define MIN_ZERO_POLL_COUNT 10 -/* around 100ms at 2 Ghz */ -#define TIMER_RESOLUTION_CYCLES 200000000ULL /* 100 ms interval */ #define TIMER_NUMBER_PER_SECOND 10 /* 100000 us */ @@ -132,9 +125,9 @@ */ #define NB_MBUF RTE_MAX ( \ - (nb_ports*nb_rx_queue*RTE_TEST_RX_DESC_DEFAULT + \ + (nb_ports*nb_rx_queue*nb_rxd + \ nb_ports*nb_lcores*MAX_PKT_BURST + \ - nb_ports*n_tx_queue*RTE_TEST_TX_DESC_DEFAULT + \ + nb_ports*n_tx_queue*nb_txd + \ nb_lcores*MEMPOOL_CACHE_SIZE), \ (unsigned)8192) @@ -148,7 +141,7 @@ /* * Configurable number of RX/TX ring descriptors */ -#define RTE_TEST_RX_DESC_DEFAULT 128 +#define RTE_TEST_RX_DESC_DEFAULT 512 #define RTE_TEST_TX_DESC_DEFAULT 512 static uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; static uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; @@ -165,6 +158,8 @@ static uint32_t enabled_port_mask = 0; static int promiscuous_on = 0; /* NUMA is enabled by default. */ static int numa_on = 1; +static int parse_ptype; /**< Parse packet type using rx callback, and */ + /**< disabled by default */ enum freq_scale_hint_t { @@ -175,7 +170,7 @@ enum freq_scale_hint_t }; struct lcore_rx_queue { - uint8_t port_id; + uint16_t port_id; uint8_t queue_id; enum freq_scale_hint_t freq_up_hint; uint32_t zero_rx_packet_count; @@ -191,7 +186,7 @@ struct lcore_rx_queue { #define MAX_LCORE_PARAMS 1024 struct lcore_params { - uint8_t port_id; + uint16_t port_id; uint8_t queue_id; uint8_t lcore_id; } __rte_cache_aligned; @@ -222,7 +217,7 @@ static struct rte_eth_conf port_conf = { .hw_ip_checksum = 1, /**< IP checksum offload enabled */ .hw_vlan_filter = 0, /**< VLAN filtering disabled */ .jumbo_frame = 0, /**< Jumbo Frame Support disabled */ - .hw_strip_crc = 0, /**< CRC stripped by hardware */ + .hw_strip_crc = 1, /**< CRC stripped by hardware */ }, .rx_adv_conf = { .rss_conf = { @@ -244,7 +239,7 @@ static struct rte_mempool * pktmbuf_pool[NB_SOCKETS]; #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) -#ifdef RTE_MACHINE_CPUFLAG_SSE4_2 +#ifdef RTE_ARCH_X86 #include #define DEFAULT_HASH_FUNC rte_hash_crc #else @@ -309,8 +304,8 @@ static lookup_struct_t *ipv6_l3fwd_lookup_struct[NB_SOCKETS]; #define IPV6_L3FWD_NUM_ROUTES \ (sizeof(ipv6_l3fwd_route_array) / sizeof(ipv6_l3fwd_route_array[0])) -static uint8_t ipv4_l3fwd_out_if[L3FWD_HASH_ENTRIES] __rte_cache_aligned; -static uint8_t ipv6_l3fwd_out_if[L3FWD_HASH_ENTRIES] __rte_cache_aligned; +static uint16_t ipv4_l3fwd_out_if[L3FWD_HASH_ENTRIES] __rte_cache_aligned; +static uint16_t ipv6_l3fwd_out_if[L3FWD_HASH_ENTRIES] __rte_cache_aligned; #endif #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM) @@ -371,13 +366,14 @@ static struct rte_timer power_timers[RTE_MAX_LCORE]; static inline uint32_t power_idle_heuristic(uint32_t zero_rx_packet_count); static inline enum freq_scale_hint_t power_freq_scaleup_heuristic( \ - unsigned lcore_id, uint8_t port_id, uint16_t queue_id); + unsigned int lcore_id, uint16_t port_id, uint16_t queue_id); /* exit signal handler */ static void signal_exit_now(int sigtype) { unsigned lcore_id; + unsigned int portid, nb_ports; int ret; if (sigtype == SIGINT) { @@ -392,6 +388,15 @@ signal_exit_now(int sigtype) "library de-initialization failed on " "core%u\n", lcore_id); } + + nb_ports = rte_eth_dev_count(); + for (portid = 0; portid < nb_ports; portid++) { + if ((enabled_port_mask & (1 << portid)) == 0) + continue; + + rte_eth_dev_stop(portid); + rte_eth_dev_close(portid); + } } rte_exit(EXIT_SUCCESS, "User forced exit\n"); @@ -442,7 +447,7 @@ power_timer_cb(__attribute__((unused)) struct rte_timer *tim, /* Enqueue a single packet, and send burst if queue is filled */ static inline int -send_single_packet(struct rte_mbuf *m, uint8_t port) +send_single_packet(struct rte_mbuf *m, uint16_t port) { uint32_t lcore_id; struct lcore_conf *qconf; @@ -514,8 +519,8 @@ print_ipv6_key(struct ipv6_5tuple key) key.port_dst, key.port_src, key.proto); } -static inline uint8_t -get_ipv4_dst_port(struct ipv4_hdr *ipv4_hdr, uint8_t portid, +static inline uint16_t +get_ipv4_dst_port(struct ipv4_hdr *ipv4_hdr, uint16_t portid, lookup_struct_t * ipv4_l3fwd_lookup_struct) { struct ipv4_5tuple key; @@ -550,11 +555,11 @@ get_ipv4_dst_port(struct ipv4_hdr *ipv4_hdr, uint8_t portid, /* Find destination port */ ret = rte_hash_lookup(ipv4_l3fwd_lookup_struct, (const void *)&key); - return (uint8_t)((ret < 0)? portid : ipv4_l3fwd_out_if[ret]); + return ((ret < 0) ? portid : ipv4_l3fwd_out_if[ret]); } -static inline uint8_t -get_ipv6_dst_port(struct ipv6_hdr *ipv6_hdr, uint8_t portid, +static inline uint16_t +get_ipv6_dst_port(struct ipv6_hdr *ipv6_hdr, uint16_t portid, lookup_struct_t *ipv6_l3fwd_lookup_struct) { struct ipv6_5tuple key; @@ -590,31 +595,73 @@ get_ipv6_dst_port(struct ipv6_hdr *ipv6_hdr, uint8_t portid, /* Find destination port */ ret = rte_hash_lookup(ipv6_l3fwd_lookup_struct, (const void *)&key); - return (uint8_t)((ret < 0)? portid : ipv6_l3fwd_out_if[ret]); + return ((ret < 0) ? portid : ipv6_l3fwd_out_if[ret]); } #endif #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM) -static inline uint8_t -get_ipv4_dst_port(struct ipv4_hdr *ipv4_hdr, uint8_t portid, +static inline uint16_t +get_ipv4_dst_port(struct ipv4_hdr *ipv4_hdr, uint16_t portid, lookup_struct_t *ipv4_l3fwd_lookup_struct) { uint32_t next_hop; - return (uint8_t) ((rte_lpm_lookup(ipv4_l3fwd_lookup_struct, + return ((rte_lpm_lookup(ipv4_l3fwd_lookup_struct, rte_be_to_cpu_32(ipv4_hdr->dst_addr), &next_hop) == 0)? next_hop : portid); } #endif static inline void -l3fwd_simple_forward(struct rte_mbuf *m, uint8_t portid, +parse_ptype_one(struct rte_mbuf *m) +{ + struct ether_hdr *eth_hdr; + uint32_t packet_type = RTE_PTYPE_UNKNOWN; + uint16_t ether_type; + + eth_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *); + ether_type = eth_hdr->ether_type; + if (ether_type == rte_cpu_to_be_16(ETHER_TYPE_IPv4)) + packet_type |= RTE_PTYPE_L3_IPV4_EXT_UNKNOWN; + else if (ether_type == rte_cpu_to_be_16(ETHER_TYPE_IPv6)) + packet_type |= RTE_PTYPE_L3_IPV6_EXT_UNKNOWN; + + m->packet_type = packet_type; +} + +static uint16_t +cb_parse_ptype(uint16_t port __rte_unused, uint16_t queue __rte_unused, + struct rte_mbuf *pkts[], uint16_t nb_pkts, + uint16_t max_pkts __rte_unused, + void *user_param __rte_unused) +{ + unsigned int i; + + for (i = 0; i < nb_pkts; ++i) + parse_ptype_one(pkts[i]); + + return nb_pkts; +} + +static int +add_cb_parse_ptype(uint16_t portid, uint16_t queueid) +{ + printf("Port %d: softly parse packet type info\n", portid); + if (rte_eth_add_rx_callback(portid, queueid, cb_parse_ptype, NULL)) + return 0; + + printf("Failed to add rx callback: port=%d\n", portid); + return -1; +} + +static inline void +l3fwd_simple_forward(struct rte_mbuf *m, uint16_t portid, struct lcore_conf *qconf) { struct ether_hdr *eth_hdr; struct ipv4_hdr *ipv4_hdr; void *d_addr_bytes; - uint8_t dst_port; + uint16_t dst_port; eth_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *); @@ -701,13 +748,11 @@ power_idle_heuristic(uint32_t zero_rx_packet_count) */ else return SUSPEND_THRESHOLD; - - return 0; } static inline enum freq_scale_hint_t power_freq_scaleup_heuristic(unsigned lcore_id, - uint8_t port_id, + uint16_t port_id, uint16_t queue_id) { /** @@ -754,7 +799,8 @@ sleep_until_rx_interrupt(int num) { struct rte_epoll_event event[num]; int n, i; - uint8_t port_id, queue_id; + uint16_t port_id; + uint8_t queue_id; void *data; RTE_LOG(INFO, L3FWD_POWER, @@ -781,7 +827,8 @@ static void turn_on_intr(struct lcore_conf *qconf) { int i; struct lcore_rx_queue *rx_queue; - uint8_t port_id, queue_id; + uint8_t queue_id; + uint16_t port_id; for (i = 0; i < qconf->n_rx_queue; ++i) { rx_queue = &(qconf->rx_queue_list[i]); @@ -797,7 +844,8 @@ static void turn_on_intr(struct lcore_conf *qconf) static int event_register(struct lcore_conf *qconf) { struct lcore_rx_queue *rx_queue; - uint8_t portid, queueid; + uint8_t queueid; + uint16_t portid; uint32_t data; int ret; int i; @@ -825,10 +873,11 @@ main_loop(__attribute__((unused)) void *dummy) { struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; unsigned lcore_id; - uint64_t prev_tsc, diff_tsc, cur_tsc; + uint64_t prev_tsc, diff_tsc, cur_tsc, tim_res_tsc, hz; uint64_t prev_tsc_power = 0, cur_tsc_power, diff_tsc_power; int i, j, nb_rx; - uint8_t portid, queueid; + uint8_t queueid; + uint16_t portid; struct lcore_conf *qconf; struct lcore_rx_queue *rx_queue; enum freq_scale_hint_t lcore_scaleup_hint; @@ -839,6 +888,8 @@ main_loop(__attribute__((unused)) void *dummy) const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / US_PER_S * BURST_TX_DRAIN_US; prev_tsc = 0; + hz = rte_get_timer_hz(); + tim_res_tsc = hz/TIMER_NUMBER_PER_SECOND; lcore_id = rte_lcore_id(); qconf = &lcore_conf[lcore_id]; @@ -853,7 +904,7 @@ main_loop(__attribute__((unused)) void *dummy) for (i = 0; i < qconf->n_rx_queue; i++) { portid = qconf->rx_queue_list[i].port_id; queueid = qconf->rx_queue_list[i].queue_id; - RTE_LOG(INFO, L3FWD_POWER, " -- lcoreid=%u portid=%hhu " + RTE_LOG(INFO, L3FWD_POWER, " -- lcoreid=%u portid=%u " "rxqueueid=%hhu\n", lcore_id, portid, queueid); } @@ -884,7 +935,7 @@ main_loop(__attribute__((unused)) void *dummy) } diff_tsc_power = cur_tsc_power - prev_tsc_power; - if (diff_tsc_power > TIMER_RESOLUTION_CYCLES) { + if (diff_tsc_power > tim_res_tsc) { rte_timer_manage(); prev_tsc_power = cur_tsc_power; } @@ -1000,9 +1051,11 @@ start_rx: turn_on_intr(qconf); sleep_until_rx_interrupt( qconf->n_rx_queue); + /** + * start receiving packets immediately + */ + goto start_rx; } - /* start receiving packets immediately */ - goto start_rx; } stats[lcore_id].sleep_time += lcore_idle_hint; } @@ -1060,7 +1113,7 @@ check_port_config(const unsigned nb_ports) } static uint8_t -get_port_n_rx_queues(const uint8_t port) +get_port_n_rx_queues(const uint16_t port) { int queue = -1; uint16_t i; @@ -1109,7 +1162,8 @@ print_usage(const char *prgname) " --config (port,queue,lcore): rx queues configuration\n" " --no-numa: optional, disable numa awareness\n" " --enable-jumbo: enable jumbo frame" - " which max packet len is PKTLEN in decimal (64-9600)\n", + " which max packet len is PKTLEN in decimal (64-9600)\n" + " --parse-ptype: parse packet type by software\n", prgname); } @@ -1203,6 +1257,8 @@ parse_config(const char *q_arg) return 0; } +#define CMD_LINE_OPT_PARSE_PTYPE "parse-ptype" + /* Parse the argument given in the command line of the application */ static int parse_args(int argc, char **argv) @@ -1215,6 +1271,7 @@ parse_args(int argc, char **argv) {"config", 1, 0, 0}, {"no-numa", 0, 0, 0}, {"enable-jumbo", 0, 0, 0}, + {CMD_LINE_OPT_PARSE_PTYPE, 0, 0, 0}, {NULL, 0, 0, 0} }; @@ -1285,6 +1342,13 @@ parse_args(int argc, char **argv) (unsigned int)port_conf.rxmode.max_rx_pkt_len); } + if (!strncmp(lgopts[option_index].name, + CMD_LINE_OPT_PARSE_PTYPE, + sizeof(CMD_LINE_OPT_PARSE_PTYPE))) { + printf("soft parse-ptype is enabled\n"); + parse_ptype = 1; + } + break; default: @@ -1297,7 +1361,7 @@ parse_args(int argc, char **argv) argv[optind-1] = prgname; ret = optind-1; - optind = 0; /* reset getopt lib */ + optind = 1; /* reset getopt lib */ return ret; } @@ -1479,11 +1543,12 @@ init_mem(unsigned nb_mbuf) /* Check the link status of all ports in up to 9s, and print them finally */ static void -check_all_ports_link_status(uint8_t port_num, uint32_t port_mask) +check_all_ports_link_status(uint16_t port_num, uint32_t port_mask) { #define CHECK_INTERVAL 100 /* 100ms */ #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ - uint8_t portid, count, all_ports_up, print_flag = 0; + uint8_t count, all_ports_up, print_flag = 0; + uint16_t portid; struct rte_eth_link link; printf("\nChecking link status"); @@ -1532,6 +1597,50 @@ check_all_ports_link_status(uint8_t port_num, uint32_t port_mask) } } +static int check_ptype(uint16_t portid) +{ + int i, ret; + int ptype_l3_ipv4 = 0; +#if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) + int ptype_l3_ipv6 = 0; +#endif + uint32_t ptype_mask = RTE_PTYPE_L3_MASK; + + ret = rte_eth_dev_get_supported_ptypes(portid, ptype_mask, NULL, 0); + if (ret <= 0) + return 0; + + uint32_t ptypes[ret]; + + ret = rte_eth_dev_get_supported_ptypes(portid, ptype_mask, ptypes, ret); + for (i = 0; i < ret; ++i) { + if (ptypes[i] & RTE_PTYPE_L3_IPV4) + ptype_l3_ipv4 = 1; +#if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) + if (ptypes[i] & RTE_PTYPE_L3_IPV6) + ptype_l3_ipv6 = 1; +#endif + } + + if (ptype_l3_ipv4 == 0) + printf("port %d cannot parse RTE_PTYPE_L3_IPV4\n", portid); + +#if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) + if (ptype_l3_ipv6 == 0) + printf("port %d cannot parse RTE_PTYPE_L3_IPV6\n", portid); +#endif + +#if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM) + if (ptype_l3_ipv4) +#else /* APP_LOOKUP_EXACT_MATCH */ + if (ptype_l3_ipv4 && ptype_l3_ipv6) +#endif + return 1; + + return 0; + +} + int main(int argc, char **argv) { @@ -1539,13 +1648,15 @@ main(int argc, char **argv) struct rte_eth_dev_info dev_info; struct rte_eth_txconf *txconf; int ret; - unsigned nb_ports; + uint16_t nb_ports; uint16_t queueid; unsigned lcore_id; uint64_t hz; uint32_t n_tx_queue, nb_lcores; uint32_t dev_rxq_num, dev_txq_num; - uint8_t portid, nb_rx_queue, queue, socketid; + uint8_t nb_rx_queue, queue, socketid; + uint16_t portid; + uint16_t org_rxq_intr = port_conf.intr_conf.rxq; /* catch SIGINT and restore cpufreq governor to ondemand */ signal(SIGINT, signal_exit_now); @@ -1606,12 +1717,24 @@ main(int argc, char **argv) n_tx_queue = dev_txq_num; printf("Creating queues: nb_rxq=%d nb_txq=%u... ", nb_rx_queue, (unsigned)n_tx_queue ); + /* If number of Rx queue is 0, no need to enable Rx interrupt */ + if (nb_rx_queue == 0) + port_conf.intr_conf.rxq = 0; ret = rte_eth_dev_configure(portid, nb_rx_queue, (uint16_t)n_tx_queue, &port_conf); + /* Revert to original value */ + port_conf.intr_conf.rxq = org_rxq_intr; if (ret < 0) rte_exit(EXIT_FAILURE, "Cannot configure device: " "err=%d, port=%d\n", ret, portid); + ret = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd, + &nb_txd); + if (ret < 0) + rte_exit(EXIT_FAILURE, + "Cannot adjust number of descriptors: err=%d, port=%d\n", + ret, portid); + rte_eth_macaddr_get(portid, &ports_eth_addr[portid]); print_ethaddr(" Address:", &ports_eth_addr[portid]); printf(", "); @@ -1632,7 +1755,7 @@ main(int argc, char **argv) rte_eth_dev_socket_id(portid)); if (qconf->tx_buffer[portid] == NULL) rte_exit(EXIT_FAILURE, "Can't allocate tx buffer for port %u\n", - (unsigned) portid); + portid); rte_eth_tx_buffer_init(qconf->tx_buffer[portid], MAX_PKT_BURST); } @@ -1717,6 +1840,14 @@ main(int argc, char **argv) rte_exit(EXIT_FAILURE, "rte_eth_rx_queue_setup: err=%d, " "port=%d\n", ret, portid); + + if (parse_ptype) { + if (add_cb_parse_ptype(portid, queueid) < 0) + rte_exit(EXIT_FAILURE, + "Fail to add ptype cb\n"); + } else if (!check_ptype(portid)) + rte_exit(EXIT_FAILURE, + "PMD can not provide needed ptypes\n"); } } @@ -1744,7 +1875,7 @@ main(int argc, char **argv) rte_spinlock_init(&(locks[portid])); } - check_all_ports_link_status((uint8_t)nb_ports, enabled_port_mask); + check_all_ports_link_status(nb_ports, enabled_port_mask); /* launch per-lcore init on every lcore */ rte_eal_mp_remote_launch(main_loop, NULL, CALL_MASTER); diff --git a/dpdk/examples/l3fwd-vf/Makefile b/dpdk/examples/l3fwd-vf/Makefile index d97611cf..989faf03 100644 --- a/dpdk/examples/l3fwd-vf/Makefile +++ b/dpdk/examples/l3fwd-vf/Makefile @@ -33,7 +33,7 @@ ifeq ($(RTE_SDK),) $(error "Please define RTE_SDK environment variable") endif -# Default target, can be overriden by command line or environment +# Default target, can be overridden by command line or environment RTE_TARGET ?= x86_64-native-linuxapp-gcc include $(RTE_SDK)/mk/rte.vars.mk diff --git a/dpdk/examples/l3fwd-vf/main.c b/dpdk/examples/l3fwd-vf/main.c index ca01b112..6ef89fc8 100644 --- a/dpdk/examples/l3fwd-vf/main.c +++ b/dpdk/examples/l3fwd-vf/main.c @@ -48,9 +48,7 @@ #include #include #include -#include #include -#include #include #include #include @@ -60,12 +58,10 @@ #include #include #include -#include #include #include #include #include -#include #include #include #include @@ -100,11 +96,11 @@ * RTE_MAX is used to ensure that NB_MBUF never goes below a minimum value of 8192 */ -#define NB_MBUF RTE_MAX ( \ - (nb_ports*nb_rx_queue*RTE_TEST_RX_DESC_DEFAULT + \ - nb_ports*nb_lcores*MAX_PKT_BURST + \ - nb_ports*n_tx_queue*RTE_TEST_TX_DESC_DEFAULT + \ - nb_lcores*MEMPOOL_CACHE_SIZE), \ +#define NB_MBUF RTE_MAX ( \ + (nb_ports*nb_rx_queue*nb_rxd + \ + nb_ports*nb_lcores*MAX_PKT_BURST + \ + nb_ports*n_tx_queue*nb_txd + \ + nb_lcores*MEMPOOL_CACHE_SIZE), \ (unsigned)8192) /* @@ -157,7 +153,7 @@ struct mbuf_table { }; struct lcore_rx_queue { - uint8_t port_id; + uint16_t port_id; uint8_t queue_id; } __rte_cache_aligned; @@ -167,7 +163,7 @@ struct lcore_rx_queue { #define MAX_LCORE_PARAMS 1024 struct lcore_params { - uint8_t port_id; + uint16_t port_id; uint8_t queue_id; uint8_t lcore_id; } __rte_cache_aligned; @@ -198,7 +194,7 @@ static struct rte_eth_conf port_conf = { .hw_ip_checksum = 1, /**< IP checksum offload enabled */ .hw_vlan_filter = 0, /**< VLAN filtering disabled */ .jumbo_frame = 0, /**< Jumbo Frame Support disabled */ - .hw_strip_crc = 0, /**< CRC stripped by hardware */ + .hw_strip_crc = 1, /**< CRC stripped by hardware */ }, .rx_adv_conf = { .rss_conf = { @@ -216,7 +212,7 @@ static struct rte_mempool * pktmbuf_pool[NB_SOCKETS]; #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) -#ifdef RTE_MACHINE_CPUFLAG_SSE4_2 +#ifdef RTE_ARCH_X86 #include #define DEFAULT_HASH_FUNC rte_hash_crc #else @@ -302,7 +298,7 @@ static struct lcore_conf lcore_conf[RTE_MAX_LCORE]; static rte_spinlock_t spinlock_conf[RTE_MAX_ETHPORTS] = {RTE_SPINLOCK_INITIALIZER}; /* Send burst of packets on an output interface */ static inline int -send_burst(struct lcore_conf *qconf, uint16_t n, uint8_t port) +send_burst(struct lcore_conf *qconf, uint16_t n, uint16_t port) { struct rte_mbuf **m_table; int ret; @@ -326,7 +322,7 @@ send_burst(struct lcore_conf *qconf, uint16_t n, uint8_t port) /* Enqueue a single packet, and send burst if queue is filled */ static inline int -send_single_packet(struct rte_mbuf *m, uint8_t port) +send_single_packet(struct rte_mbuf *m, uint16_t port) { uint32_t lcore_id; uint16_t len; @@ -398,8 +394,9 @@ print_key(struct ipv4_5tuple key) (unsigned)key.ip_dst, (unsigned)key.ip_src, key.port_dst, key.port_src, key.proto); } -static inline uint8_t -get_dst_port(struct ipv4_hdr *ipv4_hdr, uint8_t portid, lookup_struct_t * l3fwd_lookup_struct) +static inline uint16_t +get_dst_port(struct ipv4_hdr *ipv4_hdr, uint16_t portid, + lookup_struct_t *l3fwd_lookup_struct) { struct ipv4_5tuple key; struct tcp_hdr *tcp; @@ -432,29 +429,31 @@ get_dst_port(struct ipv4_hdr *ipv4_hdr, uint8_t portid, lookup_struct_t * l3fwd /* Find destination port */ ret = rte_hash_lookup(l3fwd_lookup_struct, (const void *)&key); - return (uint8_t)((ret < 0)? portid : l3fwd_out_if[ret]); + return ((ret < 0) ? portid : l3fwd_out_if[ret]); } #endif #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM) -static inline uint8_t -get_dst_port(struct ipv4_hdr *ipv4_hdr, uint8_t portid, lookup_struct_t * l3fwd_lookup_struct) +static inline uint32_t +get_dst_port(struct ipv4_hdr *ipv4_hdr, uint16_t portid, + lookup_struct_t *l3fwd_lookup_struct) { uint32_t next_hop; - return (uint8_t) ((rte_lpm_lookup(l3fwd_lookup_struct, - rte_be_to_cpu_32(ipv4_hdr->dst_addr), &next_hop) == 0)? - next_hop : portid); + return ((rte_lpm_lookup(l3fwd_lookup_struct, + rte_be_to_cpu_32(ipv4_hdr->dst_addr), &next_hop) == 0) ? + next_hop : portid); } #endif static inline void -l3fwd_simple_forward(struct rte_mbuf *m, uint8_t portid, lookup_struct_t * l3fwd_lookup_struct) +l3fwd_simple_forward(struct rte_mbuf *m, uint16_t portid, + lookup_struct_t *l3fwd_lookup_struct) { struct ether_hdr *eth_hdr; struct ipv4_hdr *ipv4_hdr; void *tmp; - uint8_t dst_port; + uint16_t dst_port; eth_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *); @@ -498,7 +497,8 @@ main_loop(__attribute__((unused)) void *dummy) unsigned lcore_id; uint64_t prev_tsc, diff_tsc, cur_tsc; int i, j, nb_rx; - uint8_t portid, queueid; + uint8_t queueid; + uint16_t portid; struct lcore_conf *qconf; const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / US_PER_S * BURST_TX_DRAIN_US; @@ -518,8 +518,8 @@ main_loop(__attribute__((unused)) void *dummy) portid = qconf->rx_queue_list[i].port_id; queueid = qconf->rx_queue_list[i].queue_id; - RTE_LOG(INFO, L3FWD, " -- lcoreid=%u portid=%hhu rxqueueid=%hhu\n", lcore_id, - portid, queueid); + RTE_LOG(INFO, L3FWD, " --lcoreid=%u portid=%u rxqueueid=%hhu\n", + lcore_id, portid, queueid); } while (1) { @@ -626,7 +626,7 @@ check_port_config(const unsigned nb_ports) } static uint8_t -get_port_n_rx_queues(const uint8_t port) +get_port_n_rx_queues(const uint16_t port) { int queue = -1; uint16_t i; @@ -678,8 +678,8 @@ print_usage(const char *prgname) static void signal_handler(int signum) { - uint8_t portid; - uint8_t nb_ports = rte_eth_dev_count(); + uint16_t portid; + uint16_t nb_ports = rte_eth_dev_count(); /* When we receive a SIGINT signal */ if (signum == SIGINT) { @@ -751,7 +751,7 @@ parse_config(const char *q_arg) nb_lcore_params); return -1; } - lcore_params_array[nb_lcore_params].port_id = (uint8_t)int_fld[FLD_PORT]; + lcore_params_array[nb_lcore_params].port_id = int_fld[FLD_PORT]; lcore_params_array[nb_lcore_params].queue_id = (uint8_t)int_fld[FLD_QUEUE]; lcore_params_array[nb_lcore_params].lcore_id = (uint8_t)int_fld[FLD_LCORE]; ++nb_lcore_params; @@ -817,7 +817,7 @@ parse_args(int argc, char **argv) argv[optind-1] = prgname; ret = optind-1; - optind = 0; /* reset getopt lib */ + optind = 1; /* reset getopt lib */ return ret; } @@ -955,11 +955,11 @@ main(int argc, char **argv) struct rte_eth_txconf *txconf; int ret; unsigned nb_ports; - uint16_t queueid; + uint16_t queueid, portid; unsigned lcore_id; uint32_t nb_lcores; uint16_t n_tx_queue; - uint8_t portid, nb_rx_queue, queue, socketid; + uint8_t nb_rx_queue, queue, socketid; signal(SIGINT, signal_handler); /* init EAL */ @@ -1011,6 +1011,13 @@ main(int argc, char **argv) rte_exit(EXIT_FAILURE, "Cannot configure device: err=%d, port=%d\n", ret, portid); + ret = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd, + &nb_txd); + if (ret < 0) + rte_exit(EXIT_FAILURE, + "Cannot adjust number of descriptors: err=%d, port=%d\n", + ret, portid); + rte_eth_macaddr_get(portid, &ports_eth_addr[portid]); print_ethaddr(" Address:", &ports_eth_addr[portid]); printf(", "); diff --git a/dpdk/examples/l3fwd/Makefile b/dpdk/examples/l3fwd/Makefile index 5ce0ce05..d99a43ad 100644 --- a/dpdk/examples/l3fwd/Makefile +++ b/dpdk/examples/l3fwd/Makefile @@ -33,7 +33,7 @@ ifeq ($(RTE_SDK),) $(error "Please define RTE_SDK environment variable") endif -# Default target, can be overriden by command line or environment +# Default target, can be overridden by command line or environment RTE_TARGET ?= x86_64-native-linuxapp-gcc include $(RTE_SDK)/mk/rte.vars.mk diff --git a/dpdk/examples/l3fwd/l3fwd.h b/dpdk/examples/l3fwd/l3fwd.h index d8798b7d..4bb15943 100644 --- a/dpdk/examples/l3fwd/l3fwd.h +++ b/dpdk/examples/l3fwd/l3fwd.h @@ -68,7 +68,7 @@ #define MASK_ETH 0x3f /* Hash parameters. */ -#ifdef RTE_ARCH_X86_64 +#ifdef RTE_ARCH_64 /* default to 4 million hash entries (approx) */ #define L3FWD_HASH_ENTRIES (1024*1024*4) #else @@ -83,7 +83,7 @@ struct mbuf_table { }; struct lcore_rx_queue { - uint8_t port_id; + uint16_t port_id; uint8_t queue_id; } __rte_cache_aligned; @@ -117,7 +117,7 @@ extern struct lcore_conf lcore_conf[RTE_MAX_LCORE]; /* Send burst of packets on an output interface */ static inline int -send_burst(struct lcore_conf *qconf, uint16_t n, uint8_t port) +send_burst(struct lcore_conf *qconf, uint16_t n, uint16_t port) { struct rte_mbuf **m_table; int ret; @@ -139,7 +139,7 @@ send_burst(struct lcore_conf *qconf, uint16_t n, uint8_t port) /* Enqueue a single packet, and send burst if queue is filled */ static inline int send_single_packet(struct lcore_conf *qconf, - struct rte_mbuf *m, uint8_t port) + struct rte_mbuf *m, uint16_t port) { uint16_t len; @@ -212,11 +212,11 @@ int lpm_check_ptype(int portid); uint16_t -em_cb_parse_ptype(uint8_t port, uint16_t queue, struct rte_mbuf *pkts[], +em_cb_parse_ptype(uint16_t port, uint16_t queue, struct rte_mbuf *pkts[], uint16_t nb_pkts, uint16_t max_pkts, void *user_param); uint16_t -lpm_cb_parse_ptype(uint8_t port, uint16_t queue, struct rte_mbuf *pkts[], +lpm_cb_parse_ptype(uint16_t port, uint16_t queue, struct rte_mbuf *pkts[], uint16_t nb_pkts, uint16_t max_pkts, void *user_param); int diff --git a/dpdk/examples/l3fwd/l3fwd_altivec.h b/dpdk/examples/l3fwd/l3fwd_altivec.h new file mode 100644 index 00000000..a1d25eaa --- /dev/null +++ b/dpdk/examples/l3fwd/l3fwd_altivec.h @@ -0,0 +1,284 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2017 IBM Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + + +#ifndef _L3FWD_ALTIVEC_H_ +#define _L3FWD_ALTIVEC_H_ + +#include "l3fwd.h" +#include "l3fwd_common.h" + +/* + * Update source and destination MAC addresses in the ethernet header. + * Perform RFC1812 checks and updates for IPV4 packets. + */ +static inline void +processx4_step3(struct rte_mbuf *pkt[FWDSTEP], uint16_t dst_port[FWDSTEP]) +{ + vector unsigned int te[FWDSTEP]; + vector unsigned int ve[FWDSTEP]; + vector unsigned int *p[FWDSTEP]; + + p[0] = rte_pktmbuf_mtod(pkt[0], vector unsigned int *); + p[1] = rte_pktmbuf_mtod(pkt[1], vector unsigned int *); + p[2] = rte_pktmbuf_mtod(pkt[2], vector unsigned int *); + p[3] = rte_pktmbuf_mtod(pkt[3], vector unsigned int *); + + ve[0] = (vector unsigned int)val_eth[dst_port[0]]; + te[0] = *p[0]; + + ve[1] = (vector unsigned int)val_eth[dst_port[1]]; + te[1] = *p[1]; + + ve[2] = (vector unsigned int)val_eth[dst_port[2]]; + te[2] = *p[2]; + + ve[3] = (vector unsigned int)val_eth[dst_port[3]]; + te[3] = *p[3]; + + /* Update first 12 bytes, keep rest bytes intact. */ + te[0] = (vector unsigned int)vec_sel( + (vector unsigned short)ve[0], + (vector unsigned short)te[0], + (vector unsigned short) {0, 0, 0, 0, + 0, 0, 0xffff, 0xffff}); + + te[1] = (vector unsigned int)vec_sel( + (vector unsigned short)ve[1], + (vector unsigned short)te[1], + (vector unsigned short) {0, 0, 0, 0, + 0, 0, 0xffff, 0xffff}); + + te[2] = (vector unsigned int)vec_sel( + (vector unsigned short)ve[2], + (vector unsigned short)te[2], + (vector unsigned short) {0, 0, 0, 0, 0, + 0, 0xffff, 0xffff}); + + te[3] = (vector unsigned int)vec_sel( + (vector unsigned short)ve[3], + (vector unsigned short)te[3], + (vector unsigned short) {0, 0, 0, 0, + 0, 0, 0xffff, 0xffff}); + + *p[0] = te[0]; + *p[1] = te[1]; + *p[2] = te[2]; + *p[3] = te[3]; + + rfc1812_process((struct ipv4_hdr *)((struct ether_hdr *)p[0] + 1), + &dst_port[0], pkt[0]->packet_type); + rfc1812_process((struct ipv4_hdr *)((struct ether_hdr *)p[1] + 1), + &dst_port[1], pkt[1]->packet_type); + rfc1812_process((struct ipv4_hdr *)((struct ether_hdr *)p[2] + 1), + &dst_port[2], pkt[2]->packet_type); + rfc1812_process((struct ipv4_hdr *)((struct ether_hdr *)p[3] + 1), + &dst_port[3], pkt[3]->packet_type); +} + +/* + * Group consecutive packets with the same destination port in bursts of 4. + * Suppose we have array of destination ports: + * dst_port[] = {a, b, c, d,, e, ... } + * dp1 should contain: , dp2: . + * We doing 4 comparisons at once and the result is 4 bit mask. + * This mask is used as an index into prebuild array of pnum values. + */ +static inline uint16_t * +port_groupx4(uint16_t pn[FWDSTEP + 1], uint16_t *lp, vector unsigned short dp1, + vector unsigned short dp2) +{ + union { + uint16_t u16[FWDSTEP + 1]; + uint64_t u64; + } *pnum = (void *)pn; + + int32_t v; + + v = vec_any_eq(dp1, dp2); + + + /* update last port counter. */ + lp[0] += gptbl[v].lpv; + + /* if dest port value has changed. */ + if (v != GRPMSK) { + pnum->u64 = gptbl[v].pnum; + pnum->u16[FWDSTEP] = 1; + lp = pnum->u16 + gptbl[v].idx; + } + + return lp; +} + +/** + * Process one packet: + * Update source and destination MAC addresses in the ethernet header. + * Perform RFC1812 checks and updates for IPV4 packets. + */ +static inline void +process_packet(struct rte_mbuf *pkt, uint16_t *dst_port) +{ + struct ether_hdr *eth_hdr; + vector unsigned int te, ve; + + eth_hdr = rte_pktmbuf_mtod(pkt, struct ether_hdr *); + + te = *(vector unsigned int *)eth_hdr; + ve = (vector unsigned int)val_eth[dst_port[0]]; + + rfc1812_process((struct ipv4_hdr *)(eth_hdr + 1), dst_port, + pkt->packet_type); + + /* dynamically vec_sel te and ve for MASK_ETH (0x3f) */ + te = (vector unsigned int)vec_sel( + (vector unsigned short)ve, + (vector unsigned short)te, + (vector unsigned short){0, 0, 0, 0, + 0, 0, 0xffff, 0xffff}); + + *(vector unsigned int *)eth_hdr = te; +} + +/** + * Send packets burst from pkts_burst to the ports in dst_port array + */ +static __rte_always_inline void +send_packets_multi(struct lcore_conf *qconf, struct rte_mbuf **pkts_burst, + uint16_t dst_port[MAX_PKT_BURST], int nb_rx) +{ + int32_t k; + int j = 0; + uint16_t dlp; + uint16_t *lp; + uint16_t pnum[MAX_PKT_BURST + 1]; + + /* + * Finish packet processing and group consecutive + * packets with the same destination port. + */ + k = RTE_ALIGN_FLOOR(nb_rx, FWDSTEP); + if (k != 0) { + vector unsigned short dp1, dp2; + + lp = pnum; + lp[0] = 1; + + processx4_step3(pkts_burst, dst_port); + + /* dp1: */ + dp1 = *(vector unsigned short *)dst_port; + + for (j = FWDSTEP; j != k; j += FWDSTEP) { + processx4_step3(&pkts_burst[j], &dst_port[j]); + + /* + * dp2: + * + */ + dp2 = *((vector unsigned short *) + &dst_port[j - FWDSTEP + 1]); + lp = port_groupx4(&pnum[j - FWDSTEP], lp, dp1, dp2); + + /* + * dp1: + * + */ + dp1 = vec_sro(dp2, (vector unsigned char) { + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, (FWDSTEP - 1) * sizeof(dst_port[0])}); + } + + /* + * dp2: + */ + dp2 = vec_perm(dp1, (vector unsigned short){}, + (vector unsigned char){0xf9}); + lp = port_groupx4(&pnum[j - FWDSTEP], lp, dp1, dp2); + + /* + * remove values added by the last repeated + * dst port. + */ + lp[0]--; + dlp = dst_port[j - 1]; + } else { + /* set dlp and lp to the never used values. */ + dlp = BAD_PORT - 1; + lp = pnum + MAX_PKT_BURST; + } + + /* Process up to last 3 packets one by one. */ + switch (nb_rx % FWDSTEP) { + case 3: + process_packet(pkts_burst[j], dst_port + j); + GROUP_PORT_STEP(dlp, dst_port, lp, pnum, j); + j++; + /* fall-through */ + case 2: + process_packet(pkts_burst[j], dst_port + j); + GROUP_PORT_STEP(dlp, dst_port, lp, pnum, j); + j++; + /* fall-through */ + case 1: + process_packet(pkts_burst[j], dst_port + j); + GROUP_PORT_STEP(dlp, dst_port, lp, pnum, j); + j++; + } + + /* + * Send packets out, through destination port. + * Consecutive packets with the same destination port + * are already grouped together. + * If destination port for the packet equals BAD_PORT, + * then free the packet without sending it out. + */ + for (j = 0; j < nb_rx; j += k) { + + int32_t m; + uint16_t pn; + + pn = dst_port[j]; + k = pnum[j]; + + if (likely(pn != BAD_PORT)) + send_packetsx4(qconf, pn, pkts_burst + j, k); + else + for (m = j; m != j + k; m++) + rte_pktmbuf_free(pkts_burst[m]); + + } +} + +#endif /* _L3FWD_ALTIVEC_H_ */ diff --git a/dpdk/examples/l3fwd/l3fwd_common.h b/dpdk/examples/l3fwd/l3fwd_common.h new file mode 100644 index 00000000..7002a43a --- /dev/null +++ b/dpdk/examples/l3fwd/l3fwd_common.h @@ -0,0 +1,293 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2017, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + + +#ifndef _L3FWD_COMMON_H_ +#define _L3FWD_COMMON_H_ + +#ifdef DO_RFC_1812_CHECKS + +#define IPV4_MIN_VER_IHL 0x45 +#define IPV4_MAX_VER_IHL 0x4f +#define IPV4_MAX_VER_IHL_DIFF (IPV4_MAX_VER_IHL - IPV4_MIN_VER_IHL) + +/* Minimum value of IPV4 total length (20B) in network byte order. */ +#define IPV4_MIN_LEN_BE (sizeof(struct ipv4_hdr) << 8) + +/* + * From http://www.rfc-editor.org/rfc/rfc1812.txt section 5.2.2: + * - The IP version number must be 4. + * - The IP header length field must be large enough to hold the + * minimum length legal IP datagram (20 bytes = 5 words). + * - The IP total length field must be large enough to hold the IP + * datagram header, whose length is specified in the IP header length + * field. + * If we encounter invalid IPV4 packet, then set destination port for it + * to BAD_PORT value. + */ +static __rte_always_inline void +rfc1812_process(struct ipv4_hdr *ipv4_hdr, uint16_t *dp, uint32_t ptype) +{ + uint8_t ihl; + + if (RTE_ETH_IS_IPV4_HDR(ptype)) { + ihl = ipv4_hdr->version_ihl - IPV4_MIN_VER_IHL; + + ipv4_hdr->time_to_live--; + ipv4_hdr->hdr_checksum++; + + if (ihl > IPV4_MAX_VER_IHL_DIFF || + ((uint8_t)ipv4_hdr->total_length == 0 && + ipv4_hdr->total_length < IPV4_MIN_LEN_BE)) + dp[0] = BAD_PORT; + + } +} + +#else +#define rfc1812_process(mb, dp, ptype) do { } while (0) +#endif /* DO_RFC_1812_CHECKS */ + +/* + * We group consecutive packets with the same destionation port into one burst. + * To avoid extra latency this is done together with some other packet + * processing, but after we made a final decision about packet's destination. + * To do this we maintain: + * pnum - array of number of consecutive packets with the same dest port for + * each packet in the input burst. + * lp - pointer to the last updated element in the pnum. + * dlp - dest port value lp corresponds to. + */ + +#define GRPSZ (1 << FWDSTEP) +#define GRPMSK (GRPSZ - 1) + +#define GROUP_PORT_STEP(dlp, dcp, lp, pn, idx) do { \ + if (likely((dlp) == (dcp)[(idx)])) { \ + (lp)[0]++; \ + } else { \ + (dlp) = (dcp)[idx]; \ + (lp) = (pn) + (idx); \ + (lp)[0] = 1; \ + } \ +} while (0) + +static const struct { + uint64_t pnum; /* prebuild 4 values for pnum[]. */ + int32_t idx; /* index for new last updated elemnet. */ + uint16_t lpv; /* add value to the last updated element. */ +} gptbl[GRPSZ] = { + { + /* 0: a != b, b != c, c != d, d != e */ + .pnum = UINT64_C(0x0001000100010001), + .idx = 4, + .lpv = 0, + }, + { + /* 1: a == b, b != c, c != d, d != e */ + .pnum = UINT64_C(0x0001000100010002), + .idx = 4, + .lpv = 1, + }, + { + /* 2: a != b, b == c, c != d, d != e */ + .pnum = UINT64_C(0x0001000100020001), + .idx = 4, + .lpv = 0, + }, + { + /* 3: a == b, b == c, c != d, d != e */ + .pnum = UINT64_C(0x0001000100020003), + .idx = 4, + .lpv = 2, + }, + { + /* 4: a != b, b != c, c == d, d != e */ + .pnum = UINT64_C(0x0001000200010001), + .idx = 4, + .lpv = 0, + }, + { + /* 5: a == b, b != c, c == d, d != e */ + .pnum = UINT64_C(0x0001000200010002), + .idx = 4, + .lpv = 1, + }, + { + /* 6: a != b, b == c, c == d, d != e */ + .pnum = UINT64_C(0x0001000200030001), + .idx = 4, + .lpv = 0, + }, + { + /* 7: a == b, b == c, c == d, d != e */ + .pnum = UINT64_C(0x0001000200030004), + .idx = 4, + .lpv = 3, + }, + { + /* 8: a != b, b != c, c != d, d == e */ + .pnum = UINT64_C(0x0002000100010001), + .idx = 3, + .lpv = 0, + }, + { + /* 9: a == b, b != c, c != d, d == e */ + .pnum = UINT64_C(0x0002000100010002), + .idx = 3, + .lpv = 1, + }, + { + /* 0xa: a != b, b == c, c != d, d == e */ + .pnum = UINT64_C(0x0002000100020001), + .idx = 3, + .lpv = 0, + }, + { + /* 0xb: a == b, b == c, c != d, d == e */ + .pnum = UINT64_C(0x0002000100020003), + .idx = 3, + .lpv = 2, + }, + { + /* 0xc: a != b, b != c, c == d, d == e */ + .pnum = UINT64_C(0x0002000300010001), + .idx = 2, + .lpv = 0, + }, + { + /* 0xd: a == b, b != c, c == d, d == e */ + .pnum = UINT64_C(0x0002000300010002), + .idx = 2, + .lpv = 1, + }, + { + /* 0xe: a != b, b == c, c == d, d == e */ + .pnum = UINT64_C(0x0002000300040001), + .idx = 1, + .lpv = 0, + }, + { + /* 0xf: a == b, b == c, c == d, d == e */ + .pnum = UINT64_C(0x0002000300040005), + .idx = 0, + .lpv = 4, + }, +}; + +static __rte_always_inline void +send_packetsx4(struct lcore_conf *qconf, uint16_t port, struct rte_mbuf *m[], + uint32_t num) +{ + uint32_t len, j, n; + + len = qconf->tx_mbufs[port].len; + + /* + * If TX buffer for that queue is empty, and we have enough packets, + * then send them straightway. + */ + if (num >= MAX_TX_BURST && len == 0) { + n = rte_eth_tx_burst(port, qconf->tx_queue_id[port], m, num); + if (unlikely(n < num)) { + do { + rte_pktmbuf_free(m[n]); + } while (++n < num); + } + return; + } + + /* + * Put packets into TX buffer for that queue. + */ + + n = len + num; + n = (n > MAX_PKT_BURST) ? MAX_PKT_BURST - len : num; + + j = 0; + switch (n % FWDSTEP) { + while (j < n) { + case 0: + qconf->tx_mbufs[port].m_table[len + j] = m[j]; + j++; + /* fallthrough */ + case 3: + qconf->tx_mbufs[port].m_table[len + j] = m[j]; + j++; + /* fallthrough */ + case 2: + qconf->tx_mbufs[port].m_table[len + j] = m[j]; + j++; + /* fallthrough */ + case 1: + qconf->tx_mbufs[port].m_table[len + j] = m[j]; + j++; + } + } + + len += n; + + /* enough pkts to be sent */ + if (unlikely(len == MAX_PKT_BURST)) { + + send_burst(qconf, MAX_PKT_BURST, port); + + /* copy rest of the packets into the TX buffer. */ + len = num - n; + j = 0; + switch (len % FWDSTEP) { + while (j < len) { + case 0: + qconf->tx_mbufs[port].m_table[j] = m[n + j]; + j++; + /* fallthrough */ + case 3: + qconf->tx_mbufs[port].m_table[j] = m[n + j]; + j++; + /* fallthrough */ + case 2: + qconf->tx_mbufs[port].m_table[j] = m[n + j]; + j++; + /* fallthrough */ + case 1: + qconf->tx_mbufs[port].m_table[j] = m[n + j]; + j++; + } + } + } + + qconf->tx_mbufs[port].len = len; +} + +#endif /* _L3FWD_COMMON_H_ */ diff --git a/dpdk/examples/l3fwd/l3fwd_em.c b/dpdk/examples/l3fwd/l3fwd_em.c index def5a024..2b7c173b 100644 --- a/dpdk/examples/l3fwd/l3fwd_em.c +++ b/dpdk/examples/l3fwd/l3fwd_em.c @@ -47,7 +47,6 @@ #include #include #include -#include #include #include #include @@ -58,13 +57,17 @@ #include "l3fwd.h" -#ifdef RTE_MACHINE_CPUFLAG_SSE4_2 +#if defined(RTE_ARCH_X86) || defined(RTE_MACHINE_CPUFLAG_CRC32) +#define EM_HASH_CRC 1 +#endif + +#ifdef EM_HASH_CRC #include #define DEFAULT_HASH_FUNC rte_hash_crc #else #include #define DEFAULT_HASH_FUNC rte_jhash -#endif /* RTE_MACHINE_CPUFLAG_SSE4_2 */ +#endif #define IPV6_ADDR_LEN 16 @@ -169,17 +172,17 @@ ipv4_hash_crc(const void *data, __rte_unused uint32_t data_len, t = k->proto; p = (const uint32_t *)&k->port_src; -#ifdef RTE_MACHINE_CPUFLAG_SSE4_2 +#ifdef EM_HASH_CRC init_val = rte_hash_crc_4byte(t, init_val); init_val = rte_hash_crc_4byte(k->ip_src, init_val); init_val = rte_hash_crc_4byte(k->ip_dst, init_val); init_val = rte_hash_crc_4byte(*p, init_val); -#else /* RTE_MACHINE_CPUFLAG_SSE4_2 */ +#else init_val = rte_jhash_1word(t, init_val); init_val = rte_jhash_1word(k->ip_src, init_val); init_val = rte_jhash_1word(k->ip_dst, init_val); init_val = rte_jhash_1word(*p, init_val); -#endif /* RTE_MACHINE_CPUFLAG_SSE4_2 */ +#endif return init_val; } @@ -191,16 +194,16 @@ ipv6_hash_crc(const void *data, __rte_unused uint32_t data_len, const union ipv6_5tuple_host *k; uint32_t t; const uint32_t *p; -#ifdef RTE_MACHINE_CPUFLAG_SSE4_2 +#ifdef EM_HASH_CRC const uint32_t *ip_src0, *ip_src1, *ip_src2, *ip_src3; const uint32_t *ip_dst0, *ip_dst1, *ip_dst2, *ip_dst3; -#endif /* RTE_MACHINE_CPUFLAG_SSE4_2 */ +#endif k = data; t = k->proto; p = (const uint32_t *)&k->port_src; -#ifdef RTE_MACHINE_CPUFLAG_SSE4_2 +#ifdef EM_HASH_CRC ip_src0 = (const uint32_t *) k->ip_src; ip_src1 = (const uint32_t *)(k->ip_src+4); ip_src2 = (const uint32_t *)(k->ip_src+8); @@ -219,14 +222,14 @@ ipv6_hash_crc(const void *data, __rte_unused uint32_t data_len, init_val = rte_hash_crc_4byte(*ip_dst2, init_val); init_val = rte_hash_crc_4byte(*ip_dst3, init_val); init_val = rte_hash_crc_4byte(*p, init_val); -#else /* RTE_MACHINE_CPUFLAG_SSE4_2 */ +#else init_val = rte_jhash_1word(t, init_val); init_val = rte_jhash(k->ip_src, sizeof(uint8_t) * IPV6_ADDR_LEN, init_val); init_val = rte_jhash(k->ip_dst, sizeof(uint8_t) * IPV6_ADDR_LEN, init_val); init_val = rte_jhash_1word(*p, init_val); -#endif /* RTE_MACHINE_CPUFLAG_SSE4_2 */ +#endif return init_val; } @@ -243,7 +246,7 @@ static rte_xmm_t mask0; static rte_xmm_t mask1; static rte_xmm_t mask2; -#if defined(__SSE2__) +#if defined(RTE_MACHINE_CPUFLAG_SSE2) static inline xmm_t em_mask_key(void *key, xmm_t mask) { @@ -259,12 +262,20 @@ em_mask_key(void *key, xmm_t mask) return vandq_s32(data, mask); } +#elif defined(RTE_MACHINE_CPUFLAG_ALTIVEC) +static inline xmm_t +em_mask_key(void *key, xmm_t mask) +{ + xmm_t data = vec_ld(0, (xmm_t *)(key)); + + return vec_and(data, mask); +} #else -#error No vector engine (SSE, NEON) available, check your toolchain +#error No vector engine (SSE, NEON, ALTIVEC) available, check your toolchain #endif -static inline uint8_t -em_get_ipv4_dst_port(void *ipv4_hdr, uint8_t portid, void *lookup_struct) +static inline uint16_t +em_get_ipv4_dst_port(void *ipv4_hdr, uint16_t portid, void *lookup_struct) { int ret = 0; union ipv4_5tuple_host key; @@ -281,11 +292,11 @@ em_get_ipv4_dst_port(void *ipv4_hdr, uint8_t portid, void *lookup_struct) /* Find destination port */ ret = rte_hash_lookup(ipv4_l3fwd_lookup_struct, (const void *)&key); - return (uint8_t)((ret < 0) ? portid : ipv4_l3fwd_out_if[ret]); + return (ret < 0) ? portid : ipv4_l3fwd_out_if[ret]; } -static inline uint8_t -em_get_ipv6_dst_port(void *ipv6_hdr, uint8_t portid, void *lookup_struct) +static inline uint16_t +em_get_ipv6_dst_port(void *ipv6_hdr, uint16_t portid, void *lookup_struct) { int ret = 0; union ipv6_5tuple_host key; @@ -314,14 +325,14 @@ em_get_ipv6_dst_port(void *ipv6_hdr, uint8_t portid, void *lookup_struct) /* Find destination port */ ret = rte_hash_lookup(ipv6_l3fwd_lookup_struct, (const void *)&key); - return (uint8_t)((ret < 0) ? portid : ipv6_l3fwd_out_if[ret]); + return (ret < 0) ? portid : ipv6_l3fwd_out_if[ret]; } -#if defined(__SSE4_1__) +#if defined RTE_ARCH_X86 || defined RTE_MACHINE_CPUFLAG_NEON #if defined(NO_HASH_MULTI_LOOKUP) -#include "l3fwd_em_sse.h" +#include "l3fwd_em_sequential.h" #else -#include "l3fwd_em_hlm_sse.h" +#include "l3fwd_em_hlm.h" #endif #else #include "l3fwd_em.h" @@ -603,7 +614,7 @@ em_parse_ptype(struct rte_mbuf *m) packet_type |= RTE_PTYPE_L4_UDP; } else packet_type |= RTE_PTYPE_L3_IPV4_EXT; - } else if (ether_type == rte_cpu_to_be_16(ETHER_TYPE_IPv4)) { + } else if (ether_type == rte_cpu_to_be_16(ETHER_TYPE_IPv6)) { ipv6_hdr = (struct ipv6_hdr *)l3; if (ipv6_hdr->proto == IPPROTO_TCP) packet_type |= RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_TCP; @@ -617,7 +628,7 @@ em_parse_ptype(struct rte_mbuf *m) } uint16_t -em_cb_parse_ptype(uint8_t port __rte_unused, uint16_t queue __rte_unused, +em_cb_parse_ptype(uint16_t port __rte_unused, uint16_t queue __rte_unused, struct rte_mbuf *pkts[], uint16_t nb_pkts, uint16_t max_pkts __rte_unused, void *user_param __rte_unused) @@ -638,7 +649,8 @@ em_main_loop(__attribute__((unused)) void *dummy) unsigned lcore_id; uint64_t prev_tsc, diff_tsc, cur_tsc; int i, nb_rx; - uint8_t portid, queueid; + uint8_t queueid; + uint16_t portid; struct lcore_conf *qconf; const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / US_PER_S * BURST_TX_DRAIN_US; @@ -660,7 +672,7 @@ em_main_loop(__attribute__((unused)) void *dummy) portid = qconf->rx_queue_list[i].port_id; queueid = qconf->rx_queue_list[i].queue_id; RTE_LOG(INFO, L3FWD, - " -- lcoreid=%u portid=%hhu rxqueueid=%hhu\n", + " -- lcoreid=%u portid=%u rxqueueid=%hhu\n", lcore_id, portid, queueid); } @@ -698,13 +710,13 @@ em_main_loop(__attribute__((unused)) void *dummy) if (nb_rx == 0) continue; -#if defined(__SSE4_1__) +#if defined RTE_ARCH_X86 || defined RTE_MACHINE_CPUFLAG_NEON l3fwd_em_send_packets(nb_rx, pkts_burst, portid, qconf); #else l3fwd_em_no_opt_send_packets(nb_rx, pkts_burst, portid, qconf); -#endif /* __SSE_4_1__ */ +#endif } } diff --git a/dpdk/examples/l3fwd/l3fwd_em.h b/dpdk/examples/l3fwd/l3fwd_em.h index 2284bbd5..302291d7 100644 --- a/dpdk/examples/l3fwd/l3fwd_em.h +++ b/dpdk/examples/l3fwd/l3fwd_em.h @@ -34,13 +34,13 @@ #ifndef __L3FWD_EM_H__ #define __L3FWD_EM_H__ -static inline __attribute__((always_inline)) void -l3fwd_em_simple_forward(struct rte_mbuf *m, uint8_t portid, +static __rte_always_inline void +l3fwd_em_simple_forward(struct rte_mbuf *m, uint16_t portid, struct lcore_conf *qconf) { struct ether_hdr *eth_hdr; struct ipv4_hdr *ipv4_hdr; - uint8_t dst_port; + uint16_t dst_port; uint32_t tcp_or_udp; uint32_t l3_ptypes; @@ -112,7 +112,7 @@ l3fwd_em_simple_forward(struct rte_mbuf *m, uint8_t portid, */ static inline void l3fwd_em_no_opt_send_packets(int nb_rx, struct rte_mbuf **pkts_burst, - uint8_t portid, struct lcore_conf *qconf) + uint16_t portid, struct lcore_conf *qconf) { int32_t j; diff --git a/dpdk/examples/l3fwd/l3fwd_em_hlm.h b/dpdk/examples/l3fwd/l3fwd_em_hlm.h new file mode 100644 index 00000000..9d7afe05 --- /dev/null +++ b/dpdk/examples/l3fwd/l3fwd_em_hlm.h @@ -0,0 +1,218 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2017, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __L3FWD_EM_HLM_H__ +#define __L3FWD_EM_HLM_H__ + +#if defined RTE_ARCH_X86 +#include "l3fwd_sse.h" +#include "l3fwd_em_hlm_sse.h" +#elif defined RTE_MACHINE_CPUFLAG_NEON +#include "l3fwd_neon.h" +#include "l3fwd_em_hlm_neon.h" +#endif + +#ifdef RTE_ARCH_ARM64 +#define EM_HASH_LOOKUP_COUNT 16 +#else +#define EM_HASH_LOOKUP_COUNT 8 +#endif + + +static __rte_always_inline void +em_get_dst_port_ipv4xN(struct lcore_conf *qconf, struct rte_mbuf *m[], + uint16_t portid, uint16_t dst_port[]) +{ + int i; + int32_t ret[EM_HASH_LOOKUP_COUNT]; + union ipv4_5tuple_host key[EM_HASH_LOOKUP_COUNT]; + const void *key_array[EM_HASH_LOOKUP_COUNT]; + + for (i = 0; i < EM_HASH_LOOKUP_COUNT; i++) { + get_ipv4_5tuple(m[i], mask0.x, &key[i]); + key_array[i] = &key[i]; + } + + rte_hash_lookup_bulk(qconf->ipv4_lookup_struct, &key_array[0], + EM_HASH_LOOKUP_COUNT, ret); + + for (i = 0; i < EM_HASH_LOOKUP_COUNT; i++) { + dst_port[i] = ((ret[i] < 0) ? + portid : ipv4_l3fwd_out_if[ret[i]]); + + if (dst_port[i] >= RTE_MAX_ETHPORTS || + (enabled_port_mask & 1 << dst_port[i]) == 0) + dst_port[i] = portid; + } +} + +static __rte_always_inline void +em_get_dst_port_ipv6xN(struct lcore_conf *qconf, struct rte_mbuf *m[], + uint16_t portid, uint16_t dst_port[]) +{ + int i; + int32_t ret[EM_HASH_LOOKUP_COUNT]; + union ipv6_5tuple_host key[EM_HASH_LOOKUP_COUNT]; + const void *key_array[EM_HASH_LOOKUP_COUNT]; + + for (i = 0; i < EM_HASH_LOOKUP_COUNT; i++) { + get_ipv6_5tuple(m[i], mask1.x, mask2.x, &key[i]); + key_array[i] = &key[i]; + } + + rte_hash_lookup_bulk(qconf->ipv6_lookup_struct, &key_array[0], + EM_HASH_LOOKUP_COUNT, ret); + + for (i = 0; i < EM_HASH_LOOKUP_COUNT; i++) { + dst_port[i] = ((ret[i] < 0) ? + portid : ipv6_l3fwd_out_if[ret[i]]); + + if (dst_port[i] >= RTE_MAX_ETHPORTS || + (enabled_port_mask & 1 << dst_port[i]) == 0) + dst_port[i] = portid; + } +} + +static __rte_always_inline uint16_t +em_get_dst_port(const struct lcore_conf *qconf, struct rte_mbuf *pkt, + uint16_t portid) +{ + uint16_t next_hop; + struct ipv4_hdr *ipv4_hdr; + struct ipv6_hdr *ipv6_hdr; + uint32_t tcp_or_udp; + uint32_t l3_ptypes; + + tcp_or_udp = pkt->packet_type & (RTE_PTYPE_L4_TCP | RTE_PTYPE_L4_UDP); + l3_ptypes = pkt->packet_type & RTE_PTYPE_L3_MASK; + + if (tcp_or_udp && (l3_ptypes == RTE_PTYPE_L3_IPV4)) { + + /* Handle IPv4 headers.*/ + ipv4_hdr = rte_pktmbuf_mtod_offset(pkt, struct ipv4_hdr *, + sizeof(struct ether_hdr)); + + next_hop = em_get_ipv4_dst_port(ipv4_hdr, portid, + qconf->ipv4_lookup_struct); + + if (next_hop >= RTE_MAX_ETHPORTS || + (enabled_port_mask & 1 << next_hop) == 0) + next_hop = portid; + + return next_hop; + + } else if (tcp_or_udp && (l3_ptypes == RTE_PTYPE_L3_IPV6)) { + + /* Handle IPv6 headers.*/ + ipv6_hdr = rte_pktmbuf_mtod_offset(pkt, struct ipv6_hdr *, + sizeof(struct ether_hdr)); + + next_hop = em_get_ipv6_dst_port(ipv6_hdr, portid, + qconf->ipv6_lookup_struct); + + if (next_hop >= RTE_MAX_ETHPORTS || + (enabled_port_mask & 1 << next_hop) == 0) + next_hop = portid; + + return next_hop; + + } + + return portid; +} + +/* + * Buffer optimized handling of packets, invoked + * from main_loop. + */ +static inline void +l3fwd_em_send_packets(int nb_rx, struct rte_mbuf **pkts_burst, + uint16_t portid, struct lcore_conf *qconf) +{ + int32_t i, j, pos; + uint16_t dst_port[MAX_PKT_BURST]; + + /* + * Send nb_rx - nb_rx % EM_HASH_LOOKUP_COUNT packets + * in groups of EM_HASH_LOOKUP_COUNT. + */ + int32_t n = RTE_ALIGN_FLOOR(nb_rx, EM_HASH_LOOKUP_COUNT); + + for (j = 0; j < EM_HASH_LOOKUP_COUNT && j < nb_rx; j++) { + rte_prefetch0(rte_pktmbuf_mtod(pkts_burst[j], + struct ether_hdr *) + 1); + } + + for (j = 0; j < n; j += EM_HASH_LOOKUP_COUNT) { + + uint32_t pkt_type = RTE_PTYPE_L3_MASK | + RTE_PTYPE_L4_TCP | RTE_PTYPE_L4_UDP; + uint32_t l3_type, tcp_or_udp; + + for (i = 0; i < EM_HASH_LOOKUP_COUNT; i++) + pkt_type &= pkts_burst[j + i]->packet_type; + + l3_type = pkt_type & RTE_PTYPE_L3_MASK; + tcp_or_udp = pkt_type & (RTE_PTYPE_L4_TCP | RTE_PTYPE_L4_UDP); + + for (i = 0, pos = j + EM_HASH_LOOKUP_COUNT; + i < EM_HASH_LOOKUP_COUNT && pos < nb_rx; i++, pos++) { + rte_prefetch0(rte_pktmbuf_mtod(pkts_burst[pos], + struct ether_hdr *) + 1); + } + + if (tcp_or_udp && (l3_type == RTE_PTYPE_L3_IPV4)) { + + em_get_dst_port_ipv4xN(qconf, &pkts_burst[j], portid, + &dst_port[j]); + + } else if (tcp_or_udp && (l3_type == RTE_PTYPE_L3_IPV6)) { + + em_get_dst_port_ipv6xN(qconf, &pkts_burst[j], portid, + &dst_port[j]); + + } else { + for (i = 0; i < EM_HASH_LOOKUP_COUNT; i++) + dst_port[j + i] = em_get_dst_port(qconf, + pkts_burst[j + i], portid); + } + } + + for (; j < nb_rx; j++) + dst_port[j] = em_get_dst_port(qconf, pkts_burst[j], portid); + + send_packets_multi(qconf, pkts_burst, dst_port, nb_rx); + +} +#endif /* __L3FWD_EM_HLM_H__ */ diff --git a/dpdk/examples/l3fwd/l3fwd_em_hlm_neon.h b/dpdk/examples/l3fwd/l3fwd_em_hlm_neon.h new file mode 100644 index 00000000..dae1acfb --- /dev/null +++ b/dpdk/examples/l3fwd/l3fwd_em_hlm_neon.h @@ -0,0 +1,74 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2017, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __L3FWD_EM_HLM_NEON_H__ +#define __L3FWD_EM_HLM_NEON_H__ + +#include + +static inline void +get_ipv4_5tuple(struct rte_mbuf *m0, int32x4_t mask0, + union ipv4_5tuple_host *key) +{ + int32x4_t tmpdata0 = vld1q_s32(rte_pktmbuf_mtod_offset(m0, int32_t *, + sizeof(struct ether_hdr) + + offsetof(struct ipv4_hdr, time_to_live))); + + key->xmm = vandq_s32(tmpdata0, mask0); +} + +static inline void +get_ipv6_5tuple(struct rte_mbuf *m0, int32x4_t mask0, + int32x4_t mask1, union ipv6_5tuple_host *key) +{ + int32x4_t tmpdata0 = vld1q_s32( + rte_pktmbuf_mtod_offset(m0, int *, + sizeof(struct ether_hdr) + + offsetof(struct ipv6_hdr, payload_len))); + + int32x4_t tmpdata1 = vld1q_s32( + rte_pktmbuf_mtod_offset(m0, int *, + sizeof(struct ether_hdr) + + offsetof(struct ipv6_hdr, payload_len) + 8)); + + int32x4_t tmpdata2 = vld1q_s32( + rte_pktmbuf_mtod_offset(m0, int *, + sizeof(struct ether_hdr) + + offsetof(struct ipv6_hdr, payload_len) + 16)); + + key->xmm[0] = vandq_s32(tmpdata0, mask0); + key->xmm[1] = tmpdata1; + key->xmm[2] = vandq_s32(tmpdata2, mask1); +} +#endif /* __L3FWD_EM_HLM_NEON_H__ */ diff --git a/dpdk/examples/l3fwd/l3fwd_em_hlm_sse.h b/dpdk/examples/l3fwd/l3fwd_em_hlm_sse.h index 7714a20c..0dd44dfa 100644 --- a/dpdk/examples/l3fwd/l3fwd_em_hlm_sse.h +++ b/dpdk/examples/l3fwd/l3fwd_em_hlm_sse.h @@ -36,102 +36,16 @@ #include "l3fwd_sse.h" -static inline __attribute__((always_inline)) void -em_get_dst_port_ipv4x8(struct lcore_conf *qconf, struct rte_mbuf *m[8], - uint8_t portid, uint16_t dst_port[8]) +static __rte_always_inline void +get_ipv4_5tuple(struct rte_mbuf *m0, __m128i mask0, + union ipv4_5tuple_host *key) { - int32_t ret[8]; - union ipv4_5tuple_host key[8]; - __m128i data[8]; - - data[0] = _mm_loadu_si128(rte_pktmbuf_mtod_offset(m[0], __m128i *, - sizeof(struct ether_hdr) + - offsetof(struct ipv4_hdr, time_to_live))); - data[1] = _mm_loadu_si128(rte_pktmbuf_mtod_offset(m[1], __m128i *, - sizeof(struct ether_hdr) + - offsetof(struct ipv4_hdr, time_to_live))); - data[2] = _mm_loadu_si128(rte_pktmbuf_mtod_offset(m[2], __m128i *, - sizeof(struct ether_hdr) + - offsetof(struct ipv4_hdr, time_to_live))); - data[3] = _mm_loadu_si128(rte_pktmbuf_mtod_offset(m[3], __m128i *, - sizeof(struct ether_hdr) + - offsetof(struct ipv4_hdr, time_to_live))); - data[4] = _mm_loadu_si128(rte_pktmbuf_mtod_offset(m[4], __m128i *, - sizeof(struct ether_hdr) + - offsetof(struct ipv4_hdr, time_to_live))); - data[5] = _mm_loadu_si128(rte_pktmbuf_mtod_offset(m[5], __m128i *, - sizeof(struct ether_hdr) + - offsetof(struct ipv4_hdr, time_to_live))); - data[6] = _mm_loadu_si128(rte_pktmbuf_mtod_offset(m[6], __m128i *, - sizeof(struct ether_hdr) + - offsetof(struct ipv4_hdr, time_to_live))); - data[7] = _mm_loadu_si128(rte_pktmbuf_mtod_offset(m[7], __m128i *, + __m128i tmpdata0 = _mm_loadu_si128( + rte_pktmbuf_mtod_offset(m0, __m128i *, sizeof(struct ether_hdr) + offsetof(struct ipv4_hdr, time_to_live))); - key[0].xmm = _mm_and_si128(data[0], mask0.x); - key[1].xmm = _mm_and_si128(data[1], mask0.x); - key[2].xmm = _mm_and_si128(data[2], mask0.x); - key[3].xmm = _mm_and_si128(data[3], mask0.x); - key[4].xmm = _mm_and_si128(data[4], mask0.x); - key[5].xmm = _mm_and_si128(data[5], mask0.x); - key[6].xmm = _mm_and_si128(data[6], mask0.x); - key[7].xmm = _mm_and_si128(data[7], mask0.x); - - const void *key_array[8] = {&key[0], &key[1], &key[2], &key[3], - &key[4], &key[5], &key[6], &key[7]}; - - rte_hash_lookup_bulk(qconf->ipv4_lookup_struct, &key_array[0], 8, ret); - - dst_port[0] = (uint8_t) ((ret[0] < 0) ? - portid : ipv4_l3fwd_out_if[ret[0]]); - dst_port[1] = (uint8_t) ((ret[1] < 0) ? - portid : ipv4_l3fwd_out_if[ret[1]]); - dst_port[2] = (uint8_t) ((ret[2] < 0) ? - portid : ipv4_l3fwd_out_if[ret[2]]); - dst_port[3] = (uint8_t) ((ret[3] < 0) ? - portid : ipv4_l3fwd_out_if[ret[3]]); - dst_port[4] = (uint8_t) ((ret[4] < 0) ? - portid : ipv4_l3fwd_out_if[ret[4]]); - dst_port[5] = (uint8_t) ((ret[5] < 0) ? - portid : ipv4_l3fwd_out_if[ret[5]]); - dst_port[6] = (uint8_t) ((ret[6] < 0) ? - portid : ipv4_l3fwd_out_if[ret[6]]); - dst_port[7] = (uint8_t) ((ret[7] < 0) ? - portid : ipv4_l3fwd_out_if[ret[7]]); - - if (dst_port[0] >= RTE_MAX_ETHPORTS || - (enabled_port_mask & 1 << dst_port[0]) == 0) - dst_port[0] = portid; - - if (dst_port[1] >= RTE_MAX_ETHPORTS || - (enabled_port_mask & 1 << dst_port[1]) == 0) - dst_port[1] = portid; - - if (dst_port[2] >= RTE_MAX_ETHPORTS || - (enabled_port_mask & 1 << dst_port[2]) == 0) - dst_port[2] = portid; - - if (dst_port[3] >= RTE_MAX_ETHPORTS || - (enabled_port_mask & 1 << dst_port[3]) == 0) - dst_port[3] = portid; - - if (dst_port[4] >= RTE_MAX_ETHPORTS || - (enabled_port_mask & 1 << dst_port[4]) == 0) - dst_port[4] = portid; - - if (dst_port[5] >= RTE_MAX_ETHPORTS || - (enabled_port_mask & 1 << dst_port[5]) == 0) - dst_port[5] = portid; - - if (dst_port[6] >= RTE_MAX_ETHPORTS || - (enabled_port_mask & 1 << dst_port[6]) == 0) - dst_port[6] = portid; - - if (dst_port[7] >= RTE_MAX_ETHPORTS || - (enabled_port_mask & 1 << dst_port[7]) == 0) - dst_port[7] = portid; - + key->xmm = _mm_and_si128(tmpdata0, mask0); } static inline void @@ -159,184 +73,4 @@ get_ipv6_5tuple(struct rte_mbuf *m0, __m128i mask0, key->xmm[1] = tmpdata1; key->xmm[2] = _mm_and_si128(tmpdata2, mask1); } - -static inline __attribute__((always_inline)) void -em_get_dst_port_ipv6x8(struct lcore_conf *qconf, struct rte_mbuf *m[8], - uint8_t portid, uint16_t dst_port[8]) -{ - int32_t ret[8]; - union ipv6_5tuple_host key[8]; - - get_ipv6_5tuple(m[0], mask1.x, mask2.x, &key[0]); - get_ipv6_5tuple(m[1], mask1.x, mask2.x, &key[1]); - get_ipv6_5tuple(m[2], mask1.x, mask2.x, &key[2]); - get_ipv6_5tuple(m[3], mask1.x, mask2.x, &key[3]); - get_ipv6_5tuple(m[4], mask1.x, mask2.x, &key[4]); - get_ipv6_5tuple(m[5], mask1.x, mask2.x, &key[5]); - get_ipv6_5tuple(m[6], mask1.x, mask2.x, &key[6]); - get_ipv6_5tuple(m[7], mask1.x, mask2.x, &key[7]); - - const void *key_array[8] = {&key[0], &key[1], &key[2], &key[3], - &key[4], &key[5], &key[6], &key[7]}; - - rte_hash_lookup_bulk(qconf->ipv6_lookup_struct, &key_array[0], 8, ret); - - dst_port[0] = (uint8_t) ((ret[0] < 0) ? - portid : ipv6_l3fwd_out_if[ret[0]]); - dst_port[1] = (uint8_t) ((ret[1] < 0) ? - portid : ipv6_l3fwd_out_if[ret[1]]); - dst_port[2] = (uint8_t) ((ret[2] < 0) ? - portid : ipv6_l3fwd_out_if[ret[2]]); - dst_port[3] = (uint8_t) ((ret[3] < 0) ? - portid : ipv6_l3fwd_out_if[ret[3]]); - dst_port[4] = (uint8_t) ((ret[4] < 0) ? - portid : ipv6_l3fwd_out_if[ret[4]]); - dst_port[5] = (uint8_t) ((ret[5] < 0) ? - portid : ipv6_l3fwd_out_if[ret[5]]); - dst_port[6] = (uint8_t) ((ret[6] < 0) ? - portid : ipv6_l3fwd_out_if[ret[6]]); - dst_port[7] = (uint8_t) ((ret[7] < 0) ? - portid : ipv6_l3fwd_out_if[ret[7]]); - - if (dst_port[0] >= RTE_MAX_ETHPORTS || - (enabled_port_mask & 1 << dst_port[0]) == 0) - dst_port[0] = portid; - - if (dst_port[1] >= RTE_MAX_ETHPORTS || - (enabled_port_mask & 1 << dst_port[1]) == 0) - dst_port[1] = portid; - - if (dst_port[2] >= RTE_MAX_ETHPORTS || - (enabled_port_mask & 1 << dst_port[2]) == 0) - dst_port[2] = portid; - - if (dst_port[3] >= RTE_MAX_ETHPORTS || - (enabled_port_mask & 1 << dst_port[3]) == 0) - dst_port[3] = portid; - - if (dst_port[4] >= RTE_MAX_ETHPORTS || - (enabled_port_mask & 1 << dst_port[4]) == 0) - dst_port[4] = portid; - - if (dst_port[5] >= RTE_MAX_ETHPORTS || - (enabled_port_mask & 1 << dst_port[5]) == 0) - dst_port[5] = portid; - - if (dst_port[6] >= RTE_MAX_ETHPORTS || - (enabled_port_mask & 1 << dst_port[6]) == 0) - dst_port[6] = portid; - - if (dst_port[7] >= RTE_MAX_ETHPORTS || - (enabled_port_mask & 1 << dst_port[7]) == 0) - dst_port[7] = portid; - -} - -static inline __attribute__((always_inline)) uint16_t -em_get_dst_port(const struct lcore_conf *qconf, struct rte_mbuf *pkt, - uint8_t portid) -{ - uint8_t next_hop; - struct ipv4_hdr *ipv4_hdr; - struct ipv6_hdr *ipv6_hdr; - uint32_t tcp_or_udp; - uint32_t l3_ptypes; - - tcp_or_udp = pkt->packet_type & (RTE_PTYPE_L4_TCP | RTE_PTYPE_L4_UDP); - l3_ptypes = pkt->packet_type & RTE_PTYPE_L3_MASK; - - if (tcp_or_udp && (l3_ptypes == RTE_PTYPE_L3_IPV4)) { - - /* Handle IPv4 headers.*/ - ipv4_hdr = rte_pktmbuf_mtod_offset(pkt, struct ipv4_hdr *, - sizeof(struct ether_hdr)); - - next_hop = em_get_ipv4_dst_port(ipv4_hdr, portid, - qconf->ipv4_lookup_struct); - - if (next_hop >= RTE_MAX_ETHPORTS || - (enabled_port_mask & 1 << next_hop) == 0) - next_hop = portid; - - return next_hop; - - } else if (tcp_or_udp && (l3_ptypes == RTE_PTYPE_L3_IPV6)) { - - /* Handle IPv6 headers.*/ - ipv6_hdr = rte_pktmbuf_mtod_offset(pkt, struct ipv6_hdr *, - sizeof(struct ether_hdr)); - - next_hop = em_get_ipv6_dst_port(ipv6_hdr, portid, - qconf->ipv6_lookup_struct); - - if (next_hop >= RTE_MAX_ETHPORTS || - (enabled_port_mask & 1 << next_hop) == 0) - next_hop = portid; - - return next_hop; - - } - - return portid; -} - -/* - * Buffer optimized handling of packets, invoked - * from main_loop. - */ -static inline void -l3fwd_em_send_packets(int nb_rx, struct rte_mbuf **pkts_burst, - uint8_t portid, struct lcore_conf *qconf) -{ - int32_t j; - uint16_t dst_port[MAX_PKT_BURST]; - - /* - * Send nb_rx - nb_rx%8 packets - * in groups of 8. - */ - int32_t n = RTE_ALIGN_FLOOR(nb_rx, 8); - - for (j = 0; j < n; j += 8) { - - uint32_t pkt_type = - pkts_burst[j]->packet_type & - pkts_burst[j+1]->packet_type & - pkts_burst[j+2]->packet_type & - pkts_burst[j+3]->packet_type & - pkts_burst[j+4]->packet_type & - pkts_burst[j+5]->packet_type & - pkts_burst[j+6]->packet_type & - pkts_burst[j+7]->packet_type; - - uint32_t l3_type = pkt_type & RTE_PTYPE_L3_MASK; - uint32_t tcp_or_udp = pkt_type & - (RTE_PTYPE_L4_TCP | RTE_PTYPE_L4_UDP); - - if (tcp_or_udp && (l3_type == RTE_PTYPE_L3_IPV4)) { - - em_get_dst_port_ipv4x8(qconf, &pkts_burst[j], portid, &dst_port[j]); - - } else if (tcp_or_udp && (l3_type == RTE_PTYPE_L3_IPV6)) { - - em_get_dst_port_ipv6x8(qconf, &pkts_burst[j], portid, &dst_port[j]); - - } else { - dst_port[j] = em_get_dst_port(qconf, pkts_burst[j], portid); - dst_port[j+1] = em_get_dst_port(qconf, pkts_burst[j+1], portid); - dst_port[j+2] = em_get_dst_port(qconf, pkts_burst[j+2], portid); - dst_port[j+3] = em_get_dst_port(qconf, pkts_burst[j+3], portid); - dst_port[j+4] = em_get_dst_port(qconf, pkts_burst[j+4], portid); - dst_port[j+5] = em_get_dst_port(qconf, pkts_burst[j+5], portid); - dst_port[j+6] = em_get_dst_port(qconf, pkts_burst[j+6], portid); - dst_port[j+7] = em_get_dst_port(qconf, pkts_burst[j+7], portid); - } - } - - for (; j < nb_rx; j++) - dst_port[j] = em_get_dst_port(qconf, pkts_burst[j], portid); - - send_packets_multi(qconf, pkts_burst, dst_port, nb_rx); - -} #endif /* __L3FWD_EM_SSE_HLM_H__ */ diff --git a/dpdk/examples/l3fwd/l3fwd_em_sequential.h b/dpdk/examples/l3fwd/l3fwd_em_sequential.h new file mode 100644 index 00000000..fa89f0f3 --- /dev/null +++ b/dpdk/examples/l3fwd/l3fwd_em_sequential.h @@ -0,0 +1,126 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __L3FWD_EM_SEQUENTIAL_H__ +#define __L3FWD_EM_SEQUENTIAL_H__ + +/** + * @file + * This is an optional implementation of packet classification in Exact-Match + * path using sequential packet classification method. + * While hash lookup multi seems to provide better performance, it's disabled + * by default and can be enabled with NO_HASH_LOOKUP_MULTI global define in + * compilation time. + */ + +#if defined RTE_ARCH_X86 +#include "l3fwd_sse.h" +#elif defined RTE_MACHINE_CPUFLAG_NEON +#include "l3fwd_neon.h" +#endif + +static __rte_always_inline uint16_t +em_get_dst_port(const struct lcore_conf *qconf, struct rte_mbuf *pkt, + uint16_t portid) +{ + uint8_t next_hop; + struct ipv4_hdr *ipv4_hdr; + struct ipv6_hdr *ipv6_hdr; + uint32_t tcp_or_udp; + uint32_t l3_ptypes; + + tcp_or_udp = pkt->packet_type & (RTE_PTYPE_L4_TCP | RTE_PTYPE_L4_UDP); + l3_ptypes = pkt->packet_type & RTE_PTYPE_L3_MASK; + + if (tcp_or_udp && (l3_ptypes == RTE_PTYPE_L3_IPV4)) { + + /* Handle IPv4 headers.*/ + ipv4_hdr = rte_pktmbuf_mtod_offset(pkt, struct ipv4_hdr *, + sizeof(struct ether_hdr)); + + next_hop = em_get_ipv4_dst_port(ipv4_hdr, portid, + qconf->ipv4_lookup_struct); + + if (next_hop >= RTE_MAX_ETHPORTS || + (enabled_port_mask & 1 << next_hop) == 0) + next_hop = portid; + + return next_hop; + + } else if (tcp_or_udp && (l3_ptypes == RTE_PTYPE_L3_IPV6)) { + + /* Handle IPv6 headers.*/ + ipv6_hdr = rte_pktmbuf_mtod_offset(pkt, struct ipv6_hdr *, + sizeof(struct ether_hdr)); + + next_hop = em_get_ipv6_dst_port(ipv6_hdr, portid, + qconf->ipv6_lookup_struct); + + if (next_hop >= RTE_MAX_ETHPORTS || + (enabled_port_mask & 1 << next_hop) == 0) + next_hop = portid; + + return next_hop; + + } + + return portid; +} + +/* + * Buffer optimized handling of packets, invoked + * from main_loop. + */ +static inline void +l3fwd_em_send_packets(int nb_rx, struct rte_mbuf **pkts_burst, + uint16_t portid, struct lcore_conf *qconf) +{ + int32_t i, j; + uint16_t dst_port[MAX_PKT_BURST]; + + if (nb_rx > 0) { + rte_prefetch0(rte_pktmbuf_mtod(pkts_burst[0], + struct ether_hdr *) + 1); + } + + for (i = 1, j = 0; j < nb_rx; i++, j++) { + if (i < nb_rx) { + rte_prefetch0(rte_pktmbuf_mtod(pkts_burst[i], + struct ether_hdr *) + 1); + } + dst_port[j] = em_get_dst_port(qconf, pkts_burst[j], portid); + } + + send_packets_multi(qconf, pkts_burst, dst_port, nb_rx); +} +#endif /* __L3FWD_EM_SEQUENTIAL_H__ */ diff --git a/dpdk/examples/l3fwd/l3fwd_em_sse.h b/dpdk/examples/l3fwd/l3fwd_em_sse.h deleted file mode 100644 index c0a9725a..00000000 --- a/dpdk/examples/l3fwd/l3fwd_em_sse.h +++ /dev/null @@ -1,112 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef __L3FWD_EM_SSE_H__ -#define __L3FWD_EM_SSE_H__ - -/** - * @file - * This is an optional implementation of packet classification in Exact-Match - * path using sequential packet classification method. - * While hash lookup multi seems to provide better performance, it's disabled - * by default and can be enabled with NO_HASH_LOOKUP_MULTI global define in - * compilation time. - */ - -#include "l3fwd_sse.h" - -static inline __attribute__((always_inline)) uint16_t -em_get_dst_port(const struct lcore_conf *qconf, struct rte_mbuf *pkt, - uint8_t portid) -{ - uint8_t next_hop; - struct ipv4_hdr *ipv4_hdr; - struct ipv6_hdr *ipv6_hdr; - uint32_t tcp_or_udp; - uint32_t l3_ptypes; - - tcp_or_udp = pkt->packet_type & (RTE_PTYPE_L4_TCP | RTE_PTYPE_L4_UDP); - l3_ptypes = pkt->packet_type & RTE_PTYPE_L3_MASK; - - if (tcp_or_udp && (l3_ptypes == RTE_PTYPE_L3_IPV4)) { - - /* Handle IPv4 headers.*/ - ipv4_hdr = rte_pktmbuf_mtod_offset(pkt, struct ipv4_hdr *, - sizeof(struct ether_hdr)); - - next_hop = em_get_ipv4_dst_port(ipv4_hdr, portid, - qconf->ipv4_lookup_struct); - - if (next_hop >= RTE_MAX_ETHPORTS || - (enabled_port_mask & 1 << next_hop) == 0) - next_hop = portid; - - return next_hop; - - } else if (tcp_or_udp && (l3_ptypes == RTE_PTYPE_L3_IPV6)) { - - /* Handle IPv6 headers.*/ - ipv6_hdr = rte_pktmbuf_mtod_offset(pkt, struct ipv6_hdr *, - sizeof(struct ether_hdr)); - - next_hop = em_get_ipv6_dst_port(ipv6_hdr, portid, - qconf->ipv6_lookup_struct); - - if (next_hop >= RTE_MAX_ETHPORTS || - (enabled_port_mask & 1 << next_hop) == 0) - next_hop = portid; - - return next_hop; - - } - - return portid; -} - -/* - * Buffer optimized handling of packets, invoked - * from main_loop. - */ -static inline void -l3fwd_em_send_packets(int nb_rx, struct rte_mbuf **pkts_burst, - uint8_t portid, struct lcore_conf *qconf) -{ - int32_t j; - uint16_t dst_port[MAX_PKT_BURST]; - - for (j = 0; j < nb_rx; j++) - dst_port[j] = em_get_dst_port(qconf, pkts_burst[j], portid); - - send_packets_multi(qconf, pkts_burst, dst_port, nb_rx); -} -#endif /* __L3FWD_EM_SSE_H__ */ diff --git a/dpdk/examples/l3fwd/l3fwd_lpm.c b/dpdk/examples/l3fwd/l3fwd_lpm.c index d941bdfd..2d0e1724 100644 --- a/dpdk/examples/l3fwd/l3fwd_lpm.c +++ b/dpdk/examples/l3fwd/l3fwd_lpm.c @@ -46,7 +46,6 @@ #include #include #include -#include #include #include #include @@ -105,8 +104,95 @@ static struct ipv6_l3fwd_lpm_route ipv6_l3fwd_lpm_route_array[] = { struct rte_lpm *ipv4_l3fwd_lpm_lookup_struct[NB_SOCKETS]; struct rte_lpm6 *ipv6_l3fwd_lpm_lookup_struct[NB_SOCKETS]; -#if defined(__SSE4_1__) +static inline uint16_t +lpm_get_ipv4_dst_port(void *ipv4_hdr, uint16_t portid, void *lookup_struct) +{ + uint32_t next_hop; + struct rte_lpm *ipv4_l3fwd_lookup_struct = + (struct rte_lpm *)lookup_struct; + + return (uint16_t) ((rte_lpm_lookup(ipv4_l3fwd_lookup_struct, + rte_be_to_cpu_32(((struct ipv4_hdr *)ipv4_hdr)->dst_addr), + &next_hop) == 0) ? next_hop : portid); +} + +static inline uint16_t +lpm_get_ipv6_dst_port(void *ipv6_hdr, uint16_t portid, void *lookup_struct) +{ + uint32_t next_hop; + struct rte_lpm6 *ipv6_l3fwd_lookup_struct = + (struct rte_lpm6 *)lookup_struct; + + return (uint16_t) ((rte_lpm6_lookup(ipv6_l3fwd_lookup_struct, + ((struct ipv6_hdr *)ipv6_hdr)->dst_addr, + &next_hop) == 0) ? next_hop : portid); +} + +static __rte_always_inline uint16_t +lpm_get_dst_port(const struct lcore_conf *qconf, struct rte_mbuf *pkt, + uint16_t portid) +{ + struct ipv6_hdr *ipv6_hdr; + struct ipv4_hdr *ipv4_hdr; + struct ether_hdr *eth_hdr; + + if (RTE_ETH_IS_IPV4_HDR(pkt->packet_type)) { + + eth_hdr = rte_pktmbuf_mtod(pkt, struct ether_hdr *); + ipv4_hdr = (struct ipv4_hdr *)(eth_hdr + 1); + + return lpm_get_ipv4_dst_port(ipv4_hdr, portid, + qconf->ipv4_lookup_struct); + } else if (RTE_ETH_IS_IPV6_HDR(pkt->packet_type)) { + + eth_hdr = rte_pktmbuf_mtod(pkt, struct ether_hdr *); + ipv6_hdr = (struct ipv6_hdr *)(eth_hdr + 1); + + return lpm_get_ipv6_dst_port(ipv6_hdr, portid, + qconf->ipv6_lookup_struct); + } + + return portid; +} + +/* + * lpm_get_dst_port optimized routine for packets where dst_ipv4 is already + * precalculated. If packet is ipv6 dst_addr is taken directly from packet + * header and dst_ipv4 value is not used. + */ +static __rte_always_inline uint16_t +lpm_get_dst_port_with_ipv4(const struct lcore_conf *qconf, struct rte_mbuf *pkt, + uint32_t dst_ipv4, uint16_t portid) +{ + uint32_t next_hop; + struct ipv6_hdr *ipv6_hdr; + struct ether_hdr *eth_hdr; + + if (RTE_ETH_IS_IPV4_HDR(pkt->packet_type)) { + return (uint16_t) ((rte_lpm_lookup(qconf->ipv4_lookup_struct, + dst_ipv4, &next_hop) == 0) + ? next_hop : portid); + + } else if (RTE_ETH_IS_IPV6_HDR(pkt->packet_type)) { + + eth_hdr = rte_pktmbuf_mtod(pkt, struct ether_hdr *); + ipv6_hdr = (struct ipv6_hdr *)(eth_hdr + 1); + + return (uint16_t) ((rte_lpm6_lookup(qconf->ipv6_lookup_struct, + ipv6_hdr->dst_addr, &next_hop) == 0) + ? next_hop : portid); + + } + + return portid; +} + +#if defined(RTE_ARCH_X86) #include "l3fwd_lpm_sse.h" +#elif defined RTE_MACHINE_CPUFLAG_NEON +#include "l3fwd_lpm_neon.h" +#elif defined(RTE_ARCH_PPC_64) +#include "l3fwd_lpm_altivec.h" #else #include "l3fwd_lpm.h" #endif @@ -119,7 +205,8 @@ lpm_main_loop(__attribute__((unused)) void *dummy) unsigned lcore_id; uint64_t prev_tsc, diff_tsc, cur_tsc; int i, nb_rx; - uint8_t portid, queueid; + uint16_t portid; + uint8_t queueid; struct lcore_conf *qconf; const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / US_PER_S * BURST_TX_DRAIN_US; @@ -141,7 +228,7 @@ lpm_main_loop(__attribute__((unused)) void *dummy) portid = qconf->rx_queue_list[i].port_id; queueid = qconf->rx_queue_list[i].queue_id; RTE_LOG(INFO, L3FWD, - " -- lcoreid=%u portid=%hhu rxqueueid=%hhu\n", + " -- lcoreid=%u portid=%u rxqueueid=%hhu\n", lcore_id, portid, queueid); } @@ -179,13 +266,14 @@ lpm_main_loop(__attribute__((unused)) void *dummy) if (nb_rx == 0) continue; -#if defined(__SSE4_1__) +#if defined RTE_ARCH_X86 || defined RTE_MACHINE_CPUFLAG_NEON \ + || defined RTE_ARCH_PPC_64 l3fwd_lpm_send_packets(nb_rx, pkts_burst, portid, qconf); #else l3fwd_lpm_no_opt_send_packets(nb_rx, pkts_burst, portid, qconf); -#endif /* __SSE_4_1__ */ +#endif /* X86 */ } } @@ -329,7 +417,7 @@ lpm_parse_ptype(struct rte_mbuf *m) } uint16_t -lpm_cb_parse_ptype(uint8_t port __rte_unused, uint16_t queue __rte_unused, +lpm_cb_parse_ptype(uint16_t port __rte_unused, uint16_t queue __rte_unused, struct rte_mbuf *pkts[], uint16_t nb_pkts, uint16_t max_pkts __rte_unused, void *user_param __rte_unused) diff --git a/dpdk/examples/l3fwd/l3fwd_lpm.h b/dpdk/examples/l3fwd/l3fwd_lpm.h index a43c5070..53b7fc80 100644 --- a/dpdk/examples/l3fwd/l3fwd_lpm.h +++ b/dpdk/examples/l3fwd/l3fwd_lpm.h @@ -34,37 +34,13 @@ #ifndef __L3FWD_LPM_H__ #define __L3FWD_LPM_H__ -static inline uint8_t -lpm_get_ipv4_dst_port(void *ipv4_hdr, uint8_t portid, void *lookup_struct) -{ - uint32_t next_hop; - struct rte_lpm *ipv4_l3fwd_lookup_struct = - (struct rte_lpm *)lookup_struct; - - return (uint8_t) ((rte_lpm_lookup(ipv4_l3fwd_lookup_struct, - rte_be_to_cpu_32(((struct ipv4_hdr *)ipv4_hdr)->dst_addr), - &next_hop) == 0) ? next_hop : portid); -} - -static inline uint8_t -lpm_get_ipv6_dst_port(void *ipv6_hdr, uint8_t portid, void *lookup_struct) -{ - uint8_t next_hop; - struct rte_lpm6 *ipv6_l3fwd_lookup_struct = - (struct rte_lpm6 *)lookup_struct; - - return (uint8_t) ((rte_lpm6_lookup(ipv6_l3fwd_lookup_struct, - ((struct ipv6_hdr *)ipv6_hdr)->dst_addr, - &next_hop) == 0) ? next_hop : portid); -} - -static inline __attribute__((always_inline)) void -l3fwd_lpm_simple_forward(struct rte_mbuf *m, uint8_t portid, +static __rte_always_inline void +l3fwd_lpm_simple_forward(struct rte_mbuf *m, uint16_t portid, struct lcore_conf *qconf) { struct ether_hdr *eth_hdr; struct ipv4_hdr *ipv4_hdr; - uint8_t dst_port; + uint16_t dst_port; eth_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *); @@ -128,7 +104,7 @@ l3fwd_lpm_simple_forward(struct rte_mbuf *m, uint8_t portid, static inline void l3fwd_lpm_no_opt_send_packets(int nb_rx, struct rte_mbuf **pkts_burst, - uint8_t portid, struct lcore_conf *qconf) + uint16_t portid, struct lcore_conf *qconf) { int32_t j; diff --git a/dpdk/examples/l3fwd/l3fwd_lpm_altivec.h b/dpdk/examples/l3fwd/l3fwd_lpm_altivec.h new file mode 100644 index 00000000..36ca983f --- /dev/null +++ b/dpdk/examples/l3fwd/l3fwd_lpm_altivec.h @@ -0,0 +1,164 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. + * Copyright(c) 2017 IBM Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __L3FWD_LPM_ALTIVEC_H__ +#define __L3FWD_LPM_ALTIVEC_H__ + +#include "l3fwd_altivec.h" + +/* + * Read packet_type and destination IPV4 addresses from 4 mbufs. + */ +static inline void +processx4_step1(struct rte_mbuf *pkt[FWDSTEP], + vector unsigned int *dip, + uint32_t *ipv4_flag) +{ + struct ipv4_hdr *ipv4_hdr; + struct ether_hdr *eth_hdr; + uint32_t x0, x1, x2, x3; + + eth_hdr = rte_pktmbuf_mtod(pkt[0], struct ether_hdr *); + ipv4_hdr = (struct ipv4_hdr *)(eth_hdr + 1); + x0 = ipv4_hdr->dst_addr; + ipv4_flag[0] = pkt[0]->packet_type & RTE_PTYPE_L3_IPV4; + + rte_compiler_barrier(); + eth_hdr = rte_pktmbuf_mtod(pkt[1], struct ether_hdr *); + ipv4_hdr = (struct ipv4_hdr *)(eth_hdr + 1); + x1 = ipv4_hdr->dst_addr; + ipv4_flag[0] &= pkt[1]->packet_type; + + rte_compiler_barrier(); + eth_hdr = rte_pktmbuf_mtod(pkt[2], struct ether_hdr *); + ipv4_hdr = (struct ipv4_hdr *)(eth_hdr + 1); + x2 = ipv4_hdr->dst_addr; + ipv4_flag[0] &= pkt[2]->packet_type; + + rte_compiler_barrier(); + eth_hdr = rte_pktmbuf_mtod(pkt[3], struct ether_hdr *); + ipv4_hdr = (struct ipv4_hdr *)(eth_hdr + 1); + x3 = ipv4_hdr->dst_addr; + ipv4_flag[0] &= pkt[3]->packet_type; + + rte_compiler_barrier(); + dip[0] = (vector unsigned int){x0, x1, x2, x3}; +} + +/* + * Lookup into LPM for destination port. + * If lookup fails, use incoming port (portid) as destination port. + */ +static inline void +processx4_step2(const struct lcore_conf *qconf, + vector unsigned int dip, + uint32_t ipv4_flag, + uint8_t portid, + struct rte_mbuf *pkt[FWDSTEP], + uint16_t dprt[FWDSTEP]) +{ + rte_xmm_t dst; + const vector unsigned char bswap_mask = (vector unsigned char){ + 3, 2, 1, 0, + 7, 6, 5, 4, + 11, 10, 9, 8, + 15, 14, 13, 12}; + + /* Byte swap 4 IPV4 addresses. */ + dip = (vector unsigned int)vec_perm(*(vector unsigned char *)&dip, + (vector unsigned char){}, bswap_mask); + + /* if all 4 packets are IPV4. */ + if (likely(ipv4_flag)) { + rte_lpm_lookupx4(qconf->ipv4_lookup_struct, (xmm_t)dip, + (uint32_t *)&dst, portid); + /* get rid of unused upper 16 bit for each dport. */ + dst.x = (xmm_t)vec_packs(dst.x, dst.x); + *(uint64_t *)dprt = dst.u64[0]; + } else { + dst.x = (xmm_t)dip; + dprt[0] = lpm_get_dst_port_with_ipv4(qconf, pkt[0], + dst.u32[0], portid); + dprt[1] = lpm_get_dst_port_with_ipv4(qconf, pkt[1], + dst.u32[1], portid); + dprt[2] = lpm_get_dst_port_with_ipv4(qconf, pkt[2], + dst.u32[2], portid); + dprt[3] = lpm_get_dst_port_with_ipv4(qconf, pkt[3], + dst.u32[3], portid); + } +} + +/* + * Buffer optimized handling of packets, invoked + * from main_loop. + */ +static inline void +l3fwd_lpm_send_packets(int nb_rx, struct rte_mbuf **pkts_burst, + uint8_t portid, struct lcore_conf *qconf) +{ + int32_t j; + uint16_t dst_port[MAX_PKT_BURST]; + vector unsigned int dip[MAX_PKT_BURST / FWDSTEP]; + uint32_t ipv4_flag[MAX_PKT_BURST / FWDSTEP]; + const int32_t k = RTE_ALIGN_FLOOR(nb_rx, FWDSTEP); + + for (j = 0; j != k; j += FWDSTEP) + processx4_step1(&pkts_burst[j], &dip[j / FWDSTEP], + &ipv4_flag[j / FWDSTEP]); + + for (j = 0; j != k; j += FWDSTEP) + processx4_step2(qconf, dip[j / FWDSTEP], + ipv4_flag[j / FWDSTEP], + portid, &pkts_burst[j], &dst_port[j]); + + /* Classify last up to 3 packets one by one */ + switch (nb_rx % FWDSTEP) { + case 3: + dst_port[j] = lpm_get_dst_port(qconf, pkts_burst[j], portid); + j++; + /* fall-through */ + case 2: + dst_port[j] = lpm_get_dst_port(qconf, pkts_burst[j], portid); + j++; + /* fall-through */ + case 1: + dst_port[j] = lpm_get_dst_port(qconf, pkts_burst[j], portid); + j++; + /* fall-through */ + } + + send_packets_multi(qconf, pkts_burst, dst_port, nb_rx); +} + +#endif /* __L3FWD_LPM_ALTIVEC_H__ */ diff --git a/dpdk/examples/l3fwd/l3fwd_lpm_neon.h b/dpdk/examples/l3fwd/l3fwd_lpm_neon.h new file mode 100644 index 00000000..85f314d1 --- /dev/null +++ b/dpdk/examples/l3fwd/l3fwd_lpm_neon.h @@ -0,0 +1,193 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. + * Copyright(c) 2017, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __L3FWD_LPM_NEON_H__ +#define __L3FWD_LPM_NEON_H__ + +#include + +#include "l3fwd_neon.h" + +/* + * Read packet_type and destination IPV4 addresses from 4 mbufs. + */ +static inline void +processx4_step1(struct rte_mbuf *pkt[FWDSTEP], + int32x4_t *dip, + uint32_t *ipv4_flag) +{ + struct ipv4_hdr *ipv4_hdr; + struct ether_hdr *eth_hdr; + int32_t dst[FWDSTEP]; + + eth_hdr = rte_pktmbuf_mtod(pkt[0], struct ether_hdr *); + ipv4_hdr = (struct ipv4_hdr *)(eth_hdr + 1); + dst[0] = ipv4_hdr->dst_addr; + ipv4_flag[0] = pkt[0]->packet_type & RTE_PTYPE_L3_IPV4; + + eth_hdr = rte_pktmbuf_mtod(pkt[1], struct ether_hdr *); + ipv4_hdr = (struct ipv4_hdr *)(eth_hdr + 1); + dst[1] = ipv4_hdr->dst_addr; + ipv4_flag[0] &= pkt[1]->packet_type; + + eth_hdr = rte_pktmbuf_mtod(pkt[2], struct ether_hdr *); + ipv4_hdr = (struct ipv4_hdr *)(eth_hdr + 1); + dst[2] = ipv4_hdr->dst_addr; + ipv4_flag[0] &= pkt[2]->packet_type; + + eth_hdr = rte_pktmbuf_mtod(pkt[3], struct ether_hdr *); + ipv4_hdr = (struct ipv4_hdr *)(eth_hdr + 1); + dst[3] = ipv4_hdr->dst_addr; + ipv4_flag[0] &= pkt[3]->packet_type; + + dip[0] = vld1q_s32(dst); +} + +/* + * Lookup into LPM for destination port. + * If lookup fails, use incoming port (portid) as destination port. + */ +static inline void +processx4_step2(const struct lcore_conf *qconf, + int32x4_t dip, + uint32_t ipv4_flag, + uint16_t portid, + struct rte_mbuf *pkt[FWDSTEP], + uint16_t dprt[FWDSTEP]) +{ + rte_xmm_t dst; + + dip = vreinterpretq_s32_u8(vrev32q_u8(vreinterpretq_u8_s32(dip))); + + /* if all 4 packets are IPV4. */ + if (likely(ipv4_flag)) { + rte_lpm_lookupx4(qconf->ipv4_lookup_struct, dip, dst.u32, + portid); + /* get rid of unused upper 16 bit for each dport. */ + vst1_s16((int16_t *)dprt, vqmovn_s32(dst.x)); + } else { + dst.x = dip; + dprt[0] = lpm_get_dst_port_with_ipv4(qconf, pkt[0], + dst.u32[0], portid); + dprt[1] = lpm_get_dst_port_with_ipv4(qconf, pkt[1], + dst.u32[1], portid); + dprt[2] = lpm_get_dst_port_with_ipv4(qconf, pkt[2], + dst.u32[2], portid); + dprt[3] = lpm_get_dst_port_with_ipv4(qconf, pkt[3], + dst.u32[3], portid); + } +} + +/* + * Buffer optimized handling of packets, invoked + * from main_loop. + */ +static inline void +l3fwd_lpm_send_packets(int nb_rx, struct rte_mbuf **pkts_burst, + uint16_t portid, struct lcore_conf *qconf) +{ + int32_t i = 0, j = 0; + uint16_t dst_port[MAX_PKT_BURST]; + int32x4_t dip; + uint32_t ipv4_flag; + const int32_t k = RTE_ALIGN_FLOOR(nb_rx, FWDSTEP); + const int32_t m = nb_rx % FWDSTEP; + + if (k) { + for (i = 0; i < FWDSTEP; i++) { + rte_prefetch0(rte_pktmbuf_mtod(pkts_burst[i], + struct ether_hdr *) + 1); + } + + for (j = 0; j != k - FWDSTEP; j += FWDSTEP) { + for (i = 0; i < FWDSTEP; i++) { + rte_prefetch0(rte_pktmbuf_mtod( + pkts_burst[j + i + FWDSTEP], + struct ether_hdr *) + 1); + } + + processx4_step1(&pkts_burst[j], &dip, &ipv4_flag); + processx4_step2(qconf, dip, ipv4_flag, portid, + &pkts_burst[j], &dst_port[j]); + } + + processx4_step1(&pkts_burst[j], &dip, &ipv4_flag); + processx4_step2(qconf, dip, ipv4_flag, portid, &pkts_burst[j], + &dst_port[j]); + + j += FWDSTEP; + } + + if (m) { + /* Prefetch last up to 3 packets one by one */ + switch (m) { + case 3: + rte_prefetch0(rte_pktmbuf_mtod(pkts_burst[j], + struct ether_hdr *) + 1); + j++; + /* fallthrough */ + case 2: + rte_prefetch0(rte_pktmbuf_mtod(pkts_burst[j], + struct ether_hdr *) + 1); + j++; + /* fallthrough */ + case 1: + rte_prefetch0(rte_pktmbuf_mtod(pkts_burst[j], + struct ether_hdr *) + 1); + j++; + } + + j -= m; + /* Classify last up to 3 packets one by one */ + switch (m) { + case 3: + dst_port[j] = lpm_get_dst_port(qconf, pkts_burst[j], + portid); + j++; + /* fallthrough */ + case 2: + dst_port[j] = lpm_get_dst_port(qconf, pkts_burst[j], + portid); + j++; + /* fallthrough */ + case 1: + dst_port[j] = lpm_get_dst_port(qconf, pkts_burst[j], + portid); + } + } + + send_packets_multi(qconf, pkts_burst, dst_port, nb_rx); +} + +#endif /* __L3FWD_LPM_NEON_H__ */ diff --git a/dpdk/examples/l3fwd/l3fwd_lpm_sse.h b/dpdk/examples/l3fwd/l3fwd_lpm_sse.h index 538fe3d7..d474396e 100644 --- a/dpdk/examples/l3fwd/l3fwd_lpm_sse.h +++ b/dpdk/examples/l3fwd/l3fwd_lpm_sse.h @@ -36,72 +36,6 @@ #include "l3fwd_sse.h" -static inline __attribute__((always_inline)) uint16_t -lpm_get_dst_port(const struct lcore_conf *qconf, struct rte_mbuf *pkt, - uint8_t portid) -{ - uint32_t next_hop_ipv4; - uint8_t next_hop_ipv6; - struct ipv6_hdr *ipv6_hdr; - struct ipv4_hdr *ipv4_hdr; - struct ether_hdr *eth_hdr; - - if (RTE_ETH_IS_IPV4_HDR(pkt->packet_type)) { - - eth_hdr = rte_pktmbuf_mtod(pkt, struct ether_hdr *); - ipv4_hdr = (struct ipv4_hdr *)(eth_hdr + 1); - - return (uint16_t) ((rte_lpm_lookup(qconf->ipv4_lookup_struct, - rte_be_to_cpu_32(ipv4_hdr->dst_addr), &next_hop_ipv4) == 0) ? - next_hop_ipv4 : portid); - - } else if (RTE_ETH_IS_IPV6_HDR(pkt->packet_type)) { - - eth_hdr = rte_pktmbuf_mtod(pkt, struct ether_hdr *); - ipv6_hdr = (struct ipv6_hdr *)(eth_hdr + 1); - - return (uint16_t) ((rte_lpm6_lookup(qconf->ipv6_lookup_struct, - ipv6_hdr->dst_addr, &next_hop_ipv6) == 0) - ? next_hop_ipv6 : portid); - - } - - return portid; -} - -/* - * lpm_get_dst_port optimized routine for packets where dst_ipv4 is already - * precalculated. If packet is ipv6 dst_addr is taken directly from packet - * header and dst_ipv4 value is not used. - */ -static inline __attribute__((always_inline)) uint16_t -lpm_get_dst_port_with_ipv4(const struct lcore_conf *qconf, struct rte_mbuf *pkt, - uint32_t dst_ipv4, uint8_t portid) -{ - uint32_t next_hop_ipv4; - uint8_t next_hop_ipv6; - struct ipv6_hdr *ipv6_hdr; - struct ether_hdr *eth_hdr; - - if (RTE_ETH_IS_IPV4_HDR(pkt->packet_type)) { - return (uint16_t) ((rte_lpm_lookup(qconf->ipv4_lookup_struct, dst_ipv4, - &next_hop_ipv4) == 0) ? next_hop_ipv4 : portid); - - } else if (RTE_ETH_IS_IPV6_HDR(pkt->packet_type)) { - - eth_hdr = rte_pktmbuf_mtod(pkt, struct ether_hdr *); - ipv6_hdr = (struct ipv6_hdr *)(eth_hdr + 1); - - return (uint16_t) ((rte_lpm6_lookup(qconf->ipv6_lookup_struct, - ipv6_hdr->dst_addr, &next_hop_ipv6) == 0) - ? next_hop_ipv6 : portid); - - } - - return portid; - -} - /* * Read packet_type and destination IPV4 addresses from 4 mbufs. */ @@ -145,7 +79,7 @@ static inline void processx4_step2(const struct lcore_conf *qconf, __m128i dip, uint32_t ipv4_flag, - uint8_t portid, + uint16_t portid, struct rte_mbuf *pkt[FWDSTEP], uint16_t dprt[FWDSTEP]) { @@ -178,7 +112,7 @@ processx4_step2(const struct lcore_conf *qconf, */ static inline void l3fwd_lpm_send_packets(int nb_rx, struct rte_mbuf **pkts_burst, - uint8_t portid, struct lcore_conf *qconf) + uint16_t portid, struct lcore_conf *qconf) { int32_t j; uint16_t dst_port[MAX_PKT_BURST]; @@ -199,9 +133,11 @@ l3fwd_lpm_send_packets(int nb_rx, struct rte_mbuf **pkts_burst, case 3: dst_port[j] = lpm_get_dst_port(qconf, pkts_burst[j], portid); j++; + /* fall-through */ case 2: dst_port[j] = lpm_get_dst_port(qconf, pkts_burst[j], portid); j++; + /* fall-through */ case 1: dst_port[j] = lpm_get_dst_port(qconf, pkts_burst[j], portid); j++; diff --git a/dpdk/examples/l3fwd/l3fwd_neon.h b/dpdk/examples/l3fwd/l3fwd_neon.h new file mode 100644 index 00000000..b319b5a9 --- /dev/null +++ b/dpdk/examples/l3fwd/l3fwd_neon.h @@ -0,0 +1,260 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2017, Linaro Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + + +#ifndef _L3FWD_NEON_H_ +#define _L3FWD_NEON_H_ + +#include "l3fwd.h" +#include "l3fwd_common.h" + +/* + * Update source and destination MAC addresses in the ethernet header. + * Perform RFC1812 checks and updates for IPV4 packets. + */ +static inline void +processx4_step3(struct rte_mbuf *pkt[FWDSTEP], uint16_t dst_port[FWDSTEP]) +{ + uint32x4_t te[FWDSTEP]; + uint32x4_t ve[FWDSTEP]; + uint32_t *p[FWDSTEP]; + + p[0] = rte_pktmbuf_mtod(pkt[0], uint32_t *); + p[1] = rte_pktmbuf_mtod(pkt[1], uint32_t *); + p[2] = rte_pktmbuf_mtod(pkt[2], uint32_t *); + p[3] = rte_pktmbuf_mtod(pkt[3], uint32_t *); + + ve[0] = vreinterpretq_u32_s32(val_eth[dst_port[0]]); + te[0] = vld1q_u32(p[0]); + + ve[1] = vreinterpretq_u32_s32(val_eth[dst_port[1]]); + te[1] = vld1q_u32(p[1]); + + ve[2] = vreinterpretq_u32_s32(val_eth[dst_port[2]]); + te[2] = vld1q_u32(p[2]); + + ve[3] = vreinterpretq_u32_s32(val_eth[dst_port[3]]); + te[3] = vld1q_u32(p[3]); + + /* Update last 4 bytes */ + ve[0] = vsetq_lane_u32(vgetq_lane_u32(te[0], 3), ve[0], 3); + ve[1] = vsetq_lane_u32(vgetq_lane_u32(te[1], 3), ve[1], 3); + ve[2] = vsetq_lane_u32(vgetq_lane_u32(te[2], 3), ve[2], 3); + ve[3] = vsetq_lane_u32(vgetq_lane_u32(te[3], 3), ve[3], 3); + + vst1q_u32(p[0], ve[0]); + vst1q_u32(p[1], ve[1]); + vst1q_u32(p[2], ve[2]); + vst1q_u32(p[3], ve[3]); + + rfc1812_process((struct ipv4_hdr *)((struct ether_hdr *)p[0] + 1), + &dst_port[0], pkt[0]->packet_type); + rfc1812_process((struct ipv4_hdr *)((struct ether_hdr *)p[1] + 1), + &dst_port[1], pkt[1]->packet_type); + rfc1812_process((struct ipv4_hdr *)((struct ether_hdr *)p[2] + 1), + &dst_port[2], pkt[2]->packet_type); + rfc1812_process((struct ipv4_hdr *)((struct ether_hdr *)p[3] + 1), + &dst_port[3], pkt[3]->packet_type); +} + +/* + * Group consecutive packets with the same destination port in bursts of 4. + * Suppose we have array of destionation ports: + * dst_port[] = {a, b, c, d,, e, ... } + * dp1 should contain: , dp2: . + * We doing 4 comparisons at once and the result is 4 bit mask. + * This mask is used as an index into prebuild array of pnum values. + */ +static inline uint16_t * +port_groupx4(uint16_t pn[FWDSTEP + 1], uint16_t *lp, uint16x8_t dp1, + uint16x8_t dp2) +{ + union { + uint16_t u16[FWDSTEP + 1]; + uint64_t u64; + } *pnum = (void *)pn; + + int32_t v; + uint16x8_t mask = {1, 2, 4, 8, 0, 0, 0, 0}; + + dp1 = vceqq_u16(dp1, dp2); + dp1 = vandq_u16(dp1, mask); + v = vaddvq_u16(dp1); + + /* update last port counter. */ + lp[0] += gptbl[v].lpv; + rte_compiler_barrier(); + + /* if dest port value has changed. */ + if (v != GRPMSK) { + pnum->u64 = gptbl[v].pnum; + pnum->u16[FWDSTEP] = 1; + lp = pnum->u16 + gptbl[v].idx; + } + + return lp; +} + +/** + * Process one packet: + * Update source and destination MAC addresses in the ethernet header. + * Perform RFC1812 checks and updates for IPV4 packets. + */ +static inline void +process_packet(struct rte_mbuf *pkt, uint16_t *dst_port) +{ + struct ether_hdr *eth_hdr; + uint32x4_t te, ve; + + eth_hdr = rte_pktmbuf_mtod(pkt, struct ether_hdr *); + + te = vld1q_u32((uint32_t *)eth_hdr); + ve = vreinterpretq_u32_s32(val_eth[dst_port[0]]); + + + rfc1812_process((struct ipv4_hdr *)(eth_hdr + 1), dst_port, + pkt->packet_type); + + ve = vcopyq_laneq_u32(ve, 3, te, 3); + vst1q_u32((uint32_t *)eth_hdr, ve); +} + +/** + * Send packets burst from pkts_burst to the ports in dst_port array + */ +static __rte_always_inline void +send_packets_multi(struct lcore_conf *qconf, struct rte_mbuf **pkts_burst, + uint16_t dst_port[MAX_PKT_BURST], int nb_rx) +{ + int32_t k; + int j = 0; + uint16_t dlp; + uint16_t *lp; + uint16_t pnum[MAX_PKT_BURST + 1]; + + /* + * Finish packet processing and group consecutive + * packets with the same destination port. + */ + k = RTE_ALIGN_FLOOR(nb_rx, FWDSTEP); + if (k != 0) { + uint16x8_t dp1, dp2; + + lp = pnum; + lp[0] = 1; + + processx4_step3(pkts_burst, dst_port); + + /* dp1: */ + dp1 = vld1q_u16(dst_port); + + for (j = FWDSTEP; j != k; j += FWDSTEP) { + processx4_step3(&pkts_burst[j], &dst_port[j]); + + /* + * dp2: + * + */ + dp2 = vld1q_u16(&dst_port[j - FWDSTEP + 1]); + lp = port_groupx4(&pnum[j - FWDSTEP], lp, dp1, dp2); + + /* + * dp1: + * + */ + dp1 = vextq_u16(dp2, dp1, FWDSTEP - 1); + } + + /* + * dp2: + */ + dp2 = vextq_u16(dp1, dp1, 1); + dp2 = vsetq_lane_u16(vgetq_lane_u16(dp2, 2), dp2, 3); + lp = port_groupx4(&pnum[j - FWDSTEP], lp, dp1, dp2); + + /* + * remove values added by the last repeated + * dst port. + */ + lp[0]--; + dlp = dst_port[j - 1]; + } else { + /* set dlp and lp to the never used values. */ + dlp = BAD_PORT - 1; + lp = pnum + MAX_PKT_BURST; + } + + /* Process up to last 3 packets one by one. */ + switch (nb_rx % FWDSTEP) { + case 3: + process_packet(pkts_burst[j], dst_port + j); + GROUP_PORT_STEP(dlp, dst_port, lp, pnum, j); + j++; + /* fallthrough */ + case 2: + process_packet(pkts_burst[j], dst_port + j); + GROUP_PORT_STEP(dlp, dst_port, lp, pnum, j); + j++; + /* fallthrough */ + case 1: + process_packet(pkts_burst[j], dst_port + j); + GROUP_PORT_STEP(dlp, dst_port, lp, pnum, j); + j++; + } + + /* + * Send packets out, through destination port. + * Consecutive packets with the same destination port + * are already grouped together. + * If destination port for the packet equals BAD_PORT, + * then free the packet without sending it out. + */ + for (j = 0; j < nb_rx; j += k) { + + int32_t m; + uint16_t pn; + + pn = dst_port[j]; + k = pnum[j]; + + if (likely(pn != BAD_PORT)) + send_packetsx4(qconf, pn, pkts_burst + j, k); + else + for (m = j; m != j + k; m++) + rte_pktmbuf_free(pkts_burst[m]); + + } +} + +#endif /* _L3FWD_NEON_H_ */ diff --git a/dpdk/examples/l3fwd/l3fwd_sse.h b/dpdk/examples/l3fwd/l3fwd_sse.h index 1afa1f00..831760f0 100644 --- a/dpdk/examples/l3fwd/l3fwd_sse.h +++ b/dpdk/examples/l3fwd/l3fwd_sse.h @@ -32,53 +32,11 @@ */ -#ifndef _L3FWD_COMMON_H_ -#define _L3FWD_COMMON_H_ +#ifndef _L3FWD_SSE_H_ +#define _L3FWD_SSE_H_ #include "l3fwd.h" - -#ifdef DO_RFC_1812_CHECKS - -#define IPV4_MIN_VER_IHL 0x45 -#define IPV4_MAX_VER_IHL 0x4f -#define IPV4_MAX_VER_IHL_DIFF (IPV4_MAX_VER_IHL - IPV4_MIN_VER_IHL) - -/* Minimum value of IPV4 total length (20B) in network byte order. */ -#define IPV4_MIN_LEN_BE (sizeof(struct ipv4_hdr) << 8) - -/* - * From http://www.rfc-editor.org/rfc/rfc1812.txt section 5.2.2: - * - The IP version number must be 4. - * - The IP header length field must be large enough to hold the - * minimum length legal IP datagram (20 bytes = 5 words). - * - The IP total length field must be large enough to hold the IP - * datagram header, whose length is specified in the IP header length - * field. - * If we encounter invalid IPV4 packet, then set destination port for it - * to BAD_PORT value. - */ -static inline __attribute__((always_inline)) void -rfc1812_process(struct ipv4_hdr *ipv4_hdr, uint16_t *dp, uint32_t ptype) -{ - uint8_t ihl; - - if (RTE_ETH_IS_IPV4_HDR(ptype)) { - ihl = ipv4_hdr->version_ihl - IPV4_MIN_VER_IHL; - - ipv4_hdr->time_to_live--; - ipv4_hdr->hdr_checksum++; - - if (ihl > IPV4_MAX_VER_IHL_DIFF || - ((uint8_t)ipv4_hdr->total_length == 0 && - ipv4_hdr->total_length < IPV4_MIN_LEN_BE)) - dp[0] = BAD_PORT; - - } -} - -#else -#define rfc1812_process(mb, dp, ptype) do { } while (0) -#endif /* DO_RFC_1812_CHECKS */ +#include "l3fwd_common.h" /* * Update source and destination MAC addresses in the ethernet header. @@ -129,144 +87,17 @@ processx4_step3(struct rte_mbuf *pkt[FWDSTEP], uint16_t dst_port[FWDSTEP]) &dst_port[3], pkt[3]->packet_type); } -/* - * We group consecutive packets with the same destionation port into one burst. - * To avoid extra latency this is done together with some other packet - * processing, but after we made a final decision about packet's destination. - * To do this we maintain: - * pnum - array of number of consecutive packets with the same dest port for - * each packet in the input burst. - * lp - pointer to the last updated element in the pnum. - * dlp - dest port value lp corresponds to. - */ - -#define GRPSZ (1 << FWDSTEP) -#define GRPMSK (GRPSZ - 1) - -#define GROUP_PORT_STEP(dlp, dcp, lp, pn, idx) do { \ - if (likely((dlp) == (dcp)[(idx)])) { \ - (lp)[0]++; \ - } else { \ - (dlp) = (dcp)[idx]; \ - (lp) = (pn) + (idx); \ - (lp)[0] = 1; \ - } \ -} while (0) - /* * Group consecutive packets with the same destination port in bursts of 4. * Suppose we have array of destionation ports: * dst_port[] = {a, b, c, d,, e, ... } * dp1 should contain: , dp2: . - * We doing 4 comparisions at once and the result is 4 bit mask. + * We doing 4 comparisons at once and the result is 4 bit mask. * This mask is used as an index into prebuild array of pnum values. */ static inline uint16_t * port_groupx4(uint16_t pn[FWDSTEP + 1], uint16_t *lp, __m128i dp1, __m128i dp2) { - static const struct { - uint64_t pnum; /* prebuild 4 values for pnum[]. */ - int32_t idx; /* index for new last updated elemnet. */ - uint16_t lpv; /* add value to the last updated element. */ - } gptbl[GRPSZ] = { - { - /* 0: a != b, b != c, c != d, d != e */ - .pnum = UINT64_C(0x0001000100010001), - .idx = 4, - .lpv = 0, - }, - { - /* 1: a == b, b != c, c != d, d != e */ - .pnum = UINT64_C(0x0001000100010002), - .idx = 4, - .lpv = 1, - }, - { - /* 2: a != b, b == c, c != d, d != e */ - .pnum = UINT64_C(0x0001000100020001), - .idx = 4, - .lpv = 0, - }, - { - /* 3: a == b, b == c, c != d, d != e */ - .pnum = UINT64_C(0x0001000100020003), - .idx = 4, - .lpv = 2, - }, - { - /* 4: a != b, b != c, c == d, d != e */ - .pnum = UINT64_C(0x0001000200010001), - .idx = 4, - .lpv = 0, - }, - { - /* 5: a == b, b != c, c == d, d != e */ - .pnum = UINT64_C(0x0001000200010002), - .idx = 4, - .lpv = 1, - }, - { - /* 6: a != b, b == c, c == d, d != e */ - .pnum = UINT64_C(0x0001000200030001), - .idx = 4, - .lpv = 0, - }, - { - /* 7: a == b, b == c, c == d, d != e */ - .pnum = UINT64_C(0x0001000200030004), - .idx = 4, - .lpv = 3, - }, - { - /* 8: a != b, b != c, c != d, d == e */ - .pnum = UINT64_C(0x0002000100010001), - .idx = 3, - .lpv = 0, - }, - { - /* 9: a == b, b != c, c != d, d == e */ - .pnum = UINT64_C(0x0002000100010002), - .idx = 3, - .lpv = 1, - }, - { - /* 0xa: a != b, b == c, c != d, d == e */ - .pnum = UINT64_C(0x0002000100020001), - .idx = 3, - .lpv = 0, - }, - { - /* 0xb: a == b, b == c, c != d, d == e */ - .pnum = UINT64_C(0x0002000100020003), - .idx = 3, - .lpv = 2, - }, - { - /* 0xc: a != b, b != c, c == d, d == e */ - .pnum = UINT64_C(0x0002000300010001), - .idx = 2, - .lpv = 0, - }, - { - /* 0xd: a == b, b != c, c == d, d == e */ - .pnum = UINT64_C(0x0002000300010002), - .idx = 2, - .lpv = 1, - }, - { - /* 0xe: a != b, b == c, c == d, d == e */ - .pnum = UINT64_C(0x0002000300040001), - .idx = 1, - .lpv = 0, - }, - { - /* 0xf: a == b, b == c, c == d, d == e */ - .pnum = UINT64_C(0x0002000300040005), - .idx = 0, - .lpv = 4, - }, - }; - union { uint16_t u16[FWDSTEP + 1]; uint64_t u64; @@ -314,88 +145,10 @@ process_packet(struct rte_mbuf *pkt, uint16_t *dst_port) _mm_storeu_si128((__m128i *)eth_hdr, te); } -static inline __attribute__((always_inline)) void -send_packetsx4(struct lcore_conf *qconf, uint8_t port, struct rte_mbuf *m[], - uint32_t num) -{ - uint32_t len, j, n; - - len = qconf->tx_mbufs[port].len; - - /* - * If TX buffer for that queue is empty, and we have enough packets, - * then send them straightway. - */ - if (num >= MAX_TX_BURST && len == 0) { - n = rte_eth_tx_burst(port, qconf->tx_queue_id[port], m, num); - if (unlikely(n < num)) { - do { - rte_pktmbuf_free(m[n]); - } while (++n < num); - } - return; - } - - /* - * Put packets into TX buffer for that queue. - */ - - n = len + num; - n = (n > MAX_PKT_BURST) ? MAX_PKT_BURST - len : num; - - j = 0; - switch (n % FWDSTEP) { - while (j < n) { - case 0: - qconf->tx_mbufs[port].m_table[len + j] = m[j]; - j++; - case 3: - qconf->tx_mbufs[port].m_table[len + j] = m[j]; - j++; - case 2: - qconf->tx_mbufs[port].m_table[len + j] = m[j]; - j++; - case 1: - qconf->tx_mbufs[port].m_table[len + j] = m[j]; - j++; - } - } - - len += n; - - /* enough pkts to be sent */ - if (unlikely(len == MAX_PKT_BURST)) { - - send_burst(qconf, MAX_PKT_BURST, port); - - /* copy rest of the packets into the TX buffer. */ - len = num - n; - j = 0; - switch (len % FWDSTEP) { - while (j < len) { - case 0: - qconf->tx_mbufs[port].m_table[j] = m[n + j]; - j++; - case 3: - qconf->tx_mbufs[port].m_table[j] = m[n + j]; - j++; - case 2: - qconf->tx_mbufs[port].m_table[j] = m[n + j]; - j++; - case 1: - qconf->tx_mbufs[port].m_table[j] = m[n + j]; - j++; - } - } - } - - qconf->tx_mbufs[port].len = len; -} - /** * Send packets burst from pkts_burst to the ports in dst_port array */ -static inline __attribute__((always_inline)) void +static __rte_always_inline void send_packets_multi(struct lcore_conf *qconf, struct rte_mbuf **pkts_burst, uint16_t dst_port[MAX_PKT_BURST], int nb_rx) { @@ -464,10 +217,12 @@ send_packets_multi(struct lcore_conf *qconf, struct rte_mbuf **pkts_burst, process_packet(pkts_burst[j], dst_port + j); GROUP_PORT_STEP(dlp, dst_port, lp, pnum, j); j++; + /* fall-through */ case 2: process_packet(pkts_burst[j], dst_port + j); GROUP_PORT_STEP(dlp, dst_port, lp, pnum, j); j++; + /* fall-through */ case 1: process_packet(pkts_burst[j], dst_port + j); GROUP_PORT_STEP(dlp, dst_port, lp, pnum, j); @@ -498,4 +253,4 @@ send_packets_multi(struct lcore_conf *qconf, struct rte_mbuf **pkts_burst, } } -#endif /* _L3FWD_COMMON_H_ */ +#endif /* _L3FWD_SSE_H_ */ diff --git a/dpdk/examples/l3fwd/main.c b/dpdk/examples/l3fwd/main.c index acedd20e..6229568f 100644 --- a/dpdk/examples/l3fwd/main.c +++ b/dpdk/examples/l3fwd/main.c @@ -50,9 +50,7 @@ #include #include #include -#include #include -#include #include #include #include @@ -61,12 +59,10 @@ #include #include #include -#include #include #include #include #include -#include #include #include #include @@ -126,7 +122,7 @@ uint32_t hash_entry_number = HASH_ENTRY_NUMBER_DEFAULT; struct lcore_conf lcore_conf[RTE_MAX_LCORE]; struct lcore_params { - uint8_t port_id; + uint16_t port_id; uint8_t queue_id; uint8_t lcore_id; } __rte_cache_aligned; @@ -157,7 +153,7 @@ static struct rte_eth_conf port_conf = { .hw_ip_checksum = 1, /**< IP checksum offload enabled */ .hw_vlan_filter = 0, /**< VLAN filtering disabled */ .jumbo_frame = 0, /**< Jumbo Frame Support disabled */ - .hw_strip_crc = 0, /**< CRC stripped by hardware */ + .hw_strip_crc = 1, /**< CRC stripped by hardware */ }, .rx_adv_conf = { .rss_conf = { @@ -247,7 +243,7 @@ check_lcore_params(void) static int check_port_config(const unsigned nb_ports) { - unsigned portid; + uint16_t portid; uint16_t i; for (i = 0; i < nb_lcore_params; ++i) { @@ -265,7 +261,7 @@ check_port_config(const unsigned nb_ports) } static uint8_t -get_port_n_rx_queues(const uint8_t port) +get_port_n_rx_queues(const uint16_t port) { int queue = -1; uint16_t i; @@ -447,7 +443,7 @@ parse_config(const char *q_arg) static void parse_eth_dest(const char *optarg) { - uint8_t portid; + uint16_t portid; char *port_end; uint8_t c, *dest, peer_addr[6]; @@ -475,6 +471,13 @@ parse_eth_dest(const char *optarg) #define MAX_JUMBO_PKT_LEN 9600 #define MEMPOOL_CACHE_SIZE 256 +static const char short_options[] = + "p:" /* portmask */ + "P" /* promiscuous */ + "L" /* enable long prefix match */ + "E" /* enable exact match */ + ; + #define CMD_LINE_OPT_CONFIG "config" #define CMD_LINE_OPT_ETH_DEST "eth-dest" #define CMD_LINE_OPT_NO_NUMA "no-numa" @@ -482,6 +485,31 @@ parse_eth_dest(const char *optarg) #define CMD_LINE_OPT_ENABLE_JUMBO "enable-jumbo" #define CMD_LINE_OPT_HASH_ENTRY_NUM "hash-entry-num" #define CMD_LINE_OPT_PARSE_PTYPE "parse-ptype" +enum { + /* long options mapped to a short option */ + + /* first long only option value must be >= 256, so that we won't + * conflict with short options */ + CMD_LINE_OPT_MIN_NUM = 256, + CMD_LINE_OPT_CONFIG_NUM, + CMD_LINE_OPT_ETH_DEST_NUM, + CMD_LINE_OPT_NO_NUMA_NUM, + CMD_LINE_OPT_IPV6_NUM, + CMD_LINE_OPT_ENABLE_JUMBO_NUM, + CMD_LINE_OPT_HASH_ENTRY_NUM_NUM, + CMD_LINE_OPT_PARSE_PTYPE_NUM, +}; + +static const struct option lgopts[] = { + {CMD_LINE_OPT_CONFIG, 1, 0, CMD_LINE_OPT_CONFIG_NUM}, + {CMD_LINE_OPT_ETH_DEST, 1, 0, CMD_LINE_OPT_ETH_DEST_NUM}, + {CMD_LINE_OPT_NO_NUMA, 0, 0, CMD_LINE_OPT_NO_NUMA_NUM}, + {CMD_LINE_OPT_IPV6, 0, 0, CMD_LINE_OPT_IPV6_NUM}, + {CMD_LINE_OPT_ENABLE_JUMBO, 0, 0, CMD_LINE_OPT_ENABLE_JUMBO_NUM}, + {CMD_LINE_OPT_HASH_ENTRY_NUM, 1, 0, CMD_LINE_OPT_HASH_ENTRY_NUM_NUM}, + {CMD_LINE_OPT_PARSE_PTYPE, 0, 0, CMD_LINE_OPT_PARSE_PTYPE_NUM}, + {NULL, 0, 0, 0} +}; /* * This expression is used to calculate the number of mbufs needed @@ -491,10 +519,10 @@ parse_eth_dest(const char *optarg) * value of 8192 */ #define NB_MBUF RTE_MAX( \ - (nb_ports*nb_rx_queue*RTE_TEST_RX_DESC_DEFAULT + \ - nb_ports*nb_lcores*MAX_PKT_BURST + \ - nb_ports*n_tx_queue*RTE_TEST_TX_DESC_DEFAULT + \ - nb_lcores*MEMPOOL_CACHE_SIZE), \ + (nb_ports*nb_rx_queue*nb_rxd + \ + nb_ports*nb_lcores*MAX_PKT_BURST + \ + nb_ports*n_tx_queue*nb_txd + \ + nb_lcores*MEMPOOL_CACHE_SIZE), \ (unsigned)8192) /* Parse the argument given in the command line of the application */ @@ -505,16 +533,6 @@ parse_args(int argc, char **argv) char **argvopt; int option_index; char *prgname = argv[0]; - static struct option lgopts[] = { - {CMD_LINE_OPT_CONFIG, 1, 0, 0}, - {CMD_LINE_OPT_ETH_DEST, 1, 0, 0}, - {CMD_LINE_OPT_NO_NUMA, 0, 0, 0}, - {CMD_LINE_OPT_IPV6, 0, 0, 0}, - {CMD_LINE_OPT_ENABLE_JUMBO, 0, 0, 0}, - {CMD_LINE_OPT_HASH_ENTRY_NUM, 1, 0, 0}, - {CMD_LINE_OPT_PARSE_PTYPE, 0, 0, 0}, - {NULL, 0, 0, 0} - }; argvopt = argv; @@ -535,7 +553,7 @@ parse_args(int argc, char **argv) "L3FWD: LPM and EM are mutually exclusive, select only one"; const char *str13 = "L3FWD: LPM or EM none selected, default LPM on"; - while ((opt = getopt_long(argc, argvopt, "p:PLE", + while ((opt = getopt_long(argc, argvopt, short_options, lgopts, &option_index)) != EOF) { switch (opt) { @@ -548,6 +566,7 @@ parse_args(int argc, char **argv) return -1; } break; + case 'P': printf("%s\n", str2); promiscuous_on = 1; @@ -564,89 +583,71 @@ parse_args(int argc, char **argv) break; /* long options */ - case 0: - if (!strncmp(lgopts[option_index].name, - CMD_LINE_OPT_CONFIG, - sizeof(CMD_LINE_OPT_CONFIG))) { + case CMD_LINE_OPT_CONFIG_NUM: + ret = parse_config(optarg); + if (ret) { + printf("%s\n", str5); + print_usage(prgname); + return -1; + } + break; - ret = parse_config(optarg); - if (ret) { - printf("%s\n", str5); + case CMD_LINE_OPT_ETH_DEST_NUM: + parse_eth_dest(optarg); + break; + + case CMD_LINE_OPT_NO_NUMA_NUM: + printf("%s\n", str6); + numa_on = 0; + break; + + case CMD_LINE_OPT_IPV6_NUM: + printf("%sn", str7); + ipv6 = 1; + break; + + case CMD_LINE_OPT_ENABLE_JUMBO_NUM: { + struct option lenopts = { + "max-pkt-len", required_argument, 0, 0 + }; + + printf("%s\n", str8); + port_conf.rxmode.jumbo_frame = 1; + + /* + * if no max-pkt-len set, use the default + * value ETHER_MAX_LEN. + */ + if (getopt_long(argc, argvopt, "", + &lenopts, &option_index) == 0) { + ret = parse_max_pkt_len(optarg); + if ((ret < 64) || + (ret > MAX_JUMBO_PKT_LEN)) { + printf("%s\n", str9); print_usage(prgname); return -1; } + port_conf.rxmode.max_rx_pkt_len = ret; } - - if (!strncmp(lgopts[option_index].name, - CMD_LINE_OPT_ETH_DEST, - sizeof(CMD_LINE_OPT_ETH_DEST))) { - parse_eth_dest(optarg); - } - - if (!strncmp(lgopts[option_index].name, - CMD_LINE_OPT_NO_NUMA, - sizeof(CMD_LINE_OPT_NO_NUMA))) { - printf("%s\n", str6); - numa_on = 0; - } - - if (!strncmp(lgopts[option_index].name, - CMD_LINE_OPT_IPV6, - sizeof(CMD_LINE_OPT_IPV6))) { - printf("%sn", str7); - ipv6 = 1; - } - - if (!strncmp(lgopts[option_index].name, - CMD_LINE_OPT_ENABLE_JUMBO, - sizeof(CMD_LINE_OPT_ENABLE_JUMBO))) { - struct option lenopts = { - "max-pkt-len", required_argument, 0, 0 - }; - - printf("%s\n", str8); - port_conf.rxmode.jumbo_frame = 1; - - /* - * if no max-pkt-len set, use the default - * value ETHER_MAX_LEN. - */ - if (0 == getopt_long(argc, argvopt, "", - &lenopts, &option_index)) { - ret = parse_max_pkt_len(optarg); - if ((ret < 64) || - (ret > MAX_JUMBO_PKT_LEN)) { - printf("%s\n", str9); - print_usage(prgname); - return -1; - } - port_conf.rxmode.max_rx_pkt_len = ret; - } - printf("%s %u\n", str10, + printf("%s %u\n", str10, (unsigned int)port_conf.rxmode.max_rx_pkt_len); + break; + } + + case CMD_LINE_OPT_HASH_ENTRY_NUM_NUM: + ret = parse_hash_entry_number(optarg); + if ((ret > 0) && (ret <= L3FWD_HASH_ENTRIES)) { + hash_entry_number = ret; + } else { + printf("%s\n", str11); + print_usage(prgname); + return -1; } + break; - if (!strncmp(lgopts[option_index].name, - CMD_LINE_OPT_HASH_ENTRY_NUM, - sizeof(CMD_LINE_OPT_HASH_ENTRY_NUM))) { - - ret = parse_hash_entry_number(optarg); - if ((ret > 0) && (ret <= L3FWD_HASH_ENTRIES)) { - hash_entry_number = ret; - } else { - printf("%s\n", str11); - print_usage(prgname); - return -1; - } - } - - if (!strncmp(lgopts[option_index].name, - CMD_LINE_OPT_PARSE_PTYPE, - sizeof(CMD_LINE_OPT_PARSE_PTYPE))) { - printf("soft parse-ptype is enabled\n"); - parse_ptype = 1; - } - + case CMD_LINE_OPT_PARSE_PTYPE_NUM: + printf("soft parse-ptype is enabled\n"); + parse_ptype = 1; break; default: @@ -684,7 +685,7 @@ parse_args(int argc, char **argv) argv[optind-1] = prgname; ret = optind-1; - optind = 0; /* reset getopt lib */ + optind = 1; /* reset getopt lib */ return ret; } @@ -747,11 +748,12 @@ init_mem(unsigned nb_mbuf) /* Check the link status of all ports in up to 9s, and print them finally */ static void -check_all_ports_link_status(uint8_t port_num, uint32_t port_mask) +check_all_ports_link_status(uint16_t port_num, uint32_t port_mask) { #define CHECK_INTERVAL 100 /* 100ms */ #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ - uint8_t portid, count, all_ports_up, print_flag = 0; + uint16_t portid; + uint8_t count, all_ports_up, print_flag = 0; struct rte_eth_link link; printf("\nChecking link status"); @@ -770,14 +772,13 @@ check_all_ports_link_status(uint8_t port_num, uint32_t port_mask) /* print link status if flag set */ if (print_flag == 1) { if (link.link_status) - printf("Port %d Link Up - speed %u " - "Mbps - %s\n", (uint8_t)portid, - (unsigned)link.link_speed, + printf( + "Port%d Link Up. Speed %u Mbps -%s\n", + portid, link.link_speed, (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? ("full-duplex") : ("half-duplex\n")); else - printf("Port %d Link Down\n", - (uint8_t)portid); + printf("Port %d Link Down\n", portid); continue; } /* clear all_ports_up flag if any link down */ @@ -815,7 +816,7 @@ signal_handler(int signum) } static int -prepare_ptype_parser(uint8_t portid, uint16_t queueid) +prepare_ptype_parser(uint16_t portid, uint16_t queueid) { if (parse_ptype) { printf("Port %d: softly parse packet type info\n", portid); @@ -844,10 +845,10 @@ main(int argc, char **argv) struct rte_eth_txconf *txconf; int ret; unsigned nb_ports; - uint16_t queueid; + uint16_t queueid, portid; unsigned lcore_id; uint32_t n_tx_queue, nb_lcores; - uint8_t portid, nb_rx_queue, queue, socketid; + uint8_t nb_rx_queue, queue, socketid; /* init EAL */ ret = rte_eal_init(argc, argv); @@ -914,6 +915,13 @@ main(int argc, char **argv) "Cannot configure device: err=%d, port=%d\n", ret, portid); + ret = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd, + &nb_txd); + if (ret < 0) + rte_exit(EXIT_FAILURE, + "Cannot adjust number of descriptors: err=%d, " + "port=%d\n", ret, portid); + rte_eth_macaddr_get(portid, &ports_eth_addr[portid]); print_ethaddr(" Address:", &ports_eth_addr[portid]); printf(", "); @@ -1038,7 +1046,7 @@ main(int argc, char **argv) } - check_all_ports_link_status((uint8_t)nb_ports, enabled_port_mask); + check_all_ports_link_status(nb_ports, enabled_port_mask); ret = 0; /* launch per-lcore init on every lcore */ diff --git a/dpdk/examples/link_status_interrupt/Makefile b/dpdk/examples/link_status_interrupt/Makefile index 9ecc7fc4..d5ee073a 100644 --- a/dpdk/examples/link_status_interrupt/Makefile +++ b/dpdk/examples/link_status_interrupt/Makefile @@ -33,7 +33,7 @@ ifeq ($(RTE_SDK),) $(error "Please define RTE_SDK environment variable") endif -# Default target, can be overriden by command line or environment +# Default target, can be overridden by command line or environment RTE_TARGET ?= x86_64-native-linuxapp-gcc include $(RTE_SDK)/mk/rte.vars.mk diff --git a/dpdk/examples/link_status_interrupt/main.c b/dpdk/examples/link_status_interrupt/main.c index 04dc3e40..bc47dcce 100644 --- a/dpdk/examples/link_status_interrupt/main.c +++ b/dpdk/examples/link_status_interrupt/main.c @@ -37,7 +37,6 @@ #include #include #include -#include #include #include #include @@ -51,9 +50,7 @@ #include #include #include -#include #include -#include #include #include #include @@ -62,12 +59,10 @@ #include #include #include -#include #include #include #include #include -#include #include #include @@ -117,7 +112,7 @@ static const struct rte_eth_conf port_conf = { .hw_ip_checksum = 0, /**< IP checksum offload disabled */ .hw_vlan_filter = 0, /**< VLAN filtering disabled */ .jumbo_frame = 0, /**< Jumbo Frame Support disabled */ - .hw_strip_crc = 0, /**< CRC stripped by hardware */ + .hw_strip_crc = 1, /**< CRC stripped by hardware */ }, .txmode = { .mq_mode = ETH_MQ_TX_NONE, @@ -148,7 +143,7 @@ print_stats(void) { struct rte_eth_link link; uint64_t total_packets_dropped, total_packets_tx, total_packets_rx; - unsigned portid; + uint16_t portid; total_packets_dropped = 0; total_packets_tx = 0; @@ -168,7 +163,7 @@ print_stats(void) continue; memset(&link, 0, sizeof(link)); - rte_eth_link_get_nowait((uint8_t)portid, &link); + rte_eth_link_get_nowait(portid, &link); printf("\nStatistics for port %u ------------------------------" "\nLink status: %25s" "\nLink speed: %26u" @@ -452,7 +447,7 @@ lsi_parse_args(int argc, char **argv) argv[optind-1] = prgname; ret = optind-1; - optind = 0; /* reset getopt lib */ + optind = 1; /* reset getopt lib */ return ret; } @@ -470,14 +465,16 @@ lsi_parse_args(int argc, char **argv) * Pointer to(address of) the parameters. * * @return - * void. + * int. */ -static void -lsi_event_callback(uint8_t port_id, enum rte_eth_event_type type, void *param) +static int +lsi_event_callback(uint16_t port_id, enum rte_eth_event_type type, void *param, + void *ret_param) { struct rte_eth_link link; RTE_SET_USED(param); + RTE_SET_USED(ret_param); printf("\n\nIn registered callback...\n"); printf("Event type: %s\n", type == RTE_ETH_EVENT_INTR_LSC ? "LSC interrupt" : "unknown event"); @@ -489,15 +486,18 @@ lsi_event_callback(uint8_t port_id, enum rte_eth_event_type type, void *param) ("full-duplex") : ("half-duplex")); } else printf("Port %d Link Down\n\n", port_id); + + return 0; } /* Check the link status of all ports in up to 9s, and print them finally */ static void -check_all_ports_link_status(uint8_t port_num, uint32_t port_mask) +check_all_ports_link_status(uint16_t port_num, uint32_t port_mask) { #define CHECK_INTERVAL 100 /* 100ms */ #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ - uint8_t portid, count, all_ports_up, print_flag = 0; + uint8_t count, all_ports_up, print_flag = 0; + uint16_t portid; struct rte_eth_link link; printf("\nChecking link status"); @@ -512,14 +512,13 @@ check_all_ports_link_status(uint8_t port_num, uint32_t port_mask) /* print link status if flag set */ if (print_flag == 1) { if (link.link_status) - printf("Port %d Link Up - speed %u " - "Mbps - %s\n", (uint8_t)portid, - (unsigned)link.link_speed, + printf( + "Port%d Link Up. Speed %u Mbps - %s\n", + portid, link.link_speed, (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? ("full-duplex") : ("half-duplex\n")); else - printf("Port %d Link Down\n", - (uint8_t)portid); + printf("Port %d Link Down\n", portid); continue; } /* clear all_ports_up flag if any link down */ @@ -552,8 +551,8 @@ main(int argc, char **argv) struct lcore_queue_conf *qconf; struct rte_eth_dev_info dev_info; int ret; - uint8_t nb_ports; - uint8_t portid, portid_last = 0; + uint16_t nb_ports; + uint16_t portid, portid_last = 0; unsigned lcore_id, rx_lcore_id; unsigned nb_ports_in_mask = 0; @@ -647,6 +646,13 @@ main(int argc, char **argv) rte_exit(EXIT_FAILURE, "Cannot configure device: err=%d, port=%u\n", ret, (unsigned) portid); + ret = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd, + &nb_txd); + if (ret < 0) + rte_exit(EXIT_FAILURE, + "rte_eth_dev_adjust_nb_rx_tx_desc: err=%d, port=%u\n", + ret, (unsigned) portid); + /* register lsi interrupt callback, need to be after * rte_eth_dev_configure(). if (intr_conf.lsc == 0), no * lsc interrupt will be present, and below callback to diff --git a/dpdk/examples/load_balancer/Makefile b/dpdk/examples/load_balancer/Makefile index 2c5fd9b0..f656e51c 100644 --- a/dpdk/examples/load_balancer/Makefile +++ b/dpdk/examples/load_balancer/Makefile @@ -33,7 +33,7 @@ ifeq ($(RTE_SDK),) $(error "Please define RTE_SDK environment variable") endif -# Default target, can be overriden by command line or environment +# Default target, can be overridden by command line or environment RTE_TARGET ?= x86_64-native-linuxapp-gcc include $(RTE_SDK)/mk/rte.vars.mk diff --git a/dpdk/examples/load_balancer/config.c b/dpdk/examples/load_balancer/config.c index 3f6ddee5..755a86e4 100644 --- a/dpdk/examples/load_balancer/config.c +++ b/dpdk/examples/load_balancer/config.c @@ -47,9 +47,7 @@ #include #include #include -#include #include -#include #include #include #include @@ -58,12 +56,10 @@ #include #include #include -#include #include #include #include #include -#include #include #include #include @@ -242,7 +238,7 @@ parse_arg_rx(const char *arg) if (lp->io.rx.n_nic_queues >= APP_MAX_NIC_RX_QUEUES_PER_IO_LCORE) { return -9; } - lp->io.rx.nic_queues[lp->io.rx.n_nic_queues].port = (uint8_t) port; + lp->io.rx.nic_queues[lp->io.rx.n_nic_queues].port = port; lp->io.rx.nic_queues[lp->io.rx.n_nic_queues].queue = (uint8_t) queue; lp->io.rx.n_nic_queues ++; @@ -320,7 +316,7 @@ parse_arg_tx(const char *arg) if (lp->io.tx.n_nic_ports >= APP_MAX_NIC_TX_PORTS_PER_IO_LCORE) { return -9; } - lp->io.tx.nic_ports[lp->io.tx.n_nic_ports] = (uint8_t) port; + lp->io.tx.nic_ports[lp->io.tx.n_nic_ports] = port; lp->io.tx.n_nic_ports ++; n_tuples ++; @@ -490,7 +486,7 @@ app_check_lpm_table(void) static int app_check_every_rx_port_is_tx_enabled(void) { - uint8_t port; + uint16_t port; for (port = 0; port < APP_MAX_NIC_PORTS; port ++) { if ((app_get_nic_rx_queues_per_port(port) > 0) && (app.nic_tx_port_mask[port] == 0)) { @@ -759,12 +755,12 @@ app_parse_args(int argc, char **argv) argv[optind - 1] = prgname; ret = optind - 1; - optind = 0; /* reset getopt lib */ + optind = 1; /* reset getopt lib */ return ret; } int -app_get_nic_rx_queues_per_port(uint8_t port) +app_get_nic_rx_queues_per_port(uint16_t port) { uint32_t i, count; @@ -783,7 +779,7 @@ app_get_nic_rx_queues_per_port(uint8_t port) } int -app_get_lcore_for_nic_rx(uint8_t port, uint8_t queue, uint32_t *lcore_out) +app_get_lcore_for_nic_rx(uint16_t port, uint8_t queue, uint32_t *lcore_out) { uint32_t lcore; @@ -810,7 +806,7 @@ app_get_lcore_for_nic_rx(uint8_t port, uint8_t queue, uint32_t *lcore_out) } int -app_get_lcore_for_nic_tx(uint8_t port, uint32_t *lcore_out) +app_get_lcore_for_nic_tx(uint16_t port, uint32_t *lcore_out) { uint32_t lcore; @@ -903,7 +899,7 @@ app_print_params(void) /* Print NIC RX configuration */ printf("NIC RX ports: "); for (port = 0; port < APP_MAX_NIC_PORTS; port ++) { - uint32_t n_rx_queues = app_get_nic_rx_queues_per_port((uint8_t) port); + uint32_t n_rx_queues = app_get_nic_rx_queues_per_port(port); if (n_rx_queues == 0) { continue; diff --git a/dpdk/examples/load_balancer/init.c b/dpdk/examples/load_balancer/init.c index e07850be..3dab7f25 100644 --- a/dpdk/examples/load_balancer/init.c +++ b/dpdk/examples/load_balancer/init.c @@ -47,9 +47,7 @@ #include #include #include -#include #include -#include #include #include #include @@ -58,7 +56,6 @@ #include #include #include -#include #include #include #include @@ -81,7 +78,7 @@ static struct rte_eth_conf port_conf = { .hw_ip_checksum = 1, /**< IP checksum offload enabled */ .hw_vlan_filter = 0, /**< VLAN filtering disabled */ .jumbo_frame = 0, /**< Jumbo Frame Support disabled */ - .hw_strip_crc = 0, /**< CRC stripped by hardware */ + .hw_strip_crc = 1, /**< CRC stripped by hardware */ }, .rx_adv_conf = { .rss_conf = { @@ -309,7 +306,7 @@ app_init_rings_tx(void) continue; } - if (app_get_lcore_for_nic_tx((uint8_t) port, &lcore_io) < 0) { + if (app_get_lcore_for_nic_tx(port, &lcore_io) < 0) { rte_panic("Algorithmic error (no I/O core to handle TX of port %u)\n", port); } @@ -360,11 +357,12 @@ app_init_rings_tx(void) /* Check the link status of all ports in up to 9s, and print them finally */ static void -check_all_ports_link_status(uint8_t port_num, uint32_t port_mask) +check_all_ports_link_status(uint16_t port_num, uint32_t port_mask) { #define CHECK_INTERVAL 100 /* 100ms */ #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ - uint8_t portid, count, all_ports_up, print_flag = 0; + uint16_t portid; + uint8_t count, all_ports_up, print_flag = 0; struct rte_eth_link link; uint32_t n_rx_queues, n_tx_queues; @@ -384,14 +382,13 @@ check_all_ports_link_status(uint8_t port_num, uint32_t port_mask) /* print link status if flag set */ if (print_flag == 1) { if (link.link_status) - printf("Port %d Link Up - speed %u " - "Mbps - %s\n", (uint8_t)portid, - (unsigned)link.link_speed, + printf( + "Port%d Link Up - speed %uMbps - %s\n", + portid, link.link_speed, (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? ("full-duplex") : ("half-duplex\n")); else - printf("Port %d Link Down\n", - (uint8_t)portid); + printf("Port %d Link Down\n", portid); continue; } /* clear all_ports_up flag if any link down */ @@ -423,13 +420,16 @@ app_init_nics(void) { unsigned socket; uint32_t lcore; - uint8_t port, queue; + uint16_t port; + uint8_t queue; int ret; uint32_t n_rx_queues, n_tx_queues; /* Init NIC ports and queues, then start the ports */ for (port = 0; port < APP_MAX_NIC_PORTS; port ++) { struct rte_mempool *pool; + uint16_t nic_rx_ring_size; + uint16_t nic_tx_ring_size; n_rx_queues = app_get_nic_rx_queues_per_port(port); n_tx_queues = app.nic_tx_port_mask[port]; @@ -439,17 +439,28 @@ app_init_nics(void) } /* Init port */ - printf("Initializing NIC port %u ...\n", (unsigned) port); + printf("Initializing NIC port %u ...\n", port); ret = rte_eth_dev_configure( port, (uint8_t) n_rx_queues, (uint8_t) n_tx_queues, &port_conf); if (ret < 0) { - rte_panic("Cannot init NIC port %u (%d)\n", (unsigned) port, ret); + rte_panic("Cannot init NIC port %u (%d)\n", port, ret); } rte_eth_promiscuous_enable(port); + nic_rx_ring_size = app.nic_rx_ring_size; + nic_tx_ring_size = app.nic_tx_ring_size; + ret = rte_eth_dev_adjust_nb_rx_tx_desc( + port, &nic_rx_ring_size, &nic_tx_ring_size); + if (ret < 0) { + rte_panic("Cannot adjust number of descriptors for port %u (%d)\n", + port, ret); + } + app.nic_rx_ring_size = nic_rx_ring_size; + app.nic_tx_ring_size = nic_tx_ring_size; + /* Init RX queues */ for (queue = 0; queue < APP_MAX_RX_QUEUES_PER_NIC_PORT; queue ++) { if (app.nic_rx_queue_mask[port][queue] == 0) { @@ -461,8 +472,7 @@ app_init_nics(void) pool = app.lcore_params[lcore].pool; printf("Initializing NIC port %u RX queue %u ...\n", - (unsigned) port, - (unsigned) queue); + port, queue); ret = rte_eth_rx_queue_setup( port, queue, @@ -472,9 +482,7 @@ app_init_nics(void) pool); if (ret < 0) { rte_panic("Cannot init RX queue %u for port %u (%d)\n", - (unsigned) queue, - (unsigned) port, - ret); + queue, port, ret); } } @@ -483,7 +491,7 @@ app_init_nics(void) app_get_lcore_for_nic_tx(port, &lcore); socket = rte_lcore_to_socket_id(lcore); printf("Initializing NIC port %u TX queue 0 ...\n", - (unsigned) port); + port); ret = rte_eth_tx_queue_setup( port, 0, diff --git a/dpdk/examples/load_balancer/main.c b/dpdk/examples/load_balancer/main.c index 7ede3585..7ced84d4 100644 --- a/dpdk/examples/load_balancer/main.c +++ b/dpdk/examples/load_balancer/main.c @@ -48,9 +48,7 @@ #include #include #include -#include #include -#include #include #include #include @@ -59,12 +57,10 @@ #include #include #include -#include #include #include #include #include -#include #include #include #include diff --git a/dpdk/examples/load_balancer/main.h b/dpdk/examples/load_balancer/main.h index d98468a7..bef2ba04 100644 --- a/dpdk/examples/load_balancer/main.h +++ b/dpdk/examples/load_balancer/main.h @@ -56,7 +56,11 @@ #endif #ifndef APP_MAX_IO_LCORES +#if (APP_MAX_LCORES > 16) #define APP_MAX_IO_LCORES 16 +#else +#define APP_MAX_IO_LCORES APP_MAX_LCORES +#endif #endif #if (APP_MAX_IO_LCORES > APP_MAX_LCORES) #error "APP_MAX_IO_LCORES is too big" @@ -74,7 +78,11 @@ #endif #ifndef APP_MAX_WORKER_LCORES +#if (APP_MAX_LCORES > 16) #define APP_MAX_WORKER_LCORES 16 +#else +#define APP_MAX_WORKER_LCORES APP_MAX_LCORES +#endif #endif #if (APP_MAX_WORKER_LCORES > APP_MAX_LCORES) #error "APP_MAX_WORKER_LCORES is too big" @@ -240,7 +248,7 @@ struct app_lcore_params_io { struct { /* NIC */ struct { - uint8_t port; + uint16_t port; uint8_t queue; } nic_queues[APP_MAX_NIC_RX_QUEUES_PER_IO_LCORE]; uint32_t n_nic_queues; @@ -267,7 +275,7 @@ struct app_lcore_params_io { struct rte_ring *rings[APP_MAX_NIC_PORTS][APP_MAX_WORKER_LCORES]; /* NIC */ - uint8_t nic_ports[APP_MAX_NIC_TX_PORTS_PER_IO_LCORE]; + uint16_t nic_ports[APP_MAX_NIC_TX_PORTS_PER_IO_LCORE]; uint32_t n_nic_ports; /* Internal buffers */ @@ -360,9 +368,10 @@ void app_print_usage(void); void app_init(void); int app_lcore_main_loop(void *arg); -int app_get_nic_rx_queues_per_port(uint8_t port); -int app_get_lcore_for_nic_rx(uint8_t port, uint8_t queue, uint32_t *lcore_out); -int app_get_lcore_for_nic_tx(uint8_t port, uint32_t *lcore_out); +int app_get_nic_rx_queues_per_port(uint16_t port); +int app_get_lcore_for_nic_rx(uint16_t port, uint8_t queue, + uint32_t *lcore_out); +int app_get_lcore_for_nic_tx(uint16_t port, uint32_t *lcore_out); int app_is_socket_used(uint32_t socket); uint32_t app_get_lcores_io_rx(void); uint32_t app_get_lcores_worker(void); diff --git a/dpdk/examples/load_balancer/runtime.c b/dpdk/examples/load_balancer/runtime.c index 6944325d..f65e14f0 100644 --- a/dpdk/examples/load_balancer/runtime.c +++ b/dpdk/examples/load_balancer/runtime.c @@ -47,9 +47,7 @@ #include #include #include -#include #include -#include #include #include #include @@ -58,7 +56,6 @@ #include #include #include -#include #include #include #include @@ -144,9 +141,10 @@ app_lcore_io_rx_buffer_to_send ( ret = rte_ring_sp_enqueue_bulk( lp->rx.rings[worker], (void **) lp->rx.mbuf_out[worker].array, - bsz); + bsz, + NULL); - if (unlikely(ret == -ENOBUFS)) { + if (unlikely(ret == 0)) { uint32_t k; for (k = 0; k < bsz; k ++) { struct rte_mbuf *m = lp->rx.mbuf_out[worker].array[k]; @@ -188,7 +186,7 @@ app_lcore_io_rx( uint32_t i; for (i = 0; i < lp->rx.n_nic_queues; i ++) { - uint8_t port = lp->rx.nic_queues[i].port; + uint16_t port = lp->rx.nic_queues[i].port; uint8_t queue = lp->rx.nic_queues[i].queue; uint32_t n_mbufs, j; @@ -213,7 +211,7 @@ app_lcore_io_rx( printf("I/O RX %u in (NIC port %u): NIC drop ratio = %.2f avg burst size = %.2f\n", lcore, - (unsigned) port, + port, (double) stats.imissed / (double) (stats.imissed + stats.ipackets), ((double) lp->rx.nic_queues_count[i]) / ((double) lp->rx.nic_queues_iters[i])); lp->rx.nic_queues_iters[i] = 0; @@ -310,9 +308,10 @@ app_lcore_io_rx_flush(struct app_lcore_params_io *lp, uint32_t n_workers) ret = rte_ring_sp_enqueue_bulk( lp->rx.rings[worker], (void **) lp->rx.mbuf_out[worker].array, - lp->rx.mbuf_out[worker].n_mbufs); + lp->rx.mbuf_out[worker].n_mbufs, + NULL); - if (unlikely(ret < 0)) { + if (unlikely(ret == 0)) { uint32_t k; for (k = 0; k < lp->rx.mbuf_out[worker].n_mbufs; k ++) { struct rte_mbuf *pkt_to_free = lp->rx.mbuf_out[worker].array[k]; @@ -338,7 +337,7 @@ app_lcore_io_tx( uint32_t i; for (i = 0; i < lp->tx.n_nic_ports; i ++) { - uint8_t port = lp->tx.nic_ports[i]; + uint16_t port = lp->tx.nic_ports[i]; struct rte_ring *ring = lp->tx.rings[port][worker]; uint32_t n_mbufs, n_pkts; int ret; @@ -347,11 +346,11 @@ app_lcore_io_tx( ret = rte_ring_sc_dequeue_bulk( ring, (void **) &lp->tx.mbuf_out[port].array[n_mbufs], - bsz_rd); + bsz_rd, + NULL); - if (unlikely(ret == -ENOENT)) { + if (unlikely(ret == 0)) continue; - } n_mbufs += bsz_rd; @@ -394,7 +393,7 @@ app_lcore_io_tx( printf("\t\t\tI/O TX %u out (port %u): avg burst size = %.2f\n", lcore, - (unsigned) port, + port, ((double) lp->tx.nic_ports_count[port]) / ((double) lp->tx.nic_ports_iters[port])); lp->tx.nic_ports_iters[port] = 0; lp->tx.nic_ports_count[port] = 0; @@ -417,11 +416,13 @@ app_lcore_io_tx( static inline void app_lcore_io_tx_flush(struct app_lcore_params_io *lp) { - uint8_t port; + uint16_t port; + uint32_t i; - for (port = 0; port < lp->tx.n_nic_ports; port ++) { + for (i = 0; i < lp->tx.n_nic_ports; i++) { uint32_t n_pkts; + port = lp->tx.nic_ports[i]; if (likely((lp->tx.mbuf_out_flush[port] == 0) || (lp->tx.mbuf_out[port].n_mbufs == 0))) { lp->tx.mbuf_out_flush[port] = 1; @@ -503,11 +504,11 @@ app_lcore_worker( ret = rte_ring_sc_dequeue_bulk( ring_in, (void **) lp->mbuf_in.array, - bsz_rd); + bsz_rd, + NULL); - if (unlikely(ret == -ENOENT)) { + if (unlikely(ret == 0)) continue; - } #if APP_WORKER_DROP_ALL_PACKETS for (j = 0; j < bsz_rd; j ++) { @@ -555,24 +556,25 @@ app_lcore_worker( ret = rte_ring_sp_enqueue_bulk( lp->rings_out[port], (void **) lp->mbuf_out[port].array, - bsz_wr); + bsz_wr, + NULL); #if APP_STATS lp->rings_out_iters[port] ++; - if (ret == 0) { + if (ret > 0) { lp->rings_out_count[port] += 1; } if (lp->rings_out_iters[port] == APP_STATS){ printf("\t\tWorker %u out (NIC port %u): enq success rate = %.2f\n", (unsigned) lp->worker_id, - (unsigned) port, + port, ((double) lp->rings_out_count[port]) / ((double) lp->rings_out_iters[port])); lp->rings_out_iters[port] = 0; lp->rings_out_count[port] = 0; } #endif - if (unlikely(ret == -ENOBUFS)) { + if (unlikely(ret == 0)) { uint32_t k; for (k = 0; k < bsz_wr; k ++) { struct rte_mbuf *pkt_to_free = lp->mbuf_out[port].array[k]; @@ -607,9 +609,10 @@ app_lcore_worker_flush(struct app_lcore_params_worker *lp) ret = rte_ring_sp_enqueue_bulk( lp->rings_out[port], (void **) lp->mbuf_out[port].array, - lp->mbuf_out[port].n_mbufs); + lp->mbuf_out[port].n_mbufs, + NULL); - if (unlikely(ret < 0)) { + if (unlikely(ret == 0)) { uint32_t k; for (k = 0; k < lp->mbuf_out[port].n_mbufs; k ++) { struct rte_mbuf *pkt_to_free = lp->mbuf_out[port].array[k]; diff --git a/dpdk/examples/multi_process/Makefile b/dpdk/examples/multi_process/Makefile index 6b315cc0..696633b9 100644 --- a/dpdk/examples/multi_process/Makefile +++ b/dpdk/examples/multi_process/Makefile @@ -33,7 +33,7 @@ ifeq ($(RTE_SDK),) $(error "Please define RTE_SDK environment variable") endif -# Default target, can be overriden by command line or environment +# Default target, can be overridden by command line or environment RTE_TARGET ?= x86_64-native-linuxapp-gcc include $(RTE_SDK)/mk/rte.vars.mk diff --git a/dpdk/examples/multi_process/client_server_mp/Makefile b/dpdk/examples/multi_process/client_server_mp/Makefile index 89cc6bf8..feb508a4 100644 --- a/dpdk/examples/multi_process/client_server_mp/Makefile +++ b/dpdk/examples/multi_process/client_server_mp/Makefile @@ -33,7 +33,7 @@ ifeq ($(RTE_SDK),) $(error "Please define RTE_SDK environment variable") endif -# Default target, can be overriden by command line or environment +# Default target, can be overridden by command line or environment RTE_TARGET ?= x86_64-native-linuxapp-gcc include $(RTE_SDK)/mk/rte.vars.mk diff --git a/dpdk/examples/multi_process/client_server_mp/mp_client/Makefile b/dpdk/examples/multi_process/client_server_mp/mp_client/Makefile index 2688fed0..2ee8cd2c 100644 --- a/dpdk/examples/multi_process/client_server_mp/mp_client/Makefile +++ b/dpdk/examples/multi_process/client_server_mp/mp_client/Makefile @@ -33,7 +33,7 @@ ifeq ($(RTE_SDK),) $(error "Please define RTE_SDK environment variable") endif -# Default target, can be overriden by command line or environment +# Default target, can be overridden by command line or environment include $(RTE_SDK)/mk/rte.vars.mk # binary name diff --git a/dpdk/examples/multi_process/client_server_mp/mp_client/client.c b/dpdk/examples/multi_process/client_server_mp/mp_client/client.c index d4f9ca37..30ce4b34 100644 --- a/dpdk/examples/multi_process/client_server_mp/mp_client/client.c +++ b/dpdk/examples/multi_process/client_server_mp/mp_client/client.c @@ -50,16 +50,13 @@ #include #include #include -#include #include #include #include -#include #include #include #include #include -#include #include #include #include @@ -76,7 +73,7 @@ static uint8_t client_id = 0; #define MBQ_CAPACITY 32 /* maps input ports to output ports for packets */ -static uint8_t output_ports[RTE_MAX_ETHPORTS]; +static uint16_t output_ports[RTE_MAX_ETHPORTS]; /* buffers up a set of packet that are ready to send */ struct rte_eth_dev_tx_buffer *tx_buffer[RTE_MAX_ETHPORTS]; @@ -152,7 +149,7 @@ static void flush_tx_error_callback(struct rte_mbuf **unsent, uint16_t count, void *userdata) { int i; - uint8_t port_id = (uintptr_t)userdata; + uint16_t port_id = (uintptr_t)userdata; tx_stats->tx_drop[port_id] += count; @@ -163,7 +160,7 @@ flush_tx_error_callback(struct rte_mbuf **unsent, uint16_t count, } static void -configure_tx_buffer(uint8_t port_id, uint16_t size) +configure_tx_buffer(uint16_t port_id, uint16_t size) { int ret; @@ -173,15 +170,16 @@ configure_tx_buffer(uint8_t port_id, uint16_t size) rte_eth_dev_socket_id(port_id)); if (tx_buffer[port_id] == NULL) rte_exit(EXIT_FAILURE, "Cannot allocate buffer for tx on port %u\n", - (unsigned) port_id); + port_id); rte_eth_tx_buffer_init(tx_buffer[port_id], size); ret = rte_eth_tx_buffer_set_err_callback(tx_buffer[port_id], flush_tx_error_callback, (void *)(intptr_t)port_id); if (ret < 0) - rte_exit(EXIT_FAILURE, "Cannot set error callback for " - "tx buffer on port %u\n", (unsigned) port_id); + rte_exit(EXIT_FAILURE, + "Cannot set error callback for tx buffer on port %u\n", + port_id); } /* @@ -197,8 +195,8 @@ configure_output_ports(const struct port_info *ports) rte_exit(EXIT_FAILURE, "Too many ethernet ports. RTE_MAX_ETHPORTS = %u\n", (unsigned)RTE_MAX_ETHPORTS); for (i = 0; i < ports->num_ports - 1; i+=2){ - uint8_t p1 = ports->id[i]; - uint8_t p2 = ports->id[i+1]; + uint16_t p1 = ports->id[i]; + uint16_t p2 = ports->id[i+1]; output_ports[p1] = p2; output_ports[p2] = p1; @@ -217,8 +215,8 @@ static void handle_packet(struct rte_mbuf *buf) { int sent; - const uint8_t in_port = buf->port; - const uint8_t out_port = output_ports[in_port]; + const uint16_t in_port = buf->port; + const uint16_t out_port = output_ports[in_port]; struct rte_eth_dev_tx_buffer *buffer = tx_buffer[out_port]; sent = rte_eth_tx_buffer(out_port, client_id, buffer, buf); @@ -276,14 +274,11 @@ main(int argc, char *argv[]) printf("[Press Ctrl-C to quit ...]\n"); for (;;) { - uint16_t i, rx_pkts = PKT_READ_SIZE; - uint8_t port; + uint16_t i, rx_pkts; + uint16_t port; - /* try dequeuing max possible packets first, if that fails, get the - * most we can. Loop body should only execute once, maximum */ - while (rx_pkts > 0 && - unlikely(rte_ring_dequeue_bulk(rx_ring, pkts, rx_pkts) != 0)) - rx_pkts = (uint16_t)RTE_MIN(rte_ring_count(rx_ring), PKT_READ_SIZE); + rx_pkts = rte_ring_dequeue_burst(rx_ring, pkts, + PKT_READ_SIZE, NULL); if (unlikely(rx_pkts == 0)){ if (need_flush) diff --git a/dpdk/examples/multi_process/client_server_mp/mp_server/Makefile b/dpdk/examples/multi_process/client_server_mp/mp_server/Makefile index c29e4783..160c17b6 100644 --- a/dpdk/examples/multi_process/client_server_mp/mp_server/Makefile +++ b/dpdk/examples/multi_process/client_server_mp/mp_server/Makefile @@ -33,7 +33,7 @@ ifeq ($(RTE_SDK),) $(error "Please define RTE_SDK environment variable") endif -# Default target, can be overriden by command line or environment +# Default target, can be overridden by command line or environment RTE_TARGET ?= x86_64-native-linuxapp-gcc include $(RTE_SDK)/mk/rte.vars.mk @@ -49,7 +49,7 @@ APP = mp_server # all source are stored in SRCS-y SRCS-y := main.c init.c args.c -INC := $(wildcard *.h) +INC := $(sort $(wildcard *.h)) CFLAGS += $(WERROR_FLAGS) -O3 CFLAGS += -I$(SRCDIR)/../shared diff --git a/dpdk/examples/multi_process/client_server_mp/mp_server/args.c b/dpdk/examples/multi_process/client_server_mp/mp_server/args.c index bf8c666c..a65884fd 100644 --- a/dpdk/examples/multi_process/client_server_mp/mp_server/args.c +++ b/dpdk/examples/multi_process/client_server_mp/mp_server/args.c @@ -74,7 +74,7 @@ parse_portmask(uint8_t max_ports, const char *portmask) { char *end = NULL; unsigned long pm; - uint8_t count = 0; + uint16_t count = 0; if (portmask == NULL || *portmask == '\0') return -1; @@ -128,7 +128,7 @@ parse_num_clients(const char *clients) * on error. */ int -parse_app_args(uint8_t max_ports, int argc, char *argv[]) +parse_app_args(uint16_t max_ports, int argc, char *argv[]) { int option_index, opt; char **argvopt = argv; diff --git a/dpdk/examples/multi_process/client_server_mp/mp_server/args.h b/dpdk/examples/multi_process/client_server_mp/mp_server/args.h index 23af1bd3..33888b89 100644 --- a/dpdk/examples/multi_process/client_server_mp/mp_server/args.h +++ b/dpdk/examples/multi_process/client_server_mp/mp_server/args.h @@ -34,6 +34,6 @@ #ifndef _ARGS_H_ #define _ARGS_H_ -int parse_app_args(uint8_t max_ports, int argc, char *argv[]); +int parse_app_args(uint16_t max_ports, int argc, char *argv[]); #endif /* ifndef _ARGS_H_ */ diff --git a/dpdk/examples/multi_process/client_server_mp/mp_server/init.c b/dpdk/examples/multi_process/client_server_mp/mp_server/init.c index ad941a7a..c8d02113 100644 --- a/dpdk/examples/multi_process/client_server_mp/mp_server/init.c +++ b/dpdk/examples/multi_process/client_server_mp/mp_server/init.c @@ -56,7 +56,6 @@ #include #include #include -#include #include #include #include @@ -114,7 +113,7 @@ init_mbuf_pools(void) * - start the port and report its status to stdout */ static int -init_port(uint8_t port_num) +init_port(uint16_t port_num) { /* for port configuration all features are off by default */ const struct rte_eth_conf port_conf = { @@ -123,13 +122,13 @@ init_port(uint8_t port_num) } }; const uint16_t rx_rings = 1, tx_rings = num_clients; - const uint16_t rx_ring_size = RTE_MP_RX_DESC_DEFAULT; - const uint16_t tx_ring_size = RTE_MP_TX_DESC_DEFAULT; + uint16_t rx_ring_size = RTE_MP_RX_DESC_DEFAULT; + uint16_t tx_ring_size = RTE_MP_TX_DESC_DEFAULT; uint16_t q; int retval; - printf("Port %u init ... ", (unsigned)port_num); + printf("Port %u init ... ", port_num); fflush(stdout); /* Standard DPDK port initialisation - config port, then set up @@ -138,6 +137,11 @@ init_port(uint8_t port_num) &port_conf)) != 0) return retval; + retval = rte_eth_dev_adjust_nb_rx_tx_desc(port_num, &rx_ring_size, + &tx_ring_size); + if (retval != 0) + return retval; + for (q = 0; q < rx_rings; q++) { retval = rte_eth_rx_queue_setup(port_num, q, rx_ring_size, rte_eth_dev_socket_id(port_num), @@ -195,11 +199,12 @@ init_shm_rings(void) /* Check the link status of all ports in up to 9s, and print them finally */ static void -check_all_ports_link_status(uint8_t port_num, uint32_t port_mask) +check_all_ports_link_status(uint16_t port_num, uint32_t port_mask) { #define CHECK_INTERVAL 100 /* 100ms */ #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ - uint8_t portid, count, all_ports_up, print_flag = 0; + uint16_t portid; + uint8_t count, all_ports_up, print_flag = 0; struct rte_eth_link link; printf("\nChecking link status"); @@ -257,7 +262,7 @@ init(int argc, char *argv[]) { int retval; const struct rte_memzone *mz; - uint8_t i, total_ports; + uint16_t i, total_ports; /* init EAL, parsing EAL args */ retval = rte_eal_init(argc, argv); diff --git a/dpdk/examples/multi_process/client_server_mp/mp_server/main.c b/dpdk/examples/multi_process/client_server_mp/mp_server/main.c index a6dc12d5..6eda556a 100644 --- a/dpdk/examples/multi_process/client_server_mp/mp_server/main.c +++ b/dpdk/examples/multi_process/client_server_mp/mp_server/main.c @@ -38,16 +38,13 @@ #include #include #include -#include #include #include #include #include #include -#include #include -#include #include #include #include @@ -61,7 +58,6 @@ #include #include #include -#include #include #include #include @@ -90,7 +86,7 @@ struct client_rx_buf { static struct client_rx_buf *cl_rx_buf; static const char * -get_printable_mac_addr(uint8_t port) +get_printable_mac_addr(uint16_t port) { static const char err_address[] = "00:00:00:00:00:00"; static char addresses[RTE_MAX_ETHPORTS][sizeof(err_address)]; @@ -227,7 +223,7 @@ flush_rx_queue(uint16_t client) cl = &clients[client]; if (rte_ring_enqueue_bulk(cl->rx_q, (void **)cl_rx_buf[client].buffer, - cl_rx_buf[client].count) != 0){ + cl_rx_buf[client].count, NULL) == 0){ for (j = 0; j < cl_rx_buf[client].count; j++) rte_pktmbuf_free(cl_rx_buf[client].buffer[j]); cl->stats.rx_drop += cl_rx_buf[client].count; diff --git a/dpdk/examples/multi_process/client_server_mp/shared/common.h b/dpdk/examples/multi_process/client_server_mp/shared/common.h index 631c4632..35a3b01d 100644 --- a/dpdk/examples/multi_process/client_server_mp/shared/common.h +++ b/dpdk/examples/multi_process/client_server_mp/shared/common.h @@ -57,8 +57,8 @@ struct tx_stats{ } __rte_cache_aligned; struct port_info { - uint8_t num_ports; - uint8_t id[RTE_MAX_ETHPORTS]; + uint16_t num_ports; + uint16_t id[RTE_MAX_ETHPORTS]; volatile struct rx_stats rx_stats; volatile struct tx_stats tx_stats[MAX_CLIENTS]; }; diff --git a/dpdk/examples/multi_process/l2fwd_fork/Makefile b/dpdk/examples/multi_process/l2fwd_fork/Makefile index ff257a35..11ae8ff4 100644 --- a/dpdk/examples/multi_process/l2fwd_fork/Makefile +++ b/dpdk/examples/multi_process/l2fwd_fork/Makefile @@ -33,7 +33,7 @@ ifeq ($(RTE_SDK),) $(error "Please define RTE_SDK environment variable") endif -# Default target, can be overriden by command line or environment +# Default target, can be overridden by command line or environment RTE_TARGET ?= x86_64-native-linuxapp-gcc include $(RTE_SDK)/mk/rte.vars.mk diff --git a/dpdk/examples/multi_process/l2fwd_fork/flib.c b/dpdk/examples/multi_process/l2fwd_fork/flib.c index 343f09f1..a3f1d275 100644 --- a/dpdk/examples/multi_process/l2fwd_fork/flib.c +++ b/dpdk/examples/multi_process/l2fwd_fork/flib.c @@ -54,9 +54,7 @@ #include #include #include -#include #include -#include #include #include #include @@ -65,12 +63,10 @@ #include #include #include -#include #include #include #include #include -#include #include #include #include diff --git a/dpdk/examples/multi_process/l2fwd_fork/flib.h b/dpdk/examples/multi_process/l2fwd_fork/flib.h index 711e3b6d..1064c9bb 100644 --- a/dpdk/examples/multi_process/l2fwd_fork/flib.h +++ b/dpdk/examples/multi_process/l2fwd_fork/flib.h @@ -120,7 +120,7 @@ int flib_register_slave_exit_notify(unsigned slave_id, /** * Assign a lcore ID to non-slave thread. Non-slave thread refers to thread that * not created by function rte_eal_remote_launch or rte_eal_mp_remote_launch. - * These threads can either bind lcore or float among differnt lcores. + * These threads can either bind lcore or float among different lcores. * This lcore ID will be unique in multi-thread or multi-process DPDK running * environment, then it can benefit from using the cache mechanism provided in * mempool library. diff --git a/dpdk/examples/multi_process/l2fwd_fork/main.c b/dpdk/examples/multi_process/l2fwd_fork/main.c index 2d951d93..deace273 100644 --- a/dpdk/examples/multi_process/l2fwd_fork/main.c +++ b/dpdk/examples/multi_process/l2fwd_fork/main.c @@ -51,9 +51,7 @@ #include #include #include -#include #include -#include #include #include #include @@ -63,7 +61,6 @@ #include #include #include -#include #include #include #include @@ -77,8 +74,7 @@ #define RTE_LOGTYPE_L2FWD RTE_LOGTYPE_USER1 #define MBUF_NAME "mbuf_pool_%d" -#define MBUF_SIZE \ -(RTE_MBUF_DEFAULT_DATAROOM + sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM) +#define MBUF_DATA_SIZE RTE_MBUF_DEFAULT_BUF_SIZE #define NB_MBUF 8192 #define RING_MASTER_NAME "l2fwd_ring_m2s_" #define RING_SLAVE_NAME "l2fwd_ring_s2m_" @@ -142,7 +138,8 @@ struct lcore_resource_struct { /* ring[1] for slave send ack, master read */ struct rte_ring *ring[2]; int port_num; /* Total port numbers */ - uint8_t port[RTE_MAX_ETHPORTS]; /* Port id for that lcore to receive packets */ + /* Port id for that lcore to receive packets */ + uint16_t port[RTE_MAX_ETHPORTS]; }__attribute__((packed)) __rte_cache_aligned; static struct lcore_resource_struct lcore_resource[RTE_MAX_LCORE]; @@ -163,7 +160,7 @@ static const struct rte_eth_conf port_conf = { .hw_ip_checksum = 0, /**< IP checksum offload disabled */ .hw_vlan_filter = 0, /**< VLAN filtering disabled */ .jumbo_frame = 0, /**< Jumbo Frame Support disabled */ - .hw_strip_crc = 0, /**< CRC stripped by hardware */ + .hw_strip_crc = 1, /**< CRC stripped by hardware */ }, .txmode = { .mq_mode = ETH_MQ_TX_NONE, @@ -672,6 +669,8 @@ l2fwd_main_loop(void) port_statistics[portid].tx += sent; } + + prev_tsc = cur_tsc; } /* @@ -865,17 +864,18 @@ l2fwd_parse_args(int argc, char **argv) return -1; } ret = optind-1; - optind = 0; /* reset getopt lib */ + optind = 1; /* reset getopt lib */ return ret; } /* Check the link status of all ports in up to 9s, and print them finally */ static void -check_all_ports_link_status(uint8_t port_num, uint32_t port_mask) +check_all_ports_link_status(uint16_t port_num, uint32_t port_mask) { #define CHECK_INTERVAL 100 /* 100ms */ #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ - uint8_t portid, count, all_ports_up, print_flag = 0; + uint16_t portid; + uint8_t count, all_ports_up, print_flag = 0; struct rte_eth_link link; printf("\nChecking link status"); @@ -890,14 +890,13 @@ check_all_ports_link_status(uint8_t port_num, uint32_t port_mask) /* print link status if flag set */ if (print_flag == 1) { if (link.link_status) - printf("Port %d Link Up - speed %u " - "Mbps - %s\n", (uint8_t)portid, - (unsigned)link.link_speed, + printf( + "Port%d Link Up- speed %u Mbps- %s\n", + portid, link.link_speed, (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? ("full-duplex") : ("half-duplex\n")); else - printf("Port %d Link Down\n", - (uint8_t)portid); + printf("Port %d Link Down\n", portid); continue; } /* clear all_ports_up flag if any link down */ @@ -930,13 +929,12 @@ main(int argc, char **argv) struct lcore_queue_conf *qconf; struct rte_eth_dev_info dev_info; int ret; - uint8_t nb_ports; - uint8_t nb_ports_available; - uint8_t portid, last_port; + uint16_t nb_ports; + uint16_t nb_ports_available; + uint16_t portid, last_port; unsigned rx_lcore_id; unsigned nb_ports_in_mask = 0; unsigned i; - int flags = 0; uint64_t prev_tsc, diff_tsc, cur_tsc, timer_tsc; /* Save cpu_affinity first, restore it in case it's floating process option */ @@ -986,17 +984,12 @@ main(int argc, char **argv) if ((l2fwd_enabled_port_mask & (1 << portid)) == 0) continue; char buf_name[RTE_MEMPOOL_NAMESIZE]; - flags = MEMPOOL_F_SP_PUT | MEMPOOL_F_SC_GET; snprintf(buf_name, RTE_MEMPOOL_NAMESIZE, MBUF_NAME, portid); l2fwd_pktmbuf_pool[portid] = - rte_mempool_create(buf_name, NB_MBUF, - MBUF_SIZE, 32, - sizeof(struct rte_pktmbuf_pool_private), - rte_pktmbuf_pool_init, NULL, - rte_pktmbuf_init, NULL, - rte_socket_id(), flags); + rte_pktmbuf_pool_create(buf_name, NB_MBUF, 32, + 0, MBUF_DATA_SIZE, rte_socket_id()); if (l2fwd_pktmbuf_pool[portid] == NULL) - rte_exit(EXIT_FAILURE, "Cannot init mbuf pool\n"); + rte_exit(EXIT_FAILURE, "Cannot create mbuf pool\n"); printf("Create mbuf %s\n", buf_name); } @@ -1085,6 +1078,13 @@ main(int argc, char **argv) rte_exit(EXIT_FAILURE, "Cannot configure device: err=%d, port=%u\n", ret, (unsigned) portid); + ret = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd, + &nb_txd); + if (ret < 0) + rte_exit(EXIT_FAILURE, + "rte_eth_dev_adjust_nb_rx_tx_desc: err=%d, port=%u\n", + ret, (unsigned) portid); + rte_eth_macaddr_get(portid,&l2fwd_ports_eth_addr[portid]); /* init one RX queue */ @@ -1203,10 +1203,7 @@ main(int argc, char **argv) message_pool = rte_mempool_create("ms_msg_pool", NB_CORE_MSGBUF * RTE_MAX_LCORE, sizeof(enum l2fwd_cmd), NB_CORE_MSGBUF / 2, - 0, - rte_pktmbuf_pool_init, NULL, - rte_pktmbuf_init, NULL, - rte_socket_id(), 0); + 0, NULL, NULL, NULL, NULL, rte_socket_id(), 0); if (message_pool == NULL) rte_exit(EXIT_FAILURE, "Create msg mempool failed\n"); diff --git a/dpdk/examples/multi_process/simple_mp/Makefile b/dpdk/examples/multi_process/simple_mp/Makefile index 31ec0c80..7ac96f2f 100644 --- a/dpdk/examples/multi_process/simple_mp/Makefile +++ b/dpdk/examples/multi_process/simple_mp/Makefile @@ -33,7 +33,7 @@ ifeq ($(RTE_SDK),) $(error "Please define RTE_SDK environment variable") endif -# Default target, can be overriden by command line or environment +# Default target, can be overridden by command line or environment RTE_TARGET ?= x86_64-native-linuxapp-gcc include $(RTE_SDK)/mk/rte.vars.mk diff --git a/dpdk/examples/multi_process/simple_mp/main.c b/dpdk/examples/multi_process/simple_mp/main.c index 2843d94e..62537b02 100644 --- a/dpdk/examples/multi_process/simple_mp/main.c +++ b/dpdk/examples/multi_process/simple_mp/main.c @@ -54,7 +54,6 @@ #include #include -#include #include #include #include @@ -67,6 +66,7 @@ #include #include #include +#include #include #include #include "mp_commands.h" @@ -76,7 +76,6 @@ static const char *_MSG_POOL = "MSG_POOL"; static const char *_SEC_2_PRI = "SEC_2_PRI"; static const char *_PRI_2_SEC = "PRI_2_SEC"; -const unsigned string_size = 64; struct rte_ring *send_ring, *recv_ring; struct rte_mempool *message_pool; @@ -121,7 +120,7 @@ main(int argc, char **argv) send_ring = rte_ring_create(_PRI_2_SEC, ring_size, rte_socket_id(), flags); recv_ring = rte_ring_create(_SEC_2_PRI, ring_size, rte_socket_id(), flags); message_pool = rte_mempool_create(_MSG_POOL, pool_size, - string_size, pool_cache, priv_data_sz, + STR_TOKEN_SIZE, pool_cache, priv_data_sz, NULL, NULL, NULL, NULL, rte_socket_id(), flags); } else { diff --git a/dpdk/examples/multi_process/simple_mp/mp_commands.c b/dpdk/examples/multi_process/simple_mp/mp_commands.c index 8da244bb..ef6dc58d 100644 --- a/dpdk/examples/multi_process/simple_mp/mp_commands.c +++ b/dpdk/examples/multi_process/simple_mp/mp_commands.c @@ -42,7 +42,6 @@ #include #include -#include #include #include #include @@ -78,7 +77,7 @@ static void cmd_send_parsed(void *parsed_result, if (rte_mempool_get(message_pool, &msg) < 0) rte_panic("Failed to get message buffer\n"); - snprintf((char *)msg, string_size, "%s", res->message); + snprintf((char *)msg, STR_TOKEN_SIZE, "%s", res->message); if (rte_ring_enqueue(send_ring, msg) < 0) { printf("Failed to send message - message discarded\n"); rte_mempool_put(message_pool, msg); diff --git a/dpdk/examples/multi_process/simple_mp/mp_commands.h b/dpdk/examples/multi_process/simple_mp/mp_commands.h index 7e9a4ab2..452b3645 100644 --- a/dpdk/examples/multi_process/simple_mp/mp_commands.h +++ b/dpdk/examples/multi_process/simple_mp/mp_commands.h @@ -34,7 +34,6 @@ #ifndef _SIMPLE_MP_COMMANDS_H_ #define _SIMPLE_MP_COMMANDS_H_ -extern const unsigned string_size; extern struct rte_ring *send_ring; extern struct rte_mempool *message_pool; extern volatile int quit; diff --git a/dpdk/examples/multi_process/symmetric_mp/Makefile b/dpdk/examples/multi_process/symmetric_mp/Makefile index c789f3c9..77d90c68 100644 --- a/dpdk/examples/multi_process/symmetric_mp/Makefile +++ b/dpdk/examples/multi_process/symmetric_mp/Makefile @@ -33,7 +33,7 @@ ifeq ($(RTE_SDK),) $(error "Please define RTE_SDK environment variable") endif -# Default target, can be overriden by command line or environment +# Default target, can be overridden by command line or environment RTE_TARGET ?= x86_64-native-linuxapp-gcc include $(RTE_SDK)/mk/rte.vars.mk diff --git a/dpdk/examples/multi_process/symmetric_mp/main.c b/dpdk/examples/multi_process/symmetric_mp/main.c index 6bbff076..6fb285c7 100644 --- a/dpdk/examples/multi_process/symmetric_mp/main.c +++ b/dpdk/examples/multi_process/symmetric_mp/main.c @@ -56,18 +56,14 @@ #include #include #include -#include #include #include #include #include -#include #include #include -#include #include #include -#include #include #include #include @@ -104,7 +100,7 @@ struct port_stats{ static int proc_id = -1; static unsigned num_procs = 0; -static uint8_t ports[RTE_MAX_ETHPORTS]; +static uint16_t ports[RTE_MAX_ETHPORTS]; static unsigned num_ports = 0; static struct lcore_ports lcore_ports[RTE_MAX_LCORE]; @@ -194,7 +190,7 @@ smp_parse_args(int argc, char **argv) ports[num_ports++] = (uint8_t)i; ret = optind-1; - optind = 0; /* reset getopt lib */ + optind = 1; /* reset getopt lib */ return ret; } @@ -204,7 +200,8 @@ smp_parse_args(int argc, char **argv) * coming from the mbuf_pool passed as parameter */ static inline int -smp_port_init(uint8_t port, struct rte_mempool *mbuf_pool, uint16_t num_queues) +smp_port_init(uint16_t port, struct rte_mempool *mbuf_pool, + uint16_t num_queues) { struct rte_eth_conf port_conf = { .rxmode = { @@ -214,7 +211,7 @@ smp_port_init(uint8_t port, struct rte_mempool *mbuf_pool, uint16_t num_queues) .hw_ip_checksum = 1, /**< IP checksum offload enabled */ .hw_vlan_filter = 0, /**< VLAN filtering disabled */ .jumbo_frame = 0, /**< Jumbo Frame Support disabled */ - .hw_strip_crc = 0, /**< CRC stripped by hardware */ + .hw_strip_crc = 1, /**< CRC stripped by hardware */ }, .rx_adv_conf = { .rss_conf = { @@ -230,6 +227,8 @@ smp_port_init(uint8_t port, struct rte_mempool *mbuf_pool, uint16_t num_queues) struct rte_eth_dev_info info; int retval; uint16_t q; + uint16_t nb_rxd = RX_RING_SIZE; + uint16_t nb_txd = TX_RING_SIZE; if (rte_eal_process_type() == RTE_PROC_SECONDARY) return 0; @@ -237,7 +236,7 @@ smp_port_init(uint8_t port, struct rte_mempool *mbuf_pool, uint16_t num_queues) if (port >= rte_eth_dev_count()) return -1; - printf("# Initialising port %u... ", (unsigned)port); + printf("# Initialising port %u... ", port); fflush(stdout); rte_eth_dev_info_get(port, &info); @@ -247,8 +246,12 @@ smp_port_init(uint8_t port, struct rte_mempool *mbuf_pool, uint16_t num_queues) if (retval < 0) return retval; + retval = rte_eth_dev_adjust_nb_rx_tx_desc(port, &nb_rxd, &nb_txd); + if (retval < 0) + return retval; + for (q = 0; q < rx_rings; q ++) { - retval = rte_eth_rx_queue_setup(port, q, RX_RING_SIZE, + retval = rte_eth_rx_queue_setup(port, q, nb_rxd, rte_eth_dev_socket_id(port), &info.default_rxconf, mbuf_pool); @@ -257,7 +260,7 @@ smp_port_init(uint8_t port, struct rte_mempool *mbuf_pool, uint16_t num_queues) } for (q = 0; q < tx_rings; q ++) { - retval = rte_eth_tx_queue_setup(port, q, TX_RING_SIZE, + retval = rte_eth_tx_queue_setup(port, q, nb_txd, rte_eth_dev_socket_id(port), NULL); if (retval < 0) @@ -358,11 +361,12 @@ lcore_main(void *arg __rte_unused) /* Check the link status of all ports in up to 9s, and print them finally */ static void -check_all_ports_link_status(uint8_t port_num, uint32_t port_mask) +check_all_ports_link_status(uint16_t port_num, uint32_t port_mask) { #define CHECK_INTERVAL 100 /* 100ms */ #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ - uint8_t portid, count, all_ports_up, print_flag = 0; + uint16_t portid; + uint8_t count, all_ports_up, print_flag = 0; struct rte_eth_link link; printf("\nChecking link status"); @@ -377,14 +381,13 @@ check_all_ports_link_status(uint8_t port_num, uint32_t port_mask) /* print link status if flag set */ if (print_flag == 1) { if (link.link_status) - printf("Port %d Link Up - speed %u " - "Mbps - %s\n", (uint8_t)portid, - (unsigned)link.link_speed, + printf( + "Port%d Link Up. Speed %u Mbps - %s\n", + portid, link.link_speed, (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? ("full-duplex") : ("half-duplex\n")); else - printf("Port %d Link Down\n", - (uint8_t)portid); + printf("Port %d Link Down\n", portid); continue; } /* clear all_ports_up flag if any link down */ diff --git a/dpdk/examples/netmap_compat/Makefile b/dpdk/examples/netmap_compat/Makefile index 52d80869..fd4630af 100644 --- a/dpdk/examples/netmap_compat/Makefile +++ b/dpdk/examples/netmap_compat/Makefile @@ -33,7 +33,7 @@ ifeq ($(RTE_SDK),) $(error "Please define RTE_SDK environment variable") endif -# Default target, can be overriden by command line or environment +# Default target, can be overridden by command line or environment RTE_TARGET ?= x86_64-native-linuxapp-gcc include $(RTE_SDK)/mk/rte.vars.mk diff --git a/dpdk/examples/netmap_compat/bridge/Makefile b/dpdk/examples/netmap_compat/bridge/Makefile index 50d96e81..ce38a345 100644 --- a/dpdk/examples/netmap_compat/bridge/Makefile +++ b/dpdk/examples/netmap_compat/bridge/Makefile @@ -33,7 +33,7 @@ ifeq ($(RTE_SDK),) $(error "Please define the RTE_SDK environment variable") endif -# Default target, can be overriden by command line or environment +# Default target, can be overridden by command line or environment RTE_TARGET ?= x86_64-native-linuxapp-gcc include $(RTE_SDK)/mk/rte.vars.mk @@ -42,6 +42,7 @@ ifneq ($(CONFIG_RTE_EXEC_ENV),"linuxapp") $(info This application can only operate in a linuxapp environment, \ please change the definition of the RTE_TARGET environment variable) all: +clean: else # binary name diff --git a/dpdk/examples/netmap_compat/bridge/bridge.c b/dpdk/examples/netmap_compat/bridge/bridge.c index 53f5fdb6..2f2b6baa 100644 --- a/dpdk/examples/netmap_compat/bridge/bridge.c +++ b/dpdk/examples/netmap_compat/bridge/bridge.c @@ -59,7 +59,7 @@ struct rte_eth_conf eth_conf = { .hw_ip_checksum = 0, .hw_vlan_filter = 0, .jumbo_frame = 0, - .hw_strip_crc = 0, + .hw_strip_crc = 1, }, .txmode = { .mq_mode = ETH_MQ_TX_NONE, diff --git a/dpdk/examples/netmap_compat/lib/compat_netmap.c b/dpdk/examples/netmap_compat/lib/compat_netmap.c index 112c551f..12b3fcbe 100644 --- a/dpdk/examples/netmap_compat/lib/compat_netmap.c +++ b/dpdk/examples/netmap_compat/lib/compat_netmap.c @@ -47,7 +47,6 @@ #include #include #include -#include #include #include @@ -129,7 +128,7 @@ static void netmap_unregif(uint32_t idx, uint32_t port); static int32_t -ifname_to_portid(const char *ifname, uint8_t *port) +ifname_to_portid(const char *ifname, uint16_t *port) { char *endptr; uint64_t portid; @@ -140,7 +139,7 @@ ifname_to_portid(const char *ifname, uint8_t *port) portid >= RTE_DIM(ports) || errno != 0) return -EINVAL; - *port = (uint8_t)portid; + *port = portid; return 0; } @@ -168,7 +167,7 @@ mbuf_to_slot(struct rte_mbuf *mbuf, struct netmap_ring *r, uint32_t index) /** * Given a Netmap ring and a slot index for that ring, construct a dpdk mbuf * from the data held in the buffer associated with the slot. - * Allocation/deallocation of the dpdk mbuf are the responsability of the + * Allocation/deallocation of the dpdk mbuf are the responsibility of the * caller. * Note that mbuf chains are not supported. */ @@ -222,10 +221,10 @@ fd_release(int32_t fd) } static int -check_nmreq(struct nmreq *req, uint8_t *port) +check_nmreq(struct nmreq *req, uint16_t *port) { int32_t rc; - uint8_t portid; + uint16_t portid; if (req == NULL) return -EINVAL; @@ -242,7 +241,7 @@ check_nmreq(struct nmreq *req, uint8_t *port) } if (ports[portid].pool == NULL) { - RTE_LOG(ERR, USER1, "Misconfigured portid %hhu\n", portid); + RTE_LOG(ERR, USER1, "Misconfigured portid %u\n", portid); return -EINVAL; } @@ -262,7 +261,7 @@ check_nmreq(struct nmreq *req, uint8_t *port) static int ioctl_niocginfo(__rte_unused int fd, void * param) { - uint8_t portid; + uint16_t portid; struct nmreq *req; int32_t rc; @@ -283,7 +282,7 @@ ioctl_niocginfo(__rte_unused int fd, void * param) } static void -netmap_ring_setup(struct netmap_ring *ring, uint8_t port, uint32_t ringid, +netmap_ring_setup(struct netmap_ring *ring, uint16_t port, uint32_t ringid, uint32_t num_slots) { uint32_t j; @@ -305,7 +304,7 @@ netmap_ring_setup(struct netmap_ring *ring, uint8_t port, uint32_t ringid, } static int -netmap_regif(struct nmreq *req, uint32_t idx, uint8_t port) +netmap_regif(struct nmreq *req, uint32_t idx, uint16_t port) { struct netmap_if *nmif; struct netmap_ring *ring; @@ -313,7 +312,7 @@ netmap_regif(struct nmreq *req, uint32_t idx, uint8_t port) int32_t rc; if (ports[port].fd < RTE_DIM(fd_port)) { - RTE_LOG(ERR, USER1, "port %hhu already in use by fd: %u\n", + RTE_LOG(ERR, USER1, "port %u already in use by fd: %u\n", port, IDX_TO_FD(ports[port].fd)); return -EBUSY; } @@ -399,7 +398,7 @@ netmap_regif(struct nmreq *req, uint32_t idx, uint8_t port) static int ioctl_niocregif(int32_t fd, void * param) { - uint8_t portid; + uint16_t portid; int32_t rc; uint32_t idx; struct nmreq *req; @@ -422,7 +421,7 @@ netmap_unregif(uint32_t idx, uint32_t port) { fd_port[idx].port = FD_PORT_RSRV; ports[port].fd = UINT32_MAX; - rte_eth_dev_stop((uint8_t)port); + rte_eth_dev_stop(port); } /** @@ -460,7 +459,7 @@ ioctl_niocunregif(int fd) * packets as it can hold coming from its dpdk port. */ static inline int -rx_sync_ring(struct netmap_ring *ring, uint8_t port, uint16_t ring_number, +rx_sync_ring(struct netmap_ring *ring, uint16_t port, uint16_t ring_number, uint16_t max_burst) { int32_t i, n_rx; @@ -513,7 +512,7 @@ rx_sync_if(uint32_t port) for (i = 0; i < nifp->ni_rx_rings + 1; i++) { r = NETMAP_RXRING(nifp, i); - rx_sync_ring(r, (uint8_t)port, (uint16_t)i, burst); + rx_sync_ring(r, port, (uint16_t)i, burst); rc += r->avail; } @@ -542,7 +541,7 @@ ioctl_niocrxsync(int fd) * buffers into rte_mbufs and sending them out on the rings's dpdk port. */ static int -tx_sync_ring(struct netmap_ring *ring, uint8_t port, uint16_t ring_number, +tx_sync_ring(struct netmap_ring *ring, uint16_t port, uint16_t ring_number, struct rte_mempool *pool, uint16_t max_burst) { uint32_t i, n_tx; @@ -608,7 +607,7 @@ tx_sync_if(uint32_t port) for (i = 0; i < nifp->ni_tx_rings + 1; i++) { r = NETMAP_TXRING(nifp, i); - tx_sync_ring(r, (uint8_t)port, (uint16_t)i, mp, burst); + tx_sync_ring(r, port, (uint16_t)i, mp, burst); rc += r->avail; } @@ -686,7 +685,7 @@ rte_netmap_init(const struct rte_netmap_conf *conf) int -rte_netmap_init_port(uint8_t portid, const struct rte_netmap_port_conf *conf) +rte_netmap_init_port(uint16_t portid, const struct rte_netmap_port_conf *conf) { int32_t ret; uint16_t i; @@ -696,17 +695,17 @@ rte_netmap_init_port(uint8_t portid, const struct rte_netmap_port_conf *conf) portid >= RTE_DIM(ports) || conf->nr_tx_rings > netmap.conf.max_rings || conf->nr_rx_rings > netmap.conf.max_rings) { - RTE_LOG(ERR, USER1, "%s(%hhu): invalid parameters\n", + RTE_LOG(ERR, USER1, "%s(%u): invalid parameters\n", __func__, portid); return -EINVAL; } - rx_slots = (uint16_t)rte_align32pow2(conf->nr_rx_slots); - tx_slots = (uint16_t)rte_align32pow2(conf->nr_tx_slots); + rx_slots = (uint16_t)rte_align32pow2(conf->nr_rx_slots); + tx_slots = (uint16_t)rte_align32pow2(conf->nr_tx_slots); if (tx_slots > netmap.conf.max_slots || rx_slots > netmap.conf.max_slots) { - RTE_LOG(ERR, USER1, "%s(%hhu): invalid parameters\n", + RTE_LOG(ERR, USER1, "%s(%u): invalid parameters\n", __func__, portid); return -EINVAL; } @@ -715,8 +714,17 @@ rte_netmap_init_port(uint8_t portid, const struct rte_netmap_port_conf *conf) conf->nr_tx_rings, conf->eth_conf); if (ret < 0) { - RTE_LOG(ERR, USER1, "Couldn't configure port %hhu\n", portid); - return ret; + RTE_LOG(ERR, USER1, "Couldn't configure port %u\n", portid); + return ret; + } + + ret = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &rx_slots, &tx_slots); + + if (ret < 0) { + RTE_LOG(ERR, USER1, + "Couldn't ot adjust number of descriptors for port %u\n", + portid); + return ret; } for (i = 0; i < conf->nr_tx_rings; i++) { @@ -725,8 +733,7 @@ rte_netmap_init_port(uint8_t portid, const struct rte_netmap_port_conf *conf) if (ret < 0) { RTE_LOG(ERR, USER1, - "Couldn't configure TX queue %"PRIu16" of " - "port %"PRIu8"\n", + "fail to configure TX queue %u of port %u\n", i, portid); return ret; } @@ -736,8 +743,7 @@ rte_netmap_init_port(uint8_t portid, const struct rte_netmap_port_conf *conf) if (ret < 0) { RTE_LOG(ERR, USER1, - "Couldn't configure RX queue %"PRIu16" of " - "port %"PRIu8"\n", + "fail to configure RX queue %u of port %u\n", i, portid); return ret; } diff --git a/dpdk/examples/netmap_compat/lib/compat_netmap.h b/dpdk/examples/netmap_compat/lib/compat_netmap.h index 3dc7a2f4..76b2d2b4 100644 --- a/dpdk/examples/netmap_compat/lib/compat_netmap.h +++ b/dpdk/examples/netmap_compat/lib/compat_netmap.h @@ -67,7 +67,7 @@ struct rte_netmap_port_conf { }; int rte_netmap_init(const struct rte_netmap_conf *conf); -int rte_netmap_init_port(uint8_t portid, +int rte_netmap_init_port(uint16_t portid, const struct rte_netmap_port_conf *conf); int rte_netmap_close(int fd); diff --git a/dpdk/examples/packet_ordering/Makefile b/dpdk/examples/packet_ordering/Makefile index 9e080a30..de066c4c 100644 --- a/dpdk/examples/packet_ordering/Makefile +++ b/dpdk/examples/packet_ordering/Makefile @@ -34,7 +34,7 @@ $(error "Please define RTE_SDK environment variable") endif # Default target, can be overridden by command line or environment -RTE_TARGET ?= x86_64-ivshmem-linuxapp-gcc +RTE_TARGET ?= x86_64-native-linuxapp-gcc include $(RTE_SDK)/mk/rte.vars.mk diff --git a/dpdk/examples/packet_ordering/main.c b/dpdk/examples/packet_ordering/main.c index 3c88b86e..3add7be4 100644 --- a/dpdk/examples/packet_ordering/main.c +++ b/dpdk/examples/packet_ordering/main.c @@ -216,7 +216,7 @@ parse_args(int argc, char **argv) } argv[optind-1] = prgname; - optind = 0; /* reset getopt lib */ + optind = 1; /* reset getopt lib */ return 0; } @@ -229,7 +229,7 @@ flush_tx_error_callback(struct rte_mbuf **unsent, uint16_t count, /* free the mbufs which failed from transmit */ app_stats.tx.ro_tx_failed_pkts += count; - RTE_LOG(DEBUG, REORDERAPP, "%s:Packet loss with tx_burst\n", __func__); + RTE_LOG_DP(DEBUG, REORDERAPP, "%s:Packet loss with tx_burst\n", __func__); pktmbuf_free_bulk(unsent, count); } @@ -269,27 +269,30 @@ configure_tx_buffers(struct rte_eth_dev_tx_buffer *tx_buffer[]) rte_eth_dev_socket_id(port_id)); if (tx_buffer[port_id] == NULL) rte_exit(EXIT_FAILURE, "Cannot allocate buffer for tx on port %u\n", - (unsigned) port_id); + port_id); rte_eth_tx_buffer_init(tx_buffer[port_id], MAX_PKTS_BURST); ret = rte_eth_tx_buffer_set_err_callback(tx_buffer[port_id], flush_tx_error_callback, NULL); if (ret < 0) - rte_exit(EXIT_FAILURE, "Cannot set error callback for " - "tx buffer on port %u\n", (unsigned) port_id); + rte_exit(EXIT_FAILURE, + "Cannot set error callback for tx buffer on port %u\n", + port_id); } return 0; } static inline int -configure_eth_port(uint8_t port_id) +configure_eth_port(uint16_t port_id) { struct ether_addr addr; const uint16_t rxRings = 1, txRings = 1; const uint8_t nb_ports = rte_eth_dev_count(); int ret; uint16_t q; + uint16_t nb_rxd = RX_DESC_PER_QUEUE; + uint16_t nb_txd = TX_DESC_PER_QUEUE; if (port_id > nb_ports) return -1; @@ -298,8 +301,12 @@ configure_eth_port(uint8_t port_id) if (ret != 0) return ret; + ret = rte_eth_dev_adjust_nb_rx_tx_desc(port_id, &nb_rxd, &nb_txd); + if (ret != 0) + return ret; + for (q = 0; q < rxRings; q++) { - ret = rte_eth_rx_queue_setup(port_id, q, RX_DESC_PER_QUEUE, + ret = rte_eth_rx_queue_setup(port_id, q, nb_rxd, rte_eth_dev_socket_id(port_id), NULL, mbuf_pool); if (ret < 0) @@ -307,7 +314,7 @@ configure_eth_port(uint8_t port_id) } for (q = 0; q < txRings; q++) { - ret = rte_eth_tx_queue_setup(port_id, q, TX_DESC_PER_QUEUE, + ret = rte_eth_tx_queue_setup(port_id, q, nb_txd, rte_eth_dev_socket_id(port_id), NULL); if (ret < 0) return ret; @@ -320,7 +327,7 @@ configure_eth_port(uint8_t port_id) rte_eth_macaddr_get(port_id, &addr); printf("Port %u MAC: %02"PRIx8" %02"PRIx8" %02"PRIx8 " %02"PRIx8" %02"PRIx8" %02"PRIx8"\n", - (unsigned)port_id, + port_id, addr.addr_bytes[0], addr.addr_bytes[1], addr.addr_bytes[2], addr.addr_bytes[3], addr.addr_bytes[4], addr.addr_bytes[5]); @@ -395,7 +402,7 @@ rx_thread(struct rte_ring *ring_out) uint32_t seqn = 0; uint16_t i, ret = 0; uint16_t nb_rx_pkts; - uint8_t port_id; + uint16_t port_id; struct rte_mbuf *pkts[MAX_PKTS_BURST]; RTE_LOG(INFO, REORDERAPP, "%s() started on lcore %u\n", __func__, @@ -410,7 +417,7 @@ rx_thread(struct rte_ring *ring_out) nb_rx_pkts = rte_eth_rx_burst(port_id, 0, pkts, MAX_PKTS_BURST); if (nb_rx_pkts == 0) { - RTE_LOG(DEBUG, REORDERAPP, + RTE_LOG_DP(DEBUG, REORDERAPP, "%s():Received zero packets\n", __func__); continue; } @@ -421,8 +428,8 @@ rx_thread(struct rte_ring *ring_out) pkts[i++]->seqn = seqn++; /* enqueue to rx_to_workers ring */ - ret = rte_ring_enqueue_burst(ring_out, (void *) pkts, - nb_rx_pkts); + ret = rte_ring_enqueue_burst(ring_out, + (void *)pkts, nb_rx_pkts, NULL); app_stats.rx.enqueue_pkts += ret; if (unlikely(ret < nb_rx_pkts)) { app_stats.rx.enqueue_failed_pkts += @@ -462,7 +469,7 @@ worker_thread(void *args_ptr) /* dequeue the mbufs from rx_to_workers ring */ burst_size = rte_ring_dequeue_burst(ring_in, - (void *)burst_buffer, MAX_PKTS_BURST); + (void *)burst_buffer, MAX_PKTS_BURST, NULL); if (unlikely(burst_size == 0)) continue; @@ -473,7 +480,8 @@ worker_thread(void *args_ptr) burst_buffer[i++]->port ^= xor_val; /* enqueue the modified mbufs to workers_to_tx ring */ - ret = rte_ring_enqueue_burst(ring_out, (void *)burst_buffer, burst_size); + ret = rte_ring_enqueue_burst(ring_out, (void *)burst_buffer, + burst_size, NULL); __sync_fetch_and_add(&app_stats.wkr.enqueue_pkts, ret); if (unlikely(ret < burst_size)) { /* Return the mbufs to their respective pool, dropping packets */ @@ -509,7 +517,7 @@ send_thread(struct send_thread_args *args) /* deque the mbufs from workers_to_tx ring */ nb_dq_mbufs = rte_ring_dequeue_burst(args->ring_in, - (void *)mbufs, MAX_PKTS_BURST); + (void *)mbufs, MAX_PKTS_BURST, NULL); if (unlikely(nb_dq_mbufs == 0)) continue; @@ -522,7 +530,7 @@ send_thread(struct send_thread_args *args) if (ret == -1 && rte_errno == ERANGE) { /* Too early pkts should be transmitted out directly */ - RTE_LOG(DEBUG, REORDERAPP, + RTE_LOG_DP(DEBUG, REORDERAPP, "%s():Cannot reorder early packet " "direct enqueuing to TX\n", __func__); outp = mbufs[i]->port; @@ -594,7 +602,7 @@ tx_thread(struct rte_ring *ring_in) /* deque the mbufs from workers_to_tx ring */ dqnum = rte_ring_dequeue_burst(ring_in, - (void *)mbufs, MAX_PKTS_BURST); + (void *)mbufs, MAX_PKTS_BURST, NULL); if (unlikely(dqnum == 0)) continue; @@ -625,8 +633,8 @@ main(int argc, char **argv) int ret; unsigned nb_ports; unsigned int lcore_id, last_lcore_id, master_lcore_id; - uint8_t port_id; - uint8_t nb_ports_available; + uint16_t port_id; + uint16_t nb_ports_available; struct worker_thread_args worker_args = {NULL, NULL}; struct send_thread_args send_args = {NULL, NULL}; struct rte_ring *rx_to_workers; @@ -680,7 +688,7 @@ main(int argc, char **argv) continue; } /* init port */ - printf("Initializing port %u... done\n", (unsigned) port_id); + printf("Initializing port %u... done\n", port_id); if (configure_eth_port(port_id) != 0) rte_exit(EXIT_FAILURE, "Cannot initialize port %"PRIu8"\n", diff --git a/dpdk/examples/performance-thread/Makefile b/dpdk/examples/performance-thread/Makefile index d19f8489..0c5edfdb 100644 --- a/dpdk/examples/performance-thread/Makefile +++ b/dpdk/examples/performance-thread/Makefile @@ -38,8 +38,8 @@ RTE_TARGET ?= x86_64-native-linuxapp-gcc include $(RTE_SDK)/mk/rte.vars.mk -ifneq ($(CONFIG_RTE_ARCH),"x86_64") -$(error This application is only supported for x86_64 targets) +ifeq ($(filter y,$(CONFIG_RTE_ARCH_X86_64) $(CONFIG_RTE_ARCH_ARM64)),) +$(error This application is only supported for x86_64 and arm64 targets) endif DIRS-y += l3fwd-thread diff --git a/dpdk/examples/performance-thread/common/arch/arm64/ctx.c b/dpdk/examples/performance-thread/common/arch/arm64/ctx.c new file mode 100644 index 00000000..d0eacaa6 --- /dev/null +++ b/dpdk/examples/performance-thread/common/arch/arm64/ctx.c @@ -0,0 +1,90 @@ +/* + * BSD LICENSE + * + * Copyright (C) Cavium, Inc. 2017. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include + +void +ctx_switch(struct ctx *new_ctx __rte_unused, struct ctx *curr_ctx __rte_unused) +{ + /* SAVE CURRENT CONTEXT */ + asm volatile ( + /* Save SP */ + "mov x3, sp\n" + "str x3, [x1, #0]\n" + + /* Save FP and LR */ + "stp x29, x30, [x1, #8]\n" + + /* Save Callee Saved Regs x19 - x28 */ + "stp x19, x20, [x1, #24]\n" + "stp x21, x22, [x1, #40]\n" + "stp x23, x24, [x1, #56]\n" + "stp x25, x26, [x1, #72]\n" + "stp x27, x28, [x1, #88]\n" + + /* + * Save bottom 64-bits of Callee Saved + * SIMD Regs v8 - v15 + */ + "stp d8, d9, [x1, #104]\n" + "stp d10, d11, [x1, #120]\n" + "stp d12, d13, [x1, #136]\n" + "stp d14, d15, [x1, #152]\n" + ); + + /* RESTORE NEW CONTEXT */ + asm volatile ( + /* Restore SP */ + "ldr x3, [x0, #0]\n" + "mov sp, x3\n" + + /* Restore FP and LR */ + "ldp x29, x30, [x0, #8]\n" + + /* Restore Callee Saved Regs x19 - x28 */ + "ldp x19, x20, [x0, #24]\n" + "ldp x21, x22, [x0, #40]\n" + "ldp x23, x24, [x0, #56]\n" + "ldp x25, x26, [x0, #72]\n" + "ldp x27, x28, [x0, #88]\n" + + /* + * Restore bottom 64-bits of Callee Saved + * SIMD Regs v8 - v15 + */ + "ldp d8, d9, [x0, #104]\n" + "ldp d10, d11, [x0, #120]\n" + "ldp d12, d13, [x0, #136]\n" + "ldp d14, d15, [x0, #152]\n" + ); +} diff --git a/dpdk/examples/performance-thread/common/arch/arm64/ctx.h b/dpdk/examples/performance-thread/common/arch/arm64/ctx.h new file mode 100644 index 00000000..38c86ce6 --- /dev/null +++ b/dpdk/examples/performance-thread/common/arch/arm64/ctx.h @@ -0,0 +1,83 @@ +/* + * BSD LICENSE + * + * Copyright (C) Cavium, Inc. 2017. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef CTX_H +#define CTX_H + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * CPU context registers + */ +struct ctx { + void *sp; /* 0 */ + void *fp; /* 8 */ + void *lr; /* 16 */ + + /* Callee Saved Generic Registers */ + void *r19; /* 24 */ + void *r20; /* 32 */ + void *r21; /* 40 */ + void *r22; /* 48 */ + void *r23; /* 56 */ + void *r24; /* 64 */ + void *r25; /* 72 */ + void *r26; /* 80 */ + void *r27; /* 88 */ + void *r28; /* 96 */ + + /* + * Callee Saved SIMD Registers. Only the bottom 64-bits + * of these registers needs to be saved. + */ + void *v8; /* 104 */ + void *v9; /* 112 */ + void *v10; /* 120 */ + void *v11; /* 128 */ + void *v12; /* 136 */ + void *v13; /* 144 */ + void *v14; /* 152 */ + void *v15; /* 160 */ +}; + + +void +ctx_switch(struct ctx *new_ctx, struct ctx *curr_ctx); + + +#ifdef __cplusplus +} +#endif + +#endif /* RTE_CTX_H_ */ diff --git a/dpdk/examples/performance-thread/common/arch/arm64/stack.h b/dpdk/examples/performance-thread/common/arch/arm64/stack.h new file mode 100644 index 00000000..fa3b31e9 --- /dev/null +++ b/dpdk/examples/performance-thread/common/arch/arm64/stack.h @@ -0,0 +1,84 @@ +/* + * BSD LICENSE + * + * Copyright (C) Cavium, Inc. 2017. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef STACK_H +#define STACK_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "lthread_int.h" + +/* + * Sets up the initial stack for the lthread. + */ +static inline void +arch_set_stack(struct lthread *lt, void *func) +{ + void **stack_top = (void *)((char *)(lt->stack) + lt->stack_size); + + /* + * Align stack_top to 16 bytes. Arm64 has the constraint that the + * stack pointer must always be quad-word aligned. + */ + stack_top = (void **)(((unsigned long)(stack_top)) & ~0xfUL); + + /* + * First Stack Frame + */ + stack_top[0] = NULL; + stack_top[-1] = NULL; + + /* + * Initialize the context + */ + lt->ctx.fp = &stack_top[-1]; + lt->ctx.sp = &stack_top[-2]; + + /* + * Here only the address of _lthread_exec is saved as the link + * register value. The argument to _lthread_exec i.e the address of + * the lthread struct is not saved. This is because the first + * argument to ctx_switch is the address of the new context, + * which also happens to be the address of required lthread struct. + * So while returning from ctx_switch into _thread_exec, parameter + * register x0 will always contain the required value. + */ + lt->ctx.lr = func; +} + +#ifdef __cplusplus +} +#endif + +#endif /* STACK_H_ */ diff --git a/dpdk/examples/performance-thread/common/arch/x86/ctx.h b/dpdk/examples/performance-thread/common/arch/x86/ctx.h index 03860508..a41ce05a 100644 --- a/dpdk/examples/performance-thread/common/arch/x86/ctx.h +++ b/dpdk/examples/performance-thread/common/arch/x86/ctx.h @@ -35,6 +35,10 @@ #ifndef CTX_H #define CTX_H +#ifdef __cplusplus +extern "C" { +#endif + /* * CPU context registers */ @@ -54,4 +58,8 @@ void ctx_switch(struct ctx *new_ctx, struct ctx *curr_ctx); +#ifdef __cplusplus +} +#endif + #endif /* RTE_CTX_H_ */ diff --git a/dpdk/examples/performance-thread/common/arch/x86/stack.h b/dpdk/examples/performance-thread/common/arch/x86/stack.h new file mode 100644 index 00000000..98723ba3 --- /dev/null +++ b/dpdk/examples/performance-thread/common/arch/x86/stack.h @@ -0,0 +1,94 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2015 Intel Corporation. All rights reserved. + * Copyright(c) Cavium, Inc. 2017. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * Some portions of this software is derived from the + * https://github.com/halayli/lthread which carrys the following license. + * + * Copyright (C) 2012, Hasan Alayli + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + + +#ifndef STACK_H +#define STACK_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "lthread_int.h" + +/* + * Sets up the initial stack for the lthread. + */ +static inline void +arch_set_stack(struct lthread *lt, void *func) +{ + char *stack_top = (char *)(lt->stack) + lt->stack_size; + void **s = (void **)stack_top; + + /* set initial context */ + s[-3] = NULL; + s[-2] = (void *)lt; + lt->ctx.rsp = (void *)(stack_top - (4 * sizeof(void *))); + lt->ctx.rbp = (void *)(stack_top - (3 * sizeof(void *))); + lt->ctx.rip = func; +} + +#ifdef __cplusplus +} +#endif + +#endif /* STACK_H_ */ diff --git a/dpdk/examples/performance-thread/common/common.mk b/dpdk/examples/performance-thread/common/common.mk index d3de5fc6..f1f05fdd 100644 --- a/dpdk/examples/performance-thread/common/common.mk +++ b/dpdk/examples/performance-thread/common/common.mk @@ -30,13 +30,21 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# list the C files belonhing to the lthread subsystem, these are common to all lthread apps -SRCS-y += ../common/lthread.c \ - ../common/lthread_sched.c \ - ../common/lthread_cond.c \ - ../common/lthread_tls.c \ - ../common/lthread_mutex.c \ - ../common/lthread_diag.c \ - ../common/arch/x86/ctx.c +# list the C files belonging to the lthread subsystem, these are common to all +# lthread apps. Any makefile including this should set VPATH to include this +# directory path +# -INCLUDES += -I$(RTE_SDK)/examples/performance-thread/common/ -I$(RTE_SDK)/examples/performance-thread/common/arch/x86/ +MKFILE_PATH=$(abspath $(dir $(lastword $(MAKEFILE_LIST)))) + +ifeq ($(CONFIG_RTE_ARCH_X86_64),y) +ARCH_PATH += $(MKFILE_PATH)/arch/x86 +else ifeq ($(CONFIG_RTE_ARCH_ARM64),y) +ARCH_PATH += $(MKFILE_PATH)/arch/arm64 +endif + +VPATH := $(MKFILE_PATH) $(ARCH_PATH) + +SRCS-y += lthread.c lthread_sched.c lthread_cond.c lthread_tls.c lthread_mutex.c lthread_diag.c ctx.c + +INCLUDES += -I$(MKFILE_PATH) -I$(ARCH_PATH) diff --git a/dpdk/examples/performance-thread/common/lthread.c b/dpdk/examples/performance-thread/common/lthread.c index 062275a4..7d76c8c4 100644 --- a/dpdk/examples/performance-thread/common/lthread.c +++ b/dpdk/examples/performance-thread/common/lthread.c @@ -76,6 +76,7 @@ #include #include +#include #include "lthread_api.h" #include "lthread.h" @@ -190,19 +191,11 @@ _lthread_init(struct lthread *lt, */ void _lthread_set_stack(struct lthread *lt, void *stack, size_t stack_size) { - char *stack_top = (char *)stack + stack_size; - void **s = (void **)stack_top; - /* set stack */ lt->stack = stack; lt->stack_size = stack_size; - /* set initial context */ - s[-3] = NULL; - s[-2] = (void *)lt; - lt->ctx.rsp = (void *)(stack_top - (4 * sizeof(void *))); - lt->ctx.rbp = (void *)(stack_top - (3 * sizeof(void *))); - lt->ctx.rip = (void *)_lthread_exec; + arch_set_stack(lt, _lthread_exec); } /* diff --git a/dpdk/examples/performance-thread/common/lthread.h b/dpdk/examples/performance-thread/common/lthread.h index 8c77af82..0cde5919 100644 --- a/dpdk/examples/performance-thread/common/lthread.h +++ b/dpdk/examples/performance-thread/common/lthread.h @@ -62,6 +62,10 @@ #ifndef LTHREAD_H_ #define LTHREAD_H_ +#ifdef __cplusplus +extern "C" { +#endif + #include #include "lthread_api.h" @@ -83,7 +87,7 @@ int _lthread_desched_sleep(struct lthread *lt); void _lthread_free(struct lthread *lt); -struct lthread_sched *_lthread_sched_get(int lcore_id); +struct lthread_sched *_lthread_sched_get(unsigned int lcore_id); struct lthread_stack *_stack_alloc(void); @@ -96,4 +100,8 @@ _lthread_init(struct lthread *lt, void _lthread_set_stack(struct lthread *lt, void *stack, size_t stack_size); +#ifdef __cplusplus +} +#endif + #endif /* LTHREAD_H_ */ diff --git a/dpdk/examples/performance-thread/common/lthread_api.h b/dpdk/examples/performance-thread/common/lthread_api.h index ec976103..ff245a08 100644 --- a/dpdk/examples/performance-thread/common/lthread_api.h +++ b/dpdk/examples/performance-thread/common/lthread_api.h @@ -124,6 +124,10 @@ #ifndef LTHREAD_H #define LTHREAD_H +#ifdef __cplusplus +extern "C" { +#endif + #include #include #include @@ -829,4 +833,8 @@ int lthread_cond_signal(struct lthread_cond *c); */ int lthread_cond_broadcast(struct lthread_cond *c); +#ifdef __cplusplus +} +#endif + #endif /* LTHREAD_H */ diff --git a/dpdk/examples/performance-thread/common/lthread_cond.h b/dpdk/examples/performance-thread/common/lthread_cond.h index 5bd02a7d..5e5f14be 100644 --- a/dpdk/examples/performance-thread/common/lthread_cond.h +++ b/dpdk/examples/performance-thread/common/lthread_cond.h @@ -62,6 +62,10 @@ #ifndef LTHREAD_COND_H_ #define LTHREAD_COND_H_ +#ifdef __cplusplus +extern "C" { +#endif + #include "lthread_queue.h" #define MAX_COND_NAME_SIZE 64 @@ -74,4 +78,8 @@ struct lthread_cond { uint64_t diag_ref; /* optional ref to user diag data */ } __rte_cache_aligned; +#ifdef __cplusplus +} +#endif + #endif /* LTHREAD_COND_H_ */ diff --git a/dpdk/examples/performance-thread/common/lthread_diag.c b/dpdk/examples/performance-thread/common/lthread_diag.c index bce1a0c3..b5007d77 100644 --- a/dpdk/examples/performance-thread/common/lthread_diag.c +++ b/dpdk/examples/performance-thread/common/lthread_diag.c @@ -296,8 +296,7 @@ _lthread_diag_default_cb(uint64_t time, struct lthread *lt, int diag_event, /* * plug in default diag callback with mask off */ -void _lthread_diag_ctor(void)__attribute__((constructor)); -void _lthread_diag_ctor(void) +RTE_INIT(_lthread_diag_ctor) { diag_cb = _lthread_diag_default_cb; diag_mask = 0; diff --git a/dpdk/examples/performance-thread/common/lthread_diag.h b/dpdk/examples/performance-thread/common/lthread_diag.h index 2877d311..3dce8e0e 100644 --- a/dpdk/examples/performance-thread/common/lthread_diag.h +++ b/dpdk/examples/performance-thread/common/lthread_diag.h @@ -34,6 +34,10 @@ #ifndef LTHREAD_DIAG_H_ #define LTHREAD_DIAG_H_ +#ifdef __cplusplus +extern "C" { +#endif + #include #include @@ -129,4 +133,9 @@ extern uint64_t diag_mask; #define DIAG_USED __rte_unused #endif /* LTHREAD_DIAG */ + +#ifdef __cplusplus +} +#endif + #endif /* LTHREAD_DIAG_H_ */ diff --git a/dpdk/examples/performance-thread/common/lthread_diag_api.h b/dpdk/examples/performance-thread/common/lthread_diag_api.h index 7ee514f8..2fda0951 100644 --- a/dpdk/examples/performance-thread/common/lthread_diag_api.h +++ b/dpdk/examples/performance-thread/common/lthread_diag_api.h @@ -33,6 +33,10 @@ #ifndef LTHREAD_DIAG_API_H_ #define LTHREAD_DIAG_API_H_ +#ifdef __cplusplus +extern "C" { +#endif + #include #include @@ -322,4 +326,8 @@ lthread_cond_diag_ref(struct lthread_cond *c); uint64_t lthread_mutex_diag_ref(struct lthread_mutex *m); +#ifdef __cplusplus +} +#endif + #endif /* LTHREAD_DIAG_API_H_ */ diff --git a/dpdk/examples/performance-thread/common/lthread_int.h b/dpdk/examples/performance-thread/common/lthread_int.h index b858b55e..e1da2462 100644 --- a/dpdk/examples/performance-thread/common/lthread_int.h +++ b/dpdk/examples/performance-thread/common/lthread_int.h @@ -59,9 +59,12 @@ * SUCH DAMAGE. */ #ifndef LTHREAD_INT_H -#include #define LTHREAD_INT_H +#ifdef __cplusplus +extern "C" { +#endif + #include #include #include @@ -69,10 +72,10 @@ #include #include +#include #include #include #include -#include #include #include #include @@ -197,4 +200,8 @@ struct lthread { uint64_t diag_ref; /* ref to user diag data */ } __rte_cache_aligned; +#ifdef __cplusplus +} +#endif + #endif /* LTHREAD_INT_H */ diff --git a/dpdk/examples/performance-thread/common/lthread_mutex.c b/dpdk/examples/performance-thread/common/lthread_mutex.c index c1bc6271..c06d3d51 100644 --- a/dpdk/examples/performance-thread/common/lthread_mutex.c +++ b/dpdk/examples/performance-thread/common/lthread_mutex.c @@ -173,7 +173,7 @@ int lthread_mutex_lock(struct lthread_mutex *m) return 0; } -/* try to lock a mutex but dont block */ +/* try to lock a mutex but don't block */ int lthread_mutex_trylock(struct lthread_mutex *m) { struct lthread *lt = THIS_LTHREAD; diff --git a/dpdk/examples/performance-thread/common/lthread_mutex.h b/dpdk/examples/performance-thread/common/lthread_mutex.h index 4d30b2e7..e78db91d 100644 --- a/dpdk/examples/performance-thread/common/lthread_mutex.h +++ b/dpdk/examples/performance-thread/common/lthread_mutex.h @@ -35,6 +35,10 @@ #ifndef LTHREAD_MUTEX_H_ #define LTHREAD_MUTEX_H_ +#ifdef __cplusplus +extern "C" { +#endif + #include "lthread_queue.h" @@ -49,4 +53,8 @@ struct lthread_mutex { uint64_t diag_ref; /* optional ref to user diag data */ } __rte_cache_aligned; +#ifdef __cplusplus +} +#endif + #endif /* LTHREAD_MUTEX_H_ */ diff --git a/dpdk/examples/performance-thread/common/lthread_objcache.h b/dpdk/examples/performance-thread/common/lthread_objcache.h index d7e35825..6e5195ba 100644 --- a/dpdk/examples/performance-thread/common/lthread_objcache.h +++ b/dpdk/examples/performance-thread/common/lthread_objcache.h @@ -33,6 +33,10 @@ #ifndef LTHREAD_OBJCACHE_H_ #define LTHREAD_OBJCACHE_H_ +#ifdef __cplusplus +extern "C" { +#endif + #include #include @@ -154,5 +158,8 @@ _lthread_objcache_free(struct lthread_objcache *c, void *obj) } +#ifdef __cplusplus +} +#endif #endif /* LTHREAD_OBJCACHE_H_ */ diff --git a/dpdk/examples/performance-thread/common/lthread_pool.h b/dpdk/examples/performance-thread/common/lthread_pool.h index 27680eab..315a2e21 100644 --- a/dpdk/examples/performance-thread/common/lthread_pool.h +++ b/dpdk/examples/performance-thread/common/lthread_pool.h @@ -69,6 +69,10 @@ #ifndef LTHREAD_POOL_H_ #define LTHREAD_POOL_H_ +#ifdef __cplusplus +extern "C" { +#endif + #include #include #include @@ -170,7 +174,7 @@ _qnode_pool_create(const char *name, int prealloc_size) { /* * Insert a node into the pool */ -static inline void __attribute__ ((always_inline)) +static __rte_always_inline void _qnode_pool_insert(struct qnode_pool *p, struct qnode *n) { n->next = NULL; @@ -194,7 +198,7 @@ _qnode_pool_insert(struct qnode_pool *p, struct qnode *n) * last item from the queue incurs the penalty of an atomic exchange. Since the * pool is maintained with a bulk pre-allocation the cost of this is amortised. */ -static inline struct qnode *__attribute__ ((always_inline)) +static __rte_always_inline struct qnode * _pool_remove(struct qnode_pool *p) { struct qnode *head; @@ -235,7 +239,7 @@ _pool_remove(struct qnode_pool *p) * This adds a retry to the _pool_remove function * defined above */ -static inline struct qnode *__attribute__ ((always_inline)) +static __rte_always_inline struct qnode * _qnode_pool_remove(struct qnode_pool *p) { struct qnode *n; @@ -255,7 +259,7 @@ _qnode_pool_remove(struct qnode_pool *p) * Allocate a node from the pool * If the pool is empty add mode nodes */ -static inline struct qnode *__attribute__ ((always_inline)) +static __rte_always_inline struct qnode * _qnode_alloc(void) { struct qnode_pool *p = (THIS_SCHED)->qnode_pool; @@ -300,7 +304,7 @@ _qnode_alloc(void) /* * free a queue node to the per scheduler pool from which it came */ -static inline void __attribute__ ((always_inline)) +static __rte_always_inline void _qnode_free(struct qnode *n) { struct qnode_pool *p = n->pool; @@ -328,5 +332,8 @@ _qnode_pool_destroy(struct qnode_pool *p) return 0; } +#ifdef __cplusplus +} +#endif #endif /* LTHREAD_POOL_H_ */ diff --git a/dpdk/examples/performance-thread/common/lthread_queue.h b/dpdk/examples/performance-thread/common/lthread_queue.h index 2c55fcec..833ed92b 100644 --- a/dpdk/examples/performance-thread/common/lthread_queue.h +++ b/dpdk/examples/performance-thread/common/lthread_queue.h @@ -69,6 +69,10 @@ #ifndef LTHREAD_QUEUE_H_ #define LTHREAD_QUEUE_H_ +#ifdef __cplusplus +extern "C" { +#endif + #include #include @@ -150,7 +154,7 @@ _lthread_queue_create(const char *name) /** * Return true if the queue is empty */ -static inline int __attribute__ ((always_inline)) +static __rte_always_inline int _lthread_queue_empty(struct lthread_queue *q) { return q->tail == q->head; @@ -181,7 +185,7 @@ RTE_DECLARE_PER_LCORE(struct lthread_sched *, this_sched); * Insert a node into a queue * this implementation is multi producer safe */ -static inline struct qnode *__attribute__ ((always_inline)) +static __rte_always_inline struct qnode * _lthread_queue_insert_mp(struct lthread_queue *q, void *data) { @@ -215,7 +219,7 @@ _lthread_queue_insert_mp(struct lthread_queue * Insert an node into a queue in single producer mode * this implementation is NOT mult producer safe */ -static inline struct qnode *__attribute__ ((always_inline)) +static __rte_always_inline struct qnode * _lthread_queue_insert_sp(struct lthread_queue *q, void *data) { @@ -243,7 +247,7 @@ _lthread_queue_insert_sp(struct lthread_queue /* * Remove a node from a queue */ -static inline void *__attribute__ ((always_inline)) +static __rte_always_inline void * _lthread_queue_poll(struct lthread_queue *q) { void *data = NULL; @@ -274,7 +278,7 @@ _lthread_queue_poll(struct lthread_queue *q) /* * Remove a node from a queue */ -static inline void *__attribute__ ((always_inline)) +static __rte_always_inline void * _lthread_queue_remove(struct lthread_queue *q) { void *data = NULL; @@ -298,5 +302,8 @@ _lthread_queue_remove(struct lthread_queue *q) return NULL; } +#ifdef __cplusplus +} +#endif #endif /* LTHREAD_QUEUE_H_ */ diff --git a/dpdk/examples/performance-thread/common/lthread_sched.c b/dpdk/examples/performance-thread/common/lthread_sched.c index c64c21ff..779aeb17 100644 --- a/dpdk/examples/performance-thread/common/lthread_sched.c +++ b/dpdk/examples/performance-thread/common/lthread_sched.c @@ -117,8 +117,7 @@ uint64_t diag_mask; /* constructor */ -void lthread_sched_ctor(void) __attribute__ ((constructor)); -void lthread_sched_ctor(void) +RTE_INIT(lthread_sched_ctor) { memset(schedcore, 0, sizeof(schedcore)); rte_atomic16_init(&num_schedulers); @@ -369,8 +368,8 @@ void lthread_scheduler_shutdown_all(void) /* * Resume a suspended lthread */ -static inline void -_lthread_resume(struct lthread *lt) __attribute__ ((always_inline)); +static __rte_always_inline void +_lthread_resume(struct lthread *lt); static inline void _lthread_resume(struct lthread *lt) { struct lthread_sched *sched = THIS_SCHED; @@ -562,11 +561,14 @@ void lthread_run(void) * Return the scheduler for this lcore * */ -struct lthread_sched *_lthread_sched_get(int lcore_id) +struct lthread_sched *_lthread_sched_get(unsigned int lcore_id) { - if (lcore_id > LTHREAD_MAX_LCORES) - return NULL; - return schedcore[lcore_id]; + struct lthread_sched *res = NULL; + + if (lcore_id < LTHREAD_MAX_LCORES) + res = schedcore[lcore_id]; + + return res; } /* @@ -578,10 +580,9 @@ int lthread_set_affinity(unsigned lcoreid) struct lthread *lt = THIS_LTHREAD; struct lthread_sched *dest_sched; - if (unlikely(lcoreid > LTHREAD_MAX_LCORES)) + if (unlikely(lcoreid >= LTHREAD_MAX_LCORES)) return POSIX_ERRNO(EINVAL); - DIAG_EVENT(lt, LT_DIAG_LTHREAD_AFFINITY, lcoreid, 0); dest_sched = schedcore[lcoreid]; diff --git a/dpdk/examples/performance-thread/common/lthread_sched.h b/dpdk/examples/performance-thread/common/lthread_sched.h index 4ce56c27..aa2f0c48 100644 --- a/dpdk/examples/performance-thread/common/lthread_sched.h +++ b/dpdk/examples/performance-thread/common/lthread_sched.h @@ -62,6 +62,10 @@ #ifndef LTHREAD_SCHED_H_ #define LTHREAD_SCHED_H_ +#ifdef __cplusplus +extern "C" { +#endif + #include "lthread_int.h" #include "lthread_queue.h" #include "lthread_objcache.h" @@ -108,8 +112,8 @@ static inline uint64_t _sched_now(void) return 1; } -static inline void -_affinitize(void) __attribute__ ((always_inline)); +static __rte_always_inline void +_affinitize(void); static inline void _affinitize(void) { @@ -119,8 +123,8 @@ _affinitize(void) ctx_switch(&(THIS_SCHED)->ctx, <->ctx); } -static inline void -_suspend(void) __attribute__ ((always_inline)); +static __rte_always_inline void +_suspend(void); static inline void _suspend(void) { @@ -132,8 +136,8 @@ _suspend(void) (THIS_SCHED)->nb_blocked_threads--; } -static inline void -_reschedule(void) __attribute__ ((always_inline)); +static __rte_always_inline void +_reschedule(void); static inline void _reschedule(void) { @@ -148,5 +152,8 @@ extern struct lthread_sched *schedcore[]; void _sched_timer_cb(struct rte_timer *tim, void *arg); void _sched_shutdown(__rte_unused void *arg); +#ifdef __cplusplus +} +#endif #endif /* LTHREAD_SCHED_H_ */ diff --git a/dpdk/examples/performance-thread/common/lthread_timer.h b/dpdk/examples/performance-thread/common/lthread_timer.h index b5e6fb0e..7c03d673 100644 --- a/dpdk/examples/performance-thread/common/lthread_timer.h +++ b/dpdk/examples/performance-thread/common/lthread_timer.h @@ -35,6 +35,10 @@ #ifndef LTHREAD_TIMER_H_ #define LTHREAD_TIMER_H_ +#ifdef __cplusplus +extern "C" { +#endif + #include "lthread_int.h" #include "lthread_sched.h" @@ -42,11 +46,22 @@ static inline uint64_t _ns_to_clks(uint64_t ns) { - unsigned __int128 clkns = rte_get_tsc_hz(); + /* + * clkns needs to be divided by 1E9 to get ns clocks. However, + * dividing by this first would lose a lot of accuracy. + * Dividing after a multiply by ns, could cause overflow of + * uint64_t if ns is about 5 seconds [if we assume a max tsc + * rate of 4GHz]. Therefore we first divide by 1E4, then + * multiply and finally divide by 1E5. This allows ns to be + * values many hours long, without overflow, while still keeping + * reasonable accuracy. + */ + uint64_t clkns = rte_get_tsc_hz() / 1e4; clkns *= ns; - clkns /= 1000000000; - return (uint64_t) clkns; + clkns /= 1e5; + + return clkns; } @@ -75,5 +90,8 @@ _timer_stop(struct lthread *lt) } } +#ifdef __cplusplus +} +#endif #endif /* LTHREAD_TIMER_H_ */ diff --git a/dpdk/examples/performance-thread/common/lthread_tls.c b/dpdk/examples/performance-thread/common/lthread_tls.c index 6876f831..2259fad4 100644 --- a/dpdk/examples/performance-thread/common/lthread_tls.c +++ b/dpdk/examples/performance-thread/common/lthread_tls.c @@ -42,7 +42,6 @@ #include #include #include -#include #include #include @@ -63,9 +62,7 @@ RTE_DEFINE_PER_LTHREAD(void *, dummy); static struct lthread_key key_table[LTHREAD_MAX_KEYS]; -void lthread_tls_ctor(void) __attribute__((constructor)); - -void lthread_tls_ctor(void) +RTE_INIT(thread_tls_ctor) { key_pool = NULL; key_pool_init = 0; @@ -199,11 +196,12 @@ void _lthread_tls_destroy(struct lthread *lt) void *lthread_getspecific(unsigned int k) { + void *res = NULL; - if (k > LTHREAD_MAX_KEYS) - return NULL; + if (k < LTHREAD_MAX_KEYS) + res = THIS_LTHREAD->tls->data[k]; - return THIS_LTHREAD->tls->data[k]; + return res; } /* @@ -213,7 +211,7 @@ void */ int lthread_setspecific(unsigned int k, const void *data) { - if (k > LTHREAD_MAX_KEYS) + if (k >= LTHREAD_MAX_KEYS) return POSIX_ERRNO(EINVAL); int n = THIS_LTHREAD->tls->nb_keys_inuse; diff --git a/dpdk/examples/performance-thread/common/lthread_tls.h b/dpdk/examples/performance-thread/common/lthread_tls.h index 86cbfadc..fff3c0db 100644 --- a/dpdk/examples/performance-thread/common/lthread_tls.h +++ b/dpdk/examples/performance-thread/common/lthread_tls.h @@ -34,6 +34,10 @@ #ifndef LTHREAD_TLS_H_ #define LTHREAD_TLS_H_ +#ifdef __cplusplus +extern "C" { +#endif + #include "lthread_api.h" #define RTE_PER_LTHREAD_SECTION_SIZE \ @@ -53,5 +57,8 @@ void _lthread_tls_destroy(struct lthread *lt); void _lthread_key_pool_init(void); void _lthread_tls_alloc(struct lthread *lt); +#ifdef __cplusplus +} +#endif #endif /* LTHREAD_TLS_H_ */ diff --git a/dpdk/examples/performance-thread/l3fwd-thread/main.c b/dpdk/examples/performance-thread/l3fwd-thread/main.c index fdc90b28..fa65234f 100644 --- a/dpdk/examples/performance-thread/l3fwd-thread/main.c +++ b/dpdk/examples/performance-thread/l3fwd-thread/main.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -50,9 +50,7 @@ #include #include #include -#include #include -#include #include #include #include @@ -61,7 +59,6 @@ #include #include #include -#include #include #include #include @@ -73,6 +70,7 @@ #include #include #include +#include #include #include @@ -90,16 +88,74 @@ #define APP_LOOKUP_METHOD APP_LOOKUP_LPM #endif +#ifndef __GLIBC__ /* sched_getcpu() is glibc specific */ +#define sched_getcpu() rte_lcore_id() +#endif + +static int +check_ptype(int portid) +{ + int i, ret; + int ipv4 = 0, ipv6 = 0; + + ret = rte_eth_dev_get_supported_ptypes(portid, RTE_PTYPE_L3_MASK, NULL, + 0); + if (ret <= 0) + return 0; + + uint32_t ptypes[ret]; + + ret = rte_eth_dev_get_supported_ptypes(portid, RTE_PTYPE_L3_MASK, + ptypes, ret); + for (i = 0; i < ret; ++i) { + if (ptypes[i] & RTE_PTYPE_L3_IPV4) + ipv4 = 1; + if (ptypes[i] & RTE_PTYPE_L3_IPV6) + ipv6 = 1; + } + + if (ipv4 && ipv6) + return 1; + + return 0; +} + +static inline void +parse_ptype(struct rte_mbuf *m) +{ + struct ether_hdr *eth_hdr; + uint32_t packet_type = RTE_PTYPE_UNKNOWN; + uint16_t ether_type; + + eth_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *); + ether_type = eth_hdr->ether_type; + if (ether_type == rte_cpu_to_be_16(ETHER_TYPE_IPv4)) + packet_type |= RTE_PTYPE_L3_IPV4_EXT_UNKNOWN; + else if (ether_type == rte_cpu_to_be_16(ETHER_TYPE_IPv6)) + packet_type |= RTE_PTYPE_L3_IPV6_EXT_UNKNOWN; + + m->packet_type = packet_type; +} + +static uint16_t +cb_parse_ptype(__rte_unused uint16_t port, __rte_unused uint16_t queue, + struct rte_mbuf *pkts[], uint16_t nb_pkts, + __rte_unused uint16_t max_pkts, __rte_unused void *user_param) +{ + unsigned int i; + + for (i = 0; i < nb_pkts; i++) + parse_ptype(pkts[i]); + + return nb_pkts; +} + /* * When set to zero, simple forwaring path is eanbled. * When set to one, optimized forwarding path is enabled. * Note that LPM optimisation path uses SSE4.1 instructions. */ -#if ((APP_LOOKUP_METHOD == APP_LOOKUP_LPM) && !defined(__SSE4_1__)) -#define ENABLE_MULTI_BUFFER_OPTIMIZE 0 -#else #define ENABLE_MULTI_BUFFER_OPTIMIZE 1 -#endif #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) #include @@ -126,10 +182,10 @@ */ #define NB_MBUF RTE_MAX(\ - (nb_ports*nb_rx_queue*RTE_TEST_RX_DESC_DEFAULT + \ - nb_ports*nb_lcores*MAX_PKT_BURST + \ - nb_ports*n_tx_queue*RTE_TEST_TX_DESC_DEFAULT + \ - nb_lcores*MEMPOOL_CACHE_SIZE), \ + (nb_ports*nb_rx_queue*nb_rxd + \ + nb_ports*nb_lcores*MAX_PKT_BURST + \ + nb_ports*n_tx_queue*nb_txd + \ + nb_lcores*MEMPOOL_CACHE_SIZE), \ (unsigned)8192) #define MAX_PKT_BURST 32 @@ -163,15 +219,16 @@ static uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; static uint64_t dest_eth_addr[RTE_MAX_ETHPORTS]; static struct ether_addr ports_eth_addr[RTE_MAX_ETHPORTS]; -static __m128i val_eth[RTE_MAX_ETHPORTS]; +static xmm_t val_eth[RTE_MAX_ETHPORTS]; /* replace first 12B of the ethernet header. */ #define MASK_ETH 0x3f /* mask of enabled ports */ static uint32_t enabled_port_mask; -static int promiscuous_on; /**< $et in promiscuous mode off by default. */ +static int promiscuous_on; /**< Set in promiscuous mode off by default. */ static int numa_on = 1; /**< NUMA is enabled by default. */ +static int parse_ptype_on; #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) static int ipv6; /**< ipv6 is false by default. */ @@ -218,7 +275,7 @@ struct mbuf_table { }; struct lcore_rx_queue { - uint8_t port_id; + uint16_t port_id; uint8_t queue_id; } __rte_cache_aligned; @@ -228,7 +285,7 @@ struct lcore_rx_queue { #define MAX_LCORE_PARAMS 1024 struct rx_thread_params { - uint8_t port_id; + uint16_t port_id; uint8_t queue_id; uint8_t lcore_id; uint8_t thread_id; @@ -282,7 +339,7 @@ static struct rte_eth_conf port_conf = { .hw_ip_checksum = 1, /**< IP checksum offload enabled */ .hw_vlan_filter = 0, /**< VLAN filtering disabled */ .jumbo_frame = 0, /**< Jumbo Frame Support disabled */ - .hw_strip_crc = 0, /**< CRC stripped by hardware */ + .hw_strip_crc = 1, /**< CRC stripped by hardware */ }, .rx_adv_conf = { .rss_conf = { @@ -299,13 +356,8 @@ static struct rte_mempool *pktmbuf_pool[NB_SOCKETS]; #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) -#ifdef RTE_MACHINE_CPUFLAG_SSE4_2 #include #define DEFAULT_HASH_FUNC rte_hash_crc -#else -#include -#define DEFAULT_HASH_FUNC rte_jhash -#endif struct ipv4_5tuple { uint32_t ip_dst; @@ -422,17 +474,10 @@ ipv4_hash_crc(const void *data, __rte_unused uint32_t data_len, t = k->proto; p = (const uint32_t *)&k->port_src; -#ifdef RTE_MACHINE_CPUFLAG_SSE4_2 init_val = rte_hash_crc_4byte(t, init_val); init_val = rte_hash_crc_4byte(k->ip_src, init_val); init_val = rte_hash_crc_4byte(k->ip_dst, init_val); init_val = rte_hash_crc_4byte(*p, init_val); -#else /* RTE_MACHINE_CPUFLAG_SSE4_2 */ - init_val = rte_jhash_1word(t, init_val); - init_val = rte_jhash_1word(k->ip_src, init_val); - init_val = rte_jhash_1word(k->ip_dst, init_val); - init_val = rte_jhash_1word(*p, init_val); -#endif /* RTE_MACHINE_CPUFLAG_SSE4_2 */ return init_val; } @@ -443,16 +488,13 @@ ipv6_hash_crc(const void *data, __rte_unused uint32_t data_len, const union ipv6_5tuple_host *k; uint32_t t; const uint32_t *p; -#ifdef RTE_MACHINE_CPUFLAG_SSE4_2 const uint32_t *ip_src0, *ip_src1, *ip_src2, *ip_src3; const uint32_t *ip_dst0, *ip_dst1, *ip_dst2, *ip_dst3; -#endif /* RTE_MACHINE_CPUFLAG_SSE4_2 */ k = data; t = k->proto; p = (const uint32_t *)&k->port_src; -#ifdef RTE_MACHINE_CPUFLAG_SSE4_2 ip_src0 = (const uint32_t *) k->ip_src; ip_src1 = (const uint32_t *)(k->ip_src + 4); ip_src2 = (const uint32_t *)(k->ip_src + 8); @@ -471,12 +513,6 @@ ipv6_hash_crc(const void *data, __rte_unused uint32_t data_len, init_val = rte_hash_crc_4byte(*ip_dst2, init_val); init_val = rte_hash_crc_4byte(*ip_dst3, init_val); init_val = rte_hash_crc_4byte(*p, init_val); -#else /* RTE_MACHINE_CPUFLAG_SSE4_2 */ - init_val = rte_jhash_1word(t, init_val); - init_val = rte_jhash(k->ip_src, sizeof(uint8_t) * IPV6_ADDR_LEN, init_val); - init_val = rte_jhash(k->ip_dst, sizeof(uint8_t) * IPV6_ADDR_LEN, init_val); - init_val = rte_jhash_1word(*p, init_val); -#endif /* RTE_MACHINE_CPUFLAG_SSE4_2 */ return init_val; } @@ -610,7 +646,7 @@ struct thread_tx_conf tx_thread[MAX_TX_THREAD]; /* Send burst of packets on an output interface */ static inline int -send_burst(struct thread_tx_conf *qconf, uint16_t n, uint8_t port) +send_burst(struct thread_tx_conf *qconf, uint16_t n, uint16_t port) { struct rte_mbuf **m_table; int ret; @@ -631,7 +667,7 @@ send_burst(struct thread_tx_conf *qconf, uint16_t n, uint8_t port) /* Enqueue a single packet, and send burst if queue is filled */ static inline int -send_single_packet(struct rte_mbuf *m, uint8_t port) +send_single_packet(struct rte_mbuf *m, uint16_t port) { uint16_t len; struct thread_tx_conf *qconf; @@ -657,8 +693,8 @@ send_single_packet(struct rte_mbuf *m, uint8_t port) #if ((APP_LOOKUP_METHOD == APP_LOOKUP_LPM) && \ (ENABLE_MULTI_BUFFER_OPTIMIZE == 1)) -static inline __attribute__((always_inline)) void -send_packetsx4(uint8_t port, +static __rte_always_inline void +send_packetsx4(uint16_t port, struct rte_mbuf *m[], uint32_t num) { uint32_t len, j, n; @@ -698,12 +734,15 @@ send_packetsx4(uint8_t port, case 0: qconf->tx_mbufs[port].m_table[len + j] = m[j]; j++; + /* fall-through */ case 3: qconf->tx_mbufs[port].m_table[len + j] = m[j]; j++; + /* fall-through */ case 2: qconf->tx_mbufs[port].m_table[len + j] = m[j]; j++; + /* fall-through */ case 1: qconf->tx_mbufs[port].m_table[len + j] = m[j]; j++; @@ -725,12 +764,15 @@ send_packetsx4(uint8_t port, case 0: qconf->tx_mbufs[port].m_table[j] = m[n + j]; j++; + /* fall-through */ case 3: qconf->tx_mbufs[port].m_table[j] = m[n + j]; j++; + /* fall-through */ case 2: qconf->tx_mbufs[port].m_table[j] = m[n + j]; j++; + /* fall-through */ case 1: qconf->tx_mbufs[port].m_table[j] = m[n + j]; j++; @@ -788,8 +830,8 @@ is_valid_ipv4_pkt(struct ipv4_hdr *pkt, uint32_t link_len) static __m128i mask0; static __m128i mask1; static __m128i mask2; -static inline uint8_t -get_ipv4_dst_port(void *ipv4_hdr, uint8_t portid, +static inline uint16_t +get_ipv4_dst_port(void *ipv4_hdr, uint16_t portid, lookup_struct_t *ipv4_l3fwd_lookup_struct) { int ret = 0; @@ -802,11 +844,11 @@ get_ipv4_dst_port(void *ipv4_hdr, uint8_t portid, key.xmm = _mm_and_si128(data, mask0); /* Find destination port */ ret = rte_hash_lookup(ipv4_l3fwd_lookup_struct, (const void *)&key); - return (uint8_t)((ret < 0) ? portid : ipv4_l3fwd_out_if[ret]); + return ((ret < 0) ? portid : ipv4_l3fwd_out_if[ret]); } -static inline uint8_t -get_ipv6_dst_port(void *ipv6_hdr, uint8_t portid, +static inline uint16_t +get_ipv6_dst_port(void *ipv6_hdr, uint16_t portid, lookup_struct_t *ipv6_l3fwd_lookup_struct) { int ret = 0; @@ -829,36 +871,36 @@ get_ipv6_dst_port(void *ipv6_hdr, uint8_t portid, /* Find destination port */ ret = rte_hash_lookup(ipv6_l3fwd_lookup_struct, (const void *)&key); - return (uint8_t)((ret < 0) ? portid : ipv6_l3fwd_out_if[ret]); + return ((ret < 0) ? portid : ipv6_l3fwd_out_if[ret]); } #endif #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM) -static inline uint8_t -get_ipv4_dst_port(void *ipv4_hdr, uint8_t portid, +static inline uint16_t +get_ipv4_dst_port(void *ipv4_hdr, uint16_t portid, lookup_struct_t *ipv4_l3fwd_lookup_struct) { uint32_t next_hop; - return (uint8_t)((rte_lpm_lookup(ipv4_l3fwd_lookup_struct, + return ((rte_lpm_lookup(ipv4_l3fwd_lookup_struct, rte_be_to_cpu_32(((struct ipv4_hdr *)ipv4_hdr)->dst_addr), &next_hop) == 0) ? next_hop : portid); } -static inline uint8_t -get_ipv6_dst_port(void *ipv6_hdr, uint8_t portid, +static inline uint16_t +get_ipv6_dst_port(void *ipv6_hdr, uint16_t portid, lookup6_struct_t *ipv6_l3fwd_lookup_struct) { - uint8_t next_hop; + uint32_t next_hop; - return (uint8_t) ((rte_lpm6_lookup(ipv6_l3fwd_lookup_struct, + return ((rte_lpm6_lookup(ipv6_l3fwd_lookup_struct, ((struct ipv6_hdr *)ipv6_hdr)->dst_addr, &next_hop) == 0) ? next_hop : portid); } #endif -static inline void l3fwd_simple_forward(struct rte_mbuf *m, uint8_t portid) +static inline void l3fwd_simple_forward(struct rte_mbuf *m, uint16_t portid) __attribute__((unused)); #if ((APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) && \ @@ -875,11 +917,11 @@ static inline void l3fwd_simple_forward(struct rte_mbuf *m, uint8_t portid) #define EXCLUDE_8TH_PKT 0x7f static inline void -simple_ipv4_fwd_8pkts(struct rte_mbuf *m[8], uint8_t portid) +simple_ipv4_fwd_8pkts(struct rte_mbuf *m[8], uint16_t portid) { struct ether_hdr *eth_hdr[8]; struct ipv4_hdr *ipv4_hdr[8]; - uint8_t dst_port[8]; + uint16_t dst_port[8]; int32_t ret[8]; union ipv4_5tuple_host key[8]; __m128i data[8]; @@ -998,14 +1040,14 @@ simple_ipv4_fwd_8pkts(struct rte_mbuf *m[8], uint8_t portid) rte_hash_lookup_bulk(RTE_PER_LCORE(lcore_conf)->ipv4_lookup_struct, &key_array[0], 8, ret); - dst_port[0] = (uint8_t) ((ret[0] < 0) ? portid : ipv4_l3fwd_out_if[ret[0]]); - dst_port[1] = (uint8_t) ((ret[1] < 0) ? portid : ipv4_l3fwd_out_if[ret[1]]); - dst_port[2] = (uint8_t) ((ret[2] < 0) ? portid : ipv4_l3fwd_out_if[ret[2]]); - dst_port[3] = (uint8_t) ((ret[3] < 0) ? portid : ipv4_l3fwd_out_if[ret[3]]); - dst_port[4] = (uint8_t) ((ret[4] < 0) ? portid : ipv4_l3fwd_out_if[ret[4]]); - dst_port[5] = (uint8_t) ((ret[5] < 0) ? portid : ipv4_l3fwd_out_if[ret[5]]); - dst_port[6] = (uint8_t) ((ret[6] < 0) ? portid : ipv4_l3fwd_out_if[ret[6]]); - dst_port[7] = (uint8_t) ((ret[7] < 0) ? portid : ipv4_l3fwd_out_if[ret[7]]); + dst_port[0] = ((ret[0] < 0) ? portid : ipv4_l3fwd_out_if[ret[0]]); + dst_port[1] = ((ret[1] < 0) ? portid : ipv4_l3fwd_out_if[ret[1]]); + dst_port[2] = ((ret[2] < 0) ? portid : ipv4_l3fwd_out_if[ret[2]]); + dst_port[3] = ((ret[3] < 0) ? portid : ipv4_l3fwd_out_if[ret[3]]); + dst_port[4] = ((ret[4] < 0) ? portid : ipv4_l3fwd_out_if[ret[4]]); + dst_port[5] = ((ret[5] < 0) ? portid : ipv4_l3fwd_out_if[ret[5]]); + dst_port[6] = ((ret[6] < 0) ? portid : ipv4_l3fwd_out_if[ret[6]]); + dst_port[7] = ((ret[7] < 0) ? portid : ipv4_l3fwd_out_if[ret[7]]); if (dst_port[0] >= RTE_MAX_ETHPORTS || (enabled_port_mask & 1 << dst_port[0]) == 0) @@ -1102,10 +1144,10 @@ static inline void get_ipv6_5tuple(struct rte_mbuf *m0, __m128i mask0, } static inline void -simple_ipv6_fwd_8pkts(struct rte_mbuf *m[8], uint8_t portid) +simple_ipv6_fwd_8pkts(struct rte_mbuf *m[8], uint16_t portid) { int32_t ret[8]; - uint8_t dst_port[8]; + uint16_t dst_port[8]; struct ether_hdr *eth_hdr[8]; union ipv6_5tuple_host key[8]; @@ -1152,14 +1194,14 @@ simple_ipv6_fwd_8pkts(struct rte_mbuf *m[8], uint8_t portid) rte_hash_lookup_bulk(RTE_PER_LCORE(lcore_conf)->ipv6_lookup_struct, &key_array[0], 4, ret); - dst_port[0] = (uint8_t) ((ret[0] < 0) ? portid : ipv6_l3fwd_out_if[ret[0]]); - dst_port[1] = (uint8_t) ((ret[1] < 0) ? portid : ipv6_l3fwd_out_if[ret[1]]); - dst_port[2] = (uint8_t) ((ret[2] < 0) ? portid : ipv6_l3fwd_out_if[ret[2]]); - dst_port[3] = (uint8_t) ((ret[3] < 0) ? portid : ipv6_l3fwd_out_if[ret[3]]); - dst_port[4] = (uint8_t) ((ret[4] < 0) ? portid : ipv6_l3fwd_out_if[ret[4]]); - dst_port[5] = (uint8_t) ((ret[5] < 0) ? portid : ipv6_l3fwd_out_if[ret[5]]); - dst_port[6] = (uint8_t) ((ret[6] < 0) ? portid : ipv6_l3fwd_out_if[ret[6]]); - dst_port[7] = (uint8_t) ((ret[7] < 0) ? portid : ipv6_l3fwd_out_if[ret[7]]); + dst_port[0] = ((ret[0] < 0) ? portid : ipv6_l3fwd_out_if[ret[0]]); + dst_port[1] = ((ret[1] < 0) ? portid : ipv6_l3fwd_out_if[ret[1]]); + dst_port[2] = ((ret[2] < 0) ? portid : ipv6_l3fwd_out_if[ret[2]]); + dst_port[3] = ((ret[3] < 0) ? portid : ipv6_l3fwd_out_if[ret[3]]); + dst_port[4] = ((ret[4] < 0) ? portid : ipv6_l3fwd_out_if[ret[4]]); + dst_port[5] = ((ret[5] < 0) ? portid : ipv6_l3fwd_out_if[ret[5]]); + dst_port[6] = ((ret[6] < 0) ? portid : ipv6_l3fwd_out_if[ret[6]]); + dst_port[7] = ((ret[7] < 0) ? portid : ipv6_l3fwd_out_if[ret[7]]); if (dst_port[0] >= RTE_MAX_ETHPORTS || (enabled_port_mask & 1 << dst_port[0]) == 0) @@ -1206,24 +1248,24 @@ simple_ipv6_fwd_8pkts(struct rte_mbuf *m[8], uint8_t portid) ether_addr_copy(&ports_eth_addr[dst_port[6]], ð_hdr[6]->s_addr); ether_addr_copy(&ports_eth_addr[dst_port[7]], ð_hdr[7]->s_addr); - send_single_packet(m[0], (uint8_t)dst_port[0]); - send_single_packet(m[1], (uint8_t)dst_port[1]); - send_single_packet(m[2], (uint8_t)dst_port[2]); - send_single_packet(m[3], (uint8_t)dst_port[3]); - send_single_packet(m[4], (uint8_t)dst_port[4]); - send_single_packet(m[5], (uint8_t)dst_port[5]); - send_single_packet(m[6], (uint8_t)dst_port[6]); - send_single_packet(m[7], (uint8_t)dst_port[7]); + send_single_packet(m[0], dst_port[0]); + send_single_packet(m[1], dst_port[1]); + send_single_packet(m[2], dst_port[2]); + send_single_packet(m[3], dst_port[3]); + send_single_packet(m[4], dst_port[4]); + send_single_packet(m[5], dst_port[5]); + send_single_packet(m[6], dst_port[6]); + send_single_packet(m[7], dst_port[7]); } #endif /* APP_LOOKUP_METHOD */ -static inline __attribute__((always_inline)) void -l3fwd_simple_forward(struct rte_mbuf *m, uint8_t portid) +static __rte_always_inline void +l3fwd_simple_forward(struct rte_mbuf *m, uint16_t portid) { struct ether_hdr *eth_hdr; struct ipv4_hdr *ipv4_hdr; - uint8_t dst_port; + uint16_t dst_port; eth_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *); @@ -1306,7 +1348,7 @@ l3fwd_simple_forward(struct rte_mbuf *m, uint8_t portid) * If we encounter invalid IPV4 packet, then set destination port for it * to BAD_PORT value. */ -static inline __attribute__((always_inline)) void +static __rte_always_inline void rfc1812_process(struct ipv4_hdr *ipv4_hdr, uint16_t *dp, uint32_t ptype) { uint8_t ihl; @@ -1334,18 +1376,17 @@ rfc1812_process(struct ipv4_hdr *ipv4_hdr, uint16_t *dp, uint32_t ptype) #if ((APP_LOOKUP_METHOD == APP_LOOKUP_LPM) && \ (ENABLE_MULTI_BUFFER_OPTIMIZE == 1)) -static inline __attribute__((always_inline)) uint16_t -get_dst_port(struct rte_mbuf *pkt, uint32_t dst_ipv4, uint8_t portid) +static __rte_always_inline uint16_t +get_dst_port(struct rte_mbuf *pkt, uint32_t dst_ipv4, uint16_t portid) { - uint32_t next_hop_ipv4; - uint8_t next_hop_ipv6; + uint32_t next_hop; struct ipv6_hdr *ipv6_hdr; struct ether_hdr *eth_hdr; if (RTE_ETH_IS_IPV4_HDR(pkt->packet_type)) { return (uint16_t) ((rte_lpm_lookup( RTE_PER_LCORE(lcore_conf)->ipv4_lookup_struct, dst_ipv4, - &next_hop_ipv4) == 0) ? next_hop_ipv4 : portid); + &next_hop) == 0) ? next_hop : portid); } else if (RTE_ETH_IS_IPV6_HDR(pkt->packet_type)) { @@ -1354,8 +1395,8 @@ get_dst_port(struct rte_mbuf *pkt, uint32_t dst_ipv4, uint8_t portid) return (uint16_t) ((rte_lpm6_lookup( RTE_PER_LCORE(lcore_conf)->ipv6_lookup_struct, - ipv6_hdr->dst_addr, &next_hop_ipv6) == 0) ? next_hop_ipv6 : - portid); + ipv6_hdr->dst_addr, &next_hop) == 0) ? + next_hop : portid); } @@ -1363,7 +1404,7 @@ get_dst_port(struct rte_mbuf *pkt, uint32_t dst_ipv4, uint8_t portid) } static inline void -process_packet(struct rte_mbuf *pkt, uint16_t *dst_port, uint8_t portid) +process_packet(struct rte_mbuf *pkt, uint16_t *dst_port, uint16_t portid) { struct ether_hdr *eth_hdr; struct ipv4_hdr *ipv4_hdr; @@ -1430,7 +1471,7 @@ processx4_step1(struct rte_mbuf *pkt[FWDSTEP], static inline void processx4_step2(__m128i dip, uint32_t ipv4_flag, - uint8_t portid, + uint16_t portid, struct rte_mbuf *pkt[FWDSTEP], uint16_t dprt[FWDSTEP]) { @@ -1536,7 +1577,7 @@ processx4_step3(struct rte_mbuf *pkt[FWDSTEP], uint16_t dst_port[FWDSTEP]) * Suppose we have array of destionation ports: * dst_port[] = {a, b, c, d,, e, ... } * dp1 should contain: , dp2: . - * We doing 4 comparisions at once and the result is 4 bit mask. + * We doing 4 comparisons at once and the result is 4 bit mask. * This mask is used as an index into prebuild array of pnum values. */ static inline uint16_t * @@ -1673,7 +1714,8 @@ port_groupx4(uint16_t pn[FWDSTEP + 1], uint16_t *lp, __m128i dp1, __m128i dp2) static void process_burst(struct rte_mbuf *pkts_burst[MAX_PKT_BURST], int nb_rx, - uint8_t portid) { + uint16_t portid) +{ int j; @@ -1798,10 +1840,12 @@ process_burst(struct rte_mbuf *pkts_burst[MAX_PKT_BURST], int nb_rx, process_packet(pkts_burst[j], dst_port + j, portid); GROUP_PORT_STEP(dlp, dst_port, lp, pnum, j); j++; + /* fall-through */ case 2: process_packet(pkts_burst[j], dst_port + j, portid); GROUP_PORT_STEP(dlp, dst_port, lp, pnum, j); j++; + /* fall-through */ case 1: process_packet(pkts_burst[j], dst_port + j, portid); GROUP_PORT_STEP(dlp, dst_port, lp, pnum, j); @@ -1991,7 +2035,7 @@ static void lthread_tx_per_ring(void *dummy) { int nb_rx; - uint8_t portid; + uint16_t portid; struct rte_ring *ring; struct thread_tx_conf *tx_conf; struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; @@ -2019,7 +2063,7 @@ lthread_tx_per_ring(void *dummy) */ SET_CPU_BUSY(tx_conf, CPU_POLL); nb_rx = rte_ring_sc_dequeue_burst(ring, (void **)pkts_burst, - MAX_PKT_BURST); + MAX_PKT_BURST, NULL); SET_CPU_IDLE(tx_conf, CPU_POLL); if (nb_rx > 0) { @@ -2046,7 +2090,7 @@ lthread_tx(void *args) struct lthread *lt; unsigned lcore_id; - uint8_t portid; + uint16_t portid; struct thread_tx_conf *tx_conf; tx_conf = (struct thread_tx_conf *)args; @@ -2093,7 +2137,8 @@ lthread_rx(void *dummy) int ret; uint16_t nb_rx; int i; - uint8_t portid, queueid; + uint16_t portid; + uint8_t queueid; int worker_id; int len[RTE_MAX_LCORE] = { 0 }; int old_len, new_len; @@ -2119,7 +2164,8 @@ lthread_rx(void *dummy) portid = rx_conf->rx_queue_list[i].port_id; queueid = rx_conf->rx_queue_list[i].queue_id; - RTE_LOG(INFO, L3FWD, " -- lcoreid=%u portid=%hhu rxqueueid=%hhu\n", + RTE_LOG(INFO, L3FWD, + " -- lcoreid=%u portid=%u rxqueueid=%hhu\n", rte_lcore_id(), portid, queueid); } @@ -2155,7 +2201,7 @@ lthread_rx(void *dummy) ret = rte_ring_sp_enqueue_burst( rx_conf->ring[worker_id], (void **) pkts_burst, - nb_rx); + nb_rx, NULL); new_len = old_len + ret; @@ -2278,7 +2324,7 @@ pthread_tx(void *dummy) struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; uint64_t prev_tsc, diff_tsc, cur_tsc; int nb_rx; - uint8_t portid; + uint16_t portid; struct thread_tx_conf *tx_conf; const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / @@ -2323,7 +2369,7 @@ pthread_tx(void *dummy) */ SET_CPU_BUSY(tx_conf, CPU_POLL); nb_rx = rte_ring_sc_dequeue_burst(tx_conf->ring, - (void **)pkts_burst, MAX_PKT_BURST); + (void **)pkts_burst, MAX_PKT_BURST, NULL); SET_CPU_IDLE(tx_conf, CPU_POLL); if (unlikely(nb_rx == 0)) { @@ -2347,7 +2393,8 @@ pthread_rx(void *dummy) uint32_t n; uint32_t nb_rx; unsigned lcore_id; - uint8_t portid, queueid; + uint8_t queueid; + uint16_t portid; struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; struct thread_rx_conf *rx_conf; @@ -2366,7 +2413,8 @@ pthread_rx(void *dummy) portid = rx_conf->rx_queue_list[i].port_id; queueid = rx_conf->rx_queue_list[i].queue_id; - RTE_LOG(INFO, L3FWD, " -- lcoreid=%u portid=%hhu rxqueueid=%hhu\n", + RTE_LOG(INFO, L3FWD, + " -- lcoreid=%u portid=%u rxqueueid=%hhu\n", lcore_id, portid, queueid); } @@ -2395,7 +2443,7 @@ pthread_rx(void *dummy) SET_CPU_BUSY(rx_conf, CPU_PROCESS); worker_id = (worker_id + 1) % rx_conf->n_ring; n = rte_ring_sp_enqueue_burst(rx_conf->ring[worker_id], - (void **)pkts_burst, nb_rx); + (void **)pkts_burst, nb_rx, NULL); if (unlikely(n != nb_rx)) { uint32_t k; @@ -2494,7 +2542,7 @@ check_port_config(const unsigned nb_ports) } static uint8_t -get_port_n_rx_queues(const uint8_t port) +get_port_n_rx_queues(const uint16_t port) { int queue = -1; uint16_t i; @@ -2610,6 +2658,7 @@ print_usage(const char *prgname) " [--rx (port,queue,lcore,thread)[,(port,queue,lcore,thread]]" " [--tx (lcore,thread)[,(lcore,thread]]" " [--enable-jumbo [--max-pkt-len PKTLEN]]\n" + " [--parse-ptype]\n\n" " -p PORTMASK: hexadecimal bitmask of ports to configure\n" " -P : enable promiscuous mode\n" " --rx (port,queue,lcore,thread): rx queues configuration\n" @@ -2621,7 +2670,8 @@ print_usage(const char *prgname) " --enable-jumbo: enable jumbo frame" " which max packet len is PKTLEN in decimal (64-9600)\n" " --hash-entry-num: specify the hash entry number in hexadecimal to be setup\n" - " --no-lthreads: turn off lthread model\n", + " --no-lthreads: turn off lthread model\n" + " --parse-ptype: set to use software to analyze packet type\n\n", prgname); } @@ -2722,7 +2772,7 @@ parse_rx_config(const char *q_arg) return -1; } rx_thread_params_array[nb_rx_thread_params].port_id = - (uint8_t)int_fld[FLD_PORT]; + int_fld[FLD_PORT]; rx_thread_params_array[nb_rx_thread_params].queue_id = (uint8_t)int_fld[FLD_QUEUE]; rx_thread_params_array[nb_rx_thread_params].lcore_id = @@ -2806,7 +2856,7 @@ parse_stat_lcore(const char *stat_lcore) static void parse_eth_dest(const char *optarg) { - uint8_t portid; + uint16_t portid; char *port_end; uint8_t c, *dest, peer_addr[6]; @@ -2840,6 +2890,7 @@ parse_eth_dest(const char *optarg) #define CMD_LINE_OPT_ENABLE_JUMBO "enable-jumbo" #define CMD_LINE_OPT_HASH_ENTRY_NUM "hash-entry-num" #define CMD_LINE_OPT_NO_LTHREADS "no-lthreads" +#define CMD_LINE_OPT_PARSE_PTYPE "parse-ptype" /* Parse the argument given in the command line of the application */ static int @@ -2859,6 +2910,7 @@ parse_args(int argc, char **argv) {CMD_LINE_OPT_ENABLE_JUMBO, 0, 0, 0}, {CMD_LINE_OPT_HASH_ENTRY_NUM, 1, 0, 0}, {CMD_LINE_OPT_NO_LTHREADS, 0, 0, 0}, + {CMD_LINE_OPT_PARSE_PTYPE, 0, 0, 0}, {NULL, 0, 0, 0} }; @@ -2935,6 +2987,12 @@ parse_args(int argc, char **argv) lthreads_on = 0; } + if (!strncmp(lgopts[option_index].name, CMD_LINE_OPT_PARSE_PTYPE, + sizeof(CMD_LINE_OPT_PARSE_PTYPE))) { + printf("software packet type parsing enabled\n"); + parse_ptype_on = 1; + } + if (!strncmp(lgopts[option_index].name, CMD_LINE_OPT_ENABLE_JUMBO, sizeof(CMD_LINE_OPT_ENABLE_JUMBO))) { struct option lenopts = {"max-pkt-len", required_argument, 0, @@ -2983,7 +3041,7 @@ parse_args(int argc, char **argv) argv[optind-1] = prgname; ret = optind-1; - optind = 0; /* reset getopt lib */ + optind = 1; /* reset getopt lib */ return ret; } @@ -3381,11 +3439,12 @@ init_mem(unsigned nb_mbuf) /* Check the link status of all ports in up to 9s, and print them finally */ static void -check_all_ports_link_status(uint8_t port_num, uint32_t port_mask) +check_all_ports_link_status(uint16_t port_num, uint32_t port_mask) { #define CHECK_INTERVAL 100 /* 100ms */ #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ - uint8_t portid, count, all_ports_up, print_flag = 0; + uint16_t portid; + uint8_t count, all_ports_up, print_flag = 0; struct rte_eth_link link; printf("\nChecking link status"); @@ -3400,14 +3459,13 @@ check_all_ports_link_status(uint8_t port_num, uint32_t port_mask) /* print link status if flag set */ if (print_flag == 1) { if (link.link_status) - printf("Port %d Link Up - speed %u " - "Mbps - %s\n", (uint8_t)portid, - (unsigned)link.link_speed, + printf( + "Port%d Link Up. Speed %u Mbps - %s\n", + portid, link.link_speed, (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? ("full-duplex") : ("half-duplex\n")); else - printf("Port %d Link Down\n", - (uint8_t)portid); + printf("Port %d Link Down\n", portid); continue; } /* clear all_ports_up flag if any link down */ @@ -3442,10 +3500,10 @@ main(int argc, char **argv) int ret; int i; unsigned nb_ports; - uint16_t queueid; + uint16_t queueid, portid; unsigned lcore_id; uint32_t n_tx_queue, nb_lcores; - uint8_t portid, nb_rx_queue, queue, socketid; + uint8_t nb_rx_queue, queue, socketid; /* init EAL */ ret = rte_eal_init(argc, argv); @@ -3515,6 +3573,13 @@ main(int argc, char **argv) rte_exit(EXIT_FAILURE, "Cannot configure device: err=%d, port=%d\n", ret, portid); + ret = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd, + &nb_txd); + if (ret < 0) + rte_exit(EXIT_FAILURE, + "rte_eth_dev_adjust_nb_rx_tx_desc: err=%d, port=%d\n", + ret, portid); + rte_eth_macaddr_get(portid, &ports_eth_addr[portid]); print_ethaddr(" Address:", &ports_eth_addr[portid]); printf(", "); @@ -3623,6 +3688,31 @@ main(int argc, char **argv) rte_eth_promiscuous_enable(portid); } + for (i = 0; i < n_rx_thread; i++) { + lcore_id = rx_thread[i].conf.lcore_id; + if (rte_lcore_is_enabled(lcore_id) == 0) + continue; + + /* check if hw packet type is supported */ + for (queue = 0; queue < rx_thread[i].n_rx_queue; ++queue) { + portid = rx_thread[i].rx_queue_list[queue].port_id; + queueid = rx_thread[i].rx_queue_list[queue].queue_id; + + if (parse_ptype_on) { + if (!rte_eth_add_rx_callback(portid, queueid, + cb_parse_ptype, NULL)) + rte_exit(EXIT_FAILURE, + "Failed to add rx callback: " + "port=%d\n", portid); + } else if (!check_ptype(portid)) + rte_exit(EXIT_FAILURE, + "Port %d cannot parse packet type.\n\n" + "Please add --parse-ptype to use sw " + "packet type analyzer.\n\n", + portid); + } + } + check_all_ports_link_status((uint8_t)nb_ports, enabled_port_mask); if (lthreads_on) { diff --git a/dpdk/examples/performance-thread/pthread_shim/main.c b/dpdk/examples/performance-thread/pthread_shim/main.c index f0357218..febae39b 100644 --- a/dpdk/examples/performance-thread/pthread_shim/main.c +++ b/dpdk/examples/performance-thread/pthread_shim/main.c @@ -59,6 +59,10 @@ #define DEBUG_APP 0 #define HELLOW_WORLD_MAX_LTHREADS 10 +#ifndef __GLIBC__ /* sched_getcpu() is glibc-specific */ +#define sched_getcpu() rte_lcore_id() +#endif + __thread int print_count; __thread pthread_mutex_t print_lock; @@ -157,6 +161,7 @@ static void initial_lthread(void *args __attribute__((unused))) pthread_override_set(1); uint64_t i; + int ret; /* initialize mutex for shared counter */ print_count = 0; @@ -175,15 +180,18 @@ static void initial_lthread(void *args __attribute__((unused))) * use an attribute to pass the desired lcore */ pthread_attr_t attr; - cpu_set_t cpuset; + rte_cpuset_t cpuset; CPU_ZERO(&cpuset); CPU_SET(lcore, &cpuset); pthread_attr_init(&attr); - pthread_attr_setaffinity_np(&attr, sizeof(cpu_set_t), &cpuset); + pthread_attr_setaffinity_np(&attr, sizeof(rte_cpuset_t), &cpuset); /* create the thread */ - pthread_create(&tid[i], &attr, helloworld_pthread, (void *) i); + ret = pthread_create(&tid[i], &attr, + helloworld_pthread, (void *) i); + if (ret != 0) + rte_exit(EXIT_FAILURE, "Cannot create helloworld thread\n"); } /* wait for 1s to allow threads diff --git a/dpdk/examples/performance-thread/pthread_shim/pthread_shim.c b/dpdk/examples/performance-thread/pthread_shim/pthread_shim.c index 0d6100c9..bc7cf2b0 100644 --- a/dpdk/examples/performance-thread/pthread_shim/pthread_shim.c +++ b/dpdk/examples/performance-thread/pthread_shim/pthread_shim.c @@ -48,6 +48,21 @@ #define POSIX_ERRNO(x) (x) +/* some releases of FreeBSD 10, e.g. 10.0, don't have CPU_COUNT macro */ +#ifndef CPU_COUNT +#define CPU_COUNT(x) __cpu_count(x) + +static inline unsigned int +__cpu_count(const rte_cpuset_t *cpuset) +{ + unsigned int i, count = 0; + for (i = 0; i < RTE_MAX_LCORE; i++) + if (CPU_ISSET(i, cpuset)) + count++; + return count; +} +#endif + /* * this flag determines at run time if we override pthread * calls and map then to equivalent lthread calls @@ -159,7 +174,7 @@ int (*f_pthread_setschedparam) int (*f_pthread_yield) (void); int (*f_pthread_setaffinity_np) - (pthread_t thread, size_t cpusetsize, const cpu_set_t *cpuset); + (pthread_t thread, size_t cpusetsize, const rte_cpuset_t *cpuset); int (*f_nanosleep) (const struct timespec *req, struct timespec *rem); } _sys_pthread_funcs = { @@ -187,10 +202,7 @@ static void *__libc_dl_handle = RTLD_NEXT; * The constructor function initialises the * function pointers for pthread library functions */ -void -pthread_intercept_ctor(void)__attribute__((constructor)); -void -pthread_intercept_ctor(void) +RTE_INIT(pthread_intercept_ctor) { override = 0; /* @@ -390,11 +402,11 @@ pthread_create(pthread_t *__restrict tid, if (attr != NULL) { /* determine CPU being requested */ - cpu_set_t cpuset; + rte_cpuset_t cpuset; CPU_ZERO(&cpuset); pthread_attr_getaffinity_np(attr, - sizeof(cpu_set_t), + sizeof(rte_cpuset_t), &cpuset); if (CPU_COUNT(&cpuset) != 1) @@ -576,15 +588,26 @@ int pthread_rwlock_wrlock(pthread_rwlock_t *a) return _sys_pthread_funcs.f_pthread_rwlock_wrlock(a); } -int pthread_yield(void) +#ifdef RTE_EXEC_ENV_LINUXAPP +int +pthread_yield(void) { if (override) { lthread_yield(); return 0; } return _sys_pthread_funcs.f_pthread_yield(); - } +#else +void +pthread_yield(void) +{ + if (override) + lthread_yield(); + else + _sys_pthread_funcs.f_pthread_yield(); +} +#endif pthread_t pthread_self(void) { @@ -686,7 +709,7 @@ int nanosleep(const struct timespec *req, struct timespec *rem) int pthread_setaffinity_np(pthread_t thread, size_t cpusetsize, - const cpu_set_t *cpuset) + const rte_cpuset_t *cpuset) { if (override) { /* we only allow affinity with a single CPU */ diff --git a/dpdk/examples/performance-thread/pthread_shim/pthread_shim.h b/dpdk/examples/performance-thread/pthread_shim/pthread_shim.h index 78bbb5ac..10f87894 100644 --- a/dpdk/examples/performance-thread/pthread_shim/pthread_shim.h +++ b/dpdk/examples/performance-thread/pthread_shim/pthread_shim.h @@ -33,7 +33,8 @@ #ifndef _PTHREAD_SHIM_H_ #define _PTHREAD_SHIM_H_ -#include + +#include /* * This pthread shim is an example that demonstrates how legacy code diff --git a/dpdk/examples/ptpclient/ptpclient.c b/dpdk/examples/ptpclient/ptpclient.c index 0af4f3b6..c53dad68 100644 --- a/dpdk/examples/ptpclient/ptpclient.c +++ b/dpdk/examples/ptpclient/ptpclient.c @@ -158,12 +158,12 @@ struct ptpv2_data_slave_ordinary { struct clock_id master_clock_id; struct timeval new_adj; int64_t delta; - uint8_t portid; + uint16_t portid; uint16_t seqID_SYNC; uint16_t seqID_FOLLOWUP; uint8_t ptpset; uint8_t kernel_time_set; - uint8_t current_ptp_port; + uint16_t current_ptp_port; }; static struct ptpv2_data_slave_ordinary ptp_data; @@ -202,7 +202,7 @@ ns_to_timeval(int64_t nsec) * coming from the mbuf_pool passed as a parameter. */ static inline int -port_init(uint8_t port, struct rte_mempool *mbuf_pool) +port_init(uint16_t port, struct rte_mempool *mbuf_pool) { struct rte_eth_dev_info dev_info; struct rte_eth_conf port_conf = port_conf_default; @@ -210,6 +210,8 @@ port_init(uint8_t port, struct rte_mempool *mbuf_pool) const uint16_t tx_rings = 1; int retval; uint16_t q; + uint16_t nb_rxd = RX_RING_SIZE; + uint16_t nb_txd = TX_RING_SIZE; if (port >= rte_eth_dev_count()) return -1; @@ -219,9 +221,13 @@ port_init(uint8_t port, struct rte_mempool *mbuf_pool) if (retval != 0) return retval; + retval = rte_eth_dev_adjust_nb_rx_tx_desc(port, &nb_rxd, &nb_txd); + if (retval != 0) + return retval; + /* Allocate and set up 1 RX queue per Ethernet port. */ for (q = 0; q < rx_rings; q++) { - retval = rte_eth_rx_queue_setup(port, q, RX_RING_SIZE, + retval = rte_eth_rx_queue_setup(port, q, nb_rxd, rte_eth_dev_socket_id(port), NULL, mbuf_pool); if (retval < 0) @@ -237,7 +243,7 @@ port_init(uint8_t port, struct rte_mempool *mbuf_pool) txconf = &dev_info.default_txconf; txconf->txq_flags = 0; - retval = rte_eth_tx_queue_setup(port, q, TX_RING_SIZE, + retval = rte_eth_tx_queue_setup(port, q, nb_txd, rte_eth_dev_socket_id(port), txconf); if (retval < 0) return retval; @@ -549,7 +555,7 @@ parse_drsp(struct ptpv2_data_slave_ordinary *ptp_data) * functionality. */ static void -parse_ptp_frames(uint8_t portid, struct rte_mbuf *m) { +parse_ptp_frames(uint16_t portid, struct rte_mbuf *m) { struct ptp_header *ptp_hdr; struct ether_hdr *eth_hdr; uint16_t eth_type; @@ -587,7 +593,7 @@ parse_ptp_frames(uint8_t portid, struct rte_mbuf *m) { static __attribute__((noreturn)) void lcore_main(void) { - uint8_t portid; + uint16_t portid; unsigned nb_rx; struct rte_mbuf *m; @@ -708,7 +714,7 @@ ptp_parse_args(int argc, char **argv) argv[optind-1] = prgname; - optind = 0; /* Reset getopt lib. */ + optind = 1; /* Reset getopt lib. */ return 0; } @@ -722,7 +728,7 @@ main(int argc, char *argv[]) { unsigned nb_ports; - uint8_t portid; + uint16_t portid; /* Initialize the Environment Abstraction Layer (EAL). */ int ret = rte_eal_init(argc, argv); diff --git a/dpdk/examples/qos_meter/Makefile b/dpdk/examples/qos_meter/Makefile index 5113a129..de1f12ce 100644 --- a/dpdk/examples/qos_meter/Makefile +++ b/dpdk/examples/qos_meter/Makefile @@ -33,7 +33,7 @@ ifeq ($(RTE_SDK),) $(error "Please define RTE_SDK environment variable") endif -# Default target, can be overriden by command line or environment +# Default target, can be overridden by command line or environment RTE_TARGET ?= x86_64-native-linuxapp-gcc include $(RTE_SDK)/mk/rte.vars.mk diff --git a/dpdk/examples/qos_meter/main.c b/dpdk/examples/qos_meter/main.c index 15656155..67b4a75b 100644 --- a/dpdk/examples/qos_meter/main.c +++ b/dpdk/examples/qos_meter/main.c @@ -89,7 +89,7 @@ static struct rte_eth_conf port_conf = { .hw_ip_checksum = 1, .hw_vlan_filter = 0, .jumbo_frame = 0, - .hw_strip_crc = 0, + .hw_strip_crc = 1, }, .rx_adv_conf = { .rss_conf = { @@ -116,8 +116,8 @@ static struct rte_eth_conf port_conf = { #define PKT_TX_BURST_MAX 32 #define TIME_TX_DRAIN 200000ULL -static uint8_t port_rx; -static uint8_t port_tx; +static uint16_t port_rx; +static uint16_t port_tx; static struct rte_mbuf *pkts_rx[PKT_RX_BURST_MAX]; struct rte_eth_dev_tx_buffer *tx_buffer; @@ -300,7 +300,7 @@ parse_args(int argc, char **argv) argv[optind-1] = prgname; - optind = 0; /* reset getopt lib */ + optind = 1; /* reset getopt lib */ return 0; } @@ -308,6 +308,8 @@ int main(int argc, char **argv) { uint32_t lcore_id; + uint16_t nb_rxd = NIC_RX_QUEUE_DESC; + uint16_t nb_txd = NIC_TX_QUEUE_DESC; int ret; /* EAL init */ @@ -337,13 +339,18 @@ main(int argc, char **argv) if (ret < 0) rte_exit(EXIT_FAILURE, "Port %d configuration error (%d)\n", port_rx, ret); - ret = rte_eth_rx_queue_setup(port_rx, NIC_RX_QUEUE, NIC_RX_QUEUE_DESC, + ret = rte_eth_dev_adjust_nb_rx_tx_desc(port_rx, &nb_rxd, &nb_txd); + if (ret < 0) + rte_exit(EXIT_FAILURE, "Port %d adjust number of descriptors error (%d)\n", + port_rx, ret); + + ret = rte_eth_rx_queue_setup(port_rx, NIC_RX_QUEUE, nb_rxd, rte_eth_dev_socket_id(port_rx), NULL, pool); if (ret < 0) rte_exit(EXIT_FAILURE, "Port %d RX queue setup error (%d)\n", port_rx, ret); - ret = rte_eth_tx_queue_setup(port_rx, NIC_TX_QUEUE, NIC_TX_QUEUE_DESC, + ret = rte_eth_tx_queue_setup(port_rx, NIC_TX_QUEUE, nb_txd, rte_eth_dev_socket_id(port_rx), NULL); if (ret < 0) @@ -353,13 +360,20 @@ main(int argc, char **argv) if (ret < 0) rte_exit(EXIT_FAILURE, "Port %d configuration error (%d)\n", port_tx, ret); - ret = rte_eth_rx_queue_setup(port_tx, NIC_RX_QUEUE, NIC_RX_QUEUE_DESC, + nb_rxd = NIC_RX_QUEUE_DESC; + nb_txd = NIC_TX_QUEUE_DESC; + ret = rte_eth_dev_adjust_nb_rx_tx_desc(port_tx, &nb_rxd, &nb_txd); + if (ret < 0) + rte_exit(EXIT_FAILURE, "Port %d adjust number of descriptors error (%d)\n", + port_tx, ret); + + ret = rte_eth_rx_queue_setup(port_tx, NIC_RX_QUEUE, nb_rxd, rte_eth_dev_socket_id(port_tx), NULL, pool); if (ret < 0) rte_exit(EXIT_FAILURE, "Port %d RX queue setup error (%d)\n", port_tx, ret); - ret = rte_eth_tx_queue_setup(port_tx, NIC_TX_QUEUE, NIC_TX_QUEUE_DESC, + ret = rte_eth_tx_queue_setup(port_tx, NIC_TX_QUEUE, nb_txd, rte_eth_dev_socket_id(port_tx), NULL); if (ret < 0) diff --git a/dpdk/examples/qos_sched/Makefile b/dpdk/examples/qos_sched/Makefile index f59645f5..56829c21 100644 --- a/dpdk/examples/qos_sched/Makefile +++ b/dpdk/examples/qos_sched/Makefile @@ -33,7 +33,7 @@ ifeq ($(RTE_SDK),) $(error "Please define RTE_SDK environment variable") endif -# Default target, can be overriden by command line or environment +# Default target, can be overridden by command line or environment RTE_TARGET ?= x86_64-native-linuxapp-gcc include $(RTE_SDK)/mk/rte.vars.mk @@ -42,6 +42,7 @@ ifneq ($(CONFIG_RTE_EXEC_ENV),"linuxapp") $(info This application can only operate in a linuxapp environment, \ please change the definition of the RTE_TARGET environment variable) all: +clean: else # binary name diff --git a/dpdk/examples/qos_sched/app_thread.c b/dpdk/examples/qos_sched/app_thread.c index 70fdcdb2..15f117f5 100644 --- a/dpdk/examples/qos_sched/app_thread.c +++ b/dpdk/examples/qos_sched/app_thread.c @@ -107,7 +107,7 @@ app_rx_thread(struct thread_conf **confs) } if (unlikely(rte_ring_sp_enqueue_bulk(conf->rx_ring, - (void **)rx_mbufs, nb_rx) != 0)) { + (void **)rx_mbufs, nb_rx, NULL) == 0)) { for(i = 0; i < nb_rx; i++) { rte_pktmbuf_free(rx_mbufs[i]); @@ -179,8 +179,8 @@ app_tx_thread(struct thread_conf **confs) while ((conf = confs[conf_idx])) { retval = rte_ring_sc_dequeue_bulk(conf->tx_ring, (void **)mbufs, - burst_conf.qos_dequeue); - if (likely(retval == 0)) { + burst_conf.qos_dequeue, NULL); + if (likely(retval != 0)) { app_send_packets(conf, mbufs, burst_conf.qos_dequeue); conf->counter = 0; /* reset empty read loop counter */ @@ -218,7 +218,7 @@ app_worker_thread(struct thread_conf **confs) /* Read packet from the ring */ nb_pkt = rte_ring_sc_dequeue_burst(conf->rx_ring, (void **)mbufs, - burst_conf.ring_burst); + burst_conf.ring_burst, NULL); if (likely(nb_pkt)) { int nb_sent = rte_sched_port_enqueue(conf->sched_port, mbufs, nb_pkt); @@ -230,7 +230,9 @@ app_worker_thread(struct thread_conf **confs) nb_pkt = rte_sched_port_dequeue(conf->sched_port, mbufs, burst_conf.qos_dequeue); if (likely(nb_pkt > 0)) - while (rte_ring_sp_enqueue_bulk(conf->tx_ring, (void **)mbufs, nb_pkt) != 0); + while (rte_ring_sp_enqueue_bulk(conf->tx_ring, + (void **)mbufs, nb_pkt, NULL) == 0) + ; /* empty body */ conf_idx++; if (confs[conf_idx] == NULL) @@ -252,7 +254,7 @@ app_mixed_thread(struct thread_conf **confs) /* Read packet from the ring */ nb_pkt = rte_ring_sc_dequeue_burst(conf->rx_ring, (void **)mbufs, - burst_conf.ring_burst); + burst_conf.ring_burst, NULL); if (likely(nb_pkt)) { int nb_sent = rte_sched_port_enqueue(conf->sched_port, mbufs, nb_pkt); diff --git a/dpdk/examples/qos_sched/args.c b/dpdk/examples/qos_sched/args.c index 476a0ee1..203a3470 100644 --- a/dpdk/examples/qos_sched/args.c +++ b/dpdk/examples/qos_sched/args.c @@ -245,14 +245,15 @@ app_parse_flow_conf(const char *conf_str) struct flow_conf *pconf; uint64_t mask; + memset(vals, 0, sizeof(vals)); ret = app_parse_opt_vals(conf_str, ',', 6, vals); if (ret < 4 || ret > 5) return ret; pconf = &qos_conf[nb_pfc]; - pconf->rx_port = (uint8_t)vals[0]; - pconf->tx_port = (uint8_t)vals[1]; + pconf->rx_port = vals[0]; + pconf->tx_port = vals[1]; pconf->rx_core = (uint8_t)vals[2]; pconf->wt_core = (uint8_t)vals[3]; if (ret == 5) @@ -266,19 +267,19 @@ app_parse_flow_conf(const char *conf_str) } if (pconf->rx_port >= RTE_MAX_ETHPORTS) { - RTE_LOG(ERR, APP, "pfc %u: invalid rx port %"PRIu8" index\n", + RTE_LOG(ERR, APP, "pfc %u: invalid rx port %"PRIu16" index\n", nb_pfc, pconf->rx_port); return -1; } if (pconf->tx_port >= RTE_MAX_ETHPORTS) { - RTE_LOG(ERR, APP, "pfc %u: invalid tx port %"PRIu8" index\n", + RTE_LOG(ERR, APP, "pfc %u: invalid tx port %"PRIu16" index\n", nb_pfc, pconf->tx_port); return -1; } mask = 1lu << pconf->rx_port; if (app_used_rx_port_mask & mask) { - RTE_LOG(ERR, APP, "pfc %u: rx port %"PRIu8" is used already\n", + RTE_LOG(ERR, APP, "pfc %u: rx port %"PRIu16" is used already\n", nb_pfc, pconf->rx_port); return -1; } @@ -287,7 +288,7 @@ app_parse_flow_conf(const char *conf_str) mask = 1lu << pconf->tx_port; if (app_used_tx_port_mask & mask) { - RTE_LOG(ERR, APP, "pfc %u: port %"PRIu8" is used already\n", + RTE_LOG(ERR, APP, "pfc %u: port %"PRIu16" is used already\n", nb_pfc, pconf->tx_port); return -1; } diff --git a/dpdk/examples/qos_sched/cmdline.c b/dpdk/examples/qos_sched/cmdline.c index f79d5246..b62d165b 100644 --- a/dpdk/examples/qos_sched/cmdline.c +++ b/dpdk/examples/qos_sched/cmdline.c @@ -191,7 +191,7 @@ cmdline_parse_inst_t cmd_appstats = { struct cmd_subportstats_result { cmdline_fixed_string_t stats_string; cmdline_fixed_string_t port_string; - uint8_t port_number; + uint16_t port_number; cmdline_fixed_string_t subport_string; uint32_t subport_number; }; @@ -220,7 +220,7 @@ cmdline_parse_token_num_t cmd_subportstats_subport_number = UINT32); cmdline_parse_token_num_t cmd_subportstats_port_number = TOKEN_NUM_INITIALIZER(struct cmd_subportstats_result, port_number, - UINT8); + UINT16); cmdline_parse_inst_t cmd_subportstats = { .f = cmd_subportstats_parsed, @@ -240,7 +240,7 @@ cmdline_parse_inst_t cmd_subportstats = { struct cmd_pipestats_result { cmdline_fixed_string_t stats_string; cmdline_fixed_string_t port_string; - uint8_t port_number; + uint16_t port_number; cmdline_fixed_string_t subport_string; uint32_t subport_number; cmdline_fixed_string_t pipe_string; @@ -265,7 +265,7 @@ cmdline_parse_token_string_t cmd_pipestats_port_string = "port"); cmdline_parse_token_num_t cmd_pipestats_port_number = TOKEN_NUM_INITIALIZER(struct cmd_pipestats_result, port_number, - UINT8); + UINT16); cmdline_parse_token_string_t cmd_pipestats_subport_string = TOKEN_STRING_INITIALIZER(struct cmd_pipestats_result, subport_string, "subport"); @@ -299,7 +299,7 @@ cmdline_parse_inst_t cmd_pipestats = { struct cmd_avg_q_result { cmdline_fixed_string_t qavg_string; cmdline_fixed_string_t port_string; - uint8_t port_number; + uint16_t port_number; cmdline_fixed_string_t subport_string; uint32_t subport_number; cmdline_fixed_string_t pipe_string; @@ -327,8 +327,8 @@ cmdline_parse_token_string_t cmd_avg_q_port_string = TOKEN_STRING_INITIALIZER(struct cmd_avg_q_result, port_string, "port"); cmdline_parse_token_num_t cmd_avg_q_port_number = - TOKEN_NUM_INITIALIZER(struct cmd_avg_q_result, port_number, - UINT8); + TOKEN_NUM_INITIALIZER(struct cmd_avg_q_result, port_number, + UINT16); cmdline_parse_token_string_t cmd_avg_q_subport_string = TOKEN_STRING_INITIALIZER(struct cmd_avg_q_result, subport_string, "subport"); @@ -378,7 +378,7 @@ cmdline_parse_inst_t cmd_avg_q = { struct cmd_avg_tcpipe_result { cmdline_fixed_string_t qavg_string; cmdline_fixed_string_t port_string; - uint8_t port_number; + uint16_t port_number; cmdline_fixed_string_t subport_string; uint32_t subport_number; cmdline_fixed_string_t pipe_string; @@ -405,7 +405,7 @@ cmdline_parse_token_string_t cmd_avg_tcpipe_port_string = "port"); cmdline_parse_token_num_t cmd_avg_tcpipe_port_number = TOKEN_NUM_INITIALIZER(struct cmd_avg_tcpipe_result, port_number, - UINT8); + UINT16); cmdline_parse_token_string_t cmd_avg_tcpipe_subport_string = TOKEN_STRING_INITIALIZER(struct cmd_avg_tcpipe_result, subport_string, "subport"); @@ -447,7 +447,7 @@ cmdline_parse_inst_t cmd_avg_tcpipe = { struct cmd_avg_pipe_result { cmdline_fixed_string_t qavg_string; cmdline_fixed_string_t port_string; - uint8_t port_number; + uint16_t port_number; cmdline_fixed_string_t subport_string; uint32_t subport_number; cmdline_fixed_string_t pipe_string; @@ -472,7 +472,7 @@ cmdline_parse_token_string_t cmd_avg_pipe_port_string = "port"); cmdline_parse_token_num_t cmd_avg_pipe_port_number = TOKEN_NUM_INITIALIZER(struct cmd_avg_pipe_result, port_number, - UINT8); + UINT16); cmdline_parse_token_string_t cmd_avg_pipe_subport_string = TOKEN_STRING_INITIALIZER(struct cmd_avg_pipe_result, subport_string, "subport"); @@ -506,7 +506,7 @@ cmdline_parse_inst_t cmd_avg_pipe = { struct cmd_avg_tcsubport_result { cmdline_fixed_string_t qavg_string; cmdline_fixed_string_t port_string; - uint8_t port_number; + uint16_t port_number; cmdline_fixed_string_t subport_string; uint32_t subport_number; cmdline_fixed_string_t tc_string; @@ -531,7 +531,7 @@ cmdline_parse_token_string_t cmd_avg_tcsubport_port_string = "port"); cmdline_parse_token_num_t cmd_avg_tcsubport_port_number = TOKEN_NUM_INITIALIZER(struct cmd_avg_tcsubport_result, port_number, - UINT8); + UINT16); cmdline_parse_token_string_t cmd_avg_tcsubport_subport_string = TOKEN_STRING_INITIALIZER(struct cmd_avg_tcsubport_result, subport_string, "subport"); @@ -565,7 +565,7 @@ cmdline_parse_inst_t cmd_avg_tcsubport = { struct cmd_avg_subport_result { cmdline_fixed_string_t qavg_string; cmdline_fixed_string_t port_string; - uint8_t port_number; + uint16_t port_number; cmdline_fixed_string_t subport_string; uint32_t subport_number; }; @@ -588,7 +588,7 @@ cmdline_parse_token_string_t cmd_avg_subport_port_string = "port"); cmdline_parse_token_num_t cmd_avg_subport_port_number = TOKEN_NUM_INITIALIZER(struct cmd_avg_subport_result, port_number, - UINT8); + UINT16); cmdline_parse_token_string_t cmd_avg_subport_subport_string = TOKEN_STRING_INITIALIZER(struct cmd_avg_subport_result, subport_string, "subport"); diff --git a/dpdk/examples/qos_sched/init.c b/dpdk/examples/qos_sched/init.c index 70e12bb4..038f0427 100644 --- a/dpdk/examples/qos_sched/init.c +++ b/dpdk/examples/qos_sched/init.c @@ -92,7 +92,7 @@ static const struct rte_eth_conf port_conf = { .hw_ip_checksum = 0, /**< IP checksum offload disabled */ .hw_vlan_filter = 0, /**< VLAN filtering disabled */ .jumbo_frame = 0, /**< Jumbo Frame Support disabled */ - .hw_strip_crc = 0, /**< CRC stripped by hardware */ + .hw_strip_crc = 1, /**< CRC stripped by hardware */ }, .txmode = { .mq_mode = ETH_DCB_NONE, @@ -100,12 +100,14 @@ static const struct rte_eth_conf port_conf = { }; static int -app_init_port(uint8_t portid, struct rte_mempool *mp) +app_init_port(uint16_t portid, struct rte_mempool *mp) { int ret; struct rte_eth_link link; struct rte_eth_rxconf rx_conf; struct rte_eth_txconf tx_conf; + uint16_t rx_size; + uint16_t tx_size; /* check if port already initialized (multistream configuration) */ if (app_inited_port_mask & (1u << portid)) @@ -116,6 +118,7 @@ app_init_port(uint8_t portid, struct rte_mempool *mp) rx_conf.rx_thresh.wthresh = rx_thresh.wthresh; rx_conf.rx_free_thresh = 32; rx_conf.rx_drop_en = 0; + rx_conf.rx_deferred_start = 0; tx_conf.tx_thresh.pthresh = tx_thresh.pthresh; tx_conf.tx_thresh.hthresh = tx_thresh.hthresh; @@ -123,36 +126,51 @@ app_init_port(uint8_t portid, struct rte_mempool *mp) tx_conf.tx_free_thresh = 0; tx_conf.tx_rs_thresh = 0; tx_conf.txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS | ETH_TXQ_FLAGS_NOOFFLOADS; + tx_conf.tx_deferred_start = 0; /* init port */ - RTE_LOG(INFO, APP, "Initializing port %"PRIu8"... ", portid); + RTE_LOG(INFO, APP, "Initializing port %"PRIu16"... ", portid); fflush(stdout); ret = rte_eth_dev_configure(portid, 1, 1, &port_conf); if (ret < 0) - rte_exit(EXIT_FAILURE, "Cannot configure device: " - "err=%d, port=%"PRIu8"\n", ret, portid); + rte_exit(EXIT_FAILURE, + "Cannot configure device: err=%d, port=%u\n", + ret, portid); + + rx_size = ring_conf.rx_size; + tx_size = ring_conf.tx_size; + ret = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &rx_size, &tx_size); + if (ret < 0) + rte_exit(EXIT_FAILURE, + "rte_eth_dev_adjust_nb_rx_tx_desc: err=%d,port=%u\n", + ret, portid); + ring_conf.rx_size = rx_size; + ring_conf.tx_size = tx_size; /* init one RX queue */ fflush(stdout); ret = rte_eth_rx_queue_setup(portid, 0, (uint16_t)ring_conf.rx_size, rte_eth_dev_socket_id(portid), &rx_conf, mp); if (ret < 0) - rte_exit(EXIT_FAILURE, "rte_eth_tx_queue_setup: " - "err=%d, port=%"PRIu8"\n", ret, portid); + rte_exit(EXIT_FAILURE, + "rte_eth_tx_queue_setup: err=%d, port=%u\n", + ret, portid); /* init one TX queue */ fflush(stdout); ret = rte_eth_tx_queue_setup(portid, 0, (uint16_t)ring_conf.tx_size, rte_eth_dev_socket_id(portid), &tx_conf); if (ret < 0) - rte_exit(EXIT_FAILURE, "rte_eth_tx_queue_setup: err=%d, " - "port=%"PRIu8" queue=%d\n", ret, portid, 0); + rte_exit(EXIT_FAILURE, + "rte_eth_tx_queue_setup: err=%d, port=%u queue=%d\n", + ret, portid, 0); /* Start device */ ret = rte_eth_dev_start(portid); if (ret < 0) - rte_exit(EXIT_FAILURE, "rte_pmd_port_start: " - "err=%d, port=%"PRIu8"\n", ret, portid); + rte_exit(EXIT_FAILURE, + "rte_pmd_port_start: err=%d, port=%u\n", + ret, portid); printf("done: "); @@ -245,7 +263,7 @@ app_init_sched_port(uint32_t portid, uint32_t socketid) uint32_t pipe, subport; int err; - rte_eth_link_get((uint8_t)portid, &link); + rte_eth_link_get(portid, &link); port_params.socket = socketid; port_params.rate = (uint64_t) link.link_speed * 1000 * 1000 / 8; diff --git a/dpdk/examples/qos_sched/main.c b/dpdk/examples/qos_sched/main.c index e10cfd44..1e2fb189 100644 --- a/dpdk/examples/qos_sched/main.c +++ b/dpdk/examples/qos_sched/main.c @@ -125,8 +125,7 @@ app_main_loop(__attribute__((unused))void *dummy) /* initialize mbuf memory */ if (mode == APP_RX_MODE) { for (i = 0; i < rx_idx; i++) { - RTE_LOG(INFO, APP, "flow %u lcoreid %u " - "reading port %"PRIu8"\n", + RTE_LOG(INFO, APP, "flow%u lcoreid%u reading port%u\n", i, lcore_id, rx_confs[i]->rx_port); } @@ -140,8 +139,8 @@ app_main_loop(__attribute__((unused))void *dummy) if (wt_confs[i]->m_table == NULL) rte_panic("flow %u unable to allocate memory buffer\n", i); - RTE_LOG(INFO, APP, "flow %u lcoreid %u sched+write " - "port %"PRIu8"\n", + RTE_LOG(INFO, APP, + "flow %u lcoreid %u sched+write port %u\n", i, lcore_id, wt_confs[i]->tx_port); } @@ -155,8 +154,7 @@ app_main_loop(__attribute__((unused))void *dummy) if (tx_confs[i]->m_table == NULL) rte_panic("flow %u unable to allocate memory buffer\n", i); - RTE_LOG(INFO, APP, "flow %u lcoreid %u " - "writing port %"PRIu8"\n", + RTE_LOG(INFO, APP, "flow%u lcoreid%u write port%u\n", i, lcore_id, tx_confs[i]->tx_port); } @@ -186,7 +184,7 @@ app_stat(void) struct flow_conf *flow = &qos_conf[i]; rte_eth_stats_get(flow->rx_port, &stats); - printf("\nRX port %"PRIu8": rx: %"PRIu64 " err: %"PRIu64 + printf("\nRX port %"PRIu16": rx: %"PRIu64 " err: %"PRIu64 " no_mbuf: %"PRIu64 "\n", flow->rx_port, stats.ipackets - rx_stats[i].ipackets, @@ -195,7 +193,7 @@ app_stat(void) memcpy(&rx_stats[i], &stats, sizeof(stats)); rte_eth_stats_get(flow->tx_port, &stats); - printf("TX port %"PRIu8": tx: %" PRIu64 " err: %" PRIu64 "\n", + printf("TX port %"PRIu16": tx: %" PRIu64 " err: %" PRIu64 "\n", flow->tx_port, stats.opackets - tx_stats[i].opackets, stats.oerrors - tx_stats[i].oerrors); diff --git a/dpdk/examples/qos_sched/main.h b/dpdk/examples/qos_sched/main.h index c7490c61..77b6e3ee 100644 --- a/dpdk/examples/qos_sched/main.h +++ b/dpdk/examples/qos_sched/main.h @@ -69,8 +69,13 @@ extern "C" { #define BURST_TX_DRAIN_US 100 #ifndef APP_MAX_LCORE +#if (RTE_MAX_LCORE > 64) #define APP_MAX_LCORE 64 +#else +#define APP_MAX_LCORE RTE_MAX_LCORE #endif +#endif + #define MAX_DATA_STREAMS (APP_MAX_LCORE/2) #define MAX_SCHED_SUBPORTS 8 #define MAX_SCHED_PIPES 4096 @@ -101,8 +106,8 @@ struct thread_conf uint32_t n_mbufs; struct rte_mbuf **m_table; - uint8_t rx_port; - uint8_t tx_port; + uint16_t rx_port; + uint16_t tx_port; uint16_t rx_queue; uint16_t tx_queue; struct rte_ring *rx_ring; @@ -120,8 +125,8 @@ struct flow_conf uint32_t rx_core; uint32_t wt_core; uint32_t tx_core; - uint8_t rx_port; - uint8_t tx_port; + uint16_t rx_port; + uint16_t tx_port; uint16_t rx_queue; uint16_t tx_queue; struct rte_ring *rx_ring; @@ -183,13 +188,15 @@ void app_worker_thread(struct thread_conf **qconf); void app_mixed_thread(struct thread_conf **qconf); void app_stat(void); -int subport_stat(uint8_t port_id, uint32_t subport_id); -int pipe_stat(uint8_t port_id, uint32_t subport_id, uint32_t pipe_id); -int qavg_q(uint8_t port_id, uint32_t subport_id, uint32_t pipe_id, uint8_t tc, uint8_t q); -int qavg_tcpipe(uint8_t port_id, uint32_t subport_id, uint32_t pipe_id, uint8_t tc); -int qavg_pipe(uint8_t port_id, uint32_t subport_id, uint32_t pipe_id); -int qavg_tcsubport(uint8_t port_id, uint32_t subport_id, uint8_t tc); -int qavg_subport(uint8_t port_id, uint32_t subport_id); +int subport_stat(uint16_t port_id, uint32_t subport_id); +int pipe_stat(uint16_t port_id, uint32_t subport_id, uint32_t pipe_id); +int qavg_q(uint16_t port_id, uint32_t subport_id, uint32_t pipe_id, + uint8_t tc, uint8_t q); +int qavg_tcpipe(uint16_t port_id, uint32_t subport_id, uint32_t pipe_id, + uint8_t tc); +int qavg_pipe(uint16_t port_id, uint32_t subport_id, uint32_t pipe_id); +int qavg_tcsubport(uint16_t port_id, uint32_t subport_id, uint8_t tc); +int qavg_subport(uint16_t port_id, uint32_t subport_id); #ifdef __cplusplus } diff --git a/dpdk/examples/qos_sched/stats.c b/dpdk/examples/qos_sched/stats.c index 5c894455..b5545e10 100644 --- a/dpdk/examples/qos_sched/stats.c +++ b/dpdk/examples/qos_sched/stats.c @@ -37,7 +37,8 @@ #include "main.h" int -qavg_q(uint8_t port_id, uint32_t subport_id, uint32_t pipe_id, uint8_t tc, uint8_t q) +qavg_q(uint16_t port_id, uint32_t subport_id, uint32_t pipe_id, uint8_t tc, + uint8_t q) { struct rte_sched_queue_stats stats; struct rte_sched_port *port; @@ -74,7 +75,8 @@ qavg_q(uint8_t port_id, uint32_t subport_id, uint32_t pipe_id, uint8_t tc, uint8 } int -qavg_tcpipe(uint8_t port_id, uint32_t subport_id, uint32_t pipe_id, uint8_t tc) +qavg_tcpipe(uint16_t port_id, uint32_t subport_id, uint32_t pipe_id, + uint8_t tc) { struct rte_sched_queue_stats stats; struct rte_sched_port *port; @@ -114,7 +116,7 @@ qavg_tcpipe(uint8_t port_id, uint32_t subport_id, uint32_t pipe_id, uint8_t tc) } int -qavg_pipe(uint8_t port_id, uint32_t subport_id, uint32_t pipe_id) +qavg_pipe(uint16_t port_id, uint32_t subport_id, uint32_t pipe_id) { struct rte_sched_queue_stats stats; struct rte_sched_port *port; @@ -153,7 +155,7 @@ qavg_pipe(uint8_t port_id, uint32_t subport_id, uint32_t pipe_id) } int -qavg_tcsubport(uint8_t port_id, uint32_t subport_id, uint8_t tc) +qavg_tcsubport(uint16_t port_id, uint32_t subport_id, uint8_t tc) { struct rte_sched_queue_stats stats; struct rte_sched_port *port; @@ -195,7 +197,7 @@ qavg_tcsubport(uint8_t port_id, uint32_t subport_id, uint8_t tc) } int -qavg_subport(uint8_t port_id, uint32_t subport_id) +qavg_subport(uint16_t port_id, uint32_t subport_id) { struct rte_sched_queue_stats stats; struct rte_sched_port *port; @@ -237,7 +239,7 @@ qavg_subport(uint8_t port_id, uint32_t subport_id) } int -subport_stat(uint8_t port_id, uint32_t subport_id) +subport_stat(uint16_t port_id, uint32_t subport_id) { struct rte_sched_subport_stats stats; struct rte_sched_port *port; @@ -273,7 +275,7 @@ subport_stat(uint8_t port_id, uint32_t subport_id) } int -pipe_stat(uint8_t port_id, uint32_t subport_id, uint32_t pipe_id) +pipe_stat(uint16_t port_id, uint32_t subport_id, uint32_t pipe_id) { struct rte_sched_queue_stats stats; struct rte_sched_port *port; diff --git a/dpdk/examples/quota_watermark/Makefile b/dpdk/examples/quota_watermark/Makefile index 17fe473b..40a01fa4 100644 --- a/dpdk/examples/quota_watermark/Makefile +++ b/dpdk/examples/quota_watermark/Makefile @@ -33,7 +33,7 @@ ifeq ($(RTE_SDK),) $(error "Please define RTE_SDK environment variable") endif -# Default target, can be overriden by command line or environment +# Default target, can be overridden by command line or environment RTE_TARGET ?= x86_64-native-linuxapp-gcc include $(RTE_SDK)/mk/rte.vars.mk diff --git a/dpdk/examples/quota_watermark/qw/Makefile b/dpdk/examples/quota_watermark/qw/Makefile index fac9328d..627897ce 100644 --- a/dpdk/examples/quota_watermark/qw/Makefile +++ b/dpdk/examples/quota_watermark/qw/Makefile @@ -33,7 +33,7 @@ ifeq ($(RTE_SDK),) $(error "Please define RTE_SDK environment variable") endif -# Default target, can be overriden by command line or environment +# Default target, can be overridden by command line or environment RTE_TARGET ?= x86_64-native-linuxapp-gcc include $(RTE_SDK)/mk/rte.vars.mk diff --git a/dpdk/examples/quota_watermark/qw/args.c b/dpdk/examples/quota_watermark/qw/args.c index 408b54d1..6ba77bc0 100644 --- a/dpdk/examples/quota_watermark/qw/args.c +++ b/dpdk/examples/quota_watermark/qw/args.c @@ -47,9 +47,9 @@ unsigned int portmask = 0; static void usage(const char *prgname) { - fprintf(stderr, "Usage: %s [EAL args] -- -p \n" - "-p PORTMASK: hexadecimal bitmask of NIC ports to configure\n", - prgname); + fprintf(stderr, "Usage: %s [EAL args] -- -p \n" + "-p PORTMASK: hexadecimal bitmask of NIC ports to configure\n", + prgname); } static unsigned long @@ -61,44 +61,47 @@ parse_portmask(const char *portmask_str) static void check_core_count(void) { - if (rte_lcore_count() < 3) - rte_exit(EXIT_FAILURE, "At least 3 cores need to be passed in the coremask\n"); + if (rte_lcore_count() < 3) + rte_exit(EXIT_FAILURE, + "At least 3 cores need to be passed in the coremask\n"); } static void check_portmask_value(unsigned int portmask) { - unsigned int port_nb = 0; + unsigned int port_nb = 0; - port_nb = __builtin_popcount(portmask); + port_nb = __builtin_popcount(portmask); - if (port_nb == 0) - rte_exit(EXIT_FAILURE, "At least 2 ports need to be passed in the portmask\n"); + if (port_nb == 0) + rte_exit(EXIT_FAILURE, + "At least 2 ports need to be passed in the portmask\n"); - if (port_nb % 2 != 0) - rte_exit(EXIT_FAILURE, "An even number of ports is required in the portmask\n"); + if (port_nb % 2 != 0) + rte_exit(EXIT_FAILURE, + "An even number of ports is required in the portmask\n"); } int parse_qw_args(int argc, char **argv) { - int opt; + int opt; - while ((opt = getopt(argc, argv, "h:p:")) != -1) { - switch (opt) { - case 'h': - usage(argv[0]); - break; - case 'p': - portmask = parse_portmask(optarg); - break; - default: - usage(argv[0]); - } - } + while ((opt = getopt(argc, argv, "h:p:")) != -1) { + switch (opt) { + case 'h': + usage(argv[0]); + break; + case 'p': + portmask = parse_portmask(optarg); + break; + default: + usage(argv[0]); + } + } - check_core_count(); - check_portmask_value(portmask); + check_core_count(); + check_portmask_value(portmask); - return 0; + return 0; } diff --git a/dpdk/examples/quota_watermark/qw/init.c b/dpdk/examples/quota_watermark/qw/init.c index c2087218..37b03626 100644 --- a/dpdk/examples/quota_watermark/qw/init.c +++ b/dpdk/examples/quota_watermark/qw/init.c @@ -51,124 +51,136 @@ static const struct rte_eth_conf port_conf = { - .rxmode = { - .split_hdr_size = 0, - .header_split = 0, /**< Header Split disabled */ - .hw_ip_checksum = 0, /**< IP checksum offload disabled */ - .hw_vlan_filter = 0, /**< VLAN filtering disabled */ - .jumbo_frame = 0, /**< Jumbo Frame Support disabled */ - .hw_strip_crc = 0, /**< CRC stripped by hardware */ - }, - .txmode = { - .mq_mode = ETH_DCB_NONE, - }, + .rxmode = { + .split_hdr_size = 0, + .header_split = 0, /**< Header Split disabled */ + .hw_ip_checksum = 0, /**< IP csum offload disabled */ + .hw_vlan_filter = 0, /**< VLAN filtering disabled */ + .jumbo_frame = 0, /**< Jumbo Frame disabled */ + .hw_strip_crc = 1, /**< CRC stripped by hardware */ + }, + .txmode = { + .mq_mode = ETH_DCB_NONE, + }, }; static struct rte_eth_fc_conf fc_conf = { - .mode = RTE_FC_TX_PAUSE, - .high_water = 80 * 510 / 100, - .low_water = 60 * 510 / 100, - .pause_time = 1337, - .send_xon = 0, + .mode = RTE_FC_TX_PAUSE, + .high_water = 80 * 510 / 100, + .low_water = 60 * 510 / 100, + .pause_time = 1337, + .send_xon = 0, }; -void configure_eth_port(uint8_t port_id) +void configure_eth_port(uint16_t port_id) { - int ret; + int ret; + uint16_t nb_rxd = RX_DESC_PER_QUEUE; + uint16_t nb_txd = TX_DESC_PER_QUEUE; - rte_eth_dev_stop(port_id); + rte_eth_dev_stop(port_id); - ret = rte_eth_dev_configure(port_id, 1, 1, &port_conf); - if (ret < 0) - rte_exit(EXIT_FAILURE, "Cannot configure port %u (error %d)\n", - (unsigned) port_id, ret); + ret = rte_eth_dev_configure(port_id, 1, 1, &port_conf); + if (ret < 0) + rte_exit(EXIT_FAILURE, "Cannot configure port %u (error %d)\n", + (unsigned int) port_id, ret); - /* Initialize the port's RX queue */ - ret = rte_eth_rx_queue_setup(port_id, 0, RX_DESC_PER_QUEUE, - rte_eth_dev_socket_id(port_id), - NULL, - mbuf_pool); - if (ret < 0) - rte_exit(EXIT_FAILURE, "Failed to setup RX queue on " - "port %u (error %d)\n", (unsigned) port_id, ret); + ret = rte_eth_dev_adjust_nb_rx_tx_desc(port_id, &nb_rxd, &nb_txd); + if (ret < 0) + rte_exit(EXIT_FAILURE, + "Cannot adjust number of descriptors for port %u (error %d)\n", + (unsigned int) port_id, ret); - /* Initialize the port's TX queue */ - ret = rte_eth_tx_queue_setup(port_id, 0, TX_DESC_PER_QUEUE, - rte_eth_dev_socket_id(port_id), - NULL); - if (ret < 0) - rte_exit(EXIT_FAILURE, "Failed to setup TX queue on " - "port %u (error %d)\n", (unsigned) port_id, ret); + /* Initialize the port's RX queue */ + ret = rte_eth_rx_queue_setup(port_id, 0, nb_rxd, + rte_eth_dev_socket_id(port_id), + NULL, + mbuf_pool); + if (ret < 0) + rte_exit(EXIT_FAILURE, + "Failed to setup RX queue on port %u (error %d)\n", + (unsigned int) port_id, ret); - /* Initialize the port's flow control */ - ret = rte_eth_dev_flow_ctrl_set(port_id, &fc_conf); - if (ret < 0) - rte_exit(EXIT_FAILURE, "Failed to setup hardware flow control on " - "port %u (error %d)\n", (unsigned) port_id, ret); + /* Initialize the port's TX queue */ + ret = rte_eth_tx_queue_setup(port_id, 0, nb_txd, + rte_eth_dev_socket_id(port_id), + NULL); + if (ret < 0) + rte_exit(EXIT_FAILURE, + "Failed to setup TX queue on port %u (error %d)\n", + (unsigned int) port_id, ret); - /* Start the port */ - ret = rte_eth_dev_start(port_id); - if (ret < 0) - rte_exit(EXIT_FAILURE, "Failed to start port %u (error %d)\n", - (unsigned) port_id, ret); + /* Initialize the port's flow control */ + ret = rte_eth_dev_flow_ctrl_set(port_id, &fc_conf); + if (ret < 0) + rte_exit(EXIT_FAILURE, + "Failed to setup hardware flow control on port %u (error %d)\n", + (unsigned int) port_id, ret); - /* Put it in promiscuous mode */ - rte_eth_promiscuous_enable(port_id); + /* Start the port */ + ret = rte_eth_dev_start(port_id); + if (ret < 0) + rte_exit(EXIT_FAILURE, "Failed to start port %u (error %d)\n", + (unsigned int) port_id, ret); + + /* Put it in promiscuous mode */ + rte_eth_promiscuous_enable(port_id); } void init_dpdk(void) { - if (rte_eth_dev_count() < 2) - rte_exit(EXIT_FAILURE, "Not enough ethernet port available\n"); + if (rte_eth_dev_count() < 2) + rte_exit(EXIT_FAILURE, "Not enough ethernet port available\n"); } -void init_ring(int lcore_id, uint8_t port_id) +void init_ring(int lcore_id, uint16_t port_id) { - struct rte_ring *ring; - char ring_name[RTE_RING_NAMESIZE]; + struct rte_ring *ring; + char ring_name[RTE_RING_NAMESIZE]; - snprintf(ring_name, RTE_RING_NAMESIZE, - "core%d_port%d", lcore_id, port_id); - ring = rte_ring_create(ring_name, RING_SIZE, rte_socket_id(), - RING_F_SP_ENQ | RING_F_SC_DEQ); + snprintf(ring_name, RTE_RING_NAMESIZE, + "core%d_port%d", lcore_id, port_id); + ring = rte_ring_create(ring_name, RING_SIZE, rte_socket_id(), + RING_F_SP_ENQ | RING_F_SC_DEQ); - if (ring == NULL) - rte_exit(EXIT_FAILURE, "%s\n", rte_strerror(rte_errno)); + if (ring == NULL) + rte_exit(EXIT_FAILURE, "%s\n", rte_strerror(rte_errno)); - rte_ring_set_water_mark(ring, 80 * RING_SIZE / 100); + *high_watermark = 80 * RING_SIZE / 100; - rings[lcore_id][port_id] = ring; + rings[lcore_id][port_id] = ring; } void pair_ports(void) { - uint8_t i, j; + uint16_t i, j; - /* Pair ports with their "closest neighbour" in the portmask */ - for (i = 0; i < RTE_MAX_ETHPORTS; i++) - if (is_bit_set(i, portmask)) - for (j = (uint8_t) (i + 1); j < RTE_MAX_ETHPORTS; j++) - if (is_bit_set(j, portmask)) { - port_pairs[i] = j; - port_pairs[j] = i; - i = j; - break; - } + /* Pair ports with their "closest neighbour" in the portmask */ + for (i = 0; i < RTE_MAX_ETHPORTS; i++) + if (is_bit_set(i, portmask)) + for (j = i + 1; j < RTE_MAX_ETHPORTS; j++) + if (is_bit_set(j, portmask)) { + port_pairs[i] = j; + port_pairs[j] = i; + i = j; + break; + } } void setup_shared_variables(void) { - const struct rte_memzone *qw_memzone; + const struct rte_memzone *qw_memzone; - qw_memzone = rte_memzone_reserve(QUOTA_WATERMARK_MEMZONE_NAME, 2 * sizeof(int), - rte_socket_id(), RTE_MEMZONE_2MB); - if (qw_memzone == NULL) - rte_exit(EXIT_FAILURE, "%s\n", rte_strerror(rte_errno)); + qw_memzone = rte_memzone_reserve(QUOTA_WATERMARK_MEMZONE_NAME, + 3 * sizeof(int), rte_socket_id(), 0); + if (qw_memzone == NULL) + rte_exit(EXIT_FAILURE, "%s\n", rte_strerror(rte_errno)); - quota = qw_memzone->addr; - low_watermark = (unsigned int *) qw_memzone->addr + 1; + quota = qw_memzone->addr; + low_watermark = (unsigned int *) qw_memzone->addr + 1; + high_watermark = (unsigned int *) qw_memzone->addr + 2; } diff --git a/dpdk/examples/quota_watermark/qw/init.h b/dpdk/examples/quota_watermark/qw/init.h index 6d0af3ab..2bfec2b1 100644 --- a/dpdk/examples/quota_watermark/qw/init.h +++ b/dpdk/examples/quota_watermark/qw/init.h @@ -34,9 +34,9 @@ #ifndef _INIT_H_ #define _INIT_H_ -void configure_eth_port(uint8_t port_id); +void configure_eth_port(uint16_t port_id); void init_dpdk(void); -void init_ring(int lcore_id, uint8_t port_id); +void init_ring(int lcore_id, uint16_t port_id); void pair_ports(void); void setup_shared_variables(void); diff --git a/dpdk/examples/quota_watermark/qw/main.c b/dpdk/examples/quota_watermark/qw/main.c index 8ed02148..fe174526 100644 --- a/dpdk/examples/quota_watermark/qw/main.c +++ b/dpdk/examples/quota_watermark/qw/main.c @@ -60,54 +60,56 @@ #define ETHER_TYPE_FLOW_CONTROL 0x8808 struct ether_fc_frame { - uint16_t opcode; - uint16_t param; + uint16_t opcode; + uint16_t param; } __attribute__((__packed__)); int *quota; unsigned int *low_watermark; +unsigned int *high_watermark; -uint8_t port_pairs[RTE_MAX_ETHPORTS]; +uint16_t port_pairs[RTE_MAX_ETHPORTS]; struct rte_ring *rings[RTE_MAX_LCORE][RTE_MAX_ETHPORTS]; struct rte_mempool *mbuf_pool; -static void send_pause_frame(uint8_t port_id, uint16_t duration) +static void send_pause_frame(uint16_t port_id, uint16_t duration) { - struct rte_mbuf *mbuf; - struct ether_fc_frame *pause_frame; - struct ether_hdr *hdr; - struct ether_addr mac_addr; + struct rte_mbuf *mbuf; + struct ether_fc_frame *pause_frame; + struct ether_hdr *hdr; + struct ether_addr mac_addr; - RTE_LOG(DEBUG, USER1, "Sending PAUSE frame (duration=%d) on port %d\n", - duration, port_id); + RTE_LOG_DP(DEBUG, USER1, + "Sending PAUSE frame (duration=%d) on port %d\n", + duration, port_id); - /* Get a mbuf from the pool */ - mbuf = rte_pktmbuf_alloc(mbuf_pool); - if (unlikely(mbuf == NULL)) - return; + /* Get a mbuf from the pool */ + mbuf = rte_pktmbuf_alloc(mbuf_pool); + if (unlikely(mbuf == NULL)) + return; - /* Prepare a PAUSE frame */ - hdr = rte_pktmbuf_mtod(mbuf, struct ether_hdr *); - pause_frame = (struct ether_fc_frame *) &hdr[1]; + /* Prepare a PAUSE frame */ + hdr = rte_pktmbuf_mtod(mbuf, struct ether_hdr *); + pause_frame = (struct ether_fc_frame *) &hdr[1]; - rte_eth_macaddr_get(port_id, &mac_addr); - ether_addr_copy(&mac_addr, &hdr->s_addr); + rte_eth_macaddr_get(port_id, &mac_addr); + ether_addr_copy(&mac_addr, &hdr->s_addr); - void *tmp = &hdr->d_addr.addr_bytes[0]; - *((uint64_t *)tmp) = 0x010000C28001ULL; + void *tmp = &hdr->d_addr.addr_bytes[0]; + *((uint64_t *)tmp) = 0x010000C28001ULL; - hdr->ether_type = rte_cpu_to_be_16(ETHER_TYPE_FLOW_CONTROL); + hdr->ether_type = rte_cpu_to_be_16(ETHER_TYPE_FLOW_CONTROL); - pause_frame->opcode = rte_cpu_to_be_16(0x0001); - pause_frame->param = rte_cpu_to_be_16(duration); + pause_frame->opcode = rte_cpu_to_be_16(0x0001); + pause_frame->param = rte_cpu_to_be_16(duration); - mbuf->pkt_len = 60; - mbuf->data_len = 60; + mbuf->pkt_len = 60; + mbuf->data_len = 60; - rte_eth_tx_burst(port_id, 0, &mbuf, 1); + rte_eth_tx_burst(port_id, 0, &mbuf, 1); } /** @@ -121,13 +123,13 @@ static void send_pause_frame(uint8_t port_id, uint16_t duration) static unsigned int get_previous_lcore_id(unsigned int lcore_id) { - int i; + int i; - for (i = lcore_id - 1; i >= 0; i--) - if (rte_lcore_is_enabled(i)) - return i; + for (i = lcore_id - 1; i >= 0; i--) + if (rte_lcore_is_enabled(i)) + return i; - return -1; + return -1; } /** @@ -139,125 +141,137 @@ get_previous_lcore_id(unsigned int lcore_id) static unsigned int get_last_lcore_id(void) { - int i; + int i; - for (i = RTE_MAX_LCORE; i >= 0; i--) - if (rte_lcore_is_enabled(i)) - return i; + for (i = RTE_MAX_LCORE; i >= 0; i--) + if (rte_lcore_is_enabled(i)) + return i; - return 0; + return 0; } static void receive_stage(__attribute__((unused)) void *args) { - int i, ret; + int i, ret; - uint8_t port_id; - uint16_t nb_rx_pkts; + uint16_t port_id; + uint16_t nb_rx_pkts; - unsigned int lcore_id; + unsigned int lcore_id; + unsigned int free; - struct rte_mbuf *pkts[MAX_PKT_QUOTA]; - struct rte_ring *ring; - enum ring_state ring_state[RTE_MAX_ETHPORTS] = { RING_READY }; + struct rte_mbuf *pkts[MAX_PKT_QUOTA]; + struct rte_ring *ring; + enum ring_state ring_state[RTE_MAX_ETHPORTS] = { RING_READY }; - lcore_id = rte_lcore_id(); + lcore_id = rte_lcore_id(); - RTE_LOG(INFO, USER1, - "%s() started on core %u\n", __func__, lcore_id); + RTE_LOG(INFO, USER1, + "%s() started on core %u\n", __func__, lcore_id); - while (1) { + while (1) { - /* Process each port round robin style */ - for (port_id = 0; port_id < RTE_MAX_ETHPORTS; port_id++) { + /* Process each port round robin style */ + for (port_id = 0; port_id < RTE_MAX_ETHPORTS; port_id++) { - if (!is_bit_set(port_id, portmask)) - continue; + if (!is_bit_set(port_id, portmask)) + continue; - ring = rings[lcore_id][port_id]; + ring = rings[lcore_id][port_id]; - if (ring_state[port_id] != RING_READY) { - if (rte_ring_count(ring) > *low_watermark) - continue; - else - ring_state[port_id] = RING_READY; - } + if (ring_state[port_id] != RING_READY) { + if (rte_ring_count(ring) > *low_watermark) + continue; + else + ring_state[port_id] = RING_READY; + } - /* Enqueue received packets on the RX ring */ - nb_rx_pkts = rte_eth_rx_burst(port_id, 0, pkts, (uint16_t) *quota); - ret = rte_ring_enqueue_bulk(ring, (void *) pkts, nb_rx_pkts); - if (ret == -EDQUOT) { - ring_state[port_id] = RING_OVERLOADED; - send_pause_frame(port_id, 1337); - } + /* Enqueue received packets on the RX ring */ + nb_rx_pkts = rte_eth_rx_burst(port_id, 0, pkts, + (uint16_t) *quota); + ret = rte_ring_enqueue_bulk(ring, (void *) pkts, + nb_rx_pkts, &free); + if (RING_SIZE - free > *high_watermark) { + ring_state[port_id] = RING_OVERLOADED; + send_pause_frame(port_id, 1337); + } - else if (ret == -ENOBUFS) { + if (ret == 0) { - /* Return mbufs to the pool, effectively dropping packets */ - for (i = 0; i < nb_rx_pkts; i++) - rte_pktmbuf_free(pkts[i]); - } - } - } + /* + * Return mbufs to the pool, + * effectively dropping packets + */ + for (i = 0; i < nb_rx_pkts; i++) + rte_pktmbuf_free(pkts[i]); + } + } + } } static void pipeline_stage(__attribute__((unused)) void *args) { - int i, ret; - int nb_dq_pkts; + int i, ret; + int nb_dq_pkts; - uint8_t port_id; + uint16_t port_id; - unsigned int lcore_id, previous_lcore_id; + unsigned int lcore_id, previous_lcore_id; + unsigned int free; - void *pkts[MAX_PKT_QUOTA]; - struct rte_ring *rx, *tx; - enum ring_state ring_state[RTE_MAX_ETHPORTS] = { RING_READY }; + void *pkts[MAX_PKT_QUOTA]; + struct rte_ring *rx, *tx; + enum ring_state ring_state[RTE_MAX_ETHPORTS] = { RING_READY }; - lcore_id = rte_lcore_id(); - previous_lcore_id = get_previous_lcore_id(lcore_id); + lcore_id = rte_lcore_id(); + previous_lcore_id = get_previous_lcore_id(lcore_id); - RTE_LOG(INFO, USER1, - "%s() started on core %u - processing packets from core %u\n", - __func__, lcore_id, previous_lcore_id); + RTE_LOG(INFO, USER1, + "%s() started on core %u - processing packets from core %u\n", + __func__, lcore_id, previous_lcore_id); - while (1) { + while (1) { - for (port_id = 0; port_id < RTE_MAX_ETHPORTS; port_id++) { + for (port_id = 0; port_id < RTE_MAX_ETHPORTS; port_id++) { - if (!is_bit_set(port_id, portmask)) - continue; + if (!is_bit_set(port_id, portmask)) + continue; - tx = rings[lcore_id][port_id]; - rx = rings[previous_lcore_id][port_id]; + tx = rings[lcore_id][port_id]; + rx = rings[previous_lcore_id][port_id]; - if (ring_state[port_id] != RING_READY) { - if (rte_ring_count(tx) > *low_watermark) - continue; - else - ring_state[port_id] = RING_READY; - } + if (ring_state[port_id] != RING_READY) { + if (rte_ring_count(tx) > *low_watermark) + continue; + else + ring_state[port_id] = RING_READY; + } - /* Dequeue up to quota mbuf from rx */ - nb_dq_pkts = rte_ring_dequeue_burst(rx, pkts, *quota); - if (unlikely(nb_dq_pkts < 0)) - continue; + /* Dequeue up to quota mbuf from rx */ + nb_dq_pkts = rte_ring_dequeue_burst(rx, pkts, + *quota, NULL); + if (unlikely(nb_dq_pkts < 0)) + continue; - /* Enqueue them on tx */ - ret = rte_ring_enqueue_bulk(tx, pkts, nb_dq_pkts); - if (ret == -EDQUOT) - ring_state[port_id] = RING_OVERLOADED; + /* Enqueue them on tx */ + ret = rte_ring_enqueue_bulk(tx, pkts, + nb_dq_pkts, &free); + if (RING_SIZE - free > *high_watermark) + ring_state[port_id] = RING_OVERLOADED; - else if (ret == -ENOBUFS) { + if (ret == 0) { - /* Return mbufs to the pool, effectively dropping packets */ - for (i = 0; i < nb_dq_pkts; i++) - rte_pktmbuf_free(pkts[i]); - } - } - } + /* + * Return mbufs to the pool, + * effectively dropping packets + */ + for (i = 0; i < nb_dq_pkts; i++) + rte_pktmbuf_free(pkts[i]); + } + } + } } static void @@ -265,108 +279,114 @@ send_stage(__attribute__((unused)) void *args) { uint16_t nb_dq_pkts; - uint8_t port_id; - uint8_t dest_port_id; + uint16_t port_id; + uint16_t dest_port_id; - unsigned int lcore_id, previous_lcore_id; + unsigned int lcore_id, previous_lcore_id; - struct rte_ring *tx; - struct rte_mbuf *tx_pkts[MAX_PKT_QUOTA]; + struct rte_ring *tx; + struct rte_mbuf *tx_pkts[MAX_PKT_QUOTA]; - lcore_id = rte_lcore_id(); - previous_lcore_id = get_previous_lcore_id(lcore_id); + lcore_id = rte_lcore_id(); + previous_lcore_id = get_previous_lcore_id(lcore_id); - RTE_LOG(INFO, USER1, - "%s() started on core %u - processing packets from core %u\n", - __func__, lcore_id, previous_lcore_id); + RTE_LOG(INFO, USER1, + "%s() started on core %u - processing packets from core %u\n", + __func__, lcore_id, previous_lcore_id); - while (1) { + while (1) { - /* Process each ring round robin style */ - for (port_id = 0; port_id < RTE_MAX_ETHPORTS; port_id++) { + /* Process each ring round robin style */ + for (port_id = 0; port_id < RTE_MAX_ETHPORTS; port_id++) { - if (!is_bit_set(port_id, portmask)) - continue; + if (!is_bit_set(port_id, portmask)) + continue; - dest_port_id = port_pairs[port_id]; - tx = rings[previous_lcore_id][port_id]; + dest_port_id = port_pairs[port_id]; + tx = rings[previous_lcore_id][port_id]; - if (rte_ring_empty(tx)) - continue; + if (rte_ring_empty(tx)) + continue; - /* Dequeue packets from tx and send them */ - nb_dq_pkts = (uint16_t) rte_ring_dequeue_burst(tx, (void *) tx_pkts, *quota); - rte_eth_tx_burst(dest_port_id, 0, tx_pkts, nb_dq_pkts); + /* Dequeue packets from tx and send them */ + nb_dq_pkts = (uint16_t) rte_ring_dequeue_burst(tx, + (void *) tx_pkts, *quota, NULL); + rte_eth_tx_burst(dest_port_id, 0, tx_pkts, nb_dq_pkts); - /* TODO: Check if nb_dq_pkts == nb_tx_pkts? */ - } - } + /* TODO: Check if nb_dq_pkts == nb_tx_pkts? */ + } + } } int main(int argc, char **argv) { - int ret; - unsigned int lcore_id, master_lcore_id, last_lcore_id; + int ret; + unsigned int lcore_id, master_lcore_id, last_lcore_id; - uint8_t port_id; + uint16_t port_id; - rte_set_log_level(RTE_LOG_INFO); + rte_log_set_global_level(RTE_LOG_INFO); - ret = rte_eal_init(argc, argv); - if (ret < 0) - rte_exit(EXIT_FAILURE, "Cannot initialize EAL\n"); + ret = rte_eal_init(argc, argv); + if (ret < 0) + rte_exit(EXIT_FAILURE, "Cannot initialize EAL\n"); - argc -= ret; - argv += ret; + argc -= ret; + argv += ret; - init_dpdk(); - setup_shared_variables(); + init_dpdk(); + setup_shared_variables(); - *quota = 32; - *low_watermark = 60 * RING_SIZE / 100; + *quota = 32; + *low_watermark = 60 * RING_SIZE / 100; - last_lcore_id = get_last_lcore_id(); - master_lcore_id = rte_get_master_lcore(); + last_lcore_id = get_last_lcore_id(); + master_lcore_id = rte_get_master_lcore(); - /* Parse the application's arguments */ - ret = parse_qw_args(argc, argv); - if (ret < 0) - rte_exit(EXIT_FAILURE, "Invalid quota/watermark argument(s)\n"); + /* Parse the application's arguments */ + ret = parse_qw_args(argc, argv); + if (ret < 0) + rte_exit(EXIT_FAILURE, "Invalid quota/watermark argument(s)\n"); - /* Create a pool of mbuf to store packets */ - mbuf_pool = rte_pktmbuf_pool_create("mbuf_pool", MBUF_PER_POOL, 32, 0, - MBUF_DATA_SIZE, rte_socket_id()); - if (mbuf_pool == NULL) - rte_panic("%s\n", rte_strerror(rte_errno)); + /* Create a pool of mbuf to store packets */ + mbuf_pool = rte_pktmbuf_pool_create("mbuf_pool", MBUF_PER_POOL, 32, 0, + MBUF_DATA_SIZE, rte_socket_id()); + if (mbuf_pool == NULL) + rte_panic("%s\n", rte_strerror(rte_errno)); - for (port_id = 0; port_id < RTE_MAX_ETHPORTS; port_id++) - if (is_bit_set(port_id, portmask)) { - configure_eth_port(port_id); - init_ring(master_lcore_id, port_id); - } + for (port_id = 0; port_id < RTE_MAX_ETHPORTS; port_id++) + if (is_bit_set(port_id, portmask)) { + configure_eth_port(port_id); + init_ring(master_lcore_id, port_id); + } - pair_ports(); + pair_ports(); - /* Start pipeline_connect() on all the available slave lcore but the last */ - for (lcore_id = 0 ; lcore_id < last_lcore_id; lcore_id++) { - if (rte_lcore_is_enabled(lcore_id) && lcore_id != master_lcore_id) { + /* + * Start pipeline_connect() on all the available slave lcores + * but the last + */ + for (lcore_id = 0 ; lcore_id < last_lcore_id; lcore_id++) { + if (rte_lcore_is_enabled(lcore_id) && + lcore_id != master_lcore_id) { - for (port_id = 0; port_id < RTE_MAX_ETHPORTS; port_id++) - if (is_bit_set(port_id, portmask)) - init_ring(lcore_id, port_id); + for (port_id = 0; port_id < RTE_MAX_ETHPORTS; port_id++) + if (is_bit_set(port_id, portmask)) + init_ring(lcore_id, port_id); - /* typecast is a workaround for GCC 4.3 bug */ - rte_eal_remote_launch((int (*)(void *))pipeline_stage, NULL, lcore_id); - } - } + /* typecast is a workaround for GCC 4.3 bug */ + rte_eal_remote_launch((int (*)(void *))pipeline_stage, + NULL, lcore_id); + } + } - /* Start send_stage() on the last slave core */ - /* typecast is a workaround for GCC 4.3 bug */ - rte_eal_remote_launch((int (*)(void *))send_stage, NULL, last_lcore_id); + /* Start send_stage() on the last slave core */ + /* typecast is a workaround for GCC 4.3 bug */ + rte_eal_remote_launch((int (*)(void *))send_stage, NULL, last_lcore_id); - /* Start receive_stage() on the master core */ - receive_stage(NULL); + /* Start receive_stage() on the master core */ + receive_stage(NULL); - return 0; + return 0; } diff --git a/dpdk/examples/quota_watermark/qw/main.h b/dpdk/examples/quota_watermark/qw/main.h index 6b364898..ebed7b2f 100644 --- a/dpdk/examples/quota_watermark/qw/main.h +++ b/dpdk/examples/quota_watermark/qw/main.h @@ -37,14 +37,15 @@ #include "../include/conf.h" enum ring_state { - RING_READY, - RING_OVERLOADED, + RING_READY, + RING_OVERLOADED, }; extern int *quota; extern unsigned int *low_watermark; +extern unsigned int *high_watermark; -extern uint8_t port_pairs[RTE_MAX_ETHPORTS]; +extern uint16_t port_pairs[RTE_MAX_ETHPORTS]; extern struct rte_ring *rings[RTE_MAX_LCORE][RTE_MAX_ETHPORTS]; extern struct rte_mempool *mbuf_pool; @@ -53,7 +54,7 @@ extern struct rte_mempool *mbuf_pool; static inline int is_bit_set(int i, unsigned int mask) { - return (1 << i) & mask; + return (1 << i) & mask; } #endif /* _MAIN_H_ */ diff --git a/dpdk/examples/quota_watermark/qwctl/Makefile b/dpdk/examples/quota_watermark/qwctl/Makefile index 1ca2f1e9..e0f0083d 100644 --- a/dpdk/examples/quota_watermark/qwctl/Makefile +++ b/dpdk/examples/quota_watermark/qwctl/Makefile @@ -33,7 +33,7 @@ ifeq ($(RTE_SDK),) $(error "Please define RTE_SDK environment variable") endif -# Default target, can be overriden by command line or environment +# Default target, can be overridden by command line or environment RTE_TARGET ?= x86_64-native-linuxapp-gcc include $(RTE_SDK)/mk/rte.vars.mk diff --git a/dpdk/examples/quota_watermark/qwctl/commands.c b/dpdk/examples/quota_watermark/qwctl/commands.c index 5348dd3d..5cac0e17 100644 --- a/dpdk/examples/quota_watermark/qwctl/commands.c +++ b/dpdk/examples/quota_watermark/qwctl/commands.c @@ -53,36 +53,36 @@ */ struct cmd_help_tokens { - cmdline_fixed_string_t verb; + cmdline_fixed_string_t verb; }; cmdline_parse_token_string_t cmd_help_verb = - TOKEN_STRING_INITIALIZER(struct cmd_help_tokens, verb, "help"); + TOKEN_STRING_INITIALIZER(struct cmd_help_tokens, verb, "help"); static void cmd_help_handler(__attribute__((unused)) void *parsed_result, - struct cmdline *cl, - __attribute__((unused)) void *data) + struct cmdline *cl, + __attribute__((unused)) void *data) { - cmdline_printf(cl, "Available commands:\n" - "- help\n" - "- set [ring_name|variable] \n" - "- show [ring_name|variable]\n" - "\n" - "Available variables:\n" - "- low_watermark\n" - "- quota\n" - "- ring names follow the core%%u_port%%u format\n"); + cmdline_printf(cl, "Available commands:\n" + "- help\n" + "- set [ring_name|variable] \n" + "- show [ring_name|variable]\n" + "\n" + "Available variables:\n" + "- low_watermark\n" + "- quota\n" + "- ring names follow the core%%u_port%%u format\n"); } cmdline_parse_inst_t cmd_help = { - .f = cmd_help_handler, - .data = NULL, - .help_str = "show help", - .tokens = { - (void *) &cmd_help_verb, - NULL, - }, + .f = cmd_help_handler, + .data = NULL, + .help_str = "show help", + .tokens = { + (void *) &cmd_help_verb, + NULL, + }, }; @@ -91,69 +91,74 @@ cmdline_parse_inst_t cmd_help = { */ struct cmd_set_tokens { - cmdline_fixed_string_t verb; - cmdline_fixed_string_t variable; - uint32_t value; + cmdline_fixed_string_t verb; + cmdline_fixed_string_t variable; + uint32_t value; }; cmdline_parse_token_string_t cmd_set_verb = - TOKEN_STRING_INITIALIZER(struct cmd_set_tokens, verb, "set"); + TOKEN_STRING_INITIALIZER(struct cmd_set_tokens, verb, "set"); cmdline_parse_token_string_t cmd_set_variable = - TOKEN_STRING_INITIALIZER(struct cmd_set_tokens, variable, NULL); + TOKEN_STRING_INITIALIZER(struct cmd_set_tokens, variable, NULL); cmdline_parse_token_num_t cmd_set_value = - TOKEN_NUM_INITIALIZER(struct cmd_set_tokens, value, UINT32); + TOKEN_NUM_INITIALIZER(struct cmd_set_tokens, value, UINT32); static void cmd_set_handler(__attribute__((unused)) void *parsed_result, - struct cmdline *cl, - __attribute__((unused)) void *data) + struct cmdline *cl, + __attribute__((unused)) void *data) { - struct cmd_set_tokens *tokens = parsed_result; - struct rte_ring *ring; + struct cmd_set_tokens *tokens = parsed_result; + struct rte_ring *ring; - if (!strcmp(tokens->variable, "quota")) { + if (!strcmp(tokens->variable, "quota")) { - if (tokens->value > 0 && tokens->value <= MAX_PKT_QUOTA) - *quota = tokens->value; - else - cmdline_printf(cl, "quota must be between 1 and %u\n", MAX_PKT_QUOTA); - } + if (tokens->value > 0 && tokens->value <= MAX_PKT_QUOTA) + *quota = tokens->value; + else + cmdline_printf(cl, "quota must be between 1 and %u\n", + MAX_PKT_QUOTA); + } - else if (!strcmp(tokens->variable, "low_watermark")) { + else if (!strcmp(tokens->variable, "low_watermark")) { - if (tokens->value <= 100) - *low_watermark = tokens->value * RING_SIZE / 100; - else - cmdline_printf(cl, "low_watermark must be between 0%% and 100%%\n"); - } + if (tokens->value <= 100) + *low_watermark = tokens->value * RING_SIZE / 100; + else + cmdline_printf(cl, + "low_watermark must be between 0%% and 100%%\n"); + } - else { + else { - ring = rte_ring_lookup(tokens->variable); - if (ring == NULL) - cmdline_printf(cl, "Cannot find ring \"%s\"\n", tokens->variable); - else - if (tokens->value >= *low_watermark * 100 / RING_SIZE - && tokens->value <= 100) - rte_ring_set_water_mark(ring, tokens->value * RING_SIZE / 100); - else - cmdline_printf(cl, "ring high watermark must be between %u%% " - "and 100%%\n", *low_watermark * 100 / RING_SIZE); - } + ring = rte_ring_lookup(tokens->variable); + if (ring == NULL) + cmdline_printf(cl, "Cannot find ring \"%s\"\n", + tokens->variable); + else + if (tokens->value >= *low_watermark * 100 / RING_SIZE + && tokens->value <= 100) + *high_watermark = tokens->value * + RING_SIZE / 100; + else + cmdline_printf(cl, + "ring high watermark must be between %u%% and 100%%\n", + *low_watermark * 100 / RING_SIZE); + } } cmdline_parse_inst_t cmd_set = { - .f = cmd_set_handler, - .data = NULL, - .help_str = "Set a variable value", - .tokens = { - (void *) &cmd_set_verb, - (void *) &cmd_set_variable, - (void *) &cmd_set_value, - NULL, - }, + .f = cmd_set_handler, + .data = NULL, + .help_str = "Set a variable value", + .tokens = { + (void *) &cmd_set_verb, + (void *) &cmd_set_variable, + (void *) &cmd_set_value, + NULL, + }, }; @@ -162,56 +167,59 @@ cmdline_parse_inst_t cmd_set = { */ struct cmd_show_tokens { - cmdline_fixed_string_t verb; - cmdline_fixed_string_t variable; + cmdline_fixed_string_t verb; + cmdline_fixed_string_t variable; }; cmdline_parse_token_string_t cmd_show_verb = - TOKEN_STRING_INITIALIZER(struct cmd_show_tokens, verb, "show"); + TOKEN_STRING_INITIALIZER(struct cmd_show_tokens, verb, "show"); cmdline_parse_token_string_t cmd_show_variable = - TOKEN_STRING_INITIALIZER(struct cmd_show_tokens, variable, NULL); + TOKEN_STRING_INITIALIZER(struct cmd_show_tokens, + variable, NULL); static void cmd_show_handler(__attribute__((unused)) void *parsed_result, - struct cmdline *cl, - __attribute__((unused)) void *data) + struct cmdline *cl, + __attribute__((unused)) void *data) { - struct cmd_show_tokens *tokens = parsed_result; - struct rte_ring *ring; + struct cmd_show_tokens *tokens = parsed_result; + struct rte_ring *ring; - if (!strcmp(tokens->variable, "quota")) - cmdline_printf(cl, "Global quota: %d\n", *quota); + if (!strcmp(tokens->variable, "quota")) + cmdline_printf(cl, "Global quota: %d\n", *quota); - else if (!strcmp(tokens->variable, "low_watermark")) - cmdline_printf(cl, "Global low_watermark: %u\n", *low_watermark); + else if (!strcmp(tokens->variable, "low_watermark")) + cmdline_printf(cl, "Global low_watermark: %u\n", + *low_watermark); - else { + else { - ring = rte_ring_lookup(tokens->variable); - if (ring == NULL) - cmdline_printf(cl, "Cannot find ring \"%s\"\n", tokens->variable); - else - rte_ring_dump(stdout, ring); - } + ring = rte_ring_lookup(tokens->variable); + if (ring == NULL) + cmdline_printf(cl, "Cannot find ring \"%s\"\n", + tokens->variable); + else + rte_ring_dump(stdout, ring); + } } cmdline_parse_inst_t cmd_show = { - .f = cmd_show_handler, - .data = NULL, - .help_str = "Show a variable value", - .tokens = { - (void *) &cmd_show_verb, - (void *) &cmd_show_variable, - NULL, - }, + .f = cmd_show_handler, + .data = NULL, + .help_str = "Show a variable value", + .tokens = { + (void *) &cmd_show_verb, + (void *) &cmd_show_variable, + NULL, + }, }; cmdline_parse_ctx_t qwctl_ctx[] = { - (cmdline_parse_inst_t *)&cmd_help, - (cmdline_parse_inst_t *)&cmd_set, - (cmdline_parse_inst_t *)&cmd_show, - NULL, + (cmdline_parse_inst_t *)&cmd_help, + (cmdline_parse_inst_t *)&cmd_set, + (cmdline_parse_inst_t *)&cmd_show, + NULL, }; diff --git a/dpdk/examples/quota_watermark/qwctl/qwctl.c b/dpdk/examples/quota_watermark/qwctl/qwctl.c index 4961089b..18ec17a1 100644 --- a/dpdk/examples/quota_watermark/qwctl/qwctl.c +++ b/dpdk/examples/quota_watermark/qwctl/qwctl.c @@ -41,7 +41,6 @@ #include #include -#include #include #include @@ -56,40 +55,42 @@ int *quota; unsigned int *low_watermark; +unsigned int *high_watermark; static void setup_shared_variables(void) { - const struct rte_memzone *qw_memzone; + const struct rte_memzone *qw_memzone; - qw_memzone = rte_memzone_lookup(QUOTA_WATERMARK_MEMZONE_NAME); - if (qw_memzone == NULL) - rte_exit(EXIT_FAILURE, "Couldn't find memzone\n"); + qw_memzone = rte_memzone_lookup(QUOTA_WATERMARK_MEMZONE_NAME); + if (qw_memzone == NULL) + rte_exit(EXIT_FAILURE, "Couldn't find memzone\n"); - quota = qw_memzone->addr; - low_watermark = (unsigned int *) qw_memzone->addr + 1; + quota = qw_memzone->addr; + low_watermark = (unsigned int *) qw_memzone->addr + 1; + high_watermark = (unsigned int *) qw_memzone->addr + 2; } int main(int argc, char **argv) { - int ret; - struct cmdline *cl; + int ret; + struct cmdline *cl; - rte_set_log_level(RTE_LOG_INFO); + rte_log_set_global_level(RTE_LOG_INFO); - ret = rte_eal_init(argc, argv); - if (ret < 0) - rte_exit(EXIT_FAILURE, "Cannot initialize EAL\n"); + ret = rte_eal_init(argc, argv); + if (ret < 0) + rte_exit(EXIT_FAILURE, "Cannot initialize EAL\n"); - setup_shared_variables(); + setup_shared_variables(); - cl = cmdline_stdin_new(qwctl_ctx, "qwctl> "); - if (cl == NULL) - rte_exit(EXIT_FAILURE, "Cannot create cmdline instance\n"); + cl = cmdline_stdin_new(qwctl_ctx, "qwctl> "); + if (cl == NULL) + rte_exit(EXIT_FAILURE, "Cannot create cmdline instance\n"); - cmdline_interact(cl); - cmdline_stdin_exit(cl); + cmdline_interact(cl); + cmdline_stdin_exit(cl); - return 0; + return 0; } diff --git a/dpdk/examples/quota_watermark/qwctl/qwctl.h b/dpdk/examples/quota_watermark/qwctl/qwctl.h index 8d146e57..545914b3 100644 --- a/dpdk/examples/quota_watermark/qwctl/qwctl.h +++ b/dpdk/examples/quota_watermark/qwctl/qwctl.h @@ -36,5 +36,6 @@ extern int *quota; extern unsigned int *low_watermark; +extern unsigned int *high_watermark; #endif /* _MAIN_H_ */ diff --git a/dpdk/examples/rxtx_callbacks/main.c b/dpdk/examples/rxtx_callbacks/main.c index 048b23f5..ca135d21 100644 --- a/dpdk/examples/rxtx_callbacks/main.c +++ b/dpdk/examples/rxtx_callbacks/main.c @@ -59,7 +59,7 @@ static struct { static uint16_t -add_timestamps(uint8_t port __rte_unused, uint16_t qidx __rte_unused, +add_timestamps(uint16_t port __rte_unused, uint16_t qidx __rte_unused, struct rte_mbuf **pkts, uint16_t nb_pkts, uint16_t max_pkts __rte_unused, void *_ __rte_unused) { @@ -72,7 +72,7 @@ add_timestamps(uint8_t port __rte_unused, uint16_t qidx __rte_unused, } static uint16_t -calc_latency(uint8_t port __rte_unused, uint16_t qidx __rte_unused, +calc_latency(uint16_t port __rte_unused, uint16_t qidx __rte_unused, struct rte_mbuf **pkts, uint16_t nb_pkts, void *_ __rte_unused) { uint64_t cycles = 0; @@ -97,10 +97,12 @@ calc_latency(uint8_t port __rte_unused, uint16_t qidx __rte_unused, * coming from the mbuf_pool passed as parameter */ static inline int -port_init(uint8_t port, struct rte_mempool *mbuf_pool) +port_init(uint16_t port, struct rte_mempool *mbuf_pool) { struct rte_eth_conf port_conf = port_conf_default; const uint16_t rx_rings = 1, tx_rings = 1; + uint16_t nb_rxd = RX_RING_SIZE; + uint16_t nb_txd = TX_RING_SIZE; int retval; uint16_t q; @@ -111,15 +113,19 @@ port_init(uint8_t port, struct rte_mempool *mbuf_pool) if (retval != 0) return retval; + retval = rte_eth_dev_adjust_nb_rx_tx_desc(port, &nb_rxd, &nb_txd); + if (retval != 0) + return retval; + for (q = 0; q < rx_rings; q++) { - retval = rte_eth_rx_queue_setup(port, q, RX_RING_SIZE, + retval = rte_eth_rx_queue_setup(port, q, nb_rxd, rte_eth_dev_socket_id(port), NULL, mbuf_pool); if (retval < 0) return retval; } for (q = 0; q < tx_rings; q++) { - retval = rte_eth_tx_queue_setup(port, q, TX_RING_SIZE, + retval = rte_eth_tx_queue_setup(port, q, nb_txd, rte_eth_dev_socket_id(port), NULL); if (retval < 0) return retval; @@ -153,7 +159,7 @@ port_init(uint8_t port, struct rte_mempool *mbuf_pool) static __attribute__((noreturn)) void lcore_main(void) { - uint8_t port; + uint16_t port; for (port = 0; port < nb_ports; port++) if (rte_eth_dev_socket_id(port) > 0 && @@ -189,7 +195,7 @@ int main(int argc, char *argv[]) { struct rte_mempool *mbuf_pool; - uint8_t portid; + uint16_t portid; /* init EAL */ int ret = rte_eal_init(argc, argv); diff --git a/dpdk/examples/server_node_efd/Makefile b/dpdk/examples/server_node_efd/Makefile new file mode 100644 index 00000000..6977ef93 --- /dev/null +++ b/dpdk/examples/server_node_efd/Makefile @@ -0,0 +1,44 @@ +# BSD LICENSE +# +# Copyright(c) 2016-2017 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +ifeq ($(RTE_SDK),) +$(error "Please define RTE_SDK environment variable") +endif + +# Default target, can be overridden by command line or environment +RTE_TARGET ?= x86_64-native-linuxapp-gcc + +include $(RTE_SDK)/mk/rte.vars.mk + +DIRS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += server +DIRS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += node + +include $(RTE_SDK)/mk/rte.extsubdir.mk diff --git a/dpdk/examples/server_node_efd/node/Makefile b/dpdk/examples/server_node_efd/node/Makefile new file mode 100644 index 00000000..8cf7b650 --- /dev/null +++ b/dpdk/examples/server_node_efd/node/Makefile @@ -0,0 +1,48 @@ +# BSD LICENSE +# +# Copyright(c) 2016-2017 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +ifeq ($(RTE_SDK),) +$(error "Please define RTE_SDK environment variable") +endif + +# Default target, can be overridden by command line or environment +include $(RTE_SDK)/mk/rte.vars.mk + +# binary name +APP = node + +# all source are stored in SRCS-y +SRCS-y := node.c + +CFLAGS += $(WERROR_FLAGS) -O3 +CFLAGS += -I$(SRCDIR)/../shared + +include $(RTE_SDK)/mk/rte.extapp.mk diff --git a/dpdk/examples/server_node_efd/node/node.c b/dpdk/examples/server_node_efd/node/node.c new file mode 100644 index 00000000..5aa1258e --- /dev/null +++ b/dpdk/examples/server_node_efd/node/node.c @@ -0,0 +1,415 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common.h" + +/* Number of packets to attempt to read from queue */ +#define PKT_READ_SIZE ((uint16_t)32) + +/* + * Our node id number - tells us which rx queue to read, and NIC TX + * queue to write to. + */ +static uint8_t node_id; + +#define MBQ_CAPACITY 32 + +/* maps input ports to output ports for packets */ +static uint16_t output_ports[RTE_MAX_ETHPORTS]; + +/* buffers up a set of packet that are ready to send */ +struct rte_eth_dev_tx_buffer *tx_buffer[RTE_MAX_ETHPORTS]; + +/* shared data from server. We update statistics here */ +static struct tx_stats *tx_stats; + +static struct filter_stats *filter_stats; + +/* + * print a usage message + */ +static void +usage(const char *progname) +{ + printf("Usage: %s [EAL args] -- -n \n\n", progname); +} + +/* + * Convert the node id number from a string to an int. + */ +static int +parse_node_num(const char *node) +{ + char *end = NULL; + unsigned long temp; + + if (node == NULL || *node == '\0') + return -1; + + temp = strtoul(node, &end, 10); + if (end == NULL || *end != '\0') + return -1; + + node_id = (uint8_t)temp; + return 0; +} + +/* + * Parse the application arguments to the node app. + */ +static int +parse_app_args(int argc, char *argv[]) +{ + int option_index, opt; + char **argvopt = argv; + const char *progname = NULL; + static struct option lgopts[] = { /* no long options */ + {NULL, 0, 0, 0 } + }; + progname = argv[0]; + + while ((opt = getopt_long(argc, argvopt, "n:", lgopts, + &option_index)) != EOF) { + switch (opt) { + case 'n': + if (parse_node_num(optarg) != 0) { + usage(progname); + return -1; + } + break; + default: + usage(progname); + return -1; + } + } + return 0; +} + +/* + * Tx buffer error callback + */ +static void +flush_tx_error_callback(struct rte_mbuf **unsent, uint16_t count, + void *userdata) { + int i; + uint16_t port_id = (uintptr_t)userdata; + + tx_stats->tx_drop[port_id] += count; + + /* free the mbufs which failed from transmit */ + for (i = 0; i < count; i++) + rte_pktmbuf_free(unsent[i]); + +} + +static void +configure_tx_buffer(uint16_t port_id, uint16_t size) +{ + int ret; + + /* Initialize TX buffers */ + tx_buffer[port_id] = rte_zmalloc_socket("tx_buffer", + RTE_ETH_TX_BUFFER_SIZE(size), 0, + rte_eth_dev_socket_id(port_id)); + if (tx_buffer[port_id] == NULL) + rte_exit(EXIT_FAILURE, + "Cannot allocate buffer for tx on port %u\n", port_id); + + rte_eth_tx_buffer_init(tx_buffer[port_id], size); + + ret = rte_eth_tx_buffer_set_err_callback(tx_buffer[port_id], + flush_tx_error_callback, (void *)(intptr_t)port_id); + if (ret < 0) + rte_exit(EXIT_FAILURE, + "Cannot set error callback for tx buffer on port %u\n", + port_id); +} + +/* + * set up output ports so that all traffic on port gets sent out + * its paired port. Index using actual port numbers since that is + * what comes in the mbuf structure. + */ +static void +configure_output_ports(const struct shared_info *info) +{ + int i; + + if (info->num_ports > RTE_MAX_ETHPORTS) + rte_exit(EXIT_FAILURE, "Too many ethernet ports. " + "RTE_MAX_ETHPORTS = %u\n", + (unsigned int)RTE_MAX_ETHPORTS); + for (i = 0; i < info->num_ports - 1; i += 2) { + uint8_t p1 = info->id[i]; + uint8_t p2 = info->id[i+1]; + + output_ports[p1] = p2; + output_ports[p2] = p1; + + configure_tx_buffer(p1, MBQ_CAPACITY); + configure_tx_buffer(p2, MBQ_CAPACITY); + + } +} + +/* + * Create the hash table that will contain the flows that + * the node will handle, which will be used to decide if packet + * is transmitted or dropped. + */ +static struct rte_hash * +create_hash_table(const struct shared_info *info) +{ + uint32_t num_flows_node = info->num_flows / info->num_nodes; + char name[RTE_HASH_NAMESIZE]; + struct rte_hash *h; + + /* create table */ + struct rte_hash_parameters hash_params = { + .entries = num_flows_node * 2, /* table load = 50% */ + .key_len = sizeof(uint32_t), /* Store IPv4 dest IP address */ + .socket_id = rte_socket_id(), + .hash_func_init_val = 0, + }; + + snprintf(name, sizeof(name), "hash_table_%d", node_id); + hash_params.name = name; + h = rte_hash_create(&hash_params); + + if (h == NULL) + rte_exit(EXIT_FAILURE, + "Problem creating the hash table for node %d\n", + node_id); + return h; +} + +static void +populate_hash_table(const struct rte_hash *h, const struct shared_info *info) +{ + unsigned int i; + int32_t ret; + uint32_t ip_dst; + uint32_t num_flows_node = 0; + uint64_t target_node; + + /* Add flows in table */ + for (i = 0; i < info->num_flows; i++) { + target_node = i % info->num_nodes; + if (target_node != node_id) + continue; + + ip_dst = rte_cpu_to_be_32(i); + + ret = rte_hash_add_key(h, (void *) &ip_dst); + if (ret < 0) + rte_exit(EXIT_FAILURE, "Unable to add entry %u " + "in hash table\n", i); + else + num_flows_node++; + + } + + printf("Hash table: Adding 0x%x keys\n", num_flows_node); +} + +/* + * This function performs routing of packets + * Just sends each input packet out an output port based solely on the input + * port it arrived on. + */ +static inline void +transmit_packet(struct rte_mbuf *buf) +{ + int sent; + const uint16_t in_port = buf->port; + const uint16_t out_port = output_ports[in_port]; + struct rte_eth_dev_tx_buffer *buffer = tx_buffer[out_port]; + + sent = rte_eth_tx_buffer(out_port, node_id, buffer, buf); + if (sent) + tx_stats->tx[out_port] += sent; + +} + +static inline void +handle_packets(struct rte_hash *h, struct rte_mbuf **bufs, uint16_t num_packets) +{ + struct ipv4_hdr *ipv4_hdr; + uint32_t ipv4_dst_ip[PKT_READ_SIZE]; + const void *key_ptrs[PKT_READ_SIZE]; + unsigned int i; + int32_t positions[PKT_READ_SIZE] = {0}; + + for (i = 0; i < num_packets; i++) { + /* Handle IPv4 header.*/ + ipv4_hdr = rte_pktmbuf_mtod_offset(bufs[i], struct ipv4_hdr *, + sizeof(struct ether_hdr)); + ipv4_dst_ip[i] = ipv4_hdr->dst_addr; + key_ptrs[i] = &ipv4_dst_ip[i]; + } + /* Check if packets belongs to any flows handled by this node */ + rte_hash_lookup_bulk(h, key_ptrs, num_packets, positions); + + for (i = 0; i < num_packets; i++) { + if (likely(positions[i] >= 0)) { + filter_stats->passed++; + transmit_packet(bufs[i]); + } else { + filter_stats->drop++; + /* Drop packet, as flow is not handled by this node */ + rte_pktmbuf_free(bufs[i]); + } + } +} + +/* + * Application main function - loops through + * receiving and processing packets. Never returns + */ +int +main(int argc, char *argv[]) +{ + const struct rte_memzone *mz; + struct rte_ring *rx_ring; + struct rte_hash *h; + struct rte_mempool *mp; + struct shared_info *info; + int need_flush = 0; /* indicates whether we have unsent packets */ + int retval; + void *pkts[PKT_READ_SIZE]; + uint16_t sent; + + retval = rte_eal_init(argc, argv); + if (retval < 0) + return -1; + argc -= retval; + argv += retval; + + if (parse_app_args(argc, argv) < 0) + rte_exit(EXIT_FAILURE, "Invalid command-line arguments\n"); + + if (rte_eth_dev_count() == 0) + rte_exit(EXIT_FAILURE, "No Ethernet ports - bye\n"); + + rx_ring = rte_ring_lookup(get_rx_queue_name(node_id)); + if (rx_ring == NULL) + rte_exit(EXIT_FAILURE, "Cannot get RX ring - " + "is server process running?\n"); + + mp = rte_mempool_lookup(PKTMBUF_POOL_NAME); + if (mp == NULL) + rte_exit(EXIT_FAILURE, "Cannot get mempool for mbufs\n"); + + mz = rte_memzone_lookup(MZ_SHARED_INFO); + if (mz == NULL) + rte_exit(EXIT_FAILURE, "Cannot get port info structure\n"); + info = mz->addr; + tx_stats = &(info->tx_stats[node_id]); + filter_stats = &(info->filter_stats[node_id]); + + configure_output_ports(info); + + h = create_hash_table(info); + + populate_hash_table(h, info); + + RTE_LOG(INFO, APP, "Finished Process Init.\n"); + + printf("\nNode process %d handling packets\n", node_id); + printf("[Press Ctrl-C to quit ...]\n"); + + for (;;) { + uint16_t rx_pkts = PKT_READ_SIZE; + uint16_t port; + + /* + * Try dequeuing max possible packets first, if that fails, + * get the most we can. Loop body should only execute once, + * maximum + */ + while (rx_pkts > 0 && + unlikely(rte_ring_dequeue_bulk(rx_ring, pkts, + rx_pkts, NULL) == 0)) + rx_pkts = (uint16_t)RTE_MIN(rte_ring_count(rx_ring), + PKT_READ_SIZE); + + if (unlikely(rx_pkts == 0)) { + if (need_flush) + for (port = 0; port < info->num_ports; port++) { + sent = rte_eth_tx_buffer_flush( + info->id[port], + node_id, + tx_buffer[port]); + if (unlikely(sent)) + tx_stats->tx[port] += sent; + } + need_flush = 0; + continue; + } + + handle_packets(h, (struct rte_mbuf **)pkts, rx_pkts); + + need_flush = 1; + } +} diff --git a/dpdk/examples/server_node_efd/server/Makefile b/dpdk/examples/server_node_efd/server/Makefile new file mode 100644 index 00000000..9f1fe289 --- /dev/null +++ b/dpdk/examples/server_node_efd/server/Makefile @@ -0,0 +1,57 @@ +# BSD LICENSE +# +# Copyright(c) 2016-2017 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +ifeq ($(RTE_SDK),) +$(error "Please define RTE_SDK environment variable") +endif + +# Default target, can be overridden by command line or environment +RTE_TARGET ?= x86_64-native-linuxapp-gcc + +include $(RTE_SDK)/mk/rte.vars.mk + +ifneq ($(CONFIG_RTE_EXEC_ENV), "linuxapp") +$(error This application can only operate in a linuxapp environment, \ +please change the definition of the RTE_TARGET environment variable) +endif + +# binary name +APP = server + +# all source are stored in SRCS-y +SRCS-y := main.c init.c args.c + +INC := $(sort $(wildcard *.h)) + +CFLAGS += $(WERROR_FLAGS) -O3 +CFLAGS += -I$(SRCDIR)/../shared + +include $(RTE_SDK)/mk/rte.extapp.mk diff --git a/dpdk/examples/server_node_efd/server/args.c b/dpdk/examples/server_node_efd/server/args.c new file mode 100644 index 00000000..ee292038 --- /dev/null +++ b/dpdk/examples/server_node_efd/server/args.c @@ -0,0 +1,200 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "common.h" +#include "args.h" +#include "init.h" + +/* 1M flows by default */ +#define DEFAULT_NUM_FLOWS 0x100000 + +/* global var for number of nodes - extern in header */ +uint8_t num_nodes; +/* global var for number of flows - extern in header */ +uint32_t num_flows = DEFAULT_NUM_FLOWS; + +static const char *progname; + +/** + * Prints out usage information to stdout + */ +static void +usage(void) +{ + printf("%s [EAL options] -- -p PORTMASK -n NUM_NODES -f NUM_FLOWS\n" + " -p PORTMASK: hexadecimal bitmask of ports to use\n" + " -n NUM_NODES: number of node processes to use\n" + " -f NUM_FLOWS: number of flows to be added in the EFD table\n", + progname); +} + +/** + * The ports to be used by the application are passed in + * the form of a bitmask. This function parses the bitmask + * and places the port numbers to be used into the port[] + * array variable + */ +static int +parse_portmask(uint8_t max_ports, const char *portmask) +{ + char *end = NULL; + unsigned long pm; + uint8_t count = 0; + + if (portmask == NULL || *portmask == '\0') + return -1; + + /* convert parameter to a number and verify */ + pm = strtoul(portmask, &end, 16); + if (end == NULL || *end != '\0' || pm == 0) + return -1; + + /* loop through bits of the mask and mark ports */ + while (pm != 0) { + if (pm & 0x01) { /* bit is set in mask, use port */ + if (count >= max_ports) + printf("WARNING: requested port %u not present" + " - ignoring\n", (unsigned int)count); + else + info->id[info->num_ports++] = count; + } + pm = (pm >> 1); + count++; + } + + return 0; +} + +/** + * Take the number of nodes parameter passed to the app + * and convert to a number to store in the num_nodes variable + */ +static int +parse_num_nodes(const char *nodes) +{ + char *end = NULL; + unsigned long temp; + + if (nodes == NULL || *nodes == '\0') + return -1; + + temp = strtoul(nodes, &end, 10); + if (end == NULL || *end != '\0' || temp == 0) + return -1; + + num_nodes = (uint8_t)temp; + return 0; +} + +static int +parse_num_flows(const char *flows) +{ + char *end = NULL; + + /* parse hexadecimal string */ + num_flows = strtoul(flows, &end, 16); + if ((flows[0] == '\0') || (end == NULL) || (*end != '\0')) + return -1; + + if (num_flows == 0) + return -1; + + return 0; +} + +/** + * The application specific arguments follow the DPDK-specific + * arguments which are stripped by the DPDK init. This function + * processes these application arguments, printing usage info + * on error. + */ +int +parse_app_args(uint8_t max_ports, int argc, char *argv[]) +{ + int option_index, opt; + char **argvopt = argv; + static struct option lgopts[] = { /* no long options */ + {NULL, 0, 0, 0 } + }; + progname = argv[0]; + + while ((opt = getopt_long(argc, argvopt, "n:f:p:", lgopts, + &option_index)) != EOF) { + switch (opt) { + case 'p': + if (parse_portmask(max_ports, optarg) != 0) { + usage(); + return -1; + } + break; + case 'n': + if (parse_num_nodes(optarg) != 0) { + usage(); + return -1; + } + break; + case 'f': + if (parse_num_flows(optarg) != 0) { + usage(); + return -1; + } + break; + default: + printf("ERROR: Unknown option '%c'\n", opt); + usage(); + return -1; + } + } + + if (info->num_ports == 0 || num_nodes == 0) { + usage(); + return -1; + } + + if (info->num_ports % 2 != 0) { + printf("ERROR: application requires an even " + "number of ports to use\n"); + return -1; + } + return 0; +} diff --git a/dpdk/examples/server_node_efd/server/args.h b/dpdk/examples/server_node_efd/server/args.h new file mode 100644 index 00000000..cacf3957 --- /dev/null +++ b/dpdk/examples/server_node_efd/server/args.h @@ -0,0 +1,39 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _ARGS_H_ +#define _ARGS_H_ + +int parse_app_args(uint8_t max_ports, int argc, char *argv[]); + +#endif /* ifndef _ARGS_H_ */ diff --git a/dpdk/examples/server_node_efd/server/init.c b/dpdk/examples/server_node_efd/server/init.c new file mode 100644 index 00000000..0bcab8cc --- /dev/null +++ b/dpdk/examples/server_node_efd/server/init.c @@ -0,0 +1,377 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common.h" +#include "args.h" +#include "init.h" + +#define MBUFS_PER_NODE 1536 +#define MBUFS_PER_PORT 1536 +#define MBUF_CACHE_SIZE 512 + +#define RTE_MP_RX_DESC_DEFAULT 512 +#define RTE_MP_TX_DESC_DEFAULT 512 +#define NODE_QUEUE_RINGSIZE 128 + +#define NO_FLAGS 0 + +/* The mbuf pool for packet rx */ +struct rte_mempool *pktmbuf_pool; + +/* array of info/queues for nodes */ +struct node *nodes; + +/* EFD table */ +struct rte_efd_table *efd_table; + +/* Shared info between server and nodes */ +struct shared_info *info; + +/** + * Initialise the mbuf pool for packet reception for the NIC, and any other + * buffer pools needed by the app - currently none. + */ +static int +init_mbuf_pools(void) +{ + const unsigned int num_mbufs = (num_nodes * MBUFS_PER_NODE) + + (info->num_ports * MBUFS_PER_PORT); + + /* + * Don't pass single-producer/single-consumer flags to mbuf create as it + * seems faster to use a cache instead + */ + printf("Creating mbuf pool '%s' [%u mbufs] ...\n", + PKTMBUF_POOL_NAME, num_mbufs); + pktmbuf_pool = rte_pktmbuf_pool_create(PKTMBUF_POOL_NAME, num_mbufs, + MBUF_CACHE_SIZE, 0, RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id()); + + return pktmbuf_pool == NULL; /* 0 on success */ +} + +/** + * Initialise an individual port: + * - configure number of rx and tx rings + * - set up each rx ring, to pull from the main mbuf pool + * - set up each tx ring + * - start the port and report its status to stdout + */ +static int +init_port(uint16_t port_num) +{ + /* for port configuration all features are off by default */ + const struct rte_eth_conf port_conf = { + .rxmode = { + .mq_mode = ETH_MQ_RX_RSS + } + }; + const uint16_t rx_rings = 1, tx_rings = num_nodes; + uint16_t rx_ring_size = RTE_MP_RX_DESC_DEFAULT; + uint16_t tx_ring_size = RTE_MP_TX_DESC_DEFAULT; + + uint16_t q; + int retval; + + printf("Port %u init ... ", port_num); + fflush(stdout); + + /* + * Standard DPDK port initialisation - config port, then set up + * rx and tx rings. + */ + retval = rte_eth_dev_configure(port_num, rx_rings, tx_rings, &port_conf); + if (retval != 0) + return retval; + + retval = rte_eth_dev_adjust_nb_rx_tx_desc(port_num, &rx_ring_size, + &tx_ring_size); + if (retval != 0) + return retval; + + for (q = 0; q < rx_rings; q++) { + retval = rte_eth_rx_queue_setup(port_num, q, rx_ring_size, + rte_eth_dev_socket_id(port_num), + NULL, pktmbuf_pool); + if (retval < 0) + return retval; + } + + for (q = 0; q < tx_rings; q++) { + retval = rte_eth_tx_queue_setup(port_num, q, tx_ring_size, + rte_eth_dev_socket_id(port_num), + NULL); + if (retval < 0) + return retval; + } + + rte_eth_promiscuous_enable(port_num); + + retval = rte_eth_dev_start(port_num); + if (retval < 0) + return retval; + + printf("done:\n"); + + return 0; +} + +/** + * Set up the DPDK rings which will be used to pass packets, via + * pointers, between the multi-process server and node processes. + * Each node needs one RX queue. + */ +static int +init_shm_rings(void) +{ + unsigned int i; + unsigned int socket_id; + const char *q_name; + const unsigned int ringsize = NODE_QUEUE_RINGSIZE; + + nodes = rte_malloc("node details", + sizeof(*nodes) * num_nodes, 0); + if (nodes == NULL) + rte_exit(EXIT_FAILURE, "Cannot allocate memory for " + "node program details\n"); + + for (i = 0; i < num_nodes; i++) { + /* Create an RX queue for each node */ + socket_id = rte_socket_id(); + q_name = get_rx_queue_name(i); + nodes[i].rx_q = rte_ring_create(q_name, + ringsize, socket_id, + RING_F_SP_ENQ | RING_F_SC_DEQ); + if (nodes[i].rx_q == NULL) + rte_exit(EXIT_FAILURE, "Cannot create rx ring queue " + "for node %u\n", i); + } + return 0; +} + +/* + * Create EFD table which will contain all the flows + * that will be distributed among the nodes + */ +static void +create_efd_table(void) +{ + uint8_t socket_id = rte_socket_id(); + + /* create table */ + efd_table = rte_efd_create("flow table", num_flows * 2, sizeof(uint32_t), + 1 << socket_id, socket_id); + + if (efd_table == NULL) + rte_exit(EXIT_FAILURE, "Problem creating the flow table\n"); +} + +static void +populate_efd_table(void) +{ + unsigned int i; + int32_t ret; + uint32_t ip_dst; + uint8_t socket_id = rte_socket_id(); + uint64_t node_id; + + /* Add flows in table */ + for (i = 0; i < num_flows; i++) { + node_id = i % num_nodes; + + ip_dst = rte_cpu_to_be_32(i); + ret = rte_efd_update(efd_table, socket_id, + (void *)&ip_dst, (efd_value_t)node_id); + if (ret < 0) + rte_exit(EXIT_FAILURE, "Unable to add entry %u in " + "EFD table\n", i); + } + + printf("EFD table: Adding 0x%x keys\n", num_flows); +} + +/* Check the link status of all ports in up to 9s, and print them finally */ +static void +check_all_ports_link_status(uint16_t port_num, uint32_t port_mask) +{ +#define CHECK_INTERVAL 100 /* 100ms */ +#define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ + uint8_t count, all_ports_up, print_flag = 0; + uint16_t portid; + struct rte_eth_link link; + + printf("\nChecking link status"); + fflush(stdout); + for (count = 0; count <= MAX_CHECK_TIME; count++) { + all_ports_up = 1; + for (portid = 0; portid < port_num; portid++) { + if ((port_mask & (1 << info->id[portid])) == 0) + continue; + memset(&link, 0, sizeof(link)); + rte_eth_link_get_nowait(info->id[portid], &link); + /* print link status if flag set */ + if (print_flag == 1) { + if (link.link_status) + printf( + "Port%d Link Up. Speed %u Mbps - %s\n", + info->id[portid], + link.link_speed, + (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? + ("full-duplex") : ("half-duplex\n")); + else + printf("Port %d Link Down\n", + info->id[portid]); + continue; + } + /* clear all_ports_up flag if any link down */ + if (link.link_status == ETH_LINK_DOWN) { + all_ports_up = 0; + break; + } + } + /* after finally printing all link status, get out */ + if (print_flag == 1) + break; + + if (all_ports_up == 0) { + printf("."); + fflush(stdout); + rte_delay_ms(CHECK_INTERVAL); + } + + /* set the print_flag if all ports up or timeout */ + if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { + print_flag = 1; + printf("done\n"); + } + } +} + +/** + * Main init function for the multi-process server app, + * calls subfunctions to do each stage of the initialisation. + */ +int +init(int argc, char *argv[]) +{ + int retval; + const struct rte_memzone *mz; + uint8_t i, total_ports; + + /* init EAL, parsing EAL args */ + retval = rte_eal_init(argc, argv); + if (retval < 0) + return -1; + argc -= retval; + argv += retval; + + /* get total number of ports */ + total_ports = rte_eth_dev_count(); + + /* set up array for port data */ + mz = rte_memzone_reserve(MZ_SHARED_INFO, sizeof(*info), + rte_socket_id(), NO_FLAGS); + if (mz == NULL) + rte_exit(EXIT_FAILURE, "Cannot reserve memory zone " + "for port information\n"); + memset(mz->addr, 0, sizeof(*info)); + info = mz->addr; + + /* parse additional, application arguments */ + retval = parse_app_args(total_ports, argc, argv); + if (retval != 0) + return -1; + + /* initialise mbuf pools */ + retval = init_mbuf_pools(); + if (retval != 0) + rte_exit(EXIT_FAILURE, "Cannot create needed mbuf pools\n"); + + /* now initialise the ports we will use */ + for (i = 0; i < info->num_ports; i++) { + retval = init_port(info->id[i]); + if (retval != 0) + rte_exit(EXIT_FAILURE, "Cannot initialise port %u\n", + (unsigned int) i); + } + + check_all_ports_link_status(info->num_ports, (~0x0)); + + /* initialise the node queues/rings for inter-eu comms */ + init_shm_rings(); + + /* Create the EFD table */ + create_efd_table(); + + /* Populate the EFD table */ + populate_efd_table(); + + /* Share the total number of nodes */ + info->num_nodes = num_nodes; + + /* Share the total number of flows */ + info->num_flows = num_flows; + return 0; +} diff --git a/dpdk/examples/server_node_efd/server/init.h b/dpdk/examples/server_node_efd/server/init.h new file mode 100644 index 00000000..8dc5885b --- /dev/null +++ b/dpdk/examples/server_node_efd/server/init.h @@ -0,0 +1,76 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _INIT_H_ +#define _INIT_H_ + +/* + * #include + * #include "args.h" + */ + +/* + * Define a node structure with all needed info, including + * stats from the nodes. + */ +struct node { + struct rte_ring *rx_q; + unsigned int node_id; + /* these stats hold how many packets the node will actually receive, + * and how many packets were dropped because the node's queue was full. + * The port-info stats, in contrast, record how many packets were received + * or transmitted on an actual NIC port. + */ + struct { + uint64_t rx; + uint64_t rx_drop; + } stats; +}; + +extern struct rte_efd_table *efd_table; +extern struct node *nodes; + +/* + * shared information between server and nodes: number of nodes, + * port numbers, rx and tx stats etc. + */ +extern struct shared_info *info; + +extern struct rte_mempool *pktmbuf_pool; +extern uint8_t num_nodes; +extern unsigned int num_sockets; +extern uint32_t num_flows; + +int init(int argc, char *argv[]); + +#endif /* ifndef _INIT_H_ */ diff --git a/dpdk/examples/server_node_efd/server/main.c b/dpdk/examples/server_node_efd/server/main.c new file mode 100644 index 00000000..aa1c6f57 --- /dev/null +++ b/dpdk/examples/server_node_efd/server/main.c @@ -0,0 +1,358 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common.h" +#include "args.h" +#include "init.h" + +/* + * When doing reads from the NIC or the node queues, + * use this batch size + */ +#define PACKET_READ_SIZE 32 + +/* + * Local buffers to put packets in, used to send packets in bursts to the + * nodes + */ +struct node_rx_buf { + struct rte_mbuf *buffer[PACKET_READ_SIZE]; + uint16_t count; +}; + +struct efd_stats { + uint64_t distributed; + uint64_t drop; +} flow_dist_stats; + +/* One buffer per node rx queue - dynamically allocate array */ +static struct node_rx_buf *cl_rx_buf; + +static const char * +get_printable_mac_addr(uint16_t port) +{ + static const char err_address[] = "00:00:00:00:00:00"; + static char addresses[RTE_MAX_ETHPORTS][sizeof(err_address)]; + struct ether_addr mac; + + if (unlikely(port >= RTE_MAX_ETHPORTS)) + return err_address; + if (unlikely(addresses[port][0] == '\0')) { + rte_eth_macaddr_get(port, &mac); + snprintf(addresses[port], sizeof(addresses[port]), + "%02x:%02x:%02x:%02x:%02x:%02x\n", + mac.addr_bytes[0], mac.addr_bytes[1], + mac.addr_bytes[2], mac.addr_bytes[3], + mac.addr_bytes[4], mac.addr_bytes[5]); + } + return addresses[port]; +} + +/* + * This function displays the recorded statistics for each port + * and for each node. It uses ANSI terminal codes to clear + * screen when called. It is called from a single non-master + * thread in the server process, when the process is run with more + * than one lcore enabled. + */ +static void +do_stats_display(void) +{ + unsigned int i, j; + const char clr[] = {27, '[', '2', 'J', '\0'}; + const char topLeft[] = {27, '[', '1', ';', '1', 'H', '\0'}; + uint64_t port_tx[RTE_MAX_ETHPORTS], port_tx_drop[RTE_MAX_ETHPORTS]; + uint64_t node_tx[MAX_NODES], node_tx_drop[MAX_NODES]; + + /* to get TX stats, we need to do some summing calculations */ + memset(port_tx, 0, sizeof(port_tx)); + memset(port_tx_drop, 0, sizeof(port_tx_drop)); + memset(node_tx, 0, sizeof(node_tx)); + memset(node_tx_drop, 0, sizeof(node_tx_drop)); + + for (i = 0; i < num_nodes; i++) { + const struct tx_stats *tx = &info->tx_stats[i]; + + for (j = 0; j < info->num_ports; j++) { + const uint64_t tx_val = tx->tx[info->id[j]]; + const uint64_t drop_val = tx->tx_drop[info->id[j]]; + + port_tx[j] += tx_val; + port_tx_drop[j] += drop_val; + node_tx[i] += tx_val; + node_tx_drop[i] += drop_val; + } + } + + /* Clear screen and move to top left */ + printf("%s%s", clr, topLeft); + + printf("PORTS\n"); + printf("-----\n"); + for (i = 0; i < info->num_ports; i++) + printf("Port %u: '%s'\t", (unsigned int)info->id[i], + get_printable_mac_addr(info->id[i])); + printf("\n\n"); + for (i = 0; i < info->num_ports; i++) { + printf("Port %u - rx: %9"PRIu64"\t" + "tx: %9"PRIu64"\n", + (unsigned int)info->id[i], info->rx_stats.rx[i], + port_tx[i]); + } + + printf("\nSERVER\n"); + printf("-----\n"); + printf("distributed: %9"PRIu64", drop: %9"PRIu64"\n", + flow_dist_stats.distributed, flow_dist_stats.drop); + + printf("\nNODES\n"); + printf("-------\n"); + for (i = 0; i < num_nodes; i++) { + const unsigned long long rx = nodes[i].stats.rx; + const unsigned long long rx_drop = nodes[i].stats.rx_drop; + const struct filter_stats *filter = &info->filter_stats[i]; + + printf("Node %2u - rx: %9llu, rx_drop: %9llu\n" + " tx: %9"PRIu64", tx_drop: %9"PRIu64"\n" + " filter_passed: %9"PRIu64", " + "filter_drop: %9"PRIu64"\n", + i, rx, rx_drop, node_tx[i], node_tx_drop[i], + filter->passed, filter->drop); + } + + printf("\n"); +} + +/* + * The function called from each non-master lcore used by the process. + * The test_and_set function is used to randomly pick a single lcore on which + * the code to display the statistics will run. Otherwise, the code just + * repeatedly sleeps. + */ +static int +sleep_lcore(__attribute__((unused)) void *dummy) +{ + /* Used to pick a display thread - static, so zero-initialised */ + static rte_atomic32_t display_stats; + + /* Only one core should display stats */ + if (rte_atomic32_test_and_set(&display_stats)) { + const unsigned int sleeptime = 1; + + printf("Core %u displaying statistics\n", rte_lcore_id()); + + /* Longer initial pause so above printf is seen */ + sleep(sleeptime * 3); + + /* Loop forever: sleep always returns 0 or <= param */ + while (sleep(sleeptime) <= sleeptime) + do_stats_display(); + } + return 0; +} + +/* + * Function to set all the node statistic values to zero. + * Called at program startup. + */ +static void +clear_stats(void) +{ + unsigned int i; + + for (i = 0; i < num_nodes; i++) + nodes[i].stats.rx = nodes[i].stats.rx_drop = 0; +} + +/* + * send a burst of traffic to a node, assuming there are packets + * available to be sent to this node + */ +static void +flush_rx_queue(uint16_t node) +{ + uint16_t j; + struct node *cl; + + if (cl_rx_buf[node].count == 0) + return; + + cl = &nodes[node]; + if (rte_ring_enqueue_bulk(cl->rx_q, (void **)cl_rx_buf[node].buffer, + cl_rx_buf[node].count, NULL) != cl_rx_buf[node].count){ + for (j = 0; j < cl_rx_buf[node].count; j++) + rte_pktmbuf_free(cl_rx_buf[node].buffer[j]); + cl->stats.rx_drop += cl_rx_buf[node].count; + } else + cl->stats.rx += cl_rx_buf[node].count; + + cl_rx_buf[node].count = 0; +} + +/* + * marks a packet down to be sent to a particular node process + */ +static inline void +enqueue_rx_packet(uint8_t node, struct rte_mbuf *buf) +{ + cl_rx_buf[node].buffer[cl_rx_buf[node].count++] = buf; +} + +/* + * This function takes a group of packets and routes them + * individually to the node process. Very simply round-robins the packets + * without checking any of the packet contents. + */ +static void +process_packets(uint32_t port_num __rte_unused, struct rte_mbuf *pkts[], + uint16_t rx_count, unsigned int socket_id) +{ + uint16_t i; + uint8_t node; + efd_value_t data[RTE_EFD_BURST_MAX]; + const void *key_ptrs[RTE_EFD_BURST_MAX]; + + struct ipv4_hdr *ipv4_hdr; + uint32_t ipv4_dst_ip[RTE_EFD_BURST_MAX]; + + for (i = 0; i < rx_count; i++) { + /* Handle IPv4 header.*/ + ipv4_hdr = rte_pktmbuf_mtod_offset(pkts[i], struct ipv4_hdr *, + sizeof(struct ether_hdr)); + ipv4_dst_ip[i] = ipv4_hdr->dst_addr; + key_ptrs[i] = (void *)&ipv4_dst_ip[i]; + } + + rte_efd_lookup_bulk(efd_table, socket_id, rx_count, + (const void **) key_ptrs, data); + for (i = 0; i < rx_count; i++) { + node = (uint8_t) ((uintptr_t)data[i]); + + if (node >= num_nodes) { + /* + * Node is out of range, which means that + * flow has not been inserted + */ + flow_dist_stats.drop++; + rte_pktmbuf_free(pkts[i]); + } else { + flow_dist_stats.distributed++; + enqueue_rx_packet(node, pkts[i]); + } + } + + for (i = 0; i < num_nodes; i++) + flush_rx_queue(i); +} + +/* + * Function called by the master lcore of the DPDK process. + */ +static void +do_packet_forwarding(void) +{ + unsigned int port_num = 0; /* indexes the port[] array */ + unsigned int socket_id = rte_socket_id(); + + for (;;) { + struct rte_mbuf *buf[PACKET_READ_SIZE]; + uint16_t rx_count; + + /* read a port */ + rx_count = rte_eth_rx_burst(info->id[port_num], 0, + buf, PACKET_READ_SIZE); + info->rx_stats.rx[port_num] += rx_count; + + /* Now process the NIC packets read */ + if (likely(rx_count > 0)) + process_packets(port_num, buf, rx_count, socket_id); + + /* move to next port */ + if (++port_num == info->num_ports) + port_num = 0; + } +} + +int +main(int argc, char *argv[]) +{ + /* initialise the system */ + if (init(argc, argv) < 0) + return -1; + RTE_LOG(INFO, APP, "Finished Process Init.\n"); + + cl_rx_buf = calloc(num_nodes, sizeof(cl_rx_buf[0])); + + /* clear statistics */ + clear_stats(); + + /* put all other cores to sleep bar master */ + rte_eal_mp_remote_launch(sleep_lcore, NULL, SKIP_MASTER); + + do_packet_forwarding(); + return 0; +} diff --git a/dpdk/examples/server_node_efd/shared/common.h b/dpdk/examples/server_node_efd/shared/common.h new file mode 100644 index 00000000..b1e0abe5 --- /dev/null +++ b/dpdk/examples/server_node_efd/shared/common.h @@ -0,0 +1,99 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _COMMON_H_ +#define _COMMON_H_ + +#include +#include + +#define MAX_NODES 16 +/* + * Shared port info, including statistics information for display by server. + * Structure will be put in a memzone. + * - All port id values share one cache line as this data will be read-only + * during operation. + * - All rx statistic values share cache lines, as this data is written only + * by the server process. (rare reads by stats display) + * - The tx statistics have values for all ports per cache line, but the stats + * themselves are written by the nodes, so we have a distinct set, on different + * cache lines for each node to use. + */ +struct rx_stats { + uint64_t rx[RTE_MAX_ETHPORTS]; +} __rte_cache_aligned; + +struct tx_stats { + uint64_t tx[RTE_MAX_ETHPORTS]; + uint64_t tx_drop[RTE_MAX_ETHPORTS]; +} __rte_cache_aligned; + +struct filter_stats { + uint64_t drop; + uint64_t passed; +} __rte_cache_aligned; + +struct shared_info { + uint8_t num_nodes; + uint16_t num_ports; + uint32_t num_flows; + uint16_t id[RTE_MAX_ETHPORTS]; + struct rx_stats rx_stats; + struct tx_stats tx_stats[MAX_NODES]; + struct filter_stats filter_stats[MAX_NODES]; +}; + +/* define common names for structures shared between server and node */ +#define MP_NODE_RXQ_NAME "MProc_Node_%u_RX" +#define PKTMBUF_POOL_NAME "MProc_pktmbuf_pool" +#define MZ_SHARED_INFO "MProc_shared_info" + +/* + * Given the rx queue name template above, get the queue name + */ +static inline const char * +get_rx_queue_name(unsigned int id) +{ + /* + * Buffer for return value. Size calculated by %u being replaced + * by maximum 3 digits (plus an extra byte for safety) + */ + static char buffer[sizeof(MP_NODE_RXQ_NAME) + 2]; + + snprintf(buffer, sizeof(buffer) - 1, MP_NODE_RXQ_NAME, id); + return buffer; +} + +#define RTE_LOGTYPE_APP RTE_LOGTYPE_USER1 + +#endif diff --git a/dpdk/examples/service_cores/Makefile b/dpdk/examples/service_cores/Makefile new file mode 100644 index 00000000..bd4a345d --- /dev/null +++ b/dpdk/examples/service_cores/Makefile @@ -0,0 +1,54 @@ +# BSD LICENSE +# +# Copyright(c) 2017 Intel Corporation. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +ifeq ($(RTE_SDK),) +$(error "Please define RTE_SDK environment variable") +endif + +# Default target, can be overridden by command line or environment +RTE_TARGET ?= x86_64-native-linuxapp-gcc + +include $(RTE_SDK)/mk/rte.vars.mk + +# binary name +APP = service_cores + +# all source are stored in SRCS-y +SRCS-y := main.c + +CFLAGS += $(WERROR_FLAGS) + +# workaround for a gcc bug with noreturn attribute +# http://gcc.gnu.org/bugzilla/show_bug.cgi?id=12603 +ifeq ($(CONFIG_RTE_TOOLCHAIN_GCC),y) +CFLAGS_main.o += -Wno-return-type +endif + +include $(RTE_SDK)/mk/rte.extapp.mk diff --git a/dpdk/examples/service_cores/main.c b/dpdk/examples/service_cores/main.c new file mode 100644 index 00000000..b617a789 --- /dev/null +++ b/dpdk/examples/service_cores/main.c @@ -0,0 +1,246 @@ +/* + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +/* allow application scheduling of the services */ +#include + +/* Allow application registration of its own services. An application does not + * have to register services, but it can be useful if it wishes to run a + * function on a core that is otherwise in use as a service core. In this + * example, all services are dummy services registered by the sample app itself. + */ +#include + +#define PROFILE_CORES_MAX 8 +#define PROFILE_SERVICE_PER_CORE 5 + +/* dummy function to do "work" */ +static int32_t service_func(void *args) +{ + RTE_SET_USED(args); + rte_delay_us(2000); + return 0; +} + +static struct rte_service_spec services[] = { + {"service_1", service_func, NULL, 0, 0}, + {"service_2", service_func, NULL, 0, 0}, + {"service_3", service_func, NULL, 0, 0}, + {"service_4", service_func, NULL, 0, 0}, + {"service_5", service_func, NULL, 0, 0}, +}; +#define NUM_SERVICES RTE_DIM(services) + +/* this struct holds the mapping of a particular core to all services */ +struct profile_for_core { + uint32_t mapped_services[PROFILE_SERVICE_PER_CORE]; +}; + +/* struct that can be applied as the service core mapping. Items in this + * struct will be passed to the ordinary rte_service_* APIs to configure the + * service cores at runtime, based on the requirements. + * + * These profiles can be considered a "configuration" for the service cores, + * where switching profile just changes the number of cores and the mappings + * for each of them. As a result, the core requirements and performance of the + * application scales. + */ +struct profile { + char name[64]; + uint32_t num_cores; + struct profile_for_core cores[PROFILE_CORES_MAX]; +}; + +static struct profile profiles[] = { + /* profile 0: high performance */ + { + .name = "High Performance", + .num_cores = 5, + .cores[0] = {.mapped_services = {1, 0, 0, 0, 0} }, + .cores[1] = {.mapped_services = {0, 1, 0, 0, 0} }, + .cores[2] = {.mapped_services = {0, 0, 1, 0, 0} }, + .cores[3] = {.mapped_services = {0, 0, 0, 1, 0} }, + .cores[4] = {.mapped_services = {0, 0, 0, 0, 1} }, + }, + /* profile 1: mid performance with single service priority */ + { + .name = "Mid-High Performance", + .num_cores = 3, + .cores[0] = {.mapped_services = {1, 1, 0, 0, 0} }, + .cores[1] = {.mapped_services = {0, 0, 1, 1, 0} }, + .cores[2] = {.mapped_services = {0, 0, 0, 0, 1} }, + .cores[3] = {.mapped_services = {0, 0, 0, 0, 0} }, + .cores[4] = {.mapped_services = {0, 0, 0, 0, 0} }, + }, + /* profile 2: mid performance with single service priority */ + { + .name = "Mid-Low Performance", + .num_cores = 2, + .cores[0] = {.mapped_services = {1, 1, 1, 0, 0} }, + .cores[1] = {.mapped_services = {1, 1, 0, 1, 1} }, + .cores[2] = {.mapped_services = {0, 0, 0, 0, 0} }, + .cores[3] = {.mapped_services = {0, 0, 0, 0, 0} }, + .cores[4] = {.mapped_services = {0, 0, 0, 0, 0} }, + }, + /* profile 3: scale down performance on single core */ + { + .name = "Scale down performance", + .num_cores = 1, + .cores[0] = {.mapped_services = {1, 1, 1, 1, 1} }, + .cores[1] = {.mapped_services = {0, 0, 0, 0, 0} }, + .cores[2] = {.mapped_services = {0, 0, 0, 0, 0} }, + .cores[3] = {.mapped_services = {0, 0, 0, 0, 0} }, + .cores[4] = {.mapped_services = {0, 0, 0, 0, 0} }, + }, +}; +#define NUM_PROFILES RTE_DIM(profiles) + +static void +apply_profile(int profile_id) +{ + uint32_t i; + uint32_t s; + int ret; + struct profile *p = &profiles[profile_id]; + const uint8_t core_off = 1; + + for (i = 0; i < p->num_cores; i++) { + uint32_t core = i + core_off; + ret = rte_service_lcore_add(core); + if (ret && ret != -EALREADY) + printf("core %d added ret %d\n", core, ret); + + ret = rte_service_lcore_start(core); + if (ret && ret != -EALREADY) + printf("core %d start ret %d\n", core, ret); + + for (s = 0; s < NUM_SERVICES; s++) { + if (rte_service_map_lcore_set(s, core, + p->cores[i].mapped_services[s])) + printf("failed to map lcore %d\n", core); + } + } + + for ( ; i < PROFILE_CORES_MAX; i++) { + uint32_t core = i + core_off; + for (s = 0; s < NUM_SERVICES; s++) { + ret = rte_service_map_lcore_set(s, core, 0); + if (ret && ret != -EINVAL) { + printf("%s %d: map lcore set = %d\n", __func__, + __LINE__, ret); + } + } + ret = rte_service_lcore_stop(core); + if (ret && ret != -EALREADY) { + printf("%s %d: lcore stop = %d\n", __func__, + __LINE__, ret); + } + ret = rte_service_lcore_del(core); + if (ret && ret != -EINVAL) { + printf("%s %d: lcore del = %d\n", __func__, + __LINE__, ret); + } + } +} + +int +main(int argc, char **argv) +{ + int ret; + + ret = rte_eal_init(argc, argv); + if (ret < 0) + rte_panic("Cannot init EAL\n"); + + uint32_t i; + for (i = 0; i < NUM_SERVICES; i++) { + services[i].callback_userdata = 0; + uint32_t id; + ret = rte_service_component_register(&services[i], &id); + if (ret) + rte_exit(-1, "service register() failed"); + + /* set the service itself to be ready to run. In the case of + * ethdev, eventdev etc PMDs, this will be set when the + * appropriate configure or setup function is called. + */ + rte_service_component_runstate_set(id, 1); + + /* Collect statistics for the service */ + rte_service_set_stats_enable(id, 1); + + /* the application sets the service to be active. Note that the + * previous component_runstate_set() is the PMD indicating + * ready, while this function is the application setting the + * service to run. Applications can choose to not run a service + * by setting runstate to 0 at any time. + */ + ret = rte_service_runstate_set(id, 1); + if (ret) + return -ENOEXEC; + } + + i = 0; + while (1) { + const char clr[] = { 27, '[', '2', 'J', '\0' }; + const char topLeft[] = { 27, '[', '1', ';', '1', 'H', '\0' }; + printf("%s%s", clr, topLeft); + + apply_profile(i); + printf("\n==> Profile: %s\n\n", profiles[i].name); + + sleep(1); + rte_service_dump(stdout, UINT32_MAX); + + sleep(5); + rte_service_dump(stdout, UINT32_MAX); + + i++; + if (i >= NUM_PROFILES) + i = 0; + } + + return 0; +} diff --git a/dpdk/examples/skeleton/basicfwd.c b/dpdk/examples/skeleton/basicfwd.c index c89822cb..e623754c 100644 --- a/dpdk/examples/skeleton/basicfwd.c +++ b/dpdk/examples/skeleton/basicfwd.c @@ -57,10 +57,12 @@ static const struct rte_eth_conf port_conf_default = { * coming from the mbuf_pool passed as a parameter. */ static inline int -port_init(uint8_t port, struct rte_mempool *mbuf_pool) +port_init(uint16_t port, struct rte_mempool *mbuf_pool) { struct rte_eth_conf port_conf = port_conf_default; const uint16_t rx_rings = 1, tx_rings = 1; + uint16_t nb_rxd = RX_RING_SIZE; + uint16_t nb_txd = TX_RING_SIZE; int retval; uint16_t q; @@ -72,9 +74,13 @@ port_init(uint8_t port, struct rte_mempool *mbuf_pool) if (retval != 0) return retval; + retval = rte_eth_dev_adjust_nb_rx_tx_desc(port, &nb_rxd, &nb_txd); + if (retval != 0) + return retval; + /* Allocate and set up 1 RX queue per Ethernet port. */ for (q = 0; q < rx_rings; q++) { - retval = rte_eth_rx_queue_setup(port, q, RX_RING_SIZE, + retval = rte_eth_rx_queue_setup(port, q, nb_rxd, rte_eth_dev_socket_id(port), NULL, mbuf_pool); if (retval < 0) return retval; @@ -82,7 +88,7 @@ port_init(uint8_t port, struct rte_mempool *mbuf_pool) /* Allocate and set up 1 TX queue per Ethernet port. */ for (q = 0; q < tx_rings; q++) { - retval = rte_eth_tx_queue_setup(port, q, TX_RING_SIZE, + retval = rte_eth_tx_queue_setup(port, q, nb_txd, rte_eth_dev_socket_id(port), NULL); if (retval < 0) return retval; @@ -98,7 +104,7 @@ port_init(uint8_t port, struct rte_mempool *mbuf_pool) rte_eth_macaddr_get(port, &addr); printf("Port %u MAC: %02" PRIx8 " %02" PRIx8 " %02" PRIx8 " %02" PRIx8 " %02" PRIx8 " %02" PRIx8 "\n", - (unsigned)port, + port, addr.addr_bytes[0], addr.addr_bytes[1], addr.addr_bytes[2], addr.addr_bytes[3], addr.addr_bytes[4], addr.addr_bytes[5]); @@ -116,8 +122,8 @@ port_init(uint8_t port, struct rte_mempool *mbuf_pool) static __attribute__((noreturn)) void lcore_main(void) { - const uint8_t nb_ports = rte_eth_dev_count(); - uint8_t port; + const uint16_t nb_ports = rte_eth_dev_count(); + uint16_t port; /* * Check that the port is on the same NUMA node as the polling thread @@ -173,7 +179,7 @@ main(int argc, char *argv[]) { struct rte_mempool *mbuf_pool; unsigned nb_ports; - uint8_t portid; + uint16_t portid; /* Initialize the Environment Abstraction Layer (EAL). */ int ret = rte_eal_init(argc, argv); @@ -198,7 +204,7 @@ main(int argc, char *argv[]) /* Initialize all ports. */ for (portid = 0; portid < nb_ports; portid++) if (port_init(portid, mbuf_pool) != 0) - rte_exit(EXIT_FAILURE, "Cannot init port %"PRIu8 "\n", + rte_exit(EXIT_FAILURE, "Cannot init port %"PRIu16 "\n", portid); if (rte_lcore_count() > 1) diff --git a/dpdk/examples/tep_termination/main.c b/dpdk/examples/tep_termination/main.c index 622f248a..a14d2e37 100644 --- a/dpdk/examples/tep_termination/main.c +++ b/dpdk/examples/tep_termination/main.c @@ -49,7 +49,8 @@ #include #include #include -#include +#include +#include #include "main.h" #include "vxlan.h" @@ -68,7 +69,7 @@ (nb_switching_cores * MBUF_CACHE_SIZE)) #define MBUF_CACHE_SIZE 128 -#define MBUF_SIZE (2048 + sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM) +#define MBUF_DATA_SIZE RTE_MBUF_DEFAULT_BUF_SIZE #define MAX_PKT_BURST 32 /* Max burst size for RX/TX */ #define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */ @@ -97,7 +98,7 @@ #define MBUF_HEADROOM_UINT32(mbuf) (*(uint32_t *)((uint8_t *)(mbuf) \ + sizeof(struct rte_mbuf))) -#define INVALID_PORT_ID 0xFF +#define INVALID_PORT_ID 0xFFFF /* Size of buffers used for snprintfs. */ #define MAX_PRINT_BUFF 6072 @@ -183,7 +184,7 @@ static uint32_t burst_rx_retry_num = BURST_RX_RETRIES; static char dev_basename[MAX_BASENAME_SZ] = "vhost-net"; static unsigned lcore_ids[RTE_MAX_LCORE]; -uint8_t ports[RTE_MAX_ETHPORTS]; +uint16_t ports[RTE_MAX_ETHPORTS]; static unsigned nb_ports; /**< The number of ports specified in command line */ @@ -559,7 +560,7 @@ check_ports_num(unsigned max_nb_ports) * This function routes the TX packet to the correct interface. This may be a local device * or the physical port. */ -static inline void __attribute__((always_inline)) +static __rte_always_inline void virtio_tx_route(struct vhost_dev *vdev, struct rte_mbuf *m) { struct mbuf_table *tx_q; @@ -567,7 +568,7 @@ virtio_tx_route(struct vhost_dev *vdev, struct rte_mbuf *m) unsigned len, ret = 0; const uint16_t lcore_id = rte_lcore_id(); - RTE_LOG(DEBUG, VHOST_DATA, "(%d) TX: MAC address is external\n", + RTE_LOG_DP(DEBUG, VHOST_DATA, "(%d) TX: MAC address is external\n", vdev->vid); /* Add packet to the port tx queue */ @@ -649,7 +650,7 @@ switch_worker(__rte_unused void *arg) if (unlikely(diff_tsc > drain_tsc)) { if (tx_q->len) { - RTE_LOG(DEBUG, VHOST_DATA, "TX queue drained after " + RTE_LOG_DP(DEBUG, VHOST_DATA, "TX queue drained after " "timeout with burst size %u\n", tx_q->len); ret = overlay_options.tx_handle(ports[0], @@ -1081,7 +1082,7 @@ new_device(int vid) * These callback allow devices to be added to the data core when configuration * has been fully complete. */ -static const struct virtio_net_device_ops virtio_net_device_ops = { +static const struct vhost_device_ops virtio_net_device_ops = { .new_device = new_device, .destroy_device = destroy_device, }; @@ -1151,8 +1152,7 @@ print_stats(void) } /** - * Main function, does initialisation and calls the per-lcore functions. The CUSE - * device is also registered here to handle the IOCTLs. + * Main function, does initialisation and calls the per-lcore functions. */ int main(int argc, char *argv[]) @@ -1161,7 +1161,7 @@ main(int argc, char *argv[]) unsigned lcore_id, core_id = 0; unsigned nb_ports, valid_nb_ports; int ret; - uint8_t portid; + uint16_t portid; uint16_t queue_id; static pthread_t tid; char thread_name[RTE_MAX_THREAD_NAME_LEN]; @@ -1200,15 +1200,13 @@ main(int argc, char *argv[]) MAX_SUP_PORTS); } /* Create the mbuf pool. */ - mbuf_pool = rte_mempool_create( + mbuf_pool = rte_pktmbuf_pool_create( "MBUF_POOL", - NUM_MBUFS_PER_PORT - * valid_nb_ports, - MBUF_SIZE, MBUF_CACHE_SIZE, - sizeof(struct rte_pktmbuf_pool_private), - rte_pktmbuf_pool_init, NULL, - rte_pktmbuf_init, NULL, - rte_socket_id(), 0); + NUM_MBUFS_PER_PORT * valid_nb_ports, + MBUF_CACHE_SIZE, + 0, + MBUF_DATA_SIZE, + rte_socket_id()); if (mbuf_pool == NULL) rte_exit(EXIT_FAILURE, "Cannot create mbuf pool\n"); @@ -1251,17 +1249,28 @@ main(int argc, char *argv[]) rte_eal_remote_launch(switch_worker, mbuf_pool, lcore_id); } - rte_vhost_feature_disable(1ULL << VIRTIO_NET_F_MRG_RXBUF); - /* Register CUSE device to handle IOCTLs. */ ret = rte_vhost_driver_register((char *)&dev_basename, 0); if (ret != 0) - rte_exit(EXIT_FAILURE, "CUSE device setup failure.\n"); + rte_exit(EXIT_FAILURE, "failed to register vhost driver.\n"); - rte_vhost_driver_callback_register(&virtio_net_device_ops); + rte_vhost_driver_disable_features(dev_basename, + 1ULL << VIRTIO_NET_F_MRG_RXBUF); - /* Start CUSE session. */ - rte_vhost_driver_session_start(); + ret = rte_vhost_driver_callback_register(dev_basename, + &virtio_net_device_ops); + if (ret != 0) { + rte_exit(EXIT_FAILURE, + "failed to register vhost driver callbacks.\n"); + } + + if (rte_vhost_driver_start(dev_basename) < 0) { + rte_exit(EXIT_FAILURE, + "failed to start vhost driver.\n"); + } + + RTE_LCORE_FOREACH_SLAVE(lcore_id) + rte_eal_wait_lcore(lcore_id); return 0; } diff --git a/dpdk/examples/tep_termination/main.h b/dpdk/examples/tep_termination/main.h index c0ea7667..8ed817d4 100644 --- a/dpdk/examples/tep_termination/main.h +++ b/dpdk/examples/tep_termination/main.h @@ -54,6 +54,8 @@ /* Max number of devices. Limited by the application. */ #define MAX_DEVICES 64 +enum {VIRTIO_RXQ, VIRTIO_TXQ, VIRTIO_QNUM}; + /* Per-device statistics struct */ struct device_statistics { uint64_t tx_total; diff --git a/dpdk/examples/tep_termination/vxlan.c b/dpdk/examples/tep_termination/vxlan.c index 9142c8d9..9f0e8362 100644 --- a/dpdk/examples/tep_termination/vxlan.c +++ b/dpdk/examples/tep_termination/vxlan.c @@ -141,14 +141,17 @@ process_inner_cksums(struct ether_hdr *eth_hdr, union tunnel_offload_info *info) ethertype, ol_flags); } else if (l4_proto == IPPROTO_TCP) { tcp_hdr = (struct tcp_hdr *)((char *)l3_hdr + info->l3_len); - ol_flags |= PKT_TX_TCP_CKSUM; - tcp_hdr->cksum = get_psd_sum(l3_hdr, ethertype, - ol_flags); + /* Put PKT_TX_TCP_SEG bit setting before get_psd_sum(), because + * it depends on PKT_TX_TCP_SEG to calculate pseudo-header + * checksum. + */ if (tso_segsz != 0) { ol_flags |= PKT_TX_TCP_SEG; info->tso_segsz = tso_segsz; info->l4_len = (tcp_hdr->data_off & 0xf0) >> 2; } + ol_flags |= PKT_TX_TCP_CKSUM; + tcp_hdr->cksum = get_psd_sum(l3_hdr, ethertype, ol_flags); } else if (l4_proto == IPPROTO_SCTP) { sctp_hdr = (struct sctp_hdr *)((char *)l3_hdr + info->l3_len); @@ -237,6 +240,8 @@ encapsulation(struct rte_mbuf *m, uint8_t queue_id) m->outer_l2_len = sizeof(struct ether_hdr); m->outer_l3_len = sizeof(struct ipv4_hdr); + ol_flags |= PKT_TX_TUNNEL_VXLAN; + m->ol_flags |= ol_flags; m->tso_segsz = tx_offload.tso_segsz; diff --git a/dpdk/examples/tep_termination/vxlan_setup.c b/dpdk/examples/tep_termination/vxlan_setup.c index 8f1f15bb..1ad4ca3c 100644 --- a/dpdk/examples/tep_termination/vxlan_setup.c +++ b/dpdk/examples/tep_termination/vxlan_setup.c @@ -49,7 +49,7 @@ #include #include "main.h" -#include "rte_virtio_net.h" +#include "rte_vhost.h" #include "vxlan.h" #include "vxlan_setup.h" @@ -102,7 +102,7 @@ static const struct rte_eth_conf port_conf = { .hw_ip_checksum = 0, /**< IP checksum offload disabled */ .hw_vlan_filter = 0, /**< VLAN filtering disabled */ .jumbo_frame = 0, /**< Jumbo Frame Support disabled */ - .hw_strip_crc = 0, /**< CRC stripped by hardware */ + .hw_strip_crc = 1, /**< CRC stripped by hardware */ }, .txmode = { .mq_mode = ETH_MQ_TX_NONE, @@ -129,14 +129,14 @@ const uint16_t tenant_id_conf[] = { * coming from the mbuf_pool passed as parameter */ int -vxlan_port_init(uint8_t port, struct rte_mempool *mbuf_pool) +vxlan_port_init(uint16_t port, struct rte_mempool *mbuf_pool) { int retval; uint16_t q; struct rte_eth_dev_info dev_info; uint16_t rx_rings, tx_rings = (uint16_t)rte_lcore_count(); - const uint16_t rx_ring_size = RTE_TEST_RX_DESC_DEFAULT; - const uint16_t tx_ring_size = RTE_TEST_TX_DESC_DEFAULT; + uint16_t rx_ring_size = RTE_TEST_RX_DESC_DEFAULT; + uint16_t tx_ring_size = RTE_TEST_TX_DESC_DEFAULT; struct rte_eth_udp_tunnel tunnel_udp; struct rte_eth_rxconf *rxconf; struct rte_eth_txconf *txconf; @@ -166,6 +166,11 @@ vxlan_port_init(uint8_t port, struct rte_mempool *mbuf_pool) if (retval != 0) return retval; + retval = rte_eth_dev_adjust_nb_rx_tx_desc(port, &rx_ring_size, + &tx_ring_size); + if (retval != 0) + return retval; + /* Setup the queues. */ for (q = 0; q < rx_rings; q++) { retval = rte_eth_rx_queue_setup(port, q, rx_ring_size, @@ -197,7 +202,7 @@ vxlan_port_init(uint8_t port, struct rte_mempool *mbuf_pool) rte_eth_macaddr_get(port, &ports_eth_addr[port]); RTE_LOG(INFO, PORT, "Port %u MAC: %02"PRIx8" %02"PRIx8" %02"PRIx8 " %02"PRIx8" %02"PRIx8" %02"PRIx8"\n", - (unsigned)port, + port, ports_eth_addr[port].addr_bytes[0], ports_eth_addr[port].addr_bytes[1], ports_eth_addr[port].addr_bytes[2], @@ -409,7 +414,7 @@ vxlan_unlink(struct vhost_dev *vdev) /* Transmit packets after encapsulating */ int -vxlan_tx_pkts(uint8_t port_id, uint16_t queue_id, +vxlan_tx_pkts(uint16_t port_id, uint16_t queue_id, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { int ret = 0; uint16_t i; diff --git a/dpdk/examples/tep_termination/vxlan_setup.h b/dpdk/examples/tep_termination/vxlan_setup.h index 8d264619..2e3550d3 100644 --- a/dpdk/examples/tep_termination/vxlan_setup.h +++ b/dpdk/examples/tep_termination/vxlan_setup.h @@ -37,14 +37,14 @@ extern uint16_t nb_devices; extern uint16_t udp_port; extern uint8_t filter_idx; -extern uint8_t ports[RTE_MAX_ETHPORTS]; +extern uint16_t ports[RTE_MAX_ETHPORTS]; extern struct ether_addr ports_eth_addr[RTE_MAX_ETHPORTS]; extern uint32_t enable_stats; extern struct device_statistics dev_statistics[MAX_DEVICES]; extern uint8_t rx_decap; extern uint8_t tx_encap; -typedef int (*ol_port_configure_t)(uint8_t port, +typedef int (*ol_port_configure_t)(uint16_t port, struct rte_mempool *mbuf_pool); typedef int (*ol_tunnel_setup_t)(struct vhost_dev *vdev, @@ -52,7 +52,7 @@ typedef int (*ol_tunnel_setup_t)(struct vhost_dev *vdev, typedef void (*ol_tunnel_destroy_t)(struct vhost_dev *vdev); -typedef int (*ol_tx_handle_t)(uint8_t port_id, uint16_t queue_id, +typedef int (*ol_tx_handle_t)(uint16_t port_id, uint16_t queue_id, struct rte_mbuf **tx_pkts, uint16_t nb_pkts); typedef int (*ol_rx_handle_t)(int vid, struct rte_mbuf **pkts, @@ -70,7 +70,7 @@ struct ol_switch_ops { }; int -vxlan_port_init(uint8_t port, struct rte_mempool *mbuf_pool); +vxlan_port_init(uint16_t port, struct rte_mempool *mbuf_pool); int vxlan_link(struct vhost_dev *vdev, struct rte_mbuf *m); @@ -79,7 +79,7 @@ void vxlan_unlink(struct vhost_dev *vdev); int -vxlan_tx_pkts(uint8_t port_id, uint16_t queue_id, +vxlan_tx_pkts(uint16_t port_id, uint16_t queue_id, struct rte_mbuf **tx_pkts, uint16_t nb_pkts); int vxlan_rx_pkts(int vid, struct rte_mbuf **pkts, uint32_t count); diff --git a/dpdk/examples/timer/Makefile b/dpdk/examples/timer/Makefile index af12b7ba..7db48ec6 100644 --- a/dpdk/examples/timer/Makefile +++ b/dpdk/examples/timer/Makefile @@ -33,7 +33,7 @@ ifeq ($(RTE_SDK),) $(error "Please define RTE_SDK environment variable") endif -# Default target, can be overriden by command line or environment +# Default target, can be overridden by command line or environment RTE_TARGET ?= x86_64-native-linuxapp-gcc include $(RTE_SDK)/mk/rte.vars.mk diff --git a/dpdk/examples/timer/main.c b/dpdk/examples/timer/main.c index 37ad559e..444e9cc1 100644 --- a/dpdk/examples/timer/main.c +++ b/dpdk/examples/timer/main.c @@ -39,7 +39,6 @@ #include #include -#include #include #include #include diff --git a/dpdk/examples/vhost/Makefile b/dpdk/examples/vhost/Makefile index e95c68ae..add9f27b 100644 --- a/dpdk/examples/vhost/Makefile +++ b/dpdk/examples/vhost/Makefile @@ -33,7 +33,7 @@ ifeq ($(RTE_SDK),) $(error "Please define RTE_SDK environment variable") endif -# Default target, can be overriden by command line or environment +# Default target, can be overridden by command line or environment RTE_TARGET ?= x86_64-native-linuxapp-gcc include $(RTE_SDK)/mk/rte.vars.mk @@ -48,7 +48,7 @@ else APP = vhost-switch # all source are stored in SRCS-y -SRCS-y := main.c +SRCS-y := main.c virtio_net.c CFLAGS += -O2 -D_FILE_OFFSET_BITS=64 CFLAGS += $(WERROR_FLAGS) diff --git a/dpdk/examples/vhost/main.c b/dpdk/examples/vhost/main.c index 92a98233..1f532fe3 100644 --- a/dpdk/examples/vhost/main.c +++ b/dpdk/examples/vhost/main.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. + * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -49,9 +49,10 @@ #include #include #include -#include +#include #include #include +#include #include "main.h" @@ -65,7 +66,6 @@ #define MBUF_CACHE_SIZE 128 #define MBUF_DATA_SIZE RTE_MBUF_DEFAULT_BUF_SIZE -#define MAX_PKT_BURST 32 /* Max burst size for RX/TX */ #define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */ #define BURST_RX_WAIT_US 15 /* Defines how long we wait between retries on RX */ @@ -90,9 +90,6 @@ /* Size of buffers used for snprintfs. */ #define MAX_PRINT_BUFF 6072 -/* Maximum character device basename size. */ -#define MAX_BASENAME_SZ 10 - /* Maximum long option length for option parsing. */ #define MAX_LONG_OPT_SZ 64 @@ -109,9 +106,6 @@ static uint32_t num_devices; static struct rte_mempool *mbuf_pool; static int mergeable; -/* Do vlan strip on host, enabled on default */ -static uint32_t vlan_strip = 1; - /* Enable VM2VM communications. If this is disabled then the MAC address compare is skipped. */ typedef enum { VM2VM_DISABLED = 0, @@ -133,14 +127,18 @@ static uint32_t enable_tx_csum; static uint32_t enable_tso; static int client_mode; +static int dequeue_zero_copy; + +static int builtin_net_driver; /* Specify timeout (in useconds) between retries on RX. */ static uint32_t burst_rx_delay_time = BURST_RX_WAIT_US; /* Specify the number of retries on RX. */ static uint32_t burst_rx_retry_num = BURST_RX_RETRIES; -/* Character device basename. Can be set by user. */ -static char dev_basename[MAX_BASENAME_SZ] = "vhost-net"; +/* Socket file paths. Can be set by user */ +static char *socket_files; +static int nb_sockets; /* empty vmdq configuration structure. Filled in programatically */ static struct rte_eth_conf vmdq_conf_default = { @@ -157,7 +155,7 @@ static struct rte_eth_conf vmdq_conf_default = { */ .hw_vlan_strip = 1, /**< VLAN strip enabled. */ .jumbo_frame = 0, /**< Jumbo Frame Support disabled */ - .hw_strip_crc = 0, /**< CRC stripped by hardware */ + .hw_strip_crc = 1, /**< CRC stripped by hardware */ }, .txmode = { @@ -179,7 +177,7 @@ static struct rte_eth_conf vmdq_conf_default = { }; static unsigned lcore_ids[RTE_MAX_LCORE]; -static uint8_t ports[RTE_MAX_ETHPORTS]; +static uint16_t ports[RTE_MAX_ETHPORTS]; static unsigned num_ports = 0; /**< The number of ports specified in command line */ static uint16_t num_pf_queues, num_vmdq_queues; static uint16_t vmdq_pool_base, vmdq_queue_base; @@ -267,7 +265,7 @@ validate_num_devices(uint32_t max_nb_devices) * coming from the mbuf_pool passed as parameter */ static inline int -port_init(uint8_t port) +port_init(uint16_t port) { struct rte_eth_dev_info dev_info; struct rte_eth_conf port_conf; @@ -281,12 +279,6 @@ port_init(uint8_t port) /* The max pool number from dev_info will be used to validate the pool number specified in cmd line */ rte_eth_dev_info_get (port, &dev_info); - if (dev_info.max_rx_queues > MAX_QUEUES) { - rte_exit(EXIT_FAILURE, - "please define MAX_QUEUES no less than %u in %s\n", - dev_info.max_rx_queues, __FILE__); - } - rxconf = &dev_info.default_rxconf; txconf = &dev_info.default_txconf; rxconf->rx_drop_en = 1; @@ -299,6 +291,17 @@ port_init(uint8_t port) rx_ring_size = RTE_TEST_RX_DESC_DEFAULT; tx_ring_size = RTE_TEST_TX_DESC_DEFAULT; + + /* + * When dequeue zero copy is enabled, guest Tx used vring will be + * updated only when corresponding mbuf is freed. Thus, the nb_tx_desc + * (tx_ring_size here) must be small enough so that the driver will + * hit the free threshold easily and free mbufs timely. Otherwise, + * guest Tx vring would be starved. + */ + if (dequeue_zero_copy) + tx_ring_size = 64; + tx_rings = (uint16_t)rte_lcore_count(); retval = validate_num_devices(MAX_DEVICES); @@ -321,16 +324,6 @@ port_init(uint8_t port) if (port >= rte_eth_dev_count()) return -1; - if (enable_tx_csum == 0) - rte_vhost_feature_disable(1ULL << VIRTIO_NET_F_CSUM); - - if (enable_tso == 0) { - rte_vhost_feature_disable(1ULL << VIRTIO_NET_F_HOST_TSO4); - rte_vhost_feature_disable(1ULL << VIRTIO_NET_F_HOST_TSO6); - rte_vhost_feature_disable(1ULL << VIRTIO_NET_F_GUEST_TSO4); - rte_vhost_feature_disable(1ULL << VIRTIO_NET_F_GUEST_TSO6); - } - rx_rings = (uint16_t)dev_info.max_rx_queues; /* Configure ethernet device. */ retval = rte_eth_dev_configure(port, rx_rings, tx_rings, &port_conf); @@ -340,6 +333,19 @@ port_init(uint8_t port) return retval; } + retval = rte_eth_dev_adjust_nb_rx_tx_desc(port, &rx_ring_size, + &tx_ring_size); + if (retval != 0) { + RTE_LOG(ERR, VHOST_PORT, "Failed to adjust number of descriptors " + "for port %u: %s.\n", port, strerror(-retval)); + return retval; + } + if (rx_ring_size > RTE_TEST_RX_DESC_DEFAULT) { + RTE_LOG(ERR, VHOST_PORT, "Mbuf pool has an insufficient size " + "for Rx queues on port %u.\n", port); + return -1; + } + /* Setup the queues. */ for (q = 0; q < rx_rings; q ++) { retval = rte_eth_rx_queue_setup(port, q, rx_ring_size, @@ -380,7 +386,7 @@ port_init(uint8_t port) RTE_LOG(INFO, VHOST_PORT, "Max virtio devices supported: %u\n", num_devices); RTE_LOG(INFO, VHOST_PORT, "Port %u MAC: %02"PRIx8" %02"PRIx8" %02"PRIx8 " %02"PRIx8" %02"PRIx8" %02"PRIx8"\n", - (unsigned)port, + port, vmdq_ports_eth_addr[port].addr_bytes[0], vmdq_ports_eth_addr[port].addr_bytes[1], vmdq_ports_eth_addr[port].addr_bytes[2], @@ -392,17 +398,18 @@ port_init(uint8_t port) } /* - * Set character device basename. + * Set socket file path. */ static int -us_vhost_parse_basename(const char *q_arg) +us_vhost_parse_socket_path(const char *q_arg) { /* parse number string */ - - if (strnlen(q_arg, MAX_BASENAME_SZ) > MAX_BASENAME_SZ) + if (strnlen(q_arg, PATH_MAX) == PATH_MAX) return -1; - else - snprintf((char*)&dev_basename, MAX_BASENAME_SZ, "%s", q_arg); + + socket_files = realloc(socket_files, PATH_MAX * (nb_sockets + 1)); + snprintf(socket_files + nb_sockets * PATH_MAX, PATH_MAX, "%s", q_arg); + nb_sockets++; return 0; } @@ -462,7 +469,7 @@ us_vhost_usage(const char *prgname) RTE_LOG(INFO, VHOST_CONFIG, "%s [EAL options] -- -p PORTMASK\n" " --vm2vm [0|1|2]\n" " --rx_retry [0|1] --mergeable [0|1] --stats [0-N]\n" - " --dev-basename \n" + " --socket-file \n" " --nb-devices ND\n" " -p PORTMASK: Set mask for ports to be used by application\n" " --vm2vm [0|1|2]: disable/software(default)/hardware vm2vm comms\n" @@ -470,12 +477,12 @@ us_vhost_usage(const char *prgname) " --rx-retry-delay [0-N]: timeout(in usecond) between retries on RX. This makes effect only if retries on rx enabled\n" " --rx-retry-num [0-N]: the number of retries on rx. This makes effect only if retries on rx enabled\n" " --mergeable [0|1]: disable(default)/enable RX mergeable buffers\n" - " --vlan-strip [0|1]: disable/enable(default) RX VLAN strip on host\n" " --stats [0-N]: 0: Disable stats, N: Time in seconds to print stats\n" - " --dev-basename: The basename to be used for the character device.\n" + " --socket-file: The path of the socket file.\n" " --tx-csum [0|1] disable/enable TX checksum offload.\n" " --tso [0|1] disable/enable TCP segment offload.\n" - " --client register a vhost-user socket as client mode.\n", + " --client register a vhost-user socket as client mode.\n" + " --dequeue-zero-copy enables dequeue zero copy\n", prgname); } @@ -495,12 +502,13 @@ us_vhost_parse_args(int argc, char **argv) {"rx-retry-delay", required_argument, NULL, 0}, {"rx-retry-num", required_argument, NULL, 0}, {"mergeable", required_argument, NULL, 0}, - {"vlan-strip", required_argument, NULL, 0}, {"stats", required_argument, NULL, 0}, - {"dev-basename", required_argument, NULL, 0}, + {"socket-file", required_argument, NULL, 0}, {"tx-csum", required_argument, NULL, 0}, {"tso", required_argument, NULL, 0}, {"client", no_argument, &client_mode, 1}, + {"dequeue-zero-copy", no_argument, &dequeue_zero_copy, 1}, + {"builtin-net-driver", no_argument, &builtin_net_driver, 1}, {NULL, 0, 0, 0}, }; @@ -523,7 +531,6 @@ us_vhost_parse_args(int argc, char **argv) vmdq_conf_default.rx_adv_conf.vmdq_rx_conf.rx_mode = ETH_VMDQ_ACCEPT_BROADCAST | ETH_VMDQ_ACCEPT_MULTICAST; - rte_vhost_feature_enable(1ULL << VIRTIO_NET_F_CTRL_RX); break; @@ -618,27 +625,12 @@ us_vhost_parse_args(int argc, char **argv) } } - /* Enable/disable RX VLAN strip on host. */ - if (!strncmp(long_option[option_index].name, - "vlan-strip", MAX_LONG_OPT_SZ)) { - ret = parse_num_opt(optarg, 1); - if (ret == -1) { - RTE_LOG(INFO, VHOST_CONFIG, - "Invalid argument for VLAN strip [0|1]\n"); - us_vhost_usage(prgname); - return -1; - } else { - vlan_strip = !!ret; - vmdq_conf_default.rxmode.hw_vlan_strip = - vlan_strip; - } - } - /* Enable/disable stats. */ if (!strncmp(long_option[option_index].name, "stats", MAX_LONG_OPT_SZ)) { ret = parse_num_opt(optarg, INT32_MAX); if (ret == -1) { - RTE_LOG(INFO, VHOST_CONFIG, "Invalid argument for stats [0..N]\n"); + RTE_LOG(INFO, VHOST_CONFIG, + "Invalid argument for stats [0..N]\n"); us_vhost_usage(prgname); return -1; } else { @@ -646,10 +638,13 @@ us_vhost_parse_args(int argc, char **argv) } } - /* Set character device basename. */ - if (!strncmp(long_option[option_index].name, "dev-basename", MAX_LONG_OPT_SZ)) { - if (us_vhost_parse_basename(optarg) == -1) { - RTE_LOG(INFO, VHOST_CONFIG, "Invalid argument for character device basename (Max %d characters)\n", MAX_BASENAME_SZ); + /* Set socket file path. */ + if (!strncmp(long_option[option_index].name, + "socket-file", MAX_LONG_OPT_SZ)) { + if (us_vhost_parse_socket_path(optarg) == -1) { + RTE_LOG(INFO, VHOST_CONFIG, + "Invalid argument for socket name (Max %d characters)\n", + PATH_MAX); us_vhost_usage(prgname); return -1; } @@ -666,7 +661,7 @@ us_vhost_parse_args(int argc, char **argv) for (i = 0; i < RTE_MAX_ETHPORTS; i++) { if (enabled_port_mask & (1 << i)) - ports[num_ports++] = (uint8_t)i; + ports[num_ports++] = i; } if ((num_ports == 0) || (num_ports > MAX_SUP_PORTS)) { @@ -704,7 +699,7 @@ static unsigned check_ports_num(unsigned nb_ports) return valid_num_ports; } -static inline struct vhost_dev *__attribute__((always_inline)) +static __rte_always_inline struct vhost_dev * find_vhost_dev(struct ether_addr *mac) { struct vhost_dev *vdev; @@ -761,10 +756,7 @@ link_vmdq(struct vhost_dev *vdev, struct rte_mbuf *m) "(%d) failed to add device MAC address to VMDQ\n", vdev->vid); - /* Enable stripping of the vlan tag as we handle routing. */ - if (vlan_strip) - rte_eth_dev_set_vlan_strip_on_queue(ports[0], - (uint16_t)vdev->vmdq_rx_q, 1); + rte_eth_dev_set_vlan_strip_on_queue(ports[0], vdev->vmdq_rx_q, 1); /* Set device as ready for RX. */ vdev->ready = DEVICE_RX; @@ -807,13 +799,18 @@ unlink_vmdq(struct vhost_dev *vdev) } } -static inline void __attribute__((always_inline)) +static __rte_always_inline void virtio_xmit(struct vhost_dev *dst_vdev, struct vhost_dev *src_vdev, struct rte_mbuf *m) { uint16_t ret; - ret = rte_vhost_enqueue_burst(dst_vdev->vid, VIRTIO_RXQ, &m, 1); + if (builtin_net_driver) { + ret = vs_enqueue_pkts(dst_vdev, VIRTIO_RXQ, &m, 1); + } else { + ret = rte_vhost_enqueue_burst(dst_vdev->vid, VIRTIO_RXQ, &m, 1); + } + if (enable_stats) { rte_atomic64_inc(&dst_vdev->stats.rx_total_atomic); rte_atomic64_add(&dst_vdev->stats.rx_atomic, ret); @@ -826,7 +823,7 @@ virtio_xmit(struct vhost_dev *dst_vdev, struct vhost_dev *src_vdev, * Check if the packet destination MAC address is for a local device. If so then put * the packet on that devices RX queue. If not then return. */ -static inline int __attribute__((always_inline)) +static __rte_always_inline int virtio_tx_local(struct vhost_dev *vdev, struct rte_mbuf *m) { struct ether_hdr *pkt_hdr; @@ -839,17 +836,17 @@ virtio_tx_local(struct vhost_dev *vdev, struct rte_mbuf *m) return -1; if (vdev->vid == dst_vdev->vid) { - RTE_LOG(DEBUG, VHOST_DATA, + RTE_LOG_DP(DEBUG, VHOST_DATA, "(%d) TX: src and dst MAC is same. Dropping packet.\n", vdev->vid); return 0; } - RTE_LOG(DEBUG, VHOST_DATA, + RTE_LOG_DP(DEBUG, VHOST_DATA, "(%d) TX: MAC address is local\n", dst_vdev->vid); if (unlikely(dst_vdev->remove)) { - RTE_LOG(DEBUG, VHOST_DATA, + RTE_LOG_DP(DEBUG, VHOST_DATA, "(%d) device is marked for removal\n", dst_vdev->vid); return 0; } @@ -862,7 +859,7 @@ virtio_tx_local(struct vhost_dev *vdev, struct rte_mbuf *m) * Check if the destination MAC of a packet is one local VM, * and get its vlan tag, and offset if it is. */ -static inline int __attribute__((always_inline)) +static __rte_always_inline int find_local_dest(struct vhost_dev *vdev, struct rte_mbuf *m, uint32_t *offset, uint16_t *vlan_tag) { @@ -874,7 +871,7 @@ find_local_dest(struct vhost_dev *vdev, struct rte_mbuf *m, return 0; if (vdev->vid == dst_vdev->vid) { - RTE_LOG(DEBUG, VHOST_DATA, + RTE_LOG_DP(DEBUG, VHOST_DATA, "(%d) TX: src and dst MAC is same. Dropping packet.\n", vdev->vid); return -1; @@ -888,7 +885,7 @@ find_local_dest(struct vhost_dev *vdev, struct rte_mbuf *m, *offset = VLAN_HLEN; *vlan_tag = vlan_tags[vdev->vid]; - RTE_LOG(DEBUG, VHOST_DATA, + RTE_LOG_DP(DEBUG, VHOST_DATA, "(%d) TX: pkt to local VM device id: (%d), vlan tag: %u.\n", vdev->vid, dst_vdev->vid, *vlan_tag); @@ -930,7 +927,7 @@ free_pkts(struct rte_mbuf **pkts, uint16_t n) rte_pktmbuf_free(pkts[n]); } -static inline void __attribute__((always_inline)) +static __rte_always_inline void do_drain_mbuf_table(struct mbuf_table *tx_q) { uint16_t count; @@ -947,7 +944,7 @@ do_drain_mbuf_table(struct mbuf_table *tx_q) * This function routes the TX packet to the correct interface. This * may be a local device or the physical port. */ -static inline void __attribute__((always_inline)) +static __rte_always_inline void virtio_tx_route(struct vhost_dev *vdev, struct rte_mbuf *m, uint16_t vlan_tag) { struct mbuf_table *tx_q; @@ -961,7 +958,8 @@ virtio_tx_route(struct vhost_dev *vdev, struct rte_mbuf *m, uint16_t vlan_tag) struct vhost_dev *vdev2; TAILQ_FOREACH(vdev2, &vhost_dev_list, global_vdev_entry) { - virtio_xmit(vdev2, vdev, m); + if (vdev2 != vdev) + virtio_xmit(vdev2, vdev, m); } goto queue2nic; } @@ -980,7 +978,7 @@ virtio_tx_route(struct vhost_dev *vdev, struct rte_mbuf *m, uint16_t vlan_tag) } } - RTE_LOG(DEBUG, VHOST_DATA, + RTE_LOG_DP(DEBUG, VHOST_DATA, "(%d) TX: MAC address is external\n", vdev->vid); queue2nic: @@ -1035,7 +1033,7 @@ queue2nic: } -static inline void __attribute__((always_inline)) +static __rte_always_inline void drain_mbuf_table(struct mbuf_table *tx_q) { static uint64_t prev_tsc; @@ -1048,14 +1046,14 @@ drain_mbuf_table(struct mbuf_table *tx_q) if (unlikely(cur_tsc - prev_tsc > MBUF_TABLE_DRAIN_TSC)) { prev_tsc = cur_tsc; - RTE_LOG(DEBUG, VHOST_DATA, + RTE_LOG_DP(DEBUG, VHOST_DATA, "TX queue drained after timeout with burst size %u\n", tx_q->len); do_drain_mbuf_table(tx_q); } } -static inline void __attribute__((always_inline)) +static __rte_always_inline void drain_eth_rx(struct vhost_dev *vdev) { uint16_t rx_count, enqueue_count; @@ -1084,8 +1082,13 @@ drain_eth_rx(struct vhost_dev *vdev) } } - enqueue_count = rte_vhost_enqueue_burst(vdev->vid, VIRTIO_RXQ, + if (builtin_net_driver) { + enqueue_count = vs_enqueue_pkts(vdev, VIRTIO_RXQ, pkts, rx_count); + } else { + enqueue_count = rte_vhost_enqueue_burst(vdev->vid, VIRTIO_RXQ, + pkts, rx_count); + } if (enable_stats) { rte_atomic64_add(&vdev->stats.rx_total_atomic, rx_count); rte_atomic64_add(&vdev->stats.rx_atomic, enqueue_count); @@ -1094,15 +1097,20 @@ drain_eth_rx(struct vhost_dev *vdev) free_pkts(pkts, rx_count); } -static inline void __attribute__((always_inline)) +static __rte_always_inline void drain_virtio_tx(struct vhost_dev *vdev) { struct rte_mbuf *pkts[MAX_PKT_BURST]; uint16_t count; uint16_t i; - count = rte_vhost_dequeue_burst(vdev->vid, VIRTIO_TXQ, mbuf_pool, + if (builtin_net_driver) { + count = vs_dequeue_pkts(vdev, VIRTIO_TXQ, mbuf_pool, pkts, MAX_PKT_BURST); + } else { + count = rte_vhost_dequeue_burst(vdev->vid, VIRTIO_TXQ, + mbuf_pool, pkts, MAX_PKT_BURST); + } /* setup VMDq for the first packet */ if (unlikely(vdev->ready == DEVICE_MAC_LEARNING) && count) { @@ -1205,6 +1213,9 @@ destroy_device(int vid) rte_pause(); } + if (builtin_net_driver) + vs_vhost_net_remove(vdev); + TAILQ_REMOVE(&lcore_info[vdev->coreid].vdev_list, vdev, lcore_vdev_entry); TAILQ_REMOVE(&vhost_dev_list, vdev, global_vdev_entry); @@ -1253,6 +1264,9 @@ new_device(int vid) } vdev->vid = vid; + if (builtin_net_driver) + vs_vhost_net_setup(vdev); + TAILQ_INSERT_TAIL(&vhost_dev_list, vdev, global_vdev_entry); vdev->vmdq_rx_q = vid * queues_per_pool + vmdq_queue_base; @@ -1288,7 +1302,7 @@ new_device(int vid) * These callback allow devices to be added to the data core when configuration * has been fully complete. */ -static const struct virtio_net_device_ops virtio_net_device_ops = +static const struct vhost_device_ops virtio_net_device_ops = { .new_device = new_device, .destroy_device = destroy_device, @@ -1340,14 +1354,27 @@ print_stats(void) } } +static void +unregister_drivers(int socket_num) +{ + int i, ret; + + for (i = 0; i < socket_num; i++) { + ret = rte_vhost_driver_unregister(socket_files + i * PATH_MAX); + if (ret != 0) + RTE_LOG(ERR, VHOST_CONFIG, + "Fail to unregister vhost driver for %s.\n", + socket_files + i * PATH_MAX); + } +} + /* When we receive a INT signal, unregister vhost driver */ static void sigint_handler(__rte_unused int signum) { /* Unregister vhost driver. */ - int ret = rte_vhost_driver_unregister((char *)&dev_basename); - if (ret != 0) - rte_exit(EXIT_FAILURE, "vhost driver unregister failure.\n"); + unregister_drivers(nb_sockets); + exit(0); } @@ -1387,7 +1414,7 @@ create_mbuf_pool(uint16_t nr_port, uint32_t nr_switch_core, uint32_t mbuf_size, mtu = 64 * 1024; nr_mbufs_per_core = (mtu + mbuf_size) * MAX_PKT_BURST / - (mbuf_size - RTE_PKTMBUF_HEADROOM) * MAX_PKT_BURST; + (mbuf_size - RTE_PKTMBUF_HEADROOM); nr_mbufs_per_core += nr_rx_desc; nr_mbufs_per_core = RTE_MAX(nr_mbufs_per_core, nr_mbuf_cache); @@ -1403,16 +1430,15 @@ create_mbuf_pool(uint16_t nr_port, uint32_t nr_switch_core, uint32_t mbuf_size, } /* - * Main function, does initialisation and calls the per-lcore functions. The CUSE - * device is also registered here to handle the IOCTLs. + * Main function, does initialisation and calls the per-lcore functions. */ int main(int argc, char *argv[]) { unsigned lcore_id, core_id = 0; unsigned nb_ports, valid_num_ports; - int ret; - uint8_t portid; + int ret, i; + uint16_t portid; static pthread_t tid; char thread_name[RTE_MAX_THREAD_NAME_LEN]; uint64_t flags = 0; @@ -1431,11 +1457,12 @@ main(int argc, char *argv[]) if (ret < 0) rte_exit(EXIT_FAILURE, "Invalid argument\n"); - for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id ++) + for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { TAILQ_INIT(&lcore_info[lcore_id].vdev_list); if (rte_lcore_is_enabled(lcore_id)) - lcore_ids[core_id ++] = lcore_id; + lcore_ids[core_id++] = lcore_id; + } if (rte_lcore_count() > RTE_MAX_LCORE) rte_exit(EXIT_FAILURE,"Not enough cores\n"); @@ -1503,21 +1530,67 @@ main(int argc, char *argv[]) RTE_LCORE_FOREACH_SLAVE(lcore_id) rte_eal_remote_launch(switch_worker, NULL, lcore_id); - if (mergeable == 0) - rte_vhost_feature_disable(1ULL << VIRTIO_NET_F_MRG_RXBUF); - if (client_mode) flags |= RTE_VHOST_USER_CLIENT; - /* Register vhost(cuse or user) driver to handle vhost messages. */ - ret = rte_vhost_driver_register(dev_basename, flags); - if (ret != 0) - rte_exit(EXIT_FAILURE, "vhost driver register failure.\n"); + if (dequeue_zero_copy) + flags |= RTE_VHOST_USER_DEQUEUE_ZERO_COPY; - rte_vhost_driver_callback_register(&virtio_net_device_ops); + /* Register vhost user driver to handle vhost messages. */ + for (i = 0; i < nb_sockets; i++) { + char *file = socket_files + i * PATH_MAX; + ret = rte_vhost_driver_register(file, flags); + if (ret != 0) { + unregister_drivers(i); + rte_exit(EXIT_FAILURE, + "vhost driver register failure.\n"); + } + + if (builtin_net_driver) + rte_vhost_driver_set_features(file, VIRTIO_NET_FEATURES); + + if (mergeable == 0) { + rte_vhost_driver_disable_features(file, + 1ULL << VIRTIO_NET_F_MRG_RXBUF); + } + + if (enable_tx_csum == 0) { + rte_vhost_driver_disable_features(file, + 1ULL << VIRTIO_NET_F_CSUM); + } + + if (enable_tso == 0) { + rte_vhost_driver_disable_features(file, + 1ULL << VIRTIO_NET_F_HOST_TSO4); + rte_vhost_driver_disable_features(file, + 1ULL << VIRTIO_NET_F_HOST_TSO6); + rte_vhost_driver_disable_features(file, + 1ULL << VIRTIO_NET_F_GUEST_TSO4); + rte_vhost_driver_disable_features(file, + 1ULL << VIRTIO_NET_F_GUEST_TSO6); + } + + if (promiscuous) { + rte_vhost_driver_enable_features(file, + 1ULL << VIRTIO_NET_F_CTRL_RX); + } + + ret = rte_vhost_driver_callback_register(file, + &virtio_net_device_ops); + if (ret != 0) { + rte_exit(EXIT_FAILURE, + "failed to register vhost driver callbacks.\n"); + } + + if (rte_vhost_driver_start(file) < 0) { + rte_exit(EXIT_FAILURE, + "failed to start vhost driver.\n"); + } + } + + RTE_LCORE_FOREACH_SLAVE(lcore_id) + rte_eal_wait_lcore(lcore_id); - /* Start CUSE session. */ - rte_vhost_driver_session_start(); return 0; } diff --git a/dpdk/examples/vhost/main.h b/dpdk/examples/vhost/main.h index 6bb42e89..9a2aca37 100644 --- a/dpdk/examples/vhost/main.h +++ b/dpdk/examples/vhost/main.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. + * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -36,11 +36,17 @@ #include +#include + /* Macros for printing using RTE_LOG */ #define RTE_LOGTYPE_VHOST_CONFIG RTE_LOGTYPE_USER1 #define RTE_LOGTYPE_VHOST_DATA RTE_LOGTYPE_USER2 #define RTE_LOGTYPE_VHOST_PORT RTE_LOGTYPE_USER3 +enum {VIRTIO_RXQ, VIRTIO_TXQ, VIRTIO_QNUM}; + +#define MAX_PKT_BURST 32 /* Max burst size for RX/TX */ + struct device_statistics { uint64_t tx; uint64_t tx_total; @@ -48,6 +54,12 @@ struct device_statistics { rte_atomic64_t rx_total_atomic; }; +struct vhost_queue { + struct rte_vhost_vring vr; + uint16_t last_avail_idx; + uint16_t last_used_idx; +}; + struct vhost_dev { /**< Number of memory regions for gpa to hpa translation. */ uint32_t nregions_hpa; @@ -65,9 +77,16 @@ struct vhost_dev { volatile uint8_t remove; int vid; + uint64_t features; + size_t hdr_len; + uint16_t nr_vrings; + struct rte_vhost_memory *mem; struct device_statistics stats; TAILQ_ENTRY(vhost_dev) global_vdev_entry; TAILQ_ENTRY(vhost_dev) lcore_vdev_entry; + +#define MAX_QUEUE_PAIRS 4 + struct vhost_queue queues[MAX_QUEUE_PAIRS * 2]; } __rte_cache_aligned; TAILQ_HEAD(vhost_dev_tailq_list, vhost_dev); @@ -88,4 +107,15 @@ struct lcore_info { struct vhost_dev_tailq_list vdev_list; }; +/* we implement non-extra virtio net features */ +#define VIRTIO_NET_FEATURES 0 + +void vs_vhost_net_setup(struct vhost_dev *dev); +void vs_vhost_net_remove(struct vhost_dev *dev); +uint16_t vs_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id, + struct rte_mbuf **pkts, uint32_t count); + +uint16_t vs_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id, + struct rte_mempool *mbuf_pool, + struct rte_mbuf **pkts, uint16_t count); #endif /* _MAIN_H_ */ diff --git a/dpdk/examples/vhost/virtio_net.c b/dpdk/examples/vhost/virtio_net.c new file mode 100644 index 00000000..31c3dd06 --- /dev/null +++ b/dpdk/examples/vhost/virtio_net.c @@ -0,0 +1,476 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include + +#include +#include +#include + +#include "main.h" + +/* + * A very simple vhost-user net driver implementation, without + * any extra features being enabled, such as TSO and mrg-Rx. + */ + +void +vs_vhost_net_setup(struct vhost_dev *dev) +{ + uint16_t i; + int vid = dev->vid; + struct vhost_queue *queue; + + RTE_LOG(INFO, VHOST_CONFIG, + "setting builtin vhost-user net driver\n"); + + rte_vhost_get_negotiated_features(vid, &dev->features); + if (dev->features & (1 << VIRTIO_NET_F_MRG_RXBUF)) + dev->hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf); + else + dev->hdr_len = sizeof(struct virtio_net_hdr); + + rte_vhost_get_mem_table(vid, &dev->mem); + + dev->nr_vrings = rte_vhost_get_vring_num(vid); + for (i = 0; i < dev->nr_vrings; i++) { + queue = &dev->queues[i]; + + queue->last_used_idx = 0; + queue->last_avail_idx = 0; + rte_vhost_get_vhost_vring(vid, i, &queue->vr); + } +} + +void +vs_vhost_net_remove(struct vhost_dev *dev) +{ + free(dev->mem); +} + +static __rte_always_inline int +enqueue_pkt(struct vhost_dev *dev, struct rte_vhost_vring *vr, + struct rte_mbuf *m, uint16_t desc_idx) +{ + uint32_t desc_avail, desc_offset; + uint64_t desc_chunck_len; + uint32_t mbuf_avail, mbuf_offset; + uint32_t cpy_len; + struct vring_desc *desc; + uint64_t desc_addr, desc_gaddr; + struct virtio_net_hdr virtio_hdr = {0, 0, 0, 0, 0, 0}; + /* A counter to avoid desc dead loop chain */ + uint16_t nr_desc = 1; + + desc = &vr->desc[desc_idx]; + desc_chunck_len = desc->len; + desc_gaddr = desc->addr; + desc_addr = rte_vhost_va_from_guest_pa( + dev->mem, desc_gaddr, &desc_chunck_len); + /* + * Checking of 'desc_addr' placed outside of 'unlikely' macro to avoid + * performance issue with some versions of gcc (4.8.4 and 5.3.0) which + * otherwise stores offset on the stack instead of in a register. + */ + if (unlikely(desc->len < dev->hdr_len) || !desc_addr) + return -1; + + rte_prefetch0((void *)(uintptr_t)desc_addr); + + /* write virtio-net header */ + if (likely(desc_chunck_len >= dev->hdr_len)) { + *(struct virtio_net_hdr *)(uintptr_t)desc_addr = virtio_hdr; + desc_offset = dev->hdr_len; + } else { + uint64_t len; + uint64_t remain = dev->hdr_len; + uint64_t src = (uint64_t)(uintptr_t)&virtio_hdr, dst; + uint64_t guest_addr = desc_gaddr; + + while (remain) { + len = remain; + dst = rte_vhost_va_from_guest_pa(dev->mem, + guest_addr, &len); + if (unlikely(!dst || !len)) + return -1; + + rte_memcpy((void *)(uintptr_t)dst, + (void *)(uintptr_t)src, + len); + + remain -= len; + guest_addr += len; + dst += len; + } + + desc_chunck_len = desc->len - dev->hdr_len; + desc_gaddr += dev->hdr_len; + desc_addr = rte_vhost_va_from_guest_pa( + dev->mem, desc_gaddr, + &desc_chunck_len); + if (unlikely(!desc_addr)) + return -1; + + desc_offset = 0; + } + + desc_avail = desc->len - dev->hdr_len; + + mbuf_avail = rte_pktmbuf_data_len(m); + mbuf_offset = 0; + while (mbuf_avail != 0 || m->next != NULL) { + /* done with current mbuf, fetch next */ + if (mbuf_avail == 0) { + m = m->next; + + mbuf_offset = 0; + mbuf_avail = rte_pktmbuf_data_len(m); + } + + /* done with current desc buf, fetch next */ + if (desc_avail == 0) { + if ((desc->flags & VRING_DESC_F_NEXT) == 0) { + /* Room in vring buffer is not enough */ + return -1; + } + if (unlikely(desc->next >= vr->size || + ++nr_desc > vr->size)) + return -1; + + desc = &vr->desc[desc->next]; + desc_chunck_len = desc->len; + desc_gaddr = desc->addr; + desc_addr = rte_vhost_va_from_guest_pa( + dev->mem, desc_gaddr, &desc_chunck_len); + if (unlikely(!desc_addr)) + return -1; + + desc_offset = 0; + desc_avail = desc->len; + } else if (unlikely(desc_chunck_len == 0)) { + desc_chunck_len = desc_avail; + desc_gaddr += desc_offset; + desc_addr = rte_vhost_va_from_guest_pa(dev->mem, + desc_gaddr, + &desc_chunck_len); + if (unlikely(!desc_addr)) + return -1; + + desc_offset = 0; + } + + cpy_len = RTE_MIN(desc_chunck_len, mbuf_avail); + rte_memcpy((void *)((uintptr_t)(desc_addr + desc_offset)), + rte_pktmbuf_mtod_offset(m, void *, mbuf_offset), + cpy_len); + + mbuf_avail -= cpy_len; + mbuf_offset += cpy_len; + desc_avail -= cpy_len; + desc_offset += cpy_len; + desc_chunck_len -= cpy_len; + } + + return 0; +} + +uint16_t +vs_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id, + struct rte_mbuf **pkts, uint32_t count) +{ + struct vhost_queue *queue; + struct rte_vhost_vring *vr; + uint16_t avail_idx, free_entries, start_idx; + uint16_t desc_indexes[MAX_PKT_BURST]; + uint16_t used_idx; + uint32_t i; + + queue = &dev->queues[queue_id]; + vr = &queue->vr; + + avail_idx = *((volatile uint16_t *)&vr->avail->idx); + start_idx = queue->last_used_idx; + free_entries = avail_idx - start_idx; + count = RTE_MIN(count, free_entries); + count = RTE_MIN(count, (uint32_t)MAX_PKT_BURST); + if (count == 0) + return 0; + + /* Retrieve all of the desc indexes first to avoid caching issues. */ + rte_prefetch0(&vr->avail->ring[start_idx & (vr->size - 1)]); + for (i = 0; i < count; i++) { + used_idx = (start_idx + i) & (vr->size - 1); + desc_indexes[i] = vr->avail->ring[used_idx]; + vr->used->ring[used_idx].id = desc_indexes[i]; + vr->used->ring[used_idx].len = pkts[i]->pkt_len + + dev->hdr_len; + } + + rte_prefetch0(&vr->desc[desc_indexes[0]]); + for (i = 0; i < count; i++) { + uint16_t desc_idx = desc_indexes[i]; + int err; + + err = enqueue_pkt(dev, vr, pkts[i], desc_idx); + if (unlikely(err)) { + used_idx = (start_idx + i) & (vr->size - 1); + vr->used->ring[used_idx].len = dev->hdr_len; + } + + if (i + 1 < count) + rte_prefetch0(&vr->desc[desc_indexes[i+1]]); + } + + rte_smp_wmb(); + + *(volatile uint16_t *)&vr->used->idx += count; + queue->last_used_idx += count; + + /* flush used->idx update before we read avail->flags. */ + rte_mb(); + + /* Kick the guest if necessary. */ + if (!(vr->avail->flags & VRING_AVAIL_F_NO_INTERRUPT) + && (vr->callfd >= 0)) + eventfd_write(vr->callfd, (eventfd_t)1); + return count; +} + +static __rte_always_inline int +dequeue_pkt(struct vhost_dev *dev, struct rte_vhost_vring *vr, + struct rte_mbuf *m, uint16_t desc_idx, + struct rte_mempool *mbuf_pool) +{ + struct vring_desc *desc; + uint64_t desc_addr, desc_gaddr; + uint32_t desc_avail, desc_offset; + uint64_t desc_chunck_len; + uint32_t mbuf_avail, mbuf_offset; + uint32_t cpy_len; + struct rte_mbuf *cur = m, *prev = m; + /* A counter to avoid desc dead loop chain */ + uint32_t nr_desc = 1; + + desc = &vr->desc[desc_idx]; + if (unlikely((desc->len < dev->hdr_len)) || + (desc->flags & VRING_DESC_F_INDIRECT)) + return -1; + + desc_chunck_len = desc->len; + desc_gaddr = desc->addr; + desc_addr = rte_vhost_va_from_guest_pa( + dev->mem, desc_gaddr, &desc_chunck_len); + if (unlikely(!desc_addr)) + return -1; + + /* + * We don't support ANY_LAYOUT, neither VERSION_1, meaning + * a Tx packet from guest must have 2 desc buffers at least: + * the first for storing the header and the others for + * storing the data. + * + * And since we don't support TSO, we could simply skip the + * header. + */ + desc = &vr->desc[desc->next]; + desc_chunck_len = desc->len; + desc_gaddr = desc->addr; + desc_addr = rte_vhost_va_from_guest_pa( + dev->mem, desc_gaddr, &desc_chunck_len); + if (unlikely(!desc_addr)) + return -1; + rte_prefetch0((void *)(uintptr_t)desc_addr); + + desc_offset = 0; + desc_avail = desc->len; + nr_desc += 1; + + mbuf_offset = 0; + mbuf_avail = m->buf_len - RTE_PKTMBUF_HEADROOM; + while (1) { + cpy_len = RTE_MIN(desc_chunck_len, mbuf_avail); + rte_memcpy(rte_pktmbuf_mtod_offset(cur, void *, + mbuf_offset), + (void *)((uintptr_t)(desc_addr + desc_offset)), + cpy_len); + + mbuf_avail -= cpy_len; + mbuf_offset += cpy_len; + desc_avail -= cpy_len; + desc_offset += cpy_len; + desc_chunck_len -= cpy_len; + + /* This desc reaches to its end, get the next one */ + if (desc_avail == 0) { + if ((desc->flags & VRING_DESC_F_NEXT) == 0) + break; + + if (unlikely(desc->next >= vr->size || + ++nr_desc > vr->size)) + return -1; + desc = &vr->desc[desc->next]; + + desc_chunck_len = desc->len; + desc_gaddr = desc->addr; + desc_addr = rte_vhost_va_from_guest_pa( + dev->mem, desc_gaddr, &desc_chunck_len); + if (unlikely(!desc_addr)) + return -1; + rte_prefetch0((void *)(uintptr_t)desc_addr); + + desc_offset = 0; + desc_avail = desc->len; + } else if (unlikely(desc_chunck_len == 0)) { + desc_chunck_len = desc_avail; + desc_gaddr += desc_offset; + desc_addr = rte_vhost_va_from_guest_pa(dev->mem, + desc_gaddr, + &desc_chunck_len); + if (unlikely(!desc_addr)) + return -1; + + desc_offset = 0; + } + + /* + * This mbuf reaches to its end, get a new one + * to hold more data. + */ + if (mbuf_avail == 0) { + cur = rte_pktmbuf_alloc(mbuf_pool); + if (unlikely(cur == NULL)) { + RTE_LOG(ERR, VHOST_DATA, "Failed to " + "allocate memory for mbuf.\n"); + return -1; + } + + prev->next = cur; + prev->data_len = mbuf_offset; + m->nb_segs += 1; + m->pkt_len += mbuf_offset; + prev = cur; + + mbuf_offset = 0; + mbuf_avail = cur->buf_len - RTE_PKTMBUF_HEADROOM; + } + } + + prev->data_len = mbuf_offset; + m->pkt_len += mbuf_offset; + + return 0; +} + +uint16_t +vs_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id, + struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count) +{ + struct vhost_queue *queue; + struct rte_vhost_vring *vr; + uint32_t desc_indexes[MAX_PKT_BURST]; + uint32_t used_idx; + uint32_t i = 0; + uint16_t free_entries; + uint16_t avail_idx; + + queue = &dev->queues[queue_id]; + vr = &queue->vr; + + free_entries = *((volatile uint16_t *)&vr->avail->idx) - + queue->last_avail_idx; + if (free_entries == 0) + return 0; + + /* Prefetch available and used ring */ + avail_idx = queue->last_avail_idx & (vr->size - 1); + used_idx = queue->last_used_idx & (vr->size - 1); + rte_prefetch0(&vr->avail->ring[avail_idx]); + rte_prefetch0(&vr->used->ring[used_idx]); + + count = RTE_MIN(count, MAX_PKT_BURST); + count = RTE_MIN(count, free_entries); + + if (unlikely(count == 0)) + return 0; + + /* + * Retrieve all of the head indexes first and pre-update used entries + * to avoid caching issues. + */ + for (i = 0; i < count; i++) { + avail_idx = (queue->last_avail_idx + i) & (vr->size - 1); + used_idx = (queue->last_used_idx + i) & (vr->size - 1); + desc_indexes[i] = vr->avail->ring[avail_idx]; + + vr->used->ring[used_idx].id = desc_indexes[i]; + vr->used->ring[used_idx].len = 0; + } + + /* Prefetch descriptor index. */ + rte_prefetch0(&vr->desc[desc_indexes[0]]); + for (i = 0; i < count; i++) { + int err; + + if (likely(i + 1 < count)) + rte_prefetch0(&vr->desc[desc_indexes[i + 1]]); + + pkts[i] = rte_pktmbuf_alloc(mbuf_pool); + if (unlikely(pkts[i] == NULL)) { + RTE_LOG(ERR, VHOST_DATA, + "Failed to allocate memory for mbuf.\n"); + break; + } + + err = dequeue_pkt(dev, vr, pkts[i], desc_indexes[i], mbuf_pool); + if (unlikely(err)) { + rte_pktmbuf_free(pkts[i]); + break; + } + + } + + queue->last_avail_idx += i; + queue->last_used_idx += i; + rte_smp_wmb(); + rte_smp_rmb(); + + vr->used->idx += i; + + if (!(vr->avail->flags & VRING_AVAIL_F_NO_INTERRUPT) + && (vr->callfd >= 0)) + eventfd_write(vr->callfd, (eventfd_t)1); + + return i; +} diff --git a/dpdk/examples/vhost_scsi/Makefile b/dpdk/examples/vhost_scsi/Makefile new file mode 100644 index 00000000..0306a6ae --- /dev/null +++ b/dpdk/examples/vhost_scsi/Makefile @@ -0,0 +1,59 @@ +# BSD LICENSE +# +# Copyright(c) 2010-2017 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +ifeq ($(RTE_SDK),) +$(error "Please define RTE_SDK environment variable") +endif + +# Default target, can be overridden by command line or environment +RTE_TARGET ?= x86_64-native-linuxapp-gcc + +include $(RTE_SDK)/mk/rte.vars.mk + +ifneq ($(CONFIG_RTE_EXEC_ENV),"linuxapp") +$(info This application can only operate in a linuxapp environment, \ +please change the definition of the RTE_TARGET environment variable) +all: +else + +# binary name +APP = vhost-scsi + +# all source are stored in SRCS-y +SRCS-y := scsi.c vhost_scsi.c + +CFLAGS += -O2 -D_FILE_OFFSET_BITS=64 +CFLAGS += $(WERROR_FLAGS) +CFLAGS += -D_GNU_SOURCE + +include $(RTE_SDK)/mk/rte.extapp.mk + +endif diff --git a/dpdk/examples/vhost_scsi/scsi.c b/dpdk/examples/vhost_scsi/scsi.c new file mode 100644 index 00000000..fd430ec2 --- /dev/null +++ b/dpdk/examples/vhost_scsi/scsi.c @@ -0,0 +1,541 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** + * This work is largely based on the "vhost-user-scsi" implementation by + * SPDK(https://github.com/spdk/spdk). + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "vhost_scsi.h" +#include "scsi_spec.h" + +#define INQ_OFFSET(field) (offsetof(struct scsi_cdb_inquiry_data, field) + \ + sizeof(((struct scsi_cdb_inquiry_data *)0x0)->field)) + +static void +vhost_strcpy_pad(void *dst, const char *src, size_t size, int pad) +{ + size_t len; + + len = strlen(src); + if (len < size) { + memcpy(dst, src, len); + memset((char *)dst + len, pad, size - len); + } else { + memcpy(dst, src, size); + } +} + +static int +vhost_hex2bin(char ch) +{ + if ((ch >= '0') && (ch <= '9')) + return ch - '0'; + ch = tolower(ch); + if ((ch >= 'a') && (ch <= 'f')) + return ch - 'a' + 10; + return (int)ch; +} + +static void +vhost_bdev_scsi_set_naa_ieee_extended(const char *name, uint8_t *buf) +{ + int i, value, count = 0; + uint64_t *temp64, local_value; + + for (i = 0; (i < 16) && (name[i] != '\0'); i++) { + value = vhost_hex2bin(name[i]); + if (i % 2) + buf[count++] |= value << 4; + else + buf[count] = value; + } + + local_value = *(uint64_t *)buf; + /* + * see spc3r23 7.6.3.6.2, + * NAA IEEE Extended identifer format + */ + local_value &= 0x0fff000000ffffffull; + /* NAA 02, and 00 03 47 for IEEE Intel */ + local_value |= 0x2000000347000000ull; + + temp64 = (uint64_t *)buf; + *temp64 = rte_cpu_to_be_64(local_value); +} + +static void +scsi_task_build_sense_data(struct vhost_scsi_task *task, int sk, + int asc, int ascq) +{ + uint8_t *cp; + int resp_code; + + resp_code = 0x70; /* Current + Fixed format */ + + /* Sense Data */ + cp = (uint8_t *)task->resp->sense; + + /* VALID(7) RESPONSE CODE(6-0) */ + cp[0] = 0x80 | resp_code; + /* Obsolete */ + cp[1] = 0; + /* FILEMARK(7) EOM(6) ILI(5) SENSE KEY(3-0) */ + cp[2] = sk & 0xf; + /* INFORMATION */ + memset(&cp[3], 0, 4); + + /* ADDITIONAL SENSE LENGTH */ + cp[7] = 10; + + /* COMMAND-SPECIFIC INFORMATION */ + memset(&cp[8], 0, 4); + /* ADDITIONAL SENSE CODE */ + cp[12] = asc; + /* ADDITIONAL SENSE CODE QUALIFIER */ + cp[13] = ascq; + /* FIELD REPLACEABLE UNIT CODE */ + cp[14] = 0; + + /* SKSV(7) SENSE KEY SPECIFIC(6-0,7-0,7-0) */ + cp[15] = 0; + cp[16] = 0; + cp[17] = 0; + + /* SenseLength */ + task->resp->sense_len = 18; +} + +static void +scsi_task_set_status(struct vhost_scsi_task *task, int sc, int sk, + int asc, int ascq) +{ + if (sc == SCSI_STATUS_CHECK_CONDITION) + scsi_task_build_sense_data(task, sk, asc, ascq); + task->resp->status = sc; +} + +static int +vhost_bdev_scsi_inquiry_command(struct vhost_block_dev *bdev, + struct vhost_scsi_task *task) +{ + int hlen = 0; + uint32_t alloc_len = 0; + uint16_t len = 0; + uint16_t *temp16; + int pc; + int pd; + int evpd; + int i; + uint8_t *buf; + struct scsi_cdb_inquiry *inq; + + inq = (struct scsi_cdb_inquiry *)task->req->cdb; + + assert(task->iovs_cnt == 1); + + /* At least 36Bytes for inquiry command */ + if (task->data_len < 0x24) + goto inq_error; + + pd = SPC_PERIPHERAL_DEVICE_TYPE_DISK; + pc = inq->page_code; + evpd = inq->evpd & 0x1; + + if (!evpd && pc) + goto inq_error; + + if (evpd) { + struct scsi_vpd_page *vpage = (struct scsi_vpd_page *) + task->iovs[0].iov_base; + + /* PERIPHERAL QUALIFIER(7-5) PERIPHERAL DEVICE TYPE(4-0) */ + vpage->peripheral = pd; + /* PAGE CODE */ + vpage->page_code = pc; + + switch (pc) { + case SPC_VPD_SUPPORTED_VPD_PAGES: + hlen = 4; + vpage->params[0] = SPC_VPD_SUPPORTED_VPD_PAGES; + vpage->params[1] = SPC_VPD_UNIT_SERIAL_NUMBER; + vpage->params[2] = SPC_VPD_DEVICE_IDENTIFICATION; + len = 3; + /* PAGE LENGTH */ + vpage->alloc_len = rte_cpu_to_be_16(len); + break; + case SPC_VPD_UNIT_SERIAL_NUMBER: + hlen = 4; + strncpy((char *)vpage->params, bdev->name, 32); + vpage->alloc_len = rte_cpu_to_be_16(32); + break; + case SPC_VPD_DEVICE_IDENTIFICATION: + buf = vpage->params; + struct scsi_desig_desc *desig; + + hlen = 4; + /* NAA designator */ + desig = (struct scsi_desig_desc *)buf; + desig->code_set = SPC_VPD_CODE_SET_BINARY; + desig->protocol_id = SPC_PROTOCOL_IDENTIFIER_ISCSI; + desig->type = SPC_VPD_IDENTIFIER_TYPE_NAA; + desig->association = SPC_VPD_ASSOCIATION_LOGICAL_UNIT; + desig->reserved0 = 0; + desig->piv = 1; + desig->reserved1 = 0; + desig->len = 8; + vhost_bdev_scsi_set_naa_ieee_extended(bdev->name, + desig->desig); + len = sizeof(struct scsi_desig_desc) + 8; + + buf += sizeof(struct scsi_desig_desc) + desig->len; + + /* T10 Vendor ID designator */ + desig = (struct scsi_desig_desc *)buf; + desig->code_set = SPC_VPD_CODE_SET_ASCII; + desig->protocol_id = SPC_PROTOCOL_IDENTIFIER_ISCSI; + desig->type = SPC_VPD_IDENTIFIER_TYPE_T10_VENDOR_ID; + desig->association = SPC_VPD_ASSOCIATION_LOGICAL_UNIT; + desig->reserved0 = 0; + desig->piv = 1; + desig->reserved1 = 0; + desig->len = 8 + 16 + 32; + strncpy((char *)desig->desig, "INTEL", 8); + vhost_strcpy_pad((char *)&desig->desig[8], + bdev->product_name, 16, ' '); + strncpy((char *)&desig->desig[24], bdev->name, 32); + len += sizeof(struct scsi_desig_desc) + 8 + 16 + 32; + + buf += sizeof(struct scsi_desig_desc) + desig->len; + + /* SCSI Device Name designator */ + desig = (struct scsi_desig_desc *)buf; + desig->code_set = SPC_VPD_CODE_SET_UTF8; + desig->protocol_id = SPC_PROTOCOL_IDENTIFIER_ISCSI; + desig->type = SPC_VPD_IDENTIFIER_TYPE_SCSI_NAME; + desig->association = SPC_VPD_ASSOCIATION_TARGET_DEVICE; + desig->reserved0 = 0; + desig->piv = 1; + desig->reserved1 = 0; + desig->len = snprintf((char *)desig->desig, + 255, "%s", bdev->name); + len += sizeof(struct scsi_desig_desc) + desig->len; + + buf += sizeof(struct scsi_desig_desc) + desig->len; + vpage->alloc_len = rte_cpu_to_be_16(len); + break; + default: + goto inq_error; + } + + } else { + struct scsi_cdb_inquiry_data *inqdata = + (struct scsi_cdb_inquiry_data *)task->iovs[0].iov_base; + /* Standard INQUIRY data */ + /* PERIPHERAL QUALIFIER(7-5) PERIPHERAL DEVICE TYPE(4-0) */ + inqdata->peripheral = pd; + + /* RMB(7) */ + inqdata->rmb = 0; + + /* VERSION */ + /* See SPC3/SBC2/MMC4/SAM2 for more details */ + inqdata->version = SPC_VERSION_SPC3; + + /* NORMACA(5) HISUP(4) RESPONSE DATA FORMAT(3-0) */ + /* format 2 */ /* hierarchical support */ + inqdata->response = 2 | 1 << 4; + + hlen = 5; + + /* SCCS(7) ACC(6) TPGS(5-4) 3PC(3) PROTECT(0) */ + /* Not support TPGS */ + inqdata->flags = 0; + + /* MULTIP */ + inqdata->flags2 = 0x10; + + /* WBUS16(5) SYNC(4) LINKED(3) CMDQUE(1) VS(0) */ + /* CMDQUE */ + inqdata->flags3 = 0x2; + + /* T10 VENDOR IDENTIFICATION */ + strncpy((char *)inqdata->t10_vendor_id, "INTEL", 8); + + /* PRODUCT IDENTIFICATION */ + snprintf((char *)inqdata->product_id, + RTE_DIM(inqdata->product_id), "%s", + bdev->product_name); + + /* PRODUCT REVISION LEVEL */ + strncpy((char *)inqdata->product_rev, "0001", 4); + + /* Standard inquiry data ends here. Only populate + * remaining fields if alloc_len indicates enough + * space to hold it. + */ + len = INQ_OFFSET(product_rev) - 5; + + if (alloc_len >= INQ_OFFSET(vendor)) { + /* Vendor specific */ + memset(inqdata->vendor, 0x20, 20); + len += sizeof(inqdata->vendor); + } + + if (alloc_len >= INQ_OFFSET(ius)) { + /* CLOCKING(3-2) QAS(1) IUS(0) */ + inqdata->ius = 0; + len += sizeof(inqdata->ius); + } + + if (alloc_len >= INQ_OFFSET(reserved)) { + /* Reserved */ + inqdata->reserved = 0; + len += sizeof(inqdata->reserved); + } + + /* VERSION DESCRIPTOR 1-8 */ + if (alloc_len >= INQ_OFFSET(reserved) + 2) { + temp16 = (uint16_t *)&inqdata->desc[0]; + *temp16 = rte_cpu_to_be_16(0x0960); + len += 2; + } + + if (alloc_len >= INQ_OFFSET(reserved) + 4) { + /* SPC-3 (no version claimed) */ + temp16 = (uint16_t *)&inqdata->desc[2]; + *temp16 = rte_cpu_to_be_16(0x0300); + len += 2; + } + + if (alloc_len >= INQ_OFFSET(reserved) + 6) { + /* SBC-2 (no version claimed) */ + temp16 = (uint16_t *)&inqdata->desc[4]; + *temp16 = rte_cpu_to_be_16(0x0320); + len += 2; + } + + if (alloc_len >= INQ_OFFSET(reserved) + 8) { + /* SAM-2 (no version claimed) */ + temp16 = (uint16_t *)&inqdata->desc[6]; + *temp16 = rte_cpu_to_be_16(0x0040); + len += 2; + } + + if (alloc_len > INQ_OFFSET(reserved) + 8) { + i = alloc_len - (INQ_OFFSET(reserved) + 8); + if (i > 30) + i = 30; + memset(&inqdata->desc[8], 0, i); + len += i; + } + + /* ADDITIONAL LENGTH */ + inqdata->add_len = len; + } + + /* STATUS GOOD */ + scsi_task_set_status(task, SCSI_STATUS_GOOD, 0, 0, 0); + return hlen + len; + +inq_error: + scsi_task_set_status(task, SCSI_STATUS_CHECK_CONDITION, + SCSI_SENSE_ILLEGAL_REQUEST, + SCSI_ASC_INVALID_FIELD_IN_CDB, + SCSI_ASCQ_CAUSE_NOT_REPORTABLE); + return 0; +} + +static int +vhost_bdev_scsi_readwrite(struct vhost_block_dev *bdev, + struct vhost_scsi_task *task, + uint64_t lba, __rte_unused uint32_t xfer_len) +{ + uint32_t i; + uint64_t offset; + uint32_t nbytes = 0; + + offset = lba * bdev->blocklen; + + for (i = 0; i < task->iovs_cnt; i++) { + if (task->dxfer_dir == SCSI_DIR_TO_DEV) + memcpy(bdev->data + offset, task->iovs[i].iov_base, + task->iovs[i].iov_len); + else + memcpy(task->iovs[i].iov_base, bdev->data + offset, + task->iovs[i].iov_len); + offset += task->iovs[i].iov_len; + nbytes += task->iovs[i].iov_len; + } + + return nbytes; +} + +static int +vhost_bdev_scsi_process_block(struct vhost_block_dev *bdev, + struct vhost_scsi_task *task) +{ + uint64_t lba, *temp64; + uint32_t xfer_len, *temp32; + uint16_t *temp16; + uint8_t *cdb = (uint8_t *)task->req->cdb; + + switch (cdb[0]) { + case SBC_READ_6: + case SBC_WRITE_6: + lba = (uint64_t)cdb[1] << 16; + lba |= (uint64_t)cdb[2] << 8; + lba |= (uint64_t)cdb[3]; + xfer_len = cdb[4]; + if (xfer_len == 0) + xfer_len = 256; + return vhost_bdev_scsi_readwrite(bdev, task, lba, xfer_len); + + case SBC_READ_10: + case SBC_WRITE_10: + temp32 = (uint32_t *)&cdb[2]; + lba = rte_be_to_cpu_32(*temp32); + temp16 = (uint16_t *)&cdb[7]; + xfer_len = rte_be_to_cpu_16(*temp16); + return vhost_bdev_scsi_readwrite(bdev, task, lba, xfer_len); + + case SBC_READ_12: + case SBC_WRITE_12: + temp32 = (uint32_t *)&cdb[2]; + lba = rte_be_to_cpu_32(*temp32); + temp32 = (uint32_t *)&cdb[6]; + xfer_len = rte_be_to_cpu_32(*temp32); + return vhost_bdev_scsi_readwrite(bdev, task, lba, xfer_len); + + case SBC_READ_16: + case SBC_WRITE_16: + temp64 = (uint64_t *)&cdb[2]; + lba = rte_be_to_cpu_64(*temp64); + temp32 = (uint32_t *)&cdb[10]; + xfer_len = rte_be_to_cpu_32(*temp32); + return vhost_bdev_scsi_readwrite(bdev, task, lba, xfer_len); + + case SBC_READ_CAPACITY_10: { + uint8_t buffer[8]; + + if (bdev->blockcnt - 1 > 0xffffffffULL) + memset(buffer, 0xff, 4); + else { + temp32 = (uint32_t *)buffer; + *temp32 = rte_cpu_to_be_32(bdev->blockcnt - 1); + } + temp32 = (uint32_t *)&buffer[4]; + *temp32 = rte_cpu_to_be_32(bdev->blocklen); + memcpy(task->iovs[0].iov_base, buffer, sizeof(buffer)); + task->resp->status = SCSI_STATUS_GOOD; + return sizeof(buffer); + } + + case SBC_SYNCHRONIZE_CACHE_10: + case SBC_SYNCHRONIZE_CACHE_16: + task->resp->status = SCSI_STATUS_GOOD; + return 0; + } + + scsi_task_set_status(task, SCSI_STATUS_CHECK_CONDITION, + SCSI_SENSE_ILLEGAL_REQUEST, + SCSI_ASC_INVALID_FIELD_IN_CDB, + SCSI_ASCQ_CAUSE_NOT_REPORTABLE); + return 0; +} + +int +vhost_bdev_process_scsi_commands(struct vhost_block_dev *bdev, + struct vhost_scsi_task *task) +{ + int len; + uint8_t *data; + uint64_t *temp64, fmt_lun = 0; + uint32_t *temp32; + const uint8_t *lun; + uint8_t *cdb = (uint8_t *)task->req->cdb; + + lun = (const uint8_t *)task->req->lun; + /* only 1 LUN supported */ + if (lun[0] != 1 || lun[1] >= 1) + return -1; + + switch (cdb[0]) { + case SPC_INQUIRY: + len = vhost_bdev_scsi_inquiry_command(bdev, task); + task->data_len = len; + break; + case SPC_REPORT_LUNS: + data = (uint8_t *)task->iovs[0].iov_base; + fmt_lun |= (0x0ULL & 0x00ffULL) << 48; + temp64 = (uint64_t *)&data[8]; + *temp64 = rte_cpu_to_be_64(fmt_lun); + temp32 = (uint32_t *)data; + *temp32 = rte_cpu_to_be_32(8); + task->data_len = 16; + scsi_task_set_status(task, SCSI_STATUS_GOOD, 0, 0, 0); + break; + case SPC_MODE_SELECT_6: + case SPC_MODE_SELECT_10: + /* don't support it now */ + scsi_task_set_status(task, SCSI_STATUS_GOOD, 0, 0, 0); + break; + case SPC_MODE_SENSE_6: + case SPC_MODE_SENSE_10: + /* don't support it now */ + scsi_task_set_status(task, SCSI_STATUS_GOOD, 0, 0, 0); + break; + case SPC_TEST_UNIT_READY: + scsi_task_set_status(task, SCSI_STATUS_GOOD, 0, 0, 0); + break; + default: + len = vhost_bdev_scsi_process_block(bdev, task); + task->data_len = len; + } + + return 0; +} diff --git a/dpdk/examples/vhost_scsi/scsi_spec.h b/dpdk/examples/vhost_scsi/scsi_spec.h new file mode 100644 index 00000000..60d761cb --- /dev/null +++ b/dpdk/examples/vhost_scsi/scsi_spec.h @@ -0,0 +1,493 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** + * SCSI specification definition + * refer http://www.t10.org/drafts.htm#SPC_Family for SPC-3 and SBC-3 + */ + +#ifndef _SCSI_SPEC_H +#define _SCSI_SPEC_H + +#include + +enum scsi_group_code { + SCSI_6BYTE_CMD = 0x00, + SCSI_10BYTE_CMD = 0x20, + SCSI_10BYTE_CMD2 = 0x40, + SCSI_16BYTE_CMD = 0x80, + SCSI_12BYTE_CMD = 0xa0, +}; + +#define SCSI_GROUP_MASK 0xe0 +#define SCSI_OPCODE_MASK 0x1f + +enum scsi_status { + SCSI_STATUS_GOOD = 0x00, + SCSI_STATUS_CHECK_CONDITION = 0x02, + SCSI_STATUS_CONDITION_MET = 0x04, + SCSI_STATUS_BUSY = 0x08, + SCSI_STATUS_INTERMEDIATE = 0x10, + SCSI_STATUS_INTERMEDIATE_CONDITION_MET = 0x14, + SCSI_STATUS_RESERVATION_CONFLICT = 0x18, + SCSI_STATUS_Obsolete = 0x22, + SCSI_STATUS_TASK_SET_FULL = 0x28, + SCSI_STATUS_ACA_ACTIVE = 0x30, + SCSI_STATUS_TASK_ABORTED = 0x40, +}; + +enum scsi_sense { + SCSI_SENSE_NO_SENSE = 0x00, + SCSI_SENSE_RECOVERED_ERROR = 0x01, + SCSI_SENSE_NOT_READY = 0x02, + SCSI_SENSE_MEDIUM_ERROR = 0x03, + SCSI_SENSE_HARDWARE_ERROR = 0x04, + SCSI_SENSE_ILLEGAL_REQUEST = 0x05, + SCSI_SENSE_UNIT_ATTENTION = 0x06, + SCSI_SENSE_DATA_PROTECT = 0x07, + SCSI_SENSE_BLANK_CHECK = 0x08, + SCSI_SENSE_VENDOR_SPECIFIC = 0x09, + SCSI_SENSE_COPY_ABORTED = 0x0a, + SCSI_SENSE_ABORTED_COMMAND = 0x0b, + SCSI_SENSE_VOLUME_OVERFLOW = 0x0d, + SCSI_SENSE_MISCOMPARE = 0x0e, +}; + +enum scsi_asc { + SCSI_ASC_NO_ADDITIONAL_SENSE = 0x00, + SCSI_ASC_PERIPHERAL_DEVICE_WRITE_FAULT = 0x03, + SCSI_ASC_LOGICAL_UNIT_NOT_READY = 0x04, + SCSI_ASC_WARNING = 0x0b, + SCSI_ASC_LOGICAL_BLOCK_GUARD_CHECK_FAILED = 0x10, + SCSI_ASC_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED = 0x10, + SCSI_ASC_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED = 0x10, + SCSI_ASC_UNRECOVERED_READ_ERROR = 0x11, + SCSI_ASC_MISCOMPARE_DURING_VERIFY_OPERATION = 0x1d, + SCSI_ASC_INVALID_COMMAND_OPERATION_CODE = 0x20, + SCSI_ASC_ACCESS_DENIED = 0x20, + SCSI_ASC_LOGICAL_BLOCK_ADDRESS_OUT_OF_RANGE = 0x21, + SCSI_ASC_INVALID_FIELD_IN_CDB = 0x24, + SCSI_ASC_LOGICAL_UNIT_NOT_SUPPORTED = 0x25, + SCSI_ASC_WRITE_PROTECTED = 0x27, + SCSI_ASC_FORMAT_COMMAND_FAILED = 0x31, + SCSI_ASC_INTERNAL_TARGET_FAILURE = 0x44, +}; + +enum scsi_ascq { + SCSI_ASCQ_CAUSE_NOT_REPORTABLE = 0x00, + SCSI_ASCQ_BECOMING_READY = 0x01, + SCSI_ASCQ_FORMAT_COMMAND_FAILED = 0x01, + SCSI_ASCQ_LOGICAL_BLOCK_GUARD_CHECK_FAILED = 0x01, + SCSI_ASCQ_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED = 0x02, + SCSI_ASCQ_NO_ACCESS_RIGHTS = 0x02, + SCSI_ASCQ_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED = 0x03, + SCSI_ASCQ_POWER_LOSS_EXPECTED = 0x08, + SCSI_ASCQ_INVALID_LU_IDENTIFIER = 0x09, +}; + +enum spc_opcode { + /* SPC3 related */ + SPC_ACCESS_CONTROL_IN = 0x86, + SPC_ACCESS_CONTROL_OUT = 0x87, + SPC_EXTENDED_COPY = 0x83, + SPC_INQUIRY = 0x12, + SPC_LOG_SELECT = 0x4c, + SPC_LOG_SENSE = 0x4d, + SPC_MODE_SELECT_6 = 0x15, + SPC_MODE_SELECT_10 = 0x55, + SPC_MODE_SENSE_6 = 0x1a, + SPC_MODE_SENSE_10 = 0x5a, + SPC_PERSISTENT_RESERVE_IN = 0x5e, + SPC_PERSISTENT_RESERVE_OUT = 0x5f, + SPC_PREVENT_ALLOW_MEDIUM_REMOVAL = 0x1e, + SPC_READ_ATTRIBUTE = 0x8c, + SPC_READ_BUFFER = 0x3c, + SPC_RECEIVE_COPY_RESULTS = 0x84, + SPC_RECEIVE_DIAGNOSTIC_RESULTS = 0x1c, + SPC_REPORT_LUNS = 0xa0, + SPC_REQUEST_SENSE = 0x03, + SPC_SEND_DIAGNOSTIC = 0x1d, + SPC_TEST_UNIT_READY = 0x00, + SPC_WRITE_ATTRIBUTE = 0x8d, + SPC_WRITE_BUFFER = 0x3b, + + SPC_SERVICE_ACTION_IN_12 = 0xab, + SPC_SERVICE_ACTION_OUT_12 = 0xa9, + SPC_SERVICE_ACTION_IN_16 = 0x9e, + SPC_SERVICE_ACTION_OUT_16 = 0x9f, + + SPC_VARIABLE_LENGTH = 0x7f, + + SPC_MO_CHANGE_ALIASES = 0x0b, + SPC_MO_SET_DEVICE_IDENTIFIER = 0x06, + SPC_MO_SET_PRIORITY = 0x0e, + SPC_MO_SET_TARGET_PORT_GROUPS = 0x0a, + SPC_MO_SET_TIMESTAMP = 0x0f, + SPC_MI_REPORT_ALIASES = 0x0b, + SPC_MI_REPORT_DEVICE_IDENTIFIER = 0x05, + SPC_MI_REPORT_PRIORITY = 0x0e, + SPC_MI_REPORT_SUPPORTED_OPERATION_CODES = 0x0c, + SPC_MI_REPORT_SUPPORTED_TASK_MANAGEMENT_FUNCTIONS = 0x0d, + SPC_MI_REPORT_TARGET_PORT_GROUPS = 0x0a, + SPC_MI_REPORT_TIMESTAMP = 0x0f, + + /* SPC2 related (Obsolete) */ + SPC2_RELEASE_6 = 0x17, + SPC2_RELEASE_10 = 0x57, + SPC2_RESERVE_6 = 0x16, + SPC2_RESERVE_10 = 0x56, +}; + +enum scc_opcode { + SCC_MAINTENANCE_IN = 0xa3, + SCC_MAINTENANCE_OUT = 0xa4, +}; + +enum sbc_opcode { + SBC_COMPARE_AND_WRITE = 0x89, + SBC_FORMAT_UNIT = 0x04, + SBC_GET_LBA_STATUS = 0x0012009e, + SBC_ORWRITE_16 = 0x8b, + SBC_PRE_FETCH_10 = 0x34, + SBC_PRE_FETCH_16 = 0x90, + SBC_READ_6 = 0x08, + SBC_READ_10 = 0x28, + SBC_READ_12 = 0xa8, + SBC_READ_16 = 0x88, + SBC_READ_ATTRIBUTE = 0x8c, + SBC_READ_BUFFER = 0x3c, + SBC_READ_CAPACITY_10 = 0x25, + SBC_READ_DEFECT_DATA_10 = 0x37, + SBC_READ_DEFECT_DATA_12 = 0xb7, + SBC_READ_LONG_10 = 0x3e, + SBC_REASSIGN_BLOCKS = 0x07, + SBC_SANITIZE = 0x48, + SBC_START_STOP_UNIT = 0x1b, + SBC_SYNCHRONIZE_CACHE_10 = 0x35, + SBC_SYNCHRONIZE_CACHE_16 = 0x91, + SBC_UNMAP = 0x42, + SBC_VERIFY_10 = 0x2f, + SBC_VERIFY_12 = 0xaf, + SBC_VERIFY_16 = 0x8f, + SBC_WRITE_6 = 0x0a, + SBC_WRITE_10 = 0x2a, + SBC_WRITE_12 = 0xaa, + SBC_WRITE_16 = 0x8a, + SBC_WRITE_AND_VERIFY_10 = 0x2e, + SBC_WRITE_AND_VERIFY_12 = 0xae, + SBC_WRITE_AND_VERIFY_16 = 0x8e, + SBC_WRITE_LONG_10 = 0x3f, + SBC_WRITE_SAME_10 = 0x41, + SBC_WRITE_SAME_16 = 0x93, + SBC_XDREAD_10 = 0x52, + SBC_XDWRITE_10 = 0x50, + SBC_XDWRITEREAD_10 = 0x53, + SBC_XPWRITE_10 = 0x51, + + SBC_SAI_READ_CAPACITY_16 = 0x10, + SBC_SAI_READ_LONG_16 = 0x11, + SBC_SAO_WRITE_LONG_16 = 0x11, + + SBC_VL_READ_32 = 0x0009, + SBC_VL_VERIFY_32 = 0x000a, + SBC_VL_WRITE_32 = 0x000b, + SBC_VL_WRITE_AND_VERIFY_32 = 0x000c, + SBC_VL_WRITE_SAME_32 = 0x000d, + SBC_VL_XDREAD_32 = 0x0003, + SBC_VL_XDWRITE_32 = 0x0004, + SBC_VL_XDWRITEREAD_32 = 0x0007, + SBC_VL_XPWRITE_32 = 0x0006, +}; + +enum mmc_opcode { + /* MMC6 */ + MMC_READ_DISC_STRUCTURE = 0xad, + + /* MMC4 */ + MMC_BLANK = 0xa1, + MMC_CLOSE_TRACK_SESSION = 0x5b, + MMC_ERASE_10 = 0x2c, + MMC_FORMAT_UNIT = 0x04, + MMC_GET_CONFIGURATION = 0x46, + MMC_GET_EVENT_STATUS_NOTIFICATION = 0x4a, + MMC_GET_PERFORMANCE = 0xac, + MMC_INQUIRY = 0x12, + MMC_LOAD_UNLOAD_MEDIUM = 0xa6, + MMC_MECHANISM_STATUS = 0xbd, + MMC_MODE_SELECT_10 = 0x55, + MMC_MODE_SENSE_10 = 0x5a, + MMC_PAUSE_RESUME = 0x4b, + MMC_PLAY_AUDIO_10 = 0x45, + MMC_PLAY_AUDIO_12 = 0xa5, + MMC_PLAY_AUDIO_MSF = 0x47, + MMC_PREVENT_ALLOW_MEDIUM_REMOVAL = 0x1e, + MMC_READ_10 = 0x28, + MMC_READ_12 = 0xa8, + MMC_READ_BUFFER = 0x3c, + MMC_READ_BUFFER_CAPACITY = 0x5c, + MMC_READ_CAPACITY = 0x25, + MMC_READ_CD = 0xbe, + MMC_READ_CD_MSF = 0xb9, + MMC_READ_DISC_INFORMATION = 0x51, + MMC_READ_DVD_STRUCTURE = 0xad, + MMC_READ_FORMAT_CAPACITIES = 0x23, + MMC_READ_SUB_CHANNEL = 0x42, + MMC_READ_TOC_PMA_ATIP = 0x43, + MMC_READ_TRACK_INFORMATION = 0x52, + MMC_REPAIR_TRACK = 0x58, + MMC_REPORT_KEY = 0xa4, + MMC_REQUEST_SENSE = 0x03, + MMC_RESERVE_TRACK = 0x53, + MMC_SCAN = 0xba, + MMC_SEEK_10 = 0x2b, + MMC_SEND_CUE_SHEET = 0x5d, + MMC_SEND_DVD_STRUCTURE = 0xbf, + MMC_SEND_KEY = 0xa3, + MMC_SEND_OPC_INFORMATION = 0x54, + MMC_SET_CD_SPEED = 0xbb, + MMC_SET_READ_AHEAD = 0xa7, + MMC_SET_STREAMING = 0xb6, + MMC_START_STOP_UNIT = 0x1b, + MMC_STOP_PLAY_SCAN = 0x4e, + MMC_SYNCHRONIZE_CACHE = 0x35, + MMC_TEST_UNIT_READY = 0x00, + MMC_VERIFY_10 = 0x2f, + MMC_WRITE_10 = 0xa2, + MMC_WRITE_12 = 0xaa, + MMC_WRITE_AND_VERIFY_10 = 0x2e, + MMC_WRITE_BUFFER = 0x3b, +}; + +enum ssc_opcode { + SSC_ERASE_6 = 0x19, + SSC_FORMAT_MEDIUM = 0x04, + SSC_LOAD_UNLOAD = 0x1b, + SSC_LOCATE_10 = 0x2b, + SSC_LOCATE_16 = 0x92, + SSC_MOVE_MEDIUM_ATTACHED = 0xa7, + SSC_READ_6 = 0x08, + SSC_READ_BLOCK_LIMITS = 0x05, + SSC_READ_ELEMENT_STATUS_ATTACHED = 0xb4, + SSC_READ_POSITION = 0x34, + SSC_READ_REVERSE_6 = 0x0f, + SSC_RECOVER_BUFFERED_DATA = 0x14, + SSC_REPORT_DENSITY_SUPPORT = 0x44, + SSC_REWIND = 0x01, + SSC_SET_CAPACITY = 0x0b, + SSC_SPACE_6 = 0x11, + SSC_SPACE_16 = 0x91, + SSC_VERIFY_6 = 0x13, + SSC_WRITE_6 = 0x0a, + SSC_WRITE_FILEMARKS_6 = 0x10, +}; + +enum spc_vpd { + SPC_VPD_DEVICE_IDENTIFICATION = 0x83, + SPC_VPD_EXTENDED_INQUIRY_DATA = 0x86, + SPC_VPD_MANAGEMENT_NETWORK_ADDRESSES = 0x85, + SPC_VPD_MODE_PAGE_POLICY = 0x87, + SPC_VPD_SCSI_PORTS = 0x88, + SPC_VPD_SOFTWARE_INTERFACE_IDENTIFICATION = 0x84, + SPC_VPD_SUPPORTED_VPD_PAGES = 0x00, + SPC_VPD_UNIT_SERIAL_NUMBER = 0x80, + SPC_VPD_BLOCK_LIMITS = 0xb0, + SPC_VPD_BLOCK_DEV_CHARS = 0xb1, + SPC_VPD_BLOCK_THIN_PROVISION = 0xb2, +}; + +enum { + SPC_PERIPHERAL_DEVICE_TYPE_DISK = 0x00, + SPC_PERIPHERAL_DEVICE_TYPE_TAPE = 0x01, + SPC_PERIPHERAL_DEVICE_TYPE_DVD = 0x05, + SPC_PERIPHERAL_DEVICE_TYPE_CHANGER = 0x08, + + SPC_VERSION_NONE = 0x00, + SPC_VERSION_SPC = 0x03, + SPC_VERSION_SPC2 = 0x04, + SPC_VERSION_SPC3 = 0x05, + SPC_VERSION_SPC4 = 0x06, + + SPC_PROTOCOL_IDENTIFIER_FC = 0x00, + SPC_PROTOCOL_IDENTIFIER_PSCSI = 0x01, + SPC_PROTOCOL_IDENTIFIER_SSA = 0x02, + SPC_PROTOCOL_IDENTIFIER_IEEE1394 = 0x03, + SPC_PROTOCOL_IDENTIFIER_RDMA = 0x04, + SPC_PROTOCOL_IDENTIFIER_ISCSI = 0x05, + SPC_PROTOCOL_IDENTIFIER_SAS = 0x06, + SPC_PROTOCOL_IDENTIFIER_ADT = 0x07, + SPC_PROTOCOL_IDENTIFIER_ATA = 0x08, + + SPC_VPD_CODE_SET_BINARY = 0x01, + SPC_VPD_CODE_SET_ASCII = 0x02, + SPC_VPD_CODE_SET_UTF8 = 0x03, + + SPC_VPD_ASSOCIATION_LOGICAL_UNIT = 0x00, + SPC_VPD_ASSOCIATION_TARGET_PORT = 0x01, + SPC_VPD_ASSOCIATION_TARGET_DEVICE = 0x02, + + SPC_VPD_IDENTIFIER_TYPE_VENDOR_SPECIFIC = 0x00, + SPC_VPD_IDENTIFIER_TYPE_T10_VENDOR_ID = 0x01, + SPC_VPD_IDENTIFIER_TYPE_EUI64 = 0x02, + SPC_VPD_IDENTIFIER_TYPE_NAA = 0x03, + SPC_VPD_IDENTIFIER_TYPE_RELATIVE_TARGET_PORT = 0x04, + SPC_VPD_IDENTIFIER_TYPE_TARGET_PORT_GROUP = 0x05, + SPC_VPD_IDENTIFIER_TYPE_LOGICAL_UNIT_GROUP = 0x06, + SPC_VPD_IDENTIFIER_TYPE_MD5_LOGICAL_UNIT = 0x07, + SPC_VPD_IDENTIFIER_TYPE_SCSI_NAME = 0x08, +}; + +struct scsi_cdb_inquiry { + uint8_t opcode; + uint8_t evpd; + uint8_t page_code; + uint16_t alloc_len; + uint8_t control; +}; + +struct scsi_cdb_inquiry_data { + uint8_t peripheral; + uint8_t rmb; + uint8_t version; + uint8_t response; + uint8_t add_len; + uint8_t flags; + uint8_t flags2; + uint8_t flags3; + uint8_t t10_vendor_id[8]; + uint8_t product_id[16]; + uint8_t product_rev[4]; + uint8_t vendor[20]; + uint8_t ius; + uint8_t reserved; + uint8_t desc[]; +}; + +struct scsi_vpd_page { + uint8_t peripheral; + uint8_t page_code; + uint16_t alloc_len; + uint8_t params[]; +}; + +#define SCSI_VEXT_REF_CHK 0x01 +#define SCSI_VEXT_APP_CHK 0x02 +#define SCSI_VEXT_GRD_CHK 0x04 +#define SCSI_VEXT_SIMPSUP 0x01 +#define SCSI_VEXT_ORDSUP 0x02 +#define SCSI_VEXT_HEADSUP 0x04 +#define SCSI_VEXT_PRIOR_SUP 0x08 +#define SCSI_VEXT_GROUP_SUP 0x10 +#define SCSI_VEXT_UASK_SUP 0x20 +#define SCSI_VEXT_V_SUP 0x01 +#define SCSI_VEXT_NV_SUP 0x02 +#define SCSI_VEXT_CRD_SUP 0x04 +#define SCSI_VEXT_WU_SUP 0x08 + +struct scsi_vpd_ext_inquiry { + uint8_t peripheral; + uint8_t page_code; + uint16_t alloc_len; + uint8_t check; + uint8_t sup; + uint8_t sup2; + uint8_t luiclr; + uint8_t cbcs; + uint8_t micro_dl; + uint8_t reserved[54]; +}; + +#define SPC_VPD_DESIG_PIV 0x80 + +/* designation descriptor */ +struct scsi_desig_desc { + uint8_t code_set : 4; + uint8_t protocol_id : 4; + uint8_t type : 4; + uint8_t association : 2; + uint8_t reserved0 : 1; + uint8_t piv : 1; + uint8_t reserved1; + uint8_t len; + uint8_t desig[]; +}; + +/* mode page policy descriptor */ +struct scsi_mpage_policy_desc { + uint8_t page_code; + uint8_t sub_page_code; + uint8_t policy; + uint8_t reserved; +}; + +/* target port descriptor */ +struct scsi_tgt_port_desc { + uint8_t code_set; + uint8_t desig_type; + uint8_t reserved; + uint8_t len; + uint8_t designator[]; +}; + +/* SCSI port designation descriptor */ +struct scsi_port_desc { + uint16_t reserved; + uint16_t rel_port_id; + uint16_t reserved2; + uint16_t init_port_len; + uint16_t init_port_id; + uint16_t reserved3; + uint16_t tgt_desc_len; + uint8_t tgt_desc[]; +}; + +/* SCSI UNMAP block descriptor */ +struct scsi_unmap_bdesc { + /* UNMAP LOGICAL BLOCK ADDRESS */ + uint64_t lba; + + /* NUMBER OF LOGICAL BLOCKS */ + uint32_t block_count; + + /* RESERVED */ + uint32_t reserved; +}; + +#define SCSI_UNMAP_LBPU (1 << 7) +#define SCSI_UNMAP_LBPWS (1 << 6) +#define SCSI_UNMAP_LBPWS10 (1 << 5) + +#define SCSI_UNMAP_FULL_PROVISIONING 0x00 +#define SCSI_UNMAP_RESOURCE_PROVISIONING 0x01 +#define SCSI_UNMAP_THIN_PROVISIONING 0x02 + +#endif /* _SCSI_SPEC_H */ diff --git a/dpdk/examples/vhost_scsi/vhost_scsi.c b/dpdk/examples/vhost_scsi/vhost_scsi.c new file mode 100644 index 00000000..b40f9936 --- /dev/null +++ b/dpdk/examples/vhost_scsi/vhost_scsi.c @@ -0,0 +1,512 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "vhost_scsi.h" +#include "scsi_spec.h" + +#define VIRTIO_SCSI_FEATURES ((1 << VIRTIO_F_NOTIFY_ON_EMPTY) |\ + (1 << VIRTIO_RING_F_EVENT_IDX) |\ + (1 << VIRTIO_SCSI_F_INOUT) |\ + (1 << VIRTIO_SCSI_F_CHANGE)) + +/* Path to folder where character device will be created. Can be set by user. */ +static char dev_pathname[PATH_MAX] = ""; + +static struct vhost_scsi_ctrlr *g_vhost_ctrlr; +static int g_should_stop; +static sem_t exit_sem; + +static struct vhost_scsi_ctrlr * +vhost_scsi_ctrlr_find(__rte_unused const char *ctrlr_name) +{ + /* currently we only support 1 socket file fd */ + return g_vhost_ctrlr; +} + +static uint64_t gpa_to_vva(int vid, uint64_t gpa, uint64_t *len) +{ + char path[PATH_MAX]; + struct vhost_scsi_ctrlr *ctrlr; + int ret = 0; + + ret = rte_vhost_get_ifname(vid, path, PATH_MAX); + if (ret) { + fprintf(stderr, "Cannot get socket name\n"); + assert(ret != 0); + } + + ctrlr = vhost_scsi_ctrlr_find(path); + if (!ctrlr) { + fprintf(stderr, "Controller is not ready\n"); + assert(ctrlr != NULL); + } + + assert(ctrlr->mem != NULL); + + return rte_vhost_va_from_guest_pa(ctrlr->mem, gpa, len); +} + +static struct vring_desc * +descriptor_get_next(struct vring_desc *vq_desc, struct vring_desc *cur_desc) +{ + return &vq_desc[cur_desc->next]; +} + +static bool +descriptor_has_next(struct vring_desc *cur_desc) +{ + return !!(cur_desc->flags & VRING_DESC_F_NEXT); +} + +static bool +descriptor_is_wr(struct vring_desc *cur_desc) +{ + return !!(cur_desc->flags & VRING_DESC_F_WRITE); +} + +static void +submit_completion(struct vhost_scsi_task *task) +{ + struct rte_vhost_vring *vq; + struct vring_used *used; + + vq = task->vq; + used = vq->used; + /* Fill out the next entry in the "used" ring. id = the + * index of the descriptor that contained the SCSI request. + * len = the total amount of data transferred for the SCSI + * request. We must report the correct len, for variable + * length SCSI CDBs, where we may return less data than + * allocated by the guest VM. + */ + used->ring[used->idx & (vq->size - 1)].id = task->req_idx; + used->ring[used->idx & (vq->size - 1)].len = task->data_len; + used->idx++; + + /* Send an interrupt back to the guest VM so that it knows + * a completion is ready to be processed. + */ + eventfd_write(vq->callfd, (eventfd_t)1); +} + +static void +vhost_process_read_payload_chain(struct vhost_scsi_task *task) +{ + void *data; + uint64_t chunck_len; + + task->iovs_cnt = 0; + chunck_len = task->desc->len; + task->resp = (void *)(uintptr_t)gpa_to_vva(task->bdev->vid, + task->desc->addr, + &chunck_len); + if (!task->resp || chunck_len != task->desc->len) { + fprintf(stderr, "failed to translate desc address.\n"); + return; + } + + while (descriptor_has_next(task->desc)) { + task->desc = descriptor_get_next(task->vq->desc, task->desc); + chunck_len = task->desc->len; + data = (void *)(uintptr_t)gpa_to_vva(task->bdev->vid, + task->desc->addr, + &chunck_len); + if (!data || chunck_len != task->desc->len) { + fprintf(stderr, "failed to translate desc address.\n"); + return; + } + + task->iovs[task->iovs_cnt].iov_base = data; + task->iovs[task->iovs_cnt].iov_len = task->desc->len; + task->data_len += task->desc->len; + task->iovs_cnt++; + } +} + +static void +vhost_process_write_payload_chain(struct vhost_scsi_task *task) +{ + void *data; + uint64_t chunck_len; + + task->iovs_cnt = 0; + + do { + chunck_len = task->desc->len; + data = (void *)(uintptr_t)gpa_to_vva(task->bdev->vid, + task->desc->addr, + &chunck_len); + if (!data || chunck_len != task->desc->len) { + fprintf(stderr, "failed to translate desc address.\n"); + return; + } + + task->iovs[task->iovs_cnt].iov_base = data; + task->iovs[task->iovs_cnt].iov_len = task->desc->len; + task->data_len += task->desc->len; + task->iovs_cnt++; + task->desc = descriptor_get_next(task->vq->desc, task->desc); + } while (descriptor_has_next(task->desc)); + + chunck_len = task->desc->len; + task->resp = (void *)(uintptr_t)gpa_to_vva(task->bdev->vid, + task->desc->addr, + &chunck_len); + if (!task->resp || chunck_len != task->desc->len) + fprintf(stderr, "failed to translate desc address.\n"); +} + +static struct vhost_block_dev * +vhost_scsi_bdev_construct(const char *bdev_name, const char *bdev_serial, + uint32_t blk_size, uint64_t blk_cnt, + bool wce_enable) +{ + struct vhost_block_dev *bdev; + + bdev = rte_zmalloc(NULL, sizeof(*bdev), RTE_CACHE_LINE_SIZE); + if (!bdev) + return NULL; + + strncpy(bdev->name, bdev_name, sizeof(bdev->name)); + strncpy(bdev->product_name, bdev_serial, sizeof(bdev->product_name)); + bdev->blocklen = blk_size; + bdev->blockcnt = blk_cnt; + bdev->write_cache = wce_enable; + + /* use memory as disk storage space */ + bdev->data = rte_zmalloc(NULL, blk_cnt * blk_size, 0); + if (!bdev->data) { + fprintf(stderr, "no enough reseverd huge memory for disk\n"); + return NULL; + } + + return bdev; +} + +static void +process_requestq(struct vhost_scsi_ctrlr *ctrlr, uint32_t q_idx) +{ + int ret; + struct vhost_scsi_queue *scsi_vq; + struct rte_vhost_vring *vq; + + scsi_vq = &ctrlr->bdev->queues[q_idx]; + vq = &scsi_vq->vq; + ret = rte_vhost_get_vhost_vring(ctrlr->bdev->vid, q_idx, vq); + assert(ret == 0); + + while (vq->avail->idx != scsi_vq->last_used_idx) { + int req_idx; + uint16_t last_idx; + struct vhost_scsi_task *task; + uint64_t chunck_len; + + last_idx = scsi_vq->last_used_idx & (vq->size - 1); + req_idx = vq->avail->ring[last_idx]; + + task = rte_zmalloc(NULL, sizeof(*task), 0); + assert(task != NULL); + + task->ctrlr = ctrlr; + task->bdev = ctrlr->bdev; + task->vq = vq; + task->req_idx = req_idx; + task->desc = &task->vq->desc[task->req_idx]; + + /* does not support indirect descriptors */ + assert((task->desc->flags & VRING_DESC_F_INDIRECT) == 0); + scsi_vq->last_used_idx++; + + chunck_len = task->desc->len; + task->req = (void *)(uintptr_t)gpa_to_vva(task->bdev->vid, + task->desc->addr, + &chunck_len); + if (!task->req || chunck_len != task->desc->len) { + fprintf(stderr, "failed to translate desc address.\n"); + return; + } + + task->desc = descriptor_get_next(task->vq->desc, task->desc); + if (!descriptor_has_next(task->desc)) { + task->dxfer_dir = SCSI_DIR_NONE; + chunck_len = task->desc->len; + task->resp = (void *)(uintptr_t) + gpa_to_vva(task->bdev->vid, + task->desc->addr, + &chunck_len); + if (!task->resp || chunck_len != task->desc->len) { + fprintf(stderr, "failed to translate desc address.\n"); + return; + } + } else if (!descriptor_is_wr(task->desc)) { + task->dxfer_dir = SCSI_DIR_TO_DEV; + vhost_process_write_payload_chain(task); + } else { + task->dxfer_dir = SCSI_DIR_FROM_DEV; + vhost_process_read_payload_chain(task); + } + + ret = vhost_bdev_process_scsi_commands(ctrlr->bdev, task); + if (ret) { + /* invalid response */ + task->resp->response = VIRTIO_SCSI_S_BAD_TARGET; + } else { + /* successfully */ + task->resp->response = VIRTIO_SCSI_S_OK; + task->resp->status = 0; + task->resp->resid = 0; + } + submit_completion(task); + rte_free(task); + } +} + +/* Main framework for processing IOs */ +static void * +ctrlr_worker(void *arg) +{ + uint32_t idx, num; + struct vhost_scsi_ctrlr *ctrlr = (struct vhost_scsi_ctrlr *)arg; + cpu_set_t cpuset; + pthread_t thread; + + thread = pthread_self(); + CPU_ZERO(&cpuset); + CPU_SET(0, &cpuset); + pthread_setaffinity_np(thread, sizeof(cpu_set_t), &cpuset); + + num = rte_vhost_get_vring_num(ctrlr->bdev->vid); + fprintf(stdout, "Ctrlr Worker Thread Started with %u Vring\n", num); + + if (num != NUM_OF_SCSI_QUEUES) { + fprintf(stderr, "Only 1 IO queue are supported\n"); + exit(0); + } + + while (!g_should_stop && ctrlr->bdev != NULL) { + /* At least 3 vrings, currently only can support 1 IO queue + * Queue 2 for IO queue, does not support TMF and hotplug + * for the example application now + */ + for (idx = 2; idx < num; idx++) + process_requestq(ctrlr, idx); + } + + fprintf(stdout, "Ctrlr Worker Thread Exiting\n"); + sem_post(&exit_sem); + return NULL; +} + +static int +new_device(int vid) +{ + char path[PATH_MAX]; + struct vhost_scsi_ctrlr *ctrlr; + struct vhost_scsi_queue *scsi_vq; + struct rte_vhost_vring *vq; + pthread_t tid; + int i, ret; + + ret = rte_vhost_get_ifname(vid, path, PATH_MAX); + if (ret) { + fprintf(stderr, "Cannot get socket name\n"); + return -1; + } + + ctrlr = vhost_scsi_ctrlr_find(path); + if (!ctrlr) { + fprintf(stderr, "Controller is not ready\n"); + return -1; + } + + ret = rte_vhost_get_mem_table(vid, &ctrlr->mem); + if (ret) { + fprintf(stderr, "Get Controller memory region failed\n"); + return -1; + } + assert(ctrlr->mem != NULL); + + /* hardcoded block device information with 128MiB */ + ctrlr->bdev = vhost_scsi_bdev_construct("malloc0", "vhost_scsi_malloc0", + 4096, 32768, 0); + if (!ctrlr->bdev) + return -1; + + ctrlr->bdev->vid = vid; + + /* Disable Notifications */ + for (i = 0; i < NUM_OF_SCSI_QUEUES; i++) { + rte_vhost_enable_guest_notification(vid, i, 0); + /* restore used index */ + scsi_vq = &ctrlr->bdev->queues[i]; + vq = &scsi_vq->vq; + ret = rte_vhost_get_vhost_vring(ctrlr->bdev->vid, i, vq); + assert(ret == 0); + scsi_vq->last_used_idx = vq->used->idx; + scsi_vq->last_avail_idx = vq->used->idx; + } + + g_should_stop = 0; + fprintf(stdout, "New Device %s, Device ID %d\n", path, vid); + if (pthread_create(&tid, NULL, &ctrlr_worker, ctrlr) < 0) { + fprintf(stderr, "Worker Thread Started Failed\n"); + return -1; + } + pthread_detach(tid); + return 0; +} + +static void +destroy_device(int vid) +{ + char path[PATH_MAX]; + struct vhost_scsi_ctrlr *ctrlr; + + rte_vhost_get_ifname(vid, path, PATH_MAX); + fprintf(stdout, "Destroy %s Device ID %d\n", path, vid); + ctrlr = vhost_scsi_ctrlr_find(path); + if (!ctrlr) { + fprintf(stderr, "Destroy Ctrlr Failed\n"); + return; + } + ctrlr->bdev = NULL; + g_should_stop = 1; + + sem_wait(&exit_sem); +} + +static const struct vhost_device_ops vhost_scsi_device_ops = { + .new_device = new_device, + .destroy_device = destroy_device, +}; + +static struct vhost_scsi_ctrlr * +vhost_scsi_ctrlr_construct(const char *ctrlr_name) +{ + int ret; + struct vhost_scsi_ctrlr *ctrlr; + char *path; + char cwd[PATH_MAX]; + + /* always use current directory */ + path = getcwd(cwd, PATH_MAX); + if (!path) { + fprintf(stderr, "Cannot get current working directory\n"); + return NULL; + } + snprintf(dev_pathname, sizeof(dev_pathname), "%s/%s", path, ctrlr_name); + + if (access(dev_pathname, F_OK) != -1) { + if (unlink(dev_pathname) != 0) + rte_exit(EXIT_FAILURE, "Cannot remove %s.\n", + dev_pathname); + } + + if (rte_vhost_driver_register(dev_pathname, 0) != 0) { + fprintf(stderr, "socket %s already exists\n", dev_pathname); + return NULL; + } + + fprintf(stdout, "socket file: %s created\n", dev_pathname); + + ret = rte_vhost_driver_set_features(dev_pathname, VIRTIO_SCSI_FEATURES); + if (ret != 0) { + fprintf(stderr, "Set vhost driver features failed\n"); + return NULL; + } + + ctrlr = rte_zmalloc(NULL, sizeof(*ctrlr), RTE_CACHE_LINE_SIZE); + if (!ctrlr) + return NULL; + + rte_vhost_driver_callback_register(dev_pathname, + &vhost_scsi_device_ops); + + return ctrlr; +} + +static void +signal_handler(__rte_unused int signum) +{ + + if (access(dev_pathname, F_OK) == 0) + unlink(dev_pathname); + exit(0); +} + +int main(int argc, char *argv[]) +{ + int ret; + + signal(SIGINT, signal_handler); + + /* init EAL */ + ret = rte_eal_init(argc, argv); + if (ret < 0) + rte_exit(EXIT_FAILURE, "Error with EAL initialization\n"); + + g_vhost_ctrlr = vhost_scsi_ctrlr_construct("vhost.socket"); + if (g_vhost_ctrlr == NULL) { + fprintf(stderr, "Construct vhost scsi controller failed\n"); + return 0; + } + + if (sem_init(&exit_sem, 0, 0) < 0) { + fprintf(stderr, "Error init exit_sem\n"); + return -1; + } + + rte_vhost_driver_start(dev_pathname); + + /* loop for exit the application */ + while (1) + sleep(1); + + return 0; +} + diff --git a/dpdk/examples/vhost_scsi/vhost_scsi.h b/dpdk/examples/vhost_scsi/vhost_scsi.h new file mode 100644 index 00000000..edb416da --- /dev/null +++ b/dpdk/examples/vhost_scsi/vhost_scsi.h @@ -0,0 +1,108 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _VHOST_SCSI_H_ +#define _VHOST_SCSI_H_ + +#include +#include +#include +#include + +#include + +struct vhost_scsi_queue { + struct rte_vhost_vring vq; + uint16_t last_avail_idx; + uint16_t last_used_idx; +}; + +#define NUM_OF_SCSI_QUEUES 3 + +struct vhost_block_dev { + /** ID for vhost library. */ + int vid; + /** Queues for the block device */ + struct vhost_scsi_queue queues[NUM_OF_SCSI_QUEUES]; + /** Unique name for this block device. */ + char name[64]; + + /** Unique product name for this kind of block device. */ + char product_name[256]; + + /** Size in bytes of a logical block for the backend */ + uint32_t blocklen; + + /** Number of blocks */ + uint64_t blockcnt; + + /** write cache enabled, not used at the moment */ + int write_cache; + + /** use memory as disk storage space */ + uint8_t *data; +}; + +struct vhost_scsi_ctrlr { + /** Only support 1 LUN for the example */ + struct vhost_block_dev *bdev; + /** VM memory region */ + struct rte_vhost_memory *mem; +} __rte_cache_aligned; + +#define VHOST_SCSI_MAX_IOVS 128 + +enum scsi_data_dir { + SCSI_DIR_NONE = 0, + SCSI_DIR_TO_DEV = 1, + SCSI_DIR_FROM_DEV = 2, +}; + +struct vhost_scsi_task { + int req_idx; + uint32_t dxfer_dir; + uint32_t data_len; + struct virtio_scsi_cmd_req *req; + struct virtio_scsi_cmd_resp *resp; + struct iovec iovs[VHOST_SCSI_MAX_IOVS]; + uint32_t iovs_cnt; + struct vring_desc *desc; + struct rte_vhost_vring *vq; + struct vhost_block_dev *bdev; + struct vhost_scsi_ctrlr *ctrlr; +}; + +int vhost_bdev_process_scsi_commands(struct vhost_block_dev *bdev, + struct vhost_scsi_task *task); + +#endif /* _VHOST_SCSI_H_ */ diff --git a/dpdk/examples/vhost_xen/Makefile b/dpdk/examples/vhost_xen/Makefile deleted file mode 100644 index 47e14898..00000000 --- a/dpdk/examples/vhost_xen/Makefile +++ /dev/null @@ -1,52 +0,0 @@ -# BSD LICENSE -# -# Copyright(c) 2010-2014 Intel Corporation. All rights reserved. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in -# the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Intel Corporation nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -ifeq ($(RTE_SDK),) -$(error "Please define RTE_SDK environment variable") -endif - -# Default target, can be overriden by command line or environment -RTE_TARGET ?= x86_64-native-linuxapp-gcc - -include $(RTE_SDK)/mk/rte.vars.mk - -# binary name -APP = vhost-switch - -# all source are stored in SRCS-y -SRCS-y := main.c vhost_monitor.c xenstore_parse.c - -CFLAGS += -O2 -I/usr/local/include -D_FILE_OFFSET_BITS=64 -Wno-unused-parameter -CFLAGS += $(WERROR_FLAGS) -CFLAGS += -D_GNU_SOURCE -LDFLAGS += -lxenstore - -include $(RTE_SDK)/mk/rte.extapp.mk diff --git a/dpdk/examples/vhost_xen/main.c b/dpdk/examples/vhost_xen/main.c deleted file mode 100644 index 2e403576..00000000 --- a/dpdk/examples/vhost_xen/main.c +++ /dev/null @@ -1,1510 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -#include "main.h" -#include "virtio-net.h" -#include "xen_vhost.h" - -#define MAX_QUEUES 128 - -/* the maximum number of external ports supported */ -#define MAX_SUP_PORTS 1 - -/* - * Calculate the number of buffers needed per port - */ -#define NUM_MBUFS_PER_PORT ((MAX_QUEUES*RTE_TEST_RX_DESC_DEFAULT) + \ - (num_switching_cores*MAX_PKT_BURST) + \ - (num_switching_cores*RTE_TEST_TX_DESC_DEFAULT) +\ - (num_switching_cores*MBUF_CACHE_SIZE)) - -#define MBUF_CACHE_SIZE 64 - -/* - * RX and TX Prefetch, Host, and Write-back threshold values should be - * carefully set for optimal performance. Consult the network - * controller's datasheet and supporting DPDK documentation for guidance - * on how these parameters should be set. - */ -#define RX_PTHRESH 8 /* Default values of RX prefetch threshold reg. */ -#define RX_HTHRESH 8 /* Default values of RX host threshold reg. */ -#define RX_WTHRESH 4 /* Default values of RX write-back threshold reg. */ - -/* - * These default values are optimized for use with the Intel(R) 82599 10 GbE - * Controller and the DPDK ixgbe PMD. Consider using other values for other - * network controllers and/or network drivers. - */ -#define TX_PTHRESH 36 /* Default values of TX prefetch threshold reg. */ -#define TX_HTHRESH 0 /* Default values of TX host threshold reg. */ -#define TX_WTHRESH 0 /* Default values of TX write-back threshold reg. */ - -#define MAX_PKT_BURST 32 /* Max burst size for RX/TX */ -#define MAX_MRG_PKT_BURST 16 /* Max burst for merge buffers. Set to 1 due to performance issue. */ -#define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */ - -/* State of virtio device. */ -#define DEVICE_NOT_READY 0 -#define DEVICE_READY 1 -#define DEVICE_SAFE_REMOVE 2 - -/* Config_core_flag status definitions. */ -#define REQUEST_DEV_REMOVAL 1 -#define ACK_DEV_REMOVAL 0 - -/* Configurable number of RX/TX ring descriptors */ -#define RTE_TEST_RX_DESC_DEFAULT 128 -#define RTE_TEST_TX_DESC_DEFAULT 512 - -#define INVALID_PORT_ID 0xFF - -/* Max number of devices. Limited by vmdq. */ -#define MAX_DEVICES 64 - -/* Size of buffers used for snprintfs. */ -#define MAX_PRINT_BUFF 6072 - - -/* Maximum long option length for option parsing. */ -#define MAX_LONG_OPT_SZ 64 - -/* Used to compare MAC addresses. */ -#define MAC_ADDR_CMP 0xFFFFFFFFFFFF - -/* mask of enabled ports */ -static uint32_t enabled_port_mask = 0; - -/*Number of switching cores enabled*/ -static uint32_t num_switching_cores = 0; - -/* number of devices/queues to support*/ -static uint32_t num_queues = 0; -uint32_t num_devices = 0; - -/* Enable VM2VM communications. If this is disabled then the MAC address compare is skipped. */ -static uint32_t enable_vm2vm = 1; -/* Enable stats. */ -static uint32_t enable_stats = 0; - -/* empty vmdq configuration structure. Filled in programatically */ -static const struct rte_eth_conf vmdq_conf_default = { - .rxmode = { - .mq_mode = ETH_MQ_RX_VMDQ_ONLY, - .split_hdr_size = 0, - .header_split = 0, /**< Header Split disabled */ - .hw_ip_checksum = 0, /**< IP checksum offload disabled */ - .hw_vlan_filter = 0, /**< VLAN filtering disabled */ - /* - * It is necessary for 1G NIC such as I350, - * this fixes bug of ipv4 forwarding in guest can't - * forward pakets from one virtio dev to another virtio dev. - */ - .hw_vlan_strip = 1, /**< VLAN strip enabled. */ - .jumbo_frame = 0, /**< Jumbo Frame Support disabled */ - .hw_strip_crc = 0, /**< CRC stripped by hardware */ - }, - - .txmode = { - .mq_mode = ETH_MQ_TX_NONE, - }, - .rx_adv_conf = { - /* - * should be overridden separately in code with - * appropriate values - */ - .vmdq_rx_conf = { - .nb_queue_pools = ETH_8_POOLS, - .enable_default_pool = 0, - .default_pool = 0, - .nb_pool_maps = 0, - .pool_map = {{0, 0},}, - }, - }, -}; - -static unsigned lcore_ids[RTE_MAX_LCORE]; -static uint8_t ports[RTE_MAX_ETHPORTS]; -static unsigned num_ports = 0; /**< The number of ports specified in command line */ - -const uint16_t vlan_tags[] = { - 1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007, - 1008, 1009, 1010, 1011, 1012, 1013, 1014, 1015, - 1016, 1017, 1018, 1019, 1020, 1021, 1022, 1023, - 1024, 1025, 1026, 1027, 1028, 1029, 1030, 1031, - 1032, 1033, 1034, 1035, 1036, 1037, 1038, 1039, - 1040, 1041, 1042, 1043, 1044, 1045, 1046, 1047, - 1048, 1049, 1050, 1051, 1052, 1053, 1054, 1055, - 1056, 1057, 1058, 1059, 1060, 1061, 1062, 1063, -}; - -/* ethernet addresses of ports */ -static struct ether_addr vmdq_ports_eth_addr[RTE_MAX_ETHPORTS]; - -/* heads for the main used and free linked lists for the data path. */ -static struct virtio_net_data_ll *ll_root_used = NULL; -static struct virtio_net_data_ll *ll_root_free = NULL; - -/* Array of data core structures containing information on individual core linked lists. */ -static struct lcore_info lcore_info[RTE_MAX_LCORE]; - -/* Used for queueing bursts of TX packets. */ -struct mbuf_table { - unsigned len; - unsigned txq_id; - struct rte_mbuf *m_table[MAX_PKT_BURST]; -}; - -/* TX queue for each data core. */ -struct mbuf_table lcore_tx_queue[RTE_MAX_LCORE]; - -/* Vlan header struct used to insert vlan tags on TX. */ -struct vlan_ethhdr { - unsigned char h_dest[ETH_ALEN]; - unsigned char h_source[ETH_ALEN]; - __be16 h_vlan_proto; - __be16 h_vlan_TCI; - __be16 h_vlan_encapsulated_proto; -}; - -/* Header lengths. */ -#define VLAN_HLEN 4 -#define VLAN_ETH_HLEN 18 - -/* Per-device statistics struct */ -struct device_statistics { - uint64_t tx_total; - rte_atomic64_t rx_total; - uint64_t tx; - rte_atomic64_t rx; -} __rte_cache_aligned; -struct device_statistics dev_statistics[MAX_DEVICES]; - -/* - * Builds up the correct configuration for VMDQ VLAN pool map - * according to the pool & queue limits. - */ -static inline int -get_eth_conf(struct rte_eth_conf *eth_conf, uint32_t num_devices) -{ - struct rte_eth_vmdq_rx_conf conf; - unsigned i; - - memset(&conf, 0, sizeof(conf)); - conf.nb_queue_pools = (enum rte_eth_nb_pools)num_devices; - conf.nb_pool_maps = num_devices; - - for (i = 0; i < conf.nb_pool_maps; i++) { - conf.pool_map[i].vlan_id = vlan_tags[ i ]; - conf.pool_map[i].pools = (1UL << i); - } - - (void)(rte_memcpy(eth_conf, &vmdq_conf_default, sizeof(*eth_conf))); - (void)(rte_memcpy(ð_conf->rx_adv_conf.vmdq_rx_conf, &conf, - sizeof(eth_conf->rx_adv_conf.vmdq_rx_conf))); - return 0; -} - -/* - * Validate the device number according to the max pool number gotten form dev_info - * If the device number is invalid, give the error message and return -1. - * Each device must have its own pool. - */ -static inline int -validate_num_devices(uint32_t max_nb_devices) -{ - if (num_devices > max_nb_devices) { - RTE_LOG(ERR, VHOST_PORT, "invalid number of devices\n"); - return -1; - } - return 0; -} - -/* - * Initialises a given port using global settings and with the rx buffers - * coming from the mbuf_pool passed as parameter - */ -static inline int -port_init(uint8_t port, struct rte_mempool *mbuf_pool) -{ - struct rte_eth_dev_info dev_info; - struct rte_eth_rxconf *rxconf; - struct rte_eth_conf port_conf; - uint16_t rx_rings, tx_rings = (uint16_t)rte_lcore_count(); - const uint16_t rx_ring_size = RTE_TEST_RX_DESC_DEFAULT, tx_ring_size = RTE_TEST_TX_DESC_DEFAULT; - int retval; - uint16_t q; - - /* The max pool number from dev_info will be used to validate the pool number specified in cmd line */ - rte_eth_dev_info_get (port, &dev_info); - - /*configure the number of supported virtio devices based on VMDQ limits */ - num_devices = dev_info.max_vmdq_pools; - num_queues = dev_info.max_rx_queues; - - retval = validate_num_devices(MAX_DEVICES); - if (retval < 0) - return retval; - - /* Get port configuration. */ - retval = get_eth_conf(&port_conf, num_devices); - if (retval < 0) - return retval; - - if (port >= rte_eth_dev_count()) return -1; - - rx_rings = (uint16_t)num_queues, - /* Configure ethernet device. */ - retval = rte_eth_dev_configure(port, rx_rings, tx_rings, &port_conf); - if (retval != 0) - return retval; - - rte_eth_dev_info_get(port, &dev_info); - rxconf = &dev_info.default_rxconf; - rxconf->rx_drop_en = 1; - /* Setup the queues. */ - for (q = 0; q < rx_rings; q ++) { - retval = rte_eth_rx_queue_setup(port, q, rx_ring_size, - rte_eth_dev_socket_id(port), rxconf, - mbuf_pool); - if (retval < 0) - return retval; - } - for (q = 0; q < tx_rings; q ++) { - retval = rte_eth_tx_queue_setup(port, q, tx_ring_size, - rte_eth_dev_socket_id(port), - NULL); - if (retval < 0) - return retval; - } - - /* Start the device. */ - retval = rte_eth_dev_start(port); - if (retval < 0) - return retval; - - rte_eth_macaddr_get(port, &vmdq_ports_eth_addr[port]); - RTE_LOG(INFO, VHOST_PORT, "Max virtio devices supported: %u\n", num_devices); - RTE_LOG(INFO, VHOST_PORT, "Port %u MAC: %02"PRIx8" %02"PRIx8" %02"PRIx8 - " %02"PRIx8" %02"PRIx8" %02"PRIx8"\n", - (unsigned)port, - vmdq_ports_eth_addr[port].addr_bytes[0], - vmdq_ports_eth_addr[port].addr_bytes[1], - vmdq_ports_eth_addr[port].addr_bytes[2], - vmdq_ports_eth_addr[port].addr_bytes[3], - vmdq_ports_eth_addr[port].addr_bytes[4], - vmdq_ports_eth_addr[port].addr_bytes[5]); - - return 0; -} - -/* - * Parse the portmask provided at run time. - */ -static int -parse_portmask(const char *portmask) -{ - char *end = NULL; - unsigned long pm; - - errno = 0; - - /* parse hexadecimal string */ - pm = strtoul(portmask, &end, 16); - if ((portmask[0] == '\0') || (end == NULL) || (*end != '\0') || (errno != 0)) - return -1; - - if (pm == 0) - return -1; - - return pm; - -} - -/* - * Parse num options at run time. - */ -static int -parse_num_opt(const char *q_arg, uint32_t max_valid_value) -{ - char *end = NULL; - unsigned long num; - - errno = 0; - - /* parse unsigned int string */ - num = strtoul(q_arg, &end, 10); - if ((q_arg[0] == '\0') || (end == NULL) || (*end != '\0') || (errno != 0)) - return -1; - - if (num > max_valid_value) - return -1; - - return num; - -} - -/* - * Display usage - */ -static void -us_vhost_usage(const char *prgname) -{ - RTE_LOG(INFO, VHOST_CONFIG, "%s [EAL options] -- -p PORTMASK --vm2vm [0|1] --stats [0-N] --nb-devices ND\n" - " -p PORTMASK: Set mask for ports to be used by application\n" - " --vm2vm [0|1]: disable/enable(default) vm2vm comms\n" - " --stats [0-N]: 0: Disable stats, N: Time in seconds to print stats\n", - prgname); -} - -/* - * Parse the arguments given in the command line of the application. - */ -static int -us_vhost_parse_args(int argc, char **argv) -{ - int opt, ret; - int option_index; - unsigned i; - const char *prgname = argv[0]; - static struct option long_option[] = { - {"vm2vm", required_argument, NULL, 0}, - {"stats", required_argument, NULL, 0}, - {NULL, 0, 0, 0} - }; - - /* Parse command line */ - while ((opt = getopt_long(argc, argv, "p:",long_option, &option_index)) != EOF) { - switch (opt) { - /* Portmask */ - case 'p': - enabled_port_mask = parse_portmask(optarg); - if (enabled_port_mask == 0) { - RTE_LOG(INFO, VHOST_CONFIG, "Invalid portmask\n"); - us_vhost_usage(prgname); - return -1; - } - break; - - case 0: - /* Enable/disable vm2vm comms. */ - if (!strncmp(long_option[option_index].name, "vm2vm", MAX_LONG_OPT_SZ)) { - ret = parse_num_opt(optarg, 1); - if (ret == -1) { - RTE_LOG(INFO, VHOST_CONFIG, "Invalid argument for vm2vm [0|1]\n"); - us_vhost_usage(prgname); - return -1; - } else { - enable_vm2vm = ret; - } - } - - /* Enable/disable stats. */ - if (!strncmp(long_option[option_index].name, "stats", MAX_LONG_OPT_SZ)) { - ret = parse_num_opt(optarg, INT32_MAX); - if (ret == -1) { - RTE_LOG(INFO, VHOST_CONFIG, "Invalid argument for stats [0..N]\n"); - us_vhost_usage(prgname); - return -1; - } else { - enable_stats = ret; - } - } - break; - - /* Invalid option - print options. */ - default: - us_vhost_usage(prgname); - return -1; - } - } - - for (i = 0; i < RTE_MAX_ETHPORTS; i++) { - if (enabled_port_mask & (1 << i)) - ports[num_ports++] = (uint8_t)i; - } - - if ((num_ports == 0) || (num_ports > MAX_SUP_PORTS)) { - RTE_LOG(INFO, VHOST_PORT, "Current enabled port number is %u," - "but only %u port can be enabled\n",num_ports, MAX_SUP_PORTS); - return -1; - } - - return 0; -} - -/* - * Update the global var NUM_PORTS and array PORTS according to system ports number - * and return valid ports number - */ -static unsigned check_ports_num(unsigned nb_ports) -{ - unsigned valid_num_ports = num_ports; - unsigned portid; - - if (num_ports > nb_ports) { - RTE_LOG(INFO, VHOST_PORT, "\nSpecified port number(%u) exceeds total system port number(%u)\n", - num_ports, nb_ports); - num_ports = nb_ports; - } - - for (portid = 0; portid < num_ports; portid ++) { - if (ports[portid] >= nb_ports) { - RTE_LOG(INFO, VHOST_PORT, "\nSpecified port ID(%u) exceeds max system port ID(%u)\n", - ports[portid], (nb_ports - 1)); - ports[portid] = INVALID_PORT_ID; - valid_num_ports--; - } - } - return valid_num_ports; -} - -/* - * Function to convert guest physical addresses to vhost virtual addresses. This - * is used to convert virtio buffer addresses. - */ -static inline uint64_t __attribute__((always_inline)) -gpa_to_vva(struct virtio_net *dev, uint64_t guest_pa) -{ - struct virtio_memory_regions *region; - uint32_t regionidx; - uint64_t vhost_va = 0; - - for (regionidx = 0; regionidx < dev->mem->nregions; regionidx++) { - region = &dev->mem->regions[regionidx]; - if ((guest_pa >= region->guest_phys_address) && - (guest_pa <= region->guest_phys_address_end)) { - vhost_va = region->address_offset + guest_pa; - break; - } - } - RTE_LOG(DEBUG, VHOST_DATA, "(%" PRIu64 ") GPA %p| VVA %p\n", - dev->device_fh, (void*)(uintptr_t)guest_pa, (void*)(uintptr_t)vhost_va); - - return vhost_va; -} - -/* - * This function adds buffers to the virtio devices RX virtqueue. Buffers can - * be received from the physical port or from another virtio device. A packet - * count is returned to indicate the number of packets that were succesfully - * added to the RX queue. - */ -static inline uint32_t __attribute__((always_inline)) -virtio_dev_rx(struct virtio_net *dev, struct rte_mbuf **pkts, uint32_t count) -{ - struct vhost_virtqueue *vq; - struct vring_desc *desc; - struct rte_mbuf *buff; - /* The virtio_hdr is initialised to 0. */ - struct virtio_net_hdr_mrg_rxbuf virtio_hdr = {{0,0,0,0,0,0},0}; - uint64_t buff_addr = 0; - uint64_t buff_hdr_addr = 0; - uint32_t head[MAX_PKT_BURST], packet_len = 0; - uint32_t head_idx, packet_success = 0; - uint16_t avail_idx, res_cur_idx; - uint16_t res_base_idx, res_end_idx; - uint16_t free_entries; - uint8_t success = 0; - void *userdata; - - RTE_LOG(DEBUG, VHOST_DATA, "(%" PRIu64 ") virtio_dev_rx()\n", dev->device_fh); - vq = dev->virtqueue_rx; - count = (count > MAX_PKT_BURST) ? MAX_PKT_BURST : count; - /* As many data cores may want access to available buffers, they need to be reserved. */ - do { - - res_base_idx = vq->last_used_idx_res; - - avail_idx = *((volatile uint16_t *)&vq->avail->idx); - - free_entries = (avail_idx - res_base_idx); - - /*check that we have enough buffers*/ - if (unlikely(count > free_entries)) - count = free_entries; - - if (count == 0) - return 0; - - res_end_idx = res_base_idx + count; - /* vq->last_used_idx_res is atomically updated. */ - success = rte_atomic16_cmpset(&vq->last_used_idx_res, res_base_idx, - res_end_idx); - } while (unlikely(success == 0)); - res_cur_idx = res_base_idx; - RTE_LOG(DEBUG, VHOST_DATA, "(%" PRIu64 ") Current Index %d| End Index %d\n", - dev->device_fh, res_cur_idx, res_end_idx); - - /* Prefetch available ring to retrieve indexes. */ - rte_prefetch0(&vq->avail->ring[res_cur_idx & (vq->size - 1)]); - - /* Retrieve all of the head indexes first to avoid caching issues. */ - for (head_idx = 0; head_idx < count; head_idx++) - head[head_idx] = vq->avail->ring[(res_cur_idx + head_idx) & (vq->size - 1)]; - - /*Prefetch descriptor index. */ - rte_prefetch0(&vq->desc[head[packet_success]]); - - while (res_cur_idx != res_end_idx) { - /* Get descriptor from available ring */ - desc = &vq->desc[head[packet_success]]; - /* Prefetch descriptor address. */ - rte_prefetch0(desc); - - buff = pkts[packet_success]; - - /* Convert from gpa to vva (guest physical addr -> vhost virtual addr) */ - buff_addr = gpa_to_vva(dev, desc->addr); - /* Prefetch buffer address. */ - rte_prefetch0((void*)(uintptr_t)buff_addr); - - { - /* Copy virtio_hdr to packet and increment buffer address */ - buff_hdr_addr = buff_addr; - packet_len = rte_pktmbuf_data_len(buff) + vq->vhost_hlen; - - /* - * If the descriptors are chained the header and data are placed in - * separate buffers. - */ - if (desc->flags & VRING_DESC_F_NEXT) { - desc->len = vq->vhost_hlen; - desc = &vq->desc[desc->next]; - /* Buffer address translation. */ - buff_addr = gpa_to_vva(dev, desc->addr); - desc->len = rte_pktmbuf_data_len(buff); - } else { - buff_addr += vq->vhost_hlen; - desc->len = packet_len; - } - } - - /* Update used ring with desc information */ - vq->used->ring[res_cur_idx & (vq->size - 1)].id = head[packet_success]; - vq->used->ring[res_cur_idx & (vq->size - 1)].len = packet_len; - - /* Copy mbuf data to buffer */ - userdata = rte_pktmbuf_mtod(buff, void *); - rte_memcpy((void *)(uintptr_t)buff_addr, userdata, rte_pktmbuf_data_len(buff)); - - res_cur_idx++; - packet_success++; - - /* mergeable is disabled then a header is required per buffer. */ - rte_memcpy((void *)(uintptr_t)buff_hdr_addr, (const void *)&virtio_hdr, vq->vhost_hlen); - if (res_cur_idx < res_end_idx) { - /* Prefetch descriptor index. */ - rte_prefetch0(&vq->desc[head[packet_success]]); - } - } - - rte_compiler_barrier(); - - /* Wait until it's our turn to add our buffer to the used ring. */ - while (unlikely(vq->last_used_idx != res_base_idx)) - rte_pause(); - - *(volatile uint16_t *)&vq->used->idx += count; - - vq->last_used_idx = res_end_idx; - - return count; -} - -/* - * Compares a packet destination MAC address to a device MAC address. - */ -static inline int __attribute__((always_inline)) -ether_addr_cmp(struct ether_addr *ea, struct ether_addr *eb) -{ - return ((*(uint64_t *)ea ^ *(uint64_t *)eb) & MAC_ADDR_CMP) == 0; -} - -/* - * This function registers mac along with a - * vlan tag to a VMDQ. - */ -static int -link_vmdq(struct virtio_net *dev) -{ - int ret; - struct virtio_net_data_ll *dev_ll; - - dev_ll = ll_root_used; - - while (dev_ll != NULL) { - if ((dev != dev_ll->dev) && ether_addr_cmp(&dev->mac_address, &dev_ll->dev->mac_address)) { - RTE_LOG(INFO, VHOST_DATA, "(%"PRIu64") WARNING: This device is using an existing MAC address and has not been registered.\n", dev->device_fh); - return -1; - } - dev_ll = dev_ll->next; - } - - /* vlan_tag currently uses the device_id. */ - dev->vlan_tag = vlan_tags[dev->device_fh]; - dev->vmdq_rx_q = dev->device_fh * (num_queues/num_devices); - - /* Print out VMDQ registration info. */ - RTE_LOG(INFO, VHOST_DATA, "(%"PRIu64") MAC_ADDRESS %02x:%02x:%02x:%02x:%02x:%02x and VLAN_TAG %d registered\n", - dev->device_fh, - dev->mac_address.addr_bytes[0], dev->mac_address.addr_bytes[1], - dev->mac_address.addr_bytes[2], dev->mac_address.addr_bytes[3], - dev->mac_address.addr_bytes[4], dev->mac_address.addr_bytes[5], - dev->vlan_tag); - - /* Register the MAC address. */ - ret = rte_eth_dev_mac_addr_add(ports[0], &dev->mac_address, (uint32_t)dev->device_fh); - if (ret) { - RTE_LOG(ERR, VHOST_DATA, "(%"PRIu64") Failed to add device MAC address to VMDQ\n", - dev->device_fh); - return -1; - } - - /* Enable stripping of the vlan tag as we handle routing. */ - rte_eth_dev_set_vlan_strip_on_queue(ports[0], dev->vmdq_rx_q, 1); - - rte_compiler_barrier(); - /* Set device as ready for RX. */ - dev->ready = DEVICE_READY; - - return 0; -} - -/* - * Removes MAC address and vlan tag from VMDQ. Ensures that nothing is adding buffers to the RX - * queue before disabling RX on the device. - */ -static inline void -unlink_vmdq(struct virtio_net *dev) -{ - unsigned i = 0; - unsigned rx_count; - struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; - - if (dev->ready == DEVICE_READY) { - /*clear MAC and VLAN settings*/ - rte_eth_dev_mac_addr_remove(ports[0], &dev->mac_address); - for (i = 0; i < 6; i++) - dev->mac_address.addr_bytes[i] = 0; - - dev->vlan_tag = 0; - - /*Clear out the receive buffers*/ - rx_count = rte_eth_rx_burst(ports[0], - (uint16_t)dev->vmdq_rx_q, pkts_burst, MAX_PKT_BURST); - - while (rx_count) { - for (i = 0; i < rx_count; i++) - rte_pktmbuf_free(pkts_burst[i]); - - rx_count = rte_eth_rx_burst(ports[0], - (uint16_t)dev->vmdq_rx_q, pkts_burst, MAX_PKT_BURST); - } - - dev->ready = DEVICE_NOT_READY; - } -} - -/* - * Check if the packet destination MAC address is for a local device. If so then put - * the packet on that devices RX queue. If not then return. - */ -static inline unsigned __attribute__((always_inline)) -virtio_tx_local(struct virtio_net *dev, struct rte_mbuf *m) -{ - struct virtio_net_data_ll *dev_ll; - struct ether_hdr *pkt_hdr; - uint64_t ret = 0; - - pkt_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *); - - /*get the used devices list*/ - dev_ll = ll_root_used; - - while (dev_ll != NULL) { - if (likely(dev_ll->dev->ready == DEVICE_READY) && ether_addr_cmp(&(pkt_hdr->d_addr), - &dev_ll->dev->mac_address)) { - - /* Drop the packet if the TX packet is destined for the TX device. */ - if (dev_ll->dev->device_fh == dev->device_fh) { - RTE_LOG(DEBUG, VHOST_DATA, "(%" PRIu64 ") TX: " - "Source and destination MAC addresses are the same. " - "Dropping packet.\n", - dev_ll->dev->device_fh); - return 0; - } - - - RTE_LOG(DEBUG, VHOST_DATA, "(%" PRIu64 ") TX: " - "MAC address is local\n", dev_ll->dev->device_fh); - - if (dev_ll->dev->remove) { - /*drop the packet if the device is marked for removal*/ - RTE_LOG(DEBUG, VHOST_DATA, "(%" PRIu64 ") " - "Device is marked for removal\n", - dev_ll->dev->device_fh); - } else { - /*send the packet to the local virtio device*/ - ret = virtio_dev_rx(dev_ll->dev, &m, 1); - if (enable_stats) { - rte_atomic64_add(&dev_statistics[dev_ll->dev->device_fh].rx_total, 1); - rte_atomic64_add(&dev_statistics[dev_ll->dev->device_fh].rx, ret); - dev_statistics[dev->device_fh].tx_total++; - dev_statistics[dev->device_fh].tx += ret; - } - } - - return 0; - } - dev_ll = dev_ll->next; - } - - return -1; -} - -/* - * This function routes the TX packet to the correct interface. This may be a local device - * or the physical port. - */ -static inline void __attribute__((always_inline)) -virtio_tx_route(struct virtio_net* dev, struct rte_mbuf *m, struct rte_mempool *mbuf_pool, uint16_t vlan_tag) -{ - struct mbuf_table *tx_q; - struct vlan_ethhdr *vlan_hdr; - struct rte_mbuf **m_table; - struct rte_mbuf *mbuf; - unsigned len, ret; - const uint16_t lcore_id = rte_lcore_id(); - - /*check if destination is local VM*/ - if (enable_vm2vm && (virtio_tx_local(dev, m) == 0)) { - return; - } - - RTE_LOG(DEBUG, VHOST_DATA, "(%" PRIu64 ") TX: " - "MAC address is external\n", dev->device_fh); - - /*Add packet to the port tx queue*/ - tx_q = &lcore_tx_queue[lcore_id]; - len = tx_q->len; - - /* Allocate an mbuf and populate the structure. */ - mbuf = rte_pktmbuf_alloc(mbuf_pool); - if(!mbuf) - return; - - mbuf->data_len = m->data_len + VLAN_HLEN; - mbuf->pkt_len = mbuf->data_len; - - /* Copy ethernet header to mbuf. */ - rte_memcpy(rte_pktmbuf_mtod(mbuf, void*), - rte_pktmbuf_mtod(m, const void*), ETH_HLEN); - - - /* Setup vlan header. Bytes need to be re-ordered for network with htons()*/ - vlan_hdr = rte_pktmbuf_mtod(mbuf, struct vlan_ethhdr *); - vlan_hdr->h_vlan_encapsulated_proto = vlan_hdr->h_vlan_proto; - vlan_hdr->h_vlan_proto = htons(ETH_P_8021Q); - vlan_hdr->h_vlan_TCI = htons(vlan_tag); - - /* Copy the remaining packet contents to the mbuf. */ - rte_memcpy(rte_pktmbuf_mtod_offset(mbuf, void *, VLAN_ETH_HLEN), - rte_pktmbuf_mtod_offset(m, const void *, ETH_HLEN), - (m->data_len - ETH_HLEN)); - tx_q->m_table[len] = mbuf; - len++; - if (enable_stats) { - dev_statistics[dev->device_fh].tx_total++; - dev_statistics[dev->device_fh].tx++; - } - - if (unlikely(len == MAX_PKT_BURST)) { - m_table = (struct rte_mbuf **)tx_q->m_table; - ret = rte_eth_tx_burst(ports[0], (uint16_t)tx_q->txq_id, m_table, (uint16_t) len); - /* Free any buffers not handled by TX and update the port stats. */ - if (unlikely(ret < len)) { - do { - rte_pktmbuf_free(m_table[ret]); - } while (++ret < len); - } - - len = 0; - } - - tx_q->len = len; - return; -} - -static inline void __attribute__((always_inline)) -virtio_dev_tx(struct virtio_net* dev, struct rte_mempool *mbuf_pool) -{ - struct rte_mbuf m; - struct vhost_virtqueue *vq; - struct vring_desc *desc; - uint64_t buff_addr = 0; - uint32_t head[MAX_PKT_BURST]; - uint32_t used_idx; - uint32_t i; - uint16_t free_entries, packet_success = 0; - uint16_t avail_idx; - - vq = dev->virtqueue_tx; - avail_idx = *((volatile uint16_t *)&vq->avail->idx); - - /* If there are no available buffers then return. */ - if (vq->last_used_idx == avail_idx) - return; - - RTE_LOG(DEBUG, VHOST_DATA, "(%" PRIu64 ") virtio_dev_tx()\n", - dev->device_fh); - - /* Prefetch available ring to retrieve head indexes. */ - rte_prefetch0(&vq->avail->ring[vq->last_used_idx & (vq->size - 1)]); - - /*get the number of free entries in the ring*/ - free_entries = avail_idx - vq->last_used_idx; - free_entries = unlikely(free_entries < MAX_PKT_BURST) ? free_entries : MAX_PKT_BURST; - - RTE_LOG(DEBUG, VHOST_DATA, "(%" PRIu64 ") Buffers available %d\n", - dev->device_fh, free_entries); - /* Retrieve all of the head indexes first to avoid caching issues. */ - for (i = 0; i < free_entries; i++) - head[i] = vq->avail->ring[(vq->last_used_idx + i) & (vq->size - 1)]; - - /* Prefetch descriptor index. */ - rte_prefetch0(&vq->desc[head[packet_success]]); - - while (packet_success < free_entries) { - desc = &vq->desc[head[packet_success]]; - /* Prefetch descriptor address. */ - rte_prefetch0(desc); - - if (packet_success < (free_entries - 1)) { - /* Prefetch descriptor index. */ - rte_prefetch0(&vq->desc[head[packet_success+1]]); - } - - /* Update used index buffer information. */ - used_idx = vq->last_used_idx & (vq->size - 1); - vq->used->ring[used_idx].id = head[packet_success]; - vq->used->ring[used_idx].len = 0; - - /* Discard first buffer as it is the virtio header */ - desc = &vq->desc[desc->next]; - - /* Buffer address translation. */ - buff_addr = gpa_to_vva(dev, desc->addr); - /* Prefetch buffer address. */ - rte_prefetch0((void*)(uintptr_t)buff_addr); - - /* Setup dummy mbuf. This is copied to a real mbuf if transmitted out the physical port. */ - m.data_len = desc->len; - m.data_off = 0; - m.nb_segs = 1; - - virtio_tx_route(dev, &m, mbuf_pool, 0); - - vq->last_used_idx++; - packet_success++; - } - - rte_compiler_barrier(); - vq->used->idx += packet_success; - /* Kick guest if required. */ -} - -/* - * This function is called by each data core. It handles all RX/TX registered with the - * core. For TX the specific lcore linked list is used. For RX, MAC addresses are compared - * with all devices in the main linked list. - */ -static int -switch_worker(__attribute__((unused)) void *arg) -{ - struct rte_mempool *mbuf_pool = arg; - struct virtio_net *dev = NULL; - struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; - struct virtio_net_data_ll *dev_ll; - struct mbuf_table *tx_q; - volatile struct lcore_ll_info *lcore_ll; - const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / US_PER_S * BURST_TX_DRAIN_US; - uint64_t prev_tsc, diff_tsc, cur_tsc, ret_count = 0; - unsigned ret, i; - const uint16_t lcore_id = rte_lcore_id(); - const uint16_t num_cores = (uint16_t)rte_lcore_count(); - uint16_t rx_count = 0; - - RTE_LOG(INFO, VHOST_DATA, "Procesing on Core %u started \n", lcore_id); - lcore_ll = lcore_info[lcore_id].lcore_ll; - prev_tsc = 0; - - tx_q = &lcore_tx_queue[lcore_id]; - for (i = 0; i < num_cores; i ++) { - if (lcore_ids[i] == lcore_id) { - tx_q->txq_id = i; - break; - } - } - - while(1) { - cur_tsc = rte_rdtsc(); - /* - * TX burst queue drain - */ - diff_tsc = cur_tsc - prev_tsc; - if (unlikely(diff_tsc > drain_tsc)) { - - if (tx_q->len) { - RTE_LOG(DEBUG, VHOST_DATA, - "TX queue drained after timeout with burst size %u\n", - tx_q->len); - - /*Tx any packets in the queue*/ - ret = rte_eth_tx_burst(ports[0], (uint16_t)tx_q->txq_id, - (struct rte_mbuf **)tx_q->m_table, - (uint16_t)tx_q->len); - if (unlikely(ret < tx_q->len)) { - do { - rte_pktmbuf_free(tx_q->m_table[ret]); - } while (++ret < tx_q->len); - } - - tx_q->len = 0; - } - - prev_tsc = cur_tsc; - - } - - /* - * Inform the configuration core that we have exited the linked list and that no devices are - * in use if requested. - */ - if (lcore_ll->dev_removal_flag == REQUEST_DEV_REMOVAL) - lcore_ll->dev_removal_flag = ACK_DEV_REMOVAL; - - /* - * Process devices - */ - dev_ll = lcore_ll->ll_root_used; - - while (dev_ll != NULL) { - /*get virtio device ID*/ - dev = dev_ll->dev; - - if (unlikely(dev->remove)) { - dev_ll = dev_ll->next; - unlink_vmdq(dev); - dev->ready = DEVICE_SAFE_REMOVE; - continue; - } - if (likely(dev->ready == DEVICE_READY)) { - /*Handle guest RX*/ - rx_count = rte_eth_rx_burst(ports[0], - (uint16_t)dev->vmdq_rx_q, pkts_burst, MAX_PKT_BURST); - - if (rx_count) { - ret_count = virtio_dev_rx(dev, pkts_burst, rx_count); - if (enable_stats) { - rte_atomic64_add(&dev_statistics[dev_ll->dev->device_fh].rx_total, rx_count); - rte_atomic64_add(&dev_statistics[dev_ll->dev->device_fh].rx, ret_count); - } - while (likely(rx_count)) { - rx_count--; - rte_pktmbuf_free_seg(pkts_burst[rx_count]); - } - - } - } - - if (likely(!dev->remove)) - /*Handle guest TX*/ - virtio_dev_tx(dev, mbuf_pool); - - /*move to the next device in the list*/ - dev_ll = dev_ll->next; - } - } - - return 0; -} - -/* - * Add an entry to a used linked list. A free entry must first be found in the free linked list - * using get_data_ll_free_entry(); - */ -static void -add_data_ll_entry(struct virtio_net_data_ll **ll_root_addr, struct virtio_net_data_ll *ll_dev) -{ - struct virtio_net_data_ll *ll = *ll_root_addr; - - /* Set next as NULL and use a compiler barrier to avoid reordering. */ - ll_dev->next = NULL; - rte_compiler_barrier(); - - /* If ll == NULL then this is the first device. */ - if (ll) { - /* Increment to the tail of the linked list. */ - while ((ll->next != NULL) ) - ll = ll->next; - - ll->next = ll_dev; - } else { - *ll_root_addr = ll_dev; - } -} - -/* - * Remove an entry from a used linked list. The entry must then be added to the free linked list - * using put_data_ll_free_entry(). - */ -static void -rm_data_ll_entry(struct virtio_net_data_ll **ll_root_addr, struct virtio_net_data_ll *ll_dev, struct virtio_net_data_ll *ll_dev_last) -{ - struct virtio_net_data_ll *ll = *ll_root_addr; - - if (ll_dev == ll) - *ll_root_addr = ll_dev->next; - else - ll_dev_last->next = ll_dev->next; -} - -/* - * Find and return an entry from the free linked list. - */ -static struct virtio_net_data_ll * -get_data_ll_free_entry(struct virtio_net_data_ll **ll_root_addr) -{ - struct virtio_net_data_ll *ll_free = *ll_root_addr; - struct virtio_net_data_ll *ll_dev; - - if (ll_free == NULL) - return NULL; - - ll_dev = ll_free; - *ll_root_addr = ll_free->next; - - return ll_dev; -} - -/* - * Place an entry back on to the free linked list. - */ -static void -put_data_ll_free_entry(struct virtio_net_data_ll **ll_root_addr, struct virtio_net_data_ll *ll_dev) -{ - struct virtio_net_data_ll *ll_free = *ll_root_addr; - - ll_dev->next = ll_free; - *ll_root_addr = ll_dev; -} - -/* - * Creates a linked list of a given size. - */ -static struct virtio_net_data_ll * -alloc_data_ll(uint32_t size) -{ - struct virtio_net_data_ll *ll_new; - uint32_t i; - - /* Malloc and then chain the linked list. */ - ll_new = malloc(size * sizeof(struct virtio_net_data_ll)); - if (ll_new == NULL) { - RTE_LOG(ERR, VHOST_CONFIG, "Failed to allocate memory for ll_new.\n"); - return NULL; - } - - for (i = 0; i < size - 1; i++) { - ll_new[i].dev = NULL; - ll_new[i].next = &ll_new[i+1]; - } - ll_new[i].next = NULL; - - return ll_new; -} - -/* - * Create the main linked list along with each individual cores linked list. A used and a free list - * are created to manage entries. - */ -static int -init_data_ll (void) -{ - int lcore; - - RTE_LCORE_FOREACH_SLAVE(lcore) { - lcore_info[lcore].lcore_ll = malloc(sizeof(struct lcore_ll_info)); - if (lcore_info[lcore].lcore_ll == NULL) { - RTE_LOG(ERR, VHOST_CONFIG, "Failed to allocate memory for lcore_ll.\n"); - return -1; - } - - lcore_info[lcore].lcore_ll->device_num = 0; - lcore_info[lcore].lcore_ll->dev_removal_flag = ACK_DEV_REMOVAL; - lcore_info[lcore].lcore_ll->ll_root_used = NULL; - if (num_devices % num_switching_cores) - lcore_info[lcore].lcore_ll->ll_root_free = alloc_data_ll((num_devices / num_switching_cores) + 1); - else - lcore_info[lcore].lcore_ll->ll_root_free = alloc_data_ll(num_devices / num_switching_cores); - } - - /* Allocate devices up to a maximum of MAX_DEVICES. */ - ll_root_free = alloc_data_ll(MIN((num_devices), MAX_DEVICES)); - - return 0; -} -/* - * Remove a device from the specific data core linked list and from the main linked list. The - * rx/tx thread must be set the flag to indicate that it is safe to remove the device. - * used. - */ -static void -destroy_device (volatile struct virtio_net *dev) -{ - struct virtio_net_data_ll *ll_lcore_dev_cur; - struct virtio_net_data_ll *ll_main_dev_cur; - struct virtio_net_data_ll *ll_lcore_dev_last = NULL; - struct virtio_net_data_ll *ll_main_dev_last = NULL; - int lcore; - - dev->flags &= ~VIRTIO_DEV_RUNNING; - - /*set the remove flag. */ - dev->remove = 1; - - while(dev->ready != DEVICE_SAFE_REMOVE) { - rte_pause(); - } - - /* Search for entry to be removed from lcore ll */ - ll_lcore_dev_cur = lcore_info[dev->coreid].lcore_ll->ll_root_used; - while (ll_lcore_dev_cur != NULL) { - if (ll_lcore_dev_cur->dev == dev) { - break; - } else { - ll_lcore_dev_last = ll_lcore_dev_cur; - ll_lcore_dev_cur = ll_lcore_dev_cur->next; - } - } - - /* Search for entry to be removed from main ll */ - ll_main_dev_cur = ll_root_used; - ll_main_dev_last = NULL; - while (ll_main_dev_cur != NULL) { - if (ll_main_dev_cur->dev == dev) { - break; - } else { - ll_main_dev_last = ll_main_dev_cur; - ll_main_dev_cur = ll_main_dev_cur->next; - } - } - - if (ll_lcore_dev_cur == NULL || ll_main_dev_cur == NULL) { - RTE_LOG(ERR, XENHOST, "%s: could find device in per_cpu list or main_list\n", __func__); - return; - } - - /* Remove entries from the lcore and main ll. */ - rm_data_ll_entry(&lcore_info[ll_lcore_dev_cur->dev->coreid].lcore_ll->ll_root_used, ll_lcore_dev_cur, ll_lcore_dev_last); - rm_data_ll_entry(&ll_root_used, ll_main_dev_cur, ll_main_dev_last); - - /* Set the dev_removal_flag on each lcore. */ - RTE_LCORE_FOREACH_SLAVE(lcore) { - lcore_info[lcore].lcore_ll->dev_removal_flag = REQUEST_DEV_REMOVAL; - } - - /* - * Once each core has set the dev_removal_flag to ACK_DEV_REMOVAL we can be sure that - * they can no longer access the device removed from the linked lists and that the devices - * are no longer in use. - */ - RTE_LCORE_FOREACH_SLAVE(lcore) { - while (lcore_info[lcore].lcore_ll->dev_removal_flag != ACK_DEV_REMOVAL) { - rte_pause(); - } - } - - /* Add the entries back to the lcore and main free ll.*/ - put_data_ll_free_entry(&lcore_info[ll_lcore_dev_cur->dev->coreid].lcore_ll->ll_root_free, ll_lcore_dev_cur); - put_data_ll_free_entry(&ll_root_free, ll_main_dev_cur); - - /* Decrement number of device on the lcore. */ - lcore_info[ll_lcore_dev_cur->dev->coreid].lcore_ll->device_num--; - - RTE_LOG(INFO, VHOST_DATA, " #####(%"PRIu64") Device has been removed from data core\n", dev->device_fh); -} - -/* - * A new device is added to a data core. First the device is added to the main linked list - * and the allocated to a specific data core. - */ -static int -new_device (struct virtio_net *dev) -{ - struct virtio_net_data_ll *ll_dev; - int lcore, core_add = 0; - uint32_t device_num_min = num_devices; - - /* Add device to main ll */ - ll_dev = get_data_ll_free_entry(&ll_root_free); - if (ll_dev == NULL) { - RTE_LOG(INFO, VHOST_DATA, "(%"PRIu64") No free entry found in linked list. Device limit " - "of %d devices per core has been reached\n", - dev->device_fh, num_devices); - return -1; - } - ll_dev->dev = dev; - add_data_ll_entry(&ll_root_used, ll_dev); - - /*reset ready flag*/ - dev->ready = DEVICE_NOT_READY; - dev->remove = 0; - - /* Find a suitable lcore to add the device. */ - RTE_LCORE_FOREACH_SLAVE(lcore) { - if (lcore_info[lcore].lcore_ll->device_num < device_num_min) { - device_num_min = lcore_info[lcore].lcore_ll->device_num; - core_add = lcore; - } - } - /* Add device to lcore ll */ - ll_dev->dev->coreid = core_add; - ll_dev = get_data_ll_free_entry(&lcore_info[ll_dev->dev->coreid].lcore_ll->ll_root_free); - if (ll_dev == NULL) { - RTE_LOG(INFO, VHOST_DATA, "(%"PRIu64") Failed to add device to data core\n", dev->device_fh); - destroy_device(dev); - return -1; - } - ll_dev->dev = dev; - add_data_ll_entry(&lcore_info[ll_dev->dev->coreid].lcore_ll->ll_root_used, ll_dev); - - /* Initialize device stats */ - memset(&dev_statistics[dev->device_fh], 0, sizeof(struct device_statistics)); - - lcore_info[ll_dev->dev->coreid].lcore_ll->device_num++; - dev->flags |= VIRTIO_DEV_RUNNING; - - RTE_LOG(INFO, VHOST_DATA, "(%"PRIu64") Device has been added to data core %d\n", dev->device_fh, dev->coreid); - - link_vmdq(dev); - - return 0; -} - -/* - * These callback allow devices to be added to the data core when configuration - * has been fully complete. - */ -static const struct virtio_net_device_ops virtio_net_device_ops = -{ - .new_device = new_device, - .destroy_device = destroy_device, -}; - -/* - * This is a thread will wake up after a period to print stats if the user has - * enabled them. - */ -static void -print_stats(void) -{ - struct virtio_net_data_ll *dev_ll; - uint64_t tx_dropped, rx_dropped; - uint64_t tx, tx_total, rx, rx_total; - uint32_t device_fh; - const char clr[] = { 27, '[', '2', 'J', '\0' }; - const char top_left[] = { 27, '[', '1', ';', '1', 'H','\0' }; - - while(1) { - sleep(enable_stats); - - /* Clear screen and move to top left */ - printf("%s%s", clr, top_left); - - printf("\nDevice statistics ===================================="); - - dev_ll = ll_root_used; - while (dev_ll != NULL) { - device_fh = (uint32_t)dev_ll->dev->device_fh; - tx_total = dev_statistics[device_fh].tx_total; - tx = dev_statistics[device_fh].tx; - tx_dropped = tx_total - tx; - rx_total = rte_atomic64_read(&dev_statistics[device_fh].rx_total); - rx = rte_atomic64_read(&dev_statistics[device_fh].rx); - rx_dropped = rx_total - rx; - - printf("\nStatistics for device %"PRIu32" ------------------------------" - "\nTX total: %"PRIu64"" - "\nTX dropped: %"PRIu64"" - "\nTX successful: %"PRIu64"" - "\nRX total: %"PRIu64"" - "\nRX dropped: %"PRIu64"" - "\nRX successful: %"PRIu64"", - device_fh, - tx_total, - tx_dropped, - tx, - rx_total, - rx_dropped, - rx); - - dev_ll = dev_ll->next; - } - printf("\n======================================================\n"); - } -} - - -int init_virtio_net(struct virtio_net_device_ops const * const ops); - -/* - * Main function, does initialisation and calls the per-lcore functions. The CUSE - * device is also registered here to handle the IOCTLs. - */ -int -main(int argc, char *argv[]) -{ - struct rte_mempool *mbuf_pool; - unsigned lcore_id, core_id = 0; - unsigned nb_ports, valid_num_ports; - int ret; - uint8_t portid; - static pthread_t tid; - char thread_name[RTE_MAX_THREAD_NAME_LEN]; - - /* init EAL */ - ret = rte_eal_init(argc, argv); - if (ret < 0) - rte_exit(EXIT_FAILURE, "Error with EAL initialization\n"); - argc -= ret; - argv += ret; - - /* parse app arguments */ - ret = us_vhost_parse_args(argc, argv); - if (ret < 0) - rte_exit(EXIT_FAILURE, "Invalid argument\n"); - - for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id ++) - if (rte_lcore_is_enabled(lcore_id)) - lcore_ids[core_id ++] = lcore_id; - - if (rte_lcore_count() > RTE_MAX_LCORE) - rte_exit(EXIT_FAILURE,"Not enough cores\n"); - - /*set the number of swithcing cores available*/ - num_switching_cores = rte_lcore_count()-1; - - /* Get the number of physical ports. */ - nb_ports = rte_eth_dev_count(); - - /* - * Update the global var NUM_PORTS and global array PORTS - * and get value of var VALID_NUM_PORTS according to system ports number - */ - valid_num_ports = check_ports_num(nb_ports); - - if ((valid_num_ports == 0) || (valid_num_ports > MAX_SUP_PORTS)) { - RTE_LOG(INFO, VHOST_PORT, "Current enabled port number is %u," - "but only %u port can be enabled\n",num_ports, MAX_SUP_PORTS); - return -1; - } - - /* Create the mbuf pool. */ - mbuf_pool = rte_pktmbuf_pool_create("MBUF_POOL", - NUM_MBUFS_PER_PORT * valid_num_ports, MBUF_CACHE_SIZE, 0, - RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id()); - if (mbuf_pool == NULL) - rte_exit(EXIT_FAILURE, "Cannot create mbuf pool\n"); - - /* initialize all ports */ - for (portid = 0; portid < nb_ports; portid++) { - /* skip ports that are not enabled */ - if ((enabled_port_mask & (1 << portid)) == 0) { - RTE_LOG(INFO, VHOST_PORT, "Skipping disabled port %d\n", portid); - continue; - } - if (port_init(portid, mbuf_pool) != 0) - rte_exit(EXIT_FAILURE, "Cannot initialize network ports\n"); - } - - /* Initialise all linked lists. */ - if (init_data_ll() == -1) - rte_exit(EXIT_FAILURE, "Failed to initialize linked list\n"); - - /* Initialize device stats */ - memset(&dev_statistics, 0, sizeof(dev_statistics)); - - /* Enable stats if the user option is set. */ - if (enable_stats) { - ret = pthread_create(&tid, NULL, (void *)print_stats, NULL); - if (ret != 0) - rte_exit(EXIT_FAILURE, - "Cannot create print-stats thread\n"); - - /* Set thread_name for aid in debugging. */ - snprintf(thread_name, RTE_MAX_THREAD_NAME_LEN, "print-xen-stats"); - ret = rte_thread_setname(tid, thread_name); - if (ret != 0) - RTE_LOG(DEBUG, VHOST_CONFIG, - "Cannot set print-stats name\n"); - } - - /* Launch all data cores. */ - RTE_LCORE_FOREACH_SLAVE(lcore_id) { - rte_eal_remote_launch(switch_worker, mbuf_pool, lcore_id); - } - - init_virtio_xen(&virtio_net_device_ops); - - virtio_monitor_loop(); - return 0; -} diff --git a/dpdk/examples/vhost_xen/main.h b/dpdk/examples/vhost_xen/main.h deleted file mode 100644 index 5ff48fd9..00000000 --- a/dpdk/examples/vhost_xen/main.h +++ /dev/null @@ -1,66 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef _MAIN_H_ -#define _MAIN_H_ - -/* Macros for printing using RTE_LOG */ -#define RTE_LOGTYPE_VHOST_CONFIG RTE_LOGTYPE_USER1 -#define RTE_LOGTYPE_VHOST_DATA RTE_LOGTYPE_USER2 -#define RTE_LOGTYPE_VHOST_PORT RTE_LOGTYPE_USER3 - -/* - * Device linked list structure for data path. - */ -struct virtio_net_data_ll -{ - struct virtio_net *dev; /* Pointer to device created by configuration core. */ - struct virtio_net_data_ll *next; /* Pointer to next device in linked list. */ -}; - -/* - * Structure containing data core specific information. - */ -struct lcore_ll_info -{ - struct virtio_net_data_ll *ll_root_free; /* Pointer to head in free linked list. */ - struct virtio_net_data_ll *ll_root_used; /* Pointer to head of used linked list. */ - uint32_t device_num; /* Number of devices on lcore. */ - volatile uint8_t dev_removal_flag; /* Flag to synchronize device removal. */ -}; - -struct lcore_info -{ - struct lcore_ll_info *lcore_ll; /* Pointer to data core specific lcore_ll_info struct */ -}; -#endif /* _MAIN_H_ */ diff --git a/dpdk/examples/vhost_xen/vhost_monitor.c b/dpdk/examples/vhost_xen/vhost_monitor.c deleted file mode 100644 index fb9606bf..00000000 --- a/dpdk/examples/vhost_xen/vhost_monitor.c +++ /dev/null @@ -1,595 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#if __XEN_LATEST_INTERFACE_VERSION__ < 0x00040200 -#include -#else -#include -#endif -#include -#include -#include - -#include -#include -#include -#include - -#include "virtio-net.h" -#include "xen_vhost.h" - -struct virtio_watch { - struct xs_handle *xs; - int watch_fd; -}; - - -/* device ops to add/remove device to/from data core. */ -static struct virtio_net_device_ops const *notify_ops; - -/* root address of the linked list in the configuration core. */ -static struct virtio_net_config_ll *ll_root = NULL; - -/* root address of VM. */ -static struct xen_guestlist guest_root; - -static struct virtio_watch watch; - -static void -vq_vring_init(struct vhost_virtqueue *vq, unsigned int num, uint8_t *p, - unsigned long align) -{ - vq->size = num; - vq->desc = (struct vring_desc *) p; - vq->avail = (struct vring_avail *) (p + - num * sizeof(struct vring_desc)); - vq->used = (void *) - RTE_ALIGN_CEIL( (uintptr_t)(&vq->avail->ring[num]), align); - -} - -static int -init_watch(void) -{ - struct xs_handle *xs; - int ret; - int fd; - - /* get a connection to the daemon */ - xs = xs_daemon_open(); - if (xs == NULL) { - RTE_LOG(ERR, XENHOST, "xs_daemon_open failed\n"); - return -1; - } - - ret = xs_watch(xs, "/local/domain", "mytoken"); - if (ret == 0) { - RTE_LOG(ERR, XENHOST, "%s: xs_watch failed\n", __func__); - xs_daemon_close(xs); - return -1; - } - - /* We are notified of read availability on the watch via the file descriptor. */ - fd = xs_fileno(xs); - watch.xs = xs; - watch.watch_fd = fd; - - TAILQ_INIT(&guest_root); - return 0; -} - -static struct xen_guest * -get_xen_guest(int dom_id) -{ - struct xen_guest *guest = NULL; - - TAILQ_FOREACH(guest, &guest_root, next) { - if(guest->dom_id == dom_id) - return guest; - } - - return NULL; -} - - -static struct xen_guest * -add_xen_guest(int32_t dom_id) -{ - struct xen_guest *guest = NULL; - - if ((guest = get_xen_guest(dom_id)) != NULL) - return guest; - - guest = calloc(1, sizeof(struct xen_guest)); - if (guest) { - RTE_LOG(ERR, XENHOST, " %s: return newly created guest with %d rings\n", __func__, guest->vring_num); - TAILQ_INSERT_TAIL(&guest_root, guest, next); - guest->dom_id = dom_id; - } - - return guest; -} - -static void -cleanup_device(struct virtio_net_config_ll *ll_dev) -{ - if (ll_dev == NULL) - return; - if (ll_dev->dev.virtqueue_rx) { - rte_free(ll_dev->dev.virtqueue_rx); - ll_dev->dev.virtqueue_rx = NULL; - } - if (ll_dev->dev.virtqueue_tx) { - rte_free(ll_dev->dev.virtqueue_tx); - ll_dev->dev.virtqueue_tx = NULL; - } - free(ll_dev); -} - -/* - * Add entry containing a device to the device configuration linked list. - */ -static void -add_config_ll_entry(struct virtio_net_config_ll *new_ll_dev) -{ - struct virtio_net_config_ll *ll_dev = ll_root; - - /* If ll_dev == NULL then this is the first device so go to else */ - if (ll_dev) { - /* If the 1st device_id != 0 then we insert our device here. */ - if (ll_dev->dev.device_fh != 0) { - new_ll_dev->dev.device_fh = 0; - new_ll_dev->next = ll_dev; - ll_root = new_ll_dev; - } else { - /* increment through the ll until we find un unused device_id, - * insert the device at that entry - */ - while ((ll_dev->next != NULL) && (ll_dev->dev.device_fh == (ll_dev->next->dev.device_fh - 1))) - ll_dev = ll_dev->next; - - new_ll_dev->dev.device_fh = ll_dev->dev.device_fh + 1; - new_ll_dev->next = ll_dev->next; - ll_dev->next = new_ll_dev; - } - } else { - ll_root = new_ll_dev; - ll_root->dev.device_fh = 0; - } -} - - -/* - * Remove an entry from the device configuration linked list. - */ -static struct virtio_net_config_ll * -rm_config_ll_entry(struct virtio_net_config_ll *ll_dev, struct virtio_net_config_ll *ll_dev_last) -{ - /* First remove the device and then clean it up. */ - if (ll_dev == ll_root) { - ll_root = ll_dev->next; - cleanup_device(ll_dev); - return ll_root; - } else { - ll_dev_last->next = ll_dev->next; - cleanup_device(ll_dev); - return ll_dev_last->next; - } -} - -/* - * Retrieves an entry from the devices configuration linked list. - */ -static struct virtio_net_config_ll * -get_config_ll_entry(unsigned int virtio_idx, unsigned int dom_id) -{ - struct virtio_net_config_ll *ll_dev = ll_root; - - /* Loop through linked list until the dom_id is found. */ - while (ll_dev != NULL) { - if (ll_dev->dev.dom_id == dom_id && ll_dev->dev.virtio_idx == virtio_idx) - return ll_dev; - ll_dev = ll_dev->next; - } - - return NULL; -} - -/* - * Initialise all variables in device structure. - */ -static void -init_dev(struct virtio_net *dev) -{ - RTE_SET_USED(dev); -} - - -static struct -virtio_net_config_ll *new_device(unsigned int virtio_idx, struct xen_guest *guest) -{ - struct virtio_net_config_ll *new_ll_dev; - struct vhost_virtqueue *virtqueue_rx, *virtqueue_tx; - size_t size, vq_ring_size, vq_size = VQ_DESC_NUM; - void *vq_ring_virt_mem; - uint64_t gpa; - uint32_t i; - - /* Setup device and virtqueues. */ - new_ll_dev = calloc(1, sizeof(struct virtio_net_config_ll)); - virtqueue_rx = rte_zmalloc(NULL, sizeof(struct vhost_virtqueue), RTE_CACHE_LINE_SIZE); - virtqueue_tx = rte_zmalloc(NULL, sizeof(struct vhost_virtqueue), RTE_CACHE_LINE_SIZE); - if (new_ll_dev == NULL || virtqueue_rx == NULL || virtqueue_tx == NULL) - goto err; - - new_ll_dev->dev.virtqueue_rx = virtqueue_rx; - new_ll_dev->dev.virtqueue_tx = virtqueue_tx; - new_ll_dev->dev.dom_id = guest->dom_id; - new_ll_dev->dev.virtio_idx = virtio_idx; - /* Initialise device and virtqueues. */ - init_dev(&new_ll_dev->dev); - - size = vring_size(vq_size, VIRTIO_PCI_VRING_ALIGN); - vq_ring_size = RTE_ALIGN_CEIL(size, VIRTIO_PCI_VRING_ALIGN); - (void)vq_ring_size; - - vq_ring_virt_mem = guest->vring[virtio_idx].rxvring_addr; - vq_vring_init(virtqueue_rx, vq_size, vq_ring_virt_mem, VIRTIO_PCI_VRING_ALIGN); - virtqueue_rx->size = vq_size; - virtqueue_rx->vhost_hlen = sizeof(struct virtio_net_hdr); - - vq_ring_virt_mem = guest->vring[virtio_idx].txvring_addr; - vq_vring_init(virtqueue_tx, vq_size, vq_ring_virt_mem, VIRTIO_PCI_VRING_ALIGN); - virtqueue_tx->size = vq_size; - memcpy(&new_ll_dev->dev.mac_address, &guest->vring[virtio_idx].addr, sizeof(struct ether_addr)); - - /* virtio_memory has to be one per domid */ - new_ll_dev->dev.mem = malloc(sizeof(struct virtio_memory) + sizeof(struct virtio_memory_regions) * MAX_XENVIRT_MEMPOOL); - new_ll_dev->dev.mem->nregions = guest->pool_num; - for (i = 0; i < guest->pool_num; i++) { - gpa = new_ll_dev->dev.mem->regions[i].guest_phys_address = - (uint64_t)((uintptr_t)guest->mempool[i].gva); - new_ll_dev->dev.mem->regions[i].guest_phys_address_end = - gpa + guest->mempool[i].mempfn_num * getpagesize(); - new_ll_dev->dev.mem->regions[i].address_offset = - (uint64_t)((uintptr_t)guest->mempool[i].hva - - (uintptr_t)gpa); - } - - new_ll_dev->next = NULL; - - /* Add entry to device configuration linked list. */ - add_config_ll_entry(new_ll_dev); - return new_ll_dev; -err: - free(new_ll_dev); - rte_free(virtqueue_rx); - rte_free(virtqueue_tx); - - return NULL; -} - -static void -destroy_guest(struct xen_guest *guest) -{ - uint32_t i; - - for (i = 0; i < guest->vring_num; i++) - cleanup_vring(&guest->vring[i]); - /* clean mempool */ - for (i = 0; i < guest->pool_num; i++) - cleanup_mempool(&guest->mempool[i]); - free(guest); - - return; -} - -/* - * This function will cleanup the device and remove it from device configuration linked list. - */ -static void -destroy_device(unsigned int virtio_idx, unsigned int dom_id) -{ - struct virtio_net_config_ll *ll_dev_cur_ctx, *ll_dev_last = NULL; - struct virtio_net_config_ll *ll_dev_cur = ll_root; - - /* clean virtio device */ - struct xen_guest *guest = NULL; - guest = get_xen_guest(dom_id); - if (guest == NULL) - return; - - /* Find the linked list entry for the device to be removed. */ - ll_dev_cur_ctx = get_config_ll_entry(virtio_idx, dom_id); - while (ll_dev_cur != NULL) { - /* If the device is found or a device that doesn't exist is found then it is removed. */ - if (ll_dev_cur == ll_dev_cur_ctx) { - if ((ll_dev_cur->dev.flags & VIRTIO_DEV_RUNNING)) - notify_ops->destroy_device(&(ll_dev_cur->dev)); - ll_dev_cur = rm_config_ll_entry(ll_dev_cur, ll_dev_last); - } else { - ll_dev_last = ll_dev_cur; - ll_dev_cur = ll_dev_cur->next; - } - } - RTE_LOG(INFO, XENHOST, " %s guest:%p vring:%p rxvring:%p txvring:%p flag:%p\n", - __func__, guest, &guest->vring[virtio_idx], guest->vring[virtio_idx].rxvring_addr, guest->vring[virtio_idx].txvring_addr, guest->vring[virtio_idx].flag); - cleanup_vring(&guest->vring[virtio_idx]); - guest->vring[virtio_idx].removed = 1; - guest->vring_num -= 1; -} - - - - -static void -watch_unmap_event(void) -{ - int i; - struct xen_guest *guest = NULL; - bool remove_request; - - TAILQ_FOREACH(guest, &guest_root, next) { - for (i = 0; i < MAX_VIRTIO; i++) { - if (guest->vring[i].dom_id && guest->vring[i].removed == 0 && *guest->vring[i].flag == 0) { - RTE_LOG(INFO, XENHOST, "\n\n"); - RTE_LOG(INFO, XENHOST, " #####%s: (%d, %d) to be removed\n", - __func__, - guest->vring[i].dom_id, - i); - destroy_device(i, guest->dom_id); - RTE_LOG(INFO, XENHOST, " %s: DOM %u, vring num: %d\n", - __func__, - guest->dom_id, - guest->vring_num); - } - } - } - -_find_next_remove: - guest = NULL; - remove_request = false; - TAILQ_FOREACH(guest, &guest_root, next) { - if (guest->vring_num == 0) { - remove_request = true; - break; - } - } - if (remove_request == true) { - TAILQ_REMOVE(&guest_root, guest, next); - RTE_LOG(INFO, XENHOST, " #####%s: destroy guest (%d)\n", __func__, guest->dom_id); - destroy_guest(guest); - goto _find_next_remove; - } - return; -} - -/* - * OK, if the guest starts first, it is ok. - * if host starts first, it is ok. - * if guest starts, and has run for sometime, and host stops and restarts, - * then last_used_idx 0? how to solve this. */ - -static void virtio_init(void) -{ - uint32_t len, e_num; - uint32_t i,j; - char **dom; - char *status; - int dom_id; - char path[PATH_MAX]; - char node[PATH_MAX]; - xs_transaction_t th; - struct xen_guest *guest; - struct virtio_net_config_ll *net_config; - char *end; - int val; - - /* init env for watch the node */ - if (init_watch() < 0) - return; - - dom = xs_directory(watch.xs, XBT_NULL, "/local/domain", &e_num); - - for (i = 0; i < e_num; i++) { - errno = 0; - dom_id = strtol(dom[i], &end, 0); - if (errno != 0 || end == NULL || dom_id == 0) - continue; - - for (j = 0; j < RTE_MAX_ETHPORTS; j++) { - snprintf(node, PATH_MAX, "%s%d", VIRTIO_START, j); - snprintf(path, PATH_MAX, XEN_VM_NODE_FMT, - dom_id, node); - - th = xs_transaction_start(watch.xs); - status = xs_read(watch.xs, th, path, &len); - xs_transaction_end(watch.xs, th, false); - - if (status == NULL) - break; - - /* if there's any valid virtio device */ - errno = 0; - val = strtol(status, &end, 0); - if (errno != 0 || end == NULL || dom_id == 0) - val = 0; - if (val == 1) { - guest = add_xen_guest(dom_id); - if (guest == NULL) - continue; - RTE_LOG(INFO, XENHOST, " there's a new virtio existed, new a virtio device\n\n"); - - RTE_LOG(INFO, XENHOST, " parse_vringnode dom_id %d virtioidx %d\n",dom_id,j); - if (parse_vringnode(guest, j)) { - RTE_LOG(ERR, XENHOST, " there is invalid information in xenstore\n"); - TAILQ_REMOVE(&guest_root, guest, next); - destroy_guest(guest); - - continue; - } - - /*if pool_num > 0, then mempool has already been parsed*/ - if (guest->pool_num == 0 && parse_mempoolnode(guest)) { - RTE_LOG(ERR, XENHOST, " there is error information in xenstore\n"); - TAILQ_REMOVE(&guest_root, guest, next); - destroy_guest(guest); - continue; - } - - net_config = new_device(j, guest); - /* every thing is ready now, added into data core */ - notify_ops->new_device(&net_config->dev); - } - } - } - - free(dom); - return; -} - -void -virtio_monitor_loop(void) -{ - char **vec; - xs_transaction_t th; - char *buf; - unsigned int len; - unsigned int dom_id; - uint32_t virtio_idx; - struct xen_guest *guest; - struct virtio_net_config_ll *net_config; - enum fieldnames { - FLD_NULL = 0, - FLD_LOCAL, - FLD_DOMAIN, - FLD_ID, - FLD_CONTROL, - FLD_DPDK, - FLD_NODE, - _NUM_FLD - }; - char *str_fld[_NUM_FLD]; - char *str; - char *end; - - virtio_init(); - while (1) { - watch_unmap_event(); - - usleep(50); - vec = xs_check_watch(watch.xs); - - if (vec == NULL) - continue; - - th = xs_transaction_start(watch.xs); - - buf = xs_read(watch.xs, th, vec[XS_WATCH_PATH],&len); - xs_transaction_end(watch.xs, th, false); - - if (buf) { - /* theres' some node for vhost existed */ - if (rte_strsplit(vec[XS_WATCH_PATH], strnlen(vec[XS_WATCH_PATH], PATH_MAX), - str_fld, _NUM_FLD, '/') == _NUM_FLD) { - if (strstr(str_fld[FLD_NODE], VIRTIO_START)) { - errno = 0; - str = str_fld[FLD_ID]; - dom_id = strtoul(str, &end, 0); - if (errno != 0 || end == NULL || end == str ) { - RTE_LOG(INFO, XENHOST, "invalid domain id\n"); - continue; - } - - errno = 0; - str = str_fld[FLD_NODE] + sizeof(VIRTIO_START) - 1; - virtio_idx = strtoul(str, &end, 0); - if (errno != 0 || end == NULL || end == str - || virtio_idx > MAX_VIRTIO) { - RTE_LOG(INFO, XENHOST, "invalid virtio idx\n"); - continue; - } - RTE_LOG(INFO, XENHOST, " #####virtio dev (%d, %d) is started\n", dom_id, virtio_idx); - - guest = add_xen_guest(dom_id); - if (guest == NULL) - continue; - guest->dom_id = dom_id; - if (parse_vringnode(guest, virtio_idx)) { - RTE_LOG(ERR, XENHOST, " there is invalid information in xenstore\n"); - /*guest newly created? guest existed ?*/ - TAILQ_REMOVE(&guest_root, guest, next); - destroy_guest(guest); - continue; - } - /*if pool_num > 0, then mempool has already been parsed*/ - if (guest->pool_num == 0 && parse_mempoolnode(guest)) { - RTE_LOG(ERR, XENHOST, " there is error information in xenstore\n"); - TAILQ_REMOVE(&guest_root, guest, next); - destroy_guest(guest); - continue; - } - - - net_config = new_device(virtio_idx, guest); - RTE_LOG(INFO, XENHOST, " Add to dataplane core\n"); - notify_ops->new_device(&net_config->dev); - - } - } - } - - free(vec); - } - return; -} - -/* - * Register ops so that we can add/remove device to data core. - */ -int -init_virtio_xen(struct virtio_net_device_ops const *const ops) -{ - notify_ops = ops; - if (xenhost_init()) - return -1; - return 0; -} diff --git a/dpdk/examples/vhost_xen/virtio-net.h b/dpdk/examples/vhost_xen/virtio-net.h deleted file mode 100644 index ab697260..00000000 --- a/dpdk/examples/vhost_xen/virtio-net.h +++ /dev/null @@ -1,113 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef _VIRTIO_NET_H_ -#define _VIRTIO_NET_H_ - -#include - -#define VQ_DESC_NUM 256 -/* Used to indicate that the device is running on a data core */ -#define VIRTIO_DEV_RUNNING 1 - -/* - * Structure contains variables relevant to TX/RX virtqueues. - */ -struct vhost_virtqueue -{ - struct vring_desc *desc; /* Virtqueue descriptor ring. */ - struct vring_avail *avail; /* Virtqueue available ring. */ - struct vring_used *used; /* Virtqueue used ring. */ - uint32_t size; /* Size of descriptor ring. */ - uint32_t vhost_hlen; /* Vhost header length (varies depending on RX merge buffers. */ - volatile uint16_t last_used_idx; /* Last index used on the available ring */ - volatile uint16_t last_used_idx_res; /* Used for multiple devices reserving buffers. */ -} __rte_cache_aligned; - -/* - * Device structure contains all configuration information relating to the device. - */ -struct virtio_net -{ - struct vhost_virtqueue *virtqueue_tx; /* Contains all TX virtqueue information. */ - struct vhost_virtqueue *virtqueue_rx; /* Contains all RX virtqueue information. */ - struct virtio_memory *mem; /* QEMU memory and memory region information. */ - struct ether_addr mac_address; /* Device MAC address (Obtained on first TX packet). */ - uint32_t flags; /* Device flags. Only used to check if device is running on data core. */ - uint32_t vlan_tag; /* Vlan tag for device. Currently set to device_id (0-63). */ - uint32_t vmdq_rx_q; - uint64_t device_fh; /* device identifier. */ - uint16_t coreid; - volatile uint8_t ready; /* A device is set as ready if the MAC address has been set. */ - volatile uint8_t remove; /* Device is marked for removal from the data core. */ - uint32_t virtio_idx; /* Index of virtio device */ - uint32_t dom_id; /* Domain id of xen guest */ -} ___rte_cache_aligned; - -/* - * Device linked list structure for configuration. - */ -struct virtio_net_config_ll -{ - struct virtio_net dev; /* Virtio device. */ - struct virtio_net_config_ll *next; /* Next entry on linked list. */ -}; - -/* - * Information relating to memory regions including offsets to addresses in QEMUs memory file. - */ -struct virtio_memory_regions { - uint64_t guest_phys_address; /* Base guest physical address of region. */ - uint64_t guest_phys_address_end; /* End guest physical address of region. */ - uint64_t memory_size; /* Size of region. */ - uint64_t userspace_address; /* Base userspace address of region. */ - uint64_t address_offset; /* Offset of region for address translation. */ -}; - -/* - * Memory structure includes region and mapping information. - */ -struct virtio_memory { - uint32_t nregions; /* Number of memory regions. */ - struct virtio_memory_regions regions[0]; /* Memory region information. */ -}; - -/* - * Device operations to add/remove device. - */ -struct virtio_net_device_ops { - int (* new_device)(struct virtio_net *); /* Add device. */ - void (* destroy_device) (volatile struct virtio_net *); /* Remove device. */ -}; - -#endif diff --git a/dpdk/examples/vhost_xen/xen_vhost.h b/dpdk/examples/vhost_xen/xen_vhost.h deleted file mode 100644 index 2fc304c7..00000000 --- a/dpdk/examples/vhost_xen/xen_vhost.h +++ /dev/null @@ -1,148 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef _XEN_VHOST_H_ -#define _XEN_VHOST_H_ - -#include - -#include - -#include "virtio-net.h" - -#define RTE_LOGTYPE_XENHOST RTE_LOGTYPE_USER1 - -#define XEN_VM_ROOTNODE_FMT "/local/domain/%d/control/dpdk" -#define XEN_VM_NODE_FMT "/local/domain/%d/control/dpdk/%s" -#define XEN_MEMPOOL_SUFFIX "mempool_gref" -#define XEN_RXVRING_SUFFIX "rx_vring_gref" -#define XEN_TXVRING_SUFFIX "tx_vring_gref" -#define XEN_GVA_SUFFIX "mempool_va" -#define XEN_VRINGFLAG_SUFFIX "vring_flag" -#define XEN_ADDR_SUFFIX "ether_addr" -#define VIRTIO_START "event_type_start_" - -#define XEN_GREF_SPLITTOKEN ',' - -#define MAX_XENVIRT_MEMPOOL 16 -#define MAX_VIRTIO 32 -#define MAX_GREF_PER_NODE 64 /* 128 MB memory */ - -#define PAGE_SIZE 4096 -#define PAGE_PFNNUM (PAGE_SIZE / sizeof(uint32_t)) - -#define XEN_GNTDEV_FNAME "/dev/xen/gntdev" - -/* xen grant reference info in one grant node */ -struct xen_gnt { - uint32_t gref; /* grant reference for this node */ - union { - int gref; /* grant reference */ - uint32_t pfn_num; /* guest pfn number of grant reference */ - } gref_pfn[PAGE_PFNNUM]; -}__attribute__((__packed__)); - - -/* structure for mempool or vring node list */ -struct xen_gntnode { - uint32_t gnt_num; /* grant reference number */ - struct xen_gnt *gnt_info; /* grant reference info */ -}; - - -struct xen_vring { - uint32_t dom_id; - uint32_t virtio_idx; /* index of virtio device */ - void *rxvring_addr; /* mapped virtual address of rxvring */ - void *txvring_addr; /* mapped virtual address of txvring */ - uint32_t rxpfn_num; /* number of gpfn for rxvring */ - uint32_t txpfn_num; /* number of gpfn for txvring */ - uint32_t *rxpfn_tbl; /* array of rxvring gpfn */ - uint32_t *txpfn_tbl; /* array of txvring gpfn */ - uint64_t *rx_pindex; /* index used to release rx grefs */ - uint64_t *tx_pindex; /* index used to release tx grefs */ - uint64_t flag_index; - uint8_t *flag; /* cleared to zero on guest unmap */ - struct ether_addr addr; /* ethernet address of virtio device */ - uint8_t removed; - -}; - -struct xen_mempool { - uint32_t dom_id; /* guest domain id */ - uint32_t pool_idx; /* index of memory pool */ - void *gva; /* guest virtual address of mbuf pool */ - void *hva; /* host virtual address of mbuf pool */ - uint32_t mempfn_num; /* number of gpfn for mbuf pool */ - uint32_t *mempfn_tbl; /* array of mbuf pool gpfn */ - uint64_t *pindex; /* index used to release grefs */ -}; - -struct xen_guest { - TAILQ_ENTRY(xen_guest) next; - int32_t dom_id; /* guest domain id */ - uint32_t pool_num; /* number of mbuf pool of the guest */ - uint32_t vring_num; /* number of virtio ports of the guest */ - /* array contain the guest mbuf pool info */ - struct xen_mempool mempool[MAX_XENVIRT_MEMPOOL]; - /* array contain the guest rx/tx vring info */ - struct xen_vring vring[MAX_VIRTIO]; -}; - -TAILQ_HEAD(xen_guestlist, xen_guest); - -int -parse_mempoolnode(struct xen_guest *guest); - -int -xenhost_init(void); - -int -parse_vringnode(struct xen_guest *guest, uint32_t virtio_idx); - -int -parse_mempoolnode(struct xen_guest *guest); - -void -cleanup_mempool(struct xen_mempool *mempool); - -void -cleanup_vring(struct xen_vring *vring); - -void -virtio_monitor_loop(void); - -int -init_virtio_xen(struct virtio_net_device_ops const * const); - -#endif diff --git a/dpdk/examples/vhost_xen/xenstore_parse.c b/dpdk/examples/vhost_xen/xenstore_parse.c deleted file mode 100644 index 26d24320..00000000 --- a/dpdk/examples/vhost_xen/xenstore_parse.c +++ /dev/null @@ -1,775 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#if __XEN_LATEST_INTERFACE_VERSION__ < 0x00040200 -#include -#else -#include -#endif - -#include -#include -#include -#include -#include -#include -#include - -#include "xen_vhost.h" - -/* xenstore handle */ -static struct xs_handle *xs = NULL; - -/* gntdev file descriptor to map grant pages */ -static int d_fd = -1; - -/* - * The grant node format in xenstore for vring/mpool is like: - * idx#_rx_vring_gref = "gref1#, gref2#, gref3#" - * idx#_mempool_gref = "gref1#, gref2#, gref3#" - * each gref# is the grant reference for a shared page. - * In each shared page, we store the grant_node_item items. - */ -struct grant_node_item { - uint32_t gref; - uint32_t pfn; -} __attribute__((packed)); - -int cmdline_parse_etheraddr(void *tk, const char *srcbuf, - void *res, unsigned ressize); - -/* Map grant ref refid at addr_ori*/ -static void * -xen_grant_mmap(void *addr_ori, int domid, int refid, uint64_t *pindex) -{ - struct ioctl_gntdev_map_grant_ref arg; - void *addr = NULL; - int pg_sz = getpagesize(); - - arg.count = 1; - arg.refs[0].domid = domid; - arg.refs[0].ref = refid; - - int rv = ioctl(d_fd, IOCTL_GNTDEV_MAP_GRANT_REF, &arg); - if (rv) { - RTE_LOG(ERR, XENHOST, " %s: (%d,%d) %s (ioctl failed)\n", __func__, - domid, refid, strerror(errno)); - return NULL; - } - - if (addr_ori == NULL) - addr = mmap(addr_ori, pg_sz, PROT_READ|PROT_WRITE, MAP_SHARED, - d_fd, arg.index); - else - addr = mmap(addr_ori, pg_sz, PROT_READ|PROT_WRITE, MAP_SHARED | MAP_FIXED, - d_fd, arg.index); - - if (addr == MAP_FAILED) { - RTE_LOG(ERR, XENHOST, " %s: (%d, %d) %s (map failed)\n", __func__, - domid, refid, strerror(errno)); - return NULL; - } - - if (pindex) - *pindex = arg.index; - - return addr; -} - -/* Unmap one grant ref, and munmap must be called before this */ -static int -xen_unmap_grant_ref(uint64_t index) -{ - struct ioctl_gntdev_unmap_grant_ref arg; - int rv; - - arg.count = 1; - arg.index = index; - rv = ioctl(d_fd, IOCTL_GNTDEV_UNMAP_GRANT_REF, &arg); - if (rv) { - RTE_LOG(ERR, XENHOST, " %s: index 0x%" PRIx64 "unmap failed\n", __func__, index); - return -1; - } - return 0; -} - -/* - * Reserve a virtual address space. - * On success, returns the pointer. On failure, returns NULL. - */ -static void * -get_xen_virtual(size_t size, size_t page_sz) -{ - void *addr; - uintptr_t aligned_addr; - - addr = mmap(NULL, size + page_sz, PROT_READ, MAP_SHARED | MAP_ANONYMOUS, -1, 0); - if (addr == MAP_FAILED) { - RTE_LOG(ERR, XENHOST, "failed get a virtual area\n"); - return NULL; - } - - aligned_addr = RTE_ALIGN_CEIL((uintptr_t)addr, page_sz); - munmap(addr, aligned_addr - (uintptr_t)addr); - munmap((void *)(aligned_addr + size), page_sz + (uintptr_t)addr - aligned_addr); - addr = (void *)(aligned_addr); - - return addr; -} - -static void -free_xen_virtual(void *addr, size_t size, size_t page_sz __rte_unused) -{ - if (addr) - munmap(addr, size); -} - -/* - * Returns val str in xenstore. - * @param path - * Full path string for key - * @return - * Pointer to Val str, NULL on failure - */ -static char * -xen_read_node(char *path, uint32_t *len) -{ - char *buf; - - buf = xs_read(xs, XBT_NULL, path, len); - return buf; -} - -static int -cal_pagenum(struct xen_gnt *gnt) -{ - unsigned int i; - /* - * the items in the page are in the format of - * gref#,pfn#,...,gref#,pfn# - * FIXME, 0 is reserved by system, use it as terminator. - */ - for (i = 0; i < (PAGE_PFNNUM) / 2; i++) { - if (gnt->gref_pfn[i * 2].gref <= 0) - break; - } - - return i; -} - -/* Frees memory allocated to a grant node */ -static void -xen_free_gntnode(struct xen_gntnode *gntnode) -{ - if (gntnode == NULL) - return; - free(gntnode->gnt_info); - free(gntnode); -} - -/* - * Parse a grant node. - * @param domid - * Guest domain id. - * @param path - * Full path string for a grant node, like for the following (key, val) pair - * idx#_mempool_gref = "gref#, gref#, gref#" - * path = 'local/domain/domid/control/dpdk/idx#_mempool_gref' - * gref# is a shared page contain packed (gref,pfn) entries - * @return - * Returns the pointer to xen_gntnode - */ -static struct xen_gntnode * -parse_gntnode(int dom_id, char *path) -{ - char **gref_list = NULL; - uint32_t i, len, gref_num; - void *addr = NULL; - char *buf = NULL; - struct xen_gntnode *gntnode = NULL; - struct xen_gnt *gnt = NULL; - int pg_sz = getpagesize(); - char *end; - uint64_t index; - - if ((buf = xen_read_node(path, &len)) == NULL) - goto err; - - gref_list = malloc(MAX_GREF_PER_NODE * sizeof(char *)); - if (gref_list == NULL) - goto err; - - gref_num = rte_strsplit(buf, len, gref_list, MAX_GREF_PER_NODE, - XEN_GREF_SPLITTOKEN); - if (gref_num == 0) { - RTE_LOG(ERR, XENHOST, " %s: invalid grant node format\n", __func__); - goto err; - } - - gntnode = calloc(1, sizeof(struct xen_gntnode)); - gnt = calloc(gref_num, sizeof(struct xen_gnt)); - if (gnt == NULL || gntnode == NULL) - goto err; - - for (i = 0; i < gref_num; i++) { - errno = 0; - gnt[i].gref = strtol(gref_list[i], &end, 0); - if (errno != 0 || end == NULL || end == gref_list[i] || - (*end != '\0' && *end != XEN_GREF_SPLITTOKEN)) { - RTE_LOG(ERR, XENHOST, " %s: parse grant node item failed\n", __func__); - goto err; - } - addr = xen_grant_mmap(NULL, dom_id, gnt[i].gref, &index); - if (addr == NULL) { - RTE_LOG(ERR, XENHOST, " %s: map gref %u failed\n", __func__, gnt[i].gref); - goto err; - } - RTE_LOG(INFO, XENHOST, " %s: map gref %u to %p\n", __func__, gnt[i].gref, addr); - memcpy(gnt[i].gref_pfn, addr, pg_sz); - if (munmap(addr, pg_sz)) { - RTE_LOG(INFO, XENHOST, " %s: unmap gref %u failed\n", __func__, gnt[i].gref); - goto err; - } - if (xen_unmap_grant_ref(index)) { - RTE_LOG(INFO, XENHOST, " %s: release gref %u failed\n", __func__, gnt[i].gref); - goto err; - } - - } - - gntnode->gnt_num = gref_num; - gntnode->gnt_info = gnt; - - free(buf); - free(gref_list); - return gntnode; - -err: - free(gnt); - free(gntnode); - free(gref_list); - free(buf); - return NULL; -} - -/* - * This function maps grant node of vring or mbuf pool to a continous virtual address space, - * and returns mapped address, pfn array, index array - * @param gntnode - * Pointer to grant node - * @param domid - * Guest domain id - * @param ppfn - * Pointer to pfn array, caller should free this array - * @param pgs - * Pointer to number of pages - * @param ppindex - * Pointer to index array, used to release grefs when to free this node - * @return - * Pointer to mapped virtual address, NULL on failure - */ -static void * -map_gntnode(struct xen_gntnode *gntnode, int domid, uint32_t **ppfn, uint32_t *pgs, uint64_t **ppindex) -{ - struct xen_gnt *gnt; - uint32_t i, j; - size_t total_pages = 0; - void *addr; - uint32_t *pfn; - uint64_t *pindex; - uint32_t pfn_num = 0; - int pg_sz; - - if (gntnode == NULL) - return NULL; - - pg_sz = getpagesize(); - for (i = 0; i < gntnode->gnt_num; i++) { - gnt = gntnode->gnt_info + i; - total_pages += cal_pagenum(gnt); - } - if ((addr = get_xen_virtual(total_pages * pg_sz, pg_sz)) == NULL) { - RTE_LOG(ERR, XENHOST, " %s: failed get_xen_virtual\n", __func__); - return NULL; - } - pfn = calloc(total_pages, (size_t)sizeof(uint32_t)); - pindex = calloc(total_pages, (size_t)sizeof(uint64_t)); - if (pfn == NULL || pindex == NULL) { - free_xen_virtual(addr, total_pages * pg_sz, pg_sz); - free(pfn); - free(pindex); - return NULL; - } - - RTE_LOG(INFO, XENHOST, " %s: total pages:%zu, map to [%p, %p]\n", __func__, total_pages, addr, RTE_PTR_ADD(addr, total_pages * pg_sz - 1)); - for (i = 0; i < gntnode->gnt_num; i++) { - gnt = gntnode->gnt_info + i; - for (j = 0; j < (PAGE_PFNNUM) / 2; j++) { - if ((gnt->gref_pfn[j * 2].gref) <= 0) - goto _end; - /*alternative: batch map, or through libxc*/ - if (xen_grant_mmap(RTE_PTR_ADD(addr, pfn_num * pg_sz), - domid, - gnt->gref_pfn[j * 2].gref, - &pindex[pfn_num]) == NULL) { - goto mmap_failed; - } - pfn[pfn_num] = gnt->gref_pfn[j * 2 + 1].pfn_num; - pfn_num++; - } - } - -mmap_failed: - if (pfn_num) - munmap(addr, pfn_num * pg_sz); - for (i = 0; i < pfn_num; i++) { - xen_unmap_grant_ref(pindex[i]); - } - free(pindex); - free(pfn); - return NULL; - -_end: - if (ppindex) - *ppindex = pindex; - else - free(pindex); - if (ppfn) - *ppfn = pfn; - else - free(pfn); - if (pgs) - *pgs = total_pages; - - return addr; -} - -static int -parse_mpool_va(struct xen_mempool *mempool) -{ - char path[PATH_MAX] = {0}; - char *buf; - uint32_t len; - char *end; - int ret = -1; - - errno = 0; - snprintf(path, sizeof(path), - XEN_VM_ROOTNODE_FMT"/%d_"XEN_GVA_SUFFIX, - mempool->dom_id, mempool->pool_idx); - - if((buf = xen_read_node(path, &len)) == NULL) - goto out; - mempool->gva = (void *)strtoul(buf, &end, 16); - if (errno != 0 || end == NULL || end == buf || *end != '\0') { - mempool->gva = NULL; - goto out; - } - ret = 0; -out: - free(buf); - return ret; -} - -/* - * map mbuf pool - */ -static int -map_mempoolnode(struct xen_gntnode *gntnode, - struct xen_mempool *mempool) -{ - if (gntnode == NULL || mempool == NULL) - return -1; - - mempool->hva = - map_gntnode(gntnode, mempool->dom_id, &mempool->mempfn_tbl, &mempool->mempfn_num, &mempool->pindex); - - RTE_LOG(INFO, XENHOST, " %s: map mempool at %p\n", __func__, (void *)mempool->hva); - if (mempool->hva) - return 0; - else { - return -1; - } -} - -void -cleanup_mempool(struct xen_mempool *mempool) -{ - int pg_sz = getpagesize(); - uint32_t i; - - if (mempool->hva) - munmap(mempool->hva, mempool->mempfn_num * pg_sz); - mempool->hva = NULL; - - if (mempool->pindex) { - RTE_LOG(INFO, XENHOST, " %s: unmap dom %02u mempool%02u %u grefs\n", - __func__, - mempool->dom_id, - mempool->pool_idx, - mempool->mempfn_num); - for (i = 0; i < mempool->mempfn_num; i ++) { - xen_unmap_grant_ref(mempool->pindex[i]); - } - } - mempool->pindex = NULL; - - free(mempool->mempfn_tbl); - mempool->mempfn_tbl = NULL; -} - -/* - * process mempool node idx#_mempool_gref, idx = 0, 1, 2... - * untill we encounter a node that doesn't exist. - */ -int -parse_mempoolnode(struct xen_guest *guest) -{ - uint32_t i, len; - char path[PATH_MAX] = {0}; - struct xen_gntnode *gntnode = NULL; - struct xen_mempool *mempool = NULL; - char *buf; - - bzero(&guest->mempool, MAX_XENVIRT_MEMPOOL * sizeof(guest->mempool[0])); - guest->pool_num = 0; - - while (1) { - /* check if null terminated */ - snprintf(path, sizeof(path), - XEN_VM_ROOTNODE_FMT"/%d_"XEN_MEMPOOL_SUFFIX, - guest->dom_id, - guest->pool_num); - - if ((buf = xen_read_node(path, &len)) != NULL) { - /* this node exists */ - free(buf); - } else { - if (guest->pool_num == 0) { - RTE_LOG(ERR, PMD, "no mempool found\n"); - return -1; - } - break; - } - - mempool = &guest->mempool[guest->pool_num]; - mempool->dom_id = guest->dom_id; - mempool->pool_idx = guest->pool_num; - - RTE_LOG(INFO, XENHOST, " %s: mempool %u parse gntnode %s\n", __func__, guest->pool_num, path); - gntnode = parse_gntnode(guest->dom_id, path); - if (gntnode == NULL) - goto err; - - if (parse_mpool_va(mempool)) - goto err; - - RTE_LOG(INFO, XENHOST, " %s: mempool %u map gntnode %s\n", __func__, guest->pool_num, path); - if (map_mempoolnode(gntnode, mempool)) - goto err; - - xen_free_gntnode(gntnode); - guest->pool_num++; - } - - return 0; -err: - if (gntnode) - xen_free_gntnode(gntnode); - for (i = 0; i < MAX_XENVIRT_MEMPOOL ; i++) { - cleanup_mempool(&guest->mempool[i]); - } - /* reinitialise mempool */ - bzero(&guest->mempool, MAX_XENVIRT_MEMPOOL * sizeof(guest->mempool[0])); - return -1; -} - -static int -xen_map_vringflag(struct xen_vring *vring) -{ - char path[PATH_MAX] = {0}; - char *buf; - uint32_t len,gref; - int pg_sz = getpagesize(); - char *end; - - snprintf(path, sizeof(path), - XEN_VM_ROOTNODE_FMT"/%d_"XEN_VRINGFLAG_SUFFIX, - vring->dom_id, vring->virtio_idx); - - if((buf = xen_read_node(path, &len)) == NULL) - goto err; - - errno = 0; - gref = strtol(buf, &end, 0); - if (errno != 0 || end == NULL || end == buf) { - goto err; - } - vring->flag = xen_grant_mmap(0, vring->dom_id, gref, &vring->flag_index); - if (vring->flag == NULL || *vring->flag == 0) - goto err; - - free(buf); - return 0; -err: - free(buf); - if (vring->flag) { - munmap(vring->flag, pg_sz); - vring->flag = NULL; - xen_unmap_grant_ref(vring->flag_index); - } - return -1; -} - - -static int -xen_map_rxvringnode(struct xen_gntnode *gntnode, - struct xen_vring *vring) -{ - vring->rxvring_addr = - map_gntnode(gntnode, vring->dom_id, &vring->rxpfn_tbl, &vring->rxpfn_num, &vring->rx_pindex); - RTE_LOG(INFO, XENHOST, " %s: map rx vring at %p\n", __func__, (void *)vring->rxvring_addr); - if (vring->rxvring_addr) - return 0; - else - return -1; -} - -static int -xen_map_txvringnode(struct xen_gntnode *gntnode, - struct xen_vring *vring) -{ - vring->txvring_addr = - map_gntnode(gntnode, vring->dom_id, &vring->txpfn_tbl, &vring->txpfn_num, &vring->tx_pindex); - RTE_LOG(INFO, XENHOST, " %s: map tx vring at %p\n", __func__, (void *)vring->txvring_addr); - if (vring->txvring_addr) - return 0; - else - return -1; -} - -void -cleanup_vring(struct xen_vring *vring) -{ - int pg_sz = getpagesize(); - uint32_t i; - - RTE_LOG(INFO, XENHOST, " %s: cleanup dom %u vring %u\n", __func__, vring->dom_id, vring->virtio_idx); - if (vring->rxvring_addr) { - munmap(vring->rxvring_addr, vring->rxpfn_num * pg_sz); - RTE_LOG(INFO, XENHOST, " %s: unmap rx vring [%p, %p]\n", - __func__, - vring->rxvring_addr, - RTE_PTR_ADD(vring->rxvring_addr, - vring->rxpfn_num * pg_sz - 1)); - } - vring->rxvring_addr = NULL; - - - if (vring->rx_pindex) { - RTE_LOG(INFO, XENHOST, " %s: unmap rx vring %u grefs\n", __func__, vring->rxpfn_num); - for (i = 0; i < vring->rxpfn_num; i++) { - xen_unmap_grant_ref(vring->rx_pindex[i]); - } - } - vring->rx_pindex = NULL; - - free(vring->rxpfn_tbl); - vring->rxpfn_tbl = NULL; - - if (vring->txvring_addr) { - munmap(vring->txvring_addr, vring->txpfn_num * pg_sz); - RTE_LOG(INFO, XENHOST, " %s: unmap tx vring [%p, %p]\n", - __func__, - vring->txvring_addr, - RTE_PTR_ADD(vring->txvring_addr, - vring->txpfn_num * pg_sz - 1)); - } - vring->txvring_addr = NULL; - - if (vring->tx_pindex) { - RTE_LOG(INFO, XENHOST, " %s: unmap tx vring %u grefs\n", __func__, vring->txpfn_num); - for (i = 0; i < vring->txpfn_num; i++) { - xen_unmap_grant_ref(vring->tx_pindex[i]); - } - } - vring->tx_pindex = NULL; - - free(vring->txpfn_tbl); - vring->txpfn_tbl = NULL; - - if (vring->flag) { - if (!munmap((void *)vring->flag, pg_sz)) - RTE_LOG(INFO, XENHOST, " %s: unmap flag page at %p\n", __func__, vring->flag); - if (!xen_unmap_grant_ref(vring->flag_index)) - RTE_LOG(INFO, XENHOST, " %s: release flag ref index 0x%" PRIx64 "\n", __func__, vring->flag_index); - } - vring->flag = NULL; - return; -} - - - -static int -xen_parse_etheraddr(struct xen_vring *vring) -{ - char path[PATH_MAX] = {0}; - char *buf; - uint32_t len; - int ret = -1; - - snprintf(path, sizeof(path), - XEN_VM_ROOTNODE_FMT"/%d_"XEN_ADDR_SUFFIX, - vring->dom_id, vring->virtio_idx); - - if ((buf = xen_read_node(path, &len)) == NULL) - goto out; - - if (cmdline_parse_etheraddr(NULL, buf, &vring->addr, - sizeof(vring->addr)) < 0) - goto out; - ret = 0; -out: - free(buf); - return ret; -} - - -int -parse_vringnode(struct xen_guest *guest, uint32_t virtio_idx) -{ - char path[PATH_MAX] = {0}; - struct xen_gntnode *rx_gntnode = NULL; - struct xen_gntnode *tx_gntnode = NULL; - struct xen_vring *vring = NULL; - - /*check if null terminated */ - snprintf(path, sizeof(path), - XEN_VM_ROOTNODE_FMT"/%d_"XEN_RXVRING_SUFFIX, - guest->dom_id, - virtio_idx); - - RTE_LOG(INFO, XENHOST, " %s: virtio %u parse rx gntnode %s\n", __func__, virtio_idx, path); - rx_gntnode = parse_gntnode(guest->dom_id, path); - if (rx_gntnode == NULL) - goto err; - - /*check if null terminated */ - snprintf(path, sizeof(path), - XEN_VM_ROOTNODE_FMT"/%d_"XEN_TXVRING_SUFFIX, - guest->dom_id, - virtio_idx); - - RTE_LOG(INFO, XENHOST, " %s: virtio %u parse tx gntnode %s\n", __func__, virtio_idx, path); - tx_gntnode = parse_gntnode(guest->dom_id, path); - if (tx_gntnode == NULL) - goto err; - - vring = &guest->vring[virtio_idx]; - bzero(vring, sizeof(*vring)); - vring->dom_id = guest->dom_id; - vring->virtio_idx = virtio_idx; - - if (xen_parse_etheraddr(vring) != 0) - goto err; - - RTE_LOG(INFO, XENHOST, " %s: virtio %u map rx gntnode %s\n", __func__, virtio_idx, path); - if (xen_map_rxvringnode(rx_gntnode, vring) != 0) - goto err; - - RTE_LOG(INFO, XENHOST, " %s: virtio %u map tx gntnode %s\n", __func__, virtio_idx, path); - if (xen_map_txvringnode(tx_gntnode, vring) != 0) - goto err; - - if (xen_map_vringflag(vring) != 0) - goto err; - - guest->vring_num++; - - xen_free_gntnode(rx_gntnode); - xen_free_gntnode(tx_gntnode); - - return 0; - -err: - if (rx_gntnode) - xen_free_gntnode(rx_gntnode); - if (tx_gntnode) - xen_free_gntnode(tx_gntnode); - if (vring) { - cleanup_vring(vring); - bzero(vring, sizeof(*vring)); - } - return -1; -} - -/* - * Open xen grant dev driver - * @return - * 0 on success, -1 on failure. - */ -static int -xen_grant_init(void) -{ - d_fd = open(XEN_GNTDEV_FNAME, O_RDWR); - - return d_fd == -1? (-1): (0); -} - -/* - * Initialise xenstore handle and open grant dev driver. - * @return - * 0 on success, -1 on failure. - */ -int -xenhost_init(void) -{ - xs = xs_daemon_open(); - if (xs == NULL) { - rte_panic("failed initialize xen daemon handler"); - return -1; - } - if (xen_grant_init()) - return -1; - return 0; -} diff --git a/dpdk/examples/vm_power_manager/Makefile b/dpdk/examples/vm_power_manager/Makefile index 59a96417..9cf20a28 100644 --- a/dpdk/examples/vm_power_manager/Makefile +++ b/dpdk/examples/vm_power_manager/Makefile @@ -54,6 +54,22 @@ CFLAGS += $(WERROR_FLAGS) LDLIBS += -lvirt +ifeq ($(CONFIG_RTE_BUILD_SHARED_LIB),y) + +ifeq ($(CONFIG_RTE_LIBRTE_IXGBE_PMD),y) +LDLIBS += -lrte_pmd_ixgbe +endif + +ifeq ($(CONFIG_RTE_LIBRTE_I40E_PMD),y) +LDLIBS += -lrte_pmd_i40e +endif + +ifeq ($(CONFIG_RTE_LIBRTE_BNXT_PMD),y) +LDLIBS += -lrte_pmd_bnxt +endif + +endif + # workaround for a gcc bug with noreturn attribute # http://gcc.gnu.org/bugzilla/show_bug.cgi?id=12603 ifeq ($(CONFIG_RTE_TOOLCHAIN_GCC),y) diff --git a/dpdk/examples/vm_power_manager/channel_manager.c b/dpdk/examples/vm_power_manager/channel_manager.c index e068ae28..ab856bdb 100644 --- a/dpdk/examples/vm_power_manager/channel_manager.c +++ b/dpdk/examples/vm_power_manager/channel_manager.c @@ -574,6 +574,73 @@ set_channel_status(const char *vm_name, unsigned *channel_list, return num_channels_changed; } +void +get_all_vm(int *num_vm, int *num_vcpu) +{ + + virNodeInfo node_info; + virDomainPtr *domptr; + uint64_t mask; + int i, ii, numVcpus[MAX_VCPUS], cpu, n_vcpus; + unsigned int jj; + const char *vm_name; + unsigned int domain_flags = VIR_CONNECT_LIST_DOMAINS_RUNNING | + VIR_CONNECT_LIST_DOMAINS_PERSISTENT; + unsigned int domain_flag = VIR_DOMAIN_VCPU_CONFIG; + + + memset(global_cpumaps, 0, CHANNEL_CMDS_MAX_CPUS*global_maplen); + if (virNodeGetInfo(global_vir_conn_ptr, &node_info)) { + RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to retrieve node Info\n"); + return; + } + + /* Returns number of pcpus */ + global_n_host_cpus = (unsigned int)node_info.cpus; + + /* Returns number of active domains */ + *num_vm = virConnectListAllDomains(global_vir_conn_ptr, &domptr, + domain_flags); + if (*num_vm <= 0) { + RTE_LOG(ERR, CHANNEL_MANAGER, "No Active Domains Running\n"); + return; + } + + for (i = 0; i < *num_vm; i++) { + + /* Get Domain Names */ + vm_name = virDomainGetName(domptr[i]); + lvm_info[i].vm_name = vm_name; + + /* Get Number of Vcpus */ + numVcpus[i] = virDomainGetVcpusFlags(domptr[i], domain_flag); + + /* Get Number of VCpus & VcpuPinInfo */ + n_vcpus = virDomainGetVcpuPinInfo(domptr[i], + numVcpus[i], global_cpumaps, + global_maplen, domain_flag); + + if ((int)n_vcpus > 0) { + *num_vcpu = n_vcpus; + lvm_info[i].num_cpus = n_vcpus; + } + + /* Save pcpu in use by libvirt VMs */ + for (ii = 0; ii < n_vcpus; ii++) { + mask = 0; + for (jj = 0; jj < global_n_host_cpus; jj++) { + if (VIR_CPU_USABLE(global_cpumaps, + global_maplen, ii, jj) > 0) { + mask |= 1ULL << jj; + } + } + ITERATIVE_BITMASK_CHECK_64(mask, cpu) { + lvm_info[i].pcpus[ii] = cpu; + } + } + } +} + int get_info_vm(const char *vm_name, struct vm_info *info) { diff --git a/dpdk/examples/vm_power_manager/channel_manager.h b/dpdk/examples/vm_power_manager/channel_manager.h index 47c3b9cd..358fb8f2 100644 --- a/dpdk/examples/vm_power_manager/channel_manager.h +++ b/dpdk/examples/vm_power_manager/channel_manager.h @@ -66,6 +66,17 @@ struct sockaddr_un _sockaddr_un; #define UNIX_PATH_MAX sizeof(_sockaddr_un.sun_path) #endif +#define MAX_VMS 4 +#define MAX_VCPUS 20 + + +struct libvirt_vm_info { + const char *vm_name; + unsigned int pcpus[MAX_VCPUS]; + uint8_t num_cpus; +}; + +struct libvirt_vm_info lvm_info[MAX_VMS]; /* Communication Channel Status */ enum channel_status { CHANNEL_MGR_CHANNEL_DISCONNECTED = 0, CHANNEL_MGR_CHANNEL_CONNECTED, @@ -319,6 +330,20 @@ int set_channel_status(const char *vm_name, unsigned *channel_list, */ int get_info_vm(const char *vm_name, struct vm_info *info); +/** + * Populates a table with all domains running and their physical cpu. + * All information is gathered through libvirt api. + * + * @param num_vm + * modified to store number of active VMs + * + * @param num_vcpu + modified to store number of vcpus active + * + * @return + * void + */ +void get_all_vm(int *num_vm, int *num_vcpu); #ifdef __cplusplus } #endif diff --git a/dpdk/examples/vm_power_manager/channel_monitor.c b/dpdk/examples/vm_power_manager/channel_monitor.c index e7f5cc4a..37e71ed9 100644 --- a/dpdk/examples/vm_power_manager/channel_monitor.c +++ b/dpdk/examples/vm_power_manager/channel_monitor.c @@ -41,13 +41,17 @@ #include #include #include +#include #include #include #include #include +#include +#include +#include - +#include #include "channel_monitor.h" #include "channel_commands.h" #include "channel_manager.h" @@ -57,10 +61,15 @@ #define MAX_EVENTS 256 +uint64_t vsi_pkt_count_prev[384]; +uint64_t rdtsc_prev[384]; +double time_period_ms = 1; static volatile unsigned run_loop = 1; static int global_event_fd; +static unsigned int policy_is_set; static struct epoll_event *global_events_list; +static struct policy policies[MAX_VMS]; void channel_monitor_exit(void) { @@ -68,6 +77,287 @@ void channel_monitor_exit(void) rte_free(global_events_list); } +static void +core_share(int pNo, int z, int x, int t) +{ + if (policies[pNo].core_share[z].pcpu == lvm_info[x].pcpus[t]) { + if (strcmp(policies[pNo].pkt.vm_name, + lvm_info[x].vm_name) != 0) { + policies[pNo].core_share[z].status = 1; + power_manager_scale_core_max( + policies[pNo].core_share[z].pcpu); + } + } +} + +static void +core_share_status(int pNo) +{ + + int noVms, noVcpus, z, x, t; + + get_all_vm(&noVms, &noVcpus); + + /* Reset Core Share Status. */ + for (z = 0; z < noVcpus; z++) + policies[pNo].core_share[z].status = 0; + + /* Foreach vcpu in a policy. */ + for (z = 0; z < policies[pNo].pkt.num_vcpu; z++) { + /* Foreach VM on the platform. */ + for (x = 0; x < noVms; x++) { + /* Foreach vcpu of VMs on platform. */ + for (t = 0; t < lvm_info[x].num_cpus; t++) + core_share(pNo, z, x, t); + } + } +} + +static void +get_pcpu_to_control(struct policy *pol) +{ + + /* Convert vcpu to pcpu. */ + struct vm_info info; + int pcpu, count; + uint64_t mask_u64b; + + RTE_LOG(INFO, CHANNEL_MONITOR, "Looking for pcpu for %s\n", + pol->pkt.vm_name); + get_info_vm(pol->pkt.vm_name, &info); + + for (count = 0; count < pol->pkt.num_vcpu; count++) { + mask_u64b = info.pcpu_mask[pol->pkt.vcpu_to_control[count]]; + for (pcpu = 0; mask_u64b; mask_u64b &= ~(1ULL << pcpu++)) { + if ((mask_u64b >> pcpu) & 1) + pol->core_share[count].pcpu = pcpu; + } + } +} + +static int +get_pfid(struct policy *pol) +{ + + int i, x, ret = 0, nb_ports; + + nb_ports = rte_eth_dev_count(); + for (i = 0; i < pol->pkt.nb_mac_to_monitor; i++) { + + for (x = 0; x < nb_ports; x++) { + ret = rte_pmd_i40e_query_vfid_by_mac(x, + (struct ether_addr *)&(pol->pkt.vfid[i])); + if (ret != -EINVAL) { + pol->port[i] = x; + break; + } + } + if (ret == -EINVAL || ret == -ENOTSUP || ret == ENODEV) { + RTE_LOG(INFO, CHANNEL_MONITOR, + "Error with Policy. MAC not found on " + "attached ports "); + pol->enabled = 0; + return ret; + } + pol->pfid[i] = ret; + } + return 1; +} + +static int +update_policy(struct channel_packet *pkt) +{ + + unsigned int updated = 0; + int i; + + for (i = 0; i < MAX_VMS; i++) { + if (strcmp(policies[i].pkt.vm_name, pkt->vm_name) == 0) { + policies[i].pkt = *pkt; + get_pcpu_to_control(&policies[i]); + if (get_pfid(&policies[i]) == -1) { + updated = 1; + break; + } + core_share_status(i); + policies[i].enabled = 1; + updated = 1; + } + } + if (!updated) { + for (i = 0; i < MAX_VMS; i++) { + if (policies[i].enabled == 0) { + policies[i].pkt = *pkt; + get_pcpu_to_control(&policies[i]); + if (get_pfid(&policies[i]) == -1) + break; + core_share_status(i); + policies[i].enabled = 1; + break; + } + } + } + return 0; +} + +static uint64_t +get_pkt_diff(struct policy *pol) +{ + + uint64_t vsi_pkt_count, + vsi_pkt_total = 0, + vsi_pkt_count_prev_total = 0; + double rdtsc_curr, rdtsc_diff, diff; + int x; + struct rte_eth_stats vf_stats; + + for (x = 0; x < pol->pkt.nb_mac_to_monitor; x++) { + + /*Read vsi stats*/ + if (rte_pmd_i40e_get_vf_stats(x, pol->pfid[x], &vf_stats) == 0) + vsi_pkt_count = vf_stats.ipackets; + else + vsi_pkt_count = -1; + + vsi_pkt_total += vsi_pkt_count; + + vsi_pkt_count_prev_total += vsi_pkt_count_prev[pol->pfid[x]]; + vsi_pkt_count_prev[pol->pfid[x]] = vsi_pkt_count; + } + + rdtsc_curr = rte_rdtsc_precise(); + rdtsc_diff = rdtsc_curr - rdtsc_prev[pol->pfid[x-1]]; + rdtsc_prev[pol->pfid[x-1]] = rdtsc_curr; + + diff = (vsi_pkt_total - vsi_pkt_count_prev_total) * + ((double)rte_get_tsc_hz() / rdtsc_diff); + + return diff; +} + +static void +apply_traffic_profile(struct policy *pol) +{ + + int count; + uint64_t diff = 0; + + diff = get_pkt_diff(pol); + + RTE_LOG(INFO, CHANNEL_MONITOR, "Applying traffic profile\n"); + + if (diff >= (pol->pkt.traffic_policy.max_max_packet_thresh)) { + for (count = 0; count < pol->pkt.num_vcpu; count++) { + if (pol->core_share[count].status != 1) + power_manager_scale_core_max( + pol->core_share[count].pcpu); + } + } else if (diff >= (pol->pkt.traffic_policy.avg_max_packet_thresh)) { + for (count = 0; count < pol->pkt.num_vcpu; count++) { + if (pol->core_share[count].status != 1) + power_manager_scale_core_med( + pol->core_share[count].pcpu); + } + } else if (diff < (pol->pkt.traffic_policy.avg_max_packet_thresh)) { + for (count = 0; count < pol->pkt.num_vcpu; count++) { + if (pol->core_share[count].status != 1) + power_manager_scale_core_min( + pol->core_share[count].pcpu); + } + } +} + +static void +apply_time_profile(struct policy *pol) +{ + + int count, x; + struct timeval tv; + struct tm *ptm; + char time_string[40]; + + /* Obtain the time of day, and convert it to a tm struct. */ + gettimeofday(&tv, NULL); + ptm = localtime(&tv.tv_sec); + /* Format the date and time, down to a single second. */ + strftime(time_string, sizeof(time_string), "%Y-%m-%d %H:%M:%S", ptm); + + for (x = 0; x < HOURS; x++) { + + if (ptm->tm_hour == pol->pkt.timer_policy.busy_hours[x]) { + for (count = 0; count < pol->pkt.num_vcpu; count++) { + if (pol->core_share[count].status != 1) { + power_manager_scale_core_max( + pol->core_share[count].pcpu); + RTE_LOG(INFO, CHANNEL_MONITOR, + "Scaling up core %d to max\n", + pol->core_share[count].pcpu); + } + } + break; + } else if (ptm->tm_hour == + pol->pkt.timer_policy.quiet_hours[x]) { + for (count = 0; count < pol->pkt.num_vcpu; count++) { + if (pol->core_share[count].status != 1) { + power_manager_scale_core_min( + pol->core_share[count].pcpu); + RTE_LOG(INFO, CHANNEL_MONITOR, + "Scaling down core %d to min\n", + pol->core_share[count].pcpu); + } + } + break; + } else if (ptm->tm_hour == + pol->pkt.timer_policy.hours_to_use_traffic_profile[x]) { + apply_traffic_profile(pol); + break; + } + } +} + +static void +apply_workload_profile(struct policy *pol) +{ + + int count; + + if (pol->pkt.workload == HIGH) { + for (count = 0; count < pol->pkt.num_vcpu; count++) { + if (pol->core_share[count].status != 1) + power_manager_scale_core_max( + pol->core_share[count].pcpu); + } + } else if (pol->pkt.workload == MEDIUM) { + for (count = 0; count < pol->pkt.num_vcpu; count++) { + if (pol->core_share[count].status != 1) + power_manager_scale_core_med( + pol->core_share[count].pcpu); + } + } else if (pol->pkt.workload == LOW) { + for (count = 0; count < pol->pkt.num_vcpu; count++) { + if (pol->core_share[count].status != 1) + power_manager_scale_core_min( + pol->core_share[count].pcpu); + } + } +} + +static void +apply_policy(struct policy *pol) +{ + + struct channel_packet *pkt = &pol->pkt; + + /*Check policy to use*/ + if (pkt->policy_to_use == TRAFFIC) + apply_traffic_profile(pol); + else if (pkt->policy_to_use == TIME) + apply_time_profile(pol); + else if (pkt->policy_to_use == WORKLOAD) + apply_workload_profile(pol); +} + + static int process_request(struct channel_packet *pkt, struct channel_info *chan_info) { @@ -105,6 +395,12 @@ process_request(struct channel_packet *pkt, struct channel_info *chan_info) case(CPU_POWER_SCALE_UP): power_manager_scale_core_up(core_num); break; + case(CPU_POWER_ENABLE_TURBO): + power_manager_enable_turbo_core(core_num); + break; + case(CPU_POWER_DISABLE_TURBO): + power_manager_disable_turbo_core(core_num); + break; default: break; } @@ -122,12 +418,25 @@ process_request(struct channel_packet *pkt, struct channel_info *chan_info) case(CPU_POWER_SCALE_UP): power_manager_scale_mask_up(core_mask); break; + case(CPU_POWER_ENABLE_TURBO): + power_manager_enable_turbo_mask(core_mask); + break; + case(CPU_POWER_DISABLE_TURBO): + power_manager_disable_turbo_mask(core_mask); + break; default: break; } } } + + if (pkt->command == PKT_POLICY) { + RTE_LOG(INFO, CHANNEL_MONITOR, "\nProcessing Policy request from Guest\n"); + update_policy(pkt); + policy_is_set = 1; + } + /* Return is not checked as channel status may have been set to DISABLED * from management thread */ @@ -197,9 +506,10 @@ run_channel_monitor(void) struct channel_info *chan_info = (struct channel_info *) global_events_list[i].data.ptr; if ((global_events_list[i].events & EPOLLERR) || - (global_events_list[i].events & EPOLLHUP)) { + (global_events_list[i].events & EPOLLHUP)) { RTE_LOG(DEBUG, CHANNEL_MONITOR, "Remote closed connection for " - "channel '%s'\n", chan_info->channel_path); + "channel '%s'\n", + chan_info->channel_path); remove_channel(&chan_info); continue; } @@ -211,14 +521,17 @@ run_channel_monitor(void) int buffer_len = sizeof(pkt); while (buffer_len > 0) { - n_bytes = read(chan_info->fd, buffer, buffer_len); + n_bytes = read(chan_info->fd, + buffer, buffer_len); if (n_bytes == buffer_len) break; if (n_bytes == -1) { err = errno; - RTE_LOG(DEBUG, CHANNEL_MONITOR, "Received error on " - "channel '%s' read: %s\n", - chan_info->channel_path, strerror(err)); + RTE_LOG(DEBUG, CHANNEL_MONITOR, + "Received error on " + "channel '%s' read: %s\n", + chan_info->channel_path, + strerror(err)); remove_channel(&chan_info); break; } @@ -229,5 +542,14 @@ run_channel_monitor(void) process_request(&pkt, chan_info); } } + rte_delay_us(time_period_ms*1000); + if (policy_is_set) { + int j; + + for (j = 0; j < MAX_VMS; j++) { + if (policies[j].enabled == 1) + apply_policy(&policies[j]); + } + } } } diff --git a/dpdk/examples/vm_power_manager/channel_monitor.h b/dpdk/examples/vm_power_manager/channel_monitor.h index c1386079..b52c1fca 100644 --- a/dpdk/examples/vm_power_manager/channel_monitor.h +++ b/dpdk/examples/vm_power_manager/channel_monitor.h @@ -35,6 +35,24 @@ #define CHANNEL_MONITOR_H_ #include "channel_manager.h" +#include "channel_commands.h" + +struct core_share { + unsigned int pcpu; + /* + * 1 CORE SHARE + * 0 NOT SHARED + */ + int status; +}; + +struct policy { + struct channel_packet pkt; + uint32_t pfid[MAX_VFS]; + uint32_t port[MAX_VFS]; + unsigned int enabled; + struct core_share core_share[MAX_VCPU_PER_VM]; +}; #ifdef __cplusplus extern "C" { diff --git a/dpdk/examples/vm_power_manager/guest_cli/main.c b/dpdk/examples/vm_power_manager/guest_cli/main.c index 5ac98ed3..ac2b1fa5 100644 --- a/dpdk/examples/vm_power_manager/guest_cli/main.c +++ b/dpdk/examples/vm_power_manager/guest_cli/main.c @@ -35,12 +35,10 @@ #include #include #include -#include #include #include #include #include -#include #include */ #include diff --git a/dpdk/examples/vm_power_manager/guest_cli/vm_power_cli_guest.c b/dpdk/examples/vm_power_manager/guest_cli/vm_power_cli_guest.c index 7931135e..63f711e0 100644 --- a/dpdk/examples/vm_power_manager/guest_cli/vm_power_cli_guest.c +++ b/dpdk/examples/vm_power_manager/guest_cli/vm_power_cli_guest.c @@ -45,8 +45,10 @@ #include #include #include +#include #include +#include #include "vm_power_cli_guest.h" @@ -108,6 +110,10 @@ cmd_set_cpu_freq_parsed(void *parsed_result, struct cmdline *cl, ret = rte_power_freq_min(res->lcore_id); else if (!strcmp(res->cmd , "max")) ret = rte_power_freq_max(res->lcore_id); + else if (!strcmp(res->cmd, "enable_turbo")) + ret = rte_power_freq_enable_turbo(res->lcore_id); + else if (!strcmp(res->cmd, "disable_turbo")) + ret = rte_power_freq_disable_turbo(res->lcore_id); if (ret != 1) cmdline_printf(cl, "Error sending message: %s\n", strerror(ret)); } @@ -120,13 +126,14 @@ cmdline_parse_token_string_t cmd_set_cpu_freq_core_num = lcore_id, UINT8); cmdline_parse_token_string_t cmd_set_cpu_freq_cmd_cmd = TOKEN_STRING_INITIALIZER(struct cmd_set_cpu_freq_result, - cmd, "up#down#min#max"); + cmd, "up#down#min#max#enable_turbo#disable_turbo"); cmdline_parse_inst_t cmd_set_cpu_freq_set = { .f = cmd_set_cpu_freq_parsed, .data = NULL, - .help_str = "set_cpu_freq , Set the current " - "frequency for the specified core by scaling up/down/min/max", + .help_str = "set_cpu_freq " + ", " + "adjust the frequency for the specified core.", .tokens = { (void *)&cmd_set_cpu_freq, (void *)&cmd_set_cpu_freq_core_num, @@ -135,8 +142,103 @@ cmdline_parse_inst_t cmd_set_cpu_freq_set = { }, }; +struct cmd_send_policy_result { + cmdline_fixed_string_t send_policy; + cmdline_fixed_string_t cmd; +}; + +union PFID { + struct ether_addr addr; + uint64_t pfid; +}; + +static inline int +send_policy(void) +{ + struct channel_packet pkt; + int ret; + + union PFID pfid; + /* Use port MAC address as the vfid */ + rte_eth_macaddr_get(0, &pfid.addr); + printf("Port %u MAC: %02" PRIx8 ":%02" PRIx8 ":%02" PRIx8 ":" + "%02" PRIx8 ":%02" PRIx8 ":%02" PRIx8 "\n", + 1, + pfid.addr.addr_bytes[0], pfid.addr.addr_bytes[1], + pfid.addr.addr_bytes[2], pfid.addr.addr_bytes[3], + pfid.addr.addr_bytes[4], pfid.addr.addr_bytes[5]); + pkt.vfid[0] = pfid.pfid; + + pkt.nb_mac_to_monitor = 1; + pkt.t_boost_status.tbEnabled = false; + + pkt.vcpu_to_control[0] = 0; + pkt.vcpu_to_control[1] = 1; + pkt.num_vcpu = 2; + /* Dummy Population. */ + pkt.traffic_policy.min_packet_thresh = 96000; + pkt.traffic_policy.avg_max_packet_thresh = 1800000; + pkt.traffic_policy.max_max_packet_thresh = 2000000; + + pkt.timer_policy.busy_hours[0] = 3; + pkt.timer_policy.busy_hours[1] = 4; + pkt.timer_policy.busy_hours[2] = 5; + pkt.timer_policy.quiet_hours[0] = 11; + pkt.timer_policy.quiet_hours[1] = 12; + pkt.timer_policy.quiet_hours[2] = 13; + + pkt.timer_policy.hours_to_use_traffic_profile[0] = 8; + pkt.timer_policy.hours_to_use_traffic_profile[1] = 10; + + pkt.workload = LOW; + pkt.policy_to_use = TIME; + pkt.command = PKT_POLICY; + strcpy(pkt.vm_name, "ubuntu2"); + ret = rte_power_guest_channel_send_msg(&pkt, 1); + if (ret == 0) + return 1; + RTE_LOG(DEBUG, POWER, "Error sending message: %s\n", + ret > 0 ? strerror(ret) : "channel not connected"); + return -1; +} + +static void +cmd_send_policy_parsed(void *parsed_result, struct cmdline *cl, + __attribute__((unused)) void *data) +{ + int ret = -1; + struct cmd_send_policy_result *res = parsed_result; + + if (!strcmp(res->cmd, "now")) { + printf("Sending Policy down now!\n"); + ret = send_policy(); + } + if (ret != 1) + cmdline_printf(cl, "Error sending message: %s\n", + strerror(ret)); +} + +cmdline_parse_token_string_t cmd_send_policy = + TOKEN_STRING_INITIALIZER(struct cmd_send_policy_result, + send_policy, "send_policy"); +cmdline_parse_token_string_t cmd_send_policy_cmd_cmd = + TOKEN_STRING_INITIALIZER(struct cmd_send_policy_result, + cmd, "now"); + +cmdline_parse_inst_t cmd_send_policy_set = { + .f = cmd_send_policy_parsed, + .data = NULL, + .help_str = "send_policy now", + .tokens = { + (void *)&cmd_send_policy, + (void *)&cmd_send_policy_cmd_cmd, + NULL, + }, +}; + cmdline_parse_ctx_t main_ctx[] = { (cmdline_parse_inst_t *)&cmd_quit, + (cmdline_parse_inst_t *)&cmd_send_policy_set, (cmdline_parse_inst_t *)&cmd_set_cpu_freq_set, NULL, }; diff --git a/dpdk/examples/vm_power_manager/guest_cli/vm_power_cli_guest.h b/dpdk/examples/vm_power_manager/guest_cli/vm_power_cli_guest.h index 0c4bdd5b..277eab3d 100644 --- a/dpdk/examples/vm_power_manager/guest_cli/vm_power_cli_guest.h +++ b/dpdk/examples/vm_power_manager/guest_cli/vm_power_cli_guest.h @@ -40,12 +40,6 @@ extern "C" { #include "channel_commands.h" -int guest_channel_host_connect(unsigned lcore_id); - -int guest_channel_send_msg(struct channel_packet *pkt, unsigned lcore_id); - -void guest_channel_host_disconnect(unsigned lcore_id); - void run_cli(__attribute__((unused)) void *arg); #ifdef __cplusplus diff --git a/dpdk/examples/vm_power_manager/main.c b/dpdk/examples/vm_power_manager/main.c index 97178d14..399fbdd4 100644 --- a/dpdk/examples/vm_power_manager/main.c +++ b/dpdk/examples/vm_power_manager/main.c @@ -34,12 +34,10 @@ #include #include #include -#include #include #include #include #include -#include #include #include @@ -51,13 +49,205 @@ #include #include #include +#include +#include +#include #include #include "channel_manager.h" #include "channel_monitor.h" #include "power_manager.h" #include "vm_power_cli.h" +#include +#include +#include +#define RX_RING_SIZE 512 +#define TX_RING_SIZE 512 + +#define NUM_MBUFS 8191 +#define MBUF_CACHE_SIZE 250 +#define BURST_SIZE 32 + +static uint32_t enabled_port_mask; +static volatile bool force_quit; + +/****************/ +static const struct rte_eth_conf port_conf_default = { + .rxmode = { .max_rx_pkt_len = ETHER_MAX_LEN } +}; + +static inline int +port_init(uint16_t port, struct rte_mempool *mbuf_pool) +{ + struct rte_eth_conf port_conf = port_conf_default; + const uint16_t rx_rings = 1, tx_rings = 1; + int retval; + uint16_t q; + + if (port >= rte_eth_dev_count()) + return -1; + + /* Configure the Ethernet device. */ + retval = rte_eth_dev_configure(port, rx_rings, tx_rings, &port_conf); + if (retval != 0) + return retval; + + /* Allocate and set up 1 RX queue per Ethernet port. */ + for (q = 0; q < rx_rings; q++) { + retval = rte_eth_rx_queue_setup(port, q, RX_RING_SIZE, + rte_eth_dev_socket_id(port), NULL, mbuf_pool); + if (retval < 0) + return retval; + } + + /* Allocate and set up 1 TX queue per Ethernet port. */ + for (q = 0; q < tx_rings; q++) { + retval = rte_eth_tx_queue_setup(port, q, TX_RING_SIZE, + rte_eth_dev_socket_id(port), NULL); + if (retval < 0) + return retval; + } + + /* Start the Ethernet port. */ + retval = rte_eth_dev_start(port); + if (retval < 0) + return retval; + + /* Display the port MAC address. */ + struct ether_addr addr; + rte_eth_macaddr_get(port, &addr); + printf("Port %u MAC: %02" PRIx8 " %02" PRIx8 " %02" PRIx8 + " %02" PRIx8 " %02" PRIx8 " %02" PRIx8 "\n", + (unsigned int)port, + addr.addr_bytes[0], addr.addr_bytes[1], + addr.addr_bytes[2], addr.addr_bytes[3], + addr.addr_bytes[4], addr.addr_bytes[5]); + + /* Enable RX in promiscuous mode for the Ethernet device. */ + rte_eth_promiscuous_enable(port); + + + return 0; +} + +static int +parse_portmask(const char *portmask) +{ + char *end = NULL; + unsigned long pm; + + /* parse hexadecimal string */ + pm = strtoul(portmask, &end, 16); + if ((portmask[0] == '\0') || (end == NULL) || (*end != '\0')) + return -1; + + if (pm == 0) + return -1; + + return pm; +} +/* Parse the argument given in the command line of the application */ +static int +parse_args(int argc, char **argv) +{ + int opt, ret; + char **argvopt; + int option_index; + char *prgname = argv[0]; + static struct option lgopts[] = { + { "mac-updating", no_argument, 0, 1}, + { "no-mac-updating", no_argument, 0, 0}, + {NULL, 0, 0, 0} + }; + argvopt = argv; + + while ((opt = getopt_long(argc, argvopt, "p:q:T:", + lgopts, &option_index)) != EOF) { + + switch (opt) { + /* portmask */ + case 'p': + enabled_port_mask = parse_portmask(optarg); + if (enabled_port_mask == 0) { + printf("invalid portmask\n"); + return -1; + } + break; + /* long options */ + case 0: + break; + + default: + return -1; + } + } + + if (optind >= 0) + argv[optind-1] = prgname; + + ret = optind-1; + optind = 0; /* reset getopt lib */ + return ret; +} + +static void +check_all_ports_link_status(uint16_t port_num, uint32_t port_mask) +{ +#define CHECK_INTERVAL 100 /* 100ms */ +#define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ + uint16_t portid, count, all_ports_up, print_flag = 0; + struct rte_eth_link link; + + printf("\nChecking link status"); + fflush(stdout); + for (count = 0; count <= MAX_CHECK_TIME; count++) { + if (force_quit) + return; + all_ports_up = 1; + for (portid = 0; portid < port_num; portid++) { + if (force_quit) + return; + if ((port_mask & (1 << portid)) == 0) + continue; + memset(&link, 0, sizeof(link)); + rte_eth_link_get_nowait(portid, &link); + /* print link status if flag set */ + if (print_flag == 1) { + if (link.link_status) + printf("Port %d Link Up - speed %u " + "Mbps - %s\n", (uint16_t)portid, + (unsigned int)link.link_speed, + (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? + ("full-duplex") : ("half-duplex\n")); + else + printf("Port %d Link Down\n", + (uint16_t)portid); + continue; + } + /* clear all_ports_up flag if any link down */ + if (link.link_status == ETH_LINK_DOWN) { + all_ports_up = 0; + break; + } + } + /* after finally printing all link status, get out */ + if (print_flag == 1) + break; + + if (all_ports_up == 0) { + printf("."); + fflush(stdout); + rte_delay_ms(CHECK_INTERVAL); + } + + /* set the print_flag if all ports up or timeout */ + if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { + print_flag = 1; + printf("done\n"); + } + } +} static int run_monitor(__attribute__((unused)) void *arg) { @@ -84,6 +274,10 @@ main(int argc, char **argv) { int ret; unsigned lcore_id; + unsigned int nb_ports; + struct rte_mempool *mbuf_pool; + uint16_t portid; + ret = rte_eal_init(argc, argv); if (ret < 0) @@ -92,12 +286,77 @@ main(int argc, char **argv) signal(SIGINT, sig_handler); signal(SIGTERM, sig_handler); + argc -= ret; + argv += ret; + + /* parse application arguments (after the EAL ones) */ + ret = parse_args(argc, argv); + if (ret < 0) + rte_exit(EXIT_FAILURE, "Invalid arguments\n"); + + nb_ports = rte_eth_dev_count(); + + mbuf_pool = rte_pktmbuf_pool_create("MBUF_POOL", NUM_MBUFS * nb_ports, + MBUF_CACHE_SIZE, 0, RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id()); + + if (mbuf_pool == NULL) + rte_exit(EXIT_FAILURE, "Cannot create mbuf pool\n"); + + /* Initialize ports. */ + for (portid = 0; portid < nb_ports; portid++) { + struct ether_addr eth; + int w, j; + int ret = -ENOTSUP; + + if ((enabled_port_mask & (1 << portid)) == 0) + continue; + + eth.addr_bytes[0] = 0xe0; + eth.addr_bytes[1] = 0xe0; + eth.addr_bytes[2] = 0xe0; + eth.addr_bytes[3] = 0xe0; + eth.addr_bytes[4] = portid + 0xf0; + + if (port_init(portid, mbuf_pool) != 0) + rte_exit(EXIT_FAILURE, "Cannot init port %"PRIu8 "\n", + portid); + + for (w = 0; w < MAX_VFS; w++) { + eth.addr_bytes[5] = w + 0xf0; + + if (ret == -ENOTSUP) + ret = rte_pmd_ixgbe_set_vf_mac_addr(portid, + w, ð); + if (ret == -ENOTSUP) + ret = rte_pmd_i40e_set_vf_mac_addr(portid, + w, ð); + if (ret == -ENOTSUP) + ret = rte_pmd_bnxt_set_vf_mac_addr(portid, + w, ð); + + switch (ret) { + case 0: + printf("Port %d VF %d MAC: ", + portid, w); + for (j = 0; j < 6; j++) { + printf("%02x", eth.addr_bytes[j]); + if (j < 5) + printf(":"); + } + printf("\n"); + break; + } + } + } + lcore_id = rte_get_next_lcore(-1, 1, 0); if (lcore_id == RTE_MAX_LCORE) { RTE_LOG(ERR, EAL, "A minimum of two cores are required to run " "application\n"); return 0; } + + check_all_ports_link_status(nb_ports, enabled_port_mask); rte_eal_remote_launch(run_monitor, NULL, lcore_id); if (power_manager_init() < 0) { diff --git a/dpdk/examples/vm_power_manager/power_manager.c b/dpdk/examples/vm_power_manager/power_manager.c index 2644fce6..1834a823 100644 --- a/dpdk/examples/vm_power_manager/power_manager.c +++ b/dpdk/examples/vm_power_manager/power_manager.c @@ -108,7 +108,7 @@ set_host_cpus_mask(void) int power_manager_init(void) { - unsigned i, num_cpus; + unsigned int i, num_cpus, num_freqs; uint64_t cpu_mask; int ret = 0; @@ -121,15 +121,21 @@ power_manager_init(void) rte_power_set_env(PM_ENV_ACPI_CPUFREQ); cpu_mask = global_enabled_cpus; for (i = 0; cpu_mask; cpu_mask &= ~(1 << i++)) { - if (rte_power_init(i) < 0 || rte_power_freqs(i, - global_core_freq_info[i].freqs, - RTE_MAX_LCORE_FREQS) == 0) { - RTE_LOG(ERR, POWER_MANAGER, "Unable to initialize power manager " + if (rte_power_init(i) < 0) + RTE_LOG(ERR, POWER_MANAGER, + "Unable to initialize power manager " "for core %u\n", i); + num_freqs = rte_power_freqs(i, global_core_freq_info[i].freqs, + RTE_MAX_LCORE_FREQS); + if (num_freqs == 0) { + RTE_LOG(ERR, POWER_MANAGER, + "Unable to get frequency list for core %u\n", + i); global_enabled_cpus &= ~(1 << i); num_cpus--; ret = -1; } + global_core_freq_info[i].num_freqs = num_freqs; rte_spinlock_init(&global_core_freq_info[i].power_sl); } RTE_LOG(INFO, POWER_MANAGER, "Detected %u host CPUs , enabled core mask:" @@ -215,6 +221,24 @@ power_manager_scale_mask_max(uint64_t core_mask) return ret; } +int +power_manager_enable_turbo_mask(uint64_t core_mask) +{ + int ret = 0; + + POWER_SCALE_MASK(enable_turbo, core_mask, ret); + return ret; +} + +int +power_manager_disable_turbo_mask(uint64_t core_mask) +{ + int ret = 0; + + POWER_SCALE_MASK(disable_turbo, core_mask, ret); + return ret; +} + int power_manager_scale_core_up(unsigned core_num) { @@ -250,3 +274,37 @@ power_manager_scale_core_max(unsigned core_num) POWER_SCALE_CORE(max, core_num, ret); return ret; } + +int +power_manager_enable_turbo_core(unsigned int core_num) +{ + int ret = 0; + + POWER_SCALE_CORE(enable_turbo, core_num, ret); + return ret; +} + +int +power_manager_disable_turbo_core(unsigned int core_num) +{ + int ret = 0; + + POWER_SCALE_CORE(disable_turbo, core_num, ret); + return ret; +} + +int +power_manager_scale_core_med(unsigned int core_num) +{ + int ret = 0; + + if (core_num >= POWER_MGR_MAX_CPUS) + return -1; + if (!(global_enabled_cpus & (1ULL << core_num))) + return -1; + rte_spinlock_lock(&global_core_freq_info[core_num].power_sl); + ret = rte_power_set_freq(core_num, + global_core_freq_info[core_num].num_freqs / 2); + rte_spinlock_unlock(&global_core_freq_info[core_num].power_sl); + return ret; +} diff --git a/dpdk/examples/vm_power_manager/power_manager.h b/dpdk/examples/vm_power_manager/power_manager.h index 1b45babf..b52fb4c7 100644 --- a/dpdk/examples/vm_power_manager/power_manager.h +++ b/dpdk/examples/vm_power_manager/power_manager.h @@ -112,6 +112,32 @@ int power_manager_scale_mask_min(uint64_t core_mask); */ int power_manager_scale_mask_max(uint64_t core_mask); +/** + * Enable Turbo Boost on the cores specified in core_mask. + * It is thread-safe. + * + * @param core_mask + * The uint64_t bit-mask of cores to change frequency. + * + * @return + * - 1 on success. + * - Negative on error. + */ +int power_manager_enable_turbo_mask(uint64_t core_mask); + +/** + * Disable Turbo Boost on the cores specified in core_mask. + * It is thread-safe. + * + * @param core_mask + * The uint64_t bit-mask of cores to change frequency. + * + * @return + * - 1 on success. + * - Negative on error. + */ +int power_manager_disable_turbo_mask(uint64_t core_mask); + /** * Scale up frequency for the core specified by core_num. * It is thread-safe. @@ -167,6 +193,32 @@ int power_manager_scale_core_min(unsigned core_num); */ int power_manager_scale_core_max(unsigned core_num); +/** + * Enable Turbo Boost for the core specified by core_num. + * It is thread-safe. + * + * @param core_num + * The core number to boost + * + * @return + * - 1 on success. + * - Negative on error. + */ +int power_manager_enable_turbo_core(unsigned int core_num); + +/** + * Disable Turbo Boost for the core specified by core_num. + * It is thread-safe. + * + * @param core_num + * The core number to boost + * + * @return + * - 1 on success. + * - Negative on error. + */ +int power_manager_disable_turbo_core(unsigned int core_num); + /** * Get the current freuency of the core specified by core_num * @@ -179,6 +231,19 @@ int power_manager_scale_core_max(unsigned core_num); */ uint32_t power_manager_get_current_frequency(unsigned core_num); +/** + * Scale to medium frequency for the core specified by core_num. + * It is thread-safe. + * + * @param core_num + * The core number to change frequency + * + * @return + * - 1 on success. + * - 0 if frequency not changed. + * - Negative on error. + */ +int power_manager_scale_core_med(unsigned int core_num); #ifdef __cplusplus } diff --git a/dpdk/examples/vm_power_manager/vm_power_cli.c b/dpdk/examples/vm_power_manager/vm_power_cli.c index c5e8d934..6f234fb7 100644 --- a/dpdk/examples/vm_power_manager/vm_power_cli.c +++ b/dpdk/examples/vm_power_manager/vm_power_cli.c @@ -520,6 +520,10 @@ cmd_set_cpu_freq_mask_parsed(void *parsed_result, struct cmdline *cl, ret = power_manager_scale_mask_min(res->core_mask); else if (!strcmp(res->cmd , "max")) ret = power_manager_scale_mask_max(res->core_mask); + else if (!strcmp(res->cmd, "enable_turbo")) + ret = power_manager_enable_turbo_mask(res->core_mask); + else if (!strcmp(res->cmd, "disable_turbo")) + ret = power_manager_disable_turbo_mask(res->core_mask); if (ret < 0) { cmdline_printf(cl, "Error scaling core_mask(0x%"PRIx64") '%s' , not " "all cores specified have been scaled\n", @@ -535,14 +539,13 @@ cmdline_parse_token_num_t cmd_set_cpu_freq_mask_core_mask = core_mask, UINT64); cmdline_parse_token_string_t cmd_set_cpu_freq_mask_result = TOKEN_STRING_INITIALIZER(struct cmd_set_cpu_freq_mask_result, - cmd, "up#down#min#max"); + cmd, "up#down#min#max#enable_turbo#disable_turbo"); cmdline_parse_inst_t cmd_set_cpu_freq_mask_set = { .f = cmd_set_cpu_freq_mask_parsed, .data = NULL, - .help_str = "set_cpu_freq , Set the current " - "frequency for the cores specified in by scaling " - "each up/down/min/max.", + .help_str = "set_cpu_freq , adjust the current " + "frequency for the cores specified in ", .tokens = { (void *)&cmd_set_cpu_freq_mask, (void *)&cmd_set_cpu_freq_mask_core_mask, @@ -614,6 +617,10 @@ cmd_set_cpu_freq_parsed(void *parsed_result, struct cmdline *cl, ret = power_manager_scale_core_min(res->core_num); else if (!strcmp(res->cmd , "max")) ret = power_manager_scale_core_max(res->core_num); + else if (!strcmp(res->cmd, "enable_turbo")) + ret = power_manager_enable_turbo_core(res->core_num); + else if (!strcmp(res->cmd, "disable_turbo")) + ret = power_manager_disable_turbo_core(res->core_num); if (ret < 0) { cmdline_printf(cl, "Error scaling core(%u) '%s'\n", res->core_num, res->cmd); @@ -628,13 +635,13 @@ cmdline_parse_token_num_t cmd_set_cpu_freq_core_num = core_num, UINT8); cmdline_parse_token_string_t cmd_set_cpu_freq_cmd_cmd = TOKEN_STRING_INITIALIZER(struct cmd_set_cpu_freq_result, - cmd, "up#down#min#max"); + cmd, "up#down#min#max#enable_turbo#disable_turbo"); cmdline_parse_inst_t cmd_set_cpu_freq_set = { .f = cmd_set_cpu_freq_parsed, .data = NULL, - .help_str = "set_cpu_freq , Set the current " - "frequency for the specified core by scaling up/down/min/max", + .help_str = "set_cpu_freq , adjust the current " + "frequency for the specified core", .tokens = { (void *)&cmd_set_cpu_freq, (void *)&cmd_set_cpu_freq_core_num, diff --git a/dpdk/examples/vmdq/Makefile b/dpdk/examples/vmdq/Makefile index 198e3bfe..50172822 100644 --- a/dpdk/examples/vmdq/Makefile +++ b/dpdk/examples/vmdq/Makefile @@ -33,7 +33,7 @@ ifeq ($(RTE_SDK),) $(error "Please define RTE_SDK environment variable") endif -# Default target, can be overriden by command line or environment +# Default target, can be overridden by command line or environment RTE_TARGET ?= x86_64-native-linuxapp-gcc include $(RTE_SDK)/mk/rte.vars.mk diff --git a/dpdk/examples/vmdq/main.c b/dpdk/examples/vmdq/main.c index 360492ba..84e9937d 100644 --- a/dpdk/examples/vmdq/main.c +++ b/dpdk/examples/vmdq/main.c @@ -47,9 +47,7 @@ #include #include #include -#include #include -#include #include #include #include @@ -58,16 +56,12 @@ #include #include #include -#include #include #include #include #include -#include -#include #include #include -#include #define MAX_QUEUES 1024 /* @@ -125,7 +119,7 @@ static const struct rte_eth_conf vmdq_conf_default = { }; static unsigned lcore_ids[RTE_MAX_LCORE]; -static uint8_t ports[RTE_MAX_ETHPORTS]; +static uint16_t ports[RTE_MAX_ETHPORTS]; static unsigned num_ports; /**< The number of ports specified in command line */ /* array used for printing out statistics */ @@ -190,13 +184,14 @@ get_eth_conf(struct rte_eth_conf *eth_conf, uint32_t num_pools) * coming from the mbuf_pool passed as parameter */ static inline int -port_init(uint8_t port, struct rte_mempool *mbuf_pool) +port_init(uint16_t port, struct rte_mempool *mbuf_pool) { struct rte_eth_dev_info dev_info; struct rte_eth_rxconf *rxconf; struct rte_eth_conf port_conf; uint16_t rxRings, txRings; - const uint16_t rxRingSize = RTE_TEST_RX_DESC_DEFAULT, txRingSize = RTE_TEST_TX_DESC_DEFAULT; + uint16_t rxRingSize = RTE_TEST_RX_DESC_DEFAULT; + uint16_t txRingSize = RTE_TEST_TX_DESC_DEFAULT; int retval; uint16_t q; uint16_t queues_per_pool; @@ -254,6 +249,17 @@ port_init(uint8_t port, struct rte_mempool *mbuf_pool) if (retval != 0) return retval; + retval = rte_eth_dev_adjust_nb_rx_tx_desc(port, &rxRingSize, + &txRingSize); + if (retval != 0) + return retval; + if (RTE_MAX(rxRingSize, txRingSize) > RTE_MAX(RTE_TEST_RX_DESC_DEFAULT, + RTE_TEST_TX_DESC_DEFAULT)) { + printf("Mbuf pool has an insufficient size for port %u.\n", + port); + return -1; + } + rte_eth_dev_info_get(port, &dev_info); rxconf = &dev_info.default_rxconf; rxconf->rx_drop_en = 1; @@ -575,7 +581,7 @@ main(int argc, char *argv[]) unsigned lcore_id, core_id = 0; int ret; unsigned nb_ports, valid_num_ports; - uint8_t portid; + uint16_t portid; signal(SIGHUP, sighup_handler); diff --git a/dpdk/examples/vmdq_dcb/Makefile b/dpdk/examples/vmdq_dcb/Makefile index 8c51131b..0c200a98 100644 --- a/dpdk/examples/vmdq_dcb/Makefile +++ b/dpdk/examples/vmdq_dcb/Makefile @@ -33,7 +33,7 @@ ifeq ($(RTE_SDK),) $(error "Please define RTE_SDK environment variable") endif -# Default target, can be overriden by command line or environment +# Default target, can be overridden by command line or environment RTE_TARGET ?= x86_64-native-linuxapp-gcc include $(RTE_SDK)/mk/rte.vars.mk diff --git a/dpdk/examples/vmdq_dcb/main.c b/dpdk/examples/vmdq_dcb/main.c index 617263b4..9dad2b8e 100644 --- a/dpdk/examples/vmdq_dcb/main.c +++ b/dpdk/examples/vmdq_dcb/main.c @@ -47,9 +47,7 @@ #include #include #include -#include #include -#include #include #include #include @@ -58,16 +56,12 @@ #include #include #include -#include #include #include #include #include -#include -#include #include #include -#include /* basic constants used in application */ #define MAX_QUEUES 1024 @@ -91,7 +85,7 @@ /* mask of enabled ports */ static uint32_t enabled_port_mask; -static uint8_t ports[RTE_MAX_ETHPORTS]; +static uint16_t ports[RTE_MAX_ETHPORTS]; static unsigned num_ports; /* number of pools (if user does not specify any, 32 by default */ @@ -224,12 +218,12 @@ get_eth_conf(struct rte_eth_conf *eth_conf) * coming from the mbuf_pool passed as parameter */ static inline int -port_init(uint8_t port, struct rte_mempool *mbuf_pool) +port_init(uint16_t port, struct rte_mempool *mbuf_pool) { struct rte_eth_dev_info dev_info; struct rte_eth_conf port_conf = {0}; - const uint16_t rxRingSize = RTE_TEST_RX_DESC_DEFAULT; - const uint16_t txRingSize = RTE_TEST_TX_DESC_DEFAULT; + uint16_t rxRingSize = RTE_TEST_RX_DESC_DEFAULT; + uint16_t txRingSize = RTE_TEST_TX_DESC_DEFAULT; int retval; uint16_t q; uint16_t queues_per_pool; @@ -300,6 +294,17 @@ port_init(uint8_t port, struct rte_mempool *mbuf_pool) if (retval != 0) return retval; + retval = rte_eth_dev_adjust_nb_rx_tx_desc(port, &rxRingSize, + &txRingSize); + if (retval != 0) + return retval; + if (RTE_MAX(rxRingSize, txRingSize) > + RTE_MAX(RTE_TEST_RX_DESC_DEFAULT, RTE_TEST_TX_DESC_DEFAULT)) { + printf("Mbuf pool has an insufficient size for port %u.\n", + port); + return -1; + } + for (q = 0; q < num_queues; q++) { retval = rte_eth_rx_queue_setup(port, q, rxRingSize, rte_eth_dev_socket_id(port), @@ -639,7 +644,7 @@ main(int argc, char *argv[]) uintptr_t i; int ret; unsigned nb_ports, valid_num_ports; - uint8_t portid; + uint16_t portid; signal(SIGHUP, sighup_handler); diff --git a/dpdk/lib/Makefile b/dpdk/lib/Makefile index ca7c02fd..26113cda 100644 --- a/dpdk/lib/Makefile +++ b/dpdk/lib/Makefile @@ -1,6 +1,6 @@ # BSD LICENSE # -# Copyright(c) 2010-2015 Intel Corporation. All rights reserved. +# Copyright(c) 2010-2017 Intel Corporation. All rights reserved. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -33,35 +33,98 @@ include $(RTE_SDK)/mk/rte.vars.mk DIRS-y += librte_compat DIRS-$(CONFIG_RTE_LIBRTE_EAL) += librte_eal +DIRS-$(CONFIG_RTE_LIBRTE_PCI) += librte_pci +DEPDIRS-librte_pci := librte_eal DIRS-$(CONFIG_RTE_LIBRTE_RING) += librte_ring +DEPDIRS-librte_ring := librte_eal DIRS-$(CONFIG_RTE_LIBRTE_MEMPOOL) += librte_mempool +DEPDIRS-librte_mempool := librte_eal librte_ring DIRS-$(CONFIG_RTE_LIBRTE_MBUF) += librte_mbuf +DEPDIRS-librte_mbuf := librte_eal librte_mempool DIRS-$(CONFIG_RTE_LIBRTE_TIMER) += librte_timer +DEPDIRS-librte_timer := librte_eal DIRS-$(CONFIG_RTE_LIBRTE_CFGFILE) += librte_cfgfile DIRS-$(CONFIG_RTE_LIBRTE_CMDLINE) += librte_cmdline +DEPDIRS-librte_cmdline := librte_eal DIRS-$(CONFIG_RTE_LIBRTE_ETHER) += librte_ether +DEPDIRS-librte_ether := librte_net librte_eal librte_mempool librte_ring +DEPDIRS-librte_ether += librte_mbuf DIRS-$(CONFIG_RTE_LIBRTE_CRYPTODEV) += librte_cryptodev +DEPDIRS-librte_cryptodev := librte_eal librte_mempool librte_ring librte_mbuf +DEPDIRS-librte_cryptodev += librte_kvargs +DIRS-$(CONFIG_RTE_LIBRTE_SECURITY) += librte_security +DEPDIRS-librte_security := librte_eal librte_mempool librte_ring librte_mbuf +DEPDIRS-librte_security += librte_ether +DEPDIRS-librte_security += librte_cryptodev +DIRS-$(CONFIG_RTE_LIBRTE_EVENTDEV) += librte_eventdev +DEPDIRS-librte_eventdev := librte_eal librte_ring librte_ether librte_hash DIRS-$(CONFIG_RTE_LIBRTE_VHOST) += librte_vhost +DEPDIRS-librte_vhost := librte_eal librte_mempool librte_mbuf librte_ether DIRS-$(CONFIG_RTE_LIBRTE_HASH) += librte_hash +DEPDIRS-librte_hash := librte_eal librte_ring +DIRS-$(CONFIG_RTE_LIBRTE_EFD) += librte_efd +DEPDIRS-librte_efd := librte_eal librte_ring librte_hash DIRS-$(CONFIG_RTE_LIBRTE_LPM) += librte_lpm +DEPDIRS-librte_lpm := librte_eal DIRS-$(CONFIG_RTE_LIBRTE_ACL) += librte_acl +DEPDIRS-librte_acl := librte_eal +DIRS-$(CONFIG_RTE_LIBRTE_MEMBER) += librte_member +DEPDIRS-librte_member := librte_eal librte_hash DIRS-$(CONFIG_RTE_LIBRTE_NET) += librte_net +DEPDIRS-librte_net := librte_mbuf librte_eal DIRS-$(CONFIG_RTE_LIBRTE_IP_FRAG) += librte_ip_frag +DEPDIRS-librte_ip_frag := librte_eal librte_mempool librte_mbuf librte_ether +DEPDIRS-librte_ip_frag += librte_hash +DIRS-$(CONFIG_RTE_LIBRTE_GRO) += librte_gro +DEPDIRS-librte_gro := librte_eal librte_mbuf librte_ether librte_net DIRS-$(CONFIG_RTE_LIBRTE_JOBSTATS) += librte_jobstats +DEPDIRS-librte_jobstats := librte_eal +DIRS-$(CONFIG_RTE_LIBRTE_METRICS) += librte_metrics +DEPDIRS-librte_metrics := librte_eal +DIRS-$(CONFIG_RTE_LIBRTE_BITRATE) += librte_bitratestats +DEPDIRS-librte_bitratestats := librte_eal librte_metrics librte_ether +DIRS-$(CONFIG_RTE_LIBRTE_LATENCY_STATS) += librte_latencystats +DEPDIRS-librte_latencystats := librte_eal librte_metrics librte_ether librte_mbuf DIRS-$(CONFIG_RTE_LIBRTE_POWER) += librte_power +DEPDIRS-librte_power := librte_eal DIRS-$(CONFIG_RTE_LIBRTE_METER) += librte_meter +DEPDIRS-librte_meter := librte_eal +DIRS-$(CONFIG_RTE_LIBRTE_FLOW_CLASSIFY) += librte_flow_classify +DEPDIRS-librte_flow_classify := librte_net librte_table librte_acl DIRS-$(CONFIG_RTE_LIBRTE_SCHED) += librte_sched +DEPDIRS-librte_sched := librte_eal librte_mempool librte_mbuf librte_net +DEPDIRS-librte_sched += librte_timer DIRS-$(CONFIG_RTE_LIBRTE_KVARGS) += librte_kvargs +DEPDIRS-librte_kvargs := librte_eal DIRS-$(CONFIG_RTE_LIBRTE_DISTRIBUTOR) += librte_distributor +DEPDIRS-librte_distributor := librte_eal librte_mbuf librte_ether DIRS-$(CONFIG_RTE_LIBRTE_PORT) += librte_port +DEPDIRS-librte_port := librte_eal librte_mempool librte_mbuf librte_ether +DEPDIRS-librte_port += librte_ip_frag librte_sched +ifeq ($(CONFIG_RTE_LIBRTE_KNI),y) +DEPDIRS-librte_port += librte_kni +endif DIRS-$(CONFIG_RTE_LIBRTE_TABLE) += librte_table +DEPDIRS-librte_table := librte_eal librte_mempool librte_mbuf +DEPDIRS-librte_table += librte_port librte_lpm librte_hash +ifeq ($(CONFIG_RTE_LIBRTE_ACL),y) +DEPDIRS-librte_table += librte_acl +endif DIRS-$(CONFIG_RTE_LIBRTE_PIPELINE) += librte_pipeline +DEPDIRS-librte_pipeline := librte_eal librte_mempool librte_mbuf +DEPDIRS-librte_pipeline += librte_table librte_port DIRS-$(CONFIG_RTE_LIBRTE_REORDER) += librte_reorder +DEPDIRS-librte_reorder := librte_eal librte_mempool librte_mbuf DIRS-$(CONFIG_RTE_LIBRTE_PDUMP) += librte_pdump +DEPDIRS-librte_pdump := librte_eal librte_mempool librte_mbuf librte_ether +DIRS-$(CONFIG_RTE_LIBRTE_GSO) += librte_gso +DEPDIRS-librte_gso := librte_eal librte_mbuf librte_ether librte_net +DEPDIRS-librte_gso += librte_mempool ifeq ($(CONFIG_RTE_EXEC_ENV_LINUXAPP),y) DIRS-$(CONFIG_RTE_LIBRTE_KNI) += librte_kni -DIRS-$(CONFIG_RTE_LIBRTE_IVSHMEM) += librte_ivshmem endif +DEPDIRS-librte_kni := librte_eal librte_mempool librte_mbuf librte_ether +DEPDIRS-librte_kni += librte_pci include $(RTE_SDK)/mk/rte.subdir.mk diff --git a/dpdk/lib/librte_acl/Makefile b/dpdk/lib/librte_acl/Makefile index 9803e9dd..e7e3c91d 100644 --- a/dpdk/lib/librte_acl/Makefile +++ b/dpdk/lib/librte_acl/Makefile @@ -36,6 +36,7 @@ LIB = librte_acl.a CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) +LDLIBS += -lrte_eal EXPORT_MAP := rte_acl_version.map @@ -51,13 +52,14 @@ SRCS-$(CONFIG_RTE_LIBRTE_ACL) += acl_run_scalar.c ifneq ($(filter y,$(CONFIG_RTE_ARCH_ARM) $(CONFIG_RTE_ARCH_ARM64)),) SRCS-$(CONFIG_RTE_LIBRTE_ACL) += acl_run_neon.c -CFLAGS_acl_run_neon.o += -flax-vector-conversions -Wno-maybe-uninitialized +CFLAGS_acl_run_neon.o += -flax-vector-conversions +ifeq ($(CONFIG_RTE_TOOLCHAIN_GCC),y) +CFLAGS_acl_run_neon.o += -Wno-maybe-uninitialized +endif +else ifeq ($(CONFIG_RTE_ARCH_PPC_64),y) +SRCS-$(CONFIG_RTE_LIBRTE_ACL) += acl_run_altivec.c else SRCS-$(CONFIG_RTE_LIBRTE_ACL) += acl_run_sse.c -#check if flag for SSE4.1 is already on, if not set it up manually - ifeq ($(findstring RTE_MACHINE_CPUFLAG_SSE4_1,$(CFLAGS)),) - CFLAGS_acl_run_sse.o += -msse4.1 - endif endif # @@ -90,7 +92,4 @@ endif SYMLINK-$(CONFIG_RTE_LIBRTE_ACL)-include := rte_acl_osdep.h SYMLINK-$(CONFIG_RTE_LIBRTE_ACL)-include += rte_acl.h -# this lib needs eal -DEPDIRS-$(CONFIG_RTE_LIBRTE_ACL) += lib/librte_eal - include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/lib/librte_acl/acl.h b/dpdk/lib/librte_acl/acl.h index 09d67841..6664a55e 100644 --- a/dpdk/lib/librte_acl/acl.h +++ b/dpdk/lib/librte_acl/acl.h @@ -234,6 +234,10 @@ int rte_acl_classify_neon(const struct rte_acl_ctx *ctx, const uint8_t **data, uint32_t *results, uint32_t num, uint32_t categories); +int +rte_acl_classify_altivec(const struct rte_acl_ctx *ctx, const uint8_t **data, + uint32_t *results, uint32_t num, uint32_t categories); + #ifdef __cplusplus } #endif /* __cplusplus */ diff --git a/dpdk/lib/librte_acl/acl_bld.c b/dpdk/lib/librte_acl/acl_bld.c index 9e5ad1c6..0768cd3b 100644 --- a/dpdk/lib/librte_acl/acl_bld.c +++ b/dpdk/lib/librte_acl/acl_bld.c @@ -830,8 +830,8 @@ acl_gen_range_trie(struct acl_build_context *context, { int32_t n; struct rte_acl_node *root; - const uint8_t *lo = (const uint8_t *)min; - const uint8_t *hi = (const uint8_t *)max; + const uint8_t *lo = min; + const uint8_t *hi = max; *pend = acl_alloc_node(context, level+size); root = acl_alloc_node(context, level++); @@ -886,8 +886,8 @@ acl_gen_mask_trie(struct acl_build_context *context, struct rte_acl_node *root; struct rte_acl_node *node, *prev; struct rte_acl_bitset bits; - const uint8_t *val = (const uint8_t *)value; - const uint8_t *msk = (const uint8_t *)mask; + const uint8_t *val = value; + const uint8_t *msk = mask; root = acl_alloc_node(context, level++); prev = root; diff --git a/dpdk/lib/librte_acl/acl_run.h b/dpdk/lib/librte_acl/acl_run.h index b2fc42c6..a862ff6e 100644 --- a/dpdk/lib/librte_acl/acl_run.h +++ b/dpdk/lib/librte_acl/acl_run.h @@ -39,7 +39,9 @@ #define MAX_SEARCHES_AVX16 16 #define MAX_SEARCHES_SSE8 8 +#define MAX_SEARCHES_ALTIVEC8 8 #define MAX_SEARCHES_SSE4 4 +#define MAX_SEARCHES_ALTIVEC4 4 #define MAX_SEARCHES_SCALAR 2 #define GET_NEXT_4BYTES(prm, idx) \ @@ -67,10 +69,10 @@ struct acl_flow_data { uint32_t trie; /* current trie index (0 to N-1) */ uint32_t cmplt_size; - uint32_t total_packets; - uint32_t categories; - /* number of result categories per packet. */ /* maximum number of packets to process */ + uint32_t total_packets; + /* number of result categories per packet. */ + uint32_t categories; const uint64_t *trans; const uint8_t **data; uint32_t *results; diff --git a/dpdk/lib/librte_acl/acl_run_altivec.c b/dpdk/lib/librte_acl/acl_run_altivec.c new file mode 100644 index 00000000..35235260 --- /dev/null +++ b/dpdk/lib/librte_acl/acl_run_altivec.c @@ -0,0 +1,47 @@ +/*- + * BSD LICENSE + * + * Copyright (C) IBM Corporation 2016. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "acl_run_altivec.h" + +int +rte_acl_classify_altivec(const struct rte_acl_ctx *ctx, const uint8_t **data, + uint32_t *results, uint32_t num, uint32_t categories) +{ + if (likely(num >= MAX_SEARCHES_ALTIVEC8)) + return search_altivec_8(ctx, data, results, num, categories); + else if (num >= MAX_SEARCHES_ALTIVEC4) + return search_altivec_4(ctx, data, results, num, categories); + else + return rte_acl_classify_scalar(ctx, data, results, num, + categories); +} diff --git a/dpdk/lib/librte_acl/acl_run_altivec.h b/dpdk/lib/librte_acl/acl_run_altivec.h new file mode 100644 index 00000000..62fd6a22 --- /dev/null +++ b/dpdk/lib/librte_acl/acl_run_altivec.h @@ -0,0 +1,329 @@ +/* + * BSD LICENSE + * + * Copyright (C) IBM Corporation 2016. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of IBM Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#include "acl_run.h" +#include "acl_vect.h" + +struct _altivec_acl_const { + rte_xmm_t xmm_shuffle_input; + rte_xmm_t xmm_index_mask; + rte_xmm_t xmm_ones_16; + rte_xmm_t range_base; +} altivec_acl_const __attribute__((aligned(RTE_CACHE_LINE_SIZE))) = { + { + .u32 = {0x00000000, 0x04040404, 0x08080808, 0x0c0c0c0c} + }, + { + .u32 = {RTE_ACL_NODE_INDEX, RTE_ACL_NODE_INDEX, + RTE_ACL_NODE_INDEX, RTE_ACL_NODE_INDEX} + }, + { + .u16 = {1, 1, 1, 1, 1, 1, 1, 1} + }, + { + .u32 = {0xffffff00, 0xffffff04, 0xffffff08, 0xffffff0c} + }, +}; + +/* + * Resolve priority for multiple results (altivec version). + * This consists comparing the priority of the current traversal with the + * running set of results for the packet. + * For each result, keep a running array of the result (rule number) and + * its priority for each category. + */ +static inline void +resolve_priority_altivec(uint64_t transition, int n, + const struct rte_acl_ctx *ctx, struct parms *parms, + const struct rte_acl_match_results *p, uint32_t categories) +{ + uint32_t x; + xmm_t results, priority, results1, priority1; + vector bool int selector; + xmm_t *saved_results, *saved_priority; + + for (x = 0; x < categories; x += RTE_ACL_RESULTS_MULTIPLIER) { + + saved_results = (xmm_t *)(&parms[n].cmplt->results[x]); + saved_priority = + (xmm_t *)(&parms[n].cmplt->priority[x]); + + /* get results and priorities for completed trie */ + results = *(const xmm_t *)&p[transition].results[x]; + priority = *(const xmm_t *)&p[transition].priority[x]; + + /* if this is not the first completed trie */ + if (parms[n].cmplt->count != ctx->num_tries) { + + /* get running best results and their priorities */ + results1 = *saved_results; + priority1 = *saved_priority; + + /* select results that are highest priority */ + selector = vec_cmpgt(priority1, priority); + results = vec_sel(results, results1, selector); + priority = vec_sel(priority, priority1, + selector); + } + + /* save running best results and their priorities */ + *saved_results = results; + *saved_priority = priority; + } +} + +/* + * Check for any match in 4 transitions + */ +static __rte_always_inline uint32_t +check_any_match_x4(uint64_t val[]) +{ + return (val[0] | val[1] | val[2] | val[3]) & RTE_ACL_NODE_MATCH; +} + +static __rte_always_inline void +acl_match_check_x4(int slot, const struct rte_acl_ctx *ctx, struct parms *parms, + struct acl_flow_data *flows, uint64_t transitions[]) +{ + while (check_any_match_x4(transitions)) { + transitions[0] = acl_match_check(transitions[0], slot, ctx, + parms, flows, resolve_priority_altivec); + transitions[1] = acl_match_check(transitions[1], slot + 1, ctx, + parms, flows, resolve_priority_altivec); + transitions[2] = acl_match_check(transitions[2], slot + 2, ctx, + parms, flows, resolve_priority_altivec); + transitions[3] = acl_match_check(transitions[3], slot + 3, ctx, + parms, flows, resolve_priority_altivec); + } +} + +/* + * Process 4 transitions (in 2 XMM registers) in parallel + */ +static inline __attribute__((optimize("O2"))) xmm_t +transition4(xmm_t next_input, const uint64_t *trans, + xmm_t *indices1, xmm_t *indices2) +{ + xmm_t addr, tr_lo, tr_hi; + xmm_t in, node_type, r, t; + xmm_t dfa_ofs, quad_ofs; + xmm_t *index_mask, *tp; + vector bool int dfa_msk; + vector signed char zeroes = {}; + union { + uint64_t d64[2]; + uint32_t d32[4]; + } v; + + /* Move low 32 into tr_lo and high 32 into tr_hi */ + tr_lo = (xmm_t){(*indices1)[0], (*indices1)[2], + (*indices2)[0], (*indices2)[2]}; + tr_hi = (xmm_t){(*indices1)[1], (*indices1)[3], + (*indices2)[1], (*indices2)[3]}; + + /* Calculate the address (array index) for all 4 transitions. */ + index_mask = (xmm_t *)&altivec_acl_const.xmm_index_mask.u32; + t = vec_xor(*index_mask, *index_mask); + in = vec_perm(next_input, (xmm_t){}, + *(vector unsigned char *)&altivec_acl_const.xmm_shuffle_input); + + /* Calc node type and node addr */ + node_type = vec_and(vec_nor(*index_mask, *index_mask), tr_lo); + addr = vec_and(tr_lo, *index_mask); + + /* mask for DFA type(0) nodes */ + dfa_msk = vec_cmpeq(node_type, t); + + /* DFA calculations. */ + r = vec_sr(in, (vector unsigned int){30, 30, 30, 30}); + tp = (xmm_t *)&altivec_acl_const.range_base.u32; + r = vec_add(r, *tp); + t = vec_sr(in, (vector unsigned int){24, 24, 24, 24}); + r = vec_perm(tr_hi, (xmm_t){(uint16_t)0 << 16}, + (vector unsigned char)r); + + dfa_ofs = vec_sub(t, r); + + /* QUAD/SINGLE caluclations. */ + t = (xmm_t)vec_cmpgt((vector signed char)in, (vector signed char)tr_hi); + t = (xmm_t)vec_sel( + vec_sel( + (vector signed char)vec_sub( + zeroes, (vector signed char)t), + (vector signed char)t, + vec_cmpgt((vector signed char)t, zeroes)), + zeroes, + vec_cmpeq((vector signed char)t, zeroes)); + + t = (xmm_t)vec_msum((vector signed char)t, + (vector unsigned char)t, (xmm_t){}); + quad_ofs = (xmm_t)vec_msum((vector signed short)t, + *(vector signed short *)&altivec_acl_const.xmm_ones_16.u16, + (xmm_t){}); + + /* blend DFA and QUAD/SINGLE. */ + t = vec_sel(quad_ofs, dfa_ofs, dfa_msk); + + /* calculate address for next transitions. */ + addr = vec_add(addr, t); + + v.d64[0] = (uint64_t)trans[addr[0]]; + v.d64[1] = (uint64_t)trans[addr[1]]; + *indices1 = (xmm_t){v.d32[0], v.d32[1], v.d32[2], v.d32[3]}; + v.d64[0] = (uint64_t)trans[addr[2]]; + v.d64[1] = (uint64_t)trans[addr[3]]; + *indices2 = (xmm_t){v.d32[0], v.d32[1], v.d32[2], v.d32[3]}; + + return vec_sr(next_input, + (vector unsigned int){CHAR_BIT, CHAR_BIT, CHAR_BIT, CHAR_BIT}); +} + +/* + * Execute trie traversal with 8 traversals in parallel + */ +static inline int +search_altivec_8(const struct rte_acl_ctx *ctx, const uint8_t **data, + uint32_t *results, uint32_t total_packets, uint32_t categories) +{ + int n; + struct acl_flow_data flows; + uint64_t index_array[MAX_SEARCHES_ALTIVEC8]; + struct completion cmplt[MAX_SEARCHES_ALTIVEC8]; + struct parms parms[MAX_SEARCHES_ALTIVEC8]; + xmm_t input0, input1; + + acl_set_flow(&flows, cmplt, RTE_DIM(cmplt), data, results, + total_packets, categories, ctx->trans_table); + + for (n = 0; n < MAX_SEARCHES_ALTIVEC8; n++) { + cmplt[n].count = 0; + index_array[n] = acl_start_next_trie(&flows, parms, n, ctx); + } + + /* Check for any matches. */ + acl_match_check_x4(0, ctx, parms, &flows, (uint64_t *)&index_array[0]); + acl_match_check_x4(4, ctx, parms, &flows, (uint64_t *)&index_array[4]); + + while (flows.started > 0) { + + /* Gather 4 bytes of input data for each stream. */ + input0 = (xmm_t){GET_NEXT_4BYTES(parms, 0), + GET_NEXT_4BYTES(parms, 1), + GET_NEXT_4BYTES(parms, 2), + GET_NEXT_4BYTES(parms, 3)}; + + input1 = (xmm_t){GET_NEXT_4BYTES(parms, 4), + GET_NEXT_4BYTES(parms, 5), + GET_NEXT_4BYTES(parms, 6), + GET_NEXT_4BYTES(parms, 7)}; + + /* Process the 4 bytes of input on each stream. */ + + input0 = transition4(input0, flows.trans, + (xmm_t *)&index_array[0], (xmm_t *)&index_array[2]); + input1 = transition4(input1, flows.trans, + (xmm_t *)&index_array[4], (xmm_t *)&index_array[6]); + + input0 = transition4(input0, flows.trans, + (xmm_t *)&index_array[0], (xmm_t *)&index_array[2]); + input1 = transition4(input1, flows.trans, + (xmm_t *)&index_array[4], (xmm_t *)&index_array[6]); + + input0 = transition4(input0, flows.trans, + (xmm_t *)&index_array[0], (xmm_t *)&index_array[2]); + input1 = transition4(input1, flows.trans, + (xmm_t *)&index_array[4], (xmm_t *)&index_array[6]); + + input0 = transition4(input0, flows.trans, + (xmm_t *)&index_array[0], (xmm_t *)&index_array[2]); + input1 = transition4(input1, flows.trans, + (xmm_t *)&index_array[4], (xmm_t *)&index_array[6]); + + /* Check for any matches. */ + acl_match_check_x4(0, ctx, parms, &flows, + (uint64_t *)&index_array[0]); + acl_match_check_x4(4, ctx, parms, &flows, + (uint64_t *)&index_array[4]); + } + + return 0; +} + +/* + * Execute trie traversal with 4 traversals in parallel + */ +static inline int +search_altivec_4(const struct rte_acl_ctx *ctx, const uint8_t **data, + uint32_t *results, int total_packets, uint32_t categories) +{ + int n; + struct acl_flow_data flows; + uint64_t index_array[MAX_SEARCHES_ALTIVEC4]; + struct completion cmplt[MAX_SEARCHES_ALTIVEC4]; + struct parms parms[MAX_SEARCHES_ALTIVEC4]; + xmm_t input; + + acl_set_flow(&flows, cmplt, RTE_DIM(cmplt), data, results, + total_packets, categories, ctx->trans_table); + + for (n = 0; n < MAX_SEARCHES_ALTIVEC4; n++) { + cmplt[n].count = 0; + index_array[n] = acl_start_next_trie(&flows, parms, n, ctx); + } + + /* Check for any matches. */ + acl_match_check_x4(0, ctx, parms, &flows, index_array); + + while (flows.started > 0) { + + /* Gather 4 bytes of input data for each stream. */ + input = (xmm_t){GET_NEXT_4BYTES(parms, 0), + GET_NEXT_4BYTES(parms, 1), + GET_NEXT_4BYTES(parms, 2), + GET_NEXT_4BYTES(parms, 3)}; + + /* Process the 4 bytes of input on each stream. */ + input = transition4(input, flows.trans, + (xmm_t *)&index_array[0], (xmm_t *)&index_array[2]); + input = transition4(input, flows.trans, + (xmm_t *)&index_array[0], (xmm_t *)&index_array[2]); + input = transition4(input, flows.trans, + (xmm_t *)&index_array[0], (xmm_t *)&index_array[2]); + input = transition4(input, flows.trans, + (xmm_t *)&index_array[0], (xmm_t *)&index_array[2]); + + /* Check for any matches. */ + acl_match_check_x4(0, ctx, parms, &flows, index_array); + } + + return 0; +} diff --git a/dpdk/lib/librte_acl/acl_run_avx2.h b/dpdk/lib/librte_acl/acl_run_avx2.h index b01a46a5..804e45af 100644 --- a/dpdk/lib/librte_acl/acl_run_avx2.h +++ b/dpdk/lib/librte_acl/acl_run_avx2.h @@ -86,7 +86,7 @@ static const rte_ymm_t ymm_range_base = { * tr_hi contains high 32 bits for 8 transition. * next_input contains up to 4 input bytes for 8 flows. */ -static inline __attribute__((always_inline)) ymm_t +static __rte_always_inline ymm_t transition8(ymm_t next_input, const uint64_t *trans, ymm_t *tr_lo, ymm_t *tr_hi) { const int32_t *tr; diff --git a/dpdk/lib/librte_acl/acl_run_neon.c b/dpdk/lib/librte_acl/acl_run_neon.c index b0144514..0b1c71c0 100644 --- a/dpdk/lib/librte_acl/acl_run_neon.c +++ b/dpdk/lib/librte_acl/acl_run_neon.c @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2015. + * Copyright (C) Cavium, Inc. 2015. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * diff --git a/dpdk/lib/librte_acl/acl_run_neon.h b/dpdk/lib/librte_acl/acl_run_neon.h index d233ff00..37881c45 100644 --- a/dpdk/lib/librte_acl/acl_run_neon.h +++ b/dpdk/lib/librte_acl/acl_run_neon.h @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2015. + * Copyright (C) Cavium, Inc. 2015. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * @@ -99,13 +99,13 @@ resolve_priority_neon(uint64_t transition, int n, const struct rte_acl_ctx *ctx, /* * Check for any match in 4 transitions */ -static inline __attribute__((always_inline)) uint32_t +static __rte_always_inline uint32_t check_any_match_x4(uint64_t val[]) { return (val[0] | val[1] | val[2] | val[3]) & RTE_ACL_NODE_MATCH; } -static inline __attribute__((always_inline)) void +static __rte_always_inline void acl_match_check_x4(int slot, const struct rte_acl_ctx *ctx, struct parms *parms, struct acl_flow_data *flows, uint64_t transitions[]) { @@ -124,7 +124,7 @@ acl_match_check_x4(int slot, const struct rte_acl_ctx *ctx, struct parms *parms, /* * Process 4 transitions (in 2 NEON Q registers) in parallel */ -static inline __attribute__((always_inline)) int32x4_t +static __rte_always_inline int32x4_t transition4(int32x4_t next_input, const uint64_t *trans, uint64_t transitions[]) { int32x4x2_t tr_hi_lo; diff --git a/dpdk/lib/librte_acl/acl_run_sse.h b/dpdk/lib/librte_acl/acl_run_sse.h index ad40a674..72f66e4f 100644 --- a/dpdk/lib/librte_acl/acl_run_sse.h +++ b/dpdk/lib/librte_acl/acl_run_sse.h @@ -149,7 +149,7 @@ acl_process_matches(xmm_t *indices, int slot, const struct rte_acl_ctx *ctx, /* * Check for any match in 4 transitions (contained in 2 SSE registers) */ -static inline __attribute__((always_inline)) void +static __rte_always_inline void acl_match_check_x4(int slot, const struct rte_acl_ctx *ctx, struct parms *parms, struct acl_flow_data *flows, xmm_t *indices1, xmm_t *indices2, xmm_t match_mask) @@ -176,7 +176,7 @@ acl_match_check_x4(int slot, const struct rte_acl_ctx *ctx, struct parms *parms, /* * Process 4 transitions (in 2 XMM registers) in parallel */ -static inline __attribute__((always_inline)) xmm_t +static __rte_always_inline xmm_t transition4(xmm_t next_input, const uint64_t *trans, xmm_t *indices1, xmm_t *indices2) { diff --git a/dpdk/lib/librte_acl/rte_acl.c b/dpdk/lib/librte_acl/rte_acl.c index 4ba9786b..67f41f3d 100644 --- a/dpdk/lib/librte_acl/rte_acl.c +++ b/dpdk/lib/librte_acl/rte_acl.c @@ -75,12 +75,23 @@ rte_acl_classify_neon(__rte_unused const struct rte_acl_ctx *ctx, return -ENOTSUP; } +int __attribute__ ((weak)) +rte_acl_classify_altivec(__rte_unused const struct rte_acl_ctx *ctx, + __rte_unused const uint8_t **data, + __rte_unused uint32_t *results, + __rte_unused uint32_t num, + __rte_unused uint32_t categories) +{ + return -ENOTSUP; +} + static const rte_acl_classify_t classify_fns[] = { [RTE_ACL_CLASSIFY_DEFAULT] = rte_acl_classify_scalar, [RTE_ACL_CLASSIFY_SCALAR] = rte_acl_classify_scalar, [RTE_ACL_CLASSIFY_SSE] = rte_acl_classify_sse, [RTE_ACL_CLASSIFY_AVX2] = rte_acl_classify_avx2, [RTE_ACL_CLASSIFY_NEON] = rte_acl_classify_neon, + [RTE_ACL_CLASSIFY_ALTIVEC] = rte_acl_classify_altivec, }; /* by default, use always available scalar code path. */ @@ -109,8 +120,7 @@ rte_acl_set_ctx_classify(struct rte_acl_ctx *ctx, enum rte_acl_classify_alg alg) * if both conditions are met: * at build time compiler supports AVX2 and target cpu supports AVX2. */ -static void __attribute__((constructor)) -rte_acl_init(void) +RTE_INIT(rte_acl_init) { enum rte_acl_classify_alg alg = RTE_ACL_CLASSIFY_DEFAULT; @@ -119,6 +129,8 @@ rte_acl_init(void) #elif defined(RTE_ARCH_ARM) if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_NEON)) alg = RTE_ACL_CLASSIFY_NEON; +#elif defined(RTE_ARCH_PPC_64) + alg = RTE_ACL_CLASSIFY_ALTIVEC; #else #ifdef CC_AVX2_SUPPORT if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2)) @@ -300,8 +312,7 @@ acl_check_rule(const struct rte_acl_rule_data *rd) if ((RTE_LEN2MASK(RTE_ACL_MAX_CATEGORIES, typeof(rd->category_mask)) & rd->category_mask) == 0 || rd->priority > RTE_ACL_MAX_PRIORITY || - rd->priority < RTE_ACL_MIN_PRIORITY || - rd->userdata == RTE_ACL_INVALID_USERDATA) + rd->priority < RTE_ACL_MIN_PRIORITY) return -EINVAL; return 0; } diff --git a/dpdk/lib/librte_acl/rte_acl.h b/dpdk/lib/librte_acl/rte_acl.h index 0979a098..b53179a8 100644 --- a/dpdk/lib/librte_acl/rte_acl.h +++ b/dpdk/lib/librte_acl/rte_acl.h @@ -120,8 +120,6 @@ enum { RTE_ACL_MIN_PRIORITY = 0, }; -#define RTE_ACL_INVALID_USERDATA 0 - #define RTE_ACL_MASKLEN_TO_BITMASK(v, s) \ ((v) == 0 ? (v) : (typeof(v))((uint64_t)-1 << ((s) * CHAR_BIT - (v)))) @@ -144,7 +142,7 @@ struct rte_acl_rule_data { struct rte_acl_field field[fld_num]; \ } -RTE_ACL_RULE_DEF(rte_acl_rule, 0); +RTE_ACL_RULE_DEF(rte_acl_rule,); #define RTE_ACL_RULE_SZ(fld_num) \ (sizeof(struct rte_acl_rule) + sizeof(struct rte_acl_field) * (fld_num)) @@ -271,6 +269,7 @@ enum rte_acl_classify_alg { RTE_ACL_CLASSIFY_SSE = 2, /**< requires SSE4.1 support. */ RTE_ACL_CLASSIFY_AVX2 = 3, /**< requires AVX2 support. */ RTE_ACL_CLASSIFY_NEON = 4, /**< requires NEON support. */ + RTE_ACL_CLASSIFY_ALTIVEC = 5, /**< requires ALTIVEC support. */ RTE_ACL_CLASSIFY_NUM /* should always be the last one. */ }; diff --git a/dpdk/lib/librte_acl/rte_acl_osdep.h b/dpdk/lib/librte_acl/rte_acl_osdep.h index 41f7e3d4..ac712bfa 100644 --- a/dpdk/lib/librte_acl/rte_acl_osdep.h +++ b/dpdk/lib/librte_acl/rte_acl_osdep.h @@ -66,7 +66,6 @@ #include #include #include -#include #include #include #include @@ -74,7 +73,6 @@ #include #include #include -#include #include #endif /* _RTE_ACL_OSDEP_H_ */ diff --git a/dpdk/lib/librte_bitratestats/Makefile b/dpdk/lib/librte_bitratestats/Makefile new file mode 100644 index 00000000..5054b679 --- /dev/null +++ b/dpdk/lib/librte_bitratestats/Makefile @@ -0,0 +1,50 @@ +# BSD LICENSE +# +# Copyright(c) 2017 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +include $(RTE_SDK)/mk/rte.vars.mk + +# library name +LIB = librte_bitratestats.a + +CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -O3 +LDLIBS += -lrte_eal -lrte_metrics -lrte_ethdev + +EXPORT_MAP := rte_bitratestats_version.map + +LIBABIVER := 2 + +# all source are stored in SRCS-y +SRCS-$(CONFIG_RTE_LIBRTE_BITRATE) := rte_bitrate.c + +# Install header file +SYMLINK-$(CONFIG_RTE_LIBRTE_BITRATE)-include += rte_bitrate.h + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/lib/librte_bitratestats/rte_bitrate.c b/dpdk/lib/librte_bitratestats/rte_bitrate.c new file mode 100644 index 00000000..f373697a --- /dev/null +++ b/dpdk/lib/librte_bitratestats/rte_bitrate.c @@ -0,0 +1,153 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include + +#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) + +/* + * Persistent bit-rate data. + * @internal + */ +struct rte_stats_bitrate { + uint64_t last_ibytes; + uint64_t last_obytes; + uint64_t peak_ibits; + uint64_t peak_obits; + uint64_t mean_ibits; + uint64_t mean_obits; + uint64_t ewma_ibits; + uint64_t ewma_obits; +}; + +struct rte_stats_bitrates { + struct rte_stats_bitrate port_stats[RTE_MAX_ETHPORTS]; + uint16_t id_stats_set; +}; + +struct rte_stats_bitrates * +rte_stats_bitrate_create(void) +{ + return rte_zmalloc(NULL, sizeof(struct rte_stats_bitrates), + RTE_CACHE_LINE_SIZE); +} + +int +rte_stats_bitrate_reg(struct rte_stats_bitrates *bitrate_data) +{ + const char * const names[] = { + "ewma_bits_in", "ewma_bits_out", + "mean_bits_in", "mean_bits_out", + "peak_bits_in", "peak_bits_out", + }; + int return_value; + + return_value = rte_metrics_reg_names(&names[0], ARRAY_SIZE(names)); + if (return_value >= 0) + bitrate_data->id_stats_set = return_value; + return return_value; +} + +int +rte_stats_bitrate_calc(struct rte_stats_bitrates *bitrate_data, + uint16_t port_id) +{ + struct rte_stats_bitrate *port_data; + struct rte_eth_stats eth_stats; + int ret_code; + uint64_t cnt_bits; + int64_t delta; + const int64_t alpha_percent = 20; + uint64_t values[6]; + + ret_code = rte_eth_stats_get(port_id, ð_stats); + if (ret_code != 0) + return ret_code; + + port_data = &bitrate_data->port_stats[port_id]; + + /* Incoming bitrate. This is an iteratively calculated EWMA + * (Exponentially Weighted Moving Average) that uses a + * weighting factor of alpha_percent. An unsmoothed mean + * for just the current time delta is also calculated for the + * benefit of people who don't understand signal processing. + */ + cnt_bits = (eth_stats.ibytes - port_data->last_ibytes) << 3; + port_data->last_ibytes = eth_stats.ibytes; + if (cnt_bits > port_data->peak_ibits) + port_data->peak_ibits = cnt_bits; + delta = cnt_bits; + delta -= port_data->ewma_ibits; + /* The +-50 fixes integer rounding during division */ + if (delta > 0) + delta = (delta * alpha_percent + 50) / 100; + else + delta = (delta * alpha_percent - 50) / 100; + port_data->ewma_ibits += delta; + /* Integer roundoff prevents EWMA between 0 and (100/alpha_percent) + * ever reaching zero in no-traffic conditions + */ + if (cnt_bits == 0 && delta == 0) + port_data->ewma_ibits = 0; + port_data->mean_ibits = cnt_bits; + + /* Outgoing bitrate (also EWMA) */ + cnt_bits = (eth_stats.obytes - port_data->last_obytes) << 3; + port_data->last_obytes = eth_stats.obytes; + if (cnt_bits > port_data->peak_obits) + port_data->peak_obits = cnt_bits; + delta = cnt_bits; + delta -= port_data->ewma_obits; + if (delta > 0) + delta = (delta * alpha_percent + 50) / 100; + else + delta = (delta * alpha_percent - 50) / 100; + port_data->ewma_obits += delta; + if (cnt_bits == 0 && delta == 0) + port_data->ewma_obits = 0; + port_data->mean_obits = cnt_bits; + + values[0] = port_data->ewma_ibits; + values[1] = port_data->ewma_obits; + values[2] = port_data->mean_ibits; + values[3] = port_data->mean_obits; + values[4] = port_data->peak_ibits; + values[5] = port_data->peak_obits; + rte_metrics_update_values(port_id, bitrate_data->id_stats_set, + values, ARRAY_SIZE(values)); + return 0; +} diff --git a/dpdk/lib/librte_bitratestats/rte_bitrate.h b/dpdk/lib/librte_bitratestats/rte_bitrate.h new file mode 100644 index 00000000..16467221 --- /dev/null +++ b/dpdk/lib/librte_bitratestats/rte_bitrate.h @@ -0,0 +1,94 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_BITRATE_H_ +#define _RTE_BITRATE_H_ + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Bitrate statistics data structure. + * This data structure is intentionally opaque. + */ +struct rte_stats_bitrates; + + +/** + * Allocate a bitrate statistics structure + * + * @return + * - Pointer to structure on success + * - NULL on error (zmalloc failure) + */ +struct rte_stats_bitrates *rte_stats_bitrate_create(void); + + +/** + * Register bitrate statistics with the metric library. + * + * @param bitrate_data + * Pointer allocated by rte_stats_create() + * + * @return + * Zero on success + * Negative on error + */ +int rte_stats_bitrate_reg(struct rte_stats_bitrates *bitrate_data); + + +/** + * Calculate statistics for current time window. The period with which + * this function is called should be the intended sampling window width. + * + * @param bitrate_data + * Bitrate statistics data pointer + * + * @param port_id + * Port id to calculate statistics for + * + * @return + * - Zero on success + * - Negative value on error + */ +int rte_stats_bitrate_calc(struct rte_stats_bitrates *bitrate_data, + uint16_t port_id); + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_BITRATE_H_ */ diff --git a/dpdk/lib/librte_bitratestats/rte_bitratestats_version.map b/dpdk/lib/librte_bitratestats/rte_bitratestats_version.map new file mode 100644 index 00000000..fe745445 --- /dev/null +++ b/dpdk/lib/librte_bitratestats/rte_bitratestats_version.map @@ -0,0 +1,9 @@ +DPDK_17.05 { + global: + + rte_stats_bitrate_calc; + rte_stats_bitrate_create; + rte_stats_bitrate_reg; + + local: *; +}; diff --git a/dpdk/lib/librte_cfgfile/Makefile b/dpdk/lib/librte_cfgfile/Makefile index 616aef09..0bee43e2 100644 --- a/dpdk/lib/librte_cfgfile/Makefile +++ b/dpdk/lib/librte_cfgfile/Makefile @@ -38,6 +38,7 @@ LIB = librte_cfgfile.a CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) +CFLAGS += -I$(SRCDIR)/../librte_eal/common/include EXPORT_MAP := rte_cfgfile_version.map @@ -51,7 +52,4 @@ SRCS-$(CONFIG_RTE_LIBRTE_CFGFILE) += rte_cfgfile.c # install includes SYMLINK-$(CONFIG_RTE_LIBRTE_CFGFILE)-include += rte_cfgfile.h -# this lib needs eal -DEPDIRS-$(CONFIG_RTE_LIBRTE_CFGFILE) += lib/librte_eal - include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/lib/librte_cfgfile/rte_cfgfile.c b/dpdk/lib/librte_cfgfile/rte_cfgfile.c index d72052a0..eacf93a8 100644 --- a/dpdk/lib/librte_cfgfile/rte_cfgfile.c +++ b/dpdk/lib/librte_cfgfile/rte_cfgfile.c @@ -35,20 +35,23 @@ #include #include #include -#include +#include +#include #include "rte_cfgfile.h" struct rte_cfgfile_section { char name[CFG_NAME_LEN]; int num_entries; - struct rte_cfgfile_entry *entries[0]; + int allocated_entries; + struct rte_cfgfile_entry *entries; }; struct rte_cfgfile { int flags; int num_sections; - struct rte_cfgfile_section *sections[0]; + int allocated_sections; + struct rte_cfgfile_section *sections; }; /** when we resize a file structure, how many extra entries @@ -58,6 +61,25 @@ struct rte_cfgfile { * for new entries do we add in */ #define CFG_ALLOC_ENTRY_BATCH 16 +/** + * Default cfgfile load parameters. + */ +static const struct rte_cfgfile_parameters default_cfgfile_params = { + .comment_character = CFG_DEFAULT_COMMENT_CHARACTER, +}; + +/** + * Defines the list of acceptable comment characters supported by this + * library. + */ +static const char valid_comment_chars[] = { + '!', + '#', + '%', + ';', + '@' +}; + static unsigned _strip(char *str, unsigned len) { @@ -85,27 +107,100 @@ _strip(char *str, unsigned len) return newlen; } +static struct rte_cfgfile_section * +_get_section(struct rte_cfgfile *cfg, const char *sectionname) +{ + int i; + + for (i = 0; i < cfg->num_sections; i++) { + if (strncmp(cfg->sections[i].name, sectionname, + sizeof(cfg->sections[0].name)) == 0) + return &cfg->sections[i]; + } + return NULL; +} + +static int +_add_entry(struct rte_cfgfile_section *section, const char *entryname, + const char *entryvalue) +{ + /* resize entry structure if we don't have room for more entries */ + if (section->num_entries == section->allocated_entries) { + struct rte_cfgfile_entry *n_entries = realloc( + section->entries, + sizeof(struct rte_cfgfile_entry) * + ((section->allocated_entries) + + CFG_ALLOC_ENTRY_BATCH)); + + if (n_entries == NULL) + return -ENOMEM; + + section->entries = n_entries; + section->allocated_entries += CFG_ALLOC_ENTRY_BATCH; + } + /* fill up entry fields with key name and value */ + struct rte_cfgfile_entry *curr_entry = + §ion->entries[section->num_entries]; + + snprintf(curr_entry->name, sizeof(curr_entry->name), "%s", entryname); + snprintf(curr_entry->value, + sizeof(curr_entry->value), "%s", entryvalue); + section->num_entries++; + + return 0; +} + +static int +rte_cfgfile_check_params(const struct rte_cfgfile_parameters *params) +{ + unsigned int valid_comment; + unsigned int i; + + if (!params) { + printf("Error - missing cfgfile parameters\n"); + return -EINVAL; + } + + valid_comment = 0; + for (i = 0; i < RTE_DIM(valid_comment_chars); i++) { + if (params->comment_character == valid_comment_chars[i]) { + valid_comment = 1; + break; + } + } + + if (valid_comment == 0) { + printf("Error - invalid comment characters %c\n", + params->comment_character); + return -ENOTSUP; + } + + return 0; +} + struct rte_cfgfile * rte_cfgfile_load(const char *filename, int flags) { - int allocated_sections = CFG_ALLOC_SECTION_BATCH; - int allocated_entries = 0; - int curr_section = -1; - int curr_entry = -1; - char buffer[256] = {0}; + return rte_cfgfile_load_with_params(filename, flags, + &default_cfgfile_params); +} + +struct rte_cfgfile * +rte_cfgfile_load_with_params(const char *filename, int flags, + const struct rte_cfgfile_parameters *params) +{ + char buffer[CFG_NAME_LEN + CFG_VALUE_LEN + 4] = {0}; int lineno = 0; struct rte_cfgfile *cfg = NULL; + if (rte_cfgfile_check_params(params)) + return NULL; + FILE *f = fopen(filename, "r"); if (f == NULL) return NULL; - cfg = malloc(sizeof(*cfg) + sizeof(cfg->sections[0]) * - allocated_sections); - if (cfg == NULL) - goto error2; - - memset(cfg->sections, 0, sizeof(cfg->sections[0]) * allocated_sections); + cfg = rte_cfgfile_create(flags); while (fgets(buffer, sizeof(buffer), f) != NULL) { char *pos = NULL; @@ -116,13 +211,15 @@ rte_cfgfile_load(const char *filename, int flags) "Check if line too long\n", lineno); goto error1; } - pos = memchr(buffer, ';', sizeof(buffer)); - if (pos != NULL) { + /* skip parsing if comment character found */ + pos = memchr(buffer, params->comment_character, len); + if (pos != NULL && (*(pos-1) != '\\')) { *pos = '\0'; len = pos - buffer; } len = _strip(buffer, len); + /* skip lines without useful content */ if (buffer[0] != '[' && memchr(buffer, '=', len) == NULL) continue; @@ -130,138 +227,252 @@ rte_cfgfile_load(const char *filename, int flags) /* section heading line */ char *end = memchr(buffer, ']', len); if (end == NULL) { - printf("Error line %d - no terminating '['" + printf("Error line %d - no terminating ']'" "character found\n", lineno); goto error1; } *end = '\0'; _strip(&buffer[1], end - &buffer[1]); - /* close off old section and add start new one */ - if (curr_section >= 0) - cfg->sections[curr_section]->num_entries = - curr_entry + 1; - curr_section++; - - /* resize overall struct if we don't have room for more - sections */ - if (curr_section == allocated_sections) { - allocated_sections += CFG_ALLOC_SECTION_BATCH; - struct rte_cfgfile *n_cfg = realloc(cfg, - sizeof(*cfg) + sizeof(cfg->sections[0]) - * allocated_sections); - if (n_cfg == NULL) { - printf("Error - no more memory\n"); - goto error1; - } - cfg = n_cfg; - } - - /* allocate space for new section */ - allocated_entries = CFG_ALLOC_ENTRY_BATCH; - curr_entry = -1; - cfg->sections[curr_section] = malloc( - sizeof(*cfg->sections[0]) + - sizeof(cfg->sections[0]->entries[0]) * - allocated_entries); - if (cfg->sections[curr_section] == NULL) { - printf("Error - no more memory\n"); - goto error1; - } - - snprintf(cfg->sections[curr_section]->name, - sizeof(cfg->sections[0]->name), - "%s", &buffer[1]); + rte_cfgfile_add_section(cfg, &buffer[1]); } else { - /* value line */ - if (curr_section < 0) { - printf("Error line %d - value outside of" - "section\n", lineno); + /* key and value line */ + char *split[2] = {NULL}; + + split[0] = buffer; + split[1] = memchr(buffer, '=', len); + if (split[1] == NULL) { + printf("Error line %d - no '='" + "character found\n", lineno); + goto error1; + } + *split[1] = '\0'; + split[1]++; + + _strip(split[0], strlen(split[0])); + _strip(split[1], strlen(split[1])); + char *end = memchr(split[1], '\\', strlen(split[1])); + + while (end != NULL) { + if (*(end+1) == params->comment_character) { + *end = '\0'; + strcat(split[1], end+1); + } else + end++; + end = memchr(end, '\\', strlen(end)); + } + + if (!(flags & CFG_FLAG_EMPTY_VALUES) && + (*split[1] == '\0')) { + printf("Error at line %d - cannot use empty " + "values\n", lineno); goto error1; } - struct rte_cfgfile_section *sect = - cfg->sections[curr_section]; - char *split[2]; - if (rte_strsplit(buffer, sizeof(buffer), split, 2, '=') - != 2) { - printf("Error at line %d - cannot split " - "string\n", lineno); + if (cfg->num_sections == 0) goto error1; - } - curr_entry++; - if (curr_entry == allocated_entries) { - allocated_entries += CFG_ALLOC_ENTRY_BATCH; - struct rte_cfgfile_section *n_sect = realloc( - sect, sizeof(*sect) + - sizeof(sect->entries[0]) * - allocated_entries); - if (n_sect == NULL) { - printf("Error - no more memory\n"); - goto error1; - } - sect = cfg->sections[curr_section] = n_sect; - } - - sect->entries[curr_entry] = malloc( - sizeof(*sect->entries[0])); - if (sect->entries[curr_entry] == NULL) { - printf("Error - no more memory\n"); - goto error1; - } - - struct rte_cfgfile_entry *entry = sect->entries[ - curr_entry]; - snprintf(entry->name, sizeof(entry->name), "%s", - split[0]); - snprintf(entry->value, sizeof(entry->value), "%s", - split[1]); - _strip(entry->name, strnlen(entry->name, - sizeof(entry->name))); - _strip(entry->value, strnlen(entry->value, - sizeof(entry->value))); + _add_entry(&cfg->sections[cfg->num_sections - 1], + split[0], split[1]); } } fclose(f); - cfg->flags = flags; - cfg->num_sections = curr_section + 1; - /* curr_section will still be -1 if we have an empty file */ - if (curr_section >= 0) - cfg->sections[curr_section]->num_entries = curr_entry + 1; return cfg; - error1: - cfg->num_sections = curr_section + 1; rte_cfgfile_close(cfg); -error2: fclose(f); return NULL; } +struct rte_cfgfile * +rte_cfgfile_create(int flags) +{ + int i; + struct rte_cfgfile *cfg = NULL; + + cfg = malloc(sizeof(*cfg)); + + if (cfg == NULL) + return NULL; + + cfg->flags = flags; + cfg->num_sections = 0; + + /* allocate first batch of sections and entries */ + cfg->sections = malloc(sizeof(struct rte_cfgfile_section) * + CFG_ALLOC_SECTION_BATCH); + + if (cfg->sections == NULL) + goto error1; + + cfg->allocated_sections = CFG_ALLOC_SECTION_BATCH; + + for (i = 0; i < CFG_ALLOC_SECTION_BATCH; i++) { + cfg->sections[i].entries = malloc(sizeof( + struct rte_cfgfile_entry) * CFG_ALLOC_ENTRY_BATCH); + + if (cfg->sections[i].entries == NULL) + goto error1; + + cfg->sections[i].num_entries = 0; + cfg->sections[i].allocated_entries = CFG_ALLOC_ENTRY_BATCH; + } + + if (flags & CFG_FLAG_GLOBAL_SECTION) + rte_cfgfile_add_section(cfg, "GLOBAL"); + + return cfg; +error1: + if (cfg->sections != NULL) { + for (i = 0; i < cfg->allocated_sections; i++) { + if (cfg->sections[i].entries != NULL) { + free(cfg->sections[i].entries); + cfg->sections[i].entries = NULL; + } + } + free(cfg->sections); + cfg->sections = NULL; + } + free(cfg); + return NULL; +} + +int +rte_cfgfile_add_section(struct rte_cfgfile *cfg, const char *sectionname) +{ + int i; + + if (cfg == NULL) + return -EINVAL; + + if (sectionname == NULL) + return -EINVAL; + + /* resize overall struct if we don't have room for more sections */ + if (cfg->num_sections == cfg->allocated_sections) { + + struct rte_cfgfile_section *n_sections = + realloc(cfg->sections, + sizeof(struct rte_cfgfile_section) * + ((cfg->allocated_sections) + + CFG_ALLOC_SECTION_BATCH)); + + if (n_sections == NULL) + return -ENOMEM; + + for (i = 0; i < CFG_ALLOC_SECTION_BATCH; i++) { + n_sections[i + cfg->allocated_sections].num_entries = 0; + n_sections[i + + cfg->allocated_sections].allocated_entries = 0; + n_sections[i + cfg->allocated_sections].entries = NULL; + } + cfg->sections = n_sections; + cfg->allocated_sections += CFG_ALLOC_SECTION_BATCH; + } + + snprintf(cfg->sections[cfg->num_sections].name, + sizeof(cfg->sections[0].name), "%s", sectionname); + cfg->sections[cfg->num_sections].num_entries = 0; + cfg->num_sections++; + + return 0; +} + +int rte_cfgfile_add_entry(struct rte_cfgfile *cfg, + const char *sectionname, const char *entryname, + const char *entryvalue) +{ + int ret; + + if ((cfg == NULL) || (sectionname == NULL) || (entryname == NULL) + || (entryvalue == NULL)) + return -EINVAL; + + if (rte_cfgfile_has_entry(cfg, sectionname, entryname) != 0) + return -EEXIST; + + /* search for section pointer by sectionname */ + struct rte_cfgfile_section *curr_section = _get_section(cfg, + sectionname); + if (curr_section == NULL) + return -EINVAL; + + ret = _add_entry(curr_section, entryname, entryvalue); + + return ret; +} + +int rte_cfgfile_set_entry(struct rte_cfgfile *cfg, const char *sectionname, + const char *entryname, const char *entryvalue) +{ + int i; + + if ((cfg == NULL) || (sectionname == NULL) || (entryname == NULL)) + return -EINVAL; + + /* search for section pointer by sectionname */ + struct rte_cfgfile_section *curr_section = _get_section(cfg, + sectionname); + if (curr_section == NULL) + return -EINVAL; + + if (entryvalue == NULL) + entryvalue = ""; + + for (i = 0; i < curr_section->num_entries; i++) + if (!strcmp(curr_section->entries[i].name, entryname)) { + snprintf(curr_section->entries[i].value, + sizeof(curr_section->entries[i].value), + "%s", entryvalue); + return 0; + } + printf("Error - entry name doesn't exist\n"); + return -EINVAL; +} + +int rte_cfgfile_save(struct rte_cfgfile *cfg, const char *filename) +{ + int i, j; + + if ((cfg == NULL) || (filename == NULL)) + return -EINVAL; + + FILE *f = fopen(filename, "w"); + + if (f == NULL) + return -EINVAL; + + for (i = 0; i < cfg->num_sections; i++) { + fprintf(f, "[%s]\n", cfg->sections[i].name); + + for (j = 0; j < cfg->sections[i].num_entries; j++) { + fprintf(f, "%s=%s\n", + cfg->sections[i].entries[j].name, + cfg->sections[i].entries[j].value); + } + } + return fclose(f); +} int rte_cfgfile_close(struct rte_cfgfile *cfg) { - int i, j; + int i; if (cfg == NULL) return -1; - for (i = 0; i < cfg->num_sections; i++) { - if (cfg->sections[i] != NULL) { - if (cfg->sections[i]->num_entries) { - for (j = 0; j < cfg->sections[i]->num_entries; - j++) { - if (cfg->sections[i]->entries[j] != - NULL) - free(cfg->sections[i]-> - entries[j]); - } + if (cfg->sections != NULL) { + for (i = 0; i < cfg->allocated_sections; i++) { + if (cfg->sections[i].entries != NULL) { + free(cfg->sections[i].entries); + cfg->sections[i].entries = NULL; } - free(cfg->sections[i]); } + free(cfg->sections); + cfg->sections = NULL; } free(cfg); + cfg = NULL; return 0; } @@ -273,7 +484,7 @@ size_t length) int i; int num_sections = 0; for (i = 0; i < cfg->num_sections; i++) { - if (strncmp(cfg->sections[i]->name, sectionname, length) == 0) + if (strncmp(cfg->sections[i].name, sectionname, length) == 0) num_sections++; } return num_sections; @@ -287,23 +498,11 @@ rte_cfgfile_sections(struct rte_cfgfile *cfg, char *sections[], for (i = 0; i < cfg->num_sections && i < max_sections; i++) snprintf(sections[i], CFG_NAME_LEN, "%s", - cfg->sections[i]->name); + cfg->sections[i].name); return i; } -static const struct rte_cfgfile_section * -_get_section(struct rte_cfgfile *cfg, const char *sectionname) -{ - int i; - for (i = 0; i < cfg->num_sections; i++) { - if (strncmp(cfg->sections[i]->name, sectionname, - sizeof(cfg->sections[0]->name)) == 0) - return cfg->sections[i]; - } - return NULL; -} - int rte_cfgfile_has_section(struct rte_cfgfile *cfg, const char *sectionname) { @@ -320,7 +519,18 @@ rte_cfgfile_section_num_entries(struct rte_cfgfile *cfg, return s->num_entries; } +int +rte_cfgfile_section_num_entries_by_index(struct rte_cfgfile *cfg, + char *sectionname, int index) +{ + if (index < 0 || index >= cfg->num_sections) + return -1; + const struct rte_cfgfile_section *sect = &(cfg->sections[index]); + + snprintf(sectionname, CFG_NAME_LEN, "%s", sect->name); + return sect->num_entries; +} int rte_cfgfile_section_entries(struct rte_cfgfile *cfg, const char *sectionname, struct rte_cfgfile_entry *entries, int max_entries) @@ -330,7 +540,7 @@ rte_cfgfile_section_entries(struct rte_cfgfile *cfg, const char *sectionname, if (sect == NULL) return -1; for (i = 0; i < max_entries && i < sect->num_entries; i++) - entries[i] = *sect->entries[i]; + entries[i] = sect->entries[i]; return i; } @@ -344,11 +554,10 @@ rte_cfgfile_section_entries_by_index(struct rte_cfgfile *cfg, int index, if (index < 0 || index >= cfg->num_sections) return -1; - - sect = cfg->sections[index]; + sect = &cfg->sections[index]; snprintf(sectionname, CFG_NAME_LEN, "%s", sect->name); for (i = 0; i < max_entries && i < sect->num_entries; i++) - entries[i] = *sect->entries[i]; + entries[i] = sect->entries[i]; return i; } @@ -361,9 +570,9 @@ rte_cfgfile_get_entry(struct rte_cfgfile *cfg, const char *sectionname, if (sect == NULL) return NULL; for (i = 0; i < sect->num_entries; i++) - if (strncmp(sect->entries[i]->name, entryname, CFG_NAME_LEN) - == 0) - return sect->entries[i]->value; + if (strncmp(sect->entries[i].name, entryname, CFG_NAME_LEN) + == 0) + return sect->entries[i].value; return NULL; } diff --git a/dpdk/lib/librte_cfgfile/rte_cfgfile.h b/dpdk/lib/librte_cfgfile/rte_cfgfile.h index f649836c..17f72757 100644 --- a/dpdk/lib/librte_cfgfile/rte_cfgfile.h +++ b/dpdk/lib/librte_cfgfile/rte_cfgfile.h @@ -34,6 +34,8 @@ #ifndef __INCLUDE_RTE_CFGFILE_H__ #define __INCLUDE_RTE_CFGFILE_H__ +#include + #ifdef __cplusplus extern "C" { #endif @@ -64,18 +66,136 @@ struct rte_cfgfile_entry { char value[CFG_VALUE_LEN]; /**< Value */ }; +/** Configuration file operation optional arguments */ +struct rte_cfgfile_parameters { + /** Config file comment character; one of '!', '#', '%', ';', '@' */ + char comment_character; +}; + +/**@{ cfgfile load operation flags */ +enum { + /** + * Indicates that the file supports key value entries before the first + * defined section. These entries can be accessed in the "GLOBAL" + * section. + */ + CFG_FLAG_GLOBAL_SECTION = 1, + + /** + * Indicates that file supports key value entries where the value can + * be zero length (e.g., "key="). + */ + CFG_FLAG_EMPTY_VALUES = 2, +}; +/**@} */ + +/** Defines the default comment character used for parsing config files. */ +#define CFG_DEFAULT_COMMENT_CHARACTER ';' + /** * Open config file * * @param filename * Config file name * @param flags -* Config file flags, Reserved for future use. Must be set to 0. +* Config file flags * @return * Handle to configuration file on success, NULL otherwise */ struct rte_cfgfile *rte_cfgfile_load(const char *filename, int flags); +/** + * Open config file with specified optional parameters. + * + * @param filename + * Config file name + * @param flags + * Config file flags + * @param params + * Additional configuration attributes. Must be configured with desired + * values prior to invoking this API. + * @return + * Handle to configuration file on success, NULL otherwise + */ +struct rte_cfgfile *rte_cfgfile_load_with_params(const char *filename, + int flags, const struct rte_cfgfile_parameters *params); + +/** + * Create new cfgfile instance with empty sections and entries + * + * @param flags + * - CFG_FLAG_GLOBAL_SECTION + * Indicates that the file supports key value entries before the first + * defined section. These entries can be accessed in the "GLOBAL" + * section. + * - CFG_FLAG_EMPTY_VALUES + * Indicates that file supports key value entries where the value can + * be zero length (e.g., "key="). + * @return + * Handle to cfgfile instance on success, NULL otherwise + */ +struct rte_cfgfile *rte_cfgfile_create(int flags); + +/** + * Add section in cfgfile instance. + * + * @param cfg + * Pointer to the cfgfile structure. + * @param sectionname + * Section name which will be add to cfgfile. + * @return + * 0 on success, -ENOMEM if can't add section + */ +int +rte_cfgfile_add_section(struct rte_cfgfile *cfg, const char *sectionname); + +/** + * Add entry to specified section in cfgfile instance. + * + * @param cfg + * Pointer to the cfgfile structure. + * @param sectionname + * Given section name to add an entry. + * @param entryname + * Entry name to add. + * @param entryvalue + * Entry value to add. + * @return + * 0 on success, -EEXIST if entry already exist, -EINVAL if bad argument + */ +int rte_cfgfile_add_entry(struct rte_cfgfile *cfg, + const char *sectionname, const char *entryname, + const char *entryvalue); + +/** + * Update value of specified entry name in given section in config file + * + * @param cfg + * Config file + * @param sectionname + * Section name + * @param entryname + * Entry name to look for the value change + * @param entryvalue + * New entry value. Can be also an empty string if CFG_FLAG_EMPTY_VALUES = 1 + * @return + * 0 on success, -EINVAL if bad argument + */ +int rte_cfgfile_set_entry(struct rte_cfgfile *cfg, const char *sectionname, + const char *entryname, const char *entryvalue); + +/** + * Save object cfgfile to file on disc + * + * @param cfg + * Config file structure + * @param filename + * File name to save data + * @return + * 0 on success, errno otherwise + */ +int rte_cfgfile_save(struct rte_cfgfile *cfg, const char *filename); + /** * Get number of sections in config file * @@ -86,7 +206,7 @@ struct rte_cfgfile *rte_cfgfile_load(const char *filename, int flags); * @param length * Maximum section name length * @return -* 0 on success, error code otherwise +* Number of sections */ int rte_cfgfile_num_sections(struct rte_cfgfile *cfg, const char *sec_name, size_t length); @@ -100,13 +220,13 @@ int rte_cfgfile_num_sections(struct rte_cfgfile *cfg, const char *sec_name, * @param cfg * Config file * @param sections -* Array containing section names after successful invocation. Each elemen +* Array containing section names after successful invocation. Each element * of this array should be preallocated by the user with at least * CFG_NAME_LEN characters. * @param max_sections * Maximum number of section names to be stored in sections array * @return -* 0 on success, error code otherwise +* Number of populated sections names */ int rte_cfgfile_sections(struct rte_cfgfile *cfg, char *sections[], int max_sections); @@ -134,12 +254,33 @@ int rte_cfgfile_has_section(struct rte_cfgfile *cfg, const char *sectionname); * @param sectionname * Section name * @return -* Number of entries in section +* Number of entries in section on success, -1 otherwise */ int rte_cfgfile_section_num_entries(struct rte_cfgfile *cfg, const char *sectionname); -/** Get section entries as key-value pairs +/** +* Get number of entries in given config file section +* +* The index of a section is the same as the index of its name in the +* result of rte_cfgfile_sections. This API can be used when there are +* multiple sections with the same name. +* +* @param cfg +* Config file +* @param sectionname +* Section name +* @param index +* Section index +* @return +* Number of entries in section on success, -1 otherwise +*/ +int rte_cfgfile_section_num_entries_by_index(struct rte_cfgfile *cfg, + char *sectionname, + int index); + +/** +* Get section entries as key-value pairs * * If multiple sections have the given name this function operates on the * first one. @@ -154,14 +295,15 @@ int rte_cfgfile_section_num_entries(struct rte_cfgfile *cfg, * @param max_entries * Maximum number of section entries to be stored in entries array * @return -* 0 on success, error code otherwise +* Number of entries populated on success, -1 otherwise */ int rte_cfgfile_section_entries(struct rte_cfgfile *cfg, const char *sectionname, struct rte_cfgfile_entry *entries, int max_entries); -/** Get section entries as key-value pairs +/** +* Get section entries as key-value pairs * * The index of a section is the same as the index of its name in the * result of rte_cfgfile_sections. This API can be used when there are @@ -180,7 +322,7 @@ int rte_cfgfile_section_entries(struct rte_cfgfile *cfg, * @param max_entries * Maximum number of section entries to be stored in entries array * @return -* Number of entries populated on success, negative error code otherwise +* Number of entries populated on success, -1 otherwise */ int rte_cfgfile_section_entries_by_index(struct rte_cfgfile *cfg, int index, @@ -188,7 +330,8 @@ int rte_cfgfile_section_entries_by_index(struct rte_cfgfile *cfg, struct rte_cfgfile_entry *entries, int max_entries); -/** Get value of the named entry in named config file section +/** +* Get value of the named entry in named config file section * * If multiple sections have the given name this function operates on the * first one. @@ -200,13 +343,14 @@ int rte_cfgfile_section_entries_by_index(struct rte_cfgfile *cfg, * @param entryname * Entry name * @return -* Entry value +* Entry value on success, NULL otherwise */ const char *rte_cfgfile_get_entry(struct rte_cfgfile *cfg, const char *sectionname, const char *entryname); -/** Check if given entry exists in named config file section +/** +* Check if given entry exists in named config file section * * If multiple sections have the given name this function operates on the * first one. @@ -223,12 +367,13 @@ const char *rte_cfgfile_get_entry(struct rte_cfgfile *cfg, int rte_cfgfile_has_entry(struct rte_cfgfile *cfg, const char *sectionname, const char *entryname); -/** Close config file +/** +* Close config file * * @param cfg * Config file * @return -* 0 on success, error code otherwise +* 0 on success, -1 otherwise */ int rte_cfgfile_close(struct rte_cfgfile *cfg); diff --git a/dpdk/lib/librte_cfgfile/rte_cfgfile_version.map b/dpdk/lib/librte_cfgfile/rte_cfgfile_version.map index 3c2f0dbf..cc4a11f6 100644 --- a/dpdk/lib/librte_cfgfile/rte_cfgfile_version.map +++ b/dpdk/lib/librte_cfgfile/rte_cfgfile_version.map @@ -20,3 +20,21 @@ DPDK_16.04 { rte_cfgfile_section_entries_by_index; } DPDK_2.0; + +DPDK_17.05 { + global: + + rte_cfgfile_load_with_params; + +} DPDK_16.04; + +DPDK_17.11 { + global: + + rte_cfgfile_add_entry; + rte_cfgfile_add_section; + rte_cfgfile_create; + rte_cfgfile_save; + rte_cfgfile_set_entry; + +} DPDK_17.05; diff --git a/dpdk/lib/librte_cmdline/Makefile b/dpdk/lib/librte_cmdline/Makefile index 7d2d148c..2c48e62b 100644 --- a/dpdk/lib/librte_cmdline/Makefile +++ b/dpdk/lib/librte_cmdline/Makefile @@ -54,6 +54,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_CMDLINE) += cmdline_socket.c SRCS-$(CONFIG_RTE_LIBRTE_CMDLINE) += cmdline_parse_portlist.c CFLAGS += -D_GNU_SOURCE +LDLIBS += -lrte_eal # install includes INCS := cmdline.h cmdline_parse.h cmdline_parse_num.h cmdline_parse_ipaddr.h @@ -61,7 +62,4 @@ INCS += cmdline_parse_etheraddr.h cmdline_parse_string.h cmdline_rdline.h INCS += cmdline_vt100.h cmdline_socket.h cmdline_cirbuf.h cmdline_parse_portlist.h SYMLINK-$(CONFIG_RTE_LIBRTE_CMDLINE)-include := $(INCS) -# this lib needs eal -DEPDIRS-$(CONFIG_RTE_LIBRTE_CMDLINE) += lib/librte_eal - include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/lib/librte_cmdline/cmdline.c b/dpdk/lib/librte_cmdline/cmdline.c index a9c47be3..d7491651 100644 --- a/dpdk/lib/librte_cmdline/cmdline.c +++ b/dpdk/lib/librte_cmdline/cmdline.c @@ -205,7 +205,8 @@ cmdline_printf(const struct cmdline *cl, const char *fmt, ...) } if (ret >= BUFSIZ) ret = BUFSIZ - 1; - write(cl->s_out, buf, ret); + ret = write(cl->s_out, buf, ret); + (void)ret; free(buf); #endif } diff --git a/dpdk/lib/librte_cmdline/cmdline.h b/dpdk/lib/librte_cmdline/cmdline.h index 2578ca81..65d73b01 100644 --- a/dpdk/lib/librte_cmdline/cmdline.h +++ b/dpdk/lib/librte_cmdline/cmdline.h @@ -63,6 +63,7 @@ #include #include +#include /** * @file diff --git a/dpdk/lib/librte_cmdline/cmdline_cirbuf.h b/dpdk/lib/librte_cmdline/cmdline_cirbuf.h index 6321dec5..87407efa 100644 --- a/dpdk/lib/librte_cmdline/cmdline_cirbuf.h +++ b/dpdk/lib/librte_cmdline/cmdline_cirbuf.h @@ -61,6 +61,8 @@ #ifndef _CIRBUF_H_ #define _CIRBUF_H_ +#include + #ifdef __cplusplus extern "C" { #endif diff --git a/dpdk/lib/librte_cmdline/cmdline_parse.c b/dpdk/lib/librte_cmdline/cmdline_parse.c index b496067a..7f799f94 100644 --- a/dpdk/lib/librte_cmdline/cmdline_parse.c +++ b/dpdk/lib/librte_cmdline/cmdline_parse.c @@ -139,6 +139,21 @@ nb_common_chars(const char * s1, const char * s2) return i; } +/** Retrieve either static or dynamic token at a given index. */ +static cmdline_parse_token_hdr_t * +get_token(cmdline_parse_inst_t *inst, unsigned int index) +{ + cmdline_parse_token_hdr_t *token_p; + + /* check presence of static tokens first */ + if (inst->tokens[0] || !inst->f) + return inst->tokens[index]; + /* generate dynamic token */ + token_p = NULL; + inst->f(&token_p, NULL, &inst->tokens[index]); + return token_p; +} + /** * try to match the buffer with an instruction (only the first * nb_match_token tokens if != 0). Return 0 if we match all the @@ -148,18 +163,20 @@ static int match_inst(cmdline_parse_inst_t *inst, const char *buf, unsigned int nb_match_token, void *resbuf, unsigned resbuf_size) { - unsigned int token_num=0; - cmdline_parse_token_hdr_t * token_p; + cmdline_parse_token_hdr_t *token_p = NULL; unsigned int i=0; int n = 0; struct cmdline_token_hdr token_hdr; - token_p = inst->tokens[token_num]; - if (token_p) + if (resbuf != NULL) + memset(resbuf, 0, resbuf_size); + /* check if we match all tokens of inst */ + while (!nb_match_token || i < nb_match_token) { + token_p = get_token(inst, i); + if (!token_p) + break; memcpy(&token_hdr, token_p, sizeof(token_hdr)); - /* check if we match all tokens of inst */ - while (token_p && (!nb_match_token || itokens[token_num]; - if (token_p) - memcpy(&token_hdr, token_p, sizeof(token_hdr)); } /* does not match */ @@ -238,7 +250,10 @@ cmdline_parse(struct cmdline *cl, const char * buf) unsigned int inst_num=0; cmdline_parse_inst_t *inst; const char *curbuf; - char result_buf[CMDLINE_PARSE_RESULT_BUFSIZE]; + union { + char buf[CMDLINE_PARSE_RESULT_BUFSIZE]; + long double align; /* strong alignment constraint for buf */ + } result, tmp_result; void (*f)(void *, struct cmdline *, void *) = NULL; void *data = NULL; int comment = 0; @@ -250,6 +265,7 @@ cmdline_parse(struct cmdline *cl, const char * buf) #ifdef RTE_LIBRTE_CMDLINE_DEBUG char debug_buf[BUFSIZ]; #endif + char *result_buf = result.buf; if (!cl || !buf) return CMDLINE_PARSE_BAD_ARGS; @@ -299,7 +315,8 @@ cmdline_parse(struct cmdline *cl, const char * buf) debug_printf("INST %d\n", inst_num); /* fully parsed */ - tok = match_inst(inst, buf, 0, result_buf, sizeof(result_buf)); + tok = match_inst(inst, buf, 0, result_buf, + CMDLINE_PARSE_RESULT_BUFSIZE); if (tok > 0) /* we matched at least one token */ err = CMDLINE_PARSE_BAD_ARGS; @@ -316,6 +333,7 @@ cmdline_parse(struct cmdline *cl, const char * buf) if (!f) { memcpy(&f, &inst->f, sizeof(f)); memcpy(&data, &inst->data, sizeof(data)); + result_buf = tmp_result.buf; } else { /* more than 1 inst matches */ @@ -333,7 +351,7 @@ cmdline_parse(struct cmdline *cl, const char * buf) /* call func */ if (f) { - f(result_buf, cl, data); + f(result.buf, cl, data); } /* no match */ @@ -396,11 +414,12 @@ cmdline_complete(struct cmdline *cl, const char *buf, int *state, inst = ctx[inst_num]; while (inst) { /* parse the first tokens of the inst */ - if (nb_token && match_inst(inst, buf, nb_token, NULL, 0)) + if (nb_token && + match_inst(inst, buf, nb_token, NULL, 0)) goto next; debug_printf("instruction match\n"); - token_p = inst->tokens[nb_token]; + token_p = get_token(inst, nb_token); if (token_p) memcpy(&token_hdr, token_p, sizeof(token_hdr)); @@ -490,10 +509,11 @@ cmdline_complete(struct cmdline *cl, const char *buf, int *state, /* we need to redo it */ inst = ctx[inst_num]; - if (nb_token && match_inst(inst, buf, nb_token, NULL, 0)) + if (nb_token && + match_inst(inst, buf, nb_token, NULL, 0)) goto next2; - token_p = inst->tokens[nb_token]; + token_p = get_token(inst, nb_token); if (token_p) memcpy(&token_hdr, token_p, sizeof(token_hdr)); diff --git a/dpdk/lib/librte_cmdline/cmdline_parse.h b/dpdk/lib/librte_cmdline/cmdline_parse.h index 4ac05d6b..13e086f2 100644 --- a/dpdk/lib/librte_cmdline/cmdline_parse.h +++ b/dpdk/lib/librte_cmdline/cmdline_parse.h @@ -130,6 +130,57 @@ struct cmdline; * Store a instruction, which is a pointer to a callback function and * its parameter that is called when the instruction is parsed, a help * string, and a list of token composing this instruction. + * + * When no tokens are defined (tokens[0] == NULL), they are retrieved + * dynamically by calling f() as follows: + * + * @code + * + * f((struct cmdline_token_hdr **)&token_p, + * NULL, + * (struct cmdline_token_hdr **)&inst->tokens[num]); + * + * @endcode + * + * The address of the resulting token is expected at the location pointed by + * the first argument. Can be set to NULL to end the list. + * + * The cmdline argument (struct cmdline *) is always NULL. + * + * The last argument points to the inst->tokens[] entry to retrieve, which + * is not necessarily inside allocated memory and should neither be read nor + * written. Its sole purpose is to deduce the token entry index of interest + * as described in the example below. + * + * Note about constraints: + * + * - Only the address of these tokens is dynamic, their storage should be + * static like normal tokens. + * - Dynamic token lists that need to maintain an internal context (e.g. in + * order to determine the next token) must store it statically also. This + * context must be reinitialized when the first token is requested, that + * is, when &inst->tokens[0] is provided as the third argument. + * - Dynamic token lists must be NULL-terminated to generate usable + * commands. + * + * @code + * + * // Assuming first and third arguments are respectively named "token_p" + * // and "token": + * + * int index = token - inst->tokens; + * + * if (!index) { + * [...] // Clean up internal context if any. + * } + * [...] // Then set up dyn_token according to index. + * + * if (no_more_tokens) + * *token_p = NULL; + * else + * *token_p = &dyn_token; + * + * @endcode */ struct cmdline_inst { /* f(parsed_struct, data) */ diff --git a/dpdk/lib/librte_cmdline/cmdline_parse_etheraddr.c b/dpdk/lib/librte_cmdline/cmdline_parse_etheraddr.c index dbfe4a61..da02d2c9 100644 --- a/dpdk/lib/librte_cmdline/cmdline_parse_etheraddr.c +++ b/dpdk/lib/librte_cmdline/cmdline_parse_etheraddr.c @@ -65,7 +65,6 @@ #include #include #include -#include #include #include diff --git a/dpdk/lib/librte_cmdline/cmdline_parse_num.c b/dpdk/lib/librte_cmdline/cmdline_parse_num.c index b0f9a35d..e507ec4f 100644 --- a/dpdk/lib/librte_cmdline/cmdline_parse_num.c +++ b/dpdk/lib/librte_cmdline/cmdline_parse_num.c @@ -250,7 +250,7 @@ cmdline_parse_num(cmdline_parse_token_hdr_t *tk, const char *srcbuf, void *res, case HEX: st = HEX_OK; - /* no break */ + /* fall-through no break */ case HEX_OK: if (c >= '0' && c <= '9') { if (add_to_res(c - '0', &res1, 16) < 0) @@ -282,7 +282,7 @@ cmdline_parse_num(cmdline_parse_token_hdr_t *tk, const char *srcbuf, void *res, case BIN: st = BIN_OK; - /* no break */ + /* fall-through */ case BIN_OK: if (c >= '0' && c <= '1') { if (add_to_res(c - '0', &res1, 2) < 0) diff --git a/dpdk/lib/librte_cmdline/cmdline_parse_portlist.h b/dpdk/lib/librte_cmdline/cmdline_parse_portlist.h index 73d70e05..058df3ee 100644 --- a/dpdk/lib/librte_cmdline/cmdline_parse_portlist.h +++ b/dpdk/lib/librte_cmdline/cmdline_parse_portlist.h @@ -61,6 +61,7 @@ #ifndef _PARSE_PORTLIST_H_ #define _PARSE_PORTLIST_H_ +#include #include #ifdef __cplusplus diff --git a/dpdk/lib/librte_cmdline/cmdline_socket.h b/dpdk/lib/librte_cmdline/cmdline_socket.h index 8cc2dfbc..aa6068e7 100644 --- a/dpdk/lib/librte_cmdline/cmdline_socket.h +++ b/dpdk/lib/librte_cmdline/cmdline_socket.h @@ -61,6 +61,9 @@ #ifndef _CMDLINE_SOCKET_H_ #define _CMDLINE_SOCKET_H_ +#include +#include + #ifdef __cplusplus extern "C" { #endif diff --git a/dpdk/lib/librte_compat/rte_compat.h b/dpdk/lib/librte_compat/rte_compat.h index 1c3c8d52..41e8032b 100644 --- a/dpdk/lib/librte_compat/rte_compat.h +++ b/dpdk/lib/librte_compat/rte_compat.h @@ -39,7 +39,7 @@ * When a symol is exported from a library to provide an API, it also provides a * calling convention (ABI) that is embodied in its name, return type, * arguments, etc. On occasion that function may need to change to accommodate - * new functionality, behavior, etc. When that occurs, it is desireable to + * new functionality, behavior, etc. When that occurs, it is desirable to * allow for backwards compatibility for a time with older binaries that are * dynamically linked to the dpdk. To support that, the __vsym and * VERSION_SYMBOL macros are created. They, in conjunction with the diff --git a/dpdk/lib/librte_cryptodev/Makefile b/dpdk/lib/librte_cryptodev/Makefile index 314a0466..8e780b83 100644 --- a/dpdk/lib/librte_cryptodev/Makefile +++ b/dpdk/lib/librte_cryptodev/Makefile @@ -34,14 +34,16 @@ include $(RTE_SDK)/mk/rte.vars.mk LIB = librte_cryptodev.a # library version -LIBABIVER := 1 +LIBABIVER := 4 # build flags CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) +LDLIBS += -lrte_eal -lrte_mempool -lrte_ring -lrte_mbuf +LDLIBS += -lrte_kvargs # library source files -SRCS-y += rte_cryptodev.c +SRCS-y += rte_cryptodev.c rte_cryptodev_pmd.c # export include files SYMLINK-y-include += rte_crypto.h @@ -52,11 +54,4 @@ SYMLINK-y-include += rte_cryptodev_pmd.h # versioning export map EXPORT_MAP := rte_cryptodev_version.map -# library dependencies -DEPDIRS-y += lib/librte_eal -DEPDIRS-y += lib/librte_mempool -DEPDIRS-y += lib/librte_ring -DEPDIRS-y += lib/librte_mbuf -DEPDIRS-y += lib/librte_kvargs - include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/lib/librte_cryptodev/rte_crypto.h b/dpdk/lib/librte_cryptodev/rte_crypto.h index 5bc3eaa7..6f0b2973 100644 --- a/dpdk/lib/librte_cryptodev/rte_crypto.h +++ b/dpdk/lib/librte_cryptodev/rte_crypto.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -48,6 +48,7 @@ extern "C" { #include #include #include +#include #include "rte_crypto_sym.h" @@ -65,8 +66,6 @@ enum rte_crypto_op_status { /**< Operation completed successfully */ RTE_CRYPTO_OP_STATUS_NOT_PROCESSED, /**< Operation has not yet been processed by a crypto device */ - RTE_CRYPTO_OP_STATUS_ENQUEUED, - /**< Operation is enqueued on device */ RTE_CRYPTO_OP_STATUS_AUTH_FAILED, /**< Authentication verification failed */ RTE_CRYPTO_OP_STATUS_INVALID_SESSION, @@ -80,6 +79,17 @@ enum rte_crypto_op_status { /**< Error handling operation */ }; +/** + * Crypto operation session type. This is used to specify whether a crypto + * operation has session structure attached for immutable parameters or if all + * operation information is included in the operation data structure. + */ +enum rte_crypto_op_sess_type { + RTE_CRYPTO_OP_WITH_SESSION, /**< Session based crypto operation */ + RTE_CRYPTO_OP_SESSIONLESS, /**< Session-less crypto operation */ + RTE_CRYPTO_OP_SECURITY_SESSION /**< Security session crypto operation */ +}; + /** * Cryptographic Operation. * @@ -91,31 +101,32 @@ enum rte_crypto_op_status { * rte_cryptodev_enqueue_burst() / rte_cryptodev_dequeue_burst() . */ struct rte_crypto_op { - enum rte_crypto_op_type type; + uint8_t type; /**< operation type */ - - enum rte_crypto_op_status status; + uint8_t status; /**< * operation status - this is reset to * RTE_CRYPTO_OP_STATUS_NOT_PROCESSED on allocation from mempool and * will be set to RTE_CRYPTO_OP_STATUS_SUCCESS after crypto operation * is successfully processed by a crypto PMD */ + uint8_t sess_type; + /**< operation session type */ + uint8_t reserved[5]; + /**< Reserved bytes to fill 64 bits for future additions */ struct rte_mempool *mempool; /**< crypto operation mempool which operation is allocated from */ - phys_addr_t phys_addr; + rte_iova_t phys_addr; /**< physical address of crypto operation */ - void *opaque_data; - /**< Opaque pointer for user data */ - + __extension__ union { - struct rte_crypto_sym_op *sym; + struct rte_crypto_sym_op sym[0]; /**< Symmetric operation parameters */ }; /**< operation specific parameters */ -} __rte_cache_aligned; +}; /** * Reset the fields of a crypto operation to their default values. @@ -128,22 +139,16 @@ __rte_crypto_op_reset(struct rte_crypto_op *op, enum rte_crypto_op_type type) { op->type = type; op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; + op->sess_type = RTE_CRYPTO_OP_SESSIONLESS; switch (type) { case RTE_CRYPTO_OP_TYPE_SYMMETRIC: - /** Symmetric operation structure starts after the end of the - * rte_crypto_op structure. - */ - op->sym = (struct rte_crypto_sym_op *)(op + 1); - op->type = type; - __rte_crypto_sym_op_reset(op->sym); break; + case RTE_CRYPTO_OP_TYPE_UNDEFINED: default: break; } - - op->opaque_data = NULL; } /** @@ -263,8 +268,9 @@ rte_crypto_op_alloc(struct rte_mempool *mempool, enum rte_crypto_op_type type) * @param nb_ops Number of crypto operations to allocate * * @returns - * - On success returns a valid rte_crypto_op structure - * - On failure returns NULL + * - nb_ops if the number of operations requested were allocated. + * - 0 if the requested number of ops are not available. + * None are allocated in this case. */ static inline unsigned @@ -405,6 +411,8 @@ rte_crypto_op_attach_sym_session(struct rte_crypto_op *op, if (unlikely(op->type != RTE_CRYPTO_OP_TYPE_SYMMETRIC)) return -1; + op->sess_type = RTE_CRYPTO_OP_WITH_SESSION; + return __rte_crypto_sym_op_attach_sym_session(op->sym, sess); } diff --git a/dpdk/lib/librte_cryptodev/rte_crypto_sym.h b/dpdk/lib/librte_cryptodev/rte_crypto_sym.h index d9bd8210..c981f0b9 100644 --- a/dpdk/lib/librte_cryptodev/rte_crypto_sym.h +++ b/dpdk/lib/librte_cryptodev/rte_crypto_sym.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -51,6 +51,7 @@ extern "C" { #include #include #include +#include /** Symmetric Cipher Algorithms */ @@ -67,28 +68,12 @@ enum rte_crypto_cipher_algorithm { RTE_CRYPTO_CIPHER_AES_CBC, /**< AES algorithm in CBC mode */ - RTE_CRYPTO_CIPHER_AES_CCM, - /**< AES algorithm in CCM mode. When this cipher algorithm is used the - * *RTE_CRYPTO_AUTH_AES_CCM* element of the - * *rte_crypto_hash_algorithm* enum MUST be used to set up the related - * *rte_crypto_auth_xform* structure in the session context or in - * the op_params of the crypto operation structure in the case of a - * session-less crypto operation - */ RTE_CRYPTO_CIPHER_AES_CTR, /**< AES algorithm in Counter mode */ RTE_CRYPTO_CIPHER_AES_ECB, /**< AES algorithm in ECB mode */ RTE_CRYPTO_CIPHER_AES_F8, /**< AES algorithm in F8 mode */ - RTE_CRYPTO_CIPHER_AES_GCM, - /**< AES algorithm in GCM mode. When this cipher algorithm is used the - * *RTE_CRYPTO_AUTH_AES_GCM* element of the - * *rte_crypto_auth_algorithm* enum MUST be used to set up the related - * *rte_crypto_auth_setup_data* structure in the session context or in - * the op_params of the crypto operation structure in the case of a - * session-less crypto operation. - */ RTE_CRYPTO_CIPHER_AES_XTS, /**< AES algorithm in XTS mode */ @@ -96,17 +81,39 @@ enum rte_crypto_cipher_algorithm { /**< (A)RC4 cipher algorithm */ RTE_CRYPTO_CIPHER_KASUMI_F8, - /**< Kasumi algorithm in F8 mode */ + /**< KASUMI algorithm in F8 mode */ RTE_CRYPTO_CIPHER_SNOW3G_UEA2, - /**< SNOW3G algorithm in UEA2 mode */ + /**< SNOW 3G algorithm in UEA2 mode */ RTE_CRYPTO_CIPHER_ZUC_EEA3, /**< ZUC algorithm in EEA3 mode */ + RTE_CRYPTO_CIPHER_DES_CBC, + /**< DES algorithm in CBC mode */ + + RTE_CRYPTO_CIPHER_AES_DOCSISBPI, + /**< AES algorithm using modes required by + * DOCSIS Baseline Privacy Plus Spec. + * Chained mbufs are not supported in this mode, i.e. rte_mbuf.next + * for m_src and m_dst in the rte_crypto_sym_op must be NULL. + */ + + RTE_CRYPTO_CIPHER_DES_DOCSISBPI, + /**< DES algorithm using modes required by + * DOCSIS Baseline Privacy Plus Spec. + * Chained mbufs are not supported in this mode, i.e. rte_mbuf.next + * for m_src and m_dst in the rte_crypto_sym_op must be NULL. + */ + RTE_CRYPTO_CIPHER_LIST_END + }; +/** Cipher algorithm name strings */ +extern const char * +rte_crypto_cipher_algorithm_strings[]; + /** Symmetric Cipher Direction */ enum rte_crypto_cipher_operation { RTE_CRYPTO_CIPHER_OP_ENCRYPT, @@ -115,6 +122,10 @@ enum rte_crypto_cipher_operation { /**< Decrypt cipher operation */ }; +/** Cipher operation name strings */ +extern const char * +rte_crypto_cipher_operation_strings[]; + /** * Symmetric Cipher Setup Data. * @@ -132,7 +143,7 @@ struct rte_crypto_cipher_xform { struct { uint8_t *data; /**< pointer to key data */ - size_t length; /**< key length in bytes */ + uint16_t length;/**< key length in bytes */ } key; /**< Cipher key * @@ -149,9 +160,6 @@ struct rte_crypto_cipher_xform { * Cipher key length is in bytes. For AES it can be 128 bits (16 bytes), * 192 bits (24 bytes) or 256 bits (32 bytes). * - * For the CCM mode of operation, the only supported key length is 128 - * bits (16 bytes). - * * For the RTE_CRYPTO_CIPHER_AES_F8 mode of operation, key.length * should be set to the combined length of the encryption key and the * keymask. Since the keymask and the encryption key are the same size, @@ -163,6 +171,57 @@ struct rte_crypto_cipher_xform { * - Each key can be either 128 bits (16 bytes) or 256 bits (32 bytes). * - Both keys must have the same size. **/ + struct { + uint16_t offset; + /**< Starting point for Initialisation Vector or Counter, + * specified as number of bytes from start of crypto + * operation (rte_crypto_op). + * + * - For block ciphers in CBC or F8 mode, or for KASUMI + * in F8 mode, or for SNOW 3G in UEA2 mode, this is the + * Initialisation Vector (IV) value. + * + * - For block ciphers in CTR mode, this is the counter. + * + * - For GCM mode, this is either the IV (if the length + * is 96 bits) or J0 (for other sizes), where J0 is as + * defined by NIST SP800-38D. Regardless of the IV + * length, a full 16 bytes needs to be allocated. + * + * - For CCM mode, the first byte is reserved, and the + * nonce should be written starting at &iv[1] (to allow + * space for the implementation to write in the flags + * in the first byte). Note that a full 16 bytes should + * be allocated, even though the length field will + * have a value less than this. Note that the PMDs may + * modify the memory reserved (the first byte and the + * final padding) + * + * - For AES-XTS, this is the 128bit tweak, i, from + * IEEE Std 1619-2007. + * + * For optimum performance, the data pointed to SHOULD + * be 8-byte aligned. + */ + uint16_t length; + /**< Length of valid IV data. + * + * - For block ciphers in CBC or F8 mode, or for KASUMI + * in F8 mode, or for SNOW 3G in UEA2 mode, this is the + * length of the IV (which must be the same as the + * block length of the cipher). + * + * - For block ciphers in CTR mode, this is the length + * of the counter (which must be the same as the block + * length of the cipher). + * + * - For GCM mode, this is either 12 (for 96-bit IVs) + * or 16, in which case data points to J0. + * + * - For CCM mode, this is the length of the nonce, + * which can be in the range 7 to 13 inclusive. + */ + } iv; /**< Initialisation vector parameters */ }; /** Symmetric Authentication / Hash Algorithms */ @@ -172,38 +231,15 @@ enum rte_crypto_auth_algorithm { RTE_CRYPTO_AUTH_AES_CBC_MAC, /**< AES-CBC-MAC algorithm. Only 128-bit keys are supported. */ - RTE_CRYPTO_AUTH_AES_CCM, - /**< AES algorithm in CCM mode. This is an authenticated cipher. When - * this hash algorithm is used, the *RTE_CRYPTO_CIPHER_AES_CCM* - * element of the *rte_crypto_cipher_algorithm* enum MUST be used to - * set up the related rte_crypto_cipher_setup_data structure in the - * session context or the corresponding parameter in the crypto - * operation data structures op_params parameter MUST be set for a - * session-less crypto operation. - */ RTE_CRYPTO_AUTH_AES_CMAC, /**< AES CMAC algorithm. */ - RTE_CRYPTO_AUTH_AES_GCM, - /**< AES algorithm in GCM mode. When this hash algorithm - * is used, the RTE_CRYPTO_CIPHER_AES_GCM element of the - * rte_crypto_cipher_algorithm enum MUST be used to set up the related - * rte_crypto_cipher_setup_data structure in the session context, or - * the corresponding parameter in the crypto operation data structures - * op_params parameter MUST be set for a session-less crypto operation. - */ RTE_CRYPTO_AUTH_AES_GMAC, - /**< AES GMAC algorithm. When this hash algorithm - * is used, the RTE_CRYPTO_CIPHER_AES_GCM element of the - * rte_crypto_cipher_algorithm enum MUST be used to set up the related - * rte_crypto_cipher_setup_data structure in the session context, or - * the corresponding parameter in the crypto operation data structures - * op_params parameter MUST be set for a session-less crypto operation. - */ + /**< AES GMAC algorithm. */ RTE_CRYPTO_AUTH_AES_XCBC_MAC, /**< AES XCBC algorithm. */ RTE_CRYPTO_AUTH_KASUMI_F9, - /**< Kasumi algorithm in F9 mode. */ + /**< KASUMI algorithm in F9 mode. */ RTE_CRYPTO_AUTH_MD5, /**< MD5 algorithm */ @@ -232,7 +268,7 @@ enum rte_crypto_auth_algorithm { /**< HMAC using 512 bit SHA algorithm. */ RTE_CRYPTO_AUTH_SNOW3G_UIA2, - /**< SNOW3G algorithm in UIA2 mode. */ + /**< SNOW 3G algorithm in UIA2 mode. */ RTE_CRYPTO_AUTH_ZUC_EIA3, /**< ZUC algorithm in EIA3 mode */ @@ -240,12 +276,20 @@ enum rte_crypto_auth_algorithm { RTE_CRYPTO_AUTH_LIST_END }; +/** Authentication algorithm name strings */ +extern const char * +rte_crypto_auth_algorithm_strings[]; + /** Symmetric Authentication / Hash Operations */ enum rte_crypto_auth_operation { RTE_CRYPTO_AUTH_OP_VERIFY, /**< Verify authentication digest */ RTE_CRYPTO_AUTH_OP_GENERATE /**< Generate authentication digest */ }; +/** Authentication operation name strings */ +extern const char * +rte_crypto_auth_operation_strings[]; + /** * Authentication / Hash transform data. * @@ -261,7 +305,7 @@ struct rte_crypto_auth_xform { struct { uint8_t *data; /**< pointer to key data */ - size_t length; /**< key length in bytes */ + uint16_t length;/**< key length in bytes */ } key; /**< Authentication key data. * The authentication key length MUST be less than or equal to the @@ -270,45 +314,122 @@ struct rte_crypto_auth_xform { * (for example RFC 2104, FIPS 198a). */ - uint32_t digest_length; + struct { + uint16_t offset; + /**< Starting point for Initialisation Vector or Counter, + * specified as number of bytes from start of crypto + * operation (rte_crypto_op). + * + * - For SNOW 3G in UIA2 mode, for ZUC in EIA3 mode and + * for AES-GMAC, this is the authentication + * Initialisation Vector (IV) value. + * + * - For KASUMI in F9 mode and other authentication + * algorithms, this field is not used. + * + * For optimum performance, the data pointed to SHOULD + * be 8-byte aligned. + */ + uint16_t length; + /**< Length of valid IV data. + * + * - For SNOW3G in UIA2 mode, for ZUC in EIA3 mode and + * for AES-GMAC, this is the length of the IV. + * + * - For KASUMI in F9 mode and other authentication + * algorithms, this field is not used. + * + */ + } iv; /**< Initialisation vector parameters */ + + uint16_t digest_length; /**< Length of the digest to be returned. If the verify option is set, * this specifies the length of the digest to be compared for the * session. * + * It is the caller's responsibility to ensure that the + * digest length is compliant with the hash algorithm being used. * If the value is less than the maximum length allowed by the hash, - * the result shall be truncated. If the value is greater than the - * maximum length allowed by the hash then an error will be generated - * by *rte_cryptodev_sym_session_create* or by the - * *rte_cryptodev_sym_enqueue_burst* if using session-less APIs. + * the result shall be truncated. */ +}; - uint32_t add_auth_data_length; + +/** Symmetric AEAD Algorithms */ +enum rte_crypto_aead_algorithm { + RTE_CRYPTO_AEAD_AES_CCM = 1, + /**< AES algorithm in CCM mode. */ + RTE_CRYPTO_AEAD_AES_GCM, + /**< AES algorithm in GCM mode. */ + RTE_CRYPTO_AEAD_LIST_END +}; + +/** AEAD algorithm name strings */ +extern const char * +rte_crypto_aead_algorithm_strings[]; + +/** Symmetric AEAD Operations */ +enum rte_crypto_aead_operation { + RTE_CRYPTO_AEAD_OP_ENCRYPT, + /**< Encrypt and generate digest */ + RTE_CRYPTO_AEAD_OP_DECRYPT + /**< Verify digest and decrypt */ +}; + +/** Authentication operation name strings */ +extern const char * +rte_crypto_aead_operation_strings[]; + +struct rte_crypto_aead_xform { + enum rte_crypto_aead_operation op; + /**< AEAD operation type */ + enum rte_crypto_aead_algorithm algo; + /**< AEAD algorithm selection */ + + struct { + uint8_t *data; /**< pointer to key data */ + uint16_t length;/**< key length in bytes */ + } key; + + struct { + uint16_t offset; + /**< Starting point for Initialisation Vector or Counter, + * specified as number of bytes from start of crypto + * operation (rte_crypto_op). + * + * - For GCM mode, this is either the IV (if the length + * is 96 bits) or J0 (for other sizes), where J0 is as + * defined by NIST SP800-38D. Regardless of the IV + * length, a full 16 bytes needs to be allocated. + * + * - For CCM mode, the first byte is reserved, and the + * nonce should be written starting at &iv[1] (to allow + * space for the implementation to write in the flags + * in the first byte). Note that a full 16 bytes should + * be allocated, even though the length field will + * have a value less than this. + * + * For optimum performance, the data pointed to SHOULD + * be 8-byte aligned. + */ + uint16_t length; + /**< Length of valid IV data. + * + * - For GCM mode, this is either 12 (for 96-bit IVs) + * or 16, in which case data points to J0. + * + * - For CCM mode, this is the length of the nonce, + * which can be in the range 7 to 13 inclusive. + */ + } iv; /**< Initialisation vector parameters */ + + uint16_t digest_length; + + uint16_t aad_length; /**< The length of the additional authenticated data (AAD) in bytes. - * The maximum permitted value is 240 bytes, unless otherwise specified - * below. - * - * This field must be specified when the hash algorithm is one of the - * following: - * - * - For SNOW3G (@ref RTE_CRYPTO_AUTH_SNOW3G_UIA2), this is the - * length of the IV (which should be 16). - * - * - For GCM (@ref RTE_CRYPTO_AUTH_AES_GCM). In this case, this is - * the length of the Additional Authenticated Data (called A, in NIST - * SP800-38D). - * - * - For CCM (@ref RTE_CRYPTO_AUTH_AES_CCM). In this case, this is - * the length of the associated data (called A, in NIST SP800-38C). - * Note that this does NOT include the length of any padding, or the - * 18 bytes reserved at the start of the above field to store the - * block B0 and the encoded length. The maximum permitted value in - * this case is 222 bytes. - * - * @note - * For AES-GMAC (@ref RTE_CRYPTO_AUTH_AES_GMAC) mode of operation - * this field is not used and should be set to 0. Instead the length - * of the AAD data is specified in the message length to hash field of - * the rte_crypto_sym_op_data structure. + * For CCM mode, this is the length of the actual AAD, even though + * it is required to reserve 18 bytes before the AAD and padding + * at the end of it, so a multiple of 16 bytes is allocated. */ }; @@ -316,7 +437,8 @@ struct rte_crypto_auth_xform { enum rte_crypto_sym_xform_type { RTE_CRYPTO_SYM_XFORM_NOT_SPECIFIED = 0, /**< No xform specified */ RTE_CRYPTO_SYM_XFORM_AUTH, /**< Authentication xform */ - RTE_CRYPTO_SYM_XFORM_CIPHER /**< Cipher xform */ + RTE_CRYPTO_SYM_XFORM_CIPHER, /**< Cipher xform */ + RTE_CRYPTO_SYM_XFORM_AEAD /**< AEAD xform */ }; /** @@ -333,25 +455,17 @@ struct rte_crypto_sym_xform { /**< next xform in chain */ enum rte_crypto_sym_xform_type type ; /**< xform type */ + RTE_STD_C11 union { struct rte_crypto_auth_xform auth; /**< Authentication / hash xform */ struct rte_crypto_cipher_xform cipher; /**< Cipher xform */ + struct rte_crypto_aead_xform aead; + /**< AEAD xform */ }; }; -/** - * Crypto operation session type. This is used to specify whether a crypto - * operation has session structure attached for immutable parameters or if all - * operation information is included in the operation data structure. - */ -enum rte_crypto_sym_op_sess_type { - RTE_CRYPTO_SYM_OP_WITH_SESSION, /**< Session based crypto operation */ - RTE_CRYPTO_SYM_OP_SESSIONLESS /**< Session-less crypto operation */ -}; - - struct rte_cryptodev_sym_session; /** @@ -364,241 +478,215 @@ struct rte_cryptodev_sym_session; * it must have a valid *rte_mbuf* structure attached, via m_src parameter, * which contains the source data which the crypto operation is to be performed * on. + * While the mbuf is in use by a crypto operation no part of the mbuf should be + * changed by the application as the device may read or write to any part of the + * mbuf. In the case of hardware crypto devices some or all of the mbuf + * may be DMAed in and out of the device, so writing over the original data, + * though only the part specified by the rte_crypto_sym_op for transformation + * will be changed. + * Out-of-place (OOP) operation, where the source mbuf is different to the + * destination mbuf, is a special case. Data will be copied from m_src to m_dst. + * The part copied includes all the parts of the source mbuf that will be + * operated on, based on the cipher.data.offset+cipher.data.length and + * auth.data.offset+auth.data.length values in the rte_crypto_sym_op. The part + * indicated by the cipher parameters will be transformed, any extra data around + * this indicated by the auth parameters will be copied unchanged from source to + * destination mbuf. + * Also in OOP operation the cipher.data.offset and auth.data.offset apply to + * both source and destination mbufs. As these offsets are relative to the + * data_off parameter in each mbuf this can result in the data written to the + * destination buffer being at a different alignment, relative to buffer start, + * to the data in the source buffer. */ struct rte_crypto_sym_op { struct rte_mbuf *m_src; /**< source mbuf */ struct rte_mbuf *m_dst; /**< destination mbuf */ - enum rte_crypto_sym_op_sess_type sess_type; - + RTE_STD_C11 union { struct rte_cryptodev_sym_session *session; /**< Handle for the initialised session context */ struct rte_crypto_sym_xform *xform; /**< Session-less API crypto operation parameters */ + struct rte_security_session *sec_session; + /**< Handle for the initialised security session context */ }; - struct { + RTE_STD_C11 + union { struct { - uint32_t offset; - /**< Starting point for cipher processing, specified - * as number of bytes from start of data in the source - * buffer. The result of the cipher operation will be - * written back into the output buffer starting at - * this location. - * - * @note - * For Snow3G @ RTE_CRYPTO_CIPHER_SNOW3G_UEA2 - * and KASUMI @ RTE_CRYPTO_CIPHER_KASUMI_F8, - * this field should be in bits. - */ - - uint32_t length; - /**< The message length, in bytes, of the source buffer - * on which the cryptographic operation will be - * computed. This must be a multiple of the block size - * if a block cipher is being used. This is also the - * same as the result length. - * - * @note - * In the case of CCM @ref RTE_CRYPTO_AUTH_AES_CCM, - * this value should not include the length of the - * padding or the length of the MAC; the driver will - * compute the actual number of bytes over which the - * encryption will occur, which will include these - * values. - * - * @note - * For AES-GMAC @ref RTE_CRYPTO_AUTH_AES_GMAC, this - * field should be set to 0. - * - * @note - * For Snow3G @ RTE_CRYPTO_AUTH_SNOW3G_UEA2 - * and KASUMI @ RTE_CRYPTO_CIPHER_KASUMI_F8, - * this field should be in bits. - */ - } data; /**< Data offsets and length for ciphering */ + struct { + uint32_t offset; + /**< Starting point for AEAD processing, specified as + * number of bytes from start of packet in source + * buffer. + */ + uint32_t length; + /**< The message length, in bytes, of the source buffer + * on which the cryptographic operation will be + * computed. This must be a multiple of the block size + */ + } data; /**< Data offsets and length for AEAD */ + struct { + uint8_t *data; + /**< This points to the location where the digest result + * should be inserted (in the case of digest generation) + * or where the purported digest exists (in the case of + * digest verification). + * + * At session creation time, the client specified the + * digest result length with the digest_length member + * of the @ref rte_crypto_auth_xform structure. For + * physical crypto devices the caller must allocate at + * least digest_length of physically contiguous memory + * at this location. + * + * For digest generation, the digest result will + * overwrite any data at this location. + * + * @note + * For GCM (@ref RTE_CRYPTO_AEAD_AES_GCM), for + * "digest result" read "authentication tag T". + */ + rte_iova_t phys_addr; + /**< Physical address of digest */ + } digest; /**< Digest parameters */ + struct { + uint8_t *data; + /**< Pointer to Additional Authenticated Data (AAD) + * needed for authenticated cipher mechanisms (CCM and + * GCM) + * + * Specifically for CCM (@ref RTE_CRYPTO_AEAD_AES_CCM), + * the caller should setup this field as follows: + * + * - the additional authentication data itself should + * be written starting at an offset of 18 bytes into + * the array, leaving room for the first block (16 bytes) + * and the length encoding in the first two bytes of the + * second block. + * + * - the array should be big enough to hold the above + * fields, plus any padding to round this up to the + * nearest multiple of the block size (16 bytes). + * Padding will be added by the implementation. + * + * - Note that PMDs may modify the memory reserved + * (first 18 bytes and the final padding). + * + * Finally, for GCM (@ref RTE_CRYPTO_AEAD_AES_GCM), the + * caller should setup this field as follows: + * + * - the AAD is written in starting at byte 0 + * - the array must be big enough to hold the AAD, plus + * any space to round this up to the nearest multiple + * of the block size (16 bytes). + * + */ + rte_iova_t phys_addr; /**< physical address */ + } aad; + /**< Additional authentication parameters */ + } aead; struct { - uint8_t *data; - /**< Initialisation Vector or Counter. - * - * - For block ciphers in CBC or F8 mode, or for Kasumi - * in F8 mode, or for SNOW3G in UEA2 mode, this is the - * Initialisation Vector (IV) value. - * - * - For block ciphers in CTR mode, this is the counter. - * - * - For GCM mode, this is either the IV (if the length - * is 96 bits) or J0 (for other sizes), where J0 is as - * defined by NIST SP800-38D. Regardless of the IV - * length, a full 16 bytes needs to be allocated. - * - * - For CCM mode, the first byte is reserved, and the - * nonce should be written starting at &iv[1] (to allow - * space for the implementation to write in the flags - * in the first byte). Note that a full 16 bytes should - * be allocated, even though the length field will - * have a value less than this. - * - * - For AES-XTS, this is the 128bit tweak, i, from - * IEEE Std 1619-2007. - * - * For optimum performance, the data pointed to SHOULD - * be 8-byte aligned. - */ - phys_addr_t phys_addr; - uint16_t length; - /**< Length of valid IV data. - * - * - For block ciphers in CBC or F8 mode, or for Kasumi - * in F8 mode, or for SNOW3G in UEA2 mode, this is the - * length of the IV (which must be the same as the - * block length of the cipher). - * - * - For block ciphers in CTR mode, this is the length - * of the counter (which must be the same as the block - * length of the cipher). - * - * - For GCM mode, this is either 12 (for 96-bit IVs) - * or 16, in which case data points to J0. - * - * - For CCM mode, this is the length of the nonce, - * which can be in the range 7 to 13 inclusive. - */ - } iv; /**< Initialisation vector parameters */ - } cipher; + struct { + struct { + uint32_t offset; + /**< Starting point for cipher processing, + * specified as number of bytes from start + * of data in the source buffer. + * The result of the cipher operation will be + * written back into the output buffer + * starting at this location. + * + * @note + * For SNOW 3G @ RTE_CRYPTO_CIPHER_SNOW3G_UEA2, + * KASUMI @ RTE_CRYPTO_CIPHER_KASUMI_F8 + * and ZUC @ RTE_CRYPTO_CIPHER_ZUC_EEA3, + * this field should be in bits. + */ + uint32_t length; + /**< The message length, in bytes, of the + * source buffer on which the cryptographic + * operation will be computed. + * This must be a multiple of the block size + * if a block cipher is being used. This is + * also the same as the result length. + * + * @note + * For SNOW 3G @ RTE_CRYPTO_AUTH_SNOW3G_UEA2, + * KASUMI @ RTE_CRYPTO_CIPHER_KASUMI_F8 + * and ZUC @ RTE_CRYPTO_CIPHER_ZUC_EEA3, + * this field should be in bits. + */ + } data; /**< Data offsets and length for ciphering */ + } cipher; - struct { - struct { - uint32_t offset; - /**< Starting point for hash processing, specified as - * number of bytes from start of packet in source - * buffer. - * - * @note - * For CCM and GCM modes of operation, this field is - * ignored. The field @ref aad field - * should be set instead. - * - * @note For AES-GMAC (@ref RTE_CRYPTO_AUTH_AES_GMAC) - * mode of operation, this field specifies the start - * of the AAD data in the source buffer. - * - * @note - * For Snow3G @ RTE_CRYPTO_AUTH_SNOW3G_UIA2 - * and KASUMI @ RTE_CRYPTO_AUTH_KASUMI_F9, - * this field should be in bits. - */ + struct { + struct { + uint32_t offset; + /**< Starting point for hash processing, + * specified as number of bytes from start of + * packet in source buffer. + * + * @note + * For SNOW 3G @ RTE_CRYPTO_AUTH_SNOW3G_UIA2, + * KASUMI @ RTE_CRYPTO_AUTH_KASUMI_F9 + * and ZUC @ RTE_CRYPTO_AUTH_ZUC_EIA3, + * this field should be in bits. + * + * @note + * For KASUMI @ RTE_CRYPTO_AUTH_KASUMI_F9, + * this offset should be such that + * data to authenticate starts at COUNT. + */ + uint32_t length; + /**< The message length, in bytes, of the source + * buffer that the hash will be computed on. + * + * @note + * For SNOW 3G @ RTE_CRYPTO_AUTH_SNOW3G_UIA2, + * KASUMI @ RTE_CRYPTO_AUTH_KASUMI_F9 + * and ZUC @ RTE_CRYPTO_AUTH_ZUC_EIA3, + * this field should be in bits. + * + * @note + * For KASUMI @ RTE_CRYPTO_AUTH_KASUMI_F9, + * the length should include the COUNT, + * FRESH, message, direction bit and padding + * (to be multiple of 8 bits). + */ + } data; + /**< Data offsets and length for authentication */ - uint32_t length; - /**< The message length, in bytes, of the source - * buffer that the hash will be computed on. - * - * @note - * For CCM and GCM modes of operation, this field is - * ignored. The field @ref aad field should be set - * instead. - * - * @note - * For AES-GMAC @ref RTE_CRYPTO_AUTH_AES_GMAC mode - * of operation, this field specifies the length of - * the AAD data in the source buffer. - * - * @note - * For Snow3G @ RTE_CRYPTO_AUTH_SNOW3G_UIA2 - * and KASUMI @ RTE_CRYPTO_AUTH_KASUMI_F9, - * this field should be in bits. - */ - } data; /**< Data offsets and length for authentication */ - - struct { - uint8_t *data; - /**< If this member of this structure is set this is a - * pointer to the location where the digest result - * should be inserted (in the case of digest generation) - * or where the purported digest exists (in the case of - * digest verification). - * - * At session creation time, the client specified the - * digest result length with the digest_length member - * of the @ref rte_crypto_auth_xform structure. For - * physical crypto devices the caller must allocate at - * least digest_length of physically contiguous memory - * at this location. - * - * For digest generation, the digest result will - * overwrite any data at this location. - * - * @note - * For GCM (@ref RTE_CRYPTO_AUTH_AES_GCM), for - * "digest result" read "authentication tag T". - * - * If this member is not set the digest result is - * understood to be in the destination buffer for - * digest generation, and in the source buffer for - * digest verification. The location of the digest - * result in this case is immediately following the - * region over which the digest is computed. - */ - phys_addr_t phys_addr; - /**< Physical address of digest */ - uint16_t length; - /**< Length of digest */ - } digest; /**< Digest parameters */ - - struct { - uint8_t *data; - /**< Pointer to Additional Authenticated Data (AAD) - * needed for authenticated cipher mechanisms (CCM and - * GCM), and to the IV for SNOW3G authentication - * (@ref RTE_CRYPTO_AUTH_SNOW3G_UIA2). For other - * authentication mechanisms this pointer is ignored. - * - * The length of the data pointed to by this field is - * set up for the session in the @ref - * rte_crypto_auth_xform structure as part of the @ref - * rte_cryptodev_sym_session_create function call. - * This length must not exceed 240 bytes. - * - * Specifically for CCM (@ref RTE_CRYPTO_AUTH_AES_CCM), - * the caller should setup this field as follows: - * - * - the nonce should be written starting at an offset - * of one byte into the array, leaving room for the - * implementation to write in the flags to the first - * byte. - * - * - the additional authentication data itself should - * be written starting at an offset of 18 bytes into - * the array, leaving room for the length encoding in - * the first two bytes of the second block. - * - * - the array should be big enough to hold the above - * fields, plus any padding to round this up to the - * nearest multiple of the block size (16 bytes). - * Padding will be added by the implementation. - * - * Finally, for GCM (@ref RTE_CRYPTO_AUTH_AES_GCM), the - * caller should setup this field as follows: - * - * - the AAD is written in starting at byte 0 - * - the array must be big enough to hold the AAD, plus - * any space to round this up to the nearest multiple - * of the block size (16 bytes). - * - * @note - * For AES-GMAC (@ref RTE_CRYPTO_AUTH_AES_GMAC) mode of - * operation, this field is not used and should be set - * to 0. Instead the AAD data should be placed in the - * source buffer. - */ - phys_addr_t phys_addr; /**< physical address */ - uint16_t length; /**< Length of digest */ - } aad; - /**< Additional authentication parameters */ - } auth; -} __rte_cache_aligned; + struct { + uint8_t *data; + /**< This points to the location where + * the digest result should be inserted + * (in the case of digest generation) + * or where the purported digest exists + * (in the case of digest verification). + * + * At session creation time, the client + * specified the digest result length with + * the digest_length member of the + * @ref rte_crypto_auth_xform structure. + * For physical crypto devices the caller + * must allocate at least digest_length of + * physically contiguous memory at this + * location. + * + * For digest generation, the digest result + * will overwrite any data at this location. + * + */ + rte_iova_t phys_addr; + /**< Physical address of digest */ + } digest; /**< Digest parameters */ + } auth; + }; + }; +}; /** @@ -610,8 +698,6 @@ static inline void __rte_crypto_sym_op_reset(struct rte_crypto_sym_op *op) { memset(op, 0, sizeof(*op)); - - op->sess_type = RTE_CRYPTO_SYM_OP_SESSIONLESS; } @@ -653,7 +739,6 @@ __rte_crypto_sym_op_attach_sym_session(struct rte_crypto_sym_op *sym_op, struct rte_cryptodev_sym_session *sess) { sym_op->session = sess; - sym_op->sess_type = RTE_CRYPTO_SYM_OP_WITH_SESSION; return 0; } diff --git a/dpdk/lib/librte_cryptodev/rte_cryptodev.c b/dpdk/lib/librte_cryptodev/rte_cryptodev.c index fc4123b6..9fe0d9da 100644 --- a/dpdk/lib/librte_cryptodev/rte_cryptodev.c +++ b/dpdk/lib/librte_cryptodev/rte_cryptodev.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2015-2016 Intel Corporation. All rights reserved. + * Copyright(c) 2015-2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -47,7 +47,6 @@ #include #include #include -#include #include #include #include @@ -59,7 +58,6 @@ #include #include #include -#include #include #include #include @@ -71,6 +69,8 @@ #include "rte_cryptodev.h" #include "rte_cryptodev_pmd.h" +static uint8_t nb_drivers; + struct rte_cryptodev rte_crypto_devices[RTE_CRYPTO_MAX_DEVS]; struct rte_cryptodev *rte_cryptodevs = &rte_crypto_devices[0]; @@ -102,94 +102,248 @@ struct rte_cryptodev_callback { uint32_t active; /**< Callback is executing */ }; -#define RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG ("max_nb_queue_pairs") -#define RTE_CRYPTODEV_VDEV_MAX_NB_SESS_ARG ("max_nb_sessions") -#define RTE_CRYPTODEV_VDEV_SOCKET_ID ("socket_id") +/** + * The crypto cipher algorithm strings identifiers. + * It could be used in application command line. + */ +const char * +rte_crypto_cipher_algorithm_strings[] = { + [RTE_CRYPTO_CIPHER_3DES_CBC] = "3des-cbc", + [RTE_CRYPTO_CIPHER_3DES_ECB] = "3des-ecb", + [RTE_CRYPTO_CIPHER_3DES_CTR] = "3des-ctr", -static const char *cryptodev_vdev_valid_params[] = { - RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG, - RTE_CRYPTODEV_VDEV_MAX_NB_SESS_ARG, - RTE_CRYPTODEV_VDEV_SOCKET_ID + [RTE_CRYPTO_CIPHER_AES_CBC] = "aes-cbc", + [RTE_CRYPTO_CIPHER_AES_CTR] = "aes-ctr", + [RTE_CRYPTO_CIPHER_AES_DOCSISBPI] = "aes-docsisbpi", + [RTE_CRYPTO_CIPHER_AES_ECB] = "aes-ecb", + [RTE_CRYPTO_CIPHER_AES_F8] = "aes-f8", + [RTE_CRYPTO_CIPHER_AES_XTS] = "aes-xts", + + [RTE_CRYPTO_CIPHER_ARC4] = "arc4", + + [RTE_CRYPTO_CIPHER_DES_CBC] = "des-cbc", + [RTE_CRYPTO_CIPHER_DES_DOCSISBPI] = "des-docsisbpi", + + [RTE_CRYPTO_CIPHER_NULL] = "null", + + [RTE_CRYPTO_CIPHER_KASUMI_F8] = "kasumi-f8", + [RTE_CRYPTO_CIPHER_SNOW3G_UEA2] = "snow3g-uea2", + [RTE_CRYPTO_CIPHER_ZUC_EEA3] = "zuc-eea3" }; -static uint8_t -number_of_sockets(void) -{ - int sockets = 0; - int i; - const struct rte_memseg *ms = rte_eal_get_physmem_layout(); +/** + * The crypto cipher operation strings identifiers. + * It could be used in application command line. + */ +const char * +rte_crypto_cipher_operation_strings[] = { + [RTE_CRYPTO_CIPHER_OP_ENCRYPT] = "encrypt", + [RTE_CRYPTO_CIPHER_OP_DECRYPT] = "decrypt" +}; - for (i = 0; ((i < RTE_MAX_MEMSEG) && (ms[i].addr != NULL)); i++) { - if (sockets < ms[i].socket_id) - sockets = ms[i].socket_id; +/** + * The crypto auth algorithm strings identifiers. + * It could be used in application command line. + */ +const char * +rte_crypto_auth_algorithm_strings[] = { + [RTE_CRYPTO_AUTH_AES_CBC_MAC] = "aes-cbc-mac", + [RTE_CRYPTO_AUTH_AES_CMAC] = "aes-cmac", + [RTE_CRYPTO_AUTH_AES_GMAC] = "aes-gmac", + [RTE_CRYPTO_AUTH_AES_XCBC_MAC] = "aes-xcbc-mac", + + [RTE_CRYPTO_AUTH_MD5] = "md5", + [RTE_CRYPTO_AUTH_MD5_HMAC] = "md5-hmac", + + [RTE_CRYPTO_AUTH_NULL] = "null", + + [RTE_CRYPTO_AUTH_SHA1] = "sha1", + [RTE_CRYPTO_AUTH_SHA1_HMAC] = "sha1-hmac", + + [RTE_CRYPTO_AUTH_SHA224] = "sha2-224", + [RTE_CRYPTO_AUTH_SHA224_HMAC] = "sha2-224-hmac", + [RTE_CRYPTO_AUTH_SHA256] = "sha2-256", + [RTE_CRYPTO_AUTH_SHA256_HMAC] = "sha2-256-hmac", + [RTE_CRYPTO_AUTH_SHA384] = "sha2-384", + [RTE_CRYPTO_AUTH_SHA384_HMAC] = "sha2-384-hmac", + [RTE_CRYPTO_AUTH_SHA512] = "sha2-512", + [RTE_CRYPTO_AUTH_SHA512_HMAC] = "sha2-512-hmac", + + [RTE_CRYPTO_AUTH_KASUMI_F9] = "kasumi-f9", + [RTE_CRYPTO_AUTH_SNOW3G_UIA2] = "snow3g-uia2", + [RTE_CRYPTO_AUTH_ZUC_EIA3] = "zuc-eia3" +}; + +/** + * The crypto AEAD algorithm strings identifiers. + * It could be used in application command line. + */ +const char * +rte_crypto_aead_algorithm_strings[] = { + [RTE_CRYPTO_AEAD_AES_CCM] = "aes-ccm", + [RTE_CRYPTO_AEAD_AES_GCM] = "aes-gcm", +}; + +/** + * The crypto AEAD operation strings identifiers. + * It could be used in application command line. + */ +const char * +rte_crypto_aead_operation_strings[] = { + [RTE_CRYPTO_AEAD_OP_ENCRYPT] = "encrypt", + [RTE_CRYPTO_AEAD_OP_DECRYPT] = "decrypt" +}; + +int +rte_cryptodev_get_cipher_algo_enum(enum rte_crypto_cipher_algorithm *algo_enum, + const char *algo_string) +{ + unsigned int i; + + for (i = 1; i < RTE_DIM(rte_crypto_cipher_algorithm_strings); i++) { + if (strcmp(algo_string, rte_crypto_cipher_algorithm_strings[i]) == 0) { + *algo_enum = (enum rte_crypto_cipher_algorithm) i; + return 0; + } } - /* Number of sockets = maximum socket_id + 1 */ - return ++sockets; + /* Invalid string */ + return -1; } -/** Parse integer from integer argument */ -static int -parse_integer_arg(const char *key __rte_unused, - const char *value, void *extra_args) +int +rte_cryptodev_get_auth_algo_enum(enum rte_crypto_auth_algorithm *algo_enum, + const char *algo_string) { - int *i = (int *) extra_args; + unsigned int i; - *i = atoi(value); - if (*i < 0) { - CDEV_LOG_ERR("Argument has to be positive."); - return -1; + for (i = 1; i < RTE_DIM(rte_crypto_auth_algorithm_strings); i++) { + if (strcmp(algo_string, rte_crypto_auth_algorithm_strings[i]) == 0) { + *algo_enum = (enum rte_crypto_auth_algorithm) i; + return 0; + } } + /* Invalid string */ + return -1; +} + +int +rte_cryptodev_get_aead_algo_enum(enum rte_crypto_aead_algorithm *algo_enum, + const char *algo_string) +{ + unsigned int i; + + for (i = 1; i < RTE_DIM(rte_crypto_aead_algorithm_strings); i++) { + if (strcmp(algo_string, rte_crypto_aead_algorithm_strings[i]) == 0) { + *algo_enum = (enum rte_crypto_aead_algorithm) i; + return 0; + } + } + + /* Invalid string */ + return -1; +} + +/** + * The crypto auth operation strings identifiers. + * It could be used in application command line. + */ +const char * +rte_crypto_auth_operation_strings[] = { + [RTE_CRYPTO_AUTH_OP_VERIFY] = "verify", + [RTE_CRYPTO_AUTH_OP_GENERATE] = "generate" +}; + +const struct rte_cryptodev_symmetric_capability * +rte_cryptodev_sym_capability_get(uint8_t dev_id, + const struct rte_cryptodev_sym_capability_idx *idx) +{ + const struct rte_cryptodev_capabilities *capability; + struct rte_cryptodev_info dev_info; + int i = 0; + + rte_cryptodev_info_get(dev_id, &dev_info); + + while ((capability = &dev_info.capabilities[i++])->op != + RTE_CRYPTO_OP_TYPE_UNDEFINED) { + if (capability->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC) + continue; + + if (capability->sym.xform_type != idx->type) + continue; + + if (idx->type == RTE_CRYPTO_SYM_XFORM_AUTH && + capability->sym.auth.algo == idx->algo.auth) + return &capability->sym; + + if (idx->type == RTE_CRYPTO_SYM_XFORM_CIPHER && + capability->sym.cipher.algo == idx->algo.cipher) + return &capability->sym; + + if (idx->type == RTE_CRYPTO_SYM_XFORM_AEAD && + capability->sym.aead.algo == idx->algo.aead) + return &capability->sym; + } + + return NULL; + +} + +#define param_range_check(x, y) \ + (((x < y.min) || (x > y.max)) || \ + (y.increment != 0 && (x % y.increment) != 0)) + +int +rte_cryptodev_sym_capability_check_cipher( + const struct rte_cryptodev_symmetric_capability *capability, + uint16_t key_size, uint16_t iv_size) +{ + if (param_range_check(key_size, capability->cipher.key_size)) + return -1; + + if (param_range_check(iv_size, capability->cipher.iv_size)) + return -1; + return 0; } int -rte_cryptodev_parse_vdev_init_params(struct rte_crypto_vdev_init_params *params, - const char *input_args) +rte_cryptodev_sym_capability_check_auth( + const struct rte_cryptodev_symmetric_capability *capability, + uint16_t key_size, uint16_t digest_size, uint16_t iv_size) { - struct rte_kvargs *kvlist = NULL; - int ret = 0; + if (param_range_check(key_size, capability->auth.key_size)) + return -1; - if (params == NULL) - return -EINVAL; + if (param_range_check(digest_size, capability->auth.digest_size)) + return -1; - if (input_args) { - kvlist = rte_kvargs_parse(input_args, - cryptodev_vdev_valid_params); - if (kvlist == NULL) - return -1; + if (param_range_check(iv_size, capability->auth.iv_size)) + return -1; - ret = rte_kvargs_process(kvlist, - RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG, - &parse_integer_arg, - ¶ms->max_nb_queue_pairs); - if (ret < 0) - goto free_kvlist; + return 0; +} - ret = rte_kvargs_process(kvlist, - RTE_CRYPTODEV_VDEV_MAX_NB_SESS_ARG, - &parse_integer_arg, - ¶ms->max_nb_sessions); - if (ret < 0) - goto free_kvlist; +int +rte_cryptodev_sym_capability_check_aead( + const struct rte_cryptodev_symmetric_capability *capability, + uint16_t key_size, uint16_t digest_size, uint16_t aad_size, + uint16_t iv_size) +{ + if (param_range_check(key_size, capability->aead.key_size)) + return -1; - ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_SOCKET_ID, - &parse_integer_arg, - ¶ms->socket_id); - if (ret < 0) - goto free_kvlist; + if (param_range_check(digest_size, capability->aead.digest_size)) + return -1; - if (params->socket_id >= number_of_sockets()) { - CDEV_LOG_ERR("Invalid socket id specified to create " - "the virtual crypto device on"); - goto free_kvlist; - } - } + if (param_range_check(aad_size, capability->aead.aad_size)) + return -1; -free_kvlist: - rte_kvargs_free(kvlist); - return ret; + if (param_range_check(iv_size, capability->aead.iv_size)) + return -1; + + return 0; } const char * @@ -208,31 +362,74 @@ rte_cryptodev_get_feature_name(uint64_t flag) return "CPU_AVX"; case RTE_CRYPTODEV_FF_CPU_AVX2: return "CPU_AVX2"; + case RTE_CRYPTODEV_FF_CPU_AVX512: + return "CPU_AVX512"; case RTE_CRYPTODEV_FF_CPU_AESNI: return "CPU_AESNI"; case RTE_CRYPTODEV_FF_HW_ACCELERATED: return "HW_ACCELERATED"; - + case RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER: + return "MBUF_SCATTER_GATHER"; + case RTE_CRYPTODEV_FF_CPU_NEON: + return "CPU_NEON"; + case RTE_CRYPTODEV_FF_CPU_ARM_CE: + return "CPU_ARM_CE"; default: return NULL; } } - -int -rte_cryptodev_create_vdev(const char *name, const char *args) +struct rte_cryptodev * +rte_cryptodev_pmd_get_dev(uint8_t dev_id) { - return rte_eal_vdev_init(name, args); + return &rte_cryptodev_globals->devs[dev_id]; } +struct rte_cryptodev * +rte_cryptodev_pmd_get_named_dev(const char *name) +{ + struct rte_cryptodev *dev; + unsigned int i; + + if (name == NULL) + return NULL; + + for (i = 0; i < rte_cryptodev_globals->max_devs; i++) { + dev = &rte_cryptodev_globals->devs[i]; + + if ((dev->attached == RTE_CRYPTODEV_ATTACHED) && + (strcmp(dev->data->name, name) == 0)) + return dev; + } + + return NULL; +} + +unsigned int +rte_cryptodev_pmd_is_valid_dev(uint8_t dev_id) +{ + struct rte_cryptodev *dev = NULL; + + if (dev_id >= rte_cryptodev_globals->nb_devs) + return 0; + + dev = rte_cryptodev_pmd_get_dev(dev_id); + if (dev->attached != RTE_CRYPTODEV_ATTACHED) + return 0; + else + return 1; +} + + int -rte_cryptodev_get_dev_id(const char *name) { +rte_cryptodev_get_dev_id(const char *name) +{ unsigned i; if (name == NULL) return -1; - for (i = 0; i < rte_cryptodev_globals->max_devs; i++) + for (i = 0; i < rte_cryptodev_globals->nb_devs; i++) if ((strcmp(rte_cryptodev_globals->devs[i].data->name, name) == 0) && (rte_cryptodev_globals->devs[i].attached == @@ -249,12 +446,12 @@ rte_cryptodev_count(void) } uint8_t -rte_cryptodev_count_devtype(enum rte_cryptodev_type type) +rte_cryptodev_device_count_by_driver(uint8_t driver_id) { uint8_t i, dev_count = 0; for (i = 0; i < rte_cryptodev_globals->max_devs; i++) - if (rte_cryptodev_globals->devs[i].dev_type == type && + if (rte_cryptodev_globals->devs[i].driver_id == driver_id && rte_cryptodev_globals->devs[i].attached == RTE_CRYPTODEV_ATTACHED) dev_count++; @@ -262,6 +459,41 @@ rte_cryptodev_count_devtype(enum rte_cryptodev_type type) return dev_count; } +uint8_t +rte_cryptodev_devices_get(const char *driver_name, uint8_t *devices, + uint8_t nb_devices) +{ + uint8_t i, count = 0; + struct rte_cryptodev *devs = rte_cryptodev_globals->devs; + uint8_t max_devs = rte_cryptodev_globals->max_devs; + + for (i = 0; i < max_devs && count < nb_devices; i++) { + + if (devs[i].attached == RTE_CRYPTODEV_ATTACHED) { + int cmp; + + cmp = strncmp(devs[i].device->driver->name, + driver_name, + strlen(driver_name)); + + if (cmp == 0) + devices[count++] = devs[i].data->dev_id; + } + } + + return count; +} + +void * +rte_cryptodev_get_sec_ctx(uint8_t dev_id) +{ + if (rte_crypto_devices[dev_id].feature_flags & + RTE_CRYPTODEV_FF_SECURITY) + return rte_crypto_devices[dev_id].security_ctx; + + return NULL; +} + int rte_cryptodev_socket_id(uint8_t dev_id) { @@ -319,7 +551,7 @@ rte_cryptodev_find_free_device_index(void) } struct rte_cryptodev * -rte_cryptodev_pmd_allocate(const char *name, enum pmd_type type, int socket_id) +rte_cryptodev_pmd_allocate(const char *name, int socket_id) { struct rte_cryptodev *cryptodev; uint8_t dev_id; @@ -357,8 +589,10 @@ rte_cryptodev_pmd_allocate(const char *name, enum pmd_type type, int socket_id) cryptodev->data->socket_id = socket_id; cryptodev->data->dev_started = 0; + /* init user callbacks */ + TAILQ_INIT(&(cryptodev->link_intr_cbs)); + cryptodev->attached = RTE_CRYPTODEV_ATTACHED; - cryptodev->pmd_type = type; cryptodev_globals.nb_devs++; } @@ -366,23 +600,6 @@ rte_cryptodev_pmd_allocate(const char *name, enum pmd_type type, int socket_id) return cryptodev; } -static inline int -rte_cryptodev_create_unique_device_name(char *name, size_t size, - struct rte_pci_device *pci_dev) -{ - int ret; - - if ((name == NULL) || (pci_dev == NULL)) - return -EINVAL; - - ret = snprintf(name, size, "%d:%d.%d", - pci_dev->addr.bus, pci_dev->addr.devid, - pci_dev->addr.function); - if (ret < 0) - return ret; - return 0; -} - int rte_cryptodev_pmd_release_device(struct rte_cryptodev *cryptodev) { @@ -391,172 +608,18 @@ rte_cryptodev_pmd_release_device(struct rte_cryptodev *cryptodev) if (cryptodev == NULL) return -EINVAL; - ret = rte_cryptodev_close(cryptodev->data->dev_id); - if (ret < 0) - return ret; - - cryptodev->attached = RTE_CRYPTODEV_DETACHED; - cryptodev_globals.nb_devs--; - return 0; -} - -struct rte_cryptodev * -rte_cryptodev_pmd_virtual_dev_init(const char *name, size_t dev_private_size, - int socket_id) -{ - struct rte_cryptodev *cryptodev; - - /* allocate device structure */ - cryptodev = rte_cryptodev_pmd_allocate(name, PMD_VDEV, socket_id); - if (cryptodev == NULL) - return NULL; - - /* allocate private device structure */ - if (rte_eal_process_type() == RTE_PROC_PRIMARY) { - cryptodev->data->dev_private = - rte_zmalloc_socket("cryptodev device private", - dev_private_size, - RTE_CACHE_LINE_SIZE, - socket_id); - - if (cryptodev->data->dev_private == NULL) - rte_panic("Cannot allocate memzone for private device" - " data"); - } - - /* initialise user call-back tail queue */ - TAILQ_INIT(&(cryptodev->link_intr_cbs)); - - return cryptodev; -} - -static int -rte_cryptodev_init(struct rte_pci_driver *pci_drv, - struct rte_pci_device *pci_dev) -{ - struct rte_cryptodev_driver *cryptodrv; - struct rte_cryptodev *cryptodev; - - char cryptodev_name[RTE_CRYPTODEV_NAME_MAX_LEN]; - - int retval; - - cryptodrv = (struct rte_cryptodev_driver *)pci_drv; - if (cryptodrv == NULL) - return -ENODEV; - - /* Create unique Crypto device name using PCI address */ - rte_cryptodev_create_unique_device_name(cryptodev_name, - sizeof(cryptodev_name), pci_dev); - - cryptodev = rte_cryptodev_pmd_allocate(cryptodev_name, PMD_PDEV, - rte_socket_id()); - if (cryptodev == NULL) - return -ENOMEM; - - if (rte_eal_process_type() == RTE_PROC_PRIMARY) { - cryptodev->data->dev_private = - rte_zmalloc_socket( - "cryptodev private structure", - cryptodrv->dev_private_size, - RTE_CACHE_LINE_SIZE, - rte_socket_id()); - - if (cryptodev->data->dev_private == NULL) - rte_panic("Cannot allocate memzone for private " - "device data"); - } - - cryptodev->pci_dev = pci_dev; - cryptodev->driver = cryptodrv; - - /* init user callbacks */ - TAILQ_INIT(&(cryptodev->link_intr_cbs)); - - /* Invoke PMD device initialization function */ - retval = (*cryptodrv->cryptodev_init)(cryptodrv, cryptodev); - if (retval == 0) - return 0; - - CDEV_LOG_ERR("driver %s: crypto_dev_init(vendor_id=0x%x device_id=0x%x)" - " failed", pci_drv->name, - (unsigned) pci_dev->id.vendor_id, - (unsigned) pci_dev->id.device_id); - - if (rte_eal_process_type() == RTE_PROC_PRIMARY) - rte_free(cryptodev->data->dev_private); - - cryptodev->attached = RTE_CRYPTODEV_DETACHED; - cryptodev_globals.nb_devs--; - - return -ENXIO; -} - -static int -rte_cryptodev_uninit(struct rte_pci_device *pci_dev) -{ - const struct rte_cryptodev_driver *cryptodrv; - struct rte_cryptodev *cryptodev; - char cryptodev_name[RTE_CRYPTODEV_NAME_MAX_LEN]; - int ret; - - if (pci_dev == NULL) - return -EINVAL; - - /* Create unique device name using PCI address */ - rte_cryptodev_create_unique_device_name(cryptodev_name, - sizeof(cryptodev_name), pci_dev); - - cryptodev = rte_cryptodev_pmd_get_named_dev(cryptodev_name); - if (cryptodev == NULL) - return -ENODEV; - - cryptodrv = (const struct rte_cryptodev_driver *)pci_dev->driver; - if (cryptodrv == NULL) - return -ENODEV; - - /* Invoke PMD device uninit function */ - if (*cryptodrv->cryptodev_uninit) { - ret = (*cryptodrv->cryptodev_uninit)(cryptodrv, cryptodev); - if (ret) + /* Close device only if device operations have been set */ + if (cryptodev->dev_ops) { + ret = rte_cryptodev_close(cryptodev->data->dev_id); + if (ret < 0) return ret; } - /* free crypto device */ - rte_cryptodev_pmd_release_device(cryptodev); - - if (rte_eal_process_type() == RTE_PROC_PRIMARY) - rte_free(cryptodev->data->dev_private); - - cryptodev->pci_dev = NULL; - cryptodev->driver = NULL; - cryptodev->data = NULL; - + cryptodev->attached = RTE_CRYPTODEV_DETACHED; + cryptodev_globals.nb_devs--; return 0; } -int -rte_cryptodev_pmd_driver_register(struct rte_cryptodev_driver *cryptodrv, - enum pmd_type type) -{ - /* Call crypto device initialization directly if device is virtual */ - if (type == PMD_VDEV) - return rte_cryptodev_init((struct rte_pci_driver *)cryptodrv, - NULL); - - /* - * Register PCI driver for physical device intialisation during - * PCI probing - */ - cryptodrv->pci_drv.devinit = rte_cryptodev_init; - cryptodrv->pci_drv.devuninit = rte_cryptodev_uninit; - - rte_eal_pci_register(&cryptodrv->pci_drv); - - return 0; -} - - uint16_t rte_cryptodev_queue_pair_count(uint8_t dev_id) { @@ -688,10 +751,6 @@ rte_cryptodev_queue_pair_stop(uint8_t dev_id, uint16_t queue_pair_id) } -static int -rte_cryptodev_sym_session_pool_create(struct rte_cryptodev *dev, - unsigned nb_objs, unsigned obj_cache_size, int socket_id); - int rte_cryptodev_configure(uint8_t dev_id, struct rte_cryptodev_config *config) { @@ -711,6 +770,8 @@ rte_cryptodev_configure(uint8_t dev_id, struct rte_cryptodev_config *config) return -EBUSY; } + RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_configure, -ENOTSUP); + /* Setup new number of queue pairs and reconfigure device. */ diag = rte_cryptodev_queue_pairs_config(dev, config->nb_queue_pairs, config->socket_id); @@ -720,11 +781,7 @@ rte_cryptodev_configure(uint8_t dev_id, struct rte_cryptodev_config *config) return diag; } - /* Setup Session mempool for device */ - return rte_cryptodev_sym_session_pool_create(dev, - config->session_mp.nb_objs, - config->session_mp.cache_size, - config->socket_id); + return (*dev->dev_ops->dev_configure)(dev, config); } @@ -780,8 +837,8 @@ rte_cryptodev_stop(uint8_t dev_id) return; } - dev->data->dev_started = 0; (*dev->dev_ops->dev_stop)(dev); + dev->data->dev_started = 0; } int @@ -826,7 +883,9 @@ rte_cryptodev_close(uint8_t dev_id) int rte_cryptodev_queue_pair_setup(uint8_t dev_id, uint16_t queue_pair_id, - const struct rte_cryptodev_qp_conf *qp_conf, int socket_id) + const struct rte_cryptodev_qp_conf *qp_conf, int socket_id, + struct rte_mempool *session_pool) + { struct rte_cryptodev *dev; @@ -850,7 +909,7 @@ rte_cryptodev_queue_pair_setup(uint8_t dev_id, uint16_t queue_pair_id, RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->queue_pair_setup, -ENOTSUP); return (*dev->dev_ops->queue_pair_setup)(dev, queue_pair_id, qp_conf, - socket_id); + socket_id, session_pool); } @@ -911,9 +970,7 @@ rte_cryptodev_info_get(uint8_t dev_id, struct rte_cryptodev_info *dev_info) RTE_FUNC_PTR_OR_RET(*dev->dev_ops->dev_infos_get); (*dev->dev_ops->dev_infos_get)(dev, dev_info); - dev_info->pci_dev = dev->pci_dev; - if (dev->driver) - dev_info->driver_name = dev->driver->pci_drv.name; + dev_info->driver_name = dev->device->driver->name; } @@ -1029,149 +1086,184 @@ rte_cryptodev_pmd_callback_process(struct rte_cryptodev *dev, } -static void -rte_cryptodev_sym_session_init(struct rte_mempool *mp, - void *opaque_arg, - void *_sess, - __rte_unused unsigned i) +int +rte_cryptodev_sym_session_init(uint8_t dev_id, + struct rte_cryptodev_sym_session *sess, + struct rte_crypto_sym_xform *xforms, + struct rte_mempool *mp) { - struct rte_cryptodev_sym_session *sess = _sess; - struct rte_cryptodev *dev = opaque_arg; + struct rte_cryptodev *dev; + uint8_t index; + int ret; - memset(sess, 0, mp->elt_size); + dev = rte_cryptodev_pmd_get_dev(dev_id); - sess->dev_id = dev->data->dev_id; - sess->dev_type = dev->dev_type; - sess->mp = mp; + if (sess == NULL || xforms == NULL || dev == NULL) + return -EINVAL; - if (dev->dev_ops->session_initialize) - (*dev->dev_ops->session_initialize)(mp, sess); -} + index = dev->driver_id; -static int -rte_cryptodev_sym_session_pool_create(struct rte_cryptodev *dev, - unsigned nb_objs, unsigned obj_cache_size, int socket_id) -{ - char mp_name[RTE_CRYPTODEV_NAME_MAX_LEN]; - unsigned priv_sess_size; - - unsigned n = snprintf(mp_name, sizeof(mp_name), "cdev_%d_sess_mp", - dev->data->dev_id); - if (n > sizeof(mp_name)) { - CDEV_LOG_ERR("Unable to create unique name for session mempool"); - return -ENOMEM; - } - - RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->session_get_size, -ENOTSUP); - priv_sess_size = (*dev->dev_ops->session_get_size)(dev); - if (priv_sess_size == 0) { - CDEV_LOG_ERR("%s returned and invalid private session size ", - dev->data->name); - return -ENOMEM; - } - - unsigned elt_size = sizeof(struct rte_cryptodev_sym_session) + - priv_sess_size; - - dev->data->session_pool = rte_mempool_lookup(mp_name); - if (dev->data->session_pool != NULL) { - if ((dev->data->session_pool->elt_size != elt_size) || - (dev->data->session_pool->cache_size < - obj_cache_size) || - (dev->data->session_pool->size < nb_objs)) { - - CDEV_LOG_ERR("%s mempool already exists with different" - " initialization parameters", mp_name); - dev->data->session_pool = NULL; - return -ENOMEM; - } - } else { - dev->data->session_pool = rte_mempool_create( - mp_name, /* mempool name */ - nb_objs, /* number of elements*/ - elt_size, /* element size*/ - obj_cache_size, /* Cache size*/ - 0, /* private data size */ - NULL, /* obj initialization constructor */ - NULL, /* obj initialization constructor arg */ - rte_cryptodev_sym_session_init, - /**< obj constructor*/ - dev, /* obj constructor arg */ - socket_id, /* socket id */ - 0); /* flags */ - - if (dev->data->session_pool == NULL) { - CDEV_LOG_ERR("%s mempool allocation failed", mp_name); - return -ENOMEM; + if (sess->sess_private_data[index] == NULL) { + ret = dev->dev_ops->session_configure(dev, xforms, sess, mp); + if (ret < 0) { + CDEV_LOG_ERR( + "dev_id %d failed to configure session details", + dev_id); + return ret; } } - CDEV_LOG_DEBUG("%s mempool created!", mp_name); return 0; } struct rte_cryptodev_sym_session * -rte_cryptodev_sym_session_create(uint8_t dev_id, - struct rte_crypto_sym_xform *xform) +rte_cryptodev_sym_session_create(struct rte_mempool *mp) { - struct rte_cryptodev *dev; struct rte_cryptodev_sym_session *sess; - void *_sess; - - if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) { - CDEV_LOG_ERR("Invalid dev_id=%d", dev_id); - return NULL; - } - - dev = &rte_crypto_devices[dev_id]; /* Allocate a session structure from the session pool */ - if (rte_mempool_get(dev->data->session_pool, &_sess)) { - CDEV_LOG_ERR("Couldn't get object from session mempool"); + if (rte_mempool_get(mp, (void **)&sess)) { + CDEV_LOG_ERR("couldn't get object from session mempool"); return NULL; } - sess = (struct rte_cryptodev_sym_session *)_sess; - - RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->session_configure, NULL); - if (dev->dev_ops->session_configure(dev, xform, sess->_private) == - NULL) { - CDEV_LOG_ERR("dev_id %d failed to configure session details", - dev_id); - - /* Return session to mempool */ - rte_mempool_put(sess->mp, _sess); - return NULL; - } + /* Clear device session pointer */ + memset(sess, 0, (sizeof(void *) * nb_drivers)); return sess; } -struct rte_cryptodev_sym_session * -rte_cryptodev_sym_session_free(uint8_t dev_id, +int +rte_cryptodev_queue_pair_attach_sym_session(uint8_t dev_id, uint16_t qp_id, struct rte_cryptodev_sym_session *sess) { struct rte_cryptodev *dev; if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) { CDEV_LOG_ERR("Invalid dev_id=%d", dev_id); - return sess; + return -EINVAL; } dev = &rte_crypto_devices[dev_id]; - /* Check the session belongs to this device type */ - if (sess->dev_type != dev->dev_type) - return sess; + /* The API is optional, not returning error if driver do not suuport */ + RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->qp_attach_session, 0); - /* Let device implementation clear session material */ - RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->session_clear, sess); - dev->dev_ops->session_clear(dev, (void *)sess->_private); + void *sess_priv = get_session_private_data(sess, dev->driver_id); + + if (dev->dev_ops->qp_attach_session(dev, qp_id, sess_priv)) { + CDEV_LOG_ERR("dev_id %d failed to attach qp: %d with session", + dev_id, qp_id); + return -EPERM; + } + + return 0; +} + +int +rte_cryptodev_queue_pair_detach_sym_session(uint8_t dev_id, uint16_t qp_id, + struct rte_cryptodev_sym_session *sess) +{ + struct rte_cryptodev *dev; + + if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) { + CDEV_LOG_ERR("Invalid dev_id=%d", dev_id); + return -EINVAL; + } + + dev = &rte_crypto_devices[dev_id]; + + /* The API is optional, not returning error if driver do not suuport */ + RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->qp_detach_session, 0); + + void *sess_priv = get_session_private_data(sess, dev->driver_id); + + if (dev->dev_ops->qp_detach_session(dev, qp_id, sess_priv)) { + CDEV_LOG_ERR("dev_id %d failed to detach qp: %d from session", + dev_id, qp_id); + return -EPERM; + } + + return 0; +} + +int +rte_cryptodev_sym_session_clear(uint8_t dev_id, + struct rte_cryptodev_sym_session *sess) +{ + struct rte_cryptodev *dev; + + dev = rte_cryptodev_pmd_get_dev(dev_id); + + if (dev == NULL || sess == NULL) + return -EINVAL; + + dev->dev_ops->session_clear(dev, sess); + + return 0; +} + +int +rte_cryptodev_sym_session_free(struct rte_cryptodev_sym_session *sess) +{ + uint8_t i; + void *sess_priv; + struct rte_mempool *sess_mp; + + if (sess == NULL) + return -EINVAL; + + /* Check that all device private data has been freed */ + for (i = 0; i < nb_drivers; i++) { + sess_priv = get_session_private_data(sess, i); + if (sess_priv != NULL) + return -EBUSY; + } /* Return session to mempool */ - rte_mempool_put(sess->mp, (void *)sess); + sess_mp = rte_mempool_from_obj(sess); + rte_mempool_put(sess_mp, sess); + + return 0; +} + +unsigned int +rte_cryptodev_get_header_session_size(void) +{ + /* + * Header contains pointers to the private data + * of all registered drivers + */ + return (sizeof(void *) * nb_drivers); +} + +unsigned int +rte_cryptodev_get_private_session_size(uint8_t dev_id) +{ + struct rte_cryptodev *dev; + unsigned int header_size = sizeof(void *) * nb_drivers; + unsigned int priv_sess_size; + + if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) + return 0; + + dev = rte_cryptodev_pmd_get_dev(dev_id); + + if (*dev->dev_ops->session_get_size == NULL) + return 0; + + priv_sess_size = (*dev->dev_ops->session_get_size)(dev); + + /* + * If size is less than session header size, + * return the latter, as this guarantees that + * sessionless operations will work + */ + if (priv_sess_size < header_size) + return header_size; + + return priv_sess_size; - return NULL; } /** Initialise rte_crypto_op mempool element */ @@ -1188,7 +1280,7 @@ rte_crypto_op_init(struct rte_mempool *mempool, __rte_crypto_op_reset(op, type); - op->phys_addr = rte_mem_virt2phy(_op_data); + op->phys_addr = rte_mem_virt2iova(_op_data); op->mempool = mempool; } @@ -1249,3 +1341,85 @@ rte_crypto_op_pool_create(const char *name, enum rte_crypto_op_type type, return mp; } + +int +rte_cryptodev_pmd_create_dev_name(char *name, const char *dev_name_prefix) +{ + struct rte_cryptodev *dev = NULL; + uint32_t i = 0; + + if (name == NULL) + return -EINVAL; + + for (i = 0; i < RTE_CRYPTO_MAX_DEVS; i++) { + int ret = snprintf(name, RTE_CRYPTODEV_NAME_MAX_LEN, + "%s_%u", dev_name_prefix, i); + + if (ret < 0) + return ret; + + dev = rte_cryptodev_pmd_get_named_dev(name); + if (!dev) + return 0; + } + + return -1; +} + +TAILQ_HEAD(cryptodev_driver_list, cryptodev_driver); + +static struct cryptodev_driver_list cryptodev_driver_list = + TAILQ_HEAD_INITIALIZER(cryptodev_driver_list); + +int +rte_cryptodev_driver_id_get(const char *name) +{ + struct cryptodev_driver *driver; + const char *driver_name; + + if (name == NULL) { + RTE_LOG(DEBUG, CRYPTODEV, "name pointer NULL"); + return -1; + } + + TAILQ_FOREACH(driver, &cryptodev_driver_list, next) { + driver_name = driver->driver->name; + if (strncmp(driver_name, name, strlen(driver_name)) == 0) + return driver->id; + } + return -1; +} + +const char * +rte_cryptodev_name_get(uint8_t dev_id) +{ + struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(dev_id); + + if (dev == NULL) + return NULL; + + return dev->data->name; +} + +const char * +rte_cryptodev_driver_name_get(uint8_t driver_id) +{ + struct cryptodev_driver *driver; + + TAILQ_FOREACH(driver, &cryptodev_driver_list, next) + if (driver->id == driver_id) + return driver->driver->name; + return NULL; +} + +uint8_t +rte_cryptodev_allocate_driver(struct cryptodev_driver *crypto_drv, + const struct rte_driver *drv) +{ + crypto_drv->driver = drv; + crypto_drv->id = nb_drivers; + + TAILQ_INSERT_TAIL(&cryptodev_driver_list, crypto_drv, next); + + return nb_drivers++; +} diff --git a/dpdk/lib/librte_cryptodev/rte_cryptodev.h b/dpdk/lib/librte_cryptodev/rte_cryptodev.h index affbdecc..ed92f982 100644 --- a/dpdk/lib/librte_cryptodev/rte_cryptodev.h +++ b/dpdk/lib/librte_cryptodev/rte_cryptodev.h @@ -1,6 +1,6 @@ /*- * - * Copyright(c) 2015-2016 Intel Corporation. All rights reserved. + * Copyright(c) 2015-2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -48,98 +48,106 @@ extern "C" { #include "rte_kvargs.h" #include "rte_crypto.h" #include "rte_dev.h" - -#define CRYPTODEV_NAME_NULL_PMD cryptodev_null_pmd -/**< Null crypto PMD device name */ -#define CRYPTODEV_NAME_AESNI_MB_PMD cryptodev_aesni_mb_pmd -/**< AES-NI Multi buffer PMD device name */ -#define CRYPTODEV_NAME_AESNI_GCM_PMD cryptodev_aesni_gcm_pmd -/**< AES-NI GCM PMD device name */ -#define CRYPTODEV_NAME_QAT_SYM_PMD cryptodev_qat_sym_pmd -/**< Intel QAT Symmetric Crypto PMD device name */ -#define CRYPTODEV_NAME_SNOW3G_PMD cryptodev_snow3g_pmd -/**< SNOW 3G PMD device name */ -#define CRYPTODEV_NAME_KASUMI_PMD cryptodev_kasumi_pmd -/**< KASUMI PMD device name */ - -/** Crypto device type */ -enum rte_cryptodev_type { - RTE_CRYPTODEV_NULL_PMD = 1, /**< Null crypto PMD */ - RTE_CRYPTODEV_AESNI_GCM_PMD, /**< AES-NI GCM PMD */ - RTE_CRYPTODEV_AESNI_MB_PMD, /**< AES-NI multi buffer PMD */ - RTE_CRYPTODEV_QAT_SYM_PMD, /**< QAT PMD Symmetric Crypto */ - RTE_CRYPTODEV_SNOW3G_PMD, /**< SNOW 3G PMD */ - RTE_CRYPTODEV_KASUMI_PMD, /**< KASUMI PMD */ -}; +#include +#include extern const char **rte_cyptodev_names; /* Logging Macros */ -#define CDEV_LOG_ERR(fmt, args...) \ - RTE_LOG(ERR, CRYPTODEV, "%s() line %u: " fmt "\n", \ - __func__, __LINE__, ## args) +#define CDEV_LOG_ERR(...) \ + RTE_LOG(ERR, CRYPTODEV, \ + RTE_FMT("%s() line %u: " RTE_FMT_HEAD(__VA_ARGS__,) "\n", \ + __func__, __LINE__, RTE_FMT_TAIL(__VA_ARGS__,))) -#define CDEV_PMD_LOG_ERR(dev, fmt, args...) \ - RTE_LOG(ERR, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \ - dev, __func__, __LINE__, ## args) +#define CDEV_LOG_INFO(...) \ + RTE_LOG(INFO, CRYPTODEV, \ + RTE_FMT(RTE_FMT_HEAD(__VA_ARGS__,) "\n", \ + RTE_FMT_TAIL(__VA_ARGS__,))) #ifdef RTE_LIBRTE_CRYPTODEV_DEBUG -#define CDEV_LOG_DEBUG(fmt, args...) \ - RTE_LOG(DEBUG, CRYPTODEV, "%s() line %u: " fmt "\n", \ - __func__, __LINE__, ## args) \ +#define CDEV_LOG_DEBUG(...) \ + RTE_LOG(DEBUG, CRYPTODEV, \ + RTE_FMT("%s() line %u: " RTE_FMT_HEAD(__VA_ARGS__,) "\n", \ + __func__, __LINE__, RTE_FMT_TAIL(__VA_ARGS__,))) -#define CDEV_PMD_TRACE(fmt, args...) \ - RTE_LOG(DEBUG, CRYPTODEV, "[%s] %s: " fmt "\n", \ - dev, __func__, ## args) +#define CDEV_PMD_TRACE(...) \ + RTE_LOG(DEBUG, CRYPTODEV, \ + RTE_FMT("[%s] %s: " RTE_FMT_HEAD(__VA_ARGS__,) "\n", \ + dev, __func__, RTE_FMT_TAIL(__VA_ARGS__,))) #else -#define CDEV_LOG_DEBUG(fmt, args...) -#define CDEV_PMD_TRACE(fmt, args...) +#define CDEV_LOG_DEBUG(...) (void)0 +#define CDEV_PMD_TRACE(...) (void)0 #endif + + +/** + * A macro that points to an offset from the start + * of the crypto operation structure (rte_crypto_op) + * + * The returned pointer is cast to type t. + * + * @param c + * The crypto operation. + * @param o + * The offset from the start of the crypto operation. + * @param t + * The type to cast the result into. + */ +#define rte_crypto_op_ctod_offset(c, t, o) \ + ((t)((char *)(c) + (o))) + +/** + * A macro that returns the physical address that points + * to an offset from the start of the crypto operation + * (rte_crypto_op) + * + * @param c + * The crypto operation. + * @param o + * The offset from the start of the crypto operation + * to calculate address from. + */ +#define rte_crypto_op_ctophys_offset(c, o) \ + (rte_iova_t)((c)->phys_addr + (o)) + +/** + * Crypto parameters range description + */ +struct rte_crypto_param_range { + uint16_t min; /**< minimum size */ + uint16_t max; /**< maximum size */ + uint16_t increment; + /**< if a range of sizes are supported, + * this parameter is used to indicate + * increments in byte size that are supported + * between the minimum and maximum + */ +}; + /** * Symmetric Crypto Capability */ struct rte_cryptodev_symmetric_capability { enum rte_crypto_sym_xform_type xform_type; - /**< Transform type : Authentication / Cipher */ + /**< Transform type : Authentication / Cipher / AEAD */ + RTE_STD_C11 union { struct { enum rte_crypto_auth_algorithm algo; /**< authentication algorithm */ uint16_t block_size; /**< algorithm block size */ - struct { - uint16_t min; /**< minimum key size */ - uint16_t max; /**< maximum key size */ - uint16_t increment; - /**< if a range of sizes are supported, - * this parameter is used to indicate - * increments in byte size that are supported - * between the minimum and maximum */ - } key_size; + struct rte_crypto_param_range key_size; /**< auth key size range */ - struct { - uint16_t min; /**< minimum digest size */ - uint16_t max; /**< maximum digest size */ - uint16_t increment; - /**< if a range of sizes are supported, - * this parameter is used to indicate - * increments in byte size that are supported - * between the minimum and maximum */ - } digest_size; + struct rte_crypto_param_range digest_size; /**< digest size range */ - struct { - uint16_t min; /**< minimum aad size */ - uint16_t max; /**< maximum aad size */ - uint16_t increment; - /**< if a range of sizes are supported, - * this parameter is used to indicate - * increments in byte size that are supported - * between the minimum and maximum */ - } aad_size; + struct rte_crypto_param_range aad_size; /**< Additional authentication data size range */ + struct rte_crypto_param_range iv_size; + /**< Initialisation vector data size range */ } auth; /**< Symmetric Authentication transform capabilities */ struct { @@ -147,28 +155,26 @@ struct rte_cryptodev_symmetric_capability { /**< cipher algorithm */ uint16_t block_size; /**< algorithm block size */ - struct { - uint16_t min; /**< minimum key size */ - uint16_t max; /**< maximum key size */ - uint16_t increment; - /**< if a range of sizes are supported, - * this parameter is used to indicate - * increments in byte size that are supported - * between the minimum and maximum */ - } key_size; + struct rte_crypto_param_range key_size; /**< cipher key size range */ - struct { - uint16_t min; /**< minimum iv size */ - uint16_t max; /**< maximum iv size */ - uint16_t increment; - /**< if a range of sizes are supported, - * this parameter is used to indicate - * increments in byte size that are supported - * between the minimum and maximum */ - } iv_size; + struct rte_crypto_param_range iv_size; /**< Initialisation vector data size range */ } cipher; /**< Symmetric Cipher transform capabilities */ + struct { + enum rte_crypto_aead_algorithm algo; + /**< AEAD algorithm */ + uint16_t block_size; + /**< algorithm block size */ + struct rte_crypto_param_range key_size; + /**< AEAD key size range */ + struct rte_crypto_param_range digest_size; + /**< digest size range */ + struct rte_crypto_param_range aad_size; + /**< Additional authentication data size range */ + struct rte_crypto_param_range iv_size; + /**< Initialisation vector data size range */ + } aead; }; }; @@ -177,12 +183,137 @@ struct rte_cryptodev_capabilities { enum rte_crypto_op_type op; /**< Operation type */ + RTE_STD_C11 union { struct rte_cryptodev_symmetric_capability sym; /**< Symmetric operation capability parameters */ }; }; +/** Structure used to describe crypto algorithms */ +struct rte_cryptodev_sym_capability_idx { + enum rte_crypto_sym_xform_type type; + union { + enum rte_crypto_cipher_algorithm cipher; + enum rte_crypto_auth_algorithm auth; + enum rte_crypto_aead_algorithm aead; + } algo; +}; + +/** + * Provide capabilities available for defined device and algorithm + * + * @param dev_id The identifier of the device. + * @param idx Description of crypto algorithms. + * + * @return + * - Return description of the symmetric crypto capability if exist. + * - Return NULL if the capability not exist. + */ +const struct rte_cryptodev_symmetric_capability * +rte_cryptodev_sym_capability_get(uint8_t dev_id, + const struct rte_cryptodev_sym_capability_idx *idx); + +/** + * Check if key size and initial vector are supported + * in crypto cipher capability + * + * @param capability Description of the symmetric crypto capability. + * @param key_size Cipher key size. + * @param iv_size Cipher initial vector size. + * + * @return + * - Return 0 if the parameters are in range of the capability. + * - Return -1 if the parameters are out of range of the capability. + */ +int +rte_cryptodev_sym_capability_check_cipher( + const struct rte_cryptodev_symmetric_capability *capability, + uint16_t key_size, uint16_t iv_size); + +/** + * Check if key size and initial vector are supported + * in crypto auth capability + * + * @param capability Description of the symmetric crypto capability. + * @param key_size Auth key size. + * @param digest_size Auth digest size. + * @param iv_size Auth initial vector size. + * + * @return + * - Return 0 if the parameters are in range of the capability. + * - Return -1 if the parameters are out of range of the capability. + */ +int +rte_cryptodev_sym_capability_check_auth( + const struct rte_cryptodev_symmetric_capability *capability, + uint16_t key_size, uint16_t digest_size, uint16_t iv_size); + +/** + * Check if key, digest, AAD and initial vector sizes are supported + * in crypto AEAD capability + * + * @param capability Description of the symmetric crypto capability. + * @param key_size AEAD key size. + * @param digest_size AEAD digest size. + * @param aad_size AEAD AAD size. + * @param iv_size AEAD IV size. + * + * @return + * - Return 0 if the parameters are in range of the capability. + * - Return -1 if the parameters are out of range of the capability. + */ +int +rte_cryptodev_sym_capability_check_aead( + const struct rte_cryptodev_symmetric_capability *capability, + uint16_t key_size, uint16_t digest_size, uint16_t aad_size, + uint16_t iv_size); + +/** + * Provide the cipher algorithm enum, given an algorithm string + * + * @param algo_enum A pointer to the cipher algorithm + * enum to be filled + * @param algo_string Authentication algo string + * + * @return + * - Return -1 if string is not valid + * - Return 0 is the string is valid + */ +int +rte_cryptodev_get_cipher_algo_enum(enum rte_crypto_cipher_algorithm *algo_enum, + const char *algo_string); + +/** + * Provide the authentication algorithm enum, given an algorithm string + * + * @param algo_enum A pointer to the authentication algorithm + * enum to be filled + * @param algo_string Authentication algo string + * + * @return + * - Return -1 if string is not valid + * - Return 0 is the string is valid + */ +int +rte_cryptodev_get_auth_algo_enum(enum rte_crypto_auth_algorithm *algo_enum, + const char *algo_string); + +/** + * Provide the AEAD algorithm enum, given an algorithm string + * + * @param algo_enum A pointer to the AEAD algorithm + * enum to be filled + * @param algo_string AEAD algorithm string + * + * @return + * - Return -1 if string is not valid + * - Return 0 is the string is valid + */ +int +rte_cryptodev_get_aead_algo_enum(enum rte_crypto_aead_algorithm *algo_enum, + const char *algo_string); + /** Macro used at end of crypto PMD list */ #define RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST() \ { RTE_CRYPTO_OP_TYPE_UNDEFINED } @@ -212,6 +343,16 @@ struct rte_cryptodev_capabilities { /**< Utilises CPU AES-NI instructions */ #define RTE_CRYPTODEV_FF_HW_ACCELERATED (1ULL << 7) /**< Operations are off-loaded to an external hardware accelerator */ +#define RTE_CRYPTODEV_FF_CPU_AVX512 (1ULL << 8) +/**< Utilises CPU SIMD AVX512 instructions */ +#define RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER (1ULL << 9) +/**< Scatter-gather mbufs are supported */ +#define RTE_CRYPTODEV_FF_CPU_NEON (1ULL << 10) +/**< Utilises CPU NEON instructions */ +#define RTE_CRYPTODEV_FF_CPU_ARM_CE (1ULL << 11) +/**< Utilises ARM CPU Cryptographic Extensions */ +#define RTE_CRYPTODEV_FF_SECURITY (1ULL << 12) +/**< Support Security Protocol Processing */ /** @@ -229,7 +370,7 @@ rte_cryptodev_get_feature_name(uint64_t flag); /** Crypto device information */ struct rte_cryptodev_info { const char *driver_name; /**< Driver name. */ - enum rte_cryptodev_type dev_type; /**< Device type */ + uint8_t driver_id; /**< Driver identifier */ struct rte_pci_device *pci_dev; /**< PCI information. */ uint64_t feature_flags; /**< Feature flags */ @@ -243,6 +384,10 @@ struct rte_cryptodev_info { struct { unsigned max_nb_sessions; /**< Maximum number of sessions supported by device. */ + unsigned int max_nb_sessions_per_qp; + /**< Maximum number of sessions per queue pair. + * Default 0 for infinite sessions + */ } sym; }; @@ -287,48 +432,8 @@ struct rte_cryptodev_stats { /**< Total error count on operations dequeued */ }; -#define RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_QUEUE_PAIRS 8 -#define RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_SESSIONS 2048 - -/** - * @internal - * Initialisation parameters for virtual crypto devices - */ -struct rte_crypto_vdev_init_params { - unsigned max_nb_queue_pairs; - unsigned max_nb_sessions; - uint8_t socket_id; -}; - -/** - * Parse virtual device initialisation parameters input arguments - * @internal - * - * @params params Initialisation parameters with defaults set. - * @params input_args Command line arguments - * - * @return - * 0 on successful parse - * <0 on failure to parse - */ -int -rte_cryptodev_parse_vdev_init_params( - struct rte_crypto_vdev_init_params *params, - const char *input_args); - -/** - * Create a virtual crypto device - * - * @param name Cryptodev PMD name of device to be created. - * @param args Options arguments for device. - * - * @return - * - On successful creation of the cryptodev the device index is returned, - * which will be between 0 and rte_cryptodev_count(). - * - In the case of a failure, returns -1. - */ -extern int -rte_cryptodev_create_vdev(const char *name, const char *args); +#define RTE_CRYPTODEV_NAME_MAX_LEN (64) +/**< Max length of name of crypto PMD */ /** * Get the device identifier for the named crypto device. @@ -342,6 +447,19 @@ rte_cryptodev_create_vdev(const char *name, const char *args); extern int rte_cryptodev_get_dev_id(const char *name); +/** + * Get the crypto device name given a device identifier. + * + * @param dev_id + * The identifier of the device + * + * @return + * - Returns crypto device name. + * - Returns NULL if crypto device is not present. + */ +extern const char * +rte_cryptodev_name_get(uint8_t dev_id); + /** * Get the total number of crypto devices that have been successfully * initialised. @@ -352,8 +470,31 @@ rte_cryptodev_get_dev_id(const char *name); extern uint8_t rte_cryptodev_count(void); +/** + * Get number of crypto device defined type. + * + * @param driver_id driver identifier. + * + * @return + * Returns number of crypto device. + */ extern uint8_t -rte_cryptodev_count_devtype(enum rte_cryptodev_type type); +rte_cryptodev_device_count_by_driver(uint8_t driver_id); + +/** + * Get number and identifiers of attached crypto devices that + * use the same crypto driver. + * + * @param driver_name driver name. + * @param devices output devices identifiers. + * @param nb_devices maximal number of devices. + * + * @return + * Returns number of attached crypto device. + */ +uint8_t +rte_cryptodev_devices_get(const char *driver_name, uint8_t *devices, + uint8_t nb_devices); /* * Return the NUMA socket to which a device is connected * @@ -372,11 +513,6 @@ struct rte_cryptodev_config { int socket_id; /**< Socket to allocate resources on */ uint16_t nb_queue_pairs; /**< Number of queue pairs to configure on device */ - - struct { - uint32_t nb_objs; /**< Number of objects in mempool */ - uint32_t cache_size; /**< l-core object cache size */ - } session_mp; /**< Session mempool configuration */ }; /** @@ -453,6 +589,8 @@ rte_cryptodev_close(uint8_t dev_id); * *SOCKET_ID_ANY* if there is no NUMA constraint * for the DMA memory allocated for the receive * queue pair. + * @param session_pool Pointer to device session mempool, used + * for session-less operations. * * @return * - 0: Success, queue pair correctly set up. @@ -460,7 +598,8 @@ rte_cryptodev_close(uint8_t dev_id); */ extern int rte_cryptodev_queue_pair_setup(uint8_t dev_id, uint16_t queue_pair_id, - const struct rte_cryptodev_qp_conf *qp_conf, int socket_id); + const struct rte_cryptodev_qp_conf *qp_conf, int socket_id, + struct rte_mempool *session_pool); /** * Start a specified queue pair of a device. It is used @@ -535,6 +674,11 @@ rte_cryptodev_stats_reset(uint8_t dev_id); * @param dev_info A pointer to a structure of type * *rte_cryptodev_info* to be filled with the * contextual information of the device. + * + * @note The capabilities field of dev_info is set to point to the first + * element of an array of struct rte_cryptodev_capabilities. The element after + * the last valid element has it's op field set to + * RTE_CRYPTO_OP_TYPE_UNDEFINED. */ extern void rte_cryptodev_info_get(uint8_t dev_id, struct rte_cryptodev_info *dev_info); @@ -600,32 +744,31 @@ struct rte_cryptodev { enqueue_pkt_burst_t enqueue_burst; /**< Pointer to PMD transmit function. */ - const struct rte_cryptodev_driver *driver; - /**< Driver for this device */ struct rte_cryptodev_data *data; /**< Pointer to device data */ struct rte_cryptodev_ops *dev_ops; /**< Functions exported by PMD */ uint64_t feature_flags; /**< Supported features */ - struct rte_pci_device *pci_dev; - /**< PCI info. supplied by probing */ + struct rte_device *device; + /**< Backing device */ - enum rte_cryptodev_type dev_type; - /**< Crypto device type */ - enum pmd_type pmd_type; - /**< PMD type - PDEV / VDEV */ + uint8_t driver_id; + /**< Crypto driver identifier*/ struct rte_cryptodev_cb_list link_intr_cbs; /**< User application callback for interrupts if present */ + void *security_ctx; + /**< Context for security ops */ + + __extension__ uint8_t attached : 1; /**< Flag indicating the device is attached */ } __rte_cache_aligned; - -#define RTE_CRYPTODEV_NAME_MAX_LEN (64) -/**< Max length of name of crypto PMD */ +void * +rte_cryptodev_get_sec_ctx(uint8_t dev_id); /** * @@ -642,6 +785,7 @@ struct rte_cryptodev_data { char name[RTE_CRYPTODEV_NAME_MAX_LEN]; /**< Unique identifier name */ + __extension__ uint8_t dev_started : 1; /**< Device state: STARTED(1)/STOPPED(0) */ @@ -749,61 +893,146 @@ rte_cryptodev_enqueue_burst(uint8_t dev_id, uint16_t qp_id, /** Cryptodev symmetric crypto session */ struct rte_cryptodev_sym_session { - struct { - uint8_t dev_id; - /**< Device Id */ - enum rte_cryptodev_type dev_type; - /** Crypto Device type session created on */ - struct rte_mempool *mp; - /**< Mempool session allocated from */ - } __rte_aligned(8); - /**< Public symmetric session details */ - - char _private[0]; + __extension__ void *sess_private_data[0]; /**< Private session material */ }; /** - * Initialise a session for symmetric cryptographic operations. - * - * This function is used by the client to initialize immutable - * parameters of symmetric cryptographic operation. - * To perform the operation the rte_cryptodev_enqueue_burst function is - * used. Each mbuf should contain a reference to the session - * pointer returned from this function contained within it's crypto_op if a - * session-based operation is being provisioned. Memory to contain the session - * information is allocated from within mempool managed by the cryptodev. - * - * The rte_cryptodev_session_free must be called to free allocated - * memory when the session is no longer required. - * - * @param dev_id The device identifier. - * @param xform Crypto transform chain. - + * Create symmetric crypto session header (generic with no private data) * + * @param mempool Symmetric session mempool to allocate session + * objects from * @return - * Pointer to the created session or NULL + * - On success return pointer to sym-session + * - On failure returns NULL */ -extern struct rte_cryptodev_sym_session * -rte_cryptodev_sym_session_create(uint8_t dev_id, - struct rte_crypto_sym_xform *xform); +struct rte_cryptodev_sym_session * +rte_cryptodev_sym_session_create(struct rte_mempool *mempool); /** - * Free the memory associated with a previously allocated session. + * Frees symmetric crypto session header, after checking that all + * the device private data has been freed, returning it + * to its original mempool. + * + * @param sess Session header to be freed. + * + * @return + * - 0 if successful. + * - -EINVAL if session is NULL. + * - -EBUSY if not all device private data has been freed. + */ +int +rte_cryptodev_sym_session_free(struct rte_cryptodev_sym_session *sess); + +/** + * Fill out private data for the device id, based on its device type. + * + * @param dev_id ID of device that we want the session to be used on + * @param sess Session where the private data will be attached to + * @param xforms Symmetric crypto transform operations to apply on flow + * processed with this session + * @param mempool Mempool where the private data is allocated. + * + * @return + * - On success, zero. + * - -EINVAL if input parameters are invalid. + * - -ENOTSUP if crypto device does not support the crypto transform. + * - -ENOMEM if the private session could not be allocated. + */ +int +rte_cryptodev_sym_session_init(uint8_t dev_id, + struct rte_cryptodev_sym_session *sess, + struct rte_crypto_sym_xform *xforms, + struct rte_mempool *mempool); + +/** + * Frees private data for the device id, based on its device type, + * returning it to its mempool. + * + * @param dev_id ID of device that uses the session. + * @param sess Session containing the reference to the private data + * + * @return + * - 0 if successful. + * - -EINVAL if device is invalid or session is NULL. + */ +int +rte_cryptodev_sym_session_clear(uint8_t dev_id, + struct rte_cryptodev_sym_session *sess); + +/** + * Get the size of the header session, for all registered drivers. + * + * @return + * Size of the header session. + */ +unsigned int +rte_cryptodev_get_header_session_size(void); + +/** + * Get the size of the private session data for a device. * * @param dev_id The device identifier. + * + * @return + * - Size of the private data, if successful + * - 0 if device is invalid or does not have private session + */ +unsigned int +rte_cryptodev_get_private_session_size(uint8_t dev_id); + +/** + * Attach queue pair with sym session. + * + * @param dev_id Device to which the session will be attached. + * @param qp_id Queue pair to which the session will be attached. * @param session Session pointer previously allocated by * *rte_cryptodev_sym_session_create*. * * @return - * NULL on successful freeing of session. - * Session pointer on failure to free session. + * - On success, zero. + * - On failure, a negative value. */ -extern struct rte_cryptodev_sym_session * -rte_cryptodev_sym_session_free(uint8_t dev_id, +int +rte_cryptodev_queue_pair_attach_sym_session(uint8_t dev_id, uint16_t qp_id, struct rte_cryptodev_sym_session *session); +/** + * Detach queue pair with sym session. + * + * @param dev_id Device to which the session is attached. + * @param qp_id Queue pair to which the session is attached. + * @param session Session pointer previously allocated by + * *rte_cryptodev_sym_session_create*. + * + * @return + * - On success, zero. + * - On failure, a negative value. + */ +int +rte_cryptodev_queue_pair_detach_sym_session(uint8_t dev_id, uint16_t qp_id, + struct rte_cryptodev_sym_session *session); + +/** + * Provide driver identifier. + * + * @param name + * The pointer to a driver name. + * @return + * The driver type identifier or -1 if no driver found + */ +int rte_cryptodev_driver_id_get(const char *name); + +/** + * Provide driver name. + * + * @param driver_id + * The driver identifier. + * @return + * The driver name or null if no driver found + */ +const char *rte_cryptodev_driver_name_get(uint8_t driver_id); #ifdef __cplusplus } diff --git a/dpdk/lib/librte_cryptodev/rte_cryptodev_pmd.c b/dpdk/lib/librte_cryptodev/rte_cryptodev_pmd.c new file mode 100644 index 00000000..b4eeb448 --- /dev/null +++ b/dpdk/lib/librte_cryptodev/rte_cryptodev_pmd.c @@ -0,0 +1,201 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +#include "rte_cryptodev_pmd.h" + +/** + * Parse name from argument + */ +static int +rte_cryptodev_pmd_parse_name_arg(const char *key __rte_unused, + const char *value, void *extra_args) +{ + struct rte_cryptodev_pmd_init_params *params = extra_args; + int n; + + n = snprintf(params->name, RTE_CRYPTODEV_NAME_MAX_LEN, "%s", value); + if (n >= RTE_CRYPTODEV_NAME_MAX_LEN) + return -EINVAL; + + return 0; +} + +/** + * Parse unsigned integer from argument + */ +static int +rte_cryptodev_pmd_parse_uint_arg(const char *key __rte_unused, + const char *value, void *extra_args) +{ + int i; + char *end; + errno = 0; + + i = strtol(value, &end, 10); + if (*end != 0 || errno != 0 || i < 0) + return -EINVAL; + + *((uint32_t *)extra_args) = i; + return 0; +} + +int +rte_cryptodev_pmd_parse_input_args( + struct rte_cryptodev_pmd_init_params *params, + const char *args) +{ + struct rte_kvargs *kvlist = NULL; + int ret = 0; + + if (params == NULL) + return -EINVAL; + + if (args) { + kvlist = rte_kvargs_parse(args, cryptodev_pmd_valid_params); + if (kvlist == NULL) + return -EINVAL; + + ret = rte_kvargs_process(kvlist, + RTE_CRYPTODEV_PMD_MAX_NB_QP_ARG, + &rte_cryptodev_pmd_parse_uint_arg, + ¶ms->max_nb_queue_pairs); + if (ret < 0) + goto free_kvlist; + + ret = rte_kvargs_process(kvlist, + RTE_CRYPTODEV_PMD_MAX_NB_SESS_ARG, + &rte_cryptodev_pmd_parse_uint_arg, + ¶ms->max_nb_sessions); + if (ret < 0) + goto free_kvlist; + + ret = rte_kvargs_process(kvlist, + RTE_CRYPTODEV_PMD_SOCKET_ID_ARG, + &rte_cryptodev_pmd_parse_uint_arg, + ¶ms->socket_id); + if (ret < 0) + goto free_kvlist; + + ret = rte_kvargs_process(kvlist, + RTE_CRYPTODEV_PMD_NAME_ARG, + &rte_cryptodev_pmd_parse_name_arg, + params); + if (ret < 0) + goto free_kvlist; + } + +free_kvlist: + rte_kvargs_free(kvlist); + return ret; +} + +struct rte_cryptodev * +rte_cryptodev_pmd_create(const char *name, + struct rte_device *device, + struct rte_cryptodev_pmd_init_params *params) +{ + struct rte_cryptodev *cryptodev; + + if (params->name[0] != '\0') { + CDEV_LOG_INFO("[%s] User specified device name = %s\n", + device->driver->name, params->name); + name = params->name; + } + + CDEV_LOG_INFO("[%s] - Creating cryptodev %s\n", + device->driver->name, name); + + CDEV_LOG_INFO("[%s] - Initialisation parameters - name: %s," + "socket id: %d, max queue pairs: %u, max sessions: %u", + device->driver->name, name, + params->socket_id, params->max_nb_queue_pairs, + params->max_nb_sessions); + + /* allocate device structure */ + cryptodev = rte_cryptodev_pmd_allocate(name, params->socket_id); + if (cryptodev == NULL) { + CDEV_LOG_ERR("[%s] Failed to allocate crypto device for %s", + device->driver->name, name); + return NULL; + } + + /* allocate private device structure */ + if (rte_eal_process_type() == RTE_PROC_PRIMARY) { + cryptodev->data->dev_private = + rte_zmalloc_socket("cryptodev device private", + params->private_data_size, + RTE_CACHE_LINE_SIZE, + params->socket_id); + + if (cryptodev->data->dev_private == NULL) { + CDEV_LOG_ERR("[%s] Cannot allocate memory for " + "cryptodev %s private data", + device->driver->name, name); + + rte_cryptodev_pmd_release_device(cryptodev); + return NULL; + } + } + + cryptodev->device = device; + + /* initialise user call-back tail queue */ + TAILQ_INIT(&(cryptodev->link_intr_cbs)); + + return cryptodev; +} + +int +rte_cryptodev_pmd_destroy(struct rte_cryptodev *cryptodev) +{ + int retval; + + CDEV_LOG_INFO("[%s] Closing crypto device %s", + cryptodev->device->driver->name, + cryptodev->device->name); + + /* free crypto device */ + retval = rte_cryptodev_pmd_release_device(cryptodev); + if (retval) + return retval; + + if (rte_eal_process_type() == RTE_PROC_PRIMARY) + rte_free(cryptodev->data->dev_private); + + + cryptodev->device = NULL; + cryptodev->data = NULL; + + return 0; +} diff --git a/dpdk/lib/librte_cryptodev/rte_cryptodev_pmd.h b/dpdk/lib/librte_cryptodev/rte_cryptodev_pmd.h index 7d049ea3..089848e0 100644 --- a/dpdk/lib/librte_cryptodev/rte_cryptodev_pmd.h +++ b/dpdk/lib/librte_cryptodev/rte_cryptodev_pmd.h @@ -46,97 +46,46 @@ extern "C" { #include +#include #include -#include #include #include #include #include +#include #include "rte_crypto.h" #include "rte_cryptodev.h" -#ifdef RTE_LIBRTE_CRYPTODEV_DEBUG -#define RTE_PMD_DEBUG_TRACE(...) \ - rte_pmd_debug_trace(__func__, __VA_ARGS__) -#else -#define RTE_PMD_DEBUG_TRACE(fmt, args...) -#endif +#define RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS 8 +#define RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_SESSIONS 2048 -struct rte_cryptodev_session { - struct { - uint8_t dev_id; - enum rte_cryptodev_type type; - struct rte_mempool *mp; - } __rte_aligned(8); +#define RTE_CRYPTODEV_PMD_NAME_ARG ("name") +#define RTE_CRYPTODEV_PMD_MAX_NB_QP_ARG ("max_nb_queue_pairs") +#define RTE_CRYPTODEV_PMD_MAX_NB_SESS_ARG ("max_nb_sessions") +#define RTE_CRYPTODEV_PMD_SOCKET_ID_ARG ("socket_id") - char _private[0]; + +static const char * const cryptodev_pmd_valid_params[] = { + RTE_CRYPTODEV_PMD_NAME_ARG, + RTE_CRYPTODEV_PMD_MAX_NB_QP_ARG, + RTE_CRYPTODEV_PMD_MAX_NB_SESS_ARG, + RTE_CRYPTODEV_PMD_SOCKET_ID_ARG }; -struct rte_cryptodev_driver; - /** - * Initialisation function of a crypto driver invoked for each matching - * crypto PCI device detected during the PCI probing phase. - * - * @param drv The pointer to the [matching] crypto driver structure - * supplied by the PMD when it registered itself. - * @param dev The dev pointer is the address of the *rte_cryptodev* - * structure associated with the matching device and which - * has been [automatically] allocated in the - * *rte_crypto_devices* array. - * - * @return - * - 0: Success, the device is properly initialised by the driver. - * In particular, the driver MUST have set up the *dev_ops* pointer - * of the *dev* structure. - * - <0: Error code of the device initialisation failure. + * @internal + * Initialisation parameters for crypto devices */ -typedef int (*cryptodev_init_t)(struct rte_cryptodev_driver *drv, - struct rte_cryptodev *dev); - -/** - * Finalisation function of a driver invoked for each matching - * PCI device detected during the PCI closing phase. - * - * @param drv The pointer to the [matching] driver structure supplied - * by the PMD when it registered itself. - * @param dev The dev pointer is the address of the *rte_cryptodev* - * structure associated with the matching device and which - * has been [automatically] allocated in the - * *rte_crypto_devices* array. - * - * * @return - * - 0: Success, the device is properly finalised by the driver. - * In particular, the driver MUST free the *dev_ops* pointer - * of the *dev* structure. - * - <0: Error code of the device initialisation failure. - */ -typedef int (*cryptodev_uninit_t)(const struct rte_cryptodev_driver *drv, - struct rte_cryptodev *dev); - -/** - * The structure associated with a PMD driver. - * - * Each driver acts as a PCI driver and is represented by a generic - * *crypto_driver* structure that holds: - * - * - An *rte_pci_driver* structure (which must be the first field). - * - * - The *cryptodev_init* function invoked for each matching PCI device. - * - * - The size of the private data to allocate for each matching device. - */ -struct rte_cryptodev_driver { - struct rte_pci_driver pci_drv; /**< The PMD is also a PCI driver. */ - unsigned dev_private_size; /**< Size of device private data. */ - - cryptodev_init_t cryptodev_init; /**< Device init function. */ - cryptodev_uninit_t cryptodev_uninit; /**< Device uninit function. */ +struct rte_cryptodev_pmd_init_params { + char name[RTE_CRYPTODEV_NAME_MAX_LEN]; + size_t private_data_size; + int socket_id; + unsigned int max_nb_queue_pairs; + unsigned int max_nb_sessions; }; - /** Global structure used for maintaining state of allocated crypto devices */ struct rte_cryptodev_global { struct rte_cryptodev *devs; /**< Device information array */ @@ -146,6 +95,13 @@ struct rte_cryptodev_global { uint8_t max_devs; /**< Max number of devices */ }; +/* Cryptodev driver, containing the driver ID */ +struct cryptodev_driver { + TAILQ_ENTRY(cryptodev_driver) next; /**< Next in list. */ + const struct rte_driver *driver; + uint8_t id; +}; + /** pointer to global crypto devices data structure. */ extern struct rte_cryptodev_global *rte_cryptodev_globals; @@ -158,11 +114,8 @@ extern struct rte_cryptodev_global *rte_cryptodev_globals; * @return * - The rte_cryptodev structure pointer for the given device ID. */ -static inline struct rte_cryptodev * -rte_cryptodev_pmd_get_dev(uint8_t dev_id) -{ - return &rte_cryptodev_globals->devs[dev_id]; -} +struct rte_cryptodev * +rte_cryptodev_pmd_get_dev(uint8_t dev_id); /** * Get the rte_cryptodev structure device pointer for the named device. @@ -172,24 +125,8 @@ rte_cryptodev_pmd_get_dev(uint8_t dev_id) * @return * - The rte_cryptodev structure pointer for the given device ID. */ -static inline struct rte_cryptodev * -rte_cryptodev_pmd_get_named_dev(const char *name) -{ - struct rte_cryptodev *dev; - unsigned i; - - if (name == NULL) - return NULL; - - for (i = 0, dev = &rte_cryptodev_globals->devs[i]; - i < rte_cryptodev_globals->max_devs; i++) { - if ((dev->attached == RTE_CRYPTODEV_ATTACHED) && - (strcmp(dev->data->name, name) == 0)) - return dev; - } - - return NULL; -} +struct rte_cryptodev * +rte_cryptodev_pmd_get_named_dev(const char *name); /** * Validate if the crypto device index is valid attached crypto device. @@ -199,20 +136,8 @@ rte_cryptodev_pmd_get_named_dev(const char *name) * @return * - If the device index is valid (1) or not (0). */ -static inline unsigned -rte_cryptodev_pmd_is_valid_dev(uint8_t dev_id) -{ - struct rte_cryptodev *dev = NULL; - - if (dev_id >= rte_cryptodev_globals->nb_devs) - return 0; - - dev = rte_cryptodev_pmd_get_dev(dev_id); - if (dev->attached != RTE_CRYPTODEV_ATTACHED) - return 0; - else - return 1; -} +unsigned int +rte_cryptodev_pmd_is_valid_dev(uint8_t dev_id); /** * The pool of rte_cryptodev structures. @@ -230,10 +155,12 @@ extern struct rte_cryptodev *rte_cryptodevs; * Function used to configure device. * * @param dev Crypto device pointer + * config Crypto device configurations * * @return Returns 0 on success */ -typedef int (*cryptodev_configure_t)(struct rte_cryptodev *dev); +typedef int (*cryptodev_configure_t)(struct rte_cryptodev *dev, + struct rte_cryptodev_config *config); /** * Function used to start a configured device. @@ -317,12 +244,13 @@ typedef int (*cryptodev_queue_pair_stop_t)(struct rte_cryptodev *dev, * @param qp_id Queue Pair Index * @param qp_conf Queue configuration structure * @param socket_id Socket Index + * @param session_pool Pointer to device session mempool * * @return Returns 0 on success. */ typedef int (*cryptodev_queue_pair_setup_t)(struct rte_cryptodev *dev, uint16_t qp_id, const struct rte_cryptodev_qp_conf *qp_conf, - int socket_id); + int socket_id, struct rte_mempool *session_pool); /** * Release memory resources allocated by given queue pair. @@ -375,41 +303,59 @@ typedef int (*cryptodev_sym_create_session_pool_t)( typedef unsigned (*cryptodev_sym_get_session_private_size_t)( struct rte_cryptodev *dev); -/** - * Initialize a Crypto session on a device. - * - * @param dev Crypto device pointer - * @param xform Single or chain of crypto xforms - * @param priv_sess Pointer to cryptodev's private session structure - * - * @return - * - Returns private session structure on success. - * - Returns NULL on failure. - */ -typedef void (*cryptodev_sym_initialize_session_t)(struct rte_mempool *mempool, - void *session_private); - /** * Configure a Crypto session on a device. * * @param dev Crypto device pointer * @param xform Single or chain of crypto xforms * @param priv_sess Pointer to cryptodev's private session structure + * @param mp Mempool where the private session is allocated * * @return - * - Returns private session structure on success. - * - Returns NULL on failure. + * - Returns 0 if private session structure have been created successfully. + * - Returns -EINVAL if input parameters are invalid. + * - Returns -ENOTSUP if crypto device does not support the crypto transform. + * - Returns -ENOMEM if the private session could not be allocated. */ -typedef void * (*cryptodev_sym_configure_session_t)(struct rte_cryptodev *dev, - struct rte_crypto_sym_xform *xform, void *session_private); +typedef int (*cryptodev_sym_configure_session_t)(struct rte_cryptodev *dev, + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *session, + struct rte_mempool *mp); /** - * Free Crypto session. - * @param session Cryptodev session structure to free + * Free driver private session data. + * + * @param dev Crypto device pointer + * @param sess Cryptodev session structure */ typedef void (*cryptodev_sym_free_session_t)(struct rte_cryptodev *dev, - void *session_private); + struct rte_cryptodev_sym_session *sess); +/** + * Optional API for drivers to attach sessions with queue pair. + * @param dev Crypto device pointer + * @param qp_id queue pair id for attaching session + * @param priv_sess Pointer to cryptodev's private session structure + * @return + * - Return 0 on success + */ +typedef int (*cryptodev_sym_queue_pair_attach_session_t)( + struct rte_cryptodev *dev, + uint16_t qp_id, + void *session_private); + +/** + * Optional API for drivers to detach sessions from queue pair. + * @param dev Crypto device pointer + * @param qp_id queue pair id for detaching session + * @param priv_sess Pointer to cryptodev's private session structure + * @return + * - Return 0 on success + */ +typedef int (*cryptodev_sym_queue_pair_detach_session_t)( + struct rte_cryptodev *dev, + uint16_t qp_id, + void *session_private); /** Crypto device operations function pointer table */ struct rte_cryptodev_ops { @@ -438,12 +384,14 @@ struct rte_cryptodev_ops { cryptodev_sym_get_session_private_size_t session_get_size; /**< Return private session. */ - cryptodev_sym_initialize_session_t session_initialize; - /**< Initialization function for private session data */ cryptodev_sym_configure_session_t session_configure; /**< Configure a Crypto session. */ cryptodev_sym_free_session_t session_clear; /**< Clear a Crypto sessions private data. */ + cryptodev_sym_queue_pair_attach_session_t qp_attach_session; + /**< Attach session to queue pair. */ + cryptodev_sym_queue_pair_detach_session_t qp_detach_session; + /**< Detach session from queue pair. */ }; @@ -454,30 +402,12 @@ struct rte_cryptodev_ops { * to that slot for the driver to use. * * @param name Unique identifier name for each device - * @param type Device type of this Crypto device * @param socket_id Socket to allocate resources on. * @return * - Slot in the rte_dev_devices array for a new device; */ struct rte_cryptodev * -rte_cryptodev_pmd_allocate(const char *name, enum pmd_type type, int socket_id); - -/** - * Creates a new virtual crypto device and returns the pointer - * to that device. - * - * @param name PMD type name - * @param dev_private_size Size of crypto PMDs private data - * @param socket_id Socket to allocate resources on. - * - * @return - * - Cryptodev pointer if device is successfully created. - * - NULL if device cannot be created. - */ -struct rte_cryptodev * -rte_cryptodev_pmd_virtual_dev_init(const char *name, size_t dev_private_size, - int socket_id); - +rte_cryptodev_pmd_allocate(const char *name, int socket_id); /** * Function for internal use by dummy drivers primarily, e.g. ring-based @@ -494,33 +424,60 @@ rte_cryptodev_pmd_release_device(struct rte_cryptodev *cryptodev); /** - * Register a Crypto [Poll Mode] driver. + * @internal * - * Function invoked by the initialization function of a Crypto driver - * to simultaneously register itself as Crypto Poll Mode Driver and to either: + * PMD assist function to parse initialisation arguments for crypto driver + * when creating a new crypto PMD device instance. * - * a - register itself as PCI driver if the crypto device is a physical - * device, by invoking the rte_eal_pci_register() function to - * register the *pci_drv* structure embedded in the *crypto_drv* - * structure, after having stored the address of the - * rte_cryptodev_init() function in the *devinit* field of the - * *pci_drv* structure. + * PMD driver should set default values for that PMD before calling function, + * these default values will be over-written with successfully parsed values + * from args string. * - * During the PCI probing phase, the rte_cryptodev_init() - * function is invoked for each PCI [device] matching the - * embedded PCI identifiers provided by the driver. + * @param params parsed PMD initialisation parameters + * @param args input argument string to parse * - * b, complete the initialization sequence if the device is a virtual - * device by calling the rte_cryptodev_init() directly passing a - * NULL parameter for the rte_pci_device structure. - * - * @param crypto_drv crypto_driver structure associated with the crypto - * driver. - * @param type pmd type + * @return + * - 0 on success + * - errno on failure */ -extern int -rte_cryptodev_pmd_driver_register(struct rte_cryptodev_driver *crypto_drv, - enum pmd_type type); +int +rte_cryptodev_pmd_parse_input_args( + struct rte_cryptodev_pmd_init_params *params, + const char *args); + +/** + * @internal + * + * PMD assist function to provide boiler plate code for crypto driver to create + * and allocate resources for a new crypto PMD device instance. + * + * @param name crypto device name. + * @param device base device instance + * @param params PMD initialisation parameters + * + * @return + * - crypto device instance on success + * - NULL on creation failure + */ +struct rte_cryptodev * +rte_cryptodev_pmd_create(const char *name, + struct rte_device *device, + struct rte_cryptodev_pmd_init_params *params); + +/** + * @internal + * + * PMD assist function to provide boiler plate code for crypto driver to + * destroy and free resources associated with a crypto PMD device instance. + * + * @param cryptodev crypto device handle. + * + * @return + * - 0 on success + * - errno on failure + */ +int +rte_cryptodev_pmd_destroy(struct rte_cryptodev *cryptodev); /** * Executes all the user application registered callbacks for the specific @@ -535,6 +492,48 @@ rte_cryptodev_pmd_driver_register(struct rte_cryptodev_driver *crypto_drv, void rte_cryptodev_pmd_callback_process(struct rte_cryptodev *dev, enum rte_cryptodev_event_type event); +/** + * @internal + * Create unique device name + */ +int +rte_cryptodev_pmd_create_dev_name(char *name, const char *dev_name_prefix); + +/** + * @internal + * Allocate Cryptodev driver. + * + * @param crypto_drv + * Pointer to cryptodev_driver. + * @param drv + * Pointer to rte_driver. + * + * @return + * The driver type identifier + */ +uint8_t rte_cryptodev_allocate_driver(struct cryptodev_driver *crypto_drv, + const struct rte_driver *drv); + + +#define RTE_PMD_REGISTER_CRYPTO_DRIVER(crypto_drv, drv, driver_id)\ +RTE_INIT(init_ ##driver_id);\ +static void init_ ##driver_id(void)\ +{\ + driver_id = rte_cryptodev_allocate_driver(&crypto_drv, &(drv).driver);\ +} + +static inline void * +get_session_private_data(const struct rte_cryptodev_sym_session *sess, + uint8_t driver_id) { + return sess->sess_private_data[driver_id]; +} + +static inline void +set_session_private_data(struct rte_cryptodev_sym_session *sess, + uint8_t driver_id, void *private_data) +{ + sess->sess_private_data[driver_id] = private_data; +} #ifdef __cplusplus } diff --git a/dpdk/lib/librte_cryptodev/rte_cryptodev_version.map b/dpdk/lib/librte_cryptodev/rte_cryptodev_version.map index a08fd202..eb47308b 100644 --- a/dpdk/lib/librte_cryptodev/rte_cryptodev_version.map +++ b/dpdk/lib/librte_cryptodev/rte_cryptodev_version.map @@ -6,17 +6,13 @@ DPDK_16.04 { rte_cryptodev_callback_unregister; rte_cryptodev_close; rte_cryptodev_count; - rte_cryptodev_count_devtype; rte_cryptodev_configure; - rte_cryptodev_create_vdev; rte_cryptodev_get_dev_id; rte_cryptodev_get_feature_name; rte_cryptodev_info_get; rte_cryptodev_pmd_allocate; rte_cryptodev_pmd_callback_process; - rte_cryptodev_pmd_driver_register; rte_cryptodev_pmd_release_device; - rte_cryptodev_pmd_virtual_dev_init; rte_cryptodev_sym_session_create; rte_cryptodev_sym_session_free; rte_cryptodev_socket_id; @@ -33,9 +29,59 @@ DPDK_16.04 { local: *; }; -DPDK_16.07 { +DPDK_17.02 { global: - rte_cryptodev_parse_vdev_init_params; + rte_cryptodev_devices_get; + rte_cryptodev_pmd_create_dev_name; + rte_cryptodev_pmd_get_dev; + rte_cryptodev_pmd_get_named_dev; + rte_cryptodev_pmd_is_valid_dev; + rte_cryptodev_sym_capability_check_auth; + rte_cryptodev_sym_capability_check_cipher; + rte_cryptodev_sym_capability_get; + rte_crypto_auth_algorithm_strings; + rte_crypto_auth_operation_strings; + rte_crypto_cipher_algorithm_strings; + rte_crypto_cipher_operation_strings; } DPDK_16.04; + +DPDK_17.05 { + global: + + rte_cryptodev_get_auth_algo_enum; + rte_cryptodev_get_cipher_algo_enum; + rte_cryptodev_queue_pair_attach_sym_session; + rte_cryptodev_queue_pair_detach_sym_session; + +} DPDK_17.02; + +DPDK_17.08 { + global: + + rte_cryptodev_allocate_driver; + rte_cryptodev_device_count_by_driver; + rte_cryptodev_driver_id_get; + rte_cryptodev_driver_name_get; + rte_cryptodev_get_aead_algo_enum; + rte_cryptodev_get_header_session_size; + rte_cryptodev_get_private_session_size; + rte_cryptodev_sym_capability_check_aead; + rte_cryptodev_sym_session_init; + rte_cryptodev_sym_session_clear; + rte_crypto_aead_algorithm_strings; + rte_crypto_aead_operation_strings; + +} DPDK_17.05; + +DPDK_17.11 { + global: + + rte_cryptodev_get_sec_ctx; + rte_cryptodev_name_get; + rte_cryptodev_pmd_create; + rte_cryptodev_pmd_destroy; + rte_cryptodev_pmd_parse_input_args; + +} DPDK_17.08; diff --git a/dpdk/lib/librte_distributor/Makefile b/dpdk/lib/librte_distributor/Makefile index 4c9af172..fee00121 100644 --- a/dpdk/lib/librte_distributor/Makefile +++ b/dpdk/lib/librte_distributor/Makefile @@ -36,19 +36,23 @@ LIB = librte_distributor.a CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) +LDLIBS += -lrte_eal -lrte_mbuf -lrte_ethdev EXPORT_MAP := rte_distributor_version.map LIBABIVER := 1 # all source are stored in SRCS-y -SRCS-$(CONFIG_RTE_LIBRTE_DISTRIBUTOR) := rte_distributor.c +SRCS-$(CONFIG_RTE_LIBRTE_DISTRIBUTOR) := rte_distributor_v20.c +SRCS-$(CONFIG_RTE_LIBRTE_DISTRIBUTOR) += rte_distributor.c +ifeq ($(CONFIG_RTE_ARCH_X86),y) +SRCS-$(CONFIG_RTE_LIBRTE_DISTRIBUTOR) += rte_distributor_match_sse.c +else +SRCS-$(CONFIG_RTE_LIBRTE_DISTRIBUTOR) += rte_distributor_match_generic.c +endif + # install this header file SYMLINK-$(CONFIG_RTE_LIBRTE_DISTRIBUTOR)-include := rte_distributor.h -# this lib needs eal -DEPDIRS-$(CONFIG_RTE_LIBRTE_DISTRIBUTOR) += lib/librte_eal -DEPDIRS-$(CONFIG_RTE_LIBRTE_DISTRIBUTOR) += lib/librte_mbuf - include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/lib/librte_distributor/rte_distributor.c b/dpdk/lib/librte_distributor/rte_distributor.c index f3f778c9..6ad23013 100644 --- a/dpdk/lib/librte_distributor/rte_distributor.c +++ b/dpdk/lib/librte_distributor/rte_distributor.c @@ -1,8 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. + * Copyright(c) 2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -36,367 +35,493 @@ #include #include #include +#include +#include #include #include #include #include +#include + +#include "rte_distributor_private.h" #include "rte_distributor.h" +#include "rte_distributor_v20.h" +#include "rte_distributor_v1705.h" -#define NO_FLAGS 0 -#define RTE_DISTRIB_PREFIX "DT_" +TAILQ_HEAD(rte_dist_burst_list, rte_distributor); -/* we will use the bottom four bits of pointer for flags, shifting out - * the top four bits to make room (since a 64-bit pointer actually only uses - * 48 bits). An arithmetic-right-shift will then appropriately restore the - * original pointer value with proper sign extension into the top bits. */ -#define RTE_DISTRIB_FLAG_BITS 4 -#define RTE_DISTRIB_FLAGS_MASK (0x0F) -#define RTE_DISTRIB_NO_BUF 0 /**< empty flags: no buffer requested */ -#define RTE_DISTRIB_GET_BUF (1) /**< worker requests a buffer, returns old */ -#define RTE_DISTRIB_RETURN_BUF (2) /**< worker returns a buffer, no request */ - -#define RTE_DISTRIB_BACKLOG_SIZE 8 -#define RTE_DISTRIB_BACKLOG_MASK (RTE_DISTRIB_BACKLOG_SIZE - 1) - -#define RTE_DISTRIB_MAX_RETURNS 128 -#define RTE_DISTRIB_RETURNS_MASK (RTE_DISTRIB_MAX_RETURNS - 1) - -/** - * Maximum number of workers allowed. - * Be aware of increasing the limit, becaus it is limited by how we track - * in-flight tags. See @in_flight_bitmask and @rte_distributor_process - */ -#define RTE_DISTRIB_MAX_WORKERS 64 - -/** - * Buffer structure used to pass the pointer data between cores. This is cache - * line aligned, but to improve performance and prevent adjacent cache-line - * prefetches of buffers for other workers, e.g. when worker 1's buffer is on - * the next cache line to worker 0, we pad this out to three cache lines. - * Only 64-bits of the memory is actually used though. - */ -union rte_distributor_buffer { - volatile int64_t bufptr64; - char pad[RTE_CACHE_LINE_SIZE*3]; -} __rte_cache_aligned; - -struct rte_distributor_backlog { - unsigned start; - unsigned count; - int64_t pkts[RTE_DISTRIB_BACKLOG_SIZE]; +static struct rte_tailq_elem rte_dist_burst_tailq = { + .name = "RTE_DIST_BURST", }; - -struct rte_distributor_returned_pkts { - unsigned start; - unsigned count; - struct rte_mbuf *mbufs[RTE_DISTRIB_MAX_RETURNS]; -}; - -struct rte_distributor { - TAILQ_ENTRY(rte_distributor) next; /**< Next in list. */ - - char name[RTE_DISTRIBUTOR_NAMESIZE]; /**< Name of the ring. */ - unsigned num_workers; /**< Number of workers polling */ - - uint32_t in_flight_tags[RTE_DISTRIB_MAX_WORKERS]; - /**< Tracks the tag being processed per core */ - uint64_t in_flight_bitmask; - /**< on/off bits for in-flight tags. - * Note that if RTE_DISTRIB_MAX_WORKERS is larger than 64 then - * the bitmask has to expand. - */ - - struct rte_distributor_backlog backlog[RTE_DISTRIB_MAX_WORKERS]; - - union rte_distributor_buffer bufs[RTE_DISTRIB_MAX_WORKERS]; - - struct rte_distributor_returned_pkts returns; -}; - -TAILQ_HEAD(rte_distributor_list, rte_distributor); - -static struct rte_tailq_elem rte_distributor_tailq = { - .name = "RTE_DISTRIBUTOR", -}; -EAL_REGISTER_TAILQ(rte_distributor_tailq) +EAL_REGISTER_TAILQ(rte_dist_burst_tailq) /**** APIs called by workers ****/ +/**** Burst Packet APIs called by workers ****/ + void -rte_distributor_request_pkt(struct rte_distributor *d, - unsigned worker_id, struct rte_mbuf *oldpkt) +rte_distributor_request_pkt_v1705(struct rte_distributor *d, + unsigned int worker_id, struct rte_mbuf **oldpkt, + unsigned int count) { - union rte_distributor_buffer *buf = &d->bufs[worker_id]; - int64_t req = (((int64_t)(uintptr_t)oldpkt) << RTE_DISTRIB_FLAG_BITS) - | RTE_DISTRIB_GET_BUF; - while (unlikely(buf->bufptr64 & RTE_DISTRIB_FLAGS_MASK)) + struct rte_distributor_buffer *buf = &(d->bufs[worker_id]); + unsigned int i; + + volatile int64_t *retptr64; + + if (unlikely(d->alg_type == RTE_DIST_ALG_SINGLE)) { + rte_distributor_request_pkt_v20(d->d_v20, + worker_id, oldpkt[0]); + return; + } + + retptr64 = &(buf->retptr64[0]); + /* Spin while handshake bits are set (scheduler clears it) */ + while (unlikely(*retptr64 & RTE_DISTRIB_GET_BUF)) { rte_pause(); - buf->bufptr64 = req; -} + uint64_t t = rte_rdtsc()+100; -struct rte_mbuf * -rte_distributor_poll_pkt(struct rte_distributor *d, - unsigned worker_id) -{ - union rte_distributor_buffer *buf = &d->bufs[worker_id]; - if (buf->bufptr64 & RTE_DISTRIB_GET_BUF) - return NULL; + while (rte_rdtsc() < t) + rte_pause(); + } - /* since bufptr64 is signed, this should be an arithmetic shift */ - int64_t ret = buf->bufptr64 >> RTE_DISTRIB_FLAG_BITS; - return (struct rte_mbuf *)((uintptr_t)ret); -} + /* + * OK, if we've got here, then the scheduler has just cleared the + * handshake bits. Populate the retptrs with returning packets. + */ -struct rte_mbuf * -rte_distributor_get_pkt(struct rte_distributor *d, - unsigned worker_id, struct rte_mbuf *oldpkt) -{ - struct rte_mbuf *ret; - rte_distributor_request_pkt(d, worker_id, oldpkt); - while ((ret = rte_distributor_poll_pkt(d, worker_id)) == NULL) - rte_pause(); - return ret; + for (i = count; i < RTE_DIST_BURST_SIZE; i++) + buf->retptr64[i] = 0; + + /* Set Return bit for each packet returned */ + for (i = count; i-- > 0; ) + buf->retptr64[i] = + (((int64_t)(uintptr_t)(oldpkt[i])) << + RTE_DISTRIB_FLAG_BITS) | RTE_DISTRIB_RETURN_BUF; + + /* + * Finally, set the GET_BUF to signal to distributor that cache + * line is ready for processing + */ + *retptr64 |= RTE_DISTRIB_GET_BUF; } +BIND_DEFAULT_SYMBOL(rte_distributor_request_pkt, _v1705, 17.05); +MAP_STATIC_SYMBOL(void rte_distributor_request_pkt(struct rte_distributor *d, + unsigned int worker_id, struct rte_mbuf **oldpkt, + unsigned int count), + rte_distributor_request_pkt_v1705); int -rte_distributor_return_pkt(struct rte_distributor *d, - unsigned worker_id, struct rte_mbuf *oldpkt) +rte_distributor_poll_pkt_v1705(struct rte_distributor *d, + unsigned int worker_id, struct rte_mbuf **pkts) { - union rte_distributor_buffer *buf = &d->bufs[worker_id]; - uint64_t req = (((int64_t)(uintptr_t)oldpkt) << RTE_DISTRIB_FLAG_BITS) - | RTE_DISTRIB_RETURN_BUF; - buf->bufptr64 = req; - return 0; -} + struct rte_distributor_buffer *buf = &d->bufs[worker_id]; + uint64_t ret; + int count = 0; + unsigned int i; -/**** APIs called on distributor core ***/ + if (unlikely(d->alg_type == RTE_DIST_ALG_SINGLE)) { + pkts[0] = rte_distributor_poll_pkt_v20(d->d_v20, worker_id); + return (pkts[0]) ? 1 : 0; + } -/* as name suggests, adds a packet to the backlog for a particular worker */ -static int -add_to_backlog(struct rte_distributor_backlog *bl, int64_t item) -{ - if (bl->count == RTE_DISTRIB_BACKLOG_SIZE) + /* If bit is set, return */ + if (buf->bufptr64[0] & RTE_DISTRIB_GET_BUF) return -1; - bl->pkts[(bl->start + bl->count++) & (RTE_DISTRIB_BACKLOG_MASK)] - = item; + /* since bufptr64 is signed, this should be an arithmetic shift */ + for (i = 0; i < RTE_DIST_BURST_SIZE; i++) { + if (likely(buf->bufptr64[i] & RTE_DISTRIB_VALID_BUF)) { + ret = buf->bufptr64[i] >> RTE_DISTRIB_FLAG_BITS; + pkts[count++] = (struct rte_mbuf *)((uintptr_t)(ret)); + } + } + + /* + * so now we've got the contents of the cacheline into an array of + * mbuf pointers, so toggle the bit so scheduler can start working + * on the next cacheline while we're working. + */ + buf->bufptr64[0] |= RTE_DISTRIB_GET_BUF; + + return count; +} +BIND_DEFAULT_SYMBOL(rte_distributor_poll_pkt, _v1705, 17.05); +MAP_STATIC_SYMBOL(int rte_distributor_poll_pkt(struct rte_distributor *d, + unsigned int worker_id, struct rte_mbuf **pkts), + rte_distributor_poll_pkt_v1705); + +int +rte_distributor_get_pkt_v1705(struct rte_distributor *d, + unsigned int worker_id, struct rte_mbuf **pkts, + struct rte_mbuf **oldpkt, unsigned int return_count) +{ + int count; + + if (unlikely(d->alg_type == RTE_DIST_ALG_SINGLE)) { + if (return_count <= 1) { + pkts[0] = rte_distributor_get_pkt_v20(d->d_v20, + worker_id, oldpkt[0]); + return (pkts[0]) ? 1 : 0; + } else + return -EINVAL; + } + + rte_distributor_request_pkt(d, worker_id, oldpkt, return_count); + + count = rte_distributor_poll_pkt(d, worker_id, pkts); + while (count == -1) { + uint64_t t = rte_rdtsc() + 100; + + while (rte_rdtsc() < t) + rte_pause(); + + count = rte_distributor_poll_pkt(d, worker_id, pkts); + } + return count; +} +BIND_DEFAULT_SYMBOL(rte_distributor_get_pkt, _v1705, 17.05); +MAP_STATIC_SYMBOL(int rte_distributor_get_pkt(struct rte_distributor *d, + unsigned int worker_id, struct rte_mbuf **pkts, + struct rte_mbuf **oldpkt, unsigned int return_count), + rte_distributor_get_pkt_v1705); + +int +rte_distributor_return_pkt_v1705(struct rte_distributor *d, + unsigned int worker_id, struct rte_mbuf **oldpkt, int num) +{ + struct rte_distributor_buffer *buf = &d->bufs[worker_id]; + unsigned int i; + + if (unlikely(d->alg_type == RTE_DIST_ALG_SINGLE)) { + if (num == 1) + return rte_distributor_return_pkt_v20(d->d_v20, + worker_id, oldpkt[0]); + else + return -EINVAL; + } + + for (i = 0; i < RTE_DIST_BURST_SIZE; i++) + /* Switch off the return bit first */ + buf->retptr64[i] &= ~RTE_DISTRIB_RETURN_BUF; + + for (i = num; i-- > 0; ) + buf->retptr64[i] = (((int64_t)(uintptr_t)oldpkt[i]) << + RTE_DISTRIB_FLAG_BITS) | RTE_DISTRIB_RETURN_BUF; + + /* set the GET_BUF but even if we got no returns */ + buf->retptr64[0] |= RTE_DISTRIB_GET_BUF; + return 0; } +BIND_DEFAULT_SYMBOL(rte_distributor_return_pkt, _v1705, 17.05); +MAP_STATIC_SYMBOL(int rte_distributor_return_pkt(struct rte_distributor *d, + unsigned int worker_id, struct rte_mbuf **oldpkt, int num), + rte_distributor_return_pkt_v1705); -/* takes the next packet for a worker off the backlog */ -static int64_t -backlog_pop(struct rte_distributor_backlog *bl) -{ - bl->count--; - return bl->pkts[bl->start++ & RTE_DISTRIB_BACKLOG_MASK]; -} +/**** APIs called on distributor core ***/ /* stores a packet returned from a worker inside the returns array */ static inline void store_return(uintptr_t oldbuf, struct rte_distributor *d, - unsigned *ret_start, unsigned *ret_count) + unsigned int *ret_start, unsigned int *ret_count) { - /* store returns in a circular buffer - code is branch-free */ + if (!oldbuf) + return; + /* store returns in a circular buffer */ d->returns.mbufs[(*ret_start + *ret_count) & RTE_DISTRIB_RETURNS_MASK] = (void *)oldbuf; - *ret_start += (*ret_count == RTE_DISTRIB_RETURNS_MASK) & !!(oldbuf); - *ret_count += (*ret_count != RTE_DISTRIB_RETURNS_MASK) & !!(oldbuf); + *ret_start += (*ret_count == RTE_DISTRIB_RETURNS_MASK); + *ret_count += (*ret_count != RTE_DISTRIB_RETURNS_MASK); } -static inline void -handle_worker_shutdown(struct rte_distributor *d, unsigned wkr) -{ - d->in_flight_tags[wkr] = 0; - d->in_flight_bitmask &= ~(1UL << wkr); - d->bufs[wkr].bufptr64 = 0; - if (unlikely(d->backlog[wkr].count != 0)) { - /* On return of a packet, we need to move the - * queued packets for this core elsewhere. - * Easiest solution is to set things up for - * a recursive call. That will cause those - * packets to be queued up for the next free - * core, i.e. it will return as soon as a - * core becomes free to accept the first - * packet, as subsequent ones will be added to - * the backlog for that core. - */ - struct rte_mbuf *pkts[RTE_DISTRIB_BACKLOG_SIZE]; - unsigned i; - struct rte_distributor_backlog *bl = &d->backlog[wkr]; - - for (i = 0; i < bl->count; i++) { - unsigned idx = (bl->start + i) & - RTE_DISTRIB_BACKLOG_MASK; - pkts[i] = (void *)((uintptr_t)(bl->pkts[idx] >> - RTE_DISTRIB_FLAG_BITS)); - } - /* recursive call. - * Note that the tags were set before first level call - * to rte_distributor_process. - */ - rte_distributor_process(d, pkts, i); - bl->count = bl->start = 0; - } -} - -/* this function is called when process() fn is called without any new - * packets. It goes through all the workers and clears any returned packets - * to do a partial flush. +/* + * Match then flow_ids (tags) of the incoming packets to the flow_ids + * of the inflight packets (both inflight on the workers and in each worker + * backlog). This will then allow us to pin those packets to the relevant + * workers to give us our atomic flow pinning. */ -static int -process_returns(struct rte_distributor *d) +void +find_match_scalar(struct rte_distributor *d, + uint16_t *data_ptr, + uint16_t *output_ptr) { - unsigned wkr; - unsigned flushed = 0; - unsigned ret_start = d->returns.start, - ret_count = d->returns.count; + struct rte_distributor_backlog *bl; + uint16_t i, j, w; - for (wkr = 0; wkr < d->num_workers; wkr++) { + /* + * Function overview: + * 1. Loop through all worker ID's + * 2. Compare the current inflights to the incoming tags + * 3. Compare the current backlog to the incoming tags + * 4. Add any matches to the output + */ - const int64_t data = d->bufs[wkr].bufptr64; - uintptr_t oldbuf = 0; + for (j = 0 ; j < RTE_DIST_BURST_SIZE; j++) + output_ptr[j] = 0; - if (data & RTE_DISTRIB_GET_BUF) { - flushed++; - if (d->backlog[wkr].count) - d->bufs[wkr].bufptr64 = - backlog_pop(&d->backlog[wkr]); - else { - d->bufs[wkr].bufptr64 = RTE_DISTRIB_GET_BUF; - d->in_flight_tags[wkr] = 0; - d->in_flight_bitmask &= ~(1UL << wkr); - } - oldbuf = data >> RTE_DISTRIB_FLAG_BITS; - } else if (data & RTE_DISTRIB_RETURN_BUF) { - handle_worker_shutdown(d, wkr); - oldbuf = data >> RTE_DISTRIB_FLAG_BITS; - } + for (i = 0; i < d->num_workers; i++) { + bl = &d->backlog[i]; - store_return(oldbuf, d, &ret_start, &ret_count); + for (j = 0; j < RTE_DIST_BURST_SIZE ; j++) + for (w = 0; w < RTE_DIST_BURST_SIZE; w++) + if (d->in_flight_tags[i][j] == data_ptr[w]) { + output_ptr[j] = i+1; + break; + } + for (j = 0; j < RTE_DIST_BURST_SIZE; j++) + for (w = 0; w < RTE_DIST_BURST_SIZE; w++) + if (bl->tags[j] == data_ptr[w]) { + output_ptr[j] = i+1; + break; + } } - d->returns.start = ret_start; - d->returns.count = ret_count; - - return flushed; + /* + * At this stage, the output contains 8 16-bit values, with + * each non-zero value containing the worker ID on which the + * corresponding flow is pinned to. + */ } + +/* + * When the handshake bits indicate that there are packets coming + * back from the worker, this function is called to copy and store + * the valid returned pointers (store_return). + */ +static unsigned int +handle_returns(struct rte_distributor *d, unsigned int wkr) +{ + struct rte_distributor_buffer *buf = &(d->bufs[wkr]); + uintptr_t oldbuf; + unsigned int ret_start = d->returns.start, + ret_count = d->returns.count; + unsigned int count = 0; + unsigned int i; + + if (buf->retptr64[0] & RTE_DISTRIB_GET_BUF) { + for (i = 0; i < RTE_DIST_BURST_SIZE; i++) { + if (buf->retptr64[i] & RTE_DISTRIB_RETURN_BUF) { + oldbuf = ((uintptr_t)(buf->retptr64[i] >> + RTE_DISTRIB_FLAG_BITS)); + /* store returns in a circular buffer */ + store_return(oldbuf, d, &ret_start, &ret_count); + count++; + buf->retptr64[i] &= ~RTE_DISTRIB_RETURN_BUF; + } + } + d->returns.start = ret_start; + d->returns.count = ret_count; + /* Clear for the worker to populate with more returns */ + buf->retptr64[0] = 0; + } + return count; +} + +/* + * This function releases a burst (cache line) to a worker. + * It is called from the process function when a cacheline is + * full to make room for more packets for that worker, or when + * all packets have been assigned to bursts and need to be flushed + * to the workers. + * It also needs to wait for any outstanding packets from the worker + * before sending out new packets. + */ +static unsigned int +release(struct rte_distributor *d, unsigned int wkr) +{ + struct rte_distributor_buffer *buf = &(d->bufs[wkr]); + unsigned int i; + + while (!(d->bufs[wkr].bufptr64[0] & RTE_DISTRIB_GET_BUF)) + rte_pause(); + + handle_returns(d, wkr); + + buf->count = 0; + + for (i = 0; i < d->backlog[wkr].count; i++) { + d->bufs[wkr].bufptr64[i] = d->backlog[wkr].pkts[i] | + RTE_DISTRIB_GET_BUF | RTE_DISTRIB_VALID_BUF; + d->in_flight_tags[wkr][i] = d->backlog[wkr].tags[i]; + } + buf->count = i; + for ( ; i < RTE_DIST_BURST_SIZE ; i++) { + buf->bufptr64[i] = RTE_DISTRIB_GET_BUF; + d->in_flight_tags[wkr][i] = 0; + } + + d->backlog[wkr].count = 0; + + /* Clear the GET bit */ + buf->bufptr64[0] &= ~RTE_DISTRIB_GET_BUF; + return buf->count; + +} + + /* process a set of packets to distribute them to workers */ int -rte_distributor_process(struct rte_distributor *d, - struct rte_mbuf **mbufs, unsigned num_mbufs) +rte_distributor_process_v1705(struct rte_distributor *d, + struct rte_mbuf **mbufs, unsigned int num_mbufs) { - unsigned next_idx = 0; - unsigned wkr = 0; + unsigned int next_idx = 0; + static unsigned int wkr; struct rte_mbuf *next_mb = NULL; int64_t next_value = 0; - uint32_t new_tag = 0; - unsigned ret_start = d->returns.start, - ret_count = d->returns.count; + uint16_t new_tag = 0; + uint16_t flows[RTE_DIST_BURST_SIZE] __rte_cache_aligned; + unsigned int i, j, w, wid; - if (unlikely(num_mbufs == 0)) - return process_returns(d); + if (d->alg_type == RTE_DIST_ALG_SINGLE) { + /* Call the old API */ + return rte_distributor_process_v20(d->d_v20, mbufs, num_mbufs); + } - while (next_idx < num_mbufs || next_mb != NULL) { + if (unlikely(num_mbufs == 0)) { + /* Flush out all non-full cache-lines to workers. */ + for (wid = 0 ; wid < d->num_workers; wid++) { + if ((d->bufs[wid].bufptr64[0] & RTE_DISTRIB_GET_BUF)) { + release(d, wid); + handle_returns(d, wid); + } + } + return 0; + } - int64_t data = d->bufs[wkr].bufptr64; - uintptr_t oldbuf = 0; + while (next_idx < num_mbufs) { + uint16_t matches[RTE_DIST_BURST_SIZE]; + unsigned int pkts; + + if (d->bufs[wkr].bufptr64[0] & RTE_DISTRIB_GET_BUF) + d->bufs[wkr].count = 0; + + if ((num_mbufs - next_idx) < RTE_DIST_BURST_SIZE) + pkts = num_mbufs - next_idx; + else + pkts = RTE_DIST_BURST_SIZE; + + for (i = 0; i < pkts; i++) { + if (mbufs[next_idx + i]) { + /* flows have to be non-zero */ + flows[i] = mbufs[next_idx + i]->hash.usr | 1; + } else + flows[i] = 0; + } + for (; i < RTE_DIST_BURST_SIZE; i++) + flows[i] = 0; + + switch (d->dist_match_fn) { + case RTE_DIST_MATCH_VECTOR: + find_match_vec(d, &flows[0], &matches[0]); + break; + default: + find_match_scalar(d, &flows[0], &matches[0]); + } + + /* + * Matches array now contain the intended worker ID (+1) of + * the incoming packets. Any zeroes need to be assigned + * workers. + */ + + for (j = 0; j < pkts; j++) { - if (!next_mb) { next_mb = mbufs[next_idx++]; - next_value = (((int64_t)(uintptr_t)next_mb) - << RTE_DISTRIB_FLAG_BITS); + next_value = (((int64_t)(uintptr_t)next_mb) << + RTE_DISTRIB_FLAG_BITS); /* - * User is advocated to set tag vaue for each + * User is advocated to set tag value for each * mbuf before calling rte_distributor_process. * User defined tags are used to identify flows, * or sessions. */ - new_tag = next_mb->hash.usr; + /* flows MUST be non-zero */ + new_tag = (uint16_t)(next_mb->hash.usr) | 1; /* - * Note that if RTE_DISTRIB_MAX_WORKERS is larger than 64 - * then the size of match has to be expanded. + * Uncommenting the next line will cause the find_match + * function to be optimized out, making this function + * do parallel (non-atomic) distribution */ - uint64_t match = 0; - unsigned i; - /* - * to scan for a match use "xor" and "not" to get a 0/1 - * value, then use shifting to merge to single "match" - * variable, where a one-bit indicates a match for the - * worker given by the bit-position - */ - for (i = 0; i < d->num_workers; i++) - match |= (!(d->in_flight_tags[i] ^ new_tag) - << i); + /* matches[j] = 0; */ - /* Only turned-on bits are considered as match */ - match &= d->in_flight_bitmask; + if (matches[j]) { + struct rte_distributor_backlog *bl = + &d->backlog[matches[j]-1]; + if (unlikely(bl->count == + RTE_DIST_BURST_SIZE)) { + release(d, matches[j]-1); + } - if (match) { - next_mb = NULL; - unsigned worker = __builtin_ctzl(match); - if (add_to_backlog(&d->backlog[worker], - next_value) < 0) - next_idx--; + /* Add to worker that already has flow */ + unsigned int idx = bl->count++; + + bl->tags[idx] = new_tag; + bl->pkts[idx] = next_value; + + } else { + struct rte_distributor_backlog *bl = + &d->backlog[wkr]; + if (unlikely(bl->count == + RTE_DIST_BURST_SIZE)) { + release(d, wkr); + } + + /* Add to current worker worker */ + unsigned int idx = bl->count++; + + bl->tags[idx] = new_tag; + bl->pkts[idx] = next_value; + /* + * Now that we've just added an unpinned flow + * to a worker, we need to ensure that all + * other packets with that same flow will go + * to the same worker in this burst. + */ + for (w = j; w < pkts; w++) + if (flows[w] == new_tag) + matches[w] = wkr+1; } } - - if ((data & RTE_DISTRIB_GET_BUF) && - (d->backlog[wkr].count || next_mb)) { - - if (d->backlog[wkr].count) - d->bufs[wkr].bufptr64 = - backlog_pop(&d->backlog[wkr]); - - else { - d->bufs[wkr].bufptr64 = next_value; - d->in_flight_tags[wkr] = new_tag; - d->in_flight_bitmask |= (1UL << wkr); - next_mb = NULL; - } - oldbuf = data >> RTE_DISTRIB_FLAG_BITS; - } else if (data & RTE_DISTRIB_RETURN_BUF) { - handle_worker_shutdown(d, wkr); - oldbuf = data >> RTE_DISTRIB_FLAG_BITS; - } - - /* store returns in a circular buffer */ - store_return(oldbuf, d, &ret_start, &ret_count); - - if (++wkr == d->num_workers) + wkr++; + if (wkr >= d->num_workers) wkr = 0; } - /* to finish, check all workers for backlog and schedule work for them - * if they are ready */ - for (wkr = 0; wkr < d->num_workers; wkr++) - if (d->backlog[wkr].count && - (d->bufs[wkr].bufptr64 & RTE_DISTRIB_GET_BUF)) { - int64_t oldbuf = d->bufs[wkr].bufptr64 >> - RTE_DISTRIB_FLAG_BITS; - store_return(oldbuf, d, &ret_start, &ret_count); + /* Flush out all non-full cache-lines to workers. */ + for (wid = 0 ; wid < d->num_workers; wid++) + if ((d->bufs[wid].bufptr64[0] & RTE_DISTRIB_GET_BUF)) + release(d, wid); - d->bufs[wkr].bufptr64 = backlog_pop(&d->backlog[wkr]); - } - - d->returns.start = ret_start; - d->returns.count = ret_count; return num_mbufs; } +BIND_DEFAULT_SYMBOL(rte_distributor_process, _v1705, 17.05); +MAP_STATIC_SYMBOL(int rte_distributor_process(struct rte_distributor *d, + struct rte_mbuf **mbufs, unsigned int num_mbufs), + rte_distributor_process_v1705); /* return to the caller, packets returned from workers */ int -rte_distributor_returned_pkts(struct rte_distributor *d, - struct rte_mbuf **mbufs, unsigned max_mbufs) +rte_distributor_returned_pkts_v1705(struct rte_distributor *d, + struct rte_mbuf **mbufs, unsigned int max_mbufs) { struct rte_distributor_returned_pkts *returns = &d->returns; - unsigned retval = (max_mbufs < returns->count) ? + unsigned int retval = (max_mbufs < returns->count) ? max_mbufs : returns->count; - unsigned i; + unsigned int i; + + if (d->alg_type == RTE_DIST_ALG_SINGLE) { + /* Call the old API */ + return rte_distributor_returned_pkts_v20(d->d_v20, + mbufs, max_mbufs); + } for (i = 0; i < retval; i++) { - unsigned idx = (returns->start + i) & RTE_DISTRIB_RETURNS_MASK; + unsigned int idx = (returns->start + i) & + RTE_DISTRIB_RETURNS_MASK; + mbufs[i] = returns->mbufs[idx]; } returns->start += i; @@ -404,15 +529,19 @@ rte_distributor_returned_pkts(struct rte_distributor *d, return retval; } +BIND_DEFAULT_SYMBOL(rte_distributor_returned_pkts, _v1705, 17.05); +MAP_STATIC_SYMBOL(int rte_distributor_returned_pkts(struct rte_distributor *d, + struct rte_mbuf **mbufs, unsigned int max_mbufs), + rte_distributor_returned_pkts_v1705); -/* return the number of packets in-flight in a distributor, i.e. packets - * being workered on or queued up in a backlog. */ -static inline unsigned +/* + * Return the number of packets in-flight in a distributor, i.e. packets + * being worked on or queued up in a backlog. + */ +static inline unsigned int total_outstanding(const struct rte_distributor *d) { - unsigned wkr, total_outstanding; - - total_outstanding = __builtin_popcountl(d->in_flight_bitmask); + unsigned int wkr, total_outstanding = 0; for (wkr = 0; wkr < d->num_workers; wkr++) total_outstanding += d->backlog[wkr].count; @@ -420,45 +549,96 @@ total_outstanding(const struct rte_distributor *d) return total_outstanding; } -/* flush the distributor, so that there are no outstanding packets in flight or - * queued up. */ +/* + * Flush the distributor, so that there are no outstanding packets in flight or + * queued up. + */ int -rte_distributor_flush(struct rte_distributor *d) +rte_distributor_flush_v1705(struct rte_distributor *d) { - const unsigned flushed = total_outstanding(d); + unsigned int flushed; + unsigned int wkr; + + if (d->alg_type == RTE_DIST_ALG_SINGLE) { + /* Call the old API */ + return rte_distributor_flush_v20(d->d_v20); + } + + flushed = total_outstanding(d); while (total_outstanding(d) > 0) rte_distributor_process(d, NULL, 0); + /* + * Send empty burst to all workers to allow them to exit + * gracefully, should they need to. + */ + rte_distributor_process(d, NULL, 0); + + for (wkr = 0; wkr < d->num_workers; wkr++) + handle_returns(d, wkr); + return flushed; } +BIND_DEFAULT_SYMBOL(rte_distributor_flush, _v1705, 17.05); +MAP_STATIC_SYMBOL(int rte_distributor_flush(struct rte_distributor *d), + rte_distributor_flush_v1705); /* clears the internal returns array in the distributor */ void -rte_distributor_clear_returns(struct rte_distributor *d) +rte_distributor_clear_returns_v1705(struct rte_distributor *d) { - d->returns.start = d->returns.count = 0; -#ifndef __OPTIMIZE__ - memset(d->returns.mbufs, 0, sizeof(d->returns.mbufs)); -#endif + unsigned int wkr; + + if (d->alg_type == RTE_DIST_ALG_SINGLE) { + /* Call the old API */ + rte_distributor_clear_returns_v20(d->d_v20); + return; + } + + /* throw away returns, so workers can exit */ + for (wkr = 0; wkr < d->num_workers; wkr++) + d->bufs[wkr].retptr64[0] = 0; } +BIND_DEFAULT_SYMBOL(rte_distributor_clear_returns, _v1705, 17.05); +MAP_STATIC_SYMBOL(void rte_distributor_clear_returns(struct rte_distributor *d), + rte_distributor_clear_returns_v1705); /* creates a distributor instance */ struct rte_distributor * -rte_distributor_create(const char *name, - unsigned socket_id, - unsigned num_workers) +rte_distributor_create_v1705(const char *name, + unsigned int socket_id, + unsigned int num_workers, + unsigned int alg_type) { struct rte_distributor *d; - struct rte_distributor_list *distributor_list; + struct rte_dist_burst_list *dist_burst_list; char mz_name[RTE_MEMZONE_NAMESIZE]; const struct rte_memzone *mz; + unsigned int i; + + /* TODO Reorganise function properly around RTE_DIST_ALG_SINGLE/BURST */ /* compilation-time checks */ RTE_BUILD_BUG_ON((sizeof(*d) & RTE_CACHE_LINE_MASK) != 0); RTE_BUILD_BUG_ON((RTE_DISTRIB_MAX_WORKERS & 7) != 0); - RTE_BUILD_BUG_ON(RTE_DISTRIB_MAX_WORKERS > - sizeof(d->in_flight_bitmask) * CHAR_BIT); + + if (alg_type == RTE_DIST_ALG_SINGLE) { + d = malloc(sizeof(struct rte_distributor)); + if (d == NULL) { + rte_errno = ENOMEM; + return NULL; + } + d->d_v20 = rte_distributor_create_v20(name, + socket_id, num_workers); + if (d->d_v20 == NULL) { + free(d); + /* rte_errno will have been set */ + return NULL; + } + d->alg_type = alg_type; + return d; + } if (name == NULL || num_workers >= RTE_DISTRIB_MAX_WORKERS) { rte_errno = EINVAL; @@ -475,13 +655,32 @@ rte_distributor_create(const char *name, d = mz->addr; snprintf(d->name, sizeof(d->name), "%s", name); d->num_workers = num_workers; + d->alg_type = alg_type; + + d->dist_match_fn = RTE_DIST_MATCH_SCALAR; +#if defined(RTE_ARCH_X86) + d->dist_match_fn = RTE_DIST_MATCH_VECTOR; +#endif + + /* + * Set up the backlog tags so they're pointing at the second cache + * line for performance during flow matching + */ + for (i = 0 ; i < num_workers ; i++) + d->backlog[i].tags = &d->in_flight_tags[i][RTE_DIST_BURST_SIZE]; + + dist_burst_list = RTE_TAILQ_CAST(rte_dist_burst_tailq.head, + rte_dist_burst_list); - distributor_list = RTE_TAILQ_CAST(rte_distributor_tailq.head, - rte_distributor_list); rte_rwlock_write_lock(RTE_EAL_TAILQ_RWLOCK); - TAILQ_INSERT_TAIL(distributor_list, d, next); + TAILQ_INSERT_TAIL(dist_burst_list, d, next); rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK); return d; } +BIND_DEFAULT_SYMBOL(rte_distributor_create, _v1705, 17.05); +MAP_STATIC_SYMBOL(struct rte_distributor *rte_distributor_create( + const char *name, unsigned int socket_id, + unsigned int num_workers, unsigned int alg_type), + rte_distributor_create_v1705); diff --git a/dpdk/lib/librte_distributor/rte_distributor.h b/dpdk/lib/librte_distributor/rte_distributor.h index 7d36bc8a..cbeed04d 100644 --- a/dpdk/lib/librte_distributor/rte_distributor.h +++ b/dpdk/lib/librte_distributor/rte_distributor.h @@ -1,8 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. + * Copyright(c) 2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -31,8 +30,8 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#ifndef _RTE_DISTRIBUTE_H_ -#define _RTE_DISTRIBUTE_H_ +#ifndef _RTE_DISTRIBUTOR_H_ +#define _RTE_DISTRIBUTOR_H_ /** * @file @@ -46,7 +45,12 @@ extern "C" { #endif -#define RTE_DISTRIBUTOR_NAMESIZE 32 /**< Length of name for instance */ +/* Type of distribution (burst/single) */ +enum rte_distributor_alg_type { + RTE_DIST_ALG_BURST = 0, + RTE_DIST_ALG_SINGLE, + RTE_DIST_NUM_ALG_TYPES +}; struct rte_distributor; struct rte_mbuf; @@ -64,12 +68,17 @@ struct rte_mbuf; * @param num_workers * The maximum number of workers that will request packets from this * distributor + * @param alg_type + * Call the legacy API, or use the new burst API. legacy uses 32-bit + * flow ID, and works on a single packet at a time. Latest uses 15- + * bit flow ID and works on up to 8 packets at a time to workers. * @return * The newly created distributor instance */ struct rte_distributor * -rte_distributor_create(const char *name, unsigned socket_id, - unsigned num_workers); +rte_distributor_create(const char *name, unsigned int socket_id, + unsigned int num_workers, + unsigned int alg_type); /* *** APIS to be called on the distributor lcore *** */ /* @@ -85,7 +94,8 @@ rte_distributor_create(const char *name, unsigned socket_id, /** * Process a set of packets by distributing them among workers that request * packets. The distributor will ensure that no two packets that have the - * same flow id, or tag, in the mbuf will be procesed at the same time. + * same flow id, or tag, in the mbuf will be processed on different cores at + * the same time. * * The user is advocated to set tag for each mbuf before calling this function. * If user doesn't set the tag, the tag value can be various values depending on @@ -104,7 +114,7 @@ rte_distributor_create(const char *name, unsigned socket_id, */ int rte_distributor_process(struct rte_distributor *d, - struct rte_mbuf **mbufs, unsigned num_mbufs); + struct rte_mbuf **mbufs, unsigned int num_mbufs); /** * Get a set of mbufs that have been returned to the distributor by workers @@ -122,7 +132,7 @@ rte_distributor_process(struct rte_distributor *d, */ int rte_distributor_returned_pkts(struct rte_distributor *d, - struct rte_mbuf **mbufs, unsigned max_mbufs); + struct rte_mbuf **mbufs, unsigned int max_mbufs); /** * Flush the distributor component, so that there are no in-flight or @@ -161,7 +171,7 @@ rte_distributor_clear_returns(struct rte_distributor *d); */ /** - * API called by a worker to get a new packet to process. Any previous packet + * API called by a worker to get new packets to process. Any previous packets * given to the worker is assumed to have completed processing, and may be * optionally returned to the distributor via the oldpkt parameter. * @@ -170,15 +180,20 @@ rte_distributor_clear_returns(struct rte_distributor *d); * @param worker_id * The worker instance number to use - must be less that num_workers passed * at distributor creation time. + * @param pkts + * The mbufs pointer array to be filled in (up to 8 packets) * @param oldpkt * The previous packet, if any, being processed by the worker + * @param retcount + * The number of packets being returned * * @return - * A new packet to be processed by the worker thread. + * The number of packets in the pkts array */ -struct rte_mbuf * +int rte_distributor_get_pkt(struct rte_distributor *d, - unsigned worker_id, struct rte_mbuf *oldpkt); + unsigned int worker_id, struct rte_mbuf **pkts, + struct rte_mbuf **oldpkt, unsigned int retcount); /** * API called by a worker to return a completed packet without requesting a @@ -189,23 +204,25 @@ rte_distributor_get_pkt(struct rte_distributor *d, * @param worker_id * The worker instance number to use - must be less that num_workers passed * at distributor creation time. - * @param mbuf - * The previous packet being processed by the worker + * @param oldpkt + * The previous packets being processed by the worker + * @param num + * The number of packets in the oldpkt array */ int -rte_distributor_return_pkt(struct rte_distributor *d, unsigned worker_id, - struct rte_mbuf *mbuf); +rte_distributor_return_pkt(struct rte_distributor *d, + unsigned int worker_id, struct rte_mbuf **oldpkt, int num); /** * API called by a worker to request a new packet to process. * Any previous packet given to the worker is assumed to have completed * processing, and may be optionally returned to the distributor via * the oldpkt parameter. - * Unlike rte_distributor_get_pkt(), this function does not wait for a new - * packet to be provided by the distributor. + * Unlike rte_distributor_get_pkt_burst(), this function does not wait for a + * new packet to be provided by the distributor. * - * NOTE: after calling this function, rte_distributor_poll_pkt() should - * be used to poll for the packet requested. The rte_distributor_get_pkt() + * NOTE: after calling this function, rte_distributor_poll_pkt_burst() should + * be used to poll for the packet requested. The rte_distributor_get_pkt_burst() * API should *not* be used to try and retrieve the new packet. * * @param d @@ -214,11 +231,14 @@ rte_distributor_return_pkt(struct rte_distributor *d, unsigned worker_id, * The worker instance number to use - must be less that num_workers passed * at distributor creation time. * @param oldpkt - * The previous packet, if any, being processed by the worker + * The returning packets, if any, processed by the worker + * @param count + * The number of returning packets */ void rte_distributor_request_pkt(struct rte_distributor *d, - unsigned worker_id, struct rte_mbuf *oldpkt); + unsigned int worker_id, struct rte_mbuf **oldpkt, + unsigned int count); /** * API called by a worker to check for a new packet that was previously @@ -231,14 +251,16 @@ rte_distributor_request_pkt(struct rte_distributor *d, * @param worker_id * The worker instance number to use - must be less that num_workers passed * at distributor creation time. + * @param mbufs + * The array of mbufs being given to the worker * * @return - * A new packet to be processed by the worker thread, or NULL if no + * The number of packets being given to the worker thread, zero if no * packet is yet available. */ -struct rte_mbuf * +int rte_distributor_poll_pkt(struct rte_distributor *d, - unsigned worker_id); + unsigned int worker_id, struct rte_mbuf **mbufs); #ifdef __cplusplus } diff --git a/dpdk/lib/librte_distributor/rte_distributor_match_generic.c b/dpdk/lib/librte_distributor/rte_distributor_match_generic.c new file mode 100644 index 00000000..4925a788 --- /dev/null +++ b/dpdk/lib/librte_distributor/rte_distributor_match_generic.c @@ -0,0 +1,43 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include "rte_distributor_private.h" +#include "rte_distributor.h" + +void +find_match_vec(struct rte_distributor *d, + uint16_t *data_ptr, + uint16_t *output_ptr) +{ + find_match_scalar(d, data_ptr, output_ptr); +} diff --git a/dpdk/lib/librte_distributor/rte_distributor_match_sse.c b/dpdk/lib/librte_distributor/rte_distributor_match_sse.c new file mode 100644 index 00000000..44935a69 --- /dev/null +++ b/dpdk/lib/librte_distributor/rte_distributor_match_sse.c @@ -0,0 +1,114 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include "rte_distributor_private.h" +#include "rte_distributor.h" +#include "smmintrin.h" +#include "nmmintrin.h" + + +void +find_match_vec(struct rte_distributor *d, + uint16_t *data_ptr, + uint16_t *output_ptr) +{ + /* Setup */ + __m128i incoming_fids; + __m128i inflight_fids; + __m128i preflight_fids; + __m128i wkr; + __m128i mask1; + __m128i mask2; + __m128i output; + struct rte_distributor_backlog *bl; + uint16_t i; + + /* + * Function overview: + * 2. Loop through all worker ID's + * 2a. Load the current inflights for that worker into an xmm reg + * 2b. Load the current backlog for that worker into an xmm reg + * 2c. use cmpestrm to intersect flow_ids with backlog and inflights + * 2d. Add any matches to the output + * 3. Write the output xmm (matching worker ids). + */ + + + output = _mm_set1_epi16(0); + incoming_fids = _mm_load_si128((__m128i *)data_ptr); + + for (i = 0; i < d->num_workers; i++) { + bl = &d->backlog[i]; + + inflight_fids = + _mm_load_si128((__m128i *)&(d->in_flight_tags[i])); + preflight_fids = + _mm_load_si128((__m128i *)(bl->tags)); + + /* + * Any incoming_fid that exists anywhere in inflight_fids will + * have 0xffff in same position of the mask as the incoming fid + * Example (shortened to bytes for brevity): + * incoming_fids 0x01 0x02 0x03 0x04 0x05 0x06 0x07 0x08 + * inflight_fids 0x03 0x05 0x07 0x00 0x00 0x00 0x00 0x00 + * mask 0x00 0x00 0xff 0x00 0xff 0x00 0xff 0x00 + */ + + mask1 = _mm_cmpestrm(inflight_fids, 8, incoming_fids, 8, + _SIDD_UWORD_OPS | + _SIDD_CMP_EQUAL_ANY | + _SIDD_UNIT_MASK); + mask2 = _mm_cmpestrm(preflight_fids, 8, incoming_fids, 8, + _SIDD_UWORD_OPS | + _SIDD_CMP_EQUAL_ANY | + _SIDD_UNIT_MASK); + + mask1 = _mm_or_si128(mask1, mask2); + /* + * Now mask contains 0xffff where there's a match. + * Next we need to store the worker_id in the relevant position + * in the output. + */ + + wkr = _mm_set1_epi16(i+1); + mask1 = _mm_and_si128(mask1, wkr); + output = _mm_or_si128(mask1, output); + } + + /* + * At this stage, the output 128-bit contains 8 16-bit values, with + * each non-zero value containing the worker ID on which the + * corresponding flow is pinned to. + */ + _mm_store_si128((__m128i *)output_ptr, output); +} diff --git a/dpdk/lib/librte_distributor/rte_distributor_private.h b/dpdk/lib/librte_distributor/rte_distributor_private.h new file mode 100644 index 00000000..24f41b95 --- /dev/null +++ b/dpdk/lib/librte_distributor/rte_distributor_private.h @@ -0,0 +1,202 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_DIST_PRIV_H_ +#define _RTE_DIST_PRIV_H_ + +/** + * @file + * RTE distributor + * + * The distributor is a component which is designed to pass packets + * one-at-a-time to workers, with dynamic load balancing. + */ + +#ifdef __cplusplus +extern "C" { +#endif + +#define NO_FLAGS 0 +#define RTE_DISTRIB_PREFIX "DT_" + +/* + * We will use the bottom four bits of pointer for flags, shifting out + * the top four bits to make room (since a 64-bit pointer actually only uses + * 48 bits). An arithmetic-right-shift will then appropriately restore the + * original pointer value with proper sign extension into the top bits. + */ +#define RTE_DISTRIB_FLAG_BITS 4 +#define RTE_DISTRIB_FLAGS_MASK (0x0F) +#define RTE_DISTRIB_NO_BUF 0 /**< empty flags: no buffer requested */ +#define RTE_DISTRIB_GET_BUF (1) /**< worker requests a buffer, returns old */ +#define RTE_DISTRIB_RETURN_BUF (2) /**< worker returns a buffer, no request */ +#define RTE_DISTRIB_VALID_BUF (4) /**< set if bufptr contains ptr */ + +#define RTE_DISTRIB_BACKLOG_SIZE 8 +#define RTE_DISTRIB_BACKLOG_MASK (RTE_DISTRIB_BACKLOG_SIZE - 1) + +#define RTE_DISTRIB_MAX_RETURNS 128 +#define RTE_DISTRIB_RETURNS_MASK (RTE_DISTRIB_MAX_RETURNS - 1) + +/** + * Maximum number of workers allowed. + * Be aware of increasing the limit, becaus it is limited by how we track + * in-flight tags. See in_flight_bitmask and rte_distributor_process + */ +#define RTE_DISTRIB_MAX_WORKERS 64 + +#define RTE_DISTRIBUTOR_NAMESIZE 32 /**< Length of name for instance */ + +/** + * Buffer structure used to pass the pointer data between cores. This is cache + * line aligned, but to improve performance and prevent adjacent cache-line + * prefetches of buffers for other workers, e.g. when worker 1's buffer is on + * the next cache line to worker 0, we pad this out to three cache lines. + * Only 64-bits of the memory is actually used though. + */ +union rte_distributor_buffer_v20 { + volatile int64_t bufptr64; + char pad[RTE_CACHE_LINE_SIZE*3]; +} __rte_cache_aligned; + +/* + * Transfer up to 8 mbufs at a time to/from workers, and + * flow matching algorithm optimized for 8 flow IDs at a time + */ +#define RTE_DIST_BURST_SIZE 8 + +struct rte_distributor_backlog { + unsigned int start; + unsigned int count; + int64_t pkts[RTE_DIST_BURST_SIZE] __rte_cache_aligned; + uint16_t *tags; /* will point to second cacheline of inflights */ +} __rte_cache_aligned; + + +struct rte_distributor_returned_pkts { + unsigned int start; + unsigned int count; + struct rte_mbuf *mbufs[RTE_DISTRIB_MAX_RETURNS]; +}; + +struct rte_distributor_v20 { + TAILQ_ENTRY(rte_distributor_v20) next; /**< Next in list. */ + + char name[RTE_DISTRIBUTOR_NAMESIZE]; /**< Name of the ring. */ + unsigned int num_workers; /**< Number of workers polling */ + + uint32_t in_flight_tags[RTE_DISTRIB_MAX_WORKERS]; + /**< Tracks the tag being processed per core */ + uint64_t in_flight_bitmask; + /**< on/off bits for in-flight tags. + * Note that if RTE_DISTRIB_MAX_WORKERS is larger than 64 then + * the bitmask has to expand. + */ + + struct rte_distributor_backlog backlog[RTE_DISTRIB_MAX_WORKERS]; + + union rte_distributor_buffer_v20 bufs[RTE_DISTRIB_MAX_WORKERS]; + + struct rte_distributor_returned_pkts returns; +}; + +/* All different signature compare functions */ +enum rte_distributor_match_function { + RTE_DIST_MATCH_SCALAR = 0, + RTE_DIST_MATCH_VECTOR, + RTE_DIST_NUM_MATCH_FNS +}; + +/** + * Buffer structure used to pass the pointer data between cores. This is cache + * line aligned, but to improve performance and prevent adjacent cache-line + * prefetches of buffers for other workers, e.g. when worker 1's buffer is on + * the next cache line to worker 0, we pad this out to two cache lines. + * We can pass up to 8 mbufs at a time in one cacheline. + * There is a separate cacheline for returns in the burst API. + */ +struct rte_distributor_buffer { + volatile int64_t bufptr64[RTE_DIST_BURST_SIZE] + __rte_cache_aligned; /* <= outgoing to worker */ + + int64_t pad1 __rte_cache_aligned; /* <= one cache line */ + + volatile int64_t retptr64[RTE_DIST_BURST_SIZE] + __rte_cache_aligned; /* <= incoming from worker */ + + int64_t pad2 __rte_cache_aligned; /* <= one cache line */ + + int count __rte_cache_aligned; /* <= number of current mbufs */ +}; + +struct rte_distributor { + TAILQ_ENTRY(rte_distributor) next; /**< Next in list. */ + + char name[RTE_DISTRIBUTOR_NAMESIZE]; /**< Name of the ring. */ + unsigned int num_workers; /**< Number of workers polling */ + unsigned int alg_type; /**< Number of alg types */ + + /**> + * First cache line in the this array are the tags inflight + * on the worker core. Second cache line are the backlog + * that are going to go to the worker core. + */ + uint16_t in_flight_tags[RTE_DISTRIB_MAX_WORKERS][RTE_DIST_BURST_SIZE*2] + __rte_cache_aligned; + + struct rte_distributor_backlog backlog[RTE_DISTRIB_MAX_WORKERS] + __rte_cache_aligned; + + struct rte_distributor_buffer bufs[RTE_DISTRIB_MAX_WORKERS]; + + struct rte_distributor_returned_pkts returns; + + enum rte_distributor_match_function dist_match_fn; + + struct rte_distributor_v20 *d_v20; +}; + +void +find_match_scalar(struct rte_distributor *d, + uint16_t *data_ptr, + uint16_t *output_ptr); + +void +find_match_vec(struct rte_distributor *d, + uint16_t *data_ptr, + uint16_t *output_ptr); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/dpdk/lib/librte_distributor/rte_distributor_v1705.h b/dpdk/lib/librte_distributor/rte_distributor_v1705.h new file mode 100644 index 00000000..81b26915 --- /dev/null +++ b/dpdk/lib/librte_distributor/rte_distributor_v1705.h @@ -0,0 +1,89 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_DISTRIB_V1705_H_ +#define _RTE_DISTRIB_V1705_H_ + +/** + * @file + * RTE distributor + * + * The distributor is a component which is designed to pass packets + * one-at-a-time to workers, with dynamic load balancing. + */ + +#ifdef __cplusplus +extern "C" { +#endif + +struct rte_distributor * +rte_distributor_create_v1705(const char *name, unsigned int socket_id, + unsigned int num_workers, + unsigned int alg_type); + +int +rte_distributor_process_v1705(struct rte_distributor *d, + struct rte_mbuf **mbufs, unsigned int num_mbufs); + +int +rte_distributor_returned_pkts_v1705(struct rte_distributor *d, + struct rte_mbuf **mbufs, unsigned int max_mbufs); + +int +rte_distributor_flush_v1705(struct rte_distributor *d); + +void +rte_distributor_clear_returns_v1705(struct rte_distributor *d); + +int +rte_distributor_get_pkt_v1705(struct rte_distributor *d, + unsigned int worker_id, struct rte_mbuf **pkts, + struct rte_mbuf **oldpkt, unsigned int retcount); + +int +rte_distributor_return_pkt_v1705(struct rte_distributor *d, + unsigned int worker_id, struct rte_mbuf **oldpkt, int num); + +void +rte_distributor_request_pkt_v1705(struct rte_distributor *d, + unsigned int worker_id, struct rte_mbuf **oldpkt, + unsigned int count); + +int +rte_distributor_poll_pkt_v1705(struct rte_distributor *d, + unsigned int worker_id, struct rte_mbuf **mbufs); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/dpdk/lib/librte_distributor/rte_distributor_v20.c b/dpdk/lib/librte_distributor/rte_distributor_v20.c new file mode 100644 index 00000000..5be6efd4 --- /dev/null +++ b/dpdk/lib/librte_distributor/rte_distributor_v20.c @@ -0,0 +1,430 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "rte_distributor_v20.h" +#include "rte_distributor_private.h" + +TAILQ_HEAD(rte_distributor_list, rte_distributor_v20); + +static struct rte_tailq_elem rte_distributor_tailq = { + .name = "RTE_DISTRIBUTOR", +}; +EAL_REGISTER_TAILQ(rte_distributor_tailq) + +/**** APIs called by workers ****/ + +void +rte_distributor_request_pkt_v20(struct rte_distributor_v20 *d, + unsigned worker_id, struct rte_mbuf *oldpkt) +{ + union rte_distributor_buffer_v20 *buf = &d->bufs[worker_id]; + int64_t req = (((int64_t)(uintptr_t)oldpkt) << RTE_DISTRIB_FLAG_BITS) + | RTE_DISTRIB_GET_BUF; + while (unlikely(buf->bufptr64 & RTE_DISTRIB_FLAGS_MASK)) + rte_pause(); + buf->bufptr64 = req; +} +VERSION_SYMBOL(rte_distributor_request_pkt, _v20, 2.0); + +struct rte_mbuf * +rte_distributor_poll_pkt_v20(struct rte_distributor_v20 *d, + unsigned worker_id) +{ + union rte_distributor_buffer_v20 *buf = &d->bufs[worker_id]; + if (buf->bufptr64 & RTE_DISTRIB_GET_BUF) + return NULL; + + /* since bufptr64 is signed, this should be an arithmetic shift */ + int64_t ret = buf->bufptr64 >> RTE_DISTRIB_FLAG_BITS; + return (struct rte_mbuf *)((uintptr_t)ret); +} +VERSION_SYMBOL(rte_distributor_poll_pkt, _v20, 2.0); + +struct rte_mbuf * +rte_distributor_get_pkt_v20(struct rte_distributor_v20 *d, + unsigned worker_id, struct rte_mbuf *oldpkt) +{ + struct rte_mbuf *ret; + rte_distributor_request_pkt_v20(d, worker_id, oldpkt); + while ((ret = rte_distributor_poll_pkt_v20(d, worker_id)) == NULL) + rte_pause(); + return ret; +} +VERSION_SYMBOL(rte_distributor_get_pkt, _v20, 2.0); + +int +rte_distributor_return_pkt_v20(struct rte_distributor_v20 *d, + unsigned worker_id, struct rte_mbuf *oldpkt) +{ + union rte_distributor_buffer_v20 *buf = &d->bufs[worker_id]; + uint64_t req = (((int64_t)(uintptr_t)oldpkt) << RTE_DISTRIB_FLAG_BITS) + | RTE_DISTRIB_RETURN_BUF; + buf->bufptr64 = req; + return 0; +} +VERSION_SYMBOL(rte_distributor_return_pkt, _v20, 2.0); + +/**** APIs called on distributor core ***/ + +/* as name suggests, adds a packet to the backlog for a particular worker */ +static int +add_to_backlog(struct rte_distributor_backlog *bl, int64_t item) +{ + if (bl->count == RTE_DISTRIB_BACKLOG_SIZE) + return -1; + + bl->pkts[(bl->start + bl->count++) & (RTE_DISTRIB_BACKLOG_MASK)] + = item; + return 0; +} + +/* takes the next packet for a worker off the backlog */ +static int64_t +backlog_pop(struct rte_distributor_backlog *bl) +{ + bl->count--; + return bl->pkts[bl->start++ & RTE_DISTRIB_BACKLOG_MASK]; +} + +/* stores a packet returned from a worker inside the returns array */ +static inline void +store_return(uintptr_t oldbuf, struct rte_distributor_v20 *d, + unsigned *ret_start, unsigned *ret_count) +{ + /* store returns in a circular buffer - code is branch-free */ + d->returns.mbufs[(*ret_start + *ret_count) & RTE_DISTRIB_RETURNS_MASK] + = (void *)oldbuf; + *ret_start += (*ret_count == RTE_DISTRIB_RETURNS_MASK) & !!(oldbuf); + *ret_count += (*ret_count != RTE_DISTRIB_RETURNS_MASK) & !!(oldbuf); +} + +static inline void +handle_worker_shutdown(struct rte_distributor_v20 *d, unsigned int wkr) +{ + d->in_flight_tags[wkr] = 0; + d->in_flight_bitmask &= ~(1UL << wkr); + d->bufs[wkr].bufptr64 = 0; + if (unlikely(d->backlog[wkr].count != 0)) { + /* On return of a packet, we need to move the + * queued packets for this core elsewhere. + * Easiest solution is to set things up for + * a recursive call. That will cause those + * packets to be queued up for the next free + * core, i.e. it will return as soon as a + * core becomes free to accept the first + * packet, as subsequent ones will be added to + * the backlog for that core. + */ + struct rte_mbuf *pkts[RTE_DISTRIB_BACKLOG_SIZE]; + unsigned i; + struct rte_distributor_backlog *bl = &d->backlog[wkr]; + + for (i = 0; i < bl->count; i++) { + unsigned idx = (bl->start + i) & + RTE_DISTRIB_BACKLOG_MASK; + pkts[i] = (void *)((uintptr_t)(bl->pkts[idx] >> + RTE_DISTRIB_FLAG_BITS)); + } + /* recursive call. + * Note that the tags were set before first level call + * to rte_distributor_process. + */ + rte_distributor_process_v20(d, pkts, i); + bl->count = bl->start = 0; + } +} + +/* this function is called when process() fn is called without any new + * packets. It goes through all the workers and clears any returned packets + * to do a partial flush. + */ +static int +process_returns(struct rte_distributor_v20 *d) +{ + unsigned wkr; + unsigned flushed = 0; + unsigned ret_start = d->returns.start, + ret_count = d->returns.count; + + for (wkr = 0; wkr < d->num_workers; wkr++) { + + const int64_t data = d->bufs[wkr].bufptr64; + uintptr_t oldbuf = 0; + + if (data & RTE_DISTRIB_GET_BUF) { + flushed++; + if (d->backlog[wkr].count) + d->bufs[wkr].bufptr64 = + backlog_pop(&d->backlog[wkr]); + else { + d->bufs[wkr].bufptr64 = RTE_DISTRIB_GET_BUF; + d->in_flight_tags[wkr] = 0; + d->in_flight_bitmask &= ~(1UL << wkr); + } + oldbuf = data >> RTE_DISTRIB_FLAG_BITS; + } else if (data & RTE_DISTRIB_RETURN_BUF) { + handle_worker_shutdown(d, wkr); + oldbuf = data >> RTE_DISTRIB_FLAG_BITS; + } + + store_return(oldbuf, d, &ret_start, &ret_count); + } + + d->returns.start = ret_start; + d->returns.count = ret_count; + + return flushed; +} + +/* process a set of packets to distribute them to workers */ +int +rte_distributor_process_v20(struct rte_distributor_v20 *d, + struct rte_mbuf **mbufs, unsigned num_mbufs) +{ + unsigned next_idx = 0; + unsigned wkr = 0; + struct rte_mbuf *next_mb = NULL; + int64_t next_value = 0; + uint32_t new_tag = 0; + unsigned ret_start = d->returns.start, + ret_count = d->returns.count; + + if (unlikely(num_mbufs == 0)) + return process_returns(d); + + while (next_idx < num_mbufs || next_mb != NULL) { + + int64_t data = d->bufs[wkr].bufptr64; + uintptr_t oldbuf = 0; + + if (!next_mb) { + next_mb = mbufs[next_idx++]; + next_value = (((int64_t)(uintptr_t)next_mb) + << RTE_DISTRIB_FLAG_BITS); + /* + * User is advocated to set tag value for each + * mbuf before calling rte_distributor_process. + * User defined tags are used to identify flows, + * or sessions. + */ + new_tag = next_mb->hash.usr; + + /* + * Note that if RTE_DISTRIB_MAX_WORKERS is larger than 64 + * then the size of match has to be expanded. + */ + uint64_t match = 0; + unsigned i; + /* + * to scan for a match use "xor" and "not" to get a 0/1 + * value, then use shifting to merge to single "match" + * variable, where a one-bit indicates a match for the + * worker given by the bit-position + */ + for (i = 0; i < d->num_workers; i++) + match |= (!(d->in_flight_tags[i] ^ new_tag) + << i); + + /* Only turned-on bits are considered as match */ + match &= d->in_flight_bitmask; + + if (match) { + next_mb = NULL; + unsigned worker = __builtin_ctzl(match); + if (add_to_backlog(&d->backlog[worker], + next_value) < 0) + next_idx--; + } + } + + if ((data & RTE_DISTRIB_GET_BUF) && + (d->backlog[wkr].count || next_mb)) { + + if (d->backlog[wkr].count) + d->bufs[wkr].bufptr64 = + backlog_pop(&d->backlog[wkr]); + + else { + d->bufs[wkr].bufptr64 = next_value; + d->in_flight_tags[wkr] = new_tag; + d->in_flight_bitmask |= (1UL << wkr); + next_mb = NULL; + } + oldbuf = data >> RTE_DISTRIB_FLAG_BITS; + } else if (data & RTE_DISTRIB_RETURN_BUF) { + handle_worker_shutdown(d, wkr); + oldbuf = data >> RTE_DISTRIB_FLAG_BITS; + } + + /* store returns in a circular buffer */ + store_return(oldbuf, d, &ret_start, &ret_count); + + if (++wkr == d->num_workers) + wkr = 0; + } + /* to finish, check all workers for backlog and schedule work for them + * if they are ready */ + for (wkr = 0; wkr < d->num_workers; wkr++) + if (d->backlog[wkr].count && + (d->bufs[wkr].bufptr64 & RTE_DISTRIB_GET_BUF)) { + + int64_t oldbuf = d->bufs[wkr].bufptr64 >> + RTE_DISTRIB_FLAG_BITS; + store_return(oldbuf, d, &ret_start, &ret_count); + + d->bufs[wkr].bufptr64 = backlog_pop(&d->backlog[wkr]); + } + + d->returns.start = ret_start; + d->returns.count = ret_count; + return num_mbufs; +} +VERSION_SYMBOL(rte_distributor_process, _v20, 2.0); + +/* return to the caller, packets returned from workers */ +int +rte_distributor_returned_pkts_v20(struct rte_distributor_v20 *d, + struct rte_mbuf **mbufs, unsigned max_mbufs) +{ + struct rte_distributor_returned_pkts *returns = &d->returns; + unsigned retval = (max_mbufs < returns->count) ? + max_mbufs : returns->count; + unsigned i; + + for (i = 0; i < retval; i++) { + unsigned idx = (returns->start + i) & RTE_DISTRIB_RETURNS_MASK; + mbufs[i] = returns->mbufs[idx]; + } + returns->start += i; + returns->count -= i; + + return retval; +} +VERSION_SYMBOL(rte_distributor_returned_pkts, _v20, 2.0); + +/* return the number of packets in-flight in a distributor, i.e. packets + * being worked on or queued up in a backlog. + */ +static inline unsigned +total_outstanding(const struct rte_distributor_v20 *d) +{ + unsigned wkr, total_outstanding; + + total_outstanding = __builtin_popcountl(d->in_flight_bitmask); + + for (wkr = 0; wkr < d->num_workers; wkr++) + total_outstanding += d->backlog[wkr].count; + + return total_outstanding; +} + +/* flush the distributor, so that there are no outstanding packets in flight or + * queued up. */ +int +rte_distributor_flush_v20(struct rte_distributor_v20 *d) +{ + const unsigned flushed = total_outstanding(d); + + while (total_outstanding(d) > 0) + rte_distributor_process_v20(d, NULL, 0); + + return flushed; +} +VERSION_SYMBOL(rte_distributor_flush, _v20, 2.0); + +/* clears the internal returns array in the distributor */ +void +rte_distributor_clear_returns_v20(struct rte_distributor_v20 *d) +{ + d->returns.start = d->returns.count = 0; +#ifndef __OPTIMIZE__ + memset(d->returns.mbufs, 0, sizeof(d->returns.mbufs)); +#endif +} +VERSION_SYMBOL(rte_distributor_clear_returns, _v20, 2.0); + +/* creates a distributor instance */ +struct rte_distributor_v20 * +rte_distributor_create_v20(const char *name, + unsigned socket_id, + unsigned num_workers) +{ + struct rte_distributor_v20 *d; + struct rte_distributor_list *distributor_list; + char mz_name[RTE_MEMZONE_NAMESIZE]; + const struct rte_memzone *mz; + + /* compilation-time checks */ + RTE_BUILD_BUG_ON((sizeof(*d) & RTE_CACHE_LINE_MASK) != 0); + RTE_BUILD_BUG_ON((RTE_DISTRIB_MAX_WORKERS & 7) != 0); + RTE_BUILD_BUG_ON(RTE_DISTRIB_MAX_WORKERS > + sizeof(d->in_flight_bitmask) * CHAR_BIT); + + if (name == NULL || num_workers >= RTE_DISTRIB_MAX_WORKERS) { + rte_errno = EINVAL; + return NULL; + } + + snprintf(mz_name, sizeof(mz_name), RTE_DISTRIB_PREFIX"%s", name); + mz = rte_memzone_reserve(mz_name, sizeof(*d), socket_id, NO_FLAGS); + if (mz == NULL) { + rte_errno = ENOMEM; + return NULL; + } + + d = mz->addr; + snprintf(d->name, sizeof(d->name), "%s", name); + d->num_workers = num_workers; + + distributor_list = RTE_TAILQ_CAST(rte_distributor_tailq.head, + rte_distributor_list); + + rte_rwlock_write_lock(RTE_EAL_TAILQ_RWLOCK); + TAILQ_INSERT_TAIL(distributor_list, d, next); + rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK); + + return d; +} +VERSION_SYMBOL(rte_distributor_create, _v20, 2.0); diff --git a/dpdk/lib/librte_distributor/rte_distributor_v20.h b/dpdk/lib/librte_distributor/rte_distributor_v20.h new file mode 100644 index 00000000..f02e6aac --- /dev/null +++ b/dpdk/lib/librte_distributor/rte_distributor_v20.h @@ -0,0 +1,247 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_DISTRIB_V20_H_ +#define _RTE_DISTRIB_V20_H_ + +/** + * @file + * RTE distributor + * + * The distributor is a component which is designed to pass packets + * one-at-a-time to workers, with dynamic load balancing. + */ + +#ifdef __cplusplus +extern "C" { +#endif + +#define RTE_DISTRIBUTOR_NAMESIZE 32 /**< Length of name for instance */ + +struct rte_distributor_v20; +struct rte_mbuf; + +/** + * Function to create a new distributor instance + * + * Reserves the memory needed for the distributor operation and + * initializes the distributor to work with the configured number of workers. + * + * @param name + * The name to be given to the distributor instance. + * @param socket_id + * The NUMA node on which the memory is to be allocated + * @param num_workers + * The maximum number of workers that will request packets from this + * distributor + * @return + * The newly created distributor instance + */ +struct rte_distributor_v20 * +rte_distributor_create_v20(const char *name, unsigned int socket_id, + unsigned int num_workers); + +/* *** APIS to be called on the distributor lcore *** */ +/* + * The following APIs are the public APIs which are designed for use on a + * single lcore which acts as the distributor lcore for a given distributor + * instance. These functions cannot be called on multiple cores simultaneously + * without using locking to protect access to the internals of the distributor. + * + * NOTE: a given lcore cannot act as both a distributor lcore and a worker lcore + * for the same distributor instance, otherwise deadlock will result. + */ + +/** + * Process a set of packets by distributing them among workers that request + * packets. The distributor will ensure that no two packets that have the + * same flow id, or tag, in the mbuf will be processed at the same time. + * + * The user is advocated to set tag for each mbuf before calling this function. + * If user doesn't set the tag, the tag value can be various values depending on + * driver implementation and configuration. + * + * This is not multi-thread safe and should only be called on a single lcore. + * + * @param d + * The distributor instance to be used + * @param mbufs + * The mbufs to be distributed + * @param num_mbufs + * The number of mbufs in the mbufs array + * @return + * The number of mbufs processed. + */ +int +rte_distributor_process_v20(struct rte_distributor_v20 *d, + struct rte_mbuf **mbufs, unsigned int num_mbufs); + +/** + * Get a set of mbufs that have been returned to the distributor by workers + * + * This should only be called on the same lcore as rte_distributor_process() + * + * @param d + * The distributor instance to be used + * @param mbufs + * The mbufs pointer array to be filled in + * @param max_mbufs + * The size of the mbufs array + * @return + * The number of mbufs returned in the mbufs array. + */ +int +rte_distributor_returned_pkts_v20(struct rte_distributor_v20 *d, + struct rte_mbuf **mbufs, unsigned int max_mbufs); + +/** + * Flush the distributor component, so that there are no in-flight or + * backlogged packets awaiting processing + * + * This should only be called on the same lcore as rte_distributor_process() + * + * @param d + * The distributor instance to be used + * @return + * The number of queued/in-flight packets that were completed by this call. + */ +int +rte_distributor_flush_v20(struct rte_distributor_v20 *d); + +/** + * Clears the array of returned packets used as the source for the + * rte_distributor_returned_pkts() API call. + * + * This should only be called on the same lcore as rte_distributor_process() + * + * @param d + * The distributor instance to be used + */ +void +rte_distributor_clear_returns_v20(struct rte_distributor_v20 *d); + +/* *** APIS to be called on the worker lcores *** */ +/* + * The following APIs are the public APIs which are designed for use on + * multiple lcores which act as workers for a distributor. Each lcore should use + * a unique worker id when requesting packets. + * + * NOTE: a given lcore cannot act as both a distributor lcore and a worker lcore + * for the same distributor instance, otherwise deadlock will result. + */ + +/** + * API called by a worker to get a new packet to process. Any previous packet + * given to the worker is assumed to have completed processing, and may be + * optionally returned to the distributor via the oldpkt parameter. + * + * @param d + * The distributor instance to be used + * @param worker_id + * The worker instance number to use - must be less that num_workers passed + * at distributor creation time. + * @param oldpkt + * The previous packet, if any, being processed by the worker + * + * @return + * A new packet to be processed by the worker thread. + */ +struct rte_mbuf * +rte_distributor_get_pkt_v20(struct rte_distributor_v20 *d, + unsigned int worker_id, struct rte_mbuf *oldpkt); + +/** + * API called by a worker to return a completed packet without requesting a + * new packet, for example, because a worker thread is shutting down + * + * @param d + * The distributor instance to be used + * @param worker_id + * The worker instance number to use - must be less that num_workers passed + * at distributor creation time. + * @param mbuf + * The previous packet being processed by the worker + */ +int +rte_distributor_return_pkt_v20(struct rte_distributor_v20 *d, + unsigned int worker_id, struct rte_mbuf *mbuf); + +/** + * API called by a worker to request a new packet to process. + * Any previous packet given to the worker is assumed to have completed + * processing, and may be optionally returned to the distributor via + * the oldpkt parameter. + * Unlike rte_distributor_get_pkt(), this function does not wait for a new + * packet to be provided by the distributor. + * + * NOTE: after calling this function, rte_distributor_poll_pkt() should + * be used to poll for the packet requested. The rte_distributor_get_pkt() + * API should *not* be used to try and retrieve the new packet. + * + * @param d + * The distributor instance to be used + * @param worker_id + * The worker instance number to use - must be less that num_workers passed + * at distributor creation time. + * @param oldpkt + * The previous packet, if any, being processed by the worker + */ +void +rte_distributor_request_pkt_v20(struct rte_distributor_v20 *d, + unsigned int worker_id, struct rte_mbuf *oldpkt); + +/** + * API called by a worker to check for a new packet that was previously + * requested by a call to rte_distributor_request_pkt(). It does not wait + * for the new packet to be available, but returns NULL if the request has + * not yet been fulfilled by the distributor. + * + * @param d + * The distributor instance to be used + * @param worker_id + * The worker instance number to use - must be less that num_workers passed + * at distributor creation time. + * + * @return + * A new packet to be processed by the worker thread, or NULL if no + * packet is yet available. + */ +struct rte_mbuf * +rte_distributor_poll_pkt_v20(struct rte_distributor_v20 *d, + unsigned int worker_id); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/dpdk/lib/librte_distributor/rte_distributor_version.map b/dpdk/lib/librte_distributor/rte_distributor_version.map index 73fdc437..3a285b39 100644 --- a/dpdk/lib/librte_distributor/rte_distributor_version.map +++ b/dpdk/lib/librte_distributor/rte_distributor_version.map @@ -13,3 +13,17 @@ DPDK_2.0 { local: *; }; + +DPDK_17.05 { + global: + + rte_distributor_clear_returns; + rte_distributor_create; + rte_distributor_flush; + rte_distributor_get_pkt; + rte_distributor_poll_pkt; + rte_distributor_process; + rte_distributor_request_pkt; + rte_distributor_return_pkt; + rte_distributor_returned_pkts; +} DPDK_2.0; diff --git a/dpdk/lib/librte_eal/Makefile b/dpdk/lib/librte_eal/Makefile index cf11a099..5690bb49 100644 --- a/dpdk/lib/librte_eal/Makefile +++ b/dpdk/lib/librte_eal/Makefile @@ -33,6 +33,8 @@ include $(RTE_SDK)/mk/rte.vars.mk DIRS-y += common DIRS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += linuxapp +DEPDIRS-linuxapp := common DIRS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += bsdapp +DEPDIRS-bsdapp := common include $(RTE_SDK)/mk/rte.subdir.mk diff --git a/dpdk/lib/librte_eal/bsdapp/contigmem/contigmem.c b/dpdk/lib/librte_eal/bsdapp/contigmem/contigmem.c index da971deb..e8fb9087 100644 --- a/dpdk/lib/librte_eal/bsdapp/contigmem/contigmem.c +++ b/dpdk/lib/librte_eal/bsdapp/contigmem/contigmem.c @@ -50,24 +50,37 @@ __FBSDID("$FreeBSD$"); #include #include +#include #include #include #include +#include + +struct contigmem_buffer { + void *addr; + int refcnt; + struct mtx mtx; +}; + +struct contigmem_vm_handle { + int buffer_index; +}; static int contigmem_load(void); static int contigmem_unload(void); static int contigmem_physaddr(SYSCTL_HANDLER_ARGS); -static d_mmap_t contigmem_mmap; static d_mmap_single_t contigmem_mmap_single; static d_open_t contigmem_open; +static d_close_t contigmem_close; static int contigmem_num_buffers = RTE_CONTIGMEM_DEFAULT_NUM_BUFS; static int64_t contigmem_buffer_size = RTE_CONTIGMEM_DEFAULT_BUF_SIZE; static eventhandler_tag contigmem_eh_tag; -static void *contigmem_buffers[RTE_CONTIGMEM_MAX_NUM_BUFS]; +static struct contigmem_buffer contigmem_buffers[RTE_CONTIGMEM_MAX_NUM_BUFS]; static struct cdev *contigmem_cdev = NULL; +static int contigmem_refcnt; TUNABLE_INT("hw.contigmem.num_buffers", &contigmem_num_buffers); TUNABLE_QUAD("hw.contigmem.buffer_size", &contigmem_buffer_size); @@ -78,6 +91,8 @@ SYSCTL_INT(_hw_contigmem, OID_AUTO, num_buffers, CTLFLAG_RD, &contigmem_num_buffers, 0, "Number of contigmem buffers allocated"); SYSCTL_QUAD(_hw_contigmem, OID_AUTO, buffer_size, CTLFLAG_RD, &contigmem_buffer_size, 0, "Size of each contiguous buffer"); +SYSCTL_INT(_hw_contigmem, OID_AUTO, num_references, CTLFLAG_RD, + &contigmem_refcnt, 0, "Number of references to contigmem"); static SYSCTL_NODE(_hw_contigmem, OID_AUTO, physaddr, CTLFLAG_RD, 0, "physaddr"); @@ -114,42 +129,49 @@ MODULE_VERSION(contigmem, 1); static struct cdevsw contigmem_ops = { .d_name = "contigmem", .d_version = D_VERSION, - .d_mmap = contigmem_mmap, + .d_flags = D_TRACKCLOSE, .d_mmap_single = contigmem_mmap_single, .d_open = contigmem_open, + .d_close = contigmem_close, }; static int contigmem_load() { char index_string[8], description[32]; - int i; + int i, error = 0; + void *addr; if (contigmem_num_buffers > RTE_CONTIGMEM_MAX_NUM_BUFS) { printf("%d buffers requested is greater than %d allowed\n", contigmem_num_buffers, RTE_CONTIGMEM_MAX_NUM_BUFS); - return EINVAL; + error = EINVAL; + goto error; } if (contigmem_buffer_size < PAGE_SIZE || (contigmem_buffer_size & (contigmem_buffer_size - 1)) != 0) { printf("buffer size 0x%lx is not greater than PAGE_SIZE and " "power of two\n", contigmem_buffer_size); - return EINVAL; + error = EINVAL; + goto error; } for (i = 0; i < contigmem_num_buffers; i++) { - contigmem_buffers[i] = - contigmalloc(contigmem_buffer_size, M_CONTIGMEM, M_ZERO, 0, - BUS_SPACE_MAXADDR, contigmem_buffer_size, 0); - - if (contigmem_buffers[i] == NULL) { + addr = contigmalloc(contigmem_buffer_size, M_CONTIGMEM, M_ZERO, + 0, BUS_SPACE_MAXADDR, contigmem_buffer_size, 0); + if (addr == NULL) { printf("contigmalloc failed for buffer %d\n", i); - return ENOMEM; + error = ENOMEM; + goto error; } - printf("%2u: virt=%p phys=%p\n", i, contigmem_buffers[i], - (void *)pmap_kextract((vm_offset_t)contigmem_buffers[i])); + printf("%2u: virt=%p phys=%p\n", i, addr, + (void *)pmap_kextract((vm_offset_t)addr)); + + mtx_init(&contigmem_buffers[i].mtx, "contigmem", NULL, MTX_DEF); + contigmem_buffers[i].addr = addr; + contigmem_buffers[i].refcnt = 0; snprintf(index_string, sizeof(index_string), "%d", i); snprintf(description, sizeof(description), @@ -165,6 +187,17 @@ contigmem_load() GID_WHEEL, 0600, "contigmem"); return 0; + +error: + for (i = 0; i < contigmem_num_buffers; i++) { + if (contigmem_buffers[i].addr != NULL) + contigfree(contigmem_buffers[i].addr, + contigmem_buffer_size, M_CONTIGMEM); + if (mtx_initialized(&contigmem_buffers[i].mtx)) + mtx_destroy(&contigmem_buffers[i].mtx); + } + + return error; } static int @@ -172,16 +205,22 @@ contigmem_unload() { int i; + if (contigmem_refcnt > 0) + return EBUSY; + if (contigmem_cdev != NULL) destroy_dev(contigmem_cdev); if (contigmem_eh_tag != NULL) EVENTHANDLER_DEREGISTER(process_exit, contigmem_eh_tag); - for (i = 0; i < RTE_CONTIGMEM_MAX_NUM_BUFS; i++) - if (contigmem_buffers[i] != NULL) - contigfree(contigmem_buffers[i], contigmem_buffer_size, - M_CONTIGMEM); + for (i = 0; i < RTE_CONTIGMEM_MAX_NUM_BUFS; i++) { + if (contigmem_buffers[i].addr != NULL) + contigfree(contigmem_buffers[i].addr, + contigmem_buffer_size, M_CONTIGMEM); + if (mtx_initialized(&contigmem_buffers[i].mtx)) + mtx_destroy(&contigmem_buffers[i].mtx); + } return 0; } @@ -192,7 +231,7 @@ contigmem_physaddr(SYSCTL_HANDLER_ARGS) uint64_t physaddr; int index = (int)(uintptr_t)arg1; - physaddr = (uint64_t)vtophys(contigmem_buffers[index]); + physaddr = (uint64_t)vtophys(contigmem_buffers[index].addr); return sysctl_handle_64(oidp, &physaddr, 0, req); } @@ -200,22 +239,121 @@ static int contigmem_open(struct cdev *cdev, int fflags, int devtype, struct thread *td) { + + atomic_add_int(&contigmem_refcnt, 1); + return 0; } static int -contigmem_mmap(struct cdev *cdev, vm_ooffset_t offset, vm_paddr_t *paddr, - int prot, vm_memattr_t *memattr) +contigmem_close(struct cdev *cdev, int fflags, int devtype, + struct thread *td) { - *paddr = offset; + atomic_subtract_int(&contigmem_refcnt, 1); + return 0; } +static int +contigmem_cdev_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot, + vm_ooffset_t foff, struct ucred *cred, u_short *color) +{ + struct contigmem_vm_handle *vmh = handle; + struct contigmem_buffer *buf; + + buf = &contigmem_buffers[vmh->buffer_index]; + + atomic_add_int(&contigmem_refcnt, 1); + + mtx_lock(&buf->mtx); + if (buf->refcnt == 0) + memset(buf->addr, 0, contigmem_buffer_size); + buf->refcnt++; + mtx_unlock(&buf->mtx); + + return 0; +} + +static void +contigmem_cdev_pager_dtor(void *handle) +{ + struct contigmem_vm_handle *vmh = handle; + struct contigmem_buffer *buf; + + buf = &contigmem_buffers[vmh->buffer_index]; + + mtx_lock(&buf->mtx); + buf->refcnt--; + mtx_unlock(&buf->mtx); + + free(vmh, M_CONTIGMEM); + + atomic_subtract_int(&contigmem_refcnt, 1); +} + +static int +contigmem_cdev_pager_fault(vm_object_t object, vm_ooffset_t offset, int prot, + vm_page_t *mres) +{ + vm_paddr_t paddr; + vm_page_t m_paddr, page; + vm_memattr_t memattr, memattr1; + + memattr = object->memattr; + + VM_OBJECT_WUNLOCK(object); + + paddr = offset; + + m_paddr = vm_phys_paddr_to_vm_page(paddr); + if (m_paddr != NULL) { + memattr1 = pmap_page_get_memattr(m_paddr); + if (memattr1 != memattr) + memattr = memattr1; + } + + if (((*mres)->flags & PG_FICTITIOUS) != 0) { + /* + * If the passed in result page is a fake page, update it with + * the new physical address. + */ + page = *mres; + VM_OBJECT_WLOCK(object); + vm_page_updatefake(page, paddr, memattr); + } else { + vm_page_t mret; + /* + * Replace the passed in reqpage page with our own fake page and + * free up the original page. + */ + page = vm_page_getfake(paddr, memattr); + VM_OBJECT_WLOCK(object); + mret = vm_page_replace(page, object, (*mres)->pindex); + KASSERT(mret == *mres, + ("invalid page replacement, old=%p, ret=%p", *mres, mret)); + vm_page_lock(mret); + vm_page_free(mret); + vm_page_unlock(mret); + *mres = page; + } + + page->valid = VM_PAGE_BITS_ALL; + + return VM_PAGER_OK; +} + +static struct cdev_pager_ops contigmem_cdev_pager_ops = { + .cdev_pg_ctor = contigmem_cdev_pager_ctor, + .cdev_pg_dtor = contigmem_cdev_pager_dtor, + .cdev_pg_fault = contigmem_cdev_pager_fault, +}; + static int contigmem_mmap_single(struct cdev *cdev, vm_ooffset_t *offset, vm_size_t size, struct vm_object **obj, int nprot) { + struct contigmem_vm_handle *vmh; uint64_t buffer_index; /* @@ -227,10 +365,17 @@ contigmem_mmap_single(struct cdev *cdev, vm_ooffset_t *offset, vm_size_t size, if (buffer_index >= contigmem_num_buffers) return EINVAL; - memset(contigmem_buffers[buffer_index], 0, contigmem_buffer_size); - *offset = (vm_ooffset_t)vtophys(contigmem_buffers[buffer_index]); - *obj = vm_pager_allocate(OBJT_DEVICE, cdev, size, nprot, *offset, - curthread->td_ucred); + if (size > contigmem_buffer_size) + return EINVAL; + + vmh = malloc(sizeof(*vmh), M_CONTIGMEM, M_NOWAIT | M_ZERO); + if (vmh == NULL) + return ENOMEM; + vmh->buffer_index = buffer_index; + + *offset = (vm_ooffset_t)vtophys(contigmem_buffers[buffer_index].addr); + *obj = cdev_pager_allocate(vmh, OBJT_DEVICE, &contigmem_cdev_pager_ops, + size, nprot, *offset, curthread->td_ucred); return 0; } diff --git a/dpdk/lib/librte_eal/bsdapp/eal/Makefile b/dpdk/lib/librte_eal/bsdapp/eal/Makefile index 988cbbce..afa117de 100644 --- a/dpdk/lib/librte_eal/bsdapp/eal/Makefile +++ b/dpdk/lib/librte_eal/bsdapp/eal/Makefile @@ -46,17 +46,15 @@ LDLIBS += -lexecinfo LDLIBS += -lpthread LDLIBS += -lgcc_s -EXPORT_MAP := rte_eal_version.map +EXPORT_MAP := ../../rte_eal_version.map -LIBABIVER := 2 +LIBABIVER := 6 -# specific to linuxapp exec-env +# specific to bsdapp exec-env SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) := eal.c SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += eal_memory.c SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += eal_hugepage_info.c SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += eal_thread.c -SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += eal_log.c -SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += eal_pci.c SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += eal_debug.c SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += eal_lcore.c SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += eal_timer.c @@ -69,8 +67,6 @@ SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += eal_common_timer.c SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += eal_common_memzone.c SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += eal_common_log.c SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += eal_common_launch.c -SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += eal_common_pci.c -SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += eal_common_pci_uio.c SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += eal_common_memory.c SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += eal_common_tailqs.c SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += eal_common_errno.c @@ -78,6 +74,7 @@ SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += eal_common_cpuflags.c SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += eal_common_string_fns.c SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += eal_common_hexdump.c SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += eal_common_devargs.c +SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += eal_common_bus.c SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += eal_common_dev.c SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += eal_common_options.c SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += eal_common_thread.c @@ -86,10 +83,12 @@ SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += rte_malloc.c SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += malloc_elem.c SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += malloc_heap.c SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += rte_keepalive.c +SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += rte_service.c # from arch dir SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += rte_cpuflags.c SRCS-$(CONFIG_RTE_ARCH_X86) += rte_spinlock.c +SRCS-y += rte_cycles.c CFLAGS_eal_common_cpuflags.o := $(CPUFLAGS_LIST) @@ -105,12 +104,9 @@ CFLAGS_eal_thread.o += -Wno-return-type CFLAGS_eal_hpet.o += -Wno-return-type endif -INC := rte_interrupts.h +INC := # no bsdapp specific headers SYMLINK-$(CONFIG_RTE_EXEC_ENV_BSDAPP)-include/exec-env := \ $(addprefix include/exec-env/,$(INC)) -DEPDIRS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += lib/librte_eal/common -DEPDIRS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += lib/librte_eal/common/arch/$(ARCH_DIR) - include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/lib/librte_eal/bsdapp/eal/eal.c b/dpdk/lib/librte_eal/bsdapp/eal/eal.c index a0c8f8c8..369a682a 100644 --- a/dpdk/lib/librte_eal/bsdapp/eal/eal.c +++ b/dpdk/lib/librte_eal/bsdapp/eal/eal.c @@ -45,29 +45,28 @@ #include #include #include -#include #include #include #include #include #include -#include #include #include #include +#include #include #include +#include #include #include #include #include #include #include -#include +#include #include #include -#include #include #include #include @@ -111,6 +110,13 @@ struct internal_config internal_config; /* used by rte_rdtsc() */ int rte_cycles_vmware_tsc_map; +/* Return mbuf pool ops name */ +const char * +rte_eal_mbuf_default_mempool_ops(void) +{ + return internal_config.mbuf_pool_ops_name; +} + /* Return a pointer to the configuration structure */ struct rte_config * rte_eal_get_configuration(void) @@ -118,6 +124,12 @@ rte_eal_get_configuration(void) return &rte_config; } +enum rte_iova_mode +rte_eal_iova_mode(void) +{ + return rte_eal_get_configuration()->iova_mode; +} + /* parse a sysfs (or other) file containing one integer value */ int eal_parse_sysfs_value(const char *filename, unsigned long *val) @@ -193,7 +205,7 @@ rte_eal_config_create(void) rte_panic("Cannot mmap memory for rte_config\n"); } memcpy(rte_mem_cfg_addr, &early_mem_config, sizeof(early_mem_config)); - rte_config.mem_config = (struct rte_mem_config *) rte_mem_cfg_addr; + rte_config.mem_config = rte_mem_cfg_addr; } /* attach to an existing shared memory config */ @@ -218,7 +230,7 @@ rte_eal_config_attach(void) if (rte_mem_cfg_addr == MAP_FAILED) rte_panic("Cannot mmap memory for rte_config\n"); - rte_config.mem_config = (struct rte_mem_config *) rte_mem_cfg_addr; + rte_config.mem_config = rte_mem_cfg_addr; } /* Detect if we are a primary or a secondary process */ @@ -321,8 +333,6 @@ eal_log_level_parse(int argc, char **argv) optind = 1; optreset = 1; - eal_reset_internal_config(&internal_config); - while ((opt = getopt_long(argc, argvopt, eal_short_options, eal_long_options, &option_index)) != EOF) { @@ -386,6 +396,9 @@ eal_parse_args(int argc, char **argv) continue; switch (opt) { + case OPT_MBUF_POOL_OPS_NAME_NUM: + internal_config.mbuf_pool_ops_name = optarg; + break; case 'h': eal_usage(prgname); exit(EXIT_SUCCESS); @@ -486,6 +499,12 @@ rte_eal_iopl_init(void) return 0; } +static void rte_eal_init_alert(const char *msg) +{ + fprintf(stderr, "EAL: FATAL: %s\n", msg); + RTE_LOG(ERR, EAL, "%s\n", msg); +} + /* Launch threads, called at application init(). */ int rte_eal_init(int argc, char **argv) @@ -496,30 +515,71 @@ rte_eal_init(int argc, char **argv) char cpuset[RTE_CPU_AFFINITY_STR_LEN]; char thread_name[RTE_MAX_THREAD_NAME_LEN]; - if (!rte_atomic32_test_and_set(&run_once)) + /* checks if the machine is adequate */ + if (!rte_cpu_is_supported()) { + rte_eal_init_alert("unsupported cpu type."); + rte_errno = ENOTSUP; return -1; + } + + if (!rte_atomic32_test_and_set(&run_once)) { + rte_eal_init_alert("already called initialization."); + rte_errno = EALREADY; + return -1; + } thread_id = pthread_self(); - if (rte_eal_log_early_init() < 0) - rte_panic("Cannot init early logs\n"); - - eal_log_level_parse(argc, argv); + eal_reset_internal_config(&internal_config); /* set log level as early as possible */ - rte_set_log_level(internal_config.log_level); + eal_log_level_parse(argc, argv); - if (rte_eal_cpu_init() < 0) - rte_panic("Cannot detect lcores\n"); + if (rte_eal_cpu_init() < 0) { + rte_eal_init_alert("Cannot detect lcores."); + rte_errno = ENOTSUP; + return -1; + } fctret = eal_parse_args(argc, argv); - if (fctret < 0) - exit(1); + if (fctret < 0) { + rte_eal_init_alert("Invalid 'command line' arguments."); + rte_errno = EINVAL; + rte_atomic32_clear(&run_once); + return -1; + } + + if (eal_plugins_init() < 0) { + rte_eal_init_alert("Cannot init plugins\n"); + rte_errno = EINVAL; + rte_atomic32_clear(&run_once); + return -1; + } + + if (eal_option_device_parse()) { + rte_errno = ENODEV; + rte_atomic32_clear(&run_once); + return -1; + } + + if (rte_bus_scan()) { + rte_eal_init_alert("Cannot scan the buses for devices\n"); + rte_errno = ENODEV; + rte_atomic32_clear(&run_once); + return -1; + } + + /* autodetect the iova mapping mode (default is iova_pa) */ + rte_eal_get_configuration()->iova_mode = rte_bus_get_iommu_class(); if (internal_config.no_hugetlbfs == 0 && internal_config.process_type != RTE_PROC_SECONDARY && - eal_hugepage_info_init() < 0) - rte_panic("Cannot get hugepage information\n"); + eal_hugepage_info_init() < 0) { + rte_eal_init_alert("Cannot get hugepage information."); + rte_errno = EACCES; + rte_atomic32_clear(&run_once); + return -1; + } if (internal_config.memory == 0 && internal_config.force_sockets == 0) { if (internal_config.no_hugetlbfs) @@ -543,35 +603,43 @@ rte_eal_init(int argc, char **argv) rte_config_init(); - if (rte_eal_memory_init() < 0) - rte_panic("Cannot init memory\n"); + if (rte_eal_memory_init() < 0) { + rte_eal_init_alert("Cannot init memory\n"); + rte_errno = ENOMEM; + return -1; + } - if (rte_eal_memzone_init() < 0) - rte_panic("Cannot init memzone\n"); + if (rte_eal_memzone_init() < 0) { + rte_eal_init_alert("Cannot init memzone\n"); + rte_errno = ENODEV; + return -1; + } - if (rte_eal_tailqs_init() < 0) - rte_panic("Cannot init tail queues for objects\n"); + if (rte_eal_tailqs_init() < 0) { + rte_eal_init_alert("Cannot init tail queues for objects\n"); + rte_errno = EFAULT; + return -1; + } -/* if (rte_eal_log_init(argv[0], internal_config.syslog_facility) < 0) - rte_panic("Cannot init logs\n");*/ + if (rte_eal_alarm_init() < 0) { + rte_eal_init_alert("Cannot init interrupt-handling thread\n"); + /* rte_eal_alarm_init sets rte_errno on failure. */ + return -1; + } - if (rte_eal_alarm_init() < 0) - rte_panic("Cannot init interrupt-handling thread\n"); + if (rte_eal_intr_init() < 0) { + rte_eal_init_alert("Cannot init interrupt-handling thread\n"); + return -1; + } - if (rte_eal_intr_init() < 0) - rte_panic("Cannot init interrupt-handling thread\n"); - - if (rte_eal_timer_init() < 0) - rte_panic("Cannot init HPET or TSC timers\n"); - - if (rte_eal_pci_init() < 0) - rte_panic("Cannot init PCI\n"); + if (rte_eal_timer_init() < 0) { + rte_eal_init_alert("Cannot init HPET or TSC timers\n"); + rte_errno = ENOTSUP; + return -1; + } eal_check_mem_on_local_socket(); - if (eal_plugins_init() < 0) - rte_panic("Cannot init plugins\n"); - eal_thread_init_master(rte_config.master_lcore); ret = eal_thread_dump_affinity(cpuset, RTE_CPU_AFFINITY_STR_LEN); @@ -580,9 +648,6 @@ rte_eal_init(int argc, char **argv) rte_config.master_lcore, thread_id, cpuset, ret == 0 ? "" : "..."); - if (rte_eal_dev_init() < 0) - rte_panic("Cannot init pmd devices\n"); - RTE_LCORE_FOREACH_SLAVE(i) { /* @@ -615,9 +680,29 @@ rte_eal_init(int argc, char **argv) rte_eal_mp_remote_launch(sync_func, NULL, SKIP_MASTER); rte_eal_mp_wait_lcore(); - /* Probe & Initialize PCI devices */ - if (rte_eal_pci_probe()) - rte_panic("Cannot probe PCI\n"); + /* initialize services so vdevs register service during bus_probe. */ + ret = rte_service_init(); + if (ret) { + rte_eal_init_alert("rte_service_init() failed\n"); + rte_errno = ENOEXEC; + return -1; + } + + /* Probe all the buses and devices/drivers on them */ + if (rte_bus_probe()) { + rte_eal_init_alert("Cannot probe devices\n"); + rte_errno = ENOTSUP; + return -1; + } + + /* initialize default service/lcore mappings and start running. Ignore + * -ENOTSUP, as it indicates no service coremask passed to EAL. + */ + ret = rte_service_start_with_defaults(); + if (ret < 0 && ret != -ENOTSUP) { + rte_errno = ENOEXEC; + return -1; + } rte_eal_mcfg_complete(); @@ -636,3 +721,60 @@ rte_eal_process_type(void) { return rte_config.process_type; } + +int rte_eal_has_pci(void) +{ + return !internal_config.no_pci; +} + +int rte_eal_create_uio_dev(void) +{ + return internal_config.create_uio_dev; +} + +enum rte_intr_mode +rte_eal_vfio_intr_mode(void) +{ + return RTE_INTR_MODE_NONE; +} + +/* dummy forward declaration. */ +struct vfio_device_info; + +/* dummy prototypes. */ +int rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr, + int *vfio_dev_fd, struct vfio_device_info *device_info); +int rte_vfio_release_device(const char *sysfs_base, const char *dev_addr, int fd); +int rte_vfio_enable(const char *modname); +int rte_vfio_is_enabled(const char *modname); +int rte_vfio_noiommu_is_enabled(void); + +int rte_vfio_setup_device(__rte_unused const char *sysfs_base, + __rte_unused const char *dev_addr, + __rte_unused int *vfio_dev_fd, + __rte_unused struct vfio_device_info *device_info) +{ + return -1; +} + +int rte_vfio_release_device(__rte_unused const char *sysfs_base, + __rte_unused const char *dev_addr, + __rte_unused int fd) +{ + return -1; +} + +int rte_vfio_enable(__rte_unused const char *modname) +{ + return -1; +} + +int rte_vfio_is_enabled(__rte_unused const char *modname) +{ + return 0; +} + +int rte_vfio_noiommu_is_enabled(void) +{ + return 0; +} diff --git a/dpdk/lib/librte_eal/bsdapp/eal/eal_debug.c b/dpdk/lib/librte_eal/bsdapp/eal/eal_debug.c index 5fbc17c5..e1c75548 100644 --- a/dpdk/lib/librte_eal/bsdapp/eal/eal_debug.c +++ b/dpdk/lib/librte_eal/bsdapp/eal/eal_debug.c @@ -31,7 +31,9 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +#ifdef RTE_BACKTRACE #include +#endif #include #include #include @@ -47,6 +49,7 @@ /* dump the stack of the calling core */ void rte_dump_stack(void) { +#ifdef RTE_BACKTRACE void *func[BACKTRACE_SIZE]; char **symb = NULL; int size; @@ -64,6 +67,7 @@ void rte_dump_stack(void) } free(symb); +#endif /* RTE_BACKTRACE */ } /* not implemented in this environment */ diff --git a/dpdk/lib/librte_eal/bsdapp/eal/eal_interrupts.c b/dpdk/lib/librte_eal/bsdapp/eal/eal_interrupts.c index 836e4836..deba8770 100644 --- a/dpdk/lib/librte_eal/bsdapp/eal/eal_interrupts.c +++ b/dpdk/lib/librte_eal/bsdapp/eal/eal_interrupts.c @@ -36,29 +36,37 @@ #include "eal_private.h" int -rte_intr_callback_register(struct rte_intr_handle *intr_handle __rte_unused, - rte_intr_callback_fn cb __rte_unused, - void *cb_arg __rte_unused) +rte_intr_callback_register(const struct rte_intr_handle *intr_handle, + rte_intr_callback_fn cb, + void *cb_arg) +{ + RTE_SET_USED(intr_handle); + RTE_SET_USED(cb); + RTE_SET_USED(cb_arg); + + return -ENOTSUP; +} + +int +rte_intr_callback_unregister(const struct rte_intr_handle *intr_handle, + rte_intr_callback_fn cb, + void *cb_arg) +{ + RTE_SET_USED(intr_handle); + RTE_SET_USED(cb); + RTE_SET_USED(cb_arg); + + return -ENOTSUP; +} + +int +rte_intr_enable(const struct rte_intr_handle *intr_handle __rte_unused) { return -ENOTSUP; } int -rte_intr_callback_unregister(struct rte_intr_handle *intr_handle __rte_unused, - rte_intr_callback_fn cb_fn __rte_unused, - void *cb_arg __rte_unused) -{ - return -ENOTSUP; -} - -int -rte_intr_enable(struct rte_intr_handle *intr_handle __rte_unused) -{ - return -ENOTSUP; -} - -int -rte_intr_disable(struct rte_intr_handle *intr_handle __rte_unused) +rte_intr_disable(const struct rte_intr_handle *intr_handle __rte_unused) { return -ENOTSUP; } @@ -117,3 +125,38 @@ rte_intr_cap_multiple(struct rte_intr_handle *intr_handle) RTE_SET_USED(intr_handle); return 0; } + +int +rte_epoll_wait(int epfd, struct rte_epoll_event *events, + int maxevents, int timeout) +{ + RTE_SET_USED(epfd); + RTE_SET_USED(events); + RTE_SET_USED(maxevents); + RTE_SET_USED(timeout); + + return -ENOTSUP; +} + +int +rte_epoll_ctl(int epfd, int op, int fd, struct rte_epoll_event *event) +{ + RTE_SET_USED(epfd); + RTE_SET_USED(op); + RTE_SET_USED(fd); + RTE_SET_USED(event); + + return -ENOTSUP; +} + +int +rte_intr_tls_epfd(void) +{ + return -ENOTSUP; +} + +void +rte_intr_free_epoll_fd(struct rte_intr_handle *intr_handle) +{ + RTE_SET_USED(intr_handle); +} diff --git a/dpdk/lib/librte_eal/bsdapp/eal/eal_lcore.c b/dpdk/lib/librte_eal/bsdapp/eal/eal_lcore.c index b8bfafde..bc584dd5 100644 --- a/dpdk/lib/librte_eal/bsdapp/eal/eal_lcore.c +++ b/dpdk/lib/librte_eal/bsdapp/eal/eal_lcore.c @@ -53,12 +53,14 @@ eal_cpu_core_id(__rte_unused unsigned lcore_id) static int eal_get_ncpus(void) { + static int ncpu = -1; int mib[2] = {CTL_HW, HW_NCPU}; - int ncpu; size_t len = sizeof(ncpu); - sysctl(mib, 2, &ncpu, &len, NULL, 0); - RTE_LOG(INFO, EAL, "Sysctl reports %d cpus\n", ncpu); + if (ncpu < 0) { + sysctl(mib, 2, &ncpu, &len, NULL, 0); + RTE_LOG(INFO, EAL, "Sysctl reports %d cpus\n", ncpu); + } return ncpu; } diff --git a/dpdk/lib/librte_eal/bsdapp/eal/eal_log.c b/dpdk/lib/librte_eal/bsdapp/eal/eal_log.c deleted file mode 100644 index a425f7a8..00000000 --- a/dpdk/lib/librte_eal/bsdapp/eal/eal_log.c +++ /dev/null @@ -1,57 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include -#include -#include - -#include - -/* - * set the log to default function, called during eal init process, - * once memzones are available. - */ -int -rte_eal_log_init(const char *id __rte_unused, int facility __rte_unused) -{ - if (rte_eal_common_log_init(stderr) < 0) - return -1; - return 0; -} - -int -rte_eal_log_early_init(void) -{ - rte_openlog_stream(stderr); - return 0; -} diff --git a/dpdk/lib/librte_eal/bsdapp/eal/eal_memory.c b/dpdk/lib/librte_eal/bsdapp/eal/eal_memory.c index 3614da8d..6ba05857 100644 --- a/dpdk/lib/librte_eal/bsdapp/eal/eal_memory.c +++ b/dpdk/lib/librte_eal/bsdapp/eal/eal_memory.c @@ -54,9 +54,14 @@ phys_addr_t rte_mem_virt2phy(const void *virtaddr) { /* XXX not implemented. This function is only used by - * rte_mempool_virt2phy() when hugepages are disabled. */ + * rte_mempool_virt2iova() when hugepages are disabled. */ (void)virtaddr; - return RTE_BAD_PHYS_ADDR; + return RTE_BAD_IOVA; +} +rte_iova_t +rte_mem_virt2iova(const void *virtaddr) +{ + return rte_mem_virt2phy(virtaddr); } int @@ -73,7 +78,7 @@ rte_eal_hugepage_init(void) /* for debug purposes, hugetlbfs can be disabled */ if (internal_config.no_hugetlbfs) { addr = malloc(internal_config.memory); - mcfg->memseg[0].phys_addr = (phys_addr_t)(uintptr_t)addr; + mcfg->memseg[0].iova = (rte_iova_t)(uintptr_t)addr; mcfg->memseg[0].addr = addr; mcfg->memseg[0].hugepage_sz = RTE_PGSIZE_4K; mcfg->memseg[0].len = internal_config.memory; @@ -88,7 +93,7 @@ rte_eal_hugepage_init(void) hpi = &internal_config.hugepage_info[i]; for (j = 0; j < hpi->num_pages[0]; j++) { struct rte_memseg *seg; - uint64_t physaddr; + rte_iova_t physaddr; int error; size_t sysctl_size = sizeof(physaddr); char physaddr_str[64]; @@ -114,7 +119,7 @@ rte_eal_hugepage_init(void) seg = &mcfg->memseg[seg_idx++]; seg->addr = addr; - seg->phys_addr = physaddr; + seg->iova = physaddr; seg->hugepage_sz = hpi->hugepage_sz; seg->len = hpi->hugepage_sz; seg->nchannel = mcfg->nchannel; @@ -192,3 +197,9 @@ error: close(fd_hugepage); return -1; } + +int +rte_eal_using_phys_addrs(void) +{ + return 0; +} diff --git a/dpdk/lib/librte_eal/bsdapp/eal/eal_pci.c b/dpdk/lib/librte_eal/bsdapp/eal/eal_pci.c deleted file mode 100644 index 374b68f2..00000000 --- a/dpdk/lib/librte_eal/bsdapp/eal/eal_pci.c +++ /dev/null @@ -1,638 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#if defined(RTE_ARCH_X86) -#include -#include -#endif - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "eal_filesystem.h" -#include "eal_private.h" - -/** - * @file - * PCI probing under linux - * - * This code is used to simulate a PCI probe by parsing information in - * sysfs. Moreover, when a registered driver matches a device, the - * kernel driver currently using it is unloaded and replaced by - * igb_uio module, which is a very minimal userland driver for Intel - * network card, only providing access to PCI BAR to applications, and - * enabling bus master. - */ - -/* unbind kernel driver for this device */ -int -pci_unbind_kernel_driver(struct rte_pci_device *dev __rte_unused) -{ - RTE_LOG(ERR, EAL, "RTE_PCI_DRV_FORCE_UNBIND flag is not implemented " - "for BSD\n"); - return -ENOTSUP; -} - -/* Map pci device */ -int -rte_eal_pci_map_device(struct rte_pci_device *dev) -{ - int ret = -1; - - /* try mapping the NIC resources */ - switch (dev->kdrv) { - case RTE_KDRV_NIC_UIO: - /* map resources for devices that use uio */ - ret = pci_uio_map_resource(dev); - break; - default: - RTE_LOG(DEBUG, EAL, - " Not managed by a supported kernel driver, skipped\n"); - ret = 1; - break; - } - - return ret; -} - -/* Unmap pci device */ -void -rte_eal_pci_unmap_device(struct rte_pci_device *dev) -{ - /* try unmapping the NIC resources */ - switch (dev->kdrv) { - case RTE_KDRV_NIC_UIO: - /* unmap resources for devices that use uio */ - pci_uio_unmap_resource(dev); - break; - default: - RTE_LOG(DEBUG, EAL, - " Not managed by a supported kernel driver, skipped\n"); - break; - } -} - -void -pci_uio_free_resource(struct rte_pci_device *dev, - struct mapped_pci_resource *uio_res) -{ - rte_free(uio_res); - - if (dev->intr_handle.fd) { - close(dev->intr_handle.fd); - dev->intr_handle.fd = -1; - dev->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN; - } -} - -int -pci_uio_alloc_resource(struct rte_pci_device *dev, - struct mapped_pci_resource **uio_res) -{ - char devname[PATH_MAX]; /* contains the /dev/uioX */ - struct rte_pci_addr *loc; - - loc = &dev->addr; - - snprintf(devname, sizeof(devname), "/dev/uio@pci:%u:%u:%u", - dev->addr.bus, dev->addr.devid, dev->addr.function); - - if (access(devname, O_RDWR) < 0) { - RTE_LOG(WARNING, EAL, " "PCI_PRI_FMT" not managed by UIO driver, " - "skipping\n", loc->domain, loc->bus, loc->devid, loc->function); - return 1; - } - - /* save fd if in primary process */ - dev->intr_handle.fd = open(devname, O_RDWR); - if (dev->intr_handle.fd < 0) { - RTE_LOG(ERR, EAL, "Cannot open %s: %s\n", - devname, strerror(errno)); - goto error; - } - dev->intr_handle.type = RTE_INTR_HANDLE_UIO; - - /* allocate the mapping details for secondary processes*/ - *uio_res = rte_zmalloc("UIO_RES", sizeof(**uio_res), 0); - if (*uio_res == NULL) { - RTE_LOG(ERR, EAL, - "%s(): cannot store uio mmap details\n", __func__); - goto error; - } - - snprintf((*uio_res)->path, sizeof((*uio_res)->path), "%s", devname); - memcpy(&(*uio_res)->pci_addr, &dev->addr, sizeof((*uio_res)->pci_addr)); - - return 0; - -error: - pci_uio_free_resource(dev, *uio_res); - return -1; -} - -int -pci_uio_map_resource_by_index(struct rte_pci_device *dev, int res_idx, - struct mapped_pci_resource *uio_res, int map_idx) -{ - int fd; - char *devname; - void *mapaddr; - uint64_t offset; - uint64_t pagesz; - struct pci_map *maps; - - maps = uio_res->maps; - devname = uio_res->path; - pagesz = sysconf(_SC_PAGESIZE); - - /* allocate memory to keep path */ - maps[map_idx].path = rte_malloc(NULL, strlen(devname) + 1, 0); - if (maps[map_idx].path == NULL) { - RTE_LOG(ERR, EAL, "Cannot allocate memory for path: %s\n", - strerror(errno)); - return -1; - } - - /* - * open resource file, to mmap it - */ - fd = open(devname, O_RDWR); - if (fd < 0) { - RTE_LOG(ERR, EAL, "Cannot open %s: %s\n", - devname, strerror(errno)); - goto error; - } - - /* if matching map is found, then use it */ - offset = res_idx * pagesz; - mapaddr = pci_map_resource(NULL, fd, (off_t)offset, - (size_t)dev->mem_resource[res_idx].len, 0); - close(fd); - if (mapaddr == MAP_FAILED) - goto error; - - maps[map_idx].phaddr = dev->mem_resource[res_idx].phys_addr; - maps[map_idx].size = dev->mem_resource[res_idx].len; - maps[map_idx].addr = mapaddr; - maps[map_idx].offset = offset; - strcpy(maps[map_idx].path, devname); - dev->mem_resource[res_idx].addr = mapaddr; - - return 0; - -error: - rte_free(maps[map_idx].path); - return -1; -} - -static int -pci_scan_one(int dev_pci_fd, struct pci_conf *conf) -{ - struct rte_pci_device *dev; - struct pci_bar_io bar; - unsigned i, max; - - dev = malloc(sizeof(*dev)); - if (dev == NULL) { - return -1; - } - - memset(dev, 0, sizeof(*dev)); - dev->addr.domain = conf->pc_sel.pc_domain; - dev->addr.bus = conf->pc_sel.pc_bus; - dev->addr.devid = conf->pc_sel.pc_dev; - dev->addr.function = conf->pc_sel.pc_func; - - /* get vendor id */ - dev->id.vendor_id = conf->pc_vendor; - - /* get device id */ - dev->id.device_id = conf->pc_device; - - /* get subsystem_vendor id */ - dev->id.subsystem_vendor_id = conf->pc_subvendor; - - /* get subsystem_device id */ - dev->id.subsystem_device_id = conf->pc_subdevice; - - /* get class id */ - dev->id.class_id = (conf->pc_class << 16) | - (conf->pc_subclass << 8) | - (conf->pc_progif); - - /* TODO: get max_vfs */ - dev->max_vfs = 0; - - /* FreeBSD has no NUMA support (yet) */ - dev->numa_node = 0; - - /* FreeBSD has only one pass through driver */ - dev->kdrv = RTE_KDRV_NIC_UIO; - - /* parse resources */ - switch (conf->pc_hdr & PCIM_HDRTYPE) { - case PCIM_HDRTYPE_NORMAL: - max = PCIR_MAX_BAR_0; - break; - case PCIM_HDRTYPE_BRIDGE: - max = PCIR_MAX_BAR_1; - break; - case PCIM_HDRTYPE_CARDBUS: - max = PCIR_MAX_BAR_2; - break; - default: - goto skipdev; - } - - for (i = 0; i <= max; i++) { - bar.pbi_sel = conf->pc_sel; - bar.pbi_reg = PCIR_BAR(i); - if (ioctl(dev_pci_fd, PCIOCGETBAR, &bar) < 0) - continue; - - dev->mem_resource[i].len = bar.pbi_length; - if (PCI_BAR_IO(bar.pbi_base)) { - dev->mem_resource[i].addr = (void *)(bar.pbi_base & ~((uint64_t)0xf)); - continue; - } - dev->mem_resource[i].phys_addr = bar.pbi_base & ~((uint64_t)0xf); - } - - /* device is valid, add in list (sorted) */ - if (TAILQ_EMPTY(&pci_device_list)) { - TAILQ_INSERT_TAIL(&pci_device_list, dev, next); - } - else { - struct rte_pci_device *dev2 = NULL; - int ret; - - TAILQ_FOREACH(dev2, &pci_device_list, next) { - ret = rte_eal_compare_pci_addr(&dev->addr, &dev2->addr); - if (ret > 0) - continue; - else if (ret < 0) { - TAILQ_INSERT_BEFORE(dev2, dev, next); - return 0; - } else { /* already registered */ - dev2->kdrv = dev->kdrv; - dev2->max_vfs = dev->max_vfs; - memmove(dev2->mem_resource, - dev->mem_resource, - sizeof(dev->mem_resource)); - free(dev); - return 0; - } - } - TAILQ_INSERT_TAIL(&pci_device_list, dev, next); - } - - return 0; - -skipdev: - free(dev); - return 0; -} - -/* - * Scan the content of the PCI bus, and add the devices in the devices - * list. Call pci_scan_one() for each pci entry found. - */ -int -rte_eal_pci_scan(void) -{ - int fd; - unsigned dev_count = 0; - struct pci_conf matches[16]; - struct pci_conf_io conf_io = { - .pat_buf_len = 0, - .num_patterns = 0, - .patterns = NULL, - .match_buf_len = sizeof(matches), - .matches = &matches[0], - }; - - fd = open("/dev/pci", O_RDONLY); - if (fd < 0) { - RTE_LOG(ERR, EAL, "%s(): error opening /dev/pci\n", __func__); - goto error; - } - - do { - unsigned i; - if (ioctl(fd, PCIOCGETCONF, &conf_io) < 0) { - RTE_LOG(ERR, EAL, "%s(): error with ioctl on /dev/pci: %s\n", - __func__, strerror(errno)); - goto error; - } - - for (i = 0; i < conf_io.num_matches; i++) - if (pci_scan_one(fd, &matches[i]) < 0) - goto error; - - dev_count += conf_io.num_matches; - } while(conf_io.status == PCI_GETCONF_MORE_DEVS); - - close(fd); - - RTE_LOG(ERR, EAL, "PCI scan found %u devices\n", dev_count); - return 0; - -error: - if (fd >= 0) - close(fd); - return -1; -} - -/* Read PCI config space. */ -int rte_eal_pci_read_config(const struct rte_pci_device *dev, - void *buf, size_t len, off_t offset) -{ - int fd = -1; - struct pci_io pi = { - .pi_sel = { - .pc_domain = dev->addr.domain, - .pc_bus = dev->addr.bus, - .pc_dev = dev->addr.devid, - .pc_func = dev->addr.function, - }, - .pi_reg = offset, - .pi_width = len, - }; - - if (len == 3 || len > sizeof(pi.pi_data)) { - RTE_LOG(ERR, EAL, "%s(): invalid pci read length\n", __func__); - goto error; - } - - fd = open("/dev/pci", O_RDWR); - if (fd < 0) { - RTE_LOG(ERR, EAL, "%s(): error opening /dev/pci\n", __func__); - goto error; - } - - if (ioctl(fd, PCIOCREAD, &pi) < 0) - goto error; - close(fd); - - memcpy(buf, &pi.pi_data, len); - return 0; - - error: - if (fd >= 0) - close(fd); - return -1; -} - -/* Write PCI config space. */ -int rte_eal_pci_write_config(const struct rte_pci_device *dev, - const void *buf, size_t len, off_t offset) -{ - int fd = -1; - - struct pci_io pi = { - .pi_sel = { - .pc_domain = dev->addr.domain, - .pc_bus = dev->addr.bus, - .pc_dev = dev->addr.devid, - .pc_func = dev->addr.function, - }, - .pi_reg = offset, - .pi_data = *(const uint32_t *)buf, - .pi_width = len, - }; - - if (len == 3 || len > sizeof(pi.pi_data)) { - RTE_LOG(ERR, EAL, "%s(): invalid pci read length\n", __func__); - goto error; - } - - memcpy(&pi.pi_data, buf, len); - - fd = open("/dev/pci", O_RDWR); - if (fd < 0) { - RTE_LOG(ERR, EAL, "%s(): error opening /dev/pci\n", __func__); - goto error; - } - - if (ioctl(fd, PCIOCWRITE, &pi) < 0) - goto error; - - close(fd); - return 0; - - error: - if (fd >= 0) - close(fd); - return -1; -} - -int -rte_eal_pci_ioport_map(struct rte_pci_device *dev, int bar, - struct rte_pci_ioport *p) -{ - int ret; - - switch (dev->kdrv) { -#if defined(RTE_ARCH_X86) - case RTE_KDRV_NIC_UIO: - if ((uintptr_t) dev->mem_resource[bar].addr <= UINT16_MAX) { - p->base = (uintptr_t)dev->mem_resource[bar].addr; - ret = 0; - } else - ret = -1; - break; -#endif - default: - ret = -1; - break; - } - - if (!ret) - p->dev = dev; - - return ret; -} - -static void -pci_uio_ioport_read(struct rte_pci_ioport *p, - void *data, size_t len, off_t offset) -{ -#if defined(RTE_ARCH_X86) - uint8_t *d; - int size; - unsigned short reg = p->base + offset; - - for (d = data; len > 0; d += size, reg += size, len -= size) { - if (len >= 4) { - size = 4; - *(uint32_t *)d = inl(reg); - } else if (len >= 2) { - size = 2; - *(uint16_t *)d = inw(reg); - } else { - size = 1; - *d = inb(reg); - } - } -#else - RTE_SET_USED(p); - RTE_SET_USED(data); - RTE_SET_USED(len); - RTE_SET_USED(offset); -#endif -} - -void -rte_eal_pci_ioport_read(struct rte_pci_ioport *p, - void *data, size_t len, off_t offset) -{ - switch (p->dev->kdrv) { - case RTE_KDRV_NIC_UIO: - pci_uio_ioport_read(p, data, len, offset); - break; - default: - break; - } -} - -static void -pci_uio_ioport_write(struct rte_pci_ioport *p, - const void *data, size_t len, off_t offset) -{ -#if defined(RTE_ARCH_X86) - const uint8_t *s; - int size; - unsigned short reg = p->base + offset; - - for (s = data; len > 0; s += size, reg += size, len -= size) { - if (len >= 4) { - size = 4; - outl(*(const uint32_t *)s, reg); - } else if (len >= 2) { - size = 2; - outw(*(const uint16_t *)s, reg); - } else { - size = 1; - outb(*s, reg); - } - } -#else - RTE_SET_USED(p); - RTE_SET_USED(data); - RTE_SET_USED(len); - RTE_SET_USED(offset); -#endif -} - -void -rte_eal_pci_ioport_write(struct rte_pci_ioport *p, - const void *data, size_t len, off_t offset) -{ - switch (p->dev->kdrv) { - case RTE_KDRV_NIC_UIO: - pci_uio_ioport_write(p, data, len, offset); - break; - default: - break; - } -} - -int -rte_eal_pci_ioport_unmap(struct rte_pci_ioport *p) -{ - int ret; - - switch (p->dev->kdrv) { -#if defined(RTE_ARCH_X86) - case RTE_KDRV_NIC_UIO: - ret = 0; - break; -#endif - default: - ret = -1; - break; - } - - return ret; -} - -/* Init the PCI EAL subsystem */ -int -rte_eal_pci_init(void) -{ - TAILQ_INIT(&pci_driver_list); - TAILQ_INIT(&pci_device_list); - - /* for debug purposes, PCI can be disabled */ - if (internal_config.no_pci) - return 0; - - if (rte_eal_pci_scan() < 0) { - RTE_LOG(ERR, EAL, "%s(): Cannot scan PCI bus\n", __func__); - return -1; - } - return 0; -} diff --git a/dpdk/lib/librte_eal/bsdapp/eal/eal_thread.c b/dpdk/lib/librte_eal/bsdapp/eal/eal_thread.c index 1b8cd8a6..2a2136a2 100644 --- a/dpdk/lib/librte_eal/bsdapp/eal/eal_thread.c +++ b/dpdk/lib/librte_eal/bsdapp/eal/eal_thread.c @@ -46,10 +46,8 @@ #include #include #include -#include #include #include -#include #include #include "eal_private.h" diff --git a/dpdk/lib/librte_eal/bsdapp/eal/eal_timer.c b/dpdk/lib/librte_eal/bsdapp/eal/eal_timer.c index f12d9bd2..14421943 100644 --- a/dpdk/lib/librte_eal/bsdapp/eal/eal_timer.c +++ b/dpdk/lib/librte_eal/bsdapp/eal/eal_timer.c @@ -42,7 +42,6 @@ #include #include #include -#include #include #include diff --git a/dpdk/lib/librte_eal/bsdapp/eal/include/exec-env/rte_dom0_common.h b/dpdk/lib/librte_eal/bsdapp/eal/include/exec-env/rte_dom0_common.h deleted file mode 100644 index 99a33432..00000000 --- a/dpdk/lib/librte_eal/bsdapp/eal/include/exec-env/rte_dom0_common.h +++ /dev/null @@ -1,107 +0,0 @@ -/*- - * This file is provided under a dual BSD/LGPLv2 license. When using or - * redistributing this file, you may do so under either license. - * - * GNU LESSER GENERAL PUBLIC LICENSE - * - * Copyright(c) 2007-2014 Intel Corporation. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of version 2.1 of the GNU Lesser General Public License - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. - * - * Contact Information: - * Intel Corporation - * - * - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -#ifndef _RTE_DOM0_COMMON_H_ -#define _RTE_DOM0_COMMON_H_ - -#ifdef __KERNEL__ -#include -#endif - -#define DOM0_NAME_MAX 256 -#define DOM0_MM_DEV "/dev/dom0_mm" - -#define DOM0_CONTIG_NUM_ORDER 9 /**< 2M order */ -#define DOM0_NUM_MEMSEG 512 /**< Maximum nb. of memory segment. */ -#define DOM0_MEMBLOCK_SIZE 0x200000 /**< Maximum nb. of memory block(2M). */ -#define DOM0_CONFIG_MEMSIZE 4096 /**< Maximum config memory size(4G). */ -#define DOM0_NUM_MEMBLOCK (DOM0_CONFIG_MEMSIZE / 2) /**< Maximum nb. of 2M memory block. */ - -#define RTE_DOM0_IOCTL_PREPARE_MEMSEG _IOWR(0, 1 , struct memory_info) -#define RTE_DOM0_IOCTL_ATTACH_TO_MEMSEG _IOWR(0, 2 , char *) -#define RTE_DOM0_IOCTL_GET_NUM_MEMSEG _IOWR(0, 3, int) -#define RTE_DOM0_IOCTL_GET_MEMSEG_INFO _IOWR(0, 4, void *) - -/** - * A structure used to store memory information. - */ -struct memory_info { - char name[DOM0_NAME_MAX]; - uint64_t size; -}; - -/** - * A structure used to store memory segment information. - */ -struct memseg_info { - uint32_t idx; - uint64_t pfn; - uint64_t size; - uint64_t mfn[DOM0_NUM_MEMBLOCK]; -}; - -/** - * A structure used to store memory block information. - */ -struct memblock_info { - uint8_t exchange_flag; - uint64_t vir_addr; - uint64_t pfn; - uint64_t mfn; -}; -#endif /* _RTE_DOM0_COMMON_H_ */ diff --git a/dpdk/lib/librte_eal/bsdapp/eal/include/exec-env/rte_interrupts.h b/dpdk/lib/librte_eal/bsdapp/eal/include/exec-env/rte_interrupts.h deleted file mode 100644 index c1995ee1..00000000 --- a/dpdk/lib/librte_eal/bsdapp/eal/include/exec-env/rte_interrupts.h +++ /dev/null @@ -1,137 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef _RTE_INTERRUPTS_H_ -#error "don't include this file directly, please include generic " -#endif - -#ifndef _RTE_BSDAPP_INTERRUPTS_H_ -#define _RTE_BSDAPP_INTERRUPTS_H_ - -#define RTE_INTR_VEC_ZERO_OFFSET 0 -#define RTE_INTR_VEC_RXTX_OFFSET 1 - -#define RTE_MAX_RXTX_INTR_VEC_ID 32 - -enum rte_intr_handle_type { - RTE_INTR_HANDLE_UNKNOWN = 0, - RTE_INTR_HANDLE_UIO, /**< uio device handle */ - RTE_INTR_HANDLE_ALARM, /**< alarm handle */ - RTE_INTR_HANDLE_MAX -}; - -/** Handle for interrupts. */ -struct rte_intr_handle { - int fd; /**< file descriptor */ - int uio_cfg_fd; /**< UIO config file descriptor */ - enum rte_intr_handle_type type; /**< handle type */ - int max_intr; /**< max interrupt requested */ - uint32_t nb_efd; /**< number of available efds */ - int *intr_vec; /**< intr vector number array */ -}; - -/** - * @param intr_handle - * Pointer to the interrupt handle. - * @param epfd - * Epoll instance fd which the intr vector associated to. - * @param op - * The operation be performed for the vector. - * Operation type of {ADD, DEL}. - * @param vec - * RX intr vector number added to the epoll instance wait list. - * @param data - * User raw data. - * @return - * - On success, zero. - * - On failure, a negative value. - */ -int -rte_intr_rx_ctl(struct rte_intr_handle *intr_handle, - int epfd, int op, unsigned int vec, void *data); - -/** - * It enables the fastpath event fds if it's necessary. - * It creates event fds when multi-vectors allowed, - * otherwise it multiplexes the single event fds. - * - * @param intr_handle - * Pointer to the interrupt handle. - * @param nb_efd - * Number of interrupt vector trying to enable. - * The value 0 is not allowed. - * @return - * - On success, zero. - * - On failure, a negative value. - */ -int -rte_intr_efd_enable(struct rte_intr_handle *intr_handle, uint32_t nb_efd); - -/** - * It disable the fastpath event fds. - * It deletes registered eventfds and closes the open fds. - * - * @param intr_handle - * Pointer to the interrupt handle. - */ -void -rte_intr_efd_disable(struct rte_intr_handle *intr_handle); - -/** - * The fastpath interrupt is enabled or not. - * - * @param intr_handle - * Pointer to the interrupt handle. - */ -int rte_intr_dp_is_en(struct rte_intr_handle *intr_handle); - -/** - * The interrupt handle instance allows other cause or not. - * Other cause stands for none fastpath interrupt. - * - * @param intr_handle - * Pointer to the interrupt handle. - */ -int rte_intr_allow_others(struct rte_intr_handle *intr_handle); - -/** - * The multiple interrupt vector capability of interrupt handle instance. - * It returns zero if no multiple interrupt vector support. - * - * @param intr_handle - * Pointer to the interrupt handle. - */ -int -rte_intr_cap_multiple(struct rte_intr_handle *intr_handle); - -#endif /* _RTE_BSDAPP_INTERRUPTS_H_ */ diff --git a/dpdk/lib/librte_eal/bsdapp/eal/rte_eal_version.map b/dpdk/lib/librte_eal/bsdapp/eal/rte_eal_version.map deleted file mode 100644 index a335e04b..00000000 --- a/dpdk/lib/librte_eal/bsdapp/eal/rte_eal_version.map +++ /dev/null @@ -1,164 +0,0 @@ -DPDK_2.0 { - global: - - __rte_panic; - devargs_list; - eal_parse_sysfs_value; - eal_timer_source; - lcore_config; - pci_device_list; - pci_driver_list; - per_lcore__lcore_id; - per_lcore__rte_errno; - rte_calloc; - rte_calloc_socket; - rte_cpu_check_supported; - rte_cpu_get_flag_enabled; - rte_cycles_vmware_tsc_map; - rte_delay_us; - rte_dump_physmem_layout; - rte_dump_registers; - rte_dump_stack; - rte_dump_tailq; - rte_eal_alarm_cancel; - rte_eal_alarm_set; - rte_eal_dev_init; - rte_eal_devargs_add; - rte_eal_devargs_dump; - rte_eal_devargs_type_count; - rte_eal_driver_register; - rte_eal_driver_unregister; - rte_eal_get_configuration; - rte_eal_get_lcore_state; - rte_eal_get_physmem_layout; - rte_eal_get_physmem_size; - rte_eal_has_hugepages; - rte_eal_hpet_init; - rte_eal_init; - rte_eal_iopl_init; - rte_eal_lcore_role; - rte_eal_mp_remote_launch; - rte_eal_mp_wait_lcore; - rte_eal_parse_devargs_str; - rte_eal_pci_dump; - rte_eal_pci_probe; - rte_eal_pci_probe_one; - rte_eal_pci_register; - rte_eal_pci_scan; - rte_eal_pci_unregister; - rte_eal_process_type; - rte_eal_remote_launch; - rte_eal_tailq_lookup; - rte_eal_tailq_register; - rte_eal_vdev_init; - rte_eal_vdev_uninit; - rte_eal_wait_lcore; - rte_exit; - rte_free; - rte_get_hpet_cycles; - rte_get_hpet_hz; - rte_get_log_level; - rte_get_log_type; - rte_get_tsc_hz; - rte_hexdump; - rte_intr_callback_register; - rte_intr_callback_unregister; - rte_intr_disable; - rte_intr_enable; - rte_log; - rte_log_add_in_history; - rte_log_cur_msg_loglevel; - rte_log_cur_msg_logtype; - rte_log_dump_history; - rte_log_set_history; - rte_logs; - rte_malloc; - rte_malloc_dump_stats; - rte_malloc_get_socket_stats; - rte_malloc_set_limit; - rte_malloc_socket; - rte_malloc_validate; - rte_malloc_virt2phy; - rte_mem_lock_page; - rte_mem_phy2mch; - rte_mem_virt2phy; - rte_memdump; - rte_memory_get_nchannel; - rte_memory_get_nrank; - rte_memzone_dump; - rte_memzone_lookup; - rte_memzone_reserve; - rte_memzone_reserve_aligned; - rte_memzone_reserve_bounded; - rte_memzone_walk; - rte_openlog_stream; - rte_realloc; - rte_set_application_usage_hook; - rte_set_log_level; - rte_set_log_type; - rte_socket_id; - rte_strerror; - rte_strsplit; - rte_sys_gettid; - rte_thread_get_affinity; - rte_thread_set_affinity; - rte_vlog; - rte_xen_dom0_memory_attach; - rte_xen_dom0_memory_init; - rte_zmalloc; - rte_zmalloc_socket; - - local: *; -}; - -DPDK_2.1 { - global: - - rte_eal_pci_detach; - rte_eal_pci_read_config; - rte_eal_pci_write_config; - rte_intr_allow_others; - rte_intr_dp_is_en; - rte_intr_efd_disable; - rte_intr_efd_enable; - rte_intr_rx_ctl; - rte_memzone_free; - -} DPDK_2.0; - -DPDK_2.2 { - global: - - rte_intr_cap_multiple; - rte_keepalive_create; - rte_keepalive_dispatch_pings; - rte_keepalive_mark_alive; - rte_keepalive_register_core; - rte_xen_dom0_supported; - -} DPDK_2.1; - -DPDK_16.04 { - global: - - rte_cpu_get_flag_name; - rte_eal_pci_ioport_map; - rte_eal_pci_ioport_read; - rte_eal_pci_ioport_unmap; - rte_eal_pci_ioport_write; - rte_eal_pci_map_device; - rte_eal_pci_unmap_device; - rte_eal_primary_proc_alive; - -} DPDK_2.2; - -DPDK_16.07 { - global: - - pci_get_sysfs_path; - rte_keepalive_mark_sleep; - rte_keepalive_register_relay_callback; - rte_rtm_supported; - rte_thread_setname; - -} DPDK_16.04; diff --git a/dpdk/lib/librte_eal/bsdapp/nic_uio/nic_uio.c b/dpdk/lib/librte_eal/bsdapp/nic_uio/nic_uio.c index 99a4975c..4bd7545a 100644 --- a/dpdk/lib/librte_eal/bsdapp/nic_uio/nic_uio.c +++ b/dpdk/lib/librte_eal/bsdapp/nic_uio/nic_uio.c @@ -180,6 +180,10 @@ nic_uio_probe (device_t dev) unsigned int device = pci_get_slot(dev); unsigned int function = pci_get_function(dev); + char bdf_str[256]; + char *token, *remaining; + + /* First check if we found this on load */ for (i = 0; i < num_detached; i++) if (bus == pci_get_bus(detached_devices[i]) && device == pci_get_slot(detached_devices[i]) && @@ -188,6 +192,45 @@ nic_uio_probe (device_t dev) return BUS_PROBE_SPECIFIC; } + /* otherwise check if it's a new device and if it matches the BDF */ + memset(bdf_str, 0, sizeof(bdf_str)); + TUNABLE_STR_FETCH("hw.nic_uio.bdfs", bdf_str, sizeof(bdf_str)); + remaining = bdf_str; + while (1) { + if (remaining == NULL || remaining[0] == '\0') + break; + token = strsep(&remaining, ",:"); + if (token == NULL) + break; + bus = strtol(token, NULL, 10); + token = strsep(&remaining, ",:"); + if (token == NULL) + break; + device = strtol(token, NULL, 10); + token = strsep(&remaining, ",:"); + if (token == NULL) + break; + function = strtol(token, NULL, 10); + + if (bus == pci_get_bus(dev) && + device == pci_get_slot(dev) && + function == pci_get_function(dev)) { + + if (num_detached < MAX_DETACHED_DEVICES) { + printf("%s: probed dev=%p\n", + __func__, dev); + detached_devices[num_detached++] = dev; + device_set_desc(dev, "DPDK PCI Device"); + return BUS_PROBE_SPECIFIC; + } else { + printf("%s: reached MAX_DETACHED_DEVICES=%d. dev=%p won't be reattached\n", + __func__, MAX_DETACHED_DEVICES, + dev); + break; + } + } + } + return ENXIO; } @@ -248,6 +291,7 @@ nic_uio_load(void) memset(bdf_str, 0, sizeof(bdf_str)); TUNABLE_STR_FETCH("hw.nic_uio.bdfs", bdf_str, sizeof(bdf_str)); remaining = bdf_str; + printf("nic_uio: hw.nic_uio.bdfs = '%s'\n", bdf_str); /* * Users should specify PCI BDFs in the format "b:d:f,b:d:f,b:d:f". * But the code below does not try differentiate between : and , diff --git a/dpdk/lib/librte_eal/common/Makefile b/dpdk/lib/librte_eal/common/Makefile index f5ea0ee8..9effd0d4 100644 --- a/dpdk/lib/librte_eal/common/Makefile +++ b/dpdk/lib/librte_eal/common/Makefile @@ -32,25 +32,26 @@ include $(RTE_SDK)/mk/rte.vars.mk INC := rte_branch_prediction.h rte_common.h -INC += rte_debug.h rte_eal.h rte_errno.h rte_launch.h rte_lcore.h -INC += rte_log.h rte_memory.h rte_memzone.h rte_pci.h -INC += rte_pci_dev_ids.h rte_per_lcore.h rte_random.h +INC += rte_debug.h rte_eal.h rte_eal_interrupts.h +INC += rte_errno.h rte_launch.h rte_lcore.h +INC += rte_log.h rte_memory.h rte_memzone.h +INC += rte_per_lcore.h rte_random.h INC += rte_tailq.h rte_interrupts.h rte_alarm.h INC += rte_string_fns.h rte_version.h INC += rte_eal_memconfig.h rte_malloc_heap.h -INC += rte_hexdump.h rte_devargs.h rte_dev.h +INC += rte_hexdump.h rte_devargs.h rte_bus.h rte_dev.h INC += rte_pci_dev_feature_defs.h rte_pci_dev_features.h INC += rte_malloc.h rte_keepalive.h rte_time.h - -ifeq ($(CONFIG_RTE_INSECURE_FUNCTION_WARNING),y) -INC += rte_warnings.h -endif +INC += rte_service.h rte_service_component.h +INC += rte_bitmap.h rte_vfio.h GENERIC_INC := rte_atomic.h rte_byteorder.h rte_cycles.h rte_prefetch.h GENERIC_INC += rte_spinlock.h rte_memcpy.h rte_cpuflags.h rte_rwlock.h +GENERIC_INC += rte_vect.h rte_pause.h rte_io.h + # defined in mk/arch/$(RTE_ARCH)/rte.vars.mk ARCH_DIR ?= $(RTE_ARCH) -ARCH_INC := $(notdir $(wildcard $(RTE_SDK)/lib/librte_eal/common/include/arch/$(ARCH_DIR)/*.h)) +ARCH_INC := $(sort $(notdir $(wildcard $(RTE_SDK)/lib/librte_eal/common/include/arch/$(ARCH_DIR)/*.h))) SYMLINK-$(CONFIG_RTE_LIBRTE_EAL)-include := $(addprefix include/,$(INC)) SYMLINK-$(CONFIG_RTE_LIBRTE_EAL)-include += \ diff --git a/dpdk/lib/librte_eal/common/arch/arm/rte_cpuflags.c b/dpdk/lib/librte_eal/common/arch/arm/rte_cpuflags.c index 79160a60..88f1cbe3 100644 --- a/dpdk/lib/librte_eal/common/arch/arm/rte_cpuflags.c +++ b/dpdk/lib/librte_eal/common/arch/arm/rte_cpuflags.c @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2015. + * Copyright (C) Cavium, Inc. 2015. * Copyright(c) 2015 RehiveTech. All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -14,7 +14,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * @@ -137,7 +137,7 @@ rte_cpu_get_features(hwcap_registers_t out) _Elfx_auxv_t auxv; auxv_fd = open("/proc/self/auxv", O_RDONLY); - assert(auxv_fd); + assert(auxv_fd != -1); while (read(auxv_fd, &auxv, sizeof(auxv)) == sizeof(auxv)) { if (auxv.a_type == AT_HWCAP) { out[REG_HWCAP] = auxv.a_un.a_val; diff --git a/dpdk/lib/librte_eal/common/arch/arm/rte_cycles.c b/dpdk/lib/librte_eal/common/arch/arm/rte_cycles.c new file mode 100644 index 00000000..3e31e5be --- /dev/null +++ b/dpdk/lib/librte_eal/common/arch/arm/rte_cycles.c @@ -0,0 +1,45 @@ +/* + * BSD LICENSE + * + * Copyright (C) Cavium, Inc. 2015. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "eal_private.h" + +uint64_t +get_tsc_freq_arch(void) +{ +#if defined RTE_ARCH_ARM64 && !defined RTE_ARM_EAL_RDTSC_USE_PMU + uint64_t freq; + asm volatile("mrs %0, cntfrq_el0" : "=r" (freq)); + return freq; +#else + return 0; +#endif +} diff --git a/dpdk/lib/librte_eal/common/arch/ppc_64/rte_cpuflags.c b/dpdk/lib/librte_eal/common/arch/ppc_64/rte_cpuflags.c index fcf96e04..970a61c5 100644 --- a/dpdk/lib/librte_eal/common/arch/ppc_64/rte_cpuflags.c +++ b/dpdk/lib/librte_eal/common/arch/ppc_64/rte_cpuflags.c @@ -108,7 +108,7 @@ rte_cpu_get_features(hwcap_registers_t out) Elf64_auxv_t auxv; auxv_fd = open("/proc/self/auxv", O_RDONLY); - assert(auxv_fd); + assert(auxv_fd != -1); while (read(auxv_fd, &auxv, sizeof(Elf64_auxv_t)) == sizeof(Elf64_auxv_t)) { if (auxv.a_type == AT_HWCAP) diff --git a/dpdk/lib/librte_eal/common/arch/ppc_64/rte_cycles.c b/dpdk/lib/librte_eal/common/arch/ppc_64/rte_cycles.c new file mode 100644 index 00000000..851fd025 --- /dev/null +++ b/dpdk/lib/librte_eal/common/arch/ppc_64/rte_cycles.c @@ -0,0 +1,7 @@ +#include "eal_private.h" + +uint64_t +get_tsc_freq_arch(void) +{ + return 0; +} diff --git a/dpdk/lib/librte_eal/common/arch/tile/rte_cpuflags.c b/dpdk/lib/librte_eal/common/arch/tile/rte_cpuflags.c deleted file mode 100644 index a2b6c51a..00000000 --- a/dpdk/lib/librte_eal/common/arch/tile/rte_cpuflags.c +++ /dev/null @@ -1,47 +0,0 @@ -/* - * BSD LICENSE - * - * Copyright (C) EZchip Semiconductor Ltd. 2015. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of EZchip Semiconductor nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - -#include "rte_cpuflags.h" - -#include - -const struct feature_entry rte_cpu_feature_table[] = { -}; - -/* - * Checks if a particular flag is available on current machine. - */ -int -rte_cpu_get_flag_enabled(__attribute__((unused)) enum rte_cpu_flag_t feature) -{ - return -ENOENT; -} diff --git a/dpdk/lib/librte_eal/common/arch/x86/rte_cpuflags.c b/dpdk/lib/librte_eal/common/arch/x86/rte_cpuflags.c index 01382571..7d4a0fef 100644 --- a/dpdk/lib/librte_eal/common/arch/x86/rte_cpuflags.c +++ b/dpdk/lib/librte_eal/common/arch/x86/rte_cpuflags.c @@ -36,6 +36,7 @@ #include #include #include +#include enum cpu_register_t { RTE_REG_EAX = 0, @@ -156,38 +157,12 @@ const struct feature_entry rte_cpu_feature_table[] = { FEAT_DEF(INVTSC, 0x80000007, 0, RTE_REG_EDX, 8) }; -/* - * Execute CPUID instruction and get contents of a specific register - * - * This function, when compiled with GCC, will generate architecture-neutral - * code, as per GCC manual. - */ -static void -rte_cpu_get_features(uint32_t leaf, uint32_t subleaf, cpuid_registers_t out) -{ -#if defined(__i386__) && defined(__PIC__) - /* %ebx is a forbidden register if we compile with -fPIC or -fPIE */ - asm volatile("movl %%ebx,%0 ; cpuid ; xchgl %%ebx,%0" - : "=r" (out[RTE_REG_EBX]), - "=a" (out[RTE_REG_EAX]), - "=c" (out[RTE_REG_ECX]), - "=d" (out[RTE_REG_EDX]) - : "a" (leaf), "c" (subleaf)); -#else - asm volatile("cpuid" - : "=a" (out[RTE_REG_EAX]), - "=b" (out[RTE_REG_EBX]), - "=c" (out[RTE_REG_ECX]), - "=d" (out[RTE_REG_EDX]) - : "a" (leaf), "c" (subleaf)); -#endif -} - int rte_cpu_get_flag_enabled(enum rte_cpu_flag_t feature) { const struct feature_entry *feat; cpuid_registers_t regs; + unsigned int maxleaf; if (feature >= RTE_CPUFLAG_NUMFLAGS) /* Flag does not match anything in the feature tables */ @@ -199,13 +174,14 @@ rte_cpu_get_flag_enabled(enum rte_cpu_flag_t feature) /* This entry in the table wasn't filled out! */ return -EFAULT; - rte_cpu_get_features(feat->leaf & 0xffff0000, 0, regs); - if (((regs[RTE_REG_EAX] ^ feat->leaf) & 0xffff0000) || - regs[RTE_REG_EAX] < feat->leaf) + maxleaf = __get_cpuid_max(feat->leaf & 0x80000000, NULL); + + if (maxleaf < feat->leaf) return 0; - /* get the cpuid leaf containing the desired feature */ - rte_cpu_get_features(feat->leaf, feat->subleaf, regs); + __cpuid_count(feat->leaf, feat->subleaf, + regs[RTE_REG_EAX], regs[RTE_REG_EBX], + regs[RTE_REG_ECX], regs[RTE_REG_EDX]); /* check if the feature is enabled */ return (regs[feat->reg] >> feat->bit) & 1; diff --git a/dpdk/lib/librte_eal/common/arch/x86/rte_cycles.c b/dpdk/lib/librte_eal/common/arch/x86/rte_cycles.c new file mode 100644 index 00000000..417850ee --- /dev/null +++ b/dpdk/lib/librte_eal/common/arch/x86/rte_cycles.c @@ -0,0 +1,152 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include + +#include + +#include "eal_private.h" + +static unsigned int +rte_cpu_get_model(uint32_t fam_mod_step) +{ + uint32_t family, model, ext_model; + + family = (fam_mod_step >> 8) & 0xf; + model = (fam_mod_step >> 4) & 0xf; + + if (family == 6 || family == 15) { + ext_model = (fam_mod_step >> 16) & 0xf; + model += (ext_model << 4); + } + + return model; +} + +static int32_t +rdmsr(int msr, uint64_t *val) +{ +#ifdef RTE_EXEC_ENV_LINUXAPP + int fd; + int ret; + + fd = open("/dev/cpu/0/msr", O_RDONLY); + if (fd < 0) + return fd; + + ret = pread(fd, val, sizeof(uint64_t), msr); + + close(fd); + + return ret; +#else + RTE_SET_USED(msr); + RTE_SET_USED(val); + + return -1; +#endif +} + +static uint32_t +check_model_wsm_nhm(uint8_t model) +{ + switch (model) { + /* Westmere */ + case 0x25: + case 0x2C: + case 0x2F: + /* Nehalem */ + case 0x1E: + case 0x1F: + case 0x1A: + case 0x2E: + return 1; + } + + return 0; +} + +static uint32_t +check_model_gdm_dnv(uint8_t model) +{ + switch (model) { + /* Goldmont */ + case 0x5C: + /* Denverton */ + case 0x5F: + return 1; + } + + return 0; +} + +uint64_t +get_tsc_freq_arch(void) +{ + uint64_t tsc_hz = 0; + uint32_t a, b, c, d, maxleaf; + uint8_t mult, model; + int32_t ret; + + /* + * Time Stamp Counter and Nominal Core Crystal Clock + * Information Leaf + */ + maxleaf = __get_cpuid_max(0, NULL); + + if (maxleaf >= 0x15) { + __cpuid(0x15, a, b, c, d); + + /* EBX : TSC/Crystal ratio, ECX : Crystal Hz */ + if (b && c) + return c * (b / a); + } + + __cpuid(0x1, a, b, c, d); + model = rte_cpu_get_model(a); + + if (check_model_wsm_nhm(model)) + mult = 133; + else if ((c & bit_AVX) || check_model_gdm_dnv(model)) + mult = 100; + else + return 0; + + ret = rdmsr(0xCE, &tsc_hz); + if (ret < 0) + return 0; + + return ((tsc_hz >> 8) & 0xff) * mult * 1E6; +} diff --git a/dpdk/lib/librte_eal/common/arch/x86/rte_memcpy.c b/dpdk/lib/librte_eal/common/arch/x86/rte_memcpy.c new file mode 100644 index 00000000..174bef15 --- /dev/null +++ b/dpdk/lib/librte_eal/common/arch/x86/rte_memcpy.c @@ -0,0 +1,58 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include + +void *(*rte_memcpy_ptr)(void *dst, const void *src, size_t n) = NULL; + +RTE_INIT(rte_memcpy_init) +{ +#ifdef CC_SUPPORT_AVX512F + if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F)) { + rte_memcpy_ptr = rte_memcpy_avx512f; + RTE_LOG(DEBUG, EAL, "AVX512 memcpy is using!\n"); + return; + } +#endif +#ifdef CC_SUPPORT_AVX2 + if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2)) { + rte_memcpy_ptr = rte_memcpy_avx2; + RTE_LOG(DEBUG, EAL, "AVX2 memcpy is using!\n"); + return; + } +#endif + rte_memcpy_ptr = rte_memcpy_sse; + RTE_LOG(DEBUG, EAL, "Default SSE/AVX memcpy is using!\n"); +} diff --git a/dpdk/lib/librte_eal/common/arch/x86/rte_spinlock.c b/dpdk/lib/librte_eal/common/arch/x86/rte_spinlock.c index c383e9f0..1244a90b 100644 --- a/dpdk/lib/librte_eal/common/arch/x86/rte_spinlock.c +++ b/dpdk/lib/librte_eal/common/arch/x86/rte_spinlock.c @@ -38,8 +38,7 @@ uint8_t rte_rtm_supported; /* cache the flag to avoid the overhead of the rte_cpu_get_flag_enabled function */ -static void __attribute__((constructor)) -rte_rtm_init(void) +RTE_INIT(rte_rtm_init) { rte_rtm_supported = rte_cpu_get_flag_enabled(RTE_CPUFLAG_RTM); } diff --git a/dpdk/lib/librte_eal/common/eal_common_bus.c b/dpdk/lib/librte_eal/common/eal_common_bus.c new file mode 100644 index 00000000..3e022d51 --- /dev/null +++ b/dpdk/lib/librte_eal/common/eal_common_bus.c @@ -0,0 +1,243 @@ +/*- + * BSD LICENSE + * + * Copyright 2016 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of NXP nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include + +#include +#include + +#include "eal_private.h" + +struct rte_bus_list rte_bus_list = + TAILQ_HEAD_INITIALIZER(rte_bus_list); + +void +rte_bus_register(struct rte_bus *bus) +{ + RTE_VERIFY(bus); + RTE_VERIFY(bus->name && strlen(bus->name)); + /* A bus should mandatorily have the scan implemented */ + RTE_VERIFY(bus->scan); + RTE_VERIFY(bus->probe); + RTE_VERIFY(bus->find_device); + /* Buses supporting driver plug also require unplug. */ + RTE_VERIFY(!bus->plug || bus->unplug); + + TAILQ_INSERT_TAIL(&rte_bus_list, bus, next); + RTE_LOG(DEBUG, EAL, "Registered [%s] bus.\n", bus->name); +} + +void +rte_bus_unregister(struct rte_bus *bus) +{ + TAILQ_REMOVE(&rte_bus_list, bus, next); + RTE_LOG(DEBUG, EAL, "Unregistered [%s] bus.\n", bus->name); +} + +/* Scan all the buses for registered devices */ +int +rte_bus_scan(void) +{ + int ret; + struct rte_bus *bus = NULL; + + TAILQ_FOREACH(bus, &rte_bus_list, next) { + ret = bus->scan(); + if (ret) + RTE_LOG(ERR, EAL, "Scan for (%s) bus failed.\n", + bus->name); + } + + return 0; +} + +/* Probe all devices of all buses */ +int +rte_bus_probe(void) +{ + int ret; + struct rte_bus *bus, *vbus = NULL; + + TAILQ_FOREACH(bus, &rte_bus_list, next) { + if (!strcmp(bus->name, "vdev")) { + vbus = bus; + continue; + } + + ret = bus->probe(); + if (ret) + RTE_LOG(ERR, EAL, "Bus (%s) probe failed.\n", + bus->name); + } + + if (vbus) { + ret = vbus->probe(); + if (ret) + RTE_LOG(ERR, EAL, "Bus (%s) probe failed.\n", + vbus->name); + } + + return 0; +} + +/* Dump information of a single bus */ +static int +bus_dump_one(FILE *f, struct rte_bus *bus) +{ + int ret; + + /* For now, dump only the bus name */ + ret = fprintf(f, " %s\n", bus->name); + + /* Error in case of inability in writing to stream */ + if (ret < 0) + return ret; + + return 0; +} + +void +rte_bus_dump(FILE *f) +{ + int ret; + struct rte_bus *bus; + + TAILQ_FOREACH(bus, &rte_bus_list, next) { + ret = bus_dump_one(f, bus); + if (ret) { + RTE_LOG(ERR, EAL, "Unable to write to stream (%d)\n", + ret); + break; + } + } +} + +struct rte_bus * +rte_bus_find(const struct rte_bus *start, rte_bus_cmp_t cmp, + const void *data) +{ + struct rte_bus *bus; + + if (start != NULL) + bus = TAILQ_NEXT(start, next); + else + bus = TAILQ_FIRST(&rte_bus_list); + while (bus != NULL) { + if (cmp(bus, data) == 0) + break; + bus = TAILQ_NEXT(bus, next); + } + return bus; +} + +static int +cmp_rte_device(const struct rte_device *dev1, const void *_dev2) +{ + const struct rte_device *dev2 = _dev2; + + return dev1 != dev2; +} + +static int +bus_find_device(const struct rte_bus *bus, const void *_dev) +{ + struct rte_device *dev; + + dev = bus->find_device(NULL, cmp_rte_device, _dev); + return dev == NULL; +} + +struct rte_bus * +rte_bus_find_by_device(const struct rte_device *dev) +{ + return rte_bus_find(NULL, bus_find_device, (const void *)dev); +} + +static int +cmp_bus_name(const struct rte_bus *bus, const void *_name) +{ + const char *name = _name; + + return strcmp(bus->name, name); +} + +struct rte_bus * +rte_bus_find_by_name(const char *busname) +{ + return rte_bus_find(NULL, cmp_bus_name, (const void *)busname); +} + +static int +bus_can_parse(const struct rte_bus *bus, const void *_name) +{ + const char *name = _name; + + return !(bus->parse && bus->parse(name, NULL) == 0); +} + +struct rte_bus * +rte_bus_find_by_device_name(const char *str) +{ + char name[RTE_DEV_NAME_MAX_LEN]; + char *c; + + snprintf(name, sizeof(name), "%s", str); + c = strchr(name, ','); + if (c != NULL) + c[0] = '\0'; + return rte_bus_find(NULL, bus_can_parse, name); +} + + +/* + * Get iommu class of devices on the bus. + */ +enum rte_iova_mode +rte_bus_get_iommu_class(void) +{ + int mode = RTE_IOVA_DC; + struct rte_bus *bus; + + TAILQ_FOREACH(bus, &rte_bus_list, next) { + + if (bus->get_iommu_class) + mode |= bus->get_iommu_class(); + } + + if (mode != RTE_IOVA_VA) { + /* Use default IOVA mode */ + mode = RTE_IOVA_PA; + } + return mode; +} diff --git a/dpdk/lib/librte_eal/common/eal_common_cpuflags.c b/dpdk/lib/librte_eal/common/eal_common_cpuflags.c index ecb12409..9a2d080a 100644 --- a/dpdk/lib/librte_eal/common/eal_common_cpuflags.c +++ b/dpdk/lib/librte_eal/common/eal_common_cpuflags.c @@ -39,15 +39,16 @@ /** * Checks if the machine is adequate for running the binary. If it is not, the * program exits with status 1. - * The function attribute forces this function to be called before main(). But - * with ICC, the check is generated by the compiler. */ -#ifndef __INTEL_COMPILER -void __attribute__ ((__constructor__)) -#else void -#endif rte_cpu_check_supported(void) +{ + if (!rte_cpu_is_supported()) + exit(1); +} + +int +rte_cpu_is_supported(void) { /* This is generated at compile-time by the build system */ static const enum rte_cpu_flag_t compile_time_flags[] = { @@ -63,14 +64,16 @@ rte_cpu_check_supported(void) fprintf(stderr, "ERROR: CPU feature flag lookup failed with error %d\n", ret); - exit(1); + return 0; } if (!ret) { fprintf(stderr, "ERROR: This system does not support \"%s\".\n" "Please check that RTE_MACHINE is set correctly.\n", rte_cpu_get_flag_name(compile_time_flags[i])); - exit(1); + return 0; } } + + return 1; } diff --git a/dpdk/lib/librte_eal/common/eal_common_dev.c b/dpdk/lib/librte_eal/common/eal_common_dev.c index a8a4146c..dda8f583 100644 --- a/dpdk/lib/librte_eal/common/eal_common_dev.c +++ b/dpdk/lib/librte_eal/common/eal_common_dev.c @@ -37,116 +37,200 @@ #include #include +#include #include #include #include -#include #include #include "eal_private.h" -/** Global list of device drivers. */ -static struct rte_driver_list dev_driver_list = - TAILQ_HEAD_INITIALIZER(dev_driver_list); - -/* register a driver */ -void -rte_eal_driver_register(struct rte_driver *driver) +static int cmp_detached_dev_name(const struct rte_device *dev, + const void *_name) { - TAILQ_INSERT_TAIL(&dev_driver_list, driver, next); + const char *name = _name; + + /* skip attached devices */ + if (dev->driver != NULL) + return 1; + + return strcmp(dev->name, name); } -/* unregister a driver */ -void -rte_eal_driver_unregister(struct rte_driver *driver) +static int cmp_dev_name(const struct rte_device *dev, const void *_name) { - TAILQ_REMOVE(&dev_driver_list, driver, next); + const char *name = _name; + + return strcmp(dev->name, name); } -int -rte_eal_vdev_init(const char *name, const char *args) +int rte_eal_dev_attach(const char *name, const char *devargs) { - struct rte_driver *driver; + struct rte_bus *bus; - if (name == NULL) + if (name == NULL || devargs == NULL) { + RTE_LOG(ERR, EAL, "Invalid device or arguments provided\n"); return -EINVAL; - - TAILQ_FOREACH(driver, &dev_driver_list, next) { - if (driver->type != PMD_VDEV) - continue; - - /* - * search a driver prefix in virtual device name. - * For example, if the driver is pcap PMD, driver->name - * will be "eth_pcap", but "name" will be "eth_pcapN". - * So use strncmp to compare. - */ - if (!strncmp(driver->name, name, strlen(driver->name))) - return driver->init(name, args); } - RTE_LOG(ERR, EAL, "no driver found for %s\n", name); - return -EINVAL; + bus = rte_bus_find_by_device_name(name); + if (bus == NULL) { + RTE_LOG(ERR, EAL, "Unable to find a bus for the device '%s'\n", + name); + return -EINVAL; + } + if (strcmp(bus->name, "pci") == 0 || strcmp(bus->name, "vdev") == 0) + return rte_eal_hotplug_add(bus->name, name, devargs); + + RTE_LOG(ERR, EAL, + "Device attach is only supported for PCI and vdev devices.\n"); + + return -ENOTSUP; } -int -rte_eal_dev_init(void) +int rte_eal_dev_detach(struct rte_device *dev) { - struct rte_devargs *devargs; - struct rte_driver *driver; + struct rte_bus *bus; + int ret; - /* - * Note that the dev_driver_list is populated here - * from calls made to rte_eal_driver_register from constructor functions - * embedded into PMD modules via the PMD_REGISTER_DRIVER macro - */ - - /* call the init function for each virtual device */ - TAILQ_FOREACH(devargs, &devargs_list, next) { - - if (devargs->type != RTE_DEVTYPE_VIRTUAL) - continue; - - if (rte_eal_vdev_init(devargs->virt.drv_name, - devargs->args)) { - RTE_LOG(ERR, EAL, "failed to initialize %s device\n", - devargs->virt.drv_name); - return -1; - } + if (dev == NULL) { + RTE_LOG(ERR, EAL, "Invalid device provided.\n"); + return -EINVAL; } - /* Once the vdevs are initalized, start calling all the pdev drivers */ - TAILQ_FOREACH(driver, &dev_driver_list, next) { - if (driver->type != PMD_PDEV) - continue; - /* PDEV drivers don't get passed any parameters */ - driver->init(NULL, NULL); + bus = rte_bus_find_by_device(dev); + if (bus == NULL) { + RTE_LOG(ERR, EAL, "Cannot find bus for device (%s)\n", + dev->name); + return -EINVAL; } + + if (bus->unplug == NULL) { + RTE_LOG(ERR, EAL, "Bus function not supported\n"); + return -ENOTSUP; + } + + ret = bus->unplug(dev); + if (ret) + RTE_LOG(ERR, EAL, "Driver cannot detach the device (%s)\n", + dev->name); + return ret; +} + +static char * +full_dev_name(const char *bus, const char *dev, const char *args) +{ + char *name; + size_t len; + + len = snprintf(NULL, 0, "%s:%s,%s", bus, dev, args) + 1; + name = calloc(1, len); + if (name == NULL) { + RTE_LOG(ERR, EAL, "Could not allocate full device name\n"); + return NULL; + } + snprintf(name, len, "%s:%s,%s", bus, dev, args); + return name; +} + +int rte_eal_hotplug_add(const char *busname, const char *devname, + const char *devargs) +{ + struct rte_bus *bus; + struct rte_device *dev; + struct rte_devargs *da; + char *name; + int ret; + + bus = rte_bus_find_by_name(busname); + if (bus == NULL) { + RTE_LOG(ERR, EAL, "Cannot find bus (%s)\n", busname); + return -ENOENT; + } + + if (bus->plug == NULL) { + RTE_LOG(ERR, EAL, "Function plug not supported by bus (%s)\n", + bus->name); + return -ENOTSUP; + } + + name = full_dev_name(busname, devname, devargs); + if (name == NULL) + return -ENOMEM; + + da = calloc(1, sizeof(*da)); + if (da == NULL) { + ret = -ENOMEM; + goto err_name; + } + + ret = rte_eal_devargs_parse(name, da); + if (ret) + goto err_devarg; + + ret = rte_eal_devargs_insert(da); + if (ret) + goto err_devarg; + + ret = bus->scan(); + if (ret) + goto err_devarg; + + dev = bus->find_device(NULL, cmp_detached_dev_name, devname); + if (dev == NULL) { + RTE_LOG(ERR, EAL, "Cannot find unplugged device (%s)\n", + devname); + ret = -ENODEV; + goto err_devarg; + } + + ret = bus->plug(dev); + if (ret) { + RTE_LOG(ERR, EAL, "Driver cannot attach the device (%s)\n", + dev->name); + goto err_devarg; + } + free(name); return 0; + +err_devarg: + if (rte_eal_devargs_remove(busname, devname)) { + free(da->args); + free(da); + } +err_name: + free(name); + return ret; } -int -rte_eal_vdev_uninit(const char *name) +int rte_eal_hotplug_remove(const char *busname, const char *devname) { - struct rte_driver *driver; + struct rte_bus *bus; + struct rte_device *dev; + int ret; - if (name == NULL) - return -EINVAL; - - TAILQ_FOREACH(driver, &dev_driver_list, next) { - if (driver->type != PMD_VDEV) - continue; - - /* - * search a driver prefix in virtual device name. - * For example, if the driver is pcap PMD, driver->name - * will be "eth_pcap", but "name" will be "eth_pcapN". - * So use strncmp to compare. - */ - if (!strncmp(driver->name, name, strlen(driver->name))) - return driver->uninit(name); + bus = rte_bus_find_by_name(busname); + if (bus == NULL) { + RTE_LOG(ERR, EAL, "Cannot find bus (%s)\n", busname); + return -ENOENT; } - RTE_LOG(ERR, EAL, "no driver found for %s\n", name); - return -EINVAL; + if (bus->unplug == NULL) { + RTE_LOG(ERR, EAL, "Function unplug not supported by bus (%s)\n", + bus->name); + return -ENOTSUP; + } + + dev = bus->find_device(NULL, cmp_dev_name, devname); + if (dev == NULL) { + RTE_LOG(ERR, EAL, "Cannot find plugged device (%s)\n", devname); + return -EINVAL; + } + + ret = bus->unplug(dev); + if (ret) + RTE_LOG(ERR, EAL, "Driver cannot detach the device (%s)\n", + dev->name); + rte_eal_devargs_remove(busname, devname); + return ret; } diff --git a/dpdk/lib/librte_eal/common/eal_common_devargs.c b/dpdk/lib/librte_eal/common/eal_common_devargs.c index e403717b..6ac88d6a 100644 --- a/dpdk/lib/librte_eal/common/eal_common_devargs.c +++ b/dpdk/lib/librte_eal/common/eal_common_devargs.c @@ -40,8 +40,9 @@ #include #include -#include +#include #include +#include #include "eal_private.h" /** Global list of user devices */ @@ -72,55 +73,113 @@ rte_eal_parse_devargs_str(const char *devargs_str, if (*drvargs == NULL) { free(*drvname); + *drvname = NULL; return -1; } return 0; } +static int +bus_name_cmp(const struct rte_bus *bus, const void *name) +{ + return strncmp(bus->name, name, strlen(bus->name)); +} + +int +rte_eal_devargs_parse(const char *dev, struct rte_devargs *da) +{ + struct rte_bus *bus = NULL; + const char *devname; + const size_t maxlen = sizeof(da->name); + size_t i; + + if (dev == NULL || da == NULL) + return -EINVAL; + /* Retrieve eventual bus info */ + do { + devname = dev; + bus = rte_bus_find(bus, bus_name_cmp, dev); + if (bus == NULL) + break; + devname = dev + strlen(bus->name) + 1; + if (rte_bus_find_by_device_name(devname) == bus) + break; + } while (1); + /* Store device name */ + i = 0; + while (devname[i] != '\0' && devname[i] != ',') { + da->name[i] = devname[i]; + i++; + if (i == maxlen) { + fprintf(stderr, "WARNING: Parsing \"%s\": device name should be shorter than %zu\n", + dev, maxlen); + da->name[i - 1] = '\0'; + return -EINVAL; + } + } + da->name[i] = '\0'; + if (bus == NULL) { + bus = rte_bus_find_by_device_name(da->name); + if (bus == NULL) { + fprintf(stderr, "ERROR: failed to parse device \"%s\"\n", + da->name); + return -EFAULT; + } + } + da->bus = bus; + /* Parse eventual device arguments */ + if (devname[i] == ',') + da->args = strdup(&devname[i + 1]); + else + da->args = strdup(""); + if (da->args == NULL) { + fprintf(stderr, "ERROR: not enough memory to parse arguments\n"); + return -ENOMEM; + } + return 0; +} + +int +rte_eal_devargs_insert(struct rte_devargs *da) +{ + int ret; + + ret = rte_eal_devargs_remove(da->bus->name, da->name); + if (ret < 0) + return ret; + TAILQ_INSERT_TAIL(&devargs_list, da, next); + return 0; +} + /* store a whitelist parameter for later parsing */ int rte_eal_devargs_add(enum rte_devtype devtype, const char *devargs_str) { struct rte_devargs *devargs = NULL; - char *buf = NULL; - int ret; + struct rte_bus *bus = NULL; + const char *dev = devargs_str; - /* use malloc instead of rte_malloc as it's called early at init */ - devargs = malloc(sizeof(*devargs)); + /* use calloc instead of rte_zmalloc as it's called early at init */ + devargs = calloc(1, sizeof(*devargs)); if (devargs == NULL) goto fail; - memset(devargs, 0, sizeof(*devargs)); - devargs->type = devtype; - - if (rte_eal_parse_devargs_str(devargs_str, &buf, &devargs->args)) + if (rte_eal_devargs_parse(dev, devargs)) goto fail; - - switch (devargs->type) { - case RTE_DEVTYPE_WHITELISTED_PCI: - case RTE_DEVTYPE_BLACKLISTED_PCI: - /* try to parse pci identifier */ - if (eal_parse_pci_BDF(buf, &devargs->pci.addr) != 0 && - eal_parse_pci_DomBDF(buf, &devargs->pci.addr) != 0) - goto fail; - - break; - case RTE_DEVTYPE_VIRTUAL: - /* save driver name */ - ret = snprintf(devargs->virt.drv_name, - sizeof(devargs->virt.drv_name), "%s", buf); - if (ret < 0 || ret >= (int)sizeof(devargs->virt.drv_name)) - goto fail; - - break; + devargs->type = devtype; + bus = devargs->bus; + if (devargs->type == RTE_DEVTYPE_BLACKLISTED_PCI) + devargs->policy = RTE_DEV_BLACKLISTED; + if (bus->conf.scan_mode == RTE_BUS_SCAN_UNDEFINED) { + if (devargs->policy == RTE_DEV_WHITELISTED) + bus->conf.scan_mode = RTE_BUS_SCAN_WHITELIST; + else if (devargs->policy == RTE_DEV_BLACKLISTED) + bus->conf.scan_mode = RTE_BUS_SCAN_BLACKLIST; } - - free(buf); TAILQ_INSERT_TAIL(&devargs_list, devargs, next); return 0; fail: - free(buf); if (devargs) { free(devargs->args); free(devargs); @@ -129,6 +188,24 @@ fail: return -1; } +int +rte_eal_devargs_remove(const char *busname, const char *devname) +{ + struct rte_devargs *d; + void *tmp; + + TAILQ_FOREACH_SAFE(d, &devargs_list, next, tmp) { + if (strcmp(d->bus->name, busname) == 0 && + strcmp(d->name, devname) == 0) { + TAILQ_REMOVE(&devargs_list, d, next); + free(d->args); + free(d); + return 0; + } + } + return 1; +} + /* count the number of devices of a specified type */ unsigned int rte_eal_devargs_type_count(enum rte_devtype devtype) @@ -150,27 +227,10 @@ rte_eal_devargs_dump(FILE *f) { struct rte_devargs *devargs; - fprintf(f, "User device white list:\n"); + fprintf(f, "User device list:\n"); TAILQ_FOREACH(devargs, &devargs_list, next) { - if (devargs->type == RTE_DEVTYPE_WHITELISTED_PCI) - fprintf(f, " PCI whitelist " PCI_PRI_FMT " %s\n", - devargs->pci.addr.domain, - devargs->pci.addr.bus, - devargs->pci.addr.devid, - devargs->pci.addr.function, - devargs->args); - else if (devargs->type == RTE_DEVTYPE_BLACKLISTED_PCI) - fprintf(f, " PCI blacklist " PCI_PRI_FMT " %s\n", - devargs->pci.addr.domain, - devargs->pci.addr.bus, - devargs->pci.addr.devid, - devargs->pci.addr.function, - devargs->args); - else if (devargs->type == RTE_DEVTYPE_VIRTUAL) - fprintf(f, " VIRTUAL %s %s\n", - devargs->virt.drv_name, - devargs->args); - else - fprintf(f, " UNKNOWN %s\n", devargs->args); + fprintf(f, " [%s]: %s %s\n", + (devargs->bus ? devargs->bus->name : "??"), + devargs->name, devargs->args); } } diff --git a/dpdk/lib/librte_eal/common/eal_common_errno.c b/dpdk/lib/librte_eal/common/eal_common_errno.c index de48d8e4..dc5b7c04 100644 --- a/dpdk/lib/librte_eal/common/eal_common_errno.c +++ b/dpdk/lib/librte_eal/common/eal_common_errno.c @@ -46,18 +46,20 @@ RTE_DEFINE_PER_LCORE(int, _rte_errno); const char * rte_strerror(int errnum) { + /* BSD puts a colon in the "unknown error" messages, Linux doesn't */ +#ifdef RTE_EXEC_ENV_BSDAPP + static const char *sep = ":"; +#else + static const char *sep = ""; +#endif #define RETVAL_SZ 256 static RTE_DEFINE_PER_LCORE(char[RETVAL_SZ], retval); + char *ret = RTE_PER_LCORE(retval); /* since some implementations of strerror_r throw an error * themselves if errnum is too big, we handle that case here */ - if (errnum > RTE_MAX_ERRNO) - snprintf(RTE_PER_LCORE(retval), RETVAL_SZ, -#ifdef RTE_EXEC_ENV_BSDAPP - "Unknown error: %d", errnum); -#else - "Unknown error %d", errnum); -#endif + if (errnum >= RTE_MAX_ERRNO) + snprintf(ret, RETVAL_SZ, "Unknown error%s %d", sep, errnum); else switch (errnum){ case E_RTE_SECONDARY: @@ -65,8 +67,10 @@ rte_strerror(int errnum) case E_RTE_NO_CONFIG: return "Missing rte_config structure"; default: - strerror_r(errnum, RTE_PER_LCORE(retval), RETVAL_SZ); + if (strerror_r(errnum, ret, RETVAL_SZ) != 0) + snprintf(ret, RETVAL_SZ, "Unknown error%s %d", + sep, errnum); } - return RTE_PER_LCORE(retval); + return ret; } diff --git a/dpdk/lib/librte_eal/common/eal_common_launch.c b/dpdk/lib/librte_eal/common/eal_common_launch.c index 229c3a03..2d5cae9f 100644 --- a/dpdk/lib/librte_eal/common/eal_common_launch.c +++ b/dpdk/lib/librte_eal/common/eal_common_launch.c @@ -38,9 +38,9 @@ #include #include -#include #include #include +#include #include #include @@ -54,7 +54,8 @@ rte_eal_wait_lcore(unsigned slave_id) return 0; while (lcore_config[slave_id].state != WAIT && - lcore_config[slave_id].state != FINISHED); + lcore_config[slave_id].state != FINISHED) + rte_pause(); rte_rmb(); diff --git a/dpdk/lib/librte_eal/common/eal_common_lcore.c b/dpdk/lib/librte_eal/common/eal_common_lcore.c index 2cd41320..0db1555b 100644 --- a/dpdk/lib/librte_eal/common/eal_common_lcore.c +++ b/dpdk/lib/librte_eal/common/eal_common_lcore.c @@ -81,18 +81,20 @@ rte_eal_cpu_init(void) /* By default, each detected core is enabled */ config->lcore_role[lcore_id] = ROLE_RTE; + lcore_config[lcore_id].core_role = ROLE_RTE; lcore_config[lcore_id].core_id = eal_cpu_core_id(lcore_id); lcore_config[lcore_id].socket_id = eal_cpu_socket_id(lcore_id); - if (lcore_config[lcore_id].socket_id >= RTE_MAX_NUMA_NODES) + if (lcore_config[lcore_id].socket_id >= RTE_MAX_NUMA_NODES) { #ifdef RTE_EAL_ALLOW_INV_SOCKET_ID lcore_config[lcore_id].socket_id = 0; #else - rte_panic("Socket ID (%u) is greater than " + RTE_LOG(ERR, EAL, "Socket ID (%u) is greater than " "RTE_MAX_NUMA_NODES (%d)\n", lcore_config[lcore_id].socket_id, RTE_MAX_NUMA_NODES); + return -1; #endif - + } RTE_LOG(DEBUG, EAL, "Detected lcore %u as " "core %u on socket %u\n", lcore_id, lcore_config[lcore_id].core_id, diff --git a/dpdk/lib/librte_eal/common/eal_common_log.c b/dpdk/lib/librte_eal/common/eal_common_log.c index 7916c781..4b2409a8 100644 --- a/dpdk/lib/librte_eal/common/eal_common_log.c +++ b/dpdk/lib/librte_eal/common/eal_common_log.c @@ -35,7 +35,11 @@ #include #include #include +#include +#include +#include +#include #include #include @@ -48,93 +52,264 @@ struct rte_logs rte_logs = { .file = NULL, }; +/* Stream to use for logging if rte_logs.file is NULL */ static FILE *default_log_stream; /** * This global structure stores some informations about the message - * that is currently beeing processed by one lcore + * that is currently being processed by one lcore */ struct log_cur_msg { uint32_t loglevel; /**< log level - see rte_log.h */ uint32_t logtype; /**< log type - see rte_log.h */ }; +struct rte_log_dynamic_type { + const char *name; + uint32_t loglevel; +}; + /* per core log */ static RTE_DEFINE_PER_LCORE(struct log_cur_msg, log_cur_msg); /* default logs */ -int -rte_log_add_in_history(const char *buf __rte_unused, size_t size __rte_unused) -{ - return 0; -} - -void -rte_log_set_history(int enable) -{ - if (enable) - RTE_LOG(WARNING, EAL, "The log history is deprecated.\n"); -} - /* Change the stream that will be used by logging system */ int rte_openlog_stream(FILE *f) { - if (f == NULL) - rte_logs.file = default_log_stream; - else - rte_logs.file = f; + rte_logs.file = f; return 0; } /* Set global log level */ void -rte_set_log_level(uint32_t level) +rte_log_set_global_level(uint32_t level) { rte_logs.level = (uint32_t)level; } /* Get global log level */ uint32_t -rte_get_log_level(void) +rte_log_get_global_level(void) { return rte_logs.level; } -/* Set global log type */ -void -rte_set_log_type(uint32_t type, int enable) +int +rte_log_get_level(uint32_t type) { - if (enable) - rte_logs.type |= type; - else - rte_logs.type &= (~type); + if (type >= rte_logs.dynamic_types_len) + return -1; + + return rte_logs.dynamic_types[type].loglevel; } -/* Get global log type */ -uint32_t -rte_get_log_type(void) +int +rte_log_set_level(uint32_t type, uint32_t level) { - return rte_logs.type; + if (type >= rte_logs.dynamic_types_len) + return -1; + if (level > RTE_LOG_DEBUG) + return -1; + + rte_logs.dynamic_types[type].loglevel = level; + + return 0; } -/* get the current loglevel for the message beeing processed */ +/* set level */ +int +rte_log_set_level_regexp(const char *pattern, uint32_t level) +{ + regex_t r; + size_t i; + + if (level > RTE_LOG_DEBUG) + return -1; + + if (regcomp(&r, pattern, 0) != 0) + return -1; + + for (i = 0; i < rte_logs.dynamic_types_len; i++) { + if (rte_logs.dynamic_types[i].name == NULL) + continue; + if (regexec(&r, rte_logs.dynamic_types[i].name, 0, + NULL, 0) == 0) + rte_logs.dynamic_types[i].loglevel = level; + } + + regfree(&r); + + return 0; +} + +/* get the current loglevel for the message being processed */ int rte_log_cur_msg_loglevel(void) { return RTE_PER_LCORE(log_cur_msg).loglevel; } -/* get the current logtype for the message beeing processed */ +/* get the current logtype for the message being processed */ int rte_log_cur_msg_logtype(void) { return RTE_PER_LCORE(log_cur_msg).logtype; } -/* Dump log history to file */ -void -rte_log_dump_history(FILE *out __rte_unused) +static int +rte_log_lookup(const char *name) { + size_t i; + + for (i = 0; i < rte_logs.dynamic_types_len; i++) { + if (rte_logs.dynamic_types[i].name == NULL) + continue; + if (strcmp(name, rte_logs.dynamic_types[i].name) == 0) + return i; + } + + return -1; +} + +/* register an extended log type, assuming table is large enough, and id + * is not yet registered. + */ +static int +__rte_log_register(const char *name, int id) +{ + char *dup_name = strdup(name); + + if (dup_name == NULL) + return -ENOMEM; + + rte_logs.dynamic_types[id].name = dup_name; + rte_logs.dynamic_types[id].loglevel = RTE_LOG_DEBUG; + + return id; +} + +/* register an extended log type */ +int +rte_log_register(const char *name) +{ + struct rte_log_dynamic_type *new_dynamic_types; + int id, ret; + + id = rte_log_lookup(name); + if (id >= 0) + return id; + + new_dynamic_types = realloc(rte_logs.dynamic_types, + sizeof(struct rte_log_dynamic_type) * + (rte_logs.dynamic_types_len + 1)); + if (new_dynamic_types == NULL) + return -ENOMEM; + rte_logs.dynamic_types = new_dynamic_types; + + ret = __rte_log_register(name, rte_logs.dynamic_types_len); + if (ret < 0) + return ret; + + rte_logs.dynamic_types_len++; + + return ret; +} + +struct logtype { + uint32_t log_id; + const char *logtype; +}; + +static const struct logtype logtype_strings[] = { + {RTE_LOGTYPE_EAL, "eal"}, + {RTE_LOGTYPE_MALLOC, "malloc"}, + {RTE_LOGTYPE_RING, "ring"}, + {RTE_LOGTYPE_MEMPOOL, "mempool"}, + {RTE_LOGTYPE_TIMER, "timer"}, + {RTE_LOGTYPE_PMD, "pmd"}, + {RTE_LOGTYPE_HASH, "hash"}, + {RTE_LOGTYPE_LPM, "lpm"}, + {RTE_LOGTYPE_KNI, "kni"}, + {RTE_LOGTYPE_ACL, "acl"}, + {RTE_LOGTYPE_POWER, "power"}, + {RTE_LOGTYPE_METER, "meter"}, + {RTE_LOGTYPE_SCHED, "sched"}, + {RTE_LOGTYPE_PORT, "port"}, + {RTE_LOGTYPE_TABLE, "table"}, + {RTE_LOGTYPE_PIPELINE, "pipeline"}, + {RTE_LOGTYPE_MBUF, "mbuf"}, + {RTE_LOGTYPE_CRYPTODEV, "cryptodev"}, + {RTE_LOGTYPE_EFD, "efd"}, + {RTE_LOGTYPE_EVENTDEV, "eventdev"}, + {RTE_LOGTYPE_USER1, "user1"}, + {RTE_LOGTYPE_USER2, "user2"}, + {RTE_LOGTYPE_USER3, "user3"}, + {RTE_LOGTYPE_USER4, "user4"}, + {RTE_LOGTYPE_USER5, "user5"}, + {RTE_LOGTYPE_USER6, "user6"}, + {RTE_LOGTYPE_USER7, "user7"}, + {RTE_LOGTYPE_USER8, "user8"} +}; + +/* Logging should be first initializer (before drivers and bus) */ +RTE_INIT_PRIO(rte_log_init, 101); +static void +rte_log_init(void) +{ + uint32_t i; + +#if RTE_LOG_LEVEL >= RTE_LOG_DEBUG + rte_log_set_global_level(RTE_LOG_INFO); +#else + rte_log_set_global_level(RTE_LOG_LEVEL); +#endif + + rte_logs.dynamic_types = calloc(RTE_LOGTYPE_FIRST_EXT_ID, + sizeof(struct rte_log_dynamic_type)); + if (rte_logs.dynamic_types == NULL) + return; + + /* register legacy log types */ + for (i = 0; i < RTE_DIM(logtype_strings); i++) + __rte_log_register(logtype_strings[i].logtype, + logtype_strings[i].log_id); + + rte_logs.dynamic_types_len = RTE_LOGTYPE_FIRST_EXT_ID; +} + +static const char * +loglevel_to_string(uint32_t level) +{ + switch (level) { + case 0: return "disabled"; + case RTE_LOG_EMERG: return "emerg"; + case RTE_LOG_ALERT: return "alert"; + case RTE_LOG_CRIT: return "critical"; + case RTE_LOG_ERR: return "error"; + case RTE_LOG_WARNING: return "warning"; + case RTE_LOG_NOTICE: return "notice"; + case RTE_LOG_INFO: return "info"; + case RTE_LOG_DEBUG: return "debug"; + default: return "unknown"; + } +} + +/* dump global level and registered log types */ +void +rte_log_dump(FILE *f) +{ + size_t i; + + fprintf(f, "global log level is %s\n", + loglevel_to_string(rte_log_get_global_level())); + + for (i = 0; i < rte_logs.dynamic_types_len; i++) { + if (rte_logs.dynamic_types[i].name == NULL) + continue; + fprintf(f, "id %zu: %s, level is %s\n", + i, rte_logs.dynamic_types[i].name, + loglevel_to_string(rte_logs.dynamic_types[i].loglevel)); + } } /* @@ -146,8 +321,25 @@ rte_vlog(uint32_t level, uint32_t logtype, const char *format, va_list ap) { int ret; FILE *f = rte_logs.file; + if (f == NULL) { + f = default_log_stream; + if (f == NULL) { + /* + * Grab the current value of stderr here, rather than + * just initializing default_log_stream to stderr. This + * ensures that we will always use the current value + * of stderr, even if the application closes and + * reopens it. + */ + f = stderr; + } + } - if ((level > rte_logs.level) || !(logtype & rte_logs.type)) + if (level > rte_logs.level) + return 0; + if (logtype >= rte_logs.dynamic_types_len) + return -1; + if (level > rte_logs.dynamic_types[logtype].loglevel) return 0; /* save loglevel and logtype in a global per-lcore variable */ @@ -177,17 +369,15 @@ rte_log(uint32_t level, uint32_t logtype, const char *format, ...) } /* - * called by environment-specific log init function + * Called by environment-specific initialization functions. */ -int -rte_eal_common_log_init(FILE *default_log) +void +eal_log_set_default(FILE *default_log) { default_log_stream = default_log; - rte_openlog_stream(default_log); -#if RTE_LOG_LEVEL >= RTE_LOG_DEBUG - RTE_LOG(NOTICE, EAL, "Debug logs available - lower performance\n"); +#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG + RTE_LOG(NOTICE, EAL, + "Debug dataplane logs available - lower performance\n"); #endif - - return 0; } diff --git a/dpdk/lib/librte_eal/common/eal_common_memory.c b/dpdk/lib/librte_eal/common/eal_common_memory.c index 6155752e..fc6c44da 100644 --- a/dpdk/lib/librte_eal/common/eal_common_memory.c +++ b/dpdk/lib/librte_eal/common/eal_common_memory.c @@ -35,11 +35,12 @@ #include #include #include +#include #include +#include #include #include -#include #include #include #include @@ -94,11 +95,11 @@ rte_dump_physmem_layout(FILE *f) if (mcfg->memseg[i].addr == NULL) break; - fprintf(f, "Segment %u: phys:0x%"PRIx64", len:%zu, " + fprintf(f, "Segment %u: IOVA:0x%"PRIx64", len:%zu, " "virt:%p, socket_id:%"PRId32", " "hugepage_sz:%"PRIu64", nchannel:%"PRIx32", " "nrank:%"PRIx32"\n", i, - mcfg->memseg[i].phys_addr, + mcfg->memseg[i].iova, mcfg->memseg[i].len, mcfg->memseg[i].addr, mcfg->memseg[i].socket_id, @@ -135,6 +136,16 @@ rte_eal_memdevice_init(void) return 0; } +/* Lock page in physical memory and prevent from swapping. */ +int +rte_mem_lock_page(const void *virt) +{ + unsigned long virtual = (unsigned long)virt; + int page_size = getpagesize(); + unsigned long aligned = (virtual & ~(page_size - 1)); + return mlock((void *)aligned, page_size); +} + /* init memory subsystem */ int rte_eal_memory_init(void) diff --git a/dpdk/lib/librte_eal/common/eal_common_memzone.c b/dpdk/lib/librte_eal/common/eal_common_memzone.c index 1bd0a33d..b682b005 100644 --- a/dpdk/lib/librte_eal/common/eal_common_memzone.c +++ b/dpdk/lib/librte_eal/common/eal_common_memzone.c @@ -189,7 +189,8 @@ memzone_reserve_aligned_thread_unsafe(const char *name, size_t len, return NULL; } - if ((socket_id != SOCKET_ID_ANY) && (socket_id >= RTE_MAX_NUMA_NODES)) { + if ((socket_id != SOCKET_ID_ANY) && + (socket_id >= RTE_MAX_NUMA_NODES || socket_id < 0)) { rte_errno = EINVAL; return NULL; } @@ -236,7 +237,7 @@ memzone_reserve_aligned_thread_unsafe(const char *name, size_t len, return NULL; } - const struct malloc_elem *elem = malloc_elem_from_data(mz_addr); + struct malloc_elem *elem = malloc_elem_from_data(mz_addr); /* fill the zone in config */ mz = get_next_free_memzone(); @@ -244,13 +245,14 @@ memzone_reserve_aligned_thread_unsafe(const char *name, size_t len, if (mz == NULL) { RTE_LOG(ERR, EAL, "%s(): Cannot find free memzone but there is room " "in config!\n", __func__); + malloc_elem_free(elem); rte_errno = ENOSPC; return NULL; } mcfg->memzone_cnt++; snprintf(mz->name, sizeof(mz->name), "%s", name); - mz->phys_addr = rte_malloc_virt2phy(mz_addr); + mz->iova = rte_malloc_virt2iova(mz_addr); mz->addr = mz_addr; mz->len = (requested_len == 0 ? elem->size : requested_len); mz->hugepage_sz = elem->ms->hugepage_sz; @@ -337,19 +339,7 @@ rte_memzone_free(const struct rte_memzone *mz) idx = ((uintptr_t)mz - (uintptr_t)mcfg->memzone); idx = idx / sizeof(struct rte_memzone); -#ifdef RTE_LIBRTE_IVSHMEM - /* - * If ioremap_addr is set, it's an IVSHMEM memzone and we cannot - * free it. - */ - if (mcfg->memzone[idx].ioremap_addr != 0) { - rte_rwlock_write_unlock(&mcfg->mlock); - return -EINVAL; - } -#endif - addr = mcfg->memzone[idx].addr; - if (addr == NULL) ret = -EINVAL; else if (mcfg->memzone_cnt == 0) { @@ -402,10 +392,10 @@ rte_memzone_dump(FILE *f) for (i=0; imemzone[i].addr == NULL) break; - fprintf(f, "Zone %u: name:<%s>, phys:0x%"PRIx64", len:0x%zx" + fprintf(f, "Zone %u: name:<%s>, IO:0x%"PRIx64", len:0x%zx" ", virt:%p, socket_id:%"PRId32", flags:%"PRIx32"\n", i, mcfg->memzone[i].name, - mcfg->memzone[i].phys_addr, + mcfg->memzone[i].iova, mcfg->memzone[i].len, mcfg->memzone[i].addr, mcfg->memzone[i].socket_id, diff --git a/dpdk/lib/librte_eal/common/eal_common_options.c b/dpdk/lib/librte_eal/common/eal_common_options.c index 1a1bab36..996a0342 100644 --- a/dpdk/lib/librte_eal/common/eal_common_options.c +++ b/dpdk/lib/librte_eal/common/eal_common_options.c @@ -47,6 +47,7 @@ #include #include #include +#include #include #include #include @@ -61,9 +62,11 @@ const char eal_short_options[] = "b:" /* pci-blacklist */ "c:" /* coremask */ + "s:" /* service coremask */ "d:" /* driver */ "h" /* help */ "l:" /* corelist */ + "S:" /* service corelist */ "m:" /* memory size */ "n:" /* memory channels */ "r:" /* memory ranks */ @@ -82,6 +85,7 @@ eal_long_options[] = { {OPT_LCORES, 1, NULL, OPT_LCORES_NUM }, {OPT_LOG_LEVEL, 1, NULL, OPT_LOG_LEVEL_NUM }, {OPT_MASTER_LCORE, 1, NULL, OPT_MASTER_LCORE_NUM }, + {OPT_MBUF_POOL_OPS_NAME, 1, NULL, OPT_MBUF_POOL_OPS_NAME_NUM}, {OPT_NO_HPET, 0, NULL, OPT_NO_HPET_NUM }, {OPT_NO_HUGE, 0, NULL, OPT_NO_HUGE_NUM }, {OPT_NO_PCI, 0, NULL, OPT_NO_PCI_NUM }, @@ -94,7 +98,6 @@ eal_long_options[] = { {OPT_VDEV, 1, NULL, OPT_VDEV_NUM }, {OPT_VFIO_INTR, 1, NULL, OPT_VFIO_INTR_NUM }, {OPT_VMWARE_TSC_MAP, 0, NULL, OPT_VMWARE_TSC_MAP_NUM }, - {OPT_XEN_DOM0, 0, NULL, OPT_XEN_DOM0_NUM }, {0, 0, NULL, 0 } }; @@ -118,14 +121,71 @@ static const char *default_solib_dir = RTE_EAL_PMD_PATH; /* * Stringified version of solib path used by dpdk-pmdinfo.py * Note: PLEASE DO NOT ALTER THIS without making a corresponding - * change to tools/dpdk-pmdinfo.py + * change to usertools/dpdk-pmdinfo.py */ static const char dpdk_solib_path[] __attribute__((used)) = "DPDK_PLUGIN_PATH=" RTE_EAL_PMD_PATH; +TAILQ_HEAD(device_option_list, device_option); + +struct device_option { + TAILQ_ENTRY(device_option) next; + + enum rte_devtype type; + char arg[]; +}; + +static struct device_option_list devopt_list = +TAILQ_HEAD_INITIALIZER(devopt_list); static int master_lcore_parsed; static int mem_parsed; +static int core_parsed; + +static int +eal_option_device_add(enum rte_devtype type, const char *optarg) +{ + struct device_option *devopt; + size_t optlen; + int ret; + + optlen = strlen(optarg) + 1; + devopt = calloc(1, sizeof(*devopt) + optlen); + if (devopt == NULL) { + RTE_LOG(ERR, EAL, "Unable to allocate device option\n"); + return -ENOMEM; + } + + devopt->type = type; + ret = snprintf(devopt->arg, optlen, "%s", optarg); + if (ret < 0) { + RTE_LOG(ERR, EAL, "Unable to copy device option\n"); + free(devopt); + return -EINVAL; + } + TAILQ_INSERT_TAIL(&devopt_list, devopt, next); + return 0; +} + +int +eal_option_device_parse(void) +{ + struct device_option *devopt; + void *tmp; + int ret = 0; + + TAILQ_FOREACH_SAFE(devopt, &devopt_list, next, tmp) { + if (ret == 0) { + ret = rte_eal_devargs_add(devopt->type, devopt->arg); + if (ret) + RTE_LOG(ERR, EAL, "Unable to parse device '%s'\n", + devopt->arg); + } + TAILQ_REMOVE(&devopt_list, devopt, next); + free(devopt); + } + return ret; +} void eal_reset_internal_config(struct internal_config *internal_cfg) @@ -147,14 +207,6 @@ eal_reset_internal_config(struct internal_config *internal_cfg) internal_cfg->base_virtaddr = 0; internal_cfg->syslog_facility = LOG_DAEMON; - /* default value from build option */ -#if RTE_LOG_LEVEL >= RTE_LOG_DEBUG - internal_cfg->log_level = RTE_LOG_INFO; -#else - internal_cfg->log_level = RTE_LOG_LEVEL; -#endif - - internal_cfg->xen_dom0_support = 0; /* if set to NONE, interrupt mode is determined automatically */ internal_cfg->vfio_intr_mode = RTE_INTR_MODE_NONE; @@ -166,6 +218,7 @@ eal_reset_internal_config(struct internal_config *internal_cfg) #endif internal_cfg->vmware_tsc_map = 0; internal_cfg->create_uio_dev = 0; + internal_cfg->mbuf_pool_ops_name = RTE_MBUF_DEFAULT_MEMPOOL_OPS; } static int @@ -225,12 +278,13 @@ int eal_plugins_init(void) { struct shared_driver *solib = NULL; + struct stat sb; - if (*default_solib_dir != '\0') + if (*default_solib_dir != '\0' && stat(default_solib_dir, &sb) == 0 && + S_ISDIR(sb.st_mode)) eal_plugin_add(default_solib_dir); TAILQ_FOREACH(solib, &solib_list, next) { - struct stat sb; if (stat(solib->name, &sb) == 0 && S_ISDIR(sb.st_mode)) { if (eal_plugindir_init(solib->name) == -1) { @@ -271,6 +325,77 @@ static int xdigit2val(unsigned char c) return val; } +static int +eal_parse_service_coremask(const char *coremask) +{ + struct rte_config *cfg = rte_eal_get_configuration(); + int i, j, idx = 0; + unsigned int count = 0; + char c; + int val; + + if (coremask == NULL) + return -1; + /* Remove all blank characters ahead and after . + * Remove 0x/0X if exists. + */ + while (isblank(*coremask)) + coremask++; + if (coremask[0] == '0' && ((coremask[1] == 'x') + || (coremask[1] == 'X'))) + coremask += 2; + i = strlen(coremask); + while ((i > 0) && isblank(coremask[i - 1])) + i--; + + if (i == 0) + return -1; + + for (i = i - 1; i >= 0 && idx < RTE_MAX_LCORE; i--) { + c = coremask[i]; + if (isxdigit(c) == 0) { + /* invalid characters */ + return -1; + } + val = xdigit2val(c); + for (j = 0; j < BITS_PER_HEX && idx < RTE_MAX_LCORE; + j++, idx++) { + if ((1 << j) & val) { + /* handle master lcore already parsed */ + uint32_t lcore = idx; + if (master_lcore_parsed && + cfg->master_lcore == lcore) { + RTE_LOG(ERR, EAL, + "Error: lcore %u is master lcore, cannot use as service core\n", + idx); + return -1; + } + + if (!lcore_config[idx].detected) { + RTE_LOG(ERR, EAL, + "lcore %u unavailable\n", idx); + return -1; + } + lcore_config[idx].core_role = ROLE_SERVICE; + count++; + } + } + } + + for (; i >= 0; i--) + if (coremask[i] != '0') + return -1; + + for (; idx < RTE_MAX_LCORE; idx++) + lcore_config[idx].core_index = -1; + + if (count == 0) + return -1; + + cfg->service_lcore_count = count; + return 0; +} + static int eal_parse_coremask(const char *coremask) { @@ -334,6 +459,72 @@ eal_parse_coremask(const char *coremask) return 0; } +static int +eal_parse_service_corelist(const char *corelist) +{ + struct rte_config *cfg = rte_eal_get_configuration(); + int i, idx = 0; + unsigned count = 0; + char *end = NULL; + int min, max; + + if (corelist == NULL) + return -1; + + /* Remove all blank characters ahead and after */ + while (isblank(*corelist)) + corelist++; + i = strlen(corelist); + while ((i > 0) && isblank(corelist[i - 1])) + i--; + + /* Get list of cores */ + min = RTE_MAX_LCORE; + do { + while (isblank(*corelist)) + corelist++; + if (*corelist == '\0') + return -1; + errno = 0; + idx = strtoul(corelist, &end, 10); + if (errno || end == NULL) + return -1; + while (isblank(*end)) + end++; + if (*end == '-') { + min = idx; + } else if ((*end == ',') || (*end == '\0')) { + max = idx; + if (min == RTE_MAX_LCORE) + min = idx; + for (idx = min; idx <= max; idx++) { + if (cfg->lcore_role[idx] != ROLE_SERVICE) { + /* handle master lcore already parsed */ + uint32_t lcore = idx; + if (cfg->master_lcore == lcore && + master_lcore_parsed) { + RTE_LOG(ERR, EAL, + "Error: lcore %u is master lcore, cannot use as service core\n", + idx); + return -1; + } + lcore_config[idx].core_role = + ROLE_SERVICE; + count++; + } + } + min = RTE_MAX_LCORE; + } else + return -1; + corelist = end + 1; + } while (*end != '\0'); + + if (count == 0) + return -1; + + return 0; +} + static int eal_parse_corelist(const char *corelist) { @@ -414,6 +605,13 @@ eal_parse_master_lcore(const char *arg) if (cfg->master_lcore >= RTE_MAX_LCORE) return -1; master_lcore_parsed = 1; + + /* ensure master core is not used as service core */ + if (lcore_config[cfg->master_lcore].core_role == ROLE_SERVICE) { + RTE_LOG(ERR, EAL, "Error: Master lcore is used as a service core.\n"); + return -1; + } + return 0; } @@ -738,25 +936,49 @@ eal_parse_syslog(const char *facility, struct internal_config *conf) } static int -eal_parse_log_level(const char *level, uint32_t *log_level) +eal_parse_log_level(const char *arg) { - char *end; + char *end, *str, *type, *level; unsigned long tmp; + str = strdup(arg); + if (str == NULL) + return -1; + + if (strchr(str, ',') == NULL) { + type = NULL; + level = str; + } else { + type = strsep(&str, ","); + level = strsep(&str, ","); + } + errno = 0; tmp = strtoul(level, &end, 0); /* check for errors */ if ((errno != 0) || (level[0] == '\0') || - end == NULL || (*end != '\0')) - return -1; + end == NULL || (*end != '\0')) + goto fail; /* log_level is a uint32_t */ if (tmp >= UINT32_MAX) - return -1; + goto fail; - *log_level = tmp; + if (type == NULL) { + rte_log_set_global_level(tmp); + } else if (rte_log_set_level_regexp(type, tmp) < 0) { + printf("cannot set log level %s,%lu\n", + type, tmp); + goto fail; + } + + free(str); return 0; + +fail: + free(str); + return -1; } static enum rte_proc_type_t @@ -776,20 +998,29 @@ int eal_parse_common_option(int opt, const char *optarg, struct internal_config *conf) { + static int b_used; + static int w_used; + switch (opt) { /* blacklist */ case 'b': - if (rte_eal_devargs_add(RTE_DEVTYPE_BLACKLISTED_PCI, + if (w_used) + goto bw_used; + if (eal_option_device_add(RTE_DEVTYPE_BLACKLISTED_PCI, optarg) < 0) { return -1; } + b_used = 1; break; /* whitelist */ case 'w': - if (rte_eal_devargs_add(RTE_DEVTYPE_WHITELISTED_PCI, + if (b_used) + goto bw_used; + if (eal_option_device_add(RTE_DEVTYPE_WHITELISTED_PCI, optarg) < 0) { return -1; } + w_used = 1; break; /* coremask */ case 'c': @@ -797,6 +1028,7 @@ eal_parse_common_option(int opt, const char *optarg, RTE_LOG(ERR, EAL, "invalid coremask\n"); return -1; } + core_parsed = 1; break; /* corelist */ case 'l': @@ -804,6 +1036,21 @@ eal_parse_common_option(int opt, const char *optarg, RTE_LOG(ERR, EAL, "invalid core list\n"); return -1; } + core_parsed = 1; + break; + /* service coremask */ + case 's': + if (eal_parse_service_coremask(optarg) < 0) { + RTE_LOG(ERR, EAL, "invalid service coremask\n"); + return -1; + } + break; + /* service corelist */ + case 'S': + if (eal_parse_service_corelist(optarg) < 0) { + RTE_LOG(ERR, EAL, "invalid service core list\n"); + return -1; + } break; /* size of memory */ case 'm': @@ -880,7 +1127,7 @@ eal_parse_common_option(int opt, const char *optarg, break; case OPT_VDEV_NUM: - if (rte_eal_devargs_add(RTE_DEVTYPE_VIRTUAL, + if (eal_option_device_add(RTE_DEVTYPE_VIRTUAL, optarg) < 0) { return -1; } @@ -895,15 +1142,12 @@ eal_parse_common_option(int opt, const char *optarg, break; case OPT_LOG_LEVEL_NUM: { - uint32_t log; - - if (eal_parse_log_level(optarg, &log) < 0) { + if (eal_parse_log_level(optarg) < 0) { RTE_LOG(ERR, EAL, "invalid parameters for --" OPT_LOG_LEVEL "\n"); return -1; } - conf->log_level = log; break; } case OPT_LCORES_NUM: @@ -912,6 +1156,7 @@ eal_parse_common_option(int opt, const char *optarg, OPT_LCORES "\n"); return -1; } + core_parsed = 1; break; /* don't know what to do, leave this to caller */ @@ -921,6 +1166,33 @@ eal_parse_common_option(int opt, const char *optarg, } return 0; +bw_used: + RTE_LOG(ERR, EAL, "Options blacklist (-b) and whitelist (-w) " + "cannot be used at the same time\n"); + return -1; +} + +static void +eal_auto_detect_cores(struct rte_config *cfg) +{ + unsigned int lcore_id; + unsigned int removed = 0; + rte_cpuset_t affinity_set; + pthread_t tid = pthread_self(); + + if (pthread_getaffinity_np(tid, sizeof(rte_cpuset_t), + &affinity_set) < 0) + CPU_ZERO(&affinity_set); + + for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { + if (cfg->lcore_role[lcore_id] == ROLE_RTE && + !CPU_ISSET(lcore_id, &affinity_set)) { + cfg->lcore_role[lcore_id] = ROLE_OFF; + removed++; + } + } + + cfg->lcore_count -= removed; } int @@ -929,12 +1201,17 @@ eal_adjust_config(struct internal_config *internal_cfg) int i; struct rte_config *cfg = rte_eal_get_configuration(); + if (!core_parsed) + eal_auto_detect_cores(cfg); + if (internal_config.process_type == RTE_PROC_AUTO) internal_config.process_type = eal_proc_type_detect(); /* default master lcore is the first one */ - if (!master_lcore_parsed) + if (!master_lcore_parsed) { cfg->master_lcore = rte_get_next_lcore(-1, 0, 0); + lcore_config[cfg->master_lcore].core_role = ROLE_RTE; + } /* if no memory amounts were requested, this will result in 0 and * will be overridden later, right after eal_hugepage_info_init() */ @@ -980,13 +1257,6 @@ eal_check_common_options(struct internal_config *internal_cfg) return -1; } - if (rte_eal_devargs_type_count(RTE_DEVTYPE_WHITELISTED_PCI) != 0 && - rte_eal_devargs_type_count(RTE_DEVTYPE_BLACKLISTED_PCI) != 0) { - RTE_LOG(ERR, EAL, "Options blacklist (-b) and whitelist (-w) " - "cannot be used at the same time\n"); - return -1; - } - return 0; } @@ -1007,7 +1277,9 @@ eal_common_usage(void) " ',' is used for single number separator.\n" " '( )' can be omitted for single element group,\n" " '@' can be omitted if cpus and lcores have the same value\n" + " -s SERVICE COREMASK Hexadecimal bitmask of cores to be used as service cores\n" " --"OPT_MASTER_LCORE" ID Core ID that is used as master\n" + " --"OPT_MBUF_POOL_OPS_NAME" Pool ops name for mbuf to use\n" " -n CHANNELS Number of memory channels\n" " -m MB Memory to allocate (see also --"OPT_SOCKET_MEM")\n" " -r RANKS Force number of memory ranks (don't detect)\n" @@ -1021,13 +1293,15 @@ eal_common_usage(void) " [NOTE: PCI whitelist cannot be used with -b option]\n" " --"OPT_VDEV" Add a virtual device.\n" " The argument format is [,key=val,...]\n" - " (ex: --vdev=eth_pcap0,iface=eth2).\n" + " (ex: --vdev=net_pcap0,iface=eth2).\n" " -d LIB.so|DIR Add a driver or driver directory\n" " (can be used multiple times)\n" " --"OPT_VMWARE_TSC_MAP" Use VMware TSC map instead of native RDTSC\n" " --"OPT_PROC_TYPE" Type of this process (primary|secondary|auto)\n" " --"OPT_SYSLOG" Set syslog facility\n" - " --"OPT_LOG_LEVEL" Set default log level\n" + " --"OPT_LOG_LEVEL"= Set global log level\n" + " --"OPT_LOG_LEVEL"=,\n" + " Set specific log level\n" " -v Display version information on startup\n" " -h, --help This help\n" "\nEAL options for DEBUG use only:\n" diff --git a/dpdk/lib/librte_eal/common/eal_common_pci.c b/dpdk/lib/librte_eal/common/eal_common_pci.c deleted file mode 100644 index 096c65e4..00000000 --- a/dpdk/lib/librte_eal/common/eal_common_pci.c +++ /dev/null @@ -1,474 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -/* BSD LICENSE - * - * Copyright 2013-2014 6WIND S.A. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of 6WIND S.A. nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "eal_private.h" - -struct pci_driver_list pci_driver_list; -struct pci_device_list pci_device_list; - -#define SYSFS_PCI_DEVICES "/sys/bus/pci/devices" - -const char *pci_get_sysfs_path(void) -{ - const char *path = NULL; - - path = getenv("SYSFS_PCI_DEVICES"); - if (path == NULL) - return SYSFS_PCI_DEVICES; - - return path; -} - -static struct rte_devargs *pci_devargs_lookup(struct rte_pci_device *dev) -{ - struct rte_devargs *devargs; - - TAILQ_FOREACH(devargs, &devargs_list, next) { - if (devargs->type != RTE_DEVTYPE_BLACKLISTED_PCI && - devargs->type != RTE_DEVTYPE_WHITELISTED_PCI) - continue; - if (!rte_eal_compare_pci_addr(&dev->addr, &devargs->pci.addr)) - return devargs; - } - return NULL; -} - -/* map a particular resource from a file */ -void * -pci_map_resource(void *requested_addr, int fd, off_t offset, size_t size, - int additional_flags) -{ - void *mapaddr; - - /* Map the PCI memory resource of device */ - mapaddr = mmap(requested_addr, size, PROT_READ | PROT_WRITE, - MAP_SHARED | additional_flags, fd, offset); - if (mapaddr == MAP_FAILED) { - RTE_LOG(ERR, EAL, "%s(): cannot mmap(%d, %p, 0x%lx, 0x%lx): %s (%p)\n", - __func__, fd, requested_addr, - (unsigned long)size, (unsigned long)offset, - strerror(errno), mapaddr); - } else - RTE_LOG(DEBUG, EAL, " PCI memory mapped at %p\n", mapaddr); - - return mapaddr; -} - -/* unmap a particular resource */ -void -pci_unmap_resource(void *requested_addr, size_t size) -{ - if (requested_addr == NULL) - return; - - /* Unmap the PCI memory resource of device */ - if (munmap(requested_addr, size)) { - RTE_LOG(ERR, EAL, "%s(): cannot munmap(%p, 0x%lx): %s\n", - __func__, requested_addr, (unsigned long)size, - strerror(errno)); - } else - RTE_LOG(DEBUG, EAL, " PCI memory unmapped at %p\n", - requested_addr); -} - -/* - * If vendor/device ID match, call the devinit() function of the - * driver. - */ -static int -rte_eal_pci_probe_one_driver(struct rte_pci_driver *dr, struct rte_pci_device *dev) -{ - int ret; - const struct rte_pci_id *id_table; - - for (id_table = dr->id_table; id_table->vendor_id != 0; id_table++) { - - /* check if device's identifiers match the driver's ones */ - if (id_table->vendor_id != dev->id.vendor_id && - id_table->vendor_id != PCI_ANY_ID) - continue; - if (id_table->device_id != dev->id.device_id && - id_table->device_id != PCI_ANY_ID) - continue; - if (id_table->subsystem_vendor_id != dev->id.subsystem_vendor_id && - id_table->subsystem_vendor_id != PCI_ANY_ID) - continue; - if (id_table->subsystem_device_id != dev->id.subsystem_device_id && - id_table->subsystem_device_id != PCI_ANY_ID) - continue; - if (id_table->class_id != dev->id.class_id && - id_table->class_id != RTE_CLASS_ANY_ID) - continue; - - struct rte_pci_addr *loc = &dev->addr; - - RTE_LOG(INFO, EAL, "PCI device "PCI_PRI_FMT" on NUMA socket %i\n", - loc->domain, loc->bus, loc->devid, loc->function, - dev->numa_node); - - /* no initialization when blacklisted, return without error */ - if (dev->devargs != NULL && - dev->devargs->type == RTE_DEVTYPE_BLACKLISTED_PCI) { - RTE_LOG(INFO, EAL, " Device is blacklisted, not initializing\n"); - return 1; - } - - RTE_LOG(INFO, EAL, " probe driver: %x:%x %s\n", dev->id.vendor_id, - dev->id.device_id, dr->name); - - if (dr->drv_flags & RTE_PCI_DRV_NEED_MAPPING) { - /* map resources for devices that use igb_uio */ - ret = rte_eal_pci_map_device(dev); - if (ret != 0) - return ret; - } else if (dr->drv_flags & RTE_PCI_DRV_FORCE_UNBIND && - rte_eal_process_type() == RTE_PROC_PRIMARY) { - /* unbind current driver */ - if (pci_unbind_kernel_driver(dev) < 0) - return -1; - } - - /* reference driver structure */ - dev->driver = dr; - - /* call the driver devinit() function */ - return dr->devinit(dr, dev); - } - /* return positive value if driver doesn't support this device */ - return 1; -} - -/* - * If vendor/device ID match, call the devuninit() function of the - * driver. - */ -static int -rte_eal_pci_detach_dev(struct rte_pci_driver *dr, - struct rte_pci_device *dev) -{ - const struct rte_pci_id *id_table; - - if ((dr == NULL) || (dev == NULL)) - return -EINVAL; - - for (id_table = dr->id_table; id_table->vendor_id != 0; id_table++) { - - /* check if device's identifiers match the driver's ones */ - if (id_table->vendor_id != dev->id.vendor_id && - id_table->vendor_id != PCI_ANY_ID) - continue; - if (id_table->device_id != dev->id.device_id && - id_table->device_id != PCI_ANY_ID) - continue; - if (id_table->subsystem_vendor_id != dev->id.subsystem_vendor_id && - id_table->subsystem_vendor_id != PCI_ANY_ID) - continue; - if (id_table->subsystem_device_id != dev->id.subsystem_device_id && - id_table->subsystem_device_id != PCI_ANY_ID) - continue; - - struct rte_pci_addr *loc = &dev->addr; - - RTE_LOG(DEBUG, EAL, "PCI device "PCI_PRI_FMT" on NUMA socket %i\n", - loc->domain, loc->bus, loc->devid, - loc->function, dev->numa_node); - - RTE_LOG(DEBUG, EAL, " remove driver: %x:%x %s\n", dev->id.vendor_id, - dev->id.device_id, dr->name); - - if (dr->devuninit && (dr->devuninit(dev) < 0)) - return -1; /* negative value is an error */ - - /* clear driver structure */ - dev->driver = NULL; - - if (dr->drv_flags & RTE_PCI_DRV_NEED_MAPPING) - /* unmap resources for devices that use igb_uio */ - rte_eal_pci_unmap_device(dev); - - return 0; - } - - /* return positive value if driver doesn't support this device */ - return 1; -} - -/* - * If vendor/device ID match, call the devinit() function of all - * registered driver for the given device. Return -1 if initialization - * failed, return 1 if no driver is found for this device. - */ -static int -pci_probe_all_drivers(struct rte_pci_device *dev) -{ - struct rte_pci_driver *dr = NULL; - int rc = 0; - - if (dev == NULL) - return -1; - - TAILQ_FOREACH(dr, &pci_driver_list, next) { - rc = rte_eal_pci_probe_one_driver(dr, dev); - if (rc < 0) - /* negative value is an error */ - return -1; - if (rc > 0) - /* positive value means driver doesn't support it */ - continue; - return 0; - } - return 1; -} - -/* - * If vendor/device ID match, call the devuninit() function of all - * registered driver for the given device. Return -1 if initialization - * failed, return 1 if no driver is found for this device. - */ -static int -pci_detach_all_drivers(struct rte_pci_device *dev) -{ - struct rte_pci_driver *dr = NULL; - int rc = 0; - - if (dev == NULL) - return -1; - - TAILQ_FOREACH(dr, &pci_driver_list, next) { - rc = rte_eal_pci_detach_dev(dr, dev); - if (rc < 0) - /* negative value is an error */ - return -1; - if (rc > 0) - /* positive value means driver doesn't support it */ - continue; - return 0; - } - return 1; -} - -/* - * Find the pci device specified by pci address, then invoke probe function of - * the driver of the devive. - */ -int -rte_eal_pci_probe_one(const struct rte_pci_addr *addr) -{ - struct rte_pci_device *dev = NULL; - int ret = 0; - - if (addr == NULL) - return -1; - - TAILQ_FOREACH(dev, &pci_device_list, next) { - if (rte_eal_compare_pci_addr(&dev->addr, addr)) - continue; - - ret = pci_probe_all_drivers(dev); - if (ret) - goto err_return; - return 0; - } - return -1; - -err_return: - RTE_LOG(WARNING, EAL, "Requested device " PCI_PRI_FMT - " cannot be used\n", dev->addr.domain, dev->addr.bus, - dev->addr.devid, dev->addr.function); - return -1; -} - -/* - * Detach device specified by its pci address. - */ -int -rte_eal_pci_detach(const struct rte_pci_addr *addr) -{ - struct rte_pci_device *dev = NULL; - int ret = 0; - - if (addr == NULL) - return -1; - - TAILQ_FOREACH(dev, &pci_device_list, next) { - if (rte_eal_compare_pci_addr(&dev->addr, addr)) - continue; - - ret = pci_detach_all_drivers(dev); - if (ret < 0) - goto err_return; - - TAILQ_REMOVE(&pci_device_list, dev, next); - free(dev); - return 0; - } - return -1; - -err_return: - RTE_LOG(WARNING, EAL, "Requested device " PCI_PRI_FMT - " cannot be used\n", dev->addr.domain, dev->addr.bus, - dev->addr.devid, dev->addr.function); - return -1; -} - -/* - * Scan the content of the PCI bus, and call the devinit() function for - * all registered drivers that have a matching entry in its id_table - * for discovered devices. - */ -int -rte_eal_pci_probe(void) -{ - struct rte_pci_device *dev = NULL; - struct rte_devargs *devargs; - int probe_all = 0; - int ret = 0; - - if (rte_eal_devargs_type_count(RTE_DEVTYPE_WHITELISTED_PCI) == 0) - probe_all = 1; - - TAILQ_FOREACH(dev, &pci_device_list, next) { - - /* set devargs in PCI structure */ - devargs = pci_devargs_lookup(dev); - if (devargs != NULL) - dev->devargs = devargs; - - /* probe all or only whitelisted devices */ - if (probe_all) - ret = pci_probe_all_drivers(dev); - else if (devargs != NULL && - devargs->type == RTE_DEVTYPE_WHITELISTED_PCI) - ret = pci_probe_all_drivers(dev); - if (ret < 0) - rte_exit(EXIT_FAILURE, "Requested device " PCI_PRI_FMT - " cannot be used\n", dev->addr.domain, dev->addr.bus, - dev->addr.devid, dev->addr.function); - } - - return 0; -} - -/* dump one device */ -static int -pci_dump_one_device(FILE *f, struct rte_pci_device *dev) -{ - int i; - - fprintf(f, PCI_PRI_FMT, dev->addr.domain, dev->addr.bus, - dev->addr.devid, dev->addr.function); - fprintf(f, " - vendor:%x device:%x\n", dev->id.vendor_id, - dev->id.device_id); - - for (i = 0; i != sizeof(dev->mem_resource) / - sizeof(dev->mem_resource[0]); i++) { - fprintf(f, " %16.16"PRIx64" %16.16"PRIx64"\n", - dev->mem_resource[i].phys_addr, - dev->mem_resource[i].len); - } - return 0; -} - -/* dump devices on the bus */ -void -rte_eal_pci_dump(FILE *f) -{ - struct rte_pci_device *dev = NULL; - - TAILQ_FOREACH(dev, &pci_device_list, next) { - pci_dump_one_device(f, dev); - } -} - -/* register a driver */ -void -rte_eal_pci_register(struct rte_pci_driver *driver) -{ - TAILQ_INSERT_TAIL(&pci_driver_list, driver, next); -} - -/* unregister a driver */ -void -rte_eal_pci_unregister(struct rte_pci_driver *driver) -{ - TAILQ_REMOVE(&pci_driver_list, driver, next); -} diff --git a/dpdk/lib/librte_eal/common/eal_common_pci_uio.c b/dpdk/lib/librte_eal/common/eal_common_pci_uio.c deleted file mode 100644 index 367a6816..00000000 --- a/dpdk/lib/librte_eal/common/eal_common_pci_uio.c +++ /dev/null @@ -1,233 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include "eal_private.h" - -static struct rte_tailq_elem rte_uio_tailq = { - .name = "UIO_RESOURCE_LIST", -}; -EAL_REGISTER_TAILQ(rte_uio_tailq) - -static int -pci_uio_map_secondary(struct rte_pci_device *dev) -{ - int fd, i, j; - struct mapped_pci_resource *uio_res; - struct mapped_pci_res_list *uio_res_list = - RTE_TAILQ_CAST(rte_uio_tailq.head, mapped_pci_res_list); - - TAILQ_FOREACH(uio_res, uio_res_list, next) { - - /* skip this element if it doesn't match our PCI address */ - if (rte_eal_compare_pci_addr(&uio_res->pci_addr, &dev->addr)) - continue; - - for (i = 0; i != uio_res->nb_maps; i++) { - /* - * open devname, to mmap it - */ - fd = open(uio_res->maps[i].path, O_RDWR); - if (fd < 0) { - RTE_LOG(ERR, EAL, "Cannot open %s: %s\n", - uio_res->maps[i].path, strerror(errno)); - return -1; - } - - void *mapaddr = pci_map_resource(uio_res->maps[i].addr, - fd, (off_t)uio_res->maps[i].offset, - (size_t)uio_res->maps[i].size, 0); - /* fd is not needed in slave process, close it */ - close(fd); - if (mapaddr != uio_res->maps[i].addr) { - RTE_LOG(ERR, EAL, - "Cannot mmap device resource file %s to address: %p\n", - uio_res->maps[i].path, - uio_res->maps[i].addr); - if (mapaddr != MAP_FAILED) { - /* unmap addrs correctly mapped */ - for (j = 0; j < i; j++) - pci_unmap_resource( - uio_res->maps[j].addr, - (size_t)uio_res->maps[j].size); - /* unmap addr wrongly mapped */ - pci_unmap_resource(mapaddr, - (size_t)uio_res->maps[i].size); - } - return -1; - } - } - return 0; - } - - RTE_LOG(ERR, EAL, "Cannot find resource for device\n"); - return 1; -} - -/* map the PCI resource of a PCI device in virtual memory */ -int -pci_uio_map_resource(struct rte_pci_device *dev) -{ - int i, map_idx = 0, ret; - uint64_t phaddr; - struct mapped_pci_resource *uio_res = NULL; - struct mapped_pci_res_list *uio_res_list = - RTE_TAILQ_CAST(rte_uio_tailq.head, mapped_pci_res_list); - - dev->intr_handle.fd = -1; - dev->intr_handle.uio_cfg_fd = -1; - dev->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN; - - /* secondary processes - use already recorded details */ - if (rte_eal_process_type() != RTE_PROC_PRIMARY) - return pci_uio_map_secondary(dev); - - /* allocate uio resource */ - ret = pci_uio_alloc_resource(dev, &uio_res); - if (ret) - return ret; - - /* Map all BARs */ - for (i = 0; i != PCI_MAX_RESOURCE; i++) { - /* skip empty BAR */ - phaddr = dev->mem_resource[i].phys_addr; - if (phaddr == 0) - continue; - - ret = pci_uio_map_resource_by_index(dev, i, - uio_res, map_idx); - if (ret) - goto error; - - map_idx++; - } - - uio_res->nb_maps = map_idx; - - TAILQ_INSERT_TAIL(uio_res_list, uio_res, next); - - return 0; -error: - for (i = 0; i < map_idx; i++) { - pci_unmap_resource(uio_res->maps[i].addr, - (size_t)uio_res->maps[i].size); - rte_free(uio_res->maps[i].path); - } - pci_uio_free_resource(dev, uio_res); - return -1; -} - -static void -pci_uio_unmap(struct mapped_pci_resource *uio_res) -{ - int i; - - if (uio_res == NULL) - return; - - for (i = 0; i != uio_res->nb_maps; i++) { - pci_unmap_resource(uio_res->maps[i].addr, - (size_t)uio_res->maps[i].size); - if (rte_eal_process_type() == RTE_PROC_PRIMARY) - rte_free(uio_res->maps[i].path); - } -} - -static struct mapped_pci_resource * -pci_uio_find_resource(struct rte_pci_device *dev) -{ - struct mapped_pci_resource *uio_res; - struct mapped_pci_res_list *uio_res_list = - RTE_TAILQ_CAST(rte_uio_tailq.head, mapped_pci_res_list); - - if (dev == NULL) - return NULL; - - TAILQ_FOREACH(uio_res, uio_res_list, next) { - - /* skip this element if it doesn't match our PCI address */ - if (!rte_eal_compare_pci_addr(&uio_res->pci_addr, &dev->addr)) - return uio_res; - } - return NULL; -} - -/* unmap the PCI resource of a PCI device in virtual memory */ -void -pci_uio_unmap_resource(struct rte_pci_device *dev) -{ - struct mapped_pci_resource *uio_res; - struct mapped_pci_res_list *uio_res_list = - RTE_TAILQ_CAST(rte_uio_tailq.head, mapped_pci_res_list); - - if (dev == NULL) - return; - - /* find an entry for the device */ - uio_res = pci_uio_find_resource(dev); - if (uio_res == NULL) - return; - - /* secondary processes - just free maps */ - if (rte_eal_process_type() != RTE_PROC_PRIMARY) - return pci_uio_unmap(uio_res); - - TAILQ_REMOVE(uio_res_list, uio_res, next); - - /* unmap all resources */ - pci_uio_unmap(uio_res); - - /* free uio resource */ - rte_free(uio_res); - - /* close fd if in primary process */ - close(dev->intr_handle.fd); - if (dev->intr_handle.uio_cfg_fd >= 0) { - close(dev->intr_handle.uio_cfg_fd); - dev->intr_handle.uio_cfg_fd = -1; - } - - dev->intr_handle.fd = -1; - dev->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN; -} diff --git a/dpdk/lib/librte_eal/common/eal_common_proc.c b/dpdk/lib/librte_eal/common/eal_common_proc.c index 12e0fcac..60526cad 100644 --- a/dpdk/lib/librte_eal/common/eal_common_proc.c +++ b/dpdk/lib/librte_eal/common/eal_common_proc.c @@ -46,10 +46,10 @@ rte_eal_primary_proc_alive(const char *config_file_path) if (config_file_path) config_fd = open(config_file_path, O_RDONLY); else { - char default_path[PATH_MAX+1]; - snprintf(default_path, PATH_MAX, RUNTIME_CONFIG_FMT, - default_config_dir, "rte"); - config_fd = open(default_path, O_RDONLY); + const char *path; + + path = eal_runtime_config_path(); + config_fd = open(path, O_RDONLY); } if (config_fd < 0) return 0; diff --git a/dpdk/lib/librte_eal/common/eal_common_tailqs.c b/dpdk/lib/librte_eal/common/eal_common_tailqs.c index bb08ec8b..6ae09fdb 100644 --- a/dpdk/lib/librte_eal/common/eal_common_tailqs.c +++ b/dpdk/lib/librte_eal/common/eal_common_tailqs.c @@ -40,13 +40,11 @@ #include #include -#include #include #include #include #include #include -#include #include #include #include @@ -188,8 +186,7 @@ rte_eal_tailqs_init(void) if (t->head == NULL) { RTE_LOG(ERR, EAL, "Cannot initialize tailq: %s\n", t->name); - /* no need to TAILQ_REMOVE, we are going to panic in - * rte_eal_init() */ + /* TAILQ_REMOVE not needed, error is already fatal */ goto fail; } } diff --git a/dpdk/lib/librte_eal/common/eal_common_thread.c b/dpdk/lib/librte_eal/common/eal_common_thread.c index 2405e93f..55e96963 100644 --- a/dpdk/lib/librte_eal/common/eal_common_thread.c +++ b/dpdk/lib/librte_eal/common/eal_common_thread.c @@ -53,6 +53,20 @@ unsigned rte_socket_id(void) return RTE_PER_LCORE(_socket_id); } +int +rte_lcore_has_role(unsigned int lcore_id, enum rte_lcore_role_t role) +{ + struct rte_config *cfg = rte_eal_get_configuration(); + + if (lcore_id >= RTE_MAX_LCORE) + return -EINVAL; + + if (cfg->lcore_role[lcore_id] == role) + return 0; + + return -EINVAL; +} + int eal_cpuset_socket_id(rte_cpuset_t *cpusetp) { unsigned cpu = 0; diff --git a/dpdk/lib/librte_eal/common/eal_common_timer.c b/dpdk/lib/librte_eal/common/eal_common_timer.c index c4227cd8..568ae2fd 100644 --- a/dpdk/lib/librte_eal/common/eal_common_timer.c +++ b/dpdk/lib/librte_eal/common/eal_common_timer.c @@ -41,14 +41,18 @@ #include #include #include +#include #include "eal_private.h" /* The frequency of the RDTSC timer resolution */ static uint64_t eal_tsc_resolution_hz; +/* Pointer to user delay function */ +void (*rte_delay_us)(unsigned int) = NULL; + void -rte_delay_us(unsigned us) +rte_delay_us_block(unsigned int us) { const uint64_t start = rte_get_timer_cycles(); const uint64_t ticks = (uint64_t)us * rte_get_timer_hz() / 1E6; @@ -76,11 +80,25 @@ estimate_tsc_freq(void) void set_tsc_freq(void) { - uint64_t freq = get_tsc_freq(); + uint64_t freq; + freq = get_tsc_freq_arch(); + if (!freq) + freq = get_tsc_freq(); if (!freq) freq = estimate_tsc_freq(); RTE_LOG(DEBUG, EAL, "TSC frequency is ~%" PRIu64 " KHz\n", freq / 1000); eal_tsc_resolution_hz = freq; } + +void rte_delay_us_callback_register(void (*userfunc)(unsigned int)) +{ + rte_delay_us = userfunc; +} + +RTE_INIT(rte_timer_init) +{ + /* set rte_delay_us_block as a delay function */ + rte_delay_us_callback_register(rte_delay_us_block); +} diff --git a/dpdk/lib/librte_eal/common/eal_filesystem.h b/dpdk/lib/librte_eal/common/eal_filesystem.h index fdb4a70b..8acbd996 100644 --- a/dpdk/lib/librte_eal/common/eal_filesystem.h +++ b/dpdk/lib/librte_eal/common/eal_filesystem.h @@ -97,17 +97,6 @@ eal_get_hugefile_path(char *buffer, size_t buflen, const char *hugedir, int f_id return buffer; } -#ifdef RTE_EAL_SINGLE_FILE_SEGMENTS -static inline const char * -eal_get_hugefile_temp_path(char *buffer, size_t buflen, const char *hugedir, int f_id) -{ - snprintf(buffer, buflen, TEMP_HUGEFILE_FMT, hugedir, - internal_config.hugefile_prefix, f_id); - buffer[buflen - 1] = '\0'; - return buffer; -} -#endif - /** define the default filename prefix for the %s values above */ #define HUGEFILE_PREFIX_DEFAULT "rte" diff --git a/dpdk/lib/librte_eal/common/eal_hugepages.h b/dpdk/lib/librte_eal/common/eal_hugepages.h index 38edac03..68369f26 100644 --- a/dpdk/lib/librte_eal/common/eal_hugepages.h +++ b/dpdk/lib/librte_eal/common/eal_hugepages.h @@ -52,9 +52,6 @@ struct hugepage_file { int socket_id; /**< NUMA socket ID */ int file_id; /**< the '%d' in HUGEFILE_FMT */ int memseg_id; /**< the memory segment to which page belongs */ -#ifdef RTE_EAL_SINGLE_FILE_SEGMENTS - int repeated; /**< number of times the page size is repeated */ -#endif char filepath[MAX_HUGEPAGE_PATH]; /**< path to backing file on filesystem */ }; diff --git a/dpdk/lib/librte_eal/common/eal_internal_cfg.h b/dpdk/lib/librte_eal/common/eal_internal_cfg.h index 5f1367eb..fa6ccbec 100644 --- a/dpdk/lib/librte_eal/common/eal_internal_cfg.h +++ b/dpdk/lib/librte_eal/common/eal_internal_cfg.h @@ -65,7 +65,6 @@ struct internal_config { volatile unsigned force_nrank; /**< force number of ranks */ volatile unsigned no_hugetlbfs; /**< true to disable hugetlbfs */ unsigned hugepage_unlink; /**< true to unlink backing files */ - volatile unsigned xen_dom0_support; /**< support app running on Xen Dom0*/ volatile unsigned no_pci; /**< true to disable PCI */ volatile unsigned no_hpet; /**< true to disable HPET */ volatile unsigned vmware_tsc_map; /**< true to use VMware TSC mapping @@ -78,12 +77,11 @@ struct internal_config { volatile uint64_t socket_mem[RTE_MAX_NUMA_NODES]; /**< amount of memory per socket */ uintptr_t base_virtaddr; /**< base address to try and reserve memory from */ volatile int syslog_facility; /**< facility passed to openlog() */ - volatile uint32_t log_level; /**< default log level */ /** default interrupt mode for VFIO */ volatile enum rte_intr_mode vfio_intr_mode; const char *hugefile_prefix; /**< the base filename of hugetlbfs files */ const char *hugepage_dir; /**< specific hugetlbfs directory to use */ - + const char *mbuf_pool_ops_name; /**< mbuf pool ops name */ unsigned num_hugepage_sizes; /**< how many sizes on this system */ struct hugepage_info hugepage_info[MAX_HUGEPAGE_SIZES]; }; diff --git a/dpdk/lib/librte_eal/common/eal_options.h b/dpdk/lib/librte_eal/common/eal_options.h index a881c62e..30e6bb41 100644 --- a/dpdk/lib/librte_eal/common/eal_options.h +++ b/dpdk/lib/librte_eal/common/eal_options.h @@ -61,6 +61,8 @@ enum { OPT_LOG_LEVEL_NUM, #define OPT_MASTER_LCORE "master-lcore" OPT_MASTER_LCORE_NUM, +#define OPT_MBUF_POOL_OPS_NAME "mbuf-pool-ops-name" + OPT_MBUF_POOL_OPS_NAME_NUM, #define OPT_PROC_TYPE "proc-type" OPT_PROC_TYPE_NUM, #define OPT_NO_HPET "no-hpet" @@ -81,8 +83,6 @@ enum { OPT_VFIO_INTR_NUM, #define OPT_VMWARE_TSC_MAP "vmware-tsc-map" OPT_VMWARE_TSC_MAP_NUM, -#define OPT_XEN_DOM0 "xen-dom0" - OPT_XEN_DOM0_NUM, OPT_LONG_MAX_NUM }; @@ -91,6 +91,7 @@ extern const struct option eal_long_options[]; int eal_parse_common_option(int opt, const char *argv, struct internal_config *conf); +int eal_option_device_parse(void); int eal_adjust_config(struct internal_config *internal_cfg); int eal_check_common_options(struct internal_config *internal_cfg); void eal_common_usage(void); diff --git a/dpdk/lib/librte_eal/common/eal_private.h b/dpdk/lib/librte_eal/common/eal_private.h index 857dc3ea..462226f1 100644 --- a/dpdk/lib/librte_eal/common/eal_private.h +++ b/dpdk/lib/librte_eal/common/eal_private.h @@ -34,8 +34,9 @@ #ifndef _EAL_PRIVATE_H_ #define _EAL_PRIVATE_H_ +#include +#include #include -#include /** * Initialize the memzone subsystem (private to eal). @@ -47,7 +48,9 @@ int rte_eal_memzone_init(void); /** - * Common log initialization function (private to eal). + * Common log initialization function (private to eal). Determines + * where log data is written when no call to rte_openlog_stream is + * in effect. * * @param default_log * The default log stream to be used. @@ -55,7 +58,7 @@ int rte_eal_memzone_init(void); * - 0 on success * - Negative on error */ -int rte_eal_common_log_init(FILE *default_log); +void eal_log_set_default(FILE *default_log); /** * Fill configuration with number of physical and logical processors @@ -96,16 +99,6 @@ int rte_eal_memory_init(void); */ int rte_eal_timer_init(void); -/** - * Init early logs - * - * This function is private to EAL. - * - * @return - * 0 on success, negative on error - */ -int rte_eal_log_early_init(void); - /** * Init the default log stream * @@ -116,116 +109,6 @@ int rte_eal_log_early_init(void); */ int rte_eal_log_init(const char *id, int facility); -/** - * Init the default log stream - * - * This function is private to EAL. - * - * @return - * 0 on success, negative on error - */ -int rte_eal_pci_init(void); - -#ifdef RTE_LIBRTE_IVSHMEM -/** - * Init the memory from IVSHMEM devices - * - * This function is private to EAL. - * - * @return - * 0 on success, negative on error - */ -int rte_eal_ivshmem_init(void); - -/** - * Init objects in IVSHMEM devices - * - * This function is private to EAL. - * - * @return - * 0 on success, negative on error - */ -int rte_eal_ivshmem_obj_init(void); -#endif - -struct rte_pci_driver; -struct rte_pci_device; - -/** - * Unbind kernel driver for this device - * - * This function is private to EAL. - * - * @return - * 0 on success, negative on error - */ -int pci_unbind_kernel_driver(struct rte_pci_device *dev); - -/** - * Map the PCI resource of a PCI device in virtual memory - * - * This function is private to EAL. - * - * @return - * 0 on success, negative on error - */ -int pci_uio_map_resource(struct rte_pci_device *dev); - -/** - * Unmap the PCI resource of a PCI device - * - * This function is private to EAL. - */ -void pci_uio_unmap_resource(struct rte_pci_device *dev); - -/** - * Allocate uio resource for PCI device - * - * This function is private to EAL. - * - * @param dev - * PCI device to allocate uio resource - * @param uio_res - * Pointer to uio resource. - * If the function returns 0, the pointer will be filled. - * @return - * 0 on success, negative on error - */ -int pci_uio_alloc_resource(struct rte_pci_device *dev, - struct mapped_pci_resource **uio_res); - -/** - * Free uio resource for PCI device - * - * This function is private to EAL. - * - * @param dev - * PCI device to free uio resource - * @param uio_res - * Pointer to uio resource. - */ -void pci_uio_free_resource(struct rte_pci_device *dev, - struct mapped_pci_resource *uio_res); - -/** - * Map device memory to uio resource - * - * This function is private to EAL. - * - * @param dev - * PCI device that has memory information. - * @param res_idx - * Memory resource index of the PCI device. - * @param uio_res - * uio resource that will keep mapping information. - * @param map_idx - * Mapping information index of the uio resource. - * @return - * 0 on success, negative on error - */ -int pci_uio_map_resource_by_index(struct rte_pci_device *dev, int res_idx, - struct mapped_pci_resource *uio_res, int map_idx); - /** * Init tail queues for non-EAL library structures. This is to allow * the rings, mempools, etc. lists to be shared among multiple processes @@ -258,13 +141,6 @@ int rte_eal_intr_init(void); */ int rte_eal_alarm_init(void); -/** - * This function initialises any virtual devices - * - * This function is private to the EAL. - */ -int rte_eal_dev_init(void); - /** * Function is to check if the kernel module(like, vfio, vfio_iommu_type1, * etc.) loaded. @@ -307,6 +183,17 @@ void set_tsc_freq(void); */ uint64_t get_tsc_freq(void); +/** + * Get TSC frequency if the architecture supports. + * + * This function is private to the EAL. + * + * @return + * The number of TSC cycles in one second. + * Returns zero if the architecture support is not available. + */ +uint64_t get_tsc_freq_arch(void); + /** * Prepare physical memory mapping * i.e. hugepages on Linux and @@ -325,4 +212,16 @@ int rte_eal_hugepage_init(void); */ int rte_eal_hugepage_attach(void); +/** + * Find a bus capable of identifying a device. + * + * @param str + * A device identifier (PCI address, virtual PMD name, ...). + * + * @return + * A valid bus handle if found. + * NULL if no bus is able to parse this device. + */ +struct rte_bus *rte_bus_find_by_device_name(const char *str); + #endif /* _EAL_PRIVATE_H_ */ diff --git a/dpdk/lib/librte_eal/common/include/arch/arm/rte_atomic.h b/dpdk/lib/librte_eal/common/include/arch/arm/rte_atomic.h index 454a12b0..f3f3b6e3 100644 --- a/dpdk/lib/librte_eal/common/include/arch/arm/rte_atomic.h +++ b/dpdk/lib/librte_eal/common/include/arch/arm/rte_atomic.h @@ -39,10 +39,4 @@ #include #endif -#define rte_smp_mb() rte_mb() - -#define rte_smp_wmb() rte_wmb() - -#define rte_smp_rmb() rte_rmb() - #endif /* _RTE_ATOMIC_ARM_H_ */ diff --git a/dpdk/lib/librte_eal/common/include/arch/arm/rte_atomic_32.h b/dpdk/lib/librte_eal/common/include/arch/arm/rte_atomic_32.h index 9ae1e78b..14c04864 100644 --- a/dpdk/lib/librte_eal/common/include/arch/arm/rte_atomic_32.h +++ b/dpdk/lib/librte_eal/common/include/arch/arm/rte_atomic_32.h @@ -67,6 +67,18 @@ extern "C" { */ #define rte_rmb() __sync_synchronize() +#define rte_smp_mb() rte_mb() + +#define rte_smp_wmb() rte_wmb() + +#define rte_smp_rmb() rte_rmb() + +#define rte_io_mb() rte_mb() + +#define rte_io_wmb() rte_wmb() + +#define rte_io_rmb() rte_rmb() + #ifdef __cplusplus } #endif diff --git a/dpdk/lib/librte_eal/common/include/arch/arm/rte_atomic_64.h b/dpdk/lib/librte_eal/common/include/arch/arm/rte_atomic_64.h index 671caa76..71da29ca 100644 --- a/dpdk/lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +++ b/dpdk/lib/librte_eal/common/include/arch/arm/rte_atomic_64.h @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2015. + * Copyright (C) Cavium, Inc. 2015. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * @@ -43,43 +43,26 @@ extern "C" { #include "generic/rte_atomic.h" -#define dmb(opt) do { asm volatile("dmb " #opt : : : "memory"); } while (0) +#define dsb(opt) asm volatile("dsb " #opt : : : "memory") +#define dmb(opt) asm volatile("dmb " #opt : : : "memory") -/** - * General memory barrier. - * - * Guarantees that the LOAD and STORE operations generated before the - * barrier occur before the LOAD and STORE operations generated after. - * This function is architecture dependent. - */ -static inline void rte_mb(void) -{ - dmb(ish); -} +#define rte_mb() dsb(sy) -/** - * Write memory barrier. - * - * Guarantees that the STORE operations generated before the barrier - * occur before the STORE operations generated after. - * This function is architecture dependent. - */ -static inline void rte_wmb(void) -{ - dmb(ishst); -} +#define rte_wmb() dsb(st) -/** - * Read memory barrier. - * - * Guarantees that the LOAD operations generated before the barrier - * occur before the LOAD operations generated after. - * This function is architecture dependent. - */ -static inline void rte_rmb(void) -{ - dmb(ishld); -} +#define rte_rmb() dsb(ld) + +#define rte_smp_mb() dmb(ish) + +#define rte_smp_wmb() dmb(ishst) + +#define rte_smp_rmb() dmb(ishld) + +#define rte_io_mb() rte_mb() + +#define rte_io_wmb() rte_wmb() + +#define rte_io_rmb() rte_rmb() #ifdef __cplusplus } diff --git a/dpdk/lib/librte_eal/common/include/arch/arm/rte_byteorder.h b/dpdk/lib/librte_eal/common/include/arch/arm/rte_byteorder.h index 3f2dd1f2..0a29f4bb 100644 --- a/dpdk/lib/librte_eal/common/include/arch/arm/rte_byteorder.h +++ b/dpdk/lib/librte_eal/common/include/arch/arm/rte_byteorder.h @@ -41,6 +41,8 @@ extern "C" { #endif +#include +#include #include "generic/rte_byteorder.h" /* fix missing __builtin_bswap16 for gcc older then 4.8 */ @@ -50,7 +52,7 @@ static inline uint16_t rte_arch_bswap16(uint16_t _x) { register uint16_t x = _x; - asm volatile ("rev16 %0,%1" + asm volatile ("rev16 %w0,%w1" : "=r" (x) : "r" (x) ); diff --git a/dpdk/lib/librte_eal/common/include/arch/arm/rte_cpuflags_64.h b/dpdk/lib/librte_eal/common/include/arch/arm/rte_cpuflags_64.h index 49aead92..5425f4c7 100644 --- a/dpdk/lib/librte_eal/common/include/arch/arm/rte_cpuflags_64.h +++ b/dpdk/lib/librte_eal/common/include/arch/arm/rte_cpuflags_64.h @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2015. + * Copyright (C) Cavium, Inc. 2015. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * diff --git a/dpdk/lib/librte_eal/common/include/arch/arm/rte_cycles_64.h b/dpdk/lib/librte_eal/common/include/arch/arm/rte_cycles_64.h index 14f26120..15457691 100644 --- a/dpdk/lib/librte_eal/common/include/arch/arm/rte_cycles_64.h +++ b/dpdk/lib/librte_eal/common/include/arch/arm/rte_cycles_64.h @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2015. + * Copyright (C) Cavium, Inc. 2015. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * @@ -45,6 +45,11 @@ extern "C" { * @return * The time base for this lcore. */ +#ifndef RTE_ARM_EAL_RDTSC_USE_PMU +/** + * This call is portable to any ARMv8 architecture, however, typically + * cntvct_el0 runs at <= 100MHz and it may be imprecise for some tasks. + */ static inline uint64_t rte_rdtsc(void) { @@ -53,6 +58,34 @@ rte_rdtsc(void) asm volatile("mrs %0, cntvct_el0" : "=r" (tsc)); return tsc; } +#else +/** + * This is an alternative method to enable rte_rdtsc() with high resolution + * PMU cycles counter.The cycle counter runs at cpu frequency and this scheme + * uses ARMv8 PMU subsystem to get the cycle counter at userspace, However, + * access to PMU cycle counter from user space is not enabled by default in + * arm64 linux kernel. + * It is possible to enable cycle counter at user space access by configuring + * the PMU from the privileged mode (kernel space). + * + * asm volatile("msr pmintenset_el1, %0" : : "r" ((u64)(0 << 31))); + * asm volatile("msr pmcntenset_el0, %0" :: "r" BIT(31)); + * asm volatile("msr pmuserenr_el0, %0" : : "r"(BIT(0) | BIT(2))); + * asm volatile("mrs %0, pmcr_el0" : "=r" (val)); + * val |= (BIT(0) | BIT(2)); + * isb(); + * asm volatile("msr pmcr_el0, %0" : : "r" (val)); + * + */ +static inline uint64_t +rte_rdtsc(void) +{ + uint64_t tsc; + + asm volatile("mrs %0, pmccntr_el0" : "=r"(tsc)); + return tsc; +} +#endif static inline uint64_t rte_rdtsc_precise(void) diff --git a/dpdk/lib/librte_eal/common/include/arch/arm/rte_io.h b/dpdk/lib/librte_eal/common/include/arch/arm/rte_io.h new file mode 100644 index 00000000..3b63ec85 --- /dev/null +++ b/dpdk/lib/librte_eal/common/include/arch/arm/rte_io.h @@ -0,0 +1,51 @@ +/* + * BSD LICENSE + * + * Copyright(c) 2016 Cavium, Inc. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_IO_ARM_H_ +#define _RTE_IO_ARM_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef RTE_ARCH_64 +#include "rte_io_64.h" +#else +#include "generic/rte_io.h" +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_IO_ARM_H_ */ diff --git a/dpdk/lib/librte_eal/common/include/arch/arm/rte_io_64.h b/dpdk/lib/librte_eal/common/include/arch/arm/rte_io_64.h new file mode 100644 index 00000000..ee9b8d55 --- /dev/null +++ b/dpdk/lib/librte_eal/common/include/arch/arm/rte_io_64.h @@ -0,0 +1,199 @@ +/* + * BSD LICENSE + * + * Copyright (C) Cavium, Inc. 2016. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_IO_ARM64_H_ +#define _RTE_IO_ARM64_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +#define RTE_OVERRIDE_IO_H + +#include "generic/rte_io.h" +#include "rte_atomic_64.h" + +static __rte_always_inline uint8_t +rte_read8_relaxed(const volatile void *addr) +{ + uint8_t val; + + asm volatile( + "ldrb %w[val], [%x[addr]]" + : [val] "=r" (val) + : [addr] "r" (addr)); + return val; +} + +static __rte_always_inline uint16_t +rte_read16_relaxed(const volatile void *addr) +{ + uint16_t val; + + asm volatile( + "ldrh %w[val], [%x[addr]]" + : [val] "=r" (val) + : [addr] "r" (addr)); + return val; +} + +static __rte_always_inline uint32_t +rte_read32_relaxed(const volatile void *addr) +{ + uint32_t val; + + asm volatile( + "ldr %w[val], [%x[addr]]" + : [val] "=r" (val) + : [addr] "r" (addr)); + return val; +} + +static __rte_always_inline uint64_t +rte_read64_relaxed(const volatile void *addr) +{ + uint64_t val; + + asm volatile( + "ldr %x[val], [%x[addr]]" + : [val] "=r" (val) + : [addr] "r" (addr)); + return val; +} + +static __rte_always_inline void +rte_write8_relaxed(uint8_t val, volatile void *addr) +{ + asm volatile( + "strb %w[val], [%x[addr]]" + : + : [val] "r" (val), [addr] "r" (addr)); +} + +static __rte_always_inline void +rte_write16_relaxed(uint16_t val, volatile void *addr) +{ + asm volatile( + "strh %w[val], [%x[addr]]" + : + : [val] "r" (val), [addr] "r" (addr)); +} + +static __rte_always_inline void +rte_write32_relaxed(uint32_t val, volatile void *addr) +{ + asm volatile( + "str %w[val], [%x[addr]]" + : + : [val] "r" (val), [addr] "r" (addr)); +} + +static __rte_always_inline void +rte_write64_relaxed(uint64_t val, volatile void *addr) +{ + asm volatile( + "str %x[val], [%x[addr]]" + : + : [val] "r" (val), [addr] "r" (addr)); +} + +static __rte_always_inline uint8_t +rte_read8(const volatile void *addr) +{ + uint8_t val; + val = rte_read8_relaxed(addr); + rte_io_rmb(); + return val; +} + +static __rte_always_inline uint16_t +rte_read16(const volatile void *addr) +{ + uint16_t val; + val = rte_read16_relaxed(addr); + rte_io_rmb(); + return val; +} + +static __rte_always_inline uint32_t +rte_read32(const volatile void *addr) +{ + uint32_t val; + val = rte_read32_relaxed(addr); + rte_io_rmb(); + return val; +} + +static __rte_always_inline uint64_t +rte_read64(const volatile void *addr) +{ + uint64_t val; + val = rte_read64_relaxed(addr); + rte_io_rmb(); + return val; +} + +static __rte_always_inline void +rte_write8(uint8_t value, volatile void *addr) +{ + rte_io_wmb(); + rte_write8_relaxed(value, addr); +} + +static __rte_always_inline void +rte_write16(uint16_t value, volatile void *addr) +{ + rte_io_wmb(); + rte_write16_relaxed(value, addr); +} + +static __rte_always_inline void +rte_write32(uint32_t value, volatile void *addr) +{ + rte_io_wmb(); + rte_write32_relaxed(value, addr); +} + +static __rte_always_inline void +rte_write64(uint64_t value, volatile void *addr) +{ + rte_io_wmb(); + rte_write64_relaxed(value, addr); +} + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_IO_ARM64_H_ */ diff --git a/dpdk/lib/librte_eal/common/include/arch/arm/rte_memcpy_32.h b/dpdk/lib/librte_eal/common/include/arch/arm/rte_memcpy_32.h index da6c233a..e4dafda1 100644 --- a/dpdk/lib/librte_eal/common/include/arch/arm/rte_memcpy_32.h +++ b/dpdk/lib/librte_eal/common/include/arch/arm/rte_memcpy_32.h @@ -148,7 +148,8 @@ rte_mov256(uint8_t *dst, const uint8_t *src) } #define rte_memcpy(dst, src, n) \ - ({ (__builtin_constant_p(n)) ? \ + __extension__ ({ \ + (__builtin_constant_p(n)) ? \ memcpy((dst), (src), (n)) : \ rte_memcpy_func((dst), (src), (n)); }) @@ -224,7 +225,7 @@ rte_memcpy_func(void *dst, const void *src, size_t n) * We split the remaining bytes (which will be less than 256) into * 64byte (2^6) chunks. * Using incrementing integers in the case labels of a switch statement - * enourages the compiler to use a jump table. To get incrementing + * encourages the compiler to use a jump table. To get incrementing * integers, we shift the 2 relevant bits to the LSB position to first * get decrementing integers, and then subtract. */ diff --git a/dpdk/lib/librte_eal/common/include/arch/arm/rte_memcpy_64.h b/dpdk/lib/librte_eal/common/include/arch/arm/rte_memcpy_64.h index 5db66b63..b80d8ba4 100644 --- a/dpdk/lib/librte_eal/common/include/arch/arm/rte_memcpy_64.h +++ b/dpdk/lib/librte_eal/common/include/arch/arm/rte_memcpy_64.h @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2015. + * Copyright (C) Cavium, Inc. 2015. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * diff --git a/dpdk/lib/librte_eal/common/include/arch/arm/rte_pause.h b/dpdk/lib/librte_eal/common/include/arch/arm/rte_pause.h new file mode 100644 index 00000000..b772ca07 --- /dev/null +++ b/dpdk/lib/librte_eal/common/include/arch/arm/rte_pause.h @@ -0,0 +1,50 @@ +/* + * BSD LICENSE + * + * Copyright(c) 2017 Cavium, Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_PAUSE_ARM_H_ +#define _RTE_PAUSE_ARM_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef RTE_ARCH_64 +#include +#else +#include +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_PAUSE_ARM_H_ */ diff --git a/dpdk/lib/librte_eal/common/include/arch/arm/rte_pause_32.h b/dpdk/lib/librte_eal/common/include/arch/arm/rte_pause_32.h new file mode 100644 index 00000000..ec680b5c --- /dev/null +++ b/dpdk/lib/librte_eal/common/include/arch/arm/rte_pause_32.h @@ -0,0 +1,51 @@ +/* + * BSD LICENSE + * + * Copyright(c) 2017 Cavium, Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_PAUSE_ARM32_H_ +#define _RTE_PAUSE_ARM32_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include "generic/rte_pause.h" + +static inline void rte_pause(void) +{ +} + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_PAUSE_ARM32_H_ */ diff --git a/dpdk/lib/librte_eal/common/include/arch/arm/rte_pause_64.h b/dpdk/lib/librte_eal/common/include/arch/arm/rte_pause_64.h new file mode 100644 index 00000000..2da46326 --- /dev/null +++ b/dpdk/lib/librte_eal/common/include/arch/arm/rte_pause_64.h @@ -0,0 +1,52 @@ +/* + * BSD LICENSE + * + * Copyright(c) 2017 Cavium, Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_PAUSE_ARM64_H_ +#define _RTE_PAUSE_ARM64_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include "generic/rte_pause.h" + +static inline void rte_pause(void) +{ + asm volatile("yield" ::: "memory"); +} + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_PAUSE_ARM64_H_ */ diff --git a/dpdk/lib/librte_eal/common/include/arch/arm/rte_prefetch_32.h b/dpdk/lib/librte_eal/common/include/arch/arm/rte_prefetch_32.h index 5aeed22d..43cde172 100644 --- a/dpdk/lib/librte_eal/common/include/arch/arm/rte_prefetch_32.h +++ b/dpdk/lib/librte_eal/common/include/arch/arm/rte_prefetch_32.h @@ -37,6 +37,7 @@ extern "C" { #endif +#include #include "generic/rte_prefetch.h" static inline void rte_prefetch0(const volatile void *p) diff --git a/dpdk/lib/librte_eal/common/include/arch/arm/rte_prefetch_64.h b/dpdk/lib/librte_eal/common/include/arch/arm/rte_prefetch_64.h index 3ed46a46..ff59509f 100644 --- a/dpdk/lib/librte_eal/common/include/arch/arm/rte_prefetch_64.h +++ b/dpdk/lib/librte_eal/common/include/arch/arm/rte_prefetch_64.h @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2015. + * Copyright (C) Cavium, Inc. 2015. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * @@ -37,6 +37,7 @@ extern "C" { #endif +#include #include "generic/rte_prefetch.h" static inline void rte_prefetch0(const volatile void *p) diff --git a/dpdk/lib/librte_eal/common/include/arch/arm/rte_vect.h b/dpdk/lib/librte_eal/common/include/arch/arm/rte_vect.h index a33c0544..aa887a97 100644 --- a/dpdk/lib/librte_eal/common/include/arch/arm/rte_vect.h +++ b/dpdk/lib/librte_eal/common/include/arch/arm/rte_vect.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2015 Cavium Networks. All rights reserved. + * Copyright(c) 2015 Cavium, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium Networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * @@ -33,6 +33,9 @@ #ifndef _RTE_VECT_ARM_H_ #define _RTE_VECT_ARM_H_ +#include +#include "generic/rte_vect.h" +#include "rte_debug.h" #include "arm_neon.h" #ifdef __cplusplus @@ -74,8 +77,122 @@ vqtbl1q_u8(uint8x16_t a, uint8x16_t b) return vld1q_u8(rte_ret.u8); } + +static inline uint16_t +vaddvq_u16(uint16x8_t a) +{ + uint32x4_t m = vpaddlq_u16(a); + uint64x2_t n = vpaddlq_u32(m); + uint64x1_t o = vget_low_u64(n) + vget_high_u64(n); + + return vget_lane_u32((uint32x2_t)o, 0); +} + #endif +#if defined(RTE_TOOLCHAIN_GCC) && (GCC_VERSION < 70000) +static inline uint32x4_t +vcopyq_laneq_u32(uint32x4_t a, const int lane_a, + uint32x4_t b, const int lane_b) +{ + return vsetq_lane_u32(vgetq_lane_u32(b, lane_b), a, lane_a); +} +#endif + +#if defined(RTE_ARCH_ARM64) +#if defined(RTE_TOOLCHAIN_GCC) && (GCC_VERSION < 70000) + +#if (GCC_VERSION < 40900) +typedef uint64_t poly64_t; +typedef uint64x2_t poly64x2_t; +typedef uint8_t poly128_t __attribute__((vector_size(16), aligned(16))); +#endif + +/* NEON intrinsic vreinterpretq_u64_p128() is supported since GCC version 7 */ +static inline uint64x2_t +vreinterpretq_u64_p128(poly128_t x) +{ + return (uint64x2_t)x; +} + +/* NEON intrinsic vreinterpretq_p64_u64() is supported since GCC version 7 */ +static inline poly64x2_t +vreinterpretq_p64_u64(uint64x2_t x) +{ + return (poly64x2_t)x; +} + +/* NEON intrinsic vgetq_lane_p64() is supported since GCC version 7 */ +static inline poly64_t +vgetq_lane_p64(poly64x2_t x, const int lane) +{ + RTE_ASSERT(lane >= 0 && lane <= 1); + + poly64_t *p = (poly64_t *)&x; + + return p[lane]; +} +#endif +#endif + +/* + * If (0 <= index <= 15), then call the ASIMD ext instruction on the + * 128 bit regs v0 and v1 with the appropriate index. + * + * Else returns a zero vector. + */ +static inline uint8x16_t +vextract(uint8x16_t v0, uint8x16_t v1, const int index) +{ + switch (index) { + case 0: return vextq_u8(v0, v1, 0); + case 1: return vextq_u8(v0, v1, 1); + case 2: return vextq_u8(v0, v1, 2); + case 3: return vextq_u8(v0, v1, 3); + case 4: return vextq_u8(v0, v1, 4); + case 5: return vextq_u8(v0, v1, 5); + case 6: return vextq_u8(v0, v1, 6); + case 7: return vextq_u8(v0, v1, 7); + case 8: return vextq_u8(v0, v1, 8); + case 9: return vextq_u8(v0, v1, 9); + case 10: return vextq_u8(v0, v1, 10); + case 11: return vextq_u8(v0, v1, 11); + case 12: return vextq_u8(v0, v1, 12); + case 13: return vextq_u8(v0, v1, 13); + case 14: return vextq_u8(v0, v1, 14); + case 15: return vextq_u8(v0, v1, 15); + } + return vdupq_n_u8(0); +} + +/** + * Shifts right 128 bit register by specified number of bytes + * + * Value of shift parameter must be in range 0 - 16 + */ +static inline uint64x2_t +vshift_bytes_right(uint64x2_t reg, const unsigned int shift) +{ + return vreinterpretq_u64_u8(vextract( + vreinterpretq_u8_u64(reg), + vdupq_n_u8(0), + shift)); +} + +/** + * Shifts left 128 bit register by specified number of bytes + * + * Value of shift parameter must be in range 0 - 16 + */ +static inline uint64x2_t +vshift_bytes_left(uint64x2_t reg, const unsigned int shift) +{ + return vreinterpretq_u64_u8(vextract( + vdupq_n_u8(0), + vreinterpretq_u8_u64(reg), + 16 - shift)); +} + #ifdef __cplusplus } #endif diff --git a/dpdk/lib/librte_eal/common/include/arch/ppc_64/rte_atomic.h b/dpdk/lib/librte_eal/common/include/arch/ppc_64/rte_atomic.h index 924e8940..6993dd29 100644 --- a/dpdk/lib/librte_eal/common/include/arch/ppc_64/rte_atomic.h +++ b/dpdk/lib/librte_eal/common/include/arch/ppc_64/rte_atomic.h @@ -46,6 +46,7 @@ extern "C" { #endif +#include #include "generic/rte_atomic.h" /** @@ -63,9 +64,9 @@ extern "C" { * occur before the STORE operations generated after. */ #ifdef RTE_ARCH_64 -#define rte_wmb() {asm volatile("lwsync" : : : "memory"); } +#define rte_wmb() asm volatile("lwsync" : : : "memory") #else -#define rte_wmb() {asm volatile("sync" : : : "memory"); } +#define rte_wmb() asm volatile("sync" : : : "memory") #endif /** @@ -75,9 +76,9 @@ extern "C" { * occur before the LOAD operations generated after. */ #ifdef RTE_ARCH_64 -#define rte_rmb() {asm volatile("lwsync" : : : "memory"); } +#define rte_rmb() asm volatile("lwsync" : : : "memory") #else -#define rte_rmb() {asm volatile("sync" : : : "memory"); } +#define rte_rmb() asm volatile("sync" : : : "memory") #endif #define rte_smp_mb() rte_mb() @@ -86,6 +87,12 @@ extern "C" { #define rte_smp_rmb() rte_rmb() +#define rte_io_mb() rte_mb() + +#define rte_io_wmb() rte_wmb() + +#define rte_io_rmb() rte_rmb() + /*------------------------- 16 bit atomic operations -------------------------*/ /* To be compatible with Power7, use GCC built-in functions for 16 bit * operations */ diff --git a/dpdk/lib/librte_eal/common/include/arch/ppc_64/rte_byteorder.h b/dpdk/lib/librte_eal/common/include/arch/ppc_64/rte_byteorder.h index 3c1734ed..544de3c2 100644 --- a/dpdk/lib/librte_eal/common/include/arch/ppc_64/rte_byteorder.h +++ b/dpdk/lib/librte_eal/common/include/arch/ppc_64/rte_byteorder.h @@ -42,6 +42,7 @@ extern "C" { #endif +#include #include "generic/rte_byteorder.h" /* diff --git a/dpdk/lib/librte_eal/common/include/arch/ppc_64/rte_cycles.h b/dpdk/lib/librte_eal/common/include/arch/ppc_64/rte_cycles.h index 64beddf9..8fa6fc60 100644 --- a/dpdk/lib/librte_eal/common/include/arch/ppc_64/rte_cycles.h +++ b/dpdk/lib/librte_eal/common/include/arch/ppc_64/rte_cycles.h @@ -40,6 +40,7 @@ extern "C" { #include "generic/rte_cycles.h" #include +#include /** * Read the time base register. @@ -52,6 +53,7 @@ rte_rdtsc(void) { union { uint64_t tsc_64; + RTE_STD_C11 struct { #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN uint32_t hi_32; diff --git a/dpdk/lib/librte_eal/common/include/arch/ppc_64/rte_io.h b/dpdk/lib/librte_eal/common/include/arch/ppc_64/rte_io.h new file mode 100644 index 00000000..1f42ced5 --- /dev/null +++ b/dpdk/lib/librte_eal/common/include/arch/ppc_64/rte_io.h @@ -0,0 +1,47 @@ +/* + * BSD LICENSE + * + * Copyright(c) 2016 Cavium, Inc. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_IO_PPC_64_H_ +#define _RTE_IO_PPC_64_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "generic/rte_io.h" + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_IO_PPC_64_H_ */ diff --git a/dpdk/lib/librte_eal/common/include/arch/ppc_64/rte_memcpy.h b/dpdk/lib/librte_eal/common/include/arch/ppc_64/rte_memcpy.h index acf7aac2..75f74897 100644 --- a/dpdk/lib/librte_eal/common/include/arch/ppc_64/rte_memcpy.h +++ b/dpdk/lib/librte_eal/common/include/arch/ppc_64/rte_memcpy.h @@ -95,7 +95,8 @@ rte_mov256(uint8_t *dst, const uint8_t *src) } #define rte_memcpy(dst, src, n) \ - ({ (__builtin_constant_p(n)) ? \ + __extension__ ({ \ + (__builtin_constant_p(n)) ? \ memcpy((dst), (src), (n)) : \ rte_memcpy_func((dst), (src), (n)); }) @@ -163,7 +164,7 @@ rte_memcpy_func(void *dst, const void *src, size_t n) * We split the remaining bytes (which will be less than 256) into * 64byte (2^6) chunks. * Using incrementing integers in the case labels of a switch statement - * enourages the compiler to use a jump table. To get incrementing + * encourages the compiler to use a jump table. To get incrementing * integers, we shift the 2 relevant bits to the LSB position to first * get decrementing integers, and then subtract. */ diff --git a/dpdk/lib/librte_eal/common/include/arch/ppc_64/rte_pause.h b/dpdk/lib/librte_eal/common/include/arch/ppc_64/rte_pause.h new file mode 100644 index 00000000..389682ca --- /dev/null +++ b/dpdk/lib/librte_eal/common/include/arch/ppc_64/rte_pause.h @@ -0,0 +1,51 @@ +/*- + * BSD LICENSE + * + * Copyright(c) Cavium, Inc. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_PAUSE_PPC64_H_ +#define _RTE_PAUSE_PPC64_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "generic/rte_pause.h" + +static inline void rte_pause(void) +{ +} + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_PAUSE_PPC64_H_ */ diff --git a/dpdk/lib/librte_eal/common/include/arch/ppc_64/rte_prefetch.h b/dpdk/lib/librte_eal/common/include/arch/ppc_64/rte_prefetch.h index 9a1995ea..fd2e53b9 100644 --- a/dpdk/lib/librte_eal/common/include/arch/ppc_64/rte_prefetch.h +++ b/dpdk/lib/librte_eal/common/include/arch/ppc_64/rte_prefetch.h @@ -37,6 +37,7 @@ extern "C" { #endif +#include #include "generic/rte_prefetch.h" static inline void rte_prefetch0(const volatile void *p) diff --git a/dpdk/lib/librte_eal/common/include/arch/ppc_64/rte_spinlock.h b/dpdk/lib/librte_eal/common/include/arch/ppc_64/rte_spinlock.h index af139c9d..39815d9e 100644 --- a/dpdk/lib/librte_eal/common/include/arch/ppc_64/rte_spinlock.h +++ b/dpdk/lib/librte_eal/common/include/arch/ppc_64/rte_spinlock.h @@ -38,6 +38,7 @@ extern "C" { #endif #include +#include #include "generic/rte_spinlock.h" /* Fixme: Use intrinsics to implement the spinlock on Power architecture */ diff --git a/dpdk/lib/librte_eal/common/include/arch/ppc_64/rte_vect.h b/dpdk/lib/librte_eal/common/include/arch/ppc_64/rte_vect.h new file mode 100644 index 00000000..99586e58 --- /dev/null +++ b/dpdk/lib/librte_eal/common/include/arch/ppc_64/rte_vect.h @@ -0,0 +1,61 @@ +/* + * BSD LICENSE + * + * Copyright (C) IBM Corporation 2016. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of IBM Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef _RTE_VECT_PPC_64_H_ +#define _RTE_VECT_PPC_64_H_ + +#include +#include "generic/rte_vect.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef vector signed int xmm_t; + +#define XMM_SIZE (sizeof(xmm_t)) +#define XMM_MASK (XMM_SIZE - 1) + +typedef union rte_xmm { + xmm_t x; + uint8_t u8[XMM_SIZE / sizeof(uint8_t)]; + uint16_t u16[XMM_SIZE / sizeof(uint16_t)]; + uint32_t u32[XMM_SIZE / sizeof(uint32_t)]; + uint64_t u64[XMM_SIZE / sizeof(uint64_t)]; + double pd[XMM_SIZE / sizeof(double)]; +} __attribute__((aligned(16))) rte_xmm_t; + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_VECT_PPC_64_H_ */ diff --git a/dpdk/lib/librte_eal/common/include/arch/tile/rte_atomic.h b/dpdk/lib/librte_eal/common/include/arch/tile/rte_atomic.h deleted file mode 100644 index 28825ff6..00000000 --- a/dpdk/lib/librte_eal/common/include/arch/tile/rte_atomic.h +++ /dev/null @@ -1,92 +0,0 @@ -/* - * BSD LICENSE - * - * Copyright (C) EZchip Semiconductor Ltd. 2015. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of EZchip Semiconductor nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - -#ifndef _RTE_ATOMIC_TILE_H_ -#define _RTE_ATOMIC_TILE_H_ - -#ifndef RTE_FORCE_INTRINSICS -# error Platform must be built with CONFIG_RTE_FORCE_INTRINSICS -#endif - -#ifdef __cplusplus -extern "C" { -#endif - -#include "generic/rte_atomic.h" - -/** - * General memory barrier. - * - * Guarantees that the LOAD and STORE operations generated before the - * barrier occur before the LOAD and STORE operations generated after. - * This function is architecture dependent. - */ -static inline void rte_mb(void) -{ - __sync_synchronize(); -} - -/** - * Write memory barrier. - * - * Guarantees that the STORE operations generated before the barrier - * occur before the STORE operations generated after. - * This function is architecture dependent. - */ -static inline void rte_wmb(void) -{ - __sync_synchronize(); -} - -/** - * Read memory barrier. - * - * Guarantees that the LOAD operations generated before the barrier - * occur before the LOAD operations generated after. - * This function is architecture dependent. - */ -static inline void rte_rmb(void) -{ - __sync_synchronize(); -} - -#define rte_smp_mb() rte_mb() - -#define rte_smp_wmb() rte_compiler_barrier() - -#define rte_smp_rmb() rte_compiler_barrier() - -#ifdef __cplusplus -} -#endif - -#endif /* _RTE_ATOMIC_TILE_H_ */ diff --git a/dpdk/lib/librte_eal/common/include/arch/tile/rte_byteorder.h b/dpdk/lib/librte_eal/common/include/arch/tile/rte_byteorder.h deleted file mode 100644 index 7239e437..00000000 --- a/dpdk/lib/librte_eal/common/include/arch/tile/rte_byteorder.h +++ /dev/null @@ -1,91 +0,0 @@ -/* - * BSD LICENSE - * - * Copyright (C) EZchip Semiconductor Ltd. 2015. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of EZchip Semiconductor nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - -#ifndef _RTE_BYTEORDER_TILE_H_ -#define _RTE_BYTEORDER_TILE_H_ - -#ifndef RTE_FORCE_INTRINSICS -# error Platform must be built with CONFIG_RTE_FORCE_INTRINSICS -#endif - -#ifdef __cplusplus -extern "C" { -#endif - -#include "generic/rte_byteorder.h" - -#if !(__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)) -#define rte_bswap16(x) rte_constant_bswap16(x) -#endif - -#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN - -#define rte_cpu_to_le_16(x) (x) -#define rte_cpu_to_le_32(x) (x) -#define rte_cpu_to_le_64(x) (x) - -#define rte_cpu_to_be_16(x) rte_bswap16(x) -#define rte_cpu_to_be_32(x) rte_bswap32(x) -#define rte_cpu_to_be_64(x) rte_bswap64(x) - -#define rte_le_to_cpu_16(x) (x) -#define rte_le_to_cpu_32(x) (x) -#define rte_le_to_cpu_64(x) (x) - -#define rte_be_to_cpu_16(x) rte_bswap16(x) -#define rte_be_to_cpu_32(x) rte_bswap32(x) -#define rte_be_to_cpu_64(x) rte_bswap64(x) - -#else /* RTE_BIG_ENDIAN */ - -#define rte_cpu_to_le_16(x) rte_bswap16(x) -#define rte_cpu_to_le_32(x) rte_bswap32(x) -#define rte_cpu_to_le_64(x) rte_bswap64(x) - -#define rte_cpu_to_be_16(x) (x) -#define rte_cpu_to_be_32(x) (x) -#define rte_cpu_to_be_64(x) (x) - -#define rte_le_to_cpu_16(x) rte_bswap16(x) -#define rte_le_to_cpu_32(x) rte_bswap32(x) -#define rte_le_to_cpu_64(x) rte_bswap64(x) - -#define rte_be_to_cpu_16(x) (x) -#define rte_be_to_cpu_32(x) (x) -#define rte_be_to_cpu_64(x) (x) -#endif - -#ifdef __cplusplus -} -#endif - -#endif /* _RTE_BYTEORDER_TILE_H_ */ diff --git a/dpdk/lib/librte_eal/common/include/arch/tile/rte_cpuflags.h b/dpdk/lib/librte_eal/common/include/arch/tile/rte_cpuflags.h deleted file mode 100644 index 1849b520..00000000 --- a/dpdk/lib/librte_eal/common/include/arch/tile/rte_cpuflags.h +++ /dev/null @@ -1,53 +0,0 @@ -/* - * BSD LICENSE - * - * Copyright (C) EZchip Semiconductor Ltd. 2015. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of EZchip Semiconductor nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - -#ifndef _RTE_CPUFLAGS_TILE_H_ -#define _RTE_CPUFLAGS_TILE_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -/** - * Enumeration of all CPU features supported - */ -enum rte_cpu_flag_t { - RTE_CPUFLAG_NUMFLAGS /**< This should always be the last! */ -}; - -#include "generic/rte_cpuflags.h" - -#ifdef __cplusplus -} -#endif - -#endif /* _RTE_CPUFLAGS_TILE_H_ */ diff --git a/dpdk/lib/librte_eal/common/include/arch/tile/rte_cycles.h b/dpdk/lib/librte_eal/common/include/arch/tile/rte_cycles.h deleted file mode 100644 index 0b2200a3..00000000 --- a/dpdk/lib/librte_eal/common/include/arch/tile/rte_cycles.h +++ /dev/null @@ -1,70 +0,0 @@ -/* - * BSD LICENSE - * - * Copyright (C) EZchip Semiconductor Ltd. 2015. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of EZchip Semiconductor nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - -#ifndef _RTE_CYCLES_TILE_H_ -#define _RTE_CYCLES_TILE_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include - -#include "generic/rte_cycles.h" - -/** - * Read the time base register. - * - * @return - * The time base for this lcore. - */ -static inline uint64_t -rte_rdtsc(void) -{ - return get_cycle_count(); -} - -static inline uint64_t -rte_rdtsc_precise(void) -{ - rte_mb(); - return rte_rdtsc(); -} - -static inline uint64_t -rte_get_tsc_cycles(void) { return rte_rdtsc(); } - -#ifdef __cplusplus -} -#endif - -#endif /* _RTE_CYCLES_TILE_H_ */ diff --git a/dpdk/lib/librte_eal/common/include/arch/tile/rte_memcpy.h b/dpdk/lib/librte_eal/common/include/arch/tile/rte_memcpy.h deleted file mode 100644 index e606957c..00000000 --- a/dpdk/lib/librte_eal/common/include/arch/tile/rte_memcpy.h +++ /dev/null @@ -1,87 +0,0 @@ -/* - * BSD LICENSE - * - * Copyright (C) EZchip Semiconductor Ltd. 2015. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of EZchip Semiconductor nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - -#ifndef _RTE_MEMCPY_TILE_H_ -#define _RTE_MEMCPY_TILE_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include - -#include "generic/rte_memcpy.h" - -static inline void -rte_mov16(uint8_t *dst, const uint8_t *src) -{ - memcpy(dst, src, 16); -} - -static inline void -rte_mov32(uint8_t *dst, const uint8_t *src) -{ - memcpy(dst, src, 32); -} - -static inline void -rte_mov48(uint8_t *dst, const uint8_t *src) -{ - memcpy(dst, src, 48); -} - -static inline void -rte_mov64(uint8_t *dst, const uint8_t *src) -{ - memcpy(dst, src, 64); -} - -static inline void -rte_mov128(uint8_t *dst, const uint8_t *src) -{ - memcpy(dst, src, 128); -} - -static inline void -rte_mov256(uint8_t *dst, const uint8_t *src) -{ - memcpy(dst, src, 256); -} - -#define rte_memcpy(d, s, n) memcpy((d), (s), (n)) - -#ifdef __cplusplus -} -#endif - -#endif /* _RTE_MEMCPY_TILE_H_ */ diff --git a/dpdk/lib/librte_eal/common/include/arch/tile/rte_prefetch.h b/dpdk/lib/librte_eal/common/include/arch/tile/rte_prefetch.h deleted file mode 100644 index 7a1bb93e..00000000 --- a/dpdk/lib/librte_eal/common/include/arch/tile/rte_prefetch.h +++ /dev/null @@ -1,67 +0,0 @@ -/* - * BSD LICENSE - * - * Copyright (C) EZchip Semiconductor Ltd. 2015. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of EZchip Semiconductor nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - -#ifndef _RTE_PREFETCH_TILE_H_ -#define _RTE_PREFETCH_TILE_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include "generic/rte_prefetch.h" - -static inline void rte_prefetch0(const volatile void *p) -{ - __builtin_prefetch((const void *)(uintptr_t)p, 0, 3); -} - -static inline void rte_prefetch1(const volatile void *p) -{ - __builtin_prefetch((const void *)(uintptr_t)p, 0, 2); -} - -static inline void rte_prefetch2(const volatile void *p) -{ - __builtin_prefetch((const void *)(uintptr_t)p, 0, 1); -} - -static inline void rte_prefetch_non_temporal(const volatile void *p) -{ - /* non-temporal version not available, fallback to rte_prefetch0 */ - rte_prefetch0(p); -} - -#ifdef __cplusplus -} -#endif - -#endif /* _RTE_PREFETCH_TILE_H_ */ diff --git a/dpdk/lib/librte_eal/common/include/arch/tile/rte_rwlock.h b/dpdk/lib/librte_eal/common/include/arch/tile/rte_rwlock.h deleted file mode 100644 index 8f67a190..00000000 --- a/dpdk/lib/librte_eal/common/include/arch/tile/rte_rwlock.h +++ /dev/null @@ -1,70 +0,0 @@ -/* - * BSD LICENSE - * - * Copyright (C) EZchip Semiconductor Ltd. 2015. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of EZchip Semiconductor nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - -#ifndef _RTE_RWLOCK_TILE_H_ -#define _RTE_RWLOCK_TILE_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include "generic/rte_rwlock.h" - -static inline void -rte_rwlock_read_lock_tm(rte_rwlock_t *rwl) -{ - rte_rwlock_read_lock(rwl); -} - -static inline void -rte_rwlock_read_unlock_tm(rte_rwlock_t *rwl) -{ - rte_rwlock_read_unlock(rwl); -} - -static inline void -rte_rwlock_write_lock_tm(rte_rwlock_t *rwl) -{ - rte_rwlock_write_lock(rwl); -} - -static inline void -rte_rwlock_write_unlock_tm(rte_rwlock_t *rwl) -{ - rte_rwlock_write_unlock(rwl); -} - -#ifdef __cplusplus -} -#endif - -#endif /* _RTE_RWLOCK_TILE_H_ */ diff --git a/dpdk/lib/librte_eal/common/include/arch/tile/rte_spinlock.h b/dpdk/lib/librte_eal/common/include/arch/tile/rte_spinlock.h deleted file mode 100644 index e91f99ee..00000000 --- a/dpdk/lib/librte_eal/common/include/arch/tile/rte_spinlock.h +++ /dev/null @@ -1,92 +0,0 @@ -/* - * BSD LICENSE - * - * Copyright (C) EZchip Semiconductor Ltd. 2015. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of EZchip Semiconductor nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - -#ifndef _RTE_SPINLOCK_TILE_H_ -#define _RTE_SPINLOCK_TILE_H_ - -#ifndef RTE_FORCE_INTRINSICS -# error Platform must be built with CONFIG_RTE_FORCE_INTRINSICS -#endif - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include "generic/rte_spinlock.h" - -static inline int rte_tm_supported(void) -{ - return 0; -} - -static inline void -rte_spinlock_lock_tm(rte_spinlock_t *sl) -{ - rte_spinlock_lock(sl); /* fall-back */ -} - -static inline int -rte_spinlock_trylock_tm(rte_spinlock_t *sl) -{ - return rte_spinlock_trylock(sl); -} - -static inline void -rte_spinlock_unlock_tm(rte_spinlock_t *sl) -{ - rte_spinlock_unlock(sl); -} - -static inline void -rte_spinlock_recursive_lock_tm(rte_spinlock_recursive_t *slr) -{ - rte_spinlock_recursive_lock(slr); /* fall-back */ -} - -static inline void -rte_spinlock_recursive_unlock_tm(rte_spinlock_recursive_t *slr) -{ - rte_spinlock_recursive_unlock(slr); -} - -static inline int -rte_spinlock_recursive_trylock_tm(rte_spinlock_recursive_t *slr) -{ - return rte_spinlock_recursive_trylock(slr); -} - -#ifdef __cplusplus -} -#endif - -#endif /* _RTE_SPINLOCK_TILE_H_ */ diff --git a/dpdk/lib/librte_eal/common/include/arch/x86/rte_atomic.h b/dpdk/lib/librte_eal/common/include/arch/x86/rte_atomic.h index b20056b8..ea665d52 100644 --- a/dpdk/lib/librte_eal/common/include/arch/x86/rte_atomic.h +++ b/dpdk/lib/librte_eal/common/include/arch/x86/rte_atomic.h @@ -38,6 +38,9 @@ extern "C" { #endif +#include +#include +#include #include #include "generic/rte_atomic.h" @@ -59,6 +62,12 @@ extern "C" { #define rte_smp_rmb() rte_compiler_barrier() +#define rte_io_mb() rte_mb() + +#define rte_io_wmb() rte_compiler_barrier() + +#define rte_io_rmb() rte_compiler_barrier() + /*------------------------- 16 bit atomic operations -------------------------*/ #ifndef RTE_FORCE_INTRINSICS diff --git a/dpdk/lib/librte_eal/common/include/arch/x86/rte_atomic_32.h b/dpdk/lib/librte_eal/common/include/arch/x86/rte_atomic_32.h index 400d8a96..fb3abf18 100644 --- a/dpdk/lib/librte_eal/common/include/arch/x86/rte_atomic_32.h +++ b/dpdk/lib/librte_eal/common/include/arch/x86/rte_atomic_32.h @@ -37,9 +37,17 @@ * All rights reserved. */ +#ifndef _RTE_ATOMIC_X86_H_ +#error do not include this file directly, use instead +#endif + #ifndef _RTE_ATOMIC_I686_H_ #define _RTE_ATOMIC_I686_H_ +#include +#include +#include + /*------------------------- 64 bit atomic operations -------------------------*/ #ifndef RTE_FORCE_INTRINSICS @@ -47,6 +55,7 @@ static inline int rte_atomic64_cmpset(volatile uint64_t *dst, uint64_t exp, uint64_t src) { uint8_t res; + RTE_STD_C11 union { struct { uint32_t l32; @@ -72,7 +81,7 @@ rte_atomic64_cmpset(volatile uint64_t *dst, uint64_t exp, uint64_t src) : "memory" ); /* no-clobber list */ #else asm volatile ( - "mov %%ebx, %%edi\n" + "xchgl %%ebx, %%edi;\n" MPLOCKED "cmpxchg8b (%[dst]);" "setz %[res];" diff --git a/dpdk/lib/librte_eal/common/include/arch/x86/rte_atomic_64.h b/dpdk/lib/librte_eal/common/include/arch/x86/rte_atomic_64.h index 4de66000..1a53a766 100644 --- a/dpdk/lib/librte_eal/common/include/arch/x86/rte_atomic_64.h +++ b/dpdk/lib/librte_eal/common/include/arch/x86/rte_atomic_64.h @@ -37,9 +37,17 @@ * All rights reserved. */ +#ifndef _RTE_ATOMIC_X86_H_ +#error do not include this file directly, use instead +#endif + #ifndef _RTE_ATOMIC_X86_64_H_ #define _RTE_ATOMIC_X86_64_H_ +#include +#include +#include + /*------------------------- 64 bit atomic operations -------------------------*/ #ifndef RTE_FORCE_INTRINSICS diff --git a/dpdk/lib/librte_eal/common/include/arch/x86/rte_byteorder.h b/dpdk/lib/librte_eal/common/include/arch/x86/rte_byteorder.h index ffdb6ef5..126f29e0 100644 --- a/dpdk/lib/librte_eal/common/include/arch/x86/rte_byteorder.h +++ b/dpdk/lib/librte_eal/common/include/arch/x86/rte_byteorder.h @@ -38,6 +38,9 @@ extern "C" { #endif +#include +#include +#include #include "generic/rte_byteorder.h" #ifndef RTE_BYTE_ORDER diff --git a/dpdk/lib/librte_eal/common/include/arch/x86/rte_byteorder_32.h b/dpdk/lib/librte_eal/common/include/arch/x86/rte_byteorder_32.h index 51c306f8..14d64834 100644 --- a/dpdk/lib/librte_eal/common/include/arch/x86/rte_byteorder_32.h +++ b/dpdk/lib/librte_eal/common/include/arch/x86/rte_byteorder_32.h @@ -31,9 +31,16 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +#ifndef _RTE_BYTEORDER_X86_H_ +#error do not include this file directly, use instead +#endif + #ifndef _RTE_BYTEORDER_I686_H_ #define _RTE_BYTEORDER_I686_H_ +#include +#include + /* * An architecture-optimized byte swap for a 64-bit value. * diff --git a/dpdk/lib/librte_eal/common/include/arch/x86/rte_byteorder_64.h b/dpdk/lib/librte_eal/common/include/arch/x86/rte_byteorder_64.h index dda572bd..516ac052 100644 --- a/dpdk/lib/librte_eal/common/include/arch/x86/rte_byteorder_64.h +++ b/dpdk/lib/librte_eal/common/include/arch/x86/rte_byteorder_64.h @@ -31,9 +31,16 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +#ifndef _RTE_BYTEORDER_X86_H_ +#error do not include this file directly, use instead +#endif + #ifndef _RTE_BYTEORDER_X86_64_H_ #define _RTE_BYTEORDER_X86_64_H_ +#include +#include + /* * An architecture-optimized byte swap for a 64-bit value. * diff --git a/dpdk/lib/librte_eal/common/include/arch/x86/rte_cycles.h b/dpdk/lib/librte_eal/common/include/arch/x86/rte_cycles.h index 6e3c7d89..b95beced 100644 --- a/dpdk/lib/librte_eal/common/include/arch/x86/rte_cycles.h +++ b/dpdk/lib/librte_eal/common/include/arch/x86/rte_cycles.h @@ -2,6 +2,7 @@ * BSD LICENSE * * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * Copyright(c) 2013 6WIND. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -30,36 +31,6 @@ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -/* BSD LICENSE - * - * Copyright(c) 2013 6WIND. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of 6WIND S.A. nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ #ifndef _RTE_CYCLES_X86_64_H_ #define _RTE_CYCLES_X86_64_H_ @@ -75,12 +46,15 @@ extern "C" { extern int rte_cycles_vmware_tsc_map; #include #endif +#include +#include static inline uint64_t rte_rdtsc(void) { union { uint64_t tsc_64; + RTE_STD_C11 struct { uint32_t lo_32; uint32_t hi_32; diff --git a/dpdk/lib/librte_eal/common/include/arch/x86/rte_io.h b/dpdk/lib/librte_eal/common/include/arch/x86/rte_io.h new file mode 100644 index 00000000..130022d0 --- /dev/null +++ b/dpdk/lib/librte_eal/common/include/arch/x86/rte_io.h @@ -0,0 +1,47 @@ +/* + * BSD LICENSE + * + * Copyright(c) 2016 Cavium, Inc. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_IO_X86_H_ +#define _RTE_IO_X86_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "generic/rte_io.h" + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_IO_X86_H_ */ diff --git a/dpdk/lib/librte_eal/common/include/arch/x86/rte_memcpy.h b/dpdk/lib/librte_eal/common/include/arch/x86/rte_memcpy.h index 413035e7..596d7779 100644 --- a/dpdk/lib/librte_eal/common/include/arch/x86/rte_memcpy.h +++ b/dpdk/lib/librte_eal/common/include/arch/x86/rte_memcpy.h @@ -44,6 +44,8 @@ #include #include #include +#include +#include #ifdef __cplusplus extern "C" { @@ -64,11 +66,13 @@ extern "C" { * @return * Pointer to the destination data. */ -static inline void * -rte_memcpy(void *dst, const void *src, size_t n) __attribute__((always_inline)); +static __rte_always_inline void * +rte_memcpy(void *dst, const void *src, size_t n); #ifdef RTE_MACHINE_CPUFLAG_AVX512F +#define ALIGNMENT_MASK 0x3F + /** * AVX512 implementation below */ @@ -189,7 +193,7 @@ rte_mov512blocks(uint8_t *dst, const uint8_t *src, size_t n) } static inline void * -rte_memcpy(void *dst, const void *src, size_t n) +rte_memcpy_generic(void *dst, const void *src, size_t n) { uintptr_t dstu = (uintptr_t)dst; uintptr_t srcu = (uintptr_t)src; @@ -308,6 +312,8 @@ COPY_BLOCK_128_BACK63: #elif defined RTE_MACHINE_CPUFLAG_AVX2 +#define ALIGNMENT_MASK 0x1F + /** * AVX2 implementation below */ @@ -387,7 +393,7 @@ rte_mov128blocks(uint8_t *dst, const uint8_t *src, size_t n) } static inline void * -rte_memcpy(void *dst, const void *src, size_t n) +rte_memcpy_generic(void *dst, const void *src, size_t n) { uintptr_t dstu = (uintptr_t)dst; uintptr_t srcu = (uintptr_t)src; @@ -499,6 +505,8 @@ COPY_BLOCK_128_BACK31: #else /* RTE_MACHINE_CPUFLAG */ +#define ALIGNMENT_MASK 0x0F + /** * SSE & AVX implementation below */ @@ -594,7 +602,7 @@ rte_mov256(uint8_t *dst, const uint8_t *src) * - __m128i ~ must be pre-defined */ #define MOVEUNALIGNED_LEFT47_IMM(dst, src, len, offset) \ -({ \ +__extension__ ({ \ int tmp; \ while (len >= 128 + 16 - offset) { \ xmm0 = _mm_loadu_si128((const __m128i *)((const uint8_t *)src - offset + 0 * 16)); \ @@ -655,7 +663,7 @@ rte_mov256(uint8_t *dst, const uint8_t *src) * - __m128i ~ used in MOVEUNALIGNED_LEFT47_IMM must be pre-defined */ #define MOVEUNALIGNED_LEFT47(dst, src, len, offset) \ -({ \ +__extension__ ({ \ switch (offset) { \ case 0x01: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x01); break; \ case 0x02: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x02); break; \ @@ -677,7 +685,7 @@ rte_mov256(uint8_t *dst, const uint8_t *src) }) static inline void * -rte_memcpy(void *dst, const void *src, size_t n) +rte_memcpy_generic(void *dst, const void *src, size_t n) { __m128i xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8; uintptr_t dstu = (uintptr_t)dst; @@ -821,6 +829,75 @@ COPY_BLOCK_64_BACK15: #endif /* RTE_MACHINE_CPUFLAG */ +static inline void * +rte_memcpy_aligned(void *dst, const void *src, size_t n) +{ + void *ret = dst; + + /* Copy size <= 16 bytes */ + if (n < 16) { + if (n & 0x01) { + *(uint8_t *)dst = *(const uint8_t *)src; + src = (const uint8_t *)src + 1; + dst = (uint8_t *)dst + 1; + } + if (n & 0x02) { + *(uint16_t *)dst = *(const uint16_t *)src; + src = (const uint16_t *)src + 1; + dst = (uint16_t *)dst + 1; + } + if (n & 0x04) { + *(uint32_t *)dst = *(const uint32_t *)src; + src = (const uint32_t *)src + 1; + dst = (uint32_t *)dst + 1; + } + if (n & 0x08) + *(uint64_t *)dst = *(const uint64_t *)src; + + return ret; + } + + /* Copy 16 <= size <= 32 bytes */ + if (n <= 32) { + rte_mov16((uint8_t *)dst, (const uint8_t *)src); + rte_mov16((uint8_t *)dst - 16 + n, + (const uint8_t *)src - 16 + n); + + return ret; + } + + /* Copy 32 < size <= 64 bytes */ + if (n <= 64) { + rte_mov32((uint8_t *)dst, (const uint8_t *)src); + rte_mov32((uint8_t *)dst - 32 + n, + (const uint8_t *)src - 32 + n); + + return ret; + } + + /* Copy 64 bytes blocks */ + for (; n >= 64; n -= 64) { + rte_mov64((uint8_t *)dst, (const uint8_t *)src); + dst = (uint8_t *)dst + 64; + src = (const uint8_t *)src + 64; + } + + /* Copy whatever left */ + rte_mov64((uint8_t *)dst - 64 + n, + (const uint8_t *)src - 64 + n); + + return ret; +} + +static inline void * +rte_memcpy(void *dst, const void *src, size_t n) +{ + if (!(((uintptr_t)dst | (uintptr_t)src) & ALIGNMENT_MASK)) + return rte_memcpy_aligned(dst, src, n); + else + return rte_memcpy_generic(dst, src, n); +} + #ifdef __cplusplus } #endif diff --git a/dpdk/lib/librte_eal/common/include/arch/x86/rte_pause.h b/dpdk/lib/librte_eal/common/include/arch/x86/rte_pause.h new file mode 100644 index 00000000..29130c4b --- /dev/null +++ b/dpdk/lib/librte_eal/common/include/arch/x86/rte_pause.h @@ -0,0 +1,53 @@ +/*- + * BSD LICENSE + * + * Copyright(c) Cavium, Inc. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_PAUSE_X86_H_ +#define _RTE_PAUSE_X86_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "generic/rte_pause.h" + +#include +static inline void rte_pause(void) +{ + _mm_pause(); +} + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_PAUSE_X86_H_ */ diff --git a/dpdk/lib/librte_eal/common/include/arch/x86/rte_prefetch.h b/dpdk/lib/librte_eal/common/include/arch/x86/rte_prefetch.h index 5dac47eb..f464398f 100644 --- a/dpdk/lib/librte_eal/common/include/arch/x86/rte_prefetch.h +++ b/dpdk/lib/librte_eal/common/include/arch/x86/rte_prefetch.h @@ -38,6 +38,7 @@ extern "C" { #endif +#include #include "generic/rte_prefetch.h" static inline void rte_prefetch0(const volatile void *p) diff --git a/dpdk/lib/librte_eal/common/include/arch/x86/rte_rtm.h b/dpdk/lib/librte_eal/common/include/arch/x86/rte_rtm.h index 0649f794..ab099952 100644 --- a/dpdk/lib/librte_eal/common/include/arch/x86/rte_rtm.h +++ b/dpdk/lib/librte_eal/common/include/arch/x86/rte_rtm.h @@ -20,6 +20,7 @@ /* Official RTM intrinsics interface matching gcc/icc, but works on older gcc compatible compilers and binutils. */ +#include #ifdef __cplusplus extern "C" { diff --git a/dpdk/lib/librte_eal/common/include/arch/x86/rte_spinlock.h b/dpdk/lib/librte_eal/common/include/arch/x86/rte_spinlock.h index 8e630c21..5675c2b4 100644 --- a/dpdk/lib/librte_eal/common/include/arch/x86/rte_spinlock.h +++ b/dpdk/lib/librte_eal/common/include/arch/x86/rte_spinlock.h @@ -43,6 +43,7 @@ extern "C" { #include "rte_cpuflags.h" #include "rte_branch_prediction.h" #include "rte_common.h" +#include "rte_pause.h" #define RTE_RTM_MAX_RETRIES (10) #define RTE_XABORT_LOCK_BUSY (0xff) diff --git a/dpdk/lib/librte_eal/common/include/arch/x86/rte_vect.h b/dpdk/lib/librte_eal/common/include/arch/x86/rte_vect.h index b698797c..893b8120 100644 --- a/dpdk/lib/librte_eal/common/include/arch/x86/rte_vect.h +++ b/dpdk/lib/librte_eal/common/include/arch/x86/rte_vect.h @@ -31,8 +31,8 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#ifndef _RTE_VECT_H_ -#define _RTE_VECT_H_ +#ifndef _RTE_VECT_X86_H_ +#define _RTE_VECT_X86_H_ /** * @file @@ -40,23 +40,13 @@ * RTE SSE/AVX related header. */ +#include +#include +#include "generic/rte_vect.h" + #if (defined(__ICC) || (__GNUC__ == 4 && __GNUC_MINOR__ < 4)) -#ifdef __SSE__ -#include -#endif - -#ifdef __SSE2__ -#include -#endif - -#ifdef __SSE3__ -#include -#endif - -#if defined(__SSE4_2__) || defined(__SSE4_1__) -#include -#endif +#include /* SSE4 */ #if defined(__AVX__) #include @@ -106,7 +96,8 @@ typedef union rte_ymm { #endif /* __AVX__ */ #ifdef RTE_ARCH_I686 -#define _mm_cvtsi128_si64(a) ({ \ +#define _mm_cvtsi128_si64(a) \ +__extension__ ({ \ rte_xmm_t m; \ m.x = (a); \ (m.u64[0]); \ @@ -117,7 +108,8 @@ typedef union rte_ymm { * Prior to version 12.1 icc doesn't support _mm_set_epi64x. */ #if (defined(__ICC) && __ICC < 1210) -#define _mm_set_epi64x(a, b) ({ \ +#define _mm_set_epi64x(a, b) \ +__extension__ ({ \ rte_xmm_t m; \ m.u64[0] = b; \ m.u64[1] = a; \ @@ -129,4 +121,4 @@ typedef union rte_ymm { } #endif -#endif /* _RTE_VECT_H_ */ +#endif /* _RTE_VECT_X86_H_ */ diff --git a/dpdk/lib/librte_eal/common/include/generic/rte_atomic.h b/dpdk/lib/librte_eal/common/include/generic/rte_atomic.h index bfb4fe44..7b81705b 100644 --- a/dpdk/lib/librte_eal/common/include/generic/rte_atomic.h +++ b/dpdk/lib/librte_eal/common/include/generic/rte_atomic.h @@ -42,6 +42,7 @@ */ #include +#include #ifdef __DOXYGEN__ @@ -99,6 +100,33 @@ static inline void rte_smp_wmb(void); */ static inline void rte_smp_rmb(void); +/** + * General memory barrier for I/O device + * + * Guarantees that the LOAD and STORE operations that precede the + * rte_io_mb() call are visible to I/O device or CPU before the + * LOAD and STORE operations that follow it. + */ +static inline void rte_io_mb(void); + +/** + * Write memory barrier for I/O device + * + * Guarantees that the STORE operations that precede the + * rte_io_wmb() call are visible to I/O device before the STORE + * operations that follow it. + */ +static inline void rte_io_wmb(void); + +/** + * Read memory barrier for IO device + * + * Guarantees that the LOAD operations on I/O device that precede the + * rte_io_rmb() call are visible to CPU before the LOAD + * operations that follow it. + */ +static inline void rte_io_rmb(void); + #endif /* __DOXYGEN__ */ /** diff --git a/dpdk/lib/librte_eal/common/include/generic/rte_byteorder.h b/dpdk/lib/librte_eal/common/include/generic/rte_byteorder.h index c46fdcf2..29e70c96 100644 --- a/dpdk/lib/librte_eal/common/include/generic/rte_byteorder.h +++ b/dpdk/lib/librte_eal/common/include/generic/rte_byteorder.h @@ -50,6 +50,9 @@ #include #endif +#include +#include + /* * Compile-time endianness detection */ @@ -72,6 +75,73 @@ #elif defined __LITTLE_ENDIAN__ #define RTE_BYTE_ORDER RTE_LITTLE_ENDIAN #endif +#if !defined(RTE_BYTE_ORDER) +#error Unknown endianness. +#endif + +#define RTE_STATIC_BSWAP16(v) \ + ((((uint16_t)(v) & UINT16_C(0x00ff)) << 8) | \ + (((uint16_t)(v) & UINT16_C(0xff00)) >> 8)) + +#define RTE_STATIC_BSWAP32(v) \ + ((((uint32_t)(v) & UINT32_C(0x000000ff)) << 24) | \ + (((uint32_t)(v) & UINT32_C(0x0000ff00)) << 8) | \ + (((uint32_t)(v) & UINT32_C(0x00ff0000)) >> 8) | \ + (((uint32_t)(v) & UINT32_C(0xff000000)) >> 24)) + +#define RTE_STATIC_BSWAP64(v) \ + ((((uint64_t)(v) & UINT64_C(0x00000000000000ff)) << 56) | \ + (((uint64_t)(v) & UINT64_C(0x000000000000ff00)) << 40) | \ + (((uint64_t)(v) & UINT64_C(0x0000000000ff0000)) << 24) | \ + (((uint64_t)(v) & UINT64_C(0x00000000ff000000)) << 8) | \ + (((uint64_t)(v) & UINT64_C(0x000000ff00000000)) >> 8) | \ + (((uint64_t)(v) & UINT64_C(0x0000ff0000000000)) >> 24) | \ + (((uint64_t)(v) & UINT64_C(0x00ff000000000000)) >> 40) | \ + (((uint64_t)(v) & UINT64_C(0xff00000000000000)) >> 56)) + +/* + * These macros are functionally similar to rte_cpu_to_(be|le)(16|32|64)(), + * they take values in host CPU order and return them converted to the + * intended endianness. + * + * They resolve at compilation time to integer constants which can safely be + * used with static initializers, since those cannot involve function calls. + * + * On the other hand, they are not as optimized as their rte_cpu_to_*() + * counterparts, therefore applications should refrain from using them on + * variable values, particularly inside performance-sensitive code. + */ +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN +#define RTE_BE16(v) (rte_be16_t)(v) +#define RTE_BE32(v) (rte_be32_t)(v) +#define RTE_BE64(v) (rte_be64_t)(v) +#define RTE_LE16(v) (rte_le16_t)(RTE_STATIC_BSWAP16(v)) +#define RTE_LE32(v) (rte_le32_t)(RTE_STATIC_BSWAP32(v)) +#define RTE_LE64(v) (rte_le64_t)(RTE_STATIC_BSWAP64(v)) +#elif RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN +#define RTE_BE16(v) (rte_be16_t)(RTE_STATIC_BSWAP16(v)) +#define RTE_BE32(v) (rte_be32_t)(RTE_STATIC_BSWAP32(v)) +#define RTE_BE64(v) (rte_be64_t)(RTE_STATIC_BSWAP64(v)) +#define RTE_LE16(v) (rte_be16_t)(v) +#define RTE_LE32(v) (rte_be32_t)(v) +#define RTE_LE64(v) (rte_be64_t)(v) +#else +#error Unsupported endianness. +#endif + +/* + * The following types should be used when handling values according to a + * specific byte ordering, which may differ from that of the host CPU. + * + * Libraries, public APIs and applications are encouraged to use them for + * documentation purposes. + */ +typedef uint16_t rte_be16_t; /**< 16-bit big-endian value. */ +typedef uint32_t rte_be32_t; /**< 32-bit big-endian value. */ +typedef uint64_t rte_be64_t; /**< 64-bit big-endian value. */ +typedef uint16_t rte_le16_t; /**< 16-bit little-endian value. */ +typedef uint32_t rte_le32_t; /**< 32-bit little-endian value. */ +typedef uint64_t rte_le64_t; /**< 64-bit little-endian value. */ /* * An internal function to swap bytes in a 16-bit value. @@ -82,8 +152,7 @@ static inline uint16_t rte_constant_bswap16(uint16_t x) { - return (uint16_t)(((x & 0x00ffU) << 8) | - ((x & 0xff00U) >> 8)); + return RTE_STATIC_BSWAP16(x); } /* @@ -95,10 +164,7 @@ rte_constant_bswap16(uint16_t x) static inline uint32_t rte_constant_bswap32(uint32_t x) { - return ((x & 0x000000ffUL) << 24) | - ((x & 0x0000ff00UL) << 8) | - ((x & 0x00ff0000UL) >> 8) | - ((x & 0xff000000UL) >> 24); + return RTE_STATIC_BSWAP32(x); } /* @@ -110,14 +176,7 @@ rte_constant_bswap32(uint32_t x) static inline uint64_t rte_constant_bswap64(uint64_t x) { - return ((x & 0x00000000000000ffULL) << 56) | - ((x & 0x000000000000ff00ULL) << 40) | - ((x & 0x0000000000ff0000ULL) << 24) | - ((x & 0x00000000ff000000ULL) << 8) | - ((x & 0x000000ff00000000ULL) >> 8) | - ((x & 0x0000ff0000000000ULL) >> 24) | - ((x & 0x00ff000000000000ULL) >> 40) | - ((x & 0xff00000000000000ULL) >> 56); + return RTE_STATIC_BSWAP64(x); } @@ -141,65 +200,65 @@ static uint64_t rte_bswap64(uint64_t x); /** * Convert a 16-bit value from CPU order to little endian. */ -static uint16_t rte_cpu_to_le_16(uint16_t x); +static rte_le16_t rte_cpu_to_le_16(uint16_t x); /** * Convert a 32-bit value from CPU order to little endian. */ -static uint32_t rte_cpu_to_le_32(uint32_t x); +static rte_le32_t rte_cpu_to_le_32(uint32_t x); /** * Convert a 64-bit value from CPU order to little endian. */ -static uint64_t rte_cpu_to_le_64(uint64_t x); +static rte_le64_t rte_cpu_to_le_64(uint64_t x); /** * Convert a 16-bit value from CPU order to big endian. */ -static uint16_t rte_cpu_to_be_16(uint16_t x); +static rte_be16_t rte_cpu_to_be_16(uint16_t x); /** * Convert a 32-bit value from CPU order to big endian. */ -static uint32_t rte_cpu_to_be_32(uint32_t x); +static rte_be32_t rte_cpu_to_be_32(uint32_t x); /** * Convert a 64-bit value from CPU order to big endian. */ -static uint64_t rte_cpu_to_be_64(uint64_t x); +static rte_be64_t rte_cpu_to_be_64(uint64_t x); /** * Convert a 16-bit value from little endian to CPU order. */ -static uint16_t rte_le_to_cpu_16(uint16_t x); +static uint16_t rte_le_to_cpu_16(rte_le16_t x); /** * Convert a 32-bit value from little endian to CPU order. */ -static uint32_t rte_le_to_cpu_32(uint32_t x); +static uint32_t rte_le_to_cpu_32(rte_le32_t x); /** * Convert a 64-bit value from little endian to CPU order. */ -static uint64_t rte_le_to_cpu_64(uint64_t x); +static uint64_t rte_le_to_cpu_64(rte_le64_t x); /** * Convert a 16-bit value from big endian to CPU order. */ -static uint16_t rte_be_to_cpu_16(uint16_t x); +static uint16_t rte_be_to_cpu_16(rte_be16_t x); /** * Convert a 32-bit value from big endian to CPU order. */ -static uint32_t rte_be_to_cpu_32(uint32_t x); +static uint32_t rte_be_to_cpu_32(rte_be32_t x); /** * Convert a 64-bit value from big endian to CPU order. */ -static uint64_t rte_be_to_cpu_64(uint64_t x); +static uint64_t rte_be_to_cpu_64(rte_be64_t x); #endif /* __DOXYGEN__ */ diff --git a/dpdk/lib/librte_eal/common/include/generic/rte_cpuflags.h b/dpdk/lib/librte_eal/common/include/generic/rte_cpuflags.h index c1da357c..c1c5551f 100644 --- a/dpdk/lib/librte_eal/common/include/generic/rte_cpuflags.h +++ b/dpdk/lib/librte_eal/common/include/generic/rte_cpuflags.h @@ -39,11 +39,13 @@ * Architecture specific API to determine available CPU features at runtime. */ +#include "rte_common.h" #include /** * Enumeration of all CPU features supported */ +__extension__ enum rte_cpu_flag_t; /** @@ -55,6 +57,7 @@ enum rte_cpu_flag_t; * flag name * NULL if flag ID is invalid */ +__extension__ const char * rte_cpu_get_flag_name(enum rte_cpu_flag_t feature); @@ -68,6 +71,7 @@ rte_cpu_get_flag_name(enum rte_cpu_flag_t feature); * 0 if flag is not available * -ENOENT if flag is invalid */ +__extension__ int rte_cpu_get_flag_enabled(enum rte_cpu_flag_t feature); @@ -76,7 +80,17 @@ rte_cpu_get_flag_enabled(enum rte_cpu_flag_t feature); * that were specified at compile time. It is called automatically within the * EAL, so does not need to be used by applications. */ +__rte_deprecated void rte_cpu_check_supported(void); +/** + * This function checks that the currently used CPU supports the CPU features + * that were specified at compile time. It is called automatically within the + * EAL, so does not need to be used by applications. This version returns a + * result so that decisions may be made (for instance, graceful shutdowns). + */ +int +rte_cpu_is_supported(void); + #endif /* _RTE_CPUFLAGS_H_ */ diff --git a/dpdk/lib/librte_eal/common/include/generic/rte_cycles.h b/dpdk/lib/librte_eal/common/include/generic/rte_cycles.h index 8cc21f20..0df90474 100644 --- a/dpdk/lib/librte_eal/common/include/generic/rte_cycles.h +++ b/dpdk/lib/librte_eal/common/include/generic/rte_cycles.h @@ -2,6 +2,7 @@ * BSD LICENSE * * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * Copyright(c) 2013 6WIND. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -30,36 +31,6 @@ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -/* BSD LICENSE - * - * Copyright(c) 2013 6WIND. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of 6WIND S.A. nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ #ifndef _RTE_CYCLES_H_ #define _RTE_CYCLES_H_ @@ -150,15 +121,17 @@ int rte_eal_hpet_init(int make_default); static inline uint64_t rte_get_timer_cycles(void) { +#ifdef RTE_LIBEAL_USE_HPET switch(eal_timer_source) { case EAL_TIMER_TSC: - return rte_get_tsc_cycles(); - case EAL_TIMER_HPET: -#ifdef RTE_LIBEAL_USE_HPET - return rte_get_hpet_cycles(); #endif + return rte_get_tsc_cycles(); +#ifdef RTE_LIBEAL_USE_HPET + case EAL_TIMER_HPET: + return rte_get_hpet_cycles(); default: rte_panic("Invalid timer source specified\n"); } +#endif } /** @@ -170,25 +143,28 @@ rte_get_timer_cycles(void) static inline uint64_t rte_get_timer_hz(void) { +#ifdef RTE_LIBEAL_USE_HPET switch(eal_timer_source) { case EAL_TIMER_TSC: - return rte_get_tsc_hz(); - case EAL_TIMER_HPET: -#ifdef RTE_LIBEAL_USE_HPET - return rte_get_hpet_hz(); #endif + return rte_get_tsc_hz(); +#ifdef RTE_LIBEAL_USE_HPET + case EAL_TIMER_HPET: + return rte_get_hpet_hz(); default: rte_panic("Invalid timer source specified\n"); } +#endif } - /** * Wait at least us microseconds. + * This function can be replaced with user-defined function. + * @see rte_delay_us_callback_register * * @param us * The number of microseconds to wait. */ -void -rte_delay_us(unsigned us); +extern void +(*rte_delay_us)(unsigned int us); /** * Wait at least ms milliseconds. @@ -202,4 +178,21 @@ rte_delay_ms(unsigned ms) rte_delay_us(ms * 1000); } +/** + * Blocking delay function. + * + * @param us + * Number of microseconds to wait. + */ +void rte_delay_us_block(unsigned int us); + +/** + * Replace rte_delay_us with user defined function. + * + * @param userfunc + * User function which replaces rte_delay_us. rte_delay_us_block restores + * buildin block delay function. + */ +void rte_delay_us_callback_register(void(*userfunc)(unsigned int)); + #endif /* _RTE_CYCLES_H_ */ diff --git a/dpdk/lib/librte_eal/common/include/generic/rte_io.h b/dpdk/lib/librte_eal/common/include/generic/rte_io.h new file mode 100644 index 00000000..0b88c341 --- /dev/null +++ b/dpdk/lib/librte_eal/common/include/generic/rte_io.h @@ -0,0 +1,379 @@ +/* + * BSD LICENSE + * + * Copyright(c) 2016 Cavium, Inc. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_IO_H_ +#define _RTE_IO_H_ + +/** + * @file + * I/O device memory operations + * + * This file defines the generic API for I/O device memory read/write operations + */ + +#include +#include +#include + +#ifdef __DOXYGEN__ + +/** + * Read a 8-bit value from I/O device memory address *addr*. + * + * The relaxed version does not have additional I/O memory barrier, useful in + * accessing the device registers of integrated controllers which implicitly + * strongly ordered with respect to memory access. + * + * @param addr + * I/O memory address to read the value from + * @return + * read value + */ +static inline uint8_t +rte_read8_relaxed(const volatile void *addr); + +/** + * Read a 16-bit value from I/O device memory address *addr*. + * + * The relaxed version does not have additional I/O memory barrier, useful in + * accessing the device registers of integrated controllers which implicitly + * strongly ordered with respect to memory access. + * + * @param addr + * I/O memory address to read the value from + * @return + * read value + */ +static inline uint16_t +rte_read16_relaxed(const volatile void *addr); + +/** + * Read a 32-bit value from I/O device memory address *addr*. + * + * The relaxed version does not have additional I/O memory barrier, useful in + * accessing the device registers of integrated controllers which implicitly + * strongly ordered with respect to memory access. + * + * @param addr + * I/O memory address to read the value from + * @return + * read value + */ +static inline uint32_t +rte_read32_relaxed(const volatile void *addr); + +/** + * Read a 64-bit value from I/O device memory address *addr*. + * + * The relaxed version does not have additional I/O memory barrier, useful in + * accessing the device registers of integrated controllers which implicitly + * strongly ordered with respect to memory access. + * + * @param addr + * I/O memory address to read the value from + * @return + * read value + */ +static inline uint64_t +rte_read64_relaxed(const volatile void *addr); + +/** + * Write a 8-bit value to I/O device memory address *addr*. + * + * The relaxed version does not have additional I/O memory barrier, useful in + * accessing the device registers of integrated controllers which implicitly + * strongly ordered with respect to memory access. + * + * @param value + * Value to write + * @param addr + * I/O memory address to write the value to + */ + +static inline void +rte_write8_relaxed(uint8_t value, volatile void *addr); + +/** + * Write a 16-bit value to I/O device memory address *addr*. + * + * The relaxed version does not have additional I/O memory barrier, useful in + * accessing the device registers of integrated controllers which implicitly + * strongly ordered with respect to memory access. + * + * @param value + * Value to write + * @param addr + * I/O memory address to write the value to + */ +static inline void +rte_write16_relaxed(uint16_t value, volatile void *addr); + +/** + * Write a 32-bit value to I/O device memory address *addr*. + * + * The relaxed version does not have additional I/O memory barrier, useful in + * accessing the device registers of integrated controllers which implicitly + * strongly ordered with respect to memory access. + * + * @param value + * Value to write + * @param addr + * I/O memory address to write the value to + */ +static inline void +rte_write32_relaxed(uint32_t value, volatile void *addr); + +/** + * Write a 64-bit value to I/O device memory address *addr*. + * + * The relaxed version does not have additional I/O memory barrier, useful in + * accessing the device registers of integrated controllers which implicitly + * strongly ordered with respect to memory access. + * + * @param value + * Value to write + * @param addr + * I/O memory address to write the value to + */ +static inline void +rte_write64_relaxed(uint64_t value, volatile void *addr); + +/** + * Read a 8-bit value from I/O device memory address *addr*. + * + * @param addr + * I/O memory address to read the value from + * @return + * read value + */ +static inline uint8_t +rte_read8(const volatile void *addr); + +/** + * Read a 16-bit value from I/O device memory address *addr*. + * + * + * @param addr + * I/O memory address to read the value from + * @return + * read value + */ +static inline uint16_t +rte_read16(const volatile void *addr); + +/** + * Read a 32-bit value from I/O device memory address *addr*. + * + * @param addr + * I/O memory address to read the value from + * @return + * read value + */ +static inline uint32_t +rte_read32(const volatile void *addr); + +/** + * Read a 64-bit value from I/O device memory address *addr*. + * + * @param addr + * I/O memory address to read the value from + * @return + * read value + */ +static inline uint64_t +rte_read64(const volatile void *addr); + +/** + * Write a 8-bit value to I/O device memory address *addr*. + * + * @param value + * Value to write + * @param addr + * I/O memory address to write the value to + */ + +static inline void +rte_write8(uint8_t value, volatile void *addr); + +/** + * Write a 16-bit value to I/O device memory address *addr*. + * + * @param value + * Value to write + * @param addr + * I/O memory address to write the value to + */ +static inline void +rte_write16(uint16_t value, volatile void *addr); + +/** + * Write a 32-bit value to I/O device memory address *addr*. + * + * @param value + * Value to write + * @param addr + * I/O memory address to write the value to + */ +static inline void +rte_write32(uint32_t value, volatile void *addr); + +/** + * Write a 64-bit value to I/O device memory address *addr*. + * + * @param value + * Value to write + * @param addr + * I/O memory address to write the value to + */ +static inline void +rte_write64(uint64_t value, volatile void *addr); + +#endif /* __DOXYGEN__ */ + +#ifndef RTE_OVERRIDE_IO_H + +static __rte_always_inline uint8_t +rte_read8_relaxed(const volatile void *addr) +{ + return *(const volatile uint8_t *)addr; +} + +static __rte_always_inline uint16_t +rte_read16_relaxed(const volatile void *addr) +{ + return *(const volatile uint16_t *)addr; +} + +static __rte_always_inline uint32_t +rte_read32_relaxed(const volatile void *addr) +{ + return *(const volatile uint32_t *)addr; +} + +static __rte_always_inline uint64_t +rte_read64_relaxed(const volatile void *addr) +{ + return *(const volatile uint64_t *)addr; +} + +static __rte_always_inline void +rte_write8_relaxed(uint8_t value, volatile void *addr) +{ + *(volatile uint8_t *)addr = value; +} + +static __rte_always_inline void +rte_write16_relaxed(uint16_t value, volatile void *addr) +{ + *(volatile uint16_t *)addr = value; +} + +static __rte_always_inline void +rte_write32_relaxed(uint32_t value, volatile void *addr) +{ + *(volatile uint32_t *)addr = value; +} + +static __rte_always_inline void +rte_write64_relaxed(uint64_t value, volatile void *addr) +{ + *(volatile uint64_t *)addr = value; +} + +static __rte_always_inline uint8_t +rte_read8(const volatile void *addr) +{ + uint8_t val; + val = rte_read8_relaxed(addr); + rte_io_rmb(); + return val; +} + +static __rte_always_inline uint16_t +rte_read16(const volatile void *addr) +{ + uint16_t val; + val = rte_read16_relaxed(addr); + rte_io_rmb(); + return val; +} + +static __rte_always_inline uint32_t +rte_read32(const volatile void *addr) +{ + uint32_t val; + val = rte_read32_relaxed(addr); + rte_io_rmb(); + return val; +} + +static __rte_always_inline uint64_t +rte_read64(const volatile void *addr) +{ + uint64_t val; + val = rte_read64_relaxed(addr); + rte_io_rmb(); + return val; +} + +static __rte_always_inline void +rte_write8(uint8_t value, volatile void *addr) +{ + rte_io_wmb(); + rte_write8_relaxed(value, addr); +} + +static __rte_always_inline void +rte_write16(uint16_t value, volatile void *addr) +{ + rte_io_wmb(); + rte_write16_relaxed(value, addr); +} + +static __rte_always_inline void +rte_write32(uint32_t value, volatile void *addr) +{ + rte_io_wmb(); + rte_write32_relaxed(value, addr); +} + +static __rte_always_inline void +rte_write64(uint64_t value, volatile void *addr) +{ + rte_io_wmb(); + rte_write64_relaxed(value, addr); +} + +#endif /* RTE_OVERRIDE_IO_H */ + +#endif /* _RTE_IO_H_ */ diff --git a/dpdk/lib/librte_eal/common/include/generic/rte_memcpy.h b/dpdk/lib/librte_eal/common/include/generic/rte_memcpy.h index afb0afe4..4e9d8794 100644 --- a/dpdk/lib/librte_eal/common/include/generic/rte_memcpy.h +++ b/dpdk/lib/librte_eal/common/include/generic/rte_memcpy.h @@ -64,6 +64,8 @@ rte_mov16(uint8_t *dst, const uint8_t *src); static inline void rte_mov32(uint8_t *dst, const uint8_t *src); +#ifdef __DOXYGEN__ + /** * Copy 48 bytes from one location to another using optimised * instructions. The locations should not overlap. @@ -76,6 +78,8 @@ rte_mov32(uint8_t *dst, const uint8_t *src); static inline void rte_mov48(uint8_t *dst, const uint8_t *src); +#endif /* __DOXYGEN__ */ + /** * Copy 64 bytes from one location to another using optimised * instructions. The locations should not overlap. diff --git a/dpdk/lib/librte_eal/common/include/generic/rte_pause.h b/dpdk/lib/librte_eal/common/include/generic/rte_pause.h new file mode 100644 index 00000000..a8374321 --- /dev/null +++ b/dpdk/lib/librte_eal/common/include/generic/rte_pause.h @@ -0,0 +1,52 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Cavium, Inc. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_PAUSE_H_ +#define _RTE_PAUSE_H_ + +/** + * @file + * + * CPU pause operation. + * + */ + +/** + * Pause CPU execution for a short while + * + * This call is intended for tight loops which poll a shared resource or wait + * for an event. A short pause within the loop may reduce the power consumption. + */ +static inline void rte_pause(void); + +#endif /* _RTE_PAUSE_H_ */ diff --git a/dpdk/lib/librte_eal/common/include/generic/rte_rwlock.h b/dpdk/lib/librte_eal/common/include/generic/rte_rwlock.h index 7a0fdc55..fdb3113d 100644 --- a/dpdk/lib/librte_eal/common/include/generic/rte_rwlock.h +++ b/dpdk/lib/librte_eal/common/include/generic/rte_rwlock.h @@ -52,6 +52,7 @@ extern "C" { #include #include +#include /** * The rte_rwlock_t type. diff --git a/dpdk/lib/librte_eal/common/include/generic/rte_spinlock.h b/dpdk/lib/librte_eal/common/include/generic/rte_spinlock.h index e51fc56b..54f83a4c 100644 --- a/dpdk/lib/librte_eal/common/include/generic/rte_spinlock.h +++ b/dpdk/lib/librte_eal/common/include/generic/rte_spinlock.h @@ -51,6 +51,7 @@ #ifdef RTE_FORCE_INTRINSICS #include #endif +#include /** * The rte_spinlock_t type. diff --git a/dpdk/lib/librte_eal/common/include/generic/rte_vect.h b/dpdk/lib/librte_eal/common/include/generic/rte_vect.h new file mode 100644 index 00000000..600ee9f3 --- /dev/null +++ b/dpdk/lib/librte_eal/common/include/generic/rte_vect.h @@ -0,0 +1,214 @@ +/*- + * BSD LICENSE + * + * Copyright 2016 6WIND S.A. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of 6WIND S.A. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_VECT_H_ +#define _RTE_VECT_H_ + +/** + * @file + * SIMD vector types + * + * This file defines types to use vector instructions with generic C code. + */ + +#include + +/* Unsigned vector types */ + +/** + * 64 bits vector size to use with unsigned 8 bits elements. + * + * a = (rte_v64u8_t){ a0, a1, a2, a3, a4, a5, a6, a7 } + */ +typedef uint8_t rte_v64u8_t __attribute__((vector_size(8), aligned(8))); + +/** + * 64 bits vector size to use with unsigned 16 bits elements. + * + * a = (rte_v64u16_t){ a0, a1, a2, a3 } + */ +typedef uint16_t rte_v64u16_t __attribute__((vector_size(8), aligned(8))); + +/** + * 64 bits vector size to use with unsigned 32 bits elements. + * + * a = (rte_v64u32_t){ a0, a1 } + */ +typedef uint32_t rte_v64u32_t __attribute__((vector_size(8), aligned(8))); + +/** + * 128 bits vector size to use with unsigned 8 bits elements. + * + * a = (rte_v128u8_t){ a00, a01, a02, a03, a04, a05, a06, a07, + * a08, a09, a10, a11, a12, a13, a14, a15 } + */ +typedef uint8_t rte_v128u8_t __attribute__((vector_size(16), aligned(16))); + +/** + * 128 bits vector size to use with unsigned 16 bits elements. + * + * a = (rte_v128u16_t){ a0, a1, a2, a3, a4, a5, a6, a7 } + */ +typedef uint16_t rte_v128u16_t __attribute__((vector_size(16), aligned(16))); + +/** + * 128 bits vector size to use with unsigned 32 bits elements. + * + * a = (rte_v128u32_t){ a0, a1, a2, a3, a4 } + */ +typedef uint32_t rte_v128u32_t __attribute__((vector_size(16), aligned(16))); + +/** + * 128 bits vector size to use with unsigned 64 bits elements. + * + * a = (rte_v128u64_t){ a0, a1 } + */ +typedef uint64_t rte_v128u64_t __attribute__((vector_size(16), aligned(16))); + +/** + * 256 bits vector size to use with unsigned 8 bits elements. + * + * a = (rte_v256u8_t){ a00, a01, a02, a03, a04, a05, a06, a07, + * a08, a09, a10, a11, a12, a13, a14, a15, + * a16, a17, a18, a19, a20, a21, a22, a23, + * a24, a25, a26, a27, a28, a29, a30, a31 } + */ +typedef uint8_t rte_v256u8_t __attribute__((vector_size(32), aligned(32))); + +/** + * 256 bits vector size to use with unsigned 16 bits elements. + * + * a = (rte_v256u16_t){ a00, a01, a02, a03, a04, a05, a06, a07, + * a08, a09, a10, a11, a12, a13, a14, a15 } + */ +typedef uint16_t rte_v256u16_t __attribute__((vector_size(32), aligned(32))); + +/** + * 256 bits vector size to use with unsigned 32 bits elements. + * + * a = (rte_v256u32_t){ a0, a1, a2, a3, a4, a5, a6, a7 } + */ +typedef uint32_t rte_v256u32_t __attribute__((vector_size(32), aligned(32))); + +/** + * 256 bits vector size to use with unsigned 64 bits elements. + * + * a = (rte_v256u64_t){ a0, a1, a2, a3 } + */ +typedef uint64_t rte_v256u64_t __attribute__((vector_size(32), aligned(32))); + + +/* Signed vector types */ + +/** + * 64 bits vector size to use with 8 bits elements. + * + * a = (rte_v64s8_t){ a0, a1, a2, a3, a4, a5, a6, a7 } + */ +typedef int8_t rte_v64s8_t __attribute__((vector_size(8), aligned(8))); + +/** + * 64 bits vector size to use with 16 bits elements. + * + * a = (rte_v64s16_t){ a0, a1, a2, a3 } + */ +typedef int16_t rte_v64s16_t __attribute__((vector_size(8), aligned(8))); + +/** + * 64 bits vector size to use with 32 bits elements. + * + * a = (rte_v64s32_t){ a0, a1 } + */ +typedef int32_t rte_v64s32_t __attribute__((vector_size(8), aligned(8))); + +/** + * 128 bits vector size to use with 8 bits elements. + * + * a = (rte_v128s8_t){ a00, a01, a02, a03, a04, a05, a06, a07, + * a08, a09, a10, a11, a12, a13, a14, a15 } + */ +typedef int8_t rte_v128s8_t __attribute__((vector_size(16), aligned(16))); + +/** + * 128 bits vector size to use with 16 bits elements. + * + * a = (rte_v128s16_t){ a0, a1, a2, a3, a4, a5, a6, a7 } + */ +typedef int16_t rte_v128s16_t __attribute__((vector_size(16), aligned(16))); + +/** + * 128 bits vector size to use with 32 bits elements. + * + * a = (rte_v128s32_t){ a0, a1, a2, a3 } + */ +typedef int32_t rte_v128s32_t __attribute__((vector_size(16), aligned(16))); + +/** + * 128 bits vector size to use with 64 bits elements. + * + * a = (rte_v128s64_t){ a1, a2 } + */ +typedef int64_t rte_v128s64_t __attribute__((vector_size(16), aligned(16))); + +/** + * 256 bits vector size to use with 8 bits elements. + * + * a = (rte_v256s8_t){ a00, a01, a02, a03, a04, a05, a06, a07, + * a08, a09, a10, a11, a12, a13, a14, a15, + * a16, a17, a18, a19, a20, a21, a22, a23, + * a24, a25, a26, a27, a28, a29, a30, a31 } + */ +typedef int8_t rte_v256s8_t __attribute__((vector_size(32), aligned(32))); + +/** + * 256 bits vector size to use with 16 bits elements. + * + * a = (rte_v256s16_t){ a00, a01, a02, a03, a04, a05, a06, a07, + * a08, a09, a10, a11, a12, a13, a14, a15 } + */ +typedef int16_t rte_v256s16_t __attribute__((vector_size(32), aligned(32))); + +/** + * 256 bits vector size to use with 32 bits elements. + * + * a = (rte_v256s32_t){ a0, a1, a2, a3, a4, a5, a6, a7 } + */ +typedef int32_t rte_v256s32_t __attribute__((vector_size(32), aligned(32))); + +/** + * 256 bits vector size to use with 64 bits elements. + * + * a = (rte_v256s64_t){ a0, a1, a2, a3 } + */ +typedef int64_t rte_v256s64_t __attribute__((vector_size(32), aligned(32))); + +#endif /* _RTE_VECT_H_ */ diff --git a/dpdk/lib/librte_eal/common/include/rte_alarm.h b/dpdk/lib/librte_eal/common/include/rte_alarm.h index 4012cd67..c275be18 100644 --- a/dpdk/lib/librte_eal/common/include/rte_alarm.h +++ b/dpdk/lib/librte_eal/common/include/rte_alarm.h @@ -91,7 +91,7 @@ int rte_eal_alarm_set(uint64_t us, rte_eal_alarm_callback cb, void *cb_arg); * the number of canceled alarm callback functions * - value greater or equal 0 and rte_errno set to EINPROGRESS, at least one * alarm could not be canceled because cancellation was requested from alarm - * callback context. Returned value is the number of succesfuly canceled + * callback context. Returned value is the number of successfully canceled * alarm callbacks * - 0 and rte_errno set to ENOENT - no alarm found * - -1 and rte_errno set to EINVAL - invalid parameter (NULL callback) diff --git a/dpdk/lib/librte_eal/common/include/rte_bitmap.h b/dpdk/lib/librte_eal/common/include/rte_bitmap.h new file mode 100644 index 00000000..b9067e67 --- /dev/null +++ b/dpdk/lib/librte_eal/common/include/rte_bitmap.h @@ -0,0 +1,562 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __INCLUDE_RTE_BITMAP_H__ +#define __INCLUDE_RTE_BITMAP_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @file + * RTE Bitmap + * + * The bitmap component provides a mechanism to manage large arrays of bits + * through bit get/set/clear and bit array scan operations. + * + * The bitmap scan operation is optimized for 64-bit CPUs using 64/128 byte cache + * lines. The bitmap is hierarchically organized using two arrays (array1 and + * array2), with each bit in array1 being associated with a full cache line + * (512/1024 bits) of bitmap bits, which are stored in array2: the bit in array1 + * is set only when there is at least one bit set within its associated array2 + * bits, otherwise the bit in array1 is cleared. The read and write operations + * for array1 and array2 are always done in slabs of 64 bits. + * + * This bitmap is not thread safe. For lock free operation on a specific bitmap + * instance, a single writer thread performing bit set/clear operations is + * allowed, only the writer thread can do bitmap scan operations, while there + * can be several reader threads performing bit get operations in parallel with + * the writer thread. When the use of locking primitives is acceptable, the + * serialization of the bit set/clear and bitmap scan operations needs to be + * enforced by the caller, while the bit get operation does not require locking + * the bitmap. + * + ***/ + +#include +#include +#include +#include +#include +#include +#include + +#ifndef RTE_BITMAP_OPTIMIZATIONS +#define RTE_BITMAP_OPTIMIZATIONS 1 +#endif + +/* Slab */ +#define RTE_BITMAP_SLAB_BIT_SIZE 64 +#define RTE_BITMAP_SLAB_BIT_SIZE_LOG2 6 +#define RTE_BITMAP_SLAB_BIT_MASK (RTE_BITMAP_SLAB_BIT_SIZE - 1) + +/* Cache line (CL) */ +#define RTE_BITMAP_CL_BIT_SIZE (RTE_CACHE_LINE_SIZE * 8) +#define RTE_BITMAP_CL_BIT_SIZE_LOG2 (RTE_CACHE_LINE_SIZE_LOG2 + 3) +#define RTE_BITMAP_CL_BIT_MASK (RTE_BITMAP_CL_BIT_SIZE - 1) + +#define RTE_BITMAP_CL_SLAB_SIZE (RTE_BITMAP_CL_BIT_SIZE / RTE_BITMAP_SLAB_BIT_SIZE) +#define RTE_BITMAP_CL_SLAB_SIZE_LOG2 (RTE_BITMAP_CL_BIT_SIZE_LOG2 - RTE_BITMAP_SLAB_BIT_SIZE_LOG2) +#define RTE_BITMAP_CL_SLAB_MASK (RTE_BITMAP_CL_SLAB_SIZE - 1) + +/** Bitmap data structure */ +struct rte_bitmap { + /* Context for array1 and array2 */ + uint64_t *array1; /**< Bitmap array1 */ + uint64_t *array2; /**< Bitmap array2 */ + uint32_t array1_size; /**< Number of 64-bit slabs in array1 that are actually used */ + uint32_t array2_size; /**< Number of 64-bit slabs in array2 */ + + /* Context for the "scan next" operation */ + uint32_t index1; /**< Bitmap scan: Index of current array1 slab */ + uint32_t offset1; /**< Bitmap scan: Offset of current bit within current array1 slab */ + uint32_t index2; /**< Bitmap scan: Index of current array2 slab */ + uint32_t go2; /**< Bitmap scan: Go/stop condition for current array2 cache line */ + + /* Storage space for array1 and array2 */ + uint8_t memory[]; +}; + +static inline void +__rte_bitmap_index1_inc(struct rte_bitmap *bmp) +{ + bmp->index1 = (bmp->index1 + 1) & (bmp->array1_size - 1); +} + +static inline uint64_t +__rte_bitmap_mask1_get(struct rte_bitmap *bmp) +{ + return (~1lu) << bmp->offset1; +} + +static inline void +__rte_bitmap_index2_set(struct rte_bitmap *bmp) +{ + bmp->index2 = (((bmp->index1 << RTE_BITMAP_SLAB_BIT_SIZE_LOG2) + bmp->offset1) << RTE_BITMAP_CL_SLAB_SIZE_LOG2); +} + +#if RTE_BITMAP_OPTIMIZATIONS + +static inline int +rte_bsf64(uint64_t slab, uint32_t *pos) +{ + if (likely(slab == 0)) { + return 0; + } + + *pos = __builtin_ctzll(slab); + return 1; +} + +#else + +static inline int +rte_bsf64(uint64_t slab, uint32_t *pos) +{ + uint64_t mask; + uint32_t i; + + if (likely(slab == 0)) { + return 0; + } + + for (i = 0, mask = 1; i < RTE_BITMAP_SLAB_BIT_SIZE; i ++, mask <<= 1) { + if (unlikely(slab & mask)) { + *pos = i; + return 1; + } + } + + return 0; +} + +#endif + +static inline uint32_t +__rte_bitmap_get_memory_footprint(uint32_t n_bits, + uint32_t *array1_byte_offset, uint32_t *array1_slabs, + uint32_t *array2_byte_offset, uint32_t *array2_slabs) +{ + uint32_t n_slabs_context, n_slabs_array1, n_cache_lines_context_and_array1; + uint32_t n_cache_lines_array2; + uint32_t n_bytes_total; + + n_cache_lines_array2 = (n_bits + RTE_BITMAP_CL_BIT_SIZE - 1) / RTE_BITMAP_CL_BIT_SIZE; + n_slabs_array1 = (n_cache_lines_array2 + RTE_BITMAP_SLAB_BIT_SIZE - 1) / RTE_BITMAP_SLAB_BIT_SIZE; + n_slabs_array1 = rte_align32pow2(n_slabs_array1); + n_slabs_context = (sizeof(struct rte_bitmap) + (RTE_BITMAP_SLAB_BIT_SIZE / 8) - 1) / (RTE_BITMAP_SLAB_BIT_SIZE / 8); + n_cache_lines_context_and_array1 = (n_slabs_context + n_slabs_array1 + RTE_BITMAP_CL_SLAB_SIZE - 1) / RTE_BITMAP_CL_SLAB_SIZE; + n_bytes_total = (n_cache_lines_context_and_array1 + n_cache_lines_array2) * RTE_CACHE_LINE_SIZE; + + if (array1_byte_offset) { + *array1_byte_offset = n_slabs_context * (RTE_BITMAP_SLAB_BIT_SIZE / 8); + } + if (array1_slabs) { + *array1_slabs = n_slabs_array1; + } + if (array2_byte_offset) { + *array2_byte_offset = n_cache_lines_context_and_array1 * RTE_CACHE_LINE_SIZE; + } + if (array2_slabs) { + *array2_slabs = n_cache_lines_array2 * RTE_BITMAP_CL_SLAB_SIZE; + } + + return n_bytes_total; +} + +static inline void +__rte_bitmap_scan_init(struct rte_bitmap *bmp) +{ + bmp->index1 = bmp->array1_size - 1; + bmp->offset1 = RTE_BITMAP_SLAB_BIT_SIZE - 1; + __rte_bitmap_index2_set(bmp); + bmp->index2 += RTE_BITMAP_CL_SLAB_SIZE; + + bmp->go2 = 0; +} + +/** + * Bitmap memory footprint calculation + * + * @param n_bits + * Number of bits in the bitmap + * @return + * Bitmap memory footprint measured in bytes on success, 0 on error + */ +static inline uint32_t +rte_bitmap_get_memory_footprint(uint32_t n_bits) { + /* Check input arguments */ + if (n_bits == 0) { + return 0; + } + + return __rte_bitmap_get_memory_footprint(n_bits, NULL, NULL, NULL, NULL); +} + +/** + * Bitmap initialization + * + * @param mem_size + * Minimum expected size of bitmap. + * @param mem + * Base address of array1 and array2. + * @param n_bits + * Number of pre-allocated bits in array2. Must be non-zero and multiple of 512. + * @return + * Handle to bitmap instance. + */ +static inline struct rte_bitmap * +rte_bitmap_init(uint32_t n_bits, uint8_t *mem, uint32_t mem_size) +{ + struct rte_bitmap *bmp; + uint32_t array1_byte_offset, array1_slabs, array2_byte_offset, array2_slabs; + uint32_t size; + + /* Check input arguments */ + if (n_bits == 0) { + return NULL; + } + + if ((mem == NULL) || (((uintptr_t) mem) & RTE_CACHE_LINE_MASK)) { + return NULL; + } + + size = __rte_bitmap_get_memory_footprint(n_bits, + &array1_byte_offset, &array1_slabs, + &array2_byte_offset, &array2_slabs); + if (size < mem_size) { + return NULL; + } + + /* Setup bitmap */ + memset(mem, 0, size); + bmp = (struct rte_bitmap *) mem; + + bmp->array1 = (uint64_t *) &mem[array1_byte_offset]; + bmp->array1_size = array1_slabs; + bmp->array2 = (uint64_t *) &mem[array2_byte_offset]; + bmp->array2_size = array2_slabs; + + __rte_bitmap_scan_init(bmp); + + return bmp; +} + +/** + * Bitmap free + * + * @param bmp + * Handle to bitmap instance + * @return + * 0 upon success, error code otherwise + */ +static inline int +rte_bitmap_free(struct rte_bitmap *bmp) +{ + /* Check input arguments */ + if (bmp == NULL) { + return -1; + } + + return 0; +} + +/** + * Bitmap reset + * + * @param bmp + * Handle to bitmap instance + */ +static inline void +rte_bitmap_reset(struct rte_bitmap *bmp) +{ + memset(bmp->array1, 0, bmp->array1_size * sizeof(uint64_t)); + memset(bmp->array2, 0, bmp->array2_size * sizeof(uint64_t)); + __rte_bitmap_scan_init(bmp); +} + +/** + * Bitmap location prefetch into CPU L1 cache + * + * @param bmp + * Handle to bitmap instance + * @param pos + * Bit position + * @return + * 0 upon success, error code otherwise + */ +static inline void +rte_bitmap_prefetch0(struct rte_bitmap *bmp, uint32_t pos) +{ + uint64_t *slab2; + uint32_t index2; + + index2 = pos >> RTE_BITMAP_SLAB_BIT_SIZE_LOG2; + slab2 = bmp->array2 + index2; + rte_prefetch0((void *) slab2); +} + +/** + * Bitmap bit get + * + * @param bmp + * Handle to bitmap instance + * @param pos + * Bit position + * @return + * 0 when bit is cleared, non-zero when bit is set + */ +static inline uint64_t +rte_bitmap_get(struct rte_bitmap *bmp, uint32_t pos) +{ + uint64_t *slab2; + uint32_t index2, offset2; + + index2 = pos >> RTE_BITMAP_SLAB_BIT_SIZE_LOG2; + offset2 = pos & RTE_BITMAP_SLAB_BIT_MASK; + slab2 = bmp->array2 + index2; + return (*slab2) & (1lu << offset2); +} + +/** + * Bitmap bit set + * + * @param bmp + * Handle to bitmap instance + * @param pos + * Bit position + */ +static inline void +rte_bitmap_set(struct rte_bitmap *bmp, uint32_t pos) +{ + uint64_t *slab1, *slab2; + uint32_t index1, index2, offset1, offset2; + + /* Set bit in array2 slab and set bit in array1 slab */ + index2 = pos >> RTE_BITMAP_SLAB_BIT_SIZE_LOG2; + offset2 = pos & RTE_BITMAP_SLAB_BIT_MASK; + index1 = pos >> (RTE_BITMAP_SLAB_BIT_SIZE_LOG2 + RTE_BITMAP_CL_BIT_SIZE_LOG2); + offset1 = (pos >> RTE_BITMAP_CL_BIT_SIZE_LOG2) & RTE_BITMAP_SLAB_BIT_MASK; + slab2 = bmp->array2 + index2; + slab1 = bmp->array1 + index1; + + *slab2 |= 1lu << offset2; + *slab1 |= 1lu << offset1; +} + +/** + * Bitmap slab set + * + * @param bmp + * Handle to bitmap instance + * @param pos + * Bit position identifying the array2 slab + * @param slab + * Value to be assigned to the 64-bit slab in array2 + */ +static inline void +rte_bitmap_set_slab(struct rte_bitmap *bmp, uint32_t pos, uint64_t slab) +{ + uint64_t *slab1, *slab2; + uint32_t index1, index2, offset1; + + /* Set bits in array2 slab and set bit in array1 slab */ + index2 = pos >> RTE_BITMAP_SLAB_BIT_SIZE_LOG2; + index1 = pos >> (RTE_BITMAP_SLAB_BIT_SIZE_LOG2 + RTE_BITMAP_CL_BIT_SIZE_LOG2); + offset1 = (pos >> RTE_BITMAP_CL_BIT_SIZE_LOG2) & RTE_BITMAP_SLAB_BIT_MASK; + slab2 = bmp->array2 + index2; + slab1 = bmp->array1 + index1; + + *slab2 |= slab; + *slab1 |= 1lu << offset1; +} + +static inline uint64_t +__rte_bitmap_line_not_empty(uint64_t *slab2) +{ + uint64_t v1, v2, v3, v4; + + v1 = slab2[0] | slab2[1]; + v2 = slab2[2] | slab2[3]; + v3 = slab2[4] | slab2[5]; + v4 = slab2[6] | slab2[7]; + v1 |= v2; + v3 |= v4; + + return v1 | v3; +} + +/** + * Bitmap bit clear + * + * @param bmp + * Handle to bitmap instance + * @param pos + * Bit position + */ +static inline void +rte_bitmap_clear(struct rte_bitmap *bmp, uint32_t pos) +{ + uint64_t *slab1, *slab2; + uint32_t index1, index2, offset1, offset2; + + /* Clear bit in array2 slab */ + index2 = pos >> RTE_BITMAP_SLAB_BIT_SIZE_LOG2; + offset2 = pos & RTE_BITMAP_SLAB_BIT_MASK; + slab2 = bmp->array2 + index2; + + /* Return if array2 slab is not all-zeros */ + *slab2 &= ~(1lu << offset2); + if (*slab2){ + return; + } + + /* Check the entire cache line of array2 for all-zeros */ + index2 &= ~ RTE_BITMAP_CL_SLAB_MASK; + slab2 = bmp->array2 + index2; + if (__rte_bitmap_line_not_empty(slab2)) { + return; + } + + /* The array2 cache line is all-zeros, so clear bit in array1 slab */ + index1 = pos >> (RTE_BITMAP_SLAB_BIT_SIZE_LOG2 + RTE_BITMAP_CL_BIT_SIZE_LOG2); + offset1 = (pos >> RTE_BITMAP_CL_BIT_SIZE_LOG2) & RTE_BITMAP_SLAB_BIT_MASK; + slab1 = bmp->array1 + index1; + *slab1 &= ~(1lu << offset1); + + return; +} + +static inline int +__rte_bitmap_scan_search(struct rte_bitmap *bmp) +{ + uint64_t value1; + uint32_t i; + + /* Check current array1 slab */ + value1 = bmp->array1[bmp->index1]; + value1 &= __rte_bitmap_mask1_get(bmp); + + if (rte_bsf64(value1, &bmp->offset1)) { + return 1; + } + + __rte_bitmap_index1_inc(bmp); + bmp->offset1 = 0; + + /* Look for another array1 slab */ + for (i = 0; i < bmp->array1_size; i ++, __rte_bitmap_index1_inc(bmp)) { + value1 = bmp->array1[bmp->index1]; + + if (rte_bsf64(value1, &bmp->offset1)) { + return 1; + } + } + + return 0; +} + +static inline void +__rte_bitmap_scan_read_init(struct rte_bitmap *bmp) +{ + __rte_bitmap_index2_set(bmp); + bmp->go2 = 1; + rte_prefetch1((void *)(bmp->array2 + bmp->index2 + 8)); +} + +static inline int +__rte_bitmap_scan_read(struct rte_bitmap *bmp, uint32_t *pos, uint64_t *slab) +{ + uint64_t *slab2; + + slab2 = bmp->array2 + bmp->index2; + for ( ; bmp->go2 ; bmp->index2 ++, slab2 ++, bmp->go2 = bmp->index2 & RTE_BITMAP_CL_SLAB_MASK) { + if (*slab2) { + *pos = bmp->index2 << RTE_BITMAP_SLAB_BIT_SIZE_LOG2; + *slab = *slab2; + + bmp->index2 ++; + slab2 ++; + bmp->go2 = bmp->index2 & RTE_BITMAP_CL_SLAB_MASK; + return 1; + } + } + + return 0; +} + +/** + * Bitmap scan (with automatic wrap-around) + * + * @param bmp + * Handle to bitmap instance + * @param pos + * When function call returns 1, pos contains the position of the next set + * bit, otherwise not modified + * @param slab + * When function call returns 1, slab contains the value of the entire 64-bit + * slab where the bit indicated by pos is located. Slabs are always 64-bit + * aligned, so the position of the first bit of the slab (this bit is not + * necessarily set) is pos / 64. Once a slab has been returned by the bitmap + * scan operation, the internal pointers of the bitmap are updated to point + * after this slab, so the same slab will not be returned again if it + * contains more than one bit which is set. When function call returns 0, + * slab is not modified. + * @return + * 0 if there is no bit set in the bitmap, 1 otherwise + */ +static inline int +rte_bitmap_scan(struct rte_bitmap *bmp, uint32_t *pos, uint64_t *slab) +{ + /* Return data from current array2 line if available */ + if (__rte_bitmap_scan_read(bmp, pos, slab)) { + return 1; + } + + /* Look for non-empty array2 line */ + if (__rte_bitmap_scan_search(bmp)) { + __rte_bitmap_scan_read_init(bmp); + __rte_bitmap_scan_read(bmp, pos, slab); + return 1; + } + + /* Empty bitmap */ + return 0; +} + +#ifdef __cplusplus +} +#endif + +#endif /* __INCLUDE_RTE_BITMAP_H__ */ diff --git a/dpdk/lib/librte_eal/common/include/rte_bus.h b/dpdk/lib/librte_eal/common/include/rte_bus.h new file mode 100644 index 00000000..6fb08341 --- /dev/null +++ b/dpdk/lib/librte_eal/common/include/rte_bus.h @@ -0,0 +1,339 @@ +/*- + * BSD LICENSE + * + * Copyright 2016 NXP + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of NXP nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_BUS_H_ +#define _RTE_BUS_H_ + +/** + * @file + * + * DPDK device bus interface + * + * This file exposes API and interfaces for bus abstraction + * over the devices and drivers in EAL. + */ + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include + +#include +#include + +/** Double linked list of buses */ +TAILQ_HEAD(rte_bus_list, rte_bus); + + +/** + * IOVA mapping mode. + * + * IOVA mapping mode is iommu programming mode of a device. + * That device (for example: IOMMU backed DMA device) based + * on rte_iova_mode will generate physical or virtual address. + * + */ +enum rte_iova_mode { + RTE_IOVA_DC = 0, /* Don't care mode */ + RTE_IOVA_PA = (1 << 0), /* DMA using physical address */ + RTE_IOVA_VA = (1 << 1) /* DMA using virtual address */ +}; + +/** + * Bus specific scan for devices attached on the bus. + * For each bus object, the scan would be responsible for finding devices and + * adding them to its private device list. + * + * A bus should mandatorily implement this method. + * + * @return + * 0 for successful scan + * <0 for unsuccessful scan with error value + */ +typedef int (*rte_bus_scan_t)(void); + +/** + * Implementation specific probe function which is responsible for linking + * devices on that bus with applicable drivers. + * + * This is called while iterating over each registered bus. + * + * @return + * 0 for successful probe + * !0 for any error while probing + */ +typedef int (*rte_bus_probe_t)(void); + +/** + * Device iterator to find a device on a bus. + * + * This function returns an rte_device if one of those held by the bus + * matches the data passed as parameter. + * + * If the comparison function returns zero this function should stop iterating + * over any more devices. To continue a search the device of a previous search + * can be passed via the start parameter. + * + * @param cmp + * Comparison function. + * + * @param data + * Data to compare each device against. + * + * @param start + * starting point for the iteration + * + * @return + * The first device matching the data, NULL if none exists. + */ +typedef struct rte_device * +(*rte_bus_find_device_t)(const struct rte_device *start, rte_dev_cmp_t cmp, + const void *data); + +/** + * Implementation specific probe function which is responsible for linking + * devices on that bus with applicable drivers. + * + * @param dev + * Device pointer that was returned by a previous call to find_device. + * + * @return + * 0 on success. + * !0 on error. + */ +typedef int (*rte_bus_plug_t)(struct rte_device *dev); + +/** + * Implementation specific remove function which is responsible for unlinking + * devices on that bus from assigned driver. + * + * @param dev + * Device pointer that was returned by a previous call to find_device. + * + * @return + * 0 on success. + * !0 on error. + */ +typedef int (*rte_bus_unplug_t)(struct rte_device *dev); + +/** + * Bus specific parsing function. + * Validates the syntax used in the textual representation of a device, + * If the syntax is valid and ``addr`` is not NULL, writes the bus-specific + * device representation to ``addr``. + * + * @param[in] name + * device textual description + * + * @param[out] addr + * device information location address, into which parsed info + * should be written. If NULL, nothing should be written, which + * is not an error. + * + * @return + * 0 if parsing was successful. + * !0 for any error. + */ +typedef int (*rte_bus_parse_t)(const char *name, void *addr); + +/** + * Bus scan policies + */ +enum rte_bus_scan_mode { + RTE_BUS_SCAN_UNDEFINED, + RTE_BUS_SCAN_WHITELIST, + RTE_BUS_SCAN_BLACKLIST, +}; + +/** + * A structure used to configure bus operations. + */ +struct rte_bus_conf { + enum rte_bus_scan_mode scan_mode; /**< Scan policy. */ +}; + + +/** + * Get common iommu class of the all the devices on the bus. The bus may + * check that those devices are attached to iommu driver. + * If no devices are attached to the bus. The bus may return with don't care + * (_DC) value. + * Otherwise, The bus will return appropriate _pa or _va iova mode. + * + * @return + * enum rte_iova_mode value. + */ +typedef enum rte_iova_mode (*rte_bus_get_iommu_class_t)(void); + + +/** + * A structure describing a generic bus. + */ +struct rte_bus { + TAILQ_ENTRY(rte_bus) next; /**< Next bus object in linked list */ + const char *name; /**< Name of the bus */ + rte_bus_scan_t scan; /**< Scan for devices attached to bus */ + rte_bus_probe_t probe; /**< Probe devices on bus */ + rte_bus_find_device_t find_device; /**< Find a device on the bus */ + rte_bus_plug_t plug; /**< Probe single device for drivers */ + rte_bus_unplug_t unplug; /**< Remove single device from driver */ + rte_bus_parse_t parse; /**< Parse a device name */ + struct rte_bus_conf conf; /**< Bus configuration */ + rte_bus_get_iommu_class_t get_iommu_class; /**< Get iommu class */ +}; + +/** + * Register a Bus handler. + * + * @param bus + * A pointer to a rte_bus structure describing the bus + * to be registered. + */ +void rte_bus_register(struct rte_bus *bus); + +/** + * Unregister a Bus handler. + * + * @param bus + * A pointer to a rte_bus structure describing the bus + * to be unregistered. + */ +void rte_bus_unregister(struct rte_bus *bus); + +/** + * Scan all the buses. + * + * @return + * 0 in case of success in scanning all buses + * !0 in case of failure to scan + */ +int rte_bus_scan(void); + +/** + * For each device on the buses, perform a driver 'match' and call the + * driver-specific probe for device initialization. + * + * @return + * 0 for successful match/probe + * !0 otherwise + */ +int rte_bus_probe(void); + +/** + * Dump information of all the buses registered with EAL. + * + * @param f + * A valid and open output stream handle + */ +void rte_bus_dump(FILE *f); + +/** + * Bus comparison function. + * + * @param bus + * Bus under test. + * + * @param data + * Data to compare against. + * + * @return + * 0 if the bus matches the data. + * !0 if the bus does not match. + * <0 if ordering is possible and the bus is lower than the data. + * >0 if ordering is possible and the bus is greater than the data. + */ +typedef int (*rte_bus_cmp_t)(const struct rte_bus *bus, const void *data); + +/** + * Bus iterator to find a particular bus. + * + * This function compares each registered bus to find one that matches + * the data passed as parameter. + * + * If the comparison function returns zero this function will stop iterating + * over any more buses. To continue a search the bus of a previous search can + * be passed via the start parameter. + * + * @param start + * Starting point for the iteration. + * + * @param cmp + * Comparison function. + * + * @param data + * Data to pass to comparison function. + * + * @return + * A pointer to a rte_bus structure or NULL in case no bus matches + */ +struct rte_bus *rte_bus_find(const struct rte_bus *start, rte_bus_cmp_t cmp, + const void *data); + +/** + * Find the registered bus for a particular device. + */ +struct rte_bus *rte_bus_find_by_device(const struct rte_device *dev); + +/** + * Find the registered bus for a given name. + */ +struct rte_bus *rte_bus_find_by_name(const char *busname); + + +/** + * Get the common iommu class of devices bound on to buses available in the + * system. The default mode is PA. + * + * @return + * enum rte_iova_mode value. + */ +enum rte_iova_mode rte_bus_get_iommu_class(void); + +/** + * Helper for Bus registration. + * The constructor has higher priority than PMD constructors. + */ +#define RTE_REGISTER_BUS(nm, bus) \ +RTE_INIT_PRIO(businitfn_ ##nm, 110); \ +static void businitfn_ ##nm(void) \ +{\ + (bus).name = RTE_STR(nm);\ + rte_bus_register(&bus); \ +} + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_BUS_H */ diff --git a/dpdk/lib/librte_eal/common/include/rte_common.h b/dpdk/lib/librte_eal/common/include/rte_common.h index 332f2a43..f1d24b86 100644 --- a/dpdk/lib/librte_eal/common/include/rte_common.h +++ b/dpdk/lib/librte_eal/common/include/rte_common.h @@ -51,6 +51,8 @@ extern "C" { #include #include +#include + #ifndef typeof #define typeof __typeof__ #endif @@ -59,6 +61,19 @@ extern "C" { #define asm __asm__ #endif +/** C extension macro for environments lacking C11 features. */ +#if !defined(__STDC_VERSION__) || __STDC_VERSION__ < 201112L +#define RTE_STD_C11 __extension__ +#else +#define RTE_STD_C11 +#endif + +/** Define GCC_VERSION **/ +#ifdef RTE_TOOLCHAIN_GCC +#define GCC_VERSION (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + \ + __GNUC_PATCHLEVEL__) +#endif + #ifdef RTE_ARCH_STRICT_ALIGN typedef uint64_t unaligned_uint64_t __attribute__ ((aligned(1))); typedef uint32_t unaligned_uint32_t __attribute__ ((aligned(1))); @@ -95,6 +110,39 @@ typedef uint16_t unaligned_uint16_t; */ #define RTE_SET_USED(x) (void)(x) +/** + * Run function before main() with low priority. + * + * The constructor will be run after prioritized constructors. + * + * @param func + * Constructor function. + */ +#define RTE_INIT(func) \ +static void __attribute__((constructor, used)) func(void) + +/** + * Run function before main() with high priority. + * + * @param func + * Constructor function. + * @param prio + * Priority number must be above 100. + * Lowest number is the first to run. + */ +#define RTE_INIT_PRIO(func, prio) \ +static void __attribute__((constructor(prio), used)) func(void) + +/** + * Force a function to be inlined + */ +#define __rte_always_inline inline __attribute__((always_inline)) + +/** + * Force a function to be noinlined + */ +#define __rte_noinline __attribute__((noinline)) + /*********** Macros for pointer arithmetic ********/ /** @@ -268,7 +316,8 @@ rte_align64pow2(uint64_t v) /** * Macro to return the minimum of two numbers */ -#define RTE_MIN(a, b) ({ \ +#define RTE_MIN(a, b) \ + __extension__ ({ \ typeof (a) _a = (a); \ typeof (b) _b = (b); \ _a < _b ? _a : _b; \ @@ -277,7 +326,8 @@ rte_align64pow2(uint64_t v) /** * Macro to return the maximum of two numbers */ -#define RTE_MAX(a, b) ({ \ +#define RTE_MAX(a, b) \ + __extension__ ({ \ typeof (a) _a = (a); \ typeof (b) _b = (b); \ _a > _b ? _a : _b; \ @@ -285,21 +335,6 @@ rte_align64pow2(uint64_t v) /*********** Other general functions / macros ********/ -#ifdef __SSE2__ -#include -/** - * PAUSE instruction for tight loops (avoid busy waiting) - */ -static inline void -rte_pause (void) -{ - _mm_pause(); -} -#else -static inline void -rte_pause(void) {} -#endif - /** * Searches the input parameter for the least significant set bit * (starting from zero). @@ -317,15 +352,64 @@ rte_bsf32(uint32_t v) return __builtin_ctz(v); } +/** + * Return the rounded-up log2 of a integer. + * + * @param v + * The input parameter. + * @return + * The rounded-up log2 of the input, or 0 if the input is 0. + */ +static inline uint32_t +rte_log2_u32(uint32_t v) +{ + if (v == 0) + return 0; + v = rte_align32pow2(v); + return rte_bsf32(v); +} + #ifndef offsetof /** Return the offset of a field in a structure. */ #define offsetof(TYPE, MEMBER) __builtin_offsetof (TYPE, MEMBER) #endif +/** + * Return pointer to the wrapping struct instance. + * + * Example: + * + * struct wrapper { + * ... + * struct child c; + * ... + * }; + * + * struct child *x = obtain(...); + * struct wrapper *w = container_of(x, struct wrapper, c); + */ +#ifndef container_of +#define container_of(ptr, type, member) __extension__ ({ \ + const typeof(((type *)0)->member) *_ptr = (ptr); \ + __attribute__((unused)) type *_target_ptr = \ + (type *)(ptr); \ + (type *)(((uintptr_t)_ptr) - offsetof(type, member)); \ + }) +#endif + #define _RTE_STR(x) #x /** Take a macro value and get a string version of it */ #define RTE_STR(x) _RTE_STR(x) +/** + * ISO C helpers to modify format strings using variadic macros. + * This is a replacement for the ", ## __VA_ARGS__" GNU extension. + * An empty %s argument is appended to avoid a dangling comma. + */ +#define RTE_FMT(fmt, ...) fmt "%.0s", __VA_ARGS__ "" +#define RTE_FMT_HEAD(fmt, ...) fmt +#define RTE_FMT_TAIL(fmt, ...) __VA_ARGS__ + /** Mask value of type "tp" for the first "ln" bit set. */ #define RTE_LEN2MASK(ln, tp) \ ((tp)((uint64_t)-1 >> (sizeof(uint64_t) * CHAR_BIT - (ln)))) diff --git a/dpdk/lib/librte_eal/common/include/rte_debug.h b/dpdk/lib/librte_eal/common/include/rte_debug.h index cab6fb4c..79b67b3e 100644 --- a/dpdk/lib/librte_eal/common/include/rte_debug.h +++ b/dpdk/lib/librte_eal/common/include/rte_debug.h @@ -79,7 +79,7 @@ void rte_dump_registers(void); #define rte_panic(...) rte_panic_(__func__, __VA_ARGS__, "dummy") #define rte_panic_(func, format, ...) __rte_panic(func, format "%.0s", __VA_ARGS__) -#if RTE_LOG_LEVEL >= RTE_LOG_DEBUG +#ifdef RTE_ENABLE_ASSERT #define RTE_ASSERT(exp) RTE_VERIFY(exp) #else #define RTE_ASSERT(exp) do {} while (0) diff --git a/dpdk/lib/librte_eal/common/include/rte_dev.h b/dpdk/lib/librte_eal/common/include/rte_dev.h index 95789f9d..8088dcc5 100644 --- a/dpdk/lib/librte_eal/common/include/rte_dev.h +++ b/dpdk/lib/librte_eal/common/include/rte_dev.h @@ -49,6 +49,7 @@ extern "C" { #include #include +#include #include __attribute__((format(printf, 2, 0))) @@ -70,6 +71,19 @@ rte_pmd_debug_trace(const char *func_name, const char *fmt, ...) rte_log(RTE_LOG_ERR, RTE_LOGTYPE_PMD, "%s: %s", func_name, buffer); } +/* + * Enable RTE_PMD_DEBUG_TRACE() when at least one component relying on the + * RTE_*_RET() macros defined below is compiled in debug mode. + */ +#if defined(RTE_LIBRTE_ETHDEV_DEBUG) || \ + defined(RTE_LIBRTE_CRYPTODEV_DEBUG) || \ + defined(RTE_LIBRTE_EVENTDEV_DEBUG) +#define RTE_PMD_DEBUG_TRACE(...) \ + rte_pmd_debug_trace(__func__, __VA_ARGS__) +#else +#define RTE_PMD_DEBUG_TRACE(...) (void)0 +#endif + /* Macros for checking for restricting functions to primary instance only */ #define RTE_PROC_PRIMARY_OR_ERR_RET(retval) do { \ if (rte_eal_process_type() != RTE_PROC_PRIMARY) { \ @@ -100,26 +114,33 @@ rte_pmd_debug_trace(const char *func_name, const char *fmt, ...) } \ } while (0) - -/** Double linked list of device drivers. */ -TAILQ_HEAD(rte_driver_list, rte_driver); +/** + * Device driver. + */ +enum rte_kernel_driver { + RTE_KDRV_UNKNOWN = 0, + RTE_KDRV_IGB_UIO, + RTE_KDRV_VFIO, + RTE_KDRV_UIO_GENERIC, + RTE_KDRV_NIC_UIO, + RTE_KDRV_NONE, +}; /** - * Initialization function called for each device driver once. + * Device policies. */ -typedef int (rte_dev_init_t)(const char *name, const char *args); +enum rte_dev_policy { + RTE_DEV_WHITELISTED, + RTE_DEV_BLACKLISTED, +}; /** - * Uninitilization function called for each device driver once. + * A generic memory resource representation. */ -typedef int (rte_dev_uninit_t)(const char *name); - -/** - * Driver type enumeration - */ -enum pmd_type { - PMD_VDEV = 0, - PMD_PDEV = 1, +struct rte_mem_resource { + uint64_t phys_addr; /**< Physical address, 0 if not resource. */ + uint64_t len; /**< Length of the resource. */ + void *addr; /**< Virtual address, NULL when not mapped. */ }; /** @@ -127,84 +148,150 @@ enum pmd_type { */ struct rte_driver { TAILQ_ENTRY(rte_driver) next; /**< Next in list. */ - enum pmd_type type; /**< PMD Driver type */ const char *name; /**< Driver name. */ - rte_dev_init_t *init; /**< Device init. function. */ - rte_dev_uninit_t *uninit; /**< Device uninit. function. */ + const char *alias; /**< Driver alias. */ +}; + +/* + * Internal identifier length + * Sufficiently large to allow for UUID or PCI address + */ +#define RTE_DEV_NAME_MAX_LEN 64 + +/** + * A structure describing a generic device. + */ +struct rte_device { + TAILQ_ENTRY(rte_device) next; /**< Next device */ + const char *name; /**< Device name */ + const struct rte_driver *driver;/**< Associated driver */ + int numa_node; /**< NUMA node connection */ + struct rte_devargs *devargs; /**< Device user arguments */ }; /** - * Register a device driver. - * - * @param driver - * A pointer to a rte_dev structure describing the driver - * to be registered. - */ -void rte_eal_driver_register(struct rte_driver *driver); - -/** - * Unregister a device driver. - * - * @param driver - * A pointer to a rte_dev structure describing the driver - * to be unregistered. - */ -void rte_eal_driver_unregister(struct rte_driver *driver); - -/** - * Initalize all the registered drivers in this process - */ -int rte_eal_dev_init(void); - -/** - * Initialize a driver specified by name. + * Attach a device to a registered driver. * * @param name - * The pointer to a driver name to be initialized. - * @param args - * The pointer to arguments used by driver initialization. + * The device name, that refers to a pci device (or some private + * way of designating a vdev device). Based on this device name, eal + * will identify a driver capable of handling it and pass it to the + * driver probing function. + * @param devargs + * Device arguments to be passed to the driver. * @return - * 0 on success, negative on error + * 0 on success, negative on error. */ -int rte_eal_vdev_init(const char *name, const char *args); +int rte_eal_dev_attach(const char *name, const char *devargs); /** - * Uninitalize a driver specified by name. + * Detach a device from its driver. * - * @param name - * The pointer to a driver name to be initialized. + * @param dev + * A pointer to a rte_device structure. * @return - * 0 on success, negative on error + * 0 on success, negative on error. */ -int rte_eal_vdev_uninit(const char *name); +int rte_eal_dev_detach(struct rte_device *dev); -#define DRIVER_EXPORT_NAME_ARRAY(n, idx) n##idx[] +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Hotplug add a given device to a specific bus. + * + * @param busname + * The bus name the device is added to. + * @param devname + * The device name. Based on this device name, eal will identify a driver + * capable of handling it and pass it to the driver probing function. + * @param devargs + * Device arguments to be passed to the driver. + * @return + * 0 on success, negative on error. + */ +int rte_eal_hotplug_add(const char *busname, const char *devname, + const char *devargs); -#define DRIVER_EXPORT_NAME(name, idx) \ -static const char DRIVER_EXPORT_NAME_ARRAY(this_pmd_name, idx) \ +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Hotplug remove a given device from a specific bus. + * + * @param busname + * The bus name the device is removed from. + * @param devname + * The device name being removed. + * @return + * 0 on success, negative on error. + */ +int rte_eal_hotplug_remove(const char *busname, const char *devname); + +/** + * Device comparison function. + * + * This type of function is used to compare an rte_device with arbitrary + * data. + * + * @param dev + * Device handle. + * + * @param data + * Data to compare against. The type of this parameter is determined by + * the kind of comparison performed by the function. + * + * @return + * 0 if the device matches the data. + * !0 if the device does not match. + * <0 if ordering is possible and the device is lower than the data. + * >0 if ordering is possible and the device is greater than the data. + */ +typedef int (*rte_dev_cmp_t)(const struct rte_device *dev, const void *data); + +#define RTE_PMD_EXPORT_NAME_ARRAY(n, idx) n##idx[] + +#define RTE_PMD_EXPORT_NAME(name, idx) \ +static const char RTE_PMD_EXPORT_NAME_ARRAY(this_pmd_name, idx) \ __attribute__((used)) = RTE_STR(name) -#define PMD_REGISTER_DRIVER(drv, nm)\ -void devinitfn_ ##drv(void);\ -void __attribute__((constructor, used)) devinitfn_ ##drv(void)\ -{\ - (drv).name = RTE_STR(nm);\ - rte_eal_driver_register(&drv);\ -} \ -DRIVER_EXPORT_NAME(nm, __COUNTER__) - #define DRV_EXP_TAG(name, tag) __##name##_##tag -#define DRIVER_REGISTER_PCI_TABLE(name, table) \ +#define RTE_PMD_REGISTER_PCI_TABLE(name, table) \ static const char DRV_EXP_TAG(name, pci_tbl_export)[] __attribute__((used)) = \ RTE_STR(table) -#define DRIVER_REGISTER_PARAM_STRING(name, str) \ +#define RTE_PMD_REGISTER_PARAM_STRING(name, str) \ static const char DRV_EXP_TAG(name, param_string_export)[] \ __attribute__((used)) = str +/** + * Advertise the list of kernel modules required to run this driver + * + * This string lists the kernel modules required for the devices + * associated to a PMD. The format of each line of the string is: + * " ". + * + * The possible formats for the device pattern are: + * "*" all devices supported by this driver + * "pci:*" all PCI devices supported by this driver + * "pci:v8086:d*:sv*:sd*" all PCI devices supported by this driver + * whose vendor id is 0x8086. + * + * The format of the kernel modules list is a parenthesed expression + * containing logical-and (&) and logical-or (|). + * + * The device pattern and the kmod expression are separated by a space. + * + * Example: + * - "* igb_uio | uio_pci_generic | vfio" + */ +#define RTE_PMD_REGISTER_KMOD_DEP(name, str) \ +static const char DRV_EXP_TAG(name, kmod_dep_export)[] \ +__attribute__((used)) = str + #ifdef __cplusplus } #endif -#endif /* _RTE_VDEV_H_ */ +#endif /* _RTE_DEV_H_ */ diff --git a/dpdk/lib/librte_eal/common/include/rte_devargs.h b/dpdk/lib/librte_eal/common/include/rte_devargs.h index 53c59f56..58d585df 100644 --- a/dpdk/lib/librte_eal/common/include/rte_devargs.h +++ b/dpdk/lib/librte_eal/common/include/rte_devargs.h @@ -50,7 +50,7 @@ extern "C" { #include #include -#include +#include /** * Type of generic device @@ -76,18 +76,12 @@ struct rte_devargs { TAILQ_ENTRY(rte_devargs) next; /** Type of device. */ enum rte_devtype type; - union { - /** Used if type is RTE_DEVTYPE_*_PCI. */ - struct { - /** PCI location. */ - struct rte_pci_addr addr; - } pci; - /** Used if type is RTE_DEVTYPE_VIRTUAL. */ - struct { - /** Driver name. */ - char drv_name[32]; - } virt; - }; + /** Device policy. */ + enum rte_dev_policy policy; + /** Bus handle for the device. */ + struct rte_bus *bus; + /** Name of the device. */ + char name[RTE_DEV_NAME_MAX_LEN]; /** Arguments string as given by user or "" for no argument. */ char *args; }; @@ -106,8 +100,8 @@ extern struct rte_devargs_list devargs_list; * "04:00.0,arg=val". * * For virtual devices, the format of arguments string is "DRIVER_NAME*" - * or "DRIVER_NAME*,key=val,key2=val2,...". Examples: "eth_ring", - * "eth_ring0", "eth_pmdAnything,arg=0:arg2=1". + * or "DRIVER_NAME*,key=val,key2=val2,...". Examples: "net_ring", + * "net_ring0", "net_pmdAnything,arg=0:arg2=1". * * The function parses the arguments string to get driver name and driver * arguments. @@ -126,6 +120,39 @@ extern struct rte_devargs_list devargs_list; int rte_eal_parse_devargs_str(const char *devargs_str, char **drvname, char **drvargs); +/** + * Parse a device string. + * + * Verify that a bus is capable of handling the device passed + * in argument. Store which bus will handle the device, its name + * and the eventual device parameters. + * + * @param dev + * The device declaration string. + * @param da + * The devargs structure holding the device information. + * + * @return + * - 0 on success. + * - Negative errno on error. + */ +int +rte_eal_devargs_parse(const char *dev, + struct rte_devargs *da); + +/** + * Insert an rte_devargs in the global list. + * + * @param da + * The devargs structure to insert. + * + * @return + * - 0 on success + * - Negative on error. + */ +int +rte_eal_devargs_insert(struct rte_devargs *da); + /** * Add a device to the user device list * @@ -134,8 +161,8 @@ int rte_eal_parse_devargs_str(const char *devargs_str, * "04:00.0,arg=val". * * For virtual devices, the format of arguments string is "DRIVER_NAME*" - * or "DRIVER_NAME*,key=val,key2=val2,...". Examples: "eth_ring", - * "eth_ring0", "eth_pmdAnything,arg=0:arg2=1". The validity of the + * or "DRIVER_NAME*,key=val,key2=val2,...". Examples: "net_ring", + * "net_ring0", "net_pmdAnything,arg=0:arg2=1". The validity of the * driver name is not checked by this function, it is done when probing * the drivers. * @@ -150,6 +177,24 @@ int rte_eal_parse_devargs_str(const char *devargs_str, */ int rte_eal_devargs_add(enum rte_devtype devtype, const char *devargs_str); +/** + * Remove a device from the user device list. + * Its resources are freed. + * If the devargs cannot be found, nothing happens. + * + * @param busname + * bus name of the devargs to remove. + * + * @param devname + * device name of the devargs to remove. + * + * @return + * 0 on success. + * <0 on error. + * >0 if the devargs was not within the user device list. + */ +int rte_eal_devargs_remove(const char *busname, const char *devname); + /** * Count the number of user devices of a specified type * diff --git a/dpdk/lib/librte_eal/common/include/rte_eal.h b/dpdk/lib/librte_eal/common/include/rte_eal.h index a71d6f57..78b0e005 100644 --- a/dpdk/lib/librte_eal/common/include/rte_eal.h +++ b/dpdk/lib/librte_eal/common/include/rte_eal.h @@ -43,7 +43,11 @@ #include #include +#include #include +#include + +#include #ifdef __cplusplus extern "C" { @@ -60,6 +64,7 @@ extern "C" { enum rte_lcore_role_t { ROLE_RTE, ROLE_OFF, + ROLE_SERVICE, }; /** @@ -79,11 +84,15 @@ enum rte_proc_type_t { struct rte_config { uint32_t master_lcore; /**< Id of the master lcore */ uint32_t lcore_count; /**< Number of available logical cores. */ + uint32_t service_lcore_count;/**< Number of available service cores. */ enum rte_lcore_role_t lcore_role[RTE_MAX_LCORE]; /**< State of cores. */ /** Primary or secondary configuration */ enum rte_proc_type_t process_type; + /** PA or VA mapping mode */ + enum rte_iova_mode iova_mode; + /** * Pointer to memory configuration, which may be shared across multiple * DPDK instances @@ -145,15 +154,47 @@ int rte_eal_iopl_init(void); * This behavior may change in the future. * * @param argc - * The argc argument that was given to the main() function. + * A non-negative value. If it is greater than 0, the array members + * for argv[0] through argv[argc] (non-inclusive) shall contain pointers + * to strings. * @param argv - * The argv argument that was given to the main() function. + * An array of strings. The contents of the array, as well as the strings + * which are pointed to by the array, may be modified by this function. * @return * - On success, the number of parsed arguments, which is greater or * equal to zero. After the call to rte_eal_init(), - * all arguments argv[x] with x < ret may be modified and should - * not be accessed by the application. - * - On failure, a negative error value. + * all arguments argv[x] with x < ret may have been modified by this + * function call and should not be further interpreted by the + * application. The EAL does not take any ownership of the memory used + * for either the argv array, or its members. + * - On failure, -1 and rte_errno is set to a value indicating the cause + * for failure. In some instances, the application will need to be + * restarted as part of clearing the issue. + * + * Error codes returned via rte_errno: + * EACCES indicates a permissions issue. + * + * EAGAIN indicates either a bus or system resource was not available, + * setup may be attempted again. + * + * EALREADY indicates that the rte_eal_init function has already been + * called, and cannot be called again. + * + * EFAULT indicates the tailq configuration name was not found in + * memory configuration. + * + * EINVAL indicates invalid parameters were passed as argv/argc. + * + * ENOMEM indicates failure likely caused by an out-of-memory condition. + * + * ENODEV indicates memory setup issues. + * + * ENOTSUP indicates that the EAL cannot initialize on this system. + * + * EPROTO indicates that the PCI bus is either not present, or is not + * readable by the eal. + * + * ENOEXEC indicates that a service core failed to launch successfully. */ int rte_eal_init(int argc, char **argv); @@ -177,7 +218,7 @@ int rte_eal_primary_proc_alive(const char *config_file_path); /** * Usage function typedef used by the application usage function. * - * Use this function typedef to define and call rte_set_applcation_usage_hook() + * Use this function typedef to define and call rte_set_application_usage_hook() * routine. */ typedef void (*rte_usage_hook_t)(const char * prgname); @@ -228,6 +269,32 @@ rte_set_application_usage_hook(rte_usage_hook_t usage_func); */ int rte_eal_has_hugepages(void); +/** + * Whether EAL is using PCI bus. + * Disabled by --no-pci option. + * + * @return + * Nonzero if the PCI bus is enabled. + */ +int rte_eal_has_pci(void); + +/** + * Whether the EAL was asked to create UIO device. + * + * @return + * Nonzero if true. + */ +int rte_eal_create_uio_dev(void); + +/** + * The user-configured vfio interrupt mode. + * + * @return + * Interrupt mode configured with the command line, + * RTE_INTR_MODE_NONE by default. + */ +enum rte_intr_mode rte_eal_vfio_intr_mode(void); + /** * A wrap API for syscall gettid. * @@ -252,6 +319,23 @@ static inline int rte_gettid(void) return RTE_PER_LCORE(_thread_id); } +/** + * Get the iova mode + * + * @return + * enum rte_iova_mode value. + */ +enum rte_iova_mode rte_eal_iova_mode(void); + +/** + * Get default pool ops name for mbuf + * + * @return + * returns default pool ops name. + */ +const char * +rte_eal_mbuf_default_mempool_ops(void); + #ifdef __cplusplus } #endif diff --git a/dpdk/lib/librte_eal/common/include/rte_eal_interrupts.h b/dpdk/lib/librte_eal/common/include/rte_eal_interrupts.h new file mode 100644 index 00000000..031f78cc --- /dev/null +++ b/dpdk/lib/librte_eal/common/include/rte_eal_interrupts.h @@ -0,0 +1,250 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_INTERRUPTS_H_ +#error "don't include this file directly, please include generic " +#endif + +/** + * @file rte_eal_interrupts.h + * @internal + * + * Contains function prototypes exposed by the EAL for interrupt handling by + * drivers and other DPDK internal consumers. + */ + +#ifndef _RTE_EAL_INTERRUPTS_H_ +#define _RTE_EAL_INTERRUPTS_H_ + +#define RTE_MAX_RXTX_INTR_VEC_ID 32 +#define RTE_INTR_VEC_ZERO_OFFSET 0 +#define RTE_INTR_VEC_RXTX_OFFSET 1 + +/** + * The interrupt source type, e.g. UIO, VFIO, ALARM etc. + */ +enum rte_intr_handle_type { + RTE_INTR_HANDLE_UNKNOWN = 0, /**< generic unknown handle */ + RTE_INTR_HANDLE_UIO, /**< uio device handle */ + RTE_INTR_HANDLE_UIO_INTX, /**< uio generic handle */ + RTE_INTR_HANDLE_VFIO_LEGACY, /**< vfio device handle (legacy) */ + RTE_INTR_HANDLE_VFIO_MSI, /**< vfio device handle (MSI) */ + RTE_INTR_HANDLE_VFIO_MSIX, /**< vfio device handle (MSIX) */ + RTE_INTR_HANDLE_ALARM, /**< alarm handle */ + RTE_INTR_HANDLE_EXT, /**< external handler */ + RTE_INTR_HANDLE_VDEV, /**< virtual device */ + RTE_INTR_HANDLE_MAX /**< count of elements */ +}; + +#define RTE_INTR_EVENT_ADD 1UL +#define RTE_INTR_EVENT_DEL 2UL + +typedef void (*rte_intr_event_cb_t)(int fd, void *arg); + +struct rte_epoll_data { + uint32_t event; /**< event type */ + void *data; /**< User data */ + rte_intr_event_cb_t cb_fun; /**< IN: callback fun */ + void *cb_arg; /**< IN: callback arg */ +}; + +enum { + RTE_EPOLL_INVALID = 0, + RTE_EPOLL_VALID, + RTE_EPOLL_EXEC, +}; + +/** interrupt epoll event obj, taken by epoll_event.ptr */ +struct rte_epoll_event { + volatile uint32_t status; /**< OUT: event status */ + int fd; /**< OUT: event fd */ + int epfd; /**< OUT: epoll instance the ev associated with */ + struct rte_epoll_data epdata; +}; + +/** Handle for interrupts. */ +struct rte_intr_handle { + RTE_STD_C11 + union { + int vfio_dev_fd; /**< VFIO device file descriptor */ + int uio_cfg_fd; /**< UIO cfg file desc for uio_pci_generic */ + }; + int fd; /**< interrupt event file descriptor */ + enum rte_intr_handle_type type; /**< handle type */ + uint32_t max_intr; /**< max interrupt requested */ + uint32_t nb_efd; /**< number of available efd(event fd) */ + uint8_t efd_counter_size; /**< size of efd counter, used for vdev */ + int efds[RTE_MAX_RXTX_INTR_VEC_ID]; /**< intr vectors/efds mapping */ + struct rte_epoll_event elist[RTE_MAX_RXTX_INTR_VEC_ID]; + /**< intr vector epoll event */ + int *intr_vec; /**< intr vector number array */ +}; + +#define RTE_EPOLL_PER_THREAD -1 /**< to hint using per thread epfd */ + +/** + * It waits for events on the epoll instance. + * + * @param epfd + * Epoll instance fd on which the caller wait for events. + * @param events + * Memory area contains the events that will be available for the caller. + * @param maxevents + * Up to maxevents are returned, must greater than zero. + * @param timeout + * Specifying a timeout of -1 causes a block indefinitely. + * Specifying a timeout equal to zero cause to return immediately. + * @return + * - On success, returns the number of available event. + * - On failure, a negative value. + */ +int +rte_epoll_wait(int epfd, struct rte_epoll_event *events, + int maxevents, int timeout); + +/** + * It performs control operations on epoll instance referred by the epfd. + * It requests that the operation op be performed for the target fd. + * + * @param epfd + * Epoll instance fd on which the caller perform control operations. + * @param op + * The operation be performed for the target fd. + * @param fd + * The target fd on which the control ops perform. + * @param event + * Describes the object linked to the fd. + * Note: The caller must take care the object deletion after CTL_DEL. + * @return + * - On success, zero. + * - On failure, a negative value. + */ +int +rte_epoll_ctl(int epfd, int op, int fd, + struct rte_epoll_event *event); + +/** + * The function returns the per thread epoll instance. + * + * @return + * epfd the epoll instance referred to. + */ +int +rte_intr_tls_epfd(void); + +/** + * @param intr_handle + * Pointer to the interrupt handle. + * @param epfd + * Epoll instance fd which the intr vector associated to. + * @param op + * The operation be performed for the vector. + * Operation type of {ADD, DEL}. + * @param vec + * RX intr vector number added to the epoll instance wait list. + * @param data + * User raw data. + * @return + * - On success, zero. + * - On failure, a negative value. + */ +int +rte_intr_rx_ctl(struct rte_intr_handle *intr_handle, + int epfd, int op, unsigned int vec, void *data); + +/** + * It deletes registered eventfds. + * + * @param intr_handle + * Pointer to the interrupt handle. + */ +void +rte_intr_free_epoll_fd(struct rte_intr_handle *intr_handle); + +/** + * It enables the packet I/O interrupt event if it's necessary. + * It creates event fd for each interrupt vector when MSIX is used, + * otherwise it multiplexes a single event fd. + * + * @param intr_handle + * Pointer to the interrupt handle. + * @param nb_efd + * Number of interrupt vector trying to enable. + * The value 0 is not allowed. + * @return + * - On success, zero. + * - On failure, a negative value. + */ +int +rte_intr_efd_enable(struct rte_intr_handle *intr_handle, uint32_t nb_efd); + +/** + * It disables the packet I/O interrupt event. + * It deletes registered eventfds and closes the open fds. + * + * @param intr_handle + * Pointer to the interrupt handle. + */ +void +rte_intr_efd_disable(struct rte_intr_handle *intr_handle); + +/** + * The packet I/O interrupt on datapath is enabled or not. + * + * @param intr_handle + * Pointer to the interrupt handle. + */ +int +rte_intr_dp_is_en(struct rte_intr_handle *intr_handle); + +/** + * The interrupt handle instance allows other causes or not. + * Other causes stand for any none packet I/O interrupts. + * + * @param intr_handle + * Pointer to the interrupt handle. + */ +int +rte_intr_allow_others(struct rte_intr_handle *intr_handle); + +/** + * The multiple interrupt vector capability of interrupt handle instance. + * It returns zero if no multiple interrupt vector support. + * + * @param intr_handle + * Pointer to the interrupt handle. + */ +int +rte_intr_cap_multiple(struct rte_intr_handle *intr_handle); + +#endif /* _RTE_EAL_INTERRUPTS_H_ */ diff --git a/dpdk/lib/librte_eal/common/include/rte_eal_memconfig.h b/dpdk/lib/librte_eal/common/include/rte_eal_memconfig.h index 2b5e0b17..c963b450 100644 --- a/dpdk/lib/librte_eal/common/include/rte_eal_memconfig.h +++ b/dpdk/lib/librte_eal/common/include/rte_eal_memconfig.h @@ -34,11 +34,13 @@ #ifndef _RTE_EAL_MEMCONFIG_H_ #define _RTE_EAL_MEMCONFIG_H_ +#include #include #include #include #include #include +#include #ifdef __cplusplus extern "C" { diff --git a/dpdk/lib/librte_eal/common/include/rte_interrupts.h b/dpdk/lib/librte_eal/common/include/rte_interrupts.h index ff11ef3a..43177c7a 100644 --- a/dpdk/lib/librte_eal/common/include/rte_interrupts.h +++ b/dpdk/lib/librte_eal/common/include/rte_interrupts.h @@ -34,6 +34,8 @@ #ifndef _RTE_INTERRUPTS_H_ #define _RTE_INTERRUPTS_H_ +#include + /** * @file * @@ -49,10 +51,9 @@ extern "C" { struct rte_intr_handle; /** Function to be registered for the specific interrupt */ -typedef void (*rte_intr_callback_fn)(struct rte_intr_handle *intr_handle, - void *cb_arg); +typedef void (*rte_intr_callback_fn)(void *cb_arg); -#include +#include "rte_eal_interrupts.h" /** * It registers the callback for the specific interrupt. Multiple @@ -68,7 +69,7 @@ typedef void (*rte_intr_callback_fn)(struct rte_intr_handle *intr_handle, * - On success, zero. * - On failure, a negative value. */ -int rte_intr_callback_register(struct rte_intr_handle *intr_handle, +int rte_intr_callback_register(const struct rte_intr_handle *intr_handle, rte_intr_callback_fn cb, void *cb_arg); /** @@ -86,7 +87,7 @@ int rte_intr_callback_register(struct rte_intr_handle *intr_handle, * - On success, return the number of callback entities removed. * - On failure, a negative value. */ -int rte_intr_callback_unregister(struct rte_intr_handle *intr_handle, +int rte_intr_callback_unregister(const struct rte_intr_handle *intr_handle, rte_intr_callback_fn cb, void *cb_arg); /** @@ -99,7 +100,7 @@ int rte_intr_callback_unregister(struct rte_intr_handle *intr_handle, * - On success, zero. * - On failure, a negative value. */ -int rte_intr_enable(struct rte_intr_handle *intr_handle); +int rte_intr_enable(const struct rte_intr_handle *intr_handle); /** * It disables the interrupt for the specified handle. @@ -111,7 +112,7 @@ int rte_intr_enable(struct rte_intr_handle *intr_handle); * - On success, zero. * - On failure, a negative value. */ -int rte_intr_disable(struct rte_intr_handle *intr_handle); +int rte_intr_disable(const struct rte_intr_handle *intr_handle); #ifdef __cplusplus } diff --git a/dpdk/lib/librte_eal/common/include/rte_keepalive.h b/dpdk/lib/librte_eal/common/include/rte_keepalive.h index 88ad8e48..e9f8f083 100644 --- a/dpdk/lib/librte_eal/common/include/rte_keepalive.h +++ b/dpdk/lib/librte_eal/common/include/rte_keepalive.h @@ -39,6 +39,7 @@ #ifndef _KEEPALIVE_H_ #define _KEEPALIVE_H_ +#include #include #ifndef RTE_KEEPALIVE_MAXCORES diff --git a/dpdk/lib/librte_eal/common/include/rte_lcore.h b/dpdk/lib/librte_eal/common/include/rte_lcore.h index fe7b5865..3735da0c 100644 --- a/dpdk/lib/librte_eal/common/include/rte_lcore.h +++ b/dpdk/lib/librte_eal/common/include/rte_lcore.h @@ -40,6 +40,7 @@ * API for lcore and socket manipulation * */ +#include #include #include #include @@ -73,6 +74,7 @@ struct lcore_config { unsigned core_id; /**< core number on socket for this lcore */ int core_index; /**< relative index, starting from 0 */ rte_cpuset_t cpuset; /**< cpu set which the lcore affinity to */ + uint8_t core_role; /**< role of core eg: OFF, RTE, SERVICE */ }; /** @@ -175,7 +177,7 @@ rte_lcore_is_enabled(unsigned lcore_id) struct rte_config *cfg = rte_eal_get_configuration(); if (lcore_id >= RTE_MAX_LCORE) return 0; - return cfg->lcore_role[lcore_id] != ROLE_OFF; + return cfg->lcore_role[lcore_id] == ROLE_RTE; } /** @@ -261,6 +263,20 @@ void rte_thread_get_affinity(rte_cpuset_t *cpusetp); */ int rte_thread_setname(pthread_t id, const char *name); +/** + * Test if the core supplied has a specific role + * + * @param lcore_id + * The identifier of the lcore, which MUST be between 0 and + * RTE_MAX_LCORE-1. + * @param role + * The role to be checked against. + * @return + * On success, return 0; otherwise return a negative value. + */ +int +rte_lcore_has_role(unsigned int lcore_id, enum rte_lcore_role_t role); + #ifdef __cplusplus } #endif diff --git a/dpdk/lib/librte_eal/common/include/rte_log.h b/dpdk/lib/librte_eal/common/include/rte_log.h index b1add04c..0bb10689 100644 --- a/dpdk/lib/librte_eal/common/include/rte_log.h +++ b/dpdk/lib/librte_eal/common/include/rte_log.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -42,8 +42,6 @@ * This file provides a log API to RTE applications. */ -#include "rte_common.h" /* for __rte_deprecated macro */ - #ifdef __cplusplus extern "C" { #endif @@ -52,45 +50,58 @@ extern "C" { #include #include +#include +#include + +struct rte_log_dynamic_type; + /** The rte_log structure. */ struct rte_logs { uint32_t type; /**< Bitfield with enabled logs. */ uint32_t level; /**< Log level. */ - FILE *file; /**< Pointer to current FILE* for logs. */ + FILE *file; /**< Output file set by rte_openlog_stream, or NULL. */ + size_t dynamic_types_len; + struct rte_log_dynamic_type *dynamic_types; }; /** Global log informations */ extern struct rte_logs rte_logs; /* SDK log type */ -#define RTE_LOGTYPE_EAL 0x00000001 /**< Log related to eal. */ -#define RTE_LOGTYPE_MALLOC 0x00000002 /**< Log related to malloc. */ -#define RTE_LOGTYPE_RING 0x00000004 /**< Log related to ring. */ -#define RTE_LOGTYPE_MEMPOOL 0x00000008 /**< Log related to mempool. */ -#define RTE_LOGTYPE_TIMER 0x00000010 /**< Log related to timers. */ -#define RTE_LOGTYPE_PMD 0x00000020 /**< Log related to poll mode driver. */ -#define RTE_LOGTYPE_HASH 0x00000040 /**< Log related to hash table. */ -#define RTE_LOGTYPE_LPM 0x00000080 /**< Log related to LPM. */ -#define RTE_LOGTYPE_KNI 0x00000100 /**< Log related to KNI. */ -#define RTE_LOGTYPE_ACL 0x00000200 /**< Log related to ACL. */ -#define RTE_LOGTYPE_POWER 0x00000400 /**< Log related to power. */ -#define RTE_LOGTYPE_METER 0x00000800 /**< Log related to QoS meter. */ -#define RTE_LOGTYPE_SCHED 0x00001000 /**< Log related to QoS port scheduler. */ -#define RTE_LOGTYPE_PORT 0x00002000 /**< Log related to port. */ -#define RTE_LOGTYPE_TABLE 0x00004000 /**< Log related to table. */ -#define RTE_LOGTYPE_PIPELINE 0x00008000 /**< Log related to pipeline. */ -#define RTE_LOGTYPE_MBUF 0x00010000 /**< Log related to mbuf. */ -#define RTE_LOGTYPE_CRYPTODEV 0x00020000 /**< Log related to cryptodev. */ +#define RTE_LOGTYPE_EAL 0 /**< Log related to eal. */ +#define RTE_LOGTYPE_MALLOC 1 /**< Log related to malloc. */ +#define RTE_LOGTYPE_RING 2 /**< Log related to ring. */ +#define RTE_LOGTYPE_MEMPOOL 3 /**< Log related to mempool. */ +#define RTE_LOGTYPE_TIMER 4 /**< Log related to timers. */ +#define RTE_LOGTYPE_PMD 5 /**< Log related to poll mode driver. */ +#define RTE_LOGTYPE_HASH 6 /**< Log related to hash table. */ +#define RTE_LOGTYPE_LPM 7 /**< Log related to LPM. */ +#define RTE_LOGTYPE_KNI 8 /**< Log related to KNI. */ +#define RTE_LOGTYPE_ACL 9 /**< Log related to ACL. */ +#define RTE_LOGTYPE_POWER 10 /**< Log related to power. */ +#define RTE_LOGTYPE_METER 11 /**< Log related to QoS meter. */ +#define RTE_LOGTYPE_SCHED 12 /**< Log related to QoS port scheduler. */ +#define RTE_LOGTYPE_PORT 13 /**< Log related to port. */ +#define RTE_LOGTYPE_TABLE 14 /**< Log related to table. */ +#define RTE_LOGTYPE_PIPELINE 15 /**< Log related to pipeline. */ +#define RTE_LOGTYPE_MBUF 16 /**< Log related to mbuf. */ +#define RTE_LOGTYPE_CRYPTODEV 17 /**< Log related to cryptodev. */ +#define RTE_LOGTYPE_EFD 18 /**< Log related to EFD. */ +#define RTE_LOGTYPE_EVENTDEV 19 /**< Log related to eventdev. */ +#define RTE_LOGTYPE_GSO 20 /**< Log related to GSO. */ /* these log types can be used in an application */ -#define RTE_LOGTYPE_USER1 0x01000000 /**< User-defined log type 1. */ -#define RTE_LOGTYPE_USER2 0x02000000 /**< User-defined log type 2. */ -#define RTE_LOGTYPE_USER3 0x04000000 /**< User-defined log type 3. */ -#define RTE_LOGTYPE_USER4 0x08000000 /**< User-defined log type 4. */ -#define RTE_LOGTYPE_USER5 0x10000000 /**< User-defined log type 5. */ -#define RTE_LOGTYPE_USER6 0x20000000 /**< User-defined log type 6. */ -#define RTE_LOGTYPE_USER7 0x40000000 /**< User-defined log type 7. */ -#define RTE_LOGTYPE_USER8 0x80000000 /**< User-defined log type 8. */ +#define RTE_LOGTYPE_USER1 24 /**< User-defined log type 1. */ +#define RTE_LOGTYPE_USER2 25 /**< User-defined log type 2. */ +#define RTE_LOGTYPE_USER3 26 /**< User-defined log type 3. */ +#define RTE_LOGTYPE_USER4 27 /**< User-defined log type 4. */ +#define RTE_LOGTYPE_USER5 28 /**< User-defined log type 5. */ +#define RTE_LOGTYPE_USER6 29 /**< User-defined log type 6. */ +#define RTE_LOGTYPE_USER7 30 /**< User-defined log type 7. */ +#define RTE_LOGTYPE_USER8 31 /**< User-defined log type 8. */ + +/** First identifier for extended logs */ +#define RTE_LOGTYPE_FIRST_EXT_ID 32 /* Can't use 0, as it gives compiler warnings */ #define RTE_LOG_EMERG 1U /**< System is unusable. */ @@ -102,9 +113,6 @@ extern struct rte_logs rte_logs; #define RTE_LOG_INFO 7U /**< Informational. */ #define RTE_LOG_DEBUG 8U /**< Debug-level messages. */ -/** The default log stream. */ -extern FILE *eal_default_log_stream; - /** * Change the stream that will be used by the logging system. * @@ -123,34 +131,55 @@ int rte_openlog_stream(FILE *f); /** * Set the global log level. * - * After this call, all logs that are lower or equal than level and - * lower or equal than the RTE_LOG_LEVEL configuration option will be - * displayed. + * After this call, logs with a level lower or equal than the level + * passed as argument will be displayed. * * @param level * Log level. A value between RTE_LOG_EMERG (1) and RTE_LOG_DEBUG (8). */ -void rte_set_log_level(uint32_t level); +void rte_log_set_global_level(uint32_t level); /** * Get the global log level. - */ -uint32_t rte_get_log_level(void); - -/** - * Enable or disable the log type. * - * @param type - * Log type, for example, RTE_LOGTYPE_EAL. - * @param enable - * True for enable; false for disable. + * @return + * The current global log level. */ -void rte_set_log_type(uint32_t type, int enable); +uint32_t rte_log_get_global_level(void); /** - * Get the global log type. + * Get the log level for a given type. + * + * @param logtype + * The log type identifier. + * @return + * 0 on success, a negative value if logtype is invalid. */ -uint32_t rte_get_log_type(void); +int rte_log_get_level(uint32_t logtype); + +/** + * Set the log level for a given type. + * + * @param pattern + * The regexp identifying the log type. + * @param level + * The level to be set. + * @return + * 0 on success, a negative value if level is invalid. + */ +int rte_log_set_level_regexp(const char *pattern, uint32_t level); + +/** + * Set the log level for a given type. + * + * @param logtype + * The log type identifier. + * @param level + * The level to be set. + * @return + * 0 on success, a negative value if logtype or level is invalid. + */ +int rte_log_set_level(uint32_t logtype, uint32_t level); /** * Get the current loglevel for the message being processed. @@ -181,43 +210,28 @@ int rte_log_cur_msg_loglevel(void); int rte_log_cur_msg_logtype(void); /** - * @deprecated - * Enable or disable the history (enabled by default) + * Register a dynamic log type * - * @param enable - * true to enable, or 0 to disable history. + * If a log is already registered with the same type, the returned value + * is the same than the previous one. + * + * @param name + * The string identifying the log type. + * @return + * - >0: success, the returned value is the log type identifier. + * - (-ENOMEM): cannot allocate memory. */ -__rte_deprecated -void rte_log_set_history(int enable); +int rte_log_register(const char *name); /** - * @deprecated - * Dump the log history to a file + * Dump log information. + * + * Dump the global level and the registered log types. * * @param f - * A pointer to a file for output + * The output stream where the dump should be sent. */ -__rte_deprecated -void rte_log_dump_history(FILE *f); - -/** - * @deprecated - * Add a log message to the history. - * - * This function can be called from a user-defined log stream. It adds - * the given message in the history that can be dumped using - * rte_log_dump_history(). - * - * @param buf - * A data buffer containing the message to be saved in the history. - * @param size - * The length of the data buffer. - * @return - * - 0: Success. - * - (-ENOBUFS) if there is no room to store the message. - */ -__rte_deprecated -int rte_log_add_in_history(const char *buf, size_t size); +void rte_log_dump(FILE *f); /** * Generates a log message. @@ -228,9 +242,8 @@ int rte_log_add_in_history(const char *buf, size_t size); * The level argument determines if the log should be displayed or * not, depending on the global rte_logs variable. * - * The preferred alternative is the RTE_LOG() function because debug logs may - * be removed at compilation time if optimization is enabled. Moreover, - * logs are automatically prefixed by type when using the macro. + * The preferred alternative is the RTE_LOG() because it adds the + * level and type in the logged string. * * @param level * Log level. A value between RTE_LOG_EMERG (1) and RTE_LOG_DEBUG (8). @@ -261,8 +274,8 @@ int rte_log(uint32_t level, uint32_t logtype, const char *format, ...) * not, depending on the global rte_logs variable. A trailing * newline may be added if needed. * - * The preferred alternative is the RTE_LOG() because debug logs may be - * removed at compilation time. + * The preferred alternative is the RTE_LOG() because it adds the + * level and type in the logged string. * * @param level * Log level. A value between RTE_LOG_EMERG (1) and RTE_LOG_DEBUG (8). @@ -283,15 +296,8 @@ int rte_vlog(uint32_t level, uint32_t logtype, const char *format, va_list ap) /** * Generates a log message. * - * The RTE_LOG() is equivalent to rte_log() with two differences: - - * - RTE_LOG() can be used to remove debug logs at compilation time, - * depending on RTE_LOG_LEVEL configuration option, and compilation - * optimization level. If optimization is enabled, the tests - * involving constants only are pre-computed. If compilation is done - * with -O0, these tests will be done at run time. - * - The log level and log type names are smaller, for example: - * RTE_LOG(INFO, EAL, "this is a %s", "log"); + * The RTE_LOG() is a helper that prefixes the string with the log level + * and type, and call rte_log(). * * @param l * Log level. A value between EMERG (1) and DEBUG (8). The short name is @@ -307,7 +313,31 @@ int rte_vlog(uint32_t level, uint32_t logtype, const char *format, va_list ap) * - Negative on error. */ #define RTE_LOG(l, t, ...) \ - (void)((RTE_LOG_ ## l <= RTE_LOG_LEVEL) ? \ + rte_log(RTE_LOG_ ## l, \ + RTE_LOGTYPE_ ## t, # t ": " __VA_ARGS__) + +/** + * Generates a log message for data path. + * + * Similar to RTE_LOG(), except that it is removed at compilation time + * if the RTE_LOG_DP_LEVEL configuration option is lower than the log + * level argument. + * + * @param l + * Log level. A value between EMERG (1) and DEBUG (8). The short name is + * expanded by the macro, so it cannot be an integer value. + * @param t + * The log type, for example, EAL. The short name is expanded by the + * macro, so it cannot be an integer value. + * @param ... + * The fmt string, as in printf(3), followed by the variable arguments + * required by the format. + * @return + * - 0: Success. + * - Negative on error. + */ +#define RTE_LOG_DP(l, t, ...) \ + (void)((RTE_LOG_ ## l <= RTE_LOG_DP_LEVEL) ? \ rte_log(RTE_LOG_ ## l, \ RTE_LOGTYPE_ ## t, # t ": " __VA_ARGS__) : \ 0) diff --git a/dpdk/lib/librte_eal/common/include/rte_malloc.h b/dpdk/lib/librte_eal/common/include/rte_malloc.h index 74bb78c7..5d4c11a7 100644 --- a/dpdk/lib/librte_eal/common/include/rte_malloc.h +++ b/dpdk/lib/librte_eal/common/include/rte_malloc.h @@ -294,7 +294,7 @@ rte_malloc_get_socket_stats(int socket, /** * Dump statistics. * - * Dump for the specified type to the console. If the type argument is + * Dump for the specified type to a file. If the type argument is * NULL, all memory types will be dumped. * * @param f @@ -323,17 +323,24 @@ int rte_malloc_set_limit(const char *type, size_t max); /** - * Return the physical address of a virtual address obtained through + * Return the IO address of a virtual address obtained through * rte_malloc * * @param addr - * Adress obtained from a previous rte_malloc call + * Address obtained from a previous rte_malloc call * @return - * NULL on error - * otherwise return physical address of the buffer + * RTE_BAD_IOVA on error + * otherwise return an address suitable for IO */ -phys_addr_t -rte_malloc_virt2phy(const void *addr); +rte_iova_t +rte_malloc_virt2iova(const void *addr); + +__rte_deprecated +static inline phys_addr_t +rte_malloc_virt2phy(const void *addr) +{ + return rte_malloc_virt2iova(addr); +} #ifdef __cplusplus } diff --git a/dpdk/lib/librte_eal/common/include/rte_memory.h b/dpdk/lib/librte_eal/common/include/rte_memory.h index 06611093..80a8fc02 100644 --- a/dpdk/lib/librte_eal/common/include/rte_memory.h +++ b/dpdk/lib/librte_eal/common/include/rte_memory.h @@ -44,16 +44,14 @@ #include #include -#ifdef RTE_EXEC_ENV_LINUXAPP -#include -#endif - #ifdef __cplusplus extern "C" { #endif #include +#include +__extension__ enum rte_page_sizes { RTE_PGSIZE_4K = 1ULL << 12, RTE_PGSIZE_64K = 1ULL << 16, @@ -95,30 +93,37 @@ enum rte_page_sizes { */ #define __rte_cache_min_aligned __rte_aligned(RTE_CACHE_LINE_MIN_SIZE) -typedef uint64_t phys_addr_t; /**< Physical address definition. */ +typedef uint64_t phys_addr_t; /**< Physical address. */ #define RTE_BAD_PHYS_ADDR ((phys_addr_t)-1) +/** + * IO virtual address type. + * When the physical addressing mode (IOVA as PA) is in use, + * the translation from an IO virtual address (IOVA) to a physical address + * is a direct mapping, i.e. the same value. + * Otherwise, in virtual mode (IOVA as VA), an IOMMU may do the translation. + */ +typedef uint64_t rte_iova_t; +#define RTE_BAD_IOVA ((rte_iova_t)-1) /** * Physical memory segment descriptor. */ struct rte_memseg { - phys_addr_t phys_addr; /**< Start physical address. */ + RTE_STD_C11 + union { + phys_addr_t phys_addr; /**< deprecated - Start physical address. */ + rte_iova_t iova; /**< Start IO address. */ + }; + RTE_STD_C11 union { void *addr; /**< Start virtual address. */ uint64_t addr_64; /**< Makes sure addr is always 64 bits */ }; -#ifdef RTE_LIBRTE_IVSHMEM - phys_addr_t ioremap_addr; /**< Real physical address inside the VM */ -#endif size_t len; /**< Length of the segment. */ uint64_t hugepage_sz; /**< The pagesize of underlying memory */ int32_t socket_id; /**< NUMA socket ID. */ uint32_t nchannel; /**< Number of channels. */ uint32_t nrank; /**< Number of ranks. */ -#ifdef RTE_LIBRTE_XEN_DOM0 - /**< store segment MFNs */ - uint64_t mfn[DOM0_NUM_MEMBLOCK]; -#endif } __rte_packed; /** @@ -139,10 +144,20 @@ int rte_mem_lock_page(const void *virt); * @param virt * The virtual address. * @return - * The physical address or RTE_BAD_PHYS_ADDR on error. + * The physical address or RTE_BAD_IOVA on error. */ phys_addr_t rte_mem_virt2phy(const void *virt); +/** + * Get IO virtual address of any mapped virtual address in the current process. + * + * @param virt + * The virtual address. + * @return + * The IO address or RTE_BAD_IOVA on error. + */ +rte_iova_t rte_mem_virt2iova(const void *virt); + /** * Get the layout of the available physical memory. * @@ -161,7 +176,7 @@ phys_addr_t rte_mem_virt2phy(const void *virt); const struct rte_memseg *rte_eal_get_physmem_layout(void); /** - * Dump the physical memory layout to the console. + * Dump the physical memory layout to a file. * * @param f * A pointer to a file for output @@ -194,68 +209,16 @@ unsigned rte_memory_get_nchannel(void); */ unsigned rte_memory_get_nrank(void); -#ifdef RTE_LIBRTE_XEN_DOM0 - -/**< Internal use only - should DOM0 memory mapping be used */ -int rte_xen_dom0_supported(void); - -/**< Internal use only - phys to virt mapping for xen */ -phys_addr_t rte_xen_mem_phy2mch(int32_t, const phys_addr_t); - /** - * Return the physical address of elt, which is an element of the pool mp. - * - * @param memseg_id - * Identifier of the memory segment owning the physical address. If - * set to -1, find it automatically. - * @param phy_addr - * physical address of elt. + * Drivers based on uio will not load unless physical + * addresses are obtainable. It is only possible to get + * physical addresses when running as a privileged user. * * @return - * The physical address or RTE_BAD_PHYS_ADDR on error. + * 1 if the system is able to obtain physical addresses. + * 0 if using DMA addresses through an IOMMU. */ -static inline phys_addr_t -rte_mem_phy2mch(int32_t memseg_id, const phys_addr_t phy_addr) -{ - if (rte_xen_dom0_supported()) - return rte_xen_mem_phy2mch(memseg_id, phy_addr); - else - return phy_addr; -} - -/** - * Memory init for supporting application running on Xen domain0. - * - * @param void - * - * @return - * 0: successfully - * negative: error - */ -int rte_xen_dom0_memory_init(void); - -/** - * Attach to memory setments of primary process on Xen domain0. - * - * @param void - * - * @return - * 0: successfully - * negative: error - */ -int rte_xen_dom0_memory_attach(void); -#else -static inline int rte_xen_dom0_supported(void) -{ - return 0; -} - -static inline phys_addr_t -rte_mem_phy2mch(int32_t memseg_id __rte_unused, const phys_addr_t phy_addr) -{ - return phy_addr; -} -#endif +int rte_eal_using_phys_addrs(void); #ifdef __cplusplus } diff --git a/dpdk/lib/librte_eal/common/include/rte_memzone.h b/dpdk/lib/librte_eal/common/include/rte_memzone.h index f69b5a87..6f0ba182 100644 --- a/dpdk/lib/librte_eal/common/include/rte_memzone.h +++ b/dpdk/lib/librte_eal/common/include/rte_memzone.h @@ -53,6 +53,7 @@ #include #include +#include #ifdef __cplusplus extern "C" { @@ -77,14 +78,16 @@ struct rte_memzone { #define RTE_MEMZONE_NAMESIZE 32 /**< Maximum length of memory zone name.*/ char name[RTE_MEMZONE_NAMESIZE]; /**< Name of the memory zone. */ - phys_addr_t phys_addr; /**< Start physical address. */ + RTE_STD_C11 + union { + phys_addr_t phys_addr; /**< deprecated - Start physical address. */ + rte_iova_t iova; /**< Start IO address. */ + }; + RTE_STD_C11 union { void *addr; /**< Start virtual address. */ uint64_t addr_64; /**< Makes sure addr is always 64-bits */ }; -#ifdef RTE_LIBRTE_IVSHMEM - phys_addr_t ioremap_addr; /**< Real physical address inside the VM */ -#endif size_t len; /**< Length of the memzone. */ uint64_t hugepage_sz; /**< The page size of underlying memory */ @@ -256,12 +259,10 @@ const struct rte_memzone *rte_memzone_reserve_bounded(const char *name, /** * Free a memzone. * - * Note: an IVSHMEM zone cannot be freed. - * * @param mz * A pointer to the memzone * @return - * -EINVAL - invalid parameter, IVSHMEM memzone. + * -EINVAL - invalid parameter. * 0 - success */ int rte_memzone_free(const struct rte_memzone *mz); @@ -280,7 +281,7 @@ int rte_memzone_free(const struct rte_memzone *mz); const struct rte_memzone *rte_memzone_lookup(const char *name); /** - * Dump all reserved memzones to the console. + * Dump all reserved memzones to a file. * * @param f * A pointer to a file for output diff --git a/dpdk/lib/librte_eal/common/include/rte_pci.h b/dpdk/lib/librte_eal/common/include/rte_pci.h deleted file mode 100644 index fa749626..00000000 --- a/dpdk/lib/librte_eal/common/include/rte_pci.h +++ /dev/null @@ -1,587 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -/* BSD LICENSE - * - * Copyright 2013-2014 6WIND S.A. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of 6WIND S.A. nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef _RTE_PCI_H_ -#define _RTE_PCI_H_ - -/** - * @file - * - * RTE PCI Interface - */ - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include -#include -#include -#include -#include - -#include - -TAILQ_HEAD(pci_device_list, rte_pci_device); /**< PCI devices in D-linked Q. */ -TAILQ_HEAD(pci_driver_list, rte_pci_driver); /**< PCI drivers in D-linked Q. */ - -extern struct pci_driver_list pci_driver_list; /**< Global list of PCI drivers. */ -extern struct pci_device_list pci_device_list; /**< Global list of PCI devices. */ - -/** Pathname of PCI devices directory. */ -const char *pci_get_sysfs_path(void); - -/** Formatting string for PCI device identifier: Ex: 0000:00:01.0 */ -#define PCI_PRI_FMT "%.4" PRIx16 ":%.2" PRIx8 ":%.2" PRIx8 ".%" PRIx8 - -/** Short formatting string, without domain, for PCI device: Ex: 00:01.0 */ -#define PCI_SHORT_PRI_FMT "%.2" PRIx8 ":%.2" PRIx8 ".%" PRIx8 - -/** Nb. of values in PCI device identifier format string. */ -#define PCI_FMT_NVAL 4 - -/** Nb. of values in PCI resource format. */ -#define PCI_RESOURCE_FMT_NVAL 3 - -/** - * A structure describing a PCI resource. - */ -struct rte_pci_resource { - uint64_t phys_addr; /**< Physical address, 0 if no resource. */ - uint64_t len; /**< Length of the resource. */ - void *addr; /**< Virtual address, NULL when not mapped. */ -}; - -/** Maximum number of PCI resources. */ -#define PCI_MAX_RESOURCE 6 - -/** - * A structure describing an ID for a PCI driver. Each driver provides a - * table of these IDs for each device that it supports. - */ -struct rte_pci_id { - uint32_t class_id; /**< Class ID (class, subclass, pi) or RTE_CLASS_ANY_ID. */ - uint16_t vendor_id; /**< Vendor ID or PCI_ANY_ID. */ - uint16_t device_id; /**< Device ID or PCI_ANY_ID. */ - uint16_t subsystem_vendor_id; /**< Subsystem vendor ID or PCI_ANY_ID. */ - uint16_t subsystem_device_id; /**< Subsystem device ID or PCI_ANY_ID. */ -}; - -/** - * A structure describing the location of a PCI device. - */ -struct rte_pci_addr { - uint16_t domain; /**< Device domain */ - uint8_t bus; /**< Device bus */ - uint8_t devid; /**< Device ID */ - uint8_t function; /**< Device function. */ -}; - -struct rte_devargs; - -enum rte_kernel_driver { - RTE_KDRV_UNKNOWN = 0, - RTE_KDRV_IGB_UIO, - RTE_KDRV_VFIO, - RTE_KDRV_UIO_GENERIC, - RTE_KDRV_NIC_UIO, - RTE_KDRV_NONE, -}; - -/** - * A structure describing a PCI device. - */ -struct rte_pci_device { - TAILQ_ENTRY(rte_pci_device) next; /**< Next probed PCI device. */ - struct rte_pci_addr addr; /**< PCI location. */ - struct rte_pci_id id; /**< PCI ID. */ - struct rte_pci_resource mem_resource[PCI_MAX_RESOURCE]; /**< PCI Memory Resource */ - struct rte_intr_handle intr_handle; /**< Interrupt handle */ - struct rte_pci_driver *driver; /**< Associated driver */ - uint16_t max_vfs; /**< sriov enable if not zero */ - int numa_node; /**< NUMA node connection */ - struct rte_devargs *devargs; /**< Device user arguments */ - enum rte_kernel_driver kdrv; /**< Kernel driver passthrough */ -}; - -/** Any PCI device identifier (vendor, device, ...) */ -#define PCI_ANY_ID (0xffff) -#define RTE_CLASS_ANY_ID (0xffffff) - -#ifdef __cplusplus -/** C++ macro used to help building up tables of device IDs */ -#define RTE_PCI_DEVICE(vend, dev) \ - RTE_CLASS_ANY_ID, \ - (vend), \ - (dev), \ - PCI_ANY_ID, \ - PCI_ANY_ID -#else -/** Macro used to help building up tables of device IDs */ -#define RTE_PCI_DEVICE(vend, dev) \ - .class_id = RTE_CLASS_ANY_ID, \ - .vendor_id = (vend), \ - .device_id = (dev), \ - .subsystem_vendor_id = PCI_ANY_ID, \ - .subsystem_device_id = PCI_ANY_ID -#endif - -struct rte_pci_driver; - -/** - * Initialisation function for the driver called during PCI probing. - */ -typedef int (pci_devinit_t)(struct rte_pci_driver *, struct rte_pci_device *); - -/** - * Uninitialisation function for the driver called during hotplugging. - */ -typedef int (pci_devuninit_t)(struct rte_pci_device *); - -/** - * A structure describing a PCI driver. - */ -struct rte_pci_driver { - TAILQ_ENTRY(rte_pci_driver) next; /**< Next in list. */ - const char *name; /**< Driver name. */ - pci_devinit_t *devinit; /**< Device init. function. */ - pci_devuninit_t *devuninit; /**< Device uninit function. */ - const struct rte_pci_id *id_table; /**< ID table, NULL terminated. */ - uint32_t drv_flags; /**< Flags contolling handling of device. */ -}; - -/** Device needs PCI BAR mapping (done with either IGB_UIO or VFIO) */ -#define RTE_PCI_DRV_NEED_MAPPING 0x0001 -/** Device needs to be unbound even if no module is provided */ -#define RTE_PCI_DRV_FORCE_UNBIND 0x0004 -/** Device driver supports link state interrupt */ -#define RTE_PCI_DRV_INTR_LSC 0x0008 -/** Device driver supports detaching capability */ -#define RTE_PCI_DRV_DETACHABLE 0x0010 - -/** - * A structure describing a PCI mapping. - */ -struct pci_map { - void *addr; - char *path; - uint64_t offset; - uint64_t size; - uint64_t phaddr; -}; - -/** - * A structure describing a mapped PCI resource. - * For multi-process we need to reproduce all PCI mappings in secondary - * processes, so save them in a tailq. - */ -struct mapped_pci_resource { - TAILQ_ENTRY(mapped_pci_resource) next; - - struct rte_pci_addr pci_addr; - char path[PATH_MAX]; - int nb_maps; - struct pci_map maps[PCI_MAX_RESOURCE]; -}; - -/** mapped pci device list */ -TAILQ_HEAD(mapped_pci_res_list, mapped_pci_resource); - -/**< Internal use only - Macro used by pci addr parsing functions **/ -#define GET_PCIADDR_FIELD(in, fd, lim, dlm) \ -do { \ - unsigned long val; \ - char *end; \ - errno = 0; \ - val = strtoul((in), &end, 16); \ - if (errno != 0 || end[0] != (dlm) || val > (lim)) \ - return -EINVAL; \ - (fd) = (typeof (fd))val; \ - (in) = end + 1; \ -} while(0) - -/** - * Utility function to produce a PCI Bus-Device-Function value - * given a string representation. Assumes that the BDF is provided without - * a domain prefix (i.e. domain returned is always 0) - * - * @param input - * The input string to be parsed. Should have the format XX:XX.X - * @param dev_addr - * The PCI Bus-Device-Function address to be returned. Domain will always be - * returned as 0 - * @return - * 0 on success, negative on error. - */ -static inline int -eal_parse_pci_BDF(const char *input, struct rte_pci_addr *dev_addr) -{ - dev_addr->domain = 0; - GET_PCIADDR_FIELD(input, dev_addr->bus, UINT8_MAX, ':'); - GET_PCIADDR_FIELD(input, dev_addr->devid, UINT8_MAX, '.'); - GET_PCIADDR_FIELD(input, dev_addr->function, UINT8_MAX, 0); - return 0; -} - -/** - * Utility function to produce a PCI Bus-Device-Function value - * given a string representation. Assumes that the BDF is provided including - * a domain prefix. - * - * @param input - * The input string to be parsed. Should have the format XXXX:XX:XX.X - * @param dev_addr - * The PCI Bus-Device-Function address to be returned - * @return - * 0 on success, negative on error. - */ -static inline int -eal_parse_pci_DomBDF(const char *input, struct rte_pci_addr *dev_addr) -{ - GET_PCIADDR_FIELD(input, dev_addr->domain, UINT16_MAX, ':'); - GET_PCIADDR_FIELD(input, dev_addr->bus, UINT8_MAX, ':'); - GET_PCIADDR_FIELD(input, dev_addr->devid, UINT8_MAX, '.'); - GET_PCIADDR_FIELD(input, dev_addr->function, UINT8_MAX, 0); - return 0; -} -#undef GET_PCIADDR_FIELD - -/* Compare two PCI device addresses. */ -/** - * Utility function to compare two PCI device addresses. - * - * @param addr - * The PCI Bus-Device-Function address to compare - * @param addr2 - * The PCI Bus-Device-Function address to compare - * @return - * 0 on equal PCI address. - * Positive on addr is greater than addr2. - * Negative on addr is less than addr2, or error. - */ -static inline int -rte_eal_compare_pci_addr(const struct rte_pci_addr *addr, - const struct rte_pci_addr *addr2) -{ - uint64_t dev_addr, dev_addr2; - - if ((addr == NULL) || (addr2 == NULL)) - return -1; - - dev_addr = (addr->domain << 24) | (addr->bus << 16) | - (addr->devid << 8) | addr->function; - dev_addr2 = (addr2->domain << 24) | (addr2->bus << 16) | - (addr2->devid << 8) | addr2->function; - - if (dev_addr > dev_addr2) - return 1; - else if (dev_addr < dev_addr2) - return -1; - else - return 0; -} - -/** - * Scan the content of the PCI bus, and the devices in the devices - * list - * - * @return - * 0 on success, negative on error - */ -int rte_eal_pci_scan(void); - -/** - * Probe the PCI bus for registered drivers. - * - * Scan the content of the PCI bus, and call the probe() function for - * all registered drivers that have a matching entry in its id_table - * for discovered devices. - * - * @return - * - 0 on success. - * - Negative on error. - */ -int rte_eal_pci_probe(void); - -/** - * Map the PCI device resources in user space virtual memory address - * - * Note that driver should not call this function when flag - * RTE_PCI_DRV_NEED_MAPPING is set, as EAL will do that for - * you when it's on. - * - * @param dev - * A pointer to a rte_pci_device structure describing the device - * to use - * - * @return - * 0 on success, negative on error and positive if no driver - * is found for the device. - */ -int rte_eal_pci_map_device(struct rte_pci_device *dev); - -/** - * Unmap this device - * - * @param dev - * A pointer to a rte_pci_device structure describing the device - * to use - */ -void rte_eal_pci_unmap_device(struct rte_pci_device *dev); - -/** - * @internal - * Map a particular resource from a file. - * - * @param requested_addr - * The starting address for the new mapping range. - * @param fd - * The file descriptor. - * @param offset - * The offset for the mapping range. - * @param size - * The size for the mapping range. - * @param additional_flags - * The additional flags for the mapping range. - * @return - * - On success, the function returns a pointer to the mapped area. - * - On error, the value MAP_FAILED is returned. - */ -void *pci_map_resource(void *requested_addr, int fd, off_t offset, - size_t size, int additional_flags); - -/** - * @internal - * Unmap a particular resource. - * - * @param requested_addr - * The address for the unmapping range. - * @param size - * The size for the unmapping range. - */ -void pci_unmap_resource(void *requested_addr, size_t size); - -/** - * Probe the single PCI device. - * - * Scan the content of the PCI bus, and find the pci device specified by pci - * address, then call the probe() function for registered driver that has a - * matching entry in its id_table for discovered device. - * - * @param addr - * The PCI Bus-Device-Function address to probe. - * @return - * - 0 on success. - * - Negative on error. - */ -int rte_eal_pci_probe_one(const struct rte_pci_addr *addr); - -/** - * Close the single PCI device. - * - * Scan the content of the PCI bus, and find the pci device specified by pci - * address, then call the devuninit() function for registered driver that has a - * matching entry in its id_table for discovered device. - * - * @param addr - * The PCI Bus-Device-Function address to close. - * @return - * - 0 on success. - * - Negative on error. - */ -int rte_eal_pci_detach(const struct rte_pci_addr *addr); - -/** - * Dump the content of the PCI bus. - * - * @param f - * A pointer to a file for output - */ -void rte_eal_pci_dump(FILE *f); - -/** - * Register a PCI driver. - * - * @param driver - * A pointer to a rte_pci_driver structure describing the driver - * to be registered. - */ -void rte_eal_pci_register(struct rte_pci_driver *driver); - -/** - * Unregister a PCI driver. - * - * @param driver - * A pointer to a rte_pci_driver structure describing the driver - * to be unregistered. - */ -void rte_eal_pci_unregister(struct rte_pci_driver *driver); - -/** - * Read PCI config space. - * - * @param device - * A pointer to a rte_pci_device structure describing the device - * to use - * @param buf - * A data buffer where the bytes should be read into - * @param len - * The length of the data buffer. - * @param offset - * The offset into PCI config space - */ -int rte_eal_pci_read_config(const struct rte_pci_device *device, - void *buf, size_t len, off_t offset); - -/** - * Write PCI config space. - * - * @param device - * A pointer to a rte_pci_device structure describing the device - * to use - * @param buf - * A data buffer containing the bytes should be written - * @param len - * The length of the data buffer. - * @param offset - * The offset into PCI config space - */ -int rte_eal_pci_write_config(const struct rte_pci_device *device, - const void *buf, size_t len, off_t offset); - -/** - * A structure used to access io resources for a pci device. - * rte_pci_ioport is arch, os, driver specific, and should not be used outside - * of pci ioport api. - */ -struct rte_pci_ioport { - struct rte_pci_device *dev; - uint64_t base; - uint64_t len; /* only filled for memory mapped ports */ -}; - -/** - * Initialize a rte_pci_ioport object for a pci device io resource. - * - * This object is then used to gain access to those io resources (see below). - * - * @param dev - * A pointer to a rte_pci_device structure describing the device - * to use. - * @param bar - * Index of the io pci resource we want to access. - * @param p - * The rte_pci_ioport object to be initialized. - * @return - * 0 on success, negative on error. - */ -int rte_eal_pci_ioport_map(struct rte_pci_device *dev, int bar, - struct rte_pci_ioport *p); - -/** - * Release any resources used in a rte_pci_ioport object. - * - * @param p - * The rte_pci_ioport object to be uninitialized. - * @return - * 0 on success, negative on error. - */ -int rte_eal_pci_ioport_unmap(struct rte_pci_ioport *p); - -/** - * Read from a io pci resource. - * - * @param p - * The rte_pci_ioport object from which we want to read. - * @param data - * A data buffer where the bytes should be read into - * @param len - * The length of the data buffer. - * @param offset - * The offset into the pci io resource. - */ -void rte_eal_pci_ioport_read(struct rte_pci_ioport *p, - void *data, size_t len, off_t offset); - -/** - * Write to a io pci resource. - * - * @param p - * The rte_pci_ioport object to which we want to write. - * @param data - * A data buffer where the bytes should be read into - * @param len - * The length of the data buffer. - * @param offset - * The offset into the pci io resource. - */ -void rte_eal_pci_ioport_write(struct rte_pci_ioport *p, - const void *data, size_t len, off_t offset); - -#ifdef __cplusplus -} -#endif - -#endif /* _RTE_PCI_H_ */ diff --git a/dpdk/lib/librte_eal/common/include/rte_pci_dev_ids.h b/dpdk/lib/librte_eal/common/include/rte_pci_dev_ids.h deleted file mode 100644 index 6ec8ae8c..00000000 --- a/dpdk/lib/librte_eal/common/include/rte_pci_dev_ids.h +++ /dev/null @@ -1,326 +0,0 @@ -/*- - * This file is provided under a dual BSD/GPLv2 license. When using or - * redistributing this file, you may do so under either license. - * - * GPL LICENSE SUMMARY - * - * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of version 2 of the GNU General Public License as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. - * The full GNU General Public License is included in this distribution - * in the file called LICENSE.GPL. - * - * Contact Information: - * Intel Corporation - * - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -#ifndef RTE_PCI_DEV_ID_DECL_IGB -#define RTE_PCI_DEV_ID_DECL_IGB(vend, dev) -#endif - -#ifndef RTE_PCI_DEV_ID_DECL_IGBVF -#define RTE_PCI_DEV_ID_DECL_IGBVF(vend, dev) -#endif - -#ifndef RTE_PCI_DEV_ID_DECL_IXGBE -#define RTE_PCI_DEV_ID_DECL_IXGBE(vend, dev) -#endif - -#ifndef RTE_PCI_DEV_ID_DECL_IXGBEVF -#define RTE_PCI_DEV_ID_DECL_IXGBEVF(vend, dev) -#endif - -#ifndef PCI_VENDOR_ID_INTEL -/** Vendor ID used by Intel devices */ -#define PCI_VENDOR_ID_INTEL 0x8086 -#endif - -/******************** Physical IGB devices from e1000_hw.h ********************/ - -#define E1000_DEV_ID_82576 0x10C9 -#define E1000_DEV_ID_82576_FIBER 0x10E6 -#define E1000_DEV_ID_82576_SERDES 0x10E7 -#define E1000_DEV_ID_82576_QUAD_COPPER 0x10E8 -#define E1000_DEV_ID_82576_QUAD_COPPER_ET2 0x1526 -#define E1000_DEV_ID_82576_NS 0x150A -#define E1000_DEV_ID_82576_NS_SERDES 0x1518 -#define E1000_DEV_ID_82576_SERDES_QUAD 0x150D -#define E1000_DEV_ID_82575EB_COPPER 0x10A7 -#define E1000_DEV_ID_82575EB_FIBER_SERDES 0x10A9 -#define E1000_DEV_ID_82575GB_QUAD_COPPER 0x10D6 -#define E1000_DEV_ID_82580_COPPER 0x150E -#define E1000_DEV_ID_82580_FIBER 0x150F -#define E1000_DEV_ID_82580_SERDES 0x1510 -#define E1000_DEV_ID_82580_SGMII 0x1511 -#define E1000_DEV_ID_82580_COPPER_DUAL 0x1516 -#define E1000_DEV_ID_82580_QUAD_FIBER 0x1527 -#define E1000_DEV_ID_I350_COPPER 0x1521 -#define E1000_DEV_ID_I350_FIBER 0x1522 -#define E1000_DEV_ID_I350_SERDES 0x1523 -#define E1000_DEV_ID_I350_SGMII 0x1524 -#define E1000_DEV_ID_I350_DA4 0x1546 -#define E1000_DEV_ID_I210_COPPER 0x1533 -#define E1000_DEV_ID_I210_COPPER_OEM1 0x1534 -#define E1000_DEV_ID_I210_COPPER_IT 0x1535 -#define E1000_DEV_ID_I210_FIBER 0x1536 -#define E1000_DEV_ID_I210_SERDES 0x1537 -#define E1000_DEV_ID_I210_SGMII 0x1538 -#define E1000_DEV_ID_I210_COPPER_FLASHLESS 0x157B -#define E1000_DEV_ID_I210_SERDES_FLASHLESS 0x157C -#define E1000_DEV_ID_I211_COPPER 0x1539 -#define E1000_DEV_ID_I354_BACKPLANE_1GBPS 0x1F40 -#define E1000_DEV_ID_I354_SGMII 0x1F41 -#define E1000_DEV_ID_I354_BACKPLANE_2_5GBPS 0x1F45 -#define E1000_DEV_ID_DH89XXCC_SGMII 0x0438 -#define E1000_DEV_ID_DH89XXCC_SERDES 0x043A -#define E1000_DEV_ID_DH89XXCC_BACKPLANE 0x043C -#define E1000_DEV_ID_DH89XXCC_SFP 0x0440 - -RTE_PCI_DEV_ID_DECL_IGB(PCI_VENDOR_ID_INTEL, E1000_DEV_ID_82576) -RTE_PCI_DEV_ID_DECL_IGB(PCI_VENDOR_ID_INTEL, E1000_DEV_ID_82576_FIBER) -RTE_PCI_DEV_ID_DECL_IGB(PCI_VENDOR_ID_INTEL, E1000_DEV_ID_82576_SERDES) -RTE_PCI_DEV_ID_DECL_IGB(PCI_VENDOR_ID_INTEL, E1000_DEV_ID_82576_QUAD_COPPER) -RTE_PCI_DEV_ID_DECL_IGB(PCI_VENDOR_ID_INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2) -RTE_PCI_DEV_ID_DECL_IGB(PCI_VENDOR_ID_INTEL, E1000_DEV_ID_82576_NS) -RTE_PCI_DEV_ID_DECL_IGB(PCI_VENDOR_ID_INTEL, E1000_DEV_ID_82576_NS_SERDES) -RTE_PCI_DEV_ID_DECL_IGB(PCI_VENDOR_ID_INTEL, E1000_DEV_ID_82576_SERDES_QUAD) - -RTE_PCI_DEV_ID_DECL_IGB(PCI_VENDOR_ID_INTEL, E1000_DEV_ID_82575EB_COPPER) -RTE_PCI_DEV_ID_DECL_IGB(PCI_VENDOR_ID_INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES) -RTE_PCI_DEV_ID_DECL_IGB(PCI_VENDOR_ID_INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER) - -RTE_PCI_DEV_ID_DECL_IGB(PCI_VENDOR_ID_INTEL, E1000_DEV_ID_82580_COPPER) -RTE_PCI_DEV_ID_DECL_IGB(PCI_VENDOR_ID_INTEL, E1000_DEV_ID_82580_FIBER) -RTE_PCI_DEV_ID_DECL_IGB(PCI_VENDOR_ID_INTEL, E1000_DEV_ID_82580_SERDES) -RTE_PCI_DEV_ID_DECL_IGB(PCI_VENDOR_ID_INTEL, E1000_DEV_ID_82580_SGMII) -RTE_PCI_DEV_ID_DECL_IGB(PCI_VENDOR_ID_INTEL, E1000_DEV_ID_82580_COPPER_DUAL) -RTE_PCI_DEV_ID_DECL_IGB(PCI_VENDOR_ID_INTEL, E1000_DEV_ID_82580_QUAD_FIBER) - -RTE_PCI_DEV_ID_DECL_IGB(PCI_VENDOR_ID_INTEL, E1000_DEV_ID_I350_COPPER) -RTE_PCI_DEV_ID_DECL_IGB(PCI_VENDOR_ID_INTEL, E1000_DEV_ID_I350_FIBER) -RTE_PCI_DEV_ID_DECL_IGB(PCI_VENDOR_ID_INTEL, E1000_DEV_ID_I350_SERDES) -RTE_PCI_DEV_ID_DECL_IGB(PCI_VENDOR_ID_INTEL, E1000_DEV_ID_I350_SGMII) -RTE_PCI_DEV_ID_DECL_IGB(PCI_VENDOR_ID_INTEL, E1000_DEV_ID_I350_DA4) -RTE_PCI_DEV_ID_DECL_IGB(PCI_VENDOR_ID_INTEL, E1000_DEV_ID_I210_COPPER) -RTE_PCI_DEV_ID_DECL_IGB(PCI_VENDOR_ID_INTEL, E1000_DEV_ID_I210_COPPER_OEM1) -RTE_PCI_DEV_ID_DECL_IGB(PCI_VENDOR_ID_INTEL, E1000_DEV_ID_I210_COPPER_IT) -RTE_PCI_DEV_ID_DECL_IGB(PCI_VENDOR_ID_INTEL, E1000_DEV_ID_I210_FIBER) -RTE_PCI_DEV_ID_DECL_IGB(PCI_VENDOR_ID_INTEL, E1000_DEV_ID_I210_SERDES) -RTE_PCI_DEV_ID_DECL_IGB(PCI_VENDOR_ID_INTEL, E1000_DEV_ID_I210_SGMII) -RTE_PCI_DEV_ID_DECL_IGB(PCI_VENDOR_ID_INTEL, E1000_DEV_ID_I211_COPPER) -RTE_PCI_DEV_ID_DECL_IGB(PCI_VENDOR_ID_INTEL, E1000_DEV_ID_I354_BACKPLANE_1GBPS) -RTE_PCI_DEV_ID_DECL_IGB(PCI_VENDOR_ID_INTEL, E1000_DEV_ID_I354_SGMII) -RTE_PCI_DEV_ID_DECL_IGB(PCI_VENDOR_ID_INTEL, E1000_DEV_ID_I354_BACKPLANE_2_5GBPS) -RTE_PCI_DEV_ID_DECL_IGB(PCI_VENDOR_ID_INTEL, E1000_DEV_ID_DH89XXCC_SGMII) -RTE_PCI_DEV_ID_DECL_IGB(PCI_VENDOR_ID_INTEL, E1000_DEV_ID_DH89XXCC_SERDES) -RTE_PCI_DEV_ID_DECL_IGB(PCI_VENDOR_ID_INTEL, E1000_DEV_ID_DH89XXCC_BACKPLANE) -RTE_PCI_DEV_ID_DECL_IGB(PCI_VENDOR_ID_INTEL, E1000_DEV_ID_DH89XXCC_SFP) - -/****************** Physical IXGBE devices from ixgbe_type.h ******************/ - -#define IXGBE_DEV_ID_82598 0x10B6 -#define IXGBE_DEV_ID_82598_BX 0x1508 -#define IXGBE_DEV_ID_82598AF_DUAL_PORT 0x10C6 -#define IXGBE_DEV_ID_82598AF_SINGLE_PORT 0x10C7 -#define IXGBE_DEV_ID_82598AT 0x10C8 -#define IXGBE_DEV_ID_82598AT2 0x150B -#define IXGBE_DEV_ID_82598EB_SFP_LOM 0x10DB -#define IXGBE_DEV_ID_82598EB_CX4 0x10DD -#define IXGBE_DEV_ID_82598_CX4_DUAL_PORT 0x10EC -#define IXGBE_DEV_ID_82598_DA_DUAL_PORT 0x10F1 -#define IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM 0x10E1 -#define IXGBE_DEV_ID_82598EB_XF_LR 0x10F4 -#define IXGBE_DEV_ID_82599_KX4 0x10F7 -#define IXGBE_DEV_ID_82599_KX4_MEZZ 0x1514 -#define IXGBE_DEV_ID_82599_KR 0x1517 -#define IXGBE_DEV_ID_82599_COMBO_BACKPLANE 0x10F8 -#define IXGBE_SUBDEV_ID_82599_KX4_KR_MEZZ 0x000C -#define IXGBE_DEV_ID_82599_CX4 0x10F9 -#define IXGBE_DEV_ID_82599_SFP 0x10FB -#define IXGBE_SUBDEV_ID_82599_SFP 0x11A9 -#define IXGBE_SUBDEV_ID_82599_RNDC 0x1F72 -#define IXGBE_SUBDEV_ID_82599_560FLR 0x17D0 -#define IXGBE_SUBDEV_ID_82599_ECNA_DP 0x0470 -#define IXGBE_DEV_ID_82599_BACKPLANE_FCOE 0x152A -#define IXGBE_DEV_ID_82599_SFP_FCOE 0x1529 -#define IXGBE_DEV_ID_82599_SFP_EM 0x1507 -#define IXGBE_DEV_ID_82599_SFP_SF2 0x154D -#define IXGBE_DEV_ID_82599_SFP_SF_QP 0x154A -#define IXGBE_DEV_ID_82599_QSFP_SF_QP 0x1558 -#define IXGBE_DEV_ID_82599EN_SFP 0x1557 -#define IXGBE_DEV_ID_82599_XAUI_LOM 0x10FC -#define IXGBE_DEV_ID_82599_T3_LOM 0x151C -#define IXGBE_DEV_ID_82599_LS 0x154F -#define IXGBE_DEV_ID_X540T 0x1528 -#define IXGBE_DEV_ID_X540T1 0x1560 -#define IXGBE_DEV_ID_X550EM_X_SFP 0x15AC -#define IXGBE_DEV_ID_X550EM_X_10G_T 0x15AD -#define IXGBE_DEV_ID_X550EM_X_1G_T 0x15AE -#define IXGBE_DEV_ID_X550T 0x1563 -#define IXGBE_DEV_ID_X550T1 0x15D1 -#define IXGBE_DEV_ID_X550EM_A_KR 0x15C2 -#define IXGBE_DEV_ID_X550EM_A_KR_L 0x15C3 -#define IXGBE_DEV_ID_X550EM_A_SFP_N 0x15C4 -#define IXGBE_DEV_ID_X550EM_A_SGMII 0x15C6 -#define IXGBE_DEV_ID_X550EM_A_SGMII_L 0x15C7 -#define IXGBE_DEV_ID_X550EM_A_10G_T 0x15C8 -#define IXGBE_DEV_ID_X550EM_A_QSFP 0x15CA -#define IXGBE_DEV_ID_X550EM_A_QSFP_N 0x15CC -#define IXGBE_DEV_ID_X550EM_A_SFP 0x15CE -#define IXGBE_DEV_ID_X550EM_A_1G_T 0x15E4 -#define IXGBE_DEV_ID_X550EM_A_1G_T_L 0x15E5 -#define IXGBE_DEV_ID_X550EM_X_KX4 0x15AA -#define IXGBE_DEV_ID_X550EM_X_KR 0x15AB - -#ifdef RTE_NIC_BYPASS -#define IXGBE_DEV_ID_82599_BYPASS 0x155D -#endif - -RTE_PCI_DEV_ID_DECL_IXGBE(PCI_VENDOR_ID_INTEL, IXGBE_DEV_ID_82598) -RTE_PCI_DEV_ID_DECL_IXGBE(PCI_VENDOR_ID_INTEL, IXGBE_DEV_ID_82598_BX) -RTE_PCI_DEV_ID_DECL_IXGBE(PCI_VENDOR_ID_INTEL, IXGBE_DEV_ID_82598AF_DUAL_PORT) -RTE_PCI_DEV_ID_DECL_IXGBE(PCI_VENDOR_ID_INTEL, \ - IXGBE_DEV_ID_82598AF_SINGLE_PORT) -RTE_PCI_DEV_ID_DECL_IXGBE(PCI_VENDOR_ID_INTEL, IXGBE_DEV_ID_82598AT) -RTE_PCI_DEV_ID_DECL_IXGBE(PCI_VENDOR_ID_INTEL, IXGBE_DEV_ID_82598AT2) -RTE_PCI_DEV_ID_DECL_IXGBE(PCI_VENDOR_ID_INTEL, IXGBE_DEV_ID_82598EB_SFP_LOM) -RTE_PCI_DEV_ID_DECL_IXGBE(PCI_VENDOR_ID_INTEL, IXGBE_DEV_ID_82598EB_CX4) -RTE_PCI_DEV_ID_DECL_IXGBE(PCI_VENDOR_ID_INTEL, IXGBE_DEV_ID_82598_CX4_DUAL_PORT) -RTE_PCI_DEV_ID_DECL_IXGBE(PCI_VENDOR_ID_INTEL, IXGBE_DEV_ID_82598_DA_DUAL_PORT) -RTE_PCI_DEV_ID_DECL_IXGBE(PCI_VENDOR_ID_INTEL, \ - IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM) -RTE_PCI_DEV_ID_DECL_IXGBE(PCI_VENDOR_ID_INTEL, IXGBE_DEV_ID_82598EB_XF_LR) -RTE_PCI_DEV_ID_DECL_IXGBE(PCI_VENDOR_ID_INTEL, IXGBE_DEV_ID_82599_KX4) -RTE_PCI_DEV_ID_DECL_IXGBE(PCI_VENDOR_ID_INTEL, IXGBE_DEV_ID_82599_KX4_MEZZ) -RTE_PCI_DEV_ID_DECL_IXGBE(PCI_VENDOR_ID_INTEL, IXGBE_DEV_ID_82599_KR) -RTE_PCI_DEV_ID_DECL_IXGBE(PCI_VENDOR_ID_INTEL, \ - IXGBE_DEV_ID_82599_COMBO_BACKPLANE) -RTE_PCI_DEV_ID_DECL_IXGBE(PCI_VENDOR_ID_INTEL, \ - IXGBE_SUBDEV_ID_82599_KX4_KR_MEZZ) -RTE_PCI_DEV_ID_DECL_IXGBE(PCI_VENDOR_ID_INTEL, IXGBE_DEV_ID_82599_CX4) -RTE_PCI_DEV_ID_DECL_IXGBE(PCI_VENDOR_ID_INTEL, IXGBE_DEV_ID_82599_SFP) -RTE_PCI_DEV_ID_DECL_IXGBE(PCI_VENDOR_ID_INTEL, IXGBE_SUBDEV_ID_82599_SFP) -RTE_PCI_DEV_ID_DECL_IXGBE(PCI_VENDOR_ID_INTEL, IXGBE_SUBDEV_ID_82599_RNDC) -RTE_PCI_DEV_ID_DECL_IXGBE(PCI_VENDOR_ID_INTEL, IXGBE_SUBDEV_ID_82599_560FLR) -RTE_PCI_DEV_ID_DECL_IXGBE(PCI_VENDOR_ID_INTEL, IXGBE_SUBDEV_ID_82599_ECNA_DP) -RTE_PCI_DEV_ID_DECL_IXGBE(PCI_VENDOR_ID_INTEL, IXGBE_DEV_ID_82599_BACKPLANE_FCOE) -RTE_PCI_DEV_ID_DECL_IXGBE(PCI_VENDOR_ID_INTEL, IXGBE_DEV_ID_82599_SFP_FCOE) -RTE_PCI_DEV_ID_DECL_IXGBE(PCI_VENDOR_ID_INTEL, IXGBE_DEV_ID_82599_SFP_EM) -RTE_PCI_DEV_ID_DECL_IXGBE(PCI_VENDOR_ID_INTEL, IXGBE_DEV_ID_82599_SFP_SF2) -RTE_PCI_DEV_ID_DECL_IXGBE(PCI_VENDOR_ID_INTEL, IXGBE_DEV_ID_82599_SFP_SF_QP) -RTE_PCI_DEV_ID_DECL_IXGBE(PCI_VENDOR_ID_INTEL, IXGBE_DEV_ID_82599_QSFP_SF_QP) -RTE_PCI_DEV_ID_DECL_IXGBE(PCI_VENDOR_ID_INTEL, IXGBE_DEV_ID_82599EN_SFP) -RTE_PCI_DEV_ID_DECL_IXGBE(PCI_VENDOR_ID_INTEL, IXGBE_DEV_ID_82599_XAUI_LOM) -RTE_PCI_DEV_ID_DECL_IXGBE(PCI_VENDOR_ID_INTEL, IXGBE_DEV_ID_82599_T3_LOM) -RTE_PCI_DEV_ID_DECL_IXGBE(PCI_VENDOR_ID_INTEL, IXGBE_DEV_ID_82599_LS) -RTE_PCI_DEV_ID_DECL_IXGBE(PCI_VENDOR_ID_INTEL, IXGBE_DEV_ID_X540T) -RTE_PCI_DEV_ID_DECL_IXGBE(PCI_VENDOR_ID_INTEL, IXGBE_DEV_ID_X540T1) -RTE_PCI_DEV_ID_DECL_IXGBE(PCI_VENDOR_ID_INTEL, IXGBE_DEV_ID_X550EM_X_SFP) -RTE_PCI_DEV_ID_DECL_IXGBE(PCI_VENDOR_ID_INTEL, IXGBE_DEV_ID_X550EM_X_10G_T) -RTE_PCI_DEV_ID_DECL_IXGBE(PCI_VENDOR_ID_INTEL, IXGBE_DEV_ID_X550EM_X_1G_T) -RTE_PCI_DEV_ID_DECL_IXGBE(PCI_VENDOR_ID_INTEL, IXGBE_DEV_ID_X550T) -RTE_PCI_DEV_ID_DECL_IXGBE(PCI_VENDOR_ID_INTEL, IXGBE_DEV_ID_X550T1) -RTE_PCI_DEV_ID_DECL_IXGBE(PCI_VENDOR_ID_INTEL, IXGBE_DEV_ID_X550EM_A_KR) -RTE_PCI_DEV_ID_DECL_IXGBE(PCI_VENDOR_ID_INTEL, IXGBE_DEV_ID_X550EM_A_KR_L) -RTE_PCI_DEV_ID_DECL_IXGBE(PCI_VENDOR_ID_INTEL, IXGBE_DEV_ID_X550EM_A_SFP_N) -RTE_PCI_DEV_ID_DECL_IXGBE(PCI_VENDOR_ID_INTEL, IXGBE_DEV_ID_X550EM_A_SGMII) -RTE_PCI_DEV_ID_DECL_IXGBE(PCI_VENDOR_ID_INTEL, IXGBE_DEV_ID_X550EM_A_SGMII_L) -RTE_PCI_DEV_ID_DECL_IXGBE(PCI_VENDOR_ID_INTEL, IXGBE_DEV_ID_X550EM_A_10G_T) -RTE_PCI_DEV_ID_DECL_IXGBE(PCI_VENDOR_ID_INTEL, IXGBE_DEV_ID_X550EM_A_QSFP) -RTE_PCI_DEV_ID_DECL_IXGBE(PCI_VENDOR_ID_INTEL, IXGBE_DEV_ID_X550EM_A_QSFP_N) -RTE_PCI_DEV_ID_DECL_IXGBE(PCI_VENDOR_ID_INTEL, IXGBE_DEV_ID_X550EM_A_SFP) -RTE_PCI_DEV_ID_DECL_IXGBE(PCI_VENDOR_ID_INTEL, IXGBE_DEV_ID_X550EM_A_1G_T) -RTE_PCI_DEV_ID_DECL_IXGBE(PCI_VENDOR_ID_INTEL, IXGBE_DEV_ID_X550EM_A_1G_T_L) -RTE_PCI_DEV_ID_DECL_IXGBE(PCI_VENDOR_ID_INTEL, IXGBE_DEV_ID_X550EM_X_KX4) -RTE_PCI_DEV_ID_DECL_IXGBE(PCI_VENDOR_ID_INTEL, IXGBE_DEV_ID_X550EM_X_KR) - -#ifdef RTE_NIC_BYPASS -RTE_PCI_DEV_ID_DECL_IXGBE(PCI_VENDOR_ID_INTEL, IXGBE_DEV_ID_82599_BYPASS) -#endif - -/****************** Virtual IGB devices from e1000_hw.h ******************/ - -#define E1000_DEV_ID_82576_VF 0x10CA -#define E1000_DEV_ID_82576_VF_HV 0x152D -#define E1000_DEV_ID_I350_VF 0x1520 -#define E1000_DEV_ID_I350_VF_HV 0x152F - -RTE_PCI_DEV_ID_DECL_IGBVF(PCI_VENDOR_ID_INTEL, E1000_DEV_ID_82576_VF) -RTE_PCI_DEV_ID_DECL_IGBVF(PCI_VENDOR_ID_INTEL, E1000_DEV_ID_82576_VF_HV) -RTE_PCI_DEV_ID_DECL_IGBVF(PCI_VENDOR_ID_INTEL, E1000_DEV_ID_I350_VF) -RTE_PCI_DEV_ID_DECL_IGBVF(PCI_VENDOR_ID_INTEL, E1000_DEV_ID_I350_VF_HV) - -/****************** Virtual IXGBE devices from ixgbe_type.h ******************/ - -#define IXGBE_DEV_ID_82599_VF 0x10ED -#define IXGBE_DEV_ID_82599_VF_HV 0x152E -#define IXGBE_DEV_ID_X540_VF 0x1515 -#define IXGBE_DEV_ID_X540_VF_HV 0x1530 -#define IXGBE_DEV_ID_X550_VF_HV 0x1564 -#define IXGBE_DEV_ID_X550_VF 0x1565 -#define IXGBE_DEV_ID_X550EM_A_VF 0x15C5 -#define IXGBE_DEV_ID_X550EM_A_VF_HV 0x15B4 -#define IXGBE_DEV_ID_X550EM_X_VF 0x15A8 -#define IXGBE_DEV_ID_X550EM_X_VF_HV 0x15A9 - -RTE_PCI_DEV_ID_DECL_IXGBEVF(PCI_VENDOR_ID_INTEL, IXGBE_DEV_ID_82599_VF) -RTE_PCI_DEV_ID_DECL_IXGBEVF(PCI_VENDOR_ID_INTEL, IXGBE_DEV_ID_82599_VF_HV) -RTE_PCI_DEV_ID_DECL_IXGBEVF(PCI_VENDOR_ID_INTEL, IXGBE_DEV_ID_X540_VF) -RTE_PCI_DEV_ID_DECL_IXGBEVF(PCI_VENDOR_ID_INTEL, IXGBE_DEV_ID_X540_VF_HV) -RTE_PCI_DEV_ID_DECL_IXGBEVF(PCI_VENDOR_ID_INTEL, IXGBE_DEV_ID_X550_VF_HV) -RTE_PCI_DEV_ID_DECL_IXGBEVF(PCI_VENDOR_ID_INTEL, IXGBE_DEV_ID_X550_VF) -RTE_PCI_DEV_ID_DECL_IXGBEVF(PCI_VENDOR_ID_INTEL, IXGBE_DEV_ID_X550EM_A_VF) -RTE_PCI_DEV_ID_DECL_IXGBEVF(PCI_VENDOR_ID_INTEL, IXGBE_DEV_ID_X550EM_A_VF_HV) -RTE_PCI_DEV_ID_DECL_IXGBEVF(PCI_VENDOR_ID_INTEL, IXGBE_DEV_ID_X550EM_X_VF) -RTE_PCI_DEV_ID_DECL_IXGBEVF(PCI_VENDOR_ID_INTEL, IXGBE_DEV_ID_X550EM_X_VF_HV) - -/* - * Undef all RTE_PCI_DEV_ID_DECL_* here. - */ -#undef RTE_PCI_DEV_ID_DECL_IGB -#undef RTE_PCI_DEV_ID_DECL_IGBVF -#undef RTE_PCI_DEV_ID_DECL_IXGBE -#undef RTE_PCI_DEV_ID_DECL_IXGBEVF diff --git a/dpdk/lib/librte_eal/common/include/rte_random.h b/dpdk/lib/librte_eal/common/include/rte_random.h index 24ae8363..aeff1f05 100644 --- a/dpdk/lib/librte_eal/common/include/rte_random.h +++ b/dpdk/lib/librte_eal/common/include/rte_random.h @@ -88,4 +88,4 @@ rte_rand(void) #endif -#endif /* _RTE_PER_LCORE_H_ */ +#endif /* _RTE_RANDOM_H_ */ diff --git a/dpdk/lib/librte_eal/common/include/rte_service.h b/dpdk/lib/librte_eal/common/include/rte_service.h new file mode 100644 index 00000000..80b90fad --- /dev/null +++ b/dpdk/lib/librte_eal/common/include/rte_service.h @@ -0,0 +1,431 @@ +/* + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_SERVICE_H_ +#define _RTE_SERVICE_H_ + +/** + * @file + * + * Service functions + * + * The service functionality provided by this header allows a DPDK component + * to indicate that it requires a function call in order for it to perform + * its processing. + * + * An example usage of this functionality would be a component that registers + * a service to perform a particular packet processing duty: for example the + * eventdev software PMD. At startup the application requests all services + * that have been registered, and the cores in the service-coremask run the + * required services. The EAL removes these number of cores from the available + * runtime cores, and dedicates them to performing service-core workloads. The + * application has access to the remaining lcores as normal. + */ + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +#include +#include + +#define RTE_SERVICE_NAME_MAX 32 + +/* Capabilities of a service. + * + * Use the *rte_service_probe_capability* function to check if a service is + * capable of a specific capability. + */ +/** When set, the service is capable of having multiple threads run it at the + * same time. + */ +#define RTE_SERVICE_CAP_MT_SAFE (1 << 0) + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Return the number of services registered. + * + * The number of services registered can be passed to *rte_service_get_by_id*, + * enabling the application to retrieve the specification of each service. + * + * @return The number of services registered. + */ +uint32_t rte_service_get_count(void); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Return the id of a service by name. + * + * This function provides the id of the service using the service name as + * lookup key. The service id is to be passed to other functions in the + * rte_service_* API. + * + * Example usage: + * @code + * uint32_t service_id; + * int32_t ret = rte_service_get_by_name("service_X", &service_id); + * if (ret) { + * // handle error + * } + * @endcode + * + * @param name The name of the service to retrieve + * @param[out] service_id A pointer to a uint32_t, to be filled in with the id. + * @retval 0 Success. The service id is provided in *service_id*. + * @retval -EINVAL Null *service_id* pointer provided + * @retval -ENODEV No such service registered + */ +int32_t rte_service_get_by_name(const char *name, uint32_t *service_id); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Return the name of the service. + * + * @return A pointer to the name of the service. The returned pointer remains + * in ownership of the service, and the application must not free it. + */ +const char *rte_service_get_name(uint32_t id); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Check if a service has a specific capability. + * + * This function returns if *service* has implements *capability*. + * See RTE_SERVICE_CAP_* defines for a list of valid capabilities. + * @retval 1 Capability supported by this service instance + * @retval 0 Capability not supported by this service instance + */ +int32_t rte_service_probe_capability(uint32_t id, uint32_t capability); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Map or unmap a lcore to a service. + * + * Each core can be added or removed from running a specific service. This + * function enables or disables *lcore* to run *service_id*. + * + * If multiple cores are enabled on a service, an atomic is used to ensure that + * only one cores runs the service at a time. The exception to this is when + * a service indicates that it is multi-thread safe by setting the capability + * called RTE_SERVICE_CAP_MT_SAFE. With the multi-thread safe capability set, + * the service function can be run on multiple threads at the same time. + * + * @param service_id the service to apply the lcore to + * @param lcore The lcore that will be mapped to service + * @param enable Zero to unmap or disable the core, non-zero to enable + * + * @retval 0 lcore map updated successfully + * @retval -EINVAL An invalid service or lcore was provided. + */ +int32_t rte_service_map_lcore_set(uint32_t service_id, uint32_t lcore, + uint32_t enable); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Retrieve the mapping of an lcore to a service. + * + * @param service_id the service to apply the lcore to + * @param lcore The lcore that will be mapped to service + * + * @retval 1 lcore is mapped to service + * @retval 0 lcore is not mapped to service + * @retval -EINVAL An invalid service or lcore was provided. + */ +int32_t rte_service_map_lcore_get(uint32_t service_id, uint32_t lcore); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Set the runstate of the service. + * + * Each service is either running or stopped. Setting a non-zero runstate + * enables the service to run, while setting runstate zero disables it. + * + * @param id The id of the service + * @param runstate The run state to apply to the service + * + * @retval 0 The service was successfully started + * @retval -EINVAL Invalid service id + */ +int32_t rte_service_runstate_set(uint32_t id, uint32_t runstate); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Get the runstate for the service with *id*. See *rte_service_runstate_set* + * for details of runstates. A service can call this function to ensure that + * the application has indicated that it will receive CPU cycles. Either a + * service-core is mapped (default case), or the application has explicitly + * disabled the check that a service-cores is mapped to the service and takes + * responsibility to run the service manually using the available function + * *rte_service_run_iter_on_app_lcore* to do so. + * + * @retval 1 Service is running + * @retval 0 Service is stopped + * @retval -EINVAL Invalid service id + */ +int32_t rte_service_runstate_get(uint32_t id); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Enable or disable the check for a service-core being mapped to the service. + * An application can disable the check when takes the responsibility to run a + * service itself using *rte_service_run_iter_on_app_lcore*. + * + * @param id The id of the service to set the check on + * @param enable When zero, the check is disabled. Non-zero enables the check. + * + * @retval 0 Success + * @retval -EINVAL Invalid service ID + */ +int32_t rte_service_set_runstate_mapped_check(uint32_t id, int32_t enable); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * This function runs a service callback from a non-service lcore. + * + * This function is designed to enable gradual porting to service cores, and + * to enable unit tests to verify a service behaves as expected. + * + * When called, this function ensures that the service identified by *id* is + * safe to run on this lcore. Multi-thread safe services are invoked even if + * other cores are simultaneously running them as they are multi-thread safe. + * + * Multi-thread unsafe services are handled depending on the variable + * *serialize_multithread_unsafe*: + * - When set, the function will check if a service is already being invoked + * on another lcore, refusing to run it and returning -EBUSY. + * - When zero, the application takes responsibility to ensure that the service + * indicated by *id* is not going to be invoked by another lcore. This setting + * avoids atomic operations, so is likely to be more performant. + * + * @param id The ID of the service to run + * @param serialize_multithread_unsafe This parameter indicates to the service + * cores library if it is required to use atomics to serialize access + * to mult-thread unsafe services. As there is an overhead in using + * atomics, applications can choose to enable or disable this feature + * + * Note that any thread calling this function MUST be a DPDK EAL thread, as + * the *rte_lcore_id* function is used to access internal data structures. + * + * @retval 0 Service was run on the calling thread successfully + * @retval -EBUSY Another lcore is executing the service, and it is not a + * multi-thread safe service, so the service was not run on this lcore + * @retval -ENOEXEC Service is not in a run-able state + * @retval -EINVAL Invalid service id + */ +int32_t rte_service_run_iter_on_app_lcore(uint32_t id, + uint32_t serialize_multithread_unsafe); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Start a service core. + * + * Starting a core makes the core begin polling. Any services assigned to it + * will be run as fast as possible. The application must ensure that the lcore + * is in a launchable state: e.g. call *rte_eal_lcore_wait* on the lcore_id + * before calling this function. + * + * @retval 0 Success + * @retval -EINVAL Failed to start core. The *lcore_id* passed in is not + * currently assigned to be a service core. + */ +int32_t rte_service_lcore_start(uint32_t lcore_id); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Stop a service core. + * + * Stopping a core makes the core become idle, but remains assigned as a + * service core. + * + * @retval 0 Success + * @retval -EINVAL Invalid *lcore_id* provided + * @retval -EALREADY Already stopped core + * @retval -EBUSY Failed to stop core, as it would cause a service to not + * be run, as this is the only core currently running the service. + * The application must stop the service first, and then stop the + * lcore. + */ +int32_t rte_service_lcore_stop(uint32_t lcore_id); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Adds lcore to the list of service cores. + * + * This functions can be used at runtime in order to modify the service core + * mask. + * + * @retval 0 Success + * @retval -EBUSY lcore is busy, and not available for service core duty + * @retval -EALREADY lcore is already added to the service core list + * @retval -EINVAL Invalid lcore provided + */ +int32_t rte_service_lcore_add(uint32_t lcore); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Removes lcore from the list of service cores. + * + * This can fail if the core is not stopped, see *rte_service_core_stop*. + * + * @retval 0 Success + * @retval -EBUSY Lcore is not stopped, stop service core before removing. + * @retval -EINVAL failed to add lcore to service core mask. + */ +int32_t rte_service_lcore_del(uint32_t lcore); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Retrieve the number of service cores currently available. + * + * This function returns the integer count of service cores available. The + * service core count can be used in mapping logic when creating mappings + * from service cores to services. + * + * See *rte_service_lcore_list* for details on retrieving the lcore_id of each + * service core. + * + * @return The number of service cores currently configured. + */ +int32_t rte_service_lcore_count(void); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Resets all service core mappings. This does not remove the service cores + * from duty, just unmaps all services / cores, and stops() the service cores. + * The runstate of services is not modified. + * + * @retval 0 Success + */ +int32_t rte_service_lcore_reset_all(void); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Enable or disable statistics collection for *service*. + * + * This function enables per core, per-service cycle count collection. + * @param id The service to enable statistics gathering on. + * @param enable Zero to disable statistics, non-zero to enable. + * @retval 0 Success + * @retval -EINVAL Invalid service pointer passed + */ +int32_t rte_service_set_stats_enable(uint32_t id, int32_t enable); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Retrieve the list of currently enabled service cores. + * + * This function fills in an application supplied array, with each element + * indicating the lcore_id of a service core. + * + * Adding and removing service cores can be performed using + * *rte_service_lcore_add* and *rte_service_lcore_del*. + * @param [out] array An array of at least *rte_service_lcore_count* items. + * If statically allocating the buffer, use RTE_MAX_LCORE. + * @param [out] n The size of *array*. + * @retval >=0 Number of service cores that have been populated in the array + * @retval -ENOMEM The provided array is not large enough to fill in the + * service core list. No items have been populated, call this function + * with a size of at least *rte_service_core_count* items. + */ +int32_t rte_service_lcore_list(uint32_t array[], uint32_t n); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Get the numer of services running on the supplied lcore. + * + * @param lcore Id of the service core. + * @retval >=0 Number of services registered to this core. + * @retval -EINVAL Invalid lcore provided + * @retval -ENOTSUP The provided lcore is not a service core. + */ +int32_t rte_service_lcore_count_services(uint32_t lcore); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Dumps any information available about the service. When id is UINT32_MAX, + * this function dumps info for all services. + * + * @retval 0 Statistics have been successfully dumped + * @retval -EINVAL Invalid service id provided + */ +int32_t rte_service_dump(FILE *f, uint32_t id); + +#ifdef __cplusplus +} +#endif + + +#endif /* _RTE_SERVICE_H_ */ diff --git a/dpdk/lib/librte_eal/common/include/rte_service_component.h b/dpdk/lib/librte_eal/common/include/rte_service_component.h new file mode 100644 index 00000000..ac965cb4 --- /dev/null +++ b/dpdk/lib/librte_eal/common/include/rte_service_component.h @@ -0,0 +1,170 @@ +/* + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_SERVICE_PRIVATE_H_ +#define _RTE_SERVICE_PRIVATE_H_ + +/* This file specifies the internal service specification. + * Include this file if you are writing a component that requires CPU cycles to + * operate, and you wish to run the component using service cores + */ + +#include + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Signature of callback function to run a service. + */ +typedef int32_t (*rte_service_func)(void *args); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * The specification of a service. + * + * This struct contains metadata about the service itself, the callback + * function to run one iteration of the service, a userdata pointer, flags etc. + */ +struct rte_service_spec { + /** The name of the service. This should be used by the application to + * understand what purpose this service provides. + */ + char name[RTE_SERVICE_NAME_MAX]; + /** The callback to invoke to run one iteration of the service. */ + rte_service_func callback; + /** The userdata pointer provided to the service callback. */ + void *callback_userdata; + /** Flags to indicate the capabilities of this service. See defines in + * the public header file for values of RTE_SERVICE_CAP_* + */ + uint32_t capabilities; + /** NUMA socket ID that this service is affinitized to */ + int socket_id; +}; + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Register a new service. + * + * A service represents a component that the requires CPU time periodically to + * achieve its purpose. + * + * For example the eventdev SW PMD requires CPU cycles to perform its + * scheduling. This can be achieved by registering it as a service, and the + * application can then assign CPU resources to that service. + * + * Note that when a service component registers itself, it is not permitted to + * add or remove service-core threads, or modify lcore-to-service mappings. The + * only API that may be called by the service-component is + * *rte_service_component_runstate_set*, which indicates that the service + * component is ready to be executed. + * + * @param spec The specification of the service to register + * @param[out] service_id A pointer to a uint32_t, which will be filled in + * during registration of the service. It is set to the integers + * service number given to the service. This parameter may be NULL. + * @retval 0 Successfully registered the service. + * -EINVAL Attempted to register an invalid service (eg, no callback + * set) + */ +int32_t rte_service_component_register(const struct rte_service_spec *spec, + uint32_t *service_id); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Unregister a service component. + * + * The service being removed must be stopped before calling this function. + * + * @retval 0 The service was successfully unregistered. + * @retval -EBUSY The service is currently running, stop the service before + * calling unregister. No action has been taken. + */ +int32_t rte_service_component_unregister(uint32_t id); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Private function to allow EAL to initialized default mappings. + * + * This function iterates all the services, and maps then to the available + * cores. Based on the capabilities of the services, they are set to run on the + * available cores in a round-robin manner. + * + * @retval 0 Success + * @retval -ENOTSUP No service lcores in use + * @retval -EINVAL Error while iterating over services + * @retval -ENODEV Error in enabling service lcore on a service + * @retval -ENOEXEC Error when starting services + */ +int32_t rte_service_start_with_defaults(void); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Set the backend runstate of a component. + * + * This function allows services to be registered at startup, but not yet + * enabled to run by default. When the service has been configured (via the + * usual method; eg rte_eventdev_configure, the service can mark itself as + * ready to run. The differentiation between backend runstate and + * service_runstate is that the backend runstate is set by the service + * component while the service runstate is reserved for application usage. + * + * @retval 0 Success + */ +int32_t rte_service_component_runstate_set(uint32_t id, uint32_t runstate); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Initialize the service library. + * + * In order to use the service library, it must be initialized. EAL initializes + * the library at startup. + * + * @retval 0 Success + * @retval -EALREADY Service library is already initialized + */ +int32_t rte_service_init(void); + +#endif /* _RTE_SERVICE_PRIVATE_H_ */ diff --git a/dpdk/lib/librte_eal/common/include/rte_tailq.h b/dpdk/lib/librte_eal/common/include/rte_tailq.h index cc3c0f1d..3aae098a 100644 --- a/dpdk/lib/librte_eal/common/include/rte_tailq.h +++ b/dpdk/lib/librte_eal/common/include/rte_tailq.h @@ -107,7 +107,7 @@ struct rte_tailq_elem { RTE_TAILQ_CAST(rte_eal_tailq_lookup(name), struct_name) /** - * Dump tail queues to the console. + * Dump tail queues to a file. * * @param f * A pointer to a file for output @@ -148,8 +148,8 @@ struct rte_tailq_head *rte_eal_tailq_lookup(const char *name); int rte_eal_tailq_register(struct rte_tailq_elem *t); #define EAL_REGISTER_TAILQ(t) \ -void tailqinitfn_ ##t(void); \ -void __attribute__((constructor, used)) tailqinitfn_ ##t(void) \ +RTE_INIT(tailqinitfn_ ##t); \ +static void tailqinitfn_ ##t(void) \ { \ if (rte_eal_tailq_register(&t) < 0) \ rte_panic("Cannot initialize tailq: %s\n", t.name); \ diff --git a/dpdk/lib/librte_eal/common/include/rte_time.h b/dpdk/lib/librte_eal/common/include/rte_time.h index 4b13b9c1..373c41ac 100644 --- a/dpdk/lib/librte_eal/common/include/rte_time.h +++ b/dpdk/lib/librte_eal/common/include/rte_time.h @@ -31,6 +31,12 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +#ifndef _RTE_TIME_H_ +#define _RTE_TIME_H_ + +#include +#include + #define NSEC_PER_SEC 1000000000L /** @@ -46,7 +52,7 @@ struct rte_timecounter { uint64_t nsec_mask; /** Sub-nanoseconds count. */ uint64_t nsec_frac; - /** Bitmask for two's complement substraction of non-64 bit counters. */ + /** Bitmask for two's complement subtraction of non-64 bit counters. */ uint64_t cc_mask; /** Cycle to nanosecond divisor (power of two). */ uint32_t cc_shift; @@ -120,3 +126,5 @@ rte_ns_to_timespec(uint64_t nsec) return ts; } + +#endif /* _RTE_TIME_H_ */ diff --git a/dpdk/lib/librte_eal/common/include/rte_version.h b/dpdk/lib/librte_eal/common/include/rte_version.h index 8187dc7b..4616a080 100644 --- a/dpdk/lib/librte_eal/common/include/rte_version.h +++ b/dpdk/lib/librte_eal/common/include/rte_version.h @@ -45,6 +45,7 @@ extern "C" { #include #include +#include #include /** @@ -55,12 +56,12 @@ extern "C" { /** * Major version/year number i.e. the yy in yy.mm.z */ -#define RTE_VER_YEAR 16 +#define RTE_VER_YEAR 17 /** * Minor version/month number i.e. the mm in yy.mm.z */ -#define RTE_VER_MONTH 7 +#define RTE_VER_MONTH 11 /** * Patch level number i.e. the z in yy.mm.z diff --git a/dpdk/lib/librte_eal/common/include/rte_vfio.h b/dpdk/lib/librte_eal/common/include/rte_vfio.h new file mode 100644 index 00000000..a69c4ff6 --- /dev/null +++ b/dpdk/lib/librte_eal/common/include/rte_vfio.h @@ -0,0 +1,153 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 6WIND S.A. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of 6WIND nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_VFIO_H_ +#define _RTE_VFIO_H_ + +/* + * determine if VFIO is present on the system + */ +#if !defined(VFIO_PRESENT) && defined(RTE_EAL_VFIO) +#include +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 6, 0) +#define VFIO_PRESENT +#endif /* kernel version >= 3.6.0 */ +#endif /* RTE_EAL_VFIO */ + +#ifdef VFIO_PRESENT + +#include + +#define VFIO_DIR "/dev/vfio" +#define VFIO_CONTAINER_PATH "/dev/vfio/vfio" +#define VFIO_GROUP_FMT "/dev/vfio/%u" +#define VFIO_NOIOMMU_GROUP_FMT "/dev/vfio/noiommu-%u" +#define VFIO_GET_REGION_ADDR(x) ((uint64_t) x << 40ULL) +#define VFIO_GET_REGION_IDX(x) (x >> 40) +#define VFIO_NOIOMMU_MODE \ + "/sys/module/vfio/parameters/enable_unsafe_noiommu_mode" + +/** + * Setup vfio_cfg for the device identified by its address. + * It discovers the configured I/O MMU groups or sets a new one for the device. + * If a new groups is assigned, the DMA mapping is performed. + * + * This function is only relevant to linux and will return + * an error on BSD. + * + * @param sysfs_base + * sysfs path prefix. + * + * @param dev_addr + * device location. + * + * @param vfio_dev_fd + * VFIO fd. + * + * @param device_info + * Device information. + * + * @return + * 0 on success. + * <0 on failure. + * >1 if the device cannot be managed this way. + */ +int rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr, + int *vfio_dev_fd, struct vfio_device_info *device_info); + +/** + * Release a device mapped to a VFIO-managed I/O MMU group. + * + * This function is only relevant to linux and will return + * an error on BSD. + * + * @param sysfs_base + * sysfs path prefix. + * + * @param dev_addr + * device location. + * + * @param fd + * VFIO fd. + * + * @return + * 0 on success. + * <0 on failure. + */ +int rte_vfio_release_device(const char *sysfs_base, const char *dev_addr, int fd); + +/** + * Enable a VFIO-related kmod. + * + * This function is only relevant to linux and will return + * an error on BSD. + * + * @param modname + * kernel module name. + * + * @return + * 0 on success. + * <0 on failure. + */ +int rte_vfio_enable(const char *modname); + +/** + * Check whether a VFIO-related kmod is enabled. + * + * This function is only relevant to linux and will return + * an error on BSD. + * + * @param modname + * kernel module name. + * + * @return + * !0 if true. + * 0 otherwise. + */ +int rte_vfio_is_enabled(const char *modname); + +/** + * Whether VFIO NOIOMMU mode is enabled. + * + * This function is only relevant to linux and will return + * an error on BSD. + * + * @return + * !0 if true. + * 0 otherwise. + */ +int rte_vfio_noiommu_is_enabled(void); + +#endif /* VFIO_PRESENT */ + +#endif /* _RTE_VFIO_H_ */ diff --git a/dpdk/lib/librte_eal/common/include/rte_warnings.h b/dpdk/lib/librte_eal/common/include/rte_warnings.h deleted file mode 100644 index 54b545c9..00000000 --- a/dpdk/lib/librte_eal/common/include/rte_warnings.h +++ /dev/null @@ -1,84 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -/** - * @file - * Definitions of warnings for use of various insecure functions - */ - -#ifndef _RTE_WARNINGS_H_ -#define _RTE_WARNINGS_H_ - -#ifdef RTE_INSECURE_FUNCTION_WARNING - -/* we need to include all used standard header files so that they appear - * _before_ we poison the function names. - */ - -#include -#include -#include -#include -#include -#ifdef RTE_EXEC_ENV_LINUXAPP -#include -#endif - -/* the following function are deemed not fully secure for use e.g. they - * do not always null-terminate arguments */ -#pragma GCC poison sprintf strtok snprintf vsnprintf -#pragma GCC poison strlen strcpy strcat -#pragma GCC poison sscanf - -/* other unsafe functions may be implemented as macros so just undef them */ -#ifdef strsep -#undef strsep -#else -#pragma GCC poison strsep -#endif - -#ifdef strncpy -#undef strncpy -#else -#pragma GCC poison strncpy -#endif - -#ifdef strncat -#undef strncat -#else -#pragma GCC poison strncat -#endif - -#endif - -#endif /* RTE_WARNINGS_H */ diff --git a/dpdk/lib/librte_eal/common/malloc_elem.c b/dpdk/lib/librte_eal/common/malloc_elem.c index 42568e1d..f6cbc422 100644 --- a/dpdk/lib/librte_eal/common/malloc_elem.c +++ b/dpdk/lib/librte_eal/common/malloc_elem.c @@ -51,7 +51,7 @@ #define MIN_DATA_SIZE (RTE_CACHE_LINE_SIZE) /* - * initialise a general malloc_elem header structure + * Initialize a general malloc_elem header structure */ void malloc_elem_init(struct malloc_elem *elem, @@ -69,7 +69,7 @@ malloc_elem_init(struct malloc_elem *elem, } /* - * initialise a dummy malloc_elem header for the end-of-memseg marker + * Initialize a dummy malloc_elem header for the end-of-memseg marker */ void malloc_elem_mkend(struct malloc_elem *elem, struct malloc_elem *prev) @@ -98,6 +98,7 @@ elem_start_pt(struct malloc_elem *elem, size_t size, unsigned align, if ((new_data_start & bmask) != ((end_pt - 1) & bmask)) { end_pt = RTE_ALIGN_FLOOR(end_pt, bound); new_data_start = RTE_ALIGN_FLOOR((end_pt - size), align); + end_pt = new_data_start + size; if (((end_pt - 1) & bmask) != (new_data_start & bmask)) return NULL; } @@ -228,7 +229,7 @@ malloc_elem_alloc(struct malloc_elem *elem, size_t size, unsigned align, elem->pad = old_elem_size; /* put a dummy header in padding, to point to real element header */ - if (elem->pad > 0){ /* pad will be at least 64-bytes, as everything + if (elem->pad > 0) { /* pad will be at least 64-bytes, as everything * is cache-line aligned */ new_elem->pad = elem->pad; new_elem->state = ELEM_PAD; @@ -252,7 +253,7 @@ malloc_elem_alloc(struct malloc_elem *elem, size_t size, unsigned align, } /* - * joing two struct malloc_elem together. elem1 and elem2 must + * join two struct malloc_elem together. elem1 and elem2 must * be contiguous in memory. */ static inline void @@ -275,14 +276,14 @@ malloc_elem_free(struct malloc_elem *elem) return -1; rte_spinlock_lock(&(elem->heap->lock)); - size_t sz = elem->size - sizeof(*elem); + size_t sz = elem->size - sizeof(*elem) - MALLOC_ELEM_TRAILER_LEN; uint8_t *ptr = (uint8_t *)&elem[1]; struct malloc_elem *next = RTE_PTR_ADD(elem, elem->size); if (next->state == ELEM_FREE){ /* remove from free list, join to this one */ elem_free_list_remove(next); join_elem(elem, next); - sz += sizeof(*elem); + sz += (sizeof(*elem) + MALLOC_ELEM_TRAILER_LEN); } /* check if previous element is free, if so join with it and return, @@ -291,8 +292,8 @@ malloc_elem_free(struct malloc_elem *elem) if (elem->prev != NULL && elem->prev->state == ELEM_FREE) { elem_free_list_remove(elem->prev); join_elem(elem->prev, elem); - sz += sizeof(*elem); - ptr -= sizeof(*elem); + sz += (sizeof(*elem) + MALLOC_ELEM_TRAILER_LEN); + ptr -= (sizeof(*elem) + MALLOC_ELEM_TRAILER_LEN); elem = elem->prev; } malloc_elem_free_list_insert(elem); @@ -314,17 +315,16 @@ malloc_elem_free(struct malloc_elem *elem) int malloc_elem_resize(struct malloc_elem *elem, size_t size) { - const size_t new_size = size + MALLOC_ELEM_OVERHEAD; + const size_t new_size = size + elem->pad + MALLOC_ELEM_OVERHEAD; /* if we request a smaller size, then always return ok */ - const size_t current_size = elem->size - elem->pad; - if (current_size >= new_size) + if (elem->size >= new_size) return 0; struct malloc_elem *next = RTE_PTR_ADD(elem, elem->size); rte_spinlock_lock(&elem->heap->lock); if (next ->state != ELEM_FREE) goto err_return; - if (current_size + next->size < new_size) + if (elem->size + next->size < new_size) goto err_return; /* we now know the element fits, so remove from free list, @@ -333,7 +333,7 @@ malloc_elem_resize(struct malloc_elem *elem, size_t size) elem_free_list_remove(next); join_elem(elem, next); - if (elem->size - new_size >= MIN_DATA_SIZE + MALLOC_ELEM_OVERHEAD){ + if (elem->size - new_size >= MIN_DATA_SIZE + MALLOC_ELEM_OVERHEAD) { /* now we have a big block together. Lets cut it down a bit, by splitting */ struct malloc_elem *split_pt = RTE_PTR_ADD(elem, new_size); split_pt = RTE_PTR_ALIGN_CEIL(split_pt, RTE_CACHE_LINE_SIZE); diff --git a/dpdk/lib/librte_eal/common/malloc_elem.h b/dpdk/lib/librte_eal/common/malloc_elem.h index f04b2d1e..ce39129d 100644 --- a/dpdk/lib/librte_eal/common/malloc_elem.h +++ b/dpdk/lib/librte_eal/common/malloc_elem.h @@ -53,13 +53,13 @@ struct malloc_elem { volatile enum elem_state state; uint32_t pad; size_t size; -#ifdef RTE_LIBRTE_MALLOC_DEBUG +#ifdef RTE_MALLOC_DEBUG uint64_t header_cookie; /* Cookie marking start of data */ /* trailer cookie at start + size */ #endif } __rte_cache_aligned; -#ifndef RTE_LIBRTE_MALLOC_DEBUG +#ifndef RTE_MALLOC_DEBUG static const unsigned MALLOC_ELEM_TRAILER_LEN = 0; /* dummy function - just check if pointer is non-null */ diff --git a/dpdk/lib/librte_eal/common/malloc_heap.c b/dpdk/lib/librte_eal/common/malloc_heap.c index 763fa324..c731f1cd 100644 --- a/dpdk/lib/librte_eal/common/malloc_heap.c +++ b/dpdk/lib/librte_eal/common/malloc_heap.c @@ -178,12 +178,14 @@ malloc_heap_alloc(struct malloc_heap *heap, * Function to retrieve data for heap on given socket */ int -malloc_heap_get_stats(const struct malloc_heap *heap, +malloc_heap_get_stats(struct malloc_heap *heap, struct rte_malloc_socket_stats *socket_stats) { size_t idx; struct malloc_elem *elem; + rte_spinlock_lock(&heap->lock); + /* Initialise variables for heap */ socket_stats->free_count = 0; socket_stats->heap_freesz_bytes = 0; @@ -205,6 +207,8 @@ malloc_heap_get_stats(const struct malloc_heap *heap, socket_stats->heap_allocsz_bytes = (socket_stats->heap_totalsz_bytes - socket_stats->heap_freesz_bytes); socket_stats->alloc_count = heap->alloc_count; + + rte_spinlock_unlock(&heap->lock); return 0; } @@ -221,14 +225,6 @@ rte_eal_malloc_heap_init(void) for (ms = &mcfg->memseg[0], ms_cnt = 0; (ms_cnt < RTE_MAX_MEMSEG) && (ms->len > 0); ms_cnt++, ms++) { -#ifdef RTE_LIBRTE_IVSHMEM - /* - * if segment has ioremap address set, it's an IVSHMEM segment and - * it is not memory to allocate from. - */ - if (ms->ioremap_addr != 0) - continue; -#endif malloc_heap_add_memseg(&mcfg->malloc_heaps[ms->socket_id], ms); } diff --git a/dpdk/lib/librte_eal/common/malloc_heap.h b/dpdk/lib/librte_eal/common/malloc_heap.h index 3ccbef0f..3b1166f0 100644 --- a/dpdk/lib/librte_eal/common/malloc_heap.h +++ b/dpdk/lib/librte_eal/common/malloc_heap.h @@ -57,7 +57,7 @@ malloc_heap_alloc(struct malloc_heap *heap, const char *type, size_t size, unsigned flags, size_t align, size_t bound); int -malloc_heap_get_stats(const struct malloc_heap *heap, +malloc_heap_get_stats(struct malloc_heap *heap, struct rte_malloc_socket_stats *socket_stats); int diff --git a/dpdk/lib/librte_eal/common/rte_keepalive.c b/dpdk/lib/librte_eal/common/rte_keepalive.c index 9765d1bd..0c7ecba2 100644 --- a/dpdk/lib/librte_eal/common/rte_keepalive.c +++ b/dpdk/lib/librte_eal/common/rte_keepalive.c @@ -38,12 +38,15 @@ #include #include #include -#include struct rte_keepalive { /** Core Liveness. */ - enum rte_keepalive_state __rte_cache_aligned state_flags[ - RTE_KEEPALIVE_MAXCORES]; + struct { + /* + * Each element must be cache aligned to prevent false sharing. + */ + enum rte_keepalive_state core_state __rte_cache_aligned; + } live_data[RTE_KEEPALIVE_MAXCORES]; /** Last-seen-alive timestamps */ uint64_t last_alive[RTE_KEEPALIVE_MAXCORES]; @@ -96,19 +99,22 @@ rte_keepalive_dispatch_pings(__rte_unused void *ptr_timer, if (keepcfg->active_cores[idx_core] == 0) continue; - switch (keepcfg->state_flags[idx_core]) { + switch (keepcfg->live_data[idx_core].core_state) { case RTE_KA_STATE_UNUSED: break; case RTE_KA_STATE_ALIVE: /* Alive */ - keepcfg->state_flags[idx_core] = RTE_KA_STATE_MISSING; + keepcfg->live_data[idx_core].core_state = + RTE_KA_STATE_MISSING; keepcfg->last_alive[idx_core] = rte_rdtsc(); break; case RTE_KA_STATE_MISSING: /* MIA */ print_trace("Core MIA. ", keepcfg, idx_core); - keepcfg->state_flags[idx_core] = RTE_KA_STATE_DEAD; + keepcfg->live_data[idx_core].core_state = + RTE_KA_STATE_DEAD; break; case RTE_KA_STATE_DEAD: /* Dead */ - keepcfg->state_flags[idx_core] = RTE_KA_STATE_GONE; + keepcfg->live_data[idx_core].core_state = + RTE_KA_STATE_GONE; print_trace("Core died. ", keepcfg, idx_core); if (keepcfg->callback) keepcfg->callback( @@ -119,7 +125,8 @@ rte_keepalive_dispatch_pings(__rte_unused void *ptr_timer, case RTE_KA_STATE_GONE: /* Buried */ break; case RTE_KA_STATE_DOZING: /* Core going idle */ - keepcfg->state_flags[idx_core] = RTE_KA_STATE_SLEEP; + keepcfg->live_data[idx_core].core_state = + RTE_KA_STATE_SLEEP; keepcfg->last_alive[idx_core] = rte_rdtsc(); break; case RTE_KA_STATE_SLEEP: /* Idled core */ @@ -129,7 +136,7 @@ rte_keepalive_dispatch_pings(__rte_unused void *ptr_timer, keepcfg->relay_callback( keepcfg->relay_callback_data, idx_core, - keepcfg->state_flags[idx_core], + keepcfg->live_data[idx_core].core_state, keepcfg->last_alive[idx_core] ); } @@ -173,11 +180,11 @@ rte_keepalive_register_core(struct rte_keepalive *keepcfg, const int id_core) void rte_keepalive_mark_alive(struct rte_keepalive *keepcfg) { - keepcfg->state_flags[rte_lcore_id()] = RTE_KA_STATE_ALIVE; + keepcfg->live_data[rte_lcore_id()].core_state = RTE_KA_STATE_ALIVE; } void rte_keepalive_mark_sleep(struct rte_keepalive *keepcfg) { - keepcfg->state_flags[rte_lcore_id()] = RTE_KA_STATE_DOZING; + keepcfg->live_data[rte_lcore_id()].core_state = RTE_KA_STATE_DOZING; } diff --git a/dpdk/lib/librte_eal/common/rte_malloc.c b/dpdk/lib/librte_eal/common/rte_malloc.c index f4a88352..fe2278bc 100644 --- a/dpdk/lib/librte_eal/common/rte_malloc.c +++ b/dpdk/lib/librte_eal/common/rte_malloc.c @@ -246,13 +246,22 @@ rte_malloc_set_limit(__rte_unused const char *type, } /* - * Return the physical address of a virtual address obtained through rte_malloc + * Return the IO address of a virtual address obtained through rte_malloc */ -phys_addr_t -rte_malloc_virt2phy(const void *addr) +rte_iova_t +rte_malloc_virt2iova(const void *addr) { + rte_iova_t iova; const struct malloc_elem *elem = malloc_elem_from_data(addr); if (elem == NULL) - return 0; - return elem->ms->phys_addr + ((uintptr_t)addr - (uintptr_t)elem->ms->addr); + return RTE_BAD_IOVA; + if (elem->ms->iova == RTE_BAD_IOVA) + return RTE_BAD_IOVA; + + if (rte_eal_iova_mode() == RTE_IOVA_VA) + iova = (uintptr_t)addr; + else + iova = elem->ms->iova + + RTE_PTR_DIFF(addr, elem->ms->addr); + return iova; } diff --git a/dpdk/lib/librte_eal/common/rte_service.c b/dpdk/lib/librte_eal/common/rte_service.c new file mode 100644 index 00000000..1f922940 --- /dev/null +++ b/dpdk/lib/librte_eal/common/rte_service.c @@ -0,0 +1,802 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include + +#include +#include "include/rte_service_component.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#define RTE_SERVICE_NUM_MAX 64 + +#define SERVICE_F_REGISTERED (1 << 0) +#define SERVICE_F_STATS_ENABLED (1 << 1) +#define SERVICE_F_START_CHECK (1 << 2) + +/* runstates for services and lcores, denoting if they are active or not */ +#define RUNSTATE_STOPPED 0 +#define RUNSTATE_RUNNING 1 + +/* internal representation of a service */ +struct rte_service_spec_impl { + /* public part of the struct */ + struct rte_service_spec spec; + + /* atomic lock that when set indicates a service core is currently + * running this service callback. When not set, a core may take the + * lock and then run the service callback. + */ + rte_atomic32_t execute_lock; + + /* API set/get-able variables */ + int8_t app_runstate; + int8_t comp_runstate; + uint8_t internal_flags; + + /* per service statistics */ + rte_atomic32_t num_mapped_cores; + uint64_t calls; + uint64_t cycles_spent; +} __rte_cache_aligned; + +/* the internal values of a service core */ +struct core_state { + /* map of services IDs are run on this core */ + uint64_t service_mask; + uint8_t runstate; /* running or stopped */ + uint8_t is_service_core; /* set if core is currently a service core */ + + /* extreme statistics */ + uint64_t calls_per_service[RTE_SERVICE_NUM_MAX]; +} __rte_cache_aligned; + +static uint32_t rte_service_count; +static struct rte_service_spec_impl *rte_services; +static struct core_state *lcore_states; +static uint32_t rte_service_library_initialized; + +int32_t rte_service_init(void) +{ + if (rte_service_library_initialized) { + printf("service library init() called, init flag %d\n", + rte_service_library_initialized); + return -EALREADY; + } + + rte_services = rte_calloc("rte_services", RTE_SERVICE_NUM_MAX, + sizeof(struct rte_service_spec_impl), + RTE_CACHE_LINE_SIZE); + if (!rte_services) { + printf("error allocating rte services array\n"); + goto fail_mem; + } + + lcore_states = rte_calloc("rte_service_core_states", RTE_MAX_LCORE, + sizeof(struct core_state), RTE_CACHE_LINE_SIZE); + if (!lcore_states) { + printf("error allocating core states array\n"); + goto fail_mem; + } + + int i; + int count = 0; + struct rte_config *cfg = rte_eal_get_configuration(); + for (i = 0; i < RTE_MAX_LCORE; i++) { + if (lcore_config[i].core_role == ROLE_SERVICE) { + if ((unsigned int)i == cfg->master_lcore) + continue; + rte_service_lcore_add(i); + count++; + } + } + + rte_service_library_initialized = 1; + return 0; +fail_mem: + if (rte_services) + rte_free(rte_services); + if (lcore_states) + rte_free(lcore_states); + return -ENOMEM; +} + +/* returns 1 if service is registered and has not been unregistered + * Returns 0 if service never registered, or has been unregistered + */ +static inline int +service_valid(uint32_t id) +{ + return !!(rte_services[id].internal_flags & SERVICE_F_REGISTERED); +} + +/* validate ID and retrieve service pointer, or return error value */ +#define SERVICE_VALID_GET_OR_ERR_RET(id, service, retval) do { \ + if (id >= RTE_SERVICE_NUM_MAX || !service_valid(id)) \ + return retval; \ + service = &rte_services[id]; \ +} while (0) + +/* returns 1 if statistics should be collected for service + * Returns 0 if statistics should not be collected for service + */ +static inline int +service_stats_enabled(struct rte_service_spec_impl *impl) +{ + return !!(impl->internal_flags & SERVICE_F_STATS_ENABLED); +} + +static inline int +service_mt_safe(struct rte_service_spec_impl *s) +{ + return !!(s->spec.capabilities & RTE_SERVICE_CAP_MT_SAFE); +} + +int32_t rte_service_set_stats_enable(uint32_t id, int32_t enabled) +{ + struct rte_service_spec_impl *s; + SERVICE_VALID_GET_OR_ERR_RET(id, s, 0); + + if (enabled) + s->internal_flags |= SERVICE_F_STATS_ENABLED; + else + s->internal_flags &= ~(SERVICE_F_STATS_ENABLED); + + return 0; +} + +int32_t rte_service_set_runstate_mapped_check(uint32_t id, int32_t enabled) +{ + struct rte_service_spec_impl *s; + SERVICE_VALID_GET_OR_ERR_RET(id, s, 0); + + if (enabled) + s->internal_flags |= SERVICE_F_START_CHECK; + else + s->internal_flags &= ~(SERVICE_F_START_CHECK); + + return 0; +} + +uint32_t +rte_service_get_count(void) +{ + return rte_service_count; +} + +int32_t rte_service_get_by_name(const char *name, uint32_t *service_id) +{ + if (!service_id) + return -EINVAL; + + int i; + for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) { + if (service_valid(i) && + strcmp(name, rte_services[i].spec.name) == 0) { + *service_id = i; + return 0; + } + } + + return -ENODEV; +} + +const char * +rte_service_get_name(uint32_t id) +{ + struct rte_service_spec_impl *s; + SERVICE_VALID_GET_OR_ERR_RET(id, s, 0); + return s->spec.name; +} + +int32_t +rte_service_probe_capability(uint32_t id, uint32_t capability) +{ + struct rte_service_spec_impl *s; + SERVICE_VALID_GET_OR_ERR_RET(id, s, -EINVAL); + return !!(s->spec.capabilities & capability); +} + +int32_t +rte_service_component_register(const struct rte_service_spec *spec, + uint32_t *id_ptr) +{ + uint32_t i; + int32_t free_slot = -1; + + if (spec->callback == NULL || strlen(spec->name) == 0) + return -EINVAL; + + for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) { + if (!service_valid(i)) { + free_slot = i; + break; + } + } + + if ((free_slot < 0) || (i == RTE_SERVICE_NUM_MAX)) + return -ENOSPC; + + struct rte_service_spec_impl *s = &rte_services[free_slot]; + s->spec = *spec; + s->internal_flags |= SERVICE_F_REGISTERED | SERVICE_F_START_CHECK; + + rte_smp_wmb(); + rte_service_count++; + + if (id_ptr) + *id_ptr = free_slot; + + return 0; +} + +int32_t +rte_service_component_unregister(uint32_t id) +{ + uint32_t i; + struct rte_service_spec_impl *s; + SERVICE_VALID_GET_OR_ERR_RET(id, s, -EINVAL); + + rte_service_count--; + rte_smp_wmb(); + + s->internal_flags &= ~(SERVICE_F_REGISTERED); + + /* clear the run-bit in all cores */ + for (i = 0; i < RTE_MAX_LCORE; i++) + lcore_states[i].service_mask &= ~(UINT64_C(1) << id); + + memset(&rte_services[id], 0, sizeof(struct rte_service_spec_impl)); + + return 0; +} + +int32_t +rte_service_component_runstate_set(uint32_t id, uint32_t runstate) +{ + struct rte_service_spec_impl *s; + SERVICE_VALID_GET_OR_ERR_RET(id, s, -EINVAL); + + if (runstate) + s->comp_runstate = RUNSTATE_RUNNING; + else + s->comp_runstate = RUNSTATE_STOPPED; + + rte_smp_wmb(); + return 0; +} + +int32_t +rte_service_runstate_set(uint32_t id, uint32_t runstate) +{ + struct rte_service_spec_impl *s; + SERVICE_VALID_GET_OR_ERR_RET(id, s, -EINVAL); + + if (runstate) + s->app_runstate = RUNSTATE_RUNNING; + else + s->app_runstate = RUNSTATE_STOPPED; + + rte_smp_wmb(); + return 0; +} + +int32_t +rte_service_runstate_get(uint32_t id) +{ + struct rte_service_spec_impl *s; + SERVICE_VALID_GET_OR_ERR_RET(id, s, -EINVAL); + rte_smp_rmb(); + + int check_disabled = !(s->internal_flags & SERVICE_F_START_CHECK); + int lcore_mapped = (rte_atomic32_read(&s->num_mapped_cores) > 0); + + return (s->app_runstate == RUNSTATE_RUNNING) && + (s->comp_runstate == RUNSTATE_RUNNING) && + (check_disabled | lcore_mapped); +} + +static inline void +rte_service_runner_do_callback(struct rte_service_spec_impl *s, + struct core_state *cs, uint32_t service_idx) +{ + void *userdata = s->spec.callback_userdata; + + if (service_stats_enabled(s)) { + uint64_t start = rte_rdtsc(); + s->spec.callback(userdata); + uint64_t end = rte_rdtsc(); + s->cycles_spent += end - start; + cs->calls_per_service[service_idx]++; + s->calls++; + } else + s->spec.callback(userdata); +} + + +static inline int32_t +service_run(uint32_t i, struct core_state *cs, uint64_t service_mask) +{ + if (!service_valid(i)) + return -EINVAL; + struct rte_service_spec_impl *s = &rte_services[i]; + if (s->comp_runstate != RUNSTATE_RUNNING || + s->app_runstate != RUNSTATE_RUNNING || + !(service_mask & (UINT64_C(1) << i))) + return -ENOEXEC; + + /* check do we need cmpset, if MT safe or <= 1 core + * mapped, atomic ops are not required. + */ + const int use_atomics = (service_mt_safe(s) == 0) && + (rte_atomic32_read(&s->num_mapped_cores) > 1); + if (use_atomics) { + if (!rte_atomic32_cmpset((uint32_t *)&s->execute_lock, 0, 1)) + return -EBUSY; + + rte_service_runner_do_callback(s, cs, i); + rte_atomic32_clear(&s->execute_lock); + } else + rte_service_runner_do_callback(s, cs, i); + + return 0; +} + +int32_t rte_service_run_iter_on_app_lcore(uint32_t id, + uint32_t serialize_mt_unsafe) +{ + /* run service on calling core, using all-ones as the service mask */ + if (!service_valid(id)) + return -EINVAL; + + struct core_state *cs = &lcore_states[rte_lcore_id()]; + struct rte_service_spec_impl *s = &rte_services[id]; + + /* Atomically add this core to the mapped cores first, then examine if + * we can run the service. This avoids a race condition between + * checking the value, and atomically adding to the mapped count. + */ + if (serialize_mt_unsafe) + rte_atomic32_inc(&s->num_mapped_cores); + + if (service_mt_safe(s) == 0 && + rte_atomic32_read(&s->num_mapped_cores) > 1) { + if (serialize_mt_unsafe) + rte_atomic32_dec(&s->num_mapped_cores); + return -EBUSY; + } + + int ret = service_run(id, cs, UINT64_MAX); + + if (serialize_mt_unsafe) + rte_atomic32_dec(&s->num_mapped_cores); + + return ret; +} + +static int32_t +rte_service_runner_func(void *arg) +{ + RTE_SET_USED(arg); + uint32_t i; + const int lcore = rte_lcore_id(); + struct core_state *cs = &lcore_states[lcore]; + + while (lcore_states[lcore].runstate == RUNSTATE_RUNNING) { + const uint64_t service_mask = cs->service_mask; + + for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) { + /* return value ignored as no change to code flow */ + service_run(i, cs, service_mask); + } + + rte_smp_rmb(); + } + + lcore_config[lcore].state = WAIT; + + return 0; +} + +int32_t +rte_service_lcore_count(void) +{ + int32_t count = 0; + uint32_t i; + for (i = 0; i < RTE_MAX_LCORE; i++) + count += lcore_states[i].is_service_core; + return count; +} + +int32_t +rte_service_lcore_list(uint32_t array[], uint32_t n) +{ + uint32_t count = rte_service_lcore_count(); + if (count > n) + return -ENOMEM; + + if (!array) + return -EINVAL; + + uint32_t i; + uint32_t idx = 0; + for (i = 0; i < RTE_MAX_LCORE; i++) { + struct core_state *cs = &lcore_states[i]; + if (cs->is_service_core) { + array[idx] = i; + idx++; + } + } + + return count; +} + +int32_t +rte_service_lcore_count_services(uint32_t lcore) +{ + if (lcore >= RTE_MAX_LCORE) + return -EINVAL; + + struct core_state *cs = &lcore_states[lcore]; + if (!cs->is_service_core) + return -ENOTSUP; + + return __builtin_popcountll(cs->service_mask); +} + +int32_t +rte_service_start_with_defaults(void) +{ + /* create a default mapping from cores to services, then start the + * services to make them transparent to unaware applications. + */ + uint32_t i; + int ret; + uint32_t count = rte_service_get_count(); + + int32_t lcore_iter = 0; + uint32_t ids[RTE_MAX_LCORE] = {0}; + int32_t lcore_count = rte_service_lcore_list(ids, RTE_MAX_LCORE); + + if (lcore_count == 0) + return -ENOTSUP; + + for (i = 0; (int)i < lcore_count; i++) + rte_service_lcore_start(ids[i]); + + for (i = 0; i < count; i++) { + /* do 1:1 core mapping here, with each service getting + * assigned a single core by default. Adding multiple services + * should multiplex to a single core, or 1:1 if there are the + * same amount of services as service-cores + */ + ret = rte_service_map_lcore_set(i, ids[lcore_iter], 1); + if (ret) + return -ENODEV; + + lcore_iter++; + if (lcore_iter >= lcore_count) + lcore_iter = 0; + + ret = rte_service_runstate_set(i, 1); + if (ret) + return -ENOEXEC; + } + + return 0; +} + +static int32_t +service_update(struct rte_service_spec *service, uint32_t lcore, + uint32_t *set, uint32_t *enabled) +{ + uint32_t i; + int32_t sid = -1; + + for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) { + if ((struct rte_service_spec *)&rte_services[i] == service && + service_valid(i)) { + sid = i; + break; + } + } + + if (sid == -1 || lcore >= RTE_MAX_LCORE) + return -EINVAL; + + if (!lcore_states[lcore].is_service_core) + return -EINVAL; + + uint64_t sid_mask = UINT64_C(1) << sid; + if (set) { + uint64_t lcore_mapped = lcore_states[lcore].service_mask & + sid_mask; + + if (*set && !lcore_mapped) { + lcore_states[lcore].service_mask |= sid_mask; + rte_atomic32_inc(&rte_services[sid].num_mapped_cores); + } + if (!*set && lcore_mapped) { + lcore_states[lcore].service_mask &= ~(sid_mask); + rte_atomic32_dec(&rte_services[sid].num_mapped_cores); + } + } + + if (enabled) + *enabled = !!(lcore_states[lcore].service_mask & (sid_mask)); + + rte_smp_wmb(); + + return 0; +} + +int32_t +rte_service_map_lcore_set(uint32_t id, uint32_t lcore, uint32_t enabled) +{ + struct rte_service_spec_impl *s; + SERVICE_VALID_GET_OR_ERR_RET(id, s, -EINVAL); + uint32_t on = enabled > 0; + return service_update(&s->spec, lcore, &on, 0); +} + +int32_t +rte_service_map_lcore_get(uint32_t id, uint32_t lcore) +{ + struct rte_service_spec_impl *s; + SERVICE_VALID_GET_OR_ERR_RET(id, s, -EINVAL); + uint32_t enabled; + int ret = service_update(&s->spec, lcore, 0, &enabled); + if (ret == 0) + return enabled; + return ret; +} + +static void +set_lcore_state(uint32_t lcore, int32_t state) +{ + /* mark core state in hugepage backed config */ + struct rte_config *cfg = rte_eal_get_configuration(); + cfg->lcore_role[lcore] = state; + + /* mark state in process local lcore_config */ + lcore_config[lcore].core_role = state; + + /* update per-lcore optimized state tracking */ + lcore_states[lcore].is_service_core = (state == ROLE_SERVICE); +} + +int32_t rte_service_lcore_reset_all(void) +{ + /* loop over cores, reset all to mask 0 */ + uint32_t i; + for (i = 0; i < RTE_MAX_LCORE; i++) { + if (lcore_states[i].is_service_core) { + lcore_states[i].service_mask = 0; + set_lcore_state(i, ROLE_RTE); + lcore_states[i].runstate = RUNSTATE_STOPPED; + } + } + for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) + rte_atomic32_set(&rte_services[i].num_mapped_cores, 0); + + rte_smp_wmb(); + + return 0; +} + +int32_t +rte_service_lcore_add(uint32_t lcore) +{ + if (lcore >= RTE_MAX_LCORE) + return -EINVAL; + if (lcore_states[lcore].is_service_core) + return -EALREADY; + + set_lcore_state(lcore, ROLE_SERVICE); + + /* ensure that after adding a core the mask and state are defaults */ + lcore_states[lcore].service_mask = 0; + lcore_states[lcore].runstate = RUNSTATE_STOPPED; + + rte_smp_wmb(); + + return rte_eal_wait_lcore(lcore); +} + +int32_t +rte_service_lcore_del(uint32_t lcore) +{ + if (lcore >= RTE_MAX_LCORE) + return -EINVAL; + + struct core_state *cs = &lcore_states[lcore]; + if (!cs->is_service_core) + return -EINVAL; + + if (cs->runstate != RUNSTATE_STOPPED) + return -EBUSY; + + set_lcore_state(lcore, ROLE_RTE); + + rte_smp_wmb(); + return 0; +} + +int32_t +rte_service_lcore_start(uint32_t lcore) +{ + if (lcore >= RTE_MAX_LCORE) + return -EINVAL; + + struct core_state *cs = &lcore_states[lcore]; + if (!cs->is_service_core) + return -EINVAL; + + if (cs->runstate == RUNSTATE_RUNNING) + return -EALREADY; + + /* set core to run state first, and then launch otherwise it will + * return immediately as runstate keeps it in the service poll loop + */ + lcore_states[lcore].runstate = RUNSTATE_RUNNING; + + int ret = rte_eal_remote_launch(rte_service_runner_func, 0, lcore); + /* returns -EBUSY if the core is already launched, 0 on success */ + return ret; +} + +int32_t +rte_service_lcore_stop(uint32_t lcore) +{ + if (lcore >= RTE_MAX_LCORE) + return -EINVAL; + + if (lcore_states[lcore].runstate == RUNSTATE_STOPPED) + return -EALREADY; + + uint32_t i; + uint64_t service_mask = lcore_states[lcore].service_mask; + for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) { + int32_t enabled = service_mask & (UINT64_C(1) << i); + int32_t service_running = rte_service_runstate_get(i); + int32_t only_core = (1 == + rte_atomic32_read(&rte_services[i].num_mapped_cores)); + + /* if the core is mapped, and the service is running, and this + * is the only core that is mapped, the service would cease to + * run if this core stopped, so fail instead. + */ + if (enabled && service_running && only_core) + return -EBUSY; + } + + lcore_states[lcore].runstate = RUNSTATE_STOPPED; + + return 0; +} + +static void +rte_service_dump_one(FILE *f, struct rte_service_spec_impl *s, + uint64_t all_cycles, uint32_t reset) +{ + /* avoid divide by zero */ + if (all_cycles == 0) + all_cycles = 1; + + int calls = 1; + if (s->calls != 0) + calls = s->calls; + + fprintf(f, " %s: stats %d\tcalls %"PRIu64"\tcycles %" + PRIu64"\tavg: %"PRIu64"\n", + s->spec.name, service_stats_enabled(s), s->calls, + s->cycles_spent, s->cycles_spent / calls); + + if (reset) { + s->cycles_spent = 0; + s->calls = 0; + } +} + +static void +service_dump_calls_per_lcore(FILE *f, uint32_t lcore, uint32_t reset) +{ + uint32_t i; + struct core_state *cs = &lcore_states[lcore]; + + fprintf(f, "%02d\t", lcore); + for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) { + if (!service_valid(i)) + continue; + fprintf(f, "%"PRIu64"\t", cs->calls_per_service[i]); + if (reset) + cs->calls_per_service[i] = 0; + } + fprintf(f, "\n"); +} + +int32_t rte_service_dump(FILE *f, uint32_t id) +{ + uint32_t i; + int print_one = (id != UINT32_MAX); + + uint64_t total_cycles = 0; + + for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) { + if (!service_valid(i)) + continue; + total_cycles += rte_services[i].cycles_spent; + } + + /* print only the specified service */ + if (print_one) { + struct rte_service_spec_impl *s; + SERVICE_VALID_GET_OR_ERR_RET(id, s, -EINVAL); + fprintf(f, "Service %s Summary\n", s->spec.name); + uint32_t reset = 0; + rte_service_dump_one(f, s, total_cycles, reset); + return 0; + } + + /* print all services, as UINT32_MAX was passed as id */ + fprintf(f, "Services Summary\n"); + for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) { + if (!service_valid(i)) + continue; + uint32_t reset = 1; + rte_service_dump_one(f, &rte_services[i], total_cycles, reset); + } + + fprintf(f, "Service Cores Summary\n"); + for (i = 0; i < RTE_MAX_LCORE; i++) { + if (lcore_config[i].core_role != ROLE_SERVICE) + continue; + + uint32_t reset = 1; + service_dump_calls_per_lcore(f, i, reset); + } + + return 0; +} diff --git a/dpdk/lib/librte_eal/linuxapp/Makefile b/dpdk/lib/librte_eal/linuxapp/Makefile index 20d2a916..2ebdf313 100644 --- a/dpdk/lib/librte_eal/linuxapp/Makefile +++ b/dpdk/lib/librte_eal/linuxapp/Makefile @@ -34,6 +34,6 @@ include $(RTE_SDK)/mk/rte.vars.mk DIRS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal DIRS-$(CONFIG_RTE_EAL_IGB_UIO) += igb_uio DIRS-$(CONFIG_RTE_KNI_KMOD) += kni -DIRS-$(CONFIG_RTE_LIBRTE_XEN_DOM0) += xen_dom0 +DEPDIRS-kni := eal include $(RTE_SDK)/mk/rte.subdir.mk diff --git a/dpdk/lib/librte_eal/linuxapp/eal/Makefile b/dpdk/lib/librte_eal/linuxapp/eal/Makefile index 182729c0..5a7b8b2a 100644 --- a/dpdk/lib/librte_eal/linuxapp/eal/Makefile +++ b/dpdk/lib/librte_eal/linuxapp/eal/Makefile @@ -34,51 +34,39 @@ include $(RTE_SDK)/mk/rte.vars.mk LIB = librte_eal.a ARCH_DIR ?= $(RTE_ARCH) -EXPORT_MAP := rte_eal_version.map +EXPORT_MAP := ../../rte_eal_version.map VPATH += $(RTE_SDK)/lib/librte_eal/common/arch/$(ARCH_DIR) -LIBABIVER := 2 +LIBABIVER := 6 VPATH += $(RTE_SDK)/lib/librte_eal/common CFLAGS += -I$(SRCDIR)/include CFLAGS += -I$(RTE_SDK)/lib/librte_eal/common CFLAGS += -I$(RTE_SDK)/lib/librte_eal/common/include -ifeq ($(CONFIG_RTE_LIBRTE_IVSHMEM),y) -# workaround for circular dependency eal -> ivshmem -> ring/mempool -> eal -CFLAGS += -I$(RTE_SDK)/lib/librte_ring -CFLAGS += -I$(RTE_SDK)/lib/librte_mempool -CFLAGS += -I$(RTE_SDK)/lib/librte_ivshmem -endif CFLAGS += $(WERROR_FLAGS) -O3 LDLIBS += -ldl LDLIBS += -lpthread LDLIBS += -lgcc_s LDLIBS += -lrt +ifeq ($(CONFIG_RTE_EAL_NUMA_AWARE_HUGEPAGES),y) +LDLIBS += -lnuma +endif # specific to linuxapp exec-env SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) := eal.c SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_hugepage_info.c SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_memory.c -ifeq ($(CONFIG_RTE_LIBRTE_XEN_DOM0),y) -SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_xen_memory.c -endif SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_thread.c SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_log.c SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_vfio.c SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_vfio_mp_sync.c -SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_pci.c -SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_pci_uio.c -SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_pci_vfio.c SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_debug.c SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_lcore.c SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_timer.c SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_interrupts.c SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_alarm.c -ifeq ($(CONFIG_RTE_LIBRTE_IVSHMEM),y) -SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_ivshmem.c -endif # from common dir SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_common_lcore.c @@ -86,8 +74,6 @@ SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_common_timer.c SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_common_memzone.c SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_common_log.c SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_common_launch.c -SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_common_pci.c -SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_common_pci_uio.c SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_common_memory.c SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_common_tailqs.c SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_common_errno.c @@ -95,6 +81,7 @@ SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_common_cpuflags.c SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_common_string_fns.c SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_common_hexdump.c SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_common_devargs.c +SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_common_bus.c SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_common_dev.c SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_common_options.c SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_common_thread.c @@ -103,10 +90,12 @@ SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += rte_malloc.c SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += malloc_elem.c SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += malloc_heap.c SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += rte_keepalive.c +SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += rte_service.c # from arch dir SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += rte_cpuflags.c SRCS-$(CONFIG_RTE_ARCH_X86) += rte_spinlock.c +SRCS-y += rte_cycles.c CFLAGS_eal_common_cpuflags.o := $(CPUFLAGS_LIST) @@ -119,13 +108,11 @@ CFLAGS_eal_thread.o := -D_GNU_SOURCE CFLAGS_eal_log.o := -D_GNU_SOURCE CFLAGS_eal_common_log.o := -D_GNU_SOURCE CFLAGS_eal_hugepage_info.o := -D_GNU_SOURCE -CFLAGS_eal_pci.o := -D_GNU_SOURCE -CFLAGS_eal_pci_uio.o := -D_GNU_SOURCE -CFLAGS_eal_pci_vfio.o := -D_GNU_SOURCE CFLAGS_eal_common_whitelist.o := -D_GNU_SOURCE CFLAGS_eal_common_options.o := -D_GNU_SOURCE CFLAGS_eal_common_thread.o := -D_GNU_SOURCE CFLAGS_eal_common_lcore.o := -D_GNU_SOURCE +CFLAGS_rte_cycles.o := -D_GNU_SOURCE # workaround for a gcc bug with noreturn attribute # http://gcc.gnu.org/bugzilla/show_bug.cgi?id=12603 @@ -133,12 +120,9 @@ ifeq ($(CONFIG_RTE_TOOLCHAIN_GCC),y) CFLAGS_eal_thread.o += -Wno-return-type endif -INC := rte_interrupts.h rte_kni_common.h rte_dom0_common.h +INC := rte_kni_common.h SYMLINK-$(CONFIG_RTE_EXEC_ENV_LINUXAPP)-include/exec-env := \ $(addprefix include/exec-env/,$(INC)) -DEPDIRS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += lib/librte_eal/common -DEPDIRS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += lib/librte_eal/common/arch/$(ARCH_DIR) - include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/lib/librte_eal/linuxapp/eal/eal.c b/dpdk/lib/librte_eal/linuxapp/eal/eal.c index 3fb2188f..229eec9f 100644 --- a/dpdk/lib/librte_eal/linuxapp/eal/eal.c +++ b/dpdk/lib/librte_eal/linuxapp/eal/eal.c @@ -46,7 +46,6 @@ #include #include #include -#include #include #include #include @@ -57,24 +56,26 @@ #include #include #include -#include #include #include #include +#include #include #include +#include #include #include #include #include #include #include -#include +#include +#include #include -#include #include #include #include +#include #include "eal_private.h" #include "eal_thread.h" @@ -119,6 +120,13 @@ struct internal_config internal_config; /* used by rte_rdtsc() */ int rte_cycles_vmware_tsc_map; +/* Return mbuf pool ops name */ +const char * +rte_eal_mbuf_default_mempool_ops(void) +{ + return internal_config.mbuf_pool_ops_name; +} + /* Return a pointer to the configuration structure */ struct rte_config * rte_eal_get_configuration(void) @@ -126,6 +134,12 @@ rte_eal_get_configuration(void) return &rte_config; } +enum rte_iova_mode +rte_eal_iova_mode(void) +{ + return rte_eal_get_configuration()->iova_mode; +} + /* parse a sysfs (or other) file containing one integer value */ int eal_parse_sysfs_value(const char *filename, unsigned long *val) @@ -209,7 +223,7 @@ rte_eal_config_create(void) rte_panic("Cannot mmap memory for rte_config\n"); } memcpy(rte_mem_cfg_addr, &early_mem_config, sizeof(early_mem_config)); - rte_config.mem_config = (struct rte_mem_config *) rte_mem_cfg_addr; + rte_config.mem_config = rte_mem_cfg_addr; /* store address of the config in the config itself so that secondary * processes could later map the config into this exact location */ @@ -238,7 +252,8 @@ rte_eal_config_attach(void) mem_config = (struct rte_mem_config *) mmap(NULL, sizeof(*mem_config), PROT_READ, MAP_SHARED, mem_cfg_fd, 0); if (mem_config == MAP_FAILED) - rte_panic("Cannot mmap memory for rte_config\n"); + rte_panic("Cannot mmap memory for rte_config! error %i (%s)\n", + errno, strerror(errno)); rte_config.mem_config = mem_config; } @@ -263,9 +278,17 @@ rte_eal_config_reattach(void) mem_config = (struct rte_mem_config *) mmap(rte_mem_cfg_addr, sizeof(*mem_config), PROT_READ | PROT_WRITE, MAP_SHARED, mem_cfg_fd, 0); + if (mem_config == MAP_FAILED || mem_config != rte_mem_cfg_addr) { + if (mem_config != MAP_FAILED) + /* errno is stale, don't use */ + rte_panic("Cannot mmap memory for rte_config at [%p], got [%p]" + " - please use '--base-virtaddr' option\n", + rte_mem_cfg_addr, mem_config); + else + rte_panic("Cannot mmap memory for rte_config! error %i (%s)\n", + errno, strerror(errno)); + } close(mem_cfg_fd); - if (mem_config == MAP_FAILED || mem_config != rte_mem_cfg_addr) - rte_panic("Cannot mmap memory for rte_config\n"); rte_config.mem_config = mem_config; } @@ -343,7 +366,6 @@ eal_usage(const char *prgname) " --"OPT_BASE_VIRTADDR" Base virtual address\n" " --"OPT_CREATE_UIO_DEV" Create /dev/uioX (usually done by hotplug)\n" " --"OPT_VFIO_INTR" Interrupt mode for VFIO (legacy|msi|msix)\n" - " --"OPT_XEN_DOM0" Support running on Xen dom0 without hugetlbfs\n" "\n"); /* Allow the application to print its usage message too if hook is set */ if ( rte_application_usage_hook ) { @@ -480,8 +502,6 @@ eal_log_level_parse(int argc, char **argv) argvopt = argv; optind = 1; - eal_reset_internal_config(&internal_config); - while ((opt = getopt_long(argc, argvopt, eal_short_options, eal_long_options, &option_index)) != EOF) { @@ -546,25 +566,12 @@ eal_parse_args(int argc, char **argv) eal_usage(prgname); exit(EXIT_SUCCESS); - /* long options */ - case OPT_XEN_DOM0_NUM: -#ifdef RTE_LIBRTE_XEN_DOM0 - internal_config.xen_dom0_support = 1; -#else - RTE_LOG(ERR, EAL, "Can't support DPDK app " - "running on Dom0, please configure" - " RTE_LIBRTE_XEN_DOM0=y\n"); - ret = -1; - goto out; -#endif - break; - case OPT_HUGE_DIR_NUM: - internal_config.hugepage_dir = optarg; + internal_config.hugepage_dir = strdup(optarg); break; case OPT_FILE_PREFIX_NUM: - internal_config.hugefile_prefix = optarg; + internal_config.hugefile_prefix = strdup(optarg); break; case OPT_SOCKET_MEM_NUM: @@ -601,6 +608,10 @@ eal_parse_args(int argc, char **argv) internal_config.create_uio_dev = 1; break; + case OPT_MBUF_POOL_OPS_NAME_NUM: + internal_config.mbuf_pool_ops_name = optarg; + break; + default: if (opt < OPT_LONG_MIN_NUM && isprint(opt)) { RTE_LOG(ERR, EAL, "Option %c is not supported " @@ -632,15 +643,6 @@ eal_parse_args(int argc, char **argv) goto out; } - /* --xen-dom0 doesn't make sense with --socket-mem */ - if (internal_config.xen_dom0_support && internal_config.force_sockets == 1) { - RTE_LOG(ERR, EAL, "Options --"OPT_SOCKET_MEM" cannot be specified " - "together with --"OPT_XEN_DOM0"\n"); - eal_usage(prgname); - ret = -1; - goto out; - } - if (optind >= 0) argv[optind-1] = prgname; ret = optind-1; @@ -707,10 +709,9 @@ static int rte_eal_vfio_setup(void) { int vfio_enabled = 0; - if (!internal_config.no_pci) { - pci_vfio_enable(); - vfio_enabled |= pci_vfio_is_enabled(); - } + if (rte_vfio_enable("vfio")) + return -1; + vfio_enabled = rte_vfio_is_enabled("vfio"); if (vfio_enabled) { @@ -729,6 +730,12 @@ static int rte_eal_vfio_setup(void) } #endif +static void rte_eal_init_alert(const char *msg) +{ + fprintf(stderr, "EAL: FATAL: %s\n", msg); + RTE_LOG(ERR, EAL, "%s\n", msg); +} + /* Launch threads, called at application init(). */ int rte_eal_init(int argc, char **argv) @@ -740,34 +747,83 @@ rte_eal_init(int argc, char **argv) char cpuset[RTE_CPU_AFFINITY_STR_LEN]; char thread_name[RTE_MAX_THREAD_NAME_LEN]; - if (!rte_atomic32_test_and_set(&run_once)) + /* checks if the machine is adequate */ + if (!rte_cpu_is_supported()) { + rte_eal_init_alert("unsupported cpu type."); + rte_errno = ENOTSUP; return -1; + } + + if (!rte_atomic32_test_and_set(&run_once)) { + rte_eal_init_alert("already called initialization."); + rte_errno = EALREADY; + return -1; + } logid = strrchr(argv[0], '/'); logid = strdup(logid ? logid + 1: argv[0]); thread_id = pthread_self(); - if (rte_eal_log_early_init() < 0) - rte_panic("Cannot init early logs\n"); - - eal_log_level_parse(argc, argv); + eal_reset_internal_config(&internal_config); /* set log level as early as possible */ - rte_set_log_level(internal_config.log_level); + eal_log_level_parse(argc, argv); - if (rte_eal_cpu_init() < 0) - rte_panic("Cannot detect lcores\n"); + if (rte_eal_cpu_init() < 0) { + rte_eal_init_alert("Cannot detect lcores."); + rte_errno = ENOTSUP; + return -1; + } fctret = eal_parse_args(argc, argv); - if (fctret < 0) - exit(1); + if (fctret < 0) { + rte_eal_init_alert("Invalid 'command line' arguments."); + rte_errno = EINVAL; + rte_atomic32_clear(&run_once); + return -1; + } + + if (eal_plugins_init() < 0) { + rte_eal_init_alert("Cannot init plugins\n"); + rte_errno = EINVAL; + rte_atomic32_clear(&run_once); + return -1; + } + + if (eal_option_device_parse()) { + rte_errno = ENODEV; + rte_atomic32_clear(&run_once); + return -1; + } + + if (rte_bus_scan()) { + rte_eal_init_alert("Cannot scan the buses for devices\n"); + rte_errno = ENODEV; + rte_atomic32_clear(&run_once); + return -1; + } + + /* autodetect the iova mapping mode (default is iova_pa) */ + rte_eal_get_configuration()->iova_mode = rte_bus_get_iommu_class(); + + /* Workaround for KNI which requires physical address to work */ + if (rte_eal_get_configuration()->iova_mode == RTE_IOVA_VA && + rte_eal_check_module("rte_kni") == 1) { + rte_eal_get_configuration()->iova_mode = RTE_IOVA_PA; + RTE_LOG(WARNING, EAL, + "Some devices want IOVA as VA but PA will be used because.. " + "KNI module inserted\n"); + } if (internal_config.no_hugetlbfs == 0 && internal_config.process_type != RTE_PROC_SECONDARY && - internal_config.xen_dom0_support == 0 && - eal_hugepage_info_init() < 0) - rte_panic("Cannot get hugepage information\n"); + eal_hugepage_info_init() < 0) { + rte_eal_init_alert("Cannot get hugepage information."); + rte_errno = EACCES; + rte_atomic32_clear(&run_once); + return -1; + } if (internal_config.memory == 0 && internal_config.force_sockets == 0) { if (internal_config.no_hugetlbfs) @@ -789,50 +845,57 @@ rte_eal_init(int argc, char **argv) rte_config_init(); - if (rte_eal_pci_init() < 0) - rte_panic("Cannot init PCI\n"); + if (rte_eal_log_init(logid, internal_config.syslog_facility) < 0) { + rte_eal_init_alert("Cannot init logging."); + rte_errno = ENOMEM; + rte_atomic32_clear(&run_once); + return -1; + } #ifdef VFIO_PRESENT - if (rte_eal_vfio_setup() < 0) - rte_panic("Cannot init VFIO\n"); + if (rte_eal_vfio_setup() < 0) { + rte_eal_init_alert("Cannot init VFIO\n"); + rte_errno = EAGAIN; + rte_atomic32_clear(&run_once); + return -1; + } #endif -#ifdef RTE_LIBRTE_IVSHMEM - if (rte_eal_ivshmem_init() < 0) - rte_panic("Cannot init IVSHMEM\n"); -#endif - - if (rte_eal_memory_init() < 0) - rte_panic("Cannot init memory\n"); + if (rte_eal_memory_init() < 0) { + rte_eal_init_alert("Cannot init memory\n"); + rte_errno = ENOMEM; + return -1; + } /* the directories are locked during eal_hugepage_info_init */ eal_hugedirs_unlock(); - if (rte_eal_memzone_init() < 0) - rte_panic("Cannot init memzone\n"); + if (rte_eal_memzone_init() < 0) { + rte_eal_init_alert("Cannot init memzone\n"); + rte_errno = ENODEV; + return -1; + } - if (rte_eal_tailqs_init() < 0) - rte_panic("Cannot init tail queues for objects\n"); + if (rte_eal_tailqs_init() < 0) { + rte_eal_init_alert("Cannot init tail queues for objects\n"); + rte_errno = EFAULT; + return -1; + } -#ifdef RTE_LIBRTE_IVSHMEM - if (rte_eal_ivshmem_obj_init() < 0) - rte_panic("Cannot init IVSHMEM objects\n"); -#endif + if (rte_eal_alarm_init() < 0) { + rte_eal_init_alert("Cannot init interrupt-handling thread\n"); + /* rte_eal_alarm_init sets rte_errno on failure. */ + return -1; + } - if (rte_eal_log_init(logid, internal_config.syslog_facility) < 0) - rte_panic("Cannot init logs\n"); - - if (rte_eal_alarm_init() < 0) - rte_panic("Cannot init interrupt-handling thread\n"); - - if (rte_eal_timer_init() < 0) - rte_panic("Cannot init HPET or TSC timers\n"); + if (rte_eal_timer_init() < 0) { + rte_eal_init_alert("Cannot init HPET or TSC timers\n"); + rte_errno = ENOTSUP; + return -1; + } eal_check_mem_on_local_socket(); - if (eal_plugins_init() < 0) - rte_panic("Cannot init plugins\n"); - eal_thread_init_master(rte_config.master_lcore); ret = eal_thread_dump_affinity(cpuset, RTE_CPU_AFFINITY_STR_LEN); @@ -841,11 +904,10 @@ rte_eal_init(int argc, char **argv) rte_config.master_lcore, (int)thread_id, cpuset, ret == 0 ? "" : "..."); - if (rte_eal_dev_init() < 0) - rte_panic("Cannot init pmd devices\n"); - - if (rte_eal_intr_init() < 0) - rte_panic("Cannot init interrupt-handling thread\n"); + if (rte_eal_intr_init() < 0) { + rte_eal_init_alert("Cannot init interrupt-handling thread\n"); + return -1; + } RTE_LCORE_FOREACH_SLAVE(i) { @@ -883,9 +945,29 @@ rte_eal_init(int argc, char **argv) rte_eal_mp_remote_launch(sync_func, NULL, SKIP_MASTER); rte_eal_mp_wait_lcore(); - /* Probe & Initialize PCI devices */ - if (rte_eal_pci_probe()) - rte_panic("Cannot probe PCI\n"); + /* initialize services so vdevs register service during bus_probe. */ + ret = rte_service_init(); + if (ret) { + rte_eal_init_alert("rte_service_init() failed\n"); + rte_errno = ENOEXEC; + return -1; + } + + /* Probe all the buses and devices/drivers on them */ + if (rte_bus_probe()) { + rte_eal_init_alert("Cannot probe devices\n"); + rte_errno = ENOTSUP; + return -1; + } + + /* initialize default service/lcore mappings and start running. Ignore + * -ENOTSUP, as it indicates no service coremask passed to EAL. + */ + ret = rte_service_start_with_defaults(); + if (ret < 0 && ret != -ENOTSUP) { + rte_errno = ENOEXEC; + return -1; + } rte_eal_mcfg_complete(); @@ -910,6 +992,22 @@ int rte_eal_has_hugepages(void) return ! internal_config.no_hugetlbfs; } +int rte_eal_has_pci(void) +{ + return !internal_config.no_pci; +} + +int rte_eal_create_uio_dev(void) +{ + return internal_config.create_uio_dev; +} + +enum rte_intr_mode +rte_eal_vfio_intr_mode(void) +{ + return internal_config.vfio_intr_mode; +} + int rte_eal_check_module(const char *module_name) { diff --git a/dpdk/lib/librte_eal/linuxapp/eal/eal_alarm.c b/dpdk/lib/librte_eal/linuxapp/eal/eal_alarm.c index 8b042abc..8e4a775b 100644 --- a/dpdk/lib/librte_eal/linuxapp/eal/eal_alarm.c +++ b/dpdk/lib/librte_eal/linuxapp/eal/eal_alarm.c @@ -40,7 +40,6 @@ #include #include -#include #include #include #include @@ -83,7 +82,7 @@ static rte_spinlock_t alarm_list_lk = RTE_SPINLOCK_INITIALIZER; static struct rte_intr_handle intr_handle = {.fd = -1 }; static int handler_registered = 0; -static void eal_alarm_callback(struct rte_intr_handle *hdl, void *arg); +static void eal_alarm_callback(void *arg); int rte_eal_alarm_init(void) @@ -102,8 +101,7 @@ error: } static void -eal_alarm_callback(struct rte_intr_handle *hdl __rte_unused, - void *arg __rte_unused) +eal_alarm_callback(void *arg __rte_unused) { struct timespec now; struct alarm_entry *ap; diff --git a/dpdk/lib/librte_eal/linuxapp/eal/eal_debug.c b/dpdk/lib/librte_eal/linuxapp/eal/eal_debug.c index 5fbc17c5..e1c75548 100644 --- a/dpdk/lib/librte_eal/linuxapp/eal/eal_debug.c +++ b/dpdk/lib/librte_eal/linuxapp/eal/eal_debug.c @@ -31,7 +31,9 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +#ifdef RTE_BACKTRACE #include +#endif #include #include #include @@ -47,6 +49,7 @@ /* dump the stack of the calling core */ void rte_dump_stack(void) { +#ifdef RTE_BACKTRACE void *func[BACKTRACE_SIZE]; char **symb = NULL; int size; @@ -64,6 +67,7 @@ void rte_dump_stack(void) } free(symb); +#endif /* RTE_BACKTRACE */ } /* not implemented in this environment */ diff --git a/dpdk/lib/librte_eal/linuxapp/eal/eal_hugepage_info.c b/dpdk/lib/librte_eal/linuxapp/eal/eal_hugepage_info.c index 18858e2d..86e174fc 100644 --- a/dpdk/lib/librte_eal/linuxapp/eal/eal_hugepage_info.c +++ b/dpdk/lib/librte_eal/linuxapp/eal/eal_hugepage_info.c @@ -46,7 +46,6 @@ #include #include -#include #include #include #include @@ -283,9 +282,12 @@ eal_hugepage_info_init(void) struct dirent *dirent; dir = opendir(sys_dir_path); - if (dir == NULL) - rte_panic("Cannot open directory %s to read system hugepage " - "info\n", sys_dir_path); + if (dir == NULL) { + RTE_LOG(ERR, EAL, + "Cannot open directory %s to read system hugepage info\n", + sys_dir_path); + return -1; + } for (dirent = readdir(dir); dirent != NULL; dirent = readdir(dir)) { struct hugepage_info *hpi; diff --git a/dpdk/lib/librte_eal/linuxapp/eal/eal_interrupts.c b/dpdk/lib/librte_eal/linuxapp/eal/eal_interrupts.c index 47a3b20a..1c20693d 100644 --- a/dpdk/lib/librte_eal/linuxapp/eal/eal_interrupts.c +++ b/dpdk/lib/librte_eal/linuxapp/eal/eal_interrupts.c @@ -46,11 +46,11 @@ #include #include #include +#include #include #include #include -#include #include #include #include @@ -59,10 +59,10 @@ #include #include #include -#include #include #include #include +#include #include "eal_private.h" #include "eal_vfio.h" @@ -136,7 +136,7 @@ static pthread_t intr_thread; /* enable legacy (INTx) interrupts */ static int -vfio_enable_intx(struct rte_intr_handle *intr_handle) { +vfio_enable_intx(const struct rte_intr_handle *intr_handle) { struct vfio_irq_set *irq_set; char irq_set_buf[IRQ_SET_BUF_LEN]; int len, ret; @@ -183,7 +183,7 @@ vfio_enable_intx(struct rte_intr_handle *intr_handle) { /* disable legacy (INTx) interrupts */ static int -vfio_disable_intx(struct rte_intr_handle *intr_handle) { +vfio_disable_intx(const struct rte_intr_handle *intr_handle) { struct vfio_irq_set *irq_set; char irq_set_buf[IRQ_SET_BUF_LEN]; int len, ret; @@ -194,14 +194,14 @@ vfio_disable_intx(struct rte_intr_handle *intr_handle) { irq_set = (struct vfio_irq_set *) irq_set_buf; irq_set->argsz = len; irq_set->count = 1; - irq_set->flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_UNMASK; + irq_set->flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_MASK; irq_set->index = VFIO_PCI_INTX_IRQ_INDEX; irq_set->start = 0; ret = ioctl(intr_handle->vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set); if (ret) { - RTE_LOG(ERR, EAL, "Error unmasking INTx interrupts for fd %d\n", + RTE_LOG(ERR, EAL, "Error masking INTx interrupts for fd %d\n", intr_handle->fd); return -1; } @@ -226,7 +226,7 @@ vfio_disable_intx(struct rte_intr_handle *intr_handle) { /* enable MSI interrupts */ static int -vfio_enable_msi(struct rte_intr_handle *intr_handle) { +vfio_enable_msi(const struct rte_intr_handle *intr_handle) { int len, ret; char irq_set_buf[IRQ_SET_BUF_LEN]; struct vfio_irq_set *irq_set; @@ -255,7 +255,7 @@ vfio_enable_msi(struct rte_intr_handle *intr_handle) { /* disable MSI interrupts */ static int -vfio_disable_msi(struct rte_intr_handle *intr_handle) { +vfio_disable_msi(const struct rte_intr_handle *intr_handle) { struct vfio_irq_set *irq_set; char irq_set_buf[IRQ_SET_BUF_LEN]; int len, ret; @@ -280,7 +280,7 @@ vfio_disable_msi(struct rte_intr_handle *intr_handle) { /* enable MSI-X interrupts */ static int -vfio_enable_msix(struct rte_intr_handle *intr_handle) { +vfio_enable_msix(const struct rte_intr_handle *intr_handle) { int len, ret; char irq_set_buf[MSIX_IRQ_SET_BUF_LEN]; struct vfio_irq_set *irq_set; @@ -290,12 +290,10 @@ vfio_enable_msix(struct rte_intr_handle *intr_handle) { irq_set = (struct vfio_irq_set *) irq_set_buf; irq_set->argsz = len; - if (!intr_handle->max_intr) - intr_handle->max_intr = 1; - else if (intr_handle->max_intr > RTE_MAX_RXTX_INTR_VEC_ID) - intr_handle->max_intr = RTE_MAX_RXTX_INTR_VEC_ID + 1; - - irq_set->count = intr_handle->max_intr; + /* 0 < irq_set->count < RTE_MAX_RXTX_INTR_VEC_ID + 1 */ + irq_set->count = intr_handle->max_intr ? + (intr_handle->max_intr > RTE_MAX_RXTX_INTR_VEC_ID + 1 ? + RTE_MAX_RXTX_INTR_VEC_ID + 1 : intr_handle->max_intr) : 1; irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER; irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX; irq_set->start = 0; @@ -318,7 +316,7 @@ vfio_enable_msix(struct rte_intr_handle *intr_handle) { /* disable MSI-X interrupts */ static int -vfio_disable_msix(struct rte_intr_handle *intr_handle) { +vfio_disable_msix(const struct rte_intr_handle *intr_handle) { struct vfio_irq_set *irq_set; char irq_set_buf[MSIX_IRQ_SET_BUF_LEN]; int len, ret; @@ -343,7 +341,7 @@ vfio_disable_msix(struct rte_intr_handle *intr_handle) { #endif static int -uio_intx_intr_disable(struct rte_intr_handle *intr_handle) +uio_intx_intr_disable(const struct rte_intr_handle *intr_handle) { unsigned char command_high; @@ -367,7 +365,7 @@ uio_intx_intr_disable(struct rte_intr_handle *intr_handle) } static int -uio_intx_intr_enable(struct rte_intr_handle *intr_handle) +uio_intx_intr_enable(const struct rte_intr_handle *intr_handle) { unsigned char command_high; @@ -391,7 +389,7 @@ uio_intx_intr_enable(struct rte_intr_handle *intr_handle) } static int -uio_intr_disable(struct rte_intr_handle *intr_handle) +uio_intr_disable(const struct rte_intr_handle *intr_handle) { const int value = 0; @@ -405,7 +403,7 @@ uio_intr_disable(struct rte_intr_handle *intr_handle) } static int -uio_intr_enable(struct rte_intr_handle *intr_handle) +uio_intr_enable(const struct rte_intr_handle *intr_handle) { const int value = 1; @@ -419,7 +417,7 @@ uio_intr_enable(struct rte_intr_handle *intr_handle) } int -rte_intr_callback_register(struct rte_intr_handle *intr_handle, +rte_intr_callback_register(const struct rte_intr_handle *intr_handle, rte_intr_callback_fn cb, void *cb_arg) { int ret, wake_thread; @@ -491,7 +489,7 @@ rte_intr_callback_register(struct rte_intr_handle *intr_handle, } int -rte_intr_callback_unregister(struct rte_intr_handle *intr_handle, +rte_intr_callback_unregister(const struct rte_intr_handle *intr_handle, rte_intr_callback_fn cb_fn, void *cb_arg) { int ret; @@ -555,8 +553,11 @@ rte_intr_callback_unregister(struct rte_intr_handle *intr_handle, } int -rte_intr_enable(struct rte_intr_handle *intr_handle) +rte_intr_enable(const struct rte_intr_handle *intr_handle) { + if (intr_handle && intr_handle->type == RTE_INTR_HANDLE_VDEV) + return 0; + if (!intr_handle || intr_handle->fd < 0 || intr_handle->uio_cfg_fd < 0) return -1; @@ -599,8 +600,11 @@ rte_intr_enable(struct rte_intr_handle *intr_handle) } int -rte_intr_disable(struct rte_intr_handle *intr_handle) +rte_intr_disable(const struct rte_intr_handle *intr_handle) { + if (intr_handle && intr_handle->type == RTE_INTR_HANDLE_VDEV) + return 0; + if (!intr_handle || intr_handle->fd < 0 || intr_handle->uio_cfg_fd < 0) return -1; @@ -645,6 +649,7 @@ rte_intr_disable(struct rte_intr_handle *intr_handle) static int eal_intr_process_interrupts(struct epoll_event *events, int nfds) { + bool call = false; int n, bytes_read; struct rte_intr_source *src; struct rte_intr_callback *cb; @@ -693,13 +698,18 @@ eal_intr_process_interrupts(struct epoll_event *events, int nfds) bytes_read = sizeof(buf.vfio_intr_count); break; #endif + case RTE_INTR_HANDLE_VDEV: case RTE_INTR_HANDLE_EXT: + bytes_read = 0; + call = true; + break; + default: bytes_read = 1; break; } - if (src->intr_handle.type != RTE_INTR_HANDLE_EXT) { + if (bytes_read > 0) { /** * read out to clear the ready-to-be-read flag * for epoll_wait. @@ -716,12 +726,14 @@ eal_intr_process_interrupts(struct epoll_event *events, int nfds) } else if (bytes_read == 0) RTE_LOG(ERR, EAL, "Read nothing from file " "descriptor %d\n", events[n].data.fd); + else + call = true; } /* grab a lock, again to call callbacks and update status. */ rte_spinlock_lock(&intr_lock); - if (bytes_read > 0) { + if (call) { /* Finally, call all callbacks. */ TAILQ_FOREACH(cb, &src->callbacks, next) { @@ -731,8 +743,7 @@ eal_intr_process_interrupts(struct epoll_event *events, int nfds) rte_spinlock_unlock(&intr_lock); /* call the actual callback */ - active_cb.cb_fn(&src->intr_handle, - active_cb.cb_arg); + active_cb.cb_fn(active_cb.cb_arg); /*get the lock back. */ rte_spinlock_lock(&intr_lock); @@ -832,7 +843,7 @@ eal_intr_thread_main(__rte_unused void *arg) TAILQ_FOREACH(src, &intr_sources, next) { if (src->callbacks.tqh_first == NULL) continue; /* skip those with no callbacks */ - ev.events = EPOLLIN | EPOLLPRI; + ev.events = EPOLLIN | EPOLLPRI | EPOLLRDHUP | EPOLLHUP; ev.data.fd = src->intr_handle.fd; /** @@ -872,13 +883,16 @@ rte_eal_intr_init(void) * create a pipe which will be waited by epoll and notified to * rebuild the wait list of epoll. */ - if (pipe(intr_pipe.pipefd) < 0) + if (pipe(intr_pipe.pipefd) < 0) { + rte_errno = errno; return -1; + } /* create the host thread to wait/handle the interrupt */ ret = pthread_create(&intr_thread, NULL, eal_intr_thread_main, NULL); if (ret != 0) { + rte_errno = ret; RTE_LOG(ERR, EAL, "Failed to create thread for interrupt handling\n"); } else { @@ -898,7 +912,7 @@ static void eal_intr_proc_rxtx_intr(int fd, const struct rte_intr_handle *intr_handle) { union rte_intr_read_buffer buf; - int bytes_read = 1; + int bytes_read = 0; int nbytes; switch (intr_handle->type) { @@ -913,6 +927,12 @@ eal_intr_proc_rxtx_intr(int fd, const struct rte_intr_handle *intr_handle) bytes_read = sizeof(buf.vfio_intr_count); break; #endif + case RTE_INTR_HANDLE_VDEV: + bytes_read = intr_handle->efd_counter_size; + /* For vdev, number of bytes to read is set by driver */ + break; + case RTE_INTR_HANDLE_EXT: + return; default: bytes_read = 1; RTE_LOG(INFO, EAL, "unexpected intr type\n"); @@ -923,6 +943,8 @@ eal_intr_proc_rxtx_intr(int fd, const struct rte_intr_handle *intr_handle) * read out to clear the ready-to-be-read flag * for epoll_wait. */ + if (bytes_read == 0) + return; do { nbytes = read(fd, &buf, bytes_read); if (nbytes < 0) { @@ -1141,6 +1163,24 @@ rte_intr_rx_ctl(struct rte_intr_handle *intr_handle, int epfd, return rc; } +void +rte_intr_free_epoll_fd(struct rte_intr_handle *intr_handle) +{ + uint32_t i; + struct rte_epoll_event *rev; + + for (i = 0; i < intr_handle->nb_efd; i++) { + rev = &intr_handle->elist[i]; + if (rev->status == RTE_EPOLL_INVALID) + continue; + if (rte_epoll_ctl(rev->epfd, EPOLL_CTL_DEL, rev->fd, rev)) { + /* force free if the entry valid */ + eal_epoll_data_safe_free(rev); + rev->status = RTE_EPOLL_INVALID; + } + } +} + int rte_intr_efd_enable(struct rte_intr_handle *intr_handle, uint32_t nb_efd) { @@ -1157,12 +1197,19 @@ rte_intr_efd_enable(struct rte_intr_handle *intr_handle, uint32_t nb_efd) RTE_LOG(ERR, EAL, "can't setup eventfd, error %i (%s)\n", errno, strerror(errno)); - return -1; + return -errno; } intr_handle->efds[i] = fd; } intr_handle->nb_efd = n; intr_handle->max_intr = NB_OTHER_INTR + n; + } else if (intr_handle->type == RTE_INTR_HANDLE_VDEV) { + /* only check, initialization would be done in vdev driver.*/ + if (intr_handle->efd_counter_size > + sizeof(union rte_intr_read_buffer)) { + RTE_LOG(ERR, EAL, "the efd_counter_size is oversized"); + return -EINVAL; + } } else { intr_handle->efds[0] = intr_handle->fd; intr_handle->nb_efd = RTE_MIN(nb_efd, 1U); @@ -1176,19 +1223,8 @@ void rte_intr_efd_disable(struct rte_intr_handle *intr_handle) { uint32_t i; - struct rte_epoll_event *rev; - - for (i = 0; i < intr_handle->nb_efd; i++) { - rev = &intr_handle->elist[i]; - if (rev->status == RTE_EPOLL_INVALID) - continue; - if (rte_epoll_ctl(rev->epfd, EPOLL_CTL_DEL, rev->fd, rev)) { - /* force free if the entry valid */ - eal_epoll_data_safe_free(rev); - rev->status = RTE_EPOLL_INVALID; - } - } + rte_intr_free_epoll_fd(intr_handle); if (intr_handle->max_intr > intr_handle->nb_efd) { for (i = 0; i < intr_handle->nb_efd; i++) close(intr_handle->efds[i]); @@ -1218,5 +1254,8 @@ rte_intr_cap_multiple(struct rte_intr_handle *intr_handle) if (intr_handle->type == RTE_INTR_HANDLE_VFIO_MSIX) return 1; + if (intr_handle->type == RTE_INTR_HANDLE_VDEV) + return 1; + return 0; } diff --git a/dpdk/lib/librte_eal/linuxapp/eal/eal_ivshmem.c b/dpdk/lib/librte_eal/linuxapp/eal/eal_ivshmem.c deleted file mode 100644 index 67b3caf2..00000000 --- a/dpdk/lib/librte_eal/linuxapp/eal/eal_ivshmem.c +++ /dev/null @@ -1,954 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifdef RTE_LIBRTE_IVSHMEM /* hide it from coverage */ - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "eal_internal_cfg.h" -#include "eal_private.h" - -#define PCI_VENDOR_ID_IVSHMEM 0x1Af4 -#define PCI_DEVICE_ID_IVSHMEM 0x1110 - -#define IVSHMEM_MAGIC 0x0BADC0DE - -#define IVSHMEM_RESOURCE_PATH "/sys/bus/pci/devices/%04x:%02x:%02x.%x/resource2" -#define IVSHMEM_CONFIG_PATH "/var/run/.%s_ivshmem_config" - -#define PHYS 0x1 -#define VIRT 0x2 -#define IOREMAP 0x4 -#define FULL (PHYS|VIRT|IOREMAP) - -#define METADATA_SIZE_ALIGNED \ - (RTE_ALIGN_CEIL(sizeof(struct rte_ivshmem_metadata),pagesz)) - -#define CONTAINS(x,y)\ - (((y).addr_64 >= (x).addr_64) && ((y).addr_64 < (x).addr_64 + (x).len)) - -#define DIM(x) (sizeof(x)/sizeof(x[0])) - -struct ivshmem_pci_device { - char path[PATH_MAX]; - phys_addr_t ioremap_addr; -}; - -/* data type to store in config */ -struct ivshmem_segment { - struct rte_ivshmem_metadata_entry entry; - uint64_t align; - char path[PATH_MAX]; -}; -struct ivshmem_shared_config { - struct ivshmem_segment segment[RTE_MAX_MEMSEG]; - uint32_t segment_idx; - struct ivshmem_pci_device pci_devs[RTE_LIBRTE_IVSHMEM_MAX_PCI_DEVS]; - uint32_t pci_devs_idx; -}; -static struct ivshmem_shared_config * ivshmem_config; -static int memseg_idx; -static int pagesz; - -/* Tailq heads to add rings to */ -TAILQ_HEAD(rte_ring_list, rte_tailq_entry); - -/* - * Utility functions - */ - -static int -is_ivshmem_device(struct rte_pci_device * dev) -{ - return dev->id.vendor_id == PCI_VENDOR_ID_IVSHMEM - && dev->id.device_id == PCI_DEVICE_ID_IVSHMEM; -} - -static void * -map_metadata(int fd, uint64_t len) -{ - size_t metadata_len = sizeof(struct rte_ivshmem_metadata); - size_t aligned_len = METADATA_SIZE_ALIGNED; - - return mmap(NULL, metadata_len, PROT_READ | PROT_WRITE, - MAP_SHARED, fd, len - aligned_len); -} - -static void -unmap_metadata(void * ptr) -{ - munmap(ptr, sizeof(struct rte_ivshmem_metadata)); -} - -static int -has_ivshmem_metadata(int fd, uint64_t len) -{ - struct rte_ivshmem_metadata metadata; - void * ptr; - - ptr = map_metadata(fd, len); - - if (ptr == MAP_FAILED) - return -1; - - metadata = *(struct rte_ivshmem_metadata*) (ptr); - - unmap_metadata(ptr); - - return metadata.magic_number == IVSHMEM_MAGIC; -} - -static void -remove_segment(struct ivshmem_segment * ms, int len, int idx) -{ - int i; - - for (i = idx; i < len - 1; i++) - memcpy(&ms[i], &ms[i+1], sizeof(struct ivshmem_segment)); - memset(&ms[len-1], 0, sizeof(struct ivshmem_segment)); -} - -static int -overlap(const struct rte_memzone * mz1, const struct rte_memzone * mz2) -{ - uint64_t start1, end1, start2, end2; - uint64_t p_start1, p_end1, p_start2, p_end2; - uint64_t i_start1, i_end1, i_start2, i_end2; - int result = 0; - - /* gather virtual addresses */ - start1 = mz1->addr_64; - end1 = mz1->addr_64 + mz1->len; - start2 = mz2->addr_64; - end2 = mz2->addr_64 + mz2->len; - - /* gather physical addresses */ - p_start1 = mz1->phys_addr; - p_end1 = mz1->phys_addr + mz1->len; - p_start2 = mz2->phys_addr; - p_end2 = mz2->phys_addr + mz2->len; - - /* gather ioremap addresses */ - i_start1 = mz1->ioremap_addr; - i_end1 = mz1->ioremap_addr + mz1->len; - i_start2 = mz2->ioremap_addr; - i_end2 = mz2->ioremap_addr + mz2->len; - - /* check for overlap in virtual addresses */ - if (start1 >= start2 && start1 < end2) - result |= VIRT; - if (start2 >= start1 && start2 < end1) - result |= VIRT; - - /* check for overlap in physical addresses */ - if (p_start1 >= p_start2 && p_start1 < p_end2) - result |= PHYS; - if (p_start2 >= p_start1 && p_start2 < p_end1) - result |= PHYS; - - /* check for overlap in ioremap addresses */ - if (i_start1 >= i_start2 && i_start1 < i_end2) - result |= IOREMAP; - if (i_start2 >= i_start1 && i_start2 < i_end1) - result |= IOREMAP; - - return result; -} - -static int -adjacent(const struct rte_memzone * mz1, const struct rte_memzone * mz2) -{ - uint64_t start1, end1, start2, end2; - uint64_t p_start1, p_end1, p_start2, p_end2; - uint64_t i_start1, i_end1, i_start2, i_end2; - int result = 0; - - /* gather virtual addresses */ - start1 = mz1->addr_64; - end1 = mz1->addr_64 + mz1->len; - start2 = mz2->addr_64; - end2 = mz2->addr_64 + mz2->len; - - /* gather physical addresses */ - p_start1 = mz1->phys_addr; - p_end1 = mz1->phys_addr + mz1->len; - p_start2 = mz2->phys_addr; - p_end2 = mz2->phys_addr + mz2->len; - - /* gather ioremap addresses */ - i_start1 = mz1->ioremap_addr; - i_end1 = mz1->ioremap_addr + mz1->len; - i_start2 = mz2->ioremap_addr; - i_end2 = mz2->ioremap_addr + mz2->len; - - /* check if segments are virtually adjacent */ - if (start1 == end2) - result |= VIRT; - if (start2 == end1) - result |= VIRT; - - /* check if segments are physically adjacent */ - if (p_start1 == p_end2) - result |= PHYS; - if (p_start2 == p_end1) - result |= PHYS; - - /* check if segments are ioremap-adjacent */ - if (i_start1 == i_end2) - result |= IOREMAP; - if (i_start2 == i_end1) - result |= IOREMAP; - - return result; -} - -static int -has_adjacent_segments(struct ivshmem_segment * ms, int len) -{ - int i, j; - - for (i = 0; i < len; i++) - for (j = i + 1; j < len; j++) { - /* we're only interested in fully adjacent segments; partially - * adjacent segments can coexist. - */ - if (adjacent(&ms[i].entry.mz, &ms[j].entry.mz) == FULL) - return 1; - } - return 0; -} - -static int -has_overlapping_segments(struct ivshmem_segment * ms, int len) -{ - int i, j; - - for (i = 0; i < len; i++) - for (j = i + 1; j < len; j++) - if (overlap(&ms[i].entry.mz, &ms[j].entry.mz)) - return 1; - return 0; -} - -static int -seg_compare(const void * a, const void * b) -{ - const struct ivshmem_segment * s1 = (const struct ivshmem_segment*) a; - const struct ivshmem_segment * s2 = (const struct ivshmem_segment*) b; - - /* move unallocated zones to the end */ - if (s1->entry.mz.addr == NULL && s2->entry.mz.addr == NULL) - return 0; - if (s1->entry.mz.addr == 0) - return 1; - if (s2->entry.mz.addr == 0) - return -1; - - return s1->entry.mz.phys_addr > s2->entry.mz.phys_addr; -} - -#ifdef RTE_LIBRTE_IVSHMEM_DEBUG -static void -entry_dump(struct rte_ivshmem_metadata_entry *e) -{ - RTE_LOG(DEBUG, EAL, "\tvirt: %p-%p\n", e->mz.addr, - RTE_PTR_ADD(e->mz.addr, e->mz.len)); - RTE_LOG(DEBUG, EAL, "\tphys: 0x%" PRIx64 "-0x%" PRIx64 "\n", - e->mz.phys_addr, - e->mz.phys_addr + e->mz.len); - RTE_LOG(DEBUG, EAL, "\tio: 0x%" PRIx64 "-0x%" PRIx64 "\n", - e->mz.ioremap_addr, - e->mz.ioremap_addr + e->mz.len); - RTE_LOG(DEBUG, EAL, "\tlen: 0x%" PRIx64 "\n", e->mz.len); - RTE_LOG(DEBUG, EAL, "\toff: 0x%" PRIx64 "\n", e->offset); -} -#endif - - - -/* - * Actual useful code - */ - -/* read through metadata mapped from the IVSHMEM device */ -static int -read_metadata(char * path, int path_len, int fd, uint64_t flen) -{ - struct rte_ivshmem_metadata metadata; - struct rte_ivshmem_metadata_entry * entry; - int idx, i; - void * ptr; - - ptr = map_metadata(fd, flen); - - if (ptr == MAP_FAILED) - return -1; - - metadata = *(struct rte_ivshmem_metadata*) (ptr); - - unmap_metadata(ptr); - - RTE_LOG(DEBUG, EAL, "Parsing metadata for \"%s\"\n", metadata.name); - - idx = ivshmem_config->segment_idx; - - for (i = 0; i < RTE_LIBRTE_IVSHMEM_MAX_ENTRIES && - idx <= RTE_MAX_MEMSEG; i++) { - - if (idx == RTE_MAX_MEMSEG) { - RTE_LOG(ERR, EAL, "Not enough memory segments!\n"); - return -1; - } - - entry = &metadata.entry[i]; - - /* stop on uninitialized memzone */ - if (entry->mz.len == 0) - break; - - /* copy metadata entry */ - memcpy(&ivshmem_config->segment[idx].entry, entry, - sizeof(struct rte_ivshmem_metadata_entry)); - - /* copy path */ - snprintf(ivshmem_config->segment[idx].path, path_len, "%s", path); - - idx++; - } - ivshmem_config->segment_idx = idx; - - return 0; -} - -/* check through each segment and look for adjacent or overlapping ones. */ -static int -cleanup_segments(struct ivshmem_segment * ms, int tbl_len) -{ - struct ivshmem_segment * s, * tmp; - int i, j, concat, seg_adjacent, seg_overlapping; - uint64_t start1, start2, end1, end2, p_start1, p_start2, i_start1, i_start2; - - qsort(ms, tbl_len, sizeof(struct ivshmem_segment), - seg_compare); - - while (has_overlapping_segments(ms, tbl_len) || - has_adjacent_segments(ms, tbl_len)) { - - for (i = 0; i < tbl_len; i++) { - s = &ms[i]; - - concat = 0; - - for (j = i + 1; j < tbl_len; j++) { - tmp = &ms[j]; - - /* check if this segment is overlapping with existing segment, - * or is adjacent to existing segment */ - seg_overlapping = overlap(&s->entry.mz, &tmp->entry.mz); - seg_adjacent = adjacent(&s->entry.mz, &tmp->entry.mz); - - /* check if segments fully overlap or are fully adjacent */ - if ((seg_adjacent == FULL) || (seg_overlapping == FULL)) { - -#ifdef RTE_LIBRTE_IVSHMEM_DEBUG - RTE_LOG(DEBUG, EAL, "Concatenating segments\n"); - RTE_LOG(DEBUG, EAL, "Segment %i:\n", i); - entry_dump(&s->entry); - RTE_LOG(DEBUG, EAL, "Segment %i:\n", j); - entry_dump(&tmp->entry); -#endif - - start1 = s->entry.mz.addr_64; - start2 = tmp->entry.mz.addr_64; - p_start1 = s->entry.mz.phys_addr; - p_start2 = tmp->entry.mz.phys_addr; - i_start1 = s->entry.mz.ioremap_addr; - i_start2 = tmp->entry.mz.ioremap_addr; - end1 = s->entry.mz.addr_64 + s->entry.mz.len; - end2 = tmp->entry.mz.addr_64 + tmp->entry.mz.len; - - /* settle for minimum start address and maximum length */ - s->entry.mz.addr_64 = RTE_MIN(start1, start2); - s->entry.mz.phys_addr = RTE_MIN(p_start1, p_start2); - s->entry.mz.ioremap_addr = RTE_MIN(i_start1, i_start2); - s->entry.offset = RTE_MIN(s->entry.offset, tmp->entry.offset); - s->entry.mz.len = RTE_MAX(end1, end2) - s->entry.mz.addr_64; - concat = 1; - -#ifdef RTE_LIBRTE_IVSHMEM_DEBUG - RTE_LOG(DEBUG, EAL, "Resulting segment:\n"); - entry_dump(&s->entry); - -#endif - } - /* if segments not fully overlap, we have an error condition. - * adjacent segments can coexist. - */ - else if (seg_overlapping > 0) { - RTE_LOG(ERR, EAL, "Segments %i and %i overlap!\n", i, j); -#ifdef RTE_LIBRTE_IVSHMEM_DEBUG - RTE_LOG(DEBUG, EAL, "Segment %i:\n", i); - entry_dump(&s->entry); - RTE_LOG(DEBUG, EAL, "Segment %i:\n", j); - entry_dump(&tmp->entry); -#endif - return -1; - } - if (concat) - break; - } - /* if we concatenated, remove segment at j */ - if (concat) { - remove_segment(ms, tbl_len, j); - tbl_len--; - break; - } - } - } - - return tbl_len; -} - -static int -create_shared_config(void) -{ - char path[PATH_MAX]; - int fd; - - /* build ivshmem config file path */ - snprintf(path, sizeof(path), IVSHMEM_CONFIG_PATH, - internal_config.hugefile_prefix); - - fd = open(path, O_CREAT | O_RDWR, 0600); - - if (fd < 0) { - RTE_LOG(ERR, EAL, "Could not open %s: %s\n", path, strerror(errno)); - return -1; - } - - /* try ex-locking first - if the file is locked, we have a problem */ - if (flock(fd, LOCK_EX | LOCK_NB) == -1) { - RTE_LOG(ERR, EAL, "Locking %s failed: %s\n", path, strerror(errno)); - close(fd); - return -1; - } - - if (ftruncate(fd, sizeof(struct ivshmem_shared_config)) < 0) { - RTE_LOG(ERR, EAL, "ftruncate failed: %s\n", strerror(errno)); - return -1; - } - - ivshmem_config = mmap(NULL, sizeof(struct ivshmem_shared_config), - PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); - - if (ivshmem_config == MAP_FAILED) - return -1; - - memset(ivshmem_config, 0, sizeof(struct ivshmem_shared_config)); - - /* change the exclusive lock we got earlier to a shared lock */ - if (flock(fd, LOCK_SH | LOCK_NB) == -1) { - RTE_LOG(ERR, EAL, "Locking %s failed: %s \n", path, strerror(errno)); - return -1; - } - - close(fd); - - return 0; -} - -/* open shared config file and, if present, map the config. - * having no config file is not an error condition, as we later check if - * ivshmem_config is NULL (if it is, that means nothing was mapped). */ -static int -open_shared_config(void) -{ - char path[PATH_MAX]; - int fd; - - /* build ivshmem config file path */ - snprintf(path, sizeof(path), IVSHMEM_CONFIG_PATH, - internal_config.hugefile_prefix); - - fd = open(path, O_RDONLY); - - /* if the file doesn't exist, just return success */ - if (fd < 0 && errno == ENOENT) - return 0; - /* else we have an error condition */ - else if (fd < 0) { - RTE_LOG(ERR, EAL, "Could not open %s: %s\n", - path, strerror(errno)); - return -1; - } - - /* try ex-locking first - if the lock *does* succeed, this means it's a - * stray config file, so it should be deleted. - */ - if (flock(fd, LOCK_EX | LOCK_NB) != -1) { - - /* if we can't remove the file, something is wrong */ - if (unlink(path) < 0) { - RTE_LOG(ERR, EAL, "Could not remove %s: %s\n", path, - strerror(errno)); - return -1; - } - - /* release the lock */ - flock(fd, LOCK_UN); - close(fd); - - /* return success as having a stray config file is equivalent to not - * having config file at all. - */ - return 0; - } - - ivshmem_config = mmap(NULL, sizeof(struct ivshmem_shared_config), - PROT_READ, MAP_SHARED, fd, 0); - - if (ivshmem_config == MAP_FAILED) - return -1; - - /* place a shared lock on config file */ - if (flock(fd, LOCK_SH | LOCK_NB) == -1) { - RTE_LOG(ERR, EAL, "Locking %s failed: %s \n", path, strerror(errno)); - return -1; - } - - close(fd); - - return 0; -} - -/* - * This function does the following: - * - * 1) Builds a table of ivshmem_segments with proper offset alignment - * 2) Cleans up that table so that we don't have any overlapping or adjacent - * memory segments - * 3) Creates memsegs from this table and maps them into memory. - */ -static inline int -map_all_segments(void) -{ - struct ivshmem_segment ms_tbl[RTE_MAX_MEMSEG]; - struct ivshmem_pci_device * pci_dev; - struct rte_mem_config * mcfg; - struct ivshmem_segment * seg; - int fd, fd_zero; - unsigned i, j; - struct rte_memzone mz; - struct rte_memseg ms; - void * base_addr; - uint64_t align, len; - phys_addr_t ioremap_addr; - - ioremap_addr = 0; - - memset(ms_tbl, 0, sizeof(ms_tbl)); - memset(&mz, 0, sizeof(struct rte_memzone)); - memset(&ms, 0, sizeof(struct rte_memseg)); - - /* first, build a table of memsegs to map, to avoid failed mmaps due to - * overlaps - */ - for (i = 0; i < ivshmem_config->segment_idx && i <= RTE_MAX_MEMSEG; i++) { - if (i == RTE_MAX_MEMSEG) { - RTE_LOG(ERR, EAL, "Too many segments requested!\n"); - return -1; - } - - seg = &ivshmem_config->segment[i]; - - /* copy segment to table */ - memcpy(&ms_tbl[i], seg, sizeof(struct ivshmem_segment)); - - /* find ioremap addr */ - for (j = 0; j < DIM(ivshmem_config->pci_devs); j++) { - pci_dev = &ivshmem_config->pci_devs[j]; - if (!strncmp(pci_dev->path, seg->path, sizeof(pci_dev->path))) { - ioremap_addr = pci_dev->ioremap_addr; - break; - } - } - if (ioremap_addr == 0) { - RTE_LOG(ERR, EAL, "Cannot find ioremap addr!\n"); - return -1; - } - - /* work out alignments */ - align = seg->entry.mz.addr_64 - - RTE_ALIGN_FLOOR(seg->entry.mz.addr_64, 0x1000); - len = RTE_ALIGN_CEIL(seg->entry.mz.len + align, 0x1000); - - /* save original alignments */ - ms_tbl[i].align = align; - - /* create a memory zone */ - mz.addr_64 = seg->entry.mz.addr_64 - align; - mz.len = len; - mz.hugepage_sz = seg->entry.mz.hugepage_sz; - mz.phys_addr = seg->entry.mz.phys_addr - align; - - /* find true physical address */ - mz.ioremap_addr = ioremap_addr + seg->entry.offset - align; - - ms_tbl[i].entry.offset = seg->entry.offset - align; - - memcpy(&ms_tbl[i].entry.mz, &mz, sizeof(struct rte_memzone)); - } - - /* clean up the segments */ - memseg_idx = cleanup_segments(ms_tbl, ivshmem_config->segment_idx); - - if (memseg_idx < 0) - return -1; - - mcfg = rte_eal_get_configuration()->mem_config; - - fd_zero = open("/dev/zero", O_RDWR); - - if (fd_zero < 0) { - RTE_LOG(ERR, EAL, "Cannot open /dev/zero: %s\n", strerror(errno)); - return -1; - } - - /* create memsegs and put them into DPDK memory */ - for (i = 0; i < (unsigned) memseg_idx; i++) { - - seg = &ms_tbl[i]; - - ms.addr_64 = seg->entry.mz.addr_64; - ms.hugepage_sz = seg->entry.mz.hugepage_sz; - ms.len = seg->entry.mz.len; - ms.nchannel = rte_memory_get_nchannel(); - ms.nrank = rte_memory_get_nrank(); - ms.phys_addr = seg->entry.mz.phys_addr; - ms.ioremap_addr = seg->entry.mz.ioremap_addr; - ms.socket_id = seg->entry.mz.socket_id; - - base_addr = mmap(ms.addr, ms.len, - PROT_READ | PROT_WRITE, MAP_PRIVATE, fd_zero, 0); - - if (base_addr == MAP_FAILED || base_addr != ms.addr) { - RTE_LOG(ERR, EAL, "Cannot map /dev/zero!\n"); - return -1; - } - - fd = open(seg->path, O_RDWR); - - if (fd < 0) { - RTE_LOG(ERR, EAL, "Cannot open %s: %s\n", seg->path, - strerror(errno)); - return -1; - } - - munmap(ms.addr, ms.len); - - base_addr = mmap(ms.addr, ms.len, - PROT_READ | PROT_WRITE, MAP_SHARED, fd, - seg->entry.offset); - - - if (base_addr == MAP_FAILED || base_addr != ms.addr) { - RTE_LOG(ERR, EAL, "Cannot map segment into memory: " - "expected %p got %p (%s)\n", ms.addr, base_addr, - strerror(errno)); - return -1; - } - - RTE_LOG(DEBUG, EAL, "Memory segment mapped: %p (len %" PRIx64 ") at " - "offset 0x%" PRIx64 "\n", - ms.addr, ms.len, seg->entry.offset); - - /* put the pointers back into their real positions using original - * alignment */ - ms.addr_64 += seg->align; - ms.phys_addr += seg->align; - ms.ioremap_addr += seg->align; - ms.len -= seg->align; - - /* at this point, the rest of DPDK memory is not initialized, so we - * expect memsegs to be empty */ - memcpy(&mcfg->memseg[i], &ms, - sizeof(struct rte_memseg)); - - close(fd); - - RTE_LOG(DEBUG, EAL, "IVSHMEM segment found, size: 0x%lx\n", - ms.len); - } - - return 0; -} - -/* this happens at a later stage, after general EAL memory initialization */ -int -rte_eal_ivshmem_obj_init(void) -{ - struct rte_ring_list* ring_list = NULL; - struct rte_mem_config * mcfg; - struct ivshmem_segment * seg; - struct rte_memzone * mz; - struct rte_ring * r; - struct rte_tailq_entry *te; - unsigned i, ms, idx; - uint64_t offset; - - /* secondary process would not need any object discovery - it'll all - * already be in shared config */ - if (rte_eal_process_type() != RTE_PROC_PRIMARY || ivshmem_config == NULL) - return 0; - - /* check that we have an initialised ring tail queue */ - ring_list = RTE_TAILQ_LOOKUP(RTE_TAILQ_RING_NAME, rte_ring_list); - if (ring_list == NULL) { - RTE_LOG(ERR, EAL, "No rte_ring tailq found!\n"); - return -1; - } - - mcfg = rte_eal_get_configuration()->mem_config; - - /* create memzones */ - for (i = 0; i < ivshmem_config->segment_idx && i <= RTE_MAX_MEMZONE; i++) { - - seg = &ivshmem_config->segment[i]; - - /* add memzone */ - if (mcfg->memzone_cnt == RTE_MAX_MEMZONE) { - RTE_LOG(ERR, EAL, "No more memory zones available!\n"); - return -1; - } - - idx = mcfg->memzone_cnt; - - RTE_LOG(DEBUG, EAL, "Found memzone: '%s' at %p (len 0x%" PRIx64 ")\n", - seg->entry.mz.name, seg->entry.mz.addr, seg->entry.mz.len); - - memcpy(&mcfg->memzone[idx], &seg->entry.mz, - sizeof(struct rte_memzone)); - - /* find ioremap address */ - for (ms = 0; ms <= RTE_MAX_MEMSEG; ms++) { - if (ms == RTE_MAX_MEMSEG) { - RTE_LOG(ERR, EAL, "Physical address of segment not found!\n"); - return -1; - } - if (CONTAINS(mcfg->memseg[ms], mcfg->memzone[idx])) { - offset = mcfg->memzone[idx].addr_64 - - mcfg->memseg[ms].addr_64; - mcfg->memzone[idx].ioremap_addr = mcfg->memseg[ms].ioremap_addr + - offset; - break; - } - } - - mcfg->memzone_cnt++; - } - - rte_rwlock_write_lock(RTE_EAL_TAILQ_RWLOCK); - - /* find rings */ - for (i = 0; i < mcfg->memzone_cnt; i++) { - mz = &mcfg->memzone[i]; - - /* check if memzone has a ring prefix */ - if (strncmp(mz->name, RTE_RING_MZ_PREFIX, - sizeof(RTE_RING_MZ_PREFIX) - 1) != 0) - continue; - - r = (struct rte_ring*) (mz->addr_64); - - te = rte_zmalloc("RING_TAILQ_ENTRY", sizeof(*te), 0); - if (te == NULL) { - RTE_LOG(ERR, EAL, "Cannot allocate ring tailq entry!\n"); - return -1; - } - - te->data = (void *) r; - - TAILQ_INSERT_TAIL(ring_list, te, next); - - RTE_LOG(DEBUG, EAL, "Found ring: '%s' at %p\n", r->name, mz->addr); - } - rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK); - -#ifdef RTE_LIBRTE_IVSHMEM_DEBUG - rte_memzone_dump(stdout); - rte_ring_list_dump(stdout); -#endif - - return 0; -} - -/* initialize ivshmem structures */ -int rte_eal_ivshmem_init(void) -{ - struct rte_pci_device * dev; - struct rte_pci_resource * res; - int fd, ret; - char path[PATH_MAX]; - - /* initialize everything to 0 */ - memset(path, 0, sizeof(path)); - ivshmem_config = NULL; - - pagesz = getpagesize(); - - RTE_LOG(DEBUG, EAL, "Searching for IVSHMEM devices...\n"); - - if (rte_eal_process_type() == RTE_PROC_SECONDARY) { - - if (open_shared_config() < 0) { - RTE_LOG(ERR, EAL, "Could not open IVSHMEM config!\n"); - return -1; - } - } - else { - - TAILQ_FOREACH(dev, &pci_device_list, next) { - - if (is_ivshmem_device(dev)) { - - /* IVSHMEM memory is always on BAR2 */ - res = &dev->mem_resource[2]; - - /* if we don't have a BAR2 */ - if (res->len == 0) - continue; - - /* construct pci device path */ - snprintf(path, sizeof(path), IVSHMEM_RESOURCE_PATH, - dev->addr.domain, dev->addr.bus, dev->addr.devid, - dev->addr.function); - - /* try to find memseg */ - fd = open(path, O_RDWR); - if (fd < 0) { - RTE_LOG(ERR, EAL, "Could not open %s\n", path); - return -1; - } - - /* check if it's a DPDK IVSHMEM device */ - ret = has_ivshmem_metadata(fd, res->len); - - /* is DPDK device */ - if (ret == 1) { - - /* config file creation is deferred until the first - * DPDK device is found. then, it has to be created - * only once. */ - if (ivshmem_config == NULL && - create_shared_config() < 0) { - RTE_LOG(ERR, EAL, "Could not create IVSHMEM config!\n"); - close(fd); - return -1; - } - - if (read_metadata(path, sizeof(path), fd, res->len) < 0) { - RTE_LOG(ERR, EAL, "Could not read metadata from" - " device %02x:%02x.%x!\n", dev->addr.bus, - dev->addr.devid, dev->addr.function); - close(fd); - return -1; - } - - if (ivshmem_config->pci_devs_idx == RTE_LIBRTE_IVSHMEM_MAX_PCI_DEVS) { - RTE_LOG(WARNING, EAL, - "IVSHMEM PCI device limit exceeded. Increase " - "CONFIG_RTE_LIBRTE_IVSHMEM_MAX_PCI_DEVS in " - "your config file.\n"); - break; - } - - RTE_LOG(INFO, EAL, "Found IVSHMEM device %02x:%02x.%x\n", - dev->addr.bus, dev->addr.devid, dev->addr.function); - - ivshmem_config->pci_devs[ivshmem_config->pci_devs_idx].ioremap_addr = res->phys_addr; - snprintf(ivshmem_config->pci_devs[ivshmem_config->pci_devs_idx].path, - sizeof(ivshmem_config->pci_devs[ivshmem_config->pci_devs_idx].path), - "%s", path); - - ivshmem_config->pci_devs_idx++; - } - /* failed to read */ - else if (ret < 0) { - RTE_LOG(ERR, EAL, "Could not read IVSHMEM device: %s\n", - strerror(errno)); - close(fd); - return -1; - } - /* not a DPDK device */ - else - RTE_LOG(DEBUG, EAL, "Skipping non-DPDK IVSHMEM device\n"); - - /* close the BAR fd */ - close(fd); - } - } - } - - /* ivshmem_config is not NULL only if config was created and/or mapped */ - if (ivshmem_config) { - if (map_all_segments() < 0) { - RTE_LOG(ERR, EAL, "Mapping IVSHMEM segments failed!\n"); - return -1; - } - } - else { - RTE_LOG(DEBUG, EAL, "No IVSHMEM configuration found! \n"); - } - - return 0; -} - -#endif diff --git a/dpdk/lib/librte_eal/linuxapp/eal/eal_log.c b/dpdk/lib/librte_eal/linuxapp/eal/eal_log.c index d3911004..c088bd9b 100644 --- a/dpdk/lib/librte_eal/linuxapp/eal/eal_log.c +++ b/dpdk/lib/librte_eal/linuxapp/eal/eal_log.c @@ -39,7 +39,6 @@ #include #include -#include #include #include #include @@ -97,45 +96,7 @@ rte_eal_log_init(const char *id, int facility) openlog(id, LOG_NDELAY | LOG_PID, facility); - if (rte_eal_common_log_init(log_stream) < 0) - return -1; + eal_log_set_default(log_stream); return 0; } - -/* early logs */ - -/* - * early log function, used before rte_eal_log_init - */ -static ssize_t -early_log_write(__attribute__((unused)) void *c, const char *buf, size_t size) -{ - ssize_t ret; - ret = fwrite(buf, size, 1, stdout); - fflush(stdout); - if (ret == 0) - return -1; - return ret; -} - -static cookie_io_functions_t early_log_func = { - .write = early_log_write, -}; -static FILE *early_log_stream; - -/* - * init the log library, called by rte_eal_init() to enable early - * logs - */ -int -rte_eal_log_early_init(void) -{ - early_log_stream = fopencookie(NULL, "w+", early_log_func); - if (early_log_stream == NULL) { - printf("Cannot configure early_log_stream\n"); - return -1; - } - rte_openlog_stream(early_log_stream); - return 0; -} diff --git a/dpdk/lib/librte_eal/linuxapp/eal/eal_memory.c b/dpdk/lib/librte_eal/linuxapp/eal/eal_memory.c index c04cff0c..16a181c3 100644 --- a/dpdk/lib/librte_eal/linuxapp/eal/eal_memory.c +++ b/dpdk/lib/librte_eal/linuxapp/eal/eal_memory.c @@ -2,6 +2,7 @@ * BSD LICENSE * * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * Copyright(c) 2013 6WIND. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -30,46 +31,16 @@ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -/* BSD LICENSE - * - * Copyright(c) 2013 6WIND. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of 6WIND S.A. nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ #define _FILE_OFFSET_BITS 64 #include #include +#include #include #include #include #include #include -#include #include #include #include @@ -77,15 +48,17 @@ #include #include #include -#include #include #include #include #include +#ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES +#include +#include +#endif #include #include -#include #include #include #include @@ -101,13 +74,6 @@ #define PFN_MASK_SIZE 8 -#ifdef RTE_LIBRTE_XEN_DOM0 -int rte_xen_dom0_supported(void) -{ - return internal_config.xen_dom0_support; -} -#endif - /** * @file * Huge page mapping under linux @@ -122,36 +88,32 @@ int rte_xen_dom0_supported(void) static uint64_t baseaddr_offset; -static unsigned proc_pagemap_readable; +static bool phys_addrs_available = true; #define RANDOMIZE_VA_SPACE_FILE "/proc/sys/kernel/randomize_va_space" static void -test_proc_pagemap_readable(void) +test_phys_addrs_available(void) { - int fd = open("/proc/self/pagemap", O_RDONLY); + uint64_t tmp; + phys_addr_t physaddr; - if (fd < 0) { + if (!rte_eal_has_hugepages()) { RTE_LOG(ERR, EAL, - "Cannot open /proc/self/pagemap: %s. " - "virt2phys address translation will not work\n", - strerror(errno)); + "Started without hugepages support, physical addresses not available\n"); + phys_addrs_available = false; return; } - /* Is readable */ - close(fd); - proc_pagemap_readable = 1; -} - -/* Lock page in physical memory and prevent from swapping. */ -int -rte_mem_lock_page(const void *virt) -{ - unsigned long virtual = (unsigned long)virt; - int page_size = getpagesize(); - unsigned long aligned = (virtual & ~ (page_size - 1)); - return mlock((void*)aligned, page_size); + physaddr = rte_mem_virt2phy(&tmp); + if (physaddr == RTE_BAD_PHYS_ADDR) { + if (rte_eal_iova_mode() == RTE_IOVA_PA) + RTE_LOG(ERR, EAL, + "Cannot obtain physical addresses: %s. " + "Only vfio will function.\n", + strerror(errno)); + phys_addrs_available = false; + } } /* @@ -166,32 +128,9 @@ rte_mem_virt2phy(const void *virtaddr) int page_size; off_t offset; - /* when using dom0, /proc/self/pagemap always returns 0, check in - * dpdk memory by browsing the memsegs */ - if (rte_xen_dom0_supported()) { - struct rte_mem_config *mcfg; - struct rte_memseg *memseg; - unsigned i; - - mcfg = rte_eal_get_configuration()->mem_config; - for (i = 0; i < RTE_MAX_MEMSEG; i++) { - memseg = &mcfg->memseg[i]; - if (memseg->addr == NULL) - break; - if (virtaddr > memseg->addr && - virtaddr < RTE_PTR_ADD(memseg->addr, - memseg->len)) { - return memseg->phys_addr + - RTE_PTR_DIFF(virtaddr, memseg->addr); - } - } - - return RTE_BAD_PHYS_ADDR; - } - /* Cannot parse /proc/self/pagemap, no need to log errors everywhere */ - if (!proc_pagemap_readable) - return RTE_BAD_PHYS_ADDR; + if (!phys_addrs_available) + return RTE_BAD_IOVA; /* standard page size */ page_size = getpagesize(); @@ -200,7 +139,7 @@ rte_mem_virt2phy(const void *virtaddr) if (fd < 0) { RTE_LOG(ERR, EAL, "%s(): cannot open /proc/self/pagemap: %s\n", __func__, strerror(errno)); - return RTE_BAD_PHYS_ADDR; + return RTE_BAD_IOVA; } virt_pfn = (unsigned long)virtaddr / page_size; @@ -209,7 +148,7 @@ rte_mem_virt2phy(const void *virtaddr) RTE_LOG(ERR, EAL, "%s(): seek error in /proc/self/pagemap: %s\n", __func__, strerror(errno)); close(fd); - return RTE_BAD_PHYS_ADDR; + return RTE_BAD_IOVA; } retval = read(fd, &page, PFN_MASK_SIZE); @@ -217,24 +156,35 @@ rte_mem_virt2phy(const void *virtaddr) if (retval < 0) { RTE_LOG(ERR, EAL, "%s(): cannot read /proc/self/pagemap: %s\n", __func__, strerror(errno)); - return RTE_BAD_PHYS_ADDR; + return RTE_BAD_IOVA; } else if (retval != PFN_MASK_SIZE) { RTE_LOG(ERR, EAL, "%s(): read %d bytes from /proc/self/pagemap " "but expected %d:\n", __func__, retval, PFN_MASK_SIZE); - return RTE_BAD_PHYS_ADDR; + return RTE_BAD_IOVA; } /* * the pfn (page frame number) are bits 0-54 (see * pagemap.txt in linux Documentation) */ + if ((page & 0x7fffffffffffffULL) == 0) + return RTE_BAD_IOVA; + physaddr = ((page & 0x7fffffffffffffULL) * page_size) + ((unsigned long)virtaddr % page_size); return physaddr; } +rte_iova_t +rte_mem_virt2iova(const void *virtaddr) +{ + if (rte_eal_iova_mode() == RTE_IOVA_VA) + return (uintptr_t)virtaddr; + return rte_mem_virt2phy(virtaddr); +} + /* * For each hugepage in hugepg_tbl, fill the physaddr value. We find * it by browsing the /proc/self/pagemap special file. @@ -242,7 +192,7 @@ rte_mem_virt2phy(const void *virtaddr) static int find_physaddrs(struct hugepage_file *hugepg_tbl, struct hugepage_info *hpi) { - unsigned i; + unsigned int i; phys_addr_t addr; for (i = 0; i < hpi->num_pages[0]; i++) { @@ -254,6 +204,22 @@ find_physaddrs(struct hugepage_file *hugepg_tbl, struct hugepage_info *hpi) return 0; } +/* + * For each hugepage in hugepg_tbl, fill the physaddr value sequentially. + */ +static int +set_physaddrs(struct hugepage_file *hugepg_tbl, struct hugepage_info *hpi) +{ + unsigned int i; + static phys_addr_t addr; + + for (i = 0; i < hpi->num_pages[0]; i++) { + hugepg_tbl[i].physaddr = addr; + addr += hugepg_tbl[i].size; + } + return 0; +} + /* * Check whether address-space layout randomization is enabled in * the kernel. This is important for multi-process as it can prevent @@ -313,7 +279,13 @@ get_virtual_area(size_t *size, size_t hugepage_sz) } do { addr = mmap(addr, - (*size) + hugepage_sz, PROT_READ, MAP_PRIVATE, fd, 0); + (*size) + hugepage_sz, PROT_READ, +#ifdef RTE_ARCH_PPC_64 + MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, +#else + MAP_PRIVATE, +#endif + fd, 0); if (addr == MAP_FAILED) *size -= hugepage_sz; } while (addr == MAP_FAILED && *size > 0); @@ -359,42 +331,100 @@ static int huge_wrap_sigsetjmp(void) return sigsetjmp(huge_jmpenv, 1); } +#ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES +/* Callback for numa library. */ +void numa_error(char *where) +{ + RTE_LOG(ERR, EAL, "%s failed: %s\n", where, strerror(errno)); +} +#endif + /* * Mmap all hugepages of hugepage table: it first open a file in * hugetlbfs, then mmap() hugepage_sz data in it. If orig is set, the * virtual address is stored in hugepg_tbl[i].orig_va, else it is stored * in hugepg_tbl[i].final_va. The second mapping (when orig is 0) tries to - * map continguous physical blocks in contiguous virtual blocks. + * map contiguous physical blocks in contiguous virtual blocks. */ static unsigned -map_all_hugepages(struct hugepage_file *hugepg_tbl, - struct hugepage_info *hpi, int orig) +map_all_hugepages(struct hugepage_file *hugepg_tbl, struct hugepage_info *hpi, + uint64_t *essential_memory __rte_unused, int orig) { int fd; unsigned i; void *virtaddr; void *vma_addr = NULL; size_t vma_len = 0; +#ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES + int node_id = -1; + int essential_prev = 0; + int oldpolicy; + struct bitmask *oldmask = numa_allocate_nodemask(); + bool have_numa = true; + unsigned long maxnode = 0; -#ifdef RTE_EAL_SINGLE_FILE_SEGMENTS - RTE_SET_USED(vma_len); + /* Check if kernel supports NUMA. */ + if (numa_available() != 0) { + RTE_LOG(DEBUG, EAL, "NUMA is not supported.\n"); + have_numa = false; + } + + if (orig && have_numa) { + RTE_LOG(DEBUG, EAL, "Trying to obtain current memory policy.\n"); + if (get_mempolicy(&oldpolicy, oldmask->maskp, + oldmask->size + 1, 0, 0) < 0) { + RTE_LOG(ERR, EAL, + "Failed to get current mempolicy: %s. " + "Assuming MPOL_DEFAULT.\n", strerror(errno)); + oldpolicy = MPOL_DEFAULT; + } + for (i = 0; i < RTE_MAX_NUMA_NODES; i++) + if (internal_config.socket_mem[i]) + maxnode = i + 1; + } #endif for (i = 0; i < hpi->num_pages[0]; i++) { uint64_t hugepage_sz = hpi->hugepage_sz; +#ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES + if (maxnode) { + unsigned int j; + + for (j = 0; j < maxnode; j++) + if (essential_memory[j]) + break; + + if (j == maxnode) { + node_id = (node_id + 1) % maxnode; + while (!internal_config.socket_mem[node_id]) { + node_id++; + node_id %= maxnode; + } + essential_prev = 0; + } else { + node_id = j; + essential_prev = essential_memory[j]; + + if (essential_memory[j] < hugepage_sz) + essential_memory[j] = 0; + else + essential_memory[j] -= hugepage_sz; + } + + RTE_LOG(DEBUG, EAL, + "Setting policy MPOL_PREFERRED for socket %d\n", + node_id); + numa_set_preferred(node_id); + } +#endif + if (orig) { hugepg_tbl[i].file_id = i; hugepg_tbl[i].size = hugepage_sz; -#ifdef RTE_EAL_SINGLE_FILE_SEGMENTS - eal_get_hugefile_temp_path(hugepg_tbl[i].filepath, - sizeof(hugepg_tbl[i].filepath), hpi->hugedir, - hugepg_tbl[i].file_id); -#else eal_get_hugefile_path(hugepg_tbl[i].filepath, sizeof(hugepg_tbl[i].filepath), hpi->hugedir, hugepg_tbl[i].file_id); -#endif hugepg_tbl[i].filepath[sizeof(hugepg_tbl[i].filepath) - 1] = '\0'; } #ifndef RTE_ARCH_64 @@ -408,8 +438,6 @@ map_all_hugepages(struct hugepage_file *hugepg_tbl, continue; } #endif - -#ifndef RTE_EAL_SINGLE_FILE_SEGMENTS else if (vma_len == 0) { unsigned j, num_pages; @@ -439,14 +467,13 @@ map_all_hugepages(struct hugepage_file *hugepg_tbl, if (vma_addr == NULL) vma_len = hugepage_sz; } -#endif /* try to create hugepage file */ - fd = open(hugepg_tbl[i].filepath, O_CREAT | O_RDWR, 0755); + fd = open(hugepg_tbl[i].filepath, O_CREAT | O_RDWR, 0600); if (fd < 0) { RTE_LOG(DEBUG, EAL, "%s(): open failed: %s\n", __func__, strerror(errno)); - return i; + goto out; } /* map the segment, and populate page tables, @@ -457,7 +484,7 @@ map_all_hugepages(struct hugepage_file *hugepg_tbl, RTE_LOG(DEBUG, EAL, "%s(): mmap failed: %s\n", __func__, strerror(errno)); close(fd); - return i; + goto out; } if (orig) { @@ -482,7 +509,12 @@ map_all_hugepages(struct hugepage_file *hugepg_tbl, munmap(virtaddr, hugepage_sz); close(fd); unlink(hugepg_tbl[i].filepath); - return i; +#ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES + if (maxnode) + essential_memory[node_id] = + essential_prev; +#endif + goto out; } *(int *)virtaddr = 0; } @@ -493,7 +525,7 @@ map_all_hugepages(struct hugepage_file *hugepg_tbl, RTE_LOG(DEBUG, EAL, "%s(): Locking file failed:%s \n", __func__, strerror(errno)); close(fd); - return i; + goto out; } close(fd); @@ -502,172 +534,25 @@ map_all_hugepages(struct hugepage_file *hugepg_tbl, vma_len -= hugepage_sz; } +out: +#ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES + if (maxnode) { + RTE_LOG(DEBUG, EAL, + "Restoring previous memory policy: %d\n", oldpolicy); + if (oldpolicy == MPOL_DEFAULT) { + numa_set_localalloc(); + } else if (set_mempolicy(oldpolicy, oldmask->maskp, + oldmask->size + 1) < 0) { + RTE_LOG(ERR, EAL, "Failed to restore mempolicy: %s\n", + strerror(errno)); + numa_set_localalloc(); + } + } + numa_free_cpumask(oldmask); +#endif return i; } -#ifdef RTE_EAL_SINGLE_FILE_SEGMENTS - -/* - * Remaps all hugepages into single file segments - */ -static int -remap_all_hugepages(struct hugepage_file *hugepg_tbl, struct hugepage_info *hpi) -{ - int fd; - unsigned i = 0, j, num_pages, page_idx = 0; - void *vma_addr = NULL, *old_addr = NULL, *page_addr = NULL; - size_t vma_len = 0; - size_t hugepage_sz = hpi->hugepage_sz; - size_t total_size, offset; - char filepath[MAX_HUGEPAGE_PATH]; - phys_addr_t physaddr; - int socket; - - while (i < hpi->num_pages[0]) { - -#ifndef RTE_ARCH_64 - /* for 32-bit systems, don't remap 1G pages and 16G pages, - * just reuse original map address as final map address. - */ - if ((hugepage_sz == RTE_PGSIZE_1G) - || (hugepage_sz == RTE_PGSIZE_16G)) { - hugepg_tbl[i].final_va = hugepg_tbl[i].orig_va; - hugepg_tbl[i].orig_va = NULL; - i++; - continue; - } -#endif - - /* reserve a virtual area for next contiguous - * physical block: count the number of - * contiguous physical pages. */ - for (j = i+1; j < hpi->num_pages[0] ; j++) { -#ifdef RTE_ARCH_PPC_64 - /* The physical addresses are sorted in descending - * order on PPC64 */ - if (hugepg_tbl[j].physaddr != - hugepg_tbl[j-1].physaddr - hugepage_sz) - break; -#else - if (hugepg_tbl[j].physaddr != - hugepg_tbl[j-1].physaddr + hugepage_sz) - break; -#endif - } - num_pages = j - i; - vma_len = num_pages * hugepage_sz; - - socket = hugepg_tbl[i].socket_id; - - /* get the biggest virtual memory area up to - * vma_len. If it fails, vma_addr is NULL, so - * let the kernel provide the address. */ - vma_addr = get_virtual_area(&vma_len, hpi->hugepage_sz); - - /* If we can't find a big enough virtual area, work out how many pages - * we are going to get */ - if (vma_addr == NULL) - j = i + 1; - else if (vma_len != num_pages * hugepage_sz) { - num_pages = vma_len / hugepage_sz; - j = i + num_pages; - - } - - hugepg_tbl[page_idx].file_id = page_idx; - eal_get_hugefile_path(filepath, - sizeof(filepath), - hpi->hugedir, - hugepg_tbl[page_idx].file_id); - - /* try to create hugepage file */ - fd = open(filepath, O_CREAT | O_RDWR, 0755); - if (fd < 0) { - RTE_LOG(ERR, EAL, "%s(): open failed: %s\n", __func__, strerror(errno)); - return -1; - } - - total_size = 0; - for (;i < j; i++) { - - /* unmap current segment */ - if (total_size > 0) - munmap(vma_addr, total_size); - - /* unmap original page */ - munmap(hugepg_tbl[i].orig_va, hugepage_sz); - unlink(hugepg_tbl[i].filepath); - - total_size += hugepage_sz; - - old_addr = vma_addr; - - /* map new, bigger segment, and populate page tables, - * the kernel fills this segment with zeros */ - vma_addr = mmap(vma_addr, total_size, - PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd, 0); - - if (vma_addr == MAP_FAILED || vma_addr != old_addr) { - RTE_LOG(ERR, EAL, "%s(): mmap failed: %s\n", __func__, strerror(errno)); - close(fd); - return -1; - } - } - - /* set shared flock on the file. */ - if (flock(fd, LOCK_SH | LOCK_NB) == -1) { - RTE_LOG(ERR, EAL, "%s(): Locking file failed:%s \n", - __func__, strerror(errno)); - close(fd); - return -1; - } - - snprintf(hugepg_tbl[page_idx].filepath, MAX_HUGEPAGE_PATH, "%s", - filepath); - - physaddr = rte_mem_virt2phy(vma_addr); - - if (physaddr == RTE_BAD_PHYS_ADDR) - return -1; - - hugepg_tbl[page_idx].final_va = vma_addr; - - hugepg_tbl[page_idx].physaddr = physaddr; - - hugepg_tbl[page_idx].repeated = num_pages; - - hugepg_tbl[page_idx].socket_id = socket; - - close(fd); - - /* verify the memory segment - that is, check that every VA corresponds - * to the physical address we expect to see - */ - for (offset = 0; offset < vma_len; offset += hugepage_sz) { - uint64_t expected_physaddr; - - expected_physaddr = hugepg_tbl[page_idx].physaddr + offset; - page_addr = RTE_PTR_ADD(vma_addr, offset); - physaddr = rte_mem_virt2phy(page_addr); - - if (physaddr != expected_physaddr) { - RTE_LOG(ERR, EAL, "Segment sanity check failed: wrong physaddr " - "at %p (offset 0x%" PRIx64 ": 0x%" PRIx64 - " (expected 0x%" PRIx64 ")\n", - page_addr, offset, physaddr, expected_physaddr); - return -1; - } - } - - page_idx++; - } - - /* zero out the rest */ - memset(&hugepg_tbl[page_idx], 0, (hpi->num_pages[0] - page_idx) * sizeof(struct hugepage_file)); - return page_idx; -} -#else/* RTE_EAL_SINGLE_FILE_SEGMENTS=n */ - /* Unmap all hugepages from original mapping */ static int unmap_all_hugepages_orig(struct hugepage_file *hugepg_tbl, struct hugepage_info *hpi) @@ -681,7 +566,6 @@ unmap_all_hugepages_orig(struct hugepage_file *hugepg_tbl, struct hugepage_info } return 0; } -#endif /* RTE_EAL_SINGLE_FILE_SEGMENTS */ /* * Parse /proc/self/numa_maps to get the NUMA socket ID for each huge @@ -700,8 +584,8 @@ find_numasocket(struct hugepage_file *hugepg_tbl, struct hugepage_info *hpi) f = fopen("/proc/self/numa_maps", "r"); if (f == NULL) { - RTE_LOG(NOTICE, EAL, "cannot open /proc/self/numa_maps," - " consider that all memory is in socket_id 0\n"); + RTE_LOG(NOTICE, EAL, "NUMA support not available" + " consider that all memory is in socket_id 0\n"); return 0; } @@ -750,6 +634,11 @@ find_numasocket(struct hugepage_file *hugepg_tbl, struct hugepage_info *hpi) if (hugepg_tbl[i].orig_va == va) { hugepg_tbl[i].socket_id = socket_id; hp_count++; +#ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES + RTE_LOG(DEBUG, EAL, + "Hugepage %s is on socket %d\n", + hugepg_tbl[i].filepath, socket_id); +#endif } } } @@ -769,12 +658,12 @@ static int cmp_physaddr(const void *a, const void *b) { #ifndef RTE_ARCH_PPC_64 - const struct hugepage_file *p1 = (const struct hugepage_file *)a; - const struct hugepage_file *p2 = (const struct hugepage_file *)b; + const struct hugepage_file *p1 = a; + const struct hugepage_file *p2 = b; #else /* PowerPC needs memory sorted in reverse order from x86 */ - const struct hugepage_file *p1 = (const struct hugepage_file *)b; - const struct hugepage_file *p2 = (const struct hugepage_file *)a; + const struct hugepage_file *p1 = b; + const struct hugepage_file *p2 = a; #endif if (p1->physaddr < p2->physaddr) return -1; @@ -801,6 +690,8 @@ create_shared_memory(const char *filename, const size_t mem_size) } retval = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); close(fd); + if (retval == MAP_FAILED) + return NULL; return retval; } @@ -875,12 +766,6 @@ unmap_unneeded_hugepages(struct hugepage_file *hugepg_tbl, for (page = 0; page < nrpages; page++) { struct hugepage_file *hp = &hugepg_tbl[page]; -#ifdef RTE_EAL_SINGLE_FILE_SEGMENTS - /* if this page was already cleared */ - if (hp->final_va == NULL) - continue; -#endif - /* find a page that matches the criteria */ if ((hp->size == hpi[size].hugepage_sz) && (hp->socket_id == (int) socket)) { @@ -889,11 +774,7 @@ unmap_unneeded_hugepages(struct hugepage_file *hugepg_tbl, if (pages_found == hpi[size].num_pages[socket]) { uint64_t unmap_len; -#ifdef RTE_EAL_SINGLE_FILE_SEGMENTS - unmap_len = hp->size * hp->repeated; -#else unmap_len = hp->size; -#endif /* get start addr and len of the remaining segment */ munmap(hp->final_va, (size_t) unmap_len); @@ -904,50 +785,10 @@ unmap_unneeded_hugepages(struct hugepage_file *hugepg_tbl, __func__, hp->filepath, strerror(errno)); return -1; } - } -#ifdef RTE_EAL_SINGLE_FILE_SEGMENTS - /* else, check how much do we need to map */ - else { - int nr_pg_left = - hpi[size].num_pages[socket] - pages_found; - - /* if we need enough memory to fit into the segment */ - if (hp->repeated <= nr_pg_left) { - pages_found += hp->repeated; - } - /* truncate the segment */ - else { - uint64_t final_size = nr_pg_left * hp->size; - uint64_t seg_size = hp->repeated * hp->size; - - void * unmap_va = RTE_PTR_ADD(hp->final_va, - final_size); - int fd; - - munmap(unmap_va, seg_size - final_size); - - fd = open(hp->filepath, O_RDWR); - if (fd < 0) { - RTE_LOG(ERR, EAL, "Cannot open %s: %s\n", - hp->filepath, strerror(errno)); - return -1; - } - if (ftruncate(fd, final_size) < 0) { - RTE_LOG(ERR, EAL, "Cannot truncate %s: %s\n", - hp->filepath, strerror(errno)); - return -1; - } - close(fd); - - pages_found += nr_pg_left; - hp->repeated = nr_pg_left; - } - } -#else - /* else, lock the page and skip */ - else + } else { + /* lock the page and skip */ pages_found++; -#endif + } } /* match page */ } /* foreach page */ @@ -1177,11 +1018,8 @@ rte_eal_hugepage_init(void) int i, j, new_memseg; int nr_hugefiles, nr_hugepages = 0; void *addr; -#ifdef RTE_EAL_SINGLE_FILE_SEGMENTS - int new_pages_count[MAX_HUGEPAGE_SIZES]; -#endif - test_proc_pagemap_readable(); + test_phys_addrs_available(); memset(used_hp, 0, sizeof(used_hp)); @@ -1197,7 +1035,10 @@ rte_eal_hugepage_init(void) strerror(errno)); return -1; } - mcfg->memseg[0].phys_addr = (phys_addr_t)(uintptr_t)addr; + if (rte_eal_iova_mode() == RTE_IOVA_VA) + mcfg->memseg[0].iova = (uintptr_t)addr; + else + mcfg->memseg[0].iova = RTE_BAD_IOVA; mcfg->memseg[0].addr = addr; mcfg->memseg[0].hugepage_sz = RTE_PGSIZE_4K; mcfg->memseg[0].len = internal_config.memory; @@ -1205,17 +1046,6 @@ rte_eal_hugepage_init(void) return 0; } -/* check if app runs on Xen Dom0 */ - if (internal_config.xen_dom0_support) { -#ifdef RTE_LIBRTE_XEN_DOM0 - /* use dom0_mm kernel driver to init memory */ - if (rte_xen_dom0_memory_init() < 0) - return -1; - else - return 0; -#endif - } - /* calculate total number of hugepages available. at this point we haven't * yet started sorting them so they all are on socket 0 */ for (i = 0; i < (int) internal_config.num_hugepage_sizes; i++) { @@ -1241,6 +1071,11 @@ rte_eal_hugepage_init(void) huge_register_sigbus(); + /* make a copy of socket_mem, needed for balanced allocation. */ + for (i = 0; i < RTE_MAX_NUMA_NODES; i++) + memory[i] = internal_config.socket_mem[i]; + + /* map all hugepages and sort them */ for (i = 0; i < (int)internal_config.num_hugepage_sizes; i ++){ unsigned pages_old, pages_new; @@ -1258,15 +1093,9 @@ rte_eal_hugepage_init(void) /* map all hugepages available */ pages_old = hpi->num_pages[0]; - pages_new = map_all_hugepages(&tmp_hp[hp_offset], hpi, 1); + pages_new = map_all_hugepages(&tmp_hp[hp_offset], hpi, + memory, 1); if (pages_new < pages_old) { -#ifdef RTE_EAL_SINGLE_FILE_SEGMENTS - RTE_LOG(ERR, EAL, - "%d not %d hugepages of size %u MB allocated\n", - pages_new, pages_old, - (unsigned)(hpi->hugepage_sz / 0x100000)); - goto fail; -#else RTE_LOG(DEBUG, EAL, "%d not %d hugepages of size %u MB allocated\n", pages_new, pages_old, @@ -1278,14 +1107,24 @@ rte_eal_hugepage_init(void) hpi->num_pages[0] = pages_new; if (pages_new == 0) continue; -#endif } - /* find physical addresses and sockets for each hugepage */ - if (find_physaddrs(&tmp_hp[hp_offset], hpi) < 0){ - RTE_LOG(DEBUG, EAL, "Failed to find phys addr for %u MB pages\n", - (unsigned)(hpi->hugepage_sz / 0x100000)); - goto fail; + if (phys_addrs_available) { + /* find physical addresses for each hugepage */ + if (find_physaddrs(&tmp_hp[hp_offset], hpi) < 0) { + RTE_LOG(DEBUG, EAL, "Failed to find phys addr " + "for %u MB pages\n", + (unsigned int)(hpi->hugepage_sz / 0x100000)); + goto fail; + } + } else { + /* set physical addresses for each hugepage */ + if (set_physaddrs(&tmp_hp[hp_offset], hpi) < 0) { + RTE_LOG(DEBUG, EAL, "Failed to set phys addr " + "for %u MB pages\n", + (unsigned int)(hpi->hugepage_sz / 0x100000)); + goto fail; + } } if (find_numasocket(&tmp_hp[hp_offset], hpi) < 0){ @@ -1297,20 +1136,8 @@ rte_eal_hugepage_init(void) qsort(&tmp_hp[hp_offset], hpi->num_pages[0], sizeof(struct hugepage_file), cmp_physaddr); -#ifdef RTE_EAL_SINGLE_FILE_SEGMENTS - /* remap all hugepages into single file segments */ - new_pages_count[i] = remap_all_hugepages(&tmp_hp[hp_offset], hpi); - if (new_pages_count[i] < 0){ - RTE_LOG(DEBUG, EAL, "Failed to remap %u MB pages\n", - (unsigned)(hpi->hugepage_sz / 0x100000)); - goto fail; - } - - /* we have processed a num of hugepages of this size, so inc offset */ - hp_offset += new_pages_count[i]; -#else /* remap all hugepages */ - if (map_all_hugepages(&tmp_hp[hp_offset], hpi, 0) != + if (map_all_hugepages(&tmp_hp[hp_offset], hpi, NULL, 0) != hpi->num_pages[0]) { RTE_LOG(ERR, EAL, "Failed to remap %u MB pages\n", (unsigned)(hpi->hugepage_sz / 0x100000)); @@ -1323,7 +1150,6 @@ rte_eal_hugepage_init(void) /* we have processed a num of hugepages of this size, so inc offset */ hp_offset += hpi->num_pages[0]; -#endif } huge_recover_sigbus(); @@ -1331,14 +1157,7 @@ rte_eal_hugepage_init(void) if (internal_config.memory == 0 && internal_config.force_sockets == 0) internal_config.memory = eal_get_hugepage_mem_size(); -#ifdef RTE_EAL_SINGLE_FILE_SEGMENTS - nr_hugefiles = 0; - for (i = 0; i < (int) internal_config.num_hugepage_sizes; i++) { - nr_hugefiles += new_pages_count[i]; - } -#else nr_hugefiles = nr_hugepages; -#endif /* clean out the numbers of pages */ @@ -1356,12 +1175,7 @@ rte_eal_hugepage_init(void) for (j = 0; j < nb_hpsizes; j++) { if (tmp_hp[i].size == internal_config.hugepage_info[j].hugepage_sz) { -#ifdef RTE_EAL_SINGLE_FILE_SEGMENTS - internal_config.hugepage_info[j].num_pages[socket] += - tmp_hp[i].repeated; -#else internal_config.hugepage_info[j].num_pages[socket]++; -#endif } } } @@ -1436,15 +1250,8 @@ rte_eal_hugepage_init(void) free(tmp_hp); tmp_hp = NULL; - /* find earliest free memseg - this is needed because in case of IVSHMEM, - * segments might have already been initialized */ - for (j = 0; j < RTE_MAX_MEMSEG; j++) - if (mcfg->memseg[j].addr == NULL) { - /* move to previous segment and exit loop */ - j--; - break; - } - + /* first memseg index shall be 0 after incrementing it below */ + j = -1; for (i = 0; i < nr_hugefiles; i++) { new_memseg = 0; @@ -1480,13 +1287,9 @@ rte_eal_hugepage_init(void) if (j == RTE_MAX_MEMSEG) break; - mcfg->memseg[j].phys_addr = hugepage[i].physaddr; + mcfg->memseg[j].iova = hugepage[i].physaddr; mcfg->memseg[j].addr = hugepage[i].final_va; -#ifdef RTE_EAL_SINGLE_FILE_SEGMENTS - mcfg->memseg[j].len = hugepage[i].size * hugepage[i].repeated; -#else mcfg->memseg[j].len = hugepage[i].size; -#endif mcfg->memseg[j].socket_id = hugepage[i].socket_id; mcfg->memseg[j].hugepage_sz = hugepage[i].size; } @@ -1495,7 +1298,7 @@ rte_eal_hugepage_init(void) #ifdef RTE_ARCH_PPC_64 /* Use the phy and virt address of the last page as segment * address for IBM Power architecture */ - mcfg->memseg[j].phys_addr = hugepage[i].physaddr; + mcfg->memseg[j].iova = hugepage[i].physaddr; mcfg->memseg[j].addr = hugepage[i].final_va; #endif mcfg->memseg[j].len += mcfg->memseg[j].hugepage_sz; @@ -1563,18 +1366,7 @@ rte_eal_hugepage_attach(void) "into secondary processes\n"); } - test_proc_pagemap_readable(); - - if (internal_config.xen_dom0_support) { -#ifdef RTE_LIBRTE_XEN_DOM0 - if (rte_xen_dom0_memory_attach() < 0) { - RTE_LOG(ERR, EAL, "Failed to attach memory segments of primary " - "process\n"); - return -1; - } - return 0; -#endif - } + test_phys_addrs_available(); fd_zero = open("/dev/zero", O_RDONLY); if (fd_zero < 0) { @@ -1598,31 +1390,36 @@ rte_eal_hugepage_attach(void) if (mcfg->memseg[s].len == 0) break; -#ifdef RTE_LIBRTE_IVSHMEM - /* - * if segment has ioremap address set, it's an IVSHMEM segment and - * doesn't need mapping as it was already mapped earlier - */ - if (mcfg->memseg[s].ioremap_addr != 0) - continue; -#endif - /* * fdzero is mmapped to get a contiguous block of virtual * addresses of the appropriate memseg size. * use mmap to get identical addresses as the primary process. */ base_addr = mmap(mcfg->memseg[s].addr, mcfg->memseg[s].len, - PROT_READ, MAP_PRIVATE, fd_zero, 0); + PROT_READ, +#ifdef RTE_ARCH_PPC_64 + MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, +#else + MAP_PRIVATE, +#endif + fd_zero, 0); if (base_addr == MAP_FAILED || base_addr != mcfg->memseg[s].addr) { - RTE_LOG(ERR, EAL, "Could not mmap %llu bytes " - "in /dev/zero to requested address [%p]: '%s'\n", - (unsigned long long)mcfg->memseg[s].len, - mcfg->memseg[s].addr, strerror(errno)); max_seg = s; - if (base_addr != MAP_FAILED) + if (base_addr != MAP_FAILED) { + /* errno is stale, don't use */ + RTE_LOG(ERR, EAL, "Could not mmap %llu bytes " + "in /dev/zero at [%p], got [%p] - " + "please use '--base-virtaddr' option\n", + (unsigned long long)mcfg->memseg[s].len, + mcfg->memseg[s].addr, base_addr); munmap(base_addr, mcfg->memseg[s].len); + } else { + RTE_LOG(ERR, EAL, "Could not mmap %llu bytes " + "in /dev/zero at [%p]: '%s'\n", + (unsigned long long)mcfg->memseg[s].len, + mcfg->memseg[s].addr, strerror(errno)); + } if (aslr_enabled() > 0) { RTE_LOG(ERR, EAL, "It is recommended to " "disable ASLR in the kernel " @@ -1648,16 +1445,6 @@ rte_eal_hugepage_attach(void) void *addr, *base_addr; uintptr_t offset = 0; size_t mapping_size; -#ifdef RTE_LIBRTE_IVSHMEM - /* - * if segment has ioremap address set, it's an IVSHMEM segment and - * doesn't need mapping as it was already mapped earlier - */ - if (mcfg->memseg[s].ioremap_addr != 0) { - s++; - continue; - } -#endif /* * free previously mapped memory so we can map the * hugepages into the space @@ -1676,11 +1463,7 @@ rte_eal_hugepage_attach(void) hp[i].filepath); goto error; } -#ifdef RTE_EAL_SINGLE_FILE_SEGMENTS - mapping_size = hp[i].size * hp[i].repeated; -#else mapping_size = hp[i].size; -#endif addr = mmap(RTE_PTR_ADD(base_addr, offset), mapping_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); @@ -1715,3 +1498,9 @@ error: close(fd_hugepage); return -1; } + +int +rte_eal_using_phys_addrs(void) +{ + return phys_addrs_available; +} diff --git a/dpdk/lib/librte_eal/linuxapp/eal/eal_pci.c b/dpdk/lib/librte_eal/linuxapp/eal/eal_pci.c deleted file mode 100644 index 706b1298..00000000 --- a/dpdk/lib/librte_eal/linuxapp/eal/eal_pci.c +++ /dev/null @@ -1,755 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include -#include - -#include -#include -#include -#include -#include -#include - -#include "eal_filesystem.h" -#include "eal_private.h" -#include "eal_pci_init.h" - -/** - * @file - * PCI probing under linux - * - * This code is used to simulate a PCI probe by parsing information in sysfs. - * When a registered device matches a driver, it is then initialized with - * IGB_UIO driver (or doesn't initialize, if the device wasn't bound to it). - */ - -/* unbind kernel driver for this device */ -int -pci_unbind_kernel_driver(struct rte_pci_device *dev) -{ - int n; - FILE *f; - char filename[PATH_MAX]; - char buf[BUFSIZ]; - struct rte_pci_addr *loc = &dev->addr; - - /* open /sys/bus/pci/devices/AAAA:BB:CC.D/driver */ - snprintf(filename, sizeof(filename), - "%s/" PCI_PRI_FMT "/driver/unbind", pci_get_sysfs_path(), - loc->domain, loc->bus, loc->devid, loc->function); - - f = fopen(filename, "w"); - if (f == NULL) /* device was not bound */ - return 0; - - n = snprintf(buf, sizeof(buf), PCI_PRI_FMT "\n", - loc->domain, loc->bus, loc->devid, loc->function); - if ((n < 0) || (n >= (int)sizeof(buf))) { - RTE_LOG(ERR, EAL, "%s(): snprintf failed\n", __func__); - goto error; - } - if (fwrite(buf, n, 1, f) == 0) { - RTE_LOG(ERR, EAL, "%s(): could not write to %s\n", __func__, - filename); - goto error; - } - - fclose(f); - return 0; - -error: - fclose(f); - return -1; -} - -static int -pci_get_kernel_driver_by_path(const char *filename, char *dri_name) -{ - int count; - char path[PATH_MAX]; - char *name; - - if (!filename || !dri_name) - return -1; - - count = readlink(filename, path, PATH_MAX); - if (count >= PATH_MAX) - return -1; - - /* For device does not have a driver */ - if (count < 0) - return 1; - - path[count] = '\0'; - - name = strrchr(path, '/'); - if (name) { - strncpy(dri_name, name + 1, strlen(name + 1) + 1); - return 0; - } - - return -1; -} - -/* Map pci device */ -int -rte_eal_pci_map_device(struct rte_pci_device *dev) -{ - int ret = -1; - - /* try mapping the NIC resources using VFIO if it exists */ - switch (dev->kdrv) { - case RTE_KDRV_VFIO: -#ifdef VFIO_PRESENT - if (pci_vfio_is_enabled()) - ret = pci_vfio_map_resource(dev); -#endif - break; - case RTE_KDRV_IGB_UIO: - case RTE_KDRV_UIO_GENERIC: - /* map resources for devices that use uio */ - ret = pci_uio_map_resource(dev); - break; - default: - RTE_LOG(DEBUG, EAL, - " Not managed by a supported kernel driver, skipped\n"); - ret = 1; - break; - } - - return ret; -} - -/* Unmap pci device */ -void -rte_eal_pci_unmap_device(struct rte_pci_device *dev) -{ - /* try unmapping the NIC resources using VFIO if it exists */ - switch (dev->kdrv) { - case RTE_KDRV_VFIO: - RTE_LOG(ERR, EAL, "Hotplug doesn't support vfio yet\n"); - break; - case RTE_KDRV_IGB_UIO: - case RTE_KDRV_UIO_GENERIC: - /* unmap resources for devices that use uio */ - pci_uio_unmap_resource(dev); - break; - default: - RTE_LOG(DEBUG, EAL, - " Not managed by a supported kernel driver, skipped\n"); - break; - } -} - -void * -pci_find_max_end_va(void) -{ - const struct rte_memseg *seg = rte_eal_get_physmem_layout(); - const struct rte_memseg *last = seg; - unsigned i = 0; - - for (i = 0; i < RTE_MAX_MEMSEG; i++, seg++) { - if (seg->addr == NULL) - break; - - if (seg->addr > last->addr) - last = seg; - - } - return RTE_PTR_ADD(last->addr, last->len); -} - -/* parse one line of the "resource" sysfs file (note that the 'line' - * string is modified) - */ -int -pci_parse_one_sysfs_resource(char *line, size_t len, uint64_t *phys_addr, - uint64_t *end_addr, uint64_t *flags) -{ - union pci_resource_info { - struct { - char *phys_addr; - char *end_addr; - char *flags; - }; - char *ptrs[PCI_RESOURCE_FMT_NVAL]; - } res_info; - - if (rte_strsplit(line, len, res_info.ptrs, 3, ' ') != 3) { - RTE_LOG(ERR, EAL, - "%s(): bad resource format\n", __func__); - return -1; - } - errno = 0; - *phys_addr = strtoull(res_info.phys_addr, NULL, 16); - *end_addr = strtoull(res_info.end_addr, NULL, 16); - *flags = strtoull(res_info.flags, NULL, 16); - if (errno != 0) { - RTE_LOG(ERR, EAL, - "%s(): bad resource format\n", __func__); - return -1; - } - - return 0; -} - -/* parse the "resource" sysfs file */ -static int -pci_parse_sysfs_resource(const char *filename, struct rte_pci_device *dev) -{ - FILE *f; - char buf[BUFSIZ]; - int i; - uint64_t phys_addr, end_addr, flags; - - f = fopen(filename, "r"); - if (f == NULL) { - RTE_LOG(ERR, EAL, "Cannot open sysfs resource\n"); - return -1; - } - - for (i = 0; imem_resource[i].phys_addr = phys_addr; - dev->mem_resource[i].len = end_addr - phys_addr + 1; - /* not mapped for now */ - dev->mem_resource[i].addr = NULL; - } - } - fclose(f); - return 0; - -error: - fclose(f); - return -1; -} - -/* Scan one pci sysfs entry, and fill the devices list from it. */ -static int -pci_scan_one(const char *dirname, uint16_t domain, uint8_t bus, - uint8_t devid, uint8_t function) -{ - char filename[PATH_MAX]; - unsigned long tmp; - struct rte_pci_device *dev; - char driver[PATH_MAX]; - int ret; - - dev = malloc(sizeof(*dev)); - if (dev == NULL) - return -1; - - memset(dev, 0, sizeof(*dev)); - dev->addr.domain = domain; - dev->addr.bus = bus; - dev->addr.devid = devid; - dev->addr.function = function; - - /* get vendor id */ - snprintf(filename, sizeof(filename), "%s/vendor", dirname); - if (eal_parse_sysfs_value(filename, &tmp) < 0) { - free(dev); - return -1; - } - dev->id.vendor_id = (uint16_t)tmp; - - /* get device id */ - snprintf(filename, sizeof(filename), "%s/device", dirname); - if (eal_parse_sysfs_value(filename, &tmp) < 0) { - free(dev); - return -1; - } - dev->id.device_id = (uint16_t)tmp; - - /* get subsystem_vendor id */ - snprintf(filename, sizeof(filename), "%s/subsystem_vendor", - dirname); - if (eal_parse_sysfs_value(filename, &tmp) < 0) { - free(dev); - return -1; - } - dev->id.subsystem_vendor_id = (uint16_t)tmp; - - /* get subsystem_device id */ - snprintf(filename, sizeof(filename), "%s/subsystem_device", - dirname); - if (eal_parse_sysfs_value(filename, &tmp) < 0) { - free(dev); - return -1; - } - dev->id.subsystem_device_id = (uint16_t)tmp; - - /* get class_id */ - snprintf(filename, sizeof(filename), "%s/class", - dirname); - if (eal_parse_sysfs_value(filename, &tmp) < 0) { - free(dev); - return -1; - } - /* the least 24 bits are valid: class, subclass, program interface */ - dev->id.class_id = (uint32_t)tmp & RTE_CLASS_ANY_ID; - - /* get max_vfs */ - dev->max_vfs = 0; - snprintf(filename, sizeof(filename), "%s/max_vfs", dirname); - if (!access(filename, F_OK) && - eal_parse_sysfs_value(filename, &tmp) == 0) - dev->max_vfs = (uint16_t)tmp; - else { - /* for non igb_uio driver, need kernel version >= 3.8 */ - snprintf(filename, sizeof(filename), - "%s/sriov_numvfs", dirname); - if (!access(filename, F_OK) && - eal_parse_sysfs_value(filename, &tmp) == 0) - dev->max_vfs = (uint16_t)tmp; - } - - /* get numa node */ - snprintf(filename, sizeof(filename), "%s/numa_node", - dirname); - if (eal_parse_sysfs_value(filename, &tmp) == 0 && - tmp < RTE_MAX_NUMA_NODES) { - dev->numa_node = tmp; - } else { - dev->numa_node = 0; - } - - /* parse resources */ - snprintf(filename, sizeof(filename), "%s/resource", dirname); - if (pci_parse_sysfs_resource(filename, dev) < 0) { - RTE_LOG(ERR, EAL, "%s(): cannot parse resource\n", __func__); - free(dev); - return -1; - } - - /* parse driver */ - snprintf(filename, sizeof(filename), "%s/driver", dirname); - ret = pci_get_kernel_driver_by_path(filename, driver); - if (ret < 0) { - RTE_LOG(ERR, EAL, "Fail to get kernel driver\n"); - free(dev); - return -1; - } - - if (!ret) { - if (!strcmp(driver, "vfio-pci")) - dev->kdrv = RTE_KDRV_VFIO; - else if (!strcmp(driver, "igb_uio")) - dev->kdrv = RTE_KDRV_IGB_UIO; - else if (!strcmp(driver, "uio_pci_generic")) - dev->kdrv = RTE_KDRV_UIO_GENERIC; - else - dev->kdrv = RTE_KDRV_UNKNOWN; - } else - dev->kdrv = RTE_KDRV_NONE; - - /* device is valid, add in list (sorted) */ - if (TAILQ_EMPTY(&pci_device_list)) { - TAILQ_INSERT_TAIL(&pci_device_list, dev, next); - } else { - struct rte_pci_device *dev2; - int ret; - - TAILQ_FOREACH(dev2, &pci_device_list, next) { - ret = rte_eal_compare_pci_addr(&dev->addr, &dev2->addr); - if (ret > 0) - continue; - - if (ret < 0) { - TAILQ_INSERT_BEFORE(dev2, dev, next); - } else { /* already registered */ - dev2->kdrv = dev->kdrv; - dev2->max_vfs = dev->max_vfs; - memmove(dev2->mem_resource, dev->mem_resource, - sizeof(dev->mem_resource)); - free(dev); - } - return 0; - } - TAILQ_INSERT_TAIL(&pci_device_list, dev, next); - } - - return 0; -} - -/* - * split up a pci address into its constituent parts. - */ -static int -parse_pci_addr_format(const char *buf, int bufsize, uint16_t *domain, - uint8_t *bus, uint8_t *devid, uint8_t *function) -{ - /* first split on ':' */ - union splitaddr { - struct { - char *domain; - char *bus; - char *devid; - char *function; - }; - char *str[PCI_FMT_NVAL]; /* last element-separator is "." not ":" */ - } splitaddr; - - char *buf_copy = strndup(buf, bufsize); - if (buf_copy == NULL) - return -1; - - if (rte_strsplit(buf_copy, bufsize, splitaddr.str, PCI_FMT_NVAL, ':') - != PCI_FMT_NVAL - 1) - goto error; - /* final split is on '.' between devid and function */ - splitaddr.function = strchr(splitaddr.devid,'.'); - if (splitaddr.function == NULL) - goto error; - *splitaddr.function++ = '\0'; - - /* now convert to int values */ - errno = 0; - *domain = (uint16_t)strtoul(splitaddr.domain, NULL, 16); - *bus = (uint8_t)strtoul(splitaddr.bus, NULL, 16); - *devid = (uint8_t)strtoul(splitaddr.devid, NULL, 16); - *function = (uint8_t)strtoul(splitaddr.function, NULL, 10); - if (errno != 0) - goto error; - - free(buf_copy); /* free the copy made with strdup */ - return 0; -error: - free(buf_copy); - return -1; -} - -/* - * Scan the content of the PCI bus, and the devices in the devices - * list - */ -int -rte_eal_pci_scan(void) -{ - struct dirent *e; - DIR *dir; - char dirname[PATH_MAX]; - uint16_t domain; - uint8_t bus, devid, function; - - dir = opendir(pci_get_sysfs_path()); - if (dir == NULL) { - RTE_LOG(ERR, EAL, "%s(): opendir failed: %s\n", - __func__, strerror(errno)); - return -1; - } - - while ((e = readdir(dir)) != NULL) { - if (e->d_name[0] == '.') - continue; - - if (parse_pci_addr_format(e->d_name, sizeof(e->d_name), &domain, - &bus, &devid, &function) != 0) - continue; - - snprintf(dirname, sizeof(dirname), "%s/%s", - pci_get_sysfs_path(), e->d_name); - if (pci_scan_one(dirname, domain, bus, devid, function) < 0) - goto error; - } - closedir(dir); - return 0; - -error: - closedir(dir); - return -1; -} - -/* Read PCI config space. */ -int rte_eal_pci_read_config(const struct rte_pci_device *device, - void *buf, size_t len, off_t offset) -{ - const struct rte_intr_handle *intr_handle = &device->intr_handle; - - switch (intr_handle->type) { - case RTE_INTR_HANDLE_UIO: - case RTE_INTR_HANDLE_UIO_INTX: - return pci_uio_read_config(intr_handle, buf, len, offset); - -#ifdef VFIO_PRESENT - case RTE_INTR_HANDLE_VFIO_MSIX: - case RTE_INTR_HANDLE_VFIO_MSI: - case RTE_INTR_HANDLE_VFIO_LEGACY: - return pci_vfio_read_config(intr_handle, buf, len, offset); -#endif - default: - RTE_LOG(ERR, EAL, - "Unknown handle type of fd %d\n", - intr_handle->fd); - return -1; - } -} - -/* Write PCI config space. */ -int rte_eal_pci_write_config(const struct rte_pci_device *device, - const void *buf, size_t len, off_t offset) -{ - const struct rte_intr_handle *intr_handle = &device->intr_handle; - - switch (intr_handle->type) { - case RTE_INTR_HANDLE_UIO: - case RTE_INTR_HANDLE_UIO_INTX: - return pci_uio_write_config(intr_handle, buf, len, offset); - -#ifdef VFIO_PRESENT - case RTE_INTR_HANDLE_VFIO_MSIX: - case RTE_INTR_HANDLE_VFIO_MSI: - case RTE_INTR_HANDLE_VFIO_LEGACY: - return pci_vfio_write_config(intr_handle, buf, len, offset); -#endif - default: - RTE_LOG(ERR, EAL, - "Unknown handle type of fd %d\n", - intr_handle->fd); - return -1; - } -} - -#if defined(RTE_ARCH_X86) -static int -pci_ioport_map(struct rte_pci_device *dev, int bar __rte_unused, - struct rte_pci_ioport *p) -{ - uint16_t start, end; - FILE *fp; - char *line = NULL; - char pci_id[16]; - int found = 0; - size_t linesz; - - snprintf(pci_id, sizeof(pci_id), PCI_PRI_FMT, - dev->addr.domain, dev->addr.bus, - dev->addr.devid, dev->addr.function); - - fp = fopen("/proc/ioports", "r"); - if (fp == NULL) { - RTE_LOG(ERR, EAL, "%s(): can't open ioports\n", __func__); - return -1; - } - - while (getdelim(&line, &linesz, '\n', fp) > 0) { - char *ptr = line; - char *left; - int n; - - n = strcspn(ptr, ":"); - ptr[n] = 0; - left = &ptr[n + 1]; - - while (*left && isspace(*left)) - left++; - - if (!strncmp(left, pci_id, strlen(pci_id))) { - found = 1; - - while (*ptr && isspace(*ptr)) - ptr++; - - sscanf(ptr, "%04hx-%04hx", &start, &end); - - break; - } - } - - free(line); - fclose(fp); - - if (!found) - return -1; - - dev->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN; - p->base = start; - RTE_LOG(DEBUG, EAL, "PCI Port IO found start=0x%x\n", start); - - return 0; -} -#endif - -int -rte_eal_pci_ioport_map(struct rte_pci_device *dev, int bar, - struct rte_pci_ioport *p) -{ - int ret = -1; - - switch (dev->kdrv) { -#ifdef VFIO_PRESENT - case RTE_KDRV_VFIO: - if (pci_vfio_is_enabled()) - ret = pci_vfio_ioport_map(dev, bar, p); - break; -#endif - case RTE_KDRV_IGB_UIO: - ret = pci_uio_ioport_map(dev, bar, p); - break; - case RTE_KDRV_UIO_GENERIC: -#if defined(RTE_ARCH_X86) - ret = pci_ioport_map(dev, bar, p); -#else - ret = pci_uio_ioport_map(dev, bar, p); -#endif - break; - case RTE_KDRV_NONE: -#if defined(RTE_ARCH_X86) - ret = pci_ioport_map(dev, bar, p); -#endif - break; - default: - break; - } - - if (!ret) - p->dev = dev; - - return ret; -} - -void -rte_eal_pci_ioport_read(struct rte_pci_ioport *p, - void *data, size_t len, off_t offset) -{ - switch (p->dev->kdrv) { -#ifdef VFIO_PRESENT - case RTE_KDRV_VFIO: - pci_vfio_ioport_read(p, data, len, offset); - break; -#endif - case RTE_KDRV_IGB_UIO: - pci_uio_ioport_read(p, data, len, offset); - break; - case RTE_KDRV_UIO_GENERIC: - pci_uio_ioport_read(p, data, len, offset); - break; - case RTE_KDRV_NONE: -#if defined(RTE_ARCH_X86) - pci_uio_ioport_read(p, data, len, offset); -#endif - break; - default: - break; - } -} - -void -rte_eal_pci_ioport_write(struct rte_pci_ioport *p, - const void *data, size_t len, off_t offset) -{ - switch (p->dev->kdrv) { -#ifdef VFIO_PRESENT - case RTE_KDRV_VFIO: - pci_vfio_ioport_write(p, data, len, offset); - break; -#endif - case RTE_KDRV_IGB_UIO: - pci_uio_ioport_write(p, data, len, offset); - break; - case RTE_KDRV_UIO_GENERIC: - pci_uio_ioport_write(p, data, len, offset); - break; - case RTE_KDRV_NONE: -#if defined(RTE_ARCH_X86) - pci_uio_ioport_write(p, data, len, offset); -#endif - break; - default: - break; - } -} - -int -rte_eal_pci_ioport_unmap(struct rte_pci_ioport *p) -{ - int ret = -1; - - switch (p->dev->kdrv) { -#ifdef VFIO_PRESENT - case RTE_KDRV_VFIO: - if (pci_vfio_is_enabled()) - ret = pci_vfio_ioport_unmap(p); - break; -#endif - case RTE_KDRV_IGB_UIO: - ret = pci_uio_ioport_unmap(p); - break; - case RTE_KDRV_UIO_GENERIC: -#if defined(RTE_ARCH_X86) - ret = 0; -#else - ret = pci_uio_ioport_unmap(p); -#endif - break; - case RTE_KDRV_NONE: -#if defined(RTE_ARCH_X86) - ret = 0; -#endif - break; - default: - break; - } - - return ret; -} - -/* Init the PCI EAL subsystem */ -int -rte_eal_pci_init(void) -{ - TAILQ_INIT(&pci_driver_list); - TAILQ_INIT(&pci_device_list); - - /* for debug purposes, PCI can be disabled */ - if (internal_config.no_pci) - return 0; - - if (rte_eal_pci_scan() < 0) { - RTE_LOG(ERR, EAL, "%s(): Cannot scan PCI bus\n", __func__); - return -1; - } - - return 0; -} diff --git a/dpdk/lib/librte_eal/linuxapp/eal/eal_pci_init.h b/dpdk/lib/librte_eal/linuxapp/eal/eal_pci_init.h deleted file mode 100644 index 6a960d1b..00000000 --- a/dpdk/lib/librte_eal/linuxapp/eal/eal_pci_init.h +++ /dev/null @@ -1,96 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef EAL_PCI_INIT_H_ -#define EAL_PCI_INIT_H_ - -#include "eal_vfio.h" - -/** IO resource type: */ -#define IORESOURCE_IO 0x00000100 -#define IORESOURCE_MEM 0x00000200 - -/* - * Helper function to map PCI resources right after hugepages in virtual memory - */ -extern void *pci_map_addr; -void *pci_find_max_end_va(void); - -/* parse one line of the "resource" sysfs file (note that the 'line' - * string is modified) - */ -int pci_parse_one_sysfs_resource(char *line, size_t len, uint64_t *phys_addr, - uint64_t *end_addr, uint64_t *flags); - -int pci_uio_alloc_resource(struct rte_pci_device *dev, - struct mapped_pci_resource **uio_res); -void pci_uio_free_resource(struct rte_pci_device *dev, - struct mapped_pci_resource *uio_res); -int pci_uio_map_resource_by_index(struct rte_pci_device *dev, int res_idx, - struct mapped_pci_resource *uio_res, int map_idx); - -int pci_uio_read_config(const struct rte_intr_handle *intr_handle, - void *buf, size_t len, off_t offs); -int pci_uio_write_config(const struct rte_intr_handle *intr_handle, - const void *buf, size_t len, off_t offs); - -int pci_uio_ioport_map(struct rte_pci_device *dev, int bar, - struct rte_pci_ioport *p); -void pci_uio_ioport_read(struct rte_pci_ioport *p, - void *data, size_t len, off_t offset); -void pci_uio_ioport_write(struct rte_pci_ioport *p, - const void *data, size_t len, off_t offset); -int pci_uio_ioport_unmap(struct rte_pci_ioport *p); - -#ifdef VFIO_PRESENT - -/* access config space */ -int pci_vfio_read_config(const struct rte_intr_handle *intr_handle, - void *buf, size_t len, off_t offs); -int pci_vfio_write_config(const struct rte_intr_handle *intr_handle, - const void *buf, size_t len, off_t offs); - -int pci_vfio_ioport_map(struct rte_pci_device *dev, int bar, - struct rte_pci_ioport *p); -void pci_vfio_ioport_read(struct rte_pci_ioport *p, - void *data, size_t len, off_t offset); -void pci_vfio_ioport_write(struct rte_pci_ioport *p, - const void *data, size_t len, off_t offset); -int pci_vfio_ioport_unmap(struct rte_pci_ioport *p); - -/* map VFIO resource prototype */ -int pci_vfio_map_resource(struct rte_pci_device *dev); - -#endif - -#endif /* EAL_PCI_INIT_H_ */ diff --git a/dpdk/lib/librte_eal/linuxapp/eal/eal_pci_uio.c b/dpdk/lib/librte_eal/linuxapp/eal/eal_pci_uio.c deleted file mode 100644 index 1786b754..00000000 --- a/dpdk/lib/librte_eal/linuxapp/eal/eal_pci_uio.c +++ /dev/null @@ -1,566 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#if defined(RTE_ARCH_X86) -#include -#endif - -#include -#include -#include -#include -#include - -#include "eal_filesystem.h" -#include "eal_pci_init.h" - -void *pci_map_addr = NULL; - -#define OFF_MAX ((uint64_t)(off_t)-1) - -int -pci_uio_read_config(const struct rte_intr_handle *intr_handle, - void *buf, size_t len, off_t offset) -{ - return pread(intr_handle->uio_cfg_fd, buf, len, offset); -} - -int -pci_uio_write_config(const struct rte_intr_handle *intr_handle, - const void *buf, size_t len, off_t offset) -{ - return pwrite(intr_handle->uio_cfg_fd, buf, len, offset); -} - -static int -pci_uio_set_bus_master(int dev_fd) -{ - uint16_t reg; - int ret; - - ret = pread(dev_fd, ®, sizeof(reg), PCI_COMMAND); - if (ret != sizeof(reg)) { - RTE_LOG(ERR, EAL, - "Cannot read command from PCI config space!\n"); - return -1; - } - - /* return if bus mastering is already on */ - if (reg & PCI_COMMAND_MASTER) - return 0; - - reg |= PCI_COMMAND_MASTER; - - ret = pwrite(dev_fd, ®, sizeof(reg), PCI_COMMAND); - if (ret != sizeof(reg)) { - RTE_LOG(ERR, EAL, - "Cannot write command to PCI config space!\n"); - return -1; - } - - return 0; -} - -static int -pci_mknod_uio_dev(const char *sysfs_uio_path, unsigned uio_num) -{ - FILE *f; - char filename[PATH_MAX]; - int ret; - unsigned major, minor; - dev_t dev; - - /* get the name of the sysfs file that contains the major and minor - * of the uio device and read its content */ - snprintf(filename, sizeof(filename), "%s/dev", sysfs_uio_path); - - f = fopen(filename, "r"); - if (f == NULL) { - RTE_LOG(ERR, EAL, "%s(): cannot open sysfs to get major:minor\n", - __func__); - return -1; - } - - ret = fscanf(f, "%u:%u", &major, &minor); - if (ret != 2) { - RTE_LOG(ERR, EAL, "%s(): cannot parse sysfs to get major:minor\n", - __func__); - fclose(f); - return -1; - } - fclose(f); - - /* create the char device "mknod /dev/uioX c major minor" */ - snprintf(filename, sizeof(filename), "/dev/uio%u", uio_num); - dev = makedev(major, minor); - ret = mknod(filename, S_IFCHR | S_IRUSR | S_IWUSR, dev); - if (f == NULL) { - RTE_LOG(ERR, EAL, "%s(): mknod() failed %s\n", - __func__, strerror(errno)); - return -1; - } - - return ret; -} - -/* - * Return the uioX char device used for a pci device. On success, return - * the UIO number and fill dstbuf string with the path of the device in - * sysfs. On error, return a negative value. In this case dstbuf is - * invalid. - */ -static int -pci_get_uio_dev(struct rte_pci_device *dev, char *dstbuf, - unsigned int buflen, int create) -{ - struct rte_pci_addr *loc = &dev->addr; - unsigned int uio_num; - struct dirent *e; - DIR *dir; - char dirname[PATH_MAX]; - - /* depending on kernel version, uio can be located in uio/uioX - * or uio:uioX */ - - snprintf(dirname, sizeof(dirname), - "%s/" PCI_PRI_FMT "/uio", pci_get_sysfs_path(), - loc->domain, loc->bus, loc->devid, loc->function); - - dir = opendir(dirname); - if (dir == NULL) { - /* retry with the parent directory */ - snprintf(dirname, sizeof(dirname), - "%s/" PCI_PRI_FMT, pci_get_sysfs_path(), - loc->domain, loc->bus, loc->devid, loc->function); - dir = opendir(dirname); - - if (dir == NULL) { - RTE_LOG(ERR, EAL, "Cannot opendir %s\n", dirname); - return -1; - } - } - - /* take the first file starting with "uio" */ - while ((e = readdir(dir)) != NULL) { - /* format could be uio%d ...*/ - int shortprefix_len = sizeof("uio") - 1; - /* ... or uio:uio%d */ - int longprefix_len = sizeof("uio:uio") - 1; - char *endptr; - - if (strncmp(e->d_name, "uio", 3) != 0) - continue; - - /* first try uio%d */ - errno = 0; - uio_num = strtoull(e->d_name + shortprefix_len, &endptr, 10); - if (errno == 0 && endptr != (e->d_name + shortprefix_len)) { - snprintf(dstbuf, buflen, "%s/uio%u", dirname, uio_num); - break; - } - - /* then try uio:uio%d */ - errno = 0; - uio_num = strtoull(e->d_name + longprefix_len, &endptr, 10); - if (errno == 0 && endptr != (e->d_name + longprefix_len)) { - snprintf(dstbuf, buflen, "%s/uio:uio%u", dirname, uio_num); - break; - } - } - closedir(dir); - - /* No uio resource found */ - if (e == NULL) - return -1; - - /* create uio device if we've been asked to */ - if (internal_config.create_uio_dev && create && - pci_mknod_uio_dev(dstbuf, uio_num) < 0) - RTE_LOG(WARNING, EAL, "Cannot create /dev/uio%u\n", uio_num); - - return uio_num; -} - -void -pci_uio_free_resource(struct rte_pci_device *dev, - struct mapped_pci_resource *uio_res) -{ - rte_free(uio_res); - - if (dev->intr_handle.uio_cfg_fd >= 0) { - close(dev->intr_handle.uio_cfg_fd); - dev->intr_handle.uio_cfg_fd = -1; - } - if (dev->intr_handle.fd) { - close(dev->intr_handle.fd); - dev->intr_handle.fd = -1; - dev->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN; - } -} - -int -pci_uio_alloc_resource(struct rte_pci_device *dev, - struct mapped_pci_resource **uio_res) -{ - char dirname[PATH_MAX]; - char cfgname[PATH_MAX]; - char devname[PATH_MAX]; /* contains the /dev/uioX */ - int uio_num; - struct rte_pci_addr *loc; - - loc = &dev->addr; - - /* find uio resource */ - uio_num = pci_get_uio_dev(dev, dirname, sizeof(dirname), 1); - if (uio_num < 0) { - RTE_LOG(WARNING, EAL, " "PCI_PRI_FMT" not managed by UIO driver, " - "skipping\n", loc->domain, loc->bus, loc->devid, loc->function); - return 1; - } - snprintf(devname, sizeof(devname), "/dev/uio%u", uio_num); - - /* save fd if in primary process */ - dev->intr_handle.fd = open(devname, O_RDWR); - if (dev->intr_handle.fd < 0) { - RTE_LOG(ERR, EAL, "Cannot open %s: %s\n", - devname, strerror(errno)); - goto error; - } - - snprintf(cfgname, sizeof(cfgname), - "/sys/class/uio/uio%u/device/config", uio_num); - dev->intr_handle.uio_cfg_fd = open(cfgname, O_RDWR); - if (dev->intr_handle.uio_cfg_fd < 0) { - RTE_LOG(ERR, EAL, "Cannot open %s: %s\n", - cfgname, strerror(errno)); - goto error; - } - - if (dev->kdrv == RTE_KDRV_IGB_UIO) - dev->intr_handle.type = RTE_INTR_HANDLE_UIO; - else { - dev->intr_handle.type = RTE_INTR_HANDLE_UIO_INTX; - - /* set bus master that is not done by uio_pci_generic */ - if (pci_uio_set_bus_master(dev->intr_handle.uio_cfg_fd)) { - RTE_LOG(ERR, EAL, "Cannot set up bus mastering!\n"); - goto error; - } - } - - /* allocate the mapping details for secondary processes*/ - *uio_res = rte_zmalloc("UIO_RES", sizeof(**uio_res), 0); - if (*uio_res == NULL) { - RTE_LOG(ERR, EAL, - "%s(): cannot store uio mmap details\n", __func__); - goto error; - } - - snprintf((*uio_res)->path, sizeof((*uio_res)->path), "%s", devname); - memcpy(&(*uio_res)->pci_addr, &dev->addr, sizeof((*uio_res)->pci_addr)); - - return 0; - -error: - pci_uio_free_resource(dev, *uio_res); - return -1; -} - -int -pci_uio_map_resource_by_index(struct rte_pci_device *dev, int res_idx, - struct mapped_pci_resource *uio_res, int map_idx) -{ - int fd; - char devname[PATH_MAX]; - void *mapaddr; - struct rte_pci_addr *loc; - struct pci_map *maps; - - loc = &dev->addr; - maps = uio_res->maps; - - /* update devname for mmap */ - snprintf(devname, sizeof(devname), - "%s/" PCI_PRI_FMT "/resource%d", - pci_get_sysfs_path(), - loc->domain, loc->bus, loc->devid, - loc->function, res_idx); - - /* allocate memory to keep path */ - maps[map_idx].path = rte_malloc(NULL, strlen(devname) + 1, 0); - if (maps[map_idx].path == NULL) { - RTE_LOG(ERR, EAL, "Cannot allocate memory for path: %s\n", - strerror(errno)); - return -1; - } - - /* - * open resource file, to mmap it - */ - fd = open(devname, O_RDWR); - if (fd < 0) { - RTE_LOG(ERR, EAL, "Cannot open %s: %s\n", - devname, strerror(errno)); - goto error; - } - - /* try mapping somewhere close to the end of hugepages */ - if (pci_map_addr == NULL) - pci_map_addr = pci_find_max_end_va(); - - mapaddr = pci_map_resource(pci_map_addr, fd, 0, - (size_t)dev->mem_resource[res_idx].len, 0); - close(fd); - if (mapaddr == MAP_FAILED) - goto error; - - pci_map_addr = RTE_PTR_ADD(mapaddr, - (size_t)dev->mem_resource[res_idx].len); - - maps[map_idx].phaddr = dev->mem_resource[res_idx].phys_addr; - maps[map_idx].size = dev->mem_resource[res_idx].len; - maps[map_idx].addr = mapaddr; - maps[map_idx].offset = 0; - strcpy(maps[map_idx].path, devname); - dev->mem_resource[res_idx].addr = mapaddr; - - return 0; - -error: - rte_free(maps[map_idx].path); - return -1; -} - -#if defined(RTE_ARCH_X86) -int -pci_uio_ioport_map(struct rte_pci_device *dev, int bar, - struct rte_pci_ioport *p) -{ - char dirname[PATH_MAX]; - char filename[PATH_MAX]; - int uio_num; - unsigned long start; - - uio_num = pci_get_uio_dev(dev, dirname, sizeof(dirname), 0); - if (uio_num < 0) - return -1; - - /* get portio start */ - snprintf(filename, sizeof(filename), - "%s/portio/port%d/start", dirname, bar); - if (eal_parse_sysfs_value(filename, &start) < 0) { - RTE_LOG(ERR, EAL, "%s(): cannot parse portio start\n", - __func__); - return -1; - } - /* ensure we don't get anything funny here, read/write will cast to - * uin16_t */ - if (start > UINT16_MAX) - return -1; - - /* FIXME only for primary process ? */ - if (dev->intr_handle.type == RTE_INTR_HANDLE_UNKNOWN) { - - snprintf(filename, sizeof(filename), "/dev/uio%u", uio_num); - dev->intr_handle.fd = open(filename, O_RDWR); - if (dev->intr_handle.fd < 0) { - RTE_LOG(ERR, EAL, "Cannot open %s: %s\n", - filename, strerror(errno)); - return -1; - } - dev->intr_handle.type = RTE_INTR_HANDLE_UIO; - } - - RTE_LOG(DEBUG, EAL, "PCI Port IO found start=0x%lx\n", start); - - p->base = start; - p->len = 0; - return 0; -} -#else -int -pci_uio_ioport_map(struct rte_pci_device *dev, int bar, - struct rte_pci_ioport *p) -{ - FILE *f; - char buf[BUFSIZ]; - char filename[PATH_MAX]; - uint64_t phys_addr, end_addr, flags; - int fd, i; - void *addr; - - /* open and read addresses of the corresponding resource in sysfs */ - snprintf(filename, sizeof(filename), "%s/" PCI_PRI_FMT "/resource", - pci_get_sysfs_path(), dev->addr.domain, dev->addr.bus, - dev->addr.devid, dev->addr.function); - f = fopen(filename, "r"); - if (f == NULL) { - RTE_LOG(ERR, EAL, "Cannot open sysfs resource: %s\n", - strerror(errno)); - return -1; - } - for (i = 0; i < bar + 1; i++) { - if (fgets(buf, sizeof(buf), f) == NULL) { - RTE_LOG(ERR, EAL, "Cannot read sysfs resource\n"); - goto error; - } - } - if (pci_parse_one_sysfs_resource(buf, sizeof(buf), &phys_addr, - &end_addr, &flags) < 0) - goto error; - if ((flags & IORESOURCE_IO) == 0) { - RTE_LOG(ERR, EAL, "BAR %d is not an IO resource\n", bar); - goto error; - } - snprintf(filename, sizeof(filename), "%s/" PCI_PRI_FMT "/resource%d", - pci_get_sysfs_path(), dev->addr.domain, dev->addr.bus, - dev->addr.devid, dev->addr.function, bar); - - /* mmap the pci resource */ - fd = open(filename, O_RDWR); - if (fd < 0) { - RTE_LOG(ERR, EAL, "Cannot open %s: %s\n", filename, - strerror(errno)); - goto error; - } - addr = mmap(NULL, end_addr + 1, PROT_READ | PROT_WRITE, - MAP_SHARED, fd, 0); - close(fd); - if (addr == MAP_FAILED) { - RTE_LOG(ERR, EAL, "Cannot mmap IO port resource: %s\n", - strerror(errno)); - goto error; - } - - /* strangely, the base address is mmap addr + phys_addr */ - p->base = (uintptr_t)addr + phys_addr; - p->len = end_addr + 1; - RTE_LOG(DEBUG, EAL, "PCI Port IO found start=0x%"PRIx64"\n", p->base); - fclose(f); - - return 0; - -error: - fclose(f); - return -1; -} -#endif - -void -pci_uio_ioport_read(struct rte_pci_ioport *p, - void *data, size_t len, off_t offset) -{ - uint8_t *d; - int size; - uintptr_t reg = p->base + offset; - - for (d = data; len > 0; d += size, reg += size, len -= size) { - if (len >= 4) { - size = 4; -#if defined(RTE_ARCH_X86) - *(uint32_t *)d = inl(reg); -#else - *(uint32_t *)d = *(volatile uint32_t *)reg; -#endif - } else if (len >= 2) { - size = 2; -#if defined(RTE_ARCH_X86) - *(uint16_t *)d = inw(reg); -#else - *(uint16_t *)d = *(volatile uint16_t *)reg; -#endif - } else { - size = 1; -#if defined(RTE_ARCH_X86) - *d = inb(reg); -#else - *d = *(volatile uint8_t *)reg; -#endif - } - } -} - -void -pci_uio_ioport_write(struct rte_pci_ioport *p, - const void *data, size_t len, off_t offset) -{ - const uint8_t *s; - int size; - uintptr_t reg = p->base + offset; - - for (s = data; len > 0; s += size, reg += size, len -= size) { - if (len >= 4) { - size = 4; -#if defined(RTE_ARCH_X86) - outl_p(*(const uint32_t *)s, reg); -#else - *(volatile uint32_t *)reg = *(const uint32_t *)s; -#endif - } else if (len >= 2) { - size = 2; -#if defined(RTE_ARCH_X86) - outw_p(*(const uint16_t *)s, reg); -#else - *(volatile uint16_t *)reg = *(const uint16_t *)s; -#endif - } else { - size = 1; -#if defined(RTE_ARCH_X86) - outb_p(*s, reg); -#else - *(volatile uint8_t *)reg = *s; -#endif - } - } -} - -int -pci_uio_ioport_unmap(struct rte_pci_ioport *p) -{ -#if defined(RTE_ARCH_X86) - RTE_SET_USED(p); - /* FIXME close intr fd ? */ - return 0; -#else - return munmap((void *)(uintptr_t)p->base, p->len); -#endif -} diff --git a/dpdk/lib/librte_eal/linuxapp/eal/eal_pci_vfio.c b/dpdk/lib/librte_eal/linuxapp/eal/eal_pci_vfio.c deleted file mode 100644 index 5f478c59..00000000 --- a/dpdk/lib/librte_eal/linuxapp/eal/eal_pci_vfio.c +++ /dev/null @@ -1,596 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include "eal_filesystem.h" -#include "eal_pci_init.h" -#include "eal_vfio.h" -#include "eal_private.h" - -/** - * @file - * PCI probing under linux (VFIO version) - * - * This code tries to determine if the PCI device is bound to VFIO driver, - * and initialize it (map BARs, set up interrupts) if that's the case. - * - * This file is only compiled if CONFIG_RTE_EAL_VFIO is set to "y". - */ - -#ifdef VFIO_PRESENT - -#define PAGE_SIZE (sysconf(_SC_PAGESIZE)) -#define PAGE_MASK (~(PAGE_SIZE - 1)) - -static struct rte_tailq_elem rte_vfio_tailq = { - .name = "VFIO_RESOURCE_LIST", -}; -EAL_REGISTER_TAILQ(rte_vfio_tailq) - -int -pci_vfio_read_config(const struct rte_intr_handle *intr_handle, - void *buf, size_t len, off_t offs) -{ - return pread64(intr_handle->vfio_dev_fd, buf, len, - VFIO_GET_REGION_ADDR(VFIO_PCI_CONFIG_REGION_INDEX) + offs); -} - -int -pci_vfio_write_config(const struct rte_intr_handle *intr_handle, - const void *buf, size_t len, off_t offs) -{ - return pwrite64(intr_handle->vfio_dev_fd, buf, len, - VFIO_GET_REGION_ADDR(VFIO_PCI_CONFIG_REGION_INDEX) + offs); -} - -/* get PCI BAR number where MSI-X interrupts are */ -static int -pci_vfio_get_msix_bar(int fd, int *msix_bar, uint32_t *msix_table_offset, - uint32_t *msix_table_size) -{ - int ret; - uint32_t reg; - uint16_t flags; - uint8_t cap_id, cap_offset; - - /* read PCI capability pointer from config space */ - ret = pread64(fd, ®, sizeof(reg), - VFIO_GET_REGION_ADDR(VFIO_PCI_CONFIG_REGION_INDEX) + - PCI_CAPABILITY_LIST); - if (ret != sizeof(reg)) { - RTE_LOG(ERR, EAL, "Cannot read capability pointer from PCI " - "config space!\n"); - return -1; - } - - /* we need first byte */ - cap_offset = reg & 0xFF; - - while (cap_offset) { - - /* read PCI capability ID */ - ret = pread64(fd, ®, sizeof(reg), - VFIO_GET_REGION_ADDR(VFIO_PCI_CONFIG_REGION_INDEX) + - cap_offset); - if (ret != sizeof(reg)) { - RTE_LOG(ERR, EAL, "Cannot read capability ID from PCI " - "config space!\n"); - return -1; - } - - /* we need first byte */ - cap_id = reg & 0xFF; - - /* if we haven't reached MSI-X, check next capability */ - if (cap_id != PCI_CAP_ID_MSIX) { - ret = pread64(fd, ®, sizeof(reg), - VFIO_GET_REGION_ADDR(VFIO_PCI_CONFIG_REGION_INDEX) + - cap_offset); - if (ret != sizeof(reg)) { - RTE_LOG(ERR, EAL, "Cannot read capability pointer from PCI " - "config space!\n"); - return -1; - } - - /* we need second byte */ - cap_offset = (reg & 0xFF00) >> 8; - - continue; - } - /* else, read table offset */ - else { - /* table offset resides in the next 4 bytes */ - ret = pread64(fd, ®, sizeof(reg), - VFIO_GET_REGION_ADDR(VFIO_PCI_CONFIG_REGION_INDEX) + - cap_offset + 4); - if (ret != sizeof(reg)) { - RTE_LOG(ERR, EAL, "Cannot read table offset from PCI config " - "space!\n"); - return -1; - } - - ret = pread64(fd, &flags, sizeof(flags), - VFIO_GET_REGION_ADDR(VFIO_PCI_CONFIG_REGION_INDEX) + - cap_offset + 2); - if (ret != sizeof(flags)) { - RTE_LOG(ERR, EAL, "Cannot read table flags from PCI config " - "space!\n"); - return -1; - } - - *msix_bar = reg & RTE_PCI_MSIX_TABLE_BIR; - *msix_table_offset = reg & RTE_PCI_MSIX_TABLE_OFFSET; - *msix_table_size = 16 * (1 + (flags & RTE_PCI_MSIX_FLAGS_QSIZE)); - - return 0; - } - } - return 0; -} - -/* set PCI bus mastering */ -static int -pci_vfio_set_bus_master(int dev_fd) -{ - uint16_t reg; - int ret; - - ret = pread64(dev_fd, ®, sizeof(reg), - VFIO_GET_REGION_ADDR(VFIO_PCI_CONFIG_REGION_INDEX) + - PCI_COMMAND); - if (ret != sizeof(reg)) { - RTE_LOG(ERR, EAL, "Cannot read command from PCI config space!\n"); - return -1; - } - - /* set the master bit */ - reg |= PCI_COMMAND_MASTER; - - ret = pwrite64(dev_fd, ®, sizeof(reg), - VFIO_GET_REGION_ADDR(VFIO_PCI_CONFIG_REGION_INDEX) + - PCI_COMMAND); - - if (ret != sizeof(reg)) { - RTE_LOG(ERR, EAL, "Cannot write command to PCI config space!\n"); - return -1; - } - - return 0; -} - -/* set up interrupt support (but not enable interrupts) */ -static int -pci_vfio_setup_interrupts(struct rte_pci_device *dev, int vfio_dev_fd) -{ - int i, ret, intr_idx; - - /* default to invalid index */ - intr_idx = VFIO_PCI_NUM_IRQS; - - /* get interrupt type from internal config (MSI-X by default, can be - * overriden from the command line - */ - switch (internal_config.vfio_intr_mode) { - case RTE_INTR_MODE_MSIX: - intr_idx = VFIO_PCI_MSIX_IRQ_INDEX; - break; - case RTE_INTR_MODE_MSI: - intr_idx = VFIO_PCI_MSI_IRQ_INDEX; - break; - case RTE_INTR_MODE_LEGACY: - intr_idx = VFIO_PCI_INTX_IRQ_INDEX; - break; - /* don't do anything if we want to automatically determine interrupt type */ - case RTE_INTR_MODE_NONE: - break; - default: - RTE_LOG(ERR, EAL, " unknown default interrupt type!\n"); - return -1; - } - - /* start from MSI-X interrupt type */ - for (i = VFIO_PCI_MSIX_IRQ_INDEX; i >= 0; i--) { - struct vfio_irq_info irq = { .argsz = sizeof(irq) }; - int fd = -1; - - /* skip interrupt modes we don't want */ - if (internal_config.vfio_intr_mode != RTE_INTR_MODE_NONE && - i != intr_idx) - continue; - - irq.index = i; - - ret = ioctl(vfio_dev_fd, VFIO_DEVICE_GET_IRQ_INFO, &irq); - if (ret < 0) { - RTE_LOG(ERR, EAL, " cannot get IRQ info, " - "error %i (%s)\n", errno, strerror(errno)); - return -1; - } - - /* if this vector cannot be used with eventfd, fail if we explicitly - * specified interrupt type, otherwise continue */ - if ((irq.flags & VFIO_IRQ_INFO_EVENTFD) == 0) { - if (internal_config.vfio_intr_mode != RTE_INTR_MODE_NONE) { - RTE_LOG(ERR, EAL, - " interrupt vector does not support eventfd!\n"); - return -1; - } else - continue; - } - - /* set up an eventfd for interrupts */ - fd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC); - if (fd < 0) { - RTE_LOG(ERR, EAL, " cannot set up eventfd, " - "error %i (%s)\n", errno, strerror(errno)); - return -1; - } - - dev->intr_handle.fd = fd; - dev->intr_handle.vfio_dev_fd = vfio_dev_fd; - - switch (i) { - case VFIO_PCI_MSIX_IRQ_INDEX: - internal_config.vfio_intr_mode = RTE_INTR_MODE_MSIX; - dev->intr_handle.type = RTE_INTR_HANDLE_VFIO_MSIX; - break; - case VFIO_PCI_MSI_IRQ_INDEX: - internal_config.vfio_intr_mode = RTE_INTR_MODE_MSI; - dev->intr_handle.type = RTE_INTR_HANDLE_VFIO_MSI; - break; - case VFIO_PCI_INTX_IRQ_INDEX: - internal_config.vfio_intr_mode = RTE_INTR_MODE_LEGACY; - dev->intr_handle.type = RTE_INTR_HANDLE_VFIO_LEGACY; - break; - default: - RTE_LOG(ERR, EAL, " unknown interrupt type!\n"); - return -1; - } - - return 0; - } - - /* if we're here, we haven't found a suitable interrupt vector */ - return -1; -} - -/* - * map the PCI resources of a PCI device in virtual memory (VFIO version). - * primary and secondary processes follow almost exactly the same path - */ -int -pci_vfio_map_resource(struct rte_pci_device *dev) -{ - struct vfio_device_info device_info = { .argsz = sizeof(device_info) }; - char pci_addr[PATH_MAX] = {0}; - int vfio_dev_fd; - struct rte_pci_addr *loc = &dev->addr; - int i, ret, msix_bar; - struct mapped_pci_resource *vfio_res = NULL; - struct mapped_pci_res_list *vfio_res_list = RTE_TAILQ_CAST(rte_vfio_tailq.head, mapped_pci_res_list); - - struct pci_map *maps; - uint32_t msix_table_offset = 0; - uint32_t msix_table_size = 0; - uint32_t ioport_bar; - - dev->intr_handle.fd = -1; - dev->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN; - - /* store PCI address string */ - snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT, - loc->domain, loc->bus, loc->devid, loc->function); - - if ((ret = vfio_setup_device(pci_get_sysfs_path(), pci_addr, - &vfio_dev_fd, &device_info))) - return ret; - - /* get MSI-X BAR, if any (we have to know where it is because we can't - * easily mmap it when using VFIO) */ - msix_bar = -1; - ret = pci_vfio_get_msix_bar(vfio_dev_fd, &msix_bar, - &msix_table_offset, &msix_table_size); - if (ret < 0) { - RTE_LOG(ERR, EAL, " %s cannot get MSI-X BAR number!\n", pci_addr); - close(vfio_dev_fd); - return -1; - } - - /* if we're in a primary process, allocate vfio_res and get region info */ - if (internal_config.process_type == RTE_PROC_PRIMARY) { - vfio_res = rte_zmalloc("VFIO_RES", sizeof(*vfio_res), 0); - if (vfio_res == NULL) { - RTE_LOG(ERR, EAL, - "%s(): cannot store uio mmap details\n", __func__); - close(vfio_dev_fd); - return -1; - } - memcpy(&vfio_res->pci_addr, &dev->addr, sizeof(vfio_res->pci_addr)); - - /* get number of registers (up to BAR5) */ - vfio_res->nb_maps = RTE_MIN((int) device_info.num_regions, - VFIO_PCI_BAR5_REGION_INDEX + 1); - } else { - /* if we're in a secondary process, just find our tailq entry */ - TAILQ_FOREACH(vfio_res, vfio_res_list, next) { - if (memcmp(&vfio_res->pci_addr, &dev->addr, sizeof(dev->addr))) - continue; - break; - } - /* if we haven't found our tailq entry, something's wrong */ - if (vfio_res == NULL) { - RTE_LOG(ERR, EAL, " %s cannot find TAILQ entry for PCI device!\n", - pci_addr); - close(vfio_dev_fd); - return -1; - } - } - - /* map BARs */ - maps = vfio_res->maps; - - for (i = 0; i < (int) vfio_res->nb_maps; i++) { - struct vfio_region_info reg = { .argsz = sizeof(reg) }; - void *bar_addr; - struct memreg { - unsigned long offset, size; - } memreg[2] = {}; - - reg.index = i; - - ret = ioctl(vfio_dev_fd, VFIO_DEVICE_GET_REGION_INFO, ®); - - if (ret) { - RTE_LOG(ERR, EAL, " %s cannot get device region info " - "error %i (%s)\n", pci_addr, errno, strerror(errno)); - close(vfio_dev_fd); - if (internal_config.process_type == RTE_PROC_PRIMARY) - rte_free(vfio_res); - return -1; - } - - /* chk for io port region */ - ret = pread64(vfio_dev_fd, &ioport_bar, sizeof(ioport_bar), - VFIO_GET_REGION_ADDR(VFIO_PCI_CONFIG_REGION_INDEX) - + PCI_BASE_ADDRESS_0 + i*4); - - if (ret != sizeof(ioport_bar)) { - RTE_LOG(ERR, EAL, - "Cannot read command (%x) from config space!\n", - PCI_BASE_ADDRESS_0 + i*4); - return -1; - } - - if (ioport_bar & PCI_BASE_ADDRESS_SPACE_IO) { - RTE_LOG(INFO, EAL, - "Ignore mapping IO port bar(%d) addr: %x\n", - i, ioport_bar); - continue; - } - - /* skip non-mmapable BARs */ - if ((reg.flags & VFIO_REGION_INFO_FLAG_MMAP) == 0) - continue; - - if (i == msix_bar) { - /* - * VFIO will not let us map the MSI-X table, - * but we can map around it. - */ - uint32_t table_start = msix_table_offset; - uint32_t table_end = table_start + msix_table_size; - table_end = (table_end + ~PAGE_MASK) & PAGE_MASK; - table_start &= PAGE_MASK; - - if (table_start == 0 && table_end >= reg.size) { - /* Cannot map this BAR */ - RTE_LOG(DEBUG, EAL, "Skipping BAR %d\n", i); - continue; - } else { - memreg[0].offset = reg.offset; - memreg[0].size = table_start; - memreg[1].offset = reg.offset + table_end; - memreg[1].size = reg.size - table_end; - - RTE_LOG(DEBUG, EAL, - "Trying to map BAR %d that contains the MSI-X " - "table. Trying offsets: " - "0x%04lx:0x%04lx, 0x%04lx:0x%04lx\n", i, - memreg[0].offset, memreg[0].size, - memreg[1].offset, memreg[1].size); - } - } else { - memreg[0].offset = reg.offset; - memreg[0].size = reg.size; - } - - /* try to figure out an address */ - if (internal_config.process_type == RTE_PROC_PRIMARY) { - /* try mapping somewhere close to the end of hugepages */ - if (pci_map_addr == NULL) - pci_map_addr = pci_find_max_end_va(); - - bar_addr = pci_map_addr; - pci_map_addr = RTE_PTR_ADD(bar_addr, (size_t) reg.size); - } else { - bar_addr = maps[i].addr; - } - - /* reserve the address using an inaccessible mapping */ - bar_addr = mmap(bar_addr, reg.size, 0, MAP_PRIVATE | - MAP_ANONYMOUS, -1, 0); - if (bar_addr != MAP_FAILED) { - void *map_addr = NULL; - if (memreg[0].size) { - /* actual map of first part */ - map_addr = pci_map_resource(bar_addr, vfio_dev_fd, - memreg[0].offset, - memreg[0].size, - MAP_FIXED); - } - - /* if there's a second part, try to map it */ - if (map_addr != MAP_FAILED - && memreg[1].offset && memreg[1].size) { - void *second_addr = RTE_PTR_ADD(bar_addr, - memreg[1].offset - - (uintptr_t)reg.offset); - map_addr = pci_map_resource(second_addr, - vfio_dev_fd, memreg[1].offset, - memreg[1].size, - MAP_FIXED); - } - - if (map_addr == MAP_FAILED || !map_addr) { - munmap(bar_addr, reg.size); - bar_addr = MAP_FAILED; - } - } - - if (bar_addr == MAP_FAILED || - (internal_config.process_type == RTE_PROC_SECONDARY && - bar_addr != maps[i].addr)) { - RTE_LOG(ERR, EAL, " %s mapping BAR%i failed: %s\n", pci_addr, i, - strerror(errno)); - close(vfio_dev_fd); - if (internal_config.process_type == RTE_PROC_PRIMARY) - rte_free(vfio_res); - return -1; - } - - maps[i].addr = bar_addr; - maps[i].offset = reg.offset; - maps[i].size = reg.size; - maps[i].path = NULL; /* vfio doesn't have per-resource paths */ - dev->mem_resource[i].addr = bar_addr; - } - - /* if secondary process, do not set up interrupts */ - if (internal_config.process_type == RTE_PROC_PRIMARY) { - if (pci_vfio_setup_interrupts(dev, vfio_dev_fd) != 0) { - RTE_LOG(ERR, EAL, " %s error setting up interrupts!\n", pci_addr); - close(vfio_dev_fd); - rte_free(vfio_res); - return -1; - } - - /* set bus mastering for the device */ - if (pci_vfio_set_bus_master(vfio_dev_fd)) { - RTE_LOG(ERR, EAL, " %s cannot set up bus mastering!\n", pci_addr); - close(vfio_dev_fd); - rte_free(vfio_res); - return -1; - } - - /* Reset the device */ - ioctl(vfio_dev_fd, VFIO_DEVICE_RESET); - } - - if (internal_config.process_type == RTE_PROC_PRIMARY) - TAILQ_INSERT_TAIL(vfio_res_list, vfio_res, next); - - return 0; -} - -int -pci_vfio_ioport_map(struct rte_pci_device *dev, int bar, - struct rte_pci_ioport *p) -{ - if (bar < VFIO_PCI_BAR0_REGION_INDEX || - bar > VFIO_PCI_BAR5_REGION_INDEX) { - RTE_LOG(ERR, EAL, "invalid bar (%d)!\n", bar); - return -1; - } - - p->dev = dev; - p->base = VFIO_GET_REGION_ADDR(bar); - return 0; -} - -void -pci_vfio_ioport_read(struct rte_pci_ioport *p, - void *data, size_t len, off_t offset) -{ - const struct rte_intr_handle *intr_handle = &p->dev->intr_handle; - - if (pread64(intr_handle->vfio_dev_fd, data, - len, p->base + offset) <= 0) - RTE_LOG(ERR, EAL, - "Can't read from PCI bar (%" PRIu64 ") : offset (%x)\n", - VFIO_GET_REGION_IDX(p->base), (int)offset); -} - -void -pci_vfio_ioport_write(struct rte_pci_ioport *p, - const void *data, size_t len, off_t offset) -{ - const struct rte_intr_handle *intr_handle = &p->dev->intr_handle; - - if (pwrite64(intr_handle->vfio_dev_fd, data, - len, p->base + offset) <= 0) - RTE_LOG(ERR, EAL, - "Can't write to PCI bar (%" PRIu64 ") : offset (%x)\n", - VFIO_GET_REGION_IDX(p->base), (int)offset); -} - -int -pci_vfio_ioport_unmap(struct rte_pci_ioport *p) -{ - RTE_SET_USED(p); - return -1; -} - -int -pci_vfio_enable(void) -{ - return vfio_enable("vfio_pci"); -} - -int -pci_vfio_is_enabled(void) -{ - return vfio_is_enabled("vfio_pci"); -} -#endif diff --git a/dpdk/lib/librte_eal/linuxapp/eal/eal_thread.c b/dpdk/lib/librte_eal/linuxapp/eal/eal_thread.c index 9f88530e..e9a579e4 100644 --- a/dpdk/lib/librte_eal/linuxapp/eal/eal_thread.c +++ b/dpdk/lib/librte_eal/linuxapp/eal/eal_thread.c @@ -46,10 +46,8 @@ #include #include #include -#include #include #include -#include #include #include "eal_private.h" @@ -184,7 +182,14 @@ eal_thread_loop(__attribute__((unused)) void *arg) ret = lcore_config[lcore_id].f(fct_arg); lcore_config[lcore_id].ret = ret; rte_wmb(); - lcore_config[lcore_id].state = FINISHED; + + /* when a service core returns, it should go directly to WAIT + * state, because the application will not lcore_wait() for it. + */ + if (lcore_config[lcore_id].core_role == ROLE_SERVICE) + lcore_config[lcore_id].state = WAIT; + else + lcore_config[lcore_id].state = FINISHED; } /* never reached */ diff --git a/dpdk/lib/librte_eal/linuxapp/eal/eal_timer.c b/dpdk/lib/librte_eal/linuxapp/eal/eal_timer.c index afa32f5c..a616928b 100644 --- a/dpdk/lib/librte_eal/linuxapp/eal/eal_timer.c +++ b/dpdk/lib/librte_eal/linuxapp/eal/eal_timer.c @@ -49,7 +49,6 @@ #include #include #include -#include #include #include @@ -114,7 +113,7 @@ static pthread_t msb_inc_thread_id; /* * This function runs on a specific thread to update a global variable - * containing used to process MSB of the HPET (unfortunatelly, we need + * containing used to process MSB of the HPET (unfortunately, we need * this because hpet is 32 bits by default under linux). */ static void diff --git a/dpdk/lib/librte_eal/linuxapp/eal/eal_vfio.c b/dpdk/lib/librte_eal/linuxapp/eal/eal_vfio.c index 702f7a2e..fb1a6226 100644 --- a/dpdk/lib/librte_eal/linuxapp/eal/eal_vfio.c +++ b/dpdk/lib/librte_eal/linuxapp/eal/eal_vfio.c @@ -39,6 +39,7 @@ #include #include #include +#include #include "eal_filesystem.h" #include "eal_vfio.h" @@ -50,12 +51,15 @@ static struct vfio_config vfio_cfg; static int vfio_type1_dma_map(int); +static int vfio_spapr_dma_map(int); static int vfio_noiommu_dma_map(int); /* IOMMU types we support */ static const struct vfio_iommu_type iommu_types[] = { /* x86 IOMMU, otherwise known as type 1 */ { RTE_VFIO_TYPE1, "Type 1", &vfio_type1_dma_map}, + /* ppc64 IOMMU, otherwise known as spapr */ + { RTE_VFIO_SPAPR, "sPAPR", &vfio_spapr_dma_map}, /* IOMMU-less mode */ { RTE_VFIO_NOIOMMU, "No-IOMMU", &vfio_noiommu_dma_map}, }; @@ -66,12 +70,31 @@ vfio_get_group_fd(int iommu_group_no) int i; int vfio_group_fd; char filename[PATH_MAX]; + struct vfio_group *cur_grp; /* check if we already have the group descriptor open */ - for (i = 0; i < vfio_cfg.vfio_group_idx; i++) + for (i = 0; i < VFIO_MAX_GROUPS; i++) if (vfio_cfg.vfio_groups[i].group_no == iommu_group_no) return vfio_cfg.vfio_groups[i].fd; + /* Lets see first if there is room for a new group */ + if (vfio_cfg.vfio_active_groups == VFIO_MAX_GROUPS) { + RTE_LOG(ERR, EAL, "Maximum number of VFIO groups reached!\n"); + return -1; + } + + /* Now lets get an index for the new group */ + for (i = 0; i < VFIO_MAX_GROUPS; i++) + if (vfio_cfg.vfio_groups[i].group_no == -1) { + cur_grp = &vfio_cfg.vfio_groups[i]; + break; + } + + /* This should not happen */ + if (i == VFIO_MAX_GROUPS) { + RTE_LOG(ERR, EAL, "No VFIO group free slot found\n"); + return -1; + } /* if primary, try to open the group */ if (internal_config.process_type == RTE_PROC_PRIMARY) { /* try regular group format */ @@ -101,14 +124,9 @@ vfio_get_group_fd(int iommu_group_no) /* noiommu group found */ } - /* if the fd is valid, create a new group for it */ - if (vfio_cfg.vfio_group_idx == VFIO_MAX_GROUPS) { - RTE_LOG(ERR, EAL, "Maximum number of VFIO groups reached!\n"); - close(vfio_group_fd); - return -1; - } - vfio_cfg.vfio_groups[vfio_cfg.vfio_group_idx].group_no = iommu_group_no; - vfio_cfg.vfio_groups[vfio_cfg.vfio_group_idx].fd = vfio_group_fd; + cur_grp->group_no = iommu_group_no; + cur_grp->fd = vfio_group_fd; + vfio_cfg.vfio_active_groups++; return vfio_group_fd; } /* if we're in a secondary process, request group fd from the primary @@ -140,9 +158,12 @@ vfio_get_group_fd(int iommu_group_no) return 0; case SOCKET_OK: vfio_group_fd = vfio_mp_sync_receive_fd(socket_fd); - /* if we got the fd, return it */ + /* if we got the fd, store it and return it */ if (vfio_group_fd > 0) { close(socket_fd); + cur_grp->group_no = iommu_group_no; + cur_grp->fd = vfio_group_fd; + vfio_cfg.vfio_active_groups++; return vfio_group_fd; } /* fall-through on error */ @@ -155,14 +176,115 @@ vfio_get_group_fd(int iommu_group_no) return -1; } -static void -clear_current_group(void) + +static int +get_vfio_group_idx(int vfio_group_fd) { - vfio_cfg.vfio_groups[vfio_cfg.vfio_group_idx].group_no = 0; - vfio_cfg.vfio_groups[vfio_cfg.vfio_group_idx].fd = -1; + int i; + for (i = 0; i < VFIO_MAX_GROUPS; i++) + if (vfio_cfg.vfio_groups[i].fd == vfio_group_fd) + return i; + return -1; } -int vfio_setup_device(const char *sysfs_base, const char *dev_addr, +static void +vfio_group_device_get(int vfio_group_fd) +{ + int i; + + i = get_vfio_group_idx(vfio_group_fd); + if (i < 0 || i > (VFIO_MAX_GROUPS - 1)) + RTE_LOG(ERR, EAL, " wrong vfio_group index (%d)\n", i); + else + vfio_cfg.vfio_groups[i].devices++; +} + +static void +vfio_group_device_put(int vfio_group_fd) +{ + int i; + + i = get_vfio_group_idx(vfio_group_fd); + if (i < 0 || i > (VFIO_MAX_GROUPS - 1)) + RTE_LOG(ERR, EAL, " wrong vfio_group index (%d)\n", i); + else + vfio_cfg.vfio_groups[i].devices--; +} + +static int +vfio_group_device_count(int vfio_group_fd) +{ + int i; + + i = get_vfio_group_idx(vfio_group_fd); + if (i < 0 || i > (VFIO_MAX_GROUPS - 1)) { + RTE_LOG(ERR, EAL, " wrong vfio_group index (%d)\n", i); + return -1; + } + + return vfio_cfg.vfio_groups[i].devices; +} + +int +clear_group(int vfio_group_fd) +{ + int i; + int socket_fd, ret; + + if (internal_config.process_type == RTE_PROC_PRIMARY) { + + i = get_vfio_group_idx(vfio_group_fd); + if (i < 0) + return -1; + vfio_cfg.vfio_groups[i].group_no = -1; + vfio_cfg.vfio_groups[i].fd = -1; + vfio_cfg.vfio_groups[i].devices = 0; + vfio_cfg.vfio_active_groups--; + return 0; + } + + /* This is just for SECONDARY processes */ + socket_fd = vfio_mp_sync_connect_to_primary(); + + if (socket_fd < 0) { + RTE_LOG(ERR, EAL, " cannot connect to primary process!\n"); + return -1; + } + + if (vfio_mp_sync_send_request(socket_fd, SOCKET_CLR_GROUP) < 0) { + RTE_LOG(ERR, EAL, " cannot request container fd!\n"); + close(socket_fd); + return -1; + } + + if (vfio_mp_sync_send_request(socket_fd, vfio_group_fd) < 0) { + RTE_LOG(ERR, EAL, " cannot send group fd!\n"); + close(socket_fd); + return -1; + } + + ret = vfio_mp_sync_receive_request(socket_fd); + switch (ret) { + case SOCKET_NO_FD: + RTE_LOG(ERR, EAL, " BAD VFIO group fd!\n"); + close(socket_fd); + break; + case SOCKET_OK: + close(socket_fd); + return 0; + case SOCKET_ERR: + RTE_LOG(ERR, EAL, " Socket error\n"); + close(socket_fd); + break; + default: + RTE_LOG(ERR, EAL, " UNKNOWN reply, %d\n", ret); + close(socket_fd); + } + return -1; +} + +int +rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr, int *vfio_dev_fd, struct vfio_device_info *device_info) { struct vfio_group_status group_status = { @@ -189,18 +311,10 @@ int vfio_setup_device(const char *sysfs_base, const char *dev_addr, if (vfio_group_fd < 0) return -1; - /* store group fd */ - vfio_cfg.vfio_groups[vfio_cfg.vfio_group_idx].group_no = iommu_group_no; - vfio_cfg.vfio_groups[vfio_cfg.vfio_group_idx].fd = vfio_group_fd; - /* if group_fd == 0, that means the device isn't managed by VFIO */ if (vfio_group_fd == 0) { - RTE_LOG(WARNING, EAL, " %s not managed by VFIO driver, skipping\n", + RTE_LOG(WARNING, EAL, " %s not managed by VFIO driver, skipping\n", dev_addr); - /* we store 0 as group fd to distinguish between existing but - * unbound VFIO groups, and groups that don't exist at all. - */ - vfio_cfg.vfio_group_idx++; return 1; } @@ -215,12 +329,12 @@ int vfio_setup_device(const char *sysfs_base, const char *dev_addr, RTE_LOG(ERR, EAL, " %s cannot get group status, " "error %i (%s)\n", dev_addr, errno, strerror(errno)); close(vfio_group_fd); - clear_current_group(); + clear_group(vfio_group_fd); return -1; } else if (!(group_status.flags & VFIO_GROUP_FLAGS_VIABLE)) { RTE_LOG(ERR, EAL, " %s VFIO group is not viable!\n", dev_addr); close(vfio_group_fd); - clear_current_group(); + clear_group(vfio_group_fd); return -1; } @@ -234,66 +348,137 @@ int vfio_setup_device(const char *sysfs_base, const char *dev_addr, RTE_LOG(ERR, EAL, " %s cannot add VFIO group to container, " "error %i (%s)\n", dev_addr, errno, strerror(errno)); close(vfio_group_fd); - clear_current_group(); + clear_group(vfio_group_fd); return -1; } - /* - * at this point we know that this group has been successfully - * initialized, so we increment vfio_group_idx to indicate that we can - * add new groups. - */ - vfio_cfg.vfio_group_idx++; - } - /* - * pick an IOMMU type and set up DMA mappings for container - * - * needs to be done only once, only when at least one group is assigned to - * a container and only in primary process - */ - if (internal_config.process_type == RTE_PROC_PRIMARY && - vfio_cfg.vfio_container_has_dma == 0) { - /* select an IOMMU type which we will be using */ - const struct vfio_iommu_type *t = + /* + * pick an IOMMU type and set up DMA mappings for container + * + * needs to be done only once, only when first group is + * assigned to a container and only in primary process. + * Note this can happen several times with the hotplug + * functionality. + */ + if (internal_config.process_type == RTE_PROC_PRIMARY && + vfio_cfg.vfio_active_groups == 1) { + /* select an IOMMU type which we will be using */ + const struct vfio_iommu_type *t = vfio_set_iommu_type(vfio_cfg.vfio_container_fd); - if (!t) { - RTE_LOG(ERR, EAL, " %s failed to select IOMMU type\n", dev_addr); - return -1; + if (!t) { + RTE_LOG(ERR, EAL, + " %s failed to select IOMMU type\n", + dev_addr); + close(vfio_group_fd); + clear_group(vfio_group_fd); + return -1; + } + ret = t->dma_map_func(vfio_cfg.vfio_container_fd); + if (ret) { + RTE_LOG(ERR, EAL, + " %s DMA remapping failed, error %i (%s)\n", + dev_addr, errno, strerror(errno)); + close(vfio_group_fd); + clear_group(vfio_group_fd); + return -1; + } } - ret = t->dma_map_func(vfio_cfg.vfio_container_fd); - if (ret) { - RTE_LOG(ERR, EAL, " %s DMA remapping failed, " - "error %i (%s)\n", dev_addr, errno, strerror(errno)); - return -1; - } - vfio_cfg.vfio_container_has_dma = 1; } /* get a file descriptor for the device */ *vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev_addr); if (*vfio_dev_fd < 0) { - /* if we cannot get a device fd, this simply means that this - * particular port is not bound to VFIO - */ - RTE_LOG(WARNING, EAL, " %s not managed by VFIO driver, skipping\n", + /* if we cannot get a device fd, this implies a problem with + * the VFIO group or the container not having IOMMU configured. + */ + + RTE_LOG(WARNING, EAL, "Getting a vfio_dev_fd for %s failed\n", dev_addr); - return 1; + close(vfio_group_fd); + clear_group(vfio_group_fd); + return -1; } /* test and setup the device */ ret = ioctl(*vfio_dev_fd, VFIO_DEVICE_GET_INFO, device_info); if (ret) { RTE_LOG(ERR, EAL, " %s cannot get device info, " - "error %i (%s)\n", dev_addr, errno, strerror(errno)); + "error %i (%s)\n", dev_addr, errno, + strerror(errno)); close(*vfio_dev_fd); + close(vfio_group_fd); + clear_group(vfio_group_fd); return -1; } + vfio_group_device_get(vfio_group_fd); + + return 0; +} + +int +rte_vfio_release_device(const char *sysfs_base, const char *dev_addr, + int vfio_dev_fd) +{ + struct vfio_group_status group_status = { + .argsz = sizeof(group_status) + }; + int vfio_group_fd; + int iommu_group_no; + int ret; + + /* get group number */ + ret = vfio_get_group_no(sysfs_base, dev_addr, &iommu_group_no); + if (ret <= 0) { + RTE_LOG(WARNING, EAL, " %s not managed by VFIO driver\n", + dev_addr); + /* This is an error at this point. */ + return -1; + } + + /* get the actual group fd */ + vfio_group_fd = vfio_get_group_fd(iommu_group_no); + if (vfio_group_fd <= 0) { + RTE_LOG(INFO, EAL, "vfio_get_group_fd failed for %s\n", + dev_addr); + return -1; + } + + /* At this point we got an active group. Closing it will make the + * container detachment. If this is the last active group, VFIO kernel + * code will unset the container and the IOMMU mappings. + */ + + /* Closing a device */ + if (close(vfio_dev_fd) < 0) { + RTE_LOG(INFO, EAL, "Error when closing vfio_dev_fd for %s\n", + dev_addr); + return -1; + } + + /* An VFIO group can have several devices attached. Just when there is + * no devices remaining should the group be closed. + */ + vfio_group_device_put(vfio_group_fd); + if (!vfio_group_device_count(vfio_group_fd)) { + + if (close(vfio_group_fd) < 0) { + RTE_LOG(INFO, EAL, "Error when closing vfio_group_fd for %s\n", + dev_addr); + return -1; + } + + if (clear_group(vfio_group_fd) < 0) { + RTE_LOG(INFO, EAL, "Error when clearing group for %s\n", + dev_addr); + return -1; + } + } return 0; } int -vfio_enable(const char *modname) +rte_vfio_enable(const char *modname) { /* initialize group list */ int i; @@ -302,12 +487,13 @@ vfio_enable(const char *modname) for (i = 0; i < VFIO_MAX_GROUPS; i++) { vfio_cfg.vfio_groups[i].fd = -1; vfio_cfg.vfio_groups[i].group_no = -1; + vfio_cfg.vfio_groups[i].devices = 0; } /* inform the user that we are probing for VFIO */ RTE_LOG(INFO, EAL, "Probing VFIO support...\n"); - /* check if vfio-pci module is loaded */ + /* check if vfio module is loaded */ vfio_available = rte_eal_check_module(modname); /* return error directly */ @@ -337,9 +523,9 @@ vfio_enable(const char *modname) } int -vfio_is_enabled(const char *modname) +rte_vfio_is_enabled(const char *modname) { - const int mod_available = rte_eal_check_module(modname); + const int mod_available = rte_eal_check_module(modname) > 0; return vfio_cfg.vfio_enabled && mod_available; } @@ -524,14 +710,18 @@ vfio_type1_dma_map(int vfio_container_fd) dma_map.argsz = sizeof(struct vfio_iommu_type1_dma_map); dma_map.vaddr = ms[i].addr_64; dma_map.size = ms[i].len; - dma_map.iova = ms[i].phys_addr; + if (rte_eal_iova_mode() == RTE_IOVA_VA) + dma_map.iova = dma_map.vaddr; + else + dma_map.iova = ms[i].iova; dma_map.flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE; ret = ioctl(vfio_container_fd, VFIO_IOMMU_MAP_DMA, &dma_map); if (ret) { RTE_LOG(ERR, EAL, " cannot set up DMA remapping, " - "error %i (%s)\n", errno, strerror(errno)); + "error %i (%s)\n", errno, + strerror(errno)); return -1; } } @@ -539,6 +729,110 @@ vfio_type1_dma_map(int vfio_container_fd) return 0; } +static int +vfio_spapr_dma_map(int vfio_container_fd) +{ + const struct rte_memseg *ms = rte_eal_get_physmem_layout(); + int i, ret; + + struct vfio_iommu_spapr_register_memory reg = { + .argsz = sizeof(reg), + .flags = 0 + }; + struct vfio_iommu_spapr_tce_info info = { + .argsz = sizeof(info), + }; + struct vfio_iommu_spapr_tce_create create = { + .argsz = sizeof(create), + }; + struct vfio_iommu_spapr_tce_remove remove = { + .argsz = sizeof(remove), + }; + + /* query spapr iommu info */ + ret = ioctl(vfio_container_fd, VFIO_IOMMU_SPAPR_TCE_GET_INFO, &info); + if (ret) { + RTE_LOG(ERR, EAL, " cannot get iommu info, " + "error %i (%s)\n", errno, strerror(errno)); + return -1; + } + + /* remove default DMA of 32 bit window */ + remove.start_addr = info.dma32_window_start; + ret = ioctl(vfio_container_fd, VFIO_IOMMU_SPAPR_TCE_REMOVE, &remove); + if (ret) { + RTE_LOG(ERR, EAL, " cannot remove default DMA window, " + "error %i (%s)\n", errno, strerror(errno)); + return -1; + } + + /* create DMA window from 0 to max(phys_addr + len) */ + for (i = 0; i < RTE_MAX_MEMSEG; i++) { + if (ms[i].addr == NULL) + break; + + create.window_size = RTE_MAX(create.window_size, + ms[i].iova + ms[i].len); + } + + /* sPAPR requires window size to be a power of 2 */ + create.window_size = rte_align64pow2(create.window_size); + create.page_shift = __builtin_ctzll(ms->hugepage_sz); + create.levels = 1; + + ret = ioctl(vfio_container_fd, VFIO_IOMMU_SPAPR_TCE_CREATE, &create); + if (ret) { + RTE_LOG(ERR, EAL, " cannot create new DMA window, " + "error %i (%s)\n", errno, strerror(errno)); + return -1; + } + + if (create.start_addr != 0) { + RTE_LOG(ERR, EAL, " DMA window start address != 0\n"); + return -1; + } + + /* map all DPDK segments for DMA. use 1:1 PA to IOVA mapping */ + for (i = 0; i < RTE_MAX_MEMSEG; i++) { + struct vfio_iommu_type1_dma_map dma_map; + + if (ms[i].addr == NULL) + break; + + reg.vaddr = (uintptr_t) ms[i].addr; + reg.size = ms[i].len; + ret = ioctl(vfio_container_fd, + VFIO_IOMMU_SPAPR_REGISTER_MEMORY, ®); + if (ret) { + RTE_LOG(ERR, EAL, " cannot register vaddr for IOMMU, " + "error %i (%s)\n", errno, strerror(errno)); + return -1; + } + + memset(&dma_map, 0, sizeof(dma_map)); + dma_map.argsz = sizeof(struct vfio_iommu_type1_dma_map); + dma_map.vaddr = ms[i].addr_64; + dma_map.size = ms[i].len; + if (rte_eal_iova_mode() == RTE_IOVA_VA) + dma_map.iova = dma_map.vaddr; + else + dma_map.iova = ms[i].iova; + dma_map.flags = VFIO_DMA_MAP_FLAG_READ | + VFIO_DMA_MAP_FLAG_WRITE; + + ret = ioctl(vfio_container_fd, VFIO_IOMMU_MAP_DMA, &dma_map); + + if (ret) { + RTE_LOG(ERR, EAL, " cannot set up DMA remapping, " + "error %i (%s)\n", errno, strerror(errno)); + return -1; + } + + } + + return 0; +} + static int vfio_noiommu_dma_map(int __rte_unused vfio_container_fd) { @@ -546,4 +840,23 @@ vfio_noiommu_dma_map(int __rte_unused vfio_container_fd) return 0; } +int +rte_vfio_noiommu_is_enabled(void) +{ + int fd, ret, cnt __rte_unused; + char c; + + ret = -1; + fd = open(VFIO_NOIOMMU_MODE, O_RDONLY); + if (fd < 0) + return -1; + + cnt = read(fd, &c, 1); + if (c == 'Y') + ret = 1; + + close(fd); + return ret; +} + #endif diff --git a/dpdk/lib/librte_eal/linuxapp/eal/eal_vfio.h b/dpdk/lib/librte_eal/linuxapp/eal/eal_vfio.h index 29f7f3ec..ba7892b7 100644 --- a/dpdk/lib/librte_eal/linuxapp/eal/eal_vfio.h +++ b/dpdk/lib/librte_eal/linuxapp/eal/eal_vfio.h @@ -37,23 +37,77 @@ /* * determine if VFIO is present on the system */ -#ifdef RTE_EAL_VFIO +#if !defined(VFIO_PRESENT) && defined(RTE_EAL_VFIO) #include #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 6, 0) +#define VFIO_PRESENT +#else +#pragma message("VFIO configured but not supported by this kernel, disabling.") +#endif /* kernel version >= 3.6.0 */ +#endif /* RTE_EAL_VFIO */ + +#ifdef VFIO_PRESENT + #include -#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 10, 0) -#define RTE_PCI_MSIX_TABLE_BIR 0x7 -#define RTE_PCI_MSIX_TABLE_OFFSET 0xfffffff8 -#define RTE_PCI_MSIX_FLAGS_QSIZE 0x07ff -#else -#define RTE_PCI_MSIX_TABLE_BIR PCI_MSIX_TABLE_BIR -#define RTE_PCI_MSIX_TABLE_OFFSET PCI_MSIX_TABLE_OFFSET -#define RTE_PCI_MSIX_FLAGS_QSIZE PCI_MSIX_FLAGS_QSIZE -#endif - #define RTE_VFIO_TYPE1 VFIO_TYPE1_IOMMU +#ifndef VFIO_SPAPR_TCE_v2_IOMMU +#define RTE_VFIO_SPAPR 7 +#define VFIO_IOMMU_SPAPR_REGISTER_MEMORY _IO(VFIO_TYPE, VFIO_BASE + 17) +#define VFIO_IOMMU_SPAPR_TCE_CREATE _IO(VFIO_TYPE, VFIO_BASE + 19) +#define VFIO_IOMMU_SPAPR_TCE_REMOVE _IO(VFIO_TYPE, VFIO_BASE + 20) + +struct vfio_iommu_spapr_register_memory { + uint32_t argsz; + uint32_t flags; + uint64_t vaddr; + uint64_t size; +}; + +struct vfio_iommu_spapr_tce_create { + uint32_t argsz; + uint32_t flags; + /* in */ + uint32_t page_shift; + uint32_t __resv1; + uint64_t window_size; + uint32_t levels; + uint32_t __resv2; + /* out */ + uint64_t start_addr; +}; + +struct vfio_iommu_spapr_tce_remove { + uint32_t argsz; + uint32_t flags; + /* in */ + uint64_t start_addr; +}; + +struct vfio_iommu_spapr_tce_ddw_info { + uint64_t pgsizes; + uint32_t max_dynamic_windows_supported; + uint32_t levels; +}; + +/* SPAPR_v2 is not present, but SPAPR might be */ +#ifndef VFIO_SPAPR_TCE_IOMMU +#define VFIO_IOMMU_SPAPR_TCE_GET_INFO _IO(VFIO_TYPE, VFIO_BASE + 12) + +struct vfio_iommu_spapr_tce_info { + uint32_t argsz; + uint32_t flags; + uint32_t dma32_window_start; + uint32_t dma32_window_size; + struct vfio_iommu_spapr_tce_ddw_info ddw; +}; +#endif /* VFIO_SPAPR_TCE_IOMMU */ + +#else /* VFIO_SPAPR_TCE_v2_IOMMU */ +#define RTE_VFIO_SPAPR VFIO_SPAPR_TCE_v2_IOMMU +#endif + #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 5, 0) #define RTE_VFIO_NOIOMMU 8 #else @@ -78,23 +132,16 @@ int vfio_mp_sync_connect_to_primary(void); struct vfio_group { int group_no; int fd; + int devices; }; struct vfio_config { int vfio_enabled; int vfio_container_fd; - int vfio_container_has_dma; - int vfio_group_idx; + int vfio_active_groups; struct vfio_group vfio_groups[VFIO_MAX_GROUPS]; }; -#define VFIO_DIR "/dev/vfio" -#define VFIO_CONTAINER_PATH "/dev/vfio/vfio" -#define VFIO_GROUP_FMT "/dev/vfio/%u" -#define VFIO_NOIOMMU_GROUP_FMT "/dev/vfio/noiommu-%u" -#define VFIO_GET_REGION_ADDR(x) ((uint64_t) x << 40ULL) -#define VFIO_GET_REGION_IDX(x) (x >> 40) - /* DMA mapping function prototype. * Takes VFIO container fd as a parameter. * Returns 0 on success, -1 on error. @@ -130,32 +177,19 @@ vfio_get_group_no(const char *sysfs_base, int vfio_get_group_fd(int iommu_group_no); -/** - * Setup vfio_cfg for the device identified by its address. It discovers - * the configured I/O MMU groups or sets a new one for the device. If a new - * groups is assigned, the DMA mapping is performed. - * Returns 0 on success, a negative value on failure and a positive value in - * case the given device cannot be managed this way. - */ -int vfio_setup_device(const char *sysfs_base, const char *dev_addr, - int *vfio_dev_fd, struct vfio_device_info *device_info); - -int vfio_enable(const char *modname); -int vfio_is_enabled(const char *modname); - -int pci_vfio_enable(void); -int pci_vfio_is_enabled(void); +/* remove group fd from internal VFIO group fd array */ +int +clear_group(int vfio_group_fd); int vfio_mp_sync_setup(void); #define SOCKET_REQ_CONTAINER 0x100 #define SOCKET_REQ_GROUP 0x200 +#define SOCKET_CLR_GROUP 0x300 #define SOCKET_OK 0x0 #define SOCKET_NO_FD 0x1 #define SOCKET_ERR 0xFF -#define VFIO_PRESENT -#endif /* kernel version */ -#endif /* RTE_EAL_VFIO */ +#endif /* VFIO_PRESENT */ #endif /* EAL_VFIO_H_ */ diff --git a/dpdk/lib/librte_eal/linuxapp/eal/eal_vfio_mp_sync.c b/dpdk/lib/librte_eal/linuxapp/eal/eal_vfio_mp_sync.c index 00cf919b..b53ed7eb 100644 --- a/dpdk/lib/librte_eal/linuxapp/eal/eal_vfio_mp_sync.c +++ b/dpdk/lib/librte_eal/linuxapp/eal/eal_vfio_mp_sync.c @@ -49,12 +49,12 @@ #endif #include -#include #include #include +#include #include "eal_filesystem.h" -#include "eal_pci_init.h" +#include "eal_vfio.h" #include "eal_thread.h" /** @@ -267,7 +267,7 @@ vfio_mp_sync_connect_to_primary(void) static __attribute__((noreturn)) void * vfio_mp_sync_thread(void __rte_unused * arg) { - int ret, fd, vfio_group_no; + int ret, fd, vfio_data; /* wait for requests on the socket */ for (;;) { @@ -301,16 +301,18 @@ vfio_mp_sync_thread(void __rte_unused * arg) vfio_mp_sync_send_request(conn_sock, SOCKET_ERR); else vfio_mp_sync_send_fd(conn_sock, fd); + if (fd >= 0) + close(fd); break; case SOCKET_REQ_GROUP: /* wait for group number */ - vfio_group_no = vfio_mp_sync_receive_request(conn_sock); - if (vfio_group_no < 0) { + vfio_data = vfio_mp_sync_receive_request(conn_sock); + if (vfio_data < 0) { close(conn_sock); continue; } - fd = vfio_get_group_fd(vfio_group_no); + fd = vfio_get_group_fd(vfio_data); if (fd < 0) vfio_mp_sync_send_request(conn_sock, SOCKET_ERR); @@ -323,6 +325,21 @@ vfio_mp_sync_thread(void __rte_unused * arg) vfio_mp_sync_send_fd(conn_sock, fd); } break; + case SOCKET_CLR_GROUP: + /* wait for group fd */ + vfio_data = vfio_mp_sync_receive_request(conn_sock); + if (vfio_data < 0) { + close(conn_sock); + continue; + } + + ret = clear_group(vfio_data); + + if (ret < 0) + vfio_mp_sync_send_request(conn_sock, SOCKET_NO_FD); + else + vfio_mp_sync_send_request(conn_sock, SOCKET_OK); + break; default: vfio_mp_sync_send_request(conn_sock, SOCKET_ERR); break; diff --git a/dpdk/lib/librte_eal/linuxapp/eal/eal_xen_memory.c b/dpdk/lib/librte_eal/linuxapp/eal/eal_xen_memory.c deleted file mode 100644 index bddbdb07..00000000 --- a/dpdk/lib/librte_eal/linuxapp/eal/eal_xen_memory.c +++ /dev/null @@ -1,383 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "eal_private.h" -#include "eal_internal_cfg.h" -#include "eal_filesystem.h" -#include - -#define PAGE_SIZE RTE_PGSIZE_4K -#define DEFAUL_DOM0_NAME "dom0-mem" - -static int xen_fd = -1; -static const char sys_dir_path[] = "/sys/kernel/mm/dom0-mm/memsize-mB"; - -/* - * Try to mmap *size bytes in /dev/zero. If it is successful, return the - * pointer to the mmap'd area and keep *size unmodified. Else, retry - * with a smaller zone: decrease *size by mem_size until it reaches - * 0. In this case, return NULL. Note: this function returns an address - * which is a multiple of mem_size size. - */ -static void * -xen_get_virtual_area(size_t *size, size_t mem_size) -{ - void *addr; - int fd; - long aligned_addr; - - RTE_LOG(DEBUG, EAL, "Ask a virtual area of 0x%zu bytes\n", *size); - - fd = open("/dev/zero", O_RDONLY); - if (fd < 0){ - RTE_LOG(ERR, EAL, "Cannot open /dev/zero\n"); - return NULL; - } - do { - addr = mmap(NULL, (*size) + mem_size, PROT_READ, - MAP_PRIVATE, fd, 0); - if (addr == MAP_FAILED) - *size -= mem_size; - } while (addr == MAP_FAILED && *size > 0); - - if (addr == MAP_FAILED) { - close(fd); - RTE_LOG(ERR, EAL, "Cannot get a virtual area\n"); - return NULL; - } - - munmap(addr, (*size) + mem_size); - close(fd); - - /* align addr to a mem_size boundary */ - aligned_addr = (uintptr_t)addr; - aligned_addr = RTE_ALIGN_CEIL(aligned_addr, mem_size); - addr = (void *)(aligned_addr); - - RTE_LOG(DEBUG, EAL, "Virtual area found at %p (size = 0x%zx)\n", - addr, *size); - - return addr; -} - -/** - * Get memory size configuration from /sys/devices/virtual/misc/dom0_mm - * /memsize-mB/memsize file, and the size unit is mB. - */ -static int -get_xen_memory_size(void) -{ - char path[PATH_MAX]; - unsigned long mem_size = 0; - static const char *file_name; - - file_name = "memsize"; - snprintf(path, sizeof(path), "%s/%s", - sys_dir_path, file_name); - - if (eal_parse_sysfs_value(path, &mem_size) < 0) - return -1; - - if (mem_size == 0) - rte_exit(EXIT_FAILURE,"XEN-DOM0:the %s/%s was not" - " configured.\n",sys_dir_path, file_name); - if (mem_size % 2) - rte_exit(EXIT_FAILURE,"XEN-DOM0:the %s/%s must be" - " even number.\n",sys_dir_path, file_name); - - if (mem_size > DOM0_CONFIG_MEMSIZE) - rte_exit(EXIT_FAILURE,"XEN-DOM0:the %s/%s should not be larger" - " than %d mB\n",sys_dir_path, file_name, DOM0_CONFIG_MEMSIZE); - - return mem_size; -} - -/** - * Based on physical address to caculate MFN in Xen Dom0. - */ -phys_addr_t -rte_xen_mem_phy2mch(int32_t memseg_id, const phys_addr_t phy_addr) -{ - int mfn_id, i; - uint64_t mfn, mfn_offset; - struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; - struct rte_memseg *memseg = mcfg->memseg; - - /* find the memory segment owning the physical address */ - if (memseg_id == -1) { - for (i = 0; i < RTE_MAX_MEMSEG; i++) { - if ((phy_addr >= memseg[i].phys_addr) && - (phy_addr < memseg[i].phys_addr + - memseg[i].len)) { - memseg_id = i; - break; - } - } - if (memseg_id == -1) - return RTE_BAD_PHYS_ADDR; - } - - mfn_id = (phy_addr - memseg[memseg_id].phys_addr) / RTE_PGSIZE_2M; - - /*the MFN is contiguous in 2M */ - mfn_offset = (phy_addr - memseg[memseg_id].phys_addr) % - RTE_PGSIZE_2M / PAGE_SIZE; - mfn = mfn_offset + memseg[memseg_id].mfn[mfn_id]; - - /** return mechine address */ - return mfn * PAGE_SIZE + phy_addr % PAGE_SIZE; -} - -int -rte_xen_dom0_memory_init(void) -{ - void *vir_addr, *vma_addr = NULL; - int err, ret = 0; - uint32_t i, requested, mem_size, memseg_idx, num_memseg = 0; - size_t vma_len = 0; - struct memory_info meminfo; - struct memseg_info seginfo[RTE_MAX_MEMSEG]; - int flags, page_size = getpagesize(); - struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; - struct rte_memseg *memseg = mcfg->memseg; - uint64_t total_mem = internal_config.memory; - - memset(seginfo, 0, sizeof(seginfo)); - memset(&meminfo, 0, sizeof(struct memory_info)); - - mem_size = get_xen_memory_size(); - requested = (unsigned) (total_mem / 0x100000); - if (requested > mem_size) - /* if we didn't satisfy total memory requirements */ - rte_exit(EXIT_FAILURE,"Not enough memory available! Requested: %uMB," - " available: %uMB\n", requested, mem_size); - else if (total_mem != 0) - mem_size = requested; - - /* Check FD and open once */ - if (xen_fd < 0) { - xen_fd = open(DOM0_MM_DEV, O_RDWR); - if (xen_fd < 0) { - RTE_LOG(ERR, EAL, "Can not open %s\n",DOM0_MM_DEV); - return -1; - } - } - - meminfo.size = mem_size; - - /* construct memory mangement name for Dom0 */ - snprintf(meminfo.name, DOM0_NAME_MAX, "%s-%s", - internal_config.hugefile_prefix, DEFAUL_DOM0_NAME); - - /* Notify kernel driver to allocate memory */ - ret = ioctl(xen_fd, RTE_DOM0_IOCTL_PREPARE_MEMSEG, &meminfo); - if (ret < 0) { - RTE_LOG(ERR, EAL, "XEN DOM0:failed to get memory\n"); - err = -EIO; - goto fail; - } - - /* Get number of memory segment from driver */ - ret = ioctl(xen_fd, RTE_DOM0_IOCTL_GET_NUM_MEMSEG, &num_memseg); - if (ret < 0) { - RTE_LOG(ERR, EAL, "XEN DOM0:failed to get memseg count.\n"); - err = -EIO; - goto fail; - } - - if(num_memseg > RTE_MAX_MEMSEG){ - RTE_LOG(ERR, EAL, "XEN DOM0: the memseg count %d is greater" - " than max memseg %d.\n",num_memseg, RTE_MAX_MEMSEG); - err = -EIO; - goto fail; - } - - /* get all memory segements information */ - ret = ioctl(xen_fd, RTE_DOM0_IOCTL_GET_MEMSEG_INFO, seginfo); - if (ret < 0) { - RTE_LOG(ERR, EAL, "XEN DOM0:failed to get memseg info.\n"); - err = -EIO; - goto fail; - } - - /* map all memory segments to contiguous user space */ - for (memseg_idx = 0; memseg_idx < num_memseg; memseg_idx++) - { - vma_len = seginfo[memseg_idx].size; - - /** - * get the biggest virtual memory area up to vma_len. If it fails, - * vma_addr is NULL, so let the kernel provide the address. - */ - vma_addr = xen_get_virtual_area(&vma_len, RTE_PGSIZE_2M); - if (vma_addr == NULL) { - flags = MAP_SHARED; - vma_len = RTE_PGSIZE_2M; - } else - flags = MAP_SHARED | MAP_FIXED; - - seginfo[memseg_idx].size = vma_len; - vir_addr = mmap(vma_addr, seginfo[memseg_idx].size, - PROT_READ|PROT_WRITE, flags, xen_fd, - memseg_idx * page_size); - if (vir_addr == MAP_FAILED) { - RTE_LOG(ERR, EAL, "XEN DOM0:Could not mmap %s\n", - DOM0_MM_DEV); - err = -EIO; - goto fail; - } - - memseg[memseg_idx].addr = vir_addr; - memseg[memseg_idx].phys_addr = page_size * - seginfo[memseg_idx].pfn ; - memseg[memseg_idx].len = seginfo[memseg_idx].size; - for ( i = 0; i < seginfo[memseg_idx].size / RTE_PGSIZE_2M; i++) - memseg[memseg_idx].mfn[i] = seginfo[memseg_idx].mfn[i]; - - /* MFNs are continuous in 2M, so assume that page size is 2M */ - memseg[memseg_idx].hugepage_sz = RTE_PGSIZE_2M; - - memseg[memseg_idx].nchannel = mcfg->nchannel; - memseg[memseg_idx].nrank = mcfg->nrank; - - /* NUMA is not suppoted in Xen Dom0, so only set socket 0*/ - memseg[memseg_idx].socket_id = 0; - } - - return 0; -fail: - if (xen_fd > 0) { - close(xen_fd); - xen_fd = -1; - } - return err; -} - -/* - * This creates the memory mappings in the secondary process to match that of - * the server process. It goes through each memory segment in the DPDK runtime - * configuration, mapping them in order to form a contiguous block in the - * virtual memory space - */ -int -rte_xen_dom0_memory_attach(void) -{ - const struct rte_mem_config *mcfg; - unsigned s = 0; /* s used to track the segment number */ - int xen_fd = -1; - int ret = -1; - void *vir_addr; - char name[DOM0_NAME_MAX] = {0}; - int page_size = getpagesize(); - - mcfg = rte_eal_get_configuration()->mem_config; - - /* Check FD and open once */ - if (xen_fd < 0) { - xen_fd = open(DOM0_MM_DEV, O_RDWR); - if (xen_fd < 0) { - RTE_LOG(ERR, EAL, "Can not open %s\n",DOM0_MM_DEV); - goto error; - } - } - - /* construct memory mangement name for Dom0 */ - snprintf(name, DOM0_NAME_MAX, "%s-%s", - internal_config.hugefile_prefix, DEFAUL_DOM0_NAME); - /* attach to memory segments of primary process */ - ret = ioctl(xen_fd, RTE_DOM0_IOCTL_ATTACH_TO_MEMSEG, name); - if (ret) { - RTE_LOG(ERR, EAL,"attach memory segments fail.\n"); - goto error; - } - - /* map all segments into memory to make sure we get the addrs */ - for (s = 0; s < RTE_MAX_MEMSEG; ++s) { - - /* - * the first memory segment with len==0 is the one that - * follows the last valid segment. - */ - if (mcfg->memseg[s].len == 0) - break; - - vir_addr = mmap(mcfg->memseg[s].addr, mcfg->memseg[s].len, - PROT_READ|PROT_WRITE, MAP_SHARED|MAP_FIXED, xen_fd, - s * page_size); - if (vir_addr == MAP_FAILED) { - RTE_LOG(ERR, EAL, "Could not mmap %llu bytes " - "in %s to requested address [%p]\n", - (unsigned long long)mcfg->memseg[s].len, DOM0_MM_DEV, - mcfg->memseg[s].addr); - goto error; - } - } - return 0; - -error: - if (xen_fd >= 0) { - close(xen_fd); - xen_fd = -1; - } - return -1; -} diff --git a/dpdk/lib/librte_eal/linuxapp/eal/include/exec-env/rte_dom0_common.h b/dpdk/lib/librte_eal/linuxapp/eal/include/exec-env/rte_dom0_common.h deleted file mode 100644 index d9707780..00000000 --- a/dpdk/lib/librte_eal/linuxapp/eal/include/exec-env/rte_dom0_common.h +++ /dev/null @@ -1,108 +0,0 @@ -/*- - * This file is provided under a dual BSD/LGPLv2 license. When using or - * redistributing this file, you may do so under either license. - * - * GNU LESSER GENERAL PUBLIC LICENSE - * - * Copyright(c) 2007-2014 Intel Corporation. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of version 2.1 of the GNU Lesser General Public License - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. - * - * Contact Information: - * Intel Corporation - * - * - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -#ifndef _RTE_DOM0_COMMON_H_ -#define _RTE_DOM0_COMMON_H_ - -#ifdef __KERNEL__ -#include -#endif - -#define DOM0_NAME_MAX 256 -#define DOM0_MM_DEV "/dev/dom0_mm" - -#define DOM0_CONTIG_NUM_ORDER 9 /**< order of 2M */ -#define DOM0_NUM_MEMSEG 512 /**< Maximum nb. of memory segment. */ -#define DOM0_MEMBLOCK_SIZE 0x200000 /**< size of memory block(2M). */ -#define DOM0_CONFIG_MEMSIZE 4096 /**< Maximum config memory size(4G). */ -#define DOM0_NUM_MEMBLOCK (DOM0_CONFIG_MEMSIZE / 2) /**< Maximum nb. of 2M memory block. */ - -#define RTE_DOM0_IOCTL_PREPARE_MEMSEG _IOWR(0, 1 , struct memory_info) -#define RTE_DOM0_IOCTL_ATTACH_TO_MEMSEG _IOWR(0, 2 , char *) -#define RTE_DOM0_IOCTL_GET_NUM_MEMSEG _IOWR(0, 3, int) -#define RTE_DOM0_IOCTL_GET_MEMSEG_INFO _IOWR(0, 4, void *) - -/** - * A structure used to store memory information. - */ -struct memory_info { - char name[DOM0_NAME_MAX]; - uint64_t size; -}; - -/** - * A structure used to store memory segment information. - */ -struct memseg_info { - uint32_t idx; - uint64_t pfn; - uint64_t size; - uint64_t mfn[DOM0_NUM_MEMBLOCK]; -}; - -/** - * A structure used to store memory block information. - */ -struct memblock_info { - uint8_t exchange_flag; - uint8_t used; - uint64_t vir_addr; - uint64_t pfn; - uint64_t mfn; -}; -#endif /* _RTE_DOM0_COMMON_H_ */ diff --git a/dpdk/lib/librte_eal/linuxapp/eal/include/exec-env/rte_interrupts.h b/dpdk/lib/librte_eal/linuxapp/eal/include/exec-env/rte_interrupts.h deleted file mode 100644 index 3dacbff8..00000000 --- a/dpdk/lib/librte_eal/linuxapp/eal/include/exec-env/rte_interrupts.h +++ /dev/null @@ -1,228 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef _RTE_INTERRUPTS_H_ -#error "don't include this file directly, please include generic " -#endif - -#ifndef _RTE_LINUXAPP_INTERRUPTS_H_ -#define _RTE_LINUXAPP_INTERRUPTS_H_ - -#define RTE_MAX_RXTX_INTR_VEC_ID 32 -#define RTE_INTR_VEC_ZERO_OFFSET 0 -#define RTE_INTR_VEC_RXTX_OFFSET 1 - -enum rte_intr_handle_type { - RTE_INTR_HANDLE_UNKNOWN = 0, - RTE_INTR_HANDLE_UIO, /**< uio device handle */ - RTE_INTR_HANDLE_UIO_INTX, /**< uio generic handle */ - RTE_INTR_HANDLE_VFIO_LEGACY, /**< vfio device handle (legacy) */ - RTE_INTR_HANDLE_VFIO_MSI, /**< vfio device handle (MSI) */ - RTE_INTR_HANDLE_VFIO_MSIX, /**< vfio device handle (MSIX) */ - RTE_INTR_HANDLE_ALARM, /**< alarm handle */ - RTE_INTR_HANDLE_EXT, /**< external handler */ - RTE_INTR_HANDLE_MAX -}; - -#define RTE_INTR_EVENT_ADD 1UL -#define RTE_INTR_EVENT_DEL 2UL - -typedef void (*rte_intr_event_cb_t)(int fd, void *arg); - -struct rte_epoll_data { - uint32_t event; /**< event type */ - void *data; /**< User data */ - rte_intr_event_cb_t cb_fun; /**< IN: callback fun */ - void *cb_arg; /**< IN: callback arg */ -}; - -enum { - RTE_EPOLL_INVALID = 0, - RTE_EPOLL_VALID, - RTE_EPOLL_EXEC, -}; - -/** interrupt epoll event obj, taken by epoll_event.ptr */ -struct rte_epoll_event { - volatile uint32_t status; /**< OUT: event status */ - int fd; /**< OUT: event fd */ - int epfd; /**< OUT: epoll instance the ev associated with */ - struct rte_epoll_data epdata; -}; - -/** Handle for interrupts. */ -struct rte_intr_handle { - union { - int vfio_dev_fd; /**< VFIO device file descriptor */ - int uio_cfg_fd; /**< UIO config file descriptor - for uio_pci_generic */ - }; - int fd; /**< interrupt event file descriptor */ - enum rte_intr_handle_type type; /**< handle type */ - uint32_t max_intr; /**< max interrupt requested */ - uint32_t nb_efd; /**< number of available efd(event fd) */ - int efds[RTE_MAX_RXTX_INTR_VEC_ID]; /**< intr vectors/efds mapping */ - struct rte_epoll_event elist[RTE_MAX_RXTX_INTR_VEC_ID]; - /**< intr vector epoll event */ - int *intr_vec; /**< intr vector number array */ -}; - -#define RTE_EPOLL_PER_THREAD -1 /**< to hint using per thread epfd */ - -/** - * It waits for events on the epoll instance. - * - * @param epfd - * Epoll instance fd on which the caller wait for events. - * @param events - * Memory area contains the events that will be available for the caller. - * @param maxevents - * Up to maxevents are returned, must greater than zero. - * @param timeout - * Specifying a timeout of -1 causes a block indefinitely. - * Specifying a timeout equal to zero cause to return immediately. - * @return - * - On success, returns the number of available event. - * - On failure, a negative value. - */ -int -rte_epoll_wait(int epfd, struct rte_epoll_event *events, - int maxevents, int timeout); - -/** - * It performs control operations on epoll instance referred by the epfd. - * It requests that the operation op be performed for the target fd. - * - * @param epfd - * Epoll instance fd on which the caller perform control operations. - * @param op - * The operation be performed for the target fd. - * @param fd - * The target fd on which the control ops perform. - * @param event - * Describes the object linked to the fd. - * Note: The caller must take care the object deletion after CTL_DEL. - * @return - * - On success, zero. - * - On failure, a negative value. - */ -int -rte_epoll_ctl(int epfd, int op, int fd, - struct rte_epoll_event *event); - -/** - * The function returns the per thread epoll instance. - * - * @return - * epfd the epoll instance referred to. - */ -int -rte_intr_tls_epfd(void); - -/** - * @param intr_handle - * Pointer to the interrupt handle. - * @param epfd - * Epoll instance fd which the intr vector associated to. - * @param op - * The operation be performed for the vector. - * Operation type of {ADD, DEL}. - * @param vec - * RX intr vector number added to the epoll instance wait list. - * @param data - * User raw data. - * @return - * - On success, zero. - * - On failure, a negative value. - */ -int -rte_intr_rx_ctl(struct rte_intr_handle *intr_handle, - int epfd, int op, unsigned int vec, void *data); - -/** - * It enables the packet I/O interrupt event if it's necessary. - * It creates event fd for each interrupt vector when MSIX is used, - * otherwise it multiplexes a single event fd. - * - * @param intr_handle - * Pointer to the interrupt handle. - * @param nb_efd - * Number of interrupt vector trying to enable. - * The value 0 is not allowed. - * @return - * - On success, zero. - * - On failure, a negative value. - */ -int -rte_intr_efd_enable(struct rte_intr_handle *intr_handle, uint32_t nb_efd); - -/** - * It disables the packet I/O interrupt event. - * It deletes registered eventfds and closes the open fds. - * - * @param intr_handle - * Pointer to the interrupt handle. - */ -void -rte_intr_efd_disable(struct rte_intr_handle *intr_handle); - -/** - * The packet I/O interrupt on datapath is enabled or not. - * - * @param intr_handle - * Pointer to the interrupt handle. - */ -int -rte_intr_dp_is_en(struct rte_intr_handle *intr_handle); - -/** - * The interrupt handle instance allows other causes or not. - * Other causes stand for any none packet I/O interrupts. - * - * @param intr_handle - * Pointer to the interrupt handle. - */ -int -rte_intr_allow_others(struct rte_intr_handle *intr_handle); - -/** - * The multiple interrupt vector capability of interrupt handle instance. - * It returns zero if no multiple interrupt vector support. - * - * @param intr_handle - * Pointer to the interrupt handle. - */ -int -rte_intr_cap_multiple(struct rte_intr_handle *intr_handle); - -#endif /* _RTE_LINUXAPP_INTERRUPTS_H_ */ diff --git a/dpdk/lib/librte_eal/linuxapp/eal/include/exec-env/rte_kni_common.h b/dpdk/lib/librte_eal/linuxapp/eal/include/exec-env/rte_kni_common.h index 2acdfd9b..794cd4f7 100644 --- a/dpdk/lib/librte_eal/linuxapp/eal/include/exec-env/rte_kni_common.h +++ b/dpdk/lib/librte_eal/linuxapp/eal/include/exec-env/rte_kni_common.h @@ -61,6 +61,10 @@ #ifdef __KERNEL__ #include +#define RTE_STD_C11 +#else +#include +#include #endif /** @@ -85,6 +89,7 @@ enum rte_kni_req_id { */ struct rte_kni_request { uint32_t req_id; /**< Request id */ + RTE_STD_C11 union { uint32_t new_mtu; /**< New MTU */ uint8_t if_up; /**< 1: interface up, 0: interface down */ @@ -102,7 +107,7 @@ struct rte_kni_fifo { volatile unsigned read; /**< Next position to be read */ unsigned len; /**< Circular buffer length */ unsigned elem_size; /**< Pointer size - for 32/64 bit OS */ - void * volatile buffer[0]; /**< The buffer contains mbuf pointers */ + void *volatile buffer[]; /**< The buffer contains mbuf pointers */ }; /* @@ -111,11 +116,11 @@ struct rte_kni_fifo { */ struct rte_kni_mbuf { void *buf_addr __attribute__((__aligned__(RTE_CACHE_LINE_SIZE))); - char pad0[10]; + uint64_t buf_physaddr; uint16_t data_off; /**< Start address of data in segment buffer. */ char pad1[2]; - uint8_t nb_segs; /**< Number of segments. */ - char pad4[1]; + uint16_t nb_segs; /**< Number of segments. */ + char pad4[2]; uint64_t ol_flags; /**< Offload features. */ char pad2[4]; uint32_t pkt_len; /**< Total pkt len: sum of all segment data_len. */ @@ -159,6 +164,7 @@ struct rte_kni_device_info { uint16_t group_id; /**< Group ID */ uint32_t core_id; /**< core ID to bind for kernel thread */ + __extension__ uint8_t force_bind : 1; /**< Flag for kernel thread binding */ /* mbuf size */ diff --git a/dpdk/lib/librte_eal/linuxapp/eal/rte_eal_version.map b/dpdk/lib/librte_eal/linuxapp/eal/rte_eal_version.map deleted file mode 100644 index db8c9845..00000000 --- a/dpdk/lib/librte_eal/linuxapp/eal/rte_eal_version.map +++ /dev/null @@ -1,168 +0,0 @@ -DPDK_2.0 { - global: - - __rte_panic; - devargs_list; - eal_parse_sysfs_value; - eal_timer_source; - lcore_config; - pci_device_list; - pci_driver_list; - per_lcore__lcore_id; - per_lcore__rte_errno; - rte_calloc; - rte_calloc_socket; - rte_cpu_check_supported; - rte_cpu_get_flag_enabled; - rte_cycles_vmware_tsc_map; - rte_delay_us; - rte_dump_physmem_layout; - rte_dump_registers; - rte_dump_stack; - rte_dump_tailq; - rte_eal_alarm_cancel; - rte_eal_alarm_set; - rte_eal_dev_init; - rte_eal_devargs_add; - rte_eal_devargs_dump; - rte_eal_devargs_type_count; - rte_eal_driver_register; - rte_eal_driver_unregister; - rte_eal_get_configuration; - rte_eal_get_lcore_state; - rte_eal_get_physmem_layout; - rte_eal_get_physmem_size; - rte_eal_has_hugepages; - rte_eal_hpet_init; - rte_eal_init; - rte_eal_iopl_init; - rte_eal_lcore_role; - rte_eal_mp_remote_launch; - rte_eal_mp_wait_lcore; - rte_eal_parse_devargs_str; - rte_eal_pci_dump; - rte_eal_pci_probe; - rte_eal_pci_probe_one; - rte_eal_pci_register; - rte_eal_pci_scan; - rte_eal_pci_unregister; - rte_eal_process_type; - rte_eal_remote_launch; - rte_eal_tailq_lookup; - rte_eal_tailq_register; - rte_eal_vdev_init; - rte_eal_vdev_uninit; - rte_eal_wait_lcore; - rte_exit; - rte_free; - rte_get_hpet_cycles; - rte_get_hpet_hz; - rte_get_log_level; - rte_get_log_type; - rte_get_tsc_hz; - rte_hexdump; - rte_intr_callback_register; - rte_intr_callback_unregister; - rte_intr_disable; - rte_intr_enable; - rte_log; - rte_log_add_in_history; - rte_log_cur_msg_loglevel; - rte_log_cur_msg_logtype; - rte_log_dump_history; - rte_log_set_history; - rte_logs; - rte_malloc; - rte_malloc_dump_stats; - rte_malloc_get_socket_stats; - rte_malloc_set_limit; - rte_malloc_socket; - rte_malloc_validate; - rte_malloc_virt2phy; - rte_mem_lock_page; - rte_mem_phy2mch; - rte_mem_virt2phy; - rte_memdump; - rte_memory_get_nchannel; - rte_memory_get_nrank; - rte_memzone_dump; - rte_memzone_lookup; - rte_memzone_reserve; - rte_memzone_reserve_aligned; - rte_memzone_reserve_bounded; - rte_memzone_walk; - rte_openlog_stream; - rte_realloc; - rte_set_application_usage_hook; - rte_set_log_level; - rte_set_log_type; - rte_socket_id; - rte_strerror; - rte_strsplit; - rte_sys_gettid; - rte_thread_get_affinity; - rte_thread_set_affinity; - rte_vlog; - rte_xen_dom0_memory_attach; - rte_xen_dom0_memory_init; - rte_zmalloc; - rte_zmalloc_socket; - - local: *; -}; - -DPDK_2.1 { - global: - - rte_eal_pci_detach; - rte_eal_pci_read_config; - rte_eal_pci_write_config; - rte_epoll_ctl; - rte_epoll_wait; - rte_intr_allow_others; - rte_intr_dp_is_en; - rte_intr_efd_disable; - rte_intr_efd_enable; - rte_intr_rx_ctl; - rte_intr_tls_epfd; - rte_memzone_free; - -} DPDK_2.0; - -DPDK_2.2 { - global: - - rte_intr_cap_multiple; - rte_keepalive_create; - rte_keepalive_dispatch_pings; - rte_keepalive_mark_alive; - rte_keepalive_register_core; - rte_xen_dom0_supported; - rte_xen_mem_phy2mch; - -} DPDK_2.1; - -DPDK_16.04 { - global: - - rte_cpu_get_flag_name; - rte_eal_pci_ioport_map; - rte_eal_pci_ioport_read; - rte_eal_pci_ioport_unmap; - rte_eal_pci_ioport_write; - rte_eal_pci_map_device; - rte_eal_pci_unmap_device; - rte_eal_primary_proc_alive; - -} DPDK_2.2; - -DPDK_16.07 { - global: - - pci_get_sysfs_path; - rte_keepalive_mark_sleep; - rte_keepalive_register_relay_callback; - rte_rtm_supported; - rte_thread_setname; - -} DPDK_16.04; diff --git a/dpdk/lib/librte_eal/linuxapp/igb_uio/compat.h b/dpdk/lib/librte_eal/linuxapp/igb_uio/compat.h index 0d781e48..ce456d4b 100644 --- a/dpdk/lib/librte_eal/linuxapp/igb_uio/compat.h +++ b/dpdk/lib/librte_eal/linuxapp/igb_uio/compat.h @@ -16,12 +16,9 @@ #endif #ifndef PCI_MSIX_ENTRY_SIZE -#define PCI_MSIX_ENTRY_SIZE 16 -#define PCI_MSIX_ENTRY_LOWER_ADDR 0 -#define PCI_MSIX_ENTRY_UPPER_ADDR 4 -#define PCI_MSIX_ENTRY_DATA 8 -#define PCI_MSIX_ENTRY_VECTOR_CTRL 12 -#define PCI_MSIX_ENTRY_CTRL_MASKBIT 1 +#define PCI_MSIX_ENTRY_SIZE 16 +#define PCI_MSIX_ENTRY_VECTOR_CTRL 12 +#define PCI_MSIX_ENTRY_CTRL_MASKBIT 1 #endif /* @@ -123,3 +120,15 @@ static bool pci_check_and_mask_intx(struct pci_dev *pdev) } #endif /* < 3.3.0 */ + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 8, 0) +#define HAVE_ALLOC_IRQ_VECTORS 1 +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 3, 0) +#define HAVE_MSI_LIST_IN_GENERIC_DEVICE 1 +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 5, 0) +#define HAVE_PCI_MSI_MASK_IRQ 1 +#endif diff --git a/dpdk/lib/librte_eal/linuxapp/igb_uio/igb_uio.c b/dpdk/lib/librte_eal/linuxapp/igb_uio/igb_uio.c index f6dd454e..1826adba 100644 --- a/dpdk/lib/librte_eal/linuxapp/igb_uio/igb_uio.c +++ b/dpdk/lib/librte_eal/linuxapp/igb_uio/igb_uio.c @@ -29,13 +29,11 @@ #include #include #include +#include #include #include #include -#ifdef CONFIG_XEN_DOM0 -#include -#endif #include #include "compat.h" @@ -47,11 +45,12 @@ struct rte_uio_pci_dev { struct uio_info info; struct pci_dev *pdev; enum rte_intr_mode mode; + struct mutex lock; + int refcnt; }; static char *intr_mode; static enum rte_intr_mode igbuio_intr_mode_preferred = RTE_INTR_MODE_MSIX; - /* sriov sysfs */ static ssize_t show_max_vfs(struct device *dev, struct device_attribute *attr, @@ -91,14 +90,16 @@ static struct attribute *dev_attrs[] = { static const struct attribute_group dev_attr_grp = { .attrs = dev_attrs, }; + +#ifndef HAVE_PCI_MSI_MASK_IRQ /* * It masks the msix on/off of generating MSI-X messages. */ static void -igbuio_msix_mask_irq(struct msi_desc *desc, int32_t state) +igbuio_msix_mask_irq(struct msi_desc *desc, s32 state) { u32 mask_bits = desc->masked; - unsigned offset = desc->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE + + unsigned int offset = desc->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE + PCI_MSIX_ENTRY_VECTOR_CTRL; if (state != 0) @@ -113,6 +114,52 @@ igbuio_msix_mask_irq(struct msi_desc *desc, int32_t state) } } +/* + * It masks the msi on/off of generating MSI messages. + */ +static void +igbuio_msi_mask_irq(struct pci_dev *pdev, struct msi_desc *desc, int32_t state) +{ + u32 mask_bits = desc->masked; + u32 offset = desc->irq - pdev->irq; + u32 mask = 1 << offset; + + if (!desc->msi_attrib.maskbit) + return; + + if (state != 0) + mask_bits &= ~mask; + else + mask_bits |= mask; + + if (mask_bits != desc->masked) { + pci_write_config_dword(pdev, desc->mask_pos, mask_bits); + desc->masked = mask_bits; + } +} + +static void +igbuio_mask_irq(struct pci_dev *pdev, enum rte_intr_mode mode, s32 irq_state) +{ + struct msi_desc *desc; + struct list_head *msi_list; + +#ifdef HAVE_MSI_LIST_IN_GENERIC_DEVICE + msi_list = &pdev->dev.msi_list; +#else + msi_list = &pdev->msi_list; +#endif + + if (mode == RTE_INTR_MODE_MSIX) { + list_for_each_entry(desc, msi_list, list) + igbuio_msix_mask_irq(desc, irq_state); + } else if (mode == RTE_INTR_MODE_MSI) { + list_for_each_entry(desc, msi_list, list) + igbuio_msi_mask_irq(pdev, desc, irq_state); + } +} +#endif + /** * This is the irqcontrol callback to be registered to uio_info. * It can be used to disable/enable interrupt from user space processes. @@ -132,21 +179,26 @@ igbuio_pci_irqcontrol(struct uio_info *info, s32 irq_state) struct rte_uio_pci_dev *udev = info->priv; struct pci_dev *pdev = udev->pdev; +#ifdef HAVE_PCI_MSI_MASK_IRQ + struct irq_data *irq = irq_get_irq_data(udev->info.irq); +#endif + pci_cfg_access_lock(pdev); + + if (udev->mode == RTE_INTR_MODE_MSIX || udev->mode == RTE_INTR_MODE_MSI) { +#ifdef HAVE_PCI_MSI_MASK_IRQ + if (irq_state == 1) + pci_msi_unmask_irq(irq); + else + pci_msi_mask_irq(irq); +#else + igbuio_mask_irq(pdev, udev->mode, irq_state); +#endif + } + if (udev->mode == RTE_INTR_MODE_LEGACY) pci_intx(pdev, !!irq_state); - else if (udev->mode == RTE_INTR_MODE_MSIX) { - struct msi_desc *desc; - -#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0)) - list_for_each_entry(desc, &pdev->msi_list, list) - igbuio_msix_mask_irq(desc, irq_state); -#else - list_for_each_entry(desc, &pdev->dev.msi_list, list) - igbuio_msix_mask_irq(desc, irq_state); -#endif - } pci_cfg_access_unlock(pdev); return 0; @@ -157,64 +209,175 @@ igbuio_pci_irqcontrol(struct uio_info *info, s32 irq_state) * If yes, disable it here and will be enable later. */ static irqreturn_t -igbuio_pci_irqhandler(int irq, struct uio_info *info) +igbuio_pci_irqhandler(int irq, void *dev_id) { - struct rte_uio_pci_dev *udev = info->priv; + struct rte_uio_pci_dev *udev = (struct rte_uio_pci_dev *)dev_id; + struct uio_info *info = &udev->info; /* Legacy mode need to mask in hardware */ if (udev->mode == RTE_INTR_MODE_LEGACY && !pci_check_and_mask_intx(udev->pdev)) return IRQ_NONE; + uio_event_notify(info); + /* Message signal mode, no share IRQ and automasked */ return IRQ_HANDLED; } -#ifdef CONFIG_XEN_DOM0 static int -igbuio_dom0_mmap_phys(struct uio_info *info, struct vm_area_struct *vma) +igbuio_pci_enable_interrupts(struct rte_uio_pci_dev *udev) { - int idx; - - idx = (int)vma->vm_pgoff; - vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); -#ifdef HAVE_PTE_MASK_PAGE_IOMAP - vma->vm_page_prot.pgprot |= _PAGE_IOMAP; + int err = 0; +#ifndef HAVE_ALLOC_IRQ_VECTORS + struct msix_entry msix_entry; #endif - return remap_pfn_range(vma, - vma->vm_start, - info->mem[idx].addr >> PAGE_SHIFT, - vma->vm_end - vma->vm_start, - vma->vm_page_prot); + switch (igbuio_intr_mode_preferred) { + case RTE_INTR_MODE_MSIX: + /* Only 1 msi-x vector needed */ +#ifndef HAVE_ALLOC_IRQ_VECTORS + msix_entry.entry = 0; + if (pci_enable_msix(udev->pdev, &msix_entry, 1) == 0) { + dev_dbg(&udev->pdev->dev, "using MSI-X"); + udev->info.irq_flags = IRQF_NO_THREAD; + udev->info.irq = msix_entry.vector; + udev->mode = RTE_INTR_MODE_MSIX; + break; + } +#else + if (pci_alloc_irq_vectors(udev->pdev, 1, 1, PCI_IRQ_MSIX) == 1) { + dev_dbg(&udev->pdev->dev, "using MSI-X"); + udev->info.irq_flags = IRQF_NO_THREAD; + udev->info.irq = pci_irq_vector(udev->pdev, 0); + udev->mode = RTE_INTR_MODE_MSIX; + break; + } +#endif + + /* fall back to MSI */ + case RTE_INTR_MODE_MSI: +#ifndef HAVE_ALLOC_IRQ_VECTORS + if (pci_enable_msi(udev->pdev) == 0) { + dev_dbg(&udev->pdev->dev, "using MSI"); + udev->info.irq_flags = IRQF_NO_THREAD; + udev->info.irq = udev->pdev->irq; + udev->mode = RTE_INTR_MODE_MSI; + break; + } +#else + if (pci_alloc_irq_vectors(udev->pdev, 1, 1, PCI_IRQ_MSI) == 1) { + dev_dbg(&udev->pdev->dev, "using MSI"); + udev->info.irq_flags = IRQF_NO_THREAD; + udev->info.irq = pci_irq_vector(udev->pdev, 0); + udev->mode = RTE_INTR_MODE_MSI; + break; + } +#endif + /* fall back to INTX */ + case RTE_INTR_MODE_LEGACY: + if (pci_intx_mask_supported(udev->pdev)) { + dev_dbg(&udev->pdev->dev, "using INTX"); + udev->info.irq_flags = IRQF_SHARED | IRQF_NO_THREAD; + udev->info.irq = udev->pdev->irq; + udev->mode = RTE_INTR_MODE_LEGACY; + break; + } + dev_notice(&udev->pdev->dev, "PCI INTX mask not supported\n"); + /* fall back to no IRQ */ + case RTE_INTR_MODE_NONE: + udev->mode = RTE_INTR_MODE_NONE; + udev->info.irq = UIO_IRQ_NONE; + break; + + default: + dev_err(&udev->pdev->dev, "invalid IRQ mode %u", + igbuio_intr_mode_preferred); + udev->info.irq = UIO_IRQ_NONE; + err = -EINVAL; + } + + if (udev->info.irq != UIO_IRQ_NONE) + err = request_irq(udev->info.irq, igbuio_pci_irqhandler, + udev->info.irq_flags, udev->info.name, + udev); + dev_info(&udev->pdev->dev, "uio device registered with irq %lx\n", + udev->info.irq); + + return err; } +static void +igbuio_pci_disable_interrupts(struct rte_uio_pci_dev *udev) +{ + if (udev->info.irq) { + free_irq(udev->info.irq, udev); + udev->info.irq = 0; + } + +#ifndef HAVE_ALLOC_IRQ_VECTORS + if (udev->mode == RTE_INTR_MODE_MSIX) + pci_disable_msix(udev->pdev); + if (udev->mode == RTE_INTR_MODE_MSI) + pci_disable_msi(udev->pdev); +#else + if (udev->mode == RTE_INTR_MODE_MSIX || + udev->mode == RTE_INTR_MODE_MSI) + pci_free_irq_vectors(udev->pdev); +#endif +} + + /** - * This is uio device mmap method which will use igbuio mmap for Xen - * Dom0 environment. + * This gets called while opening uio device file. */ static int -igbuio_dom0_pci_mmap(struct uio_info *info, struct vm_area_struct *vma) +igbuio_pci_open(struct uio_info *info, struct inode *inode) { - int idx; + struct rte_uio_pci_dev *udev = info->priv; + struct pci_dev *dev = udev->pdev; + int err; - if (vma->vm_pgoff >= MAX_UIO_MAPS) - return -EINVAL; - - if (info->mem[vma->vm_pgoff].size == 0) - return -EINVAL; - - idx = (int)vma->vm_pgoff; - switch (info->mem[idx].memtype) { - case UIO_MEM_PHYS: - return igbuio_dom0_mmap_phys(info, vma); - case UIO_MEM_LOGICAL: - case UIO_MEM_VIRTUAL: - default: - return -EINVAL; + mutex_lock(&udev->lock); + if (++udev->refcnt > 1) { + mutex_unlock(&udev->lock); + return 0; } + + /* set bus master, which was cleared by the reset function */ + pci_set_master(dev); + + /* enable interrupts */ + err = igbuio_pci_enable_interrupts(udev); + mutex_unlock(&udev->lock); + if (err) { + dev_err(&dev->dev, "Enable interrupt fails\n"); + return err; + } + return 0; +} + +static int +igbuio_pci_release(struct uio_info *info, struct inode *inode) +{ + struct rte_uio_pci_dev *udev = info->priv; + struct pci_dev *dev = udev->pdev; + + mutex_lock(&udev->lock); + if (--udev->refcnt > 0) { + mutex_unlock(&udev->lock); + return 0; + } + + /* disable interrupts */ + igbuio_pci_disable_interrupts(udev); + + /* stop the device from further DMA */ + pci_clear_master(dev); + + mutex_unlock(&udev->lock); + return 0; } -#endif /* Remap pci resources described by bar #pci_bar in uio resource n. */ static int @@ -314,7 +477,7 @@ igbuio_setup_bars(struct pci_dev *dev, struct uio_info *info) } } - return (iom != 0) ? ret : -ENOENT; + return (iom != 0 || iop != 0) ? ret : -ENOENT; } #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 8, 0) @@ -325,13 +488,15 @@ static int igbuio_pci_probe(struct pci_dev *dev, const struct pci_device_id *id) { struct rte_uio_pci_dev *udev; - struct msix_entry msix_entry; + dma_addr_t map_dma_addr; + void *map_addr; int err; udev = kzalloc(sizeof(struct rte_uio_pci_dev), GFP_KERNEL); if (!udev) return -ENOMEM; + mutex_init(&udev->lock); /* * enable device: ask low-level code to enable I/O and * memory @@ -366,53 +531,12 @@ igbuio_pci_probe(struct pci_dev *dev, const struct pci_device_id *id) /* fill uio infos */ udev->info.name = "igb_uio"; udev->info.version = "0.1"; - udev->info.handler = igbuio_pci_irqhandler; udev->info.irqcontrol = igbuio_pci_irqcontrol; -#ifdef CONFIG_XEN_DOM0 - /* check if the driver run on Xen Dom0 */ - if (xen_initial_domain()) - udev->info.mmap = igbuio_dom0_pci_mmap; -#endif + udev->info.open = igbuio_pci_open; + udev->info.release = igbuio_pci_release; udev->info.priv = udev; udev->pdev = dev; - switch (igbuio_intr_mode_preferred) { - case RTE_INTR_MODE_MSIX: - /* Only 1 msi-x vector needed */ - msix_entry.entry = 0; -#ifdef HAVE_PCI_ENABLE_MSIX - if (pci_enable_msix(dev, &msix_entry, 1) == 0) { -#else - if (pci_enable_msix_range(dev, &msix_entry, 1, 1) == 0) { -#endif - dev_dbg(&dev->dev, "using MSI-X"); - udev->info.irq = msix_entry.vector; - udev->mode = RTE_INTR_MODE_MSIX; - break; - } - /* fall back to INTX */ - case RTE_INTR_MODE_LEGACY: - if (pci_intx_mask_supported(dev)) { - dev_dbg(&dev->dev, "using INTX"); - udev->info.irq_flags = IRQF_SHARED; - udev->info.irq = dev->irq; - udev->mode = RTE_INTR_MODE_LEGACY; - break; - } - dev_notice(&dev->dev, "PCI INTX mask not supported\n"); - /* fall back to no IRQ */ - case RTE_INTR_MODE_NONE: - udev->mode = RTE_INTR_MODE_NONE; - udev->info.irq = 0; - break; - - default: - dev_err(&dev->dev, "invalid IRQ mode %u", - igbuio_intr_mode_preferred); - err = -EINVAL; - goto fail_release_iomem; - } - err = sysfs_create_group(&dev->dev.kobj, &dev_attr_grp); if (err != 0) goto fail_release_iomem; @@ -424,8 +548,26 @@ igbuio_pci_probe(struct pci_dev *dev, const struct pci_device_id *id) pci_set_drvdata(dev, udev); - dev_info(&dev->dev, "uio device registered with irq %lx\n", - udev->info.irq); + /* + * Doing a harmless dma mapping for attaching the device to + * the iommu identity mapping if kernel boots with iommu=pt. + * Note this is not a problem if no IOMMU at all. + */ + map_addr = dma_alloc_coherent(&dev->dev, 1024, &map_dma_addr, + GFP_KERNEL); + if (map_addr) + memset(map_addr, 0, 1024); + + if (!map_addr) + dev_info(&dev->dev, "dma mapping failed\n"); + else { + dev_info(&dev->dev, "mapping 1K dma=%#llx host=%p\n", + (unsigned long long)map_dma_addr, map_addr); + + dma_free_coherent(&dev->dev, 1024, map_addr, map_dma_addr); + dev_info(&dev->dev, "unmapping 1K dma=%#llx host=%p\n", + (unsigned long long)map_dma_addr, map_addr); + } return 0; @@ -433,8 +575,6 @@ fail_remove_group: sysfs_remove_group(&dev->dev.kobj, &dev_attr_grp); fail_release_iomem: igbuio_pci_release_iomem(&udev->info); - if (udev->mode == RTE_INTR_MODE_MSIX) - pci_disable_msix(udev->pdev); pci_disable_device(dev); fail_free: kfree(udev); @@ -447,11 +587,10 @@ igbuio_pci_remove(struct pci_dev *dev) { struct rte_uio_pci_dev *udev = pci_get_drvdata(dev); + mutex_destroy(&udev->lock); sysfs_remove_group(&dev->dev.kobj, &dev_attr_grp); uio_unregister_device(&udev->info); igbuio_pci_release_iomem(&udev->info); - if (udev->mode == RTE_INTR_MODE_MSIX) - pci_disable_msix(dev); pci_disable_device(dev); pci_set_drvdata(dev, NULL); kfree(udev); @@ -468,6 +607,9 @@ igbuio_config_intr_mode(char *intr_str) if (!strcmp(intr_str, RTE_INTR_MODE_MSIX_NAME)) { igbuio_intr_mode_preferred = RTE_INTR_MODE_MSIX; pr_info("Use MSIX interrupt\n"); + } else if (!strcmp(intr_str, RTE_INTR_MODE_MSI_NAME)) { + igbuio_intr_mode_preferred = RTE_INTR_MODE_MSI; + pr_info("Use MSI interrupt\n"); } else if (!strcmp(intr_str, RTE_INTR_MODE_LEGACY_NAME)) { igbuio_intr_mode_preferred = RTE_INTR_MODE_LEGACY; pr_info("Use legacy interrupt\n"); @@ -511,6 +653,7 @@ module_param(intr_mode, charp, S_IRUGO); MODULE_PARM_DESC(intr_mode, "igb_uio interrupt mode (default=msix):\n" " " RTE_INTR_MODE_MSIX_NAME " Use MSIX interrupt\n" +" " RTE_INTR_MODE_MSI_NAME " Use MSI interrupt\n" " " RTE_INTR_MODE_LEGACY_NAME " Use Legacy interrupt\n" "\n"); diff --git a/dpdk/lib/librte_eal/linuxapp/kni/Makefile b/dpdk/lib/librte_eal/linuxapp/kni/Makefile index 8cc6b61c..154c528d 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/Makefile +++ b/dpdk/lib/librte_eal/linuxapp/kni/Makefile @@ -44,50 +44,43 @@ MODULE_CFLAGS += -I$(RTE_OUTPUT)/include -I$(SRCDIR)/ethtool/ixgbe -I$(SRCDIR)/e MODULE_CFLAGS += -include $(RTE_OUTPUT)/include/rte_config.h MODULE_CFLAGS += -Wall -Werror -ifeq ($(shell lsb_release -si 2>/dev/null),Ubuntu) -MODULE_CFLAGS += -DUBUNTU_RELEASE_CODE=$(shell lsb_release -sr | tr -d .) +-include /etc/lsb-release + +ifeq ($(DISTRIB_ID),Ubuntu) +MODULE_CFLAGS += -DUBUNTU_RELEASE_CODE=$(subst .,,$(DISTRIB_RELEASE)) UBUNTU_KERNEL_CODE := $(shell echo `grep UTS_RELEASE $(RTE_KERNELDIR)/include/generated/utsrelease.h \ | cut -d '"' -f2 | cut -d- -f1,2 | tr .- ,`,1) MODULE_CFLAGS += -D"UBUNTU_KERNEL_CODE=UBUNTU_KERNEL_VERSION($(UBUNTU_KERNEL_CODE))" endif -# this lib needs main eal -DEPDIRS-y += lib/librte_eal/linuxapp/eal - # # all source are stored in SRCS-y # -SRCS-y := ethtool/ixgbe/ixgbe_main.c -SRCS-y += ethtool/ixgbe/ixgbe_api.c -SRCS-y += ethtool/ixgbe/ixgbe_common.c -SRCS-y += ethtool/ixgbe/ixgbe_ethtool.c -SRCS-y += ethtool/ixgbe/ixgbe_82599.c -SRCS-y += ethtool/ixgbe/ixgbe_82598.c -SRCS-y += ethtool/ixgbe/ixgbe_x540.c -SRCS-y += ethtool/ixgbe/ixgbe_phy.c -SRCS-y += ethtool/ixgbe/kcompat.c - -SRCS-y += ethtool/igb/e1000_82575.c -SRCS-y += ethtool/igb/e1000_i210.c -SRCS-y += ethtool/igb/e1000_api.c -SRCS-y += ethtool/igb/e1000_mac.c -SRCS-y += ethtool/igb/e1000_manage.c -SRCS-y += ethtool/igb/e1000_mbx.c -SRCS-y += ethtool/igb/e1000_nvm.c -SRCS-y += ethtool/igb/e1000_phy.c -SRCS-y += ethtool/igb/igb_ethtool.c -SRCS-y += ethtool/igb/igb_hwmon.c -SRCS-y += ethtool/igb/igb_main.c -SRCS-y += ethtool/igb/igb_debugfs.c -SRCS-y += ethtool/igb/igb_param.c -SRCS-y += ethtool/igb/igb_procfs.c -SRCS-y += ethtool/igb/igb_vmdq.c -#SRCS-y += ethtool/igb/igb_ptp.c -#SRCS-y += ethtool/igb/kcompat.c - -SRCS-y += kni_misc.c +SRCS-y := kni_misc.c SRCS-y += kni_net.c -SRCS-y += kni_ethtool.c -SRCS-$(CONFIG_RTE_KNI_VHOST) += kni_vhost.c +SRCS-$(CONFIG_RTE_KNI_KMOD_ETHTOOL) += kni_ethtool.c + +SRCS-$(CONFIG_RTE_KNI_KMOD_ETHTOOL) += ethtool/ixgbe/ixgbe_main.c +SRCS-$(CONFIG_RTE_KNI_KMOD_ETHTOOL) += ethtool/ixgbe/ixgbe_api.c +SRCS-$(CONFIG_RTE_KNI_KMOD_ETHTOOL) += ethtool/ixgbe/ixgbe_common.c +SRCS-$(CONFIG_RTE_KNI_KMOD_ETHTOOL) += ethtool/ixgbe/ixgbe_ethtool.c +SRCS-$(CONFIG_RTE_KNI_KMOD_ETHTOOL) += ethtool/ixgbe/ixgbe_82599.c +SRCS-$(CONFIG_RTE_KNI_KMOD_ETHTOOL) += ethtool/ixgbe/ixgbe_82598.c +SRCS-$(CONFIG_RTE_KNI_KMOD_ETHTOOL) += ethtool/ixgbe/ixgbe_x540.c +SRCS-$(CONFIG_RTE_KNI_KMOD_ETHTOOL) += ethtool/ixgbe/ixgbe_phy.c +SRCS-$(CONFIG_RTE_KNI_KMOD_ETHTOOL) += ethtool/ixgbe/kcompat.c + +SRCS-$(CONFIG_RTE_KNI_KMOD_ETHTOOL) += ethtool/igb/e1000_82575.c +SRCS-$(CONFIG_RTE_KNI_KMOD_ETHTOOL) += ethtool/igb/e1000_i210.c +SRCS-$(CONFIG_RTE_KNI_KMOD_ETHTOOL) += ethtool/igb/e1000_api.c +SRCS-$(CONFIG_RTE_KNI_KMOD_ETHTOOL) += ethtool/igb/e1000_mac.c +SRCS-$(CONFIG_RTE_KNI_KMOD_ETHTOOL) += ethtool/igb/e1000_manage.c +SRCS-$(CONFIG_RTE_KNI_KMOD_ETHTOOL) += ethtool/igb/e1000_mbx.c +SRCS-$(CONFIG_RTE_KNI_KMOD_ETHTOOL) += ethtool/igb/e1000_nvm.c +SRCS-$(CONFIG_RTE_KNI_KMOD_ETHTOOL) += ethtool/igb/e1000_phy.c +SRCS-$(CONFIG_RTE_KNI_KMOD_ETHTOOL) += ethtool/igb/igb_ethtool.c +SRCS-$(CONFIG_RTE_KNI_KMOD_ETHTOOL) += ethtool/igb/igb_main.c +SRCS-$(CONFIG_RTE_KNI_KMOD_ETHTOOL) += ethtool/igb/igb_param.c +SRCS-$(CONFIG_RTE_KNI_KMOD_ETHTOOL) += ethtool/igb/igb_vmdq.c include $(RTE_SDK)/mk/rte.module.mk diff --git a/dpdk/lib/librte_eal/linuxapp/kni/compat.h b/dpdk/lib/librte_eal/linuxapp/kni/compat.h index 9bd11770..3f8c0bc8 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/compat.h +++ b/dpdk/lib/librte_eal/linuxapp/kni/compat.h @@ -8,6 +8,34 @@ #define RHEL_RELEASE_VERSION(a, b) (((a) << 8) + (b)) #endif +/* SuSE version macro is the same as Linux kernel version */ +#ifndef SLE_VERSION +#define SLE_VERSION(a, b, c) KERNEL_VERSION(a, b, c) +#endif +#ifdef CONFIG_SUSE_KERNEL +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 57)) +/* SLES12SP3 is at least 4.4.57+ based */ +#define SLE_VERSION_CODE SLE_VERSION(12, 3, 0) +#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 12, 28)) +/* SLES12 is at least 3.12.28+ based */ +#define SLE_VERSION_CODE SLE_VERSION(12, 0, 0) +#elif ((LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 61)) && \ + (LINUX_VERSION_CODE < KERNEL_VERSION(3, 1, 0))) +/* SLES11 SP3 is at least 3.0.61+ based */ +#define SLE_VERSION_CODE SLE_VERSION(11, 3, 0) +#elif (LINUX_VERSION_CODE == KERNEL_VERSION(2, 6, 32)) +/* SLES11 SP1 is 2.6.32 based */ +#define SLE_VERSION_CODE SLE_VERSION(11, 1, 0) +#elif (LINUX_VERSION_CODE == KERNEL_VERSION(2, 6, 27)) +/* SLES11 GA is 2.6.27 based */ +#define SLE_VERSION_CODE SLE_VERSION(11, 0, 0) +#endif /* LINUX_VERSION_CODE == KERNEL_VERSION(x,y,z) */ +#endif /* CONFIG_SUSE_KERNEL */ +#ifndef SLE_VERSION_CODE +#define SLE_VERSION_CODE 0 +#endif /* SLE_VERSION_CODE */ + + #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 39) && \ (!(defined(RHEL_RELEASE_CODE) && \ RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(6, 4))) @@ -21,13 +49,25 @@ #endif #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 35) -#define sk_sleep(s) (s)->sk_sleep +#define sk_sleep(s) ((s)->sk_sleep) +#else +#define HAVE_SOCKET_WQ +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0) +#define HAVE_STATIC_SOCK_MAP_FD +#else +#define kni_sock_map_fd(s) sock_map_fd(s, 0) #endif #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 9, 0) #define HAVE_CHANGE_CARRIER_CB #endif +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 14, 0) +#define ether_addr_copy(dst, src) memcpy(dst, src, ETH_ALEN) +#endif + #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0) #define HAVE_IOV_ITER_MSGHDR #endif @@ -37,10 +77,30 @@ #define HAVE_REBUILD_HEADER #endif -#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 7, 0) +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 2, 0) +#define HAVE_SK_ALLOC_KERN_PARAM +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 7, 0) || \ + (defined(RHEL_RELEASE_CODE) && \ + RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(7, 4)) || \ + (SLE_VERSION_CODE && SLE_VERSION_CODE == SLE_VERSION(12, 3, 0)) #define HAVE_TRANS_START_HELPER #endif +/* + * KNI uses NET_NAME_UNKNOWN macro to select correct version of alloc_netdev() + * For old kernels just backported the commit that enables the macro + * (685343fc3ba6) but still uses old API, it is required to undefine macro to + * select correct version of API, this is safe since KNI doesn't use the value. + * This fix is specific to RedHat/CentOS kernels. + */ +#if (defined(RHEL_RELEASE_CODE) && \ + (RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(6, 8)) && \ + (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 34))) +#undef NET_NAME_UNKNOWN +#endif + #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0) #define HAVE_SIGNAL_FUNCTIONS_OWN_HEADER #endif diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/COPYING b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/COPYING deleted file mode 100644 index 5f297e5b..00000000 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/COPYING +++ /dev/null @@ -1,339 +0,0 @@ - -"This software program is licensed subject to the GNU General Public License -(GPL). Version 2, June 1991, available at -" - -GNU General Public License - -Version 2, June 1991 - -Copyright (C) 1989, 1991 Free Software Foundation, Inc. -59 Temple Place - Suite 330, Boston, MA 02111-1307, USA - -Everyone is permitted to copy and distribute verbatim copies of this license -document, but changing it is not allowed. - -Preamble - -The licenses for most software are designed to take away your freedom to -share and change it. By contrast, the GNU General Public License is intended -to guarantee your freedom to share and change free software--to make sure -the software is free for all its users. This General Public License applies -to most of the Free Software Foundation's software and to any other program -whose authors commit to using it. (Some other Free Software Foundation -software is covered by the GNU Library General Public License instead.) You -can apply it to your programs, too. - -When we speak of free software, we are referring to freedom, not price. Our -General Public Licenses are designed to make sure that you have the freedom -to distribute copies of free software (and charge for this service if you -wish), that you receive source code or can get it if you want it, that you -can change the software or use pieces of it in new free programs; and that -you know you can do these things. - -To protect your rights, we need to make restrictions that forbid anyone to -deny you these rights or to ask you to surrender the rights. These -restrictions translate to certain responsibilities for you if you distribute -copies of the software, or if you modify it. - -For example, if you distribute copies of such a program, whether gratis or -for a fee, you must give the recipients all the rights that you have. You -must make sure that they, too, receive or can get the source code. And you -must show them these terms so they know their rights. - -We protect your rights with two steps: (1) copyright the software, and (2) -offer you this license which gives you legal permission to copy, distribute -and/or modify the software. - -Also, for each author's protection and ours, we want to make certain that -everyone understands that there is no warranty for this free software. If -the software is modified by someone else and passed on, we want its -recipients to know that what they have is not the original, so that any -problems introduced by others will not reflect on the original authors' -reputations. - -Finally, any free program is threatened constantly by software patents. We -wish to avoid the danger that redistributors of a free program will -individually obtain patent licenses, in effect making the program -proprietary. To prevent this, we have made it clear that any patent must be -licensed for everyone's free use or not licensed at all. - -The precise terms and conditions for copying, distribution and modification -follow. - -TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - -0. This License applies to any program or other work which contains a notice - placed by the copyright holder saying it may be distributed under the - terms of this General Public License. The "Program", below, refers to any - such program or work, and a "work based on the Program" means either the - Program or any derivative work under copyright law: that is to say, a - work containing the Program or a portion of it, either verbatim or with - modifications and/or translated into another language. (Hereinafter, - translation is included without limitation in the term "modification".) - Each licensee is addressed as "you". - - Activities other than copying, distribution and modification are not - covered by this License; they are outside its scope. The act of running - the Program is not restricted, and the output from the Program is covered - only if its contents constitute a work based on the Program (independent - of having been made by running the Program). Whether that is true depends - on what the Program does. - -1. You may copy and distribute verbatim copies of the Program's source code - as you receive it, in any medium, provided that you conspicuously and - appropriately publish on each copy an appropriate copyright notice and - disclaimer of warranty; keep intact all the notices that refer to this - License and to the absence of any warranty; and give any other recipients - of the Program a copy of this License along with the Program. - - You may charge a fee for the physical act of transferring a copy, and you - may at your option offer warranty protection in exchange for a fee. - -2. You may modify your copy or copies of the Program or any portion of it, - thus forming a work based on the Program, and copy and distribute such - modifications or work under the terms of Section 1 above, provided that - you also meet all of these conditions: - - * a) You must cause the modified files to carry prominent notices stating - that you changed the files and the date of any change. - - * b) You must cause any work that you distribute or publish, that in - whole or in part contains or is derived from the Program or any part - thereof, to be licensed as a whole at no charge to all third parties - under the terms of this License. - - * c) If the modified program normally reads commands interactively when - run, you must cause it, when started running for such interactive - use in the most ordinary way, to print or display an announcement - including an appropriate copyright notice and a notice that there is - no warranty (or else, saying that you provide a warranty) and that - users may redistribute the program under these conditions, and - telling the user how to view a copy of this License. (Exception: if - the Program itself is interactive but does not normally print such - an announcement, your work based on the Program is not required to - print an announcement.) - - These requirements apply to the modified work as a whole. If identifiable - sections of that work are not derived from the Program, and can be - reasonably considered independent and separate works in themselves, then - this License, and its terms, do not apply to those sections when you - distribute them as separate works. But when you distribute the same - sections as part of a whole which is a work based on the Program, the - distribution of the whole must be on the terms of this License, whose - permissions for other licensees extend to the entire whole, and thus to - each and every part regardless of who wrote it. - - Thus, it is not the intent of this section to claim rights or contest - your rights to work written entirely by you; rather, the intent is to - exercise the right to control the distribution of derivative or - collective works based on the Program. - - In addition, mere aggregation of another work not based on the Program - with the Program (or with a work based on the Program) on a volume of a - storage or distribution medium does not bring the other work under the - scope of this License. - -3. You may copy and distribute the Program (or a work based on it, under - Section 2) in object code or executable form under the terms of Sections - 1 and 2 above provided that you also do one of the following: - - * a) Accompany it with the complete corresponding machine-readable source - code, which must be distributed under the terms of Sections 1 and 2 - above on a medium customarily used for software interchange; or, - - * b) Accompany it with a written offer, valid for at least three years, - to give any third party, for a charge no more than your cost of - physically performing source distribution, a complete machine- - readable copy of the corresponding source code, to be distributed - under the terms of Sections 1 and 2 above on a medium customarily - used for software interchange; or, - - * c) Accompany it with the information you received as to the offer to - distribute corresponding source code. (This alternative is allowed - only for noncommercial distribution and only if you received the - program in object code or executable form with such an offer, in - accord with Subsection b above.) - - The source code for a work means the preferred form of the work for - making modifications to it. For an executable work, complete source code - means all the source code for all modules it contains, plus any - associated interface definition files, plus the scripts used to control - compilation and installation of the executable. However, as a special - exception, the source code distributed need not include anything that is - normally distributed (in either source or binary form) with the major - components (compiler, kernel, and so on) of the operating system on which - the executable runs, unless that component itself accompanies the - executable. - - If distribution of executable or object code is made by offering access - to copy from a designated place, then offering equivalent access to copy - the source code from the same place counts as distribution of the source - code, even though third parties are not compelled to copy the source - along with the object code. - -4. You may not copy, modify, sublicense, or distribute the Program except as - expressly provided under this License. Any attempt otherwise to copy, - modify, sublicense or distribute the Program is void, and will - automatically terminate your rights under this License. However, parties - who have received copies, or rights, from you under this License will not - have their licenses terminated so long as such parties remain in full - compliance. - -5. You are not required to accept this License, since you have not signed - it. However, nothing else grants you permission to modify or distribute - the Program or its derivative works. These actions are prohibited by law - if you do not accept this License. Therefore, by modifying or - distributing the Program (or any work based on the Program), you - indicate your acceptance of this License to do so, and all its terms and - conditions for copying, distributing or modifying the Program or works - based on it. - -6. Each time you redistribute the Program (or any work based on the - Program), the recipient automatically receives a license from the - original licensor to copy, distribute or modify the Program subject to - these terms and conditions. You may not impose any further restrictions - on the recipients' exercise of the rights granted herein. You are not - responsible for enforcing compliance by third parties to this License. - -7. If, as a consequence of a court judgment or allegation of patent - infringement or for any other reason (not limited to patent issues), - conditions are imposed on you (whether by court order, agreement or - otherwise) that contradict the conditions of this License, they do not - excuse you from the conditions of this License. If you cannot distribute - so as to satisfy simultaneously your obligations under this License and - any other pertinent obligations, then as a consequence you may not - distribute the Program at all. For example, if a patent license would - not permit royalty-free redistribution of the Program by all those who - receive copies directly or indirectly through you, then the only way you - could satisfy both it and this License would be to refrain entirely from - distribution of the Program. - - If any portion of this section is held invalid or unenforceable under any - particular circumstance, the balance of the section is intended to apply - and the section as a whole is intended to apply in other circumstances. - - It is not the purpose of this section to induce you to infringe any - patents or other property right claims or to contest validity of any - such claims; this section has the sole purpose of protecting the - integrity of the free software distribution system, which is implemented - by public license practices. Many people have made generous contributions - to the wide range of software distributed through that system in - reliance on consistent application of that system; it is up to the - author/donor to decide if he or she is willing to distribute software - through any other system and a licensee cannot impose that choice. - - This section is intended to make thoroughly clear what is believed to be - a consequence of the rest of this License. - -8. If the distribution and/or use of the Program is restricted in certain - countries either by patents or by copyrighted interfaces, the original - copyright holder who places the Program under this License may add an - explicit geographical distribution limitation excluding those countries, - so that distribution is permitted only in or among countries not thus - excluded. In such case, this License incorporates the limitation as if - written in the body of this License. - -9. The Free Software Foundation may publish revised and/or new versions of - the General Public License from time to time. Such new versions will be - similar in spirit to the present version, but may differ in detail to - address new problems or concerns. - - Each version is given a distinguishing version number. If the Program - specifies a version number of this License which applies to it and "any - later version", you have the option of following the terms and - conditions either of that version or of any later version published by - the Free Software Foundation. If the Program does not specify a version - number of this License, you may choose any version ever published by the - Free Software Foundation. - -10. If you wish to incorporate parts of the Program into other free programs - whose distribution conditions are different, write to the author to ask - for permission. For software which is copyrighted by the Free Software - Foundation, write to the Free Software Foundation; we sometimes make - exceptions for this. Our decision will be guided by the two goals of - preserving the free status of all derivatives of our free software and - of promoting the sharing and reuse of software generally. - - NO WARRANTY - -11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY - FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN - OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES - PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER - EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE - ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH - YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL - NECESSARY SERVICING, REPAIR OR CORRECTION. - -12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING - WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR - REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR - DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL - DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM - (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED - INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF - THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR - OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. - -END OF TERMS AND CONDITIONS - -How to Apply These Terms to Your New Programs - -If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it free -software which everyone can redistribute and change under these terms. - -To do so, attach the following notices to the program. It is safest to -attach them to the start of each source file to most effectively convey the -exclusion of warranty; and each file should have at least the "copyright" -line and a pointer to where the full notice is found. - -one line to give the program's name and an idea of what it does. -Copyright (C) yyyy name of author - -This program is free software; you can redistribute it and/or modify it -under the terms of the GNU General Public License as published by the Free -Software Foundation; either version 2 of the License, or (at your option) -any later version. - -This program is distributed in the hope that it will be useful, but WITHOUT -ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -more details. - -You should have received a copy of the GNU General Public License along with -this program; if not, write to the Free Software Foundation, Inc., 59 -Temple Place - Suite 330, Boston, MA 02111-1307, USA. - -Also add information on how to contact you by electronic and paper mail. - -If the program is interactive, make it output a short notice like this when -it starts in an interactive mode: - -Gnomovision version 69, Copyright (C) year name of author Gnomovision comes -with ABSOLUTELY NO WARRANTY; for details type 'show w'. This is free -software, and you are welcome to redistribute it under certain conditions; -type 'show c' for details. - -The hypothetical commands 'show w' and 'show c' should show the appropriate -parts of the General Public License. Of course, the commands you use may be -called something other than 'show w' and 'show c'; they could even be -mouse-clicks or menu items--whatever suits your program. - -You should also get your employer (if you work as a programmer) or your -school, if any, to sign a "copyright disclaimer" for the program, if -necessary. Here is a sample; alter the names: - -Yoyodyne, Inc., hereby disclaims all copyright interest in the program -'Gnomovision' (which makes passes at compilers) written by James Hacker. - -signature of Ty Coon, 1 April 1989 -Ty Coon, President of Vice - -This General Public License does not permit incorporating your program into -proprietary programs. If your program is a subroutine library, you may -consider it more useful to permit linking proprietary applications with the -library. If this is what you want to do, use the GNU Library General Public -License instead of this License. diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_82575.c b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_82575.c index b8c9a13f..5da7f91f 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_82575.c +++ b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_82575.c @@ -17,7 +17,7 @@ 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. The full GNU General Public License is included in this distribution in - the file called "COPYING". + the file called "LICENSE.GPL". Contact Information: e1000-devel Mailing List @@ -241,7 +241,7 @@ static s32 e1000_init_phy_params_82575(struct e1000_hw *hw) else phy->ops.get_cable_length = e1000_get_cable_length_m88; phy->ops.force_speed_duplex = e1000_phy_force_speed_duplex_m88; - /* Check if this PHY is confgured for media swap. */ + /* Check if this PHY is configured for media swap. */ if (phy->id == M88E1112_E_PHY_ID) { u16 data; @@ -1357,7 +1357,7 @@ static s32 e1000_get_pcs_speed_and_duplex_82575(struct e1000_hw *hw, * @hw: pointer to the HW structure * * In the case of serdes shut down sfp and PCS on driver unload - * when management pass thru is not enabled. + * when management pass through is not enabled. **/ void e1000_shutdown_serdes_link_82575(struct e1000_hw *hw) { diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_82575.h b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_82575.h index 1aec75ab..185ccdf1 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_82575.h +++ b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_82575.h @@ -17,7 +17,7 @@ 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. The full GNU General Public License is included in this distribution in - the file called "COPYING". + the file called "LICENSE.GPL". Contact Information: e1000-devel Mailing List diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_api.c b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_api.c index 6095d3b4..220c9a40 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_api.c +++ b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_api.c @@ -17,7 +17,7 @@ 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. The full GNU General Public License is included in this distribution in - the file called "COPYING". + the file called "LICENSE.GPL". Contact Information: e1000-devel Mailing List diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_api.h b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_api.h index b21294ec..55c8a5f4 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_api.h +++ b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_api.h @@ -17,7 +17,7 @@ 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. The full GNU General Public License is included in this distribution in - the file called "COPYING". + the file called "LICENSE.GPL". Contact Information: e1000-devel Mailing List diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_defines.h b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_defines.h index 63b228c5..d42c7998 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_defines.h +++ b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_defines.h @@ -17,7 +17,7 @@ 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. The full GNU General Public License is included in this distribution in - the file called "COPYING". + the file called "LICENSE.GPL". Contact Information: e1000-devel Mailing List diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_hw.h b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_hw.h index 347cef71..35886e93 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_hw.h +++ b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_hw.h @@ -17,7 +17,7 @@ 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. The full GNU General Public License is included in this distribution in - the file called "COPYING". + the file called "LICENSE.GPL". Contact Information: e1000-devel Mailing List diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_i210.c b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_i210.c index 1e9f3e6e..7e4c20a9 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_i210.c +++ b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_i210.c @@ -17,7 +17,7 @@ 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. The full GNU General Public License is included in this distribution in - the file called "COPYING". + the file called "LICENSE.GPL". Contact Information: e1000-devel Mailing List diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_i210.h b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_i210.h index 57b2eb56..b8fa70d0 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_i210.h +++ b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_i210.h @@ -17,7 +17,7 @@ 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. The full GNU General Public License is included in this distribution in - the file called "COPYING". + the file called "LICENSE.GPL". Contact Information: e1000-devel Mailing List diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_mac.c b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_mac.c index 4ee59ba9..74319def 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_mac.c +++ b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_mac.c @@ -17,7 +17,7 @@ 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. The full GNU General Public License is included in this distribution in - the file called "COPYING". + the file called "LICENSE.GPL". Contact Information: e1000-devel Mailing List diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_mac.h b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_mac.h index 6a1b0f52..3bcdd88c 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_mac.h +++ b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_mac.h @@ -17,7 +17,7 @@ 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. The full GNU General Public License is included in this distribution in - the file called "COPYING". + the file called "LICENSE.GPL". Contact Information: e1000-devel Mailing List diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_manage.c b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_manage.c index a1700398..51dfae5d 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_manage.c +++ b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_manage.c @@ -17,7 +17,7 @@ 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. The full GNU General Public License is included in this distribution in - the file called "COPYING". + the file called "LICENSE.GPL". Contact Information: e1000-devel Mailing List diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_manage.h b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_manage.h index c94b2185..0627f271 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_manage.h +++ b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_manage.h @@ -17,7 +17,7 @@ 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. The full GNU General Public License is included in this distribution in - the file called "COPYING". + the file called "LICENSE.GPL". Contact Information: e1000-devel Mailing List diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_mbx.c b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_mbx.c index 3ef0d98b..bd64429f 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_mbx.c +++ b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_mbx.c @@ -17,7 +17,7 @@ 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. The full GNU General Public License is included in this distribution in - the file called "COPYING". + the file called "LICENSE.GPL". Contact Information: e1000-devel Mailing List diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_mbx.h b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_mbx.h index bbf838c8..64685d9d 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_mbx.h +++ b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_mbx.h @@ -17,7 +17,7 @@ 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. The full GNU General Public License is included in this distribution in - the file called "COPYING". + the file called "LICENSE.GPL". Contact Information: e1000-devel Mailing List diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_nvm.c b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_nvm.c index 6188d007..1ce59154 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_nvm.c +++ b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_nvm.c @@ -17,7 +17,7 @@ 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. The full GNU General Public License is included in this distribution in - the file called "COPYING". + the file called "LICENSE.GPL". Contact Information: e1000-devel Mailing List diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_nvm.h b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_nvm.h index fe62785a..17bc53c3 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_nvm.h +++ b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_nvm.h @@ -17,7 +17,7 @@ 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. The full GNU General Public License is included in this distribution in - the file called "COPYING". + the file called "LICENSE.GPL". Contact Information: e1000-devel Mailing List diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_osdep.h b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_osdep.h index d1cf98e2..c1ab60c4 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_osdep.h +++ b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_osdep.h @@ -17,7 +17,7 @@ 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. The full GNU General Public License is included in this distribution in - the file called "COPYING". + the file called "LICENSE.GPL". Contact Information: e1000-devel Mailing List diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_phy.c b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_phy.c index 140a2a47..d8a77c45 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_phy.c +++ b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_phy.c @@ -17,7 +17,7 @@ 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. The full GNU General Public License is included in this distribution in - the file called "COPYING". + the file called "LICENSE.GPL". Contact Information: e1000-devel Mailing List diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_phy.h b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_phy.h index 5387c5e7..db24fb0b 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_phy.h +++ b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_phy.h @@ -17,7 +17,7 @@ 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. The full GNU General Public License is included in this distribution in - the file called "COPYING". + the file called "LICENSE.GPL". Contact Information: e1000-devel Mailing List diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_regs.h b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_regs.h index 0e083c54..830ec991 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_regs.h +++ b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_regs.h @@ -17,7 +17,7 @@ 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. The full GNU General Public License is included in this distribution in - the file called "COPYING". + the file called "LICENSE.GPL". Contact Information: e1000-devel Mailing List diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/igb.h b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/igb.h index e5554ca3..8667f29c 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/igb.h +++ b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/igb.h @@ -17,7 +17,7 @@ 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. The full GNU General Public License is included in this distribution in - the file called "COPYING". + the file called "LICENSE.GPL". Contact Information: e1000-devel Mailing List @@ -607,7 +607,7 @@ struct igb_adapter { int int_mode; u32 rss_queues; u32 vmdq_pools; - char fw_version[32]; + char fw_version[43]; u32 wvbr; struct igb_mac_addr *mac_table; #ifdef CONFIG_IGB_VMDQ_NETDEV diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/igb_debugfs.c b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/igb_debugfs.c deleted file mode 100644 index c07f9f53..00000000 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/igb_debugfs.c +++ /dev/null @@ -1,28 +0,0 @@ -/******************************************************************************* - - Intel(R) Gigabit Ethernet Linux driver - Copyright(c) 2007-2013 Intel Corporation. - - This program is free software; you can redistribute it and/or modify it - under the terms and conditions of the GNU General Public License, - version 2, as published by the Free Software Foundation. - - This program is distributed in the hope it will be useful, but WITHOUT - ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - more details. - - You should have received a copy of the GNU General Public License along with - this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. - - The full GNU General Public License is included in this distribution in - the file called "COPYING". - - Contact Information: - e1000-devel Mailing List - Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 - -*******************************************************************************/ - -#include "igb.h" diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/igb_ethtool.c b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/igb_ethtool.c index af7e68a5..95e262b7 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/igb_ethtool.c +++ b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/igb_ethtool.c @@ -17,7 +17,7 @@ 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. The full GNU General Public License is included in this distribution in - the file called "COPYING". + the file called "LICENSE.GPL". Contact Information: e1000-devel Mailing List @@ -1126,7 +1126,7 @@ static int igb_eeprom_test(struct igb_adapter *adapter, u64 *data) static irqreturn_t igb_test_intr(int irq, void *data) { - struct igb_adapter *adapter = (struct igb_adapter *) data; + struct igb_adapter *adapter = data; struct e1000_hw *hw = &adapter->hw; adapter->test_icr |= E1000_READ_REG(hw, E1000_ICR); diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/igb_hwmon.c b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/igb_hwmon.c deleted file mode 100644 index 07a1ae07..00000000 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/igb_hwmon.c +++ /dev/null @@ -1,260 +0,0 @@ -/******************************************************************************* - - Intel(R) Gigabit Ethernet Linux driver - Copyright(c) 2007-2013 Intel Corporation. - - This program is free software; you can redistribute it and/or modify it - under the terms and conditions of the GNU General Public License, - version 2, as published by the Free Software Foundation. - - This program is distributed in the hope it will be useful, but WITHOUT - ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - more details. - - You should have received a copy of the GNU General Public License along with - this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. - - The full GNU General Public License is included in this distribution in - the file called "COPYING". - - Contact Information: - e1000-devel Mailing List - Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 - -*******************************************************************************/ - -#include "igb.h" -#include "e1000_82575.h" -#include "e1000_hw.h" -#ifdef IGB_HWMON -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef HAVE_I2C_SUPPORT -static struct i2c_board_info i350_sensor_info = { - I2C_BOARD_INFO("i350bb", (0Xf8 >> 1)), -}; -#endif /* HAVE_I2C_SUPPORT */ - -/* hwmon callback functions */ -static ssize_t igb_hwmon_show_location(struct device *dev, - struct device_attribute *attr, - char *buf) -{ - struct hwmon_attr *igb_attr = container_of(attr, struct hwmon_attr, - dev_attr); - return sprintf(buf, "loc%u\n", - igb_attr->sensor->location); -} - -static ssize_t igb_hwmon_show_temp(struct device *dev, - struct device_attribute *attr, - char *buf) -{ - struct hwmon_attr *igb_attr = container_of(attr, struct hwmon_attr, - dev_attr); - unsigned int value; - - /* reset the temp field */ - igb_attr->hw->mac.ops.get_thermal_sensor_data(igb_attr->hw); - - value = igb_attr->sensor->temp; - - /* display millidegree */ - value *= 1000; - - return sprintf(buf, "%u\n", value); -} - -static ssize_t igb_hwmon_show_cautionthresh(struct device *dev, - struct device_attribute *attr, - char *buf) -{ - struct hwmon_attr *igb_attr = container_of(attr, struct hwmon_attr, - dev_attr); - unsigned int value = igb_attr->sensor->caution_thresh; - - /* display millidegree */ - value *= 1000; - - return sprintf(buf, "%u\n", value); -} - -static ssize_t igb_hwmon_show_maxopthresh(struct device *dev, - struct device_attribute *attr, - char *buf) -{ - struct hwmon_attr *igb_attr = container_of(attr, struct hwmon_attr, - dev_attr); - unsigned int value = igb_attr->sensor->max_op_thresh; - - /* display millidegree */ - value *= 1000; - - return sprintf(buf, "%u\n", value); -} - -/* igb_add_hwmon_attr - Create hwmon attr table for a hwmon sysfs file. - * @ adapter: pointer to the adapter structure - * @ offset: offset in the eeprom sensor data table - * @ type: type of sensor data to display - * - * For each file we want in hwmon's sysfs interface we need a device_attribute - * This is included in our hwmon_attr struct that contains the references to - * the data structures we need to get the data to display. - */ -static int igb_add_hwmon_attr(struct igb_adapter *adapter, - unsigned int offset, int type) { - int rc; - unsigned int n_attr; - struct hwmon_attr *igb_attr; - - n_attr = adapter->igb_hwmon_buff.n_hwmon; - igb_attr = &adapter->igb_hwmon_buff.hwmon_list[n_attr]; - - switch (type) { - case IGB_HWMON_TYPE_LOC: - igb_attr->dev_attr.show = igb_hwmon_show_location; - snprintf(igb_attr->name, sizeof(igb_attr->name), - "temp%u_label", offset); - break; - case IGB_HWMON_TYPE_TEMP: - igb_attr->dev_attr.show = igb_hwmon_show_temp; - snprintf(igb_attr->name, sizeof(igb_attr->name), - "temp%u_input", offset); - break; - case IGB_HWMON_TYPE_CAUTION: - igb_attr->dev_attr.show = igb_hwmon_show_cautionthresh; - snprintf(igb_attr->name, sizeof(igb_attr->name), - "temp%u_max", offset); - break; - case IGB_HWMON_TYPE_MAX: - igb_attr->dev_attr.show = igb_hwmon_show_maxopthresh; - snprintf(igb_attr->name, sizeof(igb_attr->name), - "temp%u_crit", offset); - break; - default: - rc = -EPERM; - return rc; - } - - /* These always the same regardless of type */ - igb_attr->sensor = - &adapter->hw.mac.thermal_sensor_data.sensor[offset]; - igb_attr->hw = &adapter->hw; - igb_attr->dev_attr.store = NULL; - igb_attr->dev_attr.attr.mode = S_IRUGO; - igb_attr->dev_attr.attr.name = igb_attr->name; - sysfs_attr_init(&igb_attr->dev_attr.attr); - rc = device_create_file(&adapter->pdev->dev, - &igb_attr->dev_attr); - if (rc == 0) - ++adapter->igb_hwmon_buff.n_hwmon; - - return rc; -} - -static void igb_sysfs_del_adapter(struct igb_adapter *adapter) -{ - int i; - - if (adapter == NULL) - return; - - for (i = 0; i < adapter->igb_hwmon_buff.n_hwmon; i++) { - device_remove_file(&adapter->pdev->dev, - &adapter->igb_hwmon_buff.hwmon_list[i].dev_attr); - } - - kfree(adapter->igb_hwmon_buff.hwmon_list); - - if (adapter->igb_hwmon_buff.device) - hwmon_device_unregister(adapter->igb_hwmon_buff.device); -} - -/* called from igb_main.c */ -void igb_sysfs_exit(struct igb_adapter *adapter) -{ - igb_sysfs_del_adapter(adapter); -} - -/* called from igb_main.c */ -int igb_sysfs_init(struct igb_adapter *adapter) -{ - struct hwmon_buff *igb_hwmon = &adapter->igb_hwmon_buff; - unsigned int i; - int n_attrs; - int rc = 0; -#ifdef HAVE_I2C_SUPPORT - struct i2c_client *client = NULL; -#endif /* HAVE_I2C_SUPPORT */ - - /* If this method isn't defined we don't support thermals */ - if (adapter->hw.mac.ops.init_thermal_sensor_thresh == NULL) - goto exit; - - /* Don't create thermal hwmon interface if no sensors present */ - rc = (adapter->hw.mac.ops.init_thermal_sensor_thresh(&adapter->hw)); - if (rc) - goto exit; -#ifdef HAVE_I2C_SUPPORT - /* init i2c_client */ - client = i2c_new_device(&adapter->i2c_adap, &i350_sensor_info); - if (client == NULL) { - dev_info(&adapter->pdev->dev, - "Failed to create new i2c device..\n"); - goto exit; - } - adapter->i2c_client = client; -#endif /* HAVE_I2C_SUPPORT */ - - /* Allocation space for max attributes - * max num sensors * values (loc, temp, max, caution) - */ - n_attrs = E1000_MAX_SENSORS * 4; - igb_hwmon->hwmon_list = kcalloc(n_attrs, sizeof(struct hwmon_attr), - GFP_KERNEL); - if (!igb_hwmon->hwmon_list) { - rc = -ENOMEM; - goto err; - } - - igb_hwmon->device = hwmon_device_register(&adapter->pdev->dev); - if (IS_ERR(igb_hwmon->device)) { - rc = PTR_ERR(igb_hwmon->device); - goto err; - } - - for (i = 0; i < E1000_MAX_SENSORS; i++) { - - /* Only create hwmon sysfs entries for sensors that have - * meaningful data. - */ - if (adapter->hw.mac.thermal_sensor_data.sensor[i].location == 0) - continue; - - /* Bail if any hwmon attr struct fails to initialize */ - rc = igb_add_hwmon_attr(adapter, i, IGB_HWMON_TYPE_CAUTION); - rc |= igb_add_hwmon_attr(adapter, i, IGB_HWMON_TYPE_LOC); - rc |= igb_add_hwmon_attr(adapter, i, IGB_HWMON_TYPE_TEMP); - rc |= igb_add_hwmon_attr(adapter, i, IGB_HWMON_TYPE_MAX); - if (rc) - goto err; - } - - goto exit; - -err: - igb_sysfs_del_adapter(adapter); -exit: - return rc; -} -#endif /* IGB_HWMON */ diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/igb_main.c b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/igb_main.c index 2aa4f28e..e0f427a3 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/igb_main.c +++ b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/igb_main.c @@ -17,7 +17,7 @@ 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. The full GNU General Public License is included in this distribution in - the file called "COPYING". + the file called "LICENSE.GPL". Contact Information: e1000-devel Mailing List @@ -137,11 +137,20 @@ static void igb_clean_all_tx_rings(struct igb_adapter *); static void igb_clean_all_rx_rings(struct igb_adapter *); static void igb_clean_tx_ring(struct igb_ring *); static void igb_set_rx_mode(struct net_device *); +#ifdef HAVE_TIMER_SETUP +static void igb_update_phy_info(struct timer_list *); +static void igb_watchdog(struct timer_list *); +#else static void igb_update_phy_info(unsigned long); static void igb_watchdog(unsigned long); +#endif static void igb_watchdog_task(struct work_struct *); static void igb_dma_err_task(struct work_struct *); +#ifdef HAVE_TIMER_SETUP +static void igb_dma_err_timer(struct timer_list *); +#else static void igb_dma_err_timer(unsigned long data); +#endif static netdev_tx_t igb_xmit_frame(struct sk_buff *skb, struct net_device *); static struct net_device_stats *igb_get_stats(struct net_device *); static int igb_change_mtu(struct net_device *, int); @@ -1035,10 +1044,10 @@ static void igb_set_interrupt_capability(struct igb_adapter *adapter, bool msix) err = pci_enable_msix(pdev, adapter->msix_entries, numvecs); #else - err = pci_enable_msix_range(pdev, - adapter->msix_entries, - numvecs, - numvecs); + err = pci_enable_msix_range(pdev, + adapter->msix_entries, + numvecs, + numvecs); #endif if (err == 0) break; @@ -1133,7 +1142,7 @@ static int igb_alloc_q_vector(struct igb_adapter *adapter, /* initialize pointer to rings */ ring = q_vector->ring; - /* intialize ITR */ + /* initialize ITR */ if (rxr_count) { /* rx or rx/tx vector */ if (!adapter->rx_itr_setting || adapter->rx_itr_setting > 3) @@ -1569,6 +1578,7 @@ static void igb_check_swap_media(struct igb_adapter *adapter) ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); connsw = E1000_READ_REG(hw, E1000_CONNSW); link = igb_has_link(adapter); + (void) link; /* need to live swap if current media is copper and we have fiber/serdes * to go to. @@ -1635,7 +1645,7 @@ static void igb_check_swap_media(struct igb_adapter *adapter) */ static int igb_get_i2c_data(void *data) { - struct igb_adapter *adapter = (struct igb_adapter *)data; + struct igb_adapter *adapter = data; struct e1000_hw *hw = &adapter->hw; s32 i2cctl = E1000_READ_REG(hw, E1000_I2CPARAMS); @@ -1650,7 +1660,7 @@ static int igb_get_i2c_data(void *data) */ static void igb_set_i2c_data(void *data, int state) { - struct igb_adapter *adapter = (struct igb_adapter *)data; + struct igb_adapter *adapter = data; struct e1000_hw *hw = &adapter->hw; s32 i2cctl = E1000_READ_REG(hw, E1000_I2CPARAMS); @@ -1675,7 +1685,7 @@ static void igb_set_i2c_data(void *data, int state) */ static void igb_set_i2c_clk(void *data, int state) { - struct igb_adapter *adapter = (struct igb_adapter *)data; + struct igb_adapter *adapter = data; struct e1000_hw *hw = &adapter->hw; s32 i2cctl = E1000_READ_REG(hw, E1000_I2CPARAMS); @@ -1697,7 +1707,7 @@ static void igb_set_i2c_clk(void *data, int state) */ static int igb_get_i2c_clk(void *data) { - struct igb_adapter *adapter = (struct igb_adapter *)data; + struct igb_adapter *adapter = data; struct e1000_hw *hw = &adapter->hw; s32 i2cctl = E1000_READ_REG(hw, E1000_I2CPARAMS); @@ -2805,6 +2815,12 @@ static int __devinit igb_probe(struct pci_dev *pdev, /* Check if Media Autosense is enabled */ if (hw->mac.type == e1000_82580) igb_init_mas(adapter); +#ifdef HAVE_TIMER_SETUP + timer_setup(&adapter->watchdog_timer, &igb_watchdog, 0); + if (adapter->flags & IGB_FLAG_DETECT_BAD_DMA) + timer_setup(&adapter->dma_err_timer, &igb_dma_err_timer, 0); + timer_setup(&adapter->phy_info_timer, &igb_update_phy_info, 0); +#else setup_timer(&adapter->watchdog_timer, &igb_watchdog, (unsigned long) adapter); if (adapter->flags & IGB_FLAG_DETECT_BAD_DMA) @@ -2812,6 +2828,7 @@ static int __devinit igb_probe(struct pci_dev *pdev, (unsigned long) adapter); setup_timer(&adapter->phy_info_timer, &igb_update_phy_info, (unsigned long) adapter); +#endif INIT_WORK(&adapter->reset_task, igb_reset_task); INIT_WORK(&adapter->watchdog_task, igb_watchdog_task); @@ -4542,9 +4559,15 @@ static void igb_spoof_check(struct igb_adapter *adapter) /* Need to wait a few seconds after link up to get diagnostic information from * the phy */ +#ifdef HAVE_TIMER_SETUP +static void igb_update_phy_info(struct timer_list *t) +{ + struct igb_adapter *adapter = from_timer(adapter, t, phy_info_timer); +#else static void igb_update_phy_info(unsigned long data) { struct igb_adapter *adapter = (struct igb_adapter *) data; +#endif e1000_get_phy_info(&adapter->hw); } @@ -4593,9 +4616,15 @@ bool igb_has_link(struct igb_adapter *adapter) * igb_watchdog - Timer Call-back * @data: pointer to adapter cast into an unsigned long **/ +#ifdef HAVE_TIMER_SETUP +static void igb_watchdog(struct timer_list *t) +{ + struct igb_adapter *adapter = from_timer(adapter, t, watchdog_timer); +#else static void igb_watchdog(unsigned long data) { struct igb_adapter *adapter = (struct igb_adapter *)data; +#endif /* Do the rest outside of interrupt context */ schedule_work(&adapter->watchdog_task); } @@ -4853,9 +4882,15 @@ dma_timer_reset: * igb_dma_err_timer - Timer Call-back * @data: pointer to adapter cast into an unsigned long **/ +#ifdef HAVE_TIMER_SETUP +static void igb_dma_err_timer(struct timer_list *t) +{ + struct igb_adapter *adapter = from_timer(adapter, t, dma_err_timer); +#else static void igb_dma_err_timer(unsigned long data) { struct igb_adapter *adapter = (struct igb_adapter *)data; +#endif /* Do the rest outside of interrupt context */ schedule_work(&adapter->dma_err_task); } @@ -10050,6 +10085,12 @@ int igb_kni_probe(struct pci_dev *pdev, igb_init_mas(adapter); #ifdef NO_KNI +#ifdef HAVE_TIMER_SETUP + timer_setup(&adapter->watchdog_timer, &igb_watchdog, 0); + if (adapter->flags & IGB_FLAG_DETECT_BAD_DMA) + timer_setup(&adapter->dma_err_timer, &igb_dma_err_timer, 0); + timer_setup(&adapter->phy_info_timer, &igb_update_phy_info, 0); +#else setup_timer(&adapter->watchdog_timer, &igb_watchdog, (unsigned long) adapter); if (adapter->flags & IGB_FLAG_DETECT_BAD_DMA) @@ -10057,6 +10098,7 @@ int igb_kni_probe(struct pci_dev *pdev, (unsigned long) adapter); setup_timer(&adapter->phy_info_timer, &igb_update_phy_info, (unsigned long) adapter); +#endif INIT_WORK(&adapter->reset_task, igb_reset_task); INIT_WORK(&adapter->watchdog_task, igb_watchdog_task); diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/igb_param.c b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/igb_param.c index f79ce7c1..c922ca2f 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/igb_param.c +++ b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/igb_param.c @@ -17,7 +17,7 @@ 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. The full GNU General Public License is included in this distribution in - the file called "COPYING". + the file called "LICENSE.GPL". Contact Information: e1000-devel Mailing List diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/igb_procfs.c b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/igb_procfs.c deleted file mode 100644 index 66236d29..00000000 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/igb_procfs.c +++ /dev/null @@ -1,363 +0,0 @@ -/******************************************************************************* - - Intel(R) Gigabit Ethernet Linux driver - Copyright(c) 2007-2013 Intel Corporation. - - This program is free software; you can redistribute it and/or modify it - under the terms and conditions of the GNU General Public License, - version 2, as published by the Free Software Foundation. - - This program is distributed in the hope it will be useful, but WITHOUT - ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - more details. - - You should have received a copy of the GNU General Public License along with - this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. - - The full GNU General Public License is included in this distribution in - the file called "COPYING". - - Contact Information: - e1000-devel Mailing List - Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 - -*******************************************************************************/ - -#include "igb.h" -#include "e1000_82575.h" -#include "e1000_hw.h" - -#ifdef IGB_PROCFS -#ifndef IGB_HWMON - -#include -#include -#include -#include -#include - -static struct proc_dir_entry *igb_top_dir = NULL; - - -bool igb_thermal_present(struct igb_adapter *adapter) -{ - s32 status; - struct e1000_hw *hw; - - if (adapter == NULL) - return false; - hw = &adapter->hw; - - /* - * Only set I2C bit-bang mode if an external thermal sensor is - * supported on this device. - */ - if (adapter->ets) { - status = e1000_set_i2c_bb(hw); - if (status != E1000_SUCCESS) - return false; - } - - status = hw->mac.ops.init_thermal_sensor_thresh(hw); - if (status != E1000_SUCCESS) - return false; - - return true; -} - - -static int igb_macburn(char *page, char **start, off_t off, int count, - int *eof, void *data) -{ - struct e1000_hw *hw; - struct igb_adapter *adapter = (struct igb_adapter *)data; - if (adapter == NULL) - return snprintf(page, count, "error: no adapter\n"); - - hw = &adapter->hw; - if (hw == NULL) - return snprintf(page, count, "error: no hw data\n"); - - return snprintf(page, count, "0x%02X%02X%02X%02X%02X%02X\n", - (unsigned int)hw->mac.perm_addr[0], - (unsigned int)hw->mac.perm_addr[1], - (unsigned int)hw->mac.perm_addr[2], - (unsigned int)hw->mac.perm_addr[3], - (unsigned int)hw->mac.perm_addr[4], - (unsigned int)hw->mac.perm_addr[5]); -} - -static int igb_macadmn(char *page, char **start, off_t off, - int count, int *eof, void *data) -{ - struct e1000_hw *hw; - struct igb_adapter *adapter = (struct igb_adapter *)data; - if (adapter == NULL) - return snprintf(page, count, "error: no adapter\n"); - - hw = &adapter->hw; - if (hw == NULL) - return snprintf(page, count, "error: no hw data\n"); - - return snprintf(page, count, "0x%02X%02X%02X%02X%02X%02X\n", - (unsigned int)hw->mac.addr[0], - (unsigned int)hw->mac.addr[1], - (unsigned int)hw->mac.addr[2], - (unsigned int)hw->mac.addr[3], - (unsigned int)hw->mac.addr[4], - (unsigned int)hw->mac.addr[5]); -} - -static int igb_numeports(char *page, char **start, off_t off, int count, - int *eof, void *data) -{ - struct e1000_hw *hw; - int ports; - struct igb_adapter *adapter = (struct igb_adapter *)data; - if (adapter == NULL) - return snprintf(page, count, "error: no adapter\n"); - - hw = &adapter->hw; - if (hw == NULL) - return snprintf(page, count, "error: no hw data\n"); - - ports = 4; - - return snprintf(page, count, "%d\n", ports); -} - -static int igb_porttype(char *page, char **start, off_t off, int count, - int *eof, void *data) -{ - struct igb_adapter *adapter = (struct igb_adapter *)data; - if (adapter == NULL) - return snprintf(page, count, "error: no adapter\n"); - - return snprintf(page, count, "%d\n", - test_bit(__IGB_DOWN, &adapter->state)); -} - -static int igb_therm_location(char *page, char **start, off_t off, - int count, int *eof, void *data) -{ - struct igb_therm_proc_data *therm_data = - (struct igb_therm_proc_data *)data; - - if (therm_data == NULL) - return snprintf(page, count, "error: no therm_data\n"); - - return snprintf(page, count, "%d\n", therm_data->sensor_data->location); -} - -static int igb_therm_maxopthresh(char *page, char **start, off_t off, - int count, int *eof, void *data) -{ - struct igb_therm_proc_data *therm_data = - (struct igb_therm_proc_data *)data; - - if (therm_data == NULL) - return snprintf(page, count, "error: no therm_data\n"); - - return snprintf(page, count, "%d\n", - therm_data->sensor_data->max_op_thresh); -} - -static int igb_therm_cautionthresh(char *page, char **start, off_t off, - int count, int *eof, void *data) -{ - struct igb_therm_proc_data *therm_data = - (struct igb_therm_proc_data *)data; - - if (therm_data == NULL) - return snprintf(page, count, "error: no therm_data\n"); - - return snprintf(page, count, "%d\n", - therm_data->sensor_data->caution_thresh); -} - -static int igb_therm_temp(char *page, char **start, off_t off, - int count, int *eof, void *data) -{ - s32 status; - struct igb_therm_proc_data *therm_data = - (struct igb_therm_proc_data *)data; - - if (therm_data == NULL) - return snprintf(page, count, "error: no therm_data\n"); - - status = e1000_get_thermal_sensor_data(therm_data->hw); - if (status != E1000_SUCCESS) - snprintf(page, count, "error: status %d returned\n", status); - - return snprintf(page, count, "%d\n", therm_data->sensor_data->temp); -} - -struct igb_proc_type{ - char name[32]; - int (*read)(char*, char**, off_t, int, int*, void*); -}; - -struct igb_proc_type igb_proc_entries[] = { - {"numeports", &igb_numeports}, - {"porttype", &igb_porttype}, - {"macburn", &igb_macburn}, - {"macadmn", &igb_macadmn}, - {"", NULL} -}; - -struct igb_proc_type igb_internal_entries[] = { - {"location", &igb_therm_location}, - {"temp", &igb_therm_temp}, - {"cautionthresh", &igb_therm_cautionthresh}, - {"maxopthresh", &igb_therm_maxopthresh}, - {"", NULL} -}; - -void igb_del_proc_entries(struct igb_adapter *adapter) -{ - int index, i; - char buf[16]; /* much larger than the sensor number will ever be */ - - if (igb_top_dir == NULL) - return; - - for (i = 0; i < E1000_MAX_SENSORS; i++) { - if (adapter->therm_dir[i] == NULL) - continue; - - for (index = 0; ; index++) { - if (igb_internal_entries[index].read == NULL) - break; - - remove_proc_entry(igb_internal_entries[index].name, - adapter->therm_dir[i]); - } - snprintf(buf, sizeof(buf), "sensor_%d", i); - remove_proc_entry(buf, adapter->info_dir); - } - - if (adapter->info_dir != NULL) { - for (index = 0; ; index++) { - if (igb_proc_entries[index].read == NULL) - break; - remove_proc_entry(igb_proc_entries[index].name, - adapter->info_dir); - } - remove_proc_entry("info", adapter->eth_dir); - } - - if (adapter->eth_dir != NULL) - remove_proc_entry(pci_name(adapter->pdev), igb_top_dir); -} - -/* called from igb_main.c */ -void igb_procfs_exit(struct igb_adapter *adapter) -{ - igb_del_proc_entries(adapter); -} - -int igb_procfs_topdir_init(void) -{ - igb_top_dir = proc_mkdir("driver/igb", NULL); - if (igb_top_dir == NULL) - return -ENOMEM; - - return 0; -} - -void igb_procfs_topdir_exit(void) -{ - remove_proc_entry("driver/igb", NULL); -} - -/* called from igb_main.c */ -int igb_procfs_init(struct igb_adapter *adapter) -{ - int rc = 0; - int i; - int index; - char buf[16]; /* much larger than the sensor number will ever be */ - - adapter->eth_dir = NULL; - adapter->info_dir = NULL; - for (i = 0; i < E1000_MAX_SENSORS; i++) - adapter->therm_dir[i] = NULL; - - if ( igb_top_dir == NULL ) { - rc = -ENOMEM; - goto fail; - } - - adapter->eth_dir = proc_mkdir(pci_name(adapter->pdev), igb_top_dir); - if (adapter->eth_dir == NULL) { - rc = -ENOMEM; - goto fail; - } - - adapter->info_dir = proc_mkdir("info", adapter->eth_dir); - if (adapter->info_dir == NULL) { - rc = -ENOMEM; - goto fail; - } - for (index = 0; ; index++) { - if (igb_proc_entries[index].read == NULL) { - break; - } - if (!(create_proc_read_entry(igb_proc_entries[index].name, - 0444, - adapter->info_dir, - igb_proc_entries[index].read, - adapter))) { - - rc = -ENOMEM; - goto fail; - } - } - if (igb_thermal_present(adapter) == false) - goto exit; - - for (i = 0; i < E1000_MAX_SENSORS; i++) { - - if (adapter->hw.mac.thermal_sensor_data.sensor[i].location== 0) - continue; - - snprintf(buf, sizeof(buf), "sensor_%d", i); - adapter->therm_dir[i] = proc_mkdir(buf, adapter->info_dir); - if (adapter->therm_dir[i] == NULL) { - rc = -ENOMEM; - goto fail; - } - for (index = 0; ; index++) { - if (igb_internal_entries[index].read == NULL) - break; - /* - * therm_data struct contains pointer the read func - * will be needing - */ - adapter->therm_data[i].hw = &adapter->hw; - adapter->therm_data[i].sensor_data = - &adapter->hw.mac.thermal_sensor_data.sensor[i]; - - if (!(create_proc_read_entry( - igb_internal_entries[index].name, - 0444, - adapter->therm_dir[i], - igb_internal_entries[index].read, - &adapter->therm_data[i]))) { - rc = -ENOMEM; - goto fail; - } - } - } - goto exit; - -fail: - igb_del_proc_entries(adapter); -exit: - return rc; -} - -#endif /* !IGB_HWMON */ -#endif /* IGB_PROCFS */ diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/igb_ptp.c b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/igb_ptp.c deleted file mode 100644 index 454b70ce..00000000 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/igb_ptp.c +++ /dev/null @@ -1,944 +0,0 @@ -/******************************************************************************* - - Intel(R) Gigabit Ethernet Linux driver - Copyright(c) 2007-2013 Intel Corporation. - - This program is free software; you can redistribute it and/or modify it - under the terms and conditions of the GNU General Public License, - version 2, as published by the Free Software Foundation. - - This program is distributed in the hope it will be useful, but WITHOUT - ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - more details. - - You should have received a copy of the GNU General Public License along with - this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. - - The full GNU General Public License is included in this distribution in - the file called "COPYING". - - Contact Information: - e1000-devel Mailing List - Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 - -*******************************************************************************/ - -/****************************************************************************** - Copyright(c) 2011 Richard Cochran for some of the - 82576 and 82580 code -******************************************************************************/ - -#include "igb.h" - -#include -#include -#include -#include - -#define INCVALUE_MASK 0x7fffffff -#define ISGN 0x80000000 - -/* - * The 82580 timesync updates the system timer every 8ns by 8ns, - * and this update value cannot be reprogrammed. - * - * Neither the 82576 nor the 82580 offer registers wide enough to hold - * nanoseconds time values for very long. For the 82580, SYSTIM always - * counts nanoseconds, but the upper 24 bits are not available. The - * frequency is adjusted by changing the 32 bit fractional nanoseconds - * register, TIMINCA. - * - * For the 82576, the SYSTIM register time unit is affect by the - * choice of the 24 bit TININCA:IV (incvalue) field. Five bits of this - * field are needed to provide the nominal 16 nanosecond period, - * leaving 19 bits for fractional nanoseconds. - * - * We scale the NIC clock cycle by a large factor so that relatively - * small clock corrections can be added or subtracted at each clock - * tick. The drawbacks of a large factor are a) that the clock - * register overflows more quickly (not such a big deal) and b) that - * the increment per tick has to fit into 24 bits. As a result we - * need to use a shift of 19 so we can fit a value of 16 into the - * TIMINCA register. - * - * - * SYSTIMH SYSTIML - * +--------------+ +---+---+------+ - * 82576 | 32 | | 8 | 5 | 19 | - * +--------------+ +---+---+------+ - * \________ 45 bits _______/ fract - * - * +----------+---+ +--------------+ - * 82580 | 24 | 8 | | 32 | - * +----------+---+ +--------------+ - * reserved \______ 40 bits _____/ - * - * - * The 45 bit 82576 SYSTIM overflows every - * 2^45 * 10^-9 / 3600 = 9.77 hours. - * - * The 40 bit 82580 SYSTIM overflows every - * 2^40 * 10^-9 / 60 = 18.3 minutes. - */ - -#define IGB_SYSTIM_OVERFLOW_PERIOD (HZ * 60 * 9) -#define IGB_PTP_TX_TIMEOUT (HZ * 15) -#define INCPERIOD_82576 (1 << E1000_TIMINCA_16NS_SHIFT) -#define INCVALUE_82576_MASK ((1 << E1000_TIMINCA_16NS_SHIFT) - 1) -#define INCVALUE_82576 (16 << IGB_82576_TSYNC_SHIFT) -#define IGB_NBITS_82580 40 - -/* - * SYSTIM read access for the 82576 - */ - -static cycle_t igb_ptp_read_82576(const struct cyclecounter *cc) -{ - struct igb_adapter *igb = container_of(cc, struct igb_adapter, cc); - struct e1000_hw *hw = &igb->hw; - u64 val; - u32 lo, hi; - - lo = E1000_READ_REG(hw, E1000_SYSTIML); - hi = E1000_READ_REG(hw, E1000_SYSTIMH); - - val = ((u64) hi) << 32; - val |= lo; - - return val; -} - -/* - * SYSTIM read access for the 82580 - */ - -static cycle_t igb_ptp_read_82580(const struct cyclecounter *cc) -{ - struct igb_adapter *igb = container_of(cc, struct igb_adapter, cc); - struct e1000_hw *hw = &igb->hw; - u64 val; - u32 lo, hi; - - /* The timestamp latches on lowest register read. For the 82580 - * the lowest register is SYSTIMR instead of SYSTIML. However we only - * need to provide nanosecond resolution, so we just ignore it. - */ - E1000_READ_REG(hw, E1000_SYSTIMR); - lo = E1000_READ_REG(hw, E1000_SYSTIML); - hi = E1000_READ_REG(hw, E1000_SYSTIMH); - - val = ((u64) hi) << 32; - val |= lo; - - return val; -} - -/* - * SYSTIM read access for I210/I211 - */ - -static void igb_ptp_read_i210(struct igb_adapter *adapter, struct timespec *ts) -{ - struct e1000_hw *hw = &adapter->hw; - u32 sec, nsec; - - /* The timestamp latches on lowest register read. For I210/I211, the - * lowest register is SYSTIMR. Since we only need to provide nanosecond - * resolution, we can ignore it. - */ - E1000_READ_REG(hw, E1000_SYSTIMR); - nsec = E1000_READ_REG(hw, E1000_SYSTIML); - sec = E1000_READ_REG(hw, E1000_SYSTIMH); - - ts->tv_sec = sec; - ts->tv_nsec = nsec; -} - -static void igb_ptp_write_i210(struct igb_adapter *adapter, - const struct timespec *ts) -{ - struct e1000_hw *hw = &adapter->hw; - - /* - * Writing the SYSTIMR register is not necessary as it only provides - * sub-nanosecond resolution. - */ - E1000_WRITE_REG(hw, E1000_SYSTIML, ts->tv_nsec); - E1000_WRITE_REG(hw, E1000_SYSTIMH, ts->tv_sec); -} - -/** - * igb_ptp_systim_to_hwtstamp - convert system time value to hw timestamp - * @adapter: board private structure - * @hwtstamps: timestamp structure to update - * @systim: unsigned 64bit system time value. - * - * We need to convert the system time value stored in the RX/TXSTMP registers - * into a hwtstamp which can be used by the upper level timestamping functions. - * - * The 'tmreg_lock' spinlock is used to protect the consistency of the - * system time value. This is needed because reading the 64 bit time - * value involves reading two (or three) 32 bit registers. The first - * read latches the value. Ditto for writing. - * - * In addition, here have extended the system time with an overflow - * counter in software. - **/ -static void igb_ptp_systim_to_hwtstamp(struct igb_adapter *adapter, - struct skb_shared_hwtstamps *hwtstamps, - u64 systim) -{ - unsigned long flags; - u64 ns; - - switch (adapter->hw.mac.type) { - case e1000_82576: - case e1000_82580: - case e1000_i350: - case e1000_i354: - spin_lock_irqsave(&adapter->tmreg_lock, flags); - - ns = timecounter_cyc2time(&adapter->tc, systim); - - spin_unlock_irqrestore(&adapter->tmreg_lock, flags); - - memset(hwtstamps, 0, sizeof(*hwtstamps)); - hwtstamps->hwtstamp = ns_to_ktime(ns); - break; - case e1000_i210: - case e1000_i211: - memset(hwtstamps, 0, sizeof(*hwtstamps)); - /* Upper 32 bits contain s, lower 32 bits contain ns. */ - hwtstamps->hwtstamp = ktime_set(systim >> 32, - systim & 0xFFFFFFFF); - break; - default: - break; - } -} - -/* - * PTP clock operations - */ - -static int igb_ptp_adjfreq_82576(struct ptp_clock_info *ptp, s32 ppb) -{ - struct igb_adapter *igb = container_of(ptp, struct igb_adapter, - ptp_caps); - struct e1000_hw *hw = &igb->hw; - int neg_adj = 0; - u64 rate; - u32 incvalue; - - if (ppb < 0) { - neg_adj = 1; - ppb = -ppb; - } - rate = ppb; - rate <<= 14; - rate = div_u64(rate, 1953125); - - incvalue = 16 << IGB_82576_TSYNC_SHIFT; - - if (neg_adj) - incvalue -= rate; - else - incvalue += rate; - - E1000_WRITE_REG(hw, E1000_TIMINCA, INCPERIOD_82576 | (incvalue & INCVALUE_82576_MASK)); - - return 0; -} - -static int igb_ptp_adjfreq_82580(struct ptp_clock_info *ptp, s32 ppb) -{ - struct igb_adapter *igb = container_of(ptp, struct igb_adapter, - ptp_caps); - struct e1000_hw *hw = &igb->hw; - int neg_adj = 0; - u64 rate; - u32 inca; - - if (ppb < 0) { - neg_adj = 1; - ppb = -ppb; - } - rate = ppb; - rate <<= 26; - rate = div_u64(rate, 1953125); - - /* At 2.5G speeds, the TIMINCA register on I354 updates the clock 2.5x - * as quickly. Account for this by dividing the adjustment by 2.5. - */ - if (hw->mac.type == e1000_i354) { - u32 status = E1000_READ_REG(hw, E1000_STATUS); - - if ((status & E1000_STATUS_2P5_SKU) && - !(status & E1000_STATUS_2P5_SKU_OVER)) { - rate <<= 1; - rate = div_u64(rate, 5); - } - } - - inca = rate & INCVALUE_MASK; - if (neg_adj) - inca |= ISGN; - - E1000_WRITE_REG(hw, E1000_TIMINCA, inca); - - return 0; -} - -static int igb_ptp_adjtime_82576(struct ptp_clock_info *ptp, s64 delta) -{ - struct igb_adapter *igb = container_of(ptp, struct igb_adapter, - ptp_caps); - unsigned long flags; - s64 now; - - spin_lock_irqsave(&igb->tmreg_lock, flags); - - now = timecounter_read(&igb->tc); - now += delta; - timecounter_init(&igb->tc, &igb->cc, now); - - spin_unlock_irqrestore(&igb->tmreg_lock, flags); - - return 0; -} - -static int igb_ptp_adjtime_i210(struct ptp_clock_info *ptp, s64 delta) -{ - struct igb_adapter *igb = container_of(ptp, struct igb_adapter, - ptp_caps); - unsigned long flags; - struct timespec now, then = ns_to_timespec(delta); - - spin_lock_irqsave(&igb->tmreg_lock, flags); - - igb_ptp_read_i210(igb, &now); - now = timespec_add(now, then); - igb_ptp_write_i210(igb, (const struct timespec *)&now); - - spin_unlock_irqrestore(&igb->tmreg_lock, flags); - - return 0; -} - -static int igb_ptp_gettime_82576(struct ptp_clock_info *ptp, - struct timespec *ts) -{ - struct igb_adapter *igb = container_of(ptp, struct igb_adapter, - ptp_caps); - unsigned long flags; - u64 ns; - u32 remainder; - - spin_lock_irqsave(&igb->tmreg_lock, flags); - - ns = timecounter_read(&igb->tc); - - spin_unlock_irqrestore(&igb->tmreg_lock, flags); - - ts->tv_sec = div_u64_rem(ns, 1000000000, &remainder); - ts->tv_nsec = remainder; - - return 0; -} - -static int igb_ptp_gettime_i210(struct ptp_clock_info *ptp, - struct timespec *ts) -{ - struct igb_adapter *igb = container_of(ptp, struct igb_adapter, - ptp_caps); - unsigned long flags; - - spin_lock_irqsave(&igb->tmreg_lock, flags); - - igb_ptp_read_i210(igb, ts); - - spin_unlock_irqrestore(&igb->tmreg_lock, flags); - - return 0; -} - -static int igb_ptp_settime_82576(struct ptp_clock_info *ptp, - const struct timespec *ts) -{ - struct igb_adapter *igb = container_of(ptp, struct igb_adapter, - ptp_caps); - unsigned long flags; - u64 ns; - - ns = ts->tv_sec * 1000000000ULL; - ns += ts->tv_nsec; - - spin_lock_irqsave(&igb->tmreg_lock, flags); - - timecounter_init(&igb->tc, &igb->cc, ns); - - spin_unlock_irqrestore(&igb->tmreg_lock, flags); - - return 0; -} - -static int igb_ptp_settime_i210(struct ptp_clock_info *ptp, - const struct timespec *ts) -{ - struct igb_adapter *igb = container_of(ptp, struct igb_adapter, - ptp_caps); - unsigned long flags; - - spin_lock_irqsave(&igb->tmreg_lock, flags); - - igb_ptp_write_i210(igb, ts); - - spin_unlock_irqrestore(&igb->tmreg_lock, flags); - - return 0; -} - -static int igb_ptp_enable(struct ptp_clock_info *ptp, - struct ptp_clock_request *rq, int on) -{ - return -EOPNOTSUPP; -} - -/** - * igb_ptp_tx_work - * @work: pointer to work struct - * - * This work function polls the TSYNCTXCTL valid bit to determine when a - * timestamp has been taken for the current stored skb. - */ -void igb_ptp_tx_work(struct work_struct *work) -{ - struct igb_adapter *adapter = container_of(work, struct igb_adapter, - ptp_tx_work); - struct e1000_hw *hw = &adapter->hw; - u32 tsynctxctl; - - if (!adapter->ptp_tx_skb) - return; - - if (time_is_before_jiffies(adapter->ptp_tx_start + - IGB_PTP_TX_TIMEOUT)) { - dev_kfree_skb_any(adapter->ptp_tx_skb); - adapter->ptp_tx_skb = NULL; - adapter->tx_hwtstamp_timeouts++; - dev_warn(&adapter->pdev->dev, "clearing Tx timestamp hang"); - return; - } - - tsynctxctl = E1000_READ_REG(hw, E1000_TSYNCTXCTL); - if (tsynctxctl & E1000_TSYNCTXCTL_VALID) - igb_ptp_tx_hwtstamp(adapter); - else - /* reschedule to check later */ - schedule_work(&adapter->ptp_tx_work); -} - -static void igb_ptp_overflow_check(struct work_struct *work) -{ - struct igb_adapter *igb = - container_of(work, struct igb_adapter, ptp_overflow_work.work); - struct timespec ts; - - igb->ptp_caps.gettime(&igb->ptp_caps, &ts); - - pr_debug("igb overflow check at %ld.%09lu\n", ts.tv_sec, ts.tv_nsec); - - schedule_delayed_work(&igb->ptp_overflow_work, - IGB_SYSTIM_OVERFLOW_PERIOD); -} - -/** - * igb_ptp_rx_hang - detect error case when Rx timestamp registers latched - * @adapter: private network adapter structure - * - * This watchdog task is scheduled to detect error case where hardware has - * dropped an Rx packet that was timestamped when the ring is full. The - * particular error is rare but leaves the device in a state unable to timestamp - * any future packets. - */ -void igb_ptp_rx_hang(struct igb_adapter *adapter) -{ - struct e1000_hw *hw = &adapter->hw; - struct igb_ring *rx_ring; - u32 tsyncrxctl = E1000_READ_REG(hw, E1000_TSYNCRXCTL); - unsigned long rx_event; - int n; - - if (hw->mac.type != e1000_82576) - return; - - /* If we don't have a valid timestamp in the registers, just update the - * timeout counter and exit - */ - if (!(tsyncrxctl & E1000_TSYNCRXCTL_VALID)) { - adapter->last_rx_ptp_check = jiffies; - return; - } - - /* Determine the most recent watchdog or rx_timestamp event */ - rx_event = adapter->last_rx_ptp_check; - for (n = 0; n < adapter->num_rx_queues; n++) { - rx_ring = adapter->rx_ring[n]; - if (time_after(rx_ring->last_rx_timestamp, rx_event)) - rx_event = rx_ring->last_rx_timestamp; - } - - /* Only need to read the high RXSTMP register to clear the lock */ - if (time_is_before_jiffies(rx_event + 5 * HZ)) { - E1000_READ_REG(hw, E1000_RXSTMPH); - adapter->last_rx_ptp_check = jiffies; - adapter->rx_hwtstamp_cleared++; - dev_warn(&adapter->pdev->dev, "clearing Rx timestamp hang"); - } -} - -/** - * igb_ptp_tx_hwtstamp - utility function which checks for TX time stamp - * @adapter: Board private structure. - * - * If we were asked to do hardware stamping and such a time stamp is - * available, then it must have been for this skb here because we only - * allow only one such packet into the queue. - */ -void igb_ptp_tx_hwtstamp(struct igb_adapter *adapter) -{ - struct e1000_hw *hw = &adapter->hw; - struct skb_shared_hwtstamps shhwtstamps; - u64 regval; - - regval = E1000_READ_REG(hw, E1000_TXSTMPL); - regval |= (u64)E1000_READ_REG(hw, E1000_TXSTMPH) << 32; - - igb_ptp_systim_to_hwtstamp(adapter, &shhwtstamps, regval); - skb_tstamp_tx(adapter->ptp_tx_skb, &shhwtstamps); - dev_kfree_skb_any(adapter->ptp_tx_skb); - adapter->ptp_tx_skb = NULL; -} - -/** - * igb_ptp_rx_pktstamp - retrieve Rx per packet timestamp - * @q_vector: Pointer to interrupt specific structure - * @va: Pointer to address containing Rx buffer - * @skb: Buffer containing timestamp and packet - * - * This function is meant to retrieve a timestamp from the first buffer of an - * incoming frame. The value is stored in little endian format starting on - * byte 8. - */ -void igb_ptp_rx_pktstamp(struct igb_q_vector *q_vector, - unsigned char *va, - struct sk_buff *skb) -{ - __le64 *regval = (__le64 *)va; - - /* - * The timestamp is recorded in little endian format. - * DWORD: 0 1 2 3 - * Field: Reserved Reserved SYSTIML SYSTIMH - */ - igb_ptp_systim_to_hwtstamp(q_vector->adapter, skb_hwtstamps(skb), - le64_to_cpu(regval[1])); -} - -/** - * igb_ptp_rx_rgtstamp - retrieve Rx timestamp stored in register - * @q_vector: Pointer to interrupt specific structure - * @skb: Buffer containing timestamp and packet - * - * This function is meant to retrieve a timestamp from the internal registers - * of the adapter and store it in the skb. - */ -void igb_ptp_rx_rgtstamp(struct igb_q_vector *q_vector, - struct sk_buff *skb) -{ - struct igb_adapter *adapter = q_vector->adapter; - struct e1000_hw *hw = &adapter->hw; - u64 regval; - - /* - * If this bit is set, then the RX registers contain the time stamp. No - * other packet will be time stamped until we read these registers, so - * read the registers to make them available again. Because only one - * packet can be time stamped at a time, we know that the register - * values must belong to this one here and therefore we don't need to - * compare any of the additional attributes stored for it. - * - * If nothing went wrong, then it should have a shared tx_flags that we - * can turn into a skb_shared_hwtstamps. - */ - if (!(E1000_READ_REG(hw, E1000_TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID)) - return; - - regval = E1000_READ_REG(hw, E1000_RXSTMPL); - regval |= (u64)E1000_READ_REG(hw, E1000_RXSTMPH) << 32; - - igb_ptp_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), regval); -} - -/** - * igb_ptp_hwtstamp_ioctl - control hardware time stamping - * @netdev: - * @ifreq: - * @cmd: - * - * Outgoing time stamping can be enabled and disabled. Play nice and - * disable it when requested, although it shouldn't case any overhead - * when no packet needs it. At most one packet in the queue may be - * marked for time stamping, otherwise it would be impossible to tell - * for sure to which packet the hardware time stamp belongs. - * - * Incoming time stamping has to be configured via the hardware - * filters. Not all combinations are supported, in particular event - * type has to be specified. Matching the kind of event packet is - * not supported, with the exception of "all V2 events regardless of - * level 2 or 4". - * - **/ -int igb_ptp_hwtstamp_ioctl(struct net_device *netdev, - struct ifreq *ifr, int cmd) -{ - struct igb_adapter *adapter = netdev_priv(netdev); - struct e1000_hw *hw = &adapter->hw; - struct hwtstamp_config config; - u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED; - u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED; - u32 tsync_rx_cfg = 0; - bool is_l4 = false; - bool is_l2 = false; - u32 regval; - - if (copy_from_user(&config, ifr->ifr_data, sizeof(config))) - return -EFAULT; - - /* reserved for future extensions */ - if (config.flags) - return -EINVAL; - - switch (config.tx_type) { - case HWTSTAMP_TX_OFF: - tsync_tx_ctl = 0; - case HWTSTAMP_TX_ON: - break; - default: - return -ERANGE; - } - - switch (config.rx_filter) { - case HWTSTAMP_FILTER_NONE: - tsync_rx_ctl = 0; - break; - case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: - tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1; - tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE; - is_l4 = true; - break; - case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: - tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1; - tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE; - is_l4 = true; - break; - case HWTSTAMP_FILTER_PTP_V2_EVENT: - case HWTSTAMP_FILTER_PTP_V2_L2_EVENT: - case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: - case HWTSTAMP_FILTER_PTP_V2_SYNC: - case HWTSTAMP_FILTER_PTP_V2_L2_SYNC: - case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: - case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: - case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ: - case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: - tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2; - config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT; - is_l2 = true; - is_l4 = true; - break; - case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: - case HWTSTAMP_FILTER_ALL: - /* - * 82576 cannot timestamp all packets, which it needs to do to - * support both V1 Sync and Delay_Req messages - */ - if (hw->mac.type != e1000_82576) { - tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL; - config.rx_filter = HWTSTAMP_FILTER_ALL; - break; - } - /* fall through */ - default: - config.rx_filter = HWTSTAMP_FILTER_NONE; - return -ERANGE; - } - - if (hw->mac.type == e1000_82575) { - if (tsync_rx_ctl | tsync_tx_ctl) - return -EINVAL; - return 0; - } - - /* - * Per-packet timestamping only works if all packets are - * timestamped, so enable timestamping in all packets as - * long as one rx filter was configured. - */ - if ((hw->mac.type >= e1000_82580) && tsync_rx_ctl) { - tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED; - tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL; - config.rx_filter = HWTSTAMP_FILTER_ALL; - is_l2 = true; - is_l4 = true; - - if ((hw->mac.type == e1000_i210) || - (hw->mac.type == e1000_i211)) { - regval = E1000_READ_REG(hw, E1000_RXPBS); - regval |= E1000_RXPBS_CFG_TS_EN; - E1000_WRITE_REG(hw, E1000_RXPBS, regval); - } - } - - /* enable/disable TX */ - regval = E1000_READ_REG(hw, E1000_TSYNCTXCTL); - regval &= ~E1000_TSYNCTXCTL_ENABLED; - regval |= tsync_tx_ctl; - E1000_WRITE_REG(hw, E1000_TSYNCTXCTL, regval); - - /* enable/disable RX */ - regval = E1000_READ_REG(hw, E1000_TSYNCRXCTL); - regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK); - regval |= tsync_rx_ctl; - E1000_WRITE_REG(hw, E1000_TSYNCRXCTL, regval); - - /* define which PTP packets are time stamped */ - E1000_WRITE_REG(hw, E1000_TSYNCRXCFG, tsync_rx_cfg); - - /* define ethertype filter for timestamped packets */ - if (is_l2) - E1000_WRITE_REG(hw, E1000_ETQF(3), - (E1000_ETQF_FILTER_ENABLE | /* enable filter */ - E1000_ETQF_1588 | /* enable timestamping */ - ETH_P_1588)); /* 1588 eth protocol type */ - else - E1000_WRITE_REG(hw, E1000_ETQF(3), 0); - - /* L4 Queue Filter[3]: filter by destination port and protocol */ - if (is_l4) { - u32 ftqf = (IPPROTO_UDP /* UDP */ - | E1000_FTQF_VF_BP /* VF not compared */ - | E1000_FTQF_1588_TIME_STAMP /* Enable Timestamping */ - | E1000_FTQF_MASK); /* mask all inputs */ - ftqf &= ~E1000_FTQF_MASK_PROTO_BP; /* enable protocol check */ - - E1000_WRITE_REG(hw, E1000_IMIR(3), htons(PTP_EV_PORT)); - E1000_WRITE_REG(hw, E1000_IMIREXT(3), - (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP)); - if (hw->mac.type == e1000_82576) { - /* enable source port check */ - E1000_WRITE_REG(hw, E1000_SPQF(3), htons(PTP_EV_PORT)); - ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP; - } - E1000_WRITE_REG(hw, E1000_FTQF(3), ftqf); - } else { - E1000_WRITE_REG(hw, E1000_FTQF(3), E1000_FTQF_MASK); - } - E1000_WRITE_FLUSH(hw); - - /* clear TX/RX time stamp registers, just to be sure */ - regval = E1000_READ_REG(hw, E1000_TXSTMPL); - regval = E1000_READ_REG(hw, E1000_TXSTMPH); - regval = E1000_READ_REG(hw, E1000_RXSTMPL); - regval = E1000_READ_REG(hw, E1000_RXSTMPH); - - return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ? - -EFAULT : 0; -} - -void igb_ptp_init(struct igb_adapter *adapter) -{ - struct e1000_hw *hw = &adapter->hw; - struct net_device *netdev = adapter->netdev; - - switch (hw->mac.type) { - case e1000_82576: - snprintf(adapter->ptp_caps.name, 16, "%pm", netdev->dev_addr); - adapter->ptp_caps.owner = THIS_MODULE; - adapter->ptp_caps.max_adj = 999999881; - adapter->ptp_caps.n_ext_ts = 0; - adapter->ptp_caps.pps = 0; - adapter->ptp_caps.adjfreq = igb_ptp_adjfreq_82576; - adapter->ptp_caps.adjtime = igb_ptp_adjtime_82576; - adapter->ptp_caps.gettime = igb_ptp_gettime_82576; - adapter->ptp_caps.settime = igb_ptp_settime_82576; - adapter->ptp_caps.enable = igb_ptp_enable; - adapter->cc.read = igb_ptp_read_82576; - adapter->cc.mask = CLOCKSOURCE_MASK(64); - adapter->cc.mult = 1; - adapter->cc.shift = IGB_82576_TSYNC_SHIFT; - /* Dial the nominal frequency. */ - E1000_WRITE_REG(hw, E1000_TIMINCA, INCPERIOD_82576 | - INCVALUE_82576); - break; - case e1000_82580: - case e1000_i350: - case e1000_i354: - snprintf(adapter->ptp_caps.name, 16, "%pm", netdev->dev_addr); - adapter->ptp_caps.owner = THIS_MODULE; - adapter->ptp_caps.max_adj = 62499999; - adapter->ptp_caps.n_ext_ts = 0; - adapter->ptp_caps.pps = 0; - adapter->ptp_caps.adjfreq = igb_ptp_adjfreq_82580; - adapter->ptp_caps.adjtime = igb_ptp_adjtime_82576; - adapter->ptp_caps.gettime = igb_ptp_gettime_82576; - adapter->ptp_caps.settime = igb_ptp_settime_82576; - adapter->ptp_caps.enable = igb_ptp_enable; - adapter->cc.read = igb_ptp_read_82580; - adapter->cc.mask = CLOCKSOURCE_MASK(IGB_NBITS_82580); - adapter->cc.mult = 1; - adapter->cc.shift = 0; - /* Enable the timer functions by clearing bit 31. */ - E1000_WRITE_REG(hw, E1000_TSAUXC, 0x0); - break; - case e1000_i210: - case e1000_i211: - snprintf(adapter->ptp_caps.name, 16, "%pm", netdev->dev_addr); - adapter->ptp_caps.owner = THIS_MODULE; - adapter->ptp_caps.max_adj = 62499999; - adapter->ptp_caps.n_ext_ts = 0; - adapter->ptp_caps.pps = 0; - adapter->ptp_caps.adjfreq = igb_ptp_adjfreq_82580; - adapter->ptp_caps.adjtime = igb_ptp_adjtime_i210; - adapter->ptp_caps.gettime = igb_ptp_gettime_i210; - adapter->ptp_caps.settime = igb_ptp_settime_i210; - adapter->ptp_caps.enable = igb_ptp_enable; - /* Enable the timer functions by clearing bit 31. */ - E1000_WRITE_REG(hw, E1000_TSAUXC, 0x0); - break; - default: - adapter->ptp_clock = NULL; - return; - } - - E1000_WRITE_FLUSH(hw); - - spin_lock_init(&adapter->tmreg_lock); - INIT_WORK(&adapter->ptp_tx_work, igb_ptp_tx_work); - - /* Initialize the clock and overflow work for devices that need it. */ - if ((hw->mac.type == e1000_i210) || (hw->mac.type == e1000_i211)) { - struct timespec ts = ktime_to_timespec(ktime_get_real()); - - igb_ptp_settime_i210(&adapter->ptp_caps, &ts); - } else { - timecounter_init(&adapter->tc, &adapter->cc, - ktime_to_ns(ktime_get_real())); - - INIT_DELAYED_WORK(&adapter->ptp_overflow_work, - igb_ptp_overflow_check); - - schedule_delayed_work(&adapter->ptp_overflow_work, - IGB_SYSTIM_OVERFLOW_PERIOD); - } - - /* Initialize the time sync interrupts for devices that support it. */ - if (hw->mac.type >= e1000_82580) { - E1000_WRITE_REG(hw, E1000_TSIM, E1000_TSIM_TXTS); - E1000_WRITE_REG(hw, E1000_IMS, E1000_IMS_TS); - } - - adapter->ptp_clock = ptp_clock_register(&adapter->ptp_caps, - &adapter->pdev->dev); - if (IS_ERR(adapter->ptp_clock)) { - adapter->ptp_clock = NULL; - dev_err(&adapter->pdev->dev, "ptp_clock_register failed\n"); - } else { - dev_info(&adapter->pdev->dev, "added PHC on %s\n", - adapter->netdev->name); - adapter->flags |= IGB_FLAG_PTP; - } -} - -/** - * igb_ptp_stop - Disable PTP device and stop the overflow check. - * @adapter: Board private structure. - * - * This function stops the PTP support and cancels the delayed work. - **/ -void igb_ptp_stop(struct igb_adapter *adapter) -{ - switch (adapter->hw.mac.type) { - case e1000_82576: - case e1000_82580: - case e1000_i350: - case e1000_i354: - cancel_delayed_work_sync(&adapter->ptp_overflow_work); - break; - case e1000_i210: - case e1000_i211: - /* No delayed work to cancel. */ - break; - default: - return; - } - - cancel_work_sync(&adapter->ptp_tx_work); - if (adapter->ptp_tx_skb) { - dev_kfree_skb_any(adapter->ptp_tx_skb); - adapter->ptp_tx_skb = NULL; - } - - if (adapter->ptp_clock) { - ptp_clock_unregister(adapter->ptp_clock); - dev_info(&adapter->pdev->dev, "removed PHC on %s\n", - adapter->netdev->name); - adapter->flags &= ~IGB_FLAG_PTP; - } -} - -/** - * igb_ptp_reset - Re-enable the adapter for PTP following a reset. - * @adapter: Board private structure. - * - * This function handles the reset work required to re-enable the PTP device. - **/ -void igb_ptp_reset(struct igb_adapter *adapter) -{ - struct e1000_hw *hw = &adapter->hw; - - if (!(adapter->flags & IGB_FLAG_PTP)) - return; - - switch (adapter->hw.mac.type) { - case e1000_82576: - /* Dial the nominal frequency. */ - E1000_WRITE_REG(hw, E1000_TIMINCA, INCPERIOD_82576 | - INCVALUE_82576); - break; - case e1000_82580: - case e1000_i350: - case e1000_i354: - case e1000_i210: - case e1000_i211: - /* Enable the timer functions and interrupts. */ - E1000_WRITE_REG(hw, E1000_TSAUXC, 0x0); - E1000_WRITE_REG(hw, E1000_TSIM, E1000_TSIM_TXTS); - E1000_WRITE_REG(hw, E1000_IMS, E1000_IMS_TS); - break; - default: - /* No work to do. */ - return; - } - - /* Re-initialize the timer. */ - if ((hw->mac.type == e1000_i210) || (hw->mac.type == e1000_i211)) { - struct timespec ts = ktime_to_timespec(ktime_get_real()); - - igb_ptp_settime_i210(&adapter->ptp_caps, &ts); - } else { - timecounter_init(&adapter->tc, &adapter->cc, - ktime_to_ns(ktime_get_real())); - } -} diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/igb_regtest.h b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/igb_regtest.h index 18da64a3..9d49b45e 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/igb_regtest.h +++ b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/igb_regtest.h @@ -17,7 +17,7 @@ 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. The full GNU General Public License is included in this distribution in - the file called "COPYING". + the file called "LICENSE.GPL". Contact Information: e1000-devel Mailing List diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/igb_vmdq.c b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/igb_vmdq.c index 015c8952..205da562 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/igb_vmdq.c +++ b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/igb_vmdq.c @@ -17,7 +17,7 @@ 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. The full GNU General Public License is included in this distribution in - the file called "COPYING". + the file called "LICENSE.GPL". Contact Information: e1000-devel Mailing List diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/igb_vmdq.h b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/igb_vmdq.h index e51e7c4e..c6d4c568 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/igb_vmdq.h +++ b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/igb_vmdq.h @@ -17,7 +17,7 @@ 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. The full GNU General Public License is included in this distribution in - the file called "COPYING". + the file called "LICENSE.GPL". Contact Information: e1000-devel Mailing List diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/kcompat.c b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/kcompat.c deleted file mode 100644 index bde3a83c..00000000 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/kcompat.c +++ /dev/null @@ -1,1482 +0,0 @@ -/******************************************************************************* - - Intel(R) Gigabit Ethernet Linux driver - Copyright(c) 2007-2013 Intel Corporation. - - This program is free software; you can redistribute it and/or modify it - under the terms and conditions of the GNU General Public License, - version 2, as published by the Free Software Foundation. - - This program is distributed in the hope it will be useful, but WITHOUT - ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - more details. - - You should have received a copy of the GNU General Public License along with - this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. - - The full GNU General Public License is included in this distribution in - the file called "COPYING". - - Contact Information: - e1000-devel Mailing List - Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 - -*******************************************************************************/ - -#include "igb.h" -#include "kcompat.h" - -/*****************************************************************************/ -#if ( LINUX_VERSION_CODE < KERNEL_VERSION(2,4,8) ) -/* From lib/vsprintf.c */ -#include - -static int skip_atoi(const char **s) -{ - int i=0; - - while (isdigit(**s)) - i = i*10 + *((*s)++) - '0'; - return i; -} - -#define _kc_ZEROPAD 1 /* pad with zero */ -#define _kc_SIGN 2 /* unsigned/signed long */ -#define _kc_PLUS 4 /* show plus */ -#define _kc_SPACE 8 /* space if plus */ -#define _kc_LEFT 16 /* left justified */ -#define _kc_SPECIAL 32 /* 0x */ -#define _kc_LARGE 64 /* use 'ABCDEF' instead of 'abcdef' */ - -static char * number(char * buf, char * end, long long num, int base, int size, int precision, int type) -{ - char c,sign,tmp[66]; - const char *digits; - const char small_digits[] = "0123456789abcdefghijklmnopqrstuvwxyz"; - const char large_digits[] = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"; - int i; - - digits = (type & _kc_LARGE) ? large_digits : small_digits; - if (type & _kc_LEFT) - type &= ~_kc_ZEROPAD; - if (base < 2 || base > 36) - return 0; - c = (type & _kc_ZEROPAD) ? '0' : ' '; - sign = 0; - if (type & _kc_SIGN) { - if (num < 0) { - sign = '-'; - num = -num; - size--; - } else if (type & _kc_PLUS) { - sign = '+'; - size--; - } else if (type & _kc_SPACE) { - sign = ' '; - size--; - } - } - if (type & _kc_SPECIAL) { - if (base == 16) - size -= 2; - else if (base == 8) - size--; - } - i = 0; - if (num == 0) - tmp[i++]='0'; - else while (num != 0) - tmp[i++] = digits[do_div(num,base)]; - if (i > precision) - precision = i; - size -= precision; - if (!(type&(_kc_ZEROPAD+_kc_LEFT))) { - while(size-->0) { - if (buf <= end) - *buf = ' '; - ++buf; - } - } - if (sign) { - if (buf <= end) - *buf = sign; - ++buf; - } - if (type & _kc_SPECIAL) { - if (base==8) { - if (buf <= end) - *buf = '0'; - ++buf; - } else if (base==16) { - if (buf <= end) - *buf = '0'; - ++buf; - if (buf <= end) - *buf = digits[33]; - ++buf; - } - } - if (!(type & _kc_LEFT)) { - while (size-- > 0) { - if (buf <= end) - *buf = c; - ++buf; - } - } - while (i < precision--) { - if (buf <= end) - *buf = '0'; - ++buf; - } - while (i-- > 0) { - if (buf <= end) - *buf = tmp[i]; - ++buf; - } - while (size-- > 0) { - if (buf <= end) - *buf = ' '; - ++buf; - } - return buf; -} - -int _kc_vsnprintf(char *buf, size_t size, const char *fmt, va_list args) -{ - int len; - unsigned long long num; - int i, base; - char *str, *end, c; - const char *s; - - int flags; /* flags to number() */ - - int field_width; /* width of output field */ - int precision; /* min. # of digits for integers; max - number of chars for from string */ - int qualifier; /* 'h', 'l', or 'L' for integer fields */ - /* 'z' support added 23/7/1999 S.H. */ - /* 'z' changed to 'Z' --davidm 1/25/99 */ - - str = buf; - end = buf + size - 1; - - if (end < buf - 1) { - end = ((void *) -1); - size = end - buf + 1; - } - - for (; *fmt ; ++fmt) { - if (*fmt != '%') { - if (str <= end) - *str = *fmt; - ++str; - continue; - } - - /* process flags */ - flags = 0; - repeat: - ++fmt; /* this also skips first '%' */ - switch (*fmt) { - case '-': flags |= _kc_LEFT; goto repeat; - case '+': flags |= _kc_PLUS; goto repeat; - case ' ': flags |= _kc_SPACE; goto repeat; - case '#': flags |= _kc_SPECIAL; goto repeat; - case '0': flags |= _kc_ZEROPAD; goto repeat; - } - - /* get field width */ - field_width = -1; - if (isdigit(*fmt)) - field_width = skip_atoi(&fmt); - else if (*fmt == '*') { - ++fmt; - /* it's the next argument */ - field_width = va_arg(args, int); - if (field_width < 0) { - field_width = -field_width; - flags |= _kc_LEFT; - } - } - - /* get the precision */ - precision = -1; - if (*fmt == '.') { - ++fmt; - if (isdigit(*fmt)) - precision = skip_atoi(&fmt); - else if (*fmt == '*') { - ++fmt; - /* it's the next argument */ - precision = va_arg(args, int); - } - if (precision < 0) - precision = 0; - } - - /* get the conversion qualifier */ - qualifier = -1; - if (*fmt == 'h' || *fmt == 'l' || *fmt == 'L' || *fmt =='Z') { - qualifier = *fmt; - ++fmt; - } - - /* default base */ - base = 10; - - switch (*fmt) { - case 'c': - if (!(flags & _kc_LEFT)) { - while (--field_width > 0) { - if (str <= end) - *str = ' '; - ++str; - } - } - c = (unsigned char) va_arg(args, int); - if (str <= end) - *str = c; - ++str; - while (--field_width > 0) { - if (str <= end) - *str = ' '; - ++str; - } - continue; - - case 's': - s = va_arg(args, char *); - if (!s) - s = ""; - - len = strnlen(s, precision); - - if (!(flags & _kc_LEFT)) { - while (len < field_width--) { - if (str <= end) - *str = ' '; - ++str; - } - } - for (i = 0; i < len; ++i) { - if (str <= end) - *str = *s; - ++str; ++s; - } - while (len < field_width--) { - if (str <= end) - *str = ' '; - ++str; - } - continue; - - case 'p': - if (field_width == -1) { - field_width = 2*sizeof(void *); - flags |= _kc_ZEROPAD; - } - str = number(str, end, - (unsigned long) va_arg(args, void *), - 16, field_width, precision, flags); - continue; - - - case 'n': - /* FIXME: - * What does C99 say about the overflow case here? */ - if (qualifier == 'l') { - long * ip = va_arg(args, long *); - *ip = (str - buf); - } else if (qualifier == 'Z') { - size_t * ip = va_arg(args, size_t *); - *ip = (str - buf); - } else { - int * ip = va_arg(args, int *); - *ip = (str - buf); - } - continue; - - case '%': - if (str <= end) - *str = '%'; - ++str; - continue; - - /* integer number formats - set up the flags and "break" */ - case 'o': - base = 8; - break; - - case 'X': - flags |= _kc_LARGE; - case 'x': - base = 16; - break; - - case 'd': - case 'i': - flags |= _kc_SIGN; - case 'u': - break; - - default: - if (str <= end) - *str = '%'; - ++str; - if (*fmt) { - if (str <= end) - *str = *fmt; - ++str; - } else { - --fmt; - } - continue; - } - if (qualifier == 'L') - num = va_arg(args, long long); - else if (qualifier == 'l') { - num = va_arg(args, unsigned long); - if (flags & _kc_SIGN) - num = (signed long) num; - } else if (qualifier == 'Z') { - num = va_arg(args, size_t); - } else if (qualifier == 'h') { - num = (unsigned short) va_arg(args, int); - if (flags & _kc_SIGN) - num = (signed short) num; - } else { - num = va_arg(args, unsigned int); - if (flags & _kc_SIGN) - num = (signed int) num; - } - str = number(str, end, num, base, - field_width, precision, flags); - } - if (str <= end) - *str = '\0'; - else if (size > 0) - /* don't write out a null byte if the buf size is zero */ - *end = '\0'; - /* the trailing null byte doesn't count towards the total - * ++str; - */ - return str-buf; -} - -int _kc_snprintf(char * buf, size_t size, const char *fmt, ...) -{ - va_list args; - int i; - - va_start(args, fmt); - i = _kc_vsnprintf(buf,size,fmt,args); - va_end(args); - return i; -} -#endif /* < 2.4.8 */ - -/*****************************************************************************/ -#if ( LINUX_VERSION_CODE < KERNEL_VERSION(2,4,13) ) - -/**************************************/ -/* PCI DMA MAPPING */ - -#if defined(CONFIG_HIGHMEM) - -#ifndef PCI_DRAM_OFFSET -#define PCI_DRAM_OFFSET 0 -#endif - -u64 -_kc_pci_map_page(struct pci_dev *dev, struct page *page, unsigned long offset, - size_t size, int direction) -{ - return (((u64) (page - mem_map) << PAGE_SHIFT) + offset + - PCI_DRAM_OFFSET); -} - -#else /* CONFIG_HIGHMEM */ - -u64 -_kc_pci_map_page(struct pci_dev *dev, struct page *page, unsigned long offset, - size_t size, int direction) -{ - return pci_map_single(dev, (void *)page_address(page) + offset, size, - direction); -} - -#endif /* CONFIG_HIGHMEM */ - -void -_kc_pci_unmap_page(struct pci_dev *dev, u64 dma_addr, size_t size, - int direction) -{ - return pci_unmap_single(dev, dma_addr, size, direction); -} - -#endif /* 2.4.13 => 2.4.3 */ - -/*****************************************************************************/ -#if ( LINUX_VERSION_CODE < KERNEL_VERSION(2,4,3) ) - -/**************************************/ -/* PCI DRIVER API */ - -int -_kc_pci_set_dma_mask(struct pci_dev *dev, dma_addr_t mask) -{ - if (!pci_dma_supported(dev, mask)) - return -EIO; - dev->dma_mask = mask; - return 0; -} - -int -_kc_pci_request_regions(struct pci_dev *dev, char *res_name) -{ - int i; - - for (i = 0; i < 6; i++) { - if (pci_resource_len(dev, i) == 0) - continue; - - if (pci_resource_flags(dev, i) & IORESOURCE_IO) { - if (!request_region(pci_resource_start(dev, i), pci_resource_len(dev, i), res_name)) { - pci_release_regions(dev); - return -EBUSY; - } - } else if (pci_resource_flags(dev, i) & IORESOURCE_MEM) { - if (!request_mem_region(pci_resource_start(dev, i), pci_resource_len(dev, i), res_name)) { - pci_release_regions(dev); - return -EBUSY; - } - } - } - return 0; -} - -void -_kc_pci_release_regions(struct pci_dev *dev) -{ - int i; - - for (i = 0; i < 6; i++) { - if (pci_resource_len(dev, i) == 0) - continue; - - if (pci_resource_flags(dev, i) & IORESOURCE_IO) - release_region(pci_resource_start(dev, i), pci_resource_len(dev, i)); - - else if (pci_resource_flags(dev, i) & IORESOURCE_MEM) - release_mem_region(pci_resource_start(dev, i), pci_resource_len(dev, i)); - } -} - -/**************************************/ -/* NETWORK DRIVER API */ - -struct net_device * -_kc_alloc_etherdev(int sizeof_priv) -{ - struct net_device *dev; - int alloc_size; - - alloc_size = sizeof(*dev) + sizeof_priv + IFNAMSIZ + 31; - dev = kzalloc(alloc_size, GFP_KERNEL); - if (!dev) - return NULL; - - if (sizeof_priv) - dev->priv = (void *) (((unsigned long)(dev + 1) + 31) & ~31); - dev->name[0] = '\0'; - ether_setup(dev); - - return dev; -} - -int -_kc_is_valid_ether_addr(u8 *addr) -{ - const char zaddr[6] = { 0, }; - - return !(addr[0] & 1) && memcmp(addr, zaddr, 6); -} - -#endif /* 2.4.3 => 2.4.0 */ - -/*****************************************************************************/ -#if ( LINUX_VERSION_CODE < KERNEL_VERSION(2,4,6) ) - -int -_kc_pci_set_power_state(struct pci_dev *dev, int state) -{ - return 0; -} - -int -_kc_pci_enable_wake(struct pci_dev *pdev, u32 state, int enable) -{ - return 0; -} - -#endif /* 2.4.6 => 2.4.3 */ - -/*****************************************************************************/ -#if ( LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0) ) -void _kc_skb_fill_page_desc(struct sk_buff *skb, int i, struct page *page, - int off, int size) -{ - skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; - frag->page = page; - frag->page_offset = off; - frag->size = size; - skb_shinfo(skb)->nr_frags = i + 1; -} - -/* - * Original Copyright: - * find_next_bit.c: fallback find next bit implementation - * - * Copyright (C) 2004 Red Hat, Inc. All Rights Reserved. - * Written by David Howells (dhowells@redhat.com) - */ - -/** - * find_next_bit - find the next set bit in a memory region - * @addr: The address to base the search on - * @offset: The bitnumber to start searching at - * @size: The maximum size to search - */ -unsigned long find_next_bit(const unsigned long *addr, unsigned long size, - unsigned long offset) -{ - const unsigned long *p = addr + BITOP_WORD(offset); - unsigned long result = offset & ~(BITS_PER_LONG-1); - unsigned long tmp; - - if (offset >= size) - return size; - size -= result; - offset %= BITS_PER_LONG; - if (offset) { - tmp = *(p++); - tmp &= (~0UL << offset); - if (size < BITS_PER_LONG) - goto found_first; - if (tmp) - goto found_middle; - size -= BITS_PER_LONG; - result += BITS_PER_LONG; - } - while (size & ~(BITS_PER_LONG-1)) { - if ((tmp = *(p++))) - goto found_middle; - result += BITS_PER_LONG; - size -= BITS_PER_LONG; - } - if (!size) - return result; - tmp = *p; - -found_first: - tmp &= (~0UL >> (BITS_PER_LONG - size)); - if (tmp == 0UL) /* Are any bits set? */ - return result + size; /* Nope. */ -found_middle: - return result + ffs(tmp); -} - -size_t _kc_strlcpy(char *dest, const char *src, size_t size) -{ - size_t ret = strlen(src); - - if (size) { - size_t len = (ret >= size) ? size - 1 : ret; - memcpy(dest, src, len); - dest[len] = '\0'; - } - return ret; -} - -#ifndef do_div -#if BITS_PER_LONG == 32 -uint32_t __attribute__((weak)) _kc__div64_32(uint64_t *n, uint32_t base) -{ - uint64_t rem = *n; - uint64_t b = base; - uint64_t res, d = 1; - uint32_t high = rem >> 32; - - /* Reduce the thing a bit first */ - res = 0; - if (high >= base) { - high /= base; - res = (uint64_t) high << 32; - rem -= (uint64_t) (high*base) << 32; - } - - while ((int64_t)b > 0 && b < rem) { - b = b+b; - d = d+d; - } - - do { - if (rem >= b) { - rem -= b; - res += d; - } - b >>= 1; - d >>= 1; - } while (d); - - *n = res; - return rem; -} -#endif /* BITS_PER_LONG == 32 */ -#endif /* do_div */ -#endif /* 2.6.0 => 2.4.6 */ - -/*****************************************************************************/ -#if ( LINUX_VERSION_CODE < KERNEL_VERSION(2,6,4) ) -int _kc_scnprintf(char * buf, size_t size, const char *fmt, ...) -{ - va_list args; - int i; - - va_start(args, fmt); - i = vsnprintf(buf, size, fmt, args); - va_end(args); - return (i >= size) ? (size - 1) : i; -} -#endif /* < 2.6.4 */ - -/*****************************************************************************/ -#if ( LINUX_VERSION_CODE < KERNEL_VERSION(2,6,10) ) -DECLARE_BITMAP(_kcompat_node_online_map, MAX_NUMNODES) = {1}; -#endif /* < 2.6.10 */ - -/*****************************************************************************/ -#if ( LINUX_VERSION_CODE < KERNEL_VERSION(2,6,13) ) -char *_kc_kstrdup(const char *s, unsigned int gfp) -{ - size_t len; - char *buf; - - if (!s) - return NULL; - - len = strlen(s) + 1; - buf = kmalloc(len, gfp); - if (buf) - memcpy(buf, s, len); - return buf; -} -#endif /* < 2.6.13 */ - -/*****************************************************************************/ -#if ( LINUX_VERSION_CODE < KERNEL_VERSION(2,6,14) ) -void *_kc_kzalloc(size_t size, int flags) -{ - void *ret = kmalloc(size, flags); - if (ret) - memset(ret, 0, size); - return ret; -} -#endif /* <= 2.6.13 */ - -/*****************************************************************************/ -#if ( LINUX_VERSION_CODE < KERNEL_VERSION(2,6,19) ) -int _kc_skb_pad(struct sk_buff *skb, int pad) -{ - int ntail; - - /* If the skbuff is non linear tailroom is always zero.. */ - if(!skb_cloned(skb) && skb_tailroom(skb) >= pad) { - memset(skb->data+skb->len, 0, pad); - return 0; - } - - ntail = skb->data_len + pad - (skb->end - skb->tail); - if (likely(skb_cloned(skb) || ntail > 0)) { - if (pskb_expand_head(skb, 0, ntail, GFP_ATOMIC)); - goto free_skb; - } - -#ifdef MAX_SKB_FRAGS - if (skb_is_nonlinear(skb) && - !__pskb_pull_tail(skb, skb->data_len)) - goto free_skb; - -#endif - memset(skb->data + skb->len, 0, pad); - return 0; - -free_skb: - kfree_skb(skb); - return -ENOMEM; -} - -#if (!(RHEL_RELEASE_CODE && RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(5,4))) -int _kc_pci_save_state(struct pci_dev *pdev) -{ - struct net_device *netdev = pci_get_drvdata(pdev); - struct adapter_struct *adapter = netdev_priv(netdev); - int size = PCI_CONFIG_SPACE_LEN, i; - u16 pcie_cap_offset, pcie_link_status; - -#if ( LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,0) ) - /* no ->dev for 2.4 kernels */ - WARN_ON(pdev->dev.driver_data == NULL); -#endif - pcie_cap_offset = pci_find_capability(pdev, PCI_CAP_ID_EXP); - if (pcie_cap_offset) { - if (!pci_read_config_word(pdev, - pcie_cap_offset + PCIE_LINK_STATUS, - &pcie_link_status)) - size = PCIE_CONFIG_SPACE_LEN; - } - pci_config_space_ich8lan(); -#ifdef HAVE_PCI_ERS - if (adapter->config_space == NULL) -#else - WARN_ON(adapter->config_space != NULL); -#endif - adapter->config_space = kmalloc(size, GFP_KERNEL); - if (!adapter->config_space) { - printk(KERN_ERR "Out of memory in pci_save_state\n"); - return -ENOMEM; - } - for (i = 0; i < (size / 4); i++) - pci_read_config_dword(pdev, i * 4, &adapter->config_space[i]); - return 0; -} - -void _kc_pci_restore_state(struct pci_dev *pdev) -{ - struct net_device *netdev = pci_get_drvdata(pdev); - struct adapter_struct *adapter = netdev_priv(netdev); - int size = PCI_CONFIG_SPACE_LEN, i; - u16 pcie_cap_offset; - u16 pcie_link_status; - - if (adapter->config_space != NULL) { - pcie_cap_offset = pci_find_capability(pdev, PCI_CAP_ID_EXP); - if (pcie_cap_offset && - !pci_read_config_word(pdev, - pcie_cap_offset + PCIE_LINK_STATUS, - &pcie_link_status)) - size = PCIE_CONFIG_SPACE_LEN; - - pci_config_space_ich8lan(); - for (i = 0; i < (size / 4); i++) - pci_write_config_dword(pdev, i * 4, adapter->config_space[i]); -#ifndef HAVE_PCI_ERS - kfree(adapter->config_space); - adapter->config_space = NULL; -#endif - } -} -#endif /* !(RHEL_RELEASE_CODE >= RHEL 5.4) */ - -#ifdef HAVE_PCI_ERS -void _kc_free_netdev(struct net_device *netdev) -{ - struct adapter_struct *adapter = netdev_priv(netdev); - - if (adapter->config_space != NULL) - kfree(adapter->config_space); -#ifdef CONFIG_SYSFS - if (netdev->reg_state == NETREG_UNINITIALIZED) { - kfree((char *)netdev - netdev->padded); - } else { - BUG_ON(netdev->reg_state != NETREG_UNREGISTERED); - netdev->reg_state = NETREG_RELEASED; - class_device_put(&netdev->class_dev); - } -#else - kfree((char *)netdev - netdev->padded); -#endif -} -#endif - -void *_kc_kmemdup(const void *src, size_t len, unsigned gfp) -{ - void *p; - - p = kzalloc(len, gfp); - if (p) - memcpy(p, src, len); - return p; -} -#endif /* <= 2.6.19 */ - -/*****************************************************************************/ -#if ( LINUX_VERSION_CODE < KERNEL_VERSION(2,6,21) ) -struct pci_dev *_kc_netdev_to_pdev(struct net_device *netdev) -{ - return ((struct adapter_struct *)netdev_priv(netdev))->pdev; -} -#endif /* < 2.6.21 */ - -/*****************************************************************************/ -#if ( LINUX_VERSION_CODE < KERNEL_VERSION(2,6,22) ) -/* hexdump code taken from lib/hexdump.c */ -static void _kc_hex_dump_to_buffer(const void *buf, size_t len, int rowsize, - int groupsize, unsigned char *linebuf, - size_t linebuflen, bool ascii) -{ - const u8 *ptr = buf; - u8 ch; - int j, lx = 0; - int ascii_column; - - if (rowsize != 16 && rowsize != 32) - rowsize = 16; - - if (!len) - goto nil; - if (len > rowsize) /* limit to one line at a time */ - len = rowsize; - if ((len % groupsize) != 0) /* no mixed size output */ - groupsize = 1; - - switch (groupsize) { - case 8: { - const u64 *ptr8 = buf; - int ngroups = len / groupsize; - - for (j = 0; j < ngroups; j++) - lx += scnprintf((char *)(linebuf + lx), linebuflen - lx, - "%s%16.16llx", j ? " " : "", - (unsigned long long)*(ptr8 + j)); - ascii_column = 17 * ngroups + 2; - break; - } - - case 4: { - const u32 *ptr4 = buf; - int ngroups = len / groupsize; - - for (j = 0; j < ngroups; j++) - lx += scnprintf((char *)(linebuf + lx), linebuflen - lx, - "%s%8.8x", j ? " " : "", *(ptr4 + j)); - ascii_column = 9 * ngroups + 2; - break; - } - - case 2: { - const u16 *ptr2 = buf; - int ngroups = len / groupsize; - - for (j = 0; j < ngroups; j++) - lx += scnprintf((char *)(linebuf + lx), linebuflen - lx, - "%s%4.4x", j ? " " : "", *(ptr2 + j)); - ascii_column = 5 * ngroups + 2; - break; - } - - default: - for (j = 0; (j < len) && (lx + 3) <= linebuflen; j++) { - ch = ptr[j]; - linebuf[lx++] = hex_asc(ch >> 4); - linebuf[lx++] = hex_asc(ch & 0x0f); - linebuf[lx++] = ' '; - } - if (j) - lx--; - - ascii_column = 3 * rowsize + 2; - break; - } - if (!ascii) - goto nil; - - while (lx < (linebuflen - 1) && lx < (ascii_column - 1)) - linebuf[lx++] = ' '; - for (j = 0; (j < len) && (lx + 2) < linebuflen; j++) - linebuf[lx++] = (isascii(ptr[j]) && isprint(ptr[j])) ? ptr[j] - : '.'; -nil: - linebuf[lx++] = '\0'; -} - -void _kc_print_hex_dump(const char *level, - const char *prefix_str, int prefix_type, - int rowsize, int groupsize, - const void *buf, size_t len, bool ascii) -{ - const u8 *ptr = buf; - int i, linelen, remaining = len; - unsigned char linebuf[200]; - - if (rowsize != 16 && rowsize != 32) - rowsize = 16; - - for (i = 0; i < len; i += rowsize) { - linelen = min(remaining, rowsize); - remaining -= rowsize; - _kc_hex_dump_to_buffer(ptr + i, linelen, rowsize, groupsize, - linebuf, sizeof(linebuf), ascii); - - switch (prefix_type) { - case DUMP_PREFIX_ADDRESS: - printk("%s%s%*p: %s\n", level, prefix_str, - (int)(2 * sizeof(void *)), ptr + i, linebuf); - break; - case DUMP_PREFIX_OFFSET: - printk("%s%s%.8x: %s\n", level, prefix_str, i, linebuf); - break; - default: - printk("%s%s%s\n", level, prefix_str, linebuf); - break; - } - } -} - -#ifdef HAVE_I2C_SUPPORT -struct i2c_client * -_kc_i2c_new_device(struct i2c_adapter *adap, struct i2c_board_info const *info) -{ - struct i2c_client *client; - int status; - - client = kzalloc(sizeof *client, GFP_KERNEL); - if (!client) - return NULL; - - client->adapter = adap; - - client->dev.platform_data = info->platform_data; - - client->flags = info->flags; - client->addr = info->addr; - - strlcpy(client->name, info->type, sizeof(client->name)); - - /* Check for address business */ - status = i2c_check_addr(adap, client->addr); - if (status) - goto out_err; - - client->dev.parent = &client->adapter->dev; - client->dev.bus = &i2c_bus_type; - - status = i2c_attach_client(client); - if (status) - goto out_err; - - dev_dbg(&adap->dev, "client [%s] registered with bus id %s\n", - client->name, dev_name(&client->dev)); - - return client; - -out_err: - dev_err(&adap->dev, "Failed to register i2c client %s at 0x%02x " - "(%d)\n", client->name, client->addr, status); - kfree(client); - return NULL; -} -#endif /* HAVE_I2C_SUPPORT */ -#endif /* < 2.6.22 */ - -/*****************************************************************************/ -#if ( LINUX_VERSION_CODE < KERNEL_VERSION(2,6,24) ) -#ifdef NAPI -struct net_device *napi_to_poll_dev(const struct napi_struct *napi) -{ - struct adapter_q_vector *q_vector = container_of(napi, - struct adapter_q_vector, - napi); - return &q_vector->poll_dev; -} - -int __kc_adapter_clean(struct net_device *netdev, int *budget) -{ - int work_done; - int work_to_do = min(*budget, netdev->quota); - /* kcompat.h netif_napi_add puts napi struct in "fake netdev->priv" */ - struct napi_struct *napi = netdev->priv; - work_done = napi->poll(napi, work_to_do); - *budget -= work_done; - netdev->quota -= work_done; - return (work_done >= work_to_do) ? 1 : 0; -} -#endif /* NAPI */ -#endif /* <= 2.6.24 */ - -/*****************************************************************************/ -#if ( LINUX_VERSION_CODE < KERNEL_VERSION(2,6,26) ) -void _kc_pci_disable_link_state(struct pci_dev *pdev, int state) -{ - struct pci_dev *parent = pdev->bus->self; - u16 link_state; - int pos; - - if (!parent) - return; - - pos = pci_find_capability(parent, PCI_CAP_ID_EXP); - if (pos) { - pci_read_config_word(parent, pos + PCI_EXP_LNKCTL, &link_state); - link_state &= ~state; - pci_write_config_word(parent, pos + PCI_EXP_LNKCTL, link_state); - } -} -#endif /* < 2.6.26 */ - -/*****************************************************************************/ -#if ( LINUX_VERSION_CODE < KERNEL_VERSION(2,6,27) ) -#ifdef HAVE_TX_MQ -void _kc_netif_tx_stop_all_queues(struct net_device *netdev) -{ - struct adapter_struct *adapter = netdev_priv(netdev); - int i; - - netif_stop_queue(netdev); - if (netif_is_multiqueue(netdev)) - for (i = 0; i < adapter->num_tx_queues; i++) - netif_stop_subqueue(netdev, i); -} -void _kc_netif_tx_wake_all_queues(struct net_device *netdev) -{ - struct adapter_struct *adapter = netdev_priv(netdev); - int i; - - netif_wake_queue(netdev); - if (netif_is_multiqueue(netdev)) - for (i = 0; i < adapter->num_tx_queues; i++) - netif_wake_subqueue(netdev, i); -} -void _kc_netif_tx_start_all_queues(struct net_device *netdev) -{ - struct adapter_struct *adapter = netdev_priv(netdev); - int i; - - netif_start_queue(netdev); - if (netif_is_multiqueue(netdev)) - for (i = 0; i < adapter->num_tx_queues; i++) - netif_start_subqueue(netdev, i); -} -#endif /* HAVE_TX_MQ */ - -#ifndef __WARN_printf -void __kc_warn_slowpath(const char *file, int line, const char *fmt, ...) -{ - va_list args; - - printk(KERN_WARNING "------------[ cut here ]------------\n"); - printk(KERN_WARNING "WARNING: at %s:%d %s()\n", file, line); - va_start(args, fmt); - vprintk(fmt, args); - va_end(args); - - dump_stack(); -} -#endif /* __WARN_printf */ -#endif /* < 2.6.27 */ - -/*****************************************************************************/ -#if ( LINUX_VERSION_CODE < KERNEL_VERSION(2,6,28) ) - -int -_kc_pci_prepare_to_sleep(struct pci_dev *dev) -{ - pci_power_t target_state; - int error; - - target_state = pci_choose_state(dev, PMSG_SUSPEND); - - pci_enable_wake(dev, target_state, true); - - error = pci_set_power_state(dev, target_state); - - if (error) - pci_enable_wake(dev, target_state, false); - - return error; -} - -int -_kc_pci_wake_from_d3(struct pci_dev *dev, bool enable) -{ - int err; - - err = pci_enable_wake(dev, PCI_D3cold, enable); - if (err) - goto out; - - err = pci_enable_wake(dev, PCI_D3hot, enable); - -out: - return err; -} -#endif /* < 2.6.28 */ - -/*****************************************************************************/ -#if ( LINUX_VERSION_CODE < KERNEL_VERSION(2,6,29) ) -static void __kc_pci_set_master(struct pci_dev *pdev, bool enable) -{ - u16 old_cmd, cmd; - - pci_read_config_word(pdev, PCI_COMMAND, &old_cmd); - if (enable) - cmd = old_cmd | PCI_COMMAND_MASTER; - else - cmd = old_cmd & ~PCI_COMMAND_MASTER; - if (cmd != old_cmd) { - dev_dbg(pci_dev_to_dev(pdev), "%s bus mastering\n", - enable ? "enabling" : "disabling"); - pci_write_config_word(pdev, PCI_COMMAND, cmd); - } -#if ( LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,7) ) - pdev->is_busmaster = enable; -#endif -} - -void _kc_pci_clear_master(struct pci_dev *dev) -{ - __kc_pci_set_master(dev, false); -} -#endif /* < 2.6.29 */ - -#if ( LINUX_VERSION_CODE < KERNEL_VERSION(2,6,34) ) -#if (RHEL_RELEASE_CODE < RHEL_RELEASE_VERSION(6,0)) -int _kc_pci_num_vf(struct pci_dev *dev) -{ - int num_vf = 0; -#ifdef CONFIG_PCI_IOV - struct pci_dev *vfdev; - - /* loop through all ethernet devices starting at PF dev */ - vfdev = pci_get_class(PCI_CLASS_NETWORK_ETHERNET << 8, NULL); - while (vfdev) { - if (vfdev->is_virtfn && vfdev->physfn == dev) - num_vf++; - - vfdev = pci_get_class(PCI_CLASS_NETWORK_ETHERNET << 8, vfdev); - } - -#endif - return num_vf; -} -#endif /* RHEL_RELEASE_CODE */ -#endif /* < 2.6.34 */ - -#if ( LINUX_VERSION_CODE < KERNEL_VERSION(2,6,35) ) -#ifdef HAVE_TX_MQ -#if (!(RHEL_RELEASE_CODE && RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(6,0))) -#ifndef CONFIG_NETDEVICES_MULTIQUEUE -void _kc_netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq) -{ - unsigned int real_num = dev->real_num_tx_queues; - struct Qdisc *qdisc; - int i; - - if (unlikely(txq > dev->num_tx_queues)) - ; - else if (txq > real_num) - dev->real_num_tx_queues = txq; - else if ( txq < real_num) { - dev->real_num_tx_queues = txq; - for (i = txq; i < dev->num_tx_queues; i++) { - qdisc = netdev_get_tx_queue(dev, i)->qdisc; - if (qdisc) { - spin_lock_bh(qdisc_lock(qdisc)); - qdisc_reset(qdisc); - spin_unlock_bh(qdisc_lock(qdisc)); - } - } - } -} -#endif /* CONFIG_NETDEVICES_MULTIQUEUE */ -#endif /* !(RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(6,0)) */ -#endif /* HAVE_TX_MQ */ - -ssize_t _kc_simple_write_to_buffer(void *to, size_t available, loff_t *ppos, - const void __user *from, size_t count) -{ - loff_t pos = *ppos; - size_t res; - - if (pos < 0) - return -EINVAL; - if (pos >= available || !count) - return 0; - if (count > available - pos) - count = available - pos; - res = copy_from_user(to + pos, from, count); - if (res == count) - return -EFAULT; - count -= res; - *ppos = pos + count; - return count; -} - -#endif /* < 2.6.35 */ - -/*****************************************************************************/ -#if ( LINUX_VERSION_CODE < KERNEL_VERSION(2,6,36) ) -static const u32 _kc_flags_dup_features = - (ETH_FLAG_LRO | ETH_FLAG_NTUPLE | ETH_FLAG_RXHASH); - -u32 _kc_ethtool_op_get_flags(struct net_device *dev) -{ - return dev->features & _kc_flags_dup_features; -} - -int _kc_ethtool_op_set_flags(struct net_device *dev, u32 data, u32 supported) -{ - if (data & ~supported) - return -EINVAL; - - dev->features = ((dev->features & ~_kc_flags_dup_features) | - (data & _kc_flags_dup_features)); - return 0; -} -#endif /* < 2.6.36 */ - -/******************************************************************************/ -#if ( LINUX_VERSION_CODE < KERNEL_VERSION(2,6,39) ) -#if (!(RHEL_RELEASE_CODE && RHEL_RELEASE_CODE > RHEL_RELEASE_VERSION(6,0))) - - - -#endif /* !(RHEL_RELEASE_CODE > RHEL_RELEASE_VERSION(6,0)) */ -#endif /* < 2.6.39 */ - -/******************************************************************************/ -#if ( LINUX_VERSION_CODE < KERNEL_VERSION(3,4,0) ) -void _kc_skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, - int off, int size, unsigned int truesize) -{ - skb_fill_page_desc(skb, i, page, off, size); - skb->len += size; - skb->data_len += size; - skb->truesize += truesize; -} - -int _kc_simple_open(struct inode *inode, struct file *file) -{ - if (inode->i_private) - file->private_data = inode->i_private; - - return 0; -} - -#endif /* < 3.4.0 */ - -/******************************************************************************/ -#if ( LINUX_VERSION_CODE < KERNEL_VERSION(3,7,0) ) -#if !(SLE_VERSION_CODE && SLE_VERSION_CODE >= SLE_VERSION(11,3,0)) && \ - !(RHEL_RELEASE_CODE && RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(6,5)) -static inline int __kc_pcie_cap_version(struct pci_dev *dev) -{ - int pos; - u16 reg16; - - pos = pci_find_capability(dev, PCI_CAP_ID_EXP); - if (!pos) - return 0; - pci_read_config_word(dev, pos + PCI_EXP_FLAGS, ®16); - return reg16 & PCI_EXP_FLAGS_VERS; -} - -static inline bool __kc_pcie_cap_has_devctl(const struct pci_dev __always_unused *dev) -{ - return true; -} - -static inline bool __kc_pcie_cap_has_lnkctl(struct pci_dev *dev) -{ - int type = pci_pcie_type(dev); - - return __kc_pcie_cap_version(dev) > 1 || - type == PCI_EXP_TYPE_ROOT_PORT || - type == PCI_EXP_TYPE_ENDPOINT || - type == PCI_EXP_TYPE_LEG_END; -} - -static inline bool __kc_pcie_cap_has_sltctl(struct pci_dev *dev) -{ - int type = pci_pcie_type(dev); - int pos; - u16 pcie_flags_reg; - - pos = pci_find_capability(dev, PCI_CAP_ID_EXP); - if (!pos) - return 0; - pci_read_config_word(dev, pos + PCI_EXP_FLAGS, &pcie_flags_reg); - - return __kc_pcie_cap_version(dev) > 1 || - type == PCI_EXP_TYPE_ROOT_PORT || - (type == PCI_EXP_TYPE_DOWNSTREAM && - pcie_flags_reg & PCI_EXP_FLAGS_SLOT); -} - -static inline bool __kc_pcie_cap_has_rtctl(struct pci_dev *dev) -{ - int type = pci_pcie_type(dev); - - return __kc_pcie_cap_version(dev) > 1 || - type == PCI_EXP_TYPE_ROOT_PORT || - type == PCI_EXP_TYPE_RC_EC; -} - -static bool __kc_pcie_capability_reg_implemented(struct pci_dev *dev, int pos) -{ - if (!pci_is_pcie(dev)) - return false; - - switch (pos) { - case PCI_EXP_FLAGS_TYPE: - return true; - case PCI_EXP_DEVCAP: - case PCI_EXP_DEVCTL: - case PCI_EXP_DEVSTA: - return __kc_pcie_cap_has_devctl(dev); - case PCI_EXP_LNKCAP: - case PCI_EXP_LNKCTL: - case PCI_EXP_LNKSTA: - return __kc_pcie_cap_has_lnkctl(dev); - case PCI_EXP_SLTCAP: - case PCI_EXP_SLTCTL: - case PCI_EXP_SLTSTA: - return __kc_pcie_cap_has_sltctl(dev); - case PCI_EXP_RTCTL: - case PCI_EXP_RTCAP: - case PCI_EXP_RTSTA: - return __kc_pcie_cap_has_rtctl(dev); - case PCI_EXP_DEVCAP2: - case PCI_EXP_DEVCTL2: - case PCI_EXP_LNKCAP2: - case PCI_EXP_LNKCTL2: - case PCI_EXP_LNKSTA2: - return __kc_pcie_cap_version(dev) > 1; - default: - return false; - } -} - -/* - * Note that these accessor functions are only for the "PCI Express - * Capability" (see PCIe spec r3.0, sec 7.8). They do not apply to the - * other "PCI Express Extended Capabilities" (AER, VC, ACS, MFVC, etc.) - */ -int __kc_pcie_capability_read_word(struct pci_dev *dev, int pos, u16 *val) -{ - int ret; - - *val = 0; - if (pos & 1) - return -EINVAL; - - if (__kc_pcie_capability_reg_implemented(dev, pos)) { - ret = pci_read_config_word(dev, pci_pcie_cap(dev) + pos, val); - /* - * Reset *val to 0 if pci_read_config_word() fails, it may - * have been written as 0xFFFF if hardware error happens - * during pci_read_config_word(). - */ - if (ret) - *val = 0; - return ret; - } - - /* - * For Functions that do not implement the Slot Capabilities, - * Slot Status, and Slot Control registers, these spaces must - * be hardwired to 0b, with the exception of the Presence Detect - * State bit in the Slot Status register of Downstream Ports, - * which must be hardwired to 1b. (PCIe Base Spec 3.0, sec 7.8) - */ - if (pci_is_pcie(dev) && pos == PCI_EXP_SLTSTA && - pci_pcie_type(dev) == PCI_EXP_TYPE_DOWNSTREAM) { - *val = PCI_EXP_SLTSTA_PDS; - } - - return 0; -} - -int __kc_pcie_capability_write_word(struct pci_dev *dev, int pos, u16 val) -{ - if (pos & 1) - return -EINVAL; - - if (!__kc_pcie_capability_reg_implemented(dev, pos)) - return 0; - - return pci_write_config_word(dev, pci_pcie_cap(dev) + pos, val); -} - -int __kc_pcie_capability_clear_and_set_word(struct pci_dev *dev, int pos, - u16 clear, u16 set) -{ - int ret; - u16 val; - - ret = __kc_pcie_capability_read_word(dev, pos, &val); - if (!ret) { - val &= ~clear; - val |= set; - ret = __kc_pcie_capability_write_word(dev, pos, val); - } - - return ret; -} -#endif /* !(SLE_VERSION_CODE && SLE_VERSION_CODE >= SLE_VERSION(11,3,0)) && \ - !(RHEL_RELEASE_CODE && RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(6,5)) */ -#endif /* < 3.7.0 */ - -/******************************************************************************/ -#if ( LINUX_VERSION_CODE < KERNEL_VERSION(3,9,0) ) -#endif /* 3.9.0 */ - -/*****************************************************************************/ -#if ( LINUX_VERSION_CODE < KERNEL_VERSION(3,10,0) ) -#ifdef CONFIG_PCI_IOV -int __kc_pci_vfs_assigned(struct pci_dev *dev) -{ - unsigned int vfs_assigned = 0; -#ifdef HAVE_PCI_DEV_FLAGS_ASSIGNED - int pos; - struct pci_dev *vfdev; - unsigned short dev_id; - - /* only search if we are a PF */ - if (!dev->is_physfn) - return 0; - - /* find SR-IOV capability */ - pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_SRIOV); - if (!pos) - return 0; - - /* - * determine the device ID for the VFs, the vendor ID will be the - * same as the PF so there is no need to check for that one - */ - pci_read_config_word(dev, pos + PCI_SRIOV_VF_DID, &dev_id); - - /* loop through all the VFs to see if we own any that are assigned */ - vfdev = pci_get_device(dev->vendor, dev_id, NULL); - while (vfdev) { - /* - * It is considered assigned if it is a virtual function with - * our dev as the physical function and the assigned bit is set - */ - if (vfdev->is_virtfn && (vfdev->physfn == dev) && - (vfdev->dev_flags & PCI_DEV_FLAGS_ASSIGNED)) - vfs_assigned++; - - vfdev = pci_get_device(dev->vendor, dev_id, vfdev); - } - -#endif /* HAVE_PCI_DEV_FLAGS_ASSIGNED */ - return vfs_assigned; -} - -#endif /* CONFIG_PCI_IOV */ -#endif /* 3.10.0 */ diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/kcompat.h b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/kcompat.h index aea253b1..443a3f28 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/kcompat.h +++ b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/kcompat.h @@ -697,19 +697,22 @@ struct _kc_ethtool_pauseparam { #define SLE_VERSION(a,b,c) KERNEL_VERSION(a,b,c) #endif #ifdef CONFIG_SUSE_KERNEL -#if ( LINUX_VERSION_CODE == KERNEL_VERSION(2,6,27) ) -/* SLES11 GA is 2.6.27 based */ -#define SLE_VERSION_CODE SLE_VERSION(11,0,0) -#elif ( LINUX_VERSION_CODE == KERNEL_VERSION(2,6,32) ) -/* SLES11 SP1 is 2.6.32 based */ -#define SLE_VERSION_CODE SLE_VERSION(11,1,0) +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 57)) +/* SLES12SP3 is at least 4.4.57+ based */ +#define SLE_VERSION_CODE SLE_VERSION(12, 3, 0) +#elif ( LINUX_VERSION_CODE >= KERNEL_VERSION(3,12,28) ) +/* SLES12 is at least 3.12.28+ based */ +#define SLE_VERSION_CODE SLE_VERSION(12,0,0) #elif ((LINUX_VERSION_CODE >= KERNEL_VERSION(3,0,61)) && \ (LINUX_VERSION_CODE < KERNEL_VERSION(3,1,0))) /* SLES11 SP3 is at least 3.0.61+ based */ #define SLE_VERSION_CODE SLE_VERSION(11,3,0) -#elif ( LINUX_VERSION_CODE >= KERNEL_VERSION(3,12,28) ) -/* SLES12 is at least 3.12.28+ based */ -#define SLE_VERSION_CODE SLE_VERSION(12,0,0) +#elif ( LINUX_VERSION_CODE == KERNEL_VERSION(2,6,32) ) +/* SLES11 SP1 is 2.6.32 based */ +#define SLE_VERSION_CODE SLE_VERSION(11,1,0) +#elif ( LINUX_VERSION_CODE == KERNEL_VERSION(2,6,27) ) +/* SLES11 GA is 2.6.27 based */ +#define SLE_VERSION_CODE SLE_VERSION(11,0,0) #endif /* LINUX_VERSION_CODE == KERNEL_VERSION(x,y,z) */ #endif /* CONFIG_SUSE_KERNEL */ #ifndef SLE_VERSION_CODE @@ -1162,7 +1165,7 @@ static inline u32 _kc_netif_msg_init(int debug_value, int default_msg_enable_bit #define pci_register_driver pci_module_init /* - * Most of the dma compat code is copied/modifed from the 2.4.37 + * Most of the dma compat code is copied/modified from the 2.4.37 * /include/linux/libata-compat.h header file */ /* These definitions mirror those in pci.h, so they can be used @@ -3929,12 +3932,17 @@ skb_set_hash(struct sk_buff *skb, __u32 hash, __always_unused int type) #define vlan_tx_tag_present skb_vlan_tag_present #endif -#if ( LINUX_VERSION_CODE >= KERNEL_VERSION(4,9,0) ) +#if ((LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 0)) || \ + (SLE_VERSION_CODE && SLE_VERSION_CODE >= SLE_VERSION(12, 3, 0))) #define HAVE_VF_VLAN_PROTO -#endif /* >= 4.9.0 */ +#endif /* >= 4.9.0, >= SLES12SP3 */ #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0) #define HAVE_PCI_ENABLE_MSIX #endif +#if defined(timer_setup) && defined(from_timer) +#define HAVE_TIMER_SETUP +#endif + #endif /* _KCOMPAT_H_ */ diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/kcompat_ethtool.c b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/kcompat_ethtool.c deleted file mode 100644 index e1a89388..00000000 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/kcompat_ethtool.c +++ /dev/null @@ -1,1171 +0,0 @@ -/******************************************************************************* - - Intel(R) Gigabit Ethernet Linux driver - Copyright(c) 2007-2013 Intel Corporation. - - This program is free software; you can redistribute it and/or modify it - under the terms and conditions of the GNU General Public License, - version 2, as published by the Free Software Foundation. - - This program is distributed in the hope it will be useful, but WITHOUT - ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - more details. - - You should have received a copy of the GNU General Public License along with - this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. - - The full GNU General Public License is included in this distribution in - the file called "COPYING". - - Contact Information: - e1000-devel Mailing List - Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 - -*******************************************************************************/ - -/* - * net/core/ethtool.c - Ethtool ioctl handler - * Copyright (c) 2003 Matthew Wilcox - * - * This file is where we call all the ethtool_ops commands to get - * the information ethtool needs. We fall back to calling do_ioctl() - * for drivers which haven't been converted to ethtool_ops yet. - * - * It's GPL, stupid. - * - * Modification by sfeldma@pobox.com to work as backward compat - * solution for pre-ethtool_ops kernels. - * - copied struct ethtool_ops from ethtool.h - * - defined SET_ETHTOOL_OPS - * - put in some #ifndef NETIF_F_xxx wrappers - * - changes refs to dev->ethtool_ops to ethtool_ops - * - changed dev_ethtool to ethtool_ioctl - * - remove EXPORT_SYMBOL()s - * - added _kc_ prefix in built-in ethtool_op_xxx ops. - */ - -#include -#include -#include -#include -#include -#include -#include - -#include "kcompat.h" - -#undef SUPPORTED_10000baseT_Full -#define SUPPORTED_10000baseT_Full (1 << 12) -#undef ADVERTISED_10000baseT_Full -#define ADVERTISED_10000baseT_Full (1 << 12) -#undef SPEED_10000 -#define SPEED_10000 10000 - -#undef ethtool_ops -#define ethtool_ops _kc_ethtool_ops - -struct _kc_ethtool_ops { - int (*get_settings)(struct net_device *, struct ethtool_cmd *); - int (*set_settings)(struct net_device *, struct ethtool_cmd *); - void (*get_drvinfo)(struct net_device *, struct ethtool_drvinfo *); - int (*get_regs_len)(struct net_device *); - void (*get_regs)(struct net_device *, struct ethtool_regs *, void *); - void (*get_wol)(struct net_device *, struct ethtool_wolinfo *); - int (*set_wol)(struct net_device *, struct ethtool_wolinfo *); - u32 (*get_msglevel)(struct net_device *); - void (*set_msglevel)(struct net_device *, u32); - int (*nway_reset)(struct net_device *); - u32 (*get_link)(struct net_device *); - int (*get_eeprom_len)(struct net_device *); - int (*get_eeprom)(struct net_device *, struct ethtool_eeprom *, u8 *); - int (*set_eeprom)(struct net_device *, struct ethtool_eeprom *, u8 *); - int (*get_coalesce)(struct net_device *, struct ethtool_coalesce *); - int (*set_coalesce)(struct net_device *, struct ethtool_coalesce *); - void (*get_ringparam)(struct net_device *, struct ethtool_ringparam *); - int (*set_ringparam)(struct net_device *, struct ethtool_ringparam *); - void (*get_pauseparam)(struct net_device *, - struct ethtool_pauseparam*); - int (*set_pauseparam)(struct net_device *, - struct ethtool_pauseparam*); - u32 (*get_rx_csum)(struct net_device *); - int (*set_rx_csum)(struct net_device *, u32); - u32 (*get_tx_csum)(struct net_device *); - int (*set_tx_csum)(struct net_device *, u32); - u32 (*get_sg)(struct net_device *); - int (*set_sg)(struct net_device *, u32); - u32 (*get_tso)(struct net_device *); - int (*set_tso)(struct net_device *, u32); - int (*self_test_count)(struct net_device *); - void (*self_test)(struct net_device *, struct ethtool_test *, u64 *); - void (*get_strings)(struct net_device *, u32 stringset, u8 *); - int (*phys_id)(struct net_device *, u32); - int (*get_stats_count)(struct net_device *); - void (*get_ethtool_stats)(struct net_device *, struct ethtool_stats *, - u64 *); -} *ethtool_ops = NULL; - -#undef SET_ETHTOOL_OPS -#define SET_ETHTOOL_OPS(netdev, ops) (ethtool_ops = (ops)) - -/* - * Some useful ethtool_ops methods that are device independent. If we find that - * all drivers want to do the same thing here, we can turn these into dev_() - * function calls. - */ - -#undef ethtool_op_get_link -#define ethtool_op_get_link _kc_ethtool_op_get_link -u32 _kc_ethtool_op_get_link(struct net_device *dev) -{ - return netif_carrier_ok(dev) ? 1 : 0; -} - -#undef ethtool_op_get_tx_csum -#define ethtool_op_get_tx_csum _kc_ethtool_op_get_tx_csum -u32 _kc_ethtool_op_get_tx_csum(struct net_device *dev) -{ -#ifdef NETIF_F_IP_CSUM - return (dev->features & NETIF_F_IP_CSUM) != 0; -#else - return 0; -#endif -} - -#undef ethtool_op_set_tx_csum -#define ethtool_op_set_tx_csum _kc_ethtool_op_set_tx_csum -int _kc_ethtool_op_set_tx_csum(struct net_device *dev, u32 data) -{ -#ifdef NETIF_F_IP_CSUM - if (data) -#ifdef NETIF_F_IPV6_CSUM - dev->features |= (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM); - else - dev->features &= ~(NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM); -#else - dev->features |= NETIF_F_IP_CSUM; - else - dev->features &= ~NETIF_F_IP_CSUM; -#endif -#endif - - return 0; -} - -#undef ethtool_op_get_sg -#define ethtool_op_get_sg _kc_ethtool_op_get_sg -u32 _kc_ethtool_op_get_sg(struct net_device *dev) -{ -#ifdef NETIF_F_SG - return (dev->features & NETIF_F_SG) != 0; -#else - return 0; -#endif -} - -#undef ethtool_op_set_sg -#define ethtool_op_set_sg _kc_ethtool_op_set_sg -int _kc_ethtool_op_set_sg(struct net_device *dev, u32 data) -{ -#ifdef NETIF_F_SG - if (data) - dev->features |= NETIF_F_SG; - else - dev->features &= ~NETIF_F_SG; -#endif - - return 0; -} - -#undef ethtool_op_get_tso -#define ethtool_op_get_tso _kc_ethtool_op_get_tso -u32 _kc_ethtool_op_get_tso(struct net_device *dev) -{ -#ifdef NETIF_F_TSO - return (dev->features & NETIF_F_TSO) != 0; -#else - return 0; -#endif -} - -#undef ethtool_op_set_tso -#define ethtool_op_set_tso _kc_ethtool_op_set_tso -int _kc_ethtool_op_set_tso(struct net_device *dev, u32 data) -{ -#ifdef NETIF_F_TSO - if (data) - dev->features |= NETIF_F_TSO; - else - dev->features &= ~NETIF_F_TSO; -#endif - - return 0; -} - -/* Handlers for each ethtool command */ - -static int ethtool_get_settings(struct net_device *dev, void *useraddr) -{ - struct ethtool_cmd cmd = { ETHTOOL_GSET }; - int err; - - if (!ethtool_ops->get_settings) - return -EOPNOTSUPP; - - err = ethtool_ops->get_settings(dev, &cmd); - if (err < 0) - return err; - - if (copy_to_user(useraddr, &cmd, sizeof(cmd))) - return -EFAULT; - return 0; -} - -static int ethtool_set_settings(struct net_device *dev, void *useraddr) -{ - struct ethtool_cmd cmd; - - if (!ethtool_ops->set_settings) - return -EOPNOTSUPP; - - if (copy_from_user(&cmd, useraddr, sizeof(cmd))) - return -EFAULT; - - return ethtool_ops->set_settings(dev, &cmd); -} - -static int ethtool_get_drvinfo(struct net_device *dev, void *useraddr) -{ - struct ethtool_drvinfo info; - struct ethtool_ops *ops = ethtool_ops; - - if (!ops->get_drvinfo) - return -EOPNOTSUPP; - - memset(&info, 0, sizeof(info)); - info.cmd = ETHTOOL_GDRVINFO; - ops->get_drvinfo(dev, &info); - - if (ops->self_test_count) - info.testinfo_len = ops->self_test_count(dev); - if (ops->get_stats_count) - info.n_stats = ops->get_stats_count(dev); - if (ops->get_regs_len) - info.regdump_len = ops->get_regs_len(dev); - if (ops->get_eeprom_len) - info.eedump_len = ops->get_eeprom_len(dev); - - if (copy_to_user(useraddr, &info, sizeof(info))) - return -EFAULT; - return 0; -} - -static int ethtool_get_regs(struct net_device *dev, char *useraddr) -{ - struct ethtool_regs regs; - struct ethtool_ops *ops = ethtool_ops; - void *regbuf; - int reglen, ret; - - if (!ops->get_regs || !ops->get_regs_len) - return -EOPNOTSUPP; - - if (copy_from_user(®s, useraddr, sizeof(regs))) - return -EFAULT; - - reglen = ops->get_regs_len(dev); - if (regs.len > reglen) - regs.len = reglen; - - regbuf = kmalloc(reglen, GFP_USER); - if (!regbuf) - return -ENOMEM; - - ops->get_regs(dev, ®s, regbuf); - - ret = -EFAULT; - if (copy_to_user(useraddr, ®s, sizeof(regs))) - goto out; - useraddr += offsetof(struct ethtool_regs, data); - if (copy_to_user(useraddr, regbuf, reglen)) - goto out; - ret = 0; - -out: - kfree(regbuf); - return ret; -} - -static int ethtool_get_wol(struct net_device *dev, char *useraddr) -{ - struct ethtool_wolinfo wol = { ETHTOOL_GWOL }; - - if (!ethtool_ops->get_wol) - return -EOPNOTSUPP; - - ethtool_ops->get_wol(dev, &wol); - - if (copy_to_user(useraddr, &wol, sizeof(wol))) - return -EFAULT; - return 0; -} - -static int ethtool_set_wol(struct net_device *dev, char *useraddr) -{ - struct ethtool_wolinfo wol; - - if (!ethtool_ops->set_wol) - return -EOPNOTSUPP; - - if (copy_from_user(&wol, useraddr, sizeof(wol))) - return -EFAULT; - - return ethtool_ops->set_wol(dev, &wol); -} - -static int ethtool_get_msglevel(struct net_device *dev, char *useraddr) -{ - struct ethtool_value edata = { ETHTOOL_GMSGLVL }; - - if (!ethtool_ops->get_msglevel) - return -EOPNOTSUPP; - - edata.data = ethtool_ops->get_msglevel(dev); - - if (copy_to_user(useraddr, &edata, sizeof(edata))) - return -EFAULT; - return 0; -} - -static int ethtool_set_msglevel(struct net_device *dev, char *useraddr) -{ - struct ethtool_value edata; - - if (!ethtool_ops->set_msglevel) - return -EOPNOTSUPP; - - if (copy_from_user(&edata, useraddr, sizeof(edata))) - return -EFAULT; - - ethtool_ops->set_msglevel(dev, edata.data); - return 0; -} - -static int ethtool_nway_reset(struct net_device *dev) -{ - if (!ethtool_ops->nway_reset) - return -EOPNOTSUPP; - - return ethtool_ops->nway_reset(dev); -} - -static int ethtool_get_link(struct net_device *dev, void *useraddr) -{ - struct ethtool_value edata = { ETHTOOL_GLINK }; - - if (!ethtool_ops->get_link) - return -EOPNOTSUPP; - - edata.data = ethtool_ops->get_link(dev); - - if (copy_to_user(useraddr, &edata, sizeof(edata))) - return -EFAULT; - return 0; -} - -static int ethtool_get_eeprom(struct net_device *dev, void *useraddr) -{ - struct ethtool_eeprom eeprom; - struct ethtool_ops *ops = ethtool_ops; - u8 *data; - int ret; - - if (!ops->get_eeprom || !ops->get_eeprom_len) - return -EOPNOTSUPP; - - if (copy_from_user(&eeprom, useraddr, sizeof(eeprom))) - return -EFAULT; - - /* Check for wrap and zero */ - if (eeprom.offset + eeprom.len <= eeprom.offset) - return -EINVAL; - - /* Check for exceeding total eeprom len */ - if (eeprom.offset + eeprom.len > ops->get_eeprom_len(dev)) - return -EINVAL; - - data = kmalloc(eeprom.len, GFP_USER); - if (!data) - return -ENOMEM; - - ret = -EFAULT; - if (copy_from_user(data, useraddr + sizeof(eeprom), eeprom.len)) - goto out; - - ret = ops->get_eeprom(dev, &eeprom, data); - if (ret) - goto out; - - ret = -EFAULT; - if (copy_to_user(useraddr, &eeprom, sizeof(eeprom))) - goto out; - if (copy_to_user(useraddr + sizeof(eeprom), data, eeprom.len)) - goto out; - ret = 0; - -out: - kfree(data); - return ret; -} - -static int ethtool_set_eeprom(struct net_device *dev, void *useraddr) -{ - struct ethtool_eeprom eeprom; - struct ethtool_ops *ops = ethtool_ops; - u8 *data; - int ret; - - if (!ops->set_eeprom || !ops->get_eeprom_len) - return -EOPNOTSUPP; - - if (copy_from_user(&eeprom, useraddr, sizeof(eeprom))) - return -EFAULT; - - /* Check for wrap and zero */ - if (eeprom.offset + eeprom.len <= eeprom.offset) - return -EINVAL; - - /* Check for exceeding total eeprom len */ - if (eeprom.offset + eeprom.len > ops->get_eeprom_len(dev)) - return -EINVAL; - - data = kmalloc(eeprom.len, GFP_USER); - if (!data) - return -ENOMEM; - - ret = -EFAULT; - if (copy_from_user(data, useraddr + sizeof(eeprom), eeprom.len)) - goto out; - - ret = ops->set_eeprom(dev, &eeprom, data); - if (ret) - goto out; - - if (copy_to_user(useraddr + sizeof(eeprom), data, eeprom.len)) - ret = -EFAULT; - -out: - kfree(data); - return ret; -} - -static int ethtool_get_coalesce(struct net_device *dev, void *useraddr) -{ - struct ethtool_coalesce coalesce = { ETHTOOL_GCOALESCE }; - - if (!ethtool_ops->get_coalesce) - return -EOPNOTSUPP; - - ethtool_ops->get_coalesce(dev, &coalesce); - - if (copy_to_user(useraddr, &coalesce, sizeof(coalesce))) - return -EFAULT; - return 0; -} - -static int ethtool_set_coalesce(struct net_device *dev, void *useraddr) -{ - struct ethtool_coalesce coalesce; - - if (!ethtool_ops->get_coalesce) - return -EOPNOTSUPP; - - if (copy_from_user(&coalesce, useraddr, sizeof(coalesce))) - return -EFAULT; - - return ethtool_ops->set_coalesce(dev, &coalesce); -} - -static int ethtool_get_ringparam(struct net_device *dev, void *useraddr) -{ - struct ethtool_ringparam ringparam = { ETHTOOL_GRINGPARAM }; - - if (!ethtool_ops->get_ringparam) - return -EOPNOTSUPP; - - ethtool_ops->get_ringparam(dev, &ringparam); - - if (copy_to_user(useraddr, &ringparam, sizeof(ringparam))) - return -EFAULT; - return 0; -} - -static int ethtool_set_ringparam(struct net_device *dev, void *useraddr) -{ - struct ethtool_ringparam ringparam; - - if (!ethtool_ops->get_ringparam) - return -EOPNOTSUPP; - - if (copy_from_user(&ringparam, useraddr, sizeof(ringparam))) - return -EFAULT; - - return ethtool_ops->set_ringparam(dev, &ringparam); -} - -static int ethtool_get_pauseparam(struct net_device *dev, void *useraddr) -{ - struct ethtool_pauseparam pauseparam = { ETHTOOL_GPAUSEPARAM }; - - if (!ethtool_ops->get_pauseparam) - return -EOPNOTSUPP; - - ethtool_ops->get_pauseparam(dev, &pauseparam); - - if (copy_to_user(useraddr, &pauseparam, sizeof(pauseparam))) - return -EFAULT; - return 0; -} - -static int ethtool_set_pauseparam(struct net_device *dev, void *useraddr) -{ - struct ethtool_pauseparam pauseparam; - - if (!ethtool_ops->get_pauseparam) - return -EOPNOTSUPP; - - if (copy_from_user(&pauseparam, useraddr, sizeof(pauseparam))) - return -EFAULT; - - return ethtool_ops->set_pauseparam(dev, &pauseparam); -} - -static int ethtool_get_rx_csum(struct net_device *dev, char *useraddr) -{ - struct ethtool_value edata = { ETHTOOL_GRXCSUM }; - - if (!ethtool_ops->get_rx_csum) - return -EOPNOTSUPP; - - edata.data = ethtool_ops->get_rx_csum(dev); - - if (copy_to_user(useraddr, &edata, sizeof(edata))) - return -EFAULT; - return 0; -} - -static int ethtool_set_rx_csum(struct net_device *dev, char *useraddr) -{ - struct ethtool_value edata; - - if (!ethtool_ops->set_rx_csum) - return -EOPNOTSUPP; - - if (copy_from_user(&edata, useraddr, sizeof(edata))) - return -EFAULT; - - ethtool_ops->set_rx_csum(dev, edata.data); - return 0; -} - -static int ethtool_get_tx_csum(struct net_device *dev, char *useraddr) -{ - struct ethtool_value edata = { ETHTOOL_GTXCSUM }; - - if (!ethtool_ops->get_tx_csum) - return -EOPNOTSUPP; - - edata.data = ethtool_ops->get_tx_csum(dev); - - if (copy_to_user(useraddr, &edata, sizeof(edata))) - return -EFAULT; - return 0; -} - -static int ethtool_set_tx_csum(struct net_device *dev, char *useraddr) -{ - struct ethtool_value edata; - - if (!ethtool_ops->set_tx_csum) - return -EOPNOTSUPP; - - if (copy_from_user(&edata, useraddr, sizeof(edata))) - return -EFAULT; - - return ethtool_ops->set_tx_csum(dev, edata.data); -} - -static int ethtool_get_sg(struct net_device *dev, char *useraddr) -{ - struct ethtool_value edata = { ETHTOOL_GSG }; - - if (!ethtool_ops->get_sg) - return -EOPNOTSUPP; - - edata.data = ethtool_ops->get_sg(dev); - - if (copy_to_user(useraddr, &edata, sizeof(edata))) - return -EFAULT; - return 0; -} - -static int ethtool_set_sg(struct net_device *dev, char *useraddr) -{ - struct ethtool_value edata; - - if (!ethtool_ops->set_sg) - return -EOPNOTSUPP; - - if (copy_from_user(&edata, useraddr, sizeof(edata))) - return -EFAULT; - - return ethtool_ops->set_sg(dev, edata.data); -} - -static int ethtool_get_tso(struct net_device *dev, char *useraddr) -{ - struct ethtool_value edata = { ETHTOOL_GTSO }; - - if (!ethtool_ops->get_tso) - return -EOPNOTSUPP; - - edata.data = ethtool_ops->get_tso(dev); - - if (copy_to_user(useraddr, &edata, sizeof(edata))) - return -EFAULT; - return 0; -} - -static int ethtool_set_tso(struct net_device *dev, char *useraddr) -{ - struct ethtool_value edata; - - if (!ethtool_ops->set_tso) - return -EOPNOTSUPP; - - if (copy_from_user(&edata, useraddr, sizeof(edata))) - return -EFAULT; - - return ethtool_ops->set_tso(dev, edata.data); -} - -static int ethtool_self_test(struct net_device *dev, char *useraddr) -{ - struct ethtool_test test; - struct ethtool_ops *ops = ethtool_ops; - u64 *data; - int ret; - - if (!ops->self_test || !ops->self_test_count) - return -EOPNOTSUPP; - - if (copy_from_user(&test, useraddr, sizeof(test))) - return -EFAULT; - - test.len = ops->self_test_count(dev); - data = kmalloc(test.len * sizeof(u64), GFP_USER); - if (!data) - return -ENOMEM; - - ops->self_test(dev, &test, data); - - ret = -EFAULT; - if (copy_to_user(useraddr, &test, sizeof(test))) - goto out; - useraddr += sizeof(test); - if (copy_to_user(useraddr, data, test.len * sizeof(u64))) - goto out; - ret = 0; - -out: - kfree(data); - return ret; -} - -static int ethtool_get_strings(struct net_device *dev, void *useraddr) -{ - struct ethtool_gstrings gstrings; - struct ethtool_ops *ops = ethtool_ops; - u8 *data; - int ret; - - if (!ops->get_strings) - return -EOPNOTSUPP; - - if (copy_from_user(&gstrings, useraddr, sizeof(gstrings))) - return -EFAULT; - - switch (gstrings.string_set) { - case ETH_SS_TEST: - if (!ops->self_test_count) - return -EOPNOTSUPP; - gstrings.len = ops->self_test_count(dev); - break; - case ETH_SS_STATS: - if (!ops->get_stats_count) - return -EOPNOTSUPP; - gstrings.len = ops->get_stats_count(dev); - break; - default: - return -EINVAL; - } - - data = kmalloc(gstrings.len * ETH_GSTRING_LEN, GFP_USER); - if (!data) - return -ENOMEM; - - ops->get_strings(dev, gstrings.string_set, data); - - ret = -EFAULT; - if (copy_to_user(useraddr, &gstrings, sizeof(gstrings))) - goto out; - useraddr += sizeof(gstrings); - if (copy_to_user(useraddr, data, gstrings.len * ETH_GSTRING_LEN)) - goto out; - ret = 0; - -out: - kfree(data); - return ret; -} - -static int ethtool_phys_id(struct net_device *dev, void *useraddr) -{ - struct ethtool_value id; - - if (!ethtool_ops->phys_id) - return -EOPNOTSUPP; - - if (copy_from_user(&id, useraddr, sizeof(id))) - return -EFAULT; - - return ethtool_ops->phys_id(dev, id.data); -} - -static int ethtool_get_stats(struct net_device *dev, void *useraddr) -{ - struct ethtool_stats stats; - struct ethtool_ops *ops = ethtool_ops; - u64 *data; - int ret; - - if (!ops->get_ethtool_stats || !ops->get_stats_count) - return -EOPNOTSUPP; - - if (copy_from_user(&stats, useraddr, sizeof(stats))) - return -EFAULT; - - stats.n_stats = ops->get_stats_count(dev); - data = kmalloc(stats.n_stats * sizeof(u64), GFP_USER); - if (!data) - return -ENOMEM; - - ops->get_ethtool_stats(dev, &stats, data); - - ret = -EFAULT; - if (copy_to_user(useraddr, &stats, sizeof(stats))) - goto out; - useraddr += sizeof(stats); - if (copy_to_user(useraddr, data, stats.n_stats * sizeof(u64))) - goto out; - ret = 0; - -out: - kfree(data); - return ret; -} - -/* The main entry point in this file. Called from net/core/dev.c */ - -#define ETHTOOL_OPS_COMPAT -int ethtool_ioctl(struct ifreq *ifr) -{ - struct net_device *dev = __dev_get_by_name(ifr->ifr_name); - void *useraddr = (void *) ifr->ifr_data; - u32 ethcmd; - - /* - * XXX: This can be pushed down into the ethtool_* handlers that - * need it. Keep existing behavior for the moment. - */ - if (!capable(CAP_NET_ADMIN)) - return -EPERM; - - if (!dev || !netif_device_present(dev)) - return -ENODEV; - - if (copy_from_user(ðcmd, useraddr, sizeof (ethcmd))) - return -EFAULT; - - switch (ethcmd) { - case ETHTOOL_GSET: - return ethtool_get_settings(dev, useraddr); - case ETHTOOL_SSET: - return ethtool_set_settings(dev, useraddr); - case ETHTOOL_GDRVINFO: - return ethtool_get_drvinfo(dev, useraddr); - case ETHTOOL_GREGS: - return ethtool_get_regs(dev, useraddr); - case ETHTOOL_GWOL: - return ethtool_get_wol(dev, useraddr); - case ETHTOOL_SWOL: - return ethtool_set_wol(dev, useraddr); - case ETHTOOL_GMSGLVL: - return ethtool_get_msglevel(dev, useraddr); - case ETHTOOL_SMSGLVL: - return ethtool_set_msglevel(dev, useraddr); - case ETHTOOL_NWAY_RST: - return ethtool_nway_reset(dev); - case ETHTOOL_GLINK: - return ethtool_get_link(dev, useraddr); - case ETHTOOL_GEEPROM: - return ethtool_get_eeprom(dev, useraddr); - case ETHTOOL_SEEPROM: - return ethtool_set_eeprom(dev, useraddr); - case ETHTOOL_GCOALESCE: - return ethtool_get_coalesce(dev, useraddr); - case ETHTOOL_SCOALESCE: - return ethtool_set_coalesce(dev, useraddr); - case ETHTOOL_GRINGPARAM: - return ethtool_get_ringparam(dev, useraddr); - case ETHTOOL_SRINGPARAM: - return ethtool_set_ringparam(dev, useraddr); - case ETHTOOL_GPAUSEPARAM: - return ethtool_get_pauseparam(dev, useraddr); - case ETHTOOL_SPAUSEPARAM: - return ethtool_set_pauseparam(dev, useraddr); - case ETHTOOL_GRXCSUM: - return ethtool_get_rx_csum(dev, useraddr); - case ETHTOOL_SRXCSUM: - return ethtool_set_rx_csum(dev, useraddr); - case ETHTOOL_GTXCSUM: - return ethtool_get_tx_csum(dev, useraddr); - case ETHTOOL_STXCSUM: - return ethtool_set_tx_csum(dev, useraddr); - case ETHTOOL_GSG: - return ethtool_get_sg(dev, useraddr); - case ETHTOOL_SSG: - return ethtool_set_sg(dev, useraddr); - case ETHTOOL_GTSO: - return ethtool_get_tso(dev, useraddr); - case ETHTOOL_STSO: - return ethtool_set_tso(dev, useraddr); - case ETHTOOL_TEST: - return ethtool_self_test(dev, useraddr); - case ETHTOOL_GSTRINGS: - return ethtool_get_strings(dev, useraddr); - case ETHTOOL_PHYS_ID: - return ethtool_phys_id(dev, useraddr); - case ETHTOOL_GSTATS: - return ethtool_get_stats(dev, useraddr); - default: - return -EOPNOTSUPP; - } - - return -EOPNOTSUPP; -} - -#define mii_if_info _kc_mii_if_info -struct _kc_mii_if_info { - int phy_id; - int advertising; - int phy_id_mask; - int reg_num_mask; - - unsigned int full_duplex : 1; /* is full duplex? */ - unsigned int force_media : 1; /* is autoneg. disabled? */ - - struct net_device *dev; - int (*mdio_read) (struct net_device *dev, int phy_id, int location); - void (*mdio_write) (struct net_device *dev, int phy_id, int location, int val); -}; - -struct ethtool_cmd; -struct mii_ioctl_data; - -#undef mii_link_ok -#define mii_link_ok _kc_mii_link_ok -#undef mii_nway_restart -#define mii_nway_restart _kc_mii_nway_restart -#undef mii_ethtool_gset -#define mii_ethtool_gset _kc_mii_ethtool_gset -#undef mii_ethtool_sset -#define mii_ethtool_sset _kc_mii_ethtool_sset -#undef mii_check_link -#define mii_check_link _kc_mii_check_link -extern int _kc_mii_link_ok (struct mii_if_info *mii); -extern int _kc_mii_nway_restart (struct mii_if_info *mii); -extern int _kc_mii_ethtool_gset(struct mii_if_info *mii, - struct ethtool_cmd *ecmd); -extern int _kc_mii_ethtool_sset(struct mii_if_info *mii, - struct ethtool_cmd *ecmd); -extern void _kc_mii_check_link (struct mii_if_info *mii); -#if ( LINUX_VERSION_CODE > KERNEL_VERSION(2,4,6) ) -#undef generic_mii_ioctl -#define generic_mii_ioctl _kc_generic_mii_ioctl -extern int _kc_generic_mii_ioctl(struct mii_if_info *mii_if, - struct mii_ioctl_data *mii_data, int cmd, - unsigned int *duplex_changed); -#endif /* > 2.4.6 */ - - -struct _kc_pci_dev_ext { - struct pci_dev *dev; - void *pci_drvdata; - struct pci_driver *driver; -}; - -struct _kc_net_dev_ext { - struct net_device *dev; - unsigned int carrier; -}; - - -/**************************************/ -/* mii support */ - -int _kc_mii_ethtool_gset(struct mii_if_info *mii, struct ethtool_cmd *ecmd) -{ - struct net_device *dev = mii->dev; - u32 advert, bmcr, lpa, nego; - - ecmd->supported = - (SUPPORTED_10baseT_Half | SUPPORTED_10baseT_Full | - SUPPORTED_100baseT_Half | SUPPORTED_100baseT_Full | - SUPPORTED_Autoneg | SUPPORTED_TP | SUPPORTED_MII); - - /* only supports twisted-pair */ - ecmd->port = PORT_MII; - - /* only supports internal transceiver */ - ecmd->transceiver = XCVR_INTERNAL; - - /* this isn't fully supported at higher layers */ - ecmd->phy_address = mii->phy_id; - - ecmd->advertising = ADVERTISED_TP | ADVERTISED_MII; - advert = mii->mdio_read(dev, mii->phy_id, MII_ADVERTISE); - if (advert & ADVERTISE_10HALF) - ecmd->advertising |= ADVERTISED_10baseT_Half; - if (advert & ADVERTISE_10FULL) - ecmd->advertising |= ADVERTISED_10baseT_Full; - if (advert & ADVERTISE_100HALF) - ecmd->advertising |= ADVERTISED_100baseT_Half; - if (advert & ADVERTISE_100FULL) - ecmd->advertising |= ADVERTISED_100baseT_Full; - - bmcr = mii->mdio_read(dev, mii->phy_id, MII_BMCR); - lpa = mii->mdio_read(dev, mii->phy_id, MII_LPA); - if (bmcr & BMCR_ANENABLE) { - ecmd->advertising |= ADVERTISED_Autoneg; - ecmd->autoneg = AUTONEG_ENABLE; - - nego = mii_nway_result(advert & lpa); - if (nego == LPA_100FULL || nego == LPA_100HALF) - ecmd->speed = SPEED_100; - else - ecmd->speed = SPEED_10; - if (nego == LPA_100FULL || nego == LPA_10FULL) { - ecmd->duplex = DUPLEX_FULL; - mii->full_duplex = 1; - } else { - ecmd->duplex = DUPLEX_HALF; - mii->full_duplex = 0; - } - } else { - ecmd->autoneg = AUTONEG_DISABLE; - - ecmd->speed = (bmcr & BMCR_SPEED100) ? SPEED_100 : SPEED_10; - ecmd->duplex = (bmcr & BMCR_FULLDPLX) ? DUPLEX_FULL : DUPLEX_HALF; - } - - /* ignore maxtxpkt, maxrxpkt for now */ - - return 0; -} - -int _kc_mii_ethtool_sset(struct mii_if_info *mii, struct ethtool_cmd *ecmd) -{ - struct net_device *dev = mii->dev; - - if (ecmd->speed != SPEED_10 && ecmd->speed != SPEED_100) - return -EINVAL; - if (ecmd->duplex != DUPLEX_HALF && ecmd->duplex != DUPLEX_FULL) - return -EINVAL; - if (ecmd->port != PORT_MII) - return -EINVAL; - if (ecmd->transceiver != XCVR_INTERNAL) - return -EINVAL; - if (ecmd->phy_address != mii->phy_id) - return -EINVAL; - if (ecmd->autoneg != AUTONEG_DISABLE && ecmd->autoneg != AUTONEG_ENABLE) - return -EINVAL; - - /* ignore supported, maxtxpkt, maxrxpkt */ - - if (ecmd->autoneg == AUTONEG_ENABLE) { - u32 bmcr, advert, tmp; - - if ((ecmd->advertising & (ADVERTISED_10baseT_Half | - ADVERTISED_10baseT_Full | - ADVERTISED_100baseT_Half | - ADVERTISED_100baseT_Full)) == 0) - return -EINVAL; - - /* advertise only what has been requested */ - advert = mii->mdio_read(dev, mii->phy_id, MII_ADVERTISE); - tmp = advert & ~(ADVERTISE_ALL | ADVERTISE_100BASE4); - if (ADVERTISED_10baseT_Half) - tmp |= ADVERTISE_10HALF; - if (ADVERTISED_10baseT_Full) - tmp |= ADVERTISE_10FULL; - if (ADVERTISED_100baseT_Half) - tmp |= ADVERTISE_100HALF; - if (ADVERTISED_100baseT_Full) - tmp |= ADVERTISE_100FULL; - if (advert != tmp) { - mii->mdio_write(dev, mii->phy_id, MII_ADVERTISE, tmp); - mii->advertising = tmp; - } - - /* turn on autonegotiation, and force a renegotiate */ - bmcr = mii->mdio_read(dev, mii->phy_id, MII_BMCR); - bmcr |= (BMCR_ANENABLE | BMCR_ANRESTART); - mii->mdio_write(dev, mii->phy_id, MII_BMCR, bmcr); - - mii->force_media = 0; - } else { - u32 bmcr, tmp; - - /* turn off auto negotiation, set speed and duplexity */ - bmcr = mii->mdio_read(dev, mii->phy_id, MII_BMCR); - tmp = bmcr & ~(BMCR_ANENABLE | BMCR_SPEED100 | BMCR_FULLDPLX); - if (ecmd->speed == SPEED_100) - tmp |= BMCR_SPEED100; - if (ecmd->duplex == DUPLEX_FULL) { - tmp |= BMCR_FULLDPLX; - mii->full_duplex = 1; - } else - mii->full_duplex = 0; - if (bmcr != tmp) - mii->mdio_write(dev, mii->phy_id, MII_BMCR, tmp); - - mii->force_media = 1; - } - return 0; -} - -int _kc_mii_link_ok (struct mii_if_info *mii) -{ - /* first, a dummy read, needed to latch some MII phys */ - mii->mdio_read(mii->dev, mii->phy_id, MII_BMSR); - if (mii->mdio_read(mii->dev, mii->phy_id, MII_BMSR) & BMSR_LSTATUS) - return 1; - return 0; -} - -int _kc_mii_nway_restart (struct mii_if_info *mii) -{ - int bmcr; - int r = -EINVAL; - - /* if autoneg is off, it's an error */ - bmcr = mii->mdio_read(mii->dev, mii->phy_id, MII_BMCR); - - if (bmcr & BMCR_ANENABLE) { - bmcr |= BMCR_ANRESTART; - mii->mdio_write(mii->dev, mii->phy_id, MII_BMCR, bmcr); - r = 0; - } - - return r; -} - -void _kc_mii_check_link (struct mii_if_info *mii) -{ - int cur_link = mii_link_ok(mii); - int prev_link = netif_carrier_ok(mii->dev); - - if (cur_link && !prev_link) - netif_carrier_on(mii->dev); - else if (prev_link && !cur_link) - netif_carrier_off(mii->dev); -} - -#if ( LINUX_VERSION_CODE > KERNEL_VERSION(2,4,6) ) -int _kc_generic_mii_ioctl(struct mii_if_info *mii_if, - struct mii_ioctl_data *mii_data, int cmd, - unsigned int *duplex_chg_out) -{ - int rc = 0; - unsigned int duplex_changed = 0; - - if (duplex_chg_out) - *duplex_chg_out = 0; - - mii_data->phy_id &= mii_if->phy_id_mask; - mii_data->reg_num &= mii_if->reg_num_mask; - - switch(cmd) { - case SIOCDEVPRIVATE: /* binary compat, remove in 2.5 */ - case SIOCGMIIPHY: - mii_data->phy_id = mii_if->phy_id; - /* fall through */ - - case SIOCDEVPRIVATE + 1:/* binary compat, remove in 2.5 */ - case SIOCGMIIREG: - mii_data->val_out = - mii_if->mdio_read(mii_if->dev, mii_data->phy_id, - mii_data->reg_num); - break; - - case SIOCDEVPRIVATE + 2:/* binary compat, remove in 2.5 */ - case SIOCSMIIREG: { - u16 val = mii_data->val_in; - - if (!capable(CAP_NET_ADMIN)) - return -EPERM; - - if (mii_data->phy_id == mii_if->phy_id) { - switch(mii_data->reg_num) { - case MII_BMCR: { - unsigned int new_duplex = 0; - if (val & (BMCR_RESET|BMCR_ANENABLE)) - mii_if->force_media = 0; - else - mii_if->force_media = 1; - if (mii_if->force_media && - (val & BMCR_FULLDPLX)) - new_duplex = 1; - if (mii_if->full_duplex != new_duplex) { - duplex_changed = 1; - mii_if->full_duplex = new_duplex; - } - break; - } - case MII_ADVERTISE: - mii_if->advertising = val; - break; - default: - /* do nothing */ - break; - } - } - - mii_if->mdio_write(mii_if->dev, mii_data->phy_id, - mii_data->reg_num, val); - break; - } - - default: - rc = -EOPNOTSUPP; - break; - } - - if ((rc == 0) && (duplex_chg_out) && (duplex_changed)) - *duplex_chg_out = 1; - - return rc; -} -#endif /* > 2.4.6 */ diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/COPYING b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/COPYING deleted file mode 100644 index 5f297e5b..00000000 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/COPYING +++ /dev/null @@ -1,339 +0,0 @@ - -"This software program is licensed subject to the GNU General Public License -(GPL). Version 2, June 1991, available at -" - -GNU General Public License - -Version 2, June 1991 - -Copyright (C) 1989, 1991 Free Software Foundation, Inc. -59 Temple Place - Suite 330, Boston, MA 02111-1307, USA - -Everyone is permitted to copy and distribute verbatim copies of this license -document, but changing it is not allowed. - -Preamble - -The licenses for most software are designed to take away your freedom to -share and change it. By contrast, the GNU General Public License is intended -to guarantee your freedom to share and change free software--to make sure -the software is free for all its users. This General Public License applies -to most of the Free Software Foundation's software and to any other program -whose authors commit to using it. (Some other Free Software Foundation -software is covered by the GNU Library General Public License instead.) You -can apply it to your programs, too. - -When we speak of free software, we are referring to freedom, not price. Our -General Public Licenses are designed to make sure that you have the freedom -to distribute copies of free software (and charge for this service if you -wish), that you receive source code or can get it if you want it, that you -can change the software or use pieces of it in new free programs; and that -you know you can do these things. - -To protect your rights, we need to make restrictions that forbid anyone to -deny you these rights or to ask you to surrender the rights. These -restrictions translate to certain responsibilities for you if you distribute -copies of the software, or if you modify it. - -For example, if you distribute copies of such a program, whether gratis or -for a fee, you must give the recipients all the rights that you have. You -must make sure that they, too, receive or can get the source code. And you -must show them these terms so they know their rights. - -We protect your rights with two steps: (1) copyright the software, and (2) -offer you this license which gives you legal permission to copy, distribute -and/or modify the software. - -Also, for each author's protection and ours, we want to make certain that -everyone understands that there is no warranty for this free software. If -the software is modified by someone else and passed on, we want its -recipients to know that what they have is not the original, so that any -problems introduced by others will not reflect on the original authors' -reputations. - -Finally, any free program is threatened constantly by software patents. We -wish to avoid the danger that redistributors of a free program will -individually obtain patent licenses, in effect making the program -proprietary. To prevent this, we have made it clear that any patent must be -licensed for everyone's free use or not licensed at all. - -The precise terms and conditions for copying, distribution and modification -follow. - -TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - -0. This License applies to any program or other work which contains a notice - placed by the copyright holder saying it may be distributed under the - terms of this General Public License. The "Program", below, refers to any - such program or work, and a "work based on the Program" means either the - Program or any derivative work under copyright law: that is to say, a - work containing the Program or a portion of it, either verbatim or with - modifications and/or translated into another language. (Hereinafter, - translation is included without limitation in the term "modification".) - Each licensee is addressed as "you". - - Activities other than copying, distribution and modification are not - covered by this License; they are outside its scope. The act of running - the Program is not restricted, and the output from the Program is covered - only if its contents constitute a work based on the Program (independent - of having been made by running the Program). Whether that is true depends - on what the Program does. - -1. You may copy and distribute verbatim copies of the Program's source code - as you receive it, in any medium, provided that you conspicuously and - appropriately publish on each copy an appropriate copyright notice and - disclaimer of warranty; keep intact all the notices that refer to this - License and to the absence of any warranty; and give any other recipients - of the Program a copy of this License along with the Program. - - You may charge a fee for the physical act of transferring a copy, and you - may at your option offer warranty protection in exchange for a fee. - -2. You may modify your copy or copies of the Program or any portion of it, - thus forming a work based on the Program, and copy and distribute such - modifications or work under the terms of Section 1 above, provided that - you also meet all of these conditions: - - * a) You must cause the modified files to carry prominent notices stating - that you changed the files and the date of any change. - - * b) You must cause any work that you distribute or publish, that in - whole or in part contains or is derived from the Program or any part - thereof, to be licensed as a whole at no charge to all third parties - under the terms of this License. - - * c) If the modified program normally reads commands interactively when - run, you must cause it, when started running for such interactive - use in the most ordinary way, to print or display an announcement - including an appropriate copyright notice and a notice that there is - no warranty (or else, saying that you provide a warranty) and that - users may redistribute the program under these conditions, and - telling the user how to view a copy of this License. (Exception: if - the Program itself is interactive but does not normally print such - an announcement, your work based on the Program is not required to - print an announcement.) - - These requirements apply to the modified work as a whole. If identifiable - sections of that work are not derived from the Program, and can be - reasonably considered independent and separate works in themselves, then - this License, and its terms, do not apply to those sections when you - distribute them as separate works. But when you distribute the same - sections as part of a whole which is a work based on the Program, the - distribution of the whole must be on the terms of this License, whose - permissions for other licensees extend to the entire whole, and thus to - each and every part regardless of who wrote it. - - Thus, it is not the intent of this section to claim rights or contest - your rights to work written entirely by you; rather, the intent is to - exercise the right to control the distribution of derivative or - collective works based on the Program. - - In addition, mere aggregation of another work not based on the Program - with the Program (or with a work based on the Program) on a volume of a - storage or distribution medium does not bring the other work under the - scope of this License. - -3. You may copy and distribute the Program (or a work based on it, under - Section 2) in object code or executable form under the terms of Sections - 1 and 2 above provided that you also do one of the following: - - * a) Accompany it with the complete corresponding machine-readable source - code, which must be distributed under the terms of Sections 1 and 2 - above on a medium customarily used for software interchange; or, - - * b) Accompany it with a written offer, valid for at least three years, - to give any third party, for a charge no more than your cost of - physically performing source distribution, a complete machine- - readable copy of the corresponding source code, to be distributed - under the terms of Sections 1 and 2 above on a medium customarily - used for software interchange; or, - - * c) Accompany it with the information you received as to the offer to - distribute corresponding source code. (This alternative is allowed - only for noncommercial distribution and only if you received the - program in object code or executable form with such an offer, in - accord with Subsection b above.) - - The source code for a work means the preferred form of the work for - making modifications to it. For an executable work, complete source code - means all the source code for all modules it contains, plus any - associated interface definition files, plus the scripts used to control - compilation and installation of the executable. However, as a special - exception, the source code distributed need not include anything that is - normally distributed (in either source or binary form) with the major - components (compiler, kernel, and so on) of the operating system on which - the executable runs, unless that component itself accompanies the - executable. - - If distribution of executable or object code is made by offering access - to copy from a designated place, then offering equivalent access to copy - the source code from the same place counts as distribution of the source - code, even though third parties are not compelled to copy the source - along with the object code. - -4. You may not copy, modify, sublicense, or distribute the Program except as - expressly provided under this License. Any attempt otherwise to copy, - modify, sublicense or distribute the Program is void, and will - automatically terminate your rights under this License. However, parties - who have received copies, or rights, from you under this License will not - have their licenses terminated so long as such parties remain in full - compliance. - -5. You are not required to accept this License, since you have not signed - it. However, nothing else grants you permission to modify or distribute - the Program or its derivative works. These actions are prohibited by law - if you do not accept this License. Therefore, by modifying or - distributing the Program (or any work based on the Program), you - indicate your acceptance of this License to do so, and all its terms and - conditions for copying, distributing or modifying the Program or works - based on it. - -6. Each time you redistribute the Program (or any work based on the - Program), the recipient automatically receives a license from the - original licensor to copy, distribute or modify the Program subject to - these terms and conditions. You may not impose any further restrictions - on the recipients' exercise of the rights granted herein. You are not - responsible for enforcing compliance by third parties to this License. - -7. If, as a consequence of a court judgment or allegation of patent - infringement or for any other reason (not limited to patent issues), - conditions are imposed on you (whether by court order, agreement or - otherwise) that contradict the conditions of this License, they do not - excuse you from the conditions of this License. If you cannot distribute - so as to satisfy simultaneously your obligations under this License and - any other pertinent obligations, then as a consequence you may not - distribute the Program at all. For example, if a patent license would - not permit royalty-free redistribution of the Program by all those who - receive copies directly or indirectly through you, then the only way you - could satisfy both it and this License would be to refrain entirely from - distribution of the Program. - - If any portion of this section is held invalid or unenforceable under any - particular circumstance, the balance of the section is intended to apply - and the section as a whole is intended to apply in other circumstances. - - It is not the purpose of this section to induce you to infringe any - patents or other property right claims or to contest validity of any - such claims; this section has the sole purpose of protecting the - integrity of the free software distribution system, which is implemented - by public license practices. Many people have made generous contributions - to the wide range of software distributed through that system in - reliance on consistent application of that system; it is up to the - author/donor to decide if he or she is willing to distribute software - through any other system and a licensee cannot impose that choice. - - This section is intended to make thoroughly clear what is believed to be - a consequence of the rest of this License. - -8. If the distribution and/or use of the Program is restricted in certain - countries either by patents or by copyrighted interfaces, the original - copyright holder who places the Program under this License may add an - explicit geographical distribution limitation excluding those countries, - so that distribution is permitted only in or among countries not thus - excluded. In such case, this License incorporates the limitation as if - written in the body of this License. - -9. The Free Software Foundation may publish revised and/or new versions of - the General Public License from time to time. Such new versions will be - similar in spirit to the present version, but may differ in detail to - address new problems or concerns. - - Each version is given a distinguishing version number. If the Program - specifies a version number of this License which applies to it and "any - later version", you have the option of following the terms and - conditions either of that version or of any later version published by - the Free Software Foundation. If the Program does not specify a version - number of this License, you may choose any version ever published by the - Free Software Foundation. - -10. If you wish to incorporate parts of the Program into other free programs - whose distribution conditions are different, write to the author to ask - for permission. For software which is copyrighted by the Free Software - Foundation, write to the Free Software Foundation; we sometimes make - exceptions for this. Our decision will be guided by the two goals of - preserving the free status of all derivatives of our free software and - of promoting the sharing and reuse of software generally. - - NO WARRANTY - -11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY - FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN - OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES - PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER - EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE - ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH - YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL - NECESSARY SERVICING, REPAIR OR CORRECTION. - -12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING - WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR - REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR - DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL - DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM - (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED - INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF - THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR - OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. - -END OF TERMS AND CONDITIONS - -How to Apply These Terms to Your New Programs - -If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it free -software which everyone can redistribute and change under these terms. - -To do so, attach the following notices to the program. It is safest to -attach them to the start of each source file to most effectively convey the -exclusion of warranty; and each file should have at least the "copyright" -line and a pointer to where the full notice is found. - -one line to give the program's name and an idea of what it does. -Copyright (C) yyyy name of author - -This program is free software; you can redistribute it and/or modify it -under the terms of the GNU General Public License as published by the Free -Software Foundation; either version 2 of the License, or (at your option) -any later version. - -This program is distributed in the hope that it will be useful, but WITHOUT -ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -more details. - -You should have received a copy of the GNU General Public License along with -this program; if not, write to the Free Software Foundation, Inc., 59 -Temple Place - Suite 330, Boston, MA 02111-1307, USA. - -Also add information on how to contact you by electronic and paper mail. - -If the program is interactive, make it output a short notice like this when -it starts in an interactive mode: - -Gnomovision version 69, Copyright (C) year name of author Gnomovision comes -with ABSOLUTELY NO WARRANTY; for details type 'show w'. This is free -software, and you are welcome to redistribute it under certain conditions; -type 'show c' for details. - -The hypothetical commands 'show w' and 'show c' should show the appropriate -parts of the General Public License. Of course, the commands you use may be -called something other than 'show w' and 'show c'; they could even be -mouse-clicks or menu items--whatever suits your program. - -You should also get your employer (if you work as a programmer) or your -school, if any, to sign a "copyright disclaimer" for the program, if -necessary. Here is a sample; alter the names: - -Yoyodyne, Inc., hereby disclaims all copyright interest in the program -'Gnomovision' (which makes passes at compilers) written by James Hacker. - -signature of Ty Coon, 1 April 1989 -Ty Coon, President of Vice - -This General Public License does not permit incorporating your program into -proprietary programs. If your program is a subroutine library, you may -consider it more useful to permit linking proprietary applications with the -library. If this is what you want to do, use the GNU Library General Public -License instead of this License. diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe.h b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe.h index 222c2c71..59415469 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe.h +++ b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe.h @@ -17,7 +17,7 @@ 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. The full GNU General Public License is included in this distribution in - the file called "COPYING". + the file called "LICENSE.GPL". Contact Information: e1000-devel Mailing List diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_82598.c b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_82598.c index 24015844..e17b7f18 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_82598.c +++ b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_82598.c @@ -17,7 +17,7 @@ 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. The full GNU General Public License is included in this distribution in - the file called "COPYING". + the file called "LICENSE.GPL". Contact Information: e1000-devel Mailing List diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_82598.h b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_82598.h index c6abb020..00a584f4 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_82598.h +++ b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_82598.h @@ -17,7 +17,7 @@ 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. The full GNU General Public License is included in this distribution in - the file called "COPYING". + the file called "LICENSE.GPL". Contact Information: e1000-devel Mailing List diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_82599.c b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_82599.c index c6f4130d..30de47eb 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_82599.c +++ b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_82599.c @@ -17,7 +17,7 @@ 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. The full GNU General Public License is included in this distribution in - the file called "COPYING". + the file called "LICENSE.GPL". Contact Information: e1000-devel Mailing List diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_82599.h b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_82599.h index 02be92ab..41024400 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_82599.h +++ b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_82599.h @@ -17,7 +17,7 @@ 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. The full GNU General Public License is included in this distribution in - the file called "COPYING". + the file called "LICENSE.GPL". Contact Information: e1000-devel Mailing List diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_api.c b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_api.c index ef7ce629..4808d06e 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_api.c +++ b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_api.c @@ -17,7 +17,7 @@ 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. The full GNU General Public License is included in this distribution in - the file called "COPYING". + the file called "LICENSE.GPL". Contact Information: e1000-devel Mailing List @@ -718,7 +718,7 @@ s32 ixgbe_update_eeprom_checksum(struct ixgbe_hw *hw) * @vmdq: VMDq pool to assign * * Puts an ethernet address into a receive address register, or - * finds the rar that it is aleady in; adds to the pool list + * finds the rar that it is already in; adds to the pool list **/ s32 ixgbe_insert_mac_addr(struct ixgbe_hw *hw, u8 *addr, u32 vmdq) { diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_api.h b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_api.h index a6ab30d2..98b74000 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_api.h +++ b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_api.h @@ -17,7 +17,7 @@ 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. The full GNU General Public License is included in this distribution in - the file called "COPYING". + the file called "LICENSE.GPL". Contact Information: e1000-devel Mailing List diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_common.c b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_common.c index 93659ca0..2c861de5 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_common.c +++ b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_common.c @@ -17,7 +17,7 @@ 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. The full GNU General Public License is included in this distribution in - the file called "COPYING". + the file called "LICENSE.GPL". Contact Information: e1000-devel Mailing List @@ -3007,7 +3007,7 @@ u16 ixgbe_get_pcie_msix_count_generic(struct ixgbe_hw *hw) * @vmdq: VMDq pool to assign * * Puts an ethernet address into a receive address register, or - * finds the rar that it is aleady in; adds to the pool list + * finds the rar that it is already in; adds to the pool list **/ s32 ixgbe_insert_mac_addr_generic(struct ixgbe_hw *hw, u8 *addr, u32 vmdq) { diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_common.h b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_common.h index 9bd6f534..6ae5926f 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_common.h +++ b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_common.h @@ -17,7 +17,7 @@ 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. The full GNU General Public License is included in this distribution in - the file called "COPYING". + the file called "LICENSE.GPL". Contact Information: e1000-devel Mailing List diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_dcb.h b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_dcb.h index a6690451..5e6f9ac9 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_dcb.h +++ b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_dcb.h @@ -17,7 +17,7 @@ 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. The full GNU General Public License is included in this distribution in - the file called "COPYING". + the file called "LICENSE.GPL". Contact Information: e1000-devel Mailing List diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_ethtool.c b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_ethtool.c index 11472bd3..cdfcb959 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_ethtool.c +++ b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_ethtool.c @@ -17,7 +17,7 @@ 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. The full GNU General Public License is included in this distribution in - the file called "COPYING". + the file called "LICENSE.GPL". Contact Information: e1000-devel Mailing List @@ -1462,7 +1462,7 @@ static int ixgbe_eeprom_test(struct ixgbe_adapter *adapter, u64 *data) static irqreturn_t ixgbe_test_intr(int irq, void *data) { - struct net_device *netdev = (struct net_device *) data; + struct net_device *netdev = data; struct ixgbe_adapter *adapter = netdev_priv(netdev); adapter->test_icr |= IXGBE_READ_REG(&adapter->hw, IXGBE_EICR); @@ -2447,7 +2447,7 @@ static int ixgbe_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd, break; case ETHTOOL_GRXCLSRLALL: ret = ixgbe_get_ethtool_fdir_all(adapter, cmd, - (u32 *)rule_locs); + rule_locs); break; case ETHTOOL_GRXFH: ret = ixgbe_get_rss_hash_opts(adapter, cmd); diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_fcoe.h b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_fcoe.h index cad28622..48f7dcfc 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_fcoe.h +++ b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_fcoe.h @@ -17,7 +17,7 @@ 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. The full GNU General Public License is included in this distribution in - the file called "COPYING". + the file called "LICENSE.GPL". Contact Information: e1000-devel Mailing List diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_main.c b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_main.c index 238028d0..d26016c9 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_main.c +++ b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_main.c @@ -17,7 +17,7 @@ 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. The full GNU General Public License is included in this distribution in - the file called "COPYING". + the file called "LICENSE.GPL". Contact Information: e1000-devel Mailing List diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_mbx.h b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_mbx.h index 124f00de..5ced84f8 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_mbx.h +++ b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_mbx.h @@ -17,7 +17,7 @@ 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. The full GNU General Public License is included in this distribution in - the file called "COPYING". + the file called "LICENSE.GPL". Contact Information: e1000-devel Mailing List diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_osdep.h b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_osdep.h index d161600b..c6f8e21f 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_osdep.h +++ b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_osdep.h @@ -17,7 +17,7 @@ 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. The full GNU General Public License is included in this distribution in - the file called "COPYING". + the file called "LICENSE.GPL". Contact Information: e1000-devel Mailing List diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_phy.c b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_phy.c index e3f5275e..234fa632 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_phy.c +++ b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_phy.c @@ -17,7 +17,7 @@ 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. The full GNU General Public License is included in this distribution in - the file called "COPYING". + the file called "LICENSE.GPL". Contact Information: e1000-devel Mailing List diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_phy.h b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_phy.h index bbe5a9e3..5ae171ac 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_phy.h +++ b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_phy.h @@ -17,7 +17,7 @@ 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. The full GNU General Public License is included in this distribution in - the file called "COPYING". + the file called "LICENSE.GPL". Contact Information: e1000-devel Mailing List diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_sriov.h b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_sriov.h deleted file mode 100644 index 5e3559fd..00000000 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_sriov.h +++ /dev/null @@ -1,73 +0,0 @@ -/******************************************************************************* - - Intel 10 Gigabit PCI Express Linux driver - Copyright(c) 1999 - 2012 Intel Corporation. - - This program is free software; you can redistribute it and/or modify it - under the terms and conditions of the GNU General Public License, - version 2, as published by the Free Software Foundation. - - This program is distributed in the hope it will be useful, but WITHOUT - ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - more details. - - You should have received a copy of the GNU General Public License along with - this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. - - The full GNU General Public License is included in this distribution in - the file called "COPYING". - - Contact Information: - e1000-devel Mailing List - Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 - -*******************************************************************************/ - - -#ifndef _IXGBE_SRIOV_H_ -#define _IXGBE_SRIOV_H_ - -int ixgbe_set_vf_multicasts(struct ixgbe_adapter *adapter, - int entries, u16 *hash_list, u32 vf); -void ixgbe_restore_vf_multicasts(struct ixgbe_adapter *adapter); -int ixgbe_set_vf_vlan(struct ixgbe_adapter *adapter, int add, int vid, u32 vf); -void ixgbe_set_vmolr(struct ixgbe_hw *hw, u32 vf, bool aupe); -void ixgbe_vf_reset_event(struct ixgbe_adapter *adapter, u32 vf); -void ixgbe_vf_reset_msg(struct ixgbe_adapter *adapter, u32 vf); -void ixgbe_msg_task(struct ixgbe_adapter *adapter); -int ixgbe_set_vf_mac(struct ixgbe_adapter *adapter, - int vf, unsigned char *mac_addr); -void ixgbe_disable_tx_rx(struct ixgbe_adapter *adapter); -void ixgbe_ping_all_vfs(struct ixgbe_adapter *adapter); -#ifdef IFLA_VF_MAX -int ixgbe_ndo_set_vf_mac(struct net_device *netdev, int queue, u8 *mac); -int ixgbe_ndo_set_vf_vlan(struct net_device *netdev, int queue, u16 vlan, - u8 qos); -int ixgbe_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate); -#ifdef HAVE_VF_SPOOFCHK_CONFIGURE -int ixgbe_ndo_set_vf_spoofchk(struct net_device *netdev, int vf, bool setting); -#endif -int ixgbe_ndo_get_vf_config(struct net_device *netdev, - int vf, struct ifla_vf_info *ivi); -#endif -void ixgbe_disable_sriov(struct ixgbe_adapter *adapter); -#ifdef CONFIG_PCI_IOV -int ixgbe_vf_configuration(struct pci_dev *pdev, unsigned int event_mask); -void ixgbe_enable_sriov(struct ixgbe_adapter *adapter); -#endif -int ixgbe_check_vf_assignment(struct ixgbe_adapter *adapter); -#ifdef IFLA_VF_MAX -void ixgbe_check_vf_rate_limit(struct ixgbe_adapter *adapter); -#endif /* IFLA_VF_MAX */ -void ixgbe_dump_registers(struct ixgbe_adapter *adapter); - -/* - * These are defined in ixgbe_type.h on behalf of the VF driver - * but we need them here unwrapped for the PF driver. - */ -#define IXGBE_DEV_ID_82599_VF 0x10ED -#define IXGBE_DEV_ID_X540_VF 0x1515 - -#endif /* _IXGBE_SRIOV_H_ */ diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_type.h b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_type.h index 6b21c879..bda61fa4 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_type.h +++ b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_type.h @@ -17,7 +17,7 @@ 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. The full GNU General Public License is included in this distribution in - the file called "COPYING". + the file called "LICENSE.GPL". Contact Information: e1000-devel Mailing List diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_x540.c b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_x540.c index b99d9e84..2affe242 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_x540.c +++ b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_x540.c @@ -17,7 +17,7 @@ 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. The full GNU General Public License is included in this distribution in - the file called "COPYING". + the file called "LICENSE.GPL". Contact Information: e1000-devel Mailing List diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_x540.h b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_x540.h index 77e8952d..38bcc87b 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_x540.h +++ b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_x540.h @@ -17,7 +17,7 @@ 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. The full GNU General Public License is included in this distribution in - the file called "COPYING". + the file called "LICENSE.GPL". Contact Information: e1000-devel Mailing List diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/kcompat.c b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/kcompat.c index 5f2523ed..d84c7ccb 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/kcompat.c +++ b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/kcompat.c @@ -17,7 +17,7 @@ 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. The full GNU General Public License is included in this distribution in - the file called "COPYING". + the file called "LICENSE.GPL". Contact Information: e1000-devel Mailing List diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/kcompat.h b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/kcompat.h index bf27579b..f62a7b56 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/kcompat.h +++ b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/kcompat.h @@ -17,7 +17,7 @@ 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. The full GNU General Public License is included in this distribution in - the file called "COPYING". + the file called "LICENSE.GPL". Contact Information: e1000-devel Mailing List @@ -1108,7 +1108,7 @@ static inline u32 _kc_netif_msg_init(int debug_value, int default_msg_enable_bit #define pci_register_driver pci_module_init /* - * Most of the dma compat code is copied/modifed from the 2.4.37 + * Most of the dma compat code is copied/modified from the 2.4.37 * /include/linux/libata-compat.h header file */ /* These definitions mirror those in pci.h, so they can be used @@ -3140,4 +3140,16 @@ static inline int __kc_pci_vfs_assigned(struct pci_dev *dev) #define SET_ETHTOOL_OPS(netdev, ops) ((netdev)->ethtool_ops = (ops)) #endif /* >= 3.16.0 */ +/* + * vlan_tx_tag_* macros renamed to skb_vlan_tag_* (Linux commit: df8a39defad4) + * For older kernels backported this commit, need to use renamed functions. + * This fix is specific to RedHat/CentOS kernels. + */ +#if (defined(RHEL_RELEASE_CODE) && \ + RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(6, 8) && \ + LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 34)) +#define vlan_tx_tag_get skb_vlan_tag_get +#define vlan_tx_tag_present skb_vlan_tag_present +#endif + #endif /* _KCOMPAT_H_ */ diff --git a/dpdk/lib/librte_eal/linuxapp/kni/kni_dev.h b/dpdk/lib/librte_eal/linuxapp/kni/kni_dev.h index 1c8fbadb..72385ab4 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/kni_dev.h +++ b/dpdk/lib/librte_eal/linuxapp/kni/kni_dev.h @@ -25,6 +25,11 @@ #ifndef _KNI_DEV_H_ #define _KNI_DEV_H_ +#ifdef pr_fmt +#undef pr_fmt +#endif +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + #include "compat.h" #include @@ -38,17 +43,14 @@ #include #include -#ifdef RTE_KNI_VHOST -#include -#endif - #include #define KNI_KTHREAD_RESCHEDULE_INTERVAL 5 /* us */ +#define MBUF_BURST_SZ 32 + /** * A structure describing the private information for a kni device. */ - struct kni_dev { /* kni list */ struct list_head list; @@ -56,7 +58,7 @@ struct kni_dev { struct net_device_stats stats; int status; uint16_t group_id; /* Group ID of a group of KNI devices */ - unsigned core_id; /* Core ID to bind */ + uint32_t core_id; /* Core ID to bind */ char name[RTE_KNI_NAMESIZE]; /* Network device name */ struct task_struct *pthread; @@ -90,66 +92,34 @@ struct kni_dev { /* response queue */ void *resp_q; - void * sync_kva; + void *sync_kva; void *sync_va; void *mbuf_kva; void *mbuf_va; /* mbuf size */ - unsigned mbuf_size; + uint32_t mbuf_size; /* synchro for request processing */ unsigned long synchro; -#ifdef RTE_KNI_VHOST - struct kni_vhost_queue* vhost_queue; - volatile enum { - BE_STOP = 0x1, - BE_START = 0x2, - BE_FINISH = 0x4, - }vq_status; -#endif + /* buffers */ + void *pa[MBUF_BURST_SZ]; + void *va[MBUF_BURST_SZ]; + void *alloc_pa[MBUF_BURST_SZ]; + void *alloc_va[MBUF_BURST_SZ]; }; -#define KNI_ERR(args...) printk(KERN_DEBUG "KNI: Error: " args) -#define KNI_PRINT(args...) printk(KERN_DEBUG "KNI: " args) -#ifdef RTE_KNI_KO_DEBUG - #define KNI_DBG(args...) printk(KERN_DEBUG "KNI: " args) -#else - #define KNI_DBG(args...) -#endif +void kni_net_rx(struct kni_dev *kni); +void kni_net_init(struct net_device *dev); +void kni_net_config_lo_mode(char *lo_str); +void kni_net_poll_resp(struct kni_dev *kni); +void kni_set_ethtool_ops(struct net_device *netdev); -#ifdef RTE_KNI_VHOST -unsigned int -kni_poll(struct file *file, struct socket *sock, poll_table * wait); -int kni_chk_vhost_rx(struct kni_dev *kni); -int kni_vhost_init(struct kni_dev *kni); -int kni_vhost_backend_release(struct kni_dev *kni); - -struct kni_vhost_queue { - struct sock sk; - struct socket *sock; - int vnet_hdr_sz; - struct kni_dev *kni; - int sockfd; - unsigned int flags; - struct sk_buff* cache; - struct rte_kni_fifo* fifo; -}; - -#endif - -#ifdef RTE_KNI_VHOST_DEBUG_RX - #define KNI_DBG_RX(args...) printk(KERN_DEBUG "KNI RX: " args) -#else - #define KNI_DBG_RX(args...) -#endif - -#ifdef RTE_KNI_VHOST_DEBUG_TX - #define KNI_DBG_TX(args...) printk(KERN_DEBUG "KNI TX: " args) -#else - #define KNI_DBG_TX(args...) -#endif +int ixgbe_kni_probe(struct pci_dev *pdev, struct net_device **lad_dev); +void ixgbe_kni_remove(struct pci_dev *pdev); +int igb_kni_probe(struct pci_dev *pdev, struct net_device **lad_dev); +void igb_kni_remove(struct pci_dev *pdev); #endif diff --git a/dpdk/lib/librte_eal/linuxapp/kni/kni_ethtool.c b/dpdk/lib/librte_eal/linuxapp/kni/kni_ethtool.c index 06b6d463..0c88589c 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/kni_ethtool.c +++ b/dpdk/lib/librte_eal/linuxapp/kni/kni_ethtool.c @@ -31,6 +31,7 @@ static int kni_check_if_running(struct net_device *dev) { struct kni_dev *priv = netdev_priv(dev); + if (priv->lad_dev) return 0; else @@ -41,6 +42,7 @@ static void kni_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) { struct kni_dev *priv = netdev_priv(dev); + priv->lad_dev->ethtool_ops->get_drvinfo(priv->lad_dev, info); } @@ -48,6 +50,7 @@ static int kni_get_settings(struct net_device *dev, struct ethtool_cmd *ecmd) { struct kni_dev *priv = netdev_priv(dev); + return priv->lad_dev->ethtool_ops->get_settings(priv->lad_dev, ecmd); } @@ -55,6 +58,7 @@ static int kni_set_settings(struct net_device *dev, struct ethtool_cmd *ecmd) { struct kni_dev *priv = netdev_priv(dev); + return priv->lad_dev->ethtool_ops->set_settings(priv->lad_dev, ecmd); } @@ -62,6 +66,7 @@ static void kni_get_wol(struct net_device *dev, struct ethtool_wolinfo *wol) { struct kni_dev *priv = netdev_priv(dev); + priv->lad_dev->ethtool_ops->get_wol(priv->lad_dev, wol); } @@ -69,6 +74,7 @@ static int kni_set_wol(struct net_device *dev, struct ethtool_wolinfo *wol) { struct kni_dev *priv = netdev_priv(dev); + return priv->lad_dev->ethtool_ops->set_wol(priv->lad_dev, wol); } @@ -76,6 +82,7 @@ static int kni_nway_reset(struct net_device *dev) { struct kni_dev *priv = netdev_priv(dev); + return priv->lad_dev->ethtool_ops->nway_reset(priv->lad_dev); } @@ -83,6 +90,7 @@ static int kni_get_eeprom_len(struct net_device *dev) { struct kni_dev *priv = netdev_priv(dev); + return priv->lad_dev->ethtool_ops->get_eeprom_len(priv->lad_dev); } @@ -91,6 +99,7 @@ kni_get_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom, u8 *bytes) { struct kni_dev *priv = netdev_priv(dev); + return priv->lad_dev->ethtool_ops->get_eeprom(priv->lad_dev, eeprom, bytes); } @@ -100,6 +109,7 @@ kni_set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom, u8 *bytes) { struct kni_dev *priv = netdev_priv(dev); + return priv->lad_dev->ethtool_ops->set_eeprom(priv->lad_dev, eeprom, bytes); } @@ -108,6 +118,7 @@ static void kni_get_ringparam(struct net_device *dev, struct ethtool_ringparam *ring) { struct kni_dev *priv = netdev_priv(dev); + priv->lad_dev->ethtool_ops->get_ringparam(priv->lad_dev, ring); } @@ -115,6 +126,7 @@ static int kni_set_ringparam(struct net_device *dev, struct ethtool_ringparam *ring) { struct kni_dev *priv = netdev_priv(dev); + return priv->lad_dev->ethtool_ops->set_ringparam(priv->lad_dev, ring); } @@ -122,6 +134,7 @@ static void kni_get_pauseparam(struct net_device *dev, struct ethtool_pauseparam *pause) { struct kni_dev *priv = netdev_priv(dev); + priv->lad_dev->ethtool_ops->get_pauseparam(priv->lad_dev, pause); } @@ -129,6 +142,7 @@ static int kni_set_pauseparam(struct net_device *dev, struct ethtool_pauseparam *pause) { struct kni_dev *priv = netdev_priv(dev); + return priv->lad_dev->ethtool_ops->set_pauseparam(priv->lad_dev, pause); } @@ -137,6 +151,7 @@ static u32 kni_get_msglevel(struct net_device *dev) { struct kni_dev *priv = netdev_priv(dev); + return priv->lad_dev->ethtool_ops->get_msglevel(priv->lad_dev); } @@ -144,6 +159,7 @@ static void kni_set_msglevel(struct net_device *dev, u32 data) { struct kni_dev *priv = netdev_priv(dev); + priv->lad_dev->ethtool_ops->set_msglevel(priv->lad_dev, data); } @@ -151,6 +167,7 @@ static int kni_get_regs_len(struct net_device *dev) { struct kni_dev *priv = netdev_priv(dev); + return priv->lad_dev->ethtool_ops->get_regs_len(priv->lad_dev); } @@ -158,6 +175,7 @@ static void kni_get_regs(struct net_device *dev, struct ethtool_regs *regs, void *p) { struct kni_dev *priv = netdev_priv(dev); + priv->lad_dev->ethtool_ops->get_regs(priv->lad_dev, regs, p); } @@ -165,6 +183,7 @@ static void kni_get_strings(struct net_device *dev, u32 stringset, u8 *data) { struct kni_dev *priv = netdev_priv(dev); + priv->lad_dev->ethtool_ops->get_strings(priv->lad_dev, stringset, data); } @@ -173,6 +192,7 @@ static int kni_get_sset_count(struct net_device *dev, int sset) { struct kni_dev *priv = netdev_priv(dev); + return priv->lad_dev->ethtool_ops->get_sset_count(priv->lad_dev, sset); } @@ -181,24 +201,25 @@ kni_get_ethtool_stats(struct net_device *dev, struct ethtool_stats *stats, u64 *data) { struct kni_dev *priv = netdev_priv(dev); + priv->lad_dev->ethtool_ops->get_ethtool_stats(priv->lad_dev, stats, data); } struct ethtool_ops kni_ethtool_ops = { - .begin = kni_check_if_running, + .begin = kni_check_if_running, .get_drvinfo = kni_get_drvinfo, .get_settings = kni_get_settings, .set_settings = kni_set_settings, .get_regs_len = kni_get_regs_len, - .get_regs = kni_get_regs, - .get_wol = kni_get_wol, - .set_wol = kni_set_wol, - .nway_reset = kni_nway_reset, - .get_link = ethtool_op_get_link, + .get_regs = kni_get_regs, + .get_wol = kni_get_wol, + .set_wol = kni_set_wol, + .nway_reset = kni_nway_reset, + .get_link = ethtool_op_get_link, .get_eeprom_len = kni_get_eeprom_len, - .get_eeprom = kni_get_eeprom, - .set_eeprom = kni_set_eeprom, + .get_eeprom = kni_get_eeprom, + .set_eeprom = kni_set_eeprom, .get_ringparam = kni_get_ringparam, .set_ringparam = kni_set_ringparam, .get_pauseparam = kni_get_pauseparam, @@ -207,7 +228,7 @@ struct ethtool_ops kni_ethtool_ops = { .set_msglevel = kni_set_msglevel, .get_strings = kni_get_strings, .get_sset_count = kni_get_sset_count, - .get_ethtool_stats = kni_get_ethtool_stats, + .get_ethtool_stats = kni_get_ethtool_stats, }; void diff --git a/dpdk/lib/librte_eal/linuxapp/kni/kni_fifo.h b/dpdk/lib/librte_eal/linuxapp/kni/kni_fifo.h index 3ea750e2..14f4141f 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/kni_fifo.h +++ b/dpdk/lib/librte_eal/linuxapp/kni/kni_fifo.h @@ -30,13 +30,13 @@ /** * Adds num elements into the fifo. Return the number actually written */ -static inline unsigned -kni_fifo_put(struct rte_kni_fifo *fifo, void **data, unsigned num) +static inline uint32_t +kni_fifo_put(struct rte_kni_fifo *fifo, void **data, uint32_t num) { - unsigned i = 0; - unsigned fifo_write = fifo->write; - unsigned fifo_read = fifo->read; - unsigned new_write = fifo_write; + uint32_t i = 0; + uint32_t fifo_write = fifo->write; + uint32_t fifo_read = fifo->read; + uint32_t new_write = fifo_write; for (i = 0; i < num; i++) { new_write = (new_write + 1) & (fifo->len - 1); @@ -54,12 +54,12 @@ kni_fifo_put(struct rte_kni_fifo *fifo, void **data, unsigned num) /** * Get up to num elements from the fifo. Return the number actully read */ -static inline unsigned -kni_fifo_get(struct rte_kni_fifo *fifo, void **data, unsigned num) +static inline uint32_t +kni_fifo_get(struct rte_kni_fifo *fifo, void **data, uint32_t num) { - unsigned i = 0; - unsigned new_read = fifo->read; - unsigned fifo_write = fifo->write; + uint32_t i = 0; + uint32_t new_read = fifo->read; + uint32_t fifo_write = fifo->write; for (i = 0; i < num; i++) { if (new_read == fifo_write) @@ -76,33 +76,19 @@ kni_fifo_get(struct rte_kni_fifo *fifo, void **data, unsigned num) /** * Get the num of elements in the fifo */ -static inline unsigned +static inline uint32_t kni_fifo_count(struct rte_kni_fifo *fifo) { - return (fifo->len + fifo->write - fifo->read) & ( fifo->len - 1); + return (fifo->len + fifo->write - fifo->read) & (fifo->len - 1); } /** * Get the num of available elements in the fifo */ -static inline unsigned +static inline uint32_t kni_fifo_free_count(struct rte_kni_fifo *fifo) { return (fifo->read - fifo->write - 1) & (fifo->len - 1); } -#ifdef RTE_KNI_VHOST -/** - * Initializes the kni fifo structure - */ -static inline void -kni_fifo_init(struct rte_kni_fifo *fifo, unsigned size) -{ - fifo->write = 0; - fifo->read = 0; - fifo->len = size; - fifo->elem_size = sizeof(void *); -} -#endif - #endif /* _KNI_FIFO_H_ */ diff --git a/dpdk/lib/librte_eal/linuxapp/kni/kni_misc.c b/dpdk/lib/librte_eal/linuxapp/kni/kni_misc.c index 59d15ca6..7590f1fd 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/kni_misc.c +++ b/dpdk/lib/librte_eal/linuxapp/kni/kni_misc.c @@ -30,6 +30,7 @@ #include #include #include +#include #include #include #include @@ -47,52 +48,15 @@ MODULE_DESCRIPTION("Kernel Module for managing kni devices"); #define KNI_MAX_DEVICES 32 -extern void kni_net_rx(struct kni_dev *kni); -extern void kni_net_init(struct net_device *dev); -extern void kni_net_config_lo_mode(char *lo_str); -extern void kni_net_poll_resp(struct kni_dev *kni); -extern void kni_set_ethtool_ops(struct net_device *netdev); - -extern int ixgbe_kni_probe(struct pci_dev *pdev, struct net_device **lad_dev); -extern void ixgbe_kni_remove(struct pci_dev *pdev); -extern int igb_kni_probe(struct pci_dev *pdev, struct net_device **lad_dev); -extern void igb_kni_remove(struct pci_dev *pdev); - -static int kni_open(struct inode *inode, struct file *file); -static int kni_release(struct inode *inode, struct file *file); -static int kni_ioctl(struct inode *inode, unsigned int ioctl_num, - unsigned long ioctl_param); -static int kni_compat_ioctl(struct inode *inode, unsigned int ioctl_num, - unsigned long ioctl_param); -static int kni_dev_remove(struct kni_dev *dev); - -static int __init kni_parse_kthread_mode(void); - -/* KNI processing for single kernel thread mode */ -static int kni_thread_single(void *unused); -/* KNI processing for multiple kernel thread mode */ -static int kni_thread_multiple(void *param); - -static struct file_operations kni_fops = { - .owner = THIS_MODULE, - .open = kni_open, - .release = kni_release, - .unlocked_ioctl = (void *)kni_ioctl, - .compat_ioctl = (void *)kni_compat_ioctl, -}; - -static struct miscdevice kni_misc = { - .minor = MISC_DYNAMIC_MINOR, - .name = KNI_DEVICE, - .fops = &kni_fops, -}; +extern const struct pci_device_id ixgbe_pci_tbl[]; +extern const struct pci_device_id igb_pci_tbl[]; /* loopback mode */ -static char *lo_mode = NULL; +static char *lo_mode; /* Kernel thread mode */ -static char *kthread_mode = NULL; -static unsigned multiple_kthread_on = 0; +static char *kthread_mode; +static uint32_t multiple_kthread_on; #define KNI_DEV_IN_USE_BIT_NUM 0 /* Bit number for device in use */ @@ -100,20 +64,24 @@ static int kni_net_id; struct kni_net { unsigned long device_in_use; /* device in use flag */ + struct mutex kni_kthread_lock; struct task_struct *kni_kthread; struct rw_semaphore kni_list_lock; struct list_head kni_list_head; }; -static int __net_init kni_init_net(struct net *net) +static int __net_init +kni_init_net(struct net *net) { #ifdef HAVE_SIMPLIFIED_PERNET_OPERATIONS struct kni_net *knet = net_generic(net, kni_net_id); + + memset(knet, 0, sizeof(*knet)); #else struct kni_net *knet; int ret; - knet = kmalloc(sizeof(struct kni_net), GFP_KERNEL); + knet = kzalloc(sizeof(struct kni_net), GFP_KERNEL); if (!knet) { ret = -ENOMEM; return ret; @@ -123,6 +91,8 @@ static int __net_init kni_init_net(struct net *net) /* Clear the bit of device in use */ clear_bit(KNI_DEV_IN_USE_BIT_NUM, &knet->device_in_use); + mutex_init(&knet->kni_kthread_lock); + init_rwsem(&knet->kni_list_lock); INIT_LIST_HEAD(&knet->kni_list_head); @@ -137,11 +107,15 @@ static int __net_init kni_init_net(struct net *net) #endif } -static void __net_exit kni_exit_net(struct net *net) +static void __net_exit +kni_exit_net(struct net *net) { -#ifndef HAVE_SIMPLIFIED_PERNET_OPERATIONS - struct kni_net *knet = net_generic(net, kni_net_id); + struct kni_net *knet __maybe_unused; + knet = net_generic(net, kni_net_id); + mutex_destroy(&knet->kni_kthread_lock); + +#ifndef HAVE_SIMPLIFIED_PERNET_OPERATIONS kfree(knet); #endif } @@ -155,72 +129,48 @@ static struct pernet_operations kni_net_ops = { #endif }; -static int __init -kni_init(void) +static int +kni_thread_single(void *data) { - int rc; + struct kni_net *knet = data; + int j; + struct kni_dev *dev; - KNI_PRINT("######## DPDK kni module loading ########\n"); - - if (kni_parse_kthread_mode() < 0) { - KNI_ERR("Invalid parameter for kthread_mode\n"); - return -EINVAL; - } - -#ifdef HAVE_SIMPLIFIED_PERNET_OPERATIONS - rc = register_pernet_subsys(&kni_net_ops); -#else - rc = register_pernet_gen_subsys(&kni_net_id, &kni_net_ops); + while (!kthread_should_stop()) { + down_read(&knet->kni_list_lock); + for (j = 0; j < KNI_RX_LOOP_NUM; j++) { + list_for_each_entry(dev, &knet->kni_list_head, list) { + kni_net_rx(dev); + kni_net_poll_resp(dev); + } + } + up_read(&knet->kni_list_lock); +#ifdef RTE_KNI_PREEMPT_DEFAULT + /* reschedule out for a while */ + schedule_timeout_interruptible( + usecs_to_jiffies(KNI_KTHREAD_RESCHEDULE_INTERVAL)); #endif - if (rc) - return -EPERM; - - rc = misc_register(&kni_misc); - if (rc != 0) { - KNI_ERR("Misc registration failed\n"); - goto out; } - /* Configure the lo mode according to the input parameter */ - kni_net_config_lo_mode(lo_mode); - - KNI_PRINT("######## DPDK kni module loaded ########\n"); - return 0; - -out: -#ifdef HAVE_SIMPLIFIED_PERNET_OPERATIONS - unregister_pernet_subsys(&kni_net_ops); -#else - register_pernet_gen_subsys(&kni_net_id, &kni_net_ops); -#endif - return rc; } -static void __exit -kni_exit(void) +static int +kni_thread_multiple(void *param) { - misc_deregister(&kni_misc); -#ifdef HAVE_SIMPLIFIED_PERNET_OPERATIONS - unregister_pernet_subsys(&kni_net_ops); -#else - register_pernet_gen_subsys(&kni_net_id, &kni_net_ops); + int j; + struct kni_dev *dev = param; + + while (!kthread_should_stop()) { + for (j = 0; j < KNI_RX_LOOP_NUM; j++) { + kni_net_rx(dev); + kni_net_poll_resp(dev); + } +#ifdef RTE_KNI_PREEMPT_DEFAULT + schedule_timeout_interruptible( + usecs_to_jiffies(KNI_KTHREAD_RESCHEDULE_INTERVAL)); #endif - KNI_PRINT("####### DPDK kni module unloaded #######\n"); -} - -static int __init -kni_parse_kthread_mode(void) -{ - if (!kthread_mode) - return 0; - - if (strcmp(kthread_mode, "single") == 0) - return 0; - else if (strcmp(kthread_mode, "multiple") == 0) - multiple_kthread_on = 1; - else - return -1; + } return 0; } @@ -235,21 +185,31 @@ kni_open(struct inode *inode, struct file *file) if (test_and_set_bit(KNI_DEV_IN_USE_BIT_NUM, &knet->device_in_use)) return -EBUSY; - /* Create kernel thread for single mode */ - if (multiple_kthread_on == 0) { - KNI_PRINT("Single kernel thread for all KNI devices\n"); - /* Create kernel thread for RX */ - knet->kni_kthread = kthread_run(kni_thread_single, (void *)knet, - "kni_single"); - if (IS_ERR(knet->kni_kthread)) { - KNI_ERR("Unable to create kernel threaed\n"); - return PTR_ERR(knet->kni_kthread); - } - } else - KNI_PRINT("Multiple kernel thread mode enabled\n"); - file->private_data = get_net(net); - KNI_PRINT("/dev/kni opened\n"); + pr_debug("/dev/kni opened\n"); + + return 0; +} + +static int +kni_dev_remove(struct kni_dev *dev) +{ + if (!dev) + return -ENODEV; + +#ifdef RTE_KNI_KMOD_ETHTOOL + if (dev->pci_dev) { + if (pci_match_id(ixgbe_pci_tbl, dev->pci_dev)) + ixgbe_kni_remove(dev->pci_dev); + else if (pci_match_id(igb_pci_tbl, dev->pci_dev)) + igb_kni_remove(dev->pci_dev); + } +#endif + + if (dev->net_dev) { + unregister_netdev(dev->net_dev); + free_netdev(dev->net_dev); + } return 0; } @@ -263,9 +223,13 @@ kni_release(struct inode *inode, struct file *file) /* Stop kernel thread for single mode */ if (multiple_kthread_on == 0) { + mutex_lock(&knet->kni_kthread_lock); /* Stop kernel thread */ - kthread_stop(knet->kni_kthread); - knet->kni_kthread = NULL; + if (knet->kni_kthread != NULL) { + kthread_stop(knet->kni_kthread); + knet->kni_kthread = NULL; + } + mutex_unlock(&knet->kni_kthread_lock); } down_write(&knet->kni_list_lock); @@ -276,9 +240,6 @@ kni_release(struct inode *inode, struct file *file) dev->pthread = NULL; } -#ifdef RTE_KNI_VHOST - kni_vhost_backend_release(dev); -#endif kni_dev_remove(dev); list_del(&dev->list); } @@ -288,88 +249,7 @@ kni_release(struct inode *inode, struct file *file) clear_bit(KNI_DEV_IN_USE_BIT_NUM, &knet->device_in_use); put_net(net); - KNI_PRINT("/dev/kni closed\n"); - - return 0; -} - -static int -kni_thread_single(void *data) -{ - struct kni_net *knet = data; - int j; - struct kni_dev *dev; - - while (!kthread_should_stop()) { - down_read(&knet->kni_list_lock); - for (j = 0; j < KNI_RX_LOOP_NUM; j++) { - list_for_each_entry(dev, &knet->kni_list_head, list) { -#ifdef RTE_KNI_VHOST - kni_chk_vhost_rx(dev); -#else - kni_net_rx(dev); -#endif - kni_net_poll_resp(dev); - } - } - up_read(&knet->kni_list_lock); -#ifdef RTE_KNI_PREEMPT_DEFAULT - /* reschedule out for a while */ - schedule_timeout_interruptible(usecs_to_jiffies( \ - KNI_KTHREAD_RESCHEDULE_INTERVAL)); -#endif - } - - return 0; -} - -static int -kni_thread_multiple(void *param) -{ - int j; - struct kni_dev *dev = (struct kni_dev *)param; - - while (!kthread_should_stop()) { - for (j = 0; j < KNI_RX_LOOP_NUM; j++) { -#ifdef RTE_KNI_VHOST - kni_chk_vhost_rx(dev); -#else - kni_net_rx(dev); -#endif - kni_net_poll_resp(dev); - } -#ifdef RTE_KNI_PREEMPT_DEFAULT - schedule_timeout_interruptible(usecs_to_jiffies( \ - KNI_KTHREAD_RESCHEDULE_INTERVAL)); -#endif - } - - return 0; -} - -static int -kni_dev_remove(struct kni_dev *dev) -{ - if (!dev) - return -ENODEV; - - switch (dev->device_id) { - #define RTE_PCI_DEV_ID_DECL_IGB(vend, dev) case (dev): - #include - igb_kni_remove(dev->pci_dev); - break; - #define RTE_PCI_DEV_ID_DECL_IXGBE(vend, dev) case (dev): - #include - ixgbe_kni_remove(dev->pci_dev); - break; - default: - break; - } - - if (dev->net_dev) { - unregister_netdev(dev->net_dev); - free_netdev(dev->net_dev); - } + pr_debug("/dev/kni closed\n"); return 0; } @@ -382,7 +262,7 @@ kni_check_param(struct kni_dev *kni, struct rte_kni_device_info *dev) /* Check if network name has been used */ if (!strncmp(kni->name, dev->name, RTE_KNI_NAMESIZE)) { - KNI_ERR("KNI name %s duplicated\n", dev->name); + pr_err("KNI name %s duplicated\n", dev->name); return -1; } @@ -390,19 +270,62 @@ kni_check_param(struct kni_dev *kni, struct rte_kni_device_info *dev) } static int -kni_ioctl_create(struct net *net, - unsigned int ioctl_num, unsigned long ioctl_param) +kni_run_thread(struct kni_net *knet, struct kni_dev *kni, uint8_t force_bind) +{ + /** + * Create a new kernel thread for multiple mode, set its core affinity, + * and finally wake it up. + */ + if (multiple_kthread_on) { + kni->pthread = kthread_create(kni_thread_multiple, + (void *)kni, "kni_%s", kni->name); + if (IS_ERR(kni->pthread)) { + kni_dev_remove(kni); + return -ECANCELED; + } + + if (force_bind) + kthread_bind(kni->pthread, kni->core_id); + wake_up_process(kni->pthread); + } else { + mutex_lock(&knet->kni_kthread_lock); + + if (knet->kni_kthread == NULL) { + knet->kni_kthread = kthread_create(kni_thread_single, + (void *)knet, "kni_single"); + if (IS_ERR(knet->kni_kthread)) { + mutex_unlock(&knet->kni_kthread_lock); + kni_dev_remove(kni); + return -ECANCELED; + } + + if (force_bind) + kthread_bind(knet->kni_kthread, kni->core_id); + wake_up_process(knet->kni_kthread); + } + + mutex_unlock(&knet->kni_kthread_lock); + } + + return 0; +} + +static int +kni_ioctl_create(struct net *net, uint32_t ioctl_num, + unsigned long ioctl_param) { struct kni_net *knet = net_generic(net, kni_net_id); int ret; struct rte_kni_device_info dev_info; - struct pci_dev *pci = NULL; - struct pci_dev *found_pci = NULL; struct net_device *net_dev = NULL; - struct net_device *lad_dev = NULL; struct kni_dev *kni, *dev, *n; +#ifdef RTE_KNI_KMOD_ETHTOOL + struct pci_dev *found_pci = NULL; + struct net_device *lad_dev = NULL; + struct pci_dev *pci = NULL; +#endif - printk(KERN_INFO "KNI: Creating kni...\n"); + pr_info("Creating kni...\n"); /* Check the buffer size, to avoid warning */ if (_IOC_SIZE(ioctl_num) > sizeof(dev_info)) return -EINVAL; @@ -410,17 +333,21 @@ kni_ioctl_create(struct net *net, /* Copy kni info from user space */ ret = copy_from_user(&dev_info, (void *)ioctl_param, sizeof(dev_info)); if (ret) { - KNI_ERR("copy_from_user in kni_ioctl_create"); + pr_err("copy_from_user in kni_ioctl_create"); return -EIO; } + /* Check if name is zero-ended */ + if (strnlen(dev_info.name, sizeof(dev_info.name)) == sizeof(dev_info.name)) { + pr_err("kni.name not zero-terminated"); + return -EINVAL; + } + /** - * Check if the cpu core id is valid for binding, - * for multiple kernel thread mode. + * Check if the cpu core id is valid for binding. */ - if (multiple_kthread_on && dev_info.force_bind && - !cpu_online(dev_info.core_id)) { - KNI_ERR("cpu %u is not online\n", dev_info.core_id); + if (dev_info.force_bind && !cpu_online(dev_info.core_id)) { + pr_err("cpu %u is not online\n", dev_info.core_id); return -EINVAL; } @@ -435,12 +362,12 @@ kni_ioctl_create(struct net *net, up_read(&knet->kni_list_lock); net_dev = alloc_netdev(sizeof(struct kni_dev), dev_info.name, -#ifdef NET_NAME_UNKNOWN - NET_NAME_UNKNOWN, +#ifdef NET_NAME_USER + NET_NAME_USER, #endif kni_net_init); if (net_dev == NULL) { - KNI_ERR("error allocating device \"%s\"\n", dev_info.name); + pr_err("error allocating device \"%s\"\n", dev_info.name); return -EBUSY; } @@ -464,44 +391,34 @@ kni_ioctl_create(struct net *net, kni->sync_va = dev_info.sync_va; kni->sync_kva = phys_to_virt(dev_info.sync_phys); - kni->mbuf_kva = phys_to_virt(dev_info.mbuf_phys); - kni->mbuf_va = dev_info.mbuf_va; - -#ifdef RTE_KNI_VHOST - kni->vhost_queue = NULL; - kni->vq_status = BE_STOP; -#endif kni->mbuf_size = dev_info.mbuf_size; - KNI_PRINT("tx_phys: 0x%016llx, tx_q addr: 0x%p\n", + pr_debug("tx_phys: 0x%016llx, tx_q addr: 0x%p\n", (unsigned long long) dev_info.tx_phys, kni->tx_q); - KNI_PRINT("rx_phys: 0x%016llx, rx_q addr: 0x%p\n", + pr_debug("rx_phys: 0x%016llx, rx_q addr: 0x%p\n", (unsigned long long) dev_info.rx_phys, kni->rx_q); - KNI_PRINT("alloc_phys: 0x%016llx, alloc_q addr: 0x%p\n", + pr_debug("alloc_phys: 0x%016llx, alloc_q addr: 0x%p\n", (unsigned long long) dev_info.alloc_phys, kni->alloc_q); - KNI_PRINT("free_phys: 0x%016llx, free_q addr: 0x%p\n", + pr_debug("free_phys: 0x%016llx, free_q addr: 0x%p\n", (unsigned long long) dev_info.free_phys, kni->free_q); - KNI_PRINT("req_phys: 0x%016llx, req_q addr: 0x%p\n", + pr_debug("req_phys: 0x%016llx, req_q addr: 0x%p\n", (unsigned long long) dev_info.req_phys, kni->req_q); - KNI_PRINT("resp_phys: 0x%016llx, resp_q addr: 0x%p\n", + pr_debug("resp_phys: 0x%016llx, resp_q addr: 0x%p\n", (unsigned long long) dev_info.resp_phys, kni->resp_q); - KNI_PRINT("mbuf_phys: 0x%016llx, mbuf_kva: 0x%p\n", - (unsigned long long) dev_info.mbuf_phys, kni->mbuf_kva); - KNI_PRINT("mbuf_va: 0x%p\n", dev_info.mbuf_va); - KNI_PRINT("mbuf_size: %u\n", kni->mbuf_size); + pr_debug("mbuf_size: %u\n", kni->mbuf_size); - KNI_DBG("PCI: %02x:%02x.%02x %04x:%04x\n", + pr_debug("PCI: %02x:%02x.%02x %04x:%04x\n", dev_info.bus, dev_info.devid, dev_info.function, dev_info.vendor_id, dev_info.device_id); - +#ifdef RTE_KNI_KMOD_ETHTOOL pci = pci_get_device(dev_info.vendor_id, dev_info.device_id, NULL); /* Support Ethtool */ while (pci) { - KNI_PRINT("pci_bus: %02x:%02x:%02x \n", + pr_debug("pci_bus: %02x:%02x:%02x\n", pci->bus->number, PCI_SLOT(pci->devfn), PCI_FUNC(pci->devfn)); @@ -510,28 +427,21 @@ kni_ioctl_create(struct net *net, (PCI_SLOT(pci->devfn) == dev_info.devid) && (PCI_FUNC(pci->devfn) == dev_info.function)) { found_pci = pci; - switch (dev_info.device_id) { - #define RTE_PCI_DEV_ID_DECL_IGB(vend, dev) case (dev): - #include - ret = igb_kni_probe(found_pci, &lad_dev); - break; - #define RTE_PCI_DEV_ID_DECL_IXGBE(vend, dev) \ - case (dev): - #include - ret = ixgbe_kni_probe(found_pci, &lad_dev); - break; - default: - ret = -1; - break; - } - KNI_DBG("PCI found: pci=0x%p, lad_dev=0x%p\n", + if (pci_match_id(ixgbe_pci_tbl, found_pci)) + ret = ixgbe_kni_probe(found_pci, &lad_dev); + else if (pci_match_id(igb_pci_tbl, found_pci)) + ret = igb_kni_probe(found_pci, &lad_dev); + else + ret = -1; + + pr_debug("PCI found: pci=0x%p, lad_dev=0x%p\n", pci, lad_dev); if (ret == 0) { kni->lad_dev = lad_dev; kni_set_ethtool_ops(kni->net_dev); } else { - KNI_ERR("Device not supported by ethtool"); + pr_err("Device not supported by ethtool"); kni->lad_dev = NULL; } @@ -544,9 +454,10 @@ kni_ioctl_create(struct net *net, } if (pci) pci_dev_put(pci); +#endif if (kni->lad_dev) - memcpy(net_dev->dev_addr, kni->lad_dev->dev_addr, ETH_ALEN); + ether_addr_copy(net_dev->dev_addr, kni->lad_dev->dev_addr); else /* * Generate random mac address. eth_random_addr() is the newer @@ -556,32 +467,17 @@ kni_ioctl_create(struct net *net, ret = register_netdev(net_dev); if (ret) { - KNI_ERR("error %i registering device \"%s\"\n", + pr_err("error %i registering device \"%s\"\n", ret, dev_info.name); + kni->net_dev = NULL; kni_dev_remove(kni); + free_netdev(net_dev); return -ENODEV; } -#ifdef RTE_KNI_VHOST - kni_vhost_init(kni); -#endif - - /** - * Create a new kernel thread for multiple mode, set its core affinity, - * and finally wake it up. - */ - if (multiple_kthread_on) { - kni->pthread = kthread_create(kni_thread_multiple, - (void *)kni, - "kni_%s", kni->name); - if (IS_ERR(kni->pthread)) { - kni_dev_remove(kni); - return -ECANCELED; - } - if (dev_info.force_bind) - kthread_bind(kni->pthread, kni->core_id); - wake_up_process(kni->pthread); - } + ret = kni_run_thread(knet, kni, dev_info.force_bind); + if (ret != 0) + return ret; down_write(&knet->kni_list_lock); list_add(&kni->list, &knet->kni_list_head); @@ -591,8 +487,8 @@ kni_ioctl_create(struct net *net, } static int -kni_ioctl_release(struct net *net, - unsigned int ioctl_num, unsigned long ioctl_param) +kni_ioctl_release(struct net *net, uint32_t ioctl_num, + unsigned long ioctl_param) { struct kni_net *knet = net_generic(net, kni_net_id); int ret = -EINVAL; @@ -600,11 +496,11 @@ kni_ioctl_release(struct net *net, struct rte_kni_device_info dev_info; if (_IOC_SIZE(ioctl_num) > sizeof(dev_info)) - return -EINVAL; + return -EINVAL; ret = copy_from_user(&dev_info, (void *)ioctl_param, sizeof(dev_info)); if (ret) { - KNI_ERR("copy_from_user in kni_ioctl_release"); + pr_err("copy_from_user in kni_ioctl_release"); return -EIO; } @@ -622,30 +518,25 @@ kni_ioctl_release(struct net *net, dev->pthread = NULL; } -#ifdef RTE_KNI_VHOST - kni_vhost_backend_release(dev); -#endif kni_dev_remove(dev); list_del(&dev->list); ret = 0; break; } up_write(&knet->kni_list_lock); - printk(KERN_INFO "KNI: %s release kni named %s\n", + pr_info("%s release kni named %s\n", (ret == 0 ? "Successfully" : "Unsuccessfully"), dev_info.name); return ret; } static int -kni_ioctl(struct inode *inode, - unsigned int ioctl_num, - unsigned long ioctl_param) +kni_ioctl(struct inode *inode, uint32_t ioctl_num, unsigned long ioctl_param) { int ret = -EINVAL; struct net *net = current->nsproxy->net_ns; - KNI_DBG("IOCTL num=0x%0x param=0x%0lx\n", ioctl_num, ioctl_param); + pr_debug("IOCTL num=0x%0x param=0x%0lx\n", ioctl_num, ioctl_param); /* * Switch according to the ioctl called @@ -661,7 +552,7 @@ kni_ioctl(struct inode *inode, ret = kni_ioctl_release(net, ioctl_num, ioctl_param); break; default: - KNI_DBG("IOCTL default\n"); + pr_debug("IOCTL default\n"); break; } @@ -669,16 +560,99 @@ kni_ioctl(struct inode *inode, } static int -kni_compat_ioctl(struct inode *inode, - unsigned int ioctl_num, +kni_compat_ioctl(struct inode *inode, uint32_t ioctl_num, unsigned long ioctl_param) { /* 32 bits app on 64 bits OS to be supported later */ - KNI_PRINT("Not implemented.\n"); + pr_debug("Not implemented.\n"); return -EINVAL; } +static const struct file_operations kni_fops = { + .owner = THIS_MODULE, + .open = kni_open, + .release = kni_release, + .unlocked_ioctl = (void *)kni_ioctl, + .compat_ioctl = (void *)kni_compat_ioctl, +}; + +static struct miscdevice kni_misc = { + .minor = MISC_DYNAMIC_MINOR, + .name = KNI_DEVICE, + .fops = &kni_fops, +}; + +static int __init +kni_parse_kthread_mode(void) +{ + if (!kthread_mode) + return 0; + + if (strcmp(kthread_mode, "single") == 0) + return 0; + else if (strcmp(kthread_mode, "multiple") == 0) + multiple_kthread_on = 1; + else + return -1; + + return 0; +} + +static int __init +kni_init(void) +{ + int rc; + + if (kni_parse_kthread_mode() < 0) { + pr_err("Invalid parameter for kthread_mode\n"); + return -EINVAL; + } + + if (multiple_kthread_on == 0) + pr_debug("Single kernel thread for all KNI devices\n"); + else + pr_debug("Multiple kernel thread mode enabled\n"); + +#ifdef HAVE_SIMPLIFIED_PERNET_OPERATIONS + rc = register_pernet_subsys(&kni_net_ops); +#else + rc = register_pernet_gen_subsys(&kni_net_id, &kni_net_ops); +#endif + if (rc) + return -EPERM; + + rc = misc_register(&kni_misc); + if (rc != 0) { + pr_err("Misc registration failed\n"); + goto out; + } + + /* Configure the lo mode according to the input parameter */ + kni_net_config_lo_mode(lo_mode); + + return 0; + +out: +#ifdef HAVE_SIMPLIFIED_PERNET_OPERATIONS + unregister_pernet_subsys(&kni_net_ops); +#else + unregister_pernet_gen_subsys(kni_net_id, &kni_net_ops); +#endif + return rc; +} + +static void __exit +kni_exit(void) +{ + misc_deregister(&kni_misc); +#ifdef HAVE_SIMPLIFIED_PERNET_OPERATIONS + unregister_pernet_subsys(&kni_net_ops); +#else + unregister_pernet_gen_subsys(kni_net_id, &kni_net_ops); +#endif +} + module_init(kni_init); module_exit(kni_exit); diff --git a/dpdk/lib/librte_eal/linuxapp/kni/kni_net.c b/dpdk/lib/librte_eal/linuxapp/kni/kni_net.c index fc82193a..db9f4898 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/kni_net.c +++ b/dpdk/lib/librte_eal/linuxapp/kni/kni_net.c @@ -44,23 +44,103 @@ #define WD_TIMEOUT 5 /*jiffies */ -#define MBUF_BURST_SZ 32 - #define KNI_WAIT_RESPONSE_TIMEOUT 300 /* 3 seconds */ /* typedef for rx function */ typedef void (*kni_net_rx_t)(struct kni_dev *kni); -static int kni_net_tx(struct sk_buff *skb, struct net_device *dev); static void kni_net_rx_normal(struct kni_dev *kni); -static void kni_net_rx_lo_fifo(struct kni_dev *kni); -static void kni_net_rx_lo_fifo_skb(struct kni_dev *kni); -static int kni_net_process_request(struct kni_dev *kni, - struct rte_kni_request *req); /* kni rx function pointer, with default to normal rx */ static kni_net_rx_t kni_net_rx_func = kni_net_rx_normal; +/* physical address to kernel virtual address */ +static void * +pa2kva(void *pa) +{ + return phys_to_virt((unsigned long)pa); +} + +/* physical address to virtual address */ +static void * +pa2va(void *pa, struct rte_kni_mbuf *m) +{ + void *va; + + va = (void *)((unsigned long)pa + + (unsigned long)m->buf_addr - + (unsigned long)m->buf_physaddr); + return va; +} + +/* mbuf data kernel virtual address from mbuf kernel virtual address */ +static void * +kva2data_kva(struct rte_kni_mbuf *m) +{ + return phys_to_virt(m->buf_physaddr + m->data_off); +} + +/* virtual address to physical address */ +static void * +va2pa(void *va, struct rte_kni_mbuf *m) +{ + void *pa; + + pa = (void *)((unsigned long)va - + ((unsigned long)m->buf_addr - + (unsigned long)m->buf_physaddr)); + return pa; +} + +/* + * It can be called to process the request. + */ +static int +kni_net_process_request(struct kni_dev *kni, struct rte_kni_request *req) +{ + int ret = -1; + void *resp_va; + uint32_t num; + int ret_val; + + if (!kni || !req) { + pr_err("No kni instance or request\n"); + return -EINVAL; + } + + mutex_lock(&kni->sync_lock); + + /* Construct data */ + memcpy(kni->sync_kva, req, sizeof(struct rte_kni_request)); + num = kni_fifo_put(kni->req_q, &kni->sync_va, 1); + if (num < 1) { + pr_err("Cannot send to req_q\n"); + ret = -EBUSY; + goto fail; + } + + ret_val = wait_event_interruptible_timeout(kni->wq, + kni_fifo_count(kni->resp_q), 3 * HZ); + if (signal_pending(current) || ret_val <= 0) { + ret = -ETIME; + goto fail; + } + num = kni_fifo_get(kni->resp_q, (void **)&resp_va, 1); + if (num != 1 || resp_va != kni->sync_va) { + /* This should never happen */ + pr_err("No data in resp_q\n"); + ret = -ENODATA; + goto fail; + } + + memcpy(req, kni->sync_kva, sizeof(struct rte_kni_request)); + ret = 0; + +fail: + mutex_unlock(&kni->sync_lock); + return ret; +} + /* * Open and close */ @@ -115,318 +195,18 @@ kni_net_config(struct net_device *dev, struct ifmap *map) return 0; } -/* - * RX: normal working mode - */ -static void -kni_net_rx_normal(struct kni_dev *kni) -{ - unsigned ret; - uint32_t len; - unsigned i, num_rx, num_fq; - struct rte_kni_mbuf *kva; - struct rte_kni_mbuf *va[MBUF_BURST_SZ]; - void * data_kva; - - struct sk_buff *skb; - struct net_device *dev = kni->net_dev; - - /* Get the number of free entries in free_q */ - num_fq = kni_fifo_free_count(kni->free_q); - if (num_fq == 0) { - /* No room on the free_q, bail out */ - return; - } - - /* Calculate the number of entries to dequeue from rx_q */ - num_rx = min(num_fq, (unsigned)MBUF_BURST_SZ); - - /* Burst dequeue from rx_q */ - num_rx = kni_fifo_get(kni->rx_q, (void **)va, num_rx); - if (num_rx == 0) - return; - - /* Transfer received packets to netif */ - for (i = 0; i < num_rx; i++) { - kva = (void *)va[i] - kni->mbuf_va + kni->mbuf_kva; - len = kva->pkt_len; - - data_kva = kva->buf_addr + kva->data_off - kni->mbuf_va - + kni->mbuf_kva; - - skb = dev_alloc_skb(len + 2); - if (!skb) { - KNI_ERR("Out of mem, dropping pkts\n"); - /* Update statistics */ - kni->stats.rx_dropped++; - continue; - } - - /* Align IP on 16B boundary */ - skb_reserve(skb, 2); - - if (kva->nb_segs == 1) { - memcpy(skb_put(skb, len), data_kva, len); - } else { - int nb_segs; - int kva_nb_segs = kva->nb_segs; - - for (nb_segs = 0; nb_segs < kva_nb_segs; nb_segs++) { - memcpy(skb_put(skb, kva->data_len), - data_kva, kva->data_len); - - if (!kva->next) - break; - - kva = kva->next - kni->mbuf_va + kni->mbuf_kva; - data_kva = kva->buf_addr + kva->data_off - - kni->mbuf_va + kni->mbuf_kva; - } - } - - skb->dev = dev; - skb->protocol = eth_type_trans(skb, dev); - skb->ip_summed = CHECKSUM_UNNECESSARY; - - /* Call netif interface */ - netif_rx_ni(skb); - - /* Update statistics */ - kni->stats.rx_bytes += len; - kni->stats.rx_packets++; - } - - /* Burst enqueue mbufs into free_q */ - ret = kni_fifo_put(kni->free_q, (void **)va, num_rx); - if (ret != num_rx) - /* Failing should not happen */ - KNI_ERR("Fail to enqueue entries into free_q\n"); -} - -/* - * RX: loopback with enqueue/dequeue fifos. - */ -static void -kni_net_rx_lo_fifo(struct kni_dev *kni) -{ - unsigned ret; - uint32_t len; - unsigned i, num, num_rq, num_tq, num_aq, num_fq; - struct rte_kni_mbuf *kva; - struct rte_kni_mbuf *va[MBUF_BURST_SZ]; - void * data_kva; - - struct rte_kni_mbuf *alloc_kva; - struct rte_kni_mbuf *alloc_va[MBUF_BURST_SZ]; - void *alloc_data_kva; - - /* Get the number of entries in rx_q */ - num_rq = kni_fifo_count(kni->rx_q); - - /* Get the number of free entrie in tx_q */ - num_tq = kni_fifo_free_count(kni->tx_q); - - /* Get the number of entries in alloc_q */ - num_aq = kni_fifo_count(kni->alloc_q); - - /* Get the number of free entries in free_q */ - num_fq = kni_fifo_free_count(kni->free_q); - - /* Calculate the number of entries to be dequeued from rx_q */ - num = min(num_rq, num_tq); - num = min(num, num_aq); - num = min(num, num_fq); - num = min(num, (unsigned)MBUF_BURST_SZ); - - /* Return if no entry to dequeue from rx_q */ - if (num == 0) - return; - - /* Burst dequeue from rx_q */ - ret = kni_fifo_get(kni->rx_q, (void **)va, num); - if (ret == 0) - return; /* Failing should not happen */ - - /* Dequeue entries from alloc_q */ - ret = kni_fifo_get(kni->alloc_q, (void **)alloc_va, num); - if (ret) { - num = ret; - /* Copy mbufs */ - for (i = 0; i < num; i++) { - kva = (void *)va[i] - kni->mbuf_va + kni->mbuf_kva; - len = kva->pkt_len; - data_kva = kva->buf_addr + kva->data_off - - kni->mbuf_va + kni->mbuf_kva; - - alloc_kva = (void *)alloc_va[i] - kni->mbuf_va + - kni->mbuf_kva; - alloc_data_kva = alloc_kva->buf_addr + - alloc_kva->data_off - kni->mbuf_va + - kni->mbuf_kva; - memcpy(alloc_data_kva, data_kva, len); - alloc_kva->pkt_len = len; - alloc_kva->data_len = len; - - kni->stats.tx_bytes += len; - kni->stats.rx_bytes += len; - } - - /* Burst enqueue mbufs into tx_q */ - ret = kni_fifo_put(kni->tx_q, (void **)alloc_va, num); - if (ret != num) - /* Failing should not happen */ - KNI_ERR("Fail to enqueue mbufs into tx_q\n"); - } - - /* Burst enqueue mbufs into free_q */ - ret = kni_fifo_put(kni->free_q, (void **)va, num); - if (ret != num) - /* Failing should not happen */ - KNI_ERR("Fail to enqueue mbufs into free_q\n"); - - /** - * Update statistic, and enqueue/dequeue failure is impossible, - * as all queues are checked at first. - */ - kni->stats.tx_packets += num; - kni->stats.rx_packets += num; -} - -/* - * RX: loopback with enqueue/dequeue fifos and sk buffer copies. - */ -static void -kni_net_rx_lo_fifo_skb(struct kni_dev *kni) -{ - unsigned ret; - uint32_t len; - unsigned i, num_rq, num_fq, num; - struct rte_kni_mbuf *kva; - struct rte_kni_mbuf *va[MBUF_BURST_SZ]; - void * data_kva; - - struct sk_buff *skb; - struct net_device *dev = kni->net_dev; - - /* Get the number of entries in rx_q */ - num_rq = kni_fifo_count(kni->rx_q); - - /* Get the number of free entries in free_q */ - num_fq = kni_fifo_free_count(kni->free_q); - - /* Calculate the number of entries to dequeue from rx_q */ - num = min(num_rq, num_fq); - num = min(num, (unsigned)MBUF_BURST_SZ); - - /* Return if no entry to dequeue from rx_q */ - if (num == 0) - return; - - /* Burst dequeue mbufs from rx_q */ - ret = kni_fifo_get(kni->rx_q, (void **)va, num); - if (ret == 0) - return; - - /* Copy mbufs to sk buffer and then call tx interface */ - for (i = 0; i < num; i++) { - kva = (void *)va[i] - kni->mbuf_va + kni->mbuf_kva; - len = kva->pkt_len; - data_kva = kva->buf_addr + kva->data_off - kni->mbuf_va + - kni->mbuf_kva; - - skb = dev_alloc_skb(len + 2); - if (skb == NULL) - KNI_ERR("Out of mem, dropping pkts\n"); - else { - /* Align IP on 16B boundary */ - skb_reserve(skb, 2); - memcpy(skb_put(skb, len), data_kva, len); - skb->dev = dev; - skb->ip_summed = CHECKSUM_UNNECESSARY; - dev_kfree_skb(skb); - } - - /* Simulate real usage, allocate/copy skb twice */ - skb = dev_alloc_skb(len + 2); - if (skb == NULL) { - KNI_ERR("Out of mem, dropping pkts\n"); - kni->stats.rx_dropped++; - continue; - } - - /* Align IP on 16B boundary */ - skb_reserve(skb, 2); - - if (kva->nb_segs == 1) { - memcpy(skb_put(skb, len), data_kva, len); - } else { - int nb_segs; - int kva_nb_segs = kva->nb_segs; - - for (nb_segs = 0; nb_segs < kva_nb_segs; nb_segs++) { - memcpy(skb_put(skb, kva->data_len), - data_kva, kva->data_len); - - if (!kva->next) - break; - - kva = kva->next - kni->mbuf_va + kni->mbuf_kva; - data_kva = kva->buf_addr + kva->data_off - - kni->mbuf_va + kni->mbuf_kva; - } - } - - skb->dev = dev; - skb->ip_summed = CHECKSUM_UNNECESSARY; - - kni->stats.rx_bytes += len; - kni->stats.rx_packets++; - - /* call tx interface */ - kni_net_tx(skb, dev); - } - - /* enqueue all the mbufs from rx_q into free_q */ - ret = kni_fifo_put(kni->free_q, (void **)&va, num); - if (ret != num) - /* Failing should not happen */ - KNI_ERR("Fail to enqueue mbufs into free_q\n"); -} - -/* rx interface */ -void -kni_net_rx(struct kni_dev *kni) -{ - /** - * It doesn't need to check if it is NULL pointer, - * as it has a default value - */ - (*kni_net_rx_func)(kni); -} - /* * Transmit a packet (called by the kernel) */ -#ifdef RTE_KNI_VHOST -static int -kni_net_tx(struct sk_buff *skb, struct net_device *dev) -{ - struct kni_dev *kni = netdev_priv(dev); - - dev_kfree_skb(skb); - kni->stats.tx_dropped++; - - return NETDEV_TX_OK; -} -#else static int kni_net_tx(struct sk_buff *skb, struct net_device *dev) { int len = 0; - unsigned ret; + uint32_t ret; struct kni_dev *kni = netdev_priv(dev); struct rte_kni_mbuf *pkt_kva = NULL; - struct rte_kni_mbuf *pkt_va = NULL; + void *pkt_pa = NULL; + void *pkt_va = NULL; /* save the timestamp */ #ifdef HAVE_TRANS_START_HELPER @@ -453,13 +233,13 @@ kni_net_tx(struct sk_buff *skb, struct net_device *dev) } /* dequeue a mbuf from alloc_q */ - ret = kni_fifo_get(kni->alloc_q, (void **)&pkt_va, 1); + ret = kni_fifo_get(kni->alloc_q, &pkt_pa, 1); if (likely(ret == 1)) { void *data_kva; - pkt_kva = (void *)pkt_va - kni->mbuf_va + kni->mbuf_kva; - data_kva = pkt_kva->buf_addr + pkt_kva->data_off - kni->mbuf_va - + kni->mbuf_kva; + pkt_kva = pa2kva(pkt_pa); + data_kva = kva2data_kva(pkt_kva); + pkt_va = pa2va(pkt_pa, pkt_kva); len = skb->len; memcpy(data_kva, skb->data, len); @@ -471,15 +251,15 @@ kni_net_tx(struct sk_buff *skb, struct net_device *dev) pkt_kva->data_len = len; /* enqueue mbuf into tx_q */ - ret = kni_fifo_put(kni->tx_q, (void **)&pkt_va, 1); + ret = kni_fifo_put(kni->tx_q, &pkt_va, 1); if (unlikely(ret != 1)) { /* Failing should not happen */ - KNI_ERR("Fail to enqueue mbuf into tx_q\n"); + pr_err("Fail to enqueue mbuf into tx_q\n"); goto drop; } } else { /* Failing should not happen */ - KNI_ERR("Fail to dequeue mbuf from alloc_q\n"); + pr_err("Fail to dequeue mbuf from alloc_q\n"); goto drop; } @@ -497,22 +277,294 @@ drop: return NETDEV_TX_OK; } -#endif + +/* + * RX: normal working mode + */ +static void +kni_net_rx_normal(struct kni_dev *kni) +{ + uint32_t ret; + uint32_t len; + uint32_t i, num_rx, num_fq; + struct rte_kni_mbuf *kva; + void *data_kva; + struct sk_buff *skb; + struct net_device *dev = kni->net_dev; + + /* Get the number of free entries in free_q */ + num_fq = kni_fifo_free_count(kni->free_q); + if (num_fq == 0) { + /* No room on the free_q, bail out */ + return; + } + + /* Calculate the number of entries to dequeue from rx_q */ + num_rx = min_t(uint32_t, num_fq, MBUF_BURST_SZ); + + /* Burst dequeue from rx_q */ + num_rx = kni_fifo_get(kni->rx_q, kni->pa, num_rx); + if (num_rx == 0) + return; + + /* Transfer received packets to netif */ + for (i = 0; i < num_rx; i++) { + kva = pa2kva(kni->pa[i]); + len = kva->pkt_len; + data_kva = kva2data_kva(kva); + kni->va[i] = pa2va(kni->pa[i], kva); + + skb = dev_alloc_skb(len + 2); + if (!skb) { + /* Update statistics */ + kni->stats.rx_dropped++; + continue; + } + + /* Align IP on 16B boundary */ + skb_reserve(skb, 2); + + if (kva->nb_segs == 1) { + memcpy(skb_put(skb, len), data_kva, len); + } else { + int nb_segs; + int kva_nb_segs = kva->nb_segs; + + for (nb_segs = 0; nb_segs < kva_nb_segs; nb_segs++) { + memcpy(skb_put(skb, kva->data_len), + data_kva, kva->data_len); + + if (!kva->next) + break; + + kva = pa2kva(va2pa(kva->next, kva)); + data_kva = kva2data_kva(kva); + } + } + + skb->dev = dev; + skb->protocol = eth_type_trans(skb, dev); + skb->ip_summed = CHECKSUM_UNNECESSARY; + + /* Call netif interface */ + netif_rx_ni(skb); + + /* Update statistics */ + kni->stats.rx_bytes += len; + kni->stats.rx_packets++; + } + + /* Burst enqueue mbufs into free_q */ + ret = kni_fifo_put(kni->free_q, kni->va, num_rx); + if (ret != num_rx) + /* Failing should not happen */ + pr_err("Fail to enqueue entries into free_q\n"); +} + +/* + * RX: loopback with enqueue/dequeue fifos. + */ +static void +kni_net_rx_lo_fifo(struct kni_dev *kni) +{ + uint32_t ret; + uint32_t len; + uint32_t i, num, num_rq, num_tq, num_aq, num_fq; + struct rte_kni_mbuf *kva; + void *data_kva; + struct rte_kni_mbuf *alloc_kva; + void *alloc_data_kva; + + /* Get the number of entries in rx_q */ + num_rq = kni_fifo_count(kni->rx_q); + + /* Get the number of free entrie in tx_q */ + num_tq = kni_fifo_free_count(kni->tx_q); + + /* Get the number of entries in alloc_q */ + num_aq = kni_fifo_count(kni->alloc_q); + + /* Get the number of free entries in free_q */ + num_fq = kni_fifo_free_count(kni->free_q); + + /* Calculate the number of entries to be dequeued from rx_q */ + num = min(num_rq, num_tq); + num = min(num, num_aq); + num = min(num, num_fq); + num = min_t(uint32_t, num, MBUF_BURST_SZ); + + /* Return if no entry to dequeue from rx_q */ + if (num == 0) + return; + + /* Burst dequeue from rx_q */ + ret = kni_fifo_get(kni->rx_q, kni->pa, num); + if (ret == 0) + return; /* Failing should not happen */ + + /* Dequeue entries from alloc_q */ + ret = kni_fifo_get(kni->alloc_q, kni->alloc_pa, num); + if (ret) { + num = ret; + /* Copy mbufs */ + for (i = 0; i < num; i++) { + kva = pa2kva(kni->pa[i]); + len = kva->pkt_len; + data_kva = kva2data_kva(kva); + kni->va[i] = pa2va(kni->pa[i], kva); + + alloc_kva = pa2kva(kni->alloc_pa[i]); + alloc_data_kva = kva2data_kva(alloc_kva); + kni->alloc_va[i] = pa2va(kni->alloc_pa[i], alloc_kva); + + memcpy(alloc_data_kva, data_kva, len); + alloc_kva->pkt_len = len; + alloc_kva->data_len = len; + + kni->stats.tx_bytes += len; + kni->stats.rx_bytes += len; + } + + /* Burst enqueue mbufs into tx_q */ + ret = kni_fifo_put(kni->tx_q, kni->alloc_va, num); + if (ret != num) + /* Failing should not happen */ + pr_err("Fail to enqueue mbufs into tx_q\n"); + } + + /* Burst enqueue mbufs into free_q */ + ret = kni_fifo_put(kni->free_q, kni->va, num); + if (ret != num) + /* Failing should not happen */ + pr_err("Fail to enqueue mbufs into free_q\n"); + + /** + * Update statistic, and enqueue/dequeue failure is impossible, + * as all queues are checked at first. + */ + kni->stats.tx_packets += num; + kni->stats.rx_packets += num; +} + +/* + * RX: loopback with enqueue/dequeue fifos and sk buffer copies. + */ +static void +kni_net_rx_lo_fifo_skb(struct kni_dev *kni) +{ + uint32_t ret; + uint32_t len; + uint32_t i, num_rq, num_fq, num; + struct rte_kni_mbuf *kva; + void *data_kva; + struct sk_buff *skb; + struct net_device *dev = kni->net_dev; + + /* Get the number of entries in rx_q */ + num_rq = kni_fifo_count(kni->rx_q); + + /* Get the number of free entries in free_q */ + num_fq = kni_fifo_free_count(kni->free_q); + + /* Calculate the number of entries to dequeue from rx_q */ + num = min(num_rq, num_fq); + num = min_t(uint32_t, num, MBUF_BURST_SZ); + + /* Return if no entry to dequeue from rx_q */ + if (num == 0) + return; + + /* Burst dequeue mbufs from rx_q */ + ret = kni_fifo_get(kni->rx_q, kni->pa, num); + if (ret == 0) + return; + + /* Copy mbufs to sk buffer and then call tx interface */ + for (i = 0; i < num; i++) { + kva = pa2kva(kni->pa[i]); + len = kva->pkt_len; + data_kva = kva2data_kva(kva); + kni->va[i] = pa2va(kni->pa[i], kva); + + skb = dev_alloc_skb(len + 2); + if (skb) { + /* Align IP on 16B boundary */ + skb_reserve(skb, 2); + memcpy(skb_put(skb, len), data_kva, len); + skb->dev = dev; + skb->ip_summed = CHECKSUM_UNNECESSARY; + dev_kfree_skb(skb); + } + + /* Simulate real usage, allocate/copy skb twice */ + skb = dev_alloc_skb(len + 2); + if (skb == NULL) { + kni->stats.rx_dropped++; + continue; + } + + /* Align IP on 16B boundary */ + skb_reserve(skb, 2); + + if (kva->nb_segs == 1) { + memcpy(skb_put(skb, len), data_kva, len); + } else { + int nb_segs; + int kva_nb_segs = kva->nb_segs; + + for (nb_segs = 0; nb_segs < kva_nb_segs; nb_segs++) { + memcpy(skb_put(skb, kva->data_len), + data_kva, kva->data_len); + + if (!kva->next) + break; + + kva = pa2kva(va2pa(kva->next, kva)); + data_kva = kva2data_kva(kva); + } + } + + skb->dev = dev; + skb->ip_summed = CHECKSUM_UNNECESSARY; + + kni->stats.rx_bytes += len; + kni->stats.rx_packets++; + + /* call tx interface */ + kni_net_tx(skb, dev); + } + + /* enqueue all the mbufs from rx_q into free_q */ + ret = kni_fifo_put(kni->free_q, kni->va, num); + if (ret != num) + /* Failing should not happen */ + pr_err("Fail to enqueue mbufs into free_q\n"); +} + +/* rx interface */ +void +kni_net_rx(struct kni_dev *kni) +{ + /** + * It doesn't need to check if it is NULL pointer, + * as it has a default value + */ + (*kni_net_rx_func)(kni); +} /* * Deal with a transmit timeout. */ static void -kni_net_tx_timeout (struct net_device *dev) +kni_net_tx_timeout(struct net_device *dev) { struct kni_dev *kni = netdev_priv(dev); - KNI_DBG("Transmit timeout at %ld, latency %ld\n", jiffies, - jiffies - dev->trans_start); + pr_debug("Transmit timeout at %ld, latency %ld\n", jiffies, + jiffies - dev_trans_start(dev)); kni->stats.tx_errors++; netif_wake_queue(dev); - return; } /* @@ -521,8 +573,8 @@ kni_net_tx_timeout (struct net_device *dev) static int kni_net_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) { - KNI_DBG("kni_net_ioctl %d\n", - ((struct kni_dev *)netdev_priv(dev))->group_id); + pr_debug("kni_net_ioctl group:%d cmd:%d\n", + ((struct kni_dev *)netdev_priv(dev))->group_id, cmd); return 0; } @@ -539,7 +591,7 @@ kni_net_change_mtu(struct net_device *dev, int new_mtu) struct rte_kni_request req; struct kni_dev *kni = netdev_priv(dev); - KNI_DBG("kni_net_change_mtu new mtu %d to be set\n", new_mtu); + pr_debug("kni_net_change_mtu new mtu %d to be set\n", new_mtu); memset(&req, 0, sizeof(req)); req.req_id = RTE_KNI_REQ_CHANGE_MTU; @@ -561,55 +613,6 @@ kni_net_poll_resp(struct kni_dev *kni) wake_up_interruptible(&kni->wq); } -/* - * It can be called to process the request. - */ -static int -kni_net_process_request(struct kni_dev *kni, struct rte_kni_request *req) -{ - int ret = -1; - void *resp_va; - unsigned num; - int ret_val; - - if (!kni || !req) { - KNI_ERR("No kni instance or request\n"); - return -EINVAL; - } - - mutex_lock(&kni->sync_lock); - - /* Construct data */ - memcpy(kni->sync_kva, req, sizeof(struct rte_kni_request)); - num = kni_fifo_put(kni->req_q, &kni->sync_va, 1); - if (num < 1) { - KNI_ERR("Cannot send to req_q\n"); - ret = -EBUSY; - goto fail; - } - - ret_val = wait_event_interruptible_timeout(kni->wq, - kni_fifo_count(kni->resp_q), 3 * HZ); - if (signal_pending(current) || ret_val <= 0) { - ret = -ETIME; - goto fail; - } - num = kni_fifo_get(kni->resp_q, (void **)&resp_va, 1); - if (num != 1 || resp_va != kni->sync_va) { - /* This should never happen */ - KNI_ERR("No data in resp_q\n"); - ret = -ENODATA; - goto fail; - } - - memcpy(req, kni->sync_kva, sizeof(struct rte_kni_request)); - ret = 0; - -fail: - mutex_unlock(&kni->sync_lock); - return ret; -} - /* * Return statistics to the caller */ @@ -617,6 +620,7 @@ static struct net_device_stats * kni_net_stats(struct net_device *dev) { struct kni_dev *kni = netdev_priv(dev); + return &kni->stats; } @@ -626,7 +630,7 @@ kni_net_stats(struct net_device *dev) static int kni_net_header(struct sk_buff *skb, struct net_device *dev, unsigned short type, const void *daddr, - const void *saddr, unsigned int len) + const void *saddr, uint32_t len) { struct ethhdr *eth = (struct ethhdr *) skb_push(skb, ETH_HLEN); @@ -637,7 +641,6 @@ kni_net_header(struct sk_buff *skb, struct net_device *dev, return dev->hard_header_len; } - /* * Re-fill the eth header */ @@ -662,9 +665,11 @@ kni_net_rebuild_header(struct sk_buff *skb) * * Returns 0 on success, negative on failure **/ -static int kni_net_set_mac(struct net_device *netdev, void *p) +static int +kni_net_set_mac(struct net_device *netdev, void *p) { struct sockaddr *addr = p; + if (!is_valid_ether_addr((unsigned char *)(addr->sa_data))) return -EADDRNOTAVAIL; memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len); @@ -672,7 +677,8 @@ static int kni_net_set_mac(struct net_device *netdev, void *p) } #ifdef HAVE_CHANGE_CARRIER_CB -static int kni_net_change_carrier(struct net_device *dev, bool new_carrier) +static int +kni_net_change_carrier(struct net_device *dev, bool new_carrier) { if (new_carrier) netif_carrier_on(dev); @@ -711,8 +717,6 @@ kni_net_init(struct net_device *dev) { struct kni_dev *kni = netdev_priv(dev); - KNI_DBG("kni_net_init\n"); - init_waitqueue_head(&kni->wq); mutex_init(&kni->sync_lock); @@ -726,18 +730,18 @@ void kni_net_config_lo_mode(char *lo_str) { if (!lo_str) { - KNI_PRINT("loopback disabled"); + pr_debug("loopback disabled"); return; } if (!strcmp(lo_str, "lo_mode_none")) - KNI_PRINT("loopback disabled"); + pr_debug("loopback disabled"); else if (!strcmp(lo_str, "lo_mode_fifo")) { - KNI_PRINT("loopback mode=lo_mode_fifo enabled"); + pr_debug("loopback mode=lo_mode_fifo enabled"); kni_net_rx_func = kni_net_rx_lo_fifo; } else if (!strcmp(lo_str, "lo_mode_fifo_skb")) { - KNI_PRINT("loopback mode=lo_mode_fifo_skb enabled"); + pr_debug("loopback mode=lo_mode_fifo_skb enabled"); kni_net_rx_func = kni_net_rx_lo_fifo_skb; } else - KNI_PRINT("Incognizant parameter, loopback disabled"); + pr_debug("Incognizant parameter, loopback disabled"); } diff --git a/dpdk/lib/librte_eal/linuxapp/kni/kni_vhost.c b/dpdk/lib/librte_eal/linuxapp/kni/kni_vhost.c deleted file mode 100644 index a3ca8499..00000000 --- a/dpdk/lib/librte_eal/linuxapp/kni/kni_vhost.c +++ /dev/null @@ -1,857 +0,0 @@ -/*- - * GPL LICENSE SUMMARY - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of version 2 of the GNU General Public License as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. - * The full GNU General Public License is included in this distribution - * in the file called LICENSE.GPL. - * - * Contact Information: - * Intel Corporation - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "compat.h" -#include "kni_dev.h" -#include "kni_fifo.h" - -#define RX_BURST_SZ 4 - -extern void put_unused_fd(unsigned int fd); - -#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,7,0) -extern struct file* -sock_alloc_file(struct socket *sock, - int flags, const char *dname); - -extern int get_unused_fd_flags(unsigned flags); - -extern void fd_install(unsigned int fd, struct file *file); - -static int kni_sock_map_fd(struct socket *sock) -{ - struct file *file; - int fd = get_unused_fd_flags(0); - if (fd < 0) - return fd; - - file = sock_alloc_file(sock, 0, NULL); - if (IS_ERR(file)) { - put_unused_fd(fd); - return PTR_ERR(file); - } - fd_install(fd, file); - return fd; -} -#else -#define kni_sock_map_fd(s) sock_map_fd(s, 0) -#endif - -static struct proto kni_raw_proto = { - .name = "kni_vhost", - .owner = THIS_MODULE, - .obj_size = sizeof(struct kni_vhost_queue), -}; - -static inline int -kni_vhost_net_tx(struct kni_dev *kni, struct msghdr *m, - unsigned offset, unsigned len) -{ - struct rte_kni_mbuf *pkt_kva = NULL; - struct rte_kni_mbuf *pkt_va = NULL; - int ret; - - KNI_DBG_TX("tx offset=%d, len=%d, iovlen=%d\n", -#ifdef HAVE_IOV_ITER_MSGHDR - offset, len, (int)m->msg_iter.iov->iov_len); -#else - offset, len, (int)m->msg_iov->iov_len); -#endif - - /** - * Check if it has at least one free entry in tx_q and - * one entry in alloc_q. - */ - if (kni_fifo_free_count(kni->tx_q) == 0 || - kni_fifo_count(kni->alloc_q) == 0) { - /** - * If no free entry in tx_q or no entry in alloc_q, - * drops skb and goes out. - */ - goto drop; - } - - /* dequeue a mbuf from alloc_q */ - ret = kni_fifo_get(kni->alloc_q, (void **)&pkt_va, 1); - if (likely(ret == 1)) { - void *data_kva; - - pkt_kva = (void *)pkt_va - kni->mbuf_va + kni->mbuf_kva; - data_kva = pkt_kva->buf_addr + pkt_kva->data_off - - kni->mbuf_va + kni->mbuf_kva; - -#ifdef HAVE_IOV_ITER_MSGHDR - copy_from_iter(data_kva, len, &m->msg_iter); -#else - memcpy_fromiovecend(data_kva, m->msg_iov, offset, len); -#endif - - if (unlikely(len < ETH_ZLEN)) { - memset(data_kva + len, 0, ETH_ZLEN - len); - len = ETH_ZLEN; - } - pkt_kva->pkt_len = len; - pkt_kva->data_len = len; - - /* enqueue mbuf into tx_q */ - ret = kni_fifo_put(kni->tx_q, (void **)&pkt_va, 1); - if (unlikely(ret != 1)) { - /* Failing should not happen */ - KNI_ERR("Fail to enqueue mbuf into tx_q\n"); - goto drop; - } - } else { - /* Failing should not happen */ - KNI_ERR("Fail to dequeue mbuf from alloc_q\n"); - goto drop; - } - - /* update statistics */ - kni->stats.tx_bytes += len; - kni->stats.tx_packets++; - - return 0; - -drop: - /* update statistics */ - kni->stats.tx_dropped++; - - return 0; -} - -static inline int -kni_vhost_net_rx(struct kni_dev *kni, struct msghdr *m, - unsigned offset, unsigned len) -{ - uint32_t pkt_len; - struct rte_kni_mbuf *kva; - struct rte_kni_mbuf *va; - void * data_kva; - struct sk_buff *skb; - struct kni_vhost_queue *q = kni->vhost_queue; - - if (unlikely(q == NULL)) - return 0; - - /* ensure at least one entry in free_q */ - if (unlikely(kni_fifo_free_count(kni->free_q) == 0)) - return 0; - - skb = skb_dequeue(&q->sk.sk_receive_queue); - if (unlikely(skb == NULL)) - return 0; - - kva = (struct rte_kni_mbuf*)skb->data; - - /* free skb to cache */ - skb->data = NULL; - if (unlikely(1 != kni_fifo_put(q->fifo, (void **)&skb, 1))) - /* Failing should not happen */ - KNI_ERR("Fail to enqueue entries into rx cache fifo\n"); - - pkt_len = kva->data_len; - if (unlikely(pkt_len > len)) - goto drop; - - KNI_DBG_RX("rx offset=%d, len=%d, pkt_len=%d, iovlen=%d\n", -#ifdef HAVE_IOV_ITER_MSGHDR - offset, len, pkt_len, (int)m->msg_iter.iov->iov_len); -#else - offset, len, pkt_len, (int)m->msg_iov->iov_len); -#endif - - data_kva = kva->buf_addr + kva->data_off - kni->mbuf_va + kni->mbuf_kva; -#ifdef HAVE_IOV_ITER_MSGHDR - if (unlikely(copy_to_iter(data_kva, pkt_len, &m->msg_iter))) -#else - if (unlikely(memcpy_toiovecend(m->msg_iov, data_kva, offset, pkt_len))) -#endif - goto drop; - - /* Update statistics */ - kni->stats.rx_bytes += pkt_len; - kni->stats.rx_packets++; - - /* enqueue mbufs into free_q */ - va = (void*)kva - kni->mbuf_kva + kni->mbuf_va; - if (unlikely(1 != kni_fifo_put(kni->free_q, (void **)&va, 1))) - /* Failing should not happen */ - KNI_ERR("Fail to enqueue entries into free_q\n"); - - KNI_DBG_RX("receive done %d\n", pkt_len); - - return pkt_len; - -drop: - /* Update drop statistics */ - kni->stats.rx_dropped++; - - return 0; -} - -static unsigned int -kni_sock_poll(struct file *file, struct socket *sock, poll_table * wait) -{ - struct kni_vhost_queue *q = - container_of(sock->sk, struct kni_vhost_queue, sk); - struct kni_dev *kni; - unsigned int mask = 0; - - if (unlikely(q == NULL || q->kni == NULL)) - return POLLERR; - - kni = q->kni; -#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,35) - KNI_DBG("start kni_poll on group %d, wq 0x%16llx\n", - kni->group_id, (uint64_t)sock->wq); -#else - KNI_DBG("start kni_poll on group %d, wait at 0x%16llx\n", - kni->group_id, (uint64_t)&sock->wait); -#endif - -#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,35) - poll_wait(file, &sock->wq->wait, wait); -#else - poll_wait(file, &sock->wait, wait); -#endif - - if (kni_fifo_count(kni->rx_q) > 0) - mask |= POLLIN | POLLRDNORM; - - if (sock_writeable(&q->sk) || -#ifdef SOCKWQ_ASYNC_NOSPACE - (!test_and_set_bit(SOCKWQ_ASYNC_NOSPACE, &q->sock->flags) && -#else - (!test_and_set_bit(SOCK_ASYNC_NOSPACE, &q->sock->flags) && -#endif - sock_writeable(&q->sk))) - mask |= POLLOUT | POLLWRNORM; - - return mask; -} - -static inline void -kni_vhost_enqueue(struct kni_dev *kni, struct kni_vhost_queue *q, - struct sk_buff *skb, struct rte_kni_mbuf *va) -{ - struct rte_kni_mbuf *kva; - - kva = (void *)(va) - kni->mbuf_va + kni->mbuf_kva; - (skb)->data = (unsigned char*)kva; - (skb)->len = kva->data_len; - skb_queue_tail(&q->sk.sk_receive_queue, skb); -} - -static inline void -kni_vhost_enqueue_burst(struct kni_dev *kni, struct kni_vhost_queue *q, - struct sk_buff **skb, struct rte_kni_mbuf **va) -{ - int i; - for (i = 0; i < RX_BURST_SZ; skb++, va++, i++) - kni_vhost_enqueue(kni, q, *skb, *va); -} - -int -kni_chk_vhost_rx(struct kni_dev *kni) -{ - struct kni_vhost_queue *q = kni->vhost_queue; - unsigned nb_in, nb_mbuf, nb_skb; - const unsigned BURST_MASK = RX_BURST_SZ - 1; - unsigned nb_burst, nb_backlog, i; - struct sk_buff *skb[RX_BURST_SZ]; - struct rte_kni_mbuf *va[RX_BURST_SZ]; - - if (unlikely(BE_STOP & kni->vq_status)) { - kni->vq_status |= BE_FINISH; - return 0; - } - - if (unlikely(q == NULL)) - return 0; - - nb_skb = kni_fifo_count(q->fifo); - nb_mbuf = kni_fifo_count(kni->rx_q); - - nb_in = min(nb_mbuf, nb_skb); - nb_in = min(nb_in, (unsigned)RX_BURST_SZ); - nb_burst = (nb_in & ~BURST_MASK); - nb_backlog = (nb_in & BURST_MASK); - - /* enqueue skb_queue per BURST_SIZE bulk */ - if (0 != nb_burst) { - if (unlikely(RX_BURST_SZ != kni_fifo_get( - kni->rx_q, (void **)&va, - RX_BURST_SZ))) - goto except; - - if (unlikely(RX_BURST_SZ != kni_fifo_get( - q->fifo, (void **)&skb, - RX_BURST_SZ))) - goto except; - - kni_vhost_enqueue_burst(kni, q, skb, va); - } - - /* all leftover, do one by one */ - for (i = 0; i < nb_backlog; ++i) { - if (unlikely(1 != kni_fifo_get( - kni->rx_q,(void **)&va, 1))) - goto except; - - if (unlikely(1 != kni_fifo_get( - q->fifo, (void **)&skb, 1))) - goto except; - - kni_vhost_enqueue(kni, q, *skb, *va); - } - - /* Ondemand wake up */ - if ((nb_in == RX_BURST_SZ) || (nb_skb == 0) || - ((nb_mbuf < RX_BURST_SZ) && (nb_mbuf != 0))) { - wake_up_interruptible_poll(sk_sleep(&q->sk), - POLLIN | POLLRDNORM | POLLRDBAND); - KNI_DBG_RX("RX CHK KICK nb_mbuf %d, nb_skb %d, nb_in %d\n", - nb_mbuf, nb_skb, nb_in); - } - - return 0; - -except: - /* Failing should not happen */ - KNI_ERR("Fail to enqueue fifo, it shouldn't happen \n"); - BUG_ON(1); - - return 0; -} - -static int -#ifdef HAVE_KIOCB_MSG_PARAM -kni_sock_sndmsg(struct kiocb *iocb, struct socket *sock, - struct msghdr *m, size_t total_len) -#else -kni_sock_sndmsg(struct socket *sock, - struct msghdr *m, size_t total_len) -#endif /* HAVE_KIOCB_MSG_PARAM */ -{ - struct kni_vhost_queue *q = - container_of(sock->sk, struct kni_vhost_queue, sk); - int vnet_hdr_len = 0; - unsigned long len = total_len; - - if (unlikely(q == NULL || q->kni == NULL)) - return 0; - - KNI_DBG_TX("kni_sndmsg len %ld, flags 0x%08x, nb_iov %d\n", -#ifdef HAVE_IOV_ITER_MSGHDR - len, q->flags, (int)m->msg_iter.iov->iov_len); -#else - len, q->flags, (int)m->msg_iovlen); -#endif - -#ifdef RTE_KNI_VHOST_VNET_HDR_EN - if (likely(q->flags & IFF_VNET_HDR)) { - vnet_hdr_len = q->vnet_hdr_sz; - if (unlikely(len < vnet_hdr_len)) - return -EINVAL; - len -= vnet_hdr_len; - } -#endif - - if (unlikely(len < ETH_HLEN + q->vnet_hdr_sz)) - return -EINVAL; - - return kni_vhost_net_tx(q->kni, m, vnet_hdr_len, len); -} - -static int -#ifdef HAVE_KIOCB_MSG_PARAM -kni_sock_rcvmsg(struct kiocb *iocb, struct socket *sock, - struct msghdr *m, size_t len, int flags) -#else -kni_sock_rcvmsg(struct socket *sock, - struct msghdr *m, size_t len, int flags) -#endif /* HAVE_KIOCB_MSG_PARAM */ -{ - int vnet_hdr_len = 0; - int pkt_len = 0; - struct kni_vhost_queue *q = - container_of(sock->sk, struct kni_vhost_queue, sk); - static struct virtio_net_hdr - __attribute__ ((unused)) vnet_hdr = { - .flags = 0, - .gso_type = VIRTIO_NET_HDR_GSO_NONE - }; - - if (unlikely(q == NULL || q->kni == NULL)) - return 0; - -#ifdef RTE_KNI_VHOST_VNET_HDR_EN - if (likely(q->flags & IFF_VNET_HDR)) { - vnet_hdr_len = q->vnet_hdr_sz; - if ((len -= vnet_hdr_len) < 0) - return -EINVAL; - } -#endif - - if (unlikely(0 == (pkt_len = kni_vhost_net_rx(q->kni, - m, vnet_hdr_len, len)))) - return 0; - -#ifdef RTE_KNI_VHOST_VNET_HDR_EN - /* no need to copy hdr when no pkt received */ -#ifdef HAVE_IOV_ITER_MSGHDR - if (unlikely(copy_to_iter((void *)&vnet_hdr, vnet_hdr_len, - &m->msg_iter))) -#else - if (unlikely(memcpy_toiovecend(m->msg_iov, - (void *)&vnet_hdr, 0, vnet_hdr_len))) -#endif /* HAVE_IOV_ITER_MSGHDR */ - return -EFAULT; -#endif /* RTE_KNI_VHOST_VNET_HDR_EN */ - KNI_DBG_RX("kni_rcvmsg expect_len %ld, flags 0x%08x, pkt_len %d\n", - (unsigned long)len, q->flags, pkt_len); - - return pkt_len + vnet_hdr_len; -} - -/* dummy tap like ioctl */ -static int -kni_sock_ioctl(struct socket *sock, unsigned int cmd, - unsigned long arg) -{ - void __user *argp = (void __user *)arg; - struct ifreq __user *ifr = argp; - unsigned int __user *up = argp; - struct kni_vhost_queue *q = - container_of(sock->sk, struct kni_vhost_queue, sk); - struct kni_dev *kni; - unsigned int u; - int __user *sp = argp; - int s; - int ret; - - KNI_DBG("tap ioctl cmd 0x%08x\n", cmd); - - switch (cmd) { - case TUNSETIFF: - KNI_DBG("TUNSETIFF\n"); - /* ignore the name, just look at flags */ - if (get_user(u, &ifr->ifr_flags)) - return -EFAULT; - - ret = 0; - if ((u & ~IFF_VNET_HDR) != (IFF_NO_PI | IFF_TAP)) - ret = -EINVAL; - else - q->flags = u; - - return ret; - - case TUNGETIFF: - KNI_DBG("TUNGETIFF\n"); - rcu_read_lock_bh(); - kni = rcu_dereference_bh(q->kni); - if (kni) - dev_hold(kni->net_dev); - rcu_read_unlock_bh(); - - if (!kni) - return -ENOLINK; - - ret = 0; - if (copy_to_user(&ifr->ifr_name, kni->net_dev->name, IFNAMSIZ) || - put_user(q->flags, &ifr->ifr_flags)) - ret = -EFAULT; - dev_put(kni->net_dev); - return ret; - - case TUNGETFEATURES: - KNI_DBG("TUNGETFEATURES\n"); - u = IFF_TAP | IFF_NO_PI; -#ifdef RTE_KNI_VHOST_VNET_HDR_EN - u |= IFF_VNET_HDR; -#endif - if (put_user(u, up)) - return -EFAULT; - return 0; - - case TUNSETSNDBUF: - KNI_DBG("TUNSETSNDBUF\n"); - if (get_user(u, up)) - return -EFAULT; - - q->sk.sk_sndbuf = u; - return 0; - - case TUNGETVNETHDRSZ: - s = q->vnet_hdr_sz; - if (put_user(s, sp)) - return -EFAULT; - KNI_DBG("TUNGETVNETHDRSZ %d\n", s); - return 0; - - case TUNSETVNETHDRSZ: - if (get_user(s, sp)) - return -EFAULT; - if (s < (int)sizeof(struct virtio_net_hdr)) - return -EINVAL; - - KNI_DBG("TUNSETVNETHDRSZ %d\n", s); - q->vnet_hdr_sz = s; - return 0; - - case TUNSETOFFLOAD: - KNI_DBG("TUNSETOFFLOAD %lx\n", arg); -#ifdef RTE_KNI_VHOST_VNET_HDR_EN - /* not support any offload yet */ - if (!(q->flags & IFF_VNET_HDR)) - return -EINVAL; - - return 0; -#else - return -EINVAL; -#endif - - default: - KNI_DBG("NOT SUPPORT\n"); - return -EINVAL; - } -} - -static int -kni_sock_compat_ioctl(struct socket *sock, unsigned int cmd, - unsigned long arg) -{ - /* 32 bits app on 64 bits OS to be supported later */ - KNI_PRINT("Not implemented.\n"); - - return -EINVAL; -} - -#define KNI_VHOST_WAIT_WQ_SAFE() \ -do { \ - while ((BE_FINISH | BE_STOP) == kni->vq_status) \ - msleep(1); \ -}while(0) \ - - -static int -kni_sock_release(struct socket *sock) -{ - struct kni_vhost_queue *q = - container_of(sock->sk, struct kni_vhost_queue, sk); - struct kni_dev *kni; - - if (q == NULL) - return 0; - - if (NULL != (kni = q->kni)) { - kni->vq_status = BE_STOP; - KNI_VHOST_WAIT_WQ_SAFE(); - kni->vhost_queue = NULL; - q->kni = NULL; - } - - if (q->sockfd != -1) - q->sockfd = -1; - - sk_set_socket(&q->sk, NULL); - sock->sk = NULL; - - sock_put(&q->sk); - - KNI_DBG("dummy sock release done\n"); - - return 0; -} - -int -kni_sock_getname (struct socket *sock, - struct sockaddr *addr, - int *sockaddr_len, int peer) -{ - KNI_DBG("dummy sock getname\n"); - ((struct sockaddr_ll*)addr)->sll_family = AF_PACKET; - return 0; -} - -static const struct proto_ops kni_socket_ops = { - .getname = kni_sock_getname, - .sendmsg = kni_sock_sndmsg, - .recvmsg = kni_sock_rcvmsg, - .release = kni_sock_release, - .poll = kni_sock_poll, - .ioctl = kni_sock_ioctl, - .compat_ioctl = kni_sock_compat_ioctl, -}; - -static void -kni_sk_write_space(struct sock *sk) -{ - wait_queue_head_t *wqueue; - - if (!sock_writeable(sk) || -#ifdef SOCKWQ_ASYNC_NOSPACE - !test_and_clear_bit(SOCKWQ_ASYNC_NOSPACE, &sk->sk_socket->flags)) -#else - !test_and_clear_bit(SOCK_ASYNC_NOSPACE, &sk->sk_socket->flags)) -#endif - return; - wqueue = sk_sleep(sk); - if (wqueue && waitqueue_active(wqueue)) - wake_up_interruptible_poll( - wqueue, POLLOUT | POLLWRNORM | POLLWRBAND); -} - -static void -kni_sk_destruct(struct sock *sk) -{ - struct kni_vhost_queue *q = - container_of(sk, struct kni_vhost_queue, sk); - - if (!q) - return; - - /* make sure there's no packet in buffer */ - while (skb_dequeue(&sk->sk_receive_queue) != NULL) - ; - - mb(); - - if (q->fifo != NULL) { - kfree(q->fifo); - q->fifo = NULL; - } - - if (q->cache != NULL) { - kfree(q->cache); - q->cache = NULL; - } -} - -static int -kni_vhost_backend_init(struct kni_dev *kni) -{ - struct kni_vhost_queue *q; - struct net *net = current->nsproxy->net_ns; - int err, i, sockfd; - struct rte_kni_fifo *fifo; - struct sk_buff *elem; - - if (kni->vhost_queue != NULL) - return -1; - -#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 2, 0) - q = (struct kni_vhost_queue *)sk_alloc(net, AF_UNSPEC, GFP_KERNEL, - &kni_raw_proto, 0); -#else - q = (struct kni_vhost_queue *)sk_alloc(net, AF_UNSPEC, GFP_KERNEL, - &kni_raw_proto); -#endif - if (!q) - return -ENOMEM; - - err = sock_create_lite(AF_UNSPEC, SOCK_RAW, IPPROTO_RAW, &q->sock); - if (err) - goto free_sk; - - sockfd = kni_sock_map_fd(q->sock); - if (sockfd < 0) { - err = sockfd; - goto free_sock; - } - - /* cache init */ - q->cache = kzalloc(RTE_KNI_VHOST_MAX_CACHE_SIZE * sizeof(struct sk_buff), - GFP_KERNEL); - if (!q->cache) - goto free_fd; - - fifo = kzalloc(RTE_KNI_VHOST_MAX_CACHE_SIZE * sizeof(void *) - + sizeof(struct rte_kni_fifo), GFP_KERNEL); - if (!fifo) - goto free_cache; - - kni_fifo_init(fifo, RTE_KNI_VHOST_MAX_CACHE_SIZE); - - for (i = 0; i < RTE_KNI_VHOST_MAX_CACHE_SIZE; i++) { - elem = &q->cache[i]; - kni_fifo_put(fifo, (void**)&elem, 1); - } - q->fifo = fifo; - - /* store sockfd in vhost_queue */ - q->sockfd = sockfd; - - /* init socket */ - q->sock->type = SOCK_RAW; - q->sock->state = SS_CONNECTED; - q->sock->ops = &kni_socket_ops; - sock_init_data(q->sock, &q->sk); - - /* init sock data */ - q->sk.sk_write_space = kni_sk_write_space; - q->sk.sk_destruct = kni_sk_destruct; - q->flags = IFF_NO_PI | IFF_TAP; - q->vnet_hdr_sz = sizeof(struct virtio_net_hdr); -#ifdef RTE_KNI_VHOST_VNET_HDR_EN - q->flags |= IFF_VNET_HDR; -#endif - - /* bind kni_dev with vhost_queue */ - q->kni = kni; - kni->vhost_queue = q; - - wmb(); - - kni->vq_status = BE_START; - -#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,35) - KNI_DBG("backend init sockfd=%d, sock->wq=0x%16llx," - "sk->sk_wq=0x%16llx", - q->sockfd, (uint64_t)q->sock->wq, - (uint64_t)q->sk.sk_wq); -#else - KNI_DBG("backend init sockfd=%d, sock->wait at 0x%16llx," - "sk->sk_sleep=0x%16llx", - q->sockfd, (uint64_t)&q->sock->wait, - (uint64_t)q->sk.sk_sleep); -#endif - - return 0; - -free_cache: - kfree(q->cache); - q->cache = NULL; - -free_fd: - put_unused_fd(sockfd); - -free_sock: - q->kni = NULL; - kni->vhost_queue = NULL; - kni->vq_status |= BE_FINISH; - sock_release(q->sock); - q->sock->ops = NULL; - q->sock = NULL; - -free_sk: - sk_free((struct sock*)q); - - return err; -} - -/* kni vhost sock sysfs */ -static ssize_t -show_sock_fd(struct device *dev, struct device_attribute *attr, - char *buf) -{ - struct net_device *net_dev = container_of(dev, struct net_device, dev); - struct kni_dev *kni = netdev_priv(net_dev); - int sockfd = -1; - if (kni->vhost_queue != NULL) - sockfd = kni->vhost_queue->sockfd; - return snprintf(buf, 10, "%d\n", sockfd); -} - -static ssize_t -show_sock_en(struct device *dev, struct device_attribute *attr, - char *buf) -{ - struct net_device *net_dev = container_of(dev, struct net_device, dev); - struct kni_dev *kni = netdev_priv(net_dev); - return snprintf(buf, 10, "%u\n", (kni->vhost_queue == NULL ? 0 : 1)); -} - -static ssize_t -set_sock_en(struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) -{ - struct net_device *net_dev = container_of(dev, struct net_device, dev); - struct kni_dev *kni = netdev_priv(net_dev); - unsigned long en; - int err = 0; - - if (0 != kstrtoul(buf, 0, &en)) - return -EINVAL; - - if (en) - err = kni_vhost_backend_init(kni); - - return err ? err : count; -} - -static DEVICE_ATTR(sock_fd, S_IRUGO | S_IRUSR, show_sock_fd, NULL); -static DEVICE_ATTR(sock_en, S_IRUGO | S_IWUSR, show_sock_en, set_sock_en); -static struct attribute *dev_attrs[] = { - &dev_attr_sock_fd.attr, - &dev_attr_sock_en.attr, - NULL, -}; - -static const struct attribute_group dev_attr_grp = { - .attrs = dev_attrs, -}; - -int -kni_vhost_backend_release(struct kni_dev *kni) -{ - struct kni_vhost_queue *q = kni->vhost_queue; - - if (q == NULL) - return 0; - - /* dettach from kni */ - q->kni = NULL; - - KNI_DBG("release backend done\n"); - - return 0; -} - -int -kni_vhost_init(struct kni_dev *kni) -{ - struct net_device *dev = kni->net_dev; - - if (sysfs_create_group(&dev->dev.kobj, &dev_attr_grp)) - sysfs_remove_group(&dev->dev.kobj, &dev_attr_grp); - - kni->vq_status = BE_STOP; - - KNI_DBG("kni_vhost_init done\n"); - - return 0; -} diff --git a/dpdk/lib/librte_eal/linuxapp/xen_dom0/Makefile b/dpdk/lib/librte_eal/linuxapp/xen_dom0/Makefile deleted file mode 100644 index 9d22fb97..00000000 --- a/dpdk/lib/librte_eal/linuxapp/xen_dom0/Makefile +++ /dev/null @@ -1,56 +0,0 @@ -# BSD LICENSE -# -# Copyright(c) 2010-2014 Intel Corporation. All rights reserved. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in -# the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Intel Corporation nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -include $(RTE_SDK)/mk/rte.vars.mk - -# -# module name and path -# -MODULE = rte_dom0_mm - -# -# CFLAGS -# -MODULE_CFLAGS += -I$(SRCDIR) --param max-inline-insns-single=50 -MODULE_CFLAGS += -I$(RTE_OUTPUT)/include -MODULE_CFLAGS += -include $(RTE_OUTPUT)/include/rte_config.h -MODULE_CFLAGS += -Wall -Werror - -# this lib needs main eal -DEPDIRS-y += lib/librte_eal/linuxapp/eal - -# -# all source are stored in SRCS-y -# - -SRCS-y += dom0_mm_misc.c - -include $(RTE_SDK)/mk/rte.module.mk diff --git a/dpdk/lib/librte_eal/linuxapp/xen_dom0/compat.h b/dpdk/lib/librte_eal/linuxapp/xen_dom0/compat.h deleted file mode 100644 index e6eb97f2..00000000 --- a/dpdk/lib/librte_eal/linuxapp/xen_dom0/compat.h +++ /dev/null @@ -1,15 +0,0 @@ -/* - * Minimal wrappers to allow compiling xen_dom0 on older kernels. - */ - -#ifndef RHEL_RELEASE_VERSION -#define RHEL_RELEASE_VERSION(a, b) (((a) << 8) + (b)) -#endif - -#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 39) && \ - (!(defined(RHEL_RELEASE_CODE) && \ - RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(6, 4))) - -#define kstrtoul strict_strtoul - -#endif /* < 2.6.39 */ diff --git a/dpdk/lib/librte_eal/linuxapp/xen_dom0/dom0_mm_dev.h b/dpdk/lib/librte_eal/linuxapp/xen_dom0/dom0_mm_dev.h deleted file mode 100644 index 9d5ffb22..00000000 --- a/dpdk/lib/librte_eal/linuxapp/xen_dom0/dom0_mm_dev.h +++ /dev/null @@ -1,107 +0,0 @@ -/*- - * This file is provided under a dual BSD/GPLv2 license. When using or - * redistributing this file, you may do so under either license. - * - * GPL LICENSE SUMMARY - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of version 2 of the GNU General Public License as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. - * The full GNU General Public License is included in this distribution - * in the file called LICENSE.GPL. - * - * Contact Information: - * Intel Corporation - * - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ -#ifndef _DOM0_MM_DEV_H_ -#define _DOM0_MM_DEV_H_ - -#include -#include -#include -#include -#include - -#define NUM_MEM_CTX 256 /**< Maximum number of memory context*/ -#define MAX_EXCHANGE_FAIL_TIME 5 /**< Maximum times of allowing exchange fail .*/ -#define MAX_MEMBLOCK_SIZE (2 * DOM0_MEMBLOCK_SIZE) -#define MAX_NUM_ORDER (DOM0_CONTIG_NUM_ORDER + 1) -#define SIZE_PER_BLOCK 2 /**< Size of memory block (2MB).*/ - -/** - * A structure describing the private information for a dom0 device. - */ -struct dom0_mm_dev { - struct miscdevice miscdev; - uint8_t fail_times; - uint32_t used_memsize; - uint32_t num_mem_ctx; - uint32_t config_memsize; - uint32_t num_bigblock; - struct dom0_mm_data *mm_data[NUM_MEM_CTX]; - struct mutex data_lock; -}; - -struct dom0_mm_data{ - uint32_t refcnt; - uint32_t num_memseg; /**< Number of memory segment. */ - uint32_t mem_size; /**< Size of requesting memory. */ - - char name[DOM0_NAME_MAX]; - - /** Store global memory block IDs used by an instance */ - uint32_t block_num[DOM0_NUM_MEMBLOCK]; - - /** Store memory block information.*/ - struct memblock_info block_info[DOM0_NUM_MEMBLOCK]; - - /** Store memory segment information.*/ - struct memseg_info seg_info[DOM0_NUM_MEMSEG]; -}; - -#define XEN_ERR(args...) printk(KERN_DEBUG "XEN_DOM0: Error: " args) -#define XEN_PRINT(args...) printk(KERN_DEBUG "XEN_DOM0: " args) -#endif diff --git a/dpdk/lib/librte_eal/linuxapp/xen_dom0/dom0_mm_misc.c b/dpdk/lib/librte_eal/linuxapp/xen_dom0/dom0_mm_misc.c deleted file mode 100644 index 79630bad..00000000 --- a/dpdk/lib/librte_eal/linuxapp/xen_dom0/dom0_mm_misc.c +++ /dev/null @@ -1,780 +0,0 @@ -/*- - * This file is provided under a dual BSD/GPLv2 license. When using or - * redistributing this file, you may do so under either license. - * - * GPL LICENSE SUMMARY - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of version 2 of the GNU General Public License as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. - * The full GNU General Public License is included in this distribution - * in the file called LICENSE.GPL. - * - * Contact Information: - * Intel Corporation - * - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include - -#include "compat.h" -#include "dom0_mm_dev.h" - -MODULE_LICENSE("Dual BSD/GPL"); -MODULE_AUTHOR("Intel Corporation"); -MODULE_DESCRIPTION("Kernel Module for supporting DPDK running on Xen Dom0"); - -static struct dom0_mm_dev dom0_dev; -static struct kobject *dom0_kobj = NULL; - -static struct memblock_info *rsv_mm_info; - -/* Default configuration for reserved memory size(2048 MB). */ -static uint32_t rsv_memsize = 2048; - -static int dom0_open(struct inode *inode, struct file *file); -static int dom0_release(struct inode *inode, struct file *file); -static int dom0_ioctl(struct file *file, unsigned int ioctl_num, - unsigned long ioctl_param); -static int dom0_mmap(struct file *file, struct vm_area_struct *vma); -static int dom0_memory_free(uint32_t size); -static int dom0_memory_release(struct dom0_mm_data *mm_data); - -static const struct file_operations data_fops = { - .owner = THIS_MODULE, - .open = dom0_open, - .release = dom0_release, - .mmap = dom0_mmap, - .unlocked_ioctl = (void *)dom0_ioctl, -}; - -static ssize_t -show_memsize_rsvd(struct device *dev, struct device_attribute *attr, char *buf) -{ - return snprintf(buf, 10, "%u\n", dom0_dev.used_memsize); -} - -static ssize_t -show_memsize(struct device *dev, struct device_attribute *attr, char *buf) -{ - return snprintf(buf, 10, "%u\n", dom0_dev.config_memsize); -} - -static ssize_t -store_memsize(struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) -{ - int err = 0; - unsigned long mem_size; - - if (0 != kstrtoul(buf, 0, &mem_size)) - return -EINVAL; - - mutex_lock(&dom0_dev.data_lock); - if (0 == mem_size) { - err = -EINVAL; - goto fail; - } else if (mem_size > (rsv_memsize - dom0_dev.used_memsize)) { - XEN_ERR("configure memory size fail\n"); - err = -EINVAL; - goto fail; - } else - dom0_dev.config_memsize = mem_size; - -fail: - mutex_unlock(&dom0_dev.data_lock); - return err ? err : count; -} - -static DEVICE_ATTR(memsize, S_IRUGO | S_IWUSR, show_memsize, store_memsize); -static DEVICE_ATTR(memsize_rsvd, S_IRUGO, show_memsize_rsvd, NULL); - -static struct attribute *dev_attrs[] = { - &dev_attr_memsize.attr, - &dev_attr_memsize_rsvd.attr, - NULL, -}; - -/* the memory size unit is MB */ -static const struct attribute_group dev_attr_grp = { - .name = "memsize-mB", - .attrs = dev_attrs, -}; - - -static void -sort_viraddr(struct memblock_info *mb, int cnt) -{ - int i,j; - uint64_t tmp_pfn; - uint64_t tmp_viraddr; - - /*sort virtual address and pfn */ - for(i = 0; i < cnt; i ++) { - for(j = cnt - 1; j > i; j--) { - if(mb[j].pfn < mb[j - 1].pfn) { - tmp_pfn = mb[j - 1].pfn; - mb[j - 1].pfn = mb[j].pfn; - mb[j].pfn = tmp_pfn; - - tmp_viraddr = mb[j - 1].vir_addr; - mb[j - 1].vir_addr = mb[j].vir_addr; - mb[j].vir_addr = tmp_viraddr; - } - } - } -} - -static int -dom0_find_memdata(const char * mem_name) -{ - unsigned i; - int idx = -1; - for(i = 0; i< NUM_MEM_CTX; i++) { - if(dom0_dev.mm_data[i] == NULL) - continue; - if (!strncmp(dom0_dev.mm_data[i]->name, mem_name, - sizeof(char) * DOM0_NAME_MAX)) { - idx = i; - break; - } - } - - return idx; -} - -static int -dom0_find_mempos(void) -{ - unsigned i; - int idx = -1; - - for(i = 0; i< NUM_MEM_CTX; i++) { - if(dom0_dev.mm_data[i] == NULL){ - idx = i; - break; - } - } - - return idx; -} - -static int -dom0_memory_release(struct dom0_mm_data *mm_data) -{ - int idx; - uint32_t num_block, block_id; - - /* each memory block is 2M */ - num_block = mm_data->mem_size / SIZE_PER_BLOCK; - if (num_block == 0) - return -EINVAL; - - /* reset global memory data */ - idx = dom0_find_memdata(mm_data->name); - if (idx >= 0) { - dom0_dev.used_memsize -= mm_data->mem_size; - dom0_dev.mm_data[idx] = NULL; - dom0_dev.num_mem_ctx--; - } - - /* reset these memory blocks status as free */ - for (idx = 0; idx < num_block; idx++) { - block_id = mm_data->block_num[idx]; - rsv_mm_info[block_id].used = 0; - } - - memset(mm_data, 0, sizeof(struct dom0_mm_data)); - vfree(mm_data); - return 0; -} - -static int -dom0_memory_free(uint32_t rsv_size) -{ - uint64_t vstart, vaddr; - uint32_t i, num_block, size; - - if (!xen_pv_domain()) - return -1; - - /* each memory block is 2M */ - num_block = rsv_size / SIZE_PER_BLOCK; - if (num_block == 0) - return -EINVAL; - - /* free all memory blocks of size of 4M and destroy contiguous region */ - for (i = 0; i < dom0_dev.num_bigblock * 2; i += 2) { - vstart = rsv_mm_info[i].vir_addr; - if (vstart) { - #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 13, 0) - if (rsv_mm_info[i].exchange_flag) - xen_destroy_contiguous_region(vstart, - DOM0_CONTIG_NUM_ORDER); - if (rsv_mm_info[i + 1].exchange_flag) - xen_destroy_contiguous_region(vstart + - DOM0_MEMBLOCK_SIZE, - DOM0_CONTIG_NUM_ORDER); - #else - if (rsv_mm_info[i].exchange_flag) - xen_destroy_contiguous_region(rsv_mm_info[i].pfn - * PAGE_SIZE, - DOM0_CONTIG_NUM_ORDER); - if (rsv_mm_info[i + 1].exchange_flag) - xen_destroy_contiguous_region(rsv_mm_info[i].pfn - * PAGE_SIZE + DOM0_MEMBLOCK_SIZE, - DOM0_CONTIG_NUM_ORDER); - #endif - - size = DOM0_MEMBLOCK_SIZE * 2; - vaddr = vstart; - while (size > 0) { - ClearPageReserved(virt_to_page(vaddr)); - vaddr += PAGE_SIZE; - size -= PAGE_SIZE; - } - free_pages(vstart, MAX_NUM_ORDER); - } - } - - /* free all memory blocks size of 2M and destroy contiguous region */ - for (; i < num_block; i++) { - vstart = rsv_mm_info[i].vir_addr; - if (vstart) { - if (rsv_mm_info[i].exchange_flag) - xen_destroy_contiguous_region(vstart, - DOM0_CONTIG_NUM_ORDER); - - size = DOM0_MEMBLOCK_SIZE; - vaddr = vstart; - while (size > 0) { - ClearPageReserved(virt_to_page(vaddr)); - vaddr += PAGE_SIZE; - size -= PAGE_SIZE; - } - free_pages(vstart, DOM0_CONTIG_NUM_ORDER); - } - } - - memset(rsv_mm_info, 0, sizeof(struct memblock_info) * num_block); - vfree(rsv_mm_info); - rsv_mm_info = NULL; - - return 0; -} - -static void -find_free_memory(uint32_t count, struct dom0_mm_data *mm_data) -{ - uint32_t i = 0; - uint32_t j = 0; - - while ((i < count) && (j < rsv_memsize / SIZE_PER_BLOCK)) { - if (rsv_mm_info[j].used == 0) { - mm_data->block_info[i].pfn = rsv_mm_info[j].pfn; - mm_data->block_info[i].vir_addr = - rsv_mm_info[j].vir_addr; - mm_data->block_info[i].mfn = rsv_mm_info[j].mfn; - mm_data->block_info[i].exchange_flag = - rsv_mm_info[j].exchange_flag; - mm_data->block_num[i] = j; - rsv_mm_info[j].used = 1; - i++; - } - j++; - } -} - -/** - * Find all memory segments in which physical addresses are contiguous. - */ -static void -find_memseg(int count, struct dom0_mm_data * mm_data) -{ - int i = 0; - int j, k, idx = 0; - uint64_t zone_len, pfn, num_block; - - while(i < count) { - if (mm_data->block_info[i].exchange_flag == 0) { - i++; - continue; - } - k = 0; - pfn = mm_data->block_info[i].pfn; - mm_data->seg_info[idx].pfn = pfn; - mm_data->seg_info[idx].mfn[k] = mm_data->block_info[i].mfn; - - for (j = i + 1; j < count; j++) { - - /* ignore exchange fail memory block */ - if (mm_data->block_info[j].exchange_flag == 0) - break; - - if (mm_data->block_info[j].pfn != - (mm_data->block_info[j - 1].pfn + - DOM0_MEMBLOCK_SIZE / PAGE_SIZE)) - break; - ++k; - mm_data->seg_info[idx].mfn[k] = mm_data->block_info[j].mfn; - } - - num_block = j - i; - zone_len = num_block * DOM0_MEMBLOCK_SIZE; - mm_data->seg_info[idx].size = zone_len; - - XEN_PRINT("memseg id=%d, size=0x%llx\n", idx, zone_len); - i = i+ num_block; - idx++; - if (idx == DOM0_NUM_MEMSEG) - break; - } - mm_data->num_memseg = idx; -} - -static int -dom0_memory_reserve(uint32_t rsv_size) -{ - uint64_t pfn, vstart, vaddr; - uint32_t i, num_block, size, allocated_size = 0; - -#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0) - dma_addr_t dma_handle; -#endif - - /* 2M as memory block */ - num_block = rsv_size / SIZE_PER_BLOCK; - - rsv_mm_info = vmalloc(sizeof(struct memblock_info) * num_block); - if (!rsv_mm_info) { - XEN_ERR("Unable to allocate device memory information\n"); - return -ENOMEM; - } - memset(rsv_mm_info, 0, sizeof(struct memblock_info) * num_block); - - /* try alloc size of 4M once */ - for (i = 0; i < num_block; i += 2) { - vstart = (unsigned long) - __get_free_pages(GFP_ATOMIC, MAX_NUM_ORDER); - if (vstart == 0) - break; - - dom0_dev.num_bigblock = i / 2 + 1; - allocated_size = SIZE_PER_BLOCK * (i + 2); - - /* size of 4M */ - size = DOM0_MEMBLOCK_SIZE * 2; - - vaddr = vstart; - while (size > 0) { - SetPageReserved(virt_to_page(vaddr)); - vaddr += PAGE_SIZE; - size -= PAGE_SIZE; - } - - pfn = virt_to_pfn(vstart); - rsv_mm_info[i].pfn = pfn; - rsv_mm_info[i].vir_addr = vstart; - rsv_mm_info[i + 1].pfn = - pfn + DOM0_MEMBLOCK_SIZE / PAGE_SIZE; - rsv_mm_info[i + 1].vir_addr = - vstart + DOM0_MEMBLOCK_SIZE; - } - - /*if it failed to alloc 4M, and continue to alloc 2M once */ - for (; i < num_block; i++) { - vstart = (unsigned long) - __get_free_pages(GFP_ATOMIC, DOM0_CONTIG_NUM_ORDER); - if (vstart == 0) { - XEN_ERR("allocate memory fail.\n"); - dom0_memory_free(allocated_size); - return -ENOMEM; - } - - allocated_size += SIZE_PER_BLOCK; - - size = DOM0_MEMBLOCK_SIZE; - vaddr = vstart; - while (size > 0) { - SetPageReserved(virt_to_page(vaddr)); - vaddr += PAGE_SIZE; - size -= PAGE_SIZE; - } - pfn = virt_to_pfn(vstart); - rsv_mm_info[i].pfn = pfn; - rsv_mm_info[i].vir_addr = vstart; - } - - sort_viraddr(rsv_mm_info, num_block); - - for (i = 0; i< num_block; i++) { - - /* - * This API is used to exchage MFN for getting a block of - * contiguous physical addresses, its maximum size is 2M. - */ - #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 13, 0) - if (xen_create_contiguous_region(rsv_mm_info[i].vir_addr, - DOM0_CONTIG_NUM_ORDER, 0) == 0) { - #else - if (xen_create_contiguous_region(rsv_mm_info[i].pfn * PAGE_SIZE, - DOM0_CONTIG_NUM_ORDER, 0, &dma_handle) == 0) { - #endif - rsv_mm_info[i].exchange_flag = 1; - rsv_mm_info[i].mfn = - pfn_to_mfn(rsv_mm_info[i].pfn); - rsv_mm_info[i].used = 0; - } else { - XEN_ERR("exchange memeory fail\n"); - rsv_mm_info[i].exchange_flag = 0; - dom0_dev.fail_times++; - if (dom0_dev.fail_times > MAX_EXCHANGE_FAIL_TIME) { - dom0_memory_free(rsv_size); - return -EFAULT; - } - } - } - - return 0; -} - -static int -dom0_prepare_memsegs(struct memory_info *meminfo, struct dom0_mm_data *mm_data) -{ - uint32_t num_block; - int idx; - - /* check if there is a free name buffer */ - memcpy(mm_data->name, meminfo->name, DOM0_NAME_MAX); - mm_data->name[DOM0_NAME_MAX - 1] = '\0'; - idx = dom0_find_mempos(); - if (idx < 0) - return -1; - - num_block = meminfo->size / SIZE_PER_BLOCK; - /* find free memory and new memory segments*/ - find_free_memory(num_block, mm_data); - find_memseg(num_block, mm_data); - - /* update private memory data */ - mm_data->refcnt++; - mm_data->mem_size = meminfo->size; - - /* update global memory data */ - dom0_dev.mm_data[idx] = mm_data; - dom0_dev.num_mem_ctx++; - dom0_dev.used_memsize += mm_data->mem_size; - - return 0; -} - -static int -dom0_check_memory (struct memory_info *meminfo) -{ - int idx; - uint64_t mem_size; - - /* round memory size to the next even number. */ - if (meminfo->size % 2) - ++meminfo->size; - - mem_size = meminfo->size; - if (dom0_dev.num_mem_ctx > NUM_MEM_CTX) { - XEN_ERR("Memory data space is full in Dom0 driver\n"); - return -1; - } - idx = dom0_find_memdata(meminfo->name); - if (idx >= 0) { - XEN_ERR("Memory data name %s has already exsited in Dom0 driver.\n", - meminfo->name); - return -1; - } - if ((dom0_dev.used_memsize + mem_size) > rsv_memsize) { - XEN_ERR("Total size can't be larger than reserved size.\n"); - return -1; - } - - return 0; -} - -static int __init -dom0_init(void) -{ - if (!xen_domain()) - return -ENODEV; - - if (rsv_memsize > DOM0_CONFIG_MEMSIZE) { - XEN_ERR("The reserved memory size cannot be greater than %d\n", - DOM0_CONFIG_MEMSIZE); - return -EINVAL; - } - - /* Setup the misc device */ - dom0_dev.miscdev.minor = MISC_DYNAMIC_MINOR; - dom0_dev.miscdev.name = "dom0_mm"; - dom0_dev.miscdev.fops = &data_fops; - - /* register misc char device */ - if (misc_register(&dom0_dev.miscdev) != 0) { - XEN_ERR("Misc device registration failed\n"); - return -EPERM; - } - - mutex_init(&dom0_dev.data_lock); - dom0_kobj = kobject_create_and_add("dom0-mm", mm_kobj); - - if (!dom0_kobj) { - XEN_ERR("dom0-mm object creation failed\n"); - misc_deregister(&dom0_dev.miscdev); - return -ENOMEM; - } - - if (sysfs_create_group(dom0_kobj, &dev_attr_grp)) { - kobject_put(dom0_kobj); - misc_deregister(&dom0_dev.miscdev); - return -EPERM; - } - - if (dom0_memory_reserve(rsv_memsize) < 0) { - sysfs_remove_group(dom0_kobj, &dev_attr_grp); - kobject_put(dom0_kobj); - misc_deregister(&dom0_dev.miscdev); - return -ENOMEM; - } - - XEN_PRINT("####### DPDK Xen Dom0 module loaded #######\n"); - - return 0; -} - -static void __exit -dom0_exit(void) -{ - if (rsv_mm_info != NULL) - dom0_memory_free(rsv_memsize); - - sysfs_remove_group(dom0_kobj, &dev_attr_grp); - kobject_put(dom0_kobj); - misc_deregister(&dom0_dev.miscdev); - - XEN_PRINT("####### DPDK Xen Dom0 module unloaded #######\n"); -} - -static int -dom0_open(struct inode *inode, struct file *file) -{ - file->private_data = NULL; - - XEN_PRINT(KERN_INFO "/dev/dom0_mm opened\n"); - return 0; -} - -static int -dom0_release(struct inode *inode, struct file *file) -{ - int ret = 0; - struct dom0_mm_data *mm_data = file->private_data; - - if (mm_data == NULL) - return ret; - - mutex_lock(&dom0_dev.data_lock); - if (--mm_data->refcnt == 0) - ret = dom0_memory_release(mm_data); - mutex_unlock(&dom0_dev.data_lock); - - file->private_data = NULL; - XEN_PRINT(KERN_INFO "/dev/dom0_mm closed\n"); - return ret; -} - -static int -dom0_mmap(struct file *file, struct vm_area_struct *vm) -{ - int status = 0; - uint32_t idx = vm->vm_pgoff; - uint64_t pfn, size = vm->vm_end - vm->vm_start; - struct dom0_mm_data *mm_data = file->private_data; - - if(mm_data == NULL) - return -EINVAL; - - mutex_lock(&dom0_dev.data_lock); - if (idx >= mm_data->num_memseg) { - mutex_unlock(&dom0_dev.data_lock); - return -EINVAL; - } - - if (size > mm_data->seg_info[idx].size){ - mutex_unlock(&dom0_dev.data_lock); - return -EINVAL; - } - - XEN_PRINT("mmap memseg idx =%d,size = 0x%llx\n", idx, size); - - pfn = mm_data->seg_info[idx].pfn; - mutex_unlock(&dom0_dev.data_lock); - - status = remap_pfn_range(vm, vm->vm_start, pfn, size, PAGE_SHARED); - - return status; -} -static int -dom0_ioctl(struct file *file, - unsigned int ioctl_num, - unsigned long ioctl_param) -{ - int idx, ret; - char name[DOM0_NAME_MAX] = {0}; - struct memory_info meminfo; - struct dom0_mm_data *mm_data = file->private_data; - - XEN_PRINT("IOCTL num=0x%0x param=0x%0lx \n", ioctl_num, ioctl_param); - - /** - * Switch according to the ioctl called - */ - switch _IOC_NR(ioctl_num) { - case _IOC_NR(RTE_DOM0_IOCTL_PREPARE_MEMSEG): - ret = copy_from_user(&meminfo, (void *)ioctl_param, - sizeof(struct memory_info)); - if (ret) - return -EFAULT; - - if (mm_data != NULL) { - XEN_ERR("Cannot create memory segment for the same" - " file descriptor\n"); - return -EINVAL; - } - - /* Allocate private data */ - mm_data = vmalloc(sizeof(struct dom0_mm_data)); - if (!mm_data) { - XEN_ERR("Unable to allocate device private data\n"); - return -ENOMEM; - } - memset(mm_data, 0, sizeof(struct dom0_mm_data)); - - mutex_lock(&dom0_dev.data_lock); - /* check if we can allocate memory*/ - if (dom0_check_memory(&meminfo) < 0) { - mutex_unlock(&dom0_dev.data_lock); - vfree(mm_data); - return -EINVAL; - } - - /* allocate memory and created memory segments*/ - if (dom0_prepare_memsegs(&meminfo, mm_data) < 0) { - XEN_ERR("create memory segment fail.\n"); - mutex_unlock(&dom0_dev.data_lock); - return -EIO; - } - - file->private_data = mm_data; - mutex_unlock(&dom0_dev.data_lock); - break; - - /* support multiple process in term of memory mapping*/ - case _IOC_NR(RTE_DOM0_IOCTL_ATTACH_TO_MEMSEG): - ret = copy_from_user(name, (void *)ioctl_param, - sizeof(char) * DOM0_NAME_MAX); - if (ret) - return -EFAULT; - - mutex_lock(&dom0_dev.data_lock); - idx = dom0_find_memdata(name); - if (idx < 0) { - mutex_unlock(&dom0_dev.data_lock); - return -EINVAL; - } - - mm_data = dom0_dev.mm_data[idx]; - mm_data->refcnt++; - file->private_data = mm_data; - mutex_unlock(&dom0_dev.data_lock); - break; - - case _IOC_NR(RTE_DOM0_IOCTL_GET_NUM_MEMSEG): - ret = copy_to_user((void *)ioctl_param, &mm_data->num_memseg, - sizeof(int)); - if (ret) - return -EFAULT; - break; - - case _IOC_NR(RTE_DOM0_IOCTL_GET_MEMSEG_INFO): - ret = copy_to_user((void *)ioctl_param, - &mm_data->seg_info[0], - sizeof(struct memseg_info) * - mm_data->num_memseg); - if (ret) - return -EFAULT; - break; - default: - XEN_PRINT("IOCTL default \n"); - break; - } - - return 0; -} - -module_init(dom0_init); -module_exit(dom0_exit); - -module_param(rsv_memsize, uint, S_IRUGO | S_IWUSR); -MODULE_PARM_DESC(rsv_memsize, "Xen-dom0 reserved memory size(MB).\n"); diff --git a/dpdk/lib/librte_eal/rte_eal_version.map b/dpdk/lib/librte_eal/rte_eal_version.map new file mode 100644 index 00000000..f4f46c1b --- /dev/null +++ b/dpdk/lib/librte_eal/rte_eal_version.map @@ -0,0 +1,238 @@ +DPDK_2.0 { + global: + + __rte_panic; + devargs_list; + eal_parse_sysfs_value; + eal_timer_source; + lcore_config; + per_lcore__lcore_id; + per_lcore__rte_errno; + rte_calloc; + rte_calloc_socket; + rte_cpu_check_supported; + rte_cpu_get_flag_enabled; + rte_cycles_vmware_tsc_map; + rte_delay_us; + rte_dump_physmem_layout; + rte_dump_registers; + rte_dump_stack; + rte_dump_tailq; + rte_eal_alarm_cancel; + rte_eal_alarm_set; + rte_eal_devargs_add; + rte_eal_devargs_dump; + rte_eal_devargs_type_count; + rte_eal_get_configuration; + rte_eal_get_lcore_state; + rte_eal_get_physmem_layout; + rte_eal_get_physmem_size; + rte_eal_has_hugepages; + rte_eal_hpet_init; + rte_eal_init; + rte_eal_iopl_init; + rte_eal_lcore_role; + rte_eal_mp_remote_launch; + rte_eal_mp_wait_lcore; + rte_eal_parse_devargs_str; + rte_eal_process_type; + rte_eal_remote_launch; + rte_eal_tailq_lookup; + rte_eal_tailq_register; + rte_eal_wait_lcore; + rte_exit; + rte_free; + rte_get_hpet_cycles; + rte_get_hpet_hz; + rte_get_tsc_hz; + rte_hexdump; + rte_intr_callback_register; + rte_intr_callback_unregister; + rte_intr_disable; + rte_intr_enable; + rte_log; + rte_log_cur_msg_loglevel; + rte_log_cur_msg_logtype; + rte_logs; + rte_malloc; + rte_malloc_dump_stats; + rte_malloc_get_socket_stats; + rte_malloc_set_limit; + rte_malloc_socket; + rte_malloc_validate; + rte_mem_lock_page; + rte_mem_virt2phy; + rte_memdump; + rte_memory_get_nchannel; + rte_memory_get_nrank; + rte_memzone_dump; + rte_memzone_lookup; + rte_memzone_reserve; + rte_memzone_reserve_aligned; + rte_memzone_reserve_bounded; + rte_memzone_walk; + rte_openlog_stream; + rte_realloc; + rte_set_application_usage_hook; + rte_socket_id; + rte_strerror; + rte_strsplit; + rte_sys_gettid; + rte_thread_get_affinity; + rte_thread_set_affinity; + rte_vlog; + rte_zmalloc; + rte_zmalloc_socket; + + local: *; +}; + +DPDK_2.1 { + global: + + rte_epoll_ctl; + rte_epoll_wait; + rte_intr_allow_others; + rte_intr_dp_is_en; + rte_intr_efd_disable; + rte_intr_efd_enable; + rte_intr_rx_ctl; + rte_intr_tls_epfd; + rte_memzone_free; + +} DPDK_2.0; + +DPDK_2.2 { + global: + + rte_intr_cap_multiple; + rte_keepalive_create; + rte_keepalive_dispatch_pings; + rte_keepalive_mark_alive; + rte_keepalive_register_core; + +} DPDK_2.1; + +DPDK_16.04 { + global: + + rte_cpu_get_flag_name; + rte_eal_primary_proc_alive; + +} DPDK_2.2; + +DPDK_16.07 { + global: + + rte_keepalive_mark_sleep; + rte_keepalive_register_relay_callback; + rte_rtm_supported; + rte_thread_setname; + +} DPDK_16.04; + +DPDK_16.11 { + global: + + rte_delay_us_block; + rte_delay_us_callback_register; + rte_eal_dev_attach; + rte_eal_dev_detach; + +} DPDK_16.07; + +DPDK_17.02 { + global: + + rte_bus_dump; + rte_bus_probe; + rte_bus_register; + rte_bus_scan; + rte_bus_unregister; + +} DPDK_16.11; + +DPDK_17.05 { + global: + + rte_cpu_is_supported; + rte_intr_free_epoll_fd; + rte_log_dump; + rte_log_get_global_level; + rte_log_register; + rte_log_set_global_level; + rte_log_set_level; + rte_log_set_level_regexp; + vfio_get_container_fd; + vfio_get_group_fd; + vfio_get_group_no; + +} DPDK_17.02; + +DPDK_17.08 { + global: + + rte_bus_find; + rte_bus_find_by_device; + rte_bus_find_by_name; + rte_log_get_level; + +} DPDK_17.05; + +DPDK_17.11 { + global: + + rte_eal_create_uio_dev; + rte_bus_get_iommu_class; + rte_eal_has_pci; + rte_eal_iova_mode; + rte_eal_mbuf_default_mempool_ops; + rte_eal_using_phys_addrs; + rte_eal_vfio_intr_mode; + rte_lcore_has_role; + rte_malloc_virt2iova; + rte_mem_virt2iova; + rte_vfio_enable; + rte_vfio_is_enabled; + rte_vfio_noiommu_is_enabled; + rte_vfio_release_device; + rte_vfio_setup_device; + +} DPDK_17.08; + +EXPERIMENTAL { + global: + + rte_eal_devargs_insert; + rte_eal_devargs_parse; + rte_eal_devargs_remove; + rte_eal_hotplug_add; + rte_eal_hotplug_remove; + rte_service_component_register; + rte_service_component_unregister; + rte_service_component_runstate_set; + rte_service_dump; + rte_service_get_by_id; + rte_service_get_by_name; + rte_service_get_count; + rte_service_get_name; + rte_service_lcore_add; + rte_service_lcore_count; + rte_service_lcore_count_services; + rte_service_lcore_del; + rte_service_lcore_list; + rte_service_lcore_reset_all; + rte_service_lcore_start; + rte_service_lcore_stop; + rte_service_map_lcore_get; + rte_service_map_lcore_set; + rte_service_probe_capability; + rte_service_reset; + rte_service_run_iter_on_app_lcore; + rte_service_runstate_get; + rte_service_runstate_set; + rte_service_set_runstate_mapped_check; + rte_service_set_stats_enable; + rte_service_start_with_defaults; + +} DPDK_17.11; diff --git a/dpdk/lib/librte_efd/Makefile b/dpdk/lib/librte_efd/Makefile new file mode 100644 index 00000000..16e450e8 --- /dev/null +++ b/dpdk/lib/librte_efd/Makefile @@ -0,0 +1,51 @@ +# BSD LICENSE +# +# Copyright(c) 2016-2017 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +include $(RTE_SDK)/mk/rte.vars.mk + +# library name +LIB = librte_efd.a + +CFLAGS += -O3 +CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) +LDLIBS += -lrte_eal -lrte_ring -lrte_hash + +EXPORT_MAP := rte_efd_version.map + +LIBABIVER := 1 + +# all source are stored in SRCS-y +SRCS-$(CONFIG_RTE_LIBRTE_EFD) := rte_efd.c + +# install this header file +SYMLINK-$(CONFIG_RTE_LIBRTE_EFD)-include := rte_efd.h + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/lib/librte_efd/rte_efd.c b/dpdk/lib/librte_efd/rte_efd.c new file mode 100644 index 00000000..7d0b5cc6 --- /dev/null +++ b/dpdk/lib/librte_efd/rte_efd.c @@ -0,0 +1,1364 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "rte_efd.h" +#if defined(RTE_ARCH_X86) +#include "rte_efd_x86.h" +#elif defined(RTE_ARCH_ARM64) +#include "rte_efd_arm64.h" +#endif + +#define EFD_KEY(key_idx, table) (table->keys + ((key_idx) * table->key_len)) +/** Hash function used to determine chunk_id and bin_id for a group */ +#define EFD_HASH(key, table) \ + (uint32_t)(rte_jhash(key, table->key_len, 0xbc9f1d34)) +/** Hash function used as constant component of perfect hash search */ +#define EFD_HASHFUNCA(key, table) \ + (uint32_t)(rte_hash_crc(key, table->key_len, 0xbc9f1d35)) +/** Hash function used as multiplicative component of perfect hash search */ +#define EFD_HASHFUNCB(key, table) \ + (uint32_t)(rte_hash_crc(key, table->key_len, 0xbc9f1d36)) + +/************************************************************************* + * Fixed constants + *************************************************************************/ + +/* These parameters are fixed by the efd_bin_to_group balancing table */ +#define EFD_CHUNK_NUM_GROUPS (64) +#define EFD_CHUNK_NUM_BINS (256) +#define EFD_CHUNK_NUM_BIN_TO_GROUP_SETS \ + (EFD_CHUNK_NUM_BINS / EFD_CHUNK_NUM_GROUPS) + +/* + * Target number of rules that each chunk is created to handle. + * Used when initially allocating the table + */ +#define EFD_TARGET_CHUNK_NUM_RULES \ + (EFD_CHUNK_NUM_GROUPS * EFD_TARGET_GROUP_NUM_RULES) +/* + * Max number of rules that each chunk is created to handle. + * Used when initially allocating the table + */ +#define EFD_TARGET_CHUNK_MAX_NUM_RULES \ + (EFD_CHUNK_NUM_GROUPS * EFD_MAX_GROUP_NUM_RULES) + +/** This is fixed based on the bin_to_group permutation array */ +#define EFD_MAX_GROUP_NUM_BINS (16) + +/** + * The end of the chunks array needs some extra padding to ensure + * that vectorization over-reads on the last online chunk stay within +allocated memory + */ +#define EFD_NUM_CHUNK_PADDING_BYTES (256) + +/* All different internal lookup functions */ +enum efd_lookup_internal_function { + EFD_LOOKUP_SCALAR = 0, + EFD_LOOKUP_AVX2, + EFD_LOOKUP_NEON, + EFD_LOOKUP_NUM +}; + +TAILQ_HEAD(rte_efd_list, rte_tailq_entry); + +static struct rte_tailq_elem rte_efd_tailq = { + .name = "RTE_EFD", +}; +EAL_REGISTER_TAILQ(rte_efd_tailq); + +/** Internal permutation array used to shuffle bins into pseudorandom groups */ +const uint32_t efd_bin_to_group[EFD_CHUNK_NUM_BIN_TO_GROUP_SETS][EFD_CHUNK_NUM_BINS] = { + { + 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, + 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, + 8, 8, 8, 8, 9, 9, 9, 9, 10, 10, 10, 10, 11, 11, 11, 11, + 12, 12, 12, 12, 13, 13, 13, 13, 14, 14, 14, 14, 15, 15, 15, 15, + 16, 16, 16, 16, 17, 17, 17, 17, 18, 18, 18, 18, 19, 19, 19, 19, + 20, 20, 20, 20, 21, 21, 21, 21, 22, 22, 22, 22, 23, 23, 23, 23, + 24, 24, 24, 24, 25, 25, 25, 25, 26, 26, 26, 26, 27, 27, 27, 27, + 28, 28, 28, 28, 29, 29, 29, 29, 30, 30, 30, 30, 31, 31, 31, 31, + 32, 32, 32, 32, 33, 33, 33, 33, 34, 34, 34, 34, 35, 35, 35, 35, + 36, 36, 36, 36, 37, 37, 37, 37, 38, 38, 38, 38, 39, 39, 39, 39, + 40, 40, 40, 40, 41, 41, 41, 41, 42, 42, 42, 42, 43, 43, 43, 43, + 44, 44, 44, 44, 45, 45, 45, 45, 46, 46, 46, 46, 47, 47, 47, 47, + 48, 48, 48, 48, 49, 49, 49, 49, 50, 50, 50, 50, 51, 51, 51, 51, + 52, 52, 52, 52, 53, 53, 53, 53, 54, 54, 54, 54, 55, 55, 55, 55, + 56, 56, 56, 56, 57, 57, 57, 57, 58, 58, 58, 58, 59, 59, 59, 59, + 60, 60, 60, 60, 61, 61, 61, 61, 62, 62, 62, 62, 63, 63, 63, 63 + }, + { + 34, 33, 48, 59, 0, 21, 36, 18, 9, 49, 54, 38, 51, 23, 31, 5, + 44, 23, 37, 52, 11, 4, 58, 20, 38, 40, 38, 22, 26, 28, 42, 6, + 46, 16, 31, 28, 46, 14, 60, 0, 35, 53, 16, 58, 16, 29, 39, 7, + 1, 54, 15, 11, 48, 3, 62, 9, 58, 5, 30, 43, 17, 7, 36, 34, + 6, 36, 2, 14, 10, 1, 47, 47, 20, 45, 62, 56, 34, 25, 39, 18, + 51, 41, 61, 25, 56, 40, 41, 37, 52, 35, 30, 57, 11, 42, 37, 27, + 54, 19, 26, 13, 48, 31, 46, 15, 12, 10, 16, 20, 43, 17, 12, 55, + 45, 18, 8, 41, 7, 31, 42, 63, 12, 14, 21, 57, 24, 40, 5, 41, + 13, 44, 23, 59, 25, 57, 52, 50, 62, 1, 2, 49, 32, 57, 26, 43, + 56, 60, 55, 5, 49, 6, 3, 50, 46, 39, 27, 33, 17, 4, 53, 13, + 2, 19, 36, 51, 63, 0, 22, 33, 59, 28, 29, 23, 45, 33, 53, 27, + 22, 21, 40, 56, 4, 18, 44, 47, 28, 17, 4, 50, 21, 62, 8, 39, + 0, 8, 15, 24, 29, 24, 9, 11, 48, 61, 35, 55, 43, 1, 54, 42, + 53, 60, 22, 3, 32, 52, 25, 8, 15, 60, 7, 55, 27, 63, 19, 10, + 63, 24, 61, 19, 12, 38, 6, 29, 13, 37, 10, 3, 45, 32, 32, 30, + 49, 61, 44, 14, 20, 58, 35, 30, 2, 26, 34, 51, 9, 59, 47, 50 + }, + { + 32, 35, 32, 34, 55, 5, 6, 23, 49, 11, 6, 23, 52, 37, 29, 54, + 55, 40, 63, 50, 29, 52, 61, 25, 12, 56, 39, 38, 29, 11, 46, 1, + 40, 11, 19, 56, 7, 28, 51, 16, 15, 48, 21, 51, 60, 31, 14, 22, + 41, 47, 59, 56, 53, 28, 58, 26, 43, 27, 41, 33, 24, 52, 44, 38, + 13, 59, 48, 51, 60, 15, 3, 30, 15, 0, 10, 62, 44, 14, 28, 51, + 38, 2, 41, 26, 25, 49, 10, 12, 55, 57, 27, 35, 19, 33, 0, 30, + 5, 36, 47, 53, 5, 53, 20, 43, 34, 37, 52, 41, 21, 63, 59, 9, + 24, 1, 45, 24, 39, 44, 45, 16, 9, 17, 7, 50, 57, 22, 18, 28, + 25, 45, 2, 40, 58, 15, 17, 3, 1, 27, 61, 39, 19, 0, 19, 21, + 57, 62, 54, 60, 54, 40, 48, 33, 36, 37, 4, 42, 1, 43, 58, 8, + 13, 42, 10, 56, 35, 22, 48, 61, 63, 10, 49, 9, 24, 9, 25, 57, + 33, 18, 13, 31, 42, 36, 36, 55, 30, 37, 53, 34, 59, 4, 4, 23, + 8, 16, 58, 14, 30, 11, 12, 63, 49, 62, 2, 39, 47, 22, 2, 60, + 18, 8, 46, 31, 6, 20, 32, 29, 46, 42, 20, 31, 32, 61, 34, 4, + 47, 26, 20, 43, 26, 21, 7, 3, 16, 35, 18, 44, 27, 62, 13, 23, + 6, 50, 12, 8, 45, 17, 3, 46, 50, 7, 14, 5, 17, 54, 38, 0 + }, + { + 29, 56, 5, 7, 54, 48, 23, 37, 35, 44, 52, 40, 33, 49, 60, 0, + 59, 51, 28, 12, 41, 26, 2, 23, 34, 5, 59, 40, 3, 19, 6, 26, + 35, 53, 45, 49, 29, 57, 28, 62, 58, 59, 19, 53, 59, 62, 6, 54, + 13, 15, 48, 50, 45, 21, 41, 12, 34, 40, 24, 56, 19, 21, 35, 18, + 55, 45, 9, 61, 47, 61, 19, 15, 16, 39, 17, 31, 3, 51, 21, 50, + 17, 25, 25, 11, 44, 16, 18, 28, 14, 2, 37, 61, 58, 27, 62, 4, + 14, 17, 1, 9, 46, 28, 37, 0, 53, 43, 57, 7, 57, 46, 21, 41, + 39, 14, 52, 60, 44, 53, 49, 60, 49, 63, 13, 11, 29, 1, 55, 47, + 55, 12, 60, 43, 54, 37, 13, 6, 42, 10, 36, 13, 9, 8, 34, 51, + 31, 32, 12, 7, 57, 2, 26, 14, 3, 30, 63, 3, 32, 1, 5, 11, + 27, 24, 26, 44, 31, 23, 56, 38, 62, 0, 40, 30, 6, 23, 38, 2, + 47, 5, 15, 27, 16, 10, 31, 25, 22, 63, 30, 25, 20, 33, 32, 50, + 29, 43, 55, 10, 50, 45, 56, 20, 4, 7, 27, 46, 11, 16, 22, 52, + 35, 20, 41, 54, 46, 33, 42, 18, 63, 8, 22, 58, 36, 4, 51, 42, + 38, 32, 38, 22, 17, 0, 47, 8, 48, 8, 48, 1, 61, 36, 33, 20, + 24, 39, 39, 18, 30, 36, 9, 43, 42, 24, 10, 58, 4, 15, 34, 52 + }, +}; + +/************************************************************************* + * Offline region structures + *************************************************************************/ + +/** Online group containing number of rules, values, keys and their bins + * for EFD_MAX_GROUP_NUM_RULES rules. + */ +struct efd_offline_group_rules { + uint32_t num_rules; + /**< Sum of the number of rules in all bins assigned to this group. */ + + uint32_t key_idx[EFD_MAX_GROUP_NUM_RULES]; + /**< Array with all keys of the group. */ + efd_value_t value[EFD_MAX_GROUP_NUM_RULES]; + /**< Array with all values of the keys of the group. */ + + uint8_t bin_id[EFD_MAX_GROUP_NUM_RULES]; + /**< Stores the bin for each correspending key to + * avoid having to recompute it + */ +}; + +/** Offline chunk record, containing EFD_TARGET_CHUNK_NUM_RULES rules. + * Those rules are split into EFD_CHUNK_NUM_GROUPS groups per chunk. + */ +struct efd_offline_chunk_rules { + uint16_t num_rules; + /**< Number of rules in the entire chunk; + * used to detect unbalanced groups + */ + + struct efd_offline_group_rules group_rules[EFD_CHUNK_NUM_GROUPS]; + /**< Array of all groups in the chunk. */ +}; + +/************************************************************************* + * Online region structures + *************************************************************************/ + +/** Online group containing values for EFD_MAX_GROUP_NUM_RULES rules. */ +struct efd_online_group_entry { + efd_hashfunc_t hash_idx[RTE_EFD_VALUE_NUM_BITS]; + efd_lookuptbl_t lookup_table[RTE_EFD_VALUE_NUM_BITS]; +} __attribute__((__packed__)); + +/** + * A single chunk record, containing EFD_TARGET_CHUNK_NUM_RULES rules. + * Those rules are split into EFD_CHUNK_NUM_GROUPS groups per chunk. + */ +struct efd_online_chunk { + uint8_t bin_choice_list[(EFD_CHUNK_NUM_BINS * 2 + 7) / 8]; + /**< This is a packed indirection index into the 'groups' array. + * Each byte contains four two-bit values which index into + * the efd_bin_to_group array. + * The efd_bin_to_group array returns the index into the groups array + */ + + struct efd_online_group_entry groups[EFD_CHUNK_NUM_GROUPS]; + /**< Array of all the groups in the chunk. */ +} __attribute__((__packed__)); + +/** + * EFD table structure + */ +struct rte_efd_table { + char name[RTE_EFD_NAMESIZE]; /**< Name of the efd table. */ + + uint32_t key_len; /**< Length of the key stored offline */ + + uint32_t max_num_rules; + /**< Static maximum number of entries the table was constructed to hold. */ + + uint32_t num_rules; + /**< Number of entries currently in the table . */ + + uint32_t num_chunks; + /**< Number of chunks in the table needed to support num_rules. */ + + uint32_t num_chunks_shift; + /**< Bits to shift to get chunk id, instead of dividing by num_chunk. */ + + enum efd_lookup_internal_function lookup_fn; + /**< Indicates which lookup function to use. */ + + struct efd_online_chunk *chunks[RTE_MAX_NUMA_NODES]; + /**< Dynamic array of size num_chunks of chunk records. */ + + struct efd_offline_chunk_rules *offline_chunks; + /**< Dynamic array of size num_chunks of key-value pairs. */ + + struct rte_ring *free_slots; + /**< Ring that stores all indexes of the free slots in the key table */ + + uint8_t *keys; /**< Dynamic array of size max_num_rules of keys */ +}; + +/** + * Computes the chunk ID for a given key hash + * + * @param table + * EFD table to reference + * @param hashed_key + * 32-bit key hash returned by EFD_HASH + * + * @return + * chunk ID containing this key hash + */ +static inline uint32_t +efd_get_chunk_id(const struct rte_efd_table * const table, + const uint32_t hashed_key) +{ + return hashed_key & (table->num_chunks - 1); +} + +/** + * Computes the bin ID for a given key hash + * + * @param table + * EFD table to reference + * @param hashed_key + * 32-bit key hash returned by EFD_HASH + * + * @return bin ID containing this key hash + */ +static inline uint32_t +efd_get_bin_id(const struct rte_efd_table * const table, + const uint32_t hashed_key) +{ + return (hashed_key >> table->num_chunks_shift) & (EFD_CHUNK_NUM_BINS - 1); +} + +/** + * Looks up the current permutation choice for a particular bin in the online table + * + * @param table + * EFD table to reference + * @param socket_id + * Socket ID to use to look up existing values (ideally caller's socket id) + * @param chunk_id + * Chunk ID of bin to look up + * @param bin_id + * Bin ID to look up + * + * @return + * Currently active permutation choice in the online table + */ +static inline uint8_t +efd_get_choice(const struct rte_efd_table * const table, + const unsigned int socket_id, const uint32_t chunk_id, + const uint32_t bin_id) +{ + struct efd_online_chunk *chunk = &table->chunks[socket_id][chunk_id]; + + /* + * Grab the chunk (byte) that contains the choices + * for four neighboring bins. + */ + uint8_t choice_chunk = + chunk->bin_choice_list[bin_id / EFD_CHUNK_NUM_BIN_TO_GROUP_SETS]; + + /* + * Compute the offset into the chunk that contains + * the group_id lookup position + */ + int offset = (bin_id & 0x3) * 2; + + /* Extract from the byte just the desired lookup position */ + return (uint8_t) ((choice_chunk >> offset) & 0x3); +} + +/** + * Compute the chunk_id and bin_id for a given key + * + * @param table + * EFD table to reference + * @param key + * Key to hash and find location of + * @param chunk_id + * Computed chunk ID + * @param bin_id + * Computed bin ID + * + */ +static inline void +efd_compute_ids(const struct rte_efd_table * const table, + const void *key, uint32_t * const chunk_id, uint32_t * const bin_id) +{ + /* Compute the position of the entry in the hash table */ + uint32_t h = EFD_HASH(key, table); + + /* Compute the chunk_id where that entry can be found */ + *chunk_id = efd_get_chunk_id(table, h); + + /* + * Compute the bin within that chunk where the entry + * can be found (0 - 255) + */ + *bin_id = efd_get_bin_id(table, h); +} + +/** + * Search for a hash function for a group that satisfies all group results + */ +static inline int +efd_search_hash(struct rte_efd_table * const table, + const struct efd_offline_group_rules * const off_group, + struct efd_online_group_entry * const on_group) +{ + efd_hashfunc_t hash_idx; + efd_hashfunc_t start_hash_idx[RTE_EFD_VALUE_NUM_BITS]; + efd_lookuptbl_t start_lookup_table[RTE_EFD_VALUE_NUM_BITS]; + + uint32_t i, j, rule_id; + uint32_t hash_val_a[EFD_MAX_GROUP_NUM_RULES]; + uint32_t hash_val_b[EFD_MAX_GROUP_NUM_RULES]; + uint32_t hash_val[EFD_MAX_GROUP_NUM_RULES]; + + + rte_prefetch0(off_group->value); + + /* + * Prepopulate the hash_val tables by running the two hash functions + * for each provided rule + */ + for (i = 0; i < off_group->num_rules; i++) { + void *key_stored = EFD_KEY(off_group->key_idx[i], table); + hash_val_b[i] = EFD_HASHFUNCB(key_stored, table); + hash_val_a[i] = EFD_HASHFUNCA(key_stored, table); + } + + for (i = 0; i < RTE_EFD_VALUE_NUM_BITS; i++) { + hash_idx = on_group->hash_idx[i]; + start_hash_idx[i] = hash_idx; + start_lookup_table[i] = on_group->lookup_table[i]; + + do { + efd_lookuptbl_t lookup_table = 0; + efd_lookuptbl_t lookup_table_complement = 0; + + for (rule_id = 0; rule_id < off_group->num_rules; rule_id++) + hash_val[rule_id] = hash_val_a[rule_id] + (hash_idx * + hash_val_b[rule_id]); + + /* + * The goal here is to find a hash function for this + * particular bit entry that meets the following criteria: + * The most significant bits of the hash result define a + * shift into the lookup table where the bit will be stored + */ + + /* Iterate over each provided rule */ + for (rule_id = 0; rule_id < off_group->num_rules; + rule_id++) { + /* + * Use the few most significant bits (number based on + * EFD_LOOKUPTBL_SIZE) to see what position the + * expected bit should be set in the lookup_table + */ + uint32_t bucket_idx = hash_val[rule_id] >> + EFD_LOOKUPTBL_SHIFT; + + /* + * Get the current bit of interest. + * This only find an appropriate hash function + * for one bit at a time of the rule + */ + efd_lookuptbl_t expected = + (off_group->value[rule_id] >> i) & 0x1; + + /* + * Add the expected bit (if set) to a map + * (lookup_table). Also set its complement + * in lookup_table_complement + */ + lookup_table |= expected << bucket_idx; + lookup_table_complement |= (1 - expected) + << bucket_idx; + + /* + * If ever the hash function of two different + * elements result in different values at the + * same location in the lookup_table, + * the current hash_idx is not valid. + */ + if (lookup_table & lookup_table_complement) + break; + } + + /* + * Check if the previous loop completed without + * breaking early + */ + if (rule_id == off_group->num_rules) { + /* + * Current hash function worked, store it + * for the current group + */ + on_group->hash_idx[i] = hash_idx; + on_group->lookup_table[i] = lookup_table; + + /* + * Make sure that the hash function has changed + * from the starting value + */ + hash_idx = start_hash_idx[i] + 1; + break; + } + hash_idx++; + + } while (hash_idx != start_hash_idx[i]); + + /* Failed to find perfect hash for this group */ + if (hash_idx == start_hash_idx[i]) { + /* + * Restore previous hash_idx and lookup_table + * for all value bits + */ + for (j = 0; j < i; j++) { + on_group->hash_idx[j] = start_hash_idx[j]; + on_group->lookup_table[j] = start_lookup_table[j]; + } + return 1; + } + } + + return 0; +} + +struct rte_efd_table * +rte_efd_create(const char *name, uint32_t max_num_rules, uint32_t key_len, + uint8_t online_cpu_socket_bitmask, uint8_t offline_cpu_socket) +{ + struct rte_efd_table *table = NULL; + uint8_t *key_array = NULL; + uint32_t num_chunks, num_chunks_shift; + uint8_t socket_id; + struct rte_efd_list *efd_list = NULL; + struct rte_tailq_entry *te; + uint64_t offline_table_size; + char ring_name[RTE_RING_NAMESIZE]; + struct rte_ring *r = NULL; + unsigned int i; + + efd_list = RTE_TAILQ_CAST(rte_efd_tailq.head, rte_efd_list); + + if (online_cpu_socket_bitmask == 0) { + RTE_LOG(ERR, EFD, "At least one CPU socket must be enabled " + "in the bitmask\n"); + return NULL; + } + + if (max_num_rules == 0) { + RTE_LOG(ERR, EFD, "Max num rules must be higher than 0\n"); + return NULL; + } + + /* + * Compute the minimum number of chunks (smallest power of 2) + * that can hold all of the rules + */ + if (max_num_rules % EFD_TARGET_CHUNK_NUM_RULES == 0) + num_chunks = rte_align32pow2(max_num_rules / + EFD_TARGET_CHUNK_NUM_RULES); + else + num_chunks = rte_align32pow2((max_num_rules / + EFD_TARGET_CHUNK_NUM_RULES) + 1); + + num_chunks_shift = rte_bsf32(num_chunks); + + rte_rwlock_write_lock(RTE_EAL_TAILQ_RWLOCK); + + /* + * Guarantee there's no existing: this is normally already checked + * by ring creation above + */ + TAILQ_FOREACH(te, efd_list, next) + { + table = (struct rte_efd_table *) te->data; + if (strncmp(name, table->name, RTE_EFD_NAMESIZE) == 0) + break; + } + + table = NULL; + if (te != NULL) { + rte_errno = EEXIST; + te = NULL; + goto error_unlock_exit; + } + + te = rte_zmalloc("EFD_TAILQ_ENTRY", sizeof(*te), 0); + if (te == NULL) { + RTE_LOG(ERR, EFD, "tailq entry allocation failed\n"); + goto error_unlock_exit; + } + + /* Create a new EFD table management structure */ + table = (struct rte_efd_table *) rte_zmalloc_socket(NULL, + sizeof(struct rte_efd_table), + RTE_CACHE_LINE_SIZE, + offline_cpu_socket); + if (table == NULL) { + RTE_LOG(ERR, EFD, "Allocating EFD table management structure" + " on socket %u failed\n", + offline_cpu_socket); + goto error_unlock_exit; + } + + + RTE_LOG(DEBUG, EFD, "Allocated EFD table management structure " + "on socket %u\n", offline_cpu_socket); + + table->max_num_rules = num_chunks * EFD_TARGET_CHUNK_MAX_NUM_RULES; + table->num_rules = 0; + table->num_chunks = num_chunks; + table->num_chunks_shift = num_chunks_shift; + table->key_len = key_len; + + /* key_array */ + key_array = (uint8_t *) rte_zmalloc_socket(NULL, + table->max_num_rules * table->key_len, + RTE_CACHE_LINE_SIZE, + offline_cpu_socket); + if (key_array == NULL) { + RTE_LOG(ERR, EFD, "Allocating key array" + " on socket %u failed\n", + offline_cpu_socket); + goto error_unlock_exit; + } + table->keys = key_array; + snprintf(table->name, sizeof(table->name), "%s", name); + + RTE_LOG(DEBUG, EFD, "Creating an EFD table with %u chunks," + " which potentially supports %u entries\n", + num_chunks, table->max_num_rules); + + /* Make sure all the allocatable table pointers are NULL initially */ + for (socket_id = 0; socket_id < RTE_MAX_NUMA_NODES; socket_id++) + table->chunks[socket_id] = NULL; + table->offline_chunks = NULL; + + /* + * Allocate one online table per socket specified + * in the user-supplied bitmask + */ + uint64_t online_table_size = num_chunks * sizeof(struct efd_online_chunk) + + EFD_NUM_CHUNK_PADDING_BYTES; + + for (socket_id = 0; socket_id < RTE_MAX_NUMA_NODES; socket_id++) { + if ((online_cpu_socket_bitmask >> socket_id) & 0x01) { + /* + * Allocate all of the EFD table chunks (the online portion) + * as a continuous block + */ + table->chunks[socket_id] = + (struct efd_online_chunk *) rte_zmalloc_socket( + NULL, + online_table_size, + RTE_CACHE_LINE_SIZE, + socket_id); + if (table->chunks[socket_id] == NULL) { + RTE_LOG(ERR, EFD, + "Allocating EFD online table on " + "socket %u failed\n", + socket_id); + goto error_unlock_exit; + } + RTE_LOG(DEBUG, EFD, + "Allocated EFD online table of size " + "%"PRIu64" bytes (%.2f MB) on socket %u\n", + online_table_size, + (float) online_table_size / + (1024.0F * 1024.0F), + socket_id); + } + } + +#if defined(RTE_ARCH_X86) + /* + * For less than 4 bits, scalar function performs better + * than vectorised version + */ + if (RTE_EFD_VALUE_NUM_BITS > 3 && rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2)) + table->lookup_fn = EFD_LOOKUP_AVX2; + else +#endif +#if defined(RTE_ARCH_ARM64) + /* + * For less than or equal to 16 bits, scalar function performs better + * than vectorised version + */ + if (RTE_EFD_VALUE_NUM_BITS > 16 && + rte_cpu_get_flag_enabled(RTE_CPUFLAG_NEON)) + table->lookup_fn = EFD_LOOKUP_NEON; + else +#endif + table->lookup_fn = EFD_LOOKUP_SCALAR; + + /* + * Allocate the EFD table offline portion (with the actual rules + * mapping keys to values) as a continuous block. + * This could be several gigabytes of memory. + */ + offline_table_size = num_chunks * sizeof(struct efd_offline_chunk_rules); + table->offline_chunks = + (struct efd_offline_chunk_rules *) rte_zmalloc_socket(NULL, + offline_table_size, + RTE_CACHE_LINE_SIZE, + offline_cpu_socket); + if (table->offline_chunks == NULL) { + RTE_LOG(ERR, EFD, "Allocating EFD offline table on socket %u " + "failed\n", offline_cpu_socket); + goto error_unlock_exit; + } + + RTE_LOG(DEBUG, EFD, + "Allocated EFD offline table of size %"PRIu64" bytes " + " (%.2f MB) on socket %u\n", offline_table_size, + (float) offline_table_size / (1024.0F * 1024.0F), + offline_cpu_socket); + + te->data = (void *) table; + TAILQ_INSERT_TAIL(efd_list, te, next); + rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK); + + snprintf(ring_name, sizeof(ring_name), "HT_%s", table->name); + /* Create ring (Dummy slot index is not enqueued) */ + r = rte_ring_create(ring_name, rte_align32pow2(table->max_num_rules), + offline_cpu_socket, 0); + if (r == NULL) { + RTE_LOG(ERR, EFD, "memory allocation failed\n"); + goto error_unlock_exit; + } + + /* Populate free slots ring. Entry zero is reserved for key misses. */ + for (i = 0; i < table->max_num_rules; i++) + rte_ring_sp_enqueue(r, (void *) ((uintptr_t) i)); + + table->free_slots = r; + return table; + +error_unlock_exit: + rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK); + rte_efd_free(table); + + return NULL; +} + +struct rte_efd_table * +rte_efd_find_existing(const char *name) +{ + struct rte_efd_table *table = NULL; + struct rte_tailq_entry *te; + struct rte_efd_list *efd_list; + + efd_list = RTE_TAILQ_CAST(rte_efd_tailq.head, rte_efd_list); + + rte_rwlock_read_lock(RTE_EAL_TAILQ_RWLOCK); + + TAILQ_FOREACH(te, efd_list, next) + { + table = (struct rte_efd_table *) te->data; + if (strncmp(name, table->name, RTE_EFD_NAMESIZE) == 0) + break; + } + rte_rwlock_read_unlock(RTE_EAL_TAILQ_RWLOCK); + + if (te == NULL) { + rte_errno = ENOENT; + return NULL; + } + return table; +} + +void +rte_efd_free(struct rte_efd_table *table) +{ + uint8_t socket_id; + + if (table == NULL) + return; + + for (socket_id = 0; socket_id < RTE_MAX_NUMA_NODES; socket_id++) + rte_free(table->chunks[socket_id]); + + rte_ring_free(table->free_slots); + rte_free(table->offline_chunks); + rte_free(table->keys); + rte_free(table); +} + +/** + * Applies a previously computed table entry to the specified table for all + * socket-local copies of the online table. + * Intended to apply an update for only a single change + * to a key/value pair at a time + * + * @param table + * EFD table to reference + * @param socket_id + * Socket ID to use to lookup existing values (ideally caller's socket id) + * @param chunk_id + * Chunk index to update + * @param group_id + * Group index to update + * @param bin_id + * Bin within the group that this update affects + * @param new_bin_choice + * Newly chosen permutation which this bin should use - only lower 2 bits + * @param new_group_entry + * Previously computed updated chunk/group entry + */ +static inline void +efd_apply_update(struct rte_efd_table * const table, const unsigned int socket_id, + const uint32_t chunk_id, const uint32_t group_id, + const uint32_t bin_id, const uint8_t new_bin_choice, + const struct efd_online_group_entry * const new_group_entry) +{ + int i; + struct efd_online_chunk *chunk = &table->chunks[socket_id][chunk_id]; + uint8_t bin_index = bin_id / EFD_CHUNK_NUM_BIN_TO_GROUP_SETS; + + /* + * Grab the current byte that contains the choices + * for four neighboring bins + */ + uint8_t choice_chunk = + chunk->bin_choice_list[bin_index]; + + + /* Compute the offset into the chunk that needs to be updated */ + int offset = (bin_id & 0x3) * 2; + + /* Zero the two bits of interest and set them to new_bin_choice */ + choice_chunk = (choice_chunk & (~(0x03 << offset))) + | ((new_bin_choice & 0x03) << offset); + + /* Update the online table with the new data across all sockets */ + for (i = 0; i < RTE_MAX_NUMA_NODES; i++) { + if (table->chunks[i] != NULL) { + memcpy(&(table->chunks[i][chunk_id].groups[group_id]), + new_group_entry, + sizeof(struct efd_online_group_entry)); + table->chunks[i][chunk_id].bin_choice_list[bin_index] = + choice_chunk; + } + } +} + +/* + * Move the bin from prev group to the new group + */ +static inline void +move_groups(uint32_t bin_id, uint8_t bin_size, + struct efd_offline_group_rules *new_group, + struct efd_offline_group_rules * const current_group) +{ + + uint8_t empty_idx = 0; + unsigned int i; + + if (new_group == current_group) + return; + + for (i = 0; i < current_group->num_rules; i++) { + /* + * Move keys that belong to the same bin + * to the new group + */ + if (current_group->bin_id[i] == bin_id) { + new_group->key_idx[new_group->num_rules] = + current_group->key_idx[i]; + new_group->value[new_group->num_rules] = + current_group->value[i]; + new_group->bin_id[new_group->num_rules] = + current_group->bin_id[i]; + new_group->num_rules++; + } else { + if (i != empty_idx) { + /* + * Need to move this key towards + * the top of the array + */ + current_group->key_idx[empty_idx] = + current_group->key_idx[i]; + current_group->value[empty_idx] = + current_group->value[i]; + current_group->bin_id[empty_idx] = + current_group->bin_id[i]; + } + empty_idx++; + } + + } + current_group->num_rules -= bin_size; +} + +/* + * Revert group/s to their previous state before + * trying to insert/add a new key + */ +static inline void +revert_groups(struct efd_offline_group_rules *previous_group, + struct efd_offline_group_rules *current_group, uint8_t bin_size) +{ + unsigned int i; + + if (current_group == previous_group) + return; + + /* Move keys back to previous group */ + for (i = current_group->num_rules - bin_size; + i < current_group->num_rules; i++) { + previous_group->key_idx[previous_group->num_rules] = + current_group->key_idx[i]; + previous_group->value[previous_group->num_rules] = + current_group->value[i]; + previous_group->bin_id[previous_group->num_rules] = + current_group->bin_id[i]; + previous_group->num_rules++; + } + + /* + * Decrease number of rules after the move + * in the new group + */ + current_group->num_rules -= bin_size; +} + +/** + * Computes an updated table entry where the supplied key points to a new host. + * If no entry exists, one is inserted. + * + * This function does NOT modify the online table(s) + * This function DOES modify the offline table + * + * @param table + * EFD table to reference + * @param socket_id + * Socket ID to use to lookup existing values (ideally caller's socket id) + * @param key + * Key to insert + * @param value + * Value to associate with key + * @param chunk_id + * Chunk ID of the chunk that was modified + * @param group_id + * Group ID of the group that was modified + * @param bin_id + * Bin ID that was modified + * @param new_bin_choice + * Newly chosen permutation which this bin will use + * @param entry + * Newly computed online entry to apply later with efd_apply_update + * + * @return + * RTE_EFD_UPDATE_WARN_GROUP_FULL + * Operation is insert, and the last available space in the + * key's group was just used. Future inserts may fail as groups fill up. + * This operation was still successful, and entry contains a valid update + * RTE_EFD_UPDATE_FAILED + * Either the EFD failed to find a suitable perfect hash or the group was full + * This is a fatal error, and the table is now in an indeterminate state + * RTE_EFD_UPDATE_NO_CHANGE + * Operation resulted in no change to the table (same value already exists) + * 0 + * Insert or update was successful, and the new efd_online_group_entry + * is stored in *entry + * + * @warning + * Note that entry will be UNCHANGED if the update has no effect, and thus any + * subsequent use of the entry content will likely be invalid + */ +static inline int +efd_compute_update(struct rte_efd_table * const table, + const unsigned int socket_id, const void *key, + const efd_value_t value, uint32_t * const chunk_id, + uint32_t * const group_id, uint32_t * const bin_id, + uint8_t * const new_bin_choice, + struct efd_online_group_entry * const entry) +{ + unsigned int i; + int ret; + uint32_t new_idx; + void *new_k, *slot_id = NULL; + int status = EXIT_SUCCESS; + unsigned int found = 0; + + efd_compute_ids(table, key, chunk_id, bin_id); + + struct efd_offline_chunk_rules * const chunk = + &table->offline_chunks[*chunk_id]; + struct efd_offline_group_rules *new_group; + + uint8_t current_choice = efd_get_choice(table, socket_id, + *chunk_id, *bin_id); + uint32_t current_group_id = efd_bin_to_group[current_choice][*bin_id]; + struct efd_offline_group_rules * const current_group = + &chunk->group_rules[current_group_id]; + uint8_t bin_size = 0; + uint8_t key_changed_index = 0; + efd_value_t key_changed_previous_value = 0; + uint32_t key_idx_previous = 0; + + /* Scan the current group and see if the key is already present */ + for (i = 0; i < current_group->num_rules; i++) { + if (current_group->bin_id[i] == *bin_id) + bin_size++; + else + continue; + + void *key_stored = EFD_KEY(current_group->key_idx[i], table); + if (found == 0 && unlikely(memcmp(key_stored, key, + table->key_len) == 0)) { + /* Key is already present */ + + /* + * If previous value is same as new value, + * no additional work is required + */ + if (current_group->value[i] == value) + return RTE_EFD_UPDATE_NO_CHANGE; + + key_idx_previous = current_group->key_idx[i]; + key_changed_previous_value = current_group->value[i]; + key_changed_index = i; + current_group->value[i] = value; + found = 1; + } + } + + if (found == 0) { + /* Key does not exist. Insert the rule into the bin/group */ + if (unlikely(current_group->num_rules >= EFD_MAX_GROUP_NUM_RULES)) { + RTE_LOG(ERR, EFD, + "Fatal: No room remaining for insert into " + "chunk %u group %u bin %u\n", + *chunk_id, + current_group_id, *bin_id); + return RTE_EFD_UPDATE_FAILED; + } + + if (unlikely(current_group->num_rules == + (EFD_MAX_GROUP_NUM_RULES - 1))) { + RTE_LOG(INFO, EFD, "Warn: Insert into last " + "available slot in chunk %u " + "group %u bin %u\n", *chunk_id, + current_group_id, *bin_id); + status = RTE_EFD_UPDATE_WARN_GROUP_FULL; + } + + if (rte_ring_sc_dequeue(table->free_slots, &slot_id) != 0) + return RTE_EFD_UPDATE_FAILED; + + new_k = RTE_PTR_ADD(table->keys, (uintptr_t) slot_id * + table->key_len); + rte_prefetch0(new_k); + new_idx = (uint32_t) ((uintptr_t) slot_id); + + rte_memcpy(EFD_KEY(new_idx, table), key, table->key_len); + current_group->key_idx[current_group->num_rules] = new_idx; + current_group->value[current_group->num_rules] = value; + current_group->bin_id[current_group->num_rules] = *bin_id; + current_group->num_rules++; + table->num_rules++; + bin_size++; + } else { + uint32_t last = current_group->num_rules - 1; + /* Swap the key with the last key inserted*/ + current_group->key_idx[key_changed_index] = + current_group->key_idx[last]; + current_group->value[key_changed_index] = + current_group->value[last]; + current_group->bin_id[key_changed_index] = + current_group->bin_id[last]; + + /* + * Key to be updated will always be available + * at the end of the group + */ + current_group->key_idx[last] = key_idx_previous; + current_group->value[last] = value; + current_group->bin_id[last] = *bin_id; + } + + *new_bin_choice = current_choice; + *group_id = current_group_id; + new_group = current_group; + + /* Group need to be rebalanced when it starts to get loaded */ + if (current_group->num_rules > EFD_MIN_BALANCED_NUM_RULES) { + + /* + * Subtract the number of entries in the bin from + * the original group + */ + current_group->num_rules -= bin_size; + + /* + * Figure out which of the available groups that this bin + * can map to is the smallest (using the current group + * as baseline) + */ + uint8_t smallest_choice = current_choice; + uint8_t smallest_size = current_group->num_rules; + uint32_t smallest_group_id = current_group_id; + unsigned char choice; + + for (choice = 0; choice < EFD_CHUNK_NUM_BIN_TO_GROUP_SETS; + choice++) { + uint32_t test_group_id = + efd_bin_to_group[choice][*bin_id]; + uint32_t num_rules = + chunk->group_rules[test_group_id].num_rules; + if (num_rules < smallest_size) { + smallest_choice = choice; + smallest_size = num_rules; + smallest_group_id = test_group_id; + } + } + + *new_bin_choice = smallest_choice; + *group_id = smallest_group_id; + new_group = &chunk->group_rules[smallest_group_id]; + current_group->num_rules += bin_size; + + } + + uint8_t choice = 0; + for (;;) { + if (current_group != new_group && + new_group->num_rules + bin_size > + EFD_MAX_GROUP_NUM_RULES) { + RTE_LOG(DEBUG, EFD, + "Unable to move_groups to dest group " + "containing %u entries." + "bin_size:%u choice:%02x\n", + new_group->num_rules, bin_size, + choice - 1); + goto next_choice; + } + move_groups(*bin_id, bin_size, new_group, current_group); + /* + * Recompute the hash function for the modified group, + * and return it to the caller + */ + ret = efd_search_hash(table, new_group, entry); + + if (!ret) + return status; + + RTE_LOG(DEBUG, EFD, + "Failed to find perfect hash for group " + "containing %u entries. bin_size:%u choice:%02x\n", + new_group->num_rules, bin_size, choice - 1); + /* Restore groups modified to their previous state */ + revert_groups(current_group, new_group, bin_size); + +next_choice: + if (choice == EFD_CHUNK_NUM_BIN_TO_GROUP_SETS) + break; + *new_bin_choice = choice; + *group_id = efd_bin_to_group[choice][*bin_id]; + new_group = &chunk->group_rules[*group_id]; + choice++; + } + + if (!found) { + current_group->num_rules--; + table->num_rules--; + } else + current_group->value[current_group->num_rules - 1] = + key_changed_previous_value; + return RTE_EFD_UPDATE_FAILED; +} + +int +rte_efd_update(struct rte_efd_table * const table, const unsigned int socket_id, + const void *key, const efd_value_t value) +{ + uint32_t chunk_id = 0, group_id = 0, bin_id = 0; + uint8_t new_bin_choice = 0; + struct efd_online_group_entry entry; + + int status = efd_compute_update(table, socket_id, key, value, + &chunk_id, &group_id, &bin_id, + &new_bin_choice, &entry); + + if (status == RTE_EFD_UPDATE_NO_CHANGE) + return EXIT_SUCCESS; + + if (status == RTE_EFD_UPDATE_FAILED) + return status; + + efd_apply_update(table, socket_id, chunk_id, group_id, bin_id, + new_bin_choice, &entry); + return status; +} + +int +rte_efd_delete(struct rte_efd_table * const table, const unsigned int socket_id, + const void *key, efd_value_t * const prev_value) +{ + unsigned int i; + uint32_t chunk_id, bin_id; + uint8_t not_found = 1; + + efd_compute_ids(table, key, &chunk_id, &bin_id); + + struct efd_offline_chunk_rules * const chunk = + &table->offline_chunks[chunk_id]; + + uint8_t current_choice = efd_get_choice(table, socket_id, + chunk_id, bin_id); + uint32_t current_group_id = efd_bin_to_group[current_choice][bin_id]; + struct efd_offline_group_rules * const current_group = + &chunk->group_rules[current_group_id]; + + /* + * Search the current group for the specified key. + * If it exists, remove it and re-pack the other values + */ + for (i = 0; i < current_group->num_rules; i++) { + if (not_found) { + /* Found key that needs to be removed */ + if (memcmp(EFD_KEY(current_group->key_idx[i], table), + key, table->key_len) == 0) { + /* Store previous value if requested by caller */ + if (prev_value != NULL) + *prev_value = current_group->value[i]; + + not_found = 0; + rte_ring_sp_enqueue(table->free_slots, + (void *)((uintptr_t)current_group->key_idx[i])); + } + } else { + /* + * If the desired key has been found, + * need to shift other values up one + */ + + /* Need to shift this entry back up one index */ + current_group->key_idx[i - 1] = current_group->key_idx[i]; + current_group->value[i - 1] = current_group->value[i]; + current_group->bin_id[i - 1] = current_group->bin_id[i]; + } + } + + if (not_found == 0) { + table->num_rules--; + current_group->num_rules--; + } + + return not_found; +} + +static inline efd_value_t +efd_lookup_internal_scalar(const efd_hashfunc_t *group_hash_idx, + const efd_lookuptbl_t *group_lookup_table, + const uint32_t hash_val_a, const uint32_t hash_val_b) +{ + efd_value_t value = 0; + uint32_t i; + + for (i = 0; i < RTE_EFD_VALUE_NUM_BITS; i++) { + value <<= 1; + uint32_t h = hash_val_a + (hash_val_b * + group_hash_idx[RTE_EFD_VALUE_NUM_BITS - i - 1]); + uint16_t bucket_idx = h >> EFD_LOOKUPTBL_SHIFT; + value |= (group_lookup_table[ + RTE_EFD_VALUE_NUM_BITS - i - 1] >> + bucket_idx) & 0x1; + } + + return value; +} + + +static inline efd_value_t +efd_lookup_internal(const struct efd_online_group_entry * const group, + const uint32_t hash_val_a, const uint32_t hash_val_b, + enum efd_lookup_internal_function lookup_fn) +{ + efd_value_t value = 0; + + switch (lookup_fn) { + +#if defined(RTE_ARCH_X86) && defined(CC_SUPPORT_AVX2) + case EFD_LOOKUP_AVX2: + return efd_lookup_internal_avx2(group->hash_idx, + group->lookup_table, + hash_val_a, + hash_val_b); + break; +#endif +#if defined(RTE_ARCH_ARM64) + case EFD_LOOKUP_NEON: + return efd_lookup_internal_neon(group->hash_idx, + group->lookup_table, + hash_val_a, + hash_val_b); + break; +#endif + case EFD_LOOKUP_SCALAR: + /* Fall-through */ + default: + return efd_lookup_internal_scalar(group->hash_idx, + group->lookup_table, + hash_val_a, + hash_val_b); + } + + return value; +} + +efd_value_t +rte_efd_lookup(const struct rte_efd_table * const table, + const unsigned int socket_id, const void *key) +{ + uint32_t chunk_id, group_id, bin_id; + uint8_t bin_choice; + const struct efd_online_group_entry *group; + const struct efd_online_chunk * const chunks = table->chunks[socket_id]; + + /* Determine the chunk and group location for the given key */ + efd_compute_ids(table, key, &chunk_id, &bin_id); + bin_choice = efd_get_choice(table, socket_id, chunk_id, bin_id); + group_id = efd_bin_to_group[bin_choice][bin_id]; + group = &chunks[chunk_id].groups[group_id]; + + return efd_lookup_internal(group, + EFD_HASHFUNCA(key, table), + EFD_HASHFUNCB(key, table), + table->lookup_fn); +} + +void rte_efd_lookup_bulk(const struct rte_efd_table * const table, + const unsigned int socket_id, const int num_keys, + const void **key_list, efd_value_t * const value_list) +{ + int i; + uint32_t chunk_id_list[RTE_EFD_BURST_MAX]; + uint32_t bin_id_list[RTE_EFD_BURST_MAX]; + uint8_t bin_choice_list[RTE_EFD_BURST_MAX]; + uint32_t group_id_list[RTE_EFD_BURST_MAX]; + struct efd_online_group_entry *group; + + struct efd_online_chunk *chunks = table->chunks[socket_id]; + + for (i = 0; i < num_keys; i++) { + efd_compute_ids(table, key_list[i], &chunk_id_list[i], + &bin_id_list[i]); + rte_prefetch0(&chunks[chunk_id_list[i]].bin_choice_list); + } + + for (i = 0; i < num_keys; i++) { + bin_choice_list[i] = efd_get_choice(table, socket_id, + chunk_id_list[i], bin_id_list[i]); + group_id_list[i] = + efd_bin_to_group[bin_choice_list[i]][bin_id_list[i]]; + group = &chunks[chunk_id_list[i]].groups[group_id_list[i]]; + rte_prefetch0(group); + } + + for (i = 0; i < num_keys; i++) { + group = &chunks[chunk_id_list[i]].groups[group_id_list[i]]; + value_list[i] = efd_lookup_internal(group, + EFD_HASHFUNCA(key_list[i], table), + EFD_HASHFUNCB(key_list[i], table), + table->lookup_fn); + } +} diff --git a/dpdk/lib/librte_efd/rte_efd.h b/dpdk/lib/librte_efd/rte_efd.h new file mode 100644 index 00000000..15968635 --- /dev/null +++ b/dpdk/lib/librte_efd/rte_efd.h @@ -0,0 +1,308 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_EFD_H_ +#define _RTE_EFD_H_ + +/** + * @file + * + * RTE EFD Table + */ + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/************************************************************************* + * User selectable constants + *************************************************************************/ + +/* + * If possible, best lookup performance will be achieved by ensuring that + * the entire table fits in the L3 cache. + * + * Some formulas for calculating various sizes are listed below: + * + * # of chunks = + * 2 ^ (ceiling(log2((requested # of rules) / + * (EFD_CHUNK_NUM_GROUPS * EFD_TARGET_GROUP_NUM_RULES)))) + * + * Target # of rules = (# of chunks) * EFD_CHUNK_NUM_GROUPS * + * EFD_TARGET_GROUP_NUM_RULES + * + * Group Size (in bytes) = 4 (per value bit) + * + * Table size (in bytes) = RTE_EFD_VALUE_NUM_BITS * (# of chunks) * + * EFD_CHUNK_NUM_GROUPS * (group size) + */ + +/** + * !!! This parameter should be adjusted for your application !!! + * + * This parameter adjusts the number of bits of value that can be + * stored in the table. + * For example, setting the number of bits to 3 will allow storing 8 values + * in the table (between 0 and 7). + * + * This number directly affects the performance of both lookups and insertion. + * In general, performance decreases as more bits are stored in the table. + * + * This number is directly proportional to the size of the online region + * used for lookups. + * + * Note that due to the way the CPU operates on memory, best lookup performance + * will be achieved when RTE_EFD_VALUE_NUM_BITS is a multiple of 8. + * These values align the hash indexes on 16-byte boundaries. + * The greatest performance drop is moving from 8->9 bits, 16->17 bits, etc. + * + * This value must be between 1 and 32 + */ +#ifndef RTE_EFD_VALUE_NUM_BITS +#define RTE_EFD_VALUE_NUM_BITS (8) +#endif + +/* + * EFD_TARGET_GROUP_NUM_RULES: + * Adjusts how many groups/chunks are allocated at table creation time + * to support the requested number of rules. Higher values pack entries + * more tightly in memory, resulting in a smaller memory footprint + * for the online table. + * This comes at the cost of lower insert/update performance. + * + * EFD_MAX_GROUP_NUM_RULES: + * This adjusts the amount of offline memory allocated to store key/value + * pairs for the table. The recommended numbers are upper-bounds for + * this parameter + * - any higher and it becomes very unlikely that a perfect hash function + * can be found for that group size. This value should be at + * least 40% larger than EFD_TARGET_GROUP_NUM_RULES + * + * Recommended values for various lookuptable and hashfunc sizes are: + * + * HASH_FUNC_SIZE = 16, LOOKUPTBL_SIZE = 16: + * EFD_TARGET_GROUP_NUM_RULES = 22 + * EFD_MAX_GROUP_NUM_RULES = 28 + */ +#define EFD_TARGET_GROUP_NUM_RULES (22) +#define EFD_MAX_GROUP_NUM_RULES (28LU) + +#define EFD_MIN_BALANCED_NUM_RULES 5 + +/** + * Maximum number of keys that can be looked up in one call to efd_lookup_bulk + */ +#ifndef RTE_EFD_BURST_MAX +#define RTE_EFD_BURST_MAX (32) +#endif + +/** Maximum number of characters in efd name.*/ +#define RTE_EFD_NAMESIZE 32 + +#if (RTE_EFD_VALUE_NUM_BITS > 0 && RTE_EFD_VALUE_NUM_BITS <= 8) +typedef uint8_t efd_value_t; +#elif (RTE_EFD_VALUE_NUM_BITS > 8 && RTE_EFD_VALUE_NUM_BITS <= 16) +typedef uint16_t efd_value_t; +#elif (RTE_EFD_VALUE_NUM_BITS > 16 && RTE_EFD_VALUE_NUM_BITS <= 32) +typedef uint32_t efd_value_t; +#else +#error("RTE_EFD_VALUE_NUM_BITS must be in the range [1:32]") +#endif + +#define EFD_LOOKUPTBL_SHIFT (32 - 4) +typedef uint16_t efd_lookuptbl_t; +typedef uint16_t efd_hashfunc_t; + +/** + * Creates an EFD table with a single offline region and multiple per-socket + * internally-managed copies of the online table used for lookups + * + * @param name + * EFD table name + * @param max_num_rules + * Minimum number of rules the table should be sized to hold. + * Will be rounded up to the next smallest valid table size + * @param key_len + * Length of the key + * @param online_cpu_socket_bitmask + * Bitmask specifying which sockets should get a copy of the online table. + * LSB = socket 0, etc. + * @param offline_cpu_socket + * Identifies the socket where the offline table will be allocated + * (and most efficiently accessed in the case of updates/insertions) + * + * @return + * EFD table, or NULL if table allocation failed or the bitmask is invalid + */ +struct rte_efd_table * +rte_efd_create(const char *name, uint32_t max_num_rules, uint32_t key_len, + uint8_t online_cpu_socket_bitmask, uint8_t offline_cpu_socket); + +/** + * Releases the resources from an EFD table + * + * @param table + * Table to free + */ +void +rte_efd_free(struct rte_efd_table *table); + +/** + * Find an existing EFD table object and return a pointer to it. + * + * @param name + * Name of the EFD table as passed to rte_efd_create() + * @return + * Pointer to EFD table or NULL if object not found + * with rte_errno set appropriately. Possible rte_errno values include: + * - ENOENT - value not available for return + */ +struct rte_efd_table* +rte_efd_find_existing(const char *name); + +#define RTE_EFD_UPDATE_WARN_GROUP_FULL (1) +#define RTE_EFD_UPDATE_NO_CHANGE (2) +#define RTE_EFD_UPDATE_FAILED (3) + +/** + * Computes an updated table entry for the supplied key/value pair. + * The update is then immediately applied to the provided table and + * all socket-local copies of the chunks are updated. + * This operation is not multi-thread safe + * and should only be called one from thread. + * + * @param table + * EFD table to reference + * @param socket_id + * Socket ID to use to lookup existing value (ideally caller's socket id) + * @param key + * EFD table key to modify + * @param value + * Value to associate with the key + * + * @return + * RTE_EFD_UPDATE_WARN_GROUP_FULL + * Operation is insert, and the last available space in the + * key's group was just used + * Future inserts may fail as groups fill up + * This operation was still successful, and entry contains a valid update + * RTE_EFD_UPDATE_FAILED + * Either the EFD failed to find a suitable perfect hash or the group was full + * This is a fatal error, and the table is now in an indeterminite state + * RTE_EFD_UPDATE_NO_CHANGE + * Operation resulted in no change to the table (same value already exists) + * 0 - success + */ +int +rte_efd_update(struct rte_efd_table *table, unsigned int socket_id, + const void *key, efd_value_t value); + +/** + * Removes any value currently associated with the specified key from the table + * This operation is not multi-thread safe + * and should only be called from one thread. + * + * @param table + * EFD table to reference + * @param socket_id + * Socket ID to use to lookup existing value (ideally caller's socket id) + * @param key + * EFD table key to delete + * @param prev_value + * If not NULL, will store the previous value here before deleting it + * + * @return + * 0 - successfully found and deleted the key + * nonzero otherwise + */ +int +rte_efd_delete(struct rte_efd_table *table, unsigned int socket_id, + const void *key, efd_value_t *prev_value); + +/** + * Looks up the value associated with a key + * This operation is multi-thread safe. + * + * NOTE: Lookups will *always* succeed - this is a property of + * using a perfect hash table. + * If the specified key was never inserted, a pseudorandom answer will be returned. + * There is no way to know based on the lookup if the key was ever inserted + * originally, so this must be tracked elsewhere. + * + * @param table + * EFD table to reference + * @param socket_id + * Socket ID to use to lookup existing value (ideally caller's socket id) + * @param key + * EFD table key to look up + * + * @return + * Value associated with the key, or random junk if they key was never inserted + */ +efd_value_t +rte_efd_lookup(const struct rte_efd_table *table, unsigned int socket_id, + const void *key); + +/** + * Looks up the value associated with several keys. + * This operation is multi-thread safe. + * + * NOTE: Lookups will *always* succeed - this is a property of + * using a perfect hash table. + * If the specified key was never inserted, a pseudorandom answer will be returned. + * There is no way to know based on the lookup if the key was ever inserted + * originally, so this must be tracked elsewhere. + * + * @param table + * EFD table to reference + * @param socket_id + * Socket ID to use to lookup existing value (ideally caller's socket id) + * @param num_keys + * Number of keys in the key_list array, must be less than RTE_EFD_BURST_MAX + * @param key_list + * Array of num_keys pointers which point to keys to look up + * @param value_list + * Array of size num_keys where lookup values will be stored + */ +void +rte_efd_lookup_bulk(const struct rte_efd_table *table, unsigned int socket_id, + int num_keys, const void **key_list, + efd_value_t *value_list); + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_EFD_H_ */ diff --git a/dpdk/lib/librte_efd/rte_efd_arm64.h b/dpdk/lib/librte_efd/rte_efd_arm64.h new file mode 100644 index 00000000..63289ac4 --- /dev/null +++ b/dpdk/lib/librte_efd/rte_efd_arm64.h @@ -0,0 +1,76 @@ +/* + * BSD LICENSE + * + * Copyright (C) Cavium, Inc. 2017. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * rte_efd_arm64.h + * This file holds all arm64 specific EFD functions + */ + +#ifndef __RTE_EFD_ARM64_H__ +#define __RTE_EFD_ARM64_H__ + +#include + +static inline efd_value_t +efd_lookup_internal_neon(const efd_hashfunc_t *group_hash_idx, + const efd_lookuptbl_t *group_lookup_table, + const uint32_t hash_val_a, const uint32_t hash_val_b) +{ + efd_value_t value = 0; + uint32_t i = 0; + uint32x4_t vhash_val_a = vmovq_n_u32(hash_val_a); + uint32x4_t vhash_val_b = vmovq_n_u32(hash_val_b); + int32x4_t vshift = {0, 1, 2, 3}; + uint32x4_t vmask = vdupq_n_u32(0x1); + int32x4_t vincr = vdupq_n_s32(4); + + for (; i < RTE_EFD_VALUE_NUM_BITS; i += 4) { + uint32x4_t vhash_idx = vshll_n_u16( + vld1_u16((uint16_t const *)&group_hash_idx[i]), 0); + uint32x4_t vlookup_table = vshll_n_u16( + vld1_u16((uint16_t const *)&group_lookup_table[i]), 0); + uint32x4_t vhash = vaddq_u32(vhash_val_a, + vmulq_u32(vhash_idx, vhash_val_b)); + int32x4_t vbucket_idx = vnegq_s32(vreinterpretq_s32_u32( + vshrq_n_u32(vhash, EFD_LOOKUPTBL_SHIFT))); + uint32x4_t vresult = vshlq_u32(vlookup_table, vbucket_idx); + + vresult = vandq_u32(vresult, vmask); + vresult = vshlq_u32(vresult, vshift); + value |= vaddvq_u32(vresult); + vshift = vaddq_s32(vshift, vincr); + } + + return value; +} + +#endif /* __RTE_EFD_ARM64_H__ */ diff --git a/dpdk/lib/librte_efd/rte_efd_version.map b/dpdk/lib/librte_efd/rte_efd_version.map new file mode 100644 index 00000000..ae60a641 --- /dev/null +++ b/dpdk/lib/librte_efd/rte_efd_version.map @@ -0,0 +1,13 @@ +DPDK_17.02 { + global: + + rte_efd_create; + rte_efd_delete; + rte_efd_find_existing; + rte_efd_free; + rte_efd_lookup; + rte_efd_lookup_bulk; + rte_efd_update; + + local: *; +}; diff --git a/dpdk/lib/librte_efd/rte_efd_x86.h b/dpdk/lib/librte_efd/rte_efd_x86.h new file mode 100644 index 00000000..34f37d73 --- /dev/null +++ b/dpdk/lib/librte_efd/rte_efd_x86.h @@ -0,0 +1,86 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* rte_efd_x86.h + * This file holds all x86 specific EFD functions + */ +#include + +#if (RTE_EFD_VALUE_NUM_BITS == 8 || RTE_EFD_VALUE_NUM_BITS == 16 || \ + RTE_EFD_VALUE_NUM_BITS == 24 || RTE_EFD_VALUE_NUM_BITS == 32) +#define EFD_LOAD_SI128(val) _mm_load_si128(val) +#else +#define EFD_LOAD_SI128(val) _mm_lddqu_si128(val) +#endif + +static inline efd_value_t +efd_lookup_internal_avx2(const efd_hashfunc_t *group_hash_idx, + const efd_lookuptbl_t *group_lookup_table, + const uint32_t hash_val_a, const uint32_t hash_val_b) +{ +#ifdef RTE_MACHINE_CPUFLAG_AVX2 + efd_value_t value = 0; + uint32_t i = 0; + __m256i vhash_val_a = _mm256_set1_epi32(hash_val_a); + __m256i vhash_val_b = _mm256_set1_epi32(hash_val_b); + + for (; i < RTE_EFD_VALUE_NUM_BITS; i += 8) { + __m256i vhash_idx = + _mm256_cvtepu16_epi32(EFD_LOAD_SI128( + (__m128i const *) &group_hash_idx[i])); + __m256i vlookup_table = _mm256_cvtepu16_epi32( + EFD_LOAD_SI128((__m128i const *) + &group_lookup_table[i])); + __m256i vhash = _mm256_add_epi32(vhash_val_a, + _mm256_mullo_epi32(vhash_idx, vhash_val_b)); + __m256i vbucket_idx = _mm256_srli_epi32(vhash, + EFD_LOOKUPTBL_SHIFT); + __m256i vresult = _mm256_srlv_epi32(vlookup_table, + vbucket_idx); + + value |= (_mm256_movemask_ps( + (__m256) _mm256_slli_epi32(vresult, 31)) + & ((1 << (RTE_EFD_VALUE_NUM_BITS - i)) - 1)) << i; + } + + return value; +#else + RTE_SET_USED(group_hash_idx); + RTE_SET_USED(group_lookup_table); + RTE_SET_USED(hash_val_a); + RTE_SET_USED(hash_val_b); + /* Return dummy value, only to avoid compilation breakage */ + return 0; +#endif + +} diff --git a/dpdk/lib/librte_ether/Makefile b/dpdk/lib/librte_ether/Makefile index 0bb5dc90..394cc9c0 100644 --- a/dpdk/lib/librte_ether/Makefile +++ b/dpdk/lib/librte_ether/Makefile @@ -1,6 +1,6 @@ # BSD LICENSE # -# Copyright(c) 2010-2014 Intel Corporation. All rights reserved. +# Copyright(c) 2010-2017 Intel Corporation. All rights reserved. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -34,26 +34,36 @@ include $(RTE_SDK)/mk/rte.vars.mk # # library name # -LIB = libethdev.a +LIB = librte_ethdev.a CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) +LDLIBS += -lrte_net -lrte_eal -lrte_mempool -lrte_ring +LDLIBS += -lrte_mbuf -EXPORT_MAP := rte_ether_version.map +EXPORT_MAP := rte_ethdev_version.map -LIBABIVER := 4 +LIBABIVER := 8 SRCS-y += rte_ethdev.c +SRCS-y += rte_flow.c +SRCS-y += rte_tm.c +SRCS-y += rte_mtr.c +SRCS-y += ethdev_profile.c # # Export include files # -SYMLINK-y-include += rte_ether.h SYMLINK-y-include += rte_ethdev.h +SYMLINK-y-include += rte_ethdev_pci.h +SYMLINK-y-include += rte_ethdev_vdev.h SYMLINK-y-include += rte_eth_ctrl.h SYMLINK-y-include += rte_dev_info.h - -# this lib depends upon: -DEPDIRS-y += lib/librte_eal lib/librte_mempool lib/librte_ring lib/librte_mbuf +SYMLINK-y-include += rte_flow.h +SYMLINK-y-include += rte_flow_driver.h +SYMLINK-y-include += rte_tm.h +SYMLINK-y-include += rte_tm_driver.h +SYMLINK-y-include += rte_mtr.h +SYMLINK-y-include += rte_mtr_driver.h include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/lib/librte_ether/ethdev_profile.c b/dpdk/lib/librte_ether/ethdev_profile.c new file mode 100644 index 00000000..c9cb8420 --- /dev/null +++ b/dpdk/lib/librte_ether/ethdev_profile.c @@ -0,0 +1,164 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "ethdev_profile.h" + +/** + * This conditional block enables RX queues profiling by tracking wasted + * iterations, i.e. iterations which yielded no RX packets. Profiling is + * performed using the Instrumentation and Tracing Technology (ITT) API, + * employed by the Intel (R) VTune (TM) Amplifier. + */ +#ifdef RTE_ETHDEV_PROFILE_ITT_WASTED_RX_ITERATIONS + +#include + +#define ITT_MAX_NAME_LEN (100) + +/** + * Auxiliary ITT structure belonging to Ethernet device and using to: + * - track RX queue state to determine whether it is wasting loop iterations + * - begin or end ITT task using task domain and task name (handle) + */ +struct itt_profile_rx_data { + /** + * ITT domains for each queue. + */ + __itt_domain *domains[RTE_MAX_QUEUES_PER_PORT]; + /** + * ITT task names for each queue. + */ + __itt_string_handle *handles[RTE_MAX_QUEUES_PER_PORT]; + /** + * Flags indicating the queues state. Possible values: + * 1 - queue is wasting iterations, + * 0 - otherwise. + */ + uint8_t queue_state[RTE_MAX_QUEUES_PER_PORT]; +}; + +/** + * The pool of *itt_profile_rx_data* structures. + */ +struct itt_profile_rx_data itt_rx_data[RTE_MAX_ETHPORTS]; + + +/** + * This callback function manages ITT tasks collection on given port and queue. + * It must be registered with rte_eth_add_rx_callback() to be called from + * rte_eth_rx_burst(). To find more comments see rte_rx_callback_fn function + * type declaration. + */ +static uint16_t +collect_itt_rx_burst_cb(uint16_t port_id, uint16_t queue_id, + __rte_unused struct rte_mbuf *pkts[], uint16_t nb_pkts, + __rte_unused uint16_t max_pkts, __rte_unused void *user_param) +{ + if (unlikely(nb_pkts == 0)) { + if (!itt_rx_data[port_id].queue_state[queue_id]) { + __itt_task_begin( + itt_rx_data[port_id].domains[queue_id], + __itt_null, __itt_null, + itt_rx_data[port_id].handles[queue_id]); + itt_rx_data[port_id].queue_state[queue_id] = 1; + } + } else { + if (unlikely(itt_rx_data[port_id].queue_state[queue_id])) { + __itt_task_end( + itt_rx_data[port_id].domains[queue_id]); + itt_rx_data[port_id].queue_state[queue_id] = 0; + } + } + return nb_pkts; +} + +/** + * Initialization of itt_profile_rx_data for a given Ethernet device. + * This function must be invoked when ethernet device is being configured. + * Result will be stored in the global array *itt_rx_data*. + * + * @param port_id + * The port identifier of the Ethernet device. + * @param port_name + * The name of the Ethernet device. + * @param rx_queue_num + * The number of RX queues on specified port. + * + * @return + * - On success, zero. + * - On failure, a negative value. + */ +static inline int +itt_profile_rx_init(uint16_t port_id, char *port_name, uint8_t rx_queue_num) +{ + uint16_t q_id; + + for (q_id = 0; q_id < rx_queue_num; ++q_id) { + char domain_name[ITT_MAX_NAME_LEN]; + + snprintf(domain_name, sizeof(domain_name), + "RXBurst.WastedIterations.Port_%s.Queue_%d", + port_name, q_id); + itt_rx_data[port_id].domains[q_id] + = __itt_domain_create(domain_name); + + char task_name[ITT_MAX_NAME_LEN]; + + snprintf(task_name, sizeof(task_name), + "port id: %d; queue id: %d", + port_id, q_id); + itt_rx_data[port_id].handles[q_id] + = __itt_string_handle_create(task_name); + + itt_rx_data[port_id].queue_state[q_id] = 0; + + if (!rte_eth_add_rx_callback( + port_id, q_id, collect_itt_rx_burst_cb, NULL)) { + return -rte_errno; + } + } + + return 0; +} +#endif /* RTE_ETHDEV_PROFILE_ITT_WASTED_RX_ITERATIONS */ + +int +__rte_eth_profile_rx_init(__rte_unused uint16_t port_id, + __rte_unused struct rte_eth_dev *dev) +{ +#ifdef RTE_ETHDEV_PROFILE_ITT_WASTED_RX_ITERATIONS + return itt_profile_rx_init( + port_id, dev->data->name, dev->data->nb_rx_queues); +#endif + return 0; +} diff --git a/dpdk/lib/librte_ether/ethdev_profile.h b/dpdk/lib/librte_ether/ethdev_profile.h new file mode 100644 index 00000000..697facff --- /dev/null +++ b/dpdk/lib/librte_ether/ethdev_profile.h @@ -0,0 +1,56 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_ETHDEV_PROFILE_H_ +#define _RTE_ETHDEV_PROFILE_H_ + +#include "rte_ethdev.h" + +/** + * Initialization of profiling RX queues for the Ethernet device. + * Implementation of this function depends on chosen profiling method, + * defined in configs. + * + * @param port_id + * The port identifier of the Ethernet device. + * @param dev + * Pointer to struct rte_eth_dev corresponding to given port_id. + * + * @return + * - On success, zero. + * - On failure, a negative value. + */ +int +__rte_eth_profile_rx_init(uint16_t port_id, struct rte_eth_dev *dev); + +#endif diff --git a/dpdk/lib/librte_ether/rte_dev_info.h b/dpdk/lib/librte_ether/rte_dev_info.h index 574683d3..aab6d1a6 100644 --- a/dpdk/lib/librte_ether/rte_dev_info.h +++ b/dpdk/lib/librte_ether/rte_dev_info.h @@ -34,6 +34,8 @@ #ifndef _RTE_DEV_INFO_H_ #define _RTE_DEV_INFO_H_ +#include + /* * Placeholder for accessing device registers */ diff --git a/dpdk/lib/librte_ether/rte_eth_ctrl.h b/dpdk/lib/librte_ether/rte_eth_ctrl.h index c3a2c9e4..83869042 100644 --- a/dpdk/lib/librte_ether/rte_eth_ctrl.h +++ b/dpdk/lib/librte_ether/rte_eth_ctrl.h @@ -34,6 +34,10 @@ #ifndef _RTE_ETH_CTRL_H_ #define _RTE_ETH_CTRL_H_ +#include +#include +#include "rte_ether.h" + /** * @file * @@ -95,6 +99,7 @@ enum rte_filter_type { RTE_ETH_FILTER_FDIR, RTE_ETH_FILTER_HASH, RTE_ETH_FILTER_L2_TUNNEL, + RTE_ETH_FILTER_GENERIC, RTE_ETH_FILTER_MAX }; diff --git a/dpdk/lib/librte_ether/rte_ethdev.c b/dpdk/lib/librte_ether/rte_ethdev.c index a5b42aa8..4d23bc1c 100644 --- a/dpdk/lib/librte_ether/rte_ethdev.c +++ b/dpdk/lib/librte_ether/rte_ethdev.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. + * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -47,7 +47,6 @@ #include #include #include -#include #include #include #include @@ -58,7 +57,6 @@ #include #include #include -#include #include #include #include @@ -68,11 +66,12 @@ #include "rte_ether.h" #include "rte_ethdev.h" +#include "ethdev_profile.h" static const char *MZ_RTE_ETH_DEV_DATA = "rte_eth_dev_data"; struct rte_eth_dev rte_eth_devices[RTE_MAX_ETHPORTS]; static struct rte_eth_dev_data *rte_eth_dev_data; -static uint8_t nb_ports; +static uint8_t eth_dev_last_created_port; /* spinlock for eth device callbacks */ static rte_spinlock_t rte_eth_dev_cb_lock = RTE_SPINLOCK_INITIALIZER; @@ -94,6 +93,7 @@ static const struct rte_eth_xstats_name_off rte_stats_strings[] = { {"tx_good_packets", offsetof(struct rte_eth_stats, opackets)}, {"rx_good_bytes", offsetof(struct rte_eth_stats, ibytes)}, {"tx_good_bytes", offsetof(struct rte_eth_stats, obytes)}, + {"rx_missed_errors", offsetof(struct rte_eth_stats, imissed)}, {"rx_errors", offsetof(struct rte_eth_stats, ierrors)}, {"tx_errors", offsetof(struct rte_eth_stats, oerrors)}, {"rx_mbuf_allocation_errors", offsetof(struct rte_eth_stats, @@ -129,6 +129,7 @@ struct rte_eth_dev_callback { TAILQ_ENTRY(rte_eth_dev_callback) next; /**< Callbacks list */ rte_eth_dev_cb_fn cb_fn; /**< Callback address */ void *cb_arg; /**< Parameter for callback */ + void *ret_param; /**< Return parameter */ enum rte_eth_event_type event; /**< Interrupt event type */ uint32_t active; /**< Callback is executing */ }; @@ -138,10 +139,18 @@ enum { STAT_QMAP_RX }; -enum { - DEV_DETACHED = 0, - DEV_ATTACHED -}; +uint16_t +rte_eth_find_next(uint16_t port_id) +{ + while (port_id < RTE_MAX_ETHPORTS && + rte_eth_devices[port_id].state != RTE_ETH_DEV_ATTACHED) + port_id++; + + if (port_id >= RTE_MAX_ETHPORTS) + return RTE_MAX_ETHPORTS; + + return port_id; +} static void rte_eth_dev_data_alloc(void) @@ -170,68 +179,101 @@ rte_eth_dev_allocated(const char *name) unsigned i; for (i = 0; i < RTE_MAX_ETHPORTS; i++) { - if ((rte_eth_devices[i].attached == DEV_ATTACHED) && + if ((rte_eth_devices[i].state == RTE_ETH_DEV_ATTACHED) && strcmp(rte_eth_devices[i].data->name, name) == 0) return &rte_eth_devices[i]; } return NULL; } -static uint8_t +static uint16_t rte_eth_dev_find_free_port(void) { unsigned i; for (i = 0; i < RTE_MAX_ETHPORTS; i++) { - if (rte_eth_devices[i].attached == DEV_DETACHED) + /* Using shared name field to find a free port. */ + if (rte_eth_dev_data[i].name[0] == '\0') { + RTE_ASSERT(rte_eth_devices[i].state == + RTE_ETH_DEV_UNUSED); return i; + } } return RTE_MAX_ETHPORTS; } -struct rte_eth_dev * -rte_eth_dev_allocate(const char *name, enum rte_eth_dev_type type) +static struct rte_eth_dev * +eth_dev_get(uint16_t port_id) { - uint8_t port_id; + struct rte_eth_dev *eth_dev = &rte_eth_devices[port_id]; + + eth_dev->data = &rte_eth_dev_data[port_id]; + eth_dev->state = RTE_ETH_DEV_ATTACHED; + TAILQ_INIT(&(eth_dev->link_intr_cbs)); + + eth_dev_last_created_port = port_id; + + return eth_dev; +} + +struct rte_eth_dev * +rte_eth_dev_allocate(const char *name) +{ + uint16_t port_id; struct rte_eth_dev *eth_dev; + if (rte_eth_dev_data == NULL) + rte_eth_dev_data_alloc(); + port_id = rte_eth_dev_find_free_port(); if (port_id == RTE_MAX_ETHPORTS) { RTE_PMD_DEBUG_TRACE("Reached maximum number of Ethernet ports\n"); return NULL; } - if (rte_eth_dev_data == NULL) - rte_eth_dev_data_alloc(); - if (rte_eth_dev_allocated(name) != NULL) { RTE_PMD_DEBUG_TRACE("Ethernet Device with name %s already allocated!\n", name); return NULL; } - eth_dev = &rte_eth_devices[port_id]; - eth_dev->data = &rte_eth_dev_data[port_id]; + eth_dev = eth_dev_get(port_id); snprintf(eth_dev->data->name, sizeof(eth_dev->data->name), "%s", name); eth_dev->data->port_id = port_id; - eth_dev->attached = DEV_ATTACHED; - eth_dev->dev_type = type; - nb_ports++; + eth_dev->data->mtu = ETHER_MTU; + return eth_dev; } -static int -rte_eth_dev_create_unique_device_name(char *name, size_t size, - struct rte_pci_device *pci_dev) +/* + * Attach to a port already registered by the primary process, which + * makes sure that the same device would have the same port id both + * in the primary and secondary process. + */ +struct rte_eth_dev * +rte_eth_dev_attach_secondary(const char *name) { - int ret; + uint16_t i; + struct rte_eth_dev *eth_dev; - ret = snprintf(name, size, "%d:%d.%d", - pci_dev->addr.bus, pci_dev->addr.devid, - pci_dev->addr.function); - if (ret < 0) - return ret; - return 0; + if (rte_eth_dev_data == NULL) + rte_eth_dev_data_alloc(); + + for (i = 0; i < RTE_MAX_ETHPORTS; i++) { + if (strcmp(rte_eth_dev_data[i].name, name) == 0) + break; + } + if (i == RTE_MAX_ETHPORTS) { + RTE_PMD_DEBUG_TRACE( + "device %s is not driven by the primary process\n", + name); + return NULL; + } + + eth_dev = eth_dev_get(i); + RTE_ASSERT(eth_dev->data->port_id == i); + + return eth_dev; } int @@ -240,174 +282,52 @@ rte_eth_dev_release_port(struct rte_eth_dev *eth_dev) if (eth_dev == NULL) return -EINVAL; - eth_dev->attached = DEV_DETACHED; - nb_ports--; + memset(eth_dev->data, 0, sizeof(struct rte_eth_dev_data)); + eth_dev->state = RTE_ETH_DEV_UNUSED; return 0; } -static int -rte_eth_dev_init(struct rte_pci_driver *pci_drv, - struct rte_pci_device *pci_dev) -{ - struct eth_driver *eth_drv; - struct rte_eth_dev *eth_dev; - char ethdev_name[RTE_ETH_NAME_MAX_LEN]; - - int diag; - - eth_drv = (struct eth_driver *)pci_drv; - - /* Create unique Ethernet device name using PCI address */ - rte_eth_dev_create_unique_device_name(ethdev_name, - sizeof(ethdev_name), pci_dev); - - eth_dev = rte_eth_dev_allocate(ethdev_name, RTE_ETH_DEV_PCI); - if (eth_dev == NULL) - return -ENOMEM; - - if (rte_eal_process_type() == RTE_PROC_PRIMARY) { - eth_dev->data->dev_private = rte_zmalloc("ethdev private structure", - eth_drv->dev_private_size, - RTE_CACHE_LINE_SIZE); - if (eth_dev->data->dev_private == NULL) - rte_panic("Cannot allocate memzone for private port data\n"); - } - eth_dev->pci_dev = pci_dev; - eth_dev->driver = eth_drv; - eth_dev->data->rx_mbuf_alloc_failed = 0; - - /* init user callbacks */ - TAILQ_INIT(&(eth_dev->link_intr_cbs)); - - /* - * Set the default MTU. - */ - eth_dev->data->mtu = ETHER_MTU; - - /* Invoke PMD device initialization function */ - diag = (*eth_drv->eth_dev_init)(eth_dev); - if (diag == 0) - return 0; - - RTE_PMD_DEBUG_TRACE("driver %s: eth_dev_init(vendor_id=0x%x device_id=0x%x) failed\n", - pci_drv->name, - (unsigned) pci_dev->id.vendor_id, - (unsigned) pci_dev->id.device_id); - if (rte_eal_process_type() == RTE_PROC_PRIMARY) - rte_free(eth_dev->data->dev_private); - rte_eth_dev_release_port(eth_dev); - return diag; -} - -static int -rte_eth_dev_uninit(struct rte_pci_device *pci_dev) -{ - const struct eth_driver *eth_drv; - struct rte_eth_dev *eth_dev; - char ethdev_name[RTE_ETH_NAME_MAX_LEN]; - int ret; - - if (pci_dev == NULL) - return -EINVAL; - - /* Create unique Ethernet device name using PCI address */ - rte_eth_dev_create_unique_device_name(ethdev_name, - sizeof(ethdev_name), pci_dev); - - eth_dev = rte_eth_dev_allocated(ethdev_name); - if (eth_dev == NULL) - return -ENODEV; - - eth_drv = (const struct eth_driver *)pci_dev->driver; - - /* Invoke PMD device uninit function */ - if (*eth_drv->eth_dev_uninit) { - ret = (*eth_drv->eth_dev_uninit)(eth_dev); - if (ret) - return ret; - } - - /* free ether device */ - rte_eth_dev_release_port(eth_dev); - - if (rte_eal_process_type() == RTE_PROC_PRIMARY) - rte_free(eth_dev->data->dev_private); - - eth_dev->pci_dev = NULL; - eth_dev->driver = NULL; - eth_dev->data = NULL; - - return 0; -} - -/** - * Register an Ethernet [Poll Mode] driver. - * - * Function invoked by the initialization function of an Ethernet driver - * to simultaneously register itself as a PCI driver and as an Ethernet - * Poll Mode Driver. - * Invokes the rte_eal_pci_register() function to register the *pci_drv* - * structure embedded in the *eth_drv* structure, after having stored the - * address of the rte_eth_dev_init() function in the *devinit* field of - * the *pci_drv* structure. - * During the PCI probing phase, the rte_eth_dev_init() function is - * invoked for each PCI [Ethernet device] matching the embedded PCI - * identifiers provided by the driver. - */ -void -rte_eth_driver_register(struct eth_driver *eth_drv) -{ - eth_drv->pci_drv.devinit = rte_eth_dev_init; - eth_drv->pci_drv.devuninit = rte_eth_dev_uninit; - rte_eal_pci_register(ð_drv->pci_drv); -} - int -rte_eth_dev_is_valid_port(uint8_t port_id) +rte_eth_dev_is_valid_port(uint16_t port_id) { if (port_id >= RTE_MAX_ETHPORTS || - rte_eth_devices[port_id].attached != DEV_ATTACHED) + (rte_eth_devices[port_id].state != RTE_ETH_DEV_ATTACHED && + rte_eth_devices[port_id].state != RTE_ETH_DEV_DEFERRED)) return 0; else return 1; } int -rte_eth_dev_socket_id(uint8_t port_id) +rte_eth_dev_socket_id(uint16_t port_id) { RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -1); return rte_eth_devices[port_id].data->numa_node; } -uint8_t +void * +rte_eth_dev_get_sec_ctx(uint8_t port_id) +{ + RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, NULL); + return rte_eth_devices[port_id].security_ctx; +} + +uint16_t rte_eth_dev_count(void) { - return nb_ports; -} + uint16_t p; + uint16_t count; -static enum rte_eth_dev_type -rte_eth_dev_get_device_type(uint8_t port_id) -{ - RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, RTE_ETH_DEV_UNKNOWN); - return rte_eth_devices[port_id].dev_type; -} + count = 0; -static int -rte_eth_dev_get_addr_by_port(uint8_t port_id, struct rte_pci_addr *addr) -{ - RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL); + RTE_ETH_FOREACH_DEV(p) + count++; - if (addr == NULL) { - RTE_PMD_DEBUG_TRACE("Null pointer is specified\n"); - return -EINVAL; - } - - *addr = rte_eth_devices[port_id].pci_dev->addr; - return 0; + return count; } int -rte_eth_dev_get_name_by_port(uint8_t port_id, char *name) +rte_eth_dev_get_name_by_port(uint16_t port_id, char *name) { char *tmp; @@ -426,7 +346,7 @@ rte_eth_dev_get_name_by_port(uint8_t port_id, char *name) } int -rte_eth_dev_get_port_by_name(const char *name, uint8_t *port_id) +rte_eth_dev_get_port_by_name(const char *name, uint16_t *port_id) { int i; @@ -435,10 +355,7 @@ rte_eth_dev_get_port_by_name(const char *name, uint8_t *port_id) return -EINVAL; } - *port_id = RTE_MAX_ETHPORTS; - - for (i = 0; i < RTE_MAX_ETHPORTS; i++) { - + RTE_ETH_FOREACH_DEV(i) { if (!strncmp(name, rte_eth_dev_data[i].name, strlen(name))) { @@ -450,222 +367,85 @@ rte_eth_dev_get_port_by_name(const char *name, uint8_t *port_id) return -ENODEV; } -static int -rte_eth_dev_get_port_by_addr(const struct rte_pci_addr *addr, uint8_t *port_id) -{ - int i; - struct rte_pci_device *pci_dev = NULL; - - if (addr == NULL) { - RTE_PMD_DEBUG_TRACE("Null pointer is specified\n"); - return -EINVAL; - } - - *port_id = RTE_MAX_ETHPORTS; - - for (i = 0; i < RTE_MAX_ETHPORTS; i++) { - - pci_dev = rte_eth_devices[i].pci_dev; - - if (pci_dev && - !rte_eal_compare_pci_addr(&pci_dev->addr, addr)) { - - *port_id = i; - - return 0; - } - } - return -ENODEV; -} - -static int -rte_eth_dev_is_detachable(uint8_t port_id) -{ - uint32_t dev_flags; - - RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL); - - switch (rte_eth_devices[port_id].data->kdrv) { - case RTE_KDRV_IGB_UIO: - case RTE_KDRV_UIO_GENERIC: - case RTE_KDRV_NIC_UIO: - case RTE_KDRV_NONE: - break; - case RTE_KDRV_VFIO: - default: - return -ENOTSUP; - } - dev_flags = rte_eth_devices[port_id].data->dev_flags; - if ((dev_flags & RTE_ETH_DEV_DETACHABLE) && - (!(dev_flags & RTE_ETH_DEV_BONDED_SLAVE))) - return 0; - else - return 1; -} - -/* attach the new physical device, then store port_id of the device */ -static int -rte_eth_dev_attach_pdev(struct rte_pci_addr *addr, uint8_t *port_id) -{ - /* re-construct pci_device_list */ - if (rte_eal_pci_scan()) - goto err; - /* Invoke probe func of the driver can handle the new device. */ - if (rte_eal_pci_probe_one(addr)) - goto err; - - if (rte_eth_dev_get_port_by_addr(addr, port_id)) - goto err; - - return 0; -err: - return -1; -} - -/* detach the new physical device, then store pci_addr of the device */ -static int -rte_eth_dev_detach_pdev(uint8_t port_id, struct rte_pci_addr *addr) -{ - struct rte_pci_addr freed_addr; - struct rte_pci_addr vp; - - /* get pci address by port id */ - if (rte_eth_dev_get_addr_by_port(port_id, &freed_addr)) - goto err; - - /* Zeroed pci addr means the port comes from virtual device */ - vp.domain = vp.bus = vp.devid = vp.function = 0; - if (rte_eal_compare_pci_addr(&vp, &freed_addr) == 0) - goto err; - - /* invoke devuninit func of the pci driver, - * also remove the device from pci_device_list */ - if (rte_eal_pci_detach(&freed_addr)) - goto err; - - *addr = freed_addr; - return 0; -err: - return -1; -} - -/* attach the new virtual device, then store port_id of the device */ -static int -rte_eth_dev_attach_vdev(const char *vdevargs, uint8_t *port_id) -{ - char *name = NULL, *args = NULL; - int ret = -1; - - /* parse vdevargs, then retrieve device name and args */ - if (rte_eal_parse_devargs_str(vdevargs, &name, &args)) - goto end; - - /* walk around dev_driver_list to find the driver of the device, - * then invoke probe function of the driver. - * rte_eal_vdev_init() updates port_id allocated after - * initialization. - */ - if (rte_eal_vdev_init(name, args)) - goto end; - - if (rte_eth_dev_get_port_by_name(name, port_id)) - goto end; - - ret = 0; -end: - free(name); - free(args); - - return ret; -} - -/* detach the new virtual device, then store the name of the device */ -static int -rte_eth_dev_detach_vdev(uint8_t port_id, char *vdevname) -{ - char name[RTE_ETH_NAME_MAX_LEN]; - - /* get device name by port id */ - if (rte_eth_dev_get_name_by_port(port_id, name)) - goto err; - /* walk around dev_driver_list to find the driver of the device, - * then invoke uninit function of the driver */ - if (rte_eal_vdev_uninit(name)) - goto err; - - strncpy(vdevname, name, sizeof(name)); - return 0; -err: - return -1; -} - /* attach the new device, then store port_id of the device */ int -rte_eth_dev_attach(const char *devargs, uint8_t *port_id) +rte_eth_dev_attach(const char *devargs, uint16_t *port_id) { - struct rte_pci_addr addr; int ret = -1; + int current = rte_eth_dev_count(); + char *name = NULL; + char *args = NULL; if ((devargs == NULL) || (port_id == NULL)) { ret = -EINVAL; goto err; } - if (eal_parse_pci_DomBDF(devargs, &addr) == 0) { - ret = rte_eth_dev_attach_pdev(&addr, port_id); - if (ret < 0) - goto err; - } else { - ret = rte_eth_dev_attach_vdev(devargs, port_id); - if (ret < 0) - goto err; + /* parse devargs, then retrieve device name and args */ + if (rte_eal_parse_devargs_str(devargs, &name, &args)) + goto err; + + ret = rte_eal_dev_attach(name, args); + if (ret < 0) + goto err; + + /* no point looking at the port count if no port exists */ + if (!rte_eth_dev_count()) { + RTE_LOG(ERR, EAL, "No port found for device (%s)\n", name); + ret = -1; + goto err; } - return 0; + /* if nothing happened, there is a bug here, since some driver told us + * it did attach a device, but did not create a port. + */ + if (current == rte_eth_dev_count()) { + ret = -1; + goto err; + } + + *port_id = eth_dev_last_created_port; + ret = 0; + err: - RTE_LOG(ERR, EAL, "Driver, cannot attach the device\n"); + free(name); + free(args); return ret; } /* detach the device, then store the name of the device */ int -rte_eth_dev_detach(uint8_t port_id, char *name) +rte_eth_dev_detach(uint16_t port_id, char *name) { - struct rte_pci_addr addr; + uint32_t dev_flags; int ret = -1; + RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL); + if (name == NULL) { ret = -EINVAL; goto err; } - /* check whether the driver supports detach feature, or not */ - if (rte_eth_dev_is_detachable(port_id)) + dev_flags = rte_eth_devices[port_id].data->dev_flags; + if (dev_flags & RTE_ETH_DEV_BONDED_SLAVE) { + RTE_LOG(ERR, EAL, "Port %" PRIu16 " is bonded, cannot detach\n", + port_id); + ret = -ENOTSUP; goto err; - - if (rte_eth_dev_get_device_type(port_id) == RTE_ETH_DEV_PCI) { - ret = rte_eth_dev_get_addr_by_port(port_id, &addr); - if (ret < 0) - goto err; - - ret = rte_eth_dev_detach_pdev(port_id, &addr); - if (ret < 0) - goto err; - - snprintf(name, RTE_ETH_NAME_MAX_LEN, - "%04x:%02x:%02x.%d", - addr.domain, addr.bus, - addr.devid, addr.function); - } else { - ret = rte_eth_dev_detach_vdev(port_id, name); - if (ret < 0) - goto err; } + snprintf(name, sizeof(rte_eth_devices[port_id].data->name), + "%s", rte_eth_devices[port_id].data->name); + + ret = rte_eal_dev_detach(rte_eth_devices[port_id].device); + if (ret < 0) + goto err; + + rte_eth_devices[port_id].state = RTE_ETH_DEV_UNUSED; return 0; err: - RTE_LOG(ERR, EAL, "Driver, cannot detach the device\n"); return ret; } @@ -711,13 +491,16 @@ rte_eth_dev_rx_queue_config(struct rte_eth_dev *dev, uint16_t nb_queues) for (i = nb_queues; i < old_nb_queues; i++) (*dev->dev_ops->rx_queue_release)(rxq[i]); + + rte_free(dev->data->rx_queues); + dev->data->rx_queues = NULL; } dev->data->nb_rx_queues = nb_queues; return 0; } int -rte_eth_dev_rx_queue_start(uint8_t port_id, uint16_t rx_queue_id) +rte_eth_dev_rx_queue_start(uint16_t port_id, uint16_t rx_queue_id) { struct rte_eth_dev *dev; @@ -743,7 +526,7 @@ rte_eth_dev_rx_queue_start(uint8_t port_id, uint16_t rx_queue_id) } int -rte_eth_dev_rx_queue_stop(uint8_t port_id, uint16_t rx_queue_id) +rte_eth_dev_rx_queue_stop(uint16_t port_id, uint16_t rx_queue_id) { struct rte_eth_dev *dev; @@ -769,7 +552,7 @@ rte_eth_dev_rx_queue_stop(uint8_t port_id, uint16_t rx_queue_id) } int -rte_eth_dev_tx_queue_start(uint8_t port_id, uint16_t tx_queue_id) +rte_eth_dev_tx_queue_start(uint16_t port_id, uint16_t tx_queue_id) { struct rte_eth_dev *dev; @@ -795,7 +578,7 @@ rte_eth_dev_tx_queue_start(uint8_t port_id, uint16_t tx_queue_id) } int -rte_eth_dev_tx_queue_stop(uint8_t port_id, uint16_t tx_queue_id) +rte_eth_dev_tx_queue_stop(uint16_t port_id, uint16_t tx_queue_id) { struct rte_eth_dev *dev; @@ -862,6 +645,9 @@ rte_eth_dev_tx_queue_config(struct rte_eth_dev *dev, uint16_t nb_queues) for (i = nb_queues; i < old_nb_queues; i++) (*dev->dev_ops->tx_queue_release)(txq[i]); + + rte_free(dev->data->tx_queues); + dev->data->tx_queues = NULL; } dev->data->nb_tx_queues = nb_queues; return 0; @@ -900,12 +686,102 @@ rte_eth_speed_bitflag(uint32_t speed, int duplex) } } +/** + * A conversion function from rxmode bitfield API. + */ +static void +rte_eth_convert_rx_offload_bitfield(const struct rte_eth_rxmode *rxmode, + uint64_t *rx_offloads) +{ + uint64_t offloads = 0; + + if (rxmode->header_split == 1) + offloads |= DEV_RX_OFFLOAD_HEADER_SPLIT; + if (rxmode->hw_ip_checksum == 1) + offloads |= DEV_RX_OFFLOAD_CHECKSUM; + if (rxmode->hw_vlan_filter == 1) + offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; + if (rxmode->hw_vlan_strip == 1) + offloads |= DEV_RX_OFFLOAD_VLAN_STRIP; + if (rxmode->hw_vlan_extend == 1) + offloads |= DEV_RX_OFFLOAD_VLAN_EXTEND; + if (rxmode->jumbo_frame == 1) + offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME; + if (rxmode->hw_strip_crc == 1) + offloads |= DEV_RX_OFFLOAD_CRC_STRIP; + if (rxmode->enable_scatter == 1) + offloads |= DEV_RX_OFFLOAD_SCATTER; + if (rxmode->enable_lro == 1) + offloads |= DEV_RX_OFFLOAD_TCP_LRO; + if (rxmode->hw_timestamp == 1) + offloads |= DEV_RX_OFFLOAD_TIMESTAMP; + if (rxmode->security == 1) + offloads |= DEV_RX_OFFLOAD_SECURITY; + + *rx_offloads = offloads; +} + +/** + * A conversion function from rxmode offloads API. + */ +static void +rte_eth_convert_rx_offloads(const uint64_t rx_offloads, + struct rte_eth_rxmode *rxmode) +{ + + if (rx_offloads & DEV_RX_OFFLOAD_HEADER_SPLIT) + rxmode->header_split = 1; + else + rxmode->header_split = 0; + if (rx_offloads & DEV_RX_OFFLOAD_CHECKSUM) + rxmode->hw_ip_checksum = 1; + else + rxmode->hw_ip_checksum = 0; + if (rx_offloads & DEV_RX_OFFLOAD_VLAN_FILTER) + rxmode->hw_vlan_filter = 1; + else + rxmode->hw_vlan_filter = 0; + if (rx_offloads & DEV_RX_OFFLOAD_VLAN_STRIP) + rxmode->hw_vlan_strip = 1; + else + rxmode->hw_vlan_strip = 0; + if (rx_offloads & DEV_RX_OFFLOAD_VLAN_EXTEND) + rxmode->hw_vlan_extend = 1; + else + rxmode->hw_vlan_extend = 0; + if (rx_offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) + rxmode->jumbo_frame = 1; + else + rxmode->jumbo_frame = 0; + if (rx_offloads & DEV_RX_OFFLOAD_CRC_STRIP) + rxmode->hw_strip_crc = 1; + else + rxmode->hw_strip_crc = 0; + if (rx_offloads & DEV_RX_OFFLOAD_SCATTER) + rxmode->enable_scatter = 1; + else + rxmode->enable_scatter = 0; + if (rx_offloads & DEV_RX_OFFLOAD_TCP_LRO) + rxmode->enable_lro = 1; + else + rxmode->enable_lro = 0; + if (rx_offloads & DEV_RX_OFFLOAD_TIMESTAMP) + rxmode->hw_timestamp = 1; + else + rxmode->hw_timestamp = 0; + if (rx_offloads & DEV_RX_OFFLOAD_SECURITY) + rxmode->security = 1; + else + rxmode->security = 0; +} + int -rte_eth_dev_configure(uint8_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q, +rte_eth_dev_configure(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q, const struct rte_eth_conf *dev_conf) { struct rte_eth_dev *dev; struct rte_eth_dev_info dev_info; + struct rte_eth_conf local_conf = *dev_conf; int diag; RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL); @@ -935,8 +811,20 @@ rte_eth_dev_configure(uint8_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q, return -EBUSY; } + /* + * Convert between the offloads API to enable PMDs to support + * only one of them. + */ + if ((dev_conf->rxmode.ignore_offload_bitfield == 0)) { + rte_eth_convert_rx_offload_bitfield( + &dev_conf->rxmode, &local_conf.rxmode.offloads); + } else { + rte_eth_convert_rx_offloads(dev_conf->rxmode.offloads, + &local_conf.rxmode); + } + /* Copy the dev_conf parameter into the dev structure */ - memcpy(&dev->data->dev_conf, dev_conf, sizeof(dev->data->dev_conf)); + memcpy(&dev->data->dev_conf, &local_conf, sizeof(dev->data->dev_conf)); /* * Check that the numbers of RX and TX queues are not greater @@ -962,22 +850,25 @@ rte_eth_dev_configure(uint8_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q, return -EINVAL; } - /* - * If link state interrupt is enabled, check that the - * device supports it. - */ + /* Check that the device supports requested interrupts */ if ((dev_conf->intr_conf.lsc == 1) && (!(dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC))) { RTE_PMD_DEBUG_TRACE("driver %s does not support lsc\n", - dev->data->drv_name); + dev->device->driver->name); return -EINVAL; } + if ((dev_conf->intr_conf.rmv == 1) && + (!(dev->data->dev_flags & RTE_ETH_DEV_INTR_RMV))) { + RTE_PMD_DEBUG_TRACE("driver %s does not support rmv\n", + dev->device->driver->name); + return -EINVAL; + } /* * If jumbo frames are enabled, check that the maximum RX packet * length is supported by the configured device. */ - if (dev_conf->rxmode.jumbo_frame == 1) { + if (local_conf.rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) { if (dev_conf->rxmode.max_rx_pkt_len > dev_info.max_rx_pktlen) { RTE_PMD_DEBUG_TRACE("ethdev port_id=%d max_rx_pkt_len %u" @@ -1029,42 +920,74 @@ rte_eth_dev_configure(uint8_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q, return diag; } + /* Initialize Rx profiling if enabled at compilation time. */ + diag = __rte_eth_profile_rx_init(port_id, dev); + if (diag != 0) { + RTE_PMD_DEBUG_TRACE("port%d __rte_eth_profile_rx_init = %d\n", + port_id, diag); + rte_eth_dev_rx_queue_config(dev, 0); + rte_eth_dev_tx_queue_config(dev, 0); + return diag; + } + return 0; } +void +_rte_eth_dev_reset(struct rte_eth_dev *dev) +{ + if (dev->data->dev_started) { + RTE_PMD_DEBUG_TRACE( + "port %d must be stopped to allow reset\n", + dev->data->port_id); + return; + } + + rte_eth_dev_rx_queue_config(dev, 0); + rte_eth_dev_tx_queue_config(dev, 0); + + memset(&dev->data->dev_conf, 0, sizeof(dev->data->dev_conf)); +} + static void -rte_eth_dev_config_restore(uint8_t port_id) +rte_eth_dev_config_restore(uint16_t port_id) { struct rte_eth_dev *dev; struct rte_eth_dev_info dev_info; - struct ether_addr addr; + struct ether_addr *addr; uint16_t i; uint32_t pool = 0; + uint64_t pool_mask; dev = &rte_eth_devices[port_id]; rte_eth_dev_info_get(port_id, &dev_info); - if (RTE_ETH_DEV_SRIOV(dev).active) - pool = RTE_ETH_DEV_SRIOV(dev).def_vmdq_idx; + /* replay MAC address configuration including default MAC */ + addr = &dev->data->mac_addrs[0]; + if (*dev->dev_ops->mac_addr_set != NULL) + (*dev->dev_ops->mac_addr_set)(dev, addr); + else if (*dev->dev_ops->mac_addr_add != NULL) + (*dev->dev_ops->mac_addr_add)(dev, addr, 0, pool); - /* replay MAC address configuration */ - for (i = 0; i < dev_info.max_mac_addrs; i++) { - addr = dev->data->mac_addrs[i]; + if (*dev->dev_ops->mac_addr_add != NULL) { + for (i = 1; i < dev_info.max_mac_addrs; i++) { + addr = &dev->data->mac_addrs[i]; - /* skip zero address */ - if (is_zero_ether_addr(&addr)) - continue; + /* skip zero address */ + if (is_zero_ether_addr(addr)) + continue; - /* add address to the hardware */ - if (*dev->dev_ops->mac_addr_add && - (dev->data->mac_pool_sel[i] & (1ULL << pool))) - (*dev->dev_ops->mac_addr_add)(dev, &addr, i, pool); - else { - RTE_PMD_DEBUG_TRACE("port %d: MAC address array not supported\n", - port_id); - /* exit the loop but not return an error */ - break; + pool = 0; + pool_mask = dev->data->mac_pool_sel[i]; + + do { + if (pool_mask & 1ULL) + (*dev->dev_ops->mac_addr_add)(dev, + addr, i, pool); + pool_mask >>= 1; + pool++; + } while (pool_mask); } } @@ -1082,7 +1005,7 @@ rte_eth_dev_config_restore(uint8_t port_id) } int -rte_eth_dev_start(uint8_t port_id) +rte_eth_dev_start(uint16_t port_id) { struct rte_eth_dev *dev; int diag; @@ -1094,7 +1017,7 @@ rte_eth_dev_start(uint8_t port_id) RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_start, -ENOTSUP); if (dev->data->dev_started != 0) { - RTE_PMD_DEBUG_TRACE("Device with port_id=%" PRIu8 + RTE_PMD_DEBUG_TRACE("Device with port_id=%" PRIu16 " already started\n", port_id); return 0; @@ -1116,7 +1039,7 @@ rte_eth_dev_start(uint8_t port_id) } void -rte_eth_dev_stop(uint8_t port_id) +rte_eth_dev_stop(uint16_t port_id) { struct rte_eth_dev *dev; @@ -1126,7 +1049,7 @@ rte_eth_dev_stop(uint8_t port_id) RTE_FUNC_PTR_OR_RET(*dev->dev_ops->dev_stop); if (dev->data->dev_started == 0) { - RTE_PMD_DEBUG_TRACE("Device with port_id=%" PRIu8 + RTE_PMD_DEBUG_TRACE("Device with port_id=%" PRIu16 " already stopped\n", port_id); return; @@ -1137,7 +1060,7 @@ rte_eth_dev_stop(uint8_t port_id) } int -rte_eth_dev_set_link_up(uint8_t port_id) +rte_eth_dev_set_link_up(uint16_t port_id) { struct rte_eth_dev *dev; @@ -1150,7 +1073,7 @@ rte_eth_dev_set_link_up(uint8_t port_id) } int -rte_eth_dev_set_link_down(uint8_t port_id) +rte_eth_dev_set_link_down(uint16_t port_id) { struct rte_eth_dev *dev; @@ -1163,7 +1086,7 @@ rte_eth_dev_set_link_down(uint8_t port_id) } void -rte_eth_dev_close(uint8_t port_id) +rte_eth_dev_close(uint16_t port_id) { struct rte_eth_dev *dev; @@ -1174,14 +1097,33 @@ rte_eth_dev_close(uint8_t port_id) dev->data->dev_started = 0; (*dev->dev_ops->dev_close)(dev); + dev->data->nb_rx_queues = 0; rte_free(dev->data->rx_queues); dev->data->rx_queues = NULL; + dev->data->nb_tx_queues = 0; rte_free(dev->data->tx_queues); dev->data->tx_queues = NULL; } int -rte_eth_rx_queue_setup(uint8_t port_id, uint16_t rx_queue_id, +rte_eth_dev_reset(uint16_t port_id) +{ + struct rte_eth_dev *dev; + int ret; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL); + dev = &rte_eth_devices[port_id]; + + RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_reset, -ENOTSUP); + + rte_eth_dev_stop(port_id); + ret = dev->dev_ops->dev_reset(dev); + + return ret; +} + +int +rte_eth_rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id, uint16_t nb_rx_desc, unsigned int socket_id, const struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp) @@ -1190,6 +1132,8 @@ rte_eth_rx_queue_setup(uint8_t port_id, uint16_t rx_queue_id, uint32_t mbp_buf_size; struct rte_eth_dev *dev; struct rte_eth_dev_info dev_info; + struct rte_eth_rxconf local_conf; + void **rxq; RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL); @@ -1248,11 +1192,29 @@ rte_eth_rx_queue_setup(uint8_t port_id, uint16_t rx_queue_id, return -EINVAL; } + rxq = dev->data->rx_queues; + if (rxq[rx_queue_id]) { + RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_release, + -ENOTSUP); + (*dev->dev_ops->rx_queue_release)(rxq[rx_queue_id]); + rxq[rx_queue_id] = NULL; + } + if (rx_conf == NULL) rx_conf = &dev_info.default_rxconf; + local_conf = *rx_conf; + if (dev->data->dev_conf.rxmode.ignore_offload_bitfield == 0) { + /** + * Reflect port offloads to queue offloads in order for + * offloads to not be discarded. + */ + rte_eth_convert_rx_offload_bitfield(&dev->data->dev_conf.rxmode, + &local_conf.offloads); + } + ret = (*dev->dev_ops->rx_queue_setup)(dev, rx_queue_id, nb_rx_desc, - socket_id, rx_conf, mp); + socket_id, &local_conf, mp); if (!ret) { if (!dev->data->min_rx_buf_size || dev->data->min_rx_buf_size > mbp_buf_size) @@ -1262,13 +1224,64 @@ rte_eth_rx_queue_setup(uint8_t port_id, uint16_t rx_queue_id, return ret; } +/** + * A conversion function from txq_flags API. + */ +static void +rte_eth_convert_txq_flags(const uint32_t txq_flags, uint64_t *tx_offloads) +{ + uint64_t offloads = 0; + + if (!(txq_flags & ETH_TXQ_FLAGS_NOMULTSEGS)) + offloads |= DEV_TX_OFFLOAD_MULTI_SEGS; + if (!(txq_flags & ETH_TXQ_FLAGS_NOVLANOFFL)) + offloads |= DEV_TX_OFFLOAD_VLAN_INSERT; + if (!(txq_flags & ETH_TXQ_FLAGS_NOXSUMSCTP)) + offloads |= DEV_TX_OFFLOAD_SCTP_CKSUM; + if (!(txq_flags & ETH_TXQ_FLAGS_NOXSUMUDP)) + offloads |= DEV_TX_OFFLOAD_UDP_CKSUM; + if (!(txq_flags & ETH_TXQ_FLAGS_NOXSUMTCP)) + offloads |= DEV_TX_OFFLOAD_TCP_CKSUM; + if ((txq_flags & ETH_TXQ_FLAGS_NOREFCOUNT) && + (txq_flags & ETH_TXQ_FLAGS_NOMULTMEMP)) + offloads |= DEV_TX_OFFLOAD_MBUF_FAST_FREE; + + *tx_offloads = offloads; +} + +/** + * A conversion function from offloads API. + */ +static void +rte_eth_convert_txq_offloads(const uint64_t tx_offloads, uint32_t *txq_flags) +{ + uint32_t flags = 0; + + if (!(tx_offloads & DEV_TX_OFFLOAD_MULTI_SEGS)) + flags |= ETH_TXQ_FLAGS_NOMULTSEGS; + if (!(tx_offloads & DEV_TX_OFFLOAD_VLAN_INSERT)) + flags |= ETH_TXQ_FLAGS_NOVLANOFFL; + if (!(tx_offloads & DEV_TX_OFFLOAD_SCTP_CKSUM)) + flags |= ETH_TXQ_FLAGS_NOXSUMSCTP; + if (!(tx_offloads & DEV_TX_OFFLOAD_UDP_CKSUM)) + flags |= ETH_TXQ_FLAGS_NOXSUMUDP; + if (!(tx_offloads & DEV_TX_OFFLOAD_TCP_CKSUM)) + flags |= ETH_TXQ_FLAGS_NOXSUMTCP; + if (tx_offloads & DEV_TX_OFFLOAD_MBUF_FAST_FREE) + flags |= (ETH_TXQ_FLAGS_NOREFCOUNT | ETH_TXQ_FLAGS_NOMULTMEMP); + + *txq_flags = flags; +} + int -rte_eth_tx_queue_setup(uint8_t port_id, uint16_t tx_queue_id, +rte_eth_tx_queue_setup(uint16_t port_id, uint16_t tx_queue_id, uint16_t nb_tx_desc, unsigned int socket_id, const struct rte_eth_txconf *tx_conf) { struct rte_eth_dev *dev; struct rte_eth_dev_info dev_info; + struct rte_eth_txconf local_conf; + void **txq; RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL); @@ -1301,11 +1314,34 @@ rte_eth_tx_queue_setup(uint8_t port_id, uint16_t tx_queue_id, return -EINVAL; } + txq = dev->data->tx_queues; + if (txq[tx_queue_id]) { + RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_release, + -ENOTSUP); + (*dev->dev_ops->tx_queue_release)(txq[tx_queue_id]); + txq[tx_queue_id] = NULL; + } + if (tx_conf == NULL) tx_conf = &dev_info.default_txconf; + /* + * Convert between the offloads API to enable PMDs to support + * only one of them. + */ + local_conf = *tx_conf; + if (tx_conf->txq_flags & ETH_TXQ_FLAGS_IGNORE) { + rte_eth_convert_txq_offloads(tx_conf->offloads, + &local_conf.txq_flags); + /* Keep the ignore flag. */ + local_conf.txq_flags |= ETH_TXQ_FLAGS_IGNORE; + } else { + rte_eth_convert_txq_flags(tx_conf->txq_flags, + &local_conf.offloads); + } + return (*dev->dev_ops->tx_queue_setup)(dev, tx_queue_id, nb_tx_desc, - socket_id, tx_conf); + socket_id, &local_conf); } void @@ -1357,8 +1393,22 @@ rte_eth_tx_buffer_init(struct rte_eth_dev_tx_buffer *buffer, uint16_t size) return ret; } +int +rte_eth_tx_done_cleanup(uint16_t port_id, uint16_t queue_id, uint32_t free_cnt) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + + /* Validate Input Data. Bail if not valid or not supported. */ + RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); + RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_done_cleanup, -ENOTSUP); + + /* Call driver to free pending mbufs. */ + return (*dev->dev_ops->tx_done_cleanup)(dev->data->tx_queues[queue_id], + free_cnt); +} + void -rte_eth_promiscuous_enable(uint8_t port_id) +rte_eth_promiscuous_enable(uint16_t port_id) { struct rte_eth_dev *dev; @@ -1371,7 +1421,7 @@ rte_eth_promiscuous_enable(uint8_t port_id) } void -rte_eth_promiscuous_disable(uint8_t port_id) +rte_eth_promiscuous_disable(uint16_t port_id) { struct rte_eth_dev *dev; @@ -1384,7 +1434,7 @@ rte_eth_promiscuous_disable(uint8_t port_id) } int -rte_eth_promiscuous_get(uint8_t port_id) +rte_eth_promiscuous_get(uint16_t port_id) { struct rte_eth_dev *dev; @@ -1395,7 +1445,7 @@ rte_eth_promiscuous_get(uint8_t port_id) } void -rte_eth_allmulticast_enable(uint8_t port_id) +rte_eth_allmulticast_enable(uint16_t port_id) { struct rte_eth_dev *dev; @@ -1408,7 +1458,7 @@ rte_eth_allmulticast_enable(uint8_t port_id) } void -rte_eth_allmulticast_disable(uint8_t port_id) +rte_eth_allmulticast_disable(uint16_t port_id) { struct rte_eth_dev *dev; @@ -1421,7 +1471,7 @@ rte_eth_allmulticast_disable(uint8_t port_id) } int -rte_eth_allmulticast_get(uint8_t port_id) +rte_eth_allmulticast_get(uint16_t port_id) { struct rte_eth_dev *dev; @@ -1446,7 +1496,7 @@ rte_eth_dev_atomic_read_link_status(struct rte_eth_dev *dev, } void -rte_eth_link_get(uint8_t port_id, struct rte_eth_link *eth_link) +rte_eth_link_get(uint16_t port_id, struct rte_eth_link *eth_link) { struct rte_eth_dev *dev; @@ -1463,7 +1513,7 @@ rte_eth_link_get(uint8_t port_id, struct rte_eth_link *eth_link) } void -rte_eth_link_get_nowait(uint8_t port_id, struct rte_eth_link *eth_link) +rte_eth_link_get_nowait(uint16_t port_id, struct rte_eth_link *eth_link) { struct rte_eth_dev *dev; @@ -1480,7 +1530,7 @@ rte_eth_link_get_nowait(uint8_t port_id, struct rte_eth_link *eth_link) } int -rte_eth_stats_get(uint8_t port_id, struct rte_eth_stats *stats) +rte_eth_stats_get(uint16_t port_id, struct rte_eth_stats *stats) { struct rte_eth_dev *dev; @@ -1491,53 +1541,211 @@ rte_eth_stats_get(uint8_t port_id, struct rte_eth_stats *stats) RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->stats_get, -ENOTSUP); stats->rx_nombuf = dev->data->rx_mbuf_alloc_failed; - (*dev->dev_ops->stats_get)(dev, stats); - return 0; + return (*dev->dev_ops->stats_get)(dev, stats); } -void -rte_eth_stats_reset(uint8_t port_id) +int +rte_eth_stats_reset(uint16_t port_id) { struct rte_eth_dev *dev; - RTE_ETH_VALID_PORTID_OR_RET(port_id); + RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); dev = &rte_eth_devices[port_id]; - RTE_FUNC_PTR_OR_RET(*dev->dev_ops->stats_reset); + RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->stats_reset, -ENOTSUP); (*dev->dev_ops->stats_reset)(dev); dev->data->rx_mbuf_alloc_failed = 0; + + return 0; +} + +static inline int +get_xstats_basic_count(struct rte_eth_dev *dev) +{ + uint16_t nb_rxqs, nb_txqs; + int count; + + nb_rxqs = RTE_MIN(dev->data->nb_rx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS); + nb_txqs = RTE_MIN(dev->data->nb_tx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS); + + count = RTE_NB_STATS; + count += nb_rxqs * RTE_NB_RXQ_STATS; + count += nb_txqs * RTE_NB_TXQ_STATS; + + return count; } static int -get_xstats_count(uint8_t port_id) +get_xstats_count(uint16_t port_id) { struct rte_eth_dev *dev; int count; RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL); dev = &rte_eth_devices[port_id]; + if (dev->dev_ops->xstats_get_names_by_id != NULL) { + count = (*dev->dev_ops->xstats_get_names_by_id)(dev, NULL, + NULL, 0); + if (count < 0) + return count; + } if (dev->dev_ops->xstats_get_names != NULL) { count = (*dev->dev_ops->xstats_get_names)(dev, NULL, 0); if (count < 0) return count; } else count = 0; - count += RTE_NB_STATS; - count += dev->data->nb_rx_queues * RTE_NB_RXQ_STATS; - count += dev->data->nb_tx_queues * RTE_NB_TXQ_STATS; + + + count += get_xstats_basic_count(dev); + return count; } int -rte_eth_xstats_get_names(uint8_t port_id, +rte_eth_xstats_get_id_by_name(uint16_t port_id, const char *xstat_name, + uint64_t *id) +{ + int cnt_xstats, idx_xstat; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); + + if (!id) { + RTE_PMD_DEBUG_TRACE("Error: id pointer is NULL\n"); + return -ENOMEM; + } + + if (!xstat_name) { + RTE_PMD_DEBUG_TRACE("Error: xstat_name pointer is NULL\n"); + return -ENOMEM; + } + + /* Get count */ + cnt_xstats = rte_eth_xstats_get_names_by_id(port_id, NULL, 0, NULL); + if (cnt_xstats < 0) { + RTE_PMD_DEBUG_TRACE("Error: Cannot get count of xstats\n"); + return -ENODEV; + } + + /* Get id-name lookup table */ + struct rte_eth_xstat_name xstats_names[cnt_xstats]; + + if (cnt_xstats != rte_eth_xstats_get_names_by_id( + port_id, xstats_names, cnt_xstats, NULL)) { + RTE_PMD_DEBUG_TRACE("Error: Cannot get xstats lookup\n"); + return -1; + } + + for (idx_xstat = 0; idx_xstat < cnt_xstats; idx_xstat++) { + if (!strcmp(xstats_names[idx_xstat].name, xstat_name)) { + *id = idx_xstat; + return 0; + }; + } + + return -EINVAL; +} + +/* retrieve ethdev extended statistics names */ +int +rte_eth_xstats_get_names_by_id(uint16_t port_id, + struct rte_eth_xstat_name *xstats_names, unsigned int size, + uint64_t *ids) +{ + struct rte_eth_xstat_name *xstats_names_copy; + unsigned int no_basic_stat_requested = 1; + unsigned int expected_entries; + struct rte_eth_dev *dev; + unsigned int i; + int ret; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); + dev = &rte_eth_devices[port_id]; + + ret = get_xstats_count(port_id); + if (ret < 0) + return ret; + expected_entries = (unsigned int)ret; + + /* Return max number of stats if no ids given */ + if (!ids) { + if (!xstats_names) + return expected_entries; + else if (xstats_names && size < expected_entries) + return expected_entries; + } + + if (ids && !xstats_names) + return -EINVAL; + + if (ids && dev->dev_ops->xstats_get_names_by_id != NULL && size > 0) { + unsigned int basic_count = get_xstats_basic_count(dev); + uint64_t ids_copy[size]; + + for (i = 0; i < size; i++) { + if (ids[i] < basic_count) { + no_basic_stat_requested = 0; + break; + } + + /* + * Convert ids to xstats ids that PMD knows. + * ids known by user are basic + extended stats. + */ + ids_copy[i] = ids[i] - basic_count; + } + + if (no_basic_stat_requested) + return (*dev->dev_ops->xstats_get_names_by_id)(dev, + xstats_names, ids_copy, size); + } + + /* Retrieve all stats */ + if (!ids) { + int num_stats = rte_eth_xstats_get_names(port_id, xstats_names, + expected_entries); + if (num_stats < 0 || num_stats > (int)expected_entries) + return num_stats; + else + return expected_entries; + } + + xstats_names_copy = calloc(expected_entries, + sizeof(struct rte_eth_xstat_name)); + + if (!xstats_names_copy) { + RTE_PMD_DEBUG_TRACE("ERROR: can't allocate memory"); + return -ENOMEM; + } + + /* Fill xstats_names_copy structure */ + rte_eth_xstats_get_names(port_id, xstats_names_copy, expected_entries); + + /* Filter stats */ + for (i = 0; i < size; i++) { + if (ids[i] >= expected_entries) { + RTE_PMD_DEBUG_TRACE("ERROR: id value isn't valid\n"); + free(xstats_names_copy); + return -1; + } + xstats_names[i] = xstats_names_copy[ids[i]]; + } + + free(xstats_names_copy); + return size; +} + +int +rte_eth_xstats_get_names(uint16_t port_id, struct rte_eth_xstat_name *xstats_names, - unsigned size) + unsigned int size) { struct rte_eth_dev *dev; int cnt_used_entries; int cnt_expected_entries; int cnt_driver_entries; uint32_t idx, id_queue; + uint16_t num_q; cnt_expected_entries = get_xstats_count(port_id); if (xstats_names == NULL || cnt_expected_entries < 0 || @@ -1554,7 +1762,8 @@ rte_eth_xstats_get_names(uint8_t port_id, "%s", rte_stats_strings[idx].name); cnt_used_entries++; } - for (id_queue = 0; id_queue < dev->data->nb_rx_queues; id_queue++) { + num_q = RTE_MIN(dev->data->nb_rx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS); + for (id_queue = 0; id_queue < num_q; id_queue++) { for (idx = 0; idx < RTE_NB_RXQ_STATS; idx++) { snprintf(xstats_names[cnt_used_entries].name, sizeof(xstats_names[0].name), @@ -1564,7 +1773,8 @@ rte_eth_xstats_get_names(uint8_t port_id, } } - for (id_queue = 0; id_queue < dev->data->nb_tx_queues; id_queue++) { + num_q = RTE_MIN(dev->data->nb_tx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS); + for (id_queue = 0; id_queue < num_q; id_queue++) { for (idx = 0; idx < RTE_NB_TXQ_STATS; idx++) { snprintf(xstats_names[cnt_used_entries].name, sizeof(xstats_names[0].name), @@ -1592,22 +1802,99 @@ rte_eth_xstats_get_names(uint8_t port_id, /* retrieve ethdev extended statistics */ int -rte_eth_xstats_get(uint8_t port_id, struct rte_eth_xstat *xstats, - unsigned n) +rte_eth_xstats_get_by_id(uint16_t port_id, const uint64_t *ids, + uint64_t *values, unsigned int size) +{ + unsigned int no_basic_stat_requested = 1; + unsigned int num_xstats_filled; + uint16_t expected_entries; + struct rte_eth_dev *dev; + unsigned int i; + int ret; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); + expected_entries = get_xstats_count(port_id); + struct rte_eth_xstat xstats[expected_entries]; + dev = &rte_eth_devices[port_id]; + + /* Return max number of stats if no ids given */ + if (!ids) { + if (!values) + return expected_entries; + else if (values && size < expected_entries) + return expected_entries; + } + + if (ids && !values) + return -EINVAL; + + if (ids && dev->dev_ops->xstats_get_by_id != NULL && size) { + unsigned int basic_count = get_xstats_basic_count(dev); + uint64_t ids_copy[size]; + + for (i = 0; i < size; i++) { + if (ids[i] < basic_count) { + no_basic_stat_requested = 0; + break; + } + + /* + * Convert ids to xstats ids that PMD knows. + * ids known by user are basic + extended stats. + */ + ids_copy[i] = ids[i] - basic_count; + } + + if (no_basic_stat_requested) + return (*dev->dev_ops->xstats_get_by_id)(dev, ids_copy, + values, size); + } + + /* Fill the xstats structure */ + ret = rte_eth_xstats_get(port_id, xstats, expected_entries); + if (ret < 0) + return ret; + num_xstats_filled = (unsigned int)ret; + + /* Return all stats */ + if (!ids) { + for (i = 0; i < num_xstats_filled; i++) + values[i] = xstats[i].value; + return expected_entries; + } + + /* Filter stats */ + for (i = 0; i < size; i++) { + if (ids[i] >= expected_entries) { + RTE_PMD_DEBUG_TRACE("ERROR: id value isn't valid\n"); + return -1; + } + values[i] = xstats[ids[i]].value; + } + return size; +} + +int +rte_eth_xstats_get(uint16_t port_id, struct rte_eth_xstat *xstats, + unsigned int n) { struct rte_eth_stats eth_stats; struct rte_eth_dev *dev; - unsigned count = 0, i, q; - signed xcount = 0; + unsigned int count = 0, i, q; + signed int xcount = 0; uint64_t val, *stats_ptr; + uint16_t nb_rxqs, nb_txqs; RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL); dev = &rte_eth_devices[port_id]; + nb_rxqs = RTE_MIN(dev->data->nb_rx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS); + nb_txqs = RTE_MIN(dev->data->nb_tx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS); + /* Return generic statistics */ - count = RTE_NB_STATS + (dev->data->nb_rx_queues * RTE_NB_RXQ_STATS) + - (dev->data->nb_tx_queues * RTE_NB_TXQ_STATS); + count = RTE_NB_STATS + (nb_rxqs * RTE_NB_RXQ_STATS) + + (nb_txqs * RTE_NB_TXQ_STATS); /* implemented by the driver */ if (dev->dev_ops->xstats_get != NULL) { @@ -1638,7 +1925,7 @@ rte_eth_xstats_get(uint8_t port_id, struct rte_eth_xstat *xstats, } /* per-rxq stats */ - for (q = 0; q < dev->data->nb_rx_queues; q++) { + for (q = 0; q < nb_rxqs; q++) { for (i = 0; i < RTE_NB_RXQ_STATS; i++) { stats_ptr = RTE_PTR_ADD(ð_stats, rte_rxq_stats_strings[i].offset + @@ -1649,7 +1936,7 @@ rte_eth_xstats_get(uint8_t port_id, struct rte_eth_xstat *xstats, } /* per-txq stats */ - for (q = 0; q < dev->data->nb_tx_queues; q++) { + for (q = 0; q < nb_txqs; q++) { for (i = 0; i < RTE_NB_TXQ_STATS; i++) { stats_ptr = RTE_PTR_ADD(ð_stats, rte_txq_stats_strings[i].offset + @@ -1659,15 +1946,18 @@ rte_eth_xstats_get(uint8_t port_id, struct rte_eth_xstat *xstats, } } - for (i = 0; i < count + xcount; i++) + for (i = 0; i < count; i++) xstats[i].id = i; + /* add an offset to driver-specific stats */ + for ( ; i < count + xcount; i++) + xstats[i].id += count; return count + xcount; } /* reset ethdev extended statistics */ void -rte_eth_xstats_reset(uint8_t port_id) +rte_eth_xstats_reset(uint16_t port_id) { struct rte_eth_dev *dev; @@ -1685,7 +1975,7 @@ rte_eth_xstats_reset(uint8_t port_id) } static int -set_queue_stats_mapping(uint8_t port_id, uint16_t queue_id, uint8_t stat_idx, +set_queue_stats_mapping(uint16_t port_id, uint16_t queue_id, uint8_t stat_idx, uint8_t is_rx) { struct rte_eth_dev *dev; @@ -1701,7 +1991,7 @@ set_queue_stats_mapping(uint8_t port_id, uint16_t queue_id, uint8_t stat_idx, int -rte_eth_dev_set_tx_queue_stats_mapping(uint8_t port_id, uint16_t tx_queue_id, +rte_eth_dev_set_tx_queue_stats_mapping(uint16_t port_id, uint16_t tx_queue_id, uint8_t stat_idx) { return set_queue_stats_mapping(port_id, tx_queue_id, stat_idx, @@ -1710,15 +2000,27 @@ rte_eth_dev_set_tx_queue_stats_mapping(uint8_t port_id, uint16_t tx_queue_id, int -rte_eth_dev_set_rx_queue_stats_mapping(uint8_t port_id, uint16_t rx_queue_id, +rte_eth_dev_set_rx_queue_stats_mapping(uint16_t port_id, uint16_t rx_queue_id, uint8_t stat_idx) { return set_queue_stats_mapping(port_id, rx_queue_id, stat_idx, STAT_QMAP_RX); } +int +rte_eth_dev_fw_version_get(uint16_t port_id, char *fw_version, size_t fw_size) +{ + struct rte_eth_dev *dev; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); + dev = &rte_eth_devices[port_id]; + + RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->fw_version_get, -ENOTSUP); + return (*dev->dev_ops->fw_version_get)(dev, fw_version, fw_size); +} + void -rte_eth_dev_info_get(uint8_t port_id, struct rte_eth_dev_info *dev_info) +rte_eth_dev_info_get(uint16_t port_id, struct rte_eth_dev_info *dev_info) { struct rte_eth_dev *dev; const struct rte_eth_desc_lim lim = { @@ -1736,14 +2038,13 @@ rte_eth_dev_info_get(uint8_t port_id, struct rte_eth_dev_info *dev_info) RTE_FUNC_PTR_OR_RET(*dev->dev_ops->dev_infos_get); (*dev->dev_ops->dev_infos_get)(dev, dev_info); - dev_info->pci_dev = dev->pci_dev; - dev_info->driver_name = dev->data->drv_name; + dev_info->driver_name = dev->device->driver->name; dev_info->nb_rx_queues = dev->data->nb_rx_queues; dev_info->nb_tx_queues = dev->data->nb_tx_queues; } int -rte_eth_dev_get_supported_ptypes(uint8_t port_id, uint32_t ptype_mask, +rte_eth_dev_get_supported_ptypes(uint16_t port_id, uint32_t ptype_mask, uint32_t *ptypes, int num) { int i, j; @@ -1769,7 +2070,7 @@ rte_eth_dev_get_supported_ptypes(uint8_t port_id, uint32_t ptype_mask, } void -rte_eth_macaddr_get(uint8_t port_id, struct ether_addr *mac_addr) +rte_eth_macaddr_get(uint16_t port_id, struct ether_addr *mac_addr) { struct rte_eth_dev *dev; @@ -1780,7 +2081,7 @@ rte_eth_macaddr_get(uint8_t port_id, struct ether_addr *mac_addr) int -rte_eth_dev_get_mtu(uint8_t port_id, uint16_t *mtu) +rte_eth_dev_get_mtu(uint16_t port_id, uint16_t *mtu) { struct rte_eth_dev *dev; @@ -1792,7 +2093,7 @@ rte_eth_dev_get_mtu(uint8_t port_id, uint16_t *mtu) } int -rte_eth_dev_set_mtu(uint8_t port_id, uint16_t mtu) +rte_eth_dev_set_mtu(uint16_t port_id, uint16_t mtu) { int ret; struct rte_eth_dev *dev; @@ -1809,13 +2110,15 @@ rte_eth_dev_set_mtu(uint8_t port_id, uint16_t mtu) } int -rte_eth_dev_vlan_filter(uint8_t port_id, uint16_t vlan_id, int on) +rte_eth_dev_vlan_filter(uint16_t port_id, uint16_t vlan_id, int on) { struct rte_eth_dev *dev; + int ret; RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); dev = &rte_eth_devices[port_id]; - if (!(dev->data->dev_conf.rxmode.hw_vlan_filter)) { + if (!(dev->data->dev_conf.rxmode.offloads & + DEV_RX_OFFLOAD_VLAN_FILTER)) { RTE_PMD_DEBUG_TRACE("port %d: vlan-filtering disabled\n", port_id); return -ENOSYS; } @@ -1827,11 +2130,28 @@ rte_eth_dev_vlan_filter(uint8_t port_id, uint16_t vlan_id, int on) } RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_filter_set, -ENOTSUP); - return (*dev->dev_ops->vlan_filter_set)(dev, vlan_id, on); + ret = (*dev->dev_ops->vlan_filter_set)(dev, vlan_id, on); + if (ret == 0) { + struct rte_vlan_filter_conf *vfc; + int vidx; + int vbit; + + vfc = &dev->data->vlan_filter_conf; + vidx = vlan_id / 64; + vbit = vlan_id % 64; + + if (on) + vfc->ids[vidx] |= UINT64_C(1) << vbit; + else + vfc->ids[vidx] &= ~(UINT64_C(1) << vbit); + } + + return ret; } int -rte_eth_dev_set_vlan_strip_on_queue(uint8_t port_id, uint16_t rx_queue_id, int on) +rte_eth_dev_set_vlan_strip_on_queue(uint16_t port_id, uint16_t rx_queue_id, + int on) { struct rte_eth_dev *dev; @@ -1849,7 +2169,7 @@ rte_eth_dev_set_vlan_strip_on_queue(uint8_t port_id, uint16_t rx_queue_id, int o } int -rte_eth_dev_set_vlan_ether_type(uint8_t port_id, +rte_eth_dev_set_vlan_ether_type(uint16_t port_id, enum rte_vlan_type vlan_type, uint16_t tpid) { @@ -1863,35 +2183,57 @@ rte_eth_dev_set_vlan_ether_type(uint8_t port_id, } int -rte_eth_dev_set_vlan_offload(uint8_t port_id, int offload_mask) +rte_eth_dev_set_vlan_offload(uint16_t port_id, int offload_mask) { struct rte_eth_dev *dev; int ret = 0; int mask = 0; int cur, org = 0; + uint64_t orig_offloads; RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); dev = &rte_eth_devices[port_id]; + /* save original values in case of failure */ + orig_offloads = dev->data->dev_conf.rxmode.offloads; + /*check which option changed by application*/ cur = !!(offload_mask & ETH_VLAN_STRIP_OFFLOAD); - org = !!(dev->data->dev_conf.rxmode.hw_vlan_strip); + org = !!(dev->data->dev_conf.rxmode.offloads & + DEV_RX_OFFLOAD_VLAN_STRIP); if (cur != org) { - dev->data->dev_conf.rxmode.hw_vlan_strip = (uint8_t)cur; + if (cur) + dev->data->dev_conf.rxmode.offloads |= + DEV_RX_OFFLOAD_VLAN_STRIP; + else + dev->data->dev_conf.rxmode.offloads &= + ~DEV_RX_OFFLOAD_VLAN_STRIP; mask |= ETH_VLAN_STRIP_MASK; } cur = !!(offload_mask & ETH_VLAN_FILTER_OFFLOAD); - org = !!(dev->data->dev_conf.rxmode.hw_vlan_filter); + org = !!(dev->data->dev_conf.rxmode.offloads & + DEV_RX_OFFLOAD_VLAN_FILTER); if (cur != org) { - dev->data->dev_conf.rxmode.hw_vlan_filter = (uint8_t)cur; + if (cur) + dev->data->dev_conf.rxmode.offloads |= + DEV_RX_OFFLOAD_VLAN_FILTER; + else + dev->data->dev_conf.rxmode.offloads &= + ~DEV_RX_OFFLOAD_VLAN_FILTER; mask |= ETH_VLAN_FILTER_MASK; } cur = !!(offload_mask & ETH_VLAN_EXTEND_OFFLOAD); - org = !!(dev->data->dev_conf.rxmode.hw_vlan_extend); + org = !!(dev->data->dev_conf.rxmode.offloads & + DEV_RX_OFFLOAD_VLAN_EXTEND); if (cur != org) { - dev->data->dev_conf.rxmode.hw_vlan_extend = (uint8_t)cur; + if (cur) + dev->data->dev_conf.rxmode.offloads |= + DEV_RX_OFFLOAD_VLAN_EXTEND; + else + dev->data->dev_conf.rxmode.offloads &= + ~DEV_RX_OFFLOAD_VLAN_EXTEND; mask |= ETH_VLAN_EXTEND_MASK; } @@ -1900,13 +2242,26 @@ rte_eth_dev_set_vlan_offload(uint8_t port_id, int offload_mask) return ret; RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_offload_set, -ENOTSUP); - (*dev->dev_ops->vlan_offload_set)(dev, mask); + + /* + * Convert to the offload bitfield API just in case the underlying PMD + * still supporting it. + */ + rte_eth_convert_rx_offloads(dev->data->dev_conf.rxmode.offloads, + &dev->data->dev_conf.rxmode); + ret = (*dev->dev_ops->vlan_offload_set)(dev, mask); + if (ret) { + /* hit an error restore original values */ + dev->data->dev_conf.rxmode.offloads = orig_offloads; + rte_eth_convert_rx_offloads(dev->data->dev_conf.rxmode.offloads, + &dev->data->dev_conf.rxmode); + } return ret; } int -rte_eth_dev_get_vlan_offload(uint8_t port_id) +rte_eth_dev_get_vlan_offload(uint16_t port_id) { struct rte_eth_dev *dev; int ret = 0; @@ -1914,20 +2269,23 @@ rte_eth_dev_get_vlan_offload(uint8_t port_id) RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); dev = &rte_eth_devices[port_id]; - if (dev->data->dev_conf.rxmode.hw_vlan_strip) + if (dev->data->dev_conf.rxmode.offloads & + DEV_RX_OFFLOAD_VLAN_STRIP) ret |= ETH_VLAN_STRIP_OFFLOAD; - if (dev->data->dev_conf.rxmode.hw_vlan_filter) + if (dev->data->dev_conf.rxmode.offloads & + DEV_RX_OFFLOAD_VLAN_FILTER) ret |= ETH_VLAN_FILTER_OFFLOAD; - if (dev->data->dev_conf.rxmode.hw_vlan_extend) + if (dev->data->dev_conf.rxmode.offloads & + DEV_RX_OFFLOAD_VLAN_EXTEND) ret |= ETH_VLAN_EXTEND_OFFLOAD; return ret; } int -rte_eth_dev_set_vlan_pvid(uint8_t port_id, uint16_t pvid, int on) +rte_eth_dev_set_vlan_pvid(uint16_t port_id, uint16_t pvid, int on) { struct rte_eth_dev *dev; @@ -1940,7 +2298,7 @@ rte_eth_dev_set_vlan_pvid(uint8_t port_id, uint16_t pvid, int on) } int -rte_eth_dev_flow_ctrl_get(uint8_t port_id, struct rte_eth_fc_conf *fc_conf) +rte_eth_dev_flow_ctrl_get(uint16_t port_id, struct rte_eth_fc_conf *fc_conf) { struct rte_eth_dev *dev; @@ -1952,7 +2310,7 @@ rte_eth_dev_flow_ctrl_get(uint8_t port_id, struct rte_eth_fc_conf *fc_conf) } int -rte_eth_dev_flow_ctrl_set(uint8_t port_id, struct rte_eth_fc_conf *fc_conf) +rte_eth_dev_flow_ctrl_set(uint16_t port_id, struct rte_eth_fc_conf *fc_conf) { struct rte_eth_dev *dev; @@ -1968,7 +2326,8 @@ rte_eth_dev_flow_ctrl_set(uint8_t port_id, struct rte_eth_fc_conf *fc_conf) } int -rte_eth_dev_priority_flow_ctrl_set(uint8_t port_id, struct rte_eth_pfc_conf *pfc_conf) +rte_eth_dev_priority_flow_ctrl_set(uint16_t port_id, + struct rte_eth_pfc_conf *pfc_conf) { struct rte_eth_dev *dev; @@ -1994,13 +2353,7 @@ rte_eth_check_reta_mask(struct rte_eth_rss_reta_entry64 *reta_conf, if (!reta_conf) return -EINVAL; - if (reta_size != RTE_ALIGN(reta_size, RTE_RETA_GROUP_SIZE)) { - RTE_PMD_DEBUG_TRACE("Invalid reta size, should be %u aligned\n", - RTE_RETA_GROUP_SIZE); - return -EINVAL; - } - - num = reta_size / RTE_RETA_GROUP_SIZE; + num = (reta_size + RTE_RETA_GROUP_SIZE - 1) / RTE_RETA_GROUP_SIZE; for (i = 0; i < num; i++) { if (reta_conf[i].mask) return 0; @@ -2040,7 +2393,7 @@ rte_eth_check_reta_entry(struct rte_eth_rss_reta_entry64 *reta_conf, } int -rte_eth_dev_rss_reta_update(uint8_t port_id, +rte_eth_dev_rss_reta_update(uint16_t port_id, struct rte_eth_rss_reta_entry64 *reta_conf, uint16_t reta_size) { @@ -2066,7 +2419,7 @@ rte_eth_dev_rss_reta_update(uint8_t port_id, } int -rte_eth_dev_rss_reta_query(uint8_t port_id, +rte_eth_dev_rss_reta_query(uint16_t port_id, struct rte_eth_rss_reta_entry64 *reta_conf, uint16_t reta_size) { @@ -2086,26 +2439,19 @@ rte_eth_dev_rss_reta_query(uint8_t port_id, } int -rte_eth_dev_rss_hash_update(uint8_t port_id, struct rte_eth_rss_conf *rss_conf) +rte_eth_dev_rss_hash_update(uint16_t port_id, + struct rte_eth_rss_conf *rss_conf) { struct rte_eth_dev *dev; - uint16_t rss_hash_protos; RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); - rss_hash_protos = rss_conf->rss_hf; - if ((rss_hash_protos != 0) && - ((rss_hash_protos & ETH_RSS_PROTO_MASK) == 0)) { - RTE_PMD_DEBUG_TRACE("Invalid rss_hash_protos=0x%x\n", - rss_hash_protos); - return -EINVAL; - } dev = &rte_eth_devices[port_id]; RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rss_hash_update, -ENOTSUP); return (*dev->dev_ops->rss_hash_update)(dev, rss_conf); } int -rte_eth_dev_rss_hash_conf_get(uint8_t port_id, +rte_eth_dev_rss_hash_conf_get(uint16_t port_id, struct rte_eth_rss_conf *rss_conf) { struct rte_eth_dev *dev; @@ -2117,7 +2463,7 @@ rte_eth_dev_rss_hash_conf_get(uint8_t port_id, } int -rte_eth_dev_udp_tunnel_port_add(uint8_t port_id, +rte_eth_dev_udp_tunnel_port_add(uint16_t port_id, struct rte_eth_udp_tunnel *udp_tunnel) { struct rte_eth_dev *dev; @@ -2139,7 +2485,7 @@ rte_eth_dev_udp_tunnel_port_add(uint8_t port_id, } int -rte_eth_dev_udp_tunnel_port_delete(uint8_t port_id, +rte_eth_dev_udp_tunnel_port_delete(uint16_t port_id, struct rte_eth_udp_tunnel *udp_tunnel) { struct rte_eth_dev *dev; @@ -2162,7 +2508,7 @@ rte_eth_dev_udp_tunnel_port_delete(uint8_t port_id, } int -rte_eth_led_on(uint8_t port_id) +rte_eth_led_on(uint16_t port_id) { struct rte_eth_dev *dev; @@ -2173,7 +2519,7 @@ rte_eth_led_on(uint8_t port_id) } int -rte_eth_led_off(uint8_t port_id) +rte_eth_led_off(uint16_t port_id) { struct rte_eth_dev *dev; @@ -2188,12 +2534,13 @@ rte_eth_led_off(uint8_t port_id) * an empty spot. */ static int -get_mac_addr_index(uint8_t port_id, const struct ether_addr *addr) +get_mac_addr_index(uint16_t port_id, const struct ether_addr *addr) { struct rte_eth_dev_info dev_info; struct rte_eth_dev *dev = &rte_eth_devices[port_id]; unsigned i; + RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); rte_eth_dev_info_get(port_id, &dev_info); for (i = 0; i < dev_info.max_mac_addrs; i++) @@ -2206,12 +2553,13 @@ get_mac_addr_index(uint8_t port_id, const struct ether_addr *addr) static const struct ether_addr null_mac_addr; int -rte_eth_dev_mac_addr_add(uint8_t port_id, struct ether_addr *addr, +rte_eth_dev_mac_addr_add(uint16_t port_id, struct ether_addr *addr, uint32_t pool) { struct rte_eth_dev *dev; int index; uint64_t pool_mask; + int ret; RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); dev = &rte_eth_devices[port_id]; @@ -2244,19 +2592,21 @@ rte_eth_dev_mac_addr_add(uint8_t port_id, struct ether_addr *addr, } /* Update NIC */ - (*dev->dev_ops->mac_addr_add)(dev, addr, index, pool); + ret = (*dev->dev_ops->mac_addr_add)(dev, addr, index, pool); - /* Update address in NIC data structure */ - ether_addr_copy(addr, &dev->data->mac_addrs[index]); + if (ret == 0) { + /* Update address in NIC data structure */ + ether_addr_copy(addr, &dev->data->mac_addrs[index]); - /* Update pool bitmap in NIC data structure */ - dev->data->mac_pool_sel[index] |= (1ULL << pool); + /* Update pool bitmap in NIC data structure */ + dev->data->mac_pool_sel[index] |= (1ULL << pool); + } - return 0; + return ret; } int -rte_eth_dev_mac_addr_remove(uint8_t port_id, struct ether_addr *addr) +rte_eth_dev_mac_addr_remove(uint16_t port_id, struct ether_addr *addr) { struct rte_eth_dev *dev; int index; @@ -2285,7 +2635,7 @@ rte_eth_dev_mac_addr_remove(uint8_t port_id, struct ether_addr *addr) } int -rte_eth_dev_default_mac_addr_set(uint8_t port_id, struct ether_addr *addr) +rte_eth_dev_default_mac_addr_set(uint16_t port_id, struct ether_addr *addr) { struct rte_eth_dev *dev; @@ -2305,39 +2655,13 @@ rte_eth_dev_default_mac_addr_set(uint8_t port_id, struct ether_addr *addr) return 0; } -int -rte_eth_dev_set_vf_rxmode(uint8_t port_id, uint16_t vf, - uint16_t rx_mode, uint8_t on) -{ - uint16_t num_vfs; - struct rte_eth_dev *dev; - struct rte_eth_dev_info dev_info; - - RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); - - dev = &rte_eth_devices[port_id]; - rte_eth_dev_info_get(port_id, &dev_info); - - num_vfs = dev_info.max_vfs; - if (vf > num_vfs) { - RTE_PMD_DEBUG_TRACE("set VF RX mode:invalid VF id %d\n", vf); - return -EINVAL; - } - - if (rx_mode == 0) { - RTE_PMD_DEBUG_TRACE("set VF RX mode:mode mask ca not be zero\n"); - return -EINVAL; - } - RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->set_vf_rx_mode, -ENOTSUP); - return (*dev->dev_ops->set_vf_rx_mode)(dev, vf, rx_mode, on); -} /* * Returns index into MAC address array of addr. Use 00:00:00:00:00:00 to find * an empty spot. */ static int -get_hash_mac_addr_index(uint8_t port_id, const struct ether_addr *addr) +get_hash_mac_addr_index(uint16_t port_id, const struct ether_addr *addr) { struct rte_eth_dev_info dev_info; struct rte_eth_dev *dev = &rte_eth_devices[port_id]; @@ -2356,7 +2680,7 @@ get_hash_mac_addr_index(uint8_t port_id, const struct ether_addr *addr) } int -rte_eth_dev_uc_hash_table_set(uint8_t port_id, struct ether_addr *addr, +rte_eth_dev_uc_hash_table_set(uint16_t port_id, struct ether_addr *addr, uint8_t on) { int index; @@ -2408,7 +2732,7 @@ rte_eth_dev_uc_hash_table_set(uint8_t port_id, struct ether_addr *addr, } int -rte_eth_dev_uc_all_hash_table_set(uint8_t port_id, uint8_t on) +rte_eth_dev_uc_all_hash_table_set(uint16_t port_id, uint8_t on) { struct rte_eth_dev *dev; @@ -2420,77 +2744,7 @@ rte_eth_dev_uc_all_hash_table_set(uint8_t port_id, uint8_t on) return (*dev->dev_ops->uc_all_hash_table_set)(dev, on); } -int -rte_eth_dev_set_vf_rx(uint8_t port_id, uint16_t vf, uint8_t on) -{ - uint16_t num_vfs; - struct rte_eth_dev *dev; - struct rte_eth_dev_info dev_info; - - RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); - - dev = &rte_eth_devices[port_id]; - rte_eth_dev_info_get(port_id, &dev_info); - - num_vfs = dev_info.max_vfs; - if (vf > num_vfs) { - RTE_PMD_DEBUG_TRACE("port %d: invalid vf id\n", port_id); - return -EINVAL; - } - - RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->set_vf_rx, -ENOTSUP); - return (*dev->dev_ops->set_vf_rx)(dev, vf, on); -} - -int -rte_eth_dev_set_vf_tx(uint8_t port_id, uint16_t vf, uint8_t on) -{ - uint16_t num_vfs; - struct rte_eth_dev *dev; - struct rte_eth_dev_info dev_info; - - RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); - - dev = &rte_eth_devices[port_id]; - rte_eth_dev_info_get(port_id, &dev_info); - - num_vfs = dev_info.max_vfs; - if (vf > num_vfs) { - RTE_PMD_DEBUG_TRACE("set pool tx:invalid pool id=%d\n", vf); - return -EINVAL; - } - - RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->set_vf_tx, -ENOTSUP); - return (*dev->dev_ops->set_vf_tx)(dev, vf, on); -} - -int -rte_eth_dev_set_vf_vlan_filter(uint8_t port_id, uint16_t vlan_id, - uint64_t vf_mask, uint8_t vlan_on) -{ - struct rte_eth_dev *dev; - - RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); - - dev = &rte_eth_devices[port_id]; - - if (vlan_id > ETHER_MAX_VLAN_ID) { - RTE_PMD_DEBUG_TRACE("VF VLAN filter:invalid VLAN id=%d\n", - vlan_id); - return -EINVAL; - } - - if (vf_mask == 0) { - RTE_PMD_DEBUG_TRACE("VF VLAN filter:pool_mask can not be 0\n"); - return -EINVAL; - } - - RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->set_vf_vlan_filter, -ENOTSUP); - return (*dev->dev_ops->set_vf_vlan_filter)(dev, vlan_id, - vf_mask, vlan_on); -} - -int rte_eth_set_queue_rate_limit(uint8_t port_id, uint16_t queue_idx, +int rte_eth_set_queue_rate_limit(uint16_t port_id, uint16_t queue_idx, uint16_t tx_rate) { struct rte_eth_dev *dev; @@ -2520,45 +2774,12 @@ int rte_eth_set_queue_rate_limit(uint8_t port_id, uint16_t queue_idx, return (*dev->dev_ops->set_queue_rate_limit)(dev, queue_idx, tx_rate); } -int rte_eth_set_vf_rate_limit(uint8_t port_id, uint16_t vf, uint16_t tx_rate, - uint64_t q_msk) -{ - struct rte_eth_dev *dev; - struct rte_eth_dev_info dev_info; - struct rte_eth_link link; - - if (q_msk == 0) - return 0; - - RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); - - dev = &rte_eth_devices[port_id]; - rte_eth_dev_info_get(port_id, &dev_info); - link = dev->data->dev_link; - - if (vf > dev_info.max_vfs) { - RTE_PMD_DEBUG_TRACE("set VF rate limit:port %d: " - "invalid vf id=%d\n", port_id, vf); - return -EINVAL; - } - - if (tx_rate > link.link_speed) { - RTE_PMD_DEBUG_TRACE("set VF rate limit:invalid tx_rate=%d, " - "bigger than link speed= %d\n", - tx_rate, link.link_speed); - return -EINVAL; - } - - RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->set_vf_rate_limit, -ENOTSUP); - return (*dev->dev_ops->set_vf_rate_limit)(dev, vf, tx_rate, q_msk); -} - int -rte_eth_mirror_rule_set(uint8_t port_id, +rte_eth_mirror_rule_set(uint16_t port_id, struct rte_eth_mirror_conf *mirror_conf, uint8_t rule_id, uint8_t on) { - struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + struct rte_eth_dev *dev; RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); if (mirror_conf->rule_type == 0) { @@ -2592,9 +2813,9 @@ rte_eth_mirror_rule_set(uint8_t port_id, } int -rte_eth_mirror_rule_reset(uint8_t port_id, uint8_t rule_id) +rte_eth_mirror_rule_reset(uint16_t port_id, uint8_t rule_id) { - struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + struct rte_eth_dev *dev; RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); @@ -2605,7 +2826,7 @@ rte_eth_mirror_rule_reset(uint8_t port_id, uint8_t rule_id) } int -rte_eth_dev_callback_register(uint8_t port_id, +rte_eth_dev_callback_register(uint16_t port_id, enum rte_eth_event_type event, rte_eth_dev_cb_fn cb_fn, void *cb_arg) { @@ -2645,7 +2866,7 @@ rte_eth_dev_callback_register(uint8_t port_id, } int -rte_eth_dev_callback_unregister(uint8_t port_id, +rte_eth_dev_callback_unregister(uint16_t port_id, enum rte_eth_event_type event, rte_eth_dev_cb_fn cb_fn, void *cb_arg) { @@ -2687,12 +2908,13 @@ rte_eth_dev_callback_unregister(uint8_t port_id, return ret; } -void +int _rte_eth_dev_callback_process(struct rte_eth_dev *dev, - enum rte_eth_event_type event) + enum rte_eth_event_type event, void *cb_arg, void *ret_param) { struct rte_eth_dev_callback *cb_lst; struct rte_eth_dev_callback dev_cb; + int rc = 0; rte_spinlock_lock(&rte_eth_dev_cb_lock); TAILQ_FOREACH(cb_lst, &(dev->link_intr_cbs), next) { @@ -2700,17 +2922,23 @@ _rte_eth_dev_callback_process(struct rte_eth_dev *dev, continue; dev_cb = *cb_lst; cb_lst->active = 1; + if (cb_arg != NULL) + dev_cb.cb_arg = cb_arg; + if (ret_param != NULL) + dev_cb.ret_param = ret_param; + rte_spinlock_unlock(&rte_eth_dev_cb_lock); - dev_cb.cb_fn(dev->data->port_id, dev_cb.event, - dev_cb.cb_arg); + rc = dev_cb.cb_fn(dev->data->port_id, dev_cb.event, + dev_cb.cb_arg, dev_cb.ret_param); rte_spinlock_lock(&rte_eth_dev_cb_lock); cb_lst->active = 0; } rte_spinlock_unlock(&rte_eth_dev_cb_lock); + return rc; } int -rte_eth_dev_rx_intr_ctl(uint8_t port_id, int epfd, int op, void *data) +rte_eth_dev_rx_intr_ctl(uint16_t port_id, int epfd, int op, void *data) { uint32_t vec; struct rte_eth_dev *dev; @@ -2721,7 +2949,13 @@ rte_eth_dev_rx_intr_ctl(uint8_t port_id, int epfd, int op, void *data) RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); dev = &rte_eth_devices[port_id]; - intr_handle = &dev->pci_dev->intr_handle; + + if (!dev->intr_handle) { + RTE_PMD_DEBUG_TRACE("RX Intr handle unset\n"); + return -ENOTSUP; + } + + intr_handle = dev->intr_handle; if (!intr_handle->intr_vec) { RTE_PMD_DEBUG_TRACE("RX Intr vector unset\n"); return -EPERM; @@ -2749,23 +2983,18 @@ rte_eth_dma_zone_reserve(const struct rte_eth_dev *dev, const char *ring_name, const struct rte_memzone *mz; snprintf(z_name, sizeof(z_name), "%s_%s_%d_%d", - dev->driver->pci_drv.name, ring_name, + dev->device->driver->name, ring_name, dev->data->port_id, queue_id); mz = rte_memzone_lookup(z_name); if (mz) return mz; - if (rte_xen_dom0_supported()) - return rte_memzone_reserve_bounded(z_name, size, socket_id, - 0, align, RTE_PGSIZE_2M); - else - return rte_memzone_reserve_aligned(z_name, size, socket_id, - 0, align); + return rte_memzone_reserve_aligned(z_name, size, socket_id, 0, align); } int -rte_eth_dev_rx_intr_ctl_q(uint8_t port_id, uint16_t queue_id, +rte_eth_dev_rx_intr_ctl_q(uint16_t port_id, uint16_t queue_id, int epfd, int op, void *data) { uint32_t vec; @@ -2781,7 +3010,12 @@ rte_eth_dev_rx_intr_ctl_q(uint8_t port_id, uint16_t queue_id, return -EINVAL; } - intr_handle = &dev->pci_dev->intr_handle; + if (!dev->intr_handle) { + RTE_PMD_DEBUG_TRACE("RX Intr handle unset\n"); + return -ENOTSUP; + } + + intr_handle = dev->intr_handle; if (!intr_handle->intr_vec) { RTE_PMD_DEBUG_TRACE("RX Intr vector unset\n"); return -EPERM; @@ -2800,7 +3034,7 @@ rte_eth_dev_rx_intr_ctl_q(uint8_t port_id, uint16_t queue_id, } int -rte_eth_dev_rx_intr_enable(uint8_t port_id, +rte_eth_dev_rx_intr_enable(uint16_t port_id, uint16_t queue_id) { struct rte_eth_dev *dev; @@ -2814,7 +3048,7 @@ rte_eth_dev_rx_intr_enable(uint8_t port_id, } int -rte_eth_dev_rx_intr_disable(uint8_t port_id, +rte_eth_dev_rx_intr_disable(uint16_t port_id, uint16_t queue_id) { struct rte_eth_dev *dev; @@ -2827,131 +3061,10 @@ rte_eth_dev_rx_intr_disable(uint8_t port_id, return (*dev->dev_ops->rx_queue_intr_disable)(dev, queue_id); } -#ifdef RTE_NIC_BYPASS -int rte_eth_dev_bypass_init(uint8_t port_id) -{ - struct rte_eth_dev *dev; - - RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); - - dev = &rte_eth_devices[port_id]; - RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->bypass_init, -ENOTSUP); - (*dev->dev_ops->bypass_init)(dev); - return 0; -} int -rte_eth_dev_bypass_state_show(uint8_t port_id, uint32_t *state) -{ - struct rte_eth_dev *dev; - - RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); - - dev = &rte_eth_devices[port_id]; - RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->bypass_state_show, -ENOTSUP); - (*dev->dev_ops->bypass_state_show)(dev, state); - return 0; -} - -int -rte_eth_dev_bypass_state_set(uint8_t port_id, uint32_t *new_state) -{ - struct rte_eth_dev *dev; - - RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); - - dev = &rte_eth_devices[port_id]; - RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->bypass_state_set, -ENOTSUP); - (*dev->dev_ops->bypass_state_set)(dev, new_state); - return 0; -} - -int -rte_eth_dev_bypass_event_show(uint8_t port_id, uint32_t event, uint32_t *state) -{ - struct rte_eth_dev *dev; - - RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); - - dev = &rte_eth_devices[port_id]; - RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->bypass_state_show, -ENOTSUP); - (*dev->dev_ops->bypass_event_show)(dev, event, state); - return 0; -} - -int -rte_eth_dev_bypass_event_store(uint8_t port_id, uint32_t event, uint32_t state) -{ - struct rte_eth_dev *dev; - - RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); - - dev = &rte_eth_devices[port_id]; - - RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->bypass_event_set, -ENOTSUP); - (*dev->dev_ops->bypass_event_set)(dev, event, state); - return 0; -} - -int -rte_eth_dev_wd_timeout_store(uint8_t port_id, uint32_t timeout) -{ - struct rte_eth_dev *dev; - - RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); - - dev = &rte_eth_devices[port_id]; - - RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->bypass_wd_timeout_set, -ENOTSUP); - (*dev->dev_ops->bypass_wd_timeout_set)(dev, timeout); - return 0; -} - -int -rte_eth_dev_bypass_ver_show(uint8_t port_id, uint32_t *ver) -{ - struct rte_eth_dev *dev; - - RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); - - dev = &rte_eth_devices[port_id]; - - RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->bypass_ver_show, -ENOTSUP); - (*dev->dev_ops->bypass_ver_show)(dev, ver); - return 0; -} - -int -rte_eth_dev_bypass_wd_timeout_show(uint8_t port_id, uint32_t *wd_timeout) -{ - struct rte_eth_dev *dev; - - RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); - - dev = &rte_eth_devices[port_id]; - - RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->bypass_wd_timeout_show, -ENOTSUP); - (*dev->dev_ops->bypass_wd_timeout_show)(dev, wd_timeout); - return 0; -} - -int -rte_eth_dev_bypass_wd_reset(uint8_t port_id) -{ - struct rte_eth_dev *dev; - - RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); - - dev = &rte_eth_devices[port_id]; - - RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->bypass_wd_reset, -ENOTSUP); - (*dev->dev_ops->bypass_wd_reset)(dev); - return 0; -} -#endif - -int -rte_eth_dev_filter_supported(uint8_t port_id, enum rte_filter_type filter_type) +rte_eth_dev_filter_supported(uint16_t port_id, + enum rte_filter_type filter_type) { struct rte_eth_dev *dev; @@ -2964,7 +3077,7 @@ rte_eth_dev_filter_supported(uint8_t port_id, enum rte_filter_type filter_type) } int -rte_eth_dev_filter_ctrl(uint8_t port_id, enum rte_filter_type filter_type, +rte_eth_dev_filter_ctrl(uint16_t port_id, enum rte_filter_type filter_type, enum rte_filter_op filter_op, void *arg) { struct rte_eth_dev *dev; @@ -2977,7 +3090,7 @@ rte_eth_dev_filter_ctrl(uint8_t port_id, enum rte_filter_type filter_type, } void * -rte_eth_add_rx_callback(uint8_t port_id, uint16_t queue_id, +rte_eth_add_rx_callback(uint16_t port_id, uint16_t queue_id, rte_rx_callback_fn fn, void *user_param) { #ifndef RTE_ETHDEV_RXTX_CALLBACKS @@ -3019,7 +3132,7 @@ rte_eth_add_rx_callback(uint8_t port_id, uint16_t queue_id, } void * -rte_eth_add_first_rx_callback(uint8_t port_id, uint16_t queue_id, +rte_eth_add_first_rx_callback(uint16_t port_id, uint16_t queue_id, rte_rx_callback_fn fn, void *user_param) { #ifndef RTE_ETHDEV_RXTX_CALLBACKS @@ -3054,7 +3167,7 @@ rte_eth_add_first_rx_callback(uint8_t port_id, uint16_t queue_id, } void * -rte_eth_add_tx_callback(uint8_t port_id, uint16_t queue_id, +rte_eth_add_tx_callback(uint16_t port_id, uint16_t queue_id, rte_tx_callback_fn fn, void *user_param) { #ifndef RTE_ETHDEV_RXTX_CALLBACKS @@ -3097,7 +3210,7 @@ rte_eth_add_tx_callback(uint8_t port_id, uint16_t queue_id, } int -rte_eth_remove_rx_callback(uint8_t port_id, uint16_t queue_id, +rte_eth_remove_rx_callback(uint16_t port_id, uint16_t queue_id, struct rte_eth_rxtx_callback *user_cb) { #ifndef RTE_ETHDEV_RXTX_CALLBACKS @@ -3131,7 +3244,7 @@ rte_eth_remove_rx_callback(uint8_t port_id, uint16_t queue_id, } int -rte_eth_remove_tx_callback(uint8_t port_id, uint16_t queue_id, +rte_eth_remove_tx_callback(uint16_t port_id, uint16_t queue_id, struct rte_eth_rxtx_callback *user_cb) { #ifndef RTE_ETHDEV_RXTX_CALLBACKS @@ -3165,7 +3278,7 @@ rte_eth_remove_tx_callback(uint8_t port_id, uint16_t queue_id, } int -rte_eth_rx_queue_info_get(uint8_t port_id, uint16_t queue_id, +rte_eth_rx_queue_info_get(uint16_t port_id, uint16_t queue_id, struct rte_eth_rxq_info *qinfo) { struct rte_eth_dev *dev; @@ -3189,7 +3302,7 @@ rte_eth_rx_queue_info_get(uint8_t port_id, uint16_t queue_id, } int -rte_eth_tx_queue_info_get(uint8_t port_id, uint16_t queue_id, +rte_eth_tx_queue_info_get(uint16_t port_id, uint16_t queue_id, struct rte_eth_txq_info *qinfo) { struct rte_eth_dev *dev; @@ -3213,7 +3326,7 @@ rte_eth_tx_queue_info_get(uint8_t port_id, uint16_t queue_id, } int -rte_eth_dev_set_mc_addr_list(uint8_t port_id, +rte_eth_dev_set_mc_addr_list(uint16_t port_id, struct ether_addr *mc_addr_set, uint32_t nb_mc_addr) { @@ -3227,7 +3340,7 @@ rte_eth_dev_set_mc_addr_list(uint8_t port_id, } int -rte_eth_timesync_enable(uint8_t port_id) +rte_eth_timesync_enable(uint16_t port_id) { struct rte_eth_dev *dev; @@ -3239,7 +3352,7 @@ rte_eth_timesync_enable(uint8_t port_id) } int -rte_eth_timesync_disable(uint8_t port_id) +rte_eth_timesync_disable(uint16_t port_id) { struct rte_eth_dev *dev; @@ -3251,7 +3364,7 @@ rte_eth_timesync_disable(uint8_t port_id) } int -rte_eth_timesync_read_rx_timestamp(uint8_t port_id, struct timespec *timestamp, +rte_eth_timesync_read_rx_timestamp(uint16_t port_id, struct timespec *timestamp, uint32_t flags) { struct rte_eth_dev *dev; @@ -3264,7 +3377,8 @@ rte_eth_timesync_read_rx_timestamp(uint8_t port_id, struct timespec *timestamp, } int -rte_eth_timesync_read_tx_timestamp(uint8_t port_id, struct timespec *timestamp) +rte_eth_timesync_read_tx_timestamp(uint16_t port_id, + struct timespec *timestamp) { struct rte_eth_dev *dev; @@ -3276,7 +3390,7 @@ rte_eth_timesync_read_tx_timestamp(uint8_t port_id, struct timespec *timestamp) } int -rte_eth_timesync_adjust_time(uint8_t port_id, int64_t delta) +rte_eth_timesync_adjust_time(uint16_t port_id, int64_t delta) { struct rte_eth_dev *dev; @@ -3288,7 +3402,7 @@ rte_eth_timesync_adjust_time(uint8_t port_id, int64_t delta) } int -rte_eth_timesync_read_time(uint8_t port_id, struct timespec *timestamp) +rte_eth_timesync_read_time(uint16_t port_id, struct timespec *timestamp) { struct rte_eth_dev *dev; @@ -3300,7 +3414,7 @@ rte_eth_timesync_read_time(uint8_t port_id, struct timespec *timestamp) } int -rte_eth_timesync_write_time(uint8_t port_id, const struct timespec *timestamp) +rte_eth_timesync_write_time(uint16_t port_id, const struct timespec *timestamp) { struct rte_eth_dev *dev; @@ -3312,7 +3426,7 @@ rte_eth_timesync_write_time(uint8_t port_id, const struct timespec *timestamp) } int -rte_eth_dev_get_reg_info(uint8_t port_id, struct rte_dev_reg_info *info) +rte_eth_dev_get_reg_info(uint16_t port_id, struct rte_dev_reg_info *info) { struct rte_eth_dev *dev; @@ -3324,7 +3438,7 @@ rte_eth_dev_get_reg_info(uint8_t port_id, struct rte_dev_reg_info *info) } int -rte_eth_dev_get_eeprom_length(uint8_t port_id) +rte_eth_dev_get_eeprom_length(uint16_t port_id) { struct rte_eth_dev *dev; @@ -3336,7 +3450,7 @@ rte_eth_dev_get_eeprom_length(uint8_t port_id) } int -rte_eth_dev_get_eeprom(uint8_t port_id, struct rte_dev_eeprom_info *info) +rte_eth_dev_get_eeprom(uint16_t port_id, struct rte_dev_eeprom_info *info) { struct rte_eth_dev *dev; @@ -3348,7 +3462,7 @@ rte_eth_dev_get_eeprom(uint8_t port_id, struct rte_dev_eeprom_info *info) } int -rte_eth_dev_set_eeprom(uint8_t port_id, struct rte_dev_eeprom_info *info) +rte_eth_dev_set_eeprom(uint16_t port_id, struct rte_dev_eeprom_info *info) { struct rte_eth_dev *dev; @@ -3360,7 +3474,7 @@ rte_eth_dev_set_eeprom(uint8_t port_id, struct rte_dev_eeprom_info *info) } int -rte_eth_dev_get_dcb_info(uint8_t port_id, +rte_eth_dev_get_dcb_info(uint16_t port_id, struct rte_eth_dcb_info *dcb_info) { struct rte_eth_dev *dev; @@ -3374,28 +3488,8 @@ rte_eth_dev_get_dcb_info(uint8_t port_id, return (*dev->dev_ops->get_dcb_info)(dev, dcb_info); } -void -rte_eth_copy_pci_info(struct rte_eth_dev *eth_dev, struct rte_pci_device *pci_dev) -{ - if ((eth_dev == NULL) || (pci_dev == NULL)) { - RTE_PMD_DEBUG_TRACE("NULL pointer eth_dev=%p pci_dev=%p\n", - eth_dev, pci_dev); - return; - } - - eth_dev->data->dev_flags = 0; - if (pci_dev->driver->drv_flags & RTE_PCI_DRV_INTR_LSC) - eth_dev->data->dev_flags |= RTE_ETH_DEV_INTR_LSC; - if (pci_dev->driver->drv_flags & RTE_PCI_DRV_DETACHABLE) - eth_dev->data->dev_flags |= RTE_ETH_DEV_DETACHABLE; - - eth_dev->data->kdrv = pci_dev->kdrv; - eth_dev->data->numa_node = pci_dev->numa_node; - eth_dev->data->drv_name = pci_dev->driver->name; -} - int -rte_eth_dev_l2_tunnel_eth_type_conf(uint8_t port_id, +rte_eth_dev_l2_tunnel_eth_type_conf(uint16_t port_id, struct rte_eth_l2_tunnel_conf *l2_tunnel) { struct rte_eth_dev *dev; @@ -3418,7 +3512,7 @@ rte_eth_dev_l2_tunnel_eth_type_conf(uint8_t port_id, } int -rte_eth_dev_l2_tunnel_offload_set(uint8_t port_id, +rte_eth_dev_l2_tunnel_offload_set(uint16_t port_id, struct rte_eth_l2_tunnel_conf *l2_tunnel, uint32_t mask, uint8_t en) @@ -3447,3 +3541,58 @@ rte_eth_dev_l2_tunnel_offload_set(uint8_t port_id, -ENOTSUP); return (*dev->dev_ops->l2_tunnel_offload_set)(dev, l2_tunnel, mask, en); } + +static void +rte_eth_dev_adjust_nb_desc(uint16_t *nb_desc, + const struct rte_eth_desc_lim *desc_lim) +{ + if (desc_lim->nb_align != 0) + *nb_desc = RTE_ALIGN_CEIL(*nb_desc, desc_lim->nb_align); + + if (desc_lim->nb_max != 0) + *nb_desc = RTE_MIN(*nb_desc, desc_lim->nb_max); + + *nb_desc = RTE_MAX(*nb_desc, desc_lim->nb_min); +} + +int +rte_eth_dev_adjust_nb_rx_tx_desc(uint16_t port_id, + uint16_t *nb_rx_desc, + uint16_t *nb_tx_desc) +{ + struct rte_eth_dev *dev; + struct rte_eth_dev_info dev_info; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); + + dev = &rte_eth_devices[port_id]; + RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP); + + rte_eth_dev_info_get(port_id, &dev_info); + + if (nb_rx_desc != NULL) + rte_eth_dev_adjust_nb_desc(nb_rx_desc, &dev_info.rx_desc_lim); + + if (nb_tx_desc != NULL) + rte_eth_dev_adjust_nb_desc(nb_tx_desc, &dev_info.tx_desc_lim); + + return 0; +} + +int +rte_eth_dev_pool_ops_supported(uint16_t port_id, const char *pool) +{ + struct rte_eth_dev *dev; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); + + if (pool == NULL) + return -EINVAL; + + dev = &rte_eth_devices[port_id]; + + if (*dev->dev_ops->pool_ops_supported == NULL) + return 1; /* all pools are supported */ + + return (*dev->dev_ops->pool_ops_supported)(dev, pool); +} diff --git a/dpdk/lib/librte_ether/rte_ethdev.h b/dpdk/lib/librte_ether/rte_ethdev.h index b0fe0334..2cc2eedf 100644 --- a/dpdk/lib/librte_ether/rte_ethdev.h +++ b/dpdk/lib/librte_ether/rte_ethdev.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. + * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -118,7 +118,7 @@ * - NIC queue statistics mappings * * Any other configuration will not be stored and will need to be re-entered - * after a call to rte_eth_dev_start(). + * before a call to rte_eth_dev_start(). * * Finally, a network application can close an Ethernet device by invoking the * rte_eth_dev_close() function. @@ -172,16 +172,17 @@ extern "C" { #include -#include - /* Use this macro to check if LRO API is supported */ #define RTE_ETHDEV_HAS_LRO_SUPPORT #include #include -#include #include #include +#include +#include +#include + #include "rte_ether.h" #include "rte_eth_ctrl.h" #include "rte_dev_info.h" @@ -190,6 +191,9 @@ struct rte_mbuf; /** * A structure used to retrieve statistics for an Ethernet port. + * Not all statistics fields in struct rte_eth_stats are supported + * by any type of network interface card (NIC). If any statistics + * field is not supported, its value is 0. */ struct rte_eth_stats { uint64_t ipackets; /**< Total number of successfully received packets. */ @@ -198,7 +202,7 @@ struct rte_eth_stats { uint64_t obytes; /**< Total number of successfully transmitted bytes. */ uint64_t imissed; /**< Total of RX packets dropped by the HW, - * because there are no available mbufs (i.e. RX queues are full). + * because there are no available buffer (i.e. RX queues are full). */ uint64_t ierrors; /**< Total number of erroneous received packets. */ uint64_t oerrors; /**< Total number of failed transmitted packets. */ @@ -255,20 +259,21 @@ struct rte_eth_stats { /** * A structure used to retrieve link-level information of an Ethernet port. */ +__extension__ struct rte_eth_link { uint32_t link_speed; /**< ETH_SPEED_NUM_ */ uint16_t link_duplex : 1; /**< ETH_LINK_[HALF/FULL]_DUPLEX */ - uint16_t link_autoneg : 1; /**< ETH_LINK_SPEED_[AUTONEG/FIXED] */ + uint16_t link_autoneg : 1; /**< ETH_LINK_[AUTONEG/FIXED] */ uint16_t link_status : 1; /**< ETH_LINK_[DOWN/UP] */ } __attribute__((aligned(8))); /**< aligned for atomic64 read/write */ /* Utility constants */ -#define ETH_LINK_HALF_DUPLEX 0 /**< Half-duplex connection. */ -#define ETH_LINK_FULL_DUPLEX 1 /**< Full-duplex connection. */ -#define ETH_LINK_DOWN 0 /**< Link is down. */ -#define ETH_LINK_UP 1 /**< Link is up. */ -#define ETH_LINK_FIXED 0 /**< No autonegotiation. */ -#define ETH_LINK_AUTONEG 1 /**< Autonegotiated. */ +#define ETH_LINK_HALF_DUPLEX 0 /**< Half-duplex connection (see link_duplex). */ +#define ETH_LINK_FULL_DUPLEX 1 /**< Full-duplex connection (see link_duplex). */ +#define ETH_LINK_DOWN 0 /**< Link is down (see link_status). */ +#define ETH_LINK_UP 1 /**< Link is up (see link_status). */ +#define ETH_LINK_FIXED 0 /**< No autonegotiation (see link_autoneg). */ +#define ETH_LINK_AUTONEG 1 /**< Autonegotiated (see link_autoneg). */ /** * A structure used to configure the ring threshold registers of an RX/TX @@ -346,6 +351,18 @@ struct rte_eth_rxmode { enum rte_eth_rx_mq_mode mq_mode; uint32_t max_rx_pkt_len; /**< Only used if jumbo_frame enabled. */ uint16_t split_hdr_size; /**< hdr buf size (header_split enabled).*/ + /** + * Per-port Rx offloads to be set using DEV_RX_OFFLOAD_* flags. + * Only offloads set on rx_offload_capa field on rte_eth_dev_info + * structure are allowed to be set. + */ + uint64_t offloads; + __extension__ + /** + * Below bitfield API is obsolete. Application should + * enable per-port offloads using the offload field + * above. + */ uint16_t header_split : 1, /**< Header Split enable. */ hw_ip_checksum : 1, /**< IP/UDP/TCP checksum offload enable. */ hw_vlan_filter : 1, /**< VLAN filter enable. */ @@ -354,7 +371,19 @@ struct rte_eth_rxmode { jumbo_frame : 1, /**< Jumbo Frame Receipt enable. */ hw_strip_crc : 1, /**< Enable CRC stripping by hardware. */ enable_scatter : 1, /**< Enable scatter packets rx handler */ - enable_lro : 1; /**< Enable LRO */ + enable_lro : 1, /**< Enable LRO */ + hw_timestamp : 1, /**< Enable HW timestamp */ + security : 1, /**< Enable rte_security offloads */ + /** + * When set the offload bitfield should be ignored. + * Instead per-port Rx offloads should be set on offloads + * field above. + * Per-queue offloads shuold be set on rte_eth_rxq_conf + * structure. + * This bit is temporary till rxmode bitfield offloads API will + * be deprecated. + */ + ignore_offload_bitfield : 1; }; /** @@ -368,6 +397,14 @@ enum rte_vlan_type { ETH_VLAN_TYPE_MAX, }; +/** + * A structure used to describe a vlan filter. + * If the bit corresponding to a VID is set, such VID is on. + */ +struct rte_vlan_filter_conf { + uint64_t ids[64]; +}; + /** * A structure used to configure the Receive Side Scaling (RSS) feature * of an Ethernet port. @@ -559,7 +596,7 @@ struct rte_eth_rss_reta_entry64 { /** * This enum indicates the possible number of traffic classes - * in DCB configratioins + * in DCB configurations */ enum rte_eth_nb_tcs { ETH_4_TCS = 4, /**< 4 TCs with DCB. */ @@ -624,6 +661,24 @@ struct rte_eth_vmdq_dcb_conf { /**< Selects a queue in a pool */ }; +/** + * A structure used to configure the VMDQ feature of an Ethernet port when + * not combined with the DCB feature. + * + * Using this feature, packets are routed to a pool of queues. By default, + * the pool selection is based on the MAC address, the vlan id in the + * vlan tag as specified in the pool_map array. + * Passing the ETH_VMDQ_ACCEPT_UNTAG in the rx_mode field allows pool + * selection using only the MAC address. MAC address to pool mapping is done + * using the rte_eth_dev_mac_addr_add function, with the pool parameter + * corresponding to the pool id. + * + * Queue selection within the selected pool will be done using RSS when + * it is enabled or revert to the first queue of the pool if not. + * + * A default pool may be used, if desired, to route all traffic which + * does not match the vlan filter rules or any pool MAC address. + */ struct rte_eth_vmdq_rx_conf { enum rte_eth_nb_pools nb_queue_pools; /**< VMDq only mode, 8 or 64 pools */ uint8_t enable_default_pool; /**< If non-zero, use a default pool */ @@ -642,9 +697,16 @@ struct rte_eth_vmdq_rx_conf { */ struct rte_eth_txmode { enum rte_eth_tx_mq_mode mq_mode; /**< TX multi-queues mode. */ + /** + * Per-port Tx offloads to be set using DEV_TX_OFFLOAD_* flags. + * Only offloads set on tx_offload_capa field on rte_eth_dev_info + * structure are allowed to be set. + */ + uint64_t offloads; /* For i40e specifically */ uint16_t pvid; + __extension__ uint8_t hw_vlan_reject_tagged : 1, /**< If set, reject sending out tagged pkts */ hw_vlan_reject_untagged : 1, @@ -661,6 +723,12 @@ struct rte_eth_rxconf { uint16_t rx_free_thresh; /**< Drives the freeing of RX descriptors. */ uint8_t rx_drop_en; /**< Drop packets if no descriptors are available. */ uint8_t rx_deferred_start; /**< Do not start queue with rte_eth_dev_start(). */ + /** + * Per-queue Rx offloads to be set using DEV_RX_OFFLOAD_* flags. + * Only offloads set on rx_queue_offload_capa or rx_offload_capa + * fields on rte_eth_dev_info structure are allowed to be set. + */ + uint64_t offloads; }; #define ETH_TXQ_FLAGS_NOMULTSEGS 0x0001 /**< nb_segs=1 for all mbufs */ @@ -676,6 +744,15 @@ struct rte_eth_rxconf { #define ETH_TXQ_FLAGS_NOXSUMS \ (ETH_TXQ_FLAGS_NOXSUMSCTP | ETH_TXQ_FLAGS_NOXSUMUDP | \ ETH_TXQ_FLAGS_NOXSUMTCP) +/** + * When set the txq_flags should be ignored, + * instead per-queue Tx offloads will be set on offloads field + * located on rte_eth_txq_conf struct. + * This flag is temporary till the rte_eth_txq_conf.txq_flags + * API will be deprecated. + */ +#define ETH_TXQ_FLAGS_IGNORE 0x8000 + /** * A structure used to configure a TX ring of an Ethernet port. */ @@ -687,6 +764,12 @@ struct rte_eth_txconf { uint32_t txq_flags; /**< Set flags for the Tx queue */ uint8_t tx_deferred_start; /**< Do not start queue with rte_eth_dev_start(). */ + /** + * Per-queue Tx offloads to be set using DEV_TX_OFFLOAD_* flags. + * Only offloads set on tx_queue_offload_capa or tx_offload_capa + * fields on rte_eth_dev_info structure are allowed to be set. + */ + uint64_t offloads; }; /** @@ -696,6 +779,29 @@ struct rte_eth_desc_lim { uint16_t nb_max; /**< Max allowed number of descriptors. */ uint16_t nb_min; /**< Min allowed number of descriptors. */ uint16_t nb_align; /**< Number of descriptors should be aligned to. */ + + /** + * Max allowed number of segments per whole packet. + * + * - For TSO packet this is the total number of data descriptors allowed + * by device. + * + * @see nb_mtu_seg_max + */ + uint16_t nb_seg_max; + + /** + * Max number of segments per one MTU. + * + * - For non-TSO packet, this is the maximum allowed number of segments + * in a single transmit packet. + * + * - For TSO packet each segment within the TSO may span up to this + * value. + * + * @see nb_seg_max + */ + uint16_t nb_mtu_seg_max; }; /** @@ -786,9 +892,11 @@ struct rte_eth_udp_tunnel { */ struct rte_intr_conf { /** enable/disable lsc interrupt. 0 (default) - disable, 1 enable */ - uint16_t lsc; + uint32_t lsc:1; /** enable/disable rxq interrupt. 0 (default) - disable, 1 enable */ - uint16_t rxq; + uint32_t rxq:1; + /** enable/disable rmv interrupt. 0 (default) - disable, 1 enable */ + uint32_t rmv:1; }; /** @@ -819,7 +927,7 @@ struct rte_eth_conf { /**< Port dcb RX configuration. */ struct rte_eth_vmdq_rx_conf vmdq_rx_conf; /**< Port vmdq RX configuration. */ - } rx_adv_conf; /**< Port RX filtering configuration (union). */ + } rx_adv_conf; /**< Port RX filtering configuration. */ union { struct rte_eth_vmdq_dcb_tx_conf vmdq_dcb_tx_conf; /**< Port vmdq+dcb TX configuration. */ @@ -851,6 +959,21 @@ struct rte_eth_conf { #define DEV_RX_OFFLOAD_TCP_LRO 0x00000010 #define DEV_RX_OFFLOAD_QINQ_STRIP 0x00000020 #define DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM 0x00000040 +#define DEV_RX_OFFLOAD_MACSEC_STRIP 0x00000080 +#define DEV_RX_OFFLOAD_HEADER_SPLIT 0x00000100 +#define DEV_RX_OFFLOAD_VLAN_FILTER 0x00000200 +#define DEV_RX_OFFLOAD_VLAN_EXTEND 0x00000400 +#define DEV_RX_OFFLOAD_JUMBO_FRAME 0x00000800 +#define DEV_RX_OFFLOAD_CRC_STRIP 0x00001000 +#define DEV_RX_OFFLOAD_SCATTER 0x00002000 +#define DEV_RX_OFFLOAD_TIMESTAMP 0x00004000 +#define DEV_RX_OFFLOAD_SECURITY 0x00008000 +#define DEV_RX_OFFLOAD_CHECKSUM (DEV_RX_OFFLOAD_IPV4_CKSUM | \ + DEV_RX_OFFLOAD_UDP_CKSUM | \ + DEV_RX_OFFLOAD_TCP_CKSUM) +#define DEV_RX_OFFLOAD_VLAN (DEV_RX_OFFLOAD_VLAN_STRIP | \ + DEV_RX_OFFLOAD_VLAN_FILTER | \ + DEV_RX_OFFLOAD_VLAN_EXTEND) /** * TX offload capabilities of a device. @@ -864,6 +987,25 @@ struct rte_eth_conf { #define DEV_TX_OFFLOAD_UDP_TSO 0x00000040 #define DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM 0x00000080 /**< Used for tunneling packet. */ #define DEV_TX_OFFLOAD_QINQ_INSERT 0x00000100 +#define DEV_TX_OFFLOAD_VXLAN_TNL_TSO 0x00000200 /**< Used for tunneling packet. */ +#define DEV_TX_OFFLOAD_GRE_TNL_TSO 0x00000400 /**< Used for tunneling packet. */ +#define DEV_TX_OFFLOAD_IPIP_TNL_TSO 0x00000800 /**< Used for tunneling packet. */ +#define DEV_TX_OFFLOAD_GENEVE_TNL_TSO 0x00001000 /**< Used for tunneling packet. */ +#define DEV_TX_OFFLOAD_MACSEC_INSERT 0x00002000 +#define DEV_TX_OFFLOAD_MT_LOCKFREE 0x00004000 +/**< Multiple threads can invoke rte_eth_tx_burst() concurrently on the same + * tx queue without SW lock. + */ +#define DEV_TX_OFFLOAD_MULTI_SEGS 0x00008000 +/**< Device supports multi segment send. */ +#define DEV_TX_OFFLOAD_MBUF_FAST_FREE 0x00010000 +/**< Device supports optimization for fast release of mbufs. + * When set application must guarantee that per-queue all mbufs comes from + * the same mempool and has refcnt = 1. + */ +#define DEV_TX_OFFLOAD_SECURITY 0x00020000 + +struct rte_pci_device; /** * Ethernet device information @@ -882,8 +1024,14 @@ struct rte_eth_dev_info { /** Maximum number of hash MAC addresses for MTA and UTA. */ uint16_t max_vfs; /**< Maximum number of VFs. */ uint16_t max_vmdq_pools; /**< Maximum number of VMDq pools. */ - uint32_t rx_offload_capa; /**< Device RX offload capabilities. */ - uint32_t tx_offload_capa; /**< Device TX offload capabilities. */ + uint64_t rx_offload_capa; + /**< Device per port RX offload capabilities. */ + uint64_t tx_offload_capa; + /**< Device per port TX offload capabilities. */ + uint64_t rx_queue_offload_capa; + /**< Device per queue RX offload capabilities. */ + uint64_t tx_queue_offload_capa; + /**< Device per queue TX offload capabilities. */ uint16_t reta_size; /**< Device redirection table size, the total number of entries. */ uint8_t hash_key_size; /**< Hash key size in bytes */ @@ -915,7 +1063,7 @@ struct rte_eth_rxq_info { /** * Ethernet device TX queue information structure. - * Used to retieve information about configured queue. + * Used to retrieve information about configured queue. */ struct rte_eth_txq_info { struct rte_eth_txconf conf; /**< queue config parameters. */ @@ -928,23 +1076,26 @@ struct rte_eth_txq_info { /** * An Ethernet device extended statistic structure * - * This structure is used by ethdev->eth_xstats_get() to provide - * statistics that are not provided in the generic rte_eth_stats + * This structure is used by rte_eth_xstats_get() to provide + * statistics that are not provided in the generic *rte_eth_stats* * structure. + * It maps a name id, corresponding to an index in the array returned + * by rte_eth_xstats_get_names(), to a statistic value. */ struct rte_eth_xstat { - uint64_t id; - uint64_t value; + uint64_t id; /**< The index in xstats name array. */ + uint64_t value; /**< The statistic counter value. */ }; /** - * A name-key lookup element for extended statistics. + * A name element for extended statistics. * - * This structure is used to map between names and ID numbers - * for extended ethernet statistics. + * An array of this structure is returned by rte_eth_xstats_get_names(). + * It lists the names of extended statistics for a PMD. The *rte_eth_xstat* + * structure references these names by their array index. */ struct rte_eth_xstat_name { - char name[RTE_ETH_XSTATS_NAME_SIZE]; + char name[RTE_ETH_XSTATS_NAME_SIZE]; /**< The statistic name. */ }; #define ETH_DCB_NUM_TCS 8 @@ -991,15 +1142,6 @@ struct rte_eth_dev_callback; /** @internal Structure to keep track of registered callbacks */ TAILQ_HEAD(rte_eth_dev_cb_list, rte_eth_dev_callback); - -#ifdef RTE_LIBRTE_ETHDEV_DEBUG -#define RTE_PMD_DEBUG_TRACE(...) \ - rte_pmd_debug_trace(__func__, __VA_ARGS__) -#else -#define RTE_PMD_DEBUG_TRACE(...) -#endif - - /* Macros to check for valid port */ #define RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, retval) do { \ if (!rte_eth_dev_is_valid_port(port_id)) { \ @@ -1052,6 +1194,9 @@ typedef int (*eth_dev_set_link_down_t)(struct rte_eth_dev *dev); typedef void (*eth_dev_close_t)(struct rte_eth_dev *dev); /**< @internal Function used to close a configured Ethernet device. */ +typedef int (*eth_dev_reset_t)(struct rte_eth_dev *dev); +/** <@internal Function used to reset a configured Ethernet device. */ + typedef void (*eth_promiscuous_enable_t)(struct rte_eth_dev *dev); /**< @internal Function used to enable the RX promiscuous mode of an Ethernet device. */ @@ -1068,7 +1213,7 @@ typedef int (*eth_link_update_t)(struct rte_eth_dev *dev, int wait_to_complete); /**< @internal Get link speed, duplex mode and state (up/down) of an Ethernet device. */ -typedef void (*eth_stats_get_t)(struct rte_eth_dev *dev, +typedef int (*eth_stats_get_t)(struct rte_eth_dev *dev, struct rte_eth_stats *igb_stats); /**< @internal Get global I/O statistics of an Ethernet device. */ @@ -1079,6 +1224,12 @@ typedef int (*eth_xstats_get_t)(struct rte_eth_dev *dev, struct rte_eth_xstat *stats, unsigned n); /**< @internal Get extended stats of an Ethernet device. */ +typedef int (*eth_xstats_get_by_id_t)(struct rte_eth_dev *dev, + const uint64_t *ids, + uint64_t *values, + unsigned int n); +/**< @internal Get extended stats of an Ethernet device. */ + typedef void (*eth_xstats_reset_t)(struct rte_eth_dev *dev); /**< @internal Reset extended stats of an Ethernet device. */ @@ -1086,6 +1237,11 @@ typedef int (*eth_xstats_get_names_t)(struct rte_eth_dev *dev, struct rte_eth_xstat_name *xstats_names, unsigned size); /**< @internal Get names of extended stats of an Ethernet device. */ +typedef int (*eth_xstats_get_names_by_id_t)(struct rte_eth_dev *dev, + struct rte_eth_xstat_name *xstats_names, const uint64_t *ids, + unsigned int size); +/**< @internal Get names of extended stats of an Ethernet device. */ + typedef int (*eth_queue_stats_mapping_set_t)(struct rte_eth_dev *dev, uint16_t queue_id, uint8_t stat_idx, @@ -1135,11 +1291,24 @@ typedef void (*eth_queue_release_t)(void *queue); typedef uint32_t (*eth_rx_queue_count_t)(struct rte_eth_dev *dev, uint16_t rx_queue_id); -/**< @internal Get number of available descriptors on a receive queue of an Ethernet device. */ +/**< @internal Get number of used descriptors on a receive queue. */ typedef int (*eth_rx_descriptor_done_t)(void *rxq, uint16_t offset); /**< @internal Check DD bit of specific RX descriptor */ +typedef int (*eth_rx_descriptor_status_t)(void *rxq, uint16_t offset); +/**< @internal Check the status of a Rx descriptor */ + +typedef int (*eth_tx_descriptor_status_t)(void *txq, uint16_t offset); +/**< @internal Check the status of a Tx descriptor */ + +typedef int (*eth_fw_version_get_t)(struct rte_eth_dev *dev, + char *fw_version, size_t fw_size); +/**< @internal Get firmware information of an Ethernet device. */ + +typedef int (*eth_tx_done_cleanup_t)(void *txq, uint32_t free_cnt); +/**< @internal Force mbufs to be from TX ring. */ + typedef void (*eth_rxq_info_get_t)(struct rte_eth_dev *dev, uint16_t rx_queue_id, struct rte_eth_rxq_info *qinfo); @@ -1158,7 +1327,7 @@ typedef int (*vlan_tpid_set_t)(struct rte_eth_dev *dev, enum rte_vlan_type type, uint16_t tpid); /**< @internal set the outer/inner VLAN-TPID by an Ethernet device. */ -typedef void (*vlan_offload_set_t)(struct rte_eth_dev *dev, int mask); +typedef int (*vlan_offload_set_t)(struct rte_eth_dev *dev, int mask); /**< @internal set VLAN offload function by an Ethernet device. */ typedef int (*vlan_pvid_set_t)(struct rte_eth_dev *dev, @@ -1181,6 +1350,11 @@ typedef uint16_t (*eth_tx_burst_t)(void *txq, uint16_t nb_pkts); /**< @internal Send output packets on a transmit queue of an Ethernet device. */ +typedef uint16_t (*eth_tx_prep_t)(void *txq, + struct rte_mbuf **tx_pkts, + uint16_t nb_pkts); +/**< @internal Prepare output packets on a transmit queue of an Ethernet device. */ + typedef int (*flow_ctrl_get_t)(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf); /**< @internal Get current flow control parameter on an Ethernet device */ @@ -1220,7 +1394,7 @@ typedef int (*eth_dev_led_off_t)(struct rte_eth_dev *dev); typedef void (*eth_mac_addr_remove_t)(struct rte_eth_dev *dev, uint32_t index); /**< @internal Remove MAC address from receive address register */ -typedef void (*eth_mac_addr_add_t)(struct rte_eth_dev *dev, +typedef int (*eth_mac_addr_add_t)(struct rte_eth_dev *dev, struct ether_addr *mac_addr, uint32_t index, uint32_t vmdq); @@ -1239,39 +1413,11 @@ typedef int (*eth_uc_all_hash_table_set_t)(struct rte_eth_dev *dev, uint8_t on); /**< @internal Set all Unicast Hash bitmap */ -typedef int (*eth_set_vf_rx_mode_t)(struct rte_eth_dev *dev, - uint16_t vf, - uint16_t rx_mode, - uint8_t on); -/**< @internal Set a VF receive mode */ - -typedef int (*eth_set_vf_rx_t)(struct rte_eth_dev *dev, - uint16_t vf, - uint8_t on); -/**< @internal Set a VF receive mode */ - -typedef int (*eth_set_vf_tx_t)(struct rte_eth_dev *dev, - uint16_t vf, - uint8_t on); -/**< @internal Enable or disable a VF transmit */ - -typedef int (*eth_set_vf_vlan_filter_t)(struct rte_eth_dev *dev, - uint16_t vlan, - uint64_t vf_mask, - uint8_t vlan_on); -/**< @internal Set VF VLAN pool filter */ - typedef int (*eth_set_queue_rate_limit_t)(struct rte_eth_dev *dev, uint16_t queue_idx, uint16_t tx_rate); /**< @internal Set queue TX rate */ -typedef int (*eth_set_vf_rate_limit_t)(struct rte_eth_dev *dev, - uint16_t vf, - uint16_t tx_rate, - uint64_t q_msk); -/**< @internal Set VF TX rate */ - typedef int (*eth_mirror_rule_set_t)(struct rte_eth_dev *dev, struct rte_eth_mirror_conf *mirror_conf, uint8_t rule_id, @@ -1347,59 +1493,6 @@ typedef int (*eth_l2_tunnel_offload_set_t) uint8_t en); /**< @internal enable/disable the l2 tunnel offload functions */ -#ifdef RTE_NIC_BYPASS - -enum { - RTE_BYPASS_MODE_NONE, - RTE_BYPASS_MODE_NORMAL, - RTE_BYPASS_MODE_BYPASS, - RTE_BYPASS_MODE_ISOLATE, - RTE_BYPASS_MODE_NUM, -}; - -#define RTE_BYPASS_MODE_VALID(x) \ - ((x) > RTE_BYPASS_MODE_NONE && (x) < RTE_BYPASS_MODE_NUM) - -enum { - RTE_BYPASS_EVENT_NONE, - RTE_BYPASS_EVENT_START, - RTE_BYPASS_EVENT_OS_ON = RTE_BYPASS_EVENT_START, - RTE_BYPASS_EVENT_POWER_ON, - RTE_BYPASS_EVENT_OS_OFF, - RTE_BYPASS_EVENT_POWER_OFF, - RTE_BYPASS_EVENT_TIMEOUT, - RTE_BYPASS_EVENT_NUM -}; - -#define RTE_BYPASS_EVENT_VALID(x) \ - ((x) > RTE_BYPASS_EVENT_NONE && (x) < RTE_BYPASS_MODE_NUM) - -enum { - RTE_BYPASS_TMT_OFF, /* timeout disabled. */ - RTE_BYPASS_TMT_1_5_SEC, /* timeout for 1.5 seconds */ - RTE_BYPASS_TMT_2_SEC, /* timeout for 2 seconds */ - RTE_BYPASS_TMT_3_SEC, /* timeout for 3 seconds */ - RTE_BYPASS_TMT_4_SEC, /* timeout for 4 seconds */ - RTE_BYPASS_TMT_8_SEC, /* timeout for 8 seconds */ - RTE_BYPASS_TMT_16_SEC, /* timeout for 16 seconds */ - RTE_BYPASS_TMT_32_SEC, /* timeout for 32 seconds */ - RTE_BYPASS_TMT_NUM -}; - -#define RTE_BYPASS_TMT_VALID(x) \ - ((x) == RTE_BYPASS_TMT_OFF || \ - ((x) > RTE_BYPASS_TMT_OFF && (x) < RTE_BYPASS_TMT_NUM)) - -typedef void (*bypass_init_t)(struct rte_eth_dev *dev); -typedef int32_t (*bypass_state_set_t)(struct rte_eth_dev *dev, uint32_t *new_state); -typedef int32_t (*bypass_state_show_t)(struct rte_eth_dev *dev, uint32_t *state); -typedef int32_t (*bypass_event_set_t)(struct rte_eth_dev *dev, uint32_t state, uint32_t event); -typedef int32_t (*bypass_event_show_t)(struct rte_eth_dev *dev, uint32_t event_shift, uint32_t *event); -typedef int32_t (*bypass_wd_timeout_set_t)(struct rte_eth_dev *dev, uint32_t timeout); -typedef int32_t (*bypass_wd_timeout_show_t)(struct rte_eth_dev *dev, uint32_t *wd_timeout); -typedef int32_t (*bypass_ver_show_t)(struct rte_eth_dev *dev, uint32_t *ver); -typedef int32_t (*bypass_wd_reset_t)(struct rte_eth_dev *dev); -#endif typedef int (*eth_filter_ctrl_t)(struct rte_eth_dev *dev, enum rte_filter_type filter_type, @@ -1407,10 +1500,20 @@ typedef int (*eth_filter_ctrl_t)(struct rte_eth_dev *dev, void *arg); /**< @internal Take operations to assigned filter type on an Ethernet device */ +typedef int (*eth_tm_ops_get_t)(struct rte_eth_dev *dev, void *ops); +/**< @internal Get Traffic Management (TM) operations on an Ethernet device */ + +typedef int (*eth_mtr_ops_get_t)(struct rte_eth_dev *dev, void *ops); +/**< @internal Get Trafffic Metering and Policing (MTR) operations */ + typedef int (*eth_get_dcb_info)(struct rte_eth_dev *dev, struct rte_eth_dcb_info *dcb_info); /**< @internal Get dcb information on an Ethernet device */ +typedef int (*eth_pool_ops_supported_t)(struct rte_eth_dev *dev, + const char *pool); +/**< @internal Test if a port supports specific mempool ops */ + /** * @internal A structure containing the functions exported by an Ethernet driver. */ @@ -1421,11 +1524,19 @@ struct eth_dev_ops { eth_dev_set_link_up_t dev_set_link_up; /**< Device link up. */ eth_dev_set_link_down_t dev_set_link_down; /**< Device link down. */ eth_dev_close_t dev_close; /**< Close device. */ + eth_dev_reset_t dev_reset; /**< Reset device. */ + eth_link_update_t link_update; /**< Get device link state. */ + eth_promiscuous_enable_t promiscuous_enable; /**< Promiscuous ON. */ eth_promiscuous_disable_t promiscuous_disable;/**< Promiscuous OFF. */ eth_allmulticast_enable_t allmulticast_enable;/**< RX multicast ON. */ - eth_allmulticast_disable_t allmulticast_disable;/**< RX multicast OF. */ - eth_link_update_t link_update; /**< Get device link state. */ + eth_allmulticast_disable_t allmulticast_disable;/**< RX multicast OFF. */ + eth_mac_addr_remove_t mac_addr_remove; /**< Remove MAC address. */ + eth_mac_addr_add_t mac_addr_add; /**< Add a MAC address. */ + eth_mac_addr_set_t mac_addr_set; /**< Set a MAC address. */ + eth_set_mc_addr_list_t set_mc_addr_list; /**< set list of mcast addrs. */ + mtu_set_t mtu_set; /**< Set MTU. */ + eth_stats_get_t stats_get; /**< Get generic device statistics. */ eth_stats_reset_t stats_reset; /**< Reset generic device statistics. */ eth_xstats_get_t xstats_get; /**< Get extended device statistics. */ @@ -1434,109 +1545,101 @@ struct eth_dev_ops { /**< Get names of extended statistics. */ eth_queue_stats_mapping_set_t queue_stats_mapping_set; /**< Configure per queue stat counter mapping. */ + eth_dev_infos_get_t dev_infos_get; /**< Get device info. */ + eth_rxq_info_get_t rxq_info_get; /**< retrieve RX queue information. */ + eth_txq_info_get_t txq_info_get; /**< retrieve TX queue information. */ + eth_fw_version_get_t fw_version_get; /**< Get firmware version. */ eth_dev_supported_ptypes_get_t dev_supported_ptypes_get; - /**< Get packet types supported and identified by device*/ - mtu_set_t mtu_set; /**< Set MTU. */ - vlan_filter_set_t vlan_filter_set; /**< Filter VLAN Setup. */ - vlan_tpid_set_t vlan_tpid_set; /**< Outer/Inner VLAN TPID Setup. */ + /**< Get packet types supported and identified by device. */ + + vlan_filter_set_t vlan_filter_set; /**< Filter VLAN Setup. */ + vlan_tpid_set_t vlan_tpid_set; /**< Outer/Inner VLAN TPID Setup. */ vlan_strip_queue_set_t vlan_strip_queue_set; /**< VLAN Stripping on queue. */ vlan_offload_set_t vlan_offload_set; /**< Set VLAN Offload. */ - vlan_pvid_set_t vlan_pvid_set; /**< Set port based TX VLAN insertion */ - eth_queue_start_t rx_queue_start;/**< Start RX for a queue.*/ - eth_queue_stop_t rx_queue_stop;/**< Stop RX for a queue.*/ - eth_queue_start_t tx_queue_start;/**< Start TX for a queue.*/ - eth_queue_stop_t tx_queue_stop;/**< Stop TX for a queue.*/ - eth_rx_queue_setup_t rx_queue_setup;/**< Set up device RX queue.*/ - eth_queue_release_t rx_queue_release;/**< Release RX queue.*/ - eth_rx_queue_count_t rx_queue_count; /**< Get Rx queue count. */ - eth_rx_descriptor_done_t rx_descriptor_done; /**< Check rxd DD bit */ - /**< Enable Rx queue interrupt. */ - eth_rx_enable_intr_t rx_queue_intr_enable; - /**< Disable Rx queue interrupt.*/ - eth_rx_disable_intr_t rx_queue_intr_disable; - eth_tx_queue_setup_t tx_queue_setup;/**< Set up device TX queue.*/ - eth_queue_release_t tx_queue_release;/**< Release TX queue.*/ + vlan_pvid_set_t vlan_pvid_set; /**< Set port based TX VLAN insertion. */ + + eth_queue_start_t rx_queue_start;/**< Start RX for a queue. */ + eth_queue_stop_t rx_queue_stop; /**< Stop RX for a queue. */ + eth_queue_start_t tx_queue_start;/**< Start TX for a queue. */ + eth_queue_stop_t tx_queue_stop; /**< Stop TX for a queue. */ + eth_rx_queue_setup_t rx_queue_setup;/**< Set up device RX queue. */ + eth_queue_release_t rx_queue_release; /**< Release RX queue. */ + eth_rx_queue_count_t rx_queue_count; + /**< Get the number of used RX descriptors. */ + eth_rx_descriptor_done_t rx_descriptor_done; /**< Check rxd DD bit. */ + eth_rx_descriptor_status_t rx_descriptor_status; + /**< Check the status of a Rx descriptor. */ + eth_tx_descriptor_status_t tx_descriptor_status; + /**< Check the status of a Tx descriptor. */ + eth_rx_enable_intr_t rx_queue_intr_enable; /**< Enable Rx queue interrupt. */ + eth_rx_disable_intr_t rx_queue_intr_disable; /**< Disable Rx queue interrupt. */ + eth_tx_queue_setup_t tx_queue_setup;/**< Set up device TX queue. */ + eth_queue_release_t tx_queue_release; /**< Release TX queue. */ + eth_tx_done_cleanup_t tx_done_cleanup;/**< Free tx ring mbufs */ + eth_dev_led_on_t dev_led_on; /**< Turn on LED. */ eth_dev_led_off_t dev_led_off; /**< Turn off LED. */ + flow_ctrl_get_t flow_ctrl_get; /**< Get flow control. */ flow_ctrl_set_t flow_ctrl_set; /**< Setup flow control. */ - priority_flow_ctrl_set_t priority_flow_ctrl_set; /**< Setup priority flow control.*/ - eth_mac_addr_remove_t mac_addr_remove; /**< Remove MAC address */ - eth_mac_addr_add_t mac_addr_add; /**< Add a MAC address */ - eth_mac_addr_set_t mac_addr_set; /**< Set a MAC address */ - eth_uc_hash_table_set_t uc_hash_table_set; /**< Set Unicast Table Array */ - eth_uc_all_hash_table_set_t uc_all_hash_table_set; /**< Set Unicast hash bitmap */ - eth_mirror_rule_set_t mirror_rule_set; /**< Add a traffic mirror rule.*/ - eth_mirror_rule_reset_t mirror_rule_reset; /**< reset a traffic mirror rule.*/ - eth_set_vf_rx_mode_t set_vf_rx_mode; /**< Set VF RX mode */ - eth_set_vf_rx_t set_vf_rx; /**< enable/disable a VF receive */ - eth_set_vf_tx_t set_vf_tx; /**< enable/disable a VF transmit */ - eth_set_vf_vlan_filter_t set_vf_vlan_filter; /**< Set VF VLAN filter */ - /** Add UDP tunnel port. */ - eth_udp_tunnel_port_add_t udp_tunnel_port_add; - /** Del UDP tunnel port. */ - eth_udp_tunnel_port_del_t udp_tunnel_port_del; - eth_set_queue_rate_limit_t set_queue_rate_limit; /**< Set queue rate limit */ - eth_set_vf_rate_limit_t set_vf_rate_limit; /**< Set VF rate limit */ - /** Update redirection table. */ - reta_update_t reta_update; - /** Query redirection table. */ - reta_query_t reta_query; + priority_flow_ctrl_set_t priority_flow_ctrl_set; /**< Setup priority flow control. */ - eth_get_reg_t get_reg; - /**< Get registers */ - eth_get_eeprom_length_t get_eeprom_length; - /**< Get eeprom length */ - eth_get_eeprom_t get_eeprom; - /**< Get eeprom data */ - eth_set_eeprom_t set_eeprom; - /**< Set eeprom */ - /* bypass control */ -#ifdef RTE_NIC_BYPASS - bypass_init_t bypass_init; - bypass_state_set_t bypass_state_set; - bypass_state_show_t bypass_state_show; - bypass_event_set_t bypass_event_set; - bypass_event_show_t bypass_event_show; - bypass_wd_timeout_set_t bypass_wd_timeout_set; - bypass_wd_timeout_show_t bypass_wd_timeout_show; - bypass_ver_show_t bypass_ver_show; - bypass_wd_reset_t bypass_wd_reset; -#endif + eth_uc_hash_table_set_t uc_hash_table_set; /**< Set Unicast Table Array. */ + eth_uc_all_hash_table_set_t uc_all_hash_table_set; /**< Set Unicast hash bitmap. */ - /** Configure RSS hash protocols. */ - rss_hash_update_t rss_hash_update; - /** Get current RSS hash configuration. */ - rss_hash_conf_get_t rss_hash_conf_get; - eth_filter_ctrl_t filter_ctrl; - /**< common filter control. */ - eth_set_mc_addr_list_t set_mc_addr_list; /**< set list of mcast addrs */ - eth_rxq_info_get_t rxq_info_get; - /**< retrieve RX queue information. */ - eth_txq_info_get_t txq_info_get; - /**< retrieve TX queue information. */ - /** Turn IEEE1588/802.1AS timestamping on. */ - eth_timesync_enable_t timesync_enable; - /** Turn IEEE1588/802.1AS timestamping off. */ - eth_timesync_disable_t timesync_disable; - /** Read the IEEE1588/802.1AS RX timestamp. */ - eth_timesync_read_rx_timestamp_t timesync_read_rx_timestamp; - /** Read the IEEE1588/802.1AS TX timestamp. */ - eth_timesync_read_tx_timestamp_t timesync_read_tx_timestamp; + eth_mirror_rule_set_t mirror_rule_set; /**< Add a traffic mirror rule. */ + eth_mirror_rule_reset_t mirror_rule_reset; /**< reset a traffic mirror rule. */ - /** Get DCB information */ - eth_get_dcb_info get_dcb_info; - /** Adjust the device clock.*/ - eth_timesync_adjust_time timesync_adjust_time; - /** Get the device clock time. */ - eth_timesync_read_time timesync_read_time; - /** Set the device clock time. */ - eth_timesync_write_time timesync_write_time; - /** Config ether type of l2 tunnel */ + eth_udp_tunnel_port_add_t udp_tunnel_port_add; /** Add UDP tunnel port. */ + eth_udp_tunnel_port_del_t udp_tunnel_port_del; /** Del UDP tunnel port. */ eth_l2_tunnel_eth_type_conf_t l2_tunnel_eth_type_conf; - /** Enable/disable l2 tunnel offload functions */ - eth_l2_tunnel_offload_set_t l2_tunnel_offload_set; + /** Config ether type of l2 tunnel. */ + eth_l2_tunnel_offload_set_t l2_tunnel_offload_set; + /** Enable/disable l2 tunnel offload functions. */ + + eth_set_queue_rate_limit_t set_queue_rate_limit; /**< Set queue rate limit. */ + + rss_hash_update_t rss_hash_update; /** Configure RSS hash protocols. */ + rss_hash_conf_get_t rss_hash_conf_get; /** Get current RSS hash configuration. */ + reta_update_t reta_update; /** Update redirection table. */ + reta_query_t reta_query; /** Query redirection table. */ + + eth_get_reg_t get_reg; /**< Get registers. */ + eth_get_eeprom_length_t get_eeprom_length; /**< Get eeprom length. */ + eth_get_eeprom_t get_eeprom; /**< Get eeprom data. */ + eth_set_eeprom_t set_eeprom; /**< Set eeprom. */ + + + eth_filter_ctrl_t filter_ctrl; /**< common filter control. */ + + eth_get_dcb_info get_dcb_info; /** Get DCB information. */ + + eth_timesync_enable_t timesync_enable; + /** Turn IEEE1588/802.1AS timestamping on. */ + eth_timesync_disable_t timesync_disable; + /** Turn IEEE1588/802.1AS timestamping off. */ + eth_timesync_read_rx_timestamp_t timesync_read_rx_timestamp; + /** Read the IEEE1588/802.1AS RX timestamp. */ + eth_timesync_read_tx_timestamp_t timesync_read_tx_timestamp; + /** Read the IEEE1588/802.1AS TX timestamp. */ + eth_timesync_adjust_time timesync_adjust_time; /** Adjust the device clock. */ + eth_timesync_read_time timesync_read_time; /** Get the device clock time. */ + eth_timesync_write_time timesync_write_time; /** Set the device clock time. */ + + eth_xstats_get_by_id_t xstats_get_by_id; + /**< Get extended device statistic values by ID. */ + eth_xstats_get_names_by_id_t xstats_get_names_by_id; + /**< Get name of extended device statistics by ID. */ + + eth_tm_ops_get_t tm_ops_get; + /**< Get Traffic Management (TM) operations. */ + + eth_mtr_ops_get_t mtr_ops_get; + /**< Get Traffic Metering and Policing (MTR) operations. */ + + eth_pool_ops_supported_t pool_ops_supported; + /**< Test if a port supports specific mempool ops */ }; /** @@ -1561,7 +1664,7 @@ struct eth_dev_ops { * @return * The number of packets returned to the user. */ -typedef uint16_t (*rte_rx_callback_fn)(uint8_t port, uint16_t queue, +typedef uint16_t (*rte_rx_callback_fn)(uint16_t port, uint16_t queue, struct rte_mbuf *pkts[], uint16_t nb_pkts, uint16_t max_pkts, void *user_param); @@ -1585,7 +1688,7 @@ typedef uint16_t (*rte_rx_callback_fn)(uint8_t port, uint16_t queue, * @return * The number of packets to be written to the NIC. */ -typedef uint16_t (*rte_tx_callback_fn)(uint8_t port, uint16_t queue, +typedef uint16_t (*rte_tx_callback_fn)(uint16_t port, uint16_t queue, struct rte_mbuf *pkts[], uint16_t nb_pkts, void *user_param); /** @@ -1603,14 +1706,12 @@ struct rte_eth_rxtx_callback { }; /** - * The eth device type. + * A set of values to describe the possible states of an eth device. */ -enum rte_eth_dev_type { - RTE_ETH_DEV_UNKNOWN, /**< unknown device type */ - RTE_ETH_DEV_PCI, - /**< Physical function and Virtual function of PCI devices */ - RTE_ETH_DEV_VIRTUAL, /**< non hardware device */ - RTE_ETH_DEV_MAX /**< max value of this enum */ +enum rte_eth_dev_state { + RTE_ETH_DEV_UNUSED = 0, + RTE_ETH_DEV_ATTACHED, + RTE_ETH_DEV_DEFERRED, }; /** @@ -1626,10 +1727,11 @@ enum rte_eth_dev_type { struct rte_eth_dev { eth_rx_burst_t rx_pkt_burst; /**< Pointer to PMD receive function. */ eth_tx_burst_t tx_pkt_burst; /**< Pointer to PMD transmit function. */ + eth_tx_prep_t tx_pkt_prepare; /**< Pointer to PMD transmit prepare function. */ struct rte_eth_dev_data *data; /**< Pointer to device data */ - const struct eth_driver *driver;/**< Driver for this device */ const struct eth_dev_ops *dev_ops; /**< Functions exported by PMD */ - struct rte_pci_device *pci_dev; /**< PCI info. supplied by probing */ + struct rte_device *device; /**< Backing device */ + struct rte_intr_handle *intr_handle; /**< Device interrupt handle */ /** User application callbacks for NIC interrupts */ struct rte_eth_dev_cb_list link_intr_cbs; /** @@ -1642,10 +1744,13 @@ struct rte_eth_dev { * received packets before passing them to the driver for transmission. */ struct rte_eth_rxtx_callback *pre_tx_burst_cbs[RTE_MAX_QUEUES_PER_PORT]; - uint8_t attached; /**< Flag indicating the port is attached */ - enum rte_eth_dev_type dev_type; /**< Flag indicating the device type */ + enum rte_eth_dev_state state; /**< Flag indicating the port state */ + void *security_ctx; /**< Context for security ops */ } __rte_cache_aligned; +void * +rte_eth_dev_get_sec_ctx(uint8_t port_id); + struct rte_eth_dev_sriov { uint8_t active; /**< SRIOV is active with 16, 32 or 64 pools */ uint8_t nb_q_per_pool; /**< rx queue number per pool */ @@ -1654,7 +1759,7 @@ struct rte_eth_dev_sriov { }; #define RTE_ETH_DEV_SRIOV(dev) ((dev)->data->sriov) -#define RTE_ETH_NAME_MAX_LEN (32) +#define RTE_ETH_NAME_MAX_LEN RTE_DEV_NAME_MAX_LEN /** * @internal @@ -1690,7 +1795,8 @@ struct rte_eth_dev_data { /** bitmap array of associating Ethernet MAC addresses to pools */ struct ether_addr* hash_mac_addrs; /** Device Ethernet MAC addresses of hash filtering. */ - uint8_t port_id; /**< Device [external] port identifier. */ + uint16_t port_id; /**< Device [external] port identifier. */ + __extension__ uint8_t promiscuous : 1, /**< RX promiscuous mode ON(1) / OFF(0). */ scattered_rx : 1, /**< RX of scattered packets is ON(1) / OFF(0) */ all_multicast : 1, /**< RX all multicast mode ON(1) / OFF(0). */ @@ -1703,15 +1809,16 @@ struct rte_eth_dev_data { uint32_t dev_flags; /**< Capabilities */ enum rte_kernel_driver kdrv; /**< Kernel driver passthrough */ int numa_node; /**< NUMA node connection */ - const char *drv_name; /**< Driver name */ + struct rte_vlan_filter_conf vlan_filter_conf; + /**< VLAN filter configuration. */ }; -/** Device supports hotplug detach */ -#define RTE_ETH_DEV_DETACHABLE 0x0001 /** Device supports link state interrupt */ #define RTE_ETH_DEV_INTR_LSC 0x0002 /** Device is a bonded slave */ #define RTE_ETH_DEV_BONDED_SLAVE 0x0004 +/** Device supports device removal interrupt */ +#define RTE_ETH_DEV_INTR_RMV 0x0008 /** * @internal @@ -1720,20 +1827,38 @@ struct rte_eth_dev_data { */ extern struct rte_eth_dev rte_eth_devices[]; +/** + * Iterates over valid ethdev ports. + * + * @param port_id + * The id of the next possible valid port. + * @return + * Next valid port id, RTE_MAX_ETHPORTS if there is none. + */ +uint16_t rte_eth_find_next(uint16_t port_id); + +/** + * Macro to iterate over all enabled ethdev ports. + */ +#define RTE_ETH_FOREACH_DEV(p) \ + for (p = rte_eth_find_next(0); \ + (unsigned int)p < (unsigned int)RTE_MAX_ETHPORTS; \ + p = rte_eth_find_next(p + 1)) + + /** * Get the total number of Ethernet devices that have been successfully - * initialized by the [matching] Ethernet driver during the PCI probing phase. - * All devices whose port identifier is in the range - * [0, rte_eth_dev_count() - 1] can be operated on by network applications - * immediately after invoking rte_eal_init(). - * If the application unplugs a port using hotplug function, The enabled port - * numbers may be noncontiguous. In the case, the applications need to manage - * enabled port by themselves. + * initialized by the matching Ethernet driver during the PCI probing phase + * and that are available for applications to use. These devices must be + * accessed by using the ``RTE_ETH_FOREACH_DEV()`` macro to deal with + * non-contiguous ranges of devices. + * These non-contiguous ranges can be created by calls to hotplug functions or + * by some PMDs. * * @return * - The total number of usable Ethernet devices. */ -uint8_t rte_eth_dev_count(void); +uint16_t rte_eth_dev_count(void); /** * @internal @@ -1756,8 +1881,20 @@ struct rte_eth_dev *rte_eth_dev_allocated(const char *name); * @return * - Slot in the rte_dev_devices array for a new device; */ -struct rte_eth_dev *rte_eth_dev_allocate(const char *name, - enum rte_eth_dev_type type); +struct rte_eth_dev *rte_eth_dev_allocate(const char *name); + +/** + * @internal + * Attach to the ethdev already initialized by the primary + * process. + * + * @param name Ethernet device's name. + * @return + * - Success: Slot in the rte_dev_devices array for attached + * device. + * - Error: Null pointer. + */ +struct rte_eth_dev *rte_eth_dev_attach_secondary(const char *name); /** * @internal @@ -1771,18 +1908,18 @@ struct rte_eth_dev *rte_eth_dev_allocate(const char *name, int rte_eth_dev_release_port(struct rte_eth_dev *eth_dev); /** - * Attach a new Ethernet device specified by aruguments. + * Attach a new Ethernet device specified by arguments. * * @param devargs * A pointer to a strings array describing the new device * to be attached. The strings should be a pci address like - * '0000:01:00.0' or virtual device name like 'eth_pcap0'. + * '0000:01:00.0' or virtual device name like 'net_pcap0'. * @param port_id * A pointer to a port identifier actually attached. * @return * 0 on success and port_id is filled, negative on error */ -int rte_eth_dev_attach(const char *devargs, uint8_t *port_id); +int rte_eth_dev_attach(const char *devargs, uint16_t *port_id); /** * Detach a Ethernet device specified by port identifier. @@ -1792,95 +1929,12 @@ int rte_eth_dev_attach(const char *devargs, uint8_t *port_id); * @param port_id * The port identifier of the device to detach. * @param devname - * A pointer to a device name actually detached. + * A pointer to a buffer that will be filled with the device name. + * This buffer must be at least RTE_DEV_NAME_MAX_LEN long. * @return * 0 on success and devname is filled, negative on error */ -int rte_eth_dev_detach(uint8_t port_id, char *devname); - -struct eth_driver; -/** - * @internal - * Initialization function of an Ethernet driver invoked for each matching - * Ethernet PCI device detected during the PCI probing phase. - * - * @param eth_dev - * The *eth_dev* pointer is the address of the *rte_eth_dev* structure - * associated with the matching device and which have been [automatically] - * allocated in the *rte_eth_devices* array. - * The *eth_dev* structure is supplied to the driver initialization function - * with the following fields already initialized: - * - * - *pci_dev*: Holds the pointers to the *rte_pci_device* structure which - * contains the generic PCI information of the matching device. - * - * - *driver*: Holds the pointer to the *eth_driver* structure. - * - * - *dev_private*: Holds a pointer to the device private data structure. - * - * - *mtu*: Contains the default Ethernet maximum frame length (1500). - * - * - *port_id*: Contains the port index of the device (actually the index - * of the *eth_dev* structure in the *rte_eth_devices* array). - * - * @return - * - 0: Success, the device is properly initialized by the driver. - * In particular, the driver MUST have set up the *dev_ops* pointer - * of the *eth_dev* structure. - * - <0: Error code of the device initialization failure. - */ -typedef int (*eth_dev_init_t)(struct rte_eth_dev *eth_dev); - -/** - * @internal - * Finalization function of an Ethernet driver invoked for each matching - * Ethernet PCI device detected during the PCI closing phase. - * - * @param eth_dev - * The *eth_dev* pointer is the address of the *rte_eth_dev* structure - * associated with the matching device and which have been [automatically] - * allocated in the *rte_eth_devices* array. - * @return - * - 0: Success, the device is properly finalized by the driver. - * In particular, the driver MUST free the *dev_ops* pointer - * of the *eth_dev* structure. - * - <0: Error code of the device initialization failure. - */ -typedef int (*eth_dev_uninit_t)(struct rte_eth_dev *eth_dev); - -/** - * @internal - * The structure associated with a PMD Ethernet driver. - * - * Each Ethernet driver acts as a PCI driver and is represented by a generic - * *eth_driver* structure that holds: - * - * - An *rte_pci_driver* structure (which must be the first field). - * - * - The *eth_dev_init* function invoked for each matching PCI device. - * - * - The *eth_dev_uninit* function invoked for each matching PCI device. - * - * - The size of the private data to allocate for each matching device. - */ -struct eth_driver { - struct rte_pci_driver pci_drv; /**< The PMD is also a PCI driver. */ - eth_dev_init_t eth_dev_init; /**< Device init function. */ - eth_dev_uninit_t eth_dev_uninit; /**< Device uninit function. */ - unsigned int dev_private_size; /**< Size of device private data. */ -}; - -/** - * @internal - * A function invoked by the initialization function of an Ethernet driver - * to simultaneously register itself as a PCI driver and as an Ethernet - * Poll Mode Driver (PMD). - * - * @param eth_drv - * The pointer to the *eth_driver* structure associated with - * the Ethernet driver. - */ -void rte_eth_driver_register(struct eth_driver *eth_drv); +int rte_eth_dev_detach(uint16_t port_id, char *devname); /** * Convert a numerical speed in Mbps to a bitmap flag that can be used in @@ -1914,6 +1968,9 @@ uint32_t rte_eth_speed_bitflag(uint32_t speed, int duplex); * each statically configurable offload hardware feature provided by * Ethernet devices, such as IP checksum or VLAN tag stripping for * example. + * The Rx offload bitfield API is obsolete and will be deprecated. + * Applications should set the ignore_bitfield_offloads bit on *rxmode* + * structure and use offloads field to set per-port offloads instead. * - the Receive Side Scaling (RSS) configuration when using multiple RX * queues per port. * @@ -1924,9 +1981,22 @@ uint32_t rte_eth_speed_bitflag(uint32_t speed, int duplex); * - 0: Success, device configured. * - <0: Error code returned by the driver configuration function. */ -int rte_eth_dev_configure(uint8_t port_id, uint16_t nb_rx_queue, +int rte_eth_dev_configure(uint16_t port_id, uint16_t nb_rx_queue, uint16_t nb_tx_queue, const struct rte_eth_conf *eth_conf); +/** + * @internal + * Release device queues and clear its configuration to force the user + * application to reconfigure it. It is for internal use only. + * + * @param dev + * Pointer to struct rte_eth_dev. + * + * @return + * void + */ +void _rte_eth_dev_reset(struct rte_eth_dev *dev); + /** * Allocate and set up a receive queue for an Ethernet device. * @@ -1954,6 +2024,8 @@ int rte_eth_dev_configure(uint8_t port_id, uint16_t nb_rx_queue, * The *rx_conf* structure contains an *rx_thresh* structure with the values * of the Prefetch, Host, and Write-Back threshold registers of the receive * ring. + * In addition it contains the hardware offloads features to activate using + * the DEV_RX_OFFLOAD_* flags. * @param mb_pool * The pointer to the memory pool from which to allocate *rte_mbuf* network * memory buffers to populate each descriptor of the receive ring. @@ -1966,7 +2038,7 @@ int rte_eth_dev_configure(uint8_t port_id, uint16_t nb_rx_queue, * allocate network memory buffers from the memory pool when * initializing receive descriptors. */ -int rte_eth_rx_queue_setup(uint8_t port_id, uint16_t rx_queue_id, +int rte_eth_rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id, uint16_t nb_rx_desc, unsigned int socket_id, const struct rte_eth_rxconf *rx_conf, struct rte_mempool *mb_pool); @@ -2007,6 +2079,11 @@ int rte_eth_rx_queue_setup(uint8_t port_id, uint16_t rx_queue_id, * - The *txq_flags* member contains flags to pass to the TX queue setup * function to configure the behavior of the TX queue. This should be set * to 0 if no special configuration is required. + * This API is obsolete and will be deprecated. Applications + * should set it to ETH_TXQ_FLAGS_IGNORE and use + * the offloads field below. + * - The *offloads* member contains Tx offloads to be enabled. + * Offloads which are not set cannot be used on the datapath. * * Note that setting *tx_free_thresh* or *tx_rs_thresh* value to 0 forces * the transmit function to use default values. @@ -2014,7 +2091,7 @@ int rte_eth_rx_queue_setup(uint8_t port_id, uint16_t rx_queue_id, * - 0: Success, the transmit queue is correctly set up. * - -ENOMEM: Unable to allocate the transmit ring descriptors. */ -int rte_eth_tx_queue_setup(uint8_t port_id, uint16_t tx_queue_id, +int rte_eth_tx_queue_setup(uint16_t port_id, uint16_t tx_queue_id, uint16_t nb_tx_desc, unsigned int socket_id, const struct rte_eth_txconf *tx_conf); @@ -2028,7 +2105,7 @@ int rte_eth_tx_queue_setup(uint8_t port_id, uint16_t tx_queue_id, * a default of zero if the socket could not be determined. * -1 is returned is the port_id value is out of range. */ -int rte_eth_dev_socket_id(uint8_t port_id); +int rte_eth_dev_socket_id(uint16_t port_id); /** * Check if port_id of device is attached @@ -2039,7 +2116,7 @@ int rte_eth_dev_socket_id(uint8_t port_id); * - 0 if port is out of range or not attached * - 1 if device is attached */ -int rte_eth_dev_is_valid_port(uint8_t port_id); +int rte_eth_dev_is_valid_port(uint16_t port_id); /** * Start specified RX queue of a port. It is used when rx_deferred_start @@ -2056,7 +2133,7 @@ int rte_eth_dev_is_valid_port(uint8_t port_id); * - -EINVAL: The port_id or the queue_id out of range. * - -ENOTSUP: The function not supported in PMD driver. */ -int rte_eth_dev_rx_queue_start(uint8_t port_id, uint16_t rx_queue_id); +int rte_eth_dev_rx_queue_start(uint16_t port_id, uint16_t rx_queue_id); /** * Stop specified RX queue of a port @@ -2072,7 +2149,7 @@ int rte_eth_dev_rx_queue_start(uint8_t port_id, uint16_t rx_queue_id); * - -EINVAL: The port_id or the queue_id out of range. * - -ENOTSUP: The function not supported in PMD driver. */ -int rte_eth_dev_rx_queue_stop(uint8_t port_id, uint16_t rx_queue_id); +int rte_eth_dev_rx_queue_stop(uint16_t port_id, uint16_t rx_queue_id); /** * Start TX for specified queue of a port. It is used when tx_deferred_start @@ -2089,7 +2166,7 @@ int rte_eth_dev_rx_queue_stop(uint8_t port_id, uint16_t rx_queue_id); * - -EINVAL: The port_id or the queue_id out of range. * - -ENOTSUP: The function not supported in PMD driver. */ -int rte_eth_dev_tx_queue_start(uint8_t port_id, uint16_t tx_queue_id); +int rte_eth_dev_tx_queue_start(uint16_t port_id, uint16_t tx_queue_id); /** * Stop specified TX queue of a port @@ -2105,7 +2182,7 @@ int rte_eth_dev_tx_queue_start(uint8_t port_id, uint16_t tx_queue_id); * - -EINVAL: The port_id or the queue_id out of range. * - -ENOTSUP: The function not supported in PMD driver. */ -int rte_eth_dev_tx_queue_stop(uint8_t port_id, uint16_t tx_queue_id); +int rte_eth_dev_tx_queue_stop(uint16_t port_id, uint16_t tx_queue_id); @@ -2124,7 +2201,7 @@ int rte_eth_dev_tx_queue_stop(uint8_t port_id, uint16_t tx_queue_id); * - 0: Success, Ethernet device started. * - <0: Error code of the driver device start function. */ -int rte_eth_dev_start(uint8_t port_id); +int rte_eth_dev_start(uint16_t port_id); /** * Stop an Ethernet device. The device can be restarted with a call to @@ -2133,7 +2210,7 @@ int rte_eth_dev_start(uint8_t port_id); * @param port_id * The port identifier of the Ethernet device. */ -void rte_eth_dev_stop(uint8_t port_id); +void rte_eth_dev_stop(uint16_t port_id); /** @@ -2148,7 +2225,7 @@ void rte_eth_dev_stop(uint8_t port_id); * - 0: Success, Ethernet device linked up. * - <0: Error code of the driver device link up function. */ -int rte_eth_dev_set_link_up(uint8_t port_id); +int rte_eth_dev_set_link_up(uint16_t port_id); /** * Link down an Ethernet device. @@ -2159,7 +2236,7 @@ int rte_eth_dev_set_link_up(uint8_t port_id); * @param port_id * The port identifier of the Ethernet device. */ -int rte_eth_dev_set_link_down(uint8_t port_id); +int rte_eth_dev_set_link_down(uint16_t port_id); /** * Close a stopped Ethernet device. The device cannot be restarted! @@ -2169,7 +2246,46 @@ int rte_eth_dev_set_link_down(uint8_t port_id); * @param port_id * The port identifier of the Ethernet device. */ -void rte_eth_dev_close(uint8_t port_id); +void rte_eth_dev_close(uint16_t port_id); + +/** + * Reset a Ethernet device and keep its port id. + * + * When a port has to be reset passively, the DPDK application can invoke + * this function. For example when a PF is reset, all its VFs should also + * be reset. Normally a DPDK application can invoke this function when + * RTE_ETH_EVENT_INTR_RESET event is detected, but can also use it to start + * a port reset in other circumstances. + * + * When this function is called, it first stops the port and then calls the + * PMD specific dev_uninit( ) and dev_init( ) to return the port to initial + * state, in which no Tx and Rx queues are setup, as if the port has been + * reset and not started. The port keeps the port id it had before the + * function call. + * + * After calling rte_eth_dev_reset( ), the application should use + * rte_eth_dev_configure( ), rte_eth_rx_queue_setup( ), + * rte_eth_tx_queue_setup( ), and rte_eth_dev_start( ) + * to reconfigure the device as appropriate. + * + * Note: To avoid unexpected behavior, the application should stop calling + * Tx and Rx functions before calling rte_eth_dev_reset( ). For thread + * safety, all these controlling functions should be called from the same + * thread. + * + * @param port_id + * The port identifier of the Ethernet device. + * + * @return + * - (0) if successful. + * - (-EINVAL) if port identifier is invalid. + * - (-ENOTSUP) if hardware doesn't support this function. + * - (-EPERM) if not ran from the primary process. + * - (-EIO) if re-initialisation failed. + * - (-ENOMEM) if the reset failed due to OOM. + * - (-EAGAIN) if the reset temporarily failed and should be retried later. + */ +int rte_eth_dev_reset(uint16_t port_id); /** * Enable receipt in promiscuous mode for an Ethernet device. @@ -2177,7 +2293,7 @@ void rte_eth_dev_close(uint8_t port_id); * @param port_id * The port identifier of the Ethernet device. */ -void rte_eth_promiscuous_enable(uint8_t port_id); +void rte_eth_promiscuous_enable(uint16_t port_id); /** * Disable receipt in promiscuous mode for an Ethernet device. @@ -2185,7 +2301,7 @@ void rte_eth_promiscuous_enable(uint8_t port_id); * @param port_id * The port identifier of the Ethernet device. */ -void rte_eth_promiscuous_disable(uint8_t port_id); +void rte_eth_promiscuous_disable(uint16_t port_id); /** * Return the value of promiscuous mode for an Ethernet device. @@ -2197,7 +2313,7 @@ void rte_eth_promiscuous_disable(uint8_t port_id); * - (0) if promiscuous is disabled. * - (-1) on error */ -int rte_eth_promiscuous_get(uint8_t port_id); +int rte_eth_promiscuous_get(uint16_t port_id); /** * Enable the receipt of any multicast frame by an Ethernet device. @@ -2205,7 +2321,7 @@ int rte_eth_promiscuous_get(uint8_t port_id); * @param port_id * The port identifier of the Ethernet device. */ -void rte_eth_allmulticast_enable(uint8_t port_id); +void rte_eth_allmulticast_enable(uint16_t port_id); /** * Disable the receipt of all multicast frames by an Ethernet device. @@ -2213,7 +2329,7 @@ void rte_eth_allmulticast_enable(uint8_t port_id); * @param port_id * The port identifier of the Ethernet device. */ -void rte_eth_allmulticast_disable(uint8_t port_id); +void rte_eth_allmulticast_disable(uint16_t port_id); /** * Return the value of allmulticast mode for an Ethernet device. @@ -2225,7 +2341,7 @@ void rte_eth_allmulticast_disable(uint8_t port_id); * - (0) if allmulticast is disabled. * - (-1) on error */ -int rte_eth_allmulticast_get(uint8_t port_id); +int rte_eth_allmulticast_get(uint16_t port_id); /** * Retrieve the status (ON/OFF), the speed (in Mbps) and the mode (HALF-DUPLEX @@ -2238,7 +2354,7 @@ int rte_eth_allmulticast_get(uint8_t port_id); * A pointer to an *rte_eth_link* structure to be filled with * the status, the speed and the mode of the Ethernet device link. */ -void rte_eth_link_get(uint8_t port_id, struct rte_eth_link *link); +void rte_eth_link_get(uint16_t port_id, struct rte_eth_link *link); /** * Retrieve the status (ON/OFF), the speed (in Mbps) and the mode (HALF-DUPLEX @@ -2251,7 +2367,7 @@ void rte_eth_link_get(uint8_t port_id, struct rte_eth_link *link); * A pointer to an *rte_eth_link* structure to be filled with * the status, the speed and the mode of the Ethernet device link. */ -void rte_eth_link_get_nowait(uint8_t port_id, struct rte_eth_link *link); +void rte_eth_link_get_nowait(uint16_t port_id, struct rte_eth_link *link); /** * Retrieve the general I/O statistics of an Ethernet device. @@ -2270,15 +2386,19 @@ void rte_eth_link_get_nowait(uint8_t port_id, struct rte_eth_link *link); * @return * Zero if successful. Non-zero otherwise. */ -int rte_eth_stats_get(uint8_t port_id, struct rte_eth_stats *stats); +int rte_eth_stats_get(uint16_t port_id, struct rte_eth_stats *stats); /** * Reset the general I/O statistics of an Ethernet device. * * @param port_id * The port identifier of the Ethernet device. + * @return + * - (0) if device notified to reset stats. + * - (-ENOTSUP) if hardware doesn't support. + * - (-ENODEV) if *port_id* invalid. */ -void rte_eth_stats_reset(uint8_t port_id); +int rte_eth_stats_reset(uint16_t port_id); /** * Retrieve names of extended statistics of an Ethernet device. @@ -2286,22 +2406,23 @@ void rte_eth_stats_reset(uint8_t port_id); * @param port_id * The port identifier of the Ethernet device. * @param xstats_names - * Block of memory to insert names into. Must be at least size in capacity. - * If set to NULL, function returns required capacity. + * An rte_eth_xstat_name array of at least *size* elements to + * be filled. If set to NULL, the function returns the required number + * of elements. * @param size - * Capacity of xstats_names (number of names). + * The size of the xstats_names array (number of elements). * @return - * - positive value lower or equal to size: success. The return value + * - A positive value lower or equal to size: success. The return value * is the number of entries filled in the stats table. - * - positive value higher than size: error, the given statistics table + * - A positive value higher than size: error, the given statistics table * is too small. The return value corresponds to the size that should * be given to succeed. The entries in the table are not valid and * shall not be used by the caller. - * - negative value on error (invalid port id) + * - A negative value on error (invalid port id). */ -int rte_eth_xstats_get_names(uint8_t port_id, +int rte_eth_xstats_get_names(uint16_t port_id, struct rte_eth_xstat_name *xstats_names, - unsigned size); + unsigned int size); /** * Retrieve extended statistics of an Ethernet device. @@ -2310,22 +2431,96 @@ int rte_eth_xstats_get_names(uint8_t port_id, * The port identifier of the Ethernet device. * @param xstats * A pointer to a table of structure of type *rte_eth_xstat* - * to be filled with device statistics ids and values. + * to be filled with device statistics ids and values: id is the + * index of the name string in xstats_names (see rte_eth_xstats_get_names()), + * and value is the statistic counter. * This parameter can be set to NULL if n is 0. * @param n - * The size of the stats table, which should be large enough to store - * all the statistics of the device. + * The size of the xstats array (number of elements). * @return - * - positive value lower or equal to n: success. The return value + * - A positive value lower or equal to n: success. The return value * is the number of entries filled in the stats table. - * - positive value higher than n: error, the given statistics table + * - A positive value higher than n: error, the given statistics table * is too small. The return value corresponds to the size that should * be given to succeed. The entries in the table are not valid and * shall not be used by the caller. - * - negative value on error (invalid port id) + * - A negative value on error (invalid port id). */ -int rte_eth_xstats_get(uint8_t port_id, struct rte_eth_xstat *xstats, - unsigned n); +int rte_eth_xstats_get(uint16_t port_id, struct rte_eth_xstat *xstats, + unsigned int n); + +/** + * Retrieve names of extended statistics of an Ethernet device. + * + * @param port_id + * The port identifier of the Ethernet device. + * @param xstats_names + * An rte_eth_xstat_name array of at least *size* elements to + * be filled. If set to NULL, the function returns the required number + * of elements. + * @param ids + * IDs array given by app to retrieve specific statistics + * @param size + * The size of the xstats_names array (number of elements). + * @return + * - A positive value lower or equal to size: success. The return value + * is the number of entries filled in the stats table. + * - A positive value higher than size: error, the given statistics table + * is too small. The return value corresponds to the size that should + * be given to succeed. The entries in the table are not valid and + * shall not be used by the caller. + * - A negative value on error (invalid port id). + */ +int +rte_eth_xstats_get_names_by_id(uint16_t port_id, + struct rte_eth_xstat_name *xstats_names, unsigned int size, + uint64_t *ids); + +/** + * Retrieve extended statistics of an Ethernet device. + * + * @param port_id + * The port identifier of the Ethernet device. + * @param ids + * A pointer to an ids array passed by application. This tells which + * statistics values function should retrieve. This parameter + * can be set to NULL if size is 0. In this case function will retrieve + * all avalible statistics. + * @param values + * A pointer to a table to be filled with device statistics values. + * @param size + * The size of the ids array (number of elements). + * @return + * - A positive value lower or equal to size: success. The return value + * is the number of entries filled in the stats table. + * - A positive value higher than size: error, the given statistics table + * is too small. The return value corresponds to the size that should + * be given to succeed. The entries in the table are not valid and + * shall not be used by the caller. + * - A negative value on error (invalid port id). + */ +int rte_eth_xstats_get_by_id(uint16_t port_id, const uint64_t *ids, + uint64_t *values, unsigned int size); + +/** + * Gets the ID of a statistic from its name. + * + * This function searches for the statistics using string compares, and + * as such should not be used on the fast-path. For fast-path retrieval of + * specific statistics, store the ID as provided in *id* from this function, + * and pass the ID to rte_eth_xstats_get() + * + * @param port_id The port to look up statistics from + * @param xstat_name The name of the statistic to return + * @param[out] id A pointer to an app-supplied uint64_t which should be + * set to the ID of the stat if the stat exists. + * @return + * 0 on success + * -ENODEV for invalid port_id, + * -EINVAL if the xstat_name doesn't exist in port_id + */ +int rte_eth_xstats_get_id_by_name(uint16_t port_id, const char *xstat_name, + uint64_t *id); /** * Reset extended statistics of an Ethernet device. @@ -2333,7 +2528,7 @@ int rte_eth_xstats_get(uint8_t port_id, struct rte_eth_xstat *xstats, * @param port_id * The port identifier of the Ethernet device. */ -void rte_eth_xstats_reset(uint8_t port_id); +void rte_eth_xstats_reset(uint16_t port_id); /** * Set a mapping for the specified transmit queue to the specified per-queue @@ -2352,7 +2547,7 @@ void rte_eth_xstats_reset(uint8_t port_id); * @return * Zero if successful. Non-zero otherwise. */ -int rte_eth_dev_set_tx_queue_stats_mapping(uint8_t port_id, +int rte_eth_dev_set_tx_queue_stats_mapping(uint16_t port_id, uint16_t tx_queue_id, uint8_t stat_idx); /** @@ -2372,7 +2567,7 @@ int rte_eth_dev_set_tx_queue_stats_mapping(uint8_t port_id, * @return * Zero if successful. Non-zero otherwise. */ -int rte_eth_dev_set_rx_queue_stats_mapping(uint8_t port_id, +int rte_eth_dev_set_rx_queue_stats_mapping(uint16_t port_id, uint16_t rx_queue_id, uint8_t stat_idx); @@ -2385,7 +2580,7 @@ int rte_eth_dev_set_rx_queue_stats_mapping(uint8_t port_id, * A pointer to a structure of type *ether_addr* to be filled with * the Ethernet address of the Ethernet device. */ -void rte_eth_macaddr_get(uint8_t port_id, struct ether_addr *mac_addr); +void rte_eth_macaddr_get(uint16_t port_id, struct ether_addr *mac_addr); /** * Retrieve the contextual information of an Ethernet device. @@ -2396,7 +2591,28 @@ void rte_eth_macaddr_get(uint8_t port_id, struct ether_addr *mac_addr); * A pointer to a structure of type *rte_eth_dev_info* to be filled with * the contextual information of the Ethernet device. */ -void rte_eth_dev_info_get(uint8_t port_id, struct rte_eth_dev_info *dev_info); +void rte_eth_dev_info_get(uint16_t port_id, struct rte_eth_dev_info *dev_info); + +/** + * Retrieve the firmware version of a device. + * + * @param port_id + * The port identifier of the device. + * @param fw_version + * A pointer to a string array storing the firmware version of a device, + * the string includes terminating null. This pointer is allocated by caller. + * @param fw_size + * The size of the string array pointed by fw_version, which should be + * large enough to store firmware version of the device. + * @return + * - (0) if successful. + * - (-ENOTSUP) if operation is not supported. + * - (-ENODEV) if *port_id* invalid. + * - (>0) if *fw_size* is not enough to store firmware version, return + * the size of the non truncated string. + */ +int rte_eth_dev_fw_version_get(uint16_t port_id, + char *fw_version, size_t fw_size); /** * Retrieve the supported packet types of an Ethernet device. @@ -2427,7 +2643,7 @@ void rte_eth_dev_info_get(uint8_t port_id, struct rte_eth_dev_info *dev_info); * @param ptype_mask * A hint of what kind of packet type which the caller is interested in. * @param ptypes - * An array pointer to store adequent packet types, allocated by caller. + * An array pointer to store adequate packet types, allocated by caller. * @param num * Size of the array pointed by param ptypes. * @return @@ -2436,7 +2652,7 @@ void rte_eth_dev_info_get(uint8_t port_id, struct rte_eth_dev_info *dev_info); * count of supported ptypes will be returned. * - (-ENODEV) if *port_id* invalid. */ -int rte_eth_dev_get_supported_ptypes(uint8_t port_id, uint32_t ptype_mask, +int rte_eth_dev_get_supported_ptypes(uint16_t port_id, uint32_t ptype_mask, uint32_t *ptypes, int num); /** @@ -2450,7 +2666,7 @@ int rte_eth_dev_get_supported_ptypes(uint8_t port_id, uint32_t ptype_mask, * - (0) if successful. * - (-ENODEV) if *port_id* invalid. */ -int rte_eth_dev_get_mtu(uint8_t port_id, uint16_t *mtu); +int rte_eth_dev_get_mtu(uint16_t port_id, uint16_t *mtu); /** * Change the MTU of an Ethernet device. @@ -2466,7 +2682,7 @@ int rte_eth_dev_get_mtu(uint8_t port_id, uint16_t *mtu); * - (-EINVAL) if *mtu* invalid. * - (-EBUSY) if operation is not allowed when the port is running */ -int rte_eth_dev_set_mtu(uint8_t port_id, uint16_t mtu); +int rte_eth_dev_set_mtu(uint16_t port_id, uint16_t mtu); /** * Enable/Disable hardware filtering by an Ethernet device of received @@ -2486,7 +2702,7 @@ int rte_eth_dev_set_mtu(uint8_t port_id, uint16_t mtu); * - (-ENOSYS) if VLAN filtering on *port_id* disabled. * - (-EINVAL) if *vlan_id* > 4095. */ -int rte_eth_dev_vlan_filter(uint8_t port_id, uint16_t vlan_id, int on); +int rte_eth_dev_vlan_filter(uint16_t port_id, uint16_t vlan_id, int on); /** * Enable/Disable hardware VLAN Strip by a rx queue of an Ethernet device. @@ -2507,7 +2723,7 @@ int rte_eth_dev_vlan_filter(uint8_t port_id, uint16_t vlan_id, int on); * - (-ENODEV) if *port_id* invalid. * - (-EINVAL) if *rx_queue_id* invalid. */ -int rte_eth_dev_set_vlan_strip_on_queue(uint8_t port_id, uint16_t rx_queue_id, +int rte_eth_dev_set_vlan_strip_on_queue(uint16_t port_id, uint16_t rx_queue_id, int on); /** @@ -2526,7 +2742,7 @@ int rte_eth_dev_set_vlan_strip_on_queue(uint8_t port_id, uint16_t rx_queue_id, * - (-ENOSUP) if hardware-assisted VLAN TPID setup is not supported. * - (-ENODEV) if *port_id* invalid. */ -int rte_eth_dev_set_vlan_ether_type(uint8_t port_id, +int rte_eth_dev_set_vlan_ether_type(uint16_t port_id, enum rte_vlan_type vlan_type, uint16_t tag_type); @@ -2550,7 +2766,7 @@ int rte_eth_dev_set_vlan_ether_type(uint8_t port_id, * - (-ENOSUP) if hardware-assisted VLAN filtering not configured. * - (-ENODEV) if *port_id* invalid. */ -int rte_eth_dev_set_vlan_offload(uint8_t port_id, int offload_mask); +int rte_eth_dev_set_vlan_offload(uint16_t port_id, int offload_mask); /** * Read VLAN Offload configuration from an Ethernet device @@ -2564,15 +2780,15 @@ int rte_eth_dev_set_vlan_offload(uint8_t port_id, int offload_mask); * ETH_VLAN_EXTEND_OFFLOAD * - (-ENODEV) if *port_id* invalid. */ -int rte_eth_dev_get_vlan_offload(uint8_t port_id); +int rte_eth_dev_get_vlan_offload(uint16_t port_id); /** - * Set port based TX VLAN insersion on or off. + * Set port based TX VLAN insertion on or off. * * @param port_id * The port identifier of the Ethernet device. * @param pvid - * Port based TX VLAN identifier togeth with user priority. + * Port based TX VLAN identifier together with user priority. * @param on * Turn on or off the port based TX VLAN insertion. * @@ -2580,7 +2796,7 @@ int rte_eth_dev_get_vlan_offload(uint8_t port_id); * - (0) if successful. * - negative if failed. */ -int rte_eth_dev_set_vlan_pvid(uint8_t port_id, uint16_t pvid, int on); +int rte_eth_dev_set_vlan_pvid(uint16_t port_id, uint16_t pvid, int on); /** * @@ -2629,7 +2845,7 @@ int rte_eth_dev_set_vlan_pvid(uint8_t port_id, uint16_t pvid, int on); * method to retrieve bursts of received packets and to immediately * queue them for further parallel processing by another logical core, * for instance. However, instead of having received packets being - * individually queued by the driver, this approach allows the invoker + * individually queued by the driver, this approach allows the caller * of the rte_eth_rx_burst() function to queue a burst of retrieved * packets at a time and therefore dramatically reduce the cost of * enqueue/dequeue operations per packet. @@ -2665,7 +2881,7 @@ int rte_eth_dev_set_vlan_pvid(uint8_t port_id, uint16_t pvid, int on); * *rx_pkts* array. */ static inline uint16_t -rte_eth_rx_burst(uint8_t port_id, uint16_t queue_id, +rte_eth_rx_burst(uint16_t port_id, uint16_t queue_id, struct rte_mbuf **rx_pkts, const uint16_t nb_pkts) { struct rte_eth_dev *dev = &rte_eth_devices[port_id]; @@ -2698,7 +2914,7 @@ rte_eth_rx_burst(uint8_t port_id, uint16_t queue_id, } /** - * Get the number of used descriptors in a specific queue + * Get the number of used descriptors of a rx queue * * @param port_id * The port identifier of the Ethernet device. @@ -2706,16 +2922,21 @@ rte_eth_rx_burst(uint8_t port_id, uint16_t queue_id, * The queue id on the specific port. * @return * The number of used descriptors in the specific queue, or: - * (-EINVAL) if *port_id* is invalid + * (-EINVAL) if *port_id* or *queue_id* is invalid * (-ENOTSUP) if the device does not support this function */ static inline int -rte_eth_rx_queue_count(uint8_t port_id, uint16_t queue_id) +rte_eth_rx_queue_count(uint16_t port_id, uint16_t queue_id) { - struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + struct rte_eth_dev *dev; + RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL); + dev = &rte_eth_devices[port_id]; RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_count, -ENOTSUP); - return (*dev->dev_ops->rx_queue_count)(dev, queue_id); + if (queue_id >= dev->data->nb_rx_queues) + return -EINVAL; + + return (*dev->dev_ops->rx_queue_count)(dev, queue_id); } /** @@ -2734,7 +2955,7 @@ rte_eth_rx_queue_count(uint8_t port_id, uint16_t queue_id) * - (-ENOTSUP) if the device does not support this function */ static inline int -rte_eth_rx_descriptor_done(uint8_t port_id, uint16_t queue_id, uint16_t offset) +rte_eth_rx_descriptor_done(uint16_t port_id, uint16_t queue_id, uint16_t offset) { struct rte_eth_dev *dev = &rte_eth_devices[port_id]; RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); @@ -2743,6 +2964,121 @@ rte_eth_rx_descriptor_done(uint8_t port_id, uint16_t queue_id, uint16_t offset) dev->data->rx_queues[queue_id], offset); } +#define RTE_ETH_RX_DESC_AVAIL 0 /**< Desc available for hw. */ +#define RTE_ETH_RX_DESC_DONE 1 /**< Desc done, filled by hw. */ +#define RTE_ETH_RX_DESC_UNAVAIL 2 /**< Desc used by driver or hw. */ + +/** + * Check the status of a Rx descriptor in the queue + * + * It should be called in a similar context than the Rx function: + * - on a dataplane core + * - not concurrently on the same queue + * + * Since it's a dataplane function, no check is performed on port_id and + * queue_id. The caller must therefore ensure that the port is enabled + * and the queue is configured and running. + * + * Note: accessing to a random descriptor in the ring may trigger cache + * misses and have a performance impact. + * + * @param port_id + * A valid port identifier of the Ethernet device which. + * @param queue_id + * A valid Rx queue identifier on this port. + * @param offset + * The offset of the descriptor starting from tail (0 is the next + * packet to be received by the driver). + * + * @return + * - (RTE_ETH_RX_DESC_AVAIL): Descriptor is available for the hardware to + * receive a packet. + * - (RTE_ETH_RX_DESC_DONE): Descriptor is done, it is filled by hw, but + * not yet processed by the driver (i.e. in the receive queue). + * - (RTE_ETH_RX_DESC_UNAVAIL): Descriptor is unavailable, either hold by + * the driver and not yet returned to hw, or reserved by the hw. + * - (-EINVAL) bad descriptor offset. + * - (-ENOTSUP) if the device does not support this function. + * - (-ENODEV) bad port or queue (only if compiled with debug). + */ +static inline int +rte_eth_rx_descriptor_status(uint16_t port_id, uint16_t queue_id, + uint16_t offset) +{ + struct rte_eth_dev *dev; + void *rxq; + +#ifdef RTE_LIBRTE_ETHDEV_DEBUG + RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); +#endif + dev = &rte_eth_devices[port_id]; +#ifdef RTE_LIBRTE_ETHDEV_DEBUG + if (queue_id >= dev->data->nb_rx_queues) + return -ENODEV; +#endif + RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_descriptor_status, -ENOTSUP); + rxq = dev->data->rx_queues[queue_id]; + + return (*dev->dev_ops->rx_descriptor_status)(rxq, offset); +} + +#define RTE_ETH_TX_DESC_FULL 0 /**< Desc filled for hw, waiting xmit. */ +#define RTE_ETH_TX_DESC_DONE 1 /**< Desc done, packet is transmitted. */ +#define RTE_ETH_TX_DESC_UNAVAIL 2 /**< Desc used by driver or hw. */ + +/** + * Check the status of a Tx descriptor in the queue. + * + * It should be called in a similar context than the Tx function: + * - on a dataplane core + * - not concurrently on the same queue + * + * Since it's a dataplane function, no check is performed on port_id and + * queue_id. The caller must therefore ensure that the port is enabled + * and the queue is configured and running. + * + * Note: accessing to a random descriptor in the ring may trigger cache + * misses and have a performance impact. + * + * @param port_id + * A valid port identifier of the Ethernet device which. + * @param queue_id + * A valid Tx queue identifier on this port. + * @param offset + * The offset of the descriptor starting from tail (0 is the place where + * the next packet will be send). + * + * @return + * - (RTE_ETH_TX_DESC_FULL) Descriptor is being processed by the hw, i.e. + * in the transmit queue. + * - (RTE_ETH_TX_DESC_DONE) Hardware is done with this descriptor, it can + * be reused by the driver. + * - (RTE_ETH_TX_DESC_UNAVAIL): Descriptor is unavailable, reserved by the + * driver or the hardware. + * - (-EINVAL) bad descriptor offset. + * - (-ENOTSUP) if the device does not support this function. + * - (-ENODEV) bad port or queue (only if compiled with debug). + */ +static inline int rte_eth_tx_descriptor_status(uint16_t port_id, + uint16_t queue_id, uint16_t offset) +{ + struct rte_eth_dev *dev; + void *txq; + +#ifdef RTE_LIBRTE_ETHDEV_DEBUG + RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); +#endif + dev = &rte_eth_devices[port_id]; +#ifdef RTE_LIBRTE_ETHDEV_DEBUG + if (queue_id >= dev->data->nb_tx_queues) + return -ENODEV; +#endif + RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_descriptor_status, -ENOTSUP); + txq = dev->data->tx_queues[queue_id]; + + return (*dev->dev_ops->tx_descriptor_status)(txq, offset); +} + /** * Send a burst of output packets on a transmit queue of an Ethernet device. * @@ -2785,6 +3121,10 @@ rte_eth_rx_descriptor_done(uint8_t port_id, uint16_t queue_id, uint16_t offset) * rte_eth_tx_burst() function must [attempt to] free the *rte_mbuf* buffers * of those packets whose transmission was effectively completed. * + * If the PMD is DEV_TX_OFFLOAD_MT_LOCKFREE capable, multiple threads can + * invoke this function concurrently on the same tx queue without SW lock. + * @see rte_eth_dev_info_get, struct rte_eth_txconf::txq_flags + * * @param port_id * The port identifier of the Ethernet device. * @param queue_id @@ -2803,7 +3143,7 @@ rte_eth_rx_descriptor_done(uint8_t port_id, uint16_t queue_id, uint16_t offset) * *tx_pkts* parameter when the transmit ring is full or has been filled up. */ static inline uint16_t -rte_eth_tx_burst(uint8_t port_id, uint16_t queue_id, +rte_eth_tx_burst(uint16_t port_id, uint16_t queue_id, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { struct rte_eth_dev *dev = &rte_eth_devices[port_id]; @@ -2833,6 +3173,116 @@ rte_eth_tx_burst(uint8_t port_id, uint16_t queue_id, return (*dev->tx_pkt_burst)(dev->data->tx_queues[queue_id], tx_pkts, nb_pkts); } +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Process a burst of output packets on a transmit queue of an Ethernet device. + * + * The rte_eth_tx_prepare() function is invoked to prepare output packets to be + * transmitted on the output queue *queue_id* of the Ethernet device designated + * by its *port_id*. + * The *nb_pkts* parameter is the number of packets to be prepared which are + * supplied in the *tx_pkts* array of *rte_mbuf* structures, each of them + * allocated from a pool created with rte_pktmbuf_pool_create(). + * For each packet to send, the rte_eth_tx_prepare() function performs + * the following operations: + * + * - Check if packet meets devices requirements for tx offloads. + * + * - Check limitations about number of segments. + * + * - Check additional requirements when debug is enabled. + * + * - Update and/or reset required checksums when tx offload is set for packet. + * + * Since this function can modify packet data, provided mbufs must be safely + * writable (e.g. modified data cannot be in shared segment). + * + * The rte_eth_tx_prepare() function returns the number of packets ready to be + * sent. A return value equal to *nb_pkts* means that all packets are valid and + * ready to be sent, otherwise stops processing on the first invalid packet and + * leaves the rest packets untouched. + * + * When this functionality is not implemented in the driver, all packets are + * are returned untouched. + * + * @param port_id + * The port identifier of the Ethernet device. + * The value must be a valid port id. + * @param queue_id + * The index of the transmit queue through which output packets must be + * sent. + * The value must be in the range [0, nb_tx_queue - 1] previously supplied + * to rte_eth_dev_configure(). + * @param tx_pkts + * The address of an array of *nb_pkts* pointers to *rte_mbuf* structures + * which contain the output packets. + * @param nb_pkts + * The maximum number of packets to process. + * @return + * The number of packets correct and ready to be sent. The return value can be + * less than the value of the *tx_pkts* parameter when some packet doesn't + * meet devices requirements with rte_errno set appropriately: + * - -EINVAL: offload flags are not correctly set + * - -ENOTSUP: the offload feature is not supported by the hardware + * + */ + +#ifndef RTE_ETHDEV_TX_PREPARE_NOOP + +static inline uint16_t +rte_eth_tx_prepare(uint16_t port_id, uint16_t queue_id, + struct rte_mbuf **tx_pkts, uint16_t nb_pkts) +{ + struct rte_eth_dev *dev; + +#ifdef RTE_LIBRTE_ETHDEV_DEBUG + if (!rte_eth_dev_is_valid_port(port_id)) { + RTE_PMD_DEBUG_TRACE("Invalid TX port_id=%d\n", port_id); + rte_errno = -EINVAL; + return 0; + } +#endif + + dev = &rte_eth_devices[port_id]; + +#ifdef RTE_LIBRTE_ETHDEV_DEBUG + if (queue_id >= dev->data->nb_tx_queues) { + RTE_PMD_DEBUG_TRACE("Invalid TX queue_id=%d\n", queue_id); + rte_errno = -EINVAL; + return 0; + } +#endif + + if (!dev->tx_pkt_prepare) + return nb_pkts; + + return (*dev->tx_pkt_prepare)(dev->data->tx_queues[queue_id], + tx_pkts, nb_pkts); +} + +#else + +/* + * Native NOOP operation for compilation targets which doesn't require any + * preparations steps, and functional NOOP may introduce unnecessary performance + * drop. + * + * Generally this is not a good idea to turn it on globally and didn't should + * be used if behavior of tx_preparation can change. + */ + +static inline uint16_t +rte_eth_tx_prepare(__rte_unused uint16_t port_id, + __rte_unused uint16_t queue_id, + __rte_unused struct rte_mbuf **tx_pkts, uint16_t nb_pkts) +{ + return nb_pkts; +} + +#endif + typedef void (*buffer_tx_error_fn)(struct rte_mbuf **unsent, uint16_t count, void *userdata); @@ -2894,7 +3344,7 @@ rte_eth_tx_buffer_init(struct rte_eth_dev_tx_buffer *buffer, uint16_t size); * callback is called for any packets which could not be sent. */ static inline uint16_t -rte_eth_tx_buffer_flush(uint8_t port_id, uint16_t queue_id, +rte_eth_tx_buffer_flush(uint16_t port_id, uint16_t queue_id, struct rte_eth_dev_tx_buffer *buffer) { uint16_t sent; @@ -2945,8 +3395,8 @@ rte_eth_tx_buffer_flush(uint8_t port_id, uint16_t queue_id, * causing N packets to be sent, and the error callback to be called for * the rest. */ -static inline uint16_t __attribute__((always_inline)) -rte_eth_tx_buffer(uint8_t port_id, uint16_t queue_id, +static __rte_always_inline uint16_t +rte_eth_tx_buffer(uint16_t port_id, uint16_t queue_id, struct rte_eth_dev_tx_buffer *buffer, struct rte_mbuf *tx_pkt) { buffer->pkts[buffer->length++] = tx_pkt; @@ -3037,6 +3487,33 @@ void rte_eth_tx_buffer_count_callback(struct rte_mbuf **pkts, uint16_t unsent, void *userdata); +/** + * Request the driver to free mbufs currently cached by the driver. The + * driver will only free the mbuf if it is no longer in use. It is the + * application's responsibity to ensure rte_eth_tx_buffer_flush(..) is + * called if needed. + * + * @param port_id + * The port identifier of the Ethernet device. + * @param queue_id + * The index of the transmit queue through which output packets must be + * sent. + * The value must be in the range [0, nb_tx_queue - 1] previously supplied + * to rte_eth_dev_configure(). + * @param free_cnt + * Maximum number of packets to free. Use 0 to indicate all possible packets + * should be freed. Note that a packet may be using multiple mbufs. + * @return + * Failure: < 0 + * -ENODEV: Invalid interface + * -ENOTSUP: Driver does not support function + * Success: >= 0 + * 0-n: Number of packets freed. More packets may still remain in ring that + * are in use. + */ +int +rte_eth_tx_done_cleanup(uint16_t port_id, uint16_t queue_id, uint32_t free_cnt); + /** * The eth device event type for interrupt, and maybe others in the future. */ @@ -3047,11 +3524,14 @@ enum rte_eth_event_type { /**< queue state event (enabled/disabled) */ RTE_ETH_EVENT_INTR_RESET, /**< reset interrupt event, sent to VF on PF reset */ + RTE_ETH_EVENT_VF_MBOX, /**< message from the VF received by PF */ + RTE_ETH_EVENT_MACSEC, /**< MACsec offload related event */ + RTE_ETH_EVENT_INTR_RMV, /**< device removal event */ RTE_ETH_EVENT_MAX /**< max value of this enum */ }; -typedef void (*rte_eth_dev_cb_fn)(uint8_t port_id, \ - enum rte_eth_event_type event, void *cb_arg); +typedef int (*rte_eth_dev_cb_fn)(uint16_t port_id, + enum rte_eth_event_type event, void *cb_arg, void *ret_param); /**< user application callback to be registered for interrupts */ @@ -3072,7 +3552,7 @@ typedef void (*rte_eth_dev_cb_fn)(uint8_t port_id, \ * - On success, zero. * - On failure, a negative value. */ -int rte_eth_dev_callback_register(uint8_t port_id, +int rte_eth_dev_callback_register(uint16_t port_id, enum rte_eth_event_type event, rte_eth_dev_cb_fn cb_fn, void *cb_arg); @@ -3093,7 +3573,7 @@ int rte_eth_dev_callback_register(uint8_t port_id, * - On success, zero. * - On failure, a negative value. */ -int rte_eth_dev_callback_unregister(uint8_t port_id, +int rte_eth_dev_callback_unregister(uint16_t port_id, enum rte_eth_event_type event, rte_eth_dev_cb_fn cb_fn, void *cb_arg); @@ -3106,17 +3586,23 @@ int rte_eth_dev_callback_unregister(uint8_t port_id, * Pointer to struct rte_eth_dev. * @param event * Eth device interrupt event type. + * @param cb_arg + * callback parameter. + * @param ret_param + * To pass data back to user application. + * This allows the user application to decide if a particular function + * is permitted or not. * * @return - * void + * int */ -void _rte_eth_dev_callback_process(struct rte_eth_dev *dev, - enum rte_eth_event_type event); +int _rte_eth_dev_callback_process(struct rte_eth_dev *dev, + enum rte_eth_event_type event, void *cb_arg, void *ret_param); /** * When there is no rx packet coming in Rx Queue for a long time, we can * sleep lcore related to RX Queue for power saving, and enable rx interrupt - * to be triggered when rx packect arrives. + * to be triggered when Rx packet arrives. * * The rte_eth_dev_rx_intr_enable() function enables rx queue * interrupt on specific rx queue of a port. @@ -3133,7 +3619,7 @@ void _rte_eth_dev_callback_process(struct rte_eth_dev *dev, * that operation. * - (-ENODEV) if *port_id* invalid. */ -int rte_eth_dev_rx_intr_enable(uint8_t port_id, uint16_t queue_id); +int rte_eth_dev_rx_intr_enable(uint16_t port_id, uint16_t queue_id); /** * When lcore wakes up from rx interrupt indicating packet coming, disable rx @@ -3154,7 +3640,7 @@ int rte_eth_dev_rx_intr_enable(uint8_t port_id, uint16_t queue_id); * that operation. * - (-ENODEV) if *port_id* invalid. */ -int rte_eth_dev_rx_intr_disable(uint8_t port_id, uint16_t queue_id); +int rte_eth_dev_rx_intr_disable(uint16_t port_id, uint16_t queue_id); /** * RX Interrupt control per port. @@ -3173,7 +3659,7 @@ int rte_eth_dev_rx_intr_disable(uint8_t port_id, uint16_t queue_id); * - On success, zero. * - On failure, a negative value. */ -int rte_eth_dev_rx_intr_ctl(uint8_t port_id, int epfd, int op, void *data); +int rte_eth_dev_rx_intr_ctl(uint16_t port_id, int epfd, int op, void *data); /** * RX Interrupt control per queue. @@ -3196,7 +3682,7 @@ int rte_eth_dev_rx_intr_ctl(uint8_t port_id, int epfd, int op, void *data); * - On success, zero. * - On failure, a negative value. */ -int rte_eth_dev_rx_intr_ctl_q(uint8_t port_id, uint16_t queue_id, +int rte_eth_dev_rx_intr_ctl_q(uint16_t port_id, uint16_t queue_id, int epfd, int op, void *data); /** @@ -3211,7 +3697,7 @@ int rte_eth_dev_rx_intr_ctl_q(uint8_t port_id, uint16_t queue_id, * that operation. * - (-ENODEV) if *port_id* invalid. */ -int rte_eth_led_on(uint8_t port_id); +int rte_eth_led_on(uint16_t port_id); /** * Turn off the LED on the Ethernet device. @@ -3225,7 +3711,7 @@ int rte_eth_led_on(uint8_t port_id); * that operation. * - (-ENODEV) if *port_id* invalid. */ -int rte_eth_led_off(uint8_t port_id); +int rte_eth_led_off(uint16_t port_id); /** * Get current status of the Ethernet link flow control for Ethernet device @@ -3239,7 +3725,7 @@ int rte_eth_led_off(uint8_t port_id); * - (-ENOTSUP) if hardware doesn't support flow control. * - (-ENODEV) if *port_id* invalid. */ -int rte_eth_dev_flow_ctrl_get(uint8_t port_id, +int rte_eth_dev_flow_ctrl_get(uint16_t port_id, struct rte_eth_fc_conf *fc_conf); /** @@ -3256,7 +3742,7 @@ int rte_eth_dev_flow_ctrl_get(uint8_t port_id, * - (-EINVAL) if bad parameter * - (-EIO) if flow control setup failure */ -int rte_eth_dev_flow_ctrl_set(uint8_t port_id, +int rte_eth_dev_flow_ctrl_set(uint16_t port_id, struct rte_eth_fc_conf *fc_conf); /** @@ -3274,7 +3760,7 @@ int rte_eth_dev_flow_ctrl_set(uint8_t port_id, * - (-EINVAL) if bad parameter * - (-EIO) if flow control setup failure */ -int rte_eth_dev_priority_flow_ctrl_set(uint8_t port_id, +int rte_eth_dev_priority_flow_ctrl_set(uint16_t port_id, struct rte_eth_pfc_conf *pfc_conf); /** @@ -3295,7 +3781,7 @@ int rte_eth_dev_priority_flow_ctrl_set(uint8_t port_id, * - (-ENOSPC) if no more MAC addresses can be added. * - (-EINVAL) if MAC address is invalid. */ -int rte_eth_dev_mac_addr_add(uint8_t port, struct ether_addr *mac_addr, +int rte_eth_dev_mac_addr_add(uint16_t port, struct ether_addr *mac_addr, uint32_t pool); /** @@ -3311,7 +3797,7 @@ int rte_eth_dev_mac_addr_add(uint8_t port, struct ether_addr *mac_addr, * - (-ENODEV) if *port* invalid. * - (-EADDRINUSE) if attempting to remove the default MAC address */ -int rte_eth_dev_mac_addr_remove(uint8_t port, struct ether_addr *mac_addr); +int rte_eth_dev_mac_addr_remove(uint16_t port, struct ether_addr *mac_addr); /** * Set the default MAC address. @@ -3326,8 +3812,8 @@ int rte_eth_dev_mac_addr_remove(uint8_t port, struct ether_addr *mac_addr); * - (-ENODEV) if *port* invalid. * - (-EINVAL) if MAC address is invalid. */ -int rte_eth_dev_default_mac_addr_set(uint8_t port, struct ether_addr *mac_addr); - +int rte_eth_dev_default_mac_addr_set(uint16_t port, + struct ether_addr *mac_addr); /** * Update Redirection Table(RETA) of Receive Side Scaling of Ethernet device. @@ -3344,7 +3830,7 @@ int rte_eth_dev_default_mac_addr_set(uint8_t port, struct ether_addr *mac_addr); * - (-ENOTSUP) if hardware doesn't support. * - (-EINVAL) if bad parameter. */ -int rte_eth_dev_rss_reta_update(uint8_t port, +int rte_eth_dev_rss_reta_update(uint16_t port, struct rte_eth_rss_reta_entry64 *reta_conf, uint16_t reta_size); @@ -3363,7 +3849,7 @@ int rte_eth_dev_rss_reta_update(uint8_t port, * - (-ENOTSUP) if hardware doesn't support. * - (-EINVAL) if bad parameter. */ -int rte_eth_dev_rss_reta_query(uint8_t port, +int rte_eth_dev_rss_reta_query(uint16_t port, struct rte_eth_rss_reta_entry64 *reta_conf, uint16_t reta_size); @@ -3385,8 +3871,8 @@ int rte_eth_dev_rss_reta_query(uint8_t port, * - (-ENODEV) if *port_id* invalid. * - (-EINVAL) if bad parameter. */ -int rte_eth_dev_uc_hash_table_set(uint8_t port,struct ether_addr *addr, - uint8_t on); +int rte_eth_dev_uc_hash_table_set(uint16_t port, struct ether_addr *addr, + uint8_t on); /** * Updates all unicast hash bitmaps for receiving packet with any Unicast @@ -3405,94 +3891,7 @@ int rte_eth_dev_uc_hash_table_set(uint8_t port,struct ether_addr *addr, * - (-ENODEV) if *port_id* invalid. * - (-EINVAL) if bad parameter. */ -int rte_eth_dev_uc_all_hash_table_set(uint8_t port,uint8_t on); - - /** - * Set RX L2 Filtering mode of a VF of an Ethernet device. - * - * @param port - * The port identifier of the Ethernet device. - * @param vf - * VF id. - * @param rx_mode - * The RX mode mask, which is one or more of accepting Untagged Packets, - * packets that match the PFUTA table, Broadcast and Multicast Promiscuous. - * ETH_VMDQ_ACCEPT_UNTAG,ETH_VMDQ_ACCEPT_HASH_UC, - * ETH_VMDQ_ACCEPT_BROADCAST and ETH_VMDQ_ACCEPT_MULTICAST will be used - * in rx_mode. - * @param on - * 1 - Enable a VF RX mode. - * 0 - Disable a VF RX mode. - * @return - * - (0) if successful. - * - (-ENOTSUP) if hardware doesn't support. - * - (-ENOTSUP) if hardware doesn't support. - * - (-EINVAL) if bad parameter. - */ -int rte_eth_dev_set_vf_rxmode(uint8_t port, uint16_t vf, uint16_t rx_mode, - uint8_t on); - -/** -* Enable or disable a VF traffic transmit of the Ethernet device. -* -* @param port -* The port identifier of the Ethernet device. -* @param vf -* VF id. -* @param on -* 1 - Enable a VF traffic transmit. -* 0 - Disable a VF traffic transmit. -* @return -* - (0) if successful. -* - (-ENODEV) if *port_id* invalid. -* - (-ENOTSUP) if hardware doesn't support. -* - (-EINVAL) if bad parameter. -*/ -int -rte_eth_dev_set_vf_tx(uint8_t port,uint16_t vf, uint8_t on); - -/** -* Enable or disable a VF traffic receive of an Ethernet device. -* -* @param port -* The port identifier of the Ethernet device. -* @param vf -* VF id. -* @param on -* 1 - Enable a VF traffic receive. -* 0 - Disable a VF traffic receive. -* @return -* - (0) if successful. -* - (-ENOTSUP) if hardware doesn't support. -* - (-ENODEV) if *port_id* invalid. -* - (-EINVAL) if bad parameter. -*/ -int -rte_eth_dev_set_vf_rx(uint8_t port,uint16_t vf, uint8_t on); - -/** -* Enable/Disable hardware VF VLAN filtering by an Ethernet device of -* received VLAN packets tagged with a given VLAN Tag Identifier. -* -* @param port id -* The port identifier of the Ethernet device. -* @param vlan_id -* The VLAN Tag Identifier whose filtering must be enabled or disabled. -* @param vf_mask -* Bitmap listing which VFs participate in the VLAN filtering. -* @param vlan_on -* 1 - Enable VFs VLAN filtering. -* 0 - Disable VFs VLAN filtering. -* @return -* - (0) if successful. -* - (-ENOTSUP) if hardware doesn't support. -* - (-ENODEV) if *port_id* invalid. -* - (-EINVAL) if bad parameter. -*/ -int -rte_eth_dev_set_vf_vlan_filter(uint8_t port, uint16_t vlan_id, - uint64_t vf_mask, - uint8_t vlan_on); +int rte_eth_dev_uc_all_hash_table_set(uint16_t port, uint8_t on); /** * Set a traffic mirroring rule on an Ethernet device @@ -3515,7 +3914,7 @@ rte_eth_dev_set_vf_vlan_filter(uint8_t port, uint16_t vlan_id, * - (-ENODEV) if *port_id* invalid. * - (-EINVAL) if the mr_conf information is not correct. */ -int rte_eth_mirror_rule_set(uint8_t port_id, +int rte_eth_mirror_rule_set(uint16_t port_id, struct rte_eth_mirror_conf *mirror_conf, uint8_t rule_id, uint8_t on); @@ -3533,7 +3932,7 @@ int rte_eth_mirror_rule_set(uint8_t port_id, * - (-ENODEV) if *port_id* invalid. * - (-EINVAL) if bad parameter. */ -int rte_eth_mirror_rule_reset(uint8_t port_id, +int rte_eth_mirror_rule_reset(uint16_t port_id, uint8_t rule_id); /** @@ -3551,194 +3950,9 @@ int rte_eth_mirror_rule_reset(uint8_t port_id, * - (-ENODEV) if *port_id* invalid. * - (-EINVAL) if bad parameter. */ -int rte_eth_set_queue_rate_limit(uint8_t port_id, uint16_t queue_idx, +int rte_eth_set_queue_rate_limit(uint16_t port_id, uint16_t queue_idx, uint16_t tx_rate); -/** - * Set the rate limitation for a vf on an Ethernet device. - * - * @param port_id - * The port identifier of the Ethernet device. - * @param vf - * VF id. - * @param tx_rate - * The tx rate allocated from the total link speed for this VF id. - * @param q_msk - * The queue mask which need to set the rate. - * @return - * - (0) if successful. - * - (-ENOTSUP) if hardware doesn't support this feature. - * - (-ENODEV) if *port_id* invalid. - * - (-EINVAL) if bad parameter. - */ -int rte_eth_set_vf_rate_limit(uint8_t port_id, uint16_t vf, - uint16_t tx_rate, uint64_t q_msk); - -/** - * Initialize bypass logic. This function needs to be called before - * executing any other bypass API. - * - * @param port - * The port identifier of the Ethernet device. - * @return - * - (0) if successful. - * - (-ENOTSUP) if hardware doesn't support. - * - (-EINVAL) if bad parameter. - */ -int rte_eth_dev_bypass_init(uint8_t port); - -/** - * Return bypass state. - * - * @param port - * The port identifier of the Ethernet device. - * @param state - * The return bypass state. - * - (1) Normal mode - * - (2) Bypass mode - * - (3) Isolate mode - * @return - * - (0) if successful. - * - (-ENOTSUP) if hardware doesn't support. - * - (-EINVAL) if bad parameter. - */ -int rte_eth_dev_bypass_state_show(uint8_t port, uint32_t *state); - -/** - * Set bypass state - * - * @param port - * The port identifier of the Ethernet device. - * @param new_state - * The current bypass state. - * - (1) Normal mode - * - (2) Bypass mode - * - (3) Isolate mode - * @return - * - (0) if successful. - * - (-ENOTSUP) if hardware doesn't support. - * - (-EINVAL) if bad parameter. - */ -int rte_eth_dev_bypass_state_set(uint8_t port, uint32_t *new_state); - -/** - * Return bypass state when given event occurs. - * - * @param port - * The port identifier of the Ethernet device. - * @param event - * The bypass event - * - (1) Main power on (power button is pushed) - * - (2) Auxiliary power on (power supply is being plugged) - * - (3) Main power off (system shutdown and power supply is left plugged in) - * - (4) Auxiliary power off (power supply is being unplugged) - * - (5) Display or set the watchdog timer - * @param state - * The bypass state when given event occurred. - * - (1) Normal mode - * - (2) Bypass mode - * - (3) Isolate mode - * @return - * - (0) if successful. - * - (-ENOTSUP) if hardware doesn't support. - * - (-EINVAL) if bad parameter. - */ -int rte_eth_dev_bypass_event_show(uint8_t port, uint32_t event, uint32_t *state); - -/** - * Set bypass state when given event occurs. - * - * @param port - * The port identifier of the Ethernet device. - * @param event - * The bypass event - * - (1) Main power on (power button is pushed) - * - (2) Auxiliary power on (power supply is being plugged) - * - (3) Main power off (system shutdown and power supply is left plugged in) - * - (4) Auxiliary power off (power supply is being unplugged) - * - (5) Display or set the watchdog timer - * @param state - * The assigned state when given event occurs. - * - (1) Normal mode - * - (2) Bypass mode - * - (3) Isolate mode - * @return - * - (0) if successful. - * - (-ENOTSUP) if hardware doesn't support. - * - (-EINVAL) if bad parameter. - */ -int rte_eth_dev_bypass_event_store(uint8_t port, uint32_t event, uint32_t state); - -/** - * Set bypass watchdog timeout count. - * - * @param port - * The port identifier of the Ethernet device. - * @param timeout - * The timeout to be set. - * - (0) 0 seconds (timer is off) - * - (1) 1.5 seconds - * - (2) 2 seconds - * - (3) 3 seconds - * - (4) 4 seconds - * - (5) 8 seconds - * - (6) 16 seconds - * - (7) 32 seconds - * @return - * - (0) if successful. - * - (-ENOTSUP) if hardware doesn't support. - * - (-EINVAL) if bad parameter. - */ -int rte_eth_dev_wd_timeout_store(uint8_t port, uint32_t timeout); - -/** - * Get bypass firmware version. - * - * @param port - * The port identifier of the Ethernet device. - * @param ver - * The firmware version - * @return - * - (0) if successful. - * - (-ENOTSUP) if hardware doesn't support. - * - (-EINVAL) if bad parameter. - */ -int rte_eth_dev_bypass_ver_show(uint8_t port, uint32_t *ver); - -/** - * Return bypass watchdog timeout in seconds - * - * @param port - * The port identifier of the Ethernet device. - * @param wd_timeout - * The return watchdog timeout. "0" represents timer expired - * - (0) 0 seconds (timer is off) - * - (1) 1.5 seconds - * - (2) 2 seconds - * - (3) 3 seconds - * - (4) 4 seconds - * - (5) 8 seconds - * - (6) 16 seconds - * - (7) 32 seconds - * @return - * - (0) if successful. - * - (-ENOTSUP) if hardware doesn't support. - * - (-EINVAL) if bad parameter. - */ -int rte_eth_dev_bypass_wd_timeout_show(uint8_t port, uint32_t *wd_timeout); - -/** - * Reset bypass watchdog timer - * - * @param port - * The port identifier of the Ethernet device. - * @return - * - (0) if successful. - * - (-ENOTSUP) if hardware doesn't support. - * - (-EINVAL) if bad parameter. - */ -int rte_eth_dev_bypass_wd_reset(uint8_t port); - /** * Configuration of Receive Side Scaling hash computation of Ethernet device. * @@ -3752,7 +3966,7 @@ int rte_eth_dev_bypass_wd_reset(uint8_t port); * - (-ENOTSUP) if hardware doesn't support. * - (-EINVAL) if bad parameter. */ -int rte_eth_dev_rss_hash_update(uint8_t port_id, +int rte_eth_dev_rss_hash_update(uint16_t port_id, struct rte_eth_rss_conf *rss_conf); /** @@ -3769,7 +3983,7 @@ int rte_eth_dev_rss_hash_update(uint8_t port_id, * - (-ENOTSUP) if hardware doesn't support RSS. */ int -rte_eth_dev_rss_hash_conf_get(uint8_t port_id, +rte_eth_dev_rss_hash_conf_get(uint16_t port_id, struct rte_eth_rss_conf *rss_conf); /** @@ -3777,7 +3991,7 @@ rte_eth_dev_rss_hash_conf_get(uint8_t port_id, * The packets with this UDP port will be identified as this type of tunnel. * Before enabling any offloading function for a tunnel, users can call this API * to change or add more UDP port for the tunnel. So the offloading function - * can take effect on the packets with the sepcific UDP port. + * can take effect on the packets with the specific UDP port. * * @param port_id * The port identifier of the Ethernet device. @@ -3790,7 +4004,7 @@ rte_eth_dev_rss_hash_conf_get(uint8_t port_id, * - (-ENOTSUP) if hardware doesn't support tunnel type. */ int -rte_eth_dev_udp_tunnel_port_add(uint8_t port_id, +rte_eth_dev_udp_tunnel_port_add(uint16_t port_id, struct rte_eth_udp_tunnel *tunnel_udp); /** @@ -3799,7 +4013,7 @@ rte_eth_dev_udp_tunnel_port_add(uint8_t port_id, * any more. * Before enabling any offloading function for a tunnel, users can call this API * to delete a UDP port for the tunnel. So the offloading function will not take - * effect on the packets with the sepcific UDP port. + * effect on the packets with the specific UDP port. * * @param port_id * The port identifier of the Ethernet device. @@ -3812,7 +4026,7 @@ rte_eth_dev_udp_tunnel_port_add(uint8_t port_id, * - (-ENOTSUP) if hardware doesn't support tunnel type. */ int -rte_eth_dev_udp_tunnel_port_delete(uint8_t port_id, +rte_eth_dev_udp_tunnel_port_delete(uint16_t port_id, struct rte_eth_udp_tunnel *tunnel_udp); /** @@ -3828,7 +4042,8 @@ rte_eth_dev_udp_tunnel_port_delete(uint8_t port_id, * - (-ENOTSUP) if hardware doesn't support this filter type. * - (-ENODEV) if *port_id* invalid. */ -int rte_eth_dev_filter_supported(uint8_t port_id, enum rte_filter_type filter_type); +int rte_eth_dev_filter_supported(uint16_t port_id, + enum rte_filter_type filter_type); /** * Take operations to assigned filter type on an Ethernet device. @@ -3848,7 +4063,7 @@ int rte_eth_dev_filter_supported(uint8_t port_id, enum rte_filter_type filter_ty * - (-ENODEV) if *port_id* invalid. * - others depends on the specific operations implementation. */ -int rte_eth_dev_filter_ctrl(uint8_t port_id, enum rte_filter_type filter_type, +int rte_eth_dev_filter_ctrl(uint16_t port_id, enum rte_filter_type filter_type, enum rte_filter_op filter_op, void *arg); /** @@ -3863,7 +4078,7 @@ int rte_eth_dev_filter_ctrl(uint8_t port_id, enum rte_filter_type filter_type, * - (-ENODEV) if port identifier is invalid. * - (-ENOTSUP) if hardware doesn't support. */ -int rte_eth_dev_get_dcb_info(uint8_t port_id, +int rte_eth_dev_get_dcb_info(uint16_t port_id, struct rte_eth_dcb_info *dcb_info); /** @@ -3890,35 +4105,35 @@ int rte_eth_dev_get_dcb_info(uint8_t port_id, * NULL on error. * On success, a pointer value which can later be used to remove the callback. */ -void *rte_eth_add_rx_callback(uint8_t port_id, uint16_t queue_id, +void *rte_eth_add_rx_callback(uint16_t port_id, uint16_t queue_id, rte_rx_callback_fn fn, void *user_param); -/* -* Add a callback that must be called first on packet RX on a given port -* and queue. -* -* This API configures a first function to be called for each burst of -* packets received on a given NIC port queue. The return value is a pointer -* that can be used to later remove the callback using -* rte_eth_remove_rx_callback(). -* -* Multiple functions are called in the order that they are added. -* -* @param port_id -* The port identifier of the Ethernet device. -* @param queue_id -* The queue on the Ethernet device on which the callback is to be added. -* @param fn -* The callback function -* @param user_param -* A generic pointer parameter which will be passed to each invocation of the -* callback function on this port and queue. -* -* @return -* NULL on error. -* On success, a pointer value which can later be used to remove the callback. -*/ -void *rte_eth_add_first_rx_callback(uint8_t port_id, uint16_t queue_id, +/** + * Add a callback that must be called first on packet RX on a given port + * and queue. + * + * This API configures a first function to be called for each burst of + * packets received on a given NIC port queue. The return value is a pointer + * that can be used to later remove the callback using + * rte_eth_remove_rx_callback(). + * + * Multiple functions are called in the order that they are added. + * + * @param port_id + * The port identifier of the Ethernet device. + * @param queue_id + * The queue on the Ethernet device on which the callback is to be added. + * @param fn + * The callback function + * @param user_param + * A generic pointer parameter which will be passed to each invocation of the + * callback function on this port and queue. + * + * @return + * NULL on error. + * On success, a pointer value which can later be used to remove the callback. + */ +void *rte_eth_add_first_rx_callback(uint16_t port_id, uint16_t queue_id, rte_rx_callback_fn fn, void *user_param); /** @@ -3945,7 +4160,7 @@ void *rte_eth_add_first_rx_callback(uint8_t port_id, uint16_t queue_id, * NULL on error. * On success, a pointer value which can later be used to remove the callback. */ -void *rte_eth_add_tx_callback(uint8_t port_id, uint16_t queue_id, +void *rte_eth_add_tx_callback(uint16_t port_id, uint16_t queue_id, rte_tx_callback_fn fn, void *user_param); /** @@ -3978,7 +4193,7 @@ void *rte_eth_add_tx_callback(uint8_t port_id, uint16_t queue_id, * - -EINVAL: The port_id or the queue_id is out of range, or the callback * is NULL or not found for the port/queue. */ -int rte_eth_remove_rx_callback(uint8_t port_id, uint16_t queue_id, +int rte_eth_remove_rx_callback(uint16_t port_id, uint16_t queue_id, struct rte_eth_rxtx_callback *user_cb); /** @@ -4011,7 +4226,7 @@ int rte_eth_remove_rx_callback(uint8_t port_id, uint16_t queue_id, * - -EINVAL: The port_id or the queue_id is out of range, or the callback * is NULL or not found for the port/queue. */ -int rte_eth_remove_tx_callback(uint8_t port_id, uint16_t queue_id, +int rte_eth_remove_tx_callback(uint16_t port_id, uint16_t queue_id, struct rte_eth_rxtx_callback *user_cb); /** @@ -4031,7 +4246,7 @@ int rte_eth_remove_tx_callback(uint8_t port_id, uint16_t queue_id, * - -ENOTSUP: routine is not supported by the device PMD. * - -EINVAL: The port_id or the queue_id is out of range. */ -int rte_eth_rx_queue_info_get(uint8_t port_id, uint16_t queue_id, +int rte_eth_rx_queue_info_get(uint16_t port_id, uint16_t queue_id, struct rte_eth_rxq_info *qinfo); /** @@ -4051,7 +4266,7 @@ int rte_eth_rx_queue_info_get(uint8_t port_id, uint16_t queue_id, * - -ENOTSUP: routine is not supported by the device PMD. * - -EINVAL: The port_id or the queue_id is out of range. */ -int rte_eth_tx_queue_info_get(uint8_t port_id, uint16_t queue_id, +int rte_eth_tx_queue_info_get(uint16_t port_id, uint16_t queue_id, struct rte_eth_txq_info *qinfo); /** @@ -4070,7 +4285,7 @@ int rte_eth_tx_queue_info_get(uint8_t port_id, uint16_t queue_id, * - (-ENODEV) if *port_id* invalid. * - others depends on the specific operations implementation. */ -int rte_eth_dev_get_reg_info(uint8_t port_id, struct rte_dev_reg_info *info); +int rte_eth_dev_get_reg_info(uint16_t port_id, struct rte_dev_reg_info *info); /** * Retrieve size of device EEPROM @@ -4083,7 +4298,7 @@ int rte_eth_dev_get_reg_info(uint8_t port_id, struct rte_dev_reg_info *info); * - (-ENODEV) if *port_id* invalid. * - others depends on the specific operations implementation. */ -int rte_eth_dev_get_eeprom_length(uint8_t port_id); +int rte_eth_dev_get_eeprom_length(uint16_t port_id); /** * Retrieve EEPROM and EEPROM attribute @@ -4099,7 +4314,7 @@ int rte_eth_dev_get_eeprom_length(uint8_t port_id); * - (-ENODEV) if *port_id* invalid. * - others depends on the specific operations implementation. */ -int rte_eth_dev_get_eeprom(uint8_t port_id, struct rte_dev_eeprom_info *info); +int rte_eth_dev_get_eeprom(uint16_t port_id, struct rte_dev_eeprom_info *info); /** * Program EEPROM with provided data @@ -4115,7 +4330,7 @@ int rte_eth_dev_get_eeprom(uint8_t port_id, struct rte_dev_eeprom_info *info); * - (-ENODEV) if *port_id* invalid. * - others depends on the specific operations implementation. */ -int rte_eth_dev_set_eeprom(uint8_t port_id, struct rte_dev_eeprom_info *info); +int rte_eth_dev_set_eeprom(uint16_t port_id, struct rte_dev_eeprom_info *info); /** * Set the list of multicast addresses to filter on an Ethernet device. @@ -4134,7 +4349,7 @@ int rte_eth_dev_set_eeprom(uint8_t port_id, struct rte_dev_eeprom_info *info); * - (-ENOTSUP) if PMD of *port_id* doesn't support multicast filtering. * - (-ENOSPC) if *port_id* has not enough multicast filtering resources. */ -int rte_eth_dev_set_mc_addr_list(uint8_t port_id, +int rte_eth_dev_set_mc_addr_list(uint16_t port_id, struct ether_addr *mc_addr_set, uint32_t nb_mc_addr); @@ -4149,7 +4364,7 @@ int rte_eth_dev_set_mc_addr_list(uint8_t port_id, * - -ENODEV: The port ID is invalid. * - -ENOTSUP: The function is not supported by the Ethernet driver. */ -int rte_eth_timesync_enable(uint8_t port_id); +int rte_eth_timesync_enable(uint16_t port_id); /** * Disable IEEE1588/802.1AS timestamping for an Ethernet device. @@ -4162,7 +4377,7 @@ int rte_eth_timesync_enable(uint8_t port_id); * - -ENODEV: The port ID is invalid. * - -ENOTSUP: The function is not supported by the Ethernet driver. */ -int rte_eth_timesync_disable(uint8_t port_id); +int rte_eth_timesync_disable(uint16_t port_id); /** * Read an IEEE1588/802.1AS RX timestamp from an Ethernet device. @@ -4181,7 +4396,7 @@ int rte_eth_timesync_disable(uint8_t port_id); * - -ENODEV: The port ID is invalid. * - -ENOTSUP: The function is not supported by the Ethernet driver. */ -int rte_eth_timesync_read_rx_timestamp(uint8_t port_id, +int rte_eth_timesync_read_rx_timestamp(uint16_t port_id, struct timespec *timestamp, uint32_t flags); /** @@ -4198,7 +4413,7 @@ int rte_eth_timesync_read_rx_timestamp(uint8_t port_id, * - -ENODEV: The port ID is invalid. * - -ENOTSUP: The function is not supported by the Ethernet driver. */ -int rte_eth_timesync_read_tx_timestamp(uint8_t port_id, +int rte_eth_timesync_read_tx_timestamp(uint16_t port_id, struct timespec *timestamp); /** @@ -4217,7 +4432,7 @@ int rte_eth_timesync_read_tx_timestamp(uint8_t port_id, * - -ENODEV: The port ID is invalid. * - -ENOTSUP: The function is not supported by the Ethernet driver. */ -int rte_eth_timesync_adjust_time(uint8_t port_id, int64_t delta); +int rte_eth_timesync_adjust_time(uint16_t port_id, int64_t delta); /** * Read the time from the timesync clock on an Ethernet device. @@ -4233,7 +4448,7 @@ int rte_eth_timesync_adjust_time(uint8_t port_id, int64_t delta); * @return * - 0: Success. */ -int rte_eth_timesync_read_time(uint8_t port_id, struct timespec *time); +int rte_eth_timesync_read_time(uint16_t port_id, struct timespec *time); /** * Set the time of the timesync clock on an Ethernet device. @@ -4252,21 +4467,7 @@ int rte_eth_timesync_read_time(uint8_t port_id, struct timespec *time); * - -ENODEV: The port ID is invalid. * - -ENOTSUP: The function is not supported by the Ethernet driver. */ -int rte_eth_timesync_write_time(uint8_t port_id, const struct timespec *time); - -/** - * Copy pci device info to the Ethernet device data. - * - * @param eth_dev - * The *eth_dev* pointer is the address of the *rte_eth_dev* structure. - * @param pci_dev - * The *pci_dev* pointer is the address of the *rte_pci_device* structure. - * - * @return - * - 0 on success, negative on error - */ -void rte_eth_copy_pci_info(struct rte_eth_dev *eth_dev, - struct rte_pci_device *pci_dev); +int rte_eth_timesync_write_time(uint16_t port_id, const struct timespec *time); /** * Create memzone for HW rings. @@ -4307,7 +4508,7 @@ rte_eth_dma_zone_reserve(const struct rte_eth_dev *eth_dev, const char *name, * - (-ENOTSUP) if hardware doesn't support tunnel type. */ int -rte_eth_dev_l2_tunnel_eth_type_conf(uint8_t port_id, +rte_eth_dev_l2_tunnel_eth_type_conf(uint16_t port_id, struct rte_eth_l2_tunnel_conf *l2_tunnel); /** @@ -4334,25 +4535,25 @@ rte_eth_dev_l2_tunnel_eth_type_conf(uint8_t port_id, * - (-ENOTSUP) if hardware doesn't support tunnel type. */ int -rte_eth_dev_l2_tunnel_offload_set(uint8_t port_id, +rte_eth_dev_l2_tunnel_offload_set(uint16_t port_id, struct rte_eth_l2_tunnel_conf *l2_tunnel, uint32_t mask, uint8_t en); /** -* Get the port id from pci adrress or device name -* Ex: 0000:2:00.0 or vdev name eth_pcap0 +* Get the port id from pci address or device name +* Ex: 0000:2:00.0 or vdev name net_pcap0 * * @param name * pci address or name of the device * @param port_id * pointer to port identifier of the device * @return -* - (0) if successful. +* - (0) if successful and port_id is filled. * - (-ENODEV or -EINVAL) on failure. */ int -rte_eth_dev_get_port_by_name(const char *name, uint8_t *port_id); +rte_eth_dev_get_port_by_name(const char *name, uint16_t *port_id); /** * Get the device name from port id @@ -4366,7 +4567,45 @@ rte_eth_dev_get_port_by_name(const char *name, uint8_t *port_id); * - (-EINVAL) on failure. */ int -rte_eth_dev_get_name_by_port(uint8_t port_id, char *name); +rte_eth_dev_get_name_by_port(uint16_t port_id, char *name); + +/** + * Check that numbers of Rx and Tx descriptors satisfy descriptors limits from + * the ethernet device information, otherwise adjust them to boundaries. + * + * @param port_id + * The port identifier of the Ethernet device. + * @param nb_rx_desc + * A pointer to a uint16_t where the number of receive + * descriptors stored. + * @param nb_tx_desc + * A pointer to a uint16_t where the number of transmit + * descriptors stored. + * @return + * - (0) if successful. + * - (-ENOTSUP, -ENODEV or -EINVAL) on failure. + */ +int rte_eth_dev_adjust_nb_rx_tx_desc(uint16_t port_id, + uint16_t *nb_rx_desc, + uint16_t *nb_tx_desc); + + +/** + * Test if a port supports specific mempool ops. + * + * @param port_id + * Port identifier of the Ethernet device. + * @param [in] pool + * The name of the pool operations to test. + * @return + * - 0: best mempool ops choice for this port. + * - 1: mempool ops are supported for this port. + * - -ENOTSUP: mempool ops not supported for this port. + * - -ENODEV: Invalid port Identifier. + * - -EINVAL: Pool param is null. + */ +int +rte_eth_dev_pool_ops_supported(uint16_t port_id, const char *pool); #ifdef __cplusplus } diff --git a/dpdk/lib/librte_ether/rte_ethdev_pci.h b/dpdk/lib/librte_ether/rte_ethdev_pci.h new file mode 100644 index 00000000..ad64a169 --- /dev/null +++ b/dpdk/lib/librte_ether/rte_ethdev_pci.h @@ -0,0 +1,196 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Brocade Communications Systems, Inc. + * Author: Jan Blunck + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_ETHDEV_PCI_H_ +#define _RTE_ETHDEV_PCI_H_ + +#include +#include +#include +#include +#include + +/** + * Copy pci device info to the Ethernet device data. + * + * @param eth_dev + * The *eth_dev* pointer is the address of the *rte_eth_dev* structure. + * @param pci_dev + * The *pci_dev* pointer is the address of the *rte_pci_device* structure. + */ +static inline void +rte_eth_copy_pci_info(struct rte_eth_dev *eth_dev, + struct rte_pci_device *pci_dev) +{ + if ((eth_dev == NULL) || (pci_dev == NULL)) { + RTE_PMD_DEBUG_TRACE("NULL pointer eth_dev=%p pci_dev=%p\n", + eth_dev, pci_dev); + return; + } + + eth_dev->intr_handle = &pci_dev->intr_handle; + + eth_dev->data->dev_flags = 0; + if (pci_dev->driver->drv_flags & RTE_PCI_DRV_INTR_LSC) + eth_dev->data->dev_flags |= RTE_ETH_DEV_INTR_LSC; + if (pci_dev->driver->drv_flags & RTE_PCI_DRV_INTR_RMV) + eth_dev->data->dev_flags |= RTE_ETH_DEV_INTR_RMV; + + eth_dev->data->kdrv = pci_dev->kdrv; + eth_dev->data->numa_node = pci_dev->device.numa_node; +} + +/** + * @internal + * Allocates a new ethdev slot for an ethernet device and returns the pointer + * to that slot for the driver to use. + * + * @param dev + * Pointer to the PCI device + * + * @param private_data_size + * Size of private data structure + * + * @return + * A pointer to a rte_eth_dev or NULL if allocation failed. + */ +static inline struct rte_eth_dev * +rte_eth_dev_pci_allocate(struct rte_pci_device *dev, size_t private_data_size) +{ + struct rte_eth_dev *eth_dev; + const char *name; + + if (!dev) + return NULL; + + name = dev->device.name; + + if (rte_eal_process_type() == RTE_PROC_PRIMARY) { + eth_dev = rte_eth_dev_allocate(name); + if (!eth_dev) + return NULL; + + if (private_data_size) { + eth_dev->data->dev_private = rte_zmalloc_socket(name, + private_data_size, RTE_CACHE_LINE_SIZE, + dev->device.numa_node); + if (!eth_dev->data->dev_private) { + rte_eth_dev_release_port(eth_dev); + return NULL; + } + } + } else { + eth_dev = rte_eth_dev_attach_secondary(name); + if (!eth_dev) + return NULL; + } + + eth_dev->device = &dev->device; + rte_eth_copy_pci_info(eth_dev, dev); + return eth_dev; +} + +static inline void +rte_eth_dev_pci_release(struct rte_eth_dev *eth_dev) +{ + /* free ether device */ + rte_eth_dev_release_port(eth_dev); + + if (rte_eal_process_type() == RTE_PROC_PRIMARY) + rte_free(eth_dev->data->dev_private); + + eth_dev->data->dev_private = NULL; + + /* + * Secondary process will check the name to attach. + * Clear this field to avoid attaching a released ports. + */ + eth_dev->data->name[0] = '\0'; + + eth_dev->device = NULL; + eth_dev->intr_handle = NULL; +} + +typedef int (*eth_dev_pci_callback_t)(struct rte_eth_dev *eth_dev); + +/** + * @internal + * Wrapper for use by pci drivers in a .probe function to attach to a ethdev + * interface. + */ +static inline int +rte_eth_dev_pci_generic_probe(struct rte_pci_device *pci_dev, + size_t private_data_size, eth_dev_pci_callback_t dev_init) +{ + struct rte_eth_dev *eth_dev; + int ret; + + eth_dev = rte_eth_dev_pci_allocate(pci_dev, private_data_size); + if (!eth_dev) + return -ENOMEM; + + RTE_FUNC_PTR_OR_ERR_RET(*dev_init, -EINVAL); + ret = dev_init(eth_dev); + if (ret) + rte_eth_dev_pci_release(eth_dev); + + return ret; +} + +/** + * @internal + * Wrapper for use by pci drivers in a .remove function to detach a ethdev + * interface. + */ +static inline int +rte_eth_dev_pci_generic_remove(struct rte_pci_device *pci_dev, + eth_dev_pci_callback_t dev_uninit) +{ + struct rte_eth_dev *eth_dev; + int ret; + + eth_dev = rte_eth_dev_allocated(pci_dev->device.name); + if (!eth_dev) + return -ENODEV; + + if (dev_uninit) { + ret = dev_uninit(eth_dev); + if (ret) + return ret; + } + + rte_eth_dev_pci_release(eth_dev); + return 0; +} + +#endif /* _RTE_ETHDEV_PCI_H_ */ diff --git a/dpdk/lib/librte_ether/rte_ethdev_vdev.h b/dpdk/lib/librte_ether/rte_ethdev_vdev.h new file mode 100644 index 00000000..feb5a3eb --- /dev/null +++ b/dpdk/lib/librte_ether/rte_ethdev_vdev.h @@ -0,0 +1,84 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Brocade Communications Systems, Inc. + * Author: Jan Blunck + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_ETHDEV_VDEV_H_ +#define _RTE_ETHDEV_VDEV_H_ + +#include +#include +#include +#include + +/** + * @internal + * Allocates a new ethdev slot for an ethernet device and returns the pointer + * to that slot for the driver to use. + * + * @param dev + * Pointer to virtual device + * + * @param private_data_size + * Size of private data structure + * + * @return + * A pointer to a rte_eth_dev or NULL if allocation failed. + */ +static inline struct rte_eth_dev * +rte_eth_vdev_allocate(struct rte_vdev_device *dev, size_t private_data_size) +{ + struct rte_eth_dev *eth_dev; + const char *name = rte_vdev_device_name(dev); + + eth_dev = rte_eth_dev_allocate(name); + if (!eth_dev) + return NULL; + + if (private_data_size) { + eth_dev->data->dev_private = rte_zmalloc_socket(name, + private_data_size, RTE_CACHE_LINE_SIZE, + dev->device.numa_node); + if (!eth_dev->data->dev_private) { + rte_eth_dev_release_port(eth_dev); + return NULL; + } + } + + eth_dev->device = &dev->device; + eth_dev->intr_handle = NULL; + + eth_dev->data->kdrv = RTE_KDRV_NONE; + eth_dev->data->numa_node = dev->device.numa_node; + return eth_dev; +} + +#endif /* _RTE_ETHDEV_VDEV_H_ */ diff --git a/dpdk/lib/librte_ether/rte_ethdev_version.map b/dpdk/lib/librte_ether/rte_ethdev_version.map new file mode 100644 index 00000000..e9681ac8 --- /dev/null +++ b/dpdk/lib/librte_ether/rte_ethdev_version.map @@ -0,0 +1,217 @@ +DPDK_2.2 { + global: + + rte_eth_add_rx_callback; + rte_eth_add_tx_callback; + rte_eth_allmulticast_disable; + rte_eth_allmulticast_enable; + rte_eth_allmulticast_get; + rte_eth_dev_allocate; + rte_eth_dev_allocated; + rte_eth_dev_attach; + rte_eth_dev_callback_register; + rte_eth_dev_callback_unregister; + rte_eth_dev_close; + rte_eth_dev_configure; + rte_eth_dev_count; + rte_eth_dev_default_mac_addr_set; + rte_eth_dev_detach; + rte_eth_dev_filter_ctrl; + rte_eth_dev_filter_supported; + rte_eth_dev_flow_ctrl_get; + rte_eth_dev_flow_ctrl_set; + rte_eth_dev_get_dcb_info; + rte_eth_dev_get_eeprom; + rte_eth_dev_get_eeprom_length; + rte_eth_dev_get_mtu; + rte_eth_dev_get_reg_info; + rte_eth_dev_get_vlan_offload; + rte_eth_devices; + rte_eth_dev_info_get; + rte_eth_dev_is_valid_port; + rte_eth_dev_mac_addr_add; + rte_eth_dev_mac_addr_remove; + rte_eth_dev_priority_flow_ctrl_set; + rte_eth_dev_release_port; + rte_eth_dev_rss_hash_conf_get; + rte_eth_dev_rss_hash_update; + rte_eth_dev_rss_reta_query; + rte_eth_dev_rss_reta_update; + rte_eth_dev_rx_intr_ctl; + rte_eth_dev_rx_intr_ctl_q; + rte_eth_dev_rx_intr_disable; + rte_eth_dev_rx_intr_enable; + rte_eth_dev_rx_queue_start; + rte_eth_dev_rx_queue_stop; + rte_eth_dev_set_eeprom; + rte_eth_dev_set_link_down; + rte_eth_dev_set_link_up; + rte_eth_dev_set_mc_addr_list; + rte_eth_dev_set_mtu; + rte_eth_dev_set_rx_queue_stats_mapping; + rte_eth_dev_set_tx_queue_stats_mapping; + rte_eth_dev_set_vlan_offload; + rte_eth_dev_set_vlan_pvid; + rte_eth_dev_set_vlan_strip_on_queue; + rte_eth_dev_socket_id; + rte_eth_dev_start; + rte_eth_dev_stop; + rte_eth_dev_tx_queue_start; + rte_eth_dev_tx_queue_stop; + rte_eth_dev_uc_all_hash_table_set; + rte_eth_dev_uc_hash_table_set; + rte_eth_dev_vlan_filter; + rte_eth_dma_zone_reserve; + rte_eth_led_off; + rte_eth_led_on; + rte_eth_link; + rte_eth_link_get; + rte_eth_link_get_nowait; + rte_eth_macaddr_get; + rte_eth_mirror_rule_reset; + rte_eth_mirror_rule_set; + rte_eth_promiscuous_disable; + rte_eth_promiscuous_enable; + rte_eth_promiscuous_get; + rte_eth_remove_rx_callback; + rte_eth_remove_tx_callback; + rte_eth_rx_queue_info_get; + rte_eth_rx_queue_setup; + rte_eth_set_queue_rate_limit; + rte_eth_stats; + rte_eth_stats_get; + rte_eth_stats_reset; + rte_eth_timesync_adjust_time; + rte_eth_timesync_disable; + rte_eth_timesync_enable; + rte_eth_timesync_read_rx_timestamp; + rte_eth_timesync_read_time; + rte_eth_timesync_read_tx_timestamp; + rte_eth_timesync_write_time; + rte_eth_tx_queue_info_get; + rte_eth_tx_queue_setup; + rte_eth_xstats_get; + rte_eth_xstats_reset; + + local: *; +}; + +DPDK_16.04 { + global: + + rte_eth_dev_get_supported_ptypes; + rte_eth_dev_l2_tunnel_eth_type_conf; + rte_eth_dev_l2_tunnel_offload_set; + rte_eth_dev_set_vlan_ether_type; + rte_eth_dev_udp_tunnel_port_add; + rte_eth_dev_udp_tunnel_port_delete; + rte_eth_speed_bitflag; + rte_eth_tx_buffer_count_callback; + rte_eth_tx_buffer_drop_callback; + rte_eth_tx_buffer_init; + rte_eth_tx_buffer_set_err_callback; + +} DPDK_2.2; + +DPDK_16.07 { + global: + + rte_eth_add_first_rx_callback; + rte_eth_dev_get_name_by_port; + rte_eth_dev_get_port_by_name; + rte_eth_xstats_get_names; + +} DPDK_16.04; + +DPDK_17.02 { + global: + + _rte_eth_dev_reset; + rte_eth_dev_fw_version_get; + rte_flow_create; + rte_flow_destroy; + rte_flow_flush; + rte_flow_query; + rte_flow_validate; + +} DPDK_16.07; + +DPDK_17.05 { + global: + + rte_eth_dev_attach_secondary; + rte_eth_find_next; + rte_eth_tx_done_cleanup; + rte_eth_xstats_get_by_id; + rte_eth_xstats_get_id_by_name; + rte_eth_xstats_get_names_by_id; + +} DPDK_17.02; + +DPDK_17.08 { + global: + + _rte_eth_dev_callback_process; + rte_eth_dev_adjust_nb_rx_tx_desc; + rte_flow_copy; + rte_flow_isolate; + rte_tm_capabilities_get; + rte_tm_get_leaf_nodes; + rte_tm_hierarchy_commit; + rte_tm_level_capabilities_get; + rte_tm_mark_ip_dscp; + rte_tm_mark_ip_ecn; + rte_tm_mark_vlan_dei; + rte_tm_node_add; + rte_tm_node_capabilities_get; + rte_tm_node_cman_update; + rte_tm_node_delete; + rte_tm_node_parent_update; + rte_tm_node_resume; + rte_tm_node_shaper_update; + rte_tm_node_shared_shaper_update; + rte_tm_node_shared_wred_context_update; + rte_tm_node_stats_read; + rte_tm_node_stats_update; + rte_tm_node_suspend; + rte_tm_node_type_get; + rte_tm_node_wfq_weight_mode_update; + rte_tm_node_wred_context_update; + rte_tm_shaper_profile_add; + rte_tm_shaper_profile_delete; + rte_tm_shared_shaper_add_update; + rte_tm_shared_shaper_delete; + rte_tm_shared_wred_context_add_update; + rte_tm_shared_wred_context_delete; + rte_tm_wred_profile_add; + rte_tm_wred_profile_delete; + +} DPDK_17.05; + +DPDK_17.11 { + global: + + rte_eth_dev_get_sec_ctx; + rte_eth_dev_pool_ops_supported; + rte_eth_dev_reset; + rte_flow_error_set; + +} DPDK_17.08; + +EXPERIMENTAL { + global: + + rte_mtr_capabilities_get; + rte_mtr_create; + rte_mtr_destroy; + rte_mtr_meter_disable; + rte_mtr_meter_dscp_table_update; + rte_mtr_meter_enable; + rte_mtr_meter_profile_add; + rte_mtr_meter_profile_delete; + rte_mtr_meter_profile_update; + rte_mtr_policer_actions_update; + rte_mtr_stats_read; + rte_mtr_stats_update; + +} DPDK_17.11; diff --git a/dpdk/lib/librte_ether/rte_ether.h b/dpdk/lib/librte_ether/rte_ether.h deleted file mode 100644 index 1d62d8e5..00000000 --- a/dpdk/lib/librte_ether/rte_ether.h +++ /dev/null @@ -1,416 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef _RTE_ETHER_H_ -#define _RTE_ETHER_H_ - -/** - * @file - * - * Ethernet Helpers in RTE - */ - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include - -#include -#include -#include -#include - -#define ETHER_ADDR_LEN 6 /**< Length of Ethernet address. */ -#define ETHER_TYPE_LEN 2 /**< Length of Ethernet type field. */ -#define ETHER_CRC_LEN 4 /**< Length of Ethernet CRC. */ -#define ETHER_HDR_LEN \ - (ETHER_ADDR_LEN * 2 + ETHER_TYPE_LEN) /**< Length of Ethernet header. */ -#define ETHER_MIN_LEN 64 /**< Minimum frame len, including CRC. */ -#define ETHER_MAX_LEN 1518 /**< Maximum frame len, including CRC. */ -#define ETHER_MTU \ - (ETHER_MAX_LEN - ETHER_HDR_LEN - ETHER_CRC_LEN) /**< Ethernet MTU. */ - -#define ETHER_MAX_VLAN_FRAME_LEN \ - (ETHER_MAX_LEN + 4) /**< Maximum VLAN frame length, including CRC. */ - -#define ETHER_MAX_JUMBO_FRAME_LEN \ - 0x3F00 /**< Maximum Jumbo frame length, including CRC. */ - -#define ETHER_MAX_VLAN_ID 4095 /**< Maximum VLAN ID. */ - -#define ETHER_MIN_MTU 68 /**< Minimum MTU for IPv4 packets, see RFC 791. */ - -/** - * Ethernet address: - * A universally administered address is uniquely assigned to a device by its - * manufacturer. The first three octets (in transmission order) contain the - * Organizationally Unique Identifier (OUI). The following three (MAC-48 and - * EUI-48) octets are assigned by that organization with the only constraint - * of uniqueness. - * A locally administered address is assigned to a device by a network - * administrator and does not contain OUIs. - * See http://standards.ieee.org/regauth/groupmac/tutorial.html - */ -struct ether_addr { - uint8_t addr_bytes[ETHER_ADDR_LEN]; /**< Address bytes in transmission order */ -} __attribute__((__packed__)); - -#define ETHER_LOCAL_ADMIN_ADDR 0x02 /**< Locally assigned Eth. address. */ -#define ETHER_GROUP_ADDR 0x01 /**< Multicast or broadcast Eth. address. */ - -/** - * Check if two Ethernet addresses are the same. - * - * @param ea1 - * A pointer to the first ether_addr structure containing - * the ethernet address. - * @param ea2 - * A pointer to the second ether_addr structure containing - * the ethernet address. - * - * @return - * True (1) if the given two ethernet address are the same; - * False (0) otherwise. - */ -static inline int is_same_ether_addr(const struct ether_addr *ea1, - const struct ether_addr *ea2) -{ - int i; - for (i = 0; i < ETHER_ADDR_LEN; i++) - if (ea1->addr_bytes[i] != ea2->addr_bytes[i]) - return 0; - return 1; -} - -/** - * Check if an Ethernet address is filled with zeros. - * - * @param ea - * A pointer to a ether_addr structure containing the ethernet address - * to check. - * @return - * True (1) if the given ethernet address is filled with zeros; - * false (0) otherwise. - */ -static inline int is_zero_ether_addr(const struct ether_addr *ea) -{ - int i; - for (i = 0; i < ETHER_ADDR_LEN; i++) - if (ea->addr_bytes[i] != 0x00) - return 0; - return 1; -} - -/** - * Check if an Ethernet address is a unicast address. - * - * @param ea - * A pointer to a ether_addr structure containing the ethernet address - * to check. - * @return - * True (1) if the given ethernet address is a unicast address; - * false (0) otherwise. - */ -static inline int is_unicast_ether_addr(const struct ether_addr *ea) -{ - return (ea->addr_bytes[0] & ETHER_GROUP_ADDR) == 0; -} - -/** - * Check if an Ethernet address is a multicast address. - * - * @param ea - * A pointer to a ether_addr structure containing the ethernet address - * to check. - * @return - * True (1) if the given ethernet address is a multicast address; - * false (0) otherwise. - */ -static inline int is_multicast_ether_addr(const struct ether_addr *ea) -{ - return ea->addr_bytes[0] & ETHER_GROUP_ADDR; -} - -/** - * Check if an Ethernet address is a broadcast address. - * - * @param ea - * A pointer to a ether_addr structure containing the ethernet address - * to check. - * @return - * True (1) if the given ethernet address is a broadcast address; - * false (0) otherwise. - */ -static inline int is_broadcast_ether_addr(const struct ether_addr *ea) -{ - const unaligned_uint16_t *ea_words = (const unaligned_uint16_t *)ea; - - return (ea_words[0] == 0xFFFF && ea_words[1] == 0xFFFF && - ea_words[2] == 0xFFFF); -} - -/** - * Check if an Ethernet address is a universally assigned address. - * - * @param ea - * A pointer to a ether_addr structure containing the ethernet address - * to check. - * @return - * True (1) if the given ethernet address is a universally assigned address; - * false (0) otherwise. - */ -static inline int is_universal_ether_addr(const struct ether_addr *ea) -{ - return (ea->addr_bytes[0] & ETHER_LOCAL_ADMIN_ADDR) == 0; -} - -/** - * Check if an Ethernet address is a locally assigned address. - * - * @param ea - * A pointer to a ether_addr structure containing the ethernet address - * to check. - * @return - * True (1) if the given ethernet address is a locally assigned address; - * false (0) otherwise. - */ -static inline int is_local_admin_ether_addr(const struct ether_addr *ea) -{ - return (ea->addr_bytes[0] & ETHER_LOCAL_ADMIN_ADDR) != 0; -} - -/** - * Check if an Ethernet address is a valid address. Checks that the address is a - * unicast address and is not filled with zeros. - * - * @param ea - * A pointer to a ether_addr structure containing the ethernet address - * to check. - * @return - * True (1) if the given ethernet address is valid; - * false (0) otherwise. - */ -static inline int is_valid_assigned_ether_addr(const struct ether_addr *ea) -{ - return is_unicast_ether_addr(ea) && (! is_zero_ether_addr(ea)); -} - -/** - * Generate a random Ethernet address that is locally administered - * and not multicast. - * @param addr - * A pointer to Ethernet address. - */ -static inline void eth_random_addr(uint8_t *addr) -{ - uint64_t rand = rte_rand(); - uint8_t *p = (uint8_t*)&rand; - - rte_memcpy(addr, p, ETHER_ADDR_LEN); - addr[0] &= ~ETHER_GROUP_ADDR; /* clear multicast bit */ - addr[0] |= ETHER_LOCAL_ADMIN_ADDR; /* set local assignment bit */ -} - -/** - * Fast copy an Ethernet address. - * - * @param ea_from - * A pointer to a ether_addr structure holding the Ethernet address to copy. - * @param ea_to - * A pointer to a ether_addr structure where to copy the Ethernet address. - */ -static inline void ether_addr_copy(const struct ether_addr *ea_from, - struct ether_addr *ea_to) -{ -#ifdef __INTEL_COMPILER - uint16_t *from_words = (uint16_t *)(ea_from->addr_bytes); - uint16_t *to_words = (uint16_t *)(ea_to->addr_bytes); - - to_words[0] = from_words[0]; - to_words[1] = from_words[1]; - to_words[2] = from_words[2]; -#else - /* - * Use the common way, because of a strange gcc warning. - */ - *ea_to = *ea_from; -#endif -} - -#define ETHER_ADDR_FMT_SIZE 18 -/** - * Format 48bits Ethernet address in pattern xx:xx:xx:xx:xx:xx. - * - * @param buf - * A pointer to buffer contains the formatted MAC address. - * @param size - * The format buffer size. - * @param eth_addr - * A pointer to a ether_addr structure. - */ -static inline void -ether_format_addr(char *buf, uint16_t size, - const struct ether_addr *eth_addr) -{ - snprintf(buf, size, "%02X:%02X:%02X:%02X:%02X:%02X", - eth_addr->addr_bytes[0], - eth_addr->addr_bytes[1], - eth_addr->addr_bytes[2], - eth_addr->addr_bytes[3], - eth_addr->addr_bytes[4], - eth_addr->addr_bytes[5]); -} - -/** - * Ethernet header: Contains the destination address, source address - * and frame type. - */ -struct ether_hdr { - struct ether_addr d_addr; /**< Destination address. */ - struct ether_addr s_addr; /**< Source address. */ - uint16_t ether_type; /**< Frame type. */ -} __attribute__((__packed__)); - -/** - * Ethernet VLAN Header. - * Contains the 16-bit VLAN Tag Control Identifier and the Ethernet type - * of the encapsulated frame. - */ -struct vlan_hdr { - uint16_t vlan_tci; /**< Priority (3) + CFI (1) + Identifier Code (12) */ - uint16_t eth_proto;/**< Ethernet type of encapsulated frame. */ -} __attribute__((__packed__)); - -/** - * VXLAN protocol header. - * Contains the 8-bit flag, 24-bit VXLAN Network Identifier and - * Reserved fields (24 bits and 8 bits) - */ -struct vxlan_hdr { - uint32_t vx_flags; /**< flag (8) + Reserved (24). */ - uint32_t vx_vni; /**< VNI (24) + Reserved (8). */ -} __attribute__((__packed__)); - -/* Ethernet frame types */ -#define ETHER_TYPE_IPv4 0x0800 /**< IPv4 Protocol. */ -#define ETHER_TYPE_IPv6 0x86DD /**< IPv6 Protocol. */ -#define ETHER_TYPE_ARP 0x0806 /**< Arp Protocol. */ -#define ETHER_TYPE_RARP 0x8035 /**< Reverse Arp Protocol. */ -#define ETHER_TYPE_VLAN 0x8100 /**< IEEE 802.1Q VLAN tagging. */ -#define ETHER_TYPE_1588 0x88F7 /**< IEEE 802.1AS 1588 Precise Time Protocol. */ -#define ETHER_TYPE_SLOW 0x8809 /**< Slow protocols (LACP and Marker). */ -#define ETHER_TYPE_TEB 0x6558 /**< Transparent Ethernet Bridging. */ - -#define ETHER_VXLAN_HLEN (sizeof(struct udp_hdr) + sizeof(struct vxlan_hdr)) -/**< VXLAN tunnel header length. */ - -/** - * Extract VLAN tag information into mbuf - * - * Software version of VLAN stripping - * - * @param m - * The packet mbuf. - * @return - * - 0: Success - * - 1: not a vlan packet - */ -static inline int rte_vlan_strip(struct rte_mbuf *m) -{ - struct ether_hdr *eh - = rte_pktmbuf_mtod(m, struct ether_hdr *); - - if (eh->ether_type != rte_cpu_to_be_16(ETHER_TYPE_VLAN)) - return -1; - - struct vlan_hdr *vh = (struct vlan_hdr *)(eh + 1); - m->ol_flags |= PKT_RX_VLAN_PKT; - m->vlan_tci = rte_be_to_cpu_16(vh->vlan_tci); - - /* Copy ether header over rather than moving whole packet */ - memmove(rte_pktmbuf_adj(m, sizeof(struct vlan_hdr)), - eh, 2 * ETHER_ADDR_LEN); - - return 0; -} - -/** - * Insert VLAN tag into mbuf. - * - * Software version of VLAN unstripping - * - * @param m - * The packet mbuf. - * @return - * - 0: On success - * -EPERM: mbuf is is shared overwriting would be unsafe - * -ENOSPC: not enough headroom in mbuf - */ -static inline int rte_vlan_insert(struct rte_mbuf **m) -{ - struct ether_hdr *oh, *nh; - struct vlan_hdr *vh; - - /* Can't insert header if mbuf is shared */ - if (rte_mbuf_refcnt_read(*m) > 1) { - struct rte_mbuf *copy; - - copy = rte_pktmbuf_clone(*m, (*m)->pool); - if (unlikely(copy == NULL)) - return -ENOMEM; - rte_pktmbuf_free(*m); - *m = copy; - } - - oh = rte_pktmbuf_mtod(*m, struct ether_hdr *); - nh = (struct ether_hdr *) - rte_pktmbuf_prepend(*m, sizeof(struct vlan_hdr)); - if (nh == NULL) - return -ENOSPC; - - memmove(nh, oh, 2 * ETHER_ADDR_LEN); - nh->ether_type = rte_cpu_to_be_16(ETHER_TYPE_VLAN); - - vh = (struct vlan_hdr *) (nh + 1); - vh->vlan_tci = rte_cpu_to_be_16((*m)->vlan_tci); - - return 0; -} - -#ifdef __cplusplus -} -#endif - -#endif /* _RTE_ETHER_H_ */ diff --git a/dpdk/lib/librte_ether/rte_ether_version.map b/dpdk/lib/librte_ether/rte_ether_version.map deleted file mode 100644 index 45ddf44c..00000000 --- a/dpdk/lib/librte_ether/rte_ether_version.map +++ /dev/null @@ -1,141 +0,0 @@ -DPDK_2.2 { - global: - - _rte_eth_dev_callback_process; - rte_eth_add_rx_callback; - rte_eth_add_tx_callback; - rte_eth_allmulticast_disable; - rte_eth_allmulticast_enable; - rte_eth_allmulticast_get; - rte_eth_copy_pci_info; - rte_eth_dev_allocate; - rte_eth_dev_allocated; - rte_eth_dev_attach; - rte_eth_dev_bypass_event_show; - rte_eth_dev_bypass_event_store; - rte_eth_dev_bypass_init; - rte_eth_dev_bypass_state_set; - rte_eth_dev_bypass_state_show; - rte_eth_dev_bypass_ver_show; - rte_eth_dev_bypass_wd_reset; - rte_eth_dev_bypass_wd_timeout_show; - rte_eth_dev_callback_process; - rte_eth_dev_callback_register; - rte_eth_dev_callback_unregister; - rte_eth_dev_close; - rte_eth_dev_configure; - rte_eth_dev_count; - rte_eth_dev_default_mac_addr_set; - rte_eth_dev_detach; - rte_eth_dev_filter_ctrl; - rte_eth_dev_filter_supported; - rte_eth_dev_flow_ctrl_get; - rte_eth_dev_flow_ctrl_set; - rte_eth_dev_get_dcb_info; - rte_eth_dev_get_eeprom; - rte_eth_dev_get_eeprom_length; - rte_eth_dev_get_mtu; - rte_eth_dev_get_reg_info; - rte_eth_dev_get_vlan_offload; - rte_eth_devices; - rte_eth_dev_info_get; - rte_eth_dev_is_valid_port; - rte_eth_dev_mac_addr_add; - rte_eth_dev_mac_addr_remove; - rte_eth_dev_priority_flow_ctrl_set; - rte_eth_dev_release_port; - rte_eth_dev_rss_hash_conf_get; - rte_eth_dev_rss_hash_update; - rte_eth_dev_rss_reta_query; - rte_eth_dev_rss_reta_update; - rte_eth_dev_rx_intr_ctl; - rte_eth_dev_rx_intr_ctl_q; - rte_eth_dev_rx_intr_disable; - rte_eth_dev_rx_intr_enable; - rte_eth_dev_rx_queue_start; - rte_eth_dev_rx_queue_stop; - rte_eth_dev_set_eeprom; - rte_eth_dev_set_link_down; - rte_eth_dev_set_link_up; - rte_eth_dev_set_mc_addr_list; - rte_eth_dev_set_mtu; - rte_eth_dev_set_rx_queue_stats_mapping; - rte_eth_dev_set_tx_queue_stats_mapping; - rte_eth_dev_set_vf_rx; - rte_eth_dev_set_vf_rxmode; - rte_eth_dev_set_vf_tx; - rte_eth_dev_set_vf_vlan_filter; - rte_eth_dev_set_vlan_offload; - rte_eth_dev_set_vlan_pvid; - rte_eth_dev_set_vlan_strip_on_queue; - rte_eth_dev_socket_id; - rte_eth_dev_start; - rte_eth_dev_stop; - rte_eth_dev_tx_queue_start; - rte_eth_dev_tx_queue_stop; - rte_eth_dev_uc_all_hash_table_set; - rte_eth_dev_uc_hash_table_set; - rte_eth_dev_vlan_filter; - rte_eth_dev_wd_timeout_store; - rte_eth_dma_zone_reserve; - rte_eth_driver_register; - rte_eth_led_off; - rte_eth_led_on; - rte_eth_link; - rte_eth_link_get; - rte_eth_link_get_nowait; - rte_eth_macaddr_get; - rte_eth_mirror_rule_reset; - rte_eth_mirror_rule_set; - rte_eth_promiscuous_disable; - rte_eth_promiscuous_enable; - rte_eth_promiscuous_get; - rte_eth_remove_rx_callback; - rte_eth_remove_tx_callback; - rte_eth_rx_queue_info_get; - rte_eth_rx_queue_setup; - rte_eth_set_queue_rate_limit; - rte_eth_set_vf_rate_limit; - rte_eth_stats; - rte_eth_stats_get; - rte_eth_stats_reset; - rte_eth_timesync_adjust_time; - rte_eth_timesync_disable; - rte_eth_timesync_enable; - rte_eth_timesync_read_rx_timestamp; - rte_eth_timesync_read_time; - rte_eth_timesync_read_tx_timestamp; - rte_eth_timesync_write_time; - rte_eth_tx_queue_info_get; - rte_eth_tx_queue_setup; - rte_eth_xstats_get; - rte_eth_xstats_reset; - - local: *; -}; - -DPDK_16.04 { - global: - - rte_eth_dev_get_supported_ptypes; - rte_eth_dev_l2_tunnel_eth_type_conf; - rte_eth_dev_l2_tunnel_offload_set; - rte_eth_dev_set_vlan_ether_type; - rte_eth_dev_udp_tunnel_port_add; - rte_eth_dev_udp_tunnel_port_delete; - rte_eth_speed_bitflag; - rte_eth_tx_buffer_count_callback; - rte_eth_tx_buffer_drop_callback; - rte_eth_tx_buffer_init; - rte_eth_tx_buffer_set_err_callback; - -} DPDK_2.2; - -DPDK_16.07 { - global: - - rte_eth_add_first_rx_callback; - rte_eth_dev_get_name_by_port; - rte_eth_dev_get_port_by_name; - rte_eth_xstats_get_names; -} DPDK_16.04; diff --git a/dpdk/lib/librte_ether/rte_flow.c b/dpdk/lib/librte_ether/rte_flow.c new file mode 100644 index 00000000..66590630 --- /dev/null +++ b/dpdk/lib/librte_ether/rte_flow.c @@ -0,0 +1,427 @@ +/*- + * BSD LICENSE + * + * Copyright 2016 6WIND S.A. + * Copyright 2016 Mellanox. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of 6WIND S.A. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include + +#include +#include +#include +#include "rte_ethdev.h" +#include "rte_flow_driver.h" +#include "rte_flow.h" + +/** + * Flow elements description tables. + */ +struct rte_flow_desc_data { + const char *name; + size_t size; +}; + +/** Generate flow_item[] entry. */ +#define MK_FLOW_ITEM(t, s) \ + [RTE_FLOW_ITEM_TYPE_ ## t] = { \ + .name = # t, \ + .size = s, \ + } + +/** Information about known flow pattern items. */ +static const struct rte_flow_desc_data rte_flow_desc_item[] = { + MK_FLOW_ITEM(END, 0), + MK_FLOW_ITEM(VOID, 0), + MK_FLOW_ITEM(INVERT, 0), + MK_FLOW_ITEM(ANY, sizeof(struct rte_flow_item_any)), + MK_FLOW_ITEM(PF, 0), + MK_FLOW_ITEM(VF, sizeof(struct rte_flow_item_vf)), + MK_FLOW_ITEM(PORT, sizeof(struct rte_flow_item_port)), + MK_FLOW_ITEM(RAW, sizeof(struct rte_flow_item_raw)), /* +pattern[] */ + MK_FLOW_ITEM(ETH, sizeof(struct rte_flow_item_eth)), + MK_FLOW_ITEM(VLAN, sizeof(struct rte_flow_item_vlan)), + MK_FLOW_ITEM(IPV4, sizeof(struct rte_flow_item_ipv4)), + MK_FLOW_ITEM(IPV6, sizeof(struct rte_flow_item_ipv6)), + MK_FLOW_ITEM(ICMP, sizeof(struct rte_flow_item_icmp)), + MK_FLOW_ITEM(UDP, sizeof(struct rte_flow_item_udp)), + MK_FLOW_ITEM(TCP, sizeof(struct rte_flow_item_tcp)), + MK_FLOW_ITEM(SCTP, sizeof(struct rte_flow_item_sctp)), + MK_FLOW_ITEM(VXLAN, sizeof(struct rte_flow_item_vxlan)), + MK_FLOW_ITEM(MPLS, sizeof(struct rte_flow_item_mpls)), + MK_FLOW_ITEM(GRE, sizeof(struct rte_flow_item_gre)), + MK_FLOW_ITEM(E_TAG, sizeof(struct rte_flow_item_e_tag)), + MK_FLOW_ITEM(NVGRE, sizeof(struct rte_flow_item_nvgre)), +}; + +/** Generate flow_action[] entry. */ +#define MK_FLOW_ACTION(t, s) \ + [RTE_FLOW_ACTION_TYPE_ ## t] = { \ + .name = # t, \ + .size = s, \ + } + +/** Information about known flow actions. */ +static const struct rte_flow_desc_data rte_flow_desc_action[] = { + MK_FLOW_ACTION(END, 0), + MK_FLOW_ACTION(VOID, 0), + MK_FLOW_ACTION(PASSTHRU, 0), + MK_FLOW_ACTION(MARK, sizeof(struct rte_flow_action_mark)), + MK_FLOW_ACTION(FLAG, 0), + MK_FLOW_ACTION(QUEUE, sizeof(struct rte_flow_action_queue)), + MK_FLOW_ACTION(DROP, 0), + MK_FLOW_ACTION(COUNT, 0), + MK_FLOW_ACTION(DUP, sizeof(struct rte_flow_action_dup)), + MK_FLOW_ACTION(RSS, sizeof(struct rte_flow_action_rss)), /* +queue[] */ + MK_FLOW_ACTION(PF, 0), + MK_FLOW_ACTION(VF, sizeof(struct rte_flow_action_vf)), +}; + +/* Get generic flow operations structure from a port. */ +const struct rte_flow_ops * +rte_flow_ops_get(uint16_t port_id, struct rte_flow_error *error) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + const struct rte_flow_ops *ops; + int code; + + if (unlikely(!rte_eth_dev_is_valid_port(port_id))) + code = ENODEV; + else if (unlikely(!dev->dev_ops->filter_ctrl || + dev->dev_ops->filter_ctrl(dev, + RTE_ETH_FILTER_GENERIC, + RTE_ETH_FILTER_GET, + &ops) || + !ops)) + code = ENOSYS; + else + return ops; + rte_flow_error_set(error, code, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, rte_strerror(code)); + return NULL; +} + +/* Check whether a flow rule can be created on a given port. */ +int +rte_flow_validate(uint16_t port_id, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error) +{ + const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error); + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + + if (unlikely(!ops)) + return -rte_errno; + if (likely(!!ops->validate)) + return ops->validate(dev, attr, pattern, actions, error); + return rte_flow_error_set(error, ENOSYS, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, rte_strerror(ENOSYS)); +} + +/* Create a flow rule on a given port. */ +struct rte_flow * +rte_flow_create(uint16_t port_id, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error); + + if (unlikely(!ops)) + return NULL; + if (likely(!!ops->create)) + return ops->create(dev, attr, pattern, actions, error); + rte_flow_error_set(error, ENOSYS, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, rte_strerror(ENOSYS)); + return NULL; +} + +/* Destroy a flow rule on a given port. */ +int +rte_flow_destroy(uint16_t port_id, + struct rte_flow *flow, + struct rte_flow_error *error) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error); + + if (unlikely(!ops)) + return -rte_errno; + if (likely(!!ops->destroy)) + return ops->destroy(dev, flow, error); + return rte_flow_error_set(error, ENOSYS, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, rte_strerror(ENOSYS)); +} + +/* Destroy all flow rules associated with a port. */ +int +rte_flow_flush(uint16_t port_id, + struct rte_flow_error *error) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error); + + if (unlikely(!ops)) + return -rte_errno; + if (likely(!!ops->flush)) + return ops->flush(dev, error); + return rte_flow_error_set(error, ENOSYS, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, rte_strerror(ENOSYS)); +} + +/* Query an existing flow rule. */ +int +rte_flow_query(uint16_t port_id, + struct rte_flow *flow, + enum rte_flow_action_type action, + void *data, + struct rte_flow_error *error) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error); + + if (!ops) + return -rte_errno; + if (likely(!!ops->query)) + return ops->query(dev, flow, action, data, error); + return rte_flow_error_set(error, ENOSYS, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, rte_strerror(ENOSYS)); +} + +/* Restrict ingress traffic to the defined flow rules. */ +int +rte_flow_isolate(uint16_t port_id, + int set, + struct rte_flow_error *error) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error); + + if (!ops) + return -rte_errno; + if (likely(!!ops->isolate)) + return ops->isolate(dev, set, error); + return rte_flow_error_set(error, ENOSYS, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, rte_strerror(ENOSYS)); +} + +/* Initialize flow error structure. */ +int +rte_flow_error_set(struct rte_flow_error *error, + int code, + enum rte_flow_error_type type, + const void *cause, + const char *message) +{ + if (error) { + *error = (struct rte_flow_error){ + .type = type, + .cause = cause, + .message = message, + }; + } + rte_errno = code; + return -code; +} + +/** Compute storage space needed by item specification. */ +static void +flow_item_spec_size(const struct rte_flow_item *item, + size_t *size, size_t *pad) +{ + if (!item->spec) { + *size = 0; + goto empty; + } + switch (item->type) { + union { + const struct rte_flow_item_raw *raw; + } spec; + + /* Not a fall-through */ + case RTE_FLOW_ITEM_TYPE_RAW: + spec.raw = item->spec; + *size = offsetof(struct rte_flow_item_raw, pattern) + + spec.raw->length * sizeof(*spec.raw->pattern); + break; + default: + *size = rte_flow_desc_item[item->type].size; + break; + } +empty: + *pad = RTE_ALIGN_CEIL(*size, sizeof(double)) - *size; +} + +/** Compute storage space needed by action configuration. */ +static void +flow_action_conf_size(const struct rte_flow_action *action, + size_t *size, size_t *pad) +{ + if (!action->conf) { + *size = 0; + goto empty; + } + switch (action->type) { + union { + const struct rte_flow_action_rss *rss; + } conf; + + /* Not a fall-through. */ + case RTE_FLOW_ACTION_TYPE_RSS: + conf.rss = action->conf; + *size = offsetof(struct rte_flow_action_rss, queue) + + conf.rss->num * sizeof(*conf.rss->queue); + break; + default: + *size = rte_flow_desc_action[action->type].size; + break; + } +empty: + *pad = RTE_ALIGN_CEIL(*size, sizeof(double)) - *size; +} + +/** Store a full rte_flow description. */ +size_t +rte_flow_copy(struct rte_flow_desc *desc, size_t len, + const struct rte_flow_attr *attr, + const struct rte_flow_item *items, + const struct rte_flow_action *actions) +{ + struct rte_flow_desc *fd = NULL; + size_t tmp; + size_t pad; + size_t off1 = 0; + size_t off2 = 0; + size_t size = 0; + +store: + if (items) { + const struct rte_flow_item *item; + + item = items; + if (fd) + fd->items = (void *)&fd->data[off1]; + do { + struct rte_flow_item *dst = NULL; + + if ((size_t)item->type >= + RTE_DIM(rte_flow_desc_item) || + !rte_flow_desc_item[item->type].name) { + rte_errno = ENOTSUP; + return 0; + } + if (fd) + dst = memcpy(fd->data + off1, item, + sizeof(*item)); + off1 += sizeof(*item); + flow_item_spec_size(item, &tmp, &pad); + if (item->spec) { + if (fd) + dst->spec = memcpy(fd->data + off2, + item->spec, tmp); + off2 += tmp + pad; + } + if (item->last) { + if (fd) + dst->last = memcpy(fd->data + off2, + item->last, tmp); + off2 += tmp + pad; + } + if (item->mask) { + if (fd) + dst->mask = memcpy(fd->data + off2, + item->mask, tmp); + off2 += tmp + pad; + } + off2 = RTE_ALIGN_CEIL(off2, sizeof(double)); + } while ((item++)->type != RTE_FLOW_ITEM_TYPE_END); + off1 = RTE_ALIGN_CEIL(off1, sizeof(double)); + } + if (actions) { + const struct rte_flow_action *action; + + action = actions; + if (fd) + fd->actions = (void *)&fd->data[off1]; + do { + struct rte_flow_action *dst = NULL; + + if ((size_t)action->type >= + RTE_DIM(rte_flow_desc_action) || + !rte_flow_desc_action[action->type].name) { + rte_errno = ENOTSUP; + return 0; + } + if (fd) + dst = memcpy(fd->data + off1, action, + sizeof(*action)); + off1 += sizeof(*action); + flow_action_conf_size(action, &tmp, &pad); + if (action->conf) { + if (fd) + dst->conf = memcpy(fd->data + off2, + action->conf, tmp); + off2 += tmp + pad; + } + off2 = RTE_ALIGN_CEIL(off2, sizeof(double)); + } while ((action++)->type != RTE_FLOW_ACTION_TYPE_END); + } + if (fd != NULL) + return size; + off1 = RTE_ALIGN_CEIL(off1, sizeof(double)); + tmp = RTE_ALIGN_CEIL(offsetof(struct rte_flow_desc, data), + sizeof(double)); + size = tmp + off1 + off2; + if (size > len) + return size; + fd = desc; + if (fd != NULL) { + *fd = (const struct rte_flow_desc) { + .size = size, + .attr = *attr, + }; + tmp -= offsetof(struct rte_flow_desc, data); + off2 = tmp + off1; + off1 = tmp; + goto store; + } + return 0; +} diff --git a/dpdk/lib/librte_ether/rte_flow.h b/dpdk/lib/librte_ether/rte_flow.h new file mode 100644 index 00000000..47c88ea5 --- /dev/null +++ b/dpdk/lib/librte_ether/rte_flow.h @@ -0,0 +1,1479 @@ +/*- + * BSD LICENSE + * + * Copyright 2016 6WIND S.A. + * Copyright 2016 Mellanox. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of 6WIND S.A. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef RTE_FLOW_H_ +#define RTE_FLOW_H_ + +/** + * @file + * RTE generic flow API + * + * This interface provides the ability to program packet matching and + * associated actions in hardware through flow rules. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Flow rule attributes. + * + * Priorities are set on two levels: per group and per rule within groups. + * + * Lower values denote higher priority, the highest priority for both levels + * is 0, so that a rule with priority 0 in group 8 is always matched after a + * rule with priority 8 in group 0. + * + * Although optional, applications are encouraged to group similar rules as + * much as possible to fully take advantage of hardware capabilities + * (e.g. optimized matching) and work around limitations (e.g. a single + * pattern type possibly allowed in a given group). + * + * Group and priority levels are arbitrary and up to the application, they + * do not need to be contiguous nor start from 0, however the maximum number + * varies between devices and may be affected by existing flow rules. + * + * If a packet is matched by several rules of a given group for a given + * priority level, the outcome is undefined. It can take any path, may be + * duplicated or even cause unrecoverable errors. + * + * Note that support for more than a single group and priority level is not + * guaranteed. + * + * Flow rules can apply to inbound and/or outbound traffic (ingress/egress). + * + * Several pattern items and actions are valid and can be used in both + * directions. Those valid for only one direction are described as such. + * + * At least one direction must be specified. + * + * Specifying both directions at once for a given rule is not recommended + * but may be valid in a few cases (e.g. shared counter). + */ +struct rte_flow_attr { + uint32_t group; /**< Priority group. */ + uint32_t priority; /**< Priority level within group. */ + uint32_t ingress:1; /**< Rule applies to ingress traffic. */ + uint32_t egress:1; /**< Rule applies to egress traffic. */ + uint32_t reserved:30; /**< Reserved, must be zero. */ +}; + +/** + * Matching pattern item types. + * + * Pattern items fall in two categories: + * + * - Matching protocol headers and packet data (ANY, RAW, ETH, VLAN, IPV4, + * IPV6, ICMP, UDP, TCP, SCTP, VXLAN and so on), usually associated with a + * specification structure. These must be stacked in the same order as the + * protocol layers to match, starting from the lowest. + * + * - Matching meta-data or affecting pattern processing (END, VOID, INVERT, + * PF, VF, PORT and so on), often without a specification structure. Since + * they do not match packet contents, these can be specified anywhere + * within item lists without affecting others. + * + * See the description of individual types for more information. Those + * marked with [META] fall into the second category. + */ +enum rte_flow_item_type { + /** + * [META] + * + * End marker for item lists. Prevents further processing of items, + * thereby ending the pattern. + * + * No associated specification structure. + */ + RTE_FLOW_ITEM_TYPE_END, + + /** + * [META] + * + * Used as a placeholder for convenience. It is ignored and simply + * discarded by PMDs. + * + * No associated specification structure. + */ + RTE_FLOW_ITEM_TYPE_VOID, + + /** + * [META] + * + * Inverted matching, i.e. process packets that do not match the + * pattern. + * + * No associated specification structure. + */ + RTE_FLOW_ITEM_TYPE_INVERT, + + /** + * Matches any protocol in place of the current layer, a single ANY + * may also stand for several protocol layers. + * + * See struct rte_flow_item_any. + */ + RTE_FLOW_ITEM_TYPE_ANY, + + /** + * [META] + * + * Matches packets addressed to the physical function of the device. + * + * If the underlying device function differs from the one that would + * normally receive the matched traffic, specifying this item + * prevents it from reaching that device unless the flow rule + * contains a PF action. Packets are not duplicated between device + * instances by default. + * + * No associated specification structure. + */ + RTE_FLOW_ITEM_TYPE_PF, + + /** + * [META] + * + * Matches packets addressed to a virtual function ID of the device. + * + * If the underlying device function differs from the one that would + * normally receive the matched traffic, specifying this item + * prevents it from reaching that device unless the flow rule + * contains a VF action. Packets are not duplicated between device + * instances by default. + * + * See struct rte_flow_item_vf. + */ + RTE_FLOW_ITEM_TYPE_VF, + + /** + * [META] + * + * Matches packets coming from the specified physical port of the + * underlying device. + * + * The first PORT item overrides the physical port normally + * associated with the specified DPDK input port (port_id). This + * item can be provided several times to match additional physical + * ports. + * + * See struct rte_flow_item_port. + */ + RTE_FLOW_ITEM_TYPE_PORT, + + /** + * Matches a byte string of a given length at a given offset. + * + * See struct rte_flow_item_raw. + */ + RTE_FLOW_ITEM_TYPE_RAW, + + /** + * Matches an Ethernet header. + * + * See struct rte_flow_item_eth. + */ + RTE_FLOW_ITEM_TYPE_ETH, + + /** + * Matches an 802.1Q/ad VLAN tag. + * + * See struct rte_flow_item_vlan. + */ + RTE_FLOW_ITEM_TYPE_VLAN, + + /** + * Matches an IPv4 header. + * + * See struct rte_flow_item_ipv4. + */ + RTE_FLOW_ITEM_TYPE_IPV4, + + /** + * Matches an IPv6 header. + * + * See struct rte_flow_item_ipv6. + */ + RTE_FLOW_ITEM_TYPE_IPV6, + + /** + * Matches an ICMP header. + * + * See struct rte_flow_item_icmp. + */ + RTE_FLOW_ITEM_TYPE_ICMP, + + /** + * Matches a UDP header. + * + * See struct rte_flow_item_udp. + */ + RTE_FLOW_ITEM_TYPE_UDP, + + /** + * Matches a TCP header. + * + * See struct rte_flow_item_tcp. + */ + RTE_FLOW_ITEM_TYPE_TCP, + + /** + * Matches a SCTP header. + * + * See struct rte_flow_item_sctp. + */ + RTE_FLOW_ITEM_TYPE_SCTP, + + /** + * Matches a VXLAN header. + * + * See struct rte_flow_item_vxlan. + */ + RTE_FLOW_ITEM_TYPE_VXLAN, + + /** + * Matches a E_TAG header. + * + * See struct rte_flow_item_e_tag. + */ + RTE_FLOW_ITEM_TYPE_E_TAG, + + /** + * Matches a NVGRE header. + * + * See struct rte_flow_item_nvgre. + */ + RTE_FLOW_ITEM_TYPE_NVGRE, + + /** + * Matches a MPLS header. + * + * See struct rte_flow_item_mpls. + */ + RTE_FLOW_ITEM_TYPE_MPLS, + + /** + * Matches a GRE header. + * + * See struct rte_flow_item_gre. + */ + RTE_FLOW_ITEM_TYPE_GRE, + + /** + * [META] + * + * Fuzzy pattern match, expect faster than default. + * + * This is for device that support fuzzy matching option. + * Usually a fuzzy matching is fast but the cost is accuracy. + * + * See struct rte_flow_item_fuzzy. + */ + RTE_FLOW_ITEM_TYPE_FUZZY, + + /** + * Matches a GTP header. + * + * Configure flow for GTP packets. + * + * See struct rte_flow_item_gtp. + */ + RTE_FLOW_ITEM_TYPE_GTP, + + /** + * Matches a GTP header. + * + * Configure flow for GTP-C packets. + * + * See struct rte_flow_item_gtp. + */ + RTE_FLOW_ITEM_TYPE_GTPC, + + /** + * Matches a GTP header. + * + * Configure flow for GTP-U packets. + * + * See struct rte_flow_item_gtp. + */ + RTE_FLOW_ITEM_TYPE_GTPU, + + /** + * Matches a ESP header. + * + * See struct rte_flow_item_esp. + */ + RTE_FLOW_ITEM_TYPE_ESP, +}; + +/** + * RTE_FLOW_ITEM_TYPE_ANY + * + * Matches any protocol in place of the current layer, a single ANY may also + * stand for several protocol layers. + * + * This is usually specified as the first pattern item when looking for a + * protocol anywhere in a packet. + * + * A zeroed mask stands for any number of layers. + */ +struct rte_flow_item_any { + uint32_t num; /**< Number of layers covered. */ +}; + +/** Default mask for RTE_FLOW_ITEM_TYPE_ANY. */ +#ifndef __cplusplus +static const struct rte_flow_item_any rte_flow_item_any_mask = { + .num = 0x00000000, +}; +#endif + +/** + * RTE_FLOW_ITEM_TYPE_VF + * + * Matches packets addressed to a virtual function ID of the device. + * + * If the underlying device function differs from the one that would + * normally receive the matched traffic, specifying this item prevents it + * from reaching that device unless the flow rule contains a VF + * action. Packets are not duplicated between device instances by default. + * + * - Likely to return an error or never match any traffic if this causes a + * VF device to match traffic addressed to a different VF. + * - Can be specified multiple times to match traffic addressed to several + * VF IDs. + * - Can be combined with a PF item to match both PF and VF traffic. + * + * A zeroed mask can be used to match any VF ID. + */ +struct rte_flow_item_vf { + uint32_t id; /**< Destination VF ID. */ +}; + +/** Default mask for RTE_FLOW_ITEM_TYPE_VF. */ +#ifndef __cplusplus +static const struct rte_flow_item_vf rte_flow_item_vf_mask = { + .id = 0x00000000, +}; +#endif + +/** + * RTE_FLOW_ITEM_TYPE_PORT + * + * Matches packets coming from the specified physical port of the underlying + * device. + * + * The first PORT item overrides the physical port normally associated with + * the specified DPDK input port (port_id). This item can be provided + * several times to match additional physical ports. + * + * Note that physical ports are not necessarily tied to DPDK input ports + * (port_id) when those are not under DPDK control. Possible values are + * specific to each device, they are not necessarily indexed from zero and + * may not be contiguous. + * + * As a device property, the list of allowed values as well as the value + * associated with a port_id should be retrieved by other means. + * + * A zeroed mask can be used to match any port index. + */ +struct rte_flow_item_port { + uint32_t index; /**< Physical port index. */ +}; + +/** Default mask for RTE_FLOW_ITEM_TYPE_PORT. */ +#ifndef __cplusplus +static const struct rte_flow_item_port rte_flow_item_port_mask = { + .index = 0x00000000, +}; +#endif + +/** + * RTE_FLOW_ITEM_TYPE_RAW + * + * Matches a byte string of a given length at a given offset. + * + * Offset is either absolute (using the start of the packet) or relative to + * the end of the previous matched item in the stack, in which case negative + * values are allowed. + * + * If search is enabled, offset is used as the starting point. The search + * area can be delimited by setting limit to a nonzero value, which is the + * maximum number of bytes after offset where the pattern may start. + * + * Matching a zero-length pattern is allowed, doing so resets the relative + * offset for subsequent items. + * + * This type does not support ranges (struct rte_flow_item.last). + */ +struct rte_flow_item_raw { + uint32_t relative:1; /**< Look for pattern after the previous item. */ + uint32_t search:1; /**< Search pattern from offset (see also limit). */ + uint32_t reserved:30; /**< Reserved, must be set to zero. */ + int32_t offset; /**< Absolute or relative offset for pattern. */ + uint16_t limit; /**< Search area limit for start of pattern. */ + uint16_t length; /**< Pattern length. */ + uint8_t pattern[]; /**< Byte string to look for. */ +}; + +/** Default mask for RTE_FLOW_ITEM_TYPE_RAW. */ +#ifndef __cplusplus +static const struct rte_flow_item_raw rte_flow_item_raw_mask = { + .relative = 1, + .search = 1, + .reserved = 0x3fffffff, + .offset = 0xffffffff, + .limit = 0xffff, + .length = 0xffff, +}; +#endif + +/** + * RTE_FLOW_ITEM_TYPE_ETH + * + * Matches an Ethernet header. + */ +struct rte_flow_item_eth { + struct ether_addr dst; /**< Destination MAC. */ + struct ether_addr src; /**< Source MAC. */ + rte_be16_t type; /**< EtherType. */ +}; + +/** Default mask for RTE_FLOW_ITEM_TYPE_ETH. */ +#ifndef __cplusplus +static const struct rte_flow_item_eth rte_flow_item_eth_mask = { + .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", + .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", + .type = RTE_BE16(0x0000), +}; +#endif + +/** + * RTE_FLOW_ITEM_TYPE_VLAN + * + * Matches an 802.1Q/ad VLAN tag. + * + * This type normally follows either RTE_FLOW_ITEM_TYPE_ETH or + * RTE_FLOW_ITEM_TYPE_VLAN. + */ +struct rte_flow_item_vlan { + rte_be16_t tpid; /**< Tag protocol identifier. */ + rte_be16_t tci; /**< Tag control information. */ +}; + +/** Default mask for RTE_FLOW_ITEM_TYPE_VLAN. */ +#ifndef __cplusplus +static const struct rte_flow_item_vlan rte_flow_item_vlan_mask = { + .tpid = RTE_BE16(0x0000), + .tci = RTE_BE16(0xffff), +}; +#endif + +/** + * RTE_FLOW_ITEM_TYPE_IPV4 + * + * Matches an IPv4 header. + * + * Note: IPv4 options are handled by dedicated pattern items. + */ +struct rte_flow_item_ipv4 { + struct ipv4_hdr hdr; /**< IPv4 header definition. */ +}; + +/** Default mask for RTE_FLOW_ITEM_TYPE_IPV4. */ +#ifndef __cplusplus +static const struct rte_flow_item_ipv4 rte_flow_item_ipv4_mask = { + .hdr = { + .src_addr = RTE_BE32(0xffffffff), + .dst_addr = RTE_BE32(0xffffffff), + }, +}; +#endif + +/** + * RTE_FLOW_ITEM_TYPE_IPV6. + * + * Matches an IPv6 header. + * + * Note: IPv6 options are handled by dedicated pattern items. + */ +struct rte_flow_item_ipv6 { + struct ipv6_hdr hdr; /**< IPv6 header definition. */ +}; + +/** Default mask for RTE_FLOW_ITEM_TYPE_IPV6. */ +#ifndef __cplusplus +static const struct rte_flow_item_ipv6 rte_flow_item_ipv6_mask = { + .hdr = { + .src_addr = + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff", + .dst_addr = + "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff", + }, +}; +#endif + +/** + * RTE_FLOW_ITEM_TYPE_ICMP. + * + * Matches an ICMP header. + */ +struct rte_flow_item_icmp { + struct icmp_hdr hdr; /**< ICMP header definition. */ +}; + +/** Default mask for RTE_FLOW_ITEM_TYPE_ICMP. */ +#ifndef __cplusplus +static const struct rte_flow_item_icmp rte_flow_item_icmp_mask = { + .hdr = { + .icmp_type = 0xff, + .icmp_code = 0xff, + }, +}; +#endif + +/** + * RTE_FLOW_ITEM_TYPE_UDP. + * + * Matches a UDP header. + */ +struct rte_flow_item_udp { + struct udp_hdr hdr; /**< UDP header definition. */ +}; + +/** Default mask for RTE_FLOW_ITEM_TYPE_UDP. */ +#ifndef __cplusplus +static const struct rte_flow_item_udp rte_flow_item_udp_mask = { + .hdr = { + .src_port = RTE_BE16(0xffff), + .dst_port = RTE_BE16(0xffff), + }, +}; +#endif + +/** + * RTE_FLOW_ITEM_TYPE_TCP. + * + * Matches a TCP header. + */ +struct rte_flow_item_tcp { + struct tcp_hdr hdr; /**< TCP header definition. */ +}; + +/** Default mask for RTE_FLOW_ITEM_TYPE_TCP. */ +#ifndef __cplusplus +static const struct rte_flow_item_tcp rte_flow_item_tcp_mask = { + .hdr = { + .src_port = RTE_BE16(0xffff), + .dst_port = RTE_BE16(0xffff), + }, +}; +#endif + +/** + * RTE_FLOW_ITEM_TYPE_SCTP. + * + * Matches a SCTP header. + */ +struct rte_flow_item_sctp { + struct sctp_hdr hdr; /**< SCTP header definition. */ +}; + +/** Default mask for RTE_FLOW_ITEM_TYPE_SCTP. */ +#ifndef __cplusplus +static const struct rte_flow_item_sctp rte_flow_item_sctp_mask = { + .hdr = { + .src_port = RTE_BE16(0xffff), + .dst_port = RTE_BE16(0xffff), + }, +}; +#endif + +/** + * RTE_FLOW_ITEM_TYPE_VXLAN. + * + * Matches a VXLAN header (RFC 7348). + */ +struct rte_flow_item_vxlan { + uint8_t flags; /**< Normally 0x08 (I flag). */ + uint8_t rsvd0[3]; /**< Reserved, normally 0x000000. */ + uint8_t vni[3]; /**< VXLAN identifier. */ + uint8_t rsvd1; /**< Reserved, normally 0x00. */ +}; + +/** Default mask for RTE_FLOW_ITEM_TYPE_VXLAN. */ +#ifndef __cplusplus +static const struct rte_flow_item_vxlan rte_flow_item_vxlan_mask = { + .vni = "\xff\xff\xff", +}; +#endif + +/** + * RTE_FLOW_ITEM_TYPE_E_TAG. + * + * Matches a E-tag header. + */ +struct rte_flow_item_e_tag { + rte_be16_t tpid; /**< Tag protocol identifier (0x893F). */ + /** + * E-Tag control information (E-TCI). + * E-PCP (3b), E-DEI (1b), ingress E-CID base (12b). + */ + rte_be16_t epcp_edei_in_ecid_b; + /** Reserved (2b), GRP (2b), E-CID base (12b). */ + rte_be16_t rsvd_grp_ecid_b; + uint8_t in_ecid_e; /**< Ingress E-CID ext. */ + uint8_t ecid_e; /**< E-CID ext. */ +}; + +/** Default mask for RTE_FLOW_ITEM_TYPE_E_TAG. */ +#ifndef __cplusplus +static const struct rte_flow_item_e_tag rte_flow_item_e_tag_mask = { + .rsvd_grp_ecid_b = RTE_BE16(0x3fff), +}; +#endif + +/** + * RTE_FLOW_ITEM_TYPE_NVGRE. + * + * Matches a NVGRE header. + */ +struct rte_flow_item_nvgre { + /** + * Checksum (1b), undefined (1b), key bit (1b), sequence number (1b), + * reserved 0 (9b), version (3b). + * + * c_k_s_rsvd0_ver must have value 0x2000 according to RFC 7637. + */ + rte_be16_t c_k_s_rsvd0_ver; + rte_be16_t protocol; /**< Protocol type (0x6558). */ + uint8_t tni[3]; /**< Virtual subnet ID. */ + uint8_t flow_id; /**< Flow ID. */ +}; + +/** Default mask for RTE_FLOW_ITEM_TYPE_NVGRE. */ +#ifndef __cplusplus +static const struct rte_flow_item_nvgre rte_flow_item_nvgre_mask = { + .tni = "\xff\xff\xff", +}; +#endif + +/** + * RTE_FLOW_ITEM_TYPE_MPLS. + * + * Matches a MPLS header. + */ +struct rte_flow_item_mpls { + /** + * Label (20b), TC (3b), Bottom of Stack (1b). + */ + uint8_t label_tc_s[3]; + uint8_t ttl; /** Time-to-Live. */ +}; + +/** Default mask for RTE_FLOW_ITEM_TYPE_MPLS. */ +#ifndef __cplusplus +static const struct rte_flow_item_mpls rte_flow_item_mpls_mask = { + .label_tc_s = "\xff\xff\xf0", +}; +#endif + +/** + * RTE_FLOW_ITEM_TYPE_GRE. + * + * Matches a GRE header. + */ +struct rte_flow_item_gre { + /** + * Checksum (1b), reserved 0 (12b), version (3b). + * Refer to RFC 2784. + */ + rte_be16_t c_rsvd0_ver; + rte_be16_t protocol; /**< Protocol type. */ +}; + +/** Default mask for RTE_FLOW_ITEM_TYPE_GRE. */ +#ifndef __cplusplus +static const struct rte_flow_item_gre rte_flow_item_gre_mask = { + .protocol = RTE_BE16(0xffff), +}; +#endif + +/** + * RTE_FLOW_ITEM_TYPE_FUZZY + * + * Fuzzy pattern match, expect faster than default. + * + * This is for device that support fuzzy match option. + * Usually a fuzzy match is fast but the cost is accuracy. + * i.e. Signature Match only match pattern's hash value, but it is + * possible two different patterns have the same hash value. + * + * Matching accuracy level can be configure by threshold. + * Driver can divide the range of threshold and map to different + * accuracy levels that device support. + * + * Threshold 0 means perfect match (no fuzziness), while threshold + * 0xffffffff means fuzziest match. + */ +struct rte_flow_item_fuzzy { + uint32_t thresh; /**< Accuracy threshold. */ +}; + +/** Default mask for RTE_FLOW_ITEM_TYPE_FUZZY. */ +#ifndef __cplusplus +static const struct rte_flow_item_fuzzy rte_flow_item_fuzzy_mask = { + .thresh = 0xffffffff, +}; +#endif + +/** + * RTE_FLOW_ITEM_TYPE_GTP. + * + * Matches a GTPv1 header. + */ +struct rte_flow_item_gtp { + /** + * Version (3b), protocol type (1b), reserved (1b), + * Extension header flag (1b), + * Sequence number flag (1b), + * N-PDU number flag (1b). + */ + uint8_t v_pt_rsv_flags; + uint8_t msg_type; /**< Message type. */ + rte_be16_t msg_len; /**< Message length. */ + rte_be32_t teid; /**< Tunnel endpoint identifier. */ +}; + +/** Default mask for RTE_FLOW_ITEM_TYPE_GTP. */ +#ifndef __cplusplus +static const struct rte_flow_item_gtp rte_flow_item_gtp_mask = { + .teid = RTE_BE32(0xffffffff), +}; +#endif + +/** + * RTE_FLOW_ITEM_TYPE_ESP + * + * Matches an ESP header. + */ +struct rte_flow_item_esp { + struct esp_hdr hdr; /**< ESP header definition. */ +}; + +/** Default mask for RTE_FLOW_ITEM_TYPE_ESP. */ +#ifndef __cplusplus +static const struct rte_flow_item_esp rte_flow_item_esp_mask = { + .hdr = { + .spi = 0xffffffff, + }, +}; +#endif + +/** + * Matching pattern item definition. + * + * A pattern is formed by stacking items starting from the lowest protocol + * layer to match. This stacking restriction does not apply to meta items + * which can be placed anywhere in the stack without affecting the meaning + * of the resulting pattern. + * + * Patterns are terminated by END items. + * + * The spec field should be a valid pointer to a structure of the related + * item type. It may remain unspecified (NULL) in many cases to request + * broad (nonspecific) matching. In such cases, last and mask must also be + * set to NULL. + * + * Optionally, last can point to a structure of the same type to define an + * inclusive range. This is mostly supported by integer and address fields, + * may cause errors otherwise. Fields that do not support ranges must be set + * to 0 or to the same value as the corresponding fields in spec. + * + * Only the fields defined to nonzero values in the default masks (see + * rte_flow_item_{name}_mask constants) are considered relevant by + * default. This can be overridden by providing a mask structure of the + * same type with applicable bits set to one. It can also be used to + * partially filter out specific fields (e.g. as an alternate mean to match + * ranges of IP addresses). + * + * Mask is a simple bit-mask applied before interpreting the contents of + * spec and last, which may yield unexpected results if not used + * carefully. For example, if for an IPv4 address field, spec provides + * 10.1.2.3, last provides 10.3.4.5 and mask provides 255.255.0.0, the + * effective range becomes 10.1.0.0 to 10.3.255.255. + */ +struct rte_flow_item { + enum rte_flow_item_type type; /**< Item type. */ + const void *spec; /**< Pointer to item specification structure. */ + const void *last; /**< Defines an inclusive range (spec to last). */ + const void *mask; /**< Bit-mask applied to spec and last. */ +}; + +/** + * Action types. + * + * Each possible action is represented by a type. Some have associated + * configuration structures. Several actions combined in a list can be + * affected to a flow rule. That list is not ordered. + * + * They fall in three categories: + * + * - Terminating actions (such as QUEUE, DROP, RSS, PF, VF) that prevent + * processing matched packets by subsequent flow rules, unless overridden + * with PASSTHRU. + * + * - Non terminating actions (PASSTHRU, DUP) that leave matched packets up + * for additional processing by subsequent flow rules. + * + * - Other non terminating meta actions that do not affect the fate of + * packets (END, VOID, MARK, FLAG, COUNT). + * + * When several actions are combined in a flow rule, they should all have + * different types (e.g. dropping a packet twice is not possible). + * + * Only the last action of a given type is taken into account. PMDs still + * perform error checking on the entire list. + * + * Note that PASSTHRU is the only action able to override a terminating + * rule. + */ +enum rte_flow_action_type { + /** + * [META] + * + * End marker for action lists. Prevents further processing of + * actions, thereby ending the list. + * + * No associated configuration structure. + */ + RTE_FLOW_ACTION_TYPE_END, + + /** + * [META] + * + * Used as a placeholder for convenience. It is ignored and simply + * discarded by PMDs. + * + * No associated configuration structure. + */ + RTE_FLOW_ACTION_TYPE_VOID, + + /** + * Leaves packets up for additional processing by subsequent flow + * rules. This is the default when a rule does not contain a + * terminating action, but can be specified to force a rule to + * become non-terminating. + * + * No associated configuration structure. + */ + RTE_FLOW_ACTION_TYPE_PASSTHRU, + + /** + * [META] + * + * Attaches an integer value to packets and sets PKT_RX_FDIR and + * PKT_RX_FDIR_ID mbuf flags. + * + * See struct rte_flow_action_mark. + */ + RTE_FLOW_ACTION_TYPE_MARK, + + /** + * [META] + * + * Flags packets. Similar to MARK without a specific value; only + * sets the PKT_RX_FDIR mbuf flag. + * + * No associated configuration structure. + */ + RTE_FLOW_ACTION_TYPE_FLAG, + + /** + * Assigns packets to a given queue index. + * + * See struct rte_flow_action_queue. + */ + RTE_FLOW_ACTION_TYPE_QUEUE, + + /** + * Drops packets. + * + * PASSTHRU overrides this action if both are specified. + * + * No associated configuration structure. + */ + RTE_FLOW_ACTION_TYPE_DROP, + + /** + * [META] + * + * Enables counters for this rule. + * + * These counters can be retrieved and reset through rte_flow_query(), + * see struct rte_flow_query_count. + * + * No associated configuration structure. + */ + RTE_FLOW_ACTION_TYPE_COUNT, + + /** + * Duplicates packets to a given queue index. + * + * This is normally combined with QUEUE, however when used alone, it + * is actually similar to QUEUE + PASSTHRU. + * + * See struct rte_flow_action_dup. + */ + RTE_FLOW_ACTION_TYPE_DUP, + + /** + * Similar to QUEUE, except RSS is additionally performed on packets + * to spread them among several queues according to the provided + * parameters. + * + * See struct rte_flow_action_rss. + */ + RTE_FLOW_ACTION_TYPE_RSS, + + /** + * Redirects packets to the physical function (PF) of the current + * device. + * + * No associated configuration structure. + */ + RTE_FLOW_ACTION_TYPE_PF, + + /** + * Redirects packets to the virtual function (VF) of the current + * device with the specified ID. + * + * See struct rte_flow_action_vf. + */ + RTE_FLOW_ACTION_TYPE_VF, + + /** + * Traffic metering and policing (MTR). + * + * See struct rte_flow_action_meter. + * See file rte_mtr.h for MTR object configuration. + */ + RTE_FLOW_ACTION_TYPE_METER, + + /** + * Redirects packets to security engine of current device for security + * processing as specified by security session. + * + * See struct rte_flow_action_security. + */ + RTE_FLOW_ACTION_TYPE_SECURITY +}; + +/** + * RTE_FLOW_ACTION_TYPE_MARK + * + * Attaches an integer value to packets and sets PKT_RX_FDIR and + * PKT_RX_FDIR_ID mbuf flags. + * + * This value is arbitrary and application-defined. Maximum allowed value + * depends on the underlying implementation. It is returned in the + * hash.fdir.hi mbuf field. + */ +struct rte_flow_action_mark { + uint32_t id; /**< Integer value to return with packets. */ +}; + +/** + * RTE_FLOW_ACTION_TYPE_QUEUE + * + * Assign packets to a given queue index. + * + * Terminating by default. + */ +struct rte_flow_action_queue { + uint16_t index; /**< Queue index to use. */ +}; + +/** + * RTE_FLOW_ACTION_TYPE_COUNT (query) + * + * Query structure to retrieve and reset flow rule counters. + */ +struct rte_flow_query_count { + uint32_t reset:1; /**< Reset counters after query [in]. */ + uint32_t hits_set:1; /**< hits field is set [out]. */ + uint32_t bytes_set:1; /**< bytes field is set [out]. */ + uint32_t reserved:29; /**< Reserved, must be zero [in, out]. */ + uint64_t hits; /**< Number of hits for this rule [out]. */ + uint64_t bytes; /**< Number of bytes through this rule [out]. */ +}; + +/** + * RTE_FLOW_ACTION_TYPE_DUP + * + * Duplicates packets to a given queue index. + * + * This is normally combined with QUEUE, however when used alone, it is + * actually similar to QUEUE + PASSTHRU. + * + * Non-terminating by default. + */ +struct rte_flow_action_dup { + uint16_t index; /**< Queue index to duplicate packets to. */ +}; + +/** + * RTE_FLOW_ACTION_TYPE_RSS + * + * Similar to QUEUE, except RSS is additionally performed on packets to + * spread them among several queues according to the provided parameters. + * + * Note: RSS hash result is stored in the hash.rss mbuf field which overlaps + * hash.fdir.lo. Since the MARK action sets the hash.fdir.hi field only, + * both can be requested simultaneously. + * + * Terminating by default. + */ +struct rte_flow_action_rss { + const struct rte_eth_rss_conf *rss_conf; /**< RSS parameters. */ + uint16_t num; /**< Number of entries in queue[]. */ + uint16_t queue[]; /**< Queues indices to use. */ +}; + +/** + * RTE_FLOW_ACTION_TYPE_VF + * + * Redirects packets to a virtual function (VF) of the current device. + * + * Packets matched by a VF pattern item can be redirected to their original + * VF ID instead of the specified one. This parameter may not be available + * and is not guaranteed to work properly if the VF part is matched by a + * prior flow rule or if packets are not addressed to a VF in the first + * place. + * + * Terminating by default. + */ +struct rte_flow_action_vf { + uint32_t original:1; /**< Use original VF ID if possible. */ + uint32_t reserved:31; /**< Reserved, must be zero. */ + uint32_t id; /**< VF ID to redirect packets to. */ +}; + +/** + * RTE_FLOW_ACTION_TYPE_METER + * + * Traffic metering and policing (MTR). + * + * Packets matched by items of this type can be either dropped or passed to the + * next item with their color set by the MTR object. + * + * Non-terminating by default. + */ +struct rte_flow_action_meter { + uint32_t mtr_id; /**< MTR object ID created with rte_mtr_create(). */ +}; + +/** + * RTE_FLOW_ACTION_TYPE_SECURITY + * + * Perform the security action on flows matched by the pattern items + * according to the configuration of the security session. + * + * This action modifies the payload of matched flows. For INLINE_CRYPTO, the + * security protocol headers and IV are fully provided by the application as + * specified in the flow pattern. The payload of matching packets is + * encrypted on egress, and decrypted and authenticated on ingress. + * For INLINE_PROTOCOL, the security protocol is fully offloaded to HW, + * providing full encapsulation and decapsulation of packets in security + * protocols. The flow pattern specifies both the outer security header fields + * and the inner packet fields. The security session specified in the action + * must match the pattern parameters. + * + * The security session specified in the action must be created on the same + * port as the flow action that is being specified. + * + * The ingress/egress flow attribute should match that specified in the + * security session if the security session supports the definition of the + * direction. + * + * Multiple flows can be configured to use the same security session. + * + * Non-terminating by default. + */ +struct rte_flow_action_security { + void *security_session; /**< Pointer to security session structure. */ +}; + +/** + * Definition of a single action. + * + * A list of actions is terminated by a END action. + * + * For simple actions without a configuration structure, conf remains NULL. + */ +struct rte_flow_action { + enum rte_flow_action_type type; /**< Action type. */ + const void *conf; /**< Pointer to action configuration structure. */ +}; + +/** + * Opaque type returned after successfully creating a flow. + * + * This handle can be used to manage and query the related flow (e.g. to + * destroy it or retrieve counters). + */ +struct rte_flow; + +/** + * Verbose error types. + * + * Most of them provide the type of the object referenced by struct + * rte_flow_error.cause. + */ +enum rte_flow_error_type { + RTE_FLOW_ERROR_TYPE_NONE, /**< No error. */ + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, /**< Cause unspecified. */ + RTE_FLOW_ERROR_TYPE_HANDLE, /**< Flow rule (handle). */ + RTE_FLOW_ERROR_TYPE_ATTR_GROUP, /**< Group field. */ + RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, /**< Priority field. */ + RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, /**< Ingress field. */ + RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, /**< Egress field. */ + RTE_FLOW_ERROR_TYPE_ATTR, /**< Attributes structure. */ + RTE_FLOW_ERROR_TYPE_ITEM_NUM, /**< Pattern length. */ + RTE_FLOW_ERROR_TYPE_ITEM, /**< Specific pattern item. */ + RTE_FLOW_ERROR_TYPE_ACTION_NUM, /**< Number of actions. */ + RTE_FLOW_ERROR_TYPE_ACTION, /**< Specific action. */ +}; + +/** + * Verbose error structure definition. + * + * This object is normally allocated by applications and set by PMDs, the + * message points to a constant string which does not need to be freed by + * the application, however its pointer can be considered valid only as long + * as its associated DPDK port remains configured. Closing the underlying + * device or unloading the PMD invalidates it. + * + * Both cause and message may be NULL regardless of the error type. + */ +struct rte_flow_error { + enum rte_flow_error_type type; /**< Cause field and error types. */ + const void *cause; /**< Object responsible for the error. */ + const char *message; /**< Human-readable error message. */ +}; + +/** + * Check whether a flow rule can be created on a given port. + * + * The flow rule is validated for correctness and whether it could be accepted + * by the device given sufficient resources. The rule is checked against the + * current device mode and queue configuration. The flow rule may also + * optionally be validated against existing flow rules and device resources. + * This function has no effect on the target device. + * + * The returned value is guaranteed to remain valid only as long as no + * successful calls to rte_flow_create() or rte_flow_destroy() are made in + * the meantime and no device parameter affecting flow rules in any way are + * modified, due to possible collisions or resource limitations (although in + * such cases EINVAL should not be returned). + * + * @param port_id + * Port identifier of Ethernet device. + * @param[in] attr + * Flow rule attributes. + * @param[in] pattern + * Pattern specification (list terminated by the END pattern item). + * @param[in] actions + * Associated actions (list terminated by the END action). + * @param[out] error + * Perform verbose error reporting if not NULL. PMDs initialize this + * structure in case of error only. + * + * @return + * 0 if flow rule is valid and can be created. A negative errno value + * otherwise (rte_errno is also set), the following errors are defined: + * + * -ENOSYS: underlying device does not support this functionality. + * + * -EINVAL: unknown or invalid rule specification. + * + * -ENOTSUP: valid but unsupported rule specification (e.g. partial + * bit-masks are unsupported). + * + * -EEXIST: collision with an existing rule. Only returned if device + * supports flow rule collision checking and there was a flow rule + * collision. Not receiving this return code is no guarantee that creating + * the rule will not fail due to a collision. + * + * -ENOMEM: not enough memory to execute the function, or if the device + * supports resource validation, resource limitation on the device. + * + * -EBUSY: action cannot be performed due to busy device resources, may + * succeed if the affected queues or even the entire port are in a stopped + * state (see rte_eth_dev_rx_queue_stop() and rte_eth_dev_stop()). + */ +int +rte_flow_validate(uint16_t port_id, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error); + +/** + * Create a flow rule on a given port. + * + * @param port_id + * Port identifier of Ethernet device. + * @param[in] attr + * Flow rule attributes. + * @param[in] pattern + * Pattern specification (list terminated by the END pattern item). + * @param[in] actions + * Associated actions (list terminated by the END action). + * @param[out] error + * Perform verbose error reporting if not NULL. PMDs initialize this + * structure in case of error only. + * + * @return + * A valid handle in case of success, NULL otherwise and rte_errno is set + * to the positive version of one of the error codes defined for + * rte_flow_validate(). + */ +struct rte_flow * +rte_flow_create(uint16_t port_id, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error); + +/** + * Destroy a flow rule on a given port. + * + * Failure to destroy a flow rule handle may occur when other flow rules + * depend on it, and destroying it would result in an inconsistent state. + * + * This function is only guaranteed to succeed if handles are destroyed in + * reverse order of their creation. + * + * @param port_id + * Port identifier of Ethernet device. + * @param flow + * Flow rule handle to destroy. + * @param[out] error + * Perform verbose error reporting if not NULL. PMDs initialize this + * structure in case of error only. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +int +rte_flow_destroy(uint16_t port_id, + struct rte_flow *flow, + struct rte_flow_error *error); + +/** + * Destroy all flow rules associated with a port. + * + * In the unlikely event of failure, handles are still considered destroyed + * and no longer valid but the port must be assumed to be in an inconsistent + * state. + * + * @param port_id + * Port identifier of Ethernet device. + * @param[out] error + * Perform verbose error reporting if not NULL. PMDs initialize this + * structure in case of error only. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +int +rte_flow_flush(uint16_t port_id, + struct rte_flow_error *error); + +/** + * Query an existing flow rule. + * + * This function allows retrieving flow-specific data such as counters. + * Data is gathered by special actions which must be present in the flow + * rule definition. + * + * \see RTE_FLOW_ACTION_TYPE_COUNT + * + * @param port_id + * Port identifier of Ethernet device. + * @param flow + * Flow rule handle to query. + * @param action + * Action type to query. + * @param[in, out] data + * Pointer to storage for the associated query data type. + * @param[out] error + * Perform verbose error reporting if not NULL. PMDs initialize this + * structure in case of error only. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +int +rte_flow_query(uint16_t port_id, + struct rte_flow *flow, + enum rte_flow_action_type action, + void *data, + struct rte_flow_error *error); + +/** + * Restrict ingress traffic to the defined flow rules. + * + * Isolated mode guarantees that all ingress traffic comes from defined flow + * rules only (current and future). + * + * Besides making ingress more deterministic, it allows PMDs to safely reuse + * resources otherwise assigned to handle the remaining traffic, such as + * global RSS configuration settings, VLAN filters, MAC address entries, + * legacy filter API rules and so on in order to expand the set of possible + * flow rule types. + * + * Calling this function as soon as possible after device initialization, + * ideally before the first call to rte_eth_dev_configure(), is recommended + * to avoid possible failures due to conflicting settings. + * + * Once effective, leaving isolated mode may not be possible depending on + * PMD implementation. + * + * Additionally, the following functionality has no effect on the underlying + * port and may return errors such as ENOTSUP ("not supported"): + * + * - Toggling promiscuous mode. + * - Toggling allmulticast mode. + * - Configuring MAC addresses. + * - Configuring multicast addresses. + * - Configuring VLAN filters. + * - Configuring Rx filters through the legacy API (e.g. FDIR). + * - Configuring global RSS settings. + * + * @param port_id + * Port identifier of Ethernet device. + * @param set + * Nonzero to enter isolated mode, attempt to leave it otherwise. + * @param[out] error + * Perform verbose error reporting if not NULL. PMDs initialize this + * structure in case of error only. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +int +rte_flow_isolate(uint16_t port_id, int set, struct rte_flow_error *error); + +/** + * Initialize flow error structure. + * + * @param[out] error + * Pointer to flow error structure (may be NULL). + * @param code + * Related error code (rte_errno). + * @param type + * Cause field and error types. + * @param cause + * Object responsible for the error. + * @param message + * Human-readable error message. + * + * @return + * Negative error code (errno value) and rte_errno is set. + */ +int +rte_flow_error_set(struct rte_flow_error *error, + int code, + enum rte_flow_error_type type, + const void *cause, + const char *message); + +/** + * Generic flow representation. + * + * This form is sufficient to describe an rte_flow independently from any + * PMD implementation and allows for replayability and identification. + */ +struct rte_flow_desc { + size_t size; /**< Allocated space including data[]. */ + struct rte_flow_attr attr; /**< Attributes. */ + struct rte_flow_item *items; /**< Items. */ + struct rte_flow_action *actions; /**< Actions. */ + uint8_t data[]; /**< Storage for items/actions. */ +}; + +/** + * Copy an rte_flow rule description. + * + * @param[in] fd + * Flow rule description. + * @param[in] len + * Total size of allocated data for the flow description. + * @param[in] attr + * Flow rule attributes. + * @param[in] items + * Pattern specification (list terminated by the END pattern item). + * @param[in] actions + * Associated actions (list terminated by the END action). + * + * @return + * If len is greater or equal to the size of the flow, the total size of the + * flow description and its data. + * If len is lower than the size of the flow, the number of bytes that would + * have been written to desc had it been sufficient. Nothing is written. + */ +size_t +rte_flow_copy(struct rte_flow_desc *fd, size_t len, + const struct rte_flow_attr *attr, + const struct rte_flow_item *items, + const struct rte_flow_action *actions); + +#ifdef __cplusplus +} +#endif + +#endif /* RTE_FLOW_H_ */ diff --git a/dpdk/lib/librte_ether/rte_flow_driver.h b/dpdk/lib/librte_ether/rte_flow_driver.h new file mode 100644 index 00000000..254d1cb2 --- /dev/null +++ b/dpdk/lib/librte_ether/rte_flow_driver.h @@ -0,0 +1,149 @@ +/*- + * BSD LICENSE + * + * Copyright 2016 6WIND S.A. + * Copyright 2016 Mellanox. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of 6WIND S.A. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef RTE_FLOW_DRIVER_H_ +#define RTE_FLOW_DRIVER_H_ + +/** + * @file + * RTE generic flow API (driver side) + * + * This file provides implementation helpers for internal use by PMDs, they + * are not intended to be exposed to applications and are not subject to ABI + * versioning. + */ + +#include + +#include "rte_ethdev.h" +#include "rte_flow.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Generic flow operations structure implemented and returned by PMDs. + * + * To implement this API, PMDs must handle the RTE_ETH_FILTER_GENERIC filter + * type in their .filter_ctrl callback function (struct eth_dev_ops) as well + * as the RTE_ETH_FILTER_GET filter operation. + * + * If successful, this operation must result in a pointer to a PMD-specific + * struct rte_flow_ops written to the argument address as described below: + * + * \code + * + * // PMD filter_ctrl callback + * + * static const struct rte_flow_ops pmd_flow_ops = { ... }; + * + * switch (filter_type) { + * case RTE_ETH_FILTER_GENERIC: + * if (filter_op != RTE_ETH_FILTER_GET) + * return -EINVAL; + * *(const void **)arg = &pmd_flow_ops; + * return 0; + * } + * + * \endcode + * + * See also rte_flow_ops_get(). + * + * These callback functions are not supposed to be used by applications + * directly, which must rely on the API defined in rte_flow.h. + * + * Public-facing wrapper functions perform a few consistency checks so that + * unimplemented (i.e. NULL) callbacks simply return -ENOTSUP. These + * callbacks otherwise only differ by their first argument (with port ID + * already resolved to a pointer to struct rte_eth_dev). + */ +struct rte_flow_ops { + /** See rte_flow_validate(). */ + int (*validate) + (struct rte_eth_dev *, + const struct rte_flow_attr *, + const struct rte_flow_item [], + const struct rte_flow_action [], + struct rte_flow_error *); + /** See rte_flow_create(). */ + struct rte_flow *(*create) + (struct rte_eth_dev *, + const struct rte_flow_attr *, + const struct rte_flow_item [], + const struct rte_flow_action [], + struct rte_flow_error *); + /** See rte_flow_destroy(). */ + int (*destroy) + (struct rte_eth_dev *, + struct rte_flow *, + struct rte_flow_error *); + /** See rte_flow_flush(). */ + int (*flush) + (struct rte_eth_dev *, + struct rte_flow_error *); + /** See rte_flow_query(). */ + int (*query) + (struct rte_eth_dev *, + struct rte_flow *, + enum rte_flow_action_type, + void *, + struct rte_flow_error *); + /** See rte_flow_isolate(). */ + int (*isolate) + (struct rte_eth_dev *, + int, + struct rte_flow_error *); +}; + +/** + * Get generic flow operations structure from a port. + * + * @param port_id + * Port identifier to query. + * @param[out] error + * Pointer to flow error structure. + * + * @return + * The flow operations structure associated with port_id, NULL in case of + * error, in which case rte_errno is set and the error structure contains + * additional details. + */ +const struct rte_flow_ops * +rte_flow_ops_get(uint16_t port_id, struct rte_flow_error *error); + +#ifdef __cplusplus +} +#endif + +#endif /* RTE_FLOW_DRIVER_H_ */ diff --git a/dpdk/lib/librte_ether/rte_mtr.c b/dpdk/lib/librte_ether/rte_mtr.c new file mode 100644 index 00000000..4f56f871 --- /dev/null +++ b/dpdk/lib/librte_ether/rte_mtr.c @@ -0,0 +1,229 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +#include +#include "rte_ethdev.h" +#include "rte_mtr_driver.h" +#include "rte_mtr.h" + +/* Get generic traffic metering & policing operations structure from a port. */ +const struct rte_mtr_ops * +rte_mtr_ops_get(uint16_t port_id, struct rte_mtr_error *error) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + const struct rte_mtr_ops *ops; + + if (!rte_eth_dev_is_valid_port(port_id)) { + rte_mtr_error_set(error, + ENODEV, + RTE_MTR_ERROR_TYPE_UNSPECIFIED, + NULL, + rte_strerror(ENODEV)); + return NULL; + } + + if ((dev->dev_ops->mtr_ops_get == NULL) || + (dev->dev_ops->mtr_ops_get(dev, &ops) != 0) || + (ops == NULL)) { + rte_mtr_error_set(error, + ENOSYS, + RTE_MTR_ERROR_TYPE_UNSPECIFIED, + NULL, + rte_strerror(ENOSYS)); + return NULL; + } + + return ops; +} + +#define RTE_MTR_FUNC(port_id, func) \ +({ \ + const struct rte_mtr_ops *ops = \ + rte_mtr_ops_get(port_id, error); \ + if (ops == NULL) \ + return -rte_errno; \ + \ + if (ops->func == NULL) \ + return -rte_mtr_error_set(error, \ + ENOSYS, \ + RTE_MTR_ERROR_TYPE_UNSPECIFIED, \ + NULL, \ + rte_strerror(ENOSYS)); \ + \ + ops->func; \ +}) + +/* MTR capabilities get */ +int +rte_mtr_capabilities_get(uint16_t port_id, + struct rte_mtr_capabilities *cap, + struct rte_mtr_error *error) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + return RTE_MTR_FUNC(port_id, capabilities_get)(dev, + cap, error); +} + +/* MTR meter profile add */ +int +rte_mtr_meter_profile_add(uint16_t port_id, + uint32_t meter_profile_id, + struct rte_mtr_meter_profile *profile, + struct rte_mtr_error *error) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + return RTE_MTR_FUNC(port_id, meter_profile_add)(dev, + meter_profile_id, profile, error); +} + +/** MTR meter profile delete */ +int +rte_mtr_meter_profile_delete(uint16_t port_id, + uint32_t meter_profile_id, + struct rte_mtr_error *error) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + return RTE_MTR_FUNC(port_id, meter_profile_delete)(dev, + meter_profile_id, error); +} + +/** MTR object create */ +int +rte_mtr_create(uint16_t port_id, + uint32_t mtr_id, + struct rte_mtr_params *params, + int shared, + struct rte_mtr_error *error) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + return RTE_MTR_FUNC(port_id, create)(dev, + mtr_id, params, shared, error); +} + +/** MTR object destroy */ +int +rte_mtr_destroy(uint16_t port_id, + uint32_t mtr_id, + struct rte_mtr_error *error) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + return RTE_MTR_FUNC(port_id, destroy)(dev, + mtr_id, error); +} + +/** MTR object meter enable */ +int +rte_mtr_meter_enable(uint16_t port_id, + uint32_t mtr_id, + struct rte_mtr_error *error) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + return RTE_MTR_FUNC(port_id, meter_enable)(dev, + mtr_id, error); +} + +/** MTR object meter disable */ +int +rte_mtr_meter_disable(uint16_t port_id, + uint32_t mtr_id, + struct rte_mtr_error *error) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + return RTE_MTR_FUNC(port_id, meter_disable)(dev, + mtr_id, error); +} + +/** MTR object meter profile update */ +int +rte_mtr_meter_profile_update(uint16_t port_id, + uint32_t mtr_id, + uint32_t meter_profile_id, + struct rte_mtr_error *error) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + return RTE_MTR_FUNC(port_id, meter_profile_update)(dev, + mtr_id, meter_profile_id, error); +} + +/** MTR object meter DSCP table update */ +int +rte_mtr_meter_dscp_table_update(uint16_t port_id, + uint32_t mtr_id, + enum rte_mtr_color *dscp_table, + struct rte_mtr_error *error) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + return RTE_MTR_FUNC(port_id, meter_dscp_table_update)(dev, + mtr_id, dscp_table, error); +} + +/** MTR object policer action update */ +int +rte_mtr_policer_actions_update(uint16_t port_id, + uint32_t mtr_id, + uint32_t action_mask, + enum rte_mtr_policer_action *actions, + struct rte_mtr_error *error) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + return RTE_MTR_FUNC(port_id, policer_actions_update)(dev, + mtr_id, action_mask, actions, error); +} + +/** MTR object enabled stats update */ +int +rte_mtr_stats_update(uint16_t port_id, + uint32_t mtr_id, + uint64_t stats_mask, + struct rte_mtr_error *error) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + return RTE_MTR_FUNC(port_id, stats_update)(dev, + mtr_id, stats_mask, error); +} + +/** MTR object stats read */ +int +rte_mtr_stats_read(uint16_t port_id, + uint32_t mtr_id, + struct rte_mtr_stats *stats, + uint64_t *stats_mask, + int clear, + struct rte_mtr_error *error) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + return RTE_MTR_FUNC(port_id, stats_read)(dev, + mtr_id, stats, stats_mask, clear, error); +} diff --git a/dpdk/lib/librte_ether/rte_mtr.h b/dpdk/lib/librte_ether/rte_mtr.h new file mode 100644 index 00000000..f6b6ef3b --- /dev/null +++ b/dpdk/lib/librte_ether/rte_mtr.h @@ -0,0 +1,730 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 Intel Corporation + * Copyright 2017 NXP + * Copyright 2017 Cavium + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __INCLUDE_RTE_MTR_H__ +#define __INCLUDE_RTE_MTR_H__ + +/** + * @file + * RTE Generic Traffic Metering and Policing API + * + * This interface provides the ability to configure the traffic metering and + * policing (MTR) in a generic way. + * + * The processing done for each input packet hitting a MTR object is: + * A) Traffic metering: The packet is assigned a color (the meter output + * color), based on the previous history of the flow reflected in the + * current state of the MTR object, according to the specific traffic + * metering algorithm. The traffic metering algorithm can typically work + * in color aware mode, in which case the input packet already has an + * initial color (the input color), or in color blind mode, which is + * equivalent to considering all input packets initially colored as green. + * B) Policing: There is a separate policer action configured for each meter + * output color, which can: + * a) Drop the packet. + * b) Keep the same packet color: the policer output color matches the + * meter output color (essentially a no-op action). + * c) Recolor the packet: the policer output color is different than + * the meter output color. + * The policer output color is the output color of the packet, which is + * set in the packet meta-data (i.e. struct rte_mbuf::sched::color). + * C) Statistics: The set of counters maintained for each MTR object is + * configurable and subject to the implementation support. This set + * includes the number of packets and bytes dropped or passed for each + * output color. + * + * Once successfully created, an MTR object is linked to one or several flows + * through the meter action of the flow API. + * A) Whether an MTR object is private to a flow or potentially shared by + * several flows has to be specified at creation time. + * B) Several meter actions can be potentially registered for the same flow. + * + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + */ +#include + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Color + */ +enum rte_mtr_color { + RTE_MTR_GREEN = 0, /**< Green */ + RTE_MTR_YELLOW, /**< Yellow */ + RTE_MTR_RED, /**< Red */ + RTE_MTR_COLORS /**< Number of colors. */ +}; + +/** + * Statistics counter type + */ +enum rte_mtr_stats_type { + /** Number of packets passed as green by the policer. */ + RTE_MTR_STATS_N_PKTS_GREEN = 1 << 0, + + /** Number of packets passed as yellow by the policer. */ + RTE_MTR_STATS_N_PKTS_YELLOW = 1 << 1, + + /** Number of packets passed as red by the policer. */ + RTE_MTR_STATS_N_PKTS_RED = 1 << 2, + + /** Number of packets dropped by the policer. */ + RTE_MTR_STATS_N_PKTS_DROPPED = 1 << 3, + + /** Number of bytes passed as green by the policer. */ + RTE_MTR_STATS_N_BYTES_GREEN = 1 << 4, + + /** Number of bytes passed as yellow by the policer. */ + RTE_MTR_STATS_N_BYTES_YELLOW = 1 << 5, + + /** Number of bytes passed as red by the policer. */ + RTE_MTR_STATS_N_BYTES_RED = 1 << 6, + + /** Number of bytes dropped by the policer. */ + RTE_MTR_STATS_N_BYTES_DROPPED = 1 << 7, +}; + +/** + * Statistics counters + */ +struct rte_mtr_stats { + /** Number of packets passed by the policer (per color). */ + uint64_t n_pkts[RTE_MTR_COLORS]; + + /** Number of bytes passed by the policer (per color). */ + uint64_t n_bytes[RTE_MTR_COLORS]; + + /** Number of packets dropped by the policer. */ + uint64_t n_pkts_dropped; + + /** Number of bytes passed by the policer. */ + uint64_t n_bytes_dropped; +}; + +/** + * Traffic metering algorithms + */ +enum rte_mtr_algorithm { + /** No traffic metering performed, the output color is the same as the + * input color for every input packet. The meter of the MTR object is + * working in pass-through mode, having same effect as meter disable. + * @see rte_mtr_meter_disable() + */ + RTE_MTR_NONE = 0, + + /** Single Rate Three Color Marker (srTCM) - IETF RFC 2697. */ + RTE_MTR_SRTCM_RFC2697, + + /** Two Rate Three Color Marker (trTCM) - IETF RFC 2698. */ + RTE_MTR_TRTCM_RFC2698, + + /** Two Rate Three Color Marker (trTCM) - IETF RFC 4115. */ + RTE_MTR_TRTCM_RFC4115, +}; + +/** + * Meter profile + */ +struct rte_mtr_meter_profile { + /** Traffic metering algorithm. */ + enum rte_mtr_algorithm alg; + + RTE_STD_C11 + union { + /** Items only valid when *alg* is set to srTCM - RFC 2697. */ + struct { + /** Committed Information Rate (CIR) (bytes/second). */ + uint64_t cir; + + /** Committed Burst Size (CBS) (bytes). */ + uint64_t cbs; + + /** Excess Burst Size (EBS) (bytes). */ + uint64_t ebs; + } srtcm_rfc2697; + + /** Items only valid when *alg* is set to trTCM - RFC 2698. */ + struct { + /** Committed Information Rate (CIR) (bytes/second). */ + uint64_t cir; + + /** Peak Information Rate (PIR) (bytes/second). */ + uint64_t pir; + + /** Committed Burst Size (CBS) (byes). */ + uint64_t cbs; + + /** Peak Burst Size (PBS) (bytes). */ + uint64_t pbs; + } trtcm_rfc2698; + + /** Items only valid when *alg* is set to trTCM - RFC 4115. */ + struct { + /** Committed Information Rate (CIR) (bytes/second). */ + uint64_t cir; + + /** Excess Information Rate (EIR) (bytes/second). */ + uint64_t eir; + + /** Committed Burst Size (CBS) (byes). */ + uint64_t cbs; + + /** Excess Burst Size (EBS) (bytes). */ + uint64_t ebs; + } trtcm_rfc4115; + }; +}; + +/** + * Policer actions + */ +enum rte_mtr_policer_action { + /** Recolor the packet as green. */ + MTR_POLICER_ACTION_COLOR_GREEN = 0, + + /** Recolor the packet as yellow. */ + MTR_POLICER_ACTION_COLOR_YELLOW, + + /** Recolor the packet as red. */ + MTR_POLICER_ACTION_COLOR_RED, + + /** Drop the packet. */ + MTR_POLICER_ACTION_DROP, +}; + +/** + * Parameters for each traffic metering & policing object + * + * @see enum rte_mtr_stats_type + */ +struct rte_mtr_params { + /** Meter profile ID. */ + uint32_t meter_profile_id; + + /** Meter input color in case of MTR object chaining. When non-zero: if + * a previous MTR object is enabled in the same flow, then the color + * determined by the latest MTR object in the same flow is used as the + * input color by the current MTR object, otherwise the current MTR + * object uses the *dscp_table* to determine the input color. When zero: + * the color determined by any previous MTR object in same flow is + * ignored by the current MTR object, which uses the *dscp_table* to + * determine the input color. + */ + int use_prev_mtr_color; + + /** Meter input color. When non-NULL: it points to a pre-allocated and + * pre-populated table with exactly 64 elements providing the input + * color for each value of the IPv4/IPv6 Differentiated Services Code + * Point (DSCP) input packet field. When NULL: it is equivalent to + * setting this parameter to an all-green populated table (i.e. table + * with all the 64 elements set to green color). The color blind mode + * is configured by setting *use_prev_mtr_color* to 0 and *dscp_table* + * to either NULL or to an all-green populated table. When + * *use_prev_mtr_color* is non-zero value or when *dscp_table* contains + * at least one yellow or red color element, then the color aware mode + * is configured. + */ + enum rte_mtr_color *dscp_table; + + /** Non-zero to enable the meter, zero to disable the meter at the time + * of MTR object creation. Ignored when the meter profile indicated by + * *meter_profile_id* is set to NONE. + * @see rte_mtr_meter_disable() + */ + int meter_enable; + + /** Policer actions (per meter output color). */ + enum rte_mtr_policer_action action[RTE_MTR_COLORS]; + + /** Set of stats counters to be enabled. + * @see enum rte_mtr_stats_type + */ + uint64_t stats_mask; +}; + +/** + * MTR capabilities + */ +struct rte_mtr_capabilities { + /** Maximum number of MTR objects. */ + uint32_t n_max; + + /** Maximum number of MTR objects that can be shared by multiple flows. + * The value of zero indicates that shared MTR objects are not + * supported. The maximum value is *n_max*. + */ + uint32_t n_shared_max; + + /** When non-zero, this flag indicates that all the MTR objects that + * cannot be shared by multiple flows have identical capability set. + */ + int identical; + + /** When non-zero, this flag indicates that all the MTR objects that + * can be shared by multiple flows have identical capability set. + */ + int shared_identical; + + /** Maximum number of flows that can share the same MTR object. The + * value of zero is invalid. The value of 1 means that shared MTR + * objects not supported. + */ + uint32_t shared_n_flows_per_mtr_max; + + /** Maximum number of MTR objects that can be part of the same flow. The + * value of zero is invalid. The value of 1 indicates that MTR object + * chaining is not supported. The maximum value is *n_max*. + */ + uint32_t chaining_n_mtrs_per_flow_max; + + /** + * When non-zero, it indicates that the packet color identified by one + * MTR object can be used as the packet input color by any subsequent + * MTR object from the same flow. When zero, it indicates that the color + * determined by one MTR object is always ignored by any subsequent MTR + * object from the same flow. Only valid when MTR chaining is supported, + * i.e. *chaining_n_mtrs_per_flow_max* is greater than 1. When non-zero, + * it also means that the color aware mode is supported by at least one + * metering algorithm. + */ + int chaining_use_prev_mtr_color_supported; + + /** + * When non-zero, it indicates that the packet color identified by one + * MTR object is always used as the packet input color by any subsequent + * MTR object that is part of the same flow. When zero, it indicates + * that whether the color determined by one MTR object is either ignored + * or used as the packet input color by any subsequent MTR object from + * the same flow is individually configurable for each MTR object. Only + * valid when *chaining_use_prev_mtr_color_supported* is non-zero. + */ + int chaining_use_prev_mtr_color_enforced; + + /** Maximum number of MTR objects that can have their meter configured + * to run the srTCM RFC 2697 algorithm. The value of 0 indicates this + * metering algorithm is not supported. The maximum value is *n_max*. + */ + uint32_t meter_srtcm_rfc2697_n_max; + + /** Maximum number of MTR objects that can have their meter configured + * to run the trTCM RFC 2698 algorithm. The value of 0 indicates this + * metering algorithm is not supported. The maximum value is *n_max*. + */ + uint32_t meter_trtcm_rfc2698_n_max; + + /** Maximum number of MTR objects that can have their meter configured + * to run the trTCM RFC 4115 algorithm. The value of 0 indicates this + * metering algorithm is not supported. The maximum value is *n_max*. + */ + uint32_t meter_trtcm_rfc4115_n_max; + + /** Maximum traffic rate that can be metered by a single MTR object. For + * srTCM RFC 2697, this is the maximum CIR rate. For trTCM RFC 2698, + * this is the maximum PIR rate. For trTCM RFC 4115, this is the maximum + * value for the sum of PIR and EIR rates. + */ + uint64_t meter_rate_max; + + /** + * When non-zero, it indicates that color aware mode is supported for + * the srTCM RFC 2697 metering algorithm. + */ + int color_aware_srtcm_rfc2697_supported; + + /** + * When non-zero, it indicates that color aware mode is supported for + * the trTCM RFC 2698 metering algorithm. + */ + int color_aware_trtcm_rfc2698_supported; + + /** + * When non-zero, it indicates that color aware mode is supported for + * the trTCM RFC 4115 metering algorithm. + */ + int color_aware_trtcm_rfc4115_supported; + + /** When non-zero, it indicates that the policer packet recolor actions + * are supported. + * @see enum rte_mtr_policer_action + */ + int policer_action_recolor_supported; + + /** When non-zero, it indicates that the policer packet drop action is + * supported. + * @see enum rte_mtr_policer_action + */ + int policer_action_drop_supported; + + /** Set of supported statistics counter types. + * @see enum rte_mtr_stats_type + */ + uint64_t stats_mask; +}; + +/** + * Verbose error types. + * + * Most of them provide the type of the object referenced by struct + * rte_mtr_error::cause. + */ +enum rte_mtr_error_type { + RTE_MTR_ERROR_TYPE_NONE, /**< No error. */ + RTE_MTR_ERROR_TYPE_UNSPECIFIED, /**< Cause unspecified. */ + RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, + RTE_MTR_ERROR_TYPE_METER_PROFILE, + RTE_MTR_ERROR_TYPE_MTR_ID, + RTE_MTR_ERROR_TYPE_MTR_PARAMS, + RTE_MTR_ERROR_TYPE_POLICER_ACTION_GREEN, + RTE_MTR_ERROR_TYPE_POLICER_ACTION_YELLOW, + RTE_MTR_ERROR_TYPE_POLICER_ACTION_RED, + RTE_MTR_ERROR_TYPE_STATS_MASK, + RTE_MTR_ERROR_TYPE_STATS, + RTE_MTR_ERROR_TYPE_SHARED, +}; + +/** + * Verbose error structure definition. + * + * This object is normally allocated by applications and set by PMDs, the + * message points to a constant string which does not need to be freed by + * the application, however its pointer can be considered valid only as long + * as its associated DPDK port remains configured. Closing the underlying + * device or unloading the PMD invalidates it. + * + * Both cause and message may be NULL regardless of the error type. + */ +struct rte_mtr_error { + enum rte_mtr_error_type type; /**< Cause field and error type. */ + const void *cause; /**< Object responsible for the error. */ + const char *message; /**< Human-readable error message. */ +}; + +/** + * MTR capabilities get + * + * @param[in] port_id + * The port identifier of the Ethernet device. + * @param[out] cap + * MTR capabilities. Needs to be pre-allocated and valid. + * @param[out] error + * Error details. Filled in only on error, when not NULL. + * @return + * 0 on success, non-zero error code otherwise. + */ +int +rte_mtr_capabilities_get(uint16_t port_id, + struct rte_mtr_capabilities *cap, + struct rte_mtr_error *error); + +/** + * Meter profile add + * + * Create a new meter profile with ID set to *meter_profile_id*. The new profile + * is used to create one or several MTR objects. + * + * @param[in] port_id + * The port identifier of the Ethernet device. + * @param[in] meter_profile_id + * ID for the new meter profile. Needs to be unused by any of the existing + * meter profiles added for the current port. + * @param[in] profile + * Meter profile parameters. Needs to be pre-allocated and valid. + * @param[out] error + * Error details. Filled in only on error, when not NULL. + * @return + * 0 on success, non-zero error code otherwise. + */ +int +rte_mtr_meter_profile_add(uint16_t port_id, + uint32_t meter_profile_id, + struct rte_mtr_meter_profile *profile, + struct rte_mtr_error *error); + +/** + * Meter profile delete + * + * Delete an existing meter profile. This operation fails when there is + * currently at least one user (i.e. MTR object) of this profile. + * + * @param[in] port_id + * The port identifier of the Ethernet device. + * @param[in] meter_profile_id + * Meter profile ID. Needs to be the valid. + * @param[out] error + * Error details. Filled in only on error, when not NULL. + * @return + * 0 on success, non-zero error code otherwise. + */ +int +rte_mtr_meter_profile_delete(uint16_t port_id, + uint32_t meter_profile_id, + struct rte_mtr_error *error); + +/** + * MTR object create + * + * Create a new MTR object for the current port. This object is run as part of + * associated flow action for traffic metering and policing. + * + * @param[in] port_id + * The port identifier of the Ethernet device. + * @param[in] mtr_id + * MTR object ID. Needs to be unused by any of the existing MTR objects. + * created for the current port. + * @param[in] params + * MTR object params. Needs to be pre-allocated and valid. + * @param[in] shared + * Non-zero when this MTR object can be shared by multiple flows, zero when + * this MTR object can be used by a single flow. + * @param[out] error + * Error details. Filled in only on error, when not NULL. + * @return + * 0 on success, non-zero error code otherwise. + * + * @see enum rte_flow_action_type::RTE_FLOW_ACTION_TYPE_METER + */ +int +rte_mtr_create(uint16_t port_id, + uint32_t mtr_id, + struct rte_mtr_params *params, + int shared, + struct rte_mtr_error *error); + +/** + * MTR object destroy + * + * Delete an existing MTR object. This operation fails when there is currently + * at least one user (i.e. flow) of this MTR object. + * + * @param[in] port_id + * The port identifier of the Ethernet device. + * @param[in] mtr_id + * MTR object ID. Needs to be valid. + * created for the current port. + * @param[out] error + * Error details. Filled in only on error, when not NULL. + * @return + * 0 on success, non-zero error code otherwise. + */ +int +rte_mtr_destroy(uint16_t port_id, + uint32_t mtr_id, + struct rte_mtr_error *error); + +/** + * MTR object meter disable + * + * Disable the meter of an existing MTR object. In disabled state, the meter of + * the current MTR object works in pass-through mode, meaning that for each + * input packet the meter output color is always the same as the input color. In + * particular, when the meter of the current MTR object is configured in color + * blind mode, the input color is always green, so the meter output color is + * also always green. Note that the policer and the statistics of the current + * MTR object are working as usual while the meter is disabled. No action is + * taken and this function returns successfully when the meter of the current + * MTR object is already disabled. + * + * @param[in] port_id + * The port identifier of the Ethernet device. + * @param[in] mtr_id + * MTR object ID. + * @param[out] error + * Error details. Filled in only on error, when not NULL. + * @return + * 0 on success, non-zero error code otherwise. + */ +int +rte_mtr_meter_disable(uint16_t port_id, + uint32_t mtr_id, + struct rte_mtr_error *error); + +/** + * MTR object meter enable + * + * Enable the meter of an existing MTR object. If the MTR object has its meter + * already enabled, then no action is taken and this function returns + * successfully. + * + * @param[in] port_id + * The port identifier of the Ethernet device. + * @param[in] mtr_id + * MTR object ID. + * @param[out] error + * Error details. Filled in only on error, when not NULL. + * @return + * 0 on success, non-zero error code otherwise. + */ +int +rte_mtr_meter_enable(uint16_t port_id, + uint32_t mtr_id, + struct rte_mtr_error *error); + +/** + * MTR object meter profile update + * + * @param[in] port_id + * The port identifier of the Ethernet device. + * @param[in] mtr_id + * MTR object ID. Needs to be valid. + * @param[in] meter_profile_id + * Meter profile ID for the current MTR object. Needs to be valid. + * @param[out] error + * Error details. Filled in only on error, when not NULL. + * @return + * 0 on success, non-zero error code otherwise. + */ +int +rte_mtr_meter_profile_update(uint16_t port_id, + uint32_t mtr_id, + uint32_t meter_profile_id, + struct rte_mtr_error *error); + +/** + * MTR object DSCP table update + * + * @param[in] port_id + * The port identifier of the Ethernet device. + * @param[in] mtr_id + * MTR object ID. Needs to be valid. + * @param[in] dscp_table + * When non-NULL: it points to a pre-allocated and pre-populated table with + * exactly 64 elements providing the input color for each value of the + * IPv4/IPv6 Differentiated Services Code Point (DSCP) input packet field. + * When NULL: it is equivalent to setting this parameter to an “all-green” + * populated table (i.e. table with all the 64 elements set to green color). + * @param[out] error + * Error details. Filled in only on error, when not NULL. + * @return + * 0 on success, non-zero error code otherwise. + */ +int +rte_mtr_meter_dscp_table_update(uint16_t port_id, + uint32_t mtr_id, + enum rte_mtr_color *dscp_table, + struct rte_mtr_error *error); + +/** + * MTR object policer actions update + * + * @param[in] port_id + * The port identifier of the Ethernet device. + * @param[in] mtr_id + * MTR object ID. Needs to be valid. + * @param[in] action_mask + * Bit mask indicating which policer actions need to be updated. One or more + * policer actions can be updated in a single function invocation. To update + * the policer action associated with color C, bit (1 << C) needs to be set in + * *action_mask* and element at position C in the *actions* array needs to be + * valid. + * @param[in] actions + * Pre-allocated and pre-populated array of policer actions. + * @param[out] error + * Error details. Filled in only on error, when not NULL. + * @return + * 0 on success, non-zero error code otherwise. + */ +int +rte_mtr_policer_actions_update(uint16_t port_id, + uint32_t mtr_id, + uint32_t action_mask, + enum rte_mtr_policer_action *actions, + struct rte_mtr_error *error); + +/** + * MTR object enabled statistics counters update + * + * @param[in] port_id + * The port identifier of the Ethernet device. + * @param[in] mtr_id + * MTR object ID. Needs to be valid. + * @param[in] stats_mask + * Mask of statistics counter types to be enabled for the current MTR object. + * Any statistics counter type not included in this set is to be disabled for + * the current MTR object. + * @param[out] error + * Error details. Filled in only on error, when not NULL. + * @return + * 0 on success, non-zero error code otherwise. + * + * @see enum rte_mtr_stats_type + */ +int +rte_mtr_stats_update(uint16_t port_id, + uint32_t mtr_id, + uint64_t stats_mask, + struct rte_mtr_error *error); + +/** + * MTR object statistics counters read + * + * @param[in] port_id + * The port identifier of the Ethernet device. + * @param[in] mtr_id + * MTR object ID. Needs to be valid. + * @param[out] stats + * When non-NULL, it contains the current value for the statistics counters + * enabled for the current MTR object. + * @param[out] stats_mask + * When non-NULL, it contains the mask of statistics counter types that are + * currently enabled for this MTR object, indicating which of the counters + * retrieved with the *stats* structure are valid. + * @param[in] clear + * When this parameter has a non-zero value, the statistics counters are + * cleared (i.e. set to zero) immediately after they have been read, + * otherwise the statistics counters are left untouched. + * @param[out] error + * Error details. Filled in only on error, when not NULL. + * @return + * 0 on success, non-zero error code otherwise. + * + * @see enum rte_mtr_stats_type + */ +int +rte_mtr_stats_read(uint16_t port_id, + uint32_t mtr_id, + struct rte_mtr_stats *stats, + uint64_t *stats_mask, + int clear, + struct rte_mtr_error *error); + +#ifdef __cplusplus +} +#endif + +#endif /* __INCLUDE_RTE_MTR_H__ */ diff --git a/dpdk/lib/librte_ether/rte_mtr_driver.h b/dpdk/lib/librte_ether/rte_mtr_driver.h new file mode 100644 index 00000000..6a289ef1 --- /dev/null +++ b/dpdk/lib/librte_ether/rte_mtr_driver.h @@ -0,0 +1,221 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __INCLUDE_RTE_MTR_DRIVER_H__ +#define __INCLUDE_RTE_MTR_DRIVER_H__ + +/** + * @file + * RTE Generic Traffic Metering and Policing API (Driver Side) + * + * This file provides implementation helpers for internal use by PMDs, they + * are not intended to be exposed to applications and are not subject to ABI + * versioning. + */ + +#include + +#include +#include "rte_ethdev.h" +#include "rte_mtr.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef int (*rte_mtr_capabilities_get_t)(struct rte_eth_dev *dev, + struct rte_mtr_capabilities *cap, + struct rte_mtr_error *error); +/**< @internal MTR capabilities get */ + +typedef int (*rte_mtr_meter_profile_add_t)(struct rte_eth_dev *dev, + uint32_t meter_profile_id, + struct rte_mtr_meter_profile *profile, + struct rte_mtr_error *error); +/**< @internal MTR meter profile add */ + +typedef int (*rte_mtr_meter_profile_delete_t)(struct rte_eth_dev *dev, + uint32_t meter_profile_id, + struct rte_mtr_error *error); +/**< @internal MTR meter profile delete */ + +typedef int (*rte_mtr_create_t)(struct rte_eth_dev *dev, + uint32_t mtr_id, + struct rte_mtr_params *params, + int shared, + struct rte_mtr_error *error); +/**< @internal MTR object create */ + +typedef int (*rte_mtr_destroy_t)(struct rte_eth_dev *dev, + uint32_t mtr_id, + struct rte_mtr_error *error); +/**< @internal MTR object destroy */ + +typedef int (*rte_mtr_meter_enable_t)(struct rte_eth_dev *dev, + uint32_t mtr_id, + struct rte_mtr_error *error); +/**< @internal MTR object meter enable */ + +typedef int (*rte_mtr_meter_disable_t)(struct rte_eth_dev *dev, + uint32_t mtr_id, + struct rte_mtr_error *error); +/**< @internal MTR object meter disable */ + +typedef int (*rte_mtr_meter_profile_update_t)(struct rte_eth_dev *dev, + uint32_t mtr_id, + uint32_t meter_profile_id, + struct rte_mtr_error *error); +/**< @internal MTR object meter profile update */ + +typedef int (*rte_mtr_meter_dscp_table_update_t)(struct rte_eth_dev *dev, + uint32_t mtr_id, + enum rte_mtr_color *dscp_table, + struct rte_mtr_error *error); +/**< @internal MTR object meter DSCP table update */ + +typedef int (*rte_mtr_policer_actions_update_t)(struct rte_eth_dev *dev, + uint32_t mtr_id, + uint32_t action_mask, + enum rte_mtr_policer_action *actions, + struct rte_mtr_error *error); +/**< @internal MTR object policer action update*/ + +typedef int (*rte_mtr_stats_update_t)(struct rte_eth_dev *dev, + uint32_t mtr_id, + uint64_t stats_mask, + struct rte_mtr_error *error); +/**< @internal MTR object enabled stats update */ + +typedef int (*rte_mtr_stats_read_t)(struct rte_eth_dev *dev, + uint32_t mtr_id, + struct rte_mtr_stats *stats, + uint64_t *stats_mask, + int clear, + struct rte_mtr_error *error); +/**< @internal MTR object stats read */ + +struct rte_mtr_ops { + /** MTR capabilities get */ + rte_mtr_capabilities_get_t capabilities_get; + + /** MTR meter profile add */ + rte_mtr_meter_profile_add_t meter_profile_add; + + /** MTR meter profile delete */ + rte_mtr_meter_profile_delete_t meter_profile_delete; + + /** MTR object create */ + rte_mtr_create_t create; + + /** MTR object destroy */ + rte_mtr_destroy_t destroy; + + /** MTR object meter enable */ + rte_mtr_meter_enable_t meter_enable; + + /** MTR object meter disable */ + rte_mtr_meter_disable_t meter_disable; + + /** MTR object meter profile update */ + rte_mtr_meter_profile_update_t meter_profile_update; + + /** MTR object meter DSCP table update */ + rte_mtr_meter_dscp_table_update_t meter_dscp_table_update; + + /** MTR object policer action update */ + rte_mtr_policer_actions_update_t policer_actions_update; + + /** MTR object enabled stats update */ + rte_mtr_stats_update_t stats_update; + + /** MTR object stats read */ + rte_mtr_stats_read_t stats_read; +}; + +/** + * Initialize generic error structure. + * + * This function also sets rte_errno to a given value. + * + * @param[out] error + * Pointer to error structure (may be NULL). + * @param[in] code + * Related error code (rte_errno). + * @param[in] type + * Cause field and error type. + * @param[in] cause + * Object responsible for the error. + * @param[in] message + * Human-readable error message. + * + * @return + * Error code. + */ +static inline int +rte_mtr_error_set(struct rte_mtr_error *error, + int code, + enum rte_mtr_error_type type, + const void *cause, + const char *message) +{ + if (error) { + *error = (struct rte_mtr_error){ + .type = type, + .cause = cause, + .message = message, + }; + } + rte_errno = code; + return code; +} + +/** + * Get generic traffic metering and policing operations structure from a port + * + * @param[in] port_id + * The port identifier of the Ethernet device. + * @param[out] error + * Error details + * + * @return + * The traffic metering and policing operations structure associated with + * port_id on success, NULL otherwise. + */ +const struct rte_mtr_ops * +rte_mtr_ops_get(uint16_t port_id, struct rte_mtr_error *error); + +#ifdef __cplusplus +} +#endif + +#endif /* __INCLUDE_RTE_MTR_DRIVER_H__ */ diff --git a/dpdk/lib/librte_ether/rte_tm.c b/dpdk/lib/librte_ether/rte_tm.c new file mode 100644 index 00000000..ceac3411 --- /dev/null +++ b/dpdk/lib/librte_ether/rte_tm.c @@ -0,0 +1,438 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +#include +#include "rte_ethdev.h" +#include "rte_tm_driver.h" +#include "rte_tm.h" + +/* Get generic traffic manager operations structure from a port. */ +const struct rte_tm_ops * +rte_tm_ops_get(uint16_t port_id, struct rte_tm_error *error) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + const struct rte_tm_ops *ops; + + if (!rte_eth_dev_is_valid_port(port_id)) { + rte_tm_error_set(error, + ENODEV, + RTE_TM_ERROR_TYPE_UNSPECIFIED, + NULL, + rte_strerror(ENODEV)); + return NULL; + } + + if ((dev->dev_ops->tm_ops_get == NULL) || + (dev->dev_ops->tm_ops_get(dev, &ops) != 0) || + (ops == NULL)) { + rte_tm_error_set(error, + ENOSYS, + RTE_TM_ERROR_TYPE_UNSPECIFIED, + NULL, + rte_strerror(ENOSYS)); + return NULL; + } + + return ops; +} + +#define RTE_TM_FUNC(port_id, func) \ +({ \ + const struct rte_tm_ops *ops = \ + rte_tm_ops_get(port_id, error); \ + if (ops == NULL) \ + return -rte_errno; \ + \ + if (ops->func == NULL) \ + return -rte_tm_error_set(error, \ + ENOSYS, \ + RTE_TM_ERROR_TYPE_UNSPECIFIED, \ + NULL, \ + rte_strerror(ENOSYS)); \ + \ + ops->func; \ +}) + +/* Get number of leaf nodes */ +int +rte_tm_get_number_of_leaf_nodes(uint16_t port_id, + uint32_t *n_leaf_nodes, + struct rte_tm_error *error) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + const struct rte_tm_ops *ops = + rte_tm_ops_get(port_id, error); + + if (ops == NULL) + return -rte_errno; + + if (n_leaf_nodes == NULL) { + rte_tm_error_set(error, + EINVAL, + RTE_TM_ERROR_TYPE_UNSPECIFIED, + NULL, + rte_strerror(EINVAL)); + return -rte_errno; + } + + *n_leaf_nodes = dev->data->nb_tx_queues; + return 0; +} + +/* Check node type (leaf or non-leaf) */ +int +rte_tm_node_type_get(uint16_t port_id, + uint32_t node_id, + int *is_leaf, + struct rte_tm_error *error) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + return RTE_TM_FUNC(port_id, node_type_get)(dev, + node_id, is_leaf, error); +} + +/* Get capabilities */ +int rte_tm_capabilities_get(uint16_t port_id, + struct rte_tm_capabilities *cap, + struct rte_tm_error *error) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + return RTE_TM_FUNC(port_id, capabilities_get)(dev, + cap, error); +} + +/* Get level capabilities */ +int rte_tm_level_capabilities_get(uint16_t port_id, + uint32_t level_id, + struct rte_tm_level_capabilities *cap, + struct rte_tm_error *error) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + return RTE_TM_FUNC(port_id, level_capabilities_get)(dev, + level_id, cap, error); +} + +/* Get node capabilities */ +int rte_tm_node_capabilities_get(uint16_t port_id, + uint32_t node_id, + struct rte_tm_node_capabilities *cap, + struct rte_tm_error *error) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + return RTE_TM_FUNC(port_id, node_capabilities_get)(dev, + node_id, cap, error); +} + +/* Add WRED profile */ +int rte_tm_wred_profile_add(uint16_t port_id, + uint32_t wred_profile_id, + struct rte_tm_wred_params *profile, + struct rte_tm_error *error) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + return RTE_TM_FUNC(port_id, wred_profile_add)(dev, + wred_profile_id, profile, error); +} + +/* Delete WRED profile */ +int rte_tm_wred_profile_delete(uint16_t port_id, + uint32_t wred_profile_id, + struct rte_tm_error *error) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + return RTE_TM_FUNC(port_id, wred_profile_delete)(dev, + wred_profile_id, error); +} + +/* Add/update shared WRED context */ +int rte_tm_shared_wred_context_add_update(uint16_t port_id, + uint32_t shared_wred_context_id, + uint32_t wred_profile_id, + struct rte_tm_error *error) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + return RTE_TM_FUNC(port_id, shared_wred_context_add_update)(dev, + shared_wred_context_id, wred_profile_id, error); +} + +/* Delete shared WRED context */ +int rte_tm_shared_wred_context_delete(uint16_t port_id, + uint32_t shared_wred_context_id, + struct rte_tm_error *error) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + return RTE_TM_FUNC(port_id, shared_wred_context_delete)(dev, + shared_wred_context_id, error); +} + +/* Add shaper profile */ +int rte_tm_shaper_profile_add(uint16_t port_id, + uint32_t shaper_profile_id, + struct rte_tm_shaper_params *profile, + struct rte_tm_error *error) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + return RTE_TM_FUNC(port_id, shaper_profile_add)(dev, + shaper_profile_id, profile, error); +} + +/* Delete WRED profile */ +int rte_tm_shaper_profile_delete(uint16_t port_id, + uint32_t shaper_profile_id, + struct rte_tm_error *error) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + return RTE_TM_FUNC(port_id, shaper_profile_delete)(dev, + shaper_profile_id, error); +} + +/* Add shared shaper */ +int rte_tm_shared_shaper_add_update(uint16_t port_id, + uint32_t shared_shaper_id, + uint32_t shaper_profile_id, + struct rte_tm_error *error) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + return RTE_TM_FUNC(port_id, shared_shaper_add_update)(dev, + shared_shaper_id, shaper_profile_id, error); +} + +/* Delete shared shaper */ +int rte_tm_shared_shaper_delete(uint16_t port_id, + uint32_t shared_shaper_id, + struct rte_tm_error *error) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + return RTE_TM_FUNC(port_id, shared_shaper_delete)(dev, + shared_shaper_id, error); +} + +/* Add node to port traffic manager hierarchy */ +int rte_tm_node_add(uint16_t port_id, + uint32_t node_id, + uint32_t parent_node_id, + uint32_t priority, + uint32_t weight, + uint32_t level_id, + struct rte_tm_node_params *params, + struct rte_tm_error *error) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + return RTE_TM_FUNC(port_id, node_add)(dev, + node_id, parent_node_id, priority, weight, level_id, + params, error); +} + +/* Delete node from traffic manager hierarchy */ +int rte_tm_node_delete(uint16_t port_id, + uint32_t node_id, + struct rte_tm_error *error) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + return RTE_TM_FUNC(port_id, node_delete)(dev, + node_id, error); +} + +/* Suspend node */ +int rte_tm_node_suspend(uint16_t port_id, + uint32_t node_id, + struct rte_tm_error *error) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + return RTE_TM_FUNC(port_id, node_suspend)(dev, + node_id, error); +} + +/* Resume node */ +int rte_tm_node_resume(uint16_t port_id, + uint32_t node_id, + struct rte_tm_error *error) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + return RTE_TM_FUNC(port_id, node_resume)(dev, + node_id, error); +} + +/* Commit the initial port traffic manager hierarchy */ +int rte_tm_hierarchy_commit(uint16_t port_id, + int clear_on_fail, + struct rte_tm_error *error) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + return RTE_TM_FUNC(port_id, hierarchy_commit)(dev, + clear_on_fail, error); +} + +/* Update node parent */ +int rte_tm_node_parent_update(uint16_t port_id, + uint32_t node_id, + uint32_t parent_node_id, + uint32_t priority, + uint32_t weight, + struct rte_tm_error *error) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + return RTE_TM_FUNC(port_id, node_parent_update)(dev, + node_id, parent_node_id, priority, weight, error); +} + +/* Update node private shaper */ +int rte_tm_node_shaper_update(uint16_t port_id, + uint32_t node_id, + uint32_t shaper_profile_id, + struct rte_tm_error *error) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + return RTE_TM_FUNC(port_id, node_shaper_update)(dev, + node_id, shaper_profile_id, error); +} + +/* Update node shared shapers */ +int rte_tm_node_shared_shaper_update(uint16_t port_id, + uint32_t node_id, + uint32_t shared_shaper_id, + int add, + struct rte_tm_error *error) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + return RTE_TM_FUNC(port_id, node_shared_shaper_update)(dev, + node_id, shared_shaper_id, add, error); +} + +/* Update node stats */ +int rte_tm_node_stats_update(uint16_t port_id, + uint32_t node_id, + uint64_t stats_mask, + struct rte_tm_error *error) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + return RTE_TM_FUNC(port_id, node_stats_update)(dev, + node_id, stats_mask, error); +} + +/* Update WFQ weight mode */ +int rte_tm_node_wfq_weight_mode_update(uint16_t port_id, + uint32_t node_id, + int *wfq_weight_mode, + uint32_t n_sp_priorities, + struct rte_tm_error *error) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + return RTE_TM_FUNC(port_id, node_wfq_weight_mode_update)(dev, + node_id, wfq_weight_mode, n_sp_priorities, error); +} + +/* Update node congestion management mode */ +int rte_tm_node_cman_update(uint16_t port_id, + uint32_t node_id, + enum rte_tm_cman_mode cman, + struct rte_tm_error *error) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + return RTE_TM_FUNC(port_id, node_cman_update)(dev, + node_id, cman, error); +} + +/* Update node private WRED context */ +int rte_tm_node_wred_context_update(uint16_t port_id, + uint32_t node_id, + uint32_t wred_profile_id, + struct rte_tm_error *error) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + return RTE_TM_FUNC(port_id, node_wred_context_update)(dev, + node_id, wred_profile_id, error); +} + +/* Update node shared WRED context */ +int rte_tm_node_shared_wred_context_update(uint16_t port_id, + uint32_t node_id, + uint32_t shared_wred_context_id, + int add, + struct rte_tm_error *error) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + return RTE_TM_FUNC(port_id, node_shared_wred_context_update)(dev, + node_id, shared_wred_context_id, add, error); +} + +/* Read and/or clear stats counters for specific node */ +int rte_tm_node_stats_read(uint16_t port_id, + uint32_t node_id, + struct rte_tm_node_stats *stats, + uint64_t *stats_mask, + int clear, + struct rte_tm_error *error) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + return RTE_TM_FUNC(port_id, node_stats_read)(dev, + node_id, stats, stats_mask, clear, error); +} + +/* Packet marking - VLAN DEI */ +int rte_tm_mark_vlan_dei(uint16_t port_id, + int mark_green, + int mark_yellow, + int mark_red, + struct rte_tm_error *error) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + return RTE_TM_FUNC(port_id, mark_vlan_dei)(dev, + mark_green, mark_yellow, mark_red, error); +} + +/* Packet marking - IPv4/IPv6 ECN */ +int rte_tm_mark_ip_ecn(uint16_t port_id, + int mark_green, + int mark_yellow, + int mark_red, + struct rte_tm_error *error) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + return RTE_TM_FUNC(port_id, mark_ip_ecn)(dev, + mark_green, mark_yellow, mark_red, error); +} + +/* Packet marking - IPv4/IPv6 DSCP */ +int rte_tm_mark_ip_dscp(uint16_t port_id, + int mark_green, + int mark_yellow, + int mark_red, + struct rte_tm_error *error) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + return RTE_TM_FUNC(port_id, mark_ip_dscp)(dev, + mark_green, mark_yellow, mark_red, error); +} diff --git a/dpdk/lib/librte_ether/rte_tm.h b/dpdk/lib/librte_ether/rte_tm.h new file mode 100644 index 00000000..2b25a871 --- /dev/null +++ b/dpdk/lib/librte_ether/rte_tm.h @@ -0,0 +1,1912 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. + * Copyright(c) 2017 Cavium. + * Copyright(c) 2017 NXP. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __INCLUDE_RTE_TM_H__ +#define __INCLUDE_RTE_TM_H__ + +/** + * @file + * RTE Generic Traffic Manager API + * + * This interface provides the ability to configure the traffic manager in a + * generic way. It includes features such as: hierarchical scheduling, + * traffic shaping, congestion management, packet marking, etc. + * + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + */ + +#include + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Ethernet framing overhead. + * + * Overhead fields per Ethernet frame: + * 1. Preamble: 7 bytes; + * 2. Start of Frame Delimiter (SFD): 1 byte; + * 3. Inter-Frame Gap (IFG): 12 bytes. + * + * One of the typical values for the *pkt_length_adjust* field of the shaper + * profile. + * + * @see struct rte_tm_shaper_params + */ +#define RTE_TM_ETH_FRAMING_OVERHEAD 20 + +/** + * Ethernet framing overhead including the Frame Check Sequence (FCS) field. + * Useful when FCS is generated and added at the end of the Ethernet frame on + * TX side without any SW intervention. + * + * One of the typical values for the pkt_length_adjust field of the shaper + * profile. + * + * @see struct rte_tm_shaper_params + */ +#define RTE_TM_ETH_FRAMING_OVERHEAD_FCS 24 + +/** + * Invalid WRED profile ID. + * + * @see struct rte_tm_node_params + * @see rte_tm_node_add() + * @see rte_tm_node_wred_context_update() + */ +#define RTE_TM_WRED_PROFILE_ID_NONE UINT32_MAX + +/** + *Invalid shaper profile ID. + * + * @see struct rte_tm_node_params + * @see rte_tm_node_add() + * @see rte_tm_node_shaper_update() + */ +#define RTE_TM_SHAPER_PROFILE_ID_NONE UINT32_MAX + +/** + * Node ID for the parent of the root node. + * + * @see rte_tm_node_add() + */ +#define RTE_TM_NODE_ID_NULL UINT32_MAX + +/** + * Node level ID used to disable level ID checking. + * + * @see rte_tm_node_add() + */ +#define RTE_TM_NODE_LEVEL_ID_ANY UINT32_MAX + +/** + * Color + */ +enum rte_tm_color { + RTE_TM_GREEN = 0, /**< Green */ + RTE_TM_YELLOW, /**< Yellow */ + RTE_TM_RED, /**< Red */ + RTE_TM_COLORS /**< Number of colors */ +}; + +/** + * Node statistics counter type + */ +enum rte_tm_stats_type { + /** Number of packets scheduled from current node. */ + RTE_TM_STATS_N_PKTS = 1 << 0, + + /** Number of bytes scheduled from current node. */ + RTE_TM_STATS_N_BYTES = 1 << 1, + + /** Number of green packets dropped by current leaf node. */ + RTE_TM_STATS_N_PKTS_GREEN_DROPPED = 1 << 2, + + /** Number of yellow packets dropped by current leaf node. */ + RTE_TM_STATS_N_PKTS_YELLOW_DROPPED = 1 << 3, + + /** Number of red packets dropped by current leaf node. */ + RTE_TM_STATS_N_PKTS_RED_DROPPED = 1 << 4, + + /** Number of green bytes dropped by current leaf node. */ + RTE_TM_STATS_N_BYTES_GREEN_DROPPED = 1 << 5, + + /** Number of yellow bytes dropped by current leaf node. */ + RTE_TM_STATS_N_BYTES_YELLOW_DROPPED = 1 << 6, + + /** Number of red bytes dropped by current leaf node. */ + RTE_TM_STATS_N_BYTES_RED_DROPPED = 1 << 7, + + /** Number of packets currently waiting in the packet queue of current + * leaf node. + */ + RTE_TM_STATS_N_PKTS_QUEUED = 1 << 8, + + /** Number of bytes currently waiting in the packet queue of current + * leaf node. + */ + RTE_TM_STATS_N_BYTES_QUEUED = 1 << 9, +}; + +/** + * Node statistics counters + */ +struct rte_tm_node_stats { + /** Number of packets scheduled from current node. */ + uint64_t n_pkts; + + /** Number of bytes scheduled from current node. */ + uint64_t n_bytes; + + /** Statistics counters for leaf nodes only. */ + struct { + /** Number of packets dropped by current leaf node per each + * color. + */ + uint64_t n_pkts_dropped[RTE_TM_COLORS]; + + /** Number of bytes dropped by current leaf node per each + * color. + */ + uint64_t n_bytes_dropped[RTE_TM_COLORS]; + + /** Number of packets currently waiting in the packet queue of + * current leaf node. + */ + uint64_t n_pkts_queued; + + /** Number of bytes currently waiting in the packet queue of + * current leaf node. + */ + uint64_t n_bytes_queued; + } leaf; +}; + +/** + * Traffic manager dynamic updates + */ +enum rte_tm_dynamic_update_type { + /** Dynamic parent node update. The new parent node is located on same + * hierarchy level as the former parent node. Consequently, the node + * whose parent is changed preserves its hierarchy level. + */ + RTE_TM_UPDATE_NODE_PARENT_KEEP_LEVEL = 1 << 0, + + /** Dynamic parent node update. The new parent node is located on + * different hierarchy level than the former parent node. Consequently, + * the node whose parent is changed also changes its hierarchy level. + */ + RTE_TM_UPDATE_NODE_PARENT_CHANGE_LEVEL = 1 << 1, + + /** Dynamic node add/delete. */ + RTE_TM_UPDATE_NODE_ADD_DELETE = 1 << 2, + + /** Suspend/resume nodes. */ + RTE_TM_UPDATE_NODE_SUSPEND_RESUME = 1 << 3, + + /** Dynamic switch between byte-based and packet-based WFQ weights. */ + RTE_TM_UPDATE_NODE_WFQ_WEIGHT_MODE = 1 << 4, + + /** Dynamic update on number of SP priorities. */ + RTE_TM_UPDATE_NODE_N_SP_PRIORITIES = 1 << 5, + + /** Dynamic update of congestion management mode for leaf nodes. */ + RTE_TM_UPDATE_NODE_CMAN = 1 << 6, + + /** Dynamic update of the set of enabled stats counter types. */ + RTE_TM_UPDATE_NODE_STATS = 1 << 7, +}; + +/** + * Traffic manager capabilities + */ +struct rte_tm_capabilities { + /** Maximum number of nodes. */ + uint32_t n_nodes_max; + + /** Maximum number of levels (i.e. number of nodes connecting the root + * node with any leaf node, including the root and the leaf). + */ + uint32_t n_levels_max; + + /** When non-zero, this flag indicates that all the non-leaf nodes + * (with the exception of the root node) have identical capability set. + */ + int non_leaf_nodes_identical; + + /** When non-zero, this flag indicates that all the leaf nodes have + * identical capability set. + */ + int leaf_nodes_identical; + + /** Maximum number of shapers, either private or shared. In case the + * implementation does not share any resources between private and + * shared shapers, it is typically equal to the sum of + * *shaper_private_n_max* and *shaper_shared_n_max*. The + * value of zero indicates that traffic shaping is not supported. + */ + uint32_t shaper_n_max; + + /** Maximum number of private shapers. Indicates the maximum number of + * nodes that can concurrently have their private shaper enabled. The + * value of zero indicates that private shapers are not supported. + */ + uint32_t shaper_private_n_max; + + /** Maximum number of private shapers that support dual rate shaping. + * Indicates the maximum number of nodes that can concurrently have + * their private shaper enabled with dual rate support. Only valid when + * private shapers are supported. The value of zero indicates that dual + * rate shaping is not available for private shapers. The maximum value + * is *shaper_private_n_max*. + */ + int shaper_private_dual_rate_n_max; + + /** Minimum committed/peak rate (bytes per second) for any private + * shaper. Valid only when private shapers are supported. + */ + uint64_t shaper_private_rate_min; + + /** Maximum committed/peak rate (bytes per second) for any private + * shaper. Valid only when private shapers are supported. + */ + uint64_t shaper_private_rate_max; + + /** Maximum number of shared shapers. The value of zero indicates that + * shared shapers are not supported. + */ + uint32_t shaper_shared_n_max; + + /** Maximum number of nodes that can share the same shared shaper. + * Only valid when shared shapers are supported. + */ + uint32_t shaper_shared_n_nodes_per_shaper_max; + + /** Maximum number of shared shapers a node can be part of. This + * parameter indicates that there is at least one node that can be + * configured with this many shared shapers, which might not be true for + * all the nodes. Only valid when shared shapers are supported, in which + * case it ranges from 1 to *shaper_shared_n_max*. + */ + uint32_t shaper_shared_n_shapers_per_node_max; + + /** Maximum number of shared shapers that can be configured with dual + * rate shaping. The value of zero indicates that dual rate shaping + * support is not available for shared shapers. + */ + uint32_t shaper_shared_dual_rate_n_max; + + /** Minimum committed/peak rate (bytes per second) for any shared + * shaper. Only valid when shared shapers are supported. + */ + uint64_t shaper_shared_rate_min; + + /** Maximum committed/peak rate (bytes per second) for any shared + * shaper. Only valid when shared shapers are supported. + */ + uint64_t shaper_shared_rate_max; + + /** Minimum value allowed for packet length adjustment for any private + * or shared shaper. + */ + int shaper_pkt_length_adjust_min; + + /** Maximum value allowed for packet length adjustment for any private + * or shared shaper. + */ + int shaper_pkt_length_adjust_max; + + /** Maximum number of children nodes. This parameter indicates that + * there is at least one non-leaf node that can be configured with this + * many children nodes, which might not be true for all the non-leaf + * nodes. + */ + uint32_t sched_n_children_max; + + /** Maximum number of supported priority levels. This parameter + * indicates that there is at least one non-leaf node that can be + * configured with this many priority levels for managing its children + * nodes, which might not be true for all the non-leaf nodes. The value + * of zero is invalid. The value of 1 indicates that only priority 0 is + * supported, which essentially means that Strict Priority (SP) + * algorithm is not supported. + */ + uint32_t sched_sp_n_priorities_max; + + /** Maximum number of sibling nodes that can have the same priority at + * any given time, i.e. maximum size of the WFQ sibling node group. This + * parameter indicates there is at least one non-leaf node that meets + * this condition, which might not be true for all the non-leaf nodes. + * The value of zero is invalid. The value of 1 indicates that WFQ + * algorithm is not supported. The maximum value is + * *sched_n_children_max*. + */ + uint32_t sched_wfq_n_children_per_group_max; + + /** Maximum number of priority levels that can have more than one child + * node at any given time, i.e. maximum number of WFQ sibling node + * groups that have two or more members. This parameter indicates there + * is at least one non-leaf node that meets this condition, which might + * not be true for all the non-leaf nodes. The value of zero states that + * WFQ algorithm is not supported. The value of 1 indicates that + * (*sched_sp_n_priorities_max* - 1) priority levels have at most one + * child node, so there can be only one priority level with two or + * more sibling nodes making up a WFQ group. The maximum value is: + * min(floor(*sched_n_children_max* / 2), *sched_sp_n_priorities_max*). + */ + uint32_t sched_wfq_n_groups_max; + + /** Maximum WFQ weight. The value of 1 indicates that all sibling nodes + * with same priority have the same WFQ weight, so WFQ is reduced to FQ. + */ + uint32_t sched_wfq_weight_max; + + /** Head drop algorithm support. When non-zero, this parameter + * indicates that there is at least one leaf node that supports the head + * drop algorithm, which might not be true for all the leaf nodes. + */ + int cman_head_drop_supported; + + /** Maximum number of WRED contexts, either private or shared. In case + * the implementation does not share any resources between private and + * shared WRED contexts, it is typically equal to the sum of + * *cman_wred_context_private_n_max* and + * *cman_wred_context_shared_n_max*. The value of zero indicates that + * WRED is not supported. + */ + uint32_t cman_wred_context_n_max; + + /** Maximum number of private WRED contexts. Indicates the maximum + * number of leaf nodes that can concurrently have their private WRED + * context enabled. The value of zero indicates that private WRED + * contexts are not supported. + */ + uint32_t cman_wred_context_private_n_max; + + /** Maximum number of shared WRED contexts. The value of zero + * indicates that shared WRED contexts are not supported. + */ + uint32_t cman_wred_context_shared_n_max; + + /** Maximum number of leaf nodes that can share the same WRED context. + * Only valid when shared WRED contexts are supported. + */ + uint32_t cman_wred_context_shared_n_nodes_per_context_max; + + /** Maximum number of shared WRED contexts a leaf node can be part of. + * This parameter indicates that there is at least one leaf node that + * can be configured with this many shared WRED contexts, which might + * not be true for all the leaf nodes. Only valid when shared WRED + * contexts are supported, in which case it ranges from 1 to + * *cman_wred_context_shared_n_max*. + */ + uint32_t cman_wred_context_shared_n_contexts_per_node_max; + + /** Support for VLAN DEI packet marking (per color). */ + int mark_vlan_dei_supported[RTE_TM_COLORS]; + + /** Support for IPv4/IPv6 ECN marking of TCP packets (per color). */ + int mark_ip_ecn_tcp_supported[RTE_TM_COLORS]; + + /** Support for IPv4/IPv6 ECN marking of SCTP packets (per color). */ + int mark_ip_ecn_sctp_supported[RTE_TM_COLORS]; + + /** Support for IPv4/IPv6 DSCP packet marking (per color). */ + int mark_ip_dscp_supported[RTE_TM_COLORS]; + + /** Set of supported dynamic update operations. + * @see enum rte_tm_dynamic_update_type + */ + uint64_t dynamic_update_mask; + + /** Set of supported statistics counter types. + * @see enum rte_tm_stats_type + */ + uint64_t stats_mask; +}; + +/** + * Traffic manager level capabilities + */ +struct rte_tm_level_capabilities { + /** Maximum number of nodes for the current hierarchy level. */ + uint32_t n_nodes_max; + + /** Maximum number of non-leaf nodes for the current hierarchy level. + * The value of 0 indicates that current level only supports leaf + * nodes. The maximum value is *n_nodes_max*. + */ + uint32_t n_nodes_nonleaf_max; + + /** Maximum number of leaf nodes for the current hierarchy level. The + * value of 0 indicates that current level only supports non-leaf + * nodes. The maximum value is *n_nodes_max*. + */ + uint32_t n_nodes_leaf_max; + + /** When non-zero, this flag indicates that all the non-leaf nodes on + * this level have identical capability set. Valid only when + * *n_nodes_nonleaf_max* is non-zero. + */ + int non_leaf_nodes_identical; + + /** When non-zero, this flag indicates that all the leaf nodes on this + * level have identical capability set. Valid only when + * *n_nodes_leaf_max* is non-zero. + */ + int leaf_nodes_identical; + + RTE_STD_C11 + union { + /** Items valid only for the non-leaf nodes on this level. */ + struct { + /** Private shaper support. When non-zero, it indicates + * there is at least one non-leaf node on this level + * with private shaper support, which may not be the + * case for all the non-leaf nodes on this level. + */ + int shaper_private_supported; + + /** Dual rate support for private shaper. Valid only + * when private shaper is supported for the non-leaf + * nodes on the current level. When non-zero, it + * indicates there is at least one non-leaf node on this + * level with dual rate private shaper support, which + * may not be the case for all the non-leaf nodes on + * this level. + */ + int shaper_private_dual_rate_supported; + + /** Minimum committed/peak rate (bytes per second) for + * private shapers of the non-leaf nodes of this level. + * Valid only when private shaper is supported on this + * level. + */ + uint64_t shaper_private_rate_min; + + /** Maximum committed/peak rate (bytes per second) for + * private shapers of the non-leaf nodes on this level. + * Valid only when private shaper is supported on this + * level. + */ + uint64_t shaper_private_rate_max; + + /** Maximum number of shared shapers that any non-leaf + * node on this level can be part of. The value of zero + * indicates that shared shapers are not supported by + * the non-leaf nodes on this level. When non-zero, it + * indicates there is at least one non-leaf node on this + * level that meets this condition, which may not be the + * case for all the non-leaf nodes on this level. + */ + uint32_t shaper_shared_n_max; + + /** Maximum number of children nodes. This parameter + * indicates that there is at least one non-leaf node on + * this level that can be configured with this many + * children nodes, which might not be true for all the + * non-leaf nodes on this level. + */ + uint32_t sched_n_children_max; + + /** Maximum number of supported priority levels. This + * parameter indicates that there is at least one + * non-leaf node on this level that can be configured + * with this many priority levels for managing its + * children nodes, which might not be true for all the + * non-leaf nodes on this level. The value of zero is + * invalid. The value of 1 indicates that only priority + * 0 is supported, which essentially means that Strict + * Priority (SP) algorithm is not supported on this + * level. + */ + uint32_t sched_sp_n_priorities_max; + + /** Maximum number of sibling nodes that can have the + * same priority at any given time, i.e. maximum size of + * the WFQ sibling node group. This parameter indicates + * there is at least one non-leaf node on this level + * that meets this condition, which may not be true for + * all the non-leaf nodes on this level. The value of + * zero is invalid. The value of 1 indicates that WFQ + * algorithm is not supported on this level. The maximum + * value is *sched_n_children_max*. + */ + uint32_t sched_wfq_n_children_per_group_max; + + /** Maximum number of priority levels that can have + * more than one child node at any given time, i.e. + * maximum number of WFQ sibling node groups that + * have two or more members. This parameter indicates + * there is at least one non-leaf node on this level + * that meets this condition, which might not be true + * for all the non-leaf nodes. The value of zero states + * that WFQ algorithm is not supported on this level. + * The value of 1 indicates that + * (*sched_sp_n_priorities_max* - 1) priority levels on + * this level have at most one child node, so there can + * be only one priority level with two or more sibling + * nodes making up a WFQ group on this level. The + * maximum value is: + * min(floor(*sched_n_children_max* / 2), + * *sched_sp_n_priorities_max*). + */ + uint32_t sched_wfq_n_groups_max; + + /** Maximum WFQ weight. The value of 1 indicates that + * all sibling nodes on this level with same priority + * have the same WFQ weight, so on this level WFQ is + * reduced to FQ. + */ + uint32_t sched_wfq_weight_max; + + /** Mask of statistics counter types supported by the + * non-leaf nodes on this level. Every supported + * statistics counter type is supported by at least one + * non-leaf node on this level, which may not be true + * for all the non-leaf nodes on this level. + * @see enum rte_tm_stats_type + */ + uint64_t stats_mask; + } nonleaf; + + /** Items valid only for the leaf nodes on this level. */ + struct { + /** Private shaper support. When non-zero, it indicates + * there is at least one leaf node on this level with + * private shaper support, which may not be the case for + * all the leaf nodes on this level. + */ + int shaper_private_supported; + + /** Dual rate support for private shaper. Valid only + * when private shaper is supported for the leaf nodes + * on this level. When non-zero, it indicates there is + * at least one leaf node on this level with dual rate + * private shaper support, which may not be the case for + * all the leaf nodes on this level. + */ + int shaper_private_dual_rate_supported; + + /** Minimum committed/peak rate (bytes per second) for + * private shapers of the leaf nodes of this level. + * Valid only when private shaper is supported for the + * leaf nodes on this level. + */ + uint64_t shaper_private_rate_min; + + /** Maximum committed/peak rate (bytes per second) for + * private shapers of the leaf nodes on this level. + * Valid only when private shaper is supported for the + * leaf nodes on this level. + */ + uint64_t shaper_private_rate_max; + + /** Maximum number of shared shapers that any leaf node + * on this level can be part of. The value of zero + * indicates that shared shapers are not supported by + * the leaf nodes on this level. When non-zero, it + * indicates there is at least one leaf node on this + * level that meets this condition, which may not be the + * case for all the leaf nodes on this level. + */ + uint32_t shaper_shared_n_max; + + /** Head drop algorithm support. When non-zero, this + * parameter indicates that there is at least one leaf + * node on this level that supports the head drop + * algorithm, which might not be true for all the leaf + * nodes on this level. + */ + int cman_head_drop_supported; + + /** Private WRED context support. When non-zero, it + * indicates there is at least one node on this level + * with private WRED context support, which may not be + * true for all the leaf nodes on this level. + */ + int cman_wred_context_private_supported; + + /** Maximum number of shared WRED contexts that any + * leaf node on this level can be part of. The value of + * zero indicates that shared WRED contexts are not + * supported by the leaf nodes on this level. When + * non-zero, it indicates there is at least one leaf + * node on this level that meets this condition, which + * may not be the case for all the leaf nodes on this + * level. + */ + uint32_t cman_wred_context_shared_n_max; + + /** Mask of statistics counter types supported by the + * leaf nodes on this level. Every supported statistics + * counter type is supported by at least one leaf node + * on this level, which may not be true for all the leaf + * nodes on this level. + * @see enum rte_tm_stats_type + */ + uint64_t stats_mask; + } leaf; + }; +}; + +/** + * Traffic manager node capabilities + */ +struct rte_tm_node_capabilities { + /** Private shaper support for the current node. */ + int shaper_private_supported; + + /** Dual rate shaping support for private shaper of current node. + * Valid only when private shaper is supported by the current node. + */ + int shaper_private_dual_rate_supported; + + /** Minimum committed/peak rate (bytes per second) for private + * shaper of current node. Valid only when private shaper is supported + * by the current node. + */ + uint64_t shaper_private_rate_min; + + /** Maximum committed/peak rate (bytes per second) for private + * shaper of current node. Valid only when private shaper is supported + * by the current node. + */ + uint64_t shaper_private_rate_max; + + /** Maximum number of shared shapers the current node can be part of. + * The value of zero indicates that shared shapers are not supported by + * the current node. + */ + uint32_t shaper_shared_n_max; + + RTE_STD_C11 + union { + /** Items valid only for non-leaf nodes. */ + struct { + /** Maximum number of children nodes. */ + uint32_t sched_n_children_max; + + /** Maximum number of supported priority levels. The + * value of zero is invalid. The value of 1 indicates + * that only priority 0 is supported, which essentially + * means that Strict Priority (SP) algorithm is not + * supported. + */ + uint32_t sched_sp_n_priorities_max; + + /** Maximum number of sibling nodes that can have the + * same priority at any given time, i.e. maximum size + * of the WFQ sibling node group. The value of zero + * is invalid. The value of 1 indicates that WFQ + * algorithm is not supported. The maximum value is + * *sched_n_children_max*. + */ + uint32_t sched_wfq_n_children_per_group_max; + + /** Maximum number of priority levels that can have + * more than one child node at any given time, i.e. + * maximum number of WFQ sibling node groups that have + * two or more members. The value of zero states that + * WFQ algorithm is not supported. The value of 1 + * indicates that (*sched_sp_n_priorities_max* - 1) + * priority levels have at most one child node, so there + * can be only one priority level with two or more + * sibling nodes making up a WFQ group. The maximum + * value is: min(floor(*sched_n_children_max* / 2), + * *sched_sp_n_priorities_max*). + */ + uint32_t sched_wfq_n_groups_max; + + /** Maximum WFQ weight. The value of 1 indicates that + * all sibling nodes with same priority have the same + * WFQ weight, so WFQ is reduced to FQ. + */ + uint32_t sched_wfq_weight_max; + } nonleaf; + + /** Items valid only for leaf nodes. */ + struct { + /** Head drop algorithm support for current node. */ + int cman_head_drop_supported; + + /** Private WRED context support for current node. */ + int cman_wred_context_private_supported; + + /** Maximum number of shared WRED contexts the current + * node can be part of. The value of zero indicates that + * shared WRED contexts are not supported by the current + * node. + */ + uint32_t cman_wred_context_shared_n_max; + } leaf; + }; + + /** Mask of statistics counter types supported by the current node. + * @see enum rte_tm_stats_type + */ + uint64_t stats_mask; +}; + +/** + * Congestion management (CMAN) mode + * + * This is used for controlling the admission of packets into a packet queue or + * group of packet queues on congestion. On request of writing a new packet + * into the current queue while the queue is full, the *tail drop* algorithm + * drops the new packet while leaving the queue unmodified, as opposed to *head + * drop* algorithm, which drops the packet at the head of the queue (the oldest + * packet waiting in the queue) and admits the new packet at the tail of the + * queue. + * + * The *Random Early Detection (RED)* algorithm works by proactively dropping + * more and more input packets as the queue occupancy builds up. When the queue + * is full or almost full, RED effectively works as *tail drop*. The *Weighted + * RED* algorithm uses a separate set of RED thresholds for each packet color. + */ +enum rte_tm_cman_mode { + RTE_TM_CMAN_TAIL_DROP = 0, /**< Tail drop */ + RTE_TM_CMAN_HEAD_DROP, /**< Head drop */ + RTE_TM_CMAN_WRED, /**< Weighted Random Early Detection (WRED) */ +}; + +/** + * Random Early Detection (RED) profile + */ +struct rte_tm_red_params { + /** Minimum queue threshold */ + uint16_t min_th; + + /** Maximum queue threshold */ + uint16_t max_th; + + /** Inverse of packet marking probability maximum value (maxp), i.e. + * maxp_inv = 1 / maxp + */ + uint16_t maxp_inv; + + /** Negated log2 of queue weight (wq), i.e. wq = 1 / (2 ^ wq_log2) */ + uint16_t wq_log2; +}; + +/** + * Weighted RED (WRED) profile + * + * Multiple WRED contexts can share the same WRED profile. Each leaf node with + * WRED enabled as its congestion management mode has zero or one private WRED + * context (only one leaf node using it) and/or zero, one or several shared + * WRED contexts (multiple leaf nodes use the same WRED context). A private + * WRED context is used to perform congestion management for a single leaf + * node, while a shared WRED context is used to perform congestion management + * for a group of leaf nodes. + */ +struct rte_tm_wred_params { + /** One set of RED parameters per packet color */ + struct rte_tm_red_params red_params[RTE_TM_COLORS]; +}; + +/** + * Token bucket + */ +struct rte_tm_token_bucket { + /** Token bucket rate (bytes per second) */ + uint64_t rate; + + /** Token bucket size (bytes), a.k.a. max burst size */ + uint64_t size; +}; + +/** + * Shaper (rate limiter) profile + * + * Multiple shaper instances can share the same shaper profile. Each node has + * zero or one private shaper (only one node using it) and/or zero, one or + * several shared shapers (multiple nodes use the same shaper instance). + * A private shaper is used to perform traffic shaping for a single node, while + * a shared shaper is used to perform traffic shaping for a group of nodes. + * + * Single rate shapers use a single token bucket. A single rate shaper can be + * configured by setting the rate of the committed bucket to zero, which + * effectively disables this bucket. The peak bucket is used to limit the rate + * and the burst size for the current shaper. + * + * Dual rate shapers use both the committed and the peak token buckets. The + * rate of the peak bucket has to be bigger than zero, as well as greater than + * or equal to the rate of the committed bucket. + */ +struct rte_tm_shaper_params { + /** Committed token bucket */ + struct rte_tm_token_bucket committed; + + /** Peak token bucket */ + struct rte_tm_token_bucket peak; + + /** Signed value to be added to the length of each packet for the + * purpose of shaping. Can be used to correct the packet length with + * the framing overhead bytes that are also consumed on the wire (e.g. + * RTE_TM_ETH_FRAMING_OVERHEAD_FCS). + */ + int32_t pkt_length_adjust; +}; + +/** + * Node parameters + * + * Each non-leaf node has multiple inputs (its children nodes) and single output + * (which is input to its parent node). It arbitrates its inputs using Strict + * Priority (SP) and Weighted Fair Queuing (WFQ) algorithms to schedule input + * packets to its output while observing its shaping (rate limiting) + * constraints. + * + * Algorithms such as Weighted Round Robin (WRR), Byte-level WRR, Deficit WRR + * (DWRR), etc. are considered approximations of the WFQ ideal and are + * assimilated to WFQ, although an associated implementation-dependent trade-off + * on accuracy, performance and resource usage might exist. + * + * Children nodes with different priorities are scheduled using the SP algorithm + * based on their priority, with zero (0) as the highest priority. Children with + * the same priority are scheduled using the WFQ algorithm according to their + * weights. The WFQ weight of a given child node is relative to the sum of the + * weights of all its sibling nodes that have the same priority, with one (1) as + * the lowest weight. For each SP priority, the WFQ weight mode can be set as + * either byte-based or packet-based. + * + * Each leaf node sits on top of a TX queue of the current Ethernet port. Hence, + * the leaf nodes are predefined, with their node IDs set to 0 .. (N-1), where N + * is the number of TX queues configured for the current Ethernet port. The + * non-leaf nodes have their IDs generated by the application. + */ +struct rte_tm_node_params { + /** Shaper profile for the private shaper. The absence of the private + * shaper for the current node is indicated by setting this parameter + * to RTE_TM_SHAPER_PROFILE_ID_NONE. + */ + uint32_t shaper_profile_id; + + /** User allocated array of valid shared shaper IDs. */ + uint32_t *shared_shaper_id; + + /** Number of shared shaper IDs in the *shared_shaper_id* array. */ + uint32_t n_shared_shapers; + + RTE_STD_C11 + union { + /** Parameters only valid for non-leaf nodes. */ + struct { + /** WFQ weight mode for each SP priority. When NULL, it + * indicates that WFQ is to be used for all priorities. + * When non-NULL, it points to a pre-allocated array of + * *n_sp_priorities* values, with non-zero value for + * byte-mode and zero for packet-mode. + */ + int *wfq_weight_mode; + + /** Number of SP priorities. */ + uint32_t n_sp_priorities; + } nonleaf; + + /** Parameters only valid for leaf nodes. */ + struct { + /** Congestion management mode */ + enum rte_tm_cman_mode cman; + + /** WRED parameters (only valid when *cman* is set to + * WRED). + */ + struct { + /** WRED profile for private WRED context. The + * absence of a private WRED context for the + * current leaf node is indicated by value + * RTE_TM_WRED_PROFILE_ID_NONE. + */ + uint32_t wred_profile_id; + + /** User allocated array of shared WRED context + * IDs. When set to NULL, it indicates that the + * current leaf node should not currently be + * part of any shared WRED contexts. + */ + uint32_t *shared_wred_context_id; + + /** Number of elements in the + * *shared_wred_context_id* array. Only valid + * when *shared_wred_context_id* is non-NULL, + * in which case it should be non-zero. + */ + uint32_t n_shared_wred_contexts; + } wred; + } leaf; + }; + + /** Mask of statistics counter types to be enabled for this node. This + * needs to be a subset of the statistics counter types available for + * the current node. Any statistics counter type not included in this + * set is to be disabled for the current node. + * @see enum rte_tm_stats_type + */ + uint64_t stats_mask; +}; + +/** + * Verbose error types. + * + * Most of them provide the type of the object referenced by struct + * rte_tm_error::cause. + */ +enum rte_tm_error_type { + RTE_TM_ERROR_TYPE_NONE, /**< No error. */ + RTE_TM_ERROR_TYPE_UNSPECIFIED, /**< Cause unspecified. */ + RTE_TM_ERROR_TYPE_CAPABILITIES, + RTE_TM_ERROR_TYPE_LEVEL_ID, + RTE_TM_ERROR_TYPE_WRED_PROFILE, + RTE_TM_ERROR_TYPE_WRED_PROFILE_GREEN, + RTE_TM_ERROR_TYPE_WRED_PROFILE_YELLOW, + RTE_TM_ERROR_TYPE_WRED_PROFILE_RED, + RTE_TM_ERROR_TYPE_WRED_PROFILE_ID, + RTE_TM_ERROR_TYPE_SHARED_WRED_CONTEXT_ID, + RTE_TM_ERROR_TYPE_SHAPER_PROFILE, + RTE_TM_ERROR_TYPE_SHAPER_PROFILE_COMMITTED_RATE, + RTE_TM_ERROR_TYPE_SHAPER_PROFILE_COMMITTED_SIZE, + RTE_TM_ERROR_TYPE_SHAPER_PROFILE_PEAK_RATE, + RTE_TM_ERROR_TYPE_SHAPER_PROFILE_PEAK_SIZE, + RTE_TM_ERROR_TYPE_SHAPER_PROFILE_PKT_ADJUST_LEN, + RTE_TM_ERROR_TYPE_SHAPER_PROFILE_ID, + RTE_TM_ERROR_TYPE_SHARED_SHAPER_ID, + RTE_TM_ERROR_TYPE_NODE_PARENT_NODE_ID, + RTE_TM_ERROR_TYPE_NODE_PRIORITY, + RTE_TM_ERROR_TYPE_NODE_WEIGHT, + RTE_TM_ERROR_TYPE_NODE_PARAMS, + RTE_TM_ERROR_TYPE_NODE_PARAMS_SHAPER_PROFILE_ID, + RTE_TM_ERROR_TYPE_NODE_PARAMS_SHARED_SHAPER_ID, + RTE_TM_ERROR_TYPE_NODE_PARAMS_N_SHARED_SHAPERS, + RTE_TM_ERROR_TYPE_NODE_PARAMS_WFQ_WEIGHT_MODE, + RTE_TM_ERROR_TYPE_NODE_PARAMS_N_SP_PRIORITIES, + RTE_TM_ERROR_TYPE_NODE_PARAMS_CMAN, + RTE_TM_ERROR_TYPE_NODE_PARAMS_WRED_PROFILE_ID, + RTE_TM_ERROR_TYPE_NODE_PARAMS_SHARED_WRED_CONTEXT_ID, + RTE_TM_ERROR_TYPE_NODE_PARAMS_N_SHARED_WRED_CONTEXTS, + RTE_TM_ERROR_TYPE_NODE_PARAMS_STATS, + RTE_TM_ERROR_TYPE_NODE_ID, +}; + +/** + * Verbose error structure definition. + * + * This object is normally allocated by applications and set by PMDs, the + * message points to a constant string which does not need to be freed by + * the application, however its pointer can be considered valid only as long + * as its associated DPDK port remains configured. Closing the underlying + * device or unloading the PMD invalidates it. + * + * Both cause and message may be NULL regardless of the error type. + */ +struct rte_tm_error { + enum rte_tm_error_type type; /**< Cause field and error type. */ + const void *cause; /**< Object responsible for the error. */ + const char *message; /**< Human-readable error message. */ +}; + +/** + * Traffic manager get number of leaf nodes + * + * Each leaf node sits on on top of a TX queue of the current Ethernet port. + * Therefore, the set of leaf nodes is predefined, their number is always equal + * to N (where N is the number of TX queues configured for the current port) + * and their IDs are 0 .. (N-1). + * + * @param[in] port_id + * The port identifier of the Ethernet device. + * @param[out] n_leaf_nodes + * Number of leaf nodes for the current port. + * @param[out] error + * Error details. Filled in only on error, when not NULL. + * @return + * 0 on success, non-zero error code otherwise. + */ +int +rte_tm_get_number_of_leaf_nodes(uint16_t port_id, + uint32_t *n_leaf_nodes, + struct rte_tm_error *error); + +/** + * Traffic manager node ID validate and type (i.e. leaf or non-leaf) get + * + * The leaf nodes have predefined IDs in the range of 0 .. (N-1), where N is + * the number of TX queues of the current Ethernet port. The non-leaf nodes + * have their IDs generated by the application outside of the above range, + * which is reserved for leaf nodes. + * + * @param[in] port_id + * The port identifier of the Ethernet device. + * @param[in] node_id + * Node ID value. Needs to be valid. + * @param[out] is_leaf + * Set to non-zero value when node is leaf and to zero otherwise (non-leaf). + * @param[out] error + * Error details. Filled in only on error, when not NULL. + * @return + * 0 on success, non-zero error code otherwise. + */ +int +rte_tm_node_type_get(uint16_t port_id, + uint32_t node_id, + int *is_leaf, + struct rte_tm_error *error); + +/** + * Traffic manager capabilities get + * + * @param[in] port_id + * The port identifier of the Ethernet device. + * @param[out] cap + * Traffic manager capabilities. Needs to be pre-allocated and valid. + * @param[out] error + * Error details. Filled in only on error, when not NULL. + * @return + * 0 on success, non-zero error code otherwise. + */ +int +rte_tm_capabilities_get(uint16_t port_id, + struct rte_tm_capabilities *cap, + struct rte_tm_error *error); + +/** + * Traffic manager level capabilities get + * + * @param[in] port_id + * The port identifier of the Ethernet device. + * @param[in] level_id + * The hierarchy level identifier. The value of 0 identifies the level of the + * root node. + * @param[out] cap + * Traffic manager level capabilities. Needs to be pre-allocated and valid. + * @param[out] error + * Error details. Filled in only on error, when not NULL. + * @return + * 0 on success, non-zero error code otherwise. + */ +int +rte_tm_level_capabilities_get(uint16_t port_id, + uint32_t level_id, + struct rte_tm_level_capabilities *cap, + struct rte_tm_error *error); + +/** + * Traffic manager node capabilities get + * + * @param[in] port_id + * The port identifier of the Ethernet device. + * @param[in] node_id + * Node ID. Needs to be valid. + * @param[out] cap + * Traffic manager node capabilities. Needs to be pre-allocated and valid. + * @param[out] error + * Error details. Filled in only on error, when not NULL. + * @return + * 0 on success, non-zero error code otherwise. + */ +int +rte_tm_node_capabilities_get(uint16_t port_id, + uint32_t node_id, + struct rte_tm_node_capabilities *cap, + struct rte_tm_error *error); + +/** + * Traffic manager WRED profile add + * + * Create a new WRED profile with ID set to *wred_profile_id*. The new profile + * is used to create one or several WRED contexts. + * + * @param[in] port_id + * The port identifier of the Ethernet device. + * @param[in] wred_profile_id + * WRED profile ID for the new profile. Needs to be unused. + * @param[in] profile + * WRED profile parameters. Needs to be pre-allocated and valid. + * @param[out] error + * Error details. Filled in only on error, when not NULL. + * @return + * 0 on success, non-zero error code otherwise. + * + * @see struct rte_tm_capabilities::cman_wred_context_n_max + */ +int +rte_tm_wred_profile_add(uint16_t port_id, + uint32_t wred_profile_id, + struct rte_tm_wred_params *profile, + struct rte_tm_error *error); + +/** + * Traffic manager WRED profile delete + * + * Delete an existing WRED profile. This operation fails when there is + * currently at least one user (i.e. WRED context) of this WRED profile. + * + * @param[in] port_id + * The port identifier of the Ethernet device. + * @param[in] wred_profile_id + * WRED profile ID. Needs to be the valid. + * @param[out] error + * Error details. Filled in only on error, when not NULL. + * @return + * 0 on success, non-zero error code otherwise. + * + * @see struct rte_tm_capabilities::cman_wred_context_n_max + */ +int +rte_tm_wred_profile_delete(uint16_t port_id, + uint32_t wred_profile_id, + struct rte_tm_error *error); + +/** + * Traffic manager shared WRED context add or update + * + * When *shared_wred_context_id* is invalid, a new WRED context with this ID is + * created by using the WRED profile identified by *wred_profile_id*. + * + * When *shared_wred_context_id* is valid, this WRED context is no longer using + * the profile previously assigned to it and is updated to use the profile + * identified by *wred_profile_id*. + * + * A valid shared WRED context can be assigned to several hierarchy leaf nodes + * configured to use WRED as the congestion management mode. + * + * @param[in] port_id + * The port identifier of the Ethernet device. + * @param[in] shared_wred_context_id + * Shared WRED context ID + * @param[in] wred_profile_id + * WRED profile ID. Needs to be the valid. + * @param[out] error + * Error details. Filled in only on error, when not NULL. + * @return + * 0 on success, non-zero error code otherwise. + * + * @see struct rte_tm_capabilities::cman_wred_context_shared_n_max + */ +int +rte_tm_shared_wred_context_add_update(uint16_t port_id, + uint32_t shared_wred_context_id, + uint32_t wred_profile_id, + struct rte_tm_error *error); + +/** + * Traffic manager shared WRED context delete + * + * Delete an existing shared WRED context. This operation fails when there is + * currently at least one user (i.e. hierarchy leaf node) of this shared WRED + * context. + * + * @param[in] port_id + * The port identifier of the Ethernet device. + * @param[in] shared_wred_context_id + * Shared WRED context ID. Needs to be the valid. + * @param[out] error + * Error details. Filled in only on error, when not NULL. + * @return + * 0 on success, non-zero error code otherwise. + * + * @see struct rte_tm_capabilities::cman_wred_context_shared_n_max + */ +int +rte_tm_shared_wred_context_delete(uint16_t port_id, + uint32_t shared_wred_context_id, + struct rte_tm_error *error); + +/** + * Traffic manager shaper profile add + * + * Create a new shaper profile with ID set to *shaper_profile_id*. The new + * shaper profile is used to create one or several shapers. + * + * @param[in] port_id + * The port identifier of the Ethernet device. + * @param[in] shaper_profile_id + * Shaper profile ID for the new profile. Needs to be unused. + * @param[in] profile + * Shaper profile parameters. Needs to be pre-allocated and valid. + * @param[out] error + * Error details. Filled in only on error, when not NULL. + * @return + * 0 on success, non-zero error code otherwise. + * + * @see struct rte_tm_capabilities::shaper_n_max + */ +int +rte_tm_shaper_profile_add(uint16_t port_id, + uint32_t shaper_profile_id, + struct rte_tm_shaper_params *profile, + struct rte_tm_error *error); + +/** + * Traffic manager shaper profile delete + * + * Delete an existing shaper profile. This operation fails when there is + * currently at least one user (i.e. shaper) of this shaper profile. + * + * @param[in] port_id + * The port identifier of the Ethernet device. + * @param[in] shaper_profile_id + * Shaper profile ID. Needs to be the valid. + * @param[out] error + * Error details. Filled in only on error, when not NULL. + * @return + * 0 on success, non-zero error code otherwise. + * + * @see struct rte_tm_capabilities::shaper_n_max + */ +int +rte_tm_shaper_profile_delete(uint16_t port_id, + uint32_t shaper_profile_id, + struct rte_tm_error *error); + +/** + * Traffic manager shared shaper add or update + * + * When *shared_shaper_id* is not a valid shared shaper ID, a new shared shaper + * with this ID is created using the shaper profile identified by + * *shaper_profile_id*. + * + * When *shared_shaper_id* is a valid shared shaper ID, this shared shaper is + * no longer using the shaper profile previously assigned to it and is updated + * to use the shaper profile identified by *shaper_profile_id*. + * + * @param[in] port_id + * The port identifier of the Ethernet device. + * @param[in] shared_shaper_id + * Shared shaper ID + * @param[in] shaper_profile_id + * Shaper profile ID. Needs to be the valid. + * @param[out] error + * Error details. Filled in only on error, when not NULL. + * @return + * 0 on success, non-zero error code otherwise. + * + * @see struct rte_tm_capabilities::shaper_shared_n_max + */ +int +rte_tm_shared_shaper_add_update(uint16_t port_id, + uint32_t shared_shaper_id, + uint32_t shaper_profile_id, + struct rte_tm_error *error); + +/** + * Traffic manager shared shaper delete + * + * Delete an existing shared shaper. This operation fails when there is + * currently at least one user (i.e. hierarchy node) of this shared shaper. + * + * @param[in] port_id + * The port identifier of the Ethernet device. + * @param[in] shared_shaper_id + * Shared shaper ID. Needs to be the valid. + * @param[out] error + * Error details. Filled in only on error, when not NULL. + * @return + * 0 on success, non-zero error code otherwise. + * + * @see struct rte_tm_capabilities::shaper_shared_n_max + */ +int +rte_tm_shared_shaper_delete(uint16_t port_id, + uint32_t shared_shaper_id, + struct rte_tm_error *error); + +/** + * Traffic manager node add + * + * Create new node and connect it as child of an existing node. The new node is + * further identified by *node_id*, which needs to be unused by any of the + * existing nodes. The parent node is identified by *parent_node_id*, which + * needs to be the valid ID of an existing non-leaf node. The parent node is + * going to use the provided SP *priority* and WFQ *weight* to schedule its new + * child node. + * + * This function has to be called for both leaf and non-leaf nodes. In the case + * of leaf nodes (i.e. *node_id* is within the range of 0 .. (N-1), with N as + * the number of configured TX queues of the current port), the leaf node is + * configured rather than created (as the set of leaf nodes is predefined) and + * it is also connected as child of an existing node. + * + * The first node that is added becomes the root node and all the nodes that + * are subsequently added have to be added as descendants of the root node. The + * parent of the root node has to be specified as RTE_TM_NODE_ID_NULL and there + * can only be one node with this parent ID (i.e. the root node). Further + * restrictions for root node: needs to be non-leaf, its private shaper profile + * needs to be valid and single rate, cannot use any shared shapers. + * + * When called before rte_tm_hierarchy_commit() invocation, this function is + * typically used to define the initial start-up hierarchy for the port. + * Provided that dynamic hierarchy updates are supported by the current port (as + * advertised in the port capability set), this function can be also called + * after the rte_tm_hierarchy_commit() invocation. + * + * @param[in] port_id + * The port identifier of the Ethernet device. + * @param[in] node_id + * Node ID. Needs to be unused by any of the existing nodes. + * @param[in] parent_node_id + * Parent node ID. Needs to be the valid. + * @param[in] priority + * Node priority. The highest node priority is zero. Used by the SP algorithm + * running on the parent of the current node for scheduling this child node. + * @param[in] weight + * Node weight. The node weight is relative to the weight sum of all siblings + * that have the same priority. The lowest weight is one. Used by the WFQ + * algorithm running on the parent of the current node for scheduling this + * child node. + * @param[in] level_id + * Level ID that should be met by this node. The hierarchy level of the + * current node is already fully specified through its parent node (i.e. the + * level of this node is equal to the level of its parent node plus one), + * therefore the reason for providing this parameter is to enable the + * application to perform step-by-step checking of the node level during + * successive invocations of this function. When not desired, this check can + * be disabled by assigning value RTE_TM_NODE_LEVEL_ID_ANY to this parameter. + * @param[in] params + * Node parameters. Needs to be pre-allocated and valid. + * @param[out] error + * Error details. Filled in only on error, when not NULL. + * @return + * 0 on success, non-zero error code otherwise. + * + * @see rte_tm_hierarchy_commit() + * @see RTE_TM_UPDATE_NODE_ADD_DELETE + * @see RTE_TM_NODE_LEVEL_ID_ANY + * @see struct rte_tm_capabilities + */ +int +rte_tm_node_add(uint16_t port_id, + uint32_t node_id, + uint32_t parent_node_id, + uint32_t priority, + uint32_t weight, + uint32_t level_id, + struct rte_tm_node_params *params, + struct rte_tm_error *error); + +/** + * Traffic manager node delete + * + * Delete an existing node. This operation fails when this node currently has + * at least one user (i.e. child node). + * + * When called before rte_tm_hierarchy_commit() invocation, this function is + * typically used to define the initial start-up hierarchy for the port. + * Provided that dynamic hierarchy updates are supported by the current port (as + * advertised in the port capability set), this function can be also called + * after the rte_tm_hierarchy_commit() invocation. + * + * @param[in] port_id + * The port identifier of the Ethernet device. + * @param[in] node_id + * Node ID. Needs to be valid. + * @param[out] error + * Error details. Filled in only on error, when not NULL. + * @return + * 0 on success, non-zero error code otherwise. + * + * @see RTE_TM_UPDATE_NODE_ADD_DELETE + */ +int +rte_tm_node_delete(uint16_t port_id, + uint32_t node_id, + struct rte_tm_error *error); + +/** + * Traffic manager node suspend + * + * Suspend an existing node. While the node is in suspended state, no packet is + * scheduled from this node and its descendants. The node exits the suspended + * state through the node resume operation. + * + * @param[in] port_id + * The port identifier of the Ethernet device. + * @param[in] node_id + * Node ID. Needs to be valid. + * @param[out] error + * Error details. Filled in only on error, when not NULL. + * @return + * 0 on success, non-zero error code otherwise. + * + * @see rte_tm_node_resume() + * @see RTE_TM_UPDATE_NODE_SUSPEND_RESUME + */ +int +rte_tm_node_suspend(uint16_t port_id, + uint32_t node_id, + struct rte_tm_error *error); + +/** + * Traffic manager node resume + * + * Resume an existing node that is currently in suspended state. The node + * entered the suspended state as result of a previous node suspend operation. + * + * @param[in] port_id + * The port identifier of the Ethernet device. + * @param[in] node_id + * Node ID. Needs to be valid. + * @param[out] error + * Error details. Filled in only on error, when not NULL. + * @return + * 0 on success, non-zero error code otherwise. + * + * @see rte_tm_node_suspend() + * @see RTE_TM_UPDATE_NODE_SUSPEND_RESUME + */ +int +rte_tm_node_resume(uint16_t port_id, + uint32_t node_id, + struct rte_tm_error *error); + +/** + * Traffic manager hierarchy commit + * + * This function is called during the port initialization phase (before the + * Ethernet port is started) to freeze the start-up hierarchy. + * + * This function typically performs the following steps: + * a) It validates the start-up hierarchy that was previously defined for the + * current port through successive rte_tm_node_add() invocations; + * b) Assuming successful validation, it performs all the necessary port + * specific configuration operations to install the specified hierarchy on + * the current port, with immediate effect once the port is started. + * + * This function fails when the currently configured hierarchy is not supported + * by the Ethernet port, in which case the user can abort or try out another + * hierarchy configuration (e.g. a hierarchy with less leaf nodes), which can be + * build from scratch (when *clear_on_fail* is enabled) or by modifying the + * existing hierarchy configuration (when *clear_on_fail* is disabled). + * + * Note that this function can still fail due to other causes (e.g. not enough + * memory available in the system, etc), even though the specified hierarchy is + * supported in principle by the current port. + * + * @param[in] port_id + * The port identifier of the Ethernet device. + * @param[in] clear_on_fail + * On function call failure, hierarchy is cleared when this parameter is + * non-zero and preserved when this parameter is equal to zero. + * @param[out] error + * Error details. Filled in only on error, when not NULL. + * @return + * 0 on success, non-zero error code otherwise. + * + * @see rte_tm_node_add() + * @see rte_tm_node_delete() + */ +int +rte_tm_hierarchy_commit(uint16_t port_id, + int clear_on_fail, + struct rte_tm_error *error); + +/** + * Traffic manager node parent update + * + * Restriction for root node: its parent cannot be changed. + * + * This function can only be called after the rte_tm_hierarchy_commit() + * invocation. Its success depends on the port support for this operation, as + * advertised through the port capability set. + * + * @param[in] port_id + * The port identifier of the Ethernet device. + * @param[in] node_id + * Node ID. Needs to be valid. + * @param[in] parent_node_id + * Node ID for the new parent. Needs to be valid. + * @param[in] priority + * Node priority. The highest node priority is zero. Used by the SP algorithm + * running on the parent of the current node for scheduling this child node. + * @param[in] weight + * Node weight. The node weight is relative to the weight sum of all siblings + * that have the same priority. The lowest weight is zero. Used by the WFQ + * algorithm running on the parent of the current node for scheduling this + * child node. + * @param[out] error + * Error details. Filled in only on error, when not NULL. + * @return + * 0 on success, non-zero error code otherwise. + * + * @see RTE_TM_UPDATE_NODE_PARENT_KEEP_LEVEL + * @see RTE_TM_UPDATE_NODE_PARENT_CHANGE_LEVEL + */ +int +rte_tm_node_parent_update(uint16_t port_id, + uint32_t node_id, + uint32_t parent_node_id, + uint32_t priority, + uint32_t weight, + struct rte_tm_error *error); + +/** + * Traffic manager node private shaper update + * + * Restriction for the root node: its private shaper profile needs to be valid + * and single rate. + * + * @param[in] port_id + * The port identifier of the Ethernet device. + * @param[in] node_id + * Node ID. Needs to be valid. + * @param[in] shaper_profile_id + * Shaper profile ID for the private shaper of the current node. Needs to be + * either valid shaper profile ID or RTE_TM_SHAPER_PROFILE_ID_NONE, with + * the latter disabling the private shaper of the current node. + * @param[out] error + * Error details. Filled in only on error, when not NULL. + * @return + * 0 on success, non-zero error code otherwise. + * + * @see struct rte_tm_capabilities::shaper_private_n_max + */ +int +rte_tm_node_shaper_update(uint16_t port_id, + uint32_t node_id, + uint32_t shaper_profile_id, + struct rte_tm_error *error); + +/** + * Traffic manager node shared shapers update + * + * Restriction for root node: cannot use any shared rate shapers. + * + * @param[in] port_id + * The port identifier of the Ethernet device. + * @param[in] node_id + * Node ID. Needs to be valid. + * @param[in] shared_shaper_id + * Shared shaper ID. Needs to be valid. + * @param[in] add + * Set to non-zero value to add this shared shaper to current node or to zero + * to delete this shared shaper from current node. + * @param[out] error + * Error details. Filled in only on error, when not NULL. + * @return + * 0 on success, non-zero error code otherwise. + * + * @see struct rte_tm_capabilities::shaper_shared_n_max + */ +int +rte_tm_node_shared_shaper_update(uint16_t port_id, + uint32_t node_id, + uint32_t shared_shaper_id, + int add, + struct rte_tm_error *error); + +/** + * Traffic manager node enabled statistics counters update + * + * @param[in] port_id + * The port identifier of the Ethernet device. + * @param[in] node_id + * Node ID. Needs to be valid. + * @param[in] stats_mask + * Mask of statistics counter types to be enabled for the current node. This + * needs to be a subset of the statistics counter types available for the + * current node. Any statistics counter type not included in this set is to + * be disabled for the current node. + * @param[out] error + * Error details. Filled in only on error, when not NULL. + * @return + * 0 on success, non-zero error code otherwise. + * + * @see enum rte_tm_stats_type + * @see RTE_TM_UPDATE_NODE_STATS + */ +int +rte_tm_node_stats_update(uint16_t port_id, + uint32_t node_id, + uint64_t stats_mask, + struct rte_tm_error *error); + +/** + * Traffic manager node WFQ weight mode update + * + * @param[in] port_id + * The port identifier of the Ethernet device. + * @param[in] node_id + * Node ID. Needs to be valid leaf node ID. + * @param[in] wfq_weight_mode + * WFQ weight mode for each SP priority. When NULL, it indicates that WFQ is + * to be used for all priorities. When non-NULL, it points to a pre-allocated + * array of *n_sp_priorities* values, with non-zero value for byte-mode and + * zero for packet-mode. + * @param[in] n_sp_priorities + * Number of SP priorities. + * @param[out] error + * Error details. Filled in only on error, when not NULL. + * @return + * 0 on success, non-zero error code otherwise. + * + * @see RTE_TM_UPDATE_NODE_WFQ_WEIGHT_MODE + * @see RTE_TM_UPDATE_NODE_N_SP_PRIORITIES + */ +int +rte_tm_node_wfq_weight_mode_update(uint16_t port_id, + uint32_t node_id, + int *wfq_weight_mode, + uint32_t n_sp_priorities, + struct rte_tm_error *error); + +/** + * Traffic manager node congestion management mode update + * + * @param[in] port_id + * The port identifier of the Ethernet device. + * @param[in] node_id + * Node ID. Needs to be valid leaf node ID. + * @param[in] cman + * Congestion management mode. + * @param[out] error + * Error details. Filled in only on error, when not NULL. + * @return + * 0 on success, non-zero error code otherwise. + * + * @see RTE_TM_UPDATE_NODE_CMAN + */ +int +rte_tm_node_cman_update(uint16_t port_id, + uint32_t node_id, + enum rte_tm_cman_mode cman, + struct rte_tm_error *error); + +/** + * Traffic manager node private WRED context update + * + * @param[in] port_id + * The port identifier of the Ethernet device. + * @param[in] node_id + * Node ID. Needs to be valid leaf node ID. + * @param[in] wred_profile_id + * WRED profile ID for the private WRED context of the current node. Needs to + * be either valid WRED profile ID or RTE_TM_WRED_PROFILE_ID_NONE, with the + * latter disabling the private WRED context of the current node. + * @param[out] error + * Error details. Filled in only on error, when not NULL. + * @return + * 0 on success, non-zero error code otherwise. + * + * @see struct rte_tm_capabilities::cman_wred_context_private_n_max +*/ +int +rte_tm_node_wred_context_update(uint16_t port_id, + uint32_t node_id, + uint32_t wred_profile_id, + struct rte_tm_error *error); + +/** + * Traffic manager node shared WRED context update + * + * @param[in] port_id + * The port identifier of the Ethernet device. + * @param[in] node_id + * Node ID. Needs to be valid leaf node ID. + * @param[in] shared_wred_context_id + * Shared WRED context ID. Needs to be valid. + * @param[in] add + * Set to non-zero value to add this shared WRED context to current node or + * to zero to delete this shared WRED context from current node. + * @param[out] error + * Error details. Filled in only on error, when not NULL. + * @return + * 0 on success, non-zero error code otherwise. + * + * @see struct rte_tm_capabilities::cman_wred_context_shared_n_max + */ +int +rte_tm_node_shared_wred_context_update(uint16_t port_id, + uint32_t node_id, + uint32_t shared_wred_context_id, + int add, + struct rte_tm_error *error); + +/** + * Traffic manager node statistics counters read + * + * @param[in] port_id + * The port identifier of the Ethernet device. + * @param[in] node_id + * Node ID. Needs to be valid. + * @param[out] stats + * When non-NULL, it contains the current value for the statistics counters + * enabled for the current node. + * @param[out] stats_mask + * When non-NULL, it contains the mask of statistics counter types that are + * currently enabled for this node, indicating which of the counters + * retrieved with the *stats* structure are valid. + * @param[in] clear + * When this parameter has a non-zero value, the statistics counters are + * cleared (i.e. set to zero) immediately after they have been read, + * otherwise the statistics counters are left untouched. + * @param[out] error + * Error details. Filled in only on error, when not NULL. + * @return + * 0 on success, non-zero error code otherwise. + * + * @see enum rte_tm_stats_type + */ +int +rte_tm_node_stats_read(uint16_t port_id, + uint32_t node_id, + struct rte_tm_node_stats *stats, + uint64_t *stats_mask, + int clear, + struct rte_tm_error *error); + +/** + * Traffic manager packet marking - VLAN DEI (IEEE 802.1Q) + * + * IEEE 802.1p maps the traffic class to the VLAN Priority Code Point (PCP) + * field (3 bits), while IEEE 802.1q maps the drop priority to the VLAN Drop + * Eligible Indicator (DEI) field (1 bit), which was previously named Canonical + * Format Indicator (CFI). + * + * All VLAN frames of a given color get their DEI bit set if marking is enabled + * for this color; otherwise, their DEI bit is left as is (either set or not). + * + * @param[in] port_id + * The port identifier of the Ethernet device. + * @param[in] mark_green + * Set to non-zero value to enable marking of green packets and to zero to + * disable it. + * @param[in] mark_yellow + * Set to non-zero value to enable marking of yellow packets and to zero to + * disable it. + * @param[in] mark_red + * Set to non-zero value to enable marking of red packets and to zero to + * disable it. + * @param[out] error + * Error details. Filled in only on error, when not NULL. + * @return + * 0 on success, non-zero error code otherwise. + * + * @see struct rte_tm_capabilities::mark_vlan_dei_supported + */ +int +rte_tm_mark_vlan_dei(uint16_t port_id, + int mark_green, + int mark_yellow, + int mark_red, + struct rte_tm_error *error); + +/** + * Traffic manager packet marking - IPv4 / IPv6 ECN (IETF RFC 3168) + * + * IETF RFCs 2474 and 3168 reorganize the IPv4 Type of Service (TOS) field + * (8 bits) and the IPv6 Traffic Class (TC) field (8 bits) into Differentiated + * Services Codepoint (DSCP) field (6 bits) and Explicit Congestion + * Notification (ECN) field (2 bits). The DSCP field is typically used to + * encode the traffic class and/or drop priority (RFC 2597), while the ECN + * field is used by RFC 3168 to implement a congestion notification mechanism + * to be leveraged by transport layer protocols such as TCP and SCTP that have + * congestion control mechanisms. + * + * When congestion is experienced, as alternative to dropping the packet, + * routers can change the ECN field of input packets from 2'b01 or 2'b10 + * (values indicating that source endpoint is ECN-capable) to 2'b11 (meaning + * that congestion is experienced). The destination endpoint can use the + * ECN-Echo (ECE) TCP flag to relay the congestion indication back to the + * source endpoint, which acknowledges it back to the destination endpoint with + * the Congestion Window Reduced (CWR) TCP flag. + * + * All IPv4/IPv6 packets of a given color with ECN set to 2’b01 or 2’b10 + * carrying TCP or SCTP have their ECN set to 2’b11 if the marking feature is + * enabled for the current color, otherwise the ECN field is left as is. + * + * @param[in] port_id + * The port identifier of the Ethernet device. + * @param[in] mark_green + * Set to non-zero value to enable marking of green packets and to zero to + * disable it. + * @param[in] mark_yellow + * Set to non-zero value to enable marking of yellow packets and to zero to + * disable it. + * @param[in] mark_red + * Set to non-zero value to enable marking of red packets and to zero to + * disable it. + * @param[out] error + * Error details. Filled in only on error, when not NULL. + * @return + * 0 on success, non-zero error code otherwise. + * + * @see struct rte_tm_capabilities::mark_ip_ecn_tcp_supported + * @see struct rte_tm_capabilities::mark_ip_ecn_sctp_supported + */ +int +rte_tm_mark_ip_ecn(uint16_t port_id, + int mark_green, + int mark_yellow, + int mark_red, + struct rte_tm_error *error); + +/** + * Traffic manager packet marking - IPv4 / IPv6 DSCP (IETF RFC 2597) + * + * IETF RFC 2597 maps the traffic class and the drop priority to the IPv4/IPv6 + * Differentiated Services Codepoint (DSCP) field (6 bits). Here are the DSCP + * values proposed by this RFC: + * + * 
                   Class 1    Class 2    Class 3    Class 4
+ * 
                 +----------+----------+----------+----------+
+ * 
Low Drop Prec    |  001010  |  010010  |  011010  |  100010  |
+ * 
Medium Drop Prec |  001100  |  010100  |  011100  |  100100  |
+ * 
High Drop Prec   |  001110  |  010110  |  011110  |  100110  |
+ * 
                 +----------+----------+----------+----------+
+ * + * There are 4 traffic classes (classes 1 .. 4) encoded by DSCP bits 1 and 2, + * as well as 3 drop priorities (low/medium/high) encoded by DSCP bits 3 and 4. + * + * All IPv4/IPv6 packets have their color marked into DSCP bits 3 and 4 as + * follows: green mapped to Low Drop Precedence (2’b01), yellow to Medium + * (2’b10) and red to High (2’b11). Marking needs to be explicitly enabled + * for each color; when not enabled for a given color, the DSCP field of all + * packets with that color is left as is. + * + * @param[in] port_id + * The port identifier of the Ethernet device. + * @param[in] mark_green + * Set to non-zero value to enable marking of green packets and to zero to + * disable it. + * @param[in] mark_yellow + * Set to non-zero value to enable marking of yellow packets and to zero to + * disable it. + * @param[in] mark_red + * Set to non-zero value to enable marking of red packets and to zero to + * disable it. + * @param[out] error + * Error details. Filled in only on error, when not NULL. + * @return + * 0 on success, non-zero error code otherwise. + * + * @see struct rte_tm_capabilities::mark_ip_dscp_supported + */ +int +rte_tm_mark_ip_dscp(uint16_t port_id, + int mark_green, + int mark_yellow, + int mark_red, + struct rte_tm_error *error); + +#ifdef __cplusplus +} +#endif + +#endif /* __INCLUDE_RTE_TM_H__ */ diff --git a/dpdk/lib/librte_ether/rte_tm_driver.h b/dpdk/lib/librte_ether/rte_tm_driver.h new file mode 100644 index 00000000..2376943d --- /dev/null +++ b/dpdk/lib/librte_ether/rte_tm_driver.h @@ -0,0 +1,366 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __INCLUDE_RTE_TM_DRIVER_H__ +#define __INCLUDE_RTE_TM_DRIVER_H__ + +/** + * @file + * RTE Generic Traffic Manager API (Driver Side) + * + * This file provides implementation helpers for internal use by PMDs, they + * are not intended to be exposed to applications and are not subject to ABI + * versioning. + */ + +#include + +#include +#include "rte_ethdev.h" +#include "rte_tm.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** @internal Traffic manager node ID validate and type get */ +typedef int (*rte_tm_node_type_get_t)(struct rte_eth_dev *dev, + uint32_t node_id, + int *is_leaf, + struct rte_tm_error *error); + +/** @internal Traffic manager capabilities get */ +typedef int (*rte_tm_capabilities_get_t)(struct rte_eth_dev *dev, + struct rte_tm_capabilities *cap, + struct rte_tm_error *error); + +/** @internal Traffic manager level capabilities get */ +typedef int (*rte_tm_level_capabilities_get_t)(struct rte_eth_dev *dev, + uint32_t level_id, + struct rte_tm_level_capabilities *cap, + struct rte_tm_error *error); + +/** @internal Traffic manager node capabilities get */ +typedef int (*rte_tm_node_capabilities_get_t)(struct rte_eth_dev *dev, + uint32_t node_id, + struct rte_tm_node_capabilities *cap, + struct rte_tm_error *error); + +/** @internal Traffic manager WRED profile add */ +typedef int (*rte_tm_wred_profile_add_t)(struct rte_eth_dev *dev, + uint32_t wred_profile_id, + struct rte_tm_wred_params *profile, + struct rte_tm_error *error); + +/** @internal Traffic manager WRED profile delete */ +typedef int (*rte_tm_wred_profile_delete_t)(struct rte_eth_dev *dev, + uint32_t wred_profile_id, + struct rte_tm_error *error); + +/** @internal Traffic manager shared WRED context add */ +typedef int (*rte_tm_shared_wred_context_add_update_t)( + struct rte_eth_dev *dev, + uint32_t shared_wred_context_id, + uint32_t wred_profile_id, + struct rte_tm_error *error); + +/** @internal Traffic manager shared WRED context delete */ +typedef int (*rte_tm_shared_wred_context_delete_t)( + struct rte_eth_dev *dev, + uint32_t shared_wred_context_id, + struct rte_tm_error *error); + +/** @internal Traffic manager shaper profile add */ +typedef int (*rte_tm_shaper_profile_add_t)(struct rte_eth_dev *dev, + uint32_t shaper_profile_id, + struct rte_tm_shaper_params *profile, + struct rte_tm_error *error); + +/** @internal Traffic manager shaper profile delete */ +typedef int (*rte_tm_shaper_profile_delete_t)(struct rte_eth_dev *dev, + uint32_t shaper_profile_id, + struct rte_tm_error *error); + +/** @internal Traffic manager shared shaper add/update */ +typedef int (*rte_tm_shared_shaper_add_update_t)(struct rte_eth_dev *dev, + uint32_t shared_shaper_id, + uint32_t shaper_profile_id, + struct rte_tm_error *error); + +/** @internal Traffic manager shared shaper delete */ +typedef int (*rte_tm_shared_shaper_delete_t)(struct rte_eth_dev *dev, + uint32_t shared_shaper_id, + struct rte_tm_error *error); + +/** @internal Traffic manager node add */ +typedef int (*rte_tm_node_add_t)(struct rte_eth_dev *dev, + uint32_t node_id, + uint32_t parent_node_id, + uint32_t priority, + uint32_t weight, + uint32_t level_id, + struct rte_tm_node_params *params, + struct rte_tm_error *error); + +/** @internal Traffic manager node delete */ +typedef int (*rte_tm_node_delete_t)(struct rte_eth_dev *dev, + uint32_t node_id, + struct rte_tm_error *error); + +/** @internal Traffic manager node suspend */ +typedef int (*rte_tm_node_suspend_t)(struct rte_eth_dev *dev, + uint32_t node_id, + struct rte_tm_error *error); + +/** @internal Traffic manager node resume */ +typedef int (*rte_tm_node_resume_t)(struct rte_eth_dev *dev, + uint32_t node_id, + struct rte_tm_error *error); + +/** @internal Traffic manager hierarchy commit */ +typedef int (*rte_tm_hierarchy_commit_t)(struct rte_eth_dev *dev, + int clear_on_fail, + struct rte_tm_error *error); + +/** @internal Traffic manager node parent update */ +typedef int (*rte_tm_node_parent_update_t)(struct rte_eth_dev *dev, + uint32_t node_id, + uint32_t parent_node_id, + uint32_t priority, + uint32_t weight, + struct rte_tm_error *error); + +/** @internal Traffic manager node shaper update */ +typedef int (*rte_tm_node_shaper_update_t)(struct rte_eth_dev *dev, + uint32_t node_id, + uint32_t shaper_profile_id, + struct rte_tm_error *error); + +/** @internal Traffic manager node shaper update */ +typedef int (*rte_tm_node_shared_shaper_update_t)(struct rte_eth_dev *dev, + uint32_t node_id, + uint32_t shared_shaper_id, + int32_t add, + struct rte_tm_error *error); + +/** @internal Traffic manager node stats update */ +typedef int (*rte_tm_node_stats_update_t)(struct rte_eth_dev *dev, + uint32_t node_id, + uint64_t stats_mask, + struct rte_tm_error *error); + +/** @internal Traffic manager node WFQ weight mode update */ +typedef int (*rte_tm_node_wfq_weight_mode_update_t)( + struct rte_eth_dev *dev, + uint32_t node_id, + int *wfq_weight_mode, + uint32_t n_sp_priorities, + struct rte_tm_error *error); + +/** @internal Traffic manager node congestion management mode update */ +typedef int (*rte_tm_node_cman_update_t)(struct rte_eth_dev *dev, + uint32_t node_id, + enum rte_tm_cman_mode cman, + struct rte_tm_error *error); + +/** @internal Traffic manager node WRED context update */ +typedef int (*rte_tm_node_wred_context_update_t)( + struct rte_eth_dev *dev, + uint32_t node_id, + uint32_t wred_profile_id, + struct rte_tm_error *error); + +/** @internal Traffic manager node WRED context update */ +typedef int (*rte_tm_node_shared_wred_context_update_t)( + struct rte_eth_dev *dev, + uint32_t node_id, + uint32_t shared_wred_context_id, + int add, + struct rte_tm_error *error); + +/** @internal Traffic manager read stats counters for specific node */ +typedef int (*rte_tm_node_stats_read_t)(struct rte_eth_dev *dev, + uint32_t node_id, + struct rte_tm_node_stats *stats, + uint64_t *stats_mask, + int clear, + struct rte_tm_error *error); + +/** @internal Traffic manager packet marking - VLAN DEI */ +typedef int (*rte_tm_mark_vlan_dei_t)(struct rte_eth_dev *dev, + int mark_green, + int mark_yellow, + int mark_red, + struct rte_tm_error *error); + +/** @internal Traffic manager packet marking - IPv4/IPv6 ECN */ +typedef int (*rte_tm_mark_ip_ecn_t)(struct rte_eth_dev *dev, + int mark_green, + int mark_yellow, + int mark_red, + struct rte_tm_error *error); + +/** @internal Traffic manager packet marking - IPv4/IPv6 DSCP */ +typedef int (*rte_tm_mark_ip_dscp_t)(struct rte_eth_dev *dev, + int mark_green, + int mark_yellow, + int mark_red, + struct rte_tm_error *error); + +struct rte_tm_ops { + /** Traffic manager node type get */ + rte_tm_node_type_get_t node_type_get; + + /** Traffic manager capabilities_get */ + rte_tm_capabilities_get_t capabilities_get; + /** Traffic manager level capabilities_get */ + rte_tm_level_capabilities_get_t level_capabilities_get; + /** Traffic manager node capabilities get */ + rte_tm_node_capabilities_get_t node_capabilities_get; + + /** Traffic manager WRED profile add */ + rte_tm_wred_profile_add_t wred_profile_add; + /** Traffic manager WRED profile delete */ + rte_tm_wred_profile_delete_t wred_profile_delete; + /** Traffic manager shared WRED context add/update */ + rte_tm_shared_wred_context_add_update_t + shared_wred_context_add_update; + /** Traffic manager shared WRED context delete */ + rte_tm_shared_wred_context_delete_t + shared_wred_context_delete; + + /** Traffic manager shaper profile add */ + rte_tm_shaper_profile_add_t shaper_profile_add; + /** Traffic manager shaper profile delete */ + rte_tm_shaper_profile_delete_t shaper_profile_delete; + /** Traffic manager shared shaper add/update */ + rte_tm_shared_shaper_add_update_t shared_shaper_add_update; + /** Traffic manager shared shaper delete */ + rte_tm_shared_shaper_delete_t shared_shaper_delete; + + /** Traffic manager node add */ + rte_tm_node_add_t node_add; + /** Traffic manager node delete */ + rte_tm_node_delete_t node_delete; + /** Traffic manager node suspend */ + rte_tm_node_suspend_t node_suspend; + /** Traffic manager node resume */ + rte_tm_node_resume_t node_resume; + /** Traffic manager hierarchy commit */ + rte_tm_hierarchy_commit_t hierarchy_commit; + + /** Traffic manager node parent update */ + rte_tm_node_parent_update_t node_parent_update; + /** Traffic manager node shaper update */ + rte_tm_node_shaper_update_t node_shaper_update; + /** Traffic manager node shared shaper update */ + rte_tm_node_shared_shaper_update_t node_shared_shaper_update; + /** Traffic manager node stats update */ + rte_tm_node_stats_update_t node_stats_update; + /** Traffic manager node WFQ weight mode update */ + rte_tm_node_wfq_weight_mode_update_t node_wfq_weight_mode_update; + /** Traffic manager node congestion management mode update */ + rte_tm_node_cman_update_t node_cman_update; + /** Traffic manager node WRED context update */ + rte_tm_node_wred_context_update_t node_wred_context_update; + /** Traffic manager node shared WRED context update */ + rte_tm_node_shared_wred_context_update_t + node_shared_wred_context_update; + /** Traffic manager read statistics counters for current node */ + rte_tm_node_stats_read_t node_stats_read; + + /** Traffic manager packet marking - VLAN DEI */ + rte_tm_mark_vlan_dei_t mark_vlan_dei; + /** Traffic manager packet marking - IPv4/IPv6 ECN */ + rte_tm_mark_ip_ecn_t mark_ip_ecn; + /** Traffic manager packet marking - IPv4/IPv6 DSCP */ + rte_tm_mark_ip_dscp_t mark_ip_dscp; +}; + +/** + * Initialize generic error structure. + * + * This function also sets rte_errno to a given value. + * + * @param[out] error + * Pointer to error structure (may be NULL). + * @param[in] code + * Related error code (rte_errno). + * @param[in] type + * Cause field and error type. + * @param[in] cause + * Object responsible for the error. + * @param[in] message + * Human-readable error message. + * + * @return + * Error code. + */ +static inline int +rte_tm_error_set(struct rte_tm_error *error, + int code, + enum rte_tm_error_type type, + const void *cause, + const char *message) +{ + if (error) { + *error = (struct rte_tm_error){ + .type = type, + .cause = cause, + .message = message, + }; + } + rte_errno = code; + return code; +} + +/** + * Get generic traffic manager operations structure from a port + * + * @param[in] port_id + * The port identifier of the Ethernet device. + * @param[out] error + * Error details + * + * @return + * The traffic manager operations structure associated with port_id on + * success, NULL otherwise. + */ +const struct rte_tm_ops * +rte_tm_ops_get(uint16_t port_id, struct rte_tm_error *error); + +#ifdef __cplusplus +} +#endif + +#endif /* __INCLUDE_RTE_TM_DRIVER_H__ */ diff --git a/dpdk/lib/librte_eventdev/Makefile b/dpdk/lib/librte_eventdev/Makefile new file mode 100644 index 00000000..5ac22cde --- /dev/null +++ b/dpdk/lib/librte_eventdev/Makefile @@ -0,0 +1,60 @@ +# BSD LICENSE +# +# Copyright(c) 2016 Cavium, Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Cavium, Inc nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +include $(RTE_SDK)/mk/rte.vars.mk + +# library name +LIB = librte_eventdev.a + +# library version +LIBABIVER := 3 + +# build flags +CFLAGS += -O3 +CFLAGS += $(WERROR_FLAGS) +LDLIBS += -lrte_eal -lrte_ring -lrte_ethdev -lrte_hash + +# library source files +SRCS-y += rte_eventdev.c +SRCS-y += rte_event_ring.c +SRCS-y += rte_event_eth_rx_adapter.c + +# export include files +SYMLINK-y-include += rte_eventdev.h +SYMLINK-y-include += rte_eventdev_pmd.h +SYMLINK-y-include += rte_eventdev_pmd_pci.h +SYMLINK-y-include += rte_eventdev_pmd_vdev.h +SYMLINK-y-include += rte_event_ring.h +SYMLINK-y-include += rte_event_eth_rx_adapter.h + +# versioning export map +EXPORT_MAP := rte_eventdev_version.map + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/lib/librte_eventdev/rte_event_eth_rx_adapter.c b/dpdk/lib/librte_eventdev/rte_event_eth_rx_adapter.c new file mode 100644 index 00000000..90106e6c --- /dev/null +++ b/dpdk/lib/librte_eventdev/rte_event_eth_rx_adapter.c @@ -0,0 +1,1240 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "rte_eventdev.h" +#include "rte_eventdev_pmd.h" +#include "rte_event_eth_rx_adapter.h" + +#define BATCH_SIZE 32 +#define BLOCK_CNT_THRESHOLD 10 +#define ETH_EVENT_BUFFER_SIZE (4*BATCH_SIZE) + +#define ETH_RX_ADAPTER_SERVICE_NAME_LEN 32 +#define ETH_RX_ADAPTER_MEM_NAME_LEN 32 + +#define RSS_KEY_SIZE 40 + +/* + * There is an instance of this struct per polled Rx queue added to the + * adapter + */ +struct eth_rx_poll_entry { + /* Eth port to poll */ + uint8_t eth_dev_id; + /* Eth rx queue to poll */ + uint16_t eth_rx_qid; +}; + +/* Instance per adapter */ +struct rte_eth_event_enqueue_buffer { + /* Count of events in this buffer */ + uint16_t count; + /* Array of events in this buffer */ + struct rte_event events[ETH_EVENT_BUFFER_SIZE]; +}; + +struct rte_event_eth_rx_adapter { + /* RSS key */ + uint8_t rss_key_be[RSS_KEY_SIZE]; + /* Event device identifier */ + uint8_t eventdev_id; + /* Per ethernet device structure */ + struct eth_device_info *eth_devices; + /* Event port identifier */ + uint8_t event_port_id; + /* Lock to serialize config updates with service function */ + rte_spinlock_t rx_lock; + /* Max mbufs processed in any service function invocation */ + uint32_t max_nb_rx; + /* Receive queues that need to be polled */ + struct eth_rx_poll_entry *eth_rx_poll; + /* Size of the eth_rx_poll array */ + uint16_t num_rx_polled; + /* Weighted round robin schedule */ + uint32_t *wrr_sched; + /* wrr_sched[] size */ + uint32_t wrr_len; + /* Next entry in wrr[] to begin polling */ + uint32_t wrr_pos; + /* Event burst buffer */ + struct rte_eth_event_enqueue_buffer event_enqueue_buffer; + /* Per adapter stats */ + struct rte_event_eth_rx_adapter_stats stats; + /* Block count, counts up to BLOCK_CNT_THRESHOLD */ + uint16_t enq_block_count; + /* Block start ts */ + uint64_t rx_enq_block_start_ts; + /* Configuration callback for rte_service configuration */ + rte_event_eth_rx_adapter_conf_cb conf_cb; + /* Configuration callback argument */ + void *conf_arg; + /* Set if default_cb is being used */ + int default_cb_arg; + /* Service initialization state */ + uint8_t service_inited; + /* Total count of Rx queues in adapter */ + uint32_t nb_queues; + /* Memory allocation name */ + char mem_name[ETH_RX_ADAPTER_MEM_NAME_LEN]; + /* Socket identifier cached from eventdev */ + int socket_id; + /* Per adapter EAL service */ + uint32_t service_id; +} __rte_cache_aligned; + +/* Per eth device */ +struct eth_device_info { + struct rte_eth_dev *dev; + struct eth_rx_queue_info *rx_queue; + /* Set if ethdev->eventdev packet transfer uses a + * hardware mechanism + */ + uint8_t internal_event_port; + /* Set if the adapter is processing rx queues for + * this eth device and packet processing has been + * started, allows for the code to know if the PMD + * rx_adapter_stop callback needs to be invoked + */ + uint8_t dev_rx_started; + /* If nb_dev_queues > 0, the start callback will + * be invoked if not already invoked + */ + uint16_t nb_dev_queues; +}; + +/* Per Rx queue */ +struct eth_rx_queue_info { + int queue_enabled; /* True if added */ + uint16_t wt; /* Polling weight */ + uint8_t event_queue_id; /* Event queue to enqueue packets to */ + uint8_t sched_type; /* Sched type for events */ + uint8_t priority; /* Event priority */ + uint32_t flow_id; /* App provided flow identifier */ + uint32_t flow_id_mask; /* Set to ~0 if app provides flow id else 0 */ +}; + +static struct rte_event_eth_rx_adapter **event_eth_rx_adapter; + +static inline int +valid_id(uint8_t id) +{ + return id < RTE_EVENT_ETH_RX_ADAPTER_MAX_INSTANCE; +} + +#define RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, retval) do { \ + if (!valid_id(id)) { \ + RTE_EDEV_LOG_ERR("Invalid eth Rx adapter id = %d\n", id); \ + return retval; \ + } \ +} while (0) + +static inline int +sw_rx_adapter_queue_count(struct rte_event_eth_rx_adapter *rx_adapter) +{ + return rx_adapter->num_rx_polled; +} + +/* Greatest common divisor */ +static uint16_t gcd_u16(uint16_t a, uint16_t b) +{ + uint16_t r = a % b; + + return r ? gcd_u16(b, r) : b; +} + +/* Returns the next queue in the polling sequence + * + * http://kb.linuxvirtualserver.org/wiki/Weighted_Round-Robin_Scheduling + */ +static int +wrr_next(struct rte_event_eth_rx_adapter *rx_adapter, + unsigned int n, int *cw, + struct eth_rx_poll_entry *eth_rx_poll, uint16_t max_wt, + uint16_t gcd, int prev) +{ + int i = prev; + uint16_t w; + + while (1) { + uint16_t q; + uint8_t d; + + i = (i + 1) % n; + if (i == 0) { + *cw = *cw - gcd; + if (*cw <= 0) + *cw = max_wt; + } + + q = eth_rx_poll[i].eth_rx_qid; + d = eth_rx_poll[i].eth_dev_id; + w = rx_adapter->eth_devices[d].rx_queue[q].wt; + + if ((int)w >= *cw) + return i; + } +} + +/* Precalculate WRR polling sequence for all queues in rx_adapter */ +static int +eth_poll_wrr_calc(struct rte_event_eth_rx_adapter *rx_adapter) +{ + uint8_t d; + uint16_t q; + unsigned int i; + + /* Initialize variables for calculation of wrr schedule */ + uint16_t max_wrr_pos = 0; + unsigned int poll_q = 0; + uint16_t max_wt = 0; + uint16_t gcd = 0; + + struct eth_rx_poll_entry *rx_poll = NULL; + uint32_t *rx_wrr = NULL; + + if (rx_adapter->num_rx_polled) { + size_t len = RTE_ALIGN(rx_adapter->num_rx_polled * + sizeof(*rx_adapter->eth_rx_poll), + RTE_CACHE_LINE_SIZE); + rx_poll = rte_zmalloc_socket(rx_adapter->mem_name, + len, + RTE_CACHE_LINE_SIZE, + rx_adapter->socket_id); + if (rx_poll == NULL) + return -ENOMEM; + + /* Generate array of all queues to poll, the size of this + * array is poll_q + */ + for (d = 0; d < rte_eth_dev_count(); d++) { + uint16_t nb_rx_queues; + struct eth_device_info *dev_info = + &rx_adapter->eth_devices[d]; + nb_rx_queues = dev_info->dev->data->nb_rx_queues; + if (dev_info->rx_queue == NULL) + continue; + for (q = 0; q < nb_rx_queues; q++) { + struct eth_rx_queue_info *queue_info = + &dev_info->rx_queue[q]; + if (queue_info->queue_enabled == 0) + continue; + + uint16_t wt = queue_info->wt; + rx_poll[poll_q].eth_dev_id = d; + rx_poll[poll_q].eth_rx_qid = q; + max_wrr_pos += wt; + max_wt = RTE_MAX(max_wt, wt); + gcd = (gcd) ? gcd_u16(gcd, wt) : wt; + poll_q++; + } + } + + len = RTE_ALIGN(max_wrr_pos * sizeof(*rx_wrr), + RTE_CACHE_LINE_SIZE); + rx_wrr = rte_zmalloc_socket(rx_adapter->mem_name, + len, + RTE_CACHE_LINE_SIZE, + rx_adapter->socket_id); + if (rx_wrr == NULL) { + rte_free(rx_poll); + return -ENOMEM; + } + + /* Generate polling sequence based on weights */ + int prev = -1; + int cw = -1; + for (i = 0; i < max_wrr_pos; i++) { + rx_wrr[i] = wrr_next(rx_adapter, poll_q, &cw, + rx_poll, max_wt, gcd, prev); + prev = rx_wrr[i]; + } + } + + rte_free(rx_adapter->eth_rx_poll); + rte_free(rx_adapter->wrr_sched); + + rx_adapter->eth_rx_poll = rx_poll; + rx_adapter->wrr_sched = rx_wrr; + rx_adapter->wrr_len = max_wrr_pos; + + return 0; +} + +static inline void +mtoip(struct rte_mbuf *m, struct ipv4_hdr **ipv4_hdr, + struct ipv6_hdr **ipv6_hdr) +{ + struct ether_hdr *eth_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *); + struct vlan_hdr *vlan_hdr; + + *ipv4_hdr = NULL; + *ipv6_hdr = NULL; + + switch (eth_hdr->ether_type) { + case RTE_BE16(ETHER_TYPE_IPv4): + *ipv4_hdr = (struct ipv4_hdr *)(eth_hdr + 1); + break; + + case RTE_BE16(ETHER_TYPE_IPv6): + *ipv6_hdr = (struct ipv6_hdr *)(eth_hdr + 1); + break; + + case RTE_BE16(ETHER_TYPE_VLAN): + vlan_hdr = (struct vlan_hdr *)(eth_hdr + 1); + switch (vlan_hdr->eth_proto) { + case RTE_BE16(ETHER_TYPE_IPv4): + *ipv4_hdr = (struct ipv4_hdr *)(vlan_hdr + 1); + break; + case RTE_BE16(ETHER_TYPE_IPv6): + *ipv6_hdr = (struct ipv6_hdr *)(vlan_hdr + 1); + break; + default: + break; + } + break; + + default: + break; + } +} + +/* Calculate RSS hash for IPv4/6 */ +static inline uint32_t +do_softrss(struct rte_mbuf *m, const uint8_t *rss_key_be) +{ + uint32_t input_len; + void *tuple; + struct rte_ipv4_tuple ipv4_tuple; + struct rte_ipv6_tuple ipv6_tuple; + struct ipv4_hdr *ipv4_hdr; + struct ipv6_hdr *ipv6_hdr; + + mtoip(m, &ipv4_hdr, &ipv6_hdr); + + if (ipv4_hdr) { + ipv4_tuple.src_addr = rte_be_to_cpu_32(ipv4_hdr->src_addr); + ipv4_tuple.dst_addr = rte_be_to_cpu_32(ipv4_hdr->dst_addr); + tuple = &ipv4_tuple; + input_len = RTE_THASH_V4_L3_LEN; + } else if (ipv6_hdr) { + rte_thash_load_v6_addrs(ipv6_hdr, + (union rte_thash_tuple *)&ipv6_tuple); + tuple = &ipv6_tuple; + input_len = RTE_THASH_V6_L3_LEN; + } else + return 0; + + return rte_softrss_be(tuple, input_len, rss_key_be); +} + +static inline int +rx_enq_blocked(struct rte_event_eth_rx_adapter *rx_adapter) +{ + return !!rx_adapter->enq_block_count; +} + +static inline void +rx_enq_block_start_ts(struct rte_event_eth_rx_adapter *rx_adapter) +{ + if (rx_adapter->rx_enq_block_start_ts) + return; + + rx_adapter->enq_block_count++; + if (rx_adapter->enq_block_count < BLOCK_CNT_THRESHOLD) + return; + + rx_adapter->rx_enq_block_start_ts = rte_get_tsc_cycles(); +} + +static inline void +rx_enq_block_end_ts(struct rte_event_eth_rx_adapter *rx_adapter, + struct rte_event_eth_rx_adapter_stats *stats) +{ + if (unlikely(!stats->rx_enq_start_ts)) + stats->rx_enq_start_ts = rte_get_tsc_cycles(); + + if (likely(!rx_enq_blocked(rx_adapter))) + return; + + rx_adapter->enq_block_count = 0; + if (rx_adapter->rx_enq_block_start_ts) { + stats->rx_enq_end_ts = rte_get_tsc_cycles(); + stats->rx_enq_block_cycles += stats->rx_enq_end_ts - + rx_adapter->rx_enq_block_start_ts; + rx_adapter->rx_enq_block_start_ts = 0; + } +} + +/* Add event to buffer, free space check is done prior to calling + * this function + */ +static inline void +buf_event_enqueue(struct rte_event_eth_rx_adapter *rx_adapter, + struct rte_event *ev) +{ + struct rte_eth_event_enqueue_buffer *buf = + &rx_adapter->event_enqueue_buffer; + rte_memcpy(&buf->events[buf->count++], ev, sizeof(struct rte_event)); +} + +/* Enqueue buffered events to event device */ +static inline uint16_t +flush_event_buffer(struct rte_event_eth_rx_adapter *rx_adapter) +{ + struct rte_eth_event_enqueue_buffer *buf = + &rx_adapter->event_enqueue_buffer; + struct rte_event_eth_rx_adapter_stats *stats = &rx_adapter->stats; + + uint16_t n = rte_event_enqueue_new_burst(rx_adapter->eventdev_id, + rx_adapter->event_port_id, + buf->events, + buf->count); + if (n != buf->count) { + memmove(buf->events, + &buf->events[n], + (buf->count - n) * sizeof(struct rte_event)); + stats->rx_enq_retry++; + } + + n ? rx_enq_block_end_ts(rx_adapter, stats) : + rx_enq_block_start_ts(rx_adapter); + + buf->count -= n; + stats->rx_enq_count += n; + + return n; +} + +static inline void +fill_event_buffer(struct rte_event_eth_rx_adapter *rx_adapter, + uint8_t dev_id, + uint16_t rx_queue_id, + struct rte_mbuf **mbufs, + uint16_t num) +{ + uint32_t i; + struct eth_device_info *eth_device_info = + &rx_adapter->eth_devices[dev_id]; + struct eth_rx_queue_info *eth_rx_queue_info = + ð_device_info->rx_queue[rx_queue_id]; + + int32_t qid = eth_rx_queue_info->event_queue_id; + uint8_t sched_type = eth_rx_queue_info->sched_type; + uint8_t priority = eth_rx_queue_info->priority; + uint32_t flow_id; + struct rte_event events[BATCH_SIZE]; + struct rte_mbuf *m = mbufs[0]; + uint32_t rss_mask; + uint32_t rss; + int do_rss; + + /* 0xffff ffff if PKT_RX_RSS_HASH is set, otherwise 0 */ + rss_mask = ~(((m->ol_flags & PKT_RX_RSS_HASH) != 0) - 1); + do_rss = !rss_mask && !eth_rx_queue_info->flow_id_mask; + + for (i = 0; i < num; i++) { + m = mbufs[i]; + struct rte_event *ev = &events[i]; + + rss = do_rss ? + do_softrss(m, rx_adapter->rss_key_be) : m->hash.rss; + flow_id = + eth_rx_queue_info->flow_id & + eth_rx_queue_info->flow_id_mask; + flow_id |= rss & ~eth_rx_queue_info->flow_id_mask; + + ev->flow_id = flow_id; + ev->op = RTE_EVENT_OP_NEW; + ev->sched_type = sched_type; + ev->queue_id = qid; + ev->event_type = RTE_EVENT_TYPE_ETH_RX_ADAPTER; + ev->sub_event_type = 0; + ev->priority = priority; + ev->mbuf = m; + + buf_event_enqueue(rx_adapter, ev); + } +} + +/* + * Polls receive queues added to the event adapter and enqueues received + * packets to the event device. + * + * The receive code enqueues initially to a temporary buffer, the + * temporary buffer is drained anytime it holds >= BATCH_SIZE packets + * + * If there isn't space available in the temporary buffer, packets from the + * Rx queue aren't dequeued from the eth device, this back pressures the + * eth device, in virtual device environments this back pressure is relayed to + * the hypervisor's switching layer where adjustments can be made to deal with + * it. + */ +static inline uint32_t +eth_rx_poll(struct rte_event_eth_rx_adapter *rx_adapter) +{ + uint32_t num_queue; + uint16_t n; + uint32_t nb_rx = 0; + struct rte_mbuf *mbufs[BATCH_SIZE]; + struct rte_eth_event_enqueue_buffer *buf; + uint32_t wrr_pos; + uint32_t max_nb_rx; + + wrr_pos = rx_adapter->wrr_pos; + max_nb_rx = rx_adapter->max_nb_rx; + buf = &rx_adapter->event_enqueue_buffer; + struct rte_event_eth_rx_adapter_stats *stats = &rx_adapter->stats; + + /* Iterate through a WRR sequence */ + for (num_queue = 0; num_queue < rx_adapter->wrr_len; num_queue++) { + unsigned int poll_idx = rx_adapter->wrr_sched[wrr_pos]; + uint16_t qid = rx_adapter->eth_rx_poll[poll_idx].eth_rx_qid; + uint8_t d = rx_adapter->eth_rx_poll[poll_idx].eth_dev_id; + + /* Don't do a batch dequeue from the rx queue if there isn't + * enough space in the enqueue buffer. + */ + if (buf->count >= BATCH_SIZE) + flush_event_buffer(rx_adapter); + if (BATCH_SIZE > (ETH_EVENT_BUFFER_SIZE - buf->count)) + break; + + stats->rx_poll_count++; + n = rte_eth_rx_burst(d, qid, mbufs, BATCH_SIZE); + + if (n) { + stats->rx_packets += n; + /* The check before rte_eth_rx_burst() ensures that + * all n mbufs can be buffered + */ + fill_event_buffer(rx_adapter, d, qid, mbufs, n); + nb_rx += n; + if (nb_rx > max_nb_rx) { + rx_adapter->wrr_pos = + (wrr_pos + 1) % rx_adapter->wrr_len; + return nb_rx; + } + } + + if (++wrr_pos == rx_adapter->wrr_len) + wrr_pos = 0; + } + + return nb_rx; +} + +static int +event_eth_rx_adapter_service_func(void *args) +{ + struct rte_event_eth_rx_adapter *rx_adapter = args; + struct rte_eth_event_enqueue_buffer *buf; + + buf = &rx_adapter->event_enqueue_buffer; + if (rte_spinlock_trylock(&rx_adapter->rx_lock) == 0) + return 0; + if (eth_rx_poll(rx_adapter) == 0 && buf->count) + flush_event_buffer(rx_adapter); + rte_spinlock_unlock(&rx_adapter->rx_lock); + return 0; +} + +static int +rte_event_eth_rx_adapter_init(void) +{ + const char *name = "rte_event_eth_rx_adapter_array"; + const struct rte_memzone *mz; + unsigned int sz; + + sz = sizeof(*event_eth_rx_adapter) * + RTE_EVENT_ETH_RX_ADAPTER_MAX_INSTANCE; + sz = RTE_ALIGN(sz, RTE_CACHE_LINE_SIZE); + + mz = rte_memzone_lookup(name); + if (mz == NULL) { + mz = rte_memzone_reserve_aligned(name, sz, rte_socket_id(), 0, + RTE_CACHE_LINE_SIZE); + if (mz == NULL) { + RTE_EDEV_LOG_ERR("failed to reserve memzone err = %" + PRId32, rte_errno); + return -rte_errno; + } + } + + event_eth_rx_adapter = mz->addr; + return 0; +} + +static inline struct rte_event_eth_rx_adapter * +id_to_rx_adapter(uint8_t id) +{ + return event_eth_rx_adapter ? + event_eth_rx_adapter[id] : NULL; +} + +static int +default_conf_cb(uint8_t id, uint8_t dev_id, + struct rte_event_eth_rx_adapter_conf *conf, void *arg) +{ + int ret; + struct rte_eventdev *dev; + struct rte_event_dev_config dev_conf; + int started; + uint8_t port_id; + struct rte_event_port_conf *port_conf = arg; + struct rte_event_eth_rx_adapter *rx_adapter = id_to_rx_adapter(id); + + dev = &rte_eventdevs[rx_adapter->eventdev_id]; + dev_conf = dev->data->dev_conf; + + started = dev->data->dev_started; + if (started) + rte_event_dev_stop(dev_id); + port_id = dev_conf.nb_event_ports; + dev_conf.nb_event_ports += 1; + ret = rte_event_dev_configure(dev_id, &dev_conf); + if (ret) { + RTE_EDEV_LOG_ERR("failed to configure event dev %u\n", + dev_id); + if (started) + rte_event_dev_start(dev_id); + return ret; + } + + ret = rte_event_port_setup(dev_id, port_id, port_conf); + if (ret) { + RTE_EDEV_LOG_ERR("failed to setup event port %u\n", + port_id); + return ret; + } + + conf->event_port_id = port_id; + conf->max_nb_rx = 128; + if (started) + rte_event_dev_start(dev_id); + rx_adapter->default_cb_arg = 1; + return ret; +} + +static int +init_service(struct rte_event_eth_rx_adapter *rx_adapter, uint8_t id) +{ + int ret; + struct rte_service_spec service; + struct rte_event_eth_rx_adapter_conf rx_adapter_conf; + + if (rx_adapter->service_inited) + return 0; + + memset(&service, 0, sizeof(service)); + snprintf(service.name, ETH_RX_ADAPTER_SERVICE_NAME_LEN, + "rte_event_eth_rx_adapter_%d", id); + service.socket_id = rx_adapter->socket_id; + service.callback = event_eth_rx_adapter_service_func; + service.callback_userdata = rx_adapter; + /* Service function handles locking for queue add/del updates */ + service.capabilities = RTE_SERVICE_CAP_MT_SAFE; + ret = rte_service_component_register(&service, &rx_adapter->service_id); + if (ret) { + RTE_EDEV_LOG_ERR("failed to register service %s err = %" PRId32, + service.name, ret); + return ret; + } + + ret = rx_adapter->conf_cb(id, rx_adapter->eventdev_id, + &rx_adapter_conf, rx_adapter->conf_arg); + if (ret) { + RTE_EDEV_LOG_ERR("configuration callback failed err = %" PRId32, + ret); + goto err_done; + } + rx_adapter->event_port_id = rx_adapter_conf.event_port_id; + rx_adapter->max_nb_rx = rx_adapter_conf.max_nb_rx; + rx_adapter->service_inited = 1; + return 0; + +err_done: + rte_service_component_unregister(rx_adapter->service_id); + return ret; +} + + +static void +update_queue_info(struct rte_event_eth_rx_adapter *rx_adapter, + struct eth_device_info *dev_info, + int32_t rx_queue_id, + uint8_t add) +{ + struct eth_rx_queue_info *queue_info; + int enabled; + uint16_t i; + + if (dev_info->rx_queue == NULL) + return; + + if (rx_queue_id == -1) { + for (i = 0; i < dev_info->dev->data->nb_rx_queues; i++) + update_queue_info(rx_adapter, dev_info, i, add); + } else { + queue_info = &dev_info->rx_queue[rx_queue_id]; + enabled = queue_info->queue_enabled; + if (add) { + rx_adapter->nb_queues += !enabled; + dev_info->nb_dev_queues += !enabled; + } else { + rx_adapter->nb_queues -= enabled; + dev_info->nb_dev_queues -= enabled; + } + queue_info->queue_enabled = !!add; + } +} + +static int +event_eth_rx_adapter_queue_del(struct rte_event_eth_rx_adapter *rx_adapter, + struct eth_device_info *dev_info, + uint16_t rx_queue_id) +{ + struct eth_rx_queue_info *queue_info; + + if (rx_adapter->nb_queues == 0) + return 0; + + queue_info = &dev_info->rx_queue[rx_queue_id]; + rx_adapter->num_rx_polled -= queue_info->queue_enabled; + update_queue_info(rx_adapter, dev_info, rx_queue_id, 0); + return 0; +} + +static void +event_eth_rx_adapter_queue_add(struct rte_event_eth_rx_adapter *rx_adapter, + struct eth_device_info *dev_info, + uint16_t rx_queue_id, + const struct rte_event_eth_rx_adapter_queue_conf *conf) + +{ + struct eth_rx_queue_info *queue_info; + const struct rte_event *ev = &conf->ev; + + queue_info = &dev_info->rx_queue[rx_queue_id]; + queue_info->event_queue_id = ev->queue_id; + queue_info->sched_type = ev->sched_type; + queue_info->priority = ev->priority; + queue_info->wt = conf->servicing_weight; + + if (conf->rx_queue_flags & + RTE_EVENT_ETH_RX_ADAPTER_QUEUE_FLOW_ID_VALID) { + queue_info->flow_id = ev->flow_id; + queue_info->flow_id_mask = ~0; + } + + /* The same queue can be added more than once */ + rx_adapter->num_rx_polled += !queue_info->queue_enabled; + update_queue_info(rx_adapter, dev_info, rx_queue_id, 1); +} + +static int add_rx_queue(struct rte_event_eth_rx_adapter *rx_adapter, + uint8_t eth_dev_id, + int rx_queue_id, + const struct rte_event_eth_rx_adapter_queue_conf *queue_conf) +{ + struct eth_device_info *dev_info = &rx_adapter->eth_devices[eth_dev_id]; + struct rte_event_eth_rx_adapter_queue_conf temp_conf; + uint32_t i; + int ret; + + if (queue_conf->servicing_weight == 0) { + + struct rte_eth_dev_data *data = dev_info->dev->data; + if (data->dev_conf.intr_conf.rxq) { + RTE_EDEV_LOG_ERR("Interrupt driven queues" + " not supported"); + return -ENOTSUP; + } + temp_conf = *queue_conf; + + /* If Rx interrupts are disabled set wt = 1 */ + temp_conf.servicing_weight = 1; + queue_conf = &temp_conf; + } + + if (dev_info->rx_queue == NULL) { + dev_info->rx_queue = + rte_zmalloc_socket(rx_adapter->mem_name, + dev_info->dev->data->nb_rx_queues * + sizeof(struct eth_rx_queue_info), 0, + rx_adapter->socket_id); + if (dev_info->rx_queue == NULL) + return -ENOMEM; + } + + if (rx_queue_id == -1) { + for (i = 0; i < dev_info->dev->data->nb_rx_queues; i++) + event_eth_rx_adapter_queue_add(rx_adapter, + dev_info, i, + queue_conf); + } else { + event_eth_rx_adapter_queue_add(rx_adapter, dev_info, + (uint16_t)rx_queue_id, + queue_conf); + } + + ret = eth_poll_wrr_calc(rx_adapter); + if (ret) { + event_eth_rx_adapter_queue_del(rx_adapter, + dev_info, rx_queue_id); + return ret; + } + + return ret; +} + +static int +rx_adapter_ctrl(uint8_t id, int start) +{ + struct rte_event_eth_rx_adapter *rx_adapter; + struct rte_eventdev *dev; + struct eth_device_info *dev_info; + uint32_t i; + int use_service = 0; + int stop = !start; + + RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL); + rx_adapter = id_to_rx_adapter(id); + if (rx_adapter == NULL) + return -EINVAL; + + dev = &rte_eventdevs[rx_adapter->eventdev_id]; + + for (i = 0; i < rte_eth_dev_count(); i++) { + dev_info = &rx_adapter->eth_devices[i]; + /* if start check for num dev queues */ + if (start && !dev_info->nb_dev_queues) + continue; + /* if stop check if dev has been started */ + if (stop && !dev_info->dev_rx_started) + continue; + use_service |= !dev_info->internal_event_port; + dev_info->dev_rx_started = start; + if (dev_info->internal_event_port == 0) + continue; + start ? (*dev->dev_ops->eth_rx_adapter_start)(dev, + &rte_eth_devices[i]) : + (*dev->dev_ops->eth_rx_adapter_stop)(dev, + &rte_eth_devices[i]); + } + + if (use_service) + rte_service_runstate_set(rx_adapter->service_id, start); + + return 0; +} + +int +rte_event_eth_rx_adapter_create_ext(uint8_t id, uint8_t dev_id, + rte_event_eth_rx_adapter_conf_cb conf_cb, + void *conf_arg) +{ + struct rte_event_eth_rx_adapter *rx_adapter; + int ret; + int socket_id; + uint8_t i; + char mem_name[ETH_RX_ADAPTER_SERVICE_NAME_LEN]; + const uint8_t default_rss_key[] = { + 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, + 0x41, 0x67, 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, + 0xd0, 0xca, 0x2b, 0xcb, 0xae, 0x7b, 0x30, 0xb4, + 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30, 0xf2, 0x0c, + 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa, + }; + + RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL); + RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(dev_id, -EINVAL); + if (conf_cb == NULL) + return -EINVAL; + + if (event_eth_rx_adapter == NULL) { + ret = rte_event_eth_rx_adapter_init(); + if (ret) + return ret; + } + + rx_adapter = id_to_rx_adapter(id); + if (rx_adapter != NULL) { + RTE_EDEV_LOG_ERR("Eth Rx adapter exists id = %" PRIu8, id); + return -EEXIST; + } + + socket_id = rte_event_dev_socket_id(dev_id); + snprintf(mem_name, ETH_RX_ADAPTER_MEM_NAME_LEN, + "rte_event_eth_rx_adapter_%d", + id); + + rx_adapter = rte_zmalloc_socket(mem_name, sizeof(*rx_adapter), + RTE_CACHE_LINE_SIZE, socket_id); + if (rx_adapter == NULL) { + RTE_EDEV_LOG_ERR("failed to get mem for rx adapter"); + return -ENOMEM; + } + + rx_adapter->eventdev_id = dev_id; + rx_adapter->socket_id = socket_id; + rx_adapter->conf_cb = conf_cb; + rx_adapter->conf_arg = conf_arg; + strcpy(rx_adapter->mem_name, mem_name); + rx_adapter->eth_devices = rte_zmalloc_socket(rx_adapter->mem_name, + rte_eth_dev_count() * + sizeof(struct eth_device_info), 0, + socket_id); + rte_convert_rss_key((const uint32_t *)default_rss_key, + (uint32_t *)rx_adapter->rss_key_be, + RTE_DIM(default_rss_key)); + + if (rx_adapter->eth_devices == NULL) { + RTE_EDEV_LOG_ERR("failed to get mem for eth devices\n"); + rte_free(rx_adapter); + return -ENOMEM; + } + rte_spinlock_init(&rx_adapter->rx_lock); + for (i = 0; i < rte_eth_dev_count(); i++) + rx_adapter->eth_devices[i].dev = &rte_eth_devices[i]; + + event_eth_rx_adapter[id] = rx_adapter; + if (conf_cb == default_conf_cb) + rx_adapter->default_cb_arg = 1; + return 0; +} + +int +rte_event_eth_rx_adapter_create(uint8_t id, uint8_t dev_id, + struct rte_event_port_conf *port_config) +{ + struct rte_event_port_conf *pc; + int ret; + + if (port_config == NULL) + return -EINVAL; + RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL); + + pc = rte_malloc(NULL, sizeof(*pc), 0); + if (pc == NULL) + return -ENOMEM; + *pc = *port_config; + ret = rte_event_eth_rx_adapter_create_ext(id, dev_id, + default_conf_cb, + pc); + if (ret) + rte_free(pc); + return ret; +} + +int +rte_event_eth_rx_adapter_free(uint8_t id) +{ + struct rte_event_eth_rx_adapter *rx_adapter; + + RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL); + + rx_adapter = id_to_rx_adapter(id); + if (rx_adapter == NULL) + return -EINVAL; + + if (rx_adapter->nb_queues) { + RTE_EDEV_LOG_ERR("%" PRIu16 " Rx queues not deleted", + rx_adapter->nb_queues); + return -EBUSY; + } + + if (rx_adapter->default_cb_arg) + rte_free(rx_adapter->conf_arg); + rte_free(rx_adapter->eth_devices); + rte_free(rx_adapter); + event_eth_rx_adapter[id] = NULL; + + return 0; +} + +int +rte_event_eth_rx_adapter_queue_add(uint8_t id, + uint8_t eth_dev_id, + int32_t rx_queue_id, + const struct rte_event_eth_rx_adapter_queue_conf *queue_conf) +{ + int ret; + uint32_t cap; + struct rte_event_eth_rx_adapter *rx_adapter; + struct rte_eventdev *dev; + struct eth_device_info *dev_info; + int start_service; + + RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL); + RTE_ETH_VALID_PORTID_OR_ERR_RET(eth_dev_id, -EINVAL); + + rx_adapter = id_to_rx_adapter(id); + if ((rx_adapter == NULL) || (queue_conf == NULL)) + return -EINVAL; + + dev = &rte_eventdevs[rx_adapter->eventdev_id]; + ret = rte_event_eth_rx_adapter_caps_get(rx_adapter->eventdev_id, + eth_dev_id, + &cap); + if (ret) { + RTE_EDEV_LOG_ERR("Failed to get adapter caps edev %" PRIu8 + "eth port %" PRIu8, id, eth_dev_id); + return ret; + } + + if ((cap & RTE_EVENT_ETH_RX_ADAPTER_CAP_OVERRIDE_FLOW_ID) == 0 + && (queue_conf->rx_queue_flags & + RTE_EVENT_ETH_RX_ADAPTER_QUEUE_FLOW_ID_VALID)) { + RTE_EDEV_LOG_ERR("Flow ID override is not supported," + " eth port: %" PRIu8 " adapter id: %" PRIu8, + eth_dev_id, id); + return -EINVAL; + } + + if ((cap & RTE_EVENT_ETH_RX_ADAPTER_CAP_MULTI_EVENTQ) == 0 && + (rx_queue_id != -1)) { + RTE_EDEV_LOG_ERR("Rx queues can only be connected to single " + "event queue id %u eth port %u", id, eth_dev_id); + return -EINVAL; + } + + if (rx_queue_id != -1 && (uint16_t)rx_queue_id >= + rte_eth_devices[eth_dev_id].data->nb_rx_queues) { + RTE_EDEV_LOG_ERR("Invalid rx queue_id %" PRIu16, + (uint16_t)rx_queue_id); + return -EINVAL; + } + + start_service = 0; + dev_info = &rx_adapter->eth_devices[eth_dev_id]; + + if (cap & RTE_EVENT_ETH_RX_ADAPTER_CAP_INTERNAL_PORT) { + RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->eth_rx_adapter_queue_add, + -ENOTSUP); + if (dev_info->rx_queue == NULL) { + dev_info->rx_queue = + rte_zmalloc_socket(rx_adapter->mem_name, + dev_info->dev->data->nb_rx_queues * + sizeof(struct eth_rx_queue_info), 0, + rx_adapter->socket_id); + if (dev_info->rx_queue == NULL) + return -ENOMEM; + } + + ret = (*dev->dev_ops->eth_rx_adapter_queue_add)(dev, + &rte_eth_devices[eth_dev_id], + rx_queue_id, queue_conf); + if (ret == 0) { + update_queue_info(rx_adapter, + &rx_adapter->eth_devices[eth_dev_id], + rx_queue_id, + 1); + } + } else { + rte_spinlock_lock(&rx_adapter->rx_lock); + ret = init_service(rx_adapter, id); + if (ret == 0) + ret = add_rx_queue(rx_adapter, eth_dev_id, rx_queue_id, + queue_conf); + rte_spinlock_unlock(&rx_adapter->rx_lock); + if (ret == 0) + start_service = !!sw_rx_adapter_queue_count(rx_adapter); + } + + if (ret) + return ret; + + if (start_service) + rte_service_component_runstate_set(rx_adapter->service_id, 1); + + return 0; +} + +int +rte_event_eth_rx_adapter_queue_del(uint8_t id, uint8_t eth_dev_id, + int32_t rx_queue_id) +{ + int ret = 0; + struct rte_eventdev *dev; + struct rte_event_eth_rx_adapter *rx_adapter; + struct eth_device_info *dev_info; + uint32_t cap; + uint16_t i; + + RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL); + RTE_ETH_VALID_PORTID_OR_ERR_RET(eth_dev_id, -EINVAL); + + rx_adapter = id_to_rx_adapter(id); + if (rx_adapter == NULL) + return -EINVAL; + + dev = &rte_eventdevs[rx_adapter->eventdev_id]; + ret = rte_event_eth_rx_adapter_caps_get(rx_adapter->eventdev_id, + eth_dev_id, + &cap); + if (ret) + return ret; + + if (rx_queue_id != -1 && (uint16_t)rx_queue_id >= + rte_eth_devices[eth_dev_id].data->nb_rx_queues) { + RTE_EDEV_LOG_ERR("Invalid rx queue_id %" PRIu16, + (uint16_t)rx_queue_id); + return -EINVAL; + } + + dev_info = &rx_adapter->eth_devices[eth_dev_id]; + + if (cap & RTE_EVENT_ETH_RX_ADAPTER_CAP_INTERNAL_PORT) { + RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->eth_rx_adapter_queue_del, + -ENOTSUP); + ret = (*dev->dev_ops->eth_rx_adapter_queue_del)(dev, + &rte_eth_devices[eth_dev_id], + rx_queue_id); + if (ret == 0) { + update_queue_info(rx_adapter, + &rx_adapter->eth_devices[eth_dev_id], + rx_queue_id, + 0); + if (dev_info->nb_dev_queues == 0) { + rte_free(dev_info->rx_queue); + dev_info->rx_queue = NULL; + } + } + } else { + int rc; + rte_spinlock_lock(&rx_adapter->rx_lock); + if (rx_queue_id == -1) { + for (i = 0; i < dev_info->dev->data->nb_rx_queues; i++) + event_eth_rx_adapter_queue_del(rx_adapter, + dev_info, + i); + } else { + event_eth_rx_adapter_queue_del(rx_adapter, + dev_info, + (uint16_t)rx_queue_id); + } + + rc = eth_poll_wrr_calc(rx_adapter); + if (rc) + RTE_EDEV_LOG_ERR("WRR recalculation failed %" PRId32, + rc); + + if (dev_info->nb_dev_queues == 0) { + rte_free(dev_info->rx_queue); + dev_info->rx_queue = NULL; + } + + rte_spinlock_unlock(&rx_adapter->rx_lock); + rte_service_component_runstate_set(rx_adapter->service_id, + sw_rx_adapter_queue_count(rx_adapter)); + } + + return ret; +} + + +int +rte_event_eth_rx_adapter_start(uint8_t id) +{ + return rx_adapter_ctrl(id, 1); +} + +int +rte_event_eth_rx_adapter_stop(uint8_t id) +{ + return rx_adapter_ctrl(id, 0); +} + +int +rte_event_eth_rx_adapter_stats_get(uint8_t id, + struct rte_event_eth_rx_adapter_stats *stats) +{ + struct rte_event_eth_rx_adapter *rx_adapter; + struct rte_event_eth_rx_adapter_stats dev_stats_sum = { 0 }; + struct rte_event_eth_rx_adapter_stats dev_stats; + struct rte_eventdev *dev; + struct eth_device_info *dev_info; + uint32_t i; + int ret; + + RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL); + + rx_adapter = id_to_rx_adapter(id); + if (rx_adapter == NULL || stats == NULL) + return -EINVAL; + + dev = &rte_eventdevs[rx_adapter->eventdev_id]; + memset(stats, 0, sizeof(*stats)); + for (i = 0; i < rte_eth_dev_count(); i++) { + dev_info = &rx_adapter->eth_devices[i]; + if (dev_info->internal_event_port == 0 || + dev->dev_ops->eth_rx_adapter_stats_get == NULL) + continue; + ret = (*dev->dev_ops->eth_rx_adapter_stats_get)(dev, + &rte_eth_devices[i], + &dev_stats); + if (ret) + continue; + dev_stats_sum.rx_packets += dev_stats.rx_packets; + dev_stats_sum.rx_enq_count += dev_stats.rx_enq_count; + } + + if (rx_adapter->service_inited) + *stats = rx_adapter->stats; + + stats->rx_packets += dev_stats_sum.rx_packets; + stats->rx_enq_count += dev_stats_sum.rx_enq_count; + return 0; +} + +int +rte_event_eth_rx_adapter_stats_reset(uint8_t id) +{ + struct rte_event_eth_rx_adapter *rx_adapter; + struct rte_eventdev *dev; + struct eth_device_info *dev_info; + uint32_t i; + + RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL); + + rx_adapter = id_to_rx_adapter(id); + if (rx_adapter == NULL) + return -EINVAL; + + dev = &rte_eventdevs[rx_adapter->eventdev_id]; + for (i = 0; i < rte_eth_dev_count(); i++) { + dev_info = &rx_adapter->eth_devices[i]; + if (dev_info->internal_event_port == 0 || + dev->dev_ops->eth_rx_adapter_stats_reset == NULL) + continue; + (*dev->dev_ops->eth_rx_adapter_stats_reset)(dev, + &rte_eth_devices[i]); + } + + memset(&rx_adapter->stats, 0, sizeof(rx_adapter->stats)); + return 0; +} + +int +rte_event_eth_rx_adapter_service_id_get(uint8_t id, uint32_t *service_id) +{ + struct rte_event_eth_rx_adapter *rx_adapter; + + RTE_EVENT_ETH_RX_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL); + + rx_adapter = id_to_rx_adapter(id); + if (rx_adapter == NULL || service_id == NULL) + return -EINVAL; + + if (rx_adapter->service_inited) + *service_id = rx_adapter->service_id; + + return rx_adapter->service_inited ? 0 : -ESRCH; +} diff --git a/dpdk/lib/librte_eventdev/rte_event_eth_rx_adapter.h b/dpdk/lib/librte_eventdev/rte_event_eth_rx_adapter.h new file mode 100644 index 00000000..6a9e7edf --- /dev/null +++ b/dpdk/lib/librte_eventdev/rte_event_eth_rx_adapter.h @@ -0,0 +1,444 @@ +/* + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_EVENT_ETH_RX_ADAPTER_ +#define _RTE_EVENT_ETH_RX_ADAPTER_ + +/** + * @file + * + * RTE Event Ethernet Rx Adapter + * + * An eventdev-based packet processing application enqueues/dequeues mbufs + * to/from the event device. Packet flow from the ethernet device to the event + * device can be accomplished using either HW or SW mechanisms depending on the + * platform and the particular combination of ethernet and event devices. The + * event ethernet Rx adapter provides common APIs to configure the packet flow + * from the ethernet devices to event devices across both these transfer + * mechanisms. + * + * The adapter uses a EAL service core function for SW based packet transfer + * and uses the eventdev PMD functions to configure HW based packet transfer + * between the ethernet device and the event device. + * + * The ethernet Rx event adapter's functions are: + * - rte_event_eth_rx_adapter_create_ext() + * - rte_event_eth_rx_adapter_create() + * - rte_event_eth_rx_adapter_free() + * - rte_event_eth_rx_adapter_queue_add() + * - rte_event_eth_rx_adapter_queue_del() + * - rte_event_eth_rx_adapter_start() + * - rte_event_eth_rx_adapter_stop() + * - rte_event_eth_rx_adapter_stats_get() + * - rte_event_eth_rx_adapter_stats_reset() + * + * The application creates an ethernet to event adapter using + * rte_event_eth_rx_adapter_create_ext() or rte_event_eth_rx_adapter_create() + * functions. + * The adapter needs to know which ethernet rx queues to poll for mbufs as well + * as event device parameters such as the event queue identifier, event + * priority and scheduling type that the adapter should use when constructing + * events. The rte_event_eth_rx_adapter_queue_add() function is provided for + * this purpose. + * The servicing weight parameter in the rte_event_eth_rx_adapter_queue_conf + * is applicable when the Rx adapter uses a service core function and is + * intended to provide application control of the frequency of polling ethernet + * device receive queues, for example, the application may want to poll higher + * priority queues with a higher frequency but at the same time not starve + * lower priority queues completely. If this parameter is zero and the receive + * interrupt is enabled when configuring the device, the receive queue is + * interrupt driven; else, the queue is assigned a servicing weight of one. + * + * The application can start/stop the adapter using the + * rte_event_eth_rx_adapter_start() and the rte_event_eth_rx_adapter_stop() + * functions. If the adapter uses a rte_service function, then the application + * is also required to assign a core to the service function and control the + * service core using the rte_service APIs. The + * rte_event_eth_rx_adapter_service_id_get() function can be used to retrieve + * the service function ID of the adapter in this case. + * + * Note: Interrupt driven receive queues are currently unimplemented. + */ + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +#include + +#include "rte_eventdev.h" + +#define RTE_EVENT_ETH_RX_ADAPTER_MAX_INSTANCE 32 + +/* struct rte_event_eth_rx_adapter_queue_conf flags definitions */ +#define RTE_EVENT_ETH_RX_ADAPTER_QUEUE_FLOW_ID_VALID 0x1 +/**< This flag indicates the flow identifier is valid + * @see rte_event_eth_rx_adapter_queue_conf::rx_queue_flags + */ + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Adapter configuration structure that the adapter configuration callback + * function is expected to fill out + * @see rte_event_eth_rx_adapter_conf_cb + */ +struct rte_event_eth_rx_adapter_conf { + uint8_t event_port_id; + /**< Event port identifier, the adapter enqueues mbuf events to this + * port. + */ + uint32_t max_nb_rx; + /**< The adapter can return early if it has processed at least + * max_nb_rx mbufs. This isn't treated as a requirement; batching may + * cause the adapter to process more than max_nb_rx mbufs. + */ +}; + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Function type used for adapter configuration callback. The callback is + * used to fill in members of the struct rte_event_eth_rx_adapter_conf, this + * callback is invoked when creating a SW service for packet transfer from + * ethdev queues to the event device. The SW service is created within the + * rte_event_eth_rx_adapter_queue_add() function if SW based packet transfers + * from ethdev queues to the event device are required. + * + * @param id + * Adapter identifier. + * + * @param dev_id + * Event device identifier. + * + * @param [out] conf + * Structure that needs to be populated by this callback. + * + * @param arg + * Argument to the callback. This is the same as the conf_arg passed to the + * rte_event_eth_rx_adapter_create_ext(). + */ +typedef int (*rte_event_eth_rx_adapter_conf_cb) (uint8_t id, uint8_t dev_id, + struct rte_event_eth_rx_adapter_conf *conf, + void *arg); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Rx queue configuration structure + */ +struct rte_event_eth_rx_adapter_queue_conf { + uint32_t rx_queue_flags; + /**< Flags for handling received packets + * @see RTE_EVENT_ETH_RX_ADAPTER_QUEUE_FLOW_ID_VALID + */ + uint16_t servicing_weight; + /**< Relative polling frequency of ethernet receive queue when the + * adapter uses a service core function for ethernet to event device + * transfers. If it is set to zero, the Rx queue is interrupt driven + * (unless rx queue interrupts are not enabled for the ethernet + * device). + */ + struct rte_event ev; + /**< + * The values from the following event fields will be used when + * queuing mbuf events: + * - event_queue_id: Targeted event queue ID for received packets. + * - event_priority: Event priority of packets from this Rx queue in + * the event queue relative to other events. + * - sched_type: Scheduling type for packets from this Rx queue. + * - flow_id: If the RTE_ETH_RX_EVENT_ADAPTER_QUEUE_FLOW_ID_VALID bit + * is set in rx_queue_flags, this flow_id is used for all + * packets received from this queue. Otherwise the flow ID + * is set to the RSS hash of the src and dst IPv4/6 + * addresses. + * + * The event adapter sets ev.event_type to RTE_EVENT_TYPE_ETHDEV in the + * enqueued event. + */ +}; + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * A structure used to retrieve statistics for an eth rx adapter instance. + */ +struct rte_event_eth_rx_adapter_stats { + uint64_t rx_poll_count; + /**< Receive queue poll count */ + uint64_t rx_packets; + /**< Received packet count */ + uint64_t rx_enq_count; + /**< Eventdev enqueue count */ + uint64_t rx_enq_retry; + /**< Eventdev enqueue retry count */ + uint64_t rx_enq_start_ts; + /**< Rx enqueue start timestamp */ + uint64_t rx_enq_block_cycles; + /**< Cycles for which the service is blocked by the event device, + * i.e, the service fails to enqueue to the event device. + */ + uint64_t rx_enq_end_ts; + /**< Latest timestamp at which the service is unblocked + * by the event device. The start, end timestamps and + * block cycles can be used to compute the percentage of + * cycles the service is blocked by the event device. + */ +}; + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Create a new ethernet Rx event adapter with the specified identifier. + * + * @param id + * The identifier of the ethernet Rx event adapter. + * + * @param dev_id + * The identifier of the device to configure. + * + * @param conf_cb + * Callback function that fills in members of a + * struct rte_event_eth_rx_adapter_conf struct passed into + * it. + * + * @param conf_arg + * Argument that is passed to the conf_cb function. + * + * @return + * - 0: Success + * - <0: Error code on failure + */ +int rte_event_eth_rx_adapter_create_ext(uint8_t id, uint8_t dev_id, + rte_event_eth_rx_adapter_conf_cb conf_cb, + void *conf_arg); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Create a new ethernet Rx event adapter with the specified identifier. + * This function uses an internal configuration function that creates an event + * port. This default function reconfigures the event device with an + * additional event port and setups up the event port using the port_config + * parameter passed into this function. In case the application needs more + * control in configuration of the service, it should use the + * rte_event_eth_rx_adapter_create_ext() version. + * + * @param id + * The identifier of the ethernet Rx event adapter. + * + * @param dev_id + * The identifier of the device to configure. + * + * @param port_config + * Argument of type *rte_event_port_conf* that is passed to the conf_cb + * function. + * + * @return + * - 0: Success + * - <0: Error code on failure + */ +int rte_event_eth_rx_adapter_create(uint8_t id, uint8_t dev_id, + struct rte_event_port_conf *port_config); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Free an event adapter + * + * @param id + * Adapter identifier. + * + * @return + * - 0: Success + * - <0: Error code on failure, If the adapter still has Rx queues + * added to it, the function returns -EBUSY. + */ +int rte_event_eth_rx_adapter_free(uint8_t id); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Add receive queue to an event adapter. After a queue has been + * added to the event adapter, the result of the application calling + * rte_eth_rx_burst(eth_dev_id, rx_queue_id, ..) is undefined. + * + * @param id + * Adapter identifier. + * + * @param eth_dev_id + * Port identifier of Ethernet device. + * + * @param rx_queue_id + * Ethernet device receive queue index. + * If rx_queue_id is -1, then all Rx queues configured for + * the device are added. If the ethdev Rx queues can only be + * connected to a single event queue then rx_queue_id is + * required to be -1. + * @see RTE_EVENT_ETH_RX_ADAPTER_CAP_MULTI_EVENTQ + * + * @param conf + * Additional configuration structure of type *rte_event_eth_rx_adapter_conf* + * + * @return + * - 0: Success, Receive queue added correctly. + * - <0: Error code on failure. + */ +int rte_event_eth_rx_adapter_queue_add(uint8_t id, + uint8_t eth_dev_id, + int32_t rx_queue_id, + const struct rte_event_eth_rx_adapter_queue_conf *conf); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Delete receive queue from an event adapter. + * + * @param id + * Adapter identifier. + * + * @param eth_dev_id + * Port identifier of Ethernet device. + * + * @param rx_queue_id + * Ethernet device receive queue index. + * If rx_queue_id is -1, then all Rx queues configured for + * the device are deleted. If the ethdev Rx queues can only be + * connected to a single event queue then rx_queue_id is + * required to be -1. + * @see RTE_EVENT_ETH_RX_ADAPTER_CAP_MULTI_EVENTQ + * + * @return + * - 0: Success, Receive queue deleted correctly. + * - <0: Error code on failure. + */ +int rte_event_eth_rx_adapter_queue_del(uint8_t id, uint8_t eth_dev_id, + int32_t rx_queue_id); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Start ethernet Rx event adapter + * + * @param id + * Adapter identifier. + * + * @return + * - 0: Success, Adapter started correctly. + * - <0: Error code on failure. + */ +int rte_event_eth_rx_adapter_start(uint8_t id); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Stop ethernet Rx event adapter + * + * @param id + * Adapter identifier. + * + * @return + * - 0: Success, Adapter started correctly. + * - <0: Error code on failure. + */ +int rte_event_eth_rx_adapter_stop(uint8_t id); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Retrieve statistics for an adapter + * + * @param id + * Adapter identifier. + * + * @param [out] stats + * A pointer to structure used to retrieve statistics for an adapter. + * + * @return + * - 0: Success, retrieved successfully. + * - <0: Error code on failure. + */ +int rte_event_eth_rx_adapter_stats_get(uint8_t id, + struct rte_event_eth_rx_adapter_stats *stats); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Reset statistics for an adapter. + * + * @param id + * Adapter identifier. + * + * @return + * - 0: Success, statistics reset successfully. + * - <0: Error code on failure. + */ +int rte_event_eth_rx_adapter_stats_reset(uint8_t id); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Retrieve the service ID of an adapter. If the adapter doesn't use + * a rte_service function, this function returns -ESRCH. + * + * @param id + * Adapter identifier. + * + * @param [out] service_id + * A pointer to a uint32_t, to be filled in with the service id. + * + * @return + * - 0: Success + * - <0: Error code on failure, if the adapter doesn't use a rte_service + * function, this function returns -ESRCH. + */ +int rte_event_eth_rx_adapter_service_id_get(uint8_t id, uint32_t *service_id); + +#ifdef __cplusplus +} +#endif +#endif /* _RTE_EVENT_ETH_RX_ADAPTER_ */ diff --git a/dpdk/lib/librte_eventdev/rte_event_ring.c b/dpdk/lib/librte_eventdev/rte_event_ring.c new file mode 100644 index 00000000..b14c2127 --- /dev/null +++ b/dpdk/lib/librte_eventdev/rte_event_ring.c @@ -0,0 +1,207 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include + +#include +#include +#include +#include +#include "rte_event_ring.h" + +TAILQ_HEAD(rte_event_ring_list, rte_tailq_entry); + +static struct rte_tailq_elem rte_event_ring_tailq = { + .name = RTE_TAILQ_EVENT_RING_NAME, +}; +EAL_REGISTER_TAILQ(rte_event_ring_tailq) + +int +rte_event_ring_init(struct rte_event_ring *r, const char *name, + unsigned int count, unsigned int flags) +{ + /* compilation-time checks */ + RTE_BUILD_BUG_ON((sizeof(struct rte_event_ring) & + RTE_CACHE_LINE_MASK) != 0); + + /* init the ring structure */ + return rte_ring_init(&r->r, name, count, flags); +} + +/* create the ring */ +struct rte_event_ring * +rte_event_ring_create(const char *name, unsigned int count, int socket_id, + unsigned int flags) +{ + char mz_name[RTE_MEMZONE_NAMESIZE]; + struct rte_event_ring *r; + struct rte_tailq_entry *te; + const struct rte_memzone *mz; + ssize_t ring_size; + int mz_flags = 0; + struct rte_event_ring_list *ring_list = NULL; + const unsigned int requested_count = count; + int ret; + + ring_list = RTE_TAILQ_CAST(rte_event_ring_tailq.head, + rte_event_ring_list); + + /* for an exact size ring, round up from count to a power of two */ + if (flags & RING_F_EXACT_SZ) + count = rte_align32pow2(count + 1); + else if (!rte_is_power_of_2(count)) { + rte_errno = EINVAL; + return NULL; + } + + ring_size = sizeof(*r) + (count * sizeof(struct rte_event)); + + ret = snprintf(mz_name, sizeof(mz_name), "%s%s", + RTE_RING_MZ_PREFIX, name); + if (ret < 0 || ret >= (int)sizeof(mz_name)) { + rte_errno = ENAMETOOLONG; + return NULL; + } + + te = rte_zmalloc("RING_TAILQ_ENTRY", sizeof(*te), 0); + if (te == NULL) { + RTE_LOG(ERR, RING, "Cannot reserve memory for tailq\n"); + rte_errno = ENOMEM; + return NULL; + } + + rte_rwlock_write_lock(RTE_EAL_TAILQ_RWLOCK); + + /* + * reserve a memory zone for this ring. If we can't get rte_config or + * we are secondary process, the memzone_reserve function will set + * rte_errno for us appropriately - hence no check in this this function + */ + mz = rte_memzone_reserve(mz_name, ring_size, socket_id, mz_flags); + if (mz != NULL) { + r = mz->addr; + /* + * no need to check return value here, we already checked the + * arguments above + */ + rte_event_ring_init(r, name, requested_count, flags); + + te->data = (void *) r; + r->r.memzone = mz; + + TAILQ_INSERT_TAIL(ring_list, te, next); + } else { + r = NULL; + RTE_LOG(ERR, RING, "Cannot reserve memory\n"); + rte_free(te); + } + rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK); + + return r; +} + + +struct rte_event_ring * +rte_event_ring_lookup(const char *name) +{ + struct rte_tailq_entry *te; + struct rte_event_ring *r = NULL; + struct rte_event_ring_list *ring_list; + + ring_list = RTE_TAILQ_CAST(rte_event_ring_tailq.head, + rte_event_ring_list); + + rte_rwlock_read_lock(RTE_EAL_TAILQ_RWLOCK); + + TAILQ_FOREACH(te, ring_list, next) { + r = (struct rte_event_ring *) te->data; + if (strncmp(name, r->r.name, RTE_RING_NAMESIZE) == 0) + break; + } + + rte_rwlock_read_unlock(RTE_EAL_TAILQ_RWLOCK); + + if (te == NULL) { + rte_errno = ENOENT; + return NULL; + } + + return r; +} + +/* free the ring */ +void +rte_event_ring_free(struct rte_event_ring *r) +{ + struct rte_event_ring_list *ring_list = NULL; + struct rte_tailq_entry *te; + + if (r == NULL) + return; + + /* + * Ring was not created with rte_event_ring_create, + * therefore, there is no memzone to free. + */ + if (r->r.memzone == NULL) { + RTE_LOG(ERR, RING, + "Cannot free ring (not created with rte_event_ring_create()"); + return; + } + + if (rte_memzone_free(r->r.memzone) != 0) { + RTE_LOG(ERR, RING, "Cannot free memory\n"); + return; + } + + ring_list = RTE_TAILQ_CAST(rte_event_ring_tailq.head, + rte_event_ring_list); + rte_rwlock_write_lock(RTE_EAL_TAILQ_RWLOCK); + + /* find out tailq entry */ + TAILQ_FOREACH(te, ring_list, next) { + if (te->data == (void *) r) + break; + } + + if (te == NULL) { + rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK); + return; + } + + TAILQ_REMOVE(ring_list, te, next); + + rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK); + + rte_free(te); +} diff --git a/dpdk/lib/librte_eventdev/rte_event_ring.h b/dpdk/lib/librte_eventdev/rte_event_ring.h new file mode 100644 index 00000000..ea9b6885 --- /dev/null +++ b/dpdk/lib/librte_eventdev/rte_event_ring.h @@ -0,0 +1,308 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** + * @file + * RTE Event Ring + * + * This provides a ring implementation for passing rte_event structures + * from one core to another. + */ + +#ifndef _RTE_EVENT_RING_ +#define _RTE_EVENT_RING_ + +#include + +#include +#include +#include +#include +#include "rte_eventdev.h" + +#define RTE_TAILQ_EVENT_RING_NAME "RTE_EVENT_RING" + +/** + * Generic ring structure for passing rte_event objects from core to core. + * + * Based on the primitives given in the rte_ring library. Designed to be + * used inside software eventdev implementations and by applications + * directly as needed. + */ +struct rte_event_ring { + struct rte_ring r; +}; + +/** + * Returns the number of events in the ring + * + * @param r + * pointer to the event ring + * @return + * the number of events in the ring + */ +static __rte_always_inline unsigned int +rte_event_ring_count(const struct rte_event_ring *r) +{ + return rte_ring_count(&r->r); +} + +/** + * Returns the amount of free space in the ring + * + * @param r + * pointer to the event ring + * @return + * the number of free slots in the ring, i.e. the number of events that + * can be successfully enqueued before dequeue must be called + */ +static __rte_always_inline unsigned int +rte_event_ring_free_count(const struct rte_event_ring *r) +{ + return rte_ring_free_count(&r->r); +} + +/** + * Enqueue a set of events onto a ring + * + * Note: this API enqueues by copying the events themselves onto the ring, + * rather than just placing a pointer to each event onto the ring. This + * means that statically-allocated events can safely be enqueued by this + * API. + * + * @param r + * pointer to the event ring + * @param events + * pointer to an array of struct rte_event objects + * @param n + * number of events in the array to enqueue + * @param free_space + * if non-null, is updated to indicate the amount of free space in the + * ring once the enqueue has completed. + * @return + * the number of elements, n', enqueued to the ring, 0 <= n' <= n + */ +static __rte_always_inline unsigned int +rte_event_ring_enqueue_burst(struct rte_event_ring *r, + const struct rte_event *events, + unsigned int n, uint16_t *free_space) +{ + uint32_t prod_head, prod_next; + uint32_t free_entries; + + n = __rte_ring_move_prod_head(&r->r, r->r.prod.single, n, + RTE_RING_QUEUE_VARIABLE, + &prod_head, &prod_next, &free_entries); + if (n == 0) + goto end; + + ENQUEUE_PTRS(&r->r, &r[1], prod_head, events, n, struct rte_event); + rte_smp_wmb(); + + update_tail(&r->r.prod, prod_head, prod_next, 1); +end: + if (free_space != NULL) + *free_space = free_entries - n; + return n; +} + +/** + * Dequeue a set of events from a ring + * + * Note: this API does not work with pointers to events, rather it copies + * the events themselves to the destination ``events`` buffer. + * + * @param r + * pointer to the event ring + * @param events + * pointer to an array to hold the struct rte_event objects + * @param n + * number of events that can be held in the ``events`` array + * @param available + * if non-null, is updated to indicate the number of events remaining in + * the ring once the dequeue has completed + * @return + * the number of elements, n', dequeued from the ring, 0 <= n' <= n + */ +static __rte_always_inline unsigned int +rte_event_ring_dequeue_burst(struct rte_event_ring *r, + struct rte_event *events, + unsigned int n, uint16_t *available) +{ + uint32_t cons_head, cons_next; + uint32_t entries; + + n = __rte_ring_move_cons_head(&r->r, r->r.cons.single, n, + RTE_RING_QUEUE_VARIABLE, + &cons_head, &cons_next, &entries); + if (n == 0) + goto end; + + DEQUEUE_PTRS(&r->r, &r[1], cons_head, events, n, struct rte_event); + rte_smp_rmb(); + + update_tail(&r->r.cons, cons_head, cons_next, 1); + +end: + if (available != NULL) + *available = entries - n; + return n; +} + +/* + * Initializes an already-allocated ring structure + * + * @param r + * pointer to the ring memory to be initialized + * @param name + * name to be given to the ring + * @param count + * the number of elements to be stored in the ring. If the flag + * ``RING_F_EXACT_SZ`` is not set, this must be a power of 2, and the actual + * usable space in the ring will be ``count - 1`` entries. If the flag + * ``RING_F_EXACT_SZ`` is set, the this can be any value up to the ring size + * limit - 1, and the usable space will be exactly that requested. + * @param flags + * An OR of the following: + * - RING_F_SP_ENQ: If this flag is set, the default behavior when + * using ``rte_ring_enqueue()`` or ``rte_ring_enqueue_bulk()`` + * is "single-producer". Otherwise, it is "multi-producers". + * - RING_F_SC_DEQ: If this flag is set, the default behavior when + * using ``rte_ring_dequeue()`` or ``rte_ring_dequeue_bulk()`` + * is "single-consumer". Otherwise, it is "multi-consumers". + * - RING_F_EXACT_SZ: If this flag is set, the ``count`` parameter is to + * be taken as the exact usable size of the ring, and as such does not + * need to be a power of 2. The underlying ring memory should be a + * power-of-2 size greater than the count value. + * @return + * 0 on success, or a negative value on error. + */ +int +rte_event_ring_init(struct rte_event_ring *r, const char *name, + unsigned int count, unsigned int flags); + +/* + * Create an event ring structure + * + * This function allocates memory and initializes an event ring inside that + * memory. + * + * @param name + * name to be given to the ring + * @param count + * the number of elements to be stored in the ring. If the flag + * ``RING_F_EXACT_SZ`` is not set, this must be a power of 2, and the actual + * usable space in the ring will be ``count - 1`` entries. If the flag + * ``RING_F_EXACT_SZ`` is set, the this can be any value up to the ring size + * limit - 1, and the usable space will be exactly that requested. + * @param socket_id + * The *socket_id* argument is the socket identifier in case of + * NUMA. The value can be *SOCKET_ID_ANY* if there is no NUMA + * constraint for the reserved zone. + * @param flags + * An OR of the following: + * - RING_F_SP_ENQ: If this flag is set, the default behavior when + * using ``rte_ring_enqueue()`` or ``rte_ring_enqueue_bulk()`` + * is "single-producer". Otherwise, it is "multi-producers". + * - RING_F_SC_DEQ: If this flag is set, the default behavior when + * using ``rte_ring_dequeue()`` or ``rte_ring_dequeue_bulk()`` + * is "single-consumer". Otherwise, it is "multi-consumers". + * - RING_F_EXACT_SZ: If this flag is set, the ``count`` parameter is to + * be taken as the exact usable size of the ring, and as such does not + * need to be a power of 2. The underlying ring memory should be a + * power-of-2 size greater than the count value. + * @return + * On success, the pointer to the new allocated ring. NULL on error with + * rte_errno set appropriately. Possible errno values include: + * - E_RTE_NO_CONFIG - function could not get pointer to rte_config structure + * - E_RTE_SECONDARY - function was called from a secondary process instance + * - EINVAL - count provided is not a power of 2 + * - ENOSPC - the maximum number of memzones has already been allocated + * - EEXIST - a memzone with the same name already exists + * - ENOMEM - no appropriate memory area found in which to create memzone + */ +struct rte_event_ring * +rte_event_ring_create(const char *name, unsigned int count, int socket_id, + unsigned int flags); + +/** + * Search for an event ring based on its name + * + * @param name + * The name of the ring. + * @return + * The pointer to the ring matching the name, or NULL if not found, + * with rte_errno set appropriately. Possible rte_errno values include: + * - ENOENT - required entry not available to return. + */ +struct rte_event_ring * +rte_event_ring_lookup(const char *name); + +/** + * De-allocate all memory used by the ring. + * + * @param r + * Ring to free + */ +void +rte_event_ring_free(struct rte_event_ring *r); + +/** + * Return the size of the event ring. + * + * @param r + * A pointer to the ring structure. + * @return + * The size of the data store used by the ring. + * NOTE: this is not the same as the usable space in the ring. To query that + * use ``rte_ring_get_capacity()``. + */ +static inline unsigned int +rte_event_ring_get_size(const struct rte_event_ring *r) +{ + return rte_ring_get_size(&r->r); +} + +/** + * Return the number of elements which can be stored in the event ring. + * + * @param r + * A pointer to the ring structure. + * @return + * The usable size of the ring. + */ +static inline unsigned int +rte_event_ring_get_capacity(const struct rte_event_ring *r) +{ + return rte_ring_get_capacity(&r->r); +} +#endif diff --git a/dpdk/lib/librte_eventdev/rte_eventdev.c b/dpdk/lib/librte_eventdev/rte_eventdev.c new file mode 100644 index 00000000..e0c2a787 --- /dev/null +++ b/dpdk/lib/librte_eventdev/rte_eventdev.c @@ -0,0 +1,1284 @@ +/* + * BSD LICENSE + * + * Copyright(c) 2016 Cavium, Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "rte_eventdev.h" +#include "rte_eventdev_pmd.h" + +struct rte_eventdev rte_event_devices[RTE_EVENT_MAX_DEVS]; + +struct rte_eventdev *rte_eventdevs = &rte_event_devices[0]; + +static struct rte_eventdev_global eventdev_globals = { + .nb_devs = 0 +}; + +struct rte_eventdev_global *rte_eventdev_globals = &eventdev_globals; + +/* Event dev north bound API implementation */ + +uint8_t +rte_event_dev_count(void) +{ + return rte_eventdev_globals->nb_devs; +} + +int +rte_event_dev_get_dev_id(const char *name) +{ + int i; + + if (!name) + return -EINVAL; + + for (i = 0; i < rte_eventdev_globals->nb_devs; i++) + if ((strcmp(rte_event_devices[i].data->name, name) + == 0) && + (rte_event_devices[i].attached == + RTE_EVENTDEV_ATTACHED)) + return i; + return -ENODEV; +} + +int +rte_event_dev_socket_id(uint8_t dev_id) +{ + struct rte_eventdev *dev; + + RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(dev_id, -EINVAL); + dev = &rte_eventdevs[dev_id]; + + return dev->data->socket_id; +} + +int +rte_event_dev_info_get(uint8_t dev_id, struct rte_event_dev_info *dev_info) +{ + struct rte_eventdev *dev; + + RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(dev_id, -EINVAL); + dev = &rte_eventdevs[dev_id]; + + if (dev_info == NULL) + return -EINVAL; + + memset(dev_info, 0, sizeof(struct rte_event_dev_info)); + + RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP); + (*dev->dev_ops->dev_infos_get)(dev, dev_info); + + dev_info->dequeue_timeout_ns = dev->data->dev_conf.dequeue_timeout_ns; + + dev_info->dev = dev->dev; + return 0; +} + +int +rte_event_eth_rx_adapter_caps_get(uint8_t dev_id, uint8_t eth_port_id, + uint32_t *caps) +{ + struct rte_eventdev *dev; + + RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(dev_id, -EINVAL); + RTE_ETH_VALID_PORTID_OR_ERR_RET(eth_port_id, -EINVAL); + + dev = &rte_eventdevs[dev_id]; + + if (caps == NULL) + return -EINVAL; + *caps = 0; + + return dev->dev_ops->eth_rx_adapter_caps_get ? + (*dev->dev_ops->eth_rx_adapter_caps_get)(dev, + &rte_eth_devices[eth_port_id], + caps) + : 0; +} + +static inline int +rte_event_dev_queue_config(struct rte_eventdev *dev, uint8_t nb_queues) +{ + uint8_t old_nb_queues = dev->data->nb_queues; + struct rte_event_queue_conf *queues_cfg; + unsigned int i; + + RTE_EDEV_LOG_DEBUG("Setup %d queues on device %u", nb_queues, + dev->data->dev_id); + + /* First time configuration */ + if (dev->data->queues_cfg == NULL && nb_queues != 0) { + /* Allocate memory to store queue configuration */ + dev->data->queues_cfg = rte_zmalloc_socket( + "eventdev->data->queues_cfg", + sizeof(dev->data->queues_cfg[0]) * nb_queues, + RTE_CACHE_LINE_SIZE, dev->data->socket_id); + if (dev->data->queues_cfg == NULL) { + dev->data->nb_queues = 0; + RTE_EDEV_LOG_ERR("failed to get mem for queue cfg," + "nb_queues %u", nb_queues); + return -(ENOMEM); + } + /* Re-configure */ + } else if (dev->data->queues_cfg != NULL && nb_queues != 0) { + RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->queue_release, -ENOTSUP); + + for (i = nb_queues; i < old_nb_queues; i++) + (*dev->dev_ops->queue_release)(dev, i); + + /* Re allocate memory to store queue configuration */ + queues_cfg = dev->data->queues_cfg; + queues_cfg = rte_realloc(queues_cfg, + sizeof(queues_cfg[0]) * nb_queues, + RTE_CACHE_LINE_SIZE); + if (queues_cfg == NULL) { + RTE_EDEV_LOG_ERR("failed to realloc queue cfg memory," + " nb_queues %u", nb_queues); + return -(ENOMEM); + } + dev->data->queues_cfg = queues_cfg; + + if (nb_queues > old_nb_queues) { + uint8_t new_qs = nb_queues - old_nb_queues; + + memset(queues_cfg + old_nb_queues, 0, + sizeof(queues_cfg[0]) * new_qs); + } + } else if (dev->data->queues_cfg != NULL && nb_queues == 0) { + RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->queue_release, -ENOTSUP); + + for (i = nb_queues; i < old_nb_queues; i++) + (*dev->dev_ops->queue_release)(dev, i); + } + + dev->data->nb_queues = nb_queues; + return 0; +} + +#define EVENT_QUEUE_SERVICE_PRIORITY_INVALID (0xdead) + +static inline int +rte_event_dev_port_config(struct rte_eventdev *dev, uint8_t nb_ports) +{ + uint8_t old_nb_ports = dev->data->nb_ports; + void **ports; + uint16_t *links_map; + struct rte_event_port_conf *ports_cfg; + unsigned int i; + + RTE_EDEV_LOG_DEBUG("Setup %d ports on device %u", nb_ports, + dev->data->dev_id); + + /* First time configuration */ + if (dev->data->ports == NULL && nb_ports != 0) { + dev->data->ports = rte_zmalloc_socket("eventdev->data->ports", + sizeof(dev->data->ports[0]) * nb_ports, + RTE_CACHE_LINE_SIZE, dev->data->socket_id); + if (dev->data->ports == NULL) { + dev->data->nb_ports = 0; + RTE_EDEV_LOG_ERR("failed to get mem for port meta data," + "nb_ports %u", nb_ports); + return -(ENOMEM); + } + + /* Allocate memory to store port configurations */ + dev->data->ports_cfg = + rte_zmalloc_socket("eventdev->ports_cfg", + sizeof(dev->data->ports_cfg[0]) * nb_ports, + RTE_CACHE_LINE_SIZE, dev->data->socket_id); + if (dev->data->ports_cfg == NULL) { + dev->data->nb_ports = 0; + RTE_EDEV_LOG_ERR("failed to get mem for port cfg," + "nb_ports %u", nb_ports); + return -(ENOMEM); + } + + /* Allocate memory to store queue to port link connection */ + dev->data->links_map = + rte_zmalloc_socket("eventdev->links_map", + sizeof(dev->data->links_map[0]) * nb_ports * + RTE_EVENT_MAX_QUEUES_PER_DEV, + RTE_CACHE_LINE_SIZE, dev->data->socket_id); + if (dev->data->links_map == NULL) { + dev->data->nb_ports = 0; + RTE_EDEV_LOG_ERR("failed to get mem for port_map area," + "nb_ports %u", nb_ports); + return -(ENOMEM); + } + for (i = 0; i < nb_ports * RTE_EVENT_MAX_QUEUES_PER_DEV; i++) + dev->data->links_map[i] = + EVENT_QUEUE_SERVICE_PRIORITY_INVALID; + } else if (dev->data->ports != NULL && nb_ports != 0) {/* re-config */ + RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->port_release, -ENOTSUP); + + ports = dev->data->ports; + ports_cfg = dev->data->ports_cfg; + links_map = dev->data->links_map; + + for (i = nb_ports; i < old_nb_ports; i++) + (*dev->dev_ops->port_release)(ports[i]); + + /* Realloc memory for ports */ + ports = rte_realloc(ports, sizeof(ports[0]) * nb_ports, + RTE_CACHE_LINE_SIZE); + if (ports == NULL) { + RTE_EDEV_LOG_ERR("failed to realloc port meta data," + " nb_ports %u", nb_ports); + return -(ENOMEM); + } + + /* Realloc memory for ports_cfg */ + ports_cfg = rte_realloc(ports_cfg, + sizeof(ports_cfg[0]) * nb_ports, + RTE_CACHE_LINE_SIZE); + if (ports_cfg == NULL) { + RTE_EDEV_LOG_ERR("failed to realloc port cfg mem," + " nb_ports %u", nb_ports); + return -(ENOMEM); + } + + /* Realloc memory to store queue to port link connection */ + links_map = rte_realloc(links_map, + sizeof(dev->data->links_map[0]) * nb_ports * + RTE_EVENT_MAX_QUEUES_PER_DEV, + RTE_CACHE_LINE_SIZE); + if (links_map == NULL) { + dev->data->nb_ports = 0; + RTE_EDEV_LOG_ERR("failed to realloc mem for port_map," + "nb_ports %u", nb_ports); + return -(ENOMEM); + } + + if (nb_ports > old_nb_ports) { + uint8_t new_ps = nb_ports - old_nb_ports; + unsigned int old_links_map_end = + old_nb_ports * RTE_EVENT_MAX_QUEUES_PER_DEV; + unsigned int links_map_end = + nb_ports * RTE_EVENT_MAX_QUEUES_PER_DEV; + + memset(ports + old_nb_ports, 0, + sizeof(ports[0]) * new_ps); + memset(ports_cfg + old_nb_ports, 0, + sizeof(ports_cfg[0]) * new_ps); + for (i = old_links_map_end; i < links_map_end; i++) + links_map[i] = + EVENT_QUEUE_SERVICE_PRIORITY_INVALID; + } + + dev->data->ports = ports; + dev->data->ports_cfg = ports_cfg; + dev->data->links_map = links_map; + } else if (dev->data->ports != NULL && nb_ports == 0) { + RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->port_release, -ENOTSUP); + + ports = dev->data->ports; + for (i = nb_ports; i < old_nb_ports; i++) + (*dev->dev_ops->port_release)(ports[i]); + } + + dev->data->nb_ports = nb_ports; + return 0; +} + +int +rte_event_dev_configure(uint8_t dev_id, + const struct rte_event_dev_config *dev_conf) +{ + struct rte_eventdev *dev; + struct rte_event_dev_info info; + int diag; + + RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(dev_id, -EINVAL); + dev = &rte_eventdevs[dev_id]; + + RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP); + RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_configure, -ENOTSUP); + + if (dev->data->dev_started) { + RTE_EDEV_LOG_ERR( + "device %d must be stopped to allow configuration", dev_id); + return -EBUSY; + } + + if (dev_conf == NULL) + return -EINVAL; + + (*dev->dev_ops->dev_infos_get)(dev, &info); + + /* Check dequeue_timeout_ns value is in limit */ + if (!(dev_conf->event_dev_cfg & RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT)) { + if (dev_conf->dequeue_timeout_ns && + (dev_conf->dequeue_timeout_ns < info.min_dequeue_timeout_ns + || dev_conf->dequeue_timeout_ns > + info.max_dequeue_timeout_ns)) { + RTE_EDEV_LOG_ERR("dev%d invalid dequeue_timeout_ns=%d" + " min_dequeue_timeout_ns=%d max_dequeue_timeout_ns=%d", + dev_id, dev_conf->dequeue_timeout_ns, + info.min_dequeue_timeout_ns, + info.max_dequeue_timeout_ns); + return -EINVAL; + } + } + + /* Check nb_events_limit is in limit */ + if (dev_conf->nb_events_limit > info.max_num_events) { + RTE_EDEV_LOG_ERR("dev%d nb_events_limit=%d > max_num_events=%d", + dev_id, dev_conf->nb_events_limit, info.max_num_events); + return -EINVAL; + } + + /* Check nb_event_queues is in limit */ + if (!dev_conf->nb_event_queues) { + RTE_EDEV_LOG_ERR("dev%d nb_event_queues cannot be zero", + dev_id); + return -EINVAL; + } + if (dev_conf->nb_event_queues > info.max_event_queues) { + RTE_EDEV_LOG_ERR("%d nb_event_queues=%d > max_event_queues=%d", + dev_id, dev_conf->nb_event_queues, info.max_event_queues); + return -EINVAL; + } + + /* Check nb_event_ports is in limit */ + if (!dev_conf->nb_event_ports) { + RTE_EDEV_LOG_ERR("dev%d nb_event_ports cannot be zero", dev_id); + return -EINVAL; + } + if (dev_conf->nb_event_ports > info.max_event_ports) { + RTE_EDEV_LOG_ERR("id%d nb_event_ports=%d > max_event_ports= %d", + dev_id, dev_conf->nb_event_ports, info.max_event_ports); + return -EINVAL; + } + + /* Check nb_event_queue_flows is in limit */ + if (!dev_conf->nb_event_queue_flows) { + RTE_EDEV_LOG_ERR("dev%d nb_flows cannot be zero", dev_id); + return -EINVAL; + } + if (dev_conf->nb_event_queue_flows > info.max_event_queue_flows) { + RTE_EDEV_LOG_ERR("dev%d nb_flows=%x > max_flows=%x", + dev_id, dev_conf->nb_event_queue_flows, + info.max_event_queue_flows); + return -EINVAL; + } + + /* Check nb_event_port_dequeue_depth is in limit */ + if (!dev_conf->nb_event_port_dequeue_depth) { + RTE_EDEV_LOG_ERR("dev%d nb_dequeue_depth cannot be zero", + dev_id); + return -EINVAL; + } + if ((info.event_dev_cap & RTE_EVENT_DEV_CAP_BURST_MODE) && + (dev_conf->nb_event_port_dequeue_depth > + info.max_event_port_dequeue_depth)) { + RTE_EDEV_LOG_ERR("dev%d nb_dq_depth=%d > max_dq_depth=%d", + dev_id, dev_conf->nb_event_port_dequeue_depth, + info.max_event_port_dequeue_depth); + return -EINVAL; + } + + /* Check nb_event_port_enqueue_depth is in limit */ + if (!dev_conf->nb_event_port_enqueue_depth) { + RTE_EDEV_LOG_ERR("dev%d nb_enqueue_depth cannot be zero", + dev_id); + return -EINVAL; + } + if ((info.event_dev_cap & RTE_EVENT_DEV_CAP_BURST_MODE) && + (dev_conf->nb_event_port_enqueue_depth > + info.max_event_port_enqueue_depth)) { + RTE_EDEV_LOG_ERR("dev%d nb_enq_depth=%d > max_enq_depth=%d", + dev_id, dev_conf->nb_event_port_enqueue_depth, + info.max_event_port_enqueue_depth); + return -EINVAL; + } + + /* Copy the dev_conf parameter into the dev structure */ + memcpy(&dev->data->dev_conf, dev_conf, sizeof(dev->data->dev_conf)); + + /* Setup new number of queues and reconfigure device. */ + diag = rte_event_dev_queue_config(dev, dev_conf->nb_event_queues); + if (diag != 0) { + RTE_EDEV_LOG_ERR("dev%d rte_event_dev_queue_config = %d", + dev_id, diag); + return diag; + } + + /* Setup new number of ports and reconfigure device. */ + diag = rte_event_dev_port_config(dev, dev_conf->nb_event_ports); + if (diag != 0) { + rte_event_dev_queue_config(dev, 0); + RTE_EDEV_LOG_ERR("dev%d rte_event_dev_port_config = %d", + dev_id, diag); + return diag; + } + + /* Configure the device */ + diag = (*dev->dev_ops->dev_configure)(dev); + if (diag != 0) { + RTE_EDEV_LOG_ERR("dev%d dev_configure = %d", dev_id, diag); + rte_event_dev_queue_config(dev, 0); + rte_event_dev_port_config(dev, 0); + } + + dev->data->event_dev_cap = info.event_dev_cap; + return diag; +} + +static inline int +is_valid_queue(struct rte_eventdev *dev, uint8_t queue_id) +{ + if (queue_id < dev->data->nb_queues && queue_id < + RTE_EVENT_MAX_QUEUES_PER_DEV) + return 1; + else + return 0; +} + +int +rte_event_queue_default_conf_get(uint8_t dev_id, uint8_t queue_id, + struct rte_event_queue_conf *queue_conf) +{ + struct rte_eventdev *dev; + + RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(dev_id, -EINVAL); + dev = &rte_eventdevs[dev_id]; + + if (queue_conf == NULL) + return -EINVAL; + + if (!is_valid_queue(dev, queue_id)) { + RTE_EDEV_LOG_ERR("Invalid queue_id=%" PRIu8, queue_id); + return -EINVAL; + } + + RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->queue_def_conf, -ENOTSUP); + memset(queue_conf, 0, sizeof(struct rte_event_queue_conf)); + (*dev->dev_ops->queue_def_conf)(dev, queue_id, queue_conf); + return 0; +} + +static inline int +is_valid_atomic_queue_conf(const struct rte_event_queue_conf *queue_conf) +{ + if (queue_conf && + !(queue_conf->event_queue_cfg & + RTE_EVENT_QUEUE_CFG_SINGLE_LINK) && + ((queue_conf->event_queue_cfg & + RTE_EVENT_QUEUE_CFG_ALL_TYPES) || + (queue_conf->schedule_type + == RTE_SCHED_TYPE_ATOMIC) + )) + return 1; + else + return 0; +} + +static inline int +is_valid_ordered_queue_conf(const struct rte_event_queue_conf *queue_conf) +{ + if (queue_conf && + !(queue_conf->event_queue_cfg & + RTE_EVENT_QUEUE_CFG_SINGLE_LINK) && + ((queue_conf->event_queue_cfg & + RTE_EVENT_QUEUE_CFG_ALL_TYPES) || + (queue_conf->schedule_type + == RTE_SCHED_TYPE_ORDERED) + )) + return 1; + else + return 0; +} + + +int +rte_event_queue_setup(uint8_t dev_id, uint8_t queue_id, + const struct rte_event_queue_conf *queue_conf) +{ + struct rte_eventdev *dev; + struct rte_event_queue_conf def_conf; + + RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(dev_id, -EINVAL); + dev = &rte_eventdevs[dev_id]; + + if (!is_valid_queue(dev, queue_id)) { + RTE_EDEV_LOG_ERR("Invalid queue_id=%" PRIu8, queue_id); + return -EINVAL; + } + + /* Check nb_atomic_flows limit */ + if (is_valid_atomic_queue_conf(queue_conf)) { + if (queue_conf->nb_atomic_flows == 0 || + queue_conf->nb_atomic_flows > + dev->data->dev_conf.nb_event_queue_flows) { + RTE_EDEV_LOG_ERR( + "dev%d queue%d Invalid nb_atomic_flows=%d max_flows=%d", + dev_id, queue_id, queue_conf->nb_atomic_flows, + dev->data->dev_conf.nb_event_queue_flows); + return -EINVAL; + } + } + + /* Check nb_atomic_order_sequences limit */ + if (is_valid_ordered_queue_conf(queue_conf)) { + if (queue_conf->nb_atomic_order_sequences == 0 || + queue_conf->nb_atomic_order_sequences > + dev->data->dev_conf.nb_event_queue_flows) { + RTE_EDEV_LOG_ERR( + "dev%d queue%d Invalid nb_atomic_order_seq=%d max_flows=%d", + dev_id, queue_id, queue_conf->nb_atomic_order_sequences, + dev->data->dev_conf.nb_event_queue_flows); + return -EINVAL; + } + } + + if (dev->data->dev_started) { + RTE_EDEV_LOG_ERR( + "device %d must be stopped to allow queue setup", dev_id); + return -EBUSY; + } + + RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->queue_setup, -ENOTSUP); + + if (queue_conf == NULL) { + RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->queue_def_conf, + -ENOTSUP); + (*dev->dev_ops->queue_def_conf)(dev, queue_id, &def_conf); + queue_conf = &def_conf; + } + + dev->data->queues_cfg[queue_id] = *queue_conf; + return (*dev->dev_ops->queue_setup)(dev, queue_id, queue_conf); +} + +static inline int +is_valid_port(struct rte_eventdev *dev, uint8_t port_id) +{ + if (port_id < dev->data->nb_ports) + return 1; + else + return 0; +} + +int +rte_event_port_default_conf_get(uint8_t dev_id, uint8_t port_id, + struct rte_event_port_conf *port_conf) +{ + struct rte_eventdev *dev; + + RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(dev_id, -EINVAL); + dev = &rte_eventdevs[dev_id]; + + if (port_conf == NULL) + return -EINVAL; + + if (!is_valid_port(dev, port_id)) { + RTE_EDEV_LOG_ERR("Invalid port_id=%" PRIu8, port_id); + return -EINVAL; + } + + RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->port_def_conf, -ENOTSUP); + memset(port_conf, 0, sizeof(struct rte_event_port_conf)); + (*dev->dev_ops->port_def_conf)(dev, port_id, port_conf); + return 0; +} + +int +rte_event_port_setup(uint8_t dev_id, uint8_t port_id, + const struct rte_event_port_conf *port_conf) +{ + struct rte_eventdev *dev; + struct rte_event_port_conf def_conf; + int diag; + + RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(dev_id, -EINVAL); + dev = &rte_eventdevs[dev_id]; + + if (!is_valid_port(dev, port_id)) { + RTE_EDEV_LOG_ERR("Invalid port_id=%" PRIu8, port_id); + return -EINVAL; + } + + /* Check new_event_threshold limit */ + if ((port_conf && !port_conf->new_event_threshold) || + (port_conf && port_conf->new_event_threshold > + dev->data->dev_conf.nb_events_limit)) { + RTE_EDEV_LOG_ERR( + "dev%d port%d Invalid event_threshold=%d nb_events_limit=%d", + dev_id, port_id, port_conf->new_event_threshold, + dev->data->dev_conf.nb_events_limit); + return -EINVAL; + } + + /* Check dequeue_depth limit */ + if ((port_conf && !port_conf->dequeue_depth) || + (port_conf && port_conf->dequeue_depth > + dev->data->dev_conf.nb_event_port_dequeue_depth)) { + RTE_EDEV_LOG_ERR( + "dev%d port%d Invalid dequeue depth=%d max_dequeue_depth=%d", + dev_id, port_id, port_conf->dequeue_depth, + dev->data->dev_conf.nb_event_port_dequeue_depth); + return -EINVAL; + } + + /* Check enqueue_depth limit */ + if ((port_conf && !port_conf->enqueue_depth) || + (port_conf && port_conf->enqueue_depth > + dev->data->dev_conf.nb_event_port_enqueue_depth)) { + RTE_EDEV_LOG_ERR( + "dev%d port%d Invalid enqueue depth=%d max_enqueue_depth=%d", + dev_id, port_id, port_conf->enqueue_depth, + dev->data->dev_conf.nb_event_port_enqueue_depth); + return -EINVAL; + } + + if (dev->data->dev_started) { + RTE_EDEV_LOG_ERR( + "device %d must be stopped to allow port setup", dev_id); + return -EBUSY; + } + + RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->port_setup, -ENOTSUP); + + if (port_conf == NULL) { + RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->port_def_conf, + -ENOTSUP); + (*dev->dev_ops->port_def_conf)(dev, port_id, &def_conf); + port_conf = &def_conf; + } + + dev->data->ports_cfg[port_id] = *port_conf; + + diag = (*dev->dev_ops->port_setup)(dev, port_id, port_conf); + + /* Unlink all the queues from this port(default state after setup) */ + if (!diag) + diag = rte_event_port_unlink(dev_id, port_id, NULL, 0); + + if (diag < 0) + return diag; + + return 0; +} + +int +rte_event_dev_attr_get(uint8_t dev_id, uint32_t attr_id, + uint32_t *attr_value) +{ + struct rte_eventdev *dev; + + if (!attr_value) + return -EINVAL; + RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(dev_id, -EINVAL); + dev = &rte_eventdevs[dev_id]; + + switch (attr_id) { + case RTE_EVENT_DEV_ATTR_PORT_COUNT: + *attr_value = dev->data->nb_ports; + break; + case RTE_EVENT_DEV_ATTR_QUEUE_COUNT: + *attr_value = dev->data->nb_queues; + break; + case RTE_EVENT_DEV_ATTR_STARTED: + *attr_value = dev->data->dev_started; + break; + default: + return -EINVAL; + } + + return 0; +} + +int +rte_event_port_attr_get(uint8_t dev_id, uint8_t port_id, uint32_t attr_id, + uint32_t *attr_value) +{ + struct rte_eventdev *dev; + + if (!attr_value) + return -EINVAL; + + RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(dev_id, -EINVAL); + dev = &rte_eventdevs[dev_id]; + if (!is_valid_port(dev, port_id)) { + RTE_EDEV_LOG_ERR("Invalid port_id=%" PRIu8, port_id); + return -EINVAL; + } + + switch (attr_id) { + case RTE_EVENT_PORT_ATTR_ENQ_DEPTH: + *attr_value = dev->data->ports_cfg[port_id].enqueue_depth; + break; + case RTE_EVENT_PORT_ATTR_DEQ_DEPTH: + *attr_value = dev->data->ports_cfg[port_id].dequeue_depth; + break; + case RTE_EVENT_PORT_ATTR_NEW_EVENT_THRESHOLD: + *attr_value = dev->data->ports_cfg[port_id].new_event_threshold; + break; + default: + return -EINVAL; + }; + return 0; +} + +int +rte_event_queue_attr_get(uint8_t dev_id, uint8_t queue_id, uint32_t attr_id, + uint32_t *attr_value) +{ + struct rte_event_queue_conf *conf; + struct rte_eventdev *dev; + + if (!attr_value) + return -EINVAL; + + RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(dev_id, -EINVAL); + dev = &rte_eventdevs[dev_id]; + if (!is_valid_queue(dev, queue_id)) { + RTE_EDEV_LOG_ERR("Invalid queue_id=%" PRIu8, queue_id); + return -EINVAL; + } + + conf = &dev->data->queues_cfg[queue_id]; + + switch (attr_id) { + case RTE_EVENT_QUEUE_ATTR_PRIORITY: + *attr_value = RTE_EVENT_DEV_PRIORITY_NORMAL; + if (dev->data->event_dev_cap & RTE_EVENT_DEV_CAP_QUEUE_QOS) + *attr_value = conf->priority; + break; + case RTE_EVENT_QUEUE_ATTR_NB_ATOMIC_FLOWS: + *attr_value = conf->nb_atomic_flows; + break; + case RTE_EVENT_QUEUE_ATTR_NB_ATOMIC_ORDER_SEQUENCES: + *attr_value = conf->nb_atomic_order_sequences; + break; + case RTE_EVENT_QUEUE_ATTR_EVENT_QUEUE_CFG: + *attr_value = conf->event_queue_cfg; + break; + case RTE_EVENT_QUEUE_ATTR_SCHEDULE_TYPE: + if (conf->event_queue_cfg & RTE_EVENT_QUEUE_CFG_ALL_TYPES) + return -EOVERFLOW; + + *attr_value = conf->schedule_type; + break; + default: + return -EINVAL; + }; + return 0; +} + +int +rte_event_port_link(uint8_t dev_id, uint8_t port_id, + const uint8_t queues[], const uint8_t priorities[], + uint16_t nb_links) +{ + struct rte_eventdev *dev; + uint8_t queues_list[RTE_EVENT_MAX_QUEUES_PER_DEV]; + uint8_t priorities_list[RTE_EVENT_MAX_QUEUES_PER_DEV]; + uint16_t *links_map; + int i, diag; + + RTE_EVENTDEV_VALID_DEVID_OR_ERRNO_RET(dev_id, -EINVAL, 0); + dev = &rte_eventdevs[dev_id]; + + if (*dev->dev_ops->port_link == NULL) { + RTE_PMD_DEBUG_TRACE("Function not supported\n"); + rte_errno = -ENOTSUP; + return 0; + } + + if (!is_valid_port(dev, port_id)) { + RTE_EDEV_LOG_ERR("Invalid port_id=%" PRIu8, port_id); + rte_errno = -EINVAL; + return 0; + } + + if (queues == NULL) { + for (i = 0; i < dev->data->nb_queues; i++) + queues_list[i] = i; + + queues = queues_list; + nb_links = dev->data->nb_queues; + } + + if (priorities == NULL) { + for (i = 0; i < nb_links; i++) + priorities_list[i] = RTE_EVENT_DEV_PRIORITY_NORMAL; + + priorities = priorities_list; + } + + for (i = 0; i < nb_links; i++) + if (queues[i] >= dev->data->nb_queues) { + rte_errno = -EINVAL; + return 0; + } + + diag = (*dev->dev_ops->port_link)(dev, dev->data->ports[port_id], + queues, priorities, nb_links); + if (diag < 0) + return diag; + + links_map = dev->data->links_map; + /* Point links_map to this port specific area */ + links_map += (port_id * RTE_EVENT_MAX_QUEUES_PER_DEV); + for (i = 0; i < diag; i++) + links_map[queues[i]] = (uint8_t)priorities[i]; + + return diag; +} + +int +rte_event_port_unlink(uint8_t dev_id, uint8_t port_id, + uint8_t queues[], uint16_t nb_unlinks) +{ + struct rte_eventdev *dev; + uint8_t all_queues[RTE_EVENT_MAX_QUEUES_PER_DEV]; + int i, diag; + uint16_t *links_map; + + RTE_EVENTDEV_VALID_DEVID_OR_ERRNO_RET(dev_id, -EINVAL, 0); + dev = &rte_eventdevs[dev_id]; + + if (*dev->dev_ops->port_unlink == NULL) { + RTE_PMD_DEBUG_TRACE("Function not supported\n"); + rte_errno = -ENOTSUP; + return 0; + } + + if (!is_valid_port(dev, port_id)) { + RTE_EDEV_LOG_ERR("Invalid port_id=%" PRIu8, port_id); + rte_errno = -EINVAL; + return 0; + } + + if (queues == NULL) { + for (i = 0; i < dev->data->nb_queues; i++) + all_queues[i] = i; + queues = all_queues; + nb_unlinks = dev->data->nb_queues; + } + + for (i = 0; i < nb_unlinks; i++) + if (queues[i] >= dev->data->nb_queues) { + rte_errno = -EINVAL; + return 0; + } + + diag = (*dev->dev_ops->port_unlink)(dev, dev->data->ports[port_id], + queues, nb_unlinks); + + if (diag < 0) + return diag; + + links_map = dev->data->links_map; + /* Point links_map to this port specific area */ + links_map += (port_id * RTE_EVENT_MAX_QUEUES_PER_DEV); + for (i = 0; i < diag; i++) + links_map[queues[i]] = EVENT_QUEUE_SERVICE_PRIORITY_INVALID; + + return diag; +} + +int +rte_event_port_links_get(uint8_t dev_id, uint8_t port_id, + uint8_t queues[], uint8_t priorities[]) +{ + struct rte_eventdev *dev; + uint16_t *links_map; + int i, count = 0; + + RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(dev_id, -EINVAL); + dev = &rte_eventdevs[dev_id]; + if (!is_valid_port(dev, port_id)) { + RTE_EDEV_LOG_ERR("Invalid port_id=%" PRIu8, port_id); + return -EINVAL; + } + + links_map = dev->data->links_map; + /* Point links_map to this port specific area */ + links_map += (port_id * RTE_EVENT_MAX_QUEUES_PER_DEV); + for (i = 0; i < dev->data->nb_queues; i++) { + if (links_map[i] != EVENT_QUEUE_SERVICE_PRIORITY_INVALID) { + queues[count] = i; + priorities[count] = (uint8_t)links_map[i]; + ++count; + } + } + return count; +} + +int +rte_event_dequeue_timeout_ticks(uint8_t dev_id, uint64_t ns, + uint64_t *timeout_ticks) +{ + struct rte_eventdev *dev; + + RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(dev_id, -EINVAL); + dev = &rte_eventdevs[dev_id]; + RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timeout_ticks, -ENOTSUP); + + if (timeout_ticks == NULL) + return -EINVAL; + + return (*dev->dev_ops->timeout_ticks)(dev, ns, timeout_ticks); +} + +int +rte_event_dev_service_id_get(uint8_t dev_id, uint32_t *service_id) +{ + struct rte_eventdev *dev; + + RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(dev_id, -EINVAL); + dev = &rte_eventdevs[dev_id]; + + if (service_id == NULL) + return -EINVAL; + + if (dev->data->service_inited) + *service_id = dev->data->service_id; + + return dev->data->service_inited ? 0 : -ESRCH; +} + +int +rte_event_dev_dump(uint8_t dev_id, FILE *f) +{ + struct rte_eventdev *dev; + + RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(dev_id, -EINVAL); + dev = &rte_eventdevs[dev_id]; + RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dump, -ENOTSUP); + + (*dev->dev_ops->dump)(dev, f); + return 0; + +} + +static int +xstats_get_count(uint8_t dev_id, enum rte_event_dev_xstats_mode mode, + uint8_t queue_port_id) +{ + struct rte_eventdev *dev = &rte_eventdevs[dev_id]; + if (dev->dev_ops->xstats_get_names != NULL) + return (*dev->dev_ops->xstats_get_names)(dev, mode, + queue_port_id, + NULL, NULL, 0); + return 0; +} + +int +rte_event_dev_xstats_names_get(uint8_t dev_id, + enum rte_event_dev_xstats_mode mode, uint8_t queue_port_id, + struct rte_event_dev_xstats_name *xstats_names, + unsigned int *ids, unsigned int size) +{ + RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(dev_id, -ENODEV); + const int cnt_expected_entries = xstats_get_count(dev_id, mode, + queue_port_id); + if (xstats_names == NULL || cnt_expected_entries < 0 || + (int)size < cnt_expected_entries) + return cnt_expected_entries; + + /* dev_id checked above */ + const struct rte_eventdev *dev = &rte_eventdevs[dev_id]; + + if (dev->dev_ops->xstats_get_names != NULL) + return (*dev->dev_ops->xstats_get_names)(dev, mode, + queue_port_id, xstats_names, ids, size); + + return -ENOTSUP; +} + +/* retrieve eventdev extended statistics */ +int +rte_event_dev_xstats_get(uint8_t dev_id, enum rte_event_dev_xstats_mode mode, + uint8_t queue_port_id, const unsigned int ids[], + uint64_t values[], unsigned int n) +{ + RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(dev_id, -ENODEV); + const struct rte_eventdev *dev = &rte_eventdevs[dev_id]; + + /* implemented by the driver */ + if (dev->dev_ops->xstats_get != NULL) + return (*dev->dev_ops->xstats_get)(dev, mode, queue_port_id, + ids, values, n); + return -ENOTSUP; +} + +uint64_t +rte_event_dev_xstats_by_name_get(uint8_t dev_id, const char *name, + unsigned int *id) +{ + RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(dev_id, 0); + const struct rte_eventdev *dev = &rte_eventdevs[dev_id]; + unsigned int temp = -1; + + if (id != NULL) + *id = (unsigned int)-1; + else + id = &temp; /* ensure driver never gets a NULL value */ + + /* implemented by driver */ + if (dev->dev_ops->xstats_get_by_name != NULL) + return (*dev->dev_ops->xstats_get_by_name)(dev, name, id); + return -ENOTSUP; +} + +int rte_event_dev_xstats_reset(uint8_t dev_id, + enum rte_event_dev_xstats_mode mode, int16_t queue_port_id, + const uint32_t ids[], uint32_t nb_ids) +{ + RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(dev_id, -EINVAL); + struct rte_eventdev *dev = &rte_eventdevs[dev_id]; + + if (dev->dev_ops->xstats_reset != NULL) + return (*dev->dev_ops->xstats_reset)(dev, mode, queue_port_id, + ids, nb_ids); + return -ENOTSUP; +} + +int +rte_event_dev_start(uint8_t dev_id) +{ + struct rte_eventdev *dev; + int diag; + + RTE_EDEV_LOG_DEBUG("Start dev_id=%" PRIu8, dev_id); + + RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(dev_id, -EINVAL); + dev = &rte_eventdevs[dev_id]; + RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_start, -ENOTSUP); + + if (dev->data->dev_started != 0) { + RTE_EDEV_LOG_ERR("Device with dev_id=%" PRIu8 "already started", + dev_id); + return 0; + } + + diag = (*dev->dev_ops->dev_start)(dev); + if (diag == 0) + dev->data->dev_started = 1; + else + return diag; + + return 0; +} + +void +rte_event_dev_stop(uint8_t dev_id) +{ + struct rte_eventdev *dev; + + RTE_EDEV_LOG_DEBUG("Stop dev_id=%" PRIu8, dev_id); + + RTE_EVENTDEV_VALID_DEVID_OR_RET(dev_id); + dev = &rte_eventdevs[dev_id]; + RTE_FUNC_PTR_OR_RET(*dev->dev_ops->dev_stop); + + if (dev->data->dev_started == 0) { + RTE_EDEV_LOG_ERR("Device with dev_id=%" PRIu8 "already stopped", + dev_id); + return; + } + + dev->data->dev_started = 0; + (*dev->dev_ops->dev_stop)(dev); +} + +int +rte_event_dev_close(uint8_t dev_id) +{ + struct rte_eventdev *dev; + + RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(dev_id, -EINVAL); + dev = &rte_eventdevs[dev_id]; + RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_close, -ENOTSUP); + + /* Device must be stopped before it can be closed */ + if (dev->data->dev_started == 1) { + RTE_EDEV_LOG_ERR("Device %u must be stopped before closing", + dev_id); + return -EBUSY; + } + + return (*dev->dev_ops->dev_close)(dev); +} + +static inline int +rte_eventdev_data_alloc(uint8_t dev_id, struct rte_eventdev_data **data, + int socket_id) +{ + char mz_name[RTE_EVENTDEV_NAME_MAX_LEN]; + const struct rte_memzone *mz; + int n; + + /* Generate memzone name */ + n = snprintf(mz_name, sizeof(mz_name), "rte_eventdev_data_%u", dev_id); + if (n >= (int)sizeof(mz_name)) + return -EINVAL; + + if (rte_eal_process_type() == RTE_PROC_PRIMARY) { + mz = rte_memzone_reserve(mz_name, + sizeof(struct rte_eventdev_data), + socket_id, 0); + } else + mz = rte_memzone_lookup(mz_name); + + if (mz == NULL) + return -ENOMEM; + + *data = mz->addr; + if (rte_eal_process_type() == RTE_PROC_PRIMARY) + memset(*data, 0, sizeof(struct rte_eventdev_data)); + + return 0; +} + +static inline uint8_t +rte_eventdev_find_free_device_index(void) +{ + uint8_t dev_id; + + for (dev_id = 0; dev_id < RTE_EVENT_MAX_DEVS; dev_id++) { + if (rte_eventdevs[dev_id].attached == + RTE_EVENTDEV_DETACHED) + return dev_id; + } + return RTE_EVENT_MAX_DEVS; +} + +struct rte_eventdev * +rte_event_pmd_allocate(const char *name, int socket_id) +{ + struct rte_eventdev *eventdev; + uint8_t dev_id; + + if (rte_event_pmd_get_named_dev(name) != NULL) { + RTE_EDEV_LOG_ERR("Event device with name %s already " + "allocated!", name); + return NULL; + } + + dev_id = rte_eventdev_find_free_device_index(); + if (dev_id == RTE_EVENT_MAX_DEVS) { + RTE_EDEV_LOG_ERR("Reached maximum number of event devices"); + return NULL; + } + + eventdev = &rte_eventdevs[dev_id]; + + if (eventdev->data == NULL) { + struct rte_eventdev_data *eventdev_data = NULL; + + int retval = rte_eventdev_data_alloc(dev_id, &eventdev_data, + socket_id); + + if (retval < 0 || eventdev_data == NULL) + return NULL; + + eventdev->data = eventdev_data; + + snprintf(eventdev->data->name, RTE_EVENTDEV_NAME_MAX_LEN, + "%s", name); + + eventdev->data->dev_id = dev_id; + eventdev->data->socket_id = socket_id; + eventdev->data->dev_started = 0; + + eventdev->attached = RTE_EVENTDEV_ATTACHED; + + eventdev_globals.nb_devs++; + } + + return eventdev; +} + +int +rte_event_pmd_release(struct rte_eventdev *eventdev) +{ + int ret; + char mz_name[RTE_EVENTDEV_NAME_MAX_LEN]; + const struct rte_memzone *mz; + + if (eventdev == NULL) + return -EINVAL; + + eventdev->attached = RTE_EVENTDEV_DETACHED; + eventdev_globals.nb_devs--; + + if (rte_eal_process_type() == RTE_PROC_PRIMARY) { + rte_free(eventdev->data->dev_private); + + /* Generate memzone name */ + ret = snprintf(mz_name, sizeof(mz_name), "rte_eventdev_data_%u", + eventdev->data->dev_id); + if (ret >= (int)sizeof(mz_name)) + return -EINVAL; + + mz = rte_memzone_lookup(mz_name); + if (mz == NULL) + return -ENOMEM; + + ret = rte_memzone_free(mz); + if (ret) + return ret; + } + + eventdev->data = NULL; + return 0; +} diff --git a/dpdk/lib/librte_eventdev/rte_eventdev.h b/dpdk/lib/librte_eventdev/rte_eventdev.h new file mode 100644 index 00000000..eb1024d9 --- /dev/null +++ b/dpdk/lib/librte_eventdev/rte_eventdev.h @@ -0,0 +1,1778 @@ +/* + * BSD LICENSE + * + * Copyright 2016 Cavium, Inc. + * Copyright 2016 Intel Corporation. + * Copyright 2016 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_EVENTDEV_H_ +#define _RTE_EVENTDEV_H_ + +/** + * @file + * + * RTE Event Device API + * + * In a polling model, lcores poll ethdev ports and associated rx queues + * directly to look for packet. In an event driven model, by contrast, lcores + * call the scheduler that selects packets for them based on programmer + * specified criteria. Eventdev library adds support for event driven + * programming model, which offer applications automatic multicore scaling, + * dynamic load balancing, pipelining, packet ingress order maintenance and + * synchronization services to simplify application packet processing. + * + * The Event Device API is composed of two parts: + * + * - The application-oriented Event API that includes functions to setup + * an event device (configure it, setup its queues, ports and start it), to + * establish the link between queues to port and to receive events, and so on. + * + * - The driver-oriented Event API that exports a function allowing + * an event poll Mode Driver (PMD) to simultaneously register itself as + * an event device driver. + * + * Event device components: + * + * +-----------------+ + * | +-------------+ | + * +-------+ | | flow 0 | | + * |Packet | | +-------------+ | + * |event | | +-------------+ | + * | | | | flow 1 | |port_link(port0, queue0) + * +-------+ | +-------------+ | | +--------+ + * +-------+ | +-------------+ o-----v-----o |dequeue +------+ + * |Crypto | | | flow n | | | event +------->|Core 0| + * |work | | +-------------+ o----+ | port 0 | | | + * |done ev| | event queue 0 | | +--------+ +------+ + * +-------+ +-----------------+ | + * +-------+ | + * |Timer | +-----------------+ | +--------+ + * |expiry | | +-------------+ | +------o |dequeue +------+ + * |event | | | flow 0 | o-----------o event +------->|Core 1| + * +-------+ | +-------------+ | +----o port 1 | | | + * Event enqueue | +-------------+ | | +--------+ +------+ + * o-------------> | | flow 1 | | | + * enqueue( | +-------------+ | | + * queue_id, | | | +--------+ +------+ + * flow_id, | +-------------+ | | | |dequeue |Core 2| + * sched_type, | | flow n | o-----------o event +------->| | + * event_type, | +-------------+ | | | port 2 | +------+ + * subev_type, | event queue 1 | | +--------+ + * event) +-----------------+ | +--------+ + * | | |dequeue +------+ + * +-------+ +-----------------+ | | event +------->|Core n| + * |Core | | +-------------+ o-----------o port n | | | + * |(SW) | | | flow 0 | | | +--------+ +--+---+ + * |event | | +-------------+ | | | + * +-------+ | +-------------+ | | | + * ^ | | flow 1 | | | | + * | | +-------------+ o------+ | + * | | +-------------+ | | + * | | | flow n | | | + * | | +-------------+ | | + * | | event queue n | | + * | +-----------------+ | + * | | + * +-----------------------------------------------------------+ + * + * Event device: A hardware or software-based event scheduler. + * + * Event: A unit of scheduling that encapsulates a packet or other datatype + * like SW generated event from the CPU, Crypto work completion notification, + * Timer expiry event notification etc as well as metadata. + * The metadata includes flow ID, scheduling type, event priority, event_type, + * sub_event_type etc. + * + * Event queue: A queue containing events that are scheduled by the event dev. + * An event queue contains events of different flows associated with scheduling + * types, such as atomic, ordered, or parallel. + * + * Event port: An application's interface into the event dev for enqueue and + * dequeue operations. Each event port can be linked with one or more + * event queues for dequeue operations. + * + * By default, all the functions of the Event Device API exported by a PMD + * are lock-free functions which assume to not be invoked in parallel on + * different logical cores to work on the same target object. For instance, + * the dequeue function of a PMD cannot be invoked in parallel on two logical + * cores to operates on same event port. Of course, this function + * can be invoked in parallel by different logical cores on different ports. + * It is the responsibility of the upper level application to enforce this rule. + * + * In all functions of the Event API, the Event device is + * designated by an integer >= 0 named the device identifier *dev_id* + * + * At the Event driver level, Event devices are represented by a generic + * data structure of type *rte_event_dev*. + * + * Event devices are dynamically registered during the PCI/SoC device probing + * phase performed at EAL initialization time. + * When an Event device is being probed, a *rte_event_dev* structure and + * a new device identifier are allocated for that device. Then, the + * event_dev_init() function supplied by the Event driver matching the probed + * device is invoked to properly initialize the device. + * + * The role of the device init function consists of resetting the hardware or + * software event driver implementations. + * + * If the device init operation is successful, the correspondence between + * the device identifier assigned to the new device and its associated + * *rte_event_dev* structure is effectively registered. + * Otherwise, both the *rte_event_dev* structure and the device identifier are + * freed. + * + * The functions exported by the application Event API to setup a device + * designated by its device identifier must be invoked in the following order: + * - rte_event_dev_configure() + * - rte_event_queue_setup() + * - rte_event_port_setup() + * - rte_event_port_link() + * - rte_event_dev_start() + * + * Then, the application can invoke, in any order, the functions + * exported by the Event API to schedule events, dequeue events, enqueue events, + * change event queue(s) to event port [un]link establishment and so on. + * + * Application may use rte_event_[queue/port]_default_conf_get() to get the + * default configuration to set up an event queue or event port by + * overriding few default values. + * + * If the application wants to change the configuration (i.e. call + * rte_event_dev_configure(), rte_event_queue_setup(), or + * rte_event_port_setup()), it must call rte_event_dev_stop() first to stop the + * device and then do the reconfiguration before calling rte_event_dev_start() + * again. The schedule, enqueue and dequeue functions should not be invoked + * when the device is stopped. + * + * Finally, an application can close an Event device by invoking the + * rte_event_dev_close() function. + * + * Each function of the application Event API invokes a specific function + * of the PMD that controls the target device designated by its device + * identifier. + * + * For this purpose, all device-specific functions of an Event driver are + * supplied through a set of pointers contained in a generic structure of type + * *event_dev_ops*. + * The address of the *event_dev_ops* structure is stored in the *rte_event_dev* + * structure by the device init function of the Event driver, which is + * invoked during the PCI/SoC device probing phase, as explained earlier. + * + * In other words, each function of the Event API simply retrieves the + * *rte_event_dev* structure associated with the device identifier and + * performs an indirect invocation of the corresponding driver function + * supplied in the *event_dev_ops* structure of the *rte_event_dev* structure. + * + * For performance reasons, the address of the fast-path functions of the + * Event driver is not contained in the *event_dev_ops* structure. + * Instead, they are directly stored at the beginning of the *rte_event_dev* + * structure to avoid an extra indirect memory access during their invocation. + * + * RTE event device drivers do not use interrupts for enqueue or dequeue + * operation. Instead, Event drivers export Poll-Mode enqueue and dequeue + * functions to applications. + * + * The events are injected to event device through *enqueue* operation by + * event producers in the system. The typical event producers are ethdev + * subsystem for generating packet events, CPU(SW) for generating events based + * on different stages of application processing, cryptodev for generating + * crypto work completion notification etc + * + * The *dequeue* operation gets one or more events from the event ports. + * The application process the events and send to downstream event queue through + * rte_event_enqueue_burst() if it is an intermediate stage of event processing, + * on the final stage, the application may send to different subsystem like + * ethdev to send the packet/event on the wire using ethdev + * rte_eth_tx_burst() API. + * + * The point at which events are scheduled to ports depends on the device. + * For hardware devices, scheduling occurs asynchronously without any software + * intervention. Software schedulers can either be distributed + * (each worker thread schedules events to its own port) or centralized + * (a dedicated thread schedules to all ports). Distributed software schedulers + * perform the scheduling in rte_event_dequeue_burst(), whereas centralized + * scheduler logic need a dedicated service core for scheduling. + * The RTE_EVENT_DEV_CAP_DISTRIBUTED_SCHED capability flag is not set + * indicates the device is centralized and thus needs a dedicated scheduling + * thread that repeatedly calls software specific scheduling function. + * + * An event driven worker thread has following typical workflow on fastpath: + * \code{.c} + * while (1) { + * rte_event_dequeue_burst(...); + * (event processing) + * rte_event_enqueue_burst(...); + * } + * \endcode + * + */ + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include + +struct rte_mbuf; /* we just use mbuf pointers; no need to include rte_mbuf.h */ + +/* Event device capability bitmap flags */ +#define RTE_EVENT_DEV_CAP_QUEUE_QOS (1ULL << 0) +/**< Event scheduling prioritization is based on the priority associated with + * each event queue. + * + * @see rte_event_queue_setup() + */ +#define RTE_EVENT_DEV_CAP_EVENT_QOS (1ULL << 1) +/**< Event scheduling prioritization is based on the priority associated with + * each event. Priority of each event is supplied in *rte_event* structure + * on each enqueue operation. + * + * @see rte_event_enqueue_burst() + */ +#define RTE_EVENT_DEV_CAP_DISTRIBUTED_SCHED (1ULL << 2) +/**< Event device operates in distributed scheduling mode. + * In distributed scheduling mode, event scheduling happens in HW or + * rte_event_dequeue_burst() or the combination of these two. + * If the flag is not set then eventdev is centralized and thus needs a + * dedicated service core that acts as a scheduling thread . + * + * @see rte_event_dequeue_burst() + */ +#define RTE_EVENT_DEV_CAP_QUEUE_ALL_TYPES (1ULL << 3) +/**< Event device is capable of enqueuing events of any type to any queue. + * If this capability is not set, the queue only supports events of the + * *RTE_SCHED_TYPE_* type that it was created with. + * + * @see RTE_SCHED_TYPE_* values + */ +#define RTE_EVENT_DEV_CAP_BURST_MODE (1ULL << 4) +/**< Event device is capable of operating in burst mode for enqueue(forward, + * release) and dequeue operation. If this capability is not set, application + * still uses the rte_event_dequeue_burst() and rte_event_enqueue_burst() but + * PMD accepts only one event at a time. + * + * @see rte_event_dequeue_burst() rte_event_enqueue_burst() + */ + +/* Event device priority levels */ +#define RTE_EVENT_DEV_PRIORITY_HIGHEST 0 +/**< Highest priority expressed across eventdev subsystem + * @see rte_event_queue_setup(), rte_event_enqueue_burst() + * @see rte_event_port_link() + */ +#define RTE_EVENT_DEV_PRIORITY_NORMAL 128 +/**< Normal priority expressed across eventdev subsystem + * @see rte_event_queue_setup(), rte_event_enqueue_burst() + * @see rte_event_port_link() + */ +#define RTE_EVENT_DEV_PRIORITY_LOWEST 255 +/**< Lowest priority expressed across eventdev subsystem + * @see rte_event_queue_setup(), rte_event_enqueue_burst() + * @see rte_event_port_link() + */ + +/** + * Get the total number of event devices that have been successfully + * initialised. + * + * @return + * The total number of usable event devices. + */ +uint8_t +rte_event_dev_count(void); + +/** + * Get the device identifier for the named event device. + * + * @param name + * Event device name to select the event device identifier. + * + * @return + * Returns event device identifier on success. + * - <0: Failure to find named event device. + */ +int +rte_event_dev_get_dev_id(const char *name); + +/** + * Return the NUMA socket to which a device is connected. + * + * @param dev_id + * The identifier of the device. + * @return + * The NUMA socket id to which the device is connected or + * a default of zero if the socket could not be determined. + * -(-EINVAL) dev_id value is out of range. + */ +int +rte_event_dev_socket_id(uint8_t dev_id); + +/** + * Event device information + */ +struct rte_event_dev_info { + const char *driver_name; /**< Event driver name */ + struct rte_device *dev; /**< Device information */ + uint32_t min_dequeue_timeout_ns; + /**< Minimum supported global dequeue timeout(ns) by this device */ + uint32_t max_dequeue_timeout_ns; + /**< Maximum supported global dequeue timeout(ns) by this device */ + uint32_t dequeue_timeout_ns; + /**< Configured global dequeue timeout(ns) for this device */ + uint8_t max_event_queues; + /**< Maximum event_queues supported by this device */ + uint32_t max_event_queue_flows; + /**< Maximum supported flows in an event queue by this device*/ + uint8_t max_event_queue_priority_levels; + /**< Maximum number of event queue priority levels by this device. + * Valid when the device has RTE_EVENT_DEV_CAP_QUEUE_QOS capability + */ + uint8_t max_event_priority_levels; + /**< Maximum number of event priority levels by this device. + * Valid when the device has RTE_EVENT_DEV_CAP_EVENT_QOS capability + */ + uint8_t max_event_ports; + /**< Maximum number of event ports supported by this device */ + uint8_t max_event_port_dequeue_depth; + /**< Maximum number of events can be dequeued at a time from an + * event port by this device. + * A device that does not support bulk dequeue will set this as 1. + */ + uint32_t max_event_port_enqueue_depth; + /**< Maximum number of events can be enqueued at a time from an + * event port by this device. + * A device that does not support bulk enqueue will set this as 1. + */ + int32_t max_num_events; + /**< A *closed system* event dev has a limit on the number of events it + * can manage at a time. An *open system* event dev does not have a + * limit and will specify this as -1. + */ + uint32_t event_dev_cap; + /**< Event device capabilities(RTE_EVENT_DEV_CAP_)*/ +}; + +/** + * Retrieve the contextual information of an event device. + * + * @param dev_id + * The identifier of the device. + * + * @param[out] dev_info + * A pointer to a structure of type *rte_event_dev_info* to be filled with the + * contextual information of the device. + * + * @return + * - 0: Success, driver updates the contextual information of the event device + * - <0: Error code returned by the driver info get function. + * + */ +int +rte_event_dev_info_get(uint8_t dev_id, struct rte_event_dev_info *dev_info); + +/** + * The count of ports. + */ +#define RTE_EVENT_DEV_ATTR_PORT_COUNT 0 +/** + * The count of queues. + */ +#define RTE_EVENT_DEV_ATTR_QUEUE_COUNT 1 +/** + * The status of the device, zero for stopped, non-zero for started. + */ +#define RTE_EVENT_DEV_ATTR_STARTED 2 + +/** + * Get an attribute from a device. + * + * @param dev_id Eventdev id + * @param attr_id The attribute ID to retrieve + * @param[out] attr_value A pointer that will be filled in with the attribute + * value if successful. + * + * @return + * - 0: Successfully retrieved attribute value + * - -EINVAL: Invalid device or *attr_id* provided, or *attr_value* is NULL + */ +int +rte_event_dev_attr_get(uint8_t dev_id, uint32_t attr_id, + uint32_t *attr_value); + + +/* Event device configuration bitmap flags */ +#define RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT (1ULL << 0) +/**< Override the global *dequeue_timeout_ns* and use per dequeue timeout in ns. + * @see rte_event_dequeue_timeout_ticks(), rte_event_dequeue_burst() + */ + +/** Event device configuration structure */ +struct rte_event_dev_config { + uint32_t dequeue_timeout_ns; + /**< rte_event_dequeue_burst() timeout on this device. + * This value should be in the range of *min_dequeue_timeout_ns* and + * *max_dequeue_timeout_ns* which previously provided in + * rte_event_dev_info_get() + * The value 0 is allowed, in which case, default dequeue timeout used. + * @see RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT + */ + int32_t nb_events_limit; + /**< In a *closed system* this field is the limit on maximum number of + * events that can be inflight in the eventdev at a given time. The + * limit is required to ensure that the finite space in a closed system + * is not overwhelmed. The value cannot exceed the *max_num_events* + * as provided by rte_event_dev_info_get(). + * This value should be set to -1 for *open system*. + */ + uint8_t nb_event_queues; + /**< Number of event queues to configure on this device. + * This value cannot exceed the *max_event_queues* which previously + * provided in rte_event_dev_info_get() + */ + uint8_t nb_event_ports; + /**< Number of event ports to configure on this device. + * This value cannot exceed the *max_event_ports* which previously + * provided in rte_event_dev_info_get() + */ + uint32_t nb_event_queue_flows; + /**< Number of flows for any event queue on this device. + * This value cannot exceed the *max_event_queue_flows* which previously + * provided in rte_event_dev_info_get() + */ + uint32_t nb_event_port_dequeue_depth; + /**< Maximum number of events can be dequeued at a time from an + * event port by this device. + * This value cannot exceed the *max_event_port_dequeue_depth* + * which previously provided in rte_event_dev_info_get(). + * Ignored when device is not RTE_EVENT_DEV_CAP_BURST_MODE capable. + * @see rte_event_port_setup() + */ + uint32_t nb_event_port_enqueue_depth; + /**< Maximum number of events can be enqueued at a time from an + * event port by this device. + * This value cannot exceed the *max_event_port_enqueue_depth* + * which previously provided in rte_event_dev_info_get(). + * Ignored when device is not RTE_EVENT_DEV_CAP_BURST_MODE capable. + * @see rte_event_port_setup() + */ + uint32_t event_dev_cfg; + /**< Event device config flags(RTE_EVENT_DEV_CFG_)*/ +}; + +/** + * Configure an event device. + * + * This function must be invoked first before any other function in the + * API. This function can also be re-invoked when a device is in the + * stopped state. + * + * The caller may use rte_event_dev_info_get() to get the capability of each + * resources available for this event device. + * + * @param dev_id + * The identifier of the device to configure. + * @param dev_conf + * The event device configuration structure. + * + * @return + * - 0: Success, device configured. + * - <0: Error code returned by the driver configuration function. + */ +int +rte_event_dev_configure(uint8_t dev_id, + const struct rte_event_dev_config *dev_conf); + + +/* Event queue specific APIs */ + +/* Event queue configuration bitmap flags */ +#define RTE_EVENT_QUEUE_CFG_ALL_TYPES (1ULL << 0) +/**< Allow ATOMIC,ORDERED,PARALLEL schedule type enqueue + * + * @see RTE_SCHED_TYPE_ORDERED, RTE_SCHED_TYPE_ATOMIC, RTE_SCHED_TYPE_PARALLEL + * @see rte_event_enqueue_burst() + */ +#define RTE_EVENT_QUEUE_CFG_SINGLE_LINK (1ULL << 1) +/**< This event queue links only to a single event port. + * + * @see rte_event_port_setup(), rte_event_port_link() + */ + +/** Event queue configuration structure */ +struct rte_event_queue_conf { + uint32_t nb_atomic_flows; + /**< The maximum number of active flows this queue can track at any + * given time. If the queue is configured for atomic scheduling (by + * applying the RTE_EVENT_QUEUE_CFG_ALL_TYPES flag to event_queue_cfg + * or RTE_SCHED_TYPE_ATOMIC flag to schedule_type), then the + * value must be in the range of [1, nb_event_queue_flows], which was + * previously provided in rte_event_dev_configure(). + */ + uint32_t nb_atomic_order_sequences; + /**< The maximum number of outstanding events waiting to be + * reordered by this queue. In other words, the number of entries in + * this queue’s reorder buffer.When the number of events in the + * reorder buffer reaches to *nb_atomic_order_sequences* then the + * scheduler cannot schedule the events from this queue and invalid + * event will be returned from dequeue until one or more entries are + * freed up/released. + * If the queue is configured for ordered scheduling (by applying the + * RTE_EVENT_QUEUE_CFG_ALL_TYPES flag to event_queue_cfg or + * RTE_SCHED_TYPE_ORDERED flag to schedule_type), then the value must + * be in the range of [1, nb_event_queue_flows], which was + * previously supplied to rte_event_dev_configure(). + */ + uint32_t event_queue_cfg; + /**< Queue cfg flags(EVENT_QUEUE_CFG_) */ + uint8_t schedule_type; + /**< Queue schedule type(RTE_SCHED_TYPE_*). + * Valid when RTE_EVENT_QUEUE_CFG_ALL_TYPES bit is not set in + * event_queue_cfg. + */ + uint8_t priority; + /**< Priority for this event queue relative to other event queues. + * The requested priority should in the range of + * [RTE_EVENT_DEV_PRIORITY_HIGHEST, RTE_EVENT_DEV_PRIORITY_LOWEST]. + * The implementation shall normalize the requested priority to + * event device supported priority value. + * Valid when the device has RTE_EVENT_DEV_CAP_QUEUE_QOS capability + */ +}; + +/** + * Retrieve the default configuration information of an event queue designated + * by its *queue_id* from the event driver for an event device. + * + * This function intended to be used in conjunction with rte_event_queue_setup() + * where caller needs to set up the queue by overriding few default values. + * + * @param dev_id + * The identifier of the device. + * @param queue_id + * The index of the event queue to get the configuration information. + * The value must be in the range [0, nb_event_queues - 1] + * previously supplied to rte_event_dev_configure(). + * @param[out] queue_conf + * The pointer to the default event queue configuration data. + * @return + * - 0: Success, driver updates the default event queue configuration data. + * - <0: Error code returned by the driver info get function. + * + * @see rte_event_queue_setup() + * + */ +int +rte_event_queue_default_conf_get(uint8_t dev_id, uint8_t queue_id, + struct rte_event_queue_conf *queue_conf); + +/** + * Allocate and set up an event queue for an event device. + * + * @param dev_id + * The identifier of the device. + * @param queue_id + * The index of the event queue to setup. The value must be in the range + * [0, nb_event_queues - 1] previously supplied to rte_event_dev_configure(). + * @param queue_conf + * The pointer to the configuration data to be used for the event queue. + * NULL value is allowed, in which case default configuration used. + * + * @see rte_event_queue_default_conf_get() + * + * @return + * - 0: Success, event queue correctly set up. + * - <0: event queue configuration failed + */ +int +rte_event_queue_setup(uint8_t dev_id, uint8_t queue_id, + const struct rte_event_queue_conf *queue_conf); + +/** + * The priority of the queue. + */ +#define RTE_EVENT_QUEUE_ATTR_PRIORITY 0 +/** + * The number of atomic flows configured for the queue. + */ +#define RTE_EVENT_QUEUE_ATTR_NB_ATOMIC_FLOWS 1 +/** + * The number of atomic order sequences configured for the queue. + */ +#define RTE_EVENT_QUEUE_ATTR_NB_ATOMIC_ORDER_SEQUENCES 2 +/** + * The cfg flags for the queue. + */ +#define RTE_EVENT_QUEUE_ATTR_EVENT_QUEUE_CFG 3 +/** + * The schedule type of the queue. + */ +#define RTE_EVENT_QUEUE_ATTR_SCHEDULE_TYPE 4 + +/** + * Get an attribute from a queue. + * + * @param dev_id + * Eventdev id + * @param queue_id + * Eventdev queue id + * @param attr_id + * The attribute ID to retrieve + * @param[out] attr_value + * A pointer that will be filled in with the attribute value if successful + * + * @return + * - 0: Successfully returned value + * - -EINVAL: invalid device, queue or attr_id provided, or attr_value was + * NULL + * - -EOVERFLOW: returned when attr_id is set to + * RTE_EVENT_QUEUE_ATTR_SCHEDULE_TYPE and event_queue_cfg is set to + * RTE_EVENT_QUEUE_CFG_ALL_TYPES + */ +int +rte_event_queue_attr_get(uint8_t dev_id, uint8_t queue_id, uint32_t attr_id, + uint32_t *attr_value); + +/* Event port specific APIs */ + +/** Event port configuration structure */ +struct rte_event_port_conf { + int32_t new_event_threshold; + /**< A backpressure threshold for new event enqueues on this port. + * Use for *closed system* event dev where event capacity is limited, + * and cannot exceed the capacity of the event dev. + * Configuring ports with different thresholds can make higher priority + * traffic less likely to be backpressured. + * For example, a port used to inject NIC Rx packets into the event dev + * can have a lower threshold so as not to overwhelm the device, + * while ports used for worker pools can have a higher threshold. + * This value cannot exceed the *nb_events_limit* + * which was previously supplied to rte_event_dev_configure(). + * This should be set to '-1' for *open system*. + */ + uint16_t dequeue_depth; + /**< Configure number of bulk dequeues for this event port. + * This value cannot exceed the *nb_event_port_dequeue_depth* + * which previously supplied to rte_event_dev_configure(). + * Ignored when device is not RTE_EVENT_DEV_CAP_BURST_MODE capable. + */ + uint16_t enqueue_depth; + /**< Configure number of bulk enqueues for this event port. + * This value cannot exceed the *nb_event_port_enqueue_depth* + * which previously supplied to rte_event_dev_configure(). + * Ignored when device is not RTE_EVENT_DEV_CAP_BURST_MODE capable. + */ +}; + +/** + * Retrieve the default configuration information of an event port designated + * by its *port_id* from the event driver for an event device. + * + * This function intended to be used in conjunction with rte_event_port_setup() + * where caller needs to set up the port by overriding few default values. + * + * @param dev_id + * The identifier of the device. + * @param port_id + * The index of the event port to get the configuration information. + * The value must be in the range [0, nb_event_ports - 1] + * previously supplied to rte_event_dev_configure(). + * @param[out] port_conf + * The pointer to the default event port configuration data + * @return + * - 0: Success, driver updates the default event port configuration data. + * - <0: Error code returned by the driver info get function. + * + * @see rte_event_port_setup() + * + */ +int +rte_event_port_default_conf_get(uint8_t dev_id, uint8_t port_id, + struct rte_event_port_conf *port_conf); + +/** + * Allocate and set up an event port for an event device. + * + * @param dev_id + * The identifier of the device. + * @param port_id + * The index of the event port to setup. The value must be in the range + * [0, nb_event_ports - 1] previously supplied to rte_event_dev_configure(). + * @param port_conf + * The pointer to the configuration data to be used for the queue. + * NULL value is allowed, in which case default configuration used. + * + * @see rte_event_port_default_conf_get() + * + * @return + * - 0: Success, event port correctly set up. + * - <0: Port configuration failed + * - (-EDQUOT) Quota exceeded(Application tried to link the queue configured + * with RTE_EVENT_QUEUE_CFG_SINGLE_LINK to more than one event ports) + */ +int +rte_event_port_setup(uint8_t dev_id, uint8_t port_id, + const struct rte_event_port_conf *port_conf); + +/** + * The queue depth of the port on the enqueue side + */ +#define RTE_EVENT_PORT_ATTR_ENQ_DEPTH 0 +/** + * The queue depth of the port on the dequeue side + */ +#define RTE_EVENT_PORT_ATTR_DEQ_DEPTH 1 +/** + * The new event threshold of the port + */ +#define RTE_EVENT_PORT_ATTR_NEW_EVENT_THRESHOLD 2 + +/** + * Get an attribute from a port. + * + * @param dev_id + * Eventdev id + * @param port_id + * Eventdev port id + * @param attr_id + * The attribute ID to retrieve + * @param[out] attr_value + * A pointer that will be filled in with the attribute value if successful + * + * @return + * - 0: Successfully returned value + * - (-EINVAL) Invalid device, port or attr_id, or attr_value was NULL + */ +int +rte_event_port_attr_get(uint8_t dev_id, uint8_t port_id, uint32_t attr_id, + uint32_t *attr_value); + +/** + * Start an event device. + * + * The device start step is the last one and consists of setting the event + * queues to start accepting the events and schedules to event ports. + * + * On success, all basic functions exported by the API (event enqueue, + * event dequeue and so on) can be invoked. + * + * @param dev_id + * Event device identifier + * @return + * - 0: Success, device started. + * - -ESTALE : Not all ports of the device are configured + * - -ENOLINK: Not all queues are linked, which could lead to deadlock. + */ +int +rte_event_dev_start(uint8_t dev_id); + +/** + * Stop an event device. The device can be restarted with a call to + * rte_event_dev_start() + * + * @param dev_id + * Event device identifier. + */ +void +rte_event_dev_stop(uint8_t dev_id); + +/** + * Close an event device. The device cannot be restarted! + * + * @param dev_id + * Event device identifier + * + * @return + * - 0 on successfully closing device + * - <0 on failure to close device + * - (-EAGAIN) if device is busy + */ +int +rte_event_dev_close(uint8_t dev_id); + +/* Scheduler type definitions */ +#define RTE_SCHED_TYPE_ORDERED 0 +/**< Ordered scheduling + * + * Events from an ordered flow of an event queue can be scheduled to multiple + * ports for concurrent processing while maintaining the original event order. + * This scheme enables the user to achieve high single flow throughput by + * avoiding SW synchronization for ordering between ports which bound to cores. + * + * The source flow ordering from an event queue is maintained when events are + * enqueued to their destination queue within the same ordered flow context. + * An event port holds the context until application call + * rte_event_dequeue_burst() from the same port, which implicitly releases + * the context. + * User may allow the scheduler to release the context earlier than that + * by invoking rte_event_enqueue_burst() with RTE_EVENT_OP_RELEASE operation. + * + * Events from the source queue appear in their original order when dequeued + * from a destination queue. + * Event ordering is based on the received event(s), but also other + * (newly allocated or stored) events are ordered when enqueued within the same + * ordered context. Events not enqueued (e.g. released or stored) within the + * context are considered missing from reordering and are skipped at this time + * (but can be ordered again within another context). + * + * @see rte_event_queue_setup(), rte_event_dequeue_burst(), RTE_EVENT_OP_RELEASE + */ + +#define RTE_SCHED_TYPE_ATOMIC 1 +/**< Atomic scheduling + * + * Events from an atomic flow of an event queue can be scheduled only to a + * single port at a time. The port is guaranteed to have exclusive (atomic) + * access to the associated flow context, which enables the user to avoid SW + * synchronization. Atomic flows also help to maintain event ordering + * since only one port at a time can process events from a flow of an + * event queue. + * + * The atomic queue synchronization context is dedicated to the port until + * application call rte_event_dequeue_burst() from the same port, + * which implicitly releases the context. User may allow the scheduler to + * release the context earlier than that by invoking rte_event_enqueue_burst() + * with RTE_EVENT_OP_RELEASE operation. + * + * @see rte_event_queue_setup(), rte_event_dequeue_burst(), RTE_EVENT_OP_RELEASE + */ + +#define RTE_SCHED_TYPE_PARALLEL 2 +/**< Parallel scheduling + * + * The scheduler performs priority scheduling, load balancing, etc. functions + * but does not provide additional event synchronization or ordering. + * It is free to schedule events from a single parallel flow of an event queue + * to multiple events ports for concurrent processing. + * The application is responsible for flow context synchronization and + * event ordering (SW synchronization). + * + * @see rte_event_queue_setup(), rte_event_dequeue_burst() + */ + +/* Event types to classify the event source */ +#define RTE_EVENT_TYPE_ETHDEV 0x0 +/**< The event generated from ethdev subsystem */ +#define RTE_EVENT_TYPE_CRYPTODEV 0x1 +/**< The event generated from crypodev subsystem */ +#define RTE_EVENT_TYPE_TIMERDEV 0x2 +/**< The event generated from timerdev subsystem */ +#define RTE_EVENT_TYPE_CPU 0x3 +/**< The event generated from cpu for pipelining. + * Application may use *sub_event_type* to further classify the event + */ +#define RTE_EVENT_TYPE_ETH_RX_ADAPTER 0x4 +/**< The event generated from event eth Rx adapter */ +#define RTE_EVENT_TYPE_MAX 0x10 +/**< Maximum number of event types */ + +/* Event enqueue operations */ +#define RTE_EVENT_OP_NEW 0 +/**< The event producers use this operation to inject a new event to the + * event device. + */ +#define RTE_EVENT_OP_FORWARD 1 +/**< The CPU use this operation to forward the event to different event queue or + * change to new application specific flow or schedule type to enable + * pipelining. + * + * This operation must only be enqueued to the same port that the + * event to be forwarded was dequeued from. + */ +#define RTE_EVENT_OP_RELEASE 2 +/**< Release the flow context associated with the schedule type. + * + * If current flow's scheduler type method is *RTE_SCHED_TYPE_ATOMIC* + * then this function hints the scheduler that the user has completed critical + * section processing in the current atomic context. + * The scheduler is now allowed to schedule events from the same flow from + * an event queue to another port. However, the context may be still held + * until the next rte_event_dequeue_burst() call, this call allows but does not + * force the scheduler to release the context early. + * + * Early atomic context release may increase parallelism and thus system + * performance, but the user needs to design carefully the split into critical + * vs non-critical sections. + * + * If current flow's scheduler type method is *RTE_SCHED_TYPE_ORDERED* + * then this function hints the scheduler that the user has done all that need + * to maintain event order in the current ordered context. + * The scheduler is allowed to release the ordered context of this port and + * avoid reordering any following enqueues. + * + * Early ordered context release may increase parallelism and thus system + * performance. + * + * If current flow's scheduler type method is *RTE_SCHED_TYPE_PARALLEL* + * or no scheduling context is held then this function may be an NOOP, + * depending on the implementation. + * + * This operation must only be enqueued to the same port that the + * event to be released was dequeued from. + * + */ + +/** + * The generic *rte_event* structure to hold the event attributes + * for dequeue and enqueue operation + */ +RTE_STD_C11 +struct rte_event { + /** WORD0 */ + union { + uint64_t event; + /** Event attributes for dequeue or enqueue operation */ + struct { + uint32_t flow_id:20; + /**< Targeted flow identifier for the enqueue and + * dequeue operation. + * The value must be in the range of + * [0, nb_event_queue_flows - 1] which + * previously supplied to rte_event_dev_configure(). + */ + uint32_t sub_event_type:8; + /**< Sub-event types based on the event source. + * @see RTE_EVENT_TYPE_CPU + */ + uint32_t event_type:4; + /**< Event type to classify the event source. + * @see RTE_EVENT_TYPE_ETHDEV, (RTE_EVENT_TYPE_*) + */ + uint8_t op:2; + /**< The type of event enqueue operation - new/forward/ + * etc.This field is not preserved across an instance + * and is undefined on dequeue. + * @see RTE_EVENT_OP_NEW, (RTE_EVENT_OP_*) + */ + uint8_t rsvd:4; + /**< Reserved for future use */ + uint8_t sched_type:2; + /**< Scheduler synchronization type (RTE_SCHED_TYPE_*) + * associated with flow id on a given event queue + * for the enqueue and dequeue operation. + */ + uint8_t queue_id; + /**< Targeted event queue identifier for the enqueue or + * dequeue operation. + * The value must be in the range of + * [0, nb_event_queues - 1] which previously supplied to + * rte_event_dev_configure(). + */ + uint8_t priority; + /**< Event priority relative to other events in the + * event queue. The requested priority should in the + * range of [RTE_EVENT_DEV_PRIORITY_HIGHEST, + * RTE_EVENT_DEV_PRIORITY_LOWEST]. + * The implementation shall normalize the requested + * priority to supported priority value. + * Valid when the device has + * RTE_EVENT_DEV_CAP_EVENT_QOS capability. + */ + uint8_t impl_opaque; + /**< Implementation specific opaque value. + * An implementation may use this field to hold + * implementation specific value to share between + * dequeue and enqueue operation. + * The application should not modify this field. + */ + }; + }; + /** WORD1 */ + union { + uint64_t u64; + /**< Opaque 64-bit value */ + void *event_ptr; + /**< Opaque event pointer */ + struct rte_mbuf *mbuf; + /**< mbuf pointer if dequeued event is associated with mbuf */ + }; +}; + +/* Ethdev Rx adapter capability bitmap flags */ +#define RTE_EVENT_ETH_RX_ADAPTER_CAP_INTERNAL_PORT 0x1 +/**< This flag is sent when the packet transfer mechanism is in HW. + * Ethdev can send packets to the event device using internal event port. + */ +#define RTE_EVENT_ETH_RX_ADAPTER_CAP_MULTI_EVENTQ 0x2 +/**< Adapter supports multiple event queues per ethdev. Every ethdev + * Rx queue can be connected to a unique event queue. + */ +#define RTE_EVENT_ETH_RX_ADAPTER_CAP_OVERRIDE_FLOW_ID 0x4 +/**< The application can override the adapter generated flow ID in the + * event. This flow ID can be specified when adding an ethdev Rx queue + * to the adapter using the ev member of struct rte_event_eth_rx_adapter + * @see struct rte_event_eth_rx_adapter_queue_conf::ev + * @see struct rte_event_eth_rx_adapter_queue_conf::rx_queue_flags + */ + +/** + * Retrieve the event device's ethdev Rx adapter capabilities for the + * specified ethernet port + * + * @param dev_id + * The identifier of the device. + * + * @param eth_port_id + * The identifier of the ethernet device. + * + * @param[out] caps + * A pointer to memory filled with Rx event adapter capabilities. + * + * @return + * - 0: Success, driver provides Rx event adapter capabilities for the + * ethernet device. + * - <0: Error code returned by the driver function. + * + */ +int +rte_event_eth_rx_adapter_caps_get(uint8_t dev_id, uint8_t eth_port_id, + uint32_t *caps); + +struct rte_eventdev_driver; +struct rte_eventdev_ops; +struct rte_eventdev; + +typedef uint16_t (*event_enqueue_t)(void *port, const struct rte_event *ev); +/**< @internal Enqueue event on port of a device */ + +typedef uint16_t (*event_enqueue_burst_t)(void *port, + const struct rte_event ev[], uint16_t nb_events); +/**< @internal Enqueue burst of events on port of a device */ + +typedef uint16_t (*event_dequeue_t)(void *port, struct rte_event *ev, + uint64_t timeout_ticks); +/**< @internal Dequeue event from port of a device */ + +typedef uint16_t (*event_dequeue_burst_t)(void *port, struct rte_event ev[], + uint16_t nb_events, uint64_t timeout_ticks); +/**< @internal Dequeue burst of events from port of a device */ + +#define RTE_EVENTDEV_NAME_MAX_LEN (64) +/**< @internal Max length of name of event PMD */ + +/** + * @internal + * The data part, with no function pointers, associated with each device. + * + * This structure is safe to place in shared memory to be common among + * different processes in a multi-process configuration. + */ +struct rte_eventdev_data { + int socket_id; + /**< Socket ID where memory is allocated */ + uint8_t dev_id; + /**< Device ID for this instance */ + uint8_t nb_queues; + /**< Number of event queues. */ + uint8_t nb_ports; + /**< Number of event ports. */ + void **ports; + /**< Array of pointers to ports. */ + struct rte_event_port_conf *ports_cfg; + /**< Array of port configuration structures. */ + struct rte_event_queue_conf *queues_cfg; + /**< Array of queue configuration structures. */ + uint16_t *links_map; + /**< Memory to store queues to port connections. */ + void *dev_private; + /**< PMD-specific private data */ + uint32_t event_dev_cap; + /**< Event device capabilities(RTE_EVENT_DEV_CAP_)*/ + struct rte_event_dev_config dev_conf; + /**< Configuration applied to device. */ + uint8_t service_inited; + /* Service initialization state */ + uint32_t service_id; + /* Service ID*/ + + RTE_STD_C11 + uint8_t dev_started : 1; + /**< Device state: STARTED(1)/STOPPED(0) */ + + char name[RTE_EVENTDEV_NAME_MAX_LEN]; + /**< Unique identifier name */ +} __rte_cache_aligned; + +/** @internal The data structure associated with each event device. */ +struct rte_eventdev { + event_enqueue_t enqueue; + /**< Pointer to PMD enqueue function. */ + event_enqueue_burst_t enqueue_burst; + /**< Pointer to PMD enqueue burst function. */ + event_enqueue_burst_t enqueue_new_burst; + /**< Pointer to PMD enqueue burst function(op new variant) */ + event_enqueue_burst_t enqueue_forward_burst; + /**< Pointer to PMD enqueue burst function(op forward variant) */ + event_dequeue_t dequeue; + /**< Pointer to PMD dequeue function. */ + event_dequeue_burst_t dequeue_burst; + /**< Pointer to PMD dequeue burst function. */ + + struct rte_eventdev_data *data; + /**< Pointer to device data */ + const struct rte_eventdev_ops *dev_ops; + /**< Functions exported by PMD */ + struct rte_device *dev; + /**< Device info. supplied by probing */ + + RTE_STD_C11 + uint8_t attached : 1; + /**< Flag indicating the device is attached */ +} __rte_cache_aligned; + +extern struct rte_eventdev *rte_eventdevs; +/** @internal The pool of rte_eventdev structures. */ + +static __rte_always_inline uint16_t +__rte_event_enqueue_burst(uint8_t dev_id, uint8_t port_id, + const struct rte_event ev[], uint16_t nb_events, + const event_enqueue_burst_t fn) +{ + const struct rte_eventdev *dev = &rte_eventdevs[dev_id]; + +#ifdef RTE_LIBRTE_EVENTDEV_DEBUG + if (dev_id >= RTE_EVENT_MAX_DEVS || !rte_eventdevs[dev_id].attached) { + rte_errno = -EINVAL; + return 0; + } + + if (port_id >= dev->data->nb_ports) { + rte_errno = -EINVAL; + return 0; + } +#endif + /* + * Allow zero cost non burst mode routine invocation if application + * requests nb_events as const one + */ + if (nb_events == 1) + return (*dev->enqueue)(dev->data->ports[port_id], ev); + else + return fn(dev->data->ports[port_id], ev, nb_events); +} + +/** + * Enqueue a burst of events objects or an event object supplied in *rte_event* + * structure on an event device designated by its *dev_id* through the event + * port specified by *port_id*. Each event object specifies the event queue on + * which it will be enqueued. + * + * The *nb_events* parameter is the number of event objects to enqueue which are + * supplied in the *ev* array of *rte_event* structure. + * + * Event operations RTE_EVENT_OP_FORWARD and RTE_EVENT_OP_RELEASE must only be + * enqueued to the same port that their associated events were dequeued from. + * + * The rte_event_enqueue_burst() function returns the number of + * events objects it actually enqueued. A return value equal to *nb_events* + * means that all event objects have been enqueued. + * + * @param dev_id + * The identifier of the device. + * @param port_id + * The identifier of the event port. + * @param ev + * Points to an array of *nb_events* objects of type *rte_event* structure + * which contain the event object enqueue operations to be processed. + * @param nb_events + * The number of event objects to enqueue, typically number of + * rte_event_port_enqueue_depth() available for this port. + * + * @return + * The number of event objects actually enqueued on the event device. The + * return value can be less than the value of the *nb_events* parameter when + * the event devices queue is full or if invalid parameters are specified in a + * *rte_event*. If the return value is less than *nb_events*, the remaining + * events at the end of ev[] are not consumed and the caller has to take care + * of them, and rte_errno is set accordingly. Possible errno values include: + * - -EINVAL The port ID is invalid, device ID is invalid, an event's queue + * ID is invalid, or an event's sched type doesn't match the + * capabilities of the destination queue. + * - -ENOSPC The event port was backpressured and unable to enqueue + * one or more events. This error code is only applicable to + * closed systems. + * @see rte_event_port_enqueue_depth() + */ +static inline uint16_t +rte_event_enqueue_burst(uint8_t dev_id, uint8_t port_id, + const struct rte_event ev[], uint16_t nb_events) +{ + const struct rte_eventdev *dev = &rte_eventdevs[dev_id]; + + return __rte_event_enqueue_burst(dev_id, port_id, ev, nb_events, + dev->enqueue_burst); +} + +/** + * Enqueue a burst of events objects of operation type *RTE_EVENT_OP_NEW* on + * an event device designated by its *dev_id* through the event port specified + * by *port_id*. + * + * Provides the same functionality as rte_event_enqueue_burst(), expect that + * application can use this API when the all objects in the burst contains + * the enqueue operation of the type *RTE_EVENT_OP_NEW*. This specialized + * function can provide the additional hint to the PMD and optimize if possible. + * + * The rte_event_enqueue_new_burst() result is undefined if the enqueue burst + * has event object of operation type != RTE_EVENT_OP_NEW. + * + * @param dev_id + * The identifier of the device. + * @param port_id + * The identifier of the event port. + * @param ev + * Points to an array of *nb_events* objects of type *rte_event* structure + * which contain the event object enqueue operations to be processed. + * @param nb_events + * The number of event objects to enqueue, typically number of + * rte_event_port_enqueue_depth() available for this port. + * + * @return + * The number of event objects actually enqueued on the event device. The + * return value can be less than the value of the *nb_events* parameter when + * the event devices queue is full or if invalid parameters are specified in a + * *rte_event*. If the return value is less than *nb_events*, the remaining + * events at the end of ev[] are not consumed and the caller has to take care + * of them, and rte_errno is set accordingly. Possible errno values include: + * - -EINVAL The port ID is invalid, device ID is invalid, an event's queue + * ID is invalid, or an event's sched type doesn't match the + * capabilities of the destination queue. + * - -ENOSPC The event port was backpressured and unable to enqueue + * one or more events. This error code is only applicable to + * closed systems. + * @see rte_event_port_enqueue_depth() rte_event_enqueue_burst() + */ +static inline uint16_t +rte_event_enqueue_new_burst(uint8_t dev_id, uint8_t port_id, + const struct rte_event ev[], uint16_t nb_events) +{ + const struct rte_eventdev *dev = &rte_eventdevs[dev_id]; + + return __rte_event_enqueue_burst(dev_id, port_id, ev, nb_events, + dev->enqueue_new_burst); +} + +/** + * Enqueue a burst of events objects of operation type *RTE_EVENT_OP_FORWARD* + * on an event device designated by its *dev_id* through the event port + * specified by *port_id*. + * + * Provides the same functionality as rte_event_enqueue_burst(), expect that + * application can use this API when the all objects in the burst contains + * the enqueue operation of the type *RTE_EVENT_OP_FORWARD*. This specialized + * function can provide the additional hint to the PMD and optimize if possible. + * + * The rte_event_enqueue_new_burst() result is undefined if the enqueue burst + * has event object of operation type != RTE_EVENT_OP_FORWARD. + * + * @param dev_id + * The identifier of the device. + * @param port_id + * The identifier of the event port. + * @param ev + * Points to an array of *nb_events* objects of type *rte_event* structure + * which contain the event object enqueue operations to be processed. + * @param nb_events + * The number of event objects to enqueue, typically number of + * rte_event_port_enqueue_depth() available for this port. + * + * @return + * The number of event objects actually enqueued on the event device. The + * return value can be less than the value of the *nb_events* parameter when + * the event devices queue is full or if invalid parameters are specified in a + * *rte_event*. If the return value is less than *nb_events*, the remaining + * events at the end of ev[] are not consumed and the caller has to take care + * of them, and rte_errno is set accordingly. Possible errno values include: + * - -EINVAL The port ID is invalid, device ID is invalid, an event's queue + * ID is invalid, or an event's sched type doesn't match the + * capabilities of the destination queue. + * - -ENOSPC The event port was backpressured and unable to enqueue + * one or more events. This error code is only applicable to + * closed systems. + * @see rte_event_port_enqueue_depth() rte_event_enqueue_burst() + */ +static inline uint16_t +rte_event_enqueue_forward_burst(uint8_t dev_id, uint8_t port_id, + const struct rte_event ev[], uint16_t nb_events) +{ + const struct rte_eventdev *dev = &rte_eventdevs[dev_id]; + + return __rte_event_enqueue_burst(dev_id, port_id, ev, nb_events, + dev->enqueue_forward_burst); +} + +/** + * Converts nanoseconds to *timeout_ticks* value for rte_event_dequeue_burst() + * + * If the device is configured with RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT flag + * then application can use this function to convert timeout value in + * nanoseconds to implementations specific timeout value supplied in + * rte_event_dequeue_burst() + * + * @param dev_id + * The identifier of the device. + * @param ns + * Wait time in nanosecond + * @param[out] timeout_ticks + * Value for the *timeout_ticks* parameter in rte_event_dequeue_burst() + * + * @return + * - 0 on success. + * - -ENOTSUP if the device doesn't support timeouts + * - -EINVAL if *dev_id* is invalid or *timeout_ticks* is NULL + * - other values < 0 on failure. + * + * @see rte_event_dequeue_burst(), RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT + * @see rte_event_dev_configure() + * + */ +int +rte_event_dequeue_timeout_ticks(uint8_t dev_id, uint64_t ns, + uint64_t *timeout_ticks); + +/** + * Dequeue a burst of events objects or an event object from the event port + * designated by its *event_port_id*, on an event device designated + * by its *dev_id*. + * + * rte_event_dequeue_burst() does not dictate the specifics of scheduling + * algorithm as each eventdev driver may have different criteria to schedule + * an event. However, in general, from an application perspective scheduler may + * use the following scheme to dispatch an event to the port. + * + * 1) Selection of event queue based on + * a) The list of event queues are linked to the event port. + * b) If the device has RTE_EVENT_DEV_CAP_QUEUE_QOS capability then event + * queue selection from list is based on event queue priority relative to + * other event queue supplied as *priority* in rte_event_queue_setup() + * c) If the device has RTE_EVENT_DEV_CAP_EVENT_QOS capability then event + * queue selection from the list is based on event priority supplied as + * *priority* in rte_event_enqueue_burst() + * 2) Selection of event + * a) The number of flows available in selected event queue. + * b) Schedule type method associated with the event + * + * The *nb_events* parameter is the maximum number of event objects to dequeue + * which are returned in the *ev* array of *rte_event* structure. + * + * The rte_event_dequeue_burst() function returns the number of events objects + * it actually dequeued. A return value equal to *nb_events* means that all + * event objects have been dequeued. + * + * The number of events dequeued is the number of scheduler contexts held by + * this port. These contexts are automatically released in the next + * rte_event_dequeue_burst() invocation, or invoking rte_event_enqueue_burst() + * with RTE_EVENT_OP_RELEASE operation can be used to release the + * contexts early. + * + * Event operations RTE_EVENT_OP_FORWARD and RTE_EVENT_OP_RELEASE must only be + * enqueued to the same port that their associated events were dequeued from. + * + * @param dev_id + * The identifier of the device. + * @param port_id + * The identifier of the event port. + * @param[out] ev + * Points to an array of *nb_events* objects of type *rte_event* structure + * for output to be populated with the dequeued event objects. + * @param nb_events + * The maximum number of event objects to dequeue, typically number of + * rte_event_port_dequeue_depth() available for this port. + * + * @param timeout_ticks + * - 0 no-wait, returns immediately if there is no event. + * - >0 wait for the event, if the device is configured with + * RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT then this function will wait until + * at least one event is available or *timeout_ticks* time. + * if the device is not configured with RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT + * then this function will wait until the event available or + * *dequeue_timeout_ns* ns which was previously supplied to + * rte_event_dev_configure() + * + * @return + * The number of event objects actually dequeued from the port. The return + * value can be less than the value of the *nb_events* parameter when the + * event port's queue is not full. + * + * @see rte_event_port_dequeue_depth() + */ +static inline uint16_t +rte_event_dequeue_burst(uint8_t dev_id, uint8_t port_id, struct rte_event ev[], + uint16_t nb_events, uint64_t timeout_ticks) +{ + struct rte_eventdev *dev = &rte_eventdevs[dev_id]; + +#ifdef RTE_LIBRTE_EVENTDEV_DEBUG + if (dev_id >= RTE_EVENT_MAX_DEVS || !rte_eventdevs[dev_id].attached) { + rte_errno = -EINVAL; + return 0; + } + + if (port_id >= dev->data->nb_ports) { + rte_errno = -EINVAL; + return 0; + } +#endif + + /* + * Allow zero cost non burst mode routine invocation if application + * requests nb_events as const one + */ + if (nb_events == 1) + return (*dev->dequeue)( + dev->data->ports[port_id], ev, timeout_ticks); + else + return (*dev->dequeue_burst)( + dev->data->ports[port_id], ev, nb_events, + timeout_ticks); +} + +/** + * Link multiple source event queues supplied in *queues* to the destination + * event port designated by its *port_id* with associated service priority + * supplied in *priorities* on the event device designated by its *dev_id*. + * + * The link establishment shall enable the event port *port_id* from + * receiving events from the specified event queue(s) supplied in *queues* + * + * An event queue may link to one or more event ports. + * The number of links can be established from an event queue to event port is + * implementation defined. + * + * Event queue(s) to event port link establishment can be changed at runtime + * without re-configuring the device to support scaling and to reduce the + * latency of critical work by establishing the link with more event ports + * at runtime. + * + * @param dev_id + * The identifier of the device. + * + * @param port_id + * Event port identifier to select the destination port to link. + * + * @param queues + * Points to an array of *nb_links* event queues to be linked + * to the event port. + * NULL value is allowed, in which case this function links all the configured + * event queues *nb_event_queues* which previously supplied to + * rte_event_dev_configure() to the event port *port_id* + * + * @param priorities + * Points to an array of *nb_links* service priorities associated with each + * event queue link to event port. + * The priority defines the event port's servicing priority for + * event queue, which may be ignored by an implementation. + * The requested priority should in the range of + * [RTE_EVENT_DEV_PRIORITY_HIGHEST, RTE_EVENT_DEV_PRIORITY_LOWEST]. + * The implementation shall normalize the requested priority to + * implementation supported priority value. + * NULL value is allowed, in which case this function links the event queues + * with RTE_EVENT_DEV_PRIORITY_NORMAL servicing priority + * + * @param nb_links + * The number of links to establish. This parameter is ignored if queues is + * NULL. + * + * @return + * The number of links actually established. The return value can be less than + * the value of the *nb_links* parameter when the implementation has the + * limitation on specific queue to port link establishment or if invalid + * parameters are specified in *queues* + * If the return value is less than *nb_links*, the remaining links at the end + * of link[] are not established, and the caller has to take care of them. + * If return value is less than *nb_links* then implementation shall update the + * rte_errno accordingly, Possible rte_errno values are + * (-EDQUOT) Quota exceeded(Application tried to link the queue configured with + * RTE_EVENT_QUEUE_CFG_SINGLE_LINK to more than one event ports) + * (-EINVAL) Invalid parameter + * + */ +int +rte_event_port_link(uint8_t dev_id, uint8_t port_id, + const uint8_t queues[], const uint8_t priorities[], + uint16_t nb_links); + +/** + * Unlink multiple source event queues supplied in *queues* from the destination + * event port designated by its *port_id* on the event device designated + * by its *dev_id*. + * + * The unlink establishment shall disable the event port *port_id* from + * receiving events from the specified event queue *queue_id* + * + * Event queue(s) to event port unlink establishment can be changed at runtime + * without re-configuring the device. + * + * @param dev_id + * The identifier of the device. + * + * @param port_id + * Event port identifier to select the destination port to unlink. + * + * @param queues + * Points to an array of *nb_unlinks* event queues to be unlinked + * from the event port. + * NULL value is allowed, in which case this function unlinks all the + * event queue(s) from the event port *port_id*. + * + * @param nb_unlinks + * The number of unlinks to establish. This parameter is ignored if queues is + * NULL. + * + * @return + * The number of unlinks actually established. The return value can be less + * than the value of the *nb_unlinks* parameter when the implementation has the + * limitation on specific queue to port unlink establishment or + * if invalid parameters are specified. + * If the return value is less than *nb_unlinks*, the remaining queues at the + * end of queues[] are not established, and the caller has to take care of them. + * If return value is less than *nb_unlinks* then implementation shall update + * the rte_errno accordingly, Possible rte_errno values are + * (-EINVAL) Invalid parameter + * + */ +int +rte_event_port_unlink(uint8_t dev_id, uint8_t port_id, + uint8_t queues[], uint16_t nb_unlinks); + +/** + * Retrieve the list of source event queues and its associated service priority + * linked to the destination event port designated by its *port_id* + * on the event device designated by its *dev_id*. + * + * @param dev_id + * The identifier of the device. + * + * @param port_id + * Event port identifier. + * + * @param[out] queues + * Points to an array of *queues* for output. + * The caller has to allocate *RTE_EVENT_MAX_QUEUES_PER_DEV* bytes to + * store the event queue(s) linked with event port *port_id* + * + * @param[out] priorities + * Points to an array of *priorities* for output. + * The caller has to allocate *RTE_EVENT_MAX_QUEUES_PER_DEV* bytes to + * store the service priority associated with each event queue linked + * + * @return + * The number of links established on the event port designated by its + * *port_id*. + * - <0 on failure. + * + */ +int +rte_event_port_links_get(uint8_t dev_id, uint8_t port_id, + uint8_t queues[], uint8_t priorities[]); + +/** + * Retrieve the service ID of the event dev. If the adapter doesn't use + * a rte_service function, this function returns -ESRCH. + * + * @param dev_id + * The identifier of the device. + * + * @param [out] service_id + * A pointer to a uint32_t, to be filled in with the service id. + * + * @return + * - 0: Success + * - <0: Error code on failure, if the event dev doesn't use a rte_service + * function, this function returns -ESRCH. + */ +int +rte_event_dev_service_id_get(uint8_t dev_id, uint32_t *service_id); + +/** + * Dump internal information about *dev_id* to the FILE* provided in *f*. + * + * @param dev_id + * The identifier of the device. + * + * @param f + * A pointer to a file for output + * + * @return + * - 0: on success + * - <0: on failure. + */ +int +rte_event_dev_dump(uint8_t dev_id, FILE *f); + +/** Maximum name length for extended statistics counters */ +#define RTE_EVENT_DEV_XSTATS_NAME_SIZE 64 + +/** + * Selects the component of the eventdev to retrieve statistics from. + */ +enum rte_event_dev_xstats_mode { + RTE_EVENT_DEV_XSTATS_DEVICE, + RTE_EVENT_DEV_XSTATS_PORT, + RTE_EVENT_DEV_XSTATS_QUEUE, +}; + +/** + * A name-key lookup element for extended statistics. + * + * This structure is used to map between names and ID numbers + * for extended ethdev statistics. + */ +struct rte_event_dev_xstats_name { + char name[RTE_EVENT_DEV_XSTATS_NAME_SIZE]; +}; + +/** + * Retrieve names of extended statistics of an event device. + * + * @param dev_id + * The identifier of the event device. + * @param mode + * The mode of statistics to retrieve. Choices include the device statistics, + * port statistics or queue statistics. + * @param queue_port_id + * Used to specify the port or queue number in queue or port mode, and is + * ignored in device mode. + * @param[out] xstats_names + * Block of memory to insert names into. Must be at least size in capacity. + * If set to NULL, function returns required capacity. + * @param[out] ids + * Block of memory to insert ids into. Must be at least size in capacity. + * If set to NULL, function returns required capacity. The id values returned + * can be passed to *rte_event_dev_xstats_get* to select statistics. + * @param size + * Capacity of xstats_names (number of names). + * @return + * - positive value lower or equal to size: success. The return value + * is the number of entries filled in the stats table. + * - positive value higher than size: error, the given statistics table + * is too small. The return value corresponds to the size that should + * be given to succeed. The entries in the table are not valid and + * shall not be used by the caller. + * - negative value on error: + * -ENODEV for invalid *dev_id* + * -EINVAL for invalid mode, queue port or id parameters + * -ENOTSUP if the device doesn't support this function. + */ +int +rte_event_dev_xstats_names_get(uint8_t dev_id, + enum rte_event_dev_xstats_mode mode, + uint8_t queue_port_id, + struct rte_event_dev_xstats_name *xstats_names, + unsigned int *ids, + unsigned int size); + +/** + * Retrieve extended statistics of an event device. + * + * @param dev_id + * The identifier of the device. + * @param mode + * The mode of statistics to retrieve. Choices include the device statistics, + * port statistics or queue statistics. + * @param queue_port_id + * Used to specify the port or queue number in queue or port mode, and is + * ignored in device mode. + * @param ids + * The id numbers of the stats to get. The ids can be got from the stat + * position in the stat list from rte_event_dev_get_xstats_names(), or + * by using rte_eventdev_get_xstats_by_name() + * @param[out] values + * The values for each stats request by ID. + * @param n + * The number of stats requested + * @return + * - positive value: number of stat entries filled into the values array + * - negative value on error: + * -ENODEV for invalid *dev_id* + * -EINVAL for invalid mode, queue port or id parameters + * -ENOTSUP if the device doesn't support this function. + */ +int +rte_event_dev_xstats_get(uint8_t dev_id, + enum rte_event_dev_xstats_mode mode, + uint8_t queue_port_id, + const unsigned int ids[], + uint64_t values[], unsigned int n); + +/** + * Retrieve the value of a single stat by requesting it by name. + * + * @param dev_id + * The identifier of the device + * @param name + * The stat name to retrieve + * @param[out] id + * If non-NULL, the numerical id of the stat will be returned, so that further + * requests for the stat can be got using rte_eventdev_xstats_get, which will + * be faster as it doesn't need to scan a list of names for the stat. + * If the stat cannot be found, the id returned will be (unsigned)-1. + * @return + * - positive value or zero: the stat value + * - negative value: -EINVAL if stat not found, -ENOTSUP if not supported. + */ +uint64_t +rte_event_dev_xstats_by_name_get(uint8_t dev_id, const char *name, + unsigned int *id); + +/** + * Reset the values of the xstats of the selected component in the device. + * + * @param dev_id + * The identifier of the device + * @param mode + * The mode of the statistics to reset. Choose from device, queue or port. + * @param queue_port_id + * The queue or port to reset. 0 and positive values select ports and queues, + * while -1 indicates all ports or queues. + * @param ids + * Selects specific statistics to be reset. When NULL, all statistics selected + * by *mode* will be reset. If non-NULL, must point to array of at least + * *nb_ids* size. + * @param nb_ids + * The number of ids available from the *ids* array. Ignored when ids is NULL. + * @return + * - zero: successfully reset the statistics to zero + * - negative value: -EINVAL invalid parameters, -ENOTSUP if not supported. + */ +int +rte_event_dev_xstats_reset(uint8_t dev_id, + enum rte_event_dev_xstats_mode mode, + int16_t queue_port_id, + const uint32_t ids[], + uint32_t nb_ids); + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_EVENTDEV_H_ */ diff --git a/dpdk/lib/librte_eventdev/rte_eventdev_pmd.h b/dpdk/lib/librte_eventdev/rte_eventdev_pmd.h new file mode 100644 index 00000000..64535f2e --- /dev/null +++ b/dpdk/lib/librte_eventdev/rte_eventdev_pmd.h @@ -0,0 +1,693 @@ +/* + * + * Copyright(c) 2016 Cavium, Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_EVENTDEV_PMD_H_ +#define _RTE_EVENTDEV_PMD_H_ + +/** @file + * RTE Event PMD APIs + * + * @note + * These API are from event PMD only and user applications should not call + * them directly. + */ + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +#include +#include +#include +#include +#include + +#include "rte_eventdev.h" + +/* Logging Macros */ +#define RTE_EDEV_LOG_ERR(...) \ + RTE_LOG(ERR, EVENTDEV, \ + RTE_FMT("%s() line %u: " RTE_FMT_HEAD(__VA_ARGS__,) "\n", \ + __func__, __LINE__, RTE_FMT_TAIL(__VA_ARGS__,))) + +#ifdef RTE_LIBRTE_EVENTDEV_DEBUG +#define RTE_EDEV_LOG_DEBUG(...) \ + RTE_LOG(DEBUG, EVENTDEV, \ + RTE_FMT("%s() line %u: " RTE_FMT_HEAD(__VA_ARGS__,) "\n", \ + __func__, __LINE__, RTE_FMT_TAIL(__VA_ARGS__,))) +#else +#define RTE_EDEV_LOG_DEBUG(...) (void)0 +#endif + +/* Macros to check for valid device */ +#define RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(dev_id, retval) do { \ + if (!rte_event_pmd_is_valid_dev((dev_id))) { \ + RTE_EDEV_LOG_ERR("Invalid dev_id=%d\n", dev_id); \ + return retval; \ + } \ +} while (0) + +#define RTE_EVENTDEV_VALID_DEVID_OR_ERRNO_RET(dev_id, errno, retval) do { \ + if (!rte_event_pmd_is_valid_dev((dev_id))) { \ + RTE_EDEV_LOG_ERR("Invalid dev_id=%d\n", dev_id); \ + rte_errno = errno; \ + return retval; \ + } \ +} while (0) + +#define RTE_EVENTDEV_VALID_DEVID_OR_RET(dev_id) do { \ + if (!rte_event_pmd_is_valid_dev((dev_id))) { \ + RTE_EDEV_LOG_ERR("Invalid dev_id=%d\n", dev_id); \ + return; \ + } \ +} while (0) + +#define RTE_EVENT_ETH_RX_ADAPTER_SW_CAP \ + ((RTE_EVENT_ETH_RX_ADAPTER_CAP_OVERRIDE_FLOW_ID) | \ + (RTE_EVENT_ETH_RX_ADAPTER_CAP_MULTI_EVENTQ)) + +/**< Ethernet Rx adapter cap to return If the packet transfers from + * the ethdev to eventdev use a SW service function + */ + +#define RTE_EVENTDEV_DETACHED (0) +#define RTE_EVENTDEV_ATTACHED (1) + +struct rte_eth_dev; + +/** Global structure used for maintaining state of allocated event devices */ +struct rte_eventdev_global { + uint8_t nb_devs; /**< Number of devices found */ +}; + +extern struct rte_eventdev_global *rte_eventdev_globals; +/** Pointer to global event devices data structure. */ +extern struct rte_eventdev *rte_eventdevs; +/** The pool of rte_eventdev structures. */ + +/** + * Get the rte_eventdev structure device pointer for the named device. + * + * @param name + * device name to select the device structure. + * + * @return + * - The rte_eventdev structure pointer for the given device ID. + */ +static inline struct rte_eventdev * +rte_event_pmd_get_named_dev(const char *name) +{ + struct rte_eventdev *dev; + unsigned int i; + + if (name == NULL) + return NULL; + + for (i = 0; i < RTE_EVENT_MAX_DEVS; i++) { + dev = &rte_eventdevs[i]; + if ((dev->attached == RTE_EVENTDEV_ATTACHED) && + (strcmp(dev->data->name, name) == 0)) + return dev; + } + + return NULL; +} + +/** + * Validate if the event device index is valid attached event device. + * + * @param dev_id + * Event device index. + * + * @return + * - If the device index is valid (1) or not (0). + */ +static inline unsigned +rte_event_pmd_is_valid_dev(uint8_t dev_id) +{ + struct rte_eventdev *dev; + + if (dev_id >= RTE_EVENT_MAX_DEVS) + return 0; + + dev = &rte_eventdevs[dev_id]; + if (dev->attached != RTE_EVENTDEV_ATTACHED) + return 0; + else + return 1; +} + +/** + * Definitions of all functions exported by a driver through the + * the generic structure of type *event_dev_ops* supplied in the + * *rte_eventdev* structure associated with a device. + */ + +/** + * Get device information of a device. + * + * @param dev + * Event device pointer + * @param dev_info + * Event device information structure + * + * @return + * Returns 0 on success + */ +typedef void (*eventdev_info_get_t)(struct rte_eventdev *dev, + struct rte_event_dev_info *dev_info); + +/** + * Configure a device. + * + * @param dev + * Event device pointer + * + * @return + * Returns 0 on success + */ +typedef int (*eventdev_configure_t)(const struct rte_eventdev *dev); + +/** + * Start a configured device. + * + * @param dev + * Event device pointer + * + * @return + * Returns 0 on success + */ +typedef int (*eventdev_start_t)(struct rte_eventdev *dev); + +/** + * Stop a configured device. + * + * @param dev + * Event device pointer + */ +typedef void (*eventdev_stop_t)(struct rte_eventdev *dev); + +/** + * Close a configured device. + * + * @param dev + * Event device pointer + * + * @return + * - 0 on success + * - (-EAGAIN) if can't close as device is busy + */ +typedef int (*eventdev_close_t)(struct rte_eventdev *dev); + +/** + * Retrieve the default event queue configuration. + * + * @param dev + * Event device pointer + * @param queue_id + * Event queue index + * @param[out] queue_conf + * Event queue configuration structure + * + */ +typedef void (*eventdev_queue_default_conf_get_t)(struct rte_eventdev *dev, + uint8_t queue_id, struct rte_event_queue_conf *queue_conf); + +/** + * Setup an event queue. + * + * @param dev + * Event device pointer + * @param queue_id + * Event queue index + * @param queue_conf + * Event queue configuration structure + * + * @return + * Returns 0 on success. + */ +typedef int (*eventdev_queue_setup_t)(struct rte_eventdev *dev, + uint8_t queue_id, + const struct rte_event_queue_conf *queue_conf); + +/** + * Release resources allocated by given event queue. + * + * @param dev + * Event device pointer + * @param queue_id + * Event queue index + * + */ +typedef void (*eventdev_queue_release_t)(struct rte_eventdev *dev, + uint8_t queue_id); + +/** + * Retrieve the default event port configuration. + * + * @param dev + * Event device pointer + * @param port_id + * Event port index + * @param[out] port_conf + * Event port configuration structure + * + */ +typedef void (*eventdev_port_default_conf_get_t)(struct rte_eventdev *dev, + uint8_t port_id, struct rte_event_port_conf *port_conf); + +/** + * Setup an event port. + * + * @param dev + * Event device pointer + * @param port_id + * Event port index + * @param port_conf + * Event port configuration structure + * + * @return + * Returns 0 on success. + */ +typedef int (*eventdev_port_setup_t)(struct rte_eventdev *dev, + uint8_t port_id, + const struct rte_event_port_conf *port_conf); + +/** + * Release memory resources allocated by given event port. + * + * @param port + * Event port pointer + * + */ +typedef void (*eventdev_port_release_t)(void *port); + +/** + * Link multiple source event queues to destination event port. + * + * @param dev + * Event device pointer + * @param port + * Event port pointer + * @param link + * Points to an array of *nb_links* event queues to be linked + * to the event port. + * @param priorities + * Points to an array of *nb_links* service priorities associated with each + * event queue link to event port. + * @param nb_links + * The number of links to establish + * + * @return + * Returns 0 on success. + * + */ +typedef int (*eventdev_port_link_t)(struct rte_eventdev *dev, void *port, + const uint8_t queues[], const uint8_t priorities[], + uint16_t nb_links); + +/** + * Unlink multiple source event queues from destination event port. + * + * @param dev + * Event device pointer + * @param port + * Event port pointer + * @param queues + * An array of *nb_unlinks* event queues to be unlinked from the event port. + * @param nb_unlinks + * The number of unlinks to establish + * + * @return + * Returns 0 on success. + * + */ +typedef int (*eventdev_port_unlink_t)(struct rte_eventdev *dev, void *port, + uint8_t queues[], uint16_t nb_unlinks); + +/** + * Converts nanoseconds to *timeout_ticks* value for rte_event_dequeue() + * + * @param dev + * Event device pointer + * @param ns + * Wait time in nanosecond + * @param[out] timeout_ticks + * Value for the *timeout_ticks* parameter in rte_event_dequeue() function + * + * @return + * Returns 0 on success. + * + */ +typedef int (*eventdev_dequeue_timeout_ticks_t)(struct rte_eventdev *dev, + uint64_t ns, uint64_t *timeout_ticks); + +/** + * Dump internal information + * + * @param dev + * Event device pointer + * @param f + * A pointer to a file for output + * + */ +typedef void (*eventdev_dump_t)(struct rte_eventdev *dev, FILE *f); + +/** + * Retrieve a set of statistics from device + * + * @param dev + * Event device pointer + * @param ids + * The stat ids to retrieve + * @param values + * The returned stat values + * @param n + * The number of id values and entries in the values array + * @return + * The number of stat values successfully filled into the values array + */ +typedef int (*eventdev_xstats_get_t)(const struct rte_eventdev *dev, + enum rte_event_dev_xstats_mode mode, uint8_t queue_port_id, + const unsigned int ids[], uint64_t values[], unsigned int n); + +/** + * Resets the statistic values in xstats for the device, based on mode. + */ +typedef int (*eventdev_xstats_reset_t)(struct rte_eventdev *dev, + enum rte_event_dev_xstats_mode mode, + int16_t queue_port_id, + const uint32_t ids[], + uint32_t nb_ids); + +/** + * Get names of extended stats of an event device + * + * @param dev + * Event device pointer + * @param xstats_names + * Array of name values to be filled in + * @param size + * Number of values in the xstats_names array + * @return + * When size >= the number of stats, return the number of stat values filled + * into the array. + * When size < the number of available stats, return the number of stats + * values, and do not fill in any data into xstats_names. + */ +typedef int (*eventdev_xstats_get_names_t)(const struct rte_eventdev *dev, + enum rte_event_dev_xstats_mode mode, uint8_t queue_port_id, + struct rte_event_dev_xstats_name *xstats_names, + unsigned int *ids, unsigned int size); + +/** + * Get value of one stats and optionally return its id + * + * @param dev + * Event device pointer + * @param name + * The name of the stat to retrieve + * @param id + * Pointer to an unsigned int where we store the stat-id for future reference. + * This pointer may be null if the id is not required. + * @return + * The value of the stat, or (uint64_t)-1 if the stat is not found. + * If the stat is not found, the id value will be returned as (unsigned)-1, + * if id pointer is non-NULL + */ +typedef uint64_t (*eventdev_xstats_get_by_name)(const struct rte_eventdev *dev, + const char *name, unsigned int *id); + + +/** + * Retrieve the event device's ethdev Rx adapter capabilities for the + * specified ethernet port + * + * @param dev + * Event device pointer + * + * @param eth_dev + * Ethernet device pointer + * + * @param[out] caps + * A pointer to memory filled with Rx event adapter capabilities. + * + * @return + * - 0: Success, driver provides Rx event adapter capabilities for the + * ethernet device. + * - <0: Error code returned by the driver function. + * + */ +typedef int (*eventdev_eth_rx_adapter_caps_get_t) + (const struct rte_eventdev *dev, + const struct rte_eth_dev *eth_dev, + uint32_t *caps); + +struct rte_event_eth_rx_adapter_queue_conf *queue_conf; + +/** + * Add ethernet Rx queues to event device. This callback is invoked if + * the caps returned from rte_eventdev_eth_rx_adapter_caps_get(, eth_port_id) + * has RTE_EVENT_ETH_RX_ADAPTER_CAP_INTERNAL_PORT set. + * + * @param dev + * Event device pointer + * + * @param eth_dev + * Ethernet device pointer + * + * @param rx_queue_id + * Ethernet device receive queue index + * + * @param queue_conf + * Additional configuration structure + + * @return + * - 0: Success, ethernet receive queue added successfully. + * - <0: Error code returned by the driver function. + * + */ +typedef int (*eventdev_eth_rx_adapter_queue_add_t)( + const struct rte_eventdev *dev, + const struct rte_eth_dev *eth_dev, + int32_t rx_queue_id, + const struct rte_event_eth_rx_adapter_queue_conf *queue_conf); + +/** + * Delete ethernet Rx queues from event device. This callback is invoked if + * the caps returned from eventdev_eth_rx_adapter_caps_get(, eth_port_id) + * has RTE_EVENT_ETH_RX_ADAPTER_CAP_INTERNAL_PORT set. + * + * @param dev + * Event device pointer + * + * @param eth_dev + * Ethernet device pointer + * + * @param rx_queue_id + * Ethernet device receive queue index + * + * @return + * - 0: Success, ethernet receive queue deleted successfully. + * - <0: Error code returned by the driver function. + * + */ +typedef int (*eventdev_eth_rx_adapter_queue_del_t) + (const struct rte_eventdev *dev, + const struct rte_eth_dev *eth_dev, + int32_t rx_queue_id); + +/** + * Start ethernet Rx adapter. This callback is invoked if + * the caps returned from eventdev_eth_rx_adapter_caps_get(.., eth_port_id) + * has RTE_EVENT_ETH_RX_ADAPTER_CAP_INTERNAL_PORT set and Rx queues + * from eth_port_id have been added to the event device. + * + * @param dev + * Event device pointer + * + * @param eth_dev + * Ethernet device pointer + * + * @return + * - 0: Success, ethernet Rx adapter started successfully. + * - <0: Error code returned by the driver function. + */ +typedef int (*eventdev_eth_rx_adapter_start_t) + (const struct rte_eventdev *dev, + const struct rte_eth_dev *eth_dev); + +/** + * Stop ethernet Rx adapter. This callback is invoked if + * the caps returned from eventdev_eth_rx_adapter_caps_get(..,eth_port_id) + * has RTE_EVENT_ETH_RX_ADAPTER_CAP_INTERNAL_PORT set and Rx queues + * from eth_port_id have been added to the event device. + * + * @param dev + * Event device pointer + * + * @param eth_dev + * Ethernet device pointer + * + * @return + * - 0: Success, ethernet Rx adapter stopped successfully. + * - <0: Error code returned by the driver function. + */ +typedef int (*eventdev_eth_rx_adapter_stop_t) + (const struct rte_eventdev *dev, + const struct rte_eth_dev *eth_dev); + +struct rte_event_eth_rx_adapter_stats *stats; + +/** + * Retrieve ethernet Rx adapter statistics. + * + * @param dev + * Event device pointer + * + * @param eth_dev + * Ethernet device pointer + * + * @param[out] stats + * Pointer to stats structure + * + * @return + * Return 0 on success. + */ + +typedef int (*eventdev_eth_rx_adapter_stats_get) + (const struct rte_eventdev *dev, + const struct rte_eth_dev *eth_dev, + struct rte_event_eth_rx_adapter_stats *stats); +/** + * Reset ethernet Rx adapter statistics. + * + * @param dev + * Event device pointer + * + * @param eth_dev + * Ethernet device pointer + * + * @return + * Return 0 on success. + */ +typedef int (*eventdev_eth_rx_adapter_stats_reset) + (const struct rte_eventdev *dev, + const struct rte_eth_dev *eth_dev); + +/** Event device operations function pointer table */ +struct rte_eventdev_ops { + eventdev_info_get_t dev_infos_get; /**< Get device info. */ + eventdev_configure_t dev_configure; /**< Configure device. */ + eventdev_start_t dev_start; /**< Start device. */ + eventdev_stop_t dev_stop; /**< Stop device. */ + eventdev_close_t dev_close; /**< Close device. */ + + eventdev_queue_default_conf_get_t queue_def_conf; + /**< Get default queue configuration. */ + eventdev_queue_setup_t queue_setup; + /**< Set up an event queue. */ + eventdev_queue_release_t queue_release; + /**< Release an event queue. */ + + eventdev_port_default_conf_get_t port_def_conf; + /**< Get default port configuration. */ + eventdev_port_setup_t port_setup; + /**< Set up an event port. */ + eventdev_port_release_t port_release; + /**< Release an event port. */ + + eventdev_port_link_t port_link; + /**< Link event queues to an event port. */ + eventdev_port_unlink_t port_unlink; + /**< Unlink event queues from an event port. */ + eventdev_dequeue_timeout_ticks_t timeout_ticks; + /**< Converts ns to *timeout_ticks* value for rte_event_dequeue() */ + eventdev_dump_t dump; + /* Dump internal information */ + + eventdev_xstats_get_t xstats_get; + /**< Get extended device statistics. */ + eventdev_xstats_get_names_t xstats_get_names; + /**< Get names of extended stats. */ + eventdev_xstats_get_by_name xstats_get_by_name; + /**< Get one value by name. */ + eventdev_xstats_reset_t xstats_reset; + /**< Reset the statistics values in xstats. */ + + eventdev_eth_rx_adapter_caps_get_t eth_rx_adapter_caps_get; + /**< Get ethernet Rx adapter capabilities */ + eventdev_eth_rx_adapter_queue_add_t eth_rx_adapter_queue_add; + /**< Add Rx queues to ethernet Rx adapter */ + eventdev_eth_rx_adapter_queue_del_t eth_rx_adapter_queue_del; + /**< Delete Rx queues from ethernet Rx adapter */ + eventdev_eth_rx_adapter_start_t eth_rx_adapter_start; + /**< Start ethernet Rx adapter */ + eventdev_eth_rx_adapter_stop_t eth_rx_adapter_stop; + /**< Stop ethernet Rx adapter */ + eventdev_eth_rx_adapter_stats_get eth_rx_adapter_stats_get; + /**< Get ethernet Rx stats */ + eventdev_eth_rx_adapter_stats_reset eth_rx_adapter_stats_reset; + /**< Reset ethernet Rx stats */ +}; + +/** + * Allocates a new eventdev slot for an event device and returns the pointer + * to that slot for the driver to use. + * + * @param name + * Unique identifier name for each device + * @param socket_id + * Socket to allocate resources on. + * @return + * - Slot in the rte_dev_devices array for a new device; + */ +struct rte_eventdev * +rte_event_pmd_allocate(const char *name, int socket_id); + +/** + * Release the specified eventdev device. + * + * @param eventdev + * The *eventdev* pointer is the address of the *rte_eventdev* structure. + * @return + * - 0 on success, negative on error + */ +int +rte_event_pmd_release(struct rte_eventdev *eventdev); + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_EVENTDEV_PMD_H_ */ diff --git a/dpdk/lib/librte_eventdev/rte_eventdev_pmd_pci.h b/dpdk/lib/librte_eventdev/rte_eventdev_pmd_pci.h new file mode 100644 index 00000000..8da40be0 --- /dev/null +++ b/dpdk/lib/librte_eventdev/rte_eventdev_pmd_pci.h @@ -0,0 +1,164 @@ +/* + * + * Copyright(c) 2016-2017 Cavium, Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_EVENTDEV_PMD_PCI_H_ +#define _RTE_EVENTDEV_PMD_PCI_H_ + +/** @file + * RTE Eventdev PCI PMD APIs + * + * @note + * These API are from event PCI PMD only and user applications should not call + * them directly. + */ + + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +#include +#include +#include +#include +#include + +#include "rte_eventdev_pmd.h" + +typedef int (*eventdev_pmd_pci_callback_t)(struct rte_eventdev *dev); + +/** + * @internal + * Wrapper for use by pci drivers as a .probe function to attach to a event + * interface. + */ +static inline int +rte_event_pmd_pci_probe(struct rte_pci_driver *pci_drv, + struct rte_pci_device *pci_dev, + size_t private_data_size, + eventdev_pmd_pci_callback_t devinit) +{ + struct rte_eventdev *eventdev; + + char eventdev_name[RTE_EVENTDEV_NAME_MAX_LEN]; + + int retval; + + if (devinit == NULL) + return -EINVAL; + + rte_pci_device_name(&pci_dev->addr, eventdev_name, + sizeof(eventdev_name)); + + eventdev = rte_event_pmd_allocate(eventdev_name, + pci_dev->device.numa_node); + if (eventdev == NULL) + return -ENOMEM; + + if (rte_eal_process_type() == RTE_PROC_PRIMARY) { + eventdev->data->dev_private = + rte_zmalloc_socket( + "eventdev private structure", + private_data_size, + RTE_CACHE_LINE_SIZE, + rte_socket_id()); + + if (eventdev->data->dev_private == NULL) + rte_panic("Cannot allocate memzone for private " + "device data"); + } + + eventdev->dev = &pci_dev->device; + + /* Invoke PMD device initialization function */ + retval = devinit(eventdev); + if (retval == 0) + return 0; + + RTE_EDEV_LOG_ERR("driver %s: (vendor_id=0x%x device_id=0x%x)" + " failed", pci_drv->driver.name, + (unsigned int) pci_dev->id.vendor_id, + (unsigned int) pci_dev->id.device_id); + + rte_event_pmd_release(eventdev); + + return -ENXIO; +} + + +/** + * @internal + * Wrapper for use by pci drivers as a .remove function to detach a event + * interface. + */ +static inline int +rte_event_pmd_pci_remove(struct rte_pci_device *pci_dev, + eventdev_pmd_pci_callback_t devuninit) +{ + struct rte_eventdev *eventdev; + char eventdev_name[RTE_EVENTDEV_NAME_MAX_LEN]; + int ret = 0; + + if (pci_dev == NULL) + return -EINVAL; + + rte_pci_device_name(&pci_dev->addr, eventdev_name, + sizeof(eventdev_name)); + + eventdev = rte_event_pmd_get_named_dev(eventdev_name); + if (eventdev == NULL) + return -ENODEV; + + ret = rte_event_dev_close(eventdev->data->dev_id); + if (ret < 0) + return ret; + + /* Invoke PMD device un-init function */ + if (devuninit) + ret = devuninit(eventdev); + if (ret) + return ret; + + /* Free event device */ + rte_event_pmd_release(eventdev); + + eventdev->dev = NULL; + + return 0; +} + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_EVENTDEV_PMD_PCI_H_ */ diff --git a/dpdk/lib/librte_eventdev/rte_eventdev_pmd_vdev.h b/dpdk/lib/librte_eventdev/rte_eventdev_pmd_vdev.h new file mode 100644 index 00000000..f77d59bd --- /dev/null +++ b/dpdk/lib/librte_eventdev/rte_eventdev_pmd_vdev.h @@ -0,0 +1,135 @@ +/* + * + * Copyright(c) 2016-2017 Cavium, Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_EVENTDEV_PMD_VDEV_H_ +#define _RTE_EVENTDEV_PMD_VDEV_H_ + +/** @file + * RTE Eventdev VDEV PMD APIs + * + * @note + * These API are from event VDEV PMD only and user applications should not call + * them directly. + */ + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +#include +#include +#include +#include + +#include "rte_eventdev_pmd.h" + +/** + * @internal + * Creates a new virtual event device and returns the pointer to that device. + * + * @param name + * PMD type name + * @param dev_private_size + * Size of event PMDs private data + * @param socket_id + * Socket to allocate resources on. + * + * @return + * - Eventdev pointer if device is successfully created. + * - NULL if device cannot be created. + */ +static inline struct rte_eventdev * +rte_event_pmd_vdev_init(const char *name, size_t dev_private_size, + int socket_id) +{ + + struct rte_eventdev *eventdev; + + /* Allocate device structure */ + eventdev = rte_event_pmd_allocate(name, socket_id); + if (eventdev == NULL) + return NULL; + + /* Allocate private device structure */ + if (rte_eal_process_type() == RTE_PROC_PRIMARY) { + eventdev->data->dev_private = + rte_zmalloc_socket("eventdev device private", + dev_private_size, + RTE_CACHE_LINE_SIZE, + socket_id); + + if (eventdev->data->dev_private == NULL) + rte_panic("Cannot allocate memzone for private device" + " data"); + } + + return eventdev; +} + +/** + * @internal + * Destroy the given virtual event device + * + * @param name + * PMD type name + * @return + * - 0 on success, negative on error + */ +static inline int +rte_event_pmd_vdev_uninit(const char *name) +{ + int ret; + struct rte_eventdev *eventdev; + + if (name == NULL) + return -EINVAL; + + eventdev = rte_event_pmd_get_named_dev(name); + if (eventdev == NULL) + return -ENODEV; + + ret = rte_event_dev_close(eventdev->data->dev_id); + if (ret < 0) + return ret; + + /* Free the event device */ + rte_event_pmd_release(eventdev); + + return 0; +} + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_EVENTDEV_PMD_VDEV_H_ */ diff --git a/dpdk/lib/librte_eventdev/rte_eventdev_version.map b/dpdk/lib/librte_eventdev/rte_eventdev_version.map new file mode 100644 index 00000000..108ae61f --- /dev/null +++ b/dpdk/lib/librte_eventdev/rte_eventdev_version.map @@ -0,0 +1,70 @@ +DPDK_17.05 { + global: + + rte_eventdevs; + + rte_event_dev_count; + rte_event_dev_get_dev_id; + rte_event_dev_socket_id; + rte_event_dev_info_get; + rte_event_dev_configure; + rte_event_dev_start; + rte_event_dev_stop; + rte_event_dev_close; + rte_event_dev_dump; + rte_event_dev_xstats_by_name_get; + rte_event_dev_xstats_get; + rte_event_dev_xstats_names_get; + rte_event_dev_xstats_reset; + + rte_event_port_default_conf_get; + rte_event_port_setup; + rte_event_port_link; + rte_event_port_unlink; + rte_event_port_links_get; + + rte_event_queue_default_conf_get; + rte_event_queue_setup; + + rte_event_dequeue_timeout_ticks; + + rte_event_pmd_allocate; + rte_event_pmd_release; + rte_event_pmd_vdev_init; + rte_event_pmd_vdev_uninit; + rte_event_pmd_pci_probe; + rte_event_pmd_pci_remove; + + local: *; +}; + +DPDK_17.08 { + global: + + rte_event_ring_create; + rte_event_ring_free; + rte_event_ring_init; + rte_event_ring_lookup; +} DPDK_17.05; + +DPDK_17.11 { + global: + + rte_event_dev_attr_get; + rte_event_dev_service_id_get; + rte_event_port_attr_get; + rte_event_queue_attr_get; + + rte_event_eth_rx_adapter_caps_get; + rte_event_eth_rx_adapter_create; + rte_event_eth_rx_adapter_create_ext; + rte_event_eth_rx_adapter_free; + rte_event_eth_rx_adapter_queue_add; + rte_event_eth_rx_adapter_queue_del; + rte_event_eth_rx_adapter_service_id_get; + rte_event_eth_rx_adapter_start; + rte_event_eth_rx_adapter_stats_get; + rte_event_eth_rx_adapter_stats_reset; + rte_event_eth_rx_adapter_stop; + +} DPDK_17.08; diff --git a/dpdk/lib/librte_flow_classify/Makefile b/dpdk/lib/librte_flow_classify/Makefile new file mode 100644 index 00000000..ea792f5d --- /dev/null +++ b/dpdk/lib/librte_flow_classify/Makefile @@ -0,0 +1,53 @@ +# BSD LICENSE +# +# Copyright(c) 2017 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +include $(RTE_SDK)/mk/rte.vars.mk + +# library name +LIB = librte_flow_classify.a + +CFLAGS += -O3 +CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) + +EXPORT_MAP := rte_flow_classify_version.map + +LIBABIVER := 1 + +LDLIBS += -lrte_eal -lrte_ethdev -lrte_net -lrte_table -lrte_acl + +# all source are stored in SRCS-y +SRCS-$(CONFIG_RTE_LIBRTE_FLOW_CLASSIFY) += rte_flow_classify.c +SRCS-$(CONFIG_RTE_LIBRTE_FLOW_CLASSIFY) += rte_flow_classify_parse.c + +# install this header file +SYMLINK-$(CONFIG_RTE_LIBRTE_FLOW_CLASSIFY)-include := rte_flow_classify.h + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/lib/librte_flow_classify/rte_flow_classify.c b/dpdk/lib/librte_flow_classify/rte_flow_classify.c new file mode 100644 index 00000000..e6f44864 --- /dev/null +++ b/dpdk/lib/librte_flow_classify/rte_flow_classify.c @@ -0,0 +1,691 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include "rte_flow_classify_parse.h" +#include +#include +#include + +int librte_flow_classify_logtype; + +static struct rte_eth_ntuple_filter ntuple_filter; +static uint32_t unique_id = 1; + + +struct rte_flow_classify_table_entry { + /* meta-data for classify rule */ + uint32_t rule_id; +}; + +struct rte_table { + /* Input parameters */ + struct rte_table_ops ops; + uint32_t entry_size; + enum rte_flow_classify_table_type type; + + /* Handle to the low-level table object */ + void *h_table; +}; + +#define RTE_FLOW_CLASSIFIER_MAX_NAME_SZ 256 + +struct rte_flow_classifier { + /* Input parameters */ + char name[RTE_FLOW_CLASSIFIER_MAX_NAME_SZ]; + int socket_id; + enum rte_flow_classify_table_type type; + + /* Internal tables */ + struct rte_table tables[RTE_FLOW_CLASSIFY_TABLE_MAX]; + uint32_t num_tables; + uint16_t nb_pkts; + struct rte_flow_classify_table_entry + *entries[RTE_PORT_IN_BURST_SIZE_MAX]; +} __rte_cache_aligned; + +enum { + PROTO_FIELD_IPV4, + SRC_FIELD_IPV4, + DST_FIELD_IPV4, + SRCP_FIELD_IPV4, + DSTP_FIELD_IPV4, + NUM_FIELDS_IPV4 +}; + +struct acl_keys { + struct rte_table_acl_rule_add_params key_add; /* add key */ + struct rte_table_acl_rule_delete_params key_del; /* delete key */ +}; + +struct classify_rules { + enum rte_flow_classify_rule_type type; + union { + struct rte_flow_classify_ipv4_5tuple ipv4_5tuple; + } u; +}; + +struct rte_flow_classify_rule { + uint32_t id; /* unique ID of classify rule */ + struct rte_flow_action action; /* action when match found */ + struct classify_rules rules; /* union of rules */ + union { + struct acl_keys key; + } u; + int key_found; /* rule key found in table */ + void *entry; /* pointer to buffer to hold rule meta data */ + void *entry_ptr; /* handle to the table entry for rule meta data */ +}; + +static int +flow_classify_parse_flow( + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error) +{ + struct rte_flow_item *items; + parse_filter_t parse_filter; + uint32_t item_num = 0; + uint32_t i = 0; + int ret; + + memset(&ntuple_filter, 0, sizeof(ntuple_filter)); + + /* Get the non-void item number of pattern */ + while ((pattern + i)->type != RTE_FLOW_ITEM_TYPE_END) { + if ((pattern + i)->type != RTE_FLOW_ITEM_TYPE_VOID) + item_num++; + i++; + } + item_num++; + + items = malloc(item_num * sizeof(struct rte_flow_item)); + if (!items) { + rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_ITEM_NUM, + NULL, "No memory for pattern items."); + return -ENOMEM; + } + + memset(items, 0, item_num * sizeof(struct rte_flow_item)); + classify_pattern_skip_void_item(items, pattern); + + parse_filter = classify_find_parse_filter_func(items); + if (!parse_filter) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + pattern, "Unsupported pattern"); + free(items); + return -EINVAL; + } + + ret = parse_filter(attr, items, actions, &ntuple_filter, error); + free(items); + return ret; +} + + +#define uint32_t_to_char(ip, a, b, c, d) do {\ + *a = (unsigned char)(ip >> 24 & 0xff);\ + *b = (unsigned char)(ip >> 16 & 0xff);\ + *c = (unsigned char)(ip >> 8 & 0xff);\ + *d = (unsigned char)(ip & 0xff);\ + } while (0) + +static inline void +print_acl_ipv4_key_add(struct rte_table_acl_rule_add_params *key) +{ + unsigned char a, b, c, d; + + printf("%s: 0x%02hhx/0x%hhx ", __func__, + key->field_value[PROTO_FIELD_IPV4].value.u8, + key->field_value[PROTO_FIELD_IPV4].mask_range.u8); + + uint32_t_to_char(key->field_value[SRC_FIELD_IPV4].value.u32, + &a, &b, &c, &d); + printf(" %hhu.%hhu.%hhu.%hhu/0x%x ", a, b, c, d, + key->field_value[SRC_FIELD_IPV4].mask_range.u32); + + uint32_t_to_char(key->field_value[DST_FIELD_IPV4].value.u32, + &a, &b, &c, &d); + printf("%hhu.%hhu.%hhu.%hhu/0x%x ", a, b, c, d, + key->field_value[DST_FIELD_IPV4].mask_range.u32); + + printf("%hu : 0x%x %hu : 0x%x", + key->field_value[SRCP_FIELD_IPV4].value.u16, + key->field_value[SRCP_FIELD_IPV4].mask_range.u16, + key->field_value[DSTP_FIELD_IPV4].value.u16, + key->field_value[DSTP_FIELD_IPV4].mask_range.u16); + + printf(" priority: 0x%x\n", key->priority); +} + +static inline void +print_acl_ipv4_key_delete(struct rte_table_acl_rule_delete_params *key) +{ + unsigned char a, b, c, d; + + printf("%s: 0x%02hhx/0x%hhx ", __func__, + key->field_value[PROTO_FIELD_IPV4].value.u8, + key->field_value[PROTO_FIELD_IPV4].mask_range.u8); + + uint32_t_to_char(key->field_value[SRC_FIELD_IPV4].value.u32, + &a, &b, &c, &d); + printf(" %hhu.%hhu.%hhu.%hhu/0x%x ", a, b, c, d, + key->field_value[SRC_FIELD_IPV4].mask_range.u32); + + uint32_t_to_char(key->field_value[DST_FIELD_IPV4].value.u32, + &a, &b, &c, &d); + printf("%hhu.%hhu.%hhu.%hhu/0x%x ", a, b, c, d, + key->field_value[DST_FIELD_IPV4].mask_range.u32); + + printf("%hu : 0x%x %hu : 0x%x\n", + key->field_value[SRCP_FIELD_IPV4].value.u16, + key->field_value[SRCP_FIELD_IPV4].mask_range.u16, + key->field_value[DSTP_FIELD_IPV4].value.u16, + key->field_value[DSTP_FIELD_IPV4].mask_range.u16); +} + +static int +rte_flow_classifier_check_params(struct rte_flow_classifier_params *params) +{ + if (params == NULL) { + RTE_FLOW_CLASSIFY_LOG(ERR, + "%s: Incorrect value for parameter params\n", __func__); + return -EINVAL; + } + + /* name */ + if (params->name == NULL) { + RTE_FLOW_CLASSIFY_LOG(ERR, + "%s: Incorrect value for parameter name\n", __func__); + return -EINVAL; + } + + /* socket */ + if ((params->socket_id < 0) || + (params->socket_id >= RTE_MAX_NUMA_NODES)) { + RTE_FLOW_CLASSIFY_LOG(ERR, + "%s: Incorrect value for parameter socket_id\n", + __func__); + return -EINVAL; + } + + return 0; +} + +struct rte_flow_classifier * +rte_flow_classifier_create(struct rte_flow_classifier_params *params) +{ + struct rte_flow_classifier *cls; + int ret; + + /* Check input parameters */ + ret = rte_flow_classifier_check_params(params); + if (ret != 0) { + RTE_FLOW_CLASSIFY_LOG(ERR, + "%s: flow classifier params check failed (%d)\n", + __func__, ret); + return NULL; + } + + /* Allocate memory for the flow classifier */ + cls = rte_zmalloc_socket("FLOW_CLASSIFIER", + sizeof(struct rte_flow_classifier), + RTE_CACHE_LINE_SIZE, params->socket_id); + + if (cls == NULL) { + RTE_FLOW_CLASSIFY_LOG(ERR, + "%s: flow classifier memory allocation failed\n", + __func__); + return NULL; + } + + /* Save input parameters */ + snprintf(cls->name, RTE_FLOW_CLASSIFIER_MAX_NAME_SZ, "%s", + params->name); + cls->socket_id = params->socket_id; + cls->type = params->type; + + /* Initialize flow classifier internal data structure */ + cls->num_tables = 0; + + return cls; +} + +static void +rte_flow_classify_table_free(struct rte_table *table) +{ + if (table->ops.f_free != NULL) + table->ops.f_free(table->h_table); +} + +int +rte_flow_classifier_free(struct rte_flow_classifier *cls) +{ + uint32_t i; + + /* Check input parameters */ + if (cls == NULL) { + RTE_FLOW_CLASSIFY_LOG(ERR, + "%s: rte_flow_classifier parameter is NULL\n", + __func__); + return -EINVAL; + } + + /* Free tables */ + for (i = 0; i < cls->num_tables; i++) { + struct rte_table *table = &cls->tables[i]; + + rte_flow_classify_table_free(table); + } + + /* Free flow classifier memory */ + rte_free(cls); + + return 0; +} + +static int +rte_table_check_params(struct rte_flow_classifier *cls, + struct rte_flow_classify_table_params *params, + uint32_t *table_id) +{ + if (cls == NULL) { + RTE_FLOW_CLASSIFY_LOG(ERR, + "%s: flow classifier parameter is NULL\n", + __func__); + return -EINVAL; + } + if (params == NULL) { + RTE_FLOW_CLASSIFY_LOG(ERR, "%s: params parameter is NULL\n", + __func__); + return -EINVAL; + } + if (table_id == NULL) { + RTE_FLOW_CLASSIFY_LOG(ERR, "%s: table_id parameter is NULL\n", + __func__); + return -EINVAL; + } + + /* ops */ + if (params->ops == NULL) { + RTE_FLOW_CLASSIFY_LOG(ERR, "%s: params->ops is NULL\n", + __func__); + return -EINVAL; + } + + if (params->ops->f_create == NULL) { + RTE_FLOW_CLASSIFY_LOG(ERR, + "%s: f_create function pointer is NULL\n", __func__); + return -EINVAL; + } + + if (params->ops->f_lookup == NULL) { + RTE_FLOW_CLASSIFY_LOG(ERR, + "%s: f_lookup function pointer is NULL\n", __func__); + return -EINVAL; + } + + /* De we have room for one more table? */ + if (cls->num_tables == RTE_FLOW_CLASSIFY_TABLE_MAX) { + RTE_FLOW_CLASSIFY_LOG(ERR, + "%s: Incorrect value for num_tables parameter\n", + __func__); + return -EINVAL; + } + + return 0; +} + +int +rte_flow_classify_table_create(struct rte_flow_classifier *cls, + struct rte_flow_classify_table_params *params, + uint32_t *table_id) +{ + struct rte_table *table; + void *h_table; + uint32_t entry_size, id; + int ret; + + /* Check input arguments */ + ret = rte_table_check_params(cls, params, table_id); + if (ret != 0) + return ret; + + id = cls->num_tables; + table = &cls->tables[id]; + + /* calculate table entry size */ + entry_size = sizeof(struct rte_flow_classify_table_entry); + + /* Create the table */ + h_table = params->ops->f_create(params->arg_create, cls->socket_id, + entry_size); + if (h_table == NULL) { + RTE_FLOW_CLASSIFY_LOG(ERR, "%s: Table creation failed\n", + __func__); + return -EINVAL; + } + + /* Commit current table to the classifier */ + cls->num_tables++; + *table_id = id; + + /* Save input parameters */ + memcpy(&table->ops, params->ops, sizeof(struct rte_table_ops)); + + /* Initialize table internal data structure */ + table->entry_size = entry_size; + table->h_table = h_table; + + return 0; +} + +static struct rte_flow_classify_rule * +allocate_acl_ipv4_5tuple_rule(void) +{ + struct rte_flow_classify_rule *rule; + int log_level; + + rule = malloc(sizeof(struct rte_flow_classify_rule)); + if (!rule) + return rule; + + memset(rule, 0, sizeof(struct rte_flow_classify_rule)); + rule->id = unique_id++; + rule->rules.type = RTE_FLOW_CLASSIFY_RULE_TYPE_IPV4_5TUPLE; + + memcpy(&rule->action, classify_get_flow_action(), + sizeof(struct rte_flow_action)); + + /* key add values */ + rule->u.key.key_add.priority = ntuple_filter.priority; + rule->u.key.key_add.field_value[PROTO_FIELD_IPV4].mask_range.u8 = + ntuple_filter.proto_mask; + rule->u.key.key_add.field_value[PROTO_FIELD_IPV4].value.u8 = + ntuple_filter.proto; + rule->rules.u.ipv4_5tuple.proto = ntuple_filter.proto; + rule->rules.u.ipv4_5tuple.proto_mask = ntuple_filter.proto_mask; + + rule->u.key.key_add.field_value[SRC_FIELD_IPV4].mask_range.u32 = + ntuple_filter.src_ip_mask; + rule->u.key.key_add.field_value[SRC_FIELD_IPV4].value.u32 = + ntuple_filter.src_ip; + rule->rules.u.ipv4_5tuple.src_ip_mask = ntuple_filter.src_ip_mask; + rule->rules.u.ipv4_5tuple.src_ip = ntuple_filter.src_ip; + + rule->u.key.key_add.field_value[DST_FIELD_IPV4].mask_range.u32 = + ntuple_filter.dst_ip_mask; + rule->u.key.key_add.field_value[DST_FIELD_IPV4].value.u32 = + ntuple_filter.dst_ip; + rule->rules.u.ipv4_5tuple.dst_ip_mask = ntuple_filter.dst_ip_mask; + rule->rules.u.ipv4_5tuple.dst_ip = ntuple_filter.dst_ip; + + rule->u.key.key_add.field_value[SRCP_FIELD_IPV4].mask_range.u16 = + ntuple_filter.src_port_mask; + rule->u.key.key_add.field_value[SRCP_FIELD_IPV4].value.u16 = + ntuple_filter.src_port; + rule->rules.u.ipv4_5tuple.src_port_mask = ntuple_filter.src_port_mask; + rule->rules.u.ipv4_5tuple.src_port = ntuple_filter.src_port; + + rule->u.key.key_add.field_value[DSTP_FIELD_IPV4].mask_range.u16 = + ntuple_filter.dst_port_mask; + rule->u.key.key_add.field_value[DSTP_FIELD_IPV4].value.u16 = + ntuple_filter.dst_port; + rule->rules.u.ipv4_5tuple.dst_port_mask = ntuple_filter.dst_port_mask; + rule->rules.u.ipv4_5tuple.dst_port = ntuple_filter.dst_port; + + log_level = rte_log_get_level(librte_flow_classify_logtype); + + if (log_level == RTE_LOG_DEBUG) + print_acl_ipv4_key_add(&rule->u.key.key_add); + + /* key delete values */ + memcpy(&rule->u.key.key_del.field_value[PROTO_FIELD_IPV4], + &rule->u.key.key_add.field_value[PROTO_FIELD_IPV4], + NUM_FIELDS_IPV4 * sizeof(struct rte_acl_field)); + + if (log_level == RTE_LOG_DEBUG) + print_acl_ipv4_key_delete(&rule->u.key.key_del); + + return rule; +} + +struct rte_flow_classify_rule * +rte_flow_classify_table_entry_add(struct rte_flow_classifier *cls, + uint32_t table_id, + int *key_found, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error) +{ + struct rte_flow_classify_rule *rule; + struct rte_flow_classify_table_entry *table_entry; + int ret; + + if (!error) + return NULL; + + if (!cls) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "NULL classifier."); + return NULL; + } + + if (table_id >= cls->num_tables) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "invalid table_id."); + return NULL; + } + + if (key_found == NULL) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "NULL key_found."); + return NULL; + } + + if (!pattern) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM_NUM, + NULL, "NULL pattern."); + return NULL; + } + + if (!actions) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION_NUM, + NULL, "NULL action."); + return NULL; + } + + if (!attr) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR, + NULL, "NULL attribute."); + return NULL; + } + + /* parse attr, pattern and actions */ + ret = flow_classify_parse_flow(attr, pattern, actions, error); + if (ret < 0) + return NULL; + + switch (cls->type) { + case RTE_FLOW_CLASSIFY_TABLE_TYPE_ACL: + rule = allocate_acl_ipv4_5tuple_rule(); + if (!rule) + return NULL; + break; + default: + return NULL; + } + + rule->entry = malloc(sizeof(struct rte_flow_classify_table_entry)); + if (!rule->entry) { + free(rule); + return NULL; + } + + table_entry = rule->entry; + table_entry->rule_id = rule->id; + + if (cls->tables[table_id].ops.f_add != NULL) { + ret = cls->tables[table_id].ops.f_add( + cls->tables[table_id].h_table, + &rule->u.key.key_add, + rule->entry, + &rule->key_found, + &rule->entry_ptr); + if (ret) { + free(rule->entry); + free(rule); + return NULL; + } + *key_found = rule->key_found; + } + return rule; +} + +int +rte_flow_classify_table_entry_delete(struct rte_flow_classifier *cls, + uint32_t table_id, + struct rte_flow_classify_rule *rule) +{ + int ret = -EINVAL; + + if (!cls || !rule || table_id >= cls->num_tables) + return ret; + + if (cls->tables[table_id].ops.f_delete != NULL) + ret = cls->tables[table_id].ops.f_delete( + cls->tables[table_id].h_table, + &rule->u.key.key_del, + &rule->key_found, + &rule->entry); + + return ret; +} + +static int +flow_classifier_lookup(struct rte_flow_classifier *cls, + uint32_t table_id, + struct rte_mbuf **pkts, + const uint16_t nb_pkts) +{ + int ret = -EINVAL; + uint64_t pkts_mask; + uint64_t lookup_hit_mask; + + pkts_mask = RTE_LEN2MASK(nb_pkts, uint64_t); + ret = cls->tables[table_id].ops.f_lookup( + cls->tables[table_id].h_table, + pkts, pkts_mask, &lookup_hit_mask, + (void **)cls->entries); + + if (!ret && lookup_hit_mask) + cls->nb_pkts = nb_pkts; + else + cls->nb_pkts = 0; + + return ret; +} + +static int +action_apply(struct rte_flow_classifier *cls, + struct rte_flow_classify_rule *rule, + struct rte_flow_classify_stats *stats) +{ + struct rte_flow_classify_ipv4_5tuple_stats *ntuple_stats; + uint64_t count = 0; + int i; + int ret = -EINVAL; + + switch (rule->action.type) { + case RTE_FLOW_ACTION_TYPE_COUNT: + for (i = 0; i < cls->nb_pkts; i++) { + if (rule->id == cls->entries[i]->rule_id) + count++; + } + if (count) { + ret = 0; + ntuple_stats = + (struct rte_flow_classify_ipv4_5tuple_stats *) + stats->stats; + ntuple_stats->counter1 = count; + ntuple_stats->ipv4_5tuple = rule->rules.u.ipv4_5tuple; + } + break; + default: + ret = -ENOTSUP; + break; + } + + return ret; +} + +int +rte_flow_classifier_query(struct rte_flow_classifier *cls, + uint32_t table_id, + struct rte_mbuf **pkts, + const uint16_t nb_pkts, + struct rte_flow_classify_rule *rule, + struct rte_flow_classify_stats *stats) +{ + int ret = -EINVAL; + + if (!cls || !rule || !stats || !pkts || nb_pkts == 0 || + table_id >= cls->num_tables) + return ret; + + ret = flow_classifier_lookup(cls, table_id, pkts, nb_pkts); + if (!ret) + ret = action_apply(cls, rule, stats); + return ret; +} + +RTE_INIT(librte_flow_classify_init_log); + +static void +librte_flow_classify_init_log(void) +{ + librte_flow_classify_logtype = + rte_log_register("librte.flow_classify"); + if (librte_flow_classify_logtype >= 0) + rte_log_set_level(librte_flow_classify_logtype, RTE_LOG_INFO); +} diff --git a/dpdk/lib/librte_flow_classify/rte_flow_classify.h b/dpdk/lib/librte_flow_classify/rte_flow_classify.h new file mode 100644 index 00000000..e14bc787 --- /dev/null +++ b/dpdk/lib/librte_flow_classify/rte_flow_classify.h @@ -0,0 +1,293 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_FLOW_CLASSIFY_H_ +#define _RTE_FLOW_CLASSIFY_H_ + +/** + * @file + * + * RTE Flow Classify Library + * + * @b EXPERIMENTAL: this API may change without prior notice + * + * This library provides flow record information with some measured properties. + * + * Application should define the flow and measurement criteria (action) for it. + * + * The Library doesn't maintain any flow records itself, instead flow + * information is returned to upper layer only for given packets. + * + * It is application's responsibility to call rte_flow_classifier_query() + * for a burst of packets, just after receiving them or before transmitting + * them. + * Application should provide the flow type interested in, measurement to apply + * to that flow in rte_flow_classify_table_entry_add() API, and should provide + * the rte_flow_classifier object and storage to put results in for the + * rte_flow_classifier_query() API. + * + * Usage: + * - application calls rte_flow_classifier_create() to create an + * rte_flow_classifier object. + * - application calls rte_flow_classify_table_create() to create a table + * in the rte_flow_classifier object. + * - application calls rte_flow_classify_table_entry_add() to add a rule to + * the table in the rte_flow_classifier object. + * - application calls rte_flow_classifier_query() in a polling manner, + * preferably after rte_eth_rx_burst(). This will cause the library to + * match packet information to flow information with some measurements. + * - rte_flow_classifier object can be destroyed when it is no longer needed + * with rte_flow_classifier_free() + */ + +#include +#include +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +extern int librte_flow_classify_logtype; + +#define RTE_FLOW_CLASSIFY_LOG(level, ...) \ + rte_log(RTE_LOG_ ## level, \ + librte_flow_classify_logtype, \ + RTE_FMT("%s(): " RTE_FMT_HEAD(__VA_ARGS__,), \ + __func__, \ + RTE_FMT_TAIL(__VA_ARGS__,))) + +/** Opaque data type for flow classifier */ +struct rte_flow_classifier; + +/** Opaque data type for flow classify rule */ +struct rte_flow_classify_rule; + +/** Flow classify rule type */ +enum rte_flow_classify_rule_type { + /** no type */ + RTE_FLOW_CLASSIFY_RULE_TYPE_NONE, + /** IPv4 5tuple type */ + RTE_FLOW_CLASSIFY_RULE_TYPE_IPV4_5TUPLE, +}; + +/** Flow classify table type */ +enum rte_flow_classify_table_type { + /** no type */ + RTE_FLOW_CLASSIFY_TABLE_TYPE_NONE, + /** ACL type */ + RTE_FLOW_CLASSIFY_TABLE_TYPE_ACL, +}; + +/** + * Maximum number of tables allowed for any Flow Classifier instance. + * The value of this parameter cannot be changed. + */ +#define RTE_FLOW_CLASSIFY_TABLE_MAX 64 + +/** Parameters for flow classifier creation */ +struct rte_flow_classifier_params { + /** flow classifier name */ + const char *name; + + /** CPU socket ID where memory for the flow classifier and its */ + /** elements (tables) should be allocated */ + int socket_id; + + /** Table type */ + enum rte_flow_classify_table_type type; +}; + +/** Parameters for table creation */ +struct rte_flow_classify_table_params { + /** Table operations (specific to each table type) */ + struct rte_table_ops *ops; + + /** Opaque param to be passed to the table create operation */ + void *arg_create; +}; + +/** IPv4 5-tuple data */ +struct rte_flow_classify_ipv4_5tuple { + uint32_t dst_ip; /**< Destination IP address in big endian. */ + uint32_t dst_ip_mask; /**< Mask of destination IP address. */ + uint32_t src_ip; /**< Source IP address in big endian. */ + uint32_t src_ip_mask; /**< Mask of destination IP address. */ + uint16_t dst_port; /**< Destination port in big endian. */ + uint16_t dst_port_mask; /**< Mask of destination port. */ + uint16_t src_port; /**< Source Port in big endian. */ + uint16_t src_port_mask; /**< Mask of source port. */ + uint8_t proto; /**< L4 protocol. */ + uint8_t proto_mask; /**< Mask of L4 protocol. */ +}; + +/** + * Flow stats + * + * For the count action, stats can be returned by the query API. + * + * Storage for stats is provided by application. + */ +struct rte_flow_classify_stats { + void *stats; +}; + +struct rte_flow_classify_ipv4_5tuple_stats { + /** count of packets that match IPv4 5tuple pattern */ + uint64_t counter1; + /** IPv4 5tuple data */ + struct rte_flow_classify_ipv4_5tuple ipv4_5tuple; +}; + +/** + * Flow classifier create + * + * @param params + * Parameters for flow classifier creation + * @return + * Handle to flow classifier instance on success or NULL otherwise + */ +struct rte_flow_classifier * +rte_flow_classifier_create(struct rte_flow_classifier_params *params); + +/** + * Flow classifier free + * + * @param cls + * Handle to flow classifier instance + * @return + * 0 on success, error code otherwise + */ +int +rte_flow_classifier_free(struct rte_flow_classifier *cls); + +/** + * Flow classify table create + * + * @param cls + * Handle to flow classifier instance + * @param params + * Parameters for flow_classify table creation + * @param table_id + * Table ID. Valid only within the scope of table IDs of the current + * classifier. Only returned after a successful invocation. + * @return + * 0 on success, error code otherwise + */ +int +rte_flow_classify_table_create(struct rte_flow_classifier *cls, + struct rte_flow_classify_table_params *params, + uint32_t *table_id); + +/** + * Add a flow classify rule to the flow_classifer table. + * + * @param[in] cls + * Flow classifier handle + * @param[in] table_id + * id of table + * @param[out] key_found + * returns 1 if key present already, 0 otherwise. + * @param[in] attr + * Flow rule attributes + * @param[in] pattern + * Pattern specification (list terminated by the END pattern item). + * @param[in] actions + * Associated actions (list terminated by the END pattern item). + * @param[out] error + * Perform verbose error reporting if not NULL. Structure + * initialised in case of error only. + * @return + * A valid handle in case of success, NULL otherwise. + */ +struct rte_flow_classify_rule * +rte_flow_classify_table_entry_add(struct rte_flow_classifier *cls, + uint32_t table_id, + int *key_found, + const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_flow_error *error); + +/** + * Delete a flow classify rule from the flow_classifer table. + * + * @param[in] cls + * Flow classifier handle + * @param[in] table_id + * id of table + * @param[in] rule + * Flow classify rule + * @return + * 0 on success, error code otherwise. + */ +int +rte_flow_classify_table_entry_delete(struct rte_flow_classifier *cls, + uint32_t table_id, + struct rte_flow_classify_rule *rule); + +/** + * Query flow classifier for given rule. + * + * @param[in] cls + * Flow classifier handle + * @param[in] table_id + * id of table + * @param[in] pkts + * Pointer to packets to process + * @param[in] nb_pkts + * Number of packets to process + * @param[in] rule + * Flow classify rule + * @param[in] stats + * Flow classify stats + * + * @return + * 0 on success, error code otherwise. + */ +int +rte_flow_classifier_query(struct rte_flow_classifier *cls, + uint32_t table_id, + struct rte_mbuf **pkts, + const uint16_t nb_pkts, + struct rte_flow_classify_rule *rule, + struct rte_flow_classify_stats *stats); + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_FLOW_CLASSIFY_H_ */ diff --git a/dpdk/lib/librte_flow_classify/rte_flow_classify_parse.c b/dpdk/lib/librte_flow_classify/rte_flow_classify_parse.c new file mode 100644 index 00000000..dbfa1115 --- /dev/null +++ b/dpdk/lib/librte_flow_classify/rte_flow_classify_parse.c @@ -0,0 +1,546 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include "rte_flow_classify_parse.h" +#include + +struct classify_valid_pattern { + enum rte_flow_item_type *items; + parse_filter_t parse_filter; +}; + +static struct rte_flow_action action; + +/* Pattern for IPv4 5-tuple UDP filter */ +static enum rte_flow_item_type pattern_ntuple_1[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_END, +}; + +/* Pattern for IPv4 5-tuple TCP filter */ +static enum rte_flow_item_type pattern_ntuple_2[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_TCP, + RTE_FLOW_ITEM_TYPE_END, +}; + +/* Pattern for IPv4 5-tuple SCTP filter */ +static enum rte_flow_item_type pattern_ntuple_3[] = { + RTE_FLOW_ITEM_TYPE_ETH, + RTE_FLOW_ITEM_TYPE_IPV4, + RTE_FLOW_ITEM_TYPE_SCTP, + RTE_FLOW_ITEM_TYPE_END, +}; + +static int +classify_parse_ntuple_filter(const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_eth_ntuple_filter *filter, + struct rte_flow_error *error); + +static struct classify_valid_pattern classify_supported_patterns[] = { + /* ntuple */ + { pattern_ntuple_1, classify_parse_ntuple_filter }, + { pattern_ntuple_2, classify_parse_ntuple_filter }, + { pattern_ntuple_3, classify_parse_ntuple_filter }, +}; + +struct rte_flow_action * +classify_get_flow_action(void) +{ + return &action; +} + +/* Find the first VOID or non-VOID item pointer */ +const struct rte_flow_item * +classify_find_first_item(const struct rte_flow_item *item, bool is_void) +{ + bool is_find; + + while (item->type != RTE_FLOW_ITEM_TYPE_END) { + if (is_void) + is_find = item->type == RTE_FLOW_ITEM_TYPE_VOID; + else + is_find = item->type != RTE_FLOW_ITEM_TYPE_VOID; + if (is_find) + break; + item++; + } + return item; +} + +/* Skip all VOID items of the pattern */ +void +classify_pattern_skip_void_item(struct rte_flow_item *items, + const struct rte_flow_item *pattern) +{ + uint32_t cpy_count = 0; + const struct rte_flow_item *pb = pattern, *pe = pattern; + + for (;;) { + /* Find a non-void item first */ + pb = classify_find_first_item(pb, false); + if (pb->type == RTE_FLOW_ITEM_TYPE_END) { + pe = pb; + break; + } + + /* Find a void item */ + pe = classify_find_first_item(pb + 1, true); + + cpy_count = pe - pb; + rte_memcpy(items, pb, sizeof(struct rte_flow_item) * cpy_count); + + items += cpy_count; + + if (pe->type == RTE_FLOW_ITEM_TYPE_END) { + pb = pe; + break; + } + + pb = pe + 1; + } + /* Copy the END item. */ + rte_memcpy(items, pe, sizeof(struct rte_flow_item)); +} + +/* Check if the pattern matches a supported item type array */ +static bool +classify_match_pattern(enum rte_flow_item_type *item_array, + struct rte_flow_item *pattern) +{ + struct rte_flow_item *item = pattern; + + while ((*item_array == item->type) && + (*item_array != RTE_FLOW_ITEM_TYPE_END)) { + item_array++; + item++; + } + + return (*item_array == RTE_FLOW_ITEM_TYPE_END && + item->type == RTE_FLOW_ITEM_TYPE_END); +} + +/* Find if there's parse filter function matched */ +parse_filter_t +classify_find_parse_filter_func(struct rte_flow_item *pattern) +{ + parse_filter_t parse_filter = NULL; + uint8_t i = 0; + + for (; i < RTE_DIM(classify_supported_patterns); i++) { + if (classify_match_pattern(classify_supported_patterns[i].items, + pattern)) { + parse_filter = + classify_supported_patterns[i].parse_filter; + break; + } + } + + return parse_filter; +} + +#define FLOW_RULE_MIN_PRIORITY 8 +#define FLOW_RULE_MAX_PRIORITY 0 + +#define NEXT_ITEM_OF_PATTERN(item, pattern, index)\ + do {\ + item = pattern + index;\ + while (item->type == RTE_FLOW_ITEM_TYPE_VOID) {\ + index++;\ + item = pattern + index;\ + } \ + } while (0) + +#define NEXT_ITEM_OF_ACTION(act, actions, index)\ + do {\ + act = actions + index;\ + while (act->type == RTE_FLOW_ACTION_TYPE_VOID) {\ + index++;\ + act = actions + index;\ + } \ + } while (0) + +/** + * Please aware there's an assumption for all the parsers. + * rte_flow_item is using big endian, rte_flow_attr and + * rte_flow_action are using CPU order. + * Because the pattern is used to describe the packets, + * normally the packets should use network order. + */ + +/** + * Parse the rule to see if it is a n-tuple rule. + * And get the n-tuple filter info BTW. + * pattern: + * The first not void item can be ETH or IPV4. + * The second not void item must be IPV4 if the first one is ETH. + * The third not void item must be UDP or TCP. + * The next not void item must be END. + * action: + * The first not void action should be QUEUE. + * The next not void action should be END. + * pattern example: + * ITEM Spec Mask + * ETH NULL NULL + * IPV4 src_addr 192.168.1.20 0xFFFFFFFF + * dst_addr 192.167.3.50 0xFFFFFFFF + * next_proto_id 17 0xFF + * UDP/TCP/ src_port 80 0xFFFF + * SCTP dst_port 80 0xFFFF + * END + * other members in mask and spec should set to 0x00. + * item->last should be NULL. + */ +static int +classify_parse_ntuple_filter(const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_eth_ntuple_filter *filter, + struct rte_flow_error *error) +{ + const struct rte_flow_item *item; + const struct rte_flow_action *act; + const struct rte_flow_item_ipv4 *ipv4_spec; + const struct rte_flow_item_ipv4 *ipv4_mask; + const struct rte_flow_item_tcp *tcp_spec; + const struct rte_flow_item_tcp *tcp_mask; + const struct rte_flow_item_udp *udp_spec; + const struct rte_flow_item_udp *udp_mask; + const struct rte_flow_item_sctp *sctp_spec; + const struct rte_flow_item_sctp *sctp_mask; + uint32_t index; + + if (!pattern) { + rte_flow_error_set(error, + EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_NUM, + NULL, "NULL pattern."); + return -EINVAL; + } + + if (!actions) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION_NUM, + NULL, "NULL action."); + return -EINVAL; + } + if (!attr) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR, + NULL, "NULL attribute."); + return -EINVAL; + } + + /* parse pattern */ + index = 0; + + /* the first not void item can be MAC or IPv4 */ + NEXT_ITEM_OF_PATTERN(item, pattern, index); + + if (item->type != RTE_FLOW_ITEM_TYPE_ETH && + item->type != RTE_FLOW_ITEM_TYPE_IPV4) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by ntuple filter"); + return -EINVAL; + } + /* Skip Ethernet */ + if (item->type == RTE_FLOW_ITEM_TYPE_ETH) { + /*Not supported last point for range*/ + if (item->last) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + item, + "Not supported last point for range"); + return -EINVAL; + + } + /* if the first item is MAC, the content should be NULL */ + if (item->spec || item->mask) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Not supported by ntuple filter"); + return -EINVAL; + } + /* check if the next not void item is IPv4 */ + index++; + NEXT_ITEM_OF_PATTERN(item, pattern, index); + if (item->type != RTE_FLOW_ITEM_TYPE_IPV4) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "Not supported by ntuple filter"); + return -EINVAL; + } + } + + /* get the IPv4 info */ + if (!item->spec || !item->mask) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Invalid ntuple mask"); + return -EINVAL; + } + /*Not supported last point for range*/ + if (item->last) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + item, "Not supported last point for range"); + return -EINVAL; + + } + + ipv4_mask = (const struct rte_flow_item_ipv4 *)item->mask; + /** + * Only support src & dst addresses, protocol, + * others should be masked. + */ + if (ipv4_mask->hdr.version_ihl || + ipv4_mask->hdr.type_of_service || + ipv4_mask->hdr.total_length || + ipv4_mask->hdr.packet_id || + ipv4_mask->hdr.fragment_offset || + ipv4_mask->hdr.time_to_live || + ipv4_mask->hdr.hdr_checksum) { + rte_flow_error_set(error, + EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by ntuple filter"); + return -EINVAL; + } + + filter->dst_ip_mask = ipv4_mask->hdr.dst_addr; + filter->src_ip_mask = ipv4_mask->hdr.src_addr; + filter->proto_mask = ipv4_mask->hdr.next_proto_id; + + ipv4_spec = (const struct rte_flow_item_ipv4 *)item->spec; + filter->dst_ip = ipv4_spec->hdr.dst_addr; + filter->src_ip = ipv4_spec->hdr.src_addr; + filter->proto = ipv4_spec->hdr.next_proto_id; + + /* check if the next not void item is TCP or UDP or SCTP */ + index++; + NEXT_ITEM_OF_PATTERN(item, pattern, index); + if (item->type != RTE_FLOW_ITEM_TYPE_TCP && + item->type != RTE_FLOW_ITEM_TYPE_UDP && + item->type != RTE_FLOW_ITEM_TYPE_SCTP) { + memset(filter, 0, sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by ntuple filter"); + return -EINVAL; + } + + /* get the TCP/UDP info */ + if (!item->spec || !item->mask) { + memset(filter, 0, sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Invalid ntuple mask"); + return -EINVAL; + } + + /*Not supported last point for range*/ + if (item->last) { + memset(filter, 0, sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + item, "Not supported last point for range"); + return -EINVAL; + + } + + if (item->type == RTE_FLOW_ITEM_TYPE_TCP) { + tcp_mask = (const struct rte_flow_item_tcp *)item->mask; + + /** + * Only support src & dst ports, tcp flags, + * others should be masked. + */ + if (tcp_mask->hdr.sent_seq || + tcp_mask->hdr.recv_ack || + tcp_mask->hdr.data_off || + tcp_mask->hdr.rx_win || + tcp_mask->hdr.cksum || + tcp_mask->hdr.tcp_urp) { + memset(filter, 0, + sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by ntuple filter"); + return -EINVAL; + } + + filter->dst_port_mask = tcp_mask->hdr.dst_port; + filter->src_port_mask = tcp_mask->hdr.src_port; + if (tcp_mask->hdr.tcp_flags == 0xFF) { + filter->flags |= RTE_NTUPLE_FLAGS_TCP_FLAG; + } else if (!tcp_mask->hdr.tcp_flags) { + filter->flags &= ~RTE_NTUPLE_FLAGS_TCP_FLAG; + } else { + memset(filter, 0, sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by ntuple filter"); + return -EINVAL; + } + + tcp_spec = (const struct rte_flow_item_tcp *)item->spec; + filter->dst_port = tcp_spec->hdr.dst_port; + filter->src_port = tcp_spec->hdr.src_port; + filter->tcp_flags = tcp_spec->hdr.tcp_flags; + } else if (item->type == RTE_FLOW_ITEM_TYPE_UDP) { + udp_mask = (const struct rte_flow_item_udp *)item->mask; + + /** + * Only support src & dst ports, + * others should be masked. + */ + if (udp_mask->hdr.dgram_len || + udp_mask->hdr.dgram_cksum) { + memset(filter, 0, + sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by ntuple filter"); + return -EINVAL; + } + + filter->dst_port_mask = udp_mask->hdr.dst_port; + filter->src_port_mask = udp_mask->hdr.src_port; + + udp_spec = (const struct rte_flow_item_udp *)item->spec; + filter->dst_port = udp_spec->hdr.dst_port; + filter->src_port = udp_spec->hdr.src_port; + } else { + sctp_mask = (const struct rte_flow_item_sctp *)item->mask; + + /** + * Only support src & dst ports, + * others should be masked. + */ + if (sctp_mask->hdr.tag || + sctp_mask->hdr.cksum) { + memset(filter, 0, + sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by ntuple filter"); + return -EINVAL; + } + + filter->dst_port_mask = sctp_mask->hdr.dst_port; + filter->src_port_mask = sctp_mask->hdr.src_port; + + sctp_spec = (const struct rte_flow_item_sctp *)item->spec; + filter->dst_port = sctp_spec->hdr.dst_port; + filter->src_port = sctp_spec->hdr.src_port; + } + + /* check if the next not void item is END */ + index++; + NEXT_ITEM_OF_PATTERN(item, pattern, index); + if (item->type != RTE_FLOW_ITEM_TYPE_END) { + memset(filter, 0, sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, "Not supported by ntuple filter"); + return -EINVAL; + } + + /* parse action */ + index = 0; + + /** + * n-tuple only supports count, + * check if the first not void action is COUNT. + */ + memset(&action, 0, sizeof(action)); + NEXT_ITEM_OF_ACTION(act, actions, index); + if (act->type != RTE_FLOW_ACTION_TYPE_COUNT) { + memset(filter, 0, sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + item, "Not supported action."); + return -EINVAL; + } + action.type = RTE_FLOW_ACTION_TYPE_COUNT; + + /* check if the next not void item is END */ + index++; + NEXT_ITEM_OF_ACTION(act, actions, index); + if (act->type != RTE_FLOW_ACTION_TYPE_END) { + memset(filter, 0, sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + act, "Not supported action."); + return -EINVAL; + } + + /* parse attr */ + /* must be input direction */ + if (!attr->ingress) { + memset(filter, 0, sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, + attr, "Only support ingress."); + return -EINVAL; + } + + /* not supported */ + if (attr->egress) { + memset(filter, 0, sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, + attr, "Not support egress."); + return -EINVAL; + } + + if (attr->priority > 0xFFFF) { + memset(filter, 0, sizeof(struct rte_eth_ntuple_filter)); + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, + attr, "Error priority."); + return -EINVAL; + } + filter->priority = (uint16_t)attr->priority; + if (attr->priority > FLOW_RULE_MIN_PRIORITY) + filter->priority = FLOW_RULE_MAX_PRIORITY; + + return 0; +} diff --git a/dpdk/lib/librte_flow_classify/rte_flow_classify_parse.h b/dpdk/lib/librte_flow_classify/rte_flow_classify_parse.h new file mode 100644 index 00000000..1d4708a7 --- /dev/null +++ b/dpdk/lib/librte_flow_classify/rte_flow_classify_parse.h @@ -0,0 +1,74 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_FLOW_CLASSIFY_PARSE_H_ +#define _RTE_FLOW_CLASSIFY_PARSE_H_ + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef int (*parse_filter_t)(const struct rte_flow_attr *attr, + const struct rte_flow_item pattern[], + const struct rte_flow_action actions[], + struct rte_eth_ntuple_filter *filter, + struct rte_flow_error *error); + +/* Skip all VOID items of the pattern */ +void +classify_pattern_skip_void_item(struct rte_flow_item *items, + const struct rte_flow_item *pattern); + +/* Find the first VOID or non-VOID item pointer */ +const struct rte_flow_item * +classify_find_first_item(const struct rte_flow_item *item, bool is_void); + + +/* Find if there's parse filter function matched */ +parse_filter_t +classify_find_parse_filter_func(struct rte_flow_item *pattern); + +/* get action data */ +struct rte_flow_action * +classify_get_flow_action(void); + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_FLOW_CLASSIFY_PARSE_H_ */ diff --git a/dpdk/lib/librte_flow_classify/rte_flow_classify_version.map b/dpdk/lib/librte_flow_classify/rte_flow_classify_version.map new file mode 100644 index 00000000..f7695cbf --- /dev/null +++ b/dpdk/lib/librte_flow_classify/rte_flow_classify_version.map @@ -0,0 +1,12 @@ +EXPERIMENTAL { + global: + + rte_flow_classifier_create; + rte_flow_classifier_free; + rte_flow_classifier_query; + rte_flow_classify_table_create; + rte_flow_classify_table_entry_add; + rte_flow_classify_table_entry_delete; + + local: *; +}; diff --git a/dpdk/lib/librte_gro/Makefile b/dpdk/lib/librte_gro/Makefile new file mode 100644 index 00000000..eb423ccb --- /dev/null +++ b/dpdk/lib/librte_gro/Makefile @@ -0,0 +1,52 @@ +# BSD LICENSE +# +# Copyright(c) 2017 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +include $(RTE_SDK)/mk/rte.vars.mk + +# library name +LIB = librte_gro.a + +CFLAGS += -O3 +CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) +LDLIBS += -lrte_eal -lrte_mbuf -lrte_ethdev -lrte_net + +EXPORT_MAP := rte_gro_version.map + +LIBABIVER := 1 + +# source files +SRCS-$(CONFIG_RTE_LIBRTE_GRO) += rte_gro.c +SRCS-$(CONFIG_RTE_LIBRTE_GRO) += gro_tcp4.c + +# install this header file +SYMLINK-$(CONFIG_RTE_LIBRTE_GRO)-include += rte_gro.h + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/lib/librte_gro/gro_tcp4.c b/dpdk/lib/librte_gro/gro_tcp4.c new file mode 100644 index 00000000..61a04232 --- /dev/null +++ b/dpdk/lib/librte_gro/gro_tcp4.c @@ -0,0 +1,505 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include + +#include "gro_tcp4.h" + +void * +gro_tcp4_tbl_create(uint16_t socket_id, + uint16_t max_flow_num, + uint16_t max_item_per_flow) +{ + struct gro_tcp4_tbl *tbl; + size_t size; + uint32_t entries_num, i; + + entries_num = max_flow_num * max_item_per_flow; + entries_num = RTE_MIN(entries_num, GRO_TCP4_TBL_MAX_ITEM_NUM); + + if (entries_num == 0) + return NULL; + + tbl = rte_zmalloc_socket(__func__, + sizeof(struct gro_tcp4_tbl), + RTE_CACHE_LINE_SIZE, + socket_id); + if (tbl == NULL) + return NULL; + + size = sizeof(struct gro_tcp4_item) * entries_num; + tbl->items = rte_zmalloc_socket(__func__, + size, + RTE_CACHE_LINE_SIZE, + socket_id); + if (tbl->items == NULL) { + rte_free(tbl); + return NULL; + } + tbl->max_item_num = entries_num; + + size = sizeof(struct gro_tcp4_key) * entries_num; + tbl->keys = rte_zmalloc_socket(__func__, + size, + RTE_CACHE_LINE_SIZE, + socket_id); + if (tbl->keys == NULL) { + rte_free(tbl->items); + rte_free(tbl); + return NULL; + } + /* INVALID_ARRAY_INDEX indicates empty key */ + for (i = 0; i < entries_num; i++) + tbl->keys[i].start_index = INVALID_ARRAY_INDEX; + tbl->max_key_num = entries_num; + + return tbl; +} + +void +gro_tcp4_tbl_destroy(void *tbl) +{ + struct gro_tcp4_tbl *tcp_tbl = tbl; + + if (tcp_tbl) { + rte_free(tcp_tbl->items); + rte_free(tcp_tbl->keys); + } + rte_free(tcp_tbl); +} + +/* + * merge two TCP/IPv4 packets without updating checksums. + * If cmp is larger than 0, append the new packet to the + * original packet. Otherwise, pre-pend the new packet to + * the original packet. + */ +static inline int +merge_two_tcp4_packets(struct gro_tcp4_item *item_src, + struct rte_mbuf *pkt, + uint16_t ip_id, + uint32_t sent_seq, + int cmp) +{ + struct rte_mbuf *pkt_head, *pkt_tail, *lastseg; + uint16_t tcp_datalen; + + if (cmp > 0) { + pkt_head = item_src->firstseg; + pkt_tail = pkt; + } else { + pkt_head = pkt; + pkt_tail = item_src->firstseg; + } + + /* check if the packet length will be beyond the max value */ + tcp_datalen = pkt_tail->pkt_len - pkt_tail->l2_len - + pkt_tail->l3_len - pkt_tail->l4_len; + if (pkt_head->pkt_len - pkt_head->l2_len + tcp_datalen > + TCP4_MAX_L3_LENGTH) + return 0; + + /* remove packet header for the tail packet */ + rte_pktmbuf_adj(pkt_tail, + pkt_tail->l2_len + + pkt_tail->l3_len + + pkt_tail->l4_len); + + /* chain two packets together */ + if (cmp > 0) { + item_src->lastseg->next = pkt; + item_src->lastseg = rte_pktmbuf_lastseg(pkt); + /* update IP ID to the larger value */ + item_src->ip_id = ip_id; + } else { + lastseg = rte_pktmbuf_lastseg(pkt); + lastseg->next = item_src->firstseg; + item_src->firstseg = pkt; + /* update sent_seq to the smaller value */ + item_src->sent_seq = sent_seq; + } + item_src->nb_merged++; + + /* update mbuf metadata for the merged packet */ + pkt_head->nb_segs += pkt_tail->nb_segs; + pkt_head->pkt_len += pkt_tail->pkt_len; + + return 1; +} + +static inline int +check_seq_option(struct gro_tcp4_item *item, + struct tcp_hdr *tcp_hdr, + uint16_t tcp_hl, + uint16_t tcp_dl, + uint16_t ip_id, + uint32_t sent_seq) +{ + struct rte_mbuf *pkt0 = item->firstseg; + struct ipv4_hdr *ipv4_hdr0; + struct tcp_hdr *tcp_hdr0; + uint16_t tcp_hl0, tcp_dl0; + uint16_t len; + + ipv4_hdr0 = (struct ipv4_hdr *)(rte_pktmbuf_mtod(pkt0, char *) + + pkt0->l2_len); + tcp_hdr0 = (struct tcp_hdr *)((char *)ipv4_hdr0 + pkt0->l3_len); + tcp_hl0 = pkt0->l4_len; + + /* check if TCP option fields equal. If not, return 0. */ + len = RTE_MAX(tcp_hl, tcp_hl0) - sizeof(struct tcp_hdr); + if ((tcp_hl != tcp_hl0) || + ((len > 0) && (memcmp(tcp_hdr + 1, + tcp_hdr0 + 1, + len) != 0))) + return 0; + + /* check if the two packets are neighbors */ + tcp_dl0 = pkt0->pkt_len - pkt0->l2_len - pkt0->l3_len - tcp_hl0; + if ((sent_seq == (item->sent_seq + tcp_dl0)) && + (ip_id == (item->ip_id + 1))) + /* append the new packet */ + return 1; + else if (((sent_seq + tcp_dl) == item->sent_seq) && + ((ip_id + item->nb_merged) == item->ip_id)) + /* pre-pend the new packet */ + return -1; + else + return 0; +} + +static inline uint32_t +find_an_empty_item(struct gro_tcp4_tbl *tbl) +{ + uint32_t i; + uint32_t max_item_num = tbl->max_item_num; + + for (i = 0; i < max_item_num; i++) + if (tbl->items[i].firstseg == NULL) + return i; + return INVALID_ARRAY_INDEX; +} + +static inline uint32_t +find_an_empty_key(struct gro_tcp4_tbl *tbl) +{ + uint32_t i; + uint32_t max_key_num = tbl->max_key_num; + + for (i = 0; i < max_key_num; i++) + if (tbl->keys[i].start_index == INVALID_ARRAY_INDEX) + return i; + return INVALID_ARRAY_INDEX; +} + +static inline uint32_t +insert_new_item(struct gro_tcp4_tbl *tbl, + struct rte_mbuf *pkt, + uint16_t ip_id, + uint32_t sent_seq, + uint32_t prev_idx, + uint64_t start_time) +{ + uint32_t item_idx; + + item_idx = find_an_empty_item(tbl); + if (item_idx == INVALID_ARRAY_INDEX) + return INVALID_ARRAY_INDEX; + + tbl->items[item_idx].firstseg = pkt; + tbl->items[item_idx].lastseg = rte_pktmbuf_lastseg(pkt); + tbl->items[item_idx].start_time = start_time; + tbl->items[item_idx].next_pkt_idx = INVALID_ARRAY_INDEX; + tbl->items[item_idx].sent_seq = sent_seq; + tbl->items[item_idx].ip_id = ip_id; + tbl->items[item_idx].nb_merged = 1; + tbl->item_num++; + + /* if the previous packet exists, chain the new one with it */ + if (prev_idx != INVALID_ARRAY_INDEX) { + tbl->items[item_idx].next_pkt_idx = + tbl->items[prev_idx].next_pkt_idx; + tbl->items[prev_idx].next_pkt_idx = item_idx; + } + + return item_idx; +} + +static inline uint32_t +delete_item(struct gro_tcp4_tbl *tbl, uint32_t item_idx, + uint32_t prev_item_idx) +{ + uint32_t next_idx = tbl->items[item_idx].next_pkt_idx; + + /* set NULL to firstseg to indicate it's an empty item */ + tbl->items[item_idx].firstseg = NULL; + tbl->item_num--; + if (prev_item_idx != INVALID_ARRAY_INDEX) + tbl->items[prev_item_idx].next_pkt_idx = next_idx; + + return next_idx; +} + +static inline uint32_t +insert_new_key(struct gro_tcp4_tbl *tbl, + struct tcp4_key *key_src, + uint32_t item_idx) +{ + struct tcp4_key *key_dst; + uint32_t key_idx; + + key_idx = find_an_empty_key(tbl); + if (key_idx == INVALID_ARRAY_INDEX) + return INVALID_ARRAY_INDEX; + + key_dst = &(tbl->keys[key_idx].key); + + ether_addr_copy(&(key_src->eth_saddr), &(key_dst->eth_saddr)); + ether_addr_copy(&(key_src->eth_daddr), &(key_dst->eth_daddr)); + key_dst->ip_src_addr = key_src->ip_src_addr; + key_dst->ip_dst_addr = key_src->ip_dst_addr; + key_dst->recv_ack = key_src->recv_ack; + key_dst->src_port = key_src->src_port; + key_dst->dst_port = key_src->dst_port; + + /* non-INVALID_ARRAY_INDEX value indicates this key is valid */ + tbl->keys[key_idx].start_index = item_idx; + tbl->key_num++; + + return key_idx; +} + +static inline int +is_same_key(struct tcp4_key k1, struct tcp4_key k2) +{ + if (is_same_ether_addr(&k1.eth_saddr, &k2.eth_saddr) == 0) + return 0; + + if (is_same_ether_addr(&k1.eth_daddr, &k2.eth_daddr) == 0) + return 0; + + return ((k1.ip_src_addr == k2.ip_src_addr) && + (k1.ip_dst_addr == k2.ip_dst_addr) && + (k1.recv_ack == k2.recv_ack) && + (k1.src_port == k2.src_port) && + (k1.dst_port == k2.dst_port)); +} + +/* + * update packet length for the flushed packet. + */ +static inline void +update_header(struct gro_tcp4_item *item) +{ + struct ipv4_hdr *ipv4_hdr; + struct rte_mbuf *pkt = item->firstseg; + + ipv4_hdr = (struct ipv4_hdr *)(rte_pktmbuf_mtod(pkt, char *) + + pkt->l2_len); + ipv4_hdr->total_length = rte_cpu_to_be_16(pkt->pkt_len - + pkt->l2_len); +} + +int32_t +gro_tcp4_reassemble(struct rte_mbuf *pkt, + struct gro_tcp4_tbl *tbl, + uint64_t start_time) +{ + struct ether_hdr *eth_hdr; + struct ipv4_hdr *ipv4_hdr; + struct tcp_hdr *tcp_hdr; + uint32_t sent_seq; + uint16_t tcp_dl, ip_id; + + struct tcp4_key key; + uint32_t cur_idx, prev_idx, item_idx; + uint32_t i, max_key_num; + int cmp; + + eth_hdr = rte_pktmbuf_mtod(pkt, struct ether_hdr *); + ipv4_hdr = (struct ipv4_hdr *)((char *)eth_hdr + pkt->l2_len); + tcp_hdr = (struct tcp_hdr *)((char *)ipv4_hdr + pkt->l3_len); + + /* + * if FIN, SYN, RST, PSH, URG, ECE or + * CWR is set, return immediately. + */ + if (tcp_hdr->tcp_flags != TCP_ACK_FLAG) + return -1; + /* if payload length is 0, return immediately */ + tcp_dl = rte_be_to_cpu_16(ipv4_hdr->total_length) - pkt->l3_len - + pkt->l4_len; + if (tcp_dl == 0) + return -1; + + ip_id = rte_be_to_cpu_16(ipv4_hdr->packet_id); + sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq); + + ether_addr_copy(&(eth_hdr->s_addr), &(key.eth_saddr)); + ether_addr_copy(&(eth_hdr->d_addr), &(key.eth_daddr)); + key.ip_src_addr = ipv4_hdr->src_addr; + key.ip_dst_addr = ipv4_hdr->dst_addr; + key.src_port = tcp_hdr->src_port; + key.dst_port = tcp_hdr->dst_port; + key.recv_ack = tcp_hdr->recv_ack; + + /* search for a key */ + max_key_num = tbl->max_key_num; + for (i = 0; i < max_key_num; i++) { + if ((tbl->keys[i].start_index != INVALID_ARRAY_INDEX) && + is_same_key(tbl->keys[i].key, key)) + break; + } + + /* can't find a key, so insert a new key and a new item. */ + if (i == tbl->max_key_num) { + item_idx = insert_new_item(tbl, pkt, ip_id, sent_seq, + INVALID_ARRAY_INDEX, start_time); + if (item_idx == INVALID_ARRAY_INDEX) + return -1; + if (insert_new_key(tbl, &key, item_idx) == + INVALID_ARRAY_INDEX) { + /* + * fail to insert a new key, so + * delete the inserted item + */ + delete_item(tbl, item_idx, INVALID_ARRAY_INDEX); + return -1; + } + return 0; + } + + /* traverse all packets in the item group to find one to merge */ + cur_idx = tbl->keys[i].start_index; + prev_idx = cur_idx; + do { + cmp = check_seq_option(&(tbl->items[cur_idx]), tcp_hdr, + pkt->l4_len, tcp_dl, ip_id, sent_seq); + if (cmp) { + if (merge_two_tcp4_packets(&(tbl->items[cur_idx]), + pkt, ip_id, + sent_seq, cmp)) + return 1; + /* + * fail to merge two packets since the packet + * length will be greater than the max value. + * So insert the packet into the item group. + */ + if (insert_new_item(tbl, pkt, ip_id, sent_seq, + prev_idx, start_time) == + INVALID_ARRAY_INDEX) + return -1; + return 0; + } + prev_idx = cur_idx; + cur_idx = tbl->items[cur_idx].next_pkt_idx; + } while (cur_idx != INVALID_ARRAY_INDEX); + + /* + * can't find a packet in the item group to merge, + * so insert the packet into the item group. + */ + if (insert_new_item(tbl, pkt, ip_id, sent_seq, prev_idx, + start_time) == INVALID_ARRAY_INDEX) + return -1; + + return 0; +} + +uint16_t +gro_tcp4_tbl_timeout_flush(struct gro_tcp4_tbl *tbl, + uint64_t flush_timestamp, + struct rte_mbuf **out, + uint16_t nb_out) +{ + uint16_t k = 0; + uint32_t i, j; + uint32_t max_key_num = tbl->max_key_num; + + for (i = 0; i < max_key_num; i++) { + /* all keys have been checked, return immediately */ + if (tbl->key_num == 0) + return k; + + j = tbl->keys[i].start_index; + while (j != INVALID_ARRAY_INDEX) { + if (tbl->items[j].start_time <= flush_timestamp) { + out[k++] = tbl->items[j].firstseg; + if (tbl->items[j].nb_merged > 1) + update_header(&(tbl->items[j])); + /* + * delete the item and get + * the next packet index + */ + j = delete_item(tbl, j, + INVALID_ARRAY_INDEX); + + /* + * delete the key as all of + * packets are flushed + */ + if (j == INVALID_ARRAY_INDEX) { + tbl->keys[i].start_index = + INVALID_ARRAY_INDEX; + tbl->key_num--; + } else + /* update start_index of the key */ + tbl->keys[i].start_index = j; + + if (k == nb_out) + return k; + } else + /* + * left packets of this key won't be + * timeout, so go to check other keys. + */ + break; + } + } + return k; +} + +uint32_t +gro_tcp4_tbl_pkt_count(void *tbl) +{ + struct gro_tcp4_tbl *gro_tbl = tbl; + + if (gro_tbl) + return gro_tbl->item_num; + + return 0; +} diff --git a/dpdk/lib/librte_gro/gro_tcp4.h b/dpdk/lib/librte_gro/gro_tcp4.h new file mode 100644 index 00000000..0a817162 --- /dev/null +++ b/dpdk/lib/librte_gro/gro_tcp4.h @@ -0,0 +1,210 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _GRO_TCP4_H_ +#define _GRO_TCP4_H_ + +#define INVALID_ARRAY_INDEX 0xffffffffUL +#define GRO_TCP4_TBL_MAX_ITEM_NUM (1024UL * 1024UL) + +/* + * the max L3 length of a TCP/IPv4 packet. The L3 length + * is the sum of ipv4 header, tcp header and L4 payload. + */ +#define TCP4_MAX_L3_LENGTH UINT16_MAX + +/* criteria of mergeing packets */ +struct tcp4_key { + struct ether_addr eth_saddr; + struct ether_addr eth_daddr; + uint32_t ip_src_addr; + uint32_t ip_dst_addr; + + uint32_t recv_ack; + uint16_t src_port; + uint16_t dst_port; +}; + +struct gro_tcp4_key { + struct tcp4_key key; + /* + * the index of the first packet in the item group. + * If the value is INVALID_ARRAY_INDEX, it means + * the key is empty. + */ + uint32_t start_index; +}; + +struct gro_tcp4_item { + /* + * first segment of the packet. If the value + * is NULL, it means the item is empty. + */ + struct rte_mbuf *firstseg; + /* last segment of the packet */ + struct rte_mbuf *lastseg; + /* + * the time when the first packet is inserted + * into the table. If a packet in the table is + * merged with an incoming packet, this value + * won't be updated. We set this value only + * when the first packet is inserted into the + * table. + */ + uint64_t start_time; + /* + * we use next_pkt_idx to chain the packets that + * have same key value but can't be merged together. + */ + uint32_t next_pkt_idx; + /* the sequence number of the packet */ + uint32_t sent_seq; + /* the IP ID of the packet */ + uint16_t ip_id; + /* the number of merged packets */ + uint16_t nb_merged; +}; + +/* + * TCP/IPv4 reassembly table structure. + */ +struct gro_tcp4_tbl { + /* item array */ + struct gro_tcp4_item *items; + /* key array */ + struct gro_tcp4_key *keys; + /* current item number */ + uint32_t item_num; + /* current key num */ + uint32_t key_num; + /* item array size */ + uint32_t max_item_num; + /* key array size */ + uint32_t max_key_num; +}; + +/** + * This function creates a TCP/IPv4 reassembly table. + * + * @param socket_id + * socket index for allocating TCP/IPv4 reassemble table + * @param max_flow_num + * the maximum number of flows in the TCP/IPv4 GRO table + * @param max_item_per_flow + * the maximum packet number per flow. + * + * @return + * if create successfully, return a pointer which points to the + * created TCP/IPv4 GRO table. Otherwise, return NULL. + */ +void *gro_tcp4_tbl_create(uint16_t socket_id, + uint16_t max_flow_num, + uint16_t max_item_per_flow); + +/** + * This function destroys a TCP/IPv4 reassembly table. + * + * @param tbl + * a pointer points to the TCP/IPv4 reassembly table. + */ +void gro_tcp4_tbl_destroy(void *tbl); + +/** + * This function searches for a packet in the TCP/IPv4 reassembly table + * to merge with the inputted one. To merge two packets is to chain them + * together and update packet headers. Packets, whose SYN, FIN, RST, PSH + * CWR, ECE or URG bit is set, are returned immediately. Packets which + * only have packet headers (i.e. without data) are also returned + * immediately. Otherwise, the packet is either merged, or inserted into + * the table. Besides, if there is no available space to insert the + * packet, this function returns immediately too. + * + * This function assumes the inputted packet is with correct IPv4 and + * TCP checksums. And if two packets are merged, it won't re-calculate + * IPv4 and TCP checksums. Besides, if the inputted packet is IP + * fragmented, it assumes the packet is complete (with TCP header). + * + * @param pkt + * packet to reassemble. + * @param tbl + * a pointer that points to a TCP/IPv4 reassembly table. + * @start_time + * the start time that the packet is inserted into the table + * + * @return + * if the packet doesn't have data, or SYN, FIN, RST, PSH, CWR, ECE + * or URG bit is set, or there is no available space in the table to + * insert a new item or a new key, return a negative value. If the + * packet is merged successfully, return an positive value. If the + * packet is inserted into the table, return 0. + */ +int32_t gro_tcp4_reassemble(struct rte_mbuf *pkt, + struct gro_tcp4_tbl *tbl, + uint64_t start_time); + +/** + * This function flushes timeout packets in a TCP/IPv4 reassembly table + * to applications, and without updating checksums for merged packets. + * The max number of flushed timeout packets is the element number of + * the array which is used to keep flushed packets. + * + * @param tbl + * a pointer that points to a TCP GRO table. + * @param flush_timestamp + * this function flushes packets which are inserted into the table + * before or at the flush_timestamp. + * @param out + * pointer array which is used to keep flushed packets. + * @param nb_out + * the element number of out. It's also the max number of timeout + * packets that can be flushed finally. + * + * @return + * the number of packets that are returned. + */ +uint16_t gro_tcp4_tbl_timeout_flush(struct gro_tcp4_tbl *tbl, + uint64_t flush_timestamp, + struct rte_mbuf **out, + uint16_t nb_out); + +/** + * This function returns the number of the packets in a TCP/IPv4 + * reassembly table. + * + * @param tbl + * pointer points to a TCP/IPv4 reassembly table. + * + * @return + * the number of packets in the table + */ +uint32_t gro_tcp4_tbl_pkt_count(void *tbl); +#endif diff --git a/dpdk/lib/librte_gro/rte_gro.c b/dpdk/lib/librte_gro/rte_gro.c new file mode 100644 index 00000000..7853246a --- /dev/null +++ b/dpdk/lib/librte_gro/rte_gro.c @@ -0,0 +1,278 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include + +#include "rte_gro.h" +#include "gro_tcp4.h" + +typedef void *(*gro_tbl_create_fn)(uint16_t socket_id, + uint16_t max_flow_num, + uint16_t max_item_per_flow); +typedef void (*gro_tbl_destroy_fn)(void *tbl); +typedef uint32_t (*gro_tbl_pkt_count_fn)(void *tbl); + +static gro_tbl_create_fn tbl_create_fn[RTE_GRO_TYPE_MAX_NUM] = { + gro_tcp4_tbl_create, NULL}; +static gro_tbl_destroy_fn tbl_destroy_fn[RTE_GRO_TYPE_MAX_NUM] = { + gro_tcp4_tbl_destroy, NULL}; +static gro_tbl_pkt_count_fn tbl_pkt_count_fn[RTE_GRO_TYPE_MAX_NUM] = { + gro_tcp4_tbl_pkt_count, NULL}; + +/* + * GRO context structure, which is used to merge packets. It keeps + * many reassembly tables of desired GRO types. Applications need to + * create GRO context objects before using rte_gro_reassemble to + * perform GRO. + */ +struct gro_ctx { + /* GRO types to perform */ + uint64_t gro_types; + /* reassembly tables */ + void *tbls[RTE_GRO_TYPE_MAX_NUM]; +}; + +void * +rte_gro_ctx_create(const struct rte_gro_param *param) +{ + struct gro_ctx *gro_ctx; + gro_tbl_create_fn create_tbl_fn; + uint64_t gro_type_flag = 0; + uint64_t gro_types = 0; + uint8_t i; + + gro_ctx = rte_zmalloc_socket(__func__, + sizeof(struct gro_ctx), + RTE_CACHE_LINE_SIZE, + param->socket_id); + if (gro_ctx == NULL) + return NULL; + + for (i = 0; i < RTE_GRO_TYPE_MAX_NUM; i++) { + gro_type_flag = 1ULL << i; + if ((param->gro_types & gro_type_flag) == 0) + continue; + + create_tbl_fn = tbl_create_fn[i]; + if (create_tbl_fn == NULL) + continue; + + gro_ctx->tbls[i] = create_tbl_fn(param->socket_id, + param->max_flow_num, + param->max_item_per_flow); + if (gro_ctx->tbls[i] == NULL) { + /* destroy all created tables */ + gro_ctx->gro_types = gro_types; + rte_gro_ctx_destroy(gro_ctx); + return NULL; + } + gro_types |= gro_type_flag; + } + gro_ctx->gro_types = param->gro_types; + + return gro_ctx; +} + +void +rte_gro_ctx_destroy(void *ctx) +{ + gro_tbl_destroy_fn destroy_tbl_fn; + struct gro_ctx *gro_ctx = ctx; + uint64_t gro_type_flag; + uint8_t i; + + if (gro_ctx == NULL) + return; + for (i = 0; i < RTE_GRO_TYPE_MAX_NUM; i++) { + gro_type_flag = 1ULL << i; + if ((gro_ctx->gro_types & gro_type_flag) == 0) + continue; + destroy_tbl_fn = tbl_destroy_fn[i]; + if (destroy_tbl_fn) + destroy_tbl_fn(gro_ctx->tbls[i]); + } + rte_free(gro_ctx); +} + +uint16_t +rte_gro_reassemble_burst(struct rte_mbuf **pkts, + uint16_t nb_pkts, + const struct rte_gro_param *param) +{ + uint16_t i; + uint16_t nb_after_gro = nb_pkts; + uint32_t item_num; + + /* allocate a reassembly table for TCP/IPv4 GRO */ + struct gro_tcp4_tbl tcp_tbl; + struct gro_tcp4_key tcp_keys[RTE_GRO_MAX_BURST_ITEM_NUM]; + struct gro_tcp4_item tcp_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0} }; + + struct rte_mbuf *unprocess_pkts[nb_pkts]; + uint16_t unprocess_num = 0; + int32_t ret; + uint64_t current_time; + + if ((param->gro_types & RTE_GRO_TCP_IPV4) == 0) + return nb_pkts; + + /* get the actual number of packets */ + item_num = RTE_MIN(nb_pkts, (param->max_flow_num * + param->max_item_per_flow)); + item_num = RTE_MIN(item_num, RTE_GRO_MAX_BURST_ITEM_NUM); + + for (i = 0; i < item_num; i++) + tcp_keys[i].start_index = INVALID_ARRAY_INDEX; + + tcp_tbl.keys = tcp_keys; + tcp_tbl.items = tcp_items; + tcp_tbl.key_num = 0; + tcp_tbl.item_num = 0; + tcp_tbl.max_key_num = item_num; + tcp_tbl.max_item_num = item_num; + + current_time = rte_rdtsc(); + + for (i = 0; i < nb_pkts; i++) { + if ((pkts[i]->packet_type & (RTE_PTYPE_L3_IPV4 | + RTE_PTYPE_L4_TCP)) == + (RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_TCP)) { + ret = gro_tcp4_reassemble(pkts[i], + &tcp_tbl, + current_time); + if (ret > 0) + /* merge successfully */ + nb_after_gro--; + else if (ret < 0) { + unprocess_pkts[unprocess_num++] = + pkts[i]; + } + } else + unprocess_pkts[unprocess_num++] = pkts[i]; + } + + /* re-arrange GROed packets */ + if (nb_after_gro < nb_pkts) { + i = gro_tcp4_tbl_timeout_flush(&tcp_tbl, current_time, + pkts, nb_pkts); + if (unprocess_num > 0) { + memcpy(&pkts[i], unprocess_pkts, + sizeof(struct rte_mbuf *) * + unprocess_num); + } + } + + return nb_after_gro; +} + +uint16_t +rte_gro_reassemble(struct rte_mbuf **pkts, + uint16_t nb_pkts, + void *ctx) +{ + uint16_t i, unprocess_num = 0; + struct rte_mbuf *unprocess_pkts[nb_pkts]; + struct gro_ctx *gro_ctx = ctx; + uint64_t current_time; + + if ((gro_ctx->gro_types & RTE_GRO_TCP_IPV4) == 0) + return nb_pkts; + + current_time = rte_rdtsc(); + + for (i = 0; i < nb_pkts; i++) { + if ((pkts[i]->packet_type & (RTE_PTYPE_L3_IPV4 | + RTE_PTYPE_L4_TCP)) == + (RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_TCP)) { + if (gro_tcp4_reassemble(pkts[i], + gro_ctx->tbls + [RTE_GRO_TCP_IPV4_INDEX], + current_time) < 0) + unprocess_pkts[unprocess_num++] = pkts[i]; + } else + unprocess_pkts[unprocess_num++] = pkts[i]; + } + if (unprocess_num > 0) { + memcpy(pkts, unprocess_pkts, + sizeof(struct rte_mbuf *) * + unprocess_num); + } + + return unprocess_num; +} + +uint16_t +rte_gro_timeout_flush(void *ctx, + uint64_t timeout_cycles, + uint64_t gro_types, + struct rte_mbuf **out, + uint16_t max_nb_out) +{ + struct gro_ctx *gro_ctx = ctx; + uint64_t flush_timestamp; + + gro_types = gro_types & gro_ctx->gro_types; + flush_timestamp = rte_rdtsc() - timeout_cycles; + + if (gro_types & RTE_GRO_TCP_IPV4) { + return gro_tcp4_tbl_timeout_flush( + gro_ctx->tbls[RTE_GRO_TCP_IPV4_INDEX], + flush_timestamp, + out, max_nb_out); + } + return 0; +} + +uint64_t +rte_gro_get_pkt_count(void *ctx) +{ + struct gro_ctx *gro_ctx = ctx; + gro_tbl_pkt_count_fn pkt_count_fn; + uint64_t item_num = 0; + uint64_t gro_type_flag; + uint8_t i; + + for (i = 0; i < RTE_GRO_TYPE_MAX_NUM; i++) { + gro_type_flag = 1ULL << i; + if ((gro_ctx->gro_types & gro_type_flag) == 0) + continue; + + pkt_count_fn = tbl_pkt_count_fn[i]; + if (pkt_count_fn == NULL) + continue; + item_num += pkt_count_fn(gro_ctx->tbls[i]); + } + return item_num; +} diff --git a/dpdk/lib/librte_gro/rte_gro.h b/dpdk/lib/librte_gro/rte_gro.h new file mode 100644 index 00000000..d57e0c5f --- /dev/null +++ b/dpdk/lib/librte_gro/rte_gro.h @@ -0,0 +1,222 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_GRO_H_ +#define _RTE_GRO_H_ + +/** + * @file + * Interface to GRO library + */ + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#define RTE_GRO_MAX_BURST_ITEM_NUM 128U +/**< the max number of packets that rte_gro_reassemble_burst() + * can process in each invocation. + */ +#define RTE_GRO_TYPE_MAX_NUM 64 +/**< the max number of supported GRO types */ +#define RTE_GRO_TYPE_SUPPORT_NUM 1 +/**< the number of currently supported GRO types */ + +#define RTE_GRO_TCP_IPV4_INDEX 0 +#define RTE_GRO_TCP_IPV4 (1ULL << RTE_GRO_TCP_IPV4_INDEX) +/**< TCP/IPv4 GRO flag */ + +/** + * A structure which is used to create GRO context objects or tell + * rte_gro_reassemble_burst() what reassembly rules are demanded. + */ +struct rte_gro_param { + uint64_t gro_types; + /**< desired GRO types */ + uint16_t max_flow_num; + /**< max flow number */ + uint16_t max_item_per_flow; + /**< max packet number per flow */ + uint16_t socket_id; + /**< socket index for allocating GRO related data structures, + * like reassembly tables. When use rte_gro_reassemble_burst(), + * applications don't need to set this value. + */ +}; + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * This function create a GRO context object, which is used to merge + * packets in rte_gro_reassemble(). + * + * @param param + * applications use it to pass needed parameters to create a GRO + * context object. + * + * @return + * if create successfully, return a pointer which points to the GRO + * context object. Otherwise, return NULL. + */ +void *rte_gro_ctx_create(const struct rte_gro_param *param); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * This function destroys a GRO context object. + * + * @param ctx + * pointer points to a GRO context object. + */ +void rte_gro_ctx_destroy(void *ctx); + +/** + * This is one of the main reassembly APIs, which merges numbers of + * packets at a time. It assumes that all inputted packets are with + * correct checksums. That is, applications should guarantee all + * inputted packets are correct. Besides, it doesn't re-calculate + * checksums for merged packets. If inputted packets are IP fragmented, + * this function assumes them are complete (i.e. with L4 header). After + * finishing processing, it returns all GROed packets to applications + * immediately. + * + * @param pkts + * a pointer array which points to the packets to reassemble. Besides, + * it keeps mbuf addresses for the GROed packets. + * @param nb_pkts + * the number of packets to reassemble. + * @param param + * applications use it to tell rte_gro_reassemble_burst() what rules + * are demanded. + * + * @return + * the number of packets after been GROed. If no packets are merged, + * the returned value is nb_pkts. + */ +uint16_t rte_gro_reassemble_burst(struct rte_mbuf **pkts, + uint16_t nb_pkts, + const struct rte_gro_param *param); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Reassembly function, which tries to merge inputted packets with + * the packets in the reassembly tables of a given GRO context. This + * function assumes all inputted packets are with correct checksums. + * And it won't update checksums if two packets are merged. Besides, + * if inputted packets are IP fragmented, this function assumes they + * are complete packets (i.e. with L4 header). + * + * If the inputted packets don't have data or are with unsupported GRO + * types etc., they won't be processed and are returned to applications. + * Otherwise, the inputted packets are either merged or inserted into + * the table. If applications want get packets in the table, they need + * to call flush API. + * + * @param pkts + * packet to reassemble. Besides, after this function finishes, it + * keeps the unprocessed packets (e.g. without data or unsupported + * GRO types). + * @param nb_pkts + * the number of packets to reassemble. + * @param ctx + * a pointer points to a GRO context object. + * + * @return + * return the number of unprocessed packets (e.g. without data or + * unsupported GRO types). If all packets are processed (merged or + * inserted into the table), return 0. + */ +uint16_t rte_gro_reassemble(struct rte_mbuf **pkts, + uint16_t nb_pkts, + void *ctx); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * This function flushes the timeout packets from reassembly tables of + * desired GRO types. The max number of flushed timeout packets is the + * element number of the array which is used to keep the flushed packets. + * + * Besides, this function won't re-calculate checksums for merged + * packets in the tables. That is, the returned packets may be with + * wrong checksums. + * + * @param ctx + * a pointer points to a GRO context object. + * @param timeout_cycles + * max TTL for packets in reassembly tables, measured in nanosecond. + * @param gro_types + * this function only flushes packets which belong to the GRO types + * specified by gro_types. + * @param out + * a pointer array that is used to keep flushed timeout packets. + * @param max_nb_out + * the element number of out. It's also the max number of timeout + * packets that can be flushed finally. + * + * @return + * the number of flushed packets. If no packets are flushed, return 0. + */ +uint16_t rte_gro_timeout_flush(void *ctx, + uint64_t timeout_cycles, + uint64_t gro_types, + struct rte_mbuf **out, + uint16_t max_nb_out); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * This function returns the number of packets in all reassembly tables + * of a given GRO context. + * + * @param ctx + * pointer points to a GRO context object. + * + * @return + * the number of packets in all reassembly tables. + */ +uint64_t rte_gro_get_pkt_count(void *ctx); + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_GRO_H_ */ diff --git a/dpdk/lib/librte_gro/rte_gro_version.map b/dpdk/lib/librte_gro/rte_gro_version.map new file mode 100644 index 00000000..1606b6dc --- /dev/null +++ b/dpdk/lib/librte_gro/rte_gro_version.map @@ -0,0 +1,12 @@ +DPDK_17.08 { + global: + + rte_gro_ctx_create; + rte_gro_ctx_destroy; + rte_gro_get_pkt_count; + rte_gro_reassemble; + rte_gro_reassemble_burst; + rte_gro_timeout_flush; + + local: *; +}; diff --git a/dpdk/lib/librte_gso/Makefile b/dpdk/lib/librte_gso/Makefile new file mode 100644 index 00000000..ea5ad742 --- /dev/null +++ b/dpdk/lib/librte_gso/Makefile @@ -0,0 +1,54 @@ +# BSD LICENSE +# +# Copyright(c) 2017 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +include $(RTE_SDK)/mk/rte.vars.mk + +# library name +LIB = librte_gso.a + +CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -O3 +LDLIBS += -lrte_eal -lrte_mbuf -lrte_ethdev -lrte_net +LDLIBS += -lrte_mempool + +EXPORT_MAP := rte_gso_version.map + +LIBABIVER := 1 + +#source files +SRCS-$(CONFIG_RTE_LIBRTE_GSO) += rte_gso.c +SRCS-$(CONFIG_RTE_LIBRTE_GSO) += gso_common.c +SRCS-$(CONFIG_RTE_LIBRTE_GSO) += gso_tcp4.c +SRCS-$(CONFIG_RTE_LIBRTE_GSO) += gso_tunnel_tcp4.c + +# install this header file +SYMLINK-$(CONFIG_RTE_LIBRTE_GSO)-include += rte_gso.h + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/lib/librte_gso/gso_common.c b/dpdk/lib/librte_gso/gso_common.c new file mode 100644 index 00000000..ee75d4cd --- /dev/null +++ b/dpdk/lib/librte_gso/gso_common.c @@ -0,0 +1,153 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include + +#include +#include + +#include "gso_common.h" + +static inline void +hdr_segment_init(struct rte_mbuf *hdr_segment, struct rte_mbuf *pkt, + uint16_t pkt_hdr_offset) +{ + /* Copy MBUF metadata */ + hdr_segment->nb_segs = 1; + hdr_segment->port = pkt->port; + hdr_segment->ol_flags = pkt->ol_flags; + hdr_segment->packet_type = pkt->packet_type; + hdr_segment->pkt_len = pkt_hdr_offset; + hdr_segment->data_len = pkt_hdr_offset; + hdr_segment->tx_offload = pkt->tx_offload; + + /* Copy the packet header */ + rte_memcpy(rte_pktmbuf_mtod(hdr_segment, char *), + rte_pktmbuf_mtod(pkt, char *), + pkt_hdr_offset); +} + +static inline void +free_gso_segment(struct rte_mbuf **pkts, uint16_t nb_pkts) +{ + uint16_t i; + + for (i = 0; i < nb_pkts; i++) + rte_pktmbuf_free(pkts[i]); +} + +int +gso_do_segment(struct rte_mbuf *pkt, + uint16_t pkt_hdr_offset, + uint16_t pyld_unit_size, + struct rte_mempool *direct_pool, + struct rte_mempool *indirect_pool, + struct rte_mbuf **pkts_out, + uint16_t nb_pkts_out) +{ + struct rte_mbuf *pkt_in; + struct rte_mbuf *hdr_segment, *pyld_segment, *prev_segment; + uint16_t pkt_in_data_pos, segment_bytes_remaining; + uint16_t pyld_len, nb_segs; + bool more_in_pkt, more_out_segs; + + pkt_in = pkt; + nb_segs = 0; + more_in_pkt = 1; + pkt_in_data_pos = pkt_hdr_offset; + + while (more_in_pkt) { + if (unlikely(nb_segs >= nb_pkts_out)) { + free_gso_segment(pkts_out, nb_segs); + return -EINVAL; + } + + /* Allocate a direct MBUF */ + hdr_segment = rte_pktmbuf_alloc(direct_pool); + if (unlikely(hdr_segment == NULL)) { + free_gso_segment(pkts_out, nb_segs); + return -ENOMEM; + } + /* Fill the packet header */ + hdr_segment_init(hdr_segment, pkt, pkt_hdr_offset); + + prev_segment = hdr_segment; + segment_bytes_remaining = pyld_unit_size; + more_out_segs = 1; + + while (more_out_segs && more_in_pkt) { + /* Allocate an indirect MBUF */ + pyld_segment = rte_pktmbuf_alloc(indirect_pool); + if (unlikely(pyld_segment == NULL)) { + rte_pktmbuf_free(hdr_segment); + free_gso_segment(pkts_out, nb_segs); + return -ENOMEM; + } + /* Attach to current MBUF segment of pkt */ + rte_pktmbuf_attach(pyld_segment, pkt_in); + + prev_segment->next = pyld_segment; + prev_segment = pyld_segment; + + pyld_len = segment_bytes_remaining; + if (pyld_len + pkt_in_data_pos > pkt_in->data_len) + pyld_len = pkt_in->data_len - pkt_in_data_pos; + + pyld_segment->data_off = pkt_in_data_pos + + pkt_in->data_off; + pyld_segment->data_len = pyld_len; + + /* Update header segment */ + hdr_segment->pkt_len += pyld_len; + hdr_segment->nb_segs++; + + pkt_in_data_pos += pyld_len; + segment_bytes_remaining -= pyld_len; + + /* Finish processing a MBUF segment of pkt */ + if (pkt_in_data_pos == pkt_in->data_len) { + pkt_in = pkt_in->next; + pkt_in_data_pos = 0; + if (pkt_in == NULL) + more_in_pkt = 0; + } + + /* Finish generating a GSO segment */ + if (segment_bytes_remaining == 0) + more_out_segs = 0; + } + pkts_out[nb_segs++] = hdr_segment; + } + return nb_segs; +} diff --git a/dpdk/lib/librte_gso/gso_common.h b/dpdk/lib/librte_gso/gso_common.h new file mode 100644 index 00000000..145ea495 --- /dev/null +++ b/dpdk/lib/librte_gso/gso_common.h @@ -0,0 +1,171 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _GSO_COMMON_H_ +#define _GSO_COMMON_H_ + +#include + +#include +#include +#include +#include + +#define IS_FRAGMENTED(frag_off) (((frag_off) & IPV4_HDR_OFFSET_MASK) != 0 \ + || ((frag_off) & IPV4_HDR_MF_FLAG) == IPV4_HDR_MF_FLAG) + +#define TCP_HDR_PSH_MASK ((uint8_t)0x08) +#define TCP_HDR_FIN_MASK ((uint8_t)0x01) + +#define IS_IPV4_TCP(flag) (((flag) & (PKT_TX_TCP_SEG | PKT_TX_IPV4)) == \ + (PKT_TX_TCP_SEG | PKT_TX_IPV4)) + +#define IS_IPV4_VXLAN_TCP4(flag) (((flag) & (PKT_TX_TCP_SEG | PKT_TX_IPV4 | \ + PKT_TX_OUTER_IPV4 | PKT_TX_TUNNEL_VXLAN)) == \ + (PKT_TX_TCP_SEG | PKT_TX_IPV4 | PKT_TX_OUTER_IPV4 | \ + PKT_TX_TUNNEL_VXLAN)) + +#define IS_IPV4_GRE_TCP4(flag) (((flag) & (PKT_TX_TCP_SEG | PKT_TX_IPV4 | \ + PKT_TX_OUTER_IPV4 | PKT_TX_TUNNEL_GRE)) == \ + (PKT_TX_TCP_SEG | PKT_TX_IPV4 | PKT_TX_OUTER_IPV4 | \ + PKT_TX_TUNNEL_GRE)) + +/** + * Internal function which updates the UDP header of a packet, following + * segmentation. This is required to update the header's datagram length field. + * + * @param pkt + * The packet containing the UDP header. + * @param udp_offset + * The offset of the UDP header from the start of the packet. + */ +static inline void +update_udp_header(struct rte_mbuf *pkt, uint16_t udp_offset) +{ + struct udp_hdr *udp_hdr; + + udp_hdr = (struct udp_hdr *)(rte_pktmbuf_mtod(pkt, char *) + + udp_offset); + udp_hdr->dgram_len = rte_cpu_to_be_16(pkt->pkt_len - udp_offset); +} + +/** + * Internal function which updates the TCP header of a packet, following + * segmentation. This is required to update the header's 'sent' sequence + * number, and also to clear 'PSH' and 'FIN' flags for non-tail segments. + * + * @param pkt + * The packet containing the TCP header. + * @param l4_offset + * The offset of the TCP header from the start of the packet. + * @param sent_seq + * The sent sequence number. + * @param non-tail + * Indicates whether or not this is a tail segment. + */ +static inline void +update_tcp_header(struct rte_mbuf *pkt, uint16_t l4_offset, uint32_t sent_seq, + uint8_t non_tail) +{ + struct tcp_hdr *tcp_hdr; + + tcp_hdr = (struct tcp_hdr *)(rte_pktmbuf_mtod(pkt, char *) + + l4_offset); + tcp_hdr->sent_seq = rte_cpu_to_be_32(sent_seq); + if (likely(non_tail)) + tcp_hdr->tcp_flags &= (~(TCP_HDR_PSH_MASK | + TCP_HDR_FIN_MASK)); +} + +/** + * Internal function which updates the IPv4 header of a packet, following + * segmentation. This is required to update the header's 'total_length' field, + * to reflect the reduced length of the now-segmented packet. Furthermore, the + * header's 'packet_id' field must be updated to reflect the new ID of the + * now-segmented packet. + * + * @param pkt + * The packet containing the IPv4 header. + * @param l3_offset + * The offset of the IPv4 header from the start of the packet. + * @param id + * The new ID of the packet. + */ +static inline void +update_ipv4_header(struct rte_mbuf *pkt, uint16_t l3_offset, uint16_t id) +{ + struct ipv4_hdr *ipv4_hdr; + + ipv4_hdr = (struct ipv4_hdr *)(rte_pktmbuf_mtod(pkt, char *) + + l3_offset); + ipv4_hdr->total_length = rte_cpu_to_be_16(pkt->pkt_len - l3_offset); + ipv4_hdr->packet_id = rte_cpu_to_be_16(id); +} + +/** + * Internal function which divides the input packet into small segments. + * Each of the newly-created segments is organized as a two-segment MBUF, + * where the first segment is a standard mbuf, which stores a copy of + * packet header, and the second is an indirect mbuf which points to a + * section of data in the input packet. + * + * @param pkt + * Packet to segment. + * @param pkt_hdr_offset + * Packet header offset, measured in bytes. + * @param pyld_unit_size + * The max payload length of a GSO segment. + * @param direct_pool + * MBUF pool used for allocating direct buffers for output segments. + * @param indirect_pool + * MBUF pool used for allocating indirect buffers for output segments. + * @param pkts_out + * Pointer array used to keep the mbuf addresses of output segments. If + * the memory space in pkts_out is insufficient, gso_do_segment() fails + * and returns -EINVAL. + * @param nb_pkts_out + * The max number of items that pkts_out can keep. + * + * @return + * - The number of segments created in the event of success. + * - Return -ENOMEM if run out of memory in MBUF pools. + * - Return -EINVAL for invalid parameters. + */ +int gso_do_segment(struct rte_mbuf *pkt, + uint16_t pkt_hdr_offset, + uint16_t pyld_unit_size, + struct rte_mempool *direct_pool, + struct rte_mempool *indirect_pool, + struct rte_mbuf **pkts_out, + uint16_t nb_pkts_out); +#endif diff --git a/dpdk/lib/librte_gso/gso_tcp4.c b/dpdk/lib/librte_gso/gso_tcp4.c new file mode 100644 index 00000000..0c628cb1 --- /dev/null +++ b/dpdk/lib/librte_gso/gso_tcp4.c @@ -0,0 +1,102 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "gso_common.h" +#include "gso_tcp4.h" + +static void +update_ipv4_tcp_headers(struct rte_mbuf *pkt, uint8_t ipid_delta, + struct rte_mbuf **segs, uint16_t nb_segs) +{ + struct ipv4_hdr *ipv4_hdr; + struct tcp_hdr *tcp_hdr; + uint32_t sent_seq; + uint16_t id, tail_idx, i; + uint16_t l3_offset = pkt->l2_len; + uint16_t l4_offset = l3_offset + pkt->l3_len; + + ipv4_hdr = (struct ipv4_hdr *)(rte_pktmbuf_mtod(pkt, char*) + + l3_offset); + tcp_hdr = (struct tcp_hdr *)((char *)ipv4_hdr + pkt->l3_len); + id = rte_be_to_cpu_16(ipv4_hdr->packet_id); + sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq); + tail_idx = nb_segs - 1; + + for (i = 0; i < nb_segs; i++) { + update_ipv4_header(segs[i], l3_offset, id); + update_tcp_header(segs[i], l4_offset, sent_seq, i < tail_idx); + id += ipid_delta; + sent_seq += (segs[i]->pkt_len - segs[i]->data_len); + } +} + +int +gso_tcp4_segment(struct rte_mbuf *pkt, + uint16_t gso_size, + uint8_t ipid_delta, + struct rte_mempool *direct_pool, + struct rte_mempool *indirect_pool, + struct rte_mbuf **pkts_out, + uint16_t nb_pkts_out) +{ + struct ipv4_hdr *ipv4_hdr; + uint16_t pyld_unit_size, hdr_offset; + uint16_t frag_off; + int ret; + + /* Don't process the fragmented packet */ + ipv4_hdr = (struct ipv4_hdr *)(rte_pktmbuf_mtod(pkt, char *) + + pkt->l2_len); + frag_off = rte_be_to_cpu_16(ipv4_hdr->fragment_offset); + if (unlikely(IS_FRAGMENTED(frag_off))) { + pkts_out[0] = pkt; + return 1; + } + + /* Don't process the packet without data */ + hdr_offset = pkt->l2_len + pkt->l3_len + pkt->l4_len; + if (unlikely(hdr_offset >= pkt->pkt_len)) { + pkts_out[0] = pkt; + return 1; + } + + pyld_unit_size = gso_size - hdr_offset; + + /* Segment the payload */ + ret = gso_do_segment(pkt, hdr_offset, pyld_unit_size, direct_pool, + indirect_pool, pkts_out, nb_pkts_out); + if (ret > 1) + update_ipv4_tcp_headers(pkt, ipid_delta, pkts_out, ret); + + return ret; +} diff --git a/dpdk/lib/librte_gso/gso_tcp4.h b/dpdk/lib/librte_gso/gso_tcp4.h new file mode 100644 index 00000000..1c574412 --- /dev/null +++ b/dpdk/lib/librte_gso/gso_tcp4.h @@ -0,0 +1,74 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _GSO_TCP4_H_ +#define _GSO_TCP4_H_ + +#include +#include + +/** + * Segment an IPv4/TCP packet. This function doesn't check if the input + * packet has correct checksums, and doesn't update checksums for output + * GSO segments. Furthermore, it doesn't process IP fragment packets. + * + * @param pkt + * The packet mbuf to segment. + * @param gso_size + * The max length of a GSO segment, measured in bytes. + * @param ipid_delta + * The increasing unit of IP ids. + * @param direct_pool + * MBUF pool used for allocating direct buffers for output segments. + * @param indirect_pool + * MBUF pool used for allocating indirect buffers for output segments. + * @param pkts_out + * Pointer array used to store the MBUF addresses of output GSO + * segments, when the function succeeds. If the memory space in + * pkts_out is insufficient, it fails and returns -EINVAL. + * @param nb_pkts_out + * The max number of items that 'pkts_out' can keep. + * + * @return + * - The number of GSO segments filled in pkts_out on success. + * - Return -ENOMEM if run out of memory in MBUF pools. + * - Return -EINVAL for invalid parameters. + */ +int gso_tcp4_segment(struct rte_mbuf *pkt, + uint16_t gso_size, + uint8_t ip_delta, + struct rte_mempool *direct_pool, + struct rte_mempool *indirect_pool, + struct rte_mbuf **pkts_out, + uint16_t nb_pkts_out); +#endif diff --git a/dpdk/lib/librte_gso/gso_tunnel_tcp4.c b/dpdk/lib/librte_gso/gso_tunnel_tcp4.c new file mode 100644 index 00000000..8d0cfd7a --- /dev/null +++ b/dpdk/lib/librte_gso/gso_tunnel_tcp4.c @@ -0,0 +1,126 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "gso_common.h" +#include "gso_tunnel_tcp4.h" + +static void +update_tunnel_ipv4_tcp_headers(struct rte_mbuf *pkt, uint8_t ipid_delta, + struct rte_mbuf **segs, uint16_t nb_segs) +{ + struct ipv4_hdr *ipv4_hdr; + struct tcp_hdr *tcp_hdr; + uint32_t sent_seq; + uint16_t outer_id, inner_id, tail_idx, i; + uint16_t outer_ipv4_offset, inner_ipv4_offset; + uint16_t udp_gre_offset, tcp_offset; + uint8_t update_udp_hdr; + + outer_ipv4_offset = pkt->outer_l2_len; + udp_gre_offset = outer_ipv4_offset + pkt->outer_l3_len; + inner_ipv4_offset = udp_gre_offset + pkt->l2_len; + tcp_offset = inner_ipv4_offset + pkt->l3_len; + + /* Outer IPv4 header. */ + ipv4_hdr = (struct ipv4_hdr *)(rte_pktmbuf_mtod(pkt, char *) + + outer_ipv4_offset); + outer_id = rte_be_to_cpu_16(ipv4_hdr->packet_id); + + /* Inner IPv4 header. */ + ipv4_hdr = (struct ipv4_hdr *)(rte_pktmbuf_mtod(pkt, char *) + + inner_ipv4_offset); + inner_id = rte_be_to_cpu_16(ipv4_hdr->packet_id); + + tcp_hdr = (struct tcp_hdr *)((char *)ipv4_hdr + pkt->l3_len); + sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq); + tail_idx = nb_segs - 1; + + /* Only update UDP header for VxLAN packets. */ + update_udp_hdr = (pkt->ol_flags & PKT_TX_TUNNEL_VXLAN) ? 1 : 0; + + for (i = 0; i < nb_segs; i++) { + update_ipv4_header(segs[i], outer_ipv4_offset, outer_id); + if (update_udp_hdr) + update_udp_header(segs[i], udp_gre_offset); + update_ipv4_header(segs[i], inner_ipv4_offset, inner_id); + update_tcp_header(segs[i], tcp_offset, sent_seq, i < tail_idx); + outer_id++; + inner_id += ipid_delta; + sent_seq += (segs[i]->pkt_len - segs[i]->data_len); + } +} + +int +gso_tunnel_tcp4_segment(struct rte_mbuf *pkt, + uint16_t gso_size, + uint8_t ipid_delta, + struct rte_mempool *direct_pool, + struct rte_mempool *indirect_pool, + struct rte_mbuf **pkts_out, + uint16_t nb_pkts_out) +{ + struct ipv4_hdr *inner_ipv4_hdr; + uint16_t pyld_unit_size, hdr_offset, frag_off; + int ret = 1; + + hdr_offset = pkt->outer_l2_len + pkt->outer_l3_len + pkt->l2_len; + inner_ipv4_hdr = (struct ipv4_hdr *)(rte_pktmbuf_mtod(pkt, char *) + + hdr_offset); + /* + * Don't process the packet whose MF bit or offset in the inner + * IPv4 header are non-zero. + */ + frag_off = rte_be_to_cpu_16(inner_ipv4_hdr->fragment_offset); + if (unlikely(IS_FRAGMENTED(frag_off))) { + pkts_out[0] = pkt; + return 1; + } + + hdr_offset += pkt->l3_len + pkt->l4_len; + /* Don't process the packet without data */ + if (hdr_offset >= pkt->pkt_len) { + pkts_out[0] = pkt; + return 1; + } + pyld_unit_size = gso_size - hdr_offset; + + /* Segment the payload */ + ret = gso_do_segment(pkt, hdr_offset, pyld_unit_size, direct_pool, + indirect_pool, pkts_out, nb_pkts_out); + if (ret <= 1) + return ret; + + update_tunnel_ipv4_tcp_headers(pkt, ipid_delta, pkts_out, ret); + + return ret; +} diff --git a/dpdk/lib/librte_gso/gso_tunnel_tcp4.h b/dpdk/lib/librte_gso/gso_tunnel_tcp4.h new file mode 100644 index 00000000..3c67f0cd --- /dev/null +++ b/dpdk/lib/librte_gso/gso_tunnel_tcp4.h @@ -0,0 +1,75 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _GSO_TUNNEL_TCP4_H_ +#define _GSO_TUNNEL_TCP4_H_ + +#include +#include + +/** + * Segment a tunneling packet with inner TCP/IPv4 headers. This function + * doesn't check if the input packet has correct checksums, and doesn't + * update checksums for output GSO segments. Furthermore, it doesn't + * process IP fragment packets. + * + * @param pkt + * The packet mbuf to segment. + * @param gso_size + * The max length of a GSO segment, measured in bytes. + * @param ipid_delta + * The increasing unit of IP ids. + * @param direct_pool + * MBUF pool used for allocating direct buffers for output segments. + * @param indirect_pool + * MBUF pool used for allocating indirect buffers for output segments. + * @param pkts_out + * Pointer array used to store the MBUF addresses of output GSO + * segments, when it succeeds. If the memory space in pkts_out is + * insufficient, it fails and returns -EINVAL. + * @param nb_pkts_out + * The max number of items that 'pkts_out' can keep. + * + * @return + * - The number of GSO segments filled in pkts_out on success. + * - Return -ENOMEM if run out of memory in MBUF pools. + * - Return -EINVAL for invalid parameters. + */ +int gso_tunnel_tcp4_segment(struct rte_mbuf *pkt, + uint16_t gso_size, + uint8_t ipid_delta, + struct rte_mempool *direct_pool, + struct rte_mempool *indirect_pool, + struct rte_mbuf **pkts_out, + uint16_t nb_pkts_out); +#endif diff --git a/dpdk/lib/librte_gso/rte_gso.c b/dpdk/lib/librte_gso/rte_gso.c new file mode 100644 index 00000000..f86e6541 --- /dev/null +++ b/dpdk/lib/librte_gso/rte_gso.c @@ -0,0 +1,110 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +#include +#include + +#include "rte_gso.h" +#include "gso_common.h" +#include "gso_tcp4.h" +#include "gso_tunnel_tcp4.h" + +int +rte_gso_segment(struct rte_mbuf *pkt, + const struct rte_gso_ctx *gso_ctx, + struct rte_mbuf **pkts_out, + uint16_t nb_pkts_out) +{ + struct rte_mempool *direct_pool, *indirect_pool; + struct rte_mbuf *pkt_seg; + uint64_t ol_flags; + uint16_t gso_size; + uint8_t ipid_delta; + int ret = 1; + + if (pkt == NULL || pkts_out == NULL || gso_ctx == NULL || + nb_pkts_out < 1 || + gso_ctx->gso_size < RTE_GSO_SEG_SIZE_MIN || + ((gso_ctx->gso_types & (DEV_TX_OFFLOAD_TCP_TSO | + DEV_TX_OFFLOAD_VXLAN_TNL_TSO | + DEV_TX_OFFLOAD_GRE_TNL_TSO)) == 0)) + return -EINVAL; + + if (gso_ctx->gso_size >= pkt->pkt_len) { + pkt->ol_flags &= (~PKT_TX_TCP_SEG); + pkts_out[0] = pkt; + return 1; + } + + direct_pool = gso_ctx->direct_pool; + indirect_pool = gso_ctx->indirect_pool; + gso_size = gso_ctx->gso_size; + ipid_delta = (gso_ctx->flag != RTE_GSO_FLAG_IPID_FIXED); + ol_flags = pkt->ol_flags; + + if ((IS_IPV4_VXLAN_TCP4(pkt->ol_flags) && + (gso_ctx->gso_types & DEV_TX_OFFLOAD_VXLAN_TNL_TSO)) || + ((IS_IPV4_GRE_TCP4(pkt->ol_flags) && + (gso_ctx->gso_types & DEV_TX_OFFLOAD_GRE_TNL_TSO)))) { + pkt->ol_flags &= (~PKT_TX_TCP_SEG); + ret = gso_tunnel_tcp4_segment(pkt, gso_size, ipid_delta, + direct_pool, indirect_pool, + pkts_out, nb_pkts_out); + } else if (IS_IPV4_TCP(pkt->ol_flags) && + (gso_ctx->gso_types & DEV_TX_OFFLOAD_TCP_TSO)) { + pkt->ol_flags &= (~PKT_TX_TCP_SEG); + ret = gso_tcp4_segment(pkt, gso_size, ipid_delta, + direct_pool, indirect_pool, + pkts_out, nb_pkts_out); + } else { + /* unsupported packet, skip */ + pkts_out[0] = pkt; + RTE_LOG(DEBUG, GSO, "Unsupported packet type\n"); + return 1; + } + + if (ret > 1) { + pkt_seg = pkt; + while (pkt_seg) { + rte_mbuf_refcnt_update(pkt_seg, -1); + pkt_seg = pkt_seg->next; + } + } else if (ret < 0) { + /* Revert the ol_flags in the event of failure. */ + pkt->ol_flags = ol_flags; + } + + return ret; +} diff --git a/dpdk/lib/librte_gso/rte_gso.h b/dpdk/lib/librte_gso/rte_gso.h new file mode 100644 index 00000000..dbaedec7 --- /dev/null +++ b/dpdk/lib/librte_gso/rte_gso.h @@ -0,0 +1,148 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_GSO_H_ +#define _RTE_GSO_H_ + +/** + * @file + * Interface to GSO library + */ + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include + +/* Minimum GSO segment size. */ +#define RTE_GSO_SEG_SIZE_MIN (sizeof(struct ether_hdr) + \ + sizeof(struct ipv4_hdr) + sizeof(struct tcp_hdr) + 1) + +/* GSO flags for rte_gso_ctx. */ +#define RTE_GSO_FLAG_IPID_FIXED (1ULL << 0) +/**< Use fixed IP ids for output GSO segments. Setting + * 0 indicates using incremental IP ids. + */ + +/** + * GSO context structure. + */ +struct rte_gso_ctx { + struct rte_mempool *direct_pool; + /**< MBUF pool for allocating direct buffers, which are used + * to store packet headers for GSO segments. + */ + struct rte_mempool *indirect_pool; + /**< MBUF pool for allocating indirect buffers, which are used + * to locate packet payloads for GSO segments. The indirect + * buffer doesn't contain any data, but simply points to an + * offset within the packet to segment. + */ + uint64_t flag; + /**< flag that controls specific attributes of output segments, + * such as the type of IP ID generated (i.e. fixed or incremental). + */ + uint32_t gso_types; + /**< the bit mask of required GSO types. The GSO library + * uses the same macros as that of describing device TX + * offloading capabilities (i.e. DEV_TX_OFFLOAD_*_TSO) for + * gso_types. + * + * For example, if applications want to segment TCP/IPv4 + * packets, set DEV_TX_OFFLOAD_TCP_TSO in gso_types. + */ + uint16_t gso_size; + /**< maximum size of an output GSO segment, including packet + * header and payload, measured in bytes. Must exceed + * RTE_GSO_SEG_SIZE_MIN. + */ +}; + +/** + * Segmentation function, which supports processing of both single- and + * multi- MBUF packets. + * + * Note that we refer to the packets that are segmented from the input + * packet as 'GSO segments'. rte_gso_segment() doesn't check if the + * input packet has correct checksums, and doesn't update checksums for + * output GSO segments. Additionally, it doesn't process IP fragment + * packets. + * + * Before calling rte_gso_segment(), applications must set proper ol_flags + * for the packet. The GSO library uses the same macros as that of TSO. + * For example, set PKT_TX_TCP_SEG and PKT_TX_IPV4 in ol_flags to segment + * a TCP/IPv4 packet. If rte_gso_segment() succeeds, the PKT_TX_TCP_SEG + * flag is removed for all GSO segments and the input packet. + * + * Each of the newly-created GSO segments is organized as a two-segment + * MBUF, where the first segment is a standard MBUF, which stores a copy + * of packet header, and the second is an indirect MBUF which points to + * a section of data in the input packet. Since each GSO segment has + * multiple MBUFs (i.e. typically 2 MBUFs), the driver of the interface which + * the GSO segments are sent to should support transmission of multi-segment + * packets. + * + * If the input packet is GSO'd, its mbuf refcnt reduces by 1. Therefore, + * when all GSO segments are freed, the input packet is freed automatically. + * + * If the memory space in pkts_out or MBUF pools is insufficient, this + * function fails, and it returns (-1) * errno. Otherwise, GSO succeeds, + * and this function returns the number of output GSO segments filled in + * pkts_out. + * + * @param pkt + * The packet mbuf to segment. + * @param ctx + * GSO context object pointer. + * @param pkts_out + * Pointer array used to store the MBUF addresses of output GSO + * segments, when rte_gso_segment() succeeds. + * @param nb_pkts_out + * The max number of items that pkts_out can keep. + * + * @return + * - The number of GSO segments filled in pkts_out on success. + * - Return -ENOMEM if run out of memory in MBUF pools. + * - Return -EINVAL for invalid parameters. + */ +int rte_gso_segment(struct rte_mbuf *pkt, + const struct rte_gso_ctx *ctx, + struct rte_mbuf **pkts_out, + uint16_t nb_pkts_out); +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_GSO_H_ */ diff --git a/dpdk/lib/librte_gso/rte_gso_version.map b/dpdk/lib/librte_gso/rte_gso_version.map new file mode 100644 index 00000000..e1fd453e --- /dev/null +++ b/dpdk/lib/librte_gso/rte_gso_version.map @@ -0,0 +1,7 @@ +DPDK_17.11 { + global: + + rte_gso_segment; + + local: *; +}; diff --git a/dpdk/lib/librte_hash/Makefile b/dpdk/lib/librte_hash/Makefile index bb1ea990..1655b601 100644 --- a/dpdk/lib/librte_hash/Makefile +++ b/dpdk/lib/librte_hash/Makefile @@ -36,6 +36,7 @@ LIB = librte_hash.a CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) +LDLIBS += -lrte_eal -lrte_ring EXPORT_MAP := rte_hash_version.map @@ -49,13 +50,12 @@ SRCS-$(CONFIG_RTE_LIBRTE_HASH) += rte_fbk_hash.c SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include := rte_hash.h SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_hash_crc.h ifeq ($(CONFIG_RTE_ARCH_ARM64),y) +ifneq ($(findstring RTE_MACHINE_CPUFLAG_CRC32,$(CFLAGS)),) SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_crc_arm64.h endif +endif SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_jhash.h SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_thash.h SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_fbk_hash.h -# this lib needs eal and ring -DEPDIRS-$(CONFIG_RTE_LIBRTE_HASH) += lib/librte_eal lib/librte_ring - include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/lib/librte_hash/rte_cmp_arm64.h b/dpdk/lib/librte_hash/rte_cmp_arm64.h index 6fd937b1..950cef3b 100644 --- a/dpdk/lib/librte_hash/rte_cmp_arm64.h +++ b/dpdk/lib/librte_hash/rte_cmp_arm64.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2015 Cavium networks. All rights reserved. + * Copyright(c) 2015 Cavium, Inc. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -14,7 +14,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * diff --git a/dpdk/lib/librte_hash/rte_cmp_x86.h b/dpdk/lib/librte_hash/rte_cmp_x86.h index e8c484d6..704c2dec 100644 --- a/dpdk/lib/librte_hash/rte_cmp_x86.h +++ b/dpdk/lib/librte_hash/rte_cmp_x86.h @@ -37,15 +37,9 @@ rte_hash_k16_cmp_eq(const void *key1, const void *key2, size_t key_len __rte_unu { const __m128i k1 = _mm_loadu_si128((const __m128i *) key1); const __m128i k2 = _mm_loadu_si128((const __m128i *) key2); -#ifdef RTE_MACHINE_CPUFLAG_SSE4_1 const __m128i x = _mm_xor_si128(k1, k2); return !_mm_test_all_zeros(x, x); -#else - const __m128i x = _mm_cmpeq_epi32(k1, k2); - - return _mm_movemask_epi8(x) != 0xffff; -#endif } static int diff --git a/dpdk/lib/librte_hash/rte_crc_arm64.h b/dpdk/lib/librte_hash/rte_crc_arm64.h index 7dd6334e..a3c216bb 100644 --- a/dpdk/lib/librte_hash/rte_crc_arm64.h +++ b/dpdk/lib/librte_hash/rte_crc_arm64.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2015 Cavium networks. All rights reserved. + * Copyright(c) 2015 Cavium, Inc. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -14,7 +14,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * @@ -52,7 +52,6 @@ extern "C" { static inline uint32_t crc32c_arm64_u8(uint8_t data, uint32_t init_val) { - asm(".arch armv8-a+crc"); __asm__ volatile( "crc32cb %w[crc], %w[crc], %w[value]" : [crc] "+r" (init_val) @@ -63,7 +62,6 @@ crc32c_arm64_u8(uint8_t data, uint32_t init_val) static inline uint32_t crc32c_arm64_u16(uint16_t data, uint32_t init_val) { - asm(".arch armv8-a+crc"); __asm__ volatile( "crc32ch %w[crc], %w[crc], %w[value]" : [crc] "+r" (init_val) @@ -74,7 +72,6 @@ crc32c_arm64_u16(uint16_t data, uint32_t init_val) static inline uint32_t crc32c_arm64_u32(uint32_t data, uint32_t init_val) { - asm(".arch armv8-a+crc"); __asm__ volatile( "crc32cw %w[crc], %w[crc], %w[value]" : [crc] "+r" (init_val) @@ -85,7 +82,6 @@ crc32c_arm64_u32(uint32_t data, uint32_t init_val) static inline uint32_t crc32c_arm64_u64(uint64_t data, uint32_t init_val) { - asm(".arch armv8-a+crc"); __asm__ volatile( "crc32cx %w[crc], %w[crc], %x[value]" : [crc] "+r" (init_val) @@ -110,16 +106,17 @@ rte_hash_crc_set_alg(uint8_t alg) case CRC32_ARM64: if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_CRC32)) alg = CRC32_SW; + /* fall-through */ case CRC32_SW: crc32_alg = alg; + /* fall-through */ default: break; } } /* Setting the best available algorithm */ -static inline void __attribute__((constructor)) -rte_hash_crc_init_alg(void) +RTE_INIT(rte_hash_crc_init_alg) { rte_hash_crc_set_alg(CRC32_ARM64); } diff --git a/dpdk/lib/librte_hash/rte_cuckoo_hash.c b/dpdk/lib/librte_hash/rte_cuckoo_hash.c index d6e68c68..55fd7bdc 100644 --- a/dpdk/lib/librte_hash/rte_cuckoo_hash.c +++ b/dpdk/lib/librte_hash/rte_cuckoo_hash.c @@ -44,7 +44,6 @@ #include #include #include -#include #include #include #include @@ -52,11 +51,11 @@ #include #include #include -#include #include #include #include #include +#include #include "rte_hash.h" #include "rte_cuckoo_hash.h" @@ -98,6 +97,7 @@ rte_hash_find_existing(const char *name) void rte_hash_set_cmp_func(struct rte_hash *h, rte_hash_cmp_eq_t func) { + h->cmp_jump_table_idx = KEY_CUSTOM; h->rte_hash_custom_cmp_eq = func; } @@ -283,6 +283,15 @@ rte_hash_create(const struct rte_hash_parameters *params) h->free_slots = r; h->hw_trans_mem_support = hw_trans_mem_support; +#if defined(RTE_ARCH_X86) + if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2)) + h->sig_cmp_fn = RTE_HASH_COMPARE_AVX2; + else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_SSE2)) + h->sig_cmp_fn = RTE_HASH_COMPARE_SSE; + else +#endif + h->sig_cmp_fn = RTE_HASH_COMPARE_SCALAR; + /* Turn on multi-writer only with explicit flat from user and TM * support. */ @@ -407,9 +416,9 @@ rte_hash_reset(struct rte_hash *h) /* Search for an entry that can be pushed to its alternative location */ static inline int -make_space_bucket(const struct rte_hash *h, struct rte_hash_bucket *bkt) +make_space_bucket(const struct rte_hash *h, struct rte_hash_bucket *bkt, + unsigned int *nr_pushes) { - static unsigned int nr_pushes; unsigned i, j; int ret; uint32_t next_bucket_idx; @@ -421,10 +430,10 @@ make_space_bucket(const struct rte_hash *h, struct rte_hash_bucket *bkt) */ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) { /* Search for space in alternative locations */ - next_bucket_idx = bkt->signatures[i].alt & h->bucket_bitmask; + next_bucket_idx = bkt->sig_alt[i] & h->bucket_bitmask; next_bkt[i] = &h->buckets[next_bucket_idx]; for (j = 0; j < RTE_HASH_BUCKET_ENTRIES; j++) { - if (next_bkt[i]->signatures[j].sig == NULL_SIGNATURE) + if (next_bkt[i]->key_idx[j] == EMPTY_SLOT) break; } @@ -434,8 +443,8 @@ make_space_bucket(const struct rte_hash *h, struct rte_hash_bucket *bkt) /* Alternative location has spare room (end of recursive function) */ if (i != RTE_HASH_BUCKET_ENTRIES) { - next_bkt[i]->signatures[j].alt = bkt->signatures[i].current; - next_bkt[i]->signatures[j].current = bkt->signatures[i].alt; + next_bkt[i]->sig_alt[j] = bkt->sig_current[i]; + next_bkt[i]->sig_current[j] = bkt->sig_alt[i]; next_bkt[i]->key_idx[j] = bkt->key_idx[i]; return i; } @@ -446,15 +455,14 @@ make_space_bucket(const struct rte_hash *h, struct rte_hash_bucket *bkt) break; /* All entries have been pushed, so entry cannot be added */ - if (i == RTE_HASH_BUCKET_ENTRIES || nr_pushes > RTE_HASH_MAX_PUSHES) + if (i == RTE_HASH_BUCKET_ENTRIES || ++(*nr_pushes) > RTE_HASH_MAX_PUSHES) return -ENOSPC; /* Set flag to indicate that this entry is going to be pushed */ bkt->flag[i] = 1; - nr_pushes++; /* Need room in alternative bucket to insert the pushed entry */ - ret = make_space_bucket(h, next_bkt[i]); + ret = make_space_bucket(h, next_bkt[i], nr_pushes); /* * After recursive function. * Clear flags and insert the pushed entry @@ -462,10 +470,9 @@ make_space_bucket(const struct rte_hash *h, struct rte_hash_bucket *bkt) * or return error */ bkt->flag[i] = 0; - nr_pushes = 0; if (ret >= 0) { - next_bkt[i]->signatures[ret].alt = bkt->signatures[i].current; - next_bkt[i]->signatures[ret].current = bkt->signatures[i].alt; + next_bkt[i]->sig_alt[ret] = bkt->sig_current[i]; + next_bkt[i]->sig_current[ret] = bkt->sig_alt[i]; next_bkt[i]->key_idx[ret] = bkt->key_idx[i]; return i; } else @@ -505,6 +512,7 @@ __rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key, unsigned n_slots; unsigned lcore_id; struct lcore_cache *cached_free_slots = NULL; + unsigned int nr_pushes = 0; if (h->add_key == ADD_KEY_MULTIWRITER) rte_spinlock_lock(h->multiwriter_lock); @@ -526,9 +534,12 @@ __rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key, if (cached_free_slots->len == 0) { /* Need to get another burst of free slots from global ring */ n_slots = rte_ring_mc_dequeue_burst(h->free_slots, - cached_free_slots->objs, LCORE_CACHE_SIZE); - if (n_slots == 0) - return -ENOSPC; + cached_free_slots->objs, + LCORE_CACHE_SIZE, NULL); + if (n_slots == 0) { + ret = -ENOSPC; + goto failure; + } cached_free_slots->len += n_slots; } @@ -537,8 +548,10 @@ __rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key, cached_free_slots->len--; slot_id = cached_free_slots->objs[cached_free_slots->len]; } else { - if (rte_ring_sc_dequeue(h->free_slots, &slot_id) != 0) - return -ENOSPC; + if (rte_ring_sc_dequeue(h->free_slots, &slot_id) != 0) { + ret = -ENOSPC; + goto failure; + } } new_k = RTE_PTR_ADD(keys, (uintptr_t)slot_id * h->key_entry_size); @@ -547,8 +560,8 @@ __rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key, /* Check if key is already inserted in primary location */ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) { - if (prim_bkt->signatures[i].current == sig && - prim_bkt->signatures[i].alt == alt_hash) { + if (prim_bkt->sig_current[i] == sig && + prim_bkt->sig_alt[i] == alt_hash) { k = (struct rte_hash_key *) ((char *)keys + prim_bkt->key_idx[i] * h->key_entry_size); if (rte_hash_cmp_eq(key, k->key, h) == 0) { @@ -558,7 +571,7 @@ __rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key, k->pdata = data; /* * Return index where key is stored, - * substracting the first dummy index + * subtracting the first dummy index */ return prim_bkt->key_idx[i] - 1; } @@ -567,8 +580,8 @@ __rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key, /* Check if key is already inserted in secondary location */ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) { - if (sec_bkt->signatures[i].alt == sig && - sec_bkt->signatures[i].current == alt_hash) { + if (sec_bkt->sig_alt[i] == sig && + sec_bkt->sig_current[i] == alt_hash) { k = (struct rte_hash_key *) ((char *)keys + sec_bkt->key_idx[i] * h->key_entry_size); if (rte_hash_cmp_eq(key, k->key, h) == 0) { @@ -578,7 +591,7 @@ __rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key, k->pdata = data; /* * Return index where key is stored, - * substracting the first dummy index + * subtracting the first dummy index */ return sec_bkt->key_idx[i] - 1; } @@ -613,9 +626,9 @@ __rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key, #endif for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) { /* Check if slot is available */ - if (likely(prim_bkt->signatures[i].sig == NULL_SIGNATURE)) { - prim_bkt->signatures[i].current = sig; - prim_bkt->signatures[i].alt = alt_hash; + if (likely(prim_bkt->key_idx[i] == EMPTY_SLOT)) { + prim_bkt->sig_current[i] = sig; + prim_bkt->sig_alt[i] = alt_hash; prim_bkt->key_idx[i] = new_idx; break; } @@ -633,10 +646,10 @@ __rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key, * if successful or return error and * store the new slot back in the ring */ - ret = make_space_bucket(h, prim_bkt); + ret = make_space_bucket(h, prim_bkt, &nr_pushes); if (ret >= 0) { - prim_bkt->signatures[ret].current = sig; - prim_bkt->signatures[ret].alt = alt_hash; + prim_bkt->sig_current[ret] = sig; + prim_bkt->sig_alt[ret] = alt_hash; prim_bkt->key_idx[ret] = new_idx; if (h->add_key == ADD_KEY_MULTIWRITER) rte_spinlock_unlock(h->multiwriter_lock); @@ -648,6 +661,7 @@ __rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key, /* Error in addition, store new slot back in the ring and return error */ enqueue_slot_back(h, cached_free_slots, (void *)((uintptr_t) new_idx)); +failure: if (h->add_key == ADD_KEY_MULTIWRITER) rte_spinlock_unlock(h->multiwriter_lock); return ret; @@ -710,8 +724,8 @@ __rte_hash_lookup_with_hash(const struct rte_hash *h, const void *key, /* Check if key is in primary location */ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) { - if (bkt->signatures[i].current == sig && - bkt->signatures[i].sig != NULL_SIGNATURE) { + if (bkt->sig_current[i] == sig && + bkt->key_idx[i] != EMPTY_SLOT) { k = (struct rte_hash_key *) ((char *)keys + bkt->key_idx[i] * h->key_entry_size); if (rte_hash_cmp_eq(key, k->key, h) == 0) { @@ -719,7 +733,7 @@ __rte_hash_lookup_with_hash(const struct rte_hash *h, const void *key, *data = k->pdata; /* * Return index where key is stored, - * substracting the first dummy index + * subtracting the first dummy index */ return bkt->key_idx[i] - 1; } @@ -733,8 +747,8 @@ __rte_hash_lookup_with_hash(const struct rte_hash *h, const void *key, /* Check if key is in secondary location */ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) { - if (bkt->signatures[i].current == alt_hash && - bkt->signatures[i].alt == sig) { + if (bkt->sig_current[i] == alt_hash && + bkt->sig_alt[i] == sig) { k = (struct rte_hash_key *) ((char *)keys + bkt->key_idx[i] * h->key_entry_size); if (rte_hash_cmp_eq(key, k->key, h) == 0) { @@ -742,7 +756,7 @@ __rte_hash_lookup_with_hash(const struct rte_hash *h, const void *key, *data = k->pdata; /* * Return index where key is stored, - * substracting the first dummy index + * subtracting the first dummy index */ return bkt->key_idx[i] - 1; } @@ -788,7 +802,8 @@ remove_entry(const struct rte_hash *h, struct rte_hash_bucket *bkt, unsigned i) unsigned lcore_id, n_slots; struct lcore_cache *cached_free_slots; - bkt->signatures[i].sig = NULL_SIGNATURE; + bkt->sig_current[i] = NULL_SIGNATURE; + bkt->sig_alt[i] = NULL_SIGNATURE; if (h->hw_trans_mem_support) { lcore_id = rte_lcore_id(); cached_free_slots = &h->local_free_slots[lcore_id]; @@ -797,7 +812,7 @@ remove_entry(const struct rte_hash *h, struct rte_hash_bucket *bkt, unsigned i) /* Need to enqueue the free slots in global ring. */ n_slots = rte_ring_mp_enqueue_burst(h->free_slots, cached_free_slots->objs, - LCORE_CACHE_SIZE); + LCORE_CACHE_SIZE, NULL); cached_free_slots->len -= n_slots; } /* Put index of new free slot in cache. */ @@ -826,8 +841,8 @@ __rte_hash_del_key_with_hash(const struct rte_hash *h, const void *key, /* Check if key is in primary location */ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) { - if (bkt->signatures[i].current == sig && - bkt->signatures[i].sig != NULL_SIGNATURE) { + if (bkt->sig_current[i] == sig && + bkt->key_idx[i] != EMPTY_SLOT) { k = (struct rte_hash_key *) ((char *)keys + bkt->key_idx[i] * h->key_entry_size); if (rte_hash_cmp_eq(key, k->key, h) == 0) { @@ -835,10 +850,10 @@ __rte_hash_del_key_with_hash(const struct rte_hash *h, const void *key, /* * Return index where key is stored, - * substracting the first dummy index + * subtracting the first dummy index */ ret = bkt->key_idx[i] - 1; - bkt->key_idx[i] = 0; + bkt->key_idx[i] = EMPTY_SLOT; return ret; } } @@ -851,8 +866,8 @@ __rte_hash_del_key_with_hash(const struct rte_hash *h, const void *key, /* Check if key is in secondary location */ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) { - if (bkt->signatures[i].current == alt_hash && - bkt->signatures[i].sig != NULL_SIGNATURE) { + if (bkt->sig_current[i] == alt_hash && + bkt->key_idx[i] != EMPTY_SLOT) { k = (struct rte_hash_key *) ((char *)keys + bkt->key_idx[i] * h->key_entry_size); if (rte_hash_cmp_eq(key, k->key, h) == 0) { @@ -860,10 +875,10 @@ __rte_hash_del_key_with_hash(const struct rte_hash *h, const void *key, /* * Return index where key is stored, - * substracting the first dummy index + * subtracting the first dummy index */ ret = bkt->key_idx[i] - 1; - bkt->key_idx[i] = 0; + bkt->key_idx[i] = EMPTY_SLOT; return ret; } } @@ -907,280 +922,189 @@ rte_hash_get_key_with_position(const struct rte_hash *h, const int32_t position, return 0; } -/* Lookup bulk stage 0: Prefetch input key */ static inline void -lookup_stage0(unsigned *idx, uint64_t *lookup_mask, - const void * const *keys) +compare_signatures(uint32_t *prim_hash_matches, uint32_t *sec_hash_matches, + const struct rte_hash_bucket *prim_bkt, + const struct rte_hash_bucket *sec_bkt, + hash_sig_t prim_hash, hash_sig_t sec_hash, + enum rte_hash_sig_compare_function sig_cmp_fn) { - *idx = __builtin_ctzl(*lookup_mask); - if (*lookup_mask == 0) - *idx = 0; + unsigned int i; - rte_prefetch0(keys[*idx]); - *lookup_mask &= ~(1llu << *idx); -} - -/* - * Lookup bulk stage 1: Calculate primary/secondary hashes - * and prefetch primary/secondary buckets - */ -static inline void -lookup_stage1(unsigned idx, hash_sig_t *prim_hash, hash_sig_t *sec_hash, - const struct rte_hash_bucket **primary_bkt, - const struct rte_hash_bucket **secondary_bkt, - hash_sig_t *hash_vals, const void * const *keys, - const struct rte_hash *h) -{ - *prim_hash = rte_hash_hash(h, keys[idx]); - hash_vals[idx] = *prim_hash; - *sec_hash = rte_hash_secondary_hash(*prim_hash); - - *primary_bkt = &h->buckets[*prim_hash & h->bucket_bitmask]; - *secondary_bkt = &h->buckets[*sec_hash & h->bucket_bitmask]; - - rte_prefetch0(*primary_bkt); - rte_prefetch0(*secondary_bkt); -} - -/* - * Lookup bulk stage 2: Search for match hashes in primary/secondary locations - * and prefetch first key slot - */ -static inline void -lookup_stage2(unsigned idx, hash_sig_t prim_hash, hash_sig_t sec_hash, - const struct rte_hash_bucket *prim_bkt, - const struct rte_hash_bucket *sec_bkt, - const struct rte_hash_key **key_slot, int32_t *positions, - uint64_t *extra_hits_mask, const void *keys, - const struct rte_hash *h) -{ - unsigned prim_hash_matches, sec_hash_matches, key_idx, i; - unsigned total_hash_matches; - - prim_hash_matches = 1 << RTE_HASH_BUCKET_ENTRIES; - sec_hash_matches = 1 << RTE_HASH_BUCKET_ENTRIES; - for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) { - prim_hash_matches |= ((prim_hash == prim_bkt->signatures[i].current) << i); - sec_hash_matches |= ((sec_hash == sec_bkt->signatures[i].current) << i); + switch (sig_cmp_fn) { +#ifdef RTE_MACHINE_CPUFLAG_AVX2 + case RTE_HASH_COMPARE_AVX2: + *prim_hash_matches = _mm256_movemask_ps((__m256)_mm256_cmpeq_epi32( + _mm256_load_si256( + (__m256i const *)prim_bkt->sig_current), + _mm256_set1_epi32(prim_hash))); + *sec_hash_matches = _mm256_movemask_ps((__m256)_mm256_cmpeq_epi32( + _mm256_load_si256( + (__m256i const *)sec_bkt->sig_current), + _mm256_set1_epi32(sec_hash))); + break; +#endif +#ifdef RTE_MACHINE_CPUFLAG_SSE2 + case RTE_HASH_COMPARE_SSE: + /* Compare the first 4 signatures in the bucket */ + *prim_hash_matches = _mm_movemask_ps((__m128)_mm_cmpeq_epi16( + _mm_load_si128( + (__m128i const *)prim_bkt->sig_current), + _mm_set1_epi32(prim_hash))); + *prim_hash_matches |= (_mm_movemask_ps((__m128)_mm_cmpeq_epi16( + _mm_load_si128( + (__m128i const *)&prim_bkt->sig_current[4]), + _mm_set1_epi32(prim_hash)))) << 4; + /* Compare the first 4 signatures in the bucket */ + *sec_hash_matches = _mm_movemask_ps((__m128)_mm_cmpeq_epi16( + _mm_load_si128( + (__m128i const *)sec_bkt->sig_current), + _mm_set1_epi32(sec_hash))); + *sec_hash_matches |= (_mm_movemask_ps((__m128)_mm_cmpeq_epi16( + _mm_load_si128( + (__m128i const *)&sec_bkt->sig_current[4]), + _mm_set1_epi32(sec_hash)))) << 4; + break; +#endif + default: + for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) { + *prim_hash_matches |= + ((prim_hash == prim_bkt->sig_current[i]) << i); + *sec_hash_matches |= + ((sec_hash == sec_bkt->sig_current[i]) << i); + } } - key_idx = prim_bkt->key_idx[__builtin_ctzl(prim_hash_matches)]; - if (key_idx == 0) - key_idx = sec_bkt->key_idx[__builtin_ctzl(sec_hash_matches)]; - - total_hash_matches = (prim_hash_matches | - (sec_hash_matches << (RTE_HASH_BUCKET_ENTRIES + 1))); - *key_slot = (const struct rte_hash_key *) ((const char *)keys + - key_idx * h->key_entry_size); - - rte_prefetch0(*key_slot); - /* - * Return index where key is stored, - * substracting the first dummy index - */ - positions[idx] = (key_idx - 1); - - *extra_hits_mask |= (uint64_t)(__builtin_popcount(total_hash_matches) > 3) << idx; - -} - - -/* Lookup bulk stage 3: Check if key matches, update hit mask and return data */ -static inline void -lookup_stage3(unsigned idx, const struct rte_hash_key *key_slot, const void * const *keys, - const int32_t *positions, void *data[], uint64_t *hits, - const struct rte_hash *h) -{ - unsigned hit; - unsigned key_idx; - - hit = !rte_hash_cmp_eq(key_slot->key, keys[idx], h); - if (data != NULL) - data[idx] = key_slot->pdata; - - key_idx = positions[idx] + 1; - /* - * If key index is 0, force hit to be 0, in case key to be looked up - * is all zero (as in the dummy slot), which would result in a wrong hit - */ - *hits |= (uint64_t)(hit && !!key_idx) << idx; } +#define PREFETCH_OFFSET 4 static inline void __rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys, - uint32_t num_keys, int32_t *positions, + int32_t num_keys, int32_t *positions, uint64_t *hit_mask, void *data[]) { uint64_t hits = 0; - uint64_t extra_hits_mask = 0; - uint64_t lookup_mask, miss_mask; - unsigned idx; - const void *key_store = h->key_store; - int ret; - hash_sig_t hash_vals[RTE_HASH_LOOKUP_BULK_MAX]; + int32_t i; + uint32_t prim_hash[RTE_HASH_LOOKUP_BULK_MAX]; + uint32_t sec_hash[RTE_HASH_LOOKUP_BULK_MAX]; + const struct rte_hash_bucket *primary_bkt[RTE_HASH_LOOKUP_BULK_MAX]; + const struct rte_hash_bucket *secondary_bkt[RTE_HASH_LOOKUP_BULK_MAX]; + uint32_t prim_hitmask[RTE_HASH_LOOKUP_BULK_MAX] = {0}; + uint32_t sec_hitmask[RTE_HASH_LOOKUP_BULK_MAX] = {0}; - unsigned idx00, idx01, idx10, idx11, idx20, idx21, idx30, idx31; - const struct rte_hash_bucket *primary_bkt10, *primary_bkt11; - const struct rte_hash_bucket *secondary_bkt10, *secondary_bkt11; - const struct rte_hash_bucket *primary_bkt20, *primary_bkt21; - const struct rte_hash_bucket *secondary_bkt20, *secondary_bkt21; - const struct rte_hash_key *k_slot20, *k_slot21, *k_slot30, *k_slot31; - hash_sig_t primary_hash10, primary_hash11; - hash_sig_t secondary_hash10, secondary_hash11; - hash_sig_t primary_hash20, primary_hash21; - hash_sig_t secondary_hash20, secondary_hash21; + /* Prefetch first keys */ + for (i = 0; i < PREFETCH_OFFSET && i < num_keys; i++) + rte_prefetch0(keys[i]); - lookup_mask = (uint64_t) -1 >> (64 - num_keys); - miss_mask = lookup_mask; + /* + * Prefetch rest of the keys, calculate primary and + * secondary bucket and prefetch them + */ + for (i = 0; i < (num_keys - PREFETCH_OFFSET); i++) { + rte_prefetch0(keys[i + PREFETCH_OFFSET]); - lookup_stage0(&idx00, &lookup_mask, keys); - lookup_stage0(&idx01, &lookup_mask, keys); + prim_hash[i] = rte_hash_hash(h, keys[i]); + sec_hash[i] = rte_hash_secondary_hash(prim_hash[i]); - idx10 = idx00, idx11 = idx01; + primary_bkt[i] = &h->buckets[prim_hash[i] & h->bucket_bitmask]; + secondary_bkt[i] = &h->buckets[sec_hash[i] & h->bucket_bitmask]; - lookup_stage0(&idx00, &lookup_mask, keys); - lookup_stage0(&idx01, &lookup_mask, keys); - lookup_stage1(idx10, &primary_hash10, &secondary_hash10, - &primary_bkt10, &secondary_bkt10, hash_vals, keys, h); - lookup_stage1(idx11, &primary_hash11, &secondary_hash11, - &primary_bkt11, &secondary_bkt11, hash_vals, keys, h); - - primary_bkt20 = primary_bkt10; - primary_bkt21 = primary_bkt11; - secondary_bkt20 = secondary_bkt10; - secondary_bkt21 = secondary_bkt11; - primary_hash20 = primary_hash10; - primary_hash21 = primary_hash11; - secondary_hash20 = secondary_hash10; - secondary_hash21 = secondary_hash11; - idx20 = idx10, idx21 = idx11; - idx10 = idx00, idx11 = idx01; - - lookup_stage0(&idx00, &lookup_mask, keys); - lookup_stage0(&idx01, &lookup_mask, keys); - lookup_stage1(idx10, &primary_hash10, &secondary_hash10, - &primary_bkt10, &secondary_bkt10, hash_vals, keys, h); - lookup_stage1(idx11, &primary_hash11, &secondary_hash11, - &primary_bkt11, &secondary_bkt11, hash_vals, keys, h); - lookup_stage2(idx20, primary_hash20, secondary_hash20, primary_bkt20, - secondary_bkt20, &k_slot20, positions, &extra_hits_mask, - key_store, h); - lookup_stage2(idx21, primary_hash21, secondary_hash21, primary_bkt21, - secondary_bkt21, &k_slot21, positions, &extra_hits_mask, - key_store, h); - - while (lookup_mask) { - k_slot30 = k_slot20, k_slot31 = k_slot21; - idx30 = idx20, idx31 = idx21; - primary_bkt20 = primary_bkt10; - primary_bkt21 = primary_bkt11; - secondary_bkt20 = secondary_bkt10; - secondary_bkt21 = secondary_bkt11; - primary_hash20 = primary_hash10; - primary_hash21 = primary_hash11; - secondary_hash20 = secondary_hash10; - secondary_hash21 = secondary_hash11; - idx20 = idx10, idx21 = idx11; - idx10 = idx00, idx11 = idx01; - - lookup_stage0(&idx00, &lookup_mask, keys); - lookup_stage0(&idx01, &lookup_mask, keys); - lookup_stage1(idx10, &primary_hash10, &secondary_hash10, - &primary_bkt10, &secondary_bkt10, hash_vals, keys, h); - lookup_stage1(idx11, &primary_hash11, &secondary_hash11, - &primary_bkt11, &secondary_bkt11, hash_vals, keys, h); - lookup_stage2(idx20, primary_hash20, secondary_hash20, - primary_bkt20, secondary_bkt20, &k_slot20, positions, - &extra_hits_mask, key_store, h); - lookup_stage2(idx21, primary_hash21, secondary_hash21, - primary_bkt21, secondary_bkt21, &k_slot21, positions, - &extra_hits_mask, key_store, h); - lookup_stage3(idx30, k_slot30, keys, positions, data, &hits, h); - lookup_stage3(idx31, k_slot31, keys, positions, data, &hits, h); + rte_prefetch0(primary_bkt[i]); + rte_prefetch0(secondary_bkt[i]); } - k_slot30 = k_slot20, k_slot31 = k_slot21; - idx30 = idx20, idx31 = idx21; - primary_bkt20 = primary_bkt10; - primary_bkt21 = primary_bkt11; - secondary_bkt20 = secondary_bkt10; - secondary_bkt21 = secondary_bkt11; - primary_hash20 = primary_hash10; - primary_hash21 = primary_hash11; - secondary_hash20 = secondary_hash10; - secondary_hash21 = secondary_hash11; - idx20 = idx10, idx21 = idx11; - idx10 = idx00, idx11 = idx01; + /* Calculate and prefetch rest of the buckets */ + for (; i < num_keys; i++) { + prim_hash[i] = rte_hash_hash(h, keys[i]); + sec_hash[i] = rte_hash_secondary_hash(prim_hash[i]); - lookup_stage1(idx10, &primary_hash10, &secondary_hash10, - &primary_bkt10, &secondary_bkt10, hash_vals, keys, h); - lookup_stage1(idx11, &primary_hash11, &secondary_hash11, - &primary_bkt11, &secondary_bkt11, hash_vals, keys, h); - lookup_stage2(idx20, primary_hash20, secondary_hash20, primary_bkt20, - secondary_bkt20, &k_slot20, positions, &extra_hits_mask, - key_store, h); - lookup_stage2(idx21, primary_hash21, secondary_hash21, primary_bkt21, - secondary_bkt21, &k_slot21, positions, &extra_hits_mask, - key_store, h); - lookup_stage3(idx30, k_slot30, keys, positions, data, &hits, h); - lookup_stage3(idx31, k_slot31, keys, positions, data, &hits, h); + primary_bkt[i] = &h->buckets[prim_hash[i] & h->bucket_bitmask]; + secondary_bkt[i] = &h->buckets[sec_hash[i] & h->bucket_bitmask]; - k_slot30 = k_slot20, k_slot31 = k_slot21; - idx30 = idx20, idx31 = idx21; - primary_bkt20 = primary_bkt10; - primary_bkt21 = primary_bkt11; - secondary_bkt20 = secondary_bkt10; - secondary_bkt21 = secondary_bkt11; - primary_hash20 = primary_hash10; - primary_hash21 = primary_hash11; - secondary_hash20 = secondary_hash10; - secondary_hash21 = secondary_hash11; - idx20 = idx10, idx21 = idx11; + rte_prefetch0(primary_bkt[i]); + rte_prefetch0(secondary_bkt[i]); + } - lookup_stage2(idx20, primary_hash20, secondary_hash20, primary_bkt20, - secondary_bkt20, &k_slot20, positions, &extra_hits_mask, - key_store, h); - lookup_stage2(idx21, primary_hash21, secondary_hash21, primary_bkt21, - secondary_bkt21, &k_slot21, positions, &extra_hits_mask, - key_store, h); - lookup_stage3(idx30, k_slot30, keys, positions, data, &hits, h); - lookup_stage3(idx31, k_slot31, keys, positions, data, &hits, h); + /* Compare signatures and prefetch key slot of first hit */ + for (i = 0; i < num_keys; i++) { + compare_signatures(&prim_hitmask[i], &sec_hitmask[i], + primary_bkt[i], secondary_bkt[i], + prim_hash[i], sec_hash[i], h->sig_cmp_fn); - k_slot30 = k_slot20, k_slot31 = k_slot21; - idx30 = idx20, idx31 = idx21; + if (prim_hitmask[i]) { + uint32_t first_hit = __builtin_ctzl(prim_hitmask[i]); + uint32_t key_idx = primary_bkt[i]->key_idx[first_hit]; + const struct rte_hash_key *key_slot = + (const struct rte_hash_key *)( + (const char *)h->key_store + + key_idx * h->key_entry_size); + rte_prefetch0(key_slot); + continue; + } - lookup_stage3(idx30, k_slot30, keys, positions, data, &hits, h); - lookup_stage3(idx31, k_slot31, keys, positions, data, &hits, h); + if (sec_hitmask[i]) { + uint32_t first_hit = __builtin_ctzl(sec_hitmask[i]); + uint32_t key_idx = secondary_bkt[i]->key_idx[first_hit]; + const struct rte_hash_key *key_slot = + (const struct rte_hash_key *)( + (const char *)h->key_store + + key_idx * h->key_entry_size); + rte_prefetch0(key_slot); + } + } - /* ignore any items we have already found */ - extra_hits_mask &= ~hits; + /* Compare keys, first hits in primary first */ + for (i = 0; i < num_keys; i++) { + positions[i] = -ENOENT; + while (prim_hitmask[i]) { + uint32_t hit_index = __builtin_ctzl(prim_hitmask[i]); - if (unlikely(extra_hits_mask)) { - /* run a single search for each remaining item */ - do { - idx = __builtin_ctzl(extra_hits_mask); - if (data != NULL) { - ret = rte_hash_lookup_with_hash_data(h, - keys[idx], hash_vals[idx], &data[idx]); - if (ret >= 0) - hits |= 1ULL << idx; - } else { - positions[idx] = rte_hash_lookup_with_hash(h, - keys[idx], hash_vals[idx]); - if (positions[idx] >= 0) - hits |= 1llu << idx; + uint32_t key_idx = primary_bkt[i]->key_idx[hit_index]; + const struct rte_hash_key *key_slot = + (const struct rte_hash_key *)( + (const char *)h->key_store + + key_idx * h->key_entry_size); + /* + * If key index is 0, do not compare key, + * as it is checking the dummy slot + */ + if (!!key_idx & !rte_hash_cmp_eq(key_slot->key, keys[i], h)) { + if (data != NULL) + data[i] = key_slot->pdata; + + hits |= 1ULL << i; + positions[i] = key_idx - 1; + goto next_key; } - extra_hits_mask &= ~(1llu << idx); - } while (extra_hits_mask); - } + prim_hitmask[i] &= ~(1 << (hit_index)); + } - miss_mask &= ~hits; - if (unlikely(miss_mask)) { - do { - idx = __builtin_ctzl(miss_mask); - positions[idx] = -ENOENT; - miss_mask &= ~(1llu << idx); - } while (miss_mask); + while (sec_hitmask[i]) { + uint32_t hit_index = __builtin_ctzl(sec_hitmask[i]); + + uint32_t key_idx = secondary_bkt[i]->key_idx[hit_index]; + const struct rte_hash_key *key_slot = + (const struct rte_hash_key *)( + (const char *)h->key_store + + key_idx * h->key_entry_size); + /* + * If key index is 0, do not compare key, + * as it is checking the dummy slot + */ + + if (!!key_idx & !rte_hash_cmp_eq(key_slot->key, keys[i], h)) { + if (data != NULL) + data[i] = key_slot->pdata; + + hits |= 1ULL << i; + positions[i] = key_idx - 1; + goto next_key; + } + sec_hitmask[i] &= ~(1 << (hit_index)); + } + +next_key: + continue; } if (hit_mask != NULL) @@ -1233,7 +1157,7 @@ rte_hash_iterate(const struct rte_hash *h, const void **key, void **data, uint32 idx = *next % RTE_HASH_BUCKET_ENTRIES; /* If current position is empty, go to the next one */ - while (h->buckets[bucket_idx].signatures[idx].sig == NULL_SIGNATURE) { + while (h->buckets[bucket_idx].key_idx[idx] == EMPTY_SLOT) { (*next)++; /* End of table */ if (*next == total_entries) diff --git a/dpdk/lib/librte_hash/rte_cuckoo_hash.h b/dpdk/lib/librte_hash/rte_cuckoo_hash.h index 9625fffe..f75392d2 100644 --- a/dpdk/lib/librte_hash/rte_cuckoo_hash.h +++ b/dpdk/lib/librte_hash/rte_cuckoo_hash.h @@ -58,7 +58,7 @@ #endif /* Hash function used if none is specified */ -#if defined(RTE_MACHINE_CPUFLAG_SSE4_2) || defined(RTE_MACHINE_CPUFLAG_CRC32) +#if defined(RTE_ARCH_X86) || defined(RTE_MACHINE_CPUFLAG_CRC32) #include #define DEFAULT_HASH_FUNC rte_hash_crc #else @@ -130,10 +130,12 @@ enum add_key_case { }; /** Number of items per bucket. */ -#define RTE_HASH_BUCKET_ENTRIES 4 +#define RTE_HASH_BUCKET_ENTRIES 8 #define NULL_SIGNATURE 0 +#define EMPTY_SLOT 0 + #define KEY_ALIGNMENT 16 #define LCORE_CACHE_SIZE 64 @@ -151,17 +153,6 @@ struct lcore_cache { void *objs[LCORE_CACHE_SIZE]; /**< Cache objects */ } __rte_cache_aligned; -/* Structure storing both primary and secondary hashes */ -struct rte_hash_signatures { - union { - struct { - hash_sig_t current; - hash_sig_t alt; - }; - uint64_t sig; - }; -}; - /* Structure that stores key-value pair */ struct rte_hash_key { union { @@ -172,11 +163,22 @@ struct rte_hash_key { char key[0]; } __attribute__((aligned(KEY_ALIGNMENT))); +/* All different signature compare functions */ +enum rte_hash_sig_compare_function { + RTE_HASH_COMPARE_SCALAR = 0, + RTE_HASH_COMPARE_SSE, + RTE_HASH_COMPARE_AVX2, + RTE_HASH_COMPARE_NUM +}; + /** Bucket structure */ struct rte_hash_bucket { - struct rte_hash_signatures signatures[RTE_HASH_BUCKET_ENTRIES]; - /* Includes dummy key index that always contains index 0 */ - uint32_t key_idx[RTE_HASH_BUCKET_ENTRIES + 1]; + hash_sig_t sig_current[RTE_HASH_BUCKET_ENTRIES]; + + uint32_t key_idx[RTE_HASH_BUCKET_ENTRIES]; + + hash_sig_t sig_alt[RTE_HASH_BUCKET_ENTRIES]; + uint8_t flag[RTE_HASH_BUCKET_ENTRIES]; } __rte_cache_aligned; @@ -185,30 +187,38 @@ struct rte_hash { char name[RTE_HASH_NAMESIZE]; /**< Name of the hash. */ uint32_t entries; /**< Total table entries. */ uint32_t num_buckets; /**< Number of buckets in table. */ - uint32_t key_len; /**< Length of hash key. */ + + struct rte_ring *free_slots; + /**< Ring that stores all indexes of the free slots in the key table */ + uint8_t hw_trans_mem_support; + /**< Hardware transactional memory support */ + struct lcore_cache *local_free_slots; + /**< Local cache per lcore, storing some indexes of the free slots */ + enum add_key_case add_key; /**< Multi-writer hash add behavior */ + + rte_spinlock_t *multiwriter_lock; /**< Multi-writer spinlock for w/o TM */ + + /* Fields used in lookup */ + + uint32_t key_len __rte_cache_aligned; + /**< Length of hash key. */ rte_hash_function hash_func; /**< Function used to calculate hash. */ uint32_t hash_func_init_val; /**< Init value used by hash_func. */ rte_hash_cmp_eq_t rte_hash_custom_cmp_eq; /**< Custom function used to compare keys. */ enum cmp_jump_table_case cmp_jump_table_idx; /**< Indicates which compare function to use. */ - uint32_t bucket_bitmask; /**< Bitmask for getting bucket index - from hash signature. */ + enum rte_hash_sig_compare_function sig_cmp_fn; + /**< Indicates which signature compare function to use. */ + uint32_t bucket_bitmask; + /**< Bitmask for getting bucket index from hash signature. */ uint32_t key_entry_size; /**< Size of each key entry. */ - struct rte_ring *free_slots; /**< Ring that stores all indexes - of the free slots in the key table */ void *key_store; /**< Table storing all keys and data */ - struct rte_hash_bucket *buckets; /**< Table with buckets storing all the - hash values and key indexes - to the key table*/ - uint8_t hw_trans_mem_support; /**< Hardware transactional - memory support */ - struct lcore_cache *local_free_slots; - /**< Local cache per lcore, storing some indexes of the free slots */ - enum add_key_case add_key; /**< Multi-writer hash add behavior */ - - rte_spinlock_t *multiwriter_lock; /**< Multi-writer spinlock for w/o TM */ + struct rte_hash_bucket *buckets; + /**< Table with buckets storing all the hash values and key indexes + * to the key table. + */ } __rte_cache_aligned; struct queue_node { diff --git a/dpdk/lib/librte_hash/rte_cuckoo_hash_x86.h b/dpdk/lib/librte_hash/rte_cuckoo_hash_x86.h index ace1bd2e..0c94244a 100644 --- a/dpdk/lib/librte_hash/rte_cuckoo_hash_x86.h +++ b/dpdk/lib/librte_hash/rte_cuckoo_hash_x86.h @@ -53,10 +53,9 @@ rte_hash_cuckoo_insert_mw_tm(struct rte_hash_bucket *prim_bkt, */ for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) { /* Check if slot is available */ - if (likely(prim_bkt->signatures[i].sig == - NULL_SIGNATURE)) { - prim_bkt->signatures[i].current = sig; - prim_bkt->signatures[i].alt = alt_hash; + if (likely(prim_bkt->key_idx[i] == EMPTY_SLOT)) { + prim_bkt->sig_current[i] = sig; + prim_bkt->sig_alt[i] = alt_hash; prim_bkt->key_idx[i] = new_idx; break; } @@ -102,7 +101,7 @@ rte_hash_cuckoo_move_insert_mw_tm(const struct rte_hash *h, prev_slot = curr_node->prev_slot; prev_alt_bkt_idx - = prev_bkt->signatures[prev_slot].alt + = prev_bkt->sig_alt[prev_slot] & h->bucket_bitmask; if (unlikely(&h->buckets[prev_alt_bkt_idx] @@ -114,10 +113,10 @@ rte_hash_cuckoo_move_insert_mw_tm(const struct rte_hash *h, * Cuckoo insert to move elements back to its * primary bucket if available */ - curr_bkt->signatures[curr_slot].alt = - prev_bkt->signatures[prev_slot].current; - curr_bkt->signatures[curr_slot].current = - prev_bkt->signatures[prev_slot].alt; + curr_bkt->sig_alt[curr_slot] = + prev_bkt->sig_current[prev_slot]; + curr_bkt->sig_current[curr_slot] = + prev_bkt->sig_alt[prev_slot]; curr_bkt->key_idx[curr_slot] = prev_bkt->key_idx[prev_slot]; @@ -126,8 +125,8 @@ rte_hash_cuckoo_move_insert_mw_tm(const struct rte_hash *h, curr_bkt = curr_node->bkt; } - curr_bkt->signatures[curr_slot].current = sig; - curr_bkt->signatures[curr_slot].alt = alt_hash; + curr_bkt->sig_current[curr_slot] = sig; + curr_bkt->sig_alt[curr_slot] = alt_hash; curr_bkt->key_idx[curr_slot] = new_idx; rte_xend(); @@ -172,7 +171,7 @@ rte_hash_cuckoo_make_space_mw_tm(const struct rte_hash *h, RTE_HASH_BUCKET_ENTRIES)) { curr_bkt = tail->bkt; for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) { - if (curr_bkt->signatures[i].sig == NULL_SIGNATURE) { + if (curr_bkt->key_idx[i] == EMPTY_SLOT) { if (likely(rte_hash_cuckoo_move_insert_mw_tm(h, tail, i, sig, alt_hash, new_idx) == 0)) @@ -180,7 +179,7 @@ rte_hash_cuckoo_make_space_mw_tm(const struct rte_hash *h, } /* Enqueue new node and keep prev node info */ - alt_bkt = &(h->buckets[curr_bkt->signatures[i].alt + alt_bkt = &(h->buckets[curr_bkt->sig_alt[i] & h->bucket_bitmask]); head->bkt = alt_bkt; head->prev = tail; diff --git a/dpdk/lib/librte_hash/rte_fbk_hash.c b/dpdk/lib/librte_hash/rte_fbk_hash.c index 55c9f358..c87719fb 100644 --- a/dpdk/lib/librte_hash/rte_fbk_hash.c +++ b/dpdk/lib/librte_hash/rte_fbk_hash.c @@ -39,7 +39,6 @@ #include #include -#include #include #include #include diff --git a/dpdk/lib/librte_hash/rte_fbk_hash.h b/dpdk/lib/librte_hash/rte_fbk_hash.h index a430961d..e0d42b45 100644 --- a/dpdk/lib/librte_hash/rte_fbk_hash.h +++ b/dpdk/lib/librte_hash/rte_fbk_hash.h @@ -54,8 +54,9 @@ extern "C" { #include +#include #ifndef RTE_FBK_HASH_FUNC_DEFAULT -#if defined(RTE_MACHINE_CPUFLAG_SSE4_2) || defined(RTE_MACHINE_CPUFLAG_CRC32) +#if defined(RTE_ARCH_X86) || defined(RTE_MACHINE_CPUFLAG_CRC32) #include /** Default four-byte key hash function if none is specified. */ #define RTE_FBK_HASH_FUNC_DEFAULT rte_hash_crc_4byte @@ -115,7 +116,7 @@ struct rte_fbk_hash_table { uint32_t init_val; /**< For initialising hash function. */ /** A flat table of all buckets. */ - union rte_fbk_hash_entry t[0]; + union rte_fbk_hash_entry t[]; }; /** diff --git a/dpdk/lib/librte_hash/rte_hash_crc.h b/dpdk/lib/librte_hash/rte_hash_crc.h index 63e74aa4..93188c29 100644 --- a/dpdk/lib/librte_hash/rte_hash_crc.h +++ b/dpdk/lib/librte_hash/rte_hash_crc.h @@ -45,6 +45,7 @@ extern "C" { #endif #include +#include #include #include #include @@ -387,7 +388,7 @@ crc32c_2words(uint64_t data, uint32_t init_val) return crc; } -#if defined(RTE_ARCH_I686) || defined(RTE_ARCH_X86_64) +#if defined(RTE_ARCH_X86) static inline uint32_t crc32c_sse42_u8(uint8_t data, uint32_t init_val) { @@ -453,7 +454,7 @@ crc32c_sse42_u64(uint64_t data, uint64_t init_val) static uint8_t crc32_alg = CRC32_SW; -#if defined(RTE_ARCH_ARM64) +#if defined(RTE_ARCH_ARM64) && defined(RTE_MACHINE_CPUFLAG_CRC32) #include "rte_crc_arm64.h" #else @@ -471,25 +472,16 @@ static uint8_t crc32_alg = CRC32_SW; static inline void rte_hash_crc_set_alg(uint8_t alg) { - switch (alg) { -#if defined(RTE_ARCH_I686) || defined(RTE_ARCH_X86_64) - case CRC32_SSE42_x64: - if (! rte_cpu_get_flag_enabled(RTE_CPUFLAG_EM64T)) - alg = CRC32_SSE42; - case CRC32_SSE42: - if (! rte_cpu_get_flag_enabled(RTE_CPUFLAG_SSE4_2)) - alg = CRC32_SW; +#if defined(RTE_ARCH_X86) + if (alg == CRC32_SSE42_x64 && + !rte_cpu_get_flag_enabled(RTE_CPUFLAG_EM64T)) + alg = CRC32_SSE42; #endif - case CRC32_SW: - crc32_alg = alg; - default: - break; - } + crc32_alg = alg; } /* Setting the best available algorithm */ -static inline void __attribute__((constructor)) -rte_hash_crc_init_alg(void) +RTE_INIT(rte_hash_crc_init_alg) { rte_hash_crc_set_alg(CRC32_SSE42_x64); } @@ -509,7 +501,7 @@ rte_hash_crc_init_alg(void) static inline uint32_t rte_hash_crc_1byte(uint8_t data, uint32_t init_val) { -#if defined RTE_ARCH_I686 || defined RTE_ARCH_X86_64 +#if defined RTE_ARCH_X86 if (likely(crc32_alg & CRC32_SSE42)) return crc32c_sse42_u8(data, init_val); #endif @@ -532,7 +524,7 @@ rte_hash_crc_1byte(uint8_t data, uint32_t init_val) static inline uint32_t rte_hash_crc_2byte(uint16_t data, uint32_t init_val) { -#if defined RTE_ARCH_I686 || defined RTE_ARCH_X86_64 +#if defined RTE_ARCH_X86 if (likely(crc32_alg & CRC32_SSE42)) return crc32c_sse42_u16(data, init_val); #endif @@ -555,7 +547,7 @@ rte_hash_crc_2byte(uint16_t data, uint32_t init_val) static inline uint32_t rte_hash_crc_4byte(uint32_t data, uint32_t init_val) { -#if defined RTE_ARCH_I686 || defined RTE_ARCH_X86_64 +#if defined RTE_ARCH_X86 if (likely(crc32_alg & CRC32_SSE42)) return crc32c_sse42_u32(data, init_val); #endif @@ -583,7 +575,7 @@ rte_hash_crc_8byte(uint64_t data, uint32_t init_val) return crc32c_sse42_u64(data, init_val); #endif -#if defined RTE_ARCH_I686 || defined RTE_ARCH_X86_64 +#if defined RTE_ARCH_X86 if (likely(crc32_alg & CRC32_SSE42)) return crc32c_sse42_u64_mimic(data, init_val); #endif diff --git a/dpdk/lib/librte_hash/rte_jhash.h b/dpdk/lib/librte_hash/rte_jhash.h index 207478c2..42c45685 100644 --- a/dpdk/lib/librte_hash/rte_jhash.h +++ b/dpdk/lib/librte_hash/rte_jhash.h @@ -48,6 +48,7 @@ extern "C" { #include #include +#include #include #include @@ -290,7 +291,10 @@ rte_jhash_32b_2hashes(const uint32_t *k, uint32_t length, uint32_t *pc, uint32_t /** * The most generic version, hashes an arbitrary sequence * of bytes. No alignment or length assumptions are made about - * the input key. + * the input key. For keys not aligned to four byte boundaries + * or a multiple of four bytes in length, the memory region + * just after may be read (but not used in the computation). + * This may cross a page boundary. * * @param key * Key to calculate hash of. diff --git a/dpdk/lib/librte_hash/rte_thash.h b/dpdk/lib/librte_hash/rte_thash.h index d98e98e7..a6ddb7bf 100644 --- a/dpdk/lib/librte_hash/rte_thash.h +++ b/dpdk/lib/librte_hash/rte_thash.h @@ -53,13 +53,15 @@ extern "C" { #include #include +#include #include +#include -#ifdef __SSE3__ +#if defined(RTE_ARCH_X86) || defined(RTE_MACHINE_CPUFLAG_NEON) #include #endif -#ifdef __SSE3__ +#ifdef RTE_ARCH_X86 /* Byte swap mask used for converting IPv6 address * 4-byte chunks to CPU byte order */ @@ -102,6 +104,7 @@ static const __m128i rte_thash_ipv6_bswap_mask = { struct rte_ipv4_tuple { uint32_t src_addr; uint32_t dst_addr; + RTE_STD_C11 union { struct { uint16_t dport; @@ -119,6 +122,7 @@ struct rte_ipv4_tuple { struct rte_ipv6_tuple { uint8_t src_addr[16]; uint8_t dst_addr[16]; + RTE_STD_C11 union { struct { uint16_t dport; @@ -131,7 +135,7 @@ struct rte_ipv6_tuple { union rte_thash_tuple { struct rte_ipv4_tuple v4; struct rte_ipv6_tuple v6; -#ifdef __SSE3__ +#ifdef RTE_ARCH_X86 } __attribute__((aligned(XMM_SIZE))); #else }; @@ -166,13 +170,18 @@ rte_convert_rss_key(const uint32_t *orig, uint32_t *targ, int len) static inline void rte_thash_load_v6_addrs(const struct ipv6_hdr *orig, union rte_thash_tuple *targ) { -#ifdef __SSE3__ +#ifdef RTE_ARCH_X86 __m128i ipv6 = _mm_loadu_si128((const __m128i *)orig->src_addr); *(__m128i *)targ->v6.src_addr = _mm_shuffle_epi8(ipv6, rte_thash_ipv6_bswap_mask); ipv6 = _mm_loadu_si128((const __m128i *)orig->dst_addr); *(__m128i *)targ->v6.dst_addr = _mm_shuffle_epi8(ipv6, rte_thash_ipv6_bswap_mask); +#elif defined(RTE_MACHINE_CPUFLAG_NEON) + uint8x16_t ipv6 = vld1q_u8((uint8_t const *)orig->src_addr); + vst1q_u8((uint8_t *)targ->v6.src_addr, vrev32q_u8(ipv6)); + ipv6 = vld1q_u8((uint8_t const *)orig->dst_addr); + vst1q_u8((uint8_t *)targ->v6.dst_addr, vrev32q_u8(ipv6)); #else int i; for (i = 0; i < 4; i++) { @@ -199,15 +208,14 @@ static inline uint32_t rte_softrss(uint32_t *input_tuple, uint32_t input_len, const uint8_t *rss_key) { - uint32_t i, j, ret = 0; + uint32_t i, j, map, ret = 0; for (j = 0; j < input_len; j++) { - for (i = 0; i < 32; i++) { - if (input_tuple[j] & (1 << (31 - i))) { - ret ^= rte_cpu_to_be_32(((const uint32_t *)rss_key)[j]) << i | + for (map = input_tuple[j]; map; map &= (map - 1)) { + i = rte_bsf32(map); + ret ^= rte_cpu_to_be_32(((const uint32_t *)rss_key)[j]) << (31 - i) | (uint32_t)((uint64_t)(rte_cpu_to_be_32(((const uint32_t *)rss_key)[j + 1])) >> - (32 - i)); - } + (i + 1)); } } return ret; @@ -230,14 +238,13 @@ static inline uint32_t rte_softrss_be(uint32_t *input_tuple, uint32_t input_len, const uint8_t *rss_key) { - uint32_t i, j, ret = 0; + uint32_t i, j, map, ret = 0; for (j = 0; j < input_len; j++) { - for (i = 0; i < 32; i++) { - if (input_tuple[j] & (1 << (31 - i))) { - ret ^= ((const uint32_t *)rss_key)[j] << i | - (uint32_t)((uint64_t)(((const uint32_t *)rss_key)[j + 1]) >> (32 - i)); - } + for (map = input_tuple[j]; map; map &= (map - 1)) { + i = rte_bsf32(map); + ret ^= ((const uint32_t *)rss_key)[j] << (31 - i) | + (uint32_t)((uint64_t)(((const uint32_t *)rss_key)[j + 1]) >> (i + 1)); } } return ret; diff --git a/dpdk/lib/librte_ip_frag/Makefile b/dpdk/lib/librte_ip_frag/Makefile index e97dfbd3..aff94b8c 100644 --- a/dpdk/lib/librte_ip_frag/Makefile +++ b/dpdk/lib/librte_ip_frag/Makefile @@ -36,8 +36,10 @@ LIB = librte_ip_frag.a CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) +LDLIBS += -lrte_eal -lrte_mempool -lrte_mbuf -lrte_ethdev +LDLIBS += -lrte_hash -EXPORT_MAP := rte_ipfrag_version.map +EXPORT_MAP := rte_ip_frag_version.map LIBABIVER := 1 @@ -52,9 +54,4 @@ SRCS-$(CONFIG_RTE_LIBRTE_IP_FRAG) += ip_frag_internal.c # install this header file SYMLINK-$(CONFIG_RTE_LIBRTE_IP_FRAG)-include += rte_ip_frag.h -DEPDIRS-$(CONFIG_RTE_LIBRTE_IP_FRAG) += lib/librte_eal -DEPDIRS-$(CONFIG_RTE_LIBRTE_IP_FRAG) += lib/librte_ether -DEPDIRS-$(CONFIG_RTE_LIBRTE_IP_FRAG) += lib/librte_mbuf -DEPDIRS-$(CONFIG_RTE_LIBRTE_IP_FRAG) += lib/librte_mempool - include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/lib/librte_ip_frag/ip_frag_common.h b/dpdk/lib/librte_ip_frag/ip_frag_common.h index 835e4f93..9f561965 100644 --- a/dpdk/lib/librte_ip_frag/ip_frag_common.h +++ b/dpdk/lib/librte_ip_frag/ip_frag_common.h @@ -130,6 +130,26 @@ ip_frag_free(struct ip_frag_pkt *fp, struct rte_ip_frag_death_row *dr) dr->cnt = k; } +/* delete fragment's mbufs immediately instead of using death row */ +static inline void +ip_frag_free_immediate(struct ip_frag_pkt *fp) +{ + uint32_t i; + + for (i = 0; i < fp->last_idx; i++) { + if (fp->frags[i].mb != NULL) { + IP_FRAG_LOG(DEBUG, "%s:%d\n" + "mbuf: %p, tms: %" PRIu64", key: <%" PRIx64 ", %#x>\n", + __func__, __LINE__, fp->frags[i].mb, fp->start, + fp->key.src_dst[0], fp->key.id); + rte_pktmbuf_free(fp->frags[i].mb); + fp->frags[i].mb = NULL; + } + } + + fp->last_idx = 0; +} + /* if key is empty, mark key as in use */ static inline void ip_frag_inuse(struct rte_ip_frag_tbl *tbl, const struct ip_frag_pkt *fp) diff --git a/dpdk/lib/librte_ip_frag/ip_frag_internal.c b/dpdk/lib/librte_ip_frag/ip_frag_internal.c index b679ff43..46c44fff 100644 --- a/dpdk/lib/librte_ip_frag/ip_frag_internal.c +++ b/dpdk/lib/librte_ip_frag/ip_frag_internal.c @@ -34,9 +34,7 @@ #include #include -#ifdef RTE_MACHINE_CPUFLAG_SSE4_2 #include -#endif /* RTE_MACHINE_CPUFLAG_SSE4_2 */ #include "ip_frag_common.h" @@ -94,14 +92,14 @@ ipv4_frag_hash(const struct ip_frag_key *key, uint32_t *v1, uint32_t *v2) p = (const uint32_t *)&key->src_dst; -#ifdef RTE_MACHINE_CPUFLAG_SSE4_2 +#ifdef RTE_ARCH_X86 v = rte_hash_crc_4byte(p[0], PRIME_VALUE); v = rte_hash_crc_4byte(p[1], v); v = rte_hash_crc_4byte(key->id, v); #else v = rte_jhash_3words(p[0], p[1], key->id, PRIME_VALUE); -#endif /* RTE_MACHINE_CPUFLAG_SSE4_2 */ +#endif /* RTE_ARCH_X86 */ *v1 = v; *v2 = (v << 7) + (v >> 14); @@ -115,7 +113,7 @@ ipv6_frag_hash(const struct ip_frag_key *key, uint32_t *v1, uint32_t *v2) p = (const uint32_t *) &key->src_dst; -#ifdef RTE_MACHINE_CPUFLAG_SSE4_2 +#ifdef RTE_ARCH_X86 v = rte_hash_crc_4byte(p[0], PRIME_VALUE); v = rte_hash_crc_4byte(p[1], v); v = rte_hash_crc_4byte(p[2], v); @@ -130,7 +128,7 @@ ipv6_frag_hash(const struct ip_frag_key *key, uint32_t *v1, uint32_t *v2) v = rte_jhash_3words(p[0], p[1], p[2], PRIME_VALUE); v = rte_jhash_3words(p[3], p[4], p[5], v); v = rte_jhash_3words(p[6], p[7], key->id, v); -#endif /* RTE_MACHINE_CPUFLAG_SSE4_2 */ +#endif /* RTE_ARCH_X86 */ *v1 = v; *v2 = (v << 7) + (v >> 14); @@ -162,7 +160,7 @@ ip_frag_process(struct ip_frag_pkt *fp, struct rte_ip_frag_death_row *dr, } /* - * errorneous packet: either exceeed max allowed number of fragments, + * erroneous packet: either exceed max allowed number of fragments, * or duplicate first/last fragment encountered. */ if (idx >= sizeof (fp->frags) / sizeof (fp->frags[0])) { diff --git a/dpdk/lib/librte_ip_frag/rte_ip_frag.h b/dpdk/lib/librte_ip_frag/rte_ip_frag.h index 9ac7081c..01441641 100644 --- a/dpdk/lib/librte_ip_frag/rte_ip_frag.h +++ b/dpdk/lib/librte_ip_frag/rte_ip_frag.h @@ -48,6 +48,7 @@ extern "C" { #include #include +#include #include #include #include @@ -70,7 +71,7 @@ struct ip_frag { struct rte_mbuf *mb; /**< fragment mbuf */ }; -/** @internal to uniquely indetify fragmented datagram. */ +/** @internal to uniquely identify fragmented datagram. */ struct ip_frag_key { uint64_t src_dst[4]; /**< src address, first 8 bytes used for IPv4 */ uint32_t id; /**< dst address */ @@ -118,13 +119,13 @@ struct rte_ip_frag_tbl { uint32_t entry_mask; /**< hash value mask. */ uint32_t max_entries; /**< max entries allowed. */ uint32_t use_entries; /**< entries in use. */ - uint32_t bucket_entries; /**< hash assocaitivity. */ + uint32_t bucket_entries; /**< hash associativity. */ uint32_t nb_entries; /**< total size of the table. */ uint32_t nb_buckets; /**< num of associativity lines. */ struct ip_frag_pkt *last; /**< last used entry. */ struct ip_pkt_list lru; /**< LRU list for table entries. */ struct ip_frag_tbl_stat stat; /**< statistics counters. */ - struct ip_frag_pkt pkt[0]; /**< hash table. */ + __extension__ struct ip_frag_pkt pkt[0]; /**< hash table. */ }; /** IPv6 fragment extension header */ @@ -180,11 +181,8 @@ struct rte_ip_frag_tbl * rte_ip_frag_table_create(uint32_t bucket_num, * @param tbl * Fragmentation table to free. */ -static inline void -rte_ip_frag_table_destroy(struct rte_ip_frag_tbl *tbl) -{ - rte_free(tbl); -} +void +rte_ip_frag_table_destroy(struct rte_ip_frag_tbl *tbl); /** * This function implements the fragmentation of IPv6 packets. @@ -233,7 +231,7 @@ rte_ipv6_fragment_packet(struct rte_mbuf *pkt_in, * Pointer to the IPv6 fragment extension header. * @return * Pointer to mbuf for reassembled packet, or NULL if: - * - an error occured. + * - an error occurred. * - not all fragments of the packet are collected yet. */ struct rte_mbuf *rte_ipv6_frag_reassemble_packet(struct rte_ip_frag_tbl *tbl, @@ -306,8 +304,8 @@ int32_t rte_ipv4_fragment_packet(struct rte_mbuf *pkt_in, * @param ip_hdr * Pointer to the IPV4 header inside the fragment. * @return - * Pointer to mbuf for reassebled packet, or NULL if: - * - an error occured. + * Pointer to mbuf for reassembled packet, or NULL if: + * - an error occurred. * - not all fragments of the packet are collected yet. */ struct rte_mbuf * rte_ipv4_frag_reassemble_packet(struct rte_ip_frag_tbl *tbl, diff --git a/dpdk/lib/librte_ip_frag/rte_ip_frag_common.c b/dpdk/lib/librte_ip_frag/rte_ip_frag_common.c index 6176ff4e..8460f8e8 100644 --- a/dpdk/lib/librte_ip_frag/rte_ip_frag_common.c +++ b/dpdk/lib/librte_ip_frag/rte_ip_frag_common.c @@ -109,6 +109,19 @@ rte_ip_frag_table_create(uint32_t bucket_num, uint32_t bucket_entries, return tbl; } +/* delete fragmentation table */ +void +rte_ip_frag_table_destroy(struct rte_ip_frag_tbl *tbl) +{ + struct ip_frag_pkt *fp; + + TAILQ_FOREACH(fp, &tbl->lru, lru) { + ip_frag_free_immediate(fp); + } + + rte_free(tbl); +} + /* dump frag table statistics to file */ void rte_ip_frag_table_statistics_dump(FILE *f, const struct rte_ip_frag_tbl *tbl) diff --git a/dpdk/lib/librte_ip_frag/rte_ip_frag_version.map b/dpdk/lib/librte_ip_frag/rte_ip_frag_version.map new file mode 100644 index 00000000..d1acf07c --- /dev/null +++ b/dpdk/lib/librte_ip_frag/rte_ip_frag_version.map @@ -0,0 +1,20 @@ +DPDK_2.0 { + global: + + rte_ip_frag_free_death_row; + rte_ip_frag_table_create; + rte_ip_frag_table_statistics_dump; + rte_ipv4_frag_reassemble_packet; + rte_ipv4_fragment_packet; + rte_ipv6_frag_reassemble_packet; + rte_ipv6_fragment_packet; + + local: *; +}; + +DPDK_17.08 { + global: + + rte_ip_frag_table_destroy; + +} DPDK_2.0; diff --git a/dpdk/lib/librte_ip_frag/rte_ipfrag_version.map b/dpdk/lib/librte_ip_frag/rte_ipfrag_version.map deleted file mode 100644 index 354fa082..00000000 --- a/dpdk/lib/librte_ip_frag/rte_ipfrag_version.map +++ /dev/null @@ -1,13 +0,0 @@ -DPDK_2.0 { - global: - - rte_ip_frag_free_death_row; - rte_ip_frag_table_create; - rte_ip_frag_table_statistics_dump; - rte_ipv4_frag_reassemble_packet; - rte_ipv4_fragment_packet; - rte_ipv6_frag_reassemble_packet; - rte_ipv6_fragment_packet; - - local: *; -}; diff --git a/dpdk/lib/librte_ip_frag/rte_ipv4_fragmentation.c b/dpdk/lib/librte_ip_frag/rte_ipv4_fragmentation.c index a2259e80..8c5f5ec4 100644 --- a/dpdk/lib/librte_ip_frag/rte_ipv4_fragmentation.c +++ b/dpdk/lib/librte_ip_frag/rte_ipv4_fragmentation.c @@ -48,7 +48,7 @@ #define IPV4_HDR_DF_MASK (1 << IPV4_HDR_DF_SHIFT) #define IPV4_HDR_MF_MASK (1 << IPV4_HDR_MF_SHIFT) -#define IPV4_HDR_FO_MASK ((1 << IPV4_HDR_FO_SHIFT) - 1) +#define IPV4_HDR_FO_ALIGN (1 << IPV4_HDR_FO_SHIFT) static inline void __fill_ipv4hdr_frag(struct ipv4_hdr *dst, const struct ipv4_hdr *src, uint16_t len, uint16_t fofs, @@ -103,11 +103,14 @@ rte_ipv4_fragment_packet(struct rte_mbuf *pkt_in, uint32_t out_pkt_pos, in_seg_data_pos; uint32_t more_in_segs; uint16_t fragment_offset, flag_offset, frag_size; + uint16_t frag_bytes_remaining; - frag_size = (uint16_t)(mtu_size - sizeof(struct ipv4_hdr)); - - /* Fragment size should be a multiply of 8. */ - RTE_ASSERT((frag_size & IPV4_HDR_FO_MASK) == 0); + /* + * Ensure the IP payload length of all fragments is aligned to a + * multiple of 8 bytes as per RFC791 section 2.3. + */ + frag_size = RTE_ALIGN_FLOOR((mtu_size - sizeof(struct ipv4_hdr)), + IPV4_HDR_FO_ALIGN); in_hdr = rte_pktmbuf_mtod(pkt_in, struct ipv4_hdr *); flag_offset = rte_cpu_to_be_16(in_hdr->fragment_offset); @@ -142,6 +145,7 @@ rte_ipv4_fragment_packet(struct rte_mbuf *pkt_in, /* Reserve space for the IP header that will be built later */ out_pkt->data_len = sizeof(struct ipv4_hdr); out_pkt->pkt_len = sizeof(struct ipv4_hdr); + frag_bytes_remaining = frag_size; out_seg_prev = out_pkt; more_out_segs = 1; @@ -161,7 +165,7 @@ rte_ipv4_fragment_packet(struct rte_mbuf *pkt_in, /* Prepare indirect buffer */ rte_pktmbuf_attach(out_seg, in_seg); - len = mtu_size - out_pkt->pkt_len; + len = frag_bytes_remaining; if (len > (in_seg->data_len - in_seg_data_pos)) { len = in_seg->data_len - in_seg_data_pos; } @@ -171,9 +175,10 @@ rte_ipv4_fragment_packet(struct rte_mbuf *pkt_in, out_pkt->pkt_len); out_pkt->nb_segs += 1; in_seg_data_pos += len; + frag_bytes_remaining -= len; /* Current output packet (i.e. fragment) done ? */ - if (unlikely(out_pkt->pkt_len >= mtu_size)) + if (unlikely(frag_bytes_remaining == 0)) more_out_segs = 0; /* Current input segment done ? */ diff --git a/dpdk/lib/librte_ip_frag/rte_ipv4_reassembly.c b/dpdk/lib/librte_ip_frag/rte_ipv4_reassembly.c index e084ca59..040bd70a 100644 --- a/dpdk/lib/librte_ip_frag/rte_ipv4_reassembly.c +++ b/dpdk/lib/librte_ip_frag/rte_ipv4_reassembly.c @@ -93,7 +93,7 @@ ipv4_frag_reassemble(struct ip_frag_pkt *fp) /* update mbuf fields for reassembled packet. */ m->ol_flags |= PKT_TX_IP_CKSUM; - /* update ipv4 header for the reassmebled packet */ + /* update ipv4 header for the reassembled packet */ ip_hdr = rte_pktmbuf_mtod_offset(m, struct ipv4_hdr *, m->l2_len); ip_hdr->total_length = rte_cpu_to_be_16((uint16_t)(fp->total_size + @@ -117,8 +117,8 @@ ipv4_frag_reassemble(struct ip_frag_pkt *fp) * @param ip_hdr * Pointer to the IPV4 header inside the fragment. * @return - * Pointer to mbuf for reassebled packet, or NULL if: - * - an error occured. + * Pointer to mbuf for reassembled packet, or NULL if: + * - an error occurred. * - not all fragments of the packet are collected yet. */ struct rte_mbuf * diff --git a/dpdk/lib/librte_ip_frag/rte_ipv6_reassembly.c b/dpdk/lib/librte_ip_frag/rte_ipv6_reassembly.c index 21a5ef5d..dde58cb7 100644 --- a/dpdk/lib/librte_ip_frag/rte_ipv6_reassembly.c +++ b/dpdk/lib/librte_ip_frag/rte_ipv6_reassembly.c @@ -155,7 +155,7 @@ ipv6_frag_reassemble(struct ip_frag_pkt *fp) * Pointer to the IPV6 fragment extension header. * @return * Pointer to mbuf for reassembled packet, or NULL if: - * - an error occured. + * - an error occurred. * - not all fragments of the packet are collected yet. */ #define MORE_FRAGS(x) (((x) & 0x100) >> 8) diff --git a/dpdk/lib/librte_ivshmem/Makefile b/dpdk/lib/librte_ivshmem/Makefile deleted file mode 100644 index c099438c..00000000 --- a/dpdk/lib/librte_ivshmem/Makefile +++ /dev/null @@ -1,54 +0,0 @@ -# BSD LICENSE -# -# Copyright(c) 2010-2014 Intel Corporation. All rights reserved. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in -# the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Intel Corporation nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -include $(RTE_SDK)/mk/rte.vars.mk - -# library name -LIB = librte_ivshmem.a - -CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -O3 - -EXPORT_MAP := rte_ivshmem_version.map - -LIBABIVER := 1 - -# all source are stored in SRCS-y -SRCS-$(CONFIG_RTE_LIBRTE_IVSHMEM) := rte_ivshmem.c - -# install includes -SYMLINK-$(CONFIG_RTE_LIBRTE_IVSHMEM)-include := rte_ivshmem.h - -# this lib needs EAL, ring and mempool -DEPDIRS-$(CONFIG_RTE_LIBRTE_IVSHMEM) += lib/librte_eal -DEPDIRS-$(CONFIG_RTE_LIBRTE_IVSHMEM) += lib/librte_ring -DEPDIRS-$(CONFIG_RTE_LIBRTE_IVSHMEM) += lib/librte_mempool - -include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/lib/librte_ivshmem/rte_ivshmem.c b/dpdk/lib/librte_ivshmem/rte_ivshmem.c deleted file mode 100644 index c26edb61..00000000 --- a/dpdk/lib/librte_ivshmem/rte_ivshmem.c +++ /dev/null @@ -1,919 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "rte_ivshmem.h" - -#define IVSHMEM_CONFIG_FILE_FMT "/var/run/.dpdk_ivshmem_metadata_%s" -#define IVSHMEM_QEMU_CMD_LINE_HEADER_FMT "-device ivshmem,size=%" PRIu64 "M,shm=fd%s" -#define IVSHMEM_QEMU_CMD_FD_FMT ":%s:0x%" PRIx64 ":0x%" PRIx64 -#define IVSHMEM_QEMU_CMDLINE_BUFSIZE 1024 -#define IVSHMEM_MAX_PAGES (1 << 12) -#define adjacent(x,y) (((x).phys_addr+(x).len)==(y).phys_addr) -#define METADATA_SIZE_ALIGNED \ - (RTE_ALIGN_CEIL(sizeof(struct rte_ivshmem_metadata),pagesz)) - -#define GET_PAGEMAP_ADDR(in,addr,dlm,err) \ -{ \ - char *end; \ - errno = 0; \ - addr = strtoull((in), &end, 16); \ - if (errno != 0 || *end != (dlm)) { \ - RTE_LOG(ERR, EAL, err); \ - goto error; \ - } \ - (in) = end + 1; \ -} - -static int pagesz; - -struct memseg_cache_entry { - char filepath[PATH_MAX]; - uint64_t offset; - uint64_t len; -}; - -struct ivshmem_config { - struct rte_ivshmem_metadata * metadata; - struct memseg_cache_entry memseg_cache[IVSHMEM_MAX_PAGES]; - /**< account for multiple files per segment case */ - struct flock lock; - rte_spinlock_t sl; -}; - -static struct ivshmem_config -ivshmem_global_config[RTE_LIBRTE_IVSHMEM_MAX_METADATA_FILES]; - -static rte_spinlock_t global_cfg_sl; - -static struct ivshmem_config * -get_config_by_name(const char * name) -{ - struct rte_ivshmem_metadata * config; - unsigned i; - - for (i = 0; i < RTE_DIM(ivshmem_global_config); i++) { - config = ivshmem_global_config[i].metadata; - if (config == NULL) - return NULL; - if (strncmp(name, config->name, IVSHMEM_NAME_LEN) == 0) - return &ivshmem_global_config[i]; - } - - return NULL; -} - -static int -overlap(const struct rte_memzone * s1, const struct rte_memzone * s2) -{ - uint64_t start1, end1, start2, end2; - - start1 = s1->addr_64; - end1 = s1->addr_64 + s1->len; - start2 = s2->addr_64; - end2 = s2->addr_64 + s2->len; - - if (start1 >= start2 && start1 < end2) - return 1; - if (start2 >= start1 && start2 < end1) - return 1; - - return 0; -} - -static struct rte_memzone * -get_memzone_by_addr(const void * addr) -{ - struct rte_memzone * tmp, * mz; - struct rte_mem_config * mcfg; - int i; - - mcfg = rte_eal_get_configuration()->mem_config; - mz = NULL; - - /* find memzone for the ring */ - for (i = 0; i < RTE_MAX_MEMZONE; i++) { - tmp = &mcfg->memzone[i]; - - if (tmp->addr_64 == (uint64_t) addr) { - mz = tmp; - break; - } - } - - return mz; -} - -static int -entry_compare(const void * a, const void * b) -{ - const struct rte_ivshmem_metadata_entry * e1 = - (const struct rte_ivshmem_metadata_entry*) a; - const struct rte_ivshmem_metadata_entry * e2 = - (const struct rte_ivshmem_metadata_entry*) b; - - /* move unallocated zones to the end */ - if (e1->mz.addr == NULL && e2->mz.addr == NULL) - return 0; - if (e1->mz.addr == 0) - return 1; - if (e2->mz.addr == 0) - return -1; - - return e1->mz.phys_addr > e2->mz.phys_addr; -} - -/* fills hugepage cache entry for a given start virt_addr */ -static int -get_hugefile_by_virt_addr(uint64_t virt_addr, struct memseg_cache_entry * e) -{ - uint64_t start_addr, end_addr; - char *start,*path_end; - char buf[PATH_MAX*2]; - FILE *f; - - start = NULL; - path_end = NULL; - start_addr = 0; - - memset(e->filepath, 0, sizeof(e->filepath)); - - /* open /proc/self/maps */ - f = fopen("/proc/self/maps", "r"); - if (f == NULL) { - RTE_LOG(ERR, EAL, "cannot open /proc/self/maps!\n"); - return -1; - } - - /* parse maps */ - while (fgets(buf, sizeof(buf), f) != NULL) { - - /* get endptr to end of start addr */ - start = buf; - - GET_PAGEMAP_ADDR(start,start_addr,'-', - "Cannot find start address in maps!\n"); - - /* if start address is bigger than our address, skip */ - if (start_addr > virt_addr) - continue; - - GET_PAGEMAP_ADDR(start,end_addr,' ', - "Cannot find end address in maps!\n"); - - /* if end address is less than our address, skip */ - if (end_addr <= virt_addr) - continue; - - /* find where the path starts */ - start = strstr(start, "/"); - - if (start == NULL) - continue; - - /* at this point, we know that this is our map. - * now let's find the file */ - path_end = strstr(start, "\n"); - break; - } - - if (path_end == NULL) { - RTE_LOG(ERR, EAL, "Hugefile path not found!\n"); - goto error; - } - - /* calculate offset and copy the file path */ - snprintf(e->filepath, RTE_PTR_DIFF(path_end, start) + 1, "%s", start); - - e->offset = virt_addr - start_addr; - - fclose(f); - - return 0; -error: - fclose(f); - return -1; -} - -/* - * This is a complex function. What it does is the following: - * 1. Goes through metadata and gets list of hugepages involved - * 2. Sorts the hugepages by size (1G first) - * 3. Goes through metadata again and writes correct offsets - * 4. Goes through pages and finds out their filenames, offsets etc. - */ -static int -build_config(struct rte_ivshmem_metadata * metadata) -{ - struct rte_ivshmem_metadata_entry * e_local; - struct memseg_cache_entry * ms_local; - struct rte_memseg pages[IVSHMEM_MAX_PAGES]; - struct rte_ivshmem_metadata_entry *entry; - struct memseg_cache_entry * c_entry, * prev_entry; - struct ivshmem_config * config; - unsigned i, j, mz_iter, ms_iter; - uint64_t biggest_len; - int biggest_idx; - - /* return error if we try to use an unknown config file */ - config = get_config_by_name(metadata->name); - if (config == NULL) { - RTE_LOG(ERR, EAL, "Cannot find IVSHMEM config %s!\n", metadata->name); - goto fail_e; - } - - memset(pages, 0, sizeof(pages)); - - e_local = malloc(sizeof(config->metadata->entry)); - if (e_local == NULL) - goto fail_e; - ms_local = malloc(sizeof(config->memseg_cache)); - if (ms_local == NULL) - goto fail_ms; - - - /* make local copies before doing anything */ - memcpy(e_local, config->metadata->entry, sizeof(config->metadata->entry)); - memcpy(ms_local, config->memseg_cache, sizeof(config->memseg_cache)); - - qsort(e_local, RTE_DIM(config->metadata->entry), sizeof(struct rte_ivshmem_metadata_entry), - entry_compare); - - /* first pass - collect all huge pages */ - for (mz_iter = 0; mz_iter < RTE_DIM(config->metadata->entry); mz_iter++) { - - entry = &e_local[mz_iter]; - - uint64_t start_addr = RTE_ALIGN_FLOOR(entry->mz.addr_64, - entry->mz.hugepage_sz); - uint64_t offset = entry->mz.addr_64 - start_addr; - uint64_t len = RTE_ALIGN_CEIL(entry->mz.len + offset, - entry->mz.hugepage_sz); - - if (entry->mz.addr_64 == 0 || start_addr == 0 || len == 0) - continue; - - int start_page; - - /* find first unused page - mz are phys_addr sorted so we don't have to - * look out for holes */ - for (i = 0; i < RTE_DIM(pages); i++) { - - /* skip if we already have this page */ - if (pages[i].addr_64 == start_addr) { - start_addr += entry->mz.hugepage_sz; - len -= entry->mz.hugepage_sz; - continue; - } - /* we found a new page */ - else if (pages[i].addr_64 == 0) { - start_page = i; - break; - } - } - if (i == RTE_DIM(pages)) { - RTE_LOG(ERR, EAL, "Cannot find unused page!\n"); - goto fail; - } - - /* populate however many pages the memzone has */ - for (i = start_page; i < RTE_DIM(pages) && len != 0; i++) { - - pages[i].addr_64 = start_addr; - pages[i].len = entry->mz.hugepage_sz; - start_addr += entry->mz.hugepage_sz; - len -= entry->mz.hugepage_sz; - } - /* if there's still length left */ - if (len != 0) { - RTE_LOG(ERR, EAL, "Not enough space for pages!\n"); - goto fail; - } - } - - /* second pass - sort pages by size */ - for (i = 0; i < RTE_DIM(pages); i++) { - - if (pages[i].addr == NULL) - break; - - biggest_len = 0; - biggest_idx = -1; - - /* - * browse all entries starting at 'i', and find the - * entry with the smallest addr - */ - for (j=i; j< RTE_DIM(pages); j++) { - if (pages[j].addr == NULL) - break; - if (biggest_len == 0 || - pages[j].len > biggest_len) { - biggest_len = pages[j].len; - biggest_idx = j; - } - } - - /* should not happen */ - if (biggest_idx == -1) { - RTE_LOG(ERR, EAL, "Error sorting by size!\n"); - goto fail; - } - if (i != (unsigned) biggest_idx) { - struct rte_memseg tmp; - - memcpy(&tmp, &pages[biggest_idx], sizeof(struct rte_memseg)); - - /* we don't want to break contiguousness, so instead of just - * swapping segments, we move all the preceding segments to the - * right and then put the old segment @ biggest_idx in place of - * segment @ i */ - for (j = biggest_idx - 1; j >= i; j--) { - memcpy(&pages[j+1], &pages[j], sizeof(struct rte_memseg)); - memset(&pages[j], 0, sizeof(struct rte_memseg)); - if (j == 0) - break; - } - - /* put old biggest segment to its new place */ - memcpy(&pages[i], &tmp, sizeof(struct rte_memseg)); - } - } - - /* third pass - write correct offsets */ - for (mz_iter = 0; mz_iter < RTE_DIM(config->metadata->entry); mz_iter++) { - - uint64_t offset = 0; - - entry = &e_local[mz_iter]; - - if (entry->mz.addr_64 == 0) - break; - - /* find page for current memzone */ - for (i = 0; i < RTE_DIM(pages); i++) { - /* we found our page */ - if (entry->mz.addr_64 >= pages[i].addr_64 && - entry->mz.addr_64 < pages[i].addr_64 + pages[i].len) { - entry->offset = (entry->mz.addr_64 - pages[i].addr_64) + - offset; - break; - } - offset += pages[i].len; - } - if (i == RTE_DIM(pages)) { - RTE_LOG(ERR, EAL, "Page not found!\n"); - goto fail; - } - } - - ms_iter = 0; - prev_entry = NULL; - - /* fourth pass - create proper memseg cache */ - for (i = 0; i < RTE_DIM(pages) && - ms_iter <= RTE_DIM(config->memseg_cache); i++) { - if (pages[i].addr_64 == 0) - break; - - - if (ms_iter == RTE_DIM(pages)) { - RTE_LOG(ERR, EAL, "The universe has collapsed!\n"); - goto fail; - } - - c_entry = &ms_local[ms_iter]; - c_entry->len = pages[i].len; - - if (get_hugefile_by_virt_addr(pages[i].addr_64, c_entry) < 0) - goto fail; - - /* if previous entry has the same filename and is contiguous, - * clear current entry and increase previous entry's length - */ - if (prev_entry != NULL && - strncmp(c_entry->filepath, prev_entry->filepath, - sizeof(c_entry->filepath)) == 0 && - prev_entry->offset + prev_entry->len == c_entry->offset) { - prev_entry->len += pages[i].len; - memset(c_entry, 0, sizeof(struct memseg_cache_entry)); - } - else { - prev_entry = c_entry; - ms_iter++; - } - } - - /* update current configuration with new valid data */ - memcpy(config->metadata->entry, e_local, sizeof(config->metadata->entry)); - memcpy(config->memseg_cache, ms_local, sizeof(config->memseg_cache)); - - free(ms_local); - free(e_local); - - return 0; -fail: - free(ms_local); -fail_ms: - free(e_local); -fail_e: - return -1; -} - -static int -add_memzone_to_metadata(const struct rte_memzone * mz, - struct ivshmem_config * config) -{ - struct rte_ivshmem_metadata_entry * entry; - unsigned i, idx; - struct rte_mem_config *mcfg; - - if (mz->len == 0) { - RTE_LOG(ERR, EAL, "Trying to add an empty memzone\n"); - return -1; - } - - rte_spinlock_lock(&config->sl); - - mcfg = rte_eal_get_configuration()->mem_config; - - /* it prevents the memzone being freed while we add it to the metadata */ - rte_rwlock_write_lock(&mcfg->mlock); - - /* find free slot in this config */ - for (i = 0; i < RTE_DIM(config->metadata->entry); i++) { - entry = &config->metadata->entry[i]; - - if (&entry->mz.addr_64 != 0 && overlap(mz, &entry->mz)) { - RTE_LOG(ERR, EAL, "Overlapping memzones!\n"); - goto fail; - } - - /* if addr is zero, the memzone is probably free */ - if (entry->mz.addr_64 == 0) { - RTE_LOG(DEBUG, EAL, "Adding memzone '%s' at %p to metadata %s\n", - mz->name, mz->addr, config->metadata->name); - memcpy(&entry->mz, mz, sizeof(struct rte_memzone)); - - /* run config file parser */ - if (build_config(config->metadata) < 0) - goto fail; - - break; - } - } - - /* if we reached the maximum, that means we have no place in config */ - if (i == RTE_DIM(config->metadata->entry)) { - RTE_LOG(ERR, EAL, "No space left in IVSHMEM metadata %s!\n", - config->metadata->name); - goto fail; - } - - idx = ((uintptr_t)mz - (uintptr_t)mcfg->memzone); - idx = idx / sizeof(struct rte_memzone); - - /* mark the memzone not freeable */ - mcfg->memzone[idx].ioremap_addr = mz->phys_addr; - - rte_rwlock_write_unlock(&mcfg->mlock); - rte_spinlock_unlock(&config->sl); - return 0; -fail: - rte_rwlock_write_unlock(&mcfg->mlock); - rte_spinlock_unlock(&config->sl); - return -1; -} - -static int -add_ring_to_metadata(const struct rte_ring * r, - struct ivshmem_config * config) -{ - struct rte_memzone * mz; - - mz = get_memzone_by_addr(r); - - if (!mz) { - RTE_LOG(ERR, EAL, "Cannot find memzone for ring!\n"); - return -1; - } - - return add_memzone_to_metadata(mz, config); -} - -static int -add_mempool_memzone_to_metadata(const void *addr, - struct ivshmem_config *config) -{ - struct rte_memzone *mz; - - mz = get_memzone_by_addr(addr); - - if (!mz) { - RTE_LOG(ERR, EAL, "Cannot find memzone for mempool!\n"); - return -1; - } - - return add_memzone_to_metadata(mz, config); -} - -static int -add_mempool_to_metadata(const struct rte_mempool *mp, - struct ivshmem_config *config) -{ - struct rte_mempool_memhdr *memhdr; - int ret; - - ret = add_mempool_memzone_to_metadata(mp, config); - if (ret < 0) - return -1; - - STAILQ_FOREACH(memhdr, &mp->mem_list, next) { - ret = add_mempool_memzone_to_metadata(memhdr->addr, config); - if (ret < 0) - return -1; - } - - /* mempool consists of memzone and ring */ - return add_ring_to_metadata(mp->pool_data, config); -} - -int -rte_ivshmem_metadata_add_ring(const struct rte_ring * r, const char * name) -{ - struct ivshmem_config * config; - - if (name == NULL || r == NULL) - return -1; - - config = get_config_by_name(name); - - if (config == NULL) { - RTE_LOG(ERR, EAL, "Cannot find IVSHMEM config %s!\n", name); - return -1; - } - - return add_ring_to_metadata(r, config); -} - -int -rte_ivshmem_metadata_add_memzone(const struct rte_memzone * mz, const char * name) -{ - struct ivshmem_config * config; - - if (name == NULL || mz == NULL) - return -1; - - config = get_config_by_name(name); - - if (config == NULL) { - RTE_LOG(ERR, EAL, "Cannot find IVSHMEM config %s!\n", name); - return -1; - } - - return add_memzone_to_metadata(mz, config); -} - -int -rte_ivshmem_metadata_add_mempool(const struct rte_mempool * mp, const char * name) -{ - struct ivshmem_config * config; - - if (name == NULL || mp == NULL) - return -1; - - config = get_config_by_name(name); - - if (config == NULL) { - RTE_LOG(ERR, EAL, "Cannot find IVSHMEM config %s!\n", name); - return -1; - } - - return add_mempool_to_metadata(mp, config); -} - -static inline void -ivshmem_config_path(char *buffer, size_t bufflen, const char *name) -{ - snprintf(buffer, bufflen, IVSHMEM_CONFIG_FILE_FMT, name); -} - - - -static inline -void *ivshmem_metadata_create(const char *name, size_t size, - struct flock *lock) -{ - int retval, fd; - void *metadata_addr; - char pathname[PATH_MAX]; - - ivshmem_config_path(pathname, sizeof(pathname), name); - - fd = open(pathname, O_RDWR | O_CREAT, 0660); - if (fd < 0) { - RTE_LOG(ERR, EAL, "Cannot open '%s'\n", pathname); - return NULL; - } - - size = METADATA_SIZE_ALIGNED; - - retval = fcntl(fd, F_SETLK, lock); - if (retval < 0){ - close(fd); - RTE_LOG(ERR, EAL, "Cannot create lock on '%s'. Is another " - "process using it?\n", pathname); - return NULL; - } - - retval = ftruncate(fd, size); - if (retval < 0){ - close(fd); - RTE_LOG(ERR, EAL, "Cannot resize '%s'\n", pathname); - return NULL; - } - - metadata_addr = mmap(NULL, size, - PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); - - if (metadata_addr == MAP_FAILED){ - RTE_LOG(ERR, EAL, "Cannot mmap memory for '%s'\n", pathname); - - /* we don't care if we can't unlock */ - fcntl(fd, F_UNLCK, lock); - close(fd); - - return NULL; - } - - return metadata_addr; -} - -int rte_ivshmem_metadata_create(const char *name) -{ - struct ivshmem_config * ivshmem_config; - unsigned index; - - if (pagesz == 0) - pagesz = getpagesize(); - - if (name == NULL) - return -1; - - rte_spinlock_lock(&global_cfg_sl); - - for (index = 0; index < RTE_DIM(ivshmem_global_config); index++) { - if (ivshmem_global_config[index].metadata == NULL) { - ivshmem_config = &ivshmem_global_config[index]; - break; - } - } - - if (index == RTE_DIM(ivshmem_global_config)) { - RTE_LOG(ERR, EAL, "Cannot create more ivshmem config files. " - "Maximum has been reached\n"); - rte_spinlock_unlock(&global_cfg_sl); - return -1; - } - - ivshmem_config->lock.l_type = F_WRLCK; - ivshmem_config->lock.l_whence = SEEK_SET; - - ivshmem_config->lock.l_start = 0; - ivshmem_config->lock.l_len = METADATA_SIZE_ALIGNED; - - ivshmem_global_config[index].metadata = ((struct rte_ivshmem_metadata *) - ivshmem_metadata_create( - name, - sizeof(struct rte_ivshmem_metadata), - &ivshmem_config->lock)); - - if (ivshmem_global_config[index].metadata == NULL) { - rte_spinlock_unlock(&global_cfg_sl); - return -1; - } - - /* Metadata setup */ - memset(ivshmem_config->metadata, 0, sizeof(struct rte_ivshmem_metadata)); - ivshmem_config->metadata->magic_number = IVSHMEM_MAGIC; - snprintf(ivshmem_config->metadata->name, - sizeof(ivshmem_config->metadata->name), "%s", name); - - rte_spinlock_unlock(&global_cfg_sl); - - return 0; -} - -int -rte_ivshmem_metadata_cmdline_generate(char *buffer, unsigned size, const char *name) -{ - const struct memseg_cache_entry * ms_cache, *entry; - struct ivshmem_config * config; - char cmdline[IVSHMEM_QEMU_CMDLINE_BUFSIZE], *cmdline_ptr; - char cfg_file_path[PATH_MAX]; - unsigned remaining_len, tmplen, iter; - uint64_t shared_mem_size, zero_size, total_size; - - if (buffer == NULL || name == NULL) - return -1; - - config = get_config_by_name(name); - - if (config == NULL) { - RTE_LOG(ERR, EAL, "Config %s not found!\n", name); - return -1; - } - - rte_spinlock_lock(&config->sl); - - /* prepare metadata file path */ - snprintf(cfg_file_path, sizeof(cfg_file_path), IVSHMEM_CONFIG_FILE_FMT, - config->metadata->name); - - ms_cache = config->memseg_cache; - - cmdline_ptr = cmdline; - remaining_len = sizeof(cmdline); - - shared_mem_size = 0; - iter = 0; - - while ((ms_cache[iter].len != 0) && (iter < RTE_DIM(config->metadata->entry))) { - - entry = &ms_cache[iter]; - - /* Offset and sizes within the current pathname */ - tmplen = snprintf(cmdline_ptr, remaining_len, IVSHMEM_QEMU_CMD_FD_FMT, - entry->filepath, entry->offset, entry->len); - - shared_mem_size += entry->len; - - cmdline_ptr = RTE_PTR_ADD(cmdline_ptr, tmplen); - remaining_len -= tmplen; - - if (remaining_len == 0) { - RTE_LOG(ERR, EAL, "Command line too long!\n"); - rte_spinlock_unlock(&config->sl); - return -1; - } - - iter++; - } - - total_size = rte_align64pow2(shared_mem_size + METADATA_SIZE_ALIGNED); - zero_size = total_size - shared_mem_size - METADATA_SIZE_ALIGNED; - - /* add /dev/zero to command-line to fill the space */ - tmplen = snprintf(cmdline_ptr, remaining_len, IVSHMEM_QEMU_CMD_FD_FMT, - "/dev/zero", - (uint64_t)0x0, - zero_size); - - cmdline_ptr = RTE_PTR_ADD(cmdline_ptr, tmplen); - remaining_len -= tmplen; - - if (remaining_len == 0) { - RTE_LOG(ERR, EAL, "Command line too long!\n"); - rte_spinlock_unlock(&config->sl); - return -1; - } - - /* add metadata file to the end of command-line */ - tmplen = snprintf(cmdline_ptr, remaining_len, IVSHMEM_QEMU_CMD_FD_FMT, - cfg_file_path, - (uint64_t)0x0, - METADATA_SIZE_ALIGNED); - - cmdline_ptr = RTE_PTR_ADD(cmdline_ptr, tmplen); - remaining_len -= tmplen; - - if (remaining_len == 0) { - RTE_LOG(ERR, EAL, "Command line too long!\n"); - rte_spinlock_unlock(&config->sl); - return -1; - } - - /* if current length of the command line is bigger than the buffer supplied - * by the user, or if command-line is bigger than what IVSHMEM accepts */ - if ((sizeof(cmdline) - remaining_len) > size) { - RTE_LOG(ERR, EAL, "Buffer is too short!\n"); - rte_spinlock_unlock(&config->sl); - return -1; - } - /* complete the command-line */ - snprintf(buffer, size, - IVSHMEM_QEMU_CMD_LINE_HEADER_FMT, - total_size >> 20, - cmdline); - - rte_spinlock_unlock(&config->sl); - - return 0; -} - -void -rte_ivshmem_metadata_dump(FILE *f, const char *name) -{ - unsigned i = 0; - struct ivshmem_config * config; - struct rte_ivshmem_metadata_entry *entry; -#ifdef RTE_LIBRTE_IVSHMEM_DEBUG - uint64_t addr; - uint64_t end, hugepage_sz; - struct memseg_cache_entry e; -#endif - - if (name == NULL) - return; - - /* return error if we try to use an unknown config file */ - config = get_config_by_name(name); - if (config == NULL) { - RTE_LOG(ERR, EAL, "Cannot find IVSHMEM config %s!\n", name); - return; - } - - rte_spinlock_lock(&config->sl); - - entry = &config->metadata->entry[0]; - - while (entry->mz.addr != NULL && i < RTE_DIM(config->metadata->entry)) { - - fprintf(f, "Entry %u: name:<%-20s>, phys:0x%-15lx, len:0x%-15lx, " - "virt:%-15p, off:0x%-15lx\n", - i, - entry->mz.name, - entry->mz.phys_addr, - entry->mz.len, - entry->mz.addr, - entry->offset); - i++; - -#ifdef RTE_LIBRTE_IVSHMEM_DEBUG - fprintf(f, "\tHugepage files:\n"); - - hugepage_sz = entry->mz.hugepage_sz; - addr = RTE_ALIGN_FLOOR(entry->mz.addr_64, hugepage_sz); - end = addr + RTE_ALIGN_CEIL(entry->mz.len + (entry->mz.addr_64 - addr), - hugepage_sz); - - for (; addr < end; addr += hugepage_sz) { - memset(&e, 0, sizeof(e)); - - get_hugefile_by_virt_addr(addr, &e); - - fprintf(f, "\t0x%"PRIx64 "-0x%" PRIx64 " offset: 0x%" PRIx64 " %s\n", - addr, addr + hugepage_sz, e.offset, e.filepath); - } -#endif - entry++; - } - - rte_spinlock_unlock(&config->sl); -} diff --git a/dpdk/lib/librte_ivshmem/rte_ivshmem.h b/dpdk/lib/librte_ivshmem/rte_ivshmem.h deleted file mode 100644 index a5d36d6b..00000000 --- a/dpdk/lib/librte_ivshmem/rte_ivshmem.h +++ /dev/null @@ -1,165 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef RTE_IVSHMEM_H_ -#define RTE_IVSHMEM_H_ - -#include -#include - -/** - * @file - * - * The RTE IVSHMEM interface provides functions to create metadata files - * describing memory segments to be shared via QEMU IVSHMEM. - */ - - -#ifdef __cplusplus -extern "C" { -#endif - -#define IVSHMEM_MAGIC 0x0BADC0DE -#define IVSHMEM_NAME_LEN 32 - -/** - * Structure that holds IVSHMEM shared metadata entry. - */ -struct rte_ivshmem_metadata_entry { - struct rte_memzone mz; /**< shared memzone */ - uint64_t offset; /**< offset of memzone within IVSHMEM device */ -}; - -/** - * Structure that holds IVSHMEM metadata. - */ -struct rte_ivshmem_metadata { - int magic_number; /**< magic number */ - char name[IVSHMEM_NAME_LEN]; /**< name of the metadata file */ - struct rte_ivshmem_metadata_entry entry[RTE_LIBRTE_IVSHMEM_MAX_ENTRIES]; - /**< metadata entries */ -}; - -/** - * Creates metadata file with a given name - * - * @param name - * Name of metadata file to be created - * - * @return - * - On success, zero - * - On failure, a negative value - */ -int rte_ivshmem_metadata_create(const char * name); - -/** - * Adds memzone to a specific metadata file - * - * @param mz - * Memzone to be added - * @param md_name - * Name of metadata file for the memzone to be added to - * - * @return - * - On success, zero - * - On failure, a negative value - */ -int rte_ivshmem_metadata_add_memzone(const struct rte_memzone * mz, - const char * md_name); - -/** - * Adds a ring descriptor to a specific metadata file - * - * @param r - * Ring descriptor to be added - * @param md_name - * Name of metadata file for the ring to be added to - * - * @return - * - On success, zero - * - On failure, a negative value - */ -int rte_ivshmem_metadata_add_ring(const struct rte_ring * r, - const char * md_name); - -/** - * Adds a mempool to a specific metadata file - * - * @param mp - * Mempool to be added - * @param md_name - * Name of metadata file for the mempool to be added to - * - * @return - * - On success, zero - * - On failure, a negative value - */ -int rte_ivshmem_metadata_add_mempool(const struct rte_mempool * mp, - const char * md_name); - - -/** - * Generates the QEMU command-line for IVSHMEM device for a given metadata file. - * This function is to be called after all the objects were added. - * - * @param buffer - * Buffer to be filled with the command line arguments. - * @param size - * Size of the buffer. - * @param name - * Name of metadata file to generate QEMU command-line parameters for - * - * @return - * - On success, zero - * - On failure, a negative value - */ -int rte_ivshmem_metadata_cmdline_generate(char *buffer, unsigned size, - const char *name); - - -/** - * Dump all metadata entries from a given metadata file to the console. - * - * @param f - * A pointer to a file for output - * @name - * Name of the metadata file to be dumped to console. - */ -void rte_ivshmem_metadata_dump(FILE *f, const char *name); - - -#ifdef __cplusplus -} -#endif - -#endif /* RTE_IVSHMEM_H_ */ diff --git a/dpdk/lib/librte_ivshmem/rte_ivshmem_version.map b/dpdk/lib/librte_ivshmem/rte_ivshmem_version.map deleted file mode 100644 index 5a393ddc..00000000 --- a/dpdk/lib/librte_ivshmem/rte_ivshmem_version.map +++ /dev/null @@ -1,12 +0,0 @@ -DPDK_2.0 { - global: - - rte_ivshmem_metadata_add_mempool; - rte_ivshmem_metadata_add_memzone; - rte_ivshmem_metadata_add_ring; - rte_ivshmem_metadata_cmdline_generate; - rte_ivshmem_metadata_create; - rte_ivshmem_metadata_dump; - - local: *; -}; diff --git a/dpdk/lib/librte_jobstats/Makefile b/dpdk/lib/librte_jobstats/Makefile index 136a448e..d0bddd12 100644 --- a/dpdk/lib/librte_jobstats/Makefile +++ b/dpdk/lib/librte_jobstats/Makefile @@ -36,6 +36,7 @@ LIB = librte_jobstats.a CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) +LDLIBS += -lrte_eal EXPORT_MAP := rte_jobstats_version.map @@ -47,7 +48,4 @@ SRCS-$(CONFIG_RTE_LIBRTE_JOBSTATS) := rte_jobstats.c # install this header file SYMLINK-$(CONFIG_RTE_LIBRTE_JOBSTATS)-include := rte_jobstats.h -# this lib needs eal -DEPDIRS-$(CONFIG_RTE_LIBRTE_JOBSTATS) += lib/librte_eal - include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/lib/librte_jobstats/rte_jobstats.h b/dpdk/lib/librte_jobstats/rte_jobstats.h index b3686030..e1591562 100644 --- a/dpdk/lib/librte_jobstats/rte_jobstats.h +++ b/dpdk/lib/librte_jobstats/rte_jobstats.h @@ -98,7 +98,7 @@ struct rte_jobstats { } __rte_cache_aligned; struct rte_jobstats_context { - /** Viariable holding time at different points: + /** Variable holding time at different points: * -# loop start time if loop was started but no job executed yet. * -# job start time if job is currently executing. * -# job finish time if job finished its execution. @@ -117,7 +117,7 @@ struct rte_jobstats_context { /**< Minimum loop execute time. */ uint64_t max_exec_time; - /**< Minimum loop execute time. */ + /**< Maximum loop execute time. */ /** * Sum of time that is not the execute time (ex: from job finish to next @@ -313,7 +313,7 @@ rte_jobstats_set_max(struct rte_jobstats *job, uint64_t period); * * @param job * Job object. - * @param update_pedriod_cb + * @param update_period_cb * Callback to set. If NULL restore default update function. */ void diff --git a/dpdk/lib/librte_kni/Makefile b/dpdk/lib/librte_kni/Makefile index 09474461..56b19760 100644 --- a/dpdk/lib/librte_kni/Makefile +++ b/dpdk/lib/librte_kni/Makefile @@ -35,6 +35,7 @@ include $(RTE_SDK)/mk/rte.vars.mk LIB = librte_kni.a CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -O3 -fno-strict-aliasing +LDLIBS += -lrte_eal -lrte_mempool -lrte_mbuf -lrte_ethdev EXPORT_MAP := rte_kni_version.map @@ -46,9 +47,4 @@ SRCS-$(CONFIG_RTE_LIBRTE_KNI) := rte_kni.c # install includes SYMLINK-$(CONFIG_RTE_LIBRTE_KNI)-include := rte_kni.h -DEPDIRS-$(CONFIG_RTE_LIBRTE_KNI) += lib/librte_eal -DEPDIRS-$(CONFIG_RTE_LIBRTE_KNI) += lib/librte_mbuf -DEPDIRS-$(CONFIG_RTE_LIBRTE_KNI) += lib/librte_mempool -DEPDIRS-$(CONFIG_RTE_LIBRTE_KNI) += lib/librte_ether - include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/lib/librte_kni/rte_kni.c b/dpdk/lib/librte_kni/rte_kni.c index 3028fd43..8eca8c03 100644 --- a/dpdk/lib/librte_kni/rte_kni.c +++ b/dpdk/lib/librte_kni/rte_kni.c @@ -119,7 +119,7 @@ struct rte_kni_memzone_pool { uint32_t max_ifaces; /**< Max. num of KNI ifaces */ struct rte_kni_memzone_slot *slots; /**< Pool slots */ - rte_spinlock_t mutex; /**< alloc/relase mutex */ + rte_spinlock_t mutex; /**< alloc/release mutex */ /* Free memzone slots linked-list */ struct rte_kni_memzone_slot *free; /**< First empty slot */ @@ -210,14 +210,18 @@ rte_kni_init(unsigned int max_kni_ifaces) if (max_kni_ifaces == 0) { RTE_LOG(ERR, KNI, "Invalid number of max_kni_ifaces %d\n", max_kni_ifaces); - rte_panic("Unable to initialize KNI\n"); + RTE_LOG(ERR, KNI, "Unable to initialize KNI\n"); + return; } /* Check FD and open */ if (kni_fd < 0) { kni_fd = open("/dev/" KNI_DEVICE, O_RDWR); - if (kni_fd < 0) - rte_panic("Can not open /dev/%s\n", KNI_DEVICE); + if (kni_fd < 0) { + RTE_LOG(ERR, KNI, + "Can not open /dev/%s\n", KNI_DEVICE); + return; + } } /* Allocate slot objects */ @@ -307,8 +311,8 @@ rte_kni_init(unsigned int max_kni_ifaces) return; kni_fail: - rte_panic("Unable to allocate memory for max_kni_ifaces:%d. Increase the amount of hugepages memory\n", - max_kni_ifaces); + RTE_LOG(ERR, KNI, "Unable to allocate memory for max_kni_ifaces:%d." + "Increase the amount of hugepages memory\n", max_kni_ifaces); } @@ -321,9 +325,7 @@ rte_kni_alloc(struct rte_mempool *pktmbuf_pool, struct rte_kni_device_info dev_info; struct rte_kni *ctx; char intf_name[RTE_KNI_NAMESIZE]; - char mz_name[RTE_MEMZONE_NAMESIZE]; const struct rte_memzone *mz; - const struct rte_mempool *mp; struct rte_kni_memzone_slot *slot = NULL; if (!pktmbuf_pool || !conf || !conf->name[0]) @@ -338,7 +340,7 @@ rte_kni_alloc(struct rte_mempool *pktmbuf_pool, /* Get an available slot from the pool */ slot = kni_memzone_pool_alloc(); if (!slot) { - RTE_LOG(ERR, KNI, "Cannot allocate more KNI interfaces; increase the number of max_kni_ifaces(current %d) or release unusued ones.\n", + RTE_LOG(ERR, KNI, "Cannot allocate more KNI interfaces; increase the number of max_kni_ifaces(current %d) or release unused ones.\n", kni_memzone_pool.max_ifaces); return NULL; } @@ -414,19 +416,6 @@ rte_kni_alloc(struct rte_mempool *pktmbuf_pool, dev_info.sync_va = mz->addr; dev_info.sync_phys = mz->phys_addr; - - /* MBUF mempool */ - snprintf(mz_name, sizeof(mz_name), RTE_MEMPOOL_MZ_FORMAT, - pktmbuf_pool->name); - mz = rte_memzone_lookup(mz_name); - KNI_MEM_CHECK(mz == NULL); - mp = (struct rte_mempool *)mz->addr; - /* KNI currently requires to have only one memory chunk */ - if (mp->nb_mem_chunks != 1) - goto kni_fail; - - dev_info.mbuf_va = STAILQ_FIRST(&mp->mem_list)->addr; - dev_info.mbuf_phys = STAILQ_FIRST(&mp->mem_list)->phys_addr; ctx->pktmbuf_pool = pktmbuf_pool; ctx->group_id = conf->group_id; ctx->slot_id = slot->id; @@ -462,11 +451,44 @@ kni_free_fifo(struct rte_kni_fifo *fifo) } while (ret); } +static void * +va2pa(struct rte_mbuf *m) +{ + return (void *)((unsigned long)m - + ((unsigned long)m->buf_addr - + (unsigned long)m->buf_iova)); +} + +static void +obj_free(struct rte_mempool *mp __rte_unused, void *opaque, void *obj, + unsigned obj_idx __rte_unused) +{ + struct rte_mbuf *m = obj; + void *mbuf_phys = opaque; + + if (va2pa(m) == mbuf_phys) + rte_pktmbuf_free(m); +} + +static void +kni_free_fifo_phy(struct rte_mempool *mp, struct rte_kni_fifo *fifo) +{ + void *mbuf_phys; + int ret; + + do { + ret = kni_fifo_get(fifo, &mbuf_phys, 1); + if (ret) + rte_mempool_obj_iter(mp, obj_free, mbuf_phys); + } while (ret); +} + int rte_kni_release(struct rte_kni *kni) { struct rte_kni_device_info dev_info; uint32_t slot_id; + uint32_t retry = 5; if (!kni || !kni->in_use) return -1; @@ -478,9 +500,16 @@ rte_kni_release(struct rte_kni *kni) } /* mbufs in all fifo should be released, except request/response */ + + /* wait until all rxq packets processed by kernel */ + while (kni_fifo_count(kni->rx_q) && retry--) + usleep(1000); + + if (kni_fifo_count(kni->rx_q)) + RTE_LOG(ERR, KNI, "Fail to free all Rx-q items\n"); + + kni_free_fifo_phy(kni->pktmbuf_pool, kni->alloc_q); kni_free_fifo(kni->tx_q); - kni_free_fifo(kni->rx_q); - kni_free_fifo(kni->alloc_q); kni_free_fifo(kni->free_q); slot_id = kni->slot_id; @@ -490,8 +519,9 @@ rte_kni_release(struct rte_kni *kni) /* Release memzone */ if (slot_id > kni_memzone_pool.max_ifaces) { - rte_panic("KNI pool: corrupted slot ID: %d, max: %d\n", + RTE_LOG(ERR, KNI, "KNI pool: corrupted slot ID: %d, max: %d\n", slot_id, kni_memzone_pool.max_ifaces); + return -1; } kni_memzone_pool_release(&kni_memzone_pool.slots[slot_id]); @@ -513,7 +543,8 @@ rte_kni_handle_request(struct rte_kni *kni) return 0; /* It is OK of can not getting the request mbuf */ if (req != kni->sync_addr) { - rte_panic("Wrong req pointer %p\n", req); + RTE_LOG(ERR, KNI, "Wrong req pointer %p\n", req); + return -1; } /* Analyze the request and call the relevant actions for it */ @@ -547,7 +578,14 @@ rte_kni_handle_request(struct rte_kni *kni) unsigned rte_kni_tx_burst(struct rte_kni *kni, struct rte_mbuf **mbufs, unsigned num) { - unsigned ret = kni_fifo_put(kni->rx_q, (void **)mbufs, num); + void *phy_mbufs[num]; + unsigned int ret; + unsigned int i; + + for (i = 0; i < num; i++) + phy_mbufs[i] = va2pa(mbufs[i]); + + ret = kni_fifo_put(kni->rx_q, phy_mbufs, num); /* Get mbufs from free_q and then free them */ kni_free_mbufs(kni); @@ -585,6 +623,8 @@ kni_allocate_mbufs(struct rte_kni *kni) { int i, ret; struct rte_mbuf *pkts[MAX_MBUF_BURST_NUM]; + void *phys[MAX_MBUF_BURST_NUM]; + int allocq_free; RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, pool) != offsetof(struct rte_kni_mbuf, pool)); @@ -607,20 +647,23 @@ kni_allocate_mbufs(struct rte_kni *kni) return; } - for (i = 0; i < MAX_MBUF_BURST_NUM; i++) { + allocq_free = (kni->alloc_q->read - kni->alloc_q->write - 1) \ + & (MAX_MBUF_BURST_NUM - 1); + for (i = 0; i < allocq_free; i++) { pkts[i] = rte_pktmbuf_alloc(kni->pktmbuf_pool); if (unlikely(pkts[i] == NULL)) { /* Out of memory */ RTE_LOG(ERR, KNI, "Out of memory\n"); break; } + phys[i] = va2pa(pkts[i]); } - /* No pkt mbuf alocated */ + /* No pkt mbuf allocated */ if (i <= 0) return; - ret = kni_fifo_put(kni->alloc_q, (void **)pkts, i); + ret = kni_fifo_put(kni->alloc_q, phys, i); /* Check if any mbufs not put into alloc_q, and then free them */ if (ret >= 0 && ret < i && ret < MAX_MBUF_BURST_NUM) { diff --git a/dpdk/lib/librte_kni/rte_kni.h b/dpdk/lib/librte_kni/rte_kni.h index 7363e6cf..d43b5b28 100644 --- a/dpdk/lib/librte_kni/rte_kni.h +++ b/dpdk/lib/librte_kni/rte_kni.h @@ -42,7 +42,7 @@ * interfaces that may be used by the RTE application to receive/transmit * packets from/to Linux kernel net interfaces. * - * This library provide two APIs to burst receive packets from KNI interfaces, + * This library provides two APIs to burst receive packets from KNI interfaces, * and burst transmit packets to KNI interfaces. */ @@ -63,13 +63,13 @@ struct rte_mbuf; * Structure which has the function pointers for KNI interface. */ struct rte_kni_ops { - uint8_t port_id; /* Port ID */ + uint16_t port_id; /* Port ID */ /* Pointer to function of changing MTU */ - int (*change_mtu)(uint8_t port_id, unsigned new_mtu); + int (*change_mtu)(uint16_t port_id, unsigned int new_mtu); /* Pointer to function of configuring network interface */ - int (*config_network_if)(uint8_t port_id, uint8_t if_up); + int (*config_network_if)(uint16_t port_id, uint8_t if_up); }; /** @@ -88,6 +88,7 @@ struct rte_kni_conf { struct rte_pci_addr addr; struct rte_pci_id id; + __extension__ uint8_t force_bind : 1; /* Flag to bind kernel thread */ }; @@ -117,7 +118,7 @@ void rte_kni_init(unsigned int max_kni_ifaces); * elements for each KNI interface allocated. * * @param pktmbuf_pool - * The mempool for allocting mbufs for packets. + * The mempool for allocating mbufs for packets. * @param conf * The pointer to the configurations of the KNI device. * @param ops @@ -227,7 +228,7 @@ const char *rte_kni_get_name(const struct rte_kni *kni); * @param kni * pointer to struct rte_kni. * @param ops - * ponter to struct rte_kni_ops. + * pointer to struct rte_kni_ops. * * @return * On success: 0 diff --git a/dpdk/lib/librte_kni/rte_kni_fifo.h b/dpdk/lib/librte_kni/rte_kni_fifo.h index 8cb85873..6f2c3cb3 100644 --- a/dpdk/lib/librte_kni/rte_kni_fifo.h +++ b/dpdk/lib/librte_kni/rte_kni_fifo.h @@ -73,7 +73,7 @@ kni_fifo_put(struct rte_kni_fifo *fifo, void **data, unsigned num) } /** - * Get up to num elements from the fifo. Return the number actully read + * Get up to num elements from the fifo. Return the number actually read */ static inline unsigned kni_fifo_get(struct rte_kni_fifo *fifo, void **data, unsigned num) @@ -91,3 +91,12 @@ kni_fifo_get(struct rte_kni_fifo *fifo, void **data, unsigned num) fifo->read = new_read; return i; } + +/** + * Get the num of elements in the fifo + */ +static inline uint32_t +kni_fifo_count(struct rte_kni_fifo *fifo) +{ + return (fifo->len + fifo->write - fifo->read) & (fifo->len - 1); +} diff --git a/dpdk/lib/librte_kvargs/Makefile b/dpdk/lib/librte_kvargs/Makefile index 87b09f20..4eaa9334 100644 --- a/dpdk/lib/librte_kvargs/Makefile +++ b/dpdk/lib/librte_kvargs/Makefile @@ -37,6 +37,7 @@ include $(RTE_SDK)/mk/rte.vars.mk LIB = librte_kvargs.a CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -O3 +LDLIBS += -lrte_eal EXPORT_MAP := rte_kvargs_version.map @@ -49,7 +50,4 @@ SRCS-$(CONFIG_RTE_LIBRTE_KVARGS) := rte_kvargs.c INCS := rte_kvargs.h SYMLINK-$(CONFIG_RTE_LIBRTE_KVARGS)-include := $(INCS) -# this lib needs eal -DEPDIRS-$(CONFIG_RTE_LIBRTE_KVARGS) += lib/librte_eal - include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/lib/librte_kvargs/rte_kvargs.c b/dpdk/lib/librte_kvargs/rte_kvargs.c index 8d56abd4..854ac83f 100644 --- a/dpdk/lib/librte_kvargs/rte_kvargs.c +++ b/dpdk/lib/librte_kvargs/rte_kvargs.c @@ -92,9 +92,9 @@ rte_kvargs_tokenize(struct rte_kvargs *kvlist, const char *params) * into a list of valid keys. */ static int -is_valid_key(const char *valid[], const char *key_match) +is_valid_key(const char * const valid[], const char *key_match) { - const char **valid_ptr; + const char * const *valid_ptr; for (valid_ptr = valid; *valid_ptr != NULL; valid_ptr++) { if (strcmp(key_match, *valid_ptr) == 0) @@ -109,7 +109,7 @@ is_valid_key(const char *valid[], const char *key_match) */ static int check_for_valid_keys(struct rte_kvargs *kvlist, - const char *valid[]) + const char * const valid[]) { unsigned i, ret; struct rte_kvargs_pair *pair; @@ -187,7 +187,7 @@ rte_kvargs_free(struct rte_kvargs *kvlist) * check if only valid keys were used. */ struct rte_kvargs * -rte_kvargs_parse(const char *args, const char *valid_keys[]) +rte_kvargs_parse(const char *args, const char * const valid_keys[]) { struct rte_kvargs *kvlist; diff --git a/dpdk/lib/librte_kvargs/rte_kvargs.h b/dpdk/lib/librte_kvargs/rte_kvargs.h index ae9ae79f..5821c726 100644 --- a/dpdk/lib/librte_kvargs/rte_kvargs.h +++ b/dpdk/lib/librte_kvargs/rte_kvargs.h @@ -97,7 +97,8 @@ struct rte_kvargs { * - A pointer to an allocated rte_kvargs structure on success * - NULL on error */ -struct rte_kvargs *rte_kvargs_parse(const char *args, const char *valid_keys[]); +struct rte_kvargs *rte_kvargs_parse(const char *args, + const char *const valid_keys[]); /** * Free a rte_kvargs structure diff --git a/dpdk/lib/librte_latencystats/Makefile b/dpdk/lib/librte_latencystats/Makefile new file mode 100644 index 00000000..665c7b41 --- /dev/null +++ b/dpdk/lib/librte_latencystats/Makefile @@ -0,0 +1,51 @@ +# BSD LICENSE +# +# Copyright(c) 2017 Intel Corporation. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +include $(RTE_SDK)/mk/rte.vars.mk + +# library name +LIB = librte_latencystats.a + +CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -O3 +LDLIBS += -lm +LDLIBS += -lpthread +LDLIBS += -lrte_eal -lrte_metrics -lrte_ethdev -lrte_mbuf + +EXPORT_MAP := rte_latencystats_version.map + +LIBABIVER := 1 + +# all source are stored in SRCS-y +SRCS-$(CONFIG_RTE_LIBRTE_LATENCY_STATS) := rte_latencystats.c + +# install this header file +SYMLINK-$(CONFIG_RTE_LIBRTE_LATENCY_STATS)-include := rte_latencystats.h + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/lib/librte_latencystats/rte_latencystats.c b/dpdk/lib/librte_latencystats/rte_latencystats.c new file mode 100644 index 00000000..d6ad13c4 --- /dev/null +++ b/dpdk/lib/librte_latencystats/rte_latencystats.c @@ -0,0 +1,360 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "rte_latencystats.h" + +/** Nano seconds per second */ +#define NS_PER_SEC 1E9 + +/** Clock cycles per nano second */ +static uint64_t +latencystat_cycles_per_ns(void) +{ + return rte_get_timer_hz() / NS_PER_SEC; +} + +/* Macros for printing using RTE_LOG */ +#define RTE_LOGTYPE_LATENCY_STATS RTE_LOGTYPE_USER1 + +static const char *MZ_RTE_LATENCY_STATS = "rte_latencystats"; +static int latency_stats_index; +static uint64_t samp_intvl; +static uint64_t timer_tsc; +static uint64_t prev_tsc; + +struct rte_latency_stats { + float min_latency; /**< Minimum latency in nano seconds */ + float avg_latency; /**< Average latency in nano seconds */ + float max_latency; /**< Maximum latency in nano seconds */ + float jitter; /** Latency variation */ +}; + +static struct rte_latency_stats *glob_stats; + +struct rxtx_cbs { + struct rte_eth_rxtx_callback *cb; +}; + +static struct rxtx_cbs rx_cbs[RTE_MAX_ETHPORTS][RTE_MAX_QUEUES_PER_PORT]; +static struct rxtx_cbs tx_cbs[RTE_MAX_ETHPORTS][RTE_MAX_QUEUES_PER_PORT]; + +struct latency_stats_nameoff { + char name[RTE_ETH_XSTATS_NAME_SIZE]; + unsigned int offset; +}; + +static const struct latency_stats_nameoff lat_stats_strings[] = { + {"min_latency_ns", offsetof(struct rte_latency_stats, min_latency)}, + {"avg_latency_ns", offsetof(struct rte_latency_stats, avg_latency)}, + {"max_latency_ns", offsetof(struct rte_latency_stats, max_latency)}, + {"jitter_ns", offsetof(struct rte_latency_stats, jitter)}, +}; + +#define NUM_LATENCY_STATS (sizeof(lat_stats_strings) / \ + sizeof(lat_stats_strings[0])) + +int32_t +rte_latencystats_update(void) +{ + unsigned int i; + float *stats_ptr = NULL; + uint64_t values[NUM_LATENCY_STATS] = {0}; + int ret; + + for (i = 0; i < NUM_LATENCY_STATS; i++) { + stats_ptr = RTE_PTR_ADD(glob_stats, + lat_stats_strings[i].offset); + values[i] = (uint64_t)floor((*stats_ptr)/ + latencystat_cycles_per_ns()); + } + + ret = rte_metrics_update_values(RTE_METRICS_GLOBAL, + latency_stats_index, + values, NUM_LATENCY_STATS); + if (ret < 0) + RTE_LOG(INFO, LATENCY_STATS, "Failed to push the stats\n"); + + return ret; +} + +static void +rte_latencystats_fill_values(struct rte_metric_value *values) +{ + unsigned int i; + float *stats_ptr = NULL; + + for (i = 0; i < NUM_LATENCY_STATS; i++) { + stats_ptr = RTE_PTR_ADD(glob_stats, + lat_stats_strings[i].offset); + values[i].key = i; + values[i].value = (uint64_t)floor((*stats_ptr)/ + latencystat_cycles_per_ns()); + } +} + +static uint16_t +add_time_stamps(uint16_t pid __rte_unused, + uint16_t qid __rte_unused, + struct rte_mbuf **pkts, + uint16_t nb_pkts, + uint16_t max_pkts __rte_unused, + void *user_cb __rte_unused) +{ + unsigned int i; + uint64_t diff_tsc, now; + + /* + * For every sample interval, + * time stamp is marked on one received packet. + */ + now = rte_rdtsc(); + for (i = 0; i < nb_pkts; i++) { + diff_tsc = now - prev_tsc; + timer_tsc += diff_tsc; + if (timer_tsc >= samp_intvl) { + pkts[i]->timestamp = now; + timer_tsc = 0; + } + prev_tsc = now; + now = rte_rdtsc(); + } + + return nb_pkts; +} + +static uint16_t +calc_latency(uint16_t pid __rte_unused, + uint16_t qid __rte_unused, + struct rte_mbuf **pkts, + uint16_t nb_pkts, + void *_ __rte_unused) +{ + unsigned int i, cnt = 0; + uint64_t now; + float latency[nb_pkts]; + static float prev_latency; + /* + * Alpha represents degree of weighting decrease in EWMA, + * a constant smoothing factor between 0 and 1. The value + * is used below for measuring average latency. + */ + const float alpha = 0.2; + + now = rte_rdtsc(); + for (i = 0; i < nb_pkts; i++) { + if (pkts[i]->timestamp) + latency[cnt++] = now - pkts[i]->timestamp; + } + + for (i = 0; i < cnt; i++) { + /* + * The jitter is calculated as statistical mean of interpacket + * delay variation. The "jitter estimate" is computed by taking + * the absolute values of the ipdv sequence and applying an + * exponential filter with parameter 1/16 to generate the + * estimate. i.e J=J+(|D(i-1,i)|-J)/16. Where J is jitter, + * D(i-1,i) is difference in latency of two consecutive packets + * i-1 and i. + * Reference: Calculated as per RFC 5481, sec 4.1, + * RFC 3393 sec 4.5, RFC 1889 sec. + */ + glob_stats->jitter += (fabsf(prev_latency - latency[i]) + - glob_stats->jitter)/16; + if (glob_stats->min_latency == 0) + glob_stats->min_latency = latency[i]; + else if (latency[i] < glob_stats->min_latency) + glob_stats->min_latency = latency[i]; + else if (latency[i] > glob_stats->max_latency) + glob_stats->max_latency = latency[i]; + /* + * The average latency is measured using exponential moving + * average, i.e. using EWMA + * https://en.wikipedia.org/wiki/Moving_average + */ + glob_stats->avg_latency += + alpha * (latency[i] - glob_stats->avg_latency); + prev_latency = latency[i]; + } + + return nb_pkts; +} + +int +rte_latencystats_init(uint64_t app_samp_intvl, + rte_latency_stats_flow_type_fn user_cb) +{ + unsigned int i; + uint16_t pid; + uint16_t qid; + struct rxtx_cbs *cbs = NULL; + const uint16_t nb_ports = rte_eth_dev_count(); + const char *ptr_strings[NUM_LATENCY_STATS] = {0}; + const struct rte_memzone *mz = NULL; + const unsigned int flags = 0; + + if (rte_memzone_lookup(MZ_RTE_LATENCY_STATS)) + return -EEXIST; + + /** Allocate stats in shared memory fo multi process support */ + mz = rte_memzone_reserve(MZ_RTE_LATENCY_STATS, sizeof(*glob_stats), + rte_socket_id(), flags); + if (mz == NULL) { + RTE_LOG(ERR, LATENCY_STATS, "Cannot reserve memory: %s:%d\n", + __func__, __LINE__); + return -ENOMEM; + } + + glob_stats = mz->addr; + samp_intvl = app_samp_intvl * latencystat_cycles_per_ns(); + + /** Register latency stats with stats library */ + for (i = 0; i < NUM_LATENCY_STATS; i++) + ptr_strings[i] = lat_stats_strings[i].name; + + latency_stats_index = rte_metrics_reg_names(ptr_strings, + NUM_LATENCY_STATS); + if (latency_stats_index < 0) { + RTE_LOG(DEBUG, LATENCY_STATS, + "Failed to register latency stats names\n"); + return -1; + } + + /** Register Rx/Tx callbacks */ + for (pid = 0; pid < nb_ports; pid++) { + struct rte_eth_dev_info dev_info; + rte_eth_dev_info_get(pid, &dev_info); + for (qid = 0; qid < dev_info.nb_rx_queues; qid++) { + cbs = &rx_cbs[pid][qid]; + cbs->cb = rte_eth_add_first_rx_callback(pid, qid, + add_time_stamps, user_cb); + if (!cbs->cb) + RTE_LOG(INFO, LATENCY_STATS, "Failed to " + "register Rx callback for pid=%d, " + "qid=%d\n", pid, qid); + } + for (qid = 0; qid < dev_info.nb_tx_queues; qid++) { + cbs = &tx_cbs[pid][qid]; + cbs->cb = rte_eth_add_tx_callback(pid, qid, + calc_latency, user_cb); + if (!cbs->cb) + RTE_LOG(INFO, LATENCY_STATS, "Failed to " + "register Tx callback for pid=%d, " + "qid=%d\n", pid, qid); + } + } + return 0; +} + +int +rte_latencystats_uninit(void) +{ + uint16_t pid; + uint16_t qid; + int ret = 0; + struct rxtx_cbs *cbs = NULL; + const uint16_t nb_ports = rte_eth_dev_count(); + + /** De register Rx/Tx callbacks */ + for (pid = 0; pid < nb_ports; pid++) { + struct rte_eth_dev_info dev_info; + rte_eth_dev_info_get(pid, &dev_info); + for (qid = 0; qid < dev_info.nb_rx_queues; qid++) { + cbs = &rx_cbs[pid][qid]; + ret = rte_eth_remove_rx_callback(pid, qid, cbs->cb); + if (ret) + RTE_LOG(INFO, LATENCY_STATS, "failed to " + "remove Rx callback for pid=%d, " + "qid=%d\n", pid, qid); + } + for (qid = 0; qid < dev_info.nb_tx_queues; qid++) { + cbs = &tx_cbs[pid][qid]; + ret = rte_eth_remove_tx_callback(pid, qid, cbs->cb); + if (ret) + RTE_LOG(INFO, LATENCY_STATS, "failed to " + "remove Tx callback for pid=%d, " + "qid=%d\n", pid, qid); + } + } + + return 0; +} + +int +rte_latencystats_get_names(struct rte_metric_name *names, uint16_t size) +{ + unsigned int i; + + if (names == NULL || size < NUM_LATENCY_STATS) + return NUM_LATENCY_STATS; + + for (i = 0; i < NUM_LATENCY_STATS; i++) + snprintf(names[i].name, sizeof(names[i].name), + "%s", lat_stats_strings[i].name); + + return NUM_LATENCY_STATS; +} + +int +rte_latencystats_get(struct rte_metric_value *values, uint16_t size) +{ + if (size < NUM_LATENCY_STATS || values == NULL) + return NUM_LATENCY_STATS; + + if (rte_eal_process_type() == RTE_PROC_SECONDARY) { + const struct rte_memzone *mz; + mz = rte_memzone_lookup(MZ_RTE_LATENCY_STATS); + if (mz == NULL) { + RTE_LOG(ERR, LATENCY_STATS, + "Latency stats memzone not found\n"); + return -ENOMEM; + } + glob_stats = mz->addr; + } + + /* Retrieve latency stats */ + rte_latencystats_fill_values(values); + + return NUM_LATENCY_STATS; +} diff --git a/dpdk/lib/librte_latencystats/rte_latencystats.h b/dpdk/lib/librte_latencystats/rte_latencystats.h new file mode 100644 index 00000000..d85cf3a5 --- /dev/null +++ b/dpdk/lib/librte_latencystats/rte_latencystats.h @@ -0,0 +1,156 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_LATENCYSTATS_H_ +#define _RTE_LATENCYSTATS_H_ + +/** + * @file + * RTE latency stats + * + * library to provide application and flow based latency stats. + */ + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Note: This function pointer is for future flow based latency stats + * implementation. + * + * Function type used for identifting flow types of a Rx packet. + * + * The callback function is called on Rx for each packet. + * This function is used for flow based latency calculations. + * + * @param pkt + * Packet that has to be identified with its flow types. + * @param user_param + * The arbitrary user parameter passed in by the application when + * the callback was originally configured. + * @return + * The flow_mask, representing the multiple flow types of a packet. + */ +typedef uint16_t (*rte_latency_stats_flow_type_fn)(struct rte_mbuf *pkt, + void *user_param); + +/** + * Registers Rx/Tx callbacks for each active port, queue. + * + * @param samp_intvl + * Sampling time period in nano seconds, at which packet + * should be marked with time stamp. + * @param user_cb + * Note: This param is for future flow based latency stats + * implementation. + * User callback to be called to get flow types of a packet. + * Used for flow based latency calculation. + * If the value is NULL, global stats will be calculated, + * else flow based latency stats will be calculated. + * For now just pass on the NULL value to this param. + * @return + * -1 : On error + * -ENOMEM: On error + * 0 : On success + */ +int rte_latencystats_init(uint64_t samp_intvl, + rte_latency_stats_flow_type_fn user_cb); + +/** + * Calculates the latency and jitter values internally, exposing the updated + * values via *rte_latencystats_get* or the rte_metrics API. + * @return: + * 0 : on Success + * < 0 : Error in updating values. + */ +int32_t rte_latencystats_update(void); + +/** + * Removes registered Rx/Tx callbacks for each active port, queue. + * + * @return + * -1: On error + * 0: On success + */ +int rte_latencystats_uninit(void); + +/** + * Retrieve names of latency statistics + * + * @param names + * Block of memory to insert names into. Must be at least size in capacity. + * If set to NULL, function returns required capacity. + * @param size + * Capacity of latency stats names (number of names). + * @return + * - positive value lower or equal to size: success. The return value + * is the number of entries filled in the stats table. + * - positive value higher than size: error, the given statistics table + * is too small. The return value corresponds to the size that should + * be given to succeed. The entries in the table are not valid and + * shall not be used by the caller. + */ +int rte_latencystats_get_names(struct rte_metric_name *names, + uint16_t size); + +/** + * Retrieve latency statistics. + * + * @param values + * A pointer to a table of structure of type *rte_metric_value* + * to be filled with latency statistics ids and values. + * This parameter can be set to NULL if size is 0. + * @param size + * The size of the stats table, which should be large enough to store + * all the latency stats. + * @return + * - positive value lower or equal to size: success. The return value + * is the number of entries filled in the stats table. + * - positive value higher than size: error, the given statistics table + * is too small. The return value corresponds to the size that should + * be given to succeed. The entries in the table are not valid and + * shall not be used by the caller. + * -ENOMEM: On failure. + */ +int rte_latencystats_get(struct rte_metric_value *values, + uint16_t size); + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_LATENCYSTATS_H_ */ diff --git a/dpdk/lib/librte_latencystats/rte_latencystats_version.map b/dpdk/lib/librte_latencystats/rte_latencystats_version.map new file mode 100644 index 00000000..ac8403e8 --- /dev/null +++ b/dpdk/lib/librte_latencystats/rte_latencystats_version.map @@ -0,0 +1,11 @@ +DPDK_17.05 { + global: + + rte_latencystats_get; + rte_latencystats_get_names; + rte_latencystats_init; + rte_latencystats_uninit; + rte_latencystats_update; + + local: *; +}; diff --git a/dpdk/lib/librte_lpm/Makefile b/dpdk/lib/librte_lpm/Makefile index 656ade27..2e8749e8 100644 --- a/dpdk/lib/librte_lpm/Makefile +++ b/dpdk/lib/librte_lpm/Makefile @@ -36,6 +36,7 @@ LIB = librte_lpm.a CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) +LDLIBS += -lrte_eal EXPORT_MAP := rte_lpm_version.map @@ -51,9 +52,8 @@ ifneq ($(filter y,$(CONFIG_RTE_ARCH_ARM) $(CONFIG_RTE_ARCH_ARM64)),) SYMLINK-$(CONFIG_RTE_LIBRTE_LPM)-include += rte_lpm_neon.h else ifeq ($(CONFIG_RTE_ARCH_X86),y) SYMLINK-$(CONFIG_RTE_LIBRTE_LPM)-include += rte_lpm_sse.h +else ifeq ($(CONFIG_RTE_ARCH_PPC_64),y) +SYMLINK-$(CONFIG_RTE_LIBRTE_LPM)-include += rte_lpm_altivec.h endif -# this lib needs eal -DEPDIRS-$(CONFIG_RTE_LIBRTE_LPM) += lib/librte_eal - include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/lib/librte_lpm/rte_lpm.c b/dpdk/lib/librte_lpm/rte_lpm.c index e1b5d94a..dda74a9a 100644 --- a/dpdk/lib/librte_lpm/rte_lpm.c +++ b/dpdk/lib/librte_lpm/rte_lpm.c @@ -36,7 +36,6 @@ #include #include #include -#include #include #include @@ -44,7 +43,6 @@ #include #include /* for definition of RTE_CACHE_LINE_SIZE */ #include -#include #include #include #include @@ -219,6 +217,7 @@ rte_lpm_create_v20(const char *name, int socket_id, int max_rules, te = rte_zmalloc("LPM_TAILQ_ENTRY", sizeof(*te), 0); if (te == NULL) { RTE_LOG(ERR, LPM, "Failed to allocate tailq entry\n"); + rte_errno = ENOMEM; goto exit; } @@ -228,6 +227,7 @@ rte_lpm_create_v20(const char *name, int socket_id, int max_rules, if (lpm == NULL) { RTE_LOG(ERR, LPM, "LPM memory allocation failed\n"); rte_free(te); + rte_errno = ENOMEM; goto exit; } @@ -293,6 +293,7 @@ rte_lpm_create_v1604(const char *name, int socket_id, te = rte_zmalloc("LPM_TAILQ_ENTRY", sizeof(*te), 0); if (te == NULL) { RTE_LOG(ERR, LPM, "Failed to allocate tailq entry\n"); + rte_errno = ENOMEM; goto exit; } @@ -302,6 +303,7 @@ rte_lpm_create_v1604(const char *name, int socket_id, if (lpm == NULL) { RTE_LOG(ERR, LPM, "LPM memory allocation failed\n"); rte_free(te); + rte_errno = ENOMEM; goto exit; } @@ -313,6 +315,7 @@ rte_lpm_create_v1604(const char *name, int socket_id, rte_free(lpm); lpm = NULL; rte_free(te); + rte_errno = ENOMEM; goto exit; } @@ -325,6 +328,7 @@ rte_lpm_create_v1604(const char *name, int socket_id, rte_free(lpm); lpm = NULL; rte_free(te); + rte_errno = ENOMEM; goto exit; } @@ -908,7 +912,7 @@ add_depth_big_v20(struct rte_lpm_v20 *lpm, uint32_t ip_masked, uint8_t depth, */ struct rte_lpm_tbl_entry_v20 new_tbl24_entry = { - { .group_idx = (uint8_t)tbl8_group_index, }, + .group_idx = (uint8_t)tbl8_group_index, .valid = VALID, .valid_group = 1, .depth = 0, @@ -942,14 +946,9 @@ add_depth_big_v20(struct rte_lpm_v20 *lpm, uint32_t ip_masked, uint8_t depth, /* Insert new rule into the tbl8 entry. */ for (i = tbl8_index; i < tbl8_index + tbl8_range; i++) { - if (!lpm->tbl8[i].valid || - lpm->tbl8[i].depth <= depth) { - lpm->tbl8[i].valid = VALID; - lpm->tbl8[i].depth = depth; - lpm->tbl8[i].next_hop = next_hop; - - continue; - } + lpm->tbl8[i].valid = VALID; + lpm->tbl8[i].depth = depth; + lpm->tbl8[i].next_hop = next_hop; } /* @@ -959,7 +958,7 @@ add_depth_big_v20(struct rte_lpm_v20 *lpm, uint32_t ip_masked, uint8_t depth, */ struct rte_lpm_tbl_entry_v20 new_tbl24_entry = { - { .group_idx = (uint8_t)tbl8_group_index, }, + .group_idx = (uint8_t)tbl8_group_index, .valid = VALID, .valid_group = 1, .depth = 0, @@ -1039,7 +1038,7 @@ add_depth_big_v1604(struct rte_lpm *lpm, uint32_t ip_masked, uint8_t depth, */ struct rte_lpm_tbl_entry new_tbl24_entry = { - .group_idx = (uint8_t)tbl8_group_index, + .group_idx = tbl8_group_index, .valid = VALID, .valid_group = 1, .depth = 0, @@ -1073,14 +1072,9 @@ add_depth_big_v1604(struct rte_lpm *lpm, uint32_t ip_masked, uint8_t depth, /* Insert new rule into the tbl8 entry. */ for (i = tbl8_index; i < tbl8_index + tbl8_range; i++) { - if (!lpm->tbl8[i].valid || - lpm->tbl8[i].depth <= depth) { - lpm->tbl8[i].valid = VALID; - lpm->tbl8[i].depth = depth; - lpm->tbl8[i].next_hop = next_hop; - - continue; - } + lpm->tbl8[i].valid = VALID; + lpm->tbl8[i].depth = depth; + lpm->tbl8[i].next_hop = next_hop; } /* @@ -1090,7 +1084,7 @@ add_depth_big_v1604(struct rte_lpm *lpm, uint32_t ip_masked, uint8_t depth, */ struct rte_lpm_tbl_entry new_tbl24_entry = { - .group_idx = (uint8_t)tbl8_group_index, + .group_idx = tbl8_group_index, .valid = VALID, .valid_group = 1, .depth = 0, @@ -1371,7 +1365,7 @@ delete_depth_small_v20(struct rte_lpm_v20 *lpm, uint32_t ip_masked, */ struct rte_lpm_tbl_entry_v20 new_tbl24_entry = { - {.next_hop = lpm->rules_tbl[sub_rule_index].next_hop,}, + .next_hop = lpm->rules_tbl[sub_rule_index].next_hop, .valid = VALID, .valid_group = 0, .depth = sub_rule_depth, @@ -1674,7 +1668,7 @@ delete_depth_big_v20(struct rte_lpm_v20 *lpm, uint32_t ip_masked, } else if (tbl8_recycle_index > -1) { /* Update tbl24 entry. */ struct rte_lpm_tbl_entry_v20 new_tbl24_entry = { - { .next_hop = lpm->tbl8[tbl8_recycle_index].next_hop, }, + .next_hop = lpm->tbl8[tbl8_recycle_index].next_hop, .valid = VALID, .valid_group = 0, .depth = lpm->tbl8[tbl8_recycle_index].depth, diff --git a/dpdk/lib/librte_lpm/rte_lpm.h b/dpdk/lib/librte_lpm/rte_lpm.h index 2df1d672..650c51d1 100644 --- a/dpdk/lib/librte_lpm/rte_lpm.h +++ b/dpdk/lib/librte_lpm/rte_lpm.h @@ -45,6 +45,7 @@ #include #include #include +#include #include #include #include @@ -93,12 +94,14 @@ extern "C" { #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN /** @internal Tbl24 entry structure. */ +__extension__ struct rte_lpm_tbl_entry_v20 { /** * Stores Next hop (tbl8 or tbl24 when valid_group is not set) or * a group index pointing to a tbl8 structure (tbl24 only, when * valid_group is set) */ + RTE_STD_C11 union { uint8_t next_hop; uint8_t group_idx; @@ -116,6 +119,7 @@ struct rte_lpm_tbl_entry_v20 { uint8_t depth :6; /**< Rule depth. */ }; +__extension__ struct rte_lpm_tbl_entry { /** * Stores Next hop (tbl8 or tbl24 when valid_group is not set) or @@ -137,6 +141,7 @@ struct rte_lpm_tbl_entry { }; #else +__extension__ struct rte_lpm_tbl_entry_v20 { uint8_t depth :6; uint8_t valid_group :1; @@ -147,6 +152,7 @@ struct rte_lpm_tbl_entry_v20 { }; }; +__extension__ struct rte_lpm_tbl_entry { uint32_t depth :6; uint32_t valid_group :1; @@ -193,7 +199,7 @@ struct rte_lpm_v20 { __rte_cache_aligned; /**< LPM tbl24 table. */ struct rte_lpm_tbl_entry_v20 tbl8[RTE_LPM_TBL8_NUM_ENTRIES] __rte_cache_aligned; /**< LPM tbl8 table. */ - struct rte_lpm_rule_v20 rules_tbl[0] \ + struct rte_lpm_rule_v20 rules_tbl[] __rte_cache_aligned; /**< LPM rules. */ }; @@ -480,6 +486,8 @@ rte_lpm_lookupx4(const struct rte_lpm *lpm, xmm_t ip, uint32_t hop[4], #if defined(RTE_ARCH_ARM) || defined(RTE_ARCH_ARM64) #include "rte_lpm_neon.h" +#elif defined(RTE_ARCH_PPC_64) +#include "rte_lpm_altivec.h" #else #include "rte_lpm_sse.h" #endif diff --git a/dpdk/lib/librte_lpm/rte_lpm6.c b/dpdk/lib/librte_lpm/rte_lpm6.c index 32fdba01..03668d9e 100644 --- a/dpdk/lib/librte_lpm/rte_lpm6.c +++ b/dpdk/lib/librte_lpm/rte_lpm6.c @@ -35,7 +35,6 @@ #include #include #include -#include #include #include @@ -43,7 +42,6 @@ #include #include #include -#include #include #include #include @@ -97,7 +95,7 @@ struct rte_lpm6_tbl_entry { /** Rules tbl entry structure. */ struct rte_lpm6_rule { uint8_t ip[RTE_LPM6_IPV6_ADDR_SIZE]; /**< Rule IP address. */ - uint8_t next_hop; /**< Rule next hop. */ + uint32_t next_hop; /**< Rule next hop. */ uint8_t depth; /**< Rule depth. */ }; @@ -192,6 +190,7 @@ rte_lpm6_create(const char *name, int socket_id, te = rte_zmalloc("LPM6_TAILQ_ENTRY", sizeof(*te), 0); if (te == NULL) { RTE_LOG(ERR, LPM, "Failed to allocate tailq entry!\n"); + rte_errno = ENOMEM; goto exit; } @@ -202,6 +201,7 @@ rte_lpm6_create(const char *name, int socket_id, if (lpm == NULL) { RTE_LOG(ERR, LPM, "LPM memory allocation failed\n"); rte_free(te); + rte_errno = ENOMEM; goto exit; } @@ -213,6 +213,7 @@ rte_lpm6_create(const char *name, int socket_id, rte_free(lpm); lpm = NULL; rte_free(te); + rte_errno = ENOMEM; goto exit; } @@ -297,7 +298,7 @@ rte_lpm6_free(struct rte_lpm6 *lpm) * the nexthop if so. Otherwise it adds a new rule if enough space is available. */ static inline int32_t -rule_add(struct rte_lpm6 *lpm, uint8_t *ip, uint8_t next_hop, uint8_t depth) +rule_add(struct rte_lpm6 *lpm, uint8_t *ip, uint32_t next_hop, uint8_t depth) { uint32_t rule_index; @@ -340,7 +341,7 @@ rule_add(struct rte_lpm6 *lpm, uint8_t *ip, uint8_t next_hop, uint8_t depth) */ static void expand_rule(struct rte_lpm6 *lpm, uint32_t tbl8_gindex, uint8_t depth, - uint8_t next_hop) + uint32_t next_hop) { uint32_t tbl8_group_end, tbl8_gindex_next, j; @@ -377,7 +378,7 @@ expand_rule(struct rte_lpm6 *lpm, uint32_t tbl8_gindex, uint8_t depth, static inline int add_step(struct rte_lpm6 *lpm, struct rte_lpm6_tbl_entry *tbl, struct rte_lpm6_tbl_entry **tbl_next, uint8_t *ip, uint8_t bytes, - uint8_t first_byte, uint8_t depth, uint8_t next_hop) + uint8_t first_byte, uint8_t depth, uint32_t next_hop) { uint32_t tbl_index, tbl_range, tbl8_group_start, tbl8_group_end, i; int32_t tbl8_gindex; @@ -507,11 +508,19 @@ add_step(struct rte_lpm6 *lpm, struct rte_lpm6_tbl_entry *tbl, * Add a route */ int -rte_lpm6_add(struct rte_lpm6 *lpm, uint8_t *ip, uint8_t depth, +rte_lpm6_add_v20(struct rte_lpm6 *lpm, uint8_t *ip, uint8_t depth, uint8_t next_hop) +{ + return rte_lpm6_add_v1705(lpm, ip, depth, next_hop); +} +VERSION_SYMBOL(rte_lpm6_add, _v20, 2.0); + +int +rte_lpm6_add_v1705(struct rte_lpm6 *lpm, uint8_t *ip, uint8_t depth, + uint32_t next_hop) { struct rte_lpm6_tbl_entry *tbl; - struct rte_lpm6_tbl_entry *tbl_next; + struct rte_lpm6_tbl_entry *tbl_next = NULL; int32_t rule_index; int status; uint8_t masked_ip[RTE_LPM6_IPV6_ADDR_SIZE]; @@ -560,6 +569,10 @@ rte_lpm6_add(struct rte_lpm6 *lpm, uint8_t *ip, uint8_t depth, return status; } +BIND_DEFAULT_SYMBOL(rte_lpm6_add, _v1705, 17.05); +MAP_STATIC_SYMBOL(int rte_lpm6_add(struct rte_lpm6 *lpm, uint8_t *ip, + uint8_t depth, uint32_t next_hop), + rte_lpm6_add_v1705); /* * Takes a pointer to a table entry and inspect one level. @@ -569,7 +582,7 @@ rte_lpm6_add(struct rte_lpm6 *lpm, uint8_t *ip, uint8_t depth, static inline int lookup_step(const struct rte_lpm6 *lpm, const struct rte_lpm6_tbl_entry *tbl, const struct rte_lpm6_tbl_entry **tbl_next, uint8_t *ip, - uint8_t first_byte, uint8_t *next_hop) + uint8_t first_byte, uint32_t *next_hop) { uint32_t tbl8_index, tbl_entry; @@ -589,7 +602,7 @@ lookup_step(const struct rte_lpm6 *lpm, const struct rte_lpm6_tbl_entry *tbl, return 1; } else { /* If not extended then we can have a match. */ - *next_hop = (uint8_t)tbl_entry; + *next_hop = ((uint32_t)tbl_entry & RTE_LPM6_TBL8_BITMASK); return (tbl_entry & RTE_LPM6_LOOKUP_SUCCESS) ? 0 : -ENOENT; } } @@ -598,7 +611,26 @@ lookup_step(const struct rte_lpm6 *lpm, const struct rte_lpm6_tbl_entry *tbl, * Looks up an IP */ int -rte_lpm6_lookup(const struct rte_lpm6 *lpm, uint8_t *ip, uint8_t *next_hop) +rte_lpm6_lookup_v20(const struct rte_lpm6 *lpm, uint8_t *ip, uint8_t *next_hop) +{ + uint32_t next_hop32 = 0; + int32_t status; + + /* DEBUG: Check user input arguments. */ + if (next_hop == NULL) + return -EINVAL; + + status = rte_lpm6_lookup_v1705(lpm, ip, &next_hop32); + if (status == 0) + *next_hop = (uint8_t)next_hop32; + + return status; +} +VERSION_SYMBOL(rte_lpm6_lookup, _v20, 2.0); + +int +rte_lpm6_lookup_v1705(const struct rte_lpm6 *lpm, uint8_t *ip, + uint32_t *next_hop) { const struct rte_lpm6_tbl_entry *tbl; const struct rte_lpm6_tbl_entry *tbl_next = NULL; @@ -625,20 +657,23 @@ rte_lpm6_lookup(const struct rte_lpm6 *lpm, uint8_t *ip, uint8_t *next_hop) return status; } +BIND_DEFAULT_SYMBOL(rte_lpm6_lookup, _v1705, 17.05); +MAP_STATIC_SYMBOL(int rte_lpm6_lookup(const struct rte_lpm6 *lpm, uint8_t *ip, + uint32_t *next_hop), rte_lpm6_lookup_v1705); /* * Looks up a group of IP addresses */ int -rte_lpm6_lookup_bulk_func(const struct rte_lpm6 *lpm, +rte_lpm6_lookup_bulk_func_v20(const struct rte_lpm6 *lpm, uint8_t ips[][RTE_LPM6_IPV6_ADDR_SIZE], int16_t * next_hops, unsigned n) { unsigned i; const struct rte_lpm6_tbl_entry *tbl; const struct rte_lpm6_tbl_entry *tbl_next = NULL; - uint32_t tbl24_index; - uint8_t first_byte, next_hop; + uint32_t tbl24_index, next_hop; + uint8_t first_byte; int status; /* DEBUG: Check user input arguments. */ @@ -664,11 +699,59 @@ rte_lpm6_lookup_bulk_func(const struct rte_lpm6 *lpm, if (status < 0) next_hops[i] = -1; else - next_hops[i] = next_hop; + next_hops[i] = (int16_t)next_hop; } return 0; } +VERSION_SYMBOL(rte_lpm6_lookup_bulk_func, _v20, 2.0); + +int +rte_lpm6_lookup_bulk_func_v1705(const struct rte_lpm6 *lpm, + uint8_t ips[][RTE_LPM6_IPV6_ADDR_SIZE], + int32_t *next_hops, unsigned int n) +{ + unsigned int i; + const struct rte_lpm6_tbl_entry *tbl; + const struct rte_lpm6_tbl_entry *tbl_next = NULL; + uint32_t tbl24_index, next_hop; + uint8_t first_byte; + int status; + + /* DEBUG: Check user input arguments. */ + if ((lpm == NULL) || (ips == NULL) || (next_hops == NULL)) + return -EINVAL; + + for (i = 0; i < n; i++) { + first_byte = LOOKUP_FIRST_BYTE; + tbl24_index = (ips[i][0] << BYTES2_SIZE) | + (ips[i][1] << BYTE_SIZE) | ips[i][2]; + + /* Calculate pointer to the first entry to be inspected */ + tbl = &lpm->tbl24[tbl24_index]; + + do { + /* Continue inspecting following levels + * until success or failure + */ + status = lookup_step(lpm, tbl, &tbl_next, ips[i], + first_byte++, &next_hop); + tbl = tbl_next; + } while (status == 1); + + if (status < 0) + next_hops[i] = -1; + else + next_hops[i] = (int32_t)next_hop; + } + + return 0; +} +BIND_DEFAULT_SYMBOL(rte_lpm6_lookup_bulk_func, _v1705, 17.05); +MAP_STATIC_SYMBOL(int rte_lpm6_lookup_bulk_func(const struct rte_lpm6 *lpm, + uint8_t ips[][RTE_LPM6_IPV6_ADDR_SIZE], + int32_t *next_hops, unsigned int n), + rte_lpm6_lookup_bulk_func_v1705); /* * Finds a rule in rule table. @@ -698,8 +781,28 @@ rule_find(struct rte_lpm6 *lpm, uint8_t *ip, uint8_t depth) * Look for a rule in the high-level rules table */ int -rte_lpm6_is_rule_present(struct rte_lpm6 *lpm, uint8_t *ip, uint8_t depth, -uint8_t *next_hop) +rte_lpm6_is_rule_present_v20(struct rte_lpm6 *lpm, uint8_t *ip, uint8_t depth, + uint8_t *next_hop) +{ + uint32_t next_hop32 = 0; + int32_t status; + + /* DEBUG: Check user input arguments. */ + if (next_hop == NULL) + return -EINVAL; + + status = rte_lpm6_is_rule_present_v1705(lpm, ip, depth, &next_hop32); + if (status > 0) + *next_hop = (uint8_t)next_hop32; + + return status; + +} +VERSION_SYMBOL(rte_lpm6_is_rule_present, _v20, 2.0); + +int +rte_lpm6_is_rule_present_v1705(struct rte_lpm6 *lpm, uint8_t *ip, uint8_t depth, + uint32_t *next_hop) { uint8_t ip_masked[RTE_LPM6_IPV6_ADDR_SIZE]; int32_t rule_index; @@ -724,6 +827,10 @@ uint8_t *next_hop) /* If rule is not found return 0. */ return 0; } +BIND_DEFAULT_SYMBOL(rte_lpm6_is_rule_present, _v1705, 17.05); +MAP_STATIC_SYMBOL(int rte_lpm6_is_rule_present(struct rte_lpm6 *lpm, + uint8_t *ip, uint8_t depth, uint32_t *next_hop), + rte_lpm6_is_rule_present_v1705); /* * Delete a rule from the rule table. diff --git a/dpdk/lib/librte_lpm/rte_lpm6.h b/dpdk/lib/librte_lpm/rte_lpm6.h index 13d027f9..3a3342da 100644 --- a/dpdk/lib/librte_lpm/rte_lpm6.h +++ b/dpdk/lib/librte_lpm/rte_lpm6.h @@ -39,6 +39,7 @@ */ #include +#include #ifdef __cplusplus extern "C" { @@ -123,7 +124,13 @@ rte_lpm6_free(struct rte_lpm6 *lpm); */ int rte_lpm6_add(struct rte_lpm6 *lpm, uint8_t *ip, uint8_t depth, + uint32_t next_hop); +int +rte_lpm6_add_v20(struct rte_lpm6 *lpm, uint8_t *ip, uint8_t depth, uint8_t next_hop); +int +rte_lpm6_add_v1705(struct rte_lpm6 *lpm, uint8_t *ip, uint8_t depth, + uint32_t next_hop); /** * Check if a rule is present in the LPM table, @@ -142,7 +149,13 @@ rte_lpm6_add(struct rte_lpm6 *lpm, uint8_t *ip, uint8_t depth, */ int rte_lpm6_is_rule_present(struct rte_lpm6 *lpm, uint8_t *ip, uint8_t depth, -uint8_t *next_hop); + uint32_t *next_hop); +int +rte_lpm6_is_rule_present_v20(struct rte_lpm6 *lpm, uint8_t *ip, uint8_t depth, + uint8_t *next_hop); +int +rte_lpm6_is_rule_present_v1705(struct rte_lpm6 *lpm, uint8_t *ip, uint8_t depth, + uint32_t *next_hop); /** * Delete a rule from the LPM table. @@ -199,7 +212,12 @@ rte_lpm6_delete_all(struct rte_lpm6 *lpm); * -EINVAL for incorrect arguments, -ENOENT on lookup miss, 0 on lookup hit */ int -rte_lpm6_lookup(const struct rte_lpm6 *lpm, uint8_t *ip, uint8_t *next_hop); +rte_lpm6_lookup(const struct rte_lpm6 *lpm, uint8_t *ip, uint32_t *next_hop); +int +rte_lpm6_lookup_v20(const struct rte_lpm6 *lpm, uint8_t *ip, uint8_t *next_hop); +int +rte_lpm6_lookup_v1705(const struct rte_lpm6 *lpm, uint8_t *ip, + uint32_t *next_hop); /** * Lookup multiple IP addresses in an LPM table. @@ -220,7 +238,15 @@ rte_lpm6_lookup(const struct rte_lpm6 *lpm, uint8_t *ip, uint8_t *next_hop); int rte_lpm6_lookup_bulk_func(const struct rte_lpm6 *lpm, uint8_t ips[][RTE_LPM6_IPV6_ADDR_SIZE], - int16_t * next_hops, unsigned n); + int32_t *next_hops, unsigned int n); +int +rte_lpm6_lookup_bulk_func_v20(const struct rte_lpm6 *lpm, + uint8_t ips[][RTE_LPM6_IPV6_ADDR_SIZE], + int16_t *next_hops, unsigned int n); +int +rte_lpm6_lookup_bulk_func_v1705(const struct rte_lpm6 *lpm, + uint8_t ips[][RTE_LPM6_IPV6_ADDR_SIZE], + int32_t *next_hops, unsigned int n); #ifdef __cplusplus } diff --git a/dpdk/lib/librte_lpm/rte_lpm_altivec.h b/dpdk/lib/librte_lpm/rte_lpm_altivec.h new file mode 100644 index 00000000..e26e0875 --- /dev/null +++ b/dpdk/lib/librte_lpm/rte_lpm_altivec.h @@ -0,0 +1,154 @@ +/* + * BSD LICENSE + * + * Copyright (C) IBM Corporation 2016. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of IBM Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef _RTE_LPM_ALTIVEC_H_ +#define _RTE_LPM_ALTIVEC_H_ + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +static inline void +rte_lpm_lookupx4(const struct rte_lpm *lpm, xmm_t ip, uint32_t hop[4], + uint32_t defv) +{ + vector signed int i24; + rte_xmm_t i8; + uint32_t tbl[4]; + uint64_t idx, pt, pt2; + const uint32_t *ptbl; + + const uint32_t mask = UINT8_MAX; + const vector signed int mask8 = (xmm_t){mask, mask, mask, mask}; + + /* + * RTE_LPM_VALID_EXT_ENTRY_BITMASK for 2 LPM entries + * as one 64-bit value (0x0300000003000000). + */ + const uint64_t mask_xv = + ((uint64_t)RTE_LPM_VALID_EXT_ENTRY_BITMASK | + (uint64_t)RTE_LPM_VALID_EXT_ENTRY_BITMASK << 32); + + /* + * RTE_LPM_LOOKUP_SUCCESS for 2 LPM entries + * as one 64-bit value (0x0100000001000000). + */ + const uint64_t mask_v = + ((uint64_t)RTE_LPM_LOOKUP_SUCCESS | + (uint64_t)RTE_LPM_LOOKUP_SUCCESS << 32); + + /* get 4 indexes for tbl24[]. */ + i24 = vec_sr((xmm_t) ip, + (vector unsigned int){CHAR_BIT, CHAR_BIT, CHAR_BIT, CHAR_BIT}); + + /* extract values from tbl24[] */ + idx = (uint32_t)i24[0]; + idx = idx < (1<<24) ? idx : (1<<24)-1; + ptbl = (const uint32_t *)&lpm->tbl24[idx]; + tbl[0] = *ptbl; + + idx = (uint32_t) i24[1]; + idx = idx < (1<<24) ? idx : (1<<24)-1; + ptbl = (const uint32_t *)&lpm->tbl24[idx]; + tbl[1] = *ptbl; + + idx = (uint32_t) i24[2]; + idx = idx < (1<<24) ? idx : (1<<24)-1; + ptbl = (const uint32_t *)&lpm->tbl24[idx]; + tbl[2] = *ptbl; + + idx = (uint32_t) i24[3]; + idx = idx < (1<<24) ? idx : (1<<24)-1; + ptbl = (const uint32_t *)&lpm->tbl24[idx]; + tbl[3] = *ptbl; + + /* get 4 indexes for tbl8[]. */ + i8.x = vec_and(ip, mask8); + + pt = (uint64_t)tbl[0] | + (uint64_t)tbl[1] << 32; + pt2 = (uint64_t)tbl[2] | + (uint64_t)tbl[3] << 32; + + /* search successfully finished for all 4 IP addresses. */ + if (likely((pt & mask_xv) == mask_v) && + likely((pt2 & mask_xv) == mask_v)) { + *(uint64_t *)hop = pt & RTE_LPM_MASKX4_RES; + *(uint64_t *)(hop + 2) = pt2 & RTE_LPM_MASKX4_RES; + return; + } + + if (unlikely((pt & RTE_LPM_VALID_EXT_ENTRY_BITMASK) == + RTE_LPM_VALID_EXT_ENTRY_BITMASK)) { + i8.u32[0] = i8.u32[0] + + (uint8_t)tbl[0] * RTE_LPM_TBL8_GROUP_NUM_ENTRIES; + ptbl = (const uint32_t *)&lpm->tbl8[i8.u32[0]]; + tbl[0] = *ptbl; + } + if (unlikely((pt >> 32 & RTE_LPM_VALID_EXT_ENTRY_BITMASK) == + RTE_LPM_VALID_EXT_ENTRY_BITMASK)) { + i8.u32[1] = i8.u32[1] + + (uint8_t)tbl[1] * RTE_LPM_TBL8_GROUP_NUM_ENTRIES; + ptbl = (const uint32_t *)&lpm->tbl8[i8.u32[1]]; + tbl[1] = *ptbl; + } + if (unlikely((pt2 & RTE_LPM_VALID_EXT_ENTRY_BITMASK) == + RTE_LPM_VALID_EXT_ENTRY_BITMASK)) { + i8.u32[2] = i8.u32[2] + + (uint8_t)tbl[2] * RTE_LPM_TBL8_GROUP_NUM_ENTRIES; + ptbl = (const uint32_t *)&lpm->tbl8[i8.u32[2]]; + tbl[2] = *ptbl; + } + if (unlikely((pt2 >> 32 & RTE_LPM_VALID_EXT_ENTRY_BITMASK) == + RTE_LPM_VALID_EXT_ENTRY_BITMASK)) { + i8.u32[3] = i8.u32[3] + + (uint8_t)tbl[3] * RTE_LPM_TBL8_GROUP_NUM_ENTRIES; + ptbl = (const uint32_t *)&lpm->tbl8[i8.u32[3]]; + tbl[3] = *ptbl; + } + + hop[0] = (tbl[0] & RTE_LPM_LOOKUP_SUCCESS) ? tbl[0] & 0x00FFFFFF : defv; + hop[1] = (tbl[1] & RTE_LPM_LOOKUP_SUCCESS) ? tbl[1] & 0x00FFFFFF : defv; + hop[2] = (tbl[2] & RTE_LPM_LOOKUP_SUCCESS) ? tbl[2] & 0x00FFFFFF : defv; + hop[3] = (tbl[3] & RTE_LPM_LOOKUP_SUCCESS) ? tbl[3] & 0x00FFFFFF : defv; +} + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_LPM_ALTIVEC_H_ */ diff --git a/dpdk/lib/librte_lpm/rte_lpm_neon.h b/dpdk/lib/librte_lpm/rte_lpm_neon.h index 7c643159..4fd33f33 100644 --- a/dpdk/lib/librte_lpm/rte_lpm_neon.h +++ b/dpdk/lib/librte_lpm/rte_lpm_neon.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2015 Cavium Networks. All rights reserved. + * Copyright(c) 2015 Cavium, Inc. All rights reserved. * All rights reserved. * * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. @@ -19,7 +19,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium Networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * @@ -43,6 +43,7 @@ #include #include #include +#include #ifdef __cplusplus extern "C" { diff --git a/dpdk/lib/librte_lpm/rte_lpm_sse.h b/dpdk/lib/librte_lpm/rte_lpm_sse.h index da830995..5f2c4d4a 100644 --- a/dpdk/lib/librte_lpm/rte_lpm_sse.h +++ b/dpdk/lib/librte_lpm/rte_lpm_sse.h @@ -38,6 +38,7 @@ #include #include #include +#include #ifdef __cplusplus extern "C" { @@ -77,7 +78,8 @@ rte_lpm_lookupx4(const struct rte_lpm *lpm, xmm_t ip, uint32_t hop[4], /* extract values from tbl24[] */ idx = _mm_cvtsi128_si64(i24); - i24 = _mm_srli_si128(i24, sizeof(uint64_t)); + /* With -O0 option, gcc 4.8 - 5.4 fails to fold sizeof() into a constant */ + i24 = _mm_srli_si128(i24, /* sizeof(uint64_t) */ 8); ptbl = (const uint32_t *)&lpm->tbl24[(uint32_t)idx]; tbl[0] = *ptbl; diff --git a/dpdk/lib/librte_lpm/rte_lpm_version.map b/dpdk/lib/librte_lpm/rte_lpm_version.map index 239b371e..90beac85 100644 --- a/dpdk/lib/librte_lpm/rte_lpm_version.map +++ b/dpdk/lib/librte_lpm/rte_lpm_version.map @@ -34,3 +34,13 @@ DPDK_16.04 { rte_lpm_delete_all; } DPDK_2.0; + +DPDK_17.05 { + global: + + rte_lpm6_add; + rte_lpm6_is_rule_present; + rte_lpm6_lookup; + rte_lpm6_lookup_bulk_func; + +} DPDK_16.04; diff --git a/dpdk/lib/librte_mbuf/Makefile b/dpdk/lib/librte_mbuf/Makefile index 8d62b0d3..f6be3536 100644 --- a/dpdk/lib/librte_mbuf/Makefile +++ b/dpdk/lib/librte_mbuf/Makefile @@ -35,18 +35,16 @@ include $(RTE_SDK)/mk/rte.vars.mk LIB = librte_mbuf.a CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -O3 +LDLIBS += -lrte_eal -lrte_mempool EXPORT_MAP := rte_mbuf_version.map -LIBABIVER := 2 +LIBABIVER := 3 # all source are stored in SRCS-y -SRCS-$(CONFIG_RTE_LIBRTE_MBUF) := rte_mbuf.c +SRCS-$(CONFIG_RTE_LIBRTE_MBUF) := rte_mbuf.c rte_mbuf_ptype.c # install includes -SYMLINK-$(CONFIG_RTE_LIBRTE_MBUF)-include := rte_mbuf.h - -# this lib needs eal -DEPDIRS-$(CONFIG_RTE_LIBRTE_MBUF) += lib/librte_eal lib/librte_mempool +SYMLINK-$(CONFIG_RTE_LIBRTE_MBUF)-include := rte_mbuf.h rte_mbuf_ptype.h include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/lib/librte_mbuf/rte_mbuf.c b/dpdk/lib/librte_mbuf/rte_mbuf.c index d2c87526..7543662f 100644 --- a/dpdk/lib/librte_mbuf/rte_mbuf.c +++ b/dpdk/lib/librte_mbuf/rte_mbuf.c @@ -46,23 +46,22 @@ #include #include #include -#include #include #include #include #include #include #include -#include #include #include #include #include #include +#include /* * ctrlmbuf constructor, given as a callback function to - * rte_mempool_create() + * rte_mempool_obj_iter() or rte_mempool_create() */ void rte_ctrlmbuf_init(struct rte_mempool *mp, @@ -77,7 +76,8 @@ rte_ctrlmbuf_init(struct rte_mempool *mp, /* * pktmbuf pool constructor, given as a callback function to - * rte_mempool_create() + * rte_mempool_create(), or called directly if using + * rte_mempool_create_empty()/rte_mempool_populate() */ void rte_pktmbuf_pool_init(struct rte_mempool *mp, void *opaque_arg) @@ -110,7 +110,7 @@ rte_pktmbuf_pool_init(struct rte_mempool *mp, void *opaque_arg) /* * pktmbuf constructor, given as a callback function to - * rte_mempool_create(). + * rte_mempool_obj_iter() or rte_mempool_create(). * Set the fields of a packet mbuf to their default values. */ void @@ -130,12 +130,11 @@ rte_pktmbuf_init(struct rte_mempool *mp, RTE_ASSERT(mp->elt_size >= mbuf_size); RTE_ASSERT(buf_len <= UINT16_MAX); - memset(m, 0, mp->elt_size); - + memset(m, 0, mbuf_size); /* start of buffer is after mbuf structure and priv data */ m->priv_size = priv_size; m->buf_addr = (char *)m + mbuf_size; - m->buf_physaddr = rte_mempool_virt2phy(mp, m) + mbuf_size; + m->buf_iova = rte_mempool_virt2iova(m) + mbuf_size; m->buf_len = (uint16_t)buf_len; /* keep some headroom between start of buffer and data */ @@ -144,7 +143,9 @@ rte_pktmbuf_init(struct rte_mempool *mp, /* init some constant fields */ m->pool = mp; m->nb_segs = 1; - m->port = 0xff; + m->port = MBUF_INVALID_PORT; + rte_mbuf_refcnt_set(m, 1); + m->next = NULL; } /* helper to create a mbuf pool */ @@ -155,6 +156,7 @@ rte_pktmbuf_pool_create(const char *name, unsigned n, { struct rte_mempool *mp; struct rte_pktmbuf_pool_private mbp_priv; + const char *mp_ops_name; unsigned elt_size; int ret; @@ -174,8 +176,8 @@ rte_pktmbuf_pool_create(const char *name, unsigned n, if (mp == NULL) return NULL; - ret = rte_mempool_set_ops_byname(mp, - RTE_MBUF_DEFAULT_MEMPOOL_OPS, NULL); + mp_ops_name = rte_eal_mbuf_default_mempool_ops(); + ret = rte_mempool_set_ops_byname(mp, mp_ops_name, NULL); if (ret != 0) { RTE_LOG(ERR, MBUF, "error setting mempool handler\n"); rte_mempool_free(mp); @@ -201,7 +203,7 @@ void rte_mbuf_sanity_check(const struct rte_mbuf *m, int is_header) { const struct rte_mbuf *m_seg; - unsigned nb_segs; + unsigned int nb_segs; if (m == NULL) rte_panic("mbuf is NULL\n"); @@ -209,8 +211,8 @@ rte_mbuf_sanity_check(const struct rte_mbuf *m, int is_header) /* generic checks */ if (m->pool == NULL) rte_panic("bad mbuf pool\n"); - if (m->buf_physaddr == 0) - rte_panic("bad phys addr\n"); + if (m->buf_iova == 0) + rte_panic("bad IO addr\n"); if (m->buf_addr == NULL) rte_panic("bad virt addr\n"); @@ -237,12 +239,12 @@ void rte_pktmbuf_dump(FILE *f, const struct rte_mbuf *m, unsigned dump_len) { unsigned int len; - unsigned nb_segs; + unsigned int nb_segs; __rte_mbuf_sanity_check(m, 1); - fprintf(f, "dump mbuf at %p, phys=%"PRIx64", buf_len=%u\n", - m, (uint64_t)m->buf_physaddr, (unsigned)m->buf_len); + fprintf(f, "dump mbuf at %p, iova=%"PRIx64", buf_len=%u\n", + m, (uint64_t)m->buf_iova, (unsigned)m->buf_len); fprintf(f, " pkt_len=%"PRIu32", ol_flags=%"PRIx64", nb_segs=%u, " "in_port=%u\n", m->pkt_len, m->ol_flags, (unsigned)m->nb_segs, (unsigned)m->port); @@ -264,6 +266,40 @@ rte_pktmbuf_dump(FILE *f, const struct rte_mbuf *m, unsigned dump_len) } } +/* read len data bytes in a mbuf at specified offset (internal) */ +const void *__rte_pktmbuf_read(const struct rte_mbuf *m, uint32_t off, + uint32_t len, void *buf) +{ + const struct rte_mbuf *seg = m; + uint32_t buf_off = 0, copy_len; + + if (off + len > rte_pktmbuf_pkt_len(m)) + return NULL; + + while (off >= rte_pktmbuf_data_len(seg)) { + off -= rte_pktmbuf_data_len(seg); + seg = seg->next; + } + + if (off + len <= rte_pktmbuf_data_len(seg)) + return rte_pktmbuf_mtod_offset(seg, char *, off); + + /* rare case: header is split among several segments */ + while (len > 0) { + copy_len = rte_pktmbuf_data_len(seg) - off; + if (copy_len > len) + copy_len = len; + rte_memcpy((char *)buf + buf_off, + rte_pktmbuf_mtod_offset(seg, char *, off), copy_len); + off = 0; + buf_off += copy_len; + len -= copy_len; + seg = seg->next; + } + + return buf; +} + /* * Get the name of a RX offload flag. Must be kept synchronized with flag * definitions in rte_mbuf.h. @@ -271,20 +307,89 @@ rte_pktmbuf_dump(FILE *f, const struct rte_mbuf *m, unsigned dump_len) const char *rte_get_rx_ol_flag_name(uint64_t mask) { switch (mask) { - case PKT_RX_VLAN_PKT: return "PKT_RX_VLAN_PKT"; + case PKT_RX_VLAN: return "PKT_RX_VLAN"; case PKT_RX_RSS_HASH: return "PKT_RX_RSS_HASH"; case PKT_RX_FDIR: return "PKT_RX_FDIR"; case PKT_RX_L4_CKSUM_BAD: return "PKT_RX_L4_CKSUM_BAD"; + case PKT_RX_L4_CKSUM_GOOD: return "PKT_RX_L4_CKSUM_GOOD"; + case PKT_RX_L4_CKSUM_NONE: return "PKT_RX_L4_CKSUM_NONE"; case PKT_RX_IP_CKSUM_BAD: return "PKT_RX_IP_CKSUM_BAD"; + case PKT_RX_IP_CKSUM_GOOD: return "PKT_RX_IP_CKSUM_GOOD"; + case PKT_RX_IP_CKSUM_NONE: return "PKT_RX_IP_CKSUM_NONE"; case PKT_RX_EIP_CKSUM_BAD: return "PKT_RX_EIP_CKSUM_BAD"; case PKT_RX_VLAN_STRIPPED: return "PKT_RX_VLAN_STRIPPED"; case PKT_RX_IEEE1588_PTP: return "PKT_RX_IEEE1588_PTP"; case PKT_RX_IEEE1588_TMST: return "PKT_RX_IEEE1588_TMST"; case PKT_RX_QINQ_STRIPPED: return "PKT_RX_QINQ_STRIPPED"; + case PKT_RX_LRO: return "PKT_RX_LRO"; + case PKT_RX_TIMESTAMP: return "PKT_RX_TIMESTAMP"; + case PKT_RX_SEC_OFFLOAD: return "PKT_RX_SEC_OFFLOAD"; + case PKT_RX_SEC_OFFLOAD_FAILED: return "PKT_RX_SEC_OFFLOAD_FAILED"; default: return NULL; } } +struct flag_mask { + uint64_t flag; + uint64_t mask; + const char *default_name; +}; + +/* write the list of rx ol flags in buffer buf */ +int +rte_get_rx_ol_flag_list(uint64_t mask, char *buf, size_t buflen) +{ + const struct flag_mask rx_flags[] = { + { PKT_RX_VLAN, PKT_RX_VLAN, NULL }, + { PKT_RX_RSS_HASH, PKT_RX_RSS_HASH, NULL }, + { PKT_RX_FDIR, PKT_RX_FDIR, NULL }, + { PKT_RX_L4_CKSUM_BAD, PKT_RX_L4_CKSUM_MASK, NULL }, + { PKT_RX_L4_CKSUM_GOOD, PKT_RX_L4_CKSUM_MASK, NULL }, + { PKT_RX_L4_CKSUM_NONE, PKT_RX_L4_CKSUM_MASK, NULL }, + { PKT_RX_L4_CKSUM_UNKNOWN, PKT_RX_L4_CKSUM_MASK, + "PKT_RX_L4_CKSUM_UNKNOWN" }, + { PKT_RX_IP_CKSUM_BAD, PKT_RX_IP_CKSUM_MASK, NULL }, + { PKT_RX_IP_CKSUM_GOOD, PKT_RX_IP_CKSUM_MASK, NULL }, + { PKT_RX_IP_CKSUM_NONE, PKT_RX_IP_CKSUM_MASK, NULL }, + { PKT_RX_IP_CKSUM_UNKNOWN, PKT_RX_IP_CKSUM_MASK, + "PKT_RX_IP_CKSUM_UNKNOWN" }, + { PKT_RX_EIP_CKSUM_BAD, PKT_RX_EIP_CKSUM_BAD, NULL }, + { PKT_RX_VLAN_STRIPPED, PKT_RX_VLAN_STRIPPED, NULL }, + { PKT_RX_IEEE1588_PTP, PKT_RX_IEEE1588_PTP, NULL }, + { PKT_RX_IEEE1588_TMST, PKT_RX_IEEE1588_TMST, NULL }, + { PKT_RX_QINQ_STRIPPED, PKT_RX_QINQ_STRIPPED, NULL }, + { PKT_RX_LRO, PKT_RX_LRO, NULL }, + { PKT_RX_TIMESTAMP, PKT_RX_TIMESTAMP, NULL }, + { PKT_RX_SEC_OFFLOAD, PKT_RX_SEC_OFFLOAD, NULL }, + { PKT_RX_SEC_OFFLOAD_FAILED, PKT_RX_SEC_OFFLOAD_FAILED, NULL }, + { PKT_RX_QINQ, PKT_RX_QINQ, NULL }, + }; + const char *name; + unsigned int i; + int ret; + + if (buflen == 0) + return -1; + + buf[0] = '\0'; + for (i = 0; i < RTE_DIM(rx_flags); i++) { + if ((mask & rx_flags[i].mask) != rx_flags[i].flag) + continue; + name = rte_get_rx_ol_flag_name(rx_flags[i].flag); + if (name == NULL) + name = rx_flags[i].default_name; + ret = snprintf(buf, buflen, "%s ", name); + if (ret < 0) + return -1; + if ((size_t)ret >= buflen) + return -1; + buf += ret; + buflen -= ret; + } + + return 0; +} + /* * Get the name of a TX offload flag. Must be kept synchronized with flag * definitions in rte_mbuf.h. @@ -304,6 +409,70 @@ const char *rte_get_tx_ol_flag_name(uint64_t mask) case PKT_TX_OUTER_IP_CKSUM: return "PKT_TX_OUTER_IP_CKSUM"; case PKT_TX_OUTER_IPV4: return "PKT_TX_OUTER_IPV4"; case PKT_TX_OUTER_IPV6: return "PKT_TX_OUTER_IPV6"; + case PKT_TX_TUNNEL_VXLAN: return "PKT_TX_TUNNEL_VXLAN"; + case PKT_TX_TUNNEL_GRE: return "PKT_TX_TUNNEL_GRE"; + case PKT_TX_TUNNEL_IPIP: return "PKT_TX_TUNNEL_IPIP"; + case PKT_TX_TUNNEL_GENEVE: return "PKT_TX_TUNNEL_GENEVE"; + case PKT_TX_TUNNEL_MPLSINUDP: return "PKT_TX_TUNNEL_MPLSINUDP"; + case PKT_TX_MACSEC: return "PKT_TX_MACSEC"; + case PKT_TX_SEC_OFFLOAD: return "PKT_TX_SEC_OFFLOAD"; default: return NULL; } } + +/* write the list of tx ol flags in buffer buf */ +int +rte_get_tx_ol_flag_list(uint64_t mask, char *buf, size_t buflen) +{ + const struct flag_mask tx_flags[] = { + { PKT_TX_VLAN_PKT, PKT_TX_VLAN_PKT, NULL }, + { PKT_TX_IP_CKSUM, PKT_TX_IP_CKSUM, NULL }, + { PKT_TX_TCP_CKSUM, PKT_TX_L4_MASK, NULL }, + { PKT_TX_SCTP_CKSUM, PKT_TX_L4_MASK, NULL }, + { PKT_TX_UDP_CKSUM, PKT_TX_L4_MASK, NULL }, + { PKT_TX_L4_NO_CKSUM, PKT_TX_L4_MASK, "PKT_TX_L4_NO_CKSUM" }, + { PKT_TX_IEEE1588_TMST, PKT_TX_IEEE1588_TMST, NULL }, + { PKT_TX_TCP_SEG, PKT_TX_TCP_SEG, NULL }, + { PKT_TX_IPV4, PKT_TX_IPV4, NULL }, + { PKT_TX_IPV6, PKT_TX_IPV6, NULL }, + { PKT_TX_OUTER_IP_CKSUM, PKT_TX_OUTER_IP_CKSUM, NULL }, + { PKT_TX_OUTER_IPV4, PKT_TX_OUTER_IPV4, NULL }, + { PKT_TX_OUTER_IPV6, PKT_TX_OUTER_IPV6, NULL }, + { PKT_TX_TUNNEL_VXLAN, PKT_TX_TUNNEL_MASK, + "PKT_TX_TUNNEL_NONE" }, + { PKT_TX_TUNNEL_GRE, PKT_TX_TUNNEL_MASK, + "PKT_TX_TUNNEL_NONE" }, + { PKT_TX_TUNNEL_IPIP, PKT_TX_TUNNEL_MASK, + "PKT_TX_TUNNEL_NONE" }, + { PKT_TX_TUNNEL_GENEVE, PKT_TX_TUNNEL_MASK, + "PKT_TX_TUNNEL_NONE" }, + { PKT_TX_TUNNEL_MPLSINUDP, PKT_TX_TUNNEL_MASK, + "PKT_TX_TUNNEL_NONE" }, + { PKT_TX_MACSEC, PKT_TX_MACSEC, NULL }, + { PKT_TX_SEC_OFFLOAD, PKT_TX_SEC_OFFLOAD, NULL }, + }; + const char *name; + unsigned int i; + int ret; + + if (buflen == 0) + return -1; + + buf[0] = '\0'; + for (i = 0; i < RTE_DIM(tx_flags); i++) { + if ((mask & tx_flags[i].mask) != tx_flags[i].flag) + continue; + name = rte_get_tx_ol_flag_name(tx_flags[i].flag); + if (name == NULL) + name = tx_flags[i].default_name; + ret = snprintf(buf, buflen, "%s ", name); + if (ret < 0) + return -1; + if ((size_t)ret >= buflen) + return -1; + buf += ret; + buflen -= ret; + } + + return 0; +} diff --git a/dpdk/lib/librte_mbuf/rte_mbuf.h b/dpdk/lib/librte_mbuf/rte_mbuf.h index 101485fb..16a6048c 100644 --- a/dpdk/lib/librte_mbuf/rte_mbuf.h +++ b/dpdk/lib/librte_mbuf/rte_mbuf.h @@ -44,7 +44,14 @@ * buffers. The message buffers are stored in a mempool, using the * RTE mempool library. * - * This library provide an API to allocate/free packet mbufs, which are + * The preferred way to create a mbuf pool is to use + * rte_pktmbuf_pool_create(). However, in some situations, an + * application may want to have more control (ex: populate the pool with + * specific memory), in this case it is possible to use functions from + * rte_mempool. See how rte_pktmbuf_pool_create() is implemented for + * details. + * + * This library provides an API to allocate/free packet mbufs, which are * used to carry network packets. * * To understand the concepts of packet buffers or mbufs, you @@ -55,11 +62,13 @@ #include #include +#include #include #include #include #include #include +#include #ifdef __cplusplus extern "C" { @@ -81,27 +90,74 @@ extern "C" { */ /** - * RX packet is a 802.1q VLAN packet. This flag was set by PMDs when - * the packet is recognized as a VLAN, but the behavior between PMDs - * was not the same. This flag is kept for some time to avoid breaking - * applications and should be replaced by PKT_RX_VLAN_STRIPPED. + * The RX packet is a 802.1q VLAN packet, and the tci has been + * saved in in mbuf->vlan_tci. + * If the flag PKT_RX_VLAN_STRIPPED is also present, the VLAN + * header has been stripped from mbuf data, else it is still + * present. */ -#define PKT_RX_VLAN_PKT (1ULL << 0) +#define PKT_RX_VLAN (1ULL << 0) #define PKT_RX_RSS_HASH (1ULL << 1) /**< RX packet with RSS hash result. */ #define PKT_RX_FDIR (1ULL << 2) /**< RX packet with FDIR match indicate. */ -#define PKT_RX_L4_CKSUM_BAD (1ULL << 3) /**< L4 cksum of RX pkt. is not OK. */ -#define PKT_RX_IP_CKSUM_BAD (1ULL << 4) /**< IP cksum of RX pkt. is not OK. */ + +/** + * Deprecated. + * Checking this flag alone is deprecated: check the 2 bits of + * PKT_RX_L4_CKSUM_MASK. + * This flag was set when the L4 checksum of a packet was detected as + * wrong by the hardware. + */ +#define PKT_RX_L4_CKSUM_BAD (1ULL << 3) + +/** + * Deprecated. + * Checking this flag alone is deprecated: check the 2 bits of + * PKT_RX_IP_CKSUM_MASK. + * This flag was set when the IP checksum of a packet was detected as + * wrong by the hardware. + */ +#define PKT_RX_IP_CKSUM_BAD (1ULL << 4) + #define PKT_RX_EIP_CKSUM_BAD (1ULL << 5) /**< External IP header checksum error. */ /** * A vlan has been stripped by the hardware and its tci is saved in * mbuf->vlan_tci. This can only happen if vlan stripping is enabled * in the RX configuration of the PMD. + * When PKT_RX_VLAN_STRIPPED is set, PKT_RX_VLAN must also be set. */ #define PKT_RX_VLAN_STRIPPED (1ULL << 6) -/* hole, some bits can be reused here */ +/** + * Mask of bits used to determine the status of RX IP checksum. + * - PKT_RX_IP_CKSUM_UNKNOWN: no information about the RX IP checksum + * - PKT_RX_IP_CKSUM_BAD: the IP checksum in the packet is wrong + * - PKT_RX_IP_CKSUM_GOOD: the IP checksum in the packet is valid + * - PKT_RX_IP_CKSUM_NONE: the IP checksum is not correct in the packet + * data, but the integrity of the IP header is verified. + */ +#define PKT_RX_IP_CKSUM_MASK ((1ULL << 4) | (1ULL << 7)) + +#define PKT_RX_IP_CKSUM_UNKNOWN 0 +#define PKT_RX_IP_CKSUM_BAD (1ULL << 4) +#define PKT_RX_IP_CKSUM_GOOD (1ULL << 7) +#define PKT_RX_IP_CKSUM_NONE ((1ULL << 4) | (1ULL << 7)) + +/** + * Mask of bits used to determine the status of RX L4 checksum. + * - PKT_RX_L4_CKSUM_UNKNOWN: no information about the RX L4 checksum + * - PKT_RX_L4_CKSUM_BAD: the L4 checksum in the packet is wrong + * - PKT_RX_L4_CKSUM_GOOD: the L4 checksum in the packet is valid + * - PKT_RX_L4_CKSUM_NONE: the L4 checksum is not correct in the packet + * data, but the integrity of the L4 data is verified. + */ +#define PKT_RX_L4_CKSUM_MASK ((1ULL << 3) | (1ULL << 8)) + +#define PKT_RX_L4_CKSUM_UNKNOWN 0 +#define PKT_RX_L4_CKSUM_BAD (1ULL << 3) +#define PKT_RX_L4_CKSUM_GOOD (1ULL << 8) +#define PKT_RX_L4_CKSUM_NONE ((1ULL << 3) | (1ULL << 8)) #define PKT_RX_IEEE1588_PTP (1ULL << 9) /**< RX IEEE1588 L2 Ethernet PT Packet. */ #define PKT_RX_IEEE1588_TMST (1ULL << 10) /**< RX IEEE1588 L2/L4 timestamped packet.*/ @@ -112,22 +168,72 @@ extern "C" { * The 2 vlans have been stripped by the hardware and their tci are * saved in mbuf->vlan_tci (inner) and mbuf->vlan_tci_outer (outer). * This can only happen if vlan stripping is enabled in the RX - * configuration of the PMD. If this flag is set, PKT_RX_VLAN_STRIPPED - * must also be set. + * configuration of the PMD. If this flag is set, + * When PKT_RX_QINQ_STRIPPED is set, the flags (PKT_RX_VLAN | + * PKT_RX_VLAN_STRIPPED | PKT_RX_QINQ) must also be set. */ #define PKT_RX_QINQ_STRIPPED (1ULL << 15) /** - * Deprecated. - * RX packet with double VLAN stripped. - * This flag is replaced by PKT_RX_QINQ_STRIPPED. + * When packets are coalesced by a hardware or virtual driver, this flag + * can be set in the RX mbuf, meaning that the m->tso_segsz field is + * valid and is set to the segment size of original packets. */ -#define PKT_RX_QINQ_PKT PKT_RX_QINQ_STRIPPED +#define PKT_RX_LRO (1ULL << 16) + +/** + * Indicate that the timestamp field in the mbuf is valid. + */ +#define PKT_RX_TIMESTAMP (1ULL << 17) + +/** + * Indicate that security offload processing was applied on the RX packet. + */ +#define PKT_RX_SEC_OFFLOAD (1ULL << 18) + +/** + * Indicate that security offload processing failed on the RX packet. + */ +#define PKT_RX_SEC_OFFLOAD_FAILED (1ULL << 19) + +/** + * The RX packet is a double VLAN, and the outer tci has been + * saved in in mbuf->vlan_tci_outer. + * If the flag PKT_RX_QINQ_STRIPPED is also present, both VLANs + * headers have been stripped from mbuf data, else they are still + * present. + */ +#define PKT_RX_QINQ (1ULL << 20) /* add new RX flags here */ /* add new TX flags here */ +/** + * Request security offload processing on the TX packet. + */ +#define PKT_TX_SEC_OFFLOAD (1ULL << 43) + +/** + * Offload the MACsec. This flag must be set by the application to enable + * this offload feature for a packet to be transmitted. + */ +#define PKT_TX_MACSEC (1ULL << 44) + +/** + * Bits 45:48 used for the tunnel type. + * When doing Tx offload like TSO or checksum, the HW needs to configure the + * tunnel type into the HW descriptors. + */ +#define PKT_TX_TUNNEL_VXLAN (0x1ULL << 45) +#define PKT_TX_TUNNEL_GRE (0x2ULL << 45) +#define PKT_TX_TUNNEL_IPIP (0x3ULL << 45) +#define PKT_TX_TUNNEL_GENEVE (0x4ULL << 45) +/**< TX packet with MPLS-in-UDP RFC 7510 header. */ +#define PKT_TX_TUNNEL_MPLSINUDP (0x5ULL << 45) +/* add new TX TUNNEL type here */ +#define PKT_TX_TUNNEL_MASK (0xFULL << 45) + /** * Second VLAN insertion (QinQ) flag. */ @@ -218,6 +324,22 @@ extern "C" { */ #define PKT_TX_OUTER_IPV6 (1ULL << 60) +/** + * Bitmask of all supported packet Tx offload features flags, + * which can be set for packet. + */ +#define PKT_TX_OFFLOAD_MASK ( \ + PKT_TX_IP_CKSUM | \ + PKT_TX_L4_MASK | \ + PKT_TX_OUTER_IP_CKSUM | \ + PKT_TX_TCP_SEG | \ + PKT_TX_IEEE1588_TMST | \ + PKT_TX_QINQ_PKT | \ + PKT_TX_VLAN_PKT | \ + PKT_TX_TUNNEL_MASK | \ + PKT_TX_MACSEC | \ + PKT_TX_SEC_OFFLOAD) + #define __RESERVED (1ULL << 61) /**< reserved for future mbuf use */ #define IND_ATTACHED_MBUF (1ULL << 62) /**< Indirect attached mbuf */ @@ -225,500 +347,6 @@ extern "C" { /* Use final bit of flags to indicate a control mbuf */ #define CTRL_MBUF_FLAG (1ULL << 63) /**< Mbuf contains control data */ -/* - * 32 bits are divided into several fields to mark packet types. Note that - * each field is indexical. - * - Bit 3:0 is for L2 types. - * - Bit 7:4 is for L3 or outer L3 (for tunneling case) types. - * - Bit 11:8 is for L4 or outer L4 (for tunneling case) types. - * - Bit 15:12 is for tunnel types. - * - Bit 19:16 is for inner L2 types. - * - Bit 23:20 is for inner L3 types. - * - Bit 27:24 is for inner L4 types. - * - Bit 31:28 is reserved. - * - * To be compatible with Vector PMD, RTE_PTYPE_L3_IPV4, RTE_PTYPE_L3_IPV4_EXT, - * RTE_PTYPE_L3_IPV6, RTE_PTYPE_L3_IPV6_EXT, RTE_PTYPE_L4_TCP, RTE_PTYPE_L4_UDP - * and RTE_PTYPE_L4_SCTP should be kept as below in a contiguous 7 bits. - * - * Note that L3 types values are selected for checking IPV4/IPV6 header from - * performance point of view. Reading annotations of RTE_ETH_IS_IPV4_HDR and - * RTE_ETH_IS_IPV6_HDR is needed for any future changes of L3 type values. - * - * Note that the packet types of the same packet recognized by different - * hardware may be different, as different hardware may have different - * capability of packet type recognition. - * - * examples: - * <'ether type'=0x0800 - * | 'version'=4, 'protocol'=0x29 - * | 'version'=6, 'next header'=0x3A - * | 'ICMPv6 header'> - * will be recognized on i40e hardware as packet type combination of, - * RTE_PTYPE_L2_ETHER | - * RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | - * RTE_PTYPE_TUNNEL_IP | - * RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | - * RTE_PTYPE_INNER_L4_ICMP. - * - * <'ether type'=0x86DD - * | 'version'=6, 'next header'=0x2F - * | 'GRE header' - * | 'version'=6, 'next header'=0x11 - * | 'UDP header'> - * will be recognized on i40e hardware as packet type combination of, - * RTE_PTYPE_L2_ETHER | - * RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | - * RTE_PTYPE_TUNNEL_GRENAT | - * RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | - * RTE_PTYPE_INNER_L4_UDP. - */ -#define RTE_PTYPE_UNKNOWN 0x00000000 -/** - * Ethernet packet type. - * It is used for outer packet for tunneling cases. - * - * Packet format: - * <'ether type'=[0x0800|0x86DD]> - */ -#define RTE_PTYPE_L2_ETHER 0x00000001 -/** - * Ethernet packet type for time sync. - * - * Packet format: - * <'ether type'=0x88F7> - */ -#define RTE_PTYPE_L2_ETHER_TIMESYNC 0x00000002 -/** - * ARP (Address Resolution Protocol) packet type. - * - * Packet format: - * <'ether type'=0x0806> - */ -#define RTE_PTYPE_L2_ETHER_ARP 0x00000003 -/** - * LLDP (Link Layer Discovery Protocol) packet type. - * - * Packet format: - * <'ether type'=0x88CC> - */ -#define RTE_PTYPE_L2_ETHER_LLDP 0x00000004 -/** - * NSH (Network Service Header) packet type. - * - * Packet format: - * <'ether type'=0x894F> - */ -#define RTE_PTYPE_L2_ETHER_NSH 0x00000005 -/** - * Mask of layer 2 packet types. - * It is used for outer packet for tunneling cases. - */ -#define RTE_PTYPE_L2_MASK 0x0000000f -/** - * IP (Internet Protocol) version 4 packet type. - * It is used for outer packet for tunneling cases, and does not contain any - * header option. - * - * Packet format: - * <'ether type'=0x0800 - * | 'version'=4, 'ihl'=5> - */ -#define RTE_PTYPE_L3_IPV4 0x00000010 -/** - * IP (Internet Protocol) version 4 packet type. - * It is used for outer packet for tunneling cases, and contains header - * options. - * - * Packet format: - * <'ether type'=0x0800 - * | 'version'=4, 'ihl'=[6-15], 'options'> - */ -#define RTE_PTYPE_L3_IPV4_EXT 0x00000030 -/** - * IP (Internet Protocol) version 6 packet type. - * It is used for outer packet for tunneling cases, and does not contain any - * extension header. - * - * Packet format: - * <'ether type'=0x86DD - * | 'version'=6, 'next header'=0x3B> - */ -#define RTE_PTYPE_L3_IPV6 0x00000040 -/** - * IP (Internet Protocol) version 4 packet type. - * It is used for outer packet for tunneling cases, and may or maynot contain - * header options. - * - * Packet format: - * <'ether type'=0x0800 - * | 'version'=4, 'ihl'=[5-15], <'options'>> - */ -#define RTE_PTYPE_L3_IPV4_EXT_UNKNOWN 0x00000090 -/** - * IP (Internet Protocol) version 6 packet type. - * It is used for outer packet for tunneling cases, and contains extension - * headers. - * - * Packet format: - * <'ether type'=0x86DD - * | 'version'=6, 'next header'=[0x0|0x2B|0x2C|0x32|0x33|0x3C|0x87], - * 'extension headers'> - */ -#define RTE_PTYPE_L3_IPV6_EXT 0x000000c0 -/** - * IP (Internet Protocol) version 6 packet type. - * It is used for outer packet for tunneling cases, and may or maynot contain - * extension headers. - * - * Packet format: - * <'ether type'=0x86DD - * | 'version'=6, 'next header'=[0x3B|0x0|0x2B|0x2C|0x32|0x33|0x3C|0x87], - * <'extension headers'>> - */ -#define RTE_PTYPE_L3_IPV6_EXT_UNKNOWN 0x000000e0 -/** - * Mask of layer 3 packet types. - * It is used for outer packet for tunneling cases. - */ -#define RTE_PTYPE_L3_MASK 0x000000f0 -/** - * TCP (Transmission Control Protocol) packet type. - * It is used for outer packet for tunneling cases. - * - * Packet format: - * <'ether type'=0x0800 - * | 'version'=4, 'protocol'=6, 'MF'=0> - * or, - * <'ether type'=0x86DD - * | 'version'=6, 'next header'=6> - */ -#define RTE_PTYPE_L4_TCP 0x00000100 -/** - * UDP (User Datagram Protocol) packet type. - * It is used for outer packet for tunneling cases. - * - * Packet format: - * <'ether type'=0x0800 - * | 'version'=4, 'protocol'=17, 'MF'=0> - * or, - * <'ether type'=0x86DD - * | 'version'=6, 'next header'=17> - */ -#define RTE_PTYPE_L4_UDP 0x00000200 -/** - * Fragmented IP (Internet Protocol) packet type. - * It is used for outer packet for tunneling cases. - * - * It refers to those packets of any IP types, which can be recognized as - * fragmented. A fragmented packet cannot be recognized as any other L4 types - * (RTE_PTYPE_L4_TCP, RTE_PTYPE_L4_UDP, RTE_PTYPE_L4_SCTP, RTE_PTYPE_L4_ICMP, - * RTE_PTYPE_L4_NONFRAG). - * - * Packet format: - * <'ether type'=0x0800 - * | 'version'=4, 'MF'=1> - * or, - * <'ether type'=0x86DD - * | 'version'=6, 'next header'=44> - */ -#define RTE_PTYPE_L4_FRAG 0x00000300 -/** - * SCTP (Stream Control Transmission Protocol) packet type. - * It is used for outer packet for tunneling cases. - * - * Packet format: - * <'ether type'=0x0800 - * | 'version'=4, 'protocol'=132, 'MF'=0> - * or, - * <'ether type'=0x86DD - * | 'version'=6, 'next header'=132> - */ -#define RTE_PTYPE_L4_SCTP 0x00000400 -/** - * ICMP (Internet Control Message Protocol) packet type. - * It is used for outer packet for tunneling cases. - * - * Packet format: - * <'ether type'=0x0800 - * | 'version'=4, 'protocol'=1, 'MF'=0> - * or, - * <'ether type'=0x86DD - * | 'version'=6, 'next header'=1> - */ -#define RTE_PTYPE_L4_ICMP 0x00000500 -/** - * Non-fragmented IP (Internet Protocol) packet type. - * It is used for outer packet for tunneling cases. - * - * It refers to those packets of any IP types, while cannot be recognized as - * any of above L4 types (RTE_PTYPE_L4_TCP, RTE_PTYPE_L4_UDP, - * RTE_PTYPE_L4_FRAG, RTE_PTYPE_L4_SCTP, RTE_PTYPE_L4_ICMP). - * - * Packet format: - * <'ether type'=0x0800 - * | 'version'=4, 'protocol'!=[6|17|132|1], 'MF'=0> - * or, - * <'ether type'=0x86DD - * | 'version'=6, 'next header'!=[6|17|44|132|1]> - */ -#define RTE_PTYPE_L4_NONFRAG 0x00000600 -/** - * Mask of layer 4 packet types. - * It is used for outer packet for tunneling cases. - */ -#define RTE_PTYPE_L4_MASK 0x00000f00 -/** - * IP (Internet Protocol) in IP (Internet Protocol) tunneling packet type. - * - * Packet format: - * <'ether type'=0x0800 - * | 'version'=4, 'protocol'=[4|41]> - * or, - * <'ether type'=0x86DD - * | 'version'=6, 'next header'=[4|41]> - */ -#define RTE_PTYPE_TUNNEL_IP 0x00001000 -/** - * GRE (Generic Routing Encapsulation) tunneling packet type. - * - * Packet format: - * <'ether type'=0x0800 - * | 'version'=4, 'protocol'=47> - * or, - * <'ether type'=0x86DD - * | 'version'=6, 'next header'=47> - */ -#define RTE_PTYPE_TUNNEL_GRE 0x00002000 -/** - * VXLAN (Virtual eXtensible Local Area Network) tunneling packet type. - * - * Packet format: - * <'ether type'=0x0800 - * | 'version'=4, 'protocol'=17 - * | 'destination port'=4798> - * or, - * <'ether type'=0x86DD - * | 'version'=6, 'next header'=17 - * | 'destination port'=4798> - */ -#define RTE_PTYPE_TUNNEL_VXLAN 0x00003000 -/** - * NVGRE (Network Virtualization using Generic Routing Encapsulation) tunneling - * packet type. - * - * Packet format: - * <'ether type'=0x0800 - * | 'version'=4, 'protocol'=47 - * | 'protocol type'=0x6558> - * or, - * <'ether type'=0x86DD - * | 'version'=6, 'next header'=47 - * | 'protocol type'=0x6558'> - */ -#define RTE_PTYPE_TUNNEL_NVGRE 0x00004000 -/** - * GENEVE (Generic Network Virtualization Encapsulation) tunneling packet type. - * - * Packet format: - * <'ether type'=0x0800 - * | 'version'=4, 'protocol'=17 - * | 'destination port'=6081> - * or, - * <'ether type'=0x86DD - * | 'version'=6, 'next header'=17 - * | 'destination port'=6081> - */ -#define RTE_PTYPE_TUNNEL_GENEVE 0x00005000 -/** - * Tunneling packet type of Teredo, VXLAN (Virtual eXtensible Local Area - * Network) or GRE (Generic Routing Encapsulation) could be recognized as this - * packet type, if they can not be recognized independently as of hardware - * capability. - */ -#define RTE_PTYPE_TUNNEL_GRENAT 0x00006000 -/** - * Mask of tunneling packet types. - */ -#define RTE_PTYPE_TUNNEL_MASK 0x0000f000 -/** - * Ethernet packet type. - * It is used for inner packet type only. - * - * Packet format (inner only): - * <'ether type'=[0x800|0x86DD]> - */ -#define RTE_PTYPE_INNER_L2_ETHER 0x00010000 -/** - * Ethernet packet type with VLAN (Virtual Local Area Network) tag. - * - * Packet format (inner only): - * <'ether type'=[0x800|0x86DD], vlan=[1-4095]> - */ -#define RTE_PTYPE_INNER_L2_ETHER_VLAN 0x00020000 -/** - * Mask of inner layer 2 packet types. - */ -#define RTE_PTYPE_INNER_L2_MASK 0x000f0000 -/** - * IP (Internet Protocol) version 4 packet type. - * It is used for inner packet only, and does not contain any header option. - * - * Packet format (inner only): - * <'ether type'=0x0800 - * | 'version'=4, 'ihl'=5> - */ -#define RTE_PTYPE_INNER_L3_IPV4 0x00100000 -/** - * IP (Internet Protocol) version 4 packet type. - * It is used for inner packet only, and contains header options. - * - * Packet format (inner only): - * <'ether type'=0x0800 - * | 'version'=4, 'ihl'=[6-15], 'options'> - */ -#define RTE_PTYPE_INNER_L3_IPV4_EXT 0x00200000 -/** - * IP (Internet Protocol) version 6 packet type. - * It is used for inner packet only, and does not contain any extension header. - * - * Packet format (inner only): - * <'ether type'=0x86DD - * | 'version'=6, 'next header'=0x3B> - */ -#define RTE_PTYPE_INNER_L3_IPV6 0x00300000 -/** - * IP (Internet Protocol) version 4 packet type. - * It is used for inner packet only, and may or maynot contain header options. - * - * Packet format (inner only): - * <'ether type'=0x0800 - * | 'version'=4, 'ihl'=[5-15], <'options'>> - */ -#define RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN 0x00400000 -/** - * IP (Internet Protocol) version 6 packet type. - * It is used for inner packet only, and contains extension headers. - * - * Packet format (inner only): - * <'ether type'=0x86DD - * | 'version'=6, 'next header'=[0x0|0x2B|0x2C|0x32|0x33|0x3C|0x87], - * 'extension headers'> - */ -#define RTE_PTYPE_INNER_L3_IPV6_EXT 0x00500000 -/** - * IP (Internet Protocol) version 6 packet type. - * It is used for inner packet only, and may or maynot contain extension - * headers. - * - * Packet format (inner only): - * <'ether type'=0x86DD - * | 'version'=6, 'next header'=[0x3B|0x0|0x2B|0x2C|0x32|0x33|0x3C|0x87], - * <'extension headers'>> - */ -#define RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN 0x00600000 -/** - * Mask of inner layer 3 packet types. - */ -#define RTE_PTYPE_INNER_L3_MASK 0x00f00000 -/** - * TCP (Transmission Control Protocol) packet type. - * It is used for inner packet only. - * - * Packet format (inner only): - * <'ether type'=0x0800 - * | 'version'=4, 'protocol'=6, 'MF'=0> - * or, - * <'ether type'=0x86DD - * | 'version'=6, 'next header'=6> - */ -#define RTE_PTYPE_INNER_L4_TCP 0x01000000 -/** - * UDP (User Datagram Protocol) packet type. - * It is used for inner packet only. - * - * Packet format (inner only): - * <'ether type'=0x0800 - * | 'version'=4, 'protocol'=17, 'MF'=0> - * or, - * <'ether type'=0x86DD - * | 'version'=6, 'next header'=17> - */ -#define RTE_PTYPE_INNER_L4_UDP 0x02000000 -/** - * Fragmented IP (Internet Protocol) packet type. - * It is used for inner packet only, and may or maynot have layer 4 packet. - * - * Packet format (inner only): - * <'ether type'=0x0800 - * | 'version'=4, 'MF'=1> - * or, - * <'ether type'=0x86DD - * | 'version'=6, 'next header'=44> - */ -#define RTE_PTYPE_INNER_L4_FRAG 0x03000000 -/** - * SCTP (Stream Control Transmission Protocol) packet type. - * It is used for inner packet only. - * - * Packet format (inner only): - * <'ether type'=0x0800 - * | 'version'=4, 'protocol'=132, 'MF'=0> - * or, - * <'ether type'=0x86DD - * | 'version'=6, 'next header'=132> - */ -#define RTE_PTYPE_INNER_L4_SCTP 0x04000000 -/** - * ICMP (Internet Control Message Protocol) packet type. - * It is used for inner packet only. - * - * Packet format (inner only): - * <'ether type'=0x0800 - * | 'version'=4, 'protocol'=1, 'MF'=0> - * or, - * <'ether type'=0x86DD - * | 'version'=6, 'next header'=1> - */ -#define RTE_PTYPE_INNER_L4_ICMP 0x05000000 -/** - * Non-fragmented IP (Internet Protocol) packet type. - * It is used for inner packet only, and may or maynot have other unknown layer - * 4 packet types. - * - * Packet format (inner only): - * <'ether type'=0x0800 - * | 'version'=4, 'protocol'!=[6|17|132|1], 'MF'=0> - * or, - * <'ether type'=0x86DD - * | 'version'=6, 'next header'!=[6|17|44|132|1]> - */ -#define RTE_PTYPE_INNER_L4_NONFRAG 0x06000000 -/** - * Mask of inner layer 4 packet types. - */ -#define RTE_PTYPE_INNER_L4_MASK 0x0f000000 - -/** - * Check if the (outer) L3 header is IPv4. To avoid comparing IPv4 types one by - * one, bit 4 is selected to be used for IPv4 only. Then checking bit 4 can - * determine if it is an IPV4 packet. - */ -#define RTE_ETH_IS_IPV4_HDR(ptype) ((ptype) & RTE_PTYPE_L3_IPV4) - -/** - * Check if the (outer) L3 header is IPv4. To avoid comparing IPv4 types one by - * one, bit 6 is selected to be used for IPv4 only. Then checking bit 6 can - * determine if it is an IPV4 packet. - */ -#define RTE_ETH_IS_IPV6_HDR(ptype) ((ptype) & RTE_PTYPE_L3_IPV6) - -/* Check if it is a tunneling packet */ -#define RTE_ETH_IS_TUNNEL_PKT(ptype) ((ptype) & (RTE_PTYPE_TUNNEL_MASK | \ - RTE_PTYPE_INNER_L2_MASK | \ - RTE_PTYPE_INNER_L3_MASK | \ - RTE_PTYPE_INNER_L4_MASK)) - /** Alignment constraint of mbuf private area. */ #define RTE_MBUF_PRIV_ALIGN 8 @@ -732,6 +360,20 @@ extern "C" { */ const char *rte_get_rx_ol_flag_name(uint64_t mask); +/** + * Dump the list of RX offload flags in a buffer + * + * @param mask + * The mask describing the RX flags. + * @param buf + * The output buffer. + * @param buflen + * The length of the buffer. + * @return + * 0 on success, (-1) on error. + */ +int rte_get_rx_ol_flag_list(uint64_t mask, char *buf, size_t buflen); + /** * Get the name of a TX offload flag * @@ -744,6 +386,20 @@ const char *rte_get_rx_ol_flag_name(uint64_t mask); */ const char *rte_get_tx_ol_flag_name(uint64_t mask); +/** + * Dump the list of TX offload flags in a buffer + * + * @param mask + * The mask describing the TX flags. + * @param buf + * The output buffer. + * @param buflen + * The length of the buffer. + * @return + * 0 on success, (-1) on error. + */ +int rte_get_tx_ol_flag_list(uint64_t mask, char *buf, size_t buflen); + /** * Some NICs need at least 2KB buffer to RX standard Ethernet frame without * splitting it into multiple segments. @@ -756,8 +412,11 @@ const char *rte_get_tx_ol_flag_name(uint64_t mask); /* define a set of marker types that can be used to refer to set points in the * mbuf */ +__extension__ typedef void *MARKER[0]; /**< generic marker for a point in a structure */ +__extension__ typedef uint8_t MARKER8[0]; /**< generic marker with 1B alignment */ +__extension__ typedef uint64_t MARKER64[0]; /**< marker that allows us to overwrite 8 bytes * with a single assignment */ @@ -768,28 +427,40 @@ struct rte_mbuf { MARKER cacheline0; void *buf_addr; /**< Virtual address of segment buffer. */ - phys_addr_t buf_physaddr; /**< Physical address of segment buffer. */ + /** + * Physical address of segment buffer. + * Force alignment to 8-bytes, so as to ensure we have the exact + * same mbuf cacheline0 layout for 32-bit and 64-bit. This makes + * working on vector drivers easier. + */ + RTE_STD_C11 + union { + rte_iova_t buf_iova; + rte_iova_t buf_physaddr; /**< deprecated */ + } __rte_aligned(sizeof(rte_iova_t)); - uint16_t buf_len; /**< Length of segment buffer. */ - - /* next 6 bytes are initialised on RX descriptor rearm */ - MARKER8 rearm_data; + /* next 8 bytes are initialised on RX descriptor rearm */ + MARKER64 rearm_data; uint16_t data_off; /** - * 16-bit Reference counter. + * Reference counter. Its size should at least equal to the size + * of port field (16 bits), to support zero-copy broadcast. * It should only be accessed using the following functions: * rte_mbuf_refcnt_update(), rte_mbuf_refcnt_read(), and * rte_mbuf_refcnt_set(). The functionality of these functions (atomic, * or non-atomic) is controlled by the CONFIG_RTE_MBUF_REFCNT_ATOMIC * config option. */ + RTE_STD_C11 union { rte_atomic16_t refcnt_atomic; /**< Atomically accessed refcnt */ uint16_t refcnt; /**< Non-atomically accessed refcnt */ }; - uint8_t nb_segs; /**< Number of segments. */ - uint8_t port; /**< Input port. */ + uint16_t nb_segs; /**< Number of segments. */ + + /** Input port (16 bits to support more than 256 virtual ports). */ + uint16_t port; uint64_t ol_flags; /**< Offload features. */ @@ -803,6 +474,7 @@ struct rte_mbuf { * would have RTE_PTYPE_L2_ETHER and not RTE_PTYPE_L2_VLAN because the * vlan is stripped from the data. */ + RTE_STD_C11 union { uint32_t packet_type; /**< L2/L3/L4 and tunnel information. */ struct { @@ -810,8 +482,21 @@ struct rte_mbuf { uint32_t l3_type:4; /**< (Outer) L3 type. */ uint32_t l4_type:4; /**< (Outer) L4 type. */ uint32_t tun_type:4; /**< Tunnel type. */ - uint32_t inner_l2_type:4; /**< Inner L2 type. */ - uint32_t inner_l3_type:4; /**< Inner L3 type. */ + RTE_STD_C11 + union { + uint8_t inner_esp_next_proto; + /**< ESP next protocol type, valid if + * RTE_PTYPE_TUNNEL_ESP tunnel type is set + * on both Tx and Rx. + */ + __extension__ + struct { + uint8_t inner_l2_type:4; + /**< Inner L2 type. */ + uint8_t inner_l3_type:4; + /**< Inner L3 type. */ + }; + }; uint32_t inner_l4_type:4; /**< Inner L4 type. */ }; }; @@ -824,6 +509,7 @@ struct rte_mbuf { union { uint32_t rss; /**< RSS hash result if RSS enabled */ struct { + RTE_STD_C11 union { struct { uint16_t hash; @@ -843,14 +529,20 @@ struct rte_mbuf { uint32_t usr; /**< User defined tags. See rte_distributor_process() */ } hash; /**< hash information */ - uint32_t seqn; /**< Sequence number. See also rte_reorder_insert() */ - /** Outer VLAN TCI (CPU order), valid if PKT_RX_QINQ_STRIPPED is set. */ uint16_t vlan_tci_outer; + uint16_t buf_len; /**< Length of segment buffer. */ + + /** Valid if PKT_RX_TIMESTAMP is set. The unit and time reference + * are not normalized but are always the same for a given port. + */ + uint64_t timestamp; + /* second cache line - fields only used in slow path or on TX */ MARKER cacheline1 __rte_cache_min_aligned; + RTE_STD_C11 union { void *userdata; /**< Can be used for external metadata */ uint64_t udata64; /**< Allow 8-byte userdata on 32-bit */ @@ -860,10 +552,15 @@ struct rte_mbuf { struct rte_mbuf *next; /**< Next segment of scattered packet. */ /* fields to support TX offloads */ + RTE_STD_C11 union { uint64_t tx_offload; /**< combined for easy fetch */ + __extension__ struct { - uint64_t l2_len:7; /**< L2 (MAC) Header Length. */ + uint64_t l2_len:7; + /**< L2 (MAC) Header Length for non-tunneling pkt. + * Outer_L4_len + ... + Inner_L2_len for tunneling pkt. + */ uint64_t l3_len:9; /**< L3 (IP) Header Length. */ uint64_t l4_len:8; /**< L4 (TCP/UDP) Header Length. */ uint64_t tso_segsz:16; /**< TCP TSO segment size */ @@ -882,8 +579,15 @@ struct rte_mbuf { /** Timesync flags for use with IEEE1588. */ uint16_t timesync; + + /** Sequence number. See also rte_reorder_insert(). */ + uint32_t seqn; + } __rte_cache_aligned; +/**< Maximum number of nb_segs allowed. */ +#define RTE_MBUF_MAX_NB_SEGS UINT16_MAX + /** * Prefetch the first part of the mbuf * @@ -925,21 +629,28 @@ rte_mbuf_prefetch_part2(struct rte_mbuf *m) static inline uint16_t rte_pktmbuf_priv_size(struct rte_mempool *mp); /** - * Return the DMA address of the beginning of the mbuf data + * Return the IO address of the beginning of the mbuf data * * @param mb * The pointer to the mbuf. * @return - * The physical address of the beginning of the mbuf data + * The IO address of the beginning of the mbuf data */ +static inline rte_iova_t +rte_mbuf_data_iova(const struct rte_mbuf *mb) +{ + return mb->buf_iova + mb->data_off; +} + +__rte_deprecated static inline phys_addr_t rte_mbuf_data_dma_addr(const struct rte_mbuf *mb) { - return mb->buf_physaddr + mb->data_off; + return rte_mbuf_data_iova(mb); } /** - * Return the default DMA address of the beginning of the mbuf data + * Return the default IO address of the beginning of the mbuf data * * This function is used by drivers in their receive function, as it * returns the location where data should be written by the NIC, taking @@ -948,12 +659,19 @@ rte_mbuf_data_dma_addr(const struct rte_mbuf *mb) * @param mb * The pointer to the mbuf. * @return - * The physical address of the beginning of the mbuf data + * The IO address of the beginning of the mbuf data */ +static inline rte_iova_t +rte_mbuf_data_iova_default(const struct rte_mbuf *mb) +{ + return mb->buf_iova + RTE_PKTMBUF_HEADROOM; +} + +__rte_deprecated static inline phys_addr_t rte_mbuf_data_dma_addr_default(const struct rte_mbuf *mb) { - return mb->buf_physaddr + RTE_PKTMBUF_HEADROOM; + return rte_mbuf_data_iova_default(mb); } /** @@ -1047,6 +765,13 @@ rte_mbuf_refcnt_set(struct rte_mbuf *m, uint16_t new_value) rte_atomic16_set(&m->refcnt_atomic, new_value); } +/* internal */ +static inline uint16_t +__rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value) +{ + return (uint16_t)(rte_atomic16_add_return(&m->refcnt_atomic, value)); +} + /** * Adds given value to an mbuf's refcnt and returns its new value. * @param m @@ -1071,19 +796,26 @@ rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value) return 1 + value; } - return (uint16_t)(rte_atomic16_add_return(&m->refcnt_atomic, value)); + return __rte_mbuf_refcnt_update(m, value); } #else /* ! RTE_MBUF_REFCNT_ATOMIC */ +/* internal */ +static inline uint16_t +__rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value) +{ + m->refcnt = (uint16_t)(m->refcnt + value); + return m->refcnt; +} + /** * Adds given value to an mbuf's refcnt and returns its new value. */ static inline uint16_t rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value) { - m->refcnt = (uint16_t)(m->refcnt + value); - return m->refcnt; + return __rte_mbuf_refcnt_update(m, value); } /** @@ -1128,14 +860,26 @@ rte_mbuf_refcnt_set(struct rte_mbuf *m, uint16_t new_value) void rte_mbuf_sanity_check(const struct rte_mbuf *m, int is_header); +#define MBUF_RAW_ALLOC_CHECK(m) do { \ + RTE_ASSERT(rte_mbuf_refcnt_read(m) == 1); \ + RTE_ASSERT((m)->next == NULL); \ + RTE_ASSERT((m)->nb_segs == 1); \ + __rte_mbuf_sanity_check(m, 0); \ +} while (0) + /** - * Allocate an unitialized mbuf from mempool *mp*. + * Allocate an uninitialized mbuf from mempool *mp*. * * This function can be used by PMDs (especially in RX functions) to - * allocate an unitialized mbuf. The driver is responsible of + * allocate an uninitialized mbuf. The driver is responsible of * initializing all the required fields. See rte_pktmbuf_reset(). * For standard needs, prefer rte_pktmbuf_alloc(). * + * The caller can expect that the following fields of the mbuf structure + * are initialized: buf_addr, buf_iova, buf_len, refcnt=1, nb_segs=1, + * next=NULL, pool, priv_size. The other fields must be initialized + * by the caller. + * * @param mp * The mempool from which mbuf is allocated. * @return @@ -1150,33 +894,41 @@ static inline struct rte_mbuf *rte_mbuf_raw_alloc(struct rte_mempool *mp) if (rte_mempool_get(mp, &mb) < 0) return NULL; m = (struct rte_mbuf *)mb; - RTE_ASSERT(rte_mbuf_refcnt_read(m) == 0); - rte_mbuf_refcnt_set(m, 1); - __rte_mbuf_sanity_check(m, 0); - + MBUF_RAW_ALLOC_CHECK(m); return m; } -/* compat with older versions */ -__rte_deprecated static inline struct rte_mbuf * -__rte_mbuf_raw_alloc(struct rte_mempool *mp) -{ - return rte_mbuf_raw_alloc(mp); -} - /** - * @internal Put mbuf back into its original mempool. - * The use of that function is reserved for RTE internal needs. - * Please use rte_pktmbuf_free(). + * Put mbuf back into its original mempool. + * + * The caller must ensure that the mbuf is direct and properly + * reinitialized (refcnt=1, next=NULL, nb_segs=1), as done by + * rte_pktmbuf_prefree_seg(). + * + * This function should be used with care, when optimization is + * required. For standard needs, prefer rte_pktmbuf_free() or + * rte_pktmbuf_free_seg(). * * @param m * The mbuf to be freed. */ -static inline void __attribute__((always_inline)) +static __rte_always_inline void +rte_mbuf_raw_free(struct rte_mbuf *m) +{ + RTE_ASSERT(RTE_MBUF_DIRECT(m)); + RTE_ASSERT(rte_mbuf_refcnt_read(m) == 1); + RTE_ASSERT(m->next == NULL); + RTE_ASSERT(m->nb_segs == 1); + __rte_mbuf_sanity_check(m, 0); + rte_mempool_put(m->pool, m); +} + +/* compat with older versions */ +__rte_deprecated +static inline void __rte_mbuf_raw_free(struct rte_mbuf *m) { - RTE_ASSERT(rte_mbuf_refcnt_read(m) == 0); - rte_mempool_put(m->pool, m); + rte_mbuf_raw_free(m); } /* Operations on ctrl mbuf */ @@ -1187,14 +939,14 @@ __rte_mbuf_raw_free(struct rte_mbuf *m) * This function initializes some fields in an mbuf structure that are * not modified by the user once created (mbuf type, origin pool, buffer * start address, and so on). This function is given as a callback function - * to rte_mempool_create() at pool creation time. + * to rte_mempool_obj_iter() or rte_mempool_create() at pool creation time. * * @param mp * The mempool from which the mbuf is allocated. * @param opaque_arg * A pointer that can be used by the user to retrieve useful information - * for mbuf initialization. This pointer comes from the ``init_arg`` - * parameter of rte_mempool_create(). + * for mbuf initialization. This pointer is the opaque argument passed to + * rte_mempool_obj_iter() or rte_mempool_create(). * @param m * The mbuf to initialize. * @param i @@ -1268,14 +1020,14 @@ rte_is_ctrlmbuf(struct rte_mbuf *m) * This function initializes some fields in the mbuf structure that are * not modified by the user once created (origin pool, buffer start * address, and so on). This function is given as a callback function to - * rte_mempool_create() at pool creation time. + * rte_mempool_obj_iter() or rte_mempool_create() at pool creation time. * * @param mp * The mempool from which mbufs originate. * @param opaque_arg * A pointer that can be used by the user to retrieve useful information - * for mbuf initialization. This pointer comes from the ``init_arg`` - * parameter of rte_mempool_create(). + * for mbuf initialization. This pointer is the opaque argument passed to + * rte_mempool_obj_iter() or rte_mempool_create(). * @param m * The mbuf to initialize. * @param i @@ -1290,7 +1042,8 @@ void rte_pktmbuf_init(struct rte_mempool *mp, void *opaque_arg, * * This function initializes the mempool private data in the case of a * pktmbuf pool. This private data is needed by the driver. The - * function is given as a callback function to rte_mempool_create() at + * function must be called on the mempool before it is used, or it + * can be given as a callback function to rte_mempool_create() at * pool creation. It can be extended by the user, for example, to * provide another packet size. * @@ -1298,8 +1051,8 @@ void rte_pktmbuf_init(struct rte_mempool *mp, void *opaque_arg, * The mempool from which mbufs originate. * @param opaque_arg * A pointer that can be used by the user to retrieve useful information - * for mbuf initialization. This pointer comes from the ``init_arg`` - * parameter of rte_mempool_create(). + * for mbuf initialization. This pointer is the opaque argument passed to + * rte_mempool_create(). */ void rte_pktmbuf_pool_init(struct rte_mempool *mp, void *opaque_arg); @@ -1307,8 +1060,7 @@ void rte_pktmbuf_pool_init(struct rte_mempool *mp, void *opaque_arg); * Create a mbuf pool. * * This function creates and initializes a packet mbuf pool. It is - * a wrapper to rte_mempool_create() with the proper packet constructor - * and mempool constructor. + * a wrapper to rte_mempool functions. * * @param name * The name of the mbuf pool. @@ -1384,6 +1136,19 @@ rte_pktmbuf_priv_size(struct rte_mempool *mp) return mbp_priv->mbuf_priv_size; } +/** + * Reset the data_off field of a packet mbuf to its default value. + * + * The given mbuf must have only one segment, which should be empty. + * + * @param m + * The packet mbuf's data_off field has to be reset. + */ +static inline void rte_pktmbuf_reset_headroom(struct rte_mbuf *m) +{ + m->data_off = RTE_MIN(RTE_PKTMBUF_HEADROOM, (uint16_t)m->buf_len); +} + /** * Reset the fields of a packet mbuf to their default values. * @@ -1392,6 +1157,8 @@ rte_pktmbuf_priv_size(struct rte_mempool *mp) * @param m * The packet mbuf to be resetted. */ +#define MBUF_INVALID_PORT UINT16_MAX + static inline void rte_pktmbuf_reset(struct rte_mbuf *m) { m->next = NULL; @@ -1400,12 +1167,11 @@ static inline void rte_pktmbuf_reset(struct rte_mbuf *m) m->vlan_tci = 0; m->vlan_tci_outer = 0; m->nb_segs = 1; - m->port = 0xff; + m->port = MBUF_INVALID_PORT; m->ol_flags = 0; m->packet_type = 0; - m->data_off = (RTE_PKTMBUF_HEADROOM <= m->buf_len) ? - RTE_PKTMBUF_HEADROOM : m->buf_len; + rte_pktmbuf_reset_headroom(m); m->data_len = 0; __rte_mbuf_sanity_check(m, 1); @@ -1444,6 +1210,7 @@ static inline struct rte_mbuf *rte_pktmbuf_alloc(struct rte_mempool *mp) * Array size * @return * - 0: Success + * - -ENOENT: Not enough entries in the mempool; no mbufs are retrieved. */ static inline int rte_pktmbuf_alloc_bulk(struct rte_mempool *pool, struct rte_mbuf **mbufs, unsigned count) @@ -1463,25 +1230,25 @@ static inline int rte_pktmbuf_alloc_bulk(struct rte_mempool *pool, switch (count % 4) { case 0: while (idx != count) { - RTE_ASSERT(rte_mbuf_refcnt_read(mbufs[idx]) == 0); - rte_mbuf_refcnt_set(mbufs[idx], 1); + MBUF_RAW_ALLOC_CHECK(mbufs[idx]); rte_pktmbuf_reset(mbufs[idx]); idx++; + /* fall-through */ case 3: - RTE_ASSERT(rte_mbuf_refcnt_read(mbufs[idx]) == 0); - rte_mbuf_refcnt_set(mbufs[idx], 1); + MBUF_RAW_ALLOC_CHECK(mbufs[idx]); rte_pktmbuf_reset(mbufs[idx]); idx++; + /* fall-through */ case 2: - RTE_ASSERT(rte_mbuf_refcnt_read(mbufs[idx]) == 0); - rte_mbuf_refcnt_set(mbufs[idx], 1); + MBUF_RAW_ALLOC_CHECK(mbufs[idx]); rte_pktmbuf_reset(mbufs[idx]); idx++; + /* fall-through */ case 1: - RTE_ASSERT(rte_mbuf_refcnt_read(mbufs[idx]) == 0); - rte_mbuf_refcnt_set(mbufs[idx], 1); + MBUF_RAW_ALLOC_CHECK(mbufs[idx]); rte_pktmbuf_reset(mbufs[idx]); idx++; + /* fall-through */ } } return 0; @@ -1519,11 +1286,10 @@ static inline void rte_pktmbuf_attach(struct rte_mbuf *mi, struct rte_mbuf *m) rte_mbuf_refcnt_update(md, 1); mi->priv_size = m->priv_size; - mi->buf_physaddr = m->buf_physaddr; + mi->buf_iova = m->buf_iova; mi->buf_addr = m->buf_addr; mi->buf_len = m->buf_len; - mi->next = m->next; mi->data_off = m->data_off; mi->data_len = m->data_len; mi->port = m->port; @@ -1537,6 +1303,7 @@ static inline void rte_pktmbuf_attach(struct rte_mbuf *mi, struct rte_mbuf *m) mi->nb_segs = 1; mi->ol_flags = m->ol_flags | IND_ATTACHED_MBUF; mi->packet_type = m->packet_type; + mi->timestamp = m->timestamp; __rte_mbuf_sanity_check(mi, 1); __rte_mbuf_sanity_check(m, 0); @@ -1567,30 +1334,75 @@ static inline void rte_pktmbuf_detach(struct rte_mbuf *m) m->priv_size = priv_size; m->buf_addr = (char *)m + mbuf_size; - m->buf_physaddr = rte_mempool_virt2phy(mp, m) + mbuf_size; + m->buf_iova = rte_mempool_virt2iova(m) + mbuf_size; m->buf_len = (uint16_t)buf_len; - m->data_off = RTE_MIN(RTE_PKTMBUF_HEADROOM, (uint16_t)m->buf_len); + rte_pktmbuf_reset_headroom(m); m->data_len = 0; m->ol_flags = 0; - if (rte_mbuf_refcnt_update(md, -1) == 0) - __rte_mbuf_raw_free(md); + if (rte_mbuf_refcnt_update(md, -1) == 0) { + md->next = NULL; + md->nb_segs = 1; + rte_mbuf_refcnt_set(md, 1); + rte_mbuf_raw_free(md); + } } -static inline struct rte_mbuf* __attribute__((always_inline)) -__rte_pktmbuf_prefree_seg(struct rte_mbuf *m) +/** + * Decrease reference counter and unlink a mbuf segment + * + * This function does the same than a free, except that it does not + * return the segment to its pool. + * It decreases the reference counter, and if it reaches 0, it is + * detached from its parent for an indirect mbuf. + * + * @param m + * The mbuf to be unlinked + * @return + * - (m) if it is the last reference. It can be recycled or freed. + * - (NULL) if the mbuf still has remaining references on it. + */ +static __rte_always_inline struct rte_mbuf * +rte_pktmbuf_prefree_seg(struct rte_mbuf *m) { __rte_mbuf_sanity_check(m, 0); - if (likely(rte_mbuf_refcnt_update(m, -1) == 0)) { - /* if this is an indirect mbuf, it is detached. */ + if (likely(rte_mbuf_refcnt_read(m) == 1)) { + if (RTE_MBUF_INDIRECT(m)) rte_pktmbuf_detach(m); + + if (m->next != NULL) { + m->next = NULL; + m->nb_segs = 1; + } + + return m; + + } else if (__rte_mbuf_refcnt_update(m, -1) == 0) { + + if (RTE_MBUF_INDIRECT(m)) + rte_pktmbuf_detach(m); + + if (m->next != NULL) { + m->next = NULL; + m->nb_segs = 1; + } + rte_mbuf_refcnt_set(m, 1); + return m; } return NULL; } +/* deprecated, replaced by rte_pktmbuf_prefree_seg() */ +__rte_deprecated +static inline struct rte_mbuf * +__rte_pktmbuf_prefree_seg(struct rte_mbuf *m) +{ + return rte_pktmbuf_prefree_seg(m); +} + /** * Free a segment of a packet mbuf into its original mempool. * @@ -1600,13 +1412,12 @@ __rte_pktmbuf_prefree_seg(struct rte_mbuf *m) * @param m * The packet mbuf segment to be freed. */ -static inline void __attribute__((always_inline)) +static __rte_always_inline void rte_pktmbuf_free_seg(struct rte_mbuf *m) { - if (likely(NULL != (m = __rte_pktmbuf_prefree_seg(m)))) { - m->next = NULL; - __rte_mbuf_raw_free(m); - } + m = rte_pktmbuf_prefree_seg(m); + if (likely(m != NULL)) + rte_mbuf_raw_free(m); } /** @@ -1616,13 +1427,14 @@ rte_pktmbuf_free_seg(struct rte_mbuf *m) * segment is added back into its original mempool. * * @param m - * The packet mbuf to be freed. + * The packet mbuf to be freed. If NULL, the function does nothing. */ static inline void rte_pktmbuf_free(struct rte_mbuf *m) { struct rte_mbuf *m_next; - __rte_mbuf_sanity_check(m, 1); + if (m != NULL) + __rte_mbuf_sanity_check(m, 1); while (m != NULL) { m_next = m->next; @@ -1653,7 +1465,7 @@ static inline struct rte_mbuf *rte_pktmbuf_clone(struct rte_mbuf *md, { struct rte_mbuf *mc, *mi, **prev; uint32_t pktlen; - uint8_t nseg; + uint16_t nseg; if (unlikely ((mc = rte_pktmbuf_alloc(mp)) == NULL)) return NULL; @@ -1715,7 +1527,7 @@ static inline void rte_pktmbuf_refcnt_update(struct rte_mbuf *m, int16_t v) */ static inline uint16_t rte_pktmbuf_headroom(const struct rte_mbuf *m) { - __rte_mbuf_sanity_check(m, 1); + __rte_mbuf_sanity_check(m, 0); return m->data_off; } @@ -1729,7 +1541,7 @@ static inline uint16_t rte_pktmbuf_headroom(const struct rte_mbuf *m) */ static inline uint16_t rte_pktmbuf_tailroom(const struct rte_mbuf *m) { - __rte_mbuf_sanity_check(m, 1); + __rte_mbuf_sanity_check(m, 0); return (uint16_t)(m->buf_len - rte_pktmbuf_headroom(m) - m->data_len); } @@ -1784,7 +1596,7 @@ static inline struct rte_mbuf *rte_pktmbuf_lastseg(struct rte_mbuf *m) #define rte_pktmbuf_mtod(m, t) rte_pktmbuf_mtod_offset(m, t, 0) /** - * A macro that returns the physical address that points to an offset of the + * A macro that returns the IO address that points to an offset of the * start of the data in the mbuf * * @param m @@ -1792,17 +1604,24 @@ static inline struct rte_mbuf *rte_pktmbuf_lastseg(struct rte_mbuf *m) * @param o * The offset into the data to calculate address from. */ +#define rte_pktmbuf_iova_offset(m, o) \ + (rte_iova_t)((m)->buf_iova + (m)->data_off + (o)) + +/* deprecated */ #define rte_pktmbuf_mtophys_offset(m, o) \ - (phys_addr_t)((m)->buf_physaddr + (m)->data_off + (o)) + rte_pktmbuf_iova_offset(m, o) /** - * A macro that returns the physical address that points to the start of the + * A macro that returns the IO address that points to the start of the * data in the mbuf * * @param m * The packet mbuf. */ -#define rte_pktmbuf_mtophys(m) rte_pktmbuf_mtophys_offset(m, 0) +#define rte_pktmbuf_iova(m) rte_pktmbuf_iova_offset(m, 0) + +/* deprecated */ +#define rte_pktmbuf_mtophys(m) rte_pktmbuf_iova(m) /** * A macro that returns the length of the packet. @@ -1957,6 +1776,41 @@ static inline int rte_pktmbuf_is_contiguous(const struct rte_mbuf *m) return !!(m->nb_segs == 1); } +/** + * @internal used by rte_pktmbuf_read(). + */ +const void *__rte_pktmbuf_read(const struct rte_mbuf *m, uint32_t off, + uint32_t len, void *buf); + +/** + * Read len data bytes in a mbuf at specified offset. + * + * If the data is contiguous, return the pointer in the mbuf data, else + * copy the data in the buffer provided by the user and return its + * pointer. + * + * @param m + * The pointer to the mbuf. + * @param off + * The offset of the data in the mbuf. + * @param len + * The amount of bytes to read. + * @param buf + * The buffer where data is copied if it is not contiguous in mbuf + * data. Its length should be at least equal to the len parameter. + * @return + * The pointer to the data, either in the mbuf if it is contiguous, + * or in the user buffer. If mbuf is too small, NULL is returned. + */ +static inline const void *rte_pktmbuf_read(const struct rte_mbuf *m, + uint32_t off, uint32_t len, void *buf) +{ + if (likely(off + len <= rte_pktmbuf_data_len(m))) + return rte_pktmbuf_mtod_offset(m, char *, off); + else + return __rte_pktmbuf_read(m, off, len, buf); +} + /** * Chain an mbuf to another, thereby creating a segmented packet. * @@ -1971,14 +1825,14 @@ static inline int rte_pktmbuf_is_contiguous(const struct rte_mbuf *m) * * @return * - 0, on success. - * - -EOVERFLOW, if the chain is full (256 entries) + * - -EOVERFLOW, if the chain segment limit exceeded */ static inline int rte_pktmbuf_chain(struct rte_mbuf *head, struct rte_mbuf *tail) { struct rte_mbuf *cur_tail; /* Check for number-of-segments-overflow */ - if (head->nb_segs + tail->nb_segs >= 1 << (sizeof(head->nb_segs) * 8)) + if (head->nb_segs + tail->nb_segs > RTE_MBUF_MAX_NB_SEGS) return -EOVERFLOW; /* Chain 'tail' onto the old tail */ @@ -1986,7 +1840,7 @@ static inline int rte_pktmbuf_chain(struct rte_mbuf *head, struct rte_mbuf *tail cur_tail->next = tail; /* accumulate number of segments and total length. */ - head->nb_segs = (uint8_t)(head->nb_segs + tail->nb_segs); + head->nb_segs += tail->nb_segs; head->pkt_len += tail->pkt_len; /* pkt_len is only set in the head */ @@ -1996,7 +1850,109 @@ static inline int rte_pktmbuf_chain(struct rte_mbuf *head, struct rte_mbuf *tail } /** - * Dump an mbuf structure to the console. + * Validate general requirements for Tx offload in mbuf. + * + * This function checks correctness and completeness of Tx offload settings. + * + * @param m + * The packet mbuf to be validated. + * @return + * 0 if packet is valid + */ +static inline int +rte_validate_tx_offload(const struct rte_mbuf *m) +{ + uint64_t ol_flags = m->ol_flags; + uint64_t inner_l3_offset = m->l2_len; + + /* Does packet set any of available offloads? */ + if (!(ol_flags & PKT_TX_OFFLOAD_MASK)) + return 0; + + if (ol_flags & PKT_TX_OUTER_IP_CKSUM) + inner_l3_offset += m->outer_l2_len + m->outer_l3_len; + + /* Headers are fragmented */ + if (rte_pktmbuf_data_len(m) < inner_l3_offset + m->l3_len + m->l4_len) + return -ENOTSUP; + + /* IP checksum can be counted only for IPv4 packet */ + if ((ol_flags & PKT_TX_IP_CKSUM) && (ol_flags & PKT_TX_IPV6)) + return -EINVAL; + + /* IP type not set when required */ + if (ol_flags & (PKT_TX_L4_MASK | PKT_TX_TCP_SEG)) + if (!(ol_flags & (PKT_TX_IPV4 | PKT_TX_IPV6))) + return -EINVAL; + + /* Check requirements for TSO packet */ + if (ol_flags & PKT_TX_TCP_SEG) + if ((m->tso_segsz == 0) || + ((ol_flags & PKT_TX_IPV4) && + !(ol_flags & PKT_TX_IP_CKSUM))) + return -EINVAL; + + /* PKT_TX_OUTER_IP_CKSUM set for non outer IPv4 packet. */ + if ((ol_flags & PKT_TX_OUTER_IP_CKSUM) && + !(ol_flags & PKT_TX_OUTER_IPV4)) + return -EINVAL; + + return 0; +} + +/** + * Linearize data in mbuf. + * + * This function moves the mbuf data in the first segment if there is enough + * tailroom. The subsequent segments are unchained and freed. + * + * @param mbuf + * mbuf to linearize + * @return + * - 0, on success + * - -1, on error + */ +static inline int +rte_pktmbuf_linearize(struct rte_mbuf *mbuf) +{ + int seg_len, copy_len; + struct rte_mbuf *m; + struct rte_mbuf *m_next; + char *buffer; + + if (rte_pktmbuf_is_contiguous(mbuf)) + return 0; + + /* Extend first segment to the total packet length */ + copy_len = rte_pktmbuf_pkt_len(mbuf) - rte_pktmbuf_data_len(mbuf); + + if (unlikely(copy_len > rte_pktmbuf_tailroom(mbuf))) + return -1; + + buffer = rte_pktmbuf_mtod_offset(mbuf, char *, mbuf->data_len); + mbuf->data_len = (uint16_t)(mbuf->pkt_len); + + /* Append data from next segments to the first one */ + m = mbuf->next; + while (m != NULL) { + m_next = m->next; + + seg_len = rte_pktmbuf_data_len(m); + rte_memcpy(buffer, rte_pktmbuf_mtod(m, char *), seg_len); + buffer += seg_len; + + rte_pktmbuf_free_seg(m); + m = m_next; + } + + mbuf->next = NULL; + mbuf->nb_segs = 1; + + return 0; +} + +/** + * Dump an mbuf structure to a file. * * Dump all fields for the given packet mbuf and all its associated * segments (in the case of a chained buffer). diff --git a/dpdk/lib/librte_mbuf/rte_mbuf_ptype.c b/dpdk/lib/librte_mbuf/rte_mbuf_ptype.c new file mode 100644 index 00000000..a623226c --- /dev/null +++ b/dpdk/lib/librte_mbuf/rte_mbuf_ptype.c @@ -0,0 +1,230 @@ +/*- + * BSD LICENSE + * + * Copyright 2016 6WIND S.A. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of 6WIND S.A. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +#include +#include + +/* get the name of the l2 packet type */ +const char *rte_get_ptype_l2_name(uint32_t ptype) +{ + switch (ptype & RTE_PTYPE_L2_MASK) { + case RTE_PTYPE_L2_ETHER: return "L2_ETHER"; + case RTE_PTYPE_L2_ETHER_TIMESYNC: return "L2_ETHER_TIMESYNC"; + case RTE_PTYPE_L2_ETHER_ARP: return "L2_ETHER_ARP"; + case RTE_PTYPE_L2_ETHER_LLDP: return "L2_ETHER_LLDP"; + case RTE_PTYPE_L2_ETHER_NSH: return "L2_ETHER_NSH"; + case RTE_PTYPE_L2_ETHER_VLAN: return "L2_ETHER_VLAN"; + case RTE_PTYPE_L2_ETHER_QINQ: return "L2_ETHER_QINQ"; + default: return "L2_UNKNOWN"; + } +} + +/* get the name of the l3 packet type */ +const char *rte_get_ptype_l3_name(uint32_t ptype) +{ + switch (ptype & RTE_PTYPE_L3_MASK) { + case RTE_PTYPE_L3_IPV4: return "L3_IPV4"; + case RTE_PTYPE_L3_IPV4_EXT: return "L3_IPV4_EXT"; + case RTE_PTYPE_L3_IPV6: return "L3_IPV6"; + case RTE_PTYPE_L3_IPV4_EXT_UNKNOWN: return "L3_IPV4_EXT_UNKNOWN"; + case RTE_PTYPE_L3_IPV6_EXT: return "L3_IPV6_EXT"; + case RTE_PTYPE_L3_IPV6_EXT_UNKNOWN: return "L3_IPV6_EXT_UNKNOWN"; + default: return "L3_UNKNOWN"; + } +} + +/* get the name of the l4 packet type */ +const char *rte_get_ptype_l4_name(uint32_t ptype) +{ + switch (ptype & RTE_PTYPE_L4_MASK) { + case RTE_PTYPE_L4_TCP: return "L4_TCP"; + case RTE_PTYPE_L4_UDP: return "L4_UDP"; + case RTE_PTYPE_L4_FRAG: return "L4_FRAG"; + case RTE_PTYPE_L4_SCTP: return "L4_SCTP"; + case RTE_PTYPE_L4_ICMP: return "L4_ICMP"; + case RTE_PTYPE_L4_NONFRAG: return "L4_NONFRAG"; + default: return "L4_UNKNOWN"; + } +} + +/* get the name of the tunnel packet type */ +const char *rte_get_ptype_tunnel_name(uint32_t ptype) +{ + switch (ptype & RTE_PTYPE_TUNNEL_MASK) { + case RTE_PTYPE_TUNNEL_IP: return "TUNNEL_IP"; + case RTE_PTYPE_TUNNEL_GRE: return "TUNNEL_GRE"; + case RTE_PTYPE_TUNNEL_VXLAN: return "TUNNEL_VXLAN"; + case RTE_PTYPE_TUNNEL_NVGRE: return "TUNNEL_NVGRE"; + case RTE_PTYPE_TUNNEL_GENEVE: return "TUNNEL_GENEVE"; + case RTE_PTYPE_TUNNEL_GRENAT: return "TUNNEL_GRENAT"; + case RTE_PTYPE_TUNNEL_GTPC: return "TUNNEL_GTPC"; + case RTE_PTYPE_TUNNEL_GTPU: return "TUNNEL_GTPU"; + case RTE_PTYPE_TUNNEL_ESP: return "TUNNEL_ESP"; + default: return "TUNNEL_UNKNOWN"; + } +} + +/* get the name of the inner_l2 packet type */ +const char *rte_get_ptype_inner_l2_name(uint32_t ptype) +{ + switch (ptype & RTE_PTYPE_INNER_L2_MASK) { + case RTE_PTYPE_INNER_L2_ETHER: return "INNER_L2_ETHER"; + case RTE_PTYPE_INNER_L2_ETHER_VLAN: return "INNER_L2_ETHER_VLAN"; + case RTE_PTYPE_INNER_L2_ETHER_QINQ: return "INNER_L2_ETHER_QINQ"; + default: return "INNER_L2_UNKNOWN"; + } +} + +/* get the name of the inner_l3 packet type */ +const char *rte_get_ptype_inner_l3_name(uint32_t ptype) +{ + switch (ptype & RTE_PTYPE_INNER_L3_MASK) { + case RTE_PTYPE_INNER_L3_IPV4: return "INNER_L3_IPV4"; + case RTE_PTYPE_INNER_L3_IPV4_EXT: return "INNER_L3_IPV4_EXT"; + case RTE_PTYPE_INNER_L3_IPV6: return "INNER_L3_IPV6"; + case RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN: + return "INNER_L3_IPV4_EXT_UNKNOWN"; + case RTE_PTYPE_INNER_L3_IPV6_EXT: return "INNER_L3_IPV6_EXT"; + case RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN: + return "INNER_L3_IPV6_EXT_UNKNOWN"; + default: return "INNER_L3_UNKNOWN"; + } +} + +/* get the name of the inner_l4 packet type */ +const char *rte_get_ptype_inner_l4_name(uint32_t ptype) +{ + switch (ptype & RTE_PTYPE_INNER_L4_MASK) { + case RTE_PTYPE_INNER_L4_TCP: return "INNER_L4_TCP"; + case RTE_PTYPE_INNER_L4_UDP: return "INNER_L4_UDP"; + case RTE_PTYPE_INNER_L4_FRAG: return "INNER_L4_FRAG"; + case RTE_PTYPE_INNER_L4_SCTP: return "INNER_L4_SCTP"; + case RTE_PTYPE_INNER_L4_ICMP: return "INNER_L4_ICMP"; + case RTE_PTYPE_INNER_L4_NONFRAG: return "INNER_L4_NONFRAG"; + default: return "INNER_L4_UNKNOWN"; + } +} + +/* write the packet type name into the buffer */ +int rte_get_ptype_name(uint32_t ptype, char *buf, size_t buflen) +{ + int ret; + + if (buflen == 0) + return -1; + + buf[0] = '\0'; + if ((ptype & RTE_PTYPE_ALL_MASK) == RTE_PTYPE_UNKNOWN) { + ret = snprintf(buf, buflen, "UNKNOWN"); + if (ret < 0) + return -1; + if ((size_t)ret >= buflen) + return -1; + return 0; + } + + if ((ptype & RTE_PTYPE_L2_MASK) != 0) { + ret = snprintf(buf, buflen, "%s ", + rte_get_ptype_l2_name(ptype)); + if (ret < 0) + return -1; + if ((size_t)ret >= buflen) + return -1; + buf += ret; + buflen -= ret; + } + if ((ptype & RTE_PTYPE_L3_MASK) != 0) { + ret = snprintf(buf, buflen, "%s ", + rte_get_ptype_l3_name(ptype)); + if (ret < 0) + return -1; + if ((size_t)ret >= buflen) + return -1; + buf += ret; + buflen -= ret; + } + if ((ptype & RTE_PTYPE_L4_MASK) != 0) { + ret = snprintf(buf, buflen, "%s ", + rte_get_ptype_l4_name(ptype)); + if (ret < 0) + return -1; + if ((size_t)ret >= buflen) + return -1; + buf += ret; + buflen -= ret; + } + if ((ptype & RTE_PTYPE_TUNNEL_MASK) != 0) { + ret = snprintf(buf, buflen, "%s ", + rte_get_ptype_tunnel_name(ptype)); + if (ret < 0) + return -1; + if ((size_t)ret >= buflen) + return -1; + buf += ret; + buflen -= ret; + } + if ((ptype & RTE_PTYPE_INNER_L2_MASK) != 0) { + ret = snprintf(buf, buflen, "%s ", + rte_get_ptype_inner_l2_name(ptype)); + if (ret < 0) + return -1; + if ((size_t)ret >= buflen) + return -1; + buf += ret; + buflen -= ret; + } + if ((ptype & RTE_PTYPE_INNER_L3_MASK) != 0) { + ret = snprintf(buf, buflen, "%s ", + rte_get_ptype_inner_l3_name(ptype)); + if (ret < 0) + return -1; + if ((size_t)ret >= buflen) + return -1; + buf += ret; + buflen -= ret; + } + if ((ptype & RTE_PTYPE_INNER_L4_MASK) != 0) { + ret = snprintf(buf, buflen, "%s ", + rte_get_ptype_inner_l4_name(ptype)); + if (ret < 0) + return -1; + if ((size_t)ret >= buflen) + return -1; + buf += ret; + buflen -= ret; + } + + return 0; +} diff --git a/dpdk/lib/librte_mbuf/rte_mbuf_ptype.h b/dpdk/lib/librte_mbuf/rte_mbuf_ptype.h new file mode 100644 index 00000000..5c62435c --- /dev/null +++ b/dpdk/lib/librte_mbuf/rte_mbuf_ptype.h @@ -0,0 +1,714 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2016 Intel Corporation. + * Copyright 2014-2016 6WIND S.A. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_MBUF_PTYPE_H_ +#define _RTE_MBUF_PTYPE_H_ + +/** + * @file + * RTE Mbuf Packet Types + * + * This file contains declarations for features related to mbuf packet + * types. The packet type gives information about the data carried by the + * mbuf, and is stored in the mbuf in a 32 bits field. + * + * The 32 bits are divided into several fields to mark packet types. Note that + * each field is indexical. + * - Bit 3:0 is for L2 types. + * - Bit 7:4 is for L3 or outer L3 (for tunneling case) types. + * - Bit 11:8 is for L4 or outer L4 (for tunneling case) types. + * - Bit 15:12 is for tunnel types. + * - Bit 19:16 is for inner L2 types. + * - Bit 23:20 is for inner L3 types. + * - Bit 27:24 is for inner L4 types. + * - Bit 31:28 is reserved. + * + * To be compatible with Vector PMD, RTE_PTYPE_L3_IPV4, RTE_PTYPE_L3_IPV4_EXT, + * RTE_PTYPE_L3_IPV6, RTE_PTYPE_L3_IPV6_EXT, RTE_PTYPE_L4_TCP, RTE_PTYPE_L4_UDP + * and RTE_PTYPE_L4_SCTP should be kept as below in a contiguous 7 bits. + * + * Note that L3 types values are selected for checking IPV4/IPV6 header from + * performance point of view. Reading annotations of RTE_ETH_IS_IPV4_HDR and + * RTE_ETH_IS_IPV6_HDR is needed for any future changes of L3 type values. + * + * Note that the packet types of the same packet recognized by different + * hardware may be different, as different hardware may have different + * capability of packet type recognition. + * + * examples: + * <'ether type'=0x0800 + * | 'version'=4, 'protocol'=0x29 + * | 'version'=6, 'next header'=0x3A + * | 'ICMPv6 header'> + * will be recognized on i40e hardware as packet type combination of, + * RTE_PTYPE_L2_ETHER | + * RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + * RTE_PTYPE_TUNNEL_IP | + * RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + * RTE_PTYPE_INNER_L4_ICMP. + * + * <'ether type'=0x86DD + * | 'version'=6, 'next header'=0x2F + * | 'GRE header' + * | 'version'=6, 'next header'=0x11 + * | 'UDP header'> + * will be recognized on i40e hardware as packet type combination of, + * RTE_PTYPE_L2_ETHER | + * RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + * RTE_PTYPE_TUNNEL_GRENAT | + * RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + * RTE_PTYPE_INNER_L4_UDP. + */ + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * No packet type information. + */ +#define RTE_PTYPE_UNKNOWN 0x00000000 +/** + * Ethernet packet type. + * It is used for outer packet for tunneling cases. + * + * Packet format: + * <'ether type'=[0x0800|0x86DD]> + */ +#define RTE_PTYPE_L2_ETHER 0x00000001 +/** + * Ethernet packet type for time sync. + * + * Packet format: + * <'ether type'=0x88F7> + */ +#define RTE_PTYPE_L2_ETHER_TIMESYNC 0x00000002 +/** + * ARP (Address Resolution Protocol) packet type. + * + * Packet format: + * <'ether type'=0x0806> + */ +#define RTE_PTYPE_L2_ETHER_ARP 0x00000003 +/** + * LLDP (Link Layer Discovery Protocol) packet type. + * + * Packet format: + * <'ether type'=0x88CC> + */ +#define RTE_PTYPE_L2_ETHER_LLDP 0x00000004 +/** + * NSH (Network Service Header) packet type. + * + * Packet format: + * <'ether type'=0x894F> + */ +#define RTE_PTYPE_L2_ETHER_NSH 0x00000005 +/** + * VLAN packet type. + * + * Packet format: + * <'ether type'=[0x8100]> + */ +#define RTE_PTYPE_L2_ETHER_VLAN 0x00000006 +/** + * QinQ packet type. + * + * Packet format: + * <'ether type'=[0x88A8]> + */ +#define RTE_PTYPE_L2_ETHER_QINQ 0x00000007 +/** + * Mask of layer 2 packet types. + * It is used for outer packet for tunneling cases. + */ +#define RTE_PTYPE_L2_MASK 0x0000000f +/** + * IP (Internet Protocol) version 4 packet type. + * It is used for outer packet for tunneling cases, and does not contain any + * header option. + * + * Packet format: + * <'ether type'=0x0800 + * | 'version'=4, 'ihl'=5> + */ +#define RTE_PTYPE_L3_IPV4 0x00000010 +/** + * IP (Internet Protocol) version 4 packet type. + * It is used for outer packet for tunneling cases, and contains header + * options. + * + * Packet format: + * <'ether type'=0x0800 + * | 'version'=4, 'ihl'=[6-15], 'options'> + */ +#define RTE_PTYPE_L3_IPV4_EXT 0x00000030 +/** + * IP (Internet Protocol) version 6 packet type. + * It is used for outer packet for tunneling cases, and does not contain any + * extension header. + * + * Packet format: + * <'ether type'=0x86DD + * | 'version'=6, 'next header'=0x3B> + */ +#define RTE_PTYPE_L3_IPV6 0x00000040 +/** + * IP (Internet Protocol) version 4 packet type. + * It is used for outer packet for tunneling cases, and may or maynot contain + * header options. + * + * Packet format: + * <'ether type'=0x0800 + * | 'version'=4, 'ihl'=[5-15], <'options'>> + */ +#define RTE_PTYPE_L3_IPV4_EXT_UNKNOWN 0x00000090 +/** + * IP (Internet Protocol) version 6 packet type. + * It is used for outer packet for tunneling cases, and contains extension + * headers. + * + * Packet format: + * <'ether type'=0x86DD + * | 'version'=6, 'next header'=[0x0|0x2B|0x2C|0x32|0x33|0x3C|0x87], + * 'extension headers'> + */ +#define RTE_PTYPE_L3_IPV6_EXT 0x000000c0 +/** + * IP (Internet Protocol) version 6 packet type. + * It is used for outer packet for tunneling cases, and may or maynot contain + * extension headers. + * + * Packet format: + * <'ether type'=0x86DD + * | 'version'=6, 'next header'=[0x3B|0x0|0x2B|0x2C|0x32|0x33|0x3C|0x87], + * <'extension headers'>> + */ +#define RTE_PTYPE_L3_IPV6_EXT_UNKNOWN 0x000000e0 +/** + * Mask of layer 3 packet types. + * It is used for outer packet for tunneling cases. + */ +#define RTE_PTYPE_L3_MASK 0x000000f0 +/** + * TCP (Transmission Control Protocol) packet type. + * It is used for outer packet for tunneling cases. + * + * Packet format: + * <'ether type'=0x0800 + * | 'version'=4, 'protocol'=6, 'MF'=0, 'frag_offset'=0> + * or, + * <'ether type'=0x86DD + * | 'version'=6, 'next header'=6> + */ +#define RTE_PTYPE_L4_TCP 0x00000100 +/** + * UDP (User Datagram Protocol) packet type. + * It is used for outer packet for tunneling cases. + * + * Packet format: + * <'ether type'=0x0800 + * | 'version'=4, 'protocol'=17, 'MF'=0, 'frag_offset'=0> + * or, + * <'ether type'=0x86DD + * | 'version'=6, 'next header'=17> + */ +#define RTE_PTYPE_L4_UDP 0x00000200 +/** + * Fragmented IP (Internet Protocol) packet type. + * It is used for outer packet for tunneling cases. + * + * It refers to those packets of any IP types, which can be recognized as + * fragmented. A fragmented packet cannot be recognized as any other L4 types + * (RTE_PTYPE_L4_TCP, RTE_PTYPE_L4_UDP, RTE_PTYPE_L4_SCTP, RTE_PTYPE_L4_ICMP, + * RTE_PTYPE_L4_NONFRAG). + * + * Packet format: + * <'ether type'=0x0800 + * | 'version'=4, 'MF'=1> + * or, + * <'ether type'=0x0800 + * | 'version'=4, 'frag_offset'!=0> + * or, + * <'ether type'=0x86DD + * | 'version'=6, 'next header'=44> + */ +#define RTE_PTYPE_L4_FRAG 0x00000300 +/** + * SCTP (Stream Control Transmission Protocol) packet type. + * It is used for outer packet for tunneling cases. + * + * Packet format: + * <'ether type'=0x0800 + * | 'version'=4, 'protocol'=132, 'MF'=0, 'frag_offset'=0> + * or, + * <'ether type'=0x86DD + * | 'version'=6, 'next header'=132> + */ +#define RTE_PTYPE_L4_SCTP 0x00000400 +/** + * ICMP (Internet Control Message Protocol) packet type. + * It is used for outer packet for tunneling cases. + * + * Packet format: + * <'ether type'=0x0800 + * | 'version'=4, 'protocol'=1, 'MF'=0, 'frag_offset'=0> + * or, + * <'ether type'=0x86DD + * | 'version'=6, 'next header'=1> + */ +#define RTE_PTYPE_L4_ICMP 0x00000500 +/** + * Non-fragmented IP (Internet Protocol) packet type. + * It is used for outer packet for tunneling cases. + * + * It refers to those packets of any IP types, while cannot be recognized as + * any of above L4 types (RTE_PTYPE_L4_TCP, RTE_PTYPE_L4_UDP, + * RTE_PTYPE_L4_FRAG, RTE_PTYPE_L4_SCTP, RTE_PTYPE_L4_ICMP). + * + * Packet format: + * <'ether type'=0x0800 + * | 'version'=4, 'protocol'!=[6|17|132|1], 'MF'=0, 'frag_offset'=0> + * or, + * <'ether type'=0x86DD + * | 'version'=6, 'next header'!=[6|17|44|132|1]> + */ +#define RTE_PTYPE_L4_NONFRAG 0x00000600 +/** + * Mask of layer 4 packet types. + * It is used for outer packet for tunneling cases. + */ +#define RTE_PTYPE_L4_MASK 0x00000f00 +/** + * IP (Internet Protocol) in IP (Internet Protocol) tunneling packet type. + * + * Packet format: + * <'ether type'=0x0800 + * | 'version'=4, 'protocol'=[4|41]> + * or, + * <'ether type'=0x86DD + * | 'version'=6, 'next header'=[4|41]> + */ +#define RTE_PTYPE_TUNNEL_IP 0x00001000 +/** + * GRE (Generic Routing Encapsulation) tunneling packet type. + * + * Packet format: + * <'ether type'=0x0800 + * | 'version'=4, 'protocol'=47> + * or, + * <'ether type'=0x86DD + * | 'version'=6, 'next header'=47> + */ +#define RTE_PTYPE_TUNNEL_GRE 0x00002000 +/** + * VXLAN (Virtual eXtensible Local Area Network) tunneling packet type. + * + * Packet format: + * <'ether type'=0x0800 + * | 'version'=4, 'protocol'=17 + * | 'destination port'=4789> + * or, + * <'ether type'=0x86DD + * | 'version'=6, 'next header'=17 + * | 'destination port'=4789> + */ +#define RTE_PTYPE_TUNNEL_VXLAN 0x00003000 +/** + * NVGRE (Network Virtualization using Generic Routing Encapsulation) tunneling + * packet type. + * + * Packet format: + * <'ether type'=0x0800 + * | 'version'=4, 'protocol'=47 + * | 'protocol type'=0x6558> + * or, + * <'ether type'=0x86DD + * | 'version'=6, 'next header'=47 + * | 'protocol type'=0x6558'> + */ +#define RTE_PTYPE_TUNNEL_NVGRE 0x00004000 +/** + * GENEVE (Generic Network Virtualization Encapsulation) tunneling packet type. + * + * Packet format: + * <'ether type'=0x0800 + * | 'version'=4, 'protocol'=17 + * | 'destination port'=6081> + * or, + * <'ether type'=0x86DD + * | 'version'=6, 'next header'=17 + * | 'destination port'=6081> + */ +#define RTE_PTYPE_TUNNEL_GENEVE 0x00005000 +/** + * Tunneling packet type of Teredo, VXLAN (Virtual eXtensible Local Area + * Network) or GRE (Generic Routing Encapsulation) could be recognized as this + * packet type, if they can not be recognized independently as of hardware + * capability. + */ +#define RTE_PTYPE_TUNNEL_GRENAT 0x00006000 +/** + * GTP-C (GPRS Tunnelling Protocol) control tunneling packet type. + * Packet format: + * <'ether type'=0x0800 + * | 'version'=4, 'protocol'=17 + * | 'destination port'=2123> + * or, + * <'ether type'=0x86DD + * | 'version'=6, 'next header'=17 + * | 'destination port'=2123> + * or, + * <'ether type'=0x0800 + * | 'version'=4, 'protocol'=17 + * | 'source port'=2123> + * or, + * <'ether type'=0x86DD + * | 'version'=6, 'next header'=17 + * | 'source port'=2123> + */ +#define RTE_PTYPE_TUNNEL_GTPC 0x00007000 +/** + * GTP-U (GPRS Tunnelling Protocol) user data tunneling packet type. + * Packet format: + * <'ether type'=0x0800 + * | 'version'=4, 'protocol'=17 + * | 'destination port'=2152> + * or, + * <'ether type'=0x86DD + * | 'version'=6, 'next header'=17 + * | 'destination port'=2152> + */ +#define RTE_PTYPE_TUNNEL_GTPU 0x00008000 +/** + * ESP (IP Encapsulating Security Payload) tunneling packet type. + * + * Packet format: + * <'ether type'=0x0800 + * | 'version'=4, 'protocol'=51> + * or, + * <'ether type'=0x86DD + * | 'version'=6, 'next header'=51> + */ +#define RTE_PTYPE_TUNNEL_ESP 0x00009000 +/** + * Mask of tunneling packet types. + */ +#define RTE_PTYPE_TUNNEL_MASK 0x0000f000 +/** + * Ethernet packet type. + * It is used for inner packet type only. + * + * Packet format (inner only): + * <'ether type'=[0x800|0x86DD]> + */ +#define RTE_PTYPE_INNER_L2_ETHER 0x00010000 +/** + * Ethernet packet type with VLAN (Virtual Local Area Network) tag. + * + * Packet format (inner only): + * <'ether type'=[0x800|0x86DD], vlan=[1-4095]> + */ +#define RTE_PTYPE_INNER_L2_ETHER_VLAN 0x00020000 +/** + * QinQ packet type. + * + * Packet format: + * <'ether type'=[0x88A8]> + */ +#define RTE_PTYPE_INNER_L2_ETHER_QINQ 0x00030000 +/** + * Mask of inner layer 2 packet types. + */ +#define RTE_PTYPE_INNER_L2_MASK 0x000f0000 +/** + * IP (Internet Protocol) version 4 packet type. + * It is used for inner packet only, and does not contain any header option. + * + * Packet format (inner only): + * <'ether type'=0x0800 + * | 'version'=4, 'ihl'=5> + */ +#define RTE_PTYPE_INNER_L3_IPV4 0x00100000 +/** + * IP (Internet Protocol) version 4 packet type. + * It is used for inner packet only, and contains header options. + * + * Packet format (inner only): + * <'ether type'=0x0800 + * | 'version'=4, 'ihl'=[6-15], 'options'> + */ +#define RTE_PTYPE_INNER_L3_IPV4_EXT 0x00200000 +/** + * IP (Internet Protocol) version 6 packet type. + * It is used for inner packet only, and does not contain any extension header. + * + * Packet format (inner only): + * <'ether type'=0x86DD + * | 'version'=6, 'next header'=0x3B> + */ +#define RTE_PTYPE_INNER_L3_IPV6 0x00300000 +/** + * IP (Internet Protocol) version 4 packet type. + * It is used for inner packet only, and may or maynot contain header options. + * + * Packet format (inner only): + * <'ether type'=0x0800 + * | 'version'=4, 'ihl'=[5-15], <'options'>> + */ +#define RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN 0x00400000 +/** + * IP (Internet Protocol) version 6 packet type. + * It is used for inner packet only, and contains extension headers. + * + * Packet format (inner only): + * <'ether type'=0x86DD + * | 'version'=6, 'next header'=[0x0|0x2B|0x2C|0x32|0x33|0x3C|0x87], + * 'extension headers'> + */ +#define RTE_PTYPE_INNER_L3_IPV6_EXT 0x00500000 +/** + * IP (Internet Protocol) version 6 packet type. + * It is used for inner packet only, and may or maynot contain extension + * headers. + * + * Packet format (inner only): + * <'ether type'=0x86DD + * | 'version'=6, 'next header'=[0x3B|0x0|0x2B|0x2C|0x32|0x33|0x3C|0x87], + * <'extension headers'>> + */ +#define RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN 0x00600000 +/** + * Mask of inner layer 3 packet types. + */ +#define RTE_PTYPE_INNER_L3_MASK 0x00f00000 +/** + * TCP (Transmission Control Protocol) packet type. + * It is used for inner packet only. + * + * Packet format (inner only): + * <'ether type'=0x0800 + * | 'version'=4, 'protocol'=6, 'MF'=0, 'frag_offset'=0> + * or, + * <'ether type'=0x86DD + * | 'version'=6, 'next header'=6> + */ +#define RTE_PTYPE_INNER_L4_TCP 0x01000000 +/** + * UDP (User Datagram Protocol) packet type. + * It is used for inner packet only. + * + * Packet format (inner only): + * <'ether type'=0x0800 + * | 'version'=4, 'protocol'=17, 'MF'=0, 'frag_offset'=0> + * or, + * <'ether type'=0x86DD + * | 'version'=6, 'next header'=17> + */ +#define RTE_PTYPE_INNER_L4_UDP 0x02000000 +/** + * Fragmented IP (Internet Protocol) packet type. + * It is used for inner packet only, and may or maynot have layer 4 packet. + * + * Packet format (inner only): + * <'ether type'=0x0800 + * | 'version'=4, 'MF'=1> + * or, + * <'ether type'=0x0800 + * | 'version'=4, 'frag_offset'!=0> + * or, + * <'ether type'=0x86DD + * | 'version'=6, 'next header'=44> + */ +#define RTE_PTYPE_INNER_L4_FRAG 0x03000000 +/** + * SCTP (Stream Control Transmission Protocol) packet type. + * It is used for inner packet only. + * + * Packet format (inner only): + * <'ether type'=0x0800 + * | 'version'=4, 'protocol'=132, 'MF'=0, 'frag_offset'=0> + * or, + * <'ether type'=0x86DD + * | 'version'=6, 'next header'=132> + */ +#define RTE_PTYPE_INNER_L4_SCTP 0x04000000 +/** + * ICMP (Internet Control Message Protocol) packet type. + * It is used for inner packet only. + * + * Packet format (inner only): + * <'ether type'=0x0800 + * | 'version'=4, 'protocol'=1, 'MF'=0, 'frag_offset'=0> + * or, + * <'ether type'=0x86DD + * | 'version'=6, 'next header'=1> + */ +#define RTE_PTYPE_INNER_L4_ICMP 0x05000000 +/** + * Non-fragmented IP (Internet Protocol) packet type. + * It is used for inner packet only, and may or maynot have other unknown layer + * 4 packet types. + * + * Packet format (inner only): + * <'ether type'=0x0800 + * | 'version'=4, 'protocol'!=[6|17|132|1], 'MF'=0, 'frag_offset'=0> + * or, + * <'ether type'=0x86DD + * | 'version'=6, 'next header'!=[6|17|44|132|1]> + */ +#define RTE_PTYPE_INNER_L4_NONFRAG 0x06000000 +/** + * Mask of inner layer 4 packet types. + */ +#define RTE_PTYPE_INNER_L4_MASK 0x0f000000 +/** + * All valid layer masks. + */ +#define RTE_PTYPE_ALL_MASK 0x0fffffff + +/** + * Check if the (outer) L3 header is IPv4. To avoid comparing IPv4 types one by + * one, bit 4 is selected to be used for IPv4 only. Then checking bit 4 can + * determine if it is an IPV4 packet. + */ +#define RTE_ETH_IS_IPV4_HDR(ptype) ((ptype) & RTE_PTYPE_L3_IPV4) + +/** + * Check if the (outer) L3 header is IPv4. To avoid comparing IPv4 types one by + * one, bit 6 is selected to be used for IPv4 only. Then checking bit 6 can + * determine if it is an IPV4 packet. + */ +#define RTE_ETH_IS_IPV6_HDR(ptype) ((ptype) & RTE_PTYPE_L3_IPV6) + +/* Check if it is a tunneling packet */ +#define RTE_ETH_IS_TUNNEL_PKT(ptype) ((ptype) & \ + (RTE_PTYPE_TUNNEL_MASK | \ + RTE_PTYPE_INNER_L2_MASK | \ + RTE_PTYPE_INNER_L3_MASK | \ + RTE_PTYPE_INNER_L4_MASK)) + +/** + * Get the name of the l2 packet type + * + * @param ptype + * The packet type value. + * @return + * A non-null string describing the packet type. + */ +const char *rte_get_ptype_l2_name(uint32_t ptype); + +/** + * Get the name of the l3 packet type + * + * @param ptype + * The packet type value. + * @return + * A non-null string describing the packet type. + */ +const char *rte_get_ptype_l3_name(uint32_t ptype); + +/** + * Get the name of the l4 packet type + * + * @param ptype + * The packet type value. + * @return + * A non-null string describing the packet type. + */ +const char *rte_get_ptype_l4_name(uint32_t ptype); + +/** + * Get the name of the tunnel packet type + * + * @param ptype + * The packet type value. + * @return + * A non-null string describing the packet type. + */ +const char *rte_get_ptype_tunnel_name(uint32_t ptype); + +/** + * Get the name of the inner_l2 packet type + * + * @param ptype + * The packet type value. + * @return + * A non-null string describing the packet type. + */ +const char *rte_get_ptype_inner_l2_name(uint32_t ptype); + +/** + * Get the name of the inner_l3 packet type + * + * @param ptype + * The packet type value. + * @return + * A non-null string describing the packet type. + */ +const char *rte_get_ptype_inner_l3_name(uint32_t ptype); + +/** + * Get the name of the inner_l4 packet type + * + * @param ptype + * The packet type value. + * @return + * A non-null string describing the packet type. + */ +const char *rte_get_ptype_inner_l4_name(uint32_t ptype); + +/** + * Write the packet type name into the buffer + * + * @param ptype + * The packet type value. + * @param buf + * The buffer where the string is written. + * @param buflen + * The length of the buffer. + * @return + * - 0 on success + * - (-1) if the buffer is too small + */ +int rte_get_ptype_name(uint32_t ptype, char *buf, size_t buflen); + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_MBUF_PTYPE_H_ */ diff --git a/dpdk/lib/librte_mbuf/rte_mbuf_version.map b/dpdk/lib/librte_mbuf/rte_mbuf_version.map index e10f6bdc..6e2ea845 100644 --- a/dpdk/lib/librte_mbuf/rte_mbuf_version.map +++ b/dpdk/lib/librte_mbuf/rte_mbuf_version.map @@ -18,3 +18,20 @@ DPDK_2.1 { rte_pktmbuf_pool_create; } DPDK_2.0; + +DPDK_16.11 { + global: + + __rte_pktmbuf_read; + rte_get_ptype_inner_l2_name; + rte_get_ptype_inner_l3_name; + rte_get_ptype_inner_l4_name; + rte_get_ptype_l2_name; + rte_get_ptype_l3_name; + rte_get_ptype_l4_name; + rte_get_ptype_name; + rte_get_ptype_tunnel_name; + rte_get_rx_ol_flag_list; + rte_get_tx_ol_flag_list; + +} DPDK_2.1; diff --git a/dpdk/lib/librte_member/Makefile b/dpdk/lib/librte_member/Makefile new file mode 100644 index 00000000..f4cf101e --- /dev/null +++ b/dpdk/lib/librte_member/Makefile @@ -0,0 +1,52 @@ +# BSD LICENSE +# +# Copyright(c) 2017 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +include $(RTE_SDK)/mk/rte.vars.mk + +# library name +LIB = librte_member.a + +CFLAGS := -I$(SRCDIR) $(CFLAGS) +CFLAGS += $(WERROR_FLAGS) -O3 + +LDLIBS += -lm +LDLIBS += -lrte_eal -lrte_hash + +EXPORT_MAP := rte_member_version.map + +LIBABIVER := 1 + +# all source are stored in SRCS-y +SRCS-$(CONFIG_RTE_LIBRTE_MEMBER) += rte_member.c rte_member_ht.c rte_member_vbf.c +# install includes +SYMLINK-$(CONFIG_RTE_LIBRTE_MEMBER)-include := rte_member.h + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/lib/librte_member/rte_member.c b/dpdk/lib/librte_member/rte_member.c new file mode 100644 index 00000000..c3a238bb --- /dev/null +++ b/dpdk/lib/librte_member/rte_member.c @@ -0,0 +1,337 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +#include +#include +#include +#include +#include + +#include "rte_member.h" +#include "rte_member_ht.h" +#include "rte_member_vbf.h" + +int librte_member_logtype; + +TAILQ_HEAD(rte_member_list, rte_tailq_entry); +static struct rte_tailq_elem rte_member_tailq = { + .name = "RTE_MEMBER", +}; +EAL_REGISTER_TAILQ(rte_member_tailq) + +struct rte_member_setsum * +rte_member_find_existing(const char *name) +{ + struct rte_member_setsum *setsum = NULL; + struct rte_tailq_entry *te; + struct rte_member_list *member_list; + + member_list = RTE_TAILQ_CAST(rte_member_tailq.head, rte_member_list); + + rte_rwlock_read_lock(RTE_EAL_TAILQ_RWLOCK); + TAILQ_FOREACH(te, member_list, next) { + setsum = (struct rte_member_setsum *) te->data; + if (strncmp(name, setsum->name, RTE_MEMBER_NAMESIZE) == 0) + break; + } + rte_rwlock_read_unlock(RTE_EAL_TAILQ_RWLOCK); + + if (te == NULL) { + rte_errno = ENOENT; + return NULL; + } + return setsum; +} + +void +rte_member_free(struct rte_member_setsum *setsum) +{ + struct rte_member_list *member_list; + struct rte_tailq_entry *te; + + if (setsum == NULL) + return; + member_list = RTE_TAILQ_CAST(rte_member_tailq.head, rte_member_list); + rte_rwlock_write_lock(RTE_EAL_TAILQ_RWLOCK); + TAILQ_FOREACH(te, member_list, next) { + if (te->data == (void *)setsum) + break; + } + if (te == NULL) { + rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK); + return; + } + TAILQ_REMOVE(member_list, te, next); + rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK); + + switch (setsum->type) { + case RTE_MEMBER_TYPE_HT: + rte_member_free_ht(setsum); + break; + case RTE_MEMBER_TYPE_VBF: + rte_member_free_vbf(setsum); + break; + default: + break; + } + rte_free(setsum); + rte_free(te); +} + +struct rte_member_setsum * +rte_member_create(const struct rte_member_parameters *params) +{ + struct rte_tailq_entry *te; + struct rte_member_list *member_list; + struct rte_member_setsum *setsum; + int ret; + + if (params == NULL) { + rte_errno = EINVAL; + return NULL; + } + + if (params->key_len == 0 || + params->prim_hash_seed == params->sec_hash_seed) { + rte_errno = EINVAL; + RTE_MEMBER_LOG(ERR, "Create setsummary with " + "invalid parameters\n"); + return NULL; + } + + member_list = RTE_TAILQ_CAST(rte_member_tailq.head, rte_member_list); + + rte_rwlock_write_lock(RTE_EAL_TAILQ_RWLOCK); + + TAILQ_FOREACH(te, member_list, next) { + setsum = (struct rte_member_setsum *) te->data; + if (strncmp(params->name, setsum->name, + RTE_MEMBER_NAMESIZE) == 0) + break; + } + setsum = NULL; + if (te != NULL) { + rte_errno = EEXIST; + te = NULL; + goto error_unlock_exit; + } + te = rte_zmalloc("MEMBER_TAILQ_ENTRY", sizeof(*te), 0); + if (te == NULL) { + RTE_MEMBER_LOG(ERR, "tailq entry allocation failed\n"); + goto error_unlock_exit; + } + + /* Create a new setsum structure */ + setsum = (struct rte_member_setsum *) rte_zmalloc_socket(params->name, + sizeof(struct rte_member_setsum), RTE_CACHE_LINE_SIZE, + params->socket_id); + if (setsum == NULL) { + RTE_MEMBER_LOG(ERR, "Create setsummary failed\n"); + goto error_unlock_exit; + } + snprintf(setsum->name, sizeof(setsum->name), "%s", params->name); + setsum->type = params->type; + setsum->socket_id = params->socket_id; + setsum->key_len = params->key_len; + setsum->num_set = params->num_set; + setsum->prim_hash_seed = params->prim_hash_seed; + setsum->sec_hash_seed = params->sec_hash_seed; + + switch (setsum->type) { + case RTE_MEMBER_TYPE_HT: + ret = rte_member_create_ht(setsum, params); + break; + case RTE_MEMBER_TYPE_VBF: + ret = rte_member_create_vbf(setsum, params); + break; + default: + goto error_unlock_exit; + } + if (ret < 0) + goto error_unlock_exit; + + RTE_MEMBER_LOG(DEBUG, "Creating a setsummary table with " + "mode %u\n", setsum->type); + + te->data = (void *)setsum; + TAILQ_INSERT_TAIL(member_list, te, next); + rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK); + return setsum; + +error_unlock_exit: + rte_free(te); + rte_free(setsum); + rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK); + return NULL; +} + +int +rte_member_add(const struct rte_member_setsum *setsum, const void *key, + member_set_t set_id) +{ + if (setsum == NULL || key == NULL) + return -EINVAL; + + switch (setsum->type) { + case RTE_MEMBER_TYPE_HT: + return rte_member_add_ht(setsum, key, set_id); + case RTE_MEMBER_TYPE_VBF: + return rte_member_add_vbf(setsum, key, set_id); + default: + return -EINVAL; + } +} + +int +rte_member_lookup(const struct rte_member_setsum *setsum, const void *key, + member_set_t *set_id) +{ + if (setsum == NULL || key == NULL || set_id == NULL) + return -EINVAL; + + switch (setsum->type) { + case RTE_MEMBER_TYPE_HT: + return rte_member_lookup_ht(setsum, key, set_id); + case RTE_MEMBER_TYPE_VBF: + return rte_member_lookup_vbf(setsum, key, set_id); + default: + return -EINVAL; + } +} + +int +rte_member_lookup_bulk(const struct rte_member_setsum *setsum, + const void **keys, uint32_t num_keys, + member_set_t *set_ids) +{ + if (setsum == NULL || keys == NULL || set_ids == NULL) + return -EINVAL; + + switch (setsum->type) { + case RTE_MEMBER_TYPE_HT: + return rte_member_lookup_bulk_ht(setsum, keys, num_keys, + set_ids); + case RTE_MEMBER_TYPE_VBF: + return rte_member_lookup_bulk_vbf(setsum, keys, num_keys, + set_ids); + default: + return -EINVAL; + } +} + +int +rte_member_lookup_multi(const struct rte_member_setsum *setsum, const void *key, + uint32_t match_per_key, member_set_t *set_id) +{ + if (setsum == NULL || key == NULL || set_id == NULL) + return -EINVAL; + + switch (setsum->type) { + case RTE_MEMBER_TYPE_HT: + return rte_member_lookup_multi_ht(setsum, key, match_per_key, + set_id); + case RTE_MEMBER_TYPE_VBF: + return rte_member_lookup_multi_vbf(setsum, key, match_per_key, + set_id); + default: + return -EINVAL; + } +} + +int +rte_member_lookup_multi_bulk(const struct rte_member_setsum *setsum, + const void **keys, uint32_t num_keys, + uint32_t max_match_per_key, uint32_t *match_count, + member_set_t *set_ids) +{ + if (setsum == NULL || keys == NULL || set_ids == NULL || + match_count == NULL) + return -EINVAL; + + switch (setsum->type) { + case RTE_MEMBER_TYPE_HT: + return rte_member_lookup_multi_bulk_ht(setsum, keys, num_keys, + max_match_per_key, match_count, set_ids); + case RTE_MEMBER_TYPE_VBF: + return rte_member_lookup_multi_bulk_vbf(setsum, keys, num_keys, + max_match_per_key, match_count, set_ids); + default: + return -EINVAL; + } +} + +int +rte_member_delete(const struct rte_member_setsum *setsum, const void *key, + member_set_t set_id) +{ + if (setsum == NULL || key == NULL) + return -EINVAL; + + switch (setsum->type) { + case RTE_MEMBER_TYPE_HT: + return rte_member_delete_ht(setsum, key, set_id); + /* current vBF implementation does not support delete function */ + case RTE_MEMBER_TYPE_VBF: + default: + return -EINVAL; + } +} + +void +rte_member_reset(const struct rte_member_setsum *setsum) +{ + if (setsum == NULL) + return; + switch (setsum->type) { + case RTE_MEMBER_TYPE_HT: + rte_member_reset_ht(setsum); + return; + case RTE_MEMBER_TYPE_VBF: + rte_member_reset_vbf(setsum); + return; + default: + return; + } +} + +RTE_INIT(librte_member_init_log); + +static void +librte_member_init_log(void) +{ + librte_member_logtype = rte_log_register("librte.member"); + if (librte_member_logtype >= 0) + rte_log_set_level(librte_member_logtype, RTE_LOG_DEBUG); +} diff --git a/dpdk/lib/librte_member/rte_member.h b/dpdk/lib/librte_member/rte_member.h new file mode 100644 index 00000000..56dd3b3a --- /dev/null +++ b/dpdk/lib/librte_member/rte_member.h @@ -0,0 +1,519 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** + * @file + * + * RTE Membership Library + * + * The Membership Library is an extension and generalization of a traditional + * filter (for example Bloom Filter and cuckoo filter) structure that has + * multiple usages in a variety of workloads and applications. The library is + * used to test if a key belongs to certain sets. Two types of such + * "set-summary" structures are implemented: hash-table based (HT) and vector + * bloom filter (vBF). For HT setsummary, two subtypes or modes are available, + * cache and non-cache modes. The table below summarize some properties of + * the different implementations. + * + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + */ + +/** + * + */ + +#ifndef _RTE_MEMBER_H_ +#define _RTE_MEMBER_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +#include +#include + +/** The set ID type that stored internally in hash table based set summary. */ +typedef uint16_t member_set_t; +/** Invalid set ID used to mean no match found. */ +#define RTE_MEMBER_NO_MATCH 0 +/** Maximum size of hash table that can be created. */ +#define RTE_MEMBER_ENTRIES_MAX (1 << 30) +/** Maximum number of keys that can be searched as a bulk */ +#define RTE_MEMBER_LOOKUP_BULK_MAX 64 +/** Entry count per bucket in hash table based mode. */ +#define RTE_MEMBER_BUCKET_ENTRIES 16 +/** Maximum number of characters in setsum name. */ +#define RTE_MEMBER_NAMESIZE 32 + +/** @internal Hash function used by membership library. */ +#if defined(RTE_ARCH_X86) || defined(RTE_MACHINE_CPUFLAG_CRC32) +#include +#define MEMBER_HASH_FUNC rte_hash_crc +#else +#include +#define MEMBER_HASH_FUNC rte_jhash +#endif + +extern int librte_member_logtype; + +#define RTE_MEMBER_LOG(level, ...) \ + rte_log(RTE_LOG_ ## level, \ + librte_member_logtype, \ + RTE_FMT("%s(): " RTE_FMT_HEAD(__VA_ARGS__,), \ + __func__, \ + RTE_FMT_TAIL(__VA_ARGS__,))) + +/** @internal setsummary structure. */ +struct rte_member_setsum; + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Parameter struct used to create set summary + */ +struct rte_member_parameters; + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Define different set summary types + */ +enum rte_member_setsum_type { + RTE_MEMBER_TYPE_HT = 0, /**< Hash table based set summary. */ + RTE_MEMBER_TYPE_VBF, /**< Vector of bloom filters. */ + RTE_MEMBER_NUM_TYPE +}; + +/** @internal compare function for different arch. */ +enum rte_member_sig_compare_function { + RTE_MEMBER_COMPARE_SCALAR = 0, + RTE_MEMBER_COMPARE_AVX2, + RTE_MEMBER_COMPARE_NUM +}; + +/** @internal setsummary structure. */ +struct rte_member_setsum { + enum rte_member_setsum_type type; /* Type of the set summary. */ + uint32_t key_len; /* Length of key. */ + uint32_t prim_hash_seed; /* Primary hash function seed. */ + uint32_t sec_hash_seed; /* Secondary hash function seed. */ + + /* Hash table based. */ + uint32_t bucket_cnt; /* Number of buckets. */ + uint32_t bucket_mask; /* Bit mask to get bucket index. */ + /* For runtime selecting AVX, scalar, etc for signature comparison. */ + enum rte_member_sig_compare_function sig_cmp_fn; + uint8_t cache; /* If it is cache mode for ht based. */ + + /* Vector bloom filter. */ + uint32_t num_set; /* Number of set (bf) in vbf. */ + uint32_t bits; /* Number of bits in each bf. */ + uint32_t bit_mask; /* Bit mask to get bit location in bf. */ + uint32_t num_hashes; /* Number of hash values to index bf. */ + + uint32_t mul_shift; /* vbf internal variable used during bit test. */ + uint32_t div_shift; /* vbf internal variable used during bit test. */ + + void *table; /* This is the handler of hash table or vBF array. */ + + + /* Second cache line should start here. */ + uint32_t socket_id; /* NUMA Socket ID for memory. */ + char name[RTE_MEMBER_NAMESIZE]; /* Name of this set summary. */ +} __rte_cache_aligned; + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Parameters used when create the set summary table. Currently user can + * specify two types of setsummary: HT based and vBF. For HT based, user can + * specify cache or non-cache mode. Here is a table to describe some differences + * + */ +struct rte_member_parameters { + const char *name; /**< Name of the hash. */ + + /** + * User to specify the type of the setsummary from one of + * rte_member_setsum_type. + * + * HT based setsummary is implemented like a hash table. User should use + * this type when there are many sets. + * + * vBF setsummary is a vector of bloom filters. It is used when number + * of sets is not big (less than 32 for current implementation). + */ + enum rte_member_setsum_type type; + + /** + * is_cache is only used for HT based setsummary. + * + * If it is HT based setsummary, user to specify the subtype or mode + * of the setsummary. It could be cache, or non-cache mode. + * Set is_cache to be 1 if to use as cache mode. + * + * For cache mode, keys can be evicted out of the HT setsummary. Keys + * with the same signature and map to the same bucket + * will overwrite each other in the setsummary table. + * This mode is useful for the case that the set-summary only + * needs to keep record of the recently inserted keys. Both + * false-negative and false-positive could happen. + * + * For non-cache mode, keys cannot be evicted out of the cache. So for + * this mode the setsummary will become full eventually. Keys with the + * same signature but map to the same bucket will still occupy multiple + * entries. This mode does not give false-negative result. + */ + uint8_t is_cache; + + /** + * For HT setsummary, num_keys equals to the number of entries of the + * table. When the number of keys inserted in the HT setsummary + * approaches this number, eviction could happen. For cache mode, + * keys could be evicted out of the table. For non-cache mode, keys will + * be evicted to other buckets like cuckoo hash. The table will also + * likely to become full before the number of inserted keys equal to the + * total number of entries. + * + * For vBF, num_keys equal to the expected number of keys that will + * be inserted into the vBF. The implementation assumes the keys are + * evenly distributed to each BF in vBF. This is used to calculate the + * number of bits we need for each BF. User does not specify the size of + * each BF directly because the optimal size depends on the num_keys + * and false positive rate. + */ + uint32_t num_keys; + + /** + * The length of key is used for hash calculation. Since key is not + * stored in set-summary, large key does not require more memory space. + */ + uint32_t key_len; + + /** + * num_set is only used for vBF, but not used for HT setsummary. + * + * num_set is equal to the number of BFs in vBF. For current + * implementation, it only supports 1,2,4,8,16,32 BFs in one vBF set + * summary. If other number of sets are needed, for example 5, the user + * should allocate the minimum available value that larger than 5, + * which is 8. + */ + uint32_t num_set; + + /** + * false_positive_rate is only used for vBF, but not used for HT + * setsummary. + * + * For vBF, false_positive_rate is the user-defined false positive rate + * given expected number of inserted keys (num_keys). It is used to + * calculate the total number of bits for each BF, and the number of + * hash values used during lookup and insertion. For details please + * refer to vBF implementation and membership library documentation. + * + * For HT, This parameter is not directly set by users. + * HT setsummary's false positive rate is in the order of: + * false_pos = (1/bucket_count)*(1/2^16), since we use 16-bit signature. + * This is because two keys needs to map to same bucket and same + * signature to have a collision (false positive). bucket_count is equal + * to number of entries (num_keys) divided by entry count per bucket + * (RTE_MEMBER_BUCKET_ENTRIES). Thus, the false_positive_rate is not + * directly set by users for HT mode. + */ + float false_positive_rate; + + /** + * We use two seeds to calculate two independent hashes for each key. + * + * For HT type, one hash is used as signature, and the other is used + * for bucket location. + * For vBF type, these two hashes and their combinations are used as + * hash locations to index the bit array. + */ + uint32_t prim_hash_seed; + + /** + * The secondary seed should be a different value from the primary seed. + */ + uint32_t sec_hash_seed; + + int socket_id; /**< NUMA Socket ID for memory. */ +}; + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Find an existing set-summary and return a pointer to it. + * + * @param name + * Name of the set-summary. + * @return + * Pointer to the set-summary or NULL if object not found + * with rte_errno set appropriately. Possible rte_errno values include: + * - ENOENT - value not available for return + */ +struct rte_member_setsum * +rte_member_find_existing(const char *name); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Create set-summary (SS). + * + * @param params + * Parameters to initialize the setsummary. + * @return + * Return the pointer to the setsummary. + * Return value is NULL if the creation failed. + */ +struct rte_member_setsum * +rte_member_create(const struct rte_member_parameters *params); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Lookup key in set-summary (SS). + * Single key lookup and return as soon as the first match found + * + * @param setsum + * Pointer of a setsummary. + * @param key + * Pointer of the key to be looked up. + * @param set_id + * Output the set id matches the key. + * @return + * Return 1 for found a match and 0 for not found a match. + */ +int +rte_member_lookup(const struct rte_member_setsum *setsum, const void *key, + member_set_t *set_id); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Lookup bulk of keys in set-summary (SS). + * Each key lookup returns as soon as the first match found + * + * @param setsum + * Pointer of a setsummary. + * @param keys + * Pointer of the bulk of keys to be looked up. + * @param num_keys + * Number of keys that will be lookup. + * @param set_ids + * Output set ids for all the keys to this array. + * User should preallocate array that can contain all results, which size is + * the num_keys. + * @return + * The number of keys that found a match. + */ +int +rte_member_lookup_bulk(const struct rte_member_setsum *setsum, + const void **keys, uint32_t num_keys, + member_set_t *set_ids); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Lookup a key in set-summary (SS) for multiple matches. + * The key lookup will find all matched entries (multiple match). + * Note that for cache mode of HT, each key can have at most one match. This is + * because keys with same signature that maps to same bucket will overwrite + * each other. So multi-match lookup should be used for vBF and non-cache HT. + * + * @param setsum + * Pointer of a set-summary. + * @param key + * Pointer of the key that to be looked up. + * @param max_match_per_key + * User specified maximum number of matches for each key. The function returns + * as soon as this number of matches found for the key. + * @param set_id + * Output set ids for all the matches of the key. User needs to preallocate + * the array that can contain max_match_per_key number of results. + * @return + * The number of matches that found for the key. + * For cache mode HT set-summary, the number should be at most 1. + */ +int +rte_member_lookup_multi(const struct rte_member_setsum *setsum, + const void *key, uint32_t max_match_per_key, + member_set_t *set_id); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Lookup a bulk of keys in set-summary (SS) for multiple matches each key. + * Each key lookup will find all matched entries (multiple match). + * Note that for cache mode HT, each key can have at most one match. So + * multi-match function is mainly used for vBF and non-cache mode HT. + * + * @param setsum + * Pointer of a setsummary. + * @param keys + * Pointer of the keys to be looked up. + * @param num_keys + * The number of keys that will be lookup. + * @param max_match_per_key + * The possible maximum number of matches for each key. + * @param match_count + * Output the number of matches for each key in an array. + * @param set_ids + * Return set ids for all the matches of all keys. Users pass in a + * preallocated 2D array with first dimension as key index and second + * dimension as match index. For example set_ids[bulk_size][max_match_per_key] + * @return + * The number of keys that found one or more matches in the set-summary. + */ +int +rte_member_lookup_multi_bulk(const struct rte_member_setsum *setsum, + const void **keys, uint32_t num_keys, + uint32_t max_match_per_key, + uint32_t *match_count, + member_set_t *set_ids); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Insert key into set-summary (SS). + * + * @param setsum + * Pointer of a set-summary. + * @param key + * Pointer of the key to be added. + * @param set_id + * The set id associated with the key that needs to be added. Different mode + * supports different set_id ranges. 0 cannot be used as set_id since + * RTE_MEMBER_NO_MATCH by default is set as 0. + * For HT mode, the set_id has range as [1, 0x7FFF], MSB is reserved. + * For vBF mode the set id is limited by the num_set parameter when create + * the set-summary. + * @return + * HT (cache mode) and vBF should never fail unless the set_id is not in the + * valid range. In such case -EINVAL is returned. + * For HT (non-cache mode) it could fail with -ENOSPC error code when table is + * full. + * For success it returns different values for different modes to provide + * extra information for users. + * Return 0 for HT (cache mode) if the add does not cause + * eviction, return 1 otherwise. Return 0 for non-cache mode if success, + * -ENOSPC for full, and 1 if cuckoo eviction happens. + * Always returns 0 for vBF mode. + */ +int +rte_member_add(const struct rte_member_setsum *setsum, const void *key, + member_set_t set_id); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * De-allocate memory used by set-summary. + * + * @param setsum + * Pointer to the set summary. + */ +void +rte_member_free(struct rte_member_setsum *setsum); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Reset the set-summary tables. E.g. reset bits to be 0 in BF, + * reset set_id in each entry to be RTE_MEMBER_NO_MATCH in HT based SS. + * + * @param setsum + * Pointer to the set-summary. + */ +void +rte_member_reset(const struct rte_member_setsum *setsum); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Delete items from the set-summary. Note that vBF does not support deletion + * in current implementation. For vBF, error code of -EINVAL will be returned. + * + * @param setsum + * Pointer to the set-summary. + * @param key + * Pointer of the key to be deleted. + * @param set_id + * For HT mode, we need both key and its corresponding set_id to + * properly delete the key. Without set_id, we may delete other keys with the + * same signature. + * @return + * If no entry found to delete, an error code of -ENOENT could be returned. + */ +int +rte_member_delete(const struct rte_member_setsum *setsum, const void *key, + member_set_t set_id); + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_MEMBER_H_ */ diff --git a/dpdk/lib/librte_member/rte_member_ht.c b/dpdk/lib/librte_member/rte_member_ht.c new file mode 100644 index 00000000..59332d56 --- /dev/null +++ b/dpdk/lib/librte_member/rte_member_ht.c @@ -0,0 +1,586 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include + +#include "rte_member.h" +#include "rte_member_ht.h" + +#if defined(RTE_ARCH_X86) +#include "rte_member_x86.h" +#endif + +/* Search bucket for entry with tmp_sig and update set_id */ +static inline int +update_entry_search(uint32_t bucket_id, member_sig_t tmp_sig, + struct member_ht_bucket *buckets, + member_set_t set_id) +{ + uint32_t i; + + for (i = 0; i < RTE_MEMBER_BUCKET_ENTRIES; i++) { + if (buckets[bucket_id].sigs[i] == tmp_sig) { + buckets[bucket_id].sets[i] = set_id; + return 1; + } + } + return 0; +} + +static inline int +search_bucket_single(uint32_t bucket_id, member_sig_t tmp_sig, + struct member_ht_bucket *buckets, + member_set_t *set_id) +{ + uint32_t iter; + + for (iter = 0; iter < RTE_MEMBER_BUCKET_ENTRIES; iter++) { + if (tmp_sig == buckets[bucket_id].sigs[iter] && + buckets[bucket_id].sets[iter] != + RTE_MEMBER_NO_MATCH) { + *set_id = buckets[bucket_id].sets[iter]; + return 1; + } + } + return 0; +} + +static inline void +search_bucket_multi(uint32_t bucket_id, member_sig_t tmp_sig, + struct member_ht_bucket *buckets, + uint32_t *counter, + uint32_t matches_per_key, + member_set_t *set_id) +{ + uint32_t iter; + + for (iter = 0; iter < RTE_MEMBER_BUCKET_ENTRIES; iter++) { + if (tmp_sig == buckets[bucket_id].sigs[iter] && + buckets[bucket_id].sets[iter] != + RTE_MEMBER_NO_MATCH) { + set_id[*counter] = buckets[bucket_id].sets[iter]; + (*counter)++; + if (*counter >= matches_per_key) + return; + } + } +} + +int +rte_member_create_ht(struct rte_member_setsum *ss, + const struct rte_member_parameters *params) +{ + uint32_t i, j; + uint32_t size_bucket_t; + uint32_t num_entries = rte_align32pow2(params->num_keys); + + if ((num_entries > RTE_MEMBER_ENTRIES_MAX) || + !rte_is_power_of_2(RTE_MEMBER_BUCKET_ENTRIES) || + num_entries < RTE_MEMBER_BUCKET_ENTRIES) { + rte_errno = EINVAL; + RTE_MEMBER_LOG(ERR, + "Membership HT create with invalid parameters\n"); + return -EINVAL; + } + + uint32_t num_buckets = num_entries / RTE_MEMBER_BUCKET_ENTRIES; + + size_bucket_t = sizeof(struct member_ht_bucket); + + struct member_ht_bucket *buckets = rte_zmalloc_socket(NULL, + num_buckets * size_bucket_t, + RTE_CACHE_LINE_SIZE, ss->socket_id); + + if (buckets == NULL) { + RTE_MEMBER_LOG(ERR, "memory allocation failed for HT " + "setsummary\n"); + return -ENOMEM; + } + + ss->table = buckets; + ss->bucket_cnt = num_buckets; + ss->bucket_mask = num_buckets - 1; + ss->cache = params->is_cache; + + for (i = 0; i < num_buckets; i++) { + for (j = 0; j < RTE_MEMBER_BUCKET_ENTRIES; j++) + buckets[i].sets[j] = RTE_MEMBER_NO_MATCH; + } +#if defined(RTE_ARCH_X86) + if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2) && + RTE_MEMBER_BUCKET_ENTRIES == 16) + ss->sig_cmp_fn = RTE_MEMBER_COMPARE_AVX2; + else +#endif + ss->sig_cmp_fn = RTE_MEMBER_COMPARE_SCALAR; + + RTE_MEMBER_LOG(DEBUG, "Hash table based filter created, " + "the table has %u entries, %u buckets\n", + num_entries, num_buckets); + return 0; +} + +static inline void +get_buckets_index(const struct rte_member_setsum *ss, const void *key, + uint32_t *prim_bkt, uint32_t *sec_bkt, member_sig_t *sig) +{ + uint32_t first_hash = MEMBER_HASH_FUNC(key, ss->key_len, + ss->prim_hash_seed); + uint32_t sec_hash = MEMBER_HASH_FUNC(&first_hash, sizeof(uint32_t), + ss->sec_hash_seed); + /* + * We use the first hash value for the signature, and the second hash + * value to derive the primary and secondary bucket locations. + * + * For non-cache mode, we use the lower bits for the primary bucket + * location. Then we xor primary bucket location and the signature + * to get the secondary bucket location. This is called "partial-key + * cuckoo hashing" proposed by B. Fan, et al's paper + * "Cuckoo Filter: Practically Better Than Bloom". The benefit to use + * xor is that one could derive the alternative bucket location + * by only using the current bucket location and the signature. This is + * generally required by non-cache mode's eviction and deletion + * process without the need to store alternative hash value nor the full + * key. + * + * For cache mode, we use the lower bits for the primary bucket + * location and the higher bits for the secondary bucket location. In + * cache mode, keys are simply overwritten if bucket is full. We do not + * use xor since lower/higher bits are more independent hash values thus + * should provide slightly better table load. + */ + *sig = first_hash; + if (ss->cache) { + *prim_bkt = sec_hash & ss->bucket_mask; + *sec_bkt = (sec_hash >> 16) & ss->bucket_mask; + } else { + *prim_bkt = sec_hash & ss->bucket_mask; + *sec_bkt = (*prim_bkt ^ *sig) & ss->bucket_mask; + } +} + +int +rte_member_lookup_ht(const struct rte_member_setsum *ss, + const void *key, member_set_t *set_id) +{ + uint32_t prim_bucket, sec_bucket; + member_sig_t tmp_sig; + struct member_ht_bucket *buckets = ss->table; + + *set_id = RTE_MEMBER_NO_MATCH; + get_buckets_index(ss, key, &prim_bucket, &sec_bucket, &tmp_sig); + + switch (ss->sig_cmp_fn) { +#if defined(RTE_ARCH_X86) && defined(RTE_MACHINE_CPUFLAG_AVX2) + case RTE_MEMBER_COMPARE_AVX2: + if (search_bucket_single_avx(prim_bucket, tmp_sig, buckets, + set_id) || + search_bucket_single_avx(sec_bucket, tmp_sig, + buckets, set_id)) + return 1; + break; +#endif + default: + if (search_bucket_single(prim_bucket, tmp_sig, buckets, + set_id) || + search_bucket_single(sec_bucket, tmp_sig, + buckets, set_id)) + return 1; + } + + return 0; +} + +uint32_t +rte_member_lookup_bulk_ht(const struct rte_member_setsum *ss, + const void **keys, uint32_t num_keys, member_set_t *set_id) +{ + uint32_t i; + uint32_t num_matches = 0; + struct member_ht_bucket *buckets = ss->table; + member_sig_t tmp_sig[RTE_MEMBER_LOOKUP_BULK_MAX]; + uint32_t prim_buckets[RTE_MEMBER_LOOKUP_BULK_MAX]; + uint32_t sec_buckets[RTE_MEMBER_LOOKUP_BULK_MAX]; + + for (i = 0; i < num_keys; i++) { + get_buckets_index(ss, keys[i], &prim_buckets[i], + &sec_buckets[i], &tmp_sig[i]); + rte_prefetch0(&buckets[prim_buckets[i]]); + rte_prefetch0(&buckets[sec_buckets[i]]); + } + + for (i = 0; i < num_keys; i++) { + switch (ss->sig_cmp_fn) { +#if defined(RTE_ARCH_X86) && defined(RTE_MACHINE_CPUFLAG_AVX2) + case RTE_MEMBER_COMPARE_AVX2: + if (search_bucket_single_avx(prim_buckets[i], + tmp_sig[i], buckets, &set_id[i]) || + search_bucket_single_avx(sec_buckets[i], + tmp_sig[i], buckets, &set_id[i])) + num_matches++; + else + set_id[i] = RTE_MEMBER_NO_MATCH; + break; +#endif + default: + if (search_bucket_single(prim_buckets[i], tmp_sig[i], + buckets, &set_id[i]) || + search_bucket_single(sec_buckets[i], + tmp_sig[i], buckets, &set_id[i])) + num_matches++; + else + set_id[i] = RTE_MEMBER_NO_MATCH; + } + } + return num_matches; +} + +uint32_t +rte_member_lookup_multi_ht(const struct rte_member_setsum *ss, + const void *key, uint32_t match_per_key, + member_set_t *set_id) +{ + uint32_t num_matches = 0; + uint32_t prim_bucket, sec_bucket; + member_sig_t tmp_sig; + struct member_ht_bucket *buckets = ss->table; + + get_buckets_index(ss, key, &prim_bucket, &sec_bucket, &tmp_sig); + + switch (ss->sig_cmp_fn) { +#if defined(RTE_ARCH_X86) && defined(RTE_MACHINE_CPUFLAG_AVX2) + case RTE_MEMBER_COMPARE_AVX2: + search_bucket_multi_avx(prim_bucket, tmp_sig, buckets, + &num_matches, match_per_key, set_id); + if (num_matches < match_per_key) + search_bucket_multi_avx(sec_bucket, tmp_sig, + buckets, &num_matches, match_per_key, set_id); + return num_matches; +#endif + default: + search_bucket_multi(prim_bucket, tmp_sig, buckets, &num_matches, + match_per_key, set_id); + if (num_matches < match_per_key) + search_bucket_multi(sec_bucket, tmp_sig, + buckets, &num_matches, match_per_key, set_id); + return num_matches; + } +} + +uint32_t +rte_member_lookup_multi_bulk_ht(const struct rte_member_setsum *ss, + const void **keys, uint32_t num_keys, uint32_t match_per_key, + uint32_t *match_count, + member_set_t *set_ids) +{ + uint32_t i; + uint32_t num_matches = 0; + struct member_ht_bucket *buckets = ss->table; + uint32_t match_cnt_tmp; + member_sig_t tmp_sig[RTE_MEMBER_LOOKUP_BULK_MAX]; + uint32_t prim_buckets[RTE_MEMBER_LOOKUP_BULK_MAX]; + uint32_t sec_buckets[RTE_MEMBER_LOOKUP_BULK_MAX]; + + for (i = 0; i < num_keys; i++) { + get_buckets_index(ss, keys[i], &prim_buckets[i], + &sec_buckets[i], &tmp_sig[i]); + rte_prefetch0(&buckets[prim_buckets[i]]); + rte_prefetch0(&buckets[sec_buckets[i]]); + } + for (i = 0; i < num_keys; i++) { + match_cnt_tmp = 0; + + switch (ss->sig_cmp_fn) { +#if defined(RTE_ARCH_X86) && defined(RTE_MACHINE_CPUFLAG_AVX2) + case RTE_MEMBER_COMPARE_AVX2: + search_bucket_multi_avx(prim_buckets[i], tmp_sig[i], + buckets, &match_cnt_tmp, match_per_key, + &set_ids[i*match_per_key]); + if (match_cnt_tmp < match_per_key) + search_bucket_multi_avx(sec_buckets[i], + tmp_sig[i], buckets, &match_cnt_tmp, + match_per_key, + &set_ids[i*match_per_key]); + match_count[i] = match_cnt_tmp; + if (match_cnt_tmp != 0) + num_matches++; + break; +#endif + default: + search_bucket_multi(prim_buckets[i], tmp_sig[i], + buckets, &match_cnt_tmp, match_per_key, + &set_ids[i*match_per_key]); + if (match_cnt_tmp < match_per_key) + search_bucket_multi(sec_buckets[i], tmp_sig[i], + buckets, &match_cnt_tmp, match_per_key, + &set_ids[i*match_per_key]); + match_count[i] = match_cnt_tmp; + if (match_cnt_tmp != 0) + num_matches++; + } + } + return num_matches; +} + +static inline int +try_insert(struct member_ht_bucket *buckets, uint32_t prim, uint32_t sec, + member_sig_t sig, member_set_t set_id) +{ + int i; + /* If not full then insert into one slot */ + for (i = 0; i < RTE_MEMBER_BUCKET_ENTRIES; i++) { + if (buckets[prim].sets[i] == RTE_MEMBER_NO_MATCH) { + buckets[prim].sigs[i] = sig; + buckets[prim].sets[i] = set_id; + return 0; + } + } + /* If prim failed, we need to access second bucket */ + for (i = 0; i < RTE_MEMBER_BUCKET_ENTRIES; i++) { + if (buckets[sec].sets[i] == RTE_MEMBER_NO_MATCH) { + buckets[sec].sigs[i] = sig; + buckets[sec].sets[i] = set_id; + return 0; + } + } + return -1; +} + +static inline int +try_update(struct member_ht_bucket *buckets, uint32_t prim, uint32_t sec, + member_sig_t sig, member_set_t set_id, + enum rte_member_sig_compare_function cmp_fn) +{ + switch (cmp_fn) { +#if defined(RTE_ARCH_X86) && defined(RTE_MACHINE_CPUFLAG_AVX2) + case RTE_MEMBER_COMPARE_AVX2: + if (update_entry_search_avx(prim, sig, buckets, set_id) || + update_entry_search_avx(sec, sig, buckets, + set_id)) + return 0; + break; +#endif + default: + if (update_entry_search(prim, sig, buckets, set_id) || + update_entry_search(sec, sig, buckets, + set_id)) + return 0; + } + return -1; +} + +static inline int +evict_from_bucket(void) +{ + /* For now, we randomly pick one entry to evict */ + return rte_rand() & (RTE_MEMBER_BUCKET_ENTRIES - 1); +} + +/* + * This function is similar to the cuckoo hash make_space function in hash + * library + */ +static inline int +make_space_bucket(const struct rte_member_setsum *ss, uint32_t bkt_idx, + unsigned int *nr_pushes) +{ + unsigned int i, j; + int ret; + struct member_ht_bucket *buckets = ss->table; + uint32_t next_bucket_idx; + struct member_ht_bucket *next_bkt[RTE_MEMBER_BUCKET_ENTRIES]; + struct member_ht_bucket *bkt = &buckets[bkt_idx]; + /* MSB is set to indicate if an entry has been already pushed */ + member_set_t flag_mask = 1U << (sizeof(member_set_t) * 8 - 1); + + /* + * Push existing item (search for bucket with space in + * alternative locations) to its alternative location + */ + for (i = 0; i < RTE_MEMBER_BUCKET_ENTRIES; i++) { + /* Search for space in alternative locations */ + next_bucket_idx = (bkt->sigs[i] ^ bkt_idx) & ss->bucket_mask; + next_bkt[i] = &buckets[next_bucket_idx]; + for (j = 0; j < RTE_MEMBER_BUCKET_ENTRIES; j++) { + if (next_bkt[i]->sets[j] == RTE_MEMBER_NO_MATCH) + break; + } + + if (j != RTE_MEMBER_BUCKET_ENTRIES) + break; + } + + /* Alternative location has spare room (end of recursive function) */ + if (i != RTE_MEMBER_BUCKET_ENTRIES) { + next_bkt[i]->sigs[j] = bkt->sigs[i]; + next_bkt[i]->sets[j] = bkt->sets[i]; + return i; + } + + /* Pick entry that has not been pushed yet */ + for (i = 0; i < RTE_MEMBER_BUCKET_ENTRIES; i++) + if ((bkt->sets[i] & flag_mask) == 0) + break; + + /* All entries have been pushed, so entry cannot be added */ + if (i == RTE_MEMBER_BUCKET_ENTRIES || + ++(*nr_pushes) > RTE_MEMBER_MAX_PUSHES) + return -ENOSPC; + + next_bucket_idx = (bkt->sigs[i] ^ bkt_idx) & ss->bucket_mask; + /* Set flag to indicate that this entry is going to be pushed */ + bkt->sets[i] |= flag_mask; + + /* Need room in alternative bucket to insert the pushed entry */ + ret = make_space_bucket(ss, next_bucket_idx, nr_pushes); + /* + * After recursive function. + * Clear flags and insert the pushed entry + * in its alternative location if successful, + * or return error + */ + bkt->sets[i] &= ~flag_mask; + if (ret >= 0) { + next_bkt[i]->sigs[ret] = bkt->sigs[i]; + next_bkt[i]->sets[ret] = bkt->sets[i]; + return i; + } else + return ret; +} + +int +rte_member_add_ht(const struct rte_member_setsum *ss, + const void *key, member_set_t set_id) +{ + int ret; + unsigned int nr_pushes = 0; + uint32_t prim_bucket, sec_bucket; + member_sig_t tmp_sig; + struct member_ht_bucket *buckets = ss->table; + member_set_t flag_mask = 1U << (sizeof(member_set_t) * 8 - 1); + + if (set_id == RTE_MEMBER_NO_MATCH || (set_id & flag_mask) != 0) + return -EINVAL; + + get_buckets_index(ss, key, &prim_bucket, &sec_bucket, &tmp_sig); + + /* + * If it is cache based setsummary, we try overwriting (updating) + * existing entry with the same signature first. In cache mode, we allow + * false negatives and only cache the most recent keys. + * + * For non-cache mode, we do not update existing entry with the same + * signature. This is because if two keys with same signature update + * each other, false negative may happen, which is not the expected + * behavior for non-cache setsummary. + */ + if (ss->cache) { + ret = try_update(buckets, prim_bucket, sec_bucket, tmp_sig, + set_id, ss->sig_cmp_fn); + if (ret != -1) + return ret; + } + /* If not full then insert into one slot */ + ret = try_insert(buckets, prim_bucket, sec_bucket, tmp_sig, set_id); + if (ret != -1) + return ret; + + /* Random pick prim or sec for recursive displacement */ + uint32_t select_bucket = (tmp_sig && 1U) ? prim_bucket : sec_bucket; + if (ss->cache) { + ret = evict_from_bucket(); + buckets[select_bucket].sigs[ret] = tmp_sig; + buckets[select_bucket].sets[ret] = set_id; + return 1; + } + + ret = make_space_bucket(ss, select_bucket, &nr_pushes); + if (ret >= 0) { + buckets[select_bucket].sigs[ret] = tmp_sig; + buckets[select_bucket].sets[ret] = set_id; + ret = 1; + } + + return ret; +} + +void +rte_member_free_ht(struct rte_member_setsum *ss) +{ + rte_free(ss->table); +} + +int +rte_member_delete_ht(const struct rte_member_setsum *ss, const void *key, + member_set_t set_id) +{ + int i; + uint32_t prim_bucket, sec_bucket; + member_sig_t tmp_sig; + struct member_ht_bucket *buckets = ss->table; + + get_buckets_index(ss, key, &prim_bucket, &sec_bucket, &tmp_sig); + + for (i = 0; i < RTE_MEMBER_BUCKET_ENTRIES; i++) { + if (tmp_sig == buckets[prim_bucket].sigs[i] && + set_id == buckets[prim_bucket].sets[i]) { + buckets[prim_bucket].sets[i] = RTE_MEMBER_NO_MATCH; + return 0; + } + } + + for (i = 0; i < RTE_MEMBER_BUCKET_ENTRIES; i++) { + if (tmp_sig == buckets[sec_bucket].sigs[i] && + set_id == buckets[sec_bucket].sets[i]) { + buckets[sec_bucket].sets[i] = RTE_MEMBER_NO_MATCH; + return 0; + } + } + return -ENOENT; +} + +void +rte_member_reset_ht(const struct rte_member_setsum *ss) +{ + uint32_t i, j; + struct member_ht_bucket *buckets = ss->table; + + for (i = 0; i < ss->bucket_cnt; i++) { + for (j = 0; j < RTE_MEMBER_BUCKET_ENTRIES; j++) + buckets[i].sets[j] = RTE_MEMBER_NO_MATCH; + } +} diff --git a/dpdk/lib/librte_member/rte_member_ht.h b/dpdk/lib/librte_member/rte_member_ht.h new file mode 100644 index 00000000..3148a492 --- /dev/null +++ b/dpdk/lib/librte_member/rte_member_ht.h @@ -0,0 +1,94 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_MEMBER_HT_H_ +#define _RTE_MEMBER_HT_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +/* Maximum number of pushes for cuckoo path in HT mode. */ +#define RTE_MEMBER_MAX_PUSHES 50 + +typedef uint16_t member_sig_t; /* signature size is 16 bit */ + +/* The bucket struct for ht setsum */ +struct member_ht_bucket { + member_sig_t sigs[RTE_MEMBER_BUCKET_ENTRIES]; /* 2-byte signature */ + member_set_t sets[RTE_MEMBER_BUCKET_ENTRIES]; /* 2-byte set */ +} __rte_cache_aligned; + +int +rte_member_create_ht(struct rte_member_setsum *ss, + const struct rte_member_parameters *params); + +int +rte_member_lookup_ht(const struct rte_member_setsum *setsum, + const void *key, member_set_t *set_id); + +uint32_t +rte_member_lookup_bulk_ht(const struct rte_member_setsum *setsum, + const void **keys, uint32_t num_keys, + member_set_t *set_ids); + +uint32_t +rte_member_lookup_multi_ht(const struct rte_member_setsum *setsum, + const void *key, uint32_t match_per_key, + member_set_t *set_id); + +uint32_t +rte_member_lookup_multi_bulk_ht(const struct rte_member_setsum *setsum, + const void **keys, uint32_t num_keys, uint32_t match_per_key, + uint32_t *match_count, + member_set_t *set_ids); + +int +rte_member_add_ht(const struct rte_member_setsum *setsum, + const void *key, member_set_t set_id); + +void +rte_member_free_ht(struct rte_member_setsum *setsum); + +int +rte_member_delete_ht(const struct rte_member_setsum *ss, const void *key, + member_set_t set_id); + +void +rte_member_reset_ht(const struct rte_member_setsum *setsum); + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_MEMBER_HT_H_ */ diff --git a/dpdk/lib/librte_member/rte_member_vbf.c b/dpdk/lib/librte_member/rte_member_vbf.c new file mode 100644 index 00000000..1a98ac84 --- /dev/null +++ b/dpdk/lib/librte_member/rte_member_vbf.c @@ -0,0 +1,350 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include + +#include +#include +#include +#include + +#include "rte_member.h" +#include "rte_member_vbf.h" + +/* + * vBF currently implemented as a big array. + * The BFs have a vertical layout. Bits in same location of all bfs will stay + * in the same cache line. + * For example, if we have 32 bloom filters, we use a uint32_t array to + * represent all of them. array[0] represent the first location of all the + * bloom filters, array[1] represents the second location of all the + * bloom filters, etc. The advantage of this layout is to minimize the average + * number of memory accesses to test all bloom filters. + * + * Currently the implementation supports vBF containing 1,2,4,8,16,32 BFs. + */ +int +rte_member_create_vbf(struct rte_member_setsum *ss, + const struct rte_member_parameters *params) +{ + + if (params->num_set > RTE_MEMBER_MAX_BF || + !rte_is_power_of_2(params->num_set) || + params->num_keys == 0 || + params->false_positive_rate == 0 || + params->false_positive_rate > 1) { + rte_errno = EINVAL; + RTE_MEMBER_LOG(ERR, "Membership vBF create with invalid parameters\n"); + return -EINVAL; + } + + /* We assume expected keys evenly distribute to all BFs */ + uint32_t num_keys_per_bf = 1 + (params->num_keys - 1) / ss->num_set; + + /* + * Note that the false positive rate is for all BFs in the vBF + * such that the single BF's false positive rate needs to be + * calculated. + * Assume each BF's False positive rate is fp_one_bf. The total false + * positive rate is fp = 1-(1-fp_one_bf)^n. + * => fp_one_bf = 1 - (1-fp)^(1/n) + */ + + float fp_one_bf = 1 - pow((1 - params->false_positive_rate), + 1.0 / ss->num_set); + + if (fp_one_bf == 0) { + rte_errno = EINVAL; + RTE_MEMBER_LOG(ERR, "Membership BF false positive rate is too small\n"); + return -EINVAL; + } + + uint32_t bits = ceil((num_keys_per_bf * + log(fp_one_bf)) / + log(1.0 / (pow(2.0, log(2.0))))); + + /* We round to power of 2 for performance during lookup */ + ss->bits = rte_align32pow2(bits); + + ss->num_hashes = (uint32_t)(log(2.0) * bits / num_keys_per_bf); + ss->bit_mask = ss->bits - 1; + + /* + * Since we round the bits to power of 2, the final false positive + * rate will probably not be same as the user specified. We log the + * new value as debug message. + */ + float new_fp = pow((1 - pow((1 - 1.0 / ss->bits), num_keys_per_bf * + ss->num_hashes)), ss->num_hashes); + new_fp = 1 - pow((1 - new_fp), ss->num_set); + + /* + * Reduce hash function count, until we approach the user specified + * false-positive rate. Otherwise it is too conservative + */ + int tmp_num_hash = ss->num_hashes; + + while (tmp_num_hash > 1) { + float tmp_fp = new_fp; + + tmp_num_hash--; + new_fp = pow((1 - pow((1 - 1.0 / ss->bits), num_keys_per_bf * + tmp_num_hash)), tmp_num_hash); + new_fp = 1 - pow((1 - new_fp), ss->num_set); + + if (new_fp > params->false_positive_rate) { + new_fp = tmp_fp; + tmp_num_hash++; + break; + } + } + + ss->num_hashes = tmp_num_hash; + + /* + * To avoid multiplication and division: + * mul_shift is used for multiplication shift during bit test + * div_shift is used for division shift, to be divided by number of bits + * represented by a uint32_t variable + */ + ss->mul_shift = __builtin_ctzl(ss->num_set); + ss->div_shift = __builtin_ctzl(32 >> ss->mul_shift); + + RTE_MEMBER_LOG(DEBUG, "vector bloom filter created, " + "each bloom filter expects %u keys, needs %u bits, %u hashes, " + "with false positive rate set as %.5f, " + "The new calculated vBF false positive rate is %.5f\n", + num_keys_per_bf, ss->bits, ss->num_hashes, fp_one_bf, new_fp); + + ss->table = rte_zmalloc_socket(NULL, ss->num_set * (ss->bits >> 3), + RTE_CACHE_LINE_SIZE, ss->socket_id); + if (ss->table == NULL) + return -ENOMEM; + + return 0; +} + +static inline uint32_t +test_bit(uint32_t bit_loc, const struct rte_member_setsum *ss) +{ + uint32_t *vbf = ss->table; + uint32_t n = ss->num_set; + uint32_t div_shift = ss->div_shift; + uint32_t mul_shift = ss->mul_shift; + /* + * a is how many bits in one BF are represented by one 32bit + * variable. + */ + uint32_t a = 32 >> mul_shift; + /* + * x>>b is the divide, x & (a-1) is the mod, & (1<> div_shift] >> + ((bit_loc & (a - 1)) << mul_shift)) & ((1ULL << n) - 1); +} + +static inline void +set_bit(uint32_t bit_loc, const struct rte_member_setsum *ss, int32_t set) +{ + uint32_t *vbf = ss->table; + uint32_t div_shift = ss->div_shift; + uint32_t mul_shift = ss->mul_shift; + uint32_t a = 32 >> mul_shift; + + vbf[bit_loc >> div_shift] |= + 1UL << (((bit_loc & (a - 1)) << mul_shift) + set - 1); +} + +int +rte_member_lookup_vbf(const struct rte_member_setsum *ss, const void *key, + member_set_t *set_id) +{ + uint32_t j; + uint32_t h1 = MEMBER_HASH_FUNC(key, ss->key_len, ss->prim_hash_seed); + uint32_t h2 = MEMBER_HASH_FUNC(&h1, sizeof(uint32_t), + ss->sec_hash_seed); + uint32_t mask = ~0; + uint32_t bit_loc; + + for (j = 0; j < ss->num_hashes; j++) { + bit_loc = (h1 + j * h2) & ss->bit_mask; + mask &= test_bit(bit_loc, ss); + } + + if (mask) { + *set_id = __builtin_ctzl(mask) + 1; + return 1; + } + + *set_id = RTE_MEMBER_NO_MATCH; + return 0; +} + +uint32_t +rte_member_lookup_bulk_vbf(const struct rte_member_setsum *ss, + const void **keys, uint32_t num_keys, member_set_t *set_ids) +{ + uint32_t i, k; + uint32_t num_matches = 0; + uint32_t mask[RTE_MEMBER_LOOKUP_BULK_MAX]; + uint32_t h1[RTE_MEMBER_LOOKUP_BULK_MAX], h2[RTE_MEMBER_LOOKUP_BULK_MAX]; + uint32_t bit_loc; + + for (i = 0; i < num_keys; i++) + h1[i] = MEMBER_HASH_FUNC(keys[i], ss->key_len, + ss->prim_hash_seed); + for (i = 0; i < num_keys; i++) + h2[i] = MEMBER_HASH_FUNC(&h1[i], sizeof(uint32_t), + ss->sec_hash_seed); + for (i = 0; i < num_keys; i++) { + mask[i] = ~0; + for (k = 0; k < ss->num_hashes; k++) { + bit_loc = (h1[i] + k * h2[i]) & ss->bit_mask; + mask[i] &= test_bit(bit_loc, ss); + } + } + for (i = 0; i < num_keys; i++) { + if (mask[i]) { + set_ids[i] = __builtin_ctzl(mask[i]) + 1; + num_matches++; + } else + set_ids[i] = RTE_MEMBER_NO_MATCH; + } + return num_matches; +} + +uint32_t +rte_member_lookup_multi_vbf(const struct rte_member_setsum *ss, + const void *key, uint32_t match_per_key, + member_set_t *set_id) +{ + uint32_t num_matches = 0; + uint32_t j; + uint32_t h1 = MEMBER_HASH_FUNC(key, ss->key_len, ss->prim_hash_seed); + uint32_t h2 = MEMBER_HASH_FUNC(&h1, sizeof(uint32_t), + ss->sec_hash_seed); + uint32_t mask = ~0; + uint32_t bit_loc; + + for (j = 0; j < ss->num_hashes; j++) { + bit_loc = (h1 + j * h2) & ss->bit_mask; + mask &= test_bit(bit_loc, ss); + } + while (mask) { + uint32_t loc = __builtin_ctzl(mask); + set_id[num_matches] = loc + 1; + num_matches++; + if (num_matches >= match_per_key) + return num_matches; + mask &= ~(1UL << loc); + } + return num_matches; +} + +uint32_t +rte_member_lookup_multi_bulk_vbf(const struct rte_member_setsum *ss, + const void **keys, uint32_t num_keys, uint32_t match_per_key, + uint32_t *match_count, + member_set_t *set_ids) +{ + uint32_t i, k; + uint32_t num_matches = 0; + uint32_t match_cnt_t; + uint32_t mask[RTE_MEMBER_LOOKUP_BULK_MAX]; + uint32_t h1[RTE_MEMBER_LOOKUP_BULK_MAX], h2[RTE_MEMBER_LOOKUP_BULK_MAX]; + uint32_t bit_loc; + + for (i = 0; i < num_keys; i++) + h1[i] = MEMBER_HASH_FUNC(keys[i], ss->key_len, + ss->prim_hash_seed); + for (i = 0; i < num_keys; i++) + h2[i] = MEMBER_HASH_FUNC(&h1[i], sizeof(uint32_t), + ss->sec_hash_seed); + for (i = 0; i < num_keys; i++) { + mask[i] = ~0; + for (k = 0; k < ss->num_hashes; k++) { + bit_loc = (h1[i] + k * h2[i]) & ss->bit_mask; + mask[i] &= test_bit(bit_loc, ss); + } + } + for (i = 0; i < num_keys; i++) { + match_cnt_t = 0; + while (mask[i]) { + uint32_t loc = __builtin_ctzl(mask[i]); + set_ids[i * match_per_key + match_cnt_t] = loc + 1; + match_cnt_t++; + if (match_cnt_t >= match_per_key) + break; + mask[i] &= ~(1UL << loc); + } + match_count[i] = match_cnt_t; + if (match_cnt_t != 0) + num_matches++; + } + return num_matches; +} + +int +rte_member_add_vbf(const struct rte_member_setsum *ss, + const void *key, member_set_t set_id) +{ + uint32_t i, h1, h2; + uint32_t bit_loc; + + if (set_id > ss->num_set || set_id == RTE_MEMBER_NO_MATCH) + return -EINVAL; + + h1 = MEMBER_HASH_FUNC(key, ss->key_len, ss->prim_hash_seed); + h2 = MEMBER_HASH_FUNC(&h1, sizeof(uint32_t), ss->sec_hash_seed); + + for (i = 0; i < ss->num_hashes; i++) { + bit_loc = (h1 + i * h2) & ss->bit_mask; + set_bit(bit_loc, ss, set_id); + } + return 0; +} + +void +rte_member_free_vbf(struct rte_member_setsum *ss) +{ + rte_free(ss->table); +} + +void +rte_member_reset_vbf(const struct rte_member_setsum *ss) +{ + uint32_t *vbf = ss->table; + memset(vbf, 0, (ss->num_set * ss->bits) >> 3); +} diff --git a/dpdk/lib/librte_member/rte_member_vbf.h b/dpdk/lib/librte_member/rte_member_vbf.h new file mode 100644 index 00000000..5bc158b9 --- /dev/null +++ b/dpdk/lib/librte_member/rte_member_vbf.h @@ -0,0 +1,82 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_MEMBER_VBF_H_ +#define _RTE_MEMBER_VBF_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +/* Currently we only support up to 32 sets in vBF */ +#define RTE_MEMBER_MAX_BF 32 + +int +rte_member_create_vbf(struct rte_member_setsum *ss, + const struct rte_member_parameters *params); + +int +rte_member_lookup_vbf(const struct rte_member_setsum *setsum, + const void *key, member_set_t *set_id); + +uint32_t +rte_member_lookup_bulk_vbf(const struct rte_member_setsum *setsum, + const void **keys, uint32_t num_keys, + member_set_t *set_ids); + +uint32_t +rte_member_lookup_multi_vbf(const struct rte_member_setsum *setsum, + const void *key, uint32_t match_per_key, + member_set_t *set_id); + +uint32_t +rte_member_lookup_multi_bulk_vbf(const struct rte_member_setsum *setsum, + const void **keys, uint32_t num_keys, uint32_t match_per_key, + uint32_t *match_count, + member_set_t *set_ids); + +int +rte_member_add_vbf(const struct rte_member_setsum *setsum, + const void *key, member_set_t set_id); + +void +rte_member_free_vbf(struct rte_member_setsum *ss); + +void +rte_member_reset_vbf(const struct rte_member_setsum *setsum); + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_MEMBER_VBF_H_ */ diff --git a/dpdk/lib/librte_member/rte_member_version.map b/dpdk/lib/librte_member/rte_member_version.map new file mode 100644 index 00000000..019e4cd9 --- /dev/null +++ b/dpdk/lib/librte_member/rte_member_version.map @@ -0,0 +1,16 @@ +DPDK_17.11 { + global: + + rte_member_add; + rte_member_create; + rte_member_delete; + rte_member_find_existing; + rte_member_free; + rte_member_lookup; + rte_member_lookup_bulk; + rte_member_lookup_multi; + rte_member_lookup_multi_bulk; + rte_member_reset; + + local: *; +}; diff --git a/dpdk/lib/librte_member/rte_member_x86.h b/dpdk/lib/librte_member/rte_member_x86.h new file mode 100644 index 00000000..d29dd3fe --- /dev/null +++ b/dpdk/lib/librte_member/rte_member_x86.h @@ -0,0 +1,107 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_MEMBER_X86_H_ +#define _RTE_MEMBER_X86_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +#if defined(RTE_MACHINE_CPUFLAG_AVX2) + +static inline int +update_entry_search_avx(uint32_t bucket_id, member_sig_t tmp_sig, + struct member_ht_bucket *buckets, + member_set_t set_id) +{ + uint32_t hitmask = _mm256_movemask_epi8((__m256i)_mm256_cmpeq_epi16( + _mm256_load_si256((__m256i const *)buckets[bucket_id].sigs), + _mm256_set1_epi16(tmp_sig))); + if (hitmask) { + uint32_t hit_idx = __builtin_ctzl(hitmask) >> 1; + buckets[bucket_id].sets[hit_idx] = set_id; + return 1; + } + return 0; +} + +static inline int +search_bucket_single_avx(uint32_t bucket_id, member_sig_t tmp_sig, + struct member_ht_bucket *buckets, + member_set_t *set_id) +{ + uint32_t hitmask = _mm256_movemask_epi8((__m256i)_mm256_cmpeq_epi16( + _mm256_load_si256((__m256i const *)buckets[bucket_id].sigs), + _mm256_set1_epi16(tmp_sig))); + while (hitmask) { + uint32_t hit_idx = __builtin_ctzl(hitmask) >> 1; + if (buckets[bucket_id].sets[hit_idx] != RTE_MEMBER_NO_MATCH) { + *set_id = buckets[bucket_id].sets[hit_idx]; + return 1; + } + hitmask &= ~(3U << ((hit_idx) << 1)); + } + return 0; +} + +static inline void +search_bucket_multi_avx(uint32_t bucket_id, member_sig_t tmp_sig, + struct member_ht_bucket *buckets, + uint32_t *counter, + uint32_t match_per_key, + member_set_t *set_id) +{ + uint32_t hitmask = _mm256_movemask_epi8((__m256i)_mm256_cmpeq_epi16( + _mm256_load_si256((__m256i const *)buckets[bucket_id].sigs), + _mm256_set1_epi16(tmp_sig))); + while (hitmask) { + uint32_t hit_idx = __builtin_ctzl(hitmask) >> 1; + if (buckets[bucket_id].sets[hit_idx] != RTE_MEMBER_NO_MATCH) { + set_id[*counter] = buckets[bucket_id].sets[hit_idx]; + (*counter)++; + if (*counter >= match_per_key) + return; + } + hitmask &= ~(3U << ((hit_idx) << 1)); + } +} +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_MEMBER_X86_H_ */ diff --git a/dpdk/lib/librte_mempool/Makefile b/dpdk/lib/librte_mempool/Makefile index 057a6ab4..46654e32 100644 --- a/dpdk/lib/librte_mempool/Makefile +++ b/dpdk/lib/librte_mempool/Makefile @@ -35,19 +35,16 @@ include $(RTE_SDK)/mk/rte.vars.mk LIB = librte_mempool.a CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -O3 +LDLIBS += -lrte_eal -lrte_ring EXPORT_MAP := rte_mempool_version.map -LIBABIVER := 2 +LIBABIVER := 3 # all source are stored in SRCS-y SRCS-$(CONFIG_RTE_LIBRTE_MEMPOOL) += rte_mempool.c SRCS-$(CONFIG_RTE_LIBRTE_MEMPOOL) += rte_mempool_ops.c -SRCS-$(CONFIG_RTE_LIBRTE_MEMPOOL) += rte_mempool_ring.c -SRCS-$(CONFIG_RTE_LIBRTE_MEMPOOL) += rte_mempool_stack.c # install includes SYMLINK-$(CONFIG_RTE_LIBRTE_MEMPOOL)-include := rte_mempool.h -DEPDIRS-$(CONFIG_RTE_LIBRTE_MEMPOOL) += lib/librte_eal lib/librte_ring - include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/lib/librte_mempool/rte_mempool.c b/dpdk/lib/librte_mempool/rte_mempool.c index ad7c470e..5bd74eac 100644 --- a/dpdk/lib/librte_mempool/rte_mempool.c +++ b/dpdk/lib/librte_mempool/rte_mempool.c @@ -55,7 +55,6 @@ #include #include #include -#include #include #include #include @@ -129,7 +128,7 @@ static unsigned optimize_object_size(unsigned obj_size) } static void -mempool_add_elem(struct rte_mempool *mp, void *obj, phys_addr_t physaddr) +mempool_add_elem(struct rte_mempool *mp, void *obj, rte_iova_t iova) { struct rte_mempool_objhdr *hdr; struct rte_mempool_objtlr *tlr __rte_unused; @@ -137,7 +136,7 @@ mempool_add_elem(struct rte_mempool *mp, void *obj, phys_addr_t physaddr) /* set mempool ptr in header */ hdr = RTE_PTR_SUB(obj, sizeof(*hdr)); hdr->mp = mp; - hdr->physaddr = physaddr; + hdr->iova = iova; STAILQ_INSERT_TAIL(&mp->elt_list, hdr, next); mp->populated_size++; @@ -239,9 +238,16 @@ rte_mempool_calc_obj_size(uint32_t elt_size, uint32_t flags, * Calculate maximum amount of memory required to store given number of objects. */ size_t -rte_mempool_xmem_size(uint32_t elt_num, size_t total_elt_sz, uint32_t pg_shift) +rte_mempool_xmem_size(uint32_t elt_num, size_t total_elt_sz, uint32_t pg_shift, + unsigned int flags) { size_t obj_per_page, pg_num, pg_sz; + unsigned int mask; + + mask = MEMPOOL_F_CAPA_BLK_ALIGNED_OBJECTS | MEMPOOL_F_CAPA_PHYS_CONTIG; + if ((flags & mask) == mask) + /* alignment need one additional object */ + elt_num += 1; if (total_elt_sz == 0) return 0; @@ -264,23 +270,29 @@ rte_mempool_xmem_size(uint32_t elt_num, size_t total_elt_sz, uint32_t pg_shift) */ ssize_t rte_mempool_xmem_usage(__rte_unused void *vaddr, uint32_t elt_num, - size_t total_elt_sz, const phys_addr_t paddr[], uint32_t pg_num, - uint32_t pg_shift) + size_t total_elt_sz, const rte_iova_t iova[], uint32_t pg_num, + uint32_t pg_shift, unsigned int flags) { uint32_t elt_cnt = 0; - phys_addr_t start, end; - uint32_t paddr_idx; + rte_iova_t start, end; + uint32_t iova_idx; size_t pg_sz = (size_t)1 << pg_shift; + unsigned int mask; - /* if paddr is NULL, assume contiguous memory */ - if (paddr == NULL) { + mask = MEMPOOL_F_CAPA_BLK_ALIGNED_OBJECTS | MEMPOOL_F_CAPA_PHYS_CONTIG; + if ((flags & mask) == mask) + /* alignment need one additional object */ + elt_num += 1; + + /* if iova is NULL, assume contiguous memory */ + if (iova == NULL) { start = 0; end = pg_sz * pg_num; - paddr_idx = pg_num; + iova_idx = pg_num; } else { - start = paddr[0]; - end = paddr[0] + pg_sz; - paddr_idx = 1; + start = iova[0]; + end = iova[0] + pg_sz; + iova_idx = 1; } while (elt_cnt < elt_num) { @@ -288,15 +300,15 @@ rte_mempool_xmem_usage(__rte_unused void *vaddr, uint32_t elt_num, /* enough contiguous memory, add an object */ start += total_elt_sz; elt_cnt++; - } else if (paddr_idx < pg_num) { + } else if (iova_idx < pg_num) { /* no room to store one obj, add a page */ - if (end == paddr[paddr_idx]) { + if (end == iova[iova_idx]) { end += pg_sz; } else { - start = paddr[paddr_idx]; - end = paddr[paddr_idx] + pg_sz; + start = iova[iova_idx]; + end = iova[iova_idx] + pg_sz; } - paddr_idx++; + iova_idx++; } else { /* no more page, return how many elements fit */ @@ -304,7 +316,7 @@ rte_mempool_xmem_usage(__rte_unused void *vaddr, uint32_t elt_num, } } - return (size_t)paddr_idx << pg_shift; + return (size_t)iova_idx << pg_shift; } /* free a memchunk allocated with rte_memzone_reserve() */ @@ -345,11 +357,12 @@ rte_mempool_free_memchunks(struct rte_mempool *mp) * on error. */ int -rte_mempool_populate_phys(struct rte_mempool *mp, char *vaddr, - phys_addr_t paddr, size_t len, rte_mempool_memchunk_free_cb_t *free_cb, +rte_mempool_populate_iova(struct rte_mempool *mp, char *vaddr, + rte_iova_t iova, size_t len, rte_mempool_memchunk_free_cb_t *free_cb, void *opaque) { unsigned total_elt_sz; + unsigned int mp_capa_flags; unsigned i = 0; size_t off; struct rte_mempool_memhdr *memhdr; @@ -363,35 +376,62 @@ rte_mempool_populate_phys(struct rte_mempool *mp, char *vaddr, mp->flags |= MEMPOOL_F_POOL_CREATED; } + /* Notify memory area to mempool */ + ret = rte_mempool_ops_register_memory_area(mp, vaddr, iova, len); + if (ret != -ENOTSUP && ret < 0) + return ret; + /* mempool is already populated */ if (mp->populated_size >= mp->size) return -ENOSPC; total_elt_sz = mp->header_size + mp->elt_size + mp->trailer_size; + /* Get mempool capabilities */ + mp_capa_flags = 0; + ret = rte_mempool_ops_get_capabilities(mp, &mp_capa_flags); + if ((ret < 0) && (ret != -ENOTSUP)) + return ret; + + /* update mempool capabilities */ + mp->flags |= mp_capa_flags; + + /* Detect pool area has sufficient space for elements */ + if (mp_capa_flags & MEMPOOL_F_CAPA_PHYS_CONTIG) { + if (len < total_elt_sz * mp->size) { + RTE_LOG(ERR, MEMPOOL, + "pool area %" PRIx64 " not enough\n", + (uint64_t)len); + return -ENOSPC; + } + } + memhdr = rte_zmalloc("MEMPOOL_MEMHDR", sizeof(*memhdr), 0); if (memhdr == NULL) return -ENOMEM; memhdr->mp = mp; memhdr->addr = vaddr; - memhdr->phys_addr = paddr; + memhdr->iova = iova; memhdr->len = len; memhdr->free_cb = free_cb; memhdr->opaque = opaque; - if (mp->flags & MEMPOOL_F_NO_CACHE_ALIGN) + if (mp_capa_flags & MEMPOOL_F_CAPA_BLK_ALIGNED_OBJECTS) + /* align object start address to a multiple of total_elt_sz */ + off = total_elt_sz - ((uintptr_t)vaddr % total_elt_sz); + else if (mp->flags & MEMPOOL_F_NO_CACHE_ALIGN) off = RTE_PTR_ALIGN_CEIL(vaddr, 8) - vaddr; else off = RTE_PTR_ALIGN_CEIL(vaddr, RTE_CACHE_LINE_SIZE) - vaddr; while (off + total_elt_sz <= len && mp->populated_size < mp->size) { off += mp->header_size; - if (paddr == RTE_BAD_PHYS_ADDR) + if (iova == RTE_BAD_IOVA) mempool_add_elem(mp, (char *)vaddr + off, - RTE_BAD_PHYS_ADDR); + RTE_BAD_IOVA); else - mempool_add_elem(mp, (char *)vaddr + off, paddr + off); + mempool_add_elem(mp, (char *)vaddr + off, iova + off); off += mp->elt_size + mp->trailer_size; i++; } @@ -405,12 +445,20 @@ rte_mempool_populate_phys(struct rte_mempool *mp, char *vaddr, return i; } +int +rte_mempool_populate_phys(struct rte_mempool *mp, char *vaddr, + phys_addr_t paddr, size_t len, rte_mempool_memchunk_free_cb_t *free_cb, + void *opaque) +{ + return rte_mempool_populate_iova(mp, vaddr, paddr, len, free_cb, opaque); +} + /* Add objects in the pool, using a table of physical pages. Return the * number of objects added, or a negative value on error. */ int -rte_mempool_populate_phys_tab(struct rte_mempool *mp, char *vaddr, - const phys_addr_t paddr[], uint32_t pg_num, uint32_t pg_shift, +rte_mempool_populate_iova_tab(struct rte_mempool *mp, char *vaddr, + const rte_iova_t iova[], uint32_t pg_num, uint32_t pg_shift, rte_mempool_memchunk_free_cb_t *free_cb, void *opaque) { uint32_t i, n; @@ -422,18 +470,18 @@ rte_mempool_populate_phys_tab(struct rte_mempool *mp, char *vaddr, return -EEXIST; if (mp->flags & MEMPOOL_F_NO_PHYS_CONTIG) - return rte_mempool_populate_phys(mp, vaddr, RTE_BAD_PHYS_ADDR, + return rte_mempool_populate_iova(mp, vaddr, RTE_BAD_IOVA, pg_num * pg_sz, free_cb, opaque); for (i = 0; i < pg_num && mp->populated_size < mp->size; i += n) { /* populate with the largest group of contiguous pages */ for (n = 1; (i + n) < pg_num && - paddr[i + n - 1] + pg_sz == paddr[i + n]; n++) + iova[i + n - 1] + pg_sz == iova[i + n]; n++) ; - ret = rte_mempool_populate_phys(mp, vaddr + i * pg_sz, - paddr[i], n * pg_sz, free_cb, opaque); + ret = rte_mempool_populate_iova(mp, vaddr + i * pg_sz, + iova[i], n * pg_sz, free_cb, opaque); if (ret < 0) { rte_mempool_free_memchunks(mp); return ret; @@ -445,6 +493,15 @@ rte_mempool_populate_phys_tab(struct rte_mempool *mp, char *vaddr, return cnt; } +int +rte_mempool_populate_phys_tab(struct rte_mempool *mp, char *vaddr, + const phys_addr_t paddr[], uint32_t pg_num, uint32_t pg_shift, + rte_mempool_memchunk_free_cb_t *free_cb, void *opaque) +{ + return rte_mempool_populate_iova_tab(mp, vaddr, paddr, pg_num, pg_shift, + free_cb, opaque); +} + /* Populate the mempool with a virtual area. Return the number of * objects added, or a negative value on error. */ @@ -453,7 +510,7 @@ rte_mempool_populate_virt(struct rte_mempool *mp, char *addr, size_t len, size_t pg_sz, rte_mempool_memchunk_free_cb_t *free_cb, void *opaque) { - phys_addr_t paddr; + rte_iova_t iova; size_t off, phys_len; int ret, cnt = 0; @@ -467,33 +524,30 @@ rte_mempool_populate_virt(struct rte_mempool *mp, char *addr, return -EINVAL; if (mp->flags & MEMPOOL_F_NO_PHYS_CONTIG) - return rte_mempool_populate_phys(mp, addr, RTE_BAD_PHYS_ADDR, + return rte_mempool_populate_iova(mp, addr, RTE_BAD_IOVA, len, free_cb, opaque); for (off = 0; off + pg_sz <= len && mp->populated_size < mp->size; off += phys_len) { - paddr = rte_mem_virt2phy(addr + off); - /* required for xen_dom0 to get the machine address */ - paddr = rte_mem_phy2mch(-1, paddr); + iova = rte_mem_virt2iova(addr + off); - if (paddr == RTE_BAD_PHYS_ADDR) { + if (iova == RTE_BAD_IOVA && rte_eal_has_hugepages()) { ret = -EINVAL; goto fail; } /* populate with the largest group of contiguous pages */ for (phys_len = pg_sz; off + phys_len < len; phys_len += pg_sz) { - phys_addr_t paddr_tmp; + rte_iova_t iova_tmp; - paddr_tmp = rte_mem_virt2phy(addr + off + phys_len); - paddr_tmp = rte_mem_phy2mch(-1, paddr_tmp); + iova_tmp = rte_mem_virt2iova(addr + off + phys_len); - if (paddr_tmp != paddr + phys_len) + if (iova_tmp != iova + phys_len) break; } - ret = rte_mempool_populate_phys(mp, addr + off, paddr, + ret = rte_mempool_populate_iova(mp, addr + off, iova, phys_len, free_cb, opaque); if (ret < 0) goto fail; @@ -516,23 +570,29 @@ rte_mempool_populate_virt(struct rte_mempool *mp, char *addr, int rte_mempool_populate_default(struct rte_mempool *mp) { - int mz_flags = RTE_MEMZONE_1GB|RTE_MEMZONE_SIZE_HINT_ONLY; + unsigned int mz_flags = RTE_MEMZONE_1GB|RTE_MEMZONE_SIZE_HINT_ONLY; char mz_name[RTE_MEMZONE_NAMESIZE]; const struct rte_memzone *mz; size_t size, total_elt_sz, align, pg_sz, pg_shift; - phys_addr_t paddr; + rte_iova_t iova; unsigned mz_id, n; + unsigned int mp_flags; int ret; /* mempool must not be populated */ if (mp->nb_mem_chunks != 0) return -EEXIST; - if (rte_xen_dom0_supported()) { - pg_sz = RTE_PGSIZE_2M; - pg_shift = rte_bsf32(pg_sz); - align = pg_sz; - } else if (rte_eal_has_hugepages()) { + /* Get mempool capabilities */ + mp_flags = 0; + ret = rte_mempool_ops_get_capabilities(mp, &mp_flags); + if ((ret < 0) && (ret != -ENOTSUP)) + return ret; + + /* update mempool capabilities */ + mp->flags |= mp_flags; + + if (rte_eal_has_hugepages()) { pg_shift = 0; /* not needed, zone is physically contiguous */ pg_sz = 0; align = RTE_CACHE_LINE_SIZE; @@ -544,7 +604,8 @@ rte_mempool_populate_default(struct rte_mempool *mp) total_elt_sz = mp->header_size + mp->elt_size + mp->trailer_size; for (mz_id = 0, n = mp->size; n > 0; mz_id++, n -= ret) { - size = rte_mempool_xmem_size(n, total_elt_sz, pg_shift); + size = rte_mempool_xmem_size(n, total_elt_sz, pg_shift, + mp->flags); ret = snprintf(mz_name, sizeof(mz_name), RTE_MEMPOOL_MZ_FORMAT "_%d", mp->name, mz_id); @@ -565,13 +626,13 @@ rte_mempool_populate_default(struct rte_mempool *mp) } if (mp->flags & MEMPOOL_F_NO_PHYS_CONTIG) - paddr = RTE_BAD_PHYS_ADDR; + iova = RTE_BAD_IOVA; else - paddr = mz->phys_addr; + iova = mz->iova; - if (rte_eal_has_hugepages() && !rte_xen_dom0_supported()) - ret = rte_mempool_populate_phys(mp, mz->addr, - paddr, mz->len, + if (rte_eal_has_hugepages()) + ret = rte_mempool_populate_iova(mp, mz->addr, + iova, mz->len, rte_mempool_memchunk_mz_free, (void *)(uintptr_t)mz); else @@ -601,7 +662,8 @@ get_anon_size(const struct rte_mempool *mp) pg_sz = getpagesize(); pg_shift = rte_bsf32(pg_sz); total_elt_sz = mp->header_size + mp->elt_size + mp->trailer_size; - size = rte_mempool_xmem_size(mp->size, total_elt_sz, pg_shift); + size = rte_mempool_xmem_size(mp->size, total_elt_sz, pg_shift, + mp->flags); return size; } @@ -743,7 +805,7 @@ rte_mempool_create_empty(const char *name, unsigned n, unsigned elt_size, struct rte_tailq_entry *te = NULL; const struct rte_memzone *mz = NULL; size_t mempool_size; - int mz_flags = RTE_MEMZONE_1GB|RTE_MEMZONE_SIZE_HINT_ONLY; + unsigned int mz_flags = RTE_MEMZONE_1GB|RTE_MEMZONE_SIZE_HINT_ONLY; struct rte_mempool_objsz objsz; unsigned lcore_id; int ret; @@ -819,7 +881,6 @@ rte_mempool_create_empty(const char *name, unsigned n, unsigned elt_size, goto exit_unlock; } mp->mz = mz; - mp->socket_id = socket_id; mp->size = n; mp->flags = flags; mp->socket_id = socket_id; @@ -870,6 +931,7 @@ rte_mempool_create(const char *name, unsigned n, unsigned elt_size, rte_mempool_obj_cb_t *obj_init, void *obj_init_arg, int socket_id, unsigned flags) { + int ret; struct rte_mempool *mp; mp = rte_mempool_create_empty(name, n, elt_size, cache_size, @@ -882,13 +944,16 @@ rte_mempool_create(const char *name, unsigned n, unsigned elt_size, * set the correct index into the table of ops structs. */ if ((flags & MEMPOOL_F_SP_PUT) && (flags & MEMPOOL_F_SC_GET)) - rte_mempool_set_ops_byname(mp, "ring_sp_sc", NULL); + ret = rte_mempool_set_ops_byname(mp, "ring_sp_sc", NULL); else if (flags & MEMPOOL_F_SP_PUT) - rte_mempool_set_ops_byname(mp, "ring_sp_mc", NULL); + ret = rte_mempool_set_ops_byname(mp, "ring_sp_mc", NULL); else if (flags & MEMPOOL_F_SC_GET) - rte_mempool_set_ops_byname(mp, "ring_mp_sc", NULL); + ret = rte_mempool_set_ops_byname(mp, "ring_mp_sc", NULL); else - rte_mempool_set_ops_byname(mp, "ring_mp_mc", NULL); + ret = rte_mempool_set_ops_byname(mp, "ring_mp_mc", NULL); + + if (ret) + goto fail; /* call the mempool priv initializer */ if (mp_init) @@ -911,9 +976,8 @@ rte_mempool_create(const char *name, unsigned n, unsigned elt_size, /* * Create the mempool over already allocated chunk of memory. * That external memory buffer can consists of physically disjoint pages. - * Setting vaddr to NULL, makes mempool to fallback to original behaviour - * and allocate space for mempool and it's elements as one big chunk of - * physically continuos memory. + * Setting vaddr to NULL, makes mempool to fallback to rte_mempool_create() + * behavior. */ struct rte_mempool * rte_mempool_xmem_create(const char *name, unsigned n, unsigned elt_size, @@ -921,7 +985,7 @@ rte_mempool_xmem_create(const char *name, unsigned n, unsigned elt_size, rte_mempool_ctor_t *mp_init, void *mp_init_arg, rte_mempool_obj_cb_t *obj_init, void *obj_init_arg, int socket_id, unsigned flags, void *vaddr, - const phys_addr_t paddr[], uint32_t pg_num, uint32_t pg_shift) + const rte_iova_t iova[], uint32_t pg_num, uint32_t pg_shift) { struct rte_mempool *mp = NULL; int ret; @@ -933,7 +997,7 @@ rte_mempool_xmem_create(const char *name, unsigned n, unsigned elt_size, obj_init, obj_init_arg, socket_id, flags); /* check that we have both VA and PA */ - if (paddr == NULL) { + if (iova == NULL) { rte_errno = EINVAL; return NULL; } @@ -953,7 +1017,7 @@ rte_mempool_xmem_create(const char *name, unsigned n, unsigned elt_size, if (mp_init) mp_init(mp, mp_init_arg); - ret = rte_mempool_populate_phys_tab(mp, vaddr, paddr, pg_num, pg_shift, + ret = rte_mempool_populate_iova_tab(mp, vaddr, iova, pg_num, pg_shift, NULL, NULL); if (ret < 0 || ret != (int)mp->size) goto fail; @@ -1000,12 +1064,6 @@ rte_mempool_in_use_count(const struct rte_mempool *mp) return mp->size - rte_mempool_avail_count(mp); } -unsigned int -rte_mempool_count(const struct rte_mempool *mp) -{ - return rte_mempool_avail_count(mp); -} - /* dump the cache status */ static unsigned rte_mempool_dump_cache(FILE *f, const struct rte_mempool *mp) @@ -1049,7 +1107,7 @@ void rte_mempool_check_cookies(const struct rte_mempool *mp, /* Force to drop the "const" attribute. This is done only when * DEBUG is enabled */ tmp = (void *) obj_table_const; - obj_table = (void **) tmp; + obj_table = tmp; while (n--) { obj = obj_table[n]; @@ -1182,7 +1240,7 @@ rte_mempool_dump(FILE *f, struct rte_mempool *mp) fprintf(f, "mempool <%s>@%p\n", mp->name, mp); fprintf(f, " flags=%x\n", mp->flags); fprintf(f, " pool=%p\n", mp->pool_data); - fprintf(f, " phys_addr=0x%" PRIx64 "\n", mp->mz->phys_addr); + fprintf(f, " iova=0x%" PRIx64 "\n", mp->mz->iova); fprintf(f, " nb_mem_chunks=%u\n", mp->nb_mem_chunks); fprintf(f, " size=%"PRIu32"\n", mp->size); fprintf(f, " populated_size=%"PRIu32"\n", mp->populated_size); diff --git a/dpdk/lib/librte_mempool/rte_mempool.h b/dpdk/lib/librte_mempool/rte_mempool.h index 059ad9e5..e21026aa 100644 --- a/dpdk/lib/librte_mempool/rte_mempool.h +++ b/dpdk/lib/librte_mempool/rte_mempool.h @@ -51,13 +51,15 @@ * meta-data in the object data and retrieve them when allocating a * new object. * - * Note: the mempool implementation is not preemptable. A lcore must - * not be interrupted by another task that uses the same mempool - * (because it uses a ring which is not preemptable). Also, mempool - * functions must not be used outside the DPDK environment: for - * example, in linuxapp environment, a thread that is not created by - * the EAL must not use mempools. This is due to the per-lcore cache - * that won't work as rte_lcore_id() will not return a correct value. + * Note: the mempool implementation is not preemptible. An lcore must not be + * interrupted by another task that uses the same mempool (because it uses a + * ring which is not preemptible). Also, usual mempool functions like + * rte_mempool_get() or rte_mempool_put() are designed to be called from an EAL + * thread due to the internal per-lcore cache. Due to the lack of caching, + * rte_mempool_get() or rte_mempool_put() performance will suffer when called + * by non-EAL threads. Instead, non-EAL threads should call + * rte_mempool_generic_get() or rte_mempool_generic_put() with a user cache + * created with rte_mempool_cache_create(). */ #include @@ -67,6 +69,7 @@ #include #include +#include #include #include #include @@ -75,6 +78,7 @@ #include #include #include +#include #ifdef __cplusplus extern "C" { @@ -154,7 +158,11 @@ struct rte_mempool_objsz { struct rte_mempool_objhdr { STAILQ_ENTRY(rte_mempool_objhdr) next; /**< Next in list. */ struct rte_mempool *mp; /**< The mempool owning the object. */ - phys_addr_t physaddr; /**< Physical address of the object. */ + RTE_STD_C11 + union { + rte_iova_t iova; /**< IO address of the object. */ + phys_addr_t physaddr; /**< deprecated - Physical address of the object. */ + }; #ifdef RTE_LIBRTE_MEMPOOL_DEBUG uint64_t cookie; /**< Debug cookie. */ #endif @@ -200,7 +208,11 @@ struct rte_mempool_memhdr { STAILQ_ENTRY(rte_mempool_memhdr) next; /**< Next in list. */ struct rte_mempool *mp; /**< The mempool owning the chunk */ void *addr; /**< Virtual address of the chunk */ - phys_addr_t phys_addr; /**< Physical address of the chunk */ + RTE_STD_C11 + union { + rte_iova_t iova; /**< IO address of the chunk */ + phys_addr_t phys_addr; /**< Physical address of the chunk */ + }; size_t len; /**< length of the chunk */ rte_mempool_memchunk_free_cb_t *free_cb; /**< Free callback */ void *opaque; /**< Argument passed to the free callback */ @@ -216,13 +228,14 @@ struct rte_mempool { * RTE_MEMPOOL_NAMESIZE next time the ABI changes */ char name[RTE_MEMZONE_NAMESIZE]; /**< Name of mempool. */ + RTE_STD_C11 union { void *pool_data; /**< Ring or pool to store objects. */ uint64_t pool_id; /**< External mempool identifier. */ }; void *pool_config; /**< optional args for ops alloc. */ const struct rte_memzone *mz; /**< Memzone where pool is alloc'd. */ - int flags; /**< Flags of the mempool. */ + unsigned int flags; /**< Flags of the mempool. */ int socket_id; /**< Socket id passed at create. */ uint32_t size; /**< Max size of the mempool. */ uint32_t cache_size; @@ -261,6 +274,24 @@ struct rte_mempool { #define MEMPOOL_F_SC_GET 0x0008 /**< Default get is "single-consumer".*/ #define MEMPOOL_F_POOL_CREATED 0x0010 /**< Internal: pool is created. */ #define MEMPOOL_F_NO_PHYS_CONTIG 0x0020 /**< Don't need physically contiguous objs. */ +/** + * This capability flag is advertised by a mempool handler, if the whole + * memory area containing the objects must be physically contiguous. + * Note: This flag should not be passed by application. + */ +#define MEMPOOL_F_CAPA_PHYS_CONTIG 0x0040 +/** + * This capability flag is advertised by a mempool handler. Used for a case + * where mempool driver wants object start address(vaddr) aligned to block + * size(/ total element size). + * + * Note: + * - This flag should not be passed by application. + * Flag used for mempool driver only. + * - Mempool driver must also set MEMPOOL_F_CAPA_PHYS_CONTIG flag along with + * MEMPOOL_F_CAPA_BLK_ALIGNED_OBJECTS. + */ +#define MEMPOOL_F_CAPA_BLK_ALIGNED_OBJECTS 0x0080 /** * @internal When debug is enabled, store some statistics. @@ -355,7 +386,7 @@ void rte_mempool_check_cookies(const struct rte_mempool *mp, * Prototype for implementation specific data provisioning function. * * The function should provide the implementation specific memory for - * for use by the other mempool ops functions in a given mempool ops struct. + * use by the other mempool ops functions in a given mempool ops struct. * E.g. the default ops provides an instance of the rte_ring for this purpose. * it will most likely point to a different type of data structure, and * will be transparent to the application programmer. @@ -385,6 +416,18 @@ typedef int (*rte_mempool_dequeue_t)(struct rte_mempool *mp, */ typedef unsigned (*rte_mempool_get_count)(const struct rte_mempool *mp); +/** + * Get the mempool capabilities. + */ +typedef int (*rte_mempool_get_capabilities_t)(const struct rte_mempool *mp, + unsigned int *flags); + +/** + * Notify new memory area to mempool. + */ +typedef int (*rte_mempool_ops_register_memory_area_t) +(const struct rte_mempool *mp, char *vaddr, rte_iova_t iova, size_t len); + /** Structure defining mempool operations structure */ struct rte_mempool_ops { char name[RTE_MEMPOOL_OPS_NAMESIZE]; /**< Name of mempool ops struct. */ @@ -393,6 +436,14 @@ struct rte_mempool_ops { rte_mempool_enqueue_t enqueue; /**< Enqueue an object. */ rte_mempool_dequeue_t dequeue; /**< Dequeue an object. */ rte_mempool_get_count get_count; /**< Get qty of available objs. */ + /** + * Get the mempool capabilities + */ + rte_mempool_get_capabilities_t get_capabilities; + /** + * Notify new memory area to mempool + */ + rte_mempool_ops_register_memory_area_t register_memory_area; } __rte_cache_aligned; #define RTE_MEMPOOL_MAX_OPS_IDX 16 /**< Max registered ops structs */ @@ -504,6 +555,43 @@ rte_mempool_ops_enqueue_bulk(struct rte_mempool *mp, void * const *obj_table, unsigned rte_mempool_ops_get_count(const struct rte_mempool *mp); +/** + * @internal wrapper for mempool_ops get_capabilities callback. + * + * @param mp [in] + * Pointer to the memory pool. + * @param flags [out] + * Pointer to the mempool flags. + * @return + * - 0: Success; The mempool driver has advertised his pool capabilities in + * flags param. + * - -ENOTSUP - doesn't support get_capabilities ops (valid case). + * - Otherwise, pool create fails. + */ +int +rte_mempool_ops_get_capabilities(const struct rte_mempool *mp, + unsigned int *flags); +/** + * @internal wrapper for mempool_ops register_memory_area callback. + * API to notify the mempool handler when a new memory area is added to pool. + * + * @param mp + * Pointer to the memory pool. + * @param vaddr + * Pointer to the buffer virtual address. + * @param iova + * Pointer to the buffer IO address. + * @param len + * Pool size. + * @return + * - 0: Success; + * - -ENOTSUP - doesn't support register_memory_area ops (valid error case). + * - Otherwise, rte_mempool_populate_phys fails thus pool create fails. + */ +int +rte_mempool_ops_register_memory_area(const struct rte_mempool *mp, + char *vaddr, rte_iova_t iova, size_t len); + /** * @internal wrapper for mempool_ops free callback. * @@ -549,7 +637,7 @@ int rte_mempool_register_ops(const struct rte_mempool_ops *ops); /** * Macro to statically register the ops of a mempool handler. * Note that the rte_mempool_register_ops fails silently here when - * more then RTE_MEMPOOL_MAX_OPS_IDX is registered. + * more than RTE_MEMPOOL_MAX_OPS_IDX is registered. */ #define MEMPOOL_REGISTER_OPS(ops) \ void mp_hdlr_init_##ops(void); \ @@ -587,10 +675,8 @@ typedef void (rte_mempool_ctor_t)(struct rte_mempool *, void *); /** * Create a new mempool named *name* in memory. * - * This function uses ``memzone_reserve()`` to allocate memory. The + * This function uses ``rte_memzone_reserve()`` to allocate memory. The * pool contains n elements of elt_size. Its size is set to n. - * All elements of the mempool are allocated together with the mempool header, - * in one physically continuous chunk of memory. * * @param name * The name of the mempool. @@ -610,9 +696,7 @@ typedef void (rte_mempool_ctor_t)(struct rte_mempool *, void *); * never be used. The access to the per-lcore table is of course * faster than the multi-producer/consumer pool. The cache can be * disabled if the cache_size argument is set to 0; it can be useful to - * avoid losing objects in cache. Note that even if not used, the - * memory space for cache is always reserved in a mempool structure, - * except if CONFIG_RTE_MEMPOOL_CACHE_MAX_SIZE is set to 0. + * avoid losing objects in cache. * @param private_data_size * The size of the private data appended after the mempool * structure. This is useful for storing some private data after the @@ -656,7 +740,7 @@ typedef void (rte_mempool_ctor_t)(struct rte_mempool *, void *); * when using rte_mempool_get() or rte_mempool_get_bulk() is * "single-consumer". Otherwise, it is "multi-consumers". * - MEMPOOL_F_NO_PHYS_CONTIG: If set, allocated objects won't - * necessarilly be contiguous in physical memory. + * necessarily be contiguous in physical memory. * @return * The pointer to the new allocated mempool, on success. NULL on error * with rte_errno set appropriately. Possible rte_errno values include: @@ -722,11 +806,10 @@ rte_mempool_create(const char *name, unsigned n, unsigned elt_size, * @param vaddr * Virtual address of the externally allocated memory buffer. * Will be used to store mempool objects. - * @param paddr - * Array of physical addresses of the pages that comprises given memory - * buffer. + * @param iova + * Array of IO addresses of the pages that comprises given memory buffer. * @param pg_num - * Number of elements in the paddr array. + * Number of elements in the iova array. * @param pg_shift * LOG2 of the physical pages size. * @return @@ -739,14 +822,14 @@ rte_mempool_xmem_create(const char *name, unsigned n, unsigned elt_size, rte_mempool_ctor_t *mp_init, void *mp_init_arg, rte_mempool_obj_cb_t *obj_init, void *obj_init_arg, int socket_id, unsigned flags, void *vaddr, - const phys_addr_t paddr[], uint32_t pg_num, uint32_t pg_shift); + const rte_iova_t iova[], uint32_t pg_num, uint32_t pg_shift); /** * Create an empty mempool * * The mempool is allocated and initialized, but it is not populated: no * memory is allocated for the mempool elements. The user has to call - * rte_mempool_populate_*() or to add memory chunks to the pool. Once + * rte_mempool_populate_*() to add memory chunks to the pool. Once * populated, the user may also want to initialize each object with * rte_mempool_obj_iter(). * @@ -796,14 +879,18 @@ rte_mempool_free(struct rte_mempool *mp); * Add physically contiguous memory for objects in the pool at init * * Add a virtually and physically contiguous memory chunk in the pool - * where objects can be instanciated. + * where objects can be instantiated. + * + * If the given IO address is unknown (iova = RTE_BAD_IOVA), + * the chunk doesn't need to be physically contiguous (only virtually), + * and allocated objects may span two pages. * * @param mp * A pointer to the mempool structure. * @param vaddr * The virtual address of memory that should be used to store objects. - * @param paddr - * The physical address + * @param iova + * The IO address * @param len * The length of memory in bytes. * @param free_cb @@ -815,6 +902,11 @@ rte_mempool_free(struct rte_mempool *mp); * On error, the chunk is not added in the memory list of the * mempool and a negative errno is returned. */ +int rte_mempool_populate_iova(struct rte_mempool *mp, char *vaddr, + rte_iova_t iova, size_t len, rte_mempool_memchunk_free_cb_t *free_cb, + void *opaque); + +__rte_deprecated int rte_mempool_populate_phys(struct rte_mempool *mp, char *vaddr, phys_addr_t paddr, size_t len, rte_mempool_memchunk_free_cb_t *free_cb, void *opaque); @@ -823,18 +915,17 @@ int rte_mempool_populate_phys(struct rte_mempool *mp, char *vaddr, * Add physical memory for objects in the pool at init * * Add a virtually contiguous memory chunk in the pool where objects can - * be instanciated. The physical addresses corresponding to the virtual - * area are described in paddr[], pg_num, pg_shift. + * be instantiated. The IO addresses corresponding to the virtual + * area are described in iova[], pg_num, pg_shift. * * @param mp * A pointer to the mempool structure. * @param vaddr * The virtual address of memory that should be used to store objects. - * @param paddr - * An array of physical addresses of each page composing the virtual - * area. + * @param iova + * An array of IO addresses of each page composing the virtual area. * @param pg_num - * Number of elements in the paddr array. + * Number of elements in the iova array. * @param pg_shift * LOG2 of the physical pages size. * @param free_cb @@ -846,6 +937,11 @@ int rte_mempool_populate_phys(struct rte_mempool *mp, char *vaddr, * On error, the chunks are not added in the memory list of the * mempool and a negative errno is returned. */ +int rte_mempool_populate_iova_tab(struct rte_mempool *mp, char *vaddr, + const rte_iova_t iova[], uint32_t pg_num, uint32_t pg_shift, + rte_mempool_memchunk_free_cb_t *free_cb, void *opaque); + +__rte_deprecated int rte_mempool_populate_phys_tab(struct rte_mempool *mp, char *vaddr, const phys_addr_t paddr[], uint32_t pg_num, uint32_t pg_shift, rte_mempool_memchunk_free_cb_t *free_cb, void *opaque); @@ -854,7 +950,7 @@ int rte_mempool_populate_phys_tab(struct rte_mempool *mp, char *vaddr, * Add virtually contiguous memory for objects in the pool at init * * Add a virtually contiguous memory chunk in the pool where objects can - * be instanciated. + * be instantiated. * * @param mp * A pointer to the mempool structure. @@ -946,7 +1042,7 @@ uint32_t rte_mempool_mem_iter(struct rte_mempool *mp, rte_mempool_mem_cb_t *mem_cb, void *mem_cb_arg); /** - * Dump the status of the mempool to the console. + * Dump the status of the mempool to a file. * * @param f * A pointer to a file for output @@ -989,7 +1085,7 @@ rte_mempool_cache_free(struct rte_mempool_cache *cache); * @param mp * A pointer to the mempool. */ -static inline void __attribute__((always_inline)) +static __rte_always_inline void rte_mempool_cache_flush(struct rte_mempool_cache *cache, struct rte_mempool *mp) { @@ -1007,7 +1103,7 @@ rte_mempool_cache_flush(struct rte_mempool_cache *cache, * @return * A pointer to the mempool cache or NULL if disabled or non-EAL thread. */ -static inline struct rte_mempool_cache *__attribute__((always_inline)) +static __rte_always_inline struct rte_mempool_cache * rte_mempool_default_cache(struct rte_mempool *mp, unsigned lcore_id) { if (mp->cache_size == 0) @@ -1030,25 +1126,18 @@ rte_mempool_default_cache(struct rte_mempool *mp, unsigned lcore_id) * positive. * @param cache * A pointer to a mempool cache structure. May be NULL if not needed. - * @param flags - * The flags used for the mempool creation. - * Single-producer (MEMPOOL_F_SP_PUT flag) or multi-producers. */ -static inline void __attribute__((always_inline)) +static __rte_always_inline void __mempool_generic_put(struct rte_mempool *mp, void * const *obj_table, - unsigned n, struct rte_mempool_cache *cache, int flags) + unsigned int n, struct rte_mempool_cache *cache) { void **cache_objs; /* increment stat now, adding in mempool always success */ __MEMPOOL_STAT_ADD(mp, put, n); - /* No cache provided or single producer */ - if (unlikely(cache == NULL || flags & MEMPOOL_F_SP_PUT)) - goto ring_enqueue; - - /* Go straight to ring if put would overflow mem allocated for cache */ - if (unlikely(n > RTE_MEMPOOL_CACHE_MAX_SIZE)) + /* No cache provided or if put would overflow mem allocated for cache */ + if (unlikely(cache == NULL || n > RTE_MEMPOOL_CACHE_MAX_SIZE)) goto ring_enqueue; cache_objs = &cache->objs[cache->len]; @@ -1096,56 +1185,13 @@ ring_enqueue: * The number of objects to add in the mempool from the obj_table. * @param cache * A pointer to a mempool cache structure. May be NULL if not needed. - * @param flags - * The flags used for the mempool creation. - * Single-producer (MEMPOOL_F_SP_PUT flag) or multi-producers. */ -static inline void __attribute__((always_inline)) +static __rte_always_inline void rte_mempool_generic_put(struct rte_mempool *mp, void * const *obj_table, - unsigned n, struct rte_mempool_cache *cache, int flags) + unsigned int n, struct rte_mempool_cache *cache) { __mempool_check_cookies(mp, obj_table, n, 0); - __mempool_generic_put(mp, obj_table, n, cache, flags); -} - -/** - * @deprecated - * Put several objects back in the mempool (multi-producers safe). - * - * @param mp - * A pointer to the mempool structure. - * @param obj_table - * A pointer to a table of void * pointers (objects). - * @param n - * The number of objects to add in the mempool from the obj_table. - */ -__rte_deprecated -static inline void __attribute__((always_inline)) -rte_mempool_mp_put_bulk(struct rte_mempool *mp, void * const *obj_table, - unsigned n) -{ - struct rte_mempool_cache *cache; - cache = rte_mempool_default_cache(mp, rte_lcore_id()); - rte_mempool_generic_put(mp, obj_table, n, cache, 0); -} - -/** - * @deprecated - * Put several objects back in the mempool (NOT multi-producers safe). - * - * @param mp - * A pointer to the mempool structure. - * @param obj_table - * A pointer to a table of void * pointers (objects). - * @param n - * The number of objects to add in the mempool from obj_table. - */ -__rte_deprecated -static inline void __attribute__((always_inline)) -rte_mempool_sp_put_bulk(struct rte_mempool *mp, void * const *obj_table, - unsigned n) -{ - rte_mempool_generic_put(mp, obj_table, n, NULL, MEMPOOL_F_SP_PUT); + __mempool_generic_put(mp, obj_table, n, cache); } /** @@ -1162,47 +1208,13 @@ rte_mempool_sp_put_bulk(struct rte_mempool *mp, void * const *obj_table, * @param n * The number of objects to add in the mempool from obj_table. */ -static inline void __attribute__((always_inline)) +static __rte_always_inline void rte_mempool_put_bulk(struct rte_mempool *mp, void * const *obj_table, - unsigned n) + unsigned int n) { struct rte_mempool_cache *cache; cache = rte_mempool_default_cache(mp, rte_lcore_id()); - rte_mempool_generic_put(mp, obj_table, n, cache, mp->flags); -} - -/** - * @deprecated - * Put one object in the mempool (multi-producers safe). - * - * @param mp - * A pointer to the mempool structure. - * @param obj - * A pointer to the object to be added. - */ -__rte_deprecated -static inline void __attribute__((always_inline)) -rte_mempool_mp_put(struct rte_mempool *mp, void *obj) -{ - struct rte_mempool_cache *cache; - cache = rte_mempool_default_cache(mp, rte_lcore_id()); - rte_mempool_generic_put(mp, &obj, 1, cache, 0); -} - -/** - * @deprecated - * Put one object back in the mempool (NOT multi-producers safe). - * - * @param mp - * A pointer to the mempool structure. - * @param obj - * A pointer to the object to be added. - */ -__rte_deprecated -static inline void __attribute__((always_inline)) -rte_mempool_sp_put(struct rte_mempool *mp, void *obj) -{ - rte_mempool_generic_put(mp, &obj, 1, NULL, MEMPOOL_F_SP_PUT); + rte_mempool_generic_put(mp, obj_table, n, cache); } /** @@ -1217,7 +1229,7 @@ rte_mempool_sp_put(struct rte_mempool *mp, void *obj) * @param obj * A pointer to the object to be added. */ -static inline void __attribute__((always_inline)) +static __rte_always_inline void rte_mempool_put(struct rte_mempool *mp, void *obj) { rte_mempool_put_bulk(mp, &obj, 1); @@ -1233,24 +1245,20 @@ rte_mempool_put(struct rte_mempool *mp, void *obj) * The number of objects to get, must be strictly positive. * @param cache * A pointer to a mempool cache structure. May be NULL if not needed. - * @param flags - * The flags used for the mempool creation. - * Single-consumer (MEMPOOL_F_SC_GET flag) or multi-consumers. * @return * - >=0: Success; number of objects supplied. * - <0: Error; code of ring dequeue function. */ -static inline int __attribute__((always_inline)) +static __rte_always_inline int __mempool_generic_get(struct rte_mempool *mp, void **obj_table, - unsigned n, struct rte_mempool_cache *cache, int flags) + unsigned int n, struct rte_mempool_cache *cache) { int ret; uint32_t index, len; void **cache_objs; - /* No cache provided or single consumer */ - if (unlikely(cache == NULL || flags & MEMPOOL_F_SC_GET || - n >= cache->size)) + /* No cache provided or cannot be satisfied from cache */ + if (unlikely(cache == NULL || n >= cache->size)) goto ring_dequeue; cache_objs = cache->objs; @@ -1315,80 +1323,21 @@ ring_dequeue: * The number of objects to get from mempool to obj_table. * @param cache * A pointer to a mempool cache structure. May be NULL if not needed. - * @param flags - * The flags used for the mempool creation. - * Single-consumer (MEMPOOL_F_SC_GET flag) or multi-consumers. * @return * - 0: Success; objects taken. * - -ENOENT: Not enough entries in the mempool; no object is retrieved. */ -static inline int __attribute__((always_inline)) -rte_mempool_generic_get(struct rte_mempool *mp, void **obj_table, unsigned n, - struct rte_mempool_cache *cache, int flags) +static __rte_always_inline int +rte_mempool_generic_get(struct rte_mempool *mp, void **obj_table, + unsigned int n, struct rte_mempool_cache *cache) { int ret; - ret = __mempool_generic_get(mp, obj_table, n, cache, flags); + ret = __mempool_generic_get(mp, obj_table, n, cache); if (ret == 0) __mempool_check_cookies(mp, obj_table, n, 1); return ret; } -/** - * @deprecated - * Get several objects from the mempool (multi-consumers safe). - * - * If cache is enabled, objects will be retrieved first from cache, - * subsequently from the common pool. Note that it can return -ENOENT when - * the local cache and common pool are empty, even if cache from other - * lcores are full. - * - * @param mp - * A pointer to the mempool structure. - * @param obj_table - * A pointer to a table of void * pointers (objects) that will be filled. - * @param n - * The number of objects to get from mempool to obj_table. - * @return - * - 0: Success; objects taken. - * - -ENOENT: Not enough entries in the mempool; no object is retrieved. - */ -__rte_deprecated -static inline int __attribute__((always_inline)) -rte_mempool_mc_get_bulk(struct rte_mempool *mp, void **obj_table, unsigned n) -{ - struct rte_mempool_cache *cache; - cache = rte_mempool_default_cache(mp, rte_lcore_id()); - return rte_mempool_generic_get(mp, obj_table, n, cache, 0); -} - -/** - * @deprecated - * Get several objects from the mempool (NOT multi-consumers safe). - * - * If cache is enabled, objects will be retrieved first from cache, - * subsequently from the common pool. Note that it can return -ENOENT when - * the local cache and common pool are empty, even if cache from other - * lcores are full. - * - * @param mp - * A pointer to the mempool structure. - * @param obj_table - * A pointer to a table of void * pointers (objects) that will be filled. - * @param n - * The number of objects to get from the mempool to obj_table. - * @return - * - 0: Success; objects taken. - * - -ENOENT: Not enough entries in the mempool; no object is - * retrieved. - */ -__rte_deprecated -static inline int __attribute__((always_inline)) -rte_mempool_sc_get_bulk(struct rte_mempool *mp, void **obj_table, unsigned n) -{ - return rte_mempool_generic_get(mp, obj_table, n, NULL, - MEMPOOL_F_SC_GET); -} - /** * Get several objects from the mempool. * @@ -1411,62 +1360,12 @@ rte_mempool_sc_get_bulk(struct rte_mempool *mp, void **obj_table, unsigned n) * - 0: Success; objects taken * - -ENOENT: Not enough entries in the mempool; no object is retrieved. */ -static inline int __attribute__((always_inline)) -rte_mempool_get_bulk(struct rte_mempool *mp, void **obj_table, unsigned n) +static __rte_always_inline int +rte_mempool_get_bulk(struct rte_mempool *mp, void **obj_table, unsigned int n) { struct rte_mempool_cache *cache; cache = rte_mempool_default_cache(mp, rte_lcore_id()); - return rte_mempool_generic_get(mp, obj_table, n, cache, mp->flags); -} - -/** - * @deprecated - * Get one object from the mempool (multi-consumers safe). - * - * If cache is enabled, objects will be retrieved first from cache, - * subsequently from the common pool. Note that it can return -ENOENT when - * the local cache and common pool are empty, even if cache from other - * lcores are full. - * - * @param mp - * A pointer to the mempool structure. - * @param obj_p - * A pointer to a void * pointer (object) that will be filled. - * @return - * - 0: Success; objects taken. - * - -ENOENT: Not enough entries in the mempool; no object is retrieved. - */ -__rte_deprecated -static inline int __attribute__((always_inline)) -rte_mempool_mc_get(struct rte_mempool *mp, void **obj_p) -{ - struct rte_mempool_cache *cache; - cache = rte_mempool_default_cache(mp, rte_lcore_id()); - return rte_mempool_generic_get(mp, obj_p, 1, cache, 0); -} - -/** - * @deprecated - * Get one object from the mempool (NOT multi-consumers safe). - * - * If cache is enabled, objects will be retrieved first from cache, - * subsequently from the common pool. Note that it can return -ENOENT when - * the local cache and common pool are empty, even if cache from other - * lcores are full. - * - * @param mp - * A pointer to the mempool structure. - * @param obj_p - * A pointer to a void * pointer (object) that will be filled. - * @return - * - 0: Success; objects taken. - * - -ENOENT: Not enough entries in the mempool; no object is retrieved. - */ -__rte_deprecated -static inline int __attribute__((always_inline)) -rte_mempool_sc_get(struct rte_mempool *mp, void **obj_p) -{ - return rte_mempool_generic_get(mp, obj_p, 1, NULL, MEMPOOL_F_SC_GET); + return rte_mempool_generic_get(mp, obj_table, n, cache); } /** @@ -1489,7 +1388,7 @@ rte_mempool_sc_get(struct rte_mempool *mp, void **obj_p) * - 0: Success; objects taken. * - -ENOENT: Not enough entries in the mempool; no object is retrieved. */ -static inline int __attribute__((always_inline)) +static __rte_always_inline int rte_mempool_get(struct rte_mempool *mp, void **obj_p) { return rte_mempool_get_bulk(mp, obj_p, 1); @@ -1509,22 +1408,6 @@ rte_mempool_get(struct rte_mempool *mp, void **obj_p) */ unsigned int rte_mempool_avail_count(const struct rte_mempool *mp); -/** - * @deprecated - * Return the number of entries in the mempool. - * - * When cache is enabled, this function has to browse the length of - * all lcores, so it should not be used in a data path, but only for - * debug purposes. - * - * @param mp - * A pointer to the mempool structure. - * @return - * The number of entries in the mempool. - */ -__rte_deprecated -unsigned rte_mempool_count(const struct rte_mempool *mp); - /** * Return the number of elements which have been allocated from the mempool * @@ -1540,31 +1423,6 @@ unsigned rte_mempool_count(const struct rte_mempool *mp); unsigned int rte_mempool_in_use_count(const struct rte_mempool *mp); -/** - * @deprecated - * Return the number of free entries in the mempool ring. - * i.e. how many entries can be freed back to the mempool. - * - * NOTE: This corresponds to the number of elements *allocated* from the - * memory pool, not the number of elements in the pool itself. To count - * the number elements currently available in the pool, use "rte_mempool_count" - * - * When cache is enabled, this function has to browse the length of - * all lcores, so it should not be used in a data path, but only for - * debug purposes. User-owned mempool caches are not accounted for. - * - * @param mp - * A pointer to the mempool structure. - * @return - * The number of free entries in the mempool. - */ -__rte_deprecated -static inline unsigned -rte_mempool_free_count(const struct rte_mempool *mp) -{ - return rte_mempool_in_use_count(mp); -} - /** * Test if the mempool is full. * @@ -1604,24 +1462,29 @@ rte_mempool_empty(const struct rte_mempool *mp) } /** - * Return the physical address of elt, which is an element of the pool mp. + * Return the IO address of elt, which is an element of the pool mp. * - * @param mp - * A pointer to the mempool structure. * @param elt * A pointer (virtual address) to the element of the pool. * @return - * The physical address of the elt element. + * The IO address of the elt element. * If the mempool was created with MEMPOOL_F_NO_PHYS_CONTIG, the - * returned value is RTE_BAD_PHYS_ADDR. + * returned value is RTE_BAD_IOVA. */ -static inline phys_addr_t -rte_mempool_virt2phy(__rte_unused const struct rte_mempool *mp, const void *elt) +static inline rte_iova_t +rte_mempool_virt2iova(const void *elt) { const struct rte_mempool_objhdr *hdr; hdr = (const struct rte_mempool_objhdr *)RTE_PTR_SUB(elt, sizeof(*hdr)); - return hdr->physaddr; + return hdr->iova; +} + +__rte_deprecated +static inline phys_addr_t +rte_mempool_virt2phy(__rte_unused const struct rte_mempool *mp, const void *elt) +{ + return rte_mempool_virt2iova(elt); } /** @@ -1710,11 +1573,13 @@ uint32_t rte_mempool_calc_obj_size(uint32_t elt_size, uint32_t flags, * by rte_mempool_calc_obj_size(). * @param pg_shift * LOG2 of the physical pages size. If set to 0, ignore page boundaries. + * @param flags + * The mempool flags. * @return * Required memory size aligned at page boundary. */ size_t rte_mempool_xmem_size(uint32_t elt_num, size_t total_elt_sz, - uint32_t pg_shift); + uint32_t pg_shift, unsigned int flags); /** * Get the size of memory required to store mempool elements. @@ -1730,13 +1595,14 @@ size_t rte_mempool_xmem_size(uint32_t elt_num, size_t total_elt_sz, * @param total_elt_sz * The size of each element, including header and trailer, as returned * by rte_mempool_calc_obj_size(). - * @param paddr - * Array of physical addresses of the pages that comprises given memory - * buffer. + * @param iova + * Array of IO addresses of the pages that comprises given memory buffer. * @param pg_num - * Number of elements in the paddr array. + * Number of elements in the iova array. * @param pg_shift * LOG2 of the physical pages size. + * @param flags + * The mempool flags. * @return * On success, the number of bytes needed to store given number of * objects, aligned to the given page size. If the provided memory @@ -1744,8 +1610,8 @@ size_t rte_mempool_xmem_size(uint32_t elt_num, size_t total_elt_sz, * is the actual number of elements that can be stored in that buffer. */ ssize_t rte_mempool_xmem_usage(void *vaddr, uint32_t elt_num, - size_t total_elt_sz, const phys_addr_t paddr[], uint32_t pg_num, - uint32_t pg_shift); + size_t total_elt_sz, const rte_iova_t iova[], uint32_t pg_num, + uint32_t pg_shift, unsigned int flags); /** * Walk list of all memory pools diff --git a/dpdk/lib/librte_mempool/rte_mempool_ops.c b/dpdk/lib/librte_mempool/rte_mempool_ops.c index 5f24de25..92b9f90c 100644 --- a/dpdk/lib/librte_mempool/rte_mempool_ops.c +++ b/dpdk/lib/librte_mempool/rte_mempool_ops.c @@ -37,6 +37,7 @@ #include #include +#include /* indirect jump table to support external memory pools. */ struct rte_mempool_ops_table rte_mempool_ops_table = { @@ -85,6 +86,8 @@ rte_mempool_register_ops(const struct rte_mempool_ops *h) ops->enqueue = h->enqueue; ops->dequeue = h->dequeue; ops->get_count = h->get_count; + ops->get_capabilities = h->get_capabilities; + ops->register_memory_area = h->register_memory_area; rte_spinlock_unlock(&rte_mempool_ops_table.sl); @@ -123,6 +126,32 @@ rte_mempool_ops_get_count(const struct rte_mempool *mp) return ops->get_count(mp); } +/* wrapper to get external mempool capabilities. */ +int +rte_mempool_ops_get_capabilities(const struct rte_mempool *mp, + unsigned int *flags) +{ + struct rte_mempool_ops *ops; + + ops = rte_mempool_get_ops(mp->ops_index); + + RTE_FUNC_PTR_OR_ERR_RET(ops->get_capabilities, -ENOTSUP); + return ops->get_capabilities(mp, flags); +} + +/* wrapper to notify new memory area to external mempool */ +int +rte_mempool_ops_register_memory_area(const struct rte_mempool *mp, char *vaddr, + rte_iova_t iova, size_t len) +{ + struct rte_mempool_ops *ops; + + ops = rte_mempool_get_ops(mp->ops_index); + + RTE_FUNC_PTR_OR_ERR_RET(ops->register_memory_area, -ENOTSUP); + return ops->register_memory_area(mp, vaddr, iova, len); +} + /* sets mempool ops previously registered by rte_mempool_register_ops. */ int rte_mempool_set_ops_byname(struct rte_mempool *mp, const char *name, diff --git a/dpdk/lib/librte_mempool/rte_mempool_ring.c b/dpdk/lib/librte_mempool/rte_mempool_ring.c deleted file mode 100644 index b9aa64dd..00000000 --- a/dpdk/lib/librte_mempool/rte_mempool_ring.c +++ /dev/null @@ -1,161 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include -#include - -#include -#include -#include - -static int -common_ring_mp_enqueue(struct rte_mempool *mp, void * const *obj_table, - unsigned n) -{ - return rte_ring_mp_enqueue_bulk(mp->pool_data, obj_table, n); -} - -static int -common_ring_sp_enqueue(struct rte_mempool *mp, void * const *obj_table, - unsigned n) -{ - return rte_ring_sp_enqueue_bulk(mp->pool_data, obj_table, n); -} - -static int -common_ring_mc_dequeue(struct rte_mempool *mp, void **obj_table, unsigned n) -{ - return rte_ring_mc_dequeue_bulk(mp->pool_data, obj_table, n); -} - -static int -common_ring_sc_dequeue(struct rte_mempool *mp, void **obj_table, unsigned n) -{ - return rte_ring_sc_dequeue_bulk(mp->pool_data, obj_table, n); -} - -static unsigned -common_ring_get_count(const struct rte_mempool *mp) -{ - return rte_ring_count(mp->pool_data); -} - - -static int -common_ring_alloc(struct rte_mempool *mp) -{ - int rg_flags = 0, ret; - char rg_name[RTE_RING_NAMESIZE]; - struct rte_ring *r; - - ret = snprintf(rg_name, sizeof(rg_name), - RTE_MEMPOOL_MZ_FORMAT, mp->name); - if (ret < 0 || ret >= (int)sizeof(rg_name)) { - rte_errno = ENAMETOOLONG; - return -rte_errno; - } - - /* ring flags */ - if (mp->flags & MEMPOOL_F_SP_PUT) - rg_flags |= RING_F_SP_ENQ; - if (mp->flags & MEMPOOL_F_SC_GET) - rg_flags |= RING_F_SC_DEQ; - - /* - * Allocate the ring that will be used to store objects. - * Ring functions will return appropriate errors if we are - * running as a secondary process etc., so no checks made - * in this function for that condition. - */ - r = rte_ring_create(rg_name, rte_align32pow2(mp->size + 1), - mp->socket_id, rg_flags); - if (r == NULL) - return -rte_errno; - - mp->pool_data = r; - - return 0; -} - -static void -common_ring_free(struct rte_mempool *mp) -{ - rte_ring_free(mp->pool_data); -} - -/* - * The following 4 declarations of mempool ops structs address - * the need for the backward compatible mempool handlers for - * single/multi producers and single/multi consumers as dictated by the - * flags provided to the rte_mempool_create function - */ -static const struct rte_mempool_ops ops_mp_mc = { - .name = "ring_mp_mc", - .alloc = common_ring_alloc, - .free = common_ring_free, - .enqueue = common_ring_mp_enqueue, - .dequeue = common_ring_mc_dequeue, - .get_count = common_ring_get_count, -}; - -static const struct rte_mempool_ops ops_sp_sc = { - .name = "ring_sp_sc", - .alloc = common_ring_alloc, - .free = common_ring_free, - .enqueue = common_ring_sp_enqueue, - .dequeue = common_ring_sc_dequeue, - .get_count = common_ring_get_count, -}; - -static const struct rte_mempool_ops ops_mp_sc = { - .name = "ring_mp_sc", - .alloc = common_ring_alloc, - .free = common_ring_free, - .enqueue = common_ring_mp_enqueue, - .dequeue = common_ring_sc_dequeue, - .get_count = common_ring_get_count, -}; - -static const struct rte_mempool_ops ops_sp_mc = { - .name = "ring_sp_mc", - .alloc = common_ring_alloc, - .free = common_ring_free, - .enqueue = common_ring_sp_enqueue, - .dequeue = common_ring_mc_dequeue, - .get_count = common_ring_get_count, -}; - -MEMPOOL_REGISTER_OPS(ops_mp_mc); -MEMPOOL_REGISTER_OPS(ops_sp_sc); -MEMPOOL_REGISTER_OPS(ops_mp_sc); -MEMPOOL_REGISTER_OPS(ops_sp_mc); diff --git a/dpdk/lib/librte_mempool/rte_mempool_stack.c b/dpdk/lib/librte_mempool/rte_mempool_stack.c deleted file mode 100644 index 5fd8af24..00000000 --- a/dpdk/lib/librte_mempool/rte_mempool_stack.c +++ /dev/null @@ -1,147 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2016 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include -#include -#include - -struct rte_mempool_stack { - rte_spinlock_t sl; - - uint32_t size; - uint32_t len; - void *objs[]; -}; - -static int -stack_alloc(struct rte_mempool *mp) -{ - struct rte_mempool_stack *s; - unsigned n = mp->size; - int size = sizeof(*s) + (n+16)*sizeof(void *); - - /* Allocate our local memory structure */ - s = rte_zmalloc_socket("mempool-stack", - size, - RTE_CACHE_LINE_SIZE, - mp->socket_id); - if (s == NULL) { - RTE_LOG(ERR, MEMPOOL, "Cannot allocate stack!\n"); - return -ENOMEM; - } - - rte_spinlock_init(&s->sl); - - s->size = n; - mp->pool_data = s; - - return 0; -} - -static int -stack_enqueue(struct rte_mempool *mp, void * const *obj_table, - unsigned n) -{ - struct rte_mempool_stack *s = mp->pool_data; - void **cache_objs; - unsigned index; - - rte_spinlock_lock(&s->sl); - cache_objs = &s->objs[s->len]; - - /* Is there sufficient space in the stack ? */ - if ((s->len + n) > s->size) { - rte_spinlock_unlock(&s->sl); - return -ENOBUFS; - } - - /* Add elements back into the cache */ - for (index = 0; index < n; ++index, obj_table++) - cache_objs[index] = *obj_table; - - s->len += n; - - rte_spinlock_unlock(&s->sl); - return 0; -} - -static int -stack_dequeue(struct rte_mempool *mp, void **obj_table, - unsigned n) -{ - struct rte_mempool_stack *s = mp->pool_data; - void **cache_objs; - unsigned index, len; - - rte_spinlock_lock(&s->sl); - - if (unlikely(n > s->len)) { - rte_spinlock_unlock(&s->sl); - return -ENOENT; - } - - cache_objs = s->objs; - - for (index = 0, len = s->len - 1; index < n; - ++index, len--, obj_table++) - *obj_table = cache_objs[len]; - - s->len -= n; - rte_spinlock_unlock(&s->sl); - return n; -} - -static unsigned -stack_get_count(const struct rte_mempool *mp) -{ - struct rte_mempool_stack *s = mp->pool_data; - - return s->len; -} - -static void -stack_free(struct rte_mempool *mp) -{ - rte_free((void *)(mp->pool_data)); -} - -static struct rte_mempool_ops ops_stack = { - .name = "stack", - .alloc = stack_alloc, - .free = stack_free, - .enqueue = stack_enqueue, - .dequeue = stack_dequeue, - .get_count = stack_get_count -}; - -MEMPOOL_REGISTER_OPS(ops_stack); diff --git a/dpdk/lib/librte_mempool/rte_mempool_version.map b/dpdk/lib/librte_mempool/rte_mempool_version.map index dee1c990..62b76f91 100644 --- a/dpdk/lib/librte_mempool/rte_mempool_version.map +++ b/dpdk/lib/librte_mempool/rte_mempool_version.map @@ -3,7 +3,6 @@ DPDK_2.0 { rte_mempool_audit; rte_mempool_calc_obj_size; - rte_mempool_count; rte_mempool_create; rte_mempool_dump; rte_mempool_list_dump; @@ -42,3 +41,13 @@ DPDK_16.07 { rte_mempool_set_ops_byname; } DPDK_2.0; + +DPDK_17.11 { + global: + + rte_mempool_ops_get_capabilities; + rte_mempool_ops_register_memory_area; + rte_mempool_populate_iova; + rte_mempool_populate_iova_tab; + +} DPDK_16.07; diff --git a/dpdk/lib/librte_meter/Makefile b/dpdk/lib/librte_meter/Makefile index f07fced7..bfeb5d60 100644 --- a/dpdk/lib/librte_meter/Makefile +++ b/dpdk/lib/librte_meter/Makefile @@ -40,6 +40,7 @@ CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) LDLIBS += -lm +LDLIBS += -lrte_eal EXPORT_MAP := rte_meter_version.map @@ -53,7 +54,4 @@ SRCS-$(CONFIG_RTE_LIBRTE_METER) := rte_meter.c # install includes SYMLINK-$(CONFIG_RTE_LIBRTE_METER)-include := rte_meter.h -# this lib depends upon: -DEPDIRS-$(CONFIG_RTE_LIBRTE_METER) += lib/librte_eal - include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/lib/librte_meter/rte_meter.h b/dpdk/lib/librte_meter/rte_meter.h index 2cd8d814..2ab71849 100644 --- a/dpdk/lib/librte_meter/rte_meter.h +++ b/dpdk/lib/librte_meter/rte_meter.h @@ -232,13 +232,15 @@ rte_meter_srtcm_color_blind_check(struct rte_meter_srtcm *m, n_periods = time_diff / m->cir_period; m->time += n_periods * m->cir_period; + /* Put the tokens overflowing from tc into te bucket */ tc = m->tc + n_periods * m->cir_bytes_per_period; - if (tc > m->cbs) + te = m->te; + if (tc > m->cbs) { + te += (tc - m->cbs); + if (te > m->ebs) + te = m->ebs; tc = m->cbs; - - te = m->te + n_periods * m->cir_bytes_per_period; - if (te > m->ebs) - te = m->ebs; + } /* Color logic */ if (tc >= pkt_len) { @@ -271,13 +273,15 @@ rte_meter_srtcm_color_aware_check(struct rte_meter_srtcm *m, n_periods = time_diff / m->cir_period; m->time += n_periods * m->cir_period; + /* Put the tokens overflowing from tc into te bucket */ tc = m->tc + n_periods * m->cir_bytes_per_period; - if (tc > m->cbs) + te = m->te; + if (tc > m->cbs) { + te += (tc - m->cbs); + if (te > m->ebs) + te = m->ebs; tc = m->cbs; - - te = m->te + n_periods * m->cir_bytes_per_period; - if (te > m->ebs) - te = m->ebs; + } /* Color logic */ if ((pkt_color == e_RTE_METER_GREEN) && (tc >= pkt_len)) { diff --git a/dpdk/lib/librte_metrics/Makefile b/dpdk/lib/librte_metrics/Makefile new file mode 100644 index 00000000..a6efba4a --- /dev/null +++ b/dpdk/lib/librte_metrics/Makefile @@ -0,0 +1,50 @@ +# BSD LICENSE +# +# Copyright(c) 2017 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +include $(RTE_SDK)/mk/rte.vars.mk + +# library name +LIB = librte_metrics.a + +CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -O3 +LDLIBS += -lrte_eal + +EXPORT_MAP := rte_metrics_version.map + +LIBABIVER := 1 + +# all source are stored in SRCS-y +SRCS-$(CONFIG_RTE_LIBRTE_METRICS) := rte_metrics.c + +# Install header file +SYMLINK-$(CONFIG_RTE_LIBRTE_METRICS)-include += rte_metrics.h + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/lib/librte_metrics/rte_metrics.c b/dpdk/lib/librte_metrics/rte_metrics.c new file mode 100644 index 00000000..d9404001 --- /dev/null +++ b/dpdk/lib/librte_metrics/rte_metrics.c @@ -0,0 +1,304 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include + +#include +#include +#include +#include +#include +#include + +#define RTE_METRICS_MAX_METRICS 256 +#define RTE_METRICS_MEMZONE_NAME "RTE_METRICS" + +/** + * Internal stats metadata and value entry. + * + * @internal + */ +struct rte_metrics_meta_s { + /** Name of metric */ + char name[RTE_METRICS_MAX_NAME_LEN]; + /** Current value for metric */ + uint64_t value[RTE_MAX_ETHPORTS]; + /** Used for global metrics */ + uint64_t global_value; + /** Index of next root element (zero for none) */ + uint16_t idx_next_set; + /** Index of next metric in set (zero for none) */ + uint16_t idx_next_stat; +}; + +/** + * Internal stats info structure. + * + * @internal + * Offsets into metadata are used instead of pointers because ASLR + * means that having the same physical addresses in different + * processes is not guaranteed. + */ +struct rte_metrics_data_s { + /** Index of last metadata entry with valid data. + * This value is not valid if cnt_stats is zero. + */ + uint16_t idx_last_set; + /** Number of metrics. */ + uint16_t cnt_stats; + /** Metric data memory block. */ + struct rte_metrics_meta_s metadata[RTE_METRICS_MAX_METRICS]; + /** Metric data access lock */ + rte_spinlock_t lock; +}; + +void +rte_metrics_init(int socket_id) +{ + struct rte_metrics_data_s *stats; + const struct rte_memzone *memzone; + + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return; + + memzone = rte_memzone_lookup(RTE_METRICS_MEMZONE_NAME); + if (memzone != NULL) + return; + memzone = rte_memzone_reserve(RTE_METRICS_MEMZONE_NAME, + sizeof(struct rte_metrics_data_s), socket_id, 0); + if (memzone == NULL) + rte_exit(EXIT_FAILURE, "Unable to allocate stats memzone\n"); + stats = memzone->addr; + memset(stats, 0, sizeof(struct rte_metrics_data_s)); + rte_spinlock_init(&stats->lock); +} + +int +rte_metrics_reg_name(const char *name) +{ + const char * const list_names[] = {name}; + + return rte_metrics_reg_names(list_names, 1); +} + +int +rte_metrics_reg_names(const char * const *names, uint16_t cnt_names) +{ + struct rte_metrics_meta_s *entry = NULL; + struct rte_metrics_data_s *stats; + const struct rte_memzone *memzone; + uint16_t idx_name; + uint16_t idx_base; + + /* Some sanity checks */ + if (cnt_names < 1 || names == NULL) + return -EINVAL; + + memzone = rte_memzone_lookup(RTE_METRICS_MEMZONE_NAME); + if (memzone == NULL) + return -EIO; + stats = memzone->addr; + + if (stats->cnt_stats + cnt_names >= RTE_METRICS_MAX_METRICS) + return -ENOMEM; + + rte_spinlock_lock(&stats->lock); + + /* Overwritten later if this is actually first set.. */ + stats->metadata[stats->idx_last_set].idx_next_set = stats->cnt_stats; + + stats->idx_last_set = idx_base = stats->cnt_stats; + + for (idx_name = 0; idx_name < cnt_names; idx_name++) { + entry = &stats->metadata[idx_name + stats->cnt_stats]; + strncpy(entry->name, names[idx_name], + RTE_METRICS_MAX_NAME_LEN); + /* Enforce NULL-termination */ + entry->name[RTE_METRICS_MAX_NAME_LEN - 1] = '\0'; + memset(entry->value, 0, sizeof(entry->value)); + entry->idx_next_stat = idx_name + stats->cnt_stats + 1; + } + entry->idx_next_stat = 0; + entry->idx_next_set = 0; + stats->cnt_stats += cnt_names; + + rte_spinlock_unlock(&stats->lock); + + return idx_base; +} + +int +rte_metrics_update_value(int port_id, uint16_t key, const uint64_t value) +{ + return rte_metrics_update_values(port_id, key, &value, 1); +} + +int +rte_metrics_update_values(int port_id, + uint16_t key, + const uint64_t *values, + uint32_t count) +{ + struct rte_metrics_meta_s *entry; + struct rte_metrics_data_s *stats; + const struct rte_memzone *memzone; + uint16_t idx_metric; + uint16_t idx_value; + uint16_t cnt_setsize; + + if (port_id != RTE_METRICS_GLOBAL && + (port_id < 0 || port_id >= RTE_MAX_ETHPORTS)) + return -EINVAL; + + if (values == NULL) + return -EINVAL; + + memzone = rte_memzone_lookup(RTE_METRICS_MEMZONE_NAME); + if (memzone == NULL) + return -EIO; + stats = memzone->addr; + + rte_spinlock_lock(&stats->lock); + idx_metric = key; + cnt_setsize = 1; + while (idx_metric < stats->cnt_stats) { + entry = &stats->metadata[idx_metric]; + if (entry->idx_next_stat == 0) + break; + cnt_setsize++; + idx_metric++; + } + /* Check update does not cross set border */ + if (count > cnt_setsize) { + rte_spinlock_unlock(&stats->lock); + return -ERANGE; + } + + if (port_id == RTE_METRICS_GLOBAL) + for (idx_value = 0; idx_value < count; idx_value++) { + idx_metric = key + idx_value; + stats->metadata[idx_metric].global_value = + values[idx_value]; + } + else + for (idx_value = 0; idx_value < count; idx_value++) { + idx_metric = key + idx_value; + stats->metadata[idx_metric].value[port_id] = + values[idx_value]; + } + rte_spinlock_unlock(&stats->lock); + return 0; +} + +int +rte_metrics_get_names(struct rte_metric_name *names, + uint16_t capacity) +{ + struct rte_metrics_data_s *stats; + const struct rte_memzone *memzone; + uint16_t idx_name; + int return_value; + + memzone = rte_memzone_lookup(RTE_METRICS_MEMZONE_NAME); + /* If not allocated, fail silently */ + if (memzone == NULL) + return 0; + + stats = memzone->addr; + rte_spinlock_lock(&stats->lock); + if (names != NULL) { + if (capacity < stats->cnt_stats) { + return_value = stats->cnt_stats; + rte_spinlock_unlock(&stats->lock); + return return_value; + } + for (idx_name = 0; idx_name < stats->cnt_stats; idx_name++) + strncpy(names[idx_name].name, + stats->metadata[idx_name].name, + RTE_METRICS_MAX_NAME_LEN); + } + return_value = stats->cnt_stats; + rte_spinlock_unlock(&stats->lock); + return return_value; +} + +int +rte_metrics_get_values(int port_id, + struct rte_metric_value *values, + uint16_t capacity) +{ + struct rte_metrics_meta_s *entry; + struct rte_metrics_data_s *stats; + const struct rte_memzone *memzone; + uint16_t idx_name; + int return_value; + + if (port_id != RTE_METRICS_GLOBAL && + (port_id < 0 || port_id >= RTE_MAX_ETHPORTS)) + return -EINVAL; + + memzone = rte_memzone_lookup(RTE_METRICS_MEMZONE_NAME); + /* If not allocated, fail silently */ + if (memzone == NULL) + return 0; + stats = memzone->addr; + rte_spinlock_lock(&stats->lock); + + if (values != NULL) { + if (capacity < stats->cnt_stats) { + return_value = stats->cnt_stats; + rte_spinlock_unlock(&stats->lock); + return return_value; + } + if (port_id == RTE_METRICS_GLOBAL) + for (idx_name = 0; + idx_name < stats->cnt_stats; + idx_name++) { + entry = &stats->metadata[idx_name]; + values[idx_name].key = idx_name; + values[idx_name].value = entry->global_value; + } + else + for (idx_name = 0; + idx_name < stats->cnt_stats; + idx_name++) { + entry = &stats->metadata[idx_name]; + values[idx_name].key = idx_name; + values[idx_name].value = entry->value[port_id]; + } + } + return_value = stats->cnt_stats; + rte_spinlock_unlock(&stats->lock); + return return_value; +} diff --git a/dpdk/lib/librte_metrics/rte_metrics.h b/dpdk/lib/librte_metrics/rte_metrics.h new file mode 100644 index 00000000..297300ad --- /dev/null +++ b/dpdk/lib/librte_metrics/rte_metrics.h @@ -0,0 +1,251 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** + * @file + * + * DPDK Metrics module + * + * Metrics are statistics that are not generated by PMDs, and hence + * are better reported through a mechanism that is independent from + * the ethdev-based extended statistics. Providers will typically + * be other libraries and consumers will typically be applications. + * + * Metric information is populated using a push model, where producers + * update the values contained within the metric library by calling + * an update function on the relevant metrics. Consumers receive + * metric information by querying the central metric data, which is + * held in shared memory. Currently only bulk querying of metrics + * by consumers is supported. + */ + +#ifndef _RTE_METRICS_H_ +#define _RTE_METRICS_H_ + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** Maximum length of metric name (including null-terminator) */ +#define RTE_METRICS_MAX_NAME_LEN 64 + +/** + * Global metric special id. + * + * When used for the port_id parameter when calling + * rte_metrics_update_metric() or rte_metrics_update_metric(), + * the global metric, which are not associated with any specific + * port (i.e. device), are updated. + */ +#define RTE_METRICS_GLOBAL -1 + + +/** + * A name-key lookup for metrics. + * + * An array of this structure is returned by rte_metrics_get_names(). + * The struct rte_metric_value references these names via their array index. + */ +struct rte_metric_name { + /** String describing metric */ + char name[RTE_METRICS_MAX_NAME_LEN]; +}; + + +/** + * Metric value structure. + * + * This structure is used by rte_metrics_get_values() to return metrics, + * which are statistics that are not generated by PMDs. It maps a name key, + * which corresponds to an index in the array returned by + * rte_metrics_get_names(). + */ +struct rte_metric_value { + /** Numeric identifier of metric. */ + uint16_t key; + /** Value for metric */ + uint64_t value; +}; + + +/** + * Initializes metric module. This function must be called from + * a primary process before metrics are used. + * + * @param socket_id + * Socket to use for shared memory allocation. + */ +void rte_metrics_init(int socket_id); + +/** + * Register a metric, making it available as a reporting parameter. + * + * Registering a metric is the way producers declare a parameter + * that they wish to be reported. Once registered, the associated + * numeric key can be obtained via rte_metrics_get_names(), which + * is required for updating said metric's value. + * + * @param name + * Metric name. If this exceeds RTE_METRICS_MAX_NAME_LEN (including + * the NULL terminator), it is truncated. + * + * @return + * - Zero or positive: Success (index key of new metric) + * - -EIO: Error, unable to access metrics shared memory + * (rte_metrics_init() not called) + * - -EINVAL: Error, invalid parameters + * - -ENOMEM: Error, maximum metrics reached + */ +int rte_metrics_reg_name(const char *name); + +/** + * Register a set of metrics. + * + * This is a bulk version of rte_metrics_reg_names() and aside from + * handling multiple keys at once is functionally identical. + * + * @param names + * List of metric names + * + * @param cnt_names + * Number of metrics in set + * + * @return + * - Zero or positive: Success (index key of start of set) + * - -EIO: Error, unable to access metrics shared memory + * (rte_metrics_init() not called) + * - -EINVAL: Error, invalid parameters + * - -ENOMEM: Error, maximum metrics reached + */ +int rte_metrics_reg_names(const char * const *names, uint16_t cnt_names); + +/** + * Get metric name-key lookup table. + * + * @param names + * A struct rte_metric_name array of at least *capacity* in size to + * receive key names. If this is NULL, function returns the required + * number of elements for this array. + * + * @param capacity + * Size (number of elements) of struct rte_metric_name array. + * Disregarded if names is NULL. + * + * @return + * - Positive value above capacity: error, *names* is too small. + * Return value is required size. + * - Positive value equal or less than capacity: Success. Return + * value is number of elements filled in. + * - Negative value: error. + */ +int rte_metrics_get_names( + struct rte_metric_name *names, + uint16_t capacity); + +/** + * Get metric value table. + * + * @param port_id + * Port id to query + * + * @param values + * A struct rte_metric_value array of at least *capacity* in size to + * receive metric ids and values. If this is NULL, function returns + * the required number of elements for this array. + * + * @param capacity + * Size (number of elements) of struct rte_metric_value array. + * Disregarded if names is NULL. + * + * @return + * - Positive value above capacity: error, *values* is too small. + * Return value is required size. + * - Positive value equal or less than capacity: Success. Return + * value is number of elements filled in. + * - Negative value: error. + */ +int rte_metrics_get_values( + int port_id, + struct rte_metric_value *values, + uint16_t capacity); + +/** + * Updates a metric + * + * @param port_id + * Port to update metrics for + * @param key + * Id of metric to update + * @param value + * New value + * + * @return + * - -EIO if unable to access shared metrics memory + * - Zero on success + */ +int rte_metrics_update_value( + int port_id, + uint16_t key, + const uint64_t value); + +/** + * Updates a metric set. Note that it is an error to try to + * update across a set boundary. + * + * @param port_id + * Port to update metrics for + * @param key + * Base id of metrics set to update + * @param values + * Set of new values + * @param count + * Number of new values + * + * @return + * - -ERANGE if count exceeds metric set size + * - -EIO if unable to access shared metrics memory + * - Zero on success + */ +int rte_metrics_update_values( + int port_id, + uint16_t key, + const uint64_t *values, + uint32_t count); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/dpdk/lib/librte_metrics/rte_metrics_version.map b/dpdk/lib/librte_metrics/rte_metrics_version.map new file mode 100644 index 00000000..4c5234cd --- /dev/null +++ b/dpdk/lib/librte_metrics/rte_metrics_version.map @@ -0,0 +1,13 @@ +DPDK_17.05 { + global: + + rte_metrics_get_names; + rte_metrics_get_values; + rte_metrics_init; + rte_metrics_reg_name; + rte_metrics_reg_names; + rte_metrics_update_value; + rte_metrics_update_values; + + local: *; +}; diff --git a/dpdk/lib/librte_net/Makefile b/dpdk/lib/librte_net/Makefile index ad2e482d..50c358e5 100644 --- a/dpdk/lib/librte_net/Makefile +++ b/dpdk/lib/librte_net/Makefile @@ -31,10 +31,21 @@ include $(RTE_SDK)/mk/rte.vars.mk +LIB = librte_net.a + CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -O3 +LDLIBS += -lrte_mbuf -lrte_eal + +EXPORT_MAP := rte_net_version.map +LIBABIVER := 1 + +SRCS-$(CONFIG_RTE_LIBRTE_NET) := rte_net.c +SRCS-$(CONFIG_RTE_LIBRTE_NET) += rte_net_crc.c # install includes -SYMLINK-$(CONFIG_RTE_LIBRTE_NET)-include := rte_ip.h rte_tcp.h rte_udp.h rte_sctp.h rte_icmp.h rte_arp.h +SYMLINK-$(CONFIG_RTE_LIBRTE_NET)-include := rte_ip.h rte_tcp.h rte_udp.h rte_esp.h +SYMLINK-$(CONFIG_RTE_LIBRTE_NET)-include += rte_sctp.h rte_icmp.h rte_arp.h +SYMLINK-$(CONFIG_RTE_LIBRTE_NET)-include += rte_ether.h rte_gre.h rte_net.h +SYMLINK-$(CONFIG_RTE_LIBRTE_NET)-include += rte_net_crc.h - -include $(RTE_SDK)/mk/rte.install.mk +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/lib/librte_net/net_crc_neon.h b/dpdk/lib/librte_net/net_crc_neon.h new file mode 100644 index 00000000..cb8f63d9 --- /dev/null +++ b/dpdk/lib/librte_net/net_crc_neon.h @@ -0,0 +1,297 @@ +/* + * BSD LICENSE + * + * Copyright (C) Cavium, Inc. 2017. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _NET_CRC_NEON_H_ +#define _NET_CRC_NEON_H_ + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** PMULL CRC computation context structure */ +struct crc_pmull_ctx { + uint64x2_t rk1_rk2; + uint64x2_t rk5_rk6; + uint64x2_t rk7_rk8; +}; + +struct crc_pmull_ctx crc32_eth_pmull __rte_aligned(16); +struct crc_pmull_ctx crc16_ccitt_pmull __rte_aligned(16); + +/** + * @brief Performs one folding round + * + * Logically function operates as follows: + * DATA = READ_NEXT_16BYTES(); + * F1 = LSB8(FOLD) + * F2 = MSB8(FOLD) + * T1 = CLMUL(F1, RK1) + * T2 = CLMUL(F2, RK2) + * FOLD = XOR(T1, T2, DATA) + * + * @param data_block 16 byte data block + * @param precomp precomputed rk1 constant + * @param fold running 16 byte folded data + * + * @return New 16 byte folded data + */ +static inline uint64x2_t +crcr32_folding_round(uint64x2_t data_block, uint64x2_t precomp, + uint64x2_t fold) +{ + uint64x2_t tmp0 = vreinterpretq_u64_p128(vmull_p64( + vgetq_lane_p64(vreinterpretq_p64_u64(fold), 1), + vgetq_lane_p64(vreinterpretq_p64_u64(precomp), 0))); + + uint64x2_t tmp1 = vreinterpretq_u64_p128(vmull_p64( + vgetq_lane_p64(vreinterpretq_p64_u64(fold), 0), + vgetq_lane_p64(vreinterpretq_p64_u64(precomp), 1))); + + return veorq_u64(tmp1, veorq_u64(data_block, tmp0)); +} + +/** + * Performs reduction from 128 bits to 64 bits + * + * @param data128 128 bits data to be reduced + * @param precomp rk5 and rk6 precomputed constants + * + * @return data reduced to 64 bits + */ +static inline uint64x2_t +crcr32_reduce_128_to_64(uint64x2_t data128, + uint64x2_t precomp) +{ + uint64x2_t tmp0, tmp1, tmp2; + + /* 64b fold */ + tmp0 = vreinterpretq_u64_p128(vmull_p64( + vgetq_lane_p64(vreinterpretq_p64_u64(data128), 0), + vgetq_lane_p64(vreinterpretq_p64_u64(precomp), 0))); + tmp1 = vshift_bytes_right(data128, 8); + tmp0 = veorq_u64(tmp0, tmp1); + + /* 32b fold */ + tmp2 = vshift_bytes_left(tmp0, 4); + tmp1 = vreinterpretq_u64_p128(vmull_p64( + vgetq_lane_p64(vreinterpretq_p64_u64(tmp2), 0), + vgetq_lane_p64(vreinterpretq_p64_u64(precomp), 1))); + + return veorq_u64(tmp1, tmp0); +} + +/** + * Performs Barret's reduction from 64 bits to 32 bits + * + * @param data64 64 bits data to be reduced + * @param precomp rk7 precomputed constant + * + * @return data reduced to 32 bits + */ +static inline uint32_t +crcr32_reduce_64_to_32(uint64x2_t data64, + uint64x2_t precomp) +{ + static uint32_t mask1[4] __rte_aligned(16) = { + 0xffffffff, 0xffffffff, 0x00000000, 0x00000000 + }; + static uint32_t mask2[4] __rte_aligned(16) = { + 0x00000000, 0xffffffff, 0xffffffff, 0xffffffff + }; + uint64x2_t tmp0, tmp1, tmp2; + + tmp0 = vandq_u64(data64, vld1q_u64((uint64_t *)mask2)); + + tmp1 = vreinterpretq_u64_p128(vmull_p64( + vgetq_lane_p64(vreinterpretq_p64_u64(tmp0), 0), + vgetq_lane_p64(vreinterpretq_p64_u64(precomp), 0))); + tmp1 = veorq_u64(tmp1, tmp0); + tmp1 = vandq_u64(tmp1, vld1q_u64((uint64_t *)mask1)); + + tmp2 = vreinterpretq_u64_p128(vmull_p64( + vgetq_lane_p64(vreinterpretq_p64_u64(tmp1), 0), + vgetq_lane_p64(vreinterpretq_p64_u64(precomp), 1))); + tmp2 = veorq_u64(tmp2, tmp1); + tmp2 = veorq_u64(tmp2, tmp0); + + return vgetq_lane_u32(vreinterpretq_u32_u64(tmp2), 2); +} + +static inline uint32_t +crc32_eth_calc_pmull( + const uint8_t *data, + uint32_t data_len, + uint32_t crc, + const struct crc_pmull_ctx *params) +{ + uint64x2_t temp, fold, k; + uint32_t n; + + /* Get CRC init value */ + temp = vreinterpretq_u64_u32(vsetq_lane_u32(crc, vmovq_n_u32(0), 0)); + + /** + * Folding all data into single 16 byte data block + * Assumes: fold holds first 16 bytes of data + */ + if (unlikely(data_len < 32)) { + if (unlikely(data_len == 16)) { + /* 16 bytes */ + fold = vld1q_u64((const uint64_t *)data); + fold = veorq_u64(fold, temp); + goto reduction_128_64; + } + + if (unlikely(data_len < 16)) { + /* 0 to 15 bytes */ + uint8_t buffer[16] __rte_aligned(16); + + memset(buffer, 0, sizeof(buffer)); + memcpy(buffer, data, data_len); + + fold = vld1q_u64((uint64_t *)buffer); + fold = veorq_u64(fold, temp); + if (unlikely(data_len < 4)) { + fold = vshift_bytes_left(fold, 8 - data_len); + goto barret_reduction; + } + fold = vshift_bytes_left(fold, 16 - data_len); + goto reduction_128_64; + } + /* 17 to 31 bytes */ + fold = vld1q_u64((const uint64_t *)data); + fold = veorq_u64(fold, temp); + n = 16; + k = params->rk1_rk2; + goto partial_bytes; + } + + /** At least 32 bytes in the buffer */ + /** Apply CRC initial value */ + fold = vld1q_u64((const uint64_t *)data); + fold = veorq_u64(fold, temp); + + /** Main folding loop - the last 16 bytes is processed separately */ + k = params->rk1_rk2; + for (n = 16; (n + 16) <= data_len; n += 16) { + temp = vld1q_u64((const uint64_t *)&data[n]); + fold = crcr32_folding_round(temp, k, fold); + } + +partial_bytes: + if (likely(n < data_len)) { + uint64x2_t last16, a, b, mask; + uint32_t rem = data_len & 15; + + last16 = vld1q_u64((const uint64_t *)&data[data_len - 16]); + a = vshift_bytes_left(fold, 16 - rem); + b = vshift_bytes_right(fold, rem); + mask = vshift_bytes_left(vdupq_n_u64(-1), 16 - rem); + b = vorrq_u64(b, vandq_u64(mask, last16)); + + /* k = rk1 & rk2 */ + temp = vreinterpretq_u64_p128(vmull_p64( + vgetq_lane_p64(vreinterpretq_p64_u64(a), 1), + vgetq_lane_p64(vreinterpretq_p64_u64(k), 0))); + fold = vreinterpretq_u64_p128(vmull_p64( + vgetq_lane_p64(vreinterpretq_p64_u64(a), 0), + vgetq_lane_p64(vreinterpretq_p64_u64(k), 1))); + fold = veorq_u64(fold, temp); + fold = veorq_u64(fold, b); + } + + /** Reduction 128 -> 32 Assumes: fold holds 128bit folded data */ +reduction_128_64: + k = params->rk5_rk6; + fold = crcr32_reduce_128_to_64(fold, k); + +barret_reduction: + k = params->rk7_rk8; + n = crcr32_reduce_64_to_32(fold, k); + + return n; +} + +static inline void +rte_net_crc_neon_init(void) +{ + /* Initialize CRC16 data */ + uint64_t ccitt_k1_k2[2] = {0x189aeLLU, 0x8e10LLU}; + uint64_t ccitt_k5_k6[2] = {0x189aeLLU, 0x114aaLLU}; + uint64_t ccitt_k7_k8[2] = {0x11c581910LLU, 0x10811LLU}; + + /* Initialize CRC32 data */ + uint64_t eth_k1_k2[2] = {0xccaa009eLLU, 0x1751997d0LLU}; + uint64_t eth_k5_k6[2] = {0xccaa009eLLU, 0x163cd6124LLU}; + uint64_t eth_k7_k8[2] = {0x1f7011640LLU, 0x1db710641LLU}; + + /** Save the params in context structure */ + crc16_ccitt_pmull.rk1_rk2 = vld1q_u64(ccitt_k1_k2); + crc16_ccitt_pmull.rk5_rk6 = vld1q_u64(ccitt_k5_k6); + crc16_ccitt_pmull.rk7_rk8 = vld1q_u64(ccitt_k7_k8); + + /** Save the params in context structure */ + crc32_eth_pmull.rk1_rk2 = vld1q_u64(eth_k1_k2); + crc32_eth_pmull.rk5_rk6 = vld1q_u64(eth_k5_k6); + crc32_eth_pmull.rk7_rk8 = vld1q_u64(eth_k7_k8); +} + +static inline uint32_t +rte_crc16_ccitt_neon_handler(const uint8_t *data, + uint32_t data_len) +{ + return (uint16_t)~crc32_eth_calc_pmull(data, + data_len, + 0xffff, + &crc16_ccitt_pmull); +} + +static inline uint32_t +rte_crc32_eth_neon_handler(const uint8_t *data, + uint32_t data_len) +{ + return ~crc32_eth_calc_pmull(data, + data_len, + 0xffffffffUL, + &crc32_eth_pmull); +} + +#ifdef __cplusplus +} +#endif + +#endif /* _NET_CRC_NEON_H_ */ diff --git a/dpdk/lib/librte_net/net_crc_sse.h b/dpdk/lib/librte_net/net_crc_sse.h new file mode 100644 index 00000000..7eae1479 --- /dev/null +++ b/dpdk/lib/librte_net/net_crc_sse.h @@ -0,0 +1,363 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_NET_CRC_SSE_H_ +#define _RTE_NET_CRC_SSE_H_ + +#include + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** PCLMULQDQ CRC computation context structure */ +struct crc_pclmulqdq_ctx { + __m128i rk1_rk2; + __m128i rk5_rk6; + __m128i rk7_rk8; +}; + +struct crc_pclmulqdq_ctx crc32_eth_pclmulqdq __rte_aligned(16); +struct crc_pclmulqdq_ctx crc16_ccitt_pclmulqdq __rte_aligned(16); +/** + * @brief Performs one folding round + * + * Logically function operates as follows: + * DATA = READ_NEXT_16BYTES(); + * F1 = LSB8(FOLD) + * F2 = MSB8(FOLD) + * T1 = CLMUL(F1, RK1) + * T2 = CLMUL(F2, RK2) + * FOLD = XOR(T1, T2, DATA) + * + * @param data_block + * 16 byte data block + * @param precomp + * Precomputed rk1 constant + * @param fold + * Current16 byte folded data + * + * @return + * New 16 byte folded data + */ +static __rte_always_inline __m128i +crcr32_folding_round(__m128i data_block, + __m128i precomp, + __m128i fold) +{ + __m128i tmp0 = _mm_clmulepi64_si128(fold, precomp, 0x01); + __m128i tmp1 = _mm_clmulepi64_si128(fold, precomp, 0x10); + + return _mm_xor_si128(tmp1, _mm_xor_si128(data_block, tmp0)); +} + +/** + * Performs reduction from 128 bits to 64 bits + * + * @param data128 + * 128 bits data to be reduced + * @param precomp + * precomputed constants rk5, rk6 + * + * @return + * 64 bits reduced data + */ + +static __rte_always_inline __m128i +crcr32_reduce_128_to_64(__m128i data128, __m128i precomp) +{ + __m128i tmp0, tmp1, tmp2; + + /* 64b fold */ + tmp0 = _mm_clmulepi64_si128(data128, precomp, 0x00); + tmp1 = _mm_srli_si128(data128, 8); + tmp0 = _mm_xor_si128(tmp0, tmp1); + + /* 32b fold */ + tmp2 = _mm_slli_si128(tmp0, 4); + tmp1 = _mm_clmulepi64_si128(tmp2, precomp, 0x10); + + return _mm_xor_si128(tmp1, tmp0); +} + +/** + * Performs Barret's reduction from 64 bits to 32 bits + * + * @param data64 + * 64 bits data to be reduced + * @param precomp + * rk7 precomputed constant + * + * @return + * reduced 32 bits data + */ + +static __rte_always_inline uint32_t +crcr32_reduce_64_to_32(__m128i data64, __m128i precomp) +{ + static const uint32_t mask1[4] __rte_aligned(16) = { + 0xffffffff, 0xffffffff, 0x00000000, 0x00000000 + }; + + static const uint32_t mask2[4] __rte_aligned(16) = { + 0x00000000, 0xffffffff, 0xffffffff, 0xffffffff + }; + __m128i tmp0, tmp1, tmp2; + + tmp0 = _mm_and_si128(data64, _mm_load_si128((const __m128i *)mask2)); + + tmp1 = _mm_clmulepi64_si128(tmp0, precomp, 0x00); + tmp1 = _mm_xor_si128(tmp1, tmp0); + tmp1 = _mm_and_si128(tmp1, _mm_load_si128((const __m128i *)mask1)); + + tmp2 = _mm_clmulepi64_si128(tmp1, precomp, 0x10); + tmp2 = _mm_xor_si128(tmp2, tmp1); + tmp2 = _mm_xor_si128(tmp2, tmp0); + + return _mm_extract_epi32(tmp2, 2); +} + +static const uint8_t crc_xmm_shift_tab[48] __rte_aligned(16) = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; + +/** + * Shifts left 128 bit register by specified number of bytes + * + * @param reg + * 128 bit value + * @param num + * number of bytes to shift left reg by (0-16) + * + * @return + * reg << (num * 8) + */ + +static __rte_always_inline __m128i +xmm_shift_left(__m128i reg, const unsigned int num) +{ + const __m128i *p = (const __m128i *)(crc_xmm_shift_tab + 16 - num); + + return _mm_shuffle_epi8(reg, _mm_loadu_si128(p)); +} + +static __rte_always_inline uint32_t +crc32_eth_calc_pclmulqdq( + const uint8_t *data, + uint32_t data_len, + uint32_t crc, + const struct crc_pclmulqdq_ctx *params) +{ + __m128i temp, fold, k; + uint32_t n; + + /* Get CRC init value */ + temp = _mm_insert_epi32(_mm_setzero_si128(), crc, 0); + + /** + * Folding all data into single 16 byte data block + * Assumes: fold holds first 16 bytes of data + */ + + if (unlikely(data_len < 32)) { + if (unlikely(data_len == 16)) { + /* 16 bytes */ + fold = _mm_loadu_si128((const __m128i *)data); + fold = _mm_xor_si128(fold, temp); + goto reduction_128_64; + } + + if (unlikely(data_len < 16)) { + /* 0 to 15 bytes */ + uint8_t buffer[16] __rte_aligned(16); + + memset(buffer, 0, sizeof(buffer)); + memcpy(buffer, data, data_len); + + fold = _mm_load_si128((const __m128i *)buffer); + fold = _mm_xor_si128(fold, temp); + if (unlikely(data_len < 4)) { + fold = xmm_shift_left(fold, 8 - data_len); + goto barret_reduction; + } + fold = xmm_shift_left(fold, 16 - data_len); + goto reduction_128_64; + } + /* 17 to 31 bytes */ + fold = _mm_loadu_si128((const __m128i *)data); + fold = _mm_xor_si128(fold, temp); + n = 16; + k = params->rk1_rk2; + goto partial_bytes; + } + + /** At least 32 bytes in the buffer */ + /** Apply CRC initial value */ + fold = _mm_loadu_si128((const __m128i *)data); + fold = _mm_xor_si128(fold, temp); + + /** Main folding loop - the last 16 bytes is processed separately */ + k = params->rk1_rk2; + for (n = 16; (n + 16) <= data_len; n += 16) { + temp = _mm_loadu_si128((const __m128i *)&data[n]); + fold = crcr32_folding_round(temp, k, fold); + } + +partial_bytes: + if (likely(n < data_len)) { + + const uint32_t mask3[4] __rte_aligned(16) = { + 0x80808080, 0x80808080, 0x80808080, 0x80808080 + }; + + const uint8_t shf_table[32] __rte_aligned(16) = { + 0x00, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f + }; + + __m128i last16, a, b; + + last16 = _mm_loadu_si128((const __m128i *)&data[data_len - 16]); + + temp = _mm_loadu_si128((const __m128i *) + &shf_table[data_len & 15]); + a = _mm_shuffle_epi8(fold, temp); + + temp = _mm_xor_si128(temp, + _mm_load_si128((const __m128i *)mask3)); + b = _mm_shuffle_epi8(fold, temp); + b = _mm_blendv_epi8(b, last16, temp); + + /* k = rk1 & rk2 */ + temp = _mm_clmulepi64_si128(a, k, 0x01); + fold = _mm_clmulepi64_si128(a, k, 0x10); + + fold = _mm_xor_si128(fold, temp); + fold = _mm_xor_si128(fold, b); + } + + /** Reduction 128 -> 32 Assumes: fold holds 128bit folded data */ +reduction_128_64: + k = params->rk5_rk6; + fold = crcr32_reduce_128_to_64(fold, k); + +barret_reduction: + k = params->rk7_rk8; + n = crcr32_reduce_64_to_32(fold, k); + + return n; +} + + +static inline void +rte_net_crc_sse42_init(void) +{ + uint64_t k1, k2, k5, k6; + uint64_t p = 0, q = 0; + + /** Initialize CRC16 data */ + k1 = 0x189aeLLU; + k2 = 0x8e10LLU; + k5 = 0x189aeLLU; + k6 = 0x114aaLLU; + q = 0x11c581910LLU; + p = 0x10811LLU; + + /** Save the params in context structure */ + crc16_ccitt_pclmulqdq.rk1_rk2 = + _mm_setr_epi64(_mm_cvtsi64_m64(k1), _mm_cvtsi64_m64(k2)); + crc16_ccitt_pclmulqdq.rk5_rk6 = + _mm_setr_epi64(_mm_cvtsi64_m64(k5), _mm_cvtsi64_m64(k6)); + crc16_ccitt_pclmulqdq.rk7_rk8 = + _mm_setr_epi64(_mm_cvtsi64_m64(q), _mm_cvtsi64_m64(p)); + + /** Initialize CRC32 data */ + k1 = 0xccaa009eLLU; + k2 = 0x1751997d0LLU; + k5 = 0xccaa009eLLU; + k6 = 0x163cd6124LLU; + q = 0x1f7011640LLU; + p = 0x1db710641LLU; + + /** Save the params in context structure */ + crc32_eth_pclmulqdq.rk1_rk2 = + _mm_setr_epi64(_mm_cvtsi64_m64(k1), _mm_cvtsi64_m64(k2)); + crc32_eth_pclmulqdq.rk5_rk6 = + _mm_setr_epi64(_mm_cvtsi64_m64(k5), _mm_cvtsi64_m64(k6)); + crc32_eth_pclmulqdq.rk7_rk8 = + _mm_setr_epi64(_mm_cvtsi64_m64(q), _mm_cvtsi64_m64(p)); + + /** + * Reset the register as following calculation may + * use other data types such as float, double, etc. + */ + _mm_empty(); + +} + +static inline uint32_t +rte_crc16_ccitt_sse42_handler(const uint8_t *data, + uint32_t data_len) +{ + /** return 16-bit CRC value */ + return (uint16_t)~crc32_eth_calc_pclmulqdq(data, + data_len, + 0xffff, + &crc16_ccitt_pclmulqdq); +} + +static inline uint32_t +rte_crc32_eth_sse42_handler(const uint8_t *data, + uint32_t data_len) +{ + return ~crc32_eth_calc_pclmulqdq(data, + data_len, + 0xffffffffUL, + &crc32_eth_pclmulqdq); +} + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_NET_CRC_SSE_H_ */ diff --git a/dpdk/lib/librte_net/rte_esp.h b/dpdk/lib/librte_net/rte_esp.h new file mode 100644 index 00000000..148c06e0 --- /dev/null +++ b/dpdk/lib/librte_net/rte_esp.h @@ -0,0 +1,60 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2016-2017, Mellanox Technologies. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_ESP_H_ +#define _RTE_ESP_H_ + +/** + * @file + * + * ESP-related defines + */ + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * ESP Header + */ +struct esp_hdr { + rte_be32_t spi; /**< Security Parameters Index */ + rte_be32_t seq; /**< packet sequence number */ +} __attribute__((__packed__)); + +#ifdef __cplusplus +} +#endif + +#endif /* RTE_ESP_H_ */ diff --git a/dpdk/lib/librte_net/rte_ether.h b/dpdk/lib/librte_net/rte_ether.h new file mode 100644 index 00000000..06d7b486 --- /dev/null +++ b/dpdk/lib/librte_net/rte_ether.h @@ -0,0 +1,420 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_ETHER_H_ +#define _RTE_ETHER_H_ + +/** + * @file + * + * Ethernet Helpers in RTE + */ + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include + +#include +#include +#include +#include + +#define ETHER_ADDR_LEN 6 /**< Length of Ethernet address. */ +#define ETHER_TYPE_LEN 2 /**< Length of Ethernet type field. */ +#define ETHER_CRC_LEN 4 /**< Length of Ethernet CRC. */ +#define ETHER_HDR_LEN \ + (ETHER_ADDR_LEN * 2 + ETHER_TYPE_LEN) /**< Length of Ethernet header. */ +#define ETHER_MIN_LEN 64 /**< Minimum frame len, including CRC. */ +#define ETHER_MAX_LEN 1518 /**< Maximum frame len, including CRC. */ +#define ETHER_MTU \ + (ETHER_MAX_LEN - ETHER_HDR_LEN - ETHER_CRC_LEN) /**< Ethernet MTU. */ + +#define ETHER_MAX_VLAN_FRAME_LEN \ + (ETHER_MAX_LEN + 4) /**< Maximum VLAN frame length, including CRC. */ + +#define ETHER_MAX_JUMBO_FRAME_LEN \ + 0x3F00 /**< Maximum Jumbo frame length, including CRC. */ + +#define ETHER_MAX_VLAN_ID 4095 /**< Maximum VLAN ID. */ + +#define ETHER_MIN_MTU 68 /**< Minimum MTU for IPv4 packets, see RFC 791. */ + +/** + * Ethernet address: + * A universally administered address is uniquely assigned to a device by its + * manufacturer. The first three octets (in transmission order) contain the + * Organizationally Unique Identifier (OUI). The following three (MAC-48 and + * EUI-48) octets are assigned by that organization with the only constraint + * of uniqueness. + * A locally administered address is assigned to a device by a network + * administrator and does not contain OUIs. + * See http://standards.ieee.org/regauth/groupmac/tutorial.html + */ +struct ether_addr { + uint8_t addr_bytes[ETHER_ADDR_LEN]; /**< Addr bytes in tx order */ +} __attribute__((__packed__)); + +#define ETHER_LOCAL_ADMIN_ADDR 0x02 /**< Locally assigned Eth. address. */ +#define ETHER_GROUP_ADDR 0x01 /**< Multicast or broadcast Eth. address. */ + +/** + * Check if two Ethernet addresses are the same. + * + * @param ea1 + * A pointer to the first ether_addr structure containing + * the ethernet address. + * @param ea2 + * A pointer to the second ether_addr structure containing + * the ethernet address. + * + * @return + * True (1) if the given two ethernet address are the same; + * False (0) otherwise. + */ +static inline int is_same_ether_addr(const struct ether_addr *ea1, + const struct ether_addr *ea2) +{ + int i; + for (i = 0; i < ETHER_ADDR_LEN; i++) + if (ea1->addr_bytes[i] != ea2->addr_bytes[i]) + return 0; + return 1; +} + +/** + * Check if an Ethernet address is filled with zeros. + * + * @param ea + * A pointer to a ether_addr structure containing the ethernet address + * to check. + * @return + * True (1) if the given ethernet address is filled with zeros; + * false (0) otherwise. + */ +static inline int is_zero_ether_addr(const struct ether_addr *ea) +{ + int i; + for (i = 0; i < ETHER_ADDR_LEN; i++) + if (ea->addr_bytes[i] != 0x00) + return 0; + return 1; +} + +/** + * Check if an Ethernet address is a unicast address. + * + * @param ea + * A pointer to a ether_addr structure containing the ethernet address + * to check. + * @return + * True (1) if the given ethernet address is a unicast address; + * false (0) otherwise. + */ +static inline int is_unicast_ether_addr(const struct ether_addr *ea) +{ + return (ea->addr_bytes[0] & ETHER_GROUP_ADDR) == 0; +} + +/** + * Check if an Ethernet address is a multicast address. + * + * @param ea + * A pointer to a ether_addr structure containing the ethernet address + * to check. + * @return + * True (1) if the given ethernet address is a multicast address; + * false (0) otherwise. + */ +static inline int is_multicast_ether_addr(const struct ether_addr *ea) +{ + return ea->addr_bytes[0] & ETHER_GROUP_ADDR; +} + +/** + * Check if an Ethernet address is a broadcast address. + * + * @param ea + * A pointer to a ether_addr structure containing the ethernet address + * to check. + * @return + * True (1) if the given ethernet address is a broadcast address; + * false (0) otherwise. + */ +static inline int is_broadcast_ether_addr(const struct ether_addr *ea) +{ + const unaligned_uint16_t *ea_words = (const unaligned_uint16_t *)ea; + + return (ea_words[0] == 0xFFFF && ea_words[1] == 0xFFFF && + ea_words[2] == 0xFFFF); +} + +/** + * Check if an Ethernet address is a universally assigned address. + * + * @param ea + * A pointer to a ether_addr structure containing the ethernet address + * to check. + * @return + * True (1) if the given ethernet address is a universally assigned address; + * false (0) otherwise. + */ +static inline int is_universal_ether_addr(const struct ether_addr *ea) +{ + return (ea->addr_bytes[0] & ETHER_LOCAL_ADMIN_ADDR) == 0; +} + +/** + * Check if an Ethernet address is a locally assigned address. + * + * @param ea + * A pointer to a ether_addr structure containing the ethernet address + * to check. + * @return + * True (1) if the given ethernet address is a locally assigned address; + * false (0) otherwise. + */ +static inline int is_local_admin_ether_addr(const struct ether_addr *ea) +{ + return (ea->addr_bytes[0] & ETHER_LOCAL_ADMIN_ADDR) != 0; +} + +/** + * Check if an Ethernet address is a valid address. Checks that the address is a + * unicast address and is not filled with zeros. + * + * @param ea + * A pointer to a ether_addr structure containing the ethernet address + * to check. + * @return + * True (1) if the given ethernet address is valid; + * false (0) otherwise. + */ +static inline int is_valid_assigned_ether_addr(const struct ether_addr *ea) +{ + return is_unicast_ether_addr(ea) && (!is_zero_ether_addr(ea)); +} + +/** + * Generate a random Ethernet address that is locally administered + * and not multicast. + * @param addr + * A pointer to Ethernet address. + */ +static inline void eth_random_addr(uint8_t *addr) +{ + uint64_t rand = rte_rand(); + uint8_t *p = (uint8_t *)&rand; + + rte_memcpy(addr, p, ETHER_ADDR_LEN); + addr[0] &= ~ETHER_GROUP_ADDR; /* clear multicast bit */ + addr[0] |= ETHER_LOCAL_ADMIN_ADDR; /* set local assignment bit */ +} + +/** + * Fast copy an Ethernet address. + * + * @param ea_from + * A pointer to a ether_addr structure holding the Ethernet address to copy. + * @param ea_to + * A pointer to a ether_addr structure where to copy the Ethernet address. + */ +static inline void ether_addr_copy(const struct ether_addr *ea_from, + struct ether_addr *ea_to) +{ +#ifdef __INTEL_COMPILER + uint16_t *from_words = (uint16_t *)(ea_from->addr_bytes); + uint16_t *to_words = (uint16_t *)(ea_to->addr_bytes); + + to_words[0] = from_words[0]; + to_words[1] = from_words[1]; + to_words[2] = from_words[2]; +#else + /* + * Use the common way, because of a strange gcc warning. + */ + *ea_to = *ea_from; +#endif +} + +#define ETHER_ADDR_FMT_SIZE 18 +/** + * Format 48bits Ethernet address in pattern xx:xx:xx:xx:xx:xx. + * + * @param buf + * A pointer to buffer contains the formatted MAC address. + * @param size + * The format buffer size. + * @param eth_addr + * A pointer to a ether_addr structure. + */ +static inline void +ether_format_addr(char *buf, uint16_t size, + const struct ether_addr *eth_addr) +{ + snprintf(buf, size, "%02X:%02X:%02X:%02X:%02X:%02X", + eth_addr->addr_bytes[0], + eth_addr->addr_bytes[1], + eth_addr->addr_bytes[2], + eth_addr->addr_bytes[3], + eth_addr->addr_bytes[4], + eth_addr->addr_bytes[5]); +} + +/** + * Ethernet header: Contains the destination address, source address + * and frame type. + */ +struct ether_hdr { + struct ether_addr d_addr; /**< Destination address. */ + struct ether_addr s_addr; /**< Source address. */ + uint16_t ether_type; /**< Frame type. */ +} __attribute__((__packed__)); + +/** + * Ethernet VLAN Header. + * Contains the 16-bit VLAN Tag Control Identifier and the Ethernet type + * of the encapsulated frame. + */ +struct vlan_hdr { + uint16_t vlan_tci; /**< Priority (3) + CFI (1) + Identifier Code (12) */ + uint16_t eth_proto;/**< Ethernet type of encapsulated frame. */ +} __attribute__((__packed__)); + +/** + * VXLAN protocol header. + * Contains the 8-bit flag, 24-bit VXLAN Network Identifier and + * Reserved fields (24 bits and 8 bits) + */ +struct vxlan_hdr { + uint32_t vx_flags; /**< flag (8) + Reserved (24). */ + uint32_t vx_vni; /**< VNI (24) + Reserved (8). */ +} __attribute__((__packed__)); + +/* Ethernet frame types */ +#define ETHER_TYPE_IPv4 0x0800 /**< IPv4 Protocol. */ +#define ETHER_TYPE_IPv6 0x86DD /**< IPv6 Protocol. */ +#define ETHER_TYPE_ARP 0x0806 /**< Arp Protocol. */ +#define ETHER_TYPE_RARP 0x8035 /**< Reverse Arp Protocol. */ +#define ETHER_TYPE_VLAN 0x8100 /**< IEEE 802.1Q VLAN tagging. */ +#define ETHER_TYPE_QINQ 0x88A8 /**< IEEE 802.1ad QinQ tagging. */ +#define ETHER_TYPE_1588 0x88F7 /**< IEEE 802.1AS 1588 Precise Time Protocol. */ +#define ETHER_TYPE_SLOW 0x8809 /**< Slow protocols (LACP and Marker). */ +#define ETHER_TYPE_TEB 0x6558 /**< Transparent Ethernet Bridging. */ +#define ETHER_TYPE_LLDP 0x88CC /**< LLDP Protocol. */ + +#define ETHER_VXLAN_HLEN (sizeof(struct udp_hdr) + sizeof(struct vxlan_hdr)) +/**< VXLAN tunnel header length. */ + +/** + * Extract VLAN tag information into mbuf + * + * Software version of VLAN stripping + * + * @param m + * The packet mbuf. + * @return + * - 0: Success + * - 1: not a vlan packet + */ +static inline int rte_vlan_strip(struct rte_mbuf *m) +{ + struct ether_hdr *eh + = rte_pktmbuf_mtod(m, struct ether_hdr *); + + if (eh->ether_type != rte_cpu_to_be_16(ETHER_TYPE_VLAN)) + return -1; + + struct vlan_hdr *vh = (struct vlan_hdr *)(eh + 1); + m->ol_flags |= PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED; + m->vlan_tci = rte_be_to_cpu_16(vh->vlan_tci); + + /* Copy ether header over rather than moving whole packet */ + memmove(rte_pktmbuf_adj(m, sizeof(struct vlan_hdr)), + eh, 2 * ETHER_ADDR_LEN); + + return 0; +} + +/** + * Insert VLAN tag into mbuf. + * + * Software version of VLAN unstripping + * + * @param m + * The packet mbuf. + * @return + * - 0: On success + * -EPERM: mbuf is is shared overwriting would be unsafe + * -ENOSPC: not enough headroom in mbuf + */ +static inline int rte_vlan_insert(struct rte_mbuf **m) +{ + struct ether_hdr *oh, *nh; + struct vlan_hdr *vh; + + /* Can't insert header if mbuf is shared */ + if (rte_mbuf_refcnt_read(*m) > 1) { + struct rte_mbuf *copy; + + copy = rte_pktmbuf_clone(*m, (*m)->pool); + if (unlikely(copy == NULL)) + return -ENOMEM; + rte_pktmbuf_free(*m); + *m = copy; + } + + oh = rte_pktmbuf_mtod(*m, struct ether_hdr *); + nh = (struct ether_hdr *) + rte_pktmbuf_prepend(*m, sizeof(struct vlan_hdr)); + if (nh == NULL) + return -ENOSPC; + + memmove(nh, oh, 2 * ETHER_ADDR_LEN); + nh->ether_type = rte_cpu_to_be_16(ETHER_TYPE_VLAN); + + vh = (struct vlan_hdr *) (nh + 1); + vh->vlan_tci = rte_cpu_to_be_16((*m)->vlan_tci); + + (*m)->ol_flags &= ~PKT_RX_VLAN_STRIPPED; + + return 0; +} + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_ETHER_H_ */ diff --git a/dpdk/lib/librte_net/rte_gre.h b/dpdk/lib/librte_net/rte_gre.h new file mode 100644 index 00000000..46568ff5 --- /dev/null +++ b/dpdk/lib/librte_net/rte_gre.h @@ -0,0 +1,71 @@ +/*- + * BSD LICENSE + * + * Copyright 2016 6WIND S.A. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_GRE_H_ +#define _RTE_GRE_H_ + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * GRE Header + */ +struct gre_hdr { +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + uint16_t res2:4; /**< Reserved */ + uint16_t s:1; /**< Sequence Number Present bit */ + uint16_t k:1; /**< Key Present bit */ + uint16_t res1:1; /**< Reserved */ + uint16_t c:1; /**< Checksum Present bit */ + uint16_t ver:3; /**< Version Number */ + uint16_t res3:5; /**< Reserved */ +#elif RTE_BYTE_ORDER == RTE_BIG_ENDIAN + uint16_t c:1; /**< Checksum Present bit */ + uint16_t res1:1; /**< Reserved */ + uint16_t k:1; /**< Key Present bit */ + uint16_t s:1; /**< Sequence Number Present bit */ + uint16_t res2:4; /**< Reserved */ + uint16_t res3:5; /**< Reserved */ + uint16_t ver:3; /**< Version Number */ +#endif + uint16_t proto; /**< Protocol Type */ +} __attribute__((__packed__)); + +#ifdef __cplusplus +} +#endif + +#endif /* RTE_GRE_H_ */ diff --git a/dpdk/lib/librte_net/rte_ip.h b/dpdk/lib/librte_net/rte_ip.h index 5b7554ab..73ec398f 100644 --- a/dpdk/lib/librte_net/rte_ip.h +++ b/dpdk/lib/librte_net/rte_ip.h @@ -229,6 +229,77 @@ rte_raw_cksum(const void *buf, size_t len) return __rte_raw_cksum_reduce(sum); } +/** + * Compute the raw (non complemented) checksum of a packet. + * + * @param m + * The pointer to the mbuf. + * @param off + * The offset in bytes to start the checksum. + * @param len + * The length in bytes of the data to checksum. + * @param cksum + * A pointer to the checksum, filled on success. + * @return + * 0 on success, -1 on error (bad length or offset). + */ +static inline int +rte_raw_cksum_mbuf(const struct rte_mbuf *m, uint32_t off, uint32_t len, + uint16_t *cksum) +{ + const struct rte_mbuf *seg; + const char *buf; + uint32_t sum, tmp; + uint32_t seglen, done; + + /* easy case: all data in the first segment */ + if (off + len <= rte_pktmbuf_data_len(m)) { + *cksum = rte_raw_cksum(rte_pktmbuf_mtod_offset(m, + const char *, off), len); + return 0; + } + + if (unlikely(off + len > rte_pktmbuf_pkt_len(m))) + return -1; /* invalid params, return a dummy value */ + + /* else browse the segment to find offset */ + seglen = 0; + for (seg = m; seg != NULL; seg = seg->next) { + seglen = rte_pktmbuf_data_len(seg); + if (off < seglen) + break; + off -= seglen; + } + seglen -= off; + buf = rte_pktmbuf_mtod_offset(seg, const char *, off); + if (seglen >= len) { + /* all in one segment */ + *cksum = rte_raw_cksum(buf, len); + return 0; + } + + /* hard case: process checksum of several segments */ + sum = 0; + done = 0; + for (;;) { + tmp = __rte_raw_cksum(buf, seglen, 0); + if (done & 1) + tmp = rte_bswap16(tmp); + sum += tmp; + done += seglen; + if (done == len) + break; + seg = seg->next; + buf = rte_pktmbuf_mtod(seg, const char *); + seglen = rte_pktmbuf_data_len(seg); + if (seglen > len - done) + seglen = len - done; + } + + *cksum = __rte_raw_cksum_reduce(sum); + return 0; +} + /** * Process the IPv4 checksum of an IPv4 header. * diff --git a/dpdk/lib/librte_net/rte_net.c b/dpdk/lib/librte_net/rte_net.c new file mode 100644 index 00000000..a3ca0403 --- /dev/null +++ b/dpdk/lib/librte_net/rte_net.c @@ -0,0 +1,518 @@ +/*- + * BSD LICENSE + * + * Copyright 2016 6WIND S.A. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of 6WIND S.A. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* get l3 packet type from ip6 next protocol */ +static uint32_t +ptype_l3_ip6(uint8_t ip6_proto) +{ + static const uint32_t ip6_ext_proto_map[256] = { + [IPPROTO_HOPOPTS] = RTE_PTYPE_L3_IPV6_EXT - RTE_PTYPE_L3_IPV6, + [IPPROTO_ROUTING] = RTE_PTYPE_L3_IPV6_EXT - RTE_PTYPE_L3_IPV6, + [IPPROTO_FRAGMENT] = RTE_PTYPE_L3_IPV6_EXT - RTE_PTYPE_L3_IPV6, + [IPPROTO_ESP] = RTE_PTYPE_L3_IPV6_EXT - RTE_PTYPE_L3_IPV6, + [IPPROTO_AH] = RTE_PTYPE_L3_IPV6_EXT - RTE_PTYPE_L3_IPV6, + [IPPROTO_DSTOPTS] = RTE_PTYPE_L3_IPV6_EXT - RTE_PTYPE_L3_IPV6, + }; + + return RTE_PTYPE_L3_IPV6 + ip6_ext_proto_map[ip6_proto]; +} + +/* get l3 packet type from ip version and header length */ +static uint32_t +ptype_l3_ip(uint8_t ipv_ihl) +{ + static const uint32_t ptype_l3_ip_proto_map[256] = { + [0x45] = RTE_PTYPE_L3_IPV4, + [0x46] = RTE_PTYPE_L3_IPV4_EXT, + [0x47] = RTE_PTYPE_L3_IPV4_EXT, + [0x48] = RTE_PTYPE_L3_IPV4_EXT, + [0x49] = RTE_PTYPE_L3_IPV4_EXT, + [0x4A] = RTE_PTYPE_L3_IPV4_EXT, + [0x4B] = RTE_PTYPE_L3_IPV4_EXT, + [0x4C] = RTE_PTYPE_L3_IPV4_EXT, + [0x4D] = RTE_PTYPE_L3_IPV4_EXT, + [0x4E] = RTE_PTYPE_L3_IPV4_EXT, + [0x4F] = RTE_PTYPE_L3_IPV4_EXT, + }; + + return ptype_l3_ip_proto_map[ipv_ihl]; +} + +/* get l4 packet type from proto */ +static uint32_t +ptype_l4(uint8_t proto) +{ + static const uint32_t ptype_l4_proto[256] = { + [IPPROTO_UDP] = RTE_PTYPE_L4_UDP, + [IPPROTO_TCP] = RTE_PTYPE_L4_TCP, + [IPPROTO_SCTP] = RTE_PTYPE_L4_SCTP, + }; + + return ptype_l4_proto[proto]; +} + +/* get inner l3 packet type from ip6 next protocol */ +static uint32_t +ptype_inner_l3_ip6(uint8_t ip6_proto) +{ + static const uint32_t ptype_inner_ip6_ext_proto_map[256] = { + [IPPROTO_HOPOPTS] = RTE_PTYPE_INNER_L3_IPV6_EXT - + RTE_PTYPE_INNER_L3_IPV6, + [IPPROTO_ROUTING] = RTE_PTYPE_INNER_L3_IPV6_EXT - + RTE_PTYPE_INNER_L3_IPV6, + [IPPROTO_FRAGMENT] = RTE_PTYPE_INNER_L3_IPV6_EXT - + RTE_PTYPE_INNER_L3_IPV6, + [IPPROTO_ESP] = RTE_PTYPE_INNER_L3_IPV6_EXT - + RTE_PTYPE_INNER_L3_IPV6, + [IPPROTO_AH] = RTE_PTYPE_INNER_L3_IPV6_EXT - + RTE_PTYPE_INNER_L3_IPV6, + [IPPROTO_DSTOPTS] = RTE_PTYPE_INNER_L3_IPV6_EXT - + RTE_PTYPE_INNER_L3_IPV6, + }; + + return RTE_PTYPE_INNER_L3_IPV6 + + ptype_inner_ip6_ext_proto_map[ip6_proto]; +} + +/* get inner l3 packet type from ip version and header length */ +static uint32_t +ptype_inner_l3_ip(uint8_t ipv_ihl) +{ + static const uint32_t ptype_inner_l3_ip_proto_map[256] = { + [0x45] = RTE_PTYPE_INNER_L3_IPV4, + [0x46] = RTE_PTYPE_INNER_L3_IPV4_EXT, + [0x47] = RTE_PTYPE_INNER_L3_IPV4_EXT, + [0x48] = RTE_PTYPE_INNER_L3_IPV4_EXT, + [0x49] = RTE_PTYPE_INNER_L3_IPV4_EXT, + [0x4A] = RTE_PTYPE_INNER_L3_IPV4_EXT, + [0x4B] = RTE_PTYPE_INNER_L3_IPV4_EXT, + [0x4C] = RTE_PTYPE_INNER_L3_IPV4_EXT, + [0x4D] = RTE_PTYPE_INNER_L3_IPV4_EXT, + [0x4E] = RTE_PTYPE_INNER_L3_IPV4_EXT, + [0x4F] = RTE_PTYPE_INNER_L3_IPV4_EXT, + }; + + return ptype_inner_l3_ip_proto_map[ipv_ihl]; +} + +/* get inner l4 packet type from proto */ +static uint32_t +ptype_inner_l4(uint8_t proto) +{ + static const uint32_t ptype_inner_l4_proto[256] = { + [IPPROTO_UDP] = RTE_PTYPE_INNER_L4_UDP, + [IPPROTO_TCP] = RTE_PTYPE_INNER_L4_TCP, + [IPPROTO_SCTP] = RTE_PTYPE_INNER_L4_SCTP, + }; + + return ptype_inner_l4_proto[proto]; +} + +/* get the tunnel packet type if any, update proto and off. */ +static uint32_t +ptype_tunnel(uint16_t *proto, const struct rte_mbuf *m, + uint32_t *off) +{ + switch (*proto) { + case IPPROTO_GRE: { + static const uint8_t opt_len[16] = { + [0x0] = 4, + [0x1] = 8, + [0x2] = 8, + [0x8] = 8, + [0x3] = 12, + [0x9] = 12, + [0xa] = 12, + [0xb] = 16, + }; + const struct gre_hdr *gh; + struct gre_hdr gh_copy; + uint16_t flags; + + gh = rte_pktmbuf_read(m, *off, sizeof(*gh), &gh_copy); + if (unlikely(gh == NULL)) + return 0; + + flags = rte_be_to_cpu_16(*(const uint16_t *)gh); + flags >>= 12; + if (opt_len[flags] == 0) + return 0; + + *off += opt_len[flags]; + *proto = gh->proto; + if (*proto == rte_cpu_to_be_16(ETHER_TYPE_TEB)) + return RTE_PTYPE_TUNNEL_NVGRE; + else + return RTE_PTYPE_TUNNEL_GRE; + } + case IPPROTO_IPIP: + *proto = rte_cpu_to_be_16(ETHER_TYPE_IPv4); + return RTE_PTYPE_TUNNEL_IP; + case IPPROTO_IPV6: + *proto = rte_cpu_to_be_16(ETHER_TYPE_IPv6); + return RTE_PTYPE_TUNNEL_IP; /* IP is also valid for IPv6 */ + default: + return 0; + } +} + +/* get the ipv4 header length */ +static uint8_t +ip4_hlen(const struct ipv4_hdr *hdr) +{ + return (hdr->version_ihl & 0xf) * 4; +} + +/* parse ipv6 extended headers, update offset and return next proto */ +static uint16_t +skip_ip6_ext(uint16_t proto, const struct rte_mbuf *m, uint32_t *off, + int *frag) +{ + struct ext_hdr { + uint8_t next_hdr; + uint8_t len; + }; + const struct ext_hdr *xh; + struct ext_hdr xh_copy; + unsigned int i; + + *frag = 0; + +#define MAX_EXT_HDRS 5 + for (i = 0; i < MAX_EXT_HDRS; i++) { + switch (proto) { + case IPPROTO_HOPOPTS: + case IPPROTO_ROUTING: + case IPPROTO_DSTOPTS: + xh = rte_pktmbuf_read(m, *off, sizeof(*xh), + &xh_copy); + if (xh == NULL) + return 0; + *off += (xh->len + 1) * 8; + proto = xh->next_hdr; + break; + case IPPROTO_FRAGMENT: + xh = rte_pktmbuf_read(m, *off, sizeof(*xh), + &xh_copy); + if (xh == NULL) + return 0; + *off += 8; + proto = xh->next_hdr; + *frag = 1; + return proto; /* this is always the last ext hdr */ + case IPPROTO_NONE: + return 0; + default: + return proto; + } + } + return 0; +} + +/* parse mbuf data to get packet type */ +uint32_t rte_net_get_ptype(const struct rte_mbuf *m, + struct rte_net_hdr_lens *hdr_lens, uint32_t layers) +{ + struct rte_net_hdr_lens local_hdr_lens; + const struct ether_hdr *eh; + struct ether_hdr eh_copy; + uint32_t pkt_type = RTE_PTYPE_L2_ETHER; + uint32_t off = 0; + uint16_t proto; + + if (hdr_lens == NULL) + hdr_lens = &local_hdr_lens; + + eh = rte_pktmbuf_read(m, off, sizeof(*eh), &eh_copy); + if (unlikely(eh == NULL)) + return 0; + proto = eh->ether_type; + off = sizeof(*eh); + hdr_lens->l2_len = off; + + if ((layers & RTE_PTYPE_L2_MASK) == 0) + return 0; + + if (proto == rte_cpu_to_be_16(ETHER_TYPE_IPv4)) + goto l3; /* fast path if packet is IPv4 */ + + if (proto == rte_cpu_to_be_16(ETHER_TYPE_VLAN)) { + const struct vlan_hdr *vh; + struct vlan_hdr vh_copy; + + pkt_type = RTE_PTYPE_L2_ETHER_VLAN; + vh = rte_pktmbuf_read(m, off, sizeof(*vh), &vh_copy); + if (unlikely(vh == NULL)) + return pkt_type; + off += sizeof(*vh); + hdr_lens->l2_len += sizeof(*vh); + proto = vh->eth_proto; + } else if (proto == rte_cpu_to_be_16(ETHER_TYPE_QINQ)) { + const struct vlan_hdr *vh; + struct vlan_hdr vh_copy; + + pkt_type = RTE_PTYPE_L2_ETHER_QINQ; + vh = rte_pktmbuf_read(m, off + sizeof(*vh), sizeof(*vh), + &vh_copy); + if (unlikely(vh == NULL)) + return pkt_type; + off += 2 * sizeof(*vh); + hdr_lens->l2_len += 2 * sizeof(*vh); + proto = vh->eth_proto; + } + + l3: + if ((layers & RTE_PTYPE_L3_MASK) == 0) + return pkt_type; + + if (proto == rte_cpu_to_be_16(ETHER_TYPE_IPv4)) { + const struct ipv4_hdr *ip4h; + struct ipv4_hdr ip4h_copy; + + ip4h = rte_pktmbuf_read(m, off, sizeof(*ip4h), &ip4h_copy); + if (unlikely(ip4h == NULL)) + return pkt_type; + + pkt_type |= ptype_l3_ip(ip4h->version_ihl); + hdr_lens->l3_len = ip4_hlen(ip4h); + off += hdr_lens->l3_len; + + if ((layers & RTE_PTYPE_L4_MASK) == 0) + return pkt_type; + + if (ip4h->fragment_offset & rte_cpu_to_be_16( + IPV4_HDR_OFFSET_MASK | IPV4_HDR_MF_FLAG)) { + pkt_type |= RTE_PTYPE_L4_FRAG; + hdr_lens->l4_len = 0; + return pkt_type; + } + proto = ip4h->next_proto_id; + pkt_type |= ptype_l4(proto); + } else if (proto == rte_cpu_to_be_16(ETHER_TYPE_IPv6)) { + const struct ipv6_hdr *ip6h; + struct ipv6_hdr ip6h_copy; + int frag = 0; + + ip6h = rte_pktmbuf_read(m, off, sizeof(*ip6h), &ip6h_copy); + if (unlikely(ip6h == NULL)) + return pkt_type; + + proto = ip6h->proto; + hdr_lens->l3_len = sizeof(*ip6h); + off += hdr_lens->l3_len; + pkt_type |= ptype_l3_ip6(proto); + if ((pkt_type & RTE_PTYPE_L3_MASK) == RTE_PTYPE_L3_IPV6_EXT) { + proto = skip_ip6_ext(proto, m, &off, &frag); + hdr_lens->l3_len = off - hdr_lens->l2_len; + } + if (proto == 0) + return pkt_type; + + if ((layers & RTE_PTYPE_L4_MASK) == 0) + return pkt_type; + + if (frag) { + pkt_type |= RTE_PTYPE_L4_FRAG; + hdr_lens->l4_len = 0; + return pkt_type; + } + pkt_type |= ptype_l4(proto); + } + + if ((pkt_type & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_UDP) { + hdr_lens->l4_len = sizeof(struct udp_hdr); + return pkt_type; + } else if ((pkt_type & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_TCP) { + const struct tcp_hdr *th; + struct tcp_hdr th_copy; + + th = rte_pktmbuf_read(m, off, sizeof(*th), &th_copy); + if (unlikely(th == NULL)) + return pkt_type & (RTE_PTYPE_L2_MASK | + RTE_PTYPE_L3_MASK); + hdr_lens->l4_len = (th->data_off & 0xf0) >> 2; + return pkt_type; + } else if ((pkt_type & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_SCTP) { + hdr_lens->l4_len = sizeof(struct sctp_hdr); + return pkt_type; + } else { + uint32_t prev_off = off; + + hdr_lens->l4_len = 0; + + if ((layers & RTE_PTYPE_TUNNEL_MASK) == 0) + return pkt_type; + + pkt_type |= ptype_tunnel(&proto, m, &off); + hdr_lens->tunnel_len = off - prev_off; + } + + /* same job for inner header: we need to duplicate the code + * because the packet types do not have the same value. + */ + if ((layers & RTE_PTYPE_INNER_L2_MASK) == 0) + return pkt_type; + + hdr_lens->inner_l2_len = 0; + if (proto == rte_cpu_to_be_16(ETHER_TYPE_TEB)) { + eh = rte_pktmbuf_read(m, off, sizeof(*eh), &eh_copy); + if (unlikely(eh == NULL)) + return pkt_type; + pkt_type |= RTE_PTYPE_INNER_L2_ETHER; + proto = eh->ether_type; + off += sizeof(*eh); + hdr_lens->inner_l2_len = sizeof(*eh); + } + + if (proto == rte_cpu_to_be_16(ETHER_TYPE_VLAN)) { + const struct vlan_hdr *vh; + struct vlan_hdr vh_copy; + + pkt_type &= ~RTE_PTYPE_INNER_L2_MASK; + pkt_type |= RTE_PTYPE_INNER_L2_ETHER_VLAN; + vh = rte_pktmbuf_read(m, off, sizeof(*vh), &vh_copy); + if (unlikely(vh == NULL)) + return pkt_type; + off += sizeof(*vh); + hdr_lens->inner_l2_len += sizeof(*vh); + proto = vh->eth_proto; + } else if (proto == rte_cpu_to_be_16(ETHER_TYPE_QINQ)) { + const struct vlan_hdr *vh; + struct vlan_hdr vh_copy; + + pkt_type &= ~RTE_PTYPE_INNER_L2_MASK; + pkt_type |= RTE_PTYPE_INNER_L2_ETHER_QINQ; + vh = rte_pktmbuf_read(m, off + sizeof(*vh), sizeof(*vh), + &vh_copy); + if (unlikely(vh == NULL)) + return pkt_type; + off += 2 * sizeof(*vh); + hdr_lens->inner_l2_len += 2 * sizeof(*vh); + proto = vh->eth_proto; + } + + if ((layers & RTE_PTYPE_INNER_L3_MASK) == 0) + return pkt_type; + + if (proto == rte_cpu_to_be_16(ETHER_TYPE_IPv4)) { + const struct ipv4_hdr *ip4h; + struct ipv4_hdr ip4h_copy; + + ip4h = rte_pktmbuf_read(m, off, sizeof(*ip4h), &ip4h_copy); + if (unlikely(ip4h == NULL)) + return pkt_type; + + pkt_type |= ptype_inner_l3_ip(ip4h->version_ihl); + hdr_lens->inner_l3_len = ip4_hlen(ip4h); + off += hdr_lens->inner_l3_len; + + if ((layers & RTE_PTYPE_INNER_L4_MASK) == 0) + return pkt_type; + if (ip4h->fragment_offset & + rte_cpu_to_be_16(IPV4_HDR_OFFSET_MASK | + IPV4_HDR_MF_FLAG)) { + pkt_type |= RTE_PTYPE_INNER_L4_FRAG; + hdr_lens->inner_l4_len = 0; + return pkt_type; + } + proto = ip4h->next_proto_id; + pkt_type |= ptype_inner_l4(proto); + } else if (proto == rte_cpu_to_be_16(ETHER_TYPE_IPv6)) { + const struct ipv6_hdr *ip6h; + struct ipv6_hdr ip6h_copy; + int frag = 0; + + ip6h = rte_pktmbuf_read(m, off, sizeof(*ip6h), &ip6h_copy); + if (unlikely(ip6h == NULL)) + return pkt_type; + + proto = ip6h->proto; + hdr_lens->inner_l3_len = sizeof(*ip6h); + off += hdr_lens->inner_l3_len; + pkt_type |= ptype_inner_l3_ip6(proto); + if ((pkt_type & RTE_PTYPE_INNER_L3_MASK) == + RTE_PTYPE_INNER_L3_IPV6_EXT) { + uint32_t prev_off; + + prev_off = off; + proto = skip_ip6_ext(proto, m, &off, &frag); + hdr_lens->inner_l3_len += off - prev_off; + } + if (proto == 0) + return pkt_type; + + if ((layers & RTE_PTYPE_INNER_L4_MASK) == 0) + return pkt_type; + + if (frag) { + pkt_type |= RTE_PTYPE_INNER_L4_FRAG; + hdr_lens->inner_l4_len = 0; + return pkt_type; + } + pkt_type |= ptype_inner_l4(proto); + } + + if ((pkt_type & RTE_PTYPE_INNER_L4_MASK) == RTE_PTYPE_INNER_L4_UDP) { + hdr_lens->inner_l4_len = sizeof(struct udp_hdr); + } else if ((pkt_type & RTE_PTYPE_INNER_L4_MASK) == + RTE_PTYPE_INNER_L4_TCP) { + const struct tcp_hdr *th; + struct tcp_hdr th_copy; + + th = rte_pktmbuf_read(m, off, sizeof(*th), &th_copy); + if (unlikely(th == NULL)) + return pkt_type & (RTE_PTYPE_INNER_L2_MASK | + RTE_PTYPE_INNER_L3_MASK); + hdr_lens->inner_l4_len = (th->data_off & 0xf0) >> 2; + } else if ((pkt_type & RTE_PTYPE_INNER_L4_MASK) == + RTE_PTYPE_INNER_L4_SCTP) { + hdr_lens->inner_l4_len = sizeof(struct sctp_hdr); + } else { + hdr_lens->inner_l4_len = 0; + } + + return pkt_type; +} diff --git a/dpdk/lib/librte_net/rte_net.h b/dpdk/lib/librte_net/rte_net.h new file mode 100644 index 00000000..79c764ad --- /dev/null +++ b/dpdk/lib/librte_net/rte_net.h @@ -0,0 +1,205 @@ +/*- + * BSD LICENSE + * + * Copyright 2016 6WIND S.A. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_NET_PTYPE_H_ +#define _RTE_NET_PTYPE_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include + +/** + * Structure containing header lengths associated to a packet, filled + * by rte_net_get_ptype(). + */ +struct rte_net_hdr_lens { + uint8_t l2_len; + uint8_t l3_len; + uint8_t l4_len; + uint8_t tunnel_len; + uint8_t inner_l2_len; + uint8_t inner_l3_len; + uint8_t inner_l4_len; +}; + +/** + * Parse an Ethernet packet to get its packet type. + * + * This function parses the network headers in mbuf data and return its + * packet type. + * + * If it is provided by the user, it also fills a rte_net_hdr_lens + * structure that contains the lengths of the parsed network + * headers. Each length field is valid only if the associated packet + * type is set. For instance, hdr_lens->l2_len is valid only if + * (retval & RTE_PTYPE_L2_MASK) != RTE_PTYPE_UNKNOWN. + * + * Supported packet types are: + * L2: Ether, Vlan, QinQ + * L3: IPv4, IPv6 + * L4: TCP, UDP, SCTP + * Tunnels: IPv4, IPv6, Gre, Nvgre + * + * @param m + * The packet mbuf to be parsed. + * @param hdr_lens + * A pointer to a structure where the header lengths will be returned, + * or NULL. + * @param layers + * List of layers to parse. The function will stop at the first + * empty layer. Examples: + * - To parse all known layers, use RTE_PTYPE_ALL_MASK. + * - To parse only L2 and L3, use RTE_PTYPE_L2_MASK | RTE_PTYPE_L3_MASK + * @return + * The packet type of the packet. + */ +uint32_t rte_net_get_ptype(const struct rte_mbuf *m, + struct rte_net_hdr_lens *hdr_lens, uint32_t layers); + +/** + * Prepare pseudo header checksum + * + * This function prepares pseudo header checksum for TSO and non-TSO tcp/udp in + * provided mbufs packet data and based on the requested offload flags. + * + * - for non-TSO tcp/udp packets full pseudo-header checksum is counted and set + * in packet data, + * - for TSO the IP payload length is not included in pseudo header. + * + * This function expects that used headers are in the first data segment of + * mbuf, are not fragmented and can be safely modified. + * + * @param m + * The packet mbuf to be fixed. + * @param ol_flags + * TX offloads flags to use with this packet. + * @return + * 0 if checksum is initialized properly + */ +static inline int +rte_net_intel_cksum_flags_prepare(struct rte_mbuf *m, uint64_t ol_flags) +{ + struct ipv4_hdr *ipv4_hdr; + struct ipv6_hdr *ipv6_hdr; + struct tcp_hdr *tcp_hdr; + struct udp_hdr *udp_hdr; + uint64_t inner_l3_offset = m->l2_len; + + if ((ol_flags & PKT_TX_OUTER_IP_CKSUM) || + (ol_flags & PKT_TX_OUTER_IPV6)) + inner_l3_offset += m->outer_l2_len + m->outer_l3_len; + + if ((ol_flags & PKT_TX_UDP_CKSUM) == PKT_TX_UDP_CKSUM) { + if (ol_flags & PKT_TX_IPV4) { + ipv4_hdr = rte_pktmbuf_mtod_offset(m, struct ipv4_hdr *, + inner_l3_offset); + + if (ol_flags & PKT_TX_IP_CKSUM) + ipv4_hdr->hdr_checksum = 0; + + udp_hdr = (struct udp_hdr *)((char *)ipv4_hdr + + m->l3_len); + udp_hdr->dgram_cksum = rte_ipv4_phdr_cksum(ipv4_hdr, + ol_flags); + } else { + ipv6_hdr = rte_pktmbuf_mtod_offset(m, struct ipv6_hdr *, + inner_l3_offset); + /* non-TSO udp */ + udp_hdr = rte_pktmbuf_mtod_offset(m, struct udp_hdr *, + inner_l3_offset + m->l3_len); + udp_hdr->dgram_cksum = rte_ipv6_phdr_cksum(ipv6_hdr, + ol_flags); + } + } else if ((ol_flags & PKT_TX_TCP_CKSUM) || + (ol_flags & PKT_TX_TCP_SEG)) { + if (ol_flags & PKT_TX_IPV4) { + ipv4_hdr = rte_pktmbuf_mtod_offset(m, struct ipv4_hdr *, + inner_l3_offset); + + if (ol_flags & PKT_TX_IP_CKSUM) + ipv4_hdr->hdr_checksum = 0; + + /* non-TSO tcp or TSO */ + tcp_hdr = (struct tcp_hdr *)((char *)ipv4_hdr + + m->l3_len); + tcp_hdr->cksum = rte_ipv4_phdr_cksum(ipv4_hdr, + ol_flags); + } else { + ipv6_hdr = rte_pktmbuf_mtod_offset(m, struct ipv6_hdr *, + inner_l3_offset); + /* non-TSO tcp or TSO */ + tcp_hdr = rte_pktmbuf_mtod_offset(m, struct tcp_hdr *, + inner_l3_offset + m->l3_len); + tcp_hdr->cksum = rte_ipv6_phdr_cksum(ipv6_hdr, + ol_flags); + } + } + + return 0; +} + +/** + * Prepare pseudo header checksum + * + * This function prepares pseudo header checksum for TSO and non-TSO tcp/udp in + * provided mbufs packet data. + * + * - for non-TSO tcp/udp packets full pseudo-header checksum is counted and set + * in packet data, + * - for TSO the IP payload length is not included in pseudo header. + * + * This function expects that used headers are in the first data segment of + * mbuf, are not fragmented and can be safely modified. + * + * @param m + * The packet mbuf to be fixed. + * @return + * 0 if checksum is initialized properly + */ +static inline int +rte_net_intel_cksum_prepare(struct rte_mbuf *m) +{ + return rte_net_intel_cksum_flags_prepare(m, m->ol_flags); +} + +#ifdef __cplusplus +} +#endif + + +#endif /* _RTE_NET_PTYPE_H_ */ diff --git a/dpdk/lib/librte_net/rte_net_crc.c b/dpdk/lib/librte_net/rte_net_crc.c new file mode 100644 index 00000000..0c1bf51a --- /dev/null +++ b/dpdk/lib/librte_net/rte_net_crc.c @@ -0,0 +1,225 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include + +#include +#include +#include + +#if defined(RTE_ARCH_X86_64) && defined(RTE_MACHINE_CPUFLAG_PCLMULQDQ) +#define X86_64_SSE42_PCLMULQDQ 1 +#elif defined(RTE_ARCH_ARM64) && defined(RTE_MACHINE_CPUFLAG_PMULL) +#define ARM64_NEON_PMULL 1 +#endif + +#ifdef X86_64_SSE42_PCLMULQDQ +#include +#elif defined ARM64_NEON_PMULL +#include +#endif + +/* crc tables */ +static uint32_t crc32_eth_lut[CRC_LUT_SIZE]; +static uint32_t crc16_ccitt_lut[CRC_LUT_SIZE]; + +static uint32_t +rte_crc16_ccitt_handler(const uint8_t *data, uint32_t data_len); + +static uint32_t +rte_crc32_eth_handler(const uint8_t *data, uint32_t data_len); + +typedef uint32_t +(*rte_net_crc_handler)(const uint8_t *data, uint32_t data_len); + +static rte_net_crc_handler *handlers; + +static rte_net_crc_handler handlers_scalar[] = { + [RTE_NET_CRC16_CCITT] = rte_crc16_ccitt_handler, + [RTE_NET_CRC32_ETH] = rte_crc32_eth_handler, +}; + +#ifdef X86_64_SSE42_PCLMULQDQ +static rte_net_crc_handler handlers_sse42[] = { + [RTE_NET_CRC16_CCITT] = rte_crc16_ccitt_sse42_handler, + [RTE_NET_CRC32_ETH] = rte_crc32_eth_sse42_handler, +}; +#elif defined ARM64_NEON_PMULL +static rte_net_crc_handler handlers_neon[] = { + [RTE_NET_CRC16_CCITT] = rte_crc16_ccitt_neon_handler, + [RTE_NET_CRC32_ETH] = rte_crc32_eth_neon_handler, +}; +#endif + +/** + * Reflect the bits about the middle + * + * @param val + * value to be reflected + * + * @return + * reflected value + */ +static uint32_t +reflect_32bits(uint32_t val) +{ + uint32_t i, res = 0; + + for (i = 0; i < 32; i++) + if ((val & (1 << i)) != 0) + res |= (uint32_t)(1 << (31 - i)); + + return res; +} + +static void +crc32_eth_init_lut(uint32_t poly, + uint32_t *lut) +{ + uint32_t i, j; + + for (i = 0; i < CRC_LUT_SIZE; i++) { + uint32_t crc = reflect_32bits(i); + + for (j = 0; j < 8; j++) { + if (crc & 0x80000000L) + crc = (crc << 1) ^ poly; + else + crc <<= 1; + } + lut[i] = reflect_32bits(crc); + } +} + +static __rte_always_inline uint32_t +crc32_eth_calc_lut(const uint8_t *data, + uint32_t data_len, + uint32_t crc, + const uint32_t *lut) +{ + while (data_len--) + crc = lut[(crc ^ *data++) & 0xffL] ^ (crc >> 8); + + return crc; +} + +static void +rte_net_crc_scalar_init(void) +{ + /* 32-bit crc init */ + crc32_eth_init_lut(CRC32_ETH_POLYNOMIAL, crc32_eth_lut); + + /* 16-bit CRC init */ + crc32_eth_init_lut(CRC16_CCITT_POLYNOMIAL << 16, crc16_ccitt_lut); +} + +static inline uint32_t +rte_crc16_ccitt_handler(const uint8_t *data, uint32_t data_len) +{ + /* return 16-bit CRC value */ + return (uint16_t)~crc32_eth_calc_lut(data, + data_len, + 0xffff, + crc16_ccitt_lut); +} + +static inline uint32_t +rte_crc32_eth_handler(const uint8_t *data, uint32_t data_len) +{ + /* return 32-bit CRC value */ + return ~crc32_eth_calc_lut(data, + data_len, + 0xffffffffUL, + crc32_eth_lut); +} + +void +rte_net_crc_set_alg(enum rte_net_crc_alg alg) +{ + switch (alg) { +#ifdef X86_64_SSE42_PCLMULQDQ + case RTE_NET_CRC_SSE42: + handlers = handlers_sse42; + break; +#elif defined ARM64_NEON_PMULL + /* fall-through */ + case RTE_NET_CRC_NEON: + if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_PMULL)) { + handlers = handlers_neon; + break; + } +#endif + /* fall-through */ + case RTE_NET_CRC_SCALAR: + /* fall-through */ + default: + handlers = handlers_scalar; + break; + } +} + +uint32_t +rte_net_crc_calc(const void *data, + uint32_t data_len, + enum rte_net_crc_type type) +{ + uint32_t ret; + rte_net_crc_handler f_handle; + + f_handle = handlers[type]; + ret = f_handle(data, data_len); + + return ret; +} + +/* Select highest available crc algorithm as default one */ +RTE_INIT(rte_net_crc_init) +{ + enum rte_net_crc_alg alg = RTE_NET_CRC_SCALAR; + + rte_net_crc_scalar_init(); + +#ifdef X86_64_SSE42_PCLMULQDQ + alg = RTE_NET_CRC_SSE42; + rte_net_crc_sse42_init(); +#elif defined ARM64_NEON_PMULL + if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_PMULL)) { + alg = RTE_NET_CRC_NEON; + rte_net_crc_neon_init(); + } +#endif + + rte_net_crc_set_alg(alg); +} diff --git a/dpdk/lib/librte_net/rte_net_crc.h b/dpdk/lib/librte_net/rte_net_crc.h new file mode 100644 index 00000000..d01cf4b4 --- /dev/null +++ b/dpdk/lib/librte_net/rte_net_crc.h @@ -0,0 +1,100 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_NET_CRC_H_ +#define _RTE_NET_CRC_H_ + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** CRC polynomials */ +#define CRC32_ETH_POLYNOMIAL 0x04c11db7UL +#define CRC16_CCITT_POLYNOMIAL 0x1021U + +#define CRC_LUT_SIZE 256 + +/** CRC types */ +enum rte_net_crc_type { + RTE_NET_CRC16_CCITT = 0, + RTE_NET_CRC32_ETH, + RTE_NET_CRC_REQS +}; + +/** CRC compute algorithm */ +enum rte_net_crc_alg { + RTE_NET_CRC_SCALAR = 0, + RTE_NET_CRC_SSE42, + RTE_NET_CRC_NEON, +}; + +/** + * This API set the CRC computation algorithm (i.e. scalar version, + * x86 64-bit sse4.2 intrinsic version, etc.) and internal data + * structure. + * + * @param alg + * This parameter is used to select the CRC implementation version. + * - RTE_NET_CRC_SCALAR + * - RTE_NET_CRC_SSE42 (Use 64-bit SSE4.2 intrinsic) + * - RTE_NET_CRC_NEON (Use ARM Neon intrinsic) + */ +void +rte_net_crc_set_alg(enum rte_net_crc_alg alg); + +/** + * CRC compute API + * + * @param data + * Pointer to the packet data for CRC computation + * @param data_len + * Data length for CRC computation + * @param type + * CRC type (enum rte_net_crc_type) + * + * @return + * CRC value + */ +uint32_t +rte_net_crc_calc(const void *data, + uint32_t data_len, + enum rte_net_crc_type type); + +#ifdef __cplusplus +} +#endif + + +#endif /* _RTE_NET_CRC_H_ */ diff --git a/dpdk/lib/librte_net/rte_net_version.map b/dpdk/lib/librte_net/rte_net_version.map new file mode 100644 index 00000000..687c40ea --- /dev/null +++ b/dpdk/lib/librte_net/rte_net_version.map @@ -0,0 +1,14 @@ +DPDK_16.11 { + global: + rte_net_get_ptype; + + local: *; +}; + +DPDK_17.05 { + global: + + rte_net_crc_calc; + rte_net_crc_set_alg; + +} DPDK_16.11; diff --git a/dpdk/lib/librte_pci/Makefile b/dpdk/lib/librte_pci/Makefile new file mode 100644 index 00000000..fe213ea6 --- /dev/null +++ b/dpdk/lib/librte_pci/Makefile @@ -0,0 +1,49 @@ +# BSD LICENSE +# +# Copyright(c) 2017 6WIND S.A. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of 6WIND nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +include $(RTE_SDK)/mk/rte.vars.mk + +# library name +LIB = librte_pci.a + +CFLAGS := -I$(SRCDIR) $(CFLAGS) +CFLAGS += $(WERROR_FLAGS) -O3 +LDLIBS += -lrte_eal + +EXPORT_MAP := rte_pci_version.map + +LIBABIVER := 1 + +SRCS-$(CONFIG_RTE_LIBRTE_PCI) += rte_pci.c + +SYMLINK-$(CONFIG_RTE_LIBRTE_PCI)-include += rte_pci.h + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/lib/librte_pci/rte_pci.c b/dpdk/lib/librte_pci/rte_pci.c new file mode 100644 index 00000000..0160fc1e --- /dev/null +++ b/dpdk/lib/librte_pci/rte_pci.c @@ -0,0 +1,212 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * Copyright 2013-2014 6WIND S.A. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "rte_pci.h" + +static inline const char * +get_u8_pciaddr_field(const char *in, void *_u8, char dlm) +{ + unsigned long val; + uint8_t *u8 = _u8; + char *end; + + errno = 0; + val = strtoul(in, &end, 16); + if (errno != 0 || end[0] != dlm || val > UINT8_MAX) { + errno = errno ? errno : EINVAL; + return NULL; + } + *u8 = (uint8_t)val; + return end + 1; +} + +static int +pci_bdf_parse(const char *input, struct rte_pci_addr *dev_addr) +{ + const char *in = input; + + dev_addr->domain = 0; + in = get_u8_pciaddr_field(in, &dev_addr->bus, ':'); + if (in == NULL) + return -EINVAL; + in = get_u8_pciaddr_field(in, &dev_addr->devid, '.'); + if (in == NULL) + return -EINVAL; + in = get_u8_pciaddr_field(in, &dev_addr->function, '\0'); + if (in == NULL) + return -EINVAL; + return 0; +} + +static int +pci_dbdf_parse(const char *input, struct rte_pci_addr *dev_addr) +{ + const char *in = input; + unsigned long val; + char *end; + + errno = 0; + val = strtoul(in, &end, 16); + if (errno != 0 || end[0] != ':' || val > UINT16_MAX) + return -EINVAL; + dev_addr->domain = (uint16_t)val; + in = end + 1; + in = get_u8_pciaddr_field(in, &dev_addr->bus, ':'); + if (in == NULL) + return -EINVAL; + in = get_u8_pciaddr_field(in, &dev_addr->devid, '.'); + if (in == NULL) + return -EINVAL; + in = get_u8_pciaddr_field(in, &dev_addr->function, '\0'); + if (in == NULL) + return -EINVAL; + return 0; +} + +int +eal_parse_pci_BDF(const char *input, struct rte_pci_addr *dev_addr) +{ + return pci_bdf_parse(input, dev_addr); +} + +int +eal_parse_pci_DomBDF(const char *input, struct rte_pci_addr *dev_addr) +{ + return pci_dbdf_parse(input, dev_addr); +} + +void +rte_pci_device_name(const struct rte_pci_addr *addr, + char *output, size_t size) +{ + RTE_VERIFY(size >= PCI_PRI_STR_SIZE); + RTE_VERIFY(snprintf(output, size, PCI_PRI_FMT, + addr->domain, addr->bus, + addr->devid, addr->function) >= 0); +} + +int +rte_eal_compare_pci_addr(const struct rte_pci_addr *addr, + const struct rte_pci_addr *addr2) +{ + return rte_pci_addr_cmp(addr, addr2); +} + +int +rte_pci_addr_cmp(const struct rte_pci_addr *addr, + const struct rte_pci_addr *addr2) +{ + uint64_t dev_addr, dev_addr2; + + if ((addr == NULL) || (addr2 == NULL)) + return -1; + + dev_addr = ((uint64_t)addr->domain << 24) | + (addr->bus << 16) | (addr->devid << 8) | addr->function; + dev_addr2 = ((uint64_t)addr2->domain << 24) | + (addr2->bus << 16) | (addr2->devid << 8) | addr2->function; + + if (dev_addr > dev_addr2) + return 1; + else if (dev_addr < dev_addr2) + return -1; + else + return 0; +} + +int +rte_pci_addr_parse(const char *str, struct rte_pci_addr *addr) +{ + if (pci_bdf_parse(str, addr) == 0 || + pci_dbdf_parse(str, addr) == 0) + return 0; + return -1; +} + + +/* map a particular resource from a file */ +void * +pci_map_resource(void *requested_addr, int fd, off_t offset, size_t size, + int additional_flags) +{ + void *mapaddr; + + /* Map the PCI memory resource of device */ + mapaddr = mmap(requested_addr, size, PROT_READ | PROT_WRITE, + MAP_SHARED | additional_flags, fd, offset); + if (mapaddr == MAP_FAILED) { + RTE_LOG(ERR, EAL, "%s(): cannot mmap(%d, %p, 0x%lx, 0x%lx): %s (%p)\n", + __func__, fd, requested_addr, + (unsigned long)size, (unsigned long)offset, + strerror(errno), mapaddr); + } else + RTE_LOG(DEBUG, EAL, " PCI memory mapped at %p\n", mapaddr); + + return mapaddr; +} + +/* unmap a particular resource */ +void +pci_unmap_resource(void *requested_addr, size_t size) +{ + if (requested_addr == NULL) + return; + + /* Unmap the PCI memory resource of device */ + if (munmap(requested_addr, size)) { + RTE_LOG(ERR, EAL, "%s(): cannot munmap(%p, 0x%lx): %s\n", + __func__, requested_addr, (unsigned long)size, + strerror(errno)); + } else + RTE_LOG(DEBUG, EAL, " PCI memory unmapped at %p\n", + requested_addr); +} diff --git a/dpdk/lib/librte_pci/rte_pci.h b/dpdk/lib/librte_pci/rte_pci.h new file mode 100644 index 00000000..4f2cd187 --- /dev/null +++ b/dpdk/lib/librte_pci/rte_pci.h @@ -0,0 +1,263 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * Copyright 2013-2014 6WIND S.A. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_PCI_H_ +#define _RTE_PCI_H_ + +/** + * @file + * + * RTE PCI Library + */ + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +/** Formatting string for PCI device identifier: Ex: 0000:00:01.0 */ +#define PCI_PRI_FMT "%.4" PRIx16 ":%.2" PRIx8 ":%.2" PRIx8 ".%" PRIx8 +#define PCI_PRI_STR_SIZE sizeof("XXXXXXXX:XX:XX.X") + +/** Short formatting string, without domain, for PCI device: Ex: 00:01.0 */ +#define PCI_SHORT_PRI_FMT "%.2" PRIx8 ":%.2" PRIx8 ".%" PRIx8 + +/** Nb. of values in PCI device identifier format string. */ +#define PCI_FMT_NVAL 4 + +/** Nb. of values in PCI resource format. */ +#define PCI_RESOURCE_FMT_NVAL 3 + +/** Maximum number of PCI resources. */ +#define PCI_MAX_RESOURCE 6 + +/** + * A structure describing an ID for a PCI driver. Each driver provides a + * table of these IDs for each device that it supports. + */ +struct rte_pci_id { + uint32_t class_id; /**< Class ID or RTE_CLASS_ANY_ID. */ + uint16_t vendor_id; /**< Vendor ID or PCI_ANY_ID. */ + uint16_t device_id; /**< Device ID or PCI_ANY_ID. */ + uint16_t subsystem_vendor_id; /**< Subsystem vendor ID or PCI_ANY_ID. */ + uint16_t subsystem_device_id; /**< Subsystem device ID or PCI_ANY_ID. */ +}; + +/** + * A structure describing the location of a PCI device. + */ +struct rte_pci_addr { + uint32_t domain; /**< Device domain */ + uint8_t bus; /**< Device bus */ + uint8_t devid; /**< Device ID */ + uint8_t function; /**< Device function. */ +}; + +/** Any PCI device identifier (vendor, device, ...) */ +#define PCI_ANY_ID (0xffff) +#define RTE_CLASS_ANY_ID (0xffffff) + +/** + * A structure describing a PCI mapping. + */ +struct pci_map { + void *addr; + char *path; + uint64_t offset; + uint64_t size; + uint64_t phaddr; +}; + +struct pci_msix_table { + int bar_index; + uint32_t offset; + uint32_t size; +}; + +/** + * A structure describing a mapped PCI resource. + * For multi-process we need to reproduce all PCI mappings in secondary + * processes, so save them in a tailq. + */ +struct mapped_pci_resource { + TAILQ_ENTRY(mapped_pci_resource) next; + + struct rte_pci_addr pci_addr; + char path[PATH_MAX]; + int nb_maps; + struct pci_map maps[PCI_MAX_RESOURCE]; + struct pci_msix_table msix_table; +}; + + +/** mapped pci device list */ +TAILQ_HEAD(mapped_pci_res_list, mapped_pci_resource); + +/** + * @deprecated + * Utility function to produce a PCI Bus-Device-Function value + * given a string representation. Assumes that the BDF is provided without + * a domain prefix (i.e. domain returned is always 0) + * + * @param input + * The input string to be parsed. Should have the format XX:XX.X + * @param dev_addr + * The PCI Bus-Device-Function address to be returned. + * Domain will always be returned as 0 + * @return + * 0 on success, negative on error. + */ +int eal_parse_pci_BDF(const char *input, struct rte_pci_addr *dev_addr); + +/** + * @deprecated + * Utility function to produce a PCI Bus-Device-Function value + * given a string representation. Assumes that the BDF is provided including + * a domain prefix. + * + * @param input + * The input string to be parsed. Should have the format XXXX:XX:XX.X + * @param dev_addr + * The PCI Bus-Device-Function address to be returned + * @return + * 0 on success, negative on error. + */ +int eal_parse_pci_DomBDF(const char *input, struct rte_pci_addr *dev_addr); + +/** + * Utility function to write a pci device name, this device name can later be + * used to retrieve the corresponding rte_pci_addr using eal_parse_pci_* + * BDF helpers. + * + * @param addr + * The PCI Bus-Device-Function address + * @param output + * The output buffer string + * @param size + * The output buffer size + */ +void rte_pci_device_name(const struct rte_pci_addr *addr, + char *output, size_t size); + +/** + * @deprecated + * Utility function to compare two PCI device addresses. + * + * @param addr + * The PCI Bus-Device-Function address to compare + * @param addr2 + * The PCI Bus-Device-Function address to compare + * @return + * 0 on equal PCI address. + * Positive on addr is greater than addr2. + * Negative on addr is less than addr2, or error. + */ +int rte_eal_compare_pci_addr(const struct rte_pci_addr *addr, + const struct rte_pci_addr *addr2); + +/** + * Utility function to compare two PCI device addresses. + * + * @param addr + * The PCI Bus-Device-Function address to compare + * @param addr2 + * The PCI Bus-Device-Function address to compare + * @return + * 0 on equal PCI address. + * Positive on addr is greater than addr2. + * Negative on addr is less than addr2, or error. + */ +int rte_pci_addr_cmp(const struct rte_pci_addr *addr, + const struct rte_pci_addr *addr2); + + +/** + * Utility function to parse a string into a PCI location. + * + * @param str + * The string to parse + * @param addr + * The reference to the structure where the location + * is stored. + * @return + * 0 on success + * <0 otherwise + */ +int rte_pci_addr_parse(const char *str, struct rte_pci_addr *addr); + +/** + * Map a particular resource from a file. + * + * @param requested_addr + * The starting address for the new mapping range. + * @param fd + * The file descriptor. + * @param offset + * The offset for the mapping range. + * @param size + * The size for the mapping range. + * @param additional_flags + * The additional flags for the mapping range. + * @return + * - On success, the function returns a pointer to the mapped area. + * - On error, the value MAP_FAILED is returned. + */ +void *pci_map_resource(void *requested_addr, int fd, off_t offset, + size_t size, int additional_flags); + +/** + * Unmap a particular resource. + * + * @param requested_addr + * The address for the unmapping range. + * @param size + * The size for the unmapping range. + */ +void pci_unmap_resource(void *requested_addr, size_t size); + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_PCI_H_ */ diff --git a/dpdk/lib/librte_pci/rte_pci_version.map b/dpdk/lib/librte_pci/rte_pci_version.map new file mode 100644 index 00000000..15d93d95 --- /dev/null +++ b/dpdk/lib/librte_pci/rte_pci_version.map @@ -0,0 +1,15 @@ +DPDK_17.11 { + global: + + eal_parse_pci_BDF; + eal_parse_pci_DomBDF; + rte_pci_addr_cmp; + rte_pci_addr_parse; + rte_pci_device_name; + pci_map_resource; + pci_unmap_resource; + rte_eal_compare_pci_addr; + rte_pci_device_name; + + local: *; +}; diff --git a/dpdk/lib/librte_pdump/Makefile b/dpdk/lib/librte_pdump/Makefile index 166441a2..11c3e4e9 100644 --- a/dpdk/lib/librte_pdump/Makefile +++ b/dpdk/lib/librte_pdump/Makefile @@ -37,10 +37,11 @@ LIB = librte_pdump.a CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -O3 CFLAGS += -D_GNU_SOURCE LDLIBS += -lpthread +LDLIBS += -lrte_eal -lrte_mempool -lrte_mbuf -lrte_ethdev EXPORT_MAP := rte_pdump_version.map -LIBABIVER := 1 +LIBABIVER := 2 # all source are stored in SRCS-y SRCS-$(CONFIG_RTE_LIBRTE_PDUMP) := rte_pdump.c @@ -48,10 +49,4 @@ SRCS-$(CONFIG_RTE_LIBRTE_PDUMP) := rte_pdump.c # install this header file SYMLINK-$(CONFIG_RTE_LIBRTE_PDUMP)-include := rte_pdump.h -# this lib depends upon: -DEPDIRS-$(CONFIG_RTE_LIBRTE_PDUMP) += lib/librte_mbuf -DEPDIRS-$(CONFIG_RTE_LIBRTE_PDUMP) += lib/librte_mempool -DEPDIRS-$(CONFIG_RTE_LIBRTE_PDUMP) += lib/librte_eal -DEPDIRS-$(CONFIG_RTE_LIBRTE_PDUMP) += lib/librte_ether - include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/lib/librte_pdump/rte_pdump.c b/dpdk/lib/librte_pdump/rte_pdump.c index ea5ccd98..44dcc953 100644 --- a/dpdk/lib/librte_pdump/rte_pdump.c +++ b/dpdk/lib/librte_pdump/rte_pdump.c @@ -46,7 +46,6 @@ #include #include #include -#include #include "rte_pdump.h" @@ -140,7 +139,7 @@ pdump_pktmbuf_copy(struct rte_mbuf *m, struct rte_mempool *mp) { struct rte_mbuf *m_dup, *seg, **prev; uint32_t pktlen; - uint8_t nseg; + uint16_t nseg; m_dup = rte_pktmbuf_alloc(mp); if (unlikely(m_dup == NULL)) @@ -154,6 +153,8 @@ pdump_pktmbuf_copy(struct rte_mbuf *m, struct rte_mempool *mp) do { nseg++; if (pdump_pktmbuf_copy_data(seg, m) < 0) { + if (seg != m_dup) + rte_pktmbuf_free_seg(seg); rte_pktmbuf_free(m_dup); return NULL; } @@ -197,7 +198,7 @@ pdump_copy(struct rte_mbuf **pkts, uint16_t nb_pkts, void *user_params) dup_bufs[d_pkts++] = p; } - ring_enq = rte_ring_enqueue_burst(ring, (void *)dup_bufs, d_pkts); + ring_enq = rte_ring_enqueue_burst(ring, (void *)dup_bufs, d_pkts, NULL); if (unlikely(ring_enq < d_pkts)) { RTE_LOG(DEBUG, PDUMP, "only %d of packets enqueued to ring\n", ring_enq); @@ -208,7 +209,7 @@ pdump_copy(struct rte_mbuf **pkts, uint16_t nb_pkts, void *user_params) } static uint16_t -pdump_rx(uint8_t port __rte_unused, uint16_t qidx __rte_unused, +pdump_rx(uint16_t port __rte_unused, uint16_t qidx __rte_unused, struct rte_mbuf **pkts, uint16_t nb_pkts, uint16_t max_pkts __rte_unused, void *user_params) @@ -218,7 +219,7 @@ pdump_rx(uint8_t port __rte_unused, uint16_t qidx __rte_unused, } static uint16_t -pdump_tx(uint8_t port __rte_unused, uint16_t qidx __rte_unused, +pdump_tx(uint16_t port __rte_unused, uint16_t qidx __rte_unused, struct rte_mbuf **pkts, uint16_t nb_pkts, void *user_params) { pdump_copy(pkts, nb_pkts, user_params); @@ -226,30 +227,7 @@ pdump_tx(uint8_t port __rte_unused, uint16_t qidx __rte_unused, } static int -pdump_get_dombdf(char *device_id, char *domBDF, size_t len) -{ - int ret; - struct rte_pci_addr dev_addr = {0}; - - /* identify if device_id is pci address or name */ - ret = eal_parse_pci_DomBDF(device_id, &dev_addr); - if (ret < 0) - return -1; - - if (dev_addr.domain) - ret = snprintf(domBDF, len, "%u:%u:%u.%u", dev_addr.domain, - dev_addr.bus, dev_addr.devid, - dev_addr.function); - else - ret = snprintf(domBDF, len, "%u:%u.%u", dev_addr.bus, - dev_addr.devid, - dev_addr.function); - - return ret; -} - -static int -pdump_regitser_rx_callbacks(uint16_t end_q, uint8_t port, uint16_t queue, +pdump_register_rx_callbacks(uint16_t end_q, uint16_t port, uint16_t queue, struct rte_ring *ring, struct rte_mempool *mp, uint16_t operation) { @@ -292,7 +270,7 @@ pdump_regitser_rx_callbacks(uint16_t end_q, uint8_t port, uint16_t queue, if (ret < 0) { RTE_LOG(ERR, PDUMP, "failed to remove rx callback, errno=%d\n", - rte_errno); + -ret); return ret; } cbs->cb = NULL; @@ -303,7 +281,7 @@ pdump_regitser_rx_callbacks(uint16_t end_q, uint8_t port, uint16_t queue, } static int -pdump_regitser_tx_callbacks(uint16_t end_q, uint8_t port, uint16_t queue, +pdump_register_tx_callbacks(uint16_t end_q, uint16_t port, uint16_t queue, struct rte_ring *ring, struct rte_mempool *mp, uint16_t operation) { @@ -347,7 +325,7 @@ pdump_regitser_tx_callbacks(uint16_t end_q, uint8_t port, uint16_t queue, if (ret < 0) { RTE_LOG(ERR, PDUMP, "failed to remove tx callback, errno=%d\n", - rte_errno); + -ret); return ret; } cbs->cb = NULL; @@ -360,8 +338,8 @@ pdump_regitser_tx_callbacks(uint16_t end_q, uint8_t port, uint16_t queue, static int set_pdump_rxtx_cbs(struct pdump_request *p) { - uint16_t nb_rx_q, nb_tx_q = 0, end_q, queue; - uint8_t port; + uint16_t nb_rx_q = 0, nb_tx_q = 0, end_q, queue; + uint16_t port; int ret = 0; uint32_t flags; uint16_t operation; @@ -375,7 +353,7 @@ set_pdump_rxtx_cbs(struct pdump_request *p) &port); if (ret < 0) { RTE_LOG(ERR, PDUMP, - "failed to get potid for device id=%s\n", + "failed to get port id for device id=%s\n", p->data.en_v1.device); return -EINVAL; } @@ -387,7 +365,7 @@ set_pdump_rxtx_cbs(struct pdump_request *p) &port); if (ret < 0) { RTE_LOG(ERR, PDUMP, - "failed to get potid for device id=%s\n", + "failed to get port id for device id=%s\n", p->data.dis_v1.device); return -EINVAL; } @@ -424,7 +402,7 @@ set_pdump_rxtx_cbs(struct pdump_request *p) /* register RX callback */ if (flags & RTE_PDUMP_FLAG_RX) { end_q = (queue == RTE_PDUMP_ALL_QUEUES) ? nb_rx_q : queue + 1; - ret = pdump_regitser_rx_callbacks(end_q, port, queue, ring, mp, + ret = pdump_register_rx_callbacks(end_q, port, queue, ring, mp, operation); if (ret < 0) return ret; @@ -433,7 +411,7 @@ set_pdump_rxtx_cbs(struct pdump_request *p) /* register TX callback */ if (flags & RTE_PDUMP_FLAG_TX) { end_q = (queue == RTE_PDUMP_ALL_QUEUES) ? nb_tx_q : queue + 1; - ret = pdump_regitser_tx_callbacks(end_q, port, queue, ring, mp, + ret = pdump_register_tx_callbacks(end_q, port, queue, ring, mp, operation); if (ret < 0) return ret; @@ -603,7 +581,7 @@ rte_pdump_init(const char *path) if (ret != 0) { RTE_LOG(ERR, PDUMP, "Failed to create the pdump thread:%s, %s:%d\n", - strerror(errno), __func__, __LINE__); + strerror(ret), __func__, __LINE__); return -1; } /* Set thread_name for aid in debugging. */ @@ -626,7 +604,7 @@ rte_pdump_uninit(void) if (ret != 0) { RTE_LOG(ERR, PDUMP, "Failed to cancel the pdump thread:%s, %s:%d\n", - strerror(errno), __func__, __LINE__); + strerror(ret), __func__, __LINE__); return -1; } @@ -763,7 +741,7 @@ pdump_validate_ring_mp(struct rte_ring *ring, struct rte_mempool *mp) rte_errno = EINVAL; return -1; } - if (ring->prod.sp_enqueue || ring->cons.sc_dequeue) { + if (ring->prod.single || ring->cons.single) { RTE_LOG(ERR, PDUMP, "ring with either SP or SC settings" " is not valid for pdump, should have MP and MC settings\n"); rte_errno = EINVAL; @@ -788,7 +766,7 @@ pdump_validate_flags(uint32_t flags) } static int -pdump_validate_port(uint8_t port, char *name) +pdump_validate_port(uint16_t port, char *name) { int ret = 0; @@ -852,7 +830,7 @@ pdump_prepare_client_request(char *device, uint16_t queue, } int -rte_pdump_enable(uint8_t port, uint16_t queue, uint32_t flags, +rte_pdump_enable(uint16_t port, uint16_t queue, uint32_t flags, struct rte_ring *ring, struct rte_mempool *mp, void *filter) @@ -885,7 +863,6 @@ rte_pdump_enable_by_deviceid(char *device_id, uint16_t queue, void *filter) { int ret = 0; - char domBDF[DEVICE_ID_SIZE]; ret = pdump_validate_ring_mp(ring, mp); if (ret < 0) @@ -894,18 +871,14 @@ rte_pdump_enable_by_deviceid(char *device_id, uint16_t queue, if (ret < 0) return ret; - if (pdump_get_dombdf(device_id, domBDF, sizeof(domBDF)) > 0) - ret = pdump_prepare_client_request(domBDF, queue, flags, - ENABLE, ring, mp, filter); - else - ret = pdump_prepare_client_request(device_id, queue, flags, + ret = pdump_prepare_client_request(device_id, queue, flags, ENABLE, ring, mp, filter); return ret; } int -rte_pdump_disable(uint8_t port, uint16_t queue, uint32_t flags) +rte_pdump_disable(uint16_t port, uint16_t queue, uint32_t flags) { int ret = 0; char name[DEVICE_ID_SIZE]; @@ -928,17 +901,12 @@ rte_pdump_disable_by_deviceid(char *device_id, uint16_t queue, uint32_t flags) { int ret = 0; - char domBDF[DEVICE_ID_SIZE]; ret = pdump_validate_flags(flags); if (ret < 0) return ret; - if (pdump_get_dombdf(device_id, domBDF, sizeof(domBDF)) > 0) - ret = pdump_prepare_client_request(domBDF, queue, flags, - DISABLE, NULL, NULL, NULL); - else - ret = pdump_prepare_client_request(device_id, queue, flags, + ret = pdump_prepare_client_request(device_id, queue, flags, DISABLE, NULL, NULL, NULL); return ret; diff --git a/dpdk/lib/librte_pdump/rte_pdump.h b/dpdk/lib/librte_pdump/rte_pdump.h index b5f4e2f3..4ec0a106 100644 --- a/dpdk/lib/librte_pdump/rte_pdump.h +++ b/dpdk/lib/librte_pdump/rte_pdump.h @@ -41,6 +41,10 @@ * packet dump library to provide packet capturing support on dpdk. */ +#include +#include +#include + #ifdef __cplusplus extern "C" { #endif @@ -109,7 +113,7 @@ rte_pdump_uninit(void); */ int -rte_pdump_enable(uint8_t port, uint16_t queue, uint32_t flags, +rte_pdump_enable(uint16_t port, uint16_t queue, uint32_t flags, struct rte_ring *ring, struct rte_mempool *mp, void *filter); @@ -132,7 +136,7 @@ rte_pdump_enable(uint8_t port, uint16_t queue, uint32_t flags, */ int -rte_pdump_disable(uint8_t port, uint16_t queue, uint32_t flags); +rte_pdump_disable(uint16_t port, uint16_t queue, uint32_t flags); /** * Enables packet capturing on given device id and queue. @@ -197,7 +201,7 @@ rte_pdump_disable_by_deviceid(char *device_id, uint16_t queue, * * @param path * directory path for server or client socket. - * @type + * @param type * specifies RTE_PDUMP_SOCKET_SERVER if socket path is for server. * (or) * specifies RTE_PDUMP_SOCKET_CLIENT if socket path is for client. diff --git a/dpdk/lib/librte_pipeline/Makefile b/dpdk/lib/librte_pipeline/Makefile index 05d64ff8..a8285738 100644 --- a/dpdk/lib/librte_pipeline/Makefile +++ b/dpdk/lib/librte_pipeline/Makefile @@ -38,6 +38,8 @@ LIB = librte_pipeline.a CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) +LDLIBS += -lrte_eal -lrte_mempool -lrte_mbuf -lrte_table +LDLIBS += -lrte_port EXPORT_MAP := rte_pipeline_version.map @@ -51,11 +53,4 @@ SRCS-$(CONFIG_RTE_LIBRTE_PIPELINE) := rte_pipeline.c # install includes SYMLINK-$(CONFIG_RTE_LIBRTE_PIPELINE)-include += rte_pipeline.h -# this lib depends upon: -DEPDIRS-$(CONFIG_RTE_LIBRTE_PIPELINE) += lib/librte_eal -DEPDIRS-$(CONFIG_RTE_LIBRTE_PIPELINE) += lib/librte_mbuf -DEPDIRS-$(CONFIG_RTE_LIBRTE_PIPELINE) += lib/librte_mempool -DEPDIRS-$(CONFIG_RTE_LIBRTE_PIPELINE) += lib/librte_table -DEPDIRS-$(CONFIG_RTE_LIBRTE_PIPELINE) += lib/librte_port - include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/lib/librte_pipeline/rte_pipeline.c b/dpdk/lib/librte_pipeline/rte_pipeline.c index 7f8fbac5..8611a88b 100644 --- a/dpdk/lib/librte_pipeline/rte_pipeline.c +++ b/dpdk/lib/librte_pipeline/rte_pipeline.c @@ -36,7 +36,6 @@ #include #include -#include #include #include #include diff --git a/dpdk/lib/librte_pipeline/rte_pipeline.h b/dpdk/lib/librte_pipeline/rte_pipeline.h index 84d18025..fdc44a79 100644 --- a/dpdk/lib/librte_pipeline/rte_pipeline.h +++ b/dpdk/lib/librte_pipeline/rte_pipeline.h @@ -87,6 +87,7 @@ extern "C" { #include #include +#include struct rte_mbuf; @@ -244,6 +245,7 @@ struct rte_pipeline_table_entry { /** Reserved action */ enum rte_pipeline_action action; + RTE_STD_C11 union { /** Output port ID (meta-data for "Send packet to output port" action) */ @@ -252,7 +254,7 @@ struct rte_pipeline_table_entry { uint32_t table_id; }; /** Start of table entry area for user defined actions and meta-data */ - uint8_t action_data[0]; + __extension__ uint8_t action_data[0]; }; /** @@ -481,7 +483,7 @@ int rte_pipeline_table_entry_delete(struct rte_pipeline *p, * @param keys * Array containing table entry keys * @param entries - * Array containung new contents for every table entry identified by key + * Array containing new contents for every table entry identified by key * @param n_keys * Number of keys to add * @param key_found diff --git a/dpdk/lib/librte_port/Makefile b/dpdk/lib/librte_port/Makefile index 3d84a0e4..139dc59a 100644 --- a/dpdk/lib/librte_port/Makefile +++ b/dpdk/lib/librte_port/Makefile @@ -38,6 +38,11 @@ LIB = librte_port.a ifeq ($(CONFIG_RTE_PORT_PCAP),y) LDLIBS += -lpcap endif +LDLIBS += -lrte_eal -lrte_mempool -lrte_mbuf -lrte_ethdev +LDLIBS += -lrte_ip_frag -lrte_sched +ifeq ($(CONFIG_RTE_LIBRTE_KNI),y) +LDLIBS += -lrte_kni +endif CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) @@ -56,6 +61,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_PORT) += rte_port_frag.c SRCS-$(CONFIG_RTE_LIBRTE_PORT) += rte_port_ras.c endif SRCS-$(CONFIG_RTE_LIBRTE_PORT) += rte_port_sched.c +SRCS-$(CONFIG_RTE_LIBRTE_PORT) += rte_port_fd.c ifeq ($(CONFIG_RTE_LIBRTE_KNI),y) SRCS-$(CONFIG_RTE_LIBRTE_PORT) += rte_port_kni.c endif @@ -70,20 +76,10 @@ SYMLINK-$(CONFIG_RTE_LIBRTE_PORT)-include += rte_port_frag.h SYMLINK-$(CONFIG_RTE_LIBRTE_PORT)-include += rte_port_ras.h endif SYMLINK-$(CONFIG_RTE_LIBRTE_PORT)-include += rte_port_sched.h +SYMLINK-$(CONFIG_RTE_LIBRTE_PORT)-include += rte_port_fd.h ifeq ($(CONFIG_RTE_LIBRTE_KNI),y) SYMLINK-$(CONFIG_RTE_LIBRTE_PORT)-include += rte_port_kni.h endif SYMLINK-$(CONFIG_RTE_LIBRTE_PORT)-include += rte_port_source_sink.h -# this lib depends upon: -DEPDIRS-$(CONFIG_RTE_LIBRTE_PORT) := lib/librte_eal -DEPDIRS-$(CONFIG_RTE_LIBRTE_PORT) += lib/librte_mbuf -DEPDIRS-$(CONFIG_RTE_LIBRTE_PORT) += lib/librte_mempool -DEPDIRS-$(CONFIG_RTE_LIBRTE_PORT) += lib/librte_ether -DEPDIRS-$(CONFIG_RTE_LIBRTE_PORT) += lib/librte_ip_frag -DEPDIRS-$(CONFIG_RTE_LIBRTE_PORT) += lib/librte_sched -ifeq ($(CONFIG_RTE_LIBRTE_KNI),y) -DEPDIRS-$(CONFIG_RTE_LIBRTE_PORT) += lib/librte_kni -endif - include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/lib/librte_port/rte_port_ethdev.c b/dpdk/lib/librte_port/rte_port_ethdev.c index 73e5f185..4ed10f27 100644 --- a/dpdk/lib/librte_port/rte_port_ethdev.c +++ b/dpdk/lib/librte_port/rte_port_ethdev.c @@ -60,14 +60,14 @@ struct rte_port_ethdev_reader { struct rte_port_in_stats stats; uint16_t queue_id; - uint8_t port_id; + uint16_t port_id; }; static void * rte_port_ethdev_reader_create(void *params, int socket_id) { struct rte_port_ethdev_reader_params *conf = - (struct rte_port_ethdev_reader_params *) params; + params; struct rte_port_ethdev_reader *port; /* Check input parameters */ @@ -95,7 +95,7 @@ static int rte_port_ethdev_reader_rx(void *port, struct rte_mbuf **pkts, uint32_t n_pkts) { struct rte_port_ethdev_reader *p = - (struct rte_port_ethdev_reader *) port; + port; uint16_t rx_pkt_cnt; rx_pkt_cnt = rte_eth_rx_burst(p->port_id, p->queue_id, pkts, n_pkts); @@ -120,7 +120,7 @@ static int rte_port_ethdev_reader_stats_read(void *port, struct rte_port_in_stats *stats, int clear) { struct rte_port_ethdev_reader *p = - (struct rte_port_ethdev_reader *) port; + port; if (stats != NULL) memcpy(stats, &p->stats, sizeof(p->stats)); @@ -156,14 +156,14 @@ struct rte_port_ethdev_writer { uint16_t tx_buf_count; uint64_t bsz_mask; uint16_t queue_id; - uint8_t port_id; + uint16_t port_id; }; static void * rte_port_ethdev_writer_create(void *params, int socket_id) { struct rte_port_ethdev_writer_params *conf = - (struct rte_port_ethdev_writer_params *) params; + params; struct rte_port_ethdev_writer *port; /* Check input parameters */ @@ -212,7 +212,7 @@ static int rte_port_ethdev_writer_tx(void *port, struct rte_mbuf *pkt) { struct rte_port_ethdev_writer *p = - (struct rte_port_ethdev_writer *) port; + port; p->tx_buf[p->tx_buf_count++] = pkt; RTE_PORT_ETHDEV_WRITER_STATS_PKTS_IN_ADD(p, 1); @@ -228,7 +228,7 @@ rte_port_ethdev_writer_tx_bulk(void *port, uint64_t pkts_mask) { struct rte_port_ethdev_writer *p = - (struct rte_port_ethdev_writer *) port; + port; uint64_t bsz_mask = p->bsz_mask; uint32_t tx_buf_count = p->tx_buf_count; uint64_t expr = (pkts_mask & (pkts_mask + 1)) | @@ -274,7 +274,7 @@ static int rte_port_ethdev_writer_flush(void *port) { struct rte_port_ethdev_writer *p = - (struct rte_port_ethdev_writer *) port; + port; if (p->tx_buf_count > 0) send_burst(p); @@ -300,7 +300,7 @@ static int rte_port_ethdev_writer_stats_read(void *port, struct rte_port_out_stats *stats, int clear) { struct rte_port_ethdev_writer *p = - (struct rte_port_ethdev_writer *) port; + port; if (stats != NULL) memcpy(stats, &p->stats, sizeof(p->stats)); @@ -337,14 +337,14 @@ struct rte_port_ethdev_writer_nodrop { uint64_t bsz_mask; uint64_t n_retries; uint16_t queue_id; - uint8_t port_id; + uint16_t port_id; }; static void * rte_port_ethdev_writer_nodrop_create(void *params, int socket_id) { struct rte_port_ethdev_writer_nodrop_params *conf = - (struct rte_port_ethdev_writer_nodrop_params *) params; + params; struct rte_port_ethdev_writer_nodrop *port; /* Check input parameters */ @@ -418,7 +418,7 @@ static int rte_port_ethdev_writer_nodrop_tx(void *port, struct rte_mbuf *pkt) { struct rte_port_ethdev_writer_nodrop *p = - (struct rte_port_ethdev_writer_nodrop *) port; + port; p->tx_buf[p->tx_buf_count++] = pkt; RTE_PORT_ETHDEV_WRITER_NODROP_STATS_PKTS_IN_ADD(p, 1); @@ -434,7 +434,7 @@ rte_port_ethdev_writer_nodrop_tx_bulk(void *port, uint64_t pkts_mask) { struct rte_port_ethdev_writer_nodrop *p = - (struct rte_port_ethdev_writer_nodrop *) port; + port; uint64_t bsz_mask = p->bsz_mask; uint32_t tx_buf_count = p->tx_buf_count; @@ -456,8 +456,8 @@ rte_port_ethdev_writer_nodrop_tx_bulk(void *port, return 0; /* - * If we didnt manage to send all packets in single burst, move - * remaining packets to the buffer and call send burst. + * If we did not manage to send all packets in single burst, + * move remaining packets to the buffer and call send burst. */ for (; n_pkts_ok < n_pkts; n_pkts_ok++) { struct rte_mbuf *pkt = pkts[n_pkts_ok]; @@ -487,7 +487,7 @@ static int rte_port_ethdev_writer_nodrop_flush(void *port) { struct rte_port_ethdev_writer_nodrop *p = - (struct rte_port_ethdev_writer_nodrop *) port; + port; if (p->tx_buf_count > 0) send_burst_nodrop(p); @@ -513,7 +513,7 @@ static int rte_port_ethdev_writer_nodrop_stats_read(void *port, struct rte_port_out_stats *stats, int clear) { struct rte_port_ethdev_writer_nodrop *p = - (struct rte_port_ethdev_writer_nodrop *) port; + port; if (stats != NULL) memcpy(stats, &p->stats, sizeof(p->stats)); diff --git a/dpdk/lib/librte_port/rte_port_ethdev.h b/dpdk/lib/librte_port/rte_port_ethdev.h index 201a79e4..f5ed9ab2 100644 --- a/dpdk/lib/librte_port/rte_port_ethdev.h +++ b/dpdk/lib/librte_port/rte_port_ethdev.h @@ -54,7 +54,7 @@ extern "C" { /** ethdev_reader port parameters */ struct rte_port_ethdev_reader_params { /** NIC RX port ID */ - uint8_t port_id; + uint16_t port_id; /** NIC RX queue ID */ uint16_t queue_id; @@ -66,7 +66,7 @@ extern struct rte_port_in_ops rte_port_ethdev_reader_ops; /** ethdev_writer port parameters */ struct rte_port_ethdev_writer_params { /** NIC RX port ID */ - uint8_t port_id; + uint16_t port_id; /** NIC RX queue ID */ uint16_t queue_id; @@ -82,7 +82,7 @@ extern struct rte_port_out_ops rte_port_ethdev_writer_ops; /** ethdev_writer_nodrop port parameters */ struct rte_port_ethdev_writer_nodrop_params { /** NIC RX port ID */ - uint8_t port_id; + uint16_t port_id; /** NIC RX queue ID */ uint16_t queue_id; diff --git a/dpdk/lib/librte_port/rte_port_fd.c b/dpdk/lib/librte_port/rte_port_fd.c new file mode 100644 index 00000000..b5b37291 --- /dev/null +++ b/dpdk/lib/librte_port/rte_port_fd.c @@ -0,0 +1,547 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include +#include +#include + +#include +#include + +#include "rte_port_fd.h" + +/* + * Port FD Reader + */ +#ifdef RTE_PORT_STATS_COLLECT + +#define RTE_PORT_FD_READER_STATS_PKTS_IN_ADD(port, val) \ + do { port->stats.n_pkts_in += val; } while (0) +#define RTE_PORT_FD_READER_STATS_PKTS_DROP_ADD(port, val) \ + do { port->stats.n_pkts_drop += val; } while (0) + +#else + +#define RTE_PORT_FD_READER_STATS_PKTS_IN_ADD(port, val) +#define RTE_PORT_FD_READER_STATS_PKTS_DROP_ADD(port, val) + +#endif + +struct rte_port_fd_reader { + struct rte_port_in_stats stats; + int fd; + uint32_t mtu; + struct rte_mempool *mempool; +}; + +static void * +rte_port_fd_reader_create(void *params, int socket_id) +{ + struct rte_port_fd_reader_params *conf = + params; + struct rte_port_fd_reader *port; + + /* Check input parameters */ + if (conf == NULL) { + RTE_LOG(ERR, PORT, "%s: params is NULL\n", __func__); + return NULL; + } + if (conf->fd < 0) { + RTE_LOG(ERR, PORT, "%s: Invalid file descriptor\n", __func__); + return NULL; + } + if (conf->mtu == 0) { + RTE_LOG(ERR, PORT, "%s: Invalid MTU\n", __func__); + return NULL; + } + if (conf->mempool == NULL) { + RTE_LOG(ERR, PORT, "%s: Invalid mempool\n", __func__); + return NULL; + } + + /* Memory allocation */ + port = rte_zmalloc_socket("PORT", sizeof(*port), + RTE_CACHE_LINE_SIZE, socket_id); + if (port == NULL) { + RTE_LOG(ERR, PORT, "%s: Failed to allocate port\n", __func__); + return NULL; + } + + /* Initialization */ + port->fd = conf->fd; + port->mtu = conf->mtu; + port->mempool = conf->mempool; + + return port; +} + +static int +rte_port_fd_reader_rx(void *port, struct rte_mbuf **pkts, uint32_t n_pkts) +{ + struct rte_port_fd_reader *p = port; + uint32_t i, j; + + if (rte_pktmbuf_alloc_bulk(p->mempool, pkts, n_pkts) != 0) + return 0; + + for (i = 0; i < n_pkts; i++) { + struct rte_mbuf *pkt = pkts[i]; + void *pkt_data = rte_pktmbuf_mtod(pkt, void *); + ssize_t n_bytes; + + n_bytes = read(p->fd, pkt_data, (size_t) p->mtu); + if (n_bytes <= 0) + break; + + pkt->data_len = n_bytes; + pkt->pkt_len = n_bytes; + } + + for (j = i; j < n_pkts; j++) + rte_pktmbuf_free(pkts[j]); + + RTE_PORT_FD_READER_STATS_PKTS_IN_ADD(p, i); + + return i; +} + +static int +rte_port_fd_reader_free(void *port) +{ + if (port == NULL) { + RTE_LOG(ERR, PORT, "%s: port is NULL\n", __func__); + return -EINVAL; + } + + rte_free(port); + + return 0; +} + +static int rte_port_fd_reader_stats_read(void *port, + struct rte_port_in_stats *stats, int clear) +{ + struct rte_port_fd_reader *p = + port; + + if (stats != NULL) + memcpy(stats, &p->stats, sizeof(p->stats)); + + if (clear) + memset(&p->stats, 0, sizeof(p->stats)); + + return 0; +} + +/* + * Port FD Writer + */ +#ifdef RTE_PORT_STATS_COLLECT + +#define RTE_PORT_FD_WRITER_STATS_PKTS_IN_ADD(port, val) \ + do { port->stats.n_pkts_in += val; } while (0) +#define RTE_PORT_FD_WRITER_STATS_PKTS_DROP_ADD(port, val) \ + do { port->stats.n_pkts_drop += val; } while (0) + +#else + +#define RTE_PORT_FD_WRITER_STATS_PKTS_IN_ADD(port, val) +#define RTE_PORT_FD_WRITER_STATS_PKTS_DROP_ADD(port, val) + +#endif + +struct rte_port_fd_writer { + struct rte_port_out_stats stats; + + struct rte_mbuf *tx_buf[2 * RTE_PORT_IN_BURST_SIZE_MAX]; + uint32_t tx_burst_sz; + uint16_t tx_buf_count; + uint32_t fd; +}; + +static void * +rte_port_fd_writer_create(void *params, int socket_id) +{ + struct rte_port_fd_writer_params *conf = + params; + struct rte_port_fd_writer *port; + + /* Check input parameters */ + if ((conf == NULL) || + (conf->tx_burst_sz == 0) || + (conf->tx_burst_sz > RTE_PORT_IN_BURST_SIZE_MAX) || + (!rte_is_power_of_2(conf->tx_burst_sz))) { + RTE_LOG(ERR, PORT, "%s: Invalid input parameters\n", __func__); + return NULL; + } + + /* Memory allocation */ + port = rte_zmalloc_socket("PORT", sizeof(*port), + RTE_CACHE_LINE_SIZE, socket_id); + if (port == NULL) { + RTE_LOG(ERR, PORT, "%s: Failed to allocate port\n", __func__); + return NULL; + } + + /* Initialization */ + port->fd = conf->fd; + port->tx_burst_sz = conf->tx_burst_sz; + port->tx_buf_count = 0; + + return port; +} + +static inline void +send_burst(struct rte_port_fd_writer *p) +{ + uint32_t i; + + for (i = 0; i < p->tx_buf_count; i++) { + struct rte_mbuf *pkt = p->tx_buf[i]; + void *pkt_data = rte_pktmbuf_mtod(pkt, void*); + size_t n_bytes = rte_pktmbuf_data_len(pkt); + ssize_t ret; + + ret = write(p->fd, pkt_data, n_bytes); + if (ret < 0) + break; + } + + RTE_PORT_FD_WRITER_STATS_PKTS_DROP_ADD(p, p->tx_buf_count - i); + + for (i = 0; i < p->tx_buf_count; i++) + rte_pktmbuf_free(p->tx_buf[i]); + + p->tx_buf_count = 0; +} + +static int +rte_port_fd_writer_tx(void *port, struct rte_mbuf *pkt) +{ + struct rte_port_fd_writer *p = + port; + + p->tx_buf[p->tx_buf_count++] = pkt; + RTE_PORT_FD_WRITER_STATS_PKTS_IN_ADD(p, 1); + if (p->tx_buf_count >= p->tx_burst_sz) + send_burst(p); + + return 0; +} + +static int +rte_port_fd_writer_tx_bulk(void *port, + struct rte_mbuf **pkts, + uint64_t pkts_mask) +{ + struct rte_port_fd_writer *p = + port; + uint32_t tx_buf_count = p->tx_buf_count; + + if ((pkts_mask & (pkts_mask + 1)) == 0) { + uint64_t n_pkts = __builtin_popcountll(pkts_mask); + uint32_t i; + + for (i = 0; i < n_pkts; i++) + p->tx_buf[tx_buf_count++] = pkts[i]; + RTE_PORT_FD_WRITER_STATS_PKTS_IN_ADD(p, n_pkts); + } else + for ( ; pkts_mask; ) { + uint32_t pkt_index = __builtin_ctzll(pkts_mask); + uint64_t pkt_mask = 1LLU << pkt_index; + struct rte_mbuf *pkt = pkts[pkt_index]; + + p->tx_buf[tx_buf_count++] = pkt; + RTE_PORT_FD_WRITER_STATS_PKTS_IN_ADD(p, 1); + pkts_mask &= ~pkt_mask; + } + + p->tx_buf_count = tx_buf_count; + if (tx_buf_count >= p->tx_burst_sz) + send_burst(p); + + return 0; +} + +static int +rte_port_fd_writer_flush(void *port) +{ + struct rte_port_fd_writer *p = + port; + + if (p->tx_buf_count > 0) + send_burst(p); + + return 0; +} + +static int +rte_port_fd_writer_free(void *port) +{ + if (port == NULL) { + RTE_LOG(ERR, PORT, "%s: Port is NULL\n", __func__); + return -EINVAL; + } + + rte_port_fd_writer_flush(port); + rte_free(port); + + return 0; +} + +static int rte_port_fd_writer_stats_read(void *port, + struct rte_port_out_stats *stats, int clear) +{ + struct rte_port_fd_writer *p = + port; + + if (stats != NULL) + memcpy(stats, &p->stats, sizeof(p->stats)); + + if (clear) + memset(&p->stats, 0, sizeof(p->stats)); + + return 0; +} + +/* + * Port FD Writer Nodrop + */ +#ifdef RTE_PORT_STATS_COLLECT + +#define RTE_PORT_FD_WRITER_NODROP_STATS_PKTS_IN_ADD(port, val) \ + do { port->stats.n_pkts_in += val; } while (0) +#define RTE_PORT_FD_WRITER_NODROP_STATS_PKTS_DROP_ADD(port, val) \ + do { port->stats.n_pkts_drop += val; } while (0) + +#else + +#define RTE_PORT_FD_WRITER_NODROP_STATS_PKTS_IN_ADD(port, val) +#define RTE_PORT_FD_WRITER_NODROP_STATS_PKTS_DROP_ADD(port, val) + +#endif + +struct rte_port_fd_writer_nodrop { + struct rte_port_out_stats stats; + + struct rte_mbuf *tx_buf[2 * RTE_PORT_IN_BURST_SIZE_MAX]; + uint32_t tx_burst_sz; + uint16_t tx_buf_count; + uint64_t n_retries; + uint32_t fd; +}; + +static void * +rte_port_fd_writer_nodrop_create(void *params, int socket_id) +{ + struct rte_port_fd_writer_nodrop_params *conf = + params; + struct rte_port_fd_writer_nodrop *port; + + /* Check input parameters */ + if ((conf == NULL) || + (conf->fd < 0) || + (conf->tx_burst_sz == 0) || + (conf->tx_burst_sz > RTE_PORT_IN_BURST_SIZE_MAX) || + (!rte_is_power_of_2(conf->tx_burst_sz))) { + RTE_LOG(ERR, PORT, "%s: Invalid input parameters\n", __func__); + return NULL; + } + + /* Memory allocation */ + port = rte_zmalloc_socket("PORT", sizeof(*port), + RTE_CACHE_LINE_SIZE, socket_id); + if (port == NULL) { + RTE_LOG(ERR, PORT, "%s: Failed to allocate port\n", __func__); + return NULL; + } + + /* Initialization */ + port->fd = conf->fd; + port->tx_burst_sz = conf->tx_burst_sz; + port->tx_buf_count = 0; + + /* + * When n_retries is 0 it means that we should wait for every packet to + * send no matter how many retries should it take. To limit number of + * branches in fast path, we use UINT64_MAX instead of branching. + */ + port->n_retries = (conf->n_retries == 0) ? UINT64_MAX : conf->n_retries; + + return port; +} + +static inline void +send_burst_nodrop(struct rte_port_fd_writer_nodrop *p) +{ + uint64_t n_retries; + uint32_t i; + + n_retries = 0; + for (i = 0; (i < p->tx_buf_count) && (n_retries < p->n_retries); i++) { + struct rte_mbuf *pkt = p->tx_buf[i]; + void *pkt_data = rte_pktmbuf_mtod(pkt, void*); + size_t n_bytes = rte_pktmbuf_data_len(pkt); + + for ( ; n_retries < p->n_retries; n_retries++) { + ssize_t ret; + + ret = write(p->fd, pkt_data, n_bytes); + if (ret) + break; + } + } + + RTE_PORT_FD_WRITER_NODROP_STATS_PKTS_DROP_ADD(p, p->tx_buf_count - i); + + for (i = 0; i < p->tx_buf_count; i++) + rte_pktmbuf_free(p->tx_buf[i]); + + p->tx_buf_count = 0; +} + +static int +rte_port_fd_writer_nodrop_tx(void *port, struct rte_mbuf *pkt) +{ + struct rte_port_fd_writer_nodrop *p = + port; + + p->tx_buf[p->tx_buf_count++] = pkt; + RTE_PORT_FD_WRITER_NODROP_STATS_PKTS_IN_ADD(p, 1); + if (p->tx_buf_count >= p->tx_burst_sz) + send_burst_nodrop(p); + + return 0; +} + +static int +rte_port_fd_writer_nodrop_tx_bulk(void *port, + struct rte_mbuf **pkts, + uint64_t pkts_mask) +{ + struct rte_port_fd_writer_nodrop *p = + port; + uint32_t tx_buf_count = p->tx_buf_count; + + if ((pkts_mask & (pkts_mask + 1)) == 0) { + uint64_t n_pkts = __builtin_popcountll(pkts_mask); + uint32_t i; + + for (i = 0; i < n_pkts; i++) + p->tx_buf[tx_buf_count++] = pkts[i]; + RTE_PORT_FD_WRITER_NODROP_STATS_PKTS_IN_ADD(p, n_pkts); + } else + for ( ; pkts_mask; ) { + uint32_t pkt_index = __builtin_ctzll(pkts_mask); + uint64_t pkt_mask = 1LLU << pkt_index; + struct rte_mbuf *pkt = pkts[pkt_index]; + + p->tx_buf[tx_buf_count++] = pkt; + RTE_PORT_FD_WRITER_NODROP_STATS_PKTS_IN_ADD(p, 1); + pkts_mask &= ~pkt_mask; + } + + p->tx_buf_count = tx_buf_count; + if (tx_buf_count >= p->tx_burst_sz) + send_burst_nodrop(p); + + return 0; +} + +static int +rte_port_fd_writer_nodrop_flush(void *port) +{ + struct rte_port_fd_writer_nodrop *p = + port; + + if (p->tx_buf_count > 0) + send_burst_nodrop(p); + + return 0; +} + +static int +rte_port_fd_writer_nodrop_free(void *port) +{ + if (port == NULL) { + RTE_LOG(ERR, PORT, "%s: Port is NULL\n", __func__); + return -EINVAL; + } + + rte_port_fd_writer_nodrop_flush(port); + rte_free(port); + +return 0; +} + +static int rte_port_fd_writer_nodrop_stats_read(void *port, + struct rte_port_out_stats *stats, int clear) +{ + struct rte_port_fd_writer_nodrop *p = + port; + + if (stats != NULL) + memcpy(stats, &p->stats, sizeof(p->stats)); + + if (clear) + memset(&p->stats, 0, sizeof(p->stats)); + + return 0; +} + +/* + * Summary of port operations + */ +struct rte_port_in_ops rte_port_fd_reader_ops = { + .f_create = rte_port_fd_reader_create, + .f_free = rte_port_fd_reader_free, + .f_rx = rte_port_fd_reader_rx, + .f_stats = rte_port_fd_reader_stats_read, +}; + +struct rte_port_out_ops rte_port_fd_writer_ops = { + .f_create = rte_port_fd_writer_create, + .f_free = rte_port_fd_writer_free, + .f_tx = rte_port_fd_writer_tx, + .f_tx_bulk = rte_port_fd_writer_tx_bulk, + .f_flush = rte_port_fd_writer_flush, + .f_stats = rte_port_fd_writer_stats_read, +}; + +struct rte_port_out_ops rte_port_fd_writer_nodrop_ops = { + .f_create = rte_port_fd_writer_nodrop_create, + .f_free = rte_port_fd_writer_nodrop_free, + .f_tx = rte_port_fd_writer_nodrop_tx, + .f_tx_bulk = rte_port_fd_writer_nodrop_tx_bulk, + .f_flush = rte_port_fd_writer_nodrop_flush, + .f_stats = rte_port_fd_writer_nodrop_stats_read, +}; diff --git a/dpdk/lib/librte_port/rte_port_fd.h b/dpdk/lib/librte_port/rte_port_fd.h new file mode 100644 index 00000000..77a2d31b --- /dev/null +++ b/dpdk/lib/librte_port/rte_port_fd.h @@ -0,0 +1,105 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __INCLUDE_RTE_PORT_FD_H__ +#define __INCLUDE_RTE_PORT_FD_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @file + * RTE Port FD Device + * + * fd_reader: input port built on top of valid non-blocking file descriptor + * fd_writer: output port built on top of valid non-blocking file descriptor + * + ***/ + +#include + +#include +#include "rte_port.h" + +/** fd_reader port parameters */ +struct rte_port_fd_reader_params { + /** File descriptor */ + int fd; + + /** Maximum Transfer Unit (MTU) */ + uint32_t mtu; + + /** Pre-initialized buffer pool */ + struct rte_mempool *mempool; +}; + +/** fd_reader port operations */ +extern struct rte_port_in_ops rte_port_fd_reader_ops; + +/** fd_writer port parameters */ +struct rte_port_fd_writer_params { + /** File descriptor */ + int fd; + + /**< Recommended write burst size. The actual burst size can be + * bigger or smaller than this value. + */ + uint32_t tx_burst_sz; +}; + +/** fd_writer port operations */ +extern struct rte_port_out_ops rte_port_fd_writer_ops; + +/** fd_writer_nodrop port parameters */ +struct rte_port_fd_writer_nodrop_params { + /** File descriptor */ + int fd; + + /**< Recommended write burst size. The actual burst size can be + * bigger or smaller than this value. + */ + uint32_t tx_burst_sz; + + /** Maximum number of retries, 0 for no limit */ + uint32_t n_retries; +}; + +/** fd_writer_nodrop port operations */ +extern struct rte_port_out_ops rte_port_fd_writer_nodrop_ops; + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/dpdk/lib/librte_port/rte_port_frag.c b/dpdk/lib/librte_port/rte_port_frag.c index 0fcace99..a00c9ae1 100644 --- a/dpdk/lib/librte_port/rte_port_frag.c +++ b/dpdk/lib/librte_port/rte_port_frag.c @@ -88,7 +88,7 @@ static void * rte_port_ring_reader_frag_create(void *params, int socket_id, int is_ipv4) { struct rte_port_ring_reader_frag_params *conf = - (struct rte_port_ring_reader_frag_params *) params; + params; struct rte_port_ring_reader_frag *port; /* Check input parameters */ @@ -159,7 +159,7 @@ rte_port_ring_reader_frag_rx(void *port, uint32_t n_pkts) { struct rte_port_ring_reader_frag *p = - (struct rte_port_ring_reader_frag *) port; + port; uint32_t n_pkts_out; n_pkts_out = 0; @@ -186,7 +186,8 @@ rte_port_ring_reader_frag_rx(void *port, /* If "pkts" buffer is empty, read packet burst from ring */ if (p->n_pkts == 0) { p->n_pkts = rte_ring_sc_dequeue_burst(p->ring, - (void **) p->pkts, RTE_PORT_IN_BURST_SIZE_MAX); + (void **) p->pkts, RTE_PORT_IN_BURST_SIZE_MAX, + NULL); RTE_PORT_RING_READER_FRAG_STATS_PKTS_IN_ADD(p, p->n_pkts); if (p->n_pkts == 0) return n_pkts_out; @@ -276,7 +277,7 @@ rte_port_frag_reader_stats_read(void *port, struct rte_port_in_stats *stats, int clear) { struct rte_port_ring_reader_frag *p = - (struct rte_port_ring_reader_frag *) port; + port; if (stats != NULL) memcpy(stats, &p->stats, sizeof(p->stats)); diff --git a/dpdk/lib/librte_port/rte_port_kni.c b/dpdk/lib/librte_port/rte_port_kni.c index 08f4ac2a..2515fb2a 100644 --- a/dpdk/lib/librte_port/rte_port_kni.c +++ b/dpdk/lib/librte_port/rte_port_kni.c @@ -66,7 +66,7 @@ static void * rte_port_kni_reader_create(void *params, int socket_id) { struct rte_port_kni_reader_params *conf = - (struct rte_port_kni_reader_params *) params; + params; struct rte_port_kni_reader *port; /* Check input parameters */ @@ -93,7 +93,7 @@ static int rte_port_kni_reader_rx(void *port, struct rte_mbuf **pkts, uint32_t n_pkts) { struct rte_port_kni_reader *p = - (struct rte_port_kni_reader *) port; + port; uint16_t rx_pkt_cnt; rx_pkt_cnt = rte_kni_rx_burst(p->kni, pkts, n_pkts); @@ -118,7 +118,7 @@ static int rte_port_kni_reader_stats_read(void *port, struct rte_port_in_stats *stats, int clear) { struct rte_port_kni_reader *p = - (struct rte_port_kni_reader *) port; + port; if (stats != NULL) memcpy(stats, &p->stats, sizeof(p->stats)); @@ -160,7 +160,7 @@ static void * rte_port_kni_writer_create(void *params, int socket_id) { struct rte_port_kni_writer_params *conf = - (struct rte_port_kni_writer_params *) params; + params; struct rte_port_kni_writer *port; /* Check input parameters */ @@ -207,7 +207,7 @@ static int rte_port_kni_writer_tx(void *port, struct rte_mbuf *pkt) { struct rte_port_kni_writer *p = - (struct rte_port_kni_writer *) port; + port; p->tx_buf[p->tx_buf_count++] = pkt; RTE_PORT_KNI_WRITER_STATS_PKTS_IN_ADD(p, 1); @@ -223,7 +223,7 @@ rte_port_kni_writer_tx_bulk(void *port, uint64_t pkts_mask) { struct rte_port_kni_writer *p = - (struct rte_port_kni_writer *) port; + port; uint64_t bsz_mask = p->bsz_mask; uint32_t tx_buf_count = p->tx_buf_count; uint64_t expr = (pkts_mask & (pkts_mask + 1)) | @@ -268,7 +268,7 @@ static int rte_port_kni_writer_flush(void *port) { struct rte_port_kni_writer *p = - (struct rte_port_kni_writer *) port; + port; if (p->tx_buf_count > 0) send_burst(p); @@ -294,7 +294,7 @@ static int rte_port_kni_writer_stats_read(void *port, struct rte_port_out_stats *stats, int clear) { struct rte_port_kni_writer *p = - (struct rte_port_kni_writer *) port; + port; if (stats != NULL) memcpy(stats, &p->stats, sizeof(p->stats)); @@ -337,7 +337,7 @@ static void * rte_port_kni_writer_nodrop_create(void *params, int socket_id) { struct rte_port_kni_writer_nodrop_params *conf = - (struct rte_port_kni_writer_nodrop_params *) params; + params; struct rte_port_kni_writer_nodrop *port; /* Check input parameters */ @@ -410,7 +410,7 @@ static int rte_port_kni_writer_nodrop_tx(void *port, struct rte_mbuf *pkt) { struct rte_port_kni_writer_nodrop *p = - (struct rte_port_kni_writer_nodrop *) port; + port; p->tx_buf[p->tx_buf_count++] = pkt; RTE_PORT_KNI_WRITER_STATS_PKTS_IN_ADD(p, 1); @@ -426,7 +426,7 @@ rte_port_kni_writer_nodrop_tx_bulk(void *port, uint64_t pkts_mask) { struct rte_port_kni_writer_nodrop *p = - (struct rte_port_kni_writer_nodrop *) port; + port; uint64_t bsz_mask = p->bsz_mask; uint32_t tx_buf_count = p->tx_buf_count; @@ -478,7 +478,7 @@ static int rte_port_kni_writer_nodrop_flush(void *port) { struct rte_port_kni_writer_nodrop *p = - (struct rte_port_kni_writer_nodrop *) port; + port; if (p->tx_buf_count > 0) send_burst_nodrop(p); @@ -504,7 +504,7 @@ static int rte_port_kni_writer_nodrop_stats_read(void *port, struct rte_port_out_stats *stats, int clear) { struct rte_port_kni_writer_nodrop *p = - (struct rte_port_kni_writer_nodrop *) port; + port; if (stats != NULL) memcpy(stats, &p->stats, sizeof(p->stats)); diff --git a/dpdk/lib/librte_port/rte_port_ras.c b/dpdk/lib/librte_port/rte_port_ras.c index c4bb5081..415fadd5 100644 --- a/dpdk/lib/librte_port/rte_port_ras.c +++ b/dpdk/lib/librte_port/rte_port_ras.c @@ -93,7 +93,7 @@ static void * rte_port_ring_writer_ras_create(void *params, int socket_id, int is_ipv4) { struct rte_port_ring_writer_ras_params *conf = - (struct rte_port_ring_writer_ras_params *) params; + params; struct rte_port_ring_writer_ras *port; uint64_t frag_cycles; @@ -167,7 +167,7 @@ send_burst(struct rte_port_ring_writer_ras *p) uint32_t nb_tx; nb_tx = rte_ring_sp_enqueue_burst(p->ring, (void **)p->tx_buf, - p->tx_buf_count); + p->tx_buf_count, NULL); RTE_PORT_RING_WRITER_RAS_STATS_PKTS_DROP_ADD(p, p->tx_buf_count - nb_tx); for ( ; nb_tx < p->tx_buf_count; nb_tx++) @@ -243,7 +243,7 @@ static int rte_port_ring_writer_ras_tx(void *port, struct rte_mbuf *pkt) { struct rte_port_ring_writer_ras *p = - (struct rte_port_ring_writer_ras *) port; + port; RTE_PORT_RING_WRITER_RAS_STATS_PKTS_IN_ADD(p, 1); p->f_ras(p, pkt); @@ -259,7 +259,7 @@ rte_port_ring_writer_ras_tx_bulk(void *port, uint64_t pkts_mask) { struct rte_port_ring_writer_ras *p = - (struct rte_port_ring_writer_ras *) port; + port; if ((pkts_mask & (pkts_mask + 1)) == 0) { uint64_t n_pkts = __builtin_popcountll(pkts_mask); @@ -295,7 +295,7 @@ static int rte_port_ring_writer_ras_flush(void *port) { struct rte_port_ring_writer_ras *p = - (struct rte_port_ring_writer_ras *) port; + port; if (p->tx_buf_count > 0) send_burst(p); @@ -307,7 +307,7 @@ static int rte_port_ring_writer_ras_free(void *port) { struct rte_port_ring_writer_ras *p = - (struct rte_port_ring_writer_ras *) port; + port; if (port == NULL) { RTE_LOG(ERR, PORT, "%s: Parameter port is NULL\n", __func__); @@ -326,7 +326,7 @@ rte_port_ras_writer_stats_read(void *port, struct rte_port_out_stats *stats, int clear) { struct rte_port_ring_writer_ras *p = - (struct rte_port_ring_writer_ras *) port; + port; if (stats != NULL) memcpy(stats, &p->stats, sizeof(p->stats)); diff --git a/dpdk/lib/librte_port/rte_port_ring.c b/dpdk/lib/librte_port/rte_port_ring.c index 3b9d3d08..a4e709c9 100644 --- a/dpdk/lib/librte_port/rte_port_ring.c +++ b/dpdk/lib/librte_port/rte_port_ring.c @@ -67,14 +67,14 @@ rte_port_ring_reader_create_internal(void *params, int socket_id, uint32_t is_multi) { struct rte_port_ring_reader_params *conf = - (struct rte_port_ring_reader_params *) params; + params; struct rte_port_ring_reader *port; /* Check input parameters */ if ((conf == NULL) || (conf->ring == NULL) || - (conf->ring->cons.sc_dequeue && is_multi) || - (!(conf->ring->cons.sc_dequeue) && !is_multi)) { + (conf->ring->cons.single && is_multi) || + (!(conf->ring->cons.single) && !is_multi)) { RTE_LOG(ERR, PORT, "%s: Invalid Parameters\n", __func__); return NULL; } @@ -108,10 +108,11 @@ rte_port_ring_multi_reader_create(void *params, int socket_id) static int rte_port_ring_reader_rx(void *port, struct rte_mbuf **pkts, uint32_t n_pkts) { - struct rte_port_ring_reader *p = (struct rte_port_ring_reader *) port; + struct rte_port_ring_reader *p = port; uint32_t nb_rx; - nb_rx = rte_ring_sc_dequeue_burst(p->ring, (void **) pkts, n_pkts); + nb_rx = rte_ring_sc_dequeue_burst(p->ring, (void **) pkts, + n_pkts, NULL); RTE_PORT_RING_READER_STATS_PKTS_IN_ADD(p, nb_rx); return nb_rx; @@ -121,10 +122,11 @@ static int rte_port_ring_multi_reader_rx(void *port, struct rte_mbuf **pkts, uint32_t n_pkts) { - struct rte_port_ring_reader *p = (struct rte_port_ring_reader *) port; + struct rte_port_ring_reader *p = port; uint32_t nb_rx; - nb_rx = rte_ring_mc_dequeue_burst(p->ring, (void **) pkts, n_pkts); + nb_rx = rte_ring_mc_dequeue_burst(p->ring, (void **) pkts, + n_pkts, NULL); RTE_PORT_RING_READER_STATS_PKTS_IN_ADD(p, nb_rx); return nb_rx; @@ -148,7 +150,7 @@ rte_port_ring_reader_stats_read(void *port, struct rte_port_in_stats *stats, int clear) { struct rte_port_ring_reader *p = - (struct rte_port_ring_reader *) port; + port; if (stats != NULL) memcpy(stats, &p->stats, sizeof(p->stats)); @@ -192,14 +194,14 @@ rte_port_ring_writer_create_internal(void *params, int socket_id, uint32_t is_multi) { struct rte_port_ring_writer_params *conf = - (struct rte_port_ring_writer_params *) params; + params; struct rte_port_ring_writer *port; /* Check input parameters */ if ((conf == NULL) || (conf->ring == NULL) || - (conf->ring->prod.sp_enqueue && is_multi) || - (!(conf->ring->prod.sp_enqueue) && !is_multi) || + (conf->ring->prod.single && is_multi) || + (!(conf->ring->prod.single) && !is_multi) || (conf->tx_burst_sz > RTE_PORT_IN_BURST_SIZE_MAX)) { RTE_LOG(ERR, PORT, "%s: Invalid Parameters\n", __func__); return NULL; @@ -241,7 +243,7 @@ send_burst(struct rte_port_ring_writer *p) uint32_t nb_tx; nb_tx = rte_ring_sp_enqueue_burst(p->ring, (void **)p->tx_buf, - p->tx_buf_count); + p->tx_buf_count, NULL); RTE_PORT_RING_WRITER_STATS_PKTS_DROP_ADD(p, p->tx_buf_count - nb_tx); for ( ; nb_tx < p->tx_buf_count; nb_tx++) @@ -256,7 +258,7 @@ send_burst_mp(struct rte_port_ring_writer *p) uint32_t nb_tx; nb_tx = rte_ring_mp_enqueue_burst(p->ring, (void **)p->tx_buf, - p->tx_buf_count); + p->tx_buf_count, NULL); RTE_PORT_RING_WRITER_STATS_PKTS_DROP_ADD(p, p->tx_buf_count - nb_tx); for ( ; nb_tx < p->tx_buf_count; nb_tx++) @@ -268,7 +270,7 @@ send_burst_mp(struct rte_port_ring_writer *p) static int rte_port_ring_writer_tx(void *port, struct rte_mbuf *pkt) { - struct rte_port_ring_writer *p = (struct rte_port_ring_writer *) port; + struct rte_port_ring_writer *p = port; p->tx_buf[p->tx_buf_count++] = pkt; RTE_PORT_RING_WRITER_STATS_PKTS_IN_ADD(p, 1); @@ -281,7 +283,7 @@ rte_port_ring_writer_tx(void *port, struct rte_mbuf *pkt) static int rte_port_ring_multi_writer_tx(void *port, struct rte_mbuf *pkt) { - struct rte_port_ring_writer *p = (struct rte_port_ring_writer *) port; + struct rte_port_ring_writer *p = port; p->tx_buf[p->tx_buf_count++] = pkt; RTE_PORT_RING_WRITER_STATS_PKTS_IN_ADD(p, 1); @@ -291,14 +293,14 @@ rte_port_ring_multi_writer_tx(void *port, struct rte_mbuf *pkt) return 0; } -static inline int __attribute__((always_inline)) +static __rte_always_inline int rte_port_ring_writer_tx_bulk_internal(void *port, struct rte_mbuf **pkts, uint64_t pkts_mask, uint32_t is_multi) { struct rte_port_ring_writer *p = - (struct rte_port_ring_writer *) port; + port; uint64_t bsz_mask = p->bsz_mask; uint32_t tx_buf_count = p->tx_buf_count; @@ -318,11 +320,11 @@ rte_port_ring_writer_tx_bulk_internal(void *port, RTE_PORT_RING_WRITER_STATS_PKTS_IN_ADD(p, n_pkts); if (is_multi) - n_pkts_ok = rte_ring_mp_enqueue_burst(p->ring, (void **)pkts, - n_pkts); + n_pkts_ok = rte_ring_mp_enqueue_burst(p->ring, + (void **)pkts, n_pkts, NULL); else - n_pkts_ok = rte_ring_sp_enqueue_burst(p->ring, (void **)pkts, - n_pkts); + n_pkts_ok = rte_ring_sp_enqueue_burst(p->ring, + (void **)pkts, n_pkts, NULL); RTE_PORT_RING_WRITER_STATS_PKTS_DROP_ADD(p, n_pkts - n_pkts_ok); for ( ; n_pkts_ok < n_pkts; n_pkts_ok++) { @@ -372,7 +374,7 @@ rte_port_ring_multi_writer_tx_bulk(void *port, static int rte_port_ring_writer_flush(void *port) { - struct rte_port_ring_writer *p = (struct rte_port_ring_writer *) port; + struct rte_port_ring_writer *p = port; if (p->tx_buf_count > 0) send_burst(p); @@ -383,7 +385,7 @@ rte_port_ring_writer_flush(void *port) static int rte_port_ring_multi_writer_flush(void *port) { - struct rte_port_ring_writer *p = (struct rte_port_ring_writer *) port; + struct rte_port_ring_writer *p = port; if (p->tx_buf_count > 0) send_burst_mp(p); @@ -394,7 +396,7 @@ rte_port_ring_multi_writer_flush(void *port) static int rte_port_ring_writer_free(void *port) { - struct rte_port_ring_writer *p = (struct rte_port_ring_writer *) port; + struct rte_port_ring_writer *p = port; if (port == NULL) { RTE_LOG(ERR, PORT, "%s: Port is NULL\n", __func__); @@ -416,7 +418,7 @@ rte_port_ring_writer_stats_read(void *port, struct rte_port_out_stats *stats, int clear) { struct rte_port_ring_writer *p = - (struct rte_port_ring_writer *) port; + port; if (stats != NULL) memcpy(stats, &p->stats, sizeof(p->stats)); @@ -461,14 +463,14 @@ rte_port_ring_writer_nodrop_create_internal(void *params, int socket_id, uint32_t is_multi) { struct rte_port_ring_writer_nodrop_params *conf = - (struct rte_port_ring_writer_nodrop_params *) params; + params; struct rte_port_ring_writer_nodrop *port; /* Check input parameters */ if ((conf == NULL) || (conf->ring == NULL) || - (conf->ring->prod.sp_enqueue && is_multi) || - (!(conf->ring->prod.sp_enqueue) && !is_multi) || + (conf->ring->prod.single && is_multi) || + (!(conf->ring->prod.single) && !is_multi) || (conf->tx_burst_sz > RTE_PORT_IN_BURST_SIZE_MAX)) { RTE_LOG(ERR, PORT, "%s: Invalid Parameters\n", __func__); return NULL; @@ -517,7 +519,7 @@ send_burst_nodrop(struct rte_port_ring_writer_nodrop *p) uint32_t nb_tx = 0, i; nb_tx = rte_ring_sp_enqueue_burst(p->ring, (void **)p->tx_buf, - p->tx_buf_count); + p->tx_buf_count, NULL); /* We sent all the packets in a first try */ if (nb_tx >= p->tx_buf_count) { @@ -527,7 +529,8 @@ send_burst_nodrop(struct rte_port_ring_writer_nodrop *p) for (i = 0; i < p->n_retries; i++) { nb_tx += rte_ring_sp_enqueue_burst(p->ring, - (void **) (p->tx_buf + nb_tx), p->tx_buf_count - nb_tx); + (void **) (p->tx_buf + nb_tx), + p->tx_buf_count - nb_tx, NULL); /* We sent all the packets in more than one try */ if (nb_tx >= p->tx_buf_count) { @@ -550,7 +553,7 @@ send_burst_mp_nodrop(struct rte_port_ring_writer_nodrop *p) uint32_t nb_tx = 0, i; nb_tx = rte_ring_mp_enqueue_burst(p->ring, (void **)p->tx_buf, - p->tx_buf_count); + p->tx_buf_count, NULL); /* We sent all the packets in a first try */ if (nb_tx >= p->tx_buf_count) { @@ -560,7 +563,8 @@ send_burst_mp_nodrop(struct rte_port_ring_writer_nodrop *p) for (i = 0; i < p->n_retries; i++) { nb_tx += rte_ring_mp_enqueue_burst(p->ring, - (void **) (p->tx_buf + nb_tx), p->tx_buf_count - nb_tx); + (void **) (p->tx_buf + nb_tx), + p->tx_buf_count - nb_tx, NULL); /* We sent all the packets in more than one try */ if (nb_tx >= p->tx_buf_count) { @@ -581,7 +585,7 @@ static int rte_port_ring_writer_nodrop_tx(void *port, struct rte_mbuf *pkt) { struct rte_port_ring_writer_nodrop *p = - (struct rte_port_ring_writer_nodrop *) port; + port; p->tx_buf[p->tx_buf_count++] = pkt; RTE_PORT_RING_WRITER_NODROP_STATS_PKTS_IN_ADD(p, 1); @@ -595,7 +599,7 @@ static int rte_port_ring_multi_writer_nodrop_tx(void *port, struct rte_mbuf *pkt) { struct rte_port_ring_writer_nodrop *p = - (struct rte_port_ring_writer_nodrop *) port; + port; p->tx_buf[p->tx_buf_count++] = pkt; RTE_PORT_RING_WRITER_NODROP_STATS_PKTS_IN_ADD(p, 1); @@ -605,14 +609,14 @@ rte_port_ring_multi_writer_nodrop_tx(void *port, struct rte_mbuf *pkt) return 0; } -static inline int __attribute__((always_inline)) +static __rte_always_inline int rte_port_ring_writer_nodrop_tx_bulk_internal(void *port, struct rte_mbuf **pkts, uint64_t pkts_mask, uint32_t is_multi) { struct rte_port_ring_writer_nodrop *p = - (struct rte_port_ring_writer_nodrop *) port; + port; uint64_t bsz_mask = p->bsz_mask; uint32_t tx_buf_count = p->tx_buf_count; @@ -633,10 +637,12 @@ rte_port_ring_writer_nodrop_tx_bulk_internal(void *port, RTE_PORT_RING_WRITER_NODROP_STATS_PKTS_IN_ADD(p, n_pkts); if (is_multi) n_pkts_ok = - rte_ring_mp_enqueue_burst(p->ring, (void **)pkts, n_pkts); + rte_ring_mp_enqueue_burst(p->ring, + (void **)pkts, n_pkts, NULL); else n_pkts_ok = - rte_ring_sp_enqueue_burst(p->ring, (void **)pkts, n_pkts); + rte_ring_sp_enqueue_burst(p->ring, + (void **)pkts, n_pkts, NULL); if (n_pkts_ok >= n_pkts) return 0; @@ -699,7 +705,7 @@ static int rte_port_ring_writer_nodrop_flush(void *port) { struct rte_port_ring_writer_nodrop *p = - (struct rte_port_ring_writer_nodrop *) port; + port; if (p->tx_buf_count > 0) send_burst_nodrop(p); @@ -711,7 +717,7 @@ static int rte_port_ring_multi_writer_nodrop_flush(void *port) { struct rte_port_ring_writer_nodrop *p = - (struct rte_port_ring_writer_nodrop *) port; + port; if (p->tx_buf_count > 0) send_burst_mp_nodrop(p); @@ -723,7 +729,7 @@ static int rte_port_ring_writer_nodrop_free(void *port) { struct rte_port_ring_writer_nodrop *p = - (struct rte_port_ring_writer_nodrop *) port; + port; if (port == NULL) { RTE_LOG(ERR, PORT, "%s: Port is NULL\n", __func__); @@ -745,7 +751,7 @@ rte_port_ring_writer_nodrop_stats_read(void *port, struct rte_port_out_stats *stats, int clear) { struct rte_port_ring_writer_nodrop *p = - (struct rte_port_ring_writer_nodrop *) port; + port; if (stats != NULL) memcpy(stats, &p->stats, sizeof(p->stats)); diff --git a/dpdk/lib/librte_port/rte_port_sched.c b/dpdk/lib/librte_port/rte_port_sched.c index 25217d62..9100a197 100644 --- a/dpdk/lib/librte_port/rte_port_sched.c +++ b/dpdk/lib/librte_port/rte_port_sched.c @@ -64,7 +64,7 @@ static void * rte_port_sched_reader_create(void *params, int socket_id) { struct rte_port_sched_reader_params *conf = - (struct rte_port_sched_reader_params *) params; + params; struct rte_port_sched_reader *port; /* Check input parameters */ @@ -91,7 +91,7 @@ rte_port_sched_reader_create(void *params, int socket_id) static int rte_port_sched_reader_rx(void *port, struct rte_mbuf **pkts, uint32_t n_pkts) { - struct rte_port_sched_reader *p = (struct rte_port_sched_reader *) port; + struct rte_port_sched_reader *p = port; uint32_t nb_rx; nb_rx = rte_sched_port_dequeue(p->sched, pkts, n_pkts); @@ -118,7 +118,7 @@ rte_port_sched_reader_stats_read(void *port, struct rte_port_in_stats *stats, int clear) { struct rte_port_sched_reader *p = - (struct rte_port_sched_reader *) port; + port; if (stats != NULL) memcpy(stats, &p->stats, sizeof(p->stats)); @@ -160,7 +160,7 @@ static void * rte_port_sched_writer_create(void *params, int socket_id) { struct rte_port_sched_writer_params *conf = - (struct rte_port_sched_writer_params *) params; + params; struct rte_port_sched_writer *port; /* Check input parameters */ @@ -292,7 +292,7 @@ rte_port_sched_writer_stats_read(void *port, struct rte_port_out_stats *stats, int clear) { struct rte_port_sched_writer *p = - (struct rte_port_sched_writer *) port; + port; if (stats != NULL) memcpy(stats, &p->stats, sizeof(p->stats)); diff --git a/dpdk/lib/librte_port/rte_port_source_sink.c b/dpdk/lib/librte_port/rte_port_source_sink.c index 4cad7109..a79f2f64 100644 --- a/dpdk/lib/librte_port/rte_port_source_sink.c +++ b/dpdk/lib/librte_port/rte_port_source_sink.c @@ -228,7 +228,7 @@ static void * rte_port_source_create(void *params, int socket_id) { struct rte_port_source_params *p = - (struct rte_port_source_params *) params; + params; struct rte_port_source *port; /* Check input arguments*/ @@ -265,7 +265,7 @@ static int rte_port_source_free(void *port) { struct rte_port_source *p = - (struct rte_port_source *)port; + port; /* Check input parameters */ if (p == NULL) @@ -286,17 +286,12 @@ rte_port_source_free(void *port) static int rte_port_source_rx(void *port, struct rte_mbuf **pkts, uint32_t n_pkts) { - struct rte_port_source *p = (struct rte_port_source *) port; + struct rte_port_source *p = port; uint32_t i; - if (rte_mempool_get_bulk(p->mempool, (void **) pkts, n_pkts) != 0) + if (rte_pktmbuf_alloc_bulk(p->mempool, pkts, n_pkts) != 0) return 0; - for (i = 0; i < n_pkts; i++) { - rte_mbuf_refcnt_set(pkts[i], 1); - rte_pktmbuf_reset(pkts[i]); - } - if (p->pkt_buff != NULL) { for (i = 0; i < n_pkts; i++) { uint8_t *pkt_data = rte_pktmbuf_mtod(pkts[i], @@ -323,7 +318,7 @@ rte_port_source_stats_read(void *port, struct rte_port_in_stats *stats, int clear) { struct rte_port_source *p = - (struct rte_port_source *) port; + port; if (stats != NULL) memcpy(stats, &p->stats, sizeof(p->stats)); @@ -400,7 +395,7 @@ pcap_sink_open(struct rte_port_sink *port, static void pcap_sink_write_pkt(struct rte_port_sink *port, struct rte_mbuf *mbuf) { - uint8_t *pcap_dumper = (uint8_t *)(port->dumper); + uint8_t *pcap_dumper = (port->dumper); struct pcap_pkthdr pcap_hdr; uint8_t jumbo_pkt_buf[ETHER_MAX_JUMBO_FRAME_LEN]; uint8_t *pkt; @@ -524,7 +519,7 @@ rte_port_sink_create(void *params, int socket_id) static int rte_port_sink_tx(void *port, struct rte_mbuf *pkt) { - struct rte_port_sink *p = (struct rte_port_sink *) port; + struct rte_port_sink *p = port; RTE_PORT_SINK_STATS_PKTS_IN_ADD(p, 1); if (p->dumper != NULL) @@ -539,7 +534,7 @@ static int rte_port_sink_tx_bulk(void *port, struct rte_mbuf **pkts, uint64_t pkts_mask) { - struct rte_port_sink *p = (struct rte_port_sink *) port; + struct rte_port_sink *p = port; if ((pkts_mask & (pkts_mask + 1)) == 0) { uint64_t n_pkts = __builtin_popcountll(pkts_mask); @@ -591,7 +586,7 @@ static int rte_port_sink_flush(void *port) { struct rte_port_sink *p = - (struct rte_port_sink *)port; + port; if (p == NULL) return 0; @@ -605,7 +600,7 @@ static int rte_port_sink_free(void *port) { struct rte_port_sink *p = - (struct rte_port_sink *)port; + port; if (p == NULL) return 0; @@ -622,7 +617,7 @@ rte_port_sink_stats_read(void *port, struct rte_port_out_stats *stats, int clear) { struct rte_port_sink *p = - (struct rte_port_sink *) port; + port; if (stats != NULL) memcpy(stats, &p->stats, sizeof(p->stats)); diff --git a/dpdk/lib/librte_port/rte_port_source_sink.h b/dpdk/lib/librte_port/rte_port_source_sink.h index 4db8a8a8..be585a77 100644 --- a/dpdk/lib/librte_port/rte_port_source_sink.h +++ b/dpdk/lib/librte_port/rte_port_source_sink.h @@ -55,7 +55,7 @@ struct rte_port_source_params { struct rte_mempool *mempool; /** The full path of the pcap file to read packets from */ - char *file_name; + const char *file_name; /** The number of bytes to be read from each packet in the * pcap file. If this value is 0, the whole packet is read; * if it is bigger than packet size, the generated packets @@ -69,7 +69,7 @@ extern struct rte_port_in_ops rte_port_source_ops; /** sink port parameters */ struct rte_port_sink_params { /** The full path of the pcap file to write the packets to */ - char *file_name; + const char *file_name; /** The maximum number of packets write to the pcap file. * If this value is 0, the "infinite" write will be carried * out. diff --git a/dpdk/lib/librte_port/rte_port_version.map b/dpdk/lib/librte_port/rte_port_version.map index 048c20d7..6470629b 100644 --- a/dpdk/lib/librte_port/rte_port_version.map +++ b/dpdk/lib/librte_port/rte_port_version.map @@ -42,3 +42,12 @@ DPDK_16.07 { rte_port_kni_writer_nodrop_ops; } DPDK_2.2; + +DPDK_16.11 { + global: + + rte_port_fd_reader_ops; + rte_port_fd_writer_ops; + rte_port_fd_writer_nodrop_ops; + +} DPDK_16.07; diff --git a/dpdk/lib/librte_power/Makefile b/dpdk/lib/librte_power/Makefile index cee95cd8..1b1491d7 100644 --- a/dpdk/lib/librte_power/Makefile +++ b/dpdk/lib/librte_power/Makefile @@ -35,6 +35,7 @@ include $(RTE_SDK)/mk/rte.vars.mk LIB = librte_power.a CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -O3 -fno-strict-aliasing +LDLIBS += -lrte_eal EXPORT_MAP := rte_power_version.map @@ -47,7 +48,4 @@ SRCS-$(CONFIG_RTE_LIBRTE_POWER) += rte_power_kvm_vm.c guest_channel.c # install this header file SYMLINK-$(CONFIG_RTE_LIBRTE_POWER)-include := rte_power.h -# this lib needs eal -DEPDIRS-$(CONFIG_RTE_LIBRTE_POWER) += lib/librte_eal - include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/lib/librte_power/channel_commands.h b/dpdk/lib/librte_power/channel_commands.h index 383897bf..f0f5f0a2 100644 --- a/dpdk/lib/librte_power/channel_commands.h +++ b/dpdk/lib/librte_power/channel_commands.h @@ -39,6 +39,7 @@ extern "C" { #endif #include +#include /* Maximum number of channels per VM */ #define CHANNEL_CMDS_MAX_VM_CHANNELS 64 @@ -46,17 +47,60 @@ extern "C" { /* Valid Commands */ #define CPU_POWER 1 #define CPU_POWER_CONNECT 2 +#define PKT_POLICY 3 /* CPU Power Command Scaling */ #define CPU_POWER_SCALE_UP 1 #define CPU_POWER_SCALE_DOWN 2 #define CPU_POWER_SCALE_MAX 3 #define CPU_POWER_SCALE_MIN 4 +#define CPU_POWER_ENABLE_TURBO 5 +#define CPU_POWER_DISABLE_TURBO 6 +#define HOURS 24 + +#define MAX_VFS 10 +#define VM_MAX_NAME_SZ 32 + +#define MAX_VCPU_PER_VM 8 + +struct t_boost_status { + bool tbEnabled; +}; + +struct timer_profile { + int busy_hours[HOURS]; + int quiet_hours[HOURS]; + int hours_to_use_traffic_profile[HOURS]; +}; + +enum workload {HIGH, MEDIUM, LOW}; +enum policy_to_use { + TRAFFIC, + TIME, + WORKLOAD +}; + +struct traffic { + uint32_t min_packet_thresh; + uint32_t avg_max_packet_thresh; + uint32_t max_max_packet_thresh; +}; struct channel_packet { uint64_t resource_id; /**< core_num, device */ uint32_t unit; /**< scale down/up/min/max */ uint32_t command; /**< Power, IO, etc */ + char vm_name[VM_MAX_NAME_SZ]; + + uint64_t vfid[MAX_VFS]; + int nb_mac_to_monitor; + struct traffic traffic_policy; + uint8_t vcpu_to_control[MAX_VCPU_PER_VM]; + uint8_t num_vcpu; + struct timer_profile timer_policy; + enum workload workload; + enum policy_to_use policy_to_use; + struct t_boost_status t_boost_status; }; diff --git a/dpdk/lib/librte_power/guest_channel.c b/dpdk/lib/librte_power/guest_channel.c index 85c92fab..fa5de0f5 100644 --- a/dpdk/lib/librte_power/guest_channel.c +++ b/dpdk/lib/librte_power/guest_channel.c @@ -148,6 +148,13 @@ guest_channel_send_msg(struct channel_packet *pkt, unsigned lcore_id) return 0; } +int rte_power_guest_channel_send_msg(struct channel_packet *pkt, + unsigned int lcore_id) +{ + return guest_channel_send_msg(pkt, lcore_id); +} + + void guest_channel_host_disconnect(unsigned lcore_id) { diff --git a/dpdk/lib/librte_power/guest_channel.h b/dpdk/lib/librte_power/guest_channel.h index 9e18af52..741339ca 100644 --- a/dpdk/lib/librte_power/guest_channel.h +++ b/dpdk/lib/librte_power/guest_channel.h @@ -81,6 +81,21 @@ void guest_channel_host_disconnect(unsigned lcore_id); */ int guest_channel_send_msg(struct channel_packet *pkt, unsigned lcore_id); +/** + * Send a message contained in pkt over the Virtio-Serial to the host endpoint. + * + * @param pkt + * Pointer to a populated struct channel_packet + * + * @param lcore_id + * lcore_id. + * + * @return + * - 0 on success. + * - Negative on error. + */ +int rte_power_guest_channel_send_msg(struct channel_packet *pkt, + unsigned int lcore_id); #ifdef __cplusplus } diff --git a/dpdk/lib/librte_power/rte_power.c b/dpdk/lib/librte_power/rte_power.c index 998ed1c9..b327a865 100644 --- a/dpdk/lib/librte_power/rte_power.c +++ b/dpdk/lib/librte_power/rte_power.c @@ -50,6 +50,9 @@ rte_power_freq_change_t rte_power_freq_up = NULL; rte_power_freq_change_t rte_power_freq_down = NULL; rte_power_freq_change_t rte_power_freq_max = NULL; rte_power_freq_change_t rte_power_freq_min = NULL; +rte_power_freq_change_t rte_power_turbo_status; +rte_power_freq_change_t rte_power_freq_enable_turbo; +rte_power_freq_change_t rte_power_freq_disable_turbo; int rte_power_set_env(enum power_management_env env) @@ -65,6 +68,9 @@ rte_power_set_env(enum power_management_env env) rte_power_freq_down = rte_power_acpi_cpufreq_freq_down; rte_power_freq_min = rte_power_acpi_cpufreq_freq_min; rte_power_freq_max = rte_power_acpi_cpufreq_freq_max; + rte_power_turbo_status = rte_power_acpi_turbo_status; + rte_power_freq_enable_turbo = rte_power_acpi_enable_turbo; + rte_power_freq_disable_turbo = rte_power_acpi_disable_turbo; } else if (env == PM_ENV_KVM_VM) { rte_power_freqs = rte_power_kvm_vm_freqs; rte_power_get_freq = rte_power_kvm_vm_get_freq; @@ -73,6 +79,9 @@ rte_power_set_env(enum power_management_env env) rte_power_freq_down = rte_power_kvm_vm_freq_down; rte_power_freq_min = rte_power_kvm_vm_freq_min; rte_power_freq_max = rte_power_kvm_vm_freq_max; + rte_power_turbo_status = rte_power_kvm_vm_turbo_status; + rte_power_freq_enable_turbo = rte_power_kvm_vm_enable_turbo; + rte_power_freq_disable_turbo = rte_power_kvm_vm_disable_turbo; } else { RTE_LOG(ERR, POWER, "Invalid Power Management Environment(%d) set\n", env); diff --git a/dpdk/lib/librte_power/rte_power.h b/dpdk/lib/librte_power/rte_power.h index 67e0ec02..b17b7a53 100644 --- a/dpdk/lib/librte_power/rte_power.h +++ b/dpdk/lib/librte_power/rte_power.h @@ -236,6 +236,47 @@ extern rte_power_freq_change_t rte_power_freq_max; */ extern rte_power_freq_change_t rte_power_freq_min; +/** + * Query the Turbo Boost status of a specific lcore. + * Review each environments specific documentation for usage.. + * + * @param lcore_id + * lcore id. + * + * @return + * - 1 Turbo Boost is enabled for this lcore. + * - 0 Turbo Boost is disabled for this lcore. + * - Negative on error. + */ +extern rte_power_freq_change_t rte_power_turbo_status; + +/** + * Enable Turbo Boost for this lcore. + * Review each environments specific documentation for usage.. + * + * @param lcore_id + * lcore id. + * + * @return + * - 0 on success. + * - Negative on error. + */ +extern rte_power_freq_change_t rte_power_freq_enable_turbo; + +/** + * Disable Turbo Boost for this lcore. + * Review each environments specific documentation for usage.. + * + * @param lcore_id + * lcore id. + * + * @return + * - 0 on success. + * - Negative on error. + */ +extern rte_power_freq_change_t rte_power_freq_disable_turbo; + + #ifdef __cplusplus } #endif diff --git a/dpdk/lib/librte_power/rte_power_acpi_cpufreq.c b/dpdk/lib/librte_power/rte_power_acpi_cpufreq.c index a56c9b59..6b0cdb2e 100644 --- a/dpdk/lib/librte_power/rte_power_acpi_cpufreq.c +++ b/dpdk/lib/librte_power/rte_power_acpi_cpufreq.c @@ -87,6 +87,14 @@ #define POWER_SYSFILE_SETSPEED \ "/sys/devices/system/cpu/cpu%u/cpufreq/scaling_setspeed" +/* + * MSR related + */ +#define PLATFORM_INFO 0x0CE +#define TURBO_RATIO_LIMIT 0x1AD +#define IA32_PERF_CTL 0x199 +#define CORE_TURBO_DISABLE_BIT ((uint64_t)1<<32) + enum power_state { POWER_IDLE = 0, POWER_ONGOING, @@ -105,6 +113,8 @@ struct rte_power_info { char governor_ori[32]; /**< Original governor name */ uint32_t curr_idx; /**< Freq index in freqs array */ volatile uint32_t state; /**< Power in use state */ + uint16_t turbo_available; /**< Turbo Boost available */ + uint16_t turbo_enable; /**< Turbo Boost enable/disable */ } __rte_cache_aligned; static struct rte_power_info lcore_power_info[RTE_MAX_LCORE]; @@ -244,8 +254,20 @@ power_get_available_freqs(struct rte_power_info *pi) POWER_CONVERT_TO_DECIMAL); } + if ((pi->freqs[0]-1000) == pi->freqs[1]) { + pi->turbo_available = 1; + pi->turbo_enable = 1; + POWER_DEBUG_TRACE("Lcore %u Can do Turbo Boost\n", + pi->lcore_id); + } else { + pi->turbo_available = 0; + pi->turbo_enable = 0; + POWER_DEBUG_TRACE("Turbo Boost not available on Lcore %u\n", + pi->lcore_id); + } + ret = 0; - POWER_DEBUG_TRACE("%d frequencie(s) of lcore %u are available\n", + POWER_DEBUG_TRACE("%d frequency(s) of lcore %u are available\n", count, pi->lcore_id); out: fclose(f); @@ -337,7 +359,7 @@ rte_power_acpi_cpufreq_init(unsigned lcore_id) } RTE_LOG(INFO, POWER, "Initialized successfully for lcore %u " - "power manamgement\n", lcore_id); + "power management\n", lcore_id); rte_atomic32_cmpset(&(pi->state), POWER_ONGOING, POWER_USED); return 0; @@ -525,7 +547,17 @@ rte_power_acpi_cpufreq_freq_max(unsigned lcore_id) } /* Frequencies in the array are from high to low. */ - return set_freq_internal(&lcore_power_info[lcore_id], 0); + if (lcore_power_info[lcore_id].turbo_available) { + if (lcore_power_info[lcore_id].turbo_enable) + /* Set to Turbo */ + return set_freq_internal( + &lcore_power_info[lcore_id], 0); + else + /* Set to max non-turbo */ + return set_freq_internal( + &lcore_power_info[lcore_id], 1); + } else + return set_freq_internal(&lcore_power_info[lcore_id], 0); } int @@ -543,3 +575,80 @@ rte_power_acpi_cpufreq_freq_min(unsigned lcore_id) /* Frequencies in the array are from high to low. */ return set_freq_internal(pi, pi->nb_freqs - 1); } + + +int +rte_power_acpi_turbo_status(unsigned int lcore_id) +{ + struct rte_power_info *pi; + + if (lcore_id >= RTE_MAX_LCORE) { + RTE_LOG(ERR, POWER, "Invalid lcore ID\n"); + return -1; + } + + pi = &lcore_power_info[lcore_id]; + + return pi->turbo_enable; +} + + +int +rte_power_acpi_enable_turbo(unsigned int lcore_id) +{ + struct rte_power_info *pi; + + if (lcore_id >= RTE_MAX_LCORE) { + RTE_LOG(ERR, POWER, "Invalid lcore ID\n"); + return -1; + } + + pi = &lcore_power_info[lcore_id]; + + if (pi->turbo_available) + pi->turbo_enable = 1; + else { + pi->turbo_enable = 0; + RTE_LOG(ERR, POWER, + "Failed to enable turbo on lcore %u\n", + lcore_id); + return -1; + } + + /* Max may have changed, so call to max function */ + if (rte_power_acpi_cpufreq_freq_max(lcore_id) < 0) { + RTE_LOG(ERR, POWER, + "Failed to set frequency of lcore %u to max\n", + lcore_id); + return -1; + } + + return 0; +} + +int +rte_power_acpi_disable_turbo(unsigned int lcore_id) +{ + struct rte_power_info *pi; + + if (lcore_id >= RTE_MAX_LCORE) { + RTE_LOG(ERR, POWER, "Invalid lcore ID\n"); + return -1; + } + + pi = &lcore_power_info[lcore_id]; + + pi->turbo_enable = 0; + + if ((pi->turbo_available) && (pi->curr_idx <= 1)) { + /* Try to set freq to max by default coming out of turbo */ + if (rte_power_acpi_cpufreq_freq_max(lcore_id) < 0) { + RTE_LOG(ERR, POWER, + "Failed to set frequency of lcore %u to max\n", + lcore_id); + return -1; + } + } + + return 0; +} diff --git a/dpdk/lib/librte_power/rte_power_acpi_cpufreq.h b/dpdk/lib/librte_power/rte_power_acpi_cpufreq.h index 68578e9b..bc20dfd6 100644 --- a/dpdk/lib/librte_power/rte_power_acpi_cpufreq.h +++ b/dpdk/lib/librte_power/rte_power_acpi_cpufreq.h @@ -180,11 +180,51 @@ int rte_power_acpi_cpufreq_freq_max(unsigned lcore_id); * * @return * - 1 on success with frequency changed. - * - 0 on success without frequency chnaged. + * - 0 on success without frequency changed. * - Negative on error. */ int rte_power_acpi_cpufreq_freq_min(unsigned lcore_id); +/** + * Get the turbo status of a specific lcore. + * It should be protected outside of this function for threadsafe. + * + * @param lcore_id + * lcore id. + * + * @return + * - 1 Turbo Boost is enabled on this lcore. + * - 0 Turbo Boost is disabled on this lcore. + * - Negative on error. + */ +int rte_power_acpi_turbo_status(unsigned int lcore_id); + +/** + * Enable Turbo Boost on a specific lcore. + * It should be protected outside of this function for threadsafe. + * + * @param lcore_id + * lcore id. + * + * @return + * - 0 Turbo Boost is enabled successfully on this lcore. + * - Negative on error. + */ +int rte_power_acpi_enable_turbo(unsigned int lcore_id); + +/** + * Disable Turbo Boost on a specific lcore. + * It should be protected outside of this function for threadsafe. + * + * @param lcore_id + * lcore id. + * + * @return + * - 0 Turbo Boost disabled successfully on this lcore. + * - Negative on error. + */ +int rte_power_acpi_disable_turbo(unsigned int lcore_id); + #ifdef __cplusplus } #endif diff --git a/dpdk/lib/librte_power/rte_power_kvm_vm.c b/dpdk/lib/librte_power/rte_power_kvm_vm.c index a1badf34..99060625 100644 --- a/dpdk/lib/librte_power/rte_power_kvm_vm.c +++ b/dpdk/lib/librte_power/rte_power_kvm_vm.c @@ -134,3 +134,22 @@ rte_power_kvm_vm_freq_min(unsigned lcore_id) { return send_msg(lcore_id, CPU_POWER_SCALE_MIN); } + +int +rte_power_kvm_vm_turbo_status(__attribute__((unused)) unsigned int lcore_id) +{ + RTE_LOG(ERR, POWER, "rte_power_turbo_status is not implemented for Virtual Machine Power Management\n"); + return -ENOTSUP; +} + +int +rte_power_kvm_vm_enable_turbo(unsigned int lcore_id) +{ + return send_msg(lcore_id, CPU_POWER_ENABLE_TURBO); +} + +int +rte_power_kvm_vm_disable_turbo(unsigned int lcore_id) +{ + return send_msg(lcore_id, CPU_POWER_DISABLE_TURBO); +} diff --git a/dpdk/lib/librte_power/rte_power_kvm_vm.h b/dpdk/lib/librte_power/rte_power_kvm_vm.h index dcbc878a..9af41d64 100644 --- a/dpdk/lib/librte_power/rte_power_kvm_vm.h +++ b/dpdk/lib/librte_power/rte_power_kvm_vm.h @@ -172,8 +172,41 @@ int rte_power_kvm_vm_freq_max(unsigned lcore_id); */ int rte_power_kvm_vm_freq_min(unsigned lcore_id); +/** + * It should be protected outside of this function for threadsafe. + * + * @param lcore_id + * lcore id. + * + * @return + * -ENOTSUP + */ +int rte_power_kvm_vm_turbo_status(unsigned int lcore_id); + +/** + * It should be protected outside of this function for threadsafe. + * + * @param lcore_id + * lcore id. + * + * @return + * - 1 on success. + * - Negative on error. + */ +int rte_power_kvm_vm_enable_turbo(unsigned int lcore_id); + +/** + * It should be protected outside of this function for threadsafe. + * + * @param lcore_id + * lcore id. + * + * @return + * - 1 on success. + * - Negative on error. + */ +int rte_power_kvm_vm_disable_turbo(unsigned int lcore_id); #ifdef __cplusplus } #endif - #endif diff --git a/dpdk/lib/librte_power/rte_power_version.map b/dpdk/lib/librte_power/rte_power_version.map index db75ff3e..96dc42ec 100644 --- a/dpdk/lib/librte_power/rte_power_version.map +++ b/dpdk/lib/librte_power/rte_power_version.map @@ -16,3 +16,13 @@ DPDK_2.0 { local: *; }; + +DPDK_17.11 { + global: + + rte_power_guest_channel_send_msg; + rte_power_freq_disable_turbo; + rte_power_freq_enable_turbo; + rte_power_turbo_status; + +} DPDK_2.0; \ No newline at end of file diff --git a/dpdk/lib/librte_reorder/Makefile b/dpdk/lib/librte_reorder/Makefile index 0d111aad..5d38d712 100644 --- a/dpdk/lib/librte_reorder/Makefile +++ b/dpdk/lib/librte_reorder/Makefile @@ -36,6 +36,7 @@ LIB = librte_reorder.a CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) +LDLIBS += -lrte_eal -lrte_mempool -lrte_mbuf EXPORT_MAP := rte_reorder_version.map @@ -47,9 +48,4 @@ SRCS-$(CONFIG_RTE_LIBRTE_REORDER) := rte_reorder.c # install this header file SYMLINK-$(CONFIG_RTE_LIBRTE_REORDER)-include := rte_reorder.h -# this lib depends upon: -DEPDIRS-$(CONFIG_RTE_LIBRTE_REORDER) += lib/librte_mbuf -DEPDIRS-$(CONFIG_RTE_LIBRTE_REORDER) += lib/librte_mempool -DEPDIRS-$(CONFIG_RTE_LIBRTE_REORDER) += lib/librte_eal - include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/lib/librte_reorder/rte_reorder.c b/dpdk/lib/librte_reorder/rte_reorder.c index 010dff68..867775da 100644 --- a/dpdk/lib/librte_reorder/rte_reorder.c +++ b/dpdk/lib/librte_reorder/rte_reorder.c @@ -36,7 +36,6 @@ #include #include -#include #include #include #include diff --git a/dpdk/lib/librte_reorder/rte_reorder.h b/dpdk/lib/librte_reorder/rte_reorder.h index c7a2934c..dc83f8e6 100644 --- a/dpdk/lib/librte_reorder/rte_reorder.h +++ b/dpdk/lib/librte_reorder/rte_reorder.h @@ -44,6 +44,8 @@ * */ +#include + #ifdef __cplusplus extern "C" { #endif @@ -145,9 +147,9 @@ rte_reorder_free(struct rte_reorder_buffer *b); * -1 on error * On error case, rte_errno will be set appropriately: * - ENOSPC - Cannot move existing mbufs from reorder buffer to accommodate - * ealry mbuf, but it can be accomodated by performing drain and then insert. + * early mbuf, but it can be accommodated by performing drain and then insert. * - ERANGE - Too early or late mbuf which is vastly out of range of expected - * window should be ingnored without any handling. + * window should be ignored without any handling. */ int rte_reorder_insert(struct rte_reorder_buffer *b, struct rte_mbuf *mbuf); diff --git a/dpdk/lib/librte_ring/Makefile b/dpdk/lib/librte_ring/Makefile index 4b1112e4..e34d9d95 100644 --- a/dpdk/lib/librte_ring/Makefile +++ b/dpdk/lib/librte_ring/Makefile @@ -35,6 +35,7 @@ include $(RTE_SDK)/mk/rte.vars.mk LIB = librte_ring.a CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -O3 +LDLIBS += -lrte_eal EXPORT_MAP := rte_ring_version.map @@ -46,6 +47,4 @@ SRCS-$(CONFIG_RTE_LIBRTE_RING) := rte_ring.c # install includes SYMLINK-$(CONFIG_RTE_LIBRTE_RING)-include := rte_ring.h -DEPDIRS-$(CONFIG_RTE_LIBRTE_RING) += lib/librte_eal - include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/lib/librte_ring/rte_ring.c b/dpdk/lib/librte_ring/rte_ring.c index ca0a1082..036467f4 100644 --- a/dpdk/lib/librte_ring/rte_ring.c +++ b/dpdk/lib/librte_ring/rte_ring.c @@ -127,18 +127,10 @@ rte_ring_init(struct rte_ring *r, const char *name, unsigned count, /* compilation-time checks */ RTE_BUILD_BUG_ON((sizeof(struct rte_ring) & RTE_CACHE_LINE_MASK) != 0); -#ifdef RTE_RING_SPLIT_PROD_CONS RTE_BUILD_BUG_ON((offsetof(struct rte_ring, cons) & RTE_CACHE_LINE_MASK) != 0); -#endif RTE_BUILD_BUG_ON((offsetof(struct rte_ring, prod) & RTE_CACHE_LINE_MASK) != 0); -#ifdef RTE_LIBRTE_RING_DEBUG - RTE_BUILD_BUG_ON((sizeof(struct rte_ring_debug_stats) & - RTE_CACHE_LINE_MASK) != 0); - RTE_BUILD_BUG_ON((offsetof(struct rte_ring, stats) & - RTE_CACHE_LINE_MASK) != 0); -#endif /* init the ring structure */ memset(r, 0, sizeof(*r)); @@ -146,11 +138,24 @@ rte_ring_init(struct rte_ring *r, const char *name, unsigned count, if (ret < 0 || ret >= (int)sizeof(r->name)) return -ENAMETOOLONG; r->flags = flags; - r->prod.watermark = count; - r->prod.sp_enqueue = !!(flags & RING_F_SP_ENQ); - r->cons.sc_dequeue = !!(flags & RING_F_SC_DEQ); - r->prod.size = r->cons.size = count; - r->prod.mask = r->cons.mask = count-1; + r->prod.single = (flags & RING_F_SP_ENQ) ? __IS_SP : __IS_MP; + r->cons.single = (flags & RING_F_SC_DEQ) ? __IS_SC : __IS_MC; + + if (flags & RING_F_EXACT_SZ) { + r->size = rte_align32pow2(count + 1); + r->mask = r->size - 1; + r->capacity = count; + } else { + if ((!POWEROF2(count)) || (count > RTE_RING_SZ_MASK)) { + RTE_LOG(ERR, RING, + "Requested size is invalid, must be power of 2, and not exceed the size limit %u\n", + RTE_RING_SZ_MASK); + return -EINVAL; + } + r->size = count; + r->mask = count - 1; + r->capacity = r->mask; + } r->prod.head = r->cons.head = 0; r->prod.tail = r->cons.tail = 0; @@ -169,10 +174,15 @@ rte_ring_create(const char *name, unsigned count, int socket_id, ssize_t ring_size; int mz_flags = 0; struct rte_ring_list* ring_list = NULL; + const unsigned int requested_count = count; int ret; ring_list = RTE_TAILQ_CAST(rte_ring_tailq.head, rte_ring_list); + /* for an exact size ring, round up from count to a power of two */ + if (flags & RING_F_EXACT_SZ) + count = rte_align32pow2(count + 1); + ring_size = rte_ring_get_memsize(count); if (ring_size < 0) { rte_errno = ring_size; @@ -198,12 +208,13 @@ rte_ring_create(const char *name, unsigned count, int socket_id, /* reserve a memory zone for this ring. If we can't get rte_config or * we are secondary process, the memzone_reserve function will set * rte_errno for us appropriately - hence no check in this this function */ - mz = rte_memzone_reserve(mz_name, ring_size, socket_id, mz_flags); + mz = rte_memzone_reserve_aligned(mz_name, ring_size, socket_id, + mz_flags, __alignof__(*r)); if (mz != NULL) { r = mz->addr; /* no need to check return value here, we already checked the * arguments above */ - rte_ring_init(r, name, count, flags); + rte_ring_init(r, name, requested_count, flags); te->data = (void *) r; r->memzone = mz; @@ -264,76 +275,20 @@ rte_ring_free(struct rte_ring *r) rte_free(te); } -/* - * change the high water mark. If *count* is 0, water marking is - * disabled - */ -int -rte_ring_set_water_mark(struct rte_ring *r, unsigned count) -{ - if (count >= r->prod.size) - return -EINVAL; - - /* if count is 0, disable the watermarking */ - if (count == 0) - count = r->prod.size; - - r->prod.watermark = count; - return 0; -} - /* dump the status of the ring on the console */ void rte_ring_dump(FILE *f, const struct rte_ring *r) { -#ifdef RTE_LIBRTE_RING_DEBUG - struct rte_ring_debug_stats sum; - unsigned lcore_id; -#endif - fprintf(f, "ring <%s>@%p\n", r->name, r); fprintf(f, " flags=%x\n", r->flags); - fprintf(f, " size=%"PRIu32"\n", r->prod.size); + fprintf(f, " size=%"PRIu32"\n", r->size); + fprintf(f, " capacity=%"PRIu32"\n", r->capacity); fprintf(f, " ct=%"PRIu32"\n", r->cons.tail); fprintf(f, " ch=%"PRIu32"\n", r->cons.head); fprintf(f, " pt=%"PRIu32"\n", r->prod.tail); fprintf(f, " ph=%"PRIu32"\n", r->prod.head); fprintf(f, " used=%u\n", rte_ring_count(r)); fprintf(f, " avail=%u\n", rte_ring_free_count(r)); - if (r->prod.watermark == r->prod.size) - fprintf(f, " watermark=0\n"); - else - fprintf(f, " watermark=%"PRIu32"\n", r->prod.watermark); - - /* sum and dump statistics */ -#ifdef RTE_LIBRTE_RING_DEBUG - memset(&sum, 0, sizeof(sum)); - for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { - sum.enq_success_bulk += r->stats[lcore_id].enq_success_bulk; - sum.enq_success_objs += r->stats[lcore_id].enq_success_objs; - sum.enq_quota_bulk += r->stats[lcore_id].enq_quota_bulk; - sum.enq_quota_objs += r->stats[lcore_id].enq_quota_objs; - sum.enq_fail_bulk += r->stats[lcore_id].enq_fail_bulk; - sum.enq_fail_objs += r->stats[lcore_id].enq_fail_objs; - sum.deq_success_bulk += r->stats[lcore_id].deq_success_bulk; - sum.deq_success_objs += r->stats[lcore_id].deq_success_objs; - sum.deq_fail_bulk += r->stats[lcore_id].deq_fail_bulk; - sum.deq_fail_objs += r->stats[lcore_id].deq_fail_objs; - } - fprintf(f, " size=%"PRIu32"\n", r->prod.size); - fprintf(f, " enq_success_bulk=%"PRIu64"\n", sum.enq_success_bulk); - fprintf(f, " enq_success_objs=%"PRIu64"\n", sum.enq_success_objs); - fprintf(f, " enq_quota_bulk=%"PRIu64"\n", sum.enq_quota_bulk); - fprintf(f, " enq_quota_objs=%"PRIu64"\n", sum.enq_quota_objs); - fprintf(f, " enq_fail_bulk=%"PRIu64"\n", sum.enq_fail_bulk); - fprintf(f, " enq_fail_objs=%"PRIu64"\n", sum.enq_fail_objs); - fprintf(f, " deq_success_bulk=%"PRIu64"\n", sum.deq_success_bulk); - fprintf(f, " deq_success_objs=%"PRIu64"\n", sum.deq_success_objs); - fprintf(f, " deq_fail_bulk=%"PRIu64"\n", sum.deq_fail_bulk); - fprintf(f, " deq_fail_objs=%"PRIu64"\n", sum.deq_fail_objs); -#else - fprintf(f, " no statistics available\n"); -#endif } /* dump the status of all rings on the console */ diff --git a/dpdk/lib/librte_ring/rte_ring.h b/dpdk/lib/librte_ring/rte_ring.h index 0e22e694..7069d52e 100644 --- a/dpdk/lib/librte_ring/rte_ring.h +++ b/dpdk/lib/librte_ring/rte_ring.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -96,48 +96,42 @@ extern "C" { #include #include #include +#include #include #include #include #include #include +#include #define RTE_TAILQ_RING_NAME "RTE_RING" enum rte_ring_queue_behavior { RTE_RING_QUEUE_FIXED = 0, /* Enq/Deq a fixed number of items from a ring */ - RTE_RING_QUEUE_VARIABLE /* Enq/Deq as many items a possible from ring */ + RTE_RING_QUEUE_VARIABLE /* Enq/Deq as many items as possible from ring */ }; -#ifdef RTE_LIBRTE_RING_DEBUG -/** - * A structure that stores the ring statistics (per-lcore). - */ -struct rte_ring_debug_stats { - uint64_t enq_success_bulk; /**< Successful enqueues number. */ - uint64_t enq_success_objs; /**< Objects successfully enqueued. */ - uint64_t enq_quota_bulk; /**< Successful enqueues above watermark. */ - uint64_t enq_quota_objs; /**< Objects enqueued above watermark. */ - uint64_t enq_fail_bulk; /**< Failed enqueues number. */ - uint64_t enq_fail_objs; /**< Objects that failed to be enqueued. */ - uint64_t deq_success_bulk; /**< Successful dequeues number. */ - uint64_t deq_success_objs; /**< Objects successfully dequeued. */ - uint64_t deq_fail_bulk; /**< Failed dequeues number. */ - uint64_t deq_fail_objs; /**< Objects that failed to be dequeued. */ -} __rte_cache_aligned; -#endif - #define RTE_RING_MZ_PREFIX "RG_" /**< The maximum length of a ring name. */ #define RTE_RING_NAMESIZE (RTE_MEMZONE_NAMESIZE - \ sizeof(RTE_RING_MZ_PREFIX) + 1) -#ifndef RTE_RING_PAUSE_REP_COUNT -#define RTE_RING_PAUSE_REP_COUNT 0 /**< Yield after pause num of times, no yield - * if RTE_RING_PAUSE_REP not defined. */ +struct rte_memzone; /* forward declaration, so as not to require memzone.h */ + +#if RTE_CACHE_LINE_SIZE < 128 +#define PROD_ALIGN (RTE_CACHE_LINE_SIZE * 2) +#define CONS_ALIGN (RTE_CACHE_LINE_SIZE * 2) +#else +#define PROD_ALIGN RTE_CACHE_LINE_SIZE +#define CONS_ALIGN RTE_CACHE_LINE_SIZE #endif -struct rte_memzone; /* forward declaration, so as not to require memzone.h */ +/* structure to hold a pair of head/tail values and other metadata */ +struct rte_ring_headtail { + volatile uint32_t head; /**< Prod/consumer head. */ + volatile uint32_t tail; /**< Prod/consumer tail. */ + uint32_t single; /**< True if single prod/cons */ +}; /** * An RTE ring structure. @@ -155,68 +149,39 @@ struct rte_ring { * compatibility requirements, it could be changed to RTE_RING_NAMESIZE * next time the ABI changes */ - char name[RTE_MEMZONE_NAMESIZE]; /**< Name of the ring. */ - int flags; /**< Flags supplied at creation. */ + char name[RTE_MEMZONE_NAMESIZE] __rte_cache_aligned; /**< Name of the ring. */ + int flags; /**< Flags supplied at creation. */ const struct rte_memzone *memzone; /**< Memzone, if any, containing the rte_ring */ + uint32_t size; /**< Size of ring. */ + uint32_t mask; /**< Mask (size-1) of ring. */ + uint32_t capacity; /**< Usable size of ring */ /** Ring producer status. */ - struct prod { - uint32_t watermark; /**< Maximum items before EDQUOT. */ - uint32_t sp_enqueue; /**< True, if single producer. */ - uint32_t size; /**< Size of ring. */ - uint32_t mask; /**< Mask (size-1) of ring. */ - volatile uint32_t head; /**< Producer head. */ - volatile uint32_t tail; /**< Producer tail. */ - } prod __rte_cache_aligned; + struct rte_ring_headtail prod __rte_aligned(PROD_ALIGN); /** Ring consumer status. */ - struct cons { - uint32_t sc_dequeue; /**< True, if single consumer. */ - uint32_t size; /**< Size of the ring. */ - uint32_t mask; /**< Mask (size-1) of ring. */ - volatile uint32_t head; /**< Consumer head. */ - volatile uint32_t tail; /**< Consumer tail. */ -#ifdef RTE_RING_SPLIT_PROD_CONS - } cons __rte_cache_aligned; -#else - } cons; -#endif - -#ifdef RTE_LIBRTE_RING_DEBUG - struct rte_ring_debug_stats stats[RTE_MAX_LCORE]; -#endif - - void * ring[0] __rte_cache_aligned; /**< Memory space of ring starts here. - * not volatile so need to be careful - * about compiler re-ordering */ + struct rte_ring_headtail cons __rte_aligned(CONS_ALIGN); }; #define RING_F_SP_ENQ 0x0001 /**< The default enqueue is "single-producer". */ #define RING_F_SC_DEQ 0x0002 /**< The default dequeue is "single-consumer". */ -#define RTE_RING_QUOT_EXCEED (1 << 31) /**< Quota exceed for burst ops */ -#define RTE_RING_SZ_MASK (unsigned)(0x0fffffff) /**< Ring size mask */ - /** - * @internal When debug is enabled, store ring statistics. - * @param r - * A pointer to the ring. - * @param name - * The name of the statistics field to increment in the ring. - * @param n - * The number to add to the object-oriented statistics. + * Ring is to hold exactly requested number of entries. + * Without this flag set, the ring size requested must be a power of 2, and the + * usable space will be that size - 1. With the flag, the requested size will + * be rounded up to the next power of two, but the usable space will be exactly + * that requested. Worst case, if a power-of-2 size is requested, half the + * ring space will be wasted. */ -#ifdef RTE_LIBRTE_RING_DEBUG -#define __RING_STAT_ADD(r, name, n) do { \ - unsigned __lcore_id = rte_lcore_id(); \ - if (__lcore_id < RTE_MAX_LCORE) { \ - r->stats[__lcore_id].name##_objs += n; \ - r->stats[__lcore_id].name##_bulk += 1; \ - } \ - } while(0) -#else -#define __RING_STAT_ADD(r, name, n) do {} while(0) -#endif +#define RING_F_EXACT_SZ 0x0004 +#define RTE_RING_SZ_MASK (0x7fffffffU) /**< Ring size mask */ + +/* @internal defines for passing to the enqueue dequeue worker functions */ +#define __IS_SP 1 +#define __IS_MP 0 +#define __IS_SC 1 +#define __IS_MC 0 /** * Calculate the memory size needed for a ring @@ -321,27 +286,7 @@ struct rte_ring *rte_ring_create(const char *name, unsigned count, void rte_ring_free(struct rte_ring *r); /** - * Change the high water mark. - * - * If *count* is 0, water marking is disabled. Otherwise, it is set to the - * *count* value. The *count* value must be greater than 0 and less - * than the ring size. - * - * This function can be called at any time (not necessarily at - * initialization). - * - * @param r - * A pointer to the ring structure. - * @param count - * The new water mark value. - * @return - * - 0: Success; water mark changed. - * - -EINVAL: Invalid water mark value. - */ -int rte_ring_set_water_mark(struct rte_ring *r, unsigned count); - -/** - * Dump the status of the ring to the console. + * Dump the status of the ring to a file. * * @param f * A pointer to a file for output @@ -353,171 +298,155 @@ void rte_ring_dump(FILE *f, const struct rte_ring *r); /* the actual enqueue of pointers on the ring. * Placed here since identical code needed in both * single and multi producer enqueue functions */ -#define ENQUEUE_PTRS() do { \ - const uint32_t size = r->prod.size; \ - uint32_t idx = prod_head & mask; \ +#define ENQUEUE_PTRS(r, ring_start, prod_head, obj_table, n, obj_type) do { \ + unsigned int i; \ + const uint32_t size = (r)->size; \ + uint32_t idx = prod_head & (r)->mask; \ + obj_type *ring = (obj_type *)ring_start; \ if (likely(idx + n < size)) { \ for (i = 0; i < (n & ((~(unsigned)0x3))); i+=4, idx+=4) { \ - r->ring[idx] = obj_table[i]; \ - r->ring[idx+1] = obj_table[i+1]; \ - r->ring[idx+2] = obj_table[i+2]; \ - r->ring[idx+3] = obj_table[i+3]; \ + ring[idx] = obj_table[i]; \ + ring[idx+1] = obj_table[i+1]; \ + ring[idx+2] = obj_table[i+2]; \ + ring[idx+3] = obj_table[i+3]; \ } \ switch (n & 0x3) { \ - case 3: r->ring[idx++] = obj_table[i++]; \ - case 2: r->ring[idx++] = obj_table[i++]; \ - case 1: r->ring[idx++] = obj_table[i++]; \ + case 3: \ + ring[idx++] = obj_table[i++]; /* fallthrough */ \ + case 2: \ + ring[idx++] = obj_table[i++]; /* fallthrough */ \ + case 1: \ + ring[idx++] = obj_table[i++]; \ } \ } else { \ for (i = 0; idx < size; i++, idx++)\ - r->ring[idx] = obj_table[i]; \ + ring[idx] = obj_table[i]; \ for (idx = 0; i < n; i++, idx++) \ - r->ring[idx] = obj_table[i]; \ + ring[idx] = obj_table[i]; \ } \ -} while(0) +} while (0) /* the actual copy of pointers on the ring to obj_table. * Placed here since identical code needed in both * single and multi consumer dequeue functions */ -#define DEQUEUE_PTRS() do { \ - uint32_t idx = cons_head & mask; \ - const uint32_t size = r->cons.size; \ +#define DEQUEUE_PTRS(r, ring_start, cons_head, obj_table, n, obj_type) do { \ + unsigned int i; \ + uint32_t idx = cons_head & (r)->mask; \ + const uint32_t size = (r)->size; \ + obj_type *ring = (obj_type *)ring_start; \ if (likely(idx + n < size)) { \ for (i = 0; i < (n & (~(unsigned)0x3)); i+=4, idx+=4) {\ - obj_table[i] = r->ring[idx]; \ - obj_table[i+1] = r->ring[idx+1]; \ - obj_table[i+2] = r->ring[idx+2]; \ - obj_table[i+3] = r->ring[idx+3]; \ + obj_table[i] = ring[idx]; \ + obj_table[i+1] = ring[idx+1]; \ + obj_table[i+2] = ring[idx+2]; \ + obj_table[i+3] = ring[idx+3]; \ } \ switch (n & 0x3) { \ - case 3: obj_table[i++] = r->ring[idx++]; \ - case 2: obj_table[i++] = r->ring[idx++]; \ - case 1: obj_table[i++] = r->ring[idx++]; \ + case 3: \ + obj_table[i++] = ring[idx++]; /* fallthrough */ \ + case 2: \ + obj_table[i++] = ring[idx++]; /* fallthrough */ \ + case 1: \ + obj_table[i++] = ring[idx++]; \ } \ } else { \ for (i = 0; idx < size; i++, idx++) \ - obj_table[i] = r->ring[idx]; \ + obj_table[i] = ring[idx]; \ for (idx = 0; i < n; i++, idx++) \ - obj_table[i] = r->ring[idx]; \ + obj_table[i] = ring[idx]; \ } \ } while (0) +static __rte_always_inline void +update_tail(struct rte_ring_headtail *ht, uint32_t old_val, uint32_t new_val, + uint32_t single) +{ + /* + * If there are other enqueues/dequeues in progress that preceded us, + * we need to wait for them to complete + */ + if (!single) + while (unlikely(ht->tail != old_val)) + rte_pause(); + + ht->tail = new_val; +} + /** - * @internal Enqueue several objects on the ring (multi-producers safe). - * - * This function uses a "compare and set" instruction to move the - * producer index atomically. + * @internal This function updates the producer head for enqueue * * @param r - * A pointer to the ring structure. - * @param obj_table - * A pointer to a table of void * pointers (objects). + * A pointer to the ring structure + * @param is_sp + * Indicates whether multi-producer path is needed or not * @param n - * The number of objects to add in the ring from the obj_table. + * The number of elements we will want to enqueue, i.e. how far should the + * head be moved * @param behavior * RTE_RING_QUEUE_FIXED: Enqueue a fixed number of items from a ring - * RTE_RING_QUEUE_VARIABLE: Enqueue as many items a possible from ring + * RTE_RING_QUEUE_VARIABLE: Enqueue as many items as possible from ring + * @param old_head + * Returns head value as it was before the move, i.e. where enqueue starts + * @param new_head + * Returns the current/new head value i.e. where enqueue finishes + * @param free_entries + * Returns the amount of free space in the ring BEFORE head was moved * @return - * Depend on the behavior value - * if behavior = RTE_RING_QUEUE_FIXED - * - 0: Success; objects enqueue. - * - -EDQUOT: Quota exceeded. The objects have been enqueued, but the - * high water mark is exceeded. - * - -ENOBUFS: Not enough room in the ring to enqueue, no object is enqueued. - * if behavior = RTE_RING_QUEUE_VARIABLE - * - n: Actual number of objects enqueued. + * Actual number of objects enqueued. + * If behavior == RTE_RING_QUEUE_FIXED, this will be 0 or n only. */ -static inline int __attribute__((always_inline)) -__rte_ring_mp_do_enqueue(struct rte_ring *r, void * const *obj_table, - unsigned n, enum rte_ring_queue_behavior behavior) +static __rte_always_inline unsigned int +__rte_ring_move_prod_head(struct rte_ring *r, int is_sp, + unsigned int n, enum rte_ring_queue_behavior behavior, + uint32_t *old_head, uint32_t *new_head, + uint32_t *free_entries) { - uint32_t prod_head, prod_next; - uint32_t cons_tail, free_entries; - const unsigned max = n; + const uint32_t capacity = r->capacity; + unsigned int max = n; int success; - unsigned i, rep = 0; - uint32_t mask = r->prod.mask; - int ret; - /* Avoid the unnecessary cmpset operation below, which is also - * potentially harmful when n equals 0. */ - if (n == 0) - return 0; - - /* move prod.head atomically */ do { /* Reset n to the initial burst count */ n = max; - prod_head = r->prod.head; - cons_tail = r->cons.tail; - /* The subtraction is done between two unsigned 32bits value + *old_head = r->prod.head; + + /* add rmb barrier to avoid load/load reorder in weak + * memory model. It is noop on x86 + */ + rte_smp_rmb(); + + const uint32_t cons_tail = r->cons.tail; + /* + * The subtraction is done between two unsigned 32bits value * (the result is always modulo 32 bits even if we have - * prod_head > cons_tail). So 'free_entries' is always between 0 - * and size(ring)-1. */ - free_entries = (mask + cons_tail - prod_head); + * *old_head > cons_tail). So 'free_entries' is always between 0 + * and capacity (which is < size). + */ + *free_entries = (capacity + cons_tail - *old_head); /* check that we have enough room in ring */ - if (unlikely(n > free_entries)) { - if (behavior == RTE_RING_QUEUE_FIXED) { - __RING_STAT_ADD(r, enq_fail, n); - return -ENOBUFS; - } - else { - /* No free entry available */ - if (unlikely(free_entries == 0)) { - __RING_STAT_ADD(r, enq_fail, n); - return 0; - } + if (unlikely(n > *free_entries)) + n = (behavior == RTE_RING_QUEUE_FIXED) ? + 0 : *free_entries; - n = free_entries; - } - } + if (n == 0) + return 0; - prod_next = prod_head + n; - success = rte_atomic32_cmpset(&r->prod.head, prod_head, - prod_next); + *new_head = *old_head + n; + if (is_sp) + r->prod.head = *new_head, success = 1; + else + success = rte_atomic32_cmpset(&r->prod.head, + *old_head, *new_head); } while (unlikely(success == 0)); - - /* write entries in ring */ - ENQUEUE_PTRS(); - rte_smp_wmb(); - - /* if we exceed the watermark */ - if (unlikely(((mask + 1) - free_entries + n) > r->prod.watermark)) { - ret = (behavior == RTE_RING_QUEUE_FIXED) ? -EDQUOT : - (int)(n | RTE_RING_QUOT_EXCEED); - __RING_STAT_ADD(r, enq_quota, n); - } - else { - ret = (behavior == RTE_RING_QUEUE_FIXED) ? 0 : n; - __RING_STAT_ADD(r, enq_success, n); - } - - /* - * If there are other enqueues in progress that preceded us, - * we need to wait for them to complete - */ - while (unlikely(r->prod.tail != prod_head)) { - rte_pause(); - - /* Set RTE_RING_PAUSE_REP_COUNT to avoid spin too long waiting - * for other thread finish. It gives pre-empted thread a chance - * to proceed and finish with ring dequeue operation. */ - if (RTE_RING_PAUSE_REP_COUNT && - ++rep == RTE_RING_PAUSE_REP_COUNT) { - rep = 0; - sched_yield(); - } - } - r->prod.tail = prod_next; - return ret; + return n; } /** - * @internal Enqueue several objects on a ring (NOT multi-producers safe). + * @internal Enqueue several objects on the ring * - * @param r + * @param r * A pointer to the ring structure. * @param obj_table * A pointer to a table of void * pointers (objects). @@ -525,242 +454,148 @@ __rte_ring_mp_do_enqueue(struct rte_ring *r, void * const *obj_table, * The number of objects to add in the ring from the obj_table. * @param behavior * RTE_RING_QUEUE_FIXED: Enqueue a fixed number of items from a ring - * RTE_RING_QUEUE_VARIABLE: Enqueue as many items a possible from ring + * RTE_RING_QUEUE_VARIABLE: Enqueue as many items as possible from ring + * @param is_sp + * Indicates whether to use single producer or multi-producer head update + * @param free_space + * returns the amount of space after the enqueue operation has finished * @return - * Depend on the behavior value - * if behavior = RTE_RING_QUEUE_FIXED - * - 0: Success; objects enqueue. - * - -EDQUOT: Quota exceeded. The objects have been enqueued, but the - * high water mark is exceeded. - * - -ENOBUFS: Not enough room in the ring to enqueue, no object is enqueued. - * if behavior = RTE_RING_QUEUE_VARIABLE - * - n: Actual number of objects enqueued. + * Actual number of objects enqueued. + * If behavior == RTE_RING_QUEUE_FIXED, this will be 0 or n only. */ -static inline int __attribute__((always_inline)) -__rte_ring_sp_do_enqueue(struct rte_ring *r, void * const *obj_table, - unsigned n, enum rte_ring_queue_behavior behavior) +static __rte_always_inline unsigned int +__rte_ring_do_enqueue(struct rte_ring *r, void * const *obj_table, + unsigned int n, enum rte_ring_queue_behavior behavior, + int is_sp, unsigned int *free_space) { - uint32_t prod_head, cons_tail; - uint32_t prod_next, free_entries; - unsigned i; - uint32_t mask = r->prod.mask; - int ret; + uint32_t prod_head, prod_next; + uint32_t free_entries; - prod_head = r->prod.head; - cons_tail = r->cons.tail; - /* The subtraction is done between two unsigned 32bits value - * (the result is always modulo 32 bits even if we have - * prod_head > cons_tail). So 'free_entries' is always between 0 - * and size(ring)-1. */ - free_entries = mask + cons_tail - prod_head; + n = __rte_ring_move_prod_head(r, is_sp, n, behavior, + &prod_head, &prod_next, &free_entries); + if (n == 0) + goto end; - /* check that we have enough room in ring */ - if (unlikely(n > free_entries)) { - if (behavior == RTE_RING_QUEUE_FIXED) { - __RING_STAT_ADD(r, enq_fail, n); - return -ENOBUFS; - } - else { - /* No free entry available */ - if (unlikely(free_entries == 0)) { - __RING_STAT_ADD(r, enq_fail, n); - return 0; - } - - n = free_entries; - } - } - - prod_next = prod_head + n; - r->prod.head = prod_next; - - /* write entries in ring */ - ENQUEUE_PTRS(); + ENQUEUE_PTRS(r, &r[1], prod_head, obj_table, n, void *); rte_smp_wmb(); - /* if we exceed the watermark */ - if (unlikely(((mask + 1) - free_entries + n) > r->prod.watermark)) { - ret = (behavior == RTE_RING_QUEUE_FIXED) ? -EDQUOT : - (int)(n | RTE_RING_QUOT_EXCEED); - __RING_STAT_ADD(r, enq_quota, n); - } - else { - ret = (behavior == RTE_RING_QUEUE_FIXED) ? 0 : n; - __RING_STAT_ADD(r, enq_success, n); - } - - r->prod.tail = prod_next; - return ret; + update_tail(&r->prod, prod_head, prod_next, is_sp); +end: + if (free_space != NULL) + *free_space = free_entries - n; + return n; } /** - * @internal Dequeue several objects from a ring (multi-consumers safe). When - * the request objects are more than the available objects, only dequeue the - * actual number of objects - * - * This function uses a "compare and set" instruction to move the - * consumer index atomically. + * @internal This function updates the consumer head for dequeue * * @param r - * A pointer to the ring structure. - * @param obj_table - * A pointer to a table of void * pointers (objects) that will be filled. + * A pointer to the ring structure + * @param is_sc + * Indicates whether multi-consumer path is needed or not * @param n - * The number of objects to dequeue from the ring to the obj_table. + * The number of elements we will want to enqueue, i.e. how far should the + * head be moved * @param behavior * RTE_RING_QUEUE_FIXED: Dequeue a fixed number of items from a ring - * RTE_RING_QUEUE_VARIABLE: Dequeue as many items a possible from ring + * RTE_RING_QUEUE_VARIABLE: Dequeue as many items as possible from ring + * @param old_head + * Returns head value as it was before the move, i.e. where dequeue starts + * @param new_head + * Returns the current/new head value i.e. where dequeue finishes + * @param entries + * Returns the number of entries in the ring BEFORE head was moved * @return - * Depend on the behavior value - * if behavior = RTE_RING_QUEUE_FIXED - * - 0: Success; objects dequeued. - * - -ENOENT: Not enough entries in the ring to dequeue; no object is - * dequeued. - * if behavior = RTE_RING_QUEUE_VARIABLE - * - n: Actual number of objects dequeued. + * - Actual number of objects dequeued. + * If behavior == RTE_RING_QUEUE_FIXED, this will be 0 or n only. */ - -static inline int __attribute__((always_inline)) -__rte_ring_mc_do_dequeue(struct rte_ring *r, void **obj_table, - unsigned n, enum rte_ring_queue_behavior behavior) +static __rte_always_inline unsigned int +__rte_ring_move_cons_head(struct rte_ring *r, int is_sc, + unsigned int n, enum rte_ring_queue_behavior behavior, + uint32_t *old_head, uint32_t *new_head, + uint32_t *entries) { - uint32_t cons_head, prod_tail; - uint32_t cons_next, entries; - const unsigned max = n; + unsigned int max = n; int success; - unsigned i, rep = 0; - uint32_t mask = r->prod.mask; - - /* Avoid the unnecessary cmpset operation below, which is also - * potentially harmful when n equals 0. */ - if (n == 0) - return 0; /* move cons.head atomically */ do { /* Restore n as it may change every loop */ n = max; - cons_head = r->cons.head; - prod_tail = r->prod.tail; + *old_head = r->cons.head; + + /* add rmb barrier to avoid load/load reorder in weak + * memory model. It is noop on x86 + */ + rte_smp_rmb(); + + const uint32_t prod_tail = r->prod.tail; /* The subtraction is done between two unsigned 32bits value * (the result is always modulo 32 bits even if we have * cons_head > prod_tail). So 'entries' is always between 0 * and size(ring)-1. */ - entries = (prod_tail - cons_head); + *entries = (prod_tail - *old_head); /* Set the actual entries for dequeue */ - if (n > entries) { - if (behavior == RTE_RING_QUEUE_FIXED) { - __RING_STAT_ADD(r, deq_fail, n); - return -ENOENT; - } - else { - if (unlikely(entries == 0)){ - __RING_STAT_ADD(r, deq_fail, n); - return 0; - } + if (n > *entries) + n = (behavior == RTE_RING_QUEUE_FIXED) ? 0 : *entries; - n = entries; - } - } + if (unlikely(n == 0)) + return 0; - cons_next = cons_head + n; - success = rte_atomic32_cmpset(&r->cons.head, cons_head, - cons_next); + *new_head = *old_head + n; + if (is_sc) + r->cons.head = *new_head, success = 1; + else + success = rte_atomic32_cmpset(&r->cons.head, *old_head, + *new_head); } while (unlikely(success == 0)); - - /* copy in table */ - DEQUEUE_PTRS(); - rte_smp_rmb(); - - /* - * If there are other dequeues in progress that preceded us, - * we need to wait for them to complete - */ - while (unlikely(r->cons.tail != cons_head)) { - rte_pause(); - - /* Set RTE_RING_PAUSE_REP_COUNT to avoid spin too long waiting - * for other thread finish. It gives pre-empted thread a chance - * to proceed and finish with ring dequeue operation. */ - if (RTE_RING_PAUSE_REP_COUNT && - ++rep == RTE_RING_PAUSE_REP_COUNT) { - rep = 0; - sched_yield(); - } - } - __RING_STAT_ADD(r, deq_success, n); - r->cons.tail = cons_next; - - return behavior == RTE_RING_QUEUE_FIXED ? 0 : n; + return n; } /** - * @internal Dequeue several objects from a ring (NOT multi-consumers safe). - * When the request objects are more than the available objects, only dequeue - * the actual number of objects + * @internal Dequeue several objects from the ring * * @param r * A pointer to the ring structure. * @param obj_table - * A pointer to a table of void * pointers (objects) that will be filled. + * A pointer to a table of void * pointers (objects). * @param n - * The number of objects to dequeue from the ring to the obj_table. + * The number of objects to pull from the ring. * @param behavior * RTE_RING_QUEUE_FIXED: Dequeue a fixed number of items from a ring - * RTE_RING_QUEUE_VARIABLE: Dequeue as many items a possible from ring + * RTE_RING_QUEUE_VARIABLE: Dequeue as many items as possible from ring + * @param is_sc + * Indicates whether to use single consumer or multi-consumer head update + * @param available + * returns the number of remaining ring entries after the dequeue has finished * @return - * Depend on the behavior value - * if behavior = RTE_RING_QUEUE_FIXED - * - 0: Success; objects dequeued. - * - -ENOENT: Not enough entries in the ring to dequeue; no object is - * dequeued. - * if behavior = RTE_RING_QUEUE_VARIABLE - * - n: Actual number of objects dequeued. + * - Actual number of objects dequeued. + * If behavior == RTE_RING_QUEUE_FIXED, this will be 0 or n only. */ -static inline int __attribute__((always_inline)) -__rte_ring_sc_do_dequeue(struct rte_ring *r, void **obj_table, - unsigned n, enum rte_ring_queue_behavior behavior) +static __rte_always_inline unsigned int +__rte_ring_do_dequeue(struct rte_ring *r, void **obj_table, + unsigned int n, enum rte_ring_queue_behavior behavior, + int is_sc, unsigned int *available) { - uint32_t cons_head, prod_tail; - uint32_t cons_next, entries; - unsigned i; - uint32_t mask = r->prod.mask; + uint32_t cons_head, cons_next; + uint32_t entries; - cons_head = r->cons.head; - prod_tail = r->prod.tail; - /* The subtraction is done between two unsigned 32bits value - * (the result is always modulo 32 bits even if we have - * cons_head > prod_tail). So 'entries' is always between 0 - * and size(ring)-1. */ - entries = prod_tail - cons_head; + n = __rte_ring_move_cons_head(r, is_sc, n, behavior, + &cons_head, &cons_next, &entries); + if (n == 0) + goto end; - if (n > entries) { - if (behavior == RTE_RING_QUEUE_FIXED) { - __RING_STAT_ADD(r, deq_fail, n); - return -ENOENT; - } - else { - if (unlikely(entries == 0)){ - __RING_STAT_ADD(r, deq_fail, n); - return 0; - } - - n = entries; - } - } - - cons_next = cons_head + n; - r->cons.head = cons_next; - - /* copy in table */ - DEQUEUE_PTRS(); + DEQUEUE_PTRS(r, &r[1], cons_head, obj_table, n, void *); rte_smp_rmb(); - __RING_STAT_ADD(r, deq_success, n); - r->cons.tail = cons_next; - return behavior == RTE_RING_QUEUE_FIXED ? 0 : n; + update_tail(&r->cons, cons_head, cons_next, is_sc); + +end: + if (available != NULL) + *available = entries - n; + return n; } /** @@ -775,17 +610,18 @@ __rte_ring_sc_do_dequeue(struct rte_ring *r, void **obj_table, * A pointer to a table of void * pointers (objects). * @param n * The number of objects to add in the ring from the obj_table. + * @param free_space + * if non-NULL, returns the amount of space in the ring after the + * enqueue operation has finished. * @return - * - 0: Success; objects enqueue. - * - -EDQUOT: Quota exceeded. The objects have been enqueued, but the - * high water mark is exceeded. - * - -ENOBUFS: Not enough room in the ring to enqueue, no object is enqueued. + * The number of objects enqueued, either 0 or n */ -static inline int __attribute__((always_inline)) +static __rte_always_inline unsigned int rte_ring_mp_enqueue_bulk(struct rte_ring *r, void * const *obj_table, - unsigned n) + unsigned int n, unsigned int *free_space) { - return __rte_ring_mp_do_enqueue(r, obj_table, n, RTE_RING_QUEUE_FIXED); + return __rte_ring_do_enqueue(r, obj_table, n, RTE_RING_QUEUE_FIXED, + __IS_MP, free_space); } /** @@ -797,17 +633,18 @@ rte_ring_mp_enqueue_bulk(struct rte_ring *r, void * const *obj_table, * A pointer to a table of void * pointers (objects). * @param n * The number of objects to add in the ring from the obj_table. + * @param free_space + * if non-NULL, returns the amount of space in the ring after the + * enqueue operation has finished. * @return - * - 0: Success; objects enqueued. - * - -EDQUOT: Quota exceeded. The objects have been enqueued, but the - * high water mark is exceeded. - * - -ENOBUFS: Not enough room in the ring to enqueue; no object is enqueued. + * The number of objects enqueued, either 0 or n */ -static inline int __attribute__((always_inline)) +static __rte_always_inline unsigned int rte_ring_sp_enqueue_bulk(struct rte_ring *r, void * const *obj_table, - unsigned n) + unsigned int n, unsigned int *free_space) { - return __rte_ring_sp_do_enqueue(r, obj_table, n, RTE_RING_QUEUE_FIXED); + return __rte_ring_do_enqueue(r, obj_table, n, RTE_RING_QUEUE_FIXED, + __IS_SP, free_space); } /** @@ -823,20 +660,18 @@ rte_ring_sp_enqueue_bulk(struct rte_ring *r, void * const *obj_table, * A pointer to a table of void * pointers (objects). * @param n * The number of objects to add in the ring from the obj_table. + * @param free_space + * if non-NULL, returns the amount of space in the ring after the + * enqueue operation has finished. * @return - * - 0: Success; objects enqueued. - * - -EDQUOT: Quota exceeded. The objects have been enqueued, but the - * high water mark is exceeded. - * - -ENOBUFS: Not enough room in the ring to enqueue; no object is enqueued. + * The number of objects enqueued, either 0 or n */ -static inline int __attribute__((always_inline)) +static __rte_always_inline unsigned int rte_ring_enqueue_bulk(struct rte_ring *r, void * const *obj_table, - unsigned n) + unsigned int n, unsigned int *free_space) { - if (r->prod.sp_enqueue) - return rte_ring_sp_enqueue_bulk(r, obj_table, n); - else - return rte_ring_mp_enqueue_bulk(r, obj_table, n); + return __rte_ring_do_enqueue(r, obj_table, n, RTE_RING_QUEUE_FIXED, + r->prod.single, free_space); } /** @@ -851,14 +686,12 @@ rte_ring_enqueue_bulk(struct rte_ring *r, void * const *obj_table, * A pointer to the object to be added. * @return * - 0: Success; objects enqueued. - * - -EDQUOT: Quota exceeded. The objects have been enqueued, but the - * high water mark is exceeded. * - -ENOBUFS: Not enough room in the ring to enqueue; no object is enqueued. */ -static inline int __attribute__((always_inline)) +static __rte_always_inline int rte_ring_mp_enqueue(struct rte_ring *r, void *obj) { - return rte_ring_mp_enqueue_bulk(r, &obj, 1); + return rte_ring_mp_enqueue_bulk(r, &obj, 1, NULL) ? 0 : -ENOBUFS; } /** @@ -870,14 +703,12 @@ rte_ring_mp_enqueue(struct rte_ring *r, void *obj) * A pointer to the object to be added. * @return * - 0: Success; objects enqueued. - * - -EDQUOT: Quota exceeded. The objects have been enqueued, but the - * high water mark is exceeded. * - -ENOBUFS: Not enough room in the ring to enqueue; no object is enqueued. */ -static inline int __attribute__((always_inline)) +static __rte_always_inline int rte_ring_sp_enqueue(struct rte_ring *r, void *obj) { - return rte_ring_sp_enqueue_bulk(r, &obj, 1); + return rte_ring_sp_enqueue_bulk(r, &obj, 1, NULL) ? 0 : -ENOBUFS; } /** @@ -893,17 +724,12 @@ rte_ring_sp_enqueue(struct rte_ring *r, void *obj) * A pointer to the object to be added. * @return * - 0: Success; objects enqueued. - * - -EDQUOT: Quota exceeded. The objects have been enqueued, but the - * high water mark is exceeded. * - -ENOBUFS: Not enough room in the ring to enqueue; no object is enqueued. */ -static inline int __attribute__((always_inline)) +static __rte_always_inline int rte_ring_enqueue(struct rte_ring *r, void *obj) { - if (r->prod.sp_enqueue) - return rte_ring_sp_enqueue(r, obj); - else - return rte_ring_mp_enqueue(r, obj); + return rte_ring_enqueue_bulk(r, &obj, 1, NULL) ? 0 : -ENOBUFS; } /** @@ -918,15 +744,18 @@ rte_ring_enqueue(struct rte_ring *r, void *obj) * A pointer to a table of void * pointers (objects) that will be filled. * @param n * The number of objects to dequeue from the ring to the obj_table. + * @param available + * If non-NULL, returns the number of remaining ring entries after the + * dequeue has finished. * @return - * - 0: Success; objects dequeued. - * - -ENOENT: Not enough entries in the ring to dequeue; no object is - * dequeued. + * The number of objects dequeued, either 0 or n */ -static inline int __attribute__((always_inline)) -rte_ring_mc_dequeue_bulk(struct rte_ring *r, void **obj_table, unsigned n) +static __rte_always_inline unsigned int +rte_ring_mc_dequeue_bulk(struct rte_ring *r, void **obj_table, + unsigned int n, unsigned int *available) { - return __rte_ring_mc_do_dequeue(r, obj_table, n, RTE_RING_QUEUE_FIXED); + return __rte_ring_do_dequeue(r, obj_table, n, RTE_RING_QUEUE_FIXED, + __IS_MC, available); } /** @@ -939,15 +768,18 @@ rte_ring_mc_dequeue_bulk(struct rte_ring *r, void **obj_table, unsigned n) * @param n * The number of objects to dequeue from the ring to the obj_table, * must be strictly positive. + * @param available + * If non-NULL, returns the number of remaining ring entries after the + * dequeue has finished. * @return - * - 0: Success; objects dequeued. - * - -ENOENT: Not enough entries in the ring to dequeue; no object is - * dequeued. + * The number of objects dequeued, either 0 or n */ -static inline int __attribute__((always_inline)) -rte_ring_sc_dequeue_bulk(struct rte_ring *r, void **obj_table, unsigned n) +static __rte_always_inline unsigned int +rte_ring_sc_dequeue_bulk(struct rte_ring *r, void **obj_table, + unsigned int n, unsigned int *available) { - return __rte_ring_sc_do_dequeue(r, obj_table, n, RTE_RING_QUEUE_FIXED); + return __rte_ring_do_dequeue(r, obj_table, n, RTE_RING_QUEUE_FIXED, + __IS_SC, available); } /** @@ -963,18 +795,18 @@ rte_ring_sc_dequeue_bulk(struct rte_ring *r, void **obj_table, unsigned n) * A pointer to a table of void * pointers (objects) that will be filled. * @param n * The number of objects to dequeue from the ring to the obj_table. + * @param available + * If non-NULL, returns the number of remaining ring entries after the + * dequeue has finished. * @return - * - 0: Success; objects dequeued. - * - -ENOENT: Not enough entries in the ring to dequeue, no object is - * dequeued. + * The number of objects dequeued, either 0 or n */ -static inline int __attribute__((always_inline)) -rte_ring_dequeue_bulk(struct rte_ring *r, void **obj_table, unsigned n) +static __rte_always_inline unsigned int +rte_ring_dequeue_bulk(struct rte_ring *r, void **obj_table, unsigned int n, + unsigned int *available) { - if (r->cons.sc_dequeue) - return rte_ring_sc_dequeue_bulk(r, obj_table, n); - else - return rte_ring_mc_dequeue_bulk(r, obj_table, n); + return __rte_ring_do_dequeue(r, obj_table, n, RTE_RING_QUEUE_FIXED, + r->cons.single, available); } /** @@ -992,10 +824,10 @@ rte_ring_dequeue_bulk(struct rte_ring *r, void **obj_table, unsigned n) * - -ENOENT: Not enough entries in the ring to dequeue; no object is * dequeued. */ -static inline int __attribute__((always_inline)) +static __rte_always_inline int rte_ring_mc_dequeue(struct rte_ring *r, void **obj_p) { - return rte_ring_mc_dequeue_bulk(r, obj_p, 1); + return rte_ring_mc_dequeue_bulk(r, obj_p, 1, NULL) ? 0 : -ENOENT; } /** @@ -1010,10 +842,10 @@ rte_ring_mc_dequeue(struct rte_ring *r, void **obj_p) * - -ENOENT: Not enough entries in the ring to dequeue, no object is * dequeued. */ -static inline int __attribute__((always_inline)) +static __rte_always_inline int rte_ring_sc_dequeue(struct rte_ring *r, void **obj_p) { - return rte_ring_sc_dequeue_bulk(r, obj_p, 1); + return rte_ring_sc_dequeue_bulk(r, obj_p, 1, NULL) ? 0 : -ENOENT; } /** @@ -1032,47 +864,10 @@ rte_ring_sc_dequeue(struct rte_ring *r, void **obj_p) * - -ENOENT: Not enough entries in the ring to dequeue, no object is * dequeued. */ -static inline int __attribute__((always_inline)) +static __rte_always_inline int rte_ring_dequeue(struct rte_ring *r, void **obj_p) { - if (r->cons.sc_dequeue) - return rte_ring_sc_dequeue(r, obj_p); - else - return rte_ring_mc_dequeue(r, obj_p); -} - -/** - * Test if a ring is full. - * - * @param r - * A pointer to the ring structure. - * @return - * - 1: The ring is full. - * - 0: The ring is not full. - */ -static inline int -rte_ring_full(const struct rte_ring *r) -{ - uint32_t prod_tail = r->prod.tail; - uint32_t cons_tail = r->cons.tail; - return ((cons_tail - prod_tail - 1) & r->prod.mask) == 0; -} - -/** - * Test if a ring is empty. - * - * @param r - * A pointer to the ring structure. - * @return - * - 1: The ring is empty. - * - 0: The ring is not empty. - */ -static inline int -rte_ring_empty(const struct rte_ring *r) -{ - uint32_t prod_tail = r->prod.tail; - uint32_t cons_tail = r->cons.tail; - return !!(cons_tail == prod_tail); + return rte_ring_dequeue_bulk(r, obj_p, 1, NULL) ? 0 : -ENOENT; } /** @@ -1088,7 +883,8 @@ rte_ring_count(const struct rte_ring *r) { uint32_t prod_tail = r->prod.tail; uint32_t cons_tail = r->cons.tail; - return (prod_tail - cons_tail) & r->prod.mask; + uint32_t count = (prod_tail - cons_tail) & r->mask; + return (count > r->capacity) ? r->capacity : count; } /** @@ -1102,9 +898,67 @@ rte_ring_count(const struct rte_ring *r) static inline unsigned rte_ring_free_count(const struct rte_ring *r) { - uint32_t prod_tail = r->prod.tail; - uint32_t cons_tail = r->cons.tail; - return (cons_tail - prod_tail - 1) & r->prod.mask; + return r->capacity - rte_ring_count(r); +} + +/** + * Test if a ring is full. + * + * @param r + * A pointer to the ring structure. + * @return + * - 1: The ring is full. + * - 0: The ring is not full. + */ +static inline int +rte_ring_full(const struct rte_ring *r) +{ + return rte_ring_free_count(r) == 0; +} + +/** + * Test if a ring is empty. + * + * @param r + * A pointer to the ring structure. + * @return + * - 1: The ring is empty. + * - 0: The ring is not empty. + */ +static inline int +rte_ring_empty(const struct rte_ring *r) +{ + return rte_ring_count(r) == 0; +} + +/** + * Return the size of the ring. + * + * @param r + * A pointer to the ring structure. + * @return + * The size of the data store used by the ring. + * NOTE: this is not the same as the usable space in the ring. To query that + * use ``rte_ring_get_capacity()``. + */ +static inline unsigned int +rte_ring_get_size(const struct rte_ring *r) +{ + return r->size; +} + +/** + * Return the number of elements which can be stored in the ring. + * + * @param r + * A pointer to the ring structure. + * @return + * The usable size of the ring. + */ +static inline unsigned int +rte_ring_get_capacity(const struct rte_ring *r) +{ + return r->capacity; } /** @@ -1139,14 +993,18 @@ struct rte_ring *rte_ring_lookup(const char *name); * A pointer to a table of void * pointers (objects). * @param n * The number of objects to add in the ring from the obj_table. + * @param free_space + * if non-NULL, returns the amount of space in the ring after the + * enqueue operation has finished. * @return * - n: Actual number of objects enqueued. */ -static inline unsigned __attribute__((always_inline)) +static __rte_always_inline unsigned rte_ring_mp_enqueue_burst(struct rte_ring *r, void * const *obj_table, - unsigned n) + unsigned int n, unsigned int *free_space) { - return __rte_ring_mp_do_enqueue(r, obj_table, n, RTE_RING_QUEUE_VARIABLE); + return __rte_ring_do_enqueue(r, obj_table, n, + RTE_RING_QUEUE_VARIABLE, __IS_MP, free_space); } /** @@ -1158,14 +1016,18 @@ rte_ring_mp_enqueue_burst(struct rte_ring *r, void * const *obj_table, * A pointer to a table of void * pointers (objects). * @param n * The number of objects to add in the ring from the obj_table. + * @param free_space + * if non-NULL, returns the amount of space in the ring after the + * enqueue operation has finished. * @return * - n: Actual number of objects enqueued. */ -static inline unsigned __attribute__((always_inline)) +static __rte_always_inline unsigned rte_ring_sp_enqueue_burst(struct rte_ring *r, void * const *obj_table, - unsigned n) + unsigned int n, unsigned int *free_space) { - return __rte_ring_sp_do_enqueue(r, obj_table, n, RTE_RING_QUEUE_VARIABLE); + return __rte_ring_do_enqueue(r, obj_table, n, + RTE_RING_QUEUE_VARIABLE, __IS_SP, free_space); } /** @@ -1181,17 +1043,18 @@ rte_ring_sp_enqueue_burst(struct rte_ring *r, void * const *obj_table, * A pointer to a table of void * pointers (objects). * @param n * The number of objects to add in the ring from the obj_table. + * @param free_space + * if non-NULL, returns the amount of space in the ring after the + * enqueue operation has finished. * @return * - n: Actual number of objects enqueued. */ -static inline unsigned __attribute__((always_inline)) +static __rte_always_inline unsigned rte_ring_enqueue_burst(struct rte_ring *r, void * const *obj_table, - unsigned n) + unsigned int n, unsigned int *free_space) { - if (r->prod.sp_enqueue) - return rte_ring_sp_enqueue_burst(r, obj_table, n); - else - return rte_ring_mp_enqueue_burst(r, obj_table, n); + return __rte_ring_do_enqueue(r, obj_table, n, RTE_RING_QUEUE_VARIABLE, + r->prod.single, free_space); } /** @@ -1208,13 +1071,18 @@ rte_ring_enqueue_burst(struct rte_ring *r, void * const *obj_table, * A pointer to a table of void * pointers (objects) that will be filled. * @param n * The number of objects to dequeue from the ring to the obj_table. + * @param available + * If non-NULL, returns the number of remaining ring entries after the + * dequeue has finished. * @return * - n: Actual number of objects dequeued, 0 if ring is empty */ -static inline unsigned __attribute__((always_inline)) -rte_ring_mc_dequeue_burst(struct rte_ring *r, void **obj_table, unsigned n) +static __rte_always_inline unsigned +rte_ring_mc_dequeue_burst(struct rte_ring *r, void **obj_table, + unsigned int n, unsigned int *available) { - return __rte_ring_mc_do_dequeue(r, obj_table, n, RTE_RING_QUEUE_VARIABLE); + return __rte_ring_do_dequeue(r, obj_table, n, + RTE_RING_QUEUE_VARIABLE, __IS_MC, available); } /** @@ -1228,13 +1096,18 @@ rte_ring_mc_dequeue_burst(struct rte_ring *r, void **obj_table, unsigned n) * A pointer to a table of void * pointers (objects) that will be filled. * @param n * The number of objects to dequeue from the ring to the obj_table. + * @param available + * If non-NULL, returns the number of remaining ring entries after the + * dequeue has finished. * @return * - n: Actual number of objects dequeued, 0 if ring is empty */ -static inline unsigned __attribute__((always_inline)) -rte_ring_sc_dequeue_burst(struct rte_ring *r, void **obj_table, unsigned n) +static __rte_always_inline unsigned +rte_ring_sc_dequeue_burst(struct rte_ring *r, void **obj_table, + unsigned int n, unsigned int *available) { - return __rte_ring_sc_do_dequeue(r, obj_table, n, RTE_RING_QUEUE_VARIABLE); + return __rte_ring_do_dequeue(r, obj_table, n, + RTE_RING_QUEUE_VARIABLE, __IS_SC, available); } /** @@ -1250,16 +1123,19 @@ rte_ring_sc_dequeue_burst(struct rte_ring *r, void **obj_table, unsigned n) * A pointer to a table of void * pointers (objects) that will be filled. * @param n * The number of objects to dequeue from the ring to the obj_table. + * @param available + * If non-NULL, returns the number of remaining ring entries after the + * dequeue has finished. * @return * - Number of objects dequeued */ -static inline unsigned __attribute__((always_inline)) -rte_ring_dequeue_burst(struct rte_ring *r, void **obj_table, unsigned n) +static __rte_always_inline unsigned +rte_ring_dequeue_burst(struct rte_ring *r, void **obj_table, + unsigned int n, unsigned int *available) { - if (r->cons.sc_dequeue) - return rte_ring_sc_dequeue_burst(r, obj_table, n); - else - return rte_ring_mc_dequeue_burst(r, obj_table, n); + return __rte_ring_do_dequeue(r, obj_table, n, + RTE_RING_QUEUE_VARIABLE, + r->cons.single, available); } #ifdef __cplusplus diff --git a/dpdk/lib/librte_sched/Makefile b/dpdk/lib/librte_sched/Makefile index 44cb780f..04785f72 100644 --- a/dpdk/lib/librte_sched/Makefile +++ b/dpdk/lib/librte_sched/Makefile @@ -43,6 +43,8 @@ CFLAGS_rte_red.o := -D_GNU_SOURCE LDLIBS += -lm LDLIBS += -lrt +LDLIBS += -lrte_eal -lrte_mempool -lrte_mbuf -lrte_net +LDLIBS += -lrte_timer EXPORT_MAP := rte_sched_version.map @@ -55,12 +57,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_SCHED) += rte_sched.c rte_red.c rte_approx.c SRCS-$(CONFIG_RTE_LIBRTE_SCHED) += rte_reciprocal.c # install includes -SYMLINK-$(CONFIG_RTE_LIBRTE_SCHED)-include := rte_sched.h rte_bitmap.h rte_sched_common.h rte_red.h rte_approx.h +SYMLINK-$(CONFIG_RTE_LIBRTE_SCHED)-include := rte_sched.h rte_sched_common.h rte_red.h rte_approx.h SYMLINK-$(CONFIG_RTE_LIBRTE_SCHED)-include += rte_reciprocal.h -# this lib depends upon: -DEPDIRS-$(CONFIG_RTE_LIBRTE_SCHED) += lib/librte_eal -DEPDIRS-$(CONFIG_RTE_LIBRTE_SCHED) += lib/librte_mempool lib/librte_mbuf -DEPDIRS-$(CONFIG_RTE_LIBRTE_SCHED) += lib/librte_net lib/librte_timer - include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/lib/librte_sched/rte_bitmap.h b/dpdk/lib/librte_sched/rte_bitmap.h deleted file mode 100644 index ff675c58..00000000 --- a/dpdk/lib/librte_sched/rte_bitmap.h +++ /dev/null @@ -1,560 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef __INCLUDE_RTE_BITMAP_H__ -#define __INCLUDE_RTE_BITMAP_H__ - -#ifdef __cplusplus -extern "C" { -#endif - -/** - * @file - * RTE Bitmap - * - * The bitmap component provides a mechanism to manage large arrays of bits - * through bit get/set/clear and bit array scan operations. - * - * The bitmap scan operation is optimized for 64-bit CPUs using 64/128 byte cache - * lines. The bitmap is hierarchically organized using two arrays (array1 and - * array2), with each bit in array1 being associated with a full cache line - * (512/1024 bits) of bitmap bits, which are stored in array2: the bit in array1 - * is set only when there is at least one bit set within its associated array2 - * bits, otherwise the bit in array1 is cleared. The read and write operations - * for array1 and array2 are always done in slabs of 64 bits. - * - * This bitmap is not thread safe. For lock free operation on a specific bitmap - * instance, a single writer thread performing bit set/clear operations is - * allowed, only the writer thread can do bitmap scan operations, while there - * can be several reader threads performing bit get operations in parallel with - * the writer thread. When the use of locking primitives is acceptable, the - * serialization of the bit set/clear and bitmap scan operations needs to be - * enforced by the caller, while the bit get operation does not require locking - * the bitmap. - * - ***/ - -#include -#include -#include -#include -#include - -#ifndef RTE_BITMAP_OPTIMIZATIONS -#define RTE_BITMAP_OPTIMIZATIONS 1 -#endif - -/* Slab */ -#define RTE_BITMAP_SLAB_BIT_SIZE 64 -#define RTE_BITMAP_SLAB_BIT_SIZE_LOG2 6 -#define RTE_BITMAP_SLAB_BIT_MASK (RTE_BITMAP_SLAB_BIT_SIZE - 1) - -/* Cache line (CL) */ -#define RTE_BITMAP_CL_BIT_SIZE (RTE_CACHE_LINE_SIZE * 8) -#define RTE_BITMAP_CL_BIT_SIZE_LOG2 (RTE_CACHE_LINE_SIZE_LOG2 + 3) -#define RTE_BITMAP_CL_BIT_MASK (RTE_BITMAP_CL_BIT_SIZE - 1) - -#define RTE_BITMAP_CL_SLAB_SIZE (RTE_BITMAP_CL_BIT_SIZE / RTE_BITMAP_SLAB_BIT_SIZE) -#define RTE_BITMAP_CL_SLAB_SIZE_LOG2 (RTE_BITMAP_CL_BIT_SIZE_LOG2 - RTE_BITMAP_SLAB_BIT_SIZE_LOG2) -#define RTE_BITMAP_CL_SLAB_MASK (RTE_BITMAP_CL_SLAB_SIZE - 1) - -/** Bitmap data structure */ -struct rte_bitmap { - /* Context for array1 and array2 */ - uint64_t *array1; /**< Bitmap array1 */ - uint64_t *array2; /**< Bitmap array2 */ - uint32_t array1_size; /**< Number of 64-bit slabs in array1 that are actually used */ - uint32_t array2_size; /**< Number of 64-bit slabs in array2 */ - - /* Context for the "scan next" operation */ - uint32_t index1; /**< Bitmap scan: Index of current array1 slab */ - uint32_t offset1; /**< Bitmap scan: Offset of current bit within current array1 slab */ - uint32_t index2; /**< Bitmap scan: Index of current array2 slab */ - uint32_t go2; /**< Bitmap scan: Go/stop condition for current array2 cache line */ - - /* Storage space for array1 and array2 */ - uint8_t memory[0]; -}; - -static inline void -__rte_bitmap_index1_inc(struct rte_bitmap *bmp) -{ - bmp->index1 = (bmp->index1 + 1) & (bmp->array1_size - 1); -} - -static inline uint64_t -__rte_bitmap_mask1_get(struct rte_bitmap *bmp) -{ - return (~1lu) << bmp->offset1; -} - -static inline void -__rte_bitmap_index2_set(struct rte_bitmap *bmp) -{ - bmp->index2 = (((bmp->index1 << RTE_BITMAP_SLAB_BIT_SIZE_LOG2) + bmp->offset1) << RTE_BITMAP_CL_SLAB_SIZE_LOG2); -} - -#if RTE_BITMAP_OPTIMIZATIONS - -static inline int -rte_bsf64(uint64_t slab, uint32_t *pos) -{ - if (likely(slab == 0)) { - return 0; - } - - *pos = __builtin_ctzll(slab); - return 1; -} - -#else - -static inline int -rte_bsf64(uint64_t slab, uint32_t *pos) -{ - uint64_t mask; - uint32_t i; - - if (likely(slab == 0)) { - return 0; - } - - for (i = 0, mask = 1; i < RTE_BITMAP_SLAB_BIT_SIZE; i ++, mask <<= 1) { - if (unlikely(slab & mask)) { - *pos = i; - return 1; - } - } - - return 0; -} - -#endif - -static inline uint32_t -__rte_bitmap_get_memory_footprint(uint32_t n_bits, - uint32_t *array1_byte_offset, uint32_t *array1_slabs, - uint32_t *array2_byte_offset, uint32_t *array2_slabs) -{ - uint32_t n_slabs_context, n_slabs_array1, n_cache_lines_context_and_array1; - uint32_t n_cache_lines_array2; - uint32_t n_bytes_total; - - n_cache_lines_array2 = (n_bits + RTE_BITMAP_CL_BIT_SIZE - 1) / RTE_BITMAP_CL_BIT_SIZE; - n_slabs_array1 = (n_cache_lines_array2 + RTE_BITMAP_SLAB_BIT_SIZE - 1) / RTE_BITMAP_SLAB_BIT_SIZE; - n_slabs_array1 = rte_align32pow2(n_slabs_array1); - n_slabs_context = (sizeof(struct rte_bitmap) + (RTE_BITMAP_SLAB_BIT_SIZE / 8) - 1) / (RTE_BITMAP_SLAB_BIT_SIZE / 8); - n_cache_lines_context_and_array1 = (n_slabs_context + n_slabs_array1 + RTE_BITMAP_CL_SLAB_SIZE - 1) / RTE_BITMAP_CL_SLAB_SIZE; - n_bytes_total = (n_cache_lines_context_and_array1 + n_cache_lines_array2) * RTE_CACHE_LINE_SIZE; - - if (array1_byte_offset) { - *array1_byte_offset = n_slabs_context * (RTE_BITMAP_SLAB_BIT_SIZE / 8); - } - if (array1_slabs) { - *array1_slabs = n_slabs_array1; - } - if (array2_byte_offset) { - *array2_byte_offset = n_cache_lines_context_and_array1 * RTE_CACHE_LINE_SIZE; - } - if (array2_slabs) { - *array2_slabs = n_cache_lines_array2 * RTE_BITMAP_CL_SLAB_SIZE; - } - - return n_bytes_total; -} - -static inline void -__rte_bitmap_scan_init(struct rte_bitmap *bmp) -{ - bmp->index1 = bmp->array1_size - 1; - bmp->offset1 = RTE_BITMAP_SLAB_BIT_SIZE - 1; - __rte_bitmap_index2_set(bmp); - bmp->index2 += RTE_BITMAP_CL_SLAB_SIZE; - - bmp->go2 = 0; -} - -/** - * Bitmap memory footprint calculation - * - * @param n_bits - * Number of bits in the bitmap - * @return - * Bitmap memory footprint measured in bytes on success, 0 on error - */ -static inline uint32_t -rte_bitmap_get_memory_footprint(uint32_t n_bits) { - /* Check input arguments */ - if (n_bits == 0) { - return 0; - } - - return __rte_bitmap_get_memory_footprint(n_bits, NULL, NULL, NULL, NULL); -} - -/** - * Bitmap initialization - * - * @param mem_size - * Minimum expected size of bitmap. - * @param mem - * Base address of array1 and array2. - * @param n_bits - * Number of pre-allocated bits in array2. Must be non-zero and multiple of 512. - * @return - * Handle to bitmap instance. - */ -static inline struct rte_bitmap * -rte_bitmap_init(uint32_t n_bits, uint8_t *mem, uint32_t mem_size) -{ - struct rte_bitmap *bmp; - uint32_t array1_byte_offset, array1_slabs, array2_byte_offset, array2_slabs; - uint32_t size; - - /* Check input arguments */ - if (n_bits == 0) { - return NULL; - } - - if ((mem == NULL) || (((uintptr_t) mem) & RTE_CACHE_LINE_MASK)) { - return NULL; - } - - size = __rte_bitmap_get_memory_footprint(n_bits, - &array1_byte_offset, &array1_slabs, - &array2_byte_offset, &array2_slabs); - if (size < mem_size) { - return NULL; - } - - /* Setup bitmap */ - memset(mem, 0, size); - bmp = (struct rte_bitmap *) mem; - - bmp->array1 = (uint64_t *) &mem[array1_byte_offset]; - bmp->array1_size = array1_slabs; - bmp->array2 = (uint64_t *) &mem[array2_byte_offset]; - bmp->array2_size = array2_slabs; - - __rte_bitmap_scan_init(bmp); - - return bmp; -} - -/** - * Bitmap free - * - * @param bmp - * Handle to bitmap instance - * @return - * 0 upon success, error code otherwise - */ -static inline int -rte_bitmap_free(struct rte_bitmap *bmp) -{ - /* Check input arguments */ - if (bmp == NULL) { - return -1; - } - - return 0; -} - -/** - * Bitmap reset - * - * @param bmp - * Handle to bitmap instance - */ -static inline void -rte_bitmap_reset(struct rte_bitmap *bmp) -{ - memset(bmp->array1, 0, bmp->array1_size * sizeof(uint64_t)); - memset(bmp->array2, 0, bmp->array2_size * sizeof(uint64_t)); - __rte_bitmap_scan_init(bmp); -} - -/** - * Bitmap location prefetch into CPU L1 cache - * - * @param bmp - * Handle to bitmap instance - * @param pos - * Bit position - * @return - * 0 upon success, error code otherwise - */ -static inline void -rte_bitmap_prefetch0(struct rte_bitmap *bmp, uint32_t pos) -{ - uint64_t *slab2; - uint32_t index2; - - index2 = pos >> RTE_BITMAP_SLAB_BIT_SIZE_LOG2; - slab2 = bmp->array2 + index2; - rte_prefetch0((void *) slab2); -} - -/** - * Bitmap bit get - * - * @param bmp - * Handle to bitmap instance - * @param pos - * Bit position - * @return - * 0 when bit is cleared, non-zero when bit is set - */ -static inline uint64_t -rte_bitmap_get(struct rte_bitmap *bmp, uint32_t pos) -{ - uint64_t *slab2; - uint32_t index2, offset2; - - index2 = pos >> RTE_BITMAP_SLAB_BIT_SIZE_LOG2; - offset2 = pos & RTE_BITMAP_SLAB_BIT_MASK; - slab2 = bmp->array2 + index2; - return (*slab2) & (1lu << offset2); -} - -/** - * Bitmap bit set - * - * @param bmp - * Handle to bitmap instance - * @param pos - * Bit position - */ -static inline void -rte_bitmap_set(struct rte_bitmap *bmp, uint32_t pos) -{ - uint64_t *slab1, *slab2; - uint32_t index1, index2, offset1, offset2; - - /* Set bit in array2 slab and set bit in array1 slab */ - index2 = pos >> RTE_BITMAP_SLAB_BIT_SIZE_LOG2; - offset2 = pos & RTE_BITMAP_SLAB_BIT_MASK; - index1 = pos >> (RTE_BITMAP_SLAB_BIT_SIZE_LOG2 + RTE_BITMAP_CL_BIT_SIZE_LOG2); - offset1 = (pos >> RTE_BITMAP_CL_BIT_SIZE_LOG2) & RTE_BITMAP_SLAB_BIT_MASK; - slab2 = bmp->array2 + index2; - slab1 = bmp->array1 + index1; - - *slab2 |= 1lu << offset2; - *slab1 |= 1lu << offset1; -} - -/** - * Bitmap slab set - * - * @param bmp - * Handle to bitmap instance - * @param pos - * Bit position identifying the array2 slab - * @param slab - * Value to be assigned to the 64-bit slab in array2 - */ -static inline void -rte_bitmap_set_slab(struct rte_bitmap *bmp, uint32_t pos, uint64_t slab) -{ - uint64_t *slab1, *slab2; - uint32_t index1, index2, offset1; - - /* Set bits in array2 slab and set bit in array1 slab */ - index2 = pos >> RTE_BITMAP_SLAB_BIT_SIZE_LOG2; - index1 = pos >> (RTE_BITMAP_SLAB_BIT_SIZE_LOG2 + RTE_BITMAP_CL_BIT_SIZE_LOG2); - offset1 = (pos >> RTE_BITMAP_CL_BIT_SIZE_LOG2) & RTE_BITMAP_SLAB_BIT_MASK; - slab2 = bmp->array2 + index2; - slab1 = bmp->array1 + index1; - - *slab2 |= slab; - *slab1 |= 1lu << offset1; -} - -static inline uint64_t -__rte_bitmap_line_not_empty(uint64_t *slab2) -{ - uint64_t v1, v2, v3, v4; - - v1 = slab2[0] | slab2[1]; - v2 = slab2[2] | slab2[3]; - v3 = slab2[4] | slab2[5]; - v4 = slab2[6] | slab2[7]; - v1 |= v2; - v3 |= v4; - - return v1 | v3; -} - -/** - * Bitmap bit clear - * - * @param bmp - * Handle to bitmap instance - * @param pos - * Bit position - */ -static inline void -rte_bitmap_clear(struct rte_bitmap *bmp, uint32_t pos) -{ - uint64_t *slab1, *slab2; - uint32_t index1, index2, offset1, offset2; - - /* Clear bit in array2 slab */ - index2 = pos >> RTE_BITMAP_SLAB_BIT_SIZE_LOG2; - offset2 = pos & RTE_BITMAP_SLAB_BIT_MASK; - slab2 = bmp->array2 + index2; - - /* Return if array2 slab is not all-zeros */ - *slab2 &= ~(1lu << offset2); - if (*slab2){ - return; - } - - /* Check the entire cache line of array2 for all-zeros */ - index2 &= ~ RTE_BITMAP_CL_SLAB_MASK; - slab2 = bmp->array2 + index2; - if (__rte_bitmap_line_not_empty(slab2)) { - return; - } - - /* The array2 cache line is all-zeros, so clear bit in array1 slab */ - index1 = pos >> (RTE_BITMAP_SLAB_BIT_SIZE_LOG2 + RTE_BITMAP_CL_BIT_SIZE_LOG2); - offset1 = (pos >> RTE_BITMAP_CL_BIT_SIZE_LOG2) & RTE_BITMAP_SLAB_BIT_MASK; - slab1 = bmp->array1 + index1; - *slab1 &= ~(1lu << offset1); - - return; -} - -static inline int -__rte_bitmap_scan_search(struct rte_bitmap *bmp) -{ - uint64_t value1; - uint32_t i; - - /* Check current array1 slab */ - value1 = bmp->array1[bmp->index1]; - value1 &= __rte_bitmap_mask1_get(bmp); - - if (rte_bsf64(value1, &bmp->offset1)) { - return 1; - } - - __rte_bitmap_index1_inc(bmp); - bmp->offset1 = 0; - - /* Look for another array1 slab */ - for (i = 0; i < bmp->array1_size; i ++, __rte_bitmap_index1_inc(bmp)) { - value1 = bmp->array1[bmp->index1]; - - if (rte_bsf64(value1, &bmp->offset1)) { - return 1; - } - } - - return 0; -} - -static inline void -__rte_bitmap_scan_read_init(struct rte_bitmap *bmp) -{ - __rte_bitmap_index2_set(bmp); - bmp->go2 = 1; - rte_prefetch1((void *)(bmp->array2 + bmp->index2 + 8)); -} - -static inline int -__rte_bitmap_scan_read(struct rte_bitmap *bmp, uint32_t *pos, uint64_t *slab) -{ - uint64_t *slab2; - - slab2 = bmp->array2 + bmp->index2; - for ( ; bmp->go2 ; bmp->index2 ++, slab2 ++, bmp->go2 = bmp->index2 & RTE_BITMAP_CL_SLAB_MASK) { - if (*slab2) { - *pos = bmp->index2 << RTE_BITMAP_SLAB_BIT_SIZE_LOG2; - *slab = *slab2; - - bmp->index2 ++; - slab2 ++; - bmp->go2 = bmp->index2 & RTE_BITMAP_CL_SLAB_MASK; - return 1; - } - } - - return 0; -} - -/** - * Bitmap scan (with automatic wrap-around) - * - * @param bmp - * Handle to bitmap instance - * @param pos - * When function call returns 1, pos contains the position of the next set - * bit, otherwise not modified - * @param slab - * When function call returns 1, slab contains the value of the entire 64-bit - * slab where the bit indicated by pos is located. Slabs are always 64-bit - * aligned, so the position of the first bit of the slab (this bit is not - * necessarily set) is pos / 64. Once a slab has been returned by the bitmap - * scan operation, the internal pointers of the bitmap are updated to point - * after this slab, so the same slab will not be returned again if it - * contains more than one bit which is set. When function call returns 0, - * slab is not modified. - * @return - * 0 if there is no bit set in the bitmap, 1 otherwise - */ -static inline int -rte_bitmap_scan(struct rte_bitmap *bmp, uint32_t *pos, uint64_t *slab) -{ - /* Return data from current array2 line if available */ - if (__rte_bitmap_scan_read(bmp, pos, slab)) { - return 1; - } - - /* Look for non-empty array2 line */ - if (__rte_bitmap_scan_search(bmp)) { - __rte_bitmap_scan_read_init(bmp); - __rte_bitmap_scan_read(bmp, pos, slab); - return 1; - } - - /* Empty bitmap */ - return 0; -} - -#ifdef __cplusplus -} -#endif - -#endif /* __INCLUDE_RTE_BITMAP_H__ */ diff --git a/dpdk/lib/librte_sched/rte_reciprocal.h b/dpdk/lib/librte_sched/rte_reciprocal.h index abd15251..5e21f096 100644 --- a/dpdk/lib/librte_sched/rte_reciprocal.h +++ b/dpdk/lib/librte_sched/rte_reciprocal.h @@ -22,6 +22,8 @@ #ifndef _RTE_RECIPROCAL_H_ #define _RTE_RECIPROCAL_H_ +#include + struct rte_reciprocal { uint32_t m; uint8_t sh1, sh2; diff --git a/dpdk/lib/librte_sched/rte_red.h b/dpdk/lib/librte_sched/rte_red.h index ca122275..6edf914f 100644 --- a/dpdk/lib/librte_sched/rte_red.h +++ b/dpdk/lib/librte_sched/rte_red.h @@ -139,7 +139,7 @@ rte_red_config_init(struct rte_red_config *red_cfg, /** * @brief Generate random number for RED * - * Implemenetation based on: + * Implementation based on: * http://software.intel.com/en-us/articles/fast-random-number-generator-on-the-intel-pentiumr-4-processor/ * * 10 bit shift has been found through empirical tests (was 16). @@ -200,7 +200,7 @@ __rte_red_calc_qempty_factor(uint8_t wq_log2, uint16_t m) * Now using basic math we compute 2^n: * 2^(f+n) = 2^f * 2^n * 2^f - we use lookup table - * 2^n - can be replaced with bit shift right oeprations + * 2^n - can be replaced with bit shift right operations */ f = (n >> 6) & 0xf; diff --git a/dpdk/lib/librte_sched/rte_sched.c b/dpdk/lib/librte_sched/rte_sched.c index e6dace2f..7252f850 100644 --- a/dpdk/lib/librte_sched/rte_sched.c +++ b/dpdk/lib/librte_sched/rte_sched.c @@ -42,9 +42,9 @@ #include #include #include +#include #include "rte_sched.h" -#include "rte_bitmap.h" #include "rte_sched_common.h" #include "rte_approx.h" #include "rte_reciprocal.h" @@ -56,8 +56,10 @@ #ifdef RTE_SCHED_VECTOR #include -#if defined(__SSE4__) +#ifdef RTE_ARCH_X86 #define SCHED_VECTOR_SSE4 +#elif defined(RTE_MACHINE_CPUFLAG_NEON) +#define SCHED_VECTOR_NEON #endif #endif @@ -735,12 +737,14 @@ void rte_sched_port_free(struct rte_sched_port *port) { uint32_t qindex; - uint32_t n_queues_per_port = rte_sched_port_queues_per_port(port); + uint32_t n_queues_per_port; /* Check user parameters */ if (port == NULL) return; + n_queues_per_port = rte_sched_port_queues_per_port(port); + /* Free enqueued mbufs */ for (qindex = 0; qindex < n_queues_per_port; qindex++) { struct rte_mbuf **mbufs = rte_sched_port_qbase(port, qindex); @@ -1016,7 +1020,7 @@ rte_sched_subport_read_stats(struct rte_sched_port *port, memcpy(stats, &s->stats, sizeof(struct rte_sched_subport_stats)); memset(&s->stats, 0, sizeof(struct rte_sched_subport_stats)); - /* Subport TC ovesubscription status */ + /* Subport TC oversubscription status */ *tc_ov = s->tc_ov; return 0; @@ -1730,6 +1734,26 @@ grinder_pipe_exists(struct rte_sched_port *port, uint32_t base_pipe) return 1; } +#elif defined(SCHED_VECTOR_NEON) + +static inline int +grinder_pipe_exists(struct rte_sched_port *port, uint32_t base_pipe) +{ + uint32x4_t index, pipes; + uint32_t *pos = (uint32_t *)port->grinder_base_bmp_pos; + + index = vmovq_n_u32(base_pipe); + pipes = vld1q_u32(pos); + if (!vminvq_u32(veorq_u32(pipes, index))) + return 1; + + pipes = vld1q_u32(pos + 4); + if (!vminvq_u32(veorq_u32(pipes, index))) + return 1; + + return 0; +} + #else static inline int diff --git a/dpdk/lib/librte_sched/rte_sched_common.h b/dpdk/lib/librte_sched/rte_sched_common.h index 8920adec..aed144ba 100644 --- a/dpdk/lib/librte_sched/rte_sched_common.h +++ b/dpdk/lib/librte_sched/rte_sched_common.h @@ -38,6 +38,7 @@ extern "C" { #endif +#include #include #define __rte_aligned_16 __attribute__((__aligned__(16))) diff --git a/dpdk/lib/librte_security/Makefile b/dpdk/lib/librte_security/Makefile new file mode 100644 index 00000000..bb93ec33 --- /dev/null +++ b/dpdk/lib/librte_security/Makefile @@ -0,0 +1,54 @@ +# BSD LICENSE +# +# Copyright(c) 2017 Intel Corporation. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +include $(RTE_SDK)/mk/rte.vars.mk + +# library name +LIB = librte_security.a + +# library version +LIBABIVER := 1 + +# build flags +CFLAGS += -O3 +CFLAGS += $(WERROR_FLAGS) +LDLIBS += -lrte_eal -lrte_mempool + +# library source files +SRCS-y += rte_security.c + +# export include files +SYMLINK-y-include += rte_security.h +SYMLINK-y-include += rte_security_driver.h + +# versioning export map +EXPORT_MAP := rte_security_version.map + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/lib/librte_security/rte_security.c b/dpdk/lib/librte_security/rte_security.c new file mode 100644 index 00000000..685729f4 --- /dev/null +++ b/dpdk/lib/librte_security/rte_security.c @@ -0,0 +1,149 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 NXP. + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of NXP nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include + +#include "rte_security.h" +#include "rte_security_driver.h" + +struct rte_security_session * +rte_security_session_create(struct rte_security_ctx *instance, + struct rte_security_session_conf *conf, + struct rte_mempool *mp) +{ + struct rte_security_session *sess = NULL; + + if (conf == NULL) + return NULL; + + RTE_FUNC_PTR_OR_ERR_RET(*instance->ops->session_create, NULL); + + if (rte_mempool_get(mp, (void **)&sess)) + return NULL; + + if (instance->ops->session_create(instance->device, conf, sess, mp)) { + rte_mempool_put(mp, (void *)sess); + return NULL; + } + instance->sess_cnt++; + + return sess; +} + +int +rte_security_session_update(struct rte_security_ctx *instance, + struct rte_security_session *sess, + struct rte_security_session_conf *conf) +{ + RTE_FUNC_PTR_OR_ERR_RET(*instance->ops->session_update, -ENOTSUP); + return instance->ops->session_update(instance->device, sess, conf); +} + +int +rte_security_session_stats_get(struct rte_security_ctx *instance, + struct rte_security_session *sess, + struct rte_security_stats *stats) +{ + RTE_FUNC_PTR_OR_ERR_RET(*instance->ops->session_stats_get, -ENOTSUP); + return instance->ops->session_stats_get(instance->device, sess, stats); +} + +int +rte_security_session_destroy(struct rte_security_ctx *instance, + struct rte_security_session *sess) +{ + int ret; + struct rte_mempool *mp = rte_mempool_from_obj(sess); + + RTE_FUNC_PTR_OR_ERR_RET(*instance->ops->session_destroy, -ENOTSUP); + + if (instance->sess_cnt) + instance->sess_cnt--; + + ret = instance->ops->session_destroy(instance->device, sess); + if (!ret) + rte_mempool_put(mp, (void *)sess); + + return ret; +} + +int +rte_security_set_pkt_metadata(struct rte_security_ctx *instance, + struct rte_security_session *sess, + struct rte_mbuf *m, void *params) +{ + RTE_FUNC_PTR_OR_ERR_RET(*instance->ops->set_pkt_metadata, -ENOTSUP); + return instance->ops->set_pkt_metadata(instance->device, + sess, m, params); +} + +const struct rte_security_capability * +rte_security_capabilities_get(struct rte_security_ctx *instance) +{ + RTE_FUNC_PTR_OR_ERR_RET(*instance->ops->capabilities_get, NULL); + return instance->ops->capabilities_get(instance->device); +} + +const struct rte_security_capability * +rte_security_capability_get(struct rte_security_ctx *instance, + struct rte_security_capability_idx *idx) +{ + const struct rte_security_capability *capabilities; + const struct rte_security_capability *capability; + uint16_t i = 0; + + RTE_FUNC_PTR_OR_ERR_RET(*instance->ops->capabilities_get, NULL); + capabilities = instance->ops->capabilities_get(instance->device); + + if (capabilities == NULL) + return NULL; + + while ((capability = &capabilities[i++])->action + != RTE_SECURITY_ACTION_TYPE_NONE) { + if (capability->action == idx->action && + capability->protocol == idx->protocol) { + if (idx->protocol == RTE_SECURITY_PROTOCOL_IPSEC) { + if (capability->ipsec.proto == + idx->ipsec.proto && + capability->ipsec.mode == + idx->ipsec.mode && + capability->ipsec.direction == + idx->ipsec.direction) + return capability; + } + } + } + + return NULL; +} diff --git a/dpdk/lib/librte_security/rte_security.h b/dpdk/lib/librte_security/rte_security.h new file mode 100644 index 00000000..2b609cbf --- /dev/null +++ b/dpdk/lib/librte_security/rte_security.h @@ -0,0 +1,534 @@ +/*- + * BSD LICENSE + * + * Copyright 2017 NXP. + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of NXP nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_SECURITY_H_ +#define _RTE_SECURITY_H_ + +/** + * @file rte_security.h + * @b EXPERIMENTAL: this API may change without prior notice + * + * RTE Security Common Definitions + * + */ + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +#include +#include +#include + +#include +#include +#include +#include +#include + +/** IPSec protocol mode */ +enum rte_security_ipsec_sa_mode { + RTE_SECURITY_IPSEC_SA_MODE_TRANSPORT = 1, + /**< IPSec Transport mode */ + RTE_SECURITY_IPSEC_SA_MODE_TUNNEL, + /**< IPSec Tunnel mode */ +}; + +/** IPSec Protocol */ +enum rte_security_ipsec_sa_protocol { + RTE_SECURITY_IPSEC_SA_PROTO_AH = 1, + /**< AH protocol */ + RTE_SECURITY_IPSEC_SA_PROTO_ESP, + /**< ESP protocol */ +}; + +/** IPSEC tunnel type */ +enum rte_security_ipsec_tunnel_type { + RTE_SECURITY_IPSEC_TUNNEL_IPV4 = 1, + /**< Outer header is IPv4 */ + RTE_SECURITY_IPSEC_TUNNEL_IPV6, + /**< Outer header is IPv6 */ +}; + +/** + * Security context for crypto/eth devices + * + * Security instance for each driver to register security operations. + * The application can get the security context from the crypto/eth device id + * using the APIs rte_cryptodev_get_sec_ctx()/rte_eth_dev_get_sec_ctx() + * This structure is used to identify the device(crypto/eth) for which the + * security operations need to be performed. + */ +struct rte_security_ctx { + void *device; + /**< Crypto/ethernet device attached */ + const struct rte_security_ops *ops; + /**< Pointer to security ops for the device */ + uint16_t sess_cnt; + /**< Number of sessions attached to this context */ +}; + +/** + * IPSEC tunnel parameters + * + * These parameters are used to build outbound tunnel headers. + */ +struct rte_security_ipsec_tunnel_param { + enum rte_security_ipsec_tunnel_type type; + /**< Tunnel type: IPv4 or IPv6 */ + RTE_STD_C11 + union { + struct { + struct in_addr src_ip; + /**< IPv4 source address */ + struct in_addr dst_ip; + /**< IPv4 destination address */ + uint8_t dscp; + /**< IPv4 Differentiated Services Code Point */ + uint8_t df; + /**< IPv4 Don't Fragment bit */ + uint8_t ttl; + /**< IPv4 Time To Live */ + } ipv4; + /**< IPv4 header parameters */ + struct { + struct in6_addr src_addr; + /**< IPv6 source address */ + struct in6_addr dst_addr; + /**< IPv6 destination address */ + uint8_t dscp; + /**< IPv6 Differentiated Services Code Point */ + uint32_t flabel; + /**< IPv6 flow label */ + uint8_t hlimit; + /**< IPv6 hop limit */ + } ipv6; + /**< IPv6 header parameters */ + }; +}; + +/** + * IPsec Security Association option flags + */ +struct rte_security_ipsec_sa_options { + /**< Extended Sequence Numbers (ESN) + * + * * 1: Use extended (64 bit) sequence numbers + * * 0: Use normal sequence numbers + */ + uint32_t esn : 1; + + /**< UDP encapsulation + * + * * 1: Do UDP encapsulation/decapsulation so that IPSEC packets can + * traverse through NAT boxes. + * * 0: No UDP encapsulation + */ + uint32_t udp_encap : 1; + + /**< Copy DSCP bits + * + * * 1: Copy IPv4 or IPv6 DSCP bits from inner IP header to + * the outer IP header in encapsulation, and vice versa in + * decapsulation. + * * 0: Do not change DSCP field. + */ + uint32_t copy_dscp : 1; + + /**< Copy IPv6 Flow Label + * + * * 1: Copy IPv6 flow label from inner IPv6 header to the + * outer IPv6 header. + * * 0: Outer header is not modified. + */ + uint32_t copy_flabel : 1; + + /**< Copy IPv4 Don't Fragment bit + * + * * 1: Copy the DF bit from the inner IPv4 header to the outer + * IPv4 header. + * * 0: Outer header is not modified. + */ + uint32_t copy_df : 1; + + /**< Decrement inner packet Time To Live (TTL) field + * + * * 1: In tunnel mode, decrement inner packet IPv4 TTL or + * IPv6 Hop Limit after tunnel decapsulation, or before tunnel + * encapsulation. + * * 0: Inner packet is not modified. + */ + uint32_t dec_ttl : 1; +}; + +/** IPSec security association direction */ +enum rte_security_ipsec_sa_direction { + RTE_SECURITY_IPSEC_SA_DIR_EGRESS, + /**< Encrypt and generate digest */ + RTE_SECURITY_IPSEC_SA_DIR_INGRESS, + /**< Verify digest and decrypt */ +}; + +/** + * IPsec security association configuration data. + * + * This structure contains data required to create an IPsec SA security session. + */ +struct rte_security_ipsec_xform { + uint32_t spi; + /**< SA security parameter index */ + uint32_t salt; + /**< SA salt */ + struct rte_security_ipsec_sa_options options; + /**< various SA options */ + enum rte_security_ipsec_sa_direction direction; + /**< IPSec SA Direction - Egress/Ingress */ + enum rte_security_ipsec_sa_protocol proto; + /**< IPsec SA Protocol - AH/ESP */ + enum rte_security_ipsec_sa_mode mode; + /**< IPsec SA Mode - transport/tunnel */ + struct rte_security_ipsec_tunnel_param tunnel; + /**< Tunnel parameters, NULL for transport mode */ +}; + +/** + * MACsec security session configuration + */ +struct rte_security_macsec_xform { + /** To be Filled */ + int dummy; +}; + +/** + * Security session action type. + */ +enum rte_security_session_action_type { + RTE_SECURITY_ACTION_TYPE_NONE, + /**< No security actions */ + RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO, + /**< Crypto processing for security protocol is processed inline + * during transmission + */ + RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL, + /**< All security protocol processing is performed inline during + * transmission + */ + RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL + /**< All security protocol processing including crypto is performed + * on a lookaside accelerator + */ +}; + +/** Security session protocol definition */ +enum rte_security_session_protocol { + RTE_SECURITY_PROTOCOL_IPSEC = 1, + /**< IPsec Protocol */ + RTE_SECURITY_PROTOCOL_MACSEC, + /**< MACSec Protocol */ +}; + +/** + * Security session configuration + */ +struct rte_security_session_conf { + enum rte_security_session_action_type action_type; + /**< Type of action to be performed on the session */ + enum rte_security_session_protocol protocol; + /**< Security protocol to be configured */ + RTE_STD_C11 + union { + struct rte_security_ipsec_xform ipsec; + struct rte_security_macsec_xform macsec; + }; + /**< Configuration parameters for security session */ + struct rte_crypto_sym_xform *crypto_xform; + /**< Security Session Crypto Transformations */ +}; + +struct rte_security_session { + void *sess_private_data; + /**< Private session material */ +}; + +/** + * Create security session as specified by the session configuration + * + * @param instance security instance + * @param conf session configuration parameters + * @param mp mempool to allocate session objects from + * @return + * - On success, pointer to session + * - On failure, NULL + */ +struct rte_security_session * +rte_security_session_create(struct rte_security_ctx *instance, + struct rte_security_session_conf *conf, + struct rte_mempool *mp); + +/** + * Update security session as specified by the session configuration + * + * @param instance security instance + * @param sess session to update parameters + * @param conf update configuration parameters + * @return + * - On success returns 0 + * - On failure return errno + */ +int +rte_security_session_update(struct rte_security_ctx *instance, + struct rte_security_session *sess, + struct rte_security_session_conf *conf); + +/** + * Free security session header and the session private data and + * return it to its original mempool. + * + * @param instance security instance + * @param sess security session to freed + * + * @return + * - 0 if successful. + * - -EINVAL if session is NULL. + * - -EBUSY if not all device private data has been freed. + */ +int +rte_security_session_destroy(struct rte_security_ctx *instance, + struct rte_security_session *sess); + +/** + * Updates the buffer with device-specific defined metadata + * + * @param instance security instance + * @param sess security session + * @param mb packet mbuf to set metadata on. + * @param params device-specific defined parameters + * required for metadata + * + * @return + * - On success, zero. + * - On failure, a negative value. + */ +int +rte_security_set_pkt_metadata(struct rte_security_ctx *instance, + struct rte_security_session *sess, + struct rte_mbuf *mb, void *params); + +/** + * Attach a session to a symmetric crypto operation + * + * @param sym_op crypto operation + * @param sess security session + */ +static inline int +__rte_security_attach_session(struct rte_crypto_sym_op *sym_op, + struct rte_security_session *sess) +{ + sym_op->sec_session = sess; + + return 0; +} + +static inline void * +get_sec_session_private_data(const struct rte_security_session *sess) +{ + return sess->sess_private_data; +} + +static inline void +set_sec_session_private_data(struct rte_security_session *sess, + void *private_data) +{ + sess->sess_private_data = private_data; +} + +/** + * Attach a session to a crypto operation. + * This API is needed only in case of RTE_SECURITY_SESS_CRYPTO_PROTO_OFFLOAD + * For other rte_security_session_action_type, ol_flags in rte_mbuf may be + * defined to perform security operations. + * + * @param op crypto operation + * @param sess security session + */ +static inline int +rte_security_attach_session(struct rte_crypto_op *op, + struct rte_security_session *sess) +{ + if (unlikely(op->type != RTE_CRYPTO_OP_TYPE_SYMMETRIC)) + return -EINVAL; + + op->sess_type = RTE_CRYPTO_OP_SECURITY_SESSION; + + return __rte_security_attach_session(op->sym, sess); +} + +struct rte_security_macsec_stats { + uint64_t reserved; +}; + +struct rte_security_ipsec_stats { + uint64_t reserved; + +}; + +struct rte_security_stats { + enum rte_security_session_protocol protocol; + /**< Security protocol to be configured */ + + RTE_STD_C11 + union { + struct rte_security_macsec_stats macsec; + struct rte_security_ipsec_stats ipsec; + }; +}; + +/** + * Get security session statistics + * + * @param instance security instance + * @param sess security session + * @param stats statistics + * @return + * - On success return 0 + * - On failure errno + */ +int +rte_security_session_stats_get(struct rte_security_ctx *instance, + struct rte_security_session *sess, + struct rte_security_stats *stats); + +/** + * Security capability definition + */ +struct rte_security_capability { + enum rte_security_session_action_type action; + /**< Security action type*/ + enum rte_security_session_protocol protocol; + /**< Security protocol */ + RTE_STD_C11 + union { + struct { + enum rte_security_ipsec_sa_protocol proto; + /**< IPsec SA protocol */ + enum rte_security_ipsec_sa_mode mode; + /**< IPsec SA mode */ + enum rte_security_ipsec_sa_direction direction; + /**< IPsec SA direction */ + struct rte_security_ipsec_sa_options options; + /**< IPsec SA supported options */ + } ipsec; + /**< IPsec capability */ + struct { + /* To be Filled */ + int dummy; + } macsec; + /**< MACsec capability */ + }; + + const struct rte_cryptodev_capabilities *crypto_capabilities; + /**< Corresponding crypto capabilities for security capability */ + + uint32_t ol_flags; + /**< Device offload flags */ +}; + +#define RTE_SECURITY_TX_OLOAD_NEED_MDATA 0x00000001 +/**< HW needs metadata update, see rte_security_set_pkt_metadata(). + */ + +#define RTE_SECURITY_TX_HW_TRAILER_OFFLOAD 0x00000002 +/**< HW constructs trailer of packets + * Transmitted packets will have the trailer added to them + * by hardawre. The next protocol field will be based on + * the mbuf->inner_esp_next_proto field. + */ +#define RTE_SECURITY_RX_HW_TRAILER_OFFLOAD 0x00010000 +/**< HW removes trailer of packets + * Received packets have no trailer, the next protocol field + * is supplied in the mbuf->inner_esp_next_proto field. + * Inner packet is not modified. + */ + +/** + * Security capability index used to query a security instance for a specific + * security capability + */ +struct rte_security_capability_idx { + enum rte_security_session_action_type action; + enum rte_security_session_protocol protocol; + + RTE_STD_C11 + union { + struct { + enum rte_security_ipsec_sa_protocol proto; + enum rte_security_ipsec_sa_mode mode; + enum rte_security_ipsec_sa_direction direction; + } ipsec; + }; +}; + +/** + * Returns array of security instance capabilities + * + * @param instance Security instance. + * + * @return + * - Returns array of security capabilities. + * - Return NULL if no capabilities available. + */ +const struct rte_security_capability * +rte_security_capabilities_get(struct rte_security_ctx *instance); + +/** + * Query if a specific capability is available on security instance + * + * @param instance security instance. + * @param idx security capability index to match against + * + * @return + * - Returns pointer to security capability on match of capability + * index criteria. + * - Return NULL if the capability not matched on security instance. + */ +const struct rte_security_capability * +rte_security_capability_get(struct rte_security_ctx *instance, + struct rte_security_capability_idx *idx); + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_SECURITY_H_ */ diff --git a/dpdk/lib/librte_security/rte_security_driver.h b/dpdk/lib/librte_security/rte_security_driver.h new file mode 100644 index 00000000..997fbe79 --- /dev/null +++ b/dpdk/lib/librte_security/rte_security_driver.h @@ -0,0 +1,156 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * Copyright 2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_SECURITY_DRIVER_H_ +#define _RTE_SECURITY_DRIVER_H_ + +/** + * @file rte_security_driver.h + * @b EXPERIMENTAL: this API may change without prior notice + * + * RTE Security Common Definitions + * + */ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "rte_security.h" + +/** + * Configure a security session on a device. + * + * @param device Crypto/eth device pointer + * @param conf Security session configuration + * @param sess Pointer to Security private session structure + * @param mp Mempool where the private session is allocated + * + * @return + * - Returns 0 if private session structure have been created successfully. + * - Returns -EINVAL if input parameters are invalid. + * - Returns -ENOTSUP if crypto device does not support the crypto transform. + * - Returns -ENOMEM if the private session could not be allocated. + */ +typedef int (*security_session_create_t)(void *device, + struct rte_security_session_conf *conf, + struct rte_security_session *sess, + struct rte_mempool *mp); + +/** + * Free driver private session data. + * + * @param dev Crypto/eth device pointer + * @param sess Security session structure + */ +typedef int (*security_session_destroy_t)(void *device, + struct rte_security_session *sess); + +/** + * Update driver private session data. + * + * @param device Crypto/eth device pointer + * @param sess Pointer to Security private session structure + * @param conf Security session configuration + * + * @return + * - Returns 0 if private session structure have been updated successfully. + * - Returns -EINVAL if input parameters are invalid. + * - Returns -ENOTSUP if crypto device does not support the crypto transform. + */ +typedef int (*security_session_update_t)(void *device, + struct rte_security_session *sess, + struct rte_security_session_conf *conf); +/** + * Get stats from the PMD. + * + * @param device Crypto/eth device pointer + * @param sess Pointer to Security private session structure + * @param stats Security stats of the driver + * + * @return + * - Returns 0 if private session structure have been updated successfully. + * - Returns -EINVAL if session parameters are invalid. + */ +typedef int (*security_session_stats_get_t)(void *device, + struct rte_security_session *sess, + struct rte_security_stats *stats); + +/** + * Update the mbuf with provided metadata. + * + * @param sess Security session structure + * @param mb Packet buffer + * @param mt Metadata + * + * @return + * - Returns 0 if metadata updated successfully. + * - Returns -ve value for errors. + */ +typedef int (*security_set_pkt_metadata_t)(void *device, + struct rte_security_session *sess, struct rte_mbuf *m, + void *params); + +/** + * Get security capabilities of the device. + * + * @param device crypto/eth device pointer + * + * @return + * - Returns rte_security_capability pointer on success. + * - Returns NULL on error. + */ +typedef const struct rte_security_capability *(*security_capabilities_get_t)( + void *device); + +/** Security operations function pointer table */ +struct rte_security_ops { + security_session_create_t session_create; + /**< Configure a security session. */ + security_session_update_t session_update; + /**< Update a security session. */ + security_session_stats_get_t session_stats_get; + /**< Get security session statistics. */ + security_session_destroy_t session_destroy; + /**< Clear a security sessions private data. */ + security_set_pkt_metadata_t set_pkt_metadata; + /**< Update mbuf metadata. */ + security_capabilities_get_t capabilities_get; + /**< Get security capabilities. */ +}; + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_SECURITY_DRIVER_H_ */ diff --git a/dpdk/lib/librte_security/rte_security_version.map b/dpdk/lib/librte_security/rte_security_version.map new file mode 100644 index 00000000..e12c04b2 --- /dev/null +++ b/dpdk/lib/librte_security/rte_security_version.map @@ -0,0 +1,14 @@ +EXPERIMENTAL { + global: + + rte_security_attach_session; + rte_security_capabilities_get; + rte_security_capability_get; + rte_security_session_create; + rte_security_session_destroy; + rte_security_session_stats_get; + rte_security_session_update; + rte_security_set_pkt_metadata; + + local: *; +}; diff --git a/dpdk/lib/librte_table/Makefile b/dpdk/lib/librte_table/Makefile index 7a8a3f3c..2e32fbf1 100644 --- a/dpdk/lib/librte_table/Makefile +++ b/dpdk/lib/librte_table/Makefile @@ -1,6 +1,6 @@ # BSD LICENSE # -# Copyright(c) 2010-2014 Intel Corporation. All rights reserved. +# Copyright(c) 2010-2016 Intel Corporation. All rights reserved. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -38,10 +38,15 @@ LIB = librte_table.a CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) +LDLIBS += -lrte_eal -lrte_mempool -lrte_mbuf -lrte_port +LDLIBS += -lrte_lpm -lrte_hash +ifeq ($(CONFIG_RTE_LIBRTE_ACL),y) +LDLIBS += -lrte_acl +endif EXPORT_MAP := rte_table_version.map -LIBABIVER := 2 +LIBABIVER := 3 # # all source are stored in SRCS-y @@ -51,6 +56,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_TABLE) += rte_table_lpm_ipv6.c ifeq ($(CONFIG_RTE_LIBRTE_ACL),y) SRCS-$(CONFIG_RTE_LIBRTE_TABLE) += rte_table_acl.c endif +SRCS-$(CONFIG_RTE_LIBRTE_TABLE) += rte_table_hash_cuckoo.c SRCS-$(CONFIG_RTE_LIBRTE_TABLE) += rte_table_hash_key8.c SRCS-$(CONFIG_RTE_LIBRTE_TABLE) += rte_table_hash_key16.c SRCS-$(CONFIG_RTE_LIBRTE_TABLE) += rte_table_hash_key32.c @@ -68,17 +74,13 @@ SYMLINK-$(CONFIG_RTE_LIBRTE_TABLE)-include += rte_table_acl.h endif SYMLINK-$(CONFIG_RTE_LIBRTE_TABLE)-include += rte_table_hash.h SYMLINK-$(CONFIG_RTE_LIBRTE_TABLE)-include += rte_lru.h +ifeq ($(CONFIG_RTE_ARCH_X86),y) +SYMLINK-$(CONFIG_RTE_LIBRTE_TABLE)-include += rte_lru_x86.h +endif +ifeq ($(CONFIG_RTE_ARCH_ARM64),y) +SYMLINK-$(CONFIG_RTE_LIBRTE_TABLE)-include += rte_lru_arm64.h +endif SYMLINK-$(CONFIG_RTE_LIBRTE_TABLE)-include += rte_table_array.h SYMLINK-$(CONFIG_RTE_LIBRTE_TABLE)-include += rte_table_stub.h -# this lib depends upon: -DEPDIRS-$(CONFIG_RTE_LIBRTE_TABLE) := lib/librte_eal -DEPDIRS-$(CONFIG_RTE_LIBRTE_TABLE) += lib/librte_mbuf -DEPDIRS-$(CONFIG_RTE_LIBRTE_TABLE) += lib/librte_mempool -DEPDIRS-$(CONFIG_RTE_LIBRTE_TABLE) += lib/librte_port -DEPDIRS-$(CONFIG_RTE_LIBRTE_TABLE) += lib/librte_lpm -ifeq ($(CONFIG_RTE_LIBRTE_ACL),y) -DEPDIRS-$(CONFIG_RTE_LIBRTE_TABLE) += lib/librte_acl -endif - include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/lib/librte_table/rte_lru.h b/dpdk/lib/librte_table/rte_lru.h index e87e062d..a73b2ffe 100644 --- a/dpdk/lib/librte_table/rte_lru.h +++ b/dpdk/lib/librte_table/rte_lru.h @@ -38,31 +38,16 @@ extern "C" { #endif -#include - -#ifdef __INTEL_COMPILER -#define GCC_VERSION (0) +#include +#ifdef RTE_ARCH_X86_64 +#include "rte_lru_x86.h" +#elif defined(RTE_ARCH_ARM64) +#include "rte_lru_arm64.h" #else -#define GCC_VERSION (__GNUC__ * 10000+__GNUC_MINOR__*100 + __GNUC_PATCHLEVEL__) -#endif - -#ifndef RTE_TABLE_HASH_LRU_STRATEGY -#ifdef __SSE4_2__ -#define RTE_TABLE_HASH_LRU_STRATEGY 2 -#else /* if no SSE, use simple scalar version */ -#define RTE_TABLE_HASH_LRU_STRATEGY 1 -#endif -#endif - -#ifndef RTE_ARCH_X86_64 #undef RTE_TABLE_HASH_LRU_STRATEGY #define RTE_TABLE_HASH_LRU_STRATEGY 1 #endif -#if (RTE_TABLE_HASH_LRU_STRATEGY < 0) || (RTE_TABLE_HASH_LRU_STRATEGY > 3) -#error Invalid value for RTE_TABLE_HASH_LRU_STRATEGY -#endif - #if RTE_TABLE_HASH_LRU_STRATEGY == 0 #define lru_init(bucket) \ @@ -118,87 +103,11 @@ do { \ bucket->lru_list = x; \ } while (0) -#elif RTE_TABLE_HASH_LRU_STRATEGY == 2 +#elif (RTE_TABLE_HASH_LRU_STRATEGY == 2) || (RTE_TABLE_HASH_LRU_STRATEGY == 3) -#if GCC_VERSION > 40306 -#include -#else -#include -#include -#include -#endif - -#define lru_init(bucket) \ -do \ - bucket->lru_list = 0x0000000100020003LLU; \ -while (0) - -#define lru_pos(bucket) (bucket->lru_list & 0xFFFFLLU) - -#define lru_update(bucket, mru_val) \ -do { \ - /* set up the masks for all possible shuffles, depends on pos */\ - static uint64_t masks[10] = { \ - /* Shuffle order; Make Zero (see _mm_shuffle_epi8 manual) */\ - 0x0100070605040302, 0x8080808080808080, \ - 0x0302070605040100, 0x8080808080808080, \ - 0x0504070603020100, 0x8080808080808080, \ - 0x0706050403020100, 0x8080808080808080, \ - 0x0706050403020100, 0x8080808080808080}; \ - /* load up one register with repeats of mru-val */ \ - uint64_t mru2 = mru_val; \ - uint64_t mru3 = mru2 | (mru2 << 16); \ - uint64_t lru = bucket->lru_list; \ - /* XOR to cause the word we're looking for to go to zero */ \ - uint64_t mru = lru ^ ((mru3 << 32) | mru3); \ - __m128i c = _mm_cvtsi64_si128(mru); \ - __m128i b = _mm_cvtsi64_si128(lru); \ - /* Find the minimum value (first zero word, if it's in there) */\ - __m128i d = _mm_minpos_epu16(c); \ - /* Second word is the index to found word (first word is the value) */\ - unsigned pos = _mm_extract_epi16(d, 1); \ - /* move the recently used location to top of list */ \ - __m128i k = _mm_shuffle_epi8(b, *((__m128i *) &masks[2 * pos]));\ - /* Finally, update the original list with the reordered data */ \ - bucket->lru_list = _mm_extract_epi64(k, 0); \ - /* Phwew! */ \ -} while (0) - -#elif RTE_TABLE_HASH_LRU_STRATEGY == 3 - -#if GCC_VERSION > 40306 -#include -#else -#include -#include -#include -#endif - -#define lru_init(bucket) \ -do \ - bucket->lru_list = ~0LLU; \ -while (0) - - -static inline int -f_lru_pos(uint64_t lru_list) -{ - __m128i lst = _mm_set_epi64x((uint64_t)-1, lru_list); - __m128i min = _mm_minpos_epu16(lst); - return _mm_extract_epi16(min, 1); -} -#define lru_pos(bucket) f_lru_pos(bucket->lru_list) - -#define lru_update(bucket, mru_val) \ -do { \ - const uint64_t orvals[] = {0xFFFFLLU, 0xFFFFLLU << 16, \ - 0xFFFFLLU << 32, 0xFFFFLLU << 48, 0LLU}; \ - const uint64_t decs[] = {0x1000100010001LLU, 0}; \ - __m128i lru = _mm_cvtsi64_si128(bucket->lru_list); \ - __m128i vdec = _mm_cvtsi64_si128(decs[mru_val>>2]); \ - lru = _mm_subs_epu16(lru, vdec); \ - bucket->lru_list = _mm_extract_epi64(lru, 0) | orvals[mru_val]; \ -} while (0) +/** + * These strategies are implemented in architecture specific header files. + */ #else diff --git a/dpdk/lib/librte_table/rte_lru_arm64.h b/dpdk/lib/librte_table/rte_lru_arm64.h new file mode 100644 index 00000000..61735238 --- /dev/null +++ b/dpdk/lib/librte_table/rte_lru_arm64.h @@ -0,0 +1,88 @@ +/* + * BSD LICENSE + * + * Copyright (C) Cavium, Inc. 2017. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __RTE_LRU_ARM64_H__ +#define __RTE_LRU_ARM64_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include + +#ifndef RTE_TABLE_HASH_LRU_STRATEGY +#ifdef RTE_MACHINE_CPUFLAG_NEON +#define RTE_TABLE_HASH_LRU_STRATEGY 3 +#else /* if no NEON, use simple scalar version */ +#define RTE_TABLE_HASH_LRU_STRATEGY 1 +#endif +#endif + +#if RTE_TABLE_HASH_LRU_STRATEGY == 3 + +#define lru_init(bucket) \ + { bucket->lru_list = ~0LLU; } + +static inline int +f_lru_pos(uint64_t lru_list) +{ + /* Compare the vector to zero vector */ + uint16x4_t lru_vec = vld1_u16((uint16_t *)&lru_list); + uint16x4_t min_vec = vmov_n_u16(vminv_u16(lru_vec)); + uint64_t mask = vget_lane_u64(vreinterpret_u64_u16( + vceq_u16(min_vec, lru_vec)), 0); + return __builtin_clzl(mask) >> 4; +} +#define lru_pos(bucket) f_lru_pos(bucket->lru_list) + +#define lru_update(bucket, mru_val) \ +do { \ + const uint64_t orvals[] = {0xFFFFLLU, 0xFFFFLLU << 16, \ + 0xFFFFLLU << 32, 0xFFFFLLU << 48, 0LLU}; \ + const uint64_t decs[] = {0x1000100010001LLU, 0}; \ + uint64x1_t lru = vdup_n_u64(bucket->lru_list); \ + uint64x1_t vdec = vdup_n_u64(decs[mru_val>>2]); \ + bucket->lru_list = vget_lane_u64(vreinterpret_u64_u16( \ + vsub_u16(vreinterpret_u16_u64(lru), \ + vreinterpret_u16_u64(vdec))), \ + 0); \ + bucket->lru_list |= orvals[mru_val]; \ +} while (0) + +#endif + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/dpdk/lib/librte_table/rte_lru_x86.h b/dpdk/lib/librte_table/rte_lru_x86.h new file mode 100644 index 00000000..b4f60525 --- /dev/null +++ b/dpdk/lib/librte_table/rte_lru_x86.h @@ -0,0 +1,132 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __INCLUDE_RTE_LRU_X86_H__ +#define __INCLUDE_RTE_LRU_X86_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +#include + +#ifndef RTE_TABLE_HASH_LRU_STRATEGY +#define RTE_TABLE_HASH_LRU_STRATEGY 2 +#endif + +#if RTE_TABLE_HASH_LRU_STRATEGY == 2 + +#if defined(RTE_TOOLCHAIN_GCC) && (GCC_VERSION > 40306) +#include +#else +#include +#include +#include +#endif + +#define lru_init(bucket) \ + { bucket->lru_list = 0x0000000100020003LLU; } + +#define lru_pos(bucket) (bucket->lru_list & 0xFFFFLLU) + +#define lru_update(bucket, mru_val) \ +do { \ + /* set up the masks for all possible shuffles, depends on pos */\ + static uint64_t masks[10] = { \ + /* Shuffle order; Make Zero (see _mm_shuffle_epi8 manual) */\ + 0x0100070605040302, 0x8080808080808080, \ + 0x0302070605040100, 0x8080808080808080, \ + 0x0504070603020100, 0x8080808080808080, \ + 0x0706050403020100, 0x8080808080808080, \ + 0x0706050403020100, 0x8080808080808080}; \ + /* load up one register with repeats of mru-val */ \ + uint64_t mru2 = mru_val; \ + uint64_t mru3 = mru2 | (mru2 << 16); \ + uint64_t lru = bucket->lru_list; \ + /* XOR to cause the word we're looking for to go to zero */ \ + uint64_t mru = lru ^ ((mru3 << 32) | mru3); \ + __m128i c = _mm_cvtsi64_si128(mru); \ + __m128i b = _mm_cvtsi64_si128(lru); \ + /* Find the minimum value (first zero word, if it's in there) */\ + __m128i d = _mm_minpos_epu16(c); \ + /* Second word is the index to found word (first word is the value) */\ + unsigned int pos = _mm_extract_epi16(d, 1); \ + /* move the recently used location to top of list */ \ + __m128i k = _mm_shuffle_epi8(b, *((__m128i *) &masks[2 * pos]));\ + /* Finally, update the original list with the reordered data */ \ + bucket->lru_list = _mm_extract_epi64(k, 0); \ + /* Phwew! */ \ +} while (0) + +#elif RTE_TABLE_HASH_LRU_STRATEGY == 3 + +#if defined(RTE_TOOLCHAIN_GCC) && (GCC_VERSION > 40306) +#include +#else +#include +#include +#include +#endif + +#define lru_init(bucket) \ + { bucket->lru_list = ~0LLU; } + +static inline int +f_lru_pos(uint64_t lru_list) +{ + __m128i lst = _mm_set_epi64x((uint64_t)-1, lru_list); + __m128i min = _mm_minpos_epu16(lst); + return _mm_extract_epi16(min, 1); +} +#define lru_pos(bucket) f_lru_pos(bucket->lru_list) + +#define lru_update(bucket, mru_val) \ +do { \ + const uint64_t orvals[] = {0xFFFFLLU, 0xFFFFLLU << 16, \ + 0xFFFFLLU << 32, 0xFFFFLLU << 48, 0LLU}; \ + const uint64_t decs[] = {0x1000100010001LLU, 0}; \ + __m128i lru = _mm_cvtsi64_si128(bucket->lru_list); \ + __m128i vdec = _mm_cvtsi64_si128(decs[mru_val>>2]); \ + lru = _mm_subs_epu16(lru, vdec); \ + bucket->lru_list = _mm_extract_epi64(lru, 0) | orvals[mru_val]; \ +} while (0) + +#endif + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/dpdk/lib/librte_table/rte_table_acl.c b/dpdk/lib/librte_table/rte_table_acl.c index 8f1f8ceb..3c05e4a8 100644 --- a/dpdk/lib/librte_table/rte_table_acl.c +++ b/dpdk/lib/librte_table/rte_table_acl.c @@ -87,7 +87,7 @@ rte_table_acl_create( int socket_id, uint32_t entry_size) { - struct rte_table_acl_params *p = (struct rte_table_acl_params *) params; + struct rte_table_acl_params *p = params; struct rte_table_acl *acl; uint32_t action_table_size, acl_rule_list_size, acl_rule_memory_size; uint32_t total_size; @@ -168,7 +168,7 @@ rte_table_acl_create( static int rte_table_acl_free(void *table) { - struct rte_table_acl *acl = (struct rte_table_acl *) table; + struct rte_table_acl *acl = table; /* Check input parameters */ if (table == NULL) { @@ -248,9 +248,9 @@ rte_table_acl_entry_add( int *key_found, void **entry_ptr) { - struct rte_table_acl *acl = (struct rte_table_acl *) table; + struct rte_table_acl *acl = table; struct rte_table_acl_rule_add_params *rule = - (struct rte_table_acl_rule_add_params *) key; + key; struct rte_pipeline_acl_rule acl_rule; struct rte_acl_rule *rule_location; struct rte_acl_ctx *ctx; @@ -366,9 +366,9 @@ rte_table_acl_entry_delete( int *key_found, void *entry) { - struct rte_table_acl *acl = (struct rte_table_acl *) table; + struct rte_table_acl *acl = table; struct rte_table_acl_rule_delete_params *rule = - (struct rte_table_acl_rule_delete_params *) key; + key; struct rte_acl_rule *deleted_rule = NULL; struct rte_acl_ctx *ctx; uint32_t pos, pos_valid, i; @@ -450,7 +450,7 @@ rte_table_acl_entry_add_bulk( int *key_found, void **entries_ptr) { - struct rte_table_acl *acl = (struct rte_table_acl *) table; + struct rte_table_acl *acl = table; struct rte_acl_ctx *ctx; uint32_t rule_pos[n_keys]; uint32_t i; @@ -507,7 +507,7 @@ rte_table_acl_entry_add_bulk( return -EINVAL; } - rule = (struct rte_table_acl_rule_add_params *) keys[i]; + rule = keys[i]; if (rule->priority > RTE_ACL_MAX_PRIORITY) { RTE_LOG(ERR, TABLE, "%s: Priority is too high\n", __func__); return -EINVAL; @@ -518,7 +518,7 @@ rte_table_acl_entry_add_bulk( memset(key_found, 0, n_keys * sizeof(int)); for (i = 0; i < n_keys; i++) { struct rte_table_acl_rule_add_params *rule = - (struct rte_table_acl_rule_add_params *) keys[i]; + keys[i]; struct rte_pipeline_acl_rule acl_rule; struct rte_acl_rule *rule_location; uint32_t free_pos, free_pos_valid, j; @@ -636,7 +636,7 @@ rte_table_acl_entry_delete_bulk( int *key_found, void **entries) { - struct rte_table_acl *acl = (struct rte_table_acl *) table; + struct rte_table_acl *acl = table; struct rte_acl_rule *deleted_rules[n_keys]; uint32_t rule_pos[n_keys]; struct rte_acl_ctx *ctx; @@ -675,7 +675,7 @@ rte_table_acl_entry_delete_bulk( memset(rule_pos, 0, n_keys * sizeof(uint32_t)); for (i = 0; i < n_keys; i++) { struct rte_table_acl_rule_delete_params *rule = - (struct rte_table_acl_rule_delete_params *) keys[i]; + keys[i]; uint32_t pos_valid, j; /* Look for the rule in the table */ @@ -792,7 +792,7 @@ rte_table_acl_lookup( pkts_mask &= ~pkt_mask; - if (action_table_pos != RTE_ACL_INVALID_USERDATA) { + if (action_table_pos != 0) { pkts_out_mask |= pkt_mask; entries[pkt_pos] = (void *) &acl->memory[action_table_pos * @@ -810,7 +810,7 @@ rte_table_acl_lookup( static int rte_table_acl_stats_read(void *table, struct rte_table_stats *stats, int clear) { - struct rte_table_acl *acl = (struct rte_table_acl *) table; + struct rte_table_acl *acl = table; if (stats != NULL) memcpy(stats, &acl->stats, sizeof(acl->stats)); diff --git a/dpdk/lib/librte_table/rte_table_array.c b/dpdk/lib/librte_table/rte_table_array.c index 3bb68d11..cf7be88a 100644 --- a/dpdk/lib/librte_table/rte_table_array.c +++ b/dpdk/lib/librte_table/rte_table_array.c @@ -74,8 +74,7 @@ struct rte_table_array { static void * rte_table_array_create(void *params, int socket_id, uint32_t entry_size) { - struct rte_table_array_params *p = - (struct rte_table_array_params *) params; + struct rte_table_array_params *p = params; struct rte_table_array *t; uint32_t total_cl_size, total_size; @@ -111,7 +110,7 @@ rte_table_array_create(void *params, int socket_id, uint32_t entry_size) static int rte_table_array_free(void *table) { - struct rte_table_array *t = (struct rte_table_array *) table; + struct rte_table_array *t = table; /* Check input parameters */ if (t == NULL) { @@ -133,8 +132,8 @@ rte_table_array_entry_add( int *key_found, void **entry_ptr) { - struct rte_table_array *t = (struct rte_table_array *) table; - struct rte_table_array_key *k = (struct rte_table_array_key *) key; + struct rte_table_array *t = table; + struct rte_table_array_key *k = key; uint8_t *table_entry; /* Check input parameters */ @@ -214,7 +213,7 @@ rte_table_array_lookup( static int rte_table_array_stats_read(void *table, struct rte_table_stats *stats, int clear) { - struct rte_table_array *array = (struct rte_table_array *) table; + struct rte_table_array *array = table; if (stats != NULL) memcpy(stats, &array->stats, sizeof(array->stats)); diff --git a/dpdk/lib/librte_table/rte_table_hash.h b/dpdk/lib/librte_table/rte_table_hash.h index 9d17516a..15f1902b 100644 --- a/dpdk/lib/librte_table/rte_table_hash.h +++ b/dpdk/lib/librte_table/rte_table_hash.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -45,8 +45,6 @@ extern "C" { * These tables use the exact match criterion to uniquely associate data to * lookup keys. * - * Use-cases: Flow classification table, Address Resolution Protocol (ARP) table - * * Hash table types: * 1. Entry add strategy on bucket full: * a. Least Recently Used (LRU): One of the existing keys in the bucket is @@ -59,7 +57,7 @@ extern "C" { * to the bucket, it also becomes the new MRU key. When a key needs to * be picked and dropped, the most likely candidate for drop, i.e. the * current LRU key, is always picked. The LRU logic requires maintaining - * specific data structures per each bucket. + * specific data structures per each bucket. Use-cases: flow cache, etc. * b. Extendible bucket (ext): The bucket is extended with space for 4 more * keys. This is done by allocating additional memory at table init time, * which is used to create a pool of free keys (the size of this pool is @@ -73,20 +71,8 @@ extern "C" { * first group of 4 keys, the search continues beyond the first group of * 4 keys, potentially until all keys in this bucket are examined. The * extendible bucket logic requires maintaining specific data structures - * per table and per each bucket. - * 2. Key signature computation: - * a. Pre-computed key signature: The key lookup operation is split between - * two CPU cores. The first CPU core (typically the CPU core performing - * packet RX) extracts the key from the input packet, computes the key - * signature and saves both the key and the key signature in the packet - * buffer as packet meta-data. The second CPU core reads both the key and - * the key signature from the packet meta-data and performs the bucket - * search step of the key lookup operation. - * b. Key signature computed on lookup (do-sig): The same CPU core reads - * the key from the packet meta-data, uses it to compute the key - * signature and also performs the bucket search step of the key lookup - * operation. - * 3. Key size: + * per table and per each bucket. Use-cases: flow table, etc. + * 2. Key size: * a. Configurable key size * b. Single key size (8-byte, 16-byte or 32-byte key size) * @@ -98,59 +84,28 @@ extern "C" { /** Hash function */ typedef uint64_t (*rte_table_hash_op_hash)( void *key, + void *key_mask, uint32_t key_size, uint64_t seed); -/** - * Hash tables with configurable key size - * - */ -/** Extendible bucket hash table parameters */ -struct rte_table_hash_ext_params { +/** Hash table parameters */ +struct rte_table_hash_params { + /** Name */ + const char *name; + /** Key size (number of bytes) */ uint32_t key_size; - /** Maximum number of keys */ - uint32_t n_keys; - - /** Number of hash table buckets. Each bucket stores up to 4 keys. */ - uint32_t n_buckets; - - /** Number of hash table bucket extensions. Each bucket extension has - space for 4 keys and each bucket can have 0, 1 or more extensions. */ - uint32_t n_buckets_ext; - - /** Hash function */ - rte_table_hash_op_hash f_hash; - - /** Seed value for the hash function */ - uint64_t seed; - - /** Byte offset within packet meta-data where the 4-byte key signature - is located. Valid for pre-computed key signature tables, ignored for - do-sig tables. */ - uint32_t signature_offset; - /** Byte offset within packet meta-data where the key is located */ uint32_t key_offset; -}; -/** Extendible bucket hash table operations for pre-computed key signature */ -extern struct rte_table_ops rte_table_hash_ext_ops; + /** Key mask */ + uint8_t *key_mask; -/** Extendible bucket hash table operations for key signature computed on - lookup ("do-sig") */ -extern struct rte_table_ops rte_table_hash_ext_dosig_ops; - -/** LRU hash table parameters */ -struct rte_table_hash_lru_params { - /** Key size (number of bytes) */ - uint32_t key_size; - - /** Maximum number of keys */ + /** Number of keys */ uint32_t n_keys; - /** Number of hash table buckets. Each bucket stores up to 4 keys. */ + /** Number of buckets */ uint32_t n_buckets; /** Hash function */ @@ -158,211 +113,24 @@ struct rte_table_hash_lru_params { /** Seed value for the hash function */ uint64_t seed; - - /** Byte offset within packet meta-data where the 4-byte key signature - is located. Valid for pre-computed key signature tables, ignored for - do-sig tables. */ - uint32_t signature_offset; - - /** Byte offset within packet meta-data where the key is located */ - uint32_t key_offset; -}; - -/** LRU hash table operations for pre-computed key signature */ -extern struct rte_table_ops rte_table_hash_lru_ops; - -/** LRU hash table operations for key signature computed on lookup ("do-sig") */ -extern struct rte_table_ops rte_table_hash_lru_dosig_ops; - -/** - * 8-byte key hash tables - * - */ -/** LRU hash table parameters */ -struct rte_table_hash_key8_lru_params { - /** Maximum number of entries (and keys) in the table */ - uint32_t n_entries; - - /** Hash function */ - rte_table_hash_op_hash f_hash; - - /** Seed for the hash function */ - uint64_t seed; - - /** Byte offset within packet meta-data where the 4-byte key signature - is located. Valid for pre-computed key signature tables, ignored for - do-sig tables. */ - uint32_t signature_offset; - - /** Byte offset within packet meta-data where the key is located */ - uint32_t key_offset; - - /** Bit-mask to be AND-ed to the key on lookup */ - uint8_t *key_mask; -}; - -/** LRU hash table operations for pre-computed key signature */ -extern struct rte_table_ops rte_table_hash_key8_lru_ops; - -/** LRU hash table operations for key signature computed on lookup ("do-sig") */ -extern struct rte_table_ops rte_table_hash_key8_lru_dosig_ops; - -/** Extendible bucket hash table parameters */ -struct rte_table_hash_key8_ext_params { - /** Maximum number of entries (and keys) in the table */ - uint32_t n_entries; - - /** Number of entries (and keys) for hash table bucket extensions. Each - bucket is extended in increments of 4 keys. */ - uint32_t n_entries_ext; - - /** Hash function */ - rte_table_hash_op_hash f_hash; - - /** Seed for the hash function */ - uint64_t seed; - - /** Byte offset within packet meta-data where the 4-byte key signature - is located. Valid for pre-computed key signature tables, ignored for - do-sig tables. */ - uint32_t signature_offset; - - /** Byte offset within packet meta-data where the key is located */ - uint32_t key_offset; - - /** Bit-mask to be AND-ed to the key on lookup */ - uint8_t *key_mask; -}; - -/** Extendible bucket hash table operations for pre-computed key signature */ -extern struct rte_table_ops rte_table_hash_key8_ext_ops; - -/** Extendible bucket hash table operations for key signature computed on - lookup ("do-sig") */ -extern struct rte_table_ops rte_table_hash_key8_ext_dosig_ops; - -/** - * 16-byte key hash tables - * - */ -/** LRU hash table parameters */ -struct rte_table_hash_key16_lru_params { - /** Maximum number of entries (and keys) in the table */ - uint32_t n_entries; - - /** Hash function */ - rte_table_hash_op_hash f_hash; - - /** Seed for the hash function */ - uint64_t seed; - - /** Byte offset within packet meta-data where the 4-byte key signature - is located. Valid for pre-computed key signature tables, ignored for - do-sig tables. */ - uint32_t signature_offset; - - /** Byte offset within packet meta-data where the key is located */ - uint32_t key_offset; - - /** Bit-mask to be AND-ed to the key on lookup */ - uint8_t *key_mask; -}; - -/** LRU hash table operations for pre-computed key signature */ -extern struct rte_table_ops rte_table_hash_key16_lru_ops; - -/** LRU hash table operations for key signature computed on lookup - ("do-sig") */ -extern struct rte_table_ops rte_table_hash_key16_lru_dosig_ops; - -/** Extendible bucket hash table parameters */ -struct rte_table_hash_key16_ext_params { - /** Maximum number of entries (and keys) in the table */ - uint32_t n_entries; - - /** Number of entries (and keys) for hash table bucket extensions. Each - bucket is extended in increments of 4 keys. */ - uint32_t n_entries_ext; - - /** Hash function */ - rte_table_hash_op_hash f_hash; - - /** Seed for the hash function */ - uint64_t seed; - - /** Byte offset within packet meta-data where the 4-byte key signature - is located. Valid for pre-computed key signature tables, ignored for - do-sig tables. */ - uint32_t signature_offset; - - /** Byte offset within packet meta-data where the key is located */ - uint32_t key_offset; - - /** Bit-mask to be AND-ed to the key on lookup */ - uint8_t *key_mask; -}; - -/** Extendible bucket operations for pre-computed key signature */ -extern struct rte_table_ops rte_table_hash_key16_ext_ops; - -/** Extendible bucket hash table operations for key signature computed on - lookup ("do-sig") */ -extern struct rte_table_ops rte_table_hash_key16_ext_dosig_ops; - -/** - * 32-byte key hash tables - * - */ -/** LRU hash table parameters */ -struct rte_table_hash_key32_lru_params { - /** Maximum number of entries (and keys) in the table */ - uint32_t n_entries; - - /** Hash function */ - rte_table_hash_op_hash f_hash; - - /** Seed for the hash function */ - uint64_t seed; - - /** Byte offset within packet meta-data where the 4-byte key signature - is located. Valid for pre-computed key signature tables, ignored for - do-sig tables. */ - uint32_t signature_offset; - - /** Byte offset within packet meta-data where the key is located */ - uint32_t key_offset; -}; - -/** LRU hash table operations for pre-computed key signature */ -extern struct rte_table_ops rte_table_hash_key32_lru_ops; - -/** Extendible bucket hash table parameters */ -struct rte_table_hash_key32_ext_params { - /** Maximum number of entries (and keys) in the table */ - uint32_t n_entries; - - /** Number of entries (and keys) for hash table bucket extensions. Each - bucket is extended in increments of 4 keys. */ - uint32_t n_entries_ext; - - /** Hash function */ - rte_table_hash_op_hash f_hash; - - /** Seed for the hash function */ - uint64_t seed; - - /** Byte offset within packet meta-data where the 4-byte key signature - is located. Valid for pre-computed key signature tables, ignored for - do-sig tables. */ - uint32_t signature_offset; - - /** Byte offset within packet meta-data where the key is located */ - uint32_t key_offset; }; /** Extendible bucket hash table operations */ +extern struct rte_table_ops rte_table_hash_ext_ops; +extern struct rte_table_ops rte_table_hash_key8_ext_ops; +extern struct rte_table_ops rte_table_hash_key16_ext_ops; extern struct rte_table_ops rte_table_hash_key32_ext_ops; +/** LRU hash table operations */ +extern struct rte_table_ops rte_table_hash_lru_ops; + +extern struct rte_table_ops rte_table_hash_key8_lru_ops; +extern struct rte_table_ops rte_table_hash_key16_lru_ops; +extern struct rte_table_ops rte_table_hash_key32_lru_ops; + +/** Cuckoo hash table operations */ +extern struct rte_table_ops rte_table_hash_cuckoo_ops; + #ifdef __cplusplus } #endif diff --git a/dpdk/lib/librte_table/rte_table_hash_cuckoo.c b/dpdk/lib/librte_table/rte_table_hash_cuckoo.c new file mode 100644 index 00000000..f3845c75 --- /dev/null +++ b/dpdk/lib/librte_table/rte_table_hash_cuckoo.c @@ -0,0 +1,353 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include +#include + +#include +#include +#include +#include +#include + +#include +#include "rte_table_hash.h" + +#ifdef RTE_TABLE_STATS_COLLECT + +#define RTE_TABLE_HASH_CUCKOO_STATS_PKTS_IN_ADD(table, val) \ + (table->stats.n_pkts_in += val) +#define RTE_TABLE_HASH_CUCKOO_STATS_PKTS_LOOKUP_MISS(table, val) \ + (table->stats.n_pkts_lookup_miss += val) + +#else + +#define RTE_TABLE_HASH_CUCKOO_STATS_PKTS_IN_ADD(table, val) +#define RTE_TABLE_HASH_CUCKOO_STATS_PKTS_LOOKUP_MISS(table, val) + +#endif + + +struct rte_table_hash { + struct rte_table_stats stats; + + /* Input parameters */ + uint32_t key_size; + uint32_t entry_size; + uint32_t n_keys; + rte_table_hash_op_hash f_hash; + uint32_t seed; + uint32_t key_offset; + + /* cuckoo hash table object */ + struct rte_hash *h_table; + + /* Lookup table */ + uint8_t memory[0] __rte_cache_aligned; +}; + +static int +check_params_create_hash_cuckoo(struct rte_table_hash_params *params) +{ + if (params == NULL) { + RTE_LOG(ERR, TABLE, "NULL Input Parameters.\n"); + return -EINVAL; + } + + if (params->name == NULL) { + RTE_LOG(ERR, TABLE, "Table name is NULL.\n"); + return -EINVAL; + } + + if (params->key_size == 0) { + RTE_LOG(ERR, TABLE, "Invalid key_size.\n"); + return -EINVAL; + } + + if (params->n_keys == 0) { + RTE_LOG(ERR, TABLE, "Invalid n_keys.\n"); + return -EINVAL; + } + + if (params->f_hash == NULL) { + RTE_LOG(ERR, TABLE, "f_hash is NULL.\n"); + return -EINVAL; + } + + return 0; +} + +static void * +rte_table_hash_cuckoo_create(void *params, + int socket_id, + uint32_t entry_size) +{ + struct rte_table_hash_params *p = params; + struct rte_hash *h_table; + struct rte_table_hash *t; + uint32_t total_size; + + /* Check input parameters */ + if (check_params_create_hash_cuckoo(params)) + return NULL; + + /* Memory allocation */ + total_size = sizeof(struct rte_table_hash) + + RTE_CACHE_LINE_ROUNDUP(p->n_keys * entry_size); + + t = rte_zmalloc_socket(p->name, total_size, RTE_CACHE_LINE_SIZE, socket_id); + if (t == NULL) { + RTE_LOG(ERR, TABLE, + "%s: Cannot allocate %u bytes for cuckoo hash table %s\n", + __func__, total_size, p->name); + return NULL; + } + + /* Create cuckoo hash table */ + struct rte_hash_parameters hash_cuckoo_params = { + .entries = p->n_keys, + .key_len = p->key_size, + .hash_func = (rte_hash_function)(p->f_hash), + .hash_func_init_val = p->seed, + .socket_id = socket_id, + .name = p->name + }; + + h_table = rte_hash_find_existing(p->name); + if (h_table == NULL) { + h_table = rte_hash_create(&hash_cuckoo_params); + if (h_table == NULL) { + RTE_LOG(ERR, TABLE, + "%s: failed to create cuckoo hash table %s\n", + __func__, p->name); + rte_free(t); + return NULL; + } + } + + /* initialize the cuckoo hash parameters */ + t->key_size = p->key_size; + t->entry_size = entry_size; + t->n_keys = p->n_keys; + t->f_hash = p->f_hash; + t->seed = p->seed; + t->key_offset = p->key_offset; + t->h_table = h_table; + + RTE_LOG(INFO, TABLE, + "%s: Cuckoo hash table %s memory footprint is %u bytes\n", + __func__, p->name, total_size); + return t; +} + +static int +rte_table_hash_cuckoo_free(void *table) { + struct rte_table_hash *t = table; + + if (table == NULL) + return -EINVAL; + + rte_hash_free(t->h_table); + rte_free(t); + + return 0; +} + +static int +rte_table_hash_cuckoo_entry_add(void *table, void *key, void *entry, + int *key_found, void **entry_ptr) +{ + struct rte_table_hash *t = table; + int pos = 0; + + /* Check input parameters */ + if ((table == NULL) || + (key == NULL) || + (entry == NULL) || + (key_found == NULL) || + (entry_ptr == NULL)) + return -EINVAL; + + /* Find Existing entries */ + pos = rte_hash_lookup(t->h_table, key); + if (pos >= 0) { + uint8_t *existing_entry; + + *key_found = 1; + existing_entry = &t->memory[pos * t->entry_size]; + memcpy(existing_entry, entry, t->entry_size); + *entry_ptr = existing_entry; + + return 0; + } + + if (pos == -ENOENT) { + /* Entry not found. Adding new entry */ + uint8_t *new_entry; + + pos = rte_hash_add_key(t->h_table, key); + if (pos < 0) + return pos; + + new_entry = &t->memory[pos * t->entry_size]; + memcpy(new_entry, entry, t->entry_size); + + *key_found = 0; + *entry_ptr = new_entry; + return 0; + } + + return pos; +} + +static int +rte_table_hash_cuckoo_entry_delete(void *table, void *key, + int *key_found, void *entry) +{ + struct rte_table_hash *t = table; + int pos = 0; + + /* Check input parameters */ + if ((table == NULL) || + (key == NULL) || + (key_found == NULL)) + return -EINVAL; + + pos = rte_hash_del_key(t->h_table, key); + if (pos >= 0) { + *key_found = 1; + uint8_t *entry_ptr = &t->memory[pos * t->entry_size]; + + if (entry) + memcpy(entry, entry_ptr, t->entry_size); + + memset(&t->memory[pos * t->entry_size], 0, t->entry_size); + return 0; + } + + *key_found = 0; + return pos; +} + +static int +rte_table_hash_cuckoo_lookup(void *table, + struct rte_mbuf **pkts, + uint64_t pkts_mask, + uint64_t *lookup_hit_mask, + void **entries) +{ + struct rte_table_hash *t = table; + uint64_t pkts_mask_out = 0; + uint32_t i; + + __rte_unused uint32_t n_pkts_in = __builtin_popcountll(pkts_mask); + + RTE_TABLE_HASH_CUCKOO_STATS_PKTS_IN_ADD(t, n_pkts_in); + + if ((pkts_mask & (pkts_mask + 1)) == 0) { + const uint8_t *keys[RTE_PORT_IN_BURST_SIZE_MAX]; + int32_t positions[RTE_PORT_IN_BURST_SIZE_MAX], status; + + /* Keys for bulk lookup */ + for (i = 0; i < n_pkts_in; i++) + keys[i] = RTE_MBUF_METADATA_UINT8_PTR(pkts[i], + t->key_offset); + + /* Bulk Lookup */ + status = rte_hash_lookup_bulk(t->h_table, + (const void **) keys, + n_pkts_in, + positions); + if (status == 0) { + for (i = 0; i < n_pkts_in; i++) { + if (likely(positions[i] >= 0)) { + uint64_t pkt_mask = 1LLU << i; + + entries[i] = &t->memory[positions[i] + * t->entry_size]; + pkts_mask_out |= pkt_mask; + } + } + } + } else + for (i = 0; i < (uint32_t)(RTE_PORT_IN_BURST_SIZE_MAX + - __builtin_clzll(pkts_mask)); i++) { + uint64_t pkt_mask = 1LLU << i; + + if (pkt_mask & pkts_mask) { + struct rte_mbuf *pkt = pkts[i]; + uint8_t *key = RTE_MBUF_METADATA_UINT8_PTR(pkt, + t->key_offset); + int pos; + + pos = rte_hash_lookup(t->h_table, key); + if (likely(pos >= 0)) { + entries[i] = &t->memory[pos + * t->entry_size]; + pkts_mask_out |= pkt_mask; + } + } + } + + *lookup_hit_mask = pkts_mask_out; + RTE_TABLE_HASH_CUCKOO_STATS_PKTS_LOOKUP_MISS(t, + n_pkts_in - __builtin_popcountll(pkts_mask_out)); + + return 0; + +} + +static int +rte_table_hash_cuckoo_stats_read(void *table, struct rte_table_stats *stats, + int clear) +{ + struct rte_table_hash *t = table; + + if (stats != NULL) + memcpy(stats, &t->stats, sizeof(t->stats)); + + if (clear) + memset(&t->stats, 0, sizeof(t->stats)); + + return 0; +} + +struct rte_table_ops rte_table_hash_cuckoo_ops = { + .f_create = rte_table_hash_cuckoo_create, + .f_free = rte_table_hash_cuckoo_free, + .f_add = rte_table_hash_cuckoo_entry_add, + .f_delete = rte_table_hash_cuckoo_entry_delete, + .f_add_bulk = NULL, + .f_delete_bulk = NULL, + .f_lookup = rte_table_hash_cuckoo_lookup, + .f_stats = rte_table_hash_cuckoo_stats_read, +}; diff --git a/dpdk/lib/librte_table/rte_table_hash_ext.c b/dpdk/lib/librte_table/rte_table_hash_ext.c index e283a3d1..3af1bcab 100644 --- a/dpdk/lib/librte_table/rte_table_hash_ext.c +++ b/dpdk/lib/librte_table/rte_table_hash_ext.c @@ -1,34 +1,34 @@ /*- - * BSD LICENSE + * BSD LICENSE * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. + * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. + * All rights reserved. * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include @@ -106,7 +106,6 @@ struct rte_table_hash { uint32_t n_buckets_ext; rte_table_hash_op_hash f_hash; uint64_t seed; - uint32_t signature_offset; uint32_t key_offset; /* Internal */ @@ -120,6 +119,7 @@ struct rte_table_hash { struct grinder grinders[RTE_PORT_IN_BURST_SIZE_MAX]; /* Tables */ + uint64_t *key_mask; struct bucket *buckets; struct bucket *buckets_ext; uint8_t *key_mem; @@ -132,29 +132,53 @@ struct rte_table_hash { }; static int -check_params_create(struct rte_table_hash_ext_params *params) +keycmp(void *a, void *b, void *b_mask, uint32_t n_bytes) { - uint32_t n_buckets_min; + uint64_t *a64 = a, *b64 = b, *b_mask64 = b_mask; + uint32_t i; + + for (i = 0; i < n_bytes / sizeof(uint64_t); i++) + if (a64[i] != (b64[i] & b_mask64[i])) + return 1; + + return 0; +} + +static void +keycpy(void *dst, void *src, void *src_mask, uint32_t n_bytes) +{ + uint64_t *dst64 = dst, *src64 = src, *src_mask64 = src_mask; + uint32_t i; + + for (i = 0; i < n_bytes / sizeof(uint64_t); i++) + dst64[i] = src64[i] & src_mask64[i]; +} + +static int +check_params_create(struct rte_table_hash_params *params) +{ + /* name */ + if (params->name == NULL) { + RTE_LOG(ERR, TABLE, "%s: name invalid value\n", __func__); + return -EINVAL; + } /* key_size */ - if ((params->key_size == 0) || + if ((params->key_size < sizeof(uint64_t)) || (!rte_is_power_of_2(params->key_size))) { RTE_LOG(ERR, TABLE, "%s: key_size invalid value\n", __func__); return -EINVAL; } /* n_keys */ - if ((params->n_keys == 0) || - (!rte_is_power_of_2(params->n_keys))) { + if (params->n_keys == 0) { RTE_LOG(ERR, TABLE, "%s: n_keys invalid value\n", __func__); return -EINVAL; } /* n_buckets */ - n_buckets_min = (params->n_keys + KEYS_PER_BUCKET - 1) / params->n_keys; if ((params->n_buckets == 0) || - (!rte_is_power_of_2(params->n_keys)) || - (params->n_buckets < n_buckets_min)) { + (!rte_is_power_of_2(params->n_buckets))) { RTE_LOG(ERR, TABLE, "%s: n_buckets invalid value\n", __func__); return -EINVAL; } @@ -171,15 +195,13 @@ check_params_create(struct rte_table_hash_ext_params *params) static void * rte_table_hash_ext_create(void *params, int socket_id, uint32_t entry_size) { - struct rte_table_hash_ext_params *p = - (struct rte_table_hash_ext_params *) params; + struct rte_table_hash_params *p = params; struct rte_table_hash *t; - uint32_t total_size, table_meta_sz; - uint32_t bucket_sz, bucket_ext_sz, key_sz; - uint32_t key_stack_sz, bkt_ext_stack_sz, data_sz; - uint32_t bucket_offset, bucket_ext_offset, key_offset; - uint32_t key_stack_offset, bkt_ext_stack_offset, data_offset; - uint32_t i; + uint64_t table_meta_sz, key_mask_sz, bucket_sz, bucket_ext_sz, key_sz; + uint64_t key_stack_sz, bkt_ext_stack_sz, data_sz, total_size; + uint64_t key_mask_offset, bucket_offset, bucket_ext_offset, key_offset; + uint64_t key_stack_offset, bkt_ext_stack_offset, data_offset; + uint32_t n_buckets_ext, i; /* Check input parameters */ if ((check_params_create(p) != 0) || @@ -188,38 +210,66 @@ rte_table_hash_ext_create(void *params, int socket_id, uint32_t entry_size) (sizeof(struct bucket) != (RTE_CACHE_LINE_SIZE / 2))) return NULL; + /* + * Table dimensioning + * + * Objective: Pick the number of bucket extensions (n_buckets_ext) so that + * it is guaranteed that n_keys keys can be stored in the table at any time. + * + * The worst case scenario takes place when all the n_keys keys fall into + * the same bucket. Actually, due to the KEYS_PER_BUCKET scheme, the worst + * case takes place when (n_keys - KEYS_PER_BUCKET + 1) keys fall into the + * same bucket, while the remaining (KEYS_PER_BUCKET - 1) keys each fall + * into a different bucket. This case defeats the purpose of the hash table. + * It indicates unsuitable f_hash or n_keys to n_buckets ratio. + * + * n_buckets_ext = n_keys / KEYS_PER_BUCKET + KEYS_PER_BUCKET - 1 + */ + n_buckets_ext = p->n_keys / KEYS_PER_BUCKET + KEYS_PER_BUCKET - 1; + /* Memory allocation */ table_meta_sz = RTE_CACHE_LINE_ROUNDUP(sizeof(struct rte_table_hash)); + key_mask_sz = RTE_CACHE_LINE_ROUNDUP(p->key_size); bucket_sz = RTE_CACHE_LINE_ROUNDUP(p->n_buckets * sizeof(struct bucket)); bucket_ext_sz = - RTE_CACHE_LINE_ROUNDUP(p->n_buckets_ext * sizeof(struct bucket)); + RTE_CACHE_LINE_ROUNDUP(n_buckets_ext * sizeof(struct bucket)); key_sz = RTE_CACHE_LINE_ROUNDUP(p->n_keys * p->key_size); key_stack_sz = RTE_CACHE_LINE_ROUNDUP(p->n_keys * sizeof(uint32_t)); bkt_ext_stack_sz = - RTE_CACHE_LINE_ROUNDUP(p->n_buckets_ext * sizeof(uint32_t)); + RTE_CACHE_LINE_ROUNDUP(n_buckets_ext * sizeof(uint32_t)); data_sz = RTE_CACHE_LINE_ROUNDUP(p->n_keys * entry_size); - total_size = table_meta_sz + bucket_sz + bucket_ext_sz + key_sz + - key_stack_sz + bkt_ext_stack_sz + data_sz; + total_size = table_meta_sz + key_mask_sz + bucket_sz + bucket_ext_sz + + key_sz + key_stack_sz + bkt_ext_stack_sz + data_sz; - t = rte_zmalloc_socket("TABLE", total_size, RTE_CACHE_LINE_SIZE, socket_id); - if (t == NULL) { - RTE_LOG(ERR, TABLE, - "%s: Cannot allocate %u bytes for hash table\n", - __func__, total_size); + if (total_size > SIZE_MAX) { + RTE_LOG(ERR, TABLE, "%s: Cannot allocate %" PRIu64 " bytes" + " for hash table %s\n", + __func__, total_size, p->name); return NULL; } - RTE_LOG(INFO, TABLE, "%s (%u-byte key): Hash table memory footprint is " - "%u bytes\n", __func__, p->key_size, total_size); + + t = rte_zmalloc_socket(p->name, + (size_t)total_size, + RTE_CACHE_LINE_SIZE, + socket_id); + if (t == NULL) { + RTE_LOG(ERR, TABLE, "%s: Cannot allocate %" PRIu64 " bytes" + " for hash table %s\n", + __func__, total_size, p->name); + return NULL; + } + RTE_LOG(INFO, TABLE, "%s (%u-byte key): Hash table %s memory " + "footprint is %" PRIu64 " bytes\n", + __func__, p->key_size, p->name, total_size); /* Memory initialization */ t->key_size = p->key_size; t->entry_size = entry_size; t->n_keys = p->n_keys; t->n_buckets = p->n_buckets; - t->n_buckets_ext = p->n_buckets_ext; + t->n_buckets_ext = n_buckets_ext; t->f_hash = p->f_hash; t->seed = p->seed; - t->signature_offset = p->signature_offset; t->key_offset = p->key_offset; /* Internal */ @@ -228,13 +278,15 @@ rte_table_hash_ext_create(void *params, int socket_id, uint32_t entry_size) t->data_size_shl = __builtin_ctzl(entry_size); /* Tables */ - bucket_offset = 0; + key_mask_offset = 0; + bucket_offset = key_mask_offset + key_mask_sz; bucket_ext_offset = bucket_offset + bucket_sz; key_offset = bucket_ext_offset + bucket_ext_sz; key_stack_offset = key_offset + key_sz; bkt_ext_stack_offset = key_stack_offset + key_stack_sz; data_offset = bkt_ext_stack_offset + bkt_ext_stack_sz; + t->key_mask = (uint64_t *) &t->memory[key_mask_offset]; t->buckets = (struct bucket *) &t->memory[bucket_offset]; t->buckets_ext = (struct bucket *) &t->memory[bucket_ext_offset]; t->key_mem = &t->memory[key_offset]; @@ -242,6 +294,12 @@ rte_table_hash_ext_create(void *params, int socket_id, uint32_t entry_size) t->bkt_ext_stack = (uint32_t *) &t->memory[bkt_ext_stack_offset]; t->data_mem = &t->memory[data_offset]; + /* Key mask */ + if (p->key_mask == NULL) + memset(t->key_mask, 0xFF, p->key_size); + else + memcpy(t->key_mask, p->key_mask, p->key_size); + /* Key stack */ for (i = 0; i < t->n_keys; i++) t->key_stack[i] = t->n_keys - 1 - i; @@ -258,7 +316,7 @@ rte_table_hash_ext_create(void *params, int socket_id, uint32_t entry_size) static int rte_table_hash_ext_free(void *table) { - struct rte_table_hash *t = (struct rte_table_hash *) table; + struct rte_table_hash *t = table; /* Check input parameters */ if (t == NULL) @@ -272,12 +330,12 @@ static int rte_table_hash_ext_entry_add(void *table, void *key, void *entry, int *key_found, void **entry_ptr) { - struct rte_table_hash *t = (struct rte_table_hash *) table; + struct rte_table_hash *t = table; struct bucket *bkt0, *bkt, *bkt_prev; uint64_t sig; uint32_t bkt_index, i; - sig = t->f_hash(key, t->key_size, t->seed); + sig = t->f_hash(key, t->key_mask, t->key_size, t->seed); bkt_index = sig & t->bucket_mask; bkt0 = &t->buckets[bkt_index]; sig = (sig >> 16) | 1LLU; @@ -290,7 +348,7 @@ rte_table_hash_ext_entry_add(void *table, void *key, void *entry, uint8_t *bkt_key = &t->key_mem[bkt_key_index << t->key_size_shl]; - if ((sig == bkt_sig) && (memcmp(key, bkt_key, + if ((sig == bkt_sig) && (keycmp(bkt_key, key, t->key_mask, t->key_size) == 0)) { uint8_t *data = &t->data_mem[bkt_key_index << t->data_size_shl]; @@ -327,7 +385,7 @@ rte_table_hash_ext_entry_add(void *table, void *key, void *entry, bkt->sig[i] = (uint16_t) sig; bkt->key_pos[i] = bkt_key_index; - memcpy(bkt_key, key, t->key_size); + keycpy(bkt_key, key, t->key_mask, t->key_size); memcpy(data, entry, t->entry_size); *key_found = 0; @@ -358,7 +416,7 @@ rte_table_hash_ext_entry_add(void *table, void *key, void *entry, /* Install new key into bucket */ bkt->sig[0] = (uint16_t) sig; bkt->key_pos[0] = bkt_key_index; - memcpy(bkt_key, key, t->key_size); + keycpy(bkt_key, key, t->key_mask, t->key_size); memcpy(data, entry, t->entry_size); *key_found = 0; @@ -373,12 +431,12 @@ static int rte_table_hash_ext_entry_delete(void *table, void *key, int *key_found, void *entry) { - struct rte_table_hash *t = (struct rte_table_hash *) table; + struct rte_table_hash *t = table; struct bucket *bkt0, *bkt, *bkt_prev; uint64_t sig; uint32_t bkt_index, i; - sig = t->f_hash(key, t->key_size, t->seed); + sig = t->f_hash(key, t->key_mask, t->key_size, t->seed); bkt_index = sig & t->bucket_mask; bkt0 = &t->buckets[bkt_index]; sig = (sig >> 16) | 1LLU; @@ -392,7 +450,7 @@ void *entry) uint8_t *bkt_key = &t->key_mem[bkt_key_index << t->key_size_shl]; - if ((sig == bkt_sig) && (memcmp(key, bkt_key, + if ((sig == bkt_sig) && (keycmp(bkt_key, key, t->key_mask, t->key_size) == 0)) { uint8_t *data = &t->data_mem[bkt_key_index << t->data_size_shl]; @@ -437,14 +495,12 @@ static int rte_table_hash_ext_lookup_unoptimized( struct rte_mbuf **pkts, uint64_t pkts_mask, uint64_t *lookup_hit_mask, - void **entries, - int dosig) + void **entries) { struct rte_table_hash *t = (struct rte_table_hash *) table; uint64_t pkts_mask_out = 0; __rte_unused uint32_t n_pkts_in = __builtin_popcountll(pkts_mask); - RTE_TABLE_HASH_EXT_STATS_PKTS_IN_ADD(t, n_pkts_in); for ( ; pkts_mask; ) { struct bucket *bkt0, *bkt; @@ -459,11 +515,7 @@ static int rte_table_hash_ext_lookup_unoptimized( pkt = pkts[pkt_index]; key = RTE_MBUF_METADATA_UINT8_PTR(pkt, t->key_offset); - if (dosig) - sig = (uint64_t) t->f_hash(key, t->key_size, t->seed); - else - sig = RTE_MBUF_METADATA_UINT32(pkt, - t->signature_offset); + sig = (uint64_t) t->f_hash(key, t->key_mask, t->key_size, t->seed); bkt_index = sig & t->bucket_mask; bkt0 = &t->buckets[bkt_index]; @@ -477,8 +529,8 @@ static int rte_table_hash_ext_lookup_unoptimized( uint8_t *bkt_key = &t->key_mem[bkt_key_index << t->key_size_shl]; - if ((sig == bkt_sig) && (memcmp(key, bkt_key, - t->key_size) == 0)) { + if ((sig == bkt_sig) && (keycmp(bkt_key, key, + t->key_mask, t->key_size) == 0)) { uint8_t *data = &t->data_mem[ bkt_key_index << t->data_size_shl]; @@ -490,7 +542,6 @@ static int rte_table_hash_ext_lookup_unoptimized( } *lookup_hit_mask = pkts_mask_out; - RTE_TABLE_HASH_EXT_STATS_PKTS_LOOKUP_MISS(t, n_pkts_in - __builtin_popcountll(pkts_mask_out)); return 0; } @@ -578,11 +629,12 @@ static int rte_table_hash_ext_lookup_unoptimized( { \ uint64_t *pkt_key = RTE_MBUF_METADATA_UINT64_PTR(mbuf, f->key_offset);\ uint64_t *bkt_key = (uint64_t *) key; \ + uint64_t *key_mask = f->key_mask; \ \ switch (f->key_size) { \ case 8: \ { \ - uint64_t xor = pkt_key[0] ^ bkt_key[0]; \ + uint64_t xor = (pkt_key[0] & key_mask[0]) ^ bkt_key[0]; \ match_key = 0; \ if (xor == 0) \ match_key = 1; \ @@ -593,8 +645,8 @@ static int rte_table_hash_ext_lookup_unoptimized( { \ uint64_t xor[2], or; \ \ - xor[0] = pkt_key[0] ^ bkt_key[0]; \ - xor[1] = pkt_key[1] ^ bkt_key[1]; \ + xor[0] = (pkt_key[0] & key_mask[0]) ^ bkt_key[0]; \ + xor[1] = (pkt_key[1] & key_mask[1]) ^ bkt_key[1]; \ or = xor[0] | xor[1]; \ match_key = 0; \ if (or == 0) \ @@ -606,10 +658,10 @@ static int rte_table_hash_ext_lookup_unoptimized( { \ uint64_t xor[4], or; \ \ - xor[0] = pkt_key[0] ^ bkt_key[0]; \ - xor[1] = pkt_key[1] ^ bkt_key[1]; \ - xor[2] = pkt_key[2] ^ bkt_key[2]; \ - xor[3] = pkt_key[3] ^ bkt_key[3]; \ + xor[0] = (pkt_key[0] & key_mask[0]) ^ bkt_key[0]; \ + xor[1] = (pkt_key[1] & key_mask[1]) ^ bkt_key[1]; \ + xor[2] = (pkt_key[2] & key_mask[2]) ^ bkt_key[2]; \ + xor[3] = (pkt_key[3] & key_mask[3]) ^ bkt_key[3]; \ or = xor[0] | xor[1] | xor[2] | xor[3]; \ match_key = 0; \ if (or == 0) \ @@ -621,14 +673,14 @@ static int rte_table_hash_ext_lookup_unoptimized( { \ uint64_t xor[8], or; \ \ - xor[0] = pkt_key[0] ^ bkt_key[0]; \ - xor[1] = pkt_key[1] ^ bkt_key[1]; \ - xor[2] = pkt_key[2] ^ bkt_key[2]; \ - xor[3] = pkt_key[3] ^ bkt_key[3]; \ - xor[4] = pkt_key[4] ^ bkt_key[4]; \ - xor[5] = pkt_key[5] ^ bkt_key[5]; \ - xor[6] = pkt_key[6] ^ bkt_key[6]; \ - xor[7] = pkt_key[7] ^ bkt_key[7]; \ + xor[0] = (pkt_key[0] & key_mask[0]) ^ bkt_key[0]; \ + xor[1] = (pkt_key[1] & key_mask[1]) ^ bkt_key[1]; \ + xor[2] = (pkt_key[2] & key_mask[2]) ^ bkt_key[2]; \ + xor[3] = (pkt_key[3] & key_mask[3]) ^ bkt_key[3]; \ + xor[4] = (pkt_key[4] & key_mask[4]) ^ bkt_key[4]; \ + xor[5] = (pkt_key[5] & key_mask[5]) ^ bkt_key[5]; \ + xor[6] = (pkt_key[6] & key_mask[6]) ^ bkt_key[6]; \ + xor[7] = (pkt_key[7] & key_mask[7]) ^ bkt_key[7]; \ or = xor[0] | xor[1] | xor[2] | xor[3] | \ xor[4] | xor[5] | xor[6] | xor[7]; \ match_key = 0; \ @@ -639,7 +691,7 @@ static int rte_table_hash_ext_lookup_unoptimized( \ default: \ match_key = 0; \ - if (memcmp(pkt_key, bkt_key, f->key_size) == 0) \ + if (keycmp(bkt_key, pkt_key, key_mask, f->key_size) == 0) \ match_key = 1; \ } \ } @@ -687,38 +739,7 @@ static int rte_table_hash_ext_lookup_unoptimized( rte_prefetch0(RTE_MBUF_METADATA_UINT8_PTR(mbuf01, key_offset));\ } -#define lookup2_stage1(t, g, pkts, pkt10_index, pkt11_index) \ -{ \ - struct grinder *g10, *g11; \ - uint64_t sig10, sig11, bkt10_index, bkt11_index; \ - struct rte_mbuf *mbuf10, *mbuf11; \ - struct bucket *bkt10, *bkt11, *buckets = t->buckets; \ - uint64_t bucket_mask = t->bucket_mask; \ - uint32_t signature_offset = t->signature_offset; \ - \ - mbuf10 = pkts[pkt10_index]; \ - sig10 = (uint64_t) RTE_MBUF_METADATA_UINT32(mbuf10, signature_offset);\ - bkt10_index = sig10 & bucket_mask; \ - bkt10 = &buckets[bkt10_index]; \ - \ - mbuf11 = pkts[pkt11_index]; \ - sig11 = (uint64_t) RTE_MBUF_METADATA_UINT32(mbuf11, signature_offset);\ - bkt11_index = sig11 & bucket_mask; \ - bkt11 = &buckets[bkt11_index]; \ - \ - rte_prefetch0(bkt10); \ - rte_prefetch0(bkt11); \ - \ - g10 = &g[pkt10_index]; \ - g10->sig = sig10; \ - g10->bkt = bkt10; \ - \ - g11 = &g[pkt11_index]; \ - g11->sig = sig11; \ - g11->bkt = bkt11; \ -} - -#define lookup2_stage1_dosig(t, g, pkts, pkt10_index, pkt11_index) \ +#define lookup2_stage1(t, g, pkts, pkt10_index, pkt11_index) \ { \ struct grinder *g10, *g11; \ uint64_t sig10, sig11, bkt10_index, bkt11_index; \ @@ -733,13 +754,13 @@ static int rte_table_hash_ext_lookup_unoptimized( \ mbuf10 = pkts[pkt10_index]; \ key10 = RTE_MBUF_METADATA_UINT8_PTR(mbuf10, key_offset); \ - sig10 = (uint64_t) f_hash(key10, key_size, seed); \ + sig10 = (uint64_t) f_hash(key10, t->key_mask, key_size, seed); \ bkt10_index = sig10 & bucket_mask; \ bkt10 = &buckets[bkt10_index]; \ \ mbuf11 = pkts[pkt11_index]; \ key11 = RTE_MBUF_METADATA_UINT8_PTR(mbuf11, key_offset); \ - sig11 = (uint64_t) f_hash(key11, key_size, seed); \ + sig11 = (uint64_t) f_hash(key11, t->key_mask, key_size, seed); \ bkt11_index = sig11 & bucket_mask; \ bkt11 = &buckets[bkt11_index]; \ \ @@ -874,9 +895,13 @@ static int rte_table_hash_ext_lookup( RTE_TABLE_HASH_EXT_STATS_PKTS_IN_ADD(t, n_pkts_in); /* Cannot run the pipeline with less than 7 packets */ - if (__builtin_popcountll(pkts_mask) < 7) - return rte_table_hash_ext_lookup_unoptimized(table, pkts, - pkts_mask, lookup_hit_mask, entries, 0); + if (__builtin_popcountll(pkts_mask) < 7) { + status = rte_table_hash_ext_lookup_unoptimized(table, pkts, + pkts_mask, lookup_hit_mask, entries); + RTE_TABLE_HASH_EXT_STATS_PKTS_LOOKUP_MISS(t, n_pkts_in - + __builtin_popcountll(*lookup_hit_mask)); + return status; + } /* Pipeline stage 0 */ lookup2_stage0(t, g, pkts, pkts_mask, pkt00_index, pkt01_index); @@ -980,140 +1005,7 @@ static int rte_table_hash_ext_lookup( uint64_t pkts_mask_out_slow = 0; status = rte_table_hash_ext_lookup_unoptimized(table, pkts, - pkts_mask_match_many, &pkts_mask_out_slow, entries, 0); - pkts_mask_out |= pkts_mask_out_slow; - } - - *lookup_hit_mask = pkts_mask_out; - RTE_TABLE_HASH_EXT_STATS_PKTS_LOOKUP_MISS(t, n_pkts_in - __builtin_popcountll(pkts_mask_out)); - return status; -} - -static int rte_table_hash_ext_lookup_dosig( - void *table, - struct rte_mbuf **pkts, - uint64_t pkts_mask, - uint64_t *lookup_hit_mask, - void **entries) -{ - struct rte_table_hash *t = (struct rte_table_hash *) table; - struct grinder *g = t->grinders; - uint64_t pkt00_index, pkt01_index, pkt10_index, pkt11_index; - uint64_t pkt20_index, pkt21_index, pkt30_index, pkt31_index; - uint64_t pkts_mask_out = 0, pkts_mask_match_many = 0; - int status = 0; - - __rte_unused uint32_t n_pkts_in = __builtin_popcountll(pkts_mask); - RTE_TABLE_HASH_EXT_STATS_PKTS_IN_ADD(t, n_pkts_in); - - /* Cannot run the pipeline with less than 7 packets */ - if (__builtin_popcountll(pkts_mask) < 7) - return rte_table_hash_ext_lookup_unoptimized(table, pkts, - pkts_mask, lookup_hit_mask, entries, 1); - - /* Pipeline stage 0 */ - lookup2_stage0(t, g, pkts, pkts_mask, pkt00_index, pkt01_index); - - /* Pipeline feed */ - pkt10_index = pkt00_index; - pkt11_index = pkt01_index; - - /* Pipeline stage 0 */ - lookup2_stage0(t, g, pkts, pkts_mask, pkt00_index, pkt01_index); - - /* Pipeline stage 1 */ - lookup2_stage1_dosig(t, g, pkts, pkt10_index, pkt11_index); - - /* Pipeline feed */ - pkt20_index = pkt10_index; - pkt21_index = pkt11_index; - pkt10_index = pkt00_index; - pkt11_index = pkt01_index; - - /* Pipeline stage 0 */ - lookup2_stage0(t, g, pkts, pkts_mask, pkt00_index, pkt01_index); - - /* Pipeline stage 1 */ - lookup2_stage1_dosig(t, g, pkts, pkt10_index, pkt11_index); - - /* Pipeline stage 2 */ - lookup2_stage2(t, g, pkt20_index, pkt21_index, pkts_mask_match_many); - - /* - * Pipeline run - * - */ - for ( ; pkts_mask; ) { - /* Pipeline feed */ - pkt30_index = pkt20_index; - pkt31_index = pkt21_index; - pkt20_index = pkt10_index; - pkt21_index = pkt11_index; - pkt10_index = pkt00_index; - pkt11_index = pkt01_index; - - /* Pipeline stage 0 */ - lookup2_stage0_with_odd_support(t, g, pkts, pkts_mask, - pkt00_index, pkt01_index); - - /* Pipeline stage 1 */ - lookup2_stage1_dosig(t, g, pkts, pkt10_index, pkt11_index); - - /* Pipeline stage 2 */ - lookup2_stage2(t, g, pkt20_index, pkt21_index, - pkts_mask_match_many); - - /* Pipeline stage 3 */ - lookup2_stage3(t, g, pkts, pkt30_index, pkt31_index, - pkts_mask_out, entries); - } - - /* Pipeline feed */ - pkt30_index = pkt20_index; - pkt31_index = pkt21_index; - pkt20_index = pkt10_index; - pkt21_index = pkt11_index; - pkt10_index = pkt00_index; - pkt11_index = pkt01_index; - - /* Pipeline stage 1 */ - lookup2_stage1_dosig(t, g, pkts, pkt10_index, pkt11_index); - - /* Pipeline stage 2 */ - lookup2_stage2(t, g, pkt20_index, pkt21_index, pkts_mask_match_many); - - /* Pipeline stage 3 */ - lookup2_stage3(t, g, pkts, pkt30_index, pkt31_index, pkts_mask_out, - entries); - - /* Pipeline feed */ - pkt30_index = pkt20_index; - pkt31_index = pkt21_index; - pkt20_index = pkt10_index; - pkt21_index = pkt11_index; - - /* Pipeline stage 2 */ - lookup2_stage2(t, g, pkt20_index, pkt21_index, pkts_mask_match_many); - - /* Pipeline stage 3 */ - lookup2_stage3(t, g, pkts, pkt30_index, pkt31_index, pkts_mask_out, - entries); - - /* Pipeline feed */ - pkt30_index = pkt20_index; - pkt31_index = pkt21_index; - - /* Pipeline stage 3 */ - lookup2_stage3(t, g, pkts, pkt30_index, pkt31_index, pkts_mask_out, - entries); - - /* Slow path */ - pkts_mask_match_many &= ~pkts_mask_out; - if (pkts_mask_match_many) { - uint64_t pkts_mask_out_slow = 0; - - status = rte_table_hash_ext_lookup_unoptimized(table, pkts, - pkts_mask_match_many, &pkts_mask_out_slow, entries, 1); + pkts_mask_match_many, &pkts_mask_out_slow, entries); pkts_mask_out |= pkts_mask_out_slow; } @@ -1125,7 +1017,7 @@ static int rte_table_hash_ext_lookup_dosig( static int rte_table_hash_ext_stats_read(void *table, struct rte_table_stats *stats, int clear) { - struct rte_table_hash *t = (struct rte_table_hash *) table; + struct rte_table_hash *t = table; if (stats != NULL) memcpy(stats, &t->stats, sizeof(t->stats)); @@ -1146,14 +1038,3 @@ struct rte_table_ops rte_table_hash_ext_ops = { .f_lookup = rte_table_hash_ext_lookup, .f_stats = rte_table_hash_ext_stats_read, }; - -struct rte_table_ops rte_table_hash_ext_dosig_ops = { - .f_create = rte_table_hash_ext_create, - .f_free = rte_table_hash_ext_free, - .f_add = rte_table_hash_ext_entry_add, - .f_delete = rte_table_hash_ext_entry_delete, - .f_add_bulk = NULL, - .f_delete_bulk = NULL, - .f_lookup = rte_table_hash_ext_lookup_dosig, - .f_stats = rte_table_hash_ext_stats_read, -}; diff --git a/dpdk/lib/librte_table/rte_table_hash_key16.c b/dpdk/lib/librte_table/rte_table_hash_key16.c index b7e000fd..b541735c 100644 --- a/dpdk/lib/librte_table/rte_table_hash_key16.c +++ b/dpdk/lib/librte_table/rte_table_hash_key16.c @@ -1,34 +1,34 @@ /*- - * BSD LICENSE + * BSD LICENSE * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. + * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. + * All rights reserved. * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include #include @@ -42,7 +42,9 @@ #include "rte_table_hash.h" #include "rte_lru.h" -#define RTE_TABLE_HASH_KEY_SIZE 16 +#define KEY_SIZE 16 + +#define KEYS_PER_BUCKET 4 #define RTE_BUCKET_ENTRY_VALID 0x1LLU @@ -79,11 +81,9 @@ struct rte_table_hash { /* Input parameters */ uint32_t n_buckets; - uint32_t n_entries_per_bucket; uint32_t key_size; uint32_t entry_size; uint32_t bucket_size; - uint32_t signature_offset; uint32_t key_offset; uint64_t key_mask[2]; rte_table_hash_op_hash f_hash; @@ -99,17 +99,55 @@ struct rte_table_hash { }; static int -check_params_create_lru(struct rte_table_hash_key16_lru_params *params) { - /* n_entries */ - if (params->n_entries == 0) { - RTE_LOG(ERR, TABLE, "%s: n_entries is zero\n", __func__); +keycmp(void *a, void *b, void *b_mask) +{ + uint64_t *a64 = a, *b64 = b, *b_mask64 = b_mask; + + return (a64[0] != (b64[0] & b_mask64[0])) || + (a64[1] != (b64[1] & b_mask64[1])); +} + +static void +keycpy(void *dst, void *src, void *src_mask) +{ + uint64_t *dst64 = dst, *src64 = src, *src_mask64 = src_mask; + + dst64[0] = src64[0] & src_mask64[0]; + dst64[1] = src64[1] & src_mask64[1]; +} + +static int +check_params_create(struct rte_table_hash_params *params) +{ + /* name */ + if (params->name == NULL) { + RTE_LOG(ERR, TABLE, "%s: name invalid value\n", __func__); + return -EINVAL; + } + + /* key_size */ + if (params->key_size != KEY_SIZE) { + RTE_LOG(ERR, TABLE, "%s: key_size invalid value\n", __func__); + return -EINVAL; + } + + /* n_keys */ + if (params->n_keys == 0) { + RTE_LOG(ERR, TABLE, "%s: n_keys is zero\n", __func__); + return -EINVAL; + } + + /* n_buckets */ + if ((params->n_buckets == 0) || + (!rte_is_power_of_2(params->n_buckets))) { + RTE_LOG(ERR, TABLE, "%s: n_buckets invalid value\n", __func__); return -EINVAL; } /* f_hash */ if (params->f_hash == NULL) { - RTE_LOG(ERR, TABLE, - "%s: f_hash function pointer is NULL\n", __func__); + RTE_LOG(ERR, TABLE, "%s: f_hash function pointer is NULL\n", + __func__); return -EINVAL; } @@ -121,46 +159,67 @@ rte_table_hash_create_key16_lru(void *params, int socket_id, uint32_t entry_size) { - struct rte_table_hash_key16_lru_params *p = - (struct rte_table_hash_key16_lru_params *) params; + struct rte_table_hash_params *p = params; struct rte_table_hash *f; - uint32_t n_buckets, n_entries_per_bucket, - key_size, bucket_size_cl, total_size, i; + uint64_t bucket_size, total_size; + uint32_t n_buckets, i; /* Check input parameters */ - if ((check_params_create_lru(p) != 0) || + if ((check_params_create(p) != 0) || ((sizeof(struct rte_table_hash) % RTE_CACHE_LINE_SIZE) != 0) || - ((sizeof(struct rte_bucket_4_16) % RTE_CACHE_LINE_SIZE) != 0)) + ((sizeof(struct rte_bucket_4_16) % 64) != 0)) return NULL; - n_entries_per_bucket = 4; - key_size = 16; + + /* + * Table dimensioning + * + * Objective: Pick the number of buckets (n_buckets) so that there a chance + * to store n_keys keys in the table. + * + * Note: Since the buckets do not get extended, it is not possible to + * guarantee that n_keys keys can be stored in the table at any time. In the + * worst case scenario when all the n_keys fall into the same bucket, only + * a maximum of KEYS_PER_BUCKET keys will be stored in the table. This case + * defeats the purpose of the hash table. It indicates unsuitable f_hash or + * n_keys to n_buckets ratio. + * + * MIN(n_buckets) = (n_keys + KEYS_PER_BUCKET - 1) / KEYS_PER_BUCKET + */ + n_buckets = rte_align32pow2( + (p->n_keys + KEYS_PER_BUCKET - 1) / KEYS_PER_BUCKET); + n_buckets = RTE_MAX(n_buckets, p->n_buckets); /* Memory allocation */ - n_buckets = rte_align32pow2((p->n_entries + n_entries_per_bucket - 1) / - n_entries_per_bucket); - bucket_size_cl = (sizeof(struct rte_bucket_4_16) + n_entries_per_bucket - * entry_size + RTE_CACHE_LINE_SIZE - 1) / RTE_CACHE_LINE_SIZE; - total_size = sizeof(struct rte_table_hash) + n_buckets * - bucket_size_cl * RTE_CACHE_LINE_SIZE; + bucket_size = RTE_CACHE_LINE_ROUNDUP(sizeof(struct rte_bucket_4_16) + + KEYS_PER_BUCKET * entry_size); + total_size = sizeof(struct rte_table_hash) + n_buckets * bucket_size; - f = rte_zmalloc_socket("TABLE", total_size, RTE_CACHE_LINE_SIZE, socket_id); - if (f == NULL) { - RTE_LOG(ERR, TABLE, - "%s: Cannot allocate %u bytes for hash table\n", - __func__, total_size); + if (total_size > SIZE_MAX) { + RTE_LOG(ERR, TABLE, "%s: Cannot allocate %" PRIu64 " bytes " + "for hash table %s\n", + __func__, total_size, p->name); return NULL; } - RTE_LOG(INFO, TABLE, - "%s: Hash table memory footprint is %u bytes\n", - __func__, total_size); + + f = rte_zmalloc_socket(p->name, + (size_t)total_size, + RTE_CACHE_LINE_SIZE, + socket_id); + if (f == NULL) { + RTE_LOG(ERR, TABLE, "%s: Cannot allocate %" PRIu64 " bytes " + "for hash table %s\n", + __func__, total_size, p->name); + return NULL; + } + RTE_LOG(INFO, TABLE, "%s: Hash table %s memory footprint " + "is %" PRIu64 " bytes\n", + __func__, p->name, total_size); /* Memory initialization */ f->n_buckets = n_buckets; - f->n_entries_per_bucket = n_entries_per_bucket; - f->key_size = key_size; + f->key_size = KEY_SIZE; f->entry_size = entry_size; - f->bucket_size = bucket_size_cl * RTE_CACHE_LINE_SIZE; - f->signature_offset = p->signature_offset; + f->bucket_size = bucket_size; f->key_offset = p->key_offset; f->f_hash = p->f_hash; f->seed = p->seed; @@ -187,7 +246,7 @@ rte_table_hash_create_key16_lru(void *params, static int rte_table_hash_free_key16_lru(void *table) { - struct rte_table_hash *f = (struct rte_table_hash *) table; + struct rte_table_hash *f = table; /* Check input parameters */ if (f == NULL) { @@ -207,24 +266,24 @@ rte_table_hash_entry_add_key16_lru( int *key_found, void **entry_ptr) { - struct rte_table_hash *f = (struct rte_table_hash *) table; + struct rte_table_hash *f = table; struct rte_bucket_4_16 *bucket; uint64_t signature, pos; uint32_t bucket_index, i; - signature = f->f_hash(key, f->key_size, f->seed); + signature = f->f_hash(key, f->key_mask, f->key_size, f->seed); bucket_index = signature & (f->n_buckets - 1); bucket = (struct rte_bucket_4_16 *) - &f->memory[bucket_index * f->bucket_size]; + &f->memory[bucket_index * f->bucket_size]; signature |= RTE_BUCKET_ENTRY_VALID; /* Key is present in the bucket */ for (i = 0; i < 4; i++) { uint64_t bucket_signature = bucket->signature[i]; - uint8_t *bucket_key = (uint8_t *) bucket->key[i]; + uint8_t *bucket_key = (uint8_t *) &bucket->key[i]; if ((bucket_signature == signature) && - (memcmp(key, bucket_key, f->key_size) == 0)) { + (keycmp(bucket_key, key, f->key_mask) == 0)) { uint8_t *bucket_data = &bucket->data[i * f->entry_size]; memcpy(bucket_data, entry, f->entry_size); @@ -238,13 +297,13 @@ rte_table_hash_entry_add_key16_lru( /* Key is not present in the bucket */ for (i = 0; i < 4; i++) { uint64_t bucket_signature = bucket->signature[i]; - uint8_t *bucket_key = (uint8_t *) bucket->key[i]; + uint8_t *bucket_key = (uint8_t *) &bucket->key[i]; if (bucket_signature == 0) { uint8_t *bucket_data = &bucket->data[i * f->entry_size]; bucket->signature[i] = signature; - memcpy(bucket_key, key, f->key_size); + keycpy(bucket_key, key, f->key_mask); memcpy(bucket_data, entry, f->entry_size); lru_update(bucket, i); *key_found = 0; @@ -257,7 +316,7 @@ rte_table_hash_entry_add_key16_lru( /* Bucket full: replace LRU entry */ pos = lru_pos(bucket); bucket->signature[pos] = signature; - memcpy(bucket->key[pos], key, f->key_size); + keycpy(&bucket->key[pos], key, f->key_mask); memcpy(&bucket->data[pos * f->entry_size], entry, f->entry_size); lru_update(bucket, pos); *key_found = 0; @@ -273,24 +332,24 @@ rte_table_hash_entry_delete_key16_lru( int *key_found, void *entry) { - struct rte_table_hash *f = (struct rte_table_hash *) table; + struct rte_table_hash *f = table; struct rte_bucket_4_16 *bucket; uint64_t signature; uint32_t bucket_index, i; - signature = f->f_hash(key, f->key_size, f->seed); + signature = f->f_hash(key, f->key_mask, f->key_size, f->seed); bucket_index = signature & (f->n_buckets - 1); bucket = (struct rte_bucket_4_16 *) - &f->memory[bucket_index * f->bucket_size]; + &f->memory[bucket_index * f->bucket_size]; signature |= RTE_BUCKET_ENTRY_VALID; /* Key is present in the bucket */ for (i = 0; i < 4; i++) { uint64_t bucket_signature = bucket->signature[i]; - uint8_t *bucket_key = (uint8_t *) bucket->key[i]; + uint8_t *bucket_key = (uint8_t *) &bucket->key[i]; if ((bucket_signature == signature) && - (memcmp(key, bucket_key, f->key_size) == 0)) { + (keycmp(bucket_key, key, f->key_mask) == 0)) { uint8_t *bucket_data = &bucket->data[i * f->entry_size]; bucket->signature[i] = 0; @@ -306,81 +365,71 @@ rte_table_hash_entry_delete_key16_lru( return 0; } -static int -check_params_create_ext(struct rte_table_hash_key16_ext_params *params) { - /* n_entries */ - if (params->n_entries == 0) { - RTE_LOG(ERR, TABLE, "%s: n_entries is zero\n", __func__); - return -EINVAL; - } - - /* n_entries_ext */ - if (params->n_entries_ext == 0) { - RTE_LOG(ERR, TABLE, "%s: n_entries_ext is zero\n", __func__); - return -EINVAL; - } - - /* f_hash */ - if (params->f_hash == NULL) { - RTE_LOG(ERR, TABLE, - "%s: f_hash function pointer is NULL\n", __func__); - return -EINVAL; - } - - return 0; -} - static void * rte_table_hash_create_key16_ext(void *params, int socket_id, uint32_t entry_size) { - struct rte_table_hash_key16_ext_params *p = - (struct rte_table_hash_key16_ext_params *) params; + struct rte_table_hash_params *p = params; struct rte_table_hash *f; - uint32_t n_buckets, n_buckets_ext, n_entries_per_bucket, key_size, - bucket_size_cl, stack_size_cl, total_size, i; + uint64_t bucket_size, stack_size, total_size; + uint32_t n_buckets_ext, i; /* Check input parameters */ - if ((check_params_create_ext(p) != 0) || + if ((check_params_create(p) != 0) || ((sizeof(struct rte_table_hash) % RTE_CACHE_LINE_SIZE) != 0) || - ((sizeof(struct rte_bucket_4_16) % RTE_CACHE_LINE_SIZE) != 0)) + ((sizeof(struct rte_bucket_4_16) % 64) != 0)) return NULL; - n_entries_per_bucket = 4; - key_size = 16; + /* + * Table dimensioning + * + * Objective: Pick the number of bucket extensions (n_buckets_ext) so that + * it is guaranteed that n_keys keys can be stored in the table at any time. + * + * The worst case scenario takes place when all the n_keys keys fall into + * the same bucket. Actually, due to the KEYS_PER_BUCKET scheme, the worst + * case takes place when (n_keys - KEYS_PER_BUCKET + 1) keys fall into the + * same bucket, while the remaining (KEYS_PER_BUCKET - 1) keys each fall + * into a different bucket. This case defeats the purpose of the hash table. + * It indicates unsuitable f_hash or n_keys to n_buckets ratio. + * + * n_buckets_ext = n_keys / KEYS_PER_BUCKET + KEYS_PER_BUCKET - 1 + */ + n_buckets_ext = p->n_keys / KEYS_PER_BUCKET + KEYS_PER_BUCKET - 1; /* Memory allocation */ - n_buckets = rte_align32pow2((p->n_entries + n_entries_per_bucket - 1) / - n_entries_per_bucket); - n_buckets_ext = (p->n_entries_ext + n_entries_per_bucket - 1) / - n_entries_per_bucket; - bucket_size_cl = (sizeof(struct rte_bucket_4_16) + n_entries_per_bucket - * entry_size + RTE_CACHE_LINE_SIZE - 1) / RTE_CACHE_LINE_SIZE; - stack_size_cl = (n_buckets_ext * sizeof(uint32_t) + RTE_CACHE_LINE_SIZE - 1) - / RTE_CACHE_LINE_SIZE; + bucket_size = RTE_CACHE_LINE_ROUNDUP(sizeof(struct rte_bucket_4_16) + + KEYS_PER_BUCKET * entry_size); + stack_size = RTE_CACHE_LINE_ROUNDUP(n_buckets_ext * sizeof(uint32_t)); total_size = sizeof(struct rte_table_hash) + - ((n_buckets + n_buckets_ext) * bucket_size_cl + stack_size_cl) * - RTE_CACHE_LINE_SIZE; - - f = rte_zmalloc_socket("TABLE", total_size, RTE_CACHE_LINE_SIZE, socket_id); - if (f == NULL) { - RTE_LOG(ERR, TABLE, - "%s: Cannot allocate %u bytes for hash table\n", - __func__, total_size); + (p->n_buckets + n_buckets_ext) * bucket_size + stack_size; + if (total_size > SIZE_MAX) { + RTE_LOG(ERR, TABLE, "%s: Cannot allocate %" PRIu64 " bytes " + "for hash table %s\n", + __func__, total_size, p->name); return NULL; } - RTE_LOG(INFO, TABLE, - "%s: Hash table memory footprint is %u bytes\n", - __func__, total_size); + + f = rte_zmalloc_socket(p->name, + (size_t)total_size, + RTE_CACHE_LINE_SIZE, + socket_id); + if (f == NULL) { + RTE_LOG(ERR, TABLE, "%s: Cannot allocate %" PRIu64 " bytes " + "for hash table %s\n", + __func__, total_size, p->name); + return NULL; + } + RTE_LOG(INFO, TABLE, "%s: Hash table %s memory footprint " + "is %" PRIu64 " bytes\n", + __func__, p->name, total_size); /* Memory initialization */ - f->n_buckets = n_buckets; - f->n_entries_per_bucket = n_entries_per_bucket; - f->key_size = key_size; + f->n_buckets = p->n_buckets; + f->key_size = KEY_SIZE; f->entry_size = entry_size; - f->bucket_size = bucket_size_cl * RTE_CACHE_LINE_SIZE; - f->signature_offset = p->signature_offset; + f->bucket_size = bucket_size; f->key_offset = p->key_offset; f->f_hash = p->f_hash; f->seed = p->seed; @@ -388,10 +437,7 @@ rte_table_hash_create_key16_ext(void *params, f->n_buckets_ext = n_buckets_ext; f->stack_pos = n_buckets_ext; f->stack = (uint32_t *) - &f->memory[(n_buckets + n_buckets_ext) * f->bucket_size]; - - for (i = 0; i < n_buckets_ext; i++) - f->stack[i] = i; + &f->memory[(p->n_buckets + n_buckets_ext) * f->bucket_size]; if (p->key_mask != NULL) { f->key_mask[0] = (((uint64_t *)p->key_mask)[0]); @@ -401,13 +447,16 @@ rte_table_hash_create_key16_ext(void *params, f->key_mask[1] = 0xFFFFFFFFFFFFFFFFLLU; } + for (i = 0; i < n_buckets_ext; i++) + f->stack[i] = i; + return f; } static int rte_table_hash_free_key16_ext(void *table) { - struct rte_table_hash *f = (struct rte_table_hash *) table; + struct rte_table_hash *f = table; /* Check input parameters */ if (f == NULL) { @@ -427,25 +476,25 @@ rte_table_hash_entry_add_key16_ext( int *key_found, void **entry_ptr) { - struct rte_table_hash *f = (struct rte_table_hash *) table; + struct rte_table_hash *f = table; struct rte_bucket_4_16 *bucket0, *bucket, *bucket_prev; uint64_t signature; uint32_t bucket_index, i; - signature = f->f_hash(key, f->key_size, f->seed); + signature = f->f_hash(key, f->key_mask, f->key_size, f->seed); bucket_index = signature & (f->n_buckets - 1); bucket0 = (struct rte_bucket_4_16 *) - &f->memory[bucket_index * f->bucket_size]; + &f->memory[bucket_index * f->bucket_size]; signature |= RTE_BUCKET_ENTRY_VALID; /* Key is present in the bucket */ for (bucket = bucket0; bucket != NULL; bucket = bucket->next) for (i = 0; i < 4; i++) { uint64_t bucket_signature = bucket->signature[i]; - uint8_t *bucket_key = (uint8_t *) bucket->key[i]; + uint8_t *bucket_key = (uint8_t *) &bucket->key[i]; if ((bucket_signature == signature) && - (memcmp(key, bucket_key, f->key_size) == 0)) { + (keycmp(bucket_key, key, f->key_mask) == 0)) { uint8_t *bucket_data = &bucket->data[i * f->entry_size]; @@ -458,17 +507,17 @@ rte_table_hash_entry_add_key16_ext( /* Key is not present in the bucket */ for (bucket_prev = NULL, bucket = bucket0; bucket != NULL; - bucket_prev = bucket, bucket = bucket->next) + bucket_prev = bucket, bucket = bucket->next) for (i = 0; i < 4; i++) { uint64_t bucket_signature = bucket->signature[i]; - uint8_t *bucket_key = (uint8_t *) bucket->key[i]; + uint8_t *bucket_key = (uint8_t *) &bucket->key[i]; if (bucket_signature == 0) { uint8_t *bucket_data = &bucket->data[i * f->entry_size]; bucket->signature[i] = signature; - memcpy(bucket_key, key, f->key_size); + keycpy(bucket_key, key, f->key_mask); memcpy(bucket_data, entry, f->entry_size); *key_found = 0; *entry_ptr = (void *) bucket_data; @@ -487,7 +536,7 @@ rte_table_hash_entry_add_key16_ext( bucket_prev->next_valid = 1; bucket->signature[0] = signature; - memcpy(bucket->key[0], key, f->key_size); + keycpy(&bucket->key[0], key, f->key_mask); memcpy(&bucket->data[0], entry, f->entry_size); *key_found = 0; *entry_ptr = (void *) &bucket->data[0]; @@ -504,12 +553,12 @@ rte_table_hash_entry_delete_key16_ext( int *key_found, void *entry) { - struct rte_table_hash *f = (struct rte_table_hash *) table; + struct rte_table_hash *f = table; struct rte_bucket_4_16 *bucket0, *bucket, *bucket_prev; uint64_t signature; uint32_t bucket_index, i; - signature = f->f_hash(key, f->key_size, f->seed); + signature = f->f_hash(key, f->key_mask, f->key_size, f->seed); bucket_index = signature & (f->n_buckets - 1); bucket0 = (struct rte_bucket_4_16 *) &f->memory[bucket_index * f->bucket_size]; @@ -520,18 +569,17 @@ rte_table_hash_entry_delete_key16_ext( bucket_prev = bucket, bucket = bucket->next) for (i = 0; i < 4; i++) { uint64_t bucket_signature = bucket->signature[i]; - uint8_t *bucket_key = (uint8_t *) bucket->key[i]; + uint8_t *bucket_key = (uint8_t *) &bucket->key[i]; if ((bucket_signature == signature) && - (memcmp(key, bucket_key, f->key_size) == 0)) { + (keycmp(bucket_key, key, f->key_mask) == 0)) { uint8_t *bucket_data = &bucket->data[i * f->entry_size]; bucket->signature[i] = 0; *key_found = 1; if (entry) - memcpy(entry, bucket_data, - f->entry_size); + memcpy(entry, bucket_data, f->entry_size); if ((bucket->signature[0] == 0) && (bucket->signature[1] == 0) && @@ -558,26 +606,28 @@ rte_table_hash_entry_delete_key16_ext( return 0; } -#define lookup_key16_cmp(key_in, bucket, pos) \ +#define lookup_key16_cmp(key_in, bucket, pos, f) \ { \ - uint64_t xor[4][2], or[4], signature[4]; \ + uint64_t xor[4][2], or[4], signature[4], k[2]; \ \ + k[0] = key_in[0] & f->key_mask[0]; \ + k[1] = key_in[1] & f->key_mask[1]; \ signature[0] = (~bucket->signature[0]) & 1; \ signature[1] = (~bucket->signature[1]) & 1; \ signature[2] = (~bucket->signature[2]) & 1; \ signature[3] = (~bucket->signature[3]) & 1; \ \ - xor[0][0] = key_in[0] ^ bucket->key[0][0]; \ - xor[0][1] = key_in[1] ^ bucket->key[0][1]; \ + xor[0][0] = k[0] ^ bucket->key[0][0]; \ + xor[0][1] = k[1] ^ bucket->key[0][1]; \ \ - xor[1][0] = key_in[0] ^ bucket->key[1][0]; \ - xor[1][1] = key_in[1] ^ bucket->key[1][1]; \ + xor[1][0] = k[0] ^ bucket->key[1][0]; \ + xor[1][1] = k[1] ^ bucket->key[1][1]; \ \ - xor[2][0] = key_in[0] ^ bucket->key[2][0]; \ - xor[2][1] = key_in[1] ^ bucket->key[2][1]; \ + xor[2][0] = k[0] ^ bucket->key[2][0]; \ + xor[2][1] = k[1] ^ bucket->key[2][1]; \ \ - xor[3][0] = key_in[0] ^ bucket->key[3][0]; \ - xor[3][1] = key_in[1] ^ bucket->key[3][1]; \ + xor[3][0] = k[0] ^ bucket->key[3][0]; \ + xor[3][1] = k[1] ^ bucket->key[3][1]; \ \ or[0] = xor[0][0] | xor[0][1] | signature[0]; \ or[1] = xor[1][0] | xor[1][1] | signature[1]; \ @@ -609,31 +659,13 @@ rte_table_hash_entry_delete_key16_ext( } #define lookup1_stage1(mbuf1, bucket1, f) \ -{ \ - uint64_t signature; \ - uint32_t bucket_index; \ - \ - signature = RTE_MBUF_METADATA_UINT32(mbuf1, f->signature_offset);\ - bucket_index = signature & (f->n_buckets - 1); \ - bucket1 = (struct rte_bucket_4_16 *) \ - &f->memory[bucket_index * f->bucket_size]; \ - rte_prefetch0(bucket1); \ - rte_prefetch0((void *)(((uintptr_t) bucket1) + RTE_CACHE_LINE_SIZE));\ -} - -#define lookup1_stage1_dosig(mbuf1, bucket1, f) \ { \ uint64_t *key; \ uint64_t signature = 0; \ uint32_t bucket_index; \ - uint64_t hash_key_buffer[2]; \ \ key = RTE_MBUF_METADATA_UINT64_PTR(mbuf1, f->key_offset);\ - \ - hash_key_buffer[0] = key[0] & f->key_mask[0]; \ - hash_key_buffer[1] = key[1] & f->key_mask[1]; \ - signature = f->f_hash(hash_key_buffer, \ - RTE_TABLE_HASH_KEY_SIZE, f->seed); \ + signature = f->f_hash(key, f->key_mask, KEY_SIZE, f->seed); \ \ bucket_index = signature & (f->n_buckets - 1); \ bucket1 = (struct rte_bucket_4_16 *) \ @@ -648,14 +680,10 @@ rte_table_hash_entry_delete_key16_ext( void *a; \ uint64_t pkt_mask; \ uint64_t *key; \ - uint64_t hash_key_buffer[2]; \ uint32_t pos; \ \ key = RTE_MBUF_METADATA_UINT64_PTR(mbuf2, f->key_offset);\ - hash_key_buffer[0] = key[0] & f->key_mask[0]; \ - hash_key_buffer[1] = key[1] & f->key_mask[1]; \ - \ - lookup_key16_cmp(hash_key_buffer, bucket2, pos); \ + lookup_key16_cmp(key, bucket2, pos, f); \ \ pkt_mask = (bucket2->signature[pos] & 1LLU) << pkt2_index;\ pkts_mask_out |= pkt_mask; \ @@ -673,14 +701,10 @@ rte_table_hash_entry_delete_key16_ext( void *a; \ uint64_t pkt_mask, bucket_mask; \ uint64_t *key; \ - uint64_t hash_key_buffer[2]; \ uint32_t pos; \ \ key = RTE_MBUF_METADATA_UINT64_PTR(mbuf2, f->key_offset);\ - hash_key_buffer[0] = key[0] & f->key_mask[0]; \ - hash_key_buffer[1] = key[1] & f->key_mask[1]; \ - \ - lookup_key16_cmp(hash_key_buffer, bucket2, pos); \ + lookup_key16_cmp(key, bucket2, pos, f); \ \ pkt_mask = (bucket2->signature[pos] & 1LLU) << pkt2_index;\ pkts_mask_out |= pkt_mask; \ @@ -703,15 +727,11 @@ rte_table_hash_entry_delete_key16_ext( void *a; \ uint64_t pkt_mask, bucket_mask; \ uint64_t *key; \ - uint64_t hash_key_buffer[2]; \ uint32_t pos; \ \ bucket = buckets[pkt_index]; \ key = keys[pkt_index]; \ - hash_key_buffer[0] = key[0] & f->key_mask[0]; \ - hash_key_buffer[1] = key[1] & f->key_mask[1]; \ - \ - lookup_key16_cmp(hash_key_buffer, bucket, pos); \ + lookup_key16_cmp(key, bucket, pos, f); \ \ pkt_mask = (bucket->signature[pos] & 1LLU) << pkt_index;\ pkts_mask_out |= pkt_mask; \ @@ -774,37 +794,13 @@ rte_table_hash_entry_delete_key16_ext( } #define lookup2_stage1(mbuf10, mbuf11, bucket10, bucket11, f) \ -{ \ - uint64_t signature10, signature11; \ - uint32_t bucket10_index, bucket11_index; \ - \ - signature10 = RTE_MBUF_METADATA_UINT32(mbuf10, f->signature_offset);\ - bucket10_index = signature10 & (f->n_buckets - 1); \ - bucket10 = (struct rte_bucket_4_16 *) \ - &f->memory[bucket10_index * f->bucket_size]; \ - rte_prefetch0(bucket10); \ - rte_prefetch0((void *)(((uintptr_t) bucket10) + RTE_CACHE_LINE_SIZE));\ - \ - signature11 = RTE_MBUF_METADATA_UINT32(mbuf11, f->signature_offset);\ - bucket11_index = signature11 & (f->n_buckets - 1); \ - bucket11 = (struct rte_bucket_4_16 *) \ - &f->memory[bucket11_index * f->bucket_size]; \ - rte_prefetch0(bucket11); \ - rte_prefetch0((void *)(((uintptr_t) bucket11) + RTE_CACHE_LINE_SIZE));\ -} - -#define lookup2_stage1_dosig(mbuf10, mbuf11, bucket10, bucket11, f) \ { \ uint64_t *key10, *key11; \ - uint64_t hash_offset_buffer[2]; \ uint64_t signature10, signature11; \ uint32_t bucket10_index, bucket11_index; \ \ key10 = RTE_MBUF_METADATA_UINT64_PTR(mbuf10, f->key_offset);\ - hash_offset_buffer[0] = key10[0] & f->key_mask[0]; \ - hash_offset_buffer[1] = key10[1] & f->key_mask[1]; \ - signature10 = f->f_hash(hash_offset_buffer, \ - RTE_TABLE_HASH_KEY_SIZE, f->seed);\ + signature10 = f->f_hash(key10, f->key_mask, KEY_SIZE, f->seed);\ bucket10_index = signature10 & (f->n_buckets - 1); \ bucket10 = (struct rte_bucket_4_16 *) \ &f->memory[bucket10_index * f->bucket_size]; \ @@ -812,10 +808,7 @@ rte_table_hash_entry_delete_key16_ext( rte_prefetch0((void *)(((uintptr_t) bucket10) + RTE_CACHE_LINE_SIZE));\ \ key11 = RTE_MBUF_METADATA_UINT64_PTR(mbuf11, f->key_offset);\ - hash_offset_buffer[0] = key11[0] & f->key_mask[0]; \ - hash_offset_buffer[1] = key11[1] & f->key_mask[1]; \ - signature11 = f->f_hash(hash_offset_buffer, \ - RTE_TABLE_HASH_KEY_SIZE, f->seed);\ + signature11 = f->f_hash(key11, f->key_mask, KEY_SIZE, f->seed);\ bucket11_index = signature11 & (f->n_buckets - 1); \ bucket11 = (struct rte_bucket_4_16 *) \ &f->memory[bucket11_index * f->bucket_size]; \ @@ -829,19 +822,13 @@ rte_table_hash_entry_delete_key16_ext( void *a20, *a21; \ uint64_t pkt20_mask, pkt21_mask; \ uint64_t *key20, *key21; \ - uint64_t hash_key_buffer20[2]; \ - uint64_t hash_key_buffer21[2]; \ uint32_t pos20, pos21; \ \ key20 = RTE_MBUF_METADATA_UINT64_PTR(mbuf20, f->key_offset);\ key21 = RTE_MBUF_METADATA_UINT64_PTR(mbuf21, f->key_offset);\ - hash_key_buffer20[0] = key20[0] & f->key_mask[0]; \ - hash_key_buffer20[1] = key20[1] & f->key_mask[1]; \ - hash_key_buffer21[0] = key21[0] & f->key_mask[0]; \ - hash_key_buffer21[1] = key21[1] & f->key_mask[1]; \ \ - lookup_key16_cmp(hash_key_buffer20, bucket20, pos20); \ - lookup_key16_cmp(hash_key_buffer21, bucket21, pos21); \ + lookup_key16_cmp(key20, bucket20, pos20, f); \ + lookup_key16_cmp(key21, bucket21, pos21, f); \ \ pkt20_mask = (bucket20->signature[pos20] & 1LLU) << pkt20_index;\ pkt21_mask = (bucket21->signature[pos21] & 1LLU) << pkt21_index;\ @@ -864,19 +851,13 @@ rte_table_hash_entry_delete_key16_ext( void *a20, *a21; \ uint64_t pkt20_mask, pkt21_mask, bucket20_mask, bucket21_mask;\ uint64_t *key20, *key21; \ - uint64_t hash_key_buffer20[2]; \ - uint64_t hash_key_buffer21[2]; \ uint32_t pos20, pos21; \ \ key20 = RTE_MBUF_METADATA_UINT64_PTR(mbuf20, f->key_offset);\ key21 = RTE_MBUF_METADATA_UINT64_PTR(mbuf21, f->key_offset);\ - hash_key_buffer20[0] = key20[0] & f->key_mask[0]; \ - hash_key_buffer20[1] = key20[1] & f->key_mask[1]; \ - hash_key_buffer21[0] = key21[0] & f->key_mask[0]; \ - hash_key_buffer21[1] = key21[1] & f->key_mask[1]; \ \ - lookup_key16_cmp(hash_key_buffer20, bucket20, pos20); \ - lookup_key16_cmp(hash_key_buffer21, bucket21, pos21); \ + lookup_key16_cmp(key20, bucket20, pos20, f); \ + lookup_key16_cmp(key21, bucket21, pos21, f); \ \ pkt20_mask = (bucket20->signature[pos20] & 1LLU) << pkt20_index;\ pkt21_mask = (bucket21->signature[pos21] & 1LLU) << pkt21_index;\ @@ -916,6 +897,7 @@ rte_table_hash_lookup_key16_lru( uint64_t pkts_mask_out = 0; __rte_unused uint32_t n_pkts_in = __builtin_popcountll(pkts_mask); + RTE_TABLE_HASH_KEY16_STATS_PKTS_IN_ADD(f, n_pkts_in); /* Cannot run the pipeline with less than 5 packets */ @@ -931,135 +913,6 @@ rte_table_hash_lookup_key16_lru( pkts_mask_out, entries, f); } - *lookup_hit_mask = pkts_mask_out; - RTE_TABLE_HASH_KEY16_STATS_PKTS_LOOKUP_MISS(f, - n_pkts_in - __builtin_popcountll(pkts_mask_out)); - return 0; - } - - /* - * Pipeline fill - * - */ - /* Pipeline stage 0 */ - lookup2_stage0(pkt00_index, pkt01_index, mbuf00, mbuf01, pkts, - pkts_mask, f); - - /* Pipeline feed */ - mbuf10 = mbuf00; - mbuf11 = mbuf01; - pkt10_index = pkt00_index; - pkt11_index = pkt01_index; - - /* Pipeline stage 0 */ - lookup2_stage0(pkt00_index, pkt01_index, mbuf00, mbuf01, pkts, - pkts_mask, f); - - /* Pipeline stage 1 */ - lookup2_stage1(mbuf10, mbuf11, bucket10, bucket11, f); - - /* - * Pipeline run - * - */ - for ( ; pkts_mask; ) { - /* Pipeline feed */ - bucket20 = bucket10; - bucket21 = bucket11; - mbuf20 = mbuf10; - mbuf21 = mbuf11; - mbuf10 = mbuf00; - mbuf11 = mbuf01; - pkt20_index = pkt10_index; - pkt21_index = pkt11_index; - pkt10_index = pkt00_index; - pkt11_index = pkt01_index; - - /* Pipeline stage 0 */ - lookup2_stage0_with_odd_support(pkt00_index, pkt01_index, - mbuf00, mbuf01, pkts, pkts_mask, f); - - /* Pipeline stage 1 */ - lookup2_stage1(mbuf10, mbuf11, bucket10, bucket11, f); - - /* Pipeline stage 2 */ - lookup2_stage2_lru(pkt20_index, pkt21_index, mbuf20, mbuf21, - bucket20, bucket21, pkts_mask_out, entries, f); - } - - /* - * Pipeline flush - * - */ - /* Pipeline feed */ - bucket20 = bucket10; - bucket21 = bucket11; - mbuf20 = mbuf10; - mbuf21 = mbuf11; - mbuf10 = mbuf00; - mbuf11 = mbuf01; - pkt20_index = pkt10_index; - pkt21_index = pkt11_index; - pkt10_index = pkt00_index; - pkt11_index = pkt01_index; - - /* Pipeline stage 1 */ - lookup2_stage1(mbuf10, mbuf11, bucket10, bucket11, f); - - /* Pipeline stage 2 */ - lookup2_stage2_lru(pkt20_index, pkt21_index, mbuf20, mbuf21, - bucket20, bucket21, pkts_mask_out, entries, f); - - /* Pipeline feed */ - bucket20 = bucket10; - bucket21 = bucket11; - mbuf20 = mbuf10; - mbuf21 = mbuf11; - pkt20_index = pkt10_index; - pkt21_index = pkt11_index; - - /* Pipeline stage 2 */ - lookup2_stage2_lru(pkt20_index, pkt21_index, mbuf20, mbuf21, - bucket20, bucket21, pkts_mask_out, entries, f); - - *lookup_hit_mask = pkts_mask_out; - RTE_TABLE_HASH_KEY16_STATS_PKTS_LOOKUP_MISS(f, n_pkts_in - - __builtin_popcountll(pkts_mask_out)); - return 0; -} /* rte_table_hash_lookup_key16_lru() */ - -static int -rte_table_hash_lookup_key16_lru_dosig( - void *table, - struct rte_mbuf **pkts, - uint64_t pkts_mask, - uint64_t *lookup_hit_mask, - void **entries) -{ - struct rte_table_hash *f = (struct rte_table_hash *) table; - struct rte_bucket_4_16 *bucket10, *bucket11, *bucket20, *bucket21; - struct rte_mbuf *mbuf00, *mbuf01, *mbuf10, *mbuf11, *mbuf20, *mbuf21; - uint32_t pkt00_index, pkt01_index, pkt10_index; - uint32_t pkt11_index, pkt20_index, pkt21_index; - uint64_t pkts_mask_out = 0; - - __rte_unused uint32_t n_pkts_in = __builtin_popcountll(pkts_mask); - - RTE_TABLE_HASH_KEY16_STATS_PKTS_IN_ADD(f, n_pkts_in); - - /* Cannot run the pipeline with less than 5 packets */ - if (__builtin_popcountll(pkts_mask) < 5) { - for ( ; pkts_mask; ) { - struct rte_bucket_4_16 *bucket; - struct rte_mbuf *mbuf; - uint32_t pkt_index; - - lookup1_stage0(pkt_index, mbuf, pkts, pkts_mask, f); - lookup1_stage1_dosig(mbuf, bucket, f); - lookup1_stage2_lru(pkt_index, mbuf, bucket, - pkts_mask_out, entries, f); - } - *lookup_hit_mask = pkts_mask_out; RTE_TABLE_HASH_KEY16_STATS_PKTS_LOOKUP_MISS(f, n_pkts_in - __builtin_popcountll(pkts_mask_out)); @@ -1085,7 +938,7 @@ rte_table_hash_lookup_key16_lru_dosig( pkts_mask, f); /* Pipeline stage 1 */ - lookup2_stage1_dosig(mbuf10, mbuf11, bucket10, bucket11, f); + lookup2_stage1(mbuf10, mbuf11, bucket10, bucket11, f); /* * Pipeline run @@ -1109,7 +962,7 @@ rte_table_hash_lookup_key16_lru_dosig( mbuf00, mbuf01, pkts, pkts_mask, f); /* Pipeline stage 1 */ - lookup2_stage1_dosig(mbuf10, mbuf11, bucket10, bucket11, f); + lookup2_stage1(mbuf10, mbuf11, bucket10, bucket11, f); /* Pipeline stage 2 */ lookup2_stage2_lru(pkt20_index, pkt21_index, mbuf20, mbuf21, @@ -1133,7 +986,7 @@ rte_table_hash_lookup_key16_lru_dosig( pkt11_index = pkt01_index; /* Pipeline stage 1 */ - lookup2_stage1_dosig(mbuf10, mbuf11, bucket10, bucket11, f); + lookup2_stage1(mbuf10, mbuf11, bucket10, bucket11, f); /* Pipeline stage 2 */ lookup2_stage2_lru(pkt20_index, pkt21_index, mbuf20, mbuf21, @@ -1155,7 +1008,7 @@ rte_table_hash_lookup_key16_lru_dosig( RTE_TABLE_HASH_KEY16_STATS_PKTS_LOOKUP_MISS(f, n_pkts_in - __builtin_popcountll(pkts_mask_out)); return 0; -} /* rte_table_hash_lookup_key16_lru_dosig() */ +} /* lookup LRU */ static int rte_table_hash_lookup_key16_ext( @@ -1175,6 +1028,7 @@ rte_table_hash_lookup_key16_ext( uint64_t *keys[RTE_PORT_IN_BURST_SIZE_MAX]; __rte_unused uint32_t n_pkts_in = __builtin_popcountll(pkts_mask); + RTE_TABLE_HASH_KEY16_STATS_PKTS_IN_ADD(f, n_pkts_in); /* Cannot run the pipeline with less than 5 packets */ @@ -1306,164 +1160,12 @@ grind_next_buckets: RTE_TABLE_HASH_KEY16_STATS_PKTS_LOOKUP_MISS(f, n_pkts_in - __builtin_popcountll(pkts_mask_out)); return 0; -} /* rte_table_hash_lookup_key16_ext() */ - -static int -rte_table_hash_lookup_key16_ext_dosig( - void *table, - struct rte_mbuf **pkts, - uint64_t pkts_mask, - uint64_t *lookup_hit_mask, - void **entries) -{ - struct rte_table_hash *f = (struct rte_table_hash *) table; - struct rte_bucket_4_16 *bucket10, *bucket11, *bucket20, *bucket21; - struct rte_mbuf *mbuf00, *mbuf01, *mbuf10, *mbuf11, *mbuf20, *mbuf21; - uint32_t pkt00_index, pkt01_index, pkt10_index; - uint32_t pkt11_index, pkt20_index, pkt21_index; - uint64_t pkts_mask_out = 0, buckets_mask = 0; - struct rte_bucket_4_16 *buckets[RTE_PORT_IN_BURST_SIZE_MAX]; - uint64_t *keys[RTE_PORT_IN_BURST_SIZE_MAX]; - - __rte_unused uint32_t n_pkts_in = __builtin_popcountll(pkts_mask); - - RTE_TABLE_HASH_KEY16_STATS_PKTS_IN_ADD(f, n_pkts_in); - - /* Cannot run the pipeline with less than 5 packets */ - if (__builtin_popcountll(pkts_mask) < 5) { - for ( ; pkts_mask; ) { - struct rte_bucket_4_16 *bucket; - struct rte_mbuf *mbuf; - uint32_t pkt_index; - - lookup1_stage0(pkt_index, mbuf, pkts, pkts_mask, f); - lookup1_stage1_dosig(mbuf, bucket, f); - lookup1_stage2_ext(pkt_index, mbuf, bucket, - pkts_mask_out, entries, buckets_mask, - buckets, keys, f); - } - - goto grind_next_buckets; - } - - /* - * Pipeline fill - * - */ - /* Pipeline stage 0 */ - lookup2_stage0(pkt00_index, pkt01_index, mbuf00, mbuf01, pkts, - pkts_mask, f); - - /* Pipeline feed */ - mbuf10 = mbuf00; - mbuf11 = mbuf01; - pkt10_index = pkt00_index; - pkt11_index = pkt01_index; - - /* Pipeline stage 0 */ - lookup2_stage0(pkt00_index, pkt01_index, mbuf00, mbuf01, pkts, - pkts_mask, f); - - /* Pipeline stage 1 */ - lookup2_stage1_dosig(mbuf10, mbuf11, bucket10, bucket11, f); - - /* - * Pipeline run - * - */ - for ( ; pkts_mask; ) { - /* Pipeline feed */ - bucket20 = bucket10; - bucket21 = bucket11; - mbuf20 = mbuf10; - mbuf21 = mbuf11; - mbuf10 = mbuf00; - mbuf11 = mbuf01; - pkt20_index = pkt10_index; - pkt21_index = pkt11_index; - pkt10_index = pkt00_index; - pkt11_index = pkt01_index; - - /* Pipeline stage 0 */ - lookup2_stage0_with_odd_support(pkt00_index, pkt01_index, - mbuf00, mbuf01, pkts, pkts_mask, f); - - /* Pipeline stage 1 */ - lookup2_stage1_dosig(mbuf10, mbuf11, bucket10, bucket11, f); - - /* Pipeline stage 2 */ - lookup2_stage2_ext(pkt20_index, pkt21_index, mbuf20, mbuf21, - bucket20, bucket21, pkts_mask_out, entries, - buckets_mask, buckets, keys, f); - } - - /* - * Pipeline flush - * - */ - /* Pipeline feed */ - bucket20 = bucket10; - bucket21 = bucket11; - mbuf20 = mbuf10; - mbuf21 = mbuf11; - mbuf10 = mbuf00; - mbuf11 = mbuf01; - pkt20_index = pkt10_index; - pkt21_index = pkt11_index; - pkt10_index = pkt00_index; - pkt11_index = pkt01_index; - - /* Pipeline stage 1 */ - lookup2_stage1_dosig(mbuf10, mbuf11, bucket10, bucket11, f); - - /* Pipeline stage 2 */ - lookup2_stage2_ext(pkt20_index, pkt21_index, mbuf20, mbuf21, - bucket20, bucket21, pkts_mask_out, entries, - buckets_mask, buckets, keys, f); - - /* Pipeline feed */ - bucket20 = bucket10; - bucket21 = bucket11; - mbuf20 = mbuf10; - mbuf21 = mbuf11; - pkt20_index = pkt10_index; - pkt21_index = pkt11_index; - - /* Pipeline stage 2 */ - lookup2_stage2_ext(pkt20_index, pkt21_index, mbuf20, mbuf21, - bucket20, bucket21, pkts_mask_out, entries, - buckets_mask, buckets, keys, f); - -grind_next_buckets: - /* Grind next buckets */ - for ( ; buckets_mask; ) { - uint64_t buckets_mask_next = 0; - - for ( ; buckets_mask; ) { - uint64_t pkt_mask; - uint32_t pkt_index; - - pkt_index = __builtin_ctzll(buckets_mask); - pkt_mask = 1LLU << pkt_index; - buckets_mask &= ~pkt_mask; - - lookup_grinder(pkt_index, buckets, keys, pkts_mask_out, - entries, buckets_mask_next, f); - } - - buckets_mask = buckets_mask_next; - } - - *lookup_hit_mask = pkts_mask_out; - RTE_TABLE_HASH_KEY16_STATS_PKTS_LOOKUP_MISS(f, n_pkts_in - - __builtin_popcountll(pkts_mask_out)); - return 0; -} /* rte_table_hash_lookup_key16_ext_dosig() */ +} /* lookup EXT */ static int rte_table_hash_key16_stats_read(void *table, struct rte_table_stats *stats, int clear) { - struct rte_table_hash *t = (struct rte_table_hash *) table; + struct rte_table_hash *t = table; if (stats != NULL) memcpy(stats, &t->stats, sizeof(t->stats)); @@ -1485,15 +1187,6 @@ struct rte_table_ops rte_table_hash_key16_lru_ops = { .f_stats = rte_table_hash_key16_stats_read, }; -struct rte_table_ops rte_table_hash_key16_lru_dosig_ops = { - .f_create = rte_table_hash_create_key16_lru, - .f_free = rte_table_hash_free_key16_lru, - .f_add = rte_table_hash_entry_add_key16_lru, - .f_delete = rte_table_hash_entry_delete_key16_lru, - .f_lookup = rte_table_hash_lookup_key16_lru_dosig, - .f_stats = rte_table_hash_key16_stats_read, -}; - struct rte_table_ops rte_table_hash_key16_ext_ops = { .f_create = rte_table_hash_create_key16_ext, .f_free = rte_table_hash_free_key16_ext, @@ -1504,12 +1197,3 @@ struct rte_table_ops rte_table_hash_key16_ext_ops = { .f_lookup = rte_table_hash_lookup_key16_ext, .f_stats = rte_table_hash_key16_stats_read, }; - -struct rte_table_ops rte_table_hash_key16_ext_dosig_ops = { - .f_create = rte_table_hash_create_key16_ext, - .f_free = rte_table_hash_free_key16_ext, - .f_add = rte_table_hash_entry_add_key16_ext, - .f_delete = rte_table_hash_entry_delete_key16_ext, - .f_lookup = rte_table_hash_lookup_key16_ext_dosig, - .f_stats = rte_table_hash_key16_stats_read, -}; diff --git a/dpdk/lib/librte_table/rte_table_hash_key32.c b/dpdk/lib/librte_table/rte_table_hash_key32.c index a7aba492..d4364d62 100644 --- a/dpdk/lib/librte_table/rte_table_hash_key32.c +++ b/dpdk/lib/librte_table/rte_table_hash_key32.c @@ -1,34 +1,34 @@ /*- - * BSD LICENSE + * BSD LICENSE * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. + * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. + * All rights reserved. * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include #include @@ -42,7 +42,9 @@ #include "rte_table_hash.h" #include "rte_lru.h" -#define RTE_TABLE_HASH_KEY_SIZE 32 +#define KEY_SIZE 32 + +#define KEYS_PER_BUCKET 4 #define RTE_BUCKET_ENTRY_VALID 0x1LLU @@ -79,12 +81,11 @@ struct rte_table_hash { /* Input parameters */ uint32_t n_buckets; - uint32_t n_entries_per_bucket; uint32_t key_size; uint32_t entry_size; uint32_t bucket_size; - uint32_t signature_offset; uint32_t key_offset; + uint64_t key_mask[4]; rte_table_hash_op_hash f_hash; uint64_t seed; @@ -98,10 +99,52 @@ struct rte_table_hash { }; static int -check_params_create_lru(struct rte_table_hash_key32_lru_params *params) { - /* n_entries */ - if (params->n_entries == 0) { - RTE_LOG(ERR, TABLE, "%s: n_entries is zero\n", __func__); +keycmp(void *a, void *b, void *b_mask) +{ + uint64_t *a64 = a, *b64 = b, *b_mask64 = b_mask; + + return (a64[0] != (b64[0] & b_mask64[0])) || + (a64[1] != (b64[1] & b_mask64[1])) || + (a64[2] != (b64[2] & b_mask64[2])) || + (a64[3] != (b64[3] & b_mask64[3])); +} + +static void +keycpy(void *dst, void *src, void *src_mask) +{ + uint64_t *dst64 = dst, *src64 = src, *src_mask64 = src_mask; + + dst64[0] = src64[0] & src_mask64[0]; + dst64[1] = src64[1] & src_mask64[1]; + dst64[2] = src64[2] & src_mask64[2]; + dst64[3] = src64[3] & src_mask64[3]; +} + +static int +check_params_create(struct rte_table_hash_params *params) +{ + /* name */ + if (params->name == NULL) { + RTE_LOG(ERR, TABLE, "%s: name invalid value\n", __func__); + return -EINVAL; + } + + /* key_size */ + if (params->key_size != KEY_SIZE) { + RTE_LOG(ERR, TABLE, "%s: key_size invalid value\n", __func__); + return -EINVAL; + } + + /* n_keys */ + if (params->n_keys == 0) { + RTE_LOG(ERR, TABLE, "%s: n_keys is zero\n", __func__); + return -EINVAL; + } + + /* n_buckets */ + if ((params->n_buckets == 0) || + (!rte_is_power_of_2(params->n_buckets))) { + RTE_LOG(ERR, TABLE, "%s: n_buckets invalid value\n", __func__); return -EINVAL; } @@ -120,51 +163,83 @@ rte_table_hash_create_key32_lru(void *params, int socket_id, uint32_t entry_size) { - struct rte_table_hash_key32_lru_params *p = - (struct rte_table_hash_key32_lru_params *) params; + struct rte_table_hash_params *p = params; struct rte_table_hash *f; - uint32_t n_buckets, n_entries_per_bucket, key_size, bucket_size_cl; - uint32_t total_size, i; + uint64_t bucket_size, total_size; + uint32_t n_buckets, i; /* Check input parameters */ - if ((check_params_create_lru(p) != 0) || + if ((check_params_create(p) != 0) || ((sizeof(struct rte_table_hash) % RTE_CACHE_LINE_SIZE) != 0) || - ((sizeof(struct rte_bucket_4_32) % RTE_CACHE_LINE_SIZE) != 0)) { + ((sizeof(struct rte_bucket_4_32) % 64) != 0)) return NULL; - } - n_entries_per_bucket = 4; - key_size = 32; + + /* + * Table dimensioning + * + * Objective: Pick the number of buckets (n_buckets) so that there a chance + * to store n_keys keys in the table. + * + * Note: Since the buckets do not get extended, it is not possible to + * guarantee that n_keys keys can be stored in the table at any time. In the + * worst case scenario when all the n_keys fall into the same bucket, only + * a maximum of KEYS_PER_BUCKET keys will be stored in the table. This case + * defeats the purpose of the hash table. It indicates unsuitable f_hash or + * n_keys to n_buckets ratio. + * + * MIN(n_buckets) = (n_keys + KEYS_PER_BUCKET - 1) / KEYS_PER_BUCKET + */ + n_buckets = rte_align32pow2( + (p->n_keys + KEYS_PER_BUCKET - 1) / KEYS_PER_BUCKET); + n_buckets = RTE_MAX(n_buckets, p->n_buckets); /* Memory allocation */ - n_buckets = rte_align32pow2((p->n_entries + n_entries_per_bucket - 1) / - n_entries_per_bucket); - bucket_size_cl = (sizeof(struct rte_bucket_4_32) + n_entries_per_bucket - * entry_size + RTE_CACHE_LINE_SIZE - 1) / RTE_CACHE_LINE_SIZE; - total_size = sizeof(struct rte_table_hash) + n_buckets * - bucket_size_cl * RTE_CACHE_LINE_SIZE; + bucket_size = RTE_CACHE_LINE_ROUNDUP(sizeof(struct rte_bucket_4_32) + + KEYS_PER_BUCKET * entry_size); + total_size = sizeof(struct rte_table_hash) + n_buckets * bucket_size; + if (total_size > SIZE_MAX) { + RTE_LOG(ERR, TABLE, "%s: Cannot allocate %" PRIu64 " bytes " + "for hash table %s\n", + __func__, total_size, p->name); + return NULL; + } - f = rte_zmalloc_socket("TABLE", total_size, RTE_CACHE_LINE_SIZE, socket_id); + f = rte_zmalloc_socket(p->name, + (size_t)total_size, + RTE_CACHE_LINE_SIZE, + socket_id); if (f == NULL) { - RTE_LOG(ERR, TABLE, - "%s: Cannot allocate %u bytes for hash table\n", - __func__, total_size); + RTE_LOG(ERR, TABLE, "%s: Cannot allocate %" PRIu64 " bytes " + "for hash table %s\n", + __func__, total_size, p->name); return NULL; } RTE_LOG(INFO, TABLE, - "%s: Hash table memory footprint is %u bytes\n", __func__, - total_size); + "%s: Hash table %s memory footprint " + "is %" PRIu64 " bytes\n", + __func__, p->name, total_size); /* Memory initialization */ f->n_buckets = n_buckets; - f->n_entries_per_bucket = n_entries_per_bucket; - f->key_size = key_size; + f->key_size = KEY_SIZE; f->entry_size = entry_size; - f->bucket_size = bucket_size_cl * RTE_CACHE_LINE_SIZE; - f->signature_offset = p->signature_offset; + f->bucket_size = bucket_size; f->key_offset = p->key_offset; f->f_hash = p->f_hash; f->seed = p->seed; + if (p->key_mask != NULL) { + f->key_mask[0] = ((uint64_t *)p->key_mask)[0]; + f->key_mask[1] = ((uint64_t *)p->key_mask)[1]; + f->key_mask[2] = ((uint64_t *)p->key_mask)[2]; + f->key_mask[3] = ((uint64_t *)p->key_mask)[3]; + } else { + f->key_mask[0] = 0xFFFFFFFFFFFFFFFFLLU; + f->key_mask[1] = 0xFFFFFFFFFFFFFFFFLLU; + f->key_mask[2] = 0xFFFFFFFFFFFFFFFFLLU; + f->key_mask[3] = 0xFFFFFFFFFFFFFFFFLLU; + } + for (i = 0; i < n_buckets; i++) { struct rte_bucket_4_32 *bucket; @@ -179,7 +254,7 @@ rte_table_hash_create_key32_lru(void *params, static int rte_table_hash_free_key32_lru(void *table) { - struct rte_table_hash *f = (struct rte_table_hash *) table; + struct rte_table_hash *f = table; /* Check input parameters */ if (f == NULL) { @@ -199,12 +274,12 @@ rte_table_hash_entry_add_key32_lru( int *key_found, void **entry_ptr) { - struct rte_table_hash *f = (struct rte_table_hash *) table; + struct rte_table_hash *f = table; struct rte_bucket_4_32 *bucket; uint64_t signature, pos; uint32_t bucket_index, i; - signature = f->f_hash(key, f->key_size, f->seed); + signature = f->f_hash(key, f->key_mask, f->key_size, f->seed); bucket_index = signature & (f->n_buckets - 1); bucket = (struct rte_bucket_4_32 *) &f->memory[bucket_index * f->bucket_size]; @@ -213,10 +288,10 @@ rte_table_hash_entry_add_key32_lru( /* Key is present in the bucket */ for (i = 0; i < 4; i++) { uint64_t bucket_signature = bucket->signature[i]; - uint8_t *bucket_key = (uint8_t *) bucket->key[i]; + uint8_t *bucket_key = (uint8_t *) &bucket->key[i]; if ((bucket_signature == signature) && - (memcmp(key, bucket_key, f->key_size) == 0)) { + (keycmp(bucket_key, key, f->key_mask) == 0)) { uint8_t *bucket_data = &bucket->data[i * f->entry_size]; memcpy(bucket_data, entry, f->entry_size); @@ -230,13 +305,13 @@ rte_table_hash_entry_add_key32_lru( /* Key is not present in the bucket */ for (i = 0; i < 4; i++) { uint64_t bucket_signature = bucket->signature[i]; - uint8_t *bucket_key = (uint8_t *) bucket->key[i]; + uint8_t *bucket_key = (uint8_t *) &bucket->key[i]; if (bucket_signature == 0) { uint8_t *bucket_data = &bucket->data[i * f->entry_size]; bucket->signature[i] = signature; - memcpy(bucket_key, key, f->key_size); + keycpy(bucket_key, key, f->key_mask); memcpy(bucket_data, entry, f->entry_size); lru_update(bucket, i); *key_found = 0; @@ -249,10 +324,10 @@ rte_table_hash_entry_add_key32_lru( /* Bucket full: replace LRU entry */ pos = lru_pos(bucket); bucket->signature[pos] = signature; - memcpy(bucket->key[pos], key, f->key_size); + keycpy(&bucket->key[pos], key, f->key_mask); memcpy(&bucket->data[pos * f->entry_size], entry, f->entry_size); lru_update(bucket, pos); - *key_found = 0; + *key_found = 0; *entry_ptr = (void *) &bucket->data[pos * f->entry_size]; return 0; @@ -265,12 +340,12 @@ rte_table_hash_entry_delete_key32_lru( int *key_found, void *entry) { - struct rte_table_hash *f = (struct rte_table_hash *) table; + struct rte_table_hash *f = table; struct rte_bucket_4_32 *bucket; uint64_t signature; uint32_t bucket_index, i; - signature = f->f_hash(key, f->key_size, f->seed); + signature = f->f_hash(key, f->key_mask, f->key_size, f->seed); bucket_index = signature & (f->n_buckets - 1); bucket = (struct rte_bucket_4_32 *) &f->memory[bucket_index * f->bucket_size]; @@ -279,10 +354,10 @@ rte_table_hash_entry_delete_key32_lru( /* Key is present in the bucket */ for (i = 0; i < 4; i++) { uint64_t bucket_signature = bucket->signature[i]; - uint8_t *bucket_key = (uint8_t *) bucket->key[i]; + uint8_t *bucket_key = (uint8_t *) &bucket->key[i]; if ((bucket_signature == signature) && - (memcmp(key, bucket_key, f->key_size) == 0)) { + (keycmp(bucket_key, key, f->key_mask) == 0)) { uint8_t *bucket_data = &bucket->data[i * f->entry_size]; bucket->signature[i] = 0; @@ -299,81 +374,72 @@ rte_table_hash_entry_delete_key32_lru( return 0; } -static int -check_params_create_ext(struct rte_table_hash_key32_ext_params *params) { - /* n_entries */ - if (params->n_entries == 0) { - RTE_LOG(ERR, TABLE, "%s: n_entries is zero\n", __func__); - return -EINVAL; - } - - /* n_entries_ext */ - if (params->n_entries_ext == 0) { - RTE_LOG(ERR, TABLE, "%s: n_entries_ext is zero\n", __func__); - return -EINVAL; - } - - /* f_hash */ - if (params->f_hash == NULL) { - RTE_LOG(ERR, TABLE, "%s: f_hash function pointer is NULL\n", - __func__); - return -EINVAL; - } - - return 0; -} - static void * rte_table_hash_create_key32_ext(void *params, int socket_id, uint32_t entry_size) { - struct rte_table_hash_key32_ext_params *p = - (struct rte_table_hash_key32_ext_params *) params; + struct rte_table_hash_params *p = params; struct rte_table_hash *f; - uint32_t n_buckets, n_buckets_ext, n_entries_per_bucket; - uint32_t key_size, bucket_size_cl, stack_size_cl, total_size, i; + uint64_t bucket_size, stack_size, total_size; + uint32_t n_buckets_ext, i; /* Check input parameters */ - if ((check_params_create_ext(p) != 0) || + if ((check_params_create(p) != 0) || ((sizeof(struct rte_table_hash) % RTE_CACHE_LINE_SIZE) != 0) || - ((sizeof(struct rte_bucket_4_32) % RTE_CACHE_LINE_SIZE) != 0)) + ((sizeof(struct rte_bucket_4_32) % 64) != 0)) return NULL; - n_entries_per_bucket = 4; - key_size = 32; + /* + * Table dimensioning + * + * Objective: Pick the number of bucket extensions (n_buckets_ext) so that + * it is guaranteed that n_keys keys can be stored in the table at any time. + * + * The worst case scenario takes place when all the n_keys keys fall into + * the same bucket. Actually, due to the KEYS_PER_BUCKET scheme, the worst + * case takes place when (n_keys - KEYS_PER_BUCKET + 1) keys fall into the + * same bucket, while the remaining (KEYS_PER_BUCKET - 1) keys each fall + * into a different bucket. This case defeats the purpose of the hash table. + * It indicates unsuitable f_hash or n_keys to n_buckets ratio. + * + * n_buckets_ext = n_keys / KEYS_PER_BUCKET + KEYS_PER_BUCKET - 1 + */ + n_buckets_ext = p->n_keys / KEYS_PER_BUCKET + KEYS_PER_BUCKET - 1; /* Memory allocation */ - n_buckets = rte_align32pow2((p->n_entries + n_entries_per_bucket - 1) / - n_entries_per_bucket); - n_buckets_ext = (p->n_entries_ext + n_entries_per_bucket - 1) / - n_entries_per_bucket; - bucket_size_cl = (sizeof(struct rte_bucket_4_32) + n_entries_per_bucket - * entry_size + RTE_CACHE_LINE_SIZE - 1) / RTE_CACHE_LINE_SIZE; - stack_size_cl = (n_buckets_ext * sizeof(uint32_t) + RTE_CACHE_LINE_SIZE - 1) - / RTE_CACHE_LINE_SIZE; + bucket_size = RTE_CACHE_LINE_ROUNDUP(sizeof(struct rte_bucket_4_32) + + KEYS_PER_BUCKET * entry_size); + stack_size = RTE_CACHE_LINE_ROUNDUP(n_buckets_ext * sizeof(uint32_t)); total_size = sizeof(struct rte_table_hash) + - ((n_buckets + n_buckets_ext) * bucket_size_cl + stack_size_cl) * - RTE_CACHE_LINE_SIZE; + (p->n_buckets + n_buckets_ext) * bucket_size + stack_size; + if (total_size > SIZE_MAX) { + RTE_LOG(ERR, TABLE, "%s: Cannot allocate %" PRIu64 " bytes " + "for hash table %s\n", + __func__, total_size, p->name); + return NULL; + } - f = rte_zmalloc_socket("TABLE", total_size, RTE_CACHE_LINE_SIZE, socket_id); + f = rte_zmalloc_socket(p->name, + (size_t)total_size, + RTE_CACHE_LINE_SIZE, + socket_id); if (f == NULL) { - RTE_LOG(ERR, TABLE, - "%s: Cannot allocate %u bytes for hash table\n", - __func__, total_size); + RTE_LOG(ERR, TABLE, "%s: Cannot allocate %" PRIu64 " bytes " + "for hash table %s\n", + __func__, total_size, p->name); return NULL; } RTE_LOG(INFO, TABLE, - "%s: Hash table memory footprint is %u bytes\n", __func__, - total_size); + "%s: Hash table %s memory footprint " + "is %" PRIu64" bytes\n", + __func__, p->name, total_size); /* Memory initialization */ - f->n_buckets = n_buckets; - f->n_entries_per_bucket = n_entries_per_bucket; - f->key_size = key_size; + f->n_buckets = p->n_buckets; + f->key_size = KEY_SIZE; f->entry_size = entry_size; - f->bucket_size = bucket_size_cl * RTE_CACHE_LINE_SIZE; - f->signature_offset = p->signature_offset; + f->bucket_size = bucket_size; f->key_offset = p->key_offset; f->f_hash = p->f_hash; f->seed = p->seed; @@ -381,7 +447,19 @@ rte_table_hash_create_key32_ext(void *params, f->n_buckets_ext = n_buckets_ext; f->stack_pos = n_buckets_ext; f->stack = (uint32_t *) - &f->memory[(n_buckets + n_buckets_ext) * f->bucket_size]; + &f->memory[(p->n_buckets + n_buckets_ext) * f->bucket_size]; + + if (p->key_mask != NULL) { + f->key_mask[0] = (((uint64_t *)p->key_mask)[0]); + f->key_mask[1] = (((uint64_t *)p->key_mask)[1]); + f->key_mask[2] = (((uint64_t *)p->key_mask)[2]); + f->key_mask[3] = (((uint64_t *)p->key_mask)[3]); + } else { + f->key_mask[0] = 0xFFFFFFFFFFFFFFFFLLU; + f->key_mask[1] = 0xFFFFFFFFFFFFFFFFLLU; + f->key_mask[2] = 0xFFFFFFFFFFFFFFFFLLU; + f->key_mask[3] = 0xFFFFFFFFFFFFFFFFLLU; + } for (i = 0; i < n_buckets_ext; i++) f->stack[i] = i; @@ -392,7 +470,7 @@ rte_table_hash_create_key32_ext(void *params, static int rte_table_hash_free_key32_ext(void *table) { - struct rte_table_hash *f = (struct rte_table_hash *) table; + struct rte_table_hash *f = table; /* Check input parameters */ if (f == NULL) { @@ -412,12 +490,12 @@ rte_table_hash_entry_add_key32_ext( int *key_found, void **entry_ptr) { - struct rte_table_hash *f = (struct rte_table_hash *) table; + struct rte_table_hash *f = table; struct rte_bucket_4_32 *bucket0, *bucket, *bucket_prev; uint64_t signature; uint32_t bucket_index, i; - signature = f->f_hash(key, f->key_size, f->seed); + signature = f->f_hash(key, f->key_mask, f->key_size, f->seed); bucket_index = signature & (f->n_buckets - 1); bucket0 = (struct rte_bucket_4_32 *) &f->memory[bucket_index * f->bucket_size]; @@ -427,10 +505,10 @@ rte_table_hash_entry_add_key32_ext( for (bucket = bucket0; bucket != NULL; bucket = bucket->next) { for (i = 0; i < 4; i++) { uint64_t bucket_signature = bucket->signature[i]; - uint8_t *bucket_key = (uint8_t *) bucket->key[i]; + uint8_t *bucket_key = (uint8_t *) &bucket->key[i]; if ((bucket_signature == signature) && - (memcmp(key, bucket_key, f->key_size) == 0)) { + (keycmp(bucket_key, key, f->key_mask) == 0)) { uint8_t *bucket_data = &bucket->data[i * f->entry_size]; @@ -448,14 +526,14 @@ rte_table_hash_entry_add_key32_ext( bucket_prev = bucket, bucket = bucket->next) for (i = 0; i < 4; i++) { uint64_t bucket_signature = bucket->signature[i]; - uint8_t *bucket_key = (uint8_t *) bucket->key[i]; + uint8_t *bucket_key = (uint8_t *) &bucket->key[i]; if (bucket_signature == 0) { uint8_t *bucket_data = &bucket->data[i * f->entry_size]; bucket->signature[i] = signature; - memcpy(bucket_key, key, f->key_size); + keycpy(bucket_key, key, f->key_mask); memcpy(bucket_data, entry, f->entry_size); *key_found = 0; *entry_ptr = (void *) bucket_data; @@ -475,7 +553,7 @@ rte_table_hash_entry_add_key32_ext( bucket_prev->next_valid = 1; bucket->signature[0] = signature; - memcpy(bucket->key[0], key, f->key_size); + keycpy(&bucket->key[0], key, f->key_mask); memcpy(&bucket->data[0], entry, f->entry_size); *key_found = 0; *entry_ptr = (void *) &bucket->data[0]; @@ -492,12 +570,12 @@ rte_table_hash_entry_delete_key32_ext( int *key_found, void *entry) { - struct rte_table_hash *f = (struct rte_table_hash *) table; + struct rte_table_hash *f = table; struct rte_bucket_4_32 *bucket0, *bucket, *bucket_prev; uint64_t signature; uint32_t bucket_index, i; - signature = f->f_hash(key, f->key_size, f->seed); + signature = f->f_hash(key, f->key_mask, f->key_size, f->seed); bucket_index = signature & (f->n_buckets - 1); bucket0 = (struct rte_bucket_4_32 *) &f->memory[bucket_index * f->bucket_size]; @@ -508,24 +586,23 @@ rte_table_hash_entry_delete_key32_ext( bucket_prev = bucket, bucket = bucket->next) for (i = 0; i < 4; i++) { uint64_t bucket_signature = bucket->signature[i]; - uint8_t *bucket_key = (uint8_t *) bucket->key[i]; + uint8_t *bucket_key = (uint8_t *) &bucket->key[i]; if ((bucket_signature == signature) && - (memcmp(key, bucket_key, f->key_size) == 0)) { + (keycmp(bucket_key, key, f->key_mask) == 0)) { uint8_t *bucket_data = &bucket->data[i * f->entry_size]; bucket->signature[i] = 0; *key_found = 1; if (entry) - memcpy(entry, bucket_data, - f->entry_size); + memcpy(entry, bucket_data, f->entry_size); if ((bucket->signature[0] == 0) && - (bucket->signature[1] == 0) && - (bucket->signature[2] == 0) && - (bucket->signature[3] == 0) && - (bucket_prev != NULL)) { + (bucket->signature[1] == 0) && + (bucket->signature[2] == 0) && + (bucket->signature[3] == 0) && + (bucket_prev != NULL)) { bucket_prev->next = bucket->next; bucket_prev->next_valid = bucket->next_valid; @@ -546,34 +623,39 @@ rte_table_hash_entry_delete_key32_ext( return 0; } -#define lookup_key32_cmp(key_in, bucket, pos) \ +#define lookup_key32_cmp(key_in, bucket, pos, f) \ { \ - uint64_t xor[4][4], or[4], signature[4]; \ + uint64_t xor[4][4], or[4], signature[4], k[4]; \ + \ + k[0] = key_in[0] & f->key_mask[0]; \ + k[1] = key_in[1] & f->key_mask[1]; \ + k[2] = key_in[2] & f->key_mask[2]; \ + k[3] = key_in[3] & f->key_mask[3]; \ \ signature[0] = ((~bucket->signature[0]) & 1); \ signature[1] = ((~bucket->signature[1]) & 1); \ signature[2] = ((~bucket->signature[2]) & 1); \ signature[3] = ((~bucket->signature[3]) & 1); \ \ - xor[0][0] = key_in[0] ^ bucket->key[0][0]; \ - xor[0][1] = key_in[1] ^ bucket->key[0][1]; \ - xor[0][2] = key_in[2] ^ bucket->key[0][2]; \ - xor[0][3] = key_in[3] ^ bucket->key[0][3]; \ + xor[0][0] = k[0] ^ bucket->key[0][0]; \ + xor[0][1] = k[1] ^ bucket->key[0][1]; \ + xor[0][2] = k[2] ^ bucket->key[0][2]; \ + xor[0][3] = k[3] ^ bucket->key[0][3]; \ \ - xor[1][0] = key_in[0] ^ bucket->key[1][0]; \ - xor[1][1] = key_in[1] ^ bucket->key[1][1]; \ - xor[1][2] = key_in[2] ^ bucket->key[1][2]; \ - xor[1][3] = key_in[3] ^ bucket->key[1][3]; \ + xor[1][0] = k[0] ^ bucket->key[1][0]; \ + xor[1][1] = k[1] ^ bucket->key[1][1]; \ + xor[1][2] = k[2] ^ bucket->key[1][2]; \ + xor[1][3] = k[3] ^ bucket->key[1][3]; \ \ - xor[2][0] = key_in[0] ^ bucket->key[2][0]; \ - xor[2][1] = key_in[1] ^ bucket->key[2][1]; \ - xor[2][2] = key_in[2] ^ bucket->key[2][2]; \ - xor[2][3] = key_in[3] ^ bucket->key[2][3]; \ + xor[2][0] = k[0] ^ bucket->key[2][0]; \ + xor[2][1] = k[1] ^ bucket->key[2][1]; \ + xor[2][2] = k[2] ^ bucket->key[2][2]; \ + xor[2][3] = k[3] ^ bucket->key[2][3]; \ \ - xor[3][0] = key_in[0] ^ bucket->key[3][0]; \ - xor[3][1] = key_in[1] ^ bucket->key[3][1]; \ - xor[3][2] = key_in[2] ^ bucket->key[3][2]; \ - xor[3][3] = key_in[3] ^ bucket->key[3][3]; \ + xor[3][0] = k[0] ^ bucket->key[3][0]; \ + xor[3][1] = k[1] ^ bucket->key[3][1]; \ + xor[3][2] = k[2] ^ bucket->key[3][2]; \ + xor[3][3] = k[3] ^ bucket->key[3][3]; \ \ or[0] = xor[0][0] | xor[0][1] | xor[0][2] | xor[0][3] | signature[0];\ or[1] = xor[1][0] | xor[1][1] | xor[1][2] | xor[1][3] | signature[1];\ @@ -604,12 +686,15 @@ rte_table_hash_entry_delete_key32_ext( rte_prefetch0(RTE_MBUF_METADATA_UINT8_PTR(mbuf0, key_offset));\ } -#define lookup1_stage1(mbuf1, bucket1, f) \ +#define lookup1_stage1(mbuf1, bucket1, f) \ { \ + uint64_t *key; \ uint64_t signature; \ uint32_t bucket_index; \ \ - signature = RTE_MBUF_METADATA_UINT32(mbuf1, f->signature_offset);\ + key = RTE_MBUF_METADATA_UINT64_PTR(mbuf1, f->key_offset); \ + signature = f->f_hash(key, f->key_mask, KEY_SIZE, f->seed); \ + \ bucket_index = signature & (f->n_buckets - 1); \ bucket1 = (struct rte_bucket_4_32 *) \ &f->memory[bucket_index * f->bucket_size]; \ @@ -627,8 +712,7 @@ rte_table_hash_entry_delete_key32_ext( uint32_t pos; \ \ key = RTE_MBUF_METADATA_UINT64_PTR(mbuf2, f->key_offset);\ - \ - lookup_key32_cmp(key, bucket2, pos); \ + lookup_key32_cmp(key, bucket2, pos, f); \ \ pkt_mask = (bucket2->signature[pos] & 1LLU) << pkt2_index;\ pkts_mask_out |= pkt_mask; \ @@ -649,8 +733,7 @@ rte_table_hash_entry_delete_key32_ext( uint32_t pos; \ \ key = RTE_MBUF_METADATA_UINT64_PTR(mbuf2, f->key_offset);\ - \ - lookup_key32_cmp(key, bucket2, pos); \ + lookup_key32_cmp(key, bucket2, pos, f); \ \ pkt_mask = (bucket2->signature[pos] & 1LLU) << pkt2_index;\ pkts_mask_out |= pkt_mask; \ @@ -678,7 +761,7 @@ rte_table_hash_entry_delete_key32_ext( bucket = buckets[pkt_index]; \ key = keys[pkt_index]; \ \ - lookup_key32_cmp(key, bucket, pos); \ + lookup_key32_cmp(key, bucket, pos, f); \ \ pkt_mask = (bucket->signature[pos] & 1LLU) << pkt_index;\ pkts_mask_out |= pkt_mask; \ @@ -745,22 +828,27 @@ rte_table_hash_entry_delete_key32_ext( #define lookup2_stage1(mbuf10, mbuf11, bucket10, bucket11, f) \ { \ - uint64_t signature10, signature11; \ - uint32_t bucket10_index, bucket11_index; \ + uint64_t *key10, *key11; \ + uint64_t signature10, signature11; \ + uint32_t bucket10_index, bucket11_index; \ \ - signature10 = RTE_MBUF_METADATA_UINT32(mbuf10, f->signature_offset);\ - bucket10_index = signature10 & (f->n_buckets - 1); \ + key10 = RTE_MBUF_METADATA_UINT64_PTR(mbuf10, f->key_offset); \ + signature10 = f->f_hash(key10, f->key_mask, KEY_SIZE, f->seed); \ + \ + bucket10_index = signature10 & (f->n_buckets - 1); \ bucket10 = (struct rte_bucket_4_32 *) \ &f->memory[bucket10_index * f->bucket_size]; \ - rte_prefetch0(bucket10); \ + rte_prefetch0(bucket10); \ rte_prefetch0((void *)(((uintptr_t) bucket10) + RTE_CACHE_LINE_SIZE));\ rte_prefetch0((void *)(((uintptr_t) bucket10) + 2 * RTE_CACHE_LINE_SIZE));\ \ - signature11 = RTE_MBUF_METADATA_UINT32(mbuf11, f->signature_offset);\ - bucket11_index = signature11 & (f->n_buckets - 1); \ + key11 = RTE_MBUF_METADATA_UINT64_PTR(mbuf11, f->key_offset); \ + signature11 = f->f_hash(key11, f->key_mask, KEY_SIZE, f->seed);\ + \ + bucket11_index = signature11 & (f->n_buckets - 1); \ bucket11 = (struct rte_bucket_4_32 *) \ &f->memory[bucket11_index * f->bucket_size]; \ - rte_prefetch0(bucket11); \ + rte_prefetch0(bucket11); \ rte_prefetch0((void *)(((uintptr_t) bucket11) + RTE_CACHE_LINE_SIZE));\ rte_prefetch0((void *)(((uintptr_t) bucket11) + 2 * RTE_CACHE_LINE_SIZE));\ } @@ -776,8 +864,8 @@ rte_table_hash_entry_delete_key32_ext( key20 = RTE_MBUF_METADATA_UINT64_PTR(mbuf20, f->key_offset);\ key21 = RTE_MBUF_METADATA_UINT64_PTR(mbuf21, f->key_offset);\ \ - lookup_key32_cmp(key20, bucket20, pos20); \ - lookup_key32_cmp(key21, bucket21, pos21); \ + lookup_key32_cmp(key20, bucket20, pos20, f); \ + lookup_key32_cmp(key21, bucket21, pos21, f); \ \ pkt20_mask = (bucket20->signature[pos20] & 1LLU) << pkt20_index;\ pkt21_mask = (bucket21->signature[pos21] & 1LLU) << pkt21_index;\ @@ -805,8 +893,8 @@ rte_table_hash_entry_delete_key32_ext( key20 = RTE_MBUF_METADATA_UINT64_PTR(mbuf20, f->key_offset);\ key21 = RTE_MBUF_METADATA_UINT64_PTR(mbuf21, f->key_offset);\ \ - lookup_key32_cmp(key20, bucket20, pos20); \ - lookup_key32_cmp(key21, bucket21, pos21); \ + lookup_key32_cmp(key20, bucket20, pos20, f); \ + lookup_key32_cmp(key21, bucket21, pos21, f); \ \ pkt20_mask = (bucket20->signature[pos20] & 1LLU) << pkt20_index;\ pkt21_mask = (bucket21->signature[pos21] & 1LLU) << pkt21_index;\ @@ -1110,7 +1198,7 @@ grind_next_buckets: static int rte_table_hash_key32_stats_read(void *table, struct rte_table_stats *stats, int clear) { - struct rte_table_hash *t = (struct rte_table_hash *) table; + struct rte_table_hash *t = table; if (stats != NULL) memcpy(stats, &t->stats, sizeof(t->stats)); diff --git a/dpdk/lib/librte_table/rte_table_hash_key8.c b/dpdk/lib/librte_table/rte_table_hash_key8.c index e2e2bdc4..94373043 100644 --- a/dpdk/lib/librte_table/rte_table_hash_key8.c +++ b/dpdk/lib/librte_table/rte_table_hash_key8.c @@ -1,34 +1,34 @@ /*- - * BSD LICENSE + * BSD LICENSE * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. + * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. + * All rights reserved. * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include #include @@ -42,7 +42,9 @@ #include "rte_table_hash.h" #include "rte_lru.h" -#define RTE_TABLE_HASH_KEY_SIZE 8 +#define KEY_SIZE 8 + +#define KEYS_PER_BUCKET 4 #ifdef RTE_TABLE_STATS_COLLECT @@ -76,11 +78,9 @@ struct rte_table_hash { /* Input parameters */ uint32_t n_buckets; - uint32_t n_entries_per_bucket; uint32_t key_size; uint32_t entry_size; uint32_t bucket_size; - uint32_t signature_offset; uint32_t key_offset; uint64_t key_mask; rte_table_hash_op_hash f_hash; @@ -96,10 +96,46 @@ struct rte_table_hash { }; static int -check_params_create_lru(struct rte_table_hash_key8_lru_params *params) { - /* n_entries */ - if (params->n_entries == 0) { - RTE_LOG(ERR, TABLE, "%s: n_entries is zero\n", __func__); +keycmp(void *a, void *b, void *b_mask) +{ + uint64_t *a64 = a, *b64 = b, *b_mask64 = b_mask; + + return a64[0] != (b64[0] & b_mask64[0]); +} + +static void +keycpy(void *dst, void *src, void *src_mask) +{ + uint64_t *dst64 = dst, *src64 = src, *src_mask64 = src_mask; + + dst64[0] = src64[0] & src_mask64[0]; +} + +static int +check_params_create(struct rte_table_hash_params *params) +{ + /* name */ + if (params->name == NULL) { + RTE_LOG(ERR, TABLE, "%s: name invalid value\n", __func__); + return -EINVAL; + } + + /* key_size */ + if (params->key_size != KEY_SIZE) { + RTE_LOG(ERR, TABLE, "%s: key_size invalid value\n", __func__); + return -EINVAL; + } + + /* n_keys */ + if (params->n_keys == 0) { + RTE_LOG(ERR, TABLE, "%s: n_keys is zero\n", __func__); + return -EINVAL; + } + + /* n_buckets */ + if ((params->n_buckets == 0) || + (!rte_is_power_of_2(params->n_buckets))) { + RTE_LOG(ERR, TABLE, "%s: n_buckets invalid value\n", __func__); return -EINVAL; } @@ -116,47 +152,68 @@ check_params_create_lru(struct rte_table_hash_key8_lru_params *params) { static void * rte_table_hash_create_key8_lru(void *params, int socket_id, uint32_t entry_size) { - struct rte_table_hash_key8_lru_params *p = - (struct rte_table_hash_key8_lru_params *) params; + struct rte_table_hash_params *p = params; struct rte_table_hash *f; - uint32_t n_buckets, n_entries_per_bucket, key_size, bucket_size_cl; - uint32_t total_size, i; + uint64_t bucket_size, total_size; + uint32_t n_buckets, i; /* Check input parameters */ - if ((check_params_create_lru(p) != 0) || + if ((check_params_create(p) != 0) || ((sizeof(struct rte_table_hash) % RTE_CACHE_LINE_SIZE) != 0) || - ((sizeof(struct rte_bucket_4_8) % RTE_CACHE_LINE_SIZE) != 0)) { + ((sizeof(struct rte_bucket_4_8) % 64) != 0)) return NULL; - } - n_entries_per_bucket = 4; - key_size = 8; + + /* + * Table dimensioning + * + * Objective: Pick the number of buckets (n_buckets) so that there a chance + * to store n_keys keys in the table. + * + * Note: Since the buckets do not get extended, it is not possible to + * guarantee that n_keys keys can be stored in the table at any time. In the + * worst case scenario when all the n_keys fall into the same bucket, only + * a maximum of KEYS_PER_BUCKET keys will be stored in the table. This case + * defeats the purpose of the hash table. It indicates unsuitable f_hash or + * n_keys to n_buckets ratio. + * + * MIN(n_buckets) = (n_keys + KEYS_PER_BUCKET - 1) / KEYS_PER_BUCKET + */ + n_buckets = rte_align32pow2( + (p->n_keys + KEYS_PER_BUCKET - 1) / KEYS_PER_BUCKET); + n_buckets = RTE_MAX(n_buckets, p->n_buckets); /* Memory allocation */ - n_buckets = rte_align32pow2((p->n_entries + n_entries_per_bucket - 1) / - n_entries_per_bucket); - bucket_size_cl = (sizeof(struct rte_bucket_4_8) + n_entries_per_bucket * - entry_size + RTE_CACHE_LINE_SIZE - 1) / RTE_CACHE_LINE_SIZE; - total_size = sizeof(struct rte_table_hash) + n_buckets * - bucket_size_cl * RTE_CACHE_LINE_SIZE; + bucket_size = RTE_CACHE_LINE_ROUNDUP(sizeof(struct rte_bucket_4_8) + + KEYS_PER_BUCKET * entry_size); + total_size = sizeof(struct rte_table_hash) + n_buckets * bucket_size; - f = rte_zmalloc_socket("TABLE", total_size, RTE_CACHE_LINE_SIZE, socket_id); - if (f == NULL) { - RTE_LOG(ERR, TABLE, - "%s: Cannot allocate %u bytes for hash table\n", - __func__, total_size); + if (total_size > SIZE_MAX) { + RTE_LOG(ERR, TABLE, "%s: Cannot allocate %" PRIu64 " bytes" + " for hash table %s\n", + __func__, total_size, p->name); return NULL; } - RTE_LOG(INFO, TABLE, - "%s: Hash table memory footprint is %u bytes\n", - __func__, total_size); + + f = rte_zmalloc_socket(p->name, + (size_t)total_size, + RTE_CACHE_LINE_SIZE, + socket_id); + if (f == NULL) { + RTE_LOG(ERR, TABLE, "%s: Cannot allocate %" PRIu64 " bytes" + " for hash table %s\n", + __func__, total_size, p->name); + return NULL; + } + + RTE_LOG(INFO, TABLE, "%s: Hash table %s memory footprint " + "is %" PRIu64 " bytes\n", + __func__, p->name, total_size); /* Memory initialization */ f->n_buckets = n_buckets; - f->n_entries_per_bucket = n_entries_per_bucket; - f->key_size = key_size; + f->key_size = KEY_SIZE; f->entry_size = entry_size; - f->bucket_size = bucket_size_cl * RTE_CACHE_LINE_SIZE; - f->signature_offset = p->signature_offset; + f->bucket_size = bucket_size; f->key_offset = p->key_offset; f->f_hash = p->f_hash; f->seed = p->seed; @@ -180,7 +237,7 @@ rte_table_hash_create_key8_lru(void *params, int socket_id, uint32_t entry_size) static int rte_table_hash_free_key8_lru(void *table) { - struct rte_table_hash *f = (struct rte_table_hash *) table; + struct rte_table_hash *f = table; /* Check input parameters */ if (f == NULL) { @@ -200,12 +257,12 @@ rte_table_hash_entry_add_key8_lru( int *key_found, void **entry_ptr) { - struct rte_table_hash *f = (struct rte_table_hash *) table; + struct rte_table_hash *f = table; struct rte_bucket_4_8 *bucket; uint64_t signature, mask, pos; uint32_t bucket_index, i; - signature = f->f_hash(key, f->key_size, f->seed); + signature = f->f_hash(key, &f->key_mask, f->key_size, f->seed); bucket_index = signature & (f->n_buckets - 1); bucket = (struct rte_bucket_4_8 *) &f->memory[bucket_index * f->bucket_size]; @@ -213,10 +270,10 @@ rte_table_hash_entry_add_key8_lru( /* Key is present in the bucket */ for (i = 0, mask = 1LLU; i < 4; i++, mask <<= 1) { uint64_t bucket_signature = bucket->signature; - uint64_t bucket_key = bucket->key[i]; + uint64_t *bucket_key = &bucket->key[i]; if ((bucket_signature & mask) && - (*((uint64_t *) key) == bucket_key)) { + (keycmp(bucket_key, key, &f->key_mask) == 0)) { uint8_t *bucket_data = &bucket->data[i * f->entry_size]; memcpy(bucket_data, entry, f->entry_size); @@ -235,7 +292,7 @@ rte_table_hash_entry_add_key8_lru( uint8_t *bucket_data = &bucket->data[i * f->entry_size]; bucket->signature |= mask; - bucket->key[i] = *((uint64_t *) key); + keycpy(&bucket->key[i], key, &f->key_mask); memcpy(bucket_data, entry, f->entry_size); lru_update(bucket, i); *key_found = 0; @@ -247,10 +304,10 @@ rte_table_hash_entry_add_key8_lru( /* Bucket full: replace LRU entry */ pos = lru_pos(bucket); - bucket->key[pos] = *((uint64_t *) key); + keycpy(&bucket->key[pos], key, &f->key_mask); memcpy(&bucket->data[pos * f->entry_size], entry, f->entry_size); lru_update(bucket, pos); - *key_found = 0; + *key_found = 0; *entry_ptr = (void *) &bucket->data[pos * f->entry_size]; return 0; @@ -263,12 +320,12 @@ rte_table_hash_entry_delete_key8_lru( int *key_found, void *entry) { - struct rte_table_hash *f = (struct rte_table_hash *) table; + struct rte_table_hash *f = table; struct rte_bucket_4_8 *bucket; uint64_t signature, mask; uint32_t bucket_index, i; - signature = f->f_hash(key, f->key_size, f->seed); + signature = f->f_hash(key, &f->key_mask, f->key_size, f->seed); bucket_index = signature & (f->n_buckets - 1); bucket = (struct rte_bucket_4_8 *) &f->memory[bucket_index * f->bucket_size]; @@ -276,10 +333,10 @@ rte_table_hash_entry_delete_key8_lru( /* Key is present in the bucket */ for (i = 0, mask = 1LLU; i < 4; i++, mask <<= 1) { uint64_t bucket_signature = bucket->signature; - uint64_t bucket_key = bucket->key[i]; + uint64_t *bucket_key = &bucket->key[i]; if ((bucket_signature & mask) && - (*((uint64_t *) key) == bucket_key)) { + (keycmp(bucket_key, key, &f->key_mask) == 0)) { uint8_t *bucket_data = &bucket->data[i * f->entry_size]; bucket->signature &= ~mask; @@ -296,79 +353,71 @@ rte_table_hash_entry_delete_key8_lru( return 0; } -static int -check_params_create_ext(struct rte_table_hash_key8_ext_params *params) { - /* n_entries */ - if (params->n_entries == 0) { - RTE_LOG(ERR, TABLE, "%s: n_entries is zero\n", __func__); - return -EINVAL; - } - - /* n_entries_ext */ - if (params->n_entries_ext == 0) { - RTE_LOG(ERR, TABLE, "%s: n_entries_ext is zero\n", __func__); - return -EINVAL; - } - - /* f_hash */ - if (params->f_hash == NULL) { - RTE_LOG(ERR, TABLE, "%s: f_hash function pointer is NULL\n", - __func__); - return -EINVAL; - } - - return 0; -} - static void * rte_table_hash_create_key8_ext(void *params, int socket_id, uint32_t entry_size) { - struct rte_table_hash_key8_ext_params *p = - (struct rte_table_hash_key8_ext_params *) params; + struct rte_table_hash_params *p = params; struct rte_table_hash *f; - uint32_t n_buckets, n_buckets_ext, n_entries_per_bucket, key_size; - uint32_t bucket_size_cl, stack_size_cl, total_size, i; + uint64_t bucket_size, stack_size, total_size; + uint32_t n_buckets_ext, i; /* Check input parameters */ - if ((check_params_create_ext(p) != 0) || + if ((check_params_create(p) != 0) || ((sizeof(struct rte_table_hash) % RTE_CACHE_LINE_SIZE) != 0) || - ((sizeof(struct rte_bucket_4_8) % RTE_CACHE_LINE_SIZE) != 0)) + ((sizeof(struct rte_bucket_4_8) % 64) != 0)) return NULL; - n_entries_per_bucket = 4; - key_size = 8; + /* + * Table dimensioning + * + * Objective: Pick the number of bucket extensions (n_buckets_ext) so that + * it is guaranteed that n_keys keys can be stored in the table at any time. + * + * The worst case scenario takes place when all the n_keys keys fall into + * the same bucket. Actually, due to the KEYS_PER_BUCKET scheme, the worst + * case takes place when (n_keys - KEYS_PER_BUCKET + 1) keys fall into the + * same bucket, while the remaining (KEYS_PER_BUCKET - 1) keys each fall + * into a different bucket. This case defeats the purpose of the hash table. + * It indicates unsuitable f_hash or n_keys to n_buckets ratio. + * + * n_buckets_ext = n_keys / KEYS_PER_BUCKET + KEYS_PER_BUCKET - 1 + */ + n_buckets_ext = p->n_keys / KEYS_PER_BUCKET + KEYS_PER_BUCKET - 1; /* Memory allocation */ - n_buckets = rte_align32pow2((p->n_entries + n_entries_per_bucket - 1) / - n_entries_per_bucket); - n_buckets_ext = (p->n_entries_ext + n_entries_per_bucket - 1) / - n_entries_per_bucket; - bucket_size_cl = (sizeof(struct rte_bucket_4_8) + n_entries_per_bucket * - entry_size + RTE_CACHE_LINE_SIZE - 1) / RTE_CACHE_LINE_SIZE; - stack_size_cl = (n_buckets_ext * sizeof(uint32_t) + RTE_CACHE_LINE_SIZE - 1) - / RTE_CACHE_LINE_SIZE; - total_size = sizeof(struct rte_table_hash) + ((n_buckets + - n_buckets_ext) * bucket_size_cl + stack_size_cl) * - RTE_CACHE_LINE_SIZE; + bucket_size = RTE_CACHE_LINE_ROUNDUP(sizeof(struct rte_bucket_4_8) + + KEYS_PER_BUCKET * entry_size); + stack_size = RTE_CACHE_LINE_ROUNDUP(n_buckets_ext * sizeof(uint32_t)); + total_size = sizeof(struct rte_table_hash) + + (p->n_buckets + n_buckets_ext) * bucket_size + stack_size; - f = rte_zmalloc_socket("TABLE", total_size, RTE_CACHE_LINE_SIZE, socket_id); - if (f == NULL) { - RTE_LOG(ERR, TABLE, - "%s: Cannot allocate %u bytes for hash table\n", - __func__, total_size); + if (total_size > SIZE_MAX) { + RTE_LOG(ERR, TABLE, "%s: Cannot allocate %" PRIu64 " bytes " + "for hash table %s\n", + __func__, total_size, p->name); return NULL; } - RTE_LOG(INFO, TABLE, - "%s: Hash table memory footprint is %u bytes\n", - __func__, total_size); + + f = rte_zmalloc_socket(p->name, + (size_t)total_size, + RTE_CACHE_LINE_SIZE, + socket_id); + if (f == NULL) { + RTE_LOG(ERR, TABLE, + "%s: Cannot allocate %" PRIu64 " bytes " + "for hash table %s\n", + __func__, total_size, p->name); + return NULL; + } + RTE_LOG(INFO, TABLE, "%s: Hash table %s memory footprint " + "is %" PRIu64 " bytes\n", + __func__, p->name, total_size); /* Memory initialization */ - f->n_buckets = n_buckets; - f->n_entries_per_bucket = n_entries_per_bucket; - f->key_size = key_size; + f->n_buckets = p->n_buckets; + f->key_size = KEY_SIZE; f->entry_size = entry_size; - f->bucket_size = bucket_size_cl * RTE_CACHE_LINE_SIZE; - f->signature_offset = p->signature_offset; + f->bucket_size = bucket_size; f->key_offset = p->key_offset; f->f_hash = p->f_hash; f->seed = p->seed; @@ -376,7 +425,7 @@ rte_table_hash_create_key8_ext(void *params, int socket_id, uint32_t entry_size) f->n_buckets_ext = n_buckets_ext; f->stack_pos = n_buckets_ext; f->stack = (uint32_t *) - &f->memory[(n_buckets + n_buckets_ext) * f->bucket_size]; + &f->memory[(p->n_buckets + n_buckets_ext) * f->bucket_size]; if (p->key_mask != NULL) f->key_mask = ((uint64_t *)p->key_mask)[0]; @@ -392,7 +441,7 @@ rte_table_hash_create_key8_ext(void *params, int socket_id, uint32_t entry_size) static int rte_table_hash_free_key8_ext(void *table) { - struct rte_table_hash *f = (struct rte_table_hash *) table; + struct rte_table_hash *f = table; /* Check input parameters */ if (f == NULL) { @@ -412,12 +461,12 @@ rte_table_hash_entry_add_key8_ext( int *key_found, void **entry_ptr) { - struct rte_table_hash *f = (struct rte_table_hash *) table; + struct rte_table_hash *f = table; struct rte_bucket_4_8 *bucket0, *bucket, *bucket_prev; uint64_t signature; uint32_t bucket_index, i; - signature = f->f_hash(key, f->key_size, f->seed); + signature = f->f_hash(key, &f->key_mask, f->key_size, f->seed); bucket_index = signature & (f->n_buckets - 1); bucket0 = (struct rte_bucket_4_8 *) &f->memory[bucket_index * f->bucket_size]; @@ -428,10 +477,10 @@ rte_table_hash_entry_add_key8_ext( for (i = 0, mask = 1LLU; i < 4; i++, mask <<= 1) { uint64_t bucket_signature = bucket->signature; - uint64_t bucket_key = bucket->key[i]; + uint64_t *bucket_key = &bucket->key[i]; if ((bucket_signature & mask) && - (*((uint64_t *) key) == bucket_key)) { + (keycmp(bucket_key, key, &f->key_mask) == 0)) { uint8_t *bucket_data = &bucket->data[i * f->entry_size]; @@ -456,7 +505,7 @@ rte_table_hash_entry_add_key8_ext( f->entry_size]; bucket->signature |= mask; - bucket->key[i] = *((uint64_t *) key); + keycpy(&bucket->key[i], key, &f->key_mask); memcpy(bucket_data, entry, f->entry_size); *key_found = 0; *entry_ptr = (void *) bucket_data; @@ -476,7 +525,7 @@ rte_table_hash_entry_add_key8_ext( bucket_prev->next_valid = 1; bucket->signature = 1; - bucket->key[0] = *((uint64_t *) key); + keycpy(&bucket->key[0], key, &f->key_mask); memcpy(&bucket->data[0], entry, f->entry_size); *key_found = 0; *entry_ptr = (void *) &bucket->data[0]; @@ -493,12 +542,12 @@ rte_table_hash_entry_delete_key8_ext( int *key_found, void *entry) { - struct rte_table_hash *f = (struct rte_table_hash *) table; + struct rte_table_hash *f = table; struct rte_bucket_4_8 *bucket0, *bucket, *bucket_prev; uint64_t signature; uint32_t bucket_index, i; - signature = f->f_hash(key, f->key_size, f->seed); + signature = f->f_hash(key, &f->key_mask, f->key_size, f->seed); bucket_index = signature & (f->n_buckets - 1); bucket0 = (struct rte_bucket_4_8 *) &f->memory[bucket_index * f->bucket_size]; @@ -510,10 +559,10 @@ rte_table_hash_entry_delete_key8_ext( for (i = 0, mask = 1LLU; i < 4; i++, mask <<= 1) { uint64_t bucket_signature = bucket->signature; - uint64_t bucket_key = bucket->key[i]; + uint64_t *bucket_key = &bucket->key[i]; if ((bucket_signature & mask) && - (*((uint64_t *) key) == bucket_key)) { + (keycmp(bucket_key, key, &f->key_mask) == 0)) { uint8_t *bucket_data = &bucket->data[i * f->entry_size]; @@ -546,16 +595,17 @@ rte_table_hash_entry_delete_key8_ext( return 0; } -#define lookup_key8_cmp(key_in, bucket, pos) \ +#define lookup_key8_cmp(key_in, bucket, pos, f) \ { \ - uint64_t xor[4], signature; \ + uint64_t xor[4], signature, k; \ \ signature = ~bucket->signature; \ \ - xor[0] = (key_in[0] ^ bucket->key[0]) | (signature & 1);\ - xor[1] = (key_in[0] ^ bucket->key[1]) | (signature & 2);\ - xor[2] = (key_in[0] ^ bucket->key[2]) | (signature & 4);\ - xor[3] = (key_in[0] ^ bucket->key[3]) | (signature & 8);\ + k = key_in[0] & f->key_mask; \ + xor[0] = (k ^ bucket->key[0]) | (signature & 1); \ + xor[1] = (k ^ bucket->key[1]) | (signature & 2); \ + xor[2] = (k ^ bucket->key[2]) | (signature & 4); \ + xor[3] = (k ^ bucket->key[3]) | (signature & 8); \ \ pos = 4; \ if (xor[0] == 0) \ @@ -582,28 +632,13 @@ rte_table_hash_entry_delete_key8_ext( } #define lookup1_stage1(mbuf1, bucket1, f) \ -{ \ - uint64_t signature; \ - uint32_t bucket_index; \ - \ - signature = RTE_MBUF_METADATA_UINT32(mbuf1, f->signature_offset);\ - bucket_index = signature & (f->n_buckets - 1); \ - bucket1 = (struct rte_bucket_4_8 *) \ - &f->memory[bucket_index * f->bucket_size]; \ - rte_prefetch0(bucket1); \ -} - -#define lookup1_stage1_dosig(mbuf1, bucket1, f) \ { \ uint64_t *key; \ uint64_t signature; \ uint32_t bucket_index; \ - uint64_t hash_key_buffer; \ \ key = RTE_MBUF_METADATA_UINT64_PTR(mbuf1, f->key_offset);\ - hash_key_buffer = *key & f->key_mask; \ - signature = f->f_hash(&hash_key_buffer, \ - RTE_TABLE_HASH_KEY_SIZE, f->seed); \ + signature = f->f_hash(key, &f->key_mask, KEY_SIZE, f->seed); \ bucket_index = signature & (f->n_buckets - 1); \ bucket1 = (struct rte_bucket_4_8 *) \ &f->memory[bucket_index * f->bucket_size]; \ @@ -617,12 +652,9 @@ rte_table_hash_entry_delete_key8_ext( uint64_t pkt_mask; \ uint64_t *key; \ uint32_t pos; \ - uint64_t hash_key_buffer; \ \ key = RTE_MBUF_METADATA_UINT64_PTR(mbuf2, f->key_offset);\ - hash_key_buffer = key[0] & f->key_mask; \ - \ - lookup_key8_cmp((&hash_key_buffer), bucket2, pos); \ + lookup_key8_cmp(key, bucket2, pos, f); \ \ pkt_mask = ((bucket2->signature >> pos) & 1LLU) << pkt2_index;\ pkts_mask_out |= pkt_mask; \ @@ -641,12 +673,9 @@ rte_table_hash_entry_delete_key8_ext( uint64_t pkt_mask, bucket_mask; \ uint64_t *key; \ uint32_t pos; \ - uint64_t hash_key_buffer; \ \ key = RTE_MBUF_METADATA_UINT64_PTR(mbuf2, f->key_offset);\ - hash_key_buffer = *key & f->key_mask; \ - \ - lookup_key8_cmp((&hash_key_buffer), bucket2, pos); \ + lookup_key8_cmp(key, bucket2, pos, f); \ \ pkt_mask = ((bucket2->signature >> pos) & 1LLU) << pkt2_index;\ pkts_mask_out |= pkt_mask; \ @@ -670,13 +699,10 @@ rte_table_hash_entry_delete_key8_ext( uint64_t pkt_mask, bucket_mask; \ uint64_t *key; \ uint32_t pos; \ - uint64_t hash_key_buffer; \ \ bucket = buckets[pkt_index]; \ key = keys[pkt_index]; \ - hash_key_buffer = (*key) & f->key_mask; \ - \ - lookup_key8_cmp((&hash_key_buffer), bucket, pos); \ + lookup_key8_cmp(key, bucket, pos, f); \ \ pkt_mask = ((bucket->signature >> pos) & 1LLU) << pkt_index;\ pkts_mask_out |= pkt_mask; \ @@ -738,29 +764,9 @@ rte_table_hash_entry_delete_key8_ext( rte_prefetch0(RTE_MBUF_METADATA_UINT8_PTR(mbuf01, key_offset));\ } -#define lookup2_stage1(mbuf10, mbuf11, bucket10, bucket11, f) \ -{ \ - uint64_t signature10, signature11; \ - uint32_t bucket10_index, bucket11_index; \ - \ - signature10 = RTE_MBUF_METADATA_UINT32(mbuf10, f->signature_offset);\ - bucket10_index = signature10 & (f->n_buckets - 1); \ - bucket10 = (struct rte_bucket_4_8 *) \ - &f->memory[bucket10_index * f->bucket_size]; \ - rte_prefetch0(bucket10); \ - \ - signature11 = RTE_MBUF_METADATA_UINT32(mbuf11, f->signature_offset);\ - bucket11_index = signature11 & (f->n_buckets - 1); \ - bucket11 = (struct rte_bucket_4_8 *) \ - &f->memory[bucket11_index * f->bucket_size]; \ - rte_prefetch0(bucket11); \ -} - -#define lookup2_stage1_dosig(mbuf10, mbuf11, bucket10, bucket11, f)\ +#define lookup2_stage1(mbuf10, mbuf11, bucket10, bucket11, f)\ { \ uint64_t *key10, *key11; \ - uint64_t hash_offset_buffer10; \ - uint64_t hash_offset_buffer11; \ uint64_t signature10, signature11; \ uint32_t bucket10_index, bucket11_index; \ rte_table_hash_op_hash f_hash = f->f_hash; \ @@ -769,18 +775,14 @@ rte_table_hash_entry_delete_key8_ext( \ key10 = RTE_MBUF_METADATA_UINT64_PTR(mbuf10, key_offset);\ key11 = RTE_MBUF_METADATA_UINT64_PTR(mbuf11, key_offset);\ - hash_offset_buffer10 = *key10 & f->key_mask; \ - hash_offset_buffer11 = *key11 & f->key_mask; \ \ - signature10 = f_hash(&hash_offset_buffer10, \ - RTE_TABLE_HASH_KEY_SIZE, seed); \ + signature10 = f_hash(key10, &f->key_mask, KEY_SIZE, seed); \ bucket10_index = signature10 & (f->n_buckets - 1); \ bucket10 = (struct rte_bucket_4_8 *) \ &f->memory[bucket10_index * f->bucket_size]; \ rte_prefetch0(bucket10); \ \ - signature11 = f_hash(&hash_offset_buffer11, \ - RTE_TABLE_HASH_KEY_SIZE, seed); \ + signature11 = f_hash(key11, &f->key_mask, KEY_SIZE, seed); \ bucket11_index = signature11 & (f->n_buckets - 1); \ bucket11 = (struct rte_bucket_4_8 *) \ &f->memory[bucket11_index * f->bucket_size]; \ @@ -793,17 +795,13 @@ rte_table_hash_entry_delete_key8_ext( void *a20, *a21; \ uint64_t pkt20_mask, pkt21_mask; \ uint64_t *key20, *key21; \ - uint64_t hash_offset_buffer20; \ - uint64_t hash_offset_buffer21; \ uint32_t pos20, pos21; \ \ key20 = RTE_MBUF_METADATA_UINT64_PTR(mbuf20, f->key_offset);\ key21 = RTE_MBUF_METADATA_UINT64_PTR(mbuf21, f->key_offset);\ - hash_offset_buffer20 = *key20 & f->key_mask; \ - hash_offset_buffer21 = *key21 & f->key_mask; \ \ - lookup_key8_cmp((&hash_offset_buffer20), bucket20, pos20);\ - lookup_key8_cmp((&hash_offset_buffer21), bucket21, pos21);\ + lookup_key8_cmp(key20, bucket20, pos20, f); \ + lookup_key8_cmp(key21, bucket21, pos21, f); \ \ pkt20_mask = ((bucket20->signature >> pos20) & 1LLU) << pkt20_index;\ pkt21_mask = ((bucket21->signature >> pos21) & 1LLU) << pkt21_index;\ @@ -826,17 +824,13 @@ rte_table_hash_entry_delete_key8_ext( void *a20, *a21; \ uint64_t pkt20_mask, pkt21_mask, bucket20_mask, bucket21_mask;\ uint64_t *key20, *key21; \ - uint64_t hash_offset_buffer20; \ - uint64_t hash_offset_buffer21; \ uint32_t pos20, pos21; \ \ key20 = RTE_MBUF_METADATA_UINT64_PTR(mbuf20, f->key_offset);\ key21 = RTE_MBUF_METADATA_UINT64_PTR(mbuf21, f->key_offset);\ - hash_offset_buffer20 = *key20 & f->key_mask; \ - hash_offset_buffer21 = *key21 & f->key_mask; \ \ - lookup_key8_cmp((&hash_offset_buffer20), bucket20, pos20);\ - lookup_key8_cmp((&hash_offset_buffer21), bucket21, pos21);\ + lookup_key8_cmp(key20, bucket20, pos20, f); \ + lookup_key8_cmp(key21, bucket21, pos21, f); \ \ pkt20_mask = ((bucket20->signature >> pos20) & 1LLU) << pkt20_index;\ pkt21_mask = ((bucket21->signature >> pos21) & 1LLU) << pkt21_index;\ @@ -871,8 +865,8 @@ rte_table_hash_lookup_key8_lru( struct rte_table_hash *f = (struct rte_table_hash *) table; struct rte_bucket_4_8 *bucket10, *bucket11, *bucket20, *bucket21; struct rte_mbuf *mbuf00, *mbuf01, *mbuf10, *mbuf11, *mbuf20, *mbuf21; - uint32_t pkt00_index, pkt01_index, pkt10_index, - pkt11_index, pkt20_index, pkt21_index; + uint32_t pkt00_index, pkt01_index, pkt10_index; + uint32_t pkt11_index, pkt20_index, pkt21_index; uint64_t pkts_mask_out = 0; __rte_unused uint32_t n_pkts_in = __builtin_popcountll(pkts_mask); @@ -888,7 +882,7 @@ rte_table_hash_lookup_key8_lru( lookup1_stage0(pkt_index, mbuf, pkts, pkts_mask, f); lookup1_stage1(mbuf, bucket, f); lookup1_stage2_lru(pkt_index, mbuf, bucket, - pkts_mask_out, entries, f); + pkts_mask_out, entries, f); } *lookup_hit_mask = pkts_mask_out; @@ -984,133 +978,7 @@ rte_table_hash_lookup_key8_lru( *lookup_hit_mask = pkts_mask_out; RTE_TABLE_HASH_KEY8_STATS_PKTS_LOOKUP_MISS(f, n_pkts_in - __builtin_popcountll(pkts_mask_out)); return 0; -} /* rte_table_hash_lookup_key8_lru() */ - -static int -rte_table_hash_lookup_key8_lru_dosig( - void *table, - struct rte_mbuf **pkts, - uint64_t pkts_mask, - uint64_t *lookup_hit_mask, - void **entries) -{ - struct rte_table_hash *f = (struct rte_table_hash *) table; - struct rte_bucket_4_8 *bucket10, *bucket11, *bucket20, *bucket21; - struct rte_mbuf *mbuf00, *mbuf01, *mbuf10, *mbuf11, *mbuf20, *mbuf21; - uint32_t pkt00_index, pkt01_index, pkt10_index; - uint32_t pkt11_index, pkt20_index, pkt21_index; - uint64_t pkts_mask_out = 0; - - __rte_unused uint32_t n_pkts_in = __builtin_popcountll(pkts_mask); - RTE_TABLE_HASH_KEY8_STATS_PKTS_IN_ADD(f, n_pkts_in); - - /* Cannot run the pipeline with less than 5 packets */ - if (__builtin_popcountll(pkts_mask) < 5) { - for ( ; pkts_mask; ) { - struct rte_bucket_4_8 *bucket; - struct rte_mbuf *mbuf; - uint32_t pkt_index; - - lookup1_stage0(pkt_index, mbuf, pkts, pkts_mask, f); - lookup1_stage1_dosig(mbuf, bucket, f); - lookup1_stage2_lru(pkt_index, mbuf, bucket, - pkts_mask_out, entries, f); - } - - *lookup_hit_mask = pkts_mask_out; - RTE_TABLE_HASH_KEY8_STATS_PKTS_LOOKUP_MISS(f, n_pkts_in - __builtin_popcountll(pkts_mask_out)); - return 0; - } - - /* - * Pipeline fill - * - */ - /* Pipeline stage 0 */ - lookup2_stage0(pkt00_index, pkt01_index, mbuf00, mbuf01, pkts, - pkts_mask, f); - - /* Pipeline feed */ - mbuf10 = mbuf00; - mbuf11 = mbuf01; - pkt10_index = pkt00_index; - pkt11_index = pkt01_index; - - /* Pipeline stage 0 */ - lookup2_stage0(pkt00_index, pkt01_index, mbuf00, mbuf01, pkts, - pkts_mask, f); - - /* Pipeline stage 1 */ - lookup2_stage1_dosig(mbuf10, mbuf11, bucket10, bucket11, f); - - /* - * Pipeline run - * - */ - for ( ; pkts_mask; ) { - /* Pipeline feed */ - bucket20 = bucket10; - bucket21 = bucket11; - mbuf20 = mbuf10; - mbuf21 = mbuf11; - mbuf10 = mbuf00; - mbuf11 = mbuf01; - pkt20_index = pkt10_index; - pkt21_index = pkt11_index; - pkt10_index = pkt00_index; - pkt11_index = pkt01_index; - - /* Pipeline stage 0 */ - lookup2_stage0_with_odd_support(pkt00_index, pkt01_index, - mbuf00, mbuf01, pkts, pkts_mask, f); - - /* Pipeline stage 1 */ - lookup2_stage1_dosig(mbuf10, mbuf11, bucket10, bucket11, f); - - /* Pipeline stage 2 */ - lookup2_stage2_lru(pkt20_index, pkt21_index, mbuf20, mbuf21, - bucket20, bucket21, pkts_mask_out, entries, f); - } - - /* - * Pipeline flush - * - */ - /* Pipeline feed */ - bucket20 = bucket10; - bucket21 = bucket11; - mbuf20 = mbuf10; - mbuf21 = mbuf11; - mbuf10 = mbuf00; - mbuf11 = mbuf01; - pkt20_index = pkt10_index; - pkt21_index = pkt11_index; - pkt10_index = pkt00_index; - pkt11_index = pkt01_index; - - /* Pipeline stage 1 */ - lookup2_stage1_dosig(mbuf10, mbuf11, bucket10, bucket11, f); - - /* Pipeline stage 2 */ - lookup2_stage2_lru(pkt20_index, pkt21_index, mbuf20, mbuf21, - bucket20, bucket21, pkts_mask_out, entries, f); - - /* Pipeline feed */ - bucket20 = bucket10; - bucket21 = bucket11; - mbuf20 = mbuf10; - mbuf21 = mbuf11; - pkt20_index = pkt10_index; - pkt21_index = pkt11_index; - - /* Pipeline stage 2 */ - lookup2_stage2_lru(pkt20_index, pkt21_index, mbuf20, mbuf21, - bucket20, bucket21, pkts_mask_out, entries, f); - - *lookup_hit_mask = pkts_mask_out; - RTE_TABLE_HASH_KEY8_STATS_PKTS_LOOKUP_MISS(f, n_pkts_in - __builtin_popcountll(pkts_mask_out)); - return 0; -} /* rte_table_hash_lookup_key8_lru_dosig() */ +} /* lookup LRU */ static int rte_table_hash_lookup_key8_ext( @@ -1142,8 +1010,8 @@ rte_table_hash_lookup_key8_ext( lookup1_stage0(pkt_index, mbuf, pkts, pkts_mask, f); lookup1_stage1(mbuf, bucket, f); lookup1_stage2_ext(pkt_index, mbuf, bucket, - pkts_mask_out, entries, buckets_mask, buckets, - keys, f); + pkts_mask_out, entries, buckets_mask, + buckets, keys, f); } goto grind_next_buckets; @@ -1260,162 +1128,12 @@ grind_next_buckets: *lookup_hit_mask = pkts_mask_out; RTE_TABLE_HASH_KEY8_STATS_PKTS_LOOKUP_MISS(f, n_pkts_in - __builtin_popcountll(pkts_mask_out)); return 0; -} /* rte_table_hash_lookup_key8_ext() */ - -static int -rte_table_hash_lookup_key8_ext_dosig( - void *table, - struct rte_mbuf **pkts, - uint64_t pkts_mask, - uint64_t *lookup_hit_mask, - void **entries) -{ - struct rte_table_hash *f = (struct rte_table_hash *) table; - struct rte_bucket_4_8 *bucket10, *bucket11, *bucket20, *bucket21; - struct rte_mbuf *mbuf00, *mbuf01, *mbuf10, *mbuf11, *mbuf20, *mbuf21; - uint32_t pkt00_index, pkt01_index, pkt10_index; - uint32_t pkt11_index, pkt20_index, pkt21_index; - uint64_t pkts_mask_out = 0, buckets_mask = 0; - struct rte_bucket_4_8 *buckets[RTE_PORT_IN_BURST_SIZE_MAX]; - uint64_t *keys[RTE_PORT_IN_BURST_SIZE_MAX]; - - __rte_unused uint32_t n_pkts_in = __builtin_popcountll(pkts_mask); - RTE_TABLE_HASH_KEY8_STATS_PKTS_IN_ADD(f, n_pkts_in); - - /* Cannot run the pipeline with less than 5 packets */ - if (__builtin_popcountll(pkts_mask) < 5) { - for ( ; pkts_mask; ) { - struct rte_bucket_4_8 *bucket; - struct rte_mbuf *mbuf; - uint32_t pkt_index; - - lookup1_stage0(pkt_index, mbuf, pkts, pkts_mask, f); - lookup1_stage1_dosig(mbuf, bucket, f); - lookup1_stage2_ext(pkt_index, mbuf, bucket, - pkts_mask_out, entries, buckets_mask, - buckets, keys, f); - } - - goto grind_next_buckets; - } - - /* - * Pipeline fill - * - */ - /* Pipeline stage 0 */ - lookup2_stage0(pkt00_index, pkt01_index, mbuf00, mbuf01, pkts, - pkts_mask, f); - - /* Pipeline feed */ - mbuf10 = mbuf00; - mbuf11 = mbuf01; - pkt10_index = pkt00_index; - pkt11_index = pkt01_index; - - /* Pipeline stage 0 */ - lookup2_stage0(pkt00_index, pkt01_index, mbuf00, mbuf01, pkts, - pkts_mask, f); - - /* Pipeline stage 1 */ - lookup2_stage1_dosig(mbuf10, mbuf11, bucket10, bucket11, f); - - /* - * Pipeline run - * - */ - for ( ; pkts_mask; ) { - /* Pipeline feed */ - bucket20 = bucket10; - bucket21 = bucket11; - mbuf20 = mbuf10; - mbuf21 = mbuf11; - mbuf10 = mbuf00; - mbuf11 = mbuf01; - pkt20_index = pkt10_index; - pkt21_index = pkt11_index; - pkt10_index = pkt00_index; - pkt11_index = pkt01_index; - - /* Pipeline stage 0 */ - lookup2_stage0_with_odd_support(pkt00_index, pkt01_index, - mbuf00, mbuf01, pkts, pkts_mask, f); - - /* Pipeline stage 1 */ - lookup2_stage1_dosig(mbuf10, mbuf11, bucket10, bucket11, f); - - /* Pipeline stage 2 */ - lookup2_stage2_ext(pkt20_index, pkt21_index, mbuf20, mbuf21, - bucket20, bucket21, pkts_mask_out, entries, - buckets_mask, buckets, keys, f); - } - - /* - * Pipeline flush - * - */ - /* Pipeline feed */ - bucket20 = bucket10; - bucket21 = bucket11; - mbuf20 = mbuf10; - mbuf21 = mbuf11; - mbuf10 = mbuf00; - mbuf11 = mbuf01; - pkt20_index = pkt10_index; - pkt21_index = pkt11_index; - pkt10_index = pkt00_index; - pkt11_index = pkt01_index; - - /* Pipeline stage 1 */ - lookup2_stage1_dosig(mbuf10, mbuf11, bucket10, bucket11, f); - - /* Pipeline stage 2 */ - lookup2_stage2_ext(pkt20_index, pkt21_index, mbuf20, mbuf21, - bucket20, bucket21, pkts_mask_out, entries, - buckets_mask, buckets, keys, f); - - /* Pipeline feed */ - bucket20 = bucket10; - bucket21 = bucket11; - mbuf20 = mbuf10; - mbuf21 = mbuf11; - pkt20_index = pkt10_index; - pkt21_index = pkt11_index; - - /* Pipeline stage 2 */ - lookup2_stage2_ext(pkt20_index, pkt21_index, mbuf20, mbuf21, - bucket20, bucket21, pkts_mask_out, entries, - buckets_mask, buckets, keys, f); - -grind_next_buckets: - /* Grind next buckets */ - for ( ; buckets_mask; ) { - uint64_t buckets_mask_next = 0; - - for ( ; buckets_mask; ) { - uint64_t pkt_mask; - uint32_t pkt_index; - - pkt_index = __builtin_ctzll(buckets_mask); - pkt_mask = 1LLU << pkt_index; - buckets_mask &= ~pkt_mask; - - lookup_grinder(pkt_index, buckets, keys, pkts_mask_out, - entries, buckets_mask_next, f); - } - - buckets_mask = buckets_mask_next; - } - - *lookup_hit_mask = pkts_mask_out; - RTE_TABLE_HASH_KEY8_STATS_PKTS_LOOKUP_MISS(f, n_pkts_in - __builtin_popcountll(pkts_mask_out)); - return 0; -} /* rte_table_hash_lookup_key8_dosig_ext() */ +} /* lookup EXT */ static int rte_table_hash_key8_stats_read(void *table, struct rte_table_stats *stats, int clear) { - struct rte_table_hash *t = (struct rte_table_hash *) table; + struct rte_table_hash *t = table; if (stats != NULL) memcpy(stats, &t->stats, sizeof(t->stats)); @@ -1437,17 +1155,6 @@ struct rte_table_ops rte_table_hash_key8_lru_ops = { .f_stats = rte_table_hash_key8_stats_read, }; -struct rte_table_ops rte_table_hash_key8_lru_dosig_ops = { - .f_create = rte_table_hash_create_key8_lru, - .f_free = rte_table_hash_free_key8_lru, - .f_add = rte_table_hash_entry_add_key8_lru, - .f_delete = rte_table_hash_entry_delete_key8_lru, - .f_add_bulk = NULL, - .f_delete_bulk = NULL, - .f_lookup = rte_table_hash_lookup_key8_lru_dosig, - .f_stats = rte_table_hash_key8_stats_read, -}; - struct rte_table_ops rte_table_hash_key8_ext_ops = { .f_create = rte_table_hash_create_key8_ext, .f_free = rte_table_hash_free_key8_ext, @@ -1458,14 +1165,3 @@ struct rte_table_ops rte_table_hash_key8_ext_ops = { .f_lookup = rte_table_hash_lookup_key8_ext, .f_stats = rte_table_hash_key8_stats_read, }; - -struct rte_table_ops rte_table_hash_key8_ext_dosig_ops = { - .f_create = rte_table_hash_create_key8_ext, - .f_free = rte_table_hash_free_key8_ext, - .f_add = rte_table_hash_entry_add_key8_ext, - .f_delete = rte_table_hash_entry_delete_key8_ext, - .f_add_bulk = NULL, - .f_delete_bulk = NULL, - .f_lookup = rte_table_hash_lookup_key8_ext_dosig, - .f_stats = rte_table_hash_key8_stats_read, -}; diff --git a/dpdk/lib/librte_table/rte_table_hash_lru.c b/dpdk/lib/librte_table/rte_table_hash_lru.c index 407c62ab..a07392fd 100644 --- a/dpdk/lib/librte_table/rte_table_hash_lru.c +++ b/dpdk/lib/librte_table/rte_table_hash_lru.c @@ -1,34 +1,34 @@ /*- - * BSD LICENSE + * BSD LICENSE * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. + * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. + * All rights reserved. * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include @@ -86,7 +86,6 @@ struct rte_table_hash { uint32_t n_buckets; rte_table_hash_op_hash f_hash; uint64_t seed; - uint32_t signature_offset; uint32_t key_offset; /* Internal */ @@ -99,6 +98,7 @@ struct rte_table_hash { struct grinder grinders[RTE_PORT_IN_BURST_SIZE_MAX]; /* Tables */ + uint64_t *key_mask; struct bucket *buckets; uint8_t *key_mem; uint8_t *data_mem; @@ -109,29 +109,53 @@ struct rte_table_hash { }; static int -check_params_create(struct rte_table_hash_lru_params *params) +keycmp(void *a, void *b, void *b_mask, uint32_t n_bytes) { - uint32_t n_buckets_min; + uint64_t *a64 = a, *b64 = b, *b_mask64 = b_mask; + uint32_t i; + + for (i = 0; i < n_bytes / sizeof(uint64_t); i++) + if (a64[i] != (b64[i] & b_mask64[i])) + return 1; + + return 0; +} + +static void +keycpy(void *dst, void *src, void *src_mask, uint32_t n_bytes) +{ + uint64_t *dst64 = dst, *src64 = src, *src_mask64 = src_mask; + uint32_t i; + + for (i = 0; i < n_bytes / sizeof(uint64_t); i++) + dst64[i] = src64[i] & src_mask64[i]; +} + +static int +check_params_create(struct rte_table_hash_params *params) +{ + /* name */ + if (params->name == NULL) { + RTE_LOG(ERR, TABLE, "%s: name invalid value\n", __func__); + return -EINVAL; + } /* key_size */ - if ((params->key_size == 0) || + if ((params->key_size < sizeof(uint64_t)) || (!rte_is_power_of_2(params->key_size))) { RTE_LOG(ERR, TABLE, "%s: key_size invalid value\n", __func__); return -EINVAL; } /* n_keys */ - if ((params->n_keys == 0) || - (!rte_is_power_of_2(params->n_keys))) { + if (params->n_keys == 0) { RTE_LOG(ERR, TABLE, "%s: n_keys invalid value\n", __func__); return -EINVAL; } /* n_buckets */ - n_buckets_min = (params->n_keys + KEYS_PER_BUCKET - 1) / params->n_keys; if ((params->n_buckets == 0) || - (!rte_is_power_of_2(params->n_keys)) || - (params->n_buckets < n_buckets_min)) { + (!rte_is_power_of_2(params->n_buckets))) { RTE_LOG(ERR, TABLE, "%s: n_buckets invalid value\n", __func__); return -EINVAL; } @@ -148,13 +172,13 @@ check_params_create(struct rte_table_hash_lru_params *params) static void * rte_table_hash_lru_create(void *params, int socket_id, uint32_t entry_size) { - struct rte_table_hash_lru_params *p = - (struct rte_table_hash_lru_params *) params; + struct rte_table_hash_params *p = params; struct rte_table_hash *t; - uint32_t total_size, table_meta_sz; - uint32_t bucket_sz, key_sz, key_stack_sz, data_sz; - uint32_t bucket_offset, key_offset, key_stack_offset, data_offset; - uint32_t i; + uint64_t table_meta_sz, key_mask_sz, bucket_sz, key_sz, key_stack_sz; + uint64_t data_sz, total_size; + uint64_t key_mask_offset, bucket_offset, key_offset, key_stack_offset; + uint64_t data_offset; + uint32_t n_buckets, i; /* Check input parameters */ if ((check_params_create(p) != 0) || @@ -164,33 +188,65 @@ rte_table_hash_lru_create(void *params, int socket_id, uint32_t entry_size) return NULL; } + /* + * Table dimensioning + * + * Objective: Pick the number of buckets (n_buckets) so that there a chance + * to store n_keys keys in the table. + * + * Note: Since the buckets do not get extended, it is not possible to + * guarantee that n_keys keys can be stored in the table at any time. In the + * worst case scenario when all the n_keys fall into the same bucket, only + * a maximum of KEYS_PER_BUCKET keys will be stored in the table. This case + * defeats the purpose of the hash table. It indicates unsuitable f_hash or + * n_keys to n_buckets ratio. + * + * MIN(n_buckets) = (n_keys + KEYS_PER_BUCKET - 1) / KEYS_PER_BUCKET + */ + n_buckets = rte_align32pow2( + (p->n_keys + KEYS_PER_BUCKET - 1) / KEYS_PER_BUCKET); + n_buckets = RTE_MAX(n_buckets, p->n_buckets); + /* Memory allocation */ table_meta_sz = RTE_CACHE_LINE_ROUNDUP(sizeof(struct rte_table_hash)); - bucket_sz = RTE_CACHE_LINE_ROUNDUP(p->n_buckets * sizeof(struct bucket)); + key_mask_sz = RTE_CACHE_LINE_ROUNDUP(p->key_size); + bucket_sz = RTE_CACHE_LINE_ROUNDUP(n_buckets * sizeof(struct bucket)); key_sz = RTE_CACHE_LINE_ROUNDUP(p->n_keys * p->key_size); key_stack_sz = RTE_CACHE_LINE_ROUNDUP(p->n_keys * sizeof(uint32_t)); data_sz = RTE_CACHE_LINE_ROUNDUP(p->n_keys * entry_size); - total_size = table_meta_sz + bucket_sz + key_sz + key_stack_sz + - data_sz; + total_size = table_meta_sz + key_mask_sz + bucket_sz + key_sz + + key_stack_sz + data_sz; - t = rte_zmalloc_socket("TABLE", total_size, RTE_CACHE_LINE_SIZE, socket_id); - if (t == NULL) { + if (total_size > SIZE_MAX) { RTE_LOG(ERR, TABLE, - "%s: Cannot allocate %u bytes for hash table\n", - __func__, total_size); + "%s: Cannot allocate %" PRIu64 " bytes for hash " + "table %s\n", + __func__, total_size, p->name); return NULL; } - RTE_LOG(INFO, TABLE, "%s (%u-byte key): Hash table memory footprint is " - "%u bytes\n", __func__, p->key_size, total_size); + + t = rte_zmalloc_socket(p->name, + (size_t)total_size, + RTE_CACHE_LINE_SIZE, + socket_id); + if (t == NULL) { + RTE_LOG(ERR, TABLE, + "%s: Cannot allocate %" PRIu64 " bytes for hash " + "table %s\n", + __func__, total_size, p->name); + return NULL; + } + RTE_LOG(INFO, TABLE, "%s (%u-byte key): Hash table %s memory footprint" + " is %" PRIu64 " bytes\n", + __func__, p->key_size, p->name, total_size); /* Memory initialization */ t->key_size = p->key_size; t->entry_size = entry_size; t->n_keys = p->n_keys; - t->n_buckets = p->n_buckets; + t->n_buckets = n_buckets; t->f_hash = p->f_hash; t->seed = p->seed; - t->signature_offset = p->signature_offset; t->key_offset = p->key_offset; /* Internal */ @@ -199,16 +255,24 @@ rte_table_hash_lru_create(void *params, int socket_id, uint32_t entry_size) t->data_size_shl = __builtin_ctzl(entry_size); /* Tables */ - bucket_offset = 0; + key_mask_offset = 0; + bucket_offset = key_mask_offset + key_mask_sz; key_offset = bucket_offset + bucket_sz; key_stack_offset = key_offset + key_sz; data_offset = key_stack_offset + key_stack_sz; + t->key_mask = (uint64_t *) &t->memory[key_mask_offset]; t->buckets = (struct bucket *) &t->memory[bucket_offset]; t->key_mem = &t->memory[key_offset]; t->key_stack = (uint32_t *) &t->memory[key_stack_offset]; t->data_mem = &t->memory[data_offset]; + /* Key mask */ + if (p->key_mask == NULL) + memset(t->key_mask, 0xFF, p->key_size); + else + memcpy(t->key_mask, p->key_mask, p->key_size); + /* Key stack */ for (i = 0; i < t->n_keys; i++) t->key_stack[i] = t->n_keys - 1 - i; @@ -227,7 +291,7 @@ rte_table_hash_lru_create(void *params, int socket_id, uint32_t entry_size) static int rte_table_hash_lru_free(void *table) { - struct rte_table_hash *t = (struct rte_table_hash *) table; + struct rte_table_hash *t = table; /* Check input parameters */ if (t == NULL) @@ -241,12 +305,12 @@ static int rte_table_hash_lru_entry_add(void *table, void *key, void *entry, int *key_found, void **entry_ptr) { - struct rte_table_hash *t = (struct rte_table_hash *) table; + struct rte_table_hash *t = table; struct bucket *bkt; uint64_t sig; uint32_t bkt_index, i; - sig = t->f_hash(key, t->key_size, t->seed); + sig = t->f_hash(key, t->key_mask, t->key_size, t->seed); bkt_index = sig & t->bucket_mask; bkt = &t->buckets[bkt_index]; sig = (sig >> 16) | 1LLU; @@ -258,8 +322,8 @@ rte_table_hash_lru_entry_add(void *table, void *key, void *entry, uint8_t *bkt_key = &t->key_mem[bkt_key_index << t->key_size_shl]; - if ((sig == bkt_sig) && (memcmp(key, bkt_key, t->key_size) - == 0)) { + if ((sig == bkt_sig) && (keycmp(bkt_key, key, t->key_mask, + t->key_size) == 0)) { uint8_t *data = &t->data_mem[bkt_key_index << t->data_size_shl]; @@ -292,7 +356,7 @@ rte_table_hash_lru_entry_add(void *table, void *key, void *entry, bkt->sig[i] = (uint16_t) sig; bkt->key_pos[i] = bkt_key_index; - memcpy(bkt_key, key, t->key_size); + keycpy(bkt_key, key, t->key_mask, t->key_size); memcpy(data, entry, t->entry_size); lru_update(bkt, i); @@ -311,7 +375,7 @@ rte_table_hash_lru_entry_add(void *table, void *key, void *entry, uint8_t *data = &t->data_mem[bkt_key_index << t->data_size_shl]; bkt->sig[pos] = (uint16_t) sig; - memcpy(bkt_key, key, t->key_size); + keycpy(bkt_key, key, t->key_mask, t->key_size); memcpy(data, entry, t->entry_size); lru_update(bkt, pos); @@ -325,12 +389,12 @@ static int rte_table_hash_lru_entry_delete(void *table, void *key, int *key_found, void *entry) { - struct rte_table_hash *t = (struct rte_table_hash *) table; + struct rte_table_hash *t = table; struct bucket *bkt; uint64_t sig; uint32_t bkt_index, i; - sig = t->f_hash(key, t->key_size, t->seed); + sig = t->f_hash(key, t->key_mask, t->key_size, t->seed); bkt_index = sig & t->bucket_mask; bkt = &t->buckets[bkt_index]; sig = (sig >> 16) | 1LLU; @@ -343,14 +407,15 @@ rte_table_hash_lru_entry_delete(void *table, void *key, int *key_found, t->key_size_shl]; if ((sig == bkt_sig) && - (memcmp(key, bkt_key, t->key_size) == 0)) { + (keycmp(bkt_key, key, t->key_mask, t->key_size) == 0)) { uint8_t *data = &t->data_mem[bkt_key_index << t->data_size_shl]; bkt->sig[i] = 0; t->key_stack[t->key_stack_tos++] = bkt_key_index; *key_found = 1; - memcpy(entry, data, t->entry_size); + if (entry) + memcpy(entry, data, t->entry_size); return 0; } } @@ -365,8 +430,7 @@ static int rte_table_hash_lru_lookup_unoptimized( struct rte_mbuf **pkts, uint64_t pkts_mask, uint64_t *lookup_hit_mask, - void **entries, - int dosig) + void **entries) { struct rte_table_hash *t = (struct rte_table_hash *) table; uint64_t pkts_mask_out = 0; @@ -387,11 +451,7 @@ static int rte_table_hash_lru_lookup_unoptimized( pkt = pkts[pkt_index]; key = RTE_MBUF_METADATA_UINT8_PTR(pkt, t->key_offset); - if (dosig) - sig = (uint64_t) t->f_hash(key, t->key_size, t->seed); - else - sig = RTE_MBUF_METADATA_UINT32(pkt, - t->signature_offset); + sig = (uint64_t) t->f_hash(key, t->key_mask, t->key_size, t->seed); bkt_index = sig & t->bucket_mask; bkt = &t->buckets[bkt_index]; @@ -404,7 +464,7 @@ static int rte_table_hash_lru_lookup_unoptimized( uint8_t *bkt_key = &t->key_mem[bkt_key_index << t->key_size_shl]; - if ((sig == bkt_sig) && (memcmp(key, bkt_key, + if ((sig == bkt_sig) && (keycmp(bkt_key, key, t->key_mask, t->key_size) == 0)) { uint8_t *data = &t->data_mem[bkt_key_index << t->data_size_shl]; @@ -502,74 +562,75 @@ static int rte_table_hash_lru_lookup_unoptimized( match_pos = (LUT_MATCH_POS >> (mask_all << 1)) & 3; \ } -#define lookup_cmp_key(mbuf, key, match_key, f) \ -{ \ +#define lookup_cmp_key(mbuf, key, match_key, f) \ +{ \ uint64_t *pkt_key = RTE_MBUF_METADATA_UINT64_PTR(mbuf, f->key_offset);\ - uint64_t *bkt_key = (uint64_t *) key; \ - \ - switch (f->key_size) { \ - case 8: \ - { \ - uint64_t xor = pkt_key[0] ^ bkt_key[0]; \ - match_key = 0; \ - if (xor == 0) \ - match_key = 1; \ - } \ - break; \ - \ - case 16: \ - { \ - uint64_t xor[2], or; \ - \ - xor[0] = pkt_key[0] ^ bkt_key[0]; \ - xor[1] = pkt_key[1] ^ bkt_key[1]; \ - or = xor[0] | xor[1]; \ - match_key = 0; \ - if (or == 0) \ - match_key = 1; \ - } \ - break; \ - \ - case 32: \ - { \ - uint64_t xor[4], or; \ - \ - xor[0] = pkt_key[0] ^ bkt_key[0]; \ - xor[1] = pkt_key[1] ^ bkt_key[1]; \ - xor[2] = pkt_key[2] ^ bkt_key[2]; \ - xor[3] = pkt_key[3] ^ bkt_key[3]; \ - or = xor[0] | xor[1] | xor[2] | xor[3]; \ - match_key = 0; \ - if (or == 0) \ - match_key = 1; \ - } \ - break; \ - \ - case 64: \ - { \ - uint64_t xor[8], or; \ - \ - xor[0] = pkt_key[0] ^ bkt_key[0]; \ - xor[1] = pkt_key[1] ^ bkt_key[1]; \ - xor[2] = pkt_key[2] ^ bkt_key[2]; \ - xor[3] = pkt_key[3] ^ bkt_key[3]; \ - xor[4] = pkt_key[4] ^ bkt_key[4]; \ - xor[5] = pkt_key[5] ^ bkt_key[5]; \ - xor[6] = pkt_key[6] ^ bkt_key[6]; \ - xor[7] = pkt_key[7] ^ bkt_key[7]; \ - or = xor[0] | xor[1] | xor[2] | xor[3] | \ - xor[4] | xor[5] | xor[6] | xor[7]; \ - match_key = 0; \ - if (or == 0) \ - match_key = 1; \ - } \ - break; \ - \ - default: \ - match_key = 0; \ - if (memcmp(pkt_key, bkt_key, f->key_size) == 0) \ - match_key = 1; \ - } \ + uint64_t *bkt_key = (uint64_t *) key; \ + uint64_t *key_mask = f->key_mask; \ + \ + switch (f->key_size) { \ + case 8: \ + { \ + uint64_t xor = (pkt_key[0] & key_mask[0]) ^ bkt_key[0]; \ + match_key = 0; \ + if (xor == 0) \ + match_key = 1; \ + } \ + break; \ + \ + case 16: \ + { \ + uint64_t xor[2], or; \ + \ + xor[0] = (pkt_key[0] & key_mask[0]) ^ bkt_key[0]; \ + xor[1] = (pkt_key[1] & key_mask[1]) ^ bkt_key[1]; \ + or = xor[0] | xor[1]; \ + match_key = 0; \ + if (or == 0) \ + match_key = 1; \ + } \ + break; \ + \ + case 32: \ + { \ + uint64_t xor[4], or; \ + \ + xor[0] = (pkt_key[0] & key_mask[0]) ^ bkt_key[0]; \ + xor[1] = (pkt_key[1] & key_mask[1]) ^ bkt_key[1]; \ + xor[2] = (pkt_key[2] & key_mask[2]) ^ bkt_key[2]; \ + xor[3] = (pkt_key[3] & key_mask[3]) ^ bkt_key[3]; \ + or = xor[0] | xor[1] | xor[2] | xor[3]; \ + match_key = 0; \ + if (or == 0) \ + match_key = 1; \ + } \ + break; \ + \ + case 64: \ + { \ + uint64_t xor[8], or; \ + \ + xor[0] = (pkt_key[0] & key_mask[0]) ^ bkt_key[0]; \ + xor[1] = (pkt_key[1] & key_mask[1]) ^ bkt_key[1]; \ + xor[2] = (pkt_key[2] & key_mask[2]) ^ bkt_key[2]; \ + xor[3] = (pkt_key[3] & key_mask[3]) ^ bkt_key[3]; \ + xor[4] = (pkt_key[4] & key_mask[4]) ^ bkt_key[4]; \ + xor[5] = (pkt_key[5] & key_mask[5]) ^ bkt_key[5]; \ + xor[6] = (pkt_key[6] & key_mask[6]) ^ bkt_key[6]; \ + xor[7] = (pkt_key[7] & key_mask[7]) ^ bkt_key[7]; \ + or = xor[0] | xor[1] | xor[2] | xor[3] | \ + xor[4] | xor[5] | xor[6] | xor[7]; \ + match_key = 0; \ + if (or == 0) \ + match_key = 1; \ + } \ + break; \ + \ + default: \ + match_key = 0; \ + if (keycmp(bkt_key, pkt_key, key_mask, f->key_size) == 0) \ + match_key = 1; \ + } \ } #define lookup2_stage0(t, g, pkts, pkts_mask, pkt00_index, pkt01_index)\ @@ -616,38 +677,7 @@ static int rte_table_hash_lru_lookup_unoptimized( rte_prefetch0(RTE_MBUF_METADATA_UINT8_PTR(mbuf01, key_offset));\ } -#define lookup2_stage1(t, g, pkts, pkt10_index, pkt11_index) \ -{ \ - struct grinder *g10, *g11; \ - uint64_t sig10, sig11, bkt10_index, bkt11_index; \ - struct rte_mbuf *mbuf10, *mbuf11; \ - struct bucket *bkt10, *bkt11, *buckets = t->buckets; \ - uint64_t bucket_mask = t->bucket_mask; \ - uint32_t signature_offset = t->signature_offset; \ - \ - mbuf10 = pkts[pkt10_index]; \ - sig10 = (uint64_t) RTE_MBUF_METADATA_UINT32(mbuf10, signature_offset);\ - bkt10_index = sig10 & bucket_mask; \ - bkt10 = &buckets[bkt10_index]; \ - \ - mbuf11 = pkts[pkt11_index]; \ - sig11 = (uint64_t) RTE_MBUF_METADATA_UINT32(mbuf11, signature_offset);\ - bkt11_index = sig11 & bucket_mask; \ - bkt11 = &buckets[bkt11_index]; \ - \ - rte_prefetch0(bkt10); \ - rte_prefetch0(bkt11); \ - \ - g10 = &g[pkt10_index]; \ - g10->sig = sig10; \ - g10->bkt = bkt10; \ - \ - g11 = &g[pkt11_index]; \ - g11->sig = sig11; \ - g11->bkt = bkt11; \ -} - -#define lookup2_stage1_dosig(t, g, pkts, pkt10_index, pkt11_index)\ +#define lookup2_stage1(t, g, pkts, pkt10_index, pkt11_index)\ { \ struct grinder *g10, *g11; \ uint64_t sig10, sig11, bkt10_index, bkt11_index; \ @@ -662,13 +692,13 @@ static int rte_table_hash_lru_lookup_unoptimized( \ mbuf10 = pkts[pkt10_index]; \ key10 = RTE_MBUF_METADATA_UINT8_PTR(mbuf10, key_offset);\ - sig10 = (uint64_t) f_hash(key10, key_size, seed); \ + sig10 = (uint64_t) f_hash(key10, t->key_mask, key_size, seed);\ bkt10_index = sig10 & bucket_mask; \ bkt10 = &buckets[bkt10_index]; \ \ mbuf11 = pkts[pkt11_index]; \ key11 = RTE_MBUF_METADATA_UINT8_PTR(mbuf11, key_offset);\ - sig11 = (uint64_t) f_hash(key11, key_size, seed); \ + sig11 = (uint64_t) f_hash(key11, t->key_mask, key_size, seed);\ bkt11_index = sig11 & bucket_mask; \ bkt11 = &buckets[bkt11_index]; \ \ @@ -819,7 +849,7 @@ static int rte_table_hash_lru_lookup( /* Cannot run the pipeline with less than 7 packets */ if (__builtin_popcountll(pkts_mask) < 7) return rte_table_hash_lru_lookup_unoptimized(table, pkts, - pkts_mask, lookup_hit_mask, entries, 0); + pkts_mask, lookup_hit_mask, entries); /* Pipeline stage 0 */ lookup2_stage0(t, g, pkts, pkts_mask, pkt00_index, pkt01_index); @@ -923,140 +953,7 @@ static int rte_table_hash_lru_lookup( uint64_t pkts_mask_out_slow = 0; status = rte_table_hash_lru_lookup_unoptimized(table, pkts, - pkts_mask_match_many, &pkts_mask_out_slow, entries, 0); - pkts_mask_out |= pkts_mask_out_slow; - } - - *lookup_hit_mask = pkts_mask_out; - RTE_TABLE_HASH_LRU_STATS_PKTS_LOOKUP_MISS(t, n_pkts_in - __builtin_popcountll(pkts_mask_out)); - return status; -} - -static int rte_table_hash_lru_lookup_dosig( - void *table, - struct rte_mbuf **pkts, - uint64_t pkts_mask, - uint64_t *lookup_hit_mask, - void **entries) -{ - struct rte_table_hash *t = (struct rte_table_hash *) table; - struct grinder *g = t->grinders; - uint64_t pkt00_index, pkt01_index, pkt10_index, pkt11_index; - uint64_t pkt20_index, pkt21_index, pkt30_index, pkt31_index; - uint64_t pkts_mask_out = 0, pkts_mask_match_many = 0; - int status = 0; - - __rte_unused uint32_t n_pkts_in = __builtin_popcountll(pkts_mask); - RTE_TABLE_HASH_LRU_STATS_PKTS_IN_ADD(t, n_pkts_in); - - /* Cannot run the pipeline with less than 7 packets */ - if (__builtin_popcountll(pkts_mask) < 7) - return rte_table_hash_lru_lookup_unoptimized(table, pkts, - pkts_mask, lookup_hit_mask, entries, 1); - - /* Pipeline stage 0 */ - lookup2_stage0(t, g, pkts, pkts_mask, pkt00_index, pkt01_index); - - /* Pipeline feed */ - pkt10_index = pkt00_index; - pkt11_index = pkt01_index; - - /* Pipeline stage 0 */ - lookup2_stage0(t, g, pkts, pkts_mask, pkt00_index, pkt01_index); - - /* Pipeline stage 1 */ - lookup2_stage1_dosig(t, g, pkts, pkt10_index, pkt11_index); - - /* Pipeline feed */ - pkt20_index = pkt10_index; - pkt21_index = pkt11_index; - pkt10_index = pkt00_index; - pkt11_index = pkt01_index; - - /* Pipeline stage 0 */ - lookup2_stage0(t, g, pkts, pkts_mask, pkt00_index, pkt01_index); - - /* Pipeline stage 1 */ - lookup2_stage1_dosig(t, g, pkts, pkt10_index, pkt11_index); - - /* Pipeline stage 2 */ - lookup2_stage2(t, g, pkt20_index, pkt21_index, pkts_mask_match_many); - - /* - * Pipeline run - * - */ - for ( ; pkts_mask; ) { - /* Pipeline feed */ - pkt30_index = pkt20_index; - pkt31_index = pkt21_index; - pkt20_index = pkt10_index; - pkt21_index = pkt11_index; - pkt10_index = pkt00_index; - pkt11_index = pkt01_index; - - /* Pipeline stage 0 */ - lookup2_stage0_with_odd_support(t, g, pkts, pkts_mask, - pkt00_index, pkt01_index); - - /* Pipeline stage 1 */ - lookup2_stage1_dosig(t, g, pkts, pkt10_index, pkt11_index); - - /* Pipeline stage 2 */ - lookup2_stage2(t, g, pkt20_index, pkt21_index, - pkts_mask_match_many); - - /* Pipeline stage 3 */ - lookup2_stage3(t, g, pkts, pkt30_index, pkt31_index, - pkts_mask_out, entries); - } - - /* Pipeline feed */ - pkt30_index = pkt20_index; - pkt31_index = pkt21_index; - pkt20_index = pkt10_index; - pkt21_index = pkt11_index; - pkt10_index = pkt00_index; - pkt11_index = pkt01_index; - - /* Pipeline stage 1 */ - lookup2_stage1_dosig(t, g, pkts, pkt10_index, pkt11_index); - - /* Pipeline stage 2 */ - lookup2_stage2(t, g, pkt20_index, pkt21_index, pkts_mask_match_many); - - /* Pipeline stage 3 */ - lookup2_stage3(t, g, pkts, pkt30_index, pkt31_index, pkts_mask_out, - entries); - - /* Pipeline feed */ - pkt30_index = pkt20_index; - pkt31_index = pkt21_index; - pkt20_index = pkt10_index; - pkt21_index = pkt11_index; - - /* Pipeline stage 2 */ - lookup2_stage2(t, g, pkt20_index, pkt21_index, pkts_mask_match_many); - - /* Pipeline stage 3 */ - lookup2_stage3(t, g, pkts, pkt30_index, pkt31_index, pkts_mask_out, - entries); - - /* Pipeline feed */ - pkt30_index = pkt20_index; - pkt31_index = pkt21_index; - - /* Pipeline stage 3 */ - lookup2_stage3(t, g, pkts, pkt30_index, pkt31_index, pkts_mask_out, - entries); - - /* Slow path */ - pkts_mask_match_many &= ~pkts_mask_out; - if (pkts_mask_match_many) { - uint64_t pkts_mask_out_slow = 0; - - status = rte_table_hash_lru_lookup_unoptimized(table, pkts, - pkts_mask_match_many, &pkts_mask_out_slow, entries, 1); + pkts_mask_match_many, &pkts_mask_out_slow, entries); pkts_mask_out |= pkts_mask_out_slow; } @@ -1068,7 +965,7 @@ static int rte_table_hash_lru_lookup_dosig( static int rte_table_hash_lru_stats_read(void *table, struct rte_table_stats *stats, int clear) { - struct rte_table_hash *t = (struct rte_table_hash *) table; + struct rte_table_hash *t = table; if (stats != NULL) memcpy(stats, &t->stats, sizeof(t->stats)); @@ -1089,14 +986,3 @@ struct rte_table_ops rte_table_hash_lru_ops = { .f_lookup = rte_table_hash_lru_lookup, .f_stats = rte_table_hash_lru_stats_read, }; - -struct rte_table_ops rte_table_hash_lru_dosig_ops = { - .f_create = rte_table_hash_lru_create, - .f_free = rte_table_hash_lru_free, - .f_add = rte_table_hash_lru_entry_add, - .f_delete = rte_table_hash_lru_entry_delete, - .f_add_bulk = NULL, - .f_delete_bulk = NULL, - .f_lookup = rte_table_hash_lru_lookup_dosig, - .f_stats = rte_table_hash_lru_stats_read, -}; diff --git a/dpdk/lib/librte_table/rte_table_lpm.c b/dpdk/lib/librte_table/rte_table_lpm.c index 598b79f5..2f472b97 100644 --- a/dpdk/lib/librte_table/rte_table_lpm.c +++ b/dpdk/lib/librte_table/rte_table_lpm.c @@ -82,7 +82,7 @@ struct rte_table_lpm { static void * rte_table_lpm_create(void *params, int socket_id, uint32_t entry_size) { - struct rte_table_lpm_params *p = (struct rte_table_lpm_params *) params; + struct rte_table_lpm_params *p = params; struct rte_table_lpm *lpm; struct rte_lpm_config lpm_config; @@ -154,7 +154,7 @@ rte_table_lpm_create(void *params, int socket_id, uint32_t entry_size) static int rte_table_lpm_free(void *table) { - struct rte_table_lpm *lpm = (struct rte_table_lpm *) table; + struct rte_table_lpm *lpm = table; /* Check input parameters */ if (lpm == NULL) { @@ -210,8 +210,8 @@ rte_table_lpm_entry_add( int *key_found, void **entry_ptr) { - struct rte_table_lpm *lpm = (struct rte_table_lpm *) table; - struct rte_table_lpm_key *ip_prefix = (struct rte_table_lpm_key *) key; + struct rte_table_lpm *lpm = table; + struct rte_table_lpm_key *ip_prefix = key; uint32_t nht_pos, nht_pos0_valid; int status; uint32_t nht_pos0 = 0; @@ -277,8 +277,8 @@ rte_table_lpm_entry_delete( int *key_found, void *entry) { - struct rte_table_lpm *lpm = (struct rte_table_lpm *) table; - struct rte_table_lpm_key *ip_prefix = (struct rte_table_lpm_key *) key; + struct rte_table_lpm *lpm = table; + struct rte_table_lpm_key *ip_prefix = key; uint32_t nht_pos; int status; @@ -372,7 +372,7 @@ rte_table_lpm_lookup( static int rte_table_lpm_stats_read(void *table, struct rte_table_stats *stats, int clear) { - struct rte_table_lpm *t = (struct rte_table_lpm *) table; + struct rte_table_lpm *t = table; if (stats != NULL) memcpy(stats, &t->stats, sizeof(t->stats)); diff --git a/dpdk/lib/librte_table/rte_table_lpm_ipv6.c b/dpdk/lib/librte_table/rte_table_lpm_ipv6.c index 836f4cf6..5def2ad9 100644 --- a/dpdk/lib/librte_table/rte_table_lpm_ipv6.c +++ b/dpdk/lib/librte_table/rte_table_lpm_ipv6.c @@ -81,7 +81,7 @@ static void * rte_table_lpm_ipv6_create(void *params, int socket_id, uint32_t entry_size) { struct rte_table_lpm_ipv6_params *p = - (struct rte_table_lpm_ipv6_params *) params; + params; struct rte_table_lpm_ipv6 *lpm; struct rte_lpm6_config lpm6_config; uint32_t total_size, nht_size; @@ -152,7 +152,7 @@ rte_table_lpm_ipv6_create(void *params, int socket_id, uint32_t entry_size) static int rte_table_lpm_ipv6_free(void *table) { - struct rte_table_lpm_ipv6 *lpm = (struct rte_table_lpm_ipv6 *) table; + struct rte_table_lpm_ipv6 *lpm = table; /* Check input parameters */ if (lpm == NULL) { @@ -208,12 +208,11 @@ rte_table_lpm_ipv6_entry_add( int *key_found, void **entry_ptr) { - struct rte_table_lpm_ipv6 *lpm = (struct rte_table_lpm_ipv6 *) table; + struct rte_table_lpm_ipv6 *lpm = table; struct rte_table_lpm_ipv6_key *ip_prefix = - (struct rte_table_lpm_ipv6_key *) key; - uint32_t nht_pos, nht_pos0_valid; + key; + uint32_t nht_pos, nht_pos0, nht_pos0_valid; int status; - uint8_t nht_pos0; /* Check input parameters */ if (lpm == NULL) { @@ -256,7 +255,7 @@ rte_table_lpm_ipv6_entry_add( /* Add rule to low level LPM table */ if (rte_lpm6_add(lpm->lpm, ip_prefix->ip, ip_prefix->depth, - (uint8_t) nht_pos) < 0) { + nht_pos) < 0) { RTE_LOG(ERR, TABLE, "%s: LPM IPv6 rule add failed\n", __func__); return -1; } @@ -277,10 +276,10 @@ rte_table_lpm_ipv6_entry_delete( int *key_found, void *entry) { - struct rte_table_lpm_ipv6 *lpm = (struct rte_table_lpm_ipv6 *) table; + struct rte_table_lpm_ipv6 *lpm = table; struct rte_table_lpm_ipv6_key *ip_prefix = - (struct rte_table_lpm_ipv6_key *) key; - uint8_t nht_pos; + key; + uint32_t nht_pos; int status; /* Check input parameters */ @@ -356,7 +355,7 @@ rte_table_lpm_ipv6_lookup( uint8_t *ip = RTE_MBUF_METADATA_UINT8_PTR(pkt, lpm->offset); int status; - uint8_t nht_pos; + uint32_t nht_pos; status = rte_lpm6_lookup(lpm->lpm, ip, &nht_pos); if (status == 0) { @@ -375,7 +374,7 @@ rte_table_lpm_ipv6_lookup( static int rte_table_lpm_ipv6_stats_read(void *table, struct rte_table_stats *stats, int clear) { - struct rte_table_lpm_ipv6 *t = (struct rte_table_lpm_ipv6 *) table; + struct rte_table_lpm_ipv6 *t = table; if (stats != NULL) memcpy(stats, &t->stats, sizeof(t->stats)); diff --git a/dpdk/lib/librte_table/rte_table_stub.c b/dpdk/lib/librte_table/rte_table_stub.c index 691d681a..1ee26bf2 100644 --- a/dpdk/lib/librte_table/rte_table_stub.c +++ b/dpdk/lib/librte_table/rte_table_stub.c @@ -98,7 +98,7 @@ rte_table_stub_lookup( static int rte_table_stub_stats_read(void *table, struct rte_table_stats *stats, int clear) { - struct rte_table_stub *t = (struct rte_table_stub *) table; + struct rte_table_stub *t = table; if (stats != NULL) memcpy(stats, &t->stats, sizeof(t->stats)); diff --git a/dpdk/lib/librte_table/rte_table_version.map b/dpdk/lib/librte_table/rte_table_version.map index 459c2da3..6237252b 100644 --- a/dpdk/lib/librte_table/rte_table_version.map +++ b/dpdk/lib/librte_table/rte_table_version.map @@ -1,19 +1,16 @@ -DPDK_2.0 { +DPDK_17.11 { global: rte_table_acl_ops; rte_table_array_ops; - rte_table_hash_ext_dosig_ops; + rte_table_hash_cuckoo_ops; rte_table_hash_ext_ops; - rte_table_hash_key8_ext_dosig_ops; - rte_table_hash_key8_ext_ops; - rte_table_hash_key8_lru_dosig_ops; - rte_table_hash_key8_lru_ops; rte_table_hash_key16_ext_ops; rte_table_hash_key16_lru_ops; rte_table_hash_key32_ext_ops; rte_table_hash_key32_lru_ops; - rte_table_hash_lru_dosig_ops; + rte_table_hash_key8_ext_ops; + rte_table_hash_key8_lru_ops; rte_table_hash_lru_ops; rte_table_lpm_ipv6_ops; rte_table_lpm_ops; @@ -21,11 +18,3 @@ DPDK_2.0 { local: *; }; - -DPDK_2.2 { - global: - - rte_table_hash_key16_ext_dosig_ops; - rte_table_hash_key16_lru_dosig_ops; - -} DPDK_2.0; diff --git a/dpdk/lib/librte_timer/Makefile b/dpdk/lib/librte_timer/Makefile index 2aabef85..eb9c5624 100644 --- a/dpdk/lib/librte_timer/Makefile +++ b/dpdk/lib/librte_timer/Makefile @@ -35,6 +35,7 @@ include $(RTE_SDK)/mk/rte.vars.mk LIB = librte_timer.a CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -O3 +LDLIBS += -lrte_eal EXPORT_MAP := rte_timer_version.map @@ -46,7 +47,4 @@ SRCS-$(CONFIG_RTE_LIBRTE_TIMER) := rte_timer.c # install this header file SYMLINK-$(CONFIG_RTE_LIBRTE_TIMER)-include := rte_timer.h -# this lib needs eal -DEPDIRS-$(CONFIG_RTE_LIBRTE_TIMER) += lib/librte_eal - include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/lib/librte_timer/rte_timer.c b/dpdk/lib/librte_timer/rte_timer.c index 18782fab..ba436cd8 100644 --- a/dpdk/lib/librte_timer/rte_timer.c +++ b/dpdk/lib/librte_timer/rte_timer.c @@ -43,14 +43,13 @@ #include #include #include -#include #include #include -#include #include #include #include #include +#include #include "rte_timer.h" @@ -183,7 +182,7 @@ timer_set_running_state(struct rte_timer *tim) return -1; /* here, we know that timer is stopped or pending, - * mark it atomically as beeing configured */ + * mark it atomically as being configured */ status.state = RTE_TIMER_RUNNING; status.owner = (int16_t)lcore_id; success = rte_atomic32_cmpset(&tim->status.u32, @@ -196,7 +195,7 @@ timer_set_running_state(struct rte_timer *tim) /* * Return a skiplist level for a new entry. - * This probabalistically gives a level with p=1/4 that an entry at level n + * This probabilistically gives a level with p=1/4 that an entry at level n * will also appear at level n+1. */ static uint32_t @@ -432,7 +431,8 @@ rte_timer_reset(struct rte_timer *tim, uint64_t ticks, uint64_t period; if (unlikely((tim_lcore != (unsigned)LCORE_ID_ANY) && - !rte_lcore_is_enabled(tim_lcore))) + !(rte_lcore_is_enabled(tim_lcore) || + rte_lcore_has_role(tim_lcore, ROLE_SERVICE) == 0))) return -1; if (type == PERIODICAL) @@ -525,7 +525,7 @@ void rte_timer_manage(void) return; cur_time = rte_get_timer_cycles(); -#ifdef RTE_ARCH_X86_64 +#ifdef RTE_ARCH_64 /* on 64-bit the value cached in the pending_head.expired will be * updated atomically, so we can consult that for a quick check here * outside the lock */ diff --git a/dpdk/lib/librte_timer/rte_timer.h b/dpdk/lib/librte_timer/rte_timer.h index 77547c6b..3b50155c 100644 --- a/dpdk/lib/librte_timer/rte_timer.h +++ b/dpdk/lib/librte_timer/rte_timer.h @@ -66,6 +66,8 @@ #include #include #include +#include +#include #ifdef __cplusplus extern "C" { @@ -91,6 +93,7 @@ enum rte_timer_type { * config) and an owner (the id of the lcore that owns the timer). */ union rte_timer_status { + RTE_STD_C11 struct { uint16_t state; /**< Stop, pending, running, config. */ int16_t owner; /**< The lcore that owns the timer. */ diff --git a/dpdk/lib/librte_vhost/Makefile b/dpdk/lib/librte_vhost/Makefile index 538adb0b..be182798 100644 --- a/dpdk/lib/librte_vhost/Makefile +++ b/dpdk/lib/librte_vhost/Makefile @@ -36,37 +36,22 @@ LIB = librte_vhost.a EXPORT_MAP := rte_vhost_version.map -LIBABIVER := 3 +LIBABIVER := 4 CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -O3 -D_FILE_OFFSET_BITS=64 -ifeq ($(CONFIG_RTE_LIBRTE_VHOST_USER),y) CFLAGS += -I vhost_user LDLIBS += -lpthread -else -CFLAGS += -I vhost_cuse -LDLIBS += -lfuse -endif ifeq ($(CONFIG_RTE_LIBRTE_VHOST_NUMA),y) LDLIBS += -lnuma endif +LDLIBS += -lrte_eal -lrte_mempool -lrte_mbuf -lrte_ethdev # all source are stored in SRCS-y -SRCS-$(CONFIG_RTE_LIBRTE_VHOST) := virtio-net.c vhost_rxtx.c -ifeq ($(CONFIG_RTE_LIBRTE_VHOST_USER),y) -SRCS-$(CONFIG_RTE_LIBRTE_VHOST) += vhost_user/vhost-net-user.c vhost_user/virtio-net-user.c vhost_user/fd_man.c -else -SRCS-$(CONFIG_RTE_LIBRTE_VHOST) += vhost_cuse/vhost-net-cdev.c vhost_cuse/virtio-net-cdev.c vhost_cuse/eventfd_copy.c -endif +SRCS-$(CONFIG_RTE_LIBRTE_VHOST) := fd_man.c iotlb.c socket.c vhost.c \ + vhost_user.c virtio_net.c # install includes -SYMLINK-$(CONFIG_RTE_LIBRTE_VHOST)-include += rte_virtio_net.h - -# dependencies -DEPDIRS-$(CONFIG_RTE_LIBRTE_VHOST) += lib/librte_eal -DEPDIRS-$(CONFIG_RTE_LIBRTE_VHOST) += lib/librte_ether -DEPDIRS-$(CONFIG_RTE_LIBRTE_VHOST) += lib/librte_mbuf -DEPDIRS-$(CONFIG_RTE_LIBRTE_VHOST) += lib/librte_mempool -DEPDIRS-$(CONFIG_RTE_LIBRTE_VHOST) += lib/librte_net +SYMLINK-$(CONFIG_RTE_LIBRTE_VHOST)-include += rte_vhost.h include $(RTE_SDK)/mk/rte.lib.mk diff --git a/dpdk/lib/librte_vhost/eventfd_link/Makefile b/dpdk/lib/librte_vhost/eventfd_link/Makefile deleted file mode 100644 index 3140e8bf..00000000 --- a/dpdk/lib/librte_vhost/eventfd_link/Makefile +++ /dev/null @@ -1,41 +0,0 @@ -# BSD LICENSE -# -# Copyright(c) 2010-2014 Intel Corporation. All rights reserved. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in -# the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Intel Corporation nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -RTE_KERNELDIR ?= /lib/modules/$(shell uname -r)/build - -obj-m += eventfd_link.o - - -all: - make -C $(RTE_KERNELDIR) M=$(PWD) modules - -clean: - make -C $(RTE_KERNELDIR) M=$(PWD) clean diff --git a/dpdk/lib/librte_vhost/eventfd_link/eventfd_link.c b/dpdk/lib/librte_vhost/eventfd_link/eventfd_link.c deleted file mode 100644 index 4b05b5a8..00000000 --- a/dpdk/lib/librte_vhost/eventfd_link/eventfd_link.c +++ /dev/null @@ -1,277 +0,0 @@ -/*- - * GPL LICENSE SUMMARY - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of version 2 of the GNU General Public License as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. - * The full GNU General Public License is included in this distribution - * in the file called LICENSE.GPL. - * - * Contact Information: - * Intel Corporation - */ - -#include -#include -#include -#include -#include - -#include "eventfd_link.h" - - -/* - * get_files_struct is copied from fs/file.c - */ -struct files_struct * -get_files_struct(struct task_struct *task) -{ - struct files_struct *files; - - task_lock(task); - files = task->files; - if (files) - atomic_inc(&files->count); - task_unlock(task); - - return files; -} - -/* - * put_files_struct is extracted from fs/file.c - */ -void -put_files_struct(struct files_struct *files) -{ - if (atomic_dec_and_test(&files->count)) - BUG(); -} - -static struct file * -fget_from_files(struct files_struct *files, unsigned fd) -{ - struct file *file; - - rcu_read_lock(); - file = fcheck_files(files, fd); - if (file) { - if (file->f_mode & FMODE_PATH || - !atomic_long_inc_not_zero(&file->f_count)) { - - file = NULL; - } - } - rcu_read_unlock(); - - return file; -} - -static long -eventfd_link_ioctl_copy2(unsigned long arg) -{ - void __user *argp = (void __user *) arg; - struct task_struct *task_target = NULL; - struct file *file; - struct files_struct *files; - struct eventfd_copy2 eventfd_copy2; - long ret = -EFAULT; - - if (copy_from_user(&eventfd_copy2, argp, sizeof(struct eventfd_copy2))) - goto out; - - /* - * Find the task struct for the target pid - */ - ret = -ESRCH; - - task_target = - get_pid_task(find_vpid(eventfd_copy2.pid), PIDTYPE_PID); - if (task_target == NULL) { - pr_info("Unable to find pid %d\n", eventfd_copy2.pid); - goto out; - } - - ret = -ESTALE; - files = get_files_struct(task_target); - if (files == NULL) { - pr_info("Failed to get target files struct\n"); - goto out_task; - } - - ret = -EBADF; - file = fget_from_files(files, eventfd_copy2.fd); - put_files_struct(files); - - if (file == NULL) { - pr_info("Failed to get fd %d from target\n", eventfd_copy2.fd); - goto out_task; - } - - /* - * Install the file struct from the target process into the - * newly allocated file desciptor of the source process. - */ - ret = get_unused_fd_flags(eventfd_copy2.flags); - if (ret < 0) { - fput(file); - goto out_task; - } - fd_install(ret, file); - -out_task: - put_task_struct(task_target); -out: - return ret; -} - -static long -eventfd_link_ioctl_copy(unsigned long arg) -{ - void __user *argp = (void __user *) arg; - struct task_struct *task_target = NULL; - struct file *file; - struct files_struct *files; - struct fdtable *fdt; - struct eventfd_copy eventfd_copy; - long ret = -EFAULT; - - if (copy_from_user(&eventfd_copy, argp, sizeof(struct eventfd_copy))) - goto out; - - /* - * Find the task struct for the target pid - */ - ret = -ESRCH; - - task_target = - get_pid_task(find_vpid(eventfd_copy.target_pid), PIDTYPE_PID); - if (task_target == NULL) { - pr_info("Unable to find pid %d\n", eventfd_copy.target_pid); - goto out; - } - - ret = -ESTALE; - files = get_files_struct(current); - if (files == NULL) { - pr_info("Failed to get current files struct\n"); - goto out_task; - } - - ret = -EBADF; - file = fget_from_files(files, eventfd_copy.source_fd); - - if (file == NULL) { - pr_info("Failed to get fd %d from source\n", - eventfd_copy.source_fd); - put_files_struct(files); - goto out_task; - } - - /* - * Release the existing eventfd in the source process - */ - spin_lock(&files->file_lock); - fput(file); - filp_close(file, files); - fdt = files_fdtable(files); - fdt->fd[eventfd_copy.source_fd] = NULL; - spin_unlock(&files->file_lock); - - put_files_struct(files); - - /* - * Find the file struct associated with the target fd. - */ - - ret = -ESTALE; - files = get_files_struct(task_target); - if (files == NULL) { - pr_info("Failed to get target files struct\n"); - goto out_task; - } - - ret = -EBADF; - file = fget_from_files(files, eventfd_copy.target_fd); - put_files_struct(files); - - if (file == NULL) { - pr_info("Failed to get fd %d from target\n", - eventfd_copy.target_fd); - goto out_task; - } - - /* - * Install the file struct from the target process into the - * file desciptor of the source process, - */ - - fd_install(eventfd_copy.source_fd, file); - ret = 0; - -out_task: - put_task_struct(task_target); -out: - return ret; -} - -static long -eventfd_link_ioctl(struct file *f, unsigned int ioctl, unsigned long arg) -{ - long ret = -ENOIOCTLCMD; - - switch (ioctl) { - case EVENTFD_COPY: - ret = eventfd_link_ioctl_copy(arg); - break; - case EVENTFD_COPY2: - ret = eventfd_link_ioctl_copy2(arg); - break; - } - - return ret; -} - -static const struct file_operations eventfd_link_fops = { - .owner = THIS_MODULE, - .unlocked_ioctl = eventfd_link_ioctl, -}; - - -static struct miscdevice eventfd_link_misc = { - .minor = MISC_DYNAMIC_MINOR, - .name = "eventfd-link", - .fops = &eventfd_link_fops, -}; - -static int __init -eventfd_link_init(void) -{ - return misc_register(&eventfd_link_misc); -} - -module_init(eventfd_link_init); - -static void __exit -eventfd_link_exit(void) -{ - misc_deregister(&eventfd_link_misc); -} - -module_exit(eventfd_link_exit); - -MODULE_VERSION("0.0.1"); -MODULE_LICENSE("GPL v2"); -MODULE_AUTHOR("Anthony Fee"); -MODULE_DESCRIPTION("Link eventfd"); -MODULE_ALIAS("devname:eventfd-link"); diff --git a/dpdk/lib/librte_vhost/eventfd_link/eventfd_link.h b/dpdk/lib/librte_vhost/eventfd_link/eventfd_link.h deleted file mode 100644 index 5ebc20b8..00000000 --- a/dpdk/lib/librte_vhost/eventfd_link/eventfd_link.h +++ /dev/null @@ -1,94 +0,0 @@ -/*- - * This file is provided under a dual BSD/GPLv2 license. When using or - * redistributing this file, you may do so under either license. - * - * GPL LICENSE SUMMARY - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of version 2 of the GNU General Public License as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. - * The full GNU General Public License is included in this distribution - * in the file called LICENSE.GPL. - * - * Contact Information: - * Intel Corporation - * - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -#ifndef _EVENTFD_LINK_H_ -#define _EVENTFD_LINK_H_ - -/* - * arguements for the EVENTFD_COPY ioctl - */ -struct eventfd_copy { - unsigned target_fd; /* fd in the target pid */ - unsigned source_fd; /* fd in the calling pid */ - pid_t target_pid; /* pid of the target pid */ -}; - -/* - * ioctl to copy an fd entry in calling process to an fd in a target process - * NOTE: this one should be - * #define EVENTFD_COPY _IOWR('D', 1, struct eventfd_copy) actually - */ -#define EVENTFD_COPY 1 - -/* - * arguments for the EVENTFD_COPY2 ioctl - */ -struct eventfd_copy2 { - unsigned fd; /* fd to steal */ - pid_t pid; /* pid of the process to steal from */ - unsigned flags; /* flags to allocate new fd with */ -}; - -/* - * ioctl to copy an fd entry from the target process into newly allocated - * fd in the calling process - */ -#define EVENTFD_COPY2 _IOW('D', 2, struct eventfd_copy2) - -#endif /* _EVENTFD_LINK_H_ */ diff --git a/dpdk/lib/librte_vhost/fd_man.c b/dpdk/lib/librte_vhost/fd_man.c new file mode 100644 index 00000000..4c6fed41 --- /dev/null +++ b/dpdk/lib/librte_vhost/fd_man.c @@ -0,0 +1,303 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "fd_man.h" + +#define FDPOLLERR (POLLERR | POLLHUP | POLLNVAL) + +static int +get_last_valid_idx(struct fdset *pfdset, int last_valid_idx) +{ + int i; + + for (i = last_valid_idx; i >= 0 && pfdset->fd[i].fd == -1; i--) + ; + + return i; +} + +static void +fdset_move(struct fdset *pfdset, int dst, int src) +{ + pfdset->fd[dst] = pfdset->fd[src]; + pfdset->rwfds[dst] = pfdset->rwfds[src]; +} + +static void +fdset_shrink_nolock(struct fdset *pfdset) +{ + int i; + int last_valid_idx = get_last_valid_idx(pfdset, pfdset->num - 1); + + for (i = 0; i < last_valid_idx; i++) { + if (pfdset->fd[i].fd != -1) + continue; + + fdset_move(pfdset, i, last_valid_idx); + last_valid_idx = get_last_valid_idx(pfdset, last_valid_idx - 1); + } + pfdset->num = last_valid_idx + 1; +} + +/* + * Find deleted fd entries and remove them + */ +static void +fdset_shrink(struct fdset *pfdset) +{ + pthread_mutex_lock(&pfdset->fd_mutex); + fdset_shrink_nolock(pfdset); + pthread_mutex_unlock(&pfdset->fd_mutex); +} + +/** + * Returns the index in the fdset for a given fd. + * @return + * index for the fd, or -1 if fd isn't in the fdset. + */ +static int +fdset_find_fd(struct fdset *pfdset, int fd) +{ + int i; + + for (i = 0; i < pfdset->num && pfdset->fd[i].fd != fd; i++) + ; + + return i == pfdset->num ? -1 : i; +} + +static void +fdset_add_fd(struct fdset *pfdset, int idx, int fd, + fd_cb rcb, fd_cb wcb, void *dat) +{ + struct fdentry *pfdentry = &pfdset->fd[idx]; + struct pollfd *pfd = &pfdset->rwfds[idx]; + + pfdentry->fd = fd; + pfdentry->rcb = rcb; + pfdentry->wcb = wcb; + pfdentry->dat = dat; + + pfd->fd = fd; + pfd->events = rcb ? POLLIN : 0; + pfd->events |= wcb ? POLLOUT : 0; + pfd->revents = 0; +} + +void +fdset_init(struct fdset *pfdset) +{ + int i; + + if (pfdset == NULL) + return; + + for (i = 0; i < MAX_FDS; i++) { + pfdset->fd[i].fd = -1; + pfdset->fd[i].dat = NULL; + } + pfdset->num = 0; +} + +/** + * Register the fd in the fdset with read/write handler and context. + */ +int +fdset_add(struct fdset *pfdset, int fd, fd_cb rcb, fd_cb wcb, void *dat) +{ + int i; + + if (pfdset == NULL || fd == -1) + return -1; + + pthread_mutex_lock(&pfdset->fd_mutex); + i = pfdset->num < MAX_FDS ? pfdset->num++ : -1; + if (i == -1) { + fdset_shrink_nolock(pfdset); + i = pfdset->num < MAX_FDS ? pfdset->num++ : -1; + if (i == -1) { + pthread_mutex_unlock(&pfdset->fd_mutex); + return -2; + } + } + + fdset_add_fd(pfdset, i, fd, rcb, wcb, dat); + pthread_mutex_unlock(&pfdset->fd_mutex); + + return 0; +} + +/** + * Unregister the fd from the fdset. + * Returns context of a given fd or NULL. + */ +void * +fdset_del(struct fdset *pfdset, int fd) +{ + int i; + void *dat = NULL; + + if (pfdset == NULL || fd == -1) + return NULL; + + do { + pthread_mutex_lock(&pfdset->fd_mutex); + + i = fdset_find_fd(pfdset, fd); + if (i != -1 && pfdset->fd[i].busy == 0) { + /* busy indicates r/wcb is executing! */ + dat = pfdset->fd[i].dat; + pfdset->fd[i].fd = -1; + pfdset->fd[i].rcb = pfdset->fd[i].wcb = NULL; + pfdset->fd[i].dat = NULL; + i = -1; + } + pthread_mutex_unlock(&pfdset->fd_mutex); + } while (i != -1); + + return dat; +} + + +/** + * This functions runs in infinite blocking loop until there is no fd in + * pfdset. It calls corresponding r/w handler if there is event on the fd. + * + * Before the callback is called, we set the flag to busy status; If other + * thread(now rte_vhost_driver_unregister) calls fdset_del concurrently, it + * will wait until the flag is reset to zero(which indicates the callback is + * finished), then it could free the context after fdset_del. + */ +void * +fdset_event_dispatch(void *arg) +{ + int i; + struct pollfd *pfd; + struct fdentry *pfdentry; + fd_cb rcb, wcb; + void *dat; + int fd, numfds; + int remove1, remove2; + int need_shrink; + struct fdset *pfdset = arg; + int val; + + if (pfdset == NULL) + return NULL; + + while (1) { + + /* + * When poll is blocked, other threads might unregister + * listenfds from and register new listenfds into fdset. + * When poll returns, the entries for listenfds in the fdset + * might have been updated. It is ok if there is unwanted call + * for new listenfds. + */ + pthread_mutex_lock(&pfdset->fd_mutex); + numfds = pfdset->num; + pthread_mutex_unlock(&pfdset->fd_mutex); + + val = poll(pfdset->rwfds, numfds, 1000 /* millisecs */); + if (val < 0) + continue; + + need_shrink = 0; + for (i = 0; i < numfds; i++) { + pthread_mutex_lock(&pfdset->fd_mutex); + + pfdentry = &pfdset->fd[i]; + fd = pfdentry->fd; + pfd = &pfdset->rwfds[i]; + + if (fd < 0) { + need_shrink = 1; + pthread_mutex_unlock(&pfdset->fd_mutex); + continue; + } + + if (!pfd->revents) { + pthread_mutex_unlock(&pfdset->fd_mutex); + continue; + } + + remove1 = remove2 = 0; + + rcb = pfdentry->rcb; + wcb = pfdentry->wcb; + dat = pfdentry->dat; + pfdentry->busy = 1; + + pthread_mutex_unlock(&pfdset->fd_mutex); + + if (rcb && pfd->revents & (POLLIN | FDPOLLERR)) + rcb(fd, dat, &remove1); + if (wcb && pfd->revents & (POLLOUT | FDPOLLERR)) + wcb(fd, dat, &remove2); + pfdentry->busy = 0; + /* + * fdset_del needs to check busy flag. + * We don't allow fdset_del to be called in callback + * directly. + */ + /* + * When we are to clean up the fd from fdset, + * because the fd is closed in the cb, + * the old fd val could be reused by when creates new + * listen fd in another thread, we couldn't call + * fd_set_del. + */ + if (remove1 || remove2) { + pfdentry->fd = -1; + need_shrink = 1; + } + } + + if (need_shrink) + fdset_shrink(pfdset); + } + + return NULL; +} diff --git a/dpdk/lib/librte_vhost/fd_man.h b/dpdk/lib/librte_vhost/fd_man.h new file mode 100644 index 00000000..90d34db1 --- /dev/null +++ b/dpdk/lib/librte_vhost/fd_man.h @@ -0,0 +1,69 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _FD_MAN_H_ +#define _FD_MAN_H_ +#include +#include +#include + +#define MAX_FDS 1024 + +typedef void (*fd_cb)(int fd, void *dat, int *remove); + +struct fdentry { + int fd; /* -1 indicates this entry is empty */ + fd_cb rcb; /* callback when this fd is readable. */ + fd_cb wcb; /* callback when this fd is writeable.*/ + void *dat; /* fd context */ + int busy; /* whether this entry is being used in cb. */ +}; + +struct fdset { + struct pollfd rwfds[MAX_FDS]; + struct fdentry fd[MAX_FDS]; + pthread_mutex_t fd_mutex; + int num; /* current fd number of this fdset */ +}; + + +void fdset_init(struct fdset *pfdset); + +int fdset_add(struct fdset *pfdset, int fd, + fd_cb rcb, fd_cb wcb, void *dat); + +void *fdset_del(struct fdset *pfdset, int fd); + +void *fdset_event_dispatch(void *arg); + +#endif diff --git a/dpdk/lib/librte_vhost/iotlb.c b/dpdk/lib/librte_vhost/iotlb.c new file mode 100644 index 00000000..c11ebcaa --- /dev/null +++ b/dpdk/lib/librte_vhost/iotlb.c @@ -0,0 +1,358 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2017 Red Hat, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifdef RTE_LIBRTE_VHOST_NUMA +#include +#endif + +#include + +#include "iotlb.h" +#include "vhost.h" + +struct vhost_iotlb_entry { + TAILQ_ENTRY(vhost_iotlb_entry) next; + + uint64_t iova; + uint64_t uaddr; + uint64_t size; + uint8_t perm; +}; + +#define IOTLB_CACHE_SIZE 2048 + +static void +vhost_user_iotlb_cache_random_evict(struct vhost_virtqueue *vq); + +static void +vhost_user_iotlb_pending_remove_all(struct vhost_virtqueue *vq) +{ + struct vhost_iotlb_entry *node, *temp_node; + + rte_rwlock_write_lock(&vq->iotlb_pending_lock); + + TAILQ_FOREACH_SAFE(node, &vq->iotlb_pending_list, next, temp_node) { + TAILQ_REMOVE(&vq->iotlb_pending_list, node, next); + rte_mempool_put(vq->iotlb_pool, node); + } + + rte_rwlock_write_unlock(&vq->iotlb_pending_lock); +} + +bool +vhost_user_iotlb_pending_miss(struct vhost_virtqueue *vq, uint64_t iova, + uint8_t perm) +{ + struct vhost_iotlb_entry *node; + bool found = false; + + rte_rwlock_read_lock(&vq->iotlb_pending_lock); + + TAILQ_FOREACH(node, &vq->iotlb_pending_list, next) { + if ((node->iova == iova) && (node->perm == perm)) { + found = true; + break; + } + } + + rte_rwlock_read_unlock(&vq->iotlb_pending_lock); + + return found; +} + +void +vhost_user_iotlb_pending_insert(struct vhost_virtqueue *vq, + uint64_t iova, uint8_t perm) +{ + struct vhost_iotlb_entry *node; + int ret; + + ret = rte_mempool_get(vq->iotlb_pool, (void **)&node); + if (ret) { + RTE_LOG(DEBUG, VHOST_CONFIG, "IOTLB pool empty, clear entries\n"); + if (!TAILQ_EMPTY(&vq->iotlb_pending_list)) + vhost_user_iotlb_pending_remove_all(vq); + else + vhost_user_iotlb_cache_random_evict(vq); + ret = rte_mempool_get(vq->iotlb_pool, (void **)&node); + if (ret) { + RTE_LOG(ERR, VHOST_CONFIG, "IOTLB pool still empty, failure\n"); + return; + } + } + + node->iova = iova; + node->perm = perm; + + rte_rwlock_write_lock(&vq->iotlb_pending_lock); + + TAILQ_INSERT_TAIL(&vq->iotlb_pending_list, node, next); + + rte_rwlock_write_unlock(&vq->iotlb_pending_lock); +} + +void +vhost_user_iotlb_pending_remove(struct vhost_virtqueue *vq, + uint64_t iova, uint64_t size, uint8_t perm) +{ + struct vhost_iotlb_entry *node, *temp_node; + + rte_rwlock_write_lock(&vq->iotlb_pending_lock); + + TAILQ_FOREACH_SAFE(node, &vq->iotlb_pending_list, next, temp_node) { + if (node->iova < iova) + continue; + if (node->iova >= iova + size) + continue; + if ((node->perm & perm) != node->perm) + continue; + TAILQ_REMOVE(&vq->iotlb_pending_list, node, next); + rte_mempool_put(vq->iotlb_pool, node); + } + + rte_rwlock_write_unlock(&vq->iotlb_pending_lock); +} + +static void +vhost_user_iotlb_cache_remove_all(struct vhost_virtqueue *vq) +{ + struct vhost_iotlb_entry *node, *temp_node; + + rte_rwlock_write_lock(&vq->iotlb_lock); + + TAILQ_FOREACH_SAFE(node, &vq->iotlb_list, next, temp_node) { + TAILQ_REMOVE(&vq->iotlb_list, node, next); + rte_mempool_put(vq->iotlb_pool, node); + } + + vq->iotlb_cache_nr = 0; + + rte_rwlock_write_unlock(&vq->iotlb_lock); +} + +static void +vhost_user_iotlb_cache_random_evict(struct vhost_virtqueue *vq) +{ + struct vhost_iotlb_entry *node, *temp_node; + int entry_idx; + + rte_rwlock_write_lock(&vq->iotlb_lock); + + entry_idx = rte_rand() % vq->iotlb_cache_nr; + + TAILQ_FOREACH_SAFE(node, &vq->iotlb_list, next, temp_node) { + if (!entry_idx) { + TAILQ_REMOVE(&vq->iotlb_list, node, next); + rte_mempool_put(vq->iotlb_pool, node); + vq->iotlb_cache_nr--; + break; + } + entry_idx--; + } + + rte_rwlock_write_unlock(&vq->iotlb_lock); +} + +void +vhost_user_iotlb_cache_insert(struct vhost_virtqueue *vq, uint64_t iova, + uint64_t uaddr, uint64_t size, uint8_t perm) +{ + struct vhost_iotlb_entry *node, *new_node; + int ret; + + ret = rte_mempool_get(vq->iotlb_pool, (void **)&new_node); + if (ret) { + RTE_LOG(DEBUG, VHOST_CONFIG, "IOTLB pool empty, clear entries\n"); + if (!TAILQ_EMPTY(&vq->iotlb_list)) + vhost_user_iotlb_cache_random_evict(vq); + else + vhost_user_iotlb_pending_remove_all(vq); + ret = rte_mempool_get(vq->iotlb_pool, (void **)&new_node); + if (ret) { + RTE_LOG(ERR, VHOST_CONFIG, "IOTLB pool still empty, failure\n"); + return; + } + } + + new_node->iova = iova; + new_node->uaddr = uaddr; + new_node->size = size; + new_node->perm = perm; + + rte_rwlock_write_lock(&vq->iotlb_lock); + + TAILQ_FOREACH(node, &vq->iotlb_list, next) { + /* + * Entries must be invalidated before being updated. + * So if iova already in list, assume identical. + */ + if (node->iova == new_node->iova) { + rte_mempool_put(vq->iotlb_pool, new_node); + goto unlock; + } else if (node->iova > new_node->iova) { + TAILQ_INSERT_BEFORE(node, new_node, next); + vq->iotlb_cache_nr++; + goto unlock; + } + } + + TAILQ_INSERT_TAIL(&vq->iotlb_list, new_node, next); + vq->iotlb_cache_nr++; + +unlock: + vhost_user_iotlb_pending_remove(vq, iova, size, perm); + + rte_rwlock_write_unlock(&vq->iotlb_lock); + +} + +void +vhost_user_iotlb_cache_remove(struct vhost_virtqueue *vq, + uint64_t iova, uint64_t size) +{ + struct vhost_iotlb_entry *node, *temp_node; + + if (unlikely(!size)) + return; + + rte_rwlock_write_lock(&vq->iotlb_lock); + + TAILQ_FOREACH_SAFE(node, &vq->iotlb_list, next, temp_node) { + /* Sorted list */ + if (unlikely(iova + size < node->iova)) + break; + + if (iova < node->iova + node->size) { + TAILQ_REMOVE(&vq->iotlb_list, node, next); + rte_mempool_put(vq->iotlb_pool, node); + vq->iotlb_cache_nr--; + } + } + + rte_rwlock_write_unlock(&vq->iotlb_lock); +} + +uint64_t +vhost_user_iotlb_cache_find(struct vhost_virtqueue *vq, uint64_t iova, + uint64_t *size, uint8_t perm) +{ + struct vhost_iotlb_entry *node; + uint64_t offset, vva = 0, mapped = 0; + + if (unlikely(!*size)) + goto out; + + TAILQ_FOREACH(node, &vq->iotlb_list, next) { + /* List sorted by iova */ + if (unlikely(iova < node->iova)) + break; + + if (iova >= node->iova + node->size) + continue; + + if (unlikely((perm & node->perm) != perm)) { + vva = 0; + break; + } + + offset = iova - node->iova; + if (!vva) + vva = node->uaddr + offset; + + mapped += node->size - offset; + iova = node->iova + node->size; + + if (mapped >= *size) + break; + } + +out: + /* Only part of the requested chunk is mapped */ + if (unlikely(mapped < *size)) + *size = mapped; + + return vva; +} + +int +vhost_user_iotlb_init(struct virtio_net *dev, int vq_index) +{ + char pool_name[RTE_MEMPOOL_NAMESIZE]; + struct vhost_virtqueue *vq = dev->virtqueue[vq_index]; + int socket = 0; + + if (vq->iotlb_pool) { + /* + * The cache has already been initialized, + * just drop all cached and pending entries. + */ + vhost_user_iotlb_cache_remove_all(vq); + vhost_user_iotlb_pending_remove_all(vq); + } + +#ifdef RTE_LIBRTE_VHOST_NUMA + if (get_mempolicy(&socket, NULL, 0, vq, MPOL_F_NODE | MPOL_F_ADDR) != 0) + socket = 0; +#endif + + rte_rwlock_init(&vq->iotlb_lock); + rte_rwlock_init(&vq->iotlb_pending_lock); + + TAILQ_INIT(&vq->iotlb_list); + TAILQ_INIT(&vq->iotlb_pending_list); + + snprintf(pool_name, sizeof(pool_name), "iotlb_cache_%d_%d", + dev->vid, vq_index); + + /* If already created, free it and recreate */ + vq->iotlb_pool = rte_mempool_lookup(pool_name); + if (vq->iotlb_pool) + rte_mempool_free(vq->iotlb_pool); + + vq->iotlb_pool = rte_mempool_create(pool_name, + IOTLB_CACHE_SIZE, sizeof(struct vhost_iotlb_entry), 0, + 0, 0, NULL, NULL, NULL, socket, + MEMPOOL_F_NO_CACHE_ALIGN | + MEMPOOL_F_SP_PUT | + MEMPOOL_F_SC_GET); + if (!vq->iotlb_pool) { + RTE_LOG(ERR, VHOST_CONFIG, + "Failed to create IOTLB cache pool (%s)\n", + pool_name); + return -1; + } + + vq->iotlb_cache_nr = 0; + + return 0; +} + diff --git a/dpdk/lib/librte_vhost/iotlb.h b/dpdk/lib/librte_vhost/iotlb.h new file mode 100644 index 00000000..e7083e37 --- /dev/null +++ b/dpdk/lib/librte_vhost/iotlb.h @@ -0,0 +1,79 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2017 Red Hat, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef _VHOST_IOTLB_H_ +#define _VHOST_IOTLB_H_ + +#include + +#include "vhost.h" + +static __rte_always_inline void +vhost_user_iotlb_rd_lock(struct vhost_virtqueue *vq) +{ + rte_rwlock_read_lock(&vq->iotlb_lock); +} + +static __rte_always_inline void +vhost_user_iotlb_rd_unlock(struct vhost_virtqueue *vq) +{ + rte_rwlock_read_unlock(&vq->iotlb_lock); +} + +static __rte_always_inline void +vhost_user_iotlb_wr_lock(struct vhost_virtqueue *vq) +{ + rte_rwlock_write_lock(&vq->iotlb_lock); +} + +static __rte_always_inline void +vhost_user_iotlb_wr_unlock(struct vhost_virtqueue *vq) +{ + rte_rwlock_write_unlock(&vq->iotlb_lock); +} + +void vhost_user_iotlb_cache_insert(struct vhost_virtqueue *vq, uint64_t iova, + uint64_t uaddr, uint64_t size, + uint8_t perm); +void vhost_user_iotlb_cache_remove(struct vhost_virtqueue *vq, + uint64_t iova, uint64_t size); +uint64_t vhost_user_iotlb_cache_find(struct vhost_virtqueue *vq, uint64_t iova, + uint64_t *size, uint8_t perm); +bool vhost_user_iotlb_pending_miss(struct vhost_virtqueue *vq, uint64_t iova, + uint8_t perm); +void vhost_user_iotlb_pending_insert(struct vhost_virtqueue *vq, uint64_t iova, + uint8_t perm); +void vhost_user_iotlb_pending_remove(struct vhost_virtqueue *vq, uint64_t iova, + uint64_t size, uint8_t perm); + +int vhost_user_iotlb_init(struct virtio_net *dev, int vq_index); + +#endif /* _VHOST_IOTLB_H_ */ diff --git a/dpdk/lib/librte_vhost/libvirt/qemu-wrap.py b/dpdk/lib/librte_vhost/libvirt/qemu-wrap.py deleted file mode 100644 index e6a2cc9d..00000000 --- a/dpdk/lib/librte_vhost/libvirt/qemu-wrap.py +++ /dev/null @@ -1,387 +0,0 @@ -#!/usr/bin/python -#/* -# * BSD LICENSE -# * -# * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. -# * All rights reserved. -# * -# * Redistribution and use in source and binary forms, with or without -# * modification, are permitted provided that the following conditions -# * are met: -# * -# * * Redistributions of source code must retain the above copyright -# * notice, this list of conditions and the following disclaimer. -# * * Redistributions in binary form must reproduce the above copyright -# * notice, this list of conditions and the following disclaimer in -# * the documentation and/or other materials provided with the -# * distribution. -# * * Neither the name of Intel Corporation nor the names of its -# * contributors may be used to endorse or promote products derived -# * from this software without specific prior written permission. -# * -# * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# */ - -##################################################################### -# This script is designed to modify the call to the QEMU emulator -# to support userspace vhost when starting a guest machine through -# libvirt with vhost enabled. The steps to enable this are as follows -# and should be run as root: -# -# 1. Place this script in a libvirtd's binary search PATH ($PATH) -# A good location would be in the same directory that the QEMU -# binary is located -# -# 2. Ensure that the script has the same owner/group and file -# permissions as the QEMU binary -# -# 3. Update the VM xml file using "virsh edit VM.xml" -# -# 3.a) Set the VM to use the launch script -# -# Set the emulator path contained in the -# tags -# -# e.g replace /usr/bin/qemu-kvm -# with /usr/bin/qemu-wrap.py -# -# 3.b) Set the VM's device's to use vhost-net offload -# -# -# -# -# -# -# 4. Enable libvirt to access our userpace device file by adding it to -# controllers cgroup for libvirtd using the following steps -# -# 4.a) In /etc/libvirt/qemu.conf add/edit the following lines: -# 1) cgroup_controllers = [ ... "devices", ... ] -# 2) clear_emulator_capabilities = 0 -# 3) user = "root" -# 4) group = "root" -# 5) cgroup_device_acl = [ -# "/dev/null", "/dev/full", "/dev/zero", -# "/dev/random", "/dev/urandom", -# "/dev/ptmx", "/dev/kvm", "/dev/kqemu", -# "/dev/rtc", "/dev/hpet", "/dev/net/tun", -# "/dev/", -# "/dev/hugepages", -# ] -# -# 4.b) Disable SELinux or set to permissive mode -# -# 4.c) Mount cgroup device controller -# "mkdir /dev/cgroup" -# "mount -t cgroup none /dev/cgroup -o devices" -# -# 4.d) Set hugetlbfs_mount variable - ( Optional ) -# VMs using userspace vhost must use hugepage backed -# memory. This can be enabled in the libvirt XML -# config by adding a memory backing section to the -# XML config e.g. -# -# -# -# This memory backing section should be added after the -# and sections. This will add -# flags "-mem-prealloc -mem-path " to the QEMU -# command line. The hugetlbfs_mount variable can be used -# to override the default passed through by libvirt. -# -# if "-mem-prealloc" or "-mem-path " are not passed -# through and a vhost device is detected then these options will -# be automatically added by this script. This script will detect -# the system hugetlbfs mount point to be used for . The -# default for this script can be overidden by the -# hugetlbfs_dir variable in the configuration section of this script. -# -# -# 4.e) Restart the libvirtd system process -# e.g. on Fedora "systemctl restart libvirtd.service" -# -# -# 4.f) Edit the Configuration Parameters section of this script -# to point to the correct emulator location and set any -# addition options -# -# The script modifies the libvirtd Qemu call by modifying/adding -# options based on the configuration parameters below. -# NOTE: -# emul_path and us_vhost_path must be set -# All other parameters are optional -##################################################################### - - -############################################# -# Configuration Parameters -############################################# -#Path to QEMU binary -emul_path = "/usr/local/bin/qemu-system-x86_64" - -#Path to userspace vhost device file -# This filename should match the --dev-basename parameters of -# the command used to launch the userspace vhost sample application e.g. -# if the sample app lauch command is: -# ./build/vhost-switch ..... --dev-basename usvhost -# then this variable should be set to: -# us_vhost_path = "/dev/usvhost" -us_vhost_path = "/dev/usvhost" - -#List of additional user defined emulation options. These options will -#be added to all Qemu calls -emul_opts_user = [] - -#List of additional user defined emulation options for vhost only. -#These options will only be added to vhost enabled guests -emul_opts_user_vhost = [] - -#For all VHOST enabled VMs, the VM memory is preallocated from hugetlbfs -# Set this variable to one to enable this option for all VMs -use_huge_all = 0 - -#Instead of autodetecting, override the hugetlbfs directory by setting -#this variable -hugetlbfs_dir = "" - -############################################# - - -############################################# -# ****** Do Not Modify Below this Line ****** -############################################# - -import sys, os, subprocess -import time -import signal - - -#List of open userspace vhost file descriptors -fd_list = [] - -#additional virtio device flags when using userspace vhost -vhost_flags = [ "csum=off", - "gso=off", - "guest_tso4=off", - "guest_tso6=off", - "guest_ecn=off" - ] - -#String of the path to the Qemu process pid -qemu_pid = "/tmp/%d-qemu.pid" % os.getpid() - -############################################# -# Signal haldler to kill Qemu subprocess -############################################# -def kill_qemu_process(signum, stack): - pidfile = open(qemu_pid, 'r') - pid = int(pidfile.read()) - os.killpg(pid, signal.SIGTERM) - pidfile.close() - - -############################################# -# Find the system hugefile mount point. -# Note: -# if multiple hugetlbfs mount points exist -# then the first one found will be used -############################################# -def find_huge_mount(): - - if (len(hugetlbfs_dir)): - return hugetlbfs_dir - - huge_mount = "" - - if (os.access("/proc/mounts", os.F_OK)): - f = open("/proc/mounts", "r") - line = f.readline() - while line: - line_split = line.split(" ") - if line_split[2] == 'hugetlbfs': - huge_mount = line_split[1] - break - line = f.readline() - else: - print "/proc/mounts not found" - exit (1) - - f.close - if len(huge_mount) == 0: - print "Failed to find hugetlbfs mount point" - exit (1) - - return huge_mount - - -############################################# -# Get a userspace Vhost file descriptor -############################################# -def get_vhost_fd(): - - if (os.access(us_vhost_path, os.F_OK)): - fd = os.open( us_vhost_path, os.O_RDWR) - else: - print ("US-Vhost file %s not found" %us_vhost_path) - exit (1) - - return fd - - -############################################# -# Check for vhostfd. if found then replace -# with our own vhost fd and append any vhost -# flags onto the end -############################################# -def modify_netdev_arg(arg): - - global fd_list - vhost_in_use = 0 - s = '' - new_opts = [] - netdev_opts = arg.split(",") - - for opt in netdev_opts: - #check if vhost is used - if "vhost" == opt[:5]: - vhost_in_use = 1 - else: - new_opts.append(opt) - - #if using vhost append vhost options - if vhost_in_use == 1: - #append vhost on option - new_opts.append('vhost=on') - #append vhostfd ption - new_fd = get_vhost_fd() - new_opts.append('vhostfd=' + str(new_fd)) - fd_list.append(new_fd) - - #concatenate all options - for opt in new_opts: - if len(s) > 0: - s+=',' - - s+=opt - - return s - - -############################################# -# Main -############################################# -def main(): - - global fd_list - global vhost_in_use - new_args = [] - num_cmd_args = len(sys.argv) - emul_call = '' - mem_prealloc_set = 0 - mem_path_set = 0 - num = 0; - - #parse the parameters - while (num < num_cmd_args): - arg = sys.argv[num] - - #Check netdev +1 parameter for vhostfd - if arg == '-netdev': - num_vhost_devs = len(fd_list) - new_args.append(arg) - - num+=1 - arg = sys.argv[num] - mod_arg = modify_netdev_arg(arg) - new_args.append(mod_arg) - - #append vhost flags if this is a vhost device - # and -device is the next arg - # i.e -device -opt1,-opt2,...,-opt3,%vhost - if (num_vhost_devs < len(fd_list)): - num+=1 - arg = sys.argv[num] - if arg == '-device': - new_args.append(arg) - num+=1 - new_arg = sys.argv[num] - for flag in vhost_flags: - new_arg = ''.join([new_arg,',',flag]) - new_args.append(new_arg) - else: - new_args.append(arg) - elif arg == '-mem-prealloc': - mem_prealloc_set = 1 - new_args.append(arg) - elif arg == '-mem-path': - mem_path_set = 1 - new_args.append(arg) - - else: - new_args.append(arg) - - num+=1 - - #Set Qemu binary location - emul_call+=emul_path - emul_call+=" " - - #Add prealloc mem options if using vhost and not already added - if ((len(fd_list) > 0) and (mem_prealloc_set == 0)): - emul_call += "-mem-prealloc " - - #Add mempath mem options if using vhost and not already added - if ((len(fd_list) > 0) and (mem_path_set == 0)): - #Detect and add hugetlbfs mount point - mp = find_huge_mount() - mp = "".join(["-mem-path ", mp]) - emul_call += mp - emul_call += " " - - #add user options - for opt in emul_opts_user: - emul_call += opt - emul_call += " " - - #Add add user vhost only options - if len(fd_list) > 0: - for opt in emul_opts_user_vhost: - emul_call += opt - emul_call += " " - - #Add updated libvirt options - iter_args = iter(new_args) - #skip 1st arg i.e. call to this script - next(iter_args) - for arg in iter_args: - emul_call+=str(arg) - emul_call+= " " - - emul_call += "-pidfile %s " % qemu_pid - #Call QEMU - process = subprocess.Popen(emul_call, shell=True, preexec_fn=os.setsid) - - for sig in [signal.SIGTERM, signal.SIGINT, signal.SIGHUP, signal.SIGQUIT]: - signal.signal(sig, kill_qemu_process) - - process.wait() - - #Close usvhost files - for fd in fd_list: - os.close(fd) - #Cleanup temporary files - if os.access(qemu_pid, os.F_OK): - os.remove(qemu_pid) - -if __name__ == "__main__": - main() diff --git a/dpdk/lib/librte_vhost/rte_vhost.h b/dpdk/lib/librte_vhost/rte_vhost.h new file mode 100644 index 00000000..3fc6034d --- /dev/null +++ b/dpdk/lib/librte_vhost/rte_vhost.h @@ -0,0 +1,501 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_VHOST_H_ +#define _RTE_VHOST_H_ + +/** + * @file + * Interface to vhost-user + */ + +#include +#include + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* These are not C++-aware. */ +#include +#include + +#define RTE_VHOST_USER_CLIENT (1ULL << 0) +#define RTE_VHOST_USER_NO_RECONNECT (1ULL << 1) +#define RTE_VHOST_USER_DEQUEUE_ZERO_COPY (1ULL << 2) +#define RTE_VHOST_USER_IOMMU_SUPPORT (1ULL << 3) + +/** + * Information relating to memory regions including offsets to + * addresses in QEMUs memory file. + */ +struct rte_vhost_mem_region { + uint64_t guest_phys_addr; + uint64_t guest_user_addr; + uint64_t host_user_addr; + uint64_t size; + void *mmap_addr; + uint64_t mmap_size; + int fd; +}; + +/** + * Memory structure includes region and mapping information. + */ +struct rte_vhost_memory { + uint32_t nregions; + struct rte_vhost_mem_region regions[]; +}; + +struct rte_vhost_vring { + struct vring_desc *desc; + struct vring_avail *avail; + struct vring_used *used; + uint64_t log_guest_addr; + + int callfd; + int kickfd; + uint16_t size; +}; + +/** + * Device and vring operations. + */ +struct vhost_device_ops { + int (*new_device)(int vid); /**< Add device. */ + void (*destroy_device)(int vid); /**< Remove device. */ + + int (*vring_state_changed)(int vid, uint16_t queue_id, int enable); /**< triggered when a vring is enabled or disabled */ + + /** + * Features could be changed after the feature negotiation. + * For example, VHOST_F_LOG_ALL will be set/cleared at the + * start/end of live migration, respectively. This callback + * is used to inform the application on such change. + */ + int (*features_changed)(int vid, uint64_t features); + + int (*new_connection)(int vid); + void (*destroy_connection)(int vid); + + void *reserved[2]; /**< Reserved for future extension */ +}; + +/** + * Convert guest physical address to host virtual address + * + * This function is deprecated because unsafe. + * New rte_vhost_va_from_guest_pa() should be used instead to ensure + * guest physical ranges are fully and contiguously mapped into + * process virtual address space. + * + * @param mem + * the guest memory regions + * @param gpa + * the guest physical address for querying + * @return + * the host virtual address on success, 0 on failure + */ +__rte_deprecated +static __rte_always_inline uint64_t +rte_vhost_gpa_to_vva(struct rte_vhost_memory *mem, uint64_t gpa) +{ + struct rte_vhost_mem_region *reg; + uint32_t i; + + for (i = 0; i < mem->nregions; i++) { + reg = &mem->regions[i]; + if (gpa >= reg->guest_phys_addr && + gpa < reg->guest_phys_addr + reg->size) { + return gpa - reg->guest_phys_addr + + reg->host_user_addr; + } + } + + return 0; +} + +/** + * Convert guest physical address to host virtual address safely + * + * This variant of rte_vhost_gpa_to_vva() takes care all the + * requested length is mapped and contiguous in process address + * space. + * + * @param mem + * the guest memory regions + * @param gpa + * the guest physical address for querying + * @param len + * the size of the requested area to map, updated with actual size mapped + * @return + * the host virtual address on success, 0 on failure + */ +static __rte_always_inline uint64_t +rte_vhost_va_from_guest_pa(struct rte_vhost_memory *mem, + uint64_t gpa, uint64_t *len) +{ + struct rte_vhost_mem_region *r; + uint32_t i; + + for (i = 0; i < mem->nregions; i++) { + r = &mem->regions[i]; + if (gpa >= r->guest_phys_addr && + gpa < r->guest_phys_addr + r->size) { + + if (unlikely(*len > r->guest_phys_addr + r->size - gpa)) + *len = r->guest_phys_addr + r->size - gpa; + + return gpa - r->guest_phys_addr + + r->host_user_addr; + } + } + *len = 0; + + return 0; +} + +#define RTE_VHOST_NEED_LOG(features) ((features) & (1ULL << VHOST_F_LOG_ALL)) + +/** + * Log the memory write start with given address. + * + * This function only need be invoked when the live migration starts. + * Therefore, we won't need call it at all in the most of time. For + * making the performance impact be minimum, it's suggested to do a + * check before calling it: + * + * if (unlikely(RTE_VHOST_NEED_LOG(features))) + * rte_vhost_log_write(vid, addr, len); + * + * @param vid + * vhost device ID + * @param addr + * the starting address for write + * @param len + * the length to write + */ +void rte_vhost_log_write(int vid, uint64_t addr, uint64_t len); + +/** + * Log the used ring update start at given offset. + * + * Same as rte_vhost_log_write, it's suggested to do a check before + * calling it: + * + * if (unlikely(RTE_VHOST_NEED_LOG(features))) + * rte_vhost_log_used_vring(vid, vring_idx, offset, len); + * + * @param vid + * vhost device ID + * @param vring_idx + * the vring index + * @param offset + * the offset inside the used ring + * @param len + * the length to write + */ +void rte_vhost_log_used_vring(int vid, uint16_t vring_idx, + uint64_t offset, uint64_t len); + +int rte_vhost_enable_guest_notification(int vid, uint16_t queue_id, int enable); + +/** + * Register vhost driver. path could be different for multiple + * instance support. + */ +int rte_vhost_driver_register(const char *path, uint64_t flags); + +/* Unregister vhost driver. This is only meaningful to vhost user. */ +int rte_vhost_driver_unregister(const char *path); + +/** + * Set the feature bits the vhost-user driver supports. + * + * @param path + * The vhost-user socket file path + * @param features + * Supported features + * @return + * 0 on success, -1 on failure + */ +int rte_vhost_driver_set_features(const char *path, uint64_t features); + +/** + * Enable vhost-user driver features. + * + * Note that + * - the param features should be a subset of the feature bits provided + * by rte_vhost_driver_set_features(). + * - it must be invoked before vhost-user negotiation starts. + * + * @param path + * The vhost-user socket file path + * @param features + * Features to enable + * @return + * 0 on success, -1 on failure + */ +int rte_vhost_driver_enable_features(const char *path, uint64_t features); + +/** + * Disable vhost-user driver features. + * + * The two notes at rte_vhost_driver_enable_features() also apply here. + * + * @param path + * The vhost-user socket file path + * @param features + * Features to disable + * @return + * 0 on success, -1 on failure + */ +int rte_vhost_driver_disable_features(const char *path, uint64_t features); + +/** + * Get the feature bits before feature negotiation. + * + * @param path + * The vhost-user socket file path + * @param features + * A pointer to store the queried feature bits + * @return + * 0 on success, -1 on failure + */ +int rte_vhost_driver_get_features(const char *path, uint64_t *features); + +/** + * Get the feature bits after negotiation + * + * @param vid + * Vhost device ID + * @param features + * A pointer to store the queried feature bits + * @return + * 0 on success, -1 on failure + */ +int rte_vhost_get_negotiated_features(int vid, uint64_t *features); + +/* Register callbacks. */ +int rte_vhost_driver_callback_register(const char *path, + struct vhost_device_ops const * const ops); + +/** + * + * Start the vhost-user driver. + * + * This function triggers the vhost-user negotiation. + * + * @param path + * The vhost-user socket file path + * @return + * 0 on success, -1 on failure + */ +int rte_vhost_driver_start(const char *path); + +/** + * Get the MTU value of the device if set in QEMU. + * + * @param vid + * virtio-net device ID + * @param mtu + * The variable to store the MTU value + * + * @return + * 0: success + * -EAGAIN: device not yet started + * -ENOTSUP: device does not support MTU feature + */ +int rte_vhost_get_mtu(int vid, uint16_t *mtu); + +/** + * Get the numa node from which the virtio net device's memory + * is allocated. + * + * @param vid + * vhost device ID + * + * @return + * The numa node, -1 on failure + */ +int rte_vhost_get_numa_node(int vid); + +/** + * @deprecated + * Get the number of queues the device supports. + * + * Note this function is deprecated, as it returns a queue pair number, + * which is vhost specific. Instead, rte_vhost_get_vring_num should + * be used. + * + * @param vid + * vhost device ID + * + * @return + * The number of queues, 0 on failure + */ +__rte_deprecated +uint32_t rte_vhost_get_queue_num(int vid); + +/** + * Get the number of vrings the device supports. + * + * @param vid + * vhost device ID + * + * @return + * The number of vrings, 0 on failure + */ +uint16_t rte_vhost_get_vring_num(int vid); + +/** + * Get the virtio net device's ifname, which is the vhost-user socket + * file path. + * + * @param vid + * vhost device ID + * @param buf + * The buffer to stored the queried ifname + * @param len + * The length of buf + * + * @return + * 0 on success, -1 on failure + */ +int rte_vhost_get_ifname(int vid, char *buf, size_t len); + +/** + * Get how many avail entries are left in the queue + * + * @param vid + * vhost device ID + * @param queue_id + * virtio queue index + * + * @return + * num of avail entires left + */ +uint16_t rte_vhost_avail_entries(int vid, uint16_t queue_id); + +struct rte_mbuf; +struct rte_mempool; +/** + * This function adds buffers to the virtio devices RX virtqueue. Buffers can + * be received from the physical port or from another virtual device. A packet + * count is returned to indicate the number of packets that were successfully + * added to the RX queue. + * @param vid + * vhost device ID + * @param queue_id + * virtio queue index in mq case + * @param pkts + * array to contain packets to be enqueued + * @param count + * packets num to be enqueued + * @return + * num of packets enqueued + */ +uint16_t rte_vhost_enqueue_burst(int vid, uint16_t queue_id, + struct rte_mbuf **pkts, uint16_t count); + +/** + * This function gets guest buffers from the virtio device TX virtqueue, + * construct host mbufs, copies guest buffer content to host mbufs and + * store them in pkts to be processed. + * @param vid + * vhost device ID + * @param queue_id + * virtio queue index in mq case + * @param mbuf_pool + * mbuf_pool where host mbuf is allocated. + * @param pkts + * array to contain packets to be dequeued + * @param count + * packets num to be dequeued + * @return + * num of packets dequeued + */ +uint16_t rte_vhost_dequeue_burst(int vid, uint16_t queue_id, + struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count); + +/** + * Get guest mem table: a list of memory regions. + * + * An rte_vhost_vhost_memory object will be allocated internaly, to hold the + * guest memory regions. Application should free it at destroy_device() + * callback. + * + * @param vid + * vhost device ID + * @param mem + * To store the returned mem regions + * @return + * 0 on success, -1 on failure + */ +int rte_vhost_get_mem_table(int vid, struct rte_vhost_memory **mem); + +/** + * Get guest vring info, including the vring address, vring size, etc. + * + * @param vid + * vhost device ID + * @param vring_idx + * vring index + * @param vring + * the structure to hold the requested vring info + * @return + * 0 on success, -1 on failure + */ +int rte_vhost_get_vhost_vring(int vid, uint16_t vring_idx, + struct rte_vhost_vring *vring); + +/** + * Get vhost RX queue avail count. + * + * @param vid + * vhost device ID + * @param qid + * virtio queue index in mq case + * @return + * num of desc available + */ +uint32_t rte_vhost_rx_queue_count(int vid, uint16_t qid); + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_VHOST_H_ */ diff --git a/dpdk/lib/librte_vhost/rte_vhost_version.map b/dpdk/lib/librte_vhost/rte_vhost_version.map index 5ceaa8a5..9cb1d8ca 100644 --- a/dpdk/lib/librte_vhost/rte_vhost_version.map +++ b/dpdk/lib/librte_vhost/rte_vhost_version.map @@ -4,12 +4,8 @@ DPDK_2.0 { rte_vhost_dequeue_burst; rte_vhost_driver_callback_register; rte_vhost_driver_register; - rte_vhost_driver_session_start; rte_vhost_enable_guest_notification; rte_vhost_enqueue_burst; - rte_vhost_feature_disable; - rte_vhost_feature_enable; - rte_vhost_feature_get; local: *; }; @@ -30,3 +26,35 @@ DPDK_16.07 { rte_vhost_get_queue_num; } DPDK_2.1; + +DPDK_17.05 { + global: + + rte_vhost_driver_disable_features; + rte_vhost_driver_enable_features; + rte_vhost_driver_get_features; + rte_vhost_driver_set_features; + rte_vhost_driver_start; + rte_vhost_get_mem_table; + rte_vhost_get_mtu; + rte_vhost_get_negotiated_features; + rte_vhost_get_vhost_vring; + rte_vhost_get_vring_num; + rte_vhost_gpa_to_vva; + rte_vhost_log_used_vring; + rte_vhost_log_write; + +} DPDK_16.07; + +DPDK_17.08 { + global: + + rte_vhost_rx_queue_count; + +} DPDK_17.05; + +DPDK_17.11.2 { + global; + + rte_vhost_va_from_guest_pa; +} DPDK_17.08; diff --git a/dpdk/lib/librte_vhost/rte_virtio_net.h b/dpdk/lib/librte_vhost/rte_virtio_net.h deleted file mode 100644 index 9caa6221..00000000 --- a/dpdk/lib/librte_vhost/rte_virtio_net.h +++ /dev/null @@ -1,197 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef _VIRTIO_NET_H_ -#define _VIRTIO_NET_H_ - -/** - * @file - * Interface to vhost net - */ - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#define RTE_VHOST_USER_CLIENT (1ULL << 0) -#define RTE_VHOST_USER_NO_RECONNECT (1ULL << 1) - -/* Enum for virtqueue management. */ -enum {VIRTIO_RXQ, VIRTIO_TXQ, VIRTIO_QNUM}; - -/** - * Device and vring operations. - * - * Make sure to set VIRTIO_DEV_RUNNING to the device flags in new_device and - * remove it in destroy_device. - * - */ -struct virtio_net_device_ops { - int (*new_device)(int vid); /**< Add device. */ - void (*destroy_device)(int vid); /**< Remove device. */ - - int (*vring_state_changed)(int vid, uint16_t queue_id, int enable); /**< triggered when a vring is enabled or disabled */ - - void *reserved[5]; /**< Reserved for future extension */ -}; - -/** - * Disable features in feature_mask. Returns 0 on success. - */ -int rte_vhost_feature_disable(uint64_t feature_mask); - -/** - * Enable features in feature_mask. Returns 0 on success. - */ -int rte_vhost_feature_enable(uint64_t feature_mask); - -/* Returns currently supported vhost features */ -uint64_t rte_vhost_feature_get(void); - -int rte_vhost_enable_guest_notification(int vid, uint16_t queue_id, int enable); - -/** - * Register vhost driver. path could be different for multiple - * instance support. - */ -int rte_vhost_driver_register(const char *path, uint64_t flags); - -/* Unregister vhost driver. This is only meaningful to vhost user. */ -int rte_vhost_driver_unregister(const char *path); - -/* Register callbacks. */ -int rte_vhost_driver_callback_register(struct virtio_net_device_ops const * const); -/* Start vhost driver session blocking loop. */ -int rte_vhost_driver_session_start(void); - -/** - * Get the numa node from which the virtio net device's memory - * is allocated. - * - * @param vid - * virtio-net device ID - * - * @return - * The numa node, -1 on failure - */ -int rte_vhost_get_numa_node(int vid); - -/** - * Get the number of queues the device supports. - * - * @param vid - * virtio-net device ID - * - * @return - * The number of queues, 0 on failure - */ -uint32_t rte_vhost_get_queue_num(int vid); - -/** - * Get the virtio net device's ifname. For vhost-cuse, ifname is the - * path of the char device. For vhost-user, ifname is the vhost-user - * socket file path. - * - * @param vid - * virtio-net device ID - * @param buf - * The buffer to stored the queried ifname - * @param len - * The length of buf - * - * @return - * 0 on success, -1 on failure - */ -int rte_vhost_get_ifname(int vid, char *buf, size_t len); - -/** - * Get how many avail entries are left in the queue - * - * @param vid - * virtio-net device ID - * @param queue_id - * virtio queue index - * - * @return - * num of avail entires left - */ -uint16_t rte_vhost_avail_entries(int vid, uint16_t queue_id); - -/** - * This function adds buffers to the virtio devices RX virtqueue. Buffers can - * be received from the physical port or from another virtual device. A packet - * count is returned to indicate the number of packets that were succesfully - * added to the RX queue. - * @param vid - * virtio-net device ID - * @param queue_id - * virtio queue index in mq case - * @param pkts - * array to contain packets to be enqueued - * @param count - * packets num to be enqueued - * @return - * num of packets enqueued - */ -uint16_t rte_vhost_enqueue_burst(int vid, uint16_t queue_id, - struct rte_mbuf **pkts, uint16_t count); - -/** - * This function gets guest buffers from the virtio device TX virtqueue, - * construct host mbufs, copies guest buffer content to host mbufs and - * store them in pkts to be processed. - * @param vid - * virtio-net device - * @param queue_id - * virtio queue index in mq case - * @param mbuf_pool - * mbuf_pool where host mbuf is allocated. - * @param pkts - * array to contain packets to be dequeued - * @param count - * packets num to be dequeued - * @return - * num of packets dequeued - */ -uint16_t rte_vhost_dequeue_burst(int vid, uint16_t queue_id, - struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count); - -#endif /* _VIRTIO_NET_H_ */ diff --git a/dpdk/lib/librte_vhost/socket.c b/dpdk/lib/librte_vhost/socket.c new file mode 100644 index 00000000..811e6bf1 --- /dev/null +++ b/dpdk/lib/librte_vhost/socket.c @@ -0,0 +1,848 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "fd_man.h" +#include "vhost.h" +#include "vhost_user.h" + + +TAILQ_HEAD(vhost_user_connection_list, vhost_user_connection); + +/* + * Every time rte_vhost_driver_register() is invoked, an associated + * vhost_user_socket struct will be created. + */ +struct vhost_user_socket { + struct vhost_user_connection_list conn_list; + pthread_mutex_t conn_mutex; + char *path; + int socket_fd; + struct sockaddr_un un; + bool is_server; + bool reconnect; + bool dequeue_zero_copy; + bool iommu_support; + + /* + * The "supported_features" indicates the feature bits the + * vhost driver supports. The "features" indicates the feature + * bits after the rte_vhost_driver_features_disable/enable(). + * It is also the final feature bits used for vhost-user + * features negotiation. + */ + uint64_t supported_features; + uint64_t features; + + struct vhost_device_ops const *notify_ops; +}; + +struct vhost_user_connection { + struct vhost_user_socket *vsocket; + int connfd; + int vid; + + TAILQ_ENTRY(vhost_user_connection) next; +}; + +#define MAX_VHOST_SOCKET 1024 +struct vhost_user { + struct vhost_user_socket *vsockets[MAX_VHOST_SOCKET]; + struct fdset fdset; + int vsocket_cnt; + pthread_mutex_t mutex; +}; + +#define MAX_VIRTIO_BACKLOG 128 + +static void vhost_user_server_new_connection(int fd, void *data, int *remove); +static void vhost_user_read_cb(int fd, void *dat, int *remove); +static int create_unix_socket(struct vhost_user_socket *vsocket); +static int vhost_user_start_client(struct vhost_user_socket *vsocket); + +static struct vhost_user vhost_user = { + .fdset = { + .fd = { [0 ... MAX_FDS - 1] = {-1, NULL, NULL, NULL, 0} }, + .fd_mutex = PTHREAD_MUTEX_INITIALIZER, + .num = 0 + }, + .vsocket_cnt = 0, + .mutex = PTHREAD_MUTEX_INITIALIZER, +}; + +/* return bytes# of read on success or negative val on failure. */ +int +read_fd_message(int sockfd, char *buf, int buflen, int *fds, int fd_num) +{ + struct iovec iov; + struct msghdr msgh; + size_t fdsize = fd_num * sizeof(int); + char control[CMSG_SPACE(fdsize)]; + struct cmsghdr *cmsg; + int ret; + + memset(&msgh, 0, sizeof(msgh)); + iov.iov_base = buf; + iov.iov_len = buflen; + + msgh.msg_iov = &iov; + msgh.msg_iovlen = 1; + msgh.msg_control = control; + msgh.msg_controllen = sizeof(control); + + ret = recvmsg(sockfd, &msgh, 0); + if (ret <= 0) { + RTE_LOG(ERR, VHOST_CONFIG, "recvmsg failed\n"); + return ret; + } + + if (msgh.msg_flags & (MSG_TRUNC | MSG_CTRUNC)) { + RTE_LOG(ERR, VHOST_CONFIG, "truncted msg\n"); + return -1; + } + + for (cmsg = CMSG_FIRSTHDR(&msgh); cmsg != NULL; + cmsg = CMSG_NXTHDR(&msgh, cmsg)) { + if ((cmsg->cmsg_level == SOL_SOCKET) && + (cmsg->cmsg_type == SCM_RIGHTS)) { + memcpy(fds, CMSG_DATA(cmsg), fdsize); + break; + } + } + + return ret; +} + +int +send_fd_message(int sockfd, char *buf, int buflen, int *fds, int fd_num) +{ + + struct iovec iov; + struct msghdr msgh; + size_t fdsize = fd_num * sizeof(int); + char control[CMSG_SPACE(fdsize)]; + struct cmsghdr *cmsg; + int ret; + + memset(&msgh, 0, sizeof(msgh)); + iov.iov_base = buf; + iov.iov_len = buflen; + + msgh.msg_iov = &iov; + msgh.msg_iovlen = 1; + + if (fds && fd_num > 0) { + msgh.msg_control = control; + msgh.msg_controllen = sizeof(control); + cmsg = CMSG_FIRSTHDR(&msgh); + cmsg->cmsg_len = CMSG_LEN(fdsize); + cmsg->cmsg_level = SOL_SOCKET; + cmsg->cmsg_type = SCM_RIGHTS; + memcpy(CMSG_DATA(cmsg), fds, fdsize); + } else { + msgh.msg_control = NULL; + msgh.msg_controllen = 0; + } + + do { + ret = sendmsg(sockfd, &msgh, 0); + } while (ret < 0 && errno == EINTR); + + if (ret < 0) { + RTE_LOG(ERR, VHOST_CONFIG, "sendmsg error\n"); + return ret; + } + + return ret; +} + +static void +vhost_user_add_connection(int fd, struct vhost_user_socket *vsocket) +{ + int vid; + size_t size; + struct vhost_user_connection *conn; + int ret; + + conn = malloc(sizeof(*conn)); + if (conn == NULL) { + close(fd); + return; + } + + vid = vhost_new_device(); + if (vid == -1) { + goto err; + } + + size = strnlen(vsocket->path, PATH_MAX); + vhost_set_ifname(vid, vsocket->path, size); + + if (vsocket->dequeue_zero_copy) + vhost_enable_dequeue_zero_copy(vid); + + RTE_LOG(INFO, VHOST_CONFIG, "new device, handle is %d\n", vid); + + if (vsocket->notify_ops->new_connection) { + ret = vsocket->notify_ops->new_connection(vid); + if (ret < 0) { + RTE_LOG(ERR, VHOST_CONFIG, + "failed to add vhost user connection with fd %d\n", + fd); + goto err; + } + } + + conn->connfd = fd; + conn->vsocket = vsocket; + conn->vid = vid; + ret = fdset_add(&vhost_user.fdset, fd, vhost_user_read_cb, + NULL, conn); + if (ret < 0) { + RTE_LOG(ERR, VHOST_CONFIG, + "failed to add fd %d into vhost server fdset\n", + fd); + + if (vsocket->notify_ops->destroy_connection) + vsocket->notify_ops->destroy_connection(conn->vid); + + goto err; + } + + pthread_mutex_lock(&vsocket->conn_mutex); + TAILQ_INSERT_TAIL(&vsocket->conn_list, conn, next); + pthread_mutex_unlock(&vsocket->conn_mutex); + return; + +err: + free(conn); + close(fd); +} + +/* call back when there is new vhost-user connection from client */ +static void +vhost_user_server_new_connection(int fd, void *dat, int *remove __rte_unused) +{ + struct vhost_user_socket *vsocket = dat; + + fd = accept(fd, NULL, NULL); + if (fd < 0) + return; + + RTE_LOG(INFO, VHOST_CONFIG, "new vhost user connection is %d\n", fd); + vhost_user_add_connection(fd, vsocket); +} + +static void +vhost_user_read_cb(int connfd, void *dat, int *remove) +{ + struct vhost_user_connection *conn = dat; + struct vhost_user_socket *vsocket = conn->vsocket; + int ret; + + ret = vhost_user_msg_handler(conn->vid, connfd); + if (ret < 0) { + close(connfd); + *remove = 1; + vhost_destroy_device(conn->vid); + + if (vsocket->notify_ops->destroy_connection) + vsocket->notify_ops->destroy_connection(conn->vid); + + pthread_mutex_lock(&vsocket->conn_mutex); + TAILQ_REMOVE(&vsocket->conn_list, conn, next); + pthread_mutex_unlock(&vsocket->conn_mutex); + + free(conn); + + if (vsocket->reconnect) { + create_unix_socket(vsocket); + vhost_user_start_client(vsocket); + } + } +} + +static int +create_unix_socket(struct vhost_user_socket *vsocket) +{ + int fd; + struct sockaddr_un *un = &vsocket->un; + + fd = socket(AF_UNIX, SOCK_STREAM, 0); + if (fd < 0) + return -1; + RTE_LOG(INFO, VHOST_CONFIG, "vhost-user %s: socket created, fd: %d\n", + vsocket->is_server ? "server" : "client", fd); + + if (!vsocket->is_server && fcntl(fd, F_SETFL, O_NONBLOCK)) { + RTE_LOG(ERR, VHOST_CONFIG, + "vhost-user: can't set nonblocking mode for socket, fd: " + "%d (%s)\n", fd, strerror(errno)); + close(fd); + return -1; + } + + memset(un, 0, sizeof(*un)); + un->sun_family = AF_UNIX; + strncpy(un->sun_path, vsocket->path, sizeof(un->sun_path)); + un->sun_path[sizeof(un->sun_path) - 1] = '\0'; + + vsocket->socket_fd = fd; + return 0; +} + +static int +vhost_user_start_server(struct vhost_user_socket *vsocket) +{ + int ret; + int fd = vsocket->socket_fd; + const char *path = vsocket->path; + + ret = bind(fd, (struct sockaddr *)&vsocket->un, sizeof(vsocket->un)); + if (ret < 0) { + RTE_LOG(ERR, VHOST_CONFIG, + "failed to bind to %s: %s; remove it and try again\n", + path, strerror(errno)); + goto err; + } + RTE_LOG(INFO, VHOST_CONFIG, "bind to %s\n", path); + + ret = listen(fd, MAX_VIRTIO_BACKLOG); + if (ret < 0) + goto err; + + ret = fdset_add(&vhost_user.fdset, fd, vhost_user_server_new_connection, + NULL, vsocket); + if (ret < 0) { + RTE_LOG(ERR, VHOST_CONFIG, + "failed to add listen fd %d to vhost server fdset\n", + fd); + goto err; + } + + return 0; + +err: + close(fd); + return -1; +} + +struct vhost_user_reconnect { + struct sockaddr_un un; + int fd; + struct vhost_user_socket *vsocket; + + TAILQ_ENTRY(vhost_user_reconnect) next; +}; + +TAILQ_HEAD(vhost_user_reconnect_tailq_list, vhost_user_reconnect); +struct vhost_user_reconnect_list { + struct vhost_user_reconnect_tailq_list head; + pthread_mutex_t mutex; +}; + +static struct vhost_user_reconnect_list reconn_list; +static pthread_t reconn_tid; + +static int +vhost_user_connect_nonblock(int fd, struct sockaddr *un, size_t sz) +{ + int ret, flags; + + ret = connect(fd, un, sz); + if (ret < 0 && errno != EISCONN) + return -1; + + flags = fcntl(fd, F_GETFL, 0); + if (flags < 0) { + RTE_LOG(ERR, VHOST_CONFIG, + "can't get flags for connfd %d\n", fd); + return -2; + } + if ((flags & O_NONBLOCK) && fcntl(fd, F_SETFL, flags & ~O_NONBLOCK)) { + RTE_LOG(ERR, VHOST_CONFIG, + "can't disable nonblocking on fd %d\n", fd); + return -2; + } + return 0; +} + +static void * +vhost_user_client_reconnect(void *arg __rte_unused) +{ + int ret; + struct vhost_user_reconnect *reconn, *next; + + while (1) { + pthread_mutex_lock(&reconn_list.mutex); + + /* + * An equal implementation of TAILQ_FOREACH_SAFE, + * which does not exist on all platforms. + */ + for (reconn = TAILQ_FIRST(&reconn_list.head); + reconn != NULL; reconn = next) { + next = TAILQ_NEXT(reconn, next); + + ret = vhost_user_connect_nonblock(reconn->fd, + (struct sockaddr *)&reconn->un, + sizeof(reconn->un)); + if (ret == -2) { + close(reconn->fd); + RTE_LOG(ERR, VHOST_CONFIG, + "reconnection for fd %d failed\n", + reconn->fd); + goto remove_fd; + } + if (ret == -1) + continue; + + RTE_LOG(INFO, VHOST_CONFIG, + "%s: connected\n", reconn->vsocket->path); + vhost_user_add_connection(reconn->fd, reconn->vsocket); +remove_fd: + TAILQ_REMOVE(&reconn_list.head, reconn, next); + free(reconn); + } + + pthread_mutex_unlock(&reconn_list.mutex); + sleep(1); + } + + return NULL; +} + +static int +vhost_user_reconnect_init(void) +{ + int ret; + + ret = pthread_mutex_init(&reconn_list.mutex, NULL); + if (ret < 0) { + RTE_LOG(ERR, VHOST_CONFIG, "failed to initialize mutex"); + return ret; + } + TAILQ_INIT(&reconn_list.head); + + ret = pthread_create(&reconn_tid, NULL, + vhost_user_client_reconnect, NULL); + if (ret != 0) { + RTE_LOG(ERR, VHOST_CONFIG, "failed to create reconnect thread"); + if (pthread_mutex_destroy(&reconn_list.mutex)) { + RTE_LOG(ERR, VHOST_CONFIG, + "failed to destroy reconnect mutex"); + } + } + + return ret; +} + +static int +vhost_user_start_client(struct vhost_user_socket *vsocket) +{ + int ret; + int fd = vsocket->socket_fd; + const char *path = vsocket->path; + struct vhost_user_reconnect *reconn; + + ret = vhost_user_connect_nonblock(fd, (struct sockaddr *)&vsocket->un, + sizeof(vsocket->un)); + if (ret == 0) { + vhost_user_add_connection(fd, vsocket); + return 0; + } + + RTE_LOG(WARNING, VHOST_CONFIG, + "failed to connect to %s: %s\n", + path, strerror(errno)); + + if (ret == -2 || !vsocket->reconnect) { + close(fd); + return -1; + } + + RTE_LOG(INFO, VHOST_CONFIG, "%s: reconnecting...\n", path); + reconn = malloc(sizeof(*reconn)); + if (reconn == NULL) { + RTE_LOG(ERR, VHOST_CONFIG, + "failed to allocate memory for reconnect\n"); + close(fd); + return -1; + } + reconn->un = vsocket->un; + reconn->fd = fd; + reconn->vsocket = vsocket; + pthread_mutex_lock(&reconn_list.mutex); + TAILQ_INSERT_TAIL(&reconn_list.head, reconn, next); + pthread_mutex_unlock(&reconn_list.mutex); + + return 0; +} + +static struct vhost_user_socket * +find_vhost_user_socket(const char *path) +{ + int i; + + for (i = 0; i < vhost_user.vsocket_cnt; i++) { + struct vhost_user_socket *vsocket = vhost_user.vsockets[i]; + + if (!strcmp(vsocket->path, path)) + return vsocket; + } + + return NULL; +} + +int +rte_vhost_driver_disable_features(const char *path, uint64_t features) +{ + struct vhost_user_socket *vsocket; + + pthread_mutex_lock(&vhost_user.mutex); + vsocket = find_vhost_user_socket(path); + if (vsocket) + vsocket->features &= ~features; + pthread_mutex_unlock(&vhost_user.mutex); + + return vsocket ? 0 : -1; +} + +int +rte_vhost_driver_enable_features(const char *path, uint64_t features) +{ + struct vhost_user_socket *vsocket; + + pthread_mutex_lock(&vhost_user.mutex); + vsocket = find_vhost_user_socket(path); + if (vsocket) { + if ((vsocket->supported_features & features) != features) { + /* + * trying to enable features the driver doesn't + * support. + */ + pthread_mutex_unlock(&vhost_user.mutex); + return -1; + } + vsocket->features |= features; + } + pthread_mutex_unlock(&vhost_user.mutex); + + return vsocket ? 0 : -1; +} + +int +rte_vhost_driver_set_features(const char *path, uint64_t features) +{ + struct vhost_user_socket *vsocket; + + pthread_mutex_lock(&vhost_user.mutex); + vsocket = find_vhost_user_socket(path); + if (vsocket) { + vsocket->supported_features = features; + vsocket->features = features; + } + pthread_mutex_unlock(&vhost_user.mutex); + + return vsocket ? 0 : -1; +} + +int +rte_vhost_driver_get_features(const char *path, uint64_t *features) +{ + struct vhost_user_socket *vsocket; + + pthread_mutex_lock(&vhost_user.mutex); + vsocket = find_vhost_user_socket(path); + if (vsocket) + *features = vsocket->features; + pthread_mutex_unlock(&vhost_user.mutex); + + if (!vsocket) { + RTE_LOG(ERR, VHOST_CONFIG, + "socket file %s is not registered yet.\n", path); + return -1; + } else { + return 0; + } +} + +/* + * Register a new vhost-user socket; here we could act as server + * (the default case), or client (when RTE_VHOST_USER_CLIENT) flag + * is set. + */ +int +rte_vhost_driver_register(const char *path, uint64_t flags) +{ + int ret = -1; + struct vhost_user_socket *vsocket; + + if (!path) + return -1; + + pthread_mutex_lock(&vhost_user.mutex); + + if (vhost_user.vsocket_cnt == MAX_VHOST_SOCKET) { + RTE_LOG(ERR, VHOST_CONFIG, + "error: the number of vhost sockets reaches maximum\n"); + goto out; + } + + vsocket = malloc(sizeof(struct vhost_user_socket)); + if (!vsocket) + goto out; + memset(vsocket, 0, sizeof(struct vhost_user_socket)); + vsocket->path = strdup(path); + if (vsocket->path == NULL) { + RTE_LOG(ERR, VHOST_CONFIG, + "error: failed to copy socket path string\n"); + free(vsocket); + goto out; + } + TAILQ_INIT(&vsocket->conn_list); + ret = pthread_mutex_init(&vsocket->conn_mutex, NULL); + if (ret) { + RTE_LOG(ERR, VHOST_CONFIG, + "error: failed to init connection mutex\n"); + goto out_free; + } + vsocket->dequeue_zero_copy = flags & RTE_VHOST_USER_DEQUEUE_ZERO_COPY; + + /* + * Set the supported features correctly for the builtin vhost-user + * net driver. + * + * Applications know nothing about features the builtin virtio net + * driver (virtio_net.c) supports, thus it's not possible for them + * to invoke rte_vhost_driver_set_features(). To workaround it, here + * we set it unconditionally. If the application want to implement + * another vhost-user driver (say SCSI), it should call the + * rte_vhost_driver_set_features(), which will overwrite following + * two values. + */ + vsocket->supported_features = VIRTIO_NET_SUPPORTED_FEATURES; + vsocket->features = VIRTIO_NET_SUPPORTED_FEATURES; + + if (!(flags & RTE_VHOST_USER_IOMMU_SUPPORT)) { + vsocket->supported_features &= ~(1ULL << VIRTIO_F_IOMMU_PLATFORM); + vsocket->features &= ~(1ULL << VIRTIO_F_IOMMU_PLATFORM); + } + + if ((flags & RTE_VHOST_USER_CLIENT) != 0) { + vsocket->reconnect = !(flags & RTE_VHOST_USER_NO_RECONNECT); + if (vsocket->reconnect && reconn_tid == 0) { + if (vhost_user_reconnect_init() != 0) + goto out_mutex; + } + } else { + vsocket->is_server = true; + } + ret = create_unix_socket(vsocket); + if (ret < 0) { + goto out_mutex; + } + + vhost_user.vsockets[vhost_user.vsocket_cnt++] = vsocket; + + pthread_mutex_unlock(&vhost_user.mutex); + return ret; + +out_mutex: + if (pthread_mutex_destroy(&vsocket->conn_mutex)) { + RTE_LOG(ERR, VHOST_CONFIG, + "error: failed to destroy connection mutex\n"); + } +out_free: + free(vsocket->path); + free(vsocket); +out: + pthread_mutex_unlock(&vhost_user.mutex); + + return ret; +} + +static bool +vhost_user_remove_reconnect(struct vhost_user_socket *vsocket) +{ + int found = false; + struct vhost_user_reconnect *reconn, *next; + + pthread_mutex_lock(&reconn_list.mutex); + + for (reconn = TAILQ_FIRST(&reconn_list.head); + reconn != NULL; reconn = next) { + next = TAILQ_NEXT(reconn, next); + + if (reconn->vsocket == vsocket) { + TAILQ_REMOVE(&reconn_list.head, reconn, next); + close(reconn->fd); + free(reconn); + found = true; + break; + } + } + pthread_mutex_unlock(&reconn_list.mutex); + return found; +} + +/** + * Unregister the specified vhost socket + */ +int +rte_vhost_driver_unregister(const char *path) +{ + int i; + int count; + struct vhost_user_connection *conn, *next; + + pthread_mutex_lock(&vhost_user.mutex); + + for (i = 0; i < vhost_user.vsocket_cnt; i++) { + struct vhost_user_socket *vsocket = vhost_user.vsockets[i]; + + if (!strcmp(vsocket->path, path)) { + if (vsocket->is_server) { + fdset_del(&vhost_user.fdset, vsocket->socket_fd); + close(vsocket->socket_fd); + unlink(path); + } else if (vsocket->reconnect) { + vhost_user_remove_reconnect(vsocket); + } + + pthread_mutex_lock(&vsocket->conn_mutex); + for (conn = TAILQ_FIRST(&vsocket->conn_list); + conn != NULL; + conn = next) { + next = TAILQ_NEXT(conn, next); + + fdset_del(&vhost_user.fdset, conn->connfd); + RTE_LOG(INFO, VHOST_CONFIG, + "free connfd = %d for device '%s'\n", + conn->connfd, path); + close(conn->connfd); + vhost_destroy_device(conn->vid); + TAILQ_REMOVE(&vsocket->conn_list, conn, next); + free(conn); + } + pthread_mutex_unlock(&vsocket->conn_mutex); + + pthread_mutex_destroy(&vsocket->conn_mutex); + free(vsocket->path); + free(vsocket); + + count = --vhost_user.vsocket_cnt; + vhost_user.vsockets[i] = vhost_user.vsockets[count]; + vhost_user.vsockets[count] = NULL; + pthread_mutex_unlock(&vhost_user.mutex); + + return 0; + } + } + pthread_mutex_unlock(&vhost_user.mutex); + + return -1; +} + +/* + * Register ops so that we can add/remove device to data core. + */ +int +rte_vhost_driver_callback_register(const char *path, + struct vhost_device_ops const * const ops) +{ + struct vhost_user_socket *vsocket; + + pthread_mutex_lock(&vhost_user.mutex); + vsocket = find_vhost_user_socket(path); + if (vsocket) + vsocket->notify_ops = ops; + pthread_mutex_unlock(&vhost_user.mutex); + + return vsocket ? 0 : -1; +} + +struct vhost_device_ops const * +vhost_driver_callback_get(const char *path) +{ + struct vhost_user_socket *vsocket; + + pthread_mutex_lock(&vhost_user.mutex); + vsocket = find_vhost_user_socket(path); + pthread_mutex_unlock(&vhost_user.mutex); + + return vsocket ? vsocket->notify_ops : NULL; +} + +int +rte_vhost_driver_start(const char *path) +{ + struct vhost_user_socket *vsocket; + static pthread_t fdset_tid; + + pthread_mutex_lock(&vhost_user.mutex); + vsocket = find_vhost_user_socket(path); + pthread_mutex_unlock(&vhost_user.mutex); + + if (!vsocket) + return -1; + + if (fdset_tid == 0) { + int ret = pthread_create(&fdset_tid, NULL, fdset_event_dispatch, + &vhost_user.fdset); + if (ret != 0) + RTE_LOG(ERR, VHOST_CONFIG, + "failed to create fdset handling thread"); + } + + if (vsocket->is_server) + return vhost_user_start_server(vsocket); + else + return vhost_user_start_client(vsocket); +} diff --git a/dpdk/lib/librte_vhost/vhost-net.h b/dpdk/lib/librte_vhost/vhost-net.h deleted file mode 100644 index 38593a29..00000000 --- a/dpdk/lib/librte_vhost/vhost-net.h +++ /dev/null @@ -1,250 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef _VHOST_NET_CDEV_H_ -#define _VHOST_NET_CDEV_H_ -#include -#include -#include -#include -#include - -#include - -#include "rte_virtio_net.h" - -/* Used to indicate that the device is running on a data core */ -#define VIRTIO_DEV_RUNNING 1 - -/* Backend value set by guest. */ -#define VIRTIO_DEV_STOPPED -1 - -#define BUF_VECTOR_MAX 256 - -/** - * Structure contains buffer address, length and descriptor index - * from vring to do scatter RX. - */ -struct buf_vector { - uint64_t buf_addr; - uint32_t buf_len; - uint32_t desc_idx; -}; - -/** - * Structure contains variables relevant to RX/TX virtqueues. - */ -struct vhost_virtqueue { - struct vring_desc *desc; - struct vring_avail *avail; - struct vring_used *used; - uint32_t size; - - /* Last index used on the available ring */ - volatile uint16_t last_used_idx; -#define VIRTIO_INVALID_EVENTFD (-1) -#define VIRTIO_UNINITIALIZED_EVENTFD (-2) - - /* Backend value to determine if device should started/stopped */ - int backend; - /* Used to notify the guest (trigger interrupt) */ - int callfd; - /* Currently unused as polling mode is enabled */ - int kickfd; - int enabled; - - /* Physical address of used ring, for logging */ - uint64_t log_guest_addr; -} __rte_cache_aligned; - -/* Old kernels have no such macro defined */ -#ifndef VIRTIO_NET_F_GUEST_ANNOUNCE - #define VIRTIO_NET_F_GUEST_ANNOUNCE 21 -#endif - - -/* - * Make an extra wrapper for VIRTIO_NET_F_MQ and - * VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX as they are - * introduced since kernel v3.8. This makes our - * code buildable for older kernel. - */ -#ifdef VIRTIO_NET_F_MQ - #define VHOST_MAX_QUEUE_PAIRS VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX - #define VHOST_SUPPORTS_MQ (1ULL << VIRTIO_NET_F_MQ) -#else - #define VHOST_MAX_QUEUE_PAIRS 1 - #define VHOST_SUPPORTS_MQ 0 -#endif - -/* - * Define virtio 1.0 for older kernels - */ -#ifndef VIRTIO_F_VERSION_1 - #define VIRTIO_F_VERSION_1 32 -#endif - -/** - * Device structure contains all configuration information relating - * to the device. - */ -struct virtio_net { - /* Frontend (QEMU) memory and memory region information */ - struct virtio_memory *mem; - uint64_t features; - uint64_t protocol_features; - int vid; - uint32_t flags; - uint16_t vhost_hlen; - /* to tell if we need broadcast rarp packet */ - rte_atomic16_t broadcast_rarp; - uint32_t virt_qp_nb; - struct vhost_virtqueue *virtqueue[VHOST_MAX_QUEUE_PAIRS * 2]; -#define IF_NAME_SZ (PATH_MAX > IFNAMSIZ ? PATH_MAX : IFNAMSIZ) - char ifname[IF_NAME_SZ]; - uint64_t log_size; - uint64_t log_base; - uint64_t log_addr; - struct ether_addr mac; - -} __rte_cache_aligned; - -/** - * Information relating to memory regions including offsets to - * addresses in QEMUs memory file. - */ -struct virtio_memory_regions { - uint64_t guest_phys_address; - uint64_t guest_phys_address_end; - uint64_t memory_size; - uint64_t userspace_address; - uint64_t address_offset; -}; - - -/** - * Memory structure includes region and mapping information. - */ -struct virtio_memory { - /* Base QEMU userspace address of the memory file. */ - uint64_t base_address; - uint64_t mapped_address; - uint64_t mapped_size; - uint32_t nregions; - struct virtio_memory_regions regions[0]; -}; - - -/* Macros for printing using RTE_LOG */ -#define RTE_LOGTYPE_VHOST_CONFIG RTE_LOGTYPE_USER1 -#define RTE_LOGTYPE_VHOST_DATA RTE_LOGTYPE_USER1 - -#ifdef RTE_LIBRTE_VHOST_DEBUG -#define VHOST_MAX_PRINT_BUFF 6072 -#define LOG_LEVEL RTE_LOG_DEBUG -#define LOG_DEBUG(log_type, fmt, args...) RTE_LOG(DEBUG, log_type, fmt, ##args) -#define PRINT_PACKET(device, addr, size, header) do { \ - char *pkt_addr = (char *)(addr); \ - unsigned int index; \ - char packet[VHOST_MAX_PRINT_BUFF]; \ - \ - if ((header)) \ - snprintf(packet, VHOST_MAX_PRINT_BUFF, "(%d) Header size %d: ", (device->vid), (size)); \ - else \ - snprintf(packet, VHOST_MAX_PRINT_BUFF, "(%d) Packet size %d: ", (device->vid), (size)); \ - for (index = 0; index < (size); index++) { \ - snprintf(packet + strnlen(packet, VHOST_MAX_PRINT_BUFF), VHOST_MAX_PRINT_BUFF - strnlen(packet, VHOST_MAX_PRINT_BUFF), \ - "%02hhx ", pkt_addr[index]); \ - } \ - snprintf(packet + strnlen(packet, VHOST_MAX_PRINT_BUFF), VHOST_MAX_PRINT_BUFF - strnlen(packet, VHOST_MAX_PRINT_BUFF), "\n"); \ - \ - LOG_DEBUG(VHOST_DATA, "%s", packet); \ -} while (0) -#else -#define LOG_LEVEL RTE_LOG_INFO -#define LOG_DEBUG(log_type, fmt, args...) do {} while (0) -#define PRINT_PACKET(device, addr, size, header) do {} while (0) -#endif - -/** - * Function to convert guest physical addresses to vhost virtual addresses. - * This is used to convert guest virtio buffer addresses. - */ -static inline uint64_t __attribute__((always_inline)) -gpa_to_vva(struct virtio_net *dev, uint64_t guest_pa) -{ - struct virtio_memory_regions *region; - uint32_t regionidx; - uint64_t vhost_va = 0; - - for (regionidx = 0; regionidx < dev->mem->nregions; regionidx++) { - region = &dev->mem->regions[regionidx]; - if ((guest_pa >= region->guest_phys_address) && - (guest_pa <= region->guest_phys_address_end)) { - vhost_va = region->address_offset + guest_pa; - break; - } - } - return vhost_va; -} - -struct virtio_net_device_ops const *notify_ops; -struct virtio_net *get_device(int vid); - -int vhost_new_device(void); -void vhost_destroy_device(int); - -void vhost_set_ifname(int, const char *if_name, unsigned int if_len); - -int vhost_get_features(int, uint64_t *); -int vhost_set_features(int, uint64_t *); - -int vhost_set_vring_num(int, struct vhost_vring_state *); -int vhost_set_vring_addr(int, struct vhost_vring_addr *); -int vhost_set_vring_base(int, struct vhost_vring_state *); -int vhost_get_vring_base(int, uint32_t, struct vhost_vring_state *); - -int vhost_set_vring_kick(int, struct vhost_vring_file *); -int vhost_set_vring_call(int, struct vhost_vring_file *); - -int vhost_set_backend(int, struct vhost_vring_file *); - -int vhost_set_owner(int); -int vhost_reset_owner(int); - -/* - * Backend-specific cleanup. Defined by vhost-cuse and vhost-user. - */ -void vhost_backend_cleanup(struct virtio_net *dev); - -#endif /* _VHOST_NET_CDEV_H_ */ diff --git a/dpdk/lib/librte_vhost/vhost.c b/dpdk/lib/librte_vhost/vhost.c new file mode 100644 index 00000000..a8ed40b1 --- /dev/null +++ b/dpdk/lib/librte_vhost/vhost.c @@ -0,0 +1,627 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#ifdef RTE_LIBRTE_VHOST_NUMA +#include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "iotlb.h" +#include "vhost.h" +#include "vhost_user.h" + +struct virtio_net *vhost_devices[MAX_VHOST_DEVICE]; + +/* Called with iotlb_lock read-locked */ +uint64_t +__vhost_iova_to_vva(struct virtio_net *dev, struct vhost_virtqueue *vq, + uint64_t iova, uint64_t *size, uint8_t perm) +{ + uint64_t vva, tmp_size; + + if (unlikely(!*size)) + return 0; + + tmp_size = *size; + + vva = vhost_user_iotlb_cache_find(vq, iova, &tmp_size, perm); + if (tmp_size == *size) + return vva; + + iova += tmp_size; + + if (!vhost_user_iotlb_pending_miss(vq, iova, perm)) { + /* + * iotlb_lock is read-locked for a full burst, + * but it only protects the iotlb cache. + * In case of IOTLB miss, we might block on the socket, + * which could cause a deadlock with QEMU if an IOTLB update + * is being handled. We can safely unlock here to avoid it. + */ + vhost_user_iotlb_rd_unlock(vq); + + vhost_user_iotlb_pending_insert(vq, iova, perm); + if (vhost_user_iotlb_miss(dev, iova, perm)) { + RTE_LOG(ERR, VHOST_CONFIG, + "IOTLB miss req failed for IOVA 0x%" PRIx64 "\n", + iova); + vhost_user_iotlb_pending_remove(vq, iova, 1, perm); + } + + vhost_user_iotlb_rd_lock(vq); + } + + return 0; +} + +struct virtio_net * +get_device(int vid) +{ + struct virtio_net *dev = vhost_devices[vid]; + + if (unlikely(!dev)) { + RTE_LOG(ERR, VHOST_CONFIG, + "(%d) device not found.\n", vid); + } + + return dev; +} + +static void +cleanup_vq(struct vhost_virtqueue *vq, int destroy) +{ + if ((vq->callfd >= 0) && (destroy != 0)) + close(vq->callfd); + if (vq->kickfd >= 0) + close(vq->kickfd); +} + +/* + * Unmap any memory, close any file descriptors and + * free any memory owned by a device. + */ +void +cleanup_device(struct virtio_net *dev, int destroy) +{ + uint32_t i; + + vhost_backend_cleanup(dev); + + for (i = 0; i < dev->nr_vring; i++) + cleanup_vq(dev->virtqueue[i], destroy); +} + +/* + * Release virtqueues and device memory. + */ +static void +free_device(struct virtio_net *dev) +{ + uint32_t i; + struct vhost_virtqueue *vq; + + for (i = 0; i < dev->nr_vring; i++) { + vq = dev->virtqueue[i]; + + rte_free(vq->shadow_used_ring); + rte_free(vq->batch_copy_elems); + rte_mempool_free(vq->iotlb_pool); + rte_free(vq); + } + + rte_free(dev); +} + +int +vring_translate(struct virtio_net *dev, struct vhost_virtqueue *vq) +{ + uint64_t req_size, size; + + if (!(dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))) + goto out; + + req_size = sizeof(struct vring_desc) * vq->size; + size = req_size; + vq->desc = (struct vring_desc *)(uintptr_t)vhost_iova_to_vva(dev, vq, + vq->ring_addrs.desc_user_addr, + &size, VHOST_ACCESS_RW); + if (!vq->desc || size != req_size) + return -1; + + req_size = sizeof(struct vring_avail); + req_size += sizeof(uint16_t) * vq->size; + if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX)) + req_size += sizeof(uint16_t); + size = req_size; + vq->avail = (struct vring_avail *)(uintptr_t)vhost_iova_to_vva(dev, vq, + vq->ring_addrs.avail_user_addr, + &size, VHOST_ACCESS_RW); + if (!vq->avail || size != req_size) + return -1; + + req_size = sizeof(struct vring_used); + req_size += sizeof(struct vring_used_elem) * vq->size; + if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX)) + req_size += sizeof(uint16_t); + size = req_size; + vq->used = (struct vring_used *)(uintptr_t)vhost_iova_to_vva(dev, vq, + vq->ring_addrs.used_user_addr, + &size, VHOST_ACCESS_RW); + if (!vq->used || size != req_size) + return -1; + +out: + vq->access_ok = 1; + + return 0; +} + +void +vring_invalidate(struct virtio_net *dev, struct vhost_virtqueue *vq) +{ + if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)) + vhost_user_iotlb_wr_lock(vq); + + vq->access_ok = 0; + vq->desc = NULL; + vq->avail = NULL; + vq->used = NULL; + + if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)) + vhost_user_iotlb_wr_unlock(vq); +} + +static void +init_vring_queue(struct virtio_net *dev, uint32_t vring_idx) +{ + struct vhost_virtqueue *vq; + + if (vring_idx >= VHOST_MAX_VRING) { + RTE_LOG(ERR, VHOST_CONFIG, + "Failed not init vring, out of bound (%d)\n", + vring_idx); + return; + } + + vq = dev->virtqueue[vring_idx]; + + memset(vq, 0, sizeof(struct vhost_virtqueue)); + + vq->kickfd = VIRTIO_UNINITIALIZED_EVENTFD; + vq->callfd = VIRTIO_UNINITIALIZED_EVENTFD; + + vhost_user_iotlb_init(dev, vring_idx); + /* Backends are set to -1 indicating an inactive device. */ + vq->backend = -1; + + TAILQ_INIT(&vq->zmbuf_list); +} + +static void +reset_vring_queue(struct virtio_net *dev, uint32_t vring_idx) +{ + struct vhost_virtqueue *vq; + int callfd; + + if (vring_idx >= VHOST_MAX_VRING) { + RTE_LOG(ERR, VHOST_CONFIG, + "Failed not init vring, out of bound (%d)\n", + vring_idx); + return; + } + + vq = dev->virtqueue[vring_idx]; + callfd = vq->callfd; + init_vring_queue(dev, vring_idx); + vq->callfd = callfd; +} + +int +alloc_vring_queue(struct virtio_net *dev, uint32_t vring_idx) +{ + struct vhost_virtqueue *vq; + + vq = rte_malloc(NULL, sizeof(struct vhost_virtqueue), 0); + if (vq == NULL) { + RTE_LOG(ERR, VHOST_CONFIG, + "Failed to allocate memory for vring:%u.\n", vring_idx); + return -1; + } + + dev->virtqueue[vring_idx] = vq; + init_vring_queue(dev, vring_idx); + rte_spinlock_init(&vq->access_lock); + + dev->nr_vring += 1; + + return 0; +} + +/* + * Reset some variables in device structure, while keeping few + * others untouched, such as vid, ifname, nr_vring: they + * should be same unless the device is removed. + */ +void +reset_device(struct virtio_net *dev) +{ + uint32_t i; + + dev->features = 0; + dev->protocol_features = 0; + dev->flags = 0; + + for (i = 0; i < dev->nr_vring; i++) + reset_vring_queue(dev, i); +} + +/* + * Invoked when there is a new vhost-user connection established (when + * there is a new virtio device being attached). + */ +int +vhost_new_device(void) +{ + struct virtio_net *dev; + int i; + + dev = rte_zmalloc(NULL, sizeof(struct virtio_net), 0); + if (dev == NULL) { + RTE_LOG(ERR, VHOST_CONFIG, + "Failed to allocate memory for new dev.\n"); + return -1; + } + + for (i = 0; i < MAX_VHOST_DEVICE; i++) { + if (vhost_devices[i] == NULL) + break; + } + if (i == MAX_VHOST_DEVICE) { + RTE_LOG(ERR, VHOST_CONFIG, + "Failed to find a free slot for new device.\n"); + rte_free(dev); + return -1; + } + + vhost_devices[i] = dev; + dev->vid = i; + dev->slave_req_fd = -1; + + return i; +} + +/* + * Invoked when there is the vhost-user connection is broken (when + * the virtio device is being detached). + */ +void +vhost_destroy_device(int vid) +{ + struct virtio_net *dev = get_device(vid); + + if (dev == NULL) + return; + + if (dev->flags & VIRTIO_DEV_RUNNING) { + dev->flags &= ~VIRTIO_DEV_RUNNING; + dev->notify_ops->destroy_device(vid); + } + + cleanup_device(dev, 1); + free_device(dev); + + vhost_devices[vid] = NULL; +} + +void +vhost_set_ifname(int vid, const char *if_name, unsigned int if_len) +{ + struct virtio_net *dev; + unsigned int len; + + dev = get_device(vid); + if (dev == NULL) + return; + + len = if_len > sizeof(dev->ifname) ? + sizeof(dev->ifname) : if_len; + + strncpy(dev->ifname, if_name, len); + dev->ifname[sizeof(dev->ifname) - 1] = '\0'; +} + +void +vhost_enable_dequeue_zero_copy(int vid) +{ + struct virtio_net *dev = get_device(vid); + + if (dev == NULL) + return; + + dev->dequeue_zero_copy = 1; +} + +int +rte_vhost_get_mtu(int vid, uint16_t *mtu) +{ + struct virtio_net *dev = get_device(vid); + + if (!dev) + return -ENODEV; + + if (!(dev->flags & VIRTIO_DEV_READY)) + return -EAGAIN; + + if (!(dev->features & (1ULL << VIRTIO_NET_F_MTU))) + return -ENOTSUP; + + *mtu = dev->mtu; + + return 0; +} + +int +rte_vhost_get_numa_node(int vid) +{ +#ifdef RTE_LIBRTE_VHOST_NUMA + struct virtio_net *dev = get_device(vid); + int numa_node; + int ret; + + if (dev == NULL) + return -1; + + ret = get_mempolicy(&numa_node, NULL, 0, dev, + MPOL_F_NODE | MPOL_F_ADDR); + if (ret < 0) { + RTE_LOG(ERR, VHOST_CONFIG, + "(%d) failed to query numa node: %s\n", + vid, rte_strerror(errno)); + return -1; + } + + return numa_node; +#else + RTE_SET_USED(vid); + return -1; +#endif +} + +uint32_t +rte_vhost_get_queue_num(int vid) +{ + struct virtio_net *dev = get_device(vid); + + if (dev == NULL) + return 0; + + return dev->nr_vring / 2; +} + +uint16_t +rte_vhost_get_vring_num(int vid) +{ + struct virtio_net *dev = get_device(vid); + + if (dev == NULL) + return 0; + + return dev->nr_vring; +} + +int +rte_vhost_get_ifname(int vid, char *buf, size_t len) +{ + struct virtio_net *dev = get_device(vid); + + if (dev == NULL) + return -1; + + len = RTE_MIN(len, sizeof(dev->ifname)); + + strncpy(buf, dev->ifname, len); + buf[len - 1] = '\0'; + + return 0; +} + +int +rte_vhost_get_negotiated_features(int vid, uint64_t *features) +{ + struct virtio_net *dev; + + dev = get_device(vid); + if (!dev) + return -1; + + *features = dev->features; + return 0; +} + +int +rte_vhost_get_mem_table(int vid, struct rte_vhost_memory **mem) +{ + struct virtio_net *dev; + struct rte_vhost_memory *m; + size_t size; + + dev = get_device(vid); + if (!dev) + return -1; + + size = dev->mem->nregions * sizeof(struct rte_vhost_mem_region); + m = malloc(sizeof(struct rte_vhost_memory) + size); + if (!m) + return -1; + + m->nregions = dev->mem->nregions; + memcpy(m->regions, dev->mem->regions, size); + *mem = m; + + return 0; +} + +int +rte_vhost_get_vhost_vring(int vid, uint16_t vring_idx, + struct rte_vhost_vring *vring) +{ + struct virtio_net *dev; + struct vhost_virtqueue *vq; + + dev = get_device(vid); + if (!dev) + return -1; + + if (vring_idx >= VHOST_MAX_VRING) + return -1; + + vq = dev->virtqueue[vring_idx]; + if (!vq) + return -1; + + vring->desc = vq->desc; + vring->avail = vq->avail; + vring->used = vq->used; + vring->log_guest_addr = vq->log_guest_addr; + + vring->callfd = vq->callfd; + vring->kickfd = vq->kickfd; + vring->size = vq->size; + + return 0; +} + +uint16_t +rte_vhost_avail_entries(int vid, uint16_t queue_id) +{ + struct virtio_net *dev; + struct vhost_virtqueue *vq; + + dev = get_device(vid); + if (!dev) + return 0; + + vq = dev->virtqueue[queue_id]; + if (!vq->enabled) + return 0; + + return *(volatile uint16_t *)&vq->avail->idx - vq->last_used_idx; +} + +int +rte_vhost_enable_guest_notification(int vid, uint16_t queue_id, int enable) +{ + struct virtio_net *dev = get_device(vid); + + if (dev == NULL) + return -1; + + if (enable) { + RTE_LOG(ERR, VHOST_CONFIG, + "guest notification isn't supported.\n"); + return -1; + } + + dev->virtqueue[queue_id]->used->flags = VRING_USED_F_NO_NOTIFY; + return 0; +} + +void +rte_vhost_log_write(int vid, uint64_t addr, uint64_t len) +{ + struct virtio_net *dev = get_device(vid); + + if (dev == NULL) + return; + + vhost_log_write(dev, addr, len); +} + +void +rte_vhost_log_used_vring(int vid, uint16_t vring_idx, + uint64_t offset, uint64_t len) +{ + struct virtio_net *dev; + struct vhost_virtqueue *vq; + + dev = get_device(vid); + if (dev == NULL) + return; + + if (vring_idx >= VHOST_MAX_VRING) + return; + vq = dev->virtqueue[vring_idx]; + if (!vq) + return; + + vhost_log_used_vring(dev, vq, offset, len); +} + +uint32_t +rte_vhost_rx_queue_count(int vid, uint16_t qid) +{ + struct virtio_net *dev; + struct vhost_virtqueue *vq; + + dev = get_device(vid); + if (dev == NULL) + return 0; + + if (unlikely(qid >= dev->nr_vring || (qid & 1) == 0)) { + RTE_LOG(ERR, VHOST_DATA, "(%d) %s: invalid virtqueue idx %d.\n", + dev->vid, __func__, qid); + return 0; + } + + vq = dev->virtqueue[qid]; + if (vq == NULL) + return 0; + + if (unlikely(vq->enabled == 0 || vq->avail == NULL)) + return 0; + + return *((volatile uint16_t *)&vq->avail->idx) - vq->last_avail_idx; +} diff --git a/dpdk/lib/librte_vhost/vhost.h b/dpdk/lib/librte_vhost/vhost.h new file mode 100644 index 00000000..16d6b891 --- /dev/null +++ b/dpdk/lib/librte_vhost/vhost.h @@ -0,0 +1,398 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _VHOST_NET_CDEV_H_ +#define _VHOST_NET_CDEV_H_ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "rte_vhost.h" + +/* Used to indicate that the device is running on a data core */ +#define VIRTIO_DEV_RUNNING 1 +/* Used to indicate that the device is ready to operate */ +#define VIRTIO_DEV_READY 2 + +/* Backend value set by guest. */ +#define VIRTIO_DEV_STOPPED -1 + +#define BUF_VECTOR_MAX 256 + +/** + * Structure contains buffer address, length and descriptor index + * from vring to do scatter RX. + */ +struct buf_vector { + uint64_t buf_addr; + uint32_t buf_len; + uint32_t desc_idx; +}; + +/* + * A structure to hold some fields needed in zero copy code path, + * mainly for associating an mbuf with the right desc_idx. + */ +struct zcopy_mbuf { + struct rte_mbuf *mbuf; + uint32_t desc_idx; + uint16_t in_use; + + TAILQ_ENTRY(zcopy_mbuf) next; +}; +TAILQ_HEAD(zcopy_mbuf_list, zcopy_mbuf); + +/* + * Structure contains the info for each batched memory copy. + */ +struct batch_copy_elem { + void *dst; + void *src; + uint32_t len; + uint64_t log_addr; +}; + +/** + * Structure contains variables relevant to RX/TX virtqueues. + */ +struct vhost_virtqueue { + struct vring_desc *desc; + struct vring_avail *avail; + struct vring_used *used; + uint32_t size; + + uint16_t last_avail_idx; + uint16_t last_used_idx; +#define VIRTIO_INVALID_EVENTFD (-1) +#define VIRTIO_UNINITIALIZED_EVENTFD (-2) + + /* Backend value to determine if device should started/stopped */ + int backend; + int enabled; + int access_ok; + rte_spinlock_t access_lock; + + /* Used to notify the guest (trigger interrupt) */ + int callfd; + /* Currently unused as polling mode is enabled */ + int kickfd; + + /* Physical address of used ring, for logging */ + uint64_t log_guest_addr; + + uint16_t nr_zmbuf; + uint16_t zmbuf_size; + uint16_t last_zmbuf_idx; + struct zcopy_mbuf *zmbufs; + struct zcopy_mbuf_list zmbuf_list; + + struct vring_used_elem *shadow_used_ring; + uint16_t shadow_used_idx; + struct vhost_vring_addr ring_addrs; + + struct batch_copy_elem *batch_copy_elems; + uint16_t batch_copy_nb_elems; + + rte_rwlock_t iotlb_lock; + rte_rwlock_t iotlb_pending_lock; + struct rte_mempool *iotlb_pool; + TAILQ_HEAD(, vhost_iotlb_entry) iotlb_list; + int iotlb_cache_nr; + TAILQ_HEAD(, vhost_iotlb_entry) iotlb_pending_list; +} __rte_cache_aligned; + +/* Old kernels have no such macros defined */ +#ifndef VIRTIO_NET_F_GUEST_ANNOUNCE + #define VIRTIO_NET_F_GUEST_ANNOUNCE 21 +#endif + +#ifndef VIRTIO_NET_F_MQ + #define VIRTIO_NET_F_MQ 22 +#endif + +#define VHOST_MAX_VRING 0x100 +#define VHOST_MAX_QUEUE_PAIRS 0x80 + +#ifndef VIRTIO_NET_F_MTU + #define VIRTIO_NET_F_MTU 3 +#endif + +/* Declare IOMMU related bits for older kernels */ +#ifndef VIRTIO_F_IOMMU_PLATFORM + +#define VIRTIO_F_IOMMU_PLATFORM 33 + +struct vhost_iotlb_msg { + __u64 iova; + __u64 size; + __u64 uaddr; +#define VHOST_ACCESS_RO 0x1 +#define VHOST_ACCESS_WO 0x2 +#define VHOST_ACCESS_RW 0x3 + __u8 perm; +#define VHOST_IOTLB_MISS 1 +#define VHOST_IOTLB_UPDATE 2 +#define VHOST_IOTLB_INVALIDATE 3 +#define VHOST_IOTLB_ACCESS_FAIL 4 + __u8 type; +}; + +#define VHOST_IOTLB_MSG 0x1 + +struct vhost_msg { + int type; + union { + struct vhost_iotlb_msg iotlb; + __u8 padding[64]; + }; +}; +#endif + +/* + * Define virtio 1.0 for older kernels + */ +#ifndef VIRTIO_F_VERSION_1 + #define VIRTIO_F_VERSION_1 32 +#endif + +#define VHOST_USER_F_PROTOCOL_FEATURES 30 + +/* Features supported by this builtin vhost-user net driver. */ +#define VIRTIO_NET_SUPPORTED_FEATURES ((1ULL << VIRTIO_NET_F_MRG_RXBUF) | \ + (1ULL << VIRTIO_NET_F_CTRL_VQ) | \ + (1ULL << VIRTIO_NET_F_CTRL_RX) | \ + (1ULL << VIRTIO_NET_F_GUEST_ANNOUNCE) | \ + (1ULL << VIRTIO_NET_F_MQ) | \ + (1ULL << VIRTIO_F_VERSION_1) | \ + (1ULL << VHOST_F_LOG_ALL) | \ + (1ULL << VHOST_USER_F_PROTOCOL_FEATURES) | \ + (1ULL << VIRTIO_NET_F_HOST_TSO4) | \ + (1ULL << VIRTIO_NET_F_HOST_TSO6) | \ + (1ULL << VIRTIO_NET_F_CSUM) | \ + (1ULL << VIRTIO_NET_F_GUEST_CSUM) | \ + (1ULL << VIRTIO_NET_F_GUEST_TSO4) | \ + (1ULL << VIRTIO_NET_F_GUEST_TSO6) | \ + (1ULL << VIRTIO_RING_F_INDIRECT_DESC) | \ + (1ULL << VIRTIO_NET_F_MTU) | \ + (1ULL << VIRTIO_F_IOMMU_PLATFORM)) + + +struct guest_page { + uint64_t guest_phys_addr; + uint64_t host_phys_addr; + uint64_t size; +}; + +/** + * Device structure contains all configuration information relating + * to the device. + */ +struct virtio_net { + /* Frontend (QEMU) memory and memory region information */ + struct rte_vhost_memory *mem; + uint64_t features; + uint64_t protocol_features; + int vid; + uint32_t flags; + uint16_t vhost_hlen; + /* to tell if we need broadcast rarp packet */ + rte_atomic16_t broadcast_rarp; + uint32_t nr_vring; + int dequeue_zero_copy; + struct vhost_virtqueue *virtqueue[VHOST_MAX_QUEUE_PAIRS * 2]; +#define IF_NAME_SZ (PATH_MAX > IFNAMSIZ ? PATH_MAX : IFNAMSIZ) + char ifname[IF_NAME_SZ]; + uint64_t log_size; + uint64_t log_base; + uint64_t log_addr; + struct ether_addr mac; + uint16_t mtu; + + struct vhost_device_ops const *notify_ops; + + uint32_t nr_guest_pages; + uint32_t max_guest_pages; + struct guest_page *guest_pages; + + int slave_req_fd; +} __rte_cache_aligned; + + +#define VHOST_LOG_PAGE 4096 + +/* + * Atomically set a bit in memory. + */ +static __rte_always_inline void +vhost_set_bit(unsigned int nr, volatile uint8_t *addr) +{ + __sync_fetch_and_or_8(addr, (1U << nr)); +} + +static __rte_always_inline void +vhost_log_page(uint8_t *log_base, uint64_t page) +{ + vhost_set_bit(page % 8, &log_base[page / 8]); +} + +static __rte_always_inline void +vhost_log_write(struct virtio_net *dev, uint64_t addr, uint64_t len) +{ + uint64_t page; + + if (likely(((dev->features & (1ULL << VHOST_F_LOG_ALL)) == 0) || + !dev->log_base || !len)) + return; + + if (unlikely(dev->log_size <= ((addr + len - 1) / VHOST_LOG_PAGE / 8))) + return; + + /* To make sure guest memory updates are committed before logging */ + rte_smp_wmb(); + + page = addr / VHOST_LOG_PAGE; + while (page * VHOST_LOG_PAGE < addr + len) { + vhost_log_page((uint8_t *)(uintptr_t)dev->log_base, page); + page += 1; + } +} + +static __rte_always_inline void +vhost_log_used_vring(struct virtio_net *dev, struct vhost_virtqueue *vq, + uint64_t offset, uint64_t len) +{ + vhost_log_write(dev, vq->log_guest_addr + offset, len); +} + +/* Macros for printing using RTE_LOG */ +#define RTE_LOGTYPE_VHOST_CONFIG RTE_LOGTYPE_USER1 +#define RTE_LOGTYPE_VHOST_DATA RTE_LOGTYPE_USER1 + +#ifdef RTE_LIBRTE_VHOST_DEBUG +#define VHOST_MAX_PRINT_BUFF 6072 +#define LOG_LEVEL RTE_LOG_DEBUG +#define LOG_DEBUG(log_type, fmt, args...) RTE_LOG(DEBUG, log_type, fmt, ##args) +#define PRINT_PACKET(device, addr, size, header) do { \ + char *pkt_addr = (char *)(addr); \ + unsigned int index; \ + char packet[VHOST_MAX_PRINT_BUFF]; \ + \ + if ((header)) \ + snprintf(packet, VHOST_MAX_PRINT_BUFF, "(%d) Header size %d: ", (device->vid), (size)); \ + else \ + snprintf(packet, VHOST_MAX_PRINT_BUFF, "(%d) Packet size %d: ", (device->vid), (size)); \ + for (index = 0; index < (size); index++) { \ + snprintf(packet + strnlen(packet, VHOST_MAX_PRINT_BUFF), VHOST_MAX_PRINT_BUFF - strnlen(packet, VHOST_MAX_PRINT_BUFF), \ + "%02hhx ", pkt_addr[index]); \ + } \ + snprintf(packet + strnlen(packet, VHOST_MAX_PRINT_BUFF), VHOST_MAX_PRINT_BUFF - strnlen(packet, VHOST_MAX_PRINT_BUFF), "\n"); \ + \ + LOG_DEBUG(VHOST_DATA, "%s", packet); \ +} while (0) +#else +#define LOG_LEVEL RTE_LOG_INFO +#define LOG_DEBUG(log_type, fmt, args...) do {} while (0) +#define PRINT_PACKET(device, addr, size, header) do {} while (0) +#endif + +extern uint64_t VHOST_FEATURES; +#define MAX_VHOST_DEVICE 1024 +extern struct virtio_net *vhost_devices[MAX_VHOST_DEVICE]; + +/* Convert guest physical address to host physical address */ +static __rte_always_inline rte_iova_t +gpa_to_hpa(struct virtio_net *dev, uint64_t gpa, uint64_t size) +{ + uint32_t i; + struct guest_page *page; + + for (i = 0; i < dev->nr_guest_pages; i++) { + page = &dev->guest_pages[i]; + + if (gpa >= page->guest_phys_addr && + gpa + size < page->guest_phys_addr + page->size) { + return gpa - page->guest_phys_addr + + page->host_phys_addr; + } + } + + return 0; +} + +struct virtio_net *get_device(int vid); + +int vhost_new_device(void); +void cleanup_device(struct virtio_net *dev, int destroy); +void reset_device(struct virtio_net *dev); +void vhost_destroy_device(int); + +int alloc_vring_queue(struct virtio_net *dev, uint32_t vring_idx); + +void vhost_set_ifname(int, const char *if_name, unsigned int if_len); +void vhost_enable_dequeue_zero_copy(int vid); + +struct vhost_device_ops const *vhost_driver_callback_get(const char *path); + +/* + * Backend-specific cleanup. + * + * TODO: fix it; we have one backend now + */ +void vhost_backend_cleanup(struct virtio_net *dev); + +uint64_t __vhost_iova_to_vva(struct virtio_net *dev, struct vhost_virtqueue *vq, + uint64_t iova, uint64_t *len, uint8_t perm); +int vring_translate(struct virtio_net *dev, struct vhost_virtqueue *vq); +void vring_invalidate(struct virtio_net *dev, struct vhost_virtqueue *vq); + +static __rte_always_inline uint64_t +vhost_iova_to_vva(struct virtio_net *dev, struct vhost_virtqueue *vq, + uint64_t iova, uint64_t *len, uint8_t perm) +{ + if (!(dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))) + return rte_vhost_va_from_guest_pa(dev->mem, iova, len); + + return __vhost_iova_to_vva(dev, vq, iova, len, perm); +} + +#endif /* _VHOST_NET_CDEV_H_ */ diff --git a/dpdk/lib/librte_vhost/vhost_cuse/eventfd_copy.c b/dpdk/lib/librte_vhost/vhost_cuse/eventfd_copy.c deleted file mode 100644 index 154b32a4..00000000 --- a/dpdk/lib/librte_vhost/vhost_cuse/eventfd_copy.c +++ /dev/null @@ -1,104 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include -#include -#include -#include -#include -#include - -#include - -#include "eventfd_link/eventfd_link.h" -#include "eventfd_copy.h" -#include "vhost-net.h" - -static const char eventfd_cdev[] = "/dev/eventfd-link"; - -static int eventfd_link = -1; - -int -eventfd_init(void) -{ - if (eventfd_link >= 0) - return 0; - - eventfd_link = open(eventfd_cdev, O_RDWR); - if (eventfd_link < 0) { - RTE_LOG(ERR, VHOST_CONFIG, - "eventfd_link module is not loaded\n"); - return -1; - } - - return 0; -} - -int -eventfd_free(void) -{ - if (eventfd_link >= 0) - close(eventfd_link); - return 0; -} - -/* - * This function uses the eventfd_link kernel module to copy an eventfd file - * descriptor provided by QEMU in to our process space. - */ -int -eventfd_copy(int target_fd, int target_pid) -{ - int ret; - struct eventfd_copy2 eventfd_copy2; - - - /* Open the character device to the kernel module. */ - /* TODO: check this earlier rather than fail until VM boots! */ - if (eventfd_init() < 0) - return -1; - - eventfd_copy2.fd = target_fd; - eventfd_copy2.pid = target_pid; - eventfd_copy2.flags = O_NONBLOCK | O_CLOEXEC; - /* Call the IOCTL to copy the eventfd. */ - ret = ioctl(eventfd_link, EVENTFD_COPY2, &eventfd_copy2); - - if (ret < 0) { - RTE_LOG(ERR, VHOST_CONFIG, - "EVENTFD_COPY2 ioctl failed\n"); - return -1; - } - - return ret; -} diff --git a/dpdk/lib/librte_vhost/vhost_cuse/eventfd_copy.h b/dpdk/lib/librte_vhost/vhost_cuse/eventfd_copy.h deleted file mode 100644 index 5f446ca0..00000000 --- a/dpdk/lib/librte_vhost/vhost_cuse/eventfd_copy.h +++ /dev/null @@ -1,45 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -#ifndef _EVENTFD_H -#define _EVENTFD_H - -int -eventfd_init(void); - -int -eventfd_free(void); - -int -eventfd_copy(int target_fd, int target_pid); - -#endif diff --git a/dpdk/lib/librte_vhost/vhost_cuse/vhost-net-cdev.c b/dpdk/lib/librte_vhost/vhost_cuse/vhost-net-cdev.c deleted file mode 100644 index 5d150116..00000000 --- a/dpdk/lib/librte_vhost/vhost_cuse/vhost-net-cdev.c +++ /dev/null @@ -1,431 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include "virtio-net-cdev.h" -#include "vhost-net.h" -#include "eventfd_copy.h" - -#define FUSE_OPT_DUMMY "\0\0" -#define FUSE_OPT_FORE "-f\0\0" -#define FUSE_OPT_NOMULTI "-s\0\0" - -static const uint32_t default_major = 231; -static const uint32_t default_minor = 1; -static const char cuse_device_name[] = "/dev/cuse"; -static const char default_cdev[] = "vhost-net"; - -static struct fuse_session *session; - -/* - * Returns vhost_cuse_device_ctx from given fuse_req_t. The - * index is populated later when the device is added to the - * device linked list. - */ -static struct vhost_cuse_device_ctx -fuse_req_to_vhost_ctx(fuse_req_t req, struct fuse_file_info *fi) -{ - struct vhost_cuse_device_ctx ctx; - struct fuse_ctx const *const req_ctx = fuse_req_ctx(req); - - ctx.pid = req_ctx->pid; - ctx.vid = (int)fi->fh; - - return ctx; -} - -/* - * When the device is created in QEMU it gets initialised here and - * added to the device linked list. - */ -static void -vhost_net_open(fuse_req_t req, struct fuse_file_info *fi) -{ - int vid = 0; - - vid = vhost_new_device(); - if (vid == -1) { - fuse_reply_err(req, EPERM); - return; - } - - fi->fh = vid; - - RTE_LOG(INFO, VHOST_CONFIG, - "(%d) device configuration started\n", vid); - fuse_reply_open(req, fi); -} - -/* - * When QEMU is shutdown or killed the device gets released. - */ -static void -vhost_net_release(fuse_req_t req, struct fuse_file_info *fi) -{ - int err = 0; - struct vhost_cuse_device_ctx ctx = fuse_req_to_vhost_ctx(req, fi); - - vhost_destroy_device(ctx.vid); - RTE_LOG(INFO, VHOST_CONFIG, "(%d) device released\n", ctx.vid); - fuse_reply_err(req, err); -} - -/* - * Boilerplate code for CUSE IOCTL - * Implicit arguments: vid, req, result. - */ -#define VHOST_IOCTL(func) do { \ - result = (func)(vid); \ - fuse_reply_ioctl(req, result, NULL, 0); \ -} while (0) - -/* - * Boilerplate IOCTL RETRY - * Implicit arguments: req. - */ -#define VHOST_IOCTL_RETRY(size_r, size_w) do { \ - struct iovec iov_r = { arg, (size_r) }; \ - struct iovec iov_w = { arg, (size_w) }; \ - fuse_reply_ioctl_retry(req, &iov_r, \ - (size_r) ? 1 : 0, &iov_w, (size_w) ? 1 : 0);\ -} while (0) - -/* - * Boilerplate code for CUSE Read IOCTL - * Implicit arguments: vid, req, result, in_bufsz, in_buf. - */ -#define VHOST_IOCTL_R(type, var, func) do { \ - if (!in_bufsz) { \ - VHOST_IOCTL_RETRY(sizeof(type), 0);\ - } else { \ - (var) = *(const type*)in_buf; \ - result = func(vid, &(var)); \ - fuse_reply_ioctl(req, result, NULL, 0);\ - } \ -} while (0) - -/* - * Boilerplate code for CUSE Write IOCTL - * Implicit arguments: vid, req, result, out_bufsz. - */ -#define VHOST_IOCTL_W(type, var, func) do { \ - if (!out_bufsz) { \ - VHOST_IOCTL_RETRY(0, sizeof(type));\ - } else { \ - result = (func)(vid, &(var));\ - fuse_reply_ioctl(req, result, &(var), sizeof(type));\ - } \ -} while (0) - -/* - * Boilerplate code for CUSE Read/Write IOCTL - * Implicit arguments: vid, req, result, in_bufsz, in_buf. - */ -#define VHOST_IOCTL_RW(type1, var1, type2, var2, func) do { \ - if (!in_bufsz) { \ - VHOST_IOCTL_RETRY(sizeof(type1), sizeof(type2));\ - } else { \ - (var1) = *(const type1*) (in_buf); \ - result = (func)(vid, (var1), &(var2)); \ - fuse_reply_ioctl(req, result, &(var2), sizeof(type2));\ - } \ -} while (0) - -/* - * The IOCTLs are handled using CUSE/FUSE in userspace. Depending on the type - * of IOCTL a buffer is requested to read or to write. This request is handled - * by FUSE and the buffer is then given to CUSE. - */ -static void -vhost_net_ioctl(fuse_req_t req, int cmd, void *arg, - struct fuse_file_info *fi, __rte_unused unsigned flags, - const void *in_buf, size_t in_bufsz, size_t out_bufsz) -{ - struct vhost_cuse_device_ctx ctx = fuse_req_to_vhost_ctx(req, fi); - struct vhost_vring_file file; - struct vhost_vring_state state; - struct vhost_vring_addr addr; - uint64_t features; - uint32_t index; - int result = 0; - int vid = ctx.vid; - - switch (cmd) { - case VHOST_NET_SET_BACKEND: - LOG_DEBUG(VHOST_CONFIG, - "(%d) IOCTL: VHOST_NET_SET_BACKEND\n", ctx.vid); - if (!in_buf) { - VHOST_IOCTL_RETRY(sizeof(file), 0); - break; - } - file = *(const struct vhost_vring_file *)in_buf; - result = cuse_set_backend(ctx, &file); - fuse_reply_ioctl(req, result, NULL, 0); - break; - - case VHOST_GET_FEATURES: - LOG_DEBUG(VHOST_CONFIG, - "(%d) IOCTL: VHOST_GET_FEATURES\n", vid); - VHOST_IOCTL_W(uint64_t, features, vhost_get_features); - break; - - case VHOST_SET_FEATURES: - LOG_DEBUG(VHOST_CONFIG, - "(%d) IOCTL: VHOST_SET_FEATURES\n", vid); - VHOST_IOCTL_R(uint64_t, features, vhost_set_features); - break; - - case VHOST_RESET_OWNER: - LOG_DEBUG(VHOST_CONFIG, - "(%d) IOCTL: VHOST_RESET_OWNER\n", vid); - VHOST_IOCTL(vhost_reset_owner); - break; - - case VHOST_SET_OWNER: - LOG_DEBUG(VHOST_CONFIG, - "(%d) IOCTL: VHOST_SET_OWNER\n", vid); - VHOST_IOCTL(vhost_set_owner); - break; - - case VHOST_SET_MEM_TABLE: - /*TODO fix race condition.*/ - LOG_DEBUG(VHOST_CONFIG, - "(%d) IOCTL: VHOST_SET_MEM_TABLE\n", vid); - static struct vhost_memory mem_temp; - - switch (in_bufsz) { - case 0: - VHOST_IOCTL_RETRY(sizeof(struct vhost_memory), 0); - break; - - case sizeof(struct vhost_memory): - mem_temp = *(const struct vhost_memory *) in_buf; - - if (mem_temp.nregions > 0) { - VHOST_IOCTL_RETRY(sizeof(struct vhost_memory) + - (sizeof(struct vhost_memory_region) * - mem_temp.nregions), 0); - } else { - result = -1; - fuse_reply_ioctl(req, result, NULL, 0); - } - break; - - default: - result = cuse_set_mem_table(ctx, in_buf, - mem_temp.nregions); - if (result) - fuse_reply_err(req, EINVAL); - else - fuse_reply_ioctl(req, result, NULL, 0); - } - break; - - case VHOST_SET_VRING_NUM: - LOG_DEBUG(VHOST_CONFIG, - "(%d) IOCTL: VHOST_SET_VRING_NUM\n", vid); - VHOST_IOCTL_R(struct vhost_vring_state, state, - vhost_set_vring_num); - break; - - case VHOST_SET_VRING_BASE: - LOG_DEBUG(VHOST_CONFIG, - "(%d) IOCTL: VHOST_SET_VRING_BASE\n", vid); - VHOST_IOCTL_R(struct vhost_vring_state, state, - vhost_set_vring_base); - break; - - case VHOST_GET_VRING_BASE: - LOG_DEBUG(VHOST_CONFIG, - "(%d) IOCTL: VHOST_GET_VRING_BASE\n", vid); - VHOST_IOCTL_RW(uint32_t, index, - struct vhost_vring_state, state, vhost_get_vring_base); - break; - - case VHOST_SET_VRING_ADDR: - LOG_DEBUG(VHOST_CONFIG, - "(%d) IOCTL: VHOST_SET_VRING_ADDR\n", vid); - VHOST_IOCTL_R(struct vhost_vring_addr, addr, - vhost_set_vring_addr); - break; - - case VHOST_SET_VRING_KICK: - case VHOST_SET_VRING_CALL: - if (cmd == VHOST_SET_VRING_KICK) - LOG_DEBUG(VHOST_CONFIG, - "(%d) IOCTL: VHOST_SET_VRING_KICK\n", vid); - else - LOG_DEBUG(VHOST_CONFIG, - "(%d) IOCTL: VHOST_SET_VRING_CALL\n", vid); - if (!in_buf) - VHOST_IOCTL_RETRY(sizeof(struct vhost_vring_file), 0); - else { - int fd; - file = *(const struct vhost_vring_file *)in_buf; - LOG_DEBUG(VHOST_CONFIG, - "idx:%d fd:%d\n", file.index, file.fd); - fd = eventfd_copy(file.fd, ctx.pid); - if (fd < 0) { - fuse_reply_ioctl(req, -1, NULL, 0); - result = -1; - break; - } - file.fd = fd; - if (cmd == VHOST_SET_VRING_KICK) { - result = vhost_set_vring_kick(vid, &file); - fuse_reply_ioctl(req, result, NULL, 0); - } else { - result = vhost_set_vring_call(vid, &file); - fuse_reply_ioctl(req, result, NULL, 0); - } - } - break; - - default: - RTE_LOG(ERR, VHOST_CONFIG, - "(%d) IOCTL: DOESN NOT EXIST\n", vid); - result = -1; - fuse_reply_ioctl(req, result, NULL, 0); - } - - if (result < 0) - LOG_DEBUG(VHOST_CONFIG, - "(%d) IOCTL: FAIL\n", vid); - else - LOG_DEBUG(VHOST_CONFIG, - "(%d) IOCTL: SUCCESS\n", vid); -} - -/* - * Structure handling open, release and ioctl function pointers is populated. - */ -static const struct cuse_lowlevel_ops vhost_net_ops = { - .open = vhost_net_open, - .release = vhost_net_release, - .ioctl = vhost_net_ioctl, -}; - -/* - * cuse_info is populated and used to register the cuse device. - * vhost_net_device_ops are also passed when the device is registered in app. - */ -int -rte_vhost_driver_register(const char *dev_name, uint64_t flags) -{ - struct cuse_info cuse_info; - char device_name[PATH_MAX] = ""; - char char_device_name[PATH_MAX] = ""; - const char *device_argv[] = { device_name }; - - char fuse_opt_dummy[] = FUSE_OPT_DUMMY; - char fuse_opt_fore[] = FUSE_OPT_FORE; - char fuse_opt_nomulti[] = FUSE_OPT_NOMULTI; - char *fuse_argv[] = {fuse_opt_dummy, fuse_opt_fore, fuse_opt_nomulti}; - - if (flags) { - RTE_LOG(ERR, VHOST_CONFIG, - "vhost-cuse does not support any flags so far\n"); - return -1; - } - - if (access(cuse_device_name, R_OK | W_OK) < 0) { - RTE_LOG(ERR, VHOST_CONFIG, - "char device %s can't be accessed, maybe not exist\n", - cuse_device_name); - return -1; - } - - if (eventfd_init() < 0) - return -1; - - /* - * The device name is created. This is passed to QEMU so that it can - * register the device with our application. - */ - snprintf(device_name, PATH_MAX, "DEVNAME=%s", dev_name); - snprintf(char_device_name, PATH_MAX, "/dev/%s", dev_name); - - /* Check if device already exists. */ - if (access(char_device_name, F_OK) != -1) { - RTE_LOG(ERR, VHOST_CONFIG, - "char device %s already exists\n", char_device_name); - return -1; - } - - memset(&cuse_info, 0, sizeof(cuse_info)); - cuse_info.dev_major = default_major; - cuse_info.dev_minor = default_minor; - cuse_info.dev_info_argc = 1; - cuse_info.dev_info_argv = device_argv; - cuse_info.flags = CUSE_UNRESTRICTED_IOCTL; - - session = cuse_lowlevel_setup(3, fuse_argv, - &cuse_info, &vhost_net_ops, 0, NULL); - if (session == NULL) - return -1; - - return 0; -} - -/** - * An empty function for unregister - */ -int -rte_vhost_driver_unregister(const char *dev_name __rte_unused) -{ - return 0; -} - -/** - * The CUSE session is launched allowing the application to receive open, - * release and ioctl calls. - */ -int -rte_vhost_driver_session_start(void) -{ - fuse_session_loop(session); - - return 0; -} diff --git a/dpdk/lib/librte_vhost/vhost_cuse/virtio-net-cdev.c b/dpdk/lib/librte_vhost/vhost_cuse/virtio-net-cdev.c deleted file mode 100644 index 552be7d4..00000000 --- a/dpdk/lib/librte_vhost/vhost_cuse/virtio-net-cdev.c +++ /dev/null @@ -1,433 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include "rte_virtio_net.h" -#include "vhost-net.h" -#include "virtio-net-cdev.h" -#include "eventfd_copy.h" - -/* Line size for reading maps file. */ -static const uint32_t BUFSIZE = PATH_MAX; - -/* Size of prot char array in procmap. */ -#define PROT_SZ 5 - -/* Number of elements in procmap struct. */ -#define PROCMAP_SZ 8 - -/* Structure containing information gathered from maps file. */ -struct procmap { - uint64_t va_start; /* Start virtual address in file. */ - uint64_t len; /* Size of file. */ - uint64_t pgoff; /* Not used. */ - uint32_t maj; /* Not used. */ - uint32_t min; /* Not used. */ - uint32_t ino; /* Not used. */ - char prot[PROT_SZ]; /* Not used. */ - char fname[PATH_MAX]; /* File name. */ -}; - -/* - * Locate the file containing QEMU's memory space and - * map it to our address space. - */ -static int -host_memory_map(pid_t pid, uint64_t addr, - uint64_t *mapped_address, uint64_t *mapped_size) -{ - struct dirent *dptr = NULL; - struct procmap procmap; - DIR *dp = NULL; - int fd; - int i; - char memfile[PATH_MAX]; - char mapfile[PATH_MAX]; - char procdir[PATH_MAX]; - char resolved_path[PATH_MAX]; - char *path = NULL; - FILE *fmap; - void *map; - uint8_t found = 0; - char line[BUFSIZE]; - char dlm[] = "- : "; - char *str, *sp, *in[PROCMAP_SZ]; - char *end = NULL; - - /* Path where mem files are located. */ - snprintf(procdir, PATH_MAX, "/proc/%u/fd/", pid); - /* Maps file used to locate mem file. */ - snprintf(mapfile, PATH_MAX, "/proc/%u/maps", pid); - - fmap = fopen(mapfile, "r"); - if (fmap == NULL) { - RTE_LOG(ERR, VHOST_CONFIG, - "Failed to open maps file for pid %d\n", - pid); - return -1; - } - - /* Read through maps file until we find out base_address. */ - while (fgets(line, BUFSIZE, fmap) != 0) { - str = line; - errno = 0; - /* Split line into fields. */ - for (i = 0; i < PROCMAP_SZ; i++) { - in[i] = strtok_r(str, &dlm[i], &sp); - if ((in[i] == NULL) || (errno != 0)) { - fclose(fmap); - return -1; - } - str = NULL; - } - - /* Convert/Copy each field as needed. */ - procmap.va_start = strtoull(in[0], &end, 16); - if ((in[0] == '\0') || (end == NULL) || (*end != '\0') || - (errno != 0)) { - fclose(fmap); - return -1; - } - - procmap.len = strtoull(in[1], &end, 16); - if ((in[1] == '\0') || (end == NULL) || (*end != '\0') || - (errno != 0)) { - fclose(fmap); - return -1; - } - - procmap.pgoff = strtoull(in[3], &end, 16); - if ((in[3] == '\0') || (end == NULL) || (*end != '\0') || - (errno != 0)) { - fclose(fmap); - return -1; - } - - procmap.maj = strtoul(in[4], &end, 16); - if ((in[4] == '\0') || (end == NULL) || (*end != '\0') || - (errno != 0)) { - fclose(fmap); - return -1; - } - - procmap.min = strtoul(in[5], &end, 16); - if ((in[5] == '\0') || (end == NULL) || (*end != '\0') || - (errno != 0)) { - fclose(fmap); - return -1; - } - - procmap.ino = strtoul(in[6], &end, 16); - if ((in[6] == '\0') || (end == NULL) || (*end != '\0') || - (errno != 0)) { - fclose(fmap); - return -1; - } - - memcpy(&procmap.prot, in[2], PROT_SZ); - memcpy(&procmap.fname, in[7], PATH_MAX); - - if (procmap.va_start == addr) { - procmap.len = procmap.len - procmap.va_start; - found = 1; - break; - } - } - fclose(fmap); - - if (!found) { - RTE_LOG(ERR, VHOST_CONFIG, - "Failed to find memory file in pid %d maps file\n", - pid); - return -1; - } - - /* Find the guest memory file among the process fds. */ - dp = opendir(procdir); - if (dp == NULL) { - RTE_LOG(ERR, VHOST_CONFIG, - "Cannot open pid %d process directory\n", - pid); - return -1; - } - - found = 0; - - /* Read the fd directory contents. */ - while (NULL != (dptr = readdir(dp))) { - snprintf(memfile, PATH_MAX, "/proc/%u/fd/%s", - pid, dptr->d_name); - path = realpath(memfile, resolved_path); - if ((path == NULL) && (strlen(resolved_path) == 0)) { - RTE_LOG(ERR, VHOST_CONFIG, - "Failed to resolve fd directory\n"); - closedir(dp); - return -1; - } - if (strncmp(resolved_path, procmap.fname, - strnlen(procmap.fname, PATH_MAX)) == 0) { - found = 1; - break; - } - } - - closedir(dp); - - if (found == 0) { - RTE_LOG(ERR, VHOST_CONFIG, - "Failed to find memory file for pid %d\n", - pid); - return -1; - } - /* Open the shared memory file and map the memory into this process. */ - fd = open(memfile, O_RDWR); - - if (fd == -1) { - RTE_LOG(ERR, VHOST_CONFIG, - "Failed to open %s for pid %d\n", - memfile, pid); - return -1; - } - - map = mmap(0, (size_t)procmap.len, PROT_READ|PROT_WRITE, - MAP_POPULATE|MAP_SHARED, fd, 0); - close(fd); - - if (map == MAP_FAILED) { - RTE_LOG(ERR, VHOST_CONFIG, - "Error mapping the file %s for pid %d\n", - memfile, pid); - return -1; - } - - /* Store the memory address and size in the device data structure */ - *mapped_address = (uint64_t)(uintptr_t)map; - *mapped_size = procmap.len; - - LOG_DEBUG(VHOST_CONFIG, - "Mem File: %s->%s - Size: %llu - VA: %p\n", - memfile, resolved_path, - (unsigned long long)*mapped_size, map); - - return 0; -} - -int -cuse_set_mem_table(struct vhost_cuse_device_ctx ctx, - const struct vhost_memory *mem_regions_addr, uint32_t nregions) -{ - uint64_t size = offsetof(struct vhost_memory, regions); - uint32_t idx, valid_regions; - struct virtio_memory_regions *pregion; - struct vhost_memory_region *mem_regions = (void *)(uintptr_t) - ((uint64_t)(uintptr_t)mem_regions_addr + size); - uint64_t base_address = 0, mapped_address, mapped_size; - struct virtio_net *dev; - - dev = get_device(ctx.vid); - if (dev == NULL) - return -1; - - if (dev->mem && dev->mem->mapped_address) { - munmap((void *)(uintptr_t)dev->mem->mapped_address, - (size_t)dev->mem->mapped_size); - free(dev->mem); - dev->mem = NULL; - } - - dev->mem = calloc(1, sizeof(struct virtio_memory) + - sizeof(struct virtio_memory_regions) * nregions); - if (dev->mem == NULL) { - RTE_LOG(ERR, VHOST_CONFIG, - "(%d) failed to allocate memory for dev->mem\n", - dev->vid); - return -1; - } - - pregion = &dev->mem->regions[0]; - - for (idx = 0; idx < nregions; idx++) { - pregion[idx].guest_phys_address = - mem_regions[idx].guest_phys_addr; - pregion[idx].guest_phys_address_end = - pregion[idx].guest_phys_address + - mem_regions[idx].memory_size; - pregion[idx].memory_size = - mem_regions[idx].memory_size; - pregion[idx].userspace_address = - mem_regions[idx].userspace_addr; - - LOG_DEBUG(VHOST_CONFIG, - "REGION: %u - GPA: %p - QVA: %p - SIZE (%"PRIu64")\n", - idx, - (void *)(uintptr_t)pregion[idx].guest_phys_address, - (void *)(uintptr_t)pregion[idx].userspace_address, - pregion[idx].memory_size); - - /*set the base address mapping*/ - if (pregion[idx].guest_phys_address == 0x0) { - base_address = - pregion[idx].userspace_address; - /* Map VM memory file */ - if (host_memory_map(ctx.pid, base_address, - &mapped_address, &mapped_size) != 0) { - free(dev->mem); - dev->mem = NULL; - return -1; - } - dev->mem->mapped_address = mapped_address; - dev->mem->base_address = base_address; - dev->mem->mapped_size = mapped_size; - } - } - - /* Check that we have a valid base address. */ - if (base_address == 0) { - RTE_LOG(ERR, VHOST_CONFIG, - "Failed to find base address of qemu memory file.\n"); - free(dev->mem); - dev->mem = NULL; - return -1; - } - - valid_regions = nregions; - for (idx = 0; idx < nregions; idx++) { - if ((pregion[idx].userspace_address < base_address) || - (pregion[idx].userspace_address > - (base_address + mapped_size))) - valid_regions--; - } - - - if (valid_regions != nregions) { - valid_regions = 0; - for (idx = nregions; 0 != idx--; ) { - if ((pregion[idx].userspace_address < base_address) || - (pregion[idx].userspace_address > - (base_address + mapped_size))) { - memmove(&pregion[idx], &pregion[idx + 1], - sizeof(struct virtio_memory_regions) * - valid_regions); - } else - valid_regions++; - } - } - - for (idx = 0; idx < valid_regions; idx++) { - pregion[idx].address_offset = - mapped_address - base_address + - pregion[idx].userspace_address - - pregion[idx].guest_phys_address; - } - dev->mem->nregions = valid_regions; - - return 0; -} - -/* - * Function to get the tap device name from the provided file descriptor and - * save it in the device structure. - */ -static int -get_ifname(int vid, int tap_fd, int pid) -{ - int fd_tap; - struct ifreq ifr; - uint32_t ifr_size; - int ret; - - fd_tap = eventfd_copy(tap_fd, pid); - if (fd_tap < 0) - return -1; - - ret = ioctl(fd_tap, TUNGETIFF, &ifr); - - if (close(fd_tap) < 0) - RTE_LOG(ERR, VHOST_CONFIG, "(%d) fd close failed\n", vid); - - if (ret >= 0) { - ifr_size = strnlen(ifr.ifr_name, sizeof(ifr.ifr_name)); - vhost_set_ifname(vid, ifr.ifr_name, ifr_size); - } else - RTE_LOG(ERR, VHOST_CONFIG, - "(%d) TUNGETIFF ioctl failed\n", vid); - - return 0; -} - -int -cuse_set_backend(struct vhost_cuse_device_ctx ctx, - struct vhost_vring_file *file) -{ - struct virtio_net *dev; - - dev = get_device(ctx.vid); - if (dev == NULL) - return -1; - - if (!(dev->flags & VIRTIO_DEV_RUNNING) && file->fd != VIRTIO_DEV_STOPPED) - get_ifname(ctx.vid, file->fd, ctx.pid); - - return vhost_set_backend(ctx.vid, file); -} - -void -vhost_backend_cleanup(struct virtio_net *dev) -{ - /* Unmap QEMU memory file if mapped. */ - if (dev->mem) { - munmap((void *)(uintptr_t)dev->mem->mapped_address, - (size_t)dev->mem->mapped_size); - free(dev->mem); - dev->mem = NULL; - } -} diff --git a/dpdk/lib/librte_vhost/vhost_cuse/virtio-net-cdev.h b/dpdk/lib/librte_vhost/vhost_cuse/virtio-net-cdev.h deleted file mode 100644 index 3f67154b..00000000 --- a/dpdk/lib/librte_vhost/vhost_cuse/virtio-net-cdev.h +++ /dev/null @@ -1,56 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -#ifndef _VIRTIO_NET_CDEV_H -#define _VIRTIO_NET_CDEV_H - -#include -#include - -#include "vhost-net.h" - -/* - * Structure used to identify device context. - */ -struct vhost_cuse_device_ctx { - pid_t pid; /* PID of process calling the IOCTL. */ - int vid; /* Virtio-net device ID */ -}; - -int -cuse_set_mem_table(struct vhost_cuse_device_ctx ctx, - const struct vhost_memory *mem_regions_addr, uint32_t nregions); - -int -cuse_set_backend(struct vhost_cuse_device_ctx ctx, struct vhost_vring_file *); - -#endif diff --git a/dpdk/lib/librte_vhost/vhost_rxtx.c b/dpdk/lib/librte_vhost/vhost_rxtx.c deleted file mode 100644 index 5806f99a..00000000 --- a/dpdk/lib/librte_vhost/vhost_rxtx.c +++ /dev/null @@ -1,931 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "vhost-net.h" - -#define MAX_PKT_BURST 32 -#define VHOST_LOG_PAGE 4096 - -static inline void __attribute__((always_inline)) -vhost_log_page(uint8_t *log_base, uint64_t page) -{ - log_base[page / 8] |= 1 << (page % 8); -} - -static inline void __attribute__((always_inline)) -vhost_log_write(struct virtio_net *dev, uint64_t addr, uint64_t len) -{ - uint64_t page; - - if (likely(((dev->features & (1ULL << VHOST_F_LOG_ALL)) == 0) || - !dev->log_base || !len)) - return; - - if (unlikely(dev->log_size <= ((addr + len - 1) / VHOST_LOG_PAGE / 8))) - return; - - /* To make sure guest memory updates are committed before logging */ - rte_smp_wmb(); - - page = addr / VHOST_LOG_PAGE; - while (page * VHOST_LOG_PAGE < addr + len) { - vhost_log_page((uint8_t *)(uintptr_t)dev->log_base, page); - page += 1; - } -} - -static inline void __attribute__((always_inline)) -vhost_log_used_vring(struct virtio_net *dev, struct vhost_virtqueue *vq, - uint64_t offset, uint64_t len) -{ - vhost_log_write(dev, vq->log_guest_addr + offset, len); -} - -static bool -is_valid_virt_queue_idx(uint32_t idx, int is_tx, uint32_t qp_nb) -{ - return (is_tx ^ (idx & 1)) == 0 && idx < qp_nb * VIRTIO_QNUM; -} - -static void -virtio_enqueue_offload(struct rte_mbuf *m_buf, struct virtio_net_hdr *net_hdr) -{ - if (m_buf->ol_flags & PKT_TX_L4_MASK) { - net_hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM; - net_hdr->csum_start = m_buf->l2_len + m_buf->l3_len; - - switch (m_buf->ol_flags & PKT_TX_L4_MASK) { - case PKT_TX_TCP_CKSUM: - net_hdr->csum_offset = (offsetof(struct tcp_hdr, - cksum)); - break; - case PKT_TX_UDP_CKSUM: - net_hdr->csum_offset = (offsetof(struct udp_hdr, - dgram_cksum)); - break; - case PKT_TX_SCTP_CKSUM: - net_hdr->csum_offset = (offsetof(struct sctp_hdr, - cksum)); - break; - } - } - - if (m_buf->ol_flags & PKT_TX_TCP_SEG) { - if (m_buf->ol_flags & PKT_TX_IPV4) - net_hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV4; - else - net_hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV6; - net_hdr->gso_size = m_buf->tso_segsz; - net_hdr->hdr_len = m_buf->l2_len + m_buf->l3_len - + m_buf->l4_len; - } -} - -static inline void -copy_virtio_net_hdr(struct virtio_net *dev, uint64_t desc_addr, - struct virtio_net_hdr_mrg_rxbuf hdr) -{ - if (dev->vhost_hlen == sizeof(struct virtio_net_hdr_mrg_rxbuf)) - *(struct virtio_net_hdr_mrg_rxbuf *)(uintptr_t)desc_addr = hdr; - else - *(struct virtio_net_hdr *)(uintptr_t)desc_addr = hdr.hdr; -} - -static inline int __attribute__((always_inline)) -copy_mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq, - struct rte_mbuf *m, uint16_t desc_idx) -{ - uint32_t desc_avail, desc_offset; - uint32_t mbuf_avail, mbuf_offset; - uint32_t cpy_len; - struct vring_desc *desc; - uint64_t desc_addr; - struct virtio_net_hdr_mrg_rxbuf virtio_hdr = {{0, 0, 0, 0, 0, 0}, 0}; - - desc = &vq->desc[desc_idx]; - desc_addr = gpa_to_vva(dev, desc->addr); - /* - * Checking of 'desc_addr' placed outside of 'unlikely' macro to avoid - * performance issue with some versions of gcc (4.8.4 and 5.3.0) which - * otherwise stores offset on the stack instead of in a register. - */ - if (unlikely(desc->len < dev->vhost_hlen) || !desc_addr) - return -1; - - rte_prefetch0((void *)(uintptr_t)desc_addr); - - virtio_enqueue_offload(m, &virtio_hdr.hdr); - copy_virtio_net_hdr(dev, desc_addr, virtio_hdr); - vhost_log_write(dev, desc->addr, dev->vhost_hlen); - PRINT_PACKET(dev, (uintptr_t)desc_addr, dev->vhost_hlen, 0); - - desc_offset = dev->vhost_hlen; - desc_avail = desc->len - dev->vhost_hlen; - - mbuf_avail = rte_pktmbuf_data_len(m); - mbuf_offset = 0; - while (mbuf_avail != 0 || m->next != NULL) { - /* done with current mbuf, fetch next */ - if (mbuf_avail == 0) { - m = m->next; - - mbuf_offset = 0; - mbuf_avail = rte_pktmbuf_data_len(m); - } - - /* done with current desc buf, fetch next */ - if (desc_avail == 0) { - if ((desc->flags & VRING_DESC_F_NEXT) == 0) { - /* Room in vring buffer is not enough */ - return -1; - } - if (unlikely(desc->next >= vq->size)) - return -1; - - desc = &vq->desc[desc->next]; - desc_addr = gpa_to_vva(dev, desc->addr); - if (unlikely(!desc_addr)) - return -1; - - desc_offset = 0; - desc_avail = desc->len; - } - - cpy_len = RTE_MIN(desc_avail, mbuf_avail); - rte_memcpy((void *)((uintptr_t)(desc_addr + desc_offset)), - rte_pktmbuf_mtod_offset(m, void *, mbuf_offset), - cpy_len); - vhost_log_write(dev, desc->addr + desc_offset, cpy_len); - PRINT_PACKET(dev, (uintptr_t)(desc_addr + desc_offset), - cpy_len, 0); - - mbuf_avail -= cpy_len; - mbuf_offset += cpy_len; - desc_avail -= cpy_len; - desc_offset += cpy_len; - } - - return 0; -} - -/** - * This function adds buffers to the virtio devices RX virtqueue. Buffers can - * be received from the physical port or from another virtio device. A packet - * count is returned to indicate the number of packets that are succesfully - * added to the RX queue. This function works when the mbuf is scattered, but - * it doesn't support the mergeable feature. - */ -static inline uint32_t __attribute__((always_inline)) -virtio_dev_rx(struct virtio_net *dev, uint16_t queue_id, - struct rte_mbuf **pkts, uint32_t count) -{ - struct vhost_virtqueue *vq; - uint16_t avail_idx, free_entries, start_idx; - uint16_t desc_indexes[MAX_PKT_BURST]; - uint16_t used_idx; - uint32_t i; - - LOG_DEBUG(VHOST_DATA, "(%d) %s\n", dev->vid, __func__); - if (unlikely(!is_valid_virt_queue_idx(queue_id, 0, dev->virt_qp_nb))) { - RTE_LOG(ERR, VHOST_DATA, "(%d) %s: invalid virtqueue idx %d.\n", - dev->vid, __func__, queue_id); - return 0; - } - - vq = dev->virtqueue[queue_id]; - if (unlikely(vq->enabled == 0)) - return 0; - - avail_idx = *((volatile uint16_t *)&vq->avail->idx); - start_idx = vq->last_used_idx; - free_entries = avail_idx - start_idx; - count = RTE_MIN(count, free_entries); - count = RTE_MIN(count, (uint32_t)MAX_PKT_BURST); - if (count == 0) - return 0; - - LOG_DEBUG(VHOST_DATA, "(%d) start_idx %d | end_idx %d\n", - dev->vid, start_idx, start_idx + count); - - /* Retrieve all of the desc indexes first to avoid caching issues. */ - rte_prefetch0(&vq->avail->ring[start_idx & (vq->size - 1)]); - for (i = 0; i < count; i++) { - used_idx = (start_idx + i) & (vq->size - 1); - desc_indexes[i] = vq->avail->ring[used_idx]; - vq->used->ring[used_idx].id = desc_indexes[i]; - vq->used->ring[used_idx].len = pkts[i]->pkt_len + - dev->vhost_hlen; - vhost_log_used_vring(dev, vq, - offsetof(struct vring_used, ring[used_idx]), - sizeof(vq->used->ring[used_idx])); - } - - rte_prefetch0(&vq->desc[desc_indexes[0]]); - for (i = 0; i < count; i++) { - uint16_t desc_idx = desc_indexes[i]; - int err; - - err = copy_mbuf_to_desc(dev, vq, pkts[i], desc_idx); - if (unlikely(err)) { - used_idx = (start_idx + i) & (vq->size - 1); - vq->used->ring[used_idx].len = dev->vhost_hlen; - vhost_log_used_vring(dev, vq, - offsetof(struct vring_used, ring[used_idx]), - sizeof(vq->used->ring[used_idx])); - } - - if (i + 1 < count) - rte_prefetch0(&vq->desc[desc_indexes[i+1]]); - } - - rte_smp_wmb(); - - *(volatile uint16_t *)&vq->used->idx += count; - vq->last_used_idx += count; - vhost_log_used_vring(dev, vq, - offsetof(struct vring_used, idx), - sizeof(vq->used->idx)); - - /* flush used->idx update before we read avail->flags. */ - rte_mb(); - - /* Kick the guest if necessary. */ - if (!(vq->avail->flags & VRING_AVAIL_F_NO_INTERRUPT) - && (vq->callfd >= 0)) - eventfd_write(vq->callfd, (eventfd_t)1); - return count; -} - -static inline int -fill_vec_buf(struct vhost_virtqueue *vq, uint32_t avail_idx, - uint32_t *allocated, uint32_t *vec_idx, - struct buf_vector *buf_vec) -{ - uint16_t idx = vq->avail->ring[avail_idx & (vq->size - 1)]; - uint32_t vec_id = *vec_idx; - uint32_t len = *allocated; - - while (1) { - if (unlikely(vec_id >= BUF_VECTOR_MAX || idx >= vq->size)) - return -1; - - len += vq->desc[idx].len; - buf_vec[vec_id].buf_addr = vq->desc[idx].addr; - buf_vec[vec_id].buf_len = vq->desc[idx].len; - buf_vec[vec_id].desc_idx = idx; - vec_id++; - - if ((vq->desc[idx].flags & VRING_DESC_F_NEXT) == 0) - break; - - idx = vq->desc[idx].next; - } - - *allocated = len; - *vec_idx = vec_id; - - return 0; -} - -/* - * Returns -1 on fail, 0 on success - */ -static inline int -reserve_avail_buf_mergeable(struct vhost_virtqueue *vq, uint32_t size, - uint16_t *end, struct buf_vector *buf_vec) -{ - uint16_t cur_idx; - uint16_t avail_idx; - uint32_t allocated = 0; - uint32_t vec_idx = 0; - uint16_t tries = 0; - - cur_idx = vq->last_used_idx; - - while (1) { - avail_idx = *((volatile uint16_t *)&vq->avail->idx); - if (unlikely(cur_idx == avail_idx)) - return -1; - - if (unlikely(fill_vec_buf(vq, cur_idx, &allocated, - &vec_idx, buf_vec) < 0)) - return -1; - - cur_idx++; - tries++; - - if (allocated >= size) - break; - - /* - * if we tried all available ring items, and still - * can't get enough buf, it means something abnormal - * happened. - */ - if (unlikely(tries >= vq->size)) - return -1; - } - - *end = cur_idx; - return 0; -} - -static inline uint32_t __attribute__((always_inline)) -copy_mbuf_to_desc_mergeable(struct virtio_net *dev, struct vhost_virtqueue *vq, - uint16_t end_idx, struct rte_mbuf *m, - struct buf_vector *buf_vec) -{ - struct virtio_net_hdr_mrg_rxbuf virtio_hdr = {{0, 0, 0, 0, 0, 0}, 0}; - uint32_t vec_idx = 0; - uint16_t start_idx = vq->last_used_idx; - uint16_t cur_idx = start_idx; - uint64_t desc_addr; - uint32_t desc_chain_head; - uint32_t desc_chain_len; - uint32_t mbuf_offset, mbuf_avail; - uint32_t desc_offset, desc_avail; - uint32_t cpy_len; - uint16_t desc_idx, used_idx; - - if (unlikely(m == NULL)) - return 0; - - LOG_DEBUG(VHOST_DATA, "(%d) current index %d | end index %d\n", - dev->vid, cur_idx, end_idx); - - desc_addr = gpa_to_vva(dev, buf_vec[vec_idx].buf_addr); - if (buf_vec[vec_idx].buf_len < dev->vhost_hlen || !desc_addr) - return 0; - - rte_prefetch0((void *)(uintptr_t)desc_addr); - - virtio_hdr.num_buffers = end_idx - start_idx; - LOG_DEBUG(VHOST_DATA, "(%d) RX: num merge buffers %d\n", - dev->vid, virtio_hdr.num_buffers); - - virtio_enqueue_offload(m, &virtio_hdr.hdr); - copy_virtio_net_hdr(dev, desc_addr, virtio_hdr); - vhost_log_write(dev, buf_vec[vec_idx].buf_addr, dev->vhost_hlen); - PRINT_PACKET(dev, (uintptr_t)desc_addr, dev->vhost_hlen, 0); - - desc_avail = buf_vec[vec_idx].buf_len - dev->vhost_hlen; - desc_offset = dev->vhost_hlen; - desc_chain_head = buf_vec[vec_idx].desc_idx; - desc_chain_len = desc_offset; - - mbuf_avail = rte_pktmbuf_data_len(m); - mbuf_offset = 0; - while (mbuf_avail != 0 || m->next != NULL) { - /* done with current desc buf, get the next one */ - if (desc_avail == 0) { - desc_idx = buf_vec[vec_idx].desc_idx; - vec_idx++; - - if (!(vq->desc[desc_idx].flags & VRING_DESC_F_NEXT)) { - /* Update used ring with desc information */ - used_idx = cur_idx++ & (vq->size - 1); - vq->used->ring[used_idx].id = desc_chain_head; - vq->used->ring[used_idx].len = desc_chain_len; - vhost_log_used_vring(dev, vq, - offsetof(struct vring_used, - ring[used_idx]), - sizeof(vq->used->ring[used_idx])); - desc_chain_head = buf_vec[vec_idx].desc_idx; - desc_chain_len = 0; - } - - desc_addr = gpa_to_vva(dev, buf_vec[vec_idx].buf_addr); - if (unlikely(!desc_addr)) - return 0; - - /* Prefetch buffer address. */ - rte_prefetch0((void *)(uintptr_t)desc_addr); - desc_offset = 0; - desc_avail = buf_vec[vec_idx].buf_len; - } - - /* done with current mbuf, get the next one */ - if (mbuf_avail == 0) { - m = m->next; - - mbuf_offset = 0; - mbuf_avail = rte_pktmbuf_data_len(m); - } - - cpy_len = RTE_MIN(desc_avail, mbuf_avail); - rte_memcpy((void *)((uintptr_t)(desc_addr + desc_offset)), - rte_pktmbuf_mtod_offset(m, void *, mbuf_offset), - cpy_len); - vhost_log_write(dev, buf_vec[vec_idx].buf_addr + desc_offset, - cpy_len); - PRINT_PACKET(dev, (uintptr_t)(desc_addr + desc_offset), - cpy_len, 0); - - mbuf_avail -= cpy_len; - mbuf_offset += cpy_len; - desc_avail -= cpy_len; - desc_offset += cpy_len; - desc_chain_len += cpy_len; - } - - used_idx = cur_idx & (vq->size - 1); - vq->used->ring[used_idx].id = desc_chain_head; - vq->used->ring[used_idx].len = desc_chain_len; - vhost_log_used_vring(dev, vq, - offsetof(struct vring_used, ring[used_idx]), - sizeof(vq->used->ring[used_idx])); - - return end_idx - start_idx; -} - -static inline uint32_t __attribute__((always_inline)) -virtio_dev_merge_rx(struct virtio_net *dev, uint16_t queue_id, - struct rte_mbuf **pkts, uint32_t count) -{ - struct vhost_virtqueue *vq; - uint32_t pkt_idx = 0, nr_used = 0; - uint16_t end; - struct buf_vector buf_vec[BUF_VECTOR_MAX]; - - LOG_DEBUG(VHOST_DATA, "(%d) %s\n", dev->vid, __func__); - if (unlikely(!is_valid_virt_queue_idx(queue_id, 0, dev->virt_qp_nb))) { - RTE_LOG(ERR, VHOST_DATA, "(%d) %s: invalid virtqueue idx %d.\n", - dev->vid, __func__, queue_id); - return 0; - } - - vq = dev->virtqueue[queue_id]; - if (unlikely(vq->enabled == 0)) - return 0; - - count = RTE_MIN((uint32_t)MAX_PKT_BURST, count); - if (count == 0) - return 0; - - for (pkt_idx = 0; pkt_idx < count; pkt_idx++) { - uint32_t pkt_len = pkts[pkt_idx]->pkt_len + dev->vhost_hlen; - - if (unlikely(reserve_avail_buf_mergeable(vq, pkt_len, - &end, buf_vec) < 0)) { - LOG_DEBUG(VHOST_DATA, - "(%d) failed to get enough desc from vring\n", - dev->vid); - break; - } - - nr_used = copy_mbuf_to_desc_mergeable(dev, vq, end, - pkts[pkt_idx], buf_vec); - rte_smp_wmb(); - - *(volatile uint16_t *)&vq->used->idx += nr_used; - vhost_log_used_vring(dev, vq, offsetof(struct vring_used, idx), - sizeof(vq->used->idx)); - vq->last_used_idx += nr_used; - } - - if (likely(pkt_idx)) { - /* flush used->idx update before we read avail->flags. */ - rte_mb(); - - /* Kick the guest if necessary. */ - if (!(vq->avail->flags & VRING_AVAIL_F_NO_INTERRUPT) - && (vq->callfd >= 0)) - eventfd_write(vq->callfd, (eventfd_t)1); - } - - return pkt_idx; -} - -uint16_t -rte_vhost_enqueue_burst(int vid, uint16_t queue_id, - struct rte_mbuf **pkts, uint16_t count) -{ - struct virtio_net *dev = get_device(vid); - - if (!dev) - return 0; - - if (dev->features & (1 << VIRTIO_NET_F_MRG_RXBUF)) - return virtio_dev_merge_rx(dev, queue_id, pkts, count); - else - return virtio_dev_rx(dev, queue_id, pkts, count); -} - -static void -parse_ethernet(struct rte_mbuf *m, uint16_t *l4_proto, void **l4_hdr) -{ - struct ipv4_hdr *ipv4_hdr; - struct ipv6_hdr *ipv6_hdr; - void *l3_hdr = NULL; - struct ether_hdr *eth_hdr; - uint16_t ethertype; - - eth_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *); - - m->l2_len = sizeof(struct ether_hdr); - ethertype = rte_be_to_cpu_16(eth_hdr->ether_type); - - if (ethertype == ETHER_TYPE_VLAN) { - struct vlan_hdr *vlan_hdr = (struct vlan_hdr *)(eth_hdr + 1); - - m->l2_len += sizeof(struct vlan_hdr); - ethertype = rte_be_to_cpu_16(vlan_hdr->eth_proto); - } - - l3_hdr = (char *)eth_hdr + m->l2_len; - - switch (ethertype) { - case ETHER_TYPE_IPv4: - ipv4_hdr = (struct ipv4_hdr *)l3_hdr; - *l4_proto = ipv4_hdr->next_proto_id; - m->l3_len = (ipv4_hdr->version_ihl & 0x0f) * 4; - *l4_hdr = (char *)l3_hdr + m->l3_len; - m->ol_flags |= PKT_TX_IPV4; - break; - case ETHER_TYPE_IPv6: - ipv6_hdr = (struct ipv6_hdr *)l3_hdr; - *l4_proto = ipv6_hdr->proto; - m->l3_len = sizeof(struct ipv6_hdr); - *l4_hdr = (char *)l3_hdr + m->l3_len; - m->ol_flags |= PKT_TX_IPV6; - break; - default: - m->l3_len = 0; - *l4_proto = 0; - break; - } -} - -static inline void __attribute__((always_inline)) -vhost_dequeue_offload(struct virtio_net_hdr *hdr, struct rte_mbuf *m) -{ - uint16_t l4_proto = 0; - void *l4_hdr = NULL; - struct tcp_hdr *tcp_hdr = NULL; - - parse_ethernet(m, &l4_proto, &l4_hdr); - if (hdr->flags == VIRTIO_NET_HDR_F_NEEDS_CSUM) { - if (hdr->csum_start == (m->l2_len + m->l3_len)) { - switch (hdr->csum_offset) { - case (offsetof(struct tcp_hdr, cksum)): - if (l4_proto == IPPROTO_TCP) - m->ol_flags |= PKT_TX_TCP_CKSUM; - break; - case (offsetof(struct udp_hdr, dgram_cksum)): - if (l4_proto == IPPROTO_UDP) - m->ol_flags |= PKT_TX_UDP_CKSUM; - break; - case (offsetof(struct sctp_hdr, cksum)): - if (l4_proto == IPPROTO_SCTP) - m->ol_flags |= PKT_TX_SCTP_CKSUM; - break; - default: - break; - } - } - } - - if (hdr->gso_type != VIRTIO_NET_HDR_GSO_NONE) { - switch (hdr->gso_type & ~VIRTIO_NET_HDR_GSO_ECN) { - case VIRTIO_NET_HDR_GSO_TCPV4: - case VIRTIO_NET_HDR_GSO_TCPV6: - tcp_hdr = (struct tcp_hdr *)l4_hdr; - m->ol_flags |= PKT_TX_TCP_SEG; - m->tso_segsz = hdr->gso_size; - m->l4_len = (tcp_hdr->data_off & 0xf0) >> 2; - break; - default: - RTE_LOG(WARNING, VHOST_DATA, - "unsupported gso type %u.\n", hdr->gso_type); - break; - } - } -} - -#define RARP_PKT_SIZE 64 - -static int -make_rarp_packet(struct rte_mbuf *rarp_mbuf, const struct ether_addr *mac) -{ - struct ether_hdr *eth_hdr; - struct arp_hdr *rarp; - - if (rarp_mbuf->buf_len < 64) { - RTE_LOG(WARNING, VHOST_DATA, - "failed to make RARP; mbuf size too small %u (< %d)\n", - rarp_mbuf->buf_len, RARP_PKT_SIZE); - return -1; - } - - /* Ethernet header. */ - eth_hdr = rte_pktmbuf_mtod_offset(rarp_mbuf, struct ether_hdr *, 0); - memset(eth_hdr->d_addr.addr_bytes, 0xff, ETHER_ADDR_LEN); - ether_addr_copy(mac, ð_hdr->s_addr); - eth_hdr->ether_type = htons(ETHER_TYPE_RARP); - - /* RARP header. */ - rarp = (struct arp_hdr *)(eth_hdr + 1); - rarp->arp_hrd = htons(ARP_HRD_ETHER); - rarp->arp_pro = htons(ETHER_TYPE_IPv4); - rarp->arp_hln = ETHER_ADDR_LEN; - rarp->arp_pln = 4; - rarp->arp_op = htons(ARP_OP_REVREQUEST); - - ether_addr_copy(mac, &rarp->arp_data.arp_sha); - ether_addr_copy(mac, &rarp->arp_data.arp_tha); - memset(&rarp->arp_data.arp_sip, 0x00, 4); - memset(&rarp->arp_data.arp_tip, 0x00, 4); - - rarp_mbuf->pkt_len = rarp_mbuf->data_len = RARP_PKT_SIZE; - - return 0; -} - -static inline int __attribute__((always_inline)) -copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq, - struct rte_mbuf *m, uint16_t desc_idx, - struct rte_mempool *mbuf_pool) -{ - struct vring_desc *desc; - uint64_t desc_addr; - uint32_t desc_avail, desc_offset; - uint32_t mbuf_avail, mbuf_offset; - uint32_t cpy_len; - struct rte_mbuf *cur = m, *prev = m; - struct virtio_net_hdr *hdr; - /* A counter to avoid desc dead loop chain */ - uint32_t nr_desc = 1; - - desc = &vq->desc[desc_idx]; - if (unlikely(desc->len < dev->vhost_hlen)) - return -1; - - desc_addr = gpa_to_vva(dev, desc->addr); - if (unlikely(!desc_addr)) - return -1; - - hdr = (struct virtio_net_hdr *)((uintptr_t)desc_addr); - rte_prefetch0(hdr); - - /* - * A virtio driver normally uses at least 2 desc buffers - * for Tx: the first for storing the header, and others - * for storing the data. - */ - if (likely((desc->len == dev->vhost_hlen) && - (desc->flags & VRING_DESC_F_NEXT) != 0)) { - desc = &vq->desc[desc->next]; - - desc_addr = gpa_to_vva(dev, desc->addr); - if (unlikely(!desc_addr)) - return -1; - - rte_prefetch0((void *)(uintptr_t)desc_addr); - - desc_offset = 0; - desc_avail = desc->len; - nr_desc += 1; - - PRINT_PACKET(dev, (uintptr_t)desc_addr, desc->len, 0); - } else { - desc_avail = desc->len - dev->vhost_hlen; - desc_offset = dev->vhost_hlen; - } - - mbuf_offset = 0; - mbuf_avail = m->buf_len - RTE_PKTMBUF_HEADROOM; - while (1) { - cpy_len = RTE_MIN(desc_avail, mbuf_avail); - rte_memcpy(rte_pktmbuf_mtod_offset(cur, void *, mbuf_offset), - (void *)((uintptr_t)(desc_addr + desc_offset)), - cpy_len); - - mbuf_avail -= cpy_len; - mbuf_offset += cpy_len; - desc_avail -= cpy_len; - desc_offset += cpy_len; - - /* This desc reaches to its end, get the next one */ - if (desc_avail == 0) { - if ((desc->flags & VRING_DESC_F_NEXT) == 0) - break; - - if (unlikely(desc->next >= vq->size || - ++nr_desc > vq->size)) - return -1; - desc = &vq->desc[desc->next]; - - desc_addr = gpa_to_vva(dev, desc->addr); - if (unlikely(!desc_addr)) - return -1; - - rte_prefetch0((void *)(uintptr_t)desc_addr); - - desc_offset = 0; - desc_avail = desc->len; - - PRINT_PACKET(dev, (uintptr_t)desc_addr, desc->len, 0); - } - - /* - * This mbuf reaches to its end, get a new one - * to hold more data. - */ - if (mbuf_avail == 0) { - cur = rte_pktmbuf_alloc(mbuf_pool); - if (unlikely(cur == NULL)) { - RTE_LOG(ERR, VHOST_DATA, "Failed to " - "allocate memory for mbuf.\n"); - return -1; - } - - prev->next = cur; - prev->data_len = mbuf_offset; - m->nb_segs += 1; - m->pkt_len += mbuf_offset; - prev = cur; - - mbuf_offset = 0; - mbuf_avail = cur->buf_len - RTE_PKTMBUF_HEADROOM; - } - } - - prev->data_len = mbuf_offset; - m->pkt_len += mbuf_offset; - - if (hdr->flags != 0 || hdr->gso_type != VIRTIO_NET_HDR_GSO_NONE) - vhost_dequeue_offload(hdr, m); - - return 0; -} - -uint16_t -rte_vhost_dequeue_burst(int vid, uint16_t queue_id, - struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count) -{ - struct virtio_net *dev; - struct rte_mbuf *rarp_mbuf = NULL; - struct vhost_virtqueue *vq; - uint32_t desc_indexes[MAX_PKT_BURST]; - uint32_t used_idx; - uint32_t i = 0; - uint16_t free_entries; - uint16_t avail_idx; - - dev = get_device(vid); - if (!dev) - return 0; - - if (unlikely(!is_valid_virt_queue_idx(queue_id, 1, dev->virt_qp_nb))) { - RTE_LOG(ERR, VHOST_DATA, "(%d) %s: invalid virtqueue idx %d.\n", - dev->vid, __func__, queue_id); - return 0; - } - - vq = dev->virtqueue[queue_id]; - if (unlikely(vq->enabled == 0)) - return 0; - - /* - * Construct a RARP broadcast packet, and inject it to the "pkts" - * array, to looks like that guest actually send such packet. - * - * Check user_send_rarp() for more information. - */ - if (unlikely(rte_atomic16_cmpset((volatile uint16_t *) - &dev->broadcast_rarp.cnt, 1, 0))) { - rarp_mbuf = rte_pktmbuf_alloc(mbuf_pool); - if (rarp_mbuf == NULL) { - RTE_LOG(ERR, VHOST_DATA, - "Failed to allocate memory for mbuf.\n"); - return 0; - } - - if (make_rarp_packet(rarp_mbuf, &dev->mac)) { - rte_pktmbuf_free(rarp_mbuf); - rarp_mbuf = NULL; - } else { - count -= 1; - } - } - - avail_idx = *((volatile uint16_t *)&vq->avail->idx); - free_entries = avail_idx - vq->last_used_idx; - if (free_entries == 0) - goto out; - - LOG_DEBUG(VHOST_DATA, "(%d) %s\n", dev->vid, __func__); - - /* Prefetch available ring to retrieve head indexes. */ - used_idx = vq->last_used_idx & (vq->size - 1); - rte_prefetch0(&vq->avail->ring[used_idx]); - rte_prefetch0(&vq->used->ring[used_idx]); - - count = RTE_MIN(count, MAX_PKT_BURST); - count = RTE_MIN(count, free_entries); - LOG_DEBUG(VHOST_DATA, "(%d) about to dequeue %u buffers\n", - dev->vid, count); - - /* Retrieve all of the head indexes first to avoid caching issues. */ - for (i = 0; i < count; i++) { - used_idx = (vq->last_used_idx + i) & (vq->size - 1); - desc_indexes[i] = vq->avail->ring[used_idx]; - - vq->used->ring[used_idx].id = desc_indexes[i]; - vq->used->ring[used_idx].len = 0; - vhost_log_used_vring(dev, vq, - offsetof(struct vring_used, ring[used_idx]), - sizeof(vq->used->ring[used_idx])); - } - - /* Prefetch descriptor index. */ - rte_prefetch0(&vq->desc[desc_indexes[0]]); - for (i = 0; i < count; i++) { - int err; - - if (likely(i + 1 < count)) - rte_prefetch0(&vq->desc[desc_indexes[i + 1]]); - - pkts[i] = rte_pktmbuf_alloc(mbuf_pool); - if (unlikely(pkts[i] == NULL)) { - RTE_LOG(ERR, VHOST_DATA, - "Failed to allocate memory for mbuf.\n"); - break; - } - err = copy_desc_to_mbuf(dev, vq, pkts[i], desc_indexes[i], - mbuf_pool); - if (unlikely(err)) { - rte_pktmbuf_free(pkts[i]); - break; - } - } - - rte_smp_wmb(); - rte_smp_rmb(); - vq->used->idx += i; - vq->last_used_idx += i; - vhost_log_used_vring(dev, vq, offsetof(struct vring_used, idx), - sizeof(vq->used->idx)); - - /* Kick guest if required. */ - if (!(vq->avail->flags & VRING_AVAIL_F_NO_INTERRUPT) - && (vq->callfd >= 0)) - eventfd_write(vq->callfd, (eventfd_t)1); - -out: - if (unlikely(rarp_mbuf != NULL)) { - /* - * Inject it to the head of "pkts" array, so that switch's mac - * learning table will get updated first. - */ - memmove(&pkts[1], pkts, i * sizeof(struct rte_mbuf *)); - pkts[0] = rarp_mbuf; - i += 1; - } - - return i; -} diff --git a/dpdk/lib/librte_vhost/vhost_user.c b/dpdk/lib/librte_vhost/vhost_user.c new file mode 100644 index 00000000..50e654db --- /dev/null +++ b/dpdk/lib/librte_vhost/vhost_user.c @@ -0,0 +1,1510 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef RTE_LIBRTE_VHOST_NUMA +#include +#endif + +#include +#include +#include + +#include "iotlb.h" +#include "vhost.h" +#include "vhost_user.h" + +#define VIRTIO_MIN_MTU 68 +#define VIRTIO_MAX_MTU 65535 + +static const char *vhost_message_str[VHOST_USER_MAX] = { + [VHOST_USER_NONE] = "VHOST_USER_NONE", + [VHOST_USER_GET_FEATURES] = "VHOST_USER_GET_FEATURES", + [VHOST_USER_SET_FEATURES] = "VHOST_USER_SET_FEATURES", + [VHOST_USER_SET_OWNER] = "VHOST_USER_SET_OWNER", + [VHOST_USER_RESET_OWNER] = "VHOST_USER_RESET_OWNER", + [VHOST_USER_SET_MEM_TABLE] = "VHOST_USER_SET_MEM_TABLE", + [VHOST_USER_SET_LOG_BASE] = "VHOST_USER_SET_LOG_BASE", + [VHOST_USER_SET_LOG_FD] = "VHOST_USER_SET_LOG_FD", + [VHOST_USER_SET_VRING_NUM] = "VHOST_USER_SET_VRING_NUM", + [VHOST_USER_SET_VRING_ADDR] = "VHOST_USER_SET_VRING_ADDR", + [VHOST_USER_SET_VRING_BASE] = "VHOST_USER_SET_VRING_BASE", + [VHOST_USER_GET_VRING_BASE] = "VHOST_USER_GET_VRING_BASE", + [VHOST_USER_SET_VRING_KICK] = "VHOST_USER_SET_VRING_KICK", + [VHOST_USER_SET_VRING_CALL] = "VHOST_USER_SET_VRING_CALL", + [VHOST_USER_SET_VRING_ERR] = "VHOST_USER_SET_VRING_ERR", + [VHOST_USER_GET_PROTOCOL_FEATURES] = "VHOST_USER_GET_PROTOCOL_FEATURES", + [VHOST_USER_SET_PROTOCOL_FEATURES] = "VHOST_USER_SET_PROTOCOL_FEATURES", + [VHOST_USER_GET_QUEUE_NUM] = "VHOST_USER_GET_QUEUE_NUM", + [VHOST_USER_SET_VRING_ENABLE] = "VHOST_USER_SET_VRING_ENABLE", + [VHOST_USER_SEND_RARP] = "VHOST_USER_SEND_RARP", + [VHOST_USER_NET_SET_MTU] = "VHOST_USER_NET_SET_MTU", + [VHOST_USER_SET_SLAVE_REQ_FD] = "VHOST_USER_SET_SLAVE_REQ_FD", + [VHOST_USER_IOTLB_MSG] = "VHOST_USER_IOTLB_MSG", +}; + +static uint64_t +get_blk_size(int fd) +{ + struct stat stat; + int ret; + + ret = fstat(fd, &stat); + return ret == -1 ? (uint64_t)-1 : (uint64_t)stat.st_blksize; +} + +static void +free_mem_region(struct virtio_net *dev) +{ + uint32_t i; + struct rte_vhost_mem_region *reg; + + if (!dev || !dev->mem) + return; + + for (i = 0; i < dev->mem->nregions; i++) { + reg = &dev->mem->regions[i]; + if (reg->host_user_addr) { + munmap(reg->mmap_addr, reg->mmap_size); + close(reg->fd); + } + } +} + +void +vhost_backend_cleanup(struct virtio_net *dev) +{ + if (dev->mem) { + free_mem_region(dev); + rte_free(dev->mem); + dev->mem = NULL; + } + + free(dev->guest_pages); + dev->guest_pages = NULL; + + if (dev->log_addr) { + munmap((void *)(uintptr_t)dev->log_addr, dev->log_size); + dev->log_addr = 0; + } + + if (dev->slave_req_fd >= 0) { + close(dev->slave_req_fd); + dev->slave_req_fd = -1; + } +} + +/* + * This function just returns success at the moment unless + * the device hasn't been initialised. + */ +static int +vhost_user_set_owner(void) +{ + return 0; +} + +static int +vhost_user_reset_owner(struct virtio_net *dev) +{ + if (dev->flags & VIRTIO_DEV_RUNNING) { + dev->flags &= ~VIRTIO_DEV_RUNNING; + dev->notify_ops->destroy_device(dev->vid); + } + + cleanup_device(dev, 0); + reset_device(dev); + return 0; +} + +/* + * The features that we support are requested. + */ +static uint64_t +vhost_user_get_features(struct virtio_net *dev) +{ + uint64_t features = 0; + + rte_vhost_driver_get_features(dev->ifname, &features); + return features; +} + +/* + * We receive the negotiated features supported by us and the virtio device. + */ +static int +vhost_user_set_features(struct virtio_net *dev, uint64_t features) +{ + uint64_t vhost_features = 0; + + rte_vhost_driver_get_features(dev->ifname, &vhost_features); + if (features & ~vhost_features) { + RTE_LOG(ERR, VHOST_CONFIG, + "(%d) received invalid negotiated features.\n", + dev->vid); + return -1; + } + + if ((dev->flags & VIRTIO_DEV_RUNNING) && dev->features != features) { + if (dev->notify_ops->features_changed) + dev->notify_ops->features_changed(dev->vid, features); + } + + dev->features = features; + if (dev->features & + ((1 << VIRTIO_NET_F_MRG_RXBUF) | (1ULL << VIRTIO_F_VERSION_1))) { + dev->vhost_hlen = sizeof(struct virtio_net_hdr_mrg_rxbuf); + } else { + dev->vhost_hlen = sizeof(struct virtio_net_hdr); + } + LOG_DEBUG(VHOST_CONFIG, + "(%d) mergeable RX buffers %s, virtio 1 %s\n", + dev->vid, + (dev->features & (1 << VIRTIO_NET_F_MRG_RXBUF)) ? "on" : "off", + (dev->features & (1ULL << VIRTIO_F_VERSION_1)) ? "on" : "off"); + + return 0; +} + +/* + * The virtio device sends us the size of the descriptor ring. + */ +static int +vhost_user_set_vring_num(struct virtio_net *dev, + VhostUserMsg *msg) +{ + struct vhost_virtqueue *vq = dev->virtqueue[msg->payload.state.index]; + + vq->size = msg->payload.state.num; + + if (dev->dequeue_zero_copy) { + vq->nr_zmbuf = 0; + vq->last_zmbuf_idx = 0; + vq->zmbuf_size = vq->size; + vq->zmbufs = rte_zmalloc(NULL, vq->zmbuf_size * + sizeof(struct zcopy_mbuf), 0); + if (vq->zmbufs == NULL) { + RTE_LOG(WARNING, VHOST_CONFIG, + "failed to allocate mem for zero copy; " + "zero copy is force disabled\n"); + dev->dequeue_zero_copy = 0; + } + } + + vq->shadow_used_ring = rte_malloc(NULL, + vq->size * sizeof(struct vring_used_elem), + RTE_CACHE_LINE_SIZE); + if (!vq->shadow_used_ring) { + RTE_LOG(ERR, VHOST_CONFIG, + "failed to allocate memory for shadow used ring.\n"); + return -1; + } + + vq->batch_copy_elems = rte_malloc(NULL, + vq->size * sizeof(struct batch_copy_elem), + RTE_CACHE_LINE_SIZE); + if (!vq->batch_copy_elems) { + RTE_LOG(ERR, VHOST_CONFIG, + "failed to allocate memory for batching copy.\n"); + return -1; + } + + return 0; +} + +/* + * Reallocate virtio_dev and vhost_virtqueue data structure to make them on the + * same numa node as the memory of vring descriptor. + */ +#ifdef RTE_LIBRTE_VHOST_NUMA +static struct virtio_net* +numa_realloc(struct virtio_net *dev, int index) +{ + int oldnode, newnode; + struct virtio_net *old_dev; + struct vhost_virtqueue *old_vq, *vq; + int ret; + + old_dev = dev; + vq = old_vq = dev->virtqueue[index]; + + ret = get_mempolicy(&newnode, NULL, 0, old_vq->desc, + MPOL_F_NODE | MPOL_F_ADDR); + + /* check if we need to reallocate vq */ + ret |= get_mempolicy(&oldnode, NULL, 0, old_vq, + MPOL_F_NODE | MPOL_F_ADDR); + if (ret) { + RTE_LOG(ERR, VHOST_CONFIG, + "Unable to get vq numa information.\n"); + return dev; + } + if (oldnode != newnode) { + RTE_LOG(INFO, VHOST_CONFIG, + "reallocate vq from %d to %d node\n", oldnode, newnode); + vq = rte_malloc_socket(NULL, sizeof(*vq), 0, newnode); + if (!vq) + return dev; + + memcpy(vq, old_vq, sizeof(*vq)); + rte_free(old_vq); + } + + /* check if we need to reallocate dev */ + ret = get_mempolicy(&oldnode, NULL, 0, old_dev, + MPOL_F_NODE | MPOL_F_ADDR); + if (ret) { + RTE_LOG(ERR, VHOST_CONFIG, + "Unable to get dev numa information.\n"); + goto out; + } + if (oldnode != newnode) { + RTE_LOG(INFO, VHOST_CONFIG, + "reallocate dev from %d to %d node\n", + oldnode, newnode); + dev = rte_malloc_socket(NULL, sizeof(*dev), 0, newnode); + if (!dev) { + dev = old_dev; + goto out; + } + + memcpy(dev, old_dev, sizeof(*dev)); + rte_free(old_dev); + } + +out: + dev->virtqueue[index] = vq; + vhost_devices[dev->vid] = dev; + + if (old_vq != vq) + vhost_user_iotlb_init(dev, index); + + return dev; +} +#else +static struct virtio_net* +numa_realloc(struct virtio_net *dev, int index __rte_unused) +{ + return dev; +} +#endif + +/* Converts QEMU virtual address to Vhost virtual address. */ +static uint64_t +qva_to_vva(struct virtio_net *dev, uint64_t qva, uint64_t *len) +{ + struct rte_vhost_mem_region *r; + uint32_t i; + + /* Find the region where the address lives. */ + for (i = 0; i < dev->mem->nregions; i++) { + r = &dev->mem->regions[i]; + + if (qva >= r->guest_user_addr && + qva < r->guest_user_addr + r->size) { + + if (unlikely(*len > r->guest_user_addr + r->size - qva)) + *len = r->guest_user_addr + r->size - qva; + + return qva - r->guest_user_addr + + r->host_user_addr; + } + } + *len = 0; + + return 0; +} + + +/* + * Converts ring address to Vhost virtual address. + * If IOMMU is enabled, the ring address is a guest IO virtual address, + * else it is a QEMU virtual address. + */ +static uint64_t +ring_addr_to_vva(struct virtio_net *dev, struct vhost_virtqueue *vq, + uint64_t ra, uint64_t *size) +{ + if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)) { + uint64_t vva; + + vva = vhost_user_iotlb_cache_find(vq, ra, + size, VHOST_ACCESS_RW); + if (!vva) + vhost_user_iotlb_miss(dev, ra, VHOST_ACCESS_RW); + + return vva; + } + + return qva_to_vva(dev, ra, size); +} + +static struct virtio_net * +translate_ring_addresses(struct virtio_net *dev, int vq_index) +{ + struct vhost_virtqueue *vq = dev->virtqueue[vq_index]; + struct vhost_vring_addr *addr = &vq->ring_addrs; + uint64_t len; + + /* The addresses are converted from QEMU virtual to Vhost virtual. */ + if (vq->desc && vq->avail && vq->used) + return dev; + + len = sizeof(struct vring_desc) * vq->size; + vq->desc = (struct vring_desc *)(uintptr_t)ring_addr_to_vva(dev, + vq, addr->desc_user_addr, &len); + if (vq->desc == 0 || len != sizeof(struct vring_desc) * vq->size) { + RTE_LOG(DEBUG, VHOST_CONFIG, + "(%d) failed to map desc ring.\n", + dev->vid); + return dev; + } + + dev = numa_realloc(dev, vq_index); + vq = dev->virtqueue[vq_index]; + addr = &vq->ring_addrs; + + len = sizeof(struct vring_avail) + sizeof(uint16_t) * vq->size; + vq->avail = (struct vring_avail *)(uintptr_t)ring_addr_to_vva(dev, + vq, addr->avail_user_addr, &len); + if (vq->avail == 0 || + len != sizeof(struct vring_avail) + + sizeof(uint16_t) * vq->size) { + RTE_LOG(DEBUG, VHOST_CONFIG, + "(%d) failed to map avail ring.\n", + dev->vid); + return dev; + } + + len = sizeof(struct vring_used) + + sizeof(struct vring_used_elem) * vq->size; + vq->used = (struct vring_used *)(uintptr_t)ring_addr_to_vva(dev, + vq, addr->used_user_addr, &len); + if (vq->used == 0 || len != sizeof(struct vring_used) + + sizeof(struct vring_used_elem) * vq->size) { + RTE_LOG(DEBUG, VHOST_CONFIG, + "(%d) failed to map used ring.\n", + dev->vid); + return dev; + } + + if (vq->last_used_idx != vq->used->idx) { + RTE_LOG(WARNING, VHOST_CONFIG, + "last_used_idx (%u) and vq->used->idx (%u) mismatches; " + "some packets maybe resent for Tx and dropped for Rx\n", + vq->last_used_idx, vq->used->idx); + vq->last_used_idx = vq->used->idx; + vq->last_avail_idx = vq->used->idx; + } + + vq->log_guest_addr = addr->log_guest_addr; + + LOG_DEBUG(VHOST_CONFIG, "(%d) mapped address desc: %p\n", + dev->vid, vq->desc); + LOG_DEBUG(VHOST_CONFIG, "(%d) mapped address avail: %p\n", + dev->vid, vq->avail); + LOG_DEBUG(VHOST_CONFIG, "(%d) mapped address used: %p\n", + dev->vid, vq->used); + LOG_DEBUG(VHOST_CONFIG, "(%d) log_guest_addr: %" PRIx64 "\n", + dev->vid, vq->log_guest_addr); + + return dev; +} + +/* + * The virtio device sends us the desc, used and avail ring addresses. + * This function then converts these to our address space. + */ +static int +vhost_user_set_vring_addr(struct virtio_net **pdev, VhostUserMsg *msg) +{ + struct vhost_virtqueue *vq; + struct vhost_vring_addr *addr = &msg->payload.addr; + struct virtio_net *dev = *pdev; + + if (dev->mem == NULL) + return -1; + + /* addr->index refers to the queue index. The txq 1, rxq is 0. */ + vq = dev->virtqueue[msg->payload.addr.index]; + + /* + * Rings addresses should not be interpreted as long as the ring is not + * started and enabled + */ + memcpy(&vq->ring_addrs, addr, sizeof(*addr)); + + vring_invalidate(dev, vq); + + if (vq->enabled && (dev->features & + (1ULL << VHOST_USER_F_PROTOCOL_FEATURES))) { + dev = translate_ring_addresses(dev, msg->payload.state.index); + if (!dev) + return -1; + + *pdev = dev; + } + + return 0; +} + +/* + * The virtio device sends us the available ring last used index. + */ +static int +vhost_user_set_vring_base(struct virtio_net *dev, + VhostUserMsg *msg) +{ + dev->virtqueue[msg->payload.state.index]->last_used_idx = + msg->payload.state.num; + dev->virtqueue[msg->payload.state.index]->last_avail_idx = + msg->payload.state.num; + + return 0; +} + +static void +add_one_guest_page(struct virtio_net *dev, uint64_t guest_phys_addr, + uint64_t host_phys_addr, uint64_t size) +{ + struct guest_page *page, *last_page; + + if (dev->nr_guest_pages == dev->max_guest_pages) { + dev->max_guest_pages *= 2; + dev->guest_pages = realloc(dev->guest_pages, + dev->max_guest_pages * sizeof(*page)); + } + + if (dev->nr_guest_pages > 0) { + last_page = &dev->guest_pages[dev->nr_guest_pages - 1]; + /* merge if the two pages are continuous */ + if (host_phys_addr == last_page->host_phys_addr + + last_page->size) { + last_page->size += size; + return; + } + } + + page = &dev->guest_pages[dev->nr_guest_pages++]; + page->guest_phys_addr = guest_phys_addr; + page->host_phys_addr = host_phys_addr; + page->size = size; +} + +static void +add_guest_pages(struct virtio_net *dev, struct rte_vhost_mem_region *reg, + uint64_t page_size) +{ + uint64_t reg_size = reg->size; + uint64_t host_user_addr = reg->host_user_addr; + uint64_t guest_phys_addr = reg->guest_phys_addr; + uint64_t host_phys_addr; + uint64_t size; + + host_phys_addr = rte_mem_virt2iova((void *)(uintptr_t)host_user_addr); + size = page_size - (guest_phys_addr & (page_size - 1)); + size = RTE_MIN(size, reg_size); + + add_one_guest_page(dev, guest_phys_addr, host_phys_addr, size); + host_user_addr += size; + guest_phys_addr += size; + reg_size -= size; + + while (reg_size > 0) { + size = RTE_MIN(reg_size, page_size); + host_phys_addr = rte_mem_virt2iova((void *)(uintptr_t) + host_user_addr); + add_one_guest_page(dev, guest_phys_addr, host_phys_addr, size); + + host_user_addr += size; + guest_phys_addr += size; + reg_size -= size; + } +} + +#ifdef RTE_LIBRTE_VHOST_DEBUG +/* TODO: enable it only in debug mode? */ +static void +dump_guest_pages(struct virtio_net *dev) +{ + uint32_t i; + struct guest_page *page; + + for (i = 0; i < dev->nr_guest_pages; i++) { + page = &dev->guest_pages[i]; + + RTE_LOG(INFO, VHOST_CONFIG, + "guest physical page region %u\n" + "\t guest_phys_addr: %" PRIx64 "\n" + "\t host_phys_addr : %" PRIx64 "\n" + "\t size : %" PRIx64 "\n", + i, + page->guest_phys_addr, + page->host_phys_addr, + page->size); + } +} +#else +#define dump_guest_pages(dev) +#endif + +static bool +vhost_memory_changed(struct VhostUserMemory *new, + struct rte_vhost_memory *old) +{ + uint32_t i; + + if (new->nregions != old->nregions) + return true; + + for (i = 0; i < new->nregions; ++i) { + VhostUserMemoryRegion *new_r = &new->regions[i]; + struct rte_vhost_mem_region *old_r = &old->regions[i]; + + if (new_r->guest_phys_addr != old_r->guest_phys_addr) + return true; + if (new_r->memory_size != old_r->size) + return true; + if (new_r->userspace_addr != old_r->guest_user_addr) + return true; + } + + return false; +} + +static int +vhost_user_set_mem_table(struct virtio_net *dev, struct VhostUserMsg *pmsg) +{ + struct VhostUserMemory memory = pmsg->payload.memory; + struct rte_vhost_mem_region *reg; + void *mmap_addr; + uint64_t mmap_size; + uint64_t mmap_offset; + uint64_t alignment; + uint32_t i; + int fd; + + if (dev->mem && !vhost_memory_changed(&memory, dev->mem)) { + RTE_LOG(INFO, VHOST_CONFIG, + "(%d) memory regions not changed\n", dev->vid); + + for (i = 0; i < memory.nregions; i++) + close(pmsg->fds[i]); + + return 0; + } + + if (dev->mem) { + free_mem_region(dev); + rte_free(dev->mem); + dev->mem = NULL; + } + + dev->nr_guest_pages = 0; + if (!dev->guest_pages) { + dev->max_guest_pages = 8; + dev->guest_pages = malloc(dev->max_guest_pages * + sizeof(struct guest_page)); + if (dev->guest_pages == NULL) { + RTE_LOG(ERR, VHOST_CONFIG, + "(%d) failed to allocate memory " + "for dev->guest_pages\n", + dev->vid); + return -1; + } + } + + dev->mem = rte_zmalloc("vhost-mem-table", sizeof(struct rte_vhost_memory) + + sizeof(struct rte_vhost_mem_region) * memory.nregions, 0); + if (dev->mem == NULL) { + RTE_LOG(ERR, VHOST_CONFIG, + "(%d) failed to allocate memory for dev->mem\n", + dev->vid); + return -1; + } + dev->mem->nregions = memory.nregions; + + for (i = 0; i < memory.nregions; i++) { + fd = pmsg->fds[i]; + reg = &dev->mem->regions[i]; + + reg->guest_phys_addr = memory.regions[i].guest_phys_addr; + reg->guest_user_addr = memory.regions[i].userspace_addr; + reg->size = memory.regions[i].memory_size; + reg->fd = fd; + + mmap_offset = memory.regions[i].mmap_offset; + mmap_size = reg->size + mmap_offset; + + /* mmap() without flag of MAP_ANONYMOUS, should be called + * with length argument aligned with hugepagesz at older + * longterm version Linux, like 2.6.32 and 3.2.72, or + * mmap() will fail with EINVAL. + * + * to avoid failure, make sure in caller to keep length + * aligned. + */ + alignment = get_blk_size(fd); + if (alignment == (uint64_t)-1) { + RTE_LOG(ERR, VHOST_CONFIG, + "couldn't get hugepage size through fstat\n"); + goto err_mmap; + } + mmap_size = RTE_ALIGN_CEIL(mmap_size, alignment); + + mmap_addr = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_POPULATE, fd, 0); + + if (mmap_addr == MAP_FAILED) { + RTE_LOG(ERR, VHOST_CONFIG, + "mmap region %u failed.\n", i); + goto err_mmap; + } + + reg->mmap_addr = mmap_addr; + reg->mmap_size = mmap_size; + reg->host_user_addr = (uint64_t)(uintptr_t)mmap_addr + + mmap_offset; + + if (dev->dequeue_zero_copy) + add_guest_pages(dev, reg, alignment); + + RTE_LOG(INFO, VHOST_CONFIG, + "guest memory region %u, size: 0x%" PRIx64 "\n" + "\t guest physical addr: 0x%" PRIx64 "\n" + "\t guest virtual addr: 0x%" PRIx64 "\n" + "\t host virtual addr: 0x%" PRIx64 "\n" + "\t mmap addr : 0x%" PRIx64 "\n" + "\t mmap size : 0x%" PRIx64 "\n" + "\t mmap align: 0x%" PRIx64 "\n" + "\t mmap off : 0x%" PRIx64 "\n", + i, reg->size, + reg->guest_phys_addr, + reg->guest_user_addr, + reg->host_user_addr, + (uint64_t)(uintptr_t)mmap_addr, + mmap_size, + alignment, + mmap_offset); + } + + dump_guest_pages(dev); + + return 0; + +err_mmap: + free_mem_region(dev); + rte_free(dev->mem); + dev->mem = NULL; + return -1; +} + +static int +vq_is_ready(struct vhost_virtqueue *vq) +{ + return vq && vq->desc && vq->avail && vq->used && + vq->kickfd != VIRTIO_UNINITIALIZED_EVENTFD && + vq->callfd != VIRTIO_UNINITIALIZED_EVENTFD; +} + +static int +virtio_is_ready(struct virtio_net *dev) +{ + struct vhost_virtqueue *vq; + uint32_t i; + + if (dev->nr_vring == 0) + return 0; + + for (i = 0; i < dev->nr_vring; i++) { + vq = dev->virtqueue[i]; + + if (!vq_is_ready(vq)) + return 0; + } + + RTE_LOG(INFO, VHOST_CONFIG, + "virtio is now ready for processing.\n"); + return 1; +} + +static void +vhost_user_set_vring_call(struct virtio_net *dev, struct VhostUserMsg *pmsg) +{ + struct vhost_vring_file file; + struct vhost_virtqueue *vq; + + file.index = pmsg->payload.u64 & VHOST_USER_VRING_IDX_MASK; + if (pmsg->payload.u64 & VHOST_USER_VRING_NOFD_MASK) + file.fd = VIRTIO_INVALID_EVENTFD; + else + file.fd = pmsg->fds[0]; + RTE_LOG(INFO, VHOST_CONFIG, + "vring call idx:%d file:%d\n", file.index, file.fd); + + vq = dev->virtqueue[file.index]; + if (vq->callfd >= 0) + close(vq->callfd); + + vq->callfd = file.fd; +} + +static void +vhost_user_set_vring_kick(struct virtio_net **pdev, struct VhostUserMsg *pmsg) +{ + struct vhost_vring_file file; + struct vhost_virtqueue *vq; + struct virtio_net *dev = *pdev; + + file.index = pmsg->payload.u64 & VHOST_USER_VRING_IDX_MASK; + if (pmsg->payload.u64 & VHOST_USER_VRING_NOFD_MASK) + file.fd = VIRTIO_INVALID_EVENTFD; + else + file.fd = pmsg->fds[0]; + RTE_LOG(INFO, VHOST_CONFIG, + "vring kick idx:%d file:%d\n", file.index, file.fd); + + /* Interpret ring addresses only when ring is started. */ + dev = translate_ring_addresses(dev, file.index); + if (!dev) + return; + + *pdev = dev; + + vq = dev->virtqueue[file.index]; + + /* + * When VHOST_USER_F_PROTOCOL_FEATURES is not negotiated, + * the ring starts already enabled. Otherwise, it is enabled via + * the SET_VRING_ENABLE message. + */ + if (!(dev->features & (1ULL << VHOST_USER_F_PROTOCOL_FEATURES))) + vq->enabled = 1; + + if (vq->kickfd >= 0) + close(vq->kickfd); + vq->kickfd = file.fd; +} + +static void +free_zmbufs(struct vhost_virtqueue *vq) +{ + struct zcopy_mbuf *zmbuf, *next; + + for (zmbuf = TAILQ_FIRST(&vq->zmbuf_list); + zmbuf != NULL; zmbuf = next) { + next = TAILQ_NEXT(zmbuf, next); + + rte_pktmbuf_free(zmbuf->mbuf); + TAILQ_REMOVE(&vq->zmbuf_list, zmbuf, next); + } + + rte_free(vq->zmbufs); +} + +/* + * when virtio is stopped, qemu will send us the GET_VRING_BASE message. + */ +static int +vhost_user_get_vring_base(struct virtio_net *dev, + VhostUserMsg *msg) +{ + struct vhost_virtqueue *vq = dev->virtqueue[msg->payload.state.index]; + + /* We have to stop the queue (virtio) if it is running. */ + if (dev->flags & VIRTIO_DEV_RUNNING) { + dev->flags &= ~VIRTIO_DEV_RUNNING; + dev->notify_ops->destroy_device(dev->vid); + } + + dev->flags &= ~VIRTIO_DEV_READY; + + /* Here we are safe to get the last used index */ + msg->payload.state.num = vq->last_used_idx; + + RTE_LOG(INFO, VHOST_CONFIG, + "vring base idx:%d file:%d\n", msg->payload.state.index, + msg->payload.state.num); + /* + * Based on current qemu vhost-user implementation, this message is + * sent and only sent in vhost_vring_stop. + * TODO: cleanup the vring, it isn't usable since here. + */ + if (vq->kickfd >= 0) + close(vq->kickfd); + + vq->kickfd = VIRTIO_UNINITIALIZED_EVENTFD; + + if (dev->dequeue_zero_copy) + free_zmbufs(vq); + rte_free(vq->shadow_used_ring); + vq->shadow_used_ring = NULL; + + rte_free(vq->batch_copy_elems); + vq->batch_copy_elems = NULL; + + return 0; +} + +/* + * when virtio queues are ready to work, qemu will send us to + * enable the virtio queue pair. + */ +static int +vhost_user_set_vring_enable(struct virtio_net *dev, + VhostUserMsg *msg) +{ + int enable = (int)msg->payload.state.num; + + RTE_LOG(INFO, VHOST_CONFIG, + "set queue enable: %d to qp idx: %d\n", + enable, msg->payload.state.index); + + if (dev->notify_ops->vring_state_changed) + dev->notify_ops->vring_state_changed(dev->vid, + msg->payload.state.index, enable); + + dev->virtqueue[msg->payload.state.index]->enabled = enable; + + return 0; +} + +static void +vhost_user_get_protocol_features(struct virtio_net *dev, + struct VhostUserMsg *msg) +{ + uint64_t features, protocol_features = VHOST_USER_PROTOCOL_FEATURES; + + rte_vhost_driver_get_features(dev->ifname, &features); + + /* + * REPLY_ACK protocol feature is only mandatory for now + * for IOMMU feature. If IOMMU is explicitly disabled by the + * application, disable also REPLY_ACK feature for older buggy + * Qemu versions (from v2.7.0 to v2.9.0). + */ + if (!(features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))) + protocol_features &= ~(1ULL << VHOST_USER_PROTOCOL_F_REPLY_ACK); + + msg->payload.u64 = protocol_features; + msg->size = sizeof(msg->payload.u64); +} + +static void +vhost_user_set_protocol_features(struct virtio_net *dev, + uint64_t protocol_features) +{ + if (protocol_features & ~VHOST_USER_PROTOCOL_FEATURES) + return; + + dev->protocol_features = protocol_features; +} + +static int +vhost_user_set_log_base(struct virtio_net *dev, struct VhostUserMsg *msg) +{ + int fd = msg->fds[0]; + uint64_t size, off; + void *addr; + + if (fd < 0) { + RTE_LOG(ERR, VHOST_CONFIG, "invalid log fd: %d\n", fd); + return -1; + } + + if (msg->size != sizeof(VhostUserLog)) { + RTE_LOG(ERR, VHOST_CONFIG, + "invalid log base msg size: %"PRId32" != %d\n", + msg->size, (int)sizeof(VhostUserLog)); + return -1; + } + + size = msg->payload.log.mmap_size; + off = msg->payload.log.mmap_offset; + RTE_LOG(INFO, VHOST_CONFIG, + "log mmap size: %"PRId64", offset: %"PRId64"\n", + size, off); + + /* + * mmap from 0 to workaround a hugepage mmap bug: mmap will + * fail when offset is not page size aligned. + */ + addr = mmap(0, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + close(fd); + if (addr == MAP_FAILED) { + RTE_LOG(ERR, VHOST_CONFIG, "mmap log base failed!\n"); + return -1; + } + + /* + * Free previously mapped log memory on occasionally + * multiple VHOST_USER_SET_LOG_BASE. + */ + if (dev->log_addr) { + munmap((void *)(uintptr_t)dev->log_addr, dev->log_size); + } + dev->log_addr = (uint64_t)(uintptr_t)addr; + dev->log_base = dev->log_addr + off; + dev->log_size = size; + + return 0; +} + +/* + * An rarp packet is constructed and broadcasted to notify switches about + * the new location of the migrated VM, so that packets from outside will + * not be lost after migration. + * + * However, we don't actually "send" a rarp packet here, instead, we set + * a flag 'broadcast_rarp' to let rte_vhost_dequeue_burst() inject it. + */ +static int +vhost_user_send_rarp(struct virtio_net *dev, struct VhostUserMsg *msg) +{ + uint8_t *mac = (uint8_t *)&msg->payload.u64; + + RTE_LOG(DEBUG, VHOST_CONFIG, + ":: mac: %02x:%02x:%02x:%02x:%02x:%02x\n", + mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]); + memcpy(dev->mac.addr_bytes, mac, 6); + + /* + * Set the flag to inject a RARP broadcast packet at + * rte_vhost_dequeue_burst(). + * + * rte_smp_wmb() is for making sure the mac is copied + * before the flag is set. + */ + rte_smp_wmb(); + rte_atomic16_set(&dev->broadcast_rarp, 1); + + return 0; +} + +static int +vhost_user_net_set_mtu(struct virtio_net *dev, struct VhostUserMsg *msg) +{ + if (msg->payload.u64 < VIRTIO_MIN_MTU || + msg->payload.u64 > VIRTIO_MAX_MTU) { + RTE_LOG(ERR, VHOST_CONFIG, "Invalid MTU size (%"PRIu64")\n", + msg->payload.u64); + + return -1; + } + + dev->mtu = msg->payload.u64; + + return 0; +} + +static int +vhost_user_set_req_fd(struct virtio_net *dev, struct VhostUserMsg *msg) +{ + int fd = msg->fds[0]; + + if (fd < 0) { + RTE_LOG(ERR, VHOST_CONFIG, + "Invalid file descriptor for slave channel (%d)\n", + fd); + return -1; + } + + dev->slave_req_fd = fd; + + return 0; +} + +static int +is_vring_iotlb_update(struct vhost_virtqueue *vq, struct vhost_iotlb_msg *imsg) +{ + struct vhost_vring_addr *ra; + uint64_t start, end; + + start = imsg->iova; + end = start + imsg->size; + + ra = &vq->ring_addrs; + if (ra->desc_user_addr >= start && ra->desc_user_addr < end) + return 1; + if (ra->avail_user_addr >= start && ra->avail_user_addr < end) + return 1; + if (ra->used_user_addr >= start && ra->used_user_addr < end) + return 1; + + return 0; +} + +static int +is_vring_iotlb_invalidate(struct vhost_virtqueue *vq, + struct vhost_iotlb_msg *imsg) +{ + uint64_t istart, iend, vstart, vend; + + istart = imsg->iova; + iend = istart + imsg->size - 1; + + vstart = (uintptr_t)vq->desc; + vend = vstart + sizeof(struct vring_desc) * vq->size - 1; + if (vstart <= iend && istart <= vend) + return 1; + + vstart = (uintptr_t)vq->avail; + vend = vstart + sizeof(struct vring_avail); + vend += sizeof(uint16_t) * vq->size - 1; + if (vstart <= iend && istart <= vend) + return 1; + + vstart = (uintptr_t)vq->used; + vend = vstart + sizeof(struct vring_used); + vend += sizeof(struct vring_used_elem) * vq->size - 1; + if (vstart <= iend && istart <= vend) + return 1; + + return 0; +} + +static int +vhost_user_iotlb_msg(struct virtio_net **pdev, struct VhostUserMsg *msg) +{ + struct virtio_net *dev = *pdev; + struct vhost_iotlb_msg *imsg = &msg->payload.iotlb; + uint16_t i; + uint64_t vva, len; + + switch (imsg->type) { + case VHOST_IOTLB_UPDATE: + len = imsg->size; + vva = qva_to_vva(dev, imsg->uaddr, &len); + if (!vva) + return -1; + + for (i = 0; i < dev->nr_vring; i++) { + struct vhost_virtqueue *vq = dev->virtqueue[i]; + + vhost_user_iotlb_cache_insert(vq, imsg->iova, vva, + len, imsg->perm); + + if (is_vring_iotlb_update(vq, imsg)) + *pdev = dev = translate_ring_addresses(dev, i); + } + break; + case VHOST_IOTLB_INVALIDATE: + for (i = 0; i < dev->nr_vring; i++) { + struct vhost_virtqueue *vq = dev->virtqueue[i]; + + vhost_user_iotlb_cache_remove(vq, imsg->iova, + imsg->size); + + if (is_vring_iotlb_invalidate(vq, imsg)) + vring_invalidate(dev, vq); + } + break; + default: + RTE_LOG(ERR, VHOST_CONFIG, "Invalid IOTLB message type (%d)\n", + imsg->type); + return -1; + } + + return 0; +} + +/* return bytes# of read on success or negative val on failure. */ +static int +read_vhost_message(int sockfd, struct VhostUserMsg *msg) +{ + int ret; + + ret = read_fd_message(sockfd, (char *)msg, VHOST_USER_HDR_SIZE, + msg->fds, VHOST_MEMORY_MAX_NREGIONS); + if (ret <= 0) + return ret; + + if (msg && msg->size) { + if (msg->size > sizeof(msg->payload)) { + RTE_LOG(ERR, VHOST_CONFIG, + "invalid msg size: %d\n", msg->size); + return -1; + } + ret = read(sockfd, &msg->payload, msg->size); + if (ret <= 0) + return ret; + if (ret != (int)msg->size) { + RTE_LOG(ERR, VHOST_CONFIG, + "read control message failed\n"); + return -1; + } + } + + return ret; +} + +static int +send_vhost_message(int sockfd, struct VhostUserMsg *msg) +{ + if (!msg) + return 0; + + return send_fd_message(sockfd, (char *)msg, + VHOST_USER_HDR_SIZE + msg->size, NULL, 0); +} + +static int +send_vhost_reply(int sockfd, struct VhostUserMsg *msg) +{ + if (!msg) + return 0; + + msg->flags &= ~VHOST_USER_VERSION_MASK; + msg->flags &= ~VHOST_USER_NEED_REPLY; + msg->flags |= VHOST_USER_VERSION; + msg->flags |= VHOST_USER_REPLY_MASK; + + return send_vhost_message(sockfd, msg); +} + +/* + * Allocate a queue pair if it hasn't been allocated yet + */ +static int +vhost_user_check_and_alloc_queue_pair(struct virtio_net *dev, VhostUserMsg *msg) +{ + uint16_t vring_idx; + + switch (msg->request.master) { + case VHOST_USER_SET_VRING_KICK: + case VHOST_USER_SET_VRING_CALL: + case VHOST_USER_SET_VRING_ERR: + vring_idx = msg->payload.u64 & VHOST_USER_VRING_IDX_MASK; + break; + case VHOST_USER_SET_VRING_NUM: + case VHOST_USER_SET_VRING_BASE: + case VHOST_USER_SET_VRING_ENABLE: + vring_idx = msg->payload.state.index; + break; + case VHOST_USER_SET_VRING_ADDR: + vring_idx = msg->payload.addr.index; + break; + default: + return 0; + } + + if (vring_idx >= VHOST_MAX_VRING) { + RTE_LOG(ERR, VHOST_CONFIG, + "invalid vring index: %u\n", vring_idx); + return -1; + } + + if (dev->virtqueue[vring_idx]) + return 0; + + return alloc_vring_queue(dev, vring_idx); +} + +static void +vhost_user_lock_all_queue_pairs(struct virtio_net *dev) +{ + unsigned int i = 0; + unsigned int vq_num = 0; + + while (vq_num < dev->nr_vring) { + struct vhost_virtqueue *vq = dev->virtqueue[i]; + + if (vq) { + rte_spinlock_lock(&vq->access_lock); + vq_num++; + } + i++; + } +} + +static void +vhost_user_unlock_all_queue_pairs(struct virtio_net *dev) +{ + unsigned int i = 0; + unsigned int vq_num = 0; + + while (vq_num < dev->nr_vring) { + struct vhost_virtqueue *vq = dev->virtqueue[i]; + + if (vq) { + rte_spinlock_unlock(&vq->access_lock); + vq_num++; + } + i++; + } +} + +int +vhost_user_msg_handler(int vid, int fd) +{ + struct virtio_net *dev; + struct VhostUserMsg msg; + int ret; + int unlock_required = 0; + + dev = get_device(vid); + if (dev == NULL) + return -1; + + if (!dev->notify_ops) { + dev->notify_ops = vhost_driver_callback_get(dev->ifname); + if (!dev->notify_ops) { + RTE_LOG(ERR, VHOST_CONFIG, + "failed to get callback ops for driver %s\n", + dev->ifname); + return -1; + } + } + + ret = read_vhost_message(fd, &msg); + if (ret <= 0 || msg.request.master >= VHOST_USER_MAX) { + if (ret < 0) + RTE_LOG(ERR, VHOST_CONFIG, + "vhost read message failed\n"); + else if (ret == 0) + RTE_LOG(INFO, VHOST_CONFIG, + "vhost peer closed\n"); + else + RTE_LOG(ERR, VHOST_CONFIG, + "vhost read incorrect message\n"); + + return -1; + } + + ret = 0; + if (msg.request.master != VHOST_USER_IOTLB_MSG) + RTE_LOG(INFO, VHOST_CONFIG, "read message %s\n", + vhost_message_str[msg.request.master]); + else + RTE_LOG(DEBUG, VHOST_CONFIG, "read message %s\n", + vhost_message_str[msg.request.master]); + + ret = vhost_user_check_and_alloc_queue_pair(dev, &msg); + if (ret < 0) { + RTE_LOG(ERR, VHOST_CONFIG, + "failed to alloc queue\n"); + return -1; + } + + /* + * Note: we don't lock all queues on VHOST_USER_GET_VRING_BASE, + * since it is sent when virtio stops and device is destroyed. + * destroy_device waits for queues to be inactive, so it is safe. + * Otherwise taking the access_lock would cause a dead lock. + */ + switch (msg.request.master) { + case VHOST_USER_SET_FEATURES: + case VHOST_USER_SET_PROTOCOL_FEATURES: + case VHOST_USER_SET_OWNER: + case VHOST_USER_RESET_OWNER: + case VHOST_USER_SET_MEM_TABLE: + case VHOST_USER_SET_LOG_BASE: + case VHOST_USER_SET_LOG_FD: + case VHOST_USER_SET_VRING_NUM: + case VHOST_USER_SET_VRING_ADDR: + case VHOST_USER_SET_VRING_BASE: + case VHOST_USER_SET_VRING_KICK: + case VHOST_USER_SET_VRING_CALL: + case VHOST_USER_SET_VRING_ERR: + case VHOST_USER_SET_VRING_ENABLE: + case VHOST_USER_SEND_RARP: + case VHOST_USER_NET_SET_MTU: + case VHOST_USER_SET_SLAVE_REQ_FD: + vhost_user_lock_all_queue_pairs(dev); + unlock_required = 1; + break; + default: + break; + + } + + switch (msg.request.master) { + case VHOST_USER_GET_FEATURES: + msg.payload.u64 = vhost_user_get_features(dev); + msg.size = sizeof(msg.payload.u64); + send_vhost_reply(fd, &msg); + break; + case VHOST_USER_SET_FEATURES: + vhost_user_set_features(dev, msg.payload.u64); + break; + + case VHOST_USER_GET_PROTOCOL_FEATURES: + vhost_user_get_protocol_features(dev, &msg); + send_vhost_reply(fd, &msg); + break; + case VHOST_USER_SET_PROTOCOL_FEATURES: + vhost_user_set_protocol_features(dev, msg.payload.u64); + break; + + case VHOST_USER_SET_OWNER: + vhost_user_set_owner(); + break; + case VHOST_USER_RESET_OWNER: + vhost_user_reset_owner(dev); + break; + + case VHOST_USER_SET_MEM_TABLE: + ret = vhost_user_set_mem_table(dev, &msg); + break; + + case VHOST_USER_SET_LOG_BASE: + vhost_user_set_log_base(dev, &msg); + + /* it needs a reply */ + msg.size = sizeof(msg.payload.u64); + send_vhost_reply(fd, &msg); + break; + case VHOST_USER_SET_LOG_FD: + close(msg.fds[0]); + RTE_LOG(INFO, VHOST_CONFIG, "not implemented.\n"); + break; + + case VHOST_USER_SET_VRING_NUM: + vhost_user_set_vring_num(dev, &msg); + break; + case VHOST_USER_SET_VRING_ADDR: + vhost_user_set_vring_addr(&dev, &msg); + break; + case VHOST_USER_SET_VRING_BASE: + vhost_user_set_vring_base(dev, &msg); + break; + + case VHOST_USER_GET_VRING_BASE: + vhost_user_get_vring_base(dev, &msg); + msg.size = sizeof(msg.payload.state); + send_vhost_reply(fd, &msg); + break; + + case VHOST_USER_SET_VRING_KICK: + vhost_user_set_vring_kick(&dev, &msg); + break; + case VHOST_USER_SET_VRING_CALL: + vhost_user_set_vring_call(dev, &msg); + break; + + case VHOST_USER_SET_VRING_ERR: + if (!(msg.payload.u64 & VHOST_USER_VRING_NOFD_MASK)) + close(msg.fds[0]); + RTE_LOG(INFO, VHOST_CONFIG, "not implemented\n"); + break; + + case VHOST_USER_GET_QUEUE_NUM: + msg.payload.u64 = VHOST_MAX_QUEUE_PAIRS; + msg.size = sizeof(msg.payload.u64); + send_vhost_reply(fd, &msg); + break; + + case VHOST_USER_SET_VRING_ENABLE: + vhost_user_set_vring_enable(dev, &msg); + break; + case VHOST_USER_SEND_RARP: + vhost_user_send_rarp(dev, &msg); + break; + + case VHOST_USER_NET_SET_MTU: + ret = vhost_user_net_set_mtu(dev, &msg); + break; + + case VHOST_USER_SET_SLAVE_REQ_FD: + ret = vhost_user_set_req_fd(dev, &msg); + break; + + case VHOST_USER_IOTLB_MSG: + ret = vhost_user_iotlb_msg(&dev, &msg); + break; + + default: + ret = -1; + break; + + } + + if (unlock_required) + vhost_user_unlock_all_queue_pairs(dev); + + if (msg.flags & VHOST_USER_NEED_REPLY) { + msg.payload.u64 = !!ret; + msg.size = sizeof(msg.payload.u64); + send_vhost_reply(fd, &msg); + } + + if (!(dev->flags & VIRTIO_DEV_RUNNING) && virtio_is_ready(dev)) { + dev->flags |= VIRTIO_DEV_READY; + + if (!(dev->flags & VIRTIO_DEV_RUNNING)) { + if (dev->dequeue_zero_copy) { + RTE_LOG(INFO, VHOST_CONFIG, + "dequeue zero copy is enabled\n"); + } + + if (dev->notify_ops->new_device(dev->vid) == 0) + dev->flags |= VIRTIO_DEV_RUNNING; + } + } + + return 0; +} + +int +vhost_user_iotlb_miss(struct virtio_net *dev, uint64_t iova, uint8_t perm) +{ + int ret; + struct VhostUserMsg msg = { + .request.slave = VHOST_USER_SLAVE_IOTLB_MSG, + .flags = VHOST_USER_VERSION, + .size = sizeof(msg.payload.iotlb), + .payload.iotlb = { + .iova = iova, + .perm = perm, + .type = VHOST_IOTLB_MISS, + }, + }; + + ret = send_vhost_message(dev->slave_req_fd, &msg); + if (ret < 0) { + RTE_LOG(ERR, VHOST_CONFIG, + "Failed to send IOTLB miss message (%d)\n", + ret); + return ret; + } + + return 0; +} diff --git a/dpdk/lib/librte_vhost/vhost_user.h b/dpdk/lib/librte_vhost/vhost_user.h new file mode 100644 index 00000000..76d9fe2f --- /dev/null +++ b/dpdk/lib/librte_vhost/vhost_user.h @@ -0,0 +1,149 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _VHOST_NET_USER_H +#define _VHOST_NET_USER_H + +#include +#include + +#include "rte_vhost.h" + +/* refer to hw/virtio/vhost-user.c */ + +#define VHOST_MEMORY_MAX_NREGIONS 8 + +#define VHOST_USER_PROTOCOL_F_MQ 0 +#define VHOST_USER_PROTOCOL_F_LOG_SHMFD 1 +#define VHOST_USER_PROTOCOL_F_RARP 2 +#define VHOST_USER_PROTOCOL_F_REPLY_ACK 3 +#define VHOST_USER_PROTOCOL_F_NET_MTU 4 +#define VHOST_USER_PROTOCOL_F_SLAVE_REQ 5 + +#define VHOST_USER_PROTOCOL_FEATURES ((1ULL << VHOST_USER_PROTOCOL_F_MQ) | \ + (1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD) |\ + (1ULL << VHOST_USER_PROTOCOL_F_RARP) | \ + (1ULL << VHOST_USER_PROTOCOL_F_REPLY_ACK) | \ + (1ULL << VHOST_USER_PROTOCOL_F_NET_MTU) | \ + (1ULL << VHOST_USER_PROTOCOL_F_SLAVE_REQ)) + +typedef enum VhostUserRequest { + VHOST_USER_NONE = 0, + VHOST_USER_GET_FEATURES = 1, + VHOST_USER_SET_FEATURES = 2, + VHOST_USER_SET_OWNER = 3, + VHOST_USER_RESET_OWNER = 4, + VHOST_USER_SET_MEM_TABLE = 5, + VHOST_USER_SET_LOG_BASE = 6, + VHOST_USER_SET_LOG_FD = 7, + VHOST_USER_SET_VRING_NUM = 8, + VHOST_USER_SET_VRING_ADDR = 9, + VHOST_USER_SET_VRING_BASE = 10, + VHOST_USER_GET_VRING_BASE = 11, + VHOST_USER_SET_VRING_KICK = 12, + VHOST_USER_SET_VRING_CALL = 13, + VHOST_USER_SET_VRING_ERR = 14, + VHOST_USER_GET_PROTOCOL_FEATURES = 15, + VHOST_USER_SET_PROTOCOL_FEATURES = 16, + VHOST_USER_GET_QUEUE_NUM = 17, + VHOST_USER_SET_VRING_ENABLE = 18, + VHOST_USER_SEND_RARP = 19, + VHOST_USER_NET_SET_MTU = 20, + VHOST_USER_SET_SLAVE_REQ_FD = 21, + VHOST_USER_IOTLB_MSG = 22, + VHOST_USER_MAX +} VhostUserRequest; + +typedef enum VhostUserSlaveRequest { + VHOST_USER_SLAVE_NONE = 0, + VHOST_USER_SLAVE_IOTLB_MSG = 1, + VHOST_USER_SLAVE_MAX +} VhostUserSlaveRequest; + +typedef struct VhostUserMemoryRegion { + uint64_t guest_phys_addr; + uint64_t memory_size; + uint64_t userspace_addr; + uint64_t mmap_offset; +} VhostUserMemoryRegion; + +typedef struct VhostUserMemory { + uint32_t nregions; + uint32_t padding; + VhostUserMemoryRegion regions[VHOST_MEMORY_MAX_NREGIONS]; +} VhostUserMemory; + +typedef struct VhostUserLog { + uint64_t mmap_size; + uint64_t mmap_offset; +} VhostUserLog; + +typedef struct VhostUserMsg { + union { + VhostUserRequest master; + VhostUserSlaveRequest slave; + } request; + +#define VHOST_USER_VERSION_MASK 0x3 +#define VHOST_USER_REPLY_MASK (0x1 << 2) +#define VHOST_USER_NEED_REPLY (0x1 << 3) + uint32_t flags; + uint32_t size; /* the following payload size */ + union { +#define VHOST_USER_VRING_IDX_MASK 0xff +#define VHOST_USER_VRING_NOFD_MASK (0x1<<8) + uint64_t u64; + struct vhost_vring_state state; + struct vhost_vring_addr addr; + VhostUserMemory memory; + VhostUserLog log; + struct vhost_iotlb_msg iotlb; + } payload; + int fds[VHOST_MEMORY_MAX_NREGIONS]; +} __attribute((packed)) VhostUserMsg; + +#define VHOST_USER_HDR_SIZE offsetof(VhostUserMsg, payload.u64) + +/* The version of the protocol we support */ +#define VHOST_USER_VERSION 0x1 + + +/* vhost_user.c */ +int vhost_user_msg_handler(int vid, int fd); +int vhost_user_iotlb_miss(struct virtio_net *dev, uint64_t iova, uint8_t perm); + +/* socket.c */ +int read_fd_message(int sockfd, char *buf, int buflen, int *fds, int fd_num); +int send_fd_message(int sockfd, char *buf, int buflen, int *fds, int fd_num); + +#endif diff --git a/dpdk/lib/librte_vhost/vhost_user/fd_man.c b/dpdk/lib/librte_vhost/vhost_user/fd_man.c deleted file mode 100644 index 2d3eeb7d..00000000 --- a/dpdk/lib/librte_vhost/vhost_user/fd_man.c +++ /dev/null @@ -1,299 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include "fd_man.h" - -/** - * Returns the index in the fdset for a given fd. - * If fd is -1, it means to search for a free entry. - * @return - * index for the fd, or -1 if fd isn't in the fdset. - */ -static int -fdset_find_fd(struct fdset *pfdset, int fd) -{ - int i; - - if (pfdset == NULL) - return -1; - - for (i = 0; i < MAX_FDS && pfdset->fd[i].fd != fd; i++) - ; - - return i == MAX_FDS ? -1 : i; -} - -static int -fdset_find_free_slot(struct fdset *pfdset) -{ - return fdset_find_fd(pfdset, -1); -} - -static int -fdset_add_fd(struct fdset *pfdset, int idx, int fd, - fd_cb rcb, fd_cb wcb, void *dat) -{ - struct fdentry *pfdentry; - - if (pfdset == NULL || idx >= MAX_FDS || fd >= FD_SETSIZE) - return -1; - - pfdentry = &pfdset->fd[idx]; - pfdentry->fd = fd; - pfdentry->rcb = rcb; - pfdentry->wcb = wcb; - pfdentry->dat = dat; - - return 0; -} - -/** - * Fill the read/write fd_set with the fds in the fdset. - * @return - * the maximum fds filled in the read/write fd_set. - */ -static int -fdset_fill(fd_set *rfset, fd_set *wfset, struct fdset *pfdset) -{ - struct fdentry *pfdentry; - int i, maxfds = -1; - int num = MAX_FDS; - - if (pfdset == NULL) - return -1; - - for (i = 0; i < num; i++) { - pfdentry = &pfdset->fd[i]; - if (pfdentry->fd != -1) { - int added = 0; - if (pfdentry->rcb && rfset) { - FD_SET(pfdentry->fd, rfset); - added = 1; - } - if (pfdentry->wcb && wfset) { - FD_SET(pfdentry->fd, wfset); - added = 1; - } - if (added) - maxfds = pfdentry->fd < maxfds ? - maxfds : pfdentry->fd; - } - } - return maxfds; -} - -void -fdset_init(struct fdset *pfdset) -{ - int i; - - if (pfdset == NULL) - return; - - for (i = 0; i < MAX_FDS; i++) { - pfdset->fd[i].fd = -1; - pfdset->fd[i].dat = NULL; - } - pfdset->num = 0; -} - -/** - * Register the fd in the fdset with read/write handler and context. - */ -int -fdset_add(struct fdset *pfdset, int fd, fd_cb rcb, fd_cb wcb, void *dat) -{ - int i; - - if (pfdset == NULL || fd == -1) - return -1; - - pthread_mutex_lock(&pfdset->fd_mutex); - - /* Find a free slot in the list. */ - i = fdset_find_free_slot(pfdset); - if (i == -1 || fdset_add_fd(pfdset, i, fd, rcb, wcb, dat) < 0) { - pthread_mutex_unlock(&pfdset->fd_mutex); - return -2; - } - - pfdset->num++; - - pthread_mutex_unlock(&pfdset->fd_mutex); - - return 0; -} - -/** - * Unregister the fd from the fdset. - * Returns context of a given fd or NULL. - */ -void * -fdset_del(struct fdset *pfdset, int fd) -{ - int i; - void *dat = NULL; - - if (pfdset == NULL || fd == -1) - return NULL; - - do { - pthread_mutex_lock(&pfdset->fd_mutex); - - i = fdset_find_fd(pfdset, fd); - if (i != -1 && pfdset->fd[i].busy == 0) { - /* busy indicates r/wcb is executing! */ - dat = pfdset->fd[i].dat; - pfdset->fd[i].fd = -1; - pfdset->fd[i].rcb = pfdset->fd[i].wcb = NULL; - pfdset->fd[i].dat = NULL; - pfdset->num--; - i = -1; - } - pthread_mutex_unlock(&pfdset->fd_mutex); - } while (i != -1); - - return dat; -} - -/** - * Unregister the fd at the specified slot from the fdset. - */ -static void -fdset_del_slot(struct fdset *pfdset, int index) -{ - if (pfdset == NULL || index < 0 || index >= MAX_FDS) - return; - - pthread_mutex_lock(&pfdset->fd_mutex); - - pfdset->fd[index].fd = -1; - pfdset->fd[index].rcb = pfdset->fd[index].wcb = NULL; - pfdset->fd[index].dat = NULL; - pfdset->num--; - - pthread_mutex_unlock(&pfdset->fd_mutex); -} - -/** - * This functions runs in infinite blocking loop until there is no fd in - * pfdset. It calls corresponding r/w handler if there is event on the fd. - * - * Before the callback is called, we set the flag to busy status; If other - * thread(now rte_vhost_driver_unregister) calls fdset_del concurrently, it - * will wait until the flag is reset to zero(which indicates the callback is - * finished), then it could free the context after fdset_del. - */ -void -fdset_event_dispatch(struct fdset *pfdset) -{ - fd_set rfds, wfds; - int i, maxfds; - struct fdentry *pfdentry; - int num = MAX_FDS; - fd_cb rcb, wcb; - void *dat; - int fd; - int remove1, remove2; - int ret; - - if (pfdset == NULL) - return; - - while (1) { - struct timeval tv; - tv.tv_sec = 1; - tv.tv_usec = 0; - FD_ZERO(&rfds); - FD_ZERO(&wfds); - pthread_mutex_lock(&pfdset->fd_mutex); - - maxfds = fdset_fill(&rfds, &wfds, pfdset); - - pthread_mutex_unlock(&pfdset->fd_mutex); - - /* - * When select is blocked, other threads might unregister - * listenfds from and register new listenfds into fdset. - * When select returns, the entries for listenfds in the fdset - * might have been updated. It is ok if there is unwanted call - * for new listenfds. - */ - ret = select(maxfds + 1, &rfds, &wfds, NULL, &tv); - if (ret <= 0) - continue; - - for (i = 0; i < num; i++) { - remove1 = remove2 = 0; - pthread_mutex_lock(&pfdset->fd_mutex); - pfdentry = &pfdset->fd[i]; - fd = pfdentry->fd; - rcb = pfdentry->rcb; - wcb = pfdentry->wcb; - dat = pfdentry->dat; - pfdentry->busy = 1; - pthread_mutex_unlock(&pfdset->fd_mutex); - if (fd >= 0 && FD_ISSET(fd, &rfds) && rcb) - rcb(fd, dat, &remove1); - if (fd >= 0 && FD_ISSET(fd, &wfds) && wcb) - wcb(fd, dat, &remove2); - pfdentry->busy = 0; - /* - * fdset_del needs to check busy flag. - * We don't allow fdset_del to be called in callback - * directly. - */ - /* - * When we are to clean up the fd from fdset, - * because the fd is closed in the cb, - * the old fd val could be reused by when creates new - * listen fd in another thread, we couldn't call - * fd_set_del. - */ - if (remove1 || remove2) - fdset_del_slot(pfdset, i); - } - } -} diff --git a/dpdk/lib/librte_vhost/vhost_user/fd_man.h b/dpdk/lib/librte_vhost/vhost_user/fd_man.h deleted file mode 100644 index bd66ed1c..00000000 --- a/dpdk/lib/librte_vhost/vhost_user/fd_man.h +++ /dev/null @@ -1,67 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef _FD_MAN_H_ -#define _FD_MAN_H_ -#include -#include - -#define MAX_FDS 1024 - -typedef void (*fd_cb)(int fd, void *dat, int *remove); - -struct fdentry { - int fd; /* -1 indicates this entry is empty */ - fd_cb rcb; /* callback when this fd is readable. */ - fd_cb wcb; /* callback when this fd is writeable.*/ - void *dat; /* fd context */ - int busy; /* whether this entry is being used in cb. */ -}; - -struct fdset { - struct fdentry fd[MAX_FDS]; - pthread_mutex_t fd_mutex; - int num; /* current fd number of this fdset */ -}; - - -void fdset_init(struct fdset *pfdset); - -int fdset_add(struct fdset *pfdset, int fd, - fd_cb rcb, fd_cb wcb, void *dat); - -void *fdset_del(struct fdset *pfdset, int fd); - -void fdset_event_dispatch(struct fdset *pfdset); - -#endif diff --git a/dpdk/lib/librte_vhost/vhost_user/vhost-net-user.c b/dpdk/lib/librte_vhost/vhost_user/vhost-net-user.c deleted file mode 100644 index b35594d9..00000000 --- a/dpdk/lib/librte_vhost/vhost_user/vhost-net-user.c +++ /dev/null @@ -1,795 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include "fd_man.h" -#include "vhost-net-user.h" -#include "vhost-net.h" -#include "virtio-net-user.h" - -/* - * Every time rte_vhost_driver_register() is invoked, an associated - * vhost_user_socket struct will be created. - */ -struct vhost_user_socket { - char *path; - int listenfd; - int connfd; - bool is_server; - bool reconnect; -}; - -struct vhost_user_connection { - struct vhost_user_socket *vsocket; - int vid; -}; - -#define MAX_VHOST_SOCKET 1024 -struct vhost_user { - struct vhost_user_socket *vsockets[MAX_VHOST_SOCKET]; - struct fdset fdset; - int vsocket_cnt; - pthread_mutex_t mutex; -}; - -#define MAX_VIRTIO_BACKLOG 128 - -static void vhost_user_server_new_connection(int fd, void *data, int *remove); -static void vhost_user_msg_handler(int fd, void *dat, int *remove); -static int vhost_user_create_client(struct vhost_user_socket *vsocket); - -static struct vhost_user vhost_user = { - .fdset = { - .fd = { [0 ... MAX_FDS - 1] = {-1, NULL, NULL, NULL, 0} }, - .fd_mutex = PTHREAD_MUTEX_INITIALIZER, - .num = 0 - }, - .vsocket_cnt = 0, - .mutex = PTHREAD_MUTEX_INITIALIZER, -}; - -static const char *vhost_message_str[VHOST_USER_MAX] = { - [VHOST_USER_NONE] = "VHOST_USER_NONE", - [VHOST_USER_GET_FEATURES] = "VHOST_USER_GET_FEATURES", - [VHOST_USER_SET_FEATURES] = "VHOST_USER_SET_FEATURES", - [VHOST_USER_SET_OWNER] = "VHOST_USER_SET_OWNER", - [VHOST_USER_RESET_OWNER] = "VHOST_USER_RESET_OWNER", - [VHOST_USER_SET_MEM_TABLE] = "VHOST_USER_SET_MEM_TABLE", - [VHOST_USER_SET_LOG_BASE] = "VHOST_USER_SET_LOG_BASE", - [VHOST_USER_SET_LOG_FD] = "VHOST_USER_SET_LOG_FD", - [VHOST_USER_SET_VRING_NUM] = "VHOST_USER_SET_VRING_NUM", - [VHOST_USER_SET_VRING_ADDR] = "VHOST_USER_SET_VRING_ADDR", - [VHOST_USER_SET_VRING_BASE] = "VHOST_USER_SET_VRING_BASE", - [VHOST_USER_GET_VRING_BASE] = "VHOST_USER_GET_VRING_BASE", - [VHOST_USER_SET_VRING_KICK] = "VHOST_USER_SET_VRING_KICK", - [VHOST_USER_SET_VRING_CALL] = "VHOST_USER_SET_VRING_CALL", - [VHOST_USER_SET_VRING_ERR] = "VHOST_USER_SET_VRING_ERR", - [VHOST_USER_GET_PROTOCOL_FEATURES] = "VHOST_USER_GET_PROTOCOL_FEATURES", - [VHOST_USER_SET_PROTOCOL_FEATURES] = "VHOST_USER_SET_PROTOCOL_FEATURES", - [VHOST_USER_GET_QUEUE_NUM] = "VHOST_USER_GET_QUEUE_NUM", - [VHOST_USER_SET_VRING_ENABLE] = "VHOST_USER_SET_VRING_ENABLE", - [VHOST_USER_SEND_RARP] = "VHOST_USER_SEND_RARP", -}; - -/* return bytes# of read on success or negative val on failure. */ -static int -read_fd_message(int sockfd, char *buf, int buflen, int *fds, int fd_num) -{ - struct iovec iov; - struct msghdr msgh; - size_t fdsize = fd_num * sizeof(int); - char control[CMSG_SPACE(fdsize)]; - struct cmsghdr *cmsg; - int ret; - - memset(&msgh, 0, sizeof(msgh)); - iov.iov_base = buf; - iov.iov_len = buflen; - - msgh.msg_iov = &iov; - msgh.msg_iovlen = 1; - msgh.msg_control = control; - msgh.msg_controllen = sizeof(control); - - ret = recvmsg(sockfd, &msgh, 0); - if (ret <= 0) { - RTE_LOG(ERR, VHOST_CONFIG, "recvmsg failed\n"); - return ret; - } - - if (msgh.msg_flags & (MSG_TRUNC | MSG_CTRUNC)) { - RTE_LOG(ERR, VHOST_CONFIG, "truncted msg\n"); - return -1; - } - - for (cmsg = CMSG_FIRSTHDR(&msgh); cmsg != NULL; - cmsg = CMSG_NXTHDR(&msgh, cmsg)) { - if ((cmsg->cmsg_level == SOL_SOCKET) && - (cmsg->cmsg_type == SCM_RIGHTS)) { - memcpy(fds, CMSG_DATA(cmsg), fdsize); - break; - } - } - - return ret; -} - -/* return bytes# of read on success or negative val on failure. */ -static int -read_vhost_message(int sockfd, struct VhostUserMsg *msg) -{ - int ret; - - ret = read_fd_message(sockfd, (char *)msg, VHOST_USER_HDR_SIZE, - msg->fds, VHOST_MEMORY_MAX_NREGIONS); - if (ret <= 0) - return ret; - - if (msg && msg->size) { - if (msg->size > sizeof(msg->payload)) { - RTE_LOG(ERR, VHOST_CONFIG, - "invalid msg size: %d\n", msg->size); - return -1; - } - ret = read(sockfd, &msg->payload, msg->size); - if (ret <= 0) - return ret; - if (ret != (int)msg->size) { - RTE_LOG(ERR, VHOST_CONFIG, - "read control message failed\n"); - return -1; - } - } - - return ret; -} - -static int -send_fd_message(int sockfd, char *buf, int buflen, int *fds, int fd_num) -{ - - struct iovec iov; - struct msghdr msgh; - size_t fdsize = fd_num * sizeof(int); - char control[CMSG_SPACE(fdsize)]; - struct cmsghdr *cmsg; - int ret; - - memset(&msgh, 0, sizeof(msgh)); - iov.iov_base = buf; - iov.iov_len = buflen; - - msgh.msg_iov = &iov; - msgh.msg_iovlen = 1; - - if (fds && fd_num > 0) { - msgh.msg_control = control; - msgh.msg_controllen = sizeof(control); - cmsg = CMSG_FIRSTHDR(&msgh); - cmsg->cmsg_len = CMSG_LEN(fdsize); - cmsg->cmsg_level = SOL_SOCKET; - cmsg->cmsg_type = SCM_RIGHTS; - memcpy(CMSG_DATA(cmsg), fds, fdsize); - } else { - msgh.msg_control = NULL; - msgh.msg_controllen = 0; - } - - do { - ret = sendmsg(sockfd, &msgh, 0); - } while (ret < 0 && errno == EINTR); - - if (ret < 0) { - RTE_LOG(ERR, VHOST_CONFIG, "sendmsg error\n"); - return ret; - } - - return ret; -} - -static int -send_vhost_message(int sockfd, struct VhostUserMsg *msg) -{ - int ret; - - if (!msg) - return 0; - - msg->flags &= ~VHOST_USER_VERSION_MASK; - msg->flags |= VHOST_USER_VERSION; - msg->flags |= VHOST_USER_REPLY_MASK; - - ret = send_fd_message(sockfd, (char *)msg, - VHOST_USER_HDR_SIZE + msg->size, NULL, 0); - - return ret; -} - - -static void -vhost_user_add_connection(int fd, struct vhost_user_socket *vsocket) -{ - int vid; - size_t size; - struct vhost_user_connection *conn; - int ret; - - conn = malloc(sizeof(*conn)); - if (conn == NULL) { - close(fd); - return; - } - - vid = vhost_new_device(); - if (vid == -1) { - close(fd); - free(conn); - return; - } - - size = strnlen(vsocket->path, PATH_MAX); - vhost_set_ifname(vid, vsocket->path, size); - - RTE_LOG(INFO, VHOST_CONFIG, "new device, handle is %d\n", vid); - - vsocket->connfd = fd; - conn->vsocket = vsocket; - conn->vid = vid; - ret = fdset_add(&vhost_user.fdset, fd, vhost_user_msg_handler, - NULL, conn); - if (ret < 0) { - vsocket->connfd = -1; - free(conn); - close(fd); - RTE_LOG(ERR, VHOST_CONFIG, - "failed to add fd %d into vhost server fdset\n", - fd); - } -} - -/* call back when there is new vhost-user connection from client */ -static void -vhost_user_server_new_connection(int fd, void *dat, int *remove __rte_unused) -{ - struct vhost_user_socket *vsocket = dat; - - fd = accept(fd, NULL, NULL); - if (fd < 0) - return; - - RTE_LOG(INFO, VHOST_CONFIG, "new vhost user connection is %d\n", fd); - vhost_user_add_connection(fd, vsocket); -} - -/* callback when there is message on the connfd */ -static void -vhost_user_msg_handler(int connfd, void *dat, int *remove) -{ - int vid; - struct vhost_user_connection *conn = dat; - struct VhostUserMsg msg; - uint64_t features; - int ret; - - vid = conn->vid; - ret = read_vhost_message(connfd, &msg); - if (ret <= 0 || msg.request >= VHOST_USER_MAX) { - struct vhost_user_socket *vsocket = conn->vsocket; - - if (ret < 0) - RTE_LOG(ERR, VHOST_CONFIG, - "vhost read message failed\n"); - else if (ret == 0) - RTE_LOG(INFO, VHOST_CONFIG, - "vhost peer closed\n"); - else - RTE_LOG(ERR, VHOST_CONFIG, - "vhost read incorrect message\n"); - - vsocket->connfd = -1; - close(connfd); - *remove = 1; - free(conn); - vhost_destroy_device(vid); - - if (vsocket->reconnect) - vhost_user_create_client(vsocket); - - return; - } - - RTE_LOG(INFO, VHOST_CONFIG, "read message %s\n", - vhost_message_str[msg.request]); - switch (msg.request) { - case VHOST_USER_GET_FEATURES: - ret = vhost_get_features(vid, &features); - msg.payload.u64 = features; - msg.size = sizeof(msg.payload.u64); - send_vhost_message(connfd, &msg); - break; - case VHOST_USER_SET_FEATURES: - features = msg.payload.u64; - vhost_set_features(vid, &features); - break; - - case VHOST_USER_GET_PROTOCOL_FEATURES: - msg.payload.u64 = VHOST_USER_PROTOCOL_FEATURES; - msg.size = sizeof(msg.payload.u64); - send_vhost_message(connfd, &msg); - break; - case VHOST_USER_SET_PROTOCOL_FEATURES: - user_set_protocol_features(vid, msg.payload.u64); - break; - - case VHOST_USER_SET_OWNER: - vhost_set_owner(vid); - break; - case VHOST_USER_RESET_OWNER: - vhost_reset_owner(vid); - break; - - case VHOST_USER_SET_MEM_TABLE: - user_set_mem_table(vid, &msg); - break; - - case VHOST_USER_SET_LOG_BASE: - user_set_log_base(vid, &msg); - - /* it needs a reply */ - msg.size = sizeof(msg.payload.u64); - send_vhost_message(connfd, &msg); - break; - case VHOST_USER_SET_LOG_FD: - close(msg.fds[0]); - RTE_LOG(INFO, VHOST_CONFIG, "not implemented.\n"); - break; - - case VHOST_USER_SET_VRING_NUM: - vhost_set_vring_num(vid, &msg.payload.state); - break; - case VHOST_USER_SET_VRING_ADDR: - vhost_set_vring_addr(vid, &msg.payload.addr); - break; - case VHOST_USER_SET_VRING_BASE: - vhost_set_vring_base(vid, &msg.payload.state); - break; - - case VHOST_USER_GET_VRING_BASE: - ret = user_get_vring_base(vid, &msg.payload.state); - msg.size = sizeof(msg.payload.state); - send_vhost_message(connfd, &msg); - break; - - case VHOST_USER_SET_VRING_KICK: - user_set_vring_kick(vid, &msg); - break; - case VHOST_USER_SET_VRING_CALL: - user_set_vring_call(vid, &msg); - break; - - case VHOST_USER_SET_VRING_ERR: - if (!(msg.payload.u64 & VHOST_USER_VRING_NOFD_MASK)) - close(msg.fds[0]); - RTE_LOG(INFO, VHOST_CONFIG, "not implemented\n"); - break; - - case VHOST_USER_GET_QUEUE_NUM: - msg.payload.u64 = VHOST_MAX_QUEUE_PAIRS; - msg.size = sizeof(msg.payload.u64); - send_vhost_message(connfd, &msg); - break; - - case VHOST_USER_SET_VRING_ENABLE: - user_set_vring_enable(vid, &msg.payload.state); - break; - case VHOST_USER_SEND_RARP: - user_send_rarp(vid, &msg); - break; - - default: - break; - - } -} - -static int -create_unix_socket(const char *path, struct sockaddr_un *un, bool is_server) -{ - int fd; - - fd = socket(AF_UNIX, SOCK_STREAM, 0); - if (fd < 0) - return -1; - RTE_LOG(INFO, VHOST_CONFIG, "vhost-user %s: socket created, fd: %d\n", - is_server ? "server" : "client", fd); - - if (!is_server && fcntl(fd, F_SETFL, O_NONBLOCK)) { - RTE_LOG(ERR, VHOST_CONFIG, - "vhost-user: can't set nonblocking mode for socket, fd: " - "%d (%s)\n", fd, strerror(errno)); - close(fd); - return -1; - } - - memset(un, 0, sizeof(*un)); - un->sun_family = AF_UNIX; - strncpy(un->sun_path, path, sizeof(un->sun_path)); - un->sun_path[sizeof(un->sun_path) - 1] = '\0'; - - return fd; -} - -static int -vhost_user_create_server(struct vhost_user_socket *vsocket) -{ - int fd; - int ret; - struct sockaddr_un un; - const char *path = vsocket->path; - - fd = create_unix_socket(path, &un, vsocket->is_server); - if (fd < 0) - return -1; - - ret = bind(fd, (struct sockaddr *)&un, sizeof(un)); - if (ret < 0) { - RTE_LOG(ERR, VHOST_CONFIG, - "failed to bind to %s: %s; remove it and try again\n", - path, strerror(errno)); - goto err; - } - RTE_LOG(INFO, VHOST_CONFIG, "bind to %s\n", path); - - ret = listen(fd, MAX_VIRTIO_BACKLOG); - if (ret < 0) - goto err; - - vsocket->listenfd = fd; - ret = fdset_add(&vhost_user.fdset, fd, vhost_user_server_new_connection, - NULL, vsocket); - if (ret < 0) { - RTE_LOG(ERR, VHOST_CONFIG, - "failed to add listen fd %d to vhost server fdset\n", - fd); - goto err; - } - - return 0; - -err: - close(fd); - return -1; -} - -struct vhost_user_reconnect { - struct sockaddr_un un; - int fd; - struct vhost_user_socket *vsocket; - - TAILQ_ENTRY(vhost_user_reconnect) next; -}; - -TAILQ_HEAD(vhost_user_reconnect_tailq_list, vhost_user_reconnect); -struct vhost_user_reconnect_list { - struct vhost_user_reconnect_tailq_list head; - pthread_mutex_t mutex; -}; - -static struct vhost_user_reconnect_list reconn_list; -static pthread_t reconn_tid; - -static int -vhost_user_connect_nonblock(int fd, struct sockaddr *un, size_t sz) -{ - int ret, flags; - - ret = connect(fd, un, sz); - if (ret < 0 && errno != EISCONN) - return -1; - - flags = fcntl(fd, F_GETFL, 0); - if (flags < 0) { - RTE_LOG(ERR, VHOST_CONFIG, - "can't get flags for connfd %d\n", fd); - return -2; - } - if ((flags & O_NONBLOCK) && fcntl(fd, F_SETFL, flags & ~O_NONBLOCK)) { - RTE_LOG(ERR, VHOST_CONFIG, - "can't disable nonblocking on fd %d\n", fd); - return -2; - } - return 0; -} - -static void * -vhost_user_client_reconnect(void *arg __rte_unused) -{ - int ret; - struct vhost_user_reconnect *reconn, *next; - - while (1) { - pthread_mutex_lock(&reconn_list.mutex); - - /* - * An equal implementation of TAILQ_FOREACH_SAFE, - * which does not exist on all platforms. - */ - for (reconn = TAILQ_FIRST(&reconn_list.head); - reconn != NULL; reconn = next) { - next = TAILQ_NEXT(reconn, next); - - ret = vhost_user_connect_nonblock(reconn->fd, - (struct sockaddr *)&reconn->un, - sizeof(reconn->un)); - if (ret == -2) { - close(reconn->fd); - RTE_LOG(ERR, VHOST_CONFIG, - "reconnection for fd %d failed\n", - reconn->fd); - goto remove_fd; - } - if (ret == -1) - continue; - - RTE_LOG(INFO, VHOST_CONFIG, - "%s: connected\n", reconn->vsocket->path); - vhost_user_add_connection(reconn->fd, reconn->vsocket); -remove_fd: - TAILQ_REMOVE(&reconn_list.head, reconn, next); - free(reconn); - } - - pthread_mutex_unlock(&reconn_list.mutex); - sleep(1); - } - - return NULL; -} - -static int -vhost_user_reconnect_init(void) -{ - int ret; - - pthread_mutex_init(&reconn_list.mutex, NULL); - TAILQ_INIT(&reconn_list.head); - - ret = pthread_create(&reconn_tid, NULL, - vhost_user_client_reconnect, NULL); - if (ret < 0) - RTE_LOG(ERR, VHOST_CONFIG, "failed to create reconnect thread"); - - return ret; -} - -static int -vhost_user_create_client(struct vhost_user_socket *vsocket) -{ - int fd; - int ret; - struct sockaddr_un un; - const char *path = vsocket->path; - struct vhost_user_reconnect *reconn; - - fd = create_unix_socket(path, &un, vsocket->is_server); - if (fd < 0) - return -1; - - ret = vhost_user_connect_nonblock(fd, (struct sockaddr *)&un, - sizeof(un)); - if (ret == 0) { - vhost_user_add_connection(fd, vsocket); - return 0; - } - - RTE_LOG(ERR, VHOST_CONFIG, - "failed to connect to %s: %s\n", - path, strerror(errno)); - - if (ret == -2 || !vsocket->reconnect) { - close(fd); - return -1; - } - - RTE_LOG(ERR, VHOST_CONFIG, "%s: reconnecting...\n", path); - reconn = malloc(sizeof(*reconn)); - if (reconn == NULL) { - RTE_LOG(ERR, VHOST_CONFIG, - "failed to allocate memory for reconnect\n"); - close(fd); - return -1; - } - reconn->un = un; - reconn->fd = fd; - reconn->vsocket = vsocket; - pthread_mutex_lock(&reconn_list.mutex); - TAILQ_INSERT_TAIL(&reconn_list.head, reconn, next); - pthread_mutex_unlock(&reconn_list.mutex); - - return 0; -} - -/* - * Register a new vhost-user socket; here we could act as server - * (the default case), or client (when RTE_VHOST_USER_CLIENT) flag - * is set. - */ -int -rte_vhost_driver_register(const char *path, uint64_t flags) -{ - int ret = -1; - struct vhost_user_socket *vsocket; - - if (!path) - return -1; - - pthread_mutex_lock(&vhost_user.mutex); - - if (vhost_user.vsocket_cnt == MAX_VHOST_SOCKET) { - RTE_LOG(ERR, VHOST_CONFIG, - "error: the number of vhost sockets reaches maximum\n"); - goto out; - } - - vsocket = malloc(sizeof(struct vhost_user_socket)); - if (!vsocket) - goto out; - memset(vsocket, 0, sizeof(struct vhost_user_socket)); - vsocket->path = strdup(path); - vsocket->connfd = -1; - - if ((flags & RTE_VHOST_USER_CLIENT) != 0) { - vsocket->reconnect = !(flags & RTE_VHOST_USER_NO_RECONNECT); - if (vsocket->reconnect && reconn_tid == 0) { - if (vhost_user_reconnect_init() < 0) { - free(vsocket->path); - free(vsocket); - goto out; - } - } - ret = vhost_user_create_client(vsocket); - } else { - vsocket->is_server = true; - ret = vhost_user_create_server(vsocket); - } - if (ret < 0) { - free(vsocket->path); - free(vsocket); - goto out; - } - - vhost_user.vsockets[vhost_user.vsocket_cnt++] = vsocket; - -out: - pthread_mutex_unlock(&vhost_user.mutex); - - return ret; -} - -static bool -vhost_user_remove_reconnect(struct vhost_user_socket *vsocket) -{ - int found = false; - struct vhost_user_reconnect *reconn, *next; - - pthread_mutex_lock(&reconn_list.mutex); - - for (reconn = TAILQ_FIRST(&reconn_list.head); - reconn != NULL; reconn = next) { - next = TAILQ_NEXT(reconn, next); - - if (reconn->vsocket == vsocket) { - TAILQ_REMOVE(&reconn_list.head, reconn, next); - close(reconn->fd); - free(reconn); - found = true; - break; - } - } - pthread_mutex_unlock(&reconn_list.mutex); - return found; -} - -/** - * Unregister the specified vhost socket - */ -int -rte_vhost_driver_unregister(const char *path) -{ - int i; - int count; - struct vhost_user_connection *conn; - - pthread_mutex_lock(&vhost_user.mutex); - - for (i = 0; i < vhost_user.vsocket_cnt; i++) { - struct vhost_user_socket *vsocket = vhost_user.vsockets[i]; - - if (!strcmp(vsocket->path, path)) { - if (vsocket->is_server) { - fdset_del(&vhost_user.fdset, vsocket->listenfd); - close(vsocket->listenfd); - unlink(path); - } else if (vsocket->reconnect) { - vhost_user_remove_reconnect(vsocket); - } - - conn = fdset_del(&vhost_user.fdset, vsocket->connfd); - if (conn) { - RTE_LOG(INFO, VHOST_CONFIG, - "free connfd = %d for device '%s'\n", - vsocket->connfd, path); - close(vsocket->connfd); - vhost_destroy_device(conn->vid); - free(conn); - } - - free(vsocket->path); - free(vsocket); - - count = --vhost_user.vsocket_cnt; - vhost_user.vsockets[i] = vhost_user.vsockets[count]; - vhost_user.vsockets[count] = NULL; - pthread_mutex_unlock(&vhost_user.mutex); - - return 0; - } - } - pthread_mutex_unlock(&vhost_user.mutex); - - return -1; -} - -int -rte_vhost_driver_session_start(void) -{ - fdset_event_dispatch(&vhost_user.fdset); - return 0; -} diff --git a/dpdk/lib/librte_vhost/vhost_user/vhost-net-user.h b/dpdk/lib/librte_vhost/vhost_user/vhost-net-user.h deleted file mode 100644 index f5332396..00000000 --- a/dpdk/lib/librte_vhost/vhost_user/vhost-net-user.h +++ /dev/null @@ -1,113 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef _VHOST_NET_USER_H -#define _VHOST_NET_USER_H - -#include -#include - -#include "rte_virtio_net.h" - -/* refer to hw/virtio/vhost-user.c */ - -#define VHOST_MEMORY_MAX_NREGIONS 8 - -typedef enum VhostUserRequest { - VHOST_USER_NONE = 0, - VHOST_USER_GET_FEATURES = 1, - VHOST_USER_SET_FEATURES = 2, - VHOST_USER_SET_OWNER = 3, - VHOST_USER_RESET_OWNER = 4, - VHOST_USER_SET_MEM_TABLE = 5, - VHOST_USER_SET_LOG_BASE = 6, - VHOST_USER_SET_LOG_FD = 7, - VHOST_USER_SET_VRING_NUM = 8, - VHOST_USER_SET_VRING_ADDR = 9, - VHOST_USER_SET_VRING_BASE = 10, - VHOST_USER_GET_VRING_BASE = 11, - VHOST_USER_SET_VRING_KICK = 12, - VHOST_USER_SET_VRING_CALL = 13, - VHOST_USER_SET_VRING_ERR = 14, - VHOST_USER_GET_PROTOCOL_FEATURES = 15, - VHOST_USER_SET_PROTOCOL_FEATURES = 16, - VHOST_USER_GET_QUEUE_NUM = 17, - VHOST_USER_SET_VRING_ENABLE = 18, - VHOST_USER_SEND_RARP = 19, - VHOST_USER_MAX -} VhostUserRequest; - -typedef struct VhostUserMemoryRegion { - uint64_t guest_phys_addr; - uint64_t memory_size; - uint64_t userspace_addr; - uint64_t mmap_offset; -} VhostUserMemoryRegion; - -typedef struct VhostUserMemory { - uint32_t nregions; - uint32_t padding; - VhostUserMemoryRegion regions[VHOST_MEMORY_MAX_NREGIONS]; -} VhostUserMemory; - -typedef struct VhostUserLog { - uint64_t mmap_size; - uint64_t mmap_offset; -} VhostUserLog; - -typedef struct VhostUserMsg { - VhostUserRequest request; - -#define VHOST_USER_VERSION_MASK 0x3 -#define VHOST_USER_REPLY_MASK (0x1 << 2) - uint32_t flags; - uint32_t size; /* the following payload size */ - union { -#define VHOST_USER_VRING_IDX_MASK 0xff -#define VHOST_USER_VRING_NOFD_MASK (0x1<<8) - uint64_t u64; - struct vhost_vring_state state; - struct vhost_vring_addr addr; - VhostUserMemory memory; - VhostUserLog log; - } payload; - int fds[VHOST_MEMORY_MAX_NREGIONS]; -} __attribute((packed)) VhostUserMsg; - -#define VHOST_USER_HDR_SIZE offsetof(VhostUserMsg, payload.u64) - -/* The version of the protocol we support */ -#define VHOST_USER_VERSION 0x1 - -/*****************************************************************************/ -#endif diff --git a/dpdk/lib/librte_vhost/vhost_user/virtio-net-user.c b/dpdk/lib/librte_vhost/vhost_user/virtio-net-user.c deleted file mode 100644 index e7c43479..00000000 --- a/dpdk/lib/librte_vhost/vhost_user/virtio-net-user.c +++ /dev/null @@ -1,470 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include "virtio-net-user.h" -#include "vhost-net-user.h" -#include "vhost-net.h" - -struct orig_region_map { - int fd; - uint64_t mapped_address; - uint64_t mapped_size; - uint64_t blksz; -}; - -#define orig_region(ptr, nregions) \ - ((struct orig_region_map *)RTE_PTR_ADD((ptr), \ - sizeof(struct virtio_memory) + \ - sizeof(struct virtio_memory_regions) * (nregions))) - -static uint64_t -get_blk_size(int fd) -{ - struct stat stat; - int ret; - - ret = fstat(fd, &stat); - return ret == -1 ? (uint64_t)-1 : (uint64_t)stat.st_blksize; -} - -static void -free_mem_region(struct virtio_net *dev) -{ - struct orig_region_map *region; - unsigned int idx; - - if (!dev || !dev->mem) - return; - - region = orig_region(dev->mem, dev->mem->nregions); - for (idx = 0; idx < dev->mem->nregions; idx++) { - if (region[idx].mapped_address) { - munmap((void *)(uintptr_t)region[idx].mapped_address, - region[idx].mapped_size); - close(region[idx].fd); - } - } -} - -void -vhost_backend_cleanup(struct virtio_net *dev) -{ - if (dev->mem) { - free_mem_region(dev); - free(dev->mem); - dev->mem = NULL; - } - if (dev->log_addr) { - munmap((void *)(uintptr_t)dev->log_addr, dev->log_size); - dev->log_addr = 0; - } -} - -int -user_set_mem_table(int vid, struct VhostUserMsg *pmsg) -{ - struct VhostUserMemory memory = pmsg->payload.memory; - struct virtio_memory_regions *pregion; - uint64_t mapped_address, mapped_size; - struct virtio_net *dev; - unsigned int idx = 0; - struct orig_region_map *pregion_orig; - uint64_t alignment; - - /* unmap old memory regions one by one*/ - dev = get_device(vid); - if (dev == NULL) - return -1; - - /* Remove from the data plane. */ - if (dev->flags & VIRTIO_DEV_RUNNING) { - dev->flags &= ~VIRTIO_DEV_RUNNING; - notify_ops->destroy_device(vid); - } - - if (dev->mem) { - free_mem_region(dev); - free(dev->mem); - dev->mem = NULL; - } - - dev->mem = calloc(1, - sizeof(struct virtio_memory) + - sizeof(struct virtio_memory_regions) * memory.nregions + - sizeof(struct orig_region_map) * memory.nregions); - if (dev->mem == NULL) { - RTE_LOG(ERR, VHOST_CONFIG, - "(%d) failed to allocate memory for dev->mem\n", - dev->vid); - return -1; - } - dev->mem->nregions = memory.nregions; - - pregion_orig = orig_region(dev->mem, memory.nregions); - for (idx = 0; idx < memory.nregions; idx++) { - pregion = &dev->mem->regions[idx]; - pregion->guest_phys_address = - memory.regions[idx].guest_phys_addr; - pregion->guest_phys_address_end = - memory.regions[idx].guest_phys_addr + - memory.regions[idx].memory_size; - pregion->memory_size = - memory.regions[idx].memory_size; - pregion->userspace_address = - memory.regions[idx].userspace_addr; - - /* This is ugly */ - mapped_size = memory.regions[idx].memory_size + - memory.regions[idx].mmap_offset; - - /* mmap() without flag of MAP_ANONYMOUS, should be called - * with length argument aligned with hugepagesz at older - * longterm version Linux, like 2.6.32 and 3.2.72, or - * mmap() will fail with EINVAL. - * - * to avoid failure, make sure in caller to keep length - * aligned. - */ - alignment = get_blk_size(pmsg->fds[idx]); - if (alignment == (uint64_t)-1) { - RTE_LOG(ERR, VHOST_CONFIG, - "couldn't get hugepage size through fstat\n"); - goto err_mmap; - } - mapped_size = RTE_ALIGN_CEIL(mapped_size, alignment); - - mapped_address = (uint64_t)(uintptr_t)mmap(NULL, - mapped_size, - PROT_READ | PROT_WRITE, MAP_SHARED, - pmsg->fds[idx], - 0); - - RTE_LOG(INFO, VHOST_CONFIG, - "mapped region %d fd:%d to:%p sz:0x%"PRIx64" " - "off:0x%"PRIx64" align:0x%"PRIx64"\n", - idx, pmsg->fds[idx], (void *)(uintptr_t)mapped_address, - mapped_size, memory.regions[idx].mmap_offset, - alignment); - - if (mapped_address == (uint64_t)(uintptr_t)MAP_FAILED) { - RTE_LOG(ERR, VHOST_CONFIG, - "mmap qemu guest failed.\n"); - goto err_mmap; - } - - pregion_orig[idx].mapped_address = mapped_address; - pregion_orig[idx].mapped_size = mapped_size; - pregion_orig[idx].blksz = alignment; - pregion_orig[idx].fd = pmsg->fds[idx]; - - mapped_address += memory.regions[idx].mmap_offset; - - pregion->address_offset = mapped_address - - pregion->guest_phys_address; - - if (memory.regions[idx].guest_phys_addr == 0) { - dev->mem->base_address = - memory.regions[idx].userspace_addr; - dev->mem->mapped_address = - pregion->address_offset; - } - - LOG_DEBUG(VHOST_CONFIG, - "REGION: %u GPA: %p QEMU VA: %p SIZE (%"PRIu64")\n", - idx, - (void *)(uintptr_t)pregion->guest_phys_address, - (void *)(uintptr_t)pregion->userspace_address, - pregion->memory_size); - } - - return 0; - -err_mmap: - while (idx--) { - munmap((void *)(uintptr_t)pregion_orig[idx].mapped_address, - pregion_orig[idx].mapped_size); - close(pregion_orig[idx].fd); - } - free(dev->mem); - dev->mem = NULL; - return -1; -} - -static int -vq_is_ready(struct vhost_virtqueue *vq) -{ - return vq && vq->desc && - vq->kickfd != VIRTIO_UNINITIALIZED_EVENTFD && - vq->callfd != VIRTIO_UNINITIALIZED_EVENTFD; -} - -static int -virtio_is_ready(struct virtio_net *dev) -{ - struct vhost_virtqueue *rvq, *tvq; - uint32_t i; - - for (i = 0; i < dev->virt_qp_nb; i++) { - rvq = dev->virtqueue[i * VIRTIO_QNUM + VIRTIO_RXQ]; - tvq = dev->virtqueue[i * VIRTIO_QNUM + VIRTIO_TXQ]; - - if (!vq_is_ready(rvq) || !vq_is_ready(tvq)) { - RTE_LOG(INFO, VHOST_CONFIG, - "virtio is not ready for processing.\n"); - return 0; - } - } - - RTE_LOG(INFO, VHOST_CONFIG, - "virtio is now ready for processing.\n"); - return 1; -} - -void -user_set_vring_call(int vid, struct VhostUserMsg *pmsg) -{ - struct vhost_vring_file file; - - file.index = pmsg->payload.u64 & VHOST_USER_VRING_IDX_MASK; - if (pmsg->payload.u64 & VHOST_USER_VRING_NOFD_MASK) - file.fd = VIRTIO_INVALID_EVENTFD; - else - file.fd = pmsg->fds[0]; - RTE_LOG(INFO, VHOST_CONFIG, - "vring call idx:%d file:%d\n", file.index, file.fd); - vhost_set_vring_call(vid, &file); -} - - -/* - * In vhost-user, when we receive kick message, will test whether virtio - * device is ready for packet processing. - */ -void -user_set_vring_kick(int vid, struct VhostUserMsg *pmsg) -{ - struct vhost_vring_file file; - struct virtio_net *dev = get_device(vid); - - if (!dev) - return; - - file.index = pmsg->payload.u64 & VHOST_USER_VRING_IDX_MASK; - if (pmsg->payload.u64 & VHOST_USER_VRING_NOFD_MASK) - file.fd = VIRTIO_INVALID_EVENTFD; - else - file.fd = pmsg->fds[0]; - RTE_LOG(INFO, VHOST_CONFIG, - "vring kick idx:%d file:%d\n", file.index, file.fd); - vhost_set_vring_kick(vid, &file); - - if (virtio_is_ready(dev) && !(dev->flags & VIRTIO_DEV_RUNNING)) { - if (notify_ops->new_device(vid) == 0) - dev->flags |= VIRTIO_DEV_RUNNING; - } -} - -/* - * when virtio is stopped, qemu will send us the GET_VRING_BASE message. - */ -int -user_get_vring_base(int vid, struct vhost_vring_state *state) -{ - struct virtio_net *dev = get_device(vid); - - if (dev == NULL) - return -1; - /* We have to stop the queue (virtio) if it is running. */ - if (dev->flags & VIRTIO_DEV_RUNNING) { - dev->flags &= ~VIRTIO_DEV_RUNNING; - notify_ops->destroy_device(vid); - } - - /* Here we are safe to get the last used index */ - vhost_get_vring_base(vid, state->index, state); - - RTE_LOG(INFO, VHOST_CONFIG, - "vring base idx:%d file:%d\n", state->index, state->num); - /* - * Based on current qemu vhost-user implementation, this message is - * sent and only sent in vhost_vring_stop. - * TODO: cleanup the vring, it isn't usable since here. - */ - if (dev->virtqueue[state->index]->kickfd >= 0) - close(dev->virtqueue[state->index]->kickfd); - - dev->virtqueue[state->index]->kickfd = VIRTIO_UNINITIALIZED_EVENTFD; - - return 0; -} - -/* - * when virtio queues are ready to work, qemu will send us to - * enable the virtio queue pair. - */ -int -user_set_vring_enable(int vid, struct vhost_vring_state *state) -{ - struct virtio_net *dev; - int enable = (int)state->num; - - dev = get_device(vid); - if (dev == NULL) - return -1; - - RTE_LOG(INFO, VHOST_CONFIG, - "set queue enable: %d to qp idx: %d\n", - enable, state->index); - - if (notify_ops->vring_state_changed) - notify_ops->vring_state_changed(vid, state->index, enable); - - dev->virtqueue[state->index]->enabled = enable; - - return 0; -} - -void -user_set_protocol_features(int vid, uint64_t protocol_features) -{ - struct virtio_net *dev; - - dev = get_device(vid); - if (dev == NULL || protocol_features & ~VHOST_USER_PROTOCOL_FEATURES) - return; - - dev->protocol_features = protocol_features; -} - -int -user_set_log_base(int vid, struct VhostUserMsg *msg) -{ - struct virtio_net *dev; - int fd = msg->fds[0]; - uint64_t size, off; - void *addr; - - dev = get_device(vid); - if (!dev) - return -1; - - if (fd < 0) { - RTE_LOG(ERR, VHOST_CONFIG, "invalid log fd: %d\n", fd); - return -1; - } - - if (msg->size != sizeof(VhostUserLog)) { - RTE_LOG(ERR, VHOST_CONFIG, - "invalid log base msg size: %"PRId32" != %d\n", - msg->size, (int)sizeof(VhostUserLog)); - return -1; - } - - size = msg->payload.log.mmap_size; - off = msg->payload.log.mmap_offset; - RTE_LOG(INFO, VHOST_CONFIG, - "log mmap size: %"PRId64", offset: %"PRId64"\n", - size, off); - - /* - * mmap from 0 to workaround a hugepage mmap bug: mmap will - * fail when offset is not page size aligned. - */ - addr = mmap(0, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); - close(fd); - if (addr == MAP_FAILED) { - RTE_LOG(ERR, VHOST_CONFIG, "mmap log base failed!\n"); - return -1; - } - - /* - * Free previously mapped log memory on occasionally - * multiple VHOST_USER_SET_LOG_BASE. - */ - if (dev->log_addr) { - munmap((void *)(uintptr_t)dev->log_addr, dev->log_size); - } - dev->log_addr = (uint64_t)(uintptr_t)addr; - dev->log_base = dev->log_addr + off; - dev->log_size = size; - - return 0; -} - -/* - * An rarp packet is constructed and broadcasted to notify switches about - * the new location of the migrated VM, so that packets from outside will - * not be lost after migration. - * - * However, we don't actually "send" a rarp packet here, instead, we set - * a flag 'broadcast_rarp' to let rte_vhost_dequeue_burst() inject it. - */ -int -user_send_rarp(int vid, struct VhostUserMsg *msg) -{ - struct virtio_net *dev; - uint8_t *mac = (uint8_t *)&msg->payload.u64; - - dev = get_device(vid); - if (!dev) - return -1; - - RTE_LOG(DEBUG, VHOST_CONFIG, - ":: mac: %02x:%02x:%02x:%02x:%02x:%02x\n", - mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]); - memcpy(dev->mac.addr_bytes, mac, 6); - - /* - * Set the flag to inject a RARP broadcast packet at - * rte_vhost_dequeue_burst(). - * - * rte_smp_wmb() is for making sure the mac is copied - * before the flag is set. - */ - rte_smp_wmb(); - rte_atomic16_set(&dev->broadcast_rarp, 1); - - return 0; -} diff --git a/dpdk/lib/librte_vhost/vhost_user/virtio-net-user.h b/dpdk/lib/librte_vhost/vhost_user/virtio-net-user.h deleted file mode 100644 index e1b967b8..00000000 --- a/dpdk/lib/librte_vhost/vhost_user/virtio-net-user.h +++ /dev/null @@ -1,62 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef _VIRTIO_NET_USER_H -#define _VIRTIO_NET_USER_H - -#include "vhost-net.h" -#include "vhost-net-user.h" - -#define VHOST_USER_PROTOCOL_F_MQ 0 -#define VHOST_USER_PROTOCOL_F_LOG_SHMFD 1 -#define VHOST_USER_PROTOCOL_F_RARP 2 - -#define VHOST_USER_PROTOCOL_FEATURES ((1ULL << VHOST_USER_PROTOCOL_F_MQ) | \ - (1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD) |\ - (1ULL << VHOST_USER_PROTOCOL_F_RARP)) - -int user_set_mem_table(int, struct VhostUserMsg *); - -void user_set_vring_call(int, struct VhostUserMsg *); - -void user_set_vring_kick(int, struct VhostUserMsg *); - -void user_set_protocol_features(int vid, uint64_t protocol_features); -int user_set_log_base(int vid, struct VhostUserMsg *); -int user_send_rarp(int vid, struct VhostUserMsg *); - -int user_get_vring_base(int, struct vhost_vring_state *); - -int user_set_vring_enable(int vid, struct vhost_vring_state *state); - -#endif diff --git a/dpdk/lib/librte_vhost/virtio-net.c b/dpdk/lib/librte_vhost/virtio-net.c deleted file mode 100644 index 1785695b..00000000 --- a/dpdk/lib/librte_vhost/virtio-net.c +++ /dev/null @@ -1,847 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#ifdef RTE_LIBRTE_VHOST_NUMA -#include -#endif - -#include - -#include -#include -#include -#include -#include -#include - -#include "vhost-net.h" - -#define MAX_VHOST_DEVICE 1024 -static struct virtio_net *vhost_devices[MAX_VHOST_DEVICE]; - -/* device ops to add/remove device to/from data core. */ -struct virtio_net_device_ops const *notify_ops; - -#define VHOST_USER_F_PROTOCOL_FEATURES 30 - -/* Features supported by this lib. */ -#define VHOST_SUPPORTED_FEATURES ((1ULL << VIRTIO_NET_F_MRG_RXBUF) | \ - (1ULL << VIRTIO_NET_F_CTRL_VQ) | \ - (1ULL << VIRTIO_NET_F_CTRL_RX) | \ - (1ULL << VIRTIO_NET_F_GUEST_ANNOUNCE) | \ - (VHOST_SUPPORTS_MQ) | \ - (1ULL << VIRTIO_F_VERSION_1) | \ - (1ULL << VHOST_F_LOG_ALL) | \ - (1ULL << VHOST_USER_F_PROTOCOL_FEATURES) | \ - (1ULL << VIRTIO_NET_F_HOST_TSO4) | \ - (1ULL << VIRTIO_NET_F_HOST_TSO6) | \ - (1ULL << VIRTIO_NET_F_CSUM) | \ - (1ULL << VIRTIO_NET_F_GUEST_CSUM) | \ - (1ULL << VIRTIO_NET_F_GUEST_TSO4) | \ - (1ULL << VIRTIO_NET_F_GUEST_TSO6)) - -static uint64_t VHOST_FEATURES = VHOST_SUPPORTED_FEATURES; - - -/* - * Converts QEMU virtual address to Vhost virtual address. This function is - * used to convert the ring addresses to our address space. - */ -static uint64_t -qva_to_vva(struct virtio_net *dev, uint64_t qemu_va) -{ - struct virtio_memory_regions *region; - uint64_t vhost_va = 0; - uint32_t regionidx = 0; - - /* Find the region where the address lives. */ - for (regionidx = 0; regionidx < dev->mem->nregions; regionidx++) { - region = &dev->mem->regions[regionidx]; - if ((qemu_va >= region->userspace_address) && - (qemu_va <= region->userspace_address + - region->memory_size)) { - vhost_va = qemu_va + region->guest_phys_address + - region->address_offset - - region->userspace_address; - break; - } - } - return vhost_va; -} - -struct virtio_net * -get_device(int vid) -{ - struct virtio_net *dev = vhost_devices[vid]; - - if (unlikely(!dev)) { - RTE_LOG(ERR, VHOST_CONFIG, - "(%d) device not found.\n", vid); - } - - return dev; -} - -static void -cleanup_vq(struct vhost_virtqueue *vq, int destroy) -{ - if ((vq->callfd >= 0) && (destroy != 0)) - close(vq->callfd); - if (vq->kickfd >= 0) - close(vq->kickfd); -} - -/* - * Unmap any memory, close any file descriptors and - * free any memory owned by a device. - */ -static void -cleanup_device(struct virtio_net *dev, int destroy) -{ - uint32_t i; - - vhost_backend_cleanup(dev); - - for (i = 0; i < dev->virt_qp_nb; i++) { - cleanup_vq(dev->virtqueue[i * VIRTIO_QNUM + VIRTIO_RXQ], destroy); - cleanup_vq(dev->virtqueue[i * VIRTIO_QNUM + VIRTIO_TXQ], destroy); - } -} - -/* - * Release virtqueues and device memory. - */ -static void -free_device(struct virtio_net *dev) -{ - uint32_t i; - - for (i = 0; i < dev->virt_qp_nb; i++) - rte_free(dev->virtqueue[i * VIRTIO_QNUM]); - - rte_free(dev); -} - -static void -init_vring_queue(struct vhost_virtqueue *vq, int qp_idx) -{ - memset(vq, 0, sizeof(struct vhost_virtqueue)); - - vq->kickfd = VIRTIO_UNINITIALIZED_EVENTFD; - vq->callfd = VIRTIO_UNINITIALIZED_EVENTFD; - - /* Backends are set to -1 indicating an inactive device. */ - vq->backend = -1; - - /* always set the default vq pair to enabled */ - if (qp_idx == 0) - vq->enabled = 1; -} - -static void -init_vring_queue_pair(struct virtio_net *dev, uint32_t qp_idx) -{ - uint32_t base_idx = qp_idx * VIRTIO_QNUM; - - init_vring_queue(dev->virtqueue[base_idx + VIRTIO_RXQ], qp_idx); - init_vring_queue(dev->virtqueue[base_idx + VIRTIO_TXQ], qp_idx); -} - -static void -reset_vring_queue(struct vhost_virtqueue *vq, int qp_idx) -{ - int callfd; - - callfd = vq->callfd; - init_vring_queue(vq, qp_idx); - vq->callfd = callfd; -} - -static void -reset_vring_queue_pair(struct virtio_net *dev, uint32_t qp_idx) -{ - uint32_t base_idx = qp_idx * VIRTIO_QNUM; - - reset_vring_queue(dev->virtqueue[base_idx + VIRTIO_RXQ], qp_idx); - reset_vring_queue(dev->virtqueue[base_idx + VIRTIO_TXQ], qp_idx); -} - -static int -alloc_vring_queue_pair(struct virtio_net *dev, uint32_t qp_idx) -{ - struct vhost_virtqueue *virtqueue = NULL; - uint32_t virt_rx_q_idx = qp_idx * VIRTIO_QNUM + VIRTIO_RXQ; - uint32_t virt_tx_q_idx = qp_idx * VIRTIO_QNUM + VIRTIO_TXQ; - - virtqueue = rte_malloc(NULL, - sizeof(struct vhost_virtqueue) * VIRTIO_QNUM, 0); - if (virtqueue == NULL) { - RTE_LOG(ERR, VHOST_CONFIG, - "Failed to allocate memory for virt qp:%d.\n", qp_idx); - return -1; - } - - dev->virtqueue[virt_rx_q_idx] = virtqueue; - dev->virtqueue[virt_tx_q_idx] = virtqueue + VIRTIO_TXQ; - - init_vring_queue_pair(dev, qp_idx); - - dev->virt_qp_nb += 1; - - return 0; -} - -/* - * Reset some variables in device structure, while keeping few - * others untouched, such as vid, ifname, virt_qp_nb: they - * should be same unless the device is removed. - */ -static void -reset_device(struct virtio_net *dev) -{ - uint32_t i; - - dev->features = 0; - dev->protocol_features = 0; - dev->flags = 0; - - for (i = 0; i < dev->virt_qp_nb; i++) - reset_vring_queue_pair(dev, i); -} - -/* - * Function is called from the CUSE open function. The device structure is - * initialised and a new entry is added to the device configuration linked - * list. - */ -int -vhost_new_device(void) -{ - struct virtio_net *dev; - int i; - - dev = rte_zmalloc(NULL, sizeof(struct virtio_net), 0); - if (dev == NULL) { - RTE_LOG(ERR, VHOST_CONFIG, - "Failed to allocate memory for new dev.\n"); - return -1; - } - - for (i = 0; i < MAX_VHOST_DEVICE; i++) { - if (vhost_devices[i] == NULL) - break; - } - if (i == MAX_VHOST_DEVICE) { - RTE_LOG(ERR, VHOST_CONFIG, - "Failed to find a free slot for new device.\n"); - return -1; - } - - vhost_devices[i] = dev; - dev->vid = i; - - return i; -} - -/* - * Function is called from the CUSE release function. This function will - * cleanup the device and remove it from device configuration linked list. - */ -void -vhost_destroy_device(int vid) -{ - struct virtio_net *dev = get_device(vid); - - if (dev == NULL) - return; - - if (dev->flags & VIRTIO_DEV_RUNNING) { - dev->flags &= ~VIRTIO_DEV_RUNNING; - notify_ops->destroy_device(vid); - } - - cleanup_device(dev, 1); - free_device(dev); - - vhost_devices[vid] = NULL; -} - -void -vhost_set_ifname(int vid, const char *if_name, unsigned int if_len) -{ - struct virtio_net *dev; - unsigned int len; - - dev = get_device(vid); - if (dev == NULL) - return; - - len = if_len > sizeof(dev->ifname) ? - sizeof(dev->ifname) : if_len; - - strncpy(dev->ifname, if_name, len); - dev->ifname[sizeof(dev->ifname) - 1] = '\0'; -} - - -/* - * Called from CUSE IOCTL: VHOST_SET_OWNER - * This function just returns success at the moment unless - * the device hasn't been initialised. - */ -int -vhost_set_owner(int vid) -{ - struct virtio_net *dev; - - dev = get_device(vid); - if (dev == NULL) - return -1; - - return 0; -} - -/* - * Called from CUSE IOCTL: VHOST_RESET_OWNER - */ -int -vhost_reset_owner(int vid) -{ - struct virtio_net *dev; - - dev = get_device(vid); - if (dev == NULL) - return -1; - - if (dev->flags & VIRTIO_DEV_RUNNING) { - dev->flags &= ~VIRTIO_DEV_RUNNING; - notify_ops->destroy_device(vid); - } - - cleanup_device(dev, 0); - reset_device(dev); - return 0; -} - -/* - * Called from CUSE IOCTL: VHOST_GET_FEATURES - * The features that we support are requested. - */ -int -vhost_get_features(int vid, uint64_t *pu) -{ - struct virtio_net *dev; - - dev = get_device(vid); - if (dev == NULL) - return -1; - - /* Send our supported features. */ - *pu = VHOST_FEATURES; - return 0; -} - -/* - * Called from CUSE IOCTL: VHOST_SET_FEATURES - * We receive the negotiated features supported by us and the virtio device. - */ -int -vhost_set_features(int vid, uint64_t *pu) -{ - struct virtio_net *dev; - - dev = get_device(vid); - if (dev == NULL) - return -1; - if (*pu & ~VHOST_FEATURES) - return -1; - - dev->features = *pu; - if (dev->features & - ((1 << VIRTIO_NET_F_MRG_RXBUF) | (1ULL << VIRTIO_F_VERSION_1))) { - dev->vhost_hlen = sizeof(struct virtio_net_hdr_mrg_rxbuf); - } else { - dev->vhost_hlen = sizeof(struct virtio_net_hdr); - } - LOG_DEBUG(VHOST_CONFIG, - "(%d) mergeable RX buffers %s, virtio 1 %s\n", - dev->vid, - (dev->features & (1 << VIRTIO_NET_F_MRG_RXBUF)) ? "on" : "off", - (dev->features & (1ULL << VIRTIO_F_VERSION_1)) ? "on" : "off"); - - return 0; -} - -/* - * Called from CUSE IOCTL: VHOST_SET_VRING_NUM - * The virtio device sends us the size of the descriptor ring. - */ -int -vhost_set_vring_num(int vid, struct vhost_vring_state *state) -{ - struct virtio_net *dev; - - dev = get_device(vid); - if (dev == NULL) - return -1; - - /* State->index refers to the queue index. The txq is 1, rxq is 0. */ - dev->virtqueue[state->index]->size = state->num; - - return 0; -} - -/* - * Reallocate virtio_dev and vhost_virtqueue data structure to make them on the - * same numa node as the memory of vring descriptor. - */ -#ifdef RTE_LIBRTE_VHOST_NUMA -static struct virtio_net* -numa_realloc(struct virtio_net *dev, int index) -{ - int oldnode, newnode; - struct virtio_net *old_dev; - struct vhost_virtqueue *old_vq, *vq; - int ret; - - /* - * vq is allocated on pairs, we should try to do realloc - * on first queue of one queue pair only. - */ - if (index % VIRTIO_QNUM != 0) - return dev; - - old_dev = dev; - vq = old_vq = dev->virtqueue[index]; - - ret = get_mempolicy(&newnode, NULL, 0, old_vq->desc, - MPOL_F_NODE | MPOL_F_ADDR); - - /* check if we need to reallocate vq */ - ret |= get_mempolicy(&oldnode, NULL, 0, old_vq, - MPOL_F_NODE | MPOL_F_ADDR); - if (ret) { - RTE_LOG(ERR, VHOST_CONFIG, - "Unable to get vq numa information.\n"); - return dev; - } - if (oldnode != newnode) { - RTE_LOG(INFO, VHOST_CONFIG, - "reallocate vq from %d to %d node\n", oldnode, newnode); - vq = rte_malloc_socket(NULL, sizeof(*vq) * VIRTIO_QNUM, 0, - newnode); - if (!vq) - return dev; - - memcpy(vq, old_vq, sizeof(*vq) * VIRTIO_QNUM); - rte_free(old_vq); - } - - /* check if we need to reallocate dev */ - ret = get_mempolicy(&oldnode, NULL, 0, old_dev, - MPOL_F_NODE | MPOL_F_ADDR); - if (ret) { - RTE_LOG(ERR, VHOST_CONFIG, - "Unable to get dev numa information.\n"); - goto out; - } - if (oldnode != newnode) { - RTE_LOG(INFO, VHOST_CONFIG, - "reallocate dev from %d to %d node\n", - oldnode, newnode); - dev = rte_malloc_socket(NULL, sizeof(*dev), 0, newnode); - if (!dev) { - dev = old_dev; - goto out; - } - - memcpy(dev, old_dev, sizeof(*dev)); - rte_free(old_dev); - } - -out: - dev->virtqueue[index] = vq; - dev->virtqueue[index + 1] = vq + 1; - vhost_devices[dev->vid] = dev; - - return dev; -} -#else -static struct virtio_net* -numa_realloc(struct virtio_net *dev, int index __rte_unused) -{ - return dev; -} -#endif - -/* - * Called from CUSE IOCTL: VHOST_SET_VRING_ADDR - * The virtio device sends us the desc, used and avail ring addresses. - * This function then converts these to our address space. - */ -int -vhost_set_vring_addr(int vid, struct vhost_vring_addr *addr) -{ - struct virtio_net *dev; - struct vhost_virtqueue *vq; - - dev = get_device(vid); - if ((dev == NULL) || (dev->mem == NULL)) - return -1; - - /* addr->index refers to the queue index. The txq 1, rxq is 0. */ - vq = dev->virtqueue[addr->index]; - - /* The addresses are converted from QEMU virtual to Vhost virtual. */ - vq->desc = (struct vring_desc *)(uintptr_t)qva_to_vva(dev, - addr->desc_user_addr); - if (vq->desc == 0) { - RTE_LOG(ERR, VHOST_CONFIG, - "(%d) failed to find desc ring address.\n", - dev->vid); - return -1; - } - - dev = numa_realloc(dev, addr->index); - vq = dev->virtqueue[addr->index]; - - vq->avail = (struct vring_avail *)(uintptr_t)qva_to_vva(dev, - addr->avail_user_addr); - if (vq->avail == 0) { - RTE_LOG(ERR, VHOST_CONFIG, - "(%d) failed to find avail ring address.\n", - dev->vid); - return -1; - } - - vq->used = (struct vring_used *)(uintptr_t)qva_to_vva(dev, - addr->used_user_addr); - if (vq->used == 0) { - RTE_LOG(ERR, VHOST_CONFIG, - "(%d) failed to find used ring address.\n", - dev->vid); - return -1; - } - - if (vq->last_used_idx != vq->used->idx) { - RTE_LOG(WARNING, VHOST_CONFIG, - "last_used_idx (%u) and vq->used->idx (%u) mismatches; " - "some packets maybe resent for Tx and dropped for Rx\n", - vq->last_used_idx, vq->used->idx); - vq->last_used_idx = vq->used->idx; - } - - vq->log_guest_addr = addr->log_guest_addr; - - LOG_DEBUG(VHOST_CONFIG, "(%d) mapped address desc: %p\n", - dev->vid, vq->desc); - LOG_DEBUG(VHOST_CONFIG, "(%d) mapped address avail: %p\n", - dev->vid, vq->avail); - LOG_DEBUG(VHOST_CONFIG, "(%d) mapped address used: %p\n", - dev->vid, vq->used); - LOG_DEBUG(VHOST_CONFIG, "(%d) log_guest_addr: %" PRIx64 "\n", - dev->vid, vq->log_guest_addr); - - return 0; -} - -/* - * Called from CUSE IOCTL: VHOST_SET_VRING_BASE - * The virtio device sends us the available ring last used index. - */ -int -vhost_set_vring_base(int vid, struct vhost_vring_state *state) -{ - struct virtio_net *dev; - - dev = get_device(vid); - if (dev == NULL) - return -1; - - /* State->index refers to the queue index. The txq is 1, rxq is 0. */ - dev->virtqueue[state->index]->last_used_idx = state->num; - - return 0; -} - -/* - * Called from CUSE IOCTL: VHOST_GET_VRING_BASE - * We send the virtio device our available ring last used index. - */ -int -vhost_get_vring_base(int vid, uint32_t index, - struct vhost_vring_state *state) -{ - struct virtio_net *dev; - - dev = get_device(vid); - if (dev == NULL) - return -1; - - state->index = index; - /* State->index refers to the queue index. The txq is 1, rxq is 0. */ - state->num = dev->virtqueue[state->index]->last_used_idx; - - return 0; -} - - -/* - * Called from CUSE IOCTL: VHOST_SET_VRING_CALL - * The virtio device sends an eventfd to interrupt the guest. This fd gets - * copied into our process space. - */ -int -vhost_set_vring_call(int vid, struct vhost_vring_file *file) -{ - struct virtio_net *dev; - struct vhost_virtqueue *vq; - uint32_t cur_qp_idx = file->index / VIRTIO_QNUM; - - dev = get_device(vid); - if (dev == NULL) - return -1; - - /* - * FIXME: VHOST_SET_VRING_CALL is the first per-vring message - * we get, so we do vring queue pair allocation here. - */ - if (cur_qp_idx + 1 > dev->virt_qp_nb) { - if (alloc_vring_queue_pair(dev, cur_qp_idx) < 0) - return -1; - } - - /* file->index refers to the queue index. The txq is 1, rxq is 0. */ - vq = dev->virtqueue[file->index]; - assert(vq != NULL); - - if (vq->callfd >= 0) - close(vq->callfd); - - vq->callfd = file->fd; - - return 0; -} - -/* - * Called from CUSE IOCTL: VHOST_SET_VRING_KICK - * The virtio device sends an eventfd that it can use to notify us. - * This fd gets copied into our process space. - */ -int -vhost_set_vring_kick(int vid, struct vhost_vring_file *file) -{ - struct virtio_net *dev; - struct vhost_virtqueue *vq; - - dev = get_device(vid); - if (dev == NULL) - return -1; - - /* file->index refers to the queue index. The txq is 1, rxq is 0. */ - vq = dev->virtqueue[file->index]; - - if (vq->kickfd >= 0) - close(vq->kickfd); - - vq->kickfd = file->fd; - - return 0; -} - -/* - * Called from CUSE IOCTL: VHOST_NET_SET_BACKEND - * To complete device initialisation when the virtio driver is loaded, - * we are provided with a valid fd for a tap device (not used by us). - * If this happens then we can add the device to a data core. - * When the virtio driver is removed we get fd=-1. - * At that point we remove the device from the data core. - * The device will still exist in the device configuration linked list. - */ -int -vhost_set_backend(int vid, struct vhost_vring_file *file) -{ - struct virtio_net *dev; - - dev = get_device(vid); - if (dev == NULL) - return -1; - - /* file->index refers to the queue index. The txq is 1, rxq is 0. */ - dev->virtqueue[file->index]->backend = file->fd; - - /* - * If the device isn't already running and both backend fds are set, - * we add the device. - */ - if (!(dev->flags & VIRTIO_DEV_RUNNING)) { - if (dev->virtqueue[VIRTIO_TXQ]->backend != VIRTIO_DEV_STOPPED && - dev->virtqueue[VIRTIO_RXQ]->backend != VIRTIO_DEV_STOPPED) { - if (notify_ops->new_device(vid) < 0) - return -1; - dev->flags |= VIRTIO_DEV_RUNNING; - } - } else if (file->fd == VIRTIO_DEV_STOPPED) { - dev->flags &= ~VIRTIO_DEV_RUNNING; - notify_ops->destroy_device(vid); - } - - return 0; -} - -int -rte_vhost_get_numa_node(int vid) -{ -#ifdef RTE_LIBRTE_VHOST_NUMA - struct virtio_net *dev = get_device(vid); - int numa_node; - int ret; - - if (dev == NULL) - return -1; - - ret = get_mempolicy(&numa_node, NULL, 0, dev, - MPOL_F_NODE | MPOL_F_ADDR); - if (ret < 0) { - RTE_LOG(ERR, VHOST_CONFIG, - "(%d) failed to query numa node: %d\n", vid, ret); - return -1; - } - - return numa_node; -#else - RTE_SET_USED(vid); - return -1; -#endif -} - -uint32_t -rte_vhost_get_queue_num(int vid) -{ - struct virtio_net *dev = get_device(vid); - - if (dev == NULL) - return 0; - - return dev->virt_qp_nb; -} - -int -rte_vhost_get_ifname(int vid, char *buf, size_t len) -{ - struct virtio_net *dev = get_device(vid); - - if (dev == NULL) - return -1; - - len = RTE_MIN(len, sizeof(dev->ifname)); - - strncpy(buf, dev->ifname, len); - buf[len - 1] = '\0'; - - return 0; -} - -uint16_t -rte_vhost_avail_entries(int vid, uint16_t queue_id) -{ - struct virtio_net *dev; - struct vhost_virtqueue *vq; - - dev = get_device(vid); - if (!dev) - return 0; - - vq = dev->virtqueue[queue_id]; - if (!vq->enabled) - return 0; - - return *(volatile uint16_t *)&vq->avail->idx - vq->last_used_idx; -} - -int -rte_vhost_enable_guest_notification(int vid, uint16_t queue_id, int enable) -{ - struct virtio_net *dev = get_device(vid); - - if (dev == NULL) - return -1; - - if (enable) { - RTE_LOG(ERR, VHOST_CONFIG, - "guest notification isn't supported.\n"); - return -1; - } - - dev->virtqueue[queue_id]->used->flags = VRING_USED_F_NO_NOTIFY; - return 0; -} - -uint64_t rte_vhost_feature_get(void) -{ - return VHOST_FEATURES; -} - -int rte_vhost_feature_disable(uint64_t feature_mask) -{ - VHOST_FEATURES = VHOST_FEATURES & ~feature_mask; - return 0; -} - -int rte_vhost_feature_enable(uint64_t feature_mask) -{ - if ((feature_mask & VHOST_SUPPORTED_FEATURES) == feature_mask) { - VHOST_FEATURES = VHOST_FEATURES | feature_mask; - return 0; - } - return -1; -} - -/* - * Register ops so that we can add/remove device to data core. - */ -int -rte_vhost_driver_callback_register(struct virtio_net_device_ops const * const ops) -{ - notify_ops = ops; - - return 0; -} diff --git a/dpdk/lib/librte_vhost/virtio_net.c b/dpdk/lib/librte_vhost/virtio_net.c new file mode 100644 index 00000000..a013c07b --- /dev/null +++ b/dpdk/lib/librte_vhost/virtio_net.c @@ -0,0 +1,1711 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "iotlb.h" +#include "vhost.h" + +#define MAX_PKT_BURST 32 + +#define MAX_BATCH_LEN 256 + +static bool +is_valid_virt_queue_idx(uint32_t idx, int is_tx, uint32_t nr_vring) +{ + return (is_tx ^ (idx & 1)) == 0 && idx < nr_vring; +} + +static __rte_always_inline struct vring_desc * +alloc_copy_ind_table(struct virtio_net *dev, struct vhost_virtqueue *vq, + struct vring_desc *desc) +{ + struct vring_desc *idesc; + uint64_t src, dst; + uint64_t len, remain = desc->len; + uint64_t desc_addr = desc->addr; + + idesc = rte_malloc(__func__, desc->len, 0); + if (unlikely(!idesc)) + return 0; + + dst = (uint64_t)(uintptr_t)idesc; + + while (remain) { + len = remain; + src = vhost_iova_to_vva(dev, vq, desc_addr, &len, + VHOST_ACCESS_RO); + if (unlikely(!src || !len)) { + rte_free(idesc); + return 0; + } + + rte_memcpy((void *)(uintptr_t)dst, (void *)(uintptr_t)src, len); + + remain -= len; + dst += len; + desc_addr += len; + } + + return idesc; +} + +static __rte_always_inline void +free_ind_table(struct vring_desc *idesc) +{ + rte_free(idesc); +} + +static __rte_always_inline void +do_flush_shadow_used_ring(struct virtio_net *dev, struct vhost_virtqueue *vq, + uint16_t to, uint16_t from, uint16_t size) +{ + rte_memcpy(&vq->used->ring[to], + &vq->shadow_used_ring[from], + size * sizeof(struct vring_used_elem)); + vhost_log_used_vring(dev, vq, + offsetof(struct vring_used, ring[to]), + size * sizeof(struct vring_used_elem)); +} + +static __rte_always_inline void +flush_shadow_used_ring(struct virtio_net *dev, struct vhost_virtqueue *vq) +{ + uint16_t used_idx = vq->last_used_idx & (vq->size - 1); + + if (used_idx + vq->shadow_used_idx <= vq->size) { + do_flush_shadow_used_ring(dev, vq, used_idx, 0, + vq->shadow_used_idx); + } else { + uint16_t size; + + /* update used ring interval [used_idx, vq->size] */ + size = vq->size - used_idx; + do_flush_shadow_used_ring(dev, vq, used_idx, 0, size); + + /* update the left half used ring interval [0, left_size] */ + do_flush_shadow_used_ring(dev, vq, 0, size, + vq->shadow_used_idx - size); + } + vq->last_used_idx += vq->shadow_used_idx; + + rte_smp_wmb(); + + *(volatile uint16_t *)&vq->used->idx += vq->shadow_used_idx; + vhost_log_used_vring(dev, vq, offsetof(struct vring_used, idx), + sizeof(vq->used->idx)); +} + +static __rte_always_inline void +update_shadow_used_ring(struct vhost_virtqueue *vq, + uint16_t desc_idx, uint16_t len) +{ + uint16_t i = vq->shadow_used_idx++; + + vq->shadow_used_ring[i].id = desc_idx; + vq->shadow_used_ring[i].len = len; +} + +static inline void +do_data_copy_enqueue(struct virtio_net *dev, struct vhost_virtqueue *vq) +{ + struct batch_copy_elem *elem = vq->batch_copy_elems; + uint16_t count = vq->batch_copy_nb_elems; + int i; + + for (i = 0; i < count; i++) { + rte_memcpy(elem[i].dst, elem[i].src, elem[i].len); + vhost_log_write(dev, elem[i].log_addr, elem[i].len); + PRINT_PACKET(dev, (uintptr_t)elem[i].dst, elem[i].len, 0); + } +} + +static inline void +do_data_copy_dequeue(struct vhost_virtqueue *vq) +{ + struct batch_copy_elem *elem = vq->batch_copy_elems; + uint16_t count = vq->batch_copy_nb_elems; + int i; + + for (i = 0; i < count; i++) + rte_memcpy(elem[i].dst, elem[i].src, elem[i].len); +} + +/* avoid write operation when necessary, to lessen cache issues */ +#define ASSIGN_UNLESS_EQUAL(var, val) do { \ + if ((var) != (val)) \ + (var) = (val); \ +} while (0) + +static void +virtio_enqueue_offload(struct rte_mbuf *m_buf, struct virtio_net_hdr *net_hdr) +{ + uint64_t csum_l4 = m_buf->ol_flags & PKT_TX_L4_MASK; + + if (m_buf->ol_flags & PKT_TX_TCP_SEG) + csum_l4 |= PKT_TX_TCP_CKSUM; + + if (csum_l4) { + net_hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM; + net_hdr->csum_start = m_buf->l2_len + m_buf->l3_len; + + switch (csum_l4) { + case PKT_TX_TCP_CKSUM: + net_hdr->csum_offset = (offsetof(struct tcp_hdr, + cksum)); + break; + case PKT_TX_UDP_CKSUM: + net_hdr->csum_offset = (offsetof(struct udp_hdr, + dgram_cksum)); + break; + case PKT_TX_SCTP_CKSUM: + net_hdr->csum_offset = (offsetof(struct sctp_hdr, + cksum)); + break; + } + } else { + ASSIGN_UNLESS_EQUAL(net_hdr->csum_start, 0); + ASSIGN_UNLESS_EQUAL(net_hdr->csum_offset, 0); + ASSIGN_UNLESS_EQUAL(net_hdr->flags, 0); + } + + /* IP cksum verification cannot be bypassed, then calculate here */ + if (m_buf->ol_flags & PKT_TX_IP_CKSUM) { + struct ipv4_hdr *ipv4_hdr; + + ipv4_hdr = rte_pktmbuf_mtod_offset(m_buf, struct ipv4_hdr *, + m_buf->l2_len); + ipv4_hdr->hdr_checksum = rte_ipv4_cksum(ipv4_hdr); + } + + if (m_buf->ol_flags & PKT_TX_TCP_SEG) { + if (m_buf->ol_flags & PKT_TX_IPV4) + net_hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV4; + else + net_hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV6; + net_hdr->gso_size = m_buf->tso_segsz; + net_hdr->hdr_len = m_buf->l2_len + m_buf->l3_len + + m_buf->l4_len; + } else { + ASSIGN_UNLESS_EQUAL(net_hdr->gso_type, 0); + ASSIGN_UNLESS_EQUAL(net_hdr->gso_size, 0); + ASSIGN_UNLESS_EQUAL(net_hdr->hdr_len, 0); + } +} + +static __rte_always_inline int +copy_mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq, + struct vring_desc *descs, struct rte_mbuf *m, + uint16_t desc_idx, uint32_t size) +{ + uint32_t desc_avail, desc_offset; + uint32_t mbuf_avail, mbuf_offset; + uint32_t cpy_len; + uint64_t desc_chunck_len; + struct vring_desc *desc; + uint64_t desc_addr, desc_gaddr; + /* A counter to avoid desc dead loop chain */ + uint16_t nr_desc = 1; + struct batch_copy_elem *batch_copy = vq->batch_copy_elems; + uint16_t copy_nb = vq->batch_copy_nb_elems; + int error = 0; + + desc = &descs[desc_idx]; + desc_chunck_len = desc->len; + desc_gaddr = desc->addr; + desc_addr = vhost_iova_to_vva(dev, vq, desc_gaddr, + &desc_chunck_len, VHOST_ACCESS_RW); + /* + * Checking of 'desc_addr' placed outside of 'unlikely' macro to avoid + * performance issue with some versions of gcc (4.8.4 and 5.3.0) which + * otherwise stores offset on the stack instead of in a register. + */ + if (unlikely(desc->len < dev->vhost_hlen) || !desc_addr) { + error = -1; + goto out; + } + + rte_prefetch0((void *)(uintptr_t)desc_addr); + + if (likely(desc_chunck_len >= dev->vhost_hlen)) { + virtio_enqueue_offload(m, + (struct virtio_net_hdr *)(uintptr_t)desc_addr); + PRINT_PACKET(dev, (uintptr_t)desc_addr, dev->vhost_hlen, 0); + vhost_log_write(dev, desc_gaddr, dev->vhost_hlen); + } else { + struct virtio_net_hdr vnet_hdr; + uint64_t remain = dev->vhost_hlen; + uint64_t len; + uint64_t src = (uint64_t)(uintptr_t)&vnet_hdr, dst; + uint64_t guest_addr = desc_gaddr; + + virtio_enqueue_offload(m, &vnet_hdr); + + while (remain) { + len = remain; + dst = vhost_iova_to_vva(dev, vq, guest_addr, + &len, VHOST_ACCESS_RW); + if (unlikely(!dst || !len)) { + error = -1; + goto out; + } + + rte_memcpy((void *)(uintptr_t)dst, + (void *)(uintptr_t)src, len); + + PRINT_PACKET(dev, (uintptr_t)dst, len, 0); + vhost_log_write(dev, guest_addr, len); + remain -= len; + guest_addr += len; + dst += len; + } + } + + desc_avail = desc->len - dev->vhost_hlen; + if (unlikely(desc_chunck_len < dev->vhost_hlen)) { + desc_chunck_len = desc_avail; + desc_gaddr = desc->addr + dev->vhost_hlen; + desc_addr = vhost_iova_to_vva(dev, + vq, desc_gaddr, + &desc_chunck_len, + VHOST_ACCESS_RW); + if (unlikely(!desc_addr)) { + error = -1; + goto out; + } + + desc_offset = 0; + } else { + desc_offset = dev->vhost_hlen; + desc_chunck_len -= dev->vhost_hlen; + } + + mbuf_avail = rte_pktmbuf_data_len(m); + mbuf_offset = 0; + while (mbuf_avail != 0 || m->next != NULL) { + /* done with current mbuf, fetch next */ + if (mbuf_avail == 0) { + m = m->next; + + mbuf_offset = 0; + mbuf_avail = rte_pktmbuf_data_len(m); + } + + /* done with current desc buf, fetch next */ + if (desc_avail == 0) { + if ((desc->flags & VRING_DESC_F_NEXT) == 0) { + /* Room in vring buffer is not enough */ + error = -1; + goto out; + } + if (unlikely(desc->next >= size || ++nr_desc > size)) { + error = -1; + goto out; + } + + desc = &descs[desc->next]; + desc_chunck_len = desc->len; + desc_gaddr = desc->addr; + desc_addr = vhost_iova_to_vva(dev, vq, desc_gaddr, + &desc_chunck_len, + VHOST_ACCESS_RW); + if (unlikely(!desc_addr)) { + error = -1; + goto out; + } + + desc_offset = 0; + desc_avail = desc->len; + } else if (unlikely(desc_chunck_len == 0)) { + desc_chunck_len = desc_avail; + desc_gaddr += desc_offset; + desc_addr = vhost_iova_to_vva(dev, + vq, desc_gaddr, + &desc_chunck_len, VHOST_ACCESS_RW); + if (unlikely(!desc_addr)) { + error = -1; + goto out; + } + desc_offset = 0; + } + + cpy_len = RTE_MIN(desc_chunck_len, mbuf_avail); + if (likely(cpy_len > MAX_BATCH_LEN || copy_nb >= vq->size)) { + rte_memcpy((void *)((uintptr_t)(desc_addr + + desc_offset)), + rte_pktmbuf_mtod_offset(m, void *, mbuf_offset), + cpy_len); + vhost_log_write(dev, desc_gaddr + desc_offset, cpy_len); + PRINT_PACKET(dev, (uintptr_t)(desc_addr + desc_offset), + cpy_len, 0); + } else { + batch_copy[copy_nb].dst = + (void *)((uintptr_t)(desc_addr + desc_offset)); + batch_copy[copy_nb].src = + rte_pktmbuf_mtod_offset(m, void *, mbuf_offset); + batch_copy[copy_nb].log_addr = desc_gaddr + desc_offset; + batch_copy[copy_nb].len = cpy_len; + copy_nb++; + } + + mbuf_avail -= cpy_len; + mbuf_offset += cpy_len; + desc_avail -= cpy_len; + desc_offset += cpy_len; + desc_chunck_len -= cpy_len; + } + +out: + vq->batch_copy_nb_elems = copy_nb; + + return error; +} + +/** + * This function adds buffers to the virtio devices RX virtqueue. Buffers can + * be received from the physical port or from another virtio device. A packet + * count is returned to indicate the number of packets that are successfully + * added to the RX queue. This function works when the mbuf is scattered, but + * it doesn't support the mergeable feature. + */ +static __rte_always_inline uint32_t +virtio_dev_rx(struct virtio_net *dev, uint16_t queue_id, + struct rte_mbuf **pkts, uint32_t count) +{ + struct vhost_virtqueue *vq; + uint16_t avail_idx, free_entries, start_idx; + uint16_t desc_indexes[MAX_PKT_BURST]; + struct vring_desc *descs; + uint16_t used_idx; + uint32_t i, sz; + + LOG_DEBUG(VHOST_DATA, "(%d) %s\n", dev->vid, __func__); + if (unlikely(!is_valid_virt_queue_idx(queue_id, 0, dev->nr_vring))) { + RTE_LOG(ERR, VHOST_DATA, "(%d) %s: invalid virtqueue idx %d.\n", + dev->vid, __func__, queue_id); + return 0; + } + + vq = dev->virtqueue[queue_id]; + + rte_spinlock_lock(&vq->access_lock); + + if (unlikely(vq->enabled == 0)) + goto out_access_unlock; + + if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)) + vhost_user_iotlb_rd_lock(vq); + + if (unlikely(vq->access_ok == 0)) { + if (unlikely(vring_translate(dev, vq) < 0)) { + count = 0; + goto out; + } + } + + avail_idx = *((volatile uint16_t *)&vq->avail->idx); + start_idx = vq->last_used_idx; + free_entries = avail_idx - start_idx; + count = RTE_MIN(count, free_entries); + count = RTE_MIN(count, (uint32_t)MAX_PKT_BURST); + if (count == 0) + goto out; + + LOG_DEBUG(VHOST_DATA, "(%d) start_idx %d | end_idx %d\n", + dev->vid, start_idx, start_idx + count); + + vq->batch_copy_nb_elems = 0; + + /* Retrieve all of the desc indexes first to avoid caching issues. */ + rte_prefetch0(&vq->avail->ring[start_idx & (vq->size - 1)]); + for (i = 0; i < count; i++) { + used_idx = (start_idx + i) & (vq->size - 1); + desc_indexes[i] = vq->avail->ring[used_idx]; + vq->used->ring[used_idx].id = desc_indexes[i]; + vq->used->ring[used_idx].len = pkts[i]->pkt_len + + dev->vhost_hlen; + vhost_log_used_vring(dev, vq, + offsetof(struct vring_used, ring[used_idx]), + sizeof(vq->used->ring[used_idx])); + } + + rte_prefetch0(&vq->desc[desc_indexes[0]]); + for (i = 0; i < count; i++) { + struct vring_desc *idesc = NULL; + uint16_t desc_idx = desc_indexes[i]; + int err; + + if (vq->desc[desc_idx].flags & VRING_DESC_F_INDIRECT) { + uint64_t dlen = vq->desc[desc_idx].len; + descs = (struct vring_desc *)(uintptr_t) + vhost_iova_to_vva(dev, + vq, vq->desc[desc_idx].addr, + &dlen, VHOST_ACCESS_RO); + if (unlikely(!descs)) { + count = i; + break; + } + + if (unlikely(dlen < vq->desc[desc_idx].len)) { + /* + * The indirect desc table is not contiguous + * in process VA space, we have to copy it. + */ + idesc = alloc_copy_ind_table(dev, vq, + &vq->desc[desc_idx]); + if (unlikely(!idesc)) + break; + + descs = idesc; + } + + desc_idx = 0; + sz = vq->desc[desc_idx].len / sizeof(*descs); + } else { + descs = vq->desc; + sz = vq->size; + } + + err = copy_mbuf_to_desc(dev, vq, descs, pkts[i], desc_idx, sz); + if (unlikely(err)) { + count = i; + free_ind_table(idesc); + break; + } + + if (i + 1 < count) + rte_prefetch0(&vq->desc[desc_indexes[i+1]]); + + if (unlikely(!!idesc)) + free_ind_table(idesc); + } + + do_data_copy_enqueue(dev, vq); + + rte_smp_wmb(); + + *(volatile uint16_t *)&vq->used->idx += count; + vq->last_used_idx += count; + vhost_log_used_vring(dev, vq, + offsetof(struct vring_used, idx), + sizeof(vq->used->idx)); + + /* flush used->idx update before we read avail->flags. */ + rte_mb(); + + /* Kick the guest if necessary. */ + if (!(vq->avail->flags & VRING_AVAIL_F_NO_INTERRUPT) + && (vq->callfd >= 0)) + eventfd_write(vq->callfd, (eventfd_t)1); +out: + if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)) + vhost_user_iotlb_rd_unlock(vq); + +out_access_unlock: + rte_spinlock_unlock(&vq->access_lock); + + return count; +} + +static __rte_always_inline int +fill_vec_buf(struct virtio_net *dev, struct vhost_virtqueue *vq, + uint32_t avail_idx, uint32_t *vec_idx, + struct buf_vector *buf_vec, uint16_t *desc_chain_head, + uint16_t *desc_chain_len) +{ + uint16_t idx = vq->avail->ring[avail_idx & (vq->size - 1)]; + uint32_t vec_id = *vec_idx; + uint32_t len = 0; + uint64_t dlen; + struct vring_desc *descs = vq->desc; + struct vring_desc *idesc = NULL; + + *desc_chain_head = idx; + + if (vq->desc[idx].flags & VRING_DESC_F_INDIRECT) { + dlen = vq->desc[idx].len; + descs = (struct vring_desc *)(uintptr_t) + vhost_iova_to_vva(dev, vq, vq->desc[idx].addr, + &dlen, + VHOST_ACCESS_RO); + if (unlikely(!descs)) + return -1; + + if (unlikely(dlen < vq->desc[idx].len)) { + /* + * The indirect desc table is not contiguous + * in process VA space, we have to copy it. + */ + idesc = alloc_copy_ind_table(dev, vq, &vq->desc[idx]); + if (unlikely(!idesc)) + return -1; + + descs = idesc; + } + + idx = 0; + } + + while (1) { + if (unlikely(vec_id >= BUF_VECTOR_MAX || idx >= vq->size)) { + free_ind_table(idesc); + return -1; + } + + len += descs[idx].len; + buf_vec[vec_id].buf_addr = descs[idx].addr; + buf_vec[vec_id].buf_len = descs[idx].len; + buf_vec[vec_id].desc_idx = idx; + vec_id++; + + if ((descs[idx].flags & VRING_DESC_F_NEXT) == 0) + break; + + idx = descs[idx].next; + } + + *desc_chain_len = len; + *vec_idx = vec_id; + + if (unlikely(!!idesc)) + free_ind_table(idesc); + + return 0; +} + +/* + * Returns -1 on fail, 0 on success + */ +static inline int +reserve_avail_buf_mergeable(struct virtio_net *dev, struct vhost_virtqueue *vq, + uint32_t size, struct buf_vector *buf_vec, + uint16_t *num_buffers, uint16_t avail_head) +{ + uint16_t cur_idx; + uint32_t vec_idx = 0; + uint16_t tries = 0; + + uint16_t head_idx = 0; + uint16_t len = 0; + + *num_buffers = 0; + cur_idx = vq->last_avail_idx; + + while (size > 0) { + if (unlikely(cur_idx == avail_head)) + return -1; + + if (unlikely(fill_vec_buf(dev, vq, cur_idx, &vec_idx, buf_vec, + &head_idx, &len) < 0)) + return -1; + len = RTE_MIN(len, size); + update_shadow_used_ring(vq, head_idx, len); + size -= len; + + cur_idx++; + tries++; + *num_buffers += 1; + + /* + * if we tried all available ring items, and still + * can't get enough buf, it means something abnormal + * happened. + */ + if (unlikely(tries >= vq->size)) + return -1; + } + + return 0; +} + +static __rte_always_inline int +copy_mbuf_to_desc_mergeable(struct virtio_net *dev, struct vhost_virtqueue *vq, + struct rte_mbuf *m, struct buf_vector *buf_vec, + uint16_t num_buffers) +{ + uint32_t vec_idx = 0; + uint64_t desc_addr, desc_gaddr; + uint32_t mbuf_offset, mbuf_avail; + uint32_t desc_offset, desc_avail; + uint32_t cpy_len; + uint64_t desc_chunck_len; + uint64_t hdr_addr, hdr_phys_addr; + struct rte_mbuf *hdr_mbuf; + struct batch_copy_elem *batch_copy = vq->batch_copy_elems; + struct virtio_net_hdr_mrg_rxbuf tmp_hdr, *hdr = NULL; + uint16_t copy_nb = vq->batch_copy_nb_elems; + int error = 0; + + if (unlikely(m == NULL)) { + error = -1; + goto out; + } + + desc_chunck_len = buf_vec[vec_idx].buf_len; + desc_gaddr = buf_vec[vec_idx].buf_addr; + desc_addr = vhost_iova_to_vva(dev, vq, + desc_gaddr, + &desc_chunck_len, + VHOST_ACCESS_RW); + if (buf_vec[vec_idx].buf_len < dev->vhost_hlen || !desc_addr) { + error = -1; + goto out; + } + + hdr_mbuf = m; + hdr_addr = desc_addr; + if (unlikely(desc_chunck_len < dev->vhost_hlen)) + hdr = &tmp_hdr; + else + hdr = (struct virtio_net_hdr_mrg_rxbuf *)(uintptr_t)hdr_addr; + hdr_phys_addr = desc_gaddr; + rte_prefetch0((void *)(uintptr_t)hdr_addr); + + LOG_DEBUG(VHOST_DATA, "(%d) RX: num merge buffers %d\n", + dev->vid, num_buffers); + + desc_avail = buf_vec[vec_idx].buf_len - dev->vhost_hlen; + if (unlikely(desc_chunck_len < dev->vhost_hlen)) { + desc_chunck_len = desc_avail; + desc_gaddr += dev->vhost_hlen; + desc_addr = vhost_iova_to_vva(dev, vq, + desc_gaddr, + &desc_chunck_len, + VHOST_ACCESS_RW); + if (unlikely(!desc_addr)) { + error = -1; + goto out; + } + + desc_offset = 0; + } else { + desc_offset = dev->vhost_hlen; + desc_chunck_len -= dev->vhost_hlen; + } + + + mbuf_avail = rte_pktmbuf_data_len(m); + mbuf_offset = 0; + while (mbuf_avail != 0 || m->next != NULL) { + /* done with current desc buf, get the next one */ + if (desc_avail == 0) { + vec_idx++; + desc_chunck_len = buf_vec[vec_idx].buf_len; + desc_gaddr = buf_vec[vec_idx].buf_addr; + desc_addr = + vhost_iova_to_vva(dev, vq, + desc_gaddr, + &desc_chunck_len, + VHOST_ACCESS_RW); + if (unlikely(!desc_addr)) { + error = -1; + goto out; + } + + /* Prefetch buffer address. */ + rte_prefetch0((void *)(uintptr_t)desc_addr); + desc_offset = 0; + desc_avail = buf_vec[vec_idx].buf_len; + } else if (unlikely(desc_chunck_len == 0)) { + desc_chunck_len = desc_avail; + desc_gaddr += desc_offset; + desc_addr = vhost_iova_to_vva(dev, vq, + desc_gaddr, + &desc_chunck_len, VHOST_ACCESS_RW); + if (unlikely(!desc_addr)) { + error = -1; + goto out; + } + desc_offset = 0; + } + + /* done with current mbuf, get the next one */ + if (mbuf_avail == 0) { + m = m->next; + + mbuf_offset = 0; + mbuf_avail = rte_pktmbuf_data_len(m); + } + + if (hdr_addr) { + virtio_enqueue_offload(hdr_mbuf, &hdr->hdr); + ASSIGN_UNLESS_EQUAL(hdr->num_buffers, num_buffers); + + if (unlikely(hdr == &tmp_hdr)) { + uint64_t len; + uint64_t remain = dev->vhost_hlen; + uint64_t src = (uint64_t)(uintptr_t)hdr, dst; + uint64_t guest_addr = hdr_phys_addr; + + while (remain) { + len = remain; + dst = vhost_iova_to_vva(dev, vq, + guest_addr, &len, + VHOST_ACCESS_RW); + if (unlikely(!dst || !len)) { + error = -1; + goto out; + } + + rte_memcpy((void *)(uintptr_t)dst, + (void *)(uintptr_t)src, + len); + + PRINT_PACKET(dev, (uintptr_t)dst, + len, 0); + vhost_log_write(dev, guest_addr, len); + + remain -= len; + guest_addr += len; + dst += len; + } + } else { + PRINT_PACKET(dev, (uintptr_t)hdr_addr, + dev->vhost_hlen, 0); + vhost_log_write(dev, hdr_phys_addr, + dev->vhost_hlen); + } + + hdr_addr = 0; + } + + cpy_len = RTE_MIN(desc_chunck_len, mbuf_avail); + + if (likely(cpy_len > MAX_BATCH_LEN || copy_nb >= vq->size)) { + rte_memcpy((void *)((uintptr_t)(desc_addr + + desc_offset)), + rte_pktmbuf_mtod_offset(m, void *, mbuf_offset), + cpy_len); + vhost_log_write(dev, desc_gaddr + desc_offset, cpy_len); + PRINT_PACKET(dev, (uintptr_t)(desc_addr + desc_offset), + cpy_len, 0); + } else { + batch_copy[copy_nb].dst = + (void *)((uintptr_t)(desc_addr + desc_offset)); + batch_copy[copy_nb].src = + rte_pktmbuf_mtod_offset(m, void *, mbuf_offset); + batch_copy[copy_nb].log_addr = desc_gaddr + desc_offset; + batch_copy[copy_nb].len = cpy_len; + copy_nb++; + } + + mbuf_avail -= cpy_len; + mbuf_offset += cpy_len; + desc_avail -= cpy_len; + desc_offset += cpy_len; + desc_chunck_len -= cpy_len; + } + +out: + vq->batch_copy_nb_elems = copy_nb; + + return error; +} + +static __rte_always_inline uint32_t +virtio_dev_merge_rx(struct virtio_net *dev, uint16_t queue_id, + struct rte_mbuf **pkts, uint32_t count) +{ + struct vhost_virtqueue *vq; + uint32_t pkt_idx = 0; + uint16_t num_buffers; + struct buf_vector buf_vec[BUF_VECTOR_MAX]; + uint16_t avail_head; + + LOG_DEBUG(VHOST_DATA, "(%d) %s\n", dev->vid, __func__); + if (unlikely(!is_valid_virt_queue_idx(queue_id, 0, dev->nr_vring))) { + RTE_LOG(ERR, VHOST_DATA, "(%d) %s: invalid virtqueue idx %d.\n", + dev->vid, __func__, queue_id); + return 0; + } + + vq = dev->virtqueue[queue_id]; + + rte_spinlock_lock(&vq->access_lock); + + if (unlikely(vq->enabled == 0)) + goto out_access_unlock; + + if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)) + vhost_user_iotlb_rd_lock(vq); + + if (unlikely(vq->access_ok == 0)) + if (unlikely(vring_translate(dev, vq) < 0)) + goto out; + + count = RTE_MIN((uint32_t)MAX_PKT_BURST, count); + if (count == 0) + goto out; + + vq->batch_copy_nb_elems = 0; + + rte_prefetch0(&vq->avail->ring[vq->last_avail_idx & (vq->size - 1)]); + + vq->shadow_used_idx = 0; + avail_head = *((volatile uint16_t *)&vq->avail->idx); + for (pkt_idx = 0; pkt_idx < count; pkt_idx++) { + uint32_t pkt_len = pkts[pkt_idx]->pkt_len + dev->vhost_hlen; + + if (unlikely(reserve_avail_buf_mergeable(dev, vq, + pkt_len, buf_vec, &num_buffers, + avail_head) < 0)) { + LOG_DEBUG(VHOST_DATA, + "(%d) failed to get enough desc from vring\n", + dev->vid); + vq->shadow_used_idx -= num_buffers; + break; + } + + LOG_DEBUG(VHOST_DATA, "(%d) current index %d | end index %d\n", + dev->vid, vq->last_avail_idx, + vq->last_avail_idx + num_buffers); + + if (copy_mbuf_to_desc_mergeable(dev, vq, pkts[pkt_idx], + buf_vec, num_buffers) < 0) { + vq->shadow_used_idx -= num_buffers; + break; + } + + vq->last_avail_idx += num_buffers; + } + + do_data_copy_enqueue(dev, vq); + + if (likely(vq->shadow_used_idx)) { + flush_shadow_used_ring(dev, vq); + + /* flush used->idx update before we read avail->flags. */ + rte_mb(); + + /* Kick the guest if necessary. */ + if (!(vq->avail->flags & VRING_AVAIL_F_NO_INTERRUPT) + && (vq->callfd >= 0)) + eventfd_write(vq->callfd, (eventfd_t)1); + } + +out: + if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)) + vhost_user_iotlb_rd_unlock(vq); + +out_access_unlock: + rte_spinlock_unlock(&vq->access_lock); + + return pkt_idx; +} + +uint16_t +rte_vhost_enqueue_burst(int vid, uint16_t queue_id, + struct rte_mbuf **pkts, uint16_t count) +{ + struct virtio_net *dev = get_device(vid); + + if (!dev) + return 0; + + if (dev->features & (1 << VIRTIO_NET_F_MRG_RXBUF)) + return virtio_dev_merge_rx(dev, queue_id, pkts, count); + else + return virtio_dev_rx(dev, queue_id, pkts, count); +} + +static inline bool +virtio_net_with_host_offload(struct virtio_net *dev) +{ + if (dev->features & + ((1ULL << VIRTIO_NET_F_CSUM) | + (1ULL << VIRTIO_NET_F_HOST_ECN) | + (1ULL << VIRTIO_NET_F_HOST_TSO4) | + (1ULL << VIRTIO_NET_F_HOST_TSO6) | + (1ULL << VIRTIO_NET_F_HOST_UFO))) + return true; + + return false; +} + +static void +parse_ethernet(struct rte_mbuf *m, uint16_t *l4_proto, void **l4_hdr) +{ + struct ipv4_hdr *ipv4_hdr; + struct ipv6_hdr *ipv6_hdr; + void *l3_hdr = NULL; + struct ether_hdr *eth_hdr; + uint16_t ethertype; + + eth_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *); + + m->l2_len = sizeof(struct ether_hdr); + ethertype = rte_be_to_cpu_16(eth_hdr->ether_type); + + if (ethertype == ETHER_TYPE_VLAN) { + struct vlan_hdr *vlan_hdr = (struct vlan_hdr *)(eth_hdr + 1); + + m->l2_len += sizeof(struct vlan_hdr); + ethertype = rte_be_to_cpu_16(vlan_hdr->eth_proto); + } + + l3_hdr = (char *)eth_hdr + m->l2_len; + + switch (ethertype) { + case ETHER_TYPE_IPv4: + ipv4_hdr = l3_hdr; + *l4_proto = ipv4_hdr->next_proto_id; + m->l3_len = (ipv4_hdr->version_ihl & 0x0f) * 4; + *l4_hdr = (char *)l3_hdr + m->l3_len; + m->ol_flags |= PKT_TX_IPV4; + break; + case ETHER_TYPE_IPv6: + ipv6_hdr = l3_hdr; + *l4_proto = ipv6_hdr->proto; + m->l3_len = sizeof(struct ipv6_hdr); + *l4_hdr = (char *)l3_hdr + m->l3_len; + m->ol_flags |= PKT_TX_IPV6; + break; + default: + m->l3_len = 0; + *l4_proto = 0; + *l4_hdr = NULL; + break; + } +} + +static __rte_always_inline void +vhost_dequeue_offload(struct virtio_net_hdr *hdr, struct rte_mbuf *m) +{ + uint16_t l4_proto = 0; + void *l4_hdr = NULL; + struct tcp_hdr *tcp_hdr = NULL; + + if (hdr->flags == 0 && hdr->gso_type == VIRTIO_NET_HDR_GSO_NONE) + return; + + parse_ethernet(m, &l4_proto, &l4_hdr); + if (hdr->flags == VIRTIO_NET_HDR_F_NEEDS_CSUM) { + if (hdr->csum_start == (m->l2_len + m->l3_len)) { + switch (hdr->csum_offset) { + case (offsetof(struct tcp_hdr, cksum)): + if (l4_proto == IPPROTO_TCP) + m->ol_flags |= PKT_TX_TCP_CKSUM; + break; + case (offsetof(struct udp_hdr, dgram_cksum)): + if (l4_proto == IPPROTO_UDP) + m->ol_flags |= PKT_TX_UDP_CKSUM; + break; + case (offsetof(struct sctp_hdr, cksum)): + if (l4_proto == IPPROTO_SCTP) + m->ol_flags |= PKT_TX_SCTP_CKSUM; + break; + default: + break; + } + } + } + + if (l4_hdr && hdr->gso_type != VIRTIO_NET_HDR_GSO_NONE) { + switch (hdr->gso_type & ~VIRTIO_NET_HDR_GSO_ECN) { + case VIRTIO_NET_HDR_GSO_TCPV4: + case VIRTIO_NET_HDR_GSO_TCPV6: + tcp_hdr = l4_hdr; + m->ol_flags |= PKT_TX_TCP_SEG; + m->tso_segsz = hdr->gso_size; + m->l4_len = (tcp_hdr->data_off & 0xf0) >> 2; + break; + default: + RTE_LOG(WARNING, VHOST_DATA, + "unsupported gso type %u.\n", hdr->gso_type); + break; + } + } +} + +#define RARP_PKT_SIZE 64 + +static int +make_rarp_packet(struct rte_mbuf *rarp_mbuf, const struct ether_addr *mac) +{ + struct ether_hdr *eth_hdr; + struct arp_hdr *rarp; + + if (rarp_mbuf->buf_len < 64) { + RTE_LOG(WARNING, VHOST_DATA, + "failed to make RARP; mbuf size too small %u (< %d)\n", + rarp_mbuf->buf_len, RARP_PKT_SIZE); + return -1; + } + + /* Ethernet header. */ + eth_hdr = rte_pktmbuf_mtod_offset(rarp_mbuf, struct ether_hdr *, 0); + memset(eth_hdr->d_addr.addr_bytes, 0xff, ETHER_ADDR_LEN); + ether_addr_copy(mac, ð_hdr->s_addr); + eth_hdr->ether_type = htons(ETHER_TYPE_RARP); + + /* RARP header. */ + rarp = (struct arp_hdr *)(eth_hdr + 1); + rarp->arp_hrd = htons(ARP_HRD_ETHER); + rarp->arp_pro = htons(ETHER_TYPE_IPv4); + rarp->arp_hln = ETHER_ADDR_LEN; + rarp->arp_pln = 4; + rarp->arp_op = htons(ARP_OP_REVREQUEST); + + ether_addr_copy(mac, &rarp->arp_data.arp_sha); + ether_addr_copy(mac, &rarp->arp_data.arp_tha); + memset(&rarp->arp_data.arp_sip, 0x00, 4); + memset(&rarp->arp_data.arp_tip, 0x00, 4); + + rarp_mbuf->pkt_len = rarp_mbuf->data_len = RARP_PKT_SIZE; + + return 0; +} + +static __rte_always_inline void +put_zmbuf(struct zcopy_mbuf *zmbuf) +{ + zmbuf->in_use = 0; +} + +static __rte_always_inline int +copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq, + struct vring_desc *descs, uint16_t max_desc, + struct rte_mbuf *m, uint16_t desc_idx, + struct rte_mempool *mbuf_pool) +{ + struct vring_desc *desc; + uint64_t desc_addr, desc_gaddr; + uint32_t desc_avail, desc_offset; + uint32_t mbuf_avail, mbuf_offset; + uint32_t cpy_len; + uint64_t desc_chunck_len; + struct rte_mbuf *cur = m, *prev = m; + struct virtio_net_hdr tmp_hdr; + struct virtio_net_hdr *hdr = NULL; + /* A counter to avoid desc dead loop chain */ + uint32_t nr_desc = 1; + struct batch_copy_elem *batch_copy = vq->batch_copy_elems; + uint16_t copy_nb = vq->batch_copy_nb_elems; + int error = 0; + + desc = &descs[desc_idx]; + if (unlikely((desc->len < dev->vhost_hlen)) || + (desc->flags & VRING_DESC_F_INDIRECT)) { + error = -1; + goto out; + } + + desc_chunck_len = desc->len; + desc_gaddr = desc->addr; + desc_addr = vhost_iova_to_vva(dev, + vq, desc_gaddr, + &desc_chunck_len, + VHOST_ACCESS_RO); + if (unlikely(!desc_addr)) { + error = -1; + goto out; + } + + if (virtio_net_with_host_offload(dev)) { + if (unlikely(desc_chunck_len < sizeof(struct virtio_net_hdr))) { + uint64_t len = desc_chunck_len; + uint64_t remain = sizeof(struct virtio_net_hdr); + uint64_t src = desc_addr; + uint64_t dst = (uint64_t)(uintptr_t)&tmp_hdr; + uint64_t guest_addr = desc_gaddr; + + /* + * No luck, the virtio-net header doesn't fit + * in a contiguous virtual area. + */ + while (remain) { + len = remain; + src = vhost_iova_to_vva(dev, vq, + guest_addr, &len, + VHOST_ACCESS_RO); + if (unlikely(!src || !len)) { + error = -1; + goto out; + } + + rte_memcpy((void *)(uintptr_t)dst, + (void *)(uintptr_t)src, len); + + guest_addr += len; + remain -= len; + dst += len; + } + + hdr = &tmp_hdr; + } else { + hdr = (struct virtio_net_hdr *)((uintptr_t)desc_addr); + rte_prefetch0(hdr); + } + } + + /* + * A virtio driver normally uses at least 2 desc buffers + * for Tx: the first for storing the header, and others + * for storing the data. + */ + if (likely((desc->len == dev->vhost_hlen) && + (desc->flags & VRING_DESC_F_NEXT) != 0)) { + desc = &descs[desc->next]; + if (unlikely(desc->flags & VRING_DESC_F_INDIRECT)) { + error = -1; + goto out; + } + + desc_chunck_len = desc->len; + desc_gaddr = desc->addr; + desc_addr = vhost_iova_to_vva(dev, + vq, desc_gaddr, + &desc_chunck_len, + VHOST_ACCESS_RO); + if (unlikely(!desc_addr)) { + error = -1; + goto out; + } + + desc_offset = 0; + desc_avail = desc->len; + nr_desc += 1; + } else { + desc_avail = desc->len - dev->vhost_hlen; + + if (unlikely(desc_chunck_len < dev->vhost_hlen)) { + desc_chunck_len = desc_avail; + desc_gaddr += dev->vhost_hlen; + desc_addr = vhost_iova_to_vva(dev, + vq, desc_gaddr, + &desc_chunck_len, + VHOST_ACCESS_RO); + if (unlikely(!desc_addr)) { + error = -1; + goto out; + } + + desc_offset = 0; + } else { + desc_offset = dev->vhost_hlen; + desc_chunck_len -= dev->vhost_hlen; + } + } + + rte_prefetch0((void *)(uintptr_t)(desc_addr + desc_offset)); + + PRINT_PACKET(dev, (uintptr_t)(desc_addr + desc_offset), + desc_chunck_len, 0); + + mbuf_offset = 0; + mbuf_avail = m->buf_len - RTE_PKTMBUF_HEADROOM; + while (1) { + uint64_t hpa; + + cpy_len = RTE_MIN(desc_chunck_len, mbuf_avail); + + /* + * A desc buf might across two host physical pages that are + * not continuous. In such case (gpa_to_hpa returns 0), data + * will be copied even though zero copy is enabled. + */ + if (unlikely(dev->dequeue_zero_copy && (hpa = gpa_to_hpa(dev, + desc_gaddr + desc_offset, cpy_len)))) { + cur->data_len = cpy_len; + cur->data_off = 0; + cur->buf_addr = (void *)(uintptr_t)(desc_addr + + desc_offset); + cur->buf_iova = hpa; + + /* + * In zero copy mode, one mbuf can only reference data + * for one or partial of one desc buff. + */ + mbuf_avail = cpy_len; + } else { + if (likely(cpy_len > MAX_BATCH_LEN || + copy_nb >= vq->size || + (hdr && cur == m) || + desc->len != desc_chunck_len)) { + rte_memcpy(rte_pktmbuf_mtod_offset(cur, void *, + mbuf_offset), + (void *)((uintptr_t)(desc_addr + + desc_offset)), + cpy_len); + } else { + batch_copy[copy_nb].dst = + rte_pktmbuf_mtod_offset(cur, void *, + mbuf_offset); + batch_copy[copy_nb].src = + (void *)((uintptr_t)(desc_addr + + desc_offset)); + batch_copy[copy_nb].len = cpy_len; + copy_nb++; + } + } + + mbuf_avail -= cpy_len; + mbuf_offset += cpy_len; + desc_avail -= cpy_len; + desc_chunck_len -= cpy_len; + desc_offset += cpy_len; + + /* This desc reaches to its end, get the next one */ + if (desc_avail == 0) { + if ((desc->flags & VRING_DESC_F_NEXT) == 0) + break; + + if (unlikely(desc->next >= max_desc || + ++nr_desc > max_desc)) { + error = -1; + goto out; + } + desc = &descs[desc->next]; + if (unlikely(desc->flags & VRING_DESC_F_INDIRECT)) { + error = -1; + goto out; + } + + desc_chunck_len = desc->len; + desc_gaddr = desc->addr; + desc_addr = vhost_iova_to_vva(dev, + vq, desc_gaddr, + &desc_chunck_len, + VHOST_ACCESS_RO); + if (unlikely(!desc_addr)) { + error = -1; + goto out; + } + + rte_prefetch0((void *)(uintptr_t)desc_addr); + + desc_offset = 0; + desc_avail = desc->len; + + PRINT_PACKET(dev, (uintptr_t)desc_addr, + desc_chunck_len, 0); + } else if (unlikely(desc_chunck_len == 0)) { + desc_chunck_len = desc_avail; + desc_gaddr += desc_offset; + desc_addr = vhost_iova_to_vva(dev, vq, + desc_gaddr, + &desc_chunck_len, + VHOST_ACCESS_RO); + if (unlikely(!desc_addr)) { + error = -1; + goto out; + } + desc_offset = 0; + + PRINT_PACKET(dev, (uintptr_t)desc_addr, + desc_chunck_len, 0); + } + + /* + * This mbuf reaches to its end, get a new one + * to hold more data. + */ + if (mbuf_avail == 0) { + cur = rte_pktmbuf_alloc(mbuf_pool); + if (unlikely(cur == NULL)) { + RTE_LOG(ERR, VHOST_DATA, "Failed to " + "allocate memory for mbuf.\n"); + error = -1; + goto out; + } + if (unlikely(dev->dequeue_zero_copy)) + rte_mbuf_refcnt_update(cur, 1); + + prev->next = cur; + prev->data_len = mbuf_offset; + m->nb_segs += 1; + m->pkt_len += mbuf_offset; + prev = cur; + + mbuf_offset = 0; + mbuf_avail = cur->buf_len - RTE_PKTMBUF_HEADROOM; + } + } + + prev->data_len = mbuf_offset; + m->pkt_len += mbuf_offset; + + if (hdr) + vhost_dequeue_offload(hdr, m); + +out: + vq->batch_copy_nb_elems = copy_nb; + + return error; +} + +static __rte_always_inline void +update_used_ring(struct virtio_net *dev, struct vhost_virtqueue *vq, + uint32_t used_idx, uint32_t desc_idx) +{ + vq->used->ring[used_idx].id = desc_idx; + vq->used->ring[used_idx].len = 0; + vhost_log_used_vring(dev, vq, + offsetof(struct vring_used, ring[used_idx]), + sizeof(vq->used->ring[used_idx])); +} + +static __rte_always_inline void +update_used_idx(struct virtio_net *dev, struct vhost_virtqueue *vq, + uint32_t count) +{ + if (unlikely(count == 0)) + return; + + rte_smp_wmb(); + rte_smp_rmb(); + + vq->used->idx += count; + vhost_log_used_vring(dev, vq, offsetof(struct vring_used, idx), + sizeof(vq->used->idx)); + + /* Kick guest if required. */ + if (!(vq->avail->flags & VRING_AVAIL_F_NO_INTERRUPT) + && (vq->callfd >= 0)) + eventfd_write(vq->callfd, (eventfd_t)1); +} + +static __rte_always_inline struct zcopy_mbuf * +get_zmbuf(struct vhost_virtqueue *vq) +{ + uint16_t i; + uint16_t last; + int tries = 0; + + /* search [last_zmbuf_idx, zmbuf_size) */ + i = vq->last_zmbuf_idx; + last = vq->zmbuf_size; + +again: + for (; i < last; i++) { + if (vq->zmbufs[i].in_use == 0) { + vq->last_zmbuf_idx = i + 1; + vq->zmbufs[i].in_use = 1; + return &vq->zmbufs[i]; + } + } + + tries++; + if (tries == 1) { + /* search [0, last_zmbuf_idx) */ + i = 0; + last = vq->last_zmbuf_idx; + goto again; + } + + return NULL; +} + +static __rte_always_inline bool +mbuf_is_consumed(struct rte_mbuf *m) +{ + while (m) { + if (rte_mbuf_refcnt_read(m) > 1) + return false; + m = m->next; + } + + return true; +} + +static __rte_always_inline void +restore_mbuf(struct rte_mbuf *m) +{ + uint32_t mbuf_size, priv_size; + + while (m) { + priv_size = rte_pktmbuf_priv_size(m->pool); + mbuf_size = sizeof(struct rte_mbuf) + priv_size; + /* start of buffer is after mbuf structure and priv data */ + + m->buf_addr = (char *)m + mbuf_size; + m->buf_iova = rte_mempool_virt2iova(m) + mbuf_size; + m = m->next; + } +} + +uint16_t +rte_vhost_dequeue_burst(int vid, uint16_t queue_id, + struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count) +{ + struct virtio_net *dev; + struct rte_mbuf *rarp_mbuf = NULL; + struct vhost_virtqueue *vq; + uint32_t desc_indexes[MAX_PKT_BURST]; + uint32_t used_idx; + uint32_t i = 0; + uint16_t free_entries; + uint16_t avail_idx; + + dev = get_device(vid); + if (!dev) + return 0; + + if (unlikely(!is_valid_virt_queue_idx(queue_id, 1, dev->nr_vring))) { + RTE_LOG(ERR, VHOST_DATA, "(%d) %s: invalid virtqueue idx %d.\n", + dev->vid, __func__, queue_id); + return 0; + } + + vq = dev->virtqueue[queue_id]; + + if (unlikely(rte_spinlock_trylock(&vq->access_lock) == 0)) + return 0; + + if (unlikely(vq->enabled == 0)) + goto out_access_unlock; + + vq->batch_copy_nb_elems = 0; + + if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)) + vhost_user_iotlb_rd_lock(vq); + + if (unlikely(vq->access_ok == 0)) + if (unlikely(vring_translate(dev, vq) < 0)) + goto out; + + if (unlikely(dev->dequeue_zero_copy)) { + struct zcopy_mbuf *zmbuf, *next; + int nr_updated = 0; + + for (zmbuf = TAILQ_FIRST(&vq->zmbuf_list); + zmbuf != NULL; zmbuf = next) { + next = TAILQ_NEXT(zmbuf, next); + + if (mbuf_is_consumed(zmbuf->mbuf)) { + used_idx = vq->last_used_idx++ & (vq->size - 1); + update_used_ring(dev, vq, used_idx, + zmbuf->desc_idx); + nr_updated += 1; + + TAILQ_REMOVE(&vq->zmbuf_list, zmbuf, next); + restore_mbuf(zmbuf->mbuf); + rte_pktmbuf_free(zmbuf->mbuf); + put_zmbuf(zmbuf); + vq->nr_zmbuf -= 1; + } + } + + update_used_idx(dev, vq, nr_updated); + } + + /* + * Construct a RARP broadcast packet, and inject it to the "pkts" + * array, to looks like that guest actually send such packet. + * + * Check user_send_rarp() for more information. + * + * broadcast_rarp shares a cacheline in the virtio_net structure + * with some fields that are accessed during enqueue and + * rte_atomic16_cmpset() causes a write if using cmpxchg. This could + * result in false sharing between enqueue and dequeue. + * + * Prevent unnecessary false sharing by reading broadcast_rarp first + * and only performing cmpset if the read indicates it is likely to + * be set. + */ + + if (unlikely(rte_atomic16_read(&dev->broadcast_rarp) && + rte_atomic16_cmpset((volatile uint16_t *) + &dev->broadcast_rarp.cnt, 1, 0))) { + + rarp_mbuf = rte_pktmbuf_alloc(mbuf_pool); + if (rarp_mbuf == NULL) { + RTE_LOG(ERR, VHOST_DATA, + "Failed to allocate memory for mbuf.\n"); + return 0; + } + + if (make_rarp_packet(rarp_mbuf, &dev->mac)) { + rte_pktmbuf_free(rarp_mbuf); + rarp_mbuf = NULL; + } else { + count -= 1; + } + } + + free_entries = *((volatile uint16_t *)&vq->avail->idx) - + vq->last_avail_idx; + if (free_entries == 0) + goto out; + + LOG_DEBUG(VHOST_DATA, "(%d) %s\n", dev->vid, __func__); + + /* Prefetch available and used ring */ + avail_idx = vq->last_avail_idx & (vq->size - 1); + used_idx = vq->last_used_idx & (vq->size - 1); + rte_prefetch0(&vq->avail->ring[avail_idx]); + rte_prefetch0(&vq->used->ring[used_idx]); + + count = RTE_MIN(count, MAX_PKT_BURST); + count = RTE_MIN(count, free_entries); + LOG_DEBUG(VHOST_DATA, "(%d) about to dequeue %u buffers\n", + dev->vid, count); + + /* Retrieve all of the head indexes first to avoid caching issues. */ + for (i = 0; i < count; i++) { + avail_idx = (vq->last_avail_idx + i) & (vq->size - 1); + used_idx = (vq->last_used_idx + i) & (vq->size - 1); + desc_indexes[i] = vq->avail->ring[avail_idx]; + + if (likely(dev->dequeue_zero_copy == 0)) + update_used_ring(dev, vq, used_idx, desc_indexes[i]); + } + + /* Prefetch descriptor index. */ + rte_prefetch0(&vq->desc[desc_indexes[0]]); + for (i = 0; i < count; i++) { + struct vring_desc *desc, *idesc = NULL; + uint16_t sz, idx; + uint64_t dlen; + int err; + + if (likely(i + 1 < count)) + rte_prefetch0(&vq->desc[desc_indexes[i + 1]]); + + if (vq->desc[desc_indexes[i]].flags & VRING_DESC_F_INDIRECT) { + dlen = vq->desc[desc_indexes[i]].len; + desc = (struct vring_desc *)(uintptr_t) + vhost_iova_to_vva(dev, vq, + vq->desc[desc_indexes[i]].addr, + &dlen, + VHOST_ACCESS_RO); + if (unlikely(!desc)) + break; + + if (unlikely(dlen < vq->desc[desc_indexes[i]].len)) { + /* + * The indirect desc table is not contiguous + * in process VA space, we have to copy it. + */ + idesc = alloc_copy_ind_table(dev, vq, + &vq->desc[desc_indexes[i]]); + if (unlikely(!idesc)) + break; + + desc = idesc; + } + + rte_prefetch0(desc); + sz = vq->desc[desc_indexes[i]].len / sizeof(*desc); + idx = 0; + } else { + desc = vq->desc; + sz = vq->size; + idx = desc_indexes[i]; + } + + pkts[i] = rte_pktmbuf_alloc(mbuf_pool); + if (unlikely(pkts[i] == NULL)) { + RTE_LOG(ERR, VHOST_DATA, + "Failed to allocate memory for mbuf.\n"); + free_ind_table(idesc); + break; + } + + err = copy_desc_to_mbuf(dev, vq, desc, sz, pkts[i], idx, + mbuf_pool); + if (unlikely(err)) { + rte_pktmbuf_free(pkts[i]); + free_ind_table(idesc); + break; + } + + if (unlikely(dev->dequeue_zero_copy)) { + struct zcopy_mbuf *zmbuf; + + zmbuf = get_zmbuf(vq); + if (!zmbuf) { + rte_pktmbuf_free(pkts[i]); + free_ind_table(idesc); + break; + } + zmbuf->mbuf = pkts[i]; + zmbuf->desc_idx = desc_indexes[i]; + + /* + * Pin lock the mbuf; we will check later to see + * whether the mbuf is freed (when we are the last + * user) or not. If that's the case, we then could + * update the used ring safely. + */ + rte_mbuf_refcnt_update(pkts[i], 1); + + vq->nr_zmbuf += 1; + TAILQ_INSERT_TAIL(&vq->zmbuf_list, zmbuf, next); + } + + if (unlikely(!!idesc)) + free_ind_table(idesc); + } + vq->last_avail_idx += i; + + if (likely(dev->dequeue_zero_copy == 0)) { + do_data_copy_dequeue(vq); + vq->last_used_idx += i; + update_used_idx(dev, vq, i); + } + +out: + if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)) + vhost_user_iotlb_rd_unlock(vq); + +out_access_unlock: + rte_spinlock_unlock(&vq->access_lock); + + if (unlikely(rarp_mbuf != NULL)) { + /* + * Inject it to the head of "pkts" array, so that switch's mac + * learning table will get updated first. + */ + memmove(&pkts[1], pkts, i * sizeof(struct rte_mbuf *)); + pkts[0] = rarp_mbuf; + i += 1; + } + + return i; +} diff --git a/dpdk/mk/arch/arm64/rte.vars.mk b/dpdk/mk/arch/arm64/rte.vars.mk index c168426d..5e7ee96c 100644 --- a/dpdk/mk/arch/arm64/rte.vars.mk +++ b/dpdk/mk/arch/arm64/rte.vars.mk @@ -1,6 +1,6 @@ # BSD LICENSE # -# Copyright (C) Cavium networks 2015. All rights reserved. +# Copyright (C) Cavium, Inc 2015. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -12,7 +12,7 @@ # notice, this list of conditions and the following disclaimer in # the documentation and/or other materials provided with the # distribution. -# * Neither the name of Cavium networks nor the names of its +# * Neither the name of Cavium, Inc nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # diff --git a/dpdk/mk/arch/i686/rte.vars.mk b/dpdk/mk/arch/i686/rte.vars.mk index 6a25312c..ec7b7a6b 100644 --- a/dpdk/mk/arch/i686/rte.vars.mk +++ b/dpdk/mk/arch/i686/rte.vars.mk @@ -32,15 +32,15 @@ # # arch: # -# - define ARCH variable (overriden by cmdline or by previous +# - define ARCH variable (overridden by cmdline or by previous # optional define in machine .mk) -# - define CROSS variable (overriden by cmdline or previous define +# - define CROSS variable (overridden by cmdline or previous define # in machine .mk) -# - define CPU_CFLAGS variable (overriden by cmdline or previous +# - define CPU_CFLAGS variable (overridden by cmdline or previous # define in machine .mk) -# - define CPU_LDFLAGS variable (overriden by cmdline or previous +# - define CPU_LDFLAGS variable (overridden by cmdline or previous # define in machine .mk) -# - define CPU_ASFLAGS variable (overriden by cmdline or previous +# - define CPU_ASFLAGS variable (overridden by cmdline or previous # define in machine .mk) # - may override any previously defined variable # diff --git a/dpdk/mk/arch/tile/rte.vars.mk b/dpdk/mk/arch/tile/rte.vars.mk deleted file mode 100644 index 5ad37389..00000000 --- a/dpdk/mk/arch/tile/rte.vars.mk +++ /dev/null @@ -1,39 +0,0 @@ -# BSD LICENSE -# -# Copyright (C) EZchip Semiconductor Ltd. 2015. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in -# the documentation and/or other materials provided with the -# distribution. -# * Neither the name of EZchip Semiconductor nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -ARCH ?= tile -CROSS ?= - -CPU_CFLAGS ?= -CPU_LDFLAGS ?= -CPU_ASFLAGS ?= - -export ARCH CROSS CPU_CFLAGS CPU_LDFLAGS CPU_ASFLAGS diff --git a/dpdk/mk/arch/x86_64/rte.vars.mk b/dpdk/mk/arch/x86_64/rte.vars.mk index 83723c8e..617fa655 100644 --- a/dpdk/mk/arch/x86_64/rte.vars.mk +++ b/dpdk/mk/arch/x86_64/rte.vars.mk @@ -32,15 +32,15 @@ # # arch: # -# - define ARCH variable (overriden by cmdline or by previous +# - define ARCH variable (overridden by cmdline or by previous # optional define in machine .mk) -# - define CROSS variable (overriden by cmdline or previous define +# - define CROSS variable (overridden by cmdline or previous define # in machine .mk) -# - define CPU_CFLAGS variable (overriden by cmdline or previous +# - define CPU_CFLAGS variable (overridden by cmdline or previous # define in machine .mk) -# - define CPU_LDFLAGS variable (overriden by cmdline or previous +# - define CPU_LDFLAGS variable (overridden by cmdline or previous # define in machine .mk) -# - define CPU_ASFLAGS variable (overriden by cmdline or previous +# - define CPU_ASFLAGS variable (overridden by cmdline or previous # define in machine .mk) # - may override any previously defined variable # diff --git a/dpdk/mk/exec-env/bsdapp/rte.vars.mk b/dpdk/mk/exec-env/bsdapp/rte.vars.mk index 47a673e7..ebae68cc 100644 --- a/dpdk/mk/exec-env/bsdapp/rte.vars.mk +++ b/dpdk/mk/exec-env/bsdapp/rte.vars.mk @@ -32,9 +32,9 @@ # # exec-env: # -# - define EXECENV_CFLAGS variable (overriden by cmdline) -# - define EXECENV_LDFLAGS variable (overriden by cmdline) -# - define EXECENV_ASFLAGS variable (overriden by cmdline) +# - define EXECENV_CFLAGS variable (overridden by cmdline) +# - define EXECENV_LDFLAGS variable (overridden by cmdline) +# - define EXECENV_ASFLAGS variable (overridden by cmdline) # - may override any previously defined variable # # examples for RTE_EXEC_ENV: linuxapp, bsdapp diff --git a/dpdk/mk/exec-env/linuxapp/rte.vars.mk b/dpdk/mk/exec-env/linuxapp/rte.vars.mk index a8a1ee4c..9a716999 100644 --- a/dpdk/mk/exec-env/linuxapp/rte.vars.mk +++ b/dpdk/mk/exec-env/linuxapp/rte.vars.mk @@ -32,9 +32,9 @@ # # exec-env: # -# - define EXECENV_CFLAGS variable (overriden by cmdline) -# - define EXECENV_LDFLAGS variable (overriden by cmdline) -# - define EXECENV_ASFLAGS variable (overriden by cmdline) +# - define EXECENV_CFLAGS variable (overridden by cmdline) +# - define EXECENV_LDFLAGS variable (overridden by cmdline) +# - define EXECENV_ASFLAGS variable (overridden by cmdline) # - may override any previously defined variable # # examples for RTE_EXEC_ENV: linuxapp, bsdapp diff --git a/dpdk/mk/internal/rte.compile-pre.mk b/dpdk/mk/internal/rte.compile-pre.mk index f7401791..da8dda49 100644 --- a/dpdk/mk/internal/rte.compile-pre.mk +++ b/dpdk/mk/internal/rte.compile-pre.mk @@ -74,20 +74,20 @@ CMDS-all := $(CMDS-y) $(CMDS-n) $(CMDS-) # command to compile a .c file to generate an object ifeq ($(USE_HOST),1) -C_TO_O = $(HOSTCC) -Wp,-MD,$(call obj2dep,$(@)).tmp $(HOST_CFLAGS) \ - $(CFLAGS_$(@)) $(HOST_EXTRA_CFLAGS) -o $@ -c $< +C_TO_O = $(HOSTCC) -Wp,-MD,$(call obj2dep,$(@)).tmp $(HOST_CPPFLAGS) $(HOST_CFLAGS) \ + $(CFLAGS_$(@)) $(HOST_EXTRA_CPPFLAGS) $(HOST_EXTRA_CFLAGS) -o $@ -c $< C_TO_O_STR = $(subst ','\'',$(C_TO_O)) #'# fix syntax highlight C_TO_O_DISP = $(if $(V),"$(C_TO_O_STR)"," HOSTCC $(@)") else -C_TO_O = $(CC) -Wp,-MD,$(call obj2dep,$(@)).tmp $(CFLAGS) \ - $(CFLAGS_$(@)) $(EXTRA_CFLAGS) -o $@ -c $< +C_TO_O = $(CC) -Wp,-MD,$(call obj2dep,$(@)).tmp $(CPPFLAGS) $(CFLAGS) \ + $(CFLAGS_$(@)) $(EXTRA_CPPFLAGS) $(EXTRA_CFLAGS) -o $@ -c $< C_TO_O_STR = $(subst ','\'',$(C_TO_O)) #'# fix syntax highlight C_TO_O_DISP = $(if $(V),"$(C_TO_O_STR)"," CC $(@)") endif PMDINFO_GEN = $(RTE_SDK_BIN)/app/dpdk-pmdinfogen $@ $@.pmd.c -PMDINFO_CC = $(CC) $(CFLAGS) -c -o $@.pmd.o $@.pmd.c +PMDINFO_CC = $(CC) $(CPPFLAGS) $(CFLAGS) -c -o $@.pmd.o $@.pmd.c PMDINFO_LD = $(CROSS)ld $(LDFLAGS) -r -o $@.o $@.pmd.o $@ -PMDINFO_TO_O = if grep -q 'PMD_REGISTER_DRIVER(.*)' $<; then \ +PMDINFO_TO_O = if grep -q 'RTE_PMD_REGISTER_.*(.*)' $<; then \ echo "$(if $V,$(PMDINFO_GEN), PMDINFO $@.pmd.c)" && \ $(PMDINFO_GEN) && \ echo "$(if $V,$(PMDINFO_CC), CC $@.pmd.o)" && \ diff --git a/dpdk/mk/internal/rte.depdirs-post.mk b/dpdk/mk/internal/rte.depdirs-post.mk deleted file mode 100644 index fc6904dd..00000000 --- a/dpdk/mk/internal/rte.depdirs-post.mk +++ /dev/null @@ -1,42 +0,0 @@ -# BSD LICENSE -# -# Copyright(c) 2010-2014 Intel Corporation. All rights reserved. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in -# the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Intel Corporation nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -.PHONY: depdirs -depdirs: - @for d in $(DEPDIRS-y); do \ - $(RTE_SDK)/scripts/depdirs-rule.sh $(S) $$d ; \ - done - -.PHONY: depgraph -depgraph: - @for d in $(DEPDIRS-y); do \ - echo " \"$(S)\" -> \"$$d\"" ; \ - done diff --git a/dpdk/mk/internal/rte.depdirs-pre.mk b/dpdk/mk/internal/rte.depdirs-pre.mk deleted file mode 100644 index 8825db03..00000000 --- a/dpdk/mk/internal/rte.depdirs-pre.mk +++ /dev/null @@ -1,32 +0,0 @@ -# BSD LICENSE -# -# Copyright(c) 2010-2014 Intel Corporation. All rights reserved. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in -# the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Intel Corporation nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# nothing diff --git a/dpdk/mk/internal/rte.extvars.mk b/dpdk/mk/internal/rte.extvars.mk index e2462fd2..ea41ebfe 100644 --- a/dpdk/mk/internal/rte.extvars.mk +++ b/dpdk/mk/internal/rte.extvars.mk @@ -48,7 +48,7 @@ ifeq ("$(origin M)", "command line") RTE_EXTMK := $(abspath $(M)) endif endif -RTE_EXTMK ?= $(RTE_SRCDIR)/Makefile +RTE_EXTMK ?= $(RTE_SRCDIR)/$(notdir $(firstword $(MAKEFILE_LIST))) export RTE_EXTMK # RTE_SDK_BIN must point to .config, include/ and lib/. diff --git a/dpdk/mk/internal/rte.install-post.mk b/dpdk/mk/internal/rte.install-post.mk index 77addee2..b99e2b2f 100644 --- a/dpdk/mk/internal/rte.install-post.mk +++ b/dpdk/mk/internal/rte.install-post.mk @@ -59,7 +59,7 @@ define symlink_rule $(addprefix $(RTE_OUTPUT)/$(1)/,$(notdir $(2))): $(2) @echo " SYMLINK-FILE $(addprefix $(1)/,$(notdir $(2)))" @[ -d $(RTE_OUTPUT)/$(1) ] || mkdir -p $(RTE_OUTPUT)/$(1) - $(Q)ln -nsf `$(RTE_SDK)/scripts/relpath.sh $$(<) $(RTE_OUTPUT)/$(1)` \ + $(Q)ln -nsf `$(RTE_SDK)/buildtools/relpath.sh $$(<) $(RTE_OUTPUT)/$(1)` \ $(RTE_OUTPUT)/$(1) endef diff --git a/dpdk/mk/machine/armv7a/rte.vars.mk b/dpdk/mk/machine/armv7a/rte.vars.mk index 36fa3dea..41c4c408 100644 --- a/dpdk/mk/machine/armv7a/rte.vars.mk +++ b/dpdk/mk/machine/armv7a/rte.vars.mk @@ -57,7 +57,7 @@ MACHINE_CFLAGS += -march=armv7-a ifdef CONFIG_RTE_ARCH_ARM_TUNE -MACHINE_CFLAGS += -mtune=$(CONFIG_RTE_ARCH_ARM_TUNE) +MACHINE_CFLAGS += -mtune=$(CONFIG_RTE_ARCH_ARM_TUNE:"%"=%) endif MACHINE_CFLAGS += -mfpu=neon diff --git a/dpdk/mk/machine/armv8a/rte.vars.mk b/dpdk/mk/machine/armv8a/rte.vars.mk index d5049e1f..500421a2 100644 --- a/dpdk/mk/machine/armv8a/rte.vars.mk +++ b/dpdk/mk/machine/armv8a/rte.vars.mk @@ -1,6 +1,6 @@ # BSD LICENSE # -# Copyright (C) Cavium networks 2015. All rights reserved. +# Copyright (C) Cavium, Inc 2015. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -12,7 +12,7 @@ # notice, this list of conditions and the following disclaimer in # the documentation and/or other materials provided with the # distribution. -# * Neither the name of Cavium networks nor the names of its +# * Neither the name of Cavium, Inc nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # @@ -55,4 +55,4 @@ # CPU_LDFLAGS = # CPU_ASFLAGS = -MACHINE_CFLAGS += -march=armv8-a+crc +MACHINE_CFLAGS += -march=armv8-a+crc+crypto diff --git a/dpdk/mk/machine/atm/rte.vars.mk b/dpdk/mk/machine/atm/rte.vars.mk deleted file mode 100644 index d6fbba0f..00000000 --- a/dpdk/mk/machine/atm/rte.vars.mk +++ /dev/null @@ -1,58 +0,0 @@ -# BSD LICENSE -# -# Copyright(c) 2010-2014 Intel Corporation. All rights reserved. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in -# the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Intel Corporation nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# -# machine: -# -# - can define ARCH variable (overriden by cmdline value) -# - can define CROSS variable (overriden by cmdline value) -# - define MACHINE_CFLAGS variable (overriden by cmdline value) -# - define MACHINE_LDFLAGS variable (overriden by cmdline value) -# - define MACHINE_ASFLAGS variable (overriden by cmdline value) -# - can define CPU_CFLAGS variable (overriden by cmdline value) that -# overrides the one defined in arch. -# - can define CPU_LDFLAGS variable (overriden by cmdline value) that -# overrides the one defined in arch. -# - can define CPU_ASFLAGS variable (overriden by cmdline value) that -# overrides the one defined in arch. -# - may override any previously defined variable -# - -# ARCH = -# CROSS = -# MACHINE_CFLAGS = -# MACHINE_LDFLAGS = -# MACHINE_ASFLAGS = -# CPU_CFLAGS = -# CPU_LDFLAGS = -# CPU_ASFLAGS = - -MACHINE_CFLAGS = -march=atom diff --git a/dpdk/mk/machine/default/rte.vars.mk b/dpdk/mk/machine/default/rte.vars.mk index 53c6af6e..71a25bca 100644 --- a/dpdk/mk/machine/default/rte.vars.mk +++ b/dpdk/mk/machine/default/rte.vars.mk @@ -32,16 +32,16 @@ # # machine: # -# - can define ARCH variable (overriden by cmdline value) -# - can define CROSS variable (overriden by cmdline value) -# - define MACHINE_CFLAGS variable (overriden by cmdline value) -# - define MACHINE_LDFLAGS variable (overriden by cmdline value) -# - define MACHINE_ASFLAGS variable (overriden by cmdline value) -# - can define CPU_CFLAGS variable (overriden by cmdline value) that +# - can define ARCH variable (overridden by cmdline value) +# - can define CROSS variable (overridden by cmdline value) +# - define MACHINE_CFLAGS variable (overridden by cmdline value) +# - define MACHINE_LDFLAGS variable (overridden by cmdline value) +# - define MACHINE_ASFLAGS variable (overridden by cmdline value) +# - can define CPU_CFLAGS variable (overridden by cmdline value) that # overrides the one defined in arch. -# - can define CPU_LDFLAGS variable (overriden by cmdline value) that +# - can define CPU_LDFLAGS variable (overridden by cmdline value) that # overrides the one defined in arch. -# - can define CPU_ASFLAGS variable (overriden by cmdline value) that +# - can define CPU_ASFLAGS variable (overridden by cmdline value) that # overrides the one defined in arch. # - may override any previously defined variable # @@ -55,4 +55,4 @@ # CPU_LDFLAGS = # CPU_ASFLAGS = -MACHINE_CFLAGS += -march=core2 +MACHINE_CFLAGS += -march=corei7 diff --git a/dpdk/mk/machine/dpaa/rte.vars.mk b/dpdk/mk/machine/dpaa/rte.vars.mk new file mode 100644 index 00000000..356a6af5 --- /dev/null +++ b/dpdk/mk/machine/dpaa/rte.vars.mk @@ -0,0 +1,61 @@ +# BSD LICENSE +# +# Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. +# Copyright 2017 NXP. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of NXP nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# +# machine: +# +# - can define ARCH variable (overridden by cmdline value) +# - can define CROSS variable (overridden by cmdline value) +# - define MACHINE_CFLAGS variable (overridden by cmdline value) +# - define MACHINE_LDFLAGS variable (overridden by cmdline value) +# - define MACHINE_ASFLAGS variable (overridden by cmdline value) +# - can define CPU_CFLAGS variable (overridden by cmdline value) that +# overrides the one defined in arch. +# - can define CPU_LDFLAGS variable (overridden by cmdline value) that +# overrides the one defined in arch. +# - can define CPU_ASFLAGS variable (overridden by cmdline value) that +# overrides the one defined in arch. +# - may override any previously defined variable +# + +# ARCH = +# CROSS = +# MACHINE_CFLAGS = +# MACHINE_LDFLAGS = +# MACHINE_ASFLAGS = +# CPU_CFLAGS = +# CPU_LDFLAGS = +# CPU_ASFLAGS = +MACHINE_CFLAGS += -march=armv8-a+crc + +ifdef CONFIG_RTE_ARCH_ARM_TUNE +MACHINE_CFLAGS += -mtune=$(CONFIG_RTE_ARCH_ARM_TUNE:"%"=%) +endif diff --git a/dpdk/mk/machine/dpaa2/rte.vars.mk b/dpdk/mk/machine/dpaa2/rte.vars.mk index 8541633c..4988f7c8 100644 --- a/dpdk/mk/machine/dpaa2/rte.vars.mk +++ b/dpdk/mk/machine/dpaa2/rte.vars.mk @@ -1,6 +1,7 @@ # BSD LICENSE # -# Copyright(c) 2016 Freescale Semiconductor, Inc. All rights reserved. +# Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. +# Copyright 2016 NXP. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -53,8 +54,8 @@ # CPU_CFLAGS = # CPU_LDFLAGS = # CPU_ASFLAGS = -MACHINE_CFLAGS += -march=armv8-a +MACHINE_CFLAGS += -march=armv8-a+crc ifdef CONFIG_RTE_ARCH_ARM_TUNE -MACHINE_CFLAGS += -mcpu=$(CONFIG_RTE_ARCH_ARM_TUNE) +MACHINE_CFLAGS += -mtune=$(CONFIG_RTE_ARCH_ARM_TUNE:"%"=%) endif diff --git a/dpdk/mk/machine/hsw/rte.vars.mk b/dpdk/mk/machine/hsw/rte.vars.mk index eedc5d02..66a562e7 100644 --- a/dpdk/mk/machine/hsw/rte.vars.mk +++ b/dpdk/mk/machine/hsw/rte.vars.mk @@ -32,16 +32,16 @@ # # machine: # -# - can define ARCH variable (overriden by cmdline value) -# - can define CROSS variable (overriden by cmdline value) -# - define MACHINE_CFLAGS variable (overriden by cmdline value) -# - define MACHINE_LDFLAGS variable (overriden by cmdline value) -# - define MACHINE_ASFLAGS variable (overriden by cmdline value) -# - can define CPU_CFLAGS variable (overriden by cmdline value) that +# - can define ARCH variable (overridden by cmdline value) +# - can define CROSS variable (overridden by cmdline value) +# - define MACHINE_CFLAGS variable (overridden by cmdline value) +# - define MACHINE_LDFLAGS variable (overridden by cmdline value) +# - define MACHINE_ASFLAGS variable (overridden by cmdline value) +# - can define CPU_CFLAGS variable (overridden by cmdline value) that # overrides the one defined in arch. -# - can define CPU_LDFLAGS variable (overriden by cmdline value) that +# - can define CPU_LDFLAGS variable (overridden by cmdline value) that # overrides the one defined in arch. -# - can define CPU_ASFLAGS variable (overriden by cmdline value) that +# - can define CPU_ASFLAGS variable (overridden by cmdline value) that # overrides the one defined in arch. # - may override any previously defined variable # diff --git a/dpdk/mk/machine/ivb/rte.vars.mk b/dpdk/mk/machine/ivb/rte.vars.mk index 932241af..768a25d2 100644 --- a/dpdk/mk/machine/ivb/rte.vars.mk +++ b/dpdk/mk/machine/ivb/rte.vars.mk @@ -32,16 +32,16 @@ # # machine: # -# - can define ARCH variable (overriden by cmdline value) -# - can define CROSS variable (overriden by cmdline value) -# - define MACHINE_CFLAGS variable (overriden by cmdline value) -# - define MACHINE_LDFLAGS variable (overriden by cmdline value) -# - define MACHINE_ASFLAGS variable (overriden by cmdline value) -# - can define CPU_CFLAGS variable (overriden by cmdline value) that +# - can define ARCH variable (overridden by cmdline value) +# - can define CROSS variable (overridden by cmdline value) +# - define MACHINE_CFLAGS variable (overridden by cmdline value) +# - define MACHINE_LDFLAGS variable (overridden by cmdline value) +# - define MACHINE_ASFLAGS variable (overridden by cmdline value) +# - can define CPU_CFLAGS variable (overridden by cmdline value) that # overrides the one defined in arch. -# - can define CPU_LDFLAGS variable (overriden by cmdline value) that +# - can define CPU_LDFLAGS variable (overridden by cmdline value) that # overrides the one defined in arch. -# - can define CPU_ASFLAGS variable (overriden by cmdline value) that +# - can define CPU_ASFLAGS variable (overridden by cmdline value) that # overrides the one defined in arch. # - may override any previously defined variable # diff --git a/dpdk/mk/machine/native/rte.vars.mk b/dpdk/mk/machine/native/rte.vars.mk index 6ce0c723..f7d98d0d 100644 --- a/dpdk/mk/machine/native/rte.vars.mk +++ b/dpdk/mk/machine/native/rte.vars.mk @@ -32,16 +32,16 @@ # # machine: # -# - can define ARCH variable (overriden by cmdline value) -# - can define CROSS variable (overriden by cmdline value) -# - define MACHINE_CFLAGS variable (overriden by cmdline value) -# - define MACHINE_LDFLAGS variable (overriden by cmdline value) -# - define MACHINE_ASFLAGS variable (overriden by cmdline value) -# - can define CPU_CFLAGS variable (overriden by cmdline value) that +# - can define ARCH variable (overridden by cmdline value) +# - can define CROSS variable (overridden by cmdline value) +# - define MACHINE_CFLAGS variable (overridden by cmdline value) +# - define MACHINE_LDFLAGS variable (overridden by cmdline value) +# - define MACHINE_ASFLAGS variable (overridden by cmdline value) +# - can define CPU_CFLAGS variable (overridden by cmdline value) that # overrides the one defined in arch. -# - can define CPU_LDFLAGS variable (overriden by cmdline value) that +# - can define CPU_LDFLAGS variable (overridden by cmdline value) that # overrides the one defined in arch. -# - can define CPU_ASFLAGS variable (overriden by cmdline value) that +# - can define CPU_ASFLAGS variable (overridden by cmdline value) that # overrides the one defined in arch. # - may override any previously defined variable # @@ -63,8 +63,5 @@ MACHINE_CFLAGS = -march=native # so, set the compilation target to be a corei7, minimum target with SSE4.2. SSE42_SUPPORT=$(shell $(CC) -march=native -dM -E - /dev/null) - ifneq ($(CPU_SSE42_SUPPORT),) MACHINE_CFLAGS = -march=corei7 - endif endif diff --git a/dpdk/mk/machine/nhm/rte.vars.mk b/dpdk/mk/machine/nhm/rte.vars.mk index 9566efd7..8921c328 100644 --- a/dpdk/mk/machine/nhm/rte.vars.mk +++ b/dpdk/mk/machine/nhm/rte.vars.mk @@ -32,16 +32,16 @@ # # machine: # -# - can define ARCH variable (overriden by cmdline value) -# - can define CROSS variable (overriden by cmdline value) -# - define MACHINE_CFLAGS variable (overriden by cmdline value) -# - define MACHINE_LDFLAGS variable (overriden by cmdline value) -# - define MACHINE_ASFLAGS variable (overriden by cmdline value) -# - can define CPU_CFLAGS variable (overriden by cmdline value) that +# - can define ARCH variable (overridden by cmdline value) +# - can define CROSS variable (overridden by cmdline value) +# - define MACHINE_CFLAGS variable (overridden by cmdline value) +# - define MACHINE_LDFLAGS variable (overridden by cmdline value) +# - define MACHINE_ASFLAGS variable (overridden by cmdline value) +# - can define CPU_CFLAGS variable (overridden by cmdline value) that # overrides the one defined in arch. -# - can define CPU_LDFLAGS variable (overriden by cmdline value) that +# - can define CPU_LDFLAGS variable (overridden by cmdline value) that # overrides the one defined in arch. -# - can define CPU_ASFLAGS variable (overriden by cmdline value) that +# - can define CPU_ASFLAGS variable (overridden by cmdline value) that # overrides the one defined in arch. # - may override any previously defined variable # diff --git a/dpdk/mk/machine/silvermont/rte.vars.mk b/dpdk/mk/machine/silvermont/rte.vars.mk new file mode 100644 index 00000000..2079d54d --- /dev/null +++ b/dpdk/mk/machine/silvermont/rte.vars.mk @@ -0,0 +1,58 @@ +# BSD LICENSE +# +# Copyright(c) 2010-2014 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# +# machine: +# +# - can define ARCH variable (overridden by cmdline value) +# - can define CROSS variable (overridden by cmdline value) +# - define MACHINE_CFLAGS variable (overridden by cmdline value) +# - define MACHINE_LDFLAGS variable (overridden by cmdline value) +# - define MACHINE_ASFLAGS variable (overridden by cmdline value) +# - can define CPU_CFLAGS variable (overridden by cmdline value) that +# overrides the one defined in arch. +# - can define CPU_LDFLAGS variable (overridden by cmdline value) that +# overrides the one defined in arch. +# - can define CPU_ASFLAGS variable (overridden by cmdline value) that +# overrides the one defined in arch. +# - may override any previously defined variable +# + +# ARCH = +# CROSS = +# MACHINE_CFLAGS = +# MACHINE_LDFLAGS = +# MACHINE_ASFLAGS = +# CPU_CFLAGS = +# CPU_LDFLAGS = +# CPU_ASFLAGS = + +MACHINE_CFLAGS = -march=silvermont diff --git a/dpdk/mk/machine/snb/rte.vars.mk b/dpdk/mk/machine/snb/rte.vars.mk index a9c1b7ca..0709f3d2 100644 --- a/dpdk/mk/machine/snb/rte.vars.mk +++ b/dpdk/mk/machine/snb/rte.vars.mk @@ -32,16 +32,16 @@ # # machine: # -# - can define ARCH variable (overriden by cmdline value) -# - can define CROSS variable (overriden by cmdline value) -# - define MACHINE_CFLAGS variable (overriden by cmdline value) -# - define MACHINE_LDFLAGS variable (overriden by cmdline value) -# - define MACHINE_ASFLAGS variable (overriden by cmdline value) -# - can define CPU_CFLAGS variable (overriden by cmdline value) that +# - can define ARCH variable (overridden by cmdline value) +# - can define CROSS variable (overridden by cmdline value) +# - define MACHINE_CFLAGS variable (overridden by cmdline value) +# - define MACHINE_LDFLAGS variable (overridden by cmdline value) +# - define MACHINE_ASFLAGS variable (overridden by cmdline value) +# - can define CPU_CFLAGS variable (overridden by cmdline value) that # overrides the one defined in arch. -# - can define CPU_LDFLAGS variable (overriden by cmdline value) that +# - can define CPU_LDFLAGS variable (overridden by cmdline value) that # overrides the one defined in arch. -# - can define CPU_ASFLAGS variable (overriden by cmdline value) that +# - can define CPU_ASFLAGS variable (overridden by cmdline value) that # overrides the one defined in arch. # - may override any previously defined variable # diff --git a/dpdk/mk/machine/thunderx/rte.vars.mk b/dpdk/mk/machine/thunderx/rte.vars.mk index 81da1b4c..cae6e3e1 100644 --- a/dpdk/mk/machine/thunderx/rte.vars.mk +++ b/dpdk/mk/machine/thunderx/rte.vars.mk @@ -1,6 +1,6 @@ # BSD LICENSE # -# Copyright (C) Cavium networks 2015. All rights reserved. +# Copyright (C) Cavium, Inc 2015. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -12,7 +12,7 @@ # notice, this list of conditions and the following disclaimer in # the documentation and/or other materials provided with the # distribution. -# * Neither the name of Cavium networks nor the names of its +# * Neither the name of Cavium, Inc nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # @@ -47,7 +47,7 @@ # # ARCH = -CROSS ?= aarch64-thunderx-linux-gnu- +# CROSS = # MACHINE_CFLAGS = # MACHINE_LDFLAGS = # MACHINE_ASFLAGS = @@ -55,4 +55,4 @@ CROSS ?= aarch64-thunderx-linux-gnu- # CPU_LDFLAGS = # CPU_ASFLAGS = -MACHINE_CFLAGS += -march=armv8-a+crc -mcpu=thunderx +MACHINE_CFLAGS += -march=armv8-a+crc+crypto -mcpu=thunderx diff --git a/dpdk/mk/machine/tilegx/rte.vars.mk b/dpdk/mk/machine/tilegx/rte.vars.mk deleted file mode 100644 index c8256f12..00000000 --- a/dpdk/mk/machine/tilegx/rte.vars.mk +++ /dev/null @@ -1,57 +0,0 @@ -# BSD LICENSE -# -# Copyright (C) EZchip Semiconductor Ltd. 2015. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in -# the documentation and/or other materials provided with the -# distribution. -# * Neither the name of EZchip Semiconductor nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# -# machine: -# -# - can define ARCH variable (overridden by cmdline value) -# - can define CROSS variable (overridden by cmdline value) -# - define MACHINE_CFLAGS variable (overridden by cmdline value) -# - define MACHINE_LDFLAGS variable (overridden by cmdline value) -# - define MACHINE_ASFLAGS variable (overridden by cmdline value) -# - can define CPU_CFLAGS variable (overridden by cmdline value) that -# overrides the one defined in arch. -# - can define CPU_LDFLAGS variable (overridden by cmdline value) that -# overrides the one defined in arch. -# - can define CPU_ASFLAGS variable (overridden by cmdline value) that -# overrides the one defined in arch. -# - may override any previously defined variable -# - -# ARCH = -# CROSS = -# MACHINE_CFLAGS = -# MACHINE_LDFLAGS = -# MACHINE_ASFLAGS = -# CPU_CFLAGS = -# CPU_LDFLAGS = -# CPU_ASFLAGS = - -MACHINE_CFLAGS = diff --git a/dpdk/mk/machine/wsm/rte.vars.mk b/dpdk/mk/machine/wsm/rte.vars.mk index c8a266ca..3a2fa2b0 100644 --- a/dpdk/mk/machine/wsm/rte.vars.mk +++ b/dpdk/mk/machine/wsm/rte.vars.mk @@ -32,16 +32,16 @@ # # machine: # -# - can define ARCH variable (overriden by cmdline value) -# - can define CROSS variable (overriden by cmdline value) -# - define MACHINE_CFLAGS variable (overriden by cmdline value) -# - define MACHINE_LDFLAGS variable (overriden by cmdline value) -# - define MACHINE_ASFLAGS variable (overriden by cmdline value) -# - can define CPU_CFLAGS variable (overriden by cmdline value) that +# - can define ARCH variable (overridden by cmdline value) +# - can define CROSS variable (overridden by cmdline value) +# - define MACHINE_CFLAGS variable (overridden by cmdline value) +# - define MACHINE_LDFLAGS variable (overridden by cmdline value) +# - define MACHINE_ASFLAGS variable (overridden by cmdline value) +# - can define CPU_CFLAGS variable (overridden by cmdline value) that # overrides the one defined in arch. -# - can define CPU_LDFLAGS variable (overriden by cmdline value) that +# - can define CPU_LDFLAGS variable (overridden by cmdline value) that # overrides the one defined in arch. -# - can define CPU_ASFLAGS variable (overriden by cmdline value) that +# - can define CPU_ASFLAGS variable (overridden by cmdline value) that # overrides the one defined in arch. # - may override any previously defined variable # diff --git a/dpdk/mk/machine/xgene1/rte.vars.mk b/dpdk/mk/machine/xgene1/rte.vars.mk index 419c2343..07f7994e 100644 --- a/dpdk/mk/machine/xgene1/rte.vars.mk +++ b/dpdk/mk/machine/xgene1/rte.vars.mk @@ -1,6 +1,6 @@ # BSD LICENSE # -# Copyright (C) Cavium networks 2015. All rights reserved. +# Copyright (C) Cavium, Inc 2015. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -12,7 +12,7 @@ # notice, this list of conditions and the following disclaimer in # the documentation and/or other materials provided with the # distribution. -# * Neither the name of Cavium networks nor the names of its +# * Neither the name of Cavium, Inc nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # diff --git a/dpdk/mk/rte.app.mk b/dpdk/mk/rte.app.mk index eb28e115..6a6a7452 100644 --- a/dpdk/mk/rte.app.mk +++ b/dpdk/mk/rte.app.mk @@ -1,6 +1,6 @@ # BSD LICENSE # -# Copyright(c) 2010-2016 Intel Corporation. All rights reserved. +# Copyright(c) 2010-2017 Intel Corporation. All rights reserved. # Copyright(c) 2014-2015 6WIND S.A. # All rights reserved. # @@ -34,7 +34,6 @@ include $(RTE_SDK)/mk/internal/rte.compile-pre.mk include $(RTE_SDK)/mk/internal/rte.install-pre.mk include $(RTE_SDK)/mk/internal/rte.clean-pre.mk include $(RTE_SDK)/mk/internal/rte.build-pre.mk -include $(RTE_SDK)/mk/internal/rte.depdirs-pre.mk # VPATH contains at least SRCDIR VPATH += $(SRCDIR) @@ -59,72 +58,104 @@ _LDLIBS-y += -L$(RTE_SDK_BIN)/lib # # Order is important: from higher level to lower level # - -ifeq ($(CONFIG_RTE_EXEC_ENV_LINUXAPP),y) -_LDLIBS-$(CONFIG_RTE_LIBRTE_KNI) += -lrte_kni -_LDLIBS-$(CONFIG_RTE_LIBRTE_IVSHMEM) += -lrte_ivshmem -endif - +_LDLIBS-$(CONFIG_RTE_LIBRTE_FLOW_CLASSIFY) += -lrte_flow_classify _LDLIBS-$(CONFIG_RTE_LIBRTE_PIPELINE) += -lrte_pipeline _LDLIBS-$(CONFIG_RTE_LIBRTE_TABLE) += -lrte_table _LDLIBS-$(CONFIG_RTE_LIBRTE_PORT) += -lrte_port _LDLIBS-$(CONFIG_RTE_LIBRTE_PDUMP) += -lrte_pdump _LDLIBS-$(CONFIG_RTE_LIBRTE_DISTRIBUTOR) += -lrte_distributor -_LDLIBS-$(CONFIG_RTE_LIBRTE_REORDER) += -lrte_reorder _LDLIBS-$(CONFIG_RTE_LIBRTE_IP_FRAG) += -lrte_ip_frag +_LDLIBS-$(CONFIG_RTE_LIBRTE_GRO) += -lrte_gro +_LDLIBS-$(CONFIG_RTE_LIBRTE_GSO) += -lrte_gso _LDLIBS-$(CONFIG_RTE_LIBRTE_METER) += -lrte_meter -_LDLIBS-$(CONFIG_RTE_LIBRTE_SCHED) += -lrte_sched _LDLIBS-$(CONFIG_RTE_LIBRTE_LPM) += -lrte_lpm # librte_acl needs --whole-archive because of weak functions _LDLIBS-$(CONFIG_RTE_LIBRTE_ACL) += --whole-archive _LDLIBS-$(CONFIG_RTE_LIBRTE_ACL) += -lrte_acl _LDLIBS-$(CONFIG_RTE_LIBRTE_ACL) += --no-whole-archive _LDLIBS-$(CONFIG_RTE_LIBRTE_JOBSTATS) += -lrte_jobstats +_LDLIBS-$(CONFIG_RTE_LIBRTE_METRICS) += -lrte_metrics +_LDLIBS-$(CONFIG_RTE_LIBRTE_BITRATE) += -lrte_bitratestats +_LDLIBS-$(CONFIG_RTE_LIBRTE_LATENCY_STATS) += -lrte_latencystats _LDLIBS-$(CONFIG_RTE_LIBRTE_POWER) += -lrte_power +_LDLIBS-$(CONFIG_RTE_LIBRTE_TIMER) += -lrte_timer +_LDLIBS-$(CONFIG_RTE_LIBRTE_EFD) += -lrte_efd + _LDLIBS-y += --whole-archive -_LDLIBS-$(CONFIG_RTE_LIBRTE_TIMER) += -lrte_timer +_LDLIBS-$(CONFIG_RTE_LIBRTE_CFGFILE) += -lrte_cfgfile _LDLIBS-$(CONFIG_RTE_LIBRTE_HASH) += -lrte_hash +_LDLIBS-$(CONFIG_RTE_LIBRTE_MEMBER) += -lrte_member _LDLIBS-$(CONFIG_RTE_LIBRTE_VHOST) += -lrte_vhost - _LDLIBS-$(CONFIG_RTE_LIBRTE_KVARGS) += -lrte_kvargs _LDLIBS-$(CONFIG_RTE_LIBRTE_MBUF) += -lrte_mbuf -_LDLIBS-$(CONFIG_RTE_LIBRTE_ETHER) += -lethdev +_LDLIBS-$(CONFIG_RTE_LIBRTE_NET) += -lrte_net +_LDLIBS-$(CONFIG_RTE_LIBRTE_ETHER) += -lrte_ethdev _LDLIBS-$(CONFIG_RTE_LIBRTE_CRYPTODEV) += -lrte_cryptodev +_LDLIBS-$(CONFIG_RTE_LIBRTE_SECURITY) += -lrte_security +_LDLIBS-$(CONFIG_RTE_LIBRTE_EVENTDEV) += -lrte_eventdev _LDLIBS-$(CONFIG_RTE_LIBRTE_MEMPOOL) += -lrte_mempool +_LDLIBS-$(CONFIG_RTE_DRIVER_MEMPOOL_RING) += -lrte_mempool_ring _LDLIBS-$(CONFIG_RTE_LIBRTE_RING) += -lrte_ring +_LDLIBS-$(CONFIG_RTE_LIBRTE_PCI) += -lrte_pci _LDLIBS-$(CONFIG_RTE_LIBRTE_EAL) += -lrte_eal _LDLIBS-$(CONFIG_RTE_LIBRTE_CMDLINE) += -lrte_cmdline -_LDLIBS-$(CONFIG_RTE_LIBRTE_CFGFILE) += -lrte_cfgfile +_LDLIBS-$(CONFIG_RTE_LIBRTE_REORDER) += -lrte_reorder +_LDLIBS-$(CONFIG_RTE_LIBRTE_SCHED) += -lrte_sched -_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_BOND) += -lrte_pmd_bond -_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_XENVIRT) += -lrte_pmd_xenvirt -lxenstore +ifeq ($(CONFIG_RTE_EXEC_ENV_LINUXAPP),y) +_LDLIBS-$(CONFIG_RTE_LIBRTE_KNI) += -lrte_kni +endif + +_LDLIBS-$(CONFIG_RTE_LIBRTE_PCI_BUS) += -lrte_bus_pci +_LDLIBS-$(CONFIG_RTE_LIBRTE_VDEV_BUS) += -lrte_bus_vdev ifeq ($(CONFIG_RTE_BUILD_SHARED_LIB),n) # plugins (link only if static libraries) +_LDLIBS-$(CONFIG_RTE_DRIVER_MEMPOOL_STACK) += -lrte_mempool_stack + _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_AF_PACKET) += -lrte_pmd_af_packet +_LDLIBS-$(CONFIG_RTE_LIBRTE_ARK_PMD) += -lrte_pmd_ark +_LDLIBS-$(CONFIG_RTE_LIBRTE_AVP_PMD) += -lrte_pmd_avp _LDLIBS-$(CONFIG_RTE_LIBRTE_BNX2X_PMD) += -lrte_pmd_bnx2x -lz _LDLIBS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += -lrte_pmd_bnxt +_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_BOND) += -lrte_pmd_bond _LDLIBS-$(CONFIG_RTE_LIBRTE_CXGBE_PMD) += -lrte_pmd_cxgbe +ifeq ($(CONFIG_RTE_LIBRTE_DPAA_BUS),y) +_LDLIBS-$(CONFIG_RTE_LIBRTE_DPAA_BUS) += -lrte_bus_dpaa +_LDLIBS-$(CONFIG_RTE_LIBRTE_DPAA_MEMPOOL) += -lrte_mempool_dpaa +_LDLIBS-$(CONFIG_RTE_LIBRTE_DPAA_PMD) += -lrte_pmd_dpaa +endif +_LDLIBS-$(CONFIG_RTE_LIBRTE_DPAA2_PMD) += -lrte_pmd_dpaa2 _LDLIBS-$(CONFIG_RTE_LIBRTE_E1000_PMD) += -lrte_pmd_e1000 _LDLIBS-$(CONFIG_RTE_LIBRTE_ENA_PMD) += -lrte_pmd_ena _LDLIBS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += -lrte_pmd_enic _LDLIBS-$(CONFIG_RTE_LIBRTE_FM10K_PMD) += -lrte_pmd_fm10k +_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_FAILSAFE) += -lrte_pmd_failsafe _LDLIBS-$(CONFIG_RTE_LIBRTE_I40E_PMD) += -lrte_pmd_i40e _LDLIBS-$(CONFIG_RTE_LIBRTE_IXGBE_PMD) += -lrte_pmd_ixgbe -_LDLIBS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += -lrte_pmd_mlx4 -libverbs -_LDLIBS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += -lrte_pmd_mlx5 -libverbs -_LDLIBS-$(CONFIG_RTE_LIBRTE_MPIPE_PMD) += -lrte_pmd_mpipe -lgxio -_LDLIBS-$(CONFIG_RTE_LIBRTE_NFP_PMD) += -lrte_pmd_nfp -lm +ifeq ($(CONFIG_RTE_LIBRTE_KNI),y) +_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_KNI) += -lrte_pmd_kni +endif +_LDLIBS-$(CONFIG_RTE_LIBRTE_LIO_PMD) += -lrte_pmd_lio +_LDLIBS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += -lrte_pmd_mlx4 -libverbs -lmlx4 +_LDLIBS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += -lrte_pmd_mlx5 -libverbs -lmlx5 +_LDLIBS-$(CONFIG_RTE_LIBRTE_MRVL_PMD) += -lrte_pmd_mrvl -L$(LIBMUSDK_PATH)/lib -lmusdk +_LDLIBS-$(CONFIG_RTE_LIBRTE_NFP_PMD) += -lrte_pmd_nfp _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_NULL) += -lrte_pmd_null _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_PCAP) += -lrte_pmd_pcap -lpcap -_LDLIBS-$(CONFIG_RTE_LIBRTE_QEDE_PMD) += -lrte_pmd_qede -lz +_LDLIBS-$(CONFIG_RTE_LIBRTE_QEDE_PMD) += -lrte_pmd_qede _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_RING) += -lrte_pmd_ring +ifeq ($(CONFIG_RTE_LIBRTE_SCHED),y) +_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_SOFTNIC) += -lrte_pmd_softnic +endif +_LDLIBS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += -lrte_pmd_sfc_efx _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_SZEDATA2) += -lrte_pmd_szedata2 -lsze2 -_LDLIBS-$(CONFIG_RTE_LIBRTE_THUNDERX_NICVF_PMD) += -lrte_pmd_thunderx_nicvf -lm +_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_TAP) += -lrte_pmd_tap +_LDLIBS-$(CONFIG_RTE_LIBRTE_THUNDERX_NICVF_PMD) += -lrte_pmd_thunderx_nicvf _LDLIBS-$(CONFIG_RTE_LIBRTE_VIRTIO_PMD) += -lrte_pmd_virtio ifeq ($(CONFIG_RTE_LIBRTE_VHOST),y) _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_VHOST) += -lrte_pmd_vhost @@ -132,18 +163,50 @@ endif # $(CONFIG_RTE_LIBRTE_VHOST) _LDLIBS-$(CONFIG_RTE_LIBRTE_VMXNET3_PMD) += -lrte_pmd_vmxnet3_uio ifeq ($(CONFIG_RTE_LIBRTE_CRYPTODEV),y) -_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_AESNI_MB) += -lrte_pmd_aesni_mb -_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_AESNI_MB) += -L$(AESNI_MULTI_BUFFER_LIB_PATH) -lIPSec_MB -_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_AESNI_GCM) += -lrte_pmd_aesni_gcm -lcrypto -_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_AESNI_GCM) += -L$(AESNI_MULTI_BUFFER_LIB_PATH) -lIPSec_MB +_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_AESNI_MB) += -lrte_pmd_aesni_mb +_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_AESNI_MB) += -L$(AESNI_MULTI_BUFFER_LIB_PATH) -lIPSec_MB +_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_AESNI_GCM) += -lrte_pmd_aesni_gcm +_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_AESNI_GCM) += -L$(AESNI_MULTI_BUFFER_LIB_PATH) -lIPSec_MB +_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_OPENSSL) += -lrte_pmd_openssl -lcrypto _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_NULL_CRYPTO) += -lrte_pmd_null_crypto -_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_QAT) += -lrte_pmd_qat -lcrypto -_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_SNOW3G) += -lrte_pmd_snow3g -_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_SNOW3G) += -L$(LIBSSO_SNOW3G_PATH)/build -lsso_snow3g -_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_KASUMI) += -lrte_pmd_kasumi -_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_KASUMI) += -L$(LIBSSO_KASUMI_PATH)/build -lsso_kasumi +_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_QAT) += -lrte_pmd_qat -lcrypto +_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_SNOW3G) += -lrte_pmd_snow3g +_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_SNOW3G) += -L$(LIBSSO_SNOW3G_PATH)/build -lsso_snow3g +_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_KASUMI) += -lrte_pmd_kasumi +_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_KASUMI) += -L$(LIBSSO_KASUMI_PATH)/build -lsso_kasumi +_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_ZUC) += -lrte_pmd_zuc +_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_ZUC) += -L$(LIBSSO_ZUC_PATH)/build -lsso_zuc +_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_ARMV8_CRYPTO) += -lrte_pmd_armv8 +_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_ARMV8_CRYPTO) += -L$(ARMV8_CRYPTO_LIB_PATH) -larmv8_crypto +_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_MRVL_CRYPTO) += -L$(LIBMUSDK_PATH)/lib -lrte_pmd_mrvl_crypto -lmusdk +_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_CRYPTO_SCHEDULER) += -lrte_pmd_crypto_scheduler +ifeq ($(CONFIG_RTE_LIBRTE_FSLMC_BUS),y) +_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_SEC) += -lrte_pmd_dpaa2_sec +_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_SEC) += -lrte_mempool_dpaa2 +_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_SEC) += -lrte_bus_fslmc +endif # CONFIG_RTE_LIBRTE_FSLMC_BUS + +ifeq ($(CONFIG_RTE_LIBRTE_DPAA_BUS),y) +_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_DPAA_SEC) += -lrte_bus_dpaa +_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_DPAA_SEC) += -lrte_pmd_dpaa_sec +endif # CONFIG_RTE_LIBRTE_DPAA_BUS + endif # CONFIG_RTE_LIBRTE_CRYPTODEV +ifeq ($(CONFIG_RTE_LIBRTE_EVENTDEV),y) +_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_SKELETON_EVENTDEV) += -lrte_pmd_skeleton_event +_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_SW_EVENTDEV) += -lrte_pmd_sw_event +_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX_SSOVF) += -lrte_pmd_octeontx_ssovf +_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_EVENTDEV) += -lrte_pmd_dpaa2_event +_LDLIBS-$(CONFIG_RTE_LIBRTE_OCTEONTX_MEMPOOL) += -lrte_mempool_octeontx +_LDLIBS-$(CONFIG_RTE_LIBRTE_OCTEONTX_PMD) += -lrte_pmd_octeontx +endif # CONFIG_RTE_LIBRTE_EVENTDEV + +ifeq ($(CONFIG_RTE_LIBRTE_DPAA2_PMD),y) +_LDLIBS-$(CONFIG_RTE_LIBRTE_DPAA2_PMD) += -lrte_bus_fslmc +_LDLIBS-$(CONFIG_RTE_LIBRTE_DPAA2_PMD) += -lrte_mempool_dpaa2 +endif # CONFIG_RTE_LIBRTE_DPAA2_PMD + endif # !CONFIG_RTE_BUILD_SHARED_LIBS _LDLIBS-y += --no-whole-archive @@ -152,15 +215,16 @@ ifeq ($(CONFIG_RTE_BUILD_SHARED_LIB),n) # The static libraries do not know their dependencies. # So linking with static library requires explicit dependencies. _LDLIBS-$(CONFIG_RTE_LIBRTE_EAL) += -lrt +ifeq ($(CONFIG_RTE_EXEC_ENV_LINUXAPP)$(CONFIG_RTE_EAL_NUMA_AWARE_HUGEPAGES),yy) +_LDLIBS-$(CONFIG_RTE_LIBRTE_EAL) += -lnuma +endif _LDLIBS-$(CONFIG_RTE_LIBRTE_SCHED) += -lm _LDLIBS-$(CONFIG_RTE_LIBRTE_SCHED) += -lrt +_LDLIBS-$(CONFIG_RTE_LIBRTE_MEMBER) += -lm _LDLIBS-$(CONFIG_RTE_LIBRTE_METER) += -lm ifeq ($(CONFIG_RTE_LIBRTE_VHOST_NUMA),y) _LDLIBS-$(CONFIG_RTE_LIBRTE_VHOST) += -lnuma endif -ifeq ($(CONFIG_RTE_LIBRTE_VHOST_USER),n) -_LDLIBS-$(CONFIG_RTE_LIBRTE_VHOST) += -lfuse -endif _LDLIBS-$(CONFIG_RTE_PORT_PCAP) += -lpcap endif # !CONFIG_RTE_BUILD_SHARED_LIBS @@ -168,10 +232,21 @@ _LDLIBS-y += $(EXECENV_LDLIBS) LDLIBS += $(_LDLIBS-y) $(CPU_LDLIBS) $(EXTRA_LDLIBS) -# Eliminate duplicates without sorting -LDLIBS := $(shell echo $(LDLIBS) | \ - awk '{for (i = 1; i <= NF; i++) { \ - if ($$i !~ /^-l.*/ || !seen[$$i]++) print $$i }}') +# all the words except the first one +allbutfirst = $(wordlist 2,$(words $(1)),$(1)) + +# Eliminate duplicates without sorting, only keep the last occurrence +filter-libs = \ + $(if $(1),$(strip\ + $(if \ + $(and \ + $(filter $(firstword $(1)),$(call allbutfirst,$(1))),\ + $(filter -l%,$(firstword $(1)))),\ + ,\ + $(firstword $(1))) \ + $(call filter-libs,$(call allbutfirst,$(1))))) + +LDLIBS := $(call filter-libs,$(LDLIBS)) ifeq ($(RTE_DEVEL_BUILD)$(CONFIG_RTE_BUILD_SHARED_LIB),yy) LDFLAGS += -rpath=$(RTE_SDK_BIN)/lib @@ -193,7 +268,7 @@ build: _postbuild exe2cmd = $(strip $(call dotfile,$(patsubst %,%.cmd,$(1)))) ifeq ($(LINK_USING_CC),1) -O_TO_EXE = $(CC) -o $@ $(CFLAGS) $(OBJS-y) $(call linkerprefix, \ +O_TO_EXE = $(CC) -o $@ $(CFLAGS) $(EXTRA_CFLAGS) $(OBJS-y) $(call linkerprefix, \ $(LDLIBS) $(LDFLAGS) $(LDFLAGS_$(@)) $(EXTRA_LDFLAGS) \ $(MAPFLAGS)) else @@ -221,8 +296,8 @@ LDLIBS_NAMES += $(patsubst -Wl$(comma)-l%,lib%.a,$(filter -Wl$(comma)-l%,$(LDLIB # list of found libraries files (useful for deps). If not found, the # library is silently ignored and dep won't be checked -LDLIBS_FILES := $(wildcard $(foreach dir,$(LDLIBS_PATH),\ - $(addprefix $(dir)/,$(LDLIBS_NAMES)))) +LDLIBS_FILES := $(sort $(wildcard $(foreach dir,$(LDLIBS_PATH),\ + $(addprefix $(dir)/,$(LDLIBS_NAMES))))) # # Compile executable file if needed @@ -268,14 +343,13 @@ clean: _postclean .PHONY: doclean doclean: $(Q)rm -rf $(APP) $(OBJS-all) $(DEPS-all) $(DEPSTMP-all) \ - $(CMDS-all) $(INSTALL-FILES-all) .$(APP).cmd + $(CMDS-all) $(INSTALL-FILES-all) .$(APP).cmd $(APP).map include $(RTE_SDK)/mk/internal/rte.compile-post.mk include $(RTE_SDK)/mk/internal/rte.install-post.mk include $(RTE_SDK)/mk/internal/rte.clean-post.mk include $(RTE_SDK)/mk/internal/rte.build-post.mk -include $(RTE_SDK)/mk/internal/rte.depdirs-post.mk ifneq ($(wildcard $(RTE_SDK)/mk/target/$(RTE_TARGET)/rte.app.mk),) include $(RTE_SDK)/mk/target/$(RTE_TARGET)/rte.app.mk diff --git a/dpdk/mk/rte.bsdmodule.mk b/dpdk/mk/rte.bsdmodule.mk index 86b92ff5..6fc137ad 100644 --- a/dpdk/mk/rte.bsdmodule.mk +++ b/dpdk/mk/rte.bsdmodule.mk @@ -43,7 +43,6 @@ else include $(RTE_SDK)/mk/internal/rte.install-pre.mk include $(RTE_SDK)/mk/internal/rte.clean-pre.mk include $(RTE_SDK)/mk/internal/rte.build-pre.mk -include $(RTE_SDK)/mk/internal/rte.depdirs-pre.mk # DPDK uses a more up-to-date gcc, so clear the override here. unexport CC @@ -111,7 +110,6 @@ doclean: include $(RTE_SDK)/mk/internal/rte.install-post.mk include $(RTE_SDK)/mk/internal/rte.clean-post.mk include $(RTE_SDK)/mk/internal/rte.build-post.mk -include $(RTE_SDK)/mk/internal/rte.depdirs-post.mk .PHONY: FORCE FORCE: diff --git a/dpdk/mk/rte.combinedlib.mk b/dpdk/mk/rte.combinedlib.mk index 449358b3..2ab7ee8a 100644 --- a/dpdk/mk/rte.combinedlib.mk +++ b/dpdk/mk/rte.combinedlib.mk @@ -42,7 +42,7 @@ endif RTE_LIBNAME := dpdk COMBINEDLIB := lib$(RTE_LIBNAME)$(EXT) -LIBS := $(filter-out $(COMBINEDLIB), $(notdir $(wildcard $(RTE_OUTPUT)/lib/*$(EXT)))) +LIBS := $(filter-out $(COMBINEDLIB), $(sort $(notdir $(wildcard $(RTE_OUTPUT)/lib/*$(EXT))))) all: FORCE $(Q)echo "GROUP ( $(LIBS) )" > $(RTE_OUTPUT)/lib/$(COMBINEDLIB) diff --git a/dpdk/mk/rte.cpuflags.mk b/dpdk/mk/rte.cpuflags.mk index e634abc3..a813c91f 100644 --- a/dpdk/mk/rte.cpuflags.mk +++ b/dpdk/mk/rte.cpuflags.mk @@ -70,8 +70,10 @@ CPUFLAGS += PCLMULQDQ endif ifneq ($(filter $(AUTO_CPUFLAGS),__AVX__),) +ifeq ($(CONFIG_RTE_ENABLE_AVX),y) CPUFLAGS += AVX endif +endif ifneq ($(filter $(AUTO_CPUFLAGS),__RDRND__),) CPUFLAGS += RDRAND @@ -86,12 +88,16 @@ CPUFLAGS += F16C endif ifneq ($(filter $(AUTO_CPUFLAGS),__AVX2__),) +ifeq ($(CONFIG_RTE_ENABLE_AVX),y) CPUFLAGS += AVX2 endif +endif ifneq ($(filter $(AUTO_CPUFLAGS),__AVX512F__),) +ifeq ($(CONFIG_RTE_ENABLE_AVX512),y) CPUFLAGS += AVX512F endif +endif # IBM Power CPU flags ifneq ($(filter $(AUTO_CPUFLAGS),__PPC64__),) @@ -119,6 +125,12 @@ ifneq ($(filter $(AUTO_CPUFLAGS),__ARM_FEATURE_CRC32),) CPUFLAGS += CRC32 endif +ifneq ($(filter $(AUTO_CPUFLAGS),__ARM_FEATURE_CRYPTO),) +CPUFLAGS += AES +CPUFLAGS += PMULL +CPUFLAGS += SHA1 +CPUFLAGS += SHA2 +endif MACHINE_CFLAGS += $(addprefix -DRTE_MACHINE_CPUFLAG_,$(CPUFLAGS)) diff --git a/dpdk/mk/rte.extsubdir.mk b/dpdk/mk/rte.extsubdir.mk index f50f0062..e2ef0132 100644 --- a/dpdk/mk/rte.extsubdir.mk +++ b/dpdk/mk/rte.extsubdir.mk @@ -30,9 +30,11 @@ MAKEFLAGS += --no-print-directory +ALL_DEPDIRS := $(patsubst DEPDIRS-%,%,$(filter DEPDIRS-%,$(.VARIABLES))) + # output directory -O ?= . -BASE_OUTPUT ?= $(O) +O ?= $(CURDIR) +BASE_OUTPUT ?= $(abspath $(O)) CUR_SUBDIR ?= . .PHONY: all @@ -51,3 +53,14 @@ $(DIRS-y): CUR_SUBDIR=$(CUR_SUBDIR)/$(@) \ S=$(CURDIR)/$(@) \ $(filter-out $(DIRS-y),$(MAKECMDGOALS)) + +define depdirs_rule +$(DEPDIRS-$(1)): + +$(1): | $(DEPDIRS-$(1)) + +$(if $(D),$(info $(1) depends on $(DEPDIRS-$(1)))) +endef + +$(foreach dir,$(ALL_DEPDIRS),\ + $(eval $(call depdirs_rule,$(dir)))) diff --git a/dpdk/mk/rte.gnuconfigure.mk b/dpdk/mk/rte.gnuconfigure.mk index 65b658c1..b5c8df0f 100644 --- a/dpdk/mk/rte.gnuconfigure.mk +++ b/dpdk/mk/rte.gnuconfigure.mk @@ -32,7 +32,6 @@ include $(RTE_SDK)/mk/internal/rte.build-pre.mk include $(RTE_SDK)/mk/internal/rte.install-pre.mk include $(RTE_SDK)/mk/internal/rte.clean-pre.mk -include $(RTE_SDK)/mk/internal/rte.depdirs-pre.mk # VPATH contains at least SRCDIR VPATH += $(SRCDIR) @@ -68,7 +67,6 @@ doclean: include $(RTE_SDK)/mk/internal/rte.build-post.mk include $(RTE_SDK)/mk/internal/rte.install-post.mk include $(RTE_SDK)/mk/internal/rte.clean-post.mk -include $(RTE_SDK)/mk/internal/rte.depdirs-post.mk .PHONY: FORCE FORCE: diff --git a/dpdk/mk/rte.hostapp.mk b/dpdk/mk/rte.hostapp.mk index 07b391c2..f58173c3 100644 --- a/dpdk/mk/rte.hostapp.mk +++ b/dpdk/mk/rte.hostapp.mk @@ -35,7 +35,6 @@ include $(RTE_SDK)/mk/internal/rte.compile-pre.mk include $(RTE_SDK)/mk/internal/rte.install-pre.mk include $(RTE_SDK)/mk/internal/rte.clean-pre.mk include $(RTE_SDK)/mk/internal/rte.build-pre.mk -include $(RTE_SDK)/mk/internal/rte.depdirs-pre.mk # VPATH contains at least SRCDIR VPATH += $(SRCDIR) @@ -70,9 +69,9 @@ O_TO_EXE_DO = @set -e; \ -include .$(HOSTAPP).cmd # list of .a files that are linked to this application -LDLIBS_FILES := $(wildcard \ +LDLIBS_FILES := $(sort $(wildcard \ $(addprefix $(RTE_OUTPUT)/lib/, \ - $(patsubst -l%,lib%.a,$(filter -l%,$(LDLIBS))))) + $(patsubst -l%,lib%.a,$(filter -l%,$(LDLIBS)))))) # # Compile executable file if needed @@ -117,7 +116,6 @@ include $(RTE_SDK)/mk/internal/rte.compile-post.mk include $(RTE_SDK)/mk/internal/rte.install-post.mk include $(RTE_SDK)/mk/internal/rte.clean-post.mk include $(RTE_SDK)/mk/internal/rte.build-post.mk -include $(RTE_SDK)/mk/internal/rte.depdirs-post.mk .PHONY: FORCE FORCE: diff --git a/dpdk/mk/rte.hostlib.mk b/dpdk/mk/rte.hostlib.mk index fe24049f..7709cff7 100644 --- a/dpdk/mk/rte.hostlib.mk +++ b/dpdk/mk/rte.hostlib.mk @@ -35,7 +35,6 @@ include $(RTE_SDK)/mk/internal/rte.compile-pre.mk include $(RTE_SDK)/mk/internal/rte.install-pre.mk include $(RTE_SDK)/mk/internal/rte.clean-pre.mk include $(RTE_SDK)/mk/internal/rte.build-pre.mk -include $(RTE_SDK)/mk/internal/rte.depdirs-pre.mk # VPATH contains at least SRCDIR VPATH += $(SRCDIR) @@ -110,7 +109,6 @@ include $(RTE_SDK)/mk/internal/rte.compile-post.mk include $(RTE_SDK)/mk/internal/rte.install-post.mk include $(RTE_SDK)/mk/internal/rte.clean-post.mk include $(RTE_SDK)/mk/internal/rte.build-post.mk -include $(RTE_SDK)/mk/internal/rte.depdirs-post.mk .PHONY: FORCE FORCE: diff --git a/dpdk/mk/rte.install.mk b/dpdk/mk/rte.install.mk index e7ac4d5b..96144fbb 100644 --- a/dpdk/mk/rte.install.mk +++ b/dpdk/mk/rte.install.mk @@ -33,7 +33,6 @@ include $(RTE_SDK)/mk/internal/rte.install-pre.mk include $(RTE_SDK)/mk/internal/rte.clean-pre.mk -include $(RTE_SDK)/mk/internal/rte.depdirs-pre.mk # VPATH contains at least SRCDIR VPATH += $(SRCDIR) @@ -55,4 +54,3 @@ doclean: include $(RTE_SDK)/mk/internal/rte.install-post.mk include $(RTE_SDK)/mk/internal/rte.clean-post.mk -include $(RTE_SDK)/mk/internal/rte.depdirs-post.mk diff --git a/dpdk/mk/rte.lib.mk b/dpdk/mk/rte.lib.mk index 830f81a6..d0979a52 100644 --- a/dpdk/mk/rte.lib.mk +++ b/dpdk/mk/rte.lib.mk @@ -33,33 +33,46 @@ include $(RTE_SDK)/mk/internal/rte.compile-pre.mk include $(RTE_SDK)/mk/internal/rte.install-pre.mk include $(RTE_SDK)/mk/internal/rte.clean-pre.mk include $(RTE_SDK)/mk/internal/rte.build-pre.mk -include $(RTE_SDK)/mk/internal/rte.depdirs-pre.mk EXTLIB_BUILD ?= n # VPATH contains at least SRCDIR VPATH += $(SRCDIR) +ifneq ($(CONFIG_RTE_MAJOR_ABI),) +ifneq ($(LIBABIVER),) +LIBABIVER := $(CONFIG_RTE_MAJOR_ABI) +endif +endif + ifeq ($(CONFIG_RTE_BUILD_SHARED_LIB),y) LIB := $(patsubst %.a,%.so.$(LIBABIVER),$(LIB)) ifeq ($(EXTLIB_BUILD),n) +ifeq ($(CONFIG_RTE_MAJOR_ABI),) ifeq ($(CONFIG_RTE_NEXT_ABI),y) LIB := $(LIB).1 endif +endif CPU_LDFLAGS += --version-script=$(SRCDIR)/$(EXPORT_MAP) endif endif _BUILD = $(LIB) -_INSTALL = $(INSTALL-FILES-y) $(SYMLINK-FILES-y) $(RTE_OUTPUT)/lib/$(LIB) +PREINSTALL = $(SYMLINK-FILES-y) +_INSTALL = $(INSTALL-FILES-y) $(RTE_OUTPUT)/lib/$(LIB) _CLEAN = doclean .PHONY: all all: install .PHONY: install +ifeq ($(SYMLINK-FILES-y),) install: build _postinstall +else +install: _preinstall build _postinstall +build: _preinstall +endif _postinstall: build @@ -70,20 +83,13 @@ exe2cmd = $(strip $(call dotfile,$(patsubst %,%.cmd,$(1)))) ifeq ($(LINK_USING_CC),1) # Override the definition of LD here, since we're linking with CC -LD := $(CC) $(CPU_CFLAGS) +LD := $(CC) $(CPU_CFLAGS) $(EXTRA_CFLAGS) _CPU_LDFLAGS := $(call linkerprefix,$(CPU_LDFLAGS)) override EXTRA_LDFLAGS := $(call linkerprefix,$(EXTRA_LDFLAGS)) else _CPU_LDFLAGS := $(CPU_LDFLAGS) endif -# Translate DEPDIRS-y into LDLIBS -# Ignore (sub)directory dependencies which do not provide an actual library -_IGNORE_DIRS = lib/librte_eal/% lib/librte_net lib/librte_compat -_DEPDIRS = $(filter-out $(_IGNORE_DIRS),$(DEPDIRS-y)) -_LDDIRS = $(subst librte_ether,libethdev,$(_DEPDIRS)) -LDLIBS += $(subst lib/lib,-l,$(_LDDIRS)) - O_TO_A = $(AR) crDs $(LIB) $(OBJS-y) O_TO_A_STR = $(subst ','\'',$(O_TO_A)) #'# fix syntax highlight O_TO_A_DISP = $(if $(V),"$(O_TO_A_STR)"," AR $(@)") @@ -156,11 +162,7 @@ $(RTE_OUTPUT)/lib/$(LIB): $(LIB) @[ -d $(RTE_OUTPUT)/lib ] || mkdir -p $(RTE_OUTPUT)/lib $(Q)cp -f $(LIB) $(RTE_OUTPUT)/lib ifeq ($(CONFIG_RTE_BUILD_SHARED_LIB),y) -ifeq ($(CONFIG_RTE_NEXT_ABI)$(EXTLIB_BUILD),yn) - $(Q)ln -s -f $< $(basename $(basename $@)) -else - $(Q)ln -s -f $< $(basename $@) -endif + $(Q)ln -s -f $< $(shell echo $@ | sed 's/\.so.*/.so/') endif # @@ -172,14 +174,13 @@ clean: _postclean .PHONY: doclean doclean: $(Q)rm -rf $(LIB) $(OBJS-all) $(DEPS-all) $(DEPSTMP-all) \ - $(CMDS-all) $(INSTALL-FILES-all) + $(CMDS-all) .$(LIB).cmd $(INSTALL-FILES-all) *.pmd.c *.pmd.o $(Q)rm -f $(_BUILD_TARGETS) $(_INSTALL_TARGETS) $(_CLEAN_TARGETS) include $(RTE_SDK)/mk/internal/rte.compile-post.mk include $(RTE_SDK)/mk/internal/rte.install-post.mk include $(RTE_SDK)/mk/internal/rte.clean-post.mk include $(RTE_SDK)/mk/internal/rte.build-post.mk -include $(RTE_SDK)/mk/internal/rte.depdirs-post.mk .PHONY: FORCE FORCE: diff --git a/dpdk/mk/rte.module.mk b/dpdk/mk/rte.module.mk index 53ed4fe9..3dd9ac78 100644 --- a/dpdk/mk/rte.module.mk +++ b/dpdk/mk/rte.module.mk @@ -43,7 +43,6 @@ else include $(RTE_SDK)/mk/internal/rte.install-pre.mk include $(RTE_SDK)/mk/internal/rte.clean-pre.mk include $(RTE_SDK)/mk/internal/rte.build-pre.mk -include $(RTE_SDK)/mk/internal/rte.depdirs-pre.mk # VPATH contains at least SRCDIR VPATH += $(SRCDIR) @@ -108,7 +107,6 @@ doclean: include $(RTE_SDK)/mk/internal/rte.install-post.mk include $(RTE_SDK)/mk/internal/rte.clean-post.mk include $(RTE_SDK)/mk/internal/rte.build-post.mk -include $(RTE_SDK)/mk/internal/rte.depdirs-post.mk .PHONY: FORCE FORCE: diff --git a/dpdk/mk/rte.obj.mk b/dpdk/mk/rte.obj.mk index 5982227d..9336d5f8 100644 --- a/dpdk/mk/rte.obj.mk +++ b/dpdk/mk/rte.obj.mk @@ -33,7 +33,6 @@ include $(RTE_SDK)/mk/internal/rte.compile-pre.mk include $(RTE_SDK)/mk/internal/rte.install-pre.mk include $(RTE_SDK)/mk/internal/rte.clean-pre.mk include $(RTE_SDK)/mk/internal/rte.build-pre.mk -include $(RTE_SDK)/mk/internal/rte.depdirs-pre.mk # VPATH contains at least SRCDIR VPATH += $(SRCDIR) @@ -106,7 +105,6 @@ include $(RTE_SDK)/mk/internal/rte.compile-post.mk include $(RTE_SDK)/mk/internal/rte.install-post.mk include $(RTE_SDK)/mk/internal/rte.clean-post.mk include $(RTE_SDK)/mk/internal/rte.build-post.mk -include $(RTE_SDK)/mk/internal/rte.depdirs-post.mk .PHONY: FORCE FORCE: diff --git a/dpdk/mk/rte.sdkbuild.mk b/dpdk/mk/rte.sdkbuild.mk index 23fcf1e1..f6068bb9 100644 --- a/dpdk/mk/rte.sdkbuild.mk +++ b/dpdk/mk/rte.sdkbuild.mk @@ -38,18 +38,13 @@ else include $(RTE_SDK)/mk/rte.vars.mk endif -# -# include .depdirs and define rules to order priorities between build -# of directories. -# --include $(RTE_OUTPUT)/.depdirs +# allow exec-env specific targets +-include $(RTE_SDK)/mk/exec-env/$(RTE_EXEC_ENV)/rte.custom.mk -define depdirs_rule -$(1): $(sort $(LOCAL_DEPDIRS-$(1))) -endef - -$(foreach d,$(ROOTDIRS-y),$(eval $(call depdirs_rule,$(d)))) -drivers: | buildtools +buildtools: | lib +drivers: | lib buildtools +app: | lib buildtools drivers +test: | lib buildtools drivers # # build and clean targets @@ -67,14 +62,17 @@ clean: $(CLEANDIRS) $(RTE_OUTPUT)/lib \ $(RTE_OUTPUT)/hostlib $(RTE_OUTPUT)/kmod @[ -d $(RTE_OUTPUT)/include ] || mkdir -p $(RTE_OUTPUT)/include - @$(RTE_SDK)/scripts/gen-config-h.sh $(RTE_OUTPUT)/.config \ + @$(RTE_SDK)/buildtools/gen-config-h.sh $(RTE_OUTPUT)/.config \ > $(RTE_OUTPUT)/include/rte_config.h $(Q)$(MAKE) -f $(RTE_SDK)/GNUmakefile gcovclean @echo Clean complete +.PHONY: test-build +test-build: test + .SECONDEXPANSION: -.PHONY: $(ROOTDIRS-y) -$(ROOTDIRS-y): +.PHONY: $(ROOTDIRS-y) $(ROOTDIRS-) +$(ROOTDIRS-y) $(ROOTDIRS-): @[ -d $(BUILDDIR)/$@ ] || mkdir -p $(BUILDDIR)/$@ @echo "== Build $@" $(Q)$(MAKE) S=$@ -f $(RTE_SRCDIR)/$@/Makefile -C $(BUILDDIR)/$@ all @@ -90,8 +88,8 @@ $(ROOTDIRS-y): RTE_MAKE_SUBTARGET ?= all -%_sub: $(addsuffix _sub,$(FULL_DEPDIRS-$(*))) - @echo $(addsuffix _sub,$(FULL_DEPDIRS-$(*))) +%_sub: $(addsuffix _sub,$(*)) + @echo $(addsuffix _sub,$(*)) @[ -d $(BUILDDIR)/$* ] || mkdir -p $(BUILDDIR)/$* @echo "== Build $*" $(Q)$(MAKE) S=$* -f $(RTE_SRCDIR)/$*/Makefile -C $(BUILDDIR)/$* \ diff --git a/dpdk/mk/rte.sdkconfig.mk b/dpdk/mk/rte.sdkconfig.mk index 5c94edf8..97363416 100644 --- a/dpdk/mk/rte.sdkconfig.mk +++ b/dpdk/mk/rte.sdkconfig.mk @@ -60,17 +60,38 @@ showconfigs: .PHONY: notemplate notemplate: - @printf "No template specified. " - @echo "Use T=template among the following list:" + @printf "No template specified. Use 'make defconfig' or " + @echo "use T=template from the following list:" @$(MAKE) -rR showconfigs | sed 's,^, ,' + +.PHONY: defconfig +defconfig: + @$(MAKE) config T=$(shell \ + uname -m | awk '{ \ + if ($$0 == "aarch64") { \ + print "arm64-armv8a"} \ + else if ($$0 == "armv7l") { \ + print "arm-armv7a"} \ + else if ($$0 == "ppc64") { \ + print "ppc_64-power8"} \ + else { \ + printf "%s-native", $$0} }')-$(shell \ + uname | awk '{ \ + if ($$0 == "Linux") { \ + print "linuxapp"} \ + else { \ + print "bsdapp"} }')-$(shell \ + ${CC} -v 2>&1 | \ + grep " version " | cut -d ' ' -f 1) + .PHONY: config ifeq ($(RTE_CONFIG_TEMPLATE),) config: notemplate else config: $(RTE_OUTPUT)/include/rte_config.h $(RTE_OUTPUT)/Makefile - $(Q)$(MAKE) depdirs - @echo "Configuration done" + @echo "Configuration done using" \ + $(patsubst defconfig_%,%,$(notdir $(RTE_CONFIG_TEMPLATE))) endif $(RTE_OUTPUT): @@ -107,12 +128,12 @@ endif # generate a Makefile for this build directory # use a relative path so it will continue to work even if we move the directory -SDK_RELPATH=$(shell $(RTE_SDK)/scripts/relpath.sh $(abspath $(RTE_SRCDIR)) \ +SDK_RELPATH=$(shell $(RTE_SDK)/buildtools/relpath.sh $(abspath $(RTE_SRCDIR)) \ $(abspath $(RTE_OUTPUT))) -OUTPUT_RELPATH=$(shell $(RTE_SDK)/scripts/relpath.sh $(abspath $(RTE_OUTPUT)) \ +OUTPUT_RELPATH=$(shell $(RTE_SDK)/buildtools/relpath.sh $(abspath $(RTE_OUTPUT)) \ $(abspath $(RTE_SRCDIR))) $(RTE_OUTPUT)/Makefile: | $(RTE_OUTPUT) - $(Q)$(RTE_SDK)/scripts/gen-build-mk.sh $(SDK_RELPATH) $(OUTPUT_RELPATH) \ + $(Q)$(RTE_SDK)/buildtools/gen-build-mk.sh $(SDK_RELPATH) $(OUTPUT_RELPATH) \ > $(RTE_OUTPUT)/Makefile # clean installed files, and generate a new config header file @@ -122,7 +143,7 @@ $(RTE_OUTPUT)/include/rte_config.h: $(RTE_OUTPUT)/.config $(RTE_OUTPUT)/lib \ $(RTE_OUTPUT)/hostlib $(RTE_OUTPUT)/kmod $(RTE_OUTPUT)/build $(Q)mkdir -p $(RTE_OUTPUT)/include - $(Q)$(RTE_SDK)/scripts/gen-config-h.sh $(RTE_OUTPUT)/.config \ + $(Q)$(RTE_SDK)/buildtools/gen-config-h.sh $(RTE_OUTPUT)/.config \ > $(RTE_OUTPUT)/include/rte_config.h # generate the rte_config.h @@ -140,7 +161,6 @@ checkconfig: fi $(Q)$(MAKE) -f $(RTE_SDK)/mk/rte.sdkconfig.mk \ headerconfig NODOTCONF=1 - $(Q)$(MAKE) -s depdirs .PHONY: FORCE FORCE: diff --git a/dpdk/mk/rte.sdkdepdirs.mk b/dpdk/mk/rte.sdkdepdirs.mk index bebaf2a2..1f27697b 100644 --- a/dpdk/mk/rte.sdkdepdirs.mk +++ b/dpdk/mk/rte.sdkdepdirs.mk @@ -35,30 +35,3 @@ endif ifeq (,$(wildcard $(RTE_OUTPUT)/Makefile)) $(error "need a make config first") endif - -# use a "for" in a shell to process dependencies: we don't want this -# task to be run in parallel. -.PHONY: depdirs -depdirs: $(RTE_OUTPUT)/.depdirs -$(RTE_OUTPUT)/.depdirs: $(RTE_OUTPUT)/.config - @rm -f $(RTE_OUTPUT)/.depdirs ; \ - for d in $(ROOTDIRS-y); do \ - if [ -f $(RTE_SRCDIR)/$$d/Makefile ]; then \ - [ -d $(BUILDDIR)/$$d ] || mkdir -p $(BUILDDIR)/$$d ; \ - $(MAKE) S=$$d -f $(RTE_SRCDIR)/$$d/Makefile depdirs \ - >> $(RTE_OUTPUT)/.depdirs ; \ - fi ; \ - done - -.PHONY: depgraph -depgraph: - @echo "digraph unix {" ; \ - echo " size=\"6,6\";" ; \ - echo " node [color=lightblue2, style=filled];" ; \ - for d in $(ROOTDIRS-y); do \ - echo " \"root\" -> \"$$d\"" ; \ - if [ -f $(RTE_SRCDIR)/$$d/Makefile ]; then \ - $(MAKE) S=$$d -f $(RTE_SRCDIR)/$$d/Makefile depgraph ; \ - fi ; \ - done ; \ - echo "}" diff --git a/dpdk/mk/rte.sdkdoc.mk b/dpdk/mk/rte.sdkdoc.mk index 9952f258..de31b78c 100644 --- a/dpdk/mk/rte.sdkdoc.mk +++ b/dpdk/mk/rte.sdkdoc.mk @@ -54,6 +54,8 @@ RTE_PDF_DPI ?= 300 RTE_GUIDES := $(filter %/, $(wildcard $(RTE_SDK)/doc/guides/*/)) +API_EXAMPLES := $(RTE_OUTPUT)/doc/html/examples.dox + .PHONY: help help: @cat $(RTE_SDK)/doc/build-sdk-quick.txt @@ -63,15 +65,16 @@ help: all: api-html guides-html guides-pdf .PHONY: clean -clean: api-html-clean guides-html-clean guides-pdf-clean +clean: api-html-clean guides-html-clean guides-pdf-clean guides-man-clean .PHONY: api-html -api-html: api-html-clean +api-html: $(API_EXAMPLES) @echo 'doxygen for API...' $(Q)mkdir -p $(RTE_OUTPUT)/doc/html $(Q)(cat $(RTE_SDK)/doc/api/doxy-api.conf && \ printf 'PROJECT_NUMBER = ' && \ - $(MAKE) -rR showversion && \ + $(MAKE) -rRs showversion && \ + echo INPUT += $(API_EXAMPLES) && \ echo OUTPUT_DIRECTORY = $(RTE_OUTPUT)/doc && \ echo HTML_OUTPUT = html/api && \ echo GENERATE_HTML = YES && \ @@ -82,9 +85,17 @@ api-html: api-html-clean .PHONY: api-html-clean api-html-clean: + $(Q)rm -f $(API_EXAMPLES) $(Q)rm -f $(RTE_OUTPUT)/doc/html/api/* $(Q)rmdir -p --ignore-fail-on-non-empty $(RTE_OUTPUT)/doc/html/api 2>&- || true +$(API_EXAMPLES): api-html-clean + $(Q)mkdir -p $(@D) + @printf '/**\n' > $(API_EXAMPLES) + @printf '@page examples DPDK Example Programs\n\n' >> $(API_EXAMPLES) + @find examples -type f -name '*.c' -printf '@example %p\n' | LC_ALL=C sort >> $(API_EXAMPLES) + @printf '*/\n' >> $(API_EXAMPLES) + guides-pdf-clean: guides-pdf-img-clean guides-pdf-img-clean: $(Q)rm -f $(RTE_SDK)/doc/guides/*/img/*.pdf diff --git a/dpdk/mk/rte.sdkinstall.mk b/dpdk/mk/rte.sdkinstall.mk index 52170635..4e97feff 100644 --- a/dpdk/mk/rte.sdkinstall.mk +++ b/dpdk/mk/rte.sdkinstall.mk @@ -66,6 +66,7 @@ includedir ?= $(prefix)/include/dpdk datarootdir ?= $(prefix)/share docdir ?= $(datarootdir)/doc/dpdk datadir ?= $(datarootdir)/dpdk +mandir ?= $(datarootdir)/man sdkdir ?= $(datadir) targetdir ?= $(datadir)/$(RTE_TARGET) @@ -76,7 +77,7 @@ rte_mkdir = test -d $1 || mkdir -p $1 # Create the relative symbolic link $2 -> $1 # May be replaced with --relative option of ln from coreutils-8.16 -rte_symlink = ln -snf $$($(RTE_SDK)/scripts/relpath.sh $1 $(dir $2)) $2 +rte_symlink = ln -snf $$($(RTE_SDK)/buildtools/relpath.sh $1 $(dir $2)) $2 .PHONY: pre_install pre_install: @@ -121,18 +122,22 @@ install-runtime: --exclude 'app/cmdline*' --exclude app/test \ --exclude app/testacl --exclude app/testpipeline app | \ tar -xf - -C $(DESTDIR)$(bindir) --strip-components=1 \ - --keep-newer-files --warning=no-ignore-newer + --keep-newer-files $(Q)$(call rte_mkdir, $(DESTDIR)$(datadir)) - $(Q)cp -a $(RTE_SDK)/tools $(DESTDIR)$(datadir) - $(Q)$(call rte_symlink, $(DESTDIR)$(datadir)/tools/dpdk-setup.sh, \ - $(DESTDIR)$(datadir)/tools/setup.sh) - $(Q)$(call rte_symlink, $(DESTDIR)$(datadir)/tools/dpdk-devbind.py, \ - $(DESTDIR)$(datadir)/tools/dpdk_nic_bind.py) + $(Q)cp -a $(RTE_SDK)/usertools $(DESTDIR)$(datadir) $(Q)$(call rte_mkdir, $(DESTDIR)$(sbindir)) - $(Q)$(call rte_symlink, $(DESTDIR)$(datadir)/tools/dpdk-devbind.py, \ + $(Q)$(call rte_symlink, $(DESTDIR)$(datadir)/usertools/dpdk-devbind.py, \ $(DESTDIR)$(sbindir)/dpdk-devbind) - $(Q)$(call rte_symlink, $(DESTDIR)$(datadir)/tools/dpdk-pmdinfo.py, \ + $(Q)$(call rte_symlink, $(DESTDIR)$(datadir)/usertools/dpdk-pmdinfo.py, \ $(DESTDIR)$(bindir)/dpdk-pmdinfo) +ifneq ($(wildcard $O/doc/man/*/*.1),) + $(Q)$(call rte_mkdir, $(DESTDIR)$(mandir)/man1) + $(Q)cp -a $O/doc/man/*/*.1 $(DESTDIR)$(mandir)/man1 +endif +ifneq ($(wildcard $O/doc/man/*/*.8),) + $(Q)$(call rte_mkdir, $(DESTDIR)$(mandir)/man8) + $(Q)cp -a $O/doc/man/*/*.8 $(DESTDIR)$(mandir)/man8 +endif install-kmod: ifneq ($(wildcard $O/kmod/*),) @@ -144,10 +149,10 @@ install-sdk: $(Q)$(call rte_mkdir, $(DESTDIR)$(includedir)) $(Q)tar -chf - -C $O include | \ tar -xf - -C $(DESTDIR)$(includedir) --strip-components=1 \ - --keep-newer-files --warning=no-ignore-newer + --keep-newer-files $(Q)$(call rte_mkdir, $(DESTDIR)$(sdkdir)) $(Q)cp -a $(RTE_SDK)/mk $(DESTDIR)$(sdkdir) - $(Q)cp -a $(RTE_SDK)/scripts $(DESTDIR)$(sdkdir) + $(Q)cp -a $(RTE_SDK)/buildtools $(DESTDIR)$(sdkdir) $(Q)$(call rte_mkdir, $(DESTDIR)$(targetdir)/app) $(Q)cp -a $O/.config $(DESTDIR)$(targetdir) $(Q)cp -a $O/app/dpdk-pmdinfogen $(DESTDIR)$(targetdir)/app @@ -155,11 +160,11 @@ install-sdk: $(Q)$(call rte_symlink, $(DESTDIR)$(libdir), $(DESTDIR)$(targetdir)/lib) install-doc: -ifneq ($(wildcard $O/doc),) +ifneq ($(wildcard $O/doc/html),) $(Q)$(call rte_mkdir, $(DESTDIR)$(docdir)) - $(Q)tar -cf - -C $O/doc html --exclude 'html/guides/.*' | \ + $(Q)tar -cf - -C $O/doc --exclude 'html/guides/.*' html | \ tar -xf - -C $(DESTDIR)$(docdir) --strip-components=1 \ - --keep-newer-files --warning=no-ignore-newer + --keep-newer-files endif ifneq ($(wildcard $O/doc/*/*/*pdf),) $(Q)$(call rte_mkdir, $(DESTDIR)$(docdir)/guides) diff --git a/dpdk/mk/rte.sdkroot.mk b/dpdk/mk/rte.sdkroot.mk index 55a9d8ae..e2e3effa 100644 --- a/dpdk/mk/rte.sdkroot.mk +++ b/dpdk/mk/rte.sdkroot.mk @@ -34,6 +34,9 @@ MAKEFLAGS += --no-print-directory # define Q to '@' or not. $(Q) is used to prefix all shell commands to # be executed silently. Q=@ +ifeq '$V' '0' +override V= +endif ifdef V ifeq ("$(origin V)", "command line") Q= @@ -85,14 +88,20 @@ export ROOTDIRS-y ROOTDIRS- ROOTDIRS-n .PHONY: default default: all -.PHONY: config showconfigs showversion showversionum -config showconfigs showversion showversionum: +.PHONY: config defconfig showconfigs showversion showversionum +config defconfig showconfigs showversion showversionum: $(Q)$(MAKE) -f $(RTE_SDK)/mk/rte.sdkconfig.mk $@ -.PHONY: test fast_test ring_test mempool_test perf_test coverage -test fast_test ring_test mempool_test perf_test coverage: +.PHONY: cscope gtags tags etags +cscope gtags tags etags: + $(Q)$(RTE_SDK)/devtools/build-tags.sh $@ $T + +.PHONY: test test-basic test-fast test-ring test-mempool test-perf coverage +test test-basic test-fast test-ring test-mempool test-perf coverage: $(Q)$(MAKE) -f $(RTE_SDK)/mk/rte.sdktest.mk $@ +test: test-build + .PHONY: install install: $(Q)$(MAKE) -f $(RTE_SDK)/mk/rte.sdkinstall.mk pre_install @@ -106,10 +115,6 @@ help: doc-help doc-%: $(Q)$(MAKE) -f $(RTE_SDK)/mk/rte.sdkdoc.mk $* -.PHONY: depdirs depgraph -depdirs depgraph: - $(Q)$(MAKE) -f $(RTE_SDK)/mk/rte.sdkdepdirs.mk $@ - .PHONY: gcov gcovclean gcov gcovclean: $(Q)$(MAKE) -f $(RTE_SDK)/mk/rte.sdkgcov.mk $@ diff --git a/dpdk/mk/rte.sdktest.mk b/dpdk/mk/rte.sdktest.mk index ddbbbf6b..daeea90f 100644 --- a/dpdk/mk/rte.sdktest.mk +++ b/dpdk/mk/rte.sdktest.mk @@ -46,23 +46,23 @@ DIR := $(shell basename $(RTE_OUTPUT)) # # test: launch auto-tests, very simple for now. # -.PHONY: test fast_test perf_test coverage +.PHONY: test test-basic test-fast test-perf coverage PERFLIST=ring_perf,mempool_perf,memcpy_perf,hash_perf,timer_perf coverage: BLACKLIST=-$(PERFLIST) -fast_test: BLACKLIST=-$(PERFLIST) -perf_test: WHITELIST=$(PERFLIST) +test-fast: BLACKLIST=-$(PERFLIST) +test-perf: WHITELIST=$(PERFLIST) -test fast_test perf_test: +test test-basic test-fast test-perf: @mkdir -p $(AUTOTEST_DIR) ; \ cd $(AUTOTEST_DIR) ; \ if [ -f $(RTE_OUTPUT)/app/test ]; then \ - python $(RTE_SDK)/app/test/autotest.py \ + python $(RTE_SDK)/test/test/autotest.py \ $(RTE_OUTPUT)/app/test \ $(RTE_TARGET) \ $(BLACKLIST) $(WHITELIST); \ else \ - echo "No test found, please do a 'make build' first, or specify O=" ; \ + echo "No test found, please do a 'make test-build' first, or specify O=" ; \ fi # this is a special target to ease the pain of running coverage tests @@ -71,14 +71,14 @@ coverage: @mkdir -p $(AUTOTEST_DIR) ; \ cd $(AUTOTEST_DIR) ; \ if [ -f $(RTE_OUTPUT)/app/test ]; then \ - python $(RTE_SDK)/app/cmdline_test/cmdline_test.py \ + python $(RTE_SDK)/test/cmdline_test/cmdline_test.py \ $(RTE_OUTPUT)/app/cmdline_test; \ ulimit -S -n 100 ; \ - python $(RTE_SDK)/app/test/autotest.py \ + python $(RTE_SDK)/test/test/autotest.py \ $(RTE_OUTPUT)/app/test \ $(RTE_TARGET) \ $(BLACKLIST) $(WHITELIST) ; \ $(RTE_OUTPUT)/app/dpdk-procinfo --file-prefix=ring_perf -- -m; \ else \ - echo "No test found, please do a 'make build' first, or specify O=" ;\ + echo "No test found, please do a 'make test-build' first, or specify O=" ;\ fi diff --git a/dpdk/mk/rte.shared.mk b/dpdk/mk/rte.shared.mk index fc6b0b44..4e680bc0 100644 --- a/dpdk/mk/rte.shared.mk +++ b/dpdk/mk/rte.shared.mk @@ -32,7 +32,6 @@ include $(RTE_SDK)/mk/internal/rte.compile-pre.mk include $(RTE_SDK)/mk/internal/rte.install-pre.mk include $(RTE_SDK)/mk/internal/rte.clean-pre.mk include $(RTE_SDK)/mk/internal/rte.build-pre.mk -include $(RTE_SDK)/mk/internal/rte.depdirs-pre.mk # VPATH contains at least SRCDIR VPATH += $(SRCDIR) @@ -86,8 +85,8 @@ LDLIBS_NAMES += $(patsubst -Wl$(comma)-l%,lib%.a,$(filter -Wl$(comma)-l%,$(LDLIB # list of found libraries files (useful for deps). If not found, the # library is silently ignored and dep won't be checked -LDLIBS_FILES := $(wildcard $(foreach dir,$(LDLIBS_PATH),\ - $(addprefix $(dir)/,$(LDLIBS_NAMES)))) +LDLIBS_FILES := $(sort $(wildcard $(foreach dir,$(LDLIBS_PATH),\ + $(addprefix $(dir)/,$(LDLIBS_NAMES))))) # # Archive objects in .so file if needed @@ -131,7 +130,6 @@ include $(RTE_SDK)/mk/internal/rte.compile-post.mk include $(RTE_SDK)/mk/internal/rte.install-post.mk include $(RTE_SDK)/mk/internal/rte.clean-post.mk include $(RTE_SDK)/mk/internal/rte.build-post.mk -include $(RTE_SDK)/mk/internal/rte.depdirs-post.mk .PHONY: FORCE FORCE: diff --git a/dpdk/mk/rte.subdir.mk b/dpdk/mk/rte.subdir.mk index 256e64e7..6c393eb7 100644 --- a/dpdk/mk/rte.subdir.mk +++ b/dpdk/mk/rte.subdir.mk @@ -37,6 +37,8 @@ include $(RTE_SDK)/mk/internal/rte.install-pre.mk include $(RTE_SDK)/mk/internal/rte.clean-pre.mk include $(RTE_SDK)/mk/internal/rte.build-pre.mk +ALL_DEPDIRS := $(patsubst DEPDIRS-%,%,$(filter DEPDIRS-%,$(.VARIABLES))) + CLEANDIRS = $(addsuffix _clean,$(DIRS-y) $(DIRS-n) $(DIRS-)) VPATH += $(SRCDIR) @@ -72,37 +74,16 @@ clean: _postclean fi @rm -f $(_BUILD_TARGETS) $(_INSTALL_TARGETS) $(_CLEAN_TARGETS) -# -# include .depdirs and define rules to order priorities between build -# of directories. -# -include $(RTE_OUTPUT)/.depdirs - define depdirs_rule -$(1): $(sort $(patsubst $(S)/%,%,$(LOCAL_DEPDIRS-$(S)/$(1)))) +$(DEPDIRS-$(1)): + +$(1): | $(DEPDIRS-$(1)) + +$(if $(D),$(info $(1) depends on $(DEPDIRS-$(1)))) endef -$(foreach d,$(DIRS-y),$(eval $(call depdirs_rule,$(d)))) - - -# use a "for" in a shell to process dependencies: we don't want this -# task to be run in parallel. -.PHONY: depdirs -depdirs: - @for d in $(DIRS-y); do \ - if [ -f $(SRCDIR)/$$d/Makefile ]; then \ - $(MAKE) S=$S/$$d -f $(SRCDIR)/$$d/Makefile depdirs ; \ - fi ; \ - done - -.PHONY: depgraph -depgraph: - @for d in $(DIRS-y); do \ - echo " \"$(S)\" -> \"$(S)/$$d\"" ; \ - if [ -f $(SRCDIR)/$$d/Makefile ]; then \ - $(MAKE) S=$S/$$d -f $(SRCDIR)/$$d/Makefile depgraph ; \ - fi ; \ - done +$(foreach dir,$(ALL_DEPDIRS),\ + $(eval $(call depdirs_rule,$(dir)))) include $(RTE_SDK)/mk/internal/rte.install-post.mk include $(RTE_SDK)/mk/internal/rte.clean-post.mk diff --git a/dpdk/mk/rte.vars.mk b/dpdk/mk/rte.vars.mk index 28982a5f..b51006b8 100644 --- a/dpdk/mk/rte.vars.mk +++ b/dpdk/mk/rte.vars.mk @@ -46,6 +46,9 @@ endif # define Q to '@' or not. $(Q) is used to prefix all shell commands to # be executed silently. Q=@ +ifeq '$V' '0' +override V= +endif ifdef V ifeq ("$(origin V)", "command line") Q= @@ -93,7 +96,7 @@ ifeq ($(RTE_TOOLCHAIN),) $(error RTE_TOOLCHAIN is not defined) endif -# can be overriden by make command line or exported environment variable +# can be overridden by make command line or exported environment variable RTE_KERNELDIR ?= /lib/modules/$(shell uname -r)/build export RTE_TARGET @@ -104,7 +107,7 @@ export RTE_TOOLCHAIN # developer build automatically enabled in a git tree ifneq ($(wildcard $(RTE_SDK)/.git),) -RTE_DEVEL_BUILD := y +RTE_DEVEL_BUILD ?= y endif # SRCDIR is the current source directory diff --git a/dpdk/mk/target/generic/rte.vars.mk b/dpdk/mk/target/generic/rte.vars.mk index 75a616ab..84649b9b 100644 --- a/dpdk/mk/target/generic/rte.vars.mk +++ b/dpdk/mk/target/generic/rte.vars.mk @@ -38,32 +38,36 @@ # # machine: # -# - can define ARCH variable (overriden by cmdline value) -# - can define CROSS variable (overriden by cmdline value) -# - define MACHINE_CFLAGS variable (overriden by cmdline value) -# - define MACHINE_LDFLAGS variable (overriden by cmdline value) -# - define MACHINE_ASFLAGS variable (overriden by cmdline value) -# - can define CPU_CFLAGS variable (overriden by cmdline value) that +# - can define ARCH variable (overridden by cmdline value) +# - can define CROSS variable (overridden by cmdline value) +# - define MACHINE_CFLAGS variable (overridden by cmdline value) +# - define MACHINE_LDFLAGS variable (overridden by cmdline value) +# - define MACHINE_ASFLAGS variable (overridden by cmdline value) +# - can define CPU_CFLAGS variable (overridden by cmdline value) that # overrides the one defined in arch. -# - can define CPU_LDFLAGS variable (overriden by cmdline value) that +# - can define CPU_LDFLAGS variable (overridden by cmdline value) that # overrides the one defined in arch. -# - can define CPU_ASFLAGS variable (overriden by cmdline value) that +# - can define CPU_ASFLAGS variable (overridden by cmdline value) that # overrides the one defined in arch. # +ifneq ($(wildcard $(RTE_SDK)/mk/machine/$(RTE_MACHINE)/rte.vars.mk),) include $(RTE_SDK)/mk/machine/$(RTE_MACHINE)/rte.vars.mk +else +MACHINE_CFLAGS := -march=$(RTE_MACHINE) +endif # # arch: # -# - define ARCH variable (overriden by cmdline or by previous +# - define ARCH variable (overridden by cmdline or by previous # optional define in machine .mk) -# - define CROSS variable (overriden by cmdline or previous define +# - define CROSS variable (overridden by cmdline or previous define # in machine .mk) -# - define CPU_CFLAGS variable (overriden by cmdline or previous +# - define CPU_CFLAGS variable (overridden by cmdline or previous # define in machine .mk) -# - define CPU_LDFLAGS variable (overriden by cmdline or previous +# - define CPU_LDFLAGS variable (overridden by cmdline or previous # define in machine .mk) -# - define CPU_ASFLAGS variable (overriden by cmdline or previous +# - define CPU_ASFLAGS variable (overridden by cmdline or previous # define in machine .mk) # - may override any previously defined variable # @@ -73,9 +77,9 @@ include $(RTE_SDK)/mk/arch/$(RTE_ARCH)/rte.vars.mk # toolchain: # # - define CC, LD, AR, AS, ... -# - define TOOLCHAIN_CFLAGS variable (overriden by cmdline value) -# - define TOOLCHAIN_LDFLAGS variable (overriden by cmdline value) -# - define TOOLCHAIN_ASFLAGS variable (overriden by cmdline value) +# - define TOOLCHAIN_CFLAGS variable (overridden by cmdline value) +# - define TOOLCHAIN_LDFLAGS variable (overridden by cmdline value) +# - define TOOLCHAIN_ASFLAGS variable (overridden by cmdline value) # - may override any previously defined variable # include $(RTE_SDK)/mk/toolchain/$(RTE_TOOLCHAIN)/rte.vars.mk @@ -83,9 +87,9 @@ include $(RTE_SDK)/mk/toolchain/$(RTE_TOOLCHAIN)/rte.vars.mk # # exec-env: # -# - define EXECENV_CFLAGS variable (overriden by cmdline) -# - define EXECENV_LDFLAGS variable (overriden by cmdline) -# - define EXECENV_ASFLAGS variable (overriden by cmdline) +# - define EXECENV_CFLAGS variable (overridden by cmdline) +# - define EXECENV_LDFLAGS variable (overridden by cmdline) +# - define EXECENV_ASFLAGS variable (overridden by cmdline) # - may override any previously defined variable # include $(RTE_SDK)/mk/exec-env/$(RTE_EXEC_ENV)/rte.vars.mk @@ -121,9 +125,6 @@ LDFLAGS += -L$(RTE_OUTPUT)/lib ifeq ($(BUILDING_RTE_SDK),1) # building sdk CFLAGS += -include $(RTE_OUTPUT)/include/rte_config.h -ifeq ($(CONFIG_RTE_INSECURE_FUNCTION_WARNING),y) -CFLAGS += -include rte_warnings.h -endif else # if we are building an external application, include SDK's lib and # includes too @@ -132,9 +133,6 @@ ifneq ($(wildcard $(RTE_OUTPUT)/include/rte_config.h),) CFLAGS += -include $(RTE_OUTPUT)/include/rte_config.h endif CFLAGS += -include $(RTE_SDK_BIN)/include/rte_config.h -ifeq ($(CONFIG_RTE_INSECURE_FUNCTION_WARNING),y) -CFLAGS += -include rte_warnings.h -endif LDFLAGS += -L$(RTE_SDK_BIN)/lib endif diff --git a/dpdk/mk/toolchain/clang/rte.toolchain-compat.mk b/dpdk/mk/toolchain/clang/rte.toolchain-compat.mk index b734413b..9e095d38 100644 --- a/dpdk/mk/toolchain/clang/rte.toolchain-compat.mk +++ b/dpdk/mk/toolchain/clang/rte.toolchain-compat.mk @@ -38,7 +38,8 @@ # find out CLANG version -CLANG_VERSION := $(shell $(CC) -v 2>&1 | grep version | sed "s/.*version \([0-9]*\.[0-9]*\).*/\1/") +CLANG_VERSION := $(shell $(CC) -v 2>&1 | \ + sed -n "s/.*version \([0-9]*\.[0-9]*\).*/\1/p") CLANG_MAJOR_VERSION := $(shell echo $(CLANG_VERSION) | cut -f1 -d.) diff --git a/dpdk/mk/toolchain/clang/rte.vars.mk b/dpdk/mk/toolchain/clang/rte.vars.mk index 7749b991..dde922d2 100644 --- a/dpdk/mk/toolchain/clang/rte.vars.mk +++ b/dpdk/mk/toolchain/clang/rte.vars.mk @@ -32,10 +32,10 @@ # # toolchain: # -# - define CC, LD, AR, AS, ... (overriden by cmdline value) -# - define TOOLCHAIN_CFLAGS variable (overriden by cmdline value) -# - define TOOLCHAIN_LDFLAGS variable (overriden by cmdline value) -# - define TOOLCHAIN_ASFLAGS variable (overriden by cmdline value) +# - define CC, LD, AR, AS, ... (overridden by cmdline value) +# - define TOOLCHAIN_CFLAGS variable (overridden by cmdline value) +# - define TOOLCHAIN_LDFLAGS variable (overridden by cmdline value) +# - define TOOLCHAIN_ASFLAGS variable (overridden by cmdline value) # CC = $(CROSS)clang @@ -79,5 +79,10 @@ include $(RTE_SDK)/mk/toolchain/$(RTE_TOOLCHAIN)/rte.toolchain-compat.mk # workaround clang bug with warning "missing field initializer" for "= {0}" WERROR_FLAGS += -Wno-missing-field-initializers +# disable packed member unalign warnings +ifeq ($(shell test $(CLANG_MAJOR_VERSION) -ge 4 && echo 1), 1) +WERROR_FLAGS += -Wno-address-of-packed-member +endif + export CC AS AR LD OBJCOPY OBJDUMP STRIP READELF export TOOLCHAIN_CFLAGS TOOLCHAIN_LDFLAGS TOOLCHAIN_ASFLAGS diff --git a/dpdk/mk/toolchain/gcc/rte.toolchain-compat.mk b/dpdk/mk/toolchain/gcc/rte.toolchain-compat.mk index 6eed20cc..01ac7e23 100644 --- a/dpdk/mk/toolchain/gcc/rte.toolchain-compat.mk +++ b/dpdk/mk/toolchain/gcc/rte.toolchain-compat.mk @@ -60,6 +60,7 @@ else # ifeq ($(shell test $(GCC_VERSION) -le 49 && echo 1), 1) MACHINE_CFLAGS := $(patsubst -march=armv8-a+crc,-march=armv8-a+crc -D__ARM_FEATURE_CRC32=1,$(MACHINE_CFLAGS)) + MACHINE_CFLAGS := $(patsubst -march=armv8-a+crc+crypto,-march=armv8-a+crc+crypto -D__ARM_FEATURE_CRC32=1,$(MACHINE_CFLAGS)) endif ifeq ($(shell test $(GCC_VERSION) -le 47 && echo 1), 1) MACHINE_CFLAGS := $(patsubst -march=core-avx-i,-march=corei7-avx,$(MACHINE_CFLAGS)) @@ -89,4 +90,9 @@ else ifeq ($(shell test $(GCC_VERSION) -lt 42 && echo 1), 1) MACHINE_CFLAGS := $(filter-out -march% -mtune% -msse%,$(MACHINE_CFLAGS)) endif + + # Disable thunderx PMD for gcc < 4.7 + ifeq ($(shell test $(GCC_VERSION) -lt 47 && echo 1), 1) + CONFIG_RTE_LIBRTE_THUNDERX_NICVF_PMD=d + endif endif diff --git a/dpdk/mk/toolchain/gcc/rte.vars.mk b/dpdk/mk/toolchain/gcc/rte.vars.mk index ff70f3d9..3b907e20 100644 --- a/dpdk/mk/toolchain/gcc/rte.vars.mk +++ b/dpdk/mk/toolchain/gcc/rte.vars.mk @@ -32,10 +32,10 @@ # # toolchain: # -# - define CC, LD, AR, AS, ... (overriden by cmdline value) -# - define TOOLCHAIN_CFLAGS variable (overriden by cmdline value) -# - define TOOLCHAIN_LDFLAGS variable (overriden by cmdline value) -# - define TOOLCHAIN_ASFLAGS variable (overriden by cmdline value) +# - define CC, LD, AR, AS, ... (overridden by cmdline value) +# - define TOOLCHAIN_CFLAGS variable (overridden by cmdline value) +# - define TOOLCHAIN_LDFLAGS variable (overridden by cmdline value) +# - define TOOLCHAIN_ASFLAGS variable (overridden by cmdline value) # CC = $(CROSS)gcc @@ -81,9 +81,9 @@ ifeq ($(RTE_DEVEL_BUILD),y) WERROR_FLAGS += -Werror endif -# There are many issues reported for ARMv7 architecture +# There are many issues reported for strict alignment architectures # which are not necessarily fatal. Report as warnings. -ifeq ($(CONFIG_RTE_ARCH_ARMv7),y) +ifeq ($(CONFIG_RTE_ARCH_STRICT_ALIGN),y) WERROR_FLAGS += -Wno-error=cast-align endif @@ -99,5 +99,12 @@ ifeq ($(shell test $(GCC_VERSION) -lt 47 && echo 1), 1) WERROR_FLAGS += -Wno-uninitialized endif +ifeq ($(shell test $(GCC_VERSION) -gt 70 && echo 1), 1) +# Tell GCC only to error for switch fallthroughs without a suitable comment +WERROR_FLAGS += -Wimplicit-fallthrough=2 +# Ignore errors for snprintf truncation +WERROR_FLAGS += -Wno-format-truncation +endif + export CC AS AR LD OBJCOPY OBJDUMP STRIP READELF export TOOLCHAIN_CFLAGS TOOLCHAIN_LDFLAGS TOOLCHAIN_ASFLAGS diff --git a/dpdk/mk/toolchain/icc/rte.toolchain-compat.mk b/dpdk/mk/toolchain/icc/rte.toolchain-compat.mk index 41344668..3c25d184 100644 --- a/dpdk/mk/toolchain/icc/rte.toolchain-compat.mk +++ b/dpdk/mk/toolchain/icc/rte.toolchain-compat.mk @@ -41,7 +41,7 @@ ICC_MAJOR_VERSION = $(shell icc -dumpversion | cut -f1 -d.) ifeq ($(shell test $(ICC_MAJOR_VERSION) -lt 12 && echo 1), 1) - MACHINE_CFLAGS = -xSSE3 + MACHINE_CFLAGS = -xSSE4.2 $(warning You are not using ICC 12.x or higher. This is neither supported, nor tested.) else @@ -72,4 +72,9 @@ else # remove march options MACHINE_CFLAGS := $(patsubst -march=%,-xSSE3,$(MACHINE_CFLAGS)) endif + + # Disable thunderx PMD for icc <= 16.0 + ifeq ($(shell test $(ICC_MAJOR_VERSION) -le 16 && echo 1), 1) + CONFIG_RTE_LIBRTE_THUNDERX_NICVF_PMD=d + endif endif diff --git a/dpdk/mk/toolchain/icc/rte.vars.mk b/dpdk/mk/toolchain/icc/rte.vars.mk index ba69f1f8..33a8ba79 100644 --- a/dpdk/mk/toolchain/icc/rte.vars.mk +++ b/dpdk/mk/toolchain/icc/rte.vars.mk @@ -32,10 +32,10 @@ # # toolchain: # -# - define CC, LD, AR, AS, ... (overriden by cmdline value) -# - define TOOLCHAIN_CFLAGS variable (overriden by cmdline value) -# - define TOOLCHAIN_LDFLAGS variable (overriden by cmdline value) -# - define TOOLCHAIN_ASFLAGS variable (overriden by cmdline value) +# - define CC, LD, AR, AS, ... (overridden by cmdline value) +# - define TOOLCHAIN_CFLAGS variable (overridden by cmdline value) +# - define TOOLCHAIN_LDFLAGS variable (overridden by cmdline value) +# - define TOOLCHAIN_ASFLAGS variable (overridden by cmdline value) # # Warning: we do not use CROSS environment variable as icc is mainly a @@ -69,8 +69,11 @@ TOOLCHAIN_ASFLAGS = # error #13368: loop was not vectorized with "vector always assert" # error #15527: loop was not vectorized: function call to fprintf cannot be vectorize # was declared "deprecated" +# Warning #11074, 11076: to prevent "inline-max-size" warnings. WERROR_FLAGS := -Wall -w2 -diag-disable 271 -diag-warning 1478 WERROR_FLAGS += -diag-disable 13368 -diag-disable 15527 +WERROR_FLAGS += -diag-disable 188 +WERROR_FLAGS += -diag-disable 11074 -diag-disable 11076 ifeq ($(RTE_DEVEL_BUILD),y) WERROR_FLAGS += -Werror-all @@ -78,10 +81,6 @@ endif # process cpu flags include $(RTE_SDK)/mk/toolchain/$(RTE_TOOLCHAIN)/rte.toolchain-compat.mk -# disable max-inline params boundaries for ICC compiler for version 15 and greater -ifeq ($(shell test $(ICC_MAJOR_VERSION) -ge 14 && echo 1), 1) - TOOLCHAIN_CFLAGS += -no-inline-max-size -no-inline-max-total-size -endif export CC AS AR LD OBJCOPY OBJDUMP STRIP READELF export TOOLCHAIN_CFLAGS TOOLCHAIN_LDFLAGS TOOLCHAIN_ASFLAGS diff --git a/dpdk/pkg/dpdk.spec b/dpdk/pkg/dpdk.spec index e6f1eaa4..8681e4c8 100644 --- a/dpdk/pkg/dpdk.spec +++ b/dpdk/pkg/dpdk.spec @@ -30,7 +30,7 @@ # OF THE POSSIBILITY OF SUCH DAMAGE. Name: dpdk -Version: 16.07.2 +Version: 17.11.2 Release: 1 Packager: packaging@6wind.com URL: http://dpdk.org @@ -40,12 +40,18 @@ Summary: Data Plane Development Kit core Group: System Environment/Libraries License: BSD and LGPLv2 and GPLv2 -ExclusiveArch: i686, x86_64 +ExclusiveArch: i686 x86_64 aarch64 +%ifarch aarch64 +%global machine armv8a +%global target arm64-%{machine}-linuxapp-gcc +%global config arm64-%{machine}-linuxapp-gcc +%else %global machine default %global target %{_arch}-%{machine}-linuxapp-gcc %global config %{_arch}-native-linuxapp-gcc +%endif -BuildRequires: kernel-devel, kernel-headers, libpcap-devel, xen-devel +BuildRequires: kernel-devel, kernel-headers, libpcap-devel BuildRequires: doxygen, python-sphinx, inkscape BuildRequires: texlive-collection-latexextra @@ -81,7 +87,6 @@ sed -ri 's,(RTE_BUILD_SHARED_LIB=).*,\1y,' %{target}/.config sed -ri 's,(RTE_NEXT_ABI=).*,\1n,' %{target}/.config sed -ri 's,(LIBRTE_VHOST=).*,\1y,' %{target}/.config sed -ri 's,(LIBRTE_PMD_PCAP=).*,\1y,' %{target}/.config -sed -ri 's,(LIBRTE_PMD_XENVIRT=).*,\1y,' %{target}/.config make O=%{target} %{?_smp_mflags} make O=%{target} doc @@ -94,7 +99,7 @@ make install O=%{target} DESTDIR=%{buildroot} \ %files %dir %{_datadir}/dpdk -%{_datadir}/dpdk/tools +%{_datadir}/dpdk/usertools /lib/modules/%(uname -r)/extra/* %{_sbindir}/* %{_bindir}/* @@ -103,7 +108,7 @@ make install O=%{target} DESTDIR=%{buildroot} \ %files devel %{_includedir}/dpdk %{_datadir}/dpdk/mk -%{_datadir}/dpdk/scripts +%{_datadir}/dpdk/buildtools %{_datadir}/dpdk/%{target} %{_datadir}/dpdk/examples diff --git a/dpdk/scripts/auto-config-h.sh b/dpdk/scripts/auto-config-h.sh deleted file mode 100755 index 4356d7e3..00000000 --- a/dpdk/scripts/auto-config-h.sh +++ /dev/null @@ -1,136 +0,0 @@ -#!/bin/sh -# -# BSD LICENSE -# -# Copyright 2014-2015 6WIND S.A. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in -# the documentation and/or other materials provided with the -# distribution. -# * Neither the name of 6WIND S.A. nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# Crude script to detect whether particular types, macros and functions are -# defined by trying to compile a file with a given header. Can be used to -# perform cross-platform checks since the resulting object file is not -# executed. -# -# Set VERBOSE=1 in the environment to display compiler output and errors. -# -# CC, CPPFLAGS, CFLAGS, EXTRA_CPPFLAGS and EXTRA_CFLAGS are taken from the -# environment. -# -# AUTO_CONFIG_CFLAGS may append additional CFLAGS without modifying the -# above variables. - -file=${1:?output file name required (config.h)} -macro=${2:?output macro name required (HAVE_*)} -include=${3:?include name required (foo.h)} -type=${4:?object type required (define, enum, type, field, func)} -name=${5:?define/type/function name required} - -: ${CC:=cc} - -temp=/tmp/${0##*/}.$$.c - -case $type in -define) - code="\ -#ifndef $name -#error $name not defined -#endif -" - ;; -enum) - code="\ -long test____ = $name; -" - ;; -type) - code="\ -$name test____; -" - ;; -field) - code="\ -void test____(void) -{ - ${name%%.*} test_____; - - (void)test_____.${name#*.}; -} -" - ;; -func) - code="\ -void (*test____)() = (void (*)())$name; -" - ;; -*) - unset error - : ${error:?unknown object type \"$type\"} - exit -esac - -if [ "${VERBOSE}" = 1 ] -then - err=2 - out=1 - eol=' -' -else - exec 3> /dev/null || - exit - err=3 - out=3 - eol=' ' -fi && -printf 'Looking for %s %s in %s.%s' \ - "${name}" "${type}" "${include}" "${eol}" && -printf "\ -#include <%s> - -%s -" "$include" "$code" > "${temp}" && -if ${CC} ${CPPFLAGS} ${EXTRA_CPPFLAGS} ${CFLAGS} ${EXTRA_CFLAGS} \ - ${AUTO_CONFIG_CFLAGS} \ - -c -o /dev/null "${temp}" 1>&${out} 2>&${err} -then - rm -f "${temp}" - printf "\ -#ifndef %s -#define %s 1 -#endif /* %s */ - -" "${macro}" "${macro}" "${macro}" >> "${file}" && - printf 'Defining %s.\n' "${macro}" -else - rm -f "${temp}" - printf "\ -/* %s is not defined. */ - -" "${macro}" >> "${file}" && - printf 'Not defining %s.\n' "${macro}" -fi - -exit diff --git a/dpdk/scripts/check-git-log.sh b/dpdk/scripts/check-git-log.sh deleted file mode 100755 index e416aea1..00000000 --- a/dpdk/scripts/check-git-log.sh +++ /dev/null @@ -1,190 +0,0 @@ -#! /bin/sh - -# BSD LICENSE -# -# Copyright 2016 6WIND S.A. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in -# the documentation and/or other materials provided with the -# distribution. -# * Neither the name of 6WIND S.A. nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# Check commit logs (headlines and references) -# -# If any doubt about the formatting, please check in the most recent history: -# git log --format='%>|(15)%cr %s' --reverse | grep -i - -if [ "$1" = '-h' -o "$1" = '--help' ] ; then - cat <<- END_OF_HELP - usage: $(basename $0) [-h] [range] - - Check commit log formatting. - The git range can be specified as a "git log" option, - e.g. -1 to check only the latest commit. - The default range starts from origin/master to HEAD. - END_OF_HELP - exit -fi - -range=${1:-origin/master..} - -commits=$(git log --format='%h' $range) -headlines=$(git log --format='%s' $range) -bodylines=$(git log --format='%b' $range) -fixes=$(git log --format='%h %s' $range | grep -i ': *fix' | cut -d' ' -f1) -tags=$(git log --format='%b' $range | grep -i -e 'by *:' -e 'fix.*:') -bytag='\(Reported\|Suggested\|Signed-off\|Acked\|Reviewed\|Tested\)-by:' - -# check headline format (spacing, no punctuation, no code) -bad=$(echo "$headlines" | grep --color=always \ - -e ' ' \ - -e '^ ' \ - -e ' $' \ - -e '\.$' \ - -e '[,;!?&|]' \ - -e ':.*_' \ - -e '^[^:]\+$' \ - -e ':[^ ]' \ - -e ' :' \ - | sed 's,^,\t,') -[ -z "$bad" ] || printf "Wrong headline format:\n$bad\n" - -# check headline prefix when touching only drivers, e.g. net/ -bad=$(for commit in $commits ; do - headline=$(git log --format='%s' -1 $commit) - files=$(git diff-tree --no-commit-id --name-only -r $commit) - [ -z "$(echo "$files" | grep -v '^\(drivers\|doc\|config\)/')" ] || - continue - drv=$(echo "$files" | grep '^drivers/' | cut -d "/" -f 2,3 | sort -u) - drvgrp=$(echo "$drv" | cut -d "/" -f 1 | uniq) - if [ $(echo "$drvgrp" | wc -l) -gt 1 ] ; then - echo "$headline" | grep -v '^drivers:' - elif [ $(echo "$drv" | wc -l) -gt 1 ] ; then - echo "$headline" | grep -v "^$drvgrp" - else - echo "$headline" | grep -v "^$drv" - fi -done | sed 's,^,\t,') -[ -z "$bad" ] || printf "Wrong headline prefix:\n$bad\n" - -# check headline label for common typos -bad=$(echo "$headlines" | grep --color=always \ - -e '^example[:/]' \ - -e '^apps/' \ - -e '^testpmd' \ - -e 'test-pmd' \ - -e '^bond:' \ - | sed 's,^,\t,') -[ -z "$bad" ] || printf "Wrong headline label:\n$bad\n" - -# check headline lowercase for first words -bad=$(echo "$headlines" | grep --color=always \ - -e '^.*[A-Z].*:' \ - -e ': *[A-Z]' \ - | sed 's,^,\t,') -[ -z "$bad" ] || printf "Wrong headline uppercase:\n$bad\n" - -# check headline uppercase (Rx/Tx, VF, L2, MAC, Linux, ARM...) -bad=$(echo "$headlines" | grep -E --color=always \ - -e '\<(rx|tx|RX|TX)\>' \ - -e '\<[pv]f\>' \ - -e '\<[hsf]w\>' \ - -e '\' \ - -e ':.*\' \ - -e ':.*\' \ - -e ':.*\' \ - -e ':.*\' \ - -e ':.*\' \ - -e ':.*\' \ - -e ':.*\' \ - -e ':.*\' \ - -e ':.*\' \ - -e ':.*\' \ - -e ':.*\' \ - -e ':.*\' \ - -e ':.*\' \ - -e ':.*\' \ - -e ':.*\' \ - | sed 's,^,\t,') -[ -z "$bad" ] || printf "Wrong headline lowercase:\n$bad\n" - -# special case check for VMDq to give good error message -bad=$(echo "$headlines" | grep -E --color=always \ - -e '\<(vmdq|VMDQ)\>' \ - | sed 's,^,\t,') -[ -z "$bad" ] || printf "Wrong headline capitalization, use 'VMDq':\n$bad\n" - -# check headline length (60 max) -bad=$(echo "$headlines" | - awk 'length>60 {print}' | - sed 's,^,\t,') -[ -z "$bad" ] || printf "Headline too long:\n$bad\n" - -# check body lines length (75 max) -bad=$(echo "$bodylines" | grep -v '^Fixes:' | - awk 'length>75 {print}' | - sed 's,^,\t,') -[ -z "$bad" ] || printf "Line too long:\n$bad\n" - -# check starting commit message with "It" -bad=$(for commit in $commits ; do - firstbodyline=$(git log --format='%b' -1 $commit | head -n1) - echo "$firstbodyline" | grep --color=always -ie '^It ' -done | sed 's,^,\t,') -[ -z "$bad" ] || printf "Wrong beginning of commit message:\n$bad\n" - -# check tags spelling -bad=$(echo "$tags" | - grep -v "^$bytag [^,]* <.*@.*>$" | - grep -v '^Fixes: [0-9a-f]\{7\}[0-9a-f]* (".*")$' | - sed 's,^.,\t&,') -[ -z "$bad" ] || printf "Wrong tag:\n$bad\n" - -# check blank line after last Fixes: tag -bad=$(echo "$bodylines" | - sed -n 'N;/\nFixes:/D;/\n$/D;/^Fixes:/P' | - sed 's,^.,\t&,') -[ -z "$bad" ] || printf "Missing blank line after 'Fixes' tag:\n$bad\n" - -# check missing Fixes: tag -bad=$(for fix in $fixes ; do - git log --format='%b' -1 $fix | grep -q '^Fixes: ' || - git log --format='\t%s' -1 $fix -done) -[ -z "$bad" ] || printf "Missing 'Fixes' tag:\n$bad\n" - -# check Fixes: reference -IFS=' -' -fixtags=$(echo "$tags" | grep '^Fixes: ') -bad=$(for fixtag in $fixtags ; do - hash=$(echo "$fixtag" | sed 's,^Fixes: \([0-9a-f]*\).*,\1,') - if git branch --contains $hash | grep -q '^\*' ; then - good="Fixes: $hash "$(git log --format='("%s")' -1 $hash 2>&-) - else - good="reference not in current branch" - fi - printf "$fixtag" | grep -v "^$good$" -done | sed 's,^,\t,') -[ -z "$bad" ] || printf "Wrong 'Fixes' reference:\n$bad\n" diff --git a/dpdk/scripts/check-maintainers.sh b/dpdk/scripts/check-maintainers.sh deleted file mode 100755 index 69c879a8..00000000 --- a/dpdk/scripts/check-maintainers.sh +++ /dev/null @@ -1,157 +0,0 @@ -#! /bin/sh - -# BSD LICENSE -# -# Copyright 2015 6WIND S.A. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in -# the documentation and/or other materials provided with the -# distribution. -# * Neither the name of 6WIND S.A. nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# Do some basic checks in MAINTAINERS file - -cd $(dirname $0)/.. - -# Get files matching paths with wildcards and / meaning recursing -files () # [ ...] -{ - if [ -z "$1" ] ; then - return - fi - if [ -d .git ] ; then - git ls-files "$1" - else - find "$1" -type f | - sed 's,^\./,,' - fi | - # if not ended by / - if ! echo "$1" | grep -q '/[[:space:]]*$' ; then - # filter out deeper directories - sed "/\(\/[^/]*\)\{$(($(echo "$1" | grep -o / | wc -l) + 1))\}/d" - else - cat - fi - # next path - shift - files "$@" -} - -# Get all files matching F: and X: fields -parse_fx () # -{ - IFS=' -' - # parse each line excepted underlining - for line in $( (sed '/^-\+$/d' $1 ; echo) | sed 's,^$,§,') ; do - if echo "$line" | grep -q '^§$' ; then - # empty line delimit end of section - whitelist=$(files $flines) - blacklist=$(files $xlines) - match=$(aminusb "$whitelist" "$blacklist") - if [ -n "$whitelist" ] ; then - printf "# $title " - maintainers=$(echo "$maintainers" | sed -r 's,.*<(.*)>.*,\1,') - maintainers=$(printf "$maintainers" | sed -e 's,^,<,' -e 's,$,>,') - echo $maintainers - fi - if [ -n "$match" ] ; then - echo "$match" - fi - # flush section - unset maintainers - unset flines - unset xlines - elif echo "$line" | grep -q '^[A-Z]: ' ; then - # maintainer - maintainers=$(add_line_to_if "$line" "$maintainers" 'M: ') - # file matching pattern - flines=$(add_line_to_if "$line" "$flines" 'F: ') - # file exclusion pattern - xlines=$(add_line_to_if "$line" "$xlines" 'X: ') - else # assume it is a title - title="$line" - fi - done -} - -# Check patterns in F: and X: -check_fx () # -{ - IFS=' -' - for line in $(sed -n 's,^[FX]: ,,p' $1 | tr '*' '#') ; do - line=$(printf "$line" | tr '#' '*') - match=$(files "$line") - if [ -z "$match" ] ; then - echo "$line" - fi - done -} - -# Add a line to a set of lines if it begins with right pattern -add_line_to_if () # -{ - ( - echo "$2" - echo "$1" | sed -rn "s,^$3(.*),\1,p" - ) | - sed '/^$/d' -} - -# Subtract two sets of lines -aminusb () # -{ - printf "$1\n$2\n$2" | sort | uniq -u | sed '/^$/d' -} - -printf 'sections: ' -parsed=$(parse_fx MAINTAINERS) -echo "$parsed" | grep -c '^#' -printf 'with maintainer: ' -echo "$parsed" | grep -c '^#.*@' -printf 'maintainers: ' -grep '^M:.*<' MAINTAINERS | sort -u | wc -l - -echo -echo '##########' -echo '# orphan areas' -echo '##########' -echo "$parsed" | sed -rn 's,^#([^@]*)$,\1,p' | uniq - -echo -echo '##########' -echo '# files not listed' -echo '##########' -all=$(files ./) -listed=$(echo "$parsed" | sed '/^#/d' | sort -u) -aminusb "$all" "$listed" - -echo -echo '##########' -echo '# wrong patterns' -echo '##########' -check_fx MAINTAINERS - -# TODO: check overlaps diff --git a/dpdk/scripts/checkpatches.sh b/dpdk/scripts/checkpatches.sh deleted file mode 100755 index 7111558f..00000000 --- a/dpdk/scripts/checkpatches.sh +++ /dev/null @@ -1,117 +0,0 @@ -#! /bin/sh - -# BSD LICENSE -# -# Copyright 2015 6WIND S.A. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in -# the documentation and/or other materials provided with the -# distribution. -# * Neither the name of 6WIND S.A. nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# Load config options: -# - DPDK_CHECKPATCH_PATH -# - DPDK_CHECKPATCH_LINE_LENGTH -. $(dirname $(readlink -e $0))/load-devel-config.sh - -length=${DPDK_CHECKPATCH_LINE_LENGTH:-80} - -# override default Linux options -options="--no-tree" -options="$options --max-line-length=$length" -options="$options --show-types" -options="$options --ignore=LINUX_VERSION_CODE,FILE_PATH_CHANGES,\ -VOLATILE,PREFER_PACKED,PREFER_ALIGNED,PREFER_PRINTF,PREFER_KERNEL_TYPES,BIT_MACRO,\ -SPLIT_STRING,LINE_SPACING,PARENTHESIS_ALIGNMENT,NETWORKING_BLOCK_COMMENT_STYLE,\ -NEW_TYPEDEFS,COMPARISON_TO_NULL" - -print_usage () { - cat <<- END_OF_HELP - usage: $(basename $0) [-q] [-v] [-nX|patch1 [patch2] ...]] - - Run Linux kernel checkpatch.pl with DPDK options. - The environment variable DPDK_CHECKPATCH_PATH must be set. - END_OF_HELP -} - -number=0 -quiet=false -verbose=false -while getopts hn:qv ARG ; do - case $ARG in - n ) number=$OPTARG ;; - q ) quiet=true && options="$options --no-summary" ;; - v ) verbose=true ;; - h ) print_usage ; exit 0 ;; - ? ) print_usage ; exit 1 ;; - esac -done -shift $(($OPTIND - 1)) - -if [ ! -x "$DPDK_CHECKPATCH_PATH" ] ; then - print_usage >&2 - echo - echo 'Cannot execute DPDK_CHECKPATCH_PATH' >&2 - exit 1 -fi - -total=0 -status=0 - -check () { # - total=$(($total + 1)) - ! $verbose || printf '\n### %s\n\n' "$3" - if [ -n "$1" ] ; then - report=$($DPDK_CHECKPATCH_PATH $options "$1" 2>/dev/null) - elif [ -n "$2" ] ; then - report=$(git format-patch --no-stat --stdout -1 $commit | - $DPDK_CHECKPATCH_PATH $options - 2>/dev/null) - fi - [ $? -ne 0 ] || continue - $verbose || printf '\n### %s\n\n' "$3" - printf '%s\n' "$report" | head -n -6 - status=$(($status + 1)) -} - -if [ -z "$1" ] ; then - if [ $number -eq 0 ] ; then - commits=$(git rev-list origin/master..) - else - commits=$(git rev-list --max-count=$number HEAD) - fi - for commit in $commits ; do - subject=$(git log --format='%s' -1 $commit) - check '' $commit "$subject" - done -else - for patch in "$@" ; do - subject=$(sed -n 's,^Subject: ,,p' "$patch") - check "$patch" '' "$subject" - done -fi -pass=$(($total - $status)) -$quiet || printf '%d/%d valid patch' $pass $total -$quiet || [ $pass -le 1 ] || printf 'es' -$quiet || printf '\n' -exit $status diff --git a/dpdk/scripts/cocci.sh b/dpdk/scripts/cocci.sh deleted file mode 100755 index 7acc256f..00000000 --- a/dpdk/scripts/cocci.sh +++ /dev/null @@ -1,64 +0,0 @@ -#! /bin/sh - -# BSD LICENSE -# -# Copyright 2015 EZchip Semiconductor Ltd. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in -# the documentation and/or other materials provided with the -# distribution. -# * Neither the name of EZchip Semiconductor nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# Apply coccinelle transforms. - -SRCTREE=$(readlink -f $(dirname $0)/..) -COCCI=$SRCTREE/scripts/cocci -[ -n "$SPATCH" ] || SPATCH=$(which spatch) - -PATCH_LIST="$@" -[ -n "$PATCH_LIST" ] || PATCH_LIST=$(echo $COCCI/*.cocci) - -[ -x "$SPATCH" ] || ( - echo "Coccinelle tools not installed." - exit 1 -) - -tmp=$(mktemp) - -for c in $PATCH_LIST; do - while true; do - echo -n "Applying $c..." - $SPATCH --sp-file $c -c --linux-spacing --very-quiet \ - --include-headers --preprocess \ - --in-place --dir $SRCTREE > $tmp - if [ -s $tmp ]; then - echo " changes applied, retrying." - else - echo " no change." - break; - fi - done -done - -rm -f $tmp diff --git a/dpdk/scripts/cocci/mtod-offset.cocci b/dpdk/scripts/cocci/mtod-offset.cocci deleted file mode 100644 index 13134e9d..00000000 --- a/dpdk/scripts/cocci/mtod-offset.cocci +++ /dev/null @@ -1,76 +0,0 @@ -// -// Replace explicit packet offset computations with rte_pktmbuf_mtod_offset(). -// -@disable paren@ -typedef uint8_t; -expression M, O; -@@ -( -- rte_pktmbuf_mtod(M, char *) + O -+ rte_pktmbuf_mtod_offset(M, char *, O) -| -- rte_pktmbuf_mtod(M, char *) - O -+ rte_pktmbuf_mtod_offset(M, char *, -O) -| -- rte_pktmbuf_mtod(M, unsigned char *) + O -+ rte_pktmbuf_mtod_offset(M, unsigned char *, O) -| -- rte_pktmbuf_mtod(M, unsigned char *) - O -+ rte_pktmbuf_mtod_offset(M, unsigned char *, -O) -| -- rte_pktmbuf_mtod(M, uint8_t *) + O -+ rte_pktmbuf_mtod_offset(M, uint8_t *, O) -| -- rte_pktmbuf_mtod(M, uint8_t *) - O -+ rte_pktmbuf_mtod_offset(M, uint8_t *, -O) -) - - -// -// Fold subsequent offset terms into pre-existing offset used in -// rte_pktmbuf_mtod_offset(). -// -@disable paren@ -expression M, O1, O2; -@@ -( -- rte_pktmbuf_mtod_offset(M, char *, O1) + O2 -+ rte_pktmbuf_mtod_offset(M, char *, O1 + O2) -| -- rte_pktmbuf_mtod_offset(M, char *, O1) - O2 -+ rte_pktmbuf_mtod_offset(M, char *, O1 - O2) -| -- rte_pktmbuf_mtod_offset(M, unsigned char *, O1) + O2 -+ rte_pktmbuf_mtod_offset(M, unsigned char *, O1 + O2) -| -- rte_pktmbuf_mtod_offset(M, unsigned char *, O1) - O2 -+ rte_pktmbuf_mtod_offset(M, unsigned char *, O1 - O2) -| -- rte_pktmbuf_mtod_offset(M, uint8_t *, O1) + O2 -+ rte_pktmbuf_mtod_offset(M, uint8_t *, O1 + O2) -| -- rte_pktmbuf_mtod_offset(M, uint8_t *, O1) - O2 -+ rte_pktmbuf_mtod_offset(M, uint8_t *, O1 - O2) -) - - -// -// Cleanup rules. Fold in double casts, remove unnecessary paranthesis, etc. -// -@disable paren@ -expression M, O; -type C, T; -@@ -( -- (C)rte_pktmbuf_mtod_offset(M, T, O) -+ rte_pktmbuf_mtod_offset(M, C, O) -| -- (rte_pktmbuf_mtod_offset(M, T, O)) -+ rte_pktmbuf_mtod_offset(M, T, O) -| -- (C)rte_pktmbuf_mtod(M, T) -+ rte_pktmbuf_mtod(M, C) -| -- (rte_pktmbuf_mtod(M, T)) -+ rte_pktmbuf_mtod(M, T) -) diff --git a/dpdk/scripts/depdirs-rule.sh b/dpdk/scripts/depdirs-rule.sh deleted file mode 100755 index 7aba0885..00000000 --- a/dpdk/scripts/depdirs-rule.sh +++ /dev/null @@ -1,95 +0,0 @@ -#!/bin/sh - -# BSD LICENSE -# -# Copyright(c) 2010-2014 Intel Corporation. All rights reserved. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in -# the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Intel Corporation nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# -# This (obscure) bash script finds the smallest different path between -# path1 and path2 given as command line argument. The given paths MUST -# be relative paths, the script is not designed to work with absolute -# paths. -# -# The script will then generate Makefile code that can be saved in a -# file and included in build system. -# -# For instance: -# depdirs-rule.sh a/b/c/d a/b/e/f -# Will print: -# FULL_DEPDIRS-a/b/c/d += a/b/e/f -# LOCAL_DEPDIRS-a/b/c += a/b/e -# -# The script returns 0 except if invalid arguments are given. -# - -if [ $# -ne 2 ]; then - echo "Bad arguments" - echo "Usage:" - echo " $0 path1 path2" - exit 1 -fi - -left1=${1%%/*} -right1=${1#*/} -prev_right1=$1 -prev_left1= - -left2=${2%%/*} -right2=${2#*/} -prev_right2=$2 -prev_left2= - -while [ "${right1}" != "" -a "${right2}" != "" ]; do - - if [ "$left1" != "$left2" ]; then - break - fi - - prev_left1=$left1 - left1=$left1/${right1%%/*} - prev_right1=$right1 - right1=${prev_right1#*/} - if [ "$right1" = "$prev_right1" ]; then - right1="" - fi - - prev_left2=$left2 - left2=$left2/${right2%%/*} - prev_right2=$right2 - right2=${prev_right2#*/} - if [ "$right2" = "$prev_right2" ]; then - right2="" - fi -done - -echo FULL_DEPDIRS-$1 += $2 -echo LOCAL_DEPDIRS-$left1 += $left2 - -exit 0 diff --git a/dpdk/scripts/load-devel-config.sh b/dpdk/scripts/load-devel-config.sh deleted file mode 100755 index 489f0075..00000000 --- a/dpdk/scripts/load-devel-config.sh +++ /dev/null @@ -1,14 +0,0 @@ -#! /bin/echo must be loaded with . - -# Load DPDK devel config and allow override -# from system file -test ! -r /etc/dpdk/devel.config || - . /etc/dpdk/devel.config -# from user file -test ! -r ~/.config/dpdk/devel.config || - . ~/.config/dpdk/devel.config -# from local file -test ! -r $(dirname $(readlink -m $0))/../.develconfig || - . $(dirname $(readlink -m $0))/../.develconfig - -# The config files must export variables in the shell style diff --git a/dpdk/scripts/relpath.sh b/dpdk/scripts/relpath.sh deleted file mode 100755 index 4ff4671e..00000000 --- a/dpdk/scripts/relpath.sh +++ /dev/null @@ -1,105 +0,0 @@ -#!/bin/sh - -# BSD LICENSE -# -# Copyright(c) 2010-2014 Intel Corporation. All rights reserved. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in -# the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Intel Corporation nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# -# print the relative path of $1 from $2 directory -# $1 and $2 MUST be absolute paths -# - -if [ $# -ne 2 ]; then - echo "Bad arguments" - echo "Usage:" - echo " $0 path1 path2" - exit 1 -fi - -# get the real absolute path, derefencing symlinks -ABS1=$(readlink -f $1) -ABS2=$(readlink -f $2) - -# remove leading slash -REL1=${ABS1#/} -REL2=${ABS2#/} - -left1=${REL1%%/*} -right1=${REL1#*/} -prev_right1=$REL1 -prev_left1= - -left2=${REL2%%/*} -right2=${REL2#*/} -prev_right2=$REL2 -prev_left2= - -prefix= - -while [ "${right1}" != "" -a "${right2}" != "" ]; do - - if [ "$left1" != "$left2" ]; then - break - fi - - prev_left1=$left1 - left1=$left1/${right1%%/*} - prev_right1=$right1 - right1=${prev_right1#*/} - if [ "$right1" = "$prev_right1" ]; then - right1="" - fi - - prev_left2=$left2 - left2=$left2/${right2%%/*} - prev_right2=$right2 - right2=${prev_right2#*/} - if [ "$right2" = "$prev_right2" ]; then - right2="" - fi -done - -if [ "${left1}" != "${left2}" ]; then - right2=${prev_right2} - right1=${prev_right1} -fi - -while [ "${right2}" != "" ]; do - prefix=${prefix}../ - prev_right2=$right2 - right2=${right2#*/} - if [ "$right2" = "$prev_right2" ]; then - right2="" - fi -done - -echo ${prefix}${right1} - -exit 0 diff --git a/dpdk/scripts/test-build.sh b/dpdk/scripts/test-build.sh deleted file mode 100755 index d2cafc19..00000000 --- a/dpdk/scripts/test-build.sh +++ /dev/null @@ -1,249 +0,0 @@ -#! /bin/sh -e - -# BSD LICENSE -# -# Copyright 2015 6WIND S.A. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in -# the documentation and/or other materials provided with the -# distribution. -# * Neither the name of 6WIND S.A. nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -default_path=$PATH - -# Load config options: -# - AESNI_MULTI_BUFFER_LIB_PATH -# - DPDK_BUILD_TEST_CONFIGS (defconfig1+option1+option2 defconfig2) -# - DPDK_DEP_ARCHIVE -# - DPDK_DEP_CFLAGS -# - DPDK_DEP_LDFLAGS -# - DPDK_DEP_MOFED (y/[n]) -# - DPDK_DEP_NUMA (y/[n]) -# - DPDK_DEP_PCAP (y/[n]) -# - DPDK_DEP_SSL (y/[n]) -# - DPDK_DEP_SZE (y/[n]) -# - DPDK_DEP_ZLIB (y/[n]) -# - DPDK_MAKE_JOBS (int) -# - DPDK_NOTIFY (notify-send) -# - LIBSSO_SNOW3G_PATH -# - LIBSSO_KASUMI_PATH -. $(dirname $(readlink -e $0))/load-devel-config.sh - -print_usage () { - echo "usage: $(basename $0) [-h] [-jX] [-s] [config1 [config2] ...]]" -} - -print_help () { - echo 'Test building several targets with different options' - echo - print_usage - cat <<- END_OF_HELP - - options: - -h this help - -jX use X parallel jobs in "make" - -s short test with only first config without examples/doc - -v verbose build - - config: defconfig[[~][+]option1[[~][+]option2...]] - Example: x86_64-native-linuxapp-gcc+debug~RXTX_CALLBACKS - The lowercase options are defined inside $(basename $0). - The uppercase options can be the end of a defconfig option - to enable if prefixed with '+' or to disable if prefixed with '~'. - Default is to automatically enable most of the options. - The external dependencies are setup with DPDK_DEP_* variables. - If no config on command line, DPDK_BUILD_TEST_CONFIGS is used. - END_OF_HELP -} - -J=$DPDK_MAKE_JOBS -short=false -unset verbose -maxerr=-Wfatal-errors -while getopts hj:sv ARG ; do - case $ARG in - j ) J=$OPTARG ;; - s ) short=true ;; - v ) verbose='V=1' ;; - h ) print_help ; exit 0 ;; - ? ) print_usage ; exit 1 ;; - esac -done -shift $(($OPTIND - 1)) -configs=${*:-$DPDK_BUILD_TEST_CONFIGS} - -success=false -on_exit () -{ - if $success ; then - [ "$DPDK_NOTIFY" != notify-send ] || \ - notify-send -u low --icon=dialog-information 'DPDK build' 'finished' - elif [ -z "$signal" ] ; then - [ -z "$dir" ] || echo "failed to build $dir" >&2 - [ "$DPDK_NOTIFY" != notify-send ] || \ - notify-send -u low --icon=dialog-error 'DPDK build' 'failed' - fi -} -# catch manual interrupt to ignore notification -trap "signal=INT ; trap - INT ; kill -INT $$" INT -# notify result on exit -trap on_exit EXIT - -cd $(dirname $(readlink -m $0))/.. - -reset_env () -{ - export PATH=$default_path - unset CROSS - unset DPDK_DEP_ARCHIVE - unset DPDK_DEP_CFLAGS - unset DPDK_DEP_LDFLAGS - unset DPDK_DEP_MOFED - unset DPDK_DEP_NUMA - unset DPDK_DEP_PCAP - unset DPDK_DEP_SSL - unset DPDK_DEP_SZE - unset DPDK_DEP_ZLIB - unset AESNI_MULTI_BUFFER_LIB_PATH - unset LIBSSO_SNOW3G_PATH - unset LIBSSO_KASUMI_PATH - unset PQOS_INSTALL_PATH -} - -config () # <directory> <target> <options> -{ - reconfig=false - if git rev-parse 2>&- && [ -n "$(git diff HEAD~ -- config)" ] ; then - echo 'Default config may have changed' - reconfig=true - fi - if [ ! -e $1/.config ] || $reconfig ; then - echo "================== Configure $1" - make T=$2 O=$1 config - - echo 'Customize configuration' - # Built-in options (lowercase) - ! echo $3 | grep -q '+default' || \ - sed -ri 's,(RTE_MACHINE=")native,\1default,' $1/.config - echo $3 | grep -q '+next' || \ - sed -ri 's,(NEXT_ABI=)y,\1n,' $1/.config - ! echo $3 | grep -q '+shared' || \ - sed -ri 's,(SHARED_LIB=)n,\1y,' $1/.config - ! echo $3 | grep -q '+debug' || ( \ - sed -ri 's,(RTE_LOG_LEVEL=).*,\1RTE_LOG_DEBUG,' $1/.config - sed -ri 's,(_DEBUG.*=)n,\1y,' $1/.config - sed -ri 's,(_STAT.*=)n,\1y,' $1/.config - sed -ri 's,(TEST_PMD_RECORD_.*=)n,\1y,' $1/.config ) - - # Automatic configuration - test "$DPDK_DEP_NUMA" != y || \ - sed -ri 's,(NUMA=)n,\1y,' $1/.config - sed -ri 's,(LIBRTE_IEEE1588=)n,\1y,' $1/.config - sed -ri 's,(BYPASS=)n,\1y,' $1/.config - test "$DPDK_DEP_ARCHIVE" != y || \ - sed -ri 's,(RESOURCE_TAR=)n,\1y,' $1/.config - test "$DPDK_DEP_MOFED" != y || \ - sed -ri 's,(MLX._PMD=)n,\1y,' $1/.config - test "$DPDK_DEP_SZE" != y || \ - sed -ri 's,(PMD_SZEDATA2=)n,\1y,' $1/.config - test "$DPDK_DEP_ZLIB" != y || \ - sed -ri 's,(BNX2X_PMD=)n,\1y,' $1/.config - test "$DPDK_DEP_ZLIB" != y || \ - sed -ri 's,(QEDE_PMD=)n,\1y,' $1/.config - sed -ri 's,(NFP_PMD=)n,\1y,' $1/.config - test "$DPDK_DEP_PCAP" != y || \ - sed -ri 's,(PCAP=)n,\1y,' $1/.config - test -z "$AESNI_MULTI_BUFFER_LIB_PATH" || \ - sed -ri 's,(PMD_AESNI_MB=)n,\1y,' $1/.config - test -z "$AESNI_MULTI_BUFFER_LIB_PATH" || \ - sed -ri 's,(PMD_AESNI_GCM=)n,\1y,' $1/.config - test -z "$LIBSSO_SNOW3G_PATH" || \ - sed -ri 's,(PMD_SNOW3G=)n,\1y,' $1/.config - test -z "$LIBSSO_KASUMI_PATH" || \ - sed -ri 's,(PMD_KASUMI=)n,\1y,' $1/.config - test "$DPDK_DEP_SSL" != y || \ - sed -ri 's,(PMD_QAT=)n,\1y,' $1/.config - sed -ri 's,(KNI_VHOST.*=)n,\1y,' $1/.config - sed -ri 's,(SCHED_.*=)n,\1y,' $1/.config - build_config_hook $1 $2 $3 - - # Explicit enabler/disabler (uppercase) - for option in $(echo $3 | sed 's,[~+], &,g') ; do - pattern=$(echo $option | cut -c2-) - if echo $option | grep -q '^~' ; then - sed -ri "s,($pattern=)y,\1n," $1/.config - elif echo $option | grep -q '^+' ; then - sed -ri "s,($pattern=)n,\1y," $1/.config - fi - done - fi -} - -# default empty hook to override in devel config -build_config_hook () # <directory> <target> <options> -{ - : -} - -for conf in $configs ; do - target=$(echo $conf | sed 's,[~+].*,,') - # reload config with DPDK_TARGET set - DPDK_TARGET=$target - reset_env - . $(dirname $(readlink -e $0))/load-devel-config.sh - - options=$(echo $conf | sed 's,[^~+]*,,') - dir=$conf - config $dir $target $options - - echo "================== Build $dir" - make -j$J EXTRA_CFLAGS="$maxerr $DPDK_DEP_CFLAGS" \ - EXTRA_LDFLAGS="$DPDK_DEP_LDFLAGS" $verbose O=$dir - ! $short || break - echo "================== Build examples for $dir" - export RTE_SDK=$(pwd) - export RTE_TARGET=$dir - make -j$J -sC examples \ - EXTRA_LDFLAGS="$DPDK_DEP_LDFLAGS" $verbose \ - O=$(readlink -m $dir/examples) - ! echo $target | grep -q '^x86_64' || \ - make -j$J -sC examples/performance-thread \ - EXTRA_LDFLAGS="$DPDK_DEP_LDFLAGS" $verbose \ - O=$(readlink -m $dir/examples/performance-thread) - unset RTE_TARGET - echo "################## $dir done." - unset dir -done - -if ! $short ; then - mkdir -p .check - echo "================== Build doxygen HTML API" - make doc-api-html >/dev/null 2>.check/doc.txt - echo "================== Build sphinx HTML guides" - make doc-guides-html >/dev/null 2>>.check/doc.txt - echo "================== Check docs" - diff -u /dev/null .check/doc.txt -fi - -success=true diff --git a/dpdk/scripts/test-null.sh b/dpdk/scripts/test-null.sh deleted file mode 100755 index 32a47b17..00000000 --- a/dpdk/scripts/test-null.sh +++ /dev/null @@ -1,45 +0,0 @@ -#! /bin/sh -e - -# BSD LICENSE -# -# Copyright 2015 6WIND S.A. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in -# the documentation and/or other materials provided with the -# distribution. -# * Neither the name of 6WIND S.A. nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# Run a quick testpmd forwarding with null PMD without hugepage - -build=${1:-build} -coremask=${2:-3} # default using cores 0 and 1 - -if grep -q SHARED_LIB=y $build/.config; then - pmd='-d librte_pmd_null.so' -fi - -(sleep 1 && echo stop) | -$build/app/testpmd -c $coremask -n 1 --no-huge \ - $pmd --vdev eth_null1 --vdev eth_null2 -- \ - --total-num-mbufs=2048 -ia diff --git a/dpdk/scripts/validate-abi.sh b/dpdk/scripts/validate-abi.sh deleted file mode 100755 index feda6c86..00000000 --- a/dpdk/scripts/validate-abi.sh +++ /dev/null @@ -1,262 +0,0 @@ -#!/bin/sh -# BSD LICENSE -# -# Copyright(c) 2015 Neil Horman. All rights reserved. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in -# the documentation and/or other materials provided with the -# distribution. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -TAG1=$1 -TAG2=$2 -TARGET=$3 -ABI_DIR=`mktemp -d -p /tmp ABI.XXXXXX` - -usage() { - echo "$0 <REV1> <REV2> <TARGET>" -} - -log() { - local level=$1 - shift - echo "$*" -} - -validate_tags() { - - if [ -z "$HASH1" ] - then - echo "invalid revision: $TAG1" - return - fi - if [ -z "$HASH2" ] - then - echo "invalid revision: $TAG2" - return - fi -} - -validate_args() { - if [ -z "$TAG1" ] - then - echo "Must Specify REV1" - return - fi - if [ -z "$TAG2" ] - then - echo "Must Specify REV2" - return - fi - if [ -z "$TARGET" ] - then - echo "Must Specify a build target" - fi -} - - -cleanup_and_exit() { - rm -rf $ABI_DIR - git checkout $CURRENT_BRANCH - exit $1 -} - -# Make sure we configure SHARED libraries -# Also turn off IGB and KNI as those require kernel headers to build -fixup_config() { - sed -i -e"$ a\CONFIG_RTE_BUILD_SHARED_LIB=y" config/defconfig_$TARGET - sed -i -e"$ a\CONFIG_RTE_NEXT_ABI=n" config/defconfig_$TARGET - sed -i -e"$ a\CONFIG_RTE_EAL_IGB_UIO=n" config/defconfig_$TARGET - sed -i -e"$ a\CONFIG_RTE_LIBRTE_KNI=n" config/defconfig_$TARGET - sed -i -e"$ a\CONFIG_RTE_KNI_KMOD=n" config/defconfig_$TARGET -} - -########################################### -#START -############################################ - -#trap on ctrl-c to clean up -trap cleanup_and_exit SIGINT - -if [ -z "$DPDK_MAKE_JOBS" ] -then - # This counts the number of cpus on the system - if [ -e /usr/bin/lscpu ] - then - DPDK_MAKE_JOBS=`lscpu -p=cpu | grep -v "#" | wc -l` - else - DPDK_MAKE_JOBS=1 - fi -fi - -#Save the current branch -CURRENT_BRANCH=`git branch | grep \* | cut -d' ' -f2` - -if [ -z "$CURRENT_BRANCH" ] -then - CURRENT_BRANCH=`git log --pretty=format:%H HEAD~1..HEAD` -fi - -if [ -n "$VERBOSE" ] -then - export VERBOSE=/dev/stdout -else - export VERBOSE=/dev/null -fi - -# Validate that we have all the arguments we need -res=$(validate_args) -if [ -n "$res" ] -then - echo $res - usage - cleanup_and_exit 1 -fi - -HASH1=$(git show -s --format=%H "$TAG1" -- 2> /dev/null | tail -1) -HASH2=$(git show -s --format=%H "$TAG2" -- 2> /dev/null | tail -1) - -# Make sure our tags exist -res=$(validate_tags) -if [ -n "$res" ] -then - echo $res - cleanup_and_exit 1 -fi - -# Make hashes available in output for non-local reference -TAG1="$TAG1 ($HASH1)" -TAG2="$TAG2 ($HASH2)" - -ABICHECK=`which abi-compliance-checker 2>/dev/null` -if [ $? -ne 0 ] -then - log "INFO" "Cant find abi-compliance-checker utility" - cleanup_and_exit 1 -fi - -ABIDUMP=`which abi-dumper 2>/dev/null` -if [ $? -ne 0 ] -then - log "INFO" "Cant find abi-dumper utility" - cleanup_and_exit 1 -fi - -log "INFO" "We're going to check and make sure that applications built" -log "INFO" "against DPDK DSOs from version $TAG1 will still run when executed" -log "INFO" "against DPDK DSOs built from version $TAG2." -log "INFO" "" - -# Check to make sure we have a clean tree -git status | grep -q clean -if [ $? -ne 0 ] -then - log "WARN" "Working directory not clean, aborting" - cleanup_and_exit 1 -fi - -# Move to the root of the git tree -cd $(dirname $0)/.. - -log "INFO" "Checking out version $TAG1 of the dpdk" -# Move to the old version of the tree -git checkout $HASH1 - -fixup_config - -# Checking abi compliance relies on using the dwarf information in -# The shared objects. Thats only included in the DSO's if we build -# with -g -export EXTRA_CFLAGS="$EXTRA_CFLAGS -g" -export EXTRA_LDFLAGS="$EXTRA_LDFLAGS -g" - -# Now configure the build -log "INFO" "Configuring DPDK $TAG1" -make config T=$TARGET O=$TARGET > $VERBOSE 2>&1 - -log "INFO" "Building DPDK $TAG1. This might take a moment" -make -j$DPDK_MAKE_JOBS O=$TARGET > $VERBOSE 2>&1 - -if [ $? -ne 0 ] -then - log "INFO" "THE BUILD FAILED. ABORTING" - cleanup_and_exit 1 -fi - -# Move to the lib directory -cd $TARGET/lib -log "INFO" "COLLECTING ABI INFORMATION FOR $TAG1" -for i in `ls *.so` -do - $ABIDUMP $i -o $ABI_DIR/$i-ABI-0.dump -lver $HASH1 -done -cd ../.. - -# Now clean the tree, checkout the second tag, and rebuild -git clean -f -d -git reset --hard -# Move to the new version of the tree -log "INFO" "Checking out version $TAG2 of the dpdk" -git checkout $HASH2 - -fixup_config - -# Now configure the build -log "INFO" "Configuring DPDK $TAG2" -make config T=$TARGET O=$TARGET > $VERBOSE 2>&1 - -log "INFO" "Building DPDK $TAG2. This might take a moment" -make -j$DPDK_MAKE_JOBS O=$TARGET > $VERBOSE 2>&1 - -if [ $? -ne 0 ] -then - log "INFO" "THE BUILD FAILED. ABORTING" - cleanup_and_exit 1 -fi - -cd $TARGET/lib -log "INFO" "COLLECTING ABI INFORMATION FOR $TAG2" -for i in `ls *.so` -do - $ABIDUMP $i -o $ABI_DIR/$i-ABI-1.dump -lver $HASH2 -done -cd ../.. - -# Start comparison of ABI dumps -for i in `ls $ABI_DIR/*-1.dump` -do - NEWNAME=`basename $i` - OLDNAME=`basename $i | sed -e"s/1.dump/0.dump/"` - LIBNAME=`basename $i | sed -e"s/-ABI-1.dump//"` - - if [ ! -f $ABI_DIR/$OLDNAME ] - then - log "INFO" "$OLDNAME DOES NOT EXIST IN $TAG1. SKIPPING..." - fi - - #compare the abi dumps - $ABICHECK -l $LIBNAME -old $ABI_DIR/$OLDNAME -new $ABI_DIR/$NEWNAME -done - -git reset --hard -log "INFO" "ABI CHECK COMPLETE. REPORTS ARE IN compat_report directory" -cleanup_and_exit 0 diff --git a/dpdk/test/Makefile b/dpdk/test/Makefile new file mode 100644 index 00000000..e996fd8c --- /dev/null +++ b/dpdk/test/Makefile @@ -0,0 +1,39 @@ +# BSD LICENSE +# +# Copyright(c) 2017 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +include $(RTE_SDK)/mk/rte.vars.mk + +DIRS-$(CONFIG_RTE_APP_TEST) += test +DIRS-$(CONFIG_RTE_LIBRTE_ACL) += test-acl +DIRS-$(CONFIG_RTE_LIBRTE_PIPELINE) += test-pipeline +DIRS-$(CONFIG_RTE_LIBRTE_CMDLINE) += cmdline_test + +include $(RTE_SDK)/mk/rte.subdir.mk diff --git a/dpdk/test/cmdline_test/Makefile b/dpdk/test/cmdline_test/Makefile new file mode 100644 index 00000000..e9eafd2d --- /dev/null +++ b/dpdk/test/cmdline_test/Makefile @@ -0,0 +1,52 @@ +# BSD LICENSE +# +# Copyright(c) 2010-2014 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +include $(RTE_SDK)/mk/rte.vars.mk + +ifeq ($(CONFIG_RTE_LIBRTE_CMDLINE),y) + +# +# library name +# +APP = cmdline_test + +# +# all sources are stored in SRCS-y +# +SRCS-y += cmdline_test.c +SRCS-y += commands.c + +CFLAGS += -O3 +CFLAGS += $(WERROR_FLAGS) + +include $(RTE_SDK)/mk/rte.app.mk + +endif diff --git a/dpdk/app/cmdline_test/cmdline_test.c b/dpdk/test/cmdline_test/cmdline_test.c similarity index 100% rename from dpdk/app/cmdline_test/cmdline_test.c rename to dpdk/test/cmdline_test/cmdline_test.c diff --git a/dpdk/app/cmdline_test/cmdline_test.h b/dpdk/test/cmdline_test/cmdline_test.h similarity index 100% rename from dpdk/app/cmdline_test/cmdline_test.h rename to dpdk/test/cmdline_test/cmdline_test.h diff --git a/dpdk/test/cmdline_test/cmdline_test.py b/dpdk/test/cmdline_test/cmdline_test.py new file mode 100644 index 00000000..229f71f3 --- /dev/null +++ b/dpdk/test/cmdline_test/cmdline_test.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python + +# BSD LICENSE +# +# Copyright(c) 2010-2014 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Script that runs cmdline_test app and feeds keystrokes into it. +from __future__ import print_function +import cmdline_test_data +import os +import pexpect +import sys + + +# +# function to run test +# +def runTest(child, test): + child.send(test["Sequence"]) + if test["Result"] is None: + return 0 + child.expect(test["Result"], 1) + + +# +# history test is a special case +# +# This test does the following: +# 1) fills the history with garbage up to its full capacity +# (just enough to remove last entry) +# 2) scrolls back history to the very beginning +# 3) checks if the output is as expected, that is, the first +# number in the sequence (not the last entry before it) +# +# This is a self-contained test, it needs only a pexpect child +# +def runHistoryTest(child): + # find out history size + child.sendline(cmdline_test_data.CMD_GET_BUFSIZE) + child.expect("History buffer size: \\d+", timeout=1) + history_size = int(child.after[len(cmdline_test_data.BUFSIZE_TEMPLATE):]) + i = 0 + + # fill the history with numbers + while i < history_size / 10: + # add 1 to prevent from parsing as octals + child.send("1" + str(i).zfill(8) + cmdline_test_data.ENTER) + # the app will simply print out the number + child.expect(str(i + 100000000), timeout=1) + i += 1 + # scroll back history + child.send(cmdline_test_data.UP * (i + 2) + cmdline_test_data.ENTER) + child.expect("100000000", timeout=1) + +# the path to cmdline_test executable is supplied via command-line. +if len(sys.argv) < 2: + print("Error: please supply cmdline_test app path") + sys.exit(1) + +test_app_path = sys.argv[1] + +if not os.path.exists(test_app_path): + print("Error: please supply cmdline_test app path") + sys.exit(1) + +child = pexpect.spawn(test_app_path) + +print("Running command-line tests...") +for test in cmdline_test_data.tests: + testname = (test["Name"] + ":").ljust(30) + try: + runTest(child, test) + print(testname, "PASS") + except: + print(testname, "FAIL") + print(child) + sys.exit(1) + +# since last test quits the app, run new instance +child = pexpect.spawn(test_app_path) + +testname = ("History fill test:").ljust(30) +try: + runHistoryTest(child) + print(testname, "PASS") +except: + print(testname, "FAIL") + print(child) + sys.exit(1) +child.close() +sys.exit(0) diff --git a/dpdk/test/cmdline_test/cmdline_test_data.py b/dpdk/test/cmdline_test/cmdline_test_data.py new file mode 100644 index 00000000..28dfefe1 --- /dev/null +++ b/dpdk/test/cmdline_test/cmdline_test_data.py @@ -0,0 +1,310 @@ +# BSD LICENSE +# +# Copyright(c) 2010-2014 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# collection of static data + +# keycode constants +CTRL_A = chr(1) +CTRL_B = chr(2) +CTRL_C = chr(3) +CTRL_D = chr(4) +CTRL_E = chr(5) +CTRL_F = chr(6) +CTRL_K = chr(11) +CTRL_L = chr(12) +CTRL_N = chr(14) +CTRL_P = chr(16) +CTRL_W = chr(23) +CTRL_Y = chr(25) +ALT_B = chr(27) + chr(98) +ALT_D = chr(27) + chr(100) +ALT_F = chr(27) + chr(102) +ALT_BKSPACE = chr(27) + chr(127) +DEL = chr(27) + chr(91) + chr(51) + chr(126) +TAB = chr(9) +HELP = chr(63) +BKSPACE = chr(127) +RIGHT = chr(27) + chr(91) + chr(67) +DOWN = chr(27) + chr(91) + chr(66) +LEFT = chr(27) + chr(91) + chr(68) +UP = chr(27) + chr(91) + chr(65) +ENTER2 = '\r' +ENTER = '\n' + +# expected result constants +NOT_FOUND = "Command not found" +BAD_ARG = "Bad arguments" +AMBIG = "Ambiguous command" +CMD1 = "Command 1 parsed!" +CMD2 = "Command 2 parsed!" +SINGLE = "Single word command parsed!" +SINGLE_LONG = "Single long word command parsed!" +AUTO1 = "Autocomplete command 1 parsed!" +AUTO2 = "Autocomplete command 2 parsed!" + +# misc defines +CMD_QUIT = "quit" +CMD_GET_BUFSIZE = "get_history_bufsize" +BUFSIZE_TEMPLATE = "History buffer size: " +PROMPT = "CMDLINE_TEST>>" + +# test defines +# each test tests progressively diverse set of keys. this way for example +# if we want to use some key sequence in the test, we first need to test +# that it itself does what it is expected to do. Most of the tests are +# designed that way. +# +# example: "arrows & delete test 1". we enter a partially valid command, +# then move 3 chars left and use delete three times. this way we get to +# know that "delete", "left" and "ctrl+B" all work (because if any of +# them fails, the whole test will fail and next tests won't be run). +# +# each test consists of name, character sequence to send to child, +# and expected output (if any). + +tests = [ + # test basic commands + {"Name": "command test 1", + "Sequence": "ambiguous first" + ENTER, + "Result": CMD1}, + {"Name": "command test 2", + "Sequence": "ambiguous second" + ENTER, + "Result": CMD2}, + {"Name": "command test 3", + "Sequence": "ambiguous ambiguous" + ENTER, + "Result": AMBIG}, + {"Name": "command test 4", + "Sequence": "ambiguous ambiguous2" + ENTER, + "Result": AMBIG}, + + {"Name": "invalid command test 1", + "Sequence": "ambiguous invalid" + ENTER, + "Result": BAD_ARG}, + # test invalid commands + {"Name": "invalid command test 2", + "Sequence": "invalid" + ENTER, + "Result": NOT_FOUND}, + {"Name": "invalid command test 3", + "Sequence": "ambiguousinvalid" + ENTER2, + "Result": NOT_FOUND}, + + # test arrows and deletes + {"Name": "arrows & delete test 1", + "Sequence": "singlebad" + LEFT*2 + CTRL_B + DEL*3 + ENTER, + "Result": SINGLE}, + {"Name": "arrows & delete test 2", + "Sequence": "singlebad" + LEFT*5 + RIGHT + CTRL_F + DEL*3 + ENTER, + "Result": SINGLE}, + + # test backspace + {"Name": "backspace test", + "Sequence": "singlebad" + BKSPACE*3 + ENTER, + "Result": SINGLE}, + + # test goto left and goto right + {"Name": "goto left test", + "Sequence": "biguous first" + CTRL_A + "am" + ENTER, + "Result": CMD1}, + {"Name": "goto right test", + "Sequence": "biguous fir" + CTRL_A + "am" + CTRL_E + "st" + ENTER, + "Result": CMD1}, + + # test goto words + {"Name": "goto left word test", + "Sequence": "ambiguous st" + ALT_B + "fir" + ENTER, + "Result": CMD1}, + {"Name": "goto right word test", + "Sequence": "ambig first" + CTRL_A + ALT_F + "uous" + ENTER, + "Result": CMD1}, + + # test removing words + {"Name": "remove left word 1", + "Sequence": "single invalid" + CTRL_W + ENTER, + "Result": SINGLE}, + {"Name": "remove left word 2", + "Sequence": "single invalid" + ALT_BKSPACE + ENTER, + "Result": SINGLE}, + {"Name": "remove right word", + "Sequence": "single invalid" + ALT_B + ALT_D + ENTER, + "Result": SINGLE}, + + # test kill buffer (copy and paste) + {"Name": "killbuffer test 1", + "Sequence": "ambiguous" + CTRL_A + CTRL_K + " first" + CTRL_A + + CTRL_Y + ENTER, + "Result": CMD1}, + {"Name": "killbuffer test 2", + "Sequence": "ambiguous" + CTRL_A + CTRL_K + CTRL_Y*26 + ENTER, + "Result": NOT_FOUND}, + + # test newline + {"Name": "newline test", + "Sequence": "invalid" + CTRL_C + "single" + ENTER, + "Result": SINGLE}, + + # test redisplay (nothing should really happen) + {"Name": "redisplay test", + "Sequence": "single" + CTRL_L + ENTER, + "Result": SINGLE}, + + # test autocomplete + {"Name": "autocomplete test 1", + "Sequence": "si" + TAB + ENTER, + "Result": SINGLE}, + {"Name": "autocomplete test 2", + "Sequence": "si" + TAB + "_" + TAB + ENTER, + "Result": SINGLE_LONG}, + {"Name": "autocomplete test 3", + "Sequence": "in" + TAB + ENTER, + "Result": NOT_FOUND}, + {"Name": "autocomplete test 4", + "Sequence": "am" + TAB + ENTER, + "Result": BAD_ARG}, + {"Name": "autocomplete test 5", + "Sequence": "am" + TAB + "fir" + TAB + ENTER, + "Result": CMD1}, + {"Name": "autocomplete test 6", + "Sequence": "am" + TAB + "fir" + TAB + TAB + ENTER, + "Result": CMD1}, + {"Name": "autocomplete test 7", + "Sequence": "am" + TAB + "fir" + TAB + " " + TAB + ENTER, + "Result": CMD1}, + {"Name": "autocomplete test 8", + "Sequence": "am" + TAB + " am" + TAB + " " + ENTER, + "Result": AMBIG}, + {"Name": "autocomplete test 9", + "Sequence": "am" + TAB + "inv" + TAB + ENTER, + "Result": BAD_ARG}, + {"Name": "autocomplete test 10", + "Sequence": "au" + TAB + ENTER, + "Result": NOT_FOUND}, + {"Name": "autocomplete test 11", + "Sequence": "au" + TAB + "1" + ENTER, + "Result": AUTO1}, + {"Name": "autocomplete test 12", + "Sequence": "au" + TAB + "2" + ENTER, + "Result": AUTO2}, + {"Name": "autocomplete test 13", + "Sequence": "au" + TAB + "2" + TAB + ENTER, + "Result": AUTO2}, + {"Name": "autocomplete test 14", + "Sequence": "au" + TAB + "2 " + TAB + ENTER, + "Result": AUTO2}, + {"Name": "autocomplete test 15", + "Sequence": "24" + TAB + ENTER, + "Result": "24"}, + + # test history + {"Name": "history test 1", + "Sequence": "invalid" + ENTER + "single" + ENTER + "invalid" + + ENTER + UP + CTRL_P + ENTER, + "Result": SINGLE}, + {"Name": "history test 2", + "Sequence": "invalid" + ENTER + "ambiguous first" + ENTER + "invalid" + + ENTER + "single" + ENTER + UP * 3 + CTRL_N + DOWN + ENTER, + "Result": SINGLE}, + + # + # tests that improve coverage + # + + # empty space tests + {"Name": "empty space test 1", + "Sequence": RIGHT + LEFT + CTRL_B + CTRL_F + ENTER, + "Result": PROMPT}, + {"Name": "empty space test 2", + "Sequence": BKSPACE + ENTER, + "Result": PROMPT}, + {"Name": "empty space test 3", + "Sequence": CTRL_E*2 + CTRL_A*2 + ENTER, + "Result": PROMPT}, + {"Name": "empty space test 4", + "Sequence": ALT_F*2 + ALT_B*2 + ENTER, + "Result": PROMPT}, + {"Name": "empty space test 5", + "Sequence": " " + CTRL_E*2 + CTRL_A*2 + ENTER, + "Result": PROMPT}, + {"Name": "empty space test 6", + "Sequence": " " + CTRL_A + ALT_F*2 + ALT_B*2 + ENTER, + "Result": PROMPT}, + {"Name": "empty space test 7", + "Sequence": " " + CTRL_A + CTRL_D + CTRL_E + CTRL_D + ENTER, + "Result": PROMPT}, + {"Name": "empty space test 8", + "Sequence": " space" + CTRL_W*2 + ENTER, + "Result": PROMPT}, + {"Name": "empty space test 9", + "Sequence": " space" + ALT_BKSPACE*2 + ENTER, + "Result": PROMPT}, + {"Name": "empty space test 10", + "Sequence": " space " + CTRL_A + ALT_D*3 + ENTER, + "Result": PROMPT}, + + # non-printable char tests + {"Name": "non-printable test 1", + "Sequence": chr(27) + chr(47) + ENTER, + "Result": PROMPT}, + {"Name": "non-printable test 2", + "Sequence": chr(27) + chr(128) + ENTER*7, + "Result": PROMPT}, + {"Name": "non-printable test 3", + "Sequence": chr(27) + chr(91) + chr(127) + ENTER*6, + "Result": PROMPT}, + + # miscellaneous tests + {"Name": "misc test 1", + "Sequence": ENTER, + "Result": PROMPT}, + {"Name": "misc test 2", + "Sequence": "single #comment" + ENTER, + "Result": SINGLE}, + {"Name": "misc test 3", + "Sequence": "#empty line" + ENTER, + "Result": PROMPT}, + {"Name": "misc test 4", + "Sequence": " single " + ENTER, + "Result": SINGLE}, + {"Name": "misc test 5", + "Sequence": "single#" + ENTER, + "Result": SINGLE}, + {"Name": "misc test 6", + "Sequence": 'a' * 257 + ENTER, + "Result": NOT_FOUND}, + {"Name": "misc test 7", + "Sequence": "clear_history" + UP*5 + DOWN*5 + ENTER, + "Result": PROMPT}, + {"Name": "misc test 8", + "Sequence": "a" + HELP + CTRL_C, + "Result": PROMPT}, + {"Name": "misc test 9", + "Sequence": CTRL_D*3, + "Result": None}, +] diff --git a/dpdk/app/cmdline_test/commands.c b/dpdk/test/cmdline_test/commands.c similarity index 100% rename from dpdk/app/cmdline_test/commands.c rename to dpdk/test/cmdline_test/commands.c diff --git a/dpdk/test/test-acl/Makefile b/dpdk/test/test-acl/Makefile new file mode 100644 index 00000000..29de80a3 --- /dev/null +++ b/dpdk/test/test-acl/Makefile @@ -0,0 +1,45 @@ +# BSD LICENSE +# +# Copyright(c) 2010-2014 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +include $(RTE_SDK)/mk/rte.vars.mk + +ifeq ($(CONFIG_RTE_LIBRTE_ACL),y) + +APP = testacl + +CFLAGS += $(WERROR_FLAGS) + +# all source are stored in SRCS-y +SRCS-y := main.c + +include $(RTE_SDK)/mk/rte.app.mk + +endif diff --git a/dpdk/test/test-acl/main.c b/dpdk/test/test-acl/main.c new file mode 100644 index 00000000..1b2b1760 --- /dev/null +++ b/dpdk/test/test-acl/main.c @@ -0,0 +1,1125 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <rte_acl.h> +#include <getopt.h> +#include <string.h> + +#include <rte_cycles.h> +#include <rte_per_lcore.h> +#include <rte_lcore.h> +#include <rte_ip.h> + +#define PRINT_USAGE_START "%s [EAL options]\n" + +#define RTE_LOGTYPE_TESTACL RTE_LOGTYPE_USER1 + +#define APP_NAME "TESTACL" + +#define GET_CB_FIELD(in, fd, base, lim, dlm) do { \ + unsigned long val; \ + char *end_fld; \ + errno = 0; \ + val = strtoul((in), &end_fld, (base)); \ + if (errno != 0 || end_fld[0] != (dlm) || val > (lim)) \ + return -EINVAL; \ + (fd) = (typeof(fd))val; \ + (in) = end_fld + 1; \ +} while (0) + +#define OPT_RULE_FILE "rulesf" +#define OPT_TRACE_FILE "tracef" +#define OPT_RULE_NUM "rulenum" +#define OPT_TRACE_NUM "tracenum" +#define OPT_TRACE_STEP "tracestep" +#define OPT_SEARCH_ALG "alg" +#define OPT_BLD_CATEGORIES "bldcat" +#define OPT_RUN_CATEGORIES "runcat" +#define OPT_MAX_SIZE "maxsize" +#define OPT_ITER_NUM "iter" +#define OPT_VERBOSE "verbose" +#define OPT_IPV6 "ipv6" + +#define TRACE_DEFAULT_NUM 0x10000 +#define TRACE_STEP_MAX 0x1000 +#define TRACE_STEP_DEF 0x100 + +#define RULE_NUM 0x10000 + +enum { + DUMP_NONE, + DUMP_SEARCH, + DUMP_PKT, + DUMP_MAX +}; + +struct acl_alg { + const char *name; + enum rte_acl_classify_alg alg; +}; + +static const struct acl_alg acl_alg[] = { + { + .name = "scalar", + .alg = RTE_ACL_CLASSIFY_SCALAR, + }, + { + .name = "sse", + .alg = RTE_ACL_CLASSIFY_SSE, + }, + { + .name = "avx2", + .alg = RTE_ACL_CLASSIFY_AVX2, + }, + { + .name = "neon", + .alg = RTE_ACL_CLASSIFY_NEON, + }, + { + .name = "altivec", + .alg = RTE_ACL_CLASSIFY_ALTIVEC, + }, +}; + +static struct { + const char *prgname; + const char *rule_file; + const char *trace_file; + size_t max_size; + uint32_t bld_categories; + uint32_t run_categories; + uint32_t nb_rules; + uint32_t nb_traces; + uint32_t trace_step; + uint32_t trace_sz; + uint32_t iter_num; + uint32_t verbose; + uint32_t ipv6; + struct acl_alg alg; + uint32_t used_traces; + void *traces; + struct rte_acl_ctx *acx; +} config = { + .bld_categories = 3, + .run_categories = 1, + .nb_rules = RULE_NUM, + .nb_traces = TRACE_DEFAULT_NUM, + .trace_step = TRACE_STEP_DEF, + .iter_num = 1, + .verbose = DUMP_MAX, + .alg = { + .name = "default", + .alg = RTE_ACL_CLASSIFY_DEFAULT, + }, + .ipv6 = 0 +}; + +static struct rte_acl_param prm = { + .name = APP_NAME, + .socket_id = SOCKET_ID_ANY, +}; + +/* + * Rule and trace formats definitions. + */ + +struct ipv4_5tuple { + uint8_t proto; + uint32_t ip_src; + uint32_t ip_dst; + uint16_t port_src; + uint16_t port_dst; +}; + +enum { + PROTO_FIELD_IPV4, + SRC_FIELD_IPV4, + DST_FIELD_IPV4, + SRCP_FIELD_IPV4, + DSTP_FIELD_IPV4, + NUM_FIELDS_IPV4 +}; + +/* + * That effectively defines order of IPV4VLAN classifications: + * - PROTO + * - VLAN (TAG and DOMAIN) + * - SRC IP ADDRESS + * - DST IP ADDRESS + * - PORTS (SRC and DST) + */ +enum { + RTE_ACL_IPV4VLAN_PROTO, + RTE_ACL_IPV4VLAN_VLAN, + RTE_ACL_IPV4VLAN_SRC, + RTE_ACL_IPV4VLAN_DST, + RTE_ACL_IPV4VLAN_PORTS, + RTE_ACL_IPV4VLAN_NUM +}; + +struct rte_acl_field_def ipv4_defs[NUM_FIELDS_IPV4] = { + { + .type = RTE_ACL_FIELD_TYPE_BITMASK, + .size = sizeof(uint8_t), + .field_index = PROTO_FIELD_IPV4, + .input_index = RTE_ACL_IPV4VLAN_PROTO, + .offset = offsetof(struct ipv4_5tuple, proto), + }, + { + .type = RTE_ACL_FIELD_TYPE_MASK, + .size = sizeof(uint32_t), + .field_index = SRC_FIELD_IPV4, + .input_index = RTE_ACL_IPV4VLAN_SRC, + .offset = offsetof(struct ipv4_5tuple, ip_src), + }, + { + .type = RTE_ACL_FIELD_TYPE_MASK, + .size = sizeof(uint32_t), + .field_index = DST_FIELD_IPV4, + .input_index = RTE_ACL_IPV4VLAN_DST, + .offset = offsetof(struct ipv4_5tuple, ip_dst), + }, + { + .type = RTE_ACL_FIELD_TYPE_RANGE, + .size = sizeof(uint16_t), + .field_index = SRCP_FIELD_IPV4, + .input_index = RTE_ACL_IPV4VLAN_PORTS, + .offset = offsetof(struct ipv4_5tuple, port_src), + }, + { + .type = RTE_ACL_FIELD_TYPE_RANGE, + .size = sizeof(uint16_t), + .field_index = DSTP_FIELD_IPV4, + .input_index = RTE_ACL_IPV4VLAN_PORTS, + .offset = offsetof(struct ipv4_5tuple, port_dst), + }, +}; + +#define IPV6_ADDR_LEN 16 +#define IPV6_ADDR_U16 (IPV6_ADDR_LEN / sizeof(uint16_t)) +#define IPV6_ADDR_U32 (IPV6_ADDR_LEN / sizeof(uint32_t)) + +struct ipv6_5tuple { + uint8_t proto; + uint32_t ip_src[IPV6_ADDR_U32]; + uint32_t ip_dst[IPV6_ADDR_U32]; + uint16_t port_src; + uint16_t port_dst; +}; + +enum { + PROTO_FIELD_IPV6, + SRC1_FIELD_IPV6, + SRC2_FIELD_IPV6, + SRC3_FIELD_IPV6, + SRC4_FIELD_IPV6, + DST1_FIELD_IPV6, + DST2_FIELD_IPV6, + DST3_FIELD_IPV6, + DST4_FIELD_IPV6, + SRCP_FIELD_IPV6, + DSTP_FIELD_IPV6, + NUM_FIELDS_IPV6 +}; + +struct rte_acl_field_def ipv6_defs[NUM_FIELDS_IPV6] = { + { + .type = RTE_ACL_FIELD_TYPE_BITMASK, + .size = sizeof(uint8_t), + .field_index = PROTO_FIELD_IPV6, + .input_index = PROTO_FIELD_IPV6, + .offset = offsetof(struct ipv6_5tuple, proto), + }, + { + .type = RTE_ACL_FIELD_TYPE_MASK, + .size = sizeof(uint32_t), + .field_index = SRC1_FIELD_IPV6, + .input_index = SRC1_FIELD_IPV6, + .offset = offsetof(struct ipv6_5tuple, ip_src[0]), + }, + { + .type = RTE_ACL_FIELD_TYPE_MASK, + .size = sizeof(uint32_t), + .field_index = SRC2_FIELD_IPV6, + .input_index = SRC2_FIELD_IPV6, + .offset = offsetof(struct ipv6_5tuple, ip_src[1]), + }, + { + .type = RTE_ACL_FIELD_TYPE_MASK, + .size = sizeof(uint32_t), + .field_index = SRC3_FIELD_IPV6, + .input_index = SRC3_FIELD_IPV6, + .offset = offsetof(struct ipv6_5tuple, ip_src[2]), + }, + { + .type = RTE_ACL_FIELD_TYPE_MASK, + .size = sizeof(uint32_t), + .field_index = SRC4_FIELD_IPV6, + .input_index = SRC4_FIELD_IPV6, + .offset = offsetof(struct ipv6_5tuple, ip_src[3]), + }, + { + .type = RTE_ACL_FIELD_TYPE_MASK, + .size = sizeof(uint32_t), + .field_index = DST1_FIELD_IPV6, + .input_index = DST1_FIELD_IPV6, + .offset = offsetof(struct ipv6_5tuple, ip_dst[0]), + }, + { + .type = RTE_ACL_FIELD_TYPE_MASK, + .size = sizeof(uint32_t), + .field_index = DST2_FIELD_IPV6, + .input_index = DST2_FIELD_IPV6, + .offset = offsetof(struct ipv6_5tuple, ip_dst[1]), + }, + { + .type = RTE_ACL_FIELD_TYPE_MASK, + .size = sizeof(uint32_t), + .field_index = DST3_FIELD_IPV6, + .input_index = DST3_FIELD_IPV6, + .offset = offsetof(struct ipv6_5tuple, ip_dst[2]), + }, + { + .type = RTE_ACL_FIELD_TYPE_MASK, + .size = sizeof(uint32_t), + .field_index = DST4_FIELD_IPV6, + .input_index = DST4_FIELD_IPV6, + .offset = offsetof(struct ipv6_5tuple, ip_dst[3]), + }, + { + .type = RTE_ACL_FIELD_TYPE_RANGE, + .size = sizeof(uint16_t), + .field_index = SRCP_FIELD_IPV6, + .input_index = SRCP_FIELD_IPV6, + .offset = offsetof(struct ipv6_5tuple, port_src), + }, + { + .type = RTE_ACL_FIELD_TYPE_RANGE, + .size = sizeof(uint16_t), + .field_index = DSTP_FIELD_IPV6, + .input_index = SRCP_FIELD_IPV6, + .offset = offsetof(struct ipv6_5tuple, port_dst), + }, +}; + + +enum { + CB_FLD_SRC_ADDR, + CB_FLD_DST_ADDR, + CB_FLD_SRC_PORT_LOW, + CB_FLD_SRC_PORT_DLM, + CB_FLD_SRC_PORT_HIGH, + CB_FLD_DST_PORT_LOW, + CB_FLD_DST_PORT_DLM, + CB_FLD_DST_PORT_HIGH, + CB_FLD_PROTO, + CB_FLD_NUM, +}; + +enum { + CB_TRC_SRC_ADDR, + CB_TRC_DST_ADDR, + CB_TRC_SRC_PORT, + CB_TRC_DST_PORT, + CB_TRC_PROTO, + CB_TRC_NUM, +}; + +RTE_ACL_RULE_DEF(acl_rule, RTE_ACL_MAX_FIELDS); + +static const char cb_port_delim[] = ":"; + +static char line[LINE_MAX]; + +#define dump_verbose(lvl, fh, fmt, args...) do { \ + if ((lvl) <= (int32_t)config.verbose) \ + fprintf(fh, fmt, ##args); \ +} while (0) + + +/* + * Parse ClassBench input trace (test vectors and expected results) file. + * Expected format: + * <src_ipv4_addr> <space> <dst_ipv4_addr> <space> \ + * <src_port> <space> <dst_port> <space> <proto> + */ +static int +parse_cb_ipv4_trace(char *str, struct ipv4_5tuple *v) +{ + int i; + char *s, *sp, *in[CB_TRC_NUM]; + static const char *dlm = " \t\n"; + + s = str; + for (i = 0; i != RTE_DIM(in); i++) { + in[i] = strtok_r(s, dlm, &sp); + if (in[i] == NULL) + return -EINVAL; + s = NULL; + } + + GET_CB_FIELD(in[CB_TRC_SRC_ADDR], v->ip_src, 0, UINT32_MAX, 0); + GET_CB_FIELD(in[CB_TRC_DST_ADDR], v->ip_dst, 0, UINT32_MAX, 0); + GET_CB_FIELD(in[CB_TRC_SRC_PORT], v->port_src, 0, UINT16_MAX, 0); + GET_CB_FIELD(in[CB_TRC_DST_PORT], v->port_dst, 0, UINT16_MAX, 0); + GET_CB_FIELD(in[CB_TRC_PROTO], v->proto, 0, UINT8_MAX, 0); + + /* convert to network byte order. */ + v->ip_src = rte_cpu_to_be_32(v->ip_src); + v->ip_dst = rte_cpu_to_be_32(v->ip_dst); + v->port_src = rte_cpu_to_be_16(v->port_src); + v->port_dst = rte_cpu_to_be_16(v->port_dst); + + return 0; +} + +/* + * Parses IPV6 address, exepcts the following format: + * XXXX:XXXX:XXXX:XXXX:XXXX:XXXX:XXXX:XXXX (where X - is a hexedecimal digit). + */ +static int +parse_ipv6_addr(const char *in, const char **end, uint32_t v[IPV6_ADDR_U32], + char dlm) +{ + uint32_t addr[IPV6_ADDR_U16]; + + GET_CB_FIELD(in, addr[0], 16, UINT16_MAX, ':'); + GET_CB_FIELD(in, addr[1], 16, UINT16_MAX, ':'); + GET_CB_FIELD(in, addr[2], 16, UINT16_MAX, ':'); + GET_CB_FIELD(in, addr[3], 16, UINT16_MAX, ':'); + GET_CB_FIELD(in, addr[4], 16, UINT16_MAX, ':'); + GET_CB_FIELD(in, addr[5], 16, UINT16_MAX, ':'); + GET_CB_FIELD(in, addr[6], 16, UINT16_MAX, ':'); + GET_CB_FIELD(in, addr[7], 16, UINT16_MAX, dlm); + + *end = in; + + v[0] = (addr[0] << 16) + addr[1]; + v[1] = (addr[2] << 16) + addr[3]; + v[2] = (addr[4] << 16) + addr[5]; + v[3] = (addr[6] << 16) + addr[7]; + + return 0; +} + +static int +parse_cb_ipv6_addr_trace(const char *in, uint32_t v[IPV6_ADDR_U32]) +{ + int32_t rc; + const char *end; + + rc = parse_ipv6_addr(in, &end, v, 0); + if (rc != 0) + return rc; + + v[0] = rte_cpu_to_be_32(v[0]); + v[1] = rte_cpu_to_be_32(v[1]); + v[2] = rte_cpu_to_be_32(v[2]); + v[3] = rte_cpu_to_be_32(v[3]); + + return 0; +} + +/* + * Parse ClassBench input trace (test vectors and expected results) file. + * Expected format: + * <src_ipv6_addr> <space> <dst_ipv6_addr> <space> \ + * <src_port> <space> <dst_port> <space> <proto> + */ +static int +parse_cb_ipv6_trace(char *str, struct ipv6_5tuple *v) +{ + int32_t i, rc; + char *s, *sp, *in[CB_TRC_NUM]; + static const char *dlm = " \t\n"; + + s = str; + for (i = 0; i != RTE_DIM(in); i++) { + in[i] = strtok_r(s, dlm, &sp); + if (in[i] == NULL) + return -EINVAL; + s = NULL; + } + + /* get ip6 src address. */ + rc = parse_cb_ipv6_addr_trace(in[CB_TRC_SRC_ADDR], v->ip_src); + if (rc != 0) + return rc; + + /* get ip6 dst address. */ + rc = parse_cb_ipv6_addr_trace(in[CB_TRC_DST_ADDR], v->ip_dst); + if (rc != 0) + return rc; + + GET_CB_FIELD(in[CB_TRC_SRC_PORT], v->port_src, 0, UINT16_MAX, 0); + GET_CB_FIELD(in[CB_TRC_DST_PORT], v->port_dst, 0, UINT16_MAX, 0); + GET_CB_FIELD(in[CB_TRC_PROTO], v->proto, 0, UINT8_MAX, 0); + + /* convert to network byte order. */ + v->port_src = rte_cpu_to_be_16(v->port_src); + v->port_dst = rte_cpu_to_be_16(v->port_dst); + + return 0; +} + +static void +tracef_init(void) +{ + static const char name[] = APP_NAME; + FILE *f; + size_t sz; + uint32_t n; + struct ipv4_5tuple *v; + struct ipv6_5tuple *w; + + sz = config.nb_traces * (config.ipv6 ? sizeof(*w) : sizeof(*v)); + config.traces = rte_zmalloc_socket(name, sz, RTE_CACHE_LINE_SIZE, + SOCKET_ID_ANY); + if (config.traces == NULL) + rte_exit(EXIT_FAILURE, "Cannot allocate %zu bytes for " + "requested %u number of trace records\n", + sz, config.nb_traces); + + f = fopen(config.trace_file, "r"); + if (f == NULL) + rte_exit(-EINVAL, "failed to open file: %s\n", + config.trace_file); + + v = config.traces; + w = config.traces; + for (n = 0; n != config.nb_traces; n++) { + + if (fgets(line, sizeof(line), f) == NULL) + break; + + if (config.ipv6) { + if (parse_cb_ipv6_trace(line, w + n) != 0) + rte_exit(EXIT_FAILURE, + "%s: failed to parse ipv6 trace " + "record at line %u\n", + config.trace_file, n + 1); + } else { + if (parse_cb_ipv4_trace(line, v + n) != 0) + rte_exit(EXIT_FAILURE, + "%s: failed to parse ipv4 trace " + "record at line %u\n", + config.trace_file, n + 1); + } + } + + config.used_traces = n; + fclose(f); +} + +static int +parse_ipv6_net(const char *in, struct rte_acl_field field[4]) +{ + int32_t rc; + const char *mp; + uint32_t i, m, v[4]; + const uint32_t nbu32 = sizeof(uint32_t) * CHAR_BIT; + + /* get address. */ + rc = parse_ipv6_addr(in, &mp, v, '/'); + if (rc != 0) + return rc; + + /* get mask. */ + GET_CB_FIELD(mp, m, 0, CHAR_BIT * sizeof(v), 0); + + /* put all together. */ + for (i = 0; i != RTE_DIM(v); i++) { + if (m >= (i + 1) * nbu32) + field[i].mask_range.u32 = nbu32; + else + field[i].mask_range.u32 = m > (i * nbu32) ? + m - (i * 32) : 0; + + field[i].value.u32 = v[i]; + } + + return 0; +} + + +static int +parse_cb_ipv6_rule(char *str, struct acl_rule *v) +{ + int i, rc; + char *s, *sp, *in[CB_FLD_NUM]; + static const char *dlm = " \t\n"; + + /* + * Skip leading '@' + */ + if (strchr(str, '@') != str) + return -EINVAL; + + s = str + 1; + + for (i = 0; i != RTE_DIM(in); i++) { + in[i] = strtok_r(s, dlm, &sp); + if (in[i] == NULL) + return -EINVAL; + s = NULL; + } + + rc = parse_ipv6_net(in[CB_FLD_SRC_ADDR], v->field + SRC1_FIELD_IPV6); + if (rc != 0) { + RTE_LOG(ERR, TESTACL, + "failed to read source address/mask: %s\n", + in[CB_FLD_SRC_ADDR]); + return rc; + } + + rc = parse_ipv6_net(in[CB_FLD_DST_ADDR], v->field + DST1_FIELD_IPV6); + if (rc != 0) { + RTE_LOG(ERR, TESTACL, + "failed to read destination address/mask: %s\n", + in[CB_FLD_DST_ADDR]); + return rc; + } + + /* source port. */ + GET_CB_FIELD(in[CB_FLD_SRC_PORT_LOW], + v->field[SRCP_FIELD_IPV6].value.u16, + 0, UINT16_MAX, 0); + GET_CB_FIELD(in[CB_FLD_SRC_PORT_HIGH], + v->field[SRCP_FIELD_IPV6].mask_range.u16, + 0, UINT16_MAX, 0); + + if (strncmp(in[CB_FLD_SRC_PORT_DLM], cb_port_delim, + sizeof(cb_port_delim)) != 0) + return -EINVAL; + + /* destination port. */ + GET_CB_FIELD(in[CB_FLD_DST_PORT_LOW], + v->field[DSTP_FIELD_IPV6].value.u16, + 0, UINT16_MAX, 0); + GET_CB_FIELD(in[CB_FLD_DST_PORT_HIGH], + v->field[DSTP_FIELD_IPV6].mask_range.u16, + 0, UINT16_MAX, 0); + + if (strncmp(in[CB_FLD_DST_PORT_DLM], cb_port_delim, + sizeof(cb_port_delim)) != 0) + return -EINVAL; + + GET_CB_FIELD(in[CB_FLD_PROTO], v->field[PROTO_FIELD_IPV6].value.u8, + 0, UINT8_MAX, '/'); + GET_CB_FIELD(in[CB_FLD_PROTO], v->field[PROTO_FIELD_IPV6].mask_range.u8, + 0, UINT8_MAX, 0); + + return 0; +} + +static int +parse_ipv4_net(const char *in, uint32_t *addr, uint32_t *mask_len) +{ + uint8_t a, b, c, d, m; + + GET_CB_FIELD(in, a, 0, UINT8_MAX, '.'); + GET_CB_FIELD(in, b, 0, UINT8_MAX, '.'); + GET_CB_FIELD(in, c, 0, UINT8_MAX, '.'); + GET_CB_FIELD(in, d, 0, UINT8_MAX, '/'); + GET_CB_FIELD(in, m, 0, sizeof(uint32_t) * CHAR_BIT, 0); + + addr[0] = IPv4(a, b, c, d); + mask_len[0] = m; + + return 0; +} +/* + * Parse ClassBench rules file. + * Expected format: + * '@'<src_ipv4_addr>'/'<masklen> <space> \ + * <dst_ipv4_addr>'/'<masklen> <space> \ + * <src_port_low> <space> ":" <src_port_high> <space> \ + * <dst_port_low> <space> ":" <dst_port_high> <space> \ + * <proto>'/'<mask> + */ +static int +parse_cb_ipv4_rule(char *str, struct acl_rule *v) +{ + int i, rc; + char *s, *sp, *in[CB_FLD_NUM]; + static const char *dlm = " \t\n"; + + /* + * Skip leading '@' + */ + if (strchr(str, '@') != str) + return -EINVAL; + + s = str + 1; + + for (i = 0; i != RTE_DIM(in); i++) { + in[i] = strtok_r(s, dlm, &sp); + if (in[i] == NULL) + return -EINVAL; + s = NULL; + } + + rc = parse_ipv4_net(in[CB_FLD_SRC_ADDR], + &v->field[SRC_FIELD_IPV4].value.u32, + &v->field[SRC_FIELD_IPV4].mask_range.u32); + if (rc != 0) { + RTE_LOG(ERR, TESTACL, + "failed to read source address/mask: %s\n", + in[CB_FLD_SRC_ADDR]); + return rc; + } + + rc = parse_ipv4_net(in[CB_FLD_DST_ADDR], + &v->field[DST_FIELD_IPV4].value.u32, + &v->field[DST_FIELD_IPV4].mask_range.u32); + if (rc != 0) { + RTE_LOG(ERR, TESTACL, + "failed to read destination address/mask: %s\n", + in[CB_FLD_DST_ADDR]); + return rc; + } + + /* source port. */ + GET_CB_FIELD(in[CB_FLD_SRC_PORT_LOW], + v->field[SRCP_FIELD_IPV4].value.u16, + 0, UINT16_MAX, 0); + GET_CB_FIELD(in[CB_FLD_SRC_PORT_HIGH], + v->field[SRCP_FIELD_IPV4].mask_range.u16, + 0, UINT16_MAX, 0); + + if (strncmp(in[CB_FLD_SRC_PORT_DLM], cb_port_delim, + sizeof(cb_port_delim)) != 0) + return -EINVAL; + + /* destination port. */ + GET_CB_FIELD(in[CB_FLD_DST_PORT_LOW], + v->field[DSTP_FIELD_IPV4].value.u16, + 0, UINT16_MAX, 0); + GET_CB_FIELD(in[CB_FLD_DST_PORT_HIGH], + v->field[DSTP_FIELD_IPV4].mask_range.u16, + 0, UINT16_MAX, 0); + + if (strncmp(in[CB_FLD_DST_PORT_DLM], cb_port_delim, + sizeof(cb_port_delim)) != 0) + return -EINVAL; + + GET_CB_FIELD(in[CB_FLD_PROTO], v->field[PROTO_FIELD_IPV4].value.u8, + 0, UINT8_MAX, '/'); + GET_CB_FIELD(in[CB_FLD_PROTO], v->field[PROTO_FIELD_IPV4].mask_range.u8, + 0, UINT8_MAX, 0); + + return 0; +} + +typedef int (*parse_5tuple)(char *text, struct acl_rule *rule); + +static int +add_cb_rules(FILE *f, struct rte_acl_ctx *ctx) +{ + int rc; + uint32_t n; + struct acl_rule v; + parse_5tuple parser; + + memset(&v, 0, sizeof(v)); + parser = (config.ipv6 != 0) ? parse_cb_ipv6_rule : parse_cb_ipv4_rule; + + for (n = 1; fgets(line, sizeof(line), f) != NULL; n++) { + + rc = parser(line, &v); + if (rc != 0) { + RTE_LOG(ERR, TESTACL, "line %u: parse_cb_ipv4vlan_rule" + " failed, error code: %d (%s)\n", + n, rc, strerror(-rc)); + return rc; + } + + v.data.category_mask = RTE_LEN2MASK(RTE_ACL_MAX_CATEGORIES, + typeof(v.data.category_mask)); + v.data.priority = RTE_ACL_MAX_PRIORITY - n; + v.data.userdata = n; + + rc = rte_acl_add_rules(ctx, (struct rte_acl_rule *)&v, 1); + if (rc != 0) { + RTE_LOG(ERR, TESTACL, "line %u: failed to add rules " + "into ACL context, error code: %d (%s)\n", + n, rc, strerror(-rc)); + return rc; + } + } + + return 0; +} + +static void +acx_init(void) +{ + int ret; + FILE *f; + struct rte_acl_config cfg; + + memset(&cfg, 0, sizeof(cfg)); + + /* setup ACL build config. */ + if (config.ipv6) { + cfg.num_fields = RTE_DIM(ipv6_defs); + memcpy(&cfg.defs, ipv6_defs, sizeof(ipv6_defs)); + } else { + cfg.num_fields = RTE_DIM(ipv4_defs); + memcpy(&cfg.defs, ipv4_defs, sizeof(ipv4_defs)); + } + cfg.num_categories = config.bld_categories; + cfg.max_size = config.max_size; + + /* setup ACL creation parameters. */ + prm.rule_size = RTE_ACL_RULE_SZ(cfg.num_fields); + prm.max_rule_num = config.nb_rules; + + config.acx = rte_acl_create(&prm); + if (config.acx == NULL) + rte_exit(rte_errno, "failed to create ACL context\n"); + + /* set default classify method for this context. */ + if (config.alg.alg != RTE_ACL_CLASSIFY_DEFAULT) { + ret = rte_acl_set_ctx_classify(config.acx, config.alg.alg); + if (ret != 0) + rte_exit(ret, "failed to setup %s method " + "for ACL context\n", config.alg.name); + } + + /* add ACL rules. */ + f = fopen(config.rule_file, "r"); + if (f == NULL) + rte_exit(-EINVAL, "failed to open file %s\n", + config.rule_file); + + ret = add_cb_rules(f, config.acx); + if (ret != 0) + rte_exit(ret, "failed to add rules into ACL context\n"); + + fclose(f); + + /* perform build. */ + ret = rte_acl_build(config.acx, &cfg); + + dump_verbose(DUMP_NONE, stdout, + "rte_acl_build(%u) finished with %d\n", + config.bld_categories, ret); + + rte_acl_dump(config.acx); + + if (ret != 0) + rte_exit(ret, "failed to build search context\n"); +} + +static uint32_t +search_ip5tuples_once(uint32_t categories, uint32_t step, const char *alg) +{ + int ret; + uint32_t i, j, k, n, r; + const uint8_t *data[step], *v; + uint32_t results[step * categories]; + + v = config.traces; + for (i = 0; i != config.used_traces; i += n) { + + n = RTE_MIN(step, config.used_traces - i); + + for (j = 0; j != n; j++) { + data[j] = v; + v += config.trace_sz; + } + + ret = rte_acl_classify(config.acx, data, results, + n, categories); + + if (ret != 0) + rte_exit(ret, "classify for ipv%c_5tuples returns %d\n", + config.ipv6 ? '6' : '4', ret); + + for (r = 0, j = 0; j != n; j++) { + for (k = 0; k != categories; k++, r++) { + dump_verbose(DUMP_PKT, stdout, + "ipv%c_5tuple: %u, category: %u, " + "result: %u\n", + config.ipv6 ? '6' : '4', + i + j + 1, k, results[r] - 1); + } + + } + } + + dump_verbose(DUMP_SEARCH, stdout, + "%s(%u, %u, %s) returns %u\n", __func__, + categories, step, alg, i); + return i; +} + +static int +search_ip5tuples(__attribute__((unused)) void *arg) +{ + uint64_t pkt, start, tm; + uint32_t i, lcore; + + lcore = rte_lcore_id(); + start = rte_rdtsc(); + pkt = 0; + + for (i = 0; i != config.iter_num; i++) { + pkt += search_ip5tuples_once(config.run_categories, + config.trace_step, config.alg.name); + } + + tm = rte_rdtsc() - start; + dump_verbose(DUMP_NONE, stdout, + "%s @lcore %u: %" PRIu32 " iterations, %" PRIu64 " pkts, %" + PRIu32 " categories, %" PRIu64 " cycles, %#Lf cycles/pkt\n", + __func__, lcore, i, pkt, config.run_categories, + tm, (pkt == 0) ? 0 : (long double)tm / pkt); + + return 0; +} + +static unsigned long +get_ulong_opt(const char *opt, const char *name, size_t min, size_t max) +{ + unsigned long val; + char *end; + + errno = 0; + val = strtoul(opt, &end, 0); + if (errno != 0 || end[0] != 0 || val > max || val < min) + rte_exit(-EINVAL, "invalid value: \"%s\" for option: %s\n", + opt, name); + return val; +} + +static void +get_alg_opt(const char *opt, const char *name) +{ + uint32_t i; + + for (i = 0; i != RTE_DIM(acl_alg); i++) { + if (strcmp(opt, acl_alg[i].name) == 0) { + config.alg = acl_alg[i]; + return; + } + } + + rte_exit(-EINVAL, "invalid value: \"%s\" for option: %s\n", + opt, name); +} + +static void +print_usage(const char *prgname) +{ + uint32_t i, n, rc; + char buf[PATH_MAX]; + + n = 0; + buf[0] = 0; + + for (i = 0; i < RTE_DIM(acl_alg) - 1; i++) { + rc = snprintf(buf + n, sizeof(buf) - n, "%s|", + acl_alg[i].name); + if (rc > sizeof(buf) - n) + break; + n += rc; + } + + snprintf(buf + n, sizeof(buf) - n, "%s", acl_alg[i].name); + + fprintf(stdout, + PRINT_USAGE_START + "--" OPT_RULE_FILE "=<rules set file>\n" + "[--" OPT_TRACE_FILE "=<input traces file>]\n" + "[--" OPT_RULE_NUM + "=<maximum number of rules for ACL context>]\n" + "[--" OPT_TRACE_NUM + "=<number of traces to read binary file in>]\n" + "[--" OPT_TRACE_STEP + "=<number of traces to classify per one call>]\n" + "[--" OPT_BLD_CATEGORIES + "=<number of categories to build with>]\n" + "[--" OPT_RUN_CATEGORIES + "=<number of categories to run with> " + "should be either 1 or multiple of %zu, " + "but not greater then %u]\n" + "[--" OPT_MAX_SIZE + "=<size limit (in bytes) for runtime ACL strucutures> " + "leave 0 for default behaviour]\n" + "[--" OPT_ITER_NUM "=<number of iterations to perform>]\n" + "[--" OPT_VERBOSE "=<verbose level>]\n" + "[--" OPT_SEARCH_ALG "=%s]\n" + "[--" OPT_IPV6 "=<IPv6 rules and trace files>]\n", + prgname, RTE_ACL_RESULTS_MULTIPLIER, + (uint32_t)RTE_ACL_MAX_CATEGORIES, + buf); +} + +static void +dump_config(FILE *f) +{ + fprintf(f, "%s:\n", __func__); + fprintf(f, "%s:%s\n", OPT_RULE_FILE, config.rule_file); + fprintf(f, "%s:%s\n", OPT_TRACE_FILE, config.trace_file); + fprintf(f, "%s:%u\n", OPT_RULE_NUM, config.nb_rules); + fprintf(f, "%s:%u\n", OPT_TRACE_NUM, config.nb_traces); + fprintf(f, "%s:%u\n", OPT_TRACE_STEP, config.trace_step); + fprintf(f, "%s:%u\n", OPT_BLD_CATEGORIES, config.bld_categories); + fprintf(f, "%s:%u\n", OPT_RUN_CATEGORIES, config.run_categories); + fprintf(f, "%s:%zu\n", OPT_MAX_SIZE, config.max_size); + fprintf(f, "%s:%u\n", OPT_ITER_NUM, config.iter_num); + fprintf(f, "%s:%u\n", OPT_VERBOSE, config.verbose); + fprintf(f, "%s:%u(%s)\n", OPT_SEARCH_ALG, config.alg.alg, + config.alg.name); + fprintf(f, "%s:%u\n", OPT_IPV6, config.ipv6); +} + +static void +check_config(void) +{ + if (config.rule_file == NULL) { + print_usage(config.prgname); + rte_exit(-EINVAL, "mandatory option %s is not specified\n", + OPT_RULE_FILE); + } +} + + +static void +get_input_opts(int argc, char **argv) +{ + static struct option lgopts[] = { + {OPT_RULE_FILE, 1, 0, 0}, + {OPT_TRACE_FILE, 1, 0, 0}, + {OPT_TRACE_NUM, 1, 0, 0}, + {OPT_RULE_NUM, 1, 0, 0}, + {OPT_MAX_SIZE, 1, 0, 0}, + {OPT_TRACE_STEP, 1, 0, 0}, + {OPT_BLD_CATEGORIES, 1, 0, 0}, + {OPT_RUN_CATEGORIES, 1, 0, 0}, + {OPT_ITER_NUM, 1, 0, 0}, + {OPT_VERBOSE, 1, 0, 0}, + {OPT_SEARCH_ALG, 1, 0, 0}, + {OPT_IPV6, 0, 0, 0}, + {NULL, 0, 0, 0} + }; + + int opt, opt_idx; + + while ((opt = getopt_long(argc, argv, "", lgopts, &opt_idx)) != EOF) { + + if (opt != 0) { + print_usage(config.prgname); + rte_exit(-EINVAL, "unknown option: %c", opt); + } + + if (strcmp(lgopts[opt_idx].name, OPT_RULE_FILE) == 0) { + config.rule_file = optarg; + } else if (strcmp(lgopts[opt_idx].name, OPT_TRACE_FILE) == 0) { + config.trace_file = optarg; + } else if (strcmp(lgopts[opt_idx].name, OPT_RULE_NUM) == 0) { + config.nb_rules = get_ulong_opt(optarg, + lgopts[opt_idx].name, 1, RTE_ACL_MAX_INDEX + 1); + } else if (strcmp(lgopts[opt_idx].name, OPT_MAX_SIZE) == 0) { + config.max_size = get_ulong_opt(optarg, + lgopts[opt_idx].name, 0, SIZE_MAX); + } else if (strcmp(lgopts[opt_idx].name, OPT_TRACE_NUM) == 0) { + config.nb_traces = get_ulong_opt(optarg, + lgopts[opt_idx].name, 1, UINT32_MAX); + } else if (strcmp(lgopts[opt_idx].name, OPT_TRACE_STEP) == 0) { + config.trace_step = get_ulong_opt(optarg, + lgopts[opt_idx].name, 1, TRACE_STEP_MAX); + } else if (strcmp(lgopts[opt_idx].name, + OPT_BLD_CATEGORIES) == 0) { + config.bld_categories = get_ulong_opt(optarg, + lgopts[opt_idx].name, 1, + RTE_ACL_MAX_CATEGORIES); + } else if (strcmp(lgopts[opt_idx].name, + OPT_RUN_CATEGORIES) == 0) { + config.run_categories = get_ulong_opt(optarg, + lgopts[opt_idx].name, 1, + RTE_ACL_MAX_CATEGORIES); + } else if (strcmp(lgopts[opt_idx].name, OPT_ITER_NUM) == 0) { + config.iter_num = get_ulong_opt(optarg, + lgopts[opt_idx].name, 1, INT32_MAX); + } else if (strcmp(lgopts[opt_idx].name, OPT_VERBOSE) == 0) { + config.verbose = get_ulong_opt(optarg, + lgopts[opt_idx].name, DUMP_NONE, DUMP_MAX); + } else if (strcmp(lgopts[opt_idx].name, + OPT_SEARCH_ALG) == 0) { + get_alg_opt(optarg, lgopts[opt_idx].name); + } else if (strcmp(lgopts[opt_idx].name, OPT_IPV6) == 0) { + config.ipv6 = 1; + } + } + config.trace_sz = config.ipv6 ? sizeof(struct ipv6_5tuple) : + sizeof(struct ipv4_5tuple); + +} + +int +main(int argc, char **argv) +{ + int ret; + uint32_t lcore; + + ret = rte_eal_init(argc, argv); + if (ret < 0) + rte_panic("Cannot init EAL\n"); + + argc -= ret; + argv += ret; + + config.prgname = argv[0]; + + get_input_opts(argc, argv); + dump_config(stdout); + check_config(); + + acx_init(); + + if (config.trace_file != NULL) + tracef_init(); + + RTE_LCORE_FOREACH_SLAVE(lcore) + rte_eal_remote_launch(search_ip5tuples, NULL, lcore); + + search_ip5tuples(NULL); + + rte_eal_mp_wait_lcore(); + + rte_acl_free(config.acx); + return 0; +} diff --git a/dpdk/test/test-pipeline/Makefile b/dpdk/test/test-pipeline/Makefile new file mode 100644 index 00000000..520a319f --- /dev/null +++ b/dpdk/test/test-pipeline/Makefile @@ -0,0 +1,61 @@ +# BSD LICENSE +# +# Copyright(c) 2010-2015 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +include $(RTE_SDK)/mk/rte.vars.mk + +ifeq ($(CONFIG_RTE_LIBRTE_PIPELINE),y) + +# +# library name +# +APP = testpipeline + +CFLAGS += -O3 +CFLAGS += $(WERROR_FLAGS) + +# +# all source are stored in SRCS-y +# +SRCS-y := main.c +SRCS-y += config.c +SRCS-y += init.c +SRCS-y += runtime.c +SRCS-y += pipeline_stub.c +SRCS-y += pipeline_hash.c +SRCS-y += pipeline_lpm.c +SRCS-y += pipeline_lpm_ipv6.c + +# include ACL lib if available +SRCS-$(CONFIG_RTE_LIBRTE_ACL) += pipeline_acl.c + +include $(RTE_SDK)/mk/rte.app.mk + +endif diff --git a/dpdk/test/test-pipeline/config.c b/dpdk/test/test-pipeline/config.c new file mode 100644 index 00000000..741bd307 --- /dev/null +++ b/dpdk/test/test-pipeline/config.c @@ -0,0 +1,262 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <stdlib.h> +#include <stdint.h> +#include <inttypes.h> +#include <sys/types.h> +#include <string.h> +#include <sys/queue.h> +#include <stdarg.h> +#include <errno.h> +#include <getopt.h> + +#include <rte_common.h> +#include <rte_byteorder.h> +#include <rte_log.h> +#include <rte_memory.h> +#include <rte_memcpy.h> +#include <rte_eal.h> +#include <rte_per_lcore.h> +#include <rte_launch.h> +#include <rte_atomic.h> +#include <rte_cycles.h> +#include <rte_prefetch.h> +#include <rte_lcore.h> +#include <rte_branch_prediction.h> +#include <rte_interrupts.h> +#include <rte_pci.h> +#include <rte_random.h> +#include <rte_debug.h> +#include <rte_ether.h> +#include <rte_ethdev.h> +#include <rte_mempool.h> +#include <rte_mbuf.h> +#include <rte_ip.h> +#include <rte_tcp.h> +#include <rte_lpm.h> +#include <rte_lpm6.h> +#include <rte_string_fns.h> + +#include "main.h" + +struct app_params app; + +static const char usage[] = "\n"; + +void +app_print_usage(void) +{ + printf(usage); +} + +static int +app_parse_port_mask(const char *arg) +{ + char *end = NULL; + uint64_t port_mask; + uint32_t i; + + if (arg[0] == '\0') + return -1; + + port_mask = strtoul(arg, &end, 16); + if ((end == NULL) || (*end != '\0')) + return -2; + + if (port_mask == 0) + return -3; + + app.n_ports = 0; + for (i = 0; i < 64; i++) { + if ((port_mask & (1LLU << i)) == 0) + continue; + + if (app.n_ports >= APP_MAX_PORTS) + return -4; + + app.ports[app.n_ports] = i; + app.n_ports++; + } + + if (!rte_is_power_of_2(app.n_ports)) + return -5; + + return 0; +} + +struct { + const char *name; + uint32_t value; +} app_args_table[] = { + {"none", e_APP_PIPELINE_NONE}, + {"stub", e_APP_PIPELINE_STUB}, + {"hash-8-ext", e_APP_PIPELINE_HASH_KEY8_EXT}, + {"hash-8-lru", e_APP_PIPELINE_HASH_KEY8_LRU}, + {"hash-16-ext", e_APP_PIPELINE_HASH_KEY16_EXT}, + {"hash-16-lru", e_APP_PIPELINE_HASH_KEY16_LRU}, + {"hash-32-ext", e_APP_PIPELINE_HASH_KEY32_EXT}, + {"hash-32-lru", e_APP_PIPELINE_HASH_KEY32_LRU}, + {"hash-spec-8-ext", e_APP_PIPELINE_HASH_SPEC_KEY8_EXT}, + {"hash-spec-8-lru", e_APP_PIPELINE_HASH_SPEC_KEY8_LRU}, + {"hash-spec-16-ext", e_APP_PIPELINE_HASH_SPEC_KEY16_EXT}, + {"hash-spec-16-lru", e_APP_PIPELINE_HASH_SPEC_KEY16_LRU}, + {"hash-spec-32-ext", e_APP_PIPELINE_HASH_SPEC_KEY32_EXT}, + {"hash-spec-32-lru", e_APP_PIPELINE_HASH_SPEC_KEY32_LRU}, + {"acl", e_APP_PIPELINE_ACL}, + {"lpm", e_APP_PIPELINE_LPM}, + {"lpm-ipv6", e_APP_PIPELINE_LPM_IPV6}, + {"hash-cuckoo-8", e_APP_PIPELINE_HASH_CUCKOO_KEY8}, + {"hash-cuckoo-16", e_APP_PIPELINE_HASH_CUCKOO_KEY16}, + {"hash-cuckoo-32", e_APP_PIPELINE_HASH_CUCKOO_KEY32}, + {"hash-cuckoo-48", e_APP_PIPELINE_HASH_CUCKOO_KEY48}, + {"hash-cuckoo-64", e_APP_PIPELINE_HASH_CUCKOO_KEY64}, + {"hash-cuckoo-80", e_APP_PIPELINE_HASH_CUCKOO_KEY80}, + {"hash-cuckoo-96", e_APP_PIPELINE_HASH_CUCKOO_KEY96}, + {"hash-cuckoo-112", e_APP_PIPELINE_HASH_CUCKOO_KEY112}, + {"hash-cuckoo-128", e_APP_PIPELINE_HASH_CUCKOO_KEY128}, +}; + +int +app_parse_args(int argc, char **argv) +{ + int opt, ret; + char **argvopt; + int option_index; + char *prgname = argv[0]; + static struct option lgopts[] = { + {"none", 0, 0, 0}, + {"stub", 0, 0, 0}, + {"hash-8-ext", 0, 0, 0}, + {"hash-8-lru", 0, 0, 0}, + {"hash-16-ext", 0, 0, 0}, + {"hash-16-lru", 0, 0, 0}, + {"hash-32-ext", 0, 0, 0}, + {"hash-32-lru", 0, 0, 0}, + {"hash-spec-8-ext", 0, 0, 0}, + {"hash-spec-8-lru", 0, 0, 0}, + {"hash-spec-16-ext", 0, 0, 0}, + {"hash-spec-16-lru", 0, 0, 0}, + {"hash-spec-32-ext", 0, 0, 0}, + {"hash-spec-32-lru", 0, 0, 0}, + {"acl", 0, 0, 0}, + {"lpm", 0, 0, 0}, + {"lpm-ipv6", 0, 0, 0}, + {"hash-cuckoo-8", 0, 0, 0}, + {"hash-cuckoo-16", 0, 0, 0}, + {"hash-cuckoo-32", 0, 0, 0}, + {"hash-cuckoo-48", 0, 0, 0}, + {"hash-cuckoo-64", 0, 0, 0}, + {"hash-cuckoo-80", 0, 0, 0}, + {"hash-cuckoo-96", 0, 0, 0}, + {"hash-cuckoo-112", 0, 0, 0}, + {"hash-cuckoo-128", 0, 0, 0}, + {NULL, 0, 0, 0} + }; + uint32_t lcores[3], n_lcores, lcore_id, pipeline_type_provided; + + /* EAL args */ + n_lcores = 0; + for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { + if (rte_lcore_is_enabled(lcore_id) == 0) + continue; + + if (n_lcores >= 3) { + RTE_LOG(ERR, USER1, "Number of cores must be 3\n"); + app_print_usage(); + return -1; + } + + lcores[n_lcores] = lcore_id; + n_lcores++; + } + + if (n_lcores != 3) { + RTE_LOG(ERR, USER1, "Number of cores must be 3\n"); + app_print_usage(); + return -1; + } + + app.core_rx = lcores[0]; + app.core_worker = lcores[1]; + app.core_tx = lcores[2]; + + /* Non-EAL args */ + argvopt = argv; + + app.pipeline_type = e_APP_PIPELINE_HASH_KEY16_LRU; + pipeline_type_provided = 0; + + while ((opt = getopt_long(argc, argvopt, "p:", + lgopts, &option_index)) != EOF) { + switch (opt) { + case 'p': + if (app_parse_port_mask(optarg) < 0) { + app_print_usage(); + return -1; + } + break; + + case 0: /* long options */ + if (!pipeline_type_provided) { + uint32_t i; + + for (i = 0; i < e_APP_PIPELINES; i++) { + if (!strcmp(lgopts[option_index].name, + app_args_table[i].name)) { + app.pipeline_type = + app_args_table[i].value; + pipeline_type_provided = 1; + break; + } + } + + break; + } + + app_print_usage(); + return -1; + + default: + return -1; + } + } + + if (optind >= 0) + argv[optind - 1] = prgname; + + ret = optind - 1; + optind = 1; /* reset getopt lib */ + return ret; +} diff --git a/dpdk/test/test-pipeline/init.c b/dpdk/test/test-pipeline/init.c new file mode 100644 index 00000000..79536609 --- /dev/null +++ b/dpdk/test/test-pipeline/init.c @@ -0,0 +1,278 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <stdlib.h> +#include <stdint.h> +#include <inttypes.h> +#include <sys/types.h> +#include <string.h> +#include <sys/queue.h> +#include <stdarg.h> +#include <errno.h> +#include <getopt.h> + +#include <rte_common.h> +#include <rte_byteorder.h> +#include <rte_log.h> +#include <rte_memory.h> +#include <rte_memcpy.h> +#include <rte_eal.h> +#include <rte_per_lcore.h> +#include <rte_launch.h> +#include <rte_atomic.h> +#include <rte_cycles.h> +#include <rte_prefetch.h> +#include <rte_lcore.h> +#include <rte_branch_prediction.h> +#include <rte_interrupts.h> +#include <rte_pci.h> +#include <rte_random.h> +#include <rte_debug.h> +#include <rte_ether.h> +#include <rte_ethdev.h> +#include <rte_ring.h> +#include <rte_mempool.h> +#include <rte_mbuf.h> +#include <rte_string_fns.h> +#include <rte_ip.h> +#include <rte_tcp.h> +#include <rte_lpm.h> +#include <rte_lpm6.h> + +#include "main.h" + +struct app_params app = { + /* Ports*/ + .n_ports = APP_MAX_PORTS, + .port_rx_ring_size = 128, + .port_tx_ring_size = 512, + + /* Rings */ + .ring_rx_size = 128, + .ring_tx_size = 128, + + /* Buffer pool */ + .pool_buffer_size = 2048 + RTE_PKTMBUF_HEADROOM, + .pool_size = 32 * 1024, + .pool_cache_size = 256, + + /* Burst sizes */ + .burst_size_rx_read = 64, + .burst_size_rx_write = 64, + .burst_size_worker_read = 64, + .burst_size_worker_write = 64, + .burst_size_tx_read = 64, + .burst_size_tx_write = 64, +}; + +static struct rte_eth_conf port_conf = { + .rxmode = { + .split_hdr_size = 0, + .header_split = 0, /* Header Split disabled */ + .hw_ip_checksum = 1, /* IP checksum offload enabled */ + .hw_vlan_filter = 0, /* VLAN filtering disabled */ + .jumbo_frame = 0, /* Jumbo Frame Support disabled */ + .hw_strip_crc = 1, /* CRC stripped by hardware */ + }, + .rx_adv_conf = { + .rss_conf = { + .rss_key = NULL, + .rss_hf = ETH_RSS_IP, + }, + }, + .txmode = { + .mq_mode = ETH_MQ_TX_NONE, + }, +}; + +static struct rte_eth_rxconf rx_conf = { + .rx_thresh = { + .pthresh = 8, + .hthresh = 8, + .wthresh = 4, + }, + .rx_free_thresh = 64, + .rx_drop_en = 0, +}; + +static struct rte_eth_txconf tx_conf = { + .tx_thresh = { + .pthresh = 36, + .hthresh = 0, + .wthresh = 0, + }, + .tx_free_thresh = 0, + .tx_rs_thresh = 0, +}; + +static void +app_init_mbuf_pools(void) +{ + /* Init the buffer pool */ + RTE_LOG(INFO, USER1, "Creating the mbuf pool ...\n"); + app.pool = rte_pktmbuf_pool_create("mempool", app.pool_size, + app.pool_cache_size, 0, app.pool_buffer_size, rte_socket_id()); + if (app.pool == NULL) + rte_panic("Cannot create mbuf pool\n"); +} + +static void +app_init_rings(void) +{ + uint32_t i; + + for (i = 0; i < app.n_ports; i++) { + char name[32]; + + snprintf(name, sizeof(name), "app_ring_rx_%u", i); + + app.rings_rx[i] = rte_ring_create( + name, + app.ring_rx_size, + rte_socket_id(), + RING_F_SP_ENQ | RING_F_SC_DEQ); + + if (app.rings_rx[i] == NULL) + rte_panic("Cannot create RX ring %u\n", i); + } + + for (i = 0; i < app.n_ports; i++) { + char name[32]; + + snprintf(name, sizeof(name), "app_ring_tx_%u", i); + + app.rings_tx[i] = rte_ring_create( + name, + app.ring_tx_size, + rte_socket_id(), + RING_F_SP_ENQ | RING_F_SC_DEQ); + + if (app.rings_tx[i] == NULL) + rte_panic("Cannot create TX ring %u\n", i); + } + +} + +static void +app_ports_check_link(void) +{ + uint32_t all_ports_up, i; + + all_ports_up = 1; + + for (i = 0; i < app.n_ports; i++) { + struct rte_eth_link link; + uint16_t port; + + port = app.ports[i]; + memset(&link, 0, sizeof(link)); + rte_eth_link_get_nowait(port, &link); + RTE_LOG(INFO, USER1, "Port %u (%u Gbps) %s\n", + port, + link.link_speed / 1000, + link.link_status ? "UP" : "DOWN"); + + if (link.link_status == ETH_LINK_DOWN) + all_ports_up = 0; + } + + if (all_ports_up == 0) + rte_panic("Some NIC ports are DOWN\n"); +} + +static void +app_init_ports(void) +{ + uint32_t i; + + /* Init NIC ports, then start the ports */ + for (i = 0; i < app.n_ports; i++) { + uint16_t port; + int ret; + + port = app.ports[i]; + RTE_LOG(INFO, USER1, "Initializing NIC port %u ...\n", port); + + /* Init port */ + ret = rte_eth_dev_configure( + port, + 1, + 1, + &port_conf); + if (ret < 0) + rte_panic("Cannot init NIC port %u (%d)\n", port, ret); + + rte_eth_promiscuous_enable(port); + + /* Init RX queues */ + ret = rte_eth_rx_queue_setup( + port, + 0, + app.port_rx_ring_size, + rte_eth_dev_socket_id(port), + &rx_conf, + app.pool); + if (ret < 0) + rte_panic("Cannot init RX for port %u (%d)\n", + (uint32_t) port, ret); + + /* Init TX queues */ + ret = rte_eth_tx_queue_setup( + port, + 0, + app.port_tx_ring_size, + rte_eth_dev_socket_id(port), + &tx_conf); + if (ret < 0) + rte_panic("Cannot init TX for port %u (%d)\n", + (uint32_t) port, ret); + + /* Start port */ + ret = rte_eth_dev_start(port); + if (ret < 0) + rte_panic("Cannot start port %u (%d)\n", port, ret); + } + + app_ports_check_link(); +} + +void +app_init(void) +{ + app_init_mbuf_pools(); + app_init_rings(); + app_init_ports(); + + RTE_LOG(INFO, USER1, "Initialization completed\n"); +} diff --git a/dpdk/test/test-pipeline/main.c b/dpdk/test/test-pipeline/main.c new file mode 100644 index 00000000..a710173b --- /dev/null +++ b/dpdk/test/test-pipeline/main.c @@ -0,0 +1,186 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <stdlib.h> +#include <stdint.h> +#include <inttypes.h> +#include <sys/types.h> +#include <string.h> +#include <sys/queue.h> +#include <stdarg.h> +#include <errno.h> +#include <getopt.h> +#include <unistd.h> + +#include <rte_common.h> +#include <rte_byteorder.h> +#include <rte_log.h> +#include <rte_memory.h> +#include <rte_memcpy.h> +#include <rte_eal.h> +#include <rte_per_lcore.h> +#include <rte_launch.h> +#include <rte_atomic.h> +#include <rte_cycles.h> +#include <rte_prefetch.h> +#include <rte_lcore.h> +#include <rte_branch_prediction.h> +#include <rte_interrupts.h> +#include <rte_pci.h> +#include <rte_random.h> +#include <rte_debug.h> +#include <rte_ether.h> +#include <rte_ethdev.h> +#include <rte_mempool.h> +#include <rte_mbuf.h> +#include <rte_ip.h> +#include <rte_tcp.h> +#include <rte_lpm.h> +#include <rte_lpm6.h> + +#include "main.h" + +int +main(int argc, char **argv) +{ + uint32_t lcore; + int ret; + + /* Init EAL */ + ret = rte_eal_init(argc, argv); + if (ret < 0) + return -1; + argc -= ret; + argv += ret; + + /* Parse application arguments (after the EAL ones) */ + ret = app_parse_args(argc, argv); + if (ret < 0) { + app_print_usage(); + return -1; + } + + /* Init */ + app_init(); + + /* Launch per-lcore init on every lcore */ + rte_eal_mp_remote_launch(app_lcore_main_loop, NULL, CALL_MASTER); + RTE_LCORE_FOREACH_SLAVE(lcore) { + if (rte_eal_wait_lcore(lcore) < 0) + return -1; + } + + return 0; +} + +int +app_lcore_main_loop(__attribute__((unused)) void *arg) +{ + unsigned lcore; + + lcore = rte_lcore_id(); + + if (lcore == app.core_rx) { + switch (app.pipeline_type) { + case e_APP_PIPELINE_ACL: + app_main_loop_rx(); + return 0; + + default: + app_main_loop_rx_metadata(); + return 0; + } + } + + if (lcore == app.core_worker) { + switch (app.pipeline_type) { + case e_APP_PIPELINE_STUB: + app_main_loop_worker_pipeline_stub(); + return 0; + + case e_APP_PIPELINE_HASH_KEY8_EXT: + case e_APP_PIPELINE_HASH_KEY8_LRU: + case e_APP_PIPELINE_HASH_KEY16_EXT: + case e_APP_PIPELINE_HASH_KEY16_LRU: + case e_APP_PIPELINE_HASH_KEY32_EXT: + case e_APP_PIPELINE_HASH_KEY32_LRU: + case e_APP_PIPELINE_HASH_SPEC_KEY8_EXT: + case e_APP_PIPELINE_HASH_SPEC_KEY8_LRU: + case e_APP_PIPELINE_HASH_SPEC_KEY16_EXT: + case e_APP_PIPELINE_HASH_SPEC_KEY16_LRU: + case e_APP_PIPELINE_HASH_SPEC_KEY32_EXT: + case e_APP_PIPELINE_HASH_SPEC_KEY32_LRU: + /* cases for cuckoo hash table types */ + case e_APP_PIPELINE_HASH_CUCKOO_KEY8: + case e_APP_PIPELINE_HASH_CUCKOO_KEY16: + case e_APP_PIPELINE_HASH_CUCKOO_KEY32: + case e_APP_PIPELINE_HASH_CUCKOO_KEY48: + case e_APP_PIPELINE_HASH_CUCKOO_KEY64: + case e_APP_PIPELINE_HASH_CUCKOO_KEY80: + case e_APP_PIPELINE_HASH_CUCKOO_KEY96: + case e_APP_PIPELINE_HASH_CUCKOO_KEY112: + case e_APP_PIPELINE_HASH_CUCKOO_KEY128: + app_main_loop_worker_pipeline_hash(); + return 0; + + case e_APP_PIPELINE_ACL: +#ifndef RTE_LIBRTE_ACL + rte_exit(EXIT_FAILURE, "ACL not present in build\n"); +#else + app_main_loop_worker_pipeline_acl(); + return 0; +#endif + + case e_APP_PIPELINE_LPM: + app_main_loop_worker_pipeline_lpm(); + return 0; + + case e_APP_PIPELINE_LPM_IPV6: + app_main_loop_worker_pipeline_lpm_ipv6(); + return 0; + + case e_APP_PIPELINE_NONE: + default: + app_main_loop_worker(); + return 0; + } + } + + if (lcore == app.core_tx) { + app_main_loop_tx(); + return 0; + } + + return 0; +} diff --git a/dpdk/test/test-pipeline/main.h b/dpdk/test/test-pipeline/main.h new file mode 100644 index 00000000..26395a35 --- /dev/null +++ b/dpdk/test/test-pipeline/main.h @@ -0,0 +1,155 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _MAIN_H_ +#define _MAIN_H_ + +#ifndef APP_MBUF_ARRAY_SIZE +#define APP_MBUF_ARRAY_SIZE 256 +#endif + +struct app_mbuf_array { + struct rte_mbuf *array[APP_MBUF_ARRAY_SIZE]; + uint16_t n_mbufs; +}; + +#ifndef APP_MAX_PORTS +#define APP_MAX_PORTS 4 +#endif + +struct app_params { + /* CPU cores */ + uint32_t core_rx; + uint32_t core_worker; + uint32_t core_tx; + + /* Ports*/ + uint32_t ports[APP_MAX_PORTS]; + uint32_t n_ports; + uint32_t port_rx_ring_size; + uint32_t port_tx_ring_size; + + /* Rings */ + struct rte_ring *rings_rx[APP_MAX_PORTS]; + struct rte_ring *rings_tx[APP_MAX_PORTS]; + uint32_t ring_rx_size; + uint32_t ring_tx_size; + + /* Internal buffers */ + struct app_mbuf_array mbuf_rx; + struct app_mbuf_array mbuf_tx[APP_MAX_PORTS]; + + /* Buffer pool */ + struct rte_mempool *pool; + uint32_t pool_buffer_size; + uint32_t pool_size; + uint32_t pool_cache_size; + + /* Burst sizes */ + uint32_t burst_size_rx_read; + uint32_t burst_size_rx_write; + uint32_t burst_size_worker_read; + uint32_t burst_size_worker_write; + uint32_t burst_size_tx_read; + uint32_t burst_size_tx_write; + + /* App behavior */ + uint32_t pipeline_type; +} __rte_cache_aligned; + +extern struct app_params app; + +int app_parse_args(int argc, char **argv); +void app_print_usage(void); +void app_init(void); +int app_lcore_main_loop(void *arg); + +/* Pipeline */ +enum { + e_APP_PIPELINE_NONE = 0, + e_APP_PIPELINE_STUB, + + e_APP_PIPELINE_HASH_KEY8_EXT, + e_APP_PIPELINE_HASH_KEY8_LRU, + e_APP_PIPELINE_HASH_KEY16_EXT, + e_APP_PIPELINE_HASH_KEY16_LRU, + e_APP_PIPELINE_HASH_KEY32_EXT, + e_APP_PIPELINE_HASH_KEY32_LRU, + + e_APP_PIPELINE_HASH_SPEC_KEY8_EXT, + e_APP_PIPELINE_HASH_SPEC_KEY8_LRU, + e_APP_PIPELINE_HASH_SPEC_KEY16_EXT, + e_APP_PIPELINE_HASH_SPEC_KEY16_LRU, + e_APP_PIPELINE_HASH_SPEC_KEY32_EXT, + e_APP_PIPELINE_HASH_SPEC_KEY32_LRU, + + e_APP_PIPELINE_ACL, + e_APP_PIPELINE_LPM, + e_APP_PIPELINE_LPM_IPV6, + + e_APP_PIPELINE_HASH_CUCKOO_KEY8, + e_APP_PIPELINE_HASH_CUCKOO_KEY16, + e_APP_PIPELINE_HASH_CUCKOO_KEY32, + e_APP_PIPELINE_HASH_CUCKOO_KEY48, + e_APP_PIPELINE_HASH_CUCKOO_KEY64, + e_APP_PIPELINE_HASH_CUCKOO_KEY80, + e_APP_PIPELINE_HASH_CUCKOO_KEY96, + e_APP_PIPELINE_HASH_CUCKOO_KEY112, + e_APP_PIPELINE_HASH_CUCKOO_KEY128, + e_APP_PIPELINES +}; + +void app_main_loop_rx(void); +void app_main_loop_rx_metadata(void); +uint64_t test_hash(void *key, + void *key_mask, + uint32_t key_size, + uint64_t seed); + +void app_main_loop_worker(void); +void app_main_loop_worker_pipeline_stub(void); +void app_main_loop_worker_pipeline_hash(void); +void app_main_loop_worker_pipeline_acl(void); +void app_main_loop_worker_pipeline_lpm(void); +void app_main_loop_worker_pipeline_lpm_ipv6(void); + +void app_main_loop_tx(void); + +#define APP_FLUSH 0 +#ifndef APP_FLUSH +#define APP_FLUSH 0x3FF +#endif + +#define APP_METADATA_OFFSET(offset) (sizeof(struct rte_mbuf) + (offset)) + +#endif /* _MAIN_H_ */ diff --git a/dpdk/app/test-pipeline/pipeline_acl.c b/dpdk/test/test-pipeline/pipeline_acl.c similarity index 100% rename from dpdk/app/test-pipeline/pipeline_acl.c rename to dpdk/test/test-pipeline/pipeline_acl.c diff --git a/dpdk/test/test-pipeline/pipeline_hash.c b/dpdk/test/test-pipeline/pipeline_hash.c new file mode 100644 index 00000000..edc1663c --- /dev/null +++ b/dpdk/test/test-pipeline/pipeline_hash.c @@ -0,0 +1,474 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <stdlib.h> +#include <stdint.h> + +#include <rte_log.h> +#include <rte_ethdev.h> +#include <rte_ether.h> +#include <rte_ip.h> +#include <rte_byteorder.h> + +#include <rte_port_ring.h> +#include <rte_table_hash.h> +#include <rte_hash.h> +#include <rte_pipeline.h> + +#include "main.h" + +static void +translate_options(uint32_t *special, uint32_t *ext, uint32_t *key_size) +{ + switch (app.pipeline_type) { + case e_APP_PIPELINE_HASH_KEY8_EXT: + *special = 0; *ext = 1; *key_size = 8; return; + case e_APP_PIPELINE_HASH_KEY8_LRU: + *special = 0; *ext = 0; *key_size = 8; return; + case e_APP_PIPELINE_HASH_KEY16_EXT: + *special = 0; *ext = 1; *key_size = 16; return; + case e_APP_PIPELINE_HASH_KEY16_LRU: + *special = 0; *ext = 0; *key_size = 16; return; + case e_APP_PIPELINE_HASH_KEY32_EXT: + *special = 0; *ext = 1; *key_size = 32; return; + case e_APP_PIPELINE_HASH_KEY32_LRU: + *special = 0; *ext = 0; *key_size = 32; return; + + case e_APP_PIPELINE_HASH_SPEC_KEY8_EXT: + *special = 1; *ext = 1; *key_size = 8; return; + case e_APP_PIPELINE_HASH_SPEC_KEY8_LRU: + *special = 1; *ext = 0; *key_size = 8; return; + case e_APP_PIPELINE_HASH_SPEC_KEY16_EXT: + *special = 1; *ext = 1; *key_size = 16; return; + case e_APP_PIPELINE_HASH_SPEC_KEY16_LRU: + *special = 1; *ext = 0; *key_size = 16; return; + case e_APP_PIPELINE_HASH_SPEC_KEY32_EXT: + *special = 1; *ext = 1; *key_size = 32; return; + case e_APP_PIPELINE_HASH_SPEC_KEY32_LRU: + *special = 1; *ext = 0; *key_size = 32; return; + + case e_APP_PIPELINE_HASH_CUCKOO_KEY8: + *special = 0; *ext = 0; *key_size = 8; return; + case e_APP_PIPELINE_HASH_CUCKOO_KEY16: + *special = 0; *ext = 0; *key_size = 16; return; + case e_APP_PIPELINE_HASH_CUCKOO_KEY32: + *special = 0; *ext = 0; *key_size = 32; return; + case e_APP_PIPELINE_HASH_CUCKOO_KEY48: + *special = 0; *ext = 0; *key_size = 48; return; + case e_APP_PIPELINE_HASH_CUCKOO_KEY64: + *special = 0; *ext = 0; *key_size = 64; return; + case e_APP_PIPELINE_HASH_CUCKOO_KEY80: + *special = 0; *ext = 0; *key_size = 80; return; + case e_APP_PIPELINE_HASH_CUCKOO_KEY96: + *special = 0; *ext = 0; *key_size = 96; return; + case e_APP_PIPELINE_HASH_CUCKOO_KEY112: + *special = 0; *ext = 0; *key_size = 112; return; + case e_APP_PIPELINE_HASH_CUCKOO_KEY128: + *special = 0; *ext = 0; *key_size = 128; return; + + default: + rte_panic("Invalid hash table type or key size\n"); + } +} +void +app_main_loop_worker_pipeline_hash(void) { + struct rte_pipeline_params pipeline_params = { + .name = "pipeline", + .socket_id = rte_socket_id(), + }; + + struct rte_pipeline *p; + uint32_t port_in_id[APP_MAX_PORTS]; + uint32_t port_out_id[APP_MAX_PORTS]; + uint32_t table_id; + uint32_t i; + uint32_t special, ext, key_size; + + translate_options(&special, &ext, &key_size); + + RTE_LOG(INFO, USER1, "Core %u is doing work " + "(pipeline with hash table, %s, %s, %d-byte key)\n", + rte_lcore_id(), + special ? "specialized" : "non-specialized", + ext ? "extendible bucket" : "LRU", + key_size); + + /* Pipeline configuration */ + p = rte_pipeline_create(&pipeline_params); + if (p == NULL) + rte_panic("Unable to configure the pipeline\n"); + + /* Input port configuration */ + for (i = 0; i < app.n_ports; i++) { + struct rte_port_ring_reader_params port_ring_params = { + .ring = app.rings_rx[i], + }; + + struct rte_pipeline_port_in_params port_params = { + .ops = &rte_port_ring_reader_ops, + .arg_create = (void *) &port_ring_params, + .f_action = NULL, + .arg_ah = NULL, + .burst_size = app.burst_size_worker_read, + }; + + if (rte_pipeline_port_in_create(p, &port_params, + &port_in_id[i])) + rte_panic("Unable to configure input port for " + "ring %d\n", i); + } + + /* Output port configuration */ + for (i = 0; i < app.n_ports; i++) { + struct rte_port_ring_writer_params port_ring_params = { + .ring = app.rings_tx[i], + .tx_burst_sz = app.burst_size_worker_write, + }; + + struct rte_pipeline_port_out_params port_params = { + .ops = &rte_port_ring_writer_ops, + .arg_create = (void *) &port_ring_params, + .f_action = NULL, + .arg_ah = NULL, + }; + + if (rte_pipeline_port_out_create(p, &port_params, + &port_out_id[i])) + rte_panic("Unable to configure output port for " + "ring %d\n", i); + } + + struct rte_table_hash_params table_hash_params = { + .name = "TABLE", + .key_size = key_size, + .key_offset = APP_METADATA_OFFSET(32), + .key_mask = NULL, + .n_keys = 1 << 24, + .n_buckets = 1 << 22, + .f_hash = test_hash, + .seed = 0, + }; + + /* Table configuration */ + switch (app.pipeline_type) { + case e_APP_PIPELINE_HASH_KEY8_EXT: + case e_APP_PIPELINE_HASH_KEY16_EXT: + case e_APP_PIPELINE_HASH_KEY32_EXT: + { + struct rte_pipeline_table_params table_params = { + .ops = &rte_table_hash_ext_ops, + .arg_create = &table_hash_params, + .f_action_hit = NULL, + .f_action_miss = NULL, + .arg_ah = NULL, + .action_data_size = 0, + }; + + if (rte_pipeline_table_create(p, &table_params, &table_id)) + rte_panic("Unable to configure the hash table\n"); + } + break; + + case e_APP_PIPELINE_HASH_KEY8_LRU: + case e_APP_PIPELINE_HASH_KEY16_LRU: + case e_APP_PIPELINE_HASH_KEY32_LRU: + { + struct rte_pipeline_table_params table_params = { + .ops = &rte_table_hash_lru_ops, + .arg_create = &table_hash_params, + .f_action_hit = NULL, + .f_action_miss = NULL, + .arg_ah = NULL, + .action_data_size = 0, + }; + + if (rte_pipeline_table_create(p, &table_params, &table_id)) + rte_panic("Unable to configure the hash table\n"); + } + break; + + case e_APP_PIPELINE_HASH_SPEC_KEY8_EXT: + { + struct rte_pipeline_table_params table_params = { + .ops = &rte_table_hash_key8_ext_ops, + .arg_create = &table_hash_params, + .f_action_hit = NULL, + .f_action_miss = NULL, + .arg_ah = NULL, + .action_data_size = 0, + }; + + if (rte_pipeline_table_create(p, &table_params, &table_id)) + rte_panic("Unable to configure the hash table\n"); + } + break; + + case e_APP_PIPELINE_HASH_SPEC_KEY8_LRU: + { + struct rte_pipeline_table_params table_params = { + .ops = &rte_table_hash_key8_lru_ops, + .arg_create = &table_hash_params, + .f_action_hit = NULL, + .f_action_miss = NULL, + .arg_ah = NULL, + .action_data_size = 0, + }; + + if (rte_pipeline_table_create(p, &table_params, &table_id)) + rte_panic("Unable to configure the hash table\n"); + } + break; + + case e_APP_PIPELINE_HASH_SPEC_KEY16_EXT: + { + struct rte_pipeline_table_params table_params = { + .ops = &rte_table_hash_key16_ext_ops, + .arg_create = &table_hash_params, + .f_action_hit = NULL, + .f_action_miss = NULL, + .arg_ah = NULL, + .action_data_size = 0, + }; + + if (rte_pipeline_table_create(p, &table_params, &table_id)) + rte_panic("Unable to configure the hash table)\n"); + } + break; + + case e_APP_PIPELINE_HASH_SPEC_KEY16_LRU: + { + struct rte_pipeline_table_params table_params = { + .ops = &rte_table_hash_key16_lru_ops, + .arg_create = &table_hash_params, + .f_action_hit = NULL, + .f_action_miss = NULL, + .arg_ah = NULL, + .action_data_size = 0, + }; + + if (rte_pipeline_table_create(p, &table_params, &table_id)) + rte_panic("Unable to configure the hash table\n"); + } + break; + + case e_APP_PIPELINE_HASH_SPEC_KEY32_EXT: + { + struct rte_pipeline_table_params table_params = { + .ops = &rte_table_hash_key32_ext_ops, + .arg_create = &table_hash_params, + .f_action_hit = NULL, + .f_action_miss = NULL, + .arg_ah = NULL, + .action_data_size = 0, + }; + + if (rte_pipeline_table_create(p, &table_params, &table_id)) + rte_panic("Unable to configure the hash table\n"); + } + break; + + + case e_APP_PIPELINE_HASH_SPEC_KEY32_LRU: + { + struct rte_pipeline_table_params table_params = { + .ops = &rte_table_hash_key32_lru_ops, + .arg_create = &table_hash_params, + .f_action_hit = NULL, + .f_action_miss = NULL, + .arg_ah = NULL, + .action_data_size = 0, + }; + + if (rte_pipeline_table_create(p, &table_params, &table_id)) + rte_panic("Unable to configure the hash table\n"); + } + break; + + case e_APP_PIPELINE_HASH_CUCKOO_KEY8: + case e_APP_PIPELINE_HASH_CUCKOO_KEY16: + case e_APP_PIPELINE_HASH_CUCKOO_KEY32: + case e_APP_PIPELINE_HASH_CUCKOO_KEY48: + case e_APP_PIPELINE_HASH_CUCKOO_KEY64: + case e_APP_PIPELINE_HASH_CUCKOO_KEY80: + case e_APP_PIPELINE_HASH_CUCKOO_KEY96: + case e_APP_PIPELINE_HASH_CUCKOO_KEY112: + case e_APP_PIPELINE_HASH_CUCKOO_KEY128: + { + struct rte_pipeline_table_params table_params = { + .ops = &rte_table_hash_cuckoo_ops, + .arg_create = &table_hash_params, + .f_action_hit = NULL, + .f_action_miss = NULL, + .arg_ah = NULL, + .action_data_size = 0, + }; + + if (rte_pipeline_table_create(p, &table_params, &table_id)) + rte_panic("Unable to configure the hash table\n"); + } + break; + + default: + rte_panic("Invalid hash table type or key size\n"); + } + + /* Interconnecting ports and tables */ + for (i = 0; i < app.n_ports; i++) + if (rte_pipeline_port_in_connect_to_table(p, port_in_id[i], + table_id)) + rte_panic("Unable to connect input port %u to " + "table %u\n", port_in_id[i], table_id); + + /* Add entries to tables */ + for (i = 0; i < (1 << 24); i++) { + struct rte_pipeline_table_entry entry = { + .action = RTE_PIPELINE_ACTION_PORT, + {.port_id = port_out_id[i & (app.n_ports - 1)]}, + }; + struct rte_pipeline_table_entry *entry_ptr; + uint8_t key[32]; + uint32_t *k32 = (uint32_t *) key; + int key_found, status; + + memset(key, 0, sizeof(key)); + k32[0] = rte_be_to_cpu_32(i); + + status = rte_pipeline_table_entry_add(p, table_id, key, &entry, + &key_found, &entry_ptr); + if (status < 0) + rte_panic("Unable to add entry to table %u (%d)\n", + table_id, status); + } + + /* Enable input ports */ + for (i = 0; i < app.n_ports; i++) + if (rte_pipeline_port_in_enable(p, port_in_id[i])) + rte_panic("Unable to enable input port %u\n", + port_in_id[i]); + + /* Check pipeline consistency */ + if (rte_pipeline_check(p) < 0) + rte_panic("Pipeline consistency check failed\n"); + + /* Run-time */ +#if APP_FLUSH == 0 + for ( ; ; ) + rte_pipeline_run(p); +#else + for (i = 0; ; i++) { + rte_pipeline_run(p); + + if ((i & APP_FLUSH) == 0) + rte_pipeline_flush(p); + } +#endif +} + +uint64_t test_hash( + void *key, + __attribute__((unused)) void *key_mask, + __attribute__((unused)) uint32_t key_size, + __attribute__((unused)) uint64_t seed) +{ + uint32_t *k32 = key; + uint32_t ip_dst = rte_be_to_cpu_32(k32[0]); + uint64_t signature = (ip_dst >> 2) | ((ip_dst & 0x3) << 30); + + return signature; +} + +void +app_main_loop_rx_metadata(void) { + uint32_t i, j; + int ret; + + RTE_LOG(INFO, USER1, "Core %u is doing RX (with meta-data)\n", + rte_lcore_id()); + + for (i = 0; ; i = ((i + 1) & (app.n_ports - 1))) { + uint16_t n_mbufs; + + n_mbufs = rte_eth_rx_burst( + app.ports[i], + 0, + app.mbuf_rx.array, + app.burst_size_rx_read); + + if (n_mbufs == 0) + continue; + + for (j = 0; j < n_mbufs; j++) { + struct rte_mbuf *m; + uint8_t *m_data, *key; + struct ipv4_hdr *ip_hdr; + struct ipv6_hdr *ipv6_hdr; + uint32_t ip_dst; + uint8_t *ipv6_dst; + uint32_t *signature, *k32; + + m = app.mbuf_rx.array[j]; + m_data = rte_pktmbuf_mtod(m, uint8_t *); + signature = RTE_MBUF_METADATA_UINT32_PTR(m, + APP_METADATA_OFFSET(0)); + key = RTE_MBUF_METADATA_UINT8_PTR(m, + APP_METADATA_OFFSET(32)); + + if (RTE_ETH_IS_IPV4_HDR(m->packet_type)) { + ip_hdr = (struct ipv4_hdr *) + &m_data[sizeof(struct ether_hdr)]; + ip_dst = ip_hdr->dst_addr; + + k32 = (uint32_t *) key; + k32[0] = ip_dst & 0xFFFFFF00; + } else if (RTE_ETH_IS_IPV6_HDR(m->packet_type)) { + ipv6_hdr = (struct ipv6_hdr *) + &m_data[sizeof(struct ether_hdr)]; + ipv6_dst = ipv6_hdr->dst_addr; + + memcpy(key, ipv6_dst, 16); + } else + continue; + + *signature = test_hash(key, NULL, 0, 0); + } + + do { + ret = rte_ring_sp_enqueue_bulk( + app.rings_rx[i], + (void **) app.mbuf_rx.array, + n_mbufs, + NULL); + } while (ret == 0); + } +} diff --git a/dpdk/app/test-pipeline/pipeline_lpm.c b/dpdk/test/test-pipeline/pipeline_lpm.c similarity index 100% rename from dpdk/app/test-pipeline/pipeline_lpm.c rename to dpdk/test/test-pipeline/pipeline_lpm.c diff --git a/dpdk/app/test-pipeline/pipeline_lpm_ipv6.c b/dpdk/test/test-pipeline/pipeline_lpm_ipv6.c similarity index 100% rename from dpdk/app/test-pipeline/pipeline_lpm_ipv6.c rename to dpdk/test/test-pipeline/pipeline_lpm_ipv6.c diff --git a/dpdk/app/test-pipeline/pipeline_stub.c b/dpdk/test/test-pipeline/pipeline_stub.c similarity index 100% rename from dpdk/app/test-pipeline/pipeline_stub.c rename to dpdk/test/test-pipeline/pipeline_stub.c diff --git a/dpdk/test/test-pipeline/runtime.c b/dpdk/test/test-pipeline/runtime.c new file mode 100644 index 00000000..c7c45379 --- /dev/null +++ b/dpdk/test/test-pipeline/runtime.c @@ -0,0 +1,184 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <stdlib.h> +#include <stdint.h> +#include <inttypes.h> +#include <sys/types.h> +#include <string.h> +#include <sys/queue.h> +#include <stdarg.h> +#include <errno.h> +#include <getopt.h> + +#include <rte_common.h> +#include <rte_byteorder.h> +#include <rte_log.h> +#include <rte_memory.h> +#include <rte_memcpy.h> +#include <rte_eal.h> +#include <rte_per_lcore.h> +#include <rte_launch.h> +#include <rte_atomic.h> +#include <rte_cycles.h> +#include <rte_prefetch.h> +#include <rte_branch_prediction.h> +#include <rte_interrupts.h> +#include <rte_pci.h> +#include <rte_random.h> +#include <rte_debug.h> +#include <rte_ether.h> +#include <rte_ethdev.h> +#include <rte_ring.h> +#include <rte_mempool.h> +#include <rte_mbuf.h> +#include <rte_ip.h> +#include <rte_tcp.h> +#include <rte_lpm.h> +#include <rte_lpm6.h> +#include <rte_malloc.h> + +#include "main.h" + +void +app_main_loop_rx(void) { + uint32_t i; + int ret; + + RTE_LOG(INFO, USER1, "Core %u is doing RX\n", rte_lcore_id()); + + for (i = 0; ; i = ((i + 1) & (app.n_ports - 1))) { + uint16_t n_mbufs; + + n_mbufs = rte_eth_rx_burst( + app.ports[i], + 0, + app.mbuf_rx.array, + app.burst_size_rx_read); + + if (n_mbufs == 0) + continue; + + do { + ret = rte_ring_sp_enqueue_bulk( + app.rings_rx[i], + (void **) app.mbuf_rx.array, + n_mbufs, NULL); + } while (ret == 0); + } +} + +void +app_main_loop_worker(void) { + struct app_mbuf_array *worker_mbuf; + uint32_t i; + + RTE_LOG(INFO, USER1, "Core %u is doing work (no pipeline)\n", + rte_lcore_id()); + + worker_mbuf = rte_malloc_socket(NULL, sizeof(struct app_mbuf_array), + RTE_CACHE_LINE_SIZE, rte_socket_id()); + if (worker_mbuf == NULL) + rte_panic("Worker thread: cannot allocate buffer space\n"); + + for (i = 0; ; i = ((i + 1) & (app.n_ports - 1))) { + int ret; + + ret = rte_ring_sc_dequeue_bulk( + app.rings_rx[i], + (void **) worker_mbuf->array, + app.burst_size_worker_read, + NULL); + + if (ret == 0) + continue; + + do { + ret = rte_ring_sp_enqueue_bulk( + app.rings_tx[i ^ 1], + (void **) worker_mbuf->array, + app.burst_size_worker_write, + NULL); + } while (ret == 0); + } +} + +void +app_main_loop_tx(void) { + uint32_t i; + + RTE_LOG(INFO, USER1, "Core %u is doing TX\n", rte_lcore_id()); + + for (i = 0; ; i = ((i + 1) & (app.n_ports - 1))) { + uint16_t n_mbufs, n_pkts; + int ret; + + n_mbufs = app.mbuf_tx[i].n_mbufs; + + ret = rte_ring_sc_dequeue_bulk( + app.rings_tx[i], + (void **) &app.mbuf_tx[i].array[n_mbufs], + app.burst_size_tx_read, + NULL); + + if (ret == 0) + continue; + + n_mbufs += app.burst_size_tx_read; + + if (n_mbufs < app.burst_size_tx_write) { + app.mbuf_tx[i].n_mbufs = n_mbufs; + continue; + } + + n_pkts = rte_eth_tx_burst( + app.ports[i], + 0, + app.mbuf_tx[i].array, + n_mbufs); + + if (n_pkts < n_mbufs) { + uint16_t k; + + for (k = n_pkts; k < n_mbufs; k++) { + struct rte_mbuf *pkt_to_free; + + pkt_to_free = app.mbuf_tx[i].array[k]; + rte_pktmbuf_free(pkt_to_free); + } + } + + app.mbuf_tx[i].n_mbufs = 0; + } +} diff --git a/dpdk/test/test/Makefile b/dpdk/test/test/Makefile new file mode 100644 index 00000000..bb54c980 --- /dev/null +++ b/dpdk/test/test/Makefile @@ -0,0 +1,272 @@ +# BSD LICENSE +# +# Copyright(c) 2010-2017 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +include $(RTE_SDK)/mk/rte.vars.mk + +ifeq ($(CONFIG_RTE_APP_TEST),y) + +# default rule +all: + +# Define an externally linked resource. A linked resource is an arbitrary +# file that is linked into the test binary. The application refers to this +# resource by name. The linked generates identifiers beg_<name> and end_<name> +# for referencing by the C code. +# +# Parameters: <unique name>, <file to be linked> +define linked_resource +SRCS-y += $(1).res.o +$(1).res.o: $(2) + @ echo ' MKRES $$@' + $Q [ "$$(<D)" = . ] || ln -fs $$< + $Q $(OBJCOPY) -I binary -B $(RTE_OBJCOPY_ARCH) -O $(RTE_OBJCOPY_TARGET) \ + --rename-section \ + .data=.rodata,alloc,load,data,contents,readonly \ + --redefine-sym _binary_$$(subst .,_,$$(<F))_start=beg_$(1) \ + --redefine-sym _binary_$$(subst .,_,$$(<F))_end=end_$(1) \ + --redefine-sym _binary_$$(subst .,_,$$(<F))_size=siz_$(1) \ + $$(<F) $$@ +endef + +ifeq ($(CONFIG_RTE_APP_TEST_RESOURCE_TAR),y) +define linked_tar_resource +$(1).tar: $(2) + @ echo ' TAR $$@' + $Q tar -C $$(dir $$<) -cf $$@ $$(notdir $$<) +$(call linked_resource,$(1),$(1).tar) +endef +else # ! CONFIG_RTE_APP_TEST_RESOURCE_TAR +linked_tar_resource = +endif # CONFIG_RTE_APP_TEST_RESOURCE_TAR + +# +# library name +# +APP = test + +# +# all sources are stored in SRCS-y +# +SRCS-$(CONFIG_RTE_LIBRTE_CMDLINE) := commands.c +SRCS-y += test.c +SRCS-y += resource.c +SRCS-y += test_resource.c +test_resource.res: test_resource.c + @ cp $< $@ +$(eval $(call linked_resource,test_resource_c,test_resource.res)) +$(eval $(call linked_tar_resource,test_resource_tar,test_resource.c)) +SRCS-$(CONFIG_RTE_APP_TEST_RESOURCE_TAR) += test_cfgfile.c +$(eval $(call linked_tar_resource,test_cfgfiles,test_cfgfiles)) +SRCS-y += test_prefetch.c +SRCS-y += test_byteorder.c +SRCS-y += test_per_lcore.c +SRCS-y += test_atomic.c +SRCS-y += test_malloc.c +SRCS-y += test_cycles.c +SRCS-y += test_spinlock.c +SRCS-y += test_memory.c +SRCS-y += test_memzone.c +SRCS-y += test_bitmap.c + +SRCS-y += test_ring.c +SRCS-y += test_ring_perf.c +SRCS-y += test_pmd_perf.c + +ifeq ($(CONFIG_RTE_LIBRTE_TABLE),y) +SRCS-y += test_table.c +SRCS-$(CONFIG_RTE_LIBRTE_PIPELINE) += test_table_pipeline.c +SRCS-y += test_table_tables.c +SRCS-y += test_table_ports.c +SRCS-y += test_table_combined.c +SRCS-$(CONFIG_RTE_LIBRTE_ACL) += test_table_acl.c +SRCS-$(CONFIG_RTE_LIBRTE_FLOW_CLASSIFY) += test_flow_classify.c +endif + +SRCS-y += test_rwlock.c + +SRCS-$(CONFIG_RTE_LIBRTE_TIMER) += test_timer.c +SRCS-$(CONFIG_RTE_LIBRTE_TIMER) += test_timer_perf.c +SRCS-$(CONFIG_RTE_LIBRTE_TIMER) += test_timer_racecond.c + +SRCS-y += test_mempool.c +SRCS-y += test_mempool_perf.c + +SRCS-y += test_mbuf.c +SRCS-y += test_logs.c + +SRCS-y += test_memcpy.c +SRCS-y += test_memcpy_perf.c + + +SRCS-$(CONFIG_RTE_LIBRTE_MEMBER) += test_member.c +SRCS-$(CONFIG_RTE_LIBRTE_MEMBER) += test_member_perf.c + +SRCS-$(CONFIG_RTE_LIBRTE_EFD) += test_efd.c +SRCS-$(CONFIG_RTE_LIBRTE_EFD) += test_efd_perf.c + +SRCS-$(CONFIG_RTE_LIBRTE_HASH) += test_hash.c +SRCS-$(CONFIG_RTE_LIBRTE_HASH) += test_thash.c +SRCS-$(CONFIG_RTE_LIBRTE_HASH) += test_hash_perf.c +SRCS-$(CONFIG_RTE_LIBRTE_HASH) += test_hash_functions.c +SRCS-$(CONFIG_RTE_LIBRTE_HASH) += test_hash_scaling.c +SRCS-$(CONFIG_RTE_LIBRTE_HASH) += test_hash_multiwriter.c + +SRCS-$(CONFIG_RTE_LIBRTE_LPM) += test_lpm.c +SRCS-$(CONFIG_RTE_LIBRTE_LPM) += test_lpm_perf.c +SRCS-$(CONFIG_RTE_LIBRTE_LPM) += test_lpm6.c +SRCS-$(CONFIG_RTE_LIBRTE_LPM) += test_lpm6_perf.c + +SRCS-y += test_debug.c +SRCS-y += test_errno.c +SRCS-y += test_tailq.c +SRCS-y += test_string_fns.c +SRCS-y += test_cpuflags.c +SRCS-y += test_mp_secondary.c +SRCS-y += test_eal_flags.c +SRCS-y += test_eal_fs.c +SRCS-y += test_alarm.c +SRCS-y += test_interrupts.c +SRCS-y += test_version.c +SRCS-y += test_func_reentrancy.c + +SRCS-y += test_service_cores.c + +SRCS-$(CONFIG_RTE_LIBRTE_CMDLINE) += test_cmdline.c +SRCS-$(CONFIG_RTE_LIBRTE_CMDLINE) += test_cmdline_num.c +SRCS-$(CONFIG_RTE_LIBRTE_CMDLINE) += test_cmdline_etheraddr.c +SRCS-$(CONFIG_RTE_LIBRTE_CMDLINE) += test_cmdline_portlist.c +SRCS-$(CONFIG_RTE_LIBRTE_CMDLINE) += test_cmdline_ipaddr.c +SRCS-$(CONFIG_RTE_LIBRTE_CMDLINE) += test_cmdline_cirbuf.c +SRCS-$(CONFIG_RTE_LIBRTE_CMDLINE) += test_cmdline_string.c +SRCS-$(CONFIG_RTE_LIBRTE_CMDLINE) += test_cmdline_lib.c + +SRCS-$(CONFIG_RTE_LIBRTE_NET) += test_crc.c + +ifeq ($(CONFIG_RTE_LIBRTE_SCHED),y) +SRCS-y += test_red.c +SRCS-y += test_sched.c +endif + +SRCS-$(CONFIG_RTE_LIBRTE_METER) += test_meter.c +SRCS-$(CONFIG_RTE_LIBRTE_KNI) += test_kni.c +SRCS-$(CONFIG_RTE_LIBRTE_POWER) += test_power.c test_power_acpi_cpufreq.c +SRCS-$(CONFIG_RTE_LIBRTE_POWER) += test_power_kvm_vm.c +SRCS-y += test_common.c + +SRCS-$(CONFIG_RTE_LIBRTE_DISTRIBUTOR) += test_distributor.c +SRCS-$(CONFIG_RTE_LIBRTE_DISTRIBUTOR) += test_distributor_perf.c + +SRCS-$(CONFIG_RTE_LIBRTE_REORDER) += test_reorder.c + +SRCS-y += test_devargs.c +SRCS-y += virtual_pmd.c +SRCS-y += packet_burst_generator.c +SRCS-$(CONFIG_RTE_LIBRTE_ACL) += test_acl.c + +ifeq ($(CONFIG_RTE_LIBRTE_PMD_RING),y) +SRCS-$(CONFIG_RTE_LIBRTE_PMD_BOND) += test_link_bonding.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_BOND) += test_link_bonding_mode4.c +endif + +ifeq ($(CONFIG_RTE_LIBRTE_PMD_NULL),y) +SRCS-$(CONFIG_RTE_LIBRTE_PMD_BOND) += test_link_bonding_rssconf.c +endif + +SRCS-$(CONFIG_RTE_LIBRTE_PMD_RING) += test_pmd_ring.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_RING) += test_pmd_ring_perf.c + +SRCS-$(CONFIG_RTE_LIBRTE_CRYPTODEV) += test_cryptodev_blockcipher.c +SRCS-$(CONFIG_RTE_LIBRTE_CRYPTODEV) += test_cryptodev.c + +ifeq ($(CONFIG_RTE_LIBRTE_EVENTDEV),y) +SRCS-y += test_eventdev.c +SRCS-y += test_event_ring.c +SRCS-y += test_event_eth_rx_adapter.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_SW_EVENTDEV) += test_eventdev_sw.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX_SSOVF) += test_eventdev_octeontx.c +endif + +SRCS-$(CONFIG_RTE_LIBRTE_KVARGS) += test_kvargs.c + +CFLAGS += -O3 +CFLAGS += $(WERROR_FLAGS) + +CFLAGS += -D_GNU_SOURCE + +LDLIBS += -lm + +# Disable VTA for memcpy test +ifeq ($(CONFIG_RTE_TOOLCHAIN_GCC),y) +ifeq ($(shell test $(GCC_VERSION) -ge 44 && echo 1), 1) +CFLAGS_test_memcpy.o += -fno-var-tracking-assignments +CFLAGS_test_memcpy_perf.o += -fno-var-tracking-assignments +# for older GCC versions, allow us to initialize an event using +# designated initializers. +ifeq ($(shell test $(GCC_VERSION) -le 50 && echo 1), 1) +CFLAGS_test_eventdev_sw.o += -Wno-missing-field-initializers +endif +endif +endif + +# Link against shared libraries when needed +ifeq ($(CONFIG_RTE_LIBRTE_PMD_BOND),y) +ifneq ($(CONFIG_RTE_LIBRTE_PMD_RING),y) +$(error Link bonding tests require CONFIG_RTE_LIBRTE_PMD_RING=y) +endif +endif + +ifeq ($(CONFIG_RTE_BUILD_SHARED_LIB),y) + +ifeq ($(CONFIG_RTE_LIBRTE_PMD_BOND),y) +LDLIBS += -lrte_pmd_bond +endif + +ifeq ($(CONFIG_RTE_LIBRTE_PMD_NULL),y) +LDLIBS += -lrte_pmd_null +endif + +ifeq ($(CONFIG_RTE_LIBRTE_PMD_RING),y) +LDLIBS += -lrte_pmd_ring +endif + +ifeq ($(CONFIG_RTE_LIBRTE_PMD_CRYPTO_SCHEDULER),y) +LDLIBS += -lrte_pmd_crypto_scheduler +endif + +endif + +ifeq ($(CONFIG_RTE_APP_TEST_RESOURCE_TAR),y) +LDLIBS += -larchive +endif + +include $(RTE_SDK)/mk/rte.app.mk + +endif diff --git a/dpdk/test/test/autotest.py b/dpdk/test/test/autotest.py new file mode 100644 index 00000000..5c19a022 --- /dev/null +++ b/dpdk/test/test/autotest.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python + +# BSD LICENSE +# +# Copyright(c) 2010-2014 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Script that uses either test app or qemu controlled by python-pexpect +from __future__ import print_function +import autotest_data +import autotest_runner +import sys + + +def usage(): + print("Usage: autotest.py [test app|test iso image] ", + "[target] [whitelist|-blacklist]") + +if len(sys.argv) < 3: + usage() + sys.exit(1) + +target = sys.argv[2] + +test_whitelist = None +test_blacklist = None + +# get blacklist/whitelist +if len(sys.argv) > 3: + testlist = sys.argv[3].split(',') + testlist = [test.lower() for test in testlist] + if testlist[0].startswith('-'): + testlist[0] = testlist[0].lstrip('-') + test_blacklist = testlist + else: + test_whitelist = testlist + +cmdline = "%s -c f -n 4" % (sys.argv[1]) + +print(cmdline) + +runner = autotest_runner.AutotestRunner(cmdline, target, test_blacklist, + test_whitelist) + +for test_group in autotest_data.parallel_test_group_list: + runner.add_parallel_test_group(test_group) + +for test_group in autotest_data.non_parallel_test_group_list: + runner.add_non_parallel_test_group(test_group) + +num_fails = runner.run_all_tests() + +sys.exit(num_fails) diff --git a/dpdk/test/test/autotest_data.py b/dpdk/test/test/autotest_data.py new file mode 100644 index 00000000..165ed6c5 --- /dev/null +++ b/dpdk/test/test/autotest_data.py @@ -0,0 +1,495 @@ +# BSD LICENSE +# +# Copyright(c) 2010-2014 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Test data for autotests + +from glob import glob +from autotest_test_funcs import * + + +# quick and dirty function to find out number of sockets +def num_sockets(): + result = len(glob("/sys/devices/system/node/node*")) + if result == 0: + return 1 + return result + + +# Assign given number to each socket +# e.g. 32 becomes 32,32 or 32,32,32,32 +def per_sockets(num): + return ",".join([str(num)] * num_sockets()) + +# groups of tests that can be run in parallel +# the grouping has been found largely empirically +parallel_test_group_list = [ + { + "Prefix": "group_1", + "Memory": per_sockets(8), + "Tests": + [ + { + "Name": "Cycles autotest", + "Command": "cycles_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Timer autotest", + "Command": "timer_autotest", + "Func": timer_autotest, + "Report": None, + }, + { + "Name": "Debug autotest", + "Command": "debug_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Errno autotest", + "Command": "errno_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Meter autotest", + "Command": "meter_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Common autotest", + "Command": "common_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Resource autotest", + "Command": "resource_autotest", + "Func": default_autotest, + "Report": None, + }, + ] + }, + { + "Prefix": "group_2", + "Memory": "16", + "Tests": + [ + { + "Name": "Memory autotest", + "Command": "memory_autotest", + "Func": memory_autotest, + "Report": None, + }, + { + "Name": "Read/write lock autotest", + "Command": "rwlock_autotest", + "Func": rwlock_autotest, + "Report": None, + }, + { + "Name": "Logs autotest", + "Command": "logs_autotest", + "Func": logs_autotest, + "Report": None, + }, + { + "Name": "CPU flags autotest", + "Command": "cpuflags_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Version autotest", + "Command": "version_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "EAL filesystem autotest", + "Command": "eal_fs_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "EAL flags autotest", + "Command": "eal_flags_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Hash autotest", + "Command": "hash_autotest", + "Func": default_autotest, + "Report": None, + }, + ], + }, + { + "Prefix": "group_3", + "Memory": per_sockets(512), + "Tests": + [ + { + "Name": "LPM autotest", + "Command": "lpm_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "LPM6 autotest", + "Command": "lpm6_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Memcpy autotest", + "Command": "memcpy_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Memzone autotest", + "Command": "memzone_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "String autotest", + "Command": "string_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Alarm autotest", + "Command": "alarm_autotest", + "Func": default_autotest, + "Report": None, + }, + ] + }, + { + "Prefix": "group_4", + "Memory": per_sockets(128), + "Tests": + [ + { + "Name": "PCI autotest", + "Command": "pci_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Malloc autotest", + "Command": "malloc_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Multi-process autotest", + "Command": "multiprocess_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Mbuf autotest", + "Command": "mbuf_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Per-lcore autotest", + "Command": "per_lcore_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Ring autotest", + "Command": "ring_autotest", + "Func": default_autotest, + "Report": None, + }, + ] + }, + { + "Prefix": "group_5", + "Memory": "32", + "Tests": + [ + { + "Name": "Spinlock autotest", + "Command": "spinlock_autotest", + "Func": spinlock_autotest, + "Report": None, + }, + { + "Name": "Byte order autotest", + "Command": "byteorder_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "TAILQ autotest", + "Command": "tailq_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Command-line autotest", + "Command": "cmdline_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Interrupts autotest", + "Command": "interrupt_autotest", + "Func": default_autotest, + "Report": None, + }, + ] + }, + { + "Prefix": "group_6", + "Memory": per_sockets(512), + "Tests": + [ + { + "Name": "Function reentrancy autotest", + "Command": "func_reentrancy_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Mempool autotest", + "Command": "mempool_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Atomics autotest", + "Command": "atomic_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Prefetch autotest", + "Command": "prefetch_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Red autotest", + "Command": "red_autotest", + "Func": default_autotest, + "Report": None, + }, + ] + }, + { + "Prefix": "group_7", + "Memory": "64", + "Tests": + [ + { + "Name": "PMD ring autotest", + "Command": "ring_pmd_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Access list control autotest", + "Command": "acl_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Sched autotest", + "Command": "sched_autotest", + "Func": default_autotest, + "Report": None, + }, + ] + }, +] + +# tests that should not be run when any other tests are running +non_parallel_test_group_list = [ + + { + "Prefix": "eventdev", + "Memory": "512", + "Tests": + [ + { + "Name": "Eventdev common autotest", + "Command": "eventdev_common_autotest", + "Func": default_autotest, + "Report": None, + }, + ] + }, + { + "Prefix": "eventdev_sw", + "Memory": "512", + "Tests": + [ + { + "Name": "Eventdev sw autotest", + "Command": "eventdev_sw_autotest", + "Func": default_autotest, + "Report": None, + }, + ] + }, + { + "Prefix": "kni", + "Memory": "512", + "Tests": + [ + { + "Name": "KNI autotest", + "Command": "kni_autotest", + "Func": default_autotest, + "Report": None, + }, + ] + }, + { + "Prefix": "mempool_perf", + "Memory": per_sockets(256), + "Tests": + [ + { + "Name": "Mempool performance autotest", + "Command": "mempool_perf_autotest", + "Func": default_autotest, + "Report": None, + }, + ] + }, + { + "Prefix": "memcpy_perf", + "Memory": per_sockets(512), + "Tests": + [ + { + "Name": "Memcpy performance autotest", + "Command": "memcpy_perf_autotest", + "Func": default_autotest, + "Report": None, + }, + ] + }, + { + "Prefix": "hash_perf", + "Memory": per_sockets(512), + "Tests": + [ + { + "Name": "Hash performance autotest", + "Command": "hash_perf_autotest", + "Func": default_autotest, + "Report": None, + }, + ] + }, + { + "Prefix": "power", + "Memory": "16", + "Tests": + [ + { + "Name": "Power autotest", + "Command": "power_autotest", + "Func": default_autotest, + "Report": None, + }, + ] + }, + { + "Prefix": "power_acpi_cpufreq", + "Memory": "16", + "Tests": + [ + { + "Name": "Power ACPI cpufreq autotest", + "Command": "power_acpi_cpufreq_autotest", + "Func": default_autotest, + "Report": None, + }, + ] + }, + { + "Prefix": "power_kvm_vm", + "Memory": "16", + "Tests": + [ + { + "Name": "Power KVM VM autotest", + "Command": "power_kvm_vm_autotest", + "Func": default_autotest, + "Report": None, + }, + ] + }, + { + "Prefix": "timer_perf", + "Memory": per_sockets(512), + "Tests": + [ + { + "Name": "Timer performance autotest", + "Command": "timer_perf_autotest", + "Func": default_autotest, + "Report": None, + }, + ] + }, + + # + # Please always make sure that ring_perf is the last test! + # + { + "Prefix": "ring_perf", + "Memory": per_sockets(512), + "Tests": + [ + { + "Name": "Ring performance autotest", + "Command": "ring_perf_autotest", + "Func": default_autotest, + "Report": None, + }, + ] + }, +] diff --git a/dpdk/test/test/autotest_runner.py b/dpdk/test/test/autotest_runner.py new file mode 100644 index 00000000..fc882ec0 --- /dev/null +++ b/dpdk/test/test/autotest_runner.py @@ -0,0 +1,428 @@ +# BSD LICENSE +# +# Copyright(c) 2010-2014 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# The main logic behind running autotests in parallel + +import StringIO +import csv +import multiprocessing +import pexpect +import re +import subprocess +import sys +import time + +# wait for prompt + + +def wait_prompt(child): + try: + child.sendline() + result = child.expect(["RTE>>", pexpect.TIMEOUT, pexpect.EOF], + timeout=120) + except: + return False + if result == 0: + return True + else: + return False + +# run a test group +# each result tuple in results list consists of: +# result value (0 or -1) +# result string +# test name +# total test run time (double) +# raw test log +# test report (if not available, should be None) +# +# this function needs to be outside AutotestRunner class +# because otherwise Pool won't work (or rather it will require +# quite a bit of effort to make it work). + + +def run_test_group(cmdline, test_group): + results = [] + child = None + start_time = time.time() + startuplog = None + + # run test app + try: + # prepare logging of init + startuplog = StringIO.StringIO() + + print >>startuplog, "\n%s %s\n" % ("=" * 20, test_group["Prefix"]) + print >>startuplog, "\ncmdline=%s" % cmdline + + child = pexpect.spawn(cmdline, logfile=startuplog) + + # wait for target to boot + if not wait_prompt(child): + child.close() + + results.append((-1, + "Fail [No prompt]", + "Start %s" % test_group["Prefix"], + time.time() - start_time, + startuplog.getvalue(), + None)) + + # mark all tests as failed + for test in test_group["Tests"]: + results.append((-1, "Fail [No prompt]", test["Name"], + time.time() - start_time, "", None)) + # exit test + return results + + except: + results.append((-1, + "Fail [Can't run]", + "Start %s" % test_group["Prefix"], + time.time() - start_time, + startuplog.getvalue(), + None)) + + # mark all tests as failed + for t in test_group["Tests"]: + results.append((-1, "Fail [Can't run]", t["Name"], + time.time() - start_time, "", None)) + # exit test + return results + + # startup was successful + results.append((0, "Success", "Start %s" % test_group["Prefix"], + time.time() - start_time, startuplog.getvalue(), None)) + + # parse the binary for available test commands + binary = cmdline.split()[0] + stripped = 'not stripped' not in subprocess.check_output(['file', binary]) + if not stripped: + symbols = subprocess.check_output(['nm', binary]).decode('utf-8') + avail_cmds = re.findall('test_register_(\w+)', symbols) + + # run all tests in test group + for test in test_group["Tests"]: + + # create log buffer for each test + # in multiprocessing environment, the logging would be + # interleaved and will create a mess, hence the buffering + logfile = StringIO.StringIO() + child.logfile = logfile + + result = () + + # make a note when the test started + start_time = time.time() + + try: + # print test name to log buffer + print >>logfile, "\n%s %s\n" % ("-" * 20, test["Name"]) + + # run test function associated with the test + if stripped or test["Command"] in avail_cmds: + result = test["Func"](child, test["Command"]) + else: + result = (0, "Skipped [Not Available]") + + # make a note when the test was finished + end_time = time.time() + + # append test data to the result tuple + result += (test["Name"], end_time - start_time, + logfile.getvalue()) + + # call report function, if any defined, and supply it with + # target and complete log for test run + if test["Report"]: + report = test["Report"](self.target, log) + + # append report to results tuple + result += (report,) + else: + # report is None + result += (None,) + except: + # make a note when the test crashed + end_time = time.time() + + # mark test as failed + result = (-1, "Fail [Crash]", test["Name"], + end_time - start_time, logfile.getvalue(), None) + finally: + # append the results to the results list + results.append(result) + + # regardless of whether test has crashed, try quitting it + try: + child.sendline("quit") + child.close() + # if the test crashed, just do nothing instead + except: + # nop + pass + + # return test results + return results + + +# class representing an instance of autotests run +class AutotestRunner: + cmdline = "" + parallel_test_groups = [] + non_parallel_test_groups = [] + logfile = None + csvwriter = None + target = "" + start = None + n_tests = 0 + fails = 0 + log_buffers = [] + blacklist = [] + whitelist = [] + + def __init__(self, cmdline, target, blacklist, whitelist): + self.cmdline = cmdline + self.target = target + self.blacklist = blacklist + self.whitelist = whitelist + + # log file filename + logfile = "%s.log" % target + csvfile = "%s.csv" % target + + self.logfile = open(logfile, "w") + csvfile = open(csvfile, "w") + self.csvwriter = csv.writer(csvfile) + + # prepare results table + self.csvwriter.writerow(["test_name", "test_result", "result_str"]) + + # set up cmdline string + def __get_cmdline(self, test): + cmdline = self.cmdline + + # append memory limitations for each test + # otherwise tests won't run in parallel + if "i686" not in self.target: + cmdline += " --socket-mem=%s" % test["Memory"] + else: + # affinitize startup so that tests don't fail on i686 + cmdline = "taskset 1 " + cmdline + cmdline += " -m " + str(sum(map(int, test["Memory"].split(",")))) + + # set group prefix for autotest group + # otherwise they won't run in parallel + cmdline += " --file-prefix=%s" % test["Prefix"] + + return cmdline + + def add_parallel_test_group(self, test_group): + self.parallel_test_groups.append(test_group) + + def add_non_parallel_test_group(self, test_group): + self.non_parallel_test_groups.append(test_group) + + def __process_results(self, results): + # this iterates over individual test results + for i, result in enumerate(results): + + # increase total number of tests that were run + # do not include "start" test + if i > 0: + self.n_tests += 1 + + # unpack result tuple + test_result, result_str, test_name, \ + test_time, log, report = result + + # get total run time + cur_time = time.time() + total_time = int(cur_time - self.start) + + # print results, test run time and total time since start + result = ("%s:" % test_name).ljust(30) + result += result_str.ljust(29) + result += "[%02dm %02ds]" % (test_time / 60, test_time % 60) + + # don't print out total time every line, it's the same anyway + if i == len(results) - 1: + print(result, + "[%02dm %02ds]" % (total_time / 60, total_time % 60)) + else: + print(result) + + # if test failed and it wasn't a "start" test + if test_result < 0 and not i == 0: + self.fails += 1 + + # collect logs + self.log_buffers.append(log) + + # create report if it exists + if report: + try: + f = open("%s_%s_report.rst" % + (self.target, test_name), "w") + except IOError: + print("Report for %s could not be created!" % test_name) + else: + with f: + f.write(report) + + # write test result to CSV file + if i != 0: + self.csvwriter.writerow([test_name, test_result, result_str]) + + # this function iterates over test groups and removes each + # test that is not in whitelist/blacklist + def __filter_groups(self, test_groups): + groups_to_remove = [] + + # filter out tests from parallel test groups + for i, test_group in enumerate(test_groups): + + # iterate over a copy so that we could safely delete individual + # tests + for test in test_group["Tests"][:]: + test_id = test["Command"] + + # dump tests are specified in full e.g. "Dump_mempool" + if "_autotest" in test_id: + test_id = test_id[:-len("_autotest")] + + # filter out blacklisted/whitelisted tests + if self.blacklist and test_id in self.blacklist: + test_group["Tests"].remove(test) + continue + if self.whitelist and test_id not in self.whitelist: + test_group["Tests"].remove(test) + continue + + # modify or remove original group + if len(test_group["Tests"]) > 0: + test_groups[i] = test_group + else: + # remember which groups should be deleted + # put the numbers backwards so that we start + # deleting from the end, not from the beginning + groups_to_remove.insert(0, i) + + # remove test groups that need to be removed + for i in groups_to_remove: + del test_groups[i] + + return test_groups + + # iterate over test groups and run tests associated with them + def run_all_tests(self): + # filter groups + self.parallel_test_groups = \ + self.__filter_groups(self.parallel_test_groups) + self.non_parallel_test_groups = \ + self.__filter_groups(self.non_parallel_test_groups) + + # create a pool of worker threads + pool = multiprocessing.Pool(processes=1) + + results = [] + + # whatever happens, try to save as much logs as possible + try: + + # create table header + print("") + print("Test name".ljust(30), "Test result".ljust(29), + "Test".center(9), "Total".center(9)) + print("=" * 80) + + # make a note of tests start time + self.start = time.time() + + # assign worker threads to run test groups + for test_group in self.parallel_test_groups: + result = pool.apply_async(run_test_group, + [self.__get_cmdline(test_group), + test_group]) + results.append(result) + + # iterate while we have group execution results to get + while len(results) > 0: + + # iterate over a copy to be able to safely delete results + # this iterates over a list of group results + for group_result in results[:]: + + # if the thread hasn't finished yet, continue + if not group_result.ready(): + continue + + res = group_result.get() + + self.__process_results(res) + + # remove result from results list once we're done with it + results.remove(group_result) + + # run non_parallel tests. they are run one by one, synchronously + for test_group in self.non_parallel_test_groups: + group_result = run_test_group( + self.__get_cmdline(test_group), test_group) + + self.__process_results(group_result) + + # get total run time + cur_time = time.time() + total_time = int(cur_time - self.start) + + # print out summary + print("=" * 80) + print("Total run time: %02dm %02ds" % (total_time / 60, + total_time % 60)) + if self.fails != 0: + print("Number of failed tests: %s" % str(self.fails)) + + # write summary to logfile + self.logfile.write("Summary\n") + self.logfile.write("Target: ".ljust(15) + "%s\n" % self.target) + self.logfile.write("Tests: ".ljust(15) + "%i\n" % self.n_tests) + self.logfile.write("Failed tests: ".ljust( + 15) + "%i\n" % self.fails) + except: + print("Exception occurred") + print(sys.exc_info()) + self.fails = 1 + + # drop logs from all executions to a logfile + for buf in self.log_buffers: + self.logfile.write(buf.replace("\r", "")) + + return self.fails diff --git a/dpdk/test/test/autotest_test_funcs.py b/dpdk/test/test/autotest_test_funcs.py new file mode 100644 index 00000000..15fcb7cf --- /dev/null +++ b/dpdk/test/test/autotest_test_funcs.py @@ -0,0 +1,304 @@ +# BSD LICENSE +# +# Copyright(c) 2010-2014 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Test functions + +import pexpect + +# default autotest, used to run most tests +# waits for "Test OK" + + +def default_autotest(child, test_name): + child.sendline(test_name) + result = child.expect(["Test OK", "Test Failed", + "Command not found", pexpect.TIMEOUT], timeout=900) + if result == 1: + return -1, "Fail" + elif result == 2: + return -1, "Fail [Not found]" + elif result == 3: + return -1, "Fail [Timeout]" + return 0, "Success" + +# autotest used to run dump commands +# just fires the command + + +def dump_autotest(child, test_name): + child.sendline(test_name) + return 0, "Success" + +# memory autotest +# reads output and waits for Test OK + + +def memory_autotest(child, test_name): + lines = 0 + error = '' + child.sendline(test_name) + while True: + regexp = "IOVA:0x[0-9a-f]*, len:([0-9]*), virt:0x[0-9a-f]*, " \ + "socket_id:[0-9]*" + index = child.expect([regexp, "Test OK", "Test Failed", + pexpect.TIMEOUT], timeout=10) + if index == 3: + return -1, "Fail [Timeout]" + elif index == 1: + break + elif index == 2: + return -1, "Fail" + else: + lines = lines + 1 + size = int(child.match.groups()[0], 10) + if size <= 0: + error = 'Bad size' + + if lines <= 0: + return -1, "Fail [No entries]" + if error != '': + return -1, "Fail [{}]".format(error) + return 0, "Success" + + +def spinlock_autotest(child, test_name): + i = 0 + ir = 0 + child.sendline(test_name) + while True: + index = child.expect(["Test OK", + "Test Failed", + "Hello from core ([0-9]*) !", + "Hello from within recursive locks " + "from ([0-9]*) !", + pexpect.TIMEOUT], timeout=5) + # ok + if index == 0: + break + + # message, check ordering + elif index == 2: + if int(child.match.groups()[0]) < i: + return -1, "Fail [Bad order]" + i = int(child.match.groups()[0]) + elif index == 3: + if int(child.match.groups()[0]) < ir: + return -1, "Fail [Bad order]" + ir = int(child.match.groups()[0]) + + # fail + elif index == 4: + return -1, "Fail [Timeout]" + elif index == 1: + return -1, "Fail" + + return 0, "Success" + + +def rwlock_autotest(child, test_name): + i = 0 + child.sendline(test_name) + while True: + index = child.expect(["Test OK", + "Test Failed", + "Hello from core ([0-9]*) !", + "Global write lock taken on master " + "core ([0-9]*)", + pexpect.TIMEOUT], timeout=10) + # ok + if index == 0: + if i != 0xffff: + return -1, "Fail [Message is missing]" + break + + # message, check ordering + elif index == 2: + if int(child.match.groups()[0]) < i: + return -1, "Fail [Bad order]" + i = int(child.match.groups()[0]) + + # must be the last message, check ordering + elif index == 3: + i = 0xffff + + elif index == 4: + return -1, "Fail [Timeout]" + + # fail + else: + return -1, "Fail" + + return 0, "Success" + + +def logs_autotest(child, test_name): + child.sendline(test_name) + + log_list = [ + "TESTAPP1: error message", + "TESTAPP1: critical message", + "TESTAPP2: critical message", + "TESTAPP1: error message", + ] + + for log_msg in log_list: + index = child.expect([log_msg, + "Test OK", + "Test Failed", + pexpect.TIMEOUT], timeout=10) + + if index == 3: + return -1, "Fail [Timeout]" + # not ok + elif index != 0: + return -1, "Fail" + + index = child.expect(["Test OK", + "Test Failed", + pexpect.TIMEOUT], timeout=10) + + return 0, "Success" + + +def timer_autotest(child, test_name): + child.sendline(test_name) + + index = child.expect(["Start timer stress tests", + "Test Failed", + pexpect.TIMEOUT], timeout=5) + + if index == 1: + return -1, "Fail" + elif index == 2: + return -1, "Fail [Timeout]" + + index = child.expect(["Start timer stress tests 2", + "Test Failed", + pexpect.TIMEOUT], timeout=5) + + if index == 1: + return -1, "Fail" + elif index == 2: + return -1, "Fail [Timeout]" + + index = child.expect(["Start timer basic tests", + "Test Failed", + pexpect.TIMEOUT], timeout=5) + + if index == 1: + return -1, "Fail" + elif index == 2: + return -1, "Fail [Timeout]" + + lcore_tim0 = -1 + lcore_tim1 = -1 + lcore_tim2 = -1 + lcore_tim3 = -1 + + while True: + index = child.expect(["TESTTIMER: ([0-9]*): callback id=([0-9]*) " + "count=([0-9]*) on core ([0-9]*)", + "Test OK", + "Test Failed", + pexpect.TIMEOUT], timeout=10) + + if index == 1: + break + + if index == 2: + return -1, "Fail" + elif index == 3: + return -1, "Fail [Timeout]" + + try: + id = int(child.match.groups()[1]) + cnt = int(child.match.groups()[2]) + lcore = int(child.match.groups()[3]) + except: + return -1, "Fail [Cannot parse]" + + # timer0 always expires on the same core when cnt < 20 + if id == 0: + if lcore_tim0 == -1: + lcore_tim0 = lcore + elif lcore != lcore_tim0 and cnt < 20: + return -1, "Fail [lcore != lcore_tim0 (%d, %d)]" \ + % (lcore, lcore_tim0) + if cnt > 21: + return -1, "Fail [tim0 cnt > 21]" + + # timer1 each time expires on a different core + if id == 1: + if lcore == lcore_tim1: + return -1, "Fail [lcore == lcore_tim1 (%d, %d)]" \ + % (lcore, lcore_tim1) + lcore_tim1 = lcore + if cnt > 10: + return -1, "Fail [tim1 cnt > 30]" + + # timer0 always expires on the same core + if id == 2: + if lcore_tim2 == -1: + lcore_tim2 = lcore + elif lcore != lcore_tim2: + return -1, "Fail [lcore != lcore_tim2 (%d, %d)]" \ + % (lcore, lcore_tim2) + if cnt > 30: + return -1, "Fail [tim2 cnt > 30]" + + # timer0 always expires on the same core + if id == 3: + if lcore_tim3 == -1: + lcore_tim3 = lcore + elif lcore != lcore_tim3: + return -1, "Fail [lcore_tim3 changed (%d -> %d)]" \ + % (lcore, lcore_tim3) + if cnt > 30: + return -1, "Fail [tim3 cnt > 30]" + + # must be 2 different cores + if lcore_tim0 == lcore_tim3: + return -1, "Fail [lcore_tim0 (%d) == lcore_tim3 (%d)]" \ + % (lcore_tim0, lcore_tim3) + + return 0, "Success" + + +def ring_autotest(child, test_name): + child.sendline(test_name) + index = child.expect(["Test OK", "Test Failed", + pexpect.TIMEOUT], timeout=2) + if index == 1: + return -1, "Fail" + elif index == 2: + return -1, "Fail [Timeout]" + + return 0, "Success" diff --git a/dpdk/test/test/commands.c b/dpdk/test/test/commands.c new file mode 100644 index 00000000..4097a331 --- /dev/null +++ b/dpdk/test/test/commands.c @@ -0,0 +1,403 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * Copyright(c) 2014 6WIND S.A. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <stdarg.h> +#include <stdint.h> +#include <string.h> +#include <stdlib.h> +#include <netinet/in.h> +#include <termios.h> +#ifndef __linux__ +#ifndef __FreeBSD__ +#include <net/socket.h> +#endif +#endif +#include <inttypes.h> +#include <errno.h> +#include <sys/queue.h> + +#include <rte_common.h> +#include <rte_log.h> +#include <rte_debug.h> +#include <rte_memory.h> +#include <rte_memcpy.h> +#include <rte_memzone.h> +#include <rte_launch.h> +#include <rte_cycles.h> +#include <rte_eal.h> +#include <rte_per_lcore.h> +#include <rte_lcore.h> +#include <rte_atomic.h> +#include <rte_branch_prediction.h> +#include <rte_ring.h> +#include <rte_mempool.h> +#include <rte_mbuf.h> +#include <rte_devargs.h> + +#include <cmdline_rdline.h> +#include <cmdline_parse.h> +#include <cmdline_parse_ipaddr.h> +#include <cmdline_parse_num.h> +#include <cmdline_parse_string.h> +#include <cmdline.h> + +#include "test.h" + +/****************/ + +static struct test_commands_list commands_list = + TAILQ_HEAD_INITIALIZER(commands_list); + +void +add_test_command(struct test_command *t) +{ + TAILQ_INSERT_TAIL(&commands_list, t, next); +} + +struct cmd_autotest_result { + cmdline_fixed_string_t autotest; +}; + +static void cmd_autotest_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct test_command *t; + struct cmd_autotest_result *res = parsed_result; + int ret = 0; + + TAILQ_FOREACH(t, &commands_list, next) { + if (!strcmp(res->autotest, t->command)) + ret = t->callback(); + } + + if (ret == 0) + printf("Test OK\n"); + else + printf("Test Failed\n"); + fflush(stdout); +} + +cmdline_parse_token_string_t cmd_autotest_autotest = + TOKEN_STRING_INITIALIZER(struct cmd_autotest_result, autotest, + ""); + +cmdline_parse_inst_t cmd_autotest = { + .f = cmd_autotest_parsed, /* function to call */ + .data = NULL, /* 2nd arg of func */ + .help_str = "launch autotest", + .tokens = { /* token list, NULL terminated */ + (void *)&cmd_autotest_autotest, + NULL, + }, +}; + +/****************/ + +struct cmd_dump_result { + cmdline_fixed_string_t dump; +}; + +static void +dump_struct_sizes(void) +{ +#define DUMP_SIZE(t) printf("sizeof(" #t ") = %u\n", (unsigned)sizeof(t)); + DUMP_SIZE(struct rte_mbuf); + DUMP_SIZE(struct rte_mempool); + DUMP_SIZE(struct rte_ring); +#undef DUMP_SIZE +} + +static void cmd_dump_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_dump_result *res = parsed_result; + + if (!strcmp(res->dump, "dump_physmem")) + rte_dump_physmem_layout(stdout); + else if (!strcmp(res->dump, "dump_memzone")) + rte_memzone_dump(stdout); + else if (!strcmp(res->dump, "dump_struct_sizes")) + dump_struct_sizes(); + else if (!strcmp(res->dump, "dump_ring")) + rte_ring_list_dump(stdout); + else if (!strcmp(res->dump, "dump_mempool")) + rte_mempool_list_dump(stdout); + else if (!strcmp(res->dump, "dump_devargs")) + rte_eal_devargs_dump(stdout); + else if (!strcmp(res->dump, "dump_log_types")) + rte_log_dump(stdout); +} + +cmdline_parse_token_string_t cmd_dump_dump = + TOKEN_STRING_INITIALIZER(struct cmd_dump_result, dump, + "dump_physmem#dump_memzone#" + "dump_struct_sizes#dump_ring#dump_mempool#" + "dump_devargs#dump_log_types"); + +cmdline_parse_inst_t cmd_dump = { + .f = cmd_dump_parsed, /* function to call */ + .data = NULL, /* 2nd arg of func */ + .help_str = "dump status", + .tokens = { /* token list, NULL terminated */ + (void *)&cmd_dump_dump, + NULL, + }, +}; + +/****************/ + +struct cmd_dump_one_result { + cmdline_fixed_string_t dump; + cmdline_fixed_string_t name; +}; + +static void cmd_dump_one_parsed(void *parsed_result, struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_dump_one_result *res = parsed_result; + + if (!strcmp(res->dump, "dump_ring")) { + struct rte_ring *r; + r = rte_ring_lookup(res->name); + if (r == NULL) { + cmdline_printf(cl, "Cannot find ring\n"); + return; + } + rte_ring_dump(stdout, r); + } + else if (!strcmp(res->dump, "dump_mempool")) { + struct rte_mempool *mp; + mp = rte_mempool_lookup(res->name); + if (mp == NULL) { + cmdline_printf(cl, "Cannot find mempool\n"); + return; + } + rte_mempool_dump(stdout, mp); + } +} + +cmdline_parse_token_string_t cmd_dump_one_dump = + TOKEN_STRING_INITIALIZER(struct cmd_dump_one_result, dump, + "dump_ring#dump_mempool"); + +cmdline_parse_token_string_t cmd_dump_one_name = + TOKEN_STRING_INITIALIZER(struct cmd_dump_one_result, name, NULL); + +cmdline_parse_inst_t cmd_dump_one = { + .f = cmd_dump_one_parsed, /* function to call */ + .data = NULL, /* 2nd arg of func */ + .help_str = "dump one ring/mempool: dump_ring|dump_mempool <name>", + .tokens = { /* token list, NULL terminated */ + (void *)&cmd_dump_one_dump, + (void *)&cmd_dump_one_name, + NULL, + }, +}; + +/****************/ + +struct cmd_quit_result { + cmdline_fixed_string_t quit; +}; + +static void +cmd_quit_parsed(__attribute__((unused)) void *parsed_result, + struct cmdline *cl, + __attribute__((unused)) void *data) +{ + cmdline_quit(cl); +} + +cmdline_parse_token_string_t cmd_quit_quit = + TOKEN_STRING_INITIALIZER(struct cmd_quit_result, quit, + "quit"); + +cmdline_parse_inst_t cmd_quit = { + .f = cmd_quit_parsed, /* function to call */ + .data = NULL, /* 2nd arg of func */ + .help_str = "exit application", + .tokens = { /* token list, NULL terminated */ + (void *)&cmd_quit_quit, + NULL, + }, +}; + +/****************/ + +struct cmd_set_rxtx_result { + cmdline_fixed_string_t set; + cmdline_fixed_string_t mode; +}; + +static void cmd_set_rxtx_parsed(void *parsed_result, struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_set_rxtx_result *res = parsed_result; + if (test_set_rxtx_conf(res->mode) < 0) + cmdline_printf(cl, "Cannot find such mode\n"); +} + +cmdline_parse_token_string_t cmd_set_rxtx_set = + TOKEN_STRING_INITIALIZER(struct cmd_set_rxtx_result, set, + "set_rxtx_mode"); + +cmdline_parse_token_string_t cmd_set_rxtx_mode = + TOKEN_STRING_INITIALIZER(struct cmd_set_rxtx_result, mode, NULL); + +cmdline_parse_inst_t cmd_set_rxtx = { + .f = cmd_set_rxtx_parsed, /* function to call */ + .data = NULL, /* 2nd arg of func */ + .help_str = "set rxtx routine: " + "set_rxtx <mode>", + .tokens = { /* token list, NULL terminated */ + (void *)&cmd_set_rxtx_set, + (void *)&cmd_set_rxtx_mode, + NULL, + }, +}; + +/****************/ + +struct cmd_set_rxtx_anchor { + cmdline_fixed_string_t set; + cmdline_fixed_string_t type; +}; + +static void +cmd_set_rxtx_anchor_parsed(void *parsed_result, + struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_set_rxtx_anchor *res = parsed_result; + if (test_set_rxtx_anchor(res->type) < 0) + cmdline_printf(cl, "Cannot find such anchor\n"); +} + +cmdline_parse_token_string_t cmd_set_rxtx_anchor_set = + TOKEN_STRING_INITIALIZER(struct cmd_set_rxtx_anchor, set, + "set_rxtx_anchor"); + +cmdline_parse_token_string_t cmd_set_rxtx_anchor_type = + TOKEN_STRING_INITIALIZER(struct cmd_set_rxtx_anchor, type, NULL); + +cmdline_parse_inst_t cmd_set_rxtx_anchor = { + .f = cmd_set_rxtx_anchor_parsed, /* function to call */ + .data = NULL, /* 2nd arg of func */ + .help_str = "set rxtx anchor: " + "set_rxtx_anchor <type>", + .tokens = { /* token list, NULL terminated */ + (void *)&cmd_set_rxtx_anchor_set, + (void *)&cmd_set_rxtx_anchor_type, + NULL, + }, +}; + +/****************/ + +/* for stream control */ +struct cmd_set_rxtx_sc { + cmdline_fixed_string_t set; + cmdline_fixed_string_t type; +}; + +static void +cmd_set_rxtx_sc_parsed(void *parsed_result, + struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_set_rxtx_sc *res = parsed_result; + if (test_set_rxtx_sc(res->type) < 0) + cmdline_printf(cl, "Cannot find such stream control\n"); +} + +cmdline_parse_token_string_t cmd_set_rxtx_sc_set = + TOKEN_STRING_INITIALIZER(struct cmd_set_rxtx_sc, set, + "set_rxtx_sc"); + +cmdline_parse_token_string_t cmd_set_rxtx_sc_type = + TOKEN_STRING_INITIALIZER(struct cmd_set_rxtx_sc, type, NULL); + +cmdline_parse_inst_t cmd_set_rxtx_sc = { + .f = cmd_set_rxtx_sc_parsed, /* function to call */ + .data = NULL, /* 2nd arg of func */ + .help_str = "set rxtx stream control: " + "set_rxtx_sc <type>", + .tokens = { /* token list, NULL terminated */ + (void *)&cmd_set_rxtx_sc_set, + (void *)&cmd_set_rxtx_sc_type, + NULL, + }, +}; + +/****************/ + + +cmdline_parse_ctx_t main_ctx[] = { + (cmdline_parse_inst_t *)&cmd_autotest, + (cmdline_parse_inst_t *)&cmd_dump, + (cmdline_parse_inst_t *)&cmd_dump_one, + (cmdline_parse_inst_t *)&cmd_quit, + (cmdline_parse_inst_t *)&cmd_set_rxtx, + (cmdline_parse_inst_t *)&cmd_set_rxtx_anchor, + (cmdline_parse_inst_t *)&cmd_set_rxtx_sc, + NULL, +}; + +int commands_init(void) +{ + struct test_command *t; + char *commands, *ptr; + int commands_len = 0; + + TAILQ_FOREACH(t, &commands_list, next) { + commands_len += strlen(t->command) + 1; + } + + commands = malloc(commands_len + 1); + if (!commands) + return -1; + + ptr = commands; + TAILQ_FOREACH(t, &commands_list, next) { + ptr += sprintf(ptr, "%s#", t->command); + } + ptr--; + ptr[0] = '\0'; + + cmd_autotest_autotest.string_data.str = commands; + return 0; +} diff --git a/dpdk/test/test/packet_burst_generator.c b/dpdk/test/test/packet_burst_generator.c new file mode 100644 index 00000000..8f4ddcc4 --- /dev/null +++ b/dpdk/test/test/packet_burst_generator.c @@ -0,0 +1,476 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <rte_byteorder.h> +#include <rte_mbuf.h> + +#include "packet_burst_generator.h" + +#define UDP_SRC_PORT 1024 +#define UDP_DST_PORT 1024 + + +#define IP_DEFTTL 64 /* from RFC 1340. */ +#define IP_VERSION 0x40 +#define IP_HDRLEN 0x05 /* default IP header length == five 32-bits words. */ +#define IP_VHL_DEF (IP_VERSION | IP_HDRLEN) + +static void +copy_buf_to_pkt_segs(void *buf, unsigned len, struct rte_mbuf *pkt, + unsigned offset) +{ + struct rte_mbuf *seg; + void *seg_buf; + unsigned copy_len; + + seg = pkt; + while (offset >= seg->data_len) { + offset -= seg->data_len; + seg = seg->next; + } + copy_len = seg->data_len - offset; + seg_buf = rte_pktmbuf_mtod_offset(seg, char *, offset); + while (len > copy_len) { + rte_memcpy(seg_buf, buf, (size_t) copy_len); + len -= copy_len; + buf = ((char *) buf + copy_len); + seg = seg->next; + seg_buf = rte_pktmbuf_mtod(seg, void *); + } + rte_memcpy(seg_buf, buf, (size_t) len); +} + +static inline void +copy_buf_to_pkt(void *buf, unsigned len, struct rte_mbuf *pkt, unsigned offset) +{ + if (offset + len <= pkt->data_len) { + rte_memcpy(rte_pktmbuf_mtod_offset(pkt, char *, offset), buf, + (size_t) len); + return; + } + copy_buf_to_pkt_segs(buf, len, pkt, offset); +} + +void +initialize_eth_header(struct ether_hdr *eth_hdr, struct ether_addr *src_mac, + struct ether_addr *dst_mac, uint16_t ether_type, + uint8_t vlan_enabled, uint16_t van_id) +{ + ether_addr_copy(dst_mac, ð_hdr->d_addr); + ether_addr_copy(src_mac, ð_hdr->s_addr); + + if (vlan_enabled) { + struct vlan_hdr *vhdr = (struct vlan_hdr *)((uint8_t *)eth_hdr + + sizeof(struct ether_hdr)); + + eth_hdr->ether_type = rte_cpu_to_be_16(ETHER_TYPE_VLAN); + + vhdr->eth_proto = rte_cpu_to_be_16(ether_type); + vhdr->vlan_tci = van_id; + } else { + eth_hdr->ether_type = rte_cpu_to_be_16(ether_type); + } +} + +void +initialize_arp_header(struct arp_hdr *arp_hdr, struct ether_addr *src_mac, + struct ether_addr *dst_mac, uint32_t src_ip, uint32_t dst_ip, + uint32_t opcode) +{ + arp_hdr->arp_hrd = rte_cpu_to_be_16(ARP_HRD_ETHER); + arp_hdr->arp_pro = rte_cpu_to_be_16(ETHER_TYPE_IPv4); + arp_hdr->arp_hln = ETHER_ADDR_LEN; + arp_hdr->arp_pln = sizeof(uint32_t); + arp_hdr->arp_op = rte_cpu_to_be_16(opcode); + ether_addr_copy(src_mac, &arp_hdr->arp_data.arp_sha); + arp_hdr->arp_data.arp_sip = src_ip; + ether_addr_copy(dst_mac, &arp_hdr->arp_data.arp_tha); + arp_hdr->arp_data.arp_tip = dst_ip; +} + +uint16_t +initialize_udp_header(struct udp_hdr *udp_hdr, uint16_t src_port, + uint16_t dst_port, uint16_t pkt_data_len) +{ + uint16_t pkt_len; + + pkt_len = (uint16_t) (pkt_data_len + sizeof(struct udp_hdr)); + + udp_hdr->src_port = rte_cpu_to_be_16(src_port); + udp_hdr->dst_port = rte_cpu_to_be_16(dst_port); + udp_hdr->dgram_len = rte_cpu_to_be_16(pkt_len); + udp_hdr->dgram_cksum = 0; /* No UDP checksum. */ + + return pkt_len; +} + +uint16_t +initialize_tcp_header(struct tcp_hdr *tcp_hdr, uint16_t src_port, + uint16_t dst_port, uint16_t pkt_data_len) +{ + uint16_t pkt_len; + + pkt_len = (uint16_t) (pkt_data_len + sizeof(struct tcp_hdr)); + + memset(tcp_hdr, 0, sizeof(struct tcp_hdr)); + tcp_hdr->src_port = rte_cpu_to_be_16(src_port); + tcp_hdr->dst_port = rte_cpu_to_be_16(dst_port); + + return pkt_len; +} + +uint16_t +initialize_sctp_header(struct sctp_hdr *sctp_hdr, uint16_t src_port, + uint16_t dst_port, uint16_t pkt_data_len) +{ + uint16_t pkt_len; + + pkt_len = (uint16_t) (pkt_data_len + sizeof(struct udp_hdr)); + + sctp_hdr->src_port = rte_cpu_to_be_16(src_port); + sctp_hdr->dst_port = rte_cpu_to_be_16(dst_port); + sctp_hdr->tag = 0; + sctp_hdr->cksum = 0; /* No SCTP checksum. */ + + return pkt_len; +} + +uint16_t +initialize_ipv6_header(struct ipv6_hdr *ip_hdr, uint8_t *src_addr, + uint8_t *dst_addr, uint16_t pkt_data_len) +{ + ip_hdr->vtc_flow = 0; + ip_hdr->payload_len = pkt_data_len; + ip_hdr->proto = IPPROTO_UDP; + ip_hdr->hop_limits = IP_DEFTTL; + + rte_memcpy(ip_hdr->src_addr, src_addr, sizeof(ip_hdr->src_addr)); + rte_memcpy(ip_hdr->dst_addr, dst_addr, sizeof(ip_hdr->dst_addr)); + + return (uint16_t) (pkt_data_len + sizeof(struct ipv6_hdr)); +} + +uint16_t +initialize_ipv4_header(struct ipv4_hdr *ip_hdr, uint32_t src_addr, + uint32_t dst_addr, uint16_t pkt_data_len) +{ + uint16_t pkt_len; + unaligned_uint16_t *ptr16; + uint32_t ip_cksum; + + /* + * Initialize IP header. + */ + pkt_len = (uint16_t) (pkt_data_len + sizeof(struct ipv4_hdr)); + + ip_hdr->version_ihl = IP_VHL_DEF; + ip_hdr->type_of_service = 0; + ip_hdr->fragment_offset = 0; + ip_hdr->time_to_live = IP_DEFTTL; + ip_hdr->next_proto_id = IPPROTO_UDP; + ip_hdr->packet_id = 0; + ip_hdr->total_length = rte_cpu_to_be_16(pkt_len); + ip_hdr->src_addr = rte_cpu_to_be_32(src_addr); + ip_hdr->dst_addr = rte_cpu_to_be_32(dst_addr); + + /* + * Compute IP header checksum. + */ + ptr16 = (unaligned_uint16_t *)ip_hdr; + ip_cksum = 0; + ip_cksum += ptr16[0]; ip_cksum += ptr16[1]; + ip_cksum += ptr16[2]; ip_cksum += ptr16[3]; + ip_cksum += ptr16[4]; + ip_cksum += ptr16[6]; ip_cksum += ptr16[7]; + ip_cksum += ptr16[8]; ip_cksum += ptr16[9]; + + /* + * Reduce 32 bit checksum to 16 bits and complement it. + */ + ip_cksum = ((ip_cksum & 0xFFFF0000) >> 16) + + (ip_cksum & 0x0000FFFF); + ip_cksum %= 65536; + ip_cksum = (~ip_cksum) & 0x0000FFFF; + if (ip_cksum == 0) + ip_cksum = 0xFFFF; + ip_hdr->hdr_checksum = (uint16_t) ip_cksum; + + return pkt_len; +} + +uint16_t +initialize_ipv4_header_proto(struct ipv4_hdr *ip_hdr, uint32_t src_addr, + uint32_t dst_addr, uint16_t pkt_data_len, uint8_t proto) +{ + uint16_t pkt_len; + unaligned_uint16_t *ptr16; + uint32_t ip_cksum; + + /* + * Initialize IP header. + */ + pkt_len = (uint16_t) (pkt_data_len + sizeof(struct ipv4_hdr)); + + ip_hdr->version_ihl = IP_VHL_DEF; + ip_hdr->type_of_service = 0; + ip_hdr->fragment_offset = 0; + ip_hdr->time_to_live = IP_DEFTTL; + ip_hdr->next_proto_id = proto; + ip_hdr->packet_id = 0; + ip_hdr->total_length = rte_cpu_to_be_16(pkt_len); + ip_hdr->src_addr = rte_cpu_to_be_32(src_addr); + ip_hdr->dst_addr = rte_cpu_to_be_32(dst_addr); + + /* + * Compute IP header checksum. + */ + ptr16 = (unaligned_uint16_t *)ip_hdr; + ip_cksum = 0; + ip_cksum += ptr16[0]; ip_cksum += ptr16[1]; + ip_cksum += ptr16[2]; ip_cksum += ptr16[3]; + ip_cksum += ptr16[4]; + ip_cksum += ptr16[6]; ip_cksum += ptr16[7]; + ip_cksum += ptr16[8]; ip_cksum += ptr16[9]; + + /* + * Reduce 32 bit checksum to 16 bits and complement it. + */ + ip_cksum = ((ip_cksum & 0xFFFF0000) >> 16) + + (ip_cksum & 0x0000FFFF); + ip_cksum %= 65536; + ip_cksum = (~ip_cksum) & 0x0000FFFF; + if (ip_cksum == 0) + ip_cksum = 0xFFFF; + ip_hdr->hdr_checksum = (uint16_t) ip_cksum; + + return pkt_len; +} + +/* + * The maximum number of segments per packet is used when creating + * scattered transmit packets composed of a list of mbufs. + */ +#define RTE_MAX_SEGS_PER_PKT 255 /**< pkt.nb_segs is a 8-bit unsigned char. */ + + +int +generate_packet_burst(struct rte_mempool *mp, struct rte_mbuf **pkts_burst, + struct ether_hdr *eth_hdr, uint8_t vlan_enabled, void *ip_hdr, + uint8_t ipv4, struct udp_hdr *udp_hdr, int nb_pkt_per_burst, + uint8_t pkt_len, uint8_t nb_pkt_segs) +{ + int i, nb_pkt = 0; + size_t eth_hdr_size; + + struct rte_mbuf *pkt_seg; + struct rte_mbuf *pkt; + + for (nb_pkt = 0; nb_pkt < nb_pkt_per_burst; nb_pkt++) { + pkt = rte_pktmbuf_alloc(mp); + if (pkt == NULL) { +nomore_mbuf: + if (nb_pkt == 0) + return -1; + break; + } + + pkt->data_len = pkt_len; + pkt_seg = pkt; + for (i = 1; i < nb_pkt_segs; i++) { + pkt_seg->next = rte_pktmbuf_alloc(mp); + if (pkt_seg->next == NULL) { + pkt->nb_segs = i; + rte_pktmbuf_free(pkt); + goto nomore_mbuf; + } + pkt_seg = pkt_seg->next; + pkt_seg->data_len = pkt_len; + } + pkt_seg->next = NULL; /* Last segment of packet. */ + + /* + * Copy headers in first packet segment(s). + */ + if (vlan_enabled) + eth_hdr_size = sizeof(struct ether_hdr) + sizeof(struct vlan_hdr); + else + eth_hdr_size = sizeof(struct ether_hdr); + + copy_buf_to_pkt(eth_hdr, eth_hdr_size, pkt, 0); + + if (ipv4) { + copy_buf_to_pkt(ip_hdr, sizeof(struct ipv4_hdr), pkt, eth_hdr_size); + copy_buf_to_pkt(udp_hdr, sizeof(*udp_hdr), pkt, eth_hdr_size + + sizeof(struct ipv4_hdr)); + } else { + copy_buf_to_pkt(ip_hdr, sizeof(struct ipv6_hdr), pkt, eth_hdr_size); + copy_buf_to_pkt(udp_hdr, sizeof(*udp_hdr), pkt, eth_hdr_size + + sizeof(struct ipv6_hdr)); + } + + /* + * Complete first mbuf of packet and append it to the + * burst of packets to be transmitted. + */ + pkt->nb_segs = nb_pkt_segs; + pkt->pkt_len = pkt_len; + pkt->l2_len = eth_hdr_size; + + if (ipv4) { + pkt->vlan_tci = ETHER_TYPE_IPv4; + pkt->l3_len = sizeof(struct ipv4_hdr); + } else { + pkt->vlan_tci = ETHER_TYPE_IPv6; + pkt->l3_len = sizeof(struct ipv6_hdr); + } + + pkts_burst[nb_pkt] = pkt; + } + + return nb_pkt; +} + +int +generate_packet_burst_proto(struct rte_mempool *mp, + struct rte_mbuf **pkts_burst, + struct ether_hdr *eth_hdr, uint8_t vlan_enabled, void *ip_hdr, + uint8_t ipv4, uint8_t proto, void *proto_hdr, + int nb_pkt_per_burst, uint8_t pkt_len, uint8_t nb_pkt_segs) +{ + int i, nb_pkt = 0; + size_t eth_hdr_size; + + struct rte_mbuf *pkt_seg; + struct rte_mbuf *pkt; + + for (nb_pkt = 0; nb_pkt < nb_pkt_per_burst; nb_pkt++) { + pkt = rte_pktmbuf_alloc(mp); + if (pkt == NULL) { +nomore_mbuf: + if (nb_pkt == 0) + return -1; + break; + } + + pkt->data_len = pkt_len; + pkt_seg = pkt; + for (i = 1; i < nb_pkt_segs; i++) { + pkt_seg->next = rte_pktmbuf_alloc(mp); + if (pkt_seg->next == NULL) { + pkt->nb_segs = i; + rte_pktmbuf_free(pkt); + goto nomore_mbuf; + } + pkt_seg = pkt_seg->next; + pkt_seg->data_len = pkt_len; + } + pkt_seg->next = NULL; /* Last segment of packet. */ + + /* + * Copy headers in first packet segment(s). + */ + if (vlan_enabled) + eth_hdr_size = sizeof(struct ether_hdr) + + sizeof(struct vlan_hdr); + else + eth_hdr_size = sizeof(struct ether_hdr); + + copy_buf_to_pkt(eth_hdr, eth_hdr_size, pkt, 0); + + if (ipv4) { + copy_buf_to_pkt(ip_hdr, sizeof(struct ipv4_hdr), pkt, + eth_hdr_size); + switch (proto) { + case IPPROTO_UDP: + copy_buf_to_pkt(proto_hdr, + sizeof(struct udp_hdr), pkt, + eth_hdr_size + sizeof(struct ipv4_hdr)); + break; + case IPPROTO_TCP: + copy_buf_to_pkt(proto_hdr, + sizeof(struct tcp_hdr), pkt, + eth_hdr_size + sizeof(struct ipv4_hdr)); + break; + case IPPROTO_SCTP: + copy_buf_to_pkt(proto_hdr, + sizeof(struct sctp_hdr), pkt, + eth_hdr_size + sizeof(struct ipv4_hdr)); + break; + default: + break; + } + } else { + copy_buf_to_pkt(ip_hdr, sizeof(struct ipv6_hdr), pkt, + eth_hdr_size); + switch (proto) { + case IPPROTO_UDP: + copy_buf_to_pkt(proto_hdr, + sizeof(struct udp_hdr), pkt, + eth_hdr_size + sizeof(struct ipv6_hdr)); + break; + case IPPROTO_TCP: + copy_buf_to_pkt(proto_hdr, + sizeof(struct tcp_hdr), pkt, + eth_hdr_size + sizeof(struct ipv6_hdr)); + break; + case IPPROTO_SCTP: + copy_buf_to_pkt(proto_hdr, + sizeof(struct sctp_hdr), pkt, + eth_hdr_size + sizeof(struct ipv6_hdr)); + break; + default: + break; + } + } + + /* + * Complete first mbuf of packet and append it to the + * burst of packets to be transmitted. + */ + pkt->nb_segs = nb_pkt_segs; + pkt->pkt_len = pkt_len; + pkt->l2_len = eth_hdr_size; + + if (ipv4) { + pkt->vlan_tci = ETHER_TYPE_IPv4; + pkt->l3_len = sizeof(struct ipv4_hdr); + } else { + pkt->vlan_tci = ETHER_TYPE_IPv6; + pkt->l3_len = sizeof(struct ipv6_hdr); + } + + pkts_burst[nb_pkt] = pkt; + } + + return nb_pkt; +} diff --git a/dpdk/test/test/packet_burst_generator.h b/dpdk/test/test/packet_burst_generator.h new file mode 100644 index 00000000..3315bfa6 --- /dev/null +++ b/dpdk/test/test/packet_burst_generator.h @@ -0,0 +1,106 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef PACKET_BURST_GENERATOR_H_ +#define PACKET_BURST_GENERATOR_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <rte_mbuf.h> +#include <rte_ether.h> +#include <rte_arp.h> +#include <rte_ip.h> +#include <rte_udp.h> +#include <rte_tcp.h> +#include <rte_sctp.h> + +#define IPV4_ADDR(a, b, c, d)(((a & 0xff) << 24) | ((b & 0xff) << 16) | \ + ((c & 0xff) << 8) | (d & 0xff)) + +#define PACKET_BURST_GEN_PKT_LEN 60 +#define PACKET_BURST_GEN_PKT_LEN_128 128 + +void +initialize_eth_header(struct ether_hdr *eth_hdr, struct ether_addr *src_mac, + struct ether_addr *dst_mac, uint16_t ether_type, + uint8_t vlan_enabled, uint16_t van_id); + +void +initialize_arp_header(struct arp_hdr *arp_hdr, struct ether_addr *src_mac, + struct ether_addr *dst_mac, uint32_t src_ip, uint32_t dst_ip, + uint32_t opcode); + +uint16_t +initialize_udp_header(struct udp_hdr *udp_hdr, uint16_t src_port, + uint16_t dst_port, uint16_t pkt_data_len); + +uint16_t +initialize_tcp_header(struct tcp_hdr *tcp_hdr, uint16_t src_port, + uint16_t dst_port, uint16_t pkt_data_len); + +uint16_t +initialize_sctp_header(struct sctp_hdr *sctp_hdr, uint16_t src_port, + uint16_t dst_port, uint16_t pkt_data_len); + +uint16_t +initialize_ipv6_header(struct ipv6_hdr *ip_hdr, uint8_t *src_addr, + uint8_t *dst_addr, uint16_t pkt_data_len); + +uint16_t +initialize_ipv4_header(struct ipv4_hdr *ip_hdr, uint32_t src_addr, + uint32_t dst_addr, uint16_t pkt_data_len); + +uint16_t +initialize_ipv4_header_proto(struct ipv4_hdr *ip_hdr, uint32_t src_addr, + uint32_t dst_addr, uint16_t pkt_data_len, uint8_t proto); + +int +generate_packet_burst(struct rte_mempool *mp, struct rte_mbuf **pkts_burst, + struct ether_hdr *eth_hdr, uint8_t vlan_enabled, void *ip_hdr, + uint8_t ipv4, struct udp_hdr *udp_hdr, int nb_pkt_per_burst, + uint8_t pkt_len, uint8_t nb_pkt_segs); + +int +generate_packet_burst_proto(struct rte_mempool *mp, + struct rte_mbuf **pkts_burst, + struct ether_hdr *eth_hdr, uint8_t vlan_enabled, void *ip_hdr, + uint8_t ipv4, uint8_t proto, void *proto_hdr, + int nb_pkt_per_burst, uint8_t pkt_len, uint8_t nb_pkt_segs); + +#ifdef __cplusplus +} +#endif + +#endif /* PACKET_BURST_GENERATOR_H_ */ diff --git a/dpdk/test/test/process.h b/dpdk/test/test/process.h new file mode 100644 index 00000000..51ced122 --- /dev/null +++ b/dpdk/test/test/process.h @@ -0,0 +1,93 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _PROCESS_H_ +#define _PROCESS_H_ + +#ifdef RTE_EXEC_ENV_BSDAPP +#define self "curproc" +#define exe "file" +#else +#define self "self" +#define exe "exe" +#endif + +/* + * launches a second copy of the test process using the given argv parameters, + * which should include argv[0] as the process name. To identify in the + * subprocess the source of the call, the env_value parameter is set in the + * environment as $RTE_TEST + */ +static inline int +process_dup(const char *const argv[], int numargs, const char *env_value) +{ + int num; + char *argv_cpy[numargs + 1]; + int i, fd, status; + char path[32]; + + pid_t pid = fork(); + if (pid < 0) + return -1; + else if (pid == 0) { + /* make a copy of the arguments to be passed to exec */ + for (i = 0; i < numargs; i++) + argv_cpy[i] = strdup(argv[i]); + argv_cpy[i] = NULL; + num = numargs; + + /* close all open file descriptors, check /proc/self/fd to only + * call close on open fds. Exclude fds 0, 1 and 2*/ + for (fd = getdtablesize(); fd > 2; fd-- ) { + snprintf(path, sizeof(path), "/proc/" exe "/fd/%d", fd); + if (access(path, F_OK) == 0) + close(fd); + } + printf("Running binary with argv[]:"); + for (i = 0; i < num; i++) + printf("'%s' ", argv_cpy[i]); + printf("\n"); + + /* set the environment variable */ + if (setenv(RECURSIVE_ENV_VAR, env_value, 1) != 0) + rte_panic("Cannot export environment variable\n"); + if (execv("/proc/" self "/" exe, argv_cpy) < 0) + rte_panic("Cannot exec\n"); + } + /* parent process does a wait */ + while (wait(&status) != pid) + ; + return status; +} + +#endif /* _PROCESS_H_ */ diff --git a/dpdk/test/test/resource.c b/dpdk/test/test/resource.c new file mode 100644 index 00000000..0e2b62cd --- /dev/null +++ b/dpdk/test/test/resource.c @@ -0,0 +1,305 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016 RehiveTech. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of RehiveTech nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <string.h> +#include <errno.h> +#include <sys/queue.h> + +#include <rte_debug.h> + +#include "resource.h" + +struct resource_list resource_list = TAILQ_HEAD_INITIALIZER(resource_list); + +size_t resource_size(const struct resource *r) +{ + return r->end - r->begin; +} + +const struct resource *resource_find(const char *name) +{ + struct resource *r; + + TAILQ_FOREACH(r, &resource_list, next) { + RTE_VERIFY(r->name); + + if (!strcmp(r->name, name)) + return r; + } + + return NULL; +} + +int resource_fwrite(const struct resource *r, FILE *f) +{ + const size_t goal = resource_size(r); + size_t total = 0; + + while (total < goal) { + size_t wlen = fwrite(r->begin + total, 1, goal - total, f); + if (wlen == 0) { + perror(__func__); + return -1; + } + + total += wlen; + } + + return 0; +} + +int resource_fwrite_file(const struct resource *r, const char *fname) +{ + FILE *f; + int ret; + + f = fopen(fname, "w"); + if (f == NULL) { + perror(__func__); + return -1; + } + + ret = resource_fwrite(r, f); + fclose(f); + return ret; +} + +#ifdef RTE_APP_TEST_RESOURCE_TAR +#include <archive.h> +#include <archive_entry.h> + +static int do_copy(struct archive *r, struct archive *w) +{ + const void *buf; + size_t len; +#if ARCHIVE_VERSION_NUMBER >= 3000000 + int64_t off; +#else + off_t off; +#endif + int ret; + + while (1) { + ret = archive_read_data_block(r, &buf, &len, &off); + if (ret == ARCHIVE_RETRY) + continue; + + if (ret == ARCHIVE_EOF) + return 0; + + if (ret != ARCHIVE_OK) + return ret; + + do { + ret = archive_write_data_block(w, buf, len, off); + if (ret != ARCHIVE_OK && ret != ARCHIVE_RETRY) + return ret; + } while (ret != ARCHIVE_OK); + } +} + +int resource_untar(const struct resource *res) +{ + struct archive *r; + struct archive *w; + struct archive_entry *e; + void *p; + int flags = 0; + int ret; + + p = malloc(resource_size(res)); + if (p == NULL) + rte_panic("Failed to malloc %zu B\n", resource_size(res)); + + memcpy(p, res->begin, resource_size(res)); + + r = archive_read_new(); + if (r == NULL) { + free(p); + return -1; + } + + archive_read_support_format_all(r); + archive_read_support_filter_all(r); + + w = archive_write_disk_new(); + if (w == NULL) { + archive_read_free(r); + free(p); + return -1; + } + + flags |= ARCHIVE_EXTRACT_PERM; + flags |= ARCHIVE_EXTRACT_FFLAGS; + archive_write_disk_set_options(w, flags); + archive_write_disk_set_standard_lookup(w); + + ret = archive_read_open_memory(r, p, resource_size(res)); + if (ret != ARCHIVE_OK) + goto fail; + + while (1) { + ret = archive_read_next_header(r, &e); + if (ret == ARCHIVE_EOF) + break; + if (ret != ARCHIVE_OK) + goto fail; + + ret = archive_write_header(w, e); + if (ret == ARCHIVE_EOF) + break; + if (ret != ARCHIVE_OK) + goto fail; + + if (archive_entry_size(e) == 0) + continue; + + ret = do_copy(r, w); + if (ret != ARCHIVE_OK) + goto fail; + + ret = archive_write_finish_entry(w); + if (ret != ARCHIVE_OK) + goto fail; + } + + archive_write_free(w); + archive_read_free(r); + free(p); + return 0; + +fail: + archive_write_free(w); + archive_read_free(r); + free(p); + rte_panic("Failed: %s\n", archive_error_string(r)); + return -1; +} + +int resource_rm_by_tar(const struct resource *res) +{ + struct archive *r; + struct archive_entry *e; + void *p; + int try_again = 1; + int attempts = 0; + int ret; + + p = malloc(resource_size(res)); + if (p == NULL) + rte_panic("Failed to malloc %zu B\n", resource_size(res)); + + memcpy(p, res->begin, resource_size(res)); + + /* + * If somebody creates a file somewhere inside the extracted TAR + * hierarchy during a test the resource_rm_by_tar might loop + * infinitely. We prevent this by adding the attempts counter there. + * In normal case, max N iteration is done where N is the depth of + * the file-hierarchy. + */ + while (try_again && attempts < 10000) { + r = archive_read_new(); + if (r == NULL) { + free(p); + return -1; + } + + archive_read_support_format_all(r); + archive_read_support_filter_all(r); + + ret = archive_read_open_memory(r, p, resource_size(res)); + if (ret != ARCHIVE_OK) { + fprintf(stderr, "Failed: %s\n", + archive_error_string(r)); + goto fail; + } + + try_again = 0; + + while (1) { + ret = archive_read_next_header(r, &e); + if (ret == ARCHIVE_EOF) + break; + if (ret != ARCHIVE_OK) + goto fail; + + ret = remove(archive_entry_pathname(e)); + if (ret < 0) { + switch (errno) { + case ENOTEMPTY: + case EEXIST: + try_again = 1; + break; + + /* should not usually happen: */ + case ENOENT: + case ENOTDIR: + case EROFS: + attempts += 1; + continue; + default: + perror("Failed to remove file"); + goto fail; + } + } + } + + archive_read_free(r); + attempts += 1; + } + + if (attempts >= 10000) { + fprintf(stderr, "Failed to remove archive\n"); + free(p); + return -1; + } + + free(p); + return 0; + +fail: + archive_read_free(r); + free(p); + + rte_panic("Failed: %s\n", archive_error_string(r)); + return -1; +} + +#endif /* RTE_APP_TEST_RESOURCE_TAR */ + +void resource_register(struct resource *r) +{ + TAILQ_INSERT_TAIL(&resource_list, r, next); +} diff --git a/dpdk/test/test/resource.h b/dpdk/test/test/resource.h new file mode 100644 index 00000000..1e961221 --- /dev/null +++ b/dpdk/test/test/resource.h @@ -0,0 +1,135 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016 RehiveTech. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of RehiveTech nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RESOURCE_H_ +#define _RESOURCE_H_ + +/** + * @file + * + * Test Resource API + * + * Each test can require and use some external resources. Usually, an external + * resource is a file or a filesystem sub-hierarchy. A resource is included + * inside the test executable. + */ + +#include <sys/queue.h> +#include <stdio.h> +#include <stddef.h> + +#include <rte_eal.h> +#include <rte_common.h> + +TAILQ_HEAD(resource_list, resource); +extern struct resource_list resource_list; + +/** + * Representation of a resource. It points to the resource's binary data. + * The semantics of the binary data are defined by the target test. + */ +struct resource { + const char *name; /**< Unique name of the resource */ + const char *begin; /**< Start of resource data */ + const char *end; /**< End of resource data */ + TAILQ_ENTRY(resource) next; +}; + +/** + * @return size of the given resource + */ +size_t resource_size(const struct resource *r); + +/** + * Find a resource by name in the global list of resources. + */ +const struct resource *resource_find(const char *name); + +/** + * Write the raw data of the resource to the given file. + * @return 0 on success + */ +int resource_fwrite(const struct resource *r, FILE *f); + +/** + * Write the raw data of the resource to the given file given by name. + * The name is relative to the current working directory. + * @return 0 on success + */ +int resource_fwrite_file(const struct resource *r, const char *fname); + +/** + * Treat the given resource as a tar archive. Extract + * the archive to the current directory. + */ +int resource_untar(const struct resource *res); + +/** + * Treat the given resource as a tar archive. Remove + * all files (related to the current directory) listed + * in the tar archive. + */ +int resource_rm_by_tar(const struct resource *res); + +/** + * Register a resource in the global list of resources. + * Not intended for direct use, please check the REGISTER_RESOURCE + * macro. + */ +void resource_register(struct resource *r); + +/** + * Definition of a resource linked externally (by means of the used toolchain). + * Only the base name of the resource is expected. The name refers to the + * linked pointers beg_<name> and end_<name> provided externally. + */ +#define REGISTER_LINKED_RESOURCE(n) \ +extern const char beg_ ##n; \ +extern const char end_ ##n; \ +REGISTER_RESOURCE(n, &beg_ ##n, &end_ ##n) \ + +/** + * Definition of a resource described by its name, and pointers begin, end. + */ +#define REGISTER_RESOURCE(n, b, e) \ +static struct resource linkres_ ##n = { \ + .name = RTE_STR(n), \ + .begin = b, \ + .end = e, \ +}; \ +static void __attribute__((constructor, used)) resinitfn_ ##n(void) \ +{ \ + resource_register(&linkres_ ##n); \ +} + +#endif diff --git a/dpdk/test/test/test.c b/dpdk/test/test/test.c new file mode 100644 index 00000000..fe41d404 --- /dev/null +++ b/dpdk/test/test/test.c @@ -0,0 +1,250 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <string.h> +#include <stdio.h> +#include <stdint.h> +#include <stdarg.h> +#include <stdlib.h> +#include <errno.h> +#include <termios.h> +#include <ctype.h> +#include <sys/queue.h> + +#ifdef RTE_LIBRTE_CMDLINE +#include <cmdline_rdline.h> +#include <cmdline_parse.h> +#include <cmdline_socket.h> +#include <cmdline.h> +extern cmdline_parse_ctx_t main_ctx[]; +#endif + +#include <rte_memory.h> +#include <rte_eal.h> +#include <rte_cycles.h> +#include <rte_log.h> +#include <rte_string_fns.h> +#ifdef RTE_LIBRTE_TIMER +#include <rte_timer.h> +#endif + +#include "test.h" + +#define RTE_LOGTYPE_APP RTE_LOGTYPE_USER1 + +const char *prgname; /* to be set to argv[0] */ + +static const char *recursive_call; /* used in linuxapp for MP and other tests */ + +static int +no_action(void){ return 0; } + +static int +do_recursive_call(void) +{ + unsigned i; + struct { + const char *env_var; + int (*action_fn)(void); + } actions[] = { + { "run_secondary_instances", test_mp_secondary }, + { "test_missing_c_flag", no_action }, + { "test_master_lcore_flag", no_action }, + { "test_invalid_n_flag", no_action }, + { "test_no_hpet_flag", no_action }, + { "test_whitelist_flag", no_action }, + { "test_invalid_b_flag", no_action }, + { "test_invalid_vdev_flag", no_action }, + { "test_invalid_r_flag", no_action }, + { "test_misc_flags", no_action }, + { "test_memory_flags", no_action }, + { "test_file_prefix", no_action }, + { "test_no_huge_flag", no_action }, + }; + + if (recursive_call == NULL) + return -1; + for (i = 0; i < sizeof(actions)/sizeof(actions[0]); i++) { + if (strcmp(actions[i].env_var, recursive_call) == 0) + return (actions[i].action_fn)(); + } + printf("ERROR - missing action to take for %s\n", recursive_call); + return -1; +} + +int +main(int argc, char **argv) +{ +#ifdef RTE_LIBRTE_CMDLINE + struct cmdline *cl; +#endif + int ret; + + ret = rte_eal_init(argc, argv); + if (ret < 0) + return -1; + +#ifdef RTE_LIBRTE_TIMER + rte_timer_subsystem_init(); +#endif + + if (commands_init() < 0) + return -1; + + argv += ret; + + prgname = argv[0]; + + if ((recursive_call = getenv(RECURSIVE_ENV_VAR)) != NULL) + return do_recursive_call(); + +#ifdef RTE_LIBEAL_USE_HPET + if (rte_eal_hpet_init(1) < 0) +#endif + RTE_LOG(INFO, APP, + "HPET is not enabled, using TSC as default timer\n"); + + +#ifdef RTE_LIBRTE_CMDLINE + cl = cmdline_stdin_new(main_ctx, "RTE>>"); + if (cl == NULL) { + return -1; + } + cmdline_interact(cl); + cmdline_stdin_exit(cl); +#endif + + return 0; +} + + +int +unit_test_suite_runner(struct unit_test_suite *suite) +{ + int test_success; + unsigned int total = 0, executed = 0, skipped = 0; + unsigned int succeeded = 0, failed = 0, unsupported = 0; + const char *status; + + if (suite->suite_name) { + printf(" + ------------------------------------------------------- +\n"); + printf(" + Test Suite : %s\n", suite->suite_name); + } + + if (suite->setup) + if (suite->setup() != 0) { + /* + * setup failed, so count all enabled tests and mark + * them as failed + */ + while (suite->unit_test_cases[total].testcase) { + if (!suite->unit_test_cases[total].enabled) + skipped++; + else + failed++; + total++; + } + goto suite_summary; + } + + printf(" + ------------------------------------------------------- +\n"); + + while (suite->unit_test_cases[total].testcase) { + if (!suite->unit_test_cases[total].enabled) { + skipped++; + total++; + continue; + } else { + executed++; + } + + /* run test case setup */ + if (suite->unit_test_cases[total].setup) + test_success = suite->unit_test_cases[total].setup(); + else + test_success = TEST_SUCCESS; + + if (test_success == TEST_SUCCESS) { + /* run the test case */ + test_success = suite->unit_test_cases[total].testcase(); + if (test_success == TEST_SUCCESS) + succeeded++; + else if (test_success == -ENOTSUP) + unsupported++; + else + failed++; + } else if (test_success == -ENOTSUP) { + unsupported++; + } else { + failed++; + } + + /* run the test case teardown */ + if (suite->unit_test_cases[total].teardown) + suite->unit_test_cases[total].teardown(); + + if (test_success == TEST_SUCCESS) + status = "succeeded"; + else if (test_success == -ENOTSUP) + status = "unsupported"; + else + status = "failed"; + + printf(" + TestCase [%2d] : %s %s\n", total, + suite->unit_test_cases[total].name, status); + + total++; + } + + /* Run test suite teardown */ + if (suite->teardown) + suite->teardown(); + + goto suite_summary; + +suite_summary: + printf(" + ------------------------------------------------------- +\n"); + printf(" + Test Suite Summary \n"); + printf(" + Tests Total : %2d\n", total); + printf(" + Tests Skipped : %2d\n", skipped); + printf(" + Tests Executed : %2d\n", executed); + printf(" + Tests Unsupported: %2d\n", unsupported); + printf(" + Tests Passed : %2d\n", succeeded); + printf(" + Tests Failed : %2d\n", failed); + printf(" + ------------------------------------------------------- +\n"); + + if (failed) + return -1; + + return 0; +} diff --git a/dpdk/test/test/test.h b/dpdk/test/test/test.h new file mode 100644 index 00000000..08ffe949 --- /dev/null +++ b/dpdk/test/test/test.h @@ -0,0 +1,259 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _TEST_H_ +#define _TEST_H_ + +#include <stddef.h> +#include <sys/queue.h> + +#include <rte_common.h> +#include <rte_log.h> + +#define TEST_SUCCESS (0) +#define TEST_FAILED (-1) + +/* Before including test.h file you can define + * TEST_TRACE_FAILURE(_file, _line, _func) macro to better trace/debug test + * failures. Mostly useful in test development phase. */ +#ifndef TEST_TRACE_FAILURE +# define TEST_TRACE_FAILURE(_file, _line, _func) +#endif + +#define TEST_ASSERT(cond, msg, ...) do { \ + if (!(cond)) { \ + printf("TestCase %s() line %d failed: " \ + msg "\n", __func__, __LINE__, ##__VA_ARGS__); \ + TEST_TRACE_FAILURE(__FILE__, __LINE__, __func__); \ + return TEST_FAILED; \ + } \ +} while (0) + +#define TEST_ASSERT_EQUAL(a, b, msg, ...) do { \ + if (!(a == b)) { \ + printf("TestCase %s() line %d failed: " \ + msg "\n", __func__, __LINE__, ##__VA_ARGS__); \ + TEST_TRACE_FAILURE(__FILE__, __LINE__, __func__); \ + return TEST_FAILED; \ + } \ +} while (0) + +/* Compare two buffers (length in bytes) */ +#define TEST_ASSERT_BUFFERS_ARE_EQUAL(a, b, len, msg, ...) do { \ + if (memcmp(a, b, len)) { \ + printf("TestCase %s() line %d failed: " \ + msg "\n", __func__, __LINE__, ##__VA_ARGS__); \ + TEST_TRACE_FAILURE(__FILE__, __LINE__, __func__); \ + return TEST_FAILED; \ + } \ +} while (0) + +/* Compare two buffers with offset (length and offset in bytes) */ +#define TEST_ASSERT_BUFFERS_ARE_EQUAL_OFFSET(a, b, len, off, msg, ...) do { \ + const uint8_t *_a_with_off = (const uint8_t *)a + off; \ + const uint8_t *_b_with_off = (const uint8_t *)b + off; \ + TEST_ASSERT_BUFFERS_ARE_EQUAL(_a_with_off, _b_with_off, len, msg); \ +} while (0) + +/* Compare two buffers (length in bits) */ +#define TEST_ASSERT_BUFFERS_ARE_EQUAL_BIT(a, b, len, msg, ...) do { \ + uint8_t _last_byte_a, _last_byte_b; \ + uint8_t _last_byte_mask, _last_byte_bits; \ + TEST_ASSERT_BUFFERS_ARE_EQUAL(a, b, (len >> 3), msg); \ + if (len % 8) { \ + _last_byte_bits = len % 8; \ + _last_byte_mask = ~((1 << (8 - _last_byte_bits)) - 1); \ + _last_byte_a = ((const uint8_t *)a)[len >> 3]; \ + _last_byte_b = ((const uint8_t *)b)[len >> 3]; \ + _last_byte_a &= _last_byte_mask; \ + _last_byte_b &= _last_byte_mask; \ + if (_last_byte_a != _last_byte_b) { \ + printf("TestCase %s() line %d failed: " \ + msg "\n", __func__, __LINE__, ##__VA_ARGS__);\ + TEST_TRACE_FAILURE(__FILE__, __LINE__, __func__); \ + return TEST_FAILED; \ + } \ + } \ +} while (0) + +/* Compare two buffers with offset (length and offset in bits) */ +#define TEST_ASSERT_BUFFERS_ARE_EQUAL_BIT_OFFSET(a, b, len, off, msg, ...) do { \ + uint8_t _first_byte_a, _first_byte_b; \ + uint8_t _first_byte_mask, _first_byte_bits; \ + uint32_t _len_without_first_byte = (off % 8) ? \ + len - (8 - (off % 8)) : \ + len; \ + uint32_t _off_in_bytes = (off % 8) ? (off >> 3) + 1 : (off >> 3); \ + const uint8_t *_a_with_off = (const uint8_t *)a + _off_in_bytes; \ + const uint8_t *_b_with_off = (const uint8_t *)b + _off_in_bytes; \ + TEST_ASSERT_BUFFERS_ARE_EQUAL_BIT(_a_with_off, _b_with_off, \ + _len_without_first_byte, msg); \ + if (off % 8) { \ + _first_byte_bits = 8 - (off % 8); \ + _first_byte_mask = (1 << _first_byte_bits) - 1; \ + _first_byte_a = *(_a_with_off - 1); \ + _first_byte_b = *(_b_with_off - 1); \ + _first_byte_a &= _first_byte_mask; \ + _first_byte_b &= _first_byte_mask; \ + if (_first_byte_a != _first_byte_b) { \ + printf("TestCase %s() line %d failed: " \ + msg "\n", __func__, __LINE__, ##__VA_ARGS__); \ + TEST_TRACE_FAILURE(__FILE__, __LINE__, __func__); \ + return TEST_FAILED; \ + } \ + } \ +} while (0) + +#define TEST_ASSERT_NOT_EQUAL(a, b, msg, ...) do { \ + if (!(a != b)) { \ + printf("TestCase %s() line %d failed: " \ + msg "\n", __func__, __LINE__, ##__VA_ARGS__); \ + TEST_TRACE_FAILURE(__FILE__, __LINE__, __func__); \ + return TEST_FAILED; \ + } \ +} while (0) + +#define TEST_ASSERT_SUCCESS(val, msg, ...) do { \ + typeof(val) _val = (val); \ + if (!(_val == 0)) { \ + printf("TestCase %s() line %d failed (err %d): " \ + msg "\n", __func__, __LINE__, _val, \ + ##__VA_ARGS__); \ + TEST_TRACE_FAILURE(__FILE__, __LINE__, __func__); \ + return TEST_FAILED; \ + } \ +} while (0) + +#define TEST_ASSERT_FAIL(val, msg, ...) do { \ + if (!(val != 0)) { \ + printf("TestCase %s() line %d failed: " \ + msg "\n", __func__, __LINE__, ##__VA_ARGS__); \ + TEST_TRACE_FAILURE(__FILE__, __LINE__, __func__); \ + return TEST_FAILED; \ + } \ +} while (0) + +#define TEST_ASSERT_NULL(val, msg, ...) do { \ + if (!(val == NULL)) { \ + printf("TestCase %s() line %d failed: " \ + msg "\n", __func__, __LINE__, ##__VA_ARGS__); \ + TEST_TRACE_FAILURE(__FILE__, __LINE__, __func__); \ + return TEST_FAILED; \ + } \ +} while (0) + +#define TEST_ASSERT_NOT_NULL(val, msg, ...) do { \ + if (!(val != NULL)) { \ + printf("TestCase %s() line %d failed: " \ + msg "\n", __func__, __LINE__, ##__VA_ARGS__); \ + TEST_TRACE_FAILURE(__FILE__, __LINE__, __func__); \ + return TEST_FAILED; \ + } \ +} while (0) + +struct unit_test_case { + int (*setup)(void); + void (*teardown)(void); + int (*testcase)(void); + const char *name; + unsigned enabled; +}; + +#define TEST_CASE(fn) { NULL, NULL, fn, #fn, 1 } + +#define TEST_CASE_NAMED(name, fn) { NULL, NULL, fn, name, 1 } + +#define TEST_CASE_ST(setup, teardown, testcase) \ + { setup, teardown, testcase, #testcase, 1 } + + +#define TEST_CASE_DISABLED(fn) { NULL, NULL, fn, #fn, 0 } + +#define TEST_CASE_ST_DISABLED(setup, teardown, testcase) \ + { setup, teardown, testcase, #testcase, 0 } + +#define TEST_CASES_END() { NULL, NULL, NULL, NULL, 0 } + +#if RTE_LOG_LEVEL >= RTE_LOG_DEBUG +#define TEST_HEXDUMP(file, title, buf, len) rte_hexdump(file, title, buf, len) +#else +#define TEST_HEXDUMP(file, title, buf, len) do {} while (0) +#endif + +struct unit_test_suite { + const char *suite_name; + int (*setup)(void); + void (*teardown)(void); + struct unit_test_case unit_test_cases[]; +}; + +int unit_test_suite_runner(struct unit_test_suite *suite); + +#define RECURSIVE_ENV_VAR "RTE_TEST_RECURSIVE" + +#include <cmdline_parse.h> +#include <cmdline_parse_string.h> + +extern const char *prgname; + +int commands_init(void); + +int test_mp_secondary(void); + +int test_set_rxtx_conf(cmdline_fixed_string_t mode); +int test_set_rxtx_anchor(cmdline_fixed_string_t type); +int test_set_rxtx_sc(cmdline_fixed_string_t type); + +typedef int (test_callback)(void); +TAILQ_HEAD(test_commands_list, test_command); +struct test_command { + TAILQ_ENTRY(test_command) next; + const char *command; + test_callback *callback; +}; + +void add_test_command(struct test_command *t); + +/* Register a test function with its command string */ +#define REGISTER_TEST_COMMAND(cmd, func) \ + static struct test_command test_struct_##cmd = { \ + .command = RTE_STR(cmd), \ + .callback = func, \ + }; \ + static void __attribute__((constructor, used)) \ + test_register_##cmd(void) \ + { \ + add_test_command(&test_struct_##cmd); \ + } + +#endif diff --git a/dpdk/test/test/test_acl.c b/dpdk/test/test/test_acl.c new file mode 100644 index 00000000..c6b511fb --- /dev/null +++ b/dpdk/test/test/test_acl.c @@ -0,0 +1,1652 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <string.h> +#include <errno.h> + +#include "test.h" + +#include <rte_string_fns.h> +#include <rte_mbuf.h> +#include <rte_byteorder.h> +#include <rte_ip.h> +#include <rte_acl.h> +#include <rte_common.h> + +#include "test_acl.h" + +#define BIT_SIZEOF(x) (sizeof(x) * CHAR_BIT) + +#define LEN RTE_ACL_MAX_CATEGORIES + +RTE_ACL_RULE_DEF(acl_ipv4vlan_rule, RTE_ACL_IPV4VLAN_NUM_FIELDS); + +struct rte_acl_param acl_param = { + .name = "acl_ctx", + .socket_id = SOCKET_ID_ANY, + .rule_size = RTE_ACL_IPV4VLAN_RULE_SZ, + .max_rule_num = 0x30000, +}; + +struct rte_acl_ipv4vlan_rule acl_rule = { + .data = { .priority = 1, .category_mask = 0xff }, + .src_port_low = 0, + .src_port_high = UINT16_MAX, + .dst_port_low = 0, + .dst_port_high = UINT16_MAX, +}; + +const uint32_t ipv4_7tuple_layout[RTE_ACL_IPV4VLAN_NUM] = { + offsetof(struct ipv4_7tuple, proto), + offsetof(struct ipv4_7tuple, vlan), + offsetof(struct ipv4_7tuple, ip_src), + offsetof(struct ipv4_7tuple, ip_dst), + offsetof(struct ipv4_7tuple, port_src), +}; + + +/* byteswap to cpu or network order */ +static void +bswap_test_data(struct ipv4_7tuple *data, int len, int to_be) +{ + int i; + + for (i = 0; i < len; i++) { + + if (to_be) { + /* swap all bytes so that they are in network order */ + data[i].ip_dst = rte_cpu_to_be_32(data[i].ip_dst); + data[i].ip_src = rte_cpu_to_be_32(data[i].ip_src); + data[i].port_dst = rte_cpu_to_be_16(data[i].port_dst); + data[i].port_src = rte_cpu_to_be_16(data[i].port_src); + data[i].vlan = rte_cpu_to_be_16(data[i].vlan); + data[i].domain = rte_cpu_to_be_16(data[i].domain); + } else { + data[i].ip_dst = rte_be_to_cpu_32(data[i].ip_dst); + data[i].ip_src = rte_be_to_cpu_32(data[i].ip_src); + data[i].port_dst = rte_be_to_cpu_16(data[i].port_dst); + data[i].port_src = rte_be_to_cpu_16(data[i].port_src); + data[i].vlan = rte_be_to_cpu_16(data[i].vlan); + data[i].domain = rte_be_to_cpu_16(data[i].domain); + } + } +} + +static int +acl_ipv4vlan_check_rule(const struct rte_acl_ipv4vlan_rule *rule) +{ + if (rule->src_port_low > rule->src_port_high || + rule->dst_port_low > rule->dst_port_high || + rule->src_mask_len > BIT_SIZEOF(rule->src_addr) || + rule->dst_mask_len > BIT_SIZEOF(rule->dst_addr)) + return -EINVAL; + return 0; +} + +static void +acl_ipv4vlan_convert_rule(const struct rte_acl_ipv4vlan_rule *ri, + struct acl_ipv4vlan_rule *ro) +{ + ro->data = ri->data; + + ro->field[RTE_ACL_IPV4VLAN_PROTO_FIELD].value.u8 = ri->proto; + ro->field[RTE_ACL_IPV4VLAN_VLAN1_FIELD].value.u16 = ri->vlan; + ro->field[RTE_ACL_IPV4VLAN_VLAN2_FIELD].value.u16 = ri->domain; + ro->field[RTE_ACL_IPV4VLAN_SRC_FIELD].value.u32 = ri->src_addr; + ro->field[RTE_ACL_IPV4VLAN_DST_FIELD].value.u32 = ri->dst_addr; + ro->field[RTE_ACL_IPV4VLAN_SRCP_FIELD].value.u16 = ri->src_port_low; + ro->field[RTE_ACL_IPV4VLAN_DSTP_FIELD].value.u16 = ri->dst_port_low; + + ro->field[RTE_ACL_IPV4VLAN_PROTO_FIELD].mask_range.u8 = ri->proto_mask; + ro->field[RTE_ACL_IPV4VLAN_VLAN1_FIELD].mask_range.u16 = ri->vlan_mask; + ro->field[RTE_ACL_IPV4VLAN_VLAN2_FIELD].mask_range.u16 = + ri->domain_mask; + ro->field[RTE_ACL_IPV4VLAN_SRC_FIELD].mask_range.u32 = + ri->src_mask_len; + ro->field[RTE_ACL_IPV4VLAN_DST_FIELD].mask_range.u32 = ri->dst_mask_len; + ro->field[RTE_ACL_IPV4VLAN_SRCP_FIELD].mask_range.u16 = + ri->src_port_high; + ro->field[RTE_ACL_IPV4VLAN_DSTP_FIELD].mask_range.u16 = + ri->dst_port_high; +} + +/* + * Add ipv4vlan rules to an existing ACL context. + * This function is not multi-thread safe. + * + * @param ctx + * ACL context to add patterns to. + * @param rules + * Array of rules to add to the ACL context. + * Note that all fields in rte_acl_ipv4vlan_rule structures are expected + * to be in host byte order. + * @param num + * Number of elements in the input array of rules. + * @return + * - -ENOMEM if there is no space in the ACL context for these rules. + * - -EINVAL if the parameters are invalid. + * - Zero if operation completed successfully. + */ +static int +rte_acl_ipv4vlan_add_rules(struct rte_acl_ctx *ctx, + const struct rte_acl_ipv4vlan_rule *rules, + uint32_t num) +{ + int32_t rc; + uint32_t i; + struct acl_ipv4vlan_rule rv; + + if (ctx == NULL || rules == NULL) + return -EINVAL; + + /* check input rules. */ + for (i = 0; i != num; i++) { + rc = acl_ipv4vlan_check_rule(rules + i); + if (rc != 0) { + RTE_LOG(ERR, ACL, "%s: rule #%u is invalid\n", + __func__, i + 1); + return rc; + } + } + + /* perform conversion to the internal format and add to the context. */ + for (i = 0, rc = 0; i != num && rc == 0; i++) { + acl_ipv4vlan_convert_rule(rules + i, &rv); + rc = rte_acl_add_rules(ctx, (struct rte_acl_rule *)&rv, 1); + } + + return rc; +} + +static void +acl_ipv4vlan_config(struct rte_acl_config *cfg, + const uint32_t layout[RTE_ACL_IPV4VLAN_NUM], + uint32_t num_categories) +{ + static const struct rte_acl_field_def + ipv4_defs[RTE_ACL_IPV4VLAN_NUM_FIELDS] = { + { + .type = RTE_ACL_FIELD_TYPE_BITMASK, + .size = sizeof(uint8_t), + .field_index = RTE_ACL_IPV4VLAN_PROTO_FIELD, + .input_index = RTE_ACL_IPV4VLAN_PROTO, + }, + { + .type = RTE_ACL_FIELD_TYPE_BITMASK, + .size = sizeof(uint16_t), + .field_index = RTE_ACL_IPV4VLAN_VLAN1_FIELD, + .input_index = RTE_ACL_IPV4VLAN_VLAN, + }, + { + .type = RTE_ACL_FIELD_TYPE_BITMASK, + .size = sizeof(uint16_t), + .field_index = RTE_ACL_IPV4VLAN_VLAN2_FIELD, + .input_index = RTE_ACL_IPV4VLAN_VLAN, + }, + { + .type = RTE_ACL_FIELD_TYPE_MASK, + .size = sizeof(uint32_t), + .field_index = RTE_ACL_IPV4VLAN_SRC_FIELD, + .input_index = RTE_ACL_IPV4VLAN_SRC, + }, + { + .type = RTE_ACL_FIELD_TYPE_MASK, + .size = sizeof(uint32_t), + .field_index = RTE_ACL_IPV4VLAN_DST_FIELD, + .input_index = RTE_ACL_IPV4VLAN_DST, + }, + { + .type = RTE_ACL_FIELD_TYPE_RANGE, + .size = sizeof(uint16_t), + .field_index = RTE_ACL_IPV4VLAN_SRCP_FIELD, + .input_index = RTE_ACL_IPV4VLAN_PORTS, + }, + { + .type = RTE_ACL_FIELD_TYPE_RANGE, + .size = sizeof(uint16_t), + .field_index = RTE_ACL_IPV4VLAN_DSTP_FIELD, + .input_index = RTE_ACL_IPV4VLAN_PORTS, + }, + }; + + memcpy(&cfg->defs, ipv4_defs, sizeof(ipv4_defs)); + cfg->num_fields = RTE_DIM(ipv4_defs); + + cfg->defs[RTE_ACL_IPV4VLAN_PROTO_FIELD].offset = + layout[RTE_ACL_IPV4VLAN_PROTO]; + cfg->defs[RTE_ACL_IPV4VLAN_VLAN1_FIELD].offset = + layout[RTE_ACL_IPV4VLAN_VLAN]; + cfg->defs[RTE_ACL_IPV4VLAN_VLAN2_FIELD].offset = + layout[RTE_ACL_IPV4VLAN_VLAN] + + cfg->defs[RTE_ACL_IPV4VLAN_VLAN1_FIELD].size; + cfg->defs[RTE_ACL_IPV4VLAN_SRC_FIELD].offset = + layout[RTE_ACL_IPV4VLAN_SRC]; + cfg->defs[RTE_ACL_IPV4VLAN_DST_FIELD].offset = + layout[RTE_ACL_IPV4VLAN_DST]; + cfg->defs[RTE_ACL_IPV4VLAN_SRCP_FIELD].offset = + layout[RTE_ACL_IPV4VLAN_PORTS]; + cfg->defs[RTE_ACL_IPV4VLAN_DSTP_FIELD].offset = + layout[RTE_ACL_IPV4VLAN_PORTS] + + cfg->defs[RTE_ACL_IPV4VLAN_SRCP_FIELD].size; + + cfg->num_categories = num_categories; +} + +/* + * Analyze set of ipv4vlan rules and build required internal + * run-time structures. + * This function is not multi-thread safe. + * + * @param ctx + * ACL context to build. + * @param layout + * Layout of input data to search through. + * @param num_categories + * Maximum number of categories to use in that build. + * @return + * - -ENOMEM if couldn't allocate enough memory. + * - -EINVAL if the parameters are invalid. + * - Negative error code if operation failed. + * - Zero if operation completed successfully. + */ +static int +rte_acl_ipv4vlan_build(struct rte_acl_ctx *ctx, + const uint32_t layout[RTE_ACL_IPV4VLAN_NUM], + uint32_t num_categories) +{ + struct rte_acl_config cfg; + + if (ctx == NULL || layout == NULL) + return -EINVAL; + + memset(&cfg, 0, sizeof(cfg)); + acl_ipv4vlan_config(&cfg, layout, num_categories); + return rte_acl_build(ctx, &cfg); +} + +/* + * Test scalar and SSE ACL lookup. + */ +static int +test_classify_run(struct rte_acl_ctx *acx) +{ + int ret, i; + uint32_t result, count; + uint32_t results[RTE_DIM(acl_test_data) * RTE_ACL_MAX_CATEGORIES]; + const uint8_t *data[RTE_DIM(acl_test_data)]; + + /* swap all bytes in the data to network order */ + bswap_test_data(acl_test_data, RTE_DIM(acl_test_data), 1); + + /* store pointers to test data */ + for (i = 0; i < (int) RTE_DIM(acl_test_data); i++) + data[i] = (uint8_t *)&acl_test_data[i]; + + /** + * these will run quite a few times, it's necessary to test code paths + * from num=0 to num>8 + */ + for (count = 0; count <= RTE_DIM(acl_test_data); count++) { + ret = rte_acl_classify(acx, data, results, + count, RTE_ACL_MAX_CATEGORIES); + if (ret != 0) { + printf("Line %i: SSE classify failed!\n", __LINE__); + goto err; + } + + /* check if we allow everything we should allow */ + for (i = 0; i < (int) count; i++) { + result = + results[i * RTE_ACL_MAX_CATEGORIES + ACL_ALLOW]; + if (result != acl_test_data[i].allow) { + printf("Line %i: Error in allow results at %i " + "(expected %"PRIu32" got %"PRIu32")!\n", + __LINE__, i, acl_test_data[i].allow, + result); + ret = -EINVAL; + goto err; + } + } + + /* check if we deny everything we should deny */ + for (i = 0; i < (int) count; i++) { + result = results[i * RTE_ACL_MAX_CATEGORIES + ACL_DENY]; + if (result != acl_test_data[i].deny) { + printf("Line %i: Error in deny results at %i " + "(expected %"PRIu32" got %"PRIu32")!\n", + __LINE__, i, acl_test_data[i].deny, + result); + ret = -EINVAL; + goto err; + } + } + } + + /* make a quick check for scalar */ + ret = rte_acl_classify_alg(acx, data, results, + RTE_DIM(acl_test_data), RTE_ACL_MAX_CATEGORIES, + RTE_ACL_CLASSIFY_SCALAR); + if (ret != 0) { + printf("Line %i: scalar classify failed!\n", __LINE__); + goto err; + } + + /* check if we allow everything we should allow */ + for (i = 0; i < (int) RTE_DIM(acl_test_data); i++) { + result = results[i * RTE_ACL_MAX_CATEGORIES + ACL_ALLOW]; + if (result != acl_test_data[i].allow) { + printf("Line %i: Error in allow results at %i " + "(expected %"PRIu32" got %"PRIu32")!\n", + __LINE__, i, acl_test_data[i].allow, + result); + ret = -EINVAL; + goto err; + } + } + + /* check if we deny everything we should deny */ + for (i = 0; i < (int) RTE_DIM(acl_test_data); i++) { + result = results[i * RTE_ACL_MAX_CATEGORIES + ACL_DENY]; + if (result != acl_test_data[i].deny) { + printf("Line %i: Error in deny results at %i " + "(expected %"PRIu32" got %"PRIu32")!\n", + __LINE__, i, acl_test_data[i].deny, + result); + ret = -EINVAL; + goto err; + } + } + + ret = 0; + +err: + /* swap data back to cpu order so that next time tests don't fail */ + bswap_test_data(acl_test_data, RTE_DIM(acl_test_data), 0); + return ret; +} + +static int +test_classify_buid(struct rte_acl_ctx *acx, + const struct rte_acl_ipv4vlan_rule *rules, uint32_t num) +{ + int ret; + + /* add rules to the context */ + ret = rte_acl_ipv4vlan_add_rules(acx, rules, num); + if (ret != 0) { + printf("Line %i: Adding rules to ACL context failed!\n", + __LINE__); + return ret; + } + + /* try building the context */ + ret = rte_acl_ipv4vlan_build(acx, ipv4_7tuple_layout, + RTE_ACL_MAX_CATEGORIES); + if (ret != 0) { + printf("Line %i: Building ACL context failed!\n", __LINE__); + return ret; + } + + return 0; +} + +#define TEST_CLASSIFY_ITER 4 + +/* + * Test scalar and SSE ACL lookup. + */ +static int +test_classify(void) +{ + struct rte_acl_ctx *acx; + int i, ret; + + acx = rte_acl_create(&acl_param); + if (acx == NULL) { + printf("Line %i: Error creating ACL context!\n", __LINE__); + return -1; + } + + ret = 0; + for (i = 0; i != TEST_CLASSIFY_ITER; i++) { + + if ((i & 1) == 0) + rte_acl_reset(acx); + else + rte_acl_reset_rules(acx); + + ret = test_classify_buid(acx, acl_test_rules, + RTE_DIM(acl_test_rules)); + if (ret != 0) { + printf("Line %i, iter: %d: " + "Adding rules to ACL context failed!\n", + __LINE__, i); + break; + } + + ret = test_classify_run(acx); + if (ret != 0) { + printf("Line %i, iter: %d: %s failed!\n", + __LINE__, i, __func__); + break; + } + + /* reset rules and make sure that classify still works ok. */ + rte_acl_reset_rules(acx); + ret = test_classify_run(acx); + if (ret != 0) { + printf("Line %i, iter: %d: %s failed!\n", + __LINE__, i, __func__); + break; + } + } + + rte_acl_free(acx); + return ret; +} + +static int +test_build_ports_range(void) +{ + static const struct rte_acl_ipv4vlan_rule test_rules[] = { + { + /* match all packets. */ + .data = { + .userdata = 1, + .category_mask = ACL_ALLOW_MASK, + .priority = 101, + }, + .src_port_low = 0, + .src_port_high = UINT16_MAX, + .dst_port_low = 0, + .dst_port_high = UINT16_MAX, + }, + { + /* match all packets with dst ports [54-65280]. */ + .data = { + .userdata = 2, + .category_mask = ACL_ALLOW_MASK, + .priority = 102, + }, + .src_port_low = 0, + .src_port_high = UINT16_MAX, + .dst_port_low = 54, + .dst_port_high = 65280, + }, + { + /* match all packets with dst ports [0-52]. */ + .data = { + .userdata = 3, + .category_mask = ACL_ALLOW_MASK, + .priority = 103, + }, + .src_port_low = 0, + .src_port_high = UINT16_MAX, + .dst_port_low = 0, + .dst_port_high = 52, + }, + { + /* match all packets with dst ports [53]. */ + .data = { + .userdata = 4, + .category_mask = ACL_ALLOW_MASK, + .priority = 99, + }, + .src_port_low = 0, + .src_port_high = UINT16_MAX, + .dst_port_low = 53, + .dst_port_high = 53, + }, + { + /* match all packets with dst ports [65279-65535]. */ + .data = { + .userdata = 5, + .category_mask = ACL_ALLOW_MASK, + .priority = 98, + }, + .src_port_low = 0, + .src_port_high = UINT16_MAX, + .dst_port_low = 65279, + .dst_port_high = UINT16_MAX, + }, + }; + + static struct ipv4_7tuple test_data[] = { + { + .proto = 6, + .ip_src = IPv4(10, 1, 1, 1), + .ip_dst = IPv4(192, 168, 0, 33), + .port_dst = 53, + .allow = 1, + }, + { + .proto = 6, + .ip_src = IPv4(127, 84, 33, 1), + .ip_dst = IPv4(1, 2, 3, 4), + .port_dst = 65281, + .allow = 1, + }, + }; + + struct rte_acl_ctx *acx; + int32_t ret, i, j; + uint32_t results[RTE_DIM(test_data)]; + const uint8_t *data[RTE_DIM(test_data)]; + + acx = rte_acl_create(&acl_param); + if (acx == NULL) { + printf("Line %i: Error creating ACL context!\n", __LINE__); + return -1; + } + + /* swap all bytes in the data to network order */ + bswap_test_data(test_data, RTE_DIM(test_data), 1); + + /* store pointers to test data */ + for (i = 0; i != RTE_DIM(test_data); i++) + data[i] = (uint8_t *)&test_data[i]; + + for (i = 0; i != RTE_DIM(test_rules); i++) { + rte_acl_reset(acx); + ret = test_classify_buid(acx, test_rules, i + 1); + if (ret != 0) { + printf("Line %i, iter: %d: " + "Adding rules to ACL context failed!\n", + __LINE__, i); + break; + } + ret = rte_acl_classify(acx, data, results, + RTE_DIM(data), 1); + if (ret != 0) { + printf("Line %i, iter: %d: classify failed!\n", + __LINE__, i); + break; + } + + /* check results */ + for (j = 0; j != RTE_DIM(results); j++) { + if (results[j] != test_data[j].allow) { + printf("Line %i: Error in allow results at %i " + "(expected %"PRIu32" got %"PRIu32")!\n", + __LINE__, j, test_data[j].allow, + results[j]); + ret = -EINVAL; + } + } + } + + bswap_test_data(test_data, RTE_DIM(test_data), 0); + + rte_acl_free(acx); + return ret; +} + +static void +convert_rule(const struct rte_acl_ipv4vlan_rule *ri, + struct acl_ipv4vlan_rule *ro) +{ + ro->data = ri->data; + + ro->field[RTE_ACL_IPV4VLAN_PROTO_FIELD].value.u8 = ri->proto; + ro->field[RTE_ACL_IPV4VLAN_VLAN1_FIELD].value.u16 = ri->vlan; + ro->field[RTE_ACL_IPV4VLAN_VLAN2_FIELD].value.u16 = ri->domain; + ro->field[RTE_ACL_IPV4VLAN_SRC_FIELD].value.u32 = ri->src_addr; + ro->field[RTE_ACL_IPV4VLAN_DST_FIELD].value.u32 = ri->dst_addr; + ro->field[RTE_ACL_IPV4VLAN_SRCP_FIELD].value.u16 = ri->src_port_low; + ro->field[RTE_ACL_IPV4VLAN_DSTP_FIELD].value.u16 = ri->dst_port_low; + + ro->field[RTE_ACL_IPV4VLAN_PROTO_FIELD].mask_range.u8 = ri->proto_mask; + ro->field[RTE_ACL_IPV4VLAN_VLAN1_FIELD].mask_range.u16 = ri->vlan_mask; + ro->field[RTE_ACL_IPV4VLAN_VLAN2_FIELD].mask_range.u16 = + ri->domain_mask; + ro->field[RTE_ACL_IPV4VLAN_SRC_FIELD].mask_range.u32 = + ri->src_mask_len; + ro->field[RTE_ACL_IPV4VLAN_DST_FIELD].mask_range.u32 = ri->dst_mask_len; + ro->field[RTE_ACL_IPV4VLAN_SRCP_FIELD].mask_range.u16 = + ri->src_port_high; + ro->field[RTE_ACL_IPV4VLAN_DSTP_FIELD].mask_range.u16 = + ri->dst_port_high; +} + +/* + * Convert IPV4 source and destination from RTE_ACL_FIELD_TYPE_MASK to + * RTE_ACL_FIELD_TYPE_BITMASK. + */ +static void +convert_rule_1(const struct rte_acl_ipv4vlan_rule *ri, + struct acl_ipv4vlan_rule *ro) +{ + uint32_t v; + + convert_rule(ri, ro); + v = ro->field[RTE_ACL_IPV4VLAN_SRC_FIELD].mask_range.u32; + ro->field[RTE_ACL_IPV4VLAN_SRC_FIELD].mask_range.u32 = + RTE_ACL_MASKLEN_TO_BITMASK(v, sizeof(v)); + v = ro->field[RTE_ACL_IPV4VLAN_DST_FIELD].mask_range.u32; + ro->field[RTE_ACL_IPV4VLAN_DST_FIELD].mask_range.u32 = + RTE_ACL_MASKLEN_TO_BITMASK(v, sizeof(v)); +} + +/* + * Convert IPV4 source and destination from RTE_ACL_FIELD_TYPE_MASK to + * RTE_ACL_FIELD_TYPE_RANGE. + */ +static void +convert_rule_2(const struct rte_acl_ipv4vlan_rule *ri, + struct acl_ipv4vlan_rule *ro) +{ + uint32_t hi, lo, mask; + + convert_rule(ri, ro); + + mask = ro->field[RTE_ACL_IPV4VLAN_SRC_FIELD].mask_range.u32; + mask = RTE_ACL_MASKLEN_TO_BITMASK(mask, sizeof(mask)); + lo = ro->field[RTE_ACL_IPV4VLAN_SRC_FIELD].value.u32 & mask; + hi = lo + ~mask; + ro->field[RTE_ACL_IPV4VLAN_SRC_FIELD].value.u32 = lo; + ro->field[RTE_ACL_IPV4VLAN_SRC_FIELD].mask_range.u32 = hi; + + mask = ro->field[RTE_ACL_IPV4VLAN_DST_FIELD].mask_range.u32; + mask = RTE_ACL_MASKLEN_TO_BITMASK(mask, sizeof(mask)); + lo = ro->field[RTE_ACL_IPV4VLAN_DST_FIELD].value.u32 & mask; + hi = lo + ~mask; + ro->field[RTE_ACL_IPV4VLAN_DST_FIELD].value.u32 = lo; + ro->field[RTE_ACL_IPV4VLAN_DST_FIELD].mask_range.u32 = hi; +} + +/* + * Convert rte_acl_ipv4vlan_rule: swap VLAN and PORTS rule fields. + */ +static void +convert_rule_3(const struct rte_acl_ipv4vlan_rule *ri, + struct acl_ipv4vlan_rule *ro) +{ + struct rte_acl_field t1, t2; + + convert_rule(ri, ro); + + t1 = ro->field[RTE_ACL_IPV4VLAN_VLAN1_FIELD]; + t2 = ro->field[RTE_ACL_IPV4VLAN_VLAN2_FIELD]; + + ro->field[RTE_ACL_IPV4VLAN_VLAN1_FIELD] = + ro->field[RTE_ACL_IPV4VLAN_SRCP_FIELD]; + ro->field[RTE_ACL_IPV4VLAN_VLAN2_FIELD] = + ro->field[RTE_ACL_IPV4VLAN_DSTP_FIELD]; + + ro->field[RTE_ACL_IPV4VLAN_SRCP_FIELD] = t1; + ro->field[RTE_ACL_IPV4VLAN_DSTP_FIELD] = t2; +} + +/* + * Convert rte_acl_ipv4vlan_rule: swap SRC and DST IPv4 address rules. + */ +static void +convert_rule_4(const struct rte_acl_ipv4vlan_rule *ri, + struct acl_ipv4vlan_rule *ro) +{ + struct rte_acl_field t; + + convert_rule(ri, ro); + + t = ro->field[RTE_ACL_IPV4VLAN_SRC_FIELD]; + ro->field[RTE_ACL_IPV4VLAN_SRC_FIELD] = + ro->field[RTE_ACL_IPV4VLAN_DST_FIELD]; + + ro->field[RTE_ACL_IPV4VLAN_DST_FIELD] = t; +} + +static void +ipv4vlan_config(struct rte_acl_config *cfg, + const uint32_t layout[RTE_ACL_IPV4VLAN_NUM], + uint32_t num_categories) +{ + static const struct rte_acl_field_def + ipv4_defs[RTE_ACL_IPV4VLAN_NUM_FIELDS] = { + { + .type = RTE_ACL_FIELD_TYPE_BITMASK, + .size = sizeof(uint8_t), + .field_index = RTE_ACL_IPV4VLAN_PROTO_FIELD, + .input_index = RTE_ACL_IPV4VLAN_PROTO, + }, + { + .type = RTE_ACL_FIELD_TYPE_BITMASK, + .size = sizeof(uint16_t), + .field_index = RTE_ACL_IPV4VLAN_VLAN1_FIELD, + .input_index = RTE_ACL_IPV4VLAN_VLAN, + }, + { + .type = RTE_ACL_FIELD_TYPE_BITMASK, + .size = sizeof(uint16_t), + .field_index = RTE_ACL_IPV4VLAN_VLAN2_FIELD, + .input_index = RTE_ACL_IPV4VLAN_VLAN, + }, + { + .type = RTE_ACL_FIELD_TYPE_MASK, + .size = sizeof(uint32_t), + .field_index = RTE_ACL_IPV4VLAN_SRC_FIELD, + .input_index = RTE_ACL_IPV4VLAN_SRC, + }, + { + .type = RTE_ACL_FIELD_TYPE_MASK, + .size = sizeof(uint32_t), + .field_index = RTE_ACL_IPV4VLAN_DST_FIELD, + .input_index = RTE_ACL_IPV4VLAN_DST, + }, + { + .type = RTE_ACL_FIELD_TYPE_RANGE, + .size = sizeof(uint16_t), + .field_index = RTE_ACL_IPV4VLAN_SRCP_FIELD, + .input_index = RTE_ACL_IPV4VLAN_PORTS, + }, + { + .type = RTE_ACL_FIELD_TYPE_RANGE, + .size = sizeof(uint16_t), + .field_index = RTE_ACL_IPV4VLAN_DSTP_FIELD, + .input_index = RTE_ACL_IPV4VLAN_PORTS, + }, + }; + + memcpy(&cfg->defs, ipv4_defs, sizeof(ipv4_defs)); + cfg->num_fields = RTE_DIM(ipv4_defs); + + cfg->defs[RTE_ACL_IPV4VLAN_PROTO_FIELD].offset = + layout[RTE_ACL_IPV4VLAN_PROTO]; + cfg->defs[RTE_ACL_IPV4VLAN_VLAN1_FIELD].offset = + layout[RTE_ACL_IPV4VLAN_VLAN]; + cfg->defs[RTE_ACL_IPV4VLAN_VLAN2_FIELD].offset = + layout[RTE_ACL_IPV4VLAN_VLAN] + + cfg->defs[RTE_ACL_IPV4VLAN_VLAN1_FIELD].size; + cfg->defs[RTE_ACL_IPV4VLAN_SRC_FIELD].offset = + layout[RTE_ACL_IPV4VLAN_SRC]; + cfg->defs[RTE_ACL_IPV4VLAN_DST_FIELD].offset = + layout[RTE_ACL_IPV4VLAN_DST]; + cfg->defs[RTE_ACL_IPV4VLAN_SRCP_FIELD].offset = + layout[RTE_ACL_IPV4VLAN_PORTS]; + cfg->defs[RTE_ACL_IPV4VLAN_DSTP_FIELD].offset = + layout[RTE_ACL_IPV4VLAN_PORTS] + + cfg->defs[RTE_ACL_IPV4VLAN_SRCP_FIELD].size; + + cfg->num_categories = num_categories; +} + +static int +convert_rules(struct rte_acl_ctx *acx, + void (*convert)(const struct rte_acl_ipv4vlan_rule *, + struct acl_ipv4vlan_rule *), + const struct rte_acl_ipv4vlan_rule *rules, uint32_t num) +{ + int32_t rc; + uint32_t i; + struct acl_ipv4vlan_rule r; + + for (i = 0; i != num; i++) { + convert(rules + i, &r); + rc = rte_acl_add_rules(acx, (struct rte_acl_rule *)&r, 1); + if (rc != 0) { + printf("Line %i: Adding rule %u to ACL context " + "failed with error code: %d\n", + __LINE__, i, rc); + return rc; + } + } + + return 0; +} + +static void +convert_config(struct rte_acl_config *cfg) +{ + ipv4vlan_config(cfg, ipv4_7tuple_layout, RTE_ACL_MAX_CATEGORIES); +} + +/* + * Convert rte_acl_ipv4vlan_rule to use RTE_ACL_FIELD_TYPE_BITMASK. + */ +static void +convert_config_1(struct rte_acl_config *cfg) +{ + ipv4vlan_config(cfg, ipv4_7tuple_layout, RTE_ACL_MAX_CATEGORIES); + cfg->defs[RTE_ACL_IPV4VLAN_SRC_FIELD].type = RTE_ACL_FIELD_TYPE_BITMASK; + cfg->defs[RTE_ACL_IPV4VLAN_DST_FIELD].type = RTE_ACL_FIELD_TYPE_BITMASK; +} + +/* + * Convert rte_acl_ipv4vlan_rule to use RTE_ACL_FIELD_TYPE_RANGE. + */ +static void +convert_config_2(struct rte_acl_config *cfg) +{ + ipv4vlan_config(cfg, ipv4_7tuple_layout, RTE_ACL_MAX_CATEGORIES); + cfg->defs[RTE_ACL_IPV4VLAN_SRC_FIELD].type = RTE_ACL_FIELD_TYPE_RANGE; + cfg->defs[RTE_ACL_IPV4VLAN_DST_FIELD].type = RTE_ACL_FIELD_TYPE_RANGE; +} + +/* + * Convert rte_acl_ipv4vlan_rule: swap VLAN and PORTS rule definitions. + */ +static void +convert_config_3(struct rte_acl_config *cfg) +{ + struct rte_acl_field_def t1, t2; + + ipv4vlan_config(cfg, ipv4_7tuple_layout, RTE_ACL_MAX_CATEGORIES); + + t1 = cfg->defs[RTE_ACL_IPV4VLAN_VLAN1_FIELD]; + t2 = cfg->defs[RTE_ACL_IPV4VLAN_VLAN2_FIELD]; + + /* swap VLAN1 and SRCP rule definition. */ + cfg->defs[RTE_ACL_IPV4VLAN_VLAN1_FIELD] = + cfg->defs[RTE_ACL_IPV4VLAN_SRCP_FIELD]; + cfg->defs[RTE_ACL_IPV4VLAN_VLAN1_FIELD].field_index = t1.field_index; + cfg->defs[RTE_ACL_IPV4VLAN_VLAN1_FIELD].input_index = t1.input_index; + + /* swap VLAN2 and DSTP rule definition. */ + cfg->defs[RTE_ACL_IPV4VLAN_VLAN2_FIELD] = + cfg->defs[RTE_ACL_IPV4VLAN_DSTP_FIELD]; + cfg->defs[RTE_ACL_IPV4VLAN_VLAN2_FIELD].field_index = t2.field_index; + cfg->defs[RTE_ACL_IPV4VLAN_VLAN2_FIELD].input_index = t2.input_index; + + cfg->defs[RTE_ACL_IPV4VLAN_SRCP_FIELD].type = t1.type; + cfg->defs[RTE_ACL_IPV4VLAN_SRCP_FIELD].size = t1.size; + cfg->defs[RTE_ACL_IPV4VLAN_SRCP_FIELD].offset = t1.offset; + + cfg->defs[RTE_ACL_IPV4VLAN_DSTP_FIELD].type = t2.type; + cfg->defs[RTE_ACL_IPV4VLAN_DSTP_FIELD].size = t2.size; + cfg->defs[RTE_ACL_IPV4VLAN_DSTP_FIELD].offset = t2.offset; +} + +/* + * Convert rte_acl_ipv4vlan_rule: swap SRC and DST ip address rule definitions. + */ +static void +convert_config_4(struct rte_acl_config *cfg) +{ + struct rte_acl_field_def t; + + ipv4vlan_config(cfg, ipv4_7tuple_layout, RTE_ACL_MAX_CATEGORIES); + + t = cfg->defs[RTE_ACL_IPV4VLAN_SRC_FIELD]; + + cfg->defs[RTE_ACL_IPV4VLAN_SRC_FIELD] = + cfg->defs[RTE_ACL_IPV4VLAN_DST_FIELD]; + cfg->defs[RTE_ACL_IPV4VLAN_SRC_FIELD].field_index = t.field_index; + cfg->defs[RTE_ACL_IPV4VLAN_SRC_FIELD].input_index = t.input_index; + + cfg->defs[RTE_ACL_IPV4VLAN_DST_FIELD].type = t.type; + cfg->defs[RTE_ACL_IPV4VLAN_DST_FIELD].size = t.size; + cfg->defs[RTE_ACL_IPV4VLAN_DST_FIELD].offset = t.offset; +} + + +static int +build_convert_rules(struct rte_acl_ctx *acx, + void (*config)(struct rte_acl_config *), + size_t max_size) +{ + struct rte_acl_config cfg; + + memset(&cfg, 0, sizeof(cfg)); + config(&cfg); + cfg.max_size = max_size; + return rte_acl_build(acx, &cfg); +} + +static int +test_convert_rules(const char *desc, + void (*config)(struct rte_acl_config *), + void (*convert)(const struct rte_acl_ipv4vlan_rule *, + struct acl_ipv4vlan_rule *)) +{ + struct rte_acl_ctx *acx; + int32_t rc; + uint32_t i; + static const size_t mem_sizes[] = {0, -1}; + + printf("running %s(%s)\n", __func__, desc); + + acx = rte_acl_create(&acl_param); + if (acx == NULL) { + printf("Line %i: Error creating ACL context!\n", __LINE__); + return -1; + } + + rc = convert_rules(acx, convert, acl_test_rules, + RTE_DIM(acl_test_rules)); + if (rc != 0) + printf("Line %i: Error converting ACL rules!\n", __LINE__); + + for (i = 0; rc == 0 && i != RTE_DIM(mem_sizes); i++) { + + rc = build_convert_rules(acx, config, mem_sizes[i]); + if (rc != 0) { + printf("Line %i: Error @ build_convert_rules(%zu)!\n", + __LINE__, mem_sizes[i]); + break; + } + + rc = test_classify_run(acx); + if (rc != 0) + printf("%s failed at line %i, max_size=%zu\n", + __func__, __LINE__, mem_sizes[i]); + } + + rte_acl_free(acx); + return rc; +} + +static int +test_convert(void) +{ + static const struct { + const char *desc; + void (*config)(struct rte_acl_config *); + void (*convert)(const struct rte_acl_ipv4vlan_rule *, + struct acl_ipv4vlan_rule *); + } convert_param[] = { + { + "acl_ipv4vlan_tuple", + convert_config, + convert_rule, + }, + { + "acl_ipv4vlan_tuple, RTE_ACL_FIELD_TYPE_BITMASK type " + "for IPv4", + convert_config_1, + convert_rule_1, + }, + { + "acl_ipv4vlan_tuple, RTE_ACL_FIELD_TYPE_RANGE type " + "for IPv4", + convert_config_2, + convert_rule_2, + }, + { + "acl_ipv4vlan_tuple: swap VLAN and PORTs order", + convert_config_3, + convert_rule_3, + }, + { + "acl_ipv4vlan_tuple: swap SRC and DST IPv4 order", + convert_config_4, + convert_rule_4, + }, + }; + + uint32_t i; + int32_t rc; + + for (i = 0; i != RTE_DIM(convert_param); i++) { + rc = test_convert_rules(convert_param[i].desc, + convert_param[i].config, + convert_param[i].convert); + if (rc != 0) { + printf("%s for test-case: %s failed, error code: %d;\n", + __func__, convert_param[i].desc, rc); + return rc; + } + } + + return 0; +} + +/* + * Test wrong layout behavior + * This test supplies the ACL context with invalid layout, which results in + * ACL matching the wrong stuff. However, it should match the wrong stuff + * the right way. We switch around source and destination addresses, + * source and destination ports, and protocol will point to first byte of + * destination port. + */ +static int +test_invalid_layout(void) +{ + struct rte_acl_ctx *acx; + int ret, i; + + uint32_t results[RTE_DIM(invalid_layout_data)]; + const uint8_t *data[RTE_DIM(invalid_layout_data)]; + + const uint32_t layout[RTE_ACL_IPV4VLAN_NUM] = { + /* proto points to destination port's first byte */ + offsetof(struct ipv4_7tuple, port_dst), + + 0, /* VLAN not used */ + + /* src and dst addresses are swapped */ + offsetof(struct ipv4_7tuple, ip_dst), + offsetof(struct ipv4_7tuple, ip_src), + + /* + * we can't swap ports here, so we will swap + * them in the data + */ + offsetof(struct ipv4_7tuple, port_src), + }; + + acx = rte_acl_create(&acl_param); + if (acx == NULL) { + printf("Line %i: Error creating ACL context!\n", __LINE__); + return -1; + } + + /* putting a lot of rules into the context results in greater + * coverage numbers. it doesn't matter if they are identical */ + for (i = 0; i < 1000; i++) { + /* add rules to the context */ + ret = rte_acl_ipv4vlan_add_rules(acx, invalid_layout_rules, + RTE_DIM(invalid_layout_rules)); + if (ret != 0) { + printf("Line %i: Adding rules to ACL context failed!\n", + __LINE__); + rte_acl_free(acx); + return -1; + } + } + + /* try building the context */ + ret = rte_acl_ipv4vlan_build(acx, layout, 1); + if (ret != 0) { + printf("Line %i: Building ACL context failed!\n", __LINE__); + rte_acl_free(acx); + return -1; + } + + /* swap all bytes in the data to network order */ + bswap_test_data(invalid_layout_data, RTE_DIM(invalid_layout_data), 1); + + /* prepare data */ + for (i = 0; i < (int) RTE_DIM(invalid_layout_data); i++) { + data[i] = (uint8_t *)&invalid_layout_data[i]; + } + + /* classify tuples */ + ret = rte_acl_classify_alg(acx, data, results, + RTE_DIM(results), 1, RTE_ACL_CLASSIFY_SCALAR); + if (ret != 0) { + printf("Line %i: SSE classify failed!\n", __LINE__); + rte_acl_free(acx); + return -1; + } + + for (i = 0; i < (int) RTE_DIM(results); i++) { + if (results[i] != invalid_layout_data[i].allow) { + printf("Line %i: Wrong results at %i " + "(result=%u, should be %u)!\n", + __LINE__, i, results[i], + invalid_layout_data[i].allow); + goto err; + } + } + + /* classify tuples (scalar) */ + ret = rte_acl_classify_alg(acx, data, results, RTE_DIM(results), 1, + RTE_ACL_CLASSIFY_SCALAR); + + if (ret != 0) { + printf("Line %i: Scalar classify failed!\n", __LINE__); + rte_acl_free(acx); + return -1; + } + + for (i = 0; i < (int) RTE_DIM(results); i++) { + if (results[i] != invalid_layout_data[i].allow) { + printf("Line %i: Wrong results at %i " + "(result=%u, should be %u)!\n", + __LINE__, i, results[i], + invalid_layout_data[i].allow); + goto err; + } + } + + rte_acl_free(acx); + + /* swap data back to cpu order so that next time tests don't fail */ + bswap_test_data(invalid_layout_data, RTE_DIM(invalid_layout_data), 0); + + return 0; +err: + + /* swap data back to cpu order so that next time tests don't fail */ + bswap_test_data(invalid_layout_data, RTE_DIM(invalid_layout_data), 0); + + rte_acl_free(acx); + + return -1; +} + +/* + * Test creating and finding ACL contexts, and adding rules + */ +static int +test_create_find_add(void) +{ + struct rte_acl_param param; + struct rte_acl_ctx *acx, *acx2, *tmp; + struct rte_acl_ipv4vlan_rule rules[LEN]; + + const uint32_t layout[RTE_ACL_IPV4VLAN_NUM] = {0}; + + const char *acx_name = "acx"; + const char *acx2_name = "acx2"; + int i, ret; + + /* create two contexts */ + memcpy(¶m, &acl_param, sizeof(param)); + param.max_rule_num = 2; + + param.name = acx_name; + acx = rte_acl_create(¶m); + if (acx == NULL) { + printf("Line %i: Error creating %s!\n", __LINE__, acx_name); + return -1; + } + + param.name = acx2_name; + acx2 = rte_acl_create(¶m); + if (acx2 == NULL || acx2 == acx) { + printf("Line %i: Error creating %s!\n", __LINE__, acx2_name); + rte_acl_free(acx); + return -1; + } + + /* try to create third one, with an existing name */ + param.name = acx_name; + tmp = rte_acl_create(¶m); + if (tmp != acx) { + printf("Line %i: Creating context with existing name " + "test failed!\n", + __LINE__); + if (tmp) + rte_acl_free(tmp); + goto err; + } + + param.name = acx2_name; + tmp = rte_acl_create(¶m); + if (tmp != acx2) { + printf("Line %i: Creating context with existing " + "name test 2 failed!\n", + __LINE__); + if (tmp) + rte_acl_free(tmp); + goto err; + } + + /* try to find existing ACL contexts */ + tmp = rte_acl_find_existing(acx_name); + if (tmp != acx) { + printf("Line %i: Finding %s failed!\n", __LINE__, acx_name); + if (tmp) + rte_acl_free(tmp); + goto err; + } + + tmp = rte_acl_find_existing(acx2_name); + if (tmp != acx2) { + printf("Line %i: Finding %s failed!\n", __LINE__, acx2_name); + if (tmp) + rte_acl_free(tmp); + goto err; + } + + /* try to find non-existing context */ + tmp = rte_acl_find_existing("invalid"); + if (tmp != NULL) { + printf("Line %i: Non-existent ACL context found!\n", __LINE__); + goto err; + } + + /* free context */ + rte_acl_free(acx); + + + /* create valid (but severely limited) acx */ + memcpy(¶m, &acl_param, sizeof(param)); + param.max_rule_num = LEN; + + acx = rte_acl_create(¶m); + if (acx == NULL) { + printf("Line %i: Error creating %s!\n", __LINE__, param.name); + goto err; + } + + /* create dummy acl */ + for (i = 0; i < LEN; i++) { + memcpy(&rules[i], &acl_rule, + sizeof(struct rte_acl_ipv4vlan_rule)); + /* skip zero */ + rules[i].data.userdata = i + 1; + /* one rule per category */ + rules[i].data.category_mask = 1 << i; + } + + /* try filling up the context */ + ret = rte_acl_ipv4vlan_add_rules(acx, rules, LEN); + if (ret != 0) { + printf("Line %i: Adding %i rules to ACL context failed!\n", + __LINE__, LEN); + goto err; + } + + /* try adding to a (supposedly) full context */ + ret = rte_acl_ipv4vlan_add_rules(acx, rules, 1); + if (ret == 0) { + printf("Line %i: Adding rules to full ACL context should" + "have failed!\n", __LINE__); + goto err; + } + + /* try building the context */ + ret = rte_acl_ipv4vlan_build(acx, layout, RTE_ACL_MAX_CATEGORIES); + if (ret != 0) { + printf("Line %i: Building ACL context failed!\n", __LINE__); + goto err; + } + + rte_acl_free(acx); + rte_acl_free(acx2); + + return 0; +err: + rte_acl_free(acx); + rte_acl_free(acx2); + return -1; +} + +/* + * test various invalid rules + */ +static int +test_invalid_rules(void) +{ + struct rte_acl_ctx *acx; + int ret; + + struct rte_acl_ipv4vlan_rule rule; + + acx = rte_acl_create(&acl_param); + if (acx == NULL) { + printf("Line %i: Error creating ACL context!\n", __LINE__); + return -1; + } + + /* test inverted high/low source and destination ports. + * originally, there was a problem with memory consumption when using + * such rules. + */ + /* create dummy acl */ + memcpy(&rule, &acl_rule, sizeof(struct rte_acl_ipv4vlan_rule)); + rule.data.userdata = 1; + rule.dst_port_low = 0xfff0; + rule.dst_port_high = 0x0010; + + /* add rules to context and try to build it */ + ret = rte_acl_ipv4vlan_add_rules(acx, &rule, 1); + if (ret == 0) { + printf("Line %i: Adding rules to ACL context " + "should have failed!\n", __LINE__); + goto err; + } + + rule.dst_port_low = 0x0; + rule.dst_port_high = 0xffff; + rule.src_port_low = 0xfff0; + rule.src_port_high = 0x0010; + + /* add rules to context and try to build it */ + ret = rte_acl_ipv4vlan_add_rules(acx, &rule, 1); + if (ret == 0) { + printf("Line %i: Adding rules to ACL context " + "should have failed!\n", __LINE__); + goto err; + } + + rule.dst_port_low = 0x0; + rule.dst_port_high = 0xffff; + rule.src_port_low = 0x0; + rule.src_port_high = 0xffff; + + rule.dst_mask_len = 33; + + /* add rules to context and try to build it */ + ret = rte_acl_ipv4vlan_add_rules(acx, &rule, 1); + if (ret == 0) { + printf("Line %i: Adding rules to ACL context " + "should have failed!\n", __LINE__); + goto err; + } + + rule.dst_mask_len = 0; + rule.src_mask_len = 33; + + /* add rules to context and try to build it */ + ret = rte_acl_ipv4vlan_add_rules(acx, &rule, 1); + if (ret == 0) { + printf("Line %i: Adding rules to ACL context " + "should have failed!\n", __LINE__); + goto err; + } + + rte_acl_free(acx); + + return 0; + +err: + rte_acl_free(acx); + + return -1; +} + +/* + * test functions by passing invalid or + * non-workable parameters. + * + * we do very limited testing of classify functions here + * because those are performance-critical and + * thus don't do much parameter checking. + */ +static int +test_invalid_parameters(void) +{ + struct rte_acl_param param; + struct rte_acl_ctx *acx; + struct rte_acl_ipv4vlan_rule rule; + int result; + + uint32_t layout[RTE_ACL_IPV4VLAN_NUM] = {0}; + + + /** + * rte_ac_create() + */ + + /* NULL param */ + acx = rte_acl_create(NULL); + if (acx != NULL) { + printf("Line %i: ACL context creation with NULL param " + "should have failed!\n", __LINE__); + rte_acl_free(acx); + return -1; + } + + /* zero rule size */ + memcpy(¶m, &acl_param, sizeof(param)); + param.rule_size = 0; + + acx = rte_acl_create(¶m); + if (acx == NULL) { + printf("Line %i: ACL context creation with zero rule len " + "failed!\n", __LINE__); + return -1; + } else + rte_acl_free(acx); + + /* zero max rule num */ + memcpy(¶m, &acl_param, sizeof(param)); + param.max_rule_num = 0; + + acx = rte_acl_create(¶m); + if (acx == NULL) { + printf("Line %i: ACL context creation with zero rule num " + "failed!\n", __LINE__); + return -1; + } else + rte_acl_free(acx); + + /* invalid NUMA node */ + memcpy(¶m, &acl_param, sizeof(param)); + param.socket_id = RTE_MAX_NUMA_NODES + 1; + + acx = rte_acl_create(¶m); + if (acx != NULL) { + printf("Line %i: ACL context creation with invalid NUMA " + "should have failed!\n", __LINE__); + rte_acl_free(acx); + return -1; + } + + /* NULL name */ + memcpy(¶m, &acl_param, sizeof(param)); + param.name = NULL; + + acx = rte_acl_create(¶m); + if (acx != NULL) { + printf("Line %i: ACL context creation with NULL name " + "should have failed!\n", __LINE__); + rte_acl_free(acx); + return -1; + } + + /** + * rte_acl_find_existing + */ + + acx = rte_acl_find_existing(NULL); + if (acx != NULL) { + printf("Line %i: NULL ACL context found!\n", __LINE__); + rte_acl_free(acx); + return -1; + } + + /** + * rte_acl_ipv4vlan_add_rules + */ + + /* initialize everything */ + memcpy(¶m, &acl_param, sizeof(param)); + acx = rte_acl_create(¶m); + if (acx == NULL) { + printf("Line %i: ACL context creation failed!\n", __LINE__); + return -1; + } + + memcpy(&rule, &acl_rule, sizeof(rule)); + + /* NULL context */ + result = rte_acl_ipv4vlan_add_rules(NULL, &rule, 1); + if (result == 0) { + printf("Line %i: Adding rules with NULL ACL context " + "should have failed!\n", __LINE__); + rte_acl_free(acx); + return -1; + } + + /* NULL rule */ + result = rte_acl_ipv4vlan_add_rules(acx, NULL, 1); + if (result == 0) { + printf("Line %i: Adding NULL rule to ACL context " + "should have failed!\n", __LINE__); + rte_acl_free(acx); + return -1; + } + + /* zero count (should succeed) */ + result = rte_acl_ipv4vlan_add_rules(acx, &rule, 0); + if (result != 0) { + printf("Line %i: Adding 0 rules to ACL context failed!\n", + __LINE__); + rte_acl_free(acx); + return -1; + } + + /* free ACL context */ + rte_acl_free(acx); + + + /** + * rte_acl_ipv4vlan_build + */ + + /* reinitialize context */ + memcpy(¶m, &acl_param, sizeof(param)); + acx = rte_acl_create(¶m); + if (acx == NULL) { + printf("Line %i: ACL context creation failed!\n", __LINE__); + return -1; + } + + /* NULL context */ + result = rte_acl_ipv4vlan_build(NULL, layout, 1); + if (result == 0) { + printf("Line %i: Building with NULL context " + "should have failed!\n", __LINE__); + rte_acl_free(acx); + return -1; + } + + /* NULL layout */ + result = rte_acl_ipv4vlan_build(acx, NULL, 1); + if (result == 0) { + printf("Line %i: Building with NULL layout " + "should have failed!\n", __LINE__); + rte_acl_free(acx); + return -1; + } + + /* zero categories (should not fail) */ + result = rte_acl_ipv4vlan_build(acx, layout, 0); + if (result == 0) { + printf("Line %i: Building with 0 categories should fail!\n", + __LINE__); + rte_acl_free(acx); + return -1; + } + + /* SSE classify test */ + + /* cover zero categories in classify (should not fail) */ + result = rte_acl_classify(acx, NULL, NULL, 0, 0); + if (result != 0) { + printf("Line %i: SSE classify with zero categories " + "failed!\n", __LINE__); + rte_acl_free(acx); + return -1; + } + + /* cover invalid but positive categories in classify */ + result = rte_acl_classify(acx, NULL, NULL, 0, 3); + if (result == 0) { + printf("Line %i: SSE classify with 3 categories " + "should have failed!\n", __LINE__); + rte_acl_free(acx); + return -1; + } + + /* scalar classify test */ + + /* cover zero categories in classify (should not fail) */ + result = rte_acl_classify_alg(acx, NULL, NULL, 0, 0, + RTE_ACL_CLASSIFY_SCALAR); + if (result != 0) { + printf("Line %i: Scalar classify with zero categories " + "failed!\n", __LINE__); + rte_acl_free(acx); + return -1; + } + + /* cover invalid but positive categories in classify */ + result = rte_acl_classify(acx, NULL, NULL, 0, 3); + if (result == 0) { + printf("Line %i: Scalar classify with 3 categories " + "should have failed!\n", __LINE__); + rte_acl_free(acx); + return -1; + } + + /* free ACL context */ + rte_acl_free(acx); + + + /** + * make sure void functions don't crash with NULL parameters + */ + + rte_acl_free(NULL); + + rte_acl_dump(NULL); + + return 0; +} + +/** + * Various tests that don't test much but improve coverage + */ +static int +test_misc(void) +{ + struct rte_acl_param param; + struct rte_acl_ctx *acx; + + /* create context */ + memcpy(¶m, &acl_param, sizeof(param)); + + acx = rte_acl_create(¶m); + if (acx == NULL) { + printf("Line %i: Error creating ACL context!\n", __LINE__); + return -1; + } + + /* dump context with rules - useful for coverage */ + rte_acl_list_dump(); + + rte_acl_dump(acx); + + rte_acl_free(acx); + + return 0; +} + +static int +test_acl(void) +{ + if (test_invalid_parameters() < 0) + return -1; + if (test_invalid_rules() < 0) + return -1; + if (test_create_find_add() < 0) + return -1; + if (test_invalid_layout() < 0) + return -1; + if (test_misc() < 0) + return -1; + if (test_classify() < 0) + return -1; + if (test_build_ports_range() < 0) + return -1; + if (test_convert() < 0) + return -1; + + return 0; +} + +REGISTER_TEST_COMMAND(acl_autotest, test_acl); diff --git a/dpdk/test/test/test_acl.h b/dpdk/test/test/test_acl.h new file mode 100644 index 00000000..421f3109 --- /dev/null +++ b/dpdk/test/test/test_acl.h @@ -0,0 +1,692 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef TEST_ACL_H_ +#define TEST_ACL_H_ + +struct ipv4_7tuple { + uint16_t vlan; + uint16_t domain; + uint8_t proto; + uint32_t ip_src; + uint32_t ip_dst; + uint16_t port_src; + uint16_t port_dst; + uint32_t allow; + uint32_t deny; +}; + +/** + * Legacy support for 7-tuple IPv4 and VLAN rule. + * This structure and corresponding API is deprecated. + */ +struct rte_acl_ipv4vlan_rule { + struct rte_acl_rule_data data; /**< Miscellaneous data for the rule. */ + uint8_t proto; /**< IPv4 protocol ID. */ + uint8_t proto_mask; /**< IPv4 protocol ID mask. */ + uint16_t vlan; /**< VLAN ID. */ + uint16_t vlan_mask; /**< VLAN ID mask. */ + uint16_t domain; /**< VLAN domain. */ + uint16_t domain_mask; /**< VLAN domain mask. */ + uint32_t src_addr; /**< IPv4 source address. */ + uint32_t src_mask_len; /**< IPv4 source address mask. */ + uint32_t dst_addr; /**< IPv4 destination address. */ + uint32_t dst_mask_len; /**< IPv4 destination address mask. */ + uint16_t src_port_low; /**< L4 source port low. */ + uint16_t src_port_high; /**< L4 source port high. */ + uint16_t dst_port_low; /**< L4 destination port low. */ + uint16_t dst_port_high; /**< L4 destination port high. */ +}; + +/** + * Specifies fields layout inside rte_acl_rule for rte_acl_ipv4vlan_rule. + */ +enum { + RTE_ACL_IPV4VLAN_PROTO_FIELD, + RTE_ACL_IPV4VLAN_VLAN1_FIELD, + RTE_ACL_IPV4VLAN_VLAN2_FIELD, + RTE_ACL_IPV4VLAN_SRC_FIELD, + RTE_ACL_IPV4VLAN_DST_FIELD, + RTE_ACL_IPV4VLAN_SRCP_FIELD, + RTE_ACL_IPV4VLAN_DSTP_FIELD, + RTE_ACL_IPV4VLAN_NUM_FIELDS +}; + +/** + * Macro to define rule size for rte_acl_ipv4vlan_rule. + */ +#define RTE_ACL_IPV4VLAN_RULE_SZ \ + RTE_ACL_RULE_SZ(RTE_ACL_IPV4VLAN_NUM_FIELDS) + +/* + * That effectively defines order of IPV4VLAN classifications: + * - PROTO + * - VLAN (TAG and DOMAIN) + * - SRC IP ADDRESS + * - DST IP ADDRESS + * - PORTS (SRC and DST) + */ +enum { + RTE_ACL_IPV4VLAN_PROTO, + RTE_ACL_IPV4VLAN_VLAN, + RTE_ACL_IPV4VLAN_SRC, + RTE_ACL_IPV4VLAN_DST, + RTE_ACL_IPV4VLAN_PORTS, + RTE_ACL_IPV4VLAN_NUM +}; + +/* rules for invalid layout test */ +struct rte_acl_ipv4vlan_rule invalid_layout_rules[] = { + /* test src and dst address */ + { + .data = {.userdata = 1, .category_mask = 1}, + .src_addr = IPv4(10,0,0,0), + .src_mask_len = 24, + }, + { + .data = {.userdata = 2, .category_mask = 1}, + .dst_addr = IPv4(10,0,0,0), + .dst_mask_len = 24, + }, + /* test src and dst ports */ + { + .data = {.userdata = 3, .category_mask = 1}, + .dst_port_low = 100, + .dst_port_high = 100, + }, + { + .data = {.userdata = 4, .category_mask = 1}, + .src_port_low = 100, + .src_port_high = 100, + }, + /* test proto */ + { + .data = {.userdata = 5, .category_mask = 1}, + .proto = 0xf, + .proto_mask = 0xf + }, + { + .data = {.userdata = 6, .category_mask = 1}, + .dst_port_low = 0xf, + .dst_port_high = 0xf, + } +}; + +/* these might look odd because they don't match up the rules. This is + * intentional, as the invalid layout test presumes returning the correct + * results using the wrong data layout. + */ +struct ipv4_7tuple invalid_layout_data[] = { + {.ip_src = IPv4(10,0,1,0)}, /* should not match */ + {.ip_src = IPv4(10,0,0,1), .allow = 2}, /* should match 2 */ + {.port_src = 100, .allow = 4}, /* should match 4 */ + {.port_dst = 0xf, .allow = 6}, /* should match 6 */ +}; + +#define ACL_ALLOW 0 +#define ACL_DENY 1 +#define ACL_ALLOW_MASK 0x1 +#define ACL_DENY_MASK 0x2 + +/* ruleset for ACL unit test */ +struct rte_acl_ipv4vlan_rule acl_test_rules[] = { +/* destination IP addresses */ + /* matches all packets traveling to 192.168.0.0/16 */ + { + .data = {.userdata = 1, .category_mask = ACL_ALLOW_MASK, + .priority = 230}, + .dst_addr = IPv4(192,168,0,0), + .dst_mask_len = 16, + .src_port_low = 0, + .src_port_high = 0xffff, + .dst_port_low = 0, + .dst_port_high = 0xffff, + }, + /* matches all packets traveling to 192.168.1.0/24 */ + { + .data = {.userdata = 2, .category_mask = ACL_ALLOW_MASK, + .priority = 330}, + .dst_addr = IPv4(192,168,1,0), + .dst_mask_len = 24, + .src_port_low = 0, + .src_port_high = 0xffff, + .dst_port_low = 0, + .dst_port_high = 0xffff, + }, + /* matches all packets traveling to 192.168.1.50 */ + { + .data = {.userdata = 3, .category_mask = ACL_DENY_MASK, + .priority = 230}, + .dst_addr = IPv4(192,168,1,50), + .dst_mask_len = 32, + .src_port_low = 0, + .src_port_high = 0xffff, + .dst_port_low = 0, + .dst_port_high = 0xffff, + }, + +/* source IP addresses */ + /* matches all packets traveling from 10.0.0.0/8 */ + { + .data = {.userdata = 4, .category_mask = ACL_ALLOW_MASK, + .priority = 240}, + .src_addr = IPv4(10,0,0,0), + .src_mask_len = 8, + .src_port_low = 0, + .src_port_high = 0xffff, + .dst_port_low = 0, + .dst_port_high = 0xffff, + }, + /* matches all packets traveling from 10.1.1.0/24 */ + { + .data = {.userdata = 5, .category_mask = ACL_ALLOW_MASK, + .priority = 340}, + .src_addr = IPv4(10,1,1,0), + .src_mask_len = 24, + .src_port_low = 0, + .src_port_high = 0xffff, + .dst_port_low = 0, + .dst_port_high = 0xffff, + }, + /* matches all packets traveling from 10.1.1.1 */ + { + .data = {.userdata = 6, .category_mask = ACL_DENY_MASK, + .priority = 240}, + .src_addr = IPv4(10,1,1,1), + .src_mask_len = 32, + .src_port_low = 0, + .src_port_high = 0xffff, + .dst_port_low = 0, + .dst_port_high = 0xffff, + }, + +/* VLAN tag */ + /* matches all packets with lower 7 bytes of VLAN tag equal to 0x64 */ + { + .data = {.userdata = 7, .category_mask = ACL_ALLOW_MASK, + .priority = 260}, + .vlan = 0x64, + .vlan_mask = 0x7f, + .src_port_low = 0, + .src_port_high = 0xffff, + .dst_port_low = 0, + .dst_port_high = 0xffff, + }, + /* matches all packets with VLAN tags that have 0x5 in them */ + { + .data = {.userdata = 8, .category_mask = ACL_ALLOW_MASK, + .priority = 260}, + .vlan = 0x5, + .vlan_mask = 0x5, + .src_port_low = 0, + .src_port_high = 0xffff, + .dst_port_low = 0, + .dst_port_high = 0xffff, + }, + /* matches all packets with VLAN tag 5 */ + { + .data = {.userdata = 9, .category_mask = ACL_DENY_MASK, + .priority = 360}, + .vlan = 0x5, + .vlan_mask = 0xffff, + .src_port_low = 0, + .src_port_high = 0xffff, + .dst_port_low = 0, + .dst_port_high = 0xffff, + }, + +/* VLAN domain */ + /* matches all packets with lower 7 bytes of domain equal to 0x64 */ + { + .data = {.userdata = 10, .category_mask = ACL_ALLOW_MASK, + .priority = 250}, + .domain = 0x64, + .domain_mask = 0x7f, + .src_port_low = 0, + .src_port_high = 0xffff, + .dst_port_low = 0, + .dst_port_high = 0xffff, + }, + /* matches all packets with domains that have 0x5 in them */ + { + .data = {.userdata = 11, .category_mask = ACL_ALLOW_MASK, + .priority = 350}, + .domain = 0x5, + .domain_mask = 0x5, + .src_port_low = 0, + .src_port_high = 0xffff, + .dst_port_low = 0, + .dst_port_high = 0xffff, + }, + /* matches all packets with domain 5 */ + { + .data = {.userdata = 12, .category_mask = ACL_DENY_MASK, + .priority = 350}, + .domain = 0x5, + .domain_mask = 0xffff, + .src_port_low = 0, + .src_port_high = 0xffff, + .dst_port_low = 0, + .dst_port_high = 0xffff, + }, + +/* destination port */ + /* matches everything with dst port 80 */ + { + .data = {.userdata = 13, .category_mask = ACL_ALLOW_MASK, + .priority = 310}, + .dst_port_low = 80, + .dst_port_high = 80, + .src_port_low = 0, + .src_port_high = 0xffff, + }, + /* matches everything with dst port 22-1023 */ + { + .data = {.userdata = 14, .category_mask = ACL_ALLOW_MASK, + .priority = 210}, + .dst_port_low = 22, + .dst_port_high = 1023, + .src_port_low = 0, + .src_port_high = 0xffff, + }, + /* matches everything with dst port 1020 */ + { + .data = {.userdata = 15, .category_mask = ACL_DENY_MASK, + .priority = 310}, + .dst_port_low = 1020, + .dst_port_high = 1020, + .src_port_low = 0, + .src_port_high = 0xffff, + }, + /* matches everything with dst portrange 1000-2000 */ + { + .data = {.userdata = 16, .category_mask = ACL_DENY_MASK, + .priority = 210}, + .dst_port_low = 1000, + .dst_port_high = 2000, + .src_port_low = 0, + .src_port_high = 0xffff, + }, + +/* source port */ + /* matches everything with src port 80 */ + { + .data = {.userdata = 17, .category_mask = ACL_ALLOW_MASK, + .priority = 320}, + .src_port_low = 80, + .src_port_high = 80, + .dst_port_low = 0, + .dst_port_high = 0xffff, + }, + /* matches everything with src port 22-1023 */ + { + .data = {.userdata = 18, .category_mask = ACL_ALLOW_MASK, + .priority = 220}, + .src_port_low = 22, + .src_port_high = 1023, + .dst_port_low = 0, + .dst_port_high = 0xffff, + }, + /* matches everything with src port 1020 */ + { + .data = {.userdata = 19, .category_mask = ACL_DENY_MASK, + .priority = 320}, + .src_port_low = 1020, + .src_port_high = 1020, + .dst_port_low = 0, + .dst_port_high = 0xffff, + }, + /* matches everything with src portrange 1000-2000 */ + { + .data = {.userdata = 20, .category_mask = ACL_DENY_MASK, + .priority = 220}, + .src_port_low = 1000, + .src_port_high = 2000, + .dst_port_low = 0, + .dst_port_high = 0xffff, + }, + +/* protocol number */ + /* matches all packets with protocol number either 0x64 or 0xE4 */ + { + .data = {.userdata = 21, .category_mask = ACL_ALLOW_MASK, + .priority = 270}, + .proto = 0x64, + .proto_mask = 0x7f, + .src_port_low = 0, + .src_port_high = 0xffff, + .dst_port_low = 0, + .dst_port_high = 0xffff, + }, + /* matches all packets with protocol that have 0x5 in them */ + { + .data = {.userdata = 22, .category_mask = ACL_ALLOW_MASK, + .priority = 1}, + .proto = 0x5, + .proto_mask = 0x5, + .src_port_low = 0, + .src_port_high = 0xffff, + .dst_port_low = 0, + .dst_port_high = 0xffff, + }, + /* matches all packets with protocol 5 */ + { + .data = {.userdata = 23, .category_mask = ACL_DENY_MASK, + .priority = 370}, + .proto = 0x5, + .proto_mask = 0xff, + .src_port_low = 0, + .src_port_high = 0xffff, + .dst_port_low = 0, + .dst_port_high = 0xffff, + }, + +/* rules combining various fields */ + { + .data = {.userdata = 24, .category_mask = ACL_ALLOW_MASK, + .priority = 400}, + /** make sure that unmasked bytes don't fail! */ + .dst_addr = IPv4(1,2,3,4), + .dst_mask_len = 16, + .src_addr = IPv4(5,6,7,8), + .src_mask_len = 24, + .proto = 0x5, + .proto_mask = 0xff, + .src_port_low = 0, + .src_port_high = 0xffff, + .dst_port_low = 22, + .dst_port_high = 1024, + .vlan = 0x8100, + .vlan_mask = 0xffff, + .domain = 0x64, + .domain_mask = 0xffff, + }, + { + .data = {.userdata = 25, .category_mask = ACL_DENY_MASK, + .priority = 400}, + .dst_addr = IPv4(5,6,7,8), + .dst_mask_len = 24, + .src_addr = IPv4(1,2,3,4), + .src_mask_len = 16, + .proto = 0x5, + .proto_mask = 0xff, + .src_port_low = 0, + .src_port_high = 0xffff, + .dst_port_low = 22, + .dst_port_high = 1024, + .vlan = 0x8100, + .vlan_mask = 0xffff, + .domain = 0x64, + .domain_mask = 0xffff, + }, + { + .data = {.userdata = 26, .category_mask = ACL_ALLOW_MASK, + .priority = 500}, + .dst_addr = IPv4(1,2,3,4), + .dst_mask_len = 8, + .src_addr = IPv4(5,6,7,8), + .src_mask_len = 32, + .proto = 0x5, + .proto_mask = 0xff, + .src_port_low = 0, + .src_port_high = 0xffff, + .dst_port_low = 22, + .dst_port_high = 1024, + .vlan = 0x64, + .vlan_mask = 0xffff, + }, + { + .data = {.userdata = 27, .category_mask = ACL_DENY_MASK, + .priority = 500}, + .dst_addr = IPv4(5,6,7,8), + .dst_mask_len = 32, + .src_addr = IPv4(1,2,3,4), + .src_mask_len = 8, + .proto = 0x5, + .proto_mask = 0xff, + .src_port_low = 0, + .src_port_high = 0xffff, + .dst_port_low = 22, + .dst_port_high = 1024, + .vlan = 0x64, + .vlan_mask = 0xffff, + }, +}; + +/* data for ACL unit test */ +struct ipv4_7tuple acl_test_data[] = { +/* testing single rule aspects */ + {.ip_src = IPv4(10,0,0,0), .allow = 4}, /* should match 4 */ + {.ip_src = IPv4(10,1,1,2), .allow = 5}, /* should match 5 */ + {.ip_src = IPv4(10,1,1,1), .allow = 5, + .deny = 6}, /* should match 5, 6 */ + {.ip_dst = IPv4(10,0,0,0)}, /* should not match */ + {.ip_dst = IPv4(10,1,1,2)}, /* should not match */ + {.ip_dst = IPv4(10,1,1,1)}, /* should not match */ + + {.ip_src = IPv4(192,168,2,50)}, /* should not match */ + {.ip_src = IPv4(192,168,1,2)}, /* should not match */ + {.ip_src = IPv4(192,168,1,50)}, /* should not match */ + {.ip_dst = IPv4(192,168,2,50), .allow = 1}, /* should match 1 */ + {.ip_dst = IPv4(192,168,1,49), .allow = 2}, /* should match 2 */ + {.ip_dst = IPv4(192,168,1,50), .allow = 2, + .deny = 3}, /* should match 2, 3 */ + + {.vlan = 0x64, .allow = 7}, /* should match 7 */ + {.vlan = 0xfE4, .allow = 7}, /* should match 7 */ + {.vlan = 0xE2}, /* should not match */ + {.vlan = 0xD, .allow = 8}, /* should match 8 */ + {.vlan = 0x6}, /* should not match */ + {.vlan = 0x5, .allow = 8, .deny = 9}, /* should match 8, 9 */ + + {.domain = 0x64, .allow = 10}, /* should match 10 */ + {.domain = 0xfE4, .allow = 10}, /* should match 10 */ + {.domain = 0xE2}, /* should not match */ + {.domain = 0xD, .allow = 11}, /* should match 11 */ + {.domain = 0x6}, /* should not match */ + {.domain = 0x5, .allow = 11, .deny = 12}, /* should match 11, 12 */ + + {.port_dst = 80, .allow = 13}, /* should match 13 */ + {.port_dst = 79, .allow = 14}, /* should match 14 */ + {.port_dst = 81, .allow = 14}, /* should match 14 */ + {.port_dst = 21}, /* should not match */ + {.port_dst = 1024, .deny = 16}, /* should match 16 */ + {.port_dst = 1020, .allow = 14, .deny = 15}, /* should match 14, 15 */ + + {.port_src = 80, .allow = 17}, /* should match 17 */ + {.port_src = 79, .allow = 18}, /* should match 18 */ + {.port_src = 81, .allow = 18}, /* should match 18 */ + {.port_src = 21}, /* should not match */ + {.port_src = 1024, .deny = 20}, /* should match 20 */ + {.port_src = 1020, .allow = 18, .deny = 19}, /* should match 18, 19 */ + + {.proto = 0x64, .allow = 21}, /* should match 21 */ + {.proto = 0xE4, .allow = 21}, /* should match 21 */ + {.proto = 0xE2}, /* should not match */ + {.proto = 0xD, .allow = 22}, /* should match 22 */ + {.proto = 0x6}, /* should not match */ + {.proto = 0x5, .allow = 22, .deny = 23}, /* should match 22, 23 */ + +/* testing matching multiple rules at once */ + {.vlan = 0x5, .ip_src = IPv4(10,1,1,1), + .allow = 5, .deny = 9}, /* should match 5, 9 */ + {.vlan = 0x5, .ip_src = IPv4(192,168,2,50), + .allow = 8, .deny = 9}, /* should match 8, 9 */ + {.vlan = 0x55, .ip_src = IPv4(192,168,1,49), + .allow = 8}, /* should match 8 */ + {.port_dst = 80, .port_src = 1024, + .allow = 13, .deny = 20}, /* should match 13,20 */ + {.port_dst = 79, .port_src = 1024, + .allow = 14, .deny = 20}, /* should match 14,20 */ + {.proto = 0x5, .ip_dst = IPv4(192,168,2,50), + .allow = 1, .deny = 23}, /* should match 1, 23 */ + + {.proto = 0x5, .ip_dst = IPv4(192,168,1,50), + .allow = 2, .deny = 23}, /* should match 2, 23 */ + {.vlan = 0x64, .domain = 0x5, + .allow = 11, .deny = 12}, /* should match 11, 12 */ + {.proto = 0x5, .port_src = 80, + .allow = 17, .deny = 23}, /* should match 17, 23 */ + {.proto = 0x5, .port_dst = 80, + .allow = 13, .deny = 23}, /* should match 13, 23 */ + {.proto = 0x51, .port_src = 5000}, /* should not match */ + {.ip_src = IPv4(192,168,1,50), + .ip_dst = IPv4(10,0,0,0), + .proto = 0x51, + .port_src = 5000, + .port_dst = 5000}, /* should not match */ + +/* test full packet rules */ + { + .ip_dst = IPv4(1,2,100,200), + .ip_src = IPv4(5,6,7,254), + .proto = 0x5, + .vlan = 0x8100, + .domain = 0x64, + .port_src = 12345, + .port_dst = 80, + .allow = 24, + .deny = 23 + }, /* should match 23, 24 */ + { + .ip_dst = IPv4(5,6,7,254), + .ip_src = IPv4(1,2,100,200), + .proto = 0x5, + .vlan = 0x8100, + .domain = 0x64, + .port_src = 12345, + .port_dst = 80, + .allow = 13, + .deny = 25 + }, /* should match 13, 25 */ + { + .ip_dst = IPv4(1,10,20,30), + .ip_src = IPv4(5,6,7,8), + .proto = 0x5, + .vlan = 0x64, + .port_src = 12345, + .port_dst = 80, + .allow = 26, + .deny = 23 + }, /* should match 23, 26 */ + { + .ip_dst = IPv4(5,6,7,8), + .ip_src = IPv4(1,10,20,30), + .proto = 0x5, + .vlan = 0x64, + .port_src = 12345, + .port_dst = 80, + .allow = 13, + .deny = 27 + }, /* should match 13, 27 */ + { + .ip_dst = IPv4(2,2,3,4), + .ip_src = IPv4(4,6,7,8), + .proto = 0x5, + .vlan = 0x64, + .port_src = 12345, + .port_dst = 80, + .allow = 13, + .deny = 23 + }, /* should match 13, 23 */ + { + .ip_dst = IPv4(1,2,3,4), + .ip_src = IPv4(4,6,7,8), + .proto = 0x5, + .vlan = 0x64, + .port_src = 12345, + .port_dst = 80, + .allow = 13, + .deny = 23 + }, /* should match 13, 23 */ + + +/* visual separator! */ + { + .ip_dst = IPv4(1,2,100,200), + .ip_src = IPv4(5,6,7,254), + .proto = 0x55, + .vlan = 0x8000, + .domain = 0x6464, + .port_src = 12345, + .port_dst = 8080, + .allow = 10 + }, /* should match 10 */ + { + .ip_dst = IPv4(5,6,7,254), + .ip_src = IPv4(1,2,100,200), + .proto = 0x55, + .vlan = 0x8100, + .domain = 0x6464, + .port_src = 12345, + .port_dst = 180, + .allow = 10 + }, /* should match 10 */ + { + .ip_dst = IPv4(1,10,20,30), + .ip_src = IPv4(5,6,7,8), + .proto = 0x55, + .vlan = 0x64, + .port_src = 12345, + .port_dst = 180, + .allow = 7 + }, /* should match 7 */ + { + .ip_dst = IPv4(5,6,7,8), + .ip_src = IPv4(1,10,20,30), + .proto = 0x55, + .vlan = 0x64, + .port_src = 12345, + .port_dst = 180, + .allow = 7 + }, /* should match 7 */ + { + .ip_dst = IPv4(2,2,3,4), + .ip_src = IPv4(4,6,7,8), + .proto = 0x55, + .vlan = 0x64, + .port_src = 12345, + .port_dst = 180, + .allow = 7 + }, /* should match 7 */ + { + .ip_dst = IPv4(1,2,3,4), + .ip_src = IPv4(4,6,7,8), + .proto = 0x50, + .vlan = 0x6466, + .port_src = 12345, + .port_dst = 12345, + }, /* should not match */ +}; + +#endif /* TEST_ACL_H_ */ diff --git a/dpdk/test/test/test_alarm.c b/dpdk/test/test/test_alarm.c new file mode 100644 index 00000000..2d16d6ff --- /dev/null +++ b/dpdk/test/test/test_alarm.c @@ -0,0 +1,260 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <stdint.h> + +#include <rte_common.h> +#include <rte_cycles.h> +#include <rte_interrupts.h> +#include <rte_atomic.h> +#include <rte_alarm.h> + +#include "test.h" + +#define US_PER_MS 1000 + +#define RTE_TEST_ALARM_TIMEOUT 10 /* ms */ +#define RTE_TEST_CHECK_PERIOD 3 /* ms */ +#define RTE_TEST_MAX_REPEAT 20 + +static volatile int flag; + +static void +test_alarm_callback(void *cb_arg) +{ + flag = 1; + printf("Callback setting flag - OK. [cb_arg = %p]\n", cb_arg); +} + +static rte_atomic32_t cb_count; + +static void +test_multi_cb(void *arg) +{ + rte_atomic32_inc(&cb_count); + printf("In %s - arg = %p\n", __func__, arg); +} + +static volatile int recursive_error = 0; + +static void +test_remove_in_callback(void *arg) +{ + printf("In %s - arg = %p\n", __func__, arg); + if (rte_eal_alarm_cancel(test_remove_in_callback, arg) || + rte_eal_alarm_cancel(test_remove_in_callback, (void *)-1)) { + printf("Error - cancelling callback from within function succeeded!\n"); + recursive_error = 1; + } + flag = (int)((uintptr_t)arg); +} + +static volatile int flag_2; + +static void +test_remove_in_callback_2(void *arg) +{ + if (rte_eal_alarm_cancel(test_remove_in_callback_2, arg) || rte_eal_alarm_cancel(test_remove_in_callback_2, (void *)-1)) { + printf("Error - cancelling callback of test_remove_in_callback_2\n"); + return; + } + flag_2 = 1; +} + +static int +test_multi_alarms(void) +{ + int rm_count = 0; + int count = 0; + cb_count.cnt = 0; + + printf("Expect 6 callbacks in order...\n"); + /* add two alarms in order */ + rte_eal_alarm_set(10 * US_PER_MS, test_multi_cb, (void *)1); + rte_eal_alarm_set(20 * US_PER_MS, test_multi_cb, (void *)2); + + /* now add in reverse order */ + rte_eal_alarm_set(60 * US_PER_MS, test_multi_cb, (void *)6); + rte_eal_alarm_set(50 * US_PER_MS, test_multi_cb, (void *)5); + rte_eal_alarm_set(40 * US_PER_MS, test_multi_cb, (void *)4); + rte_eal_alarm_set(30 * US_PER_MS, test_multi_cb, (void *)3); + + /* wait for expiry */ + rte_delay_ms(65); + if (cb_count.cnt != 6) { + printf("Missing callbacks\n"); + /* remove any callbacks that might remain */ + rte_eal_alarm_cancel(test_multi_cb, (void *)-1); + return -1; + } + + cb_count.cnt = 0; + printf("Expect only callbacks with args 1 and 3...\n"); + /* Add 3 flags, then delete one */ + rte_eal_alarm_set(30 * US_PER_MS, test_multi_cb, (void *)3); + rte_eal_alarm_set(20 * US_PER_MS, test_multi_cb, (void *)2); + rte_eal_alarm_set(10 * US_PER_MS, test_multi_cb, (void *)1); + rm_count = rte_eal_alarm_cancel(test_multi_cb, (void *)2); + + rte_delay_ms(35); + if (cb_count.cnt != 2 || rm_count != 1) { + printf("Error: invalid flags count or alarm removal failure" + " - flags value = %d, expected = %d\n", + (int)cb_count.cnt, 2); + /* remove any callbacks that might remain */ + rte_eal_alarm_cancel(test_multi_cb, (void *)-1); + return -1; + } + + printf("Testing adding and then removing multiple alarms\n"); + /* finally test that no callbacks are called if we delete them all*/ + rte_eal_alarm_set(10 * US_PER_MS, test_multi_cb, (void *)1); + rte_eal_alarm_set(10 * US_PER_MS, test_multi_cb, (void *)2); + rte_eal_alarm_set(10 * US_PER_MS, test_multi_cb, (void *)3); + rm_count = rte_eal_alarm_cancel(test_alarm_callback, (void *)-1); + if (rm_count != 0) { + printf("Error removing non-existant alarm succeeded\n"); + rte_eal_alarm_cancel(test_multi_cb, (void *) -1); + return -1; + } + rm_count = rte_eal_alarm_cancel(test_multi_cb, (void *) -1); + if (rm_count != 3) { + printf("Error removing all pending alarm callbacks\n"); + return -1; + } + + /* Test that we cannot cancel an alarm from within the callback itself + * Also test that we can cancel head-of-line callbacks ok.*/ + flag = 0; + recursive_error = 0; + rte_eal_alarm_set(10 * US_PER_MS, test_remove_in_callback, (void *)1); + rte_eal_alarm_set(20 * US_PER_MS, test_remove_in_callback, (void *)2); + rm_count = rte_eal_alarm_cancel(test_remove_in_callback, (void *)1); + if (rm_count != 1) { + printf("Error cancelling head-of-list callback\n"); + return -1; + } + rte_delay_ms(15); + if (flag != 0) { + printf("Error, cancelling head-of-list leads to premature callback\n"); + return -1; + } + + while (flag != 2 && count++ < RTE_TEST_MAX_REPEAT) + rte_delay_ms(10); + + if (flag != 2) { + printf("Error - expected callback not called\n"); + rte_eal_alarm_cancel(test_remove_in_callback, (void *)-1); + return -1; + } + if (recursive_error == 1) + return -1; + + /* Check if it can cancel all for the same callback */ + printf("Testing canceling all for the same callback\n"); + flag_2 = 0; + rte_eal_alarm_set(10 * US_PER_MS, test_remove_in_callback, (void *)1); + rte_eal_alarm_set(20 * US_PER_MS, test_remove_in_callback_2, (void *)2); + rte_eal_alarm_set(30 * US_PER_MS, test_remove_in_callback_2, (void *)3); + rte_eal_alarm_set(40 * US_PER_MS, test_remove_in_callback, (void *)4); + rm_count = rte_eal_alarm_cancel(test_remove_in_callback_2, (void *)-1); + if (rm_count != 2) { + printf("Error, cannot cancel all for the same callback\n"); + return -1; + } + rm_count = rte_eal_alarm_cancel(test_remove_in_callback, (void *)-1); + if (rm_count != 2) { + printf("Error, cannot cancel all for the same callback\n"); + return -1; + } + + return 0; +} + +static int +test_alarm(void) +{ + int count = 0; + + /* check if the callback will be called */ + printf("check if the callback will be called\n"); + flag = 0; + if (rte_eal_alarm_set(RTE_TEST_ALARM_TIMEOUT * US_PER_MS, + test_alarm_callback, NULL) < 0) { + printf("fail to set alarm callback\n"); + return -1; + } + while (flag == 0 && count++ < RTE_TEST_MAX_REPEAT) + rte_delay_ms(RTE_TEST_CHECK_PERIOD); + + if (flag == 0){ + printf("Callback not called\n"); + return -1; + } + + /* check if it will fail to set alarm with wrong us value */ + printf("check if it will fail to set alarm with wrong ms values\n"); + if (rte_eal_alarm_set(0, test_alarm_callback, + NULL) >= 0) { + printf("should not be successful with 0 us value\n"); + return -1; + } + if (rte_eal_alarm_set(UINT64_MAX - 1, test_alarm_callback, + NULL) >= 0) { + printf("should not be successful with (UINT64_MAX-1) us value\n"); + return -1; + } + + /* check if it will fail to set alarm with null callback parameter */ + printf("check if it will fail to set alarm with null callback parameter\n"); + if (rte_eal_alarm_set(RTE_TEST_ALARM_TIMEOUT, NULL, NULL) >= 0) { + printf("should not be successful to set alarm with null callback parameter\n"); + return -1; + } + + /* check if it will fail to remove alarm with null callback parameter */ + printf("check if it will fail to remove alarm with null callback parameter\n"); + if (rte_eal_alarm_cancel(NULL, NULL) == 0) { + printf("should not be successful to remove alarm with null callback parameter"); + return -1; + } + + if (test_multi_alarms() != 0) + return -1; + + return 0; +} + +REGISTER_TEST_COMMAND(alarm_autotest, test_alarm); diff --git a/dpdk/test/test/test_atomic.c b/dpdk/test/test/test_atomic.c new file mode 100644 index 00000000..39c9a595 --- /dev/null +++ b/dpdk/test/test/test_atomic.c @@ -0,0 +1,375 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <stdint.h> +#include <unistd.h> +#include <sys/queue.h> + +#include <rte_memory.h> +#include <rte_per_lcore.h> +#include <rte_launch.h> +#include <rte_atomic.h> +#include <rte_eal.h> +#include <rte_lcore.h> + +#include "test.h" + +/* + * Atomic Variables + * ================ + * + * - The main test function performs three subtests. The first test + * checks that the usual inc/dec/add/sub functions are working + * correctly: + * + * - Initialize 16-bit, 32-bit and 64-bit atomic variables to specific + * values. + * + * - These variables are incremented and decremented on each core at + * the same time in ``test_atomic_usual()``. + * + * - The function checks that once all lcores finish their function, + * the value of the atomic variables are still the same. + * + * - The second test verifies the behavior of "test and set" functions. + * + * - Initialize 16-bit, 32-bit and 64-bit atomic variables to zero. + * + * - Invoke ``test_atomic_tas()`` on each lcore: before doing anything + * else. The cores are waiting a synchro using ``while + * (rte_atomic32_read(&val) == 0)`` which is triggered by the main test + * function. Then all cores do a + * ``rte_atomicXX_test_and_set()`` at the same time. If it is successful, + * it increments another atomic counter. + * + * - The main function checks that the atomic counter was incremented + * twice only (one for 16-bit, one for 32-bit and one for 64-bit values). + * + * - Test "add/sub and return" + * + * - Initialize 16-bit, 32-bit and 64-bit atomic variables to zero. + * + * - Invoke ``test_atomic_addsub_return()`` on each lcore. Before doing + * anything else, the cores are waiting a synchro. Each lcore does + * this operation several times:: + * + * tmp = rte_atomicXX_add_return(&a, 1); + * atomic_add(&count, tmp); + * tmp = rte_atomicXX_sub_return(&a, 1); + * atomic_sub(&count, tmp+1); + * + * - At the end of the test, the *count* value must be 0. + */ + +#define NUM_ATOMIC_TYPES 3 + +#define N 10000 + +static rte_atomic16_t a16; +static rte_atomic32_t a32; +static rte_atomic64_t a64; +static rte_atomic64_t count; +static rte_atomic32_t synchro; + +static int +test_atomic_usual(__attribute__((unused)) void *arg) +{ + unsigned i; + + while (rte_atomic32_read(&synchro) == 0) + ; + + for (i = 0; i < N; i++) + rte_atomic16_inc(&a16); + for (i = 0; i < N; i++) + rte_atomic16_dec(&a16); + for (i = 0; i < (N / 5); i++) + rte_atomic16_add(&a16, 5); + for (i = 0; i < (N / 5); i++) + rte_atomic16_sub(&a16, 5); + + for (i = 0; i < N; i++) + rte_atomic32_inc(&a32); + for (i = 0; i < N; i++) + rte_atomic32_dec(&a32); + for (i = 0; i < (N / 5); i++) + rte_atomic32_add(&a32, 5); + for (i = 0; i < (N / 5); i++) + rte_atomic32_sub(&a32, 5); + + for (i = 0; i < N; i++) + rte_atomic64_inc(&a64); + for (i = 0; i < N; i++) + rte_atomic64_dec(&a64); + for (i = 0; i < (N / 5); i++) + rte_atomic64_add(&a64, 5); + for (i = 0; i < (N / 5); i++) + rte_atomic64_sub(&a64, 5); + + return 0; +} + +static int +test_atomic_tas(__attribute__((unused)) void *arg) +{ + while (rte_atomic32_read(&synchro) == 0) + ; + + if (rte_atomic16_test_and_set(&a16)) + rte_atomic64_inc(&count); + if (rte_atomic32_test_and_set(&a32)) + rte_atomic64_inc(&count); + if (rte_atomic64_test_and_set(&a64)) + rte_atomic64_inc(&count); + + return 0; +} + +static int +test_atomic_addsub_and_return(__attribute__((unused)) void *arg) +{ + uint32_t tmp16; + uint32_t tmp32; + uint64_t tmp64; + unsigned i; + + while (rte_atomic32_read(&synchro) == 0) + ; + + for (i = 0; i < N; i++) { + tmp16 = rte_atomic16_add_return(&a16, 1); + rte_atomic64_add(&count, tmp16); + + tmp16 = rte_atomic16_sub_return(&a16, 1); + rte_atomic64_sub(&count, tmp16+1); + + tmp32 = rte_atomic32_add_return(&a32, 1); + rte_atomic64_add(&count, tmp32); + + tmp32 = rte_atomic32_sub_return(&a32, 1); + rte_atomic64_sub(&count, tmp32+1); + + tmp64 = rte_atomic64_add_return(&a64, 1); + rte_atomic64_add(&count, tmp64); + + tmp64 = rte_atomic64_sub_return(&a64, 1); + rte_atomic64_sub(&count, tmp64+1); + } + + return 0; +} + +/* + * rte_atomic32_inc_and_test() would increase a 32 bits counter by one and then + * test if that counter is equal to 0. It would return true if the counter is 0 + * and false if the counter is not 0. rte_atomic64_inc_and_test() could do the + * same thing but for a 64 bits counter. + * Here checks that if the 32/64 bits counter is equal to 0 after being atomically + * increased by one. If it is, increase the variable of "count" by one which would + * be checked as the result later. + * + */ +static int +test_atomic_inc_and_test(__attribute__((unused)) void *arg) +{ + while (rte_atomic32_read(&synchro) == 0) + ; + + if (rte_atomic16_inc_and_test(&a16)) { + rte_atomic64_inc(&count); + } + if (rte_atomic32_inc_and_test(&a32)) { + rte_atomic64_inc(&count); + } + if (rte_atomic64_inc_and_test(&a64)) { + rte_atomic64_inc(&count); + } + + return 0; +} + +/* + * rte_atomicXX_dec_and_test() should decrease a 32 bits counter by one and then + * test if that counter is equal to 0. It should return true if the counter is 0 + * and false if the counter is not 0. + * This test checks if the counter is equal to 0 after being atomically + * decreased by one. If it is, increase the value of "count" by one which is to + * be checked as the result later. + */ +static int +test_atomic_dec_and_test(__attribute__((unused)) void *arg) +{ + while (rte_atomic32_read(&synchro) == 0) + ; + + if (rte_atomic16_dec_and_test(&a16)) + rte_atomic64_inc(&count); + + if (rte_atomic32_dec_and_test(&a32)) + rte_atomic64_inc(&count); + + if (rte_atomic64_dec_and_test(&a64)) + rte_atomic64_inc(&count); + + return 0; +} + +static int +test_atomic(void) +{ + rte_atomic16_init(&a16); + rte_atomic32_init(&a32); + rte_atomic64_init(&a64); + rte_atomic64_init(&count); + rte_atomic32_init(&synchro); + + rte_atomic16_set(&a16, 1UL << 10); + rte_atomic32_set(&a32, 1UL << 10); + rte_atomic64_set(&a64, 1ULL << 33); + + printf("usual inc/dec/add/sub functions\n"); + + rte_eal_mp_remote_launch(test_atomic_usual, NULL, SKIP_MASTER); + rte_atomic32_set(&synchro, 1); + rte_eal_mp_wait_lcore(); + rte_atomic32_set(&synchro, 0); + + if (rte_atomic16_read(&a16) != 1UL << 10) { + printf("Atomic16 usual functions failed\n"); + return -1; + } + + if (rte_atomic32_read(&a32) != 1UL << 10) { + printf("Atomic32 usual functions failed\n"); + return -1; + } + + if (rte_atomic64_read(&a64) != 1ULL << 33) { + printf("Atomic64 usual functions failed\n"); + return -1; + } + + printf("test and set\n"); + + rte_atomic64_set(&a64, 0); + rte_atomic32_set(&a32, 0); + rte_atomic16_set(&a16, 0); + rte_atomic64_set(&count, 0); + rte_eal_mp_remote_launch(test_atomic_tas, NULL, SKIP_MASTER); + rte_atomic32_set(&synchro, 1); + rte_eal_mp_wait_lcore(); + rte_atomic32_set(&synchro, 0); + + if (rte_atomic64_read(&count) != NUM_ATOMIC_TYPES) { + printf("Atomic test and set failed\n"); + return -1; + } + + printf("add/sub and return\n"); + + rte_atomic64_set(&a64, 0); + rte_atomic32_set(&a32, 0); + rte_atomic16_set(&a16, 0); + rte_atomic64_set(&count, 0); + rte_eal_mp_remote_launch(test_atomic_addsub_and_return, NULL, + SKIP_MASTER); + rte_atomic32_set(&synchro, 1); + rte_eal_mp_wait_lcore(); + rte_atomic32_set(&synchro, 0); + + if (rte_atomic64_read(&count) != 0) { + printf("Atomic add/sub+return failed\n"); + return -1; + } + + /* + * Set a64, a32 and a16 with the same value of minus "number of slave + * lcores", launch all slave lcores to atomically increase by one and + * test them respectively. + * Each lcore should have only one chance to increase a64 by one and + * then check if it is equal to 0, but there should be only one lcore + * that finds that it is 0. It is similar for a32 and a16. + * Then a variable of "count", initialized to zero, is increased by + * one if a64, a32 or a16 is 0 after being increased and tested + * atomically. + * We can check if "count" is finally equal to 3 to see if all slave + * lcores performed "atomic inc and test" right. + */ + printf("inc and test\n"); + + rte_atomic64_clear(&a64); + rte_atomic32_clear(&a32); + rte_atomic16_clear(&a16); + rte_atomic32_clear(&synchro); + rte_atomic64_clear(&count); + + rte_atomic64_set(&a64, (int64_t)(1 - (int64_t)rte_lcore_count())); + rte_atomic32_set(&a32, (int32_t)(1 - (int32_t)rte_lcore_count())); + rte_atomic16_set(&a16, (int16_t)(1 - (int16_t)rte_lcore_count())); + rte_eal_mp_remote_launch(test_atomic_inc_and_test, NULL, SKIP_MASTER); + rte_atomic32_set(&synchro, 1); + rte_eal_mp_wait_lcore(); + rte_atomic32_clear(&synchro); + + if (rte_atomic64_read(&count) != NUM_ATOMIC_TYPES) { + printf("Atomic inc and test failed %d\n", (int)count.cnt); + return -1; + } + + /* + * Same as above, but this time we set the values to "number of slave + * lcores", and decrement instead of increment. + */ + printf("dec and test\n"); + + rte_atomic32_clear(&synchro); + rte_atomic64_clear(&count); + + rte_atomic64_set(&a64, (int64_t)(rte_lcore_count() - 1)); + rte_atomic32_set(&a32, (int32_t)(rte_lcore_count() - 1)); + rte_atomic16_set(&a16, (int16_t)(rte_lcore_count() - 1)); + rte_eal_mp_remote_launch(test_atomic_dec_and_test, NULL, SKIP_MASTER); + rte_atomic32_set(&synchro, 1); + rte_eal_mp_wait_lcore(); + rte_atomic32_clear(&synchro); + + if (rte_atomic64_read(&count) != NUM_ATOMIC_TYPES) { + printf("Atomic dec and test failed\n"); + return -1; + } + + return 0; +} + +REGISTER_TEST_COMMAND(atomic_autotest, test_atomic); diff --git a/dpdk/test/test/test_bitmap.c b/dpdk/test/test/test_bitmap.c new file mode 100644 index 00000000..7045d332 --- /dev/null +++ b/dpdk/test/test/test_bitmap.c @@ -0,0 +1,195 @@ +/* + * BSD LICENSE + * + * Copyright (C) Cavium, Inc. 2017. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <inttypes.h> + +#include <rte_common.h> +#include <rte_bitmap.h> +#include <rte_malloc.h> + +#include "test.h" + +#define MAX_BITS 1000 + +static int +test_bitmap_scan_operations(struct rte_bitmap *bmp) +{ + uint32_t pos = 0; + uint64_t slab1_magic = 0xBADC0FFEEBADF00D; + uint64_t slab2_magic = 0xFEEDDEADDEADF00D; + uint64_t out_slab = 0; + + rte_bitmap_reset(bmp); + + rte_bitmap_set_slab(bmp, pos, slab1_magic); + rte_bitmap_set_slab(bmp, pos + RTE_BITMAP_SLAB_BIT_SIZE, slab2_magic); + + if (!rte_bitmap_scan(bmp, &pos, &out_slab)) { + printf("Failed to get slab from bitmap.\n"); + return TEST_FAILED; + } + + if (slab1_magic != out_slab) { + printf("Scan operation sanity failed.\n"); + return TEST_FAILED; + } + + if (!rte_bitmap_scan(bmp, &pos, &out_slab)) { + printf("Failed to get slab from bitmap.\n"); + return TEST_FAILED; + } + + if (slab2_magic != out_slab) { + printf("Scan operation sanity failed.\n"); + return TEST_FAILED; + } + + /* Wrap around */ + if (!rte_bitmap_scan(bmp, &pos, &out_slab)) { + printf("Failed to get slab from bitmap.\n"); + return TEST_FAILED; + } + + if (slab1_magic != out_slab) { + printf("Scan operation wrap around failed.\n"); + return TEST_FAILED; + } + + /* Scan reset check. */ + __rte_bitmap_scan_init(bmp); + + if (!rte_bitmap_scan(bmp, &pos, &out_slab)) { + printf("Failed to get slab from bitmap.\n"); + return TEST_FAILED; + } + + if (slab1_magic != out_slab) { + printf("Scan reset operation failed.\n"); + return TEST_FAILED; + } + + return TEST_SUCCESS; +} + +static int +test_bitmap_slab_set_get(struct rte_bitmap *bmp) +{ + uint32_t pos = 0; + uint64_t slab_magic = 0xBADC0FFEEBADF00D; + uint64_t out_slab = 0; + + rte_bitmap_reset(bmp); + rte_bitmap_set_slab(bmp, pos, slab_magic); + + if (!rte_bitmap_scan(bmp, &pos, &out_slab)) { + printf("Failed to get slab from bitmap.\n"); + return TEST_FAILED; + } + + + if (slab_magic != out_slab) { + printf("Invalid slab in bitmap.\n"); + return TEST_FAILED; + } + + + return TEST_SUCCESS; +} + +static int +test_bitmap_set_get_clear(struct rte_bitmap *bmp) +{ + int i; + + rte_bitmap_reset(bmp); + for (i = 0; i < MAX_BITS; i++) + rte_bitmap_set(bmp, i); + + for (i = 0; i < MAX_BITS; i++) { + if (!rte_bitmap_get(bmp, i)) { + printf("Failed to get set bit.\n"); + return TEST_FAILED; + } + } + + for (i = 0; i < MAX_BITS; i++) + rte_bitmap_clear(bmp, i); + + for (i = 0; i < MAX_BITS; i++) { + if (rte_bitmap_get(bmp, i)) { + printf("Failed to clear set bit.\n"); + return TEST_FAILED; + } + } + + return TEST_SUCCESS; +} + +static int +test_bitmap(void) +{ + void *mem; + uint32_t bmp_size; + struct rte_bitmap *bmp; + + bmp_size = + rte_bitmap_get_memory_footprint(MAX_BITS); + + mem = rte_zmalloc("test_bmap", bmp_size, RTE_CACHE_LINE_SIZE); + if (mem == NULL) { + printf("Failed to allocate memory for bitmap\n"); + return TEST_FAILED; + } + + bmp = rte_bitmap_init(MAX_BITS, mem, bmp_size); + if (bmp == NULL) { + printf("Failed to init bitmap\n"); + return TEST_FAILED; + } + + if (test_bitmap_set_get_clear(bmp) < 0) + return TEST_FAILED; + + if (test_bitmap_slab_set_get(bmp) < 0) + return TEST_FAILED; + + if (test_bitmap_scan_operations(bmp) < 0) + return TEST_FAILED; + + rte_bitmap_free(bmp); + rte_free(mem); + + return TEST_SUCCESS; +} + +REGISTER_TEST_COMMAND(bitmap_test, test_bitmap); diff --git a/dpdk/test/test/test_byteorder.c b/dpdk/test/test/test_byteorder.c new file mode 100644 index 00000000..8ae31142 --- /dev/null +++ b/dpdk/test/test/test_byteorder.c @@ -0,0 +1,95 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <stdint.h> +#include <unistd.h> +#include <inttypes.h> + +#include <rte_byteorder.h> + +#include "test.h" + +static volatile uint16_t u16 = 0x1337; +static volatile uint32_t u32 = 0xdeadbeefUL; +static volatile uint64_t u64 = 0xdeadcafebabefaceULL; + +/* + * Byteorder functions + * =================== + * + * - check that optimized byte swap functions are working for each + * size (16, 32, 64 bits) + */ + +static int +test_byteorder(void) +{ + uint16_t res_u16; + uint32_t res_u32; + uint64_t res_u64; + + res_u16 = rte_bswap16(u16); + printf("%"PRIx16" -> %"PRIx16"\n", u16, res_u16); + if (res_u16 != 0x3713) + return -1; + + res_u32 = rte_bswap32(u32); + printf("%"PRIx32" -> %"PRIx32"\n", u32, res_u32); + if (res_u32 != 0xefbeaddeUL) + return -1; + + res_u64 = rte_bswap64(u64); + printf("%"PRIx64" -> %"PRIx64"\n", u64, res_u64); + if (res_u64 != 0xcefabebafecaaddeULL) + return -1; + + res_u16 = rte_bswap16(0x1337); + printf("const %"PRIx16" -> %"PRIx16"\n", 0x1337, res_u16); + if (res_u16 != 0x3713) + return -1; + + res_u32 = rte_bswap32(0xdeadbeefUL); + printf("const %"PRIx32" -> %"PRIx32"\n", (uint32_t) 0xdeadbeef, res_u32); + if (res_u32 != 0xefbeaddeUL) + return -1; + + res_u64 = rte_bswap64(0xdeadcafebabefaceULL); + printf("const %"PRIx64" -> %"PRIx64"\n", (uint64_t) 0xdeadcafebabefaceULL, res_u64); + if (res_u64 != 0xcefabebafecaaddeULL) + return -1; + + return 0; +} + +REGISTER_TEST_COMMAND(byteorder_autotest, test_byteorder); diff --git a/dpdk/test/test/test_cfgfile.c b/dpdk/test/test/test_cfgfile.c new file mode 100644 index 00000000..37435b39 --- /dev/null +++ b/dpdk/test/test/test_cfgfile.c @@ -0,0 +1,362 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Wind River Systems Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <string.h> +#include <stdint.h> +#include <sys/queue.h> + +#include <rte_cfgfile.h> + +#include "test.h" +#include "resource.h" + + +#define CFG_FILES_ETC "test_cfgfiles/etc" + +REGISTER_LINKED_RESOURCE(test_cfgfiles); + +static int +test_cfgfile_setup(void) +{ + const struct resource *r; + int ret; + + r = resource_find("test_cfgfiles"); + TEST_ASSERT_NOT_NULL(r, "missing resource test_cfgfiles"); + + ret = resource_untar(r); + TEST_ASSERT_SUCCESS(ret, "failed to untar %s", r->name); + + return 0; +} + +static int +test_cfgfile_cleanup(void) +{ + const struct resource *r; + int ret; + + r = resource_find("test_cfgfiles"); + TEST_ASSERT_NOT_NULL(r, "missing resource test_cfgfiles"); + + ret = resource_rm_by_tar(r); + TEST_ASSERT_SUCCESS(ret, "Failed to delete resource %s", r->name); + + return 0; +} + +static int +_test_cfgfile_sample(struct rte_cfgfile *cfgfile) +{ + const char *value; + int ret; + + ret = rte_cfgfile_num_sections(cfgfile, NULL, 0); + TEST_ASSERT(ret == 2, "Unexpected number of sections: %d", ret); + + ret = rte_cfgfile_has_section(cfgfile, "section1"); + TEST_ASSERT(ret, "section1 section missing"); + + ret = rte_cfgfile_section_num_entries(cfgfile, "section1"); + TEST_ASSERT(ret == 1, "section1 unexpected number of entries: %d", ret); + + value = rte_cfgfile_get_entry(cfgfile, "section1", "key1"); + TEST_ASSERT(strcmp("value1", value) == 0, + "key1 unexpected value: %s", value); + + ret = rte_cfgfile_has_section(cfgfile, "section2"); + TEST_ASSERT(ret, "section2 section missing"); + + ret = rte_cfgfile_section_num_entries(cfgfile, "section2"); + TEST_ASSERT(ret == 2, "section2 unexpected number of entries: %d", ret); + + value = rte_cfgfile_get_entry(cfgfile, "section2", "key2"); + TEST_ASSERT(strcmp("value2", value) == 0, + "key2 unexpected value: %s", value); + + value = rte_cfgfile_get_entry(cfgfile, "section2", "key3"); + TEST_ASSERT(strcmp("value3", value) == 0, + "key3 unexpected value: %s", value); + + return 0; +} + +static int +test_cfgfile_sample1(void) +{ + struct rte_cfgfile *cfgfile; + int ret; + + cfgfile = rte_cfgfile_load(CFG_FILES_ETC "/sample1.ini", 0); + TEST_ASSERT_NOT_NULL(cfgfile, "Failed to load config file"); + + ret = _test_cfgfile_sample(cfgfile); + TEST_ASSERT_SUCCESS(ret, "Failed to validate sample file: %d", ret); + + ret = rte_cfgfile_close(cfgfile); + TEST_ASSERT_SUCCESS(ret, "Failed to close cfgfile"); + + return 0; +} + +static int +test_cfgfile_sample2(void) +{ + struct rte_cfgfile_parameters params; + struct rte_cfgfile *cfgfile; + int ret; + + /* override comment character */ + memset(¶ms, 0, sizeof(params)); + params.comment_character = '#'; + + cfgfile = rte_cfgfile_load_with_params(CFG_FILES_ETC "/sample2.ini", 0, + ¶ms); + TEST_ASSERT_NOT_NULL(cfgfile, "Failed to parse sample2.ini"); + + ret = _test_cfgfile_sample(cfgfile); + TEST_ASSERT_SUCCESS(ret, "Failed to validate sample file: %d", ret); + + ret = rte_cfgfile_close(cfgfile); + TEST_ASSERT_SUCCESS(ret, "Failed to close cfgfile"); + + return 0; +} + +static int +test_cfgfile_realloc_sections(void) +{ + struct rte_cfgfile *cfgfile; + int ret; + const char *value; + + cfgfile = rte_cfgfile_load(CFG_FILES_ETC "/realloc_sections.ini", 0); + TEST_ASSERT_NOT_NULL(cfgfile, "Failed to load config file"); + + ret = rte_cfgfile_num_sections(cfgfile, NULL, 0); + TEST_ASSERT(ret == 9, "Unexpected number of sections: %d", ret); + + ret = rte_cfgfile_has_section(cfgfile, "section9"); + TEST_ASSERT(ret, "section9 missing"); + + ret = rte_cfgfile_section_num_entries(cfgfile, "section3"); + TEST_ASSERT(ret == 21, + "section3 unexpected number of entries: %d", ret); + + ret = rte_cfgfile_section_num_entries(cfgfile, "section9"); + TEST_ASSERT(ret == 8, "section9 unexpected number of entries: %d", ret); + + value = rte_cfgfile_get_entry(cfgfile, "section9", "key8"); + TEST_ASSERT(strcmp("value8_section9", value) == 0, + "key unexpected value: %s", value); + + ret = rte_cfgfile_save(cfgfile, "/tmp/cfgfile_save.ini"); + TEST_ASSERT_SUCCESS(ret, "Failed to save *.ini file"); + remove("/tmp/cfgfile_save.ini"); + + ret = rte_cfgfile_close(cfgfile); + TEST_ASSERT_SUCCESS(ret, "Failed to close cfgfile"); + + return 0; +} + +static int +test_cfgfile_invalid_section_header(void) +{ + struct rte_cfgfile *cfgfile; + + cfgfile = rte_cfgfile_load(CFG_FILES_ETC "/invalid_section.ini", 0); + TEST_ASSERT_NULL(cfgfile, "Expected failured did not occur"); + + return 0; +} + +static int +test_cfgfile_invalid_comment(void) +{ + struct rte_cfgfile_parameters params; + struct rte_cfgfile *cfgfile; + + /* override comment character with an invalid one */ + memset(¶ms, 0, sizeof(params)); + params.comment_character = '$'; + + cfgfile = rte_cfgfile_load_with_params(CFG_FILES_ETC "/sample2.ini", 0, + ¶ms); + TEST_ASSERT_NULL(cfgfile, "Expected failured did not occur"); + + return 0; +} + +static int +test_cfgfile_invalid_key_value_pair(void) +{ + struct rte_cfgfile *cfgfile; + + cfgfile = rte_cfgfile_load(CFG_FILES_ETC "/empty_key_value.ini", 0); + TEST_ASSERT_NULL(cfgfile, "Expected failured did not occur"); + + return 0; +} + +static int +test_cfgfile_empty_key_value_pair(void) +{ + struct rte_cfgfile *cfgfile; + const char *value; + int ret; + + cfgfile = rte_cfgfile_load(CFG_FILES_ETC "/empty_key_value.ini", + CFG_FLAG_EMPTY_VALUES); + TEST_ASSERT_NOT_NULL(cfgfile, "Failed to parse empty_key_value.ini"); + + ret = rte_cfgfile_num_sections(cfgfile, NULL, 0); + TEST_ASSERT(ret == 1, "Unexpected number of sections: %d", ret); + + ret = rte_cfgfile_has_section(cfgfile, "section1"); + TEST_ASSERT(ret, "section1 missing"); + + ret = rte_cfgfile_section_num_entries(cfgfile, "section1"); + TEST_ASSERT(ret == 1, "section1 unexpected number of entries: %d", ret); + + value = rte_cfgfile_get_entry(cfgfile, "section1", "key"); + TEST_ASSERT(strlen(value) == 0, "key unexpected value: %s", value); + + ret = rte_cfgfile_close(cfgfile); + TEST_ASSERT_SUCCESS(ret, "Failed to close cfgfile"); + + return 0; +} + +static int +test_cfgfile_missing_section(void) +{ + struct rte_cfgfile *cfgfile; + + cfgfile = rte_cfgfile_load(CFG_FILES_ETC "/missing_section.ini", 0); + TEST_ASSERT_NULL(cfgfile, "Expected failured did not occur"); + + return 0; +} + +static int +test_cfgfile_global_properties(void) +{ + struct rte_cfgfile *cfgfile; + const char *value; + int ret; + + cfgfile = rte_cfgfile_load(CFG_FILES_ETC "/missing_section.ini", + CFG_FLAG_GLOBAL_SECTION); + TEST_ASSERT_NOT_NULL(cfgfile, "Failed to load config file"); + + ret = rte_cfgfile_num_sections(cfgfile, NULL, 0); + TEST_ASSERT(ret == 1, "Unexpected number of sections: %d", ret); + + ret = rte_cfgfile_has_section(cfgfile, "GLOBAL"); + TEST_ASSERT(ret, "global section missing"); + + ret = rte_cfgfile_section_num_entries(cfgfile, "GLOBAL"); + TEST_ASSERT(ret == 1, "GLOBAL unexpected number of entries: %d", ret); + + value = rte_cfgfile_get_entry(cfgfile, "GLOBAL", "key"); + TEST_ASSERT(strcmp("value", value) == 0, + "key unexpected value: %s", value); + + ret = rte_cfgfile_close(cfgfile); + TEST_ASSERT_SUCCESS(ret, "Failed to close cfgfile"); + + return 0; +} + +static int +test_cfgfile_empty_file(void) +{ + struct rte_cfgfile *cfgfile; + int ret; + + cfgfile = rte_cfgfile_load(CFG_FILES_ETC "/empty.ini", 0); + TEST_ASSERT_NOT_NULL(cfgfile, "Failed to load config file"); + + ret = rte_cfgfile_num_sections(cfgfile, NULL, 0); + TEST_ASSERT(ret == 0, "Unexpected number of sections: %d", ret); + + ret = rte_cfgfile_close(cfgfile); + TEST_ASSERT_SUCCESS(ret, "Failed to close cfgfile"); + + return 0; +} + +static int +test_cfgfile(void) +{ + if (test_cfgfile_setup()) + return -1; + + if (test_cfgfile_sample1()) + return -1; + + if (test_cfgfile_sample2()) + return -1; + + if (test_cfgfile_realloc_sections()) + return -1; + + if (test_cfgfile_invalid_section_header()) + return -1; + + if (test_cfgfile_invalid_comment()) + return -1; + + if (test_cfgfile_invalid_key_value_pair()) + return -1; + + if (test_cfgfile_empty_key_value_pair()) + return -1; + + if (test_cfgfile_missing_section()) + return -1; + + if (test_cfgfile_global_properties()) + return -1; + + if (test_cfgfile_empty_file()) + return -1; + + if (test_cfgfile_cleanup()) + return -1; + + return 0; +} + +REGISTER_TEST_COMMAND(cfgfile_autotest, test_cfgfile); diff --git a/dpdk/test/test/test_cfgfiles/etc/empty.ini b/dpdk/test/test/test_cfgfiles/etc/empty.ini new file mode 100644 index 00000000..e69de29b diff --git a/dpdk/test/test/test_cfgfiles/etc/empty_key_value.ini b/dpdk/test/test/test_cfgfiles/etc/empty_key_value.ini new file mode 100644 index 00000000..53284467 --- /dev/null +++ b/dpdk/test/test/test_cfgfiles/etc/empty_key_value.ini @@ -0,0 +1,3 @@ +[section1] +; this is section 1 +key= diff --git a/dpdk/test/test/test_cfgfiles/etc/invalid_section.ini b/dpdk/test/test/test_cfgfiles/etc/invalid_section.ini new file mode 100644 index 00000000..95d68039 --- /dev/null +++ b/dpdk/test/test/test_cfgfiles/etc/invalid_section.ini @@ -0,0 +1,3 @@ +[invalid +; this is section 1 +key1=value1 diff --git a/dpdk/test/test/test_cfgfiles/etc/line_too_long.ini b/dpdk/test/test/test_cfgfiles/etc/line_too_long.ini new file mode 100644 index 00000000..1dce1648 --- /dev/null +++ b/dpdk/test/test/test_cfgfiles/etc/line_too_long.ini @@ -0,0 +1,3 @@ +[section1] +; this is section 1 +012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789 diff --git a/dpdk/test/test/test_cfgfiles/etc/missing_section.ini b/dpdk/test/test/test_cfgfiles/etc/missing_section.ini new file mode 100644 index 00000000..c78e131b --- /dev/null +++ b/dpdk/test/test/test_cfgfiles/etc/missing_section.ini @@ -0,0 +1,2 @@ +; no section +key=value diff --git a/dpdk/test/test/test_cfgfiles/etc/realloc_sections.ini b/dpdk/test/test/test_cfgfiles/etc/realloc_sections.ini new file mode 100644 index 00000000..e653e40c --- /dev/null +++ b/dpdk/test/test/test_cfgfiles/etc/realloc_sections.ini @@ -0,0 +1,128 @@ +[section1] +key1=value1_section1 +key2=value2_section1 +key3=value3_section1 +key4=value4_section1 +key5=value5_section1 +key6=value6_section1 +key7=value7_section1 +key8=value8_section1 +key9=value9_section1 +key10=value10_section1 +key11=value11_section1 +key12=value12_section1 +key13=value13_section1 +key14=value14_section1 +key15=value15_section1 +key16=value16_section1 +key17=value17_section1 +key18=value18_section1 +key19=value19_section1 +key20=value20_section1 +key21=value21_section1 + +[section2] +key1=value1_section2 +key2=value2_section2 +key3=value3_section2 +key4=value4_section2 +key5=value5_section2 +key6=value6_section2 +key7=value7_section2 +key8=value8_section2 +key9=value9_section2 +key10=value10_section2 +key11=value11_section2 +key12=value12_section2 +key13=value13_section2 +key14=value14_section2 +key15=value15_section2 +key16=value16_section2 +key17=value17_section2 +key18=value18_section2 +key19=value19_section2 +key20=value20_section2 +key21=value21_section2 + +[section3] +key1=value1_section3 +key2=value2_section3 +key3=value3_section3 +key4=value4_section3 +key5=value5_section3 +key6=value6_section3 +key7=value7_section3 +key8=value8_section3 +key9=value9_section3 +key10=value10_section3 +key11=value11_section3 +key12=value12_section3 +key13=value13_section3 +key14=value14_section3 +key15=value15_section3 +key16=value16_section3 +key17=value17_section3 +key18=value18_section3 +key19=value19_section3 +key20=value20_section3 +key21=value21_section3 + +[section4] +key1=value1_section4 +key2=value2_section4 +key3=value3_section4 +key4=value4_section4 +key5=value5_section4 +key6=value6_section4 +key7=value7_section4 +key8=value8_section4 + +[section5] +key1=value1_section5 +key2=value2_section5 +key3=value3_section5 +key4=value4_section5 +key5=value5_section5 +key6=value6_section5 +key7=value7_section5 +key8=value8_section5 + +[section6] +key1=value1_section6 +key2=value2_section6 +key3=value3_section6 +key4=value4_section6 +key5=value5_section6 +key6=value6_section6 +key7=value7_section6 +key8=value8_section6 + +[section7] +key1=value1_section7 +key2=value2_section7 +key3=value3_section7 +key4=value4_section7 +key5=value5_section7 +key6=value6_section7 +key7=value7_section7 +key8=value8_section7 + +[section8] +key1=value1_section8 +key2=value2_section8 +key3=value3_section8 +key4=value4_section8 +key5=value5_section8 +key6=value6_section8 +key7=value7_section8 +key8=value8_section8 + +[section9] +key1=value1_section9 +key2=value2_section9 +key3=value3_section9 +key4=value4_section9 +key5=value5_section9 +key6=value6_section9 +key7=value7_section9 +key8=value8_section9 diff --git a/dpdk/test/test/test_cfgfiles/etc/sample1.ini b/dpdk/test/test/test_cfgfiles/etc/sample1.ini new file mode 100644 index 00000000..aef91c24 --- /dev/null +++ b/dpdk/test/test/test_cfgfiles/etc/sample1.ini @@ -0,0 +1,12 @@ +; this is a global comment + +[section1] +; this is section 1 +key1=value1 + +[section2] +; this is section 2 +;key1=value1 +key2=value2 +key3=value3 ; this is key3 +ignore-missing-separator diff --git a/dpdk/test/test/test_cfgfiles/etc/sample2.ini b/dpdk/test/test/test_cfgfiles/etc/sample2.ini new file mode 100644 index 00000000..21075e97 --- /dev/null +++ b/dpdk/test/test/test_cfgfiles/etc/sample2.ini @@ -0,0 +1,12 @@ +# this is a global comment + +[section1] +# this is section 1 +key1=value1 + +[section2] +# this is section 2 +#key1=value1 +key2=value2 +key3=value3 # this is key3 +ignore-missing-separator diff --git a/dpdk/test/test/test_cmdline.c b/dpdk/test/test/test_cmdline.c new file mode 100644 index 00000000..38c7256f --- /dev/null +++ b/dpdk/test/test/test_cmdline.c @@ -0,0 +1,92 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> + +#include "test.h" +#include "test_cmdline.h" + +static int +test_cmdline(void) +{ + printf("Testind parsing ethernet addresses...\n"); + if (test_parse_etheraddr_valid() < 0) + return -1; + if (test_parse_etheraddr_invalid_data() < 0) + return -1; + if (test_parse_etheraddr_invalid_param() < 0) + return -1; + printf("Testind parsing port lists...\n"); + if (test_parse_portlist_valid() < 0) + return -1; + if (test_parse_portlist_invalid_data() < 0) + return -1; + if (test_parse_portlist_invalid_param() < 0) + return -1; + printf("Testind parsing numbers...\n"); + if (test_parse_num_valid() < 0) + return -1; + if (test_parse_num_invalid_data() < 0) + return -1; + if (test_parse_num_invalid_param() < 0) + return -1; + printf("Testing parsing IP addresses...\n"); + if (test_parse_ipaddr_valid() < 0) + return -1; + if (test_parse_ipaddr_invalid_data() < 0) + return -1; + if (test_parse_ipaddr_invalid_param() < 0) + return -1; + printf("Testing parsing strings...\n"); + if (test_parse_string_valid() < 0) + return -1; + if (test_parse_string_invalid_data() < 0) + return -1; + if (test_parse_string_invalid_param() < 0) + return -1; + printf("Testing circular buffer...\n"); + if (test_cirbuf_char() < 0) + return -1; + if (test_cirbuf_string() < 0) + return -1; + if (test_cirbuf_align() < 0) + return -1; + if (test_cirbuf_invalid_param() < 0) + return -1; + printf("Testing library functions...\n"); + if (test_cmdline_lib() < 0) + return -1; + return 0; +} + +REGISTER_TEST_COMMAND(cmdline_autotest, test_cmdline); diff --git a/dpdk/test/test/test_cmdline.h b/dpdk/test/test/test_cmdline.h new file mode 100644 index 00000000..0ee91c17 --- /dev/null +++ b/dpdk/test/test/test_cmdline.h @@ -0,0 +1,73 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef TEST_CMDLINE_H_ +#define TEST_CMDLINE_H_ + +#define CMDLINE_TEST_BUFSIZE 64 + +/* cmdline_parse_num tests */ +int test_parse_num_valid(void); +int test_parse_num_invalid_data(void); +int test_parse_num_invalid_param(void); + +/* cmdline_parse_etheraddr tests */ +int test_parse_etheraddr_valid(void); +int test_parse_etheraddr_invalid_data(void); +int test_parse_etheraddr_invalid_param(void); + +/* cmdline_parse_portlist tests */ +int test_parse_portlist_valid(void); +int test_parse_portlist_invalid_data(void); +int test_parse_portlist_invalid_param(void); + +/* cmdline_parse_ipaddr tests */ +int test_parse_ipaddr_valid(void); +int test_parse_ipaddr_invalid_data(void); +int test_parse_ipaddr_invalid_param(void); + +/* cmdline_parse_string tests */ +int test_parse_string_valid(void); +int test_parse_string_invalid_data(void); +int test_parse_string_invalid_param(void); + +/* cmdline_cirbuf tests */ +int test_cirbuf_invalid_param(void); +int test_cirbuf_char(void); +int test_cirbuf_string(void); +int test_cirbuf_align(void); + +/* test the rest of the library */ +int test_cmdline_lib(void); + +#endif /* TEST_CMDLINE_H_ */ diff --git a/dpdk/test/test/test_cmdline_cirbuf.c b/dpdk/test/test/test_cmdline_cirbuf.c new file mode 100644 index 00000000..2c321457 --- /dev/null +++ b/dpdk/test/test/test_cmdline_cirbuf.c @@ -0,0 +1,1330 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include <rte_string_fns.h> + +#include <cmdline_cirbuf.h> + +#include "test_cmdline.h" + +/* different length strings */ +#define CIRBUF_STR_HEAD " HEAD" +#define CIRBUF_STR_TAIL "TAIL" + +/* miscellaneous tests - they make bullseye happy */ +static int +test_cirbuf_string_misc(void) +{ + struct cirbuf cb; + char buf[CMDLINE_TEST_BUFSIZE]; + char tmp[CMDLINE_TEST_BUFSIZE]; + + /* initialize buffers */ + memset(buf, 0, sizeof(buf)); + memset(tmp, 0, sizeof(tmp)); + + /* + * initialize circular buffer + */ + if (cirbuf_init(&cb, buf, 0, sizeof(buf)) < 0) { + printf("Error: failed to initialize circular buffer!\n"); + return -1; + } + + /* + * add strings to head and tail, but read only tail + * this results in read operation that does not transcend + * from buffer end to buffer beginning (in other words, + * strlen <= cb->maxlen - cb->end) + */ + + /* add string to head */ + if (cirbuf_add_buf_head(&cb, CIRBUF_STR_HEAD, sizeof(CIRBUF_STR_HEAD)) + != (sizeof(CIRBUF_STR_HEAD))) { + printf("Error: failed to add string to head!\n"); + return -1; + } + /* add string to tail */ + if (cirbuf_add_buf_tail(&cb, CIRBUF_STR_TAIL, sizeof(CIRBUF_STR_TAIL)) + != (sizeof(CIRBUF_STR_TAIL))) { + printf("Error: failed to add string to head!\n"); + return -1; + } + /* read string from tail */ + if (cirbuf_get_buf_tail(&cb, tmp, sizeof(CIRBUF_STR_TAIL)) + != (sizeof(CIRBUF_STR_TAIL))) { + printf("Error: failed to get string from tail!\n"); + return -1; + } + /* verify string */ + if (strncmp(tmp, CIRBUF_STR_TAIL, sizeof(CIRBUF_STR_TAIL)) != 0) { + printf("Error: tail strings do not match!\n"); + return -1; + } + /* clear buffers */ + memset(tmp, 0, sizeof(tmp)); + memset(buf, 0, sizeof(buf)); + + + + /* + * add a string to buffer when start/end is at end of buffer + */ + + /* + * reinitialize circular buffer with start at the end of cirbuf + */ + if (cirbuf_init(&cb, buf, CMDLINE_TEST_BUFSIZE - 2, sizeof(buf)) < 0) { + printf("Error: failed to reinitialize circular buffer!\n"); + return -1; + } + + + /* add string to tail */ + if (cirbuf_add_buf_tail(&cb, CIRBUF_STR_TAIL, sizeof(CIRBUF_STR_TAIL)) + != (sizeof(CIRBUF_STR_TAIL))) { + printf("Error: failed to add string to tail!\n"); + return -1; + } + /* read string from tail */ + if (cirbuf_get_buf_tail(&cb, tmp, sizeof(CIRBUF_STR_TAIL)) + != (sizeof(CIRBUF_STR_TAIL))) { + printf("Error: failed to get string from tail!\n"); + return -1; + } + /* verify string */ + if (strncmp(tmp, CIRBUF_STR_TAIL, sizeof(CIRBUF_STR_TAIL)) != 0) { + printf("Error: tail strings do not match!\n"); + return -1; + } + /* clear tmp buffer */ + memset(tmp, 0, sizeof(tmp)); + + + /* add string to head */ + if (cirbuf_add_buf_head(&cb, CIRBUF_STR_HEAD, sizeof(CIRBUF_STR_HEAD)) + != (sizeof(CIRBUF_STR_HEAD))) { + printf("Error: failed to add string to head!\n"); + return -1; + } + /* read string from tail */ + if (cirbuf_get_buf_head(&cb, tmp, sizeof(CIRBUF_STR_HEAD)) + != (sizeof(CIRBUF_STR_HEAD))) { + printf("Error: failed to get string from head!\n"); + return -1; + } + /* verify string */ + if (strncmp(tmp, CIRBUF_STR_HEAD, sizeof(CIRBUF_STR_HEAD)) != 0) { + printf("Error: headstrings do not match!\n"); + return -1; + } + + return 0; +} + +/* test adding and deleting strings */ +static int +test_cirbuf_string_add_del(void) +{ + struct cirbuf cb; + char buf[CMDLINE_TEST_BUFSIZE]; + char tmp[CMDLINE_TEST_BUFSIZE]; + + /* initialize buffers */ + memset(buf, 0, sizeof(buf)); + memset(tmp, 0, sizeof(tmp)); + + /* + * initialize circular buffer + */ + if (cirbuf_init(&cb, buf, 0, sizeof(buf)) < 0) { + printf("Error: failed to initialize circular buffer!\n"); + return -1; + } + + /* add string to head */ + if (cirbuf_add_buf_head(&cb, CIRBUF_STR_HEAD, sizeof(CIRBUF_STR_HEAD)) + != (sizeof(CIRBUF_STR_HEAD))) { + printf("Error: failed to add string to head!\n"); + return -1; + } + /* read string from head */ + if (cirbuf_get_buf_head(&cb, tmp, sizeof(CIRBUF_STR_HEAD)) + != (sizeof(CIRBUF_STR_HEAD))) { + printf("Error: failed to get string from head!\n"); + return -1; + } + /* verify string */ + if (strncmp(tmp, CIRBUF_STR_HEAD, sizeof(CIRBUF_STR_HEAD)) != 0) { + printf("Error: head strings do not match!\n"); + return -1; + } + /* clear tmp buffer */ + memset(tmp, 0, sizeof(tmp)); + /* read string from tail */ + if (cirbuf_get_buf_tail(&cb, tmp, sizeof(CIRBUF_STR_HEAD)) + != (sizeof(CIRBUF_STR_HEAD))) { + printf("Error: failed to get string from head!\n"); + return -1; + } + /* verify string */ + if (strncmp(tmp, CIRBUF_STR_HEAD, sizeof(CIRBUF_STR_HEAD)) != 0) { + printf("Error: head strings do not match!\n"); + return -1; + } + /* delete string from head*/ + if (cirbuf_del_buf_head(&cb, sizeof(CIRBUF_STR_HEAD)) < 0) { + printf("Error: failed to delete string from head!\n"); + return -1; + } + /* verify string was deleted */ + if (cirbuf_del_head_safe(&cb) == 0) { + printf("Error: buffer should have been empty!\n"); + return -1; + } + /* clear tmp buffer */ + memset(tmp, 0, sizeof(tmp)); + + + + /* + * reinitialize circular buffer + */ + memset(buf, 0, sizeof(buf)); + if (cirbuf_init(&cb, buf, 0, sizeof(buf)) < 0) { + printf("Error: failed to reinitialize circular buffer!\n"); + return -1; + } + + /* add string to tail */ + if (cirbuf_add_buf_tail(&cb, CIRBUF_STR_TAIL, sizeof(CIRBUF_STR_TAIL)) + != (sizeof(CIRBUF_STR_TAIL))) { + printf("Error: failed to add string to tail!\n"); + return -1; + } + /* get string from tail */ + if (cirbuf_get_buf_tail(&cb, tmp, sizeof(CIRBUF_STR_TAIL)) + != (sizeof(CIRBUF_STR_TAIL))) { + printf("Error: failed to get string from tail!\n"); + return -1; + } + /* verify string */ + if (strncmp(tmp, CIRBUF_STR_TAIL, sizeof(CIRBUF_STR_TAIL)) != 0) { + printf("Error: tail strings do not match!\n"); + return -1; + } + /* clear tmp buffer */ + memset(tmp, 0, sizeof(tmp)); + /* get string from head */ + if (cirbuf_get_buf_head(&cb, tmp, sizeof(CIRBUF_STR_TAIL)) + != (sizeof(CIRBUF_STR_TAIL))) { + printf("Error: failed to get string from tail!\n"); + return -1; + } + /* verify string */ + if (strncmp(tmp, CIRBUF_STR_TAIL, sizeof(CIRBUF_STR_TAIL)) != 0) { + printf("Error: tail strings do not match!\n"); + return -1; + } + /* delete string from tail */ + if (cirbuf_del_buf_tail(&cb, sizeof(CIRBUF_STR_TAIL)) < 0) { + printf("Error: failed to delete string from tail!\n"); + return -1; + } + /* verify string was deleted */ + if (cirbuf_del_tail_safe(&cb) == 0) { + printf("Error: buffer should have been empty!\n"); + return -1; + } + + return 0; +} + +/* test adding from head and deleting from tail, and vice versa */ +static int +test_cirbuf_string_add_del_reverse(void) +{ + struct cirbuf cb; + char buf[CMDLINE_TEST_BUFSIZE]; + char tmp[CMDLINE_TEST_BUFSIZE]; + + /* initialize buffers */ + memset(buf, 0, sizeof(buf)); + memset(tmp, 0, sizeof(tmp)); + + /* + * initialize circular buffer + */ + if (cirbuf_init(&cb, buf, 0, sizeof(buf)) < 0) { + printf("Error: failed to initialize circular buffer!\n"); + return -1; + } + + /* add string to head */ + if (cirbuf_add_buf_head(&cb, CIRBUF_STR_HEAD, sizeof(CIRBUF_STR_HEAD)) + != (sizeof(CIRBUF_STR_HEAD))) { + printf("Error: failed to add string to head!\n"); + return -1; + } + /* delete string from tail */ + if (cirbuf_del_buf_tail(&cb, sizeof(CIRBUF_STR_HEAD)) < 0) { + printf("Error: failed to delete string from tail!\n"); + return -1; + } + /* verify string was deleted */ + if (cirbuf_del_tail_safe(&cb) == 0) { + printf("Error: buffer should have been empty!\n"); + return -1; + } + /* clear tmp buffer */ + memset(tmp, 0, sizeof(tmp)); + + /* + * reinitialize circular buffer + */ + memset(buf, 0, sizeof(buf)); + if (cirbuf_init(&cb, buf, 0, sizeof(buf)) < 0) { + printf("Error: failed to reinitialize circular buffer!\n"); + return -1; + } + + /* add string to tail */ + if (cirbuf_add_buf_tail(&cb, CIRBUF_STR_TAIL, sizeof(CIRBUF_STR_TAIL)) + != (sizeof(CIRBUF_STR_TAIL))) { + printf("Error: failed to add string to tail!\n"); + return -1; + } + /* delete string from head */ + if (cirbuf_del_buf_head(&cb, sizeof(CIRBUF_STR_TAIL)) < 0) { + printf("Error: failed to delete string from head!\n"); + return -1; + } + /* verify string was deleted */ + if (cirbuf_del_head_safe(&cb) == 0) { + printf("Error: buffer should have been empty!\n"); + return -1; + } + + return 0; +} + +/* try to write more than available */ +static int +test_cirbuf_string_add_boundaries(void) +{ + struct cirbuf cb; + char buf[CMDLINE_TEST_BUFSIZE]; + unsigned i; + + /* initialize buffers */ + memset(buf, 0, sizeof(buf)); + + /* + * initialize circular buffer + */ + if (cirbuf_init(&cb, buf, 0, sizeof(buf)) < 0) { + printf("Error: failed to initialize circular buffer!\n"); + return -1; + } + + /* fill the buffer from tail */ + for (i = 0; i < CMDLINE_TEST_BUFSIZE - sizeof(CIRBUF_STR_TAIL) + 1; i++) + cirbuf_add_tail_safe(&cb, 't'); + + /* try adding a string to tail */ + if (cirbuf_add_buf_tail(&cb, CIRBUF_STR_TAIL, sizeof(CIRBUF_STR_TAIL)) + > 0) { + printf("Error: buffer should have been full!\n"); + return -1; + } + /* try adding a string to head */ + if (cirbuf_add_buf_head(&cb, CIRBUF_STR_TAIL, sizeof(CIRBUF_STR_TAIL)) + > 0) { + printf("Error: buffer should have been full!\n"); + return -1; + } + + /* + * reinitialize circular buffer + */ + memset(buf, 0, sizeof(buf)); + if (cirbuf_init(&cb, buf, 0, sizeof(buf)) < 0) { + printf("Error: failed to reinitialize circular buffer!\n"); + return -1; + } + + /* fill the buffer from head */ + for (i = 0; i < CMDLINE_TEST_BUFSIZE - sizeof(CIRBUF_STR_HEAD) + 1; i++) + cirbuf_add_head_safe(&cb, 'h'); + + /* try adding a string to head */ + if (cirbuf_add_buf_head(&cb, CIRBUF_STR_HEAD, sizeof(CIRBUF_STR_HEAD)) + > 0) { + printf("Error: buffer should have been full!\n"); + return -1; + } + /* try adding a string to tail */ + if (cirbuf_add_buf_tail(&cb, CIRBUF_STR_HEAD, sizeof(CIRBUF_STR_HEAD)) + > 0) { + printf("Error: buffer should have been full!\n"); + return -1; + } + + return 0; +} + +/* try to read/delete more than written */ +static int +test_cirbuf_string_get_del_boundaries(void) +{ + struct cirbuf cb; + char buf[CMDLINE_TEST_BUFSIZE]; + char tmp[CMDLINE_TEST_BUFSIZE]; + + /* initialize buffers */ + memset(buf, 0, sizeof(buf)); + memset(tmp, 0, sizeof(tmp)); + + /* + * initialize circular buffer + */ + if (cirbuf_init(&cb, buf, 0, sizeof(buf)) < 0) { + printf("Error: failed to initialize circular buffer!\n"); + return -1; + } + + + /* add string to head */ + if (cirbuf_add_buf_head(&cb, CIRBUF_STR_HEAD, sizeof(CIRBUF_STR_HEAD)) + != (sizeof(CIRBUF_STR_HEAD))) { + printf("Error: failed to add string to head!\n"); + return -1; + } + /* read more than written (head) */ + if (cirbuf_get_buf_head(&cb, tmp, sizeof(CIRBUF_STR_HEAD) + 1) + != sizeof(CIRBUF_STR_HEAD)) { + printf("Error: unexpected result when reading too much data!\n"); + return -1; + } + /* read more than written (tail) */ + if (cirbuf_get_buf_tail(&cb, tmp, sizeof(CIRBUF_STR_HEAD) + 1) + != sizeof(CIRBUF_STR_HEAD)) { + printf("Error: unexpected result when reading too much data!\n"); + return -1; + } + /* delete more than written (head) */ + if (cirbuf_del_buf_head(&cb, sizeof(CIRBUF_STR_HEAD) + 1) == 0) { + printf("Error: unexpected result when deleting too much data!\n"); + return -1; + } + /* delete more than written (tail) */ + if (cirbuf_del_buf_tail(&cb, sizeof(CIRBUF_STR_HEAD) + 1) == 0) { + printf("Error: unexpected result when deleting too much data!\n"); + return -1; + } + + /* + * reinitialize circular buffer + */ + memset(buf, 0, sizeof(buf)); + if (cirbuf_init(&cb, buf, 0, sizeof(buf)) < 0) { + printf("Error: failed to reinitialize circular buffer!\n"); + return -1; + } + + /* add string to tail */ + if (cirbuf_add_buf_tail(&cb, CIRBUF_STR_TAIL, sizeof(CIRBUF_STR_TAIL)) + != (sizeof(CIRBUF_STR_TAIL))) { + printf("Error: failed to add string to tail!\n"); + return -1; + } + /* read more than written (tail) */ + if (cirbuf_get_buf_tail(&cb, tmp, sizeof(CIRBUF_STR_TAIL) + 1) + != sizeof(CIRBUF_STR_TAIL)) { + printf("Error: unexpected result when reading too much data!\n"); + return -1; + } + /* read more than written (head) */ + if (cirbuf_get_buf_head(&cb, tmp, sizeof(CIRBUF_STR_TAIL) + 1) + != sizeof(CIRBUF_STR_TAIL)) { + printf("Error: unexpected result when reading too much data!\n"); + return -1; + } + /* delete more than written (tail) */ + if (cirbuf_del_buf_tail(&cb, sizeof(CIRBUF_STR_TAIL) + 1) == 0) { + printf("Error: unexpected result when deleting too much data!\n"); + return -1; + } + /* delete more than written (head) */ + if (cirbuf_del_buf_tail(&cb, sizeof(CIRBUF_STR_TAIL) + 1) == 0) { + printf("Error: unexpected result when deleting too much data!\n"); + return -1; + } + + return 0; +} + +/* try to read/delete less than written */ +static int +test_cirbuf_string_get_del_partial(void) +{ + struct cirbuf cb; + char buf[CMDLINE_TEST_BUFSIZE]; + char tmp[CMDLINE_TEST_BUFSIZE]; + char tmp2[CMDLINE_TEST_BUFSIZE]; + + /* initialize buffers */ + memset(buf, 0, sizeof(buf)); + memset(tmp, 0, sizeof(tmp)); + memset(tmp2, 0, sizeof(tmp)); + + snprintf(tmp2, sizeof(tmp2), "%s", CIRBUF_STR_HEAD); + + /* + * initialize circular buffer + */ + if (cirbuf_init(&cb, buf, 0, sizeof(buf)) < 0) { + printf("Error: failed to initialize circular buffer!\n"); + return -1; + } + + /* add string to head */ + if (cirbuf_add_buf_head(&cb, CIRBUF_STR_HEAD, sizeof(CIRBUF_STR_HEAD)) + != (sizeof(CIRBUF_STR_HEAD))) { + printf("Error: failed to add string to head!\n"); + return -1; + } + /* read less than written (head) */ + if (cirbuf_get_buf_head(&cb, tmp, sizeof(CIRBUF_STR_HEAD) - 1) + != sizeof(CIRBUF_STR_HEAD) - 1) { + printf("Error: unexpected result when reading from head!\n"); + return -1; + } + /* verify string */ + if (strncmp(tmp, tmp2, sizeof(CIRBUF_STR_HEAD) - 1) != 0) { + printf("Error: strings mismatch!\n"); + return -1; + } + memset(tmp, 0, sizeof(tmp)); + /* read less than written (tail) */ + if (cirbuf_get_buf_tail(&cb, tmp, sizeof(CIRBUF_STR_HEAD) - 1) + != sizeof(CIRBUF_STR_HEAD) - 1) { + printf("Error: unexpected result when reading from tail!\n"); + return -1; + } + /* verify string */ + if (strncmp(tmp, &tmp2[1], sizeof(CIRBUF_STR_HEAD) - 1) != 0) { + printf("Error: strings mismatch!\n"); + return -1; + } + + /* + * verify correct deletion + */ + + /* clear buffer */ + memset(tmp, 0, sizeof(tmp)); + + /* delete less than written (head) */ + if (cirbuf_del_buf_head(&cb, 1) != 0) { + printf("Error: delete from head failed!\n"); + return -1; + } + /* read from head */ + if (cirbuf_get_buf_head(&cb, tmp, sizeof(CIRBUF_STR_HEAD) - 1) + != sizeof(CIRBUF_STR_HEAD) - 1) { + printf("Error: unexpected result when reading from head!\n"); + return -1; + } + /* since we deleted from head, first char should be deleted */ + if (strncmp(tmp, &tmp2[1], sizeof(CIRBUF_STR_HEAD) - 1) != 0) { + printf("Error: strings mismatch!\n"); + return -1; + } + /* clear buffer */ + memset(tmp, 0, sizeof(tmp)); + + /* delete less than written (tail) */ + if (cirbuf_del_buf_tail(&cb, 1) != 0) { + printf("Error: delete from tail failed!\n"); + return -1; + } + /* read from tail */ + if (cirbuf_get_buf_tail(&cb, tmp, sizeof(CIRBUF_STR_HEAD) - 2) + != sizeof(CIRBUF_STR_HEAD) - 2) { + printf("Error: unexpected result when reading from head!\n"); + return -1; + } + /* since we deleted from tail, last char should be deleted */ + if (strncmp(tmp, &tmp2[1], sizeof(CIRBUF_STR_HEAD) - 2) != 0) { + printf("Error: strings mismatch!\n"); + return -1; + } + + return 0; +} + +/* test cmdline_cirbuf char add/del functions */ +static int +test_cirbuf_char_add_del(void) +{ + struct cirbuf cb; + char buf[CMDLINE_TEST_BUFSIZE]; + char tmp[CMDLINE_TEST_BUFSIZE]; + + /* clear buffer */ + memset(buf, 0, sizeof(buf)); + memset(tmp, 0, sizeof(tmp)); + + /* + * initialize circular buffer + */ + if (cirbuf_init(&cb, buf, 0, sizeof(buf)) < 0) { + printf("Error: failed to initialize circular buffer!\n"); + return -1; + } + + /* + * try to delete something from cirbuf. since it's empty, + * these should fail. + */ + if (cirbuf_del_head_safe(&cb) == 0) { + printf("Error: deleting from empty cirbuf head succeeded!\n"); + return -1; + } + if (cirbuf_del_tail_safe(&cb) == 0) { + printf("Error: deleting from empty cirbuf tail succeeded!\n"); + return -1; + } + + /* + * add, verify and delete. these should pass. + */ + if (cirbuf_add_head_safe(&cb,'h') < 0) { + printf("Error: adding to cirbuf head failed!\n"); + return -1; + } + if (cirbuf_get_head(&cb) != 'h') { + printf("Error: wrong head content!\n"); + return -1; + } + if (cirbuf_del_head_safe(&cb) < 0) { + printf("Error: deleting from cirbuf head failed!\n"); + return -1; + } + if (cirbuf_add_tail_safe(&cb,'t') < 0) { + printf("Error: adding to cirbuf tail failed!\n"); + return -1; + } + if (cirbuf_get_tail(&cb) != 't') { + printf("Error: wrong tail content!\n"); + return -1; + } + if (cirbuf_del_tail_safe(&cb) < 0) { + printf("Error: deleting from cirbuf tail failed!\n"); + return -1; + } + /* do the same for unsafe versions. those are void. */ + cirbuf_add_head(&cb,'h'); + if (cirbuf_get_head(&cb) != 'h') { + printf("Error: wrong head content!\n"); + return -1; + } + cirbuf_del_head(&cb); + + /* test if char has been deleted. we can't call cirbuf_get_head + * because it's unsafe, but we can call cirbuf_get_buf_head. + */ + if (cirbuf_get_buf_head(&cb, tmp, 1) > 0) { + printf("Error: buffer should have been empty!\n"); + return -1; + } + + cirbuf_add_tail(&cb,'t'); + if (cirbuf_get_tail(&cb) != 't') { + printf("Error: wrong tail content!\n"); + return -1; + } + cirbuf_del_tail(&cb); + + /* test if char has been deleted. we can't call cirbuf_get_tail + * because it's unsafe, but we can call cirbuf_get_buf_tail. + */ + if (cirbuf_get_buf_tail(&cb, tmp, 1) > 0) { + printf("Error: buffer should have been empty!\n"); + return -1; + } + + return 0; +} + +/* test filling up buffer with chars */ +static int +test_cirbuf_char_fill(void) +{ + struct cirbuf cb; + char buf[CMDLINE_TEST_BUFSIZE]; + unsigned i; + + /* clear buffer */ + memset(buf, 0, sizeof(buf)); + + /* + * initialize circular buffer + */ + if (cirbuf_init(&cb, buf, 0, sizeof(buf)) < 0) { + printf("Error: failed to initialize circular buffer!\n"); + return -1; + } + + /* + * fill the buffer from head or tail, verify contents, test boundaries + * and clear the buffer + */ + + /* fill the buffer from tail */ + for (i = 0; i < CMDLINE_TEST_BUFSIZE; i++) + cirbuf_add_tail_safe(&cb, 't'); + /* verify that contents of the buffer are what they are supposed to be */ + for (i = 0; i < sizeof(buf); i++) { + if (buf[i] != 't') { + printf("Error: wrong content in buffer!\n"); + return -1; + } + } + /* try to add to a full buffer from tail */ + if (cirbuf_add_tail_safe(&cb, 't') == 0) { + printf("Error: buffer should have been full!\n"); + return -1; + } + /* try to add to a full buffer from head */ + if (cirbuf_add_head_safe(&cb, 'h') == 0) { + printf("Error: buffer should have been full!\n"); + return -1; + } + /* delete buffer from tail */ + for(i = 0; i < CMDLINE_TEST_BUFSIZE; i++) + cirbuf_del_tail_safe(&cb); + /* try to delete from an empty buffer */ + if (cirbuf_del_tail_safe(&cb) >= 0) { + printf("Error: buffer should have been empty!\n"); + return -1; + } + + /* fill the buffer from head */ + for (i = 0; i < CMDLINE_TEST_BUFSIZE; i++) + cirbuf_add_head_safe(&cb, 'h'); + /* verify that contents of the buffer are what they are supposed to be */ + for (i = 0; i < sizeof(buf); i++) { + if (buf[i] != 'h') { + printf("Error: wrong content in buffer!\n"); + return -1; + } + } + /* try to add to a full buffer from head */ + if (cirbuf_add_head_safe(&cb,'h') >= 0) { + printf("Error: buffer should have been full!\n"); + return -1; + } + /* try to add to a full buffer from tail */ + if (cirbuf_add_tail_safe(&cb, 't') == 0) { + printf("Error: buffer should have been full!\n"); + return -1; + } + /* delete buffer from head */ + for(i = 0; i < CMDLINE_TEST_BUFSIZE; i++) + cirbuf_del_head_safe(&cb); + /* try to delete from an empty buffer */ + if (cirbuf_del_head_safe(&cb) >= 0) { + printf("Error: buffer should have been empty!\n"); + return -1; + } + + /* + * fill the buffer from both head and tail, with alternating characters, + * verify contents and clear the buffer + */ + + /* fill half of buffer from tail */ + for (i = 0; i < CMDLINE_TEST_BUFSIZE / 2; i++) + cirbuf_add_tail_safe(&cb, (char) (i % 2 ? 't' : 'T')); + /* fill other half of the buffer from head */ + for (i = 0; i < CMDLINE_TEST_BUFSIZE / 2; i++) + cirbuf_add_head_safe(&cb, (char) (i % 2 ? 'H' : 'h')); /* added in reverse */ + + /* verify that contents of the buffer are what they are supposed to be */ + for (i = 0; i < sizeof(buf) / 2; i++) { + if (buf[i] != (char) (i % 2 ? 't' : 'T')) { + printf("Error: wrong content in buffer at %u!\n", i); + return -1; + } + } + for (i = sizeof(buf) / 2; i < sizeof(buf); i++) { + if (buf[i] != (char) (i % 2 ? 'h' : 'H')) { + printf("Error: wrong content in buffer %u!\n", i); + return -1; + } + } + + return 0; +} + +/* test left alignment */ +static int +test_cirbuf_align_left(void) +{ +#define HALF_OFFSET CMDLINE_TEST_BUFSIZE / 2 +#define SMALL_OFFSET HALF_OFFSET / 2 +/* resulting buffer lengths for each of the test cases */ +#define LEN1 HALF_OFFSET - SMALL_OFFSET - 1 +#define LEN2 HALF_OFFSET + SMALL_OFFSET + 2 +#define LEN3 HALF_OFFSET - SMALL_OFFSET +#define LEN4 HALF_OFFSET + SMALL_OFFSET - 1 + + struct cirbuf cb; + char buf[CMDLINE_TEST_BUFSIZE]; + char tmp[CMDLINE_TEST_BUFSIZE]; + unsigned i; + + /* + * align left when start < end and start in left half + */ + + /* + * initialize circular buffer + */ + memset(buf, 0, sizeof(buf)); + if (cirbuf_init(&cb, buf, 0, sizeof(buf)) < 0) { + printf("Error: failed to initialize circular buffer!\n"); + return -1; + } + + /* push end into left half */ + for (i = 0; i < HALF_OFFSET - 1; i++) + cirbuf_add_tail_safe(&cb, 't'); + + /* push start into left half < end */ + for (i = 0; i < SMALL_OFFSET; i++) + cirbuf_del_head_safe(&cb); + + /* align */ + if (cirbuf_align_left(&cb) < 0) { + printf("Error: alignment failed!\n"); + return -1; + } + + /* verify result */ + if (cb.start != 0 || cb.len != LEN1 || cb.end != cb.len - 1) { + printf("Error: buffer alignment is wrong!\n"); + return -1; + } + + /* + * align left when start > end and start in left half + */ + + /* + * reinitialize circular buffer + */ + memset(buf, 0, sizeof(buf)); + if (cirbuf_init(&cb, buf, 0, sizeof(buf)) < 0) { + printf("Error: failed to reinitialize circular buffer!\n"); + return -1; + } + + /* push start into left half */ + for (i = 0; i < HALF_OFFSET + 2; i++) + cirbuf_add_head_safe(&cb, 'h'); + + /* push end into left half > start */ + for (i = 0; i < SMALL_OFFSET; i++) + cirbuf_add_tail_safe(&cb, 't'); + + /* align */ + if (cirbuf_align_left(&cb) < 0) { + printf("Error: alignment failed!\n"); + return -1; + } + + /* verify result */ + if (cb.start != 0 || cb.len != LEN2 || cb.end != cb.len - 1) { + printf("Error: buffer alignment is wrong!"); + return -1; + } + + /* + * align left when start < end and start in right half + */ + + /* + * reinitialize circular buffer + */ + memset(buf, 0, sizeof(buf)); + if (cirbuf_init(&cb, buf, 0, sizeof(buf)) < 0) { + printf("Error: failed to reinitialize circular buffer!\n"); + return -1; + } + + /* push start into the right half */ + for (i = 0; i < HALF_OFFSET; i++) + cirbuf_add_head_safe(&cb, 'h'); + + /* push end into left half > start */ + for (i = 0; i < SMALL_OFFSET; i++) + cirbuf_del_tail_safe(&cb); + + /* align */ + if (cirbuf_align_left(&cb) < 0) { + printf("Error: alignment failed!\n"); + return -1; + } + + /* verify result */ + if (cb.start != 0 || cb.len != LEN3 || cb.end != cb.len - 1) { + printf("Error: buffer alignment is wrong!"); + return -1; + } + + /* + * align left when start > end and start in right half + */ + + /* + * reinitialize circular buffer + */ + memset(buf, 0, sizeof(buf)); + if (cirbuf_init(&cb, buf, 0, sizeof(buf)) < 0) { + printf("Error: failed to reinitialize circular buffer!\n"); + return -1; + } + + /* push start into the right half */ + for (i = 0; i < HALF_OFFSET - 1; i++) + cirbuf_add_head_safe(&cb, 'h'); + + /* push end into left half < start */ + for (i = 0; i < SMALL_OFFSET; i++) + cirbuf_add_tail_safe(&cb, 't'); + + /* align */ + if (cirbuf_align_left(&cb) < 0) { + printf("Error: alignment failed!\n"); + return -1; + } + + /* verify result */ + if (cb.start != 0 || cb.len != LEN4 || + cb.end != cb.len - 1) { + printf("Error: buffer alignment is wrong!"); + return -1; + } + + /* + * Verify that alignment doesn't corrupt data + */ + + /* + * reinitialize circular buffer + */ + memset(buf, 0, sizeof(buf)); + if (cirbuf_init(&cb, buf, 0, sizeof(buf)) < 0) { + printf("Error: failed to reinitialize circular buffer!\n"); + return -1; + } + + /* add string to tail and head */ + if (cirbuf_add_buf_head(&cb, CIRBUF_STR_HEAD, + sizeof(CIRBUF_STR_HEAD)) < 0 || cirbuf_add_buf_tail(&cb, + CIRBUF_STR_TAIL, sizeof(CIRBUF_STR_TAIL)) < 0) { + printf("Error: failed to add strings!\n"); + return -1; + } + + /* align */ + if (cirbuf_align_left(&cb) < 0) { + printf("Error: alignment failed!\n"); + return -1; + } + + /* get string from head */ + if (cirbuf_get_buf_head(&cb, tmp, + sizeof(CIRBUF_STR_HEAD) + sizeof(CIRBUF_STR_TAIL)) < 0) { + printf("Error: failed to read string from head!\n"); + return -1; + } + + /* verify string */ + if (strncmp(tmp, CIRBUF_STR_HEAD "\0" CIRBUF_STR_TAIL, + sizeof(CIRBUF_STR_HEAD) + sizeof(CIRBUF_STR_TAIL)) != 0) { + printf("Error: strings mismatch!\n"); + return -1; + } + + /* reset tmp buffer */ + memset(tmp, 0, sizeof(tmp)); + + /* get string from tail */ + if (cirbuf_get_buf_tail(&cb, tmp, + sizeof(CIRBUF_STR_HEAD) + sizeof(CIRBUF_STR_TAIL)) < 0) { + printf("Error: failed to read string from head!\n"); + return -1; + } + + /* verify string */ + if (strncmp(tmp, CIRBUF_STR_HEAD "\0" CIRBUF_STR_TAIL, + sizeof(CIRBUF_STR_HEAD) + sizeof(CIRBUF_STR_TAIL)) != 0) { + printf("Error: strings mismatch!\n"); + return -1; + } + + return 0; +} + +/* test right alignment */ +static int +test_cirbuf_align_right(void) +{ +#define END_OFFSET CMDLINE_TEST_BUFSIZE - 1 + struct cirbuf cb; + char buf[CMDLINE_TEST_BUFSIZE]; + char tmp[CMDLINE_TEST_BUFSIZE]; + unsigned i; + + + /* + * align right when start < end and start in left half + */ + + /* + * initialize circular buffer + */ + memset(buf, 0, sizeof(buf)); + if (cirbuf_init(&cb, buf, 0, sizeof(buf)) < 0) { + printf("Error: failed to initialize circular buffer!\n"); + return -1; + } + + /* push end into left half */ + for (i = 0; i < HALF_OFFSET - 1; i++) + cirbuf_add_tail_safe(&cb, 't'); + + /* push start into left half < end */ + for (i = 0; i < SMALL_OFFSET; i++) + cirbuf_del_head_safe(&cb); + + /* align */ + cirbuf_align_right(&cb); + + /* verify result */ + if (cb.start != END_OFFSET || cb.len != LEN1 || cb.end != cb.len - 2) { + printf("Error: buffer alignment is wrong!\n"); + return -1; + } + + /* + * align right when start > end and start in left half + */ + + /* + * reinitialize circular buffer + */ + memset(buf, 0, sizeof(buf)); + if (cirbuf_init(&cb, buf, 0, sizeof(buf)) < 0) { + printf("Error: failed to reinitialize circular buffer!\n"); + return -1; + } + + /* push start into left half */ + for (i = 0; i < HALF_OFFSET + 2; i++) + cirbuf_add_head_safe(&cb, 'h'); + + /* push end into left half > start */ + for (i = 0; i < SMALL_OFFSET; i++) + cirbuf_add_tail_safe(&cb, 't'); + + /* align */ + cirbuf_align_right(&cb); + + /* verify result */ + if (cb.start != END_OFFSET || cb.len != LEN2 || cb.end != cb.len - 2) { + printf("Error: buffer alignment is wrong!"); + return -1; + } + + /* + * align right when start < end and start in right half + */ + + /* + * reinitialize circular buffer + */ + memset(buf, 0, sizeof(buf)); + if (cirbuf_init(&cb, buf, 0, sizeof(buf)) < 0) { + printf("Error: failed to reinitialize circular buffer!\n"); + return -1; + } + + /* push start into the right half */ + for (i = 0; i < HALF_OFFSET; i++) + cirbuf_add_head_safe(&cb, 'h'); + + /* push end into left half > start */ + for (i = 0; i < SMALL_OFFSET; i++) + cirbuf_del_tail_safe(&cb); + + /* align */ + cirbuf_align_right(&cb); + + /* verify result */ + if (cb.end != END_OFFSET || cb.len != LEN3 || cb.start != cb.end - cb.len + 1) { + printf("Error: buffer alignment is wrong!"); + return -1; + } + + /* + * align right when start > end and start in right half + */ + + /* + * reinitialize circular buffer + */ + memset(buf, 0, sizeof(buf)); + if (cirbuf_init(&cb, buf, 0, sizeof(buf)) < 0) { + printf("Error: failed to reinitialize circular buffer!\n"); + return -1; + } + + /* push start into the right half */ + for (i = 0; i < HALF_OFFSET - 1; i++) + cirbuf_add_head_safe(&cb, 'h'); + + /* push end into left half < start */ + for (i = 0; i < SMALL_OFFSET; i++) + cirbuf_add_tail_safe(&cb, 't'); + + /* align */ + cirbuf_align_right(&cb); + + /* verify result */ + if (cb.end != END_OFFSET || cb.len != LEN4 || cb.start != cb.end - cb.len + 1) { + printf("Error: buffer alignment is wrong!"); + return -1; + } + + /* + * Verify that alignment doesn't corrupt data + */ + + /* + * reinitialize circular buffer + */ + memset(buf, 0, sizeof(buf)); + if (cirbuf_init(&cb, buf, 0, sizeof(buf)) < 0) { + printf("Error: failed to reinitialize circular buffer!\n"); + return -1; + } + + /* add string to tail and head */ + if (cirbuf_add_buf_tail(&cb, CIRBUF_STR_TAIL, + sizeof(CIRBUF_STR_TAIL)) < 0 || cirbuf_add_buf_head(&cb, + CIRBUF_STR_HEAD, sizeof(CIRBUF_STR_HEAD)) < 0) { + printf("Error: failed to add strings!\n"); + return -1; + } + + /* align */ + if (cirbuf_align_right(&cb) < 0) { + printf("Error: alignment failed!\n"); + return -1; + } + + /* get string from head */ + if (cirbuf_get_buf_head(&cb, tmp, + sizeof(CIRBUF_STR_HEAD) + sizeof(CIRBUF_STR_TAIL)) < 0) { + printf("Error: failed to read string from head!\n"); + return -1; + } + + /* verify string */ + if (strncmp(tmp, CIRBUF_STR_HEAD "\0" CIRBUF_STR_TAIL, + sizeof(CIRBUF_STR_HEAD) + sizeof(CIRBUF_STR_TAIL)) != 0) { + printf("Error: strings mismatch!\n"); + return -1; + } + + /* reset tmp buffer */ + memset(tmp, 0, sizeof(tmp)); + + /* get string from tail */ + if (cirbuf_get_buf_tail(&cb, tmp, + sizeof(CIRBUF_STR_HEAD) + sizeof(CIRBUF_STR_TAIL)) < 0) { + printf("Error: failed to read string from head!\n"); + return -1; + } + /* verify string */ + if (strncmp(tmp, CIRBUF_STR_HEAD "\0" CIRBUF_STR_TAIL, + sizeof(CIRBUF_STR_HEAD) + sizeof(CIRBUF_STR_TAIL)) != 0) { + printf("Error: strings mismatch!\n"); + return -1; + } + + return 0; +} + +/* call functions with invalid parameters */ +int +test_cirbuf_invalid_param(void) +{ + struct cirbuf cb; + char buf[CMDLINE_TEST_BUFSIZE]; + + /* null cirbuf */ + if (cirbuf_init(0, buf, 0, sizeof(buf)) == 0) + return -1; + /* null buffer */ + if (cirbuf_init(&cb, 0, 0, sizeof(buf)) == 0) + return -1; + /* null cirbuf */ + if (cirbuf_add_head_safe(0, 'h') == 0) + return -1; + if (cirbuf_add_tail_safe(0, 't') == 0) + return -1; + if (cirbuf_del_head_safe(0) == 0) + return -1; + if (cirbuf_del_tail_safe(0) == 0) + return -1; + /* null buffer */ + if (cirbuf_add_buf_head(&cb, 0, 0) == 0) + return -1; + if (cirbuf_add_buf_tail(&cb, 0, 0) == 0) + return -1; + /* null cirbuf */ + if (cirbuf_add_buf_head(0, buf, 0) == 0) + return -1; + if (cirbuf_add_buf_tail(0, buf, 0) == 0) + return -1; + /* null size */ + if (cirbuf_add_buf_head(&cb, buf, 0) == 0) + return -1; + if (cirbuf_add_buf_tail(&cb, buf, 0) == 0) + return -1; + /* null cirbuf */ + if (cirbuf_del_buf_head(0, 0) == 0) + return -1; + if (cirbuf_del_buf_tail(0, 0) == 0) + return -1; + /* null size */ + if (cirbuf_del_buf_head(&cb, 0) == 0) + return -1; + if (cirbuf_del_buf_tail(&cb, 0) == 0) + return -1; + /* null cirbuf */ + if (cirbuf_get_buf_head(0, 0, 0) == 0) + return -1; + if (cirbuf_get_buf_tail(0, 0, 0) == 0) + return -1; + /* null buffer */ + if (cirbuf_get_buf_head(&cb, 0, 0) == 0) + return -1; + if (cirbuf_get_buf_tail(&cb, 0, 0) == 0) + return -1; + /* null size, this is valid but should return 0 */ + if (cirbuf_get_buf_head(&cb, buf, 0) != 0) + return -1; + if (cirbuf_get_buf_tail(&cb, buf, 0) != 0) + return -1; + /* null cirbuf */ + if (cirbuf_align_left(0) == 0) + return -1; + if (cirbuf_align_right(0) == 0) + return -1; + + return 0; +} + +/* test cmdline_cirbuf char functions */ +int +test_cirbuf_char(void) +{ + int ret; + + ret = test_cirbuf_char_add_del(); + if (ret < 0) + return -1; + + ret = test_cirbuf_char_fill(); + if (ret < 0) + return -1; + + return 0; +} + +/* test cmdline_cirbuf string functions */ +int +test_cirbuf_string(void) +{ + if (test_cirbuf_string_add_del() < 0) + return -1; + + if (test_cirbuf_string_add_del_reverse() < 0) + return -1; + + if (test_cirbuf_string_add_boundaries() < 0) + return -1; + + if (test_cirbuf_string_get_del_boundaries() < 0) + return -1; + + if (test_cirbuf_string_get_del_partial() < 0) + return -1; + + if (test_cirbuf_string_misc() < 0) + return -1; + + return 0; +} + +/* test cmdline_cirbuf align functions */ +int +test_cirbuf_align(void) +{ + if (test_cirbuf_align_left() < 0) + return -1; + if (test_cirbuf_align_right() < 0) + return -1; + return 0; +} diff --git a/dpdk/test/test/test_cmdline_etheraddr.c b/dpdk/test/test/test_cmdline_etheraddr.c new file mode 100644 index 00000000..e4f42317 --- /dev/null +++ b/dpdk/test/test/test_cmdline_etheraddr.c @@ -0,0 +1,247 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <string.h> +#include <inttypes.h> + +#include <rte_ether.h> +#include <rte_string_fns.h> + +#include <cmdline_parse.h> +#include <cmdline_parse_etheraddr.h> + +#include "test_cmdline.h" + +struct ether_addr_str { + const char * str; + uint64_t address; +}; + +/* valid strings */ +const struct ether_addr_str ether_addr_valid_strs[] = { + {"01:23:45:67:89:AB", 0xAB8967452301ULL}, + {"4567:89AB:CDEF", 0xEFCDAB896745ULL}, +}; + +/* valid strings with various garbage at the end. + * these strings are still valid because parser checks for + * end of token, which is either space chars, null char or + * a hash sign. + */ +const char * ether_addr_garbage_strs[] = { + "00:11:22:33:44:55\0garbage", + "00:11:22:33:44:55#garbage", + "00:11:22:33:44:55 garbage", + "00:11:22:33:44:55\tgarbage", + "00:11:22:33:44:55\ngarbage", + "00:11:22:33:44:55\rgarbage", + "00:11:22:33:44:55#", + "00:11:22:33:44:55 ", + "00:11:22:33:44:55\t", + "00:11:22:33:44:55\n", + "00:11:22:33:44:55\r", +}; +#define GARBAGE_ETHERADDR 0x554433221100ULL /* corresponding address */ + + +const char * ether_addr_invalid_strs[] = { + /* valid chars, invalid syntax */ + "0123:45:67:89:AB", + "01:23:4567:89:AB", + "01:23:45:67:89AB", + "012:345:678:9AB", + "01:23:45:67:89:ABC", + "01:23:45:67:89:A", + "01:23:45:67:89", + "01:23:45:67:89:AB:CD", + /* invalid chars, valid syntax */ + "IN:VA:LI:DC:HA:RS", + "INVA:LIDC:HARS", + /* misc */ + "01 23 45 67 89 AB", + "01.23.45.67.89.AB", + "01,23,45,67,89,AB", + "01:23:45\0:67:89:AB", + "01:23:45#:67:89:AB", + "random invalid text", + "random text", + "", + "\0", + " ", +}; + +#define ETHERADDR_VALID_STRS_SIZE \ + (sizeof(ether_addr_valid_strs) / sizeof(ether_addr_valid_strs[0])) +#define ETHERADDR_GARBAGE_STRS_SIZE \ + (sizeof(ether_addr_garbage_strs) / sizeof(ether_addr_garbage_strs[0])) +#define ETHERADDR_INVALID_STRS_SIZE \ + (sizeof(ether_addr_invalid_strs) / sizeof(ether_addr_invalid_strs[0])) + + + +static int +is_addr_different(const struct ether_addr addr, uint64_t num) +{ + int i; + for (i = 0; i < ETHER_ADDR_LEN; i++, num >>= 8) + if (addr.addr_bytes[i] != (num & 0xFF)) { + return 1; + } + return 0; +} + +/* test invalid parameters */ +int +test_parse_etheraddr_invalid_param(void) +{ + char buf[CMDLINE_TEST_BUFSIZE]; + struct ether_addr result; + int ret = 0; + + /* try all null */ + ret = cmdline_parse_etheraddr(NULL, NULL, NULL, 0); + if (ret != -1) { + printf("Error: parser accepted null parameters!\n"); + return -1; + } + + /* try null buf */ + ret = cmdline_parse_etheraddr(NULL, NULL, (void*)&result, + sizeof(result)); + if (ret != -1) { + printf("Error: parser accepted null string!\n"); + return -1; + } + + /* try null result */ + + /* copy string to buffer */ + snprintf(buf, sizeof(buf), "%s", + ether_addr_valid_strs[0].str); + + ret = cmdline_parse_etheraddr(NULL, buf, NULL, 0); + if (ret == -1) { + printf("Error: parser rejected null result!\n"); + return -1; + } + + /* token is not used in ether_parse anyway so there's no point in + * testing it */ + + /* test help function */ + memset(&buf, 0, sizeof(buf)); + + /* coverage! */ + ret = cmdline_get_help_etheraddr(NULL, buf, sizeof(buf)); + if (ret < 0) { + printf("Error: help function failed with valid parameters!\n"); + return -1; + } + + return 0; +} + +/* test valid parameters but invalid data */ +int +test_parse_etheraddr_invalid_data(void) +{ + int ret = 0; + unsigned i; + struct ether_addr result; + + /* test full strings */ + for (i = 0; i < ETHERADDR_INVALID_STRS_SIZE; i++) { + + memset(&result, 0, sizeof(struct ether_addr)); + + ret = cmdline_parse_etheraddr(NULL, ether_addr_invalid_strs[i], + (void*)&result, sizeof(result)); + if (ret != -1) { + printf("Error: parsing %s succeeded!\n", + ether_addr_invalid_strs[i]); + return -1; + } + } + + return 0; +} + +/* test valid parameters and data */ +int +test_parse_etheraddr_valid(void) +{ + int ret = 0; + unsigned i; + struct ether_addr result; + + /* test full strings */ + for (i = 0; i < ETHERADDR_VALID_STRS_SIZE; i++) { + + memset(&result, 0, sizeof(struct ether_addr)); + + ret = cmdline_parse_etheraddr(NULL, ether_addr_valid_strs[i].str, + (void*)&result, sizeof(result)); + if (ret < 0) { + printf("Error: parsing %s failed!\n", + ether_addr_valid_strs[i].str); + return -1; + } + if (is_addr_different(result, ether_addr_valid_strs[i].address)) { + printf("Error: parsing %s failed: address mismatch!\n", + ether_addr_valid_strs[i].str); + return -1; + } + } + + /* test garbage strings */ + for (i = 0; i < ETHERADDR_GARBAGE_STRS_SIZE; i++) { + + memset(&result, 0, sizeof(struct ether_addr)); + + ret = cmdline_parse_etheraddr(NULL, ether_addr_garbage_strs[i], + (void*)&result, sizeof(result)); + if (ret < 0) { + printf("Error: parsing %s failed!\n", + ether_addr_garbage_strs[i]); + return -1; + } + if (is_addr_different(result, GARBAGE_ETHERADDR)) { + printf("Error: parsing %s failed: address mismatch!\n", + ether_addr_garbage_strs[i]); + return -1; + } + } + + return 0; +} diff --git a/dpdk/test/test/test_cmdline_ipaddr.c b/dpdk/test/test/test_cmdline_ipaddr.c new file mode 100644 index 00000000..471d2ff1 --- /dev/null +++ b/dpdk/test/test/test_cmdline_ipaddr.c @@ -0,0 +1,722 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <string.h> +#include <inttypes.h> +#include <netinet/in.h> + +#ifndef __linux__ +#ifndef __FreeBSD__ +#include <net/socket.h> +#else +#include <sys/socket.h> +#endif +#endif + +#include <rte_string_fns.h> + +#include <cmdline_parse.h> +#include <cmdline_parse_ipaddr.h> + +#include "test_cmdline.h" + +#define IP4(a,b,c,d) {((uint32_t)(((a) & 0xff)) | \ + (((b) & 0xff) << 8) | \ + (((c) & 0xff) << 16) | \ + ((d) & 0xff) << 24)} + +#define U16_SWAP(x) \ + (((x & 0xFF) << 8) | ((x & 0xFF00) >> 8)) + +/* create IPv6 address, swapping bytes where needed */ +#ifndef s6_addr16 +# define s6_addr16 __u6_addr.__u6_addr16 +#endif +#define IP6(a,b,c,d,e,f,g,h) .ipv6 = \ + {.s6_addr16 = \ + {U16_SWAP(a),U16_SWAP(b),U16_SWAP(c),U16_SWAP(d),\ + U16_SWAP(e),U16_SWAP(f),U16_SWAP(g),U16_SWAP(h)}} + +/** these are defined in netinet/in.h but not present in linux headers */ +#ifndef NIPQUAD + +#define NIPQUAD_FMT "%u.%u.%u.%u" +#define NIPQUAD(addr) \ + (unsigned)((unsigned char *)&addr)[0], \ + (unsigned)((unsigned char *)&addr)[1], \ + (unsigned)((unsigned char *)&addr)[2], \ + (unsigned)((unsigned char *)&addr)[3] + +#define NIP6_FMT "%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x" +#define NIP6(addr) \ + (unsigned)((addr).s6_addr[0]), \ + (unsigned)((addr).s6_addr[1]), \ + (unsigned)((addr).s6_addr[2]), \ + (unsigned)((addr).s6_addr[3]), \ + (unsigned)((addr).s6_addr[4]), \ + (unsigned)((addr).s6_addr[5]), \ + (unsigned)((addr).s6_addr[6]), \ + (unsigned)((addr).s6_addr[7]), \ + (unsigned)((addr).s6_addr[8]), \ + (unsigned)((addr).s6_addr[9]), \ + (unsigned)((addr).s6_addr[10]), \ + (unsigned)((addr).s6_addr[11]), \ + (unsigned)((addr).s6_addr[12]), \ + (unsigned)((addr).s6_addr[13]), \ + (unsigned)((addr).s6_addr[14]), \ + (unsigned)((addr).s6_addr[15]) + +#endif + + + +struct ipaddr_str { + const char * str; + cmdline_ipaddr_t addr; + unsigned flags; +}; + +const struct ipaddr_str ipaddr_valid_strs[] = { + {"0.0.0.0", {AF_INET, {IP4(0,0,0,0)}, 0}, + CMDLINE_IPADDR_V4}, + {"0.0.0.0/0", {AF_INET, {IP4(0,0,0,0)}, 0}, + CMDLINE_IPADDR_V4 | CMDLINE_IPADDR_NETWORK}, + {"0.0.0.0/24", {AF_INET, {IP4(0,0,0,0)}, 24}, + CMDLINE_IPADDR_V4 | CMDLINE_IPADDR_NETWORK}, + {"192.168.1.0/24", {AF_INET, {IP4(192,168,1,0)}, 24}, + CMDLINE_IPADDR_V4 | CMDLINE_IPADDR_NETWORK}, + {"012.34.56.78/24", {AF_INET, {IP4(12,34,56,78)}, 24}, + CMDLINE_IPADDR_V4 | CMDLINE_IPADDR_NETWORK}, + {"34.56.78.90/1", {AF_INET, {IP4(34,56,78,90)}, 1}, + CMDLINE_IPADDR_V4 | CMDLINE_IPADDR_NETWORK}, + {"::", {AF_INET6, {IP6(0,0,0,0,0,0,0,0)}, 0}, + CMDLINE_IPADDR_V6}, + {"::1", {AF_INET6, {IP6(0,0,0,0,0,0,0,1)}, 0}, + CMDLINE_IPADDR_V6}, + {"::1/32", {AF_INET6, {IP6(0,0,0,0,0,0,0,1)}, 32}, + CMDLINE_IPADDR_V6 | CMDLINE_IPADDR_NETWORK}, + {"::/32", {AF_INET6, {IP6(0,0,0,0,0,0,0,0)}, 32}, + CMDLINE_IPADDR_V6 | CMDLINE_IPADDR_NETWORK}, + /* RFC5952 requests that only lowercase should be used */ + {"1234:5678:90ab:cdef:4321:8765:BA09:FEDC", {AF_INET6, + {IP6(0x1234,0x5678,0x90AB,0xCDEF,0x4321,0x8765,0xBA09,0xFEDC)}, + 0}, + CMDLINE_IPADDR_V6}, + {"1234::1234/64", {AF_INET6, + {IP6(0x1234,0,0,0,0,0,0,0x1234)}, + 64}, + CMDLINE_IPADDR_V6 | CMDLINE_IPADDR_NETWORK}, + {"1234::/64", {AF_INET6, + {IP6(0x1234,0,0,0,0,0,0,0)}, + 64}, + CMDLINE_IPADDR_V6 | CMDLINE_IPADDR_NETWORK}, + {"1:1::1/32", {AF_INET6, + {IP6(1,1,0,0,0,0,0,1)}, + 32}, + CMDLINE_IPADDR_V6 | CMDLINE_IPADDR_NETWORK}, + {"1:2:3:4::/64", {AF_INET6, + {IP6(1,2,3,4,0,0,0,0)}, + 64}, + CMDLINE_IPADDR_V6 | CMDLINE_IPADDR_NETWORK}, + {"::ffff:192.168.1.0/64", {AF_INET6, + {IP6(0,0,0,0,0,0xFFFF,0xC0A8,0x100)}, + 64}, + CMDLINE_IPADDR_V6 | CMDLINE_IPADDR_NETWORK}, + /* RFC5952 requests not using :: to skip one block of zeros*/ + {"1::2:3:4:5:6:7", {AF_INET6, + {IP6(1,0,2,3,4,5,6,7)}, + 0}, + CMDLINE_IPADDR_V6}, +}; + +const char * ipaddr_garbage_addr4_strs[] = { + /* IPv4 */ + "192.168.1.0 garbage", + "192.168.1.0\0garbage", + "192.168.1.0#garbage", + "192.168.1.0\tgarbage", + "192.168.1.0\rgarbage", + "192.168.1.0\ngarbage", +}; +#define IPv4_GARBAGE_ADDR IP4(192,168,1,0) + +const char * ipaddr_garbage_addr6_strs[] = { + /* IPv6 */ + "1:2:3:4::8 garbage", + "1:2:3:4::8#garbage", + "1:2:3:4::8\0garbage", + "1:2:3:4::8\rgarbage", + "1:2:3:4::8\ngarbage", + "1:2:3:4::8\tgarbage", +}; +#define IPv6_GARBAGE_ADDR {IP6(1,2,3,4,0,0,0,8)} + +const char * ipaddr_garbage_network4_strs[] = { + /* IPv4 */ + "192.168.1.0/24 garbage", + "192.168.1.0/24\0garbage", + "192.168.1.0/24#garbage", + "192.168.1.0/24\tgarbage", + "192.168.1.0/24\rgarbage", + "192.168.1.0/24\ngarbage", +}; +#define IPv4_GARBAGE_PREFIX 24 + +const char * ipaddr_garbage_network6_strs[] = { + /* IPv6 */ + "1:2:3:4::8/64 garbage", + "1:2:3:4::8/64#garbage", + "1:2:3:4::8/64\0garbage", + "1:2:3:4::8/64\rgarbage", + "1:2:3:4::8/64\ngarbage", + "1:2:3:4::8/64\tgarbage", +}; +#define IPv6_GARBAGE_PREFIX 64 + + + +const char * ipaddr_invalid_strs[] = { + /** IPv4 **/ + + /* invalid numbers */ + "0.0.0.-1", + "0.0.-1.0", + "0.-1.0.0", + "-1.0.0.0", + "0.0.0.-1/24", + "256.123.123.123", + "255.256.123.123", + "255.255.256.123", + "255.255.255.256", + "256.123.123.123/24", + "255.256.123.123/24", + "255.255.256.123/24", + "255.255.255.256/24", + /* invalid network mask */ + "1.2.3.4/33", + "1.2.3.4/33231313", + "1.2.3.4/-1", + "1.2.3.4/24/33", + "1.2.3.4/24/-1", + "1.2.3.4/24/", + /* wrong format */ + "1/24" + "/24" + "123.123.123", + "123.123.123.", + "123.123.123.123.", + "123.123.123..123", + "123.123.123.123.123", + ".123.123.123", + ".123.123.123.123", + "123.123.123/24", + "123.123.123./24", + "123.123.123.123./24", + "123.123.123..123/24", + "123.123.123.123.123/24", + ".123.123.123/24", + ".123.123.123.123/24", + /* invalid characters */ + "123.123.123.12F", + "123.123.12F.123", + "123.12F.123.123", + "12F.123.123.123", + "12J.123.123.123", + "123,123,123,123", + "123!123!123!12F", + "123.123.123.123/4F", + + /** IPv6 **/ + + /* wrong format */ + "::fffff", + "ffff:", + "1:2:3:4:5:6:7:192.168.1.1", + "1234:192.168.1.1:ffff::", + "1:2:3:4:5:6:7:890ab", + "1:2:3:4:5:6:7890a:b", + "1:2:3:4:5:67890:a:b", + "1:2:3:4:56789:0:a:b", + "1:2:3:45678:9:0:a:b", + "1:2:34567:8:9:0:a:b", + "1:23456:7:8:9:0:a:b", + "12345:6:7:8:9:0:a:b", + "1:::2", + "1::::2", + "::fffff/64", + "1::2::3", + "1::2::3/64", + ":1:2", + ":1:2/64", + ":1::2", + ":1::2/64", + "1::2:3:4:5:6:7:8/64", + + /* invalid network mask */ + "1:2:3:4:5:6:7:8/129", + "1:2:3:4:5:6:7:8/-1", + + /* invalid characters */ + "a:b:c:d:e:f:g::", + + /** misc **/ + + /* too long */ + "1234:1234:1234:1234:1234:1234:1234:1234:1234:1234:1234" + "random invalid text", + "", + "\0", + " ", +}; + +#define IPADDR_VALID_STRS_SIZE \ + (sizeof(ipaddr_valid_strs) / sizeof(ipaddr_valid_strs[0])) +#define IPADDR_GARBAGE_ADDR4_STRS_SIZE \ + (sizeof(ipaddr_garbage_addr4_strs) / sizeof(ipaddr_garbage_addr4_strs[0])) +#define IPADDR_GARBAGE_ADDR6_STRS_SIZE \ + (sizeof(ipaddr_garbage_addr6_strs) / sizeof(ipaddr_garbage_addr6_strs[0])) +#define IPADDR_GARBAGE_NETWORK4_STRS_SIZE \ + (sizeof(ipaddr_garbage_network4_strs) / sizeof(ipaddr_garbage_network4_strs[0])) +#define IPADDR_GARBAGE_NETWORK6_STRS_SIZE \ + (sizeof(ipaddr_garbage_network6_strs) / sizeof(ipaddr_garbage_network6_strs[0])) +#define IPADDR_INVALID_STRS_SIZE \ + (sizeof(ipaddr_invalid_strs) / sizeof(ipaddr_invalid_strs[0])) + +static void +dump_addr(cmdline_ipaddr_t addr) +{ + switch (addr.family) { + case AF_INET: + { + printf(NIPQUAD_FMT " prefixlen=%u\n", + NIPQUAD(addr.addr.ipv4.s_addr), addr.prefixlen); + break; + } + case AF_INET6: + { + printf(NIP6_FMT " prefixlen=%u\n", + NIP6(addr.addr.ipv6), addr.prefixlen); + break; + } + default: + printf("Can't dump: unknown address family.\n"); + return; + } +} + + +static int +is_addr_different(cmdline_ipaddr_t addr1, cmdline_ipaddr_t addr2) +{ + if (addr1.family != addr2.family) + return 1; + + if (addr1.prefixlen != addr2.prefixlen) + return 1; + + switch (addr1.family) { + /* IPv4 */ + case AF_INET: + if (memcmp(&addr1.addr.ipv4, &addr2.addr.ipv4, + sizeof(struct in_addr)) != 0) + return 1; + break; + /* IPv6 */ + case AF_INET6: + { + if (memcmp(&addr1.addr.ipv6, &addr2.addr.ipv6, + sizeof(struct in6_addr)) != 0) + return 1; + break; + } + /* thing that should not be */ + default: + return -1; + } + return 0; +} + +static int +can_parse_addr(unsigned addr_flags, unsigned test_flags) +{ + if ((test_flags & addr_flags) == addr_flags) { + /* if we are not trying to parse network addresses */ + if (test_flags < CMDLINE_IPADDR_NETWORK) + return 1; + /* if this is a network address */ + else if (addr_flags & CMDLINE_IPADDR_NETWORK) + return 1; + } + return 0; +} + +int +test_parse_ipaddr_valid(void) +{ + cmdline_parse_token_ipaddr_t token; + char buf[CMDLINE_TEST_BUFSIZE]; + cmdline_ipaddr_t result; + unsigned i; + uint8_t flags; + int ret; + + /* cover all cases in help */ + for (flags = 0x1; flags < 0x8; flags++) { + token.ipaddr_data.flags = flags; + + memset(buf, 0, sizeof(buf)); + + if (cmdline_get_help_ipaddr((cmdline_parse_token_hdr_t*)&token, + buf, sizeof(buf)) == -1) { + printf("Error: help rejected valid parameters!\n"); + return -1; + } + } + + /* test valid strings */ + for (i = 0; i < IPADDR_VALID_STRS_SIZE; i++) { + + /* test each valid string against different flags */ + for (flags = 1; flags < 0x8; flags++) { + + /* skip bad flag */ + if (flags == CMDLINE_IPADDR_NETWORK) + continue; + + /* clear out everything */ + memset(buf, 0, sizeof(buf)); + memset(&result, 0, sizeof(result)); + memset(&token, 0, sizeof(token)); + + token.ipaddr_data.flags = flags; + + cmdline_get_help_ipaddr((cmdline_parse_token_hdr_t*)&token, + buf, sizeof(buf)); + + ret = cmdline_parse_ipaddr((cmdline_parse_token_hdr_t*)&token, + ipaddr_valid_strs[i].str, (void*)&result, + sizeof(result)); + + /* if should have passed, or should have failed */ + if ((ret < 0) == + (can_parse_addr(ipaddr_valid_strs[i].flags, flags))) { + printf("Error: unexpected behavior when parsing %s as %s!\n", + ipaddr_valid_strs[i].str, buf); + printf("Parsed result: "); + dump_addr(result); + printf("Expected result: "); + dump_addr(ipaddr_valid_strs[i].addr); + return -1; + } + if (ret != -1 && + is_addr_different(result, ipaddr_valid_strs[i].addr)) { + printf("Error: result mismatch when parsing %s as %s!\n", + ipaddr_valid_strs[i].str, buf); + printf("Parsed result: "); + dump_addr(result); + printf("Expected result: "); + dump_addr(ipaddr_valid_strs[i].addr); + return -1; + } + } + } + + /* test garbage ipv4 address strings */ + for (i = 0; i < IPADDR_GARBAGE_ADDR4_STRS_SIZE; i++) { + + struct in_addr tmp = IPv4_GARBAGE_ADDR; + + /* test each valid string against different flags */ + for (flags = 1; flags < 0x8; flags++) { + + /* skip bad flag */ + if (flags == CMDLINE_IPADDR_NETWORK) + continue; + + /* clear out everything */ + memset(buf, 0, sizeof(buf)); + memset(&result, 0, sizeof(result)); + memset(&token, 0, sizeof(token)); + + token.ipaddr_data.flags = flags; + + cmdline_get_help_ipaddr((cmdline_parse_token_hdr_t*)&token, + buf, sizeof(buf)); + + ret = cmdline_parse_ipaddr((cmdline_parse_token_hdr_t*)&token, + ipaddr_garbage_addr4_strs[i], (void*)&result, + sizeof(result)); + + /* if should have passed, or should have failed */ + if ((ret < 0) == + (can_parse_addr(CMDLINE_IPADDR_V4, flags))) { + printf("Error: unexpected behavior when parsing %s as %s!\n", + ipaddr_garbage_addr4_strs[i], buf); + return -1; + } + if (ret != -1 && + memcmp(&result.addr.ipv4, &tmp, sizeof(tmp))) { + printf("Error: result mismatch when parsing %s as %s!\n", + ipaddr_garbage_addr4_strs[i], buf); + return -1; + } + } + } + + /* test garbage ipv6 address strings */ + for (i = 0; i < IPADDR_GARBAGE_ADDR6_STRS_SIZE; i++) { + + cmdline_ipaddr_t tmp = {.addr = IPv6_GARBAGE_ADDR}; + + /* test each valid string against different flags */ + for (flags = 1; flags < 0x8; flags++) { + + /* skip bad flag */ + if (flags == CMDLINE_IPADDR_NETWORK) + continue; + + /* clear out everything */ + memset(buf, 0, sizeof(buf)); + memset(&result, 0, sizeof(result)); + memset(&token, 0, sizeof(token)); + + token.ipaddr_data.flags = flags; + + cmdline_get_help_ipaddr((cmdline_parse_token_hdr_t*)&token, + buf, sizeof(buf)); + + ret = cmdline_parse_ipaddr((cmdline_parse_token_hdr_t*)&token, + ipaddr_garbage_addr6_strs[i], (void*)&result, + sizeof(result)); + + /* if should have passed, or should have failed */ + if ((ret < 0) == + (can_parse_addr(CMDLINE_IPADDR_V6, flags))) { + printf("Error: unexpected behavior when parsing %s as %s!\n", + ipaddr_garbage_addr6_strs[i], buf); + return -1; + } + if (ret != -1 && + memcmp(&result.addr.ipv6, &tmp.addr.ipv6, sizeof(struct in6_addr))) { + printf("Error: result mismatch when parsing %s as %s!\n", + ipaddr_garbage_addr6_strs[i], buf); + return -1; + } + } + } + + + /* test garbage ipv4 network strings */ + for (i = 0; i < IPADDR_GARBAGE_NETWORK4_STRS_SIZE; i++) { + + struct in_addr tmp = IPv4_GARBAGE_ADDR; + + /* test each valid string against different flags */ + for (flags = 1; flags < 0x8; flags++) { + + /* skip bad flag */ + if (flags == CMDLINE_IPADDR_NETWORK) + continue; + + /* clear out everything */ + memset(buf, 0, sizeof(buf)); + memset(&result, 0, sizeof(result)); + memset(&token, 0, sizeof(token)); + + token.ipaddr_data.flags = flags; + + cmdline_get_help_ipaddr((cmdline_parse_token_hdr_t*)&token, + buf, sizeof(buf)); + + ret = cmdline_parse_ipaddr((cmdline_parse_token_hdr_t*)&token, + ipaddr_garbage_network4_strs[i], (void*)&result, + sizeof(result)); + + /* if should have passed, or should have failed */ + if ((ret < 0) == + (can_parse_addr(CMDLINE_IPADDR_V4 | CMDLINE_IPADDR_NETWORK, flags))) { + printf("Error: unexpected behavior when parsing %s as %s!\n", + ipaddr_garbage_network4_strs[i], buf); + return -1; + } + if (ret != -1 && + memcmp(&result.addr.ipv4, &tmp, sizeof(tmp))) { + printf("Error: result mismatch when parsing %s as %s!\n", + ipaddr_garbage_network4_strs[i], buf); + return -1; + } + } + } + + /* test garbage ipv6 address strings */ + for (i = 0; i < IPADDR_GARBAGE_NETWORK6_STRS_SIZE; i++) { + + cmdline_ipaddr_t tmp = {.addr = IPv6_GARBAGE_ADDR}; + + /* test each valid string against different flags */ + for (flags = 1; flags < 0x8; flags++) { + + /* skip bad flag */ + if (flags == CMDLINE_IPADDR_NETWORK) + continue; + + /* clear out everything */ + memset(buf, 0, sizeof(buf)); + memset(&result, 0, sizeof(result)); + memset(&token, 0, sizeof(token)); + + token.ipaddr_data.flags = flags; + + cmdline_get_help_ipaddr((cmdline_parse_token_hdr_t*)&token, + buf, sizeof(buf)); + + ret = cmdline_parse_ipaddr((cmdline_parse_token_hdr_t*)&token, + ipaddr_garbage_network6_strs[i], (void*)&result, + sizeof(result)); + + /* if should have passed, or should have failed */ + if ((ret < 0) == + (can_parse_addr(CMDLINE_IPADDR_V6 | CMDLINE_IPADDR_NETWORK, flags))) { + printf("Error: unexpected behavior when parsing %s as %s!\n", + ipaddr_garbage_network6_strs[i], buf); + return -1; + } + if (ret != -1 && + memcmp(&result.addr.ipv6, &tmp.addr.ipv6, sizeof(struct in6_addr))) { + printf("Error: result mismatch when parsing %s as %s!\n", + ipaddr_garbage_network6_strs[i], buf); + return -1; + } + } + } + + return 0; +} + +int +test_parse_ipaddr_invalid_data(void) +{ + cmdline_parse_token_ipaddr_t token; + char buf[CMDLINE_TEST_BUFSIZE]; + cmdline_ipaddr_t result; + unsigned i; + uint8_t flags; + int ret; + + memset(&result, 0, sizeof(result)); + + /* test invalid strings */ + for (i = 0; i < IPADDR_INVALID_STRS_SIZE; i++) { + + /* test each valid string against different flags */ + for (flags = 1; flags < 0x8; flags++) { + + /* skip bad flag */ + if (flags == CMDLINE_IPADDR_NETWORK) + continue; + + /* clear out everything */ + memset(buf, 0, sizeof(buf)); + memset(&token, 0, sizeof(token)); + + token.ipaddr_data.flags = flags; + + cmdline_get_help_ipaddr((cmdline_parse_token_hdr_t*)&token, + buf, sizeof(buf)); + + ret = cmdline_parse_ipaddr((cmdline_parse_token_hdr_t*)&token, + ipaddr_invalid_strs[i], (void*)&result, + sizeof(result)); + + if (ret != -1) { + printf("Error: parsing %s as %s succeeded!\n", + ipaddr_invalid_strs[i], buf); + printf("Parsed result: "); + dump_addr(result); + return -1; + } + } + } + + return 0; +} + +int +test_parse_ipaddr_invalid_param(void) +{ + cmdline_parse_token_ipaddr_t token; + char buf[CMDLINE_TEST_BUFSIZE]; + cmdline_ipaddr_t result; + + snprintf(buf, sizeof(buf), "1.2.3.4"); + token.ipaddr_data.flags = CMDLINE_IPADDR_V4; + + /* null token */ + if (cmdline_parse_ipaddr(NULL, buf, (void*)&result, + sizeof(result)) != -1) { + printf("Error: parser accepted invalid parameters!\n"); + return -1; + } + /* null buffer */ + if (cmdline_parse_ipaddr((cmdline_parse_token_hdr_t*)&token, + NULL, (void*)&result, sizeof(result)) != -1) { + printf("Error: parser accepted invalid parameters!\n"); + return -1; + } + /* empty buffer */ + if (cmdline_parse_ipaddr((cmdline_parse_token_hdr_t*)&token, + "", (void*)&result, sizeof(result)) != -1) { + printf("Error: parser accepted invalid parameters!\n"); + return -1; + } + /* null result */ + if (cmdline_parse_ipaddr((cmdline_parse_token_hdr_t*)&token, + buf, NULL, 0) == -1) { + printf("Error: parser rejected null result!\n"); + return -1; + } + + /* null token */ + if (cmdline_get_help_ipaddr(NULL, buf, 0) != -1) { + printf("Error: help accepted invalid parameters!\n"); + return -1; + } + /* null buffer */ + if (cmdline_get_help_ipaddr((cmdline_parse_token_hdr_t*)&token, + NULL, 0) != -1) { + printf("Error: help accepted invalid parameters!\n"); + return -1; + } + return 0; +} diff --git a/dpdk/test/test/test_cmdline_lib.c b/dpdk/test/test/test_cmdline_lib.c new file mode 100644 index 00000000..65b823a7 --- /dev/null +++ b/dpdk/test/test/test_cmdline_lib.c @@ -0,0 +1,263 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <string.h> +#include <stdio.h> +#include <stdint.h> +#include <stdarg.h> +#include <stdlib.h> +#include <errno.h> +#include <termios.h> +#include <ctype.h> +#include <sys/queue.h> + +#include <cmdline_vt100.h> +#include <cmdline_rdline.h> +#include <cmdline_parse.h> +#include <cmdline_socket.h> +#include <cmdline.h> + +#include "test_cmdline.h" + +/****************************************************************/ +/* static functions required for some tests */ +static void +valid_buffer(__attribute__((unused))struct rdline *rdl, + __attribute__((unused))const char *buf, + __attribute__((unused)) unsigned int size) +{ +} + +static int +complete_buffer(__attribute__((unused)) struct rdline *rdl, + __attribute__((unused)) const char *buf, + __attribute__((unused)) char *dstbuf, + __attribute__((unused)) unsigned int dstsize, + __attribute__((unused)) int *state) +{ + return 0; +} + +/****************************************************************/ + +static int +test_cmdline_parse_fns(void) +{ + struct cmdline cl; + int i = 0; + char dst[CMDLINE_TEST_BUFSIZE]; + + if (cmdline_parse(NULL, "buffer") >= 0) + goto error; + if (cmdline_parse(&cl, NULL) >= 0) + goto error; + + if (cmdline_complete(NULL, "buffer", &i, dst, sizeof(dst)) >= 0) + goto error; + if (cmdline_complete(&cl, NULL, &i, dst, sizeof(dst)) >= 0) + goto error; + if (cmdline_complete(&cl, "buffer", NULL, dst, sizeof(dst)) >= 0) + goto error; + if (cmdline_complete(&cl, "buffer", &i, NULL, sizeof(dst)) >= 0) + goto error; + + return 0; + +error: + printf("Error: function accepted null parameter!\n"); + return -1; +} + +static int +test_cmdline_rdline_fns(void) +{ + struct rdline rdl; + rdline_write_char_t *wc = &cmdline_write_char; + rdline_validate_t *v = &valid_buffer; + rdline_complete_t *c = &complete_buffer; + + if (rdline_init(NULL, wc, v, c) >= 0) + goto error; + if (rdline_init(&rdl, NULL, v, c) >= 0) + goto error; + if (rdline_init(&rdl, wc, NULL, c) >= 0) + goto error; + if (rdline_init(&rdl, wc, v, NULL) >= 0) + goto error; + if (rdline_char_in(NULL, 0) >= 0) + goto error; + if (rdline_get_buffer(NULL) != NULL) + goto error; + if (rdline_add_history(NULL, "history") >= 0) + goto error; + if (rdline_add_history(&rdl, NULL) >= 0) + goto error; + if (rdline_get_history_item(NULL, 0) != NULL) + goto error; + + /* void functions */ + rdline_newline(NULL, "prompt"); + rdline_newline(&rdl, NULL); + rdline_stop(NULL); + rdline_quit(NULL); + rdline_restart(NULL); + rdline_redisplay(NULL); + rdline_reset(NULL); + rdline_clear_history(NULL); + + return 0; + +error: + printf("Error: function accepted null parameter!\n"); + return -1; +} + +static int +test_cmdline_vt100_fns(void) +{ + if (vt100_parser(NULL, 0) >= 0) { + printf("Error: function accepted null parameter!\n"); + return -1; + } + + /* void functions */ + vt100_init(NULL); + + return 0; +} + +static int +test_cmdline_socket_fns(void) +{ + cmdline_parse_ctx_t ctx; + + if (cmdline_stdin_new(NULL, "prompt") != NULL) + goto error; + if (cmdline_stdin_new(&ctx, NULL) != NULL) + goto error; + if (cmdline_file_new(NULL, "prompt", "/dev/null") != NULL) + goto error; + if (cmdline_file_new(&ctx, NULL, "/dev/null") != NULL) + goto error; + if (cmdline_file_new(&ctx, "prompt", NULL) != NULL) + goto error; + if (cmdline_file_new(&ctx, "prompt", "-/invalid/~/path") != NULL) { + printf("Error: succeeded in opening invalid file for reading!"); + return -1; + } + if (cmdline_file_new(&ctx, "prompt", "/dev/null") == NULL) { + printf("Error: failed to open /dev/null for reading!"); + return -1; + } + + /* void functions */ + cmdline_stdin_exit(NULL); + + return 0; +error: + printf("Error: function accepted null parameter!\n"); + return -1; +} + +static int +test_cmdline_fns(void) +{ + cmdline_parse_ctx_t ctx; + struct cmdline cl, *tmp; + + memset(&ctx, 0, sizeof(ctx)); + tmp = cmdline_new(&ctx, "test", -1, -1); + if (tmp == NULL) + goto error; + + if (cmdline_new(NULL, "prompt", 0, 0) != NULL) + goto error; + if (cmdline_new(&ctx, NULL, 0, 0) != NULL) + goto error; + if (cmdline_in(NULL, "buffer", CMDLINE_TEST_BUFSIZE) >= 0) + goto error; + if (cmdline_in(&cl, NULL, CMDLINE_TEST_BUFSIZE) >= 0) + goto error; + if (cmdline_write_char(NULL, 0) >= 0) + goto error; + + /* void functions */ + cmdline_set_prompt(NULL, "prompt"); + cmdline_free(NULL); + cmdline_printf(NULL, "format"); + /* this should fail as stream handles are invalid */ + cmdline_printf(tmp, "format"); + cmdline_interact(NULL); + cmdline_quit(NULL); + + /* check if void calls change anything when they should fail */ + cl = *tmp; + + cmdline_printf(&cl, NULL); + if (memcmp(&cl, tmp, sizeof(cl))) goto mismatch; + cmdline_set_prompt(&cl, NULL); + if (memcmp(&cl, tmp, sizeof(cl))) goto mismatch; + cmdline_in(&cl, NULL, CMDLINE_TEST_BUFSIZE); + if (memcmp(&cl, tmp, sizeof(cl))) goto mismatch; + + cmdline_free(tmp); + + return 0; + +error: + printf("Error: function accepted null parameter!\n"); + return -1; +mismatch: + printf("Error: data changed!\n"); + return -1; +} + +/* test library functions. the point of these tests is not so much to test + * functions' behaviour as it is to make sure there are no segfaults if + * they are called with invalid parameters. + */ +int +test_cmdline_lib(void) +{ + if (test_cmdline_parse_fns() < 0) + return -1; + if (test_cmdline_rdline_fns() < 0) + return -1; + if (test_cmdline_vt100_fns() < 0) + return -1; + if (test_cmdline_socket_fns() < 0) + return -1; + if (test_cmdline_fns() < 0) + return -1; + return 0; +} diff --git a/dpdk/test/test/test_cmdline_num.c b/dpdk/test/test/test_cmdline_num.c new file mode 100644 index 00000000..e8f60cfa --- /dev/null +++ b/dpdk/test/test/test_cmdline_num.c @@ -0,0 +1,623 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <string.h> +#include <inttypes.h> + +#include <rte_string_fns.h> + +#include <cmdline_parse.h> +#include <cmdline_parse_num.h> + +#include "test_cmdline.h" + +struct num_unsigned_str { + const char * str; + uint64_t result; +}; + +struct num_signed_str { + const char * str; + int64_t result; +}; + +const struct num_unsigned_str num_valid_positive_strs[] = { + /* decimal positive */ + {"0", 0 }, + {"127", INT8_MAX }, + {"128", INT8_MAX + 1 }, + {"255", UINT8_MAX }, + {"256", UINT8_MAX + 1 }, + {"32767", INT16_MAX }, + {"32768", INT16_MAX + 1 }, + {"65535", UINT16_MAX }, + {"65536", UINT16_MAX + 1 }, + {"2147483647", INT32_MAX }, + {"2147483648", INT32_MAX + 1U }, + {"4294967295", UINT32_MAX }, + {"4294967296", UINT32_MAX + 1ULL }, + {"9223372036854775807", INT64_MAX }, + {"9223372036854775808", INT64_MAX + 1ULL}, + {"18446744073709551615", UINT64_MAX }, + /* hexadecimal (no leading zeroes) */ + {"0x0", 0 }, + {"0x7F", INT8_MAX }, + {"0x80", INT8_MAX + 1 }, + {"0xFF", UINT8_MAX }, + {"0x100", UINT8_MAX + 1 }, + {"0x7FFF", INT16_MAX }, + {"0x8000", INT16_MAX + 1 }, + {"0xFFFF", UINT16_MAX }, + {"0x10000", UINT16_MAX + 1 }, + {"0x7FFFFFFF", INT32_MAX }, + {"0x80000000", INT32_MAX + 1U }, + {"0xFFFFFFFF", UINT32_MAX }, + {"0x100000000", UINT32_MAX + 1ULL }, + {"0x7FFFFFFFFFFFFFFF", INT64_MAX }, + {"0x8000000000000000", INT64_MAX + 1ULL}, + {"0xFFFFFFFFFFFFFFFF", UINT64_MAX }, + /* hexadecimal (with leading zeroes) */ + {"0x00", 0 }, + {"0x7F", INT8_MAX }, + {"0x80", INT8_MAX + 1 }, + {"0xFF", UINT8_MAX }, + {"0x0100", UINT8_MAX + 1 }, + {"0x7FFF", INT16_MAX }, + {"0x8000", INT16_MAX + 1 }, + {"0xFFFF", UINT16_MAX }, + {"0x00010000", UINT16_MAX + 1 }, + {"0x7FFFFFFF", INT32_MAX }, + {"0x80000000", INT32_MAX + 1U }, + {"0xFFFFFFFF", UINT32_MAX }, + {"0x0000000100000000", UINT32_MAX + 1ULL }, + {"0x7FFFFFFFFFFFFFFF", INT64_MAX }, + {"0x8000000000000000", INT64_MAX + 1ULL}, + {"0xFFFFFFFFFFFFFFFF", UINT64_MAX }, + /* check all characters */ + {"0x1234567890ABCDEF", 0x1234567890ABCDEFULL }, + {"0x1234567890abcdef", 0x1234567890ABCDEFULL }, + /* binary (no leading zeroes) */ + {"0b0", 0 }, + {"0b1111111", INT8_MAX }, + {"0b10000000", INT8_MAX + 1 }, + {"0b11111111", UINT8_MAX }, + {"0b100000000", UINT8_MAX + 1 }, + {"0b111111111111111", INT16_MAX }, + {"0b1000000000000000", INT16_MAX + 1 }, + {"0b1111111111111111", UINT16_MAX }, + {"0b10000000000000000", UINT16_MAX + 1 }, + {"0b1111111111111111111111111111111", INT32_MAX }, + {"0b10000000000000000000000000000000", INT32_MAX + 1U }, + {"0b11111111111111111111111111111111", UINT32_MAX }, + {"0b100000000000000000000000000000000", UINT32_MAX + 1ULL }, + {"0b111111111111111111111111111111111111111111111111111111111111111", + INT64_MAX }, + {"0b1000000000000000000000000000000000000000000000000000000000000000", + INT64_MAX + 1ULL}, + {"0b1111111111111111111111111111111111111111111111111111111111111111", + UINT64_MAX }, + /* binary (with leading zeroes) */ + {"0b01111111", INT8_MAX }, + {"0b0000000100000000", UINT8_MAX + 1 }, + {"0b0111111111111111", INT16_MAX }, + {"0b00000000000000010000000000000000", UINT16_MAX + 1 }, + {"0b01111111111111111111111111111111", INT32_MAX }, + {"0b0000000000000000000000000000000100000000000000000000000000000000", + UINT32_MAX + 1ULL }, + {"0b0111111111111111111111111111111111111111111111111111111111111111", + INT64_MAX }, + /* octal */ + {"00", 0 }, + {"0177", INT8_MAX }, + {"0200", INT8_MAX + 1 }, + {"0377", UINT8_MAX }, + {"0400", UINT8_MAX + 1 }, + {"077777", INT16_MAX }, + {"0100000", INT16_MAX + 1 }, + {"0177777", UINT16_MAX }, + {"0200000", UINT16_MAX + 1 }, + {"017777777777", INT32_MAX }, + {"020000000000", INT32_MAX + 1U }, + {"037777777777", UINT32_MAX }, + {"040000000000", UINT32_MAX + 1ULL }, + {"0777777777777777777777", INT64_MAX }, + {"01000000000000000000000", INT64_MAX + 1ULL}, + {"01777777777777777777777", UINT64_MAX }, + /* check all numbers */ + {"012345670", 012345670 }, + {"076543210", 076543210 }, +}; + +const struct num_signed_str num_valid_negative_strs[] = { + /* deciman negative */ + {"-128", INT8_MIN }, + {"-129", INT8_MIN - 1 }, + {"-32768", INT16_MIN }, + {"-32769", INT16_MIN - 1 }, + {"-2147483648", INT32_MIN }, + {"-2147483649", INT32_MIN - 1LL }, + {"-9223372036854775808", INT64_MIN }, +}; + +const struct num_unsigned_str num_garbage_positive_strs[] = { + /* valid strings with garbage on the end, should still be valid */ + /* decimal */ + {"9223372036854775807\0garbage", INT64_MAX }, + {"9223372036854775807\tgarbage", INT64_MAX }, + {"9223372036854775807\rgarbage", INT64_MAX }, + {"9223372036854775807\ngarbage", INT64_MAX }, + {"9223372036854775807#garbage", INT64_MAX }, + {"9223372036854775807 garbage", INT64_MAX }, + /* hex */ + {"0x7FFFFFFFFFFFFFFF\0garbage", INT64_MAX }, + {"0x7FFFFFFFFFFFFFFF\tgarbage", INT64_MAX }, + {"0x7FFFFFFFFFFFFFFF\rgarbage", INT64_MAX }, + {"0x7FFFFFFFFFFFFFFF\ngarbage", INT64_MAX }, + {"0x7FFFFFFFFFFFFFFF#garbage", INT64_MAX }, + {"0x7FFFFFFFFFFFFFFF garbage", INT64_MAX }, + /* binary */ + {"0b1111111111111111111111111111111\0garbage", INT32_MAX }, + {"0b1111111111111111111111111111111\rgarbage", INT32_MAX }, + {"0b1111111111111111111111111111111\tgarbage", INT32_MAX }, + {"0b1111111111111111111111111111111\ngarbage", INT32_MAX }, + {"0b1111111111111111111111111111111#garbage", INT32_MAX }, + {"0b1111111111111111111111111111111 garbage", INT32_MAX }, + /* octal */ + {"01777777777777777777777\0garbage", UINT64_MAX }, + {"01777777777777777777777\rgarbage", UINT64_MAX }, + {"01777777777777777777777\tgarbage", UINT64_MAX }, + {"01777777777777777777777\ngarbage", UINT64_MAX }, + {"01777777777777777777777#garbage", UINT64_MAX }, + {"01777777777777777777777 garbage", UINT64_MAX }, +}; + +const struct num_signed_str num_garbage_negative_strs[] = { + /* valid strings with garbage on the end, should still be valid */ + {"-9223372036854775808\0garbage", INT64_MIN }, + {"-9223372036854775808\rgarbage", INT64_MIN }, + {"-9223372036854775808\tgarbage", INT64_MIN }, + {"-9223372036854775808\ngarbage", INT64_MIN }, + {"-9223372036854775808#garbage", INT64_MIN }, + {"-9223372036854775808 garbage", INT64_MIN }, +}; + +const char * num_invalid_strs[] = { + "18446744073709551616", /* out of range unsigned */ + "-9223372036854775809", /* out of range negative signed */ + "0x10000000000000000", /* out of range hex */ + /* out of range binary */ + "0b10000000000000000000000000000000000000000000000000000000000000000", + "020000000000000000000000", /* out of range octal */ + /* wrong chars */ + "0123456239", + "0x1234580AGE", + "0b0111010101g001", + "0b01110101017001", + /* false negative numbers */ + "-12345F623", + "-0x1234580A", + "-0b0111010101", + /* too long (128+ chars) */ + "0b1111000011110000111100001111000011110000111100001111000011110000" + "1111000011110000111100001111000011110000111100001111000011110000", + "1E3", + "0A", + "-B", + "+4", + "1.23G", + "", + " ", + "#", + "\r", + "\t", + "\n", + "\0", +}; + +#define NUM_POSITIVE_STRS_SIZE \ + (sizeof(num_valid_positive_strs) / sizeof(num_valid_positive_strs[0])) +#define NUM_NEGATIVE_STRS_SIZE \ + (sizeof(num_valid_negative_strs) / sizeof(num_valid_negative_strs[0])) +#define NUM_POSITIVE_GARBAGE_STRS_SIZE \ + (sizeof(num_garbage_positive_strs) / sizeof(num_garbage_positive_strs[0])) +#define NUM_NEGATIVE_GARBAGE_STRS_SIZE \ + (sizeof(num_garbage_negative_strs) / sizeof(num_garbage_negative_strs[0])) +#define NUM_INVALID_STRS_SIZE \ + (sizeof(num_invalid_strs) / sizeof(num_invalid_strs[0])) + + + +static int +can_parse_unsigned(uint64_t expected_result, enum cmdline_numtype type) +{ + switch (type) { + case UINT8: + if (expected_result > UINT8_MAX) + return 0; + break; + case UINT16: + if (expected_result > UINT16_MAX) + return 0; + break; + case UINT32: + if (expected_result > UINT32_MAX) + return 0; + break; + case INT8: + if (expected_result > INT8_MAX) + return 0; + break; + case INT16: + if (expected_result > INT16_MAX) + return 0; + break; + case INT32: + if (expected_result > INT32_MAX) + return 0; + break; + case INT64: + if (expected_result > INT64_MAX) + return 0; + break; + default: + return 1; + } + return 1; +} + +static int +can_parse_signed(int64_t expected_result, enum cmdline_numtype type) +{ + switch (type) { + case UINT8: + if (expected_result > UINT8_MAX || expected_result < 0) + return 0; + break; + case UINT16: + if (expected_result > UINT16_MAX || expected_result < 0) + return 0; + break; + case UINT32: + if (expected_result > UINT32_MAX || expected_result < 0) + return 0; + break; + case UINT64: + if (expected_result < 0) + return 0; + break; + case INT8: + if (expected_result > INT8_MAX || expected_result < INT8_MIN) + return 0; + break; + case INT16: + if (expected_result > INT16_MAX || expected_result < INT16_MIN) + return 0; + break; + case INT32: + if (expected_result > INT32_MAX || expected_result < INT32_MIN) + return 0; + break; + default: + return 1; + } + return 1; +} + +/* test invalid parameters */ +int +test_parse_num_invalid_param(void) +{ + char buf[CMDLINE_TEST_BUFSIZE]; + uint32_t result; + cmdline_parse_token_num_t token; + int ret = 0; + + /* set up a token */ + token.num_data.type = UINT32; + + /* copy string to buffer */ + snprintf(buf, sizeof(buf), "%s", + num_valid_positive_strs[0].str); + + /* try all null */ + ret = cmdline_parse_num(NULL, NULL, NULL, 0); + if (ret != -1) { + printf("Error: parser accepted null parameters!\n"); + return -1; + } + + /* try null token */ + ret = cmdline_parse_num(NULL, buf, (void*)&result, sizeof(result)); + if (ret != -1) { + printf("Error: parser accepted null token!\n"); + return -1; + } + + /* try null buf */ + ret = cmdline_parse_num((cmdline_parse_token_hdr_t*)&token, NULL, + (void*)&result, sizeof(result)); + if (ret != -1) { + printf("Error: parser accepted null string!\n"); + return -1; + } + + /* try null result */ + ret = cmdline_parse_num((cmdline_parse_token_hdr_t*)&token, buf, + NULL, 0); + if (ret == -1) { + printf("Error: parser rejected null result!\n"); + return -1; + } + + /* test help function */ + memset(&buf, 0, sizeof(buf)); + + /* try all null */ + ret = cmdline_get_help_num(NULL, NULL, 0); + if (ret != -1) { + printf("Error: help function accepted null parameters!\n"); + return -1; + } + + /* try null token */ + ret = cmdline_get_help_num(NULL, buf, sizeof(buf)); + if (ret != -1) { + printf("Error: help function accepted null token!\n"); + return -1; + } + + /* coverage! */ + ret = cmdline_get_help_num((cmdline_parse_token_hdr_t*)&token, buf, sizeof(buf)); + if (ret < 0) { + printf("Error: help function failed with valid parameters!\n"); + return -1; + } + + return 0; +} +/* test valid parameters but invalid data */ +int +test_parse_num_invalid_data(void) +{ + enum cmdline_numtype type; + int ret = 0; + unsigned i; + char buf[CMDLINE_TEST_BUFSIZE]; + uint64_t result; /* pick largest buffer */ + cmdline_parse_token_num_t token; + + /* cycle through all possible parsed types */ + for (type = UINT8; type <= INT64; type++) { + token.num_data.type = type; + + /* test full strings */ + for (i = 0; i < NUM_INVALID_STRS_SIZE; i++) { + + memset(&result, 0, sizeof(uint64_t)); + memset(&buf, 0, sizeof(buf)); + + ret = cmdline_parse_num((cmdline_parse_token_hdr_t*)&token, + num_invalid_strs[i], (void*)&result, sizeof(result)); + if (ret != -1) { + /* get some info about what we are trying to parse */ + cmdline_get_help_num((cmdline_parse_token_hdr_t*)&token, + buf, sizeof(buf)); + + printf("Error: parsing %s as %s succeeded!\n", + num_invalid_strs[i], buf); + return -1; + } + } + } + return 0; +} + +/* test valid parameters and data */ +int +test_parse_num_valid(void) +{ + int ret = 0; + enum cmdline_numtype type; + unsigned i; + char buf[CMDLINE_TEST_BUFSIZE]; + uint64_t result; + cmdline_parse_token_num_t token; + + /** valid strings **/ + + /* cycle through all possible parsed types */ + for (type = UINT8; type <= INT64; type++) { + token.num_data.type = type; + + /* test positive strings */ + for (i = 0; i < NUM_POSITIVE_STRS_SIZE; i++) { + result = 0; + memset(&buf, 0, sizeof(buf)); + + cmdline_get_help_num((cmdline_parse_token_hdr_t*)&token, + buf, sizeof(buf)); + + ret = cmdline_parse_num((cmdline_parse_token_hdr_t*) &token, + num_valid_positive_strs[i].str, + (void*)&result, sizeof(result)); + + /* if it should have passed but didn't, or if it should have failed but didn't */ + if ((ret < 0) == (can_parse_unsigned(num_valid_positive_strs[i].result, type) > 0)) { + printf("Error: parser behaves unexpectedly when parsing %s as %s!\n", + num_valid_positive_strs[i].str, buf); + return -1; + } + /* check if result matches what it should have matched + * since unsigned numbers don't care about number of bits, we can just convert + * everything to uint64_t without any worries. */ + if (ret > 0 && num_valid_positive_strs[i].result != result) { + printf("Error: parsing %s as %s failed: result mismatch!\n", + num_valid_positive_strs[i].str, buf); + return -1; + } + } + + /* test negative strings */ + for (i = 0; i < NUM_NEGATIVE_STRS_SIZE; i++) { + result = 0; + memset(&buf, 0, sizeof(buf)); + + cmdline_get_help_num((cmdline_parse_token_hdr_t*)&token, + buf, sizeof(buf)); + + ret = cmdline_parse_num((cmdline_parse_token_hdr_t*) &token, + num_valid_negative_strs[i].str, + (void*)&result, sizeof(result)); + + /* if it should have passed but didn't, or if it should have failed but didn't */ + if ((ret < 0) == (can_parse_signed(num_valid_negative_strs[i].result, type) > 0)) { + printf("Error: parser behaves unexpectedly when parsing %s as %s!\n", + num_valid_negative_strs[i].str, buf); + return -1; + } + /* check if result matches what it should have matched + * the result is signed in this case, so we have to account for that */ + if (ret > 0) { + /* detect negative */ + switch (type) { + case INT8: + result = (int8_t) result; + break; + case INT16: + result = (int16_t) result; + break; + case INT32: + result = (int32_t) result; + break; + default: + break; + } + if (num_valid_negative_strs[i].result == (int64_t) result) + continue; + printf("Error: parsing %s as %s failed: result mismatch!\n", + num_valid_negative_strs[i].str, buf); + return -1; + } + } + } + + /** garbage strings **/ + + /* cycle through all possible parsed types */ + for (type = UINT8; type <= INT64; type++) { + token.num_data.type = type; + + /* test positive garbage strings */ + for (i = 0; i < NUM_POSITIVE_GARBAGE_STRS_SIZE; i++) { + result = 0; + memset(&buf, 0, sizeof(buf)); + + cmdline_get_help_num((cmdline_parse_token_hdr_t*)&token, + buf, sizeof(buf)); + + ret = cmdline_parse_num((cmdline_parse_token_hdr_t*) &token, + num_garbage_positive_strs[i].str, + (void*)&result, sizeof(result)); + + /* if it should have passed but didn't, or if it should have failed but didn't */ + if ((ret < 0) == (can_parse_unsigned(num_garbage_positive_strs[i].result, type) > 0)) { + printf("Error: parser behaves unexpectedly when parsing %s as %s!\n", + num_garbage_positive_strs[i].str, buf); + return -1; + } + /* check if result matches what it should have matched + * since unsigned numbers don't care about number of bits, we can just convert + * everything to uint64_t without any worries. */ + if (ret > 0 && num_garbage_positive_strs[i].result != result) { + printf("Error: parsing %s as %s failed: result mismatch!\n", + num_garbage_positive_strs[i].str, buf); + return -1; + } + } + + /* test negative strings */ + for (i = 0; i < NUM_NEGATIVE_GARBAGE_STRS_SIZE; i++) { + result = 0; + memset(&buf, 0, sizeof(buf)); + + cmdline_get_help_num((cmdline_parse_token_hdr_t*)&token, + buf, sizeof(buf)); + + ret = cmdline_parse_num((cmdline_parse_token_hdr_t*) &token, + num_garbage_negative_strs[i].str, + (void*)&result, sizeof(result)); + + /* if it should have passed but didn't, or if it should have failed but didn't */ + if ((ret < 0) == (can_parse_signed(num_garbage_negative_strs[i].result, type) > 0)) { + printf("Error: parser behaves unexpectedly when parsing %s as %s!\n", + num_garbage_negative_strs[i].str, buf); + return -1; + } + /* check if result matches what it should have matched + * the result is signed in this case, so we have to account for that */ + if (ret > 0) { + /* detect negative */ + switch (type) { + case INT8: + if (result & (INT8_MAX + 1)) + result |= 0xFFFFFFFFFFFFFF00ULL; + break; + case INT16: + if (result & (INT16_MAX + 1)) + result |= 0xFFFFFFFFFFFF0000ULL; + break; + case INT32: + if (result & (INT32_MAX + 1ULL)) + result |= 0xFFFFFFFF00000000ULL; + break; + default: + break; + } + if (num_garbage_negative_strs[i].result == (int64_t) result) + continue; + printf("Error: parsing %s as %s failed: result mismatch!\n", + num_garbage_negative_strs[i].str, buf); + return -1; + } + } + } + + memset(&buf, 0, sizeof(buf)); + + /* coverage! */ + cmdline_get_help_num((cmdline_parse_token_hdr_t*)&token, + buf, sizeof(buf)); + + return 0; +} diff --git a/dpdk/test/test/test_cmdline_portlist.c b/dpdk/test/test/test_cmdline_portlist.c new file mode 100644 index 00000000..b9664b0e --- /dev/null +++ b/dpdk/test/test/test_cmdline_portlist.c @@ -0,0 +1,250 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <string.h> +#include <inttypes.h> + +#include <cmdline_parse.h> +#include <cmdline_parse_portlist.h> + +#include "test_cmdline.h" + +struct portlist_str { + const char * str; + uint32_t portmap; +}; + +/* valid strings */ +const struct portlist_str portlist_valid_strs[] = { + {"0", 0x1U }, + {"0-10", 0x7FFU}, + {"10-20", 0x1FFC00U}, + {"all", UINT32_MAX}, + {"0,1,2,3", 0xFU}, + {"0,1-5", 0x3FU}, + {"0,0,0", 0x1U}, + {"31,0-10,15", 0x800087FFU}, + {"0000", 0x1U}, + {"00,01,02,03", 0xFU}, + {"000,001,002,003", 0xFU}, +}; + +/* valid strings but with garbage at the end. + * these strings should still be valid because parser checks + * for end of token, which is either a space/tab, a newline/return, + * or a hash sign. + */ + +const char * portlist_garbage_strs[] = { + "0-31 garbage", + "0-31#garbage", + "0-31\0garbage", + "0-31\ngarbage", + "0-31\rgarbage", + "0-31\tgarbage", + "0,1,2,3-31 garbage", + "0,1,2,3-31#garbage", + "0,1,2,3-31\0garbage", + "0,1,2,3-31\ngarbage", + "0,1,2,3-31\rgarbage", + "0,1,2,3-31\tgarbage", + "all garbage", + "all#garbage", + "all\0garbage", + "all\ngarbage", + "all\rgarbage", + "all\tgarbage", +}; + +/* invalid strings */ +const char * portlist_invalid_strs[] = { + /* valid syntax, invalid chars */ + "A-B", + "0-S", + "1,2,3,4,Q", + "A-4,3-15", + "0-31invalid", + /* valid chars, invalid syntax */ + "1, 2", + "1- 4", + ",2", + ",2 ", + "-1, 4", + "5-1", + "2-", + /* misc */ + "-" + "a", + "A", + ",", + "#", + " ", + "\0", + "", + /* too long */ + "0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1," + "0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,2", +}; + +#define PORTLIST_VALID_STRS_SIZE \ + (sizeof(portlist_valid_strs) / sizeof(portlist_valid_strs[0])) +#define PORTLIST_GARBAGE_STRS_SIZE \ + (sizeof(portlist_garbage_strs) / sizeof(portlist_garbage_strs[0])) +#define PORTLIST_INVALID_STRS_SIZE \ + (sizeof(portlist_invalid_strs) / sizeof(portlist_invalid_strs[0])) + + + + +/* test invalid parameters */ +int +test_parse_portlist_invalid_param(void) +{ + cmdline_portlist_t result; + char buf[CMDLINE_TEST_BUFSIZE]; + int ret; + + memset(&buf, 0, sizeof(buf)); + memset(&result, 0, sizeof(cmdline_portlist_t)); + + /* try all null */ + ret = cmdline_parse_portlist(NULL, NULL, NULL, 0); + if (ret != -1) { + printf("Error: parser accepted null parameters!\n"); + return -1; + } + + /* try null buf */ + ret = cmdline_parse_portlist(NULL, NULL, (void*)&result, + sizeof(result)); + if (ret != -1) { + printf("Error: parser accepted null string!\n"); + return -1; + } + + /* try null result */ + ret = cmdline_parse_portlist(NULL, portlist_valid_strs[0].str, NULL, 0); + if (ret == -1) { + printf("Error: parser rejected null result!\n"); + return -1; + } + + /* token is not used in ether_parse anyway so there's no point in + * testing it */ + + /* test help function */ + + /* coverage! */ + ret = cmdline_get_help_portlist(NULL, buf, sizeof(buf)); + if (ret < 0) { + printf("Error: help function failed with valid parameters!\n"); + return -1; + } + + return 0; +} + +/* test valid parameters but invalid data */ +int +test_parse_portlist_invalid_data(void) +{ + int ret = 0; + unsigned i; + cmdline_portlist_t result; + + /* test invalid strings */ + for (i = 0; i < PORTLIST_INVALID_STRS_SIZE; i++) { + + memset(&result, 0, sizeof(cmdline_portlist_t)); + + ret = cmdline_parse_portlist(NULL, portlist_invalid_strs[i], + (void*)&result, sizeof(result)); + if (ret != -1) { + printf("Error: parsing %s succeeded!\n", + portlist_invalid_strs[i]); + return -1; + } + } + + return 0; +} + +/* test valid parameters and data */ +int +test_parse_portlist_valid(void) +{ + int ret = 0; + unsigned i; + cmdline_portlist_t result; + + /* test full strings */ + for (i = 0; i < PORTLIST_VALID_STRS_SIZE; i++) { + + memset(&result, 0, sizeof(cmdline_portlist_t)); + + ret = cmdline_parse_portlist(NULL, portlist_valid_strs[i].str, + (void*)&result, sizeof(result)); + if (ret < 0) { + printf("Error: parsing %s failed!\n", + portlist_valid_strs[i].str); + return -1; + } + if (result.map != portlist_valid_strs[i].portmap) { + printf("Error: parsing %s failed: map mismatch!\n", + portlist_valid_strs[i].str); + return -1; + } + } + + /* test garbage strings */ + for (i = 0; i < PORTLIST_GARBAGE_STRS_SIZE; i++) { + + memset(&result, 0, sizeof(cmdline_portlist_t)); + + ret = cmdline_parse_portlist(NULL, portlist_garbage_strs[i], + (void*)&result, sizeof(result)); + if (ret < 0) { + printf("Error: parsing %s failed!\n", + portlist_garbage_strs[i]); + return -1; + } + if (result.map != UINT32_MAX) { + printf("Error: parsing %s failed: map mismatch!\n", + portlist_garbage_strs[i]); + return -1; + } + } + + return 0; +} diff --git a/dpdk/test/test/test_cmdline_string.c b/dpdk/test/test/test_cmdline_string.c new file mode 100644 index 00000000..c5bb9c0c --- /dev/null +++ b/dpdk/test/test/test_cmdline_string.c @@ -0,0 +1,412 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <string.h> +#include <inttypes.h> + +#include <rte_common.h> +#include <rte_string_fns.h> + +#include <cmdline_parse.h> +#include <cmdline_parse_string.h> + +#include "test_cmdline.h" + +/* structures needed to run tests */ + +struct string_elt_str { + const char * str; /* parsed string */ + const char * result; /* expected string */ + int idx; /* position at which result is expected to be */ +}; + +struct string_elt_str string_elt_strs[] = { + {"one#two#three", "three", 2}, + {"one#two with spaces#three", "three", 2}, + {"one#two\twith\ttabs#three", "three", 2}, + {"one#two\rwith\rreturns#three", "three", 2}, + {"one#two\nwith\nnewlines#three", "three", 2}, + {"one#two#three", "one", 0}, + {"one#two#three", "two", 1}, + {"one#two\0three", "two", 1}, + {"one#two with spaces#three", "two with spaces", 1}, + {"one#two\twith\ttabs#three", "two\twith\ttabs", 1}, + {"one#two\rwith\rreturns#three", "two\rwith\rreturns", 1}, + {"one#two\nwith\nnewlines#three", "two\nwith\nnewlines", 1}, +}; + +#if (CMDLINE_TEST_BUFSIZE < STR_TOKEN_SIZE) \ +|| (CMDLINE_TEST_BUFSIZE < STR_MULTI_TOKEN_SIZE) +#undef CMDLINE_TEST_BUFSIZE +#define CMDLINE_TEST_BUFSIZE RTE_MAX(STR_TOKEN_SIZE, STR_MULTI_TOKEN_SIZE) +#endif + +struct string_nb_str { + const char * str; /* parsed string */ + int nb_strs; /* expected number of strings in str */ +}; + +struct string_nb_str string_nb_strs[] = { + {"one#two#three", 3}, + {"one", 1}, + {"one# \t two \r # three \n #four", 4}, +}; + + + +struct string_parse_str { + const char * str; /* parsed string */ + const char * fixed_str; /* parsing mode (any, fixed or multi) */ + const char * result; /* expected result */ +}; + +struct string_parse_str string_parse_strs[] = { + {"one", NULL, "one"}, /* any string */ + {"two", "one#two#three", "two"}, /* multiple choice string */ + {"three", "three", "three"}, /* fixed string */ + {"three", "one#two with\rgarbage\tcharacters\n#three", "three"}, + {"two with\rgarbage\tcharacters\n", + "one#two with\rgarbage\tcharacters\n#three", + "two with\rgarbage\tcharacters\n"}, + {"one two", "one", "one"}, /* fixed string */ + {"one two", TOKEN_STRING_MULTI, "one two"}, /* multi string */ + {"one two", NULL, "one"}, /* any string */ + {"one two #three", TOKEN_STRING_MULTI, "one two "}, + /* multi string with comment */ +}; + + + +struct string_invalid_str { + const char * str; /* parsed string */ + const char * fixed_str; /* parsing mode (any, fixed or multi) */ +}; + +struct string_invalid_str string_invalid_strs[] = { + {"invalid", "one"}, /* fixed string */ + {"invalid", "one#two#three"}, /* multiple choice string */ + {"invalid", "invalidone"}, /* string that starts the same */ + {"invalidone", "invalid"}, /* string that starts the same */ + {"toolong!!!toolong!!!toolong!!!toolong!!!toolong!!!toolong!!!" + "toolong!!!toolong!!!toolong!!!toolong!!!toolong!!!toolong!!!" + "toolong!!!", NULL }, + {"toolong!!!toolong!!!toolong!!!toolong!!!toolong!!!toolong!!!" + "toolong!!!toolong!!!toolong!!!toolong!!!toolong!!!toolong!!!" + "toolong!!!", "fixed" }, + {"toolong!!!toolong!!!toolong!!!toolong!!!toolong!!!toolong!!!" + "toolong!!!toolong!!!toolong!!!toolong!!!toolong!!!toolong!!!" + "toolong!!!", "multi#choice#string" }, + {"invalid", + "toolong!!!toolong!!!toolong!!!toolong!!!toolong!!!toolong!!!" + "toolong!!!toolong!!!toolong!!!toolong!!!toolong!!!toolong!!!" + "toolong!!!" }, + {"", "invalid"} +}; + + + +const char * string_help_strs[] = { + NULL, + "fixed_str", + "multi#str", +}; + + + +#define STRING_PARSE_STRS_SIZE \ + (sizeof(string_parse_strs) / sizeof(string_parse_strs[0])) +#define STRING_HELP_STRS_SIZE \ + (sizeof(string_help_strs) / sizeof(string_help_strs[0])) +#define STRING_ELT_STRS_SIZE \ + (sizeof(string_elt_strs) / sizeof(string_elt_strs[0])) +#define STRING_NB_STRS_SIZE \ + (sizeof(string_nb_strs) / sizeof(string_nb_strs[0])) +#define STRING_INVALID_STRS_SIZE \ + (sizeof(string_invalid_strs) / sizeof(string_invalid_strs[0])) + +#define SMALL_BUF 8 + +/* test invalid parameters */ +int +test_parse_string_invalid_param(void) +{ + cmdline_parse_token_string_t token; + int result; + char buf[CMDLINE_TEST_BUFSIZE]; + + memset(&token, 0, sizeof(token)); + + snprintf(buf, sizeof(buf), "buffer"); + + /* test null token */ + if (cmdline_get_help_string( + NULL, buf, 0) != -1) { + printf("Error: function accepted null token!\n"); + return -1; + } + if (cmdline_complete_get_elt_string( + NULL, 0, buf, 0) != -1) { + printf("Error: function accepted null token!\n"); + return -1; + } + if (cmdline_complete_get_nb_string(NULL) != -1) { + printf("Error: function accepted null token!\n"); + return -1; + } + if (cmdline_parse_string(NULL, buf, NULL, 0) != -1) { + printf("Error: function accepted null token!\n"); + return -1; + } + /* test null buffer */ + if (cmdline_complete_get_elt_string( + (cmdline_parse_token_hdr_t*)&token, 0, NULL, 0) != -1) { + printf("Error: function accepted null buffer!\n"); + return -1; + } + if (cmdline_parse_string( + (cmdline_parse_token_hdr_t*)&token, NULL, + (void*)&result, sizeof(result)) != -1) { + printf("Error: function accepted null buffer!\n"); + return -1; + } + if (cmdline_get_help_string( + (cmdline_parse_token_hdr_t*)&token, NULL, 0) != -1) { + printf("Error: function accepted null buffer!\n"); + return -1; + } + /* test null result */ + if (cmdline_parse_string( + (cmdline_parse_token_hdr_t*)&token, buf, NULL, 0) == -1) { + printf("Error: function rejected null result!\n"); + return -1; + } + /* test negative index */ + if (cmdline_complete_get_elt_string( + (cmdline_parse_token_hdr_t*)&token, -1, buf, 0) != -1) { + printf("Error: function accepted negative index!\n"); + return -1; + } + return 0; +} + +/* test valid parameters but invalid data */ +int +test_parse_string_invalid_data(void) +{ + cmdline_parse_token_string_t token; + cmdline_parse_token_string_t help_token; + char buf[CMDLINE_TEST_BUFSIZE]; + char help_str[CMDLINE_TEST_BUFSIZE]; + char small_buf[SMALL_BUF]; + unsigned i; + + /* test parsing invalid strings */ + for (i = 0; i < STRING_INVALID_STRS_SIZE; i++) { + memset(&token, 0, sizeof(token)); + memset(buf, 0, sizeof(buf)); + + /* prepare test token data */ + token.string_data.str = string_invalid_strs[i].fixed_str; + + if (cmdline_parse_string((cmdline_parse_token_hdr_t*)&token, + string_invalid_strs[i].str, (void*)buf, + sizeof(buf)) != -1) { + memset(help_str, 0, sizeof(help_str)); + memset(&help_token, 0, sizeof(help_token)); + + help_token.string_data.str = string_invalid_strs[i].fixed_str; + + /* get parse type so we can give a good error message */ + cmdline_get_help_string((cmdline_parse_token_hdr_t*)&token, help_str, + sizeof(help_str)); + + printf("Error: parsing %s as %s succeeded!\n", + string_invalid_strs[i].str, help_str); + return -1; + } + } + + /* misc tests (big comments signify test cases) */ + memset(&token, 0, sizeof(token)); + memset(small_buf, 0, sizeof(small_buf)); + + /* + * try to get element from a null token + */ + token.string_data.str = NULL; + if (cmdline_complete_get_elt_string( + (cmdline_parse_token_hdr_t*)&token, 1, + buf, sizeof(buf)) != -1) { + printf("Error: getting token from null token string!\n"); + return -1; + } + + /* + * try to get element into a buffer that is too small + */ + token.string_data.str = "too_small_buffer"; + if (cmdline_complete_get_elt_string( + (cmdline_parse_token_hdr_t*)&token, 0, + small_buf, sizeof(small_buf)) != -1) { + printf("Error: writing token into too small a buffer succeeded!\n"); + return -1; + } + + /* + * get help string written into a buffer smaller than help string + * truncation should occur + */ + token.string_data.str = NULL; + if (cmdline_get_help_string( + (cmdline_parse_token_hdr_t*)&token, + small_buf, sizeof(small_buf)) == -1) { + printf("Error: writing help string into too small a buffer failed!\n"); + return -1; + } + /* get help string for "any string" so we can compare it with small_buf */ + cmdline_get_help_string((cmdline_parse_token_hdr_t*)&token, help_str, + sizeof(help_str)); + if (strncmp(small_buf, help_str, sizeof(small_buf) - 1)) { + printf("Error: help string mismatch!\n"); + return -1; + } + /* check null terminator */ + if (small_buf[sizeof(small_buf) - 1] != '\0') { + printf("Error: small buffer doesn't have a null terminator!\n"); + return -1; + } + + /* + * try to count tokens in a null token + */ + token.string_data.str = NULL; + if (cmdline_complete_get_nb_string( + (cmdline_parse_token_hdr_t*)&token) != 0) { + printf("Error: getting token count from null token succeeded!\n"); + return -1; + } + + return 0; +} + +/* test valid parameters and data */ +int +test_parse_string_valid(void) +{ + cmdline_parse_token_string_t token; + cmdline_parse_token_string_t help_token; + char buf[CMDLINE_TEST_BUFSIZE]; + char help_str[CMDLINE_TEST_BUFSIZE]; + unsigned i; + + /* test parsing strings */ + for (i = 0; i < STRING_PARSE_STRS_SIZE; i++) { + memset(&token, 0, sizeof(token)); + memset(buf, 0, sizeof(buf)); + + token.string_data.str = string_parse_strs[i].fixed_str; + + if (cmdline_parse_string((cmdline_parse_token_hdr_t*)&token, + string_parse_strs[i].str, (void*)buf, + sizeof(buf)) < 0) { + + /* clean help data */ + memset(&help_token, 0, sizeof(help_token)); + memset(help_str, 0, sizeof(help_str)); + + /* prepare help token */ + help_token.string_data.str = string_parse_strs[i].fixed_str; + + /* get help string so that we get an informative error message */ + cmdline_get_help_string((cmdline_parse_token_hdr_t*)&token, help_str, + sizeof(help_str)); + + printf("Error: parsing %s as %s failed!\n", + string_parse_strs[i].str, help_str); + return -1; + } + if (strcmp(buf, string_parse_strs[i].result) != 0) { + printf("Error: result mismatch!\n"); + return -1; + } + } + + /* get number of string tokens and verify it's correct */ + for (i = 0; i < STRING_NB_STRS_SIZE; i++) { + memset(&token, 0, sizeof(token)); + + token.string_data.str = string_nb_strs[i].str; + + if (cmdline_complete_get_nb_string( + (cmdline_parse_token_hdr_t*)&token) < + string_nb_strs[i].nb_strs) { + printf("Error: strings count mismatch!\n"); + return -1; + } + } + + /* get token at specified position and verify it's correct */ + for (i = 0; i < STRING_ELT_STRS_SIZE; i++) { + memset(&token, 0, sizeof(token)); + memset(buf, 0, sizeof(buf)); + + token.string_data.str = string_elt_strs[i].str; + + if (cmdline_complete_get_elt_string( + (cmdline_parse_token_hdr_t*)&token, string_elt_strs[i].idx, + buf, sizeof(buf)) < 0) { + printf("Error: getting string element failed!\n"); + return -1; + } + if (strncmp(buf, string_elt_strs[i].result, + sizeof(buf)) != 0) { + printf("Error: result mismatch!\n"); + return -1; + } + } + + /* cover all cases with help strings */ + for (i = 0; i < STRING_HELP_STRS_SIZE; i++) { + memset(&help_token, 0, sizeof(help_token)); + memset(help_str, 0, sizeof(help_str)); + help_token.string_data.str = string_help_strs[i]; + if (cmdline_get_help_string((cmdline_parse_token_hdr_t*)&help_token, + help_str, sizeof(help_str)) < 0) { + printf("Error: help operation failed!\n"); + return -1; + } + } + + return 0; +} diff --git a/dpdk/test/test/test_common.c b/dpdk/test/test/test_common.c new file mode 100644 index 00000000..ae3482da --- /dev/null +++ b/dpdk/test/test/test_common.c @@ -0,0 +1,194 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <string.h> +#include <math.h> +#include <rte_common.h> +#include <rte_hexdump.h> +#include <rte_pause.h> + +#include "test.h" + +#define MAX_NUM 1 << 20 + +#define FAIL(x)\ + {printf(x "() test failed!\n");\ + return -1;} + +/* this is really a sanity check */ +static int +test_macros(int __rte_unused unused_parm) +{ +#define SMALLER 0x1000U +#define BIGGER 0x2000U +#define PTR_DIFF BIGGER - SMALLER +#define FAIL_MACRO(x)\ + {printf(#x "() test failed!\n");\ + return -1;} + + uintptr_t unused = 0; + + RTE_SET_USED(unused); + + if ((uintptr_t)RTE_PTR_ADD(SMALLER, PTR_DIFF) != BIGGER) + FAIL_MACRO(RTE_PTR_ADD); + if ((uintptr_t)RTE_PTR_SUB(BIGGER, PTR_DIFF) != SMALLER) + FAIL_MACRO(RTE_PTR_SUB); + if (RTE_PTR_DIFF(BIGGER, SMALLER) != PTR_DIFF) + FAIL_MACRO(RTE_PTR_DIFF); + if (RTE_MAX(SMALLER, BIGGER) != BIGGER) + FAIL_MACRO(RTE_MAX); + if (RTE_MIN(SMALLER, BIGGER) != SMALLER) + FAIL_MACRO(RTE_MIN); + + if (strncmp(RTE_STR(test), "test", sizeof("test"))) + FAIL_MACRO(RTE_STR); + + return 0; +} + +static int +test_misc(void) +{ + char memdump[] = "memdump_test"; + if (rte_bsf32(129)) + FAIL("rte_bsf32"); + + rte_memdump(stdout, "test", memdump, sizeof(memdump)); + rte_hexdump(stdout, "test", memdump, sizeof(memdump)); + + rte_pause(); + + return 0; +} + +static int +test_align(void) +{ +#define FAIL_ALIGN(x, i, p)\ + {printf(x "() test failed: %u %u\n", i, p);\ + return -1;} +#define ERROR_FLOOR(res, i, pow) \ + (res % pow) || /* check if not aligned */ \ + ((res / pow) != (i / pow)) /* check if correct alignment */ +#define ERROR_CEIL(res, i, pow) \ + (res % pow) || /* check if not aligned */ \ + ((i % pow) == 0 ? /* check if ceiling is invoked */ \ + val / pow != i / pow : /* if aligned */ \ + val / pow != (i / pow) + 1) /* if not aligned, hence +1 */ + + uint32_t i, p, val; + + for (i = 1, p = 1; i <= MAX_NUM; i ++) { + if (rte_align32pow2(i) != p) + FAIL_ALIGN("rte_align32pow2", i, p); + if (i == p) + p <<= 1; + } + + for (p = 2; p <= MAX_NUM; p <<= 1) { + + if (!rte_is_power_of_2(p)) + FAIL("rte_is_power_of_2"); + + for (i = 1; i <= MAX_NUM; i++) { + /* align floor */ + if (RTE_ALIGN_FLOOR((uintptr_t)i, p) % p) + FAIL_ALIGN("RTE_ALIGN_FLOOR", i, p); + + val = RTE_PTR_ALIGN_FLOOR((uintptr_t) i, p); + if (ERROR_FLOOR(val, i, p)) + FAIL_ALIGN("RTE_PTR_ALIGN_FLOOR", i, p); + + val = RTE_ALIGN_FLOOR(i, p); + if (ERROR_FLOOR(val, i, p)) + FAIL_ALIGN("RTE_ALIGN_FLOOR", i, p); + + /* align ceiling */ + val = RTE_PTR_ALIGN((uintptr_t) i, p); + if (ERROR_CEIL(val, i, p)) + FAIL_ALIGN("RTE_PTR_ALIGN", i, p); + + val = RTE_ALIGN(i, p); + if (ERROR_CEIL(val, i, p)) + FAIL_ALIGN("RTE_ALIGN", i, p); + + val = RTE_ALIGN_CEIL(i, p); + if (ERROR_CEIL(val, i, p)) + FAIL_ALIGN("RTE_ALIGN_CEIL", i, p); + + val = RTE_PTR_ALIGN_CEIL((uintptr_t)i, p); + if (ERROR_CEIL(val, i, p)) + FAIL_ALIGN("RTE_PTR_ALIGN_CEIL", i, p); + + /* by this point we know that val is aligned to p */ + if (!rte_is_aligned((void*)(uintptr_t) val, p)) + FAIL("rte_is_aligned"); + } + } + return 0; +} + +static int +test_log2(void) +{ + uint32_t i, base, compare; + const uint32_t max = 0x10000; + const uint32_t step = 1; + + for (i = 0; i < max; i = i + step) { + base = (uint32_t)ceilf(log2((uint32_t)i)); + compare = rte_log2_u32(i); + if (base != compare) { + printf("Wrong rte_log2_u32(%x) val %x, expected %x\n", + i, compare, base); + return TEST_FAILED; + } + } + return 0; +} + +static int +test_common(void) +{ + int ret = 0; + ret |= test_align(); + ret |= test_macros(0); + ret |= test_misc(); + ret |= test_log2(); + + return ret; +} + +REGISTER_TEST_COMMAND(common_autotest, test_common); diff --git a/dpdk/test/test/test_cpuflags.c b/dpdk/test/test/test_cpuflags.c new file mode 100644 index 00000000..08c16613 --- /dev/null +++ b/dpdk/test/test/test_cpuflags.c @@ -0,0 +1,205 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> + +#include <errno.h> +#include <stdint.h> +#include <rte_cpuflags.h> +#include <rte_debug.h> + +#include "test.h" + + +/* convenience define */ +#define CHECK_FOR_FLAG(x) \ + result = rte_cpu_get_flag_enabled(x); \ + printf("%s\n", cpu_flag_result(result)); \ + if (result == -ENOENT) \ + return -1; + +/* + * Helper function to display result + */ +static inline const char * +cpu_flag_result(int result) +{ + switch (result) { + case 0: + return "NOT PRESENT"; + case 1: + return "OK"; + default: + return "ERROR"; + } +} + + + +/* + * CPUID test + * =========== + * + * - Check flags from different registers with rte_cpu_get_flag_enabled() + * - Check if register and CPUID functions fail properly + */ + +static int +test_cpuflags(void) +{ + int result; + printf("\nChecking for flags from different registers...\n"); + +#ifdef RTE_ARCH_PPC_64 + printf("Check for PPC64:\t\t"); + CHECK_FOR_FLAG(RTE_CPUFLAG_PPC64); + + printf("Check for PPC32:\t\t"); + CHECK_FOR_FLAG(RTE_CPUFLAG_PPC32); + + printf("Check for VSX:\t\t"); + CHECK_FOR_FLAG(RTE_CPUFLAG_VSX); + + printf("Check for DFP:\t\t"); + CHECK_FOR_FLAG(RTE_CPUFLAG_DFP); + + printf("Check for FPU:\t\t"); + CHECK_FOR_FLAG(RTE_CPUFLAG_FPU); + + printf("Check for SMT:\t\t"); + CHECK_FOR_FLAG(RTE_CPUFLAG_SMT); + + printf("Check for MMU:\t\t"); + CHECK_FOR_FLAG(RTE_CPUFLAG_MMU); + + printf("Check for ALTIVEC:\t\t"); + CHECK_FOR_FLAG(RTE_CPUFLAG_ALTIVEC); + + printf("Check for ARCH_2_06:\t\t"); + CHECK_FOR_FLAG(RTE_CPUFLAG_ARCH_2_06); + + printf("Check for ARCH_2_07:\t\t"); + CHECK_FOR_FLAG(RTE_CPUFLAG_ARCH_2_07); + + printf("Check for ICACHE_SNOOP:\t\t"); + CHECK_FOR_FLAG(RTE_CPUFLAG_ICACHE_SNOOP); +#endif + +#if defined(RTE_ARCH_ARM) + printf("Check for NEON:\t\t"); + CHECK_FOR_FLAG(RTE_CPUFLAG_NEON); +#endif + +#if defined(RTE_ARCH_ARM64) + printf("Check for FP:\t\t"); + CHECK_FOR_FLAG(RTE_CPUFLAG_FP); + + printf("Check for ASIMD:\t"); + CHECK_FOR_FLAG(RTE_CPUFLAG_NEON); + + printf("Check for EVTSTRM:\t"); + CHECK_FOR_FLAG(RTE_CPUFLAG_EVTSTRM); + + printf("Check for AES:\t\t"); + CHECK_FOR_FLAG(RTE_CPUFLAG_AES); + + printf("Check for PMULL:\t"); + CHECK_FOR_FLAG(RTE_CPUFLAG_PMULL); + + printf("Check for SHA1:\t\t"); + CHECK_FOR_FLAG(RTE_CPUFLAG_SHA1); + + printf("Check for SHA2:\t\t"); + CHECK_FOR_FLAG(RTE_CPUFLAG_SHA2); + + printf("Check for CRC32:\t"); + CHECK_FOR_FLAG(RTE_CPUFLAG_CRC32); + + printf("Check for ATOMICS:\t"); + CHECK_FOR_FLAG(RTE_CPUFLAG_ATOMICS); +#endif + +#if defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_I686) + printf("Check for SSE:\t\t"); + CHECK_FOR_FLAG(RTE_CPUFLAG_SSE); + + printf("Check for SSE2:\t\t"); + CHECK_FOR_FLAG(RTE_CPUFLAG_SSE2); + + printf("Check for SSE3:\t\t"); + CHECK_FOR_FLAG(RTE_CPUFLAG_SSE3); + + printf("Check for SSE4.1:\t"); + CHECK_FOR_FLAG(RTE_CPUFLAG_SSE4_1); + + printf("Check for SSE4.2:\t"); + CHECK_FOR_FLAG(RTE_CPUFLAG_SSE4_2); + + printf("Check for AVX:\t\t"); + CHECK_FOR_FLAG(RTE_CPUFLAG_AVX); + + printf("Check for AVX2:\t\t"); + CHECK_FOR_FLAG(RTE_CPUFLAG_AVX2); + + printf("Check for AVX512F:\t"); + CHECK_FOR_FLAG(RTE_CPUFLAG_AVX512F); + + printf("Check for TRBOBST:\t"); + CHECK_FOR_FLAG(RTE_CPUFLAG_TRBOBST); + + printf("Check for ENERGY_EFF:\t"); + CHECK_FOR_FLAG(RTE_CPUFLAG_ENERGY_EFF); + + printf("Check for LAHF_SAHF:\t"); + CHECK_FOR_FLAG(RTE_CPUFLAG_LAHF_SAHF); + + printf("Check for 1GB_PG:\t"); + CHECK_FOR_FLAG(RTE_CPUFLAG_1GB_PG); + + printf("Check for INVTSC:\t"); + CHECK_FOR_FLAG(RTE_CPUFLAG_INVTSC); +#endif + + /* + * Check if invalid data is handled properly + */ + printf("\nCheck for invalid flag:\t"); + result = rte_cpu_get_flag_enabled(RTE_CPUFLAG_NUMFLAGS); + printf("%s\n", cpu_flag_result(result)); + if (result != -ENOENT) + return -1; + + return 0; +} + +REGISTER_TEST_COMMAND(cpuflags_autotest, test_cpuflags); diff --git a/dpdk/test/test/test_crc.c b/dpdk/test/test/test_crc.c new file mode 100644 index 00000000..9f2a17d4 --- /dev/null +++ b/dpdk/test/test/test_crc.c @@ -0,0 +1,193 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "test.h" + +#include <rte_hexdump.h> +#include <rte_malloc.h> +#include <rte_memcpy.h> +#include <rte_net_crc.h> + +#define CRC_VEC_LEN 32 +#define CRC32_VEC_LEN1 1512 +#define CRC32_VEC_LEN2 348 +#define CRC16_VEC_LEN1 12 +#define CRC16_VEC_LEN2 2 +#define LINE_LEN 75 + +/* CRC test vector */ +static const uint8_t crc_vec[CRC_VEC_LEN] = { + '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', + 'g', 'h', 'i', 'j', 'A', 'B', 'C', 'D', + 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', +}; + +/* 32-bit CRC test vector */ +static const uint8_t crc32_vec1[12] = { + 0xBE, 0xD7, 0x23, 0x47, 0x6B, 0x8F, + 0xB3, 0x14, 0x5E, 0xFB, 0x35, 0x59, +}; + +/* 16-bit CRC test vector 1 */ +static const uint8_t crc16_vec1[CRC16_VEC_LEN1] = { + 0x0D, 0x01, 0x01, 0x23, 0x45, 0x67, + 0x89, 0x01, 0x23, 0x45, 0x00, 0x01, +}; + +/* 16-bit CRC test vector 2 */ +static const uint8_t crc16_vec2[CRC16_VEC_LEN2] = { + 0x03, 0x3f, +}; +/** CRC results */ +static const uint32_t crc32_vec_res = 0xb491aab4; +static const uint32_t crc32_vec1_res = 0xac54d294; +static const uint32_t crc32_vec2_res = 0xefaae02f; +static const uint32_t crc16_vec_res = 0x6bec; +static const uint16_t crc16_vec1_res = 0x8cdd; +static const uint16_t crc16_vec2_res = 0xec5b; + +static int +crc_calc(const uint8_t *vec, + uint32_t vec_len, + enum rte_net_crc_type type) +{ + /* compute CRC */ + uint32_t ret = rte_net_crc_calc(vec, vec_len, type); + + /* dump data on console */ + TEST_HEXDUMP(stdout, NULL, vec, vec_len); + + return ret; +} + +static int +test_crc_calc(void) +{ + uint32_t i; + enum rte_net_crc_type type; + uint8_t *test_data; + uint32_t result; + int error; + + /* 32-bit ethernet CRC: Test 1 */ + type = RTE_NET_CRC32_ETH; + + result = crc_calc(crc_vec, CRC_VEC_LEN, type); + if (result != crc32_vec_res) + return -1; + + /* 32-bit ethernet CRC: Test 2 */ + test_data = rte_zmalloc(NULL, CRC32_VEC_LEN1, 0); + + for (i = 0; i < CRC32_VEC_LEN1; i += 12) + rte_memcpy(&test_data[i], crc32_vec1, 12); + + result = crc_calc(test_data, CRC32_VEC_LEN1, type); + if (result != crc32_vec1_res) { + error = -2; + goto fail; + } + + /* 32-bit ethernet CRC: Test 3 */ + for (i = 0; i < CRC32_VEC_LEN2; i += 12) + rte_memcpy(&test_data[i], crc32_vec1, 12); + + result = crc_calc(test_data, CRC32_VEC_LEN2, type); + if (result != crc32_vec2_res) { + error = -3; + goto fail; + } + + /* 16-bit CCITT CRC: Test 4 */ + type = RTE_NET_CRC16_CCITT; + result = crc_calc(crc_vec, CRC_VEC_LEN, type); + if (result != crc16_vec_res) { + error = -4; + goto fail; + } + /* 16-bit CCITT CRC: Test 5 */ + result = crc_calc(crc16_vec1, CRC16_VEC_LEN1, type); + if (result != crc16_vec1_res) { + error = -5; + goto fail; + } + /* 16-bit CCITT CRC: Test 6 */ + result = crc_calc(crc16_vec2, CRC16_VEC_LEN2, type); + if (result != crc16_vec2_res) { + error = -6; + goto fail; + } + + rte_free(test_data); + return 0; + +fail: + rte_free(test_data); + return error; +} + +static int +test_crc(void) +{ + int ret; + /* set CRC scalar mode */ + rte_net_crc_set_alg(RTE_NET_CRC_SCALAR); + + ret = test_crc_calc(); + if (ret < 0) { + printf("test_crc (scalar): failed (%d)\n", ret); + return ret; + } + /* set CRC sse4.2 mode */ + rte_net_crc_set_alg(RTE_NET_CRC_SSE42); + + ret = test_crc_calc(); + if (ret < 0) { + printf("test_crc (x86_64_SSE4.2): failed (%d)\n", ret); + return ret; + } + + /* set CRC neon mode */ + rte_net_crc_set_alg(RTE_NET_CRC_NEON); + + ret = test_crc_calc(); + if (ret < 0) { + printf("test crc (arm64 neon pmull): failed (%d)\n", ret); + return ret; + } + + return 0; +} + +REGISTER_TEST_COMMAND(crc_autotest, test_crc); diff --git a/dpdk/test/test/test_cryptodev.c b/dpdk/test/test/test_cryptodev.c new file mode 100644 index 00000000..7997b488 --- /dev/null +++ b/dpdk/test/test/test_cryptodev.c @@ -0,0 +1,9817 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2015-2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <time.h> + +#include <rte_common.h> +#include <rte_hexdump.h> +#include <rte_mbuf.h> +#include <rte_malloc.h> +#include <rte_memcpy.h> +#include <rte_pause.h> +#include <rte_bus_vdev.h> + +#include <rte_crypto.h> +#include <rte_cryptodev.h> +#include <rte_cryptodev_pmd.h> + +#ifdef RTE_LIBRTE_PMD_CRYPTO_SCHEDULER +#include <rte_cryptodev_scheduler.h> +#include <rte_cryptodev_scheduler_operations.h> +#endif + +#include "test.h" +#include "test_cryptodev.h" + +#include "test_cryptodev_blockcipher.h" +#include "test_cryptodev_aes_test_vectors.h" +#include "test_cryptodev_des_test_vectors.h" +#include "test_cryptodev_hash_test_vectors.h" +#include "test_cryptodev_kasumi_test_vectors.h" +#include "test_cryptodev_kasumi_hash_test_vectors.h" +#include "test_cryptodev_snow3g_test_vectors.h" +#include "test_cryptodev_snow3g_hash_test_vectors.h" +#include "test_cryptodev_zuc_test_vectors.h" +#include "test_cryptodev_aead_test_vectors.h" +#include "test_cryptodev_hmac_test_vectors.h" + +static int gbl_driver_id; + +struct crypto_testsuite_params { + struct rte_mempool *mbuf_pool; + struct rte_mempool *large_mbuf_pool; + struct rte_mempool *op_mpool; + struct rte_mempool *session_mpool; + struct rte_cryptodev_config conf; + struct rte_cryptodev_qp_conf qp_conf; + + uint8_t valid_devs[RTE_CRYPTO_MAX_DEVS]; + uint8_t valid_dev_count; +}; + +struct crypto_unittest_params { + struct rte_crypto_sym_xform cipher_xform; + struct rte_crypto_sym_xform auth_xform; + struct rte_crypto_sym_xform aead_xform; + + struct rte_cryptodev_sym_session *sess; + + struct rte_crypto_op *op; + + struct rte_mbuf *obuf, *ibuf; + + uint8_t *digest; +}; + +#define ALIGN_POW2_ROUNDUP(num, align) \ + (((num) + (align) - 1) & ~((align) - 1)) + +/* + * Forward declarations. + */ +static int +test_AES_CBC_HMAC_SHA512_decrypt_create_session_params( + struct crypto_unittest_params *ut_params, uint8_t *cipher_key, + uint8_t *hmac_key); + +static int +test_AES_CBC_HMAC_SHA512_decrypt_perform(struct rte_cryptodev_sym_session *sess, + struct crypto_unittest_params *ut_params, + struct crypto_testsuite_params *ts_param, + const uint8_t *cipher, + const uint8_t *digest, + const uint8_t *iv); + +static struct rte_mbuf * +setup_test_string(struct rte_mempool *mpool, + const char *string, size_t len, uint8_t blocksize) +{ + struct rte_mbuf *m = rte_pktmbuf_alloc(mpool); + size_t t_len = len - (blocksize ? (len % blocksize) : 0); + + memset(m->buf_addr, 0, m->buf_len); + if (m) { + char *dst = rte_pktmbuf_append(m, t_len); + + if (!dst) { + rte_pktmbuf_free(m); + return NULL; + } + if (string != NULL) + rte_memcpy(dst, string, t_len); + else + memset(dst, 0, t_len); + } + + return m; +} + +/* Get number of bytes in X bits (rounding up) */ +static uint32_t +ceil_byte_length(uint32_t num_bits) +{ + if (num_bits % 8) + return ((num_bits >> 3) + 1); + else + return (num_bits >> 3); +} + +static struct rte_crypto_op * +process_crypto_request(uint8_t dev_id, struct rte_crypto_op *op) +{ + if (rte_cryptodev_enqueue_burst(dev_id, 0, &op, 1) != 1) { + printf("Error sending packet for encryption"); + return NULL; + } + + op = NULL; + + while (rte_cryptodev_dequeue_burst(dev_id, 0, &op, 1) == 0) + rte_pause(); + + return op; +} + +static struct crypto_testsuite_params testsuite_params = { NULL }; +static struct crypto_unittest_params unittest_params; + +static int +testsuite_setup(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct rte_cryptodev_info info; + uint32_t i = 0, nb_devs, dev_id; + int ret; + uint16_t qp_id; + + memset(ts_params, 0, sizeof(*ts_params)); + + ts_params->mbuf_pool = rte_mempool_lookup("CRYPTO_MBUFPOOL"); + if (ts_params->mbuf_pool == NULL) { + /* Not already created so create */ + ts_params->mbuf_pool = rte_pktmbuf_pool_create( + "CRYPTO_MBUFPOOL", + NUM_MBUFS, MBUF_CACHE_SIZE, 0, MBUF_SIZE, + rte_socket_id()); + if (ts_params->mbuf_pool == NULL) { + RTE_LOG(ERR, USER1, "Can't create CRYPTO_MBUFPOOL\n"); + return TEST_FAILED; + } + } + + ts_params->large_mbuf_pool = rte_mempool_lookup( + "CRYPTO_LARGE_MBUFPOOL"); + if (ts_params->large_mbuf_pool == NULL) { + /* Not already created so create */ + ts_params->large_mbuf_pool = rte_pktmbuf_pool_create( + "CRYPTO_LARGE_MBUFPOOL", + 1, 0, 0, UINT16_MAX, + rte_socket_id()); + if (ts_params->large_mbuf_pool == NULL) { + RTE_LOG(ERR, USER1, + "Can't create CRYPTO_LARGE_MBUFPOOL\n"); + return TEST_FAILED; + } + } + + ts_params->op_mpool = rte_crypto_op_pool_create( + "MBUF_CRYPTO_SYM_OP_POOL", + RTE_CRYPTO_OP_TYPE_SYMMETRIC, + NUM_MBUFS, MBUF_CACHE_SIZE, + DEFAULT_NUM_XFORMS * + sizeof(struct rte_crypto_sym_xform) + + MAXIMUM_IV_LENGTH, + rte_socket_id()); + if (ts_params->op_mpool == NULL) { + RTE_LOG(ERR, USER1, "Can't create CRYPTO_OP_POOL\n"); + return TEST_FAILED; + } + + /* Create an AESNI MB device if required */ + if (gbl_driver_id == rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_AESNI_MB_PMD))) { + nb_devs = rte_cryptodev_device_count_by_driver( + rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_AESNI_MB_PMD))); + if (nb_devs < 1) { + ret = rte_vdev_init( + RTE_STR(CRYPTODEV_NAME_AESNI_MB_PMD), NULL); + + TEST_ASSERT(ret == 0, + "Failed to create instance of" + " pmd : %s", + RTE_STR(CRYPTODEV_NAME_AESNI_MB_PMD)); + } + } + + /* Create an AESNI GCM device if required */ + if (gbl_driver_id == rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_AESNI_GCM_PMD))) { + nb_devs = rte_cryptodev_device_count_by_driver( + rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_AESNI_GCM_PMD))); + if (nb_devs < 1) { + TEST_ASSERT_SUCCESS(rte_vdev_init( + RTE_STR(CRYPTODEV_NAME_AESNI_GCM_PMD), NULL), + "Failed to create instance of" + " pmd : %s", + RTE_STR(CRYPTODEV_NAME_AESNI_GCM_PMD)); + } + } + + /* Create a SNOW 3G device if required */ + if (gbl_driver_id == rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_SNOW3G_PMD))) { + nb_devs = rte_cryptodev_device_count_by_driver( + rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_SNOW3G_PMD))); + if (nb_devs < 1) { + TEST_ASSERT_SUCCESS(rte_vdev_init( + RTE_STR(CRYPTODEV_NAME_SNOW3G_PMD), NULL), + "Failed to create instance of" + " pmd : %s", + RTE_STR(CRYPTODEV_NAME_SNOW3G_PMD)); + } + } + + /* Create a KASUMI device if required */ + if (gbl_driver_id == rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_KASUMI_PMD))) { + nb_devs = rte_cryptodev_device_count_by_driver( + rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_KASUMI_PMD))); + if (nb_devs < 1) { + TEST_ASSERT_SUCCESS(rte_vdev_init( + RTE_STR(CRYPTODEV_NAME_KASUMI_PMD), NULL), + "Failed to create instance of" + " pmd : %s", + RTE_STR(CRYPTODEV_NAME_KASUMI_PMD)); + } + } + + /* Create a ZUC device if required */ + if (gbl_driver_id == rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_ZUC_PMD))) { + nb_devs = rte_cryptodev_device_count_by_driver( + rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_ZUC_PMD))); + if (nb_devs < 1) { + TEST_ASSERT_SUCCESS(rte_vdev_init( + RTE_STR(CRYPTODEV_NAME_ZUC_PMD), NULL), + "Failed to create instance of" + " pmd : %s", + RTE_STR(CRYPTODEV_NAME_ZUC_PMD)); + } + } + + /* Create a NULL device if required */ + if (gbl_driver_id == rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_NULL_PMD))) { + nb_devs = rte_cryptodev_device_count_by_driver( + rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_NULL_PMD))); + if (nb_devs < 1) { + ret = rte_vdev_init( + RTE_STR(CRYPTODEV_NAME_NULL_PMD), NULL); + + TEST_ASSERT(ret == 0, + "Failed to create instance of" + " pmd : %s", + RTE_STR(CRYPTODEV_NAME_NULL_PMD)); + } + } + + /* Create an OPENSSL device if required */ + if (gbl_driver_id == rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_OPENSSL_PMD))) { + nb_devs = rte_cryptodev_device_count_by_driver( + rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_OPENSSL_PMD))); + if (nb_devs < 1) { + ret = rte_vdev_init( + RTE_STR(CRYPTODEV_NAME_OPENSSL_PMD), + NULL); + + TEST_ASSERT(ret == 0, "Failed to create " + "instance of pmd : %s", + RTE_STR(CRYPTODEV_NAME_OPENSSL_PMD)); + } + } + + /* Create a ARMv8 device if required */ + if (gbl_driver_id == rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_ARMV8_PMD))) { + nb_devs = rte_cryptodev_device_count_by_driver( + rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_ARMV8_PMD))); + if (nb_devs < 1) { + ret = rte_vdev_init( + RTE_STR(CRYPTODEV_NAME_ARMV8_PMD), + NULL); + + TEST_ASSERT(ret == 0, "Failed to create " + "instance of pmd : %s", + RTE_STR(CRYPTODEV_NAME_ARMV8_PMD)); + } + } + + /* Create a MRVL device if required */ + if (gbl_driver_id == rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_MRVL_PMD))) { +#ifndef RTE_LIBRTE_PMD_MRVL_CRYPTO + RTE_LOG(ERR, USER1, "CONFIG_RTE_LIBRTE_PMD_MRVL_CRYPTO must be" + " enabled in config file to run this testsuite.\n"); + return TEST_FAILED; +#endif + nb_devs = rte_cryptodev_device_count_by_driver( + rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_MRVL_PMD))); + if (nb_devs < 1) { + ret = rte_vdev_init( + RTE_STR(CRYPTODEV_NAME_MRVL_PMD), + NULL); + + TEST_ASSERT(ret == 0, "Failed to create " + "instance of pmd : %s", + RTE_STR(CRYPTODEV_NAME_MRVL_PMD)); + } + } + +#ifdef RTE_LIBRTE_PMD_CRYPTO_SCHEDULER + if (gbl_driver_id == rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_SCHEDULER_PMD))) { + + nb_devs = rte_cryptodev_device_count_by_driver( + rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_SCHEDULER_PMD))); + if (nb_devs < 1) { + ret = rte_vdev_init( + RTE_STR(CRYPTODEV_NAME_SCHEDULER_PMD), + NULL); + + TEST_ASSERT(ret == 0, + "Failed to create instance %u of" + " pmd : %s", + i, RTE_STR(CRYPTODEV_NAME_SCHEDULER_PMD)); + } + } +#endif /* RTE_LIBRTE_PMD_CRYPTO_SCHEDULER */ + + nb_devs = rte_cryptodev_count(); + if (nb_devs < 1) { + RTE_LOG(ERR, USER1, "No crypto devices found?\n"); + return TEST_FAILED; + } + + /* Create list of valid crypto devs */ + for (i = 0; i < nb_devs; i++) { + rte_cryptodev_info_get(i, &info); + if (info.driver_id == gbl_driver_id) + ts_params->valid_devs[ts_params->valid_dev_count++] = i; + } + + if (ts_params->valid_dev_count < 1) + return TEST_FAILED; + + /* Set up all the qps on the first of the valid devices found */ + + dev_id = ts_params->valid_devs[0]; + + rte_cryptodev_info_get(dev_id, &info); + + ts_params->conf.nb_queue_pairs = info.max_nb_queue_pairs; + ts_params->conf.socket_id = SOCKET_ID_ANY; + + unsigned int session_size = rte_cryptodev_get_private_session_size(dev_id); + + /* + * Create mempool with maximum number of sessions * 2, + * to include the session headers + */ + ts_params->session_mpool = rte_mempool_create( + "test_sess_mp", + info.sym.max_nb_sessions * 2, + session_size, + 0, 0, NULL, NULL, NULL, + NULL, SOCKET_ID_ANY, + 0); + + TEST_ASSERT_NOT_NULL(ts_params->session_mpool, + "session mempool allocation failed"); + + TEST_ASSERT_SUCCESS(rte_cryptodev_configure(dev_id, + &ts_params->conf), + "Failed to configure cryptodev %u with %u qps", + dev_id, ts_params->conf.nb_queue_pairs); + + ts_params->qp_conf.nb_descriptors = DEFAULT_NUM_OPS_INFLIGHT; + + for (qp_id = 0; qp_id < info.max_nb_queue_pairs; qp_id++) { + TEST_ASSERT_SUCCESS(rte_cryptodev_queue_pair_setup( + dev_id, qp_id, &ts_params->qp_conf, + rte_cryptodev_socket_id(dev_id), + ts_params->session_mpool), + "Failed to setup queue pair %u on cryptodev %u", + qp_id, dev_id); + } + + return TEST_SUCCESS; +} + +static void +testsuite_teardown(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + + if (ts_params->mbuf_pool != NULL) { + RTE_LOG(DEBUG, USER1, "CRYPTO_MBUFPOOL count %u\n", + rte_mempool_avail_count(ts_params->mbuf_pool)); + } + + if (ts_params->op_mpool != NULL) { + RTE_LOG(DEBUG, USER1, "CRYPTO_OP_POOL count %u\n", + rte_mempool_avail_count(ts_params->op_mpool)); + } + + /* Free session mempools */ + if (ts_params->session_mpool != NULL) { + rte_mempool_free(ts_params->session_mpool); + ts_params->session_mpool = NULL; + } +} + +static int +ut_setup(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + + uint16_t qp_id; + + /* Clear unit test parameters before running test */ + memset(ut_params, 0, sizeof(*ut_params)); + + /* Reconfigure device to default parameters */ + ts_params->conf.socket_id = SOCKET_ID_ANY; + + TEST_ASSERT_SUCCESS(rte_cryptodev_configure(ts_params->valid_devs[0], + &ts_params->conf), + "Failed to configure cryptodev %u", + ts_params->valid_devs[0]); + + for (qp_id = 0; qp_id < ts_params->conf.nb_queue_pairs ; qp_id++) { + TEST_ASSERT_SUCCESS(rte_cryptodev_queue_pair_setup( + ts_params->valid_devs[0], qp_id, + &ts_params->qp_conf, + rte_cryptodev_socket_id(ts_params->valid_devs[0]), + ts_params->session_mpool), + "Failed to setup queue pair %u on cryptodev %u", + qp_id, ts_params->valid_devs[0]); + } + + + rte_cryptodev_stats_reset(ts_params->valid_devs[0]); + + /* Start the device */ + TEST_ASSERT_SUCCESS(rte_cryptodev_start(ts_params->valid_devs[0]), + "Failed to start cryptodev %u", + ts_params->valid_devs[0]); + + return TEST_SUCCESS; +} + +static void +ut_teardown(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + struct rte_cryptodev_stats stats; + + /* free crypto session structure */ + if (ut_params->sess) { + rte_cryptodev_sym_session_clear(ts_params->valid_devs[0], + ut_params->sess); + rte_cryptodev_sym_session_free(ut_params->sess); + ut_params->sess = NULL; + } + + /* free crypto operation structure */ + if (ut_params->op) + rte_crypto_op_free(ut_params->op); + + /* + * free mbuf - both obuf and ibuf are usually the same, + * so check if they point at the same address is necessary, + * to avoid freeing the mbuf twice. + */ + if (ut_params->obuf) { + rte_pktmbuf_free(ut_params->obuf); + if (ut_params->ibuf == ut_params->obuf) + ut_params->ibuf = 0; + ut_params->obuf = 0; + } + if (ut_params->ibuf) { + rte_pktmbuf_free(ut_params->ibuf); + ut_params->ibuf = 0; + } + + if (ts_params->mbuf_pool != NULL) + RTE_LOG(DEBUG, USER1, "CRYPTO_MBUFPOOL count %u\n", + rte_mempool_avail_count(ts_params->mbuf_pool)); + + rte_cryptodev_stats_get(ts_params->valid_devs[0], &stats); + + /* Stop the device */ + rte_cryptodev_stop(ts_params->valid_devs[0]); +} + +static int +test_device_configure_invalid_dev_id(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + uint16_t dev_id, num_devs = 0; + + TEST_ASSERT((num_devs = rte_cryptodev_count()) >= 1, + "Need at least %d devices for test", 1); + + /* valid dev_id values */ + dev_id = ts_params->valid_devs[ts_params->valid_dev_count - 1]; + + /* Stop the device in case it's started so it can be configured */ + rte_cryptodev_stop(ts_params->valid_devs[dev_id]); + + TEST_ASSERT_SUCCESS(rte_cryptodev_configure(dev_id, &ts_params->conf), + "Failed test for rte_cryptodev_configure: " + "invalid dev_num %u", dev_id); + + /* invalid dev_id values */ + dev_id = num_devs; + + TEST_ASSERT_FAIL(rte_cryptodev_configure(dev_id, &ts_params->conf), + "Failed test for rte_cryptodev_configure: " + "invalid dev_num %u", dev_id); + + dev_id = 0xff; + + TEST_ASSERT_FAIL(rte_cryptodev_configure(dev_id, &ts_params->conf), + "Failed test for rte_cryptodev_configure:" + "invalid dev_num %u", dev_id); + + return TEST_SUCCESS; +} + +static int +test_device_configure_invalid_queue_pair_ids(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + uint16_t orig_nb_qps = ts_params->conf.nb_queue_pairs; + + /* Stop the device in case it's started so it can be configured */ + rte_cryptodev_stop(ts_params->valid_devs[0]); + + /* valid - one queue pairs */ + ts_params->conf.nb_queue_pairs = 1; + + TEST_ASSERT_SUCCESS(rte_cryptodev_configure(ts_params->valid_devs[0], + &ts_params->conf), + "Failed to configure cryptodev: dev_id %u, qp_id %u", + ts_params->valid_devs[0], ts_params->conf.nb_queue_pairs); + + + /* valid - max value queue pairs */ + ts_params->conf.nb_queue_pairs = MAX_NUM_QPS_PER_QAT_DEVICE; + + TEST_ASSERT_SUCCESS(rte_cryptodev_configure(ts_params->valid_devs[0], + &ts_params->conf), + "Failed to configure cryptodev: dev_id %u, qp_id %u", + ts_params->valid_devs[0], + ts_params->conf.nb_queue_pairs); + + + /* invalid - zero queue pairs */ + ts_params->conf.nb_queue_pairs = 0; + + TEST_ASSERT_FAIL(rte_cryptodev_configure(ts_params->valid_devs[0], + &ts_params->conf), + "Failed test for rte_cryptodev_configure, dev_id %u," + " invalid qps: %u", + ts_params->valid_devs[0], + ts_params->conf.nb_queue_pairs); + + + /* invalid - max value supported by field queue pairs */ + ts_params->conf.nb_queue_pairs = UINT16_MAX; + + TEST_ASSERT_FAIL(rte_cryptodev_configure(ts_params->valid_devs[0], + &ts_params->conf), + "Failed test for rte_cryptodev_configure, dev_id %u," + " invalid qps: %u", + ts_params->valid_devs[0], + ts_params->conf.nb_queue_pairs); + + + /* invalid - max value + 1 queue pairs */ + ts_params->conf.nb_queue_pairs = MAX_NUM_QPS_PER_QAT_DEVICE + 1; + + TEST_ASSERT_FAIL(rte_cryptodev_configure(ts_params->valid_devs[0], + &ts_params->conf), + "Failed test for rte_cryptodev_configure, dev_id %u," + " invalid qps: %u", + ts_params->valid_devs[0], + ts_params->conf.nb_queue_pairs); + + /* revert to original testsuite value */ + ts_params->conf.nb_queue_pairs = orig_nb_qps; + + return TEST_SUCCESS; +} + +static int +test_queue_pair_descriptor_setup(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct rte_cryptodev_info dev_info; + struct rte_cryptodev_qp_conf qp_conf = { + .nb_descriptors = MAX_NUM_OPS_INFLIGHT + }; + + uint16_t qp_id; + + /* Stop the device in case it's started so it can be configured */ + rte_cryptodev_stop(ts_params->valid_devs[0]); + + + rte_cryptodev_info_get(ts_params->valid_devs[0], &dev_info); + + TEST_ASSERT_SUCCESS(rte_cryptodev_configure(ts_params->valid_devs[0], + &ts_params->conf), + "Failed to configure cryptodev %u", + ts_params->valid_devs[0]); + + /* + * Test various ring sizes on this device. memzones can't be + * freed so are re-used if ring is released and re-created. + */ + qp_conf.nb_descriptors = MIN_NUM_OPS_INFLIGHT; /* min size*/ + + for (qp_id = 0; qp_id < ts_params->conf.nb_queue_pairs; qp_id++) { + TEST_ASSERT_SUCCESS(rte_cryptodev_queue_pair_setup( + ts_params->valid_devs[0], qp_id, &qp_conf, + rte_cryptodev_socket_id( + ts_params->valid_devs[0]), + ts_params->session_mpool), + "Failed test for " + "rte_cryptodev_queue_pair_setup: num_inflights " + "%u on qp %u on cryptodev %u", + qp_conf.nb_descriptors, qp_id, + ts_params->valid_devs[0]); + } + + qp_conf.nb_descriptors = (uint32_t)(MAX_NUM_OPS_INFLIGHT / 2); + + for (qp_id = 0; qp_id < ts_params->conf.nb_queue_pairs; qp_id++) { + TEST_ASSERT_SUCCESS(rte_cryptodev_queue_pair_setup( + ts_params->valid_devs[0], qp_id, &qp_conf, + rte_cryptodev_socket_id( + ts_params->valid_devs[0]), + ts_params->session_mpool), + "Failed test for" + " rte_cryptodev_queue_pair_setup: num_inflights" + " %u on qp %u on cryptodev %u", + qp_conf.nb_descriptors, qp_id, + ts_params->valid_devs[0]); + } + + qp_conf.nb_descriptors = MAX_NUM_OPS_INFLIGHT; /* valid */ + + for (qp_id = 0; qp_id < ts_params->conf.nb_queue_pairs; qp_id++) { + TEST_ASSERT_SUCCESS(rte_cryptodev_queue_pair_setup( + ts_params->valid_devs[0], qp_id, &qp_conf, + rte_cryptodev_socket_id( + ts_params->valid_devs[0]), + ts_params->session_mpool), + "Failed test for " + "rte_cryptodev_queue_pair_setup: num_inflights" + " %u on qp %u on cryptodev %u", + qp_conf.nb_descriptors, qp_id, + ts_params->valid_devs[0]); + } + + /* invalid number of descriptors - max supported + 2 */ + qp_conf.nb_descriptors = MAX_NUM_OPS_INFLIGHT + 2; + + for (qp_id = 0; qp_id < ts_params->conf.nb_queue_pairs; qp_id++) { + TEST_ASSERT_FAIL(rte_cryptodev_queue_pair_setup( + ts_params->valid_devs[0], qp_id, &qp_conf, + rte_cryptodev_socket_id( + ts_params->valid_devs[0]), + ts_params->session_mpool), + "Unexpectedly passed test for " + "rte_cryptodev_queue_pair_setup:" + "num_inflights %u on qp %u on cryptodev %u", + qp_conf.nb_descriptors, qp_id, + ts_params->valid_devs[0]); + } + + /* invalid number of descriptors - max value of parameter */ + qp_conf.nb_descriptors = UINT32_MAX-1; + + for (qp_id = 0; qp_id < ts_params->conf.nb_queue_pairs; qp_id++) { + TEST_ASSERT_FAIL(rte_cryptodev_queue_pair_setup( + ts_params->valid_devs[0], qp_id, &qp_conf, + rte_cryptodev_socket_id( + ts_params->valid_devs[0]), + ts_params->session_mpool), + "Unexpectedly passed test for " + "rte_cryptodev_queue_pair_setup:" + "num_inflights %u on qp %u on cryptodev %u", + qp_conf.nb_descriptors, qp_id, + ts_params->valid_devs[0]); + } + + qp_conf.nb_descriptors = DEFAULT_NUM_OPS_INFLIGHT; + + for (qp_id = 0; qp_id < ts_params->conf.nb_queue_pairs; qp_id++) { + TEST_ASSERT_SUCCESS(rte_cryptodev_queue_pair_setup( + ts_params->valid_devs[0], qp_id, &qp_conf, + rte_cryptodev_socket_id( + ts_params->valid_devs[0]), + ts_params->session_mpool), + "Failed test for" + " rte_cryptodev_queue_pair_setup:" + "num_inflights %u on qp %u on cryptodev %u", + qp_conf.nb_descriptors, qp_id, + ts_params->valid_devs[0]); + } + + /* invalid number of descriptors - max supported + 1 */ + qp_conf.nb_descriptors = DEFAULT_NUM_OPS_INFLIGHT + 1; + + for (qp_id = 0; qp_id < ts_params->conf.nb_queue_pairs; qp_id++) { + TEST_ASSERT_FAIL(rte_cryptodev_queue_pair_setup( + ts_params->valid_devs[0], qp_id, &qp_conf, + rte_cryptodev_socket_id( + ts_params->valid_devs[0]), + ts_params->session_mpool), + "Unexpectedly passed test for " + "rte_cryptodev_queue_pair_setup:" + "num_inflights %u on qp %u on cryptodev %u", + qp_conf.nb_descriptors, qp_id, + ts_params->valid_devs[0]); + } + + /* test invalid queue pair id */ + qp_conf.nb_descriptors = DEFAULT_NUM_OPS_INFLIGHT; /*valid */ + + qp_id = DEFAULT_NUM_QPS_PER_QAT_DEVICE; /*invalid */ + + TEST_ASSERT_FAIL(rte_cryptodev_queue_pair_setup( + ts_params->valid_devs[0], + qp_id, &qp_conf, + rte_cryptodev_socket_id(ts_params->valid_devs[0]), + ts_params->session_mpool), + "Failed test for rte_cryptodev_queue_pair_setup:" + "invalid qp %u on cryptodev %u", + qp_id, ts_params->valid_devs[0]); + + qp_id = 0xffff; /*invalid*/ + + TEST_ASSERT_FAIL(rte_cryptodev_queue_pair_setup( + ts_params->valid_devs[0], + qp_id, &qp_conf, + rte_cryptodev_socket_id(ts_params->valid_devs[0]), + ts_params->session_mpool), + "Failed test for rte_cryptodev_queue_pair_setup:" + "invalid qp %u on cryptodev %u", + qp_id, ts_params->valid_devs[0]); + + return TEST_SUCCESS; +} + +/* ***** Plaintext data for tests ***** */ + +const char catch_22_quote_1[] = + "There was only one catch and that was Catch-22, which " + "specified that a concern for one's safety in the face of " + "dangers that were real and immediate was the process of a " + "rational mind. Orr was crazy and could be grounded. All he " + "had to do was ask; and as soon as he did, he would no longer " + "be crazy and would have to fly more missions. Orr would be " + "crazy to fly more missions and sane if he didn't, but if he " + "was sane he had to fly them. If he flew them he was crazy " + "and didn't have to; but if he didn't want to he was sane and " + "had to. Yossarian was moved very deeply by the absolute " + "simplicity of this clause of Catch-22 and let out a " + "respectful whistle. \"That's some catch, that Catch-22\", he " + "observed. \"It's the best there is,\" Doc Daneeka agreed."; + +const char catch_22_quote[] = + "What a lousy earth! He wondered how many people were " + "destitute that same night even in his own prosperous country, " + "how many homes were shanties, how many husbands were drunk " + "and wives socked, and how many children were bullied, abused, " + "or abandoned. How many families hungered for food they could " + "not afford to buy? How many hearts were broken? How many " + "suicides would take place that same night, how many people " + "would go insane? How many cockroaches and landlords would " + "triumph? How many winners were losers, successes failures, " + "and rich men poor men? How many wise guys were stupid? How " + "many happy endings were unhappy endings? How many honest men " + "were liars, brave men cowards, loyal men traitors, how many " + "sainted men were corrupt, how many people in positions of " + "trust had sold their souls to bodyguards, how many had never " + "had souls? How many straight-and-narrow paths were crooked " + "paths? How many best families were worst families and how " + "many good people were bad people? When you added them all up " + "and then subtracted, you might be left with only the children, " + "and perhaps with Albert Einstein and an old violinist or " + "sculptor somewhere."; + +#define QUOTE_480_BYTES (480) +#define QUOTE_512_BYTES (512) +#define QUOTE_768_BYTES (768) +#define QUOTE_1024_BYTES (1024) + + + +/* ***** SHA1 Hash Tests ***** */ + +#define HMAC_KEY_LENGTH_SHA1 (DIGEST_BYTE_LENGTH_SHA1) + +static uint8_t hmac_sha1_key[] = { + 0xF8, 0x2A, 0xC7, 0x54, 0xDB, 0x96, 0x18, 0xAA, + 0xC3, 0xA1, 0x53, 0xF6, 0x1F, 0x17, 0x60, 0xBD, + 0xDE, 0xF4, 0xDE, 0xAD }; + +/* ***** SHA224 Hash Tests ***** */ + +#define HMAC_KEY_LENGTH_SHA224 (DIGEST_BYTE_LENGTH_SHA224) + + +/* ***** AES-CBC Cipher Tests ***** */ + +#define CIPHER_KEY_LENGTH_AES_CBC (16) +#define CIPHER_IV_LENGTH_AES_CBC (CIPHER_KEY_LENGTH_AES_CBC) + +static uint8_t aes_cbc_key[] = { + 0xE4, 0x23, 0x33, 0x8A, 0x35, 0x64, 0x61, 0xE2, + 0x49, 0x03, 0xDD, 0xC6, 0xB8, 0xCA, 0x55, 0x7A }; + +static uint8_t aes_cbc_iv[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f }; + + +/* ***** AES-CBC / HMAC-SHA1 Hash Tests ***** */ + +static const uint8_t catch_22_quote_2_512_bytes_AES_CBC_ciphertext[] = { + 0x8B, 0x4D, 0xDA, 0x1B, 0xCF, 0x04, 0xA0, 0x31, + 0xB4, 0xBF, 0xBD, 0x68, 0x43, 0x20, 0x7E, 0x76, + 0xB1, 0x96, 0x8B, 0xA2, 0x7C, 0xA2, 0x83, 0x9E, + 0x39, 0x5A, 0x2F, 0x7E, 0x92, 0xB4, 0x48, 0x1A, + 0x3F, 0x6B, 0x5D, 0xDF, 0x52, 0x85, 0x5F, 0x8E, + 0x42, 0x3C, 0xFB, 0xE9, 0x1A, 0x24, 0xD6, 0x08, + 0xDD, 0xFD, 0x16, 0xFB, 0xE9, 0x55, 0xEF, 0xF0, + 0xA0, 0x8D, 0x13, 0xAB, 0x81, 0xC6, 0x90, 0x01, + 0xB5, 0x18, 0x84, 0xB3, 0xF6, 0xE6, 0x11, 0x57, + 0xD6, 0x71, 0xC6, 0x3C, 0x3F, 0x2F, 0x33, 0xEE, + 0x24, 0x42, 0x6E, 0xAC, 0x0B, 0xCA, 0xEC, 0xF9, + 0x84, 0xF8, 0x22, 0xAA, 0x60, 0xF0, 0x32, 0xA9, + 0x75, 0x75, 0x3B, 0xCB, 0x70, 0x21, 0x0A, 0x8D, + 0x0F, 0xE0, 0xC4, 0x78, 0x2B, 0xF8, 0x97, 0xE3, + 0xE4, 0x26, 0x4B, 0x29, 0xDA, 0x88, 0xCD, 0x46, + 0xEC, 0xAA, 0xF9, 0x7F, 0xF1, 0x15, 0xEA, 0xC3, + 0x87, 0xE6, 0x31, 0xF2, 0xCF, 0xDE, 0x4D, 0x80, + 0x70, 0x91, 0x7E, 0x0C, 0xF7, 0x26, 0x3A, 0x92, + 0x4F, 0x18, 0x83, 0xC0, 0x8F, 0x59, 0x01, 0xA5, + 0x88, 0xD1, 0xDB, 0x26, 0x71, 0x27, 0x16, 0xF5, + 0xEE, 0x10, 0x82, 0xAC, 0x68, 0x26, 0x9B, 0xE2, + 0x6D, 0xD8, 0x9A, 0x80, 0xDF, 0x04, 0x31, 0xD5, + 0xF1, 0x35, 0x5C, 0x3B, 0xDD, 0x9A, 0x65, 0xBA, + 0x58, 0x34, 0x85, 0x61, 0x1C, 0x42, 0x10, 0x76, + 0x73, 0x02, 0x42, 0xC9, 0x23, 0x18, 0x8E, 0xB4, + 0x6F, 0xB4, 0xA3, 0x54, 0x6E, 0x88, 0x3B, 0x62, + 0x7C, 0x02, 0x8D, 0x4C, 0x9F, 0xC8, 0x45, 0xF4, + 0xC9, 0xDE, 0x4F, 0xEB, 0x22, 0x83, 0x1B, 0xE4, + 0x49, 0x37, 0xE4, 0xAD, 0xE7, 0xCD, 0x21, 0x54, + 0xBC, 0x1C, 0xC2, 0x04, 0x97, 0xB4, 0x10, 0x61, + 0xF0, 0xE4, 0xEF, 0x27, 0x63, 0x3A, 0xDA, 0x91, + 0x41, 0x25, 0x62, 0x1C, 0x5C, 0xB6, 0x38, 0x4A, + 0x88, 0x71, 0x59, 0x5A, 0x8D, 0xA0, 0x09, 0xAF, + 0x72, 0x94, 0xD7, 0x79, 0x5C, 0x60, 0x7C, 0x8F, + 0x4C, 0xF5, 0xD9, 0xA1, 0x39, 0x6D, 0x81, 0x28, + 0xEF, 0x13, 0x28, 0xDF, 0xF5, 0x3E, 0xF7, 0x8E, + 0x09, 0x9C, 0x78, 0x18, 0x79, 0xB8, 0x68, 0xD7, + 0xA8, 0x29, 0x62, 0xAD, 0xDE, 0xE1, 0x61, 0x76, + 0x1B, 0x05, 0x16, 0xCD, 0xBF, 0x02, 0x8E, 0xA6, + 0x43, 0x6E, 0x92, 0x55, 0x4F, 0x60, 0x9C, 0x03, + 0xB8, 0x4F, 0xA3, 0x02, 0xAC, 0xA8, 0xA7, 0x0C, + 0x1E, 0xB5, 0x6B, 0xF8, 0xC8, 0x4D, 0xDE, 0xD2, + 0xB0, 0x29, 0x6E, 0x40, 0xE6, 0xD6, 0xC9, 0xE6, + 0xB9, 0x0F, 0xB6, 0x63, 0xF5, 0xAA, 0x2B, 0x96, + 0xA7, 0x16, 0xAC, 0x4E, 0x0A, 0x33, 0x1C, 0xA6, + 0xE6, 0xBD, 0x8A, 0xCF, 0x40, 0xA9, 0xB2, 0xFA, + 0x63, 0x27, 0xFD, 0x9B, 0xD9, 0xFC, 0xD5, 0x87, + 0x8D, 0x4C, 0xB6, 0xA4, 0xCB, 0xE7, 0x74, 0x55, + 0xF4, 0xFB, 0x41, 0x25, 0xB5, 0x4B, 0x0A, 0x1B, + 0xB1, 0xD6, 0xB7, 0xD9, 0x47, 0x2A, 0xC3, 0x98, + 0x6A, 0xC4, 0x03, 0x73, 0x1F, 0x93, 0x6E, 0x53, + 0x19, 0x25, 0x64, 0x15, 0x83, 0xF9, 0x73, 0x2A, + 0x74, 0xB4, 0x93, 0x69, 0xC4, 0x72, 0xFC, 0x26, + 0xA2, 0x9F, 0x43, 0x45, 0xDD, 0xB9, 0xEF, 0x36, + 0xC8, 0x3A, 0xCD, 0x99, 0x9B, 0x54, 0x1A, 0x36, + 0xC1, 0x59, 0xF8, 0x98, 0xA8, 0xCC, 0x28, 0x0D, + 0x73, 0x4C, 0xEE, 0x98, 0xCB, 0x7C, 0x58, 0x7E, + 0x20, 0x75, 0x1E, 0xB7, 0xC9, 0xF8, 0xF2, 0x0E, + 0x63, 0x9E, 0x05, 0x78, 0x1A, 0xB6, 0xA8, 0x7A, + 0xF9, 0x98, 0x6A, 0xA6, 0x46, 0x84, 0x2E, 0xF6, + 0x4B, 0xDC, 0x9B, 0x8F, 0x9B, 0x8F, 0xEE, 0xB4, + 0xAA, 0x3F, 0xEE, 0xC0, 0x37, 0x27, 0x76, 0xC7, + 0x95, 0xBB, 0x26, 0x74, 0x69, 0x12, 0x7F, 0xF1, + 0xBB, 0xFF, 0xAE, 0xB5, 0x99, 0x6E, 0xCB, 0x0C +}; + +static const uint8_t catch_22_quote_2_512_bytes_AES_CBC_HMAC_SHA1_digest[] = { + 0x9a, 0x4f, 0x88, 0x1b, 0xb6, 0x8f, 0xd8, 0x60, + 0x42, 0x1a, 0x7d, 0x3d, 0xf5, 0x82, 0x80, 0xf1, + 0x18, 0x8c, 0x1d, 0x32 +}; + + +/* Multisession Vector context Test */ +/*Begin Session 0 */ +static uint8_t ms_aes_cbc_key0[] = { + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff +}; + +static uint8_t ms_aes_cbc_iv0[] = { + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff +}; + +static const uint8_t ms_aes_cbc_cipher0[] = { + 0x3C, 0xE4, 0xEE, 0x42, 0xB6, 0x9B, 0xC3, 0x38, + 0x5F, 0xAD, 0x54, 0xDC, 0xA8, 0x32, 0x81, 0xDC, + 0x7A, 0x6F, 0x85, 0x58, 0x07, 0x35, 0xED, 0xEB, + 0xAD, 0x79, 0x79, 0x96, 0xD3, 0x0E, 0xA6, 0xD9, + 0xAA, 0x86, 0xA4, 0x8F, 0xB5, 0xD6, 0x6E, 0x6D, + 0x0C, 0x91, 0x2F, 0xC4, 0x67, 0x98, 0x0E, 0xC4, + 0x8D, 0x83, 0x68, 0x69, 0xC4, 0xD3, 0x94, 0x34, + 0xC4, 0x5D, 0x60, 0x55, 0x22, 0x87, 0x8F, 0x6F, + 0x17, 0x8E, 0x75, 0xE4, 0x02, 0xF5, 0x1B, 0x99, + 0xC8, 0x39, 0xA9, 0xAB, 0x23, 0x91, 0x12, 0xED, + 0x08, 0xE7, 0xD9, 0x25, 0x89, 0x24, 0x4F, 0x8D, + 0x68, 0xF3, 0x10, 0x39, 0x0A, 0xEE, 0x45, 0x24, + 0xDF, 0x7A, 0x9D, 0x00, 0x25, 0xE5, 0x35, 0x71, + 0x4E, 0x40, 0x59, 0x6F, 0x0A, 0x13, 0xB3, 0x72, + 0x1D, 0x98, 0x63, 0x94, 0x89, 0xA5, 0x39, 0x8E, + 0xD3, 0x9C, 0x8A, 0x7F, 0x71, 0x2F, 0xC7, 0xCD, + 0x81, 0x05, 0xDC, 0xC0, 0x8D, 0xCE, 0x6D, 0x18, + 0x30, 0xC4, 0x72, 0x51, 0xF0, 0x27, 0xC8, 0xF6, + 0x60, 0x5B, 0x7C, 0xB2, 0xE3, 0x49, 0x0C, 0x29, + 0xC6, 0x9F, 0x39, 0x57, 0x80, 0x55, 0x24, 0x2C, + 0x9B, 0x0F, 0x5A, 0xB3, 0x89, 0x55, 0x31, 0x96, + 0x0D, 0xCD, 0xF6, 0x51, 0x03, 0x2D, 0x89, 0x26, + 0x74, 0x44, 0xD6, 0xE8, 0xDC, 0xEA, 0x44, 0x55, + 0x64, 0x71, 0x9C, 0x9F, 0x5D, 0xBA, 0x39, 0x46, + 0xA8, 0x17, 0xA1, 0x9C, 0x52, 0x9D, 0xBC, 0x6B, + 0x4A, 0x98, 0xE6, 0xEA, 0x33, 0xEC, 0x58, 0xB4, + 0x43, 0xF0, 0x32, 0x45, 0xA4, 0xC1, 0x55, 0xB7, + 0x5D, 0xB5, 0x59, 0xB2, 0xE3, 0x96, 0xFF, 0xA5, + 0xAF, 0xE1, 0x86, 0x1B, 0x42, 0xE6, 0x3B, 0xA0, + 0x90, 0x4A, 0xE8, 0x8C, 0x21, 0x7F, 0x36, 0x1E, + 0x5B, 0x65, 0x25, 0xD1, 0xC1, 0x5A, 0xCA, 0x3D, + 0x10, 0xED, 0x2D, 0x79, 0xD0, 0x0F, 0x58, 0x44, + 0x69, 0x81, 0xF5, 0xD4, 0xC9, 0x0F, 0x90, 0x76, + 0x1F, 0x54, 0xD2, 0xD5, 0x97, 0xCE, 0x2C, 0xE3, + 0xEF, 0xF4, 0xB7, 0xC6, 0x3A, 0x87, 0x7F, 0x83, + 0x2A, 0xAF, 0xCD, 0x90, 0x12, 0xA7, 0x7D, 0x85, + 0x1D, 0x62, 0xD3, 0x85, 0x25, 0x05, 0xDB, 0x45, + 0x92, 0xA3, 0xF6, 0xA2, 0xA8, 0x41, 0xE4, 0x25, + 0x86, 0x87, 0x67, 0x24, 0xEC, 0x89, 0x23, 0x2A, + 0x9B, 0x20, 0x4D, 0x93, 0xEE, 0xE2, 0x2E, 0xC1, + 0x0B, 0x15, 0x33, 0xCF, 0x00, 0xD1, 0x1A, 0xDA, + 0x93, 0xFD, 0x28, 0x21, 0x5B, 0xCF, 0xD1, 0xF3, + 0x5A, 0x81, 0xBA, 0x82, 0x5E, 0x2F, 0x61, 0xB4, + 0x05, 0x71, 0xB5, 0xF4, 0x39, 0x3C, 0x1F, 0x60, + 0x00, 0x7A, 0xC4, 0xF8, 0x35, 0x20, 0x6C, 0x3A, + 0xCC, 0x03, 0x8F, 0x7B, 0xA2, 0xB6, 0x65, 0x8A, + 0xB6, 0x5F, 0xFD, 0x25, 0xD3, 0x5F, 0x92, 0xF9, + 0xAE, 0x17, 0x9B, 0x5E, 0x6E, 0x9A, 0xE4, 0x55, + 0x10, 0x25, 0x07, 0xA4, 0xAF, 0x21, 0x69, 0x13, + 0xD8, 0xFA, 0x31, 0xED, 0xF7, 0xA7, 0xA7, 0x3B, + 0xB8, 0x96, 0x8E, 0x10, 0x86, 0x74, 0xD8, 0xB1, + 0x34, 0x9E, 0x9B, 0x6A, 0x26, 0xA8, 0xD4, 0xD0, + 0xB5, 0xF6, 0xDE, 0xE7, 0xCA, 0x06, 0xDC, 0xA3, + 0x6F, 0xEE, 0x6B, 0x1E, 0xB5, 0x30, 0x99, 0x23, + 0xF9, 0x76, 0xF0, 0xA0, 0xCF, 0x3B, 0x94, 0x7B, + 0x19, 0x8D, 0xA5, 0x0C, 0x18, 0xA6, 0x1D, 0x07, + 0x89, 0xBE, 0x5B, 0x61, 0xE5, 0xF1, 0x42, 0xDB, + 0xD4, 0x2E, 0x02, 0x1F, 0xCE, 0xEF, 0x92, 0xB1, + 0x1B, 0x56, 0x50, 0xF2, 0x16, 0xE5, 0xE7, 0x4F, + 0xFD, 0xBB, 0x3E, 0xD2, 0xFC, 0x3C, 0xC6, 0x0F, + 0xF9, 0x12, 0x4E, 0xCB, 0x1E, 0x0C, 0x15, 0x84, + 0x2A, 0x14, 0x8A, 0x02, 0xE4, 0x7E, 0x95, 0x5B, + 0x86, 0xDB, 0x9B, 0x62, 0x5B, 0x19, 0xD2, 0x17, + 0xFA, 0x13, 0xBB, 0x6B, 0x3F, 0x45, 0x9F, 0xBF +}; + + +static uint8_t ms_hmac_key0[] = { + 0xFF, 0x1A, 0x7D, 0x3D, 0xF5, 0x82, 0x80, 0xF1, + 0xF1, 0x35, 0x5C, 0x3B, 0xDD, 0x9A, 0x65, 0xBA, + 0x58, 0x34, 0x85, 0x65, 0x1C, 0x42, 0x50, 0x76, + 0x9A, 0xAF, 0x88, 0x1B, 0xB6, 0x8F, 0xF8, 0x60, + 0xA2, 0x5A, 0x7F, 0x3F, 0xF4, 0x72, 0x70, 0xF1, + 0xF5, 0x35, 0x4C, 0x3B, 0xDD, 0x90, 0x65, 0xB0, + 0x47, 0x3A, 0x75, 0x61, 0x5C, 0xA2, 0x10, 0x76, + 0x9A, 0xAF, 0x77, 0x5B, 0xB6, 0x7F, 0xF7, 0x60 +}; + +static const uint8_t ms_hmac_digest0[] = { + 0x43, 0x52, 0xED, 0x34, 0xAB, 0x36, 0xB2, 0x51, + 0xFB, 0xA3, 0xA6, 0x7C, 0x38, 0xFC, 0x42, 0x8F, + 0x57, 0x64, 0xAB, 0x81, 0xA7, 0x89, 0xB7, 0x6C, + 0xA0, 0xDC, 0xB9, 0x4D, 0xC4, 0x30, 0xF9, 0xD4, + 0x10, 0x82, 0x55, 0xD0, 0xAB, 0x32, 0xFB, 0x56, + 0x0D, 0xE4, 0x68, 0x3D, 0x76, 0xD0, 0x7B, 0xE4, + 0xA6, 0x2C, 0x34, 0x9E, 0x8C, 0x41, 0xF8, 0x23, + 0x28, 0x1B, 0x3A, 0x90, 0x26, 0x34, 0x47, 0x90 + }; + +/* End Session 0 */ +/* Begin session 1 */ + +static uint8_t ms_aes_cbc_key1[] = { + 0xf1, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff +}; + +static uint8_t ms_aes_cbc_iv1[] = { + 0xf1, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff +}; + +static const uint8_t ms_aes_cbc_cipher1[] = { + 0x5A, 0x7A, 0x67, 0x5D, 0xB8, 0xE1, 0xDC, 0x71, + 0x39, 0xA8, 0x74, 0x93, 0x9C, 0x4C, 0xFE, 0x23, + 0x61, 0xCD, 0xA4, 0xB3, 0xD9, 0xCE, 0x99, 0x09, + 0x2A, 0x23, 0xF3, 0x29, 0xBF, 0x4C, 0xB4, 0x6A, + 0x1B, 0x6B, 0x73, 0x4D, 0x48, 0x0C, 0xCF, 0x6C, + 0x5E, 0x34, 0x9E, 0x7F, 0xBC, 0x8F, 0xCC, 0x8F, + 0x75, 0x1D, 0x3D, 0x77, 0x10, 0x76, 0xC8, 0xB9, + 0x99, 0x6F, 0xD6, 0x56, 0x75, 0xA9, 0xB2, 0x66, + 0xC2, 0x24, 0x2B, 0x9C, 0xFE, 0x40, 0x8E, 0x43, + 0x20, 0x97, 0x1B, 0xFA, 0xD0, 0xCF, 0x04, 0xAB, + 0xBB, 0xF6, 0x5D, 0xF5, 0xA0, 0x19, 0x7C, 0x23, + 0x5D, 0x80, 0x8C, 0x49, 0xF6, 0x76, 0x88, 0x29, + 0x27, 0x4C, 0x59, 0x2B, 0x43, 0xA6, 0xB2, 0x26, + 0x27, 0x78, 0xBE, 0x1B, 0xE1, 0x4F, 0x5A, 0x1F, + 0xFC, 0x68, 0x08, 0xE7, 0xC4, 0xD1, 0x34, 0x68, + 0xB7, 0x13, 0x14, 0x41, 0x62, 0x6B, 0x1F, 0x77, + 0x0C, 0x68, 0x1D, 0x0D, 0xED, 0x89, 0xAA, 0xD8, + 0x97, 0x02, 0xBA, 0x5E, 0xD4, 0x84, 0x25, 0x97, + 0x03, 0xA5, 0xA6, 0x13, 0x66, 0x02, 0xF4, 0xC3, + 0xF3, 0xD3, 0xCC, 0x95, 0xC3, 0x87, 0x46, 0x90, + 0x1F, 0x6E, 0x14, 0xA8, 0x00, 0xF2, 0x6F, 0xD5, + 0xA1, 0xAD, 0xD5, 0x40, 0xA2, 0x0F, 0x32, 0x7E, + 0x99, 0xA3, 0xF5, 0x53, 0xC3, 0x26, 0xA1, 0x45, + 0x01, 0x88, 0x57, 0x84, 0x3E, 0x7B, 0x4E, 0x0B, + 0x3C, 0xB5, 0x3E, 0x9E, 0xE9, 0x78, 0x77, 0xC5, + 0xC0, 0x89, 0xA8, 0xF8, 0xF1, 0xA5, 0x2D, 0x5D, + 0xF9, 0xC6, 0xFB, 0xCB, 0x05, 0x23, 0xBD, 0x6E, + 0x5E, 0x14, 0xC6, 0x57, 0x73, 0xCF, 0x98, 0xBD, + 0x10, 0x8B, 0x18, 0xA6, 0x01, 0x5B, 0x13, 0xAE, + 0x8E, 0xDE, 0x1F, 0xB5, 0xB7, 0x40, 0x6C, 0xC1, + 0x1E, 0xA1, 0x19, 0x20, 0x9E, 0x95, 0xE0, 0x2F, + 0x1C, 0xF5, 0xD9, 0xD0, 0x2B, 0x1E, 0x82, 0x25, + 0x62, 0xB4, 0xEB, 0xA1, 0x1F, 0xCE, 0x44, 0xA1, + 0xCB, 0x92, 0x01, 0x6B, 0xE4, 0x26, 0x23, 0xE3, + 0xC5, 0x67, 0x35, 0x55, 0xDA, 0xE5, 0x27, 0xEE, + 0x8D, 0x12, 0x84, 0xB7, 0xBA, 0xA7, 0x1C, 0xD6, + 0x32, 0x3F, 0x67, 0xED, 0xFB, 0x5B, 0x8B, 0x52, + 0x46, 0x8C, 0xF9, 0x69, 0xCD, 0xAE, 0x79, 0xAA, + 0x37, 0x78, 0x49, 0xEB, 0xC6, 0x8E, 0x76, 0x63, + 0x84, 0xFF, 0x9D, 0x22, 0x99, 0x51, 0xB7, 0x5E, + 0x83, 0x4C, 0x8B, 0xDF, 0x5A, 0x07, 0xCC, 0xBA, + 0x42, 0xA5, 0x98, 0xB6, 0x47, 0x0E, 0x66, 0xEB, + 0x23, 0x0E, 0xBA, 0x44, 0xA8, 0xAA, 0x20, 0x71, + 0x79, 0x9C, 0x77, 0x5F, 0xF5, 0xFE, 0xEC, 0xEF, + 0xC6, 0x64, 0x3D, 0x84, 0xD0, 0x2B, 0xA7, 0x0A, + 0xC3, 0x72, 0x5B, 0x9C, 0xFA, 0xA8, 0x87, 0x95, + 0x94, 0x11, 0x38, 0xA7, 0x1E, 0x58, 0xE3, 0x73, + 0xC6, 0xC9, 0xD1, 0x7B, 0x92, 0xDB, 0x0F, 0x49, + 0x74, 0xC2, 0xA2, 0x0E, 0x35, 0x57, 0xAC, 0xDB, + 0x9A, 0x1C, 0xCF, 0x5A, 0x32, 0x3E, 0x26, 0x9B, + 0xEC, 0xB3, 0xEF, 0x9C, 0xFE, 0xBE, 0x52, 0xAC, + 0xB1, 0x29, 0xDD, 0xFD, 0x07, 0xE2, 0xEE, 0xED, + 0xE4, 0x46, 0x37, 0xFE, 0xD1, 0xDC, 0xCD, 0x02, + 0xF9, 0x31, 0xB0, 0xFB, 0x36, 0xB7, 0x34, 0xA4, + 0x76, 0xE8, 0x57, 0xBF, 0x99, 0x92, 0xC7, 0xAF, + 0x98, 0x10, 0xE2, 0x70, 0xCA, 0xC9, 0x2B, 0x82, + 0x06, 0x96, 0x88, 0x0D, 0xB3, 0xAC, 0x9E, 0x6D, + 0x43, 0xBC, 0x5B, 0x31, 0xCF, 0x65, 0x8D, 0xA6, + 0xC7, 0xFE, 0x73, 0xE1, 0x54, 0xF7, 0x10, 0xF9, + 0x86, 0xF7, 0xDF, 0xA1, 0xA1, 0xD8, 0xAE, 0x35, + 0xB3, 0x90, 0xDC, 0x6F, 0x43, 0x7A, 0x8B, 0xE0, + 0xFE, 0x8F, 0x33, 0x4D, 0x29, 0x6C, 0x45, 0x53, + 0x73, 0xDD, 0x21, 0x0B, 0x85, 0x30, 0xB5, 0xA5, + 0xF3, 0x5D, 0xEC, 0x79, 0x61, 0x9D, 0x9E, 0xB3 + +}; + +static uint8_t ms_hmac_key1[] = { + 0xFE, 0x1A, 0x7D, 0x3D, 0xF5, 0x82, 0x80, 0xF1, + 0xF1, 0x35, 0x5C, 0x3B, 0xDD, 0x9A, 0x65, 0xBA, + 0x58, 0x34, 0x85, 0x65, 0x1C, 0x42, 0x50, 0x76, + 0x9A, 0xAF, 0x88, 0x1B, 0xB6, 0x8F, 0xF8, 0x60, + 0xA2, 0x5A, 0x7F, 0x3F, 0xF4, 0x72, 0x70, 0xF1, + 0xF5, 0x35, 0x4C, 0x3B, 0xDD, 0x90, 0x65, 0xB0, + 0x47, 0x3A, 0x75, 0x61, 0x5C, 0xA2, 0x10, 0x76, + 0x9A, 0xAF, 0x77, 0x5B, 0xB6, 0x7F, 0xF7, 0x60 +}; + +static const uint8_t ms_hmac_digest1[] = { + 0xCE, 0x6E, 0x5F, 0x77, 0x96, 0x9A, 0xB1, 0x69, + 0x2D, 0x5E, 0xF3, 0x2F, 0x32, 0x10, 0xCB, 0x50, + 0x0E, 0x09, 0x56, 0x25, 0x07, 0x34, 0xC9, 0x20, + 0xEC, 0x13, 0x43, 0x23, 0x5C, 0x08, 0x8B, 0xCD, + 0xDC, 0x86, 0x8C, 0xEE, 0x0A, 0x95, 0x2E, 0xB9, + 0x8C, 0x7B, 0x02, 0x7A, 0xD4, 0xE1, 0x49, 0xB4, + 0x45, 0xB5, 0x52, 0x37, 0xC6, 0xFF, 0xFE, 0xAA, + 0x0A, 0x87, 0xB8, 0x51, 0xF9, 0x2A, 0x01, 0x8F +}; +/* End Session 1 */ +/* Begin Session 2 */ +static uint8_t ms_aes_cbc_key2[] = { + 0xff, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff +}; + +static uint8_t ms_aes_cbc_iv2[] = { + 0xff, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff +}; + +static const uint8_t ms_aes_cbc_cipher2[] = { + 0xBB, 0x3C, 0x68, 0x25, 0xFD, 0xB6, 0xA2, 0x91, + 0x20, 0x56, 0xF6, 0x30, 0x35, 0xFC, 0x9E, 0x97, + 0xF2, 0x90, 0xFC, 0x7E, 0x3E, 0x0A, 0x75, 0xC8, + 0x4C, 0xF2, 0x2D, 0xAC, 0xD3, 0x93, 0xF0, 0xC5, + 0x14, 0x88, 0x8A, 0x23, 0xC2, 0x59, 0x9A, 0x98, + 0x4B, 0xD5, 0x2C, 0xDA, 0x43, 0xA9, 0x34, 0x69, + 0x7C, 0x6D, 0xDB, 0xDC, 0xCB, 0xC0, 0xA0, 0x09, + 0xA7, 0x86, 0x16, 0x4B, 0xBF, 0xA8, 0xB6, 0xCF, + 0x7F, 0x74, 0x1F, 0x22, 0xF0, 0xF6, 0xBB, 0x44, + 0x8B, 0x4C, 0x9E, 0x23, 0xF8, 0x9F, 0xFC, 0x5B, + 0x9E, 0x9C, 0x2A, 0x79, 0x30, 0x8F, 0xBF, 0xA9, + 0x68, 0xA1, 0x20, 0x71, 0x7C, 0x77, 0x22, 0x34, + 0x07, 0xCD, 0xC6, 0xF6, 0x50, 0x0A, 0x08, 0x99, + 0x17, 0x98, 0xE3, 0x93, 0x8A, 0xB0, 0xEE, 0xDF, + 0xC2, 0xBA, 0x3B, 0x44, 0x73, 0xDF, 0xDD, 0xDC, + 0x14, 0x4D, 0x3B, 0xBB, 0x5E, 0x58, 0xC1, 0x26, + 0xA7, 0xAE, 0x47, 0xF3, 0x24, 0x6D, 0x4F, 0xD3, + 0x6E, 0x3E, 0x33, 0xE6, 0x7F, 0xCA, 0x50, 0xAF, + 0x5D, 0x3D, 0xA0, 0xDD, 0xC9, 0xF3, 0x30, 0xD3, + 0x6E, 0x8B, 0x2E, 0x12, 0x24, 0x34, 0xF0, 0xD3, + 0xC7, 0x8D, 0x23, 0x29, 0xAA, 0x05, 0xE1, 0xFA, + 0x2E, 0xF6, 0x8D, 0x37, 0x86, 0xC0, 0x6D, 0x13, + 0x2D, 0x98, 0xF3, 0x52, 0x39, 0x22, 0xCE, 0x38, + 0xC2, 0x1A, 0x72, 0xED, 0xFB, 0xCC, 0xE4, 0x71, + 0x5A, 0x0C, 0x0D, 0x09, 0xF8, 0xE8, 0x1B, 0xBC, + 0x53, 0xC8, 0xD8, 0x8F, 0xE5, 0x98, 0x5A, 0xB1, + 0x06, 0xA6, 0x5B, 0xE6, 0xA2, 0x88, 0x21, 0x9E, + 0x36, 0xC0, 0x34, 0xF9, 0xFB, 0x3B, 0x0A, 0x22, + 0x00, 0x00, 0x39, 0x48, 0x8D, 0x23, 0x74, 0x62, + 0x72, 0x91, 0xE6, 0x36, 0xAA, 0x77, 0x9C, 0x72, + 0x9D, 0xA8, 0xC3, 0xA9, 0xD5, 0x44, 0x72, 0xA6, + 0xB9, 0x28, 0x8F, 0x64, 0x4C, 0x8A, 0x64, 0xE6, + 0x4E, 0xFA, 0xEF, 0x87, 0xDE, 0x7B, 0x22, 0x44, + 0xB0, 0xDF, 0x2E, 0x5F, 0x0B, 0xA5, 0xF2, 0x24, + 0x07, 0x5C, 0x2D, 0x39, 0xB7, 0x3D, 0x8A, 0xE5, + 0x0E, 0x9D, 0x4E, 0x50, 0xED, 0x03, 0x99, 0x8E, + 0xF0, 0x06, 0x55, 0x4E, 0xA2, 0x24, 0xE7, 0x17, + 0x46, 0xDF, 0x6C, 0xCD, 0xC6, 0x44, 0xE8, 0xF9, + 0xB9, 0x1B, 0x36, 0xF6, 0x7F, 0x10, 0xA4, 0x7D, + 0x90, 0xBD, 0xE4, 0xAA, 0xD6, 0x9E, 0x18, 0x9D, + 0x22, 0x35, 0xD6, 0x55, 0x54, 0xAA, 0xF7, 0x22, + 0xA3, 0x3E, 0xEF, 0xC8, 0xA2, 0x34, 0x8D, 0xA9, + 0x37, 0x63, 0xA6, 0xC3, 0x57, 0xCB, 0x0C, 0x49, + 0x7D, 0x02, 0xBE, 0xAA, 0x13, 0x75, 0xB7, 0x4E, + 0x52, 0x62, 0xA5, 0xC2, 0x33, 0xC7, 0x6C, 0x1B, + 0xF6, 0x34, 0xF6, 0x09, 0xA5, 0x0C, 0xC7, 0xA2, + 0x61, 0x48, 0x62, 0x7D, 0x17, 0x15, 0xE3, 0x95, + 0xC8, 0x63, 0xD2, 0xA4, 0x43, 0xA9, 0x49, 0x07, + 0xB2, 0x3B, 0x2B, 0x62, 0x7D, 0xCB, 0x51, 0xB3, + 0x25, 0x33, 0x47, 0x0E, 0x14, 0x67, 0xDC, 0x6A, + 0x9B, 0x51, 0xAC, 0x9D, 0x8F, 0xA2, 0x2B, 0x57, + 0x8C, 0x5C, 0x5F, 0x76, 0x23, 0x92, 0x0F, 0x84, + 0x46, 0x0E, 0x40, 0x85, 0x38, 0x60, 0xFA, 0x61, + 0x20, 0xC5, 0xE3, 0xF1, 0x70, 0xAC, 0x1B, 0xBF, + 0xC4, 0x2B, 0xC5, 0x67, 0xD1, 0x43, 0xC5, 0x17, + 0x74, 0x71, 0x69, 0x6F, 0x82, 0x89, 0x19, 0x8A, + 0x70, 0x43, 0x92, 0x01, 0xC4, 0x63, 0x7E, 0xB1, + 0x59, 0x4E, 0xCD, 0xEA, 0x93, 0xA4, 0x52, 0x53, + 0x9B, 0x61, 0x5B, 0xD2, 0x3E, 0x19, 0x39, 0xB7, + 0x32, 0xEA, 0x8E, 0xF8, 0x1D, 0x76, 0x5C, 0xB2, + 0x73, 0x2D, 0x91, 0xC0, 0x18, 0xED, 0x25, 0x2A, + 0x53, 0x64, 0xF0, 0x92, 0x31, 0x55, 0x21, 0xA8, + 0x24, 0xA9, 0xD1, 0x02, 0xF6, 0x6C, 0x2B, 0x70, + 0xA9, 0x59, 0xC1, 0xD6, 0xC3, 0x57, 0x5B, 0x92 +}; + +static uint8_t ms_hmac_key2[] = { + 0xFC, 0x1A, 0x7D, 0x3D, 0xF5, 0x82, 0x80, 0xF1, + 0xF1, 0x35, 0x5C, 0x3B, 0xDD, 0x9A, 0x65, 0xBA, + 0x58, 0x34, 0x85, 0x65, 0x1C, 0x42, 0x50, 0x76, + 0x9A, 0xAF, 0x88, 0x1B, 0xB6, 0x8F, 0xF8, 0x60, + 0xA2, 0x5A, 0x7F, 0x3F, 0xF4, 0x72, 0x70, 0xF1, + 0xF5, 0x35, 0x4C, 0x3B, 0xDD, 0x90, 0x65, 0xB0, + 0x47, 0x3A, 0x75, 0x61, 0x5C, 0xA2, 0x10, 0x76, + 0x9A, 0xAF, 0x77, 0x5B, 0xB6, 0x7F, 0xF7, 0x60 +}; + +static const uint8_t ms_hmac_digest2[] = { + 0xA5, 0x0F, 0x9C, 0xFB, 0x08, 0x62, 0x59, 0xFF, + 0x80, 0x2F, 0xEB, 0x4B, 0xE1, 0x46, 0x21, 0xD6, + 0x02, 0x98, 0xF2, 0x8E, 0xF4, 0xEC, 0xD4, 0x77, + 0x86, 0x4C, 0x31, 0x28, 0xC8, 0x25, 0x80, 0x27, + 0x3A, 0x72, 0x5D, 0x6A, 0x56, 0x8A, 0xD3, 0x82, + 0xB0, 0xEC, 0x31, 0x6D, 0x8B, 0x6B, 0xB4, 0x24, + 0xE7, 0x62, 0xC1, 0x52, 0xBC, 0x14, 0x1B, 0x8E, + 0xEC, 0x9A, 0xF1, 0x47, 0x80, 0xD2, 0xB0, 0x59 +}; + +/* End Session 2 */ + + +static int +test_AES_CBC_HMAC_SHA1_encrypt_digest(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + + /* Generate test mbuf data and space for digest */ + ut_params->ibuf = setup_test_string(ts_params->mbuf_pool, + catch_22_quote, QUOTE_512_BYTES, 0); + + ut_params->digest = (uint8_t *)rte_pktmbuf_append(ut_params->ibuf, + DIGEST_BYTE_LENGTH_SHA1); + TEST_ASSERT_NOT_NULL(ut_params->digest, "no room to append digest"); + + /* Setup Cipher Parameters */ + ut_params->cipher_xform.type = RTE_CRYPTO_SYM_XFORM_CIPHER; + ut_params->cipher_xform.next = &ut_params->auth_xform; + + ut_params->cipher_xform.cipher.algo = RTE_CRYPTO_CIPHER_AES_CBC; + ut_params->cipher_xform.cipher.op = RTE_CRYPTO_CIPHER_OP_ENCRYPT; + ut_params->cipher_xform.cipher.key.data = aes_cbc_key; + ut_params->cipher_xform.cipher.key.length = CIPHER_KEY_LENGTH_AES_CBC; + ut_params->cipher_xform.cipher.iv.offset = IV_OFFSET; + ut_params->cipher_xform.cipher.iv.length = CIPHER_IV_LENGTH_AES_CBC; + + /* Setup HMAC Parameters */ + ut_params->auth_xform.type = RTE_CRYPTO_SYM_XFORM_AUTH; + + ut_params->auth_xform.next = NULL; + + ut_params->auth_xform.auth.op = RTE_CRYPTO_AUTH_OP_GENERATE; + ut_params->auth_xform.auth.algo = RTE_CRYPTO_AUTH_SHA1_HMAC; + ut_params->auth_xform.auth.key.length = HMAC_KEY_LENGTH_SHA1; + ut_params->auth_xform.auth.key.data = hmac_sha1_key; + ut_params->auth_xform.auth.digest_length = DIGEST_BYTE_LENGTH_SHA1; + + ut_params->sess = rte_cryptodev_sym_session_create( + ts_params->session_mpool); + + /* Create crypto session*/ + rte_cryptodev_sym_session_init(ts_params->valid_devs[0], + ut_params->sess, &ut_params->cipher_xform, + ts_params->session_mpool); + TEST_ASSERT_NOT_NULL(ut_params->sess, "Session creation failed"); + + /* Generate crypto op data structure */ + ut_params->op = rte_crypto_op_alloc(ts_params->op_mpool, + RTE_CRYPTO_OP_TYPE_SYMMETRIC); + TEST_ASSERT_NOT_NULL(ut_params->op, + "Failed to allocate symmetric crypto operation struct"); + + rte_crypto_op_attach_sym_session(ut_params->op, ut_params->sess); + + struct rte_crypto_sym_op *sym_op = ut_params->op->sym; + + /* set crypto operation source mbuf */ + sym_op->m_src = ut_params->ibuf; + + /* Set crypto operation authentication parameters */ + sym_op->auth.digest.data = ut_params->digest; + sym_op->auth.digest.phys_addr = rte_pktmbuf_iova_offset( + ut_params->ibuf, QUOTE_512_BYTES); + + sym_op->auth.data.offset = 0; + sym_op->auth.data.length = QUOTE_512_BYTES; + + /* Copy IV at the end of the crypto operation */ + rte_memcpy(rte_crypto_op_ctod_offset(ut_params->op, uint8_t *, IV_OFFSET), + aes_cbc_iv, CIPHER_IV_LENGTH_AES_CBC); + + /* Set crypto operation cipher parameters */ + sym_op->cipher.data.offset = 0; + sym_op->cipher.data.length = QUOTE_512_BYTES; + + /* Process crypto operation */ + TEST_ASSERT_NOT_NULL(process_crypto_request(ts_params->valid_devs[0], + ut_params->op), "failed to process sym crypto op"); + + TEST_ASSERT_EQUAL(ut_params->op->status, RTE_CRYPTO_OP_STATUS_SUCCESS, + "crypto op processing failed"); + + /* Validate obuf */ + uint8_t *ciphertext = rte_pktmbuf_mtod(ut_params->op->sym->m_src, + uint8_t *); + + TEST_ASSERT_BUFFERS_ARE_EQUAL(ciphertext, + catch_22_quote_2_512_bytes_AES_CBC_ciphertext, + QUOTE_512_BYTES, + "ciphertext data not as expected"); + + uint8_t *digest = ciphertext + QUOTE_512_BYTES; + + TEST_ASSERT_BUFFERS_ARE_EQUAL(digest, + catch_22_quote_2_512_bytes_AES_CBC_HMAC_SHA1_digest, + gbl_driver_id == rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_AESNI_MB_PMD)) ? + TRUNCATED_DIGEST_BYTE_LENGTH_SHA1 : + DIGEST_BYTE_LENGTH_SHA1, + "Generated digest data not as expected"); + + return TEST_SUCCESS; +} + +/* ***** AES-CBC / HMAC-SHA512 Hash Tests ***** */ + +#define HMAC_KEY_LENGTH_SHA512 (DIGEST_BYTE_LENGTH_SHA512) + +static uint8_t hmac_sha512_key[] = { + 0x42, 0x1a, 0x7d, 0x3d, 0xf5, 0x82, 0x80, 0xf1, + 0xF1, 0x35, 0x5C, 0x3B, 0xDD, 0x9A, 0x65, 0xBA, + 0x58, 0x34, 0x85, 0x65, 0x1C, 0x42, 0x50, 0x76, + 0x9a, 0xaf, 0x88, 0x1b, 0xb6, 0x8f, 0xf8, 0x60, + 0xa2, 0x5a, 0x7f, 0x3f, 0xf4, 0x72, 0x70, 0xf1, + 0xF5, 0x35, 0x4C, 0x3B, 0xDD, 0x90, 0x65, 0xB0, + 0x47, 0x3a, 0x75, 0x61, 0x5C, 0xa2, 0x10, 0x76, + 0x9a, 0xaf, 0x77, 0x5b, 0xb6, 0x7f, 0xf7, 0x60 }; + +static const uint8_t catch_22_quote_2_512_bytes_AES_CBC_HMAC_SHA512_digest[] = { + 0x5D, 0x54, 0x66, 0xC1, 0x6E, 0xBC, 0x04, 0xB8, + 0x46, 0xB8, 0x08, 0x6E, 0xE0, 0xF0, 0x43, 0x48, + 0x37, 0x96, 0x9C, 0xC6, 0x9C, 0xC2, 0x1E, 0xE8, + 0xF2, 0x0C, 0x0B, 0xEF, 0x86, 0xA2, 0xE3, 0x70, + 0x95, 0xC8, 0xB3, 0x06, 0x47, 0xA9, 0x90, 0xE8, + 0xA0, 0xC6, 0x72, 0x69, 0x05, 0xC0, 0x0D, 0x0E, + 0x21, 0x96, 0x65, 0x93, 0x74, 0x43, 0x2A, 0x1D, + 0x2E, 0xBF, 0xC2, 0xC2, 0xEE, 0xCC, 0x2F, 0x0A }; + + + +static int +test_AES_CBC_HMAC_SHA512_decrypt_create_session_params( + struct crypto_unittest_params *ut_params, + uint8_t *cipher_key, + uint8_t *hmac_key); + +static int +test_AES_CBC_HMAC_SHA512_decrypt_perform(struct rte_cryptodev_sym_session *sess, + struct crypto_unittest_params *ut_params, + struct crypto_testsuite_params *ts_params, + const uint8_t *cipher, + const uint8_t *digest, + const uint8_t *iv); + + +static int +test_AES_CBC_HMAC_SHA512_decrypt_create_session_params( + struct crypto_unittest_params *ut_params, + uint8_t *cipher_key, + uint8_t *hmac_key) +{ + + /* Setup Cipher Parameters */ + ut_params->cipher_xform.type = RTE_CRYPTO_SYM_XFORM_CIPHER; + ut_params->cipher_xform.next = NULL; + + ut_params->cipher_xform.cipher.algo = RTE_CRYPTO_CIPHER_AES_CBC; + ut_params->cipher_xform.cipher.op = RTE_CRYPTO_CIPHER_OP_DECRYPT; + ut_params->cipher_xform.cipher.key.data = cipher_key; + ut_params->cipher_xform.cipher.key.length = CIPHER_KEY_LENGTH_AES_CBC; + ut_params->cipher_xform.cipher.iv.offset = IV_OFFSET; + ut_params->cipher_xform.cipher.iv.length = CIPHER_IV_LENGTH_AES_CBC; + + /* Setup HMAC Parameters */ + ut_params->auth_xform.type = RTE_CRYPTO_SYM_XFORM_AUTH; + ut_params->auth_xform.next = &ut_params->cipher_xform; + + ut_params->auth_xform.auth.op = RTE_CRYPTO_AUTH_OP_VERIFY; + ut_params->auth_xform.auth.algo = RTE_CRYPTO_AUTH_SHA512_HMAC; + ut_params->auth_xform.auth.key.data = hmac_key; + ut_params->auth_xform.auth.key.length = HMAC_KEY_LENGTH_SHA512; + ut_params->auth_xform.auth.digest_length = DIGEST_BYTE_LENGTH_SHA512; + + return TEST_SUCCESS; +} + + +static int +test_AES_CBC_HMAC_SHA512_decrypt_perform(struct rte_cryptodev_sym_session *sess, + struct crypto_unittest_params *ut_params, + struct crypto_testsuite_params *ts_params, + const uint8_t *cipher, + const uint8_t *digest, + const uint8_t *iv) +{ + /* Generate test mbuf data and digest */ + ut_params->ibuf = setup_test_string(ts_params->mbuf_pool, + (const char *) + cipher, + QUOTE_512_BYTES, 0); + + ut_params->digest = (uint8_t *)rte_pktmbuf_append(ut_params->ibuf, + DIGEST_BYTE_LENGTH_SHA512); + TEST_ASSERT_NOT_NULL(ut_params->digest, "no room to append digest"); + + rte_memcpy(ut_params->digest, + digest, + DIGEST_BYTE_LENGTH_SHA512); + + /* Generate Crypto op data structure */ + ut_params->op = rte_crypto_op_alloc(ts_params->op_mpool, + RTE_CRYPTO_OP_TYPE_SYMMETRIC); + TEST_ASSERT_NOT_NULL(ut_params->op, + "Failed to allocate symmetric crypto operation struct"); + + rte_crypto_op_attach_sym_session(ut_params->op, sess); + + struct rte_crypto_sym_op *sym_op = ut_params->op->sym; + + /* set crypto operation source mbuf */ + sym_op->m_src = ut_params->ibuf; + + sym_op->auth.digest.data = ut_params->digest; + sym_op->auth.digest.phys_addr = rte_pktmbuf_iova_offset( + ut_params->ibuf, QUOTE_512_BYTES); + + sym_op->auth.data.offset = 0; + sym_op->auth.data.length = QUOTE_512_BYTES; + + /* Copy IV at the end of the crypto operation */ + rte_memcpy(rte_crypto_op_ctod_offset(ut_params->op, uint8_t *, IV_OFFSET), + iv, CIPHER_IV_LENGTH_AES_CBC); + + sym_op->cipher.data.offset = 0; + sym_op->cipher.data.length = QUOTE_512_BYTES; + + /* Process crypto operation */ + TEST_ASSERT_NOT_NULL(process_crypto_request(ts_params->valid_devs[0], + ut_params->op), "failed to process sym crypto op"); + + TEST_ASSERT_EQUAL(ut_params->op->status, RTE_CRYPTO_OP_STATUS_SUCCESS, + "crypto op processing failed"); + + ut_params->obuf = ut_params->op->sym->m_src; + + /* Validate obuf */ + TEST_ASSERT_BUFFERS_ARE_EQUAL( + rte_pktmbuf_mtod(ut_params->obuf, uint8_t *), + catch_22_quote, + QUOTE_512_BYTES, + "Plaintext data not as expected"); + + /* Validate obuf */ + TEST_ASSERT_EQUAL(ut_params->op->status, RTE_CRYPTO_OP_STATUS_SUCCESS, + "Digest verification failed"); + + return TEST_SUCCESS; +} + +static int +test_AES_cipheronly_mb_all(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + int status; + + status = test_blockcipher_all_tests(ts_params->mbuf_pool, + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], + rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_AESNI_MB_PMD)), + BLKCIPHER_AES_CIPHERONLY_TYPE); + + TEST_ASSERT_EQUAL(status, 0, "Test failed"); + + return TEST_SUCCESS; +} + +static int +test_AES_docsis_mb_all(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + int status; + + status = test_blockcipher_all_tests(ts_params->mbuf_pool, + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], + rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_AESNI_MB_PMD)), + BLKCIPHER_AES_DOCSIS_TYPE); + + TEST_ASSERT_EQUAL(status, 0, "Test failed"); + + return TEST_SUCCESS; +} + +static int +test_AES_docsis_qat_all(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + int status; + + status = test_blockcipher_all_tests(ts_params->mbuf_pool, + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], + rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_QAT_SYM_PMD)), + BLKCIPHER_AES_DOCSIS_TYPE); + + TEST_ASSERT_EQUAL(status, 0, "Test failed"); + + return TEST_SUCCESS; +} + +static int +test_DES_docsis_qat_all(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + int status; + + status = test_blockcipher_all_tests(ts_params->mbuf_pool, + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], + rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_QAT_SYM_PMD)), + BLKCIPHER_DES_DOCSIS_TYPE); + + TEST_ASSERT_EQUAL(status, 0, "Test failed"); + + return TEST_SUCCESS; +} + +static int +test_authonly_mb_all(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + int status; + + status = test_blockcipher_all_tests(ts_params->mbuf_pool, + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], + rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_AESNI_MB_PMD)), + BLKCIPHER_AUTHONLY_TYPE); + + TEST_ASSERT_EQUAL(status, 0, "Test failed"); + + return TEST_SUCCESS; +} + +static int +test_authonly_qat_all(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + int status; + + status = test_blockcipher_all_tests(ts_params->mbuf_pool, + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], + rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_QAT_SYM_PMD)), + BLKCIPHER_AUTHONLY_TYPE); + + TEST_ASSERT_EQUAL(status, 0, "Test failed"); + + return TEST_SUCCESS; +} +static int +test_AES_chain_mb_all(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + int status; + + status = test_blockcipher_all_tests(ts_params->mbuf_pool, + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], + rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_AESNI_MB_PMD)), + BLKCIPHER_AES_CHAIN_TYPE); + + TEST_ASSERT_EQUAL(status, 0, "Test failed"); + + return TEST_SUCCESS; +} + +#ifdef RTE_LIBRTE_PMD_CRYPTO_SCHEDULER + +static int +test_AES_cipheronly_scheduler_all(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + int status; + + status = test_blockcipher_all_tests(ts_params->mbuf_pool, + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], + rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_SCHEDULER_PMD)), + BLKCIPHER_AES_CIPHERONLY_TYPE); + + TEST_ASSERT_EQUAL(status, 0, "Test failed"); + + return TEST_SUCCESS; +} + +static int +test_AES_chain_scheduler_all(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + int status; + + status = test_blockcipher_all_tests(ts_params->mbuf_pool, + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], + rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_SCHEDULER_PMD)), + BLKCIPHER_AES_CHAIN_TYPE); + + TEST_ASSERT_EQUAL(status, 0, "Test failed"); + + return TEST_SUCCESS; +} + +static int +test_authonly_scheduler_all(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + int status; + + status = test_blockcipher_all_tests(ts_params->mbuf_pool, + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], + rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_SCHEDULER_PMD)), + BLKCIPHER_AUTHONLY_TYPE); + + TEST_ASSERT_EQUAL(status, 0, "Test failed"); + + return TEST_SUCCESS; +} + +#endif /* RTE_LIBRTE_PMD_CRYPTO_SCHEDULER */ + +static int +test_AES_chain_openssl_all(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + int status; + + status = test_blockcipher_all_tests(ts_params->mbuf_pool, + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], + rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_OPENSSL_PMD)), + BLKCIPHER_AES_CHAIN_TYPE); + + TEST_ASSERT_EQUAL(status, 0, "Test failed"); + + return TEST_SUCCESS; +} + +static int +test_AES_cipheronly_openssl_all(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + int status; + + status = test_blockcipher_all_tests(ts_params->mbuf_pool, + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], + rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_OPENSSL_PMD)), + BLKCIPHER_AES_CIPHERONLY_TYPE); + + TEST_ASSERT_EQUAL(status, 0, "Test failed"); + + return TEST_SUCCESS; +} + +static int +test_AES_chain_qat_all(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + int status; + + status = test_blockcipher_all_tests(ts_params->mbuf_pool, + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], + rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_QAT_SYM_PMD)), + BLKCIPHER_AES_CHAIN_TYPE); + + TEST_ASSERT_EQUAL(status, 0, "Test failed"); + + return TEST_SUCCESS; +} + +static int +test_AES_cipheronly_qat_all(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + int status; + + status = test_blockcipher_all_tests(ts_params->mbuf_pool, + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], + rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_QAT_SYM_PMD)), + BLKCIPHER_AES_CIPHERONLY_TYPE); + + TEST_ASSERT_EQUAL(status, 0, "Test failed"); + + return TEST_SUCCESS; +} + +static int +test_AES_chain_dpaa_sec_all(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + int status; + + status = test_blockcipher_all_tests(ts_params->mbuf_pool, + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], + rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_DPAA_SEC_PMD)), + BLKCIPHER_AES_CHAIN_TYPE); + + TEST_ASSERT_EQUAL(status, 0, "Test failed"); + + return TEST_SUCCESS; +} + +static int +test_AES_cipheronly_dpaa_sec_all(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + int status; + + status = test_blockcipher_all_tests(ts_params->mbuf_pool, + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], + rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_DPAA_SEC_PMD)), + BLKCIPHER_AES_CIPHERONLY_TYPE); + + TEST_ASSERT_EQUAL(status, 0, "Test failed"); + + return TEST_SUCCESS; +} + +static int +test_authonly_dpaa_sec_all(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + int status; + + status = test_blockcipher_all_tests(ts_params->mbuf_pool, + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], + rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_DPAA_SEC_PMD)), + BLKCIPHER_AUTHONLY_TYPE); + + TEST_ASSERT_EQUAL(status, 0, "Test failed"); + + return TEST_SUCCESS; +} + +static int +test_AES_chain_dpaa2_sec_all(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + int status; + + status = test_blockcipher_all_tests(ts_params->mbuf_pool, + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], + rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_DPAA2_SEC_PMD)), + BLKCIPHER_AES_CHAIN_TYPE); + + TEST_ASSERT_EQUAL(status, 0, "Test failed"); + + return TEST_SUCCESS; +} + +static int +test_AES_cipheronly_dpaa2_sec_all(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + int status; + + status = test_blockcipher_all_tests(ts_params->mbuf_pool, + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], + rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_DPAA2_SEC_PMD)), + BLKCIPHER_AES_CIPHERONLY_TYPE); + + TEST_ASSERT_EQUAL(status, 0, "Test failed"); + + return TEST_SUCCESS; +} + +static int +test_authonly_dpaa2_sec_all(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + int status; + + status = test_blockcipher_all_tests(ts_params->mbuf_pool, + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], + rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_DPAA2_SEC_PMD)), + BLKCIPHER_AUTHONLY_TYPE); + + TEST_ASSERT_EQUAL(status, 0, "Test failed"); + + return TEST_SUCCESS; +} + +static int +test_authonly_openssl_all(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + int status; + + status = test_blockcipher_all_tests(ts_params->mbuf_pool, + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], + rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_OPENSSL_PMD)), + BLKCIPHER_AUTHONLY_TYPE); + + TEST_ASSERT_EQUAL(status, 0, "Test failed"); + + return TEST_SUCCESS; +} + +static int +test_AES_chain_armv8_all(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + int status; + + status = test_blockcipher_all_tests(ts_params->mbuf_pool, + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], + rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_ARMV8_PMD)), + BLKCIPHER_AES_CHAIN_TYPE); + + TEST_ASSERT_EQUAL(status, 0, "Test failed"); + + return TEST_SUCCESS; +} + +static int +test_AES_chain_mrvl_all(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + int status; + + status = test_blockcipher_all_tests(ts_params->mbuf_pool, + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], + rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_MRVL_PMD)), + BLKCIPHER_AES_CHAIN_TYPE); + + TEST_ASSERT_EQUAL(status, 0, "Test failed"); + + return TEST_SUCCESS; +} + +static int +test_AES_cipheronly_mrvl_all(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + int status; + + status = test_blockcipher_all_tests(ts_params->mbuf_pool, + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], + rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_MRVL_PMD)), + BLKCIPHER_AES_CIPHERONLY_TYPE); + + TEST_ASSERT_EQUAL(status, 0, "Test failed"); + + return TEST_SUCCESS; +} + +static int +test_authonly_mrvl_all(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + int status; + + status = test_blockcipher_all_tests(ts_params->mbuf_pool, + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], + rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_MRVL_PMD)), + BLKCIPHER_AUTHONLY_TYPE); + + TEST_ASSERT_EQUAL(status, 0, "Test failed"); + + return TEST_SUCCESS; +} + +static int +test_3DES_chain_mrvl_all(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + int status; + + status = test_blockcipher_all_tests(ts_params->mbuf_pool, + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], + rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_MRVL_PMD)), + BLKCIPHER_3DES_CHAIN_TYPE); + + TEST_ASSERT_EQUAL(status, 0, "Test failed"); + + return TEST_SUCCESS; +} + +static int +test_3DES_cipheronly_mrvl_all(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + int status; + + status = test_blockcipher_all_tests(ts_params->mbuf_pool, + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], + rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_MRVL_PMD)), + BLKCIPHER_3DES_CIPHERONLY_TYPE); + + TEST_ASSERT_EQUAL(status, 0, "Test failed"); + + return TEST_SUCCESS; +} + +/* ***** SNOW 3G Tests ***** */ +static int +create_wireless_algo_hash_session(uint8_t dev_id, + const uint8_t *key, const uint8_t key_len, + const uint8_t iv_len, const uint8_t auth_len, + enum rte_crypto_auth_operation op, + enum rte_crypto_auth_algorithm algo) +{ + uint8_t hash_key[key_len]; + + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + + memcpy(hash_key, key, key_len); + + TEST_HEXDUMP(stdout, "key:", key, key_len); + + /* Setup Authentication Parameters */ + ut_params->auth_xform.type = RTE_CRYPTO_SYM_XFORM_AUTH; + ut_params->auth_xform.next = NULL; + + ut_params->auth_xform.auth.op = op; + ut_params->auth_xform.auth.algo = algo; + ut_params->auth_xform.auth.key.length = key_len; + ut_params->auth_xform.auth.key.data = hash_key; + ut_params->auth_xform.auth.digest_length = auth_len; + ut_params->auth_xform.auth.iv.offset = IV_OFFSET; + ut_params->auth_xform.auth.iv.length = iv_len; + ut_params->sess = rte_cryptodev_sym_session_create( + ts_params->session_mpool); + + rte_cryptodev_sym_session_init(dev_id, ut_params->sess, + &ut_params->auth_xform, ts_params->session_mpool); + TEST_ASSERT_NOT_NULL(ut_params->sess, "Session creation failed"); + return 0; +} + +static int +create_wireless_algo_cipher_session(uint8_t dev_id, + enum rte_crypto_cipher_operation op, + enum rte_crypto_cipher_algorithm algo, + const uint8_t *key, const uint8_t key_len, + uint8_t iv_len) +{ + uint8_t cipher_key[key_len]; + + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + + memcpy(cipher_key, key, key_len); + + /* Setup Cipher Parameters */ + ut_params->cipher_xform.type = RTE_CRYPTO_SYM_XFORM_CIPHER; + ut_params->cipher_xform.next = NULL; + + ut_params->cipher_xform.cipher.algo = algo; + ut_params->cipher_xform.cipher.op = op; + ut_params->cipher_xform.cipher.key.data = cipher_key; + ut_params->cipher_xform.cipher.key.length = key_len; + ut_params->cipher_xform.cipher.iv.offset = IV_OFFSET; + ut_params->cipher_xform.cipher.iv.length = iv_len; + + TEST_HEXDUMP(stdout, "key:", key, key_len); + + /* Create Crypto session */ + ut_params->sess = rte_cryptodev_sym_session_create( + ts_params->session_mpool); + + rte_cryptodev_sym_session_init(dev_id, ut_params->sess, + &ut_params->cipher_xform, ts_params->session_mpool); + TEST_ASSERT_NOT_NULL(ut_params->sess, "Session creation failed"); + return 0; +} + +static int +create_wireless_algo_cipher_operation(const uint8_t *iv, uint8_t iv_len, + unsigned int cipher_len, + unsigned int cipher_offset) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + + /* Generate Crypto op data structure */ + ut_params->op = rte_crypto_op_alloc(ts_params->op_mpool, + RTE_CRYPTO_OP_TYPE_SYMMETRIC); + TEST_ASSERT_NOT_NULL(ut_params->op, + "Failed to allocate pktmbuf offload"); + + /* Set crypto operation data parameters */ + rte_crypto_op_attach_sym_session(ut_params->op, ut_params->sess); + + struct rte_crypto_sym_op *sym_op = ut_params->op->sym; + + /* set crypto operation source mbuf */ + sym_op->m_src = ut_params->ibuf; + + /* iv */ + rte_memcpy(rte_crypto_op_ctod_offset(ut_params->op, uint8_t *, IV_OFFSET), + iv, iv_len); + sym_op->cipher.data.length = cipher_len; + sym_op->cipher.data.offset = cipher_offset; + return 0; +} + +static int +create_wireless_algo_cipher_operation_oop(const uint8_t *iv, uint8_t iv_len, + unsigned int cipher_len, + unsigned int cipher_offset) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + + /* Generate Crypto op data structure */ + ut_params->op = rte_crypto_op_alloc(ts_params->op_mpool, + RTE_CRYPTO_OP_TYPE_SYMMETRIC); + TEST_ASSERT_NOT_NULL(ut_params->op, + "Failed to allocate pktmbuf offload"); + + /* Set crypto operation data parameters */ + rte_crypto_op_attach_sym_session(ut_params->op, ut_params->sess); + + struct rte_crypto_sym_op *sym_op = ut_params->op->sym; + + /* set crypto operation source mbuf */ + sym_op->m_src = ut_params->ibuf; + sym_op->m_dst = ut_params->obuf; + + /* iv */ + rte_memcpy(rte_crypto_op_ctod_offset(ut_params->op, uint8_t *, IV_OFFSET), + iv, iv_len); + sym_op->cipher.data.length = cipher_len; + sym_op->cipher.data.offset = cipher_offset; + return 0; +} + +static int +create_wireless_algo_cipher_auth_session(uint8_t dev_id, + enum rte_crypto_cipher_operation cipher_op, + enum rte_crypto_auth_operation auth_op, + enum rte_crypto_auth_algorithm auth_algo, + enum rte_crypto_cipher_algorithm cipher_algo, + const uint8_t *key, uint8_t key_len, + uint8_t auth_iv_len, uint8_t auth_len, + uint8_t cipher_iv_len) + +{ + uint8_t cipher_auth_key[key_len]; + + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + + memcpy(cipher_auth_key, key, key_len); + + /* Setup Authentication Parameters */ + ut_params->auth_xform.type = RTE_CRYPTO_SYM_XFORM_AUTH; + ut_params->auth_xform.next = NULL; + + ut_params->auth_xform.auth.op = auth_op; + ut_params->auth_xform.auth.algo = auth_algo; + ut_params->auth_xform.auth.key.length = key_len; + /* Hash key = cipher key */ + ut_params->auth_xform.auth.key.data = cipher_auth_key; + ut_params->auth_xform.auth.digest_length = auth_len; + /* Auth IV will be after cipher IV */ + ut_params->auth_xform.auth.iv.offset = IV_OFFSET + cipher_iv_len; + ut_params->auth_xform.auth.iv.length = auth_iv_len; + + /* Setup Cipher Parameters */ + ut_params->cipher_xform.type = RTE_CRYPTO_SYM_XFORM_CIPHER; + ut_params->cipher_xform.next = &ut_params->auth_xform; + + ut_params->cipher_xform.cipher.algo = cipher_algo; + ut_params->cipher_xform.cipher.op = cipher_op; + ut_params->cipher_xform.cipher.key.data = cipher_auth_key; + ut_params->cipher_xform.cipher.key.length = key_len; + ut_params->cipher_xform.cipher.iv.offset = IV_OFFSET; + ut_params->cipher_xform.cipher.iv.length = cipher_iv_len; + + TEST_HEXDUMP(stdout, "key:", key, key_len); + + /* Create Crypto session*/ + ut_params->sess = rte_cryptodev_sym_session_create( + ts_params->session_mpool); + + rte_cryptodev_sym_session_init(dev_id, ut_params->sess, + &ut_params->cipher_xform, ts_params->session_mpool); + + TEST_ASSERT_NOT_NULL(ut_params->sess, "Session creation failed"); + return 0; +} + +static int +create_wireless_cipher_auth_session(uint8_t dev_id, + enum rte_crypto_cipher_operation cipher_op, + enum rte_crypto_auth_operation auth_op, + enum rte_crypto_auth_algorithm auth_algo, + enum rte_crypto_cipher_algorithm cipher_algo, + const struct wireless_test_data *tdata) +{ + const uint8_t key_len = tdata->key.len; + uint8_t cipher_auth_key[key_len]; + + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + const uint8_t *key = tdata->key.data; + const uint8_t auth_len = tdata->digest.len; + uint8_t cipher_iv_len = tdata->cipher_iv.len; + uint8_t auth_iv_len = tdata->auth_iv.len; + + memcpy(cipher_auth_key, key, key_len); + + /* Setup Authentication Parameters */ + ut_params->auth_xform.type = RTE_CRYPTO_SYM_XFORM_AUTH; + ut_params->auth_xform.next = NULL; + + ut_params->auth_xform.auth.op = auth_op; + ut_params->auth_xform.auth.algo = auth_algo; + ut_params->auth_xform.auth.key.length = key_len; + /* Hash key = cipher key */ + ut_params->auth_xform.auth.key.data = cipher_auth_key; + ut_params->auth_xform.auth.digest_length = auth_len; + /* Auth IV will be after cipher IV */ + ut_params->auth_xform.auth.iv.offset = IV_OFFSET + cipher_iv_len; + ut_params->auth_xform.auth.iv.length = auth_iv_len; + + /* Setup Cipher Parameters */ + ut_params->cipher_xform.type = RTE_CRYPTO_SYM_XFORM_CIPHER; + ut_params->cipher_xform.next = &ut_params->auth_xform; + + ut_params->cipher_xform.cipher.algo = cipher_algo; + ut_params->cipher_xform.cipher.op = cipher_op; + ut_params->cipher_xform.cipher.key.data = cipher_auth_key; + ut_params->cipher_xform.cipher.key.length = key_len; + ut_params->cipher_xform.cipher.iv.offset = IV_OFFSET; + ut_params->cipher_xform.cipher.iv.length = cipher_iv_len; + + + TEST_HEXDUMP(stdout, "key:", key, key_len); + + /* Create Crypto session*/ + ut_params->sess = rte_cryptodev_sym_session_create( + ts_params->session_mpool); + + rte_cryptodev_sym_session_init(dev_id, ut_params->sess, + &ut_params->cipher_xform, ts_params->session_mpool); + + TEST_ASSERT_NOT_NULL(ut_params->sess, "Session creation failed"); + return 0; +} + +static int +create_zuc_cipher_auth_encrypt_generate_session(uint8_t dev_id, + const struct wireless_test_data *tdata) +{ + return create_wireless_cipher_auth_session(dev_id, + RTE_CRYPTO_CIPHER_OP_ENCRYPT, + RTE_CRYPTO_AUTH_OP_GENERATE, RTE_CRYPTO_AUTH_ZUC_EIA3, + RTE_CRYPTO_CIPHER_ZUC_EEA3, tdata); +} + +static int +create_wireless_algo_auth_cipher_session(uint8_t dev_id, + enum rte_crypto_cipher_operation cipher_op, + enum rte_crypto_auth_operation auth_op, + enum rte_crypto_auth_algorithm auth_algo, + enum rte_crypto_cipher_algorithm cipher_algo, + const uint8_t *key, const uint8_t key_len, + uint8_t auth_iv_len, uint8_t auth_len, + uint8_t cipher_iv_len) +{ + uint8_t auth_cipher_key[key_len]; + + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + + memcpy(auth_cipher_key, key, key_len); + + /* Setup Authentication Parameters */ + ut_params->auth_xform.type = RTE_CRYPTO_SYM_XFORM_AUTH; + ut_params->auth_xform.auth.op = auth_op; + ut_params->auth_xform.next = &ut_params->cipher_xform; + ut_params->auth_xform.auth.algo = auth_algo; + ut_params->auth_xform.auth.key.length = key_len; + ut_params->auth_xform.auth.key.data = auth_cipher_key; + ut_params->auth_xform.auth.digest_length = auth_len; + /* Auth IV will be after cipher IV */ + ut_params->auth_xform.auth.iv.offset = IV_OFFSET + cipher_iv_len; + ut_params->auth_xform.auth.iv.length = auth_iv_len; + + /* Setup Cipher Parameters */ + ut_params->cipher_xform.type = RTE_CRYPTO_SYM_XFORM_CIPHER; + ut_params->cipher_xform.next = NULL; + ut_params->cipher_xform.cipher.algo = cipher_algo; + ut_params->cipher_xform.cipher.op = cipher_op; + ut_params->cipher_xform.cipher.key.data = auth_cipher_key; + ut_params->cipher_xform.cipher.key.length = key_len; + ut_params->cipher_xform.cipher.iv.offset = IV_OFFSET; + ut_params->cipher_xform.cipher.iv.length = cipher_iv_len; + + TEST_HEXDUMP(stdout, "key:", key, key_len); + + /* Create Crypto session*/ + ut_params->sess = rte_cryptodev_sym_session_create( + ts_params->session_mpool); + + rte_cryptodev_sym_session_init(dev_id, ut_params->sess, + &ut_params->auth_xform, ts_params->session_mpool); + + TEST_ASSERT_NOT_NULL(ut_params->sess, "Session creation failed"); + + return 0; +} + +static int +create_wireless_algo_hash_operation(const uint8_t *auth_tag, + unsigned int auth_tag_len, + const uint8_t *iv, unsigned int iv_len, + unsigned int data_pad_len, + enum rte_crypto_auth_operation op, + unsigned int auth_len, unsigned int auth_offset) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + + struct crypto_unittest_params *ut_params = &unittest_params; + + /* Generate Crypto op data structure */ + ut_params->op = rte_crypto_op_alloc(ts_params->op_mpool, + RTE_CRYPTO_OP_TYPE_SYMMETRIC); + TEST_ASSERT_NOT_NULL(ut_params->op, + "Failed to allocate pktmbuf offload"); + + /* Set crypto operation data parameters */ + rte_crypto_op_attach_sym_session(ut_params->op, ut_params->sess); + + struct rte_crypto_sym_op *sym_op = ut_params->op->sym; + + /* set crypto operation source mbuf */ + sym_op->m_src = ut_params->ibuf; + + /* iv */ + rte_memcpy(rte_crypto_op_ctod_offset(ut_params->op, uint8_t *, IV_OFFSET), + iv, iv_len); + /* digest */ + sym_op->auth.digest.data = (uint8_t *)rte_pktmbuf_append( + ut_params->ibuf, auth_tag_len); + + TEST_ASSERT_NOT_NULL(sym_op->auth.digest.data, + "no room to append auth tag"); + ut_params->digest = sym_op->auth.digest.data; + sym_op->auth.digest.phys_addr = rte_pktmbuf_iova_offset( + ut_params->ibuf, data_pad_len); + if (op == RTE_CRYPTO_AUTH_OP_GENERATE) + memset(sym_op->auth.digest.data, 0, auth_tag_len); + else + rte_memcpy(sym_op->auth.digest.data, auth_tag, auth_tag_len); + + TEST_HEXDUMP(stdout, "digest:", + sym_op->auth.digest.data, + auth_tag_len); + + sym_op->auth.data.length = auth_len; + sym_op->auth.data.offset = auth_offset; + + return 0; +} + +static int +create_wireless_cipher_hash_operation(const struct wireless_test_data *tdata, + enum rte_crypto_auth_operation op) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + + const uint8_t *auth_tag = tdata->digest.data; + const unsigned int auth_tag_len = tdata->digest.len; + unsigned int plaintext_len = ceil_byte_length(tdata->plaintext.len); + unsigned int data_pad_len = RTE_ALIGN_CEIL(plaintext_len, 16); + + const uint8_t *cipher_iv = tdata->cipher_iv.data; + const uint8_t cipher_iv_len = tdata->cipher_iv.len; + const uint8_t *auth_iv = tdata->auth_iv.data; + const uint8_t auth_iv_len = tdata->auth_iv.len; + const unsigned int cipher_len = tdata->validCipherLenInBits.len; + const unsigned int auth_len = tdata->validAuthLenInBits.len; + + /* Generate Crypto op data structure */ + ut_params->op = rte_crypto_op_alloc(ts_params->op_mpool, + RTE_CRYPTO_OP_TYPE_SYMMETRIC); + TEST_ASSERT_NOT_NULL(ut_params->op, + "Failed to allocate pktmbuf offload"); + /* Set crypto operation data parameters */ + rte_crypto_op_attach_sym_session(ut_params->op, ut_params->sess); + + struct rte_crypto_sym_op *sym_op = ut_params->op->sym; + + /* set crypto operation source mbuf */ + sym_op->m_src = ut_params->ibuf; + + /* digest */ + sym_op->auth.digest.data = (uint8_t *)rte_pktmbuf_append( + ut_params->ibuf, auth_tag_len); + + TEST_ASSERT_NOT_NULL(sym_op->auth.digest.data, + "no room to append auth tag"); + ut_params->digest = sym_op->auth.digest.data; + sym_op->auth.digest.phys_addr = rte_pktmbuf_iova_offset( + ut_params->ibuf, data_pad_len); + if (op == RTE_CRYPTO_AUTH_OP_GENERATE) + memset(sym_op->auth.digest.data, 0, auth_tag_len); + else + rte_memcpy(sym_op->auth.digest.data, auth_tag, auth_tag_len); + + TEST_HEXDUMP(stdout, "digest:", + sym_op->auth.digest.data, + auth_tag_len); + + /* Copy cipher and auth IVs at the end of the crypto operation */ + uint8_t *iv_ptr = rte_crypto_op_ctod_offset(ut_params->op, uint8_t *, + IV_OFFSET); + rte_memcpy(iv_ptr, cipher_iv, cipher_iv_len); + iv_ptr += cipher_iv_len; + rte_memcpy(iv_ptr, auth_iv, auth_iv_len); + + sym_op->cipher.data.length = cipher_len; + sym_op->cipher.data.offset = 0; + sym_op->auth.data.length = auth_len; + sym_op->auth.data.offset = 0; + + return 0; +} + +static int +create_zuc_cipher_hash_generate_operation( + const struct wireless_test_data *tdata) +{ + return create_wireless_cipher_hash_operation(tdata, + RTE_CRYPTO_AUTH_OP_GENERATE); +} + +static int +create_wireless_algo_cipher_hash_operation(const uint8_t *auth_tag, + const unsigned auth_tag_len, + const uint8_t *auth_iv, uint8_t auth_iv_len, + unsigned data_pad_len, + enum rte_crypto_auth_operation op, + const uint8_t *cipher_iv, uint8_t cipher_iv_len, + const unsigned cipher_len, const unsigned cipher_offset, + const unsigned auth_len, const unsigned auth_offset) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + + /* Generate Crypto op data structure */ + ut_params->op = rte_crypto_op_alloc(ts_params->op_mpool, + RTE_CRYPTO_OP_TYPE_SYMMETRIC); + TEST_ASSERT_NOT_NULL(ut_params->op, + "Failed to allocate pktmbuf offload"); + /* Set crypto operation data parameters */ + rte_crypto_op_attach_sym_session(ut_params->op, ut_params->sess); + + struct rte_crypto_sym_op *sym_op = ut_params->op->sym; + + /* set crypto operation source mbuf */ + sym_op->m_src = ut_params->ibuf; + + /* digest */ + sym_op->auth.digest.data = (uint8_t *)rte_pktmbuf_append( + ut_params->ibuf, auth_tag_len); + + TEST_ASSERT_NOT_NULL(sym_op->auth.digest.data, + "no room to append auth tag"); + ut_params->digest = sym_op->auth.digest.data; + sym_op->auth.digest.phys_addr = rte_pktmbuf_iova_offset( + ut_params->ibuf, data_pad_len); + if (op == RTE_CRYPTO_AUTH_OP_GENERATE) + memset(sym_op->auth.digest.data, 0, auth_tag_len); + else + rte_memcpy(sym_op->auth.digest.data, auth_tag, auth_tag_len); + + TEST_HEXDUMP(stdout, "digest:", + sym_op->auth.digest.data, + auth_tag_len); + + /* Copy cipher and auth IVs at the end of the crypto operation */ + uint8_t *iv_ptr = rte_crypto_op_ctod_offset(ut_params->op, uint8_t *, + IV_OFFSET); + rte_memcpy(iv_ptr, cipher_iv, cipher_iv_len); + iv_ptr += cipher_iv_len; + rte_memcpy(iv_ptr, auth_iv, auth_iv_len); + + sym_op->cipher.data.length = cipher_len; + sym_op->cipher.data.offset = cipher_offset; + sym_op->auth.data.length = auth_len; + sym_op->auth.data.offset = auth_offset; + + return 0; +} + +static int +create_wireless_algo_auth_cipher_operation(unsigned int auth_tag_len, + const uint8_t *cipher_iv, uint8_t cipher_iv_len, + const uint8_t *auth_iv, uint8_t auth_iv_len, + unsigned int data_pad_len, + unsigned int cipher_len, unsigned int cipher_offset, + unsigned int auth_len, unsigned int auth_offset) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + + /* Generate Crypto op data structure */ + ut_params->op = rte_crypto_op_alloc(ts_params->op_mpool, + RTE_CRYPTO_OP_TYPE_SYMMETRIC); + TEST_ASSERT_NOT_NULL(ut_params->op, + "Failed to allocate pktmbuf offload"); + + /* Set crypto operation data parameters */ + rte_crypto_op_attach_sym_session(ut_params->op, ut_params->sess); + + struct rte_crypto_sym_op *sym_op = ut_params->op->sym; + + /* set crypto operation source mbuf */ + sym_op->m_src = ut_params->ibuf; + + /* digest */ + sym_op->auth.digest.data = (uint8_t *)rte_pktmbuf_append( + ut_params->ibuf, auth_tag_len); + + TEST_ASSERT_NOT_NULL(sym_op->auth.digest.data, + "no room to append auth tag"); + + sym_op->auth.digest.phys_addr = rte_pktmbuf_iova_offset( + ut_params->ibuf, data_pad_len); + + memset(sym_op->auth.digest.data, 0, auth_tag_len); + + TEST_HEXDUMP(stdout, "digest:", + sym_op->auth.digest.data, + auth_tag_len); + + /* Copy cipher and auth IVs at the end of the crypto operation */ + uint8_t *iv_ptr = rte_crypto_op_ctod_offset(ut_params->op, uint8_t *, + IV_OFFSET); + rte_memcpy(iv_ptr, cipher_iv, cipher_iv_len); + iv_ptr += cipher_iv_len; + rte_memcpy(iv_ptr, auth_iv, auth_iv_len); + + sym_op->cipher.data.length = cipher_len; + sym_op->cipher.data.offset = cipher_offset; + + sym_op->auth.data.length = auth_len; + sym_op->auth.data.offset = auth_offset; + + return 0; +} + +static int +test_snow3g_authentication(const struct snow3g_hash_test_data *tdata) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + + int retval; + unsigned plaintext_pad_len; + unsigned plaintext_len; + uint8_t *plaintext; + + /* Create SNOW 3G session */ + retval = create_wireless_algo_hash_session(ts_params->valid_devs[0], + tdata->key.data, tdata->key.len, + tdata->auth_iv.len, tdata->digest.len, + RTE_CRYPTO_AUTH_OP_GENERATE, + RTE_CRYPTO_AUTH_SNOW3G_UIA2); + if (retval < 0) + return retval; + + /* alloc mbuf and set payload */ + ut_params->ibuf = rte_pktmbuf_alloc(ts_params->mbuf_pool); + + memset(rte_pktmbuf_mtod(ut_params->ibuf, uint8_t *), 0, + rte_pktmbuf_tailroom(ut_params->ibuf)); + + plaintext_len = ceil_byte_length(tdata->plaintext.len); + /* Append data which is padded to a multiple of */ + /* the algorithms block size */ + plaintext_pad_len = RTE_ALIGN_CEIL(plaintext_len, 16); + plaintext = (uint8_t *)rte_pktmbuf_append(ut_params->ibuf, + plaintext_pad_len); + memcpy(plaintext, tdata->plaintext.data, plaintext_len); + + /* Create SNOW 3G operation */ + retval = create_wireless_algo_hash_operation(NULL, tdata->digest.len, + tdata->auth_iv.data, tdata->auth_iv.len, + plaintext_pad_len, RTE_CRYPTO_AUTH_OP_GENERATE, + tdata->validAuthLenInBits.len, + 0); + if (retval < 0) + return retval; + + ut_params->op = process_crypto_request(ts_params->valid_devs[0], + ut_params->op); + ut_params->obuf = ut_params->op->sym->m_src; + TEST_ASSERT_NOT_NULL(ut_params->op, "failed to retrieve obuf"); + ut_params->digest = rte_pktmbuf_mtod(ut_params->obuf, uint8_t *) + + plaintext_pad_len; + + /* Validate obuf */ + TEST_ASSERT_BUFFERS_ARE_EQUAL( + ut_params->digest, + tdata->digest.data, + DIGEST_BYTE_LENGTH_SNOW3G_UIA2, + "SNOW 3G Generated auth tag not as expected"); + + return 0; +} + +static int +test_snow3g_authentication_verify(const struct snow3g_hash_test_data *tdata) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + + int retval; + unsigned plaintext_pad_len; + unsigned plaintext_len; + uint8_t *plaintext; + + /* Create SNOW 3G session */ + retval = create_wireless_algo_hash_session(ts_params->valid_devs[0], + tdata->key.data, tdata->key.len, + tdata->auth_iv.len, tdata->digest.len, + RTE_CRYPTO_AUTH_OP_VERIFY, + RTE_CRYPTO_AUTH_SNOW3G_UIA2); + if (retval < 0) + return retval; + /* alloc mbuf and set payload */ + ut_params->ibuf = rte_pktmbuf_alloc(ts_params->mbuf_pool); + + memset(rte_pktmbuf_mtod(ut_params->ibuf, uint8_t *), 0, + rte_pktmbuf_tailroom(ut_params->ibuf)); + + plaintext_len = ceil_byte_length(tdata->plaintext.len); + /* Append data which is padded to a multiple of */ + /* the algorithms block size */ + plaintext_pad_len = RTE_ALIGN_CEIL(plaintext_len, 16); + plaintext = (uint8_t *)rte_pktmbuf_append(ut_params->ibuf, + plaintext_pad_len); + memcpy(plaintext, tdata->plaintext.data, plaintext_len); + + /* Create SNOW 3G operation */ + retval = create_wireless_algo_hash_operation(tdata->digest.data, + tdata->digest.len, + tdata->auth_iv.data, tdata->auth_iv.len, + plaintext_pad_len, + RTE_CRYPTO_AUTH_OP_VERIFY, + tdata->validAuthLenInBits.len, + 0); + if (retval < 0) + return retval; + + ut_params->op = process_crypto_request(ts_params->valid_devs[0], + ut_params->op); + TEST_ASSERT_NOT_NULL(ut_params->op, "failed to retrieve obuf"); + ut_params->obuf = ut_params->op->sym->m_src; + ut_params->digest = rte_pktmbuf_mtod(ut_params->obuf, uint8_t *) + + plaintext_pad_len; + + /* Validate obuf */ + if (ut_params->op->status == RTE_CRYPTO_OP_STATUS_SUCCESS) + return 0; + else + return -1; + + return 0; +} + +static int +test_kasumi_authentication(const struct kasumi_hash_test_data *tdata) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + + int retval; + unsigned plaintext_pad_len; + unsigned plaintext_len; + uint8_t *plaintext; + + /* Create KASUMI session */ + retval = create_wireless_algo_hash_session(ts_params->valid_devs[0], + tdata->key.data, tdata->key.len, + 0, tdata->digest.len, + RTE_CRYPTO_AUTH_OP_GENERATE, + RTE_CRYPTO_AUTH_KASUMI_F9); + if (retval < 0) + return retval; + + /* alloc mbuf and set payload */ + ut_params->ibuf = rte_pktmbuf_alloc(ts_params->mbuf_pool); + + memset(rte_pktmbuf_mtod(ut_params->ibuf, uint8_t *), 0, + rte_pktmbuf_tailroom(ut_params->ibuf)); + + plaintext_len = ceil_byte_length(tdata->plaintext.len); + /* Append data which is padded to a multiple of */ + /* the algorithms block size */ + plaintext_pad_len = RTE_ALIGN_CEIL(plaintext_len, 8); + plaintext = (uint8_t *)rte_pktmbuf_append(ut_params->ibuf, + plaintext_pad_len); + memcpy(plaintext, tdata->plaintext.data, plaintext_len); + + /* Create KASUMI operation */ + retval = create_wireless_algo_hash_operation(NULL, tdata->digest.len, + NULL, 0, + plaintext_pad_len, RTE_CRYPTO_AUTH_OP_GENERATE, + tdata->plaintext.len, + 0); + if (retval < 0) + return retval; + + ut_params->op = process_crypto_request(ts_params->valid_devs[0], + ut_params->op); + ut_params->obuf = ut_params->op->sym->m_src; + TEST_ASSERT_NOT_NULL(ut_params->op, "failed to retrieve obuf"); + ut_params->digest = rte_pktmbuf_mtod(ut_params->obuf, uint8_t *) + + plaintext_pad_len; + + /* Validate obuf */ + TEST_ASSERT_BUFFERS_ARE_EQUAL( + ut_params->digest, + tdata->digest.data, + DIGEST_BYTE_LENGTH_KASUMI_F9, + "KASUMI Generated auth tag not as expected"); + + return 0; +} + +static int +test_kasumi_authentication_verify(const struct kasumi_hash_test_data *tdata) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + + int retval; + unsigned plaintext_pad_len; + unsigned plaintext_len; + uint8_t *plaintext; + + /* Create KASUMI session */ + retval = create_wireless_algo_hash_session(ts_params->valid_devs[0], + tdata->key.data, tdata->key.len, + 0, tdata->digest.len, + RTE_CRYPTO_AUTH_OP_VERIFY, + RTE_CRYPTO_AUTH_KASUMI_F9); + if (retval < 0) + return retval; + /* alloc mbuf and set payload */ + ut_params->ibuf = rte_pktmbuf_alloc(ts_params->mbuf_pool); + + memset(rte_pktmbuf_mtod(ut_params->ibuf, uint8_t *), 0, + rte_pktmbuf_tailroom(ut_params->ibuf)); + + plaintext_len = ceil_byte_length(tdata->plaintext.len); + /* Append data which is padded to a multiple */ + /* of the algorithms block size */ + plaintext_pad_len = RTE_ALIGN_CEIL(plaintext_len, 8); + plaintext = (uint8_t *)rte_pktmbuf_append(ut_params->ibuf, + plaintext_pad_len); + memcpy(plaintext, tdata->plaintext.data, plaintext_len); + + /* Create KASUMI operation */ + retval = create_wireless_algo_hash_operation(tdata->digest.data, + tdata->digest.len, + NULL, 0, + plaintext_pad_len, + RTE_CRYPTO_AUTH_OP_VERIFY, + tdata->plaintext.len, + 0); + if (retval < 0) + return retval; + + ut_params->op = process_crypto_request(ts_params->valid_devs[0], + ut_params->op); + TEST_ASSERT_NOT_NULL(ut_params->op, "failed to retrieve obuf"); + ut_params->obuf = ut_params->op->sym->m_src; + ut_params->digest = rte_pktmbuf_mtod(ut_params->obuf, uint8_t *) + + plaintext_pad_len; + + /* Validate obuf */ + if (ut_params->op->status == RTE_CRYPTO_OP_STATUS_SUCCESS) + return 0; + else + return -1; + + return 0; +} + +static int +test_snow3g_hash_generate_test_case_1(void) +{ + return test_snow3g_authentication(&snow3g_hash_test_case_1); +} + +static int +test_snow3g_hash_generate_test_case_2(void) +{ + return test_snow3g_authentication(&snow3g_hash_test_case_2); +} + +static int +test_snow3g_hash_generate_test_case_3(void) +{ + return test_snow3g_authentication(&snow3g_hash_test_case_3); +} + +static int +test_snow3g_hash_generate_test_case_4(void) +{ + return test_snow3g_authentication(&snow3g_hash_test_case_4); +} + +static int +test_snow3g_hash_generate_test_case_5(void) +{ + return test_snow3g_authentication(&snow3g_hash_test_case_5); +} + +static int +test_snow3g_hash_generate_test_case_6(void) +{ + return test_snow3g_authentication(&snow3g_hash_test_case_6); +} + +static int +test_snow3g_hash_verify_test_case_1(void) +{ + return test_snow3g_authentication_verify(&snow3g_hash_test_case_1); + +} + +static int +test_snow3g_hash_verify_test_case_2(void) +{ + return test_snow3g_authentication_verify(&snow3g_hash_test_case_2); +} + +static int +test_snow3g_hash_verify_test_case_3(void) +{ + return test_snow3g_authentication_verify(&snow3g_hash_test_case_3); +} + +static int +test_snow3g_hash_verify_test_case_4(void) +{ + return test_snow3g_authentication_verify(&snow3g_hash_test_case_4); +} + +static int +test_snow3g_hash_verify_test_case_5(void) +{ + return test_snow3g_authentication_verify(&snow3g_hash_test_case_5); +} + +static int +test_snow3g_hash_verify_test_case_6(void) +{ + return test_snow3g_authentication_verify(&snow3g_hash_test_case_6); +} + +static int +test_kasumi_hash_generate_test_case_1(void) +{ + return test_kasumi_authentication(&kasumi_hash_test_case_1); +} + +static int +test_kasumi_hash_generate_test_case_2(void) +{ + return test_kasumi_authentication(&kasumi_hash_test_case_2); +} + +static int +test_kasumi_hash_generate_test_case_3(void) +{ + return test_kasumi_authentication(&kasumi_hash_test_case_3); +} + +static int +test_kasumi_hash_generate_test_case_4(void) +{ + return test_kasumi_authentication(&kasumi_hash_test_case_4); +} + +static int +test_kasumi_hash_generate_test_case_5(void) +{ + return test_kasumi_authentication(&kasumi_hash_test_case_5); +} + +static int +test_kasumi_hash_generate_test_case_6(void) +{ + return test_kasumi_authentication(&kasumi_hash_test_case_6); +} + +static int +test_kasumi_hash_verify_test_case_1(void) +{ + return test_kasumi_authentication_verify(&kasumi_hash_test_case_1); +} + +static int +test_kasumi_hash_verify_test_case_2(void) +{ + return test_kasumi_authentication_verify(&kasumi_hash_test_case_2); +} + +static int +test_kasumi_hash_verify_test_case_3(void) +{ + return test_kasumi_authentication_verify(&kasumi_hash_test_case_3); +} + +static int +test_kasumi_hash_verify_test_case_4(void) +{ + return test_kasumi_authentication_verify(&kasumi_hash_test_case_4); +} + +static int +test_kasumi_hash_verify_test_case_5(void) +{ + return test_kasumi_authentication_verify(&kasumi_hash_test_case_5); +} + +static int +test_kasumi_encryption(const struct kasumi_test_data *tdata) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + + int retval; + uint8_t *plaintext, *ciphertext; + unsigned plaintext_pad_len; + unsigned plaintext_len; + + /* Create KASUMI session */ + retval = create_wireless_algo_cipher_session(ts_params->valid_devs[0], + RTE_CRYPTO_CIPHER_OP_ENCRYPT, + RTE_CRYPTO_CIPHER_KASUMI_F8, + tdata->key.data, tdata->key.len, + tdata->cipher_iv.len); + if (retval < 0) + return retval; + + ut_params->ibuf = rte_pktmbuf_alloc(ts_params->mbuf_pool); + + /* Clear mbuf payload */ + memset(rte_pktmbuf_mtod(ut_params->ibuf, uint8_t *), 0, + rte_pktmbuf_tailroom(ut_params->ibuf)); + + plaintext_len = ceil_byte_length(tdata->plaintext.len); + /* Append data which is padded to a multiple */ + /* of the algorithms block size */ + plaintext_pad_len = RTE_ALIGN_CEIL(plaintext_len, 8); + plaintext = (uint8_t *)rte_pktmbuf_append(ut_params->ibuf, + plaintext_pad_len); + memcpy(plaintext, tdata->plaintext.data, plaintext_len); + + TEST_HEXDUMP(stdout, "plaintext:", plaintext, plaintext_len); + + /* Create KASUMI operation */ + retval = create_wireless_algo_cipher_operation(tdata->cipher_iv.data, + tdata->cipher_iv.len, + RTE_ALIGN_CEIL(tdata->validCipherLenInBits.len, 8), + tdata->validCipherOffsetInBits.len); + if (retval < 0) + return retval; + + ut_params->op = process_crypto_request(ts_params->valid_devs[0], + ut_params->op); + TEST_ASSERT_NOT_NULL(ut_params->op, "failed to retrieve obuf"); + + ut_params->obuf = ut_params->op->sym->m_dst; + if (ut_params->obuf) + ciphertext = rte_pktmbuf_mtod(ut_params->obuf, uint8_t *); + else + ciphertext = plaintext + (tdata->validCipherOffsetInBits.len >> 3); + + TEST_HEXDUMP(stdout, "ciphertext:", ciphertext, plaintext_len); + + const uint8_t *reference_ciphertext = tdata->ciphertext.data + + (tdata->validCipherOffsetInBits.len >> 3); + /* Validate obuf */ + TEST_ASSERT_BUFFERS_ARE_EQUAL_BIT( + ciphertext, + reference_ciphertext, + tdata->validCipherLenInBits.len, + "KASUMI Ciphertext data not as expected"); + return 0; +} + +static int +test_kasumi_encryption_sgl(const struct kasumi_test_data *tdata) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + + int retval; + + unsigned int plaintext_pad_len; + unsigned int plaintext_len; + + uint8_t buffer[10000]; + const uint8_t *ciphertext; + + struct rte_cryptodev_info dev_info; + + rte_cryptodev_info_get(ts_params->valid_devs[0], &dev_info); + if (!(dev_info.feature_flags & RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER)) { + printf("Device doesn't support scatter-gather. " + "Test Skipped.\n"); + return 0; + } + + /* Create KASUMI session */ + retval = create_wireless_algo_cipher_session(ts_params->valid_devs[0], + RTE_CRYPTO_CIPHER_OP_ENCRYPT, + RTE_CRYPTO_CIPHER_KASUMI_F8, + tdata->key.data, tdata->key.len, + tdata->cipher_iv.len); + if (retval < 0) + return retval; + + plaintext_len = ceil_byte_length(tdata->plaintext.len); + + + /* Append data which is padded to a multiple */ + /* of the algorithms block size */ + plaintext_pad_len = RTE_ALIGN_CEIL(plaintext_len, 8); + + ut_params->ibuf = create_segmented_mbuf(ts_params->mbuf_pool, + plaintext_pad_len, 10, 0); + + pktmbuf_write(ut_params->ibuf, 0, plaintext_len, tdata->plaintext.data); + + /* Create KASUMI operation */ + retval = create_wireless_algo_cipher_operation(tdata->cipher_iv.data, + tdata->cipher_iv.len, + RTE_ALIGN_CEIL(tdata->validCipherLenInBits.len, 8), + tdata->validCipherOffsetInBits.len); + if (retval < 0) + return retval; + + ut_params->op = process_crypto_request(ts_params->valid_devs[0], + ut_params->op); + TEST_ASSERT_NOT_NULL(ut_params->op, "failed to retrieve obuf"); + + ut_params->obuf = ut_params->op->sym->m_dst; + + if (ut_params->obuf) + ciphertext = rte_pktmbuf_read(ut_params->obuf, 0, + plaintext_len, buffer); + else + ciphertext = rte_pktmbuf_read(ut_params->ibuf, + tdata->validCipherOffsetInBits.len >> 3, + plaintext_len, buffer); + + /* Validate obuf */ + TEST_HEXDUMP(stdout, "ciphertext:", ciphertext, plaintext_len); + + const uint8_t *reference_ciphertext = tdata->ciphertext.data + + (tdata->validCipherOffsetInBits.len >> 3); + /* Validate obuf */ + TEST_ASSERT_BUFFERS_ARE_EQUAL_BIT( + ciphertext, + reference_ciphertext, + tdata->validCipherLenInBits.len, + "KASUMI Ciphertext data not as expected"); + return 0; +} + +static int +test_kasumi_encryption_oop(const struct kasumi_test_data *tdata) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + + int retval; + uint8_t *plaintext, *ciphertext; + unsigned plaintext_pad_len; + unsigned plaintext_len; + + /* Create KASUMI session */ + retval = create_wireless_algo_cipher_session(ts_params->valid_devs[0], + RTE_CRYPTO_CIPHER_OP_ENCRYPT, + RTE_CRYPTO_CIPHER_KASUMI_F8, + tdata->key.data, tdata->key.len, + tdata->cipher_iv.len); + if (retval < 0) + return retval; + + ut_params->ibuf = rte_pktmbuf_alloc(ts_params->mbuf_pool); + ut_params->obuf = rte_pktmbuf_alloc(ts_params->mbuf_pool); + + /* Clear mbuf payload */ + memset(rte_pktmbuf_mtod(ut_params->ibuf, uint8_t *), 0, + rte_pktmbuf_tailroom(ut_params->ibuf)); + + plaintext_len = ceil_byte_length(tdata->plaintext.len); + /* Append data which is padded to a multiple */ + /* of the algorithms block size */ + plaintext_pad_len = RTE_ALIGN_CEIL(plaintext_len, 8); + plaintext = (uint8_t *)rte_pktmbuf_append(ut_params->ibuf, + plaintext_pad_len); + rte_pktmbuf_append(ut_params->obuf, plaintext_pad_len); + memcpy(plaintext, tdata->plaintext.data, plaintext_len); + + TEST_HEXDUMP(stdout, "plaintext:", plaintext, plaintext_len); + + /* Create KASUMI operation */ + retval = create_wireless_algo_cipher_operation_oop(tdata->cipher_iv.data, + tdata->cipher_iv.len, + RTE_ALIGN_CEIL(tdata->validCipherLenInBits.len, 8), + tdata->validCipherOffsetInBits.len); + if (retval < 0) + return retval; + + ut_params->op = process_crypto_request(ts_params->valid_devs[0], + ut_params->op); + TEST_ASSERT_NOT_NULL(ut_params->op, "failed to retrieve obuf"); + + ut_params->obuf = ut_params->op->sym->m_dst; + if (ut_params->obuf) + ciphertext = rte_pktmbuf_mtod(ut_params->obuf, uint8_t *); + else + ciphertext = plaintext + (tdata->validCipherOffsetInBits.len >> 3); + + TEST_HEXDUMP(stdout, "ciphertext:", ciphertext, plaintext_len); + + const uint8_t *reference_ciphertext = tdata->ciphertext.data + + (tdata->validCipherOffsetInBits.len >> 3); + /* Validate obuf */ + TEST_ASSERT_BUFFERS_ARE_EQUAL_BIT( + ciphertext, + reference_ciphertext, + tdata->validCipherLenInBits.len, + "KASUMI Ciphertext data not as expected"); + return 0; +} + +static int +test_kasumi_encryption_oop_sgl(const struct kasumi_test_data *tdata) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + + int retval; + unsigned int plaintext_pad_len; + unsigned int plaintext_len; + + const uint8_t *ciphertext; + uint8_t buffer[2048]; + + struct rte_cryptodev_info dev_info; + + rte_cryptodev_info_get(ts_params->valid_devs[0], &dev_info); + if (!(dev_info.feature_flags & RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER)) { + printf("Device doesn't support scatter-gather. " + "Test Skipped.\n"); + return 0; + } + + /* Create KASUMI session */ + retval = create_wireless_algo_cipher_session(ts_params->valid_devs[0], + RTE_CRYPTO_CIPHER_OP_ENCRYPT, + RTE_CRYPTO_CIPHER_KASUMI_F8, + tdata->key.data, tdata->key.len, + tdata->cipher_iv.len); + if (retval < 0) + return retval; + + plaintext_len = ceil_byte_length(tdata->plaintext.len); + /* Append data which is padded to a multiple */ + /* of the algorithms block size */ + plaintext_pad_len = RTE_ALIGN_CEIL(plaintext_len, 8); + + ut_params->ibuf = create_segmented_mbuf(ts_params->mbuf_pool, + plaintext_pad_len, 10, 0); + ut_params->obuf = create_segmented_mbuf(ts_params->mbuf_pool, + plaintext_pad_len, 3, 0); + + /* Append data which is padded to a multiple */ + /* of the algorithms block size */ + pktmbuf_write(ut_params->ibuf, 0, plaintext_len, tdata->plaintext.data); + + /* Create KASUMI operation */ + retval = create_wireless_algo_cipher_operation_oop(tdata->cipher_iv.data, + tdata->cipher_iv.len, + RTE_ALIGN_CEIL(tdata->validCipherLenInBits.len, 8), + tdata->validCipherOffsetInBits.len); + if (retval < 0) + return retval; + + ut_params->op = process_crypto_request(ts_params->valid_devs[0], + ut_params->op); + TEST_ASSERT_NOT_NULL(ut_params->op, "failed to retrieve obuf"); + + ut_params->obuf = ut_params->op->sym->m_dst; + if (ut_params->obuf) + ciphertext = rte_pktmbuf_read(ut_params->obuf, 0, + plaintext_pad_len, buffer); + else + ciphertext = rte_pktmbuf_read(ut_params->ibuf, + tdata->validCipherOffsetInBits.len >> 3, + plaintext_pad_len, buffer); + + const uint8_t *reference_ciphertext = tdata->ciphertext.data + + (tdata->validCipherOffsetInBits.len >> 3); + /* Validate obuf */ + TEST_ASSERT_BUFFERS_ARE_EQUAL_BIT( + ciphertext, + reference_ciphertext, + tdata->validCipherLenInBits.len, + "KASUMI Ciphertext data not as expected"); + return 0; +} + + +static int +test_kasumi_decryption_oop(const struct kasumi_test_data *tdata) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + + int retval; + uint8_t *ciphertext, *plaintext; + unsigned ciphertext_pad_len; + unsigned ciphertext_len; + + /* Create KASUMI session */ + retval = create_wireless_algo_cipher_session(ts_params->valid_devs[0], + RTE_CRYPTO_CIPHER_OP_DECRYPT, + RTE_CRYPTO_CIPHER_KASUMI_F8, + tdata->key.data, tdata->key.len, + tdata->cipher_iv.len); + if (retval < 0) + return retval; + + ut_params->ibuf = rte_pktmbuf_alloc(ts_params->mbuf_pool); + ut_params->obuf = rte_pktmbuf_alloc(ts_params->mbuf_pool); + + /* Clear mbuf payload */ + memset(rte_pktmbuf_mtod(ut_params->ibuf, uint8_t *), 0, + rte_pktmbuf_tailroom(ut_params->ibuf)); + + ciphertext_len = ceil_byte_length(tdata->ciphertext.len); + /* Append data which is padded to a multiple */ + /* of the algorithms block size */ + ciphertext_pad_len = RTE_ALIGN_CEIL(ciphertext_len, 8); + ciphertext = (uint8_t *)rte_pktmbuf_append(ut_params->ibuf, + ciphertext_pad_len); + rte_pktmbuf_append(ut_params->obuf, ciphertext_pad_len); + memcpy(ciphertext, tdata->ciphertext.data, ciphertext_len); + + TEST_HEXDUMP(stdout, "ciphertext:", ciphertext, ciphertext_len); + + /* Create KASUMI operation */ + retval = create_wireless_algo_cipher_operation_oop(tdata->cipher_iv.data, + tdata->cipher_iv.len, + RTE_ALIGN_CEIL(tdata->validCipherLenInBits.len, 8), + tdata->validCipherOffsetInBits.len); + if (retval < 0) + return retval; + + ut_params->op = process_crypto_request(ts_params->valid_devs[0], + ut_params->op); + TEST_ASSERT_NOT_NULL(ut_params->op, "failed to retrieve obuf"); + + ut_params->obuf = ut_params->op->sym->m_dst; + if (ut_params->obuf) + plaintext = rte_pktmbuf_mtod(ut_params->obuf, uint8_t *); + else + plaintext = ciphertext + (tdata->validCipherOffsetInBits.len >> 3); + + TEST_HEXDUMP(stdout, "plaintext:", plaintext, ciphertext_len); + + const uint8_t *reference_plaintext = tdata->plaintext.data + + (tdata->validCipherOffsetInBits.len >> 3); + /* Validate obuf */ + TEST_ASSERT_BUFFERS_ARE_EQUAL_BIT( + plaintext, + reference_plaintext, + tdata->validCipherLenInBits.len, + "KASUMI Plaintext data not as expected"); + return 0; +} + +static int +test_kasumi_decryption(const struct kasumi_test_data *tdata) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + + int retval; + uint8_t *ciphertext, *plaintext; + unsigned ciphertext_pad_len; + unsigned ciphertext_len; + + /* Create KASUMI session */ + retval = create_wireless_algo_cipher_session(ts_params->valid_devs[0], + RTE_CRYPTO_CIPHER_OP_DECRYPT, + RTE_CRYPTO_CIPHER_KASUMI_F8, + tdata->key.data, tdata->key.len, + tdata->cipher_iv.len); + if (retval < 0) + return retval; + + ut_params->ibuf = rte_pktmbuf_alloc(ts_params->mbuf_pool); + + /* Clear mbuf payload */ + memset(rte_pktmbuf_mtod(ut_params->ibuf, uint8_t *), 0, + rte_pktmbuf_tailroom(ut_params->ibuf)); + + ciphertext_len = ceil_byte_length(tdata->ciphertext.len); + /* Append data which is padded to a multiple */ + /* of the algorithms block size */ + ciphertext_pad_len = RTE_ALIGN_CEIL(ciphertext_len, 8); + ciphertext = (uint8_t *)rte_pktmbuf_append(ut_params->ibuf, + ciphertext_pad_len); + memcpy(ciphertext, tdata->ciphertext.data, ciphertext_len); + + TEST_HEXDUMP(stdout, "ciphertext:", ciphertext, ciphertext_len); + + /* Create KASUMI operation */ + retval = create_wireless_algo_cipher_operation(tdata->cipher_iv.data, + tdata->cipher_iv.len, + tdata->ciphertext.len, + tdata->validCipherOffsetInBits.len); + if (retval < 0) + return retval; + + ut_params->op = process_crypto_request(ts_params->valid_devs[0], + ut_params->op); + TEST_ASSERT_NOT_NULL(ut_params->op, "failed to retrieve obuf"); + + ut_params->obuf = ut_params->op->sym->m_dst; + if (ut_params->obuf) + plaintext = rte_pktmbuf_mtod(ut_params->obuf, uint8_t *); + else + plaintext = ciphertext + (tdata->validCipherOffsetInBits.len >> 3); + + TEST_HEXDUMP(stdout, "plaintext:", plaintext, ciphertext_len); + + const uint8_t *reference_plaintext = tdata->plaintext.data + + (tdata->validCipherOffsetInBits.len >> 3); + /* Validate obuf */ + TEST_ASSERT_BUFFERS_ARE_EQUAL_BIT( + plaintext, + reference_plaintext, + tdata->validCipherLenInBits.len, + "KASUMI Plaintext data not as expected"); + return 0; +} + +static int +test_snow3g_encryption(const struct snow3g_test_data *tdata) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + + int retval; + uint8_t *plaintext, *ciphertext; + unsigned plaintext_pad_len; + unsigned plaintext_len; + + /* Create SNOW 3G session */ + retval = create_wireless_algo_cipher_session(ts_params->valid_devs[0], + RTE_CRYPTO_CIPHER_OP_ENCRYPT, + RTE_CRYPTO_CIPHER_SNOW3G_UEA2, + tdata->key.data, tdata->key.len, + tdata->cipher_iv.len); + if (retval < 0) + return retval; + + ut_params->ibuf = rte_pktmbuf_alloc(ts_params->mbuf_pool); + + /* Clear mbuf payload */ + memset(rte_pktmbuf_mtod(ut_params->ibuf, uint8_t *), 0, + rte_pktmbuf_tailroom(ut_params->ibuf)); + + plaintext_len = ceil_byte_length(tdata->plaintext.len); + /* Append data which is padded to a multiple of */ + /* the algorithms block size */ + plaintext_pad_len = RTE_ALIGN_CEIL(plaintext_len, 16); + plaintext = (uint8_t *)rte_pktmbuf_append(ut_params->ibuf, + plaintext_pad_len); + memcpy(plaintext, tdata->plaintext.data, plaintext_len); + + TEST_HEXDUMP(stdout, "plaintext:", plaintext, plaintext_len); + + /* Create SNOW 3G operation */ + retval = create_wireless_algo_cipher_operation(tdata->cipher_iv.data, + tdata->cipher_iv.len, + tdata->validCipherLenInBits.len, + 0); + if (retval < 0) + return retval; + + ut_params->op = process_crypto_request(ts_params->valid_devs[0], + ut_params->op); + TEST_ASSERT_NOT_NULL(ut_params->op, "failed to retrieve obuf"); + + ut_params->obuf = ut_params->op->sym->m_dst; + if (ut_params->obuf) + ciphertext = rte_pktmbuf_mtod(ut_params->obuf, uint8_t *); + else + ciphertext = plaintext; + + TEST_HEXDUMP(stdout, "ciphertext:", ciphertext, plaintext_len); + + /* Validate obuf */ + TEST_ASSERT_BUFFERS_ARE_EQUAL_BIT( + ciphertext, + tdata->ciphertext.data, + tdata->validDataLenInBits.len, + "SNOW 3G Ciphertext data not as expected"); + return 0; +} + + +static int +test_snow3g_encryption_oop(const struct snow3g_test_data *tdata) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + uint8_t *plaintext, *ciphertext; + + int retval; + unsigned plaintext_pad_len; + unsigned plaintext_len; + + /* Create SNOW 3G session */ + retval = create_wireless_algo_cipher_session(ts_params->valid_devs[0], + RTE_CRYPTO_CIPHER_OP_ENCRYPT, + RTE_CRYPTO_CIPHER_SNOW3G_UEA2, + tdata->key.data, tdata->key.len, + tdata->cipher_iv.len); + if (retval < 0) + return retval; + + ut_params->ibuf = rte_pktmbuf_alloc(ts_params->mbuf_pool); + ut_params->obuf = rte_pktmbuf_alloc(ts_params->mbuf_pool); + + TEST_ASSERT_NOT_NULL(ut_params->ibuf, + "Failed to allocate input buffer in mempool"); + TEST_ASSERT_NOT_NULL(ut_params->obuf, + "Failed to allocate output buffer in mempool"); + + /* Clear mbuf payload */ + memset(rte_pktmbuf_mtod(ut_params->ibuf, uint8_t *), 0, + rte_pktmbuf_tailroom(ut_params->ibuf)); + + plaintext_len = ceil_byte_length(tdata->plaintext.len); + /* Append data which is padded to a multiple of */ + /* the algorithms block size */ + plaintext_pad_len = RTE_ALIGN_CEIL(plaintext_len, 16); + plaintext = (uint8_t *)rte_pktmbuf_append(ut_params->ibuf, + plaintext_pad_len); + rte_pktmbuf_append(ut_params->obuf, plaintext_pad_len); + memcpy(plaintext, tdata->plaintext.data, plaintext_len); + + TEST_HEXDUMP(stdout, "plaintext:", plaintext, plaintext_len); + + /* Create SNOW 3G operation */ + retval = create_wireless_algo_cipher_operation_oop(tdata->cipher_iv.data, + tdata->cipher_iv.len, + tdata->validCipherLenInBits.len, + 0); + if (retval < 0) + return retval; + + ut_params->op = process_crypto_request(ts_params->valid_devs[0], + ut_params->op); + TEST_ASSERT_NOT_NULL(ut_params->op, "failed to retrieve obuf"); + + ut_params->obuf = ut_params->op->sym->m_dst; + if (ut_params->obuf) + ciphertext = rte_pktmbuf_mtod(ut_params->obuf, uint8_t *); + else + ciphertext = plaintext; + + TEST_HEXDUMP(stdout, "ciphertext:", ciphertext, plaintext_len); + + /* Validate obuf */ + TEST_ASSERT_BUFFERS_ARE_EQUAL_BIT( + ciphertext, + tdata->ciphertext.data, + tdata->validDataLenInBits.len, + "SNOW 3G Ciphertext data not as expected"); + return 0; +} + +static int +test_snow3g_encryption_oop_sgl(const struct snow3g_test_data *tdata) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + + int retval; + unsigned int plaintext_pad_len; + unsigned int plaintext_len; + uint8_t buffer[10000]; + const uint8_t *ciphertext; + + struct rte_cryptodev_info dev_info; + + rte_cryptodev_info_get(ts_params->valid_devs[0], &dev_info); + if (!(dev_info.feature_flags & RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER)) { + printf("Device doesn't support scatter-gather. " + "Test Skipped.\n"); + return 0; + } + + /* Create SNOW 3G session */ + retval = create_wireless_algo_cipher_session(ts_params->valid_devs[0], + RTE_CRYPTO_CIPHER_OP_ENCRYPT, + RTE_CRYPTO_CIPHER_SNOW3G_UEA2, + tdata->key.data, tdata->key.len, + tdata->cipher_iv.len); + if (retval < 0) + return retval; + + plaintext_len = ceil_byte_length(tdata->plaintext.len); + /* Append data which is padded to a multiple of */ + /* the algorithms block size */ + plaintext_pad_len = RTE_ALIGN_CEIL(plaintext_len, 16); + + ut_params->ibuf = create_segmented_mbuf(ts_params->mbuf_pool, + plaintext_pad_len, 10, 0); + ut_params->obuf = create_segmented_mbuf(ts_params->mbuf_pool, + plaintext_pad_len, 3, 0); + + TEST_ASSERT_NOT_NULL(ut_params->ibuf, + "Failed to allocate input buffer in mempool"); + TEST_ASSERT_NOT_NULL(ut_params->obuf, + "Failed to allocate output buffer in mempool"); + + pktmbuf_write(ut_params->ibuf, 0, plaintext_len, tdata->plaintext.data); + + /* Create SNOW 3G operation */ + retval = create_wireless_algo_cipher_operation_oop(tdata->cipher_iv.data, + tdata->cipher_iv.len, + tdata->validCipherLenInBits.len, + 0); + if (retval < 0) + return retval; + + ut_params->op = process_crypto_request(ts_params->valid_devs[0], + ut_params->op); + TEST_ASSERT_NOT_NULL(ut_params->op, "failed to retrieve obuf"); + + ut_params->obuf = ut_params->op->sym->m_dst; + if (ut_params->obuf) + ciphertext = rte_pktmbuf_read(ut_params->obuf, 0, + plaintext_len, buffer); + else + ciphertext = rte_pktmbuf_read(ut_params->ibuf, 0, + plaintext_len, buffer); + + TEST_HEXDUMP(stdout, "ciphertext:", ciphertext, plaintext_len); + + /* Validate obuf */ + TEST_ASSERT_BUFFERS_ARE_EQUAL_BIT( + ciphertext, + tdata->ciphertext.data, + tdata->validDataLenInBits.len, + "SNOW 3G Ciphertext data not as expected"); + + return 0; +} + +/* Shift right a buffer by "offset" bits, "offset" < 8 */ +static void +buffer_shift_right(uint8_t *buffer, uint32_t length, uint8_t offset) +{ + uint8_t curr_byte, prev_byte; + uint32_t length_in_bytes = ceil_byte_length(length + offset); + uint8_t lower_byte_mask = (1 << offset) - 1; + unsigned i; + + prev_byte = buffer[0]; + buffer[0] >>= offset; + + for (i = 1; i < length_in_bytes; i++) { + curr_byte = buffer[i]; + buffer[i] = ((prev_byte & lower_byte_mask) << (8 - offset)) | + (curr_byte >> offset); + prev_byte = curr_byte; + } +} + +static int +test_snow3g_encryption_offset_oop(const struct snow3g_test_data *tdata) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + uint8_t *plaintext, *ciphertext; + int retval; + uint32_t plaintext_len; + uint32_t plaintext_pad_len; + uint8_t extra_offset = 4; + uint8_t *expected_ciphertext_shifted; + + /* Create SNOW 3G session */ + retval = create_wireless_algo_cipher_session(ts_params->valid_devs[0], + RTE_CRYPTO_CIPHER_OP_ENCRYPT, + RTE_CRYPTO_CIPHER_SNOW3G_UEA2, + tdata->key.data, tdata->key.len, + tdata->cipher_iv.len); + if (retval < 0) + return retval; + + ut_params->ibuf = rte_pktmbuf_alloc(ts_params->mbuf_pool); + ut_params->obuf = rte_pktmbuf_alloc(ts_params->mbuf_pool); + + TEST_ASSERT_NOT_NULL(ut_params->ibuf, + "Failed to allocate input buffer in mempool"); + TEST_ASSERT_NOT_NULL(ut_params->obuf, + "Failed to allocate output buffer in mempool"); + + /* Clear mbuf payload */ + memset(rte_pktmbuf_mtod(ut_params->ibuf, uint8_t *), 0, + rte_pktmbuf_tailroom(ut_params->ibuf)); + + plaintext_len = ceil_byte_length(tdata->plaintext.len + extra_offset); + /* + * Append data which is padded to a + * multiple of the algorithms block size + */ + plaintext_pad_len = RTE_ALIGN_CEIL(plaintext_len, 16); + + plaintext = (uint8_t *) rte_pktmbuf_append(ut_params->ibuf, + plaintext_pad_len); + + rte_pktmbuf_append(ut_params->obuf, plaintext_pad_len); + + memcpy(plaintext, tdata->plaintext.data, (tdata->plaintext.len >> 3)); + buffer_shift_right(plaintext, tdata->plaintext.len, extra_offset); + +#ifdef RTE_APP_TEST_DEBUG + rte_hexdump(stdout, "plaintext:", plaintext, tdata->plaintext.len); +#endif + /* Create SNOW 3G operation */ + retval = create_wireless_algo_cipher_operation_oop(tdata->cipher_iv.data, + tdata->cipher_iv.len, + tdata->validCipherLenInBits.len, + extra_offset); + if (retval < 0) + return retval; + + ut_params->op = process_crypto_request(ts_params->valid_devs[0], + ut_params->op); + TEST_ASSERT_NOT_NULL(ut_params->op, "failed to retrieve obuf"); + + ut_params->obuf = ut_params->op->sym->m_dst; + if (ut_params->obuf) + ciphertext = rte_pktmbuf_mtod(ut_params->obuf, uint8_t *); + else + ciphertext = plaintext; + +#ifdef RTE_APP_TEST_DEBUG + rte_hexdump(stdout, "ciphertext:", ciphertext, plaintext_len); +#endif + + expected_ciphertext_shifted = rte_malloc(NULL, plaintext_len, 8); + + TEST_ASSERT_NOT_NULL(expected_ciphertext_shifted, + "failed to reserve memory for ciphertext shifted\n"); + + memcpy(expected_ciphertext_shifted, tdata->ciphertext.data, + ceil_byte_length(tdata->ciphertext.len)); + buffer_shift_right(expected_ciphertext_shifted, tdata->ciphertext.len, + extra_offset); + /* Validate obuf */ + TEST_ASSERT_BUFFERS_ARE_EQUAL_BIT_OFFSET( + ciphertext, + expected_ciphertext_shifted, + tdata->validDataLenInBits.len, + extra_offset, + "SNOW 3G Ciphertext data not as expected"); + return 0; +} + +static int test_snow3g_decryption(const struct snow3g_test_data *tdata) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + + int retval; + + uint8_t *plaintext, *ciphertext; + unsigned ciphertext_pad_len; + unsigned ciphertext_len; + + /* Create SNOW 3G session */ + retval = create_wireless_algo_cipher_session(ts_params->valid_devs[0], + RTE_CRYPTO_CIPHER_OP_DECRYPT, + RTE_CRYPTO_CIPHER_SNOW3G_UEA2, + tdata->key.data, tdata->key.len, + tdata->cipher_iv.len); + if (retval < 0) + return retval; + + ut_params->ibuf = rte_pktmbuf_alloc(ts_params->mbuf_pool); + + /* Clear mbuf payload */ + memset(rte_pktmbuf_mtod(ut_params->ibuf, uint8_t *), 0, + rte_pktmbuf_tailroom(ut_params->ibuf)); + + ciphertext_len = ceil_byte_length(tdata->ciphertext.len); + /* Append data which is padded to a multiple of */ + /* the algorithms block size */ + ciphertext_pad_len = RTE_ALIGN_CEIL(ciphertext_len, 16); + ciphertext = (uint8_t *)rte_pktmbuf_append(ut_params->ibuf, + ciphertext_pad_len); + memcpy(ciphertext, tdata->ciphertext.data, ciphertext_len); + + TEST_HEXDUMP(stdout, "ciphertext:", ciphertext, ciphertext_len); + + /* Create SNOW 3G operation */ + retval = create_wireless_algo_cipher_operation(tdata->cipher_iv.data, + tdata->cipher_iv.len, + tdata->validCipherLenInBits.len, + 0); + if (retval < 0) + return retval; + + ut_params->op = process_crypto_request(ts_params->valid_devs[0], + ut_params->op); + TEST_ASSERT_NOT_NULL(ut_params->op, "failed to retrieve obuf"); + ut_params->obuf = ut_params->op->sym->m_dst; + if (ut_params->obuf) + plaintext = rte_pktmbuf_mtod(ut_params->obuf, uint8_t *); + else + plaintext = ciphertext; + + TEST_HEXDUMP(stdout, "plaintext:", plaintext, ciphertext_len); + + /* Validate obuf */ + TEST_ASSERT_BUFFERS_ARE_EQUAL_BIT(plaintext, + tdata->plaintext.data, + tdata->validDataLenInBits.len, + "SNOW 3G Plaintext data not as expected"); + return 0; +} + +static int test_snow3g_decryption_oop(const struct snow3g_test_data *tdata) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + + int retval; + + uint8_t *plaintext, *ciphertext; + unsigned ciphertext_pad_len; + unsigned ciphertext_len; + + /* Create SNOW 3G session */ + retval = create_wireless_algo_cipher_session(ts_params->valid_devs[0], + RTE_CRYPTO_CIPHER_OP_DECRYPT, + RTE_CRYPTO_CIPHER_SNOW3G_UEA2, + tdata->key.data, tdata->key.len, + tdata->cipher_iv.len); + if (retval < 0) + return retval; + + ut_params->ibuf = rte_pktmbuf_alloc(ts_params->mbuf_pool); + ut_params->obuf = rte_pktmbuf_alloc(ts_params->mbuf_pool); + + TEST_ASSERT_NOT_NULL(ut_params->ibuf, + "Failed to allocate input buffer"); + TEST_ASSERT_NOT_NULL(ut_params->obuf, + "Failed to allocate output buffer"); + + /* Clear mbuf payload */ + memset(rte_pktmbuf_mtod(ut_params->ibuf, uint8_t *), 0, + rte_pktmbuf_tailroom(ut_params->ibuf)); + + memset(rte_pktmbuf_mtod(ut_params->obuf, uint8_t *), 0, + rte_pktmbuf_tailroom(ut_params->obuf)); + + ciphertext_len = ceil_byte_length(tdata->ciphertext.len); + /* Append data which is padded to a multiple of */ + /* the algorithms block size */ + ciphertext_pad_len = RTE_ALIGN_CEIL(ciphertext_len, 16); + ciphertext = (uint8_t *)rte_pktmbuf_append(ut_params->ibuf, + ciphertext_pad_len); + rte_pktmbuf_append(ut_params->obuf, ciphertext_pad_len); + memcpy(ciphertext, tdata->ciphertext.data, ciphertext_len); + + TEST_HEXDUMP(stdout, "ciphertext:", ciphertext, ciphertext_len); + + /* Create SNOW 3G operation */ + retval = create_wireless_algo_cipher_operation_oop(tdata->cipher_iv.data, + tdata->cipher_iv.len, + tdata->validCipherLenInBits.len, + 0); + if (retval < 0) + return retval; + + ut_params->op = process_crypto_request(ts_params->valid_devs[0], + ut_params->op); + TEST_ASSERT_NOT_NULL(ut_params->op, "failed to retrieve obuf"); + ut_params->obuf = ut_params->op->sym->m_dst; + if (ut_params->obuf) + plaintext = rte_pktmbuf_mtod(ut_params->obuf, uint8_t *); + else + plaintext = ciphertext; + + TEST_HEXDUMP(stdout, "plaintext:", plaintext, ciphertext_len); + + /* Validate obuf */ + TEST_ASSERT_BUFFERS_ARE_EQUAL_BIT(plaintext, + tdata->plaintext.data, + tdata->validDataLenInBits.len, + "SNOW 3G Plaintext data not as expected"); + return 0; +} + +static int +test_zuc_cipher_auth(const struct wireless_test_data *tdata) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + + int retval; + + uint8_t *plaintext, *ciphertext; + unsigned int plaintext_pad_len; + unsigned int plaintext_len; + + struct rte_cryptodev_sym_capability_idx cap_idx; + + /* Check if device supports ZUC EEA3 */ + cap_idx.type = RTE_CRYPTO_SYM_XFORM_CIPHER; + cap_idx.algo.cipher = RTE_CRYPTO_CIPHER_ZUC_EEA3; + + if (rte_cryptodev_sym_capability_get(ts_params->valid_devs[0], + &cap_idx) == NULL) + return -ENOTSUP; + + /* Check if device supports ZUC EIA3 */ + cap_idx.type = RTE_CRYPTO_SYM_XFORM_AUTH; + cap_idx.algo.auth = RTE_CRYPTO_AUTH_ZUC_EIA3; + + if (rte_cryptodev_sym_capability_get(ts_params->valid_devs[0], + &cap_idx) == NULL) + return -ENOTSUP; + + /* Create ZUC session */ + retval = create_zuc_cipher_auth_encrypt_generate_session( + ts_params->valid_devs[0], + tdata); + if (retval < 0) + return retval; + ut_params->ibuf = rte_pktmbuf_alloc(ts_params->mbuf_pool); + + /* clear mbuf payload */ + memset(rte_pktmbuf_mtod(ut_params->ibuf, uint8_t *), 0, + rte_pktmbuf_tailroom(ut_params->ibuf)); + + plaintext_len = ceil_byte_length(tdata->plaintext.len); + /* Append data which is padded to a multiple of */ + /* the algorithms block size */ + plaintext_pad_len = RTE_ALIGN_CEIL(plaintext_len, 16); + plaintext = (uint8_t *)rte_pktmbuf_append(ut_params->ibuf, + plaintext_pad_len); + memcpy(plaintext, tdata->plaintext.data, plaintext_len); + + TEST_HEXDUMP(stdout, "plaintext:", plaintext, plaintext_len); + + /* Create ZUC operation */ + retval = create_zuc_cipher_hash_generate_operation(tdata); + if (retval < 0) + return retval; + + ut_params->op = process_crypto_request(ts_params->valid_devs[0], + ut_params->op); + TEST_ASSERT_NOT_NULL(ut_params->op, "failed to retrieve obuf"); + ut_params->obuf = ut_params->op->sym->m_src; + if (ut_params->obuf) + ciphertext = rte_pktmbuf_mtod(ut_params->obuf, uint8_t *); + else + ciphertext = plaintext; + + TEST_HEXDUMP(stdout, "ciphertext:", ciphertext, plaintext_len); + /* Validate obuf */ + TEST_ASSERT_BUFFERS_ARE_EQUAL_BIT( + ciphertext, + tdata->ciphertext.data, + tdata->validDataLenInBits.len, + "ZUC Ciphertext data not as expected"); + + ut_params->digest = rte_pktmbuf_mtod(ut_params->obuf, uint8_t *) + + plaintext_pad_len; + + /* Validate obuf */ + TEST_ASSERT_BUFFERS_ARE_EQUAL( + ut_params->digest, + tdata->digest.data, + 4, + "ZUC Generated auth tag not as expected"); + return 0; +} + +static int +test_snow3g_cipher_auth(const struct snow3g_test_data *tdata) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + + int retval; + + uint8_t *plaintext, *ciphertext; + unsigned plaintext_pad_len; + unsigned plaintext_len; + + /* Create SNOW 3G session */ + retval = create_wireless_algo_cipher_auth_session(ts_params->valid_devs[0], + RTE_CRYPTO_CIPHER_OP_ENCRYPT, + RTE_CRYPTO_AUTH_OP_GENERATE, + RTE_CRYPTO_AUTH_SNOW3G_UIA2, + RTE_CRYPTO_CIPHER_SNOW3G_UEA2, + tdata->key.data, tdata->key.len, + tdata->auth_iv.len, tdata->digest.len, + tdata->cipher_iv.len); + if (retval < 0) + return retval; + ut_params->ibuf = rte_pktmbuf_alloc(ts_params->mbuf_pool); + + /* clear mbuf payload */ + memset(rte_pktmbuf_mtod(ut_params->ibuf, uint8_t *), 0, + rte_pktmbuf_tailroom(ut_params->ibuf)); + + plaintext_len = ceil_byte_length(tdata->plaintext.len); + /* Append data which is padded to a multiple of */ + /* the algorithms block size */ + plaintext_pad_len = RTE_ALIGN_CEIL(plaintext_len, 16); + plaintext = (uint8_t *)rte_pktmbuf_append(ut_params->ibuf, + plaintext_pad_len); + memcpy(plaintext, tdata->plaintext.data, plaintext_len); + + TEST_HEXDUMP(stdout, "plaintext:", plaintext, plaintext_len); + + /* Create SNOW 3G operation */ + retval = create_wireless_algo_cipher_hash_operation(tdata->digest.data, + tdata->digest.len, tdata->auth_iv.data, + tdata->auth_iv.len, + plaintext_pad_len, RTE_CRYPTO_AUTH_OP_GENERATE, + tdata->cipher_iv.data, tdata->cipher_iv.len, + tdata->validCipherLenInBits.len, + 0, + tdata->validAuthLenInBits.len, + 0 + ); + if (retval < 0) + return retval; + + ut_params->op = process_crypto_request(ts_params->valid_devs[0], + ut_params->op); + TEST_ASSERT_NOT_NULL(ut_params->op, "failed to retrieve obuf"); + ut_params->obuf = ut_params->op->sym->m_src; + if (ut_params->obuf) + ciphertext = rte_pktmbuf_mtod(ut_params->obuf, uint8_t *); + else + ciphertext = plaintext; + + TEST_HEXDUMP(stdout, "ciphertext:", ciphertext, plaintext_len); + /* Validate obuf */ + TEST_ASSERT_BUFFERS_ARE_EQUAL_BIT( + ciphertext, + tdata->ciphertext.data, + tdata->validDataLenInBits.len, + "SNOW 3G Ciphertext data not as expected"); + + ut_params->digest = rte_pktmbuf_mtod(ut_params->obuf, uint8_t *) + + plaintext_pad_len; + + /* Validate obuf */ + TEST_ASSERT_BUFFERS_ARE_EQUAL( + ut_params->digest, + tdata->digest.data, + DIGEST_BYTE_LENGTH_SNOW3G_UIA2, + "SNOW 3G Generated auth tag not as expected"); + return 0; +} +static int +test_snow3g_auth_cipher(const struct snow3g_test_data *tdata) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + + int retval; + + uint8_t *plaintext, *ciphertext; + unsigned plaintext_pad_len; + unsigned plaintext_len; + + /* Create SNOW 3G session */ + retval = create_wireless_algo_auth_cipher_session(ts_params->valid_devs[0], + RTE_CRYPTO_CIPHER_OP_ENCRYPT, + RTE_CRYPTO_AUTH_OP_GENERATE, + RTE_CRYPTO_AUTH_SNOW3G_UIA2, + RTE_CRYPTO_CIPHER_SNOW3G_UEA2, + tdata->key.data, tdata->key.len, + tdata->auth_iv.len, tdata->digest.len, + tdata->cipher_iv.len); + if (retval < 0) + return retval; + + ut_params->ibuf = rte_pktmbuf_alloc(ts_params->mbuf_pool); + + /* clear mbuf payload */ + memset(rte_pktmbuf_mtod(ut_params->ibuf, uint8_t *), 0, + rte_pktmbuf_tailroom(ut_params->ibuf)); + + plaintext_len = ceil_byte_length(tdata->plaintext.len); + /* Append data which is padded to a multiple of */ + /* the algorithms block size */ + plaintext_pad_len = RTE_ALIGN_CEIL(plaintext_len, 16); + plaintext = (uint8_t *)rte_pktmbuf_append(ut_params->ibuf, + plaintext_pad_len); + memcpy(plaintext, tdata->plaintext.data, plaintext_len); + + TEST_HEXDUMP(stdout, "plaintext:", plaintext, plaintext_len); + + /* Create SNOW 3G operation */ + retval = create_wireless_algo_auth_cipher_operation( + tdata->digest.len, + tdata->cipher_iv.data, tdata->cipher_iv.len, + tdata->auth_iv.data, tdata->auth_iv.len, + plaintext_pad_len, + tdata->validCipherLenInBits.len, + 0, + tdata->validAuthLenInBits.len, + 0); + + if (retval < 0) + return retval; + + ut_params->op = process_crypto_request(ts_params->valid_devs[0], + ut_params->op); + TEST_ASSERT_NOT_NULL(ut_params->op, "failed to retrieve obuf"); + ut_params->obuf = ut_params->op->sym->m_src; + if (ut_params->obuf) + ciphertext = rte_pktmbuf_mtod(ut_params->obuf, uint8_t *); + else + ciphertext = plaintext; + + ut_params->digest = rte_pktmbuf_mtod(ut_params->obuf, uint8_t *) + + plaintext_pad_len; + TEST_HEXDUMP(stdout, "ciphertext:", ciphertext, plaintext_len); + + /* Validate obuf */ + TEST_ASSERT_BUFFERS_ARE_EQUAL_BIT( + ciphertext, + tdata->ciphertext.data, + tdata->validDataLenInBits.len, + "SNOW 3G Ciphertext data not as expected"); + + /* Validate obuf */ + TEST_ASSERT_BUFFERS_ARE_EQUAL( + ut_params->digest, + tdata->digest.data, + DIGEST_BYTE_LENGTH_SNOW3G_UIA2, + "SNOW 3G Generated auth tag not as expected"); + return 0; +} + +static int +test_kasumi_auth_cipher(const struct kasumi_test_data *tdata) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + + int retval; + + uint8_t *plaintext, *ciphertext; + unsigned plaintext_pad_len; + unsigned plaintext_len; + + /* Create KASUMI session */ + retval = create_wireless_algo_auth_cipher_session( + ts_params->valid_devs[0], + RTE_CRYPTO_CIPHER_OP_ENCRYPT, + RTE_CRYPTO_AUTH_OP_GENERATE, + RTE_CRYPTO_AUTH_KASUMI_F9, + RTE_CRYPTO_CIPHER_KASUMI_F8, + tdata->key.data, tdata->key.len, + 0, tdata->digest.len, + tdata->cipher_iv.len); + if (retval < 0) + return retval; + ut_params->ibuf = rte_pktmbuf_alloc(ts_params->mbuf_pool); + + /* clear mbuf payload */ + memset(rte_pktmbuf_mtod(ut_params->ibuf, uint8_t *), 0, + rte_pktmbuf_tailroom(ut_params->ibuf)); + + plaintext_len = ceil_byte_length(tdata->plaintext.len); + /* Append data which is padded to a multiple of */ + /* the algorithms block size */ + plaintext_pad_len = RTE_ALIGN_CEIL(plaintext_len, 16); + plaintext = (uint8_t *)rte_pktmbuf_append(ut_params->ibuf, + plaintext_pad_len); + memcpy(plaintext, tdata->plaintext.data, plaintext_len); + + TEST_HEXDUMP(stdout, "plaintext:", plaintext, plaintext_len); + + /* Create KASUMI operation */ + retval = create_wireless_algo_auth_cipher_operation(tdata->digest.len, + tdata->cipher_iv.data, tdata->cipher_iv.len, + NULL, 0, + plaintext_pad_len, + tdata->validCipherLenInBits.len, + tdata->validCipherOffsetInBits.len, + tdata->validAuthLenInBits.len, + 0 + ); + + if (retval < 0) + return retval; + + ut_params->op = process_crypto_request(ts_params->valid_devs[0], + ut_params->op); + TEST_ASSERT_NOT_NULL(ut_params->op, "failed to retrieve obuf"); + if (ut_params->op->sym->m_dst) + ut_params->obuf = ut_params->op->sym->m_dst; + else + ut_params->obuf = ut_params->op->sym->m_src; + + ciphertext = rte_pktmbuf_mtod_offset(ut_params->obuf, uint8_t *, + tdata->validCipherOffsetInBits.len >> 3); + + const uint8_t *reference_ciphertext = tdata->ciphertext.data + + (tdata->validCipherOffsetInBits.len >> 3); + /* Validate obuf */ + TEST_ASSERT_BUFFERS_ARE_EQUAL_BIT( + ciphertext, + reference_ciphertext, + tdata->validCipherLenInBits.len, + "KASUMI Ciphertext data not as expected"); + ut_params->digest = rte_pktmbuf_mtod(ut_params->ibuf, uint8_t *) + + plaintext_pad_len; + + /* Validate obuf */ + TEST_ASSERT_BUFFERS_ARE_EQUAL( + ut_params->digest, + tdata->digest.data, + DIGEST_BYTE_LENGTH_KASUMI_F9, + "KASUMI Generated auth tag not as expected"); + return 0; +} + +static int +test_kasumi_cipher_auth(const struct kasumi_test_data *tdata) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + + int retval; + + uint8_t *plaintext, *ciphertext; + unsigned plaintext_pad_len; + unsigned plaintext_len; + + /* Create KASUMI session */ + retval = create_wireless_algo_cipher_auth_session( + ts_params->valid_devs[0], + RTE_CRYPTO_CIPHER_OP_ENCRYPT, + RTE_CRYPTO_AUTH_OP_GENERATE, + RTE_CRYPTO_AUTH_KASUMI_F9, + RTE_CRYPTO_CIPHER_KASUMI_F8, + tdata->key.data, tdata->key.len, + 0, tdata->digest.len, + tdata->cipher_iv.len); + if (retval < 0) + return retval; + + ut_params->ibuf = rte_pktmbuf_alloc(ts_params->mbuf_pool); + + /* clear mbuf payload */ + memset(rte_pktmbuf_mtod(ut_params->ibuf, uint8_t *), 0, + rte_pktmbuf_tailroom(ut_params->ibuf)); + + plaintext_len = ceil_byte_length(tdata->plaintext.len); + /* Append data which is padded to a multiple of */ + /* the algorithms block size */ + plaintext_pad_len = RTE_ALIGN_CEIL(plaintext_len, 16); + plaintext = (uint8_t *)rte_pktmbuf_append(ut_params->ibuf, + plaintext_pad_len); + memcpy(plaintext, tdata->plaintext.data, plaintext_len); + + TEST_HEXDUMP(stdout, "plaintext:", plaintext, plaintext_len); + + /* Create KASUMI operation */ + retval = create_wireless_algo_cipher_hash_operation(tdata->digest.data, + tdata->digest.len, NULL, 0, + plaintext_pad_len, RTE_CRYPTO_AUTH_OP_GENERATE, + tdata->cipher_iv.data, tdata->cipher_iv.len, + RTE_ALIGN_CEIL(tdata->validCipherLenInBits.len, 8), + tdata->validCipherOffsetInBits.len, + tdata->validAuthLenInBits.len, + 0 + ); + if (retval < 0) + return retval; + + ut_params->op = process_crypto_request(ts_params->valid_devs[0], + ut_params->op); + TEST_ASSERT_NOT_NULL(ut_params->op, "failed to retrieve obuf"); + + if (ut_params->op->sym->m_dst) + ut_params->obuf = ut_params->op->sym->m_dst; + else + ut_params->obuf = ut_params->op->sym->m_src; + + ciphertext = rte_pktmbuf_mtod_offset(ut_params->obuf, uint8_t *, + tdata->validCipherOffsetInBits.len >> 3); + + ut_params->digest = rte_pktmbuf_mtod(ut_params->obuf, uint8_t *) + + plaintext_pad_len; + + const uint8_t *reference_ciphertext = tdata->ciphertext.data + + (tdata->validCipherOffsetInBits.len >> 3); + /* Validate obuf */ + TEST_ASSERT_BUFFERS_ARE_EQUAL_BIT( + ciphertext, + reference_ciphertext, + tdata->validCipherLenInBits.len, + "KASUMI Ciphertext data not as expected"); + + /* Validate obuf */ + TEST_ASSERT_BUFFERS_ARE_EQUAL( + ut_params->digest, + tdata->digest.data, + DIGEST_BYTE_LENGTH_SNOW3G_UIA2, + "KASUMI Generated auth tag not as expected"); + return 0; +} + +static int +test_zuc_encryption(const struct wireless_test_data *tdata) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + + int retval; + uint8_t *plaintext, *ciphertext; + unsigned plaintext_pad_len; + unsigned plaintext_len; + + struct rte_cryptodev_sym_capability_idx cap_idx; + + /* Check if device supports ZUC EEA3 */ + cap_idx.type = RTE_CRYPTO_SYM_XFORM_CIPHER; + cap_idx.algo.cipher = RTE_CRYPTO_CIPHER_ZUC_EEA3; + + if (rte_cryptodev_sym_capability_get(ts_params->valid_devs[0], + &cap_idx) == NULL) + return -ENOTSUP; + + /* Create ZUC session */ + retval = create_wireless_algo_cipher_session(ts_params->valid_devs[0], + RTE_CRYPTO_CIPHER_OP_ENCRYPT, + RTE_CRYPTO_CIPHER_ZUC_EEA3, + tdata->key.data, tdata->key.len, + tdata->cipher_iv.len); + if (retval < 0) + return retval; + + ut_params->ibuf = rte_pktmbuf_alloc(ts_params->mbuf_pool); + + /* Clear mbuf payload */ + memset(rte_pktmbuf_mtod(ut_params->ibuf, uint8_t *), 0, + rte_pktmbuf_tailroom(ut_params->ibuf)); + + plaintext_len = ceil_byte_length(tdata->plaintext.len); + /* Append data which is padded to a multiple */ + /* of the algorithms block size */ + plaintext_pad_len = RTE_ALIGN_CEIL(plaintext_len, 8); + plaintext = (uint8_t *)rte_pktmbuf_append(ut_params->ibuf, + plaintext_pad_len); + memcpy(plaintext, tdata->plaintext.data, plaintext_len); + + TEST_HEXDUMP(stdout, "plaintext:", plaintext, plaintext_len); + + /* Create ZUC operation */ + retval = create_wireless_algo_cipher_operation(tdata->cipher_iv.data, + tdata->cipher_iv.len, + tdata->plaintext.len, + 0); + if (retval < 0) + return retval; + + ut_params->op = process_crypto_request(ts_params->valid_devs[0], + ut_params->op); + TEST_ASSERT_NOT_NULL(ut_params->op, "failed to retrieve obuf"); + + ut_params->obuf = ut_params->op->sym->m_dst; + if (ut_params->obuf) + ciphertext = rte_pktmbuf_mtod(ut_params->obuf, uint8_t *); + else + ciphertext = plaintext; + + TEST_HEXDUMP(stdout, "ciphertext:", ciphertext, plaintext_len); + + /* Validate obuf */ + TEST_ASSERT_BUFFERS_ARE_EQUAL_BIT( + ciphertext, + tdata->ciphertext.data, + tdata->validCipherLenInBits.len, + "ZUC Ciphertext data not as expected"); + return 0; +} + +static int +test_zuc_encryption_sgl(const struct wireless_test_data *tdata) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + + int retval; + + unsigned int plaintext_pad_len; + unsigned int plaintext_len; + const uint8_t *ciphertext; + uint8_t ciphertext_buffer[2048]; + struct rte_cryptodev_info dev_info; + + struct rte_cryptodev_sym_capability_idx cap_idx; + + /* Check if device supports ZUC EEA3 */ + cap_idx.type = RTE_CRYPTO_SYM_XFORM_CIPHER; + cap_idx.algo.cipher = RTE_CRYPTO_CIPHER_ZUC_EEA3; + + if (rte_cryptodev_sym_capability_get(ts_params->valid_devs[0], + &cap_idx) == NULL) + return -ENOTSUP; + + rte_cryptodev_info_get(ts_params->valid_devs[0], &dev_info); + if (!(dev_info.feature_flags & RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER)) { + printf("Device doesn't support scatter-gather. " + "Test Skipped.\n"); + return -ENOTSUP; + } + + plaintext_len = ceil_byte_length(tdata->plaintext.len); + + /* Append data which is padded to a multiple */ + /* of the algorithms block size */ + plaintext_pad_len = RTE_ALIGN_CEIL(plaintext_len, 8); + + ut_params->ibuf = create_segmented_mbuf(ts_params->mbuf_pool, + plaintext_pad_len, 10, 0); + + pktmbuf_write(ut_params->ibuf, 0, plaintext_len, + tdata->plaintext.data); + + /* Create ZUC session */ + retval = create_wireless_algo_cipher_session(ts_params->valid_devs[0], + RTE_CRYPTO_CIPHER_OP_ENCRYPT, + RTE_CRYPTO_CIPHER_ZUC_EEA3, + tdata->key.data, tdata->key.len, + tdata->cipher_iv.len); + if (retval < 0) + return retval; + + /* Clear mbuf payload */ + + pktmbuf_write(ut_params->ibuf, 0, plaintext_len, tdata->plaintext.data); + + /* Create ZUC operation */ + retval = create_wireless_algo_cipher_operation(tdata->cipher_iv.data, + tdata->cipher_iv.len, tdata->plaintext.len, + 0); + if (retval < 0) + return retval; + + ut_params->op = process_crypto_request(ts_params->valid_devs[0], + ut_params->op); + TEST_ASSERT_NOT_NULL(ut_params->op, "failed to retrieve obuf"); + + ut_params->obuf = ut_params->op->sym->m_dst; + if (ut_params->obuf) + ciphertext = rte_pktmbuf_read(ut_params->obuf, + 0, plaintext_len, ciphertext_buffer); + else + ciphertext = rte_pktmbuf_read(ut_params->ibuf, + 0, plaintext_len, ciphertext_buffer); + + /* Validate obuf */ + TEST_HEXDUMP(stdout, "ciphertext:", ciphertext, plaintext_len); + + /* Validate obuf */ + TEST_ASSERT_BUFFERS_ARE_EQUAL_BIT( + ciphertext, + tdata->ciphertext.data, + tdata->validCipherLenInBits.len, + "ZUC Ciphertext data not as expected"); + + return 0; +} + +static int +test_zuc_authentication(const struct wireless_test_data *tdata) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + + int retval; + unsigned plaintext_pad_len; + unsigned plaintext_len; + uint8_t *plaintext; + + struct rte_cryptodev_sym_capability_idx cap_idx; + + /* Check if device supports ZUC EIA3 */ + cap_idx.type = RTE_CRYPTO_SYM_XFORM_AUTH; + cap_idx.algo.auth = RTE_CRYPTO_AUTH_ZUC_EIA3; + + if (rte_cryptodev_sym_capability_get(ts_params->valid_devs[0], + &cap_idx) == NULL) + return -ENOTSUP; + + /* Create ZUC session */ + retval = create_wireless_algo_hash_session(ts_params->valid_devs[0], + tdata->key.data, tdata->key.len, + tdata->auth_iv.len, tdata->digest.len, + RTE_CRYPTO_AUTH_OP_GENERATE, + RTE_CRYPTO_AUTH_ZUC_EIA3); + if (retval < 0) + return retval; + + /* alloc mbuf and set payload */ + ut_params->ibuf = rte_pktmbuf_alloc(ts_params->mbuf_pool); + + memset(rte_pktmbuf_mtod(ut_params->ibuf, uint8_t *), 0, + rte_pktmbuf_tailroom(ut_params->ibuf)); + + plaintext_len = ceil_byte_length(tdata->plaintext.len); + /* Append data which is padded to a multiple of */ + /* the algorithms block size */ + plaintext_pad_len = RTE_ALIGN_CEIL(plaintext_len, 8); + plaintext = (uint8_t *)rte_pktmbuf_append(ut_params->ibuf, + plaintext_pad_len); + memcpy(plaintext, tdata->plaintext.data, plaintext_len); + + /* Create ZUC operation */ + retval = create_wireless_algo_hash_operation(NULL, tdata->digest.len, + tdata->auth_iv.data, tdata->auth_iv.len, + plaintext_pad_len, RTE_CRYPTO_AUTH_OP_GENERATE, + tdata->validAuthLenInBits.len, + 0); + if (retval < 0) + return retval; + + ut_params->op = process_crypto_request(ts_params->valid_devs[0], + ut_params->op); + ut_params->obuf = ut_params->op->sym->m_src; + TEST_ASSERT_NOT_NULL(ut_params->op, "failed to retrieve obuf"); + ut_params->digest = rte_pktmbuf_mtod(ut_params->obuf, uint8_t *) + + plaintext_pad_len; + + /* Validate obuf */ + TEST_ASSERT_BUFFERS_ARE_EQUAL( + ut_params->digest, + tdata->digest.data, + DIGEST_BYTE_LENGTH_KASUMI_F9, + "ZUC Generated auth tag not as expected"); + + return 0; +} + +static int +test_kasumi_encryption_test_case_1(void) +{ + return test_kasumi_encryption(&kasumi_test_case_1); +} + +static int +test_kasumi_encryption_test_case_1_sgl(void) +{ + return test_kasumi_encryption_sgl(&kasumi_test_case_1); +} + +static int +test_kasumi_encryption_test_case_1_oop(void) +{ + return test_kasumi_encryption_oop(&kasumi_test_case_1); +} + +static int +test_kasumi_encryption_test_case_1_oop_sgl(void) +{ + return test_kasumi_encryption_oop_sgl(&kasumi_test_case_1); +} + +static int +test_kasumi_encryption_test_case_2(void) +{ + return test_kasumi_encryption(&kasumi_test_case_2); +} + +static int +test_kasumi_encryption_test_case_3(void) +{ + return test_kasumi_encryption(&kasumi_test_case_3); +} + +static int +test_kasumi_encryption_test_case_4(void) +{ + return test_kasumi_encryption(&kasumi_test_case_4); +} + +static int +test_kasumi_encryption_test_case_5(void) +{ + return test_kasumi_encryption(&kasumi_test_case_5); +} + +static int +test_kasumi_decryption_test_case_1(void) +{ + return test_kasumi_decryption(&kasumi_test_case_1); +} + +static int +test_kasumi_decryption_test_case_1_oop(void) +{ + return test_kasumi_decryption_oop(&kasumi_test_case_1); +} + +static int +test_kasumi_decryption_test_case_2(void) +{ + return test_kasumi_decryption(&kasumi_test_case_2); +} + +static int +test_kasumi_decryption_test_case_3(void) +{ + return test_kasumi_decryption(&kasumi_test_case_3); +} + +static int +test_kasumi_decryption_test_case_4(void) +{ + return test_kasumi_decryption(&kasumi_test_case_4); +} + +static int +test_kasumi_decryption_test_case_5(void) +{ + return test_kasumi_decryption(&kasumi_test_case_5); +} +static int +test_snow3g_encryption_test_case_1(void) +{ + return test_snow3g_encryption(&snow3g_test_case_1); +} + +static int +test_snow3g_encryption_test_case_1_oop(void) +{ + return test_snow3g_encryption_oop(&snow3g_test_case_1); +} + +static int +test_snow3g_encryption_test_case_1_oop_sgl(void) +{ + return test_snow3g_encryption_oop_sgl(&snow3g_test_case_1); +} + + +static int +test_snow3g_encryption_test_case_1_offset_oop(void) +{ + return test_snow3g_encryption_offset_oop(&snow3g_test_case_1); +} + +static int +test_snow3g_encryption_test_case_2(void) +{ + return test_snow3g_encryption(&snow3g_test_case_2); +} + +static int +test_snow3g_encryption_test_case_3(void) +{ + return test_snow3g_encryption(&snow3g_test_case_3); +} + +static int +test_snow3g_encryption_test_case_4(void) +{ + return test_snow3g_encryption(&snow3g_test_case_4); +} + +static int +test_snow3g_encryption_test_case_5(void) +{ + return test_snow3g_encryption(&snow3g_test_case_5); +} + +static int +test_snow3g_decryption_test_case_1(void) +{ + return test_snow3g_decryption(&snow3g_test_case_1); +} + +static int +test_snow3g_decryption_test_case_1_oop(void) +{ + return test_snow3g_decryption_oop(&snow3g_test_case_1); +} + +static int +test_snow3g_decryption_test_case_2(void) +{ + return test_snow3g_decryption(&snow3g_test_case_2); +} + +static int +test_snow3g_decryption_test_case_3(void) +{ + return test_snow3g_decryption(&snow3g_test_case_3); +} + +static int +test_snow3g_decryption_test_case_4(void) +{ + return test_snow3g_decryption(&snow3g_test_case_4); +} + +static int +test_snow3g_decryption_test_case_5(void) +{ + return test_snow3g_decryption(&snow3g_test_case_5); +} +static int +test_snow3g_cipher_auth_test_case_1(void) +{ + return test_snow3g_cipher_auth(&snow3g_test_case_3); +} + +static int +test_snow3g_auth_cipher_test_case_1(void) +{ + return test_snow3g_auth_cipher(&snow3g_test_case_6); +} + +static int +test_kasumi_auth_cipher_test_case_1(void) +{ + return test_kasumi_auth_cipher(&kasumi_test_case_3); +} + +static int +test_kasumi_cipher_auth_test_case_1(void) +{ + return test_kasumi_cipher_auth(&kasumi_test_case_6); +} + +static int +test_zuc_encryption_test_case_1(void) +{ + return test_zuc_encryption(&zuc_test_case_cipher_193b); +} + +static int +test_zuc_encryption_test_case_2(void) +{ + return test_zuc_encryption(&zuc_test_case_cipher_800b); +} + +static int +test_zuc_encryption_test_case_3(void) +{ + return test_zuc_encryption(&zuc_test_case_cipher_1570b); +} + +static int +test_zuc_encryption_test_case_4(void) +{ + return test_zuc_encryption(&zuc_test_case_cipher_2798b); +} + +static int +test_zuc_encryption_test_case_5(void) +{ + return test_zuc_encryption(&zuc_test_case_cipher_4019b); +} + +static int +test_zuc_encryption_test_case_6_sgl(void) +{ + return test_zuc_encryption_sgl(&zuc_test_case_cipher_193b); +} + +static int +test_zuc_hash_generate_test_case_1(void) +{ + return test_zuc_authentication(&zuc_test_case_auth_1b); +} + +static int +test_zuc_hash_generate_test_case_2(void) +{ + return test_zuc_authentication(&zuc_test_case_auth_90b); +} + +static int +test_zuc_hash_generate_test_case_3(void) +{ + return test_zuc_authentication(&zuc_test_case_auth_577b); +} + +static int +test_zuc_hash_generate_test_case_4(void) +{ + return test_zuc_authentication(&zuc_test_case_auth_2079b); +} + +static int +test_zuc_hash_generate_test_case_5(void) +{ + return test_zuc_authentication(&zuc_test_auth_5670b); +} + +static int +test_zuc_hash_generate_test_case_6(void) +{ + return test_zuc_authentication(&zuc_test_case_auth_128b); +} + +static int +test_zuc_hash_generate_test_case_7(void) +{ + return test_zuc_authentication(&zuc_test_case_auth_2080b); +} + +static int +test_zuc_hash_generate_test_case_8(void) +{ + return test_zuc_authentication(&zuc_test_case_auth_584b); +} + +static int +test_zuc_cipher_auth_test_case_1(void) +{ + return test_zuc_cipher_auth(&zuc_test_case_cipher_200b_auth_200b); +} + +static int +test_zuc_cipher_auth_test_case_2(void) +{ + return test_zuc_cipher_auth(&zuc_test_case_cipher_800b_auth_120b); +} + +static int +test_3DES_chain_qat_all(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + int status; + + status = test_blockcipher_all_tests(ts_params->mbuf_pool, + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], + rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_QAT_SYM_PMD)), + BLKCIPHER_3DES_CHAIN_TYPE); + + TEST_ASSERT_EQUAL(status, 0, "Test failed"); + + return TEST_SUCCESS; +} + +static int +test_DES_cipheronly_qat_all(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + int status; + + status = test_blockcipher_all_tests(ts_params->mbuf_pool, + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], + rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_QAT_SYM_PMD)), + BLKCIPHER_DES_CIPHERONLY_TYPE); + + TEST_ASSERT_EQUAL(status, 0, "Test failed"); + + return TEST_SUCCESS; +} + +static int +test_DES_cipheronly_openssl_all(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + int status; + + status = test_blockcipher_all_tests(ts_params->mbuf_pool, + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], + rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_OPENSSL_PMD)), + BLKCIPHER_DES_CIPHERONLY_TYPE); + + TEST_ASSERT_EQUAL(status, 0, "Test failed"); + + return TEST_SUCCESS; +} + +static int +test_DES_docsis_openssl_all(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + int status; + + status = test_blockcipher_all_tests(ts_params->mbuf_pool, + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], + rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_OPENSSL_PMD)), + BLKCIPHER_DES_DOCSIS_TYPE); + + TEST_ASSERT_EQUAL(status, 0, "Test failed"); + + return TEST_SUCCESS; +} + +static int +test_DES_cipheronly_mb_all(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + int status; + + status = test_blockcipher_all_tests(ts_params->mbuf_pool, + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], + rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_AESNI_MB_PMD)), + BLKCIPHER_DES_CIPHERONLY_TYPE); + + TEST_ASSERT_EQUAL(status, 0, "Test failed"); + + return TEST_SUCCESS; +} + +static int +test_DES_docsis_mb_all(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + int status; + + status = test_blockcipher_all_tests(ts_params->mbuf_pool, + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], + rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_AESNI_MB_PMD)), + BLKCIPHER_DES_DOCSIS_TYPE); + + TEST_ASSERT_EQUAL(status, 0, "Test failed"); + + return TEST_SUCCESS; +} + +static int +test_3DES_chain_dpaa_sec_all(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + int status; + + status = test_blockcipher_all_tests(ts_params->mbuf_pool, + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], + rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_DPAA_SEC_PMD)), + BLKCIPHER_3DES_CHAIN_TYPE); + + TEST_ASSERT_EQUAL(status, 0, "Test failed"); + + return TEST_SUCCESS; +} + +static int +test_3DES_cipheronly_dpaa_sec_all(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + int status; + + status = test_blockcipher_all_tests(ts_params->mbuf_pool, + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], + rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_DPAA_SEC_PMD)), + BLKCIPHER_3DES_CIPHERONLY_TYPE); + + TEST_ASSERT_EQUAL(status, 0, "Test failed"); + + return TEST_SUCCESS; +} + +static int +test_3DES_chain_dpaa2_sec_all(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + int status; + + status = test_blockcipher_all_tests(ts_params->mbuf_pool, + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], + rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_DPAA2_SEC_PMD)), + BLKCIPHER_3DES_CHAIN_TYPE); + + TEST_ASSERT_EQUAL(status, 0, "Test failed"); + + return TEST_SUCCESS; +} + +static int +test_3DES_cipheronly_dpaa2_sec_all(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + int status; + + status = test_blockcipher_all_tests(ts_params->mbuf_pool, + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], + rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_DPAA2_SEC_PMD)), + BLKCIPHER_3DES_CIPHERONLY_TYPE); + + TEST_ASSERT_EQUAL(status, 0, "Test failed"); + + return TEST_SUCCESS; +} + +static int +test_3DES_cipheronly_qat_all(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + int status; + + status = test_blockcipher_all_tests(ts_params->mbuf_pool, + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], + rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_QAT_SYM_PMD)), + BLKCIPHER_3DES_CIPHERONLY_TYPE); + + TEST_ASSERT_EQUAL(status, 0, "Test failed"); + + return TEST_SUCCESS; +} + +static int +test_3DES_chain_openssl_all(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + int status; + + status = test_blockcipher_all_tests(ts_params->mbuf_pool, + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], + rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_OPENSSL_PMD)), + BLKCIPHER_3DES_CHAIN_TYPE); + + TEST_ASSERT_EQUAL(status, 0, "Test failed"); + + return TEST_SUCCESS; +} + +static int +test_3DES_cipheronly_openssl_all(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + int status; + + status = test_blockcipher_all_tests(ts_params->mbuf_pool, + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], + rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_OPENSSL_PMD)), + BLKCIPHER_3DES_CIPHERONLY_TYPE); + + TEST_ASSERT_EQUAL(status, 0, "Test failed"); + + return TEST_SUCCESS; +} + +/* ***** AEAD algorithm Tests ***** */ + +static int +create_aead_session(uint8_t dev_id, enum rte_crypto_aead_algorithm algo, + enum rte_crypto_aead_operation op, + const uint8_t *key, const uint8_t key_len, + const uint16_t aad_len, const uint8_t auth_len, + uint8_t iv_len) +{ + uint8_t aead_key[key_len]; + + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + + memcpy(aead_key, key, key_len); + + /* Setup AEAD Parameters */ + ut_params->aead_xform.type = RTE_CRYPTO_SYM_XFORM_AEAD; + ut_params->aead_xform.next = NULL; + ut_params->aead_xform.aead.algo = algo; + ut_params->aead_xform.aead.op = op; + ut_params->aead_xform.aead.key.data = aead_key; + ut_params->aead_xform.aead.key.length = key_len; + ut_params->aead_xform.aead.iv.offset = IV_OFFSET; + ut_params->aead_xform.aead.iv.length = iv_len; + ut_params->aead_xform.aead.digest_length = auth_len; + ut_params->aead_xform.aead.aad_length = aad_len; + + TEST_HEXDUMP(stdout, "key:", key, key_len); + + /* Create Crypto session*/ + ut_params->sess = rte_cryptodev_sym_session_create( + ts_params->session_mpool); + + rte_cryptodev_sym_session_init(dev_id, ut_params->sess, + &ut_params->aead_xform, ts_params->session_mpool); + + TEST_ASSERT_NOT_NULL(ut_params->sess, "Session creation failed"); + + return 0; +} + +static int +create_aead_xform(struct rte_crypto_op *op, + enum rte_crypto_aead_algorithm algo, + enum rte_crypto_aead_operation aead_op, + uint8_t *key, const uint8_t key_len, + const uint8_t aad_len, const uint8_t auth_len, + uint8_t iv_len) +{ + TEST_ASSERT_NOT_NULL(rte_crypto_op_sym_xforms_alloc(op, 1), + "failed to allocate space for crypto transform"); + + struct rte_crypto_sym_op *sym_op = op->sym; + + /* Setup AEAD Parameters */ + sym_op->xform->type = RTE_CRYPTO_SYM_XFORM_AEAD; + sym_op->xform->next = NULL; + sym_op->xform->aead.algo = algo; + sym_op->xform->aead.op = aead_op; + sym_op->xform->aead.key.data = key; + sym_op->xform->aead.key.length = key_len; + sym_op->xform->aead.iv.offset = IV_OFFSET; + sym_op->xform->aead.iv.length = iv_len; + sym_op->xform->aead.digest_length = auth_len; + sym_op->xform->aead.aad_length = aad_len; + + TEST_HEXDUMP(stdout, "key:", key, key_len); + + return 0; +} + +static int +create_aead_operation(enum rte_crypto_aead_operation op, + const struct aead_test_data *tdata) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + + uint8_t *plaintext, *ciphertext; + unsigned int aad_pad_len, plaintext_pad_len; + + /* Generate Crypto op data structure */ + ut_params->op = rte_crypto_op_alloc(ts_params->op_mpool, + RTE_CRYPTO_OP_TYPE_SYMMETRIC); + TEST_ASSERT_NOT_NULL(ut_params->op, + "Failed to allocate symmetric crypto operation struct"); + + struct rte_crypto_sym_op *sym_op = ut_params->op->sym; + + /* Append aad data */ + if (tdata->algo == RTE_CRYPTO_AEAD_AES_CCM) { + aad_pad_len = RTE_ALIGN_CEIL(tdata->aad.len + 18, 16); + sym_op->aead.aad.data = (uint8_t *)rte_pktmbuf_append(ut_params->ibuf, + aad_pad_len); + TEST_ASSERT_NOT_NULL(sym_op->aead.aad.data, + "no room to append aad"); + + sym_op->aead.aad.phys_addr = + rte_pktmbuf_iova(ut_params->ibuf); + /* Copy AAD 18 bytes after the AAD pointer, according to the API */ + memcpy(sym_op->aead.aad.data + 18, tdata->aad.data, tdata->aad.len); + TEST_HEXDUMP(stdout, "aad:", sym_op->aead.aad.data, + tdata->aad.len); + + /* Append IV at the end of the crypto operation*/ + uint8_t *iv_ptr = rte_crypto_op_ctod_offset(ut_params->op, + uint8_t *, IV_OFFSET); + + /* Copy IV 1 byte after the IV pointer, according to the API */ + rte_memcpy(iv_ptr + 1, tdata->iv.data, tdata->iv.len); + TEST_HEXDUMP(stdout, "iv:", iv_ptr, + tdata->iv.len); + } else { + aad_pad_len = RTE_ALIGN_CEIL(tdata->aad.len, 16); + sym_op->aead.aad.data = (uint8_t *)rte_pktmbuf_append(ut_params->ibuf, + aad_pad_len); + TEST_ASSERT_NOT_NULL(sym_op->aead.aad.data, + "no room to append aad"); + + sym_op->aead.aad.phys_addr = + rte_pktmbuf_iova(ut_params->ibuf); + memcpy(sym_op->aead.aad.data, tdata->aad.data, tdata->aad.len); + TEST_HEXDUMP(stdout, "aad:", sym_op->aead.aad.data, + tdata->aad.len); + + /* Append IV at the end of the crypto operation*/ + uint8_t *iv_ptr = rte_crypto_op_ctod_offset(ut_params->op, + uint8_t *, IV_OFFSET); + + rte_memcpy(iv_ptr, tdata->iv.data, tdata->iv.len); + TEST_HEXDUMP(stdout, "iv:", iv_ptr, + tdata->iv.len); + } + + /* Append plaintext/ciphertext */ + if (op == RTE_CRYPTO_AEAD_OP_ENCRYPT) { + plaintext_pad_len = RTE_ALIGN_CEIL(tdata->plaintext.len, 16); + plaintext = (uint8_t *)rte_pktmbuf_append(ut_params->ibuf, + plaintext_pad_len); + TEST_ASSERT_NOT_NULL(plaintext, "no room to append plaintext"); + + memcpy(plaintext, tdata->plaintext.data, tdata->plaintext.len); + TEST_HEXDUMP(stdout, "plaintext:", plaintext, + tdata->plaintext.len); + + if (ut_params->obuf) { + ciphertext = (uint8_t *)rte_pktmbuf_append( + ut_params->obuf, + plaintext_pad_len + aad_pad_len); + TEST_ASSERT_NOT_NULL(ciphertext, + "no room to append ciphertext"); + + memset(ciphertext + aad_pad_len, 0, + tdata->ciphertext.len); + } + } else { + plaintext_pad_len = RTE_ALIGN_CEIL(tdata->ciphertext.len, 16); + ciphertext = (uint8_t *)rte_pktmbuf_append(ut_params->ibuf, + plaintext_pad_len); + TEST_ASSERT_NOT_NULL(ciphertext, + "no room to append ciphertext"); + + memcpy(ciphertext, tdata->ciphertext.data, + tdata->ciphertext.len); + TEST_HEXDUMP(stdout, "ciphertext:", ciphertext, + tdata->ciphertext.len); + + if (ut_params->obuf) { + plaintext = (uint8_t *)rte_pktmbuf_append( + ut_params->obuf, + plaintext_pad_len + aad_pad_len); + TEST_ASSERT_NOT_NULL(plaintext, + "no room to append plaintext"); + + memset(plaintext + aad_pad_len, 0, + tdata->plaintext.len); + } + } + + /* Append digest data */ + if (op == RTE_CRYPTO_AEAD_OP_ENCRYPT) { + sym_op->aead.digest.data = (uint8_t *)rte_pktmbuf_append( + ut_params->obuf ? ut_params->obuf : + ut_params->ibuf, + tdata->auth_tag.len); + TEST_ASSERT_NOT_NULL(sym_op->aead.digest.data, + "no room to append digest"); + memset(sym_op->aead.digest.data, 0, tdata->auth_tag.len); + sym_op->aead.digest.phys_addr = rte_pktmbuf_iova_offset( + ut_params->obuf ? ut_params->obuf : + ut_params->ibuf, + plaintext_pad_len + + aad_pad_len); + } else { + sym_op->aead.digest.data = (uint8_t *)rte_pktmbuf_append( + ut_params->ibuf, tdata->auth_tag.len); + TEST_ASSERT_NOT_NULL(sym_op->aead.digest.data, + "no room to append digest"); + sym_op->aead.digest.phys_addr = rte_pktmbuf_iova_offset( + ut_params->ibuf, + plaintext_pad_len + aad_pad_len); + + rte_memcpy(sym_op->aead.digest.data, tdata->auth_tag.data, + tdata->auth_tag.len); + TEST_HEXDUMP(stdout, "digest:", + sym_op->aead.digest.data, + tdata->auth_tag.len); + } + + sym_op->aead.data.length = tdata->plaintext.len; + sym_op->aead.data.offset = aad_pad_len; + + return 0; +} + +static int +test_authenticated_encryption(const struct aead_test_data *tdata) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + + int retval; + uint8_t *ciphertext, *auth_tag; + uint16_t plaintext_pad_len; + uint32_t i; + + /* Create AEAD session */ + retval = create_aead_session(ts_params->valid_devs[0], + tdata->algo, + RTE_CRYPTO_AEAD_OP_ENCRYPT, + tdata->key.data, tdata->key.len, + tdata->aad.len, tdata->auth_tag.len, + tdata->iv.len); + if (retval < 0) + return retval; + + if (tdata->aad.len > MBUF_SIZE) { + ut_params->ibuf = rte_pktmbuf_alloc(ts_params->large_mbuf_pool); + /* Populate full size of add data */ + for (i = 32; i < MAX_AAD_LENGTH; i += 32) + memcpy(&tdata->aad.data[i], &tdata->aad.data[0], 32); + } else + ut_params->ibuf = rte_pktmbuf_alloc(ts_params->mbuf_pool); + + /* clear mbuf payload */ + memset(rte_pktmbuf_mtod(ut_params->ibuf, uint8_t *), 0, + rte_pktmbuf_tailroom(ut_params->ibuf)); + + /* Create AEAD operation */ + retval = create_aead_operation(RTE_CRYPTO_AEAD_OP_ENCRYPT, tdata); + if (retval < 0) + return retval; + + rte_crypto_op_attach_sym_session(ut_params->op, ut_params->sess); + + ut_params->op->sym->m_src = ut_params->ibuf; + + /* Process crypto operation */ + TEST_ASSERT_NOT_NULL(process_crypto_request(ts_params->valid_devs[0], + ut_params->op), "failed to process sym crypto op"); + + TEST_ASSERT_EQUAL(ut_params->op->status, RTE_CRYPTO_OP_STATUS_SUCCESS, + "crypto op processing failed"); + + plaintext_pad_len = RTE_ALIGN_CEIL(tdata->plaintext.len, 16); + + if (ut_params->op->sym->m_dst) { + ciphertext = rte_pktmbuf_mtod(ut_params->op->sym->m_dst, + uint8_t *); + auth_tag = rte_pktmbuf_mtod_offset(ut_params->op->sym->m_dst, + uint8_t *, plaintext_pad_len); + } else { + ciphertext = rte_pktmbuf_mtod_offset(ut_params->op->sym->m_src, + uint8_t *, + ut_params->op->sym->cipher.data.offset); + auth_tag = ciphertext + plaintext_pad_len; + } + + TEST_HEXDUMP(stdout, "ciphertext:", ciphertext, tdata->ciphertext.len); + TEST_HEXDUMP(stdout, "auth tag:", auth_tag, tdata->auth_tag.len); + + /* Validate obuf */ + TEST_ASSERT_BUFFERS_ARE_EQUAL( + ciphertext, + tdata->ciphertext.data, + tdata->ciphertext.len, + "Ciphertext data not as expected"); + + TEST_ASSERT_BUFFERS_ARE_EQUAL( + auth_tag, + tdata->auth_tag.data, + tdata->auth_tag.len, + "Generated auth tag not as expected"); + + return 0; + +} + +static int +test_AES_GCM_authenticated_encryption_test_case_1(void) +{ + return test_authenticated_encryption(&gcm_test_case_1); +} + +static int +test_AES_GCM_authenticated_encryption_test_case_2(void) +{ + return test_authenticated_encryption(&gcm_test_case_2); +} + +static int +test_AES_GCM_authenticated_encryption_test_case_3(void) +{ + return test_authenticated_encryption(&gcm_test_case_3); +} + +static int +test_AES_GCM_authenticated_encryption_test_case_4(void) +{ + return test_authenticated_encryption(&gcm_test_case_4); +} + +static int +test_AES_GCM_authenticated_encryption_test_case_5(void) +{ + return test_authenticated_encryption(&gcm_test_case_5); +} + +static int +test_AES_GCM_authenticated_encryption_test_case_6(void) +{ + return test_authenticated_encryption(&gcm_test_case_6); +} + +static int +test_AES_GCM_authenticated_encryption_test_case_7(void) +{ + return test_authenticated_encryption(&gcm_test_case_7); +} + +static int +test_AES_GCM_auth_encryption_test_case_192_1(void) +{ + return test_authenticated_encryption(&gcm_test_case_192_1); +} + +static int +test_AES_GCM_auth_encryption_test_case_192_2(void) +{ + return test_authenticated_encryption(&gcm_test_case_192_2); +} + +static int +test_AES_GCM_auth_encryption_test_case_192_3(void) +{ + return test_authenticated_encryption(&gcm_test_case_192_3); +} + +static int +test_AES_GCM_auth_encryption_test_case_192_4(void) +{ + return test_authenticated_encryption(&gcm_test_case_192_4); +} + +static int +test_AES_GCM_auth_encryption_test_case_192_5(void) +{ + return test_authenticated_encryption(&gcm_test_case_192_5); +} + +static int +test_AES_GCM_auth_encryption_test_case_192_6(void) +{ + return test_authenticated_encryption(&gcm_test_case_192_6); +} + +static int +test_AES_GCM_auth_encryption_test_case_192_7(void) +{ + return test_authenticated_encryption(&gcm_test_case_192_7); +} + +static int +test_AES_GCM_auth_encryption_test_case_256_1(void) +{ + return test_authenticated_encryption(&gcm_test_case_256_1); +} + +static int +test_AES_GCM_auth_encryption_test_case_256_2(void) +{ + return test_authenticated_encryption(&gcm_test_case_256_2); +} + +static int +test_AES_GCM_auth_encryption_test_case_256_3(void) +{ + return test_authenticated_encryption(&gcm_test_case_256_3); +} + +static int +test_AES_GCM_auth_encryption_test_case_256_4(void) +{ + return test_authenticated_encryption(&gcm_test_case_256_4); +} + +static int +test_AES_GCM_auth_encryption_test_case_256_5(void) +{ + return test_authenticated_encryption(&gcm_test_case_256_5); +} + +static int +test_AES_GCM_auth_encryption_test_case_256_6(void) +{ + return test_authenticated_encryption(&gcm_test_case_256_6); +} + +static int +test_AES_GCM_auth_encryption_test_case_256_7(void) +{ + return test_authenticated_encryption(&gcm_test_case_256_7); +} + +static int +test_AES_GCM_auth_encryption_test_case_aad_1(void) +{ + return test_authenticated_encryption(&gcm_test_case_aad_1); +} + +static int +test_AES_GCM_auth_encryption_test_case_aad_2(void) +{ + return test_authenticated_encryption(&gcm_test_case_aad_2); +} + +static int +test_authenticated_decryption(const struct aead_test_data *tdata) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + + int retval; + uint8_t *plaintext; + uint32_t i; + + /* Create AEAD session */ + retval = create_aead_session(ts_params->valid_devs[0], + tdata->algo, + RTE_CRYPTO_AEAD_OP_DECRYPT, + tdata->key.data, tdata->key.len, + tdata->aad.len, tdata->auth_tag.len, + tdata->iv.len); + if (retval < 0) + return retval; + + /* alloc mbuf and set payload */ + if (tdata->aad.len > MBUF_SIZE) { + ut_params->ibuf = rte_pktmbuf_alloc(ts_params->large_mbuf_pool); + /* Populate full size of add data */ + for (i = 32; i < MAX_AAD_LENGTH; i += 32) + memcpy(&tdata->aad.data[i], &tdata->aad.data[0], 32); + } else + ut_params->ibuf = rte_pktmbuf_alloc(ts_params->mbuf_pool); + + memset(rte_pktmbuf_mtod(ut_params->ibuf, uint8_t *), 0, + rte_pktmbuf_tailroom(ut_params->ibuf)); + + /* Create AEAD operation */ + retval = create_aead_operation(RTE_CRYPTO_AEAD_OP_DECRYPT, tdata); + if (retval < 0) + return retval; + + rte_crypto_op_attach_sym_session(ut_params->op, ut_params->sess); + + ut_params->op->sym->m_src = ut_params->ibuf; + + /* Process crypto operation */ + TEST_ASSERT_NOT_NULL(process_crypto_request(ts_params->valid_devs[0], + ut_params->op), "failed to process sym crypto op"); + + TEST_ASSERT_EQUAL(ut_params->op->status, RTE_CRYPTO_OP_STATUS_SUCCESS, + "crypto op processing failed"); + + if (ut_params->op->sym->m_dst) + plaintext = rte_pktmbuf_mtod(ut_params->op->sym->m_dst, + uint8_t *); + else + plaintext = rte_pktmbuf_mtod_offset(ut_params->op->sym->m_src, + uint8_t *, + ut_params->op->sym->cipher.data.offset); + + TEST_HEXDUMP(stdout, "plaintext:", plaintext, tdata->ciphertext.len); + + /* Validate obuf */ + TEST_ASSERT_BUFFERS_ARE_EQUAL( + plaintext, + tdata->plaintext.data, + tdata->plaintext.len, + "Plaintext data not as expected"); + + TEST_ASSERT_EQUAL(ut_params->op->status, + RTE_CRYPTO_OP_STATUS_SUCCESS, + "Authentication failed"); + return 0; +} + +static int +test_AES_GCM_authenticated_decryption_test_case_1(void) +{ + return test_authenticated_decryption(&gcm_test_case_1); +} + +static int +test_AES_GCM_authenticated_decryption_test_case_2(void) +{ + return test_authenticated_decryption(&gcm_test_case_2); +} + +static int +test_AES_GCM_authenticated_decryption_test_case_3(void) +{ + return test_authenticated_decryption(&gcm_test_case_3); +} + +static int +test_AES_GCM_authenticated_decryption_test_case_4(void) +{ + return test_authenticated_decryption(&gcm_test_case_4); +} + +static int +test_AES_GCM_authenticated_decryption_test_case_5(void) +{ + return test_authenticated_decryption(&gcm_test_case_5); +} + +static int +test_AES_GCM_authenticated_decryption_test_case_6(void) +{ + return test_authenticated_decryption(&gcm_test_case_6); +} + +static int +test_AES_GCM_authenticated_decryption_test_case_7(void) +{ + return test_authenticated_decryption(&gcm_test_case_7); +} + +static int +test_AES_GCM_auth_decryption_test_case_192_1(void) +{ + return test_authenticated_decryption(&gcm_test_case_192_1); +} + +static int +test_AES_GCM_auth_decryption_test_case_192_2(void) +{ + return test_authenticated_decryption(&gcm_test_case_192_2); +} + +static int +test_AES_GCM_auth_decryption_test_case_192_3(void) +{ + return test_authenticated_decryption(&gcm_test_case_192_3); +} + +static int +test_AES_GCM_auth_decryption_test_case_192_4(void) +{ + return test_authenticated_decryption(&gcm_test_case_192_4); +} + +static int +test_AES_GCM_auth_decryption_test_case_192_5(void) +{ + return test_authenticated_decryption(&gcm_test_case_192_5); +} + +static int +test_AES_GCM_auth_decryption_test_case_192_6(void) +{ + return test_authenticated_decryption(&gcm_test_case_192_6); +} + +static int +test_AES_GCM_auth_decryption_test_case_192_7(void) +{ + return test_authenticated_decryption(&gcm_test_case_192_7); +} + +static int +test_AES_GCM_auth_decryption_test_case_256_1(void) +{ + return test_authenticated_decryption(&gcm_test_case_256_1); +} + +static int +test_AES_GCM_auth_decryption_test_case_256_2(void) +{ + return test_authenticated_decryption(&gcm_test_case_256_2); +} + +static int +test_AES_GCM_auth_decryption_test_case_256_3(void) +{ + return test_authenticated_decryption(&gcm_test_case_256_3); +} + +static int +test_AES_GCM_auth_decryption_test_case_256_4(void) +{ + return test_authenticated_decryption(&gcm_test_case_256_4); +} + +static int +test_AES_GCM_auth_decryption_test_case_256_5(void) +{ + return test_authenticated_decryption(&gcm_test_case_256_5); +} + +static int +test_AES_GCM_auth_decryption_test_case_256_6(void) +{ + return test_authenticated_decryption(&gcm_test_case_256_6); +} + +static int +test_AES_GCM_auth_decryption_test_case_256_7(void) +{ + return test_authenticated_decryption(&gcm_test_case_256_7); +} + +static int +test_AES_GCM_auth_decryption_test_case_aad_1(void) +{ + return test_authenticated_decryption(&gcm_test_case_aad_1); +} + +static int +test_AES_GCM_auth_decryption_test_case_aad_2(void) +{ + return test_authenticated_decryption(&gcm_test_case_aad_2); +} + +static int +test_authenticated_encryption_oop(const struct aead_test_data *tdata) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + + int retval; + uint8_t *ciphertext, *auth_tag; + uint16_t plaintext_pad_len; + + /* Create AEAD session */ + retval = create_aead_session(ts_params->valid_devs[0], + tdata->algo, + RTE_CRYPTO_AEAD_OP_ENCRYPT, + tdata->key.data, tdata->key.len, + tdata->aad.len, tdata->auth_tag.len, + tdata->iv.len); + if (retval < 0) + return retval; + + ut_params->ibuf = rte_pktmbuf_alloc(ts_params->mbuf_pool); + ut_params->obuf = rte_pktmbuf_alloc(ts_params->mbuf_pool); + + /* clear mbuf payload */ + memset(rte_pktmbuf_mtod(ut_params->ibuf, uint8_t *), 0, + rte_pktmbuf_tailroom(ut_params->ibuf)); + memset(rte_pktmbuf_mtod(ut_params->obuf, uint8_t *), 0, + rte_pktmbuf_tailroom(ut_params->obuf)); + + /* Create AEAD operation */ + retval = create_aead_operation(RTE_CRYPTO_AEAD_OP_ENCRYPT, tdata); + if (retval < 0) + return retval; + + rte_crypto_op_attach_sym_session(ut_params->op, ut_params->sess); + + ut_params->op->sym->m_src = ut_params->ibuf; + ut_params->op->sym->m_dst = ut_params->obuf; + + /* Process crypto operation */ + TEST_ASSERT_NOT_NULL(process_crypto_request(ts_params->valid_devs[0], + ut_params->op), "failed to process sym crypto op"); + + TEST_ASSERT_EQUAL(ut_params->op->status, RTE_CRYPTO_OP_STATUS_SUCCESS, + "crypto op processing failed"); + + plaintext_pad_len = RTE_ALIGN_CEIL(tdata->plaintext.len, 16); + + ciphertext = rte_pktmbuf_mtod_offset(ut_params->obuf, uint8_t *, + ut_params->op->sym->cipher.data.offset); + auth_tag = ciphertext + plaintext_pad_len; + + TEST_HEXDUMP(stdout, "ciphertext:", ciphertext, tdata->ciphertext.len); + TEST_HEXDUMP(stdout, "auth tag:", auth_tag, tdata->auth_tag.len); + + /* Validate obuf */ + TEST_ASSERT_BUFFERS_ARE_EQUAL( + ciphertext, + tdata->ciphertext.data, + tdata->ciphertext.len, + "Ciphertext data not as expected"); + + TEST_ASSERT_BUFFERS_ARE_EQUAL( + auth_tag, + tdata->auth_tag.data, + tdata->auth_tag.len, + "Generated auth tag not as expected"); + + return 0; + +} + +static int +test_AES_GCM_authenticated_encryption_oop_test_case_1(void) +{ + return test_authenticated_encryption_oop(&gcm_test_case_5); +} + +static int +test_authenticated_decryption_oop(const struct aead_test_data *tdata) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + + int retval; + uint8_t *plaintext; + + /* Create AEAD session */ + retval = create_aead_session(ts_params->valid_devs[0], + tdata->algo, + RTE_CRYPTO_AEAD_OP_DECRYPT, + tdata->key.data, tdata->key.len, + tdata->aad.len, tdata->auth_tag.len, + tdata->iv.len); + if (retval < 0) + return retval; + + /* alloc mbuf and set payload */ + ut_params->ibuf = rte_pktmbuf_alloc(ts_params->mbuf_pool); + ut_params->obuf = rte_pktmbuf_alloc(ts_params->mbuf_pool); + + memset(rte_pktmbuf_mtod(ut_params->ibuf, uint8_t *), 0, + rte_pktmbuf_tailroom(ut_params->ibuf)); + memset(rte_pktmbuf_mtod(ut_params->obuf, uint8_t *), 0, + rte_pktmbuf_tailroom(ut_params->obuf)); + + /* Create AEAD operation */ + retval = create_aead_operation(RTE_CRYPTO_AEAD_OP_DECRYPT, tdata); + if (retval < 0) + return retval; + + rte_crypto_op_attach_sym_session(ut_params->op, ut_params->sess); + + ut_params->op->sym->m_src = ut_params->ibuf; + ut_params->op->sym->m_dst = ut_params->obuf; + + /* Process crypto operation */ + TEST_ASSERT_NOT_NULL(process_crypto_request(ts_params->valid_devs[0], + ut_params->op), "failed to process sym crypto op"); + + TEST_ASSERT_EQUAL(ut_params->op->status, RTE_CRYPTO_OP_STATUS_SUCCESS, + "crypto op processing failed"); + + plaintext = rte_pktmbuf_mtod_offset(ut_params->obuf, uint8_t *, + ut_params->op->sym->cipher.data.offset); + + TEST_HEXDUMP(stdout, "plaintext:", plaintext, tdata->ciphertext.len); + + /* Validate obuf */ + TEST_ASSERT_BUFFERS_ARE_EQUAL( + plaintext, + tdata->plaintext.data, + tdata->plaintext.len, + "Plaintext data not as expected"); + + TEST_ASSERT_EQUAL(ut_params->op->status, + RTE_CRYPTO_OP_STATUS_SUCCESS, + "Authentication failed"); + return 0; +} + +static int +test_AES_GCM_authenticated_decryption_oop_test_case_1(void) +{ + return test_authenticated_decryption_oop(&gcm_test_case_5); +} + +static int +test_authenticated_encryption_sessionless( + const struct aead_test_data *tdata) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + + int retval; + uint8_t *ciphertext, *auth_tag; + uint16_t plaintext_pad_len; + uint8_t key[tdata->key.len + 1]; + + ut_params->ibuf = rte_pktmbuf_alloc(ts_params->mbuf_pool); + + /* clear mbuf payload */ + memset(rte_pktmbuf_mtod(ut_params->ibuf, uint8_t *), 0, + rte_pktmbuf_tailroom(ut_params->ibuf)); + + /* Create AEAD operation */ + retval = create_aead_operation(RTE_CRYPTO_AEAD_OP_ENCRYPT, tdata); + if (retval < 0) + return retval; + + /* Create GCM xform */ + memcpy(key, tdata->key.data, tdata->key.len); + retval = create_aead_xform(ut_params->op, + tdata->algo, + RTE_CRYPTO_AEAD_OP_ENCRYPT, + key, tdata->key.len, + tdata->aad.len, tdata->auth_tag.len, + tdata->iv.len); + if (retval < 0) + return retval; + + ut_params->op->sym->m_src = ut_params->ibuf; + + TEST_ASSERT_EQUAL(ut_params->op->sess_type, + RTE_CRYPTO_OP_SESSIONLESS, + "crypto op session type not sessionless"); + + /* Process crypto operation */ + TEST_ASSERT_NOT_NULL(process_crypto_request(ts_params->valid_devs[0], + ut_params->op), "failed to process sym crypto op"); + + TEST_ASSERT_NOT_NULL(ut_params->op, "failed crypto process"); + + TEST_ASSERT_EQUAL(ut_params->op->status, RTE_CRYPTO_OP_STATUS_SUCCESS, + "crypto op status not success"); + + plaintext_pad_len = RTE_ALIGN_CEIL(tdata->plaintext.len, 16); + + ciphertext = rte_pktmbuf_mtod_offset(ut_params->ibuf, uint8_t *, + ut_params->op->sym->cipher.data.offset); + auth_tag = ciphertext + plaintext_pad_len; + + TEST_HEXDUMP(stdout, "ciphertext:", ciphertext, tdata->ciphertext.len); + TEST_HEXDUMP(stdout, "auth tag:", auth_tag, tdata->auth_tag.len); + + /* Validate obuf */ + TEST_ASSERT_BUFFERS_ARE_EQUAL( + ciphertext, + tdata->ciphertext.data, + tdata->ciphertext.len, + "Ciphertext data not as expected"); + + TEST_ASSERT_BUFFERS_ARE_EQUAL( + auth_tag, + tdata->auth_tag.data, + tdata->auth_tag.len, + "Generated auth tag not as expected"); + + return 0; + +} + +static int +test_AES_GCM_authenticated_encryption_sessionless_test_case_1(void) +{ + return test_authenticated_encryption_sessionless( + &gcm_test_case_5); +} + +static int +test_authenticated_decryption_sessionless( + const struct aead_test_data *tdata) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + + int retval; + uint8_t *plaintext; + uint8_t key[tdata->key.len + 1]; + + /* alloc mbuf and set payload */ + ut_params->ibuf = rte_pktmbuf_alloc(ts_params->mbuf_pool); + + memset(rte_pktmbuf_mtod(ut_params->ibuf, uint8_t *), 0, + rte_pktmbuf_tailroom(ut_params->ibuf)); + + /* Create AEAD operation */ + retval = create_aead_operation(RTE_CRYPTO_AEAD_OP_DECRYPT, tdata); + if (retval < 0) + return retval; + + /* Create AEAD xform */ + memcpy(key, tdata->key.data, tdata->key.len); + retval = create_aead_xform(ut_params->op, + tdata->algo, + RTE_CRYPTO_AEAD_OP_DECRYPT, + key, tdata->key.len, + tdata->aad.len, tdata->auth_tag.len, + tdata->iv.len); + if (retval < 0) + return retval; + + ut_params->op->sym->m_src = ut_params->ibuf; + + TEST_ASSERT_EQUAL(ut_params->op->sess_type, + RTE_CRYPTO_OP_SESSIONLESS, + "crypto op session type not sessionless"); + + /* Process crypto operation */ + TEST_ASSERT_NOT_NULL(process_crypto_request(ts_params->valid_devs[0], + ut_params->op), "failed to process sym crypto op"); + + TEST_ASSERT_NOT_NULL(ut_params->op, "failed crypto process"); + + TEST_ASSERT_EQUAL(ut_params->op->status, RTE_CRYPTO_OP_STATUS_SUCCESS, + "crypto op status not success"); + + plaintext = rte_pktmbuf_mtod_offset(ut_params->ibuf, uint8_t *, + ut_params->op->sym->cipher.data.offset); + + TEST_HEXDUMP(stdout, "plaintext:", plaintext, tdata->ciphertext.len); + + /* Validate obuf */ + TEST_ASSERT_BUFFERS_ARE_EQUAL( + plaintext, + tdata->plaintext.data, + tdata->plaintext.len, + "Plaintext data not as expected"); + + TEST_ASSERT_EQUAL(ut_params->op->status, + RTE_CRYPTO_OP_STATUS_SUCCESS, + "Authentication failed"); + return 0; +} + +static int +test_AES_GCM_authenticated_decryption_sessionless_test_case_1(void) +{ + return test_authenticated_decryption_sessionless( + &gcm_test_case_5); +} + +static int +test_AES_CCM_authenticated_encryption_test_case_128_1(void) +{ + return test_authenticated_encryption(&ccm_test_case_128_1); +} + +static int +test_AES_CCM_authenticated_encryption_test_case_128_2(void) +{ + return test_authenticated_encryption(&ccm_test_case_128_2); +} + +static int +test_AES_CCM_authenticated_encryption_test_case_128_3(void) +{ + return test_authenticated_encryption(&ccm_test_case_128_3); +} + +static int +test_AES_CCM_authenticated_decryption_test_case_128_1(void) +{ + return test_authenticated_decryption(&ccm_test_case_128_1); +} + +static int +test_AES_CCM_authenticated_decryption_test_case_128_2(void) +{ + return test_authenticated_decryption(&ccm_test_case_128_2); +} + +static int +test_AES_CCM_authenticated_decryption_test_case_128_3(void) +{ + return test_authenticated_decryption(&ccm_test_case_128_3); +} + +static int +test_AES_CCM_authenticated_encryption_test_case_192_1(void) +{ + return test_authenticated_encryption(&ccm_test_case_192_1); +} + +static int +test_AES_CCM_authenticated_encryption_test_case_192_2(void) +{ + return test_authenticated_encryption(&ccm_test_case_192_2); +} + +static int +test_AES_CCM_authenticated_encryption_test_case_192_3(void) +{ + return test_authenticated_encryption(&ccm_test_case_192_3); +} + +static int +test_AES_CCM_authenticated_decryption_test_case_192_1(void) +{ + return test_authenticated_decryption(&ccm_test_case_192_1); +} + +static int +test_AES_CCM_authenticated_decryption_test_case_192_2(void) +{ + return test_authenticated_decryption(&ccm_test_case_192_2); +} + +static int +test_AES_CCM_authenticated_decryption_test_case_192_3(void) +{ + return test_authenticated_decryption(&ccm_test_case_192_3); +} + +static int +test_AES_CCM_authenticated_encryption_test_case_256_1(void) +{ + return test_authenticated_encryption(&ccm_test_case_256_1); +} + +static int +test_AES_CCM_authenticated_encryption_test_case_256_2(void) +{ + return test_authenticated_encryption(&ccm_test_case_256_2); +} + +static int +test_AES_CCM_authenticated_encryption_test_case_256_3(void) +{ + return test_authenticated_encryption(&ccm_test_case_256_3); +} + +static int +test_AES_CCM_authenticated_decryption_test_case_256_1(void) +{ + return test_authenticated_decryption(&ccm_test_case_256_1); +} + +static int +test_AES_CCM_authenticated_decryption_test_case_256_2(void) +{ + return test_authenticated_decryption(&ccm_test_case_256_2); +} + +static int +test_AES_CCM_authenticated_decryption_test_case_256_3(void) +{ + return test_authenticated_decryption(&ccm_test_case_256_3); +} + +static int +test_stats(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct rte_cryptodev_stats stats; + struct rte_cryptodev *dev; + cryptodev_stats_get_t temp_pfn; + + rte_cryptodev_stats_reset(ts_params->valid_devs[0]); + TEST_ASSERT((rte_cryptodev_stats_get(ts_params->valid_devs[0] + 600, + &stats) == -ENODEV), + "rte_cryptodev_stats_get invalid dev failed"); + TEST_ASSERT((rte_cryptodev_stats_get(ts_params->valid_devs[0], 0) != 0), + "rte_cryptodev_stats_get invalid Param failed"); + dev = &rte_cryptodevs[ts_params->valid_devs[0]]; + temp_pfn = dev->dev_ops->stats_get; + dev->dev_ops->stats_get = (cryptodev_stats_get_t)0; + TEST_ASSERT((rte_cryptodev_stats_get(ts_params->valid_devs[0], &stats) + == -ENOTSUP), + "rte_cryptodev_stats_get invalid Param failed"); + dev->dev_ops->stats_get = temp_pfn; + + /* Test expected values */ + ut_setup(); + test_AES_CBC_HMAC_SHA1_encrypt_digest(); + ut_teardown(); + TEST_ASSERT_SUCCESS(rte_cryptodev_stats_get(ts_params->valid_devs[0], + &stats), + "rte_cryptodev_stats_get failed"); + TEST_ASSERT((stats.enqueued_count == 1), + "rte_cryptodev_stats_get returned unexpected enqueued stat"); + TEST_ASSERT((stats.dequeued_count == 1), + "rte_cryptodev_stats_get returned unexpected enqueued stat"); + TEST_ASSERT((stats.enqueue_err_count == 0), + "rte_cryptodev_stats_get returned unexpected enqueued stat"); + TEST_ASSERT((stats.dequeue_err_count == 0), + "rte_cryptodev_stats_get returned unexpected enqueued stat"); + + /* invalid device but should ignore and not reset device stats*/ + rte_cryptodev_stats_reset(ts_params->valid_devs[0] + 300); + TEST_ASSERT_SUCCESS(rte_cryptodev_stats_get(ts_params->valid_devs[0], + &stats), + "rte_cryptodev_stats_get failed"); + TEST_ASSERT((stats.enqueued_count == 1), + "rte_cryptodev_stats_get returned unexpected enqueued stat"); + + /* check that a valid reset clears stats */ + rte_cryptodev_stats_reset(ts_params->valid_devs[0]); + TEST_ASSERT_SUCCESS(rte_cryptodev_stats_get(ts_params->valid_devs[0], + &stats), + "rte_cryptodev_stats_get failed"); + TEST_ASSERT((stats.enqueued_count == 0), + "rte_cryptodev_stats_get returned unexpected enqueued stat"); + TEST_ASSERT((stats.dequeued_count == 0), + "rte_cryptodev_stats_get returned unexpected enqueued stat"); + + return TEST_SUCCESS; +} + +static int MD5_HMAC_create_session(struct crypto_testsuite_params *ts_params, + struct crypto_unittest_params *ut_params, + enum rte_crypto_auth_operation op, + const struct HMAC_MD5_vector *test_case) +{ + uint8_t key[64]; + + memcpy(key, test_case->key.data, test_case->key.len); + + ut_params->auth_xform.type = RTE_CRYPTO_SYM_XFORM_AUTH; + ut_params->auth_xform.next = NULL; + ut_params->auth_xform.auth.op = op; + + ut_params->auth_xform.auth.algo = RTE_CRYPTO_AUTH_MD5_HMAC; + + ut_params->auth_xform.auth.digest_length = MD5_DIGEST_LEN; + ut_params->auth_xform.auth.key.length = test_case->key.len; + ut_params->auth_xform.auth.key.data = key; + + ut_params->sess = rte_cryptodev_sym_session_create( + ts_params->session_mpool); + + rte_cryptodev_sym_session_init(ts_params->valid_devs[0], + ut_params->sess, &ut_params->auth_xform, + ts_params->session_mpool); + + if (ut_params->sess == NULL) + return TEST_FAILED; + + ut_params->ibuf = rte_pktmbuf_alloc(ts_params->mbuf_pool); + + memset(rte_pktmbuf_mtod(ut_params->ibuf, uint8_t *), 0, + rte_pktmbuf_tailroom(ut_params->ibuf)); + + return 0; +} + +static int MD5_HMAC_create_op(struct crypto_unittest_params *ut_params, + const struct HMAC_MD5_vector *test_case, + uint8_t **plaintext) +{ + uint16_t plaintext_pad_len; + + struct rte_crypto_sym_op *sym_op = ut_params->op->sym; + + plaintext_pad_len = RTE_ALIGN_CEIL(test_case->plaintext.len, + 16); + + *plaintext = (uint8_t *)rte_pktmbuf_append(ut_params->ibuf, + plaintext_pad_len); + memcpy(*plaintext, test_case->plaintext.data, + test_case->plaintext.len); + + sym_op->auth.digest.data = (uint8_t *)rte_pktmbuf_append( + ut_params->ibuf, MD5_DIGEST_LEN); + TEST_ASSERT_NOT_NULL(sym_op->auth.digest.data, + "no room to append digest"); + sym_op->auth.digest.phys_addr = rte_pktmbuf_iova_offset( + ut_params->ibuf, plaintext_pad_len); + + if (ut_params->auth_xform.auth.op == RTE_CRYPTO_AUTH_OP_VERIFY) { + rte_memcpy(sym_op->auth.digest.data, test_case->auth_tag.data, + test_case->auth_tag.len); + } + + sym_op->auth.data.offset = 0; + sym_op->auth.data.length = test_case->plaintext.len; + + rte_crypto_op_attach_sym_session(ut_params->op, ut_params->sess); + ut_params->op->sym->m_src = ut_params->ibuf; + + return 0; +} + +static int +test_MD5_HMAC_generate(const struct HMAC_MD5_vector *test_case) +{ + uint16_t plaintext_pad_len; + uint8_t *plaintext, *auth_tag; + + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + + if (MD5_HMAC_create_session(ts_params, ut_params, + RTE_CRYPTO_AUTH_OP_GENERATE, test_case)) + return TEST_FAILED; + + /* Generate Crypto op data structure */ + ut_params->op = rte_crypto_op_alloc(ts_params->op_mpool, + RTE_CRYPTO_OP_TYPE_SYMMETRIC); + TEST_ASSERT_NOT_NULL(ut_params->op, + "Failed to allocate symmetric crypto operation struct"); + + plaintext_pad_len = RTE_ALIGN_CEIL(test_case->plaintext.len, + 16); + + if (MD5_HMAC_create_op(ut_params, test_case, &plaintext)) + return TEST_FAILED; + + TEST_ASSERT_NOT_NULL(process_crypto_request(ts_params->valid_devs[0], + ut_params->op), "failed to process sym crypto op"); + + TEST_ASSERT_EQUAL(ut_params->op->status, RTE_CRYPTO_OP_STATUS_SUCCESS, + "crypto op processing failed"); + + if (ut_params->op->sym->m_dst) { + auth_tag = rte_pktmbuf_mtod_offset(ut_params->op->sym->m_dst, + uint8_t *, plaintext_pad_len); + } else { + auth_tag = plaintext + plaintext_pad_len; + } + + TEST_ASSERT_BUFFERS_ARE_EQUAL( + auth_tag, + test_case->auth_tag.data, + test_case->auth_tag.len, + "HMAC_MD5 generated tag not as expected"); + + return TEST_SUCCESS; +} + +static int +test_MD5_HMAC_verify(const struct HMAC_MD5_vector *test_case) +{ + uint8_t *plaintext; + + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + + if (MD5_HMAC_create_session(ts_params, ut_params, + RTE_CRYPTO_AUTH_OP_VERIFY, test_case)) { + return TEST_FAILED; + } + + /* Generate Crypto op data structure */ + ut_params->op = rte_crypto_op_alloc(ts_params->op_mpool, + RTE_CRYPTO_OP_TYPE_SYMMETRIC); + TEST_ASSERT_NOT_NULL(ut_params->op, + "Failed to allocate symmetric crypto operation struct"); + + if (MD5_HMAC_create_op(ut_params, test_case, &plaintext)) + return TEST_FAILED; + + TEST_ASSERT_NOT_NULL(process_crypto_request(ts_params->valid_devs[0], + ut_params->op), "failed to process sym crypto op"); + + TEST_ASSERT_EQUAL(ut_params->op->status, RTE_CRYPTO_OP_STATUS_SUCCESS, + "HMAC_MD5 crypto op processing failed"); + + return TEST_SUCCESS; +} + +static int +test_MD5_HMAC_generate_case_1(void) +{ + return test_MD5_HMAC_generate(&HMAC_MD5_test_case_1); +} + +static int +test_MD5_HMAC_verify_case_1(void) +{ + return test_MD5_HMAC_verify(&HMAC_MD5_test_case_1); +} + +static int +test_MD5_HMAC_generate_case_2(void) +{ + return test_MD5_HMAC_generate(&HMAC_MD5_test_case_2); +} + +static int +test_MD5_HMAC_verify_case_2(void) +{ + return test_MD5_HMAC_verify(&HMAC_MD5_test_case_2); +} + +static int +test_multi_session(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + + struct rte_cryptodev_info dev_info; + struct rte_cryptodev_sym_session **sessions; + + uint16_t i; + + test_AES_CBC_HMAC_SHA512_decrypt_create_session_params(ut_params, + aes_cbc_key, hmac_sha512_key); + + + rte_cryptodev_info_get(ts_params->valid_devs[0], &dev_info); + + sessions = rte_malloc(NULL, + (sizeof(struct rte_cryptodev_sym_session *) * + dev_info.sym.max_nb_sessions) + 1, 0); + + /* Create multiple crypto sessions*/ + for (i = 0; i < dev_info.sym.max_nb_sessions; i++) { + + sessions[i] = rte_cryptodev_sym_session_create( + ts_params->session_mpool); + + rte_cryptodev_sym_session_init(ts_params->valid_devs[0], + sessions[i], &ut_params->auth_xform, + ts_params->session_mpool); + TEST_ASSERT_NOT_NULL(sessions[i], + "Session creation failed at session number %u", + i); + + /* Attempt to send a request on each session */ + TEST_ASSERT_SUCCESS( test_AES_CBC_HMAC_SHA512_decrypt_perform( + sessions[i], + ut_params, + ts_params, + catch_22_quote_2_512_bytes_AES_CBC_ciphertext, + catch_22_quote_2_512_bytes_AES_CBC_HMAC_SHA512_digest, + aes_cbc_iv), + "Failed to perform decrypt on request number %u.", i); + /* free crypto operation structure */ + if (ut_params->op) + rte_crypto_op_free(ut_params->op); + + /* + * free mbuf - both obuf and ibuf are usually the same, + * so check if they point at the same address is necessary, + * to avoid freeing the mbuf twice. + */ + if (ut_params->obuf) { + rte_pktmbuf_free(ut_params->obuf); + if (ut_params->ibuf == ut_params->obuf) + ut_params->ibuf = 0; + ut_params->obuf = 0; + } + if (ut_params->ibuf) { + rte_pktmbuf_free(ut_params->ibuf); + ut_params->ibuf = 0; + } + } + + /* Next session create should fail */ + rte_cryptodev_sym_session_init(ts_params->valid_devs[0], + sessions[i], &ut_params->auth_xform, + ts_params->session_mpool); + TEST_ASSERT_NULL(sessions[i], + "Session creation succeeded unexpectedly!"); + + for (i = 0; i < dev_info.sym.max_nb_sessions; i++) { + rte_cryptodev_sym_session_clear(ts_params->valid_devs[0], + sessions[i]); + rte_cryptodev_sym_session_free(sessions[i]); + } + + rte_free(sessions); + + return TEST_SUCCESS; +} + +struct multi_session_params { + struct crypto_unittest_params ut_params; + uint8_t *cipher_key; + uint8_t *hmac_key; + const uint8_t *cipher; + const uint8_t *digest; + uint8_t *iv; +}; + +#define MB_SESSION_NUMBER 3 + +static int +test_multi_session_random_usage(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct rte_cryptodev_info dev_info; + struct rte_cryptodev_sym_session **sessions; + uint32_t i, j; + struct multi_session_params ut_paramz[] = { + + { + .cipher_key = ms_aes_cbc_key0, + .hmac_key = ms_hmac_key0, + .cipher = ms_aes_cbc_cipher0, + .digest = ms_hmac_digest0, + .iv = ms_aes_cbc_iv0 + }, + { + .cipher_key = ms_aes_cbc_key1, + .hmac_key = ms_hmac_key1, + .cipher = ms_aes_cbc_cipher1, + .digest = ms_hmac_digest1, + .iv = ms_aes_cbc_iv1 + }, + { + .cipher_key = ms_aes_cbc_key2, + .hmac_key = ms_hmac_key2, + .cipher = ms_aes_cbc_cipher2, + .digest = ms_hmac_digest2, + .iv = ms_aes_cbc_iv2 + }, + + }; + + rte_cryptodev_info_get(ts_params->valid_devs[0], &dev_info); + + sessions = rte_malloc(NULL, + (sizeof(struct rte_cryptodev_sym_session *) + * dev_info.sym.max_nb_sessions) + 1, 0); + + for (i = 0; i < MB_SESSION_NUMBER; i++) { + sessions[i] = rte_cryptodev_sym_session_create( + ts_params->session_mpool); + + rte_memcpy(&ut_paramz[i].ut_params, &testsuite_params, + sizeof(struct crypto_unittest_params)); + + test_AES_CBC_HMAC_SHA512_decrypt_create_session_params( + &ut_paramz[i].ut_params, + ut_paramz[i].cipher_key, ut_paramz[i].hmac_key); + + /* Create multiple crypto sessions*/ + rte_cryptodev_sym_session_init( + ts_params->valid_devs[0], + sessions[i], + &ut_paramz[i].ut_params.auth_xform, + ts_params->session_mpool); + + TEST_ASSERT_NOT_NULL(sessions[i], + "Session creation failed at session number %u", + i); + + } + + srand(time(NULL)); + for (i = 0; i < 40000; i++) { + + j = rand() % MB_SESSION_NUMBER; + + TEST_ASSERT_SUCCESS( + test_AES_CBC_HMAC_SHA512_decrypt_perform( + sessions[j], + &ut_paramz[j].ut_params, + ts_params, ut_paramz[j].cipher, + ut_paramz[j].digest, + ut_paramz[j].iv), + "Failed to perform decrypt on request number %u.", i); + + if (ut_paramz[j].ut_params.op) + rte_crypto_op_free(ut_paramz[j].ut_params.op); + + /* + * free mbuf - both obuf and ibuf are usually the same, + * so check if they point at the same address is necessary, + * to avoid freeing the mbuf twice. + */ + if (ut_paramz[j].ut_params.obuf) { + rte_pktmbuf_free(ut_paramz[j].ut_params.obuf); + if (ut_paramz[j].ut_params.ibuf + == ut_paramz[j].ut_params.obuf) + ut_paramz[j].ut_params.ibuf = 0; + ut_paramz[j].ut_params.obuf = 0; + } + if (ut_paramz[j].ut_params.ibuf) { + rte_pktmbuf_free(ut_paramz[j].ut_params.ibuf); + ut_paramz[j].ut_params.ibuf = 0; + } + } + + for (i = 0; i < MB_SESSION_NUMBER; i++) { + rte_cryptodev_sym_session_clear(ts_params->valid_devs[0], + sessions[i]); + rte_cryptodev_sym_session_free(sessions[i]); + } + + rte_free(sessions); + + return TEST_SUCCESS; +} + +static int +test_null_cipher_only_operation(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + + /* Generate test mbuf data and space for digest */ + ut_params->ibuf = setup_test_string(ts_params->mbuf_pool, + catch_22_quote, QUOTE_512_BYTES, 0); + + /* Setup Cipher Parameters */ + ut_params->cipher_xform.type = RTE_CRYPTO_SYM_XFORM_CIPHER; + ut_params->cipher_xform.next = NULL; + + ut_params->cipher_xform.cipher.algo = RTE_CRYPTO_CIPHER_NULL; + ut_params->cipher_xform.cipher.op = RTE_CRYPTO_CIPHER_OP_ENCRYPT; + + ut_params->sess = rte_cryptodev_sym_session_create( + ts_params->session_mpool); + + /* Create Crypto session*/ + rte_cryptodev_sym_session_init(ts_params->valid_devs[0], + ut_params->sess, + &ut_params->cipher_xform, + ts_params->session_mpool); + TEST_ASSERT_NOT_NULL(ut_params->sess, "Session creation failed"); + + /* Generate Crypto op data structure */ + ut_params->op = rte_crypto_op_alloc(ts_params->op_mpool, + RTE_CRYPTO_OP_TYPE_SYMMETRIC); + TEST_ASSERT_NOT_NULL(ut_params->op, + "Failed to allocate symmetric crypto operation struct"); + + /* Set crypto operation data parameters */ + rte_crypto_op_attach_sym_session(ut_params->op, ut_params->sess); + + struct rte_crypto_sym_op *sym_op = ut_params->op->sym; + + /* set crypto operation source mbuf */ + sym_op->m_src = ut_params->ibuf; + + sym_op->cipher.data.offset = 0; + sym_op->cipher.data.length = QUOTE_512_BYTES; + + /* Process crypto operation */ + ut_params->op = process_crypto_request(ts_params->valid_devs[0], + ut_params->op); + TEST_ASSERT_NOT_NULL(ut_params->op, "no crypto operation returned"); + + TEST_ASSERT_EQUAL(ut_params->op->status, RTE_CRYPTO_OP_STATUS_SUCCESS, + "crypto operation processing failed"); + + /* Validate obuf */ + TEST_ASSERT_BUFFERS_ARE_EQUAL( + rte_pktmbuf_mtod(ut_params->op->sym->m_src, uint8_t *), + catch_22_quote, + QUOTE_512_BYTES, + "Ciphertext data not as expected"); + + return TEST_SUCCESS; +} + +static int +test_null_auth_only_operation(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + + /* Generate test mbuf data and space for digest */ + ut_params->ibuf = setup_test_string(ts_params->mbuf_pool, + catch_22_quote, QUOTE_512_BYTES, 0); + + /* Setup HMAC Parameters */ + ut_params->auth_xform.type = RTE_CRYPTO_SYM_XFORM_AUTH; + ut_params->auth_xform.next = NULL; + + ut_params->auth_xform.auth.algo = RTE_CRYPTO_AUTH_NULL; + ut_params->auth_xform.auth.op = RTE_CRYPTO_AUTH_OP_GENERATE; + + ut_params->sess = rte_cryptodev_sym_session_create( + ts_params->session_mpool); + + /* Create Crypto session*/ + rte_cryptodev_sym_session_init(ts_params->valid_devs[0], + ut_params->sess, &ut_params->auth_xform, + ts_params->session_mpool); + TEST_ASSERT_NOT_NULL(ut_params->sess, "Session creation failed"); + + /* Generate Crypto op data structure */ + ut_params->op = rte_crypto_op_alloc(ts_params->op_mpool, + RTE_CRYPTO_OP_TYPE_SYMMETRIC); + TEST_ASSERT_NOT_NULL(ut_params->op, + "Failed to allocate symmetric crypto operation struct"); + + /* Set crypto operation data parameters */ + rte_crypto_op_attach_sym_session(ut_params->op, ut_params->sess); + + struct rte_crypto_sym_op *sym_op = ut_params->op->sym; + + sym_op->m_src = ut_params->ibuf; + + sym_op->auth.data.offset = 0; + sym_op->auth.data.length = QUOTE_512_BYTES; + + /* Process crypto operation */ + ut_params->op = process_crypto_request(ts_params->valid_devs[0], + ut_params->op); + TEST_ASSERT_NOT_NULL(ut_params->op, "no crypto operation returned"); + + TEST_ASSERT_EQUAL(ut_params->op->status, RTE_CRYPTO_OP_STATUS_SUCCESS, + "crypto operation processing failed"); + + return TEST_SUCCESS; +} + +static int +test_null_cipher_auth_operation(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + + /* Generate test mbuf data and space for digest */ + ut_params->ibuf = setup_test_string(ts_params->mbuf_pool, + catch_22_quote, QUOTE_512_BYTES, 0); + + /* Setup Cipher Parameters */ + ut_params->cipher_xform.type = RTE_CRYPTO_SYM_XFORM_CIPHER; + ut_params->cipher_xform.next = &ut_params->auth_xform; + + ut_params->cipher_xform.cipher.algo = RTE_CRYPTO_CIPHER_NULL; + ut_params->cipher_xform.cipher.op = RTE_CRYPTO_CIPHER_OP_ENCRYPT; + + /* Setup HMAC Parameters */ + ut_params->auth_xform.type = RTE_CRYPTO_SYM_XFORM_AUTH; + ut_params->auth_xform.next = NULL; + + ut_params->auth_xform.auth.algo = RTE_CRYPTO_AUTH_NULL; + ut_params->auth_xform.auth.op = RTE_CRYPTO_AUTH_OP_GENERATE; + + ut_params->sess = rte_cryptodev_sym_session_create( + ts_params->session_mpool); + + /* Create Crypto session*/ + rte_cryptodev_sym_session_init(ts_params->valid_devs[0], + ut_params->sess, &ut_params->cipher_xform, + ts_params->session_mpool); + TEST_ASSERT_NOT_NULL(ut_params->sess, "Session creation failed"); + + /* Generate Crypto op data structure */ + ut_params->op = rte_crypto_op_alloc(ts_params->op_mpool, + RTE_CRYPTO_OP_TYPE_SYMMETRIC); + TEST_ASSERT_NOT_NULL(ut_params->op, + "Failed to allocate symmetric crypto operation struct"); + + /* Set crypto operation data parameters */ + rte_crypto_op_attach_sym_session(ut_params->op, ut_params->sess); + + struct rte_crypto_sym_op *sym_op = ut_params->op->sym; + + sym_op->m_src = ut_params->ibuf; + + sym_op->cipher.data.offset = 0; + sym_op->cipher.data.length = QUOTE_512_BYTES; + + sym_op->auth.data.offset = 0; + sym_op->auth.data.length = QUOTE_512_BYTES; + + /* Process crypto operation */ + ut_params->op = process_crypto_request(ts_params->valid_devs[0], + ut_params->op); + TEST_ASSERT_NOT_NULL(ut_params->op, "no crypto operation returned"); + + TEST_ASSERT_EQUAL(ut_params->op->status, RTE_CRYPTO_OP_STATUS_SUCCESS, + "crypto operation processing failed"); + + /* Validate obuf */ + TEST_ASSERT_BUFFERS_ARE_EQUAL( + rte_pktmbuf_mtod(ut_params->op->sym->m_src, uint8_t *), + catch_22_quote, + QUOTE_512_BYTES, + "Ciphertext data not as expected"); + + return TEST_SUCCESS; +} + +static int +test_null_auth_cipher_operation(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + + /* Generate test mbuf data and space for digest */ + ut_params->ibuf = setup_test_string(ts_params->mbuf_pool, + catch_22_quote, QUOTE_512_BYTES, 0); + + /* Setup Cipher Parameters */ + ut_params->cipher_xform.type = RTE_CRYPTO_SYM_XFORM_CIPHER; + ut_params->cipher_xform.next = NULL; + + ut_params->cipher_xform.cipher.algo = RTE_CRYPTO_CIPHER_NULL; + ut_params->cipher_xform.cipher.op = RTE_CRYPTO_CIPHER_OP_ENCRYPT; + + /* Setup HMAC Parameters */ + ut_params->auth_xform.type = RTE_CRYPTO_SYM_XFORM_AUTH; + ut_params->auth_xform.next = &ut_params->cipher_xform; + + ut_params->auth_xform.auth.algo = RTE_CRYPTO_AUTH_NULL; + ut_params->auth_xform.auth.op = RTE_CRYPTO_AUTH_OP_GENERATE; + + ut_params->sess = rte_cryptodev_sym_session_create( + ts_params->session_mpool); + + /* Create Crypto session*/ + rte_cryptodev_sym_session_init(ts_params->valid_devs[0], + ut_params->sess, &ut_params->cipher_xform, + ts_params->session_mpool); + TEST_ASSERT_NOT_NULL(ut_params->sess, "Session creation failed"); + + /* Generate Crypto op data structure */ + ut_params->op = rte_crypto_op_alloc(ts_params->op_mpool, + RTE_CRYPTO_OP_TYPE_SYMMETRIC); + TEST_ASSERT_NOT_NULL(ut_params->op, + "Failed to allocate symmetric crypto operation struct"); + + /* Set crypto operation data parameters */ + rte_crypto_op_attach_sym_session(ut_params->op, ut_params->sess); + + struct rte_crypto_sym_op *sym_op = ut_params->op->sym; + + sym_op->m_src = ut_params->ibuf; + + sym_op->cipher.data.offset = 0; + sym_op->cipher.data.length = QUOTE_512_BYTES; + + sym_op->auth.data.offset = 0; + sym_op->auth.data.length = QUOTE_512_BYTES; + + /* Process crypto operation */ + ut_params->op = process_crypto_request(ts_params->valid_devs[0], + ut_params->op); + TEST_ASSERT_NOT_NULL(ut_params->op, "no crypto operation returned"); + + TEST_ASSERT_EQUAL(ut_params->op->status, RTE_CRYPTO_OP_STATUS_SUCCESS, + "crypto operation processing failed"); + + /* Validate obuf */ + TEST_ASSERT_BUFFERS_ARE_EQUAL( + rte_pktmbuf_mtod(ut_params->op->sym->m_src, uint8_t *), + catch_22_quote, + QUOTE_512_BYTES, + "Ciphertext data not as expected"); + + return TEST_SUCCESS; +} + + +static int +test_null_invalid_operation(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + int ret; + + /* Setup Cipher Parameters */ + ut_params->cipher_xform.type = RTE_CRYPTO_SYM_XFORM_CIPHER; + ut_params->cipher_xform.next = NULL; + + ut_params->cipher_xform.cipher.algo = RTE_CRYPTO_CIPHER_AES_CBC; + ut_params->cipher_xform.cipher.op = RTE_CRYPTO_CIPHER_OP_ENCRYPT; + + ut_params->sess = rte_cryptodev_sym_session_create( + ts_params->session_mpool); + + /* Create Crypto session*/ + ret = rte_cryptodev_sym_session_init(ts_params->valid_devs[0], + ut_params->sess, &ut_params->cipher_xform, + ts_params->session_mpool); + TEST_ASSERT(ret < 0, + "Session creation succeeded unexpectedly"); + + + /* Setup HMAC Parameters */ + ut_params->auth_xform.type = RTE_CRYPTO_SYM_XFORM_AUTH; + ut_params->auth_xform.next = NULL; + + ut_params->auth_xform.auth.algo = RTE_CRYPTO_AUTH_SHA1_HMAC; + ut_params->auth_xform.auth.op = RTE_CRYPTO_AUTH_OP_GENERATE; + + ut_params->sess = rte_cryptodev_sym_session_create( + ts_params->session_mpool); + + /* Create Crypto session*/ + ret = rte_cryptodev_sym_session_init(ts_params->valid_devs[0], + ut_params->sess, &ut_params->auth_xform, + ts_params->session_mpool); + TEST_ASSERT(ret < 0, + "Session creation succeeded unexpectedly"); + + return TEST_SUCCESS; +} + + +#define NULL_BURST_LENGTH (32) + +static int +test_null_burst_operation(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + + unsigned i, burst_len = NULL_BURST_LENGTH; + + struct rte_crypto_op *burst[NULL_BURST_LENGTH] = { NULL }; + struct rte_crypto_op *burst_dequeued[NULL_BURST_LENGTH] = { NULL }; + + /* Setup Cipher Parameters */ + ut_params->cipher_xform.type = RTE_CRYPTO_SYM_XFORM_CIPHER; + ut_params->cipher_xform.next = &ut_params->auth_xform; + + ut_params->cipher_xform.cipher.algo = RTE_CRYPTO_CIPHER_NULL; + ut_params->cipher_xform.cipher.op = RTE_CRYPTO_CIPHER_OP_ENCRYPT; + + /* Setup HMAC Parameters */ + ut_params->auth_xform.type = RTE_CRYPTO_SYM_XFORM_AUTH; + ut_params->auth_xform.next = NULL; + + ut_params->auth_xform.auth.algo = RTE_CRYPTO_AUTH_NULL; + ut_params->auth_xform.auth.op = RTE_CRYPTO_AUTH_OP_GENERATE; + + ut_params->sess = rte_cryptodev_sym_session_create( + ts_params->session_mpool); + + /* Create Crypto session*/ + rte_cryptodev_sym_session_init(ts_params->valid_devs[0], + ut_params->sess, &ut_params->cipher_xform, + ts_params->session_mpool); + TEST_ASSERT_NOT_NULL(ut_params->sess, "Session creation failed"); + + TEST_ASSERT_EQUAL(rte_crypto_op_bulk_alloc(ts_params->op_mpool, + RTE_CRYPTO_OP_TYPE_SYMMETRIC, burst, burst_len), + burst_len, "failed to generate burst of crypto ops"); + + /* Generate an operation for each mbuf in burst */ + for (i = 0; i < burst_len; i++) { + struct rte_mbuf *m = rte_pktmbuf_alloc(ts_params->mbuf_pool); + + TEST_ASSERT_NOT_NULL(m, "Failed to allocate mbuf"); + + unsigned *data = (unsigned *)rte_pktmbuf_append(m, + sizeof(unsigned)); + *data = i; + + rte_crypto_op_attach_sym_session(burst[i], ut_params->sess); + + burst[i]->sym->m_src = m; + } + + /* Process crypto operation */ + TEST_ASSERT_EQUAL(rte_cryptodev_enqueue_burst(ts_params->valid_devs[0], + 0, burst, burst_len), + burst_len, + "Error enqueuing burst"); + + TEST_ASSERT_EQUAL(rte_cryptodev_dequeue_burst(ts_params->valid_devs[0], + 0, burst_dequeued, burst_len), + burst_len, + "Error dequeuing burst"); + + + for (i = 0; i < burst_len; i++) { + TEST_ASSERT_EQUAL( + *rte_pktmbuf_mtod(burst[i]->sym->m_src, uint32_t *), + *rte_pktmbuf_mtod(burst_dequeued[i]->sym->m_src, + uint32_t *), + "data not as expected"); + + rte_pktmbuf_free(burst[i]->sym->m_src); + rte_crypto_op_free(burst[i]); + } + + return TEST_SUCCESS; +} + +static void +generate_gmac_large_plaintext(uint8_t *data) +{ + uint16_t i; + + for (i = 32; i < GMAC_LARGE_PLAINTEXT_LENGTH; i += 32) + memcpy(&data[i], &data[0], 32); +} + +static int +create_gmac_operation(enum rte_crypto_auth_operation op, + const struct gmac_test_data *tdata) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + struct rte_crypto_sym_op *sym_op; + + uint32_t plaintext_pad_len = RTE_ALIGN_CEIL(tdata->plaintext.len, 16); + + /* Generate Crypto op data structure */ + ut_params->op = rte_crypto_op_alloc(ts_params->op_mpool, + RTE_CRYPTO_OP_TYPE_SYMMETRIC); + TEST_ASSERT_NOT_NULL(ut_params->op, + "Failed to allocate symmetric crypto operation struct"); + + sym_op = ut_params->op->sym; + + sym_op->auth.digest.data = (uint8_t *)rte_pktmbuf_append( + ut_params->ibuf, tdata->gmac_tag.len); + TEST_ASSERT_NOT_NULL(sym_op->auth.digest.data, + "no room to append digest"); + + sym_op->auth.digest.phys_addr = rte_pktmbuf_iova_offset( + ut_params->ibuf, plaintext_pad_len); + + if (op == RTE_CRYPTO_AUTH_OP_VERIFY) { + rte_memcpy(sym_op->auth.digest.data, tdata->gmac_tag.data, + tdata->gmac_tag.len); + TEST_HEXDUMP(stdout, "digest:", + sym_op->auth.digest.data, + tdata->gmac_tag.len); + } + + uint8_t *iv_ptr = rte_crypto_op_ctod_offset(ut_params->op, + uint8_t *, IV_OFFSET); + + rte_memcpy(iv_ptr, tdata->iv.data, tdata->iv.len); + + TEST_HEXDUMP(stdout, "iv:", iv_ptr, tdata->iv.len); + + sym_op->cipher.data.length = 0; + sym_op->cipher.data.offset = 0; + + sym_op->auth.data.offset = 0; + sym_op->auth.data.length = tdata->plaintext.len; + + return 0; +} + +static int create_gmac_session(uint8_t dev_id, + const struct gmac_test_data *tdata, + enum rte_crypto_auth_operation auth_op) +{ + uint8_t auth_key[tdata->key.len]; + + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + + memcpy(auth_key, tdata->key.data, tdata->key.len); + + ut_params->auth_xform.type = RTE_CRYPTO_SYM_XFORM_AUTH; + ut_params->auth_xform.next = NULL; + + ut_params->auth_xform.auth.algo = RTE_CRYPTO_AUTH_AES_GMAC; + ut_params->auth_xform.auth.op = auth_op; + ut_params->auth_xform.auth.digest_length = tdata->gmac_tag.len; + ut_params->auth_xform.auth.key.length = tdata->key.len; + ut_params->auth_xform.auth.key.data = auth_key; + ut_params->auth_xform.auth.iv.offset = IV_OFFSET; + ut_params->auth_xform.auth.iv.length = tdata->iv.len; + + + ut_params->sess = rte_cryptodev_sym_session_create( + ts_params->session_mpool); + + rte_cryptodev_sym_session_init(dev_id, ut_params->sess, + &ut_params->auth_xform, + ts_params->session_mpool); + + TEST_ASSERT_NOT_NULL(ut_params->sess, "Session creation failed"); + + return 0; +} + +static int +test_AES_GMAC_authentication(const struct gmac_test_data *tdata) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + + int retval; + + uint8_t *auth_tag, *plaintext; + uint16_t plaintext_pad_len; + + TEST_ASSERT_NOT_EQUAL(tdata->gmac_tag.len, 0, + "No GMAC length in the source data"); + + retval = create_gmac_session(ts_params->valid_devs[0], + tdata, RTE_CRYPTO_AUTH_OP_GENERATE); + + if (retval < 0) + return retval; + + if (tdata->plaintext.len > MBUF_SIZE) + ut_params->ibuf = rte_pktmbuf_alloc(ts_params->large_mbuf_pool); + else + ut_params->ibuf = rte_pktmbuf_alloc(ts_params->mbuf_pool); + TEST_ASSERT_NOT_NULL(ut_params->ibuf, + "Failed to allocate input buffer in mempool"); + + memset(rte_pktmbuf_mtod(ut_params->ibuf, uint8_t *), 0, + rte_pktmbuf_tailroom(ut_params->ibuf)); + + plaintext_pad_len = RTE_ALIGN_CEIL(tdata->plaintext.len, 16); + /* + * Runtime generate the large plain text instead of use hard code + * plain text vector. It is done to avoid create huge source file + * with the test vector. + */ + if (tdata->plaintext.len == GMAC_LARGE_PLAINTEXT_LENGTH) + generate_gmac_large_plaintext(tdata->plaintext.data); + + plaintext = (uint8_t *)rte_pktmbuf_append(ut_params->ibuf, + plaintext_pad_len); + TEST_ASSERT_NOT_NULL(plaintext, "no room to append plaintext"); + + memcpy(plaintext, tdata->plaintext.data, tdata->plaintext.len); + TEST_HEXDUMP(stdout, "plaintext:", plaintext, + tdata->plaintext.len); + + retval = create_gmac_operation(RTE_CRYPTO_AUTH_OP_GENERATE, + tdata); + + if (retval < 0) + return retval; + + rte_crypto_op_attach_sym_session(ut_params->op, ut_params->sess); + + ut_params->op->sym->m_src = ut_params->ibuf; + + TEST_ASSERT_NOT_NULL(process_crypto_request(ts_params->valid_devs[0], + ut_params->op), "failed to process sym crypto op"); + + TEST_ASSERT_EQUAL(ut_params->op->status, RTE_CRYPTO_OP_STATUS_SUCCESS, + "crypto op processing failed"); + + if (ut_params->op->sym->m_dst) { + auth_tag = rte_pktmbuf_mtod_offset(ut_params->op->sym->m_dst, + uint8_t *, plaintext_pad_len); + } else { + auth_tag = plaintext + plaintext_pad_len; + } + + TEST_HEXDUMP(stdout, "auth tag:", auth_tag, tdata->gmac_tag.len); + + TEST_ASSERT_BUFFERS_ARE_EQUAL( + auth_tag, + tdata->gmac_tag.data, + tdata->gmac_tag.len, + "GMAC Generated auth tag not as expected"); + + return 0; +} + +static int +test_AES_GMAC_authentication_test_case_1(void) +{ + return test_AES_GMAC_authentication(&gmac_test_case_1); +} + +static int +test_AES_GMAC_authentication_test_case_2(void) +{ + return test_AES_GMAC_authentication(&gmac_test_case_2); +} + +static int +test_AES_GMAC_authentication_test_case_3(void) +{ + return test_AES_GMAC_authentication(&gmac_test_case_3); +} + +static int +test_AES_GMAC_authentication_test_case_4(void) +{ + return test_AES_GMAC_authentication(&gmac_test_case_4); +} + +static int +test_AES_GMAC_authentication_verify(const struct gmac_test_data *tdata) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + int retval; + uint32_t plaintext_pad_len; + uint8_t *plaintext; + + TEST_ASSERT_NOT_EQUAL(tdata->gmac_tag.len, 0, + "No GMAC length in the source data"); + + retval = create_gmac_session(ts_params->valid_devs[0], + tdata, RTE_CRYPTO_AUTH_OP_VERIFY); + + if (retval < 0) + return retval; + + if (tdata->plaintext.len > MBUF_SIZE) + ut_params->ibuf = rte_pktmbuf_alloc(ts_params->large_mbuf_pool); + else + ut_params->ibuf = rte_pktmbuf_alloc(ts_params->mbuf_pool); + TEST_ASSERT_NOT_NULL(ut_params->ibuf, + "Failed to allocate input buffer in mempool"); + + memset(rte_pktmbuf_mtod(ut_params->ibuf, uint8_t *), 0, + rte_pktmbuf_tailroom(ut_params->ibuf)); + + plaintext_pad_len = RTE_ALIGN_CEIL(tdata->plaintext.len, 16); + + /* + * Runtime generate the large plain text instead of use hard code + * plain text vector. It is done to avoid create huge source file + * with the test vector. + */ + if (tdata->plaintext.len == GMAC_LARGE_PLAINTEXT_LENGTH) + generate_gmac_large_plaintext(tdata->plaintext.data); + + plaintext = (uint8_t *)rte_pktmbuf_append(ut_params->ibuf, + plaintext_pad_len); + TEST_ASSERT_NOT_NULL(plaintext, "no room to append plaintext"); + + memcpy(plaintext, tdata->plaintext.data, tdata->plaintext.len); + TEST_HEXDUMP(stdout, "plaintext:", plaintext, + tdata->plaintext.len); + + retval = create_gmac_operation(RTE_CRYPTO_AUTH_OP_VERIFY, + tdata); + + if (retval < 0) + return retval; + + rte_crypto_op_attach_sym_session(ut_params->op, ut_params->sess); + + ut_params->op->sym->m_src = ut_params->ibuf; + + TEST_ASSERT_NOT_NULL(process_crypto_request(ts_params->valid_devs[0], + ut_params->op), "failed to process sym crypto op"); + + TEST_ASSERT_EQUAL(ut_params->op->status, RTE_CRYPTO_OP_STATUS_SUCCESS, + "crypto op processing failed"); + + return 0; + +} + +static int +test_AES_GMAC_authentication_verify_test_case_1(void) +{ + return test_AES_GMAC_authentication_verify(&gmac_test_case_1); +} + +static int +test_AES_GMAC_authentication_verify_test_case_2(void) +{ + return test_AES_GMAC_authentication_verify(&gmac_test_case_2); +} + +static int +test_AES_GMAC_authentication_verify_test_case_3(void) +{ + return test_AES_GMAC_authentication_verify(&gmac_test_case_3); +} + +static int +test_AES_GMAC_authentication_verify_test_case_4(void) +{ + return test_AES_GMAC_authentication_verify(&gmac_test_case_4); +} + +struct test_crypto_vector { + enum rte_crypto_cipher_algorithm crypto_algo; + + struct { + uint8_t data[64]; + unsigned int len; + } cipher_key; + + struct { + uint8_t data[64]; + unsigned int len; + } iv; + + struct { + const uint8_t *data; + unsigned int len; + } plaintext; + + struct { + const uint8_t *data; + unsigned int len; + } ciphertext; + + enum rte_crypto_auth_algorithm auth_algo; + + struct { + uint8_t data[128]; + unsigned int len; + } auth_key; + + struct { + const uint8_t *data; + unsigned int len; + } aad; + + struct { + uint8_t data[128]; + unsigned int len; + } digest; +}; + +static const struct test_crypto_vector +hmac_sha1_test_crypto_vector = { + .auth_algo = RTE_CRYPTO_AUTH_SHA1_HMAC, + .plaintext = { + .data = plaintext_hash, + .len = 512 + }, + .auth_key = { + .data = { + 0xF8, 0x2A, 0xC7, 0x54, 0xDB, 0x96, 0x18, 0xAA, + 0xC3, 0xA1, 0x53, 0xF6, 0x1F, 0x17, 0x60, 0xBD, + 0xDE, 0xF4, 0xDE, 0xAD + }, + .len = 20 + }, + .digest = { + .data = { + 0xC4, 0xB7, 0x0E, 0x6B, 0xDE, 0xD1, 0xE7, 0x77, + 0x7E, 0x2E, 0x8F, 0xFC, 0x48, 0x39, 0x46, 0x17, + 0x3F, 0x91, 0x64, 0x59 + }, + .len = 20 + } +}; + +static const struct test_crypto_vector +aes128_gmac_test_vector = { + .auth_algo = RTE_CRYPTO_AUTH_AES_GMAC, + .plaintext = { + .data = plaintext_hash, + .len = 512 + }, + .iv = { + .data = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0A, 0x0B + }, + .len = 12 + }, + .auth_key = { + .data = { + 0x42, 0x1A, 0x7D, 0x3D, 0xF5, 0x82, 0x80, 0xF1, + 0xF1, 0x35, 0x5C, 0x3B, 0xDD, 0x9A, 0x65, 0xBA + }, + .len = 16 + }, + .digest = { + .data = { + 0xCA, 0x00, 0x99, 0x8B, 0x30, 0x7E, 0x74, 0x56, + 0x32, 0xA7, 0x87, 0xB5, 0xE9, 0xB2, 0x34, 0x5A + }, + .len = 16 + } +}; + +static const struct test_crypto_vector +aes128cbc_hmac_sha1_test_vector = { + .crypto_algo = RTE_CRYPTO_CIPHER_AES_CBC, + .cipher_key = { + .data = { + 0xE4, 0x23, 0x33, 0x8A, 0x35, 0x64, 0x61, 0xE2, + 0x49, 0x03, 0xDD, 0xC6, 0xB8, 0xCA, 0x55, 0x7A + }, + .len = 16 + }, + .iv = { + .data = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F + }, + .len = 16 + }, + .plaintext = { + .data = plaintext_hash, + .len = 512 + }, + .ciphertext = { + .data = ciphertext512_aes128cbc, + .len = 512 + }, + .auth_algo = RTE_CRYPTO_AUTH_SHA1_HMAC, + .auth_key = { + .data = { + 0xF8, 0x2A, 0xC7, 0x54, 0xDB, 0x96, 0x18, 0xAA, + 0xC3, 0xA1, 0x53, 0xF6, 0x1F, 0x17, 0x60, 0xBD, + 0xDE, 0xF4, 0xDE, 0xAD + }, + .len = 20 + }, + .digest = { + .data = { + 0x9A, 0x4F, 0x88, 0x1B, 0xB6, 0x8F, 0xD8, 0x60, + 0x42, 0x1A, 0x7D, 0x3D, 0xF5, 0x82, 0x80, 0xF1, + 0x18, 0x8C, 0x1D, 0x32 + }, + .len = 20 + } +}; + +static void +data_corruption(uint8_t *data) +{ + data[0] += 1; +} + +static void +tag_corruption(uint8_t *data, unsigned int tag_offset) +{ + data[tag_offset] += 1; +} + +static int +create_auth_session(struct crypto_unittest_params *ut_params, + uint8_t dev_id, + const struct test_crypto_vector *reference, + enum rte_crypto_auth_operation auth_op) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + uint8_t auth_key[reference->auth_key.len + 1]; + + memcpy(auth_key, reference->auth_key.data, reference->auth_key.len); + + /* Setup Authentication Parameters */ + ut_params->auth_xform.type = RTE_CRYPTO_SYM_XFORM_AUTH; + ut_params->auth_xform.auth.op = auth_op; + ut_params->auth_xform.next = NULL; + ut_params->auth_xform.auth.algo = reference->auth_algo; + ut_params->auth_xform.auth.key.length = reference->auth_key.len; + ut_params->auth_xform.auth.key.data = auth_key; + ut_params->auth_xform.auth.digest_length = reference->digest.len; + + /* Create Crypto session*/ + ut_params->sess = rte_cryptodev_sym_session_create( + ts_params->session_mpool); + + rte_cryptodev_sym_session_init(dev_id, ut_params->sess, + &ut_params->auth_xform, + ts_params->session_mpool); + + TEST_ASSERT_NOT_NULL(ut_params->sess, "Session creation failed"); + + return 0; +} + +static int +create_auth_cipher_session(struct crypto_unittest_params *ut_params, + uint8_t dev_id, + const struct test_crypto_vector *reference, + enum rte_crypto_auth_operation auth_op, + enum rte_crypto_cipher_operation cipher_op) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + uint8_t cipher_key[reference->cipher_key.len + 1]; + uint8_t auth_key[reference->auth_key.len + 1]; + + memcpy(cipher_key, reference->cipher_key.data, + reference->cipher_key.len); + memcpy(auth_key, reference->auth_key.data, reference->auth_key.len); + + /* Setup Authentication Parameters */ + ut_params->auth_xform.type = RTE_CRYPTO_SYM_XFORM_AUTH; + ut_params->auth_xform.auth.op = auth_op; + ut_params->auth_xform.auth.algo = reference->auth_algo; + ut_params->auth_xform.auth.key.length = reference->auth_key.len; + ut_params->auth_xform.auth.key.data = auth_key; + ut_params->auth_xform.auth.digest_length = reference->digest.len; + + if (reference->auth_algo == RTE_CRYPTO_AUTH_AES_GMAC) { + ut_params->auth_xform.auth.iv.offset = IV_OFFSET; + ut_params->auth_xform.auth.iv.length = reference->iv.len; + } else { + ut_params->auth_xform.next = &ut_params->cipher_xform; + + /* Setup Cipher Parameters */ + ut_params->cipher_xform.type = RTE_CRYPTO_SYM_XFORM_CIPHER; + ut_params->cipher_xform.next = NULL; + ut_params->cipher_xform.cipher.algo = reference->crypto_algo; + ut_params->cipher_xform.cipher.op = cipher_op; + ut_params->cipher_xform.cipher.key.data = cipher_key; + ut_params->cipher_xform.cipher.key.length = reference->cipher_key.len; + ut_params->cipher_xform.cipher.iv.offset = IV_OFFSET; + ut_params->cipher_xform.cipher.iv.length = reference->iv.len; + } + + /* Create Crypto session*/ + ut_params->sess = rte_cryptodev_sym_session_create( + ts_params->session_mpool); + + rte_cryptodev_sym_session_init(dev_id, ut_params->sess, + &ut_params->auth_xform, + ts_params->session_mpool); + + TEST_ASSERT_NOT_NULL(ut_params->sess, "Session creation failed"); + + return 0; +} + +static int +create_auth_operation(struct crypto_testsuite_params *ts_params, + struct crypto_unittest_params *ut_params, + const struct test_crypto_vector *reference, + unsigned int auth_generate) +{ + /* Generate Crypto op data structure */ + ut_params->op = rte_crypto_op_alloc(ts_params->op_mpool, + RTE_CRYPTO_OP_TYPE_SYMMETRIC); + TEST_ASSERT_NOT_NULL(ut_params->op, + "Failed to allocate pktmbuf offload"); + + /* Set crypto operation data parameters */ + rte_crypto_op_attach_sym_session(ut_params->op, ut_params->sess); + + struct rte_crypto_sym_op *sym_op = ut_params->op->sym; + + /* set crypto operation source mbuf */ + sym_op->m_src = ut_params->ibuf; + + /* digest */ + sym_op->auth.digest.data = (uint8_t *)rte_pktmbuf_append( + ut_params->ibuf, reference->digest.len); + + TEST_ASSERT_NOT_NULL(sym_op->auth.digest.data, + "no room to append auth tag"); + + sym_op->auth.digest.phys_addr = rte_pktmbuf_iova_offset( + ut_params->ibuf, reference->plaintext.len); + + if (auth_generate) + memset(sym_op->auth.digest.data, 0, reference->digest.len); + else + memcpy(sym_op->auth.digest.data, + reference->digest.data, + reference->digest.len); + + TEST_HEXDUMP(stdout, "digest:", + sym_op->auth.digest.data, + reference->digest.len); + + sym_op->auth.data.length = reference->plaintext.len; + sym_op->auth.data.offset = 0; + + return 0; +} + +static int +create_auth_GMAC_operation(struct crypto_testsuite_params *ts_params, + struct crypto_unittest_params *ut_params, + const struct test_crypto_vector *reference, + unsigned int auth_generate) +{ + /* Generate Crypto op data structure */ + ut_params->op = rte_crypto_op_alloc(ts_params->op_mpool, + RTE_CRYPTO_OP_TYPE_SYMMETRIC); + TEST_ASSERT_NOT_NULL(ut_params->op, + "Failed to allocate pktmbuf offload"); + + /* Set crypto operation data parameters */ + rte_crypto_op_attach_sym_session(ut_params->op, ut_params->sess); + + struct rte_crypto_sym_op *sym_op = ut_params->op->sym; + + /* set crypto operation source mbuf */ + sym_op->m_src = ut_params->ibuf; + + /* digest */ + sym_op->auth.digest.data = (uint8_t *)rte_pktmbuf_append( + ut_params->ibuf, reference->digest.len); + + TEST_ASSERT_NOT_NULL(sym_op->auth.digest.data, + "no room to append auth tag"); + + sym_op->auth.digest.phys_addr = rte_pktmbuf_iova_offset( + ut_params->ibuf, reference->ciphertext.len); + + if (auth_generate) + memset(sym_op->auth.digest.data, 0, reference->digest.len); + else + memcpy(sym_op->auth.digest.data, + reference->digest.data, + reference->digest.len); + + TEST_HEXDUMP(stdout, "digest:", + sym_op->auth.digest.data, + reference->digest.len); + + rte_memcpy(rte_crypto_op_ctod_offset(ut_params->op, uint8_t *, IV_OFFSET), + reference->iv.data, reference->iv.len); + + sym_op->cipher.data.length = 0; + sym_op->cipher.data.offset = 0; + + sym_op->auth.data.length = reference->plaintext.len; + sym_op->auth.data.offset = 0; + + return 0; +} + +static int +create_cipher_auth_operation(struct crypto_testsuite_params *ts_params, + struct crypto_unittest_params *ut_params, + const struct test_crypto_vector *reference, + unsigned int auth_generate) +{ + /* Generate Crypto op data structure */ + ut_params->op = rte_crypto_op_alloc(ts_params->op_mpool, + RTE_CRYPTO_OP_TYPE_SYMMETRIC); + TEST_ASSERT_NOT_NULL(ut_params->op, + "Failed to allocate pktmbuf offload"); + + /* Set crypto operation data parameters */ + rte_crypto_op_attach_sym_session(ut_params->op, ut_params->sess); + + struct rte_crypto_sym_op *sym_op = ut_params->op->sym; + + /* set crypto operation source mbuf */ + sym_op->m_src = ut_params->ibuf; + + /* digest */ + sym_op->auth.digest.data = (uint8_t *)rte_pktmbuf_append( + ut_params->ibuf, reference->digest.len); + + TEST_ASSERT_NOT_NULL(sym_op->auth.digest.data, + "no room to append auth tag"); + + sym_op->auth.digest.phys_addr = rte_pktmbuf_iova_offset( + ut_params->ibuf, reference->ciphertext.len); + + if (auth_generate) + memset(sym_op->auth.digest.data, 0, reference->digest.len); + else + memcpy(sym_op->auth.digest.data, + reference->digest.data, + reference->digest.len); + + TEST_HEXDUMP(stdout, "digest:", + sym_op->auth.digest.data, + reference->digest.len); + + rte_memcpy(rte_crypto_op_ctod_offset(ut_params->op, uint8_t *, IV_OFFSET), + reference->iv.data, reference->iv.len); + + sym_op->cipher.data.length = reference->ciphertext.len; + sym_op->cipher.data.offset = 0; + + sym_op->auth.data.length = reference->ciphertext.len; + sym_op->auth.data.offset = 0; + + return 0; +} + +static int +create_auth_verify_operation(struct crypto_testsuite_params *ts_params, + struct crypto_unittest_params *ut_params, + const struct test_crypto_vector *reference) +{ + return create_auth_operation(ts_params, ut_params, reference, 0); +} + +static int +create_auth_verify_GMAC_operation( + struct crypto_testsuite_params *ts_params, + struct crypto_unittest_params *ut_params, + const struct test_crypto_vector *reference) +{ + return create_auth_GMAC_operation(ts_params, ut_params, reference, 0); +} + +static int +create_cipher_auth_verify_operation(struct crypto_testsuite_params *ts_params, + struct crypto_unittest_params *ut_params, + const struct test_crypto_vector *reference) +{ + return create_cipher_auth_operation(ts_params, ut_params, reference, 0); +} + +static int +test_authentication_verify_fail_when_data_corruption( + struct crypto_testsuite_params *ts_params, + struct crypto_unittest_params *ut_params, + const struct test_crypto_vector *reference, + unsigned int data_corrupted) +{ + int retval; + + uint8_t *plaintext; + + /* Create session */ + retval = create_auth_session(ut_params, + ts_params->valid_devs[0], + reference, + RTE_CRYPTO_AUTH_OP_VERIFY); + if (retval < 0) + return retval; + + ut_params->ibuf = rte_pktmbuf_alloc(ts_params->mbuf_pool); + TEST_ASSERT_NOT_NULL(ut_params->ibuf, + "Failed to allocate input buffer in mempool"); + + /* clear mbuf payload */ + memset(rte_pktmbuf_mtod(ut_params->ibuf, uint8_t *), 0, + rte_pktmbuf_tailroom(ut_params->ibuf)); + + plaintext = (uint8_t *)rte_pktmbuf_append(ut_params->ibuf, + reference->plaintext.len); + TEST_ASSERT_NOT_NULL(plaintext, "no room to append plaintext"); + memcpy(plaintext, reference->plaintext.data, reference->plaintext.len); + + TEST_HEXDUMP(stdout, "plaintext:", plaintext, reference->plaintext.len); + + /* Create operation */ + retval = create_auth_verify_operation(ts_params, ut_params, reference); + + if (retval < 0) + return retval; + + if (data_corrupted) + data_corruption(plaintext); + else + tag_corruption(plaintext, reference->plaintext.len); + + ut_params->op = process_crypto_request(ts_params->valid_devs[0], + ut_params->op); + TEST_ASSERT_NOT_NULL(ut_params->op, "failed crypto process"); + TEST_ASSERT_EQUAL(ut_params->op->status, + RTE_CRYPTO_OP_STATUS_AUTH_FAILED, + "authentication not failed"); + + ut_params->obuf = ut_params->op->sym->m_src; + TEST_ASSERT_NOT_NULL(ut_params->obuf, "failed to retrieve obuf"); + + return 0; +} + +static int +test_authentication_verify_GMAC_fail_when_corruption( + struct crypto_testsuite_params *ts_params, + struct crypto_unittest_params *ut_params, + const struct test_crypto_vector *reference, + unsigned int data_corrupted) +{ + int retval; + uint8_t *plaintext; + + /* Create session */ + retval = create_auth_cipher_session(ut_params, + ts_params->valid_devs[0], + reference, + RTE_CRYPTO_AUTH_OP_VERIFY, + RTE_CRYPTO_CIPHER_OP_DECRYPT); + if (retval < 0) + return retval; + + ut_params->ibuf = rte_pktmbuf_alloc(ts_params->mbuf_pool); + TEST_ASSERT_NOT_NULL(ut_params->ibuf, + "Failed to allocate input buffer in mempool"); + + /* clear mbuf payload */ + memset(rte_pktmbuf_mtod(ut_params->ibuf, uint8_t *), 0, + rte_pktmbuf_tailroom(ut_params->ibuf)); + + plaintext = (uint8_t *)rte_pktmbuf_append(ut_params->ibuf, + reference->plaintext.len); + TEST_ASSERT_NOT_NULL(plaintext, "no room to append plaintext"); + memcpy(plaintext, reference->plaintext.data, reference->plaintext.len); + + TEST_HEXDUMP(stdout, "plaintext:", plaintext, reference->plaintext.len); + + /* Create operation */ + retval = create_auth_verify_GMAC_operation(ts_params, + ut_params, + reference); + + if (retval < 0) + return retval; + + if (data_corrupted) + data_corruption(plaintext); + else + tag_corruption(plaintext, reference->aad.len); + + ut_params->op = process_crypto_request(ts_params->valid_devs[0], + ut_params->op); + TEST_ASSERT_NOT_NULL(ut_params->op, "failed crypto process"); + TEST_ASSERT_EQUAL(ut_params->op->status, + RTE_CRYPTO_OP_STATUS_AUTH_FAILED, + "authentication not failed"); + + ut_params->obuf = ut_params->op->sym->m_src; + TEST_ASSERT_NOT_NULL(ut_params->obuf, "failed to retrieve obuf"); + + return 0; +} + +static int +test_authenticated_decryption_fail_when_corruption( + struct crypto_testsuite_params *ts_params, + struct crypto_unittest_params *ut_params, + const struct test_crypto_vector *reference, + unsigned int data_corrupted) +{ + int retval; + + uint8_t *ciphertext; + + /* Create session */ + retval = create_auth_cipher_session(ut_params, + ts_params->valid_devs[0], + reference, + RTE_CRYPTO_AUTH_OP_VERIFY, + RTE_CRYPTO_CIPHER_OP_DECRYPT); + if (retval < 0) + return retval; + + ut_params->ibuf = rte_pktmbuf_alloc(ts_params->mbuf_pool); + TEST_ASSERT_NOT_NULL(ut_params->ibuf, + "Failed to allocate input buffer in mempool"); + + /* clear mbuf payload */ + memset(rte_pktmbuf_mtod(ut_params->ibuf, uint8_t *), 0, + rte_pktmbuf_tailroom(ut_params->ibuf)); + + ciphertext = (uint8_t *)rte_pktmbuf_append(ut_params->ibuf, + reference->ciphertext.len); + TEST_ASSERT_NOT_NULL(ciphertext, "no room to append ciphertext"); + memcpy(ciphertext, reference->ciphertext.data, + reference->ciphertext.len); + + /* Create operation */ + retval = create_cipher_auth_verify_operation(ts_params, + ut_params, + reference); + + if (retval < 0) + return retval; + + if (data_corrupted) + data_corruption(ciphertext); + else + tag_corruption(ciphertext, reference->ciphertext.len); + + ut_params->op = process_crypto_request(ts_params->valid_devs[0], + ut_params->op); + + TEST_ASSERT_NOT_NULL(ut_params->op, "failed crypto process"); + TEST_ASSERT_EQUAL(ut_params->op->status, + RTE_CRYPTO_OP_STATUS_AUTH_FAILED, + "authentication not failed"); + + ut_params->obuf = ut_params->op->sym->m_src; + TEST_ASSERT_NOT_NULL(ut_params->obuf, "failed to retrieve obuf"); + + return 0; +} + +static int +create_aead_operation_SGL(enum rte_crypto_aead_operation op, + const struct aead_test_data *tdata, + void *digest_mem, uint64_t digest_phys) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + + const unsigned int auth_tag_len = tdata->auth_tag.len; + const unsigned int iv_len = tdata->iv.len; + unsigned int aad_len = tdata->aad.len; + + /* Generate Crypto op data structure */ + ut_params->op = rte_crypto_op_alloc(ts_params->op_mpool, + RTE_CRYPTO_OP_TYPE_SYMMETRIC); + TEST_ASSERT_NOT_NULL(ut_params->op, + "Failed to allocate symmetric crypto operation struct"); + + struct rte_crypto_sym_op *sym_op = ut_params->op->sym; + + sym_op->aead.digest.data = digest_mem; + + TEST_ASSERT_NOT_NULL(sym_op->aead.digest.data, + "no room to append digest"); + + sym_op->aead.digest.phys_addr = digest_phys; + + if (op == RTE_CRYPTO_AEAD_OP_DECRYPT) { + rte_memcpy(sym_op->aead.digest.data, tdata->auth_tag.data, + auth_tag_len); + TEST_HEXDUMP(stdout, "digest:", + sym_op->aead.digest.data, + auth_tag_len); + } + + /* Append aad data */ + if (tdata->algo == RTE_CRYPTO_AEAD_AES_CCM) { + uint8_t *iv_ptr = rte_crypto_op_ctod_offset(ut_params->op, + uint8_t *, IV_OFFSET); + + /* Copy IV 1 byte after the IV pointer, according to the API */ + rte_memcpy(iv_ptr + 1, tdata->iv.data, iv_len); + + aad_len = RTE_ALIGN_CEIL(aad_len + 18, 16); + + sym_op->aead.aad.data = (uint8_t *)rte_pktmbuf_prepend( + ut_params->ibuf, aad_len); + TEST_ASSERT_NOT_NULL(sym_op->aead.aad.data, + "no room to prepend aad"); + sym_op->aead.aad.phys_addr = rte_pktmbuf_iova( + ut_params->ibuf); + + memset(sym_op->aead.aad.data, 0, aad_len); + /* Copy AAD 18 bytes after the AAD pointer, according to the API */ + rte_memcpy(sym_op->aead.aad.data, tdata->aad.data, aad_len); + + TEST_HEXDUMP(stdout, "iv:", iv_ptr, iv_len); + TEST_HEXDUMP(stdout, "aad:", + sym_op->aead.aad.data, aad_len); + } else { + uint8_t *iv_ptr = rte_crypto_op_ctod_offset(ut_params->op, + uint8_t *, IV_OFFSET); + + rte_memcpy(iv_ptr, tdata->iv.data, iv_len); + + sym_op->aead.aad.data = (uint8_t *)rte_pktmbuf_prepend( + ut_params->ibuf, aad_len); + TEST_ASSERT_NOT_NULL(sym_op->aead.aad.data, + "no room to prepend aad"); + sym_op->aead.aad.phys_addr = rte_pktmbuf_iova( + ut_params->ibuf); + + memset(sym_op->aead.aad.data, 0, aad_len); + rte_memcpy(sym_op->aead.aad.data, tdata->aad.data, aad_len); + + TEST_HEXDUMP(stdout, "iv:", iv_ptr, iv_len); + TEST_HEXDUMP(stdout, "aad:", + sym_op->aead.aad.data, aad_len); + } + + sym_op->aead.data.length = tdata->plaintext.len; + sym_op->aead.data.offset = aad_len; + + return 0; +} + +#define SGL_MAX_NO 16 + +static int +test_authenticated_encryption_SGL(const struct aead_test_data *tdata, + const int oop, uint32_t fragsz, uint32_t fragsz_oop) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + struct crypto_unittest_params *ut_params = &unittest_params; + struct rte_mbuf *buf, *buf_oop = NULL, *buf_last_oop = NULL; + int retval; + int to_trn = 0; + int to_trn_tbl[SGL_MAX_NO]; + int segs = 1; + unsigned int trn_data = 0; + uint8_t *plaintext, *ciphertext, *auth_tag; + + if (fragsz > tdata->plaintext.len) + fragsz = tdata->plaintext.len; + + uint16_t plaintext_len = fragsz; + uint16_t frag_size_oop = fragsz_oop ? fragsz_oop : fragsz; + + if (fragsz_oop > tdata->plaintext.len) + frag_size_oop = tdata->plaintext.len; + + int ecx = 0; + void *digest_mem = NULL; + + uint32_t prepend_len = tdata->aad.len; + + if (tdata->plaintext.len % fragsz != 0) { + if (tdata->plaintext.len / fragsz + 1 > SGL_MAX_NO) + return 1; + } else { + if (tdata->plaintext.len / fragsz > SGL_MAX_NO) + return 1; + } + + /* + * For out-op-place we need to alloc another mbuf + */ + if (oop) { + ut_params->obuf = rte_pktmbuf_alloc(ts_params->mbuf_pool); + rte_pktmbuf_append(ut_params->obuf, + frag_size_oop + prepend_len); + buf_oop = ut_params->obuf; + } + + /* Create AEAD session */ + retval = create_aead_session(ts_params->valid_devs[0], + tdata->algo, + RTE_CRYPTO_AEAD_OP_ENCRYPT, + tdata->key.data, tdata->key.len, + tdata->aad.len, tdata->auth_tag.len, + tdata->iv.len); + if (retval < 0) + return retval; + + ut_params->ibuf = rte_pktmbuf_alloc(ts_params->mbuf_pool); + + /* clear mbuf payload */ + memset(rte_pktmbuf_mtod(ut_params->ibuf, uint8_t *), 0, + rte_pktmbuf_tailroom(ut_params->ibuf)); + + plaintext = (uint8_t *)rte_pktmbuf_append(ut_params->ibuf, + plaintext_len); + + memcpy(plaintext, tdata->plaintext.data, plaintext_len); + + trn_data += plaintext_len; + + buf = ut_params->ibuf; + + /* + * Loop until no more fragments + */ + + while (trn_data < tdata->plaintext.len) { + ++segs; + to_trn = (tdata->plaintext.len - trn_data < fragsz) ? + (tdata->plaintext.len - trn_data) : fragsz; + + to_trn_tbl[ecx++] = to_trn; + + buf->next = rte_pktmbuf_alloc(ts_params->mbuf_pool); + buf = buf->next; + + memset(rte_pktmbuf_mtod(buf, uint8_t *), 0, + rte_pktmbuf_tailroom(buf)); + + /* OOP */ + if (oop && !fragsz_oop) { + buf_last_oop = buf_oop->next = + rte_pktmbuf_alloc(ts_params->mbuf_pool); + buf_oop = buf_oop->next; + memset(rte_pktmbuf_mtod(buf_oop, uint8_t *), + 0, rte_pktmbuf_tailroom(buf_oop)); + rte_pktmbuf_append(buf_oop, to_trn); + } + + plaintext = (uint8_t *)rte_pktmbuf_append(buf, + to_trn); + + memcpy(plaintext, tdata->plaintext.data + trn_data, + to_trn); + trn_data += to_trn; + if (trn_data == tdata->plaintext.len) { + if (oop) { + if (!fragsz_oop) + digest_mem = rte_pktmbuf_append(buf_oop, + tdata->auth_tag.len); + } else + digest_mem = (uint8_t *)rte_pktmbuf_append(buf, + tdata->auth_tag.len); + } + } + + uint64_t digest_phys = 0; + + ut_params->ibuf->nb_segs = segs; + + segs = 1; + if (fragsz_oop && oop) { + to_trn = 0; + ecx = 0; + + if (frag_size_oop == tdata->plaintext.len) { + digest_mem = rte_pktmbuf_append(ut_params->obuf, + tdata->auth_tag.len); + + digest_phys = rte_pktmbuf_iova_offset( + ut_params->obuf, + tdata->plaintext.len + prepend_len); + } + + trn_data = frag_size_oop; + while (trn_data < tdata->plaintext.len) { + ++segs; + to_trn = + (tdata->plaintext.len - trn_data < + frag_size_oop) ? + (tdata->plaintext.len - trn_data) : + frag_size_oop; + + to_trn_tbl[ecx++] = to_trn; + + buf_last_oop = buf_oop->next = + rte_pktmbuf_alloc(ts_params->mbuf_pool); + buf_oop = buf_oop->next; + memset(rte_pktmbuf_mtod(buf_oop, uint8_t *), + 0, rte_pktmbuf_tailroom(buf_oop)); + rte_pktmbuf_append(buf_oop, to_trn); + + trn_data += to_trn; + + if (trn_data == tdata->plaintext.len) { + digest_mem = rte_pktmbuf_append(buf_oop, + tdata->auth_tag.len); + } + } + + ut_params->obuf->nb_segs = segs; + } + + /* + * Place digest at the end of the last buffer + */ + if (!digest_phys) + digest_phys = rte_pktmbuf_iova(buf) + to_trn; + if (oop && buf_last_oop) + digest_phys = rte_pktmbuf_iova(buf_last_oop) + to_trn; + + if (!digest_mem && !oop) { + digest_mem = (uint8_t *)rte_pktmbuf_append(ut_params->ibuf, + + tdata->auth_tag.len); + digest_phys = rte_pktmbuf_iova_offset(ut_params->ibuf, + tdata->plaintext.len); + } + + /* Create AEAD operation */ + retval = create_aead_operation_SGL(RTE_CRYPTO_AEAD_OP_ENCRYPT, + tdata, digest_mem, digest_phys); + + if (retval < 0) + return retval; + + rte_crypto_op_attach_sym_session(ut_params->op, ut_params->sess); + + ut_params->op->sym->m_src = ut_params->ibuf; + if (oop) + ut_params->op->sym->m_dst = ut_params->obuf; + + /* Process crypto operation */ + TEST_ASSERT_NOT_NULL(process_crypto_request(ts_params->valid_devs[0], + ut_params->op), "failed to process sym crypto op"); + + TEST_ASSERT_EQUAL(ut_params->op->status, RTE_CRYPTO_OP_STATUS_SUCCESS, + "crypto op processing failed"); + + + ciphertext = rte_pktmbuf_mtod_offset(ut_params->op->sym->m_src, + uint8_t *, prepend_len); + if (oop) { + ciphertext = rte_pktmbuf_mtod_offset(ut_params->op->sym->m_dst, + uint8_t *, prepend_len); + } + + if (fragsz_oop) + fragsz = fragsz_oop; + + TEST_ASSERT_BUFFERS_ARE_EQUAL( + ciphertext, + tdata->ciphertext.data, + fragsz, + "Ciphertext data not as expected"); + + buf = ut_params->op->sym->m_src->next; + if (oop) + buf = ut_params->op->sym->m_dst->next; + + unsigned int off = fragsz; + + ecx = 0; + while (buf) { + ciphertext = rte_pktmbuf_mtod(buf, + uint8_t *); + + TEST_ASSERT_BUFFERS_ARE_EQUAL( + ciphertext, + tdata->ciphertext.data + off, + to_trn_tbl[ecx], + "Ciphertext data not as expected"); + + off += to_trn_tbl[ecx++]; + buf = buf->next; + } + + auth_tag = digest_mem; + TEST_ASSERT_BUFFERS_ARE_EQUAL( + auth_tag, + tdata->auth_tag.data, + tdata->auth_tag.len, + "Generated auth tag not as expected"); + + return 0; +} + +#define IN_PLACE 0 +#define OUT_OF_PLACE 1 + +static int +test_AES_GCM_auth_encrypt_SGL_out_of_place_400B_400B(void) +{ + return test_authenticated_encryption_SGL( + &gcm_test_case_SGL_1, OUT_OF_PLACE, 400, 400); +} + +static int +test_AES_GCM_auth_encrypt_SGL_out_of_place_1500B_2000B(void) +{ + return test_authenticated_encryption_SGL( + &gcm_test_case_SGL_1, OUT_OF_PLACE, 1500, 2000); +} + +static int +test_AES_GCM_auth_encrypt_SGL_out_of_place_400B_1seg(void) +{ + return test_authenticated_encryption_SGL( + &gcm_test_case_8, OUT_OF_PLACE, 400, + gcm_test_case_8.plaintext.len); +} + +static int +test_AES_GCM_auth_encrypt_SGL_in_place_1500B(void) +{ + + return test_authenticated_encryption_SGL( + &gcm_test_case_SGL_1, IN_PLACE, 1500, 0); +} + +static int +test_authentication_verify_fail_when_data_corrupted( + struct crypto_testsuite_params *ts_params, + struct crypto_unittest_params *ut_params, + const struct test_crypto_vector *reference) +{ + return test_authentication_verify_fail_when_data_corruption( + ts_params, ut_params, reference, 1); +} + +static int +test_authentication_verify_fail_when_tag_corrupted( + struct crypto_testsuite_params *ts_params, + struct crypto_unittest_params *ut_params, + const struct test_crypto_vector *reference) +{ + return test_authentication_verify_fail_when_data_corruption( + ts_params, ut_params, reference, 0); +} + +static int +test_authentication_verify_GMAC_fail_when_data_corrupted( + struct crypto_testsuite_params *ts_params, + struct crypto_unittest_params *ut_params, + const struct test_crypto_vector *reference) +{ + return test_authentication_verify_GMAC_fail_when_corruption( + ts_params, ut_params, reference, 1); +} + +static int +test_authentication_verify_GMAC_fail_when_tag_corrupted( + struct crypto_testsuite_params *ts_params, + struct crypto_unittest_params *ut_params, + const struct test_crypto_vector *reference) +{ + return test_authentication_verify_GMAC_fail_when_corruption( + ts_params, ut_params, reference, 0); +} + +static int +test_authenticated_decryption_fail_when_data_corrupted( + struct crypto_testsuite_params *ts_params, + struct crypto_unittest_params *ut_params, + const struct test_crypto_vector *reference) +{ + return test_authenticated_decryption_fail_when_corruption( + ts_params, ut_params, reference, 1); +} + +static int +test_authenticated_decryption_fail_when_tag_corrupted( + struct crypto_testsuite_params *ts_params, + struct crypto_unittest_params *ut_params, + const struct test_crypto_vector *reference) +{ + return test_authenticated_decryption_fail_when_corruption( + ts_params, ut_params, reference, 0); +} + +static int +authentication_verify_HMAC_SHA1_fail_data_corrupt(void) +{ + return test_authentication_verify_fail_when_data_corrupted( + &testsuite_params, &unittest_params, + &hmac_sha1_test_crypto_vector); +} + +static int +authentication_verify_HMAC_SHA1_fail_tag_corrupt(void) +{ + return test_authentication_verify_fail_when_tag_corrupted( + &testsuite_params, &unittest_params, + &hmac_sha1_test_crypto_vector); +} + +static int +authentication_verify_AES128_GMAC_fail_data_corrupt(void) +{ + return test_authentication_verify_GMAC_fail_when_data_corrupted( + &testsuite_params, &unittest_params, + &aes128_gmac_test_vector); +} + +static int +authentication_verify_AES128_GMAC_fail_tag_corrupt(void) +{ + return test_authentication_verify_GMAC_fail_when_tag_corrupted( + &testsuite_params, &unittest_params, + &aes128_gmac_test_vector); +} + +static int +auth_decryption_AES128CBC_HMAC_SHA1_fail_data_corrupt(void) +{ + return test_authenticated_decryption_fail_when_data_corrupted( + &testsuite_params, + &unittest_params, + &aes128cbc_hmac_sha1_test_vector); +} + +static int +auth_decryption_AES128CBC_HMAC_SHA1_fail_tag_corrupt(void) +{ + return test_authenticated_decryption_fail_when_tag_corrupted( + &testsuite_params, + &unittest_params, + &aes128cbc_hmac_sha1_test_vector); +} + +#ifdef RTE_LIBRTE_PMD_CRYPTO_SCHEDULER + +/* global AESNI slave IDs for the scheduler test */ +uint8_t aesni_ids[2]; + +static int +test_scheduler_attach_slave_op(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + uint8_t sched_id = ts_params->valid_devs[0]; + uint32_t nb_devs, i, nb_devs_attached = 0; + int ret; + char vdev_name[32]; + + /* create 2 AESNI_MB if necessary */ + nb_devs = rte_cryptodev_device_count_by_driver( + rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_AESNI_MB_PMD))); + if (nb_devs < 2) { + for (i = nb_devs; i < 2; i++) { + snprintf(vdev_name, sizeof(vdev_name), "%s_%u", + RTE_STR(CRYPTODEV_NAME_AESNI_MB_PMD), + i); + ret = rte_vdev_init(vdev_name, NULL); + + TEST_ASSERT(ret == 0, + "Failed to create instance %u of" + " pmd : %s", + i, RTE_STR(CRYPTODEV_NAME_AESNI_MB_PMD)); + } + } + + /* attach 2 AESNI_MB cdevs */ + for (i = 0; i < rte_cryptodev_count() && nb_devs_attached < 2; + i++) { + struct rte_cryptodev_info info; + + rte_cryptodev_info_get(i, &info); + if (info.driver_id != rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_AESNI_MB_PMD))) + continue; + + /* + * Create the session mempool again, since now there are new devices + * to use the mempool. + */ + if (ts_params->session_mpool) { + rte_mempool_free(ts_params->session_mpool); + ts_params->session_mpool = NULL; + } + unsigned int session_size = rte_cryptodev_get_private_session_size(i); + + /* + * Create mempool with maximum number of sessions * 2, + * to include the session headers + */ + if (ts_params->session_mpool == NULL) { + ts_params->session_mpool = rte_mempool_create( + "test_sess_mp", + info.sym.max_nb_sessions * 2, + session_size, + 0, 0, NULL, NULL, NULL, + NULL, SOCKET_ID_ANY, + 0); + + TEST_ASSERT_NOT_NULL(ts_params->session_mpool, + "session mempool allocation failed"); + } + + ret = rte_cryptodev_scheduler_slave_attach(sched_id, + (uint8_t)i); + + TEST_ASSERT(ret == 0, + "Failed to attach device %u of pmd : %s", i, + RTE_STR(CRYPTODEV_NAME_AESNI_MB_PMD)); + + aesni_ids[nb_devs_attached] = (uint8_t)i; + + nb_devs_attached++; + } + + return 0; +} + +static int +test_scheduler_detach_slave_op(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + uint8_t sched_id = ts_params->valid_devs[0]; + uint32_t i; + int ret; + + for (i = 0; i < 2; i++) { + ret = rte_cryptodev_scheduler_slave_detach(sched_id, + aesni_ids[i]); + TEST_ASSERT(ret == 0, + "Failed to detach device %u", aesni_ids[i]); + } + + return 0; +} + +static int +test_scheduler_mode_op(void) +{ + struct crypto_testsuite_params *ts_params = &testsuite_params; + uint8_t sched_id = ts_params->valid_devs[0]; + struct rte_cryptodev_scheduler_ops op = {0}; + struct rte_cryptodev_scheduler dummy_scheduler = { + .description = "dummy scheduler to test mode", + .name = "dummy scheduler", + .mode = CDEV_SCHED_MODE_USERDEFINED, + .ops = &op + }; + int ret; + + /* set user defined mode */ + ret = rte_cryptodev_scheduler_load_user_scheduler(sched_id, + &dummy_scheduler); + TEST_ASSERT(ret == 0, + "Failed to set cdev %u to user defined mode", sched_id); + + /* set round robin mode */ + ret = rte_cryptodev_scheduler_mode_set(sched_id, + CDEV_SCHED_MODE_ROUNDROBIN); + TEST_ASSERT(ret == 0, + "Failed to set cdev %u to round-robin mode", sched_id); + TEST_ASSERT(rte_cryptodev_scheduler_mode_get(sched_id) == + CDEV_SCHED_MODE_ROUNDROBIN, "Scheduling Mode " + "not match"); + + return 0; +} + +static struct unit_test_suite cryptodev_scheduler_testsuite = { + .suite_name = "Crypto Device Scheduler Unit Test Suite", + .setup = testsuite_setup, + .teardown = testsuite_teardown, + .unit_test_cases = { + TEST_CASE_ST(NULL, NULL, test_scheduler_attach_slave_op), + TEST_CASE_ST(NULL, NULL, test_scheduler_mode_op), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_chain_scheduler_all), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_cipheronly_scheduler_all), + TEST_CASE_ST(ut_setup, ut_teardown, + test_authonly_scheduler_all), + TEST_CASE_ST(NULL, NULL, test_scheduler_detach_slave_op), + TEST_CASES_END() /**< NULL terminate unit test array */ + } +}; + +#endif /* RTE_LIBRTE_PMD_CRYPTO_SCHEDULER */ + +static struct unit_test_suite cryptodev_qat_testsuite = { + .suite_name = "Crypto QAT Unit Test Suite", + .setup = testsuite_setup, + .teardown = testsuite_teardown, + .unit_test_cases = { + TEST_CASE_ST(ut_setup, ut_teardown, + test_device_configure_invalid_dev_id), + TEST_CASE_ST(ut_setup, ut_teardown, + test_device_configure_invalid_queue_pair_ids), + TEST_CASE_ST(ut_setup, ut_teardown, + test_queue_pair_descriptor_setup), + TEST_CASE_ST(ut_setup, ut_teardown, + test_multi_session), + + TEST_CASE_ST(ut_setup, ut_teardown, test_AES_chain_qat_all), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_cipheronly_qat_all), + TEST_CASE_ST(ut_setup, ut_teardown, test_3DES_chain_qat_all), + TEST_CASE_ST(ut_setup, ut_teardown, + test_3DES_cipheronly_qat_all), + TEST_CASE_ST(ut_setup, ut_teardown, + test_DES_cipheronly_qat_all), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_docsis_qat_all), + TEST_CASE_ST(ut_setup, ut_teardown, + test_DES_docsis_qat_all), + TEST_CASE_ST(ut_setup, ut_teardown, test_authonly_qat_all), + TEST_CASE_ST(ut_setup, ut_teardown, test_stats), + + /** AES CCM Authenticated Encryption 128 bits key */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_CCM_authenticated_encryption_test_case_128_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_CCM_authenticated_encryption_test_case_128_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_CCM_authenticated_encryption_test_case_128_3), + + /** AES CCM Authenticated Decryption 128 bits key*/ + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_CCM_authenticated_decryption_test_case_128_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_CCM_authenticated_decryption_test_case_128_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_CCM_authenticated_decryption_test_case_128_3), + + /** AES GCM Authenticated Encryption */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encrypt_SGL_in_place_1500B), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encrypt_SGL_out_of_place_400B_400B), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encrypt_SGL_out_of_place_1500B_2000B), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_encryption_test_case_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_encryption_test_case_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_encryption_test_case_3), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_encryption_test_case_4), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_encryption_test_case_5), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_encryption_test_case_6), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_encryption_test_case_7), + + /** AES GCM Authenticated Decryption */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_decryption_test_case_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_decryption_test_case_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_decryption_test_case_3), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_decryption_test_case_4), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_decryption_test_case_5), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_decryption_test_case_6), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_decryption_test_case_7), + + /** AES GCM Authenticated Encryption 192 bits key */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_192_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_192_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_192_3), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_192_4), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_192_5), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_192_6), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_192_7), + + /** AES GCM Authenticated Decryption 192 bits key */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_192_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_192_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_192_3), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_192_4), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_192_5), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_192_6), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_192_7), + + /** AES GCM Authenticated Encryption 256 bits key */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_256_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_256_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_256_3), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_256_4), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_256_5), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_256_6), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_256_7), + + /** AES GMAC Authentication */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GMAC_authentication_test_case_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GMAC_authentication_verify_test_case_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GMAC_authentication_test_case_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GMAC_authentication_verify_test_case_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GMAC_authentication_test_case_3), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GMAC_authentication_verify_test_case_3), + + /** SNOW 3G encrypt only (UEA2) */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_snow3g_encryption_test_case_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_snow3g_encryption_test_case_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_snow3g_encryption_test_case_3), + TEST_CASE_ST(ut_setup, ut_teardown, + test_snow3g_encryption_test_case_4), + TEST_CASE_ST(ut_setup, ut_teardown, + test_snow3g_encryption_test_case_5), + + TEST_CASE_ST(ut_setup, ut_teardown, + test_snow3g_encryption_test_case_1_oop), + TEST_CASE_ST(ut_setup, ut_teardown, + test_snow3g_decryption_test_case_1_oop), + + /** SNOW 3G decrypt only (UEA2) */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_snow3g_decryption_test_case_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_snow3g_decryption_test_case_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_snow3g_decryption_test_case_3), + TEST_CASE_ST(ut_setup, ut_teardown, + test_snow3g_decryption_test_case_4), + TEST_CASE_ST(ut_setup, ut_teardown, + test_snow3g_decryption_test_case_5), + TEST_CASE_ST(ut_setup, ut_teardown, + test_snow3g_hash_generate_test_case_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_snow3g_hash_generate_test_case_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_snow3g_hash_generate_test_case_3), + TEST_CASE_ST(ut_setup, ut_teardown, + test_snow3g_hash_verify_test_case_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_snow3g_hash_verify_test_case_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_snow3g_hash_verify_test_case_3), + TEST_CASE_ST(ut_setup, ut_teardown, + test_snow3g_cipher_auth_test_case_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_snow3g_auth_cipher_test_case_1), + + /** ZUC encrypt only (EEA3) */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_zuc_encryption_test_case_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_zuc_encryption_test_case_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_zuc_encryption_test_case_3), + TEST_CASE_ST(ut_setup, ut_teardown, + test_zuc_encryption_test_case_4), + TEST_CASE_ST(ut_setup, ut_teardown, + test_zuc_encryption_test_case_5), + + /** ZUC authenticate (EIA3) */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_zuc_hash_generate_test_case_6), + TEST_CASE_ST(ut_setup, ut_teardown, + test_zuc_hash_generate_test_case_7), + TEST_CASE_ST(ut_setup, ut_teardown, + test_zuc_hash_generate_test_case_8), + + /** ZUC alg-chain (EEA3/EIA3) */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_zuc_cipher_auth_test_case_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_zuc_cipher_auth_test_case_2), + + /** HMAC_MD5 Authentication */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_MD5_HMAC_generate_case_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_MD5_HMAC_verify_case_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_MD5_HMAC_generate_case_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_MD5_HMAC_verify_case_2), + + /** NULL tests */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_null_auth_only_operation), + TEST_CASE_ST(ut_setup, ut_teardown, + test_null_cipher_only_operation), + TEST_CASE_ST(ut_setup, ut_teardown, + test_null_cipher_auth_operation), + TEST_CASE_ST(ut_setup, ut_teardown, + test_null_auth_cipher_operation), + + /** KASUMI tests */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_kasumi_hash_generate_test_case_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_kasumi_hash_generate_test_case_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_kasumi_hash_generate_test_case_3), + TEST_CASE_ST(ut_setup, ut_teardown, + test_kasumi_hash_generate_test_case_4), + TEST_CASE_ST(ut_setup, ut_teardown, + test_kasumi_hash_generate_test_case_5), + TEST_CASE_ST(ut_setup, ut_teardown, + test_kasumi_hash_generate_test_case_6), + + TEST_CASE_ST(ut_setup, ut_teardown, + test_kasumi_hash_verify_test_case_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_kasumi_hash_verify_test_case_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_kasumi_hash_verify_test_case_3), + TEST_CASE_ST(ut_setup, ut_teardown, + test_kasumi_hash_verify_test_case_4), + TEST_CASE_ST(ut_setup, ut_teardown, + test_kasumi_hash_verify_test_case_5), + + TEST_CASE_ST(ut_setup, ut_teardown, + test_kasumi_encryption_test_case_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_kasumi_encryption_test_case_3), + TEST_CASE_ST(ut_setup, ut_teardown, + test_kasumi_auth_cipher_test_case_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_kasumi_cipher_auth_test_case_1), + + /** Negative tests */ + TEST_CASE_ST(ut_setup, ut_teardown, + authentication_verify_HMAC_SHA1_fail_data_corrupt), + TEST_CASE_ST(ut_setup, ut_teardown, + authentication_verify_HMAC_SHA1_fail_tag_corrupt), + TEST_CASE_ST(ut_setup, ut_teardown, + authentication_verify_AES128_GMAC_fail_data_corrupt), + TEST_CASE_ST(ut_setup, ut_teardown, + authentication_verify_AES128_GMAC_fail_tag_corrupt), + TEST_CASE_ST(ut_setup, ut_teardown, + auth_decryption_AES128CBC_HMAC_SHA1_fail_data_corrupt), + TEST_CASE_ST(ut_setup, ut_teardown, + auth_decryption_AES128CBC_HMAC_SHA1_fail_tag_corrupt), + + TEST_CASES_END() /**< NULL terminate unit test array */ + } +}; + +static struct unit_test_suite cryptodev_aesni_mb_testsuite = { + .suite_name = "Crypto Device AESNI MB Unit Test Suite", + .setup = testsuite_setup, + .teardown = testsuite_teardown, + .unit_test_cases = { + TEST_CASE_ST(ut_setup, ut_teardown, test_AES_chain_mb_all), + TEST_CASE_ST(ut_setup, ut_teardown, test_AES_cipheronly_mb_all), + TEST_CASE_ST(ut_setup, ut_teardown, test_AES_docsis_mb_all), + TEST_CASE_ST(ut_setup, ut_teardown, test_authonly_mb_all), + TEST_CASE_ST(ut_setup, ut_teardown, + test_DES_cipheronly_mb_all), + TEST_CASE_ST(ut_setup, ut_teardown, + test_DES_docsis_mb_all), + + TEST_CASES_END() /**< NULL terminate unit test array */ + } +}; + +static struct unit_test_suite cryptodev_openssl_testsuite = { + .suite_name = "Crypto Device OPENSSL Unit Test Suite", + .setup = testsuite_setup, + .teardown = testsuite_teardown, + .unit_test_cases = { + TEST_CASE_ST(ut_setup, ut_teardown, test_multi_session), + TEST_CASE_ST(ut_setup, ut_teardown, + test_multi_session_random_usage), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_chain_openssl_all), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_cipheronly_openssl_all), + TEST_CASE_ST(ut_setup, ut_teardown, + test_3DES_chain_openssl_all), + TEST_CASE_ST(ut_setup, ut_teardown, + test_3DES_cipheronly_openssl_all), + TEST_CASE_ST(ut_setup, ut_teardown, + test_DES_cipheronly_openssl_all), + TEST_CASE_ST(ut_setup, ut_teardown, + test_DES_docsis_openssl_all), + TEST_CASE_ST(ut_setup, ut_teardown, + test_authonly_openssl_all), + + /** AES GCM Authenticated Encryption */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_encryption_test_case_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_encryption_test_case_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_encryption_test_case_3), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_encryption_test_case_4), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_encryption_test_case_5), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_encryption_test_case_6), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_encryption_test_case_7), + + /** AES GCM Authenticated Decryption */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_decryption_test_case_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_decryption_test_case_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_decryption_test_case_3), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_decryption_test_case_4), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_decryption_test_case_5), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_decryption_test_case_6), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_decryption_test_case_7), + + + /** AES GCM Authenticated Encryption 192 bits key */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_192_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_192_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_192_3), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_192_4), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_192_5), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_192_6), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_192_7), + + /** AES GCM Authenticated Decryption 192 bits key */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_192_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_192_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_192_3), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_192_4), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_192_5), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_192_6), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_192_7), + + /** AES GCM Authenticated Encryption 256 bits key */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_256_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_256_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_256_3), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_256_4), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_256_5), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_256_6), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_256_7), + + /** AES GCM Authenticated Decryption 256 bits key */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_256_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_256_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_256_3), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_256_4), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_256_5), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_256_6), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_256_7), + + /** AES GMAC Authentication */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GMAC_authentication_test_case_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GMAC_authentication_verify_test_case_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GMAC_authentication_test_case_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GMAC_authentication_verify_test_case_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GMAC_authentication_test_case_3), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GMAC_authentication_verify_test_case_3), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GMAC_authentication_test_case_4), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GMAC_authentication_verify_test_case_4), + + /** AES CCM Authenticated Encryption 128 bits key */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_CCM_authenticated_encryption_test_case_128_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_CCM_authenticated_encryption_test_case_128_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_CCM_authenticated_encryption_test_case_128_3), + + /** AES CCM Authenticated Decryption 128 bits key*/ + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_CCM_authenticated_decryption_test_case_128_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_CCM_authenticated_decryption_test_case_128_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_CCM_authenticated_decryption_test_case_128_3), + + /** AES CCM Authenticated Encryption 192 bits key */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_CCM_authenticated_encryption_test_case_192_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_CCM_authenticated_encryption_test_case_192_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_CCM_authenticated_encryption_test_case_192_3), + + /** AES CCM Authenticated Decryption 192 bits key*/ + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_CCM_authenticated_decryption_test_case_192_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_CCM_authenticated_decryption_test_case_192_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_CCM_authenticated_decryption_test_case_192_3), + + /** AES CCM Authenticated Encryption 256 bits key */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_CCM_authenticated_encryption_test_case_256_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_CCM_authenticated_encryption_test_case_256_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_CCM_authenticated_encryption_test_case_256_3), + + /** AES CCM Authenticated Decryption 256 bits key*/ + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_CCM_authenticated_decryption_test_case_256_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_CCM_authenticated_decryption_test_case_256_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_CCM_authenticated_decryption_test_case_256_3), + + /** Scatter-Gather */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encrypt_SGL_out_of_place_400B_1seg), + + /** Negative tests */ + TEST_CASE_ST(ut_setup, ut_teardown, + authentication_verify_HMAC_SHA1_fail_data_corrupt), + TEST_CASE_ST(ut_setup, ut_teardown, + authentication_verify_HMAC_SHA1_fail_tag_corrupt), + TEST_CASE_ST(ut_setup, ut_teardown, + authentication_verify_AES128_GMAC_fail_data_corrupt), + TEST_CASE_ST(ut_setup, ut_teardown, + authentication_verify_AES128_GMAC_fail_tag_corrupt), + TEST_CASE_ST(ut_setup, ut_teardown, + auth_decryption_AES128CBC_HMAC_SHA1_fail_data_corrupt), + TEST_CASE_ST(ut_setup, ut_teardown, + auth_decryption_AES128CBC_HMAC_SHA1_fail_tag_corrupt), + + TEST_CASES_END() /**< NULL terminate unit test array */ + } +}; + +static struct unit_test_suite cryptodev_aesni_gcm_testsuite = { + .suite_name = "Crypto Device AESNI GCM Unit Test Suite", + .setup = testsuite_setup, + .teardown = testsuite_teardown, + .unit_test_cases = { + /** AES GCM Authenticated Encryption */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_encryption_test_case_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_encryption_test_case_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_encryption_test_case_3), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_encryption_test_case_4), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_encryption_test_case_5), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_encryption_test_case_6), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_encryption_test_case_7), + + /** AES GCM Authenticated Decryption */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_decryption_test_case_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_decryption_test_case_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_decryption_test_case_3), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_decryption_test_case_4), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_decryption_test_case_5), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_decryption_test_case_6), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_decryption_test_case_7), + + /** AES GCM Authenticated Encryption 192 bits key */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_192_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_192_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_192_3), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_192_4), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_192_5), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_192_6), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_192_7), + + /** AES GCM Authenticated Decryption 192 bits key */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_192_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_192_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_192_3), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_192_4), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_192_5), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_192_6), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_192_7), + + /** AES GCM Authenticated Encryption 256 bits key */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_256_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_256_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_256_3), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_256_4), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_256_5), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_256_6), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_256_7), + + /** AES GCM Authenticated Decryption 256 bits key */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_256_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_256_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_256_3), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_256_4), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_256_5), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_256_6), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_256_7), + + /** AES GCM Authenticated Encryption big aad size */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_aad_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_aad_2), + + /** AES GCM Authenticated Decryption big aad size */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_aad_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_aad_2), + + /** AES GMAC Authentication */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GMAC_authentication_test_case_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GMAC_authentication_verify_test_case_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GMAC_authentication_test_case_3), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GMAC_authentication_verify_test_case_3), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GMAC_authentication_test_case_4), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GMAC_authentication_verify_test_case_4), + + /** Negative tests */ + TEST_CASE_ST(ut_setup, ut_teardown, + authentication_verify_AES128_GMAC_fail_data_corrupt), + TEST_CASE_ST(ut_setup, ut_teardown, + authentication_verify_AES128_GMAC_fail_tag_corrupt), + + /** Out of place tests */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_encryption_oop_test_case_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_decryption_oop_test_case_1), + + /** Session-less tests */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_encryption_sessionless_test_case_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_decryption_sessionless_test_case_1), + + /** Scatter-Gather */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encrypt_SGL_out_of_place_400B_1seg), + + TEST_CASES_END() /**< NULL terminate unit test array */ + } +}; + +static struct unit_test_suite cryptodev_sw_kasumi_testsuite = { + .suite_name = "Crypto Device SW KASUMI Unit Test Suite", + .setup = testsuite_setup, + .teardown = testsuite_teardown, + .unit_test_cases = { + /** KASUMI encrypt only (UEA1) */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_kasumi_encryption_test_case_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_kasumi_encryption_test_case_1_sgl), + TEST_CASE_ST(ut_setup, ut_teardown, + test_kasumi_encryption_test_case_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_kasumi_encryption_test_case_3), + TEST_CASE_ST(ut_setup, ut_teardown, + test_kasumi_encryption_test_case_4), + TEST_CASE_ST(ut_setup, ut_teardown, + test_kasumi_encryption_test_case_5), + /** KASUMI decrypt only (UEA1) */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_kasumi_decryption_test_case_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_kasumi_decryption_test_case_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_kasumi_decryption_test_case_3), + TEST_CASE_ST(ut_setup, ut_teardown, + test_kasumi_decryption_test_case_4), + TEST_CASE_ST(ut_setup, ut_teardown, + test_kasumi_decryption_test_case_5), + + TEST_CASE_ST(ut_setup, ut_teardown, + test_kasumi_encryption_test_case_1_oop), + TEST_CASE_ST(ut_setup, ut_teardown, + test_kasumi_encryption_test_case_1_oop_sgl), + + + TEST_CASE_ST(ut_setup, ut_teardown, + test_kasumi_decryption_test_case_1_oop), + + /** KASUMI hash only (UIA1) */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_kasumi_hash_generate_test_case_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_kasumi_hash_generate_test_case_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_kasumi_hash_generate_test_case_3), + TEST_CASE_ST(ut_setup, ut_teardown, + test_kasumi_hash_generate_test_case_4), + TEST_CASE_ST(ut_setup, ut_teardown, + test_kasumi_hash_generate_test_case_5), + TEST_CASE_ST(ut_setup, ut_teardown, + test_kasumi_hash_generate_test_case_6), + TEST_CASE_ST(ut_setup, ut_teardown, + test_kasumi_hash_verify_test_case_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_kasumi_hash_verify_test_case_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_kasumi_hash_verify_test_case_3), + TEST_CASE_ST(ut_setup, ut_teardown, + test_kasumi_hash_verify_test_case_4), + TEST_CASE_ST(ut_setup, ut_teardown, + test_kasumi_hash_verify_test_case_5), + TEST_CASE_ST(ut_setup, ut_teardown, + test_kasumi_auth_cipher_test_case_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_kasumi_cipher_auth_test_case_1), + TEST_CASES_END() /**< NULL terminate unit test array */ + } +}; +static struct unit_test_suite cryptodev_sw_snow3g_testsuite = { + .suite_name = "Crypto Device SW SNOW 3G Unit Test Suite", + .setup = testsuite_setup, + .teardown = testsuite_teardown, + .unit_test_cases = { + /** SNOW 3G encrypt only (UEA2) */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_snow3g_encryption_test_case_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_snow3g_encryption_test_case_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_snow3g_encryption_test_case_3), + TEST_CASE_ST(ut_setup, ut_teardown, + test_snow3g_encryption_test_case_4), + TEST_CASE_ST(ut_setup, ut_teardown, + test_snow3g_encryption_test_case_5), + + TEST_CASE_ST(ut_setup, ut_teardown, + test_snow3g_encryption_test_case_1_oop), + TEST_CASE_ST(ut_setup, ut_teardown, + test_snow3g_encryption_test_case_1_oop_sgl), + TEST_CASE_ST(ut_setup, ut_teardown, + test_snow3g_decryption_test_case_1_oop), + + TEST_CASE_ST(ut_setup, ut_teardown, + test_snow3g_encryption_test_case_1_offset_oop), + + /** SNOW 3G decrypt only (UEA2) */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_snow3g_decryption_test_case_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_snow3g_decryption_test_case_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_snow3g_decryption_test_case_3), + TEST_CASE_ST(ut_setup, ut_teardown, + test_snow3g_decryption_test_case_4), + TEST_CASE_ST(ut_setup, ut_teardown, + test_snow3g_decryption_test_case_5), + TEST_CASE_ST(ut_setup, ut_teardown, + test_snow3g_hash_generate_test_case_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_snow3g_hash_generate_test_case_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_snow3g_hash_generate_test_case_3), + /* Tests with buffers which length is not byte-aligned */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_snow3g_hash_generate_test_case_4), + TEST_CASE_ST(ut_setup, ut_teardown, + test_snow3g_hash_generate_test_case_5), + TEST_CASE_ST(ut_setup, ut_teardown, + test_snow3g_hash_generate_test_case_6), + TEST_CASE_ST(ut_setup, ut_teardown, + test_snow3g_hash_verify_test_case_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_snow3g_hash_verify_test_case_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_snow3g_hash_verify_test_case_3), + /* Tests with buffers which length is not byte-aligned */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_snow3g_hash_verify_test_case_4), + TEST_CASE_ST(ut_setup, ut_teardown, + test_snow3g_hash_verify_test_case_5), + TEST_CASE_ST(ut_setup, ut_teardown, + test_snow3g_hash_verify_test_case_6), + TEST_CASE_ST(ut_setup, ut_teardown, + test_snow3g_cipher_auth_test_case_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_snow3g_auth_cipher_test_case_1), + + TEST_CASES_END() /**< NULL terminate unit test array */ + } +}; + +static struct unit_test_suite cryptodev_sw_zuc_testsuite = { + .suite_name = "Crypto Device SW ZUC Unit Test Suite", + .setup = testsuite_setup, + .teardown = testsuite_teardown, + .unit_test_cases = { + /** ZUC encrypt only (EEA3) */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_zuc_encryption_test_case_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_zuc_encryption_test_case_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_zuc_encryption_test_case_3), + TEST_CASE_ST(ut_setup, ut_teardown, + test_zuc_encryption_test_case_4), + TEST_CASE_ST(ut_setup, ut_teardown, + test_zuc_encryption_test_case_5), + TEST_CASE_ST(ut_setup, ut_teardown, + test_zuc_hash_generate_test_case_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_zuc_hash_generate_test_case_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_zuc_hash_generate_test_case_3), + TEST_CASE_ST(ut_setup, ut_teardown, + test_zuc_hash_generate_test_case_4), + TEST_CASE_ST(ut_setup, ut_teardown, + test_zuc_hash_generate_test_case_5), + TEST_CASE_ST(ut_setup, ut_teardown, + test_zuc_encryption_test_case_6_sgl), + TEST_CASES_END() /**< NULL terminate unit test array */ + } +}; + +static struct unit_test_suite cryptodev_dpaa_sec_testsuite = { + .suite_name = "Crypto DPAA_SEC Unit Test Suite", + .setup = testsuite_setup, + .teardown = testsuite_teardown, + .unit_test_cases = { + TEST_CASE_ST(ut_setup, ut_teardown, + test_device_configure_invalid_dev_id), + TEST_CASE_ST(ut_setup, ut_teardown, + test_multi_session), + + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_chain_dpaa_sec_all), + TEST_CASE_ST(ut_setup, ut_teardown, + test_3DES_chain_dpaa_sec_all), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_cipheronly_dpaa_sec_all), + TEST_CASE_ST(ut_setup, ut_teardown, + test_3DES_cipheronly_dpaa_sec_all), + TEST_CASE_ST(ut_setup, ut_teardown, + test_authonly_dpaa_sec_all), + + /** AES GCM Authenticated Encryption */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_encryption_test_case_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_encryption_test_case_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_encryption_test_case_3), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_encryption_test_case_4), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_encryption_test_case_5), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_encryption_test_case_6), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_encryption_test_case_7), + + /** AES GCM Authenticated Decryption */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_decryption_test_case_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_decryption_test_case_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_decryption_test_case_3), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_decryption_test_case_4), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_decryption_test_case_5), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_decryption_test_case_6), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_decryption_test_case_7), + + /** AES GCM Authenticated Encryption 256 bits key */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_256_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_256_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_256_3), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_256_4), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_256_5), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_256_6), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_256_7), + + /** AES GCM Authenticated Decryption 256 bits key */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_256_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_256_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_256_3), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_256_4), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_256_5), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_256_6), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_256_7), + + /** Out of place tests */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_encryption_oop_test_case_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_decryption_oop_test_case_1), + + TEST_CASES_END() /**< NULL terminate unit test array */ + } +}; + +static struct unit_test_suite cryptodev_dpaa2_sec_testsuite = { + .suite_name = "Crypto DPAA2_SEC Unit Test Suite", + .setup = testsuite_setup, + .teardown = testsuite_teardown, + .unit_test_cases = { + TEST_CASE_ST(ut_setup, ut_teardown, + test_device_configure_invalid_dev_id), + TEST_CASE_ST(ut_setup, ut_teardown, + test_multi_session), + + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_chain_dpaa2_sec_all), + TEST_CASE_ST(ut_setup, ut_teardown, + test_3DES_chain_dpaa2_sec_all), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_cipheronly_dpaa2_sec_all), + TEST_CASE_ST(ut_setup, ut_teardown, + test_3DES_cipheronly_dpaa2_sec_all), + TEST_CASE_ST(ut_setup, ut_teardown, + test_authonly_dpaa2_sec_all), + + /** AES GCM Authenticated Encryption */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_encryption_test_case_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_encryption_test_case_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_encryption_test_case_3), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_encryption_test_case_4), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_encryption_test_case_5), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_encryption_test_case_6), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_encryption_test_case_7), + + /** AES GCM Authenticated Decryption */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_decryption_test_case_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_decryption_test_case_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_decryption_test_case_3), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_decryption_test_case_4), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_decryption_test_case_5), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_decryption_test_case_6), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_decryption_test_case_7), + + /** AES GCM Authenticated Encryption 192 bits key */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_192_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_192_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_192_3), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_192_4), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_192_5), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_192_6), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_192_7), + + /** AES GCM Authenticated Decryption 192 bits key */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_192_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_192_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_192_3), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_192_4), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_192_5), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_192_6), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_192_7), + + /** AES GCM Authenticated Encryption 256 bits key */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_256_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_256_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_256_3), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_256_4), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_256_5), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_256_6), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_encryption_test_case_256_7), + + /** AES GCM Authenticated Decryption 256 bits key */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_256_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_256_2), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_256_3), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_256_4), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_256_5), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_256_6), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_auth_decryption_test_case_256_7), + + /** Out of place tests */ + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_encryption_oop_test_case_1), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_GCM_authenticated_decryption_oop_test_case_1), + + TEST_CASES_END() /**< NULL terminate unit test array */ + } +}; + +static struct unit_test_suite cryptodev_null_testsuite = { + .suite_name = "Crypto Device NULL Unit Test Suite", + .setup = testsuite_setup, + .teardown = testsuite_teardown, + .unit_test_cases = { + TEST_CASE_ST(ut_setup, ut_teardown, + test_null_auth_only_operation), + TEST_CASE_ST(ut_setup, ut_teardown, + test_null_cipher_only_operation), + TEST_CASE_ST(ut_setup, ut_teardown, + test_null_cipher_auth_operation), + TEST_CASE_ST(ut_setup, ut_teardown, + test_null_auth_cipher_operation), + TEST_CASE_ST(ut_setup, ut_teardown, + test_null_invalid_operation), + TEST_CASE_ST(ut_setup, ut_teardown, + test_null_burst_operation), + + TEST_CASES_END() /**< NULL terminate unit test array */ + } +}; + +static struct unit_test_suite cryptodev_armv8_testsuite = { + .suite_name = "Crypto Device ARMv8 Unit Test Suite", + .setup = testsuite_setup, + .teardown = testsuite_teardown, + .unit_test_cases = { + TEST_CASE_ST(ut_setup, ut_teardown, test_AES_chain_armv8_all), + + /** Negative tests */ + TEST_CASE_ST(ut_setup, ut_teardown, + auth_decryption_AES128CBC_HMAC_SHA1_fail_data_corrupt), + TEST_CASE_ST(ut_setup, ut_teardown, + auth_decryption_AES128CBC_HMAC_SHA1_fail_tag_corrupt), + + TEST_CASES_END() /**< NULL terminate unit test array */ + } +}; + +static struct unit_test_suite cryptodev_mrvl_testsuite = { + .suite_name = "Crypto Device Marvell Component Test Suite", + .setup = testsuite_setup, + .teardown = testsuite_teardown, + .unit_test_cases = { + TEST_CASE_ST(ut_setup, ut_teardown, test_multi_session), + TEST_CASE_ST(ut_setup, ut_teardown, + test_multi_session_random_usage), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_chain_mrvl_all), + TEST_CASE_ST(ut_setup, ut_teardown, + test_AES_cipheronly_mrvl_all), + TEST_CASE_ST(ut_setup, ut_teardown, + test_authonly_mrvl_all), + TEST_CASE_ST(ut_setup, ut_teardown, + test_3DES_chain_mrvl_all), + TEST_CASE_ST(ut_setup, ut_teardown, + test_3DES_cipheronly_mrvl_all), + + /** Negative tests */ + TEST_CASE_ST(ut_setup, ut_teardown, + authentication_verify_HMAC_SHA1_fail_data_corrupt), + TEST_CASE_ST(ut_setup, ut_teardown, + authentication_verify_HMAC_SHA1_fail_tag_corrupt), + TEST_CASE_ST(ut_setup, ut_teardown, + auth_decryption_AES128CBC_HMAC_SHA1_fail_data_corrupt), + TEST_CASE_ST(ut_setup, ut_teardown, + auth_decryption_AES128CBC_HMAC_SHA1_fail_tag_corrupt), + + TEST_CASES_END() /**< NULL terminate unit test array */ + } +}; + + +static int +test_cryptodev_qat(void /*argv __rte_unused, int argc __rte_unused*/) +{ + gbl_driver_id = rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_QAT_SYM_PMD)); + + if (gbl_driver_id == -1) { + RTE_LOG(ERR, USER1, "QAT PMD must be loaded. Check if " + "CONFIG_RTE_LIBRTE_PMD_QAT is enabled " + "in config file to run this testsuite.\n"); + return TEST_FAILED; + } + + return unit_test_suite_runner(&cryptodev_qat_testsuite); +} + +static int +test_cryptodev_aesni_mb(void /*argv __rte_unused, int argc __rte_unused*/) +{ + gbl_driver_id = rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_AESNI_MB_PMD)); + + if (gbl_driver_id == -1) { + RTE_LOG(ERR, USER1, "AESNI MB PMD must be loaded. Check if " + "CONFIG_RTE_LIBRTE_PMD_AESNI_MB is enabled " + "in config file to run this testsuite.\n"); + return TEST_FAILED; + } + + return unit_test_suite_runner(&cryptodev_aesni_mb_testsuite); +} + +static int +test_cryptodev_openssl(void) +{ + gbl_driver_id = rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_OPENSSL_PMD)); + + if (gbl_driver_id == -1) { + RTE_LOG(ERR, USER1, "OPENSSL PMD must be loaded. Check if " + "CONFIG_RTE_LIBRTE_PMD_OPENSSL is enabled " + "in config file to run this testsuite.\n"); + return TEST_FAILED; + } + + return unit_test_suite_runner(&cryptodev_openssl_testsuite); +} + +static int +test_cryptodev_aesni_gcm(void) +{ + gbl_driver_id = rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_AESNI_GCM_PMD)); + + if (gbl_driver_id == -1) { + RTE_LOG(ERR, USER1, "AESNI GCM PMD must be loaded. Check if " + "CONFIG_RTE_LIBRTE_PMD_AESNI_GCM is enabled " + "in config file to run this testsuite.\n"); + return TEST_FAILED; + } + + return unit_test_suite_runner(&cryptodev_aesni_gcm_testsuite); +} + +static int +test_cryptodev_null(void) +{ + gbl_driver_id = rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_NULL_PMD)); + + if (gbl_driver_id == -1) { + RTE_LOG(ERR, USER1, "NULL PMD must be loaded. Check if " + "CONFIG_RTE_LIBRTE_PMD_NULL is enabled " + "in config file to run this testsuite.\n"); + return TEST_FAILED; + } + + return unit_test_suite_runner(&cryptodev_null_testsuite); +} + +static int +test_cryptodev_sw_snow3g(void /*argv __rte_unused, int argc __rte_unused*/) +{ + gbl_driver_id = rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_SNOW3G_PMD)); + + if (gbl_driver_id == -1) { + RTE_LOG(ERR, USER1, "SNOW3G PMD must be loaded. Check if " + "CONFIG_RTE_LIBRTE_PMD_SNOW3G is enabled " + "in config file to run this testsuite.\n"); + return TEST_FAILED; + } + + return unit_test_suite_runner(&cryptodev_sw_snow3g_testsuite); +} + +static int +test_cryptodev_sw_kasumi(void /*argv __rte_unused, int argc __rte_unused*/) +{ + gbl_driver_id = rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_KASUMI_PMD)); + + if (gbl_driver_id == -1) { + RTE_LOG(ERR, USER1, "ZUC PMD must be loaded. Check if " + "CONFIG_RTE_LIBRTE_PMD_KASUMI is enabled " + "in config file to run this testsuite.\n"); + return TEST_FAILED; + } + + return unit_test_suite_runner(&cryptodev_sw_kasumi_testsuite); +} + +static int +test_cryptodev_sw_zuc(void /*argv __rte_unused, int argc __rte_unused*/) +{ + gbl_driver_id = rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_ZUC_PMD)); + + if (gbl_driver_id == -1) { + RTE_LOG(ERR, USER1, "ZUC PMD must be loaded. Check if " + "CONFIG_RTE_LIBRTE_PMD_ZUC is enabled " + "in config file to run this testsuite.\n"); + return TEST_FAILED; + } + + return unit_test_suite_runner(&cryptodev_sw_zuc_testsuite); +} + +static int +test_cryptodev_armv8(void) +{ + gbl_driver_id = rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_ARMV8_PMD)); + + if (gbl_driver_id == -1) { + RTE_LOG(ERR, USER1, "ARMV8 PMD must be loaded. Check if " + "CONFIG_RTE_LIBRTE_PMD_ARMV8 is enabled " + "in config file to run this testsuite.\n"); + return TEST_FAILED; + } + + return unit_test_suite_runner(&cryptodev_armv8_testsuite); +} + +static int +test_cryptodev_mrvl(void) +{ + gbl_driver_id = rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_MRVL_PMD)); + + if (gbl_driver_id == -1) { + RTE_LOG(ERR, USER1, "MRVL PMD must be loaded. Check if " + "CONFIG_RTE_LIBRTE_PMD_MRVL_CRYPTO is enabled " + "in config file to run this testsuite.\n"); + return TEST_FAILED; + } + + return unit_test_suite_runner(&cryptodev_mrvl_testsuite); +} + +#ifdef RTE_LIBRTE_PMD_CRYPTO_SCHEDULER + +static int +test_cryptodev_scheduler(void /*argv __rte_unused, int argc __rte_unused*/) +{ + gbl_driver_id = rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_SCHEDULER_PMD)); + + if (gbl_driver_id == -1) { + RTE_LOG(ERR, USER1, "SCHEDULER PMD must be loaded. Check if " + "CONFIG_RTE_LIBRTE_PMD_SCHEDULER is enabled " + "in config file to run this testsuite.\n"); + return TEST_FAILED; + } + + if (rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_AESNI_MB_PMD)) == -1) { + RTE_LOG(ERR, USER1, "CONFIG_RTE_LIBRTE_PMD_AESNI_MB must be" + " enabled in config file to run this testsuite.\n"); + return TEST_FAILED; +} + return unit_test_suite_runner(&cryptodev_scheduler_testsuite); +} + +REGISTER_TEST_COMMAND(cryptodev_scheduler_autotest, test_cryptodev_scheduler); + +#endif + +static int +test_cryptodev_dpaa2_sec(void /*argv __rte_unused, int argc __rte_unused*/) +{ + gbl_driver_id = rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_DPAA2_SEC_PMD)); + + if (gbl_driver_id == -1) { + RTE_LOG(ERR, USER1, "DPAA2 SEC PMD must be loaded. Check if " + "CONFIG_RTE_LIBRTE_PMD_DPAA2_SEC is enabled " + "in config file to run this testsuite.\n"); + return TEST_FAILED; + } + + return unit_test_suite_runner(&cryptodev_dpaa2_sec_testsuite); +} + +static int +test_cryptodev_dpaa_sec(void /*argv __rte_unused, int argc __rte_unused*/) +{ + gbl_driver_id = rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_DPAA_SEC_PMD)); + + if (gbl_driver_id == -1) { + RTE_LOG(ERR, USER1, "DPAA SEC PMD must be loaded. Check if " + "CONFIG_RTE_LIBRTE_PMD_DPAA_SEC is enabled " + "in config file to run this testsuite.\n"); + return TEST_FAILED; + } + + return unit_test_suite_runner(&cryptodev_dpaa_sec_testsuite); +} + +REGISTER_TEST_COMMAND(cryptodev_qat_autotest, test_cryptodev_qat); +REGISTER_TEST_COMMAND(cryptodev_aesni_mb_autotest, test_cryptodev_aesni_mb); +REGISTER_TEST_COMMAND(cryptodev_openssl_autotest, test_cryptodev_openssl); +REGISTER_TEST_COMMAND(cryptodev_aesni_gcm_autotest, test_cryptodev_aesni_gcm); +REGISTER_TEST_COMMAND(cryptodev_null_autotest, test_cryptodev_null); +REGISTER_TEST_COMMAND(cryptodev_sw_snow3g_autotest, test_cryptodev_sw_snow3g); +REGISTER_TEST_COMMAND(cryptodev_sw_kasumi_autotest, test_cryptodev_sw_kasumi); +REGISTER_TEST_COMMAND(cryptodev_sw_zuc_autotest, test_cryptodev_sw_zuc); +REGISTER_TEST_COMMAND(cryptodev_sw_armv8_autotest, test_cryptodev_armv8); +REGISTER_TEST_COMMAND(cryptodev_sw_mrvl_autotest, test_cryptodev_mrvl); +REGISTER_TEST_COMMAND(cryptodev_dpaa2_sec_autotest, test_cryptodev_dpaa2_sec); +REGISTER_TEST_COMMAND(cryptodev_dpaa_sec_autotest, test_cryptodev_dpaa_sec); diff --git a/dpdk/test/test/test_cryptodev.h b/dpdk/test/test/test_cryptodev.h new file mode 100644 index 00000000..26bfbe60 --- /dev/null +++ b/dpdk/test/test/test_cryptodev.h @@ -0,0 +1,231 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2015-2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef TEST_CRYPTODEV_H_ +#define TEST_CRYPTODEV_H_ + +#define HEX_DUMP 0 + +#define FALSE 0 +#define TRUE 1 + +#define MAX_NUM_OPS_INFLIGHT (4096) +#define MIN_NUM_OPS_INFLIGHT (128) +#define DEFAULT_NUM_OPS_INFLIGHT (128) + +#define MAX_NUM_QPS_PER_QAT_DEVICE (2) +#define DEFAULT_NUM_QPS_PER_QAT_DEVICE (2) +#define DEFAULT_BURST_SIZE (64) +#define DEFAULT_NUM_XFORMS (2) +#define NUM_MBUFS (8191) +#define MBUF_CACHE_SIZE (256) +#define MBUF_DATAPAYLOAD_SIZE (2048 + DIGEST_BYTE_LENGTH_SHA512) +#define MBUF_SIZE (sizeof(struct rte_mbuf) + \ + RTE_PKTMBUF_HEADROOM + MBUF_DATAPAYLOAD_SIZE) + +#define BYTE_LENGTH(x) (x/8) +/* HASH DIGEST LENGTHS */ +#define DIGEST_BYTE_LENGTH_MD5 (BYTE_LENGTH(128)) +#define DIGEST_BYTE_LENGTH_SHA1 (BYTE_LENGTH(160)) +#define DIGEST_BYTE_LENGTH_SHA224 (BYTE_LENGTH(224)) +#define DIGEST_BYTE_LENGTH_SHA256 (BYTE_LENGTH(256)) +#define DIGEST_BYTE_LENGTH_SHA384 (BYTE_LENGTH(384)) +#define DIGEST_BYTE_LENGTH_SHA512 (BYTE_LENGTH(512)) +#define DIGEST_BYTE_LENGTH_AES_XCBC (BYTE_LENGTH(96)) +#define DIGEST_BYTE_LENGTH_SNOW3G_UIA2 (BYTE_LENGTH(32)) +#define DIGEST_BYTE_LENGTH_KASUMI_F9 (BYTE_LENGTH(32)) +#define AES_XCBC_MAC_KEY_SZ (16) +#define DIGEST_BYTE_LENGTH_AES_GCM (BYTE_LENGTH(128)) + +#define TRUNCATED_DIGEST_BYTE_LENGTH_SHA1 (12) +#define TRUNCATED_DIGEST_BYTE_LENGTH_SHA224 (16) +#define TRUNCATED_DIGEST_BYTE_LENGTH_SHA256 (16) +#define TRUNCATED_DIGEST_BYTE_LENGTH_SHA384 (24) +#define TRUNCATED_DIGEST_BYTE_LENGTH_SHA512 (32) + +#define MAXIMUM_IV_LENGTH (16) + +#define IV_OFFSET (sizeof(struct rte_crypto_op) + \ + sizeof(struct rte_crypto_sym_op) + DEFAULT_NUM_XFORMS * \ + sizeof(struct rte_crypto_sym_xform)) + +#define CRYPTODEV_NAME_NULL_PMD crypto_null +#define CRYPTODEV_NAME_AESNI_MB_PMD crypto_aesni_mb +#define CRYPTODEV_NAME_AESNI_GCM_PMD crypto_aesni_gcm +#define CRYPTODEV_NAME_OPENSSL_PMD crypto_openssl +#define CRYPTODEV_NAME_QAT_SYM_PMD crypto_qat +#define CRYPTODEV_NAME_SNOW3G_PMD crypto_snow3g +#define CRYPTODEV_NAME_KASUMI_PMD crypto_kasumi +#define CRYPTODEV_NAME_ZUC_PMD crypto_zuc +#define CRYPTODEV_NAME_ARMV8_PMD crypto_armv8 +#define CRYPTODEV_NAME_DPAA2_SEC_PMD crypto_dpaa2_sec +#define CRYPTODEV_NAME_SCHEDULER_PMD crypto_scheduler +#define CRYPTODEV_NAME_MRVL_PMD crypto_mrvl + +/** + * Write (spread) data from buffer to mbuf data + * + * @param mbuf + * Destination mbuf + * @param offset + * Start offset in mbuf + * @param len + * Number of bytes to copy + * @param buffer + * Continuous source buffer + */ +static inline void +pktmbuf_write(struct rte_mbuf *mbuf, int offset, int len, const uint8_t *buffer) +{ + int n = len; + int l; + struct rte_mbuf *m; + char *dst; + + for (m = mbuf; (m != NULL) && (offset > m->data_len); m = m->next) + offset -= m->data_len; + + l = m->data_len - offset; + + /* copy data from first segment */ + dst = rte_pktmbuf_mtod_offset(m, char *, offset); + if (len <= l) { + rte_memcpy(dst, buffer, len); + return; + } + + rte_memcpy(dst, buffer, l); + buffer += l; + n -= l; + + for (m = m->next; (m != NULL) && (n > 0); m = m->next) { + dst = rte_pktmbuf_mtod(m, char *); + l = m->data_len; + if (n < l) { + rte_memcpy(dst, buffer, n); + return; + } + rte_memcpy(dst, buffer, l); + buffer += l; + n -= l; + } +} + +static inline uint8_t * +pktmbuf_mtod_offset(struct rte_mbuf *mbuf, int offset) { + struct rte_mbuf *m; + + for (m = mbuf; (m != NULL) && (offset > m->data_len); m = m->next) + offset -= m->data_len; + + if (m == NULL) { + printf("pktmbuf_mtod_offset: offset out of buffer\n"); + return NULL; + } + return rte_pktmbuf_mtod_offset(m, uint8_t *, offset); +} + +static inline rte_iova_t +pktmbuf_iova_offset(struct rte_mbuf *mbuf, int offset) { + struct rte_mbuf *m; + + for (m = mbuf; (m != NULL) && (offset > m->data_len); m = m->next) + offset -= m->data_len; + + if (m == NULL) { + printf("pktmbuf_iova_offset: offset out of buffer\n"); + return 0; + } + return rte_pktmbuf_iova_offset(m, offset); +} + +static inline struct rte_mbuf * +create_segmented_mbuf(struct rte_mempool *mbuf_pool, int pkt_len, + int nb_segs, uint8_t pattern) { + + struct rte_mbuf *m = NULL, *mbuf = NULL; + uint8_t *dst; + int data_len = 0; + int i, size; + int t_len; + + if (pkt_len < 1) { + printf("Packet size must be 1 or more (is %d)\n", pkt_len); + return NULL; + } + + if (nb_segs < 1) { + printf("Number of segments must be 1 or more (is %d)\n", + nb_segs); + return NULL; + } + + t_len = pkt_len >= nb_segs ? pkt_len / nb_segs : 1; + size = pkt_len; + + /* Create chained mbuf_src and fill it generated data */ + for (i = 0; size > 0; i++) { + + m = rte_pktmbuf_alloc(mbuf_pool); + if (i == 0) + mbuf = m; + + if (m == NULL) { + printf("Cannot create segment for source mbuf"); + goto fail; + } + + /* Make sure if tailroom is zeroed */ + memset(m->buf_addr, pattern, m->buf_len); + + data_len = size > t_len ? t_len : size; + dst = (uint8_t *)rte_pktmbuf_append(m, data_len); + if (dst == NULL) { + printf("Cannot append %d bytes to the mbuf\n", + data_len); + goto fail; + } + + if (mbuf != m) + rte_pktmbuf_chain(mbuf, m); + + size -= data_len; + + } + return mbuf; + +fail: + if (mbuf) + rte_pktmbuf_free(mbuf); + return NULL; +} + +#endif /* TEST_CRYPTODEV_H_ */ diff --git a/dpdk/test/test/test_cryptodev_aead_test_vectors.h b/dpdk/test/test/test_cryptodev_aead_test_vectors.h new file mode 100644 index 00000000..e760ef93 --- /dev/null +++ b/dpdk/test/test/test_cryptodev_aead_test_vectors.h @@ -0,0 +1,3803 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2015-2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef TEST_CRYPTODEV_AEAD_TEST_VECTORS_H_ +#define TEST_CRYPTODEV_AEAD_TEST_VECTORS_H_ + +#define GMAC_LARGE_PLAINTEXT_LENGTH 65344 +#define MAX_AAD_LENGTH 65536 +#define GCM_LARGE_AAD_LENGTH 65296 + +static uint8_t gcm_aad_zero_text[MAX_AAD_LENGTH] = { 0 }; + +static uint8_t gcm_aad_text[MAX_AAD_LENGTH] = { + 0xfe, 0xed, 0xfa, 0xce, 0xde, 0xad, 0xbe, 0xef, + 0xfe, 0xed, 0xfa, 0xce, 0xde, 0xad, 0xbe, 0xef, + 0x00, 0xf1, 0xe2, 0xd3, 0xc4, 0xb5, 0xa6, 0x97, + 0x88, 0x79, 0x6a, 0x5b, 0x4c, 0x3d, 0x2e, 0x1f }; + +static uint8_t ccm_aad_test_1[8] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07 +}; + +static uint8_t ccm_aad_test_2[22] = { + 0x08, 0x40, 0x0F, 0xD2, 0xE1, 0x28, 0xA5, 0x7C, + 0x50, 0x30, 0xF1, 0x84, 0x44, 0x08, 0xAB, 0xAE, + 0xA5, 0xB8, 0xFC, 0xBA, 0x00, 0x00 +}; + +struct aead_test_data { + enum rte_crypto_aead_algorithm algo; + + struct { + uint8_t data[64]; + unsigned len; + } key; + + struct { + uint8_t data[64] __rte_aligned(16); + unsigned len; + } iv; + + struct { + uint8_t *data; + unsigned len; + } aad; + + struct { + uint8_t data[8096]; + unsigned len; + } plaintext; + + struct { + uint8_t data[8096]; + unsigned len; + } ciphertext; + + struct { + uint8_t data[16]; + unsigned len; + } auth_tag; +}; + +struct gmac_test_data { + struct { + uint8_t data[64]; + unsigned len; + } key; + + struct { + uint8_t data[64] __rte_aligned(16); + unsigned len; + } iv; + + struct { + uint8_t *data; + unsigned len; + } plaintext; + + struct { + uint8_t data[16]; + unsigned len; + } gmac_tag; + +}; + +/** AES-GCM-128 Test Vectors */ +static const struct aead_test_data gcm_test_case_1 = { + .algo = RTE_CRYPTO_AEAD_AES_GCM, + .key = { + .data = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .len = 16 + }, + .iv = { + .data = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00 }, + .len = 12 + }, + .aad = { + .data = gcm_aad_zero_text, + .len = 0 + }, + .plaintext = { + .data = { + 0x00 }, + .len = 0 + }, + .ciphertext = { + .data = { + 0x00 + }, + .len = 0 + }, + .auth_tag = { + .data = { + 0x58, 0xe2, 0xfc, 0xce, 0xfa, 0x7e, 0x30, 0x61, + 0x36, 0x7f, 0x1d, 0x57, 0xa4, 0xe7, 0x45, 0x5a }, + .len = 16 + } +}; + +static const struct aead_test_data gcm_test_case_2 = { + .algo = RTE_CRYPTO_AEAD_AES_GCM, + .key = { + .data = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .len = 16 + }, + .iv = { + .data = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00 }, + .len = 12 + }, + .aad = { + .data = gcm_aad_zero_text, + .len = 0 + }, + .plaintext = { + .data = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .len = 16 + }, + .ciphertext = { + .data = { + 0x03, 0x88, 0xda, 0xce, 0x60, 0xb6, 0xa3, 0x92, + 0xf3, 0x28, 0xc2, 0xb9, 0x71, 0xb2, 0xfe, 0x78 }, + .len = 16 + }, + .auth_tag = { + .data = { + 0xab, 0x6e, 0x47, 0xd4, 0x2c, 0xec, 0x13, 0xbd, + 0xf5, 0x3a, 0x67, 0xb2, 0x12, 0x57, 0xbd, 0xdf }, + .len = 16 + } +}; + +static const struct aead_test_data gcm_test_case_3 = { + .algo = RTE_CRYPTO_AEAD_AES_GCM, + .key = { + .data = { + 0xfe, 0xff, 0xe9, 0x92, 0x86, 0x65, 0x73, 0x1c, + 0x6d, 0x6a, 0x8f, 0x94, 0x67, 0x30, 0x83, 0x08 }, + .len = 16 + }, + .iv = { + .data = { + 0xca, 0xfe, 0xba, 0xbe, 0xfa, 0xce, 0xdb, 0xad, + 0xde, 0xca, 0xf8, 0x88 }, + .len = 12 + }, + .aad = { + .data = gcm_aad_zero_text, + .len = 0 + }, + .plaintext = { + .data = { + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x55 }, + .len = 64 + }, + .ciphertext = { + .data = { + 0x42, 0x83, 0x1e, 0xc2, 0x21, 0x77, 0x74, 0x24, + 0x4b, 0x72, 0x21, 0xb7, 0x84, 0xd0, 0xd4, 0x9c, + 0xe3, 0xaa, 0x21, 0x2f, 0x2c, 0x02, 0xa4, 0xe0, + 0x35, 0xc1, 0x7e, 0x23, 0x29, 0xac, 0xa1, 0x2e, + 0x21, 0xd5, 0x14, 0xb2, 0x54, 0x66, 0x93, 0x1c, + 0x7d, 0x8f, 0x6a, 0x5a, 0xac, 0x84, 0xaa, 0x05, + 0x1b, 0xa3, 0x0b, 0x39, 0x6a, 0x0a, 0xac, 0x97, + 0x3d, 0x58, 0xe0, 0x91, 0x47, 0x3f, 0x59, 0x85 + }, + .len = 64 + }, + .auth_tag = { + .data = { + 0x4d, 0x5c, 0x2a, 0xf3, 0x27, 0xcd, 0x64, 0xa6, + 0x2c, 0xf3, 0x5a, 0xbd, 0x2b, 0xa6, 0xfa, 0xb4 }, + .len = 16 + } +}; + +static const struct aead_test_data gcm_test_case_4 = { + .algo = RTE_CRYPTO_AEAD_AES_GCM, + .key = { + .data = { + 0xfe, 0xff, 0xe9, 0x92, 0x86, 0x65, 0x73, 0x1c, + 0x6d, 0x6a, 0x8f, 0x94, 0x67, 0x30, 0x83, 0x08 + }, + .len = 16 + }, + .iv = { + .data = { + 0xca, 0xfe, 0xba, 0xbe, 0xfa, 0xce, 0xdb, 0xad, + 0xde, 0xca, 0xf8, 0x88 }, + .len = 12 + }, + .aad = { + .data = gcm_aad_zero_text, + .len = 8 + }, + .plaintext = { + .data = { + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39 + }, + .len = 60 + }, + .ciphertext = { + .data = { + 0x42, 0x83, 0x1e, 0xc2, 0x21, 0x77, 0x74, 0x24, + 0x4b, 0x72, 0x21, 0xb7, 0x84, 0xd0, 0xd4, 0x9c, + 0xe3, 0xaa, 0x21, 0x2f, 0x2c, 0x02, 0xa4, 0xe0, + 0x35, 0xc1, 0x7e, 0x23, 0x29, 0xac, 0xa1, 0x2e, + 0x21, 0xd5, 0x14, 0xb2, 0x54, 0x66, 0x93, 0x1c, + 0x7d, 0x8f, 0x6a, 0x5a, 0xac, 0x84, 0xaa, 0x05, + 0x1b, 0xa3, 0x0b, 0x39, 0x6a, 0x0a, 0xac, 0x97, + 0x3d, 0x58, 0xe0, 0x91 + }, + .len = 60 + }, + .auth_tag = { + .data = { + 0xA2, 0xA4, 0x35, 0x75, 0xDC, 0xB0, 0x57, 0x74, + 0x07, 0x02, 0x30, 0xC2, 0xE7, 0x52, 0x02, 0x00 + }, + .len = 16 + } + +}; + +static const struct aead_test_data gcm_test_case_5 = { + .algo = RTE_CRYPTO_AEAD_AES_GCM, + .key = { + .data = { + 0xfe, 0xff, 0xe9, 0x92, 0x86, 0x65, 0x73, 0x1c, + 0x6d, 0x6a, 0x8f, 0x94, 0x67, 0x30, 0x83, 0x08 + }, + .len = 16 + }, + .iv = { + .data = { + 0xca, 0xfe, 0xba, 0xbe, 0xfa, 0xce, 0xdb, 0xad, + 0xde, 0xca, 0xf8, 0x88 }, + .len = 12 + }, + .aad = { + .data = gcm_aad_text, + .len = 8 + }, + .plaintext = { + .data = { + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39 + }, + .len = 60 + }, + .ciphertext = { + .data = { + 0x42, 0x83, 0x1e, 0xc2, 0x21, 0x77, 0x74, 0x24, + 0x4b, 0x72, 0x21, 0xb7, 0x84, 0xd0, 0xd4, 0x9c, + 0xe3, 0xaa, 0x21, 0x2f, 0x2c, 0x02, 0xa4, 0xe0, + 0x35, 0xc1, 0x7e, 0x23, 0x29, 0xac, 0xa1, 0x2e, + 0x21, 0xd5, 0x14, 0xb2, 0x54, 0x66, 0x93, 0x1c, + 0x7d, 0x8f, 0x6a, 0x5a, 0xac, 0x84, 0xaa, 0x05, + 0x1b, 0xa3, 0x0b, 0x39, 0x6a, 0x0a, 0xac, 0x97, + 0x3d, 0x58, 0xe0, 0x91 + }, + .len = 60 + }, + .auth_tag = { + .data = { + 0xC5, 0x2D, 0xFB, 0x54, 0xAF, 0xBB, 0x07, 0xA1, + 0x9A, 0xFF, 0xBE, 0xE0, 0x61, 0x4C, 0xE7, 0xA5 + }, + .len = 16 + } + +}; + +static const struct aead_test_data gcm_test_case_6 = { + .algo = RTE_CRYPTO_AEAD_AES_GCM, + .key = { + .data = { + 0xfe, 0xff, 0xe9, 0x92, 0x86, 0x65, 0x73, 0x1c, + 0x6d, 0x6a, 0x8f, 0x94, 0x67, 0x30, 0x83, 0x08 + }, + .len = 16 + }, + .iv = { + .data = { + 0xca, 0xfe, 0xba, 0xbe, 0xfa, 0xce, 0xdb, 0xad, + 0xde, 0xca, 0xf8, 0x88 + }, + .len = 12 + }, + .aad = { + .data = gcm_aad_zero_text, + .len = 12 + }, + .plaintext = { + .data = { + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39 + }, + .len = 60 + }, + .ciphertext = { + .data = { + 0x42, 0x83, 0x1e, 0xc2, 0x21, 0x77, 0x74, 0x24, + 0x4b, 0x72, 0x21, 0xb7, 0x84, 0xd0, 0xd4, 0x9c, + 0xe3, 0xaa, 0x21, 0x2f, 0x2c, 0x02, 0xa4, 0xe0, + 0x35, 0xc1, 0x7e, 0x23, 0x29, 0xac, 0xa1, 0x2e, + 0x21, 0xd5, 0x14, 0xb2, 0x54, 0x66, 0x93, 0x1c, + 0x7d, 0x8f, 0x6a, 0x5a, 0xac, 0x84, 0xaa, 0x05, + 0x1b, 0xa3, 0x0b, 0x39, 0x6a, 0x0a, 0xac, 0x97, + 0x3d, 0x58, 0xe0, 0x91 + }, + .len = 60 + }, + .auth_tag = { + .data = { + 0x74, 0xFC, 0xFA, 0x29, 0x3E, 0x60, 0xCC, 0x66, + 0x09, 0xD6, 0xFD, 0x00, 0xC8, 0x86, 0xD5, 0x42 + }, + .len = 16 + } +}; + +static const struct aead_test_data gcm_test_case_7 = { + .algo = RTE_CRYPTO_AEAD_AES_GCM, + .key = { + .data = { + 0xfe, 0xff, 0xe9, 0x92, 0x86, 0x65, 0x73, 0x1c, + 0x6d, 0x6a, 0x8f, 0x94, 0x67, 0x30, 0x83, 0x08 + }, + .len = 16 + }, + .iv = { + .data = { + 0xca, 0xfe, 0xba, 0xbe, 0xfa, 0xce, 0xdb, 0xad, + 0xde, 0xca, 0xf8, 0x88 + }, + .len = 12 + }, + .aad = { + .data = gcm_aad_text, + .len = 12 + }, + .plaintext = { + .data = { + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39 + }, + .len = 60 + }, + .ciphertext = { + .data = { + 0x42, 0x83, 0x1e, 0xc2, 0x21, 0x77, 0x74, 0x24, + 0x4b, 0x72, 0x21, 0xb7, 0x84, 0xd0, 0xd4, 0x9c, + 0xe3, 0xaa, 0x21, 0x2f, 0x2c, 0x02, 0xa4, 0xe0, + 0x35, 0xc1, 0x7e, 0x23, 0x29, 0xac, 0xa1, 0x2e, + 0x21, 0xd5, 0x14, 0xb2, 0x54, 0x66, 0x93, 0x1c, + 0x7d, 0x8f, 0x6a, 0x5a, 0xac, 0x84, 0xaa, 0x05, + 0x1b, 0xa3, 0x0b, 0x39, 0x6a, 0x0a, 0xac, 0x97, + 0x3d, 0x58, 0xe0, 0x91 + }, + .len = 60 + }, + .auth_tag = { + .data = { + 0xE9, 0xE4, 0xAB, 0x76, 0xB7, 0xFF, 0xEA, 0xDC, + 0x69, 0x79, 0x38, 0xA2, 0x0D, 0xCA, 0xF5, 0x92 + }, + .len = 16 + } +}; + +static const struct aead_test_data gcm_test_case_8 = { + .algo = RTE_CRYPTO_AEAD_AES_GCM, + .key = { + .data = { + 0xfe, 0xff, 0xe9, 0x92, 0x86, 0x65, 0x73, 0x1c, + 0x6d, 0x6a, 0x8f, 0x94, 0x67, 0x30, 0x83, 0x08 + }, + .len = 16 + }, + .iv = { + .data = { + 0xca, 0xfe, 0xba, 0xbe, 0xfa, 0xce, 0xdb, 0xad, + 0xde, 0xca, 0xf8, 0x88 + }, + .len = 12 + }, + .aad = { + .data = gcm_aad_text, + .len = 12 + }, + .plaintext = { + .data = { + 0xC5, 0x34, 0x2E, 0x83, 0xEB, 0x4C, 0x02, 0x03, + 0xF7, 0xB2, 0x57, 0x35, 0x26, 0x81, 0x63, 0xAE, + 0x1F, 0xCD, 0x2D, 0x02, 0x91, 0x5A, 0xDB, 0x3A, + 0xF1, 0x38, 0xD8, 0x75, 0x86, 0x20, 0xCC, 0x1E, + 0xE6, 0xDC, 0xFF, 0xB5, 0xEA, 0x0E, 0x18, 0x7A, + 0x86, 0x6C, 0xAB, 0x39, 0x2D, 0x90, 0xAC, 0x77, + 0x5D, 0xED, 0x65, 0xB3, 0x05, 0x29, 0xBB, 0x09, + 0xD0, 0x21, 0x74, 0x6A, 0x67, 0x1C, 0x95, 0x42, + 0x55, 0xAD, 0xC8, 0x91, 0x28, 0xFE, 0x16, 0x9A, + 0xE1, 0xCB, 0xCD, 0x68, 0x3B, 0xDF, 0x3E, 0x3A, + 0x34, 0xFE, 0x9B, 0xFB, 0xF5, 0x15, 0x2A, 0x29, + 0x18, 0x99, 0x24, 0xBF, 0xB6, 0x43, 0xDB, 0xD1, + 0x69, 0x26, 0x1E, 0x31, 0x2C, 0x8C, 0x3C, 0x6B, + 0x7F, 0x06, 0xA6, 0x03, 0xE2, 0x1A, 0x50, 0xFE, + 0x7C, 0x69, 0xE5, 0x5F, 0x35, 0x93, 0xE9, 0x20, + 0x14, 0xB1, 0xCA, 0x61, 0xE7, 0x9C, 0x89, 0x08, + 0xD6, 0xB1, 0xC2, 0x63, 0x1B, 0x86, 0x5E, 0xF1, + 0xF5, 0x23, 0x0E, 0x9B, 0xE5, 0xBD, 0x5D, 0x04, + 0xF7, 0xEF, 0x8E, 0x46, 0xB0, 0x11, 0x4F, 0x69, + 0x62, 0x35, 0x51, 0xB7, 0x24, 0xA2, 0x31, 0xD0, + 0x32, 0x4E, 0xB8, 0x44, 0xC7, 0x59, 0xDE, 0x25, + 0xEA, 0x2D, 0x00, 0x0E, 0xF1, 0x07, 0xBA, 0xBB, + 0x9A, 0xBC, 0x4F, 0x57, 0xB7, 0x13, 0x57, 0xEF, + 0xD9, 0xF6, 0x80, 0x69, 0xEA, 0xE8, 0x47, 0x9C, + 0x51, 0x71, 0xE6, 0x8F, 0x69, 0x29, 0xB4, 0x60, + 0xE8, 0x50, 0xE5, 0xD0, 0x9B, 0xD2, 0x62, 0x6F, + 0x09, 0x5C, 0xD1, 0x4B, 0x85, 0xE2, 0xFD, 0xD3, + 0xEB, 0x28, 0x55, 0x77, 0x97, 0xCA, 0xD6, 0xA8, + 0xDC, 0x35, 0x68, 0xF7, 0x6A, 0xCF, 0x48, 0x3F, + 0x49, 0x31, 0x00, 0x65, 0xB7, 0x31, 0x1A, 0x49, + 0x75, 0xDE, 0xCE, 0x7F, 0x18, 0xB5, 0x31, 0x9A, + 0x64, 0x6D, 0xE5, 0x49, 0x1D, 0x6D, 0xF2, 0x21, + 0x9F, 0xF5, 0xFF, 0x7C, 0x41, 0x30, 0x33, 0x06, + 0x7B, 0xA4, 0xD8, 0x99, 0xF6, 0xCC, 0xDF, 0xC4, + 0x3F, 0xF3, 0xCD, 0xE7, 0x74, 0xC4, 0x4A, 0x19, + 0x5C, 0xCA, 0x42, 0x31, 0xF1, 0x3B, 0x65, 0x1C, + 0x3D, 0x56, 0x08, 0xBE, 0x15, 0x37, 0x23, 0x50, + 0xD6, 0xA3, 0x57, 0x64, 0x25, 0xBE, 0xDA, 0xC2, + 0x4E, 0xF5, 0x1A, 0xAD, 0x6F, 0x43, 0x78, 0x21, + 0xF9, 0x36, 0x39, 0x1F, 0x5F, 0xF7, 0x1B, 0xA0, + 0xEE, 0x8B, 0x4F, 0x8A, 0x9D, 0xD8, 0xED, 0x37, + 0xCE, 0x0D, 0x70, 0xE0, 0x3F, 0xE7, 0x11, 0x30, + 0x17, 0x1D, 0x03, 0x5E, 0xA0, 0x3D, 0x3F, 0x9E, + 0xF5, 0xD3, 0x74, 0x2E, 0xC1, 0xD6, 0xFF, 0xF7, + 0x2E, 0xE7, 0x80, 0x88, 0xCF, 0x0E, 0x7F, 0x12, + 0x71, 0x62, 0xC7, 0xF1, 0xC4, 0x2B, 0x64, 0x5D, + 0x1C, 0x9A, 0xB4, 0xCB, 0xB8, 0x24, 0xB3, 0x0B, + 0x33, 0xF2, 0x8A, 0x8F, 0x76, 0xC8, 0x81, 0xDA, + 0x1A, 0x10, 0xB5, 0xA9, 0xCD, 0xDC, 0x1A, 0x02, + 0xC1, 0xAE, 0x4F, 0x02, 0x1B, 0x13, 0x96, 0x5A, + 0x2E, 0x03, 0xA2, 0x68, 0xB2, 0x29, 0xAC, 0x28, + 0xB8, 0xDC, 0xD5, 0x27, 0x55, 0xEC, 0x43, 0xDC, + 0xB7, 0x49, 0x1D, 0xE1, 0x30, 0x25, 0x81, 0xA6, + 0x90, 0x1F, 0x75, 0xBA, 0x19, 0x1E, 0xF7, 0xC5, + 0x77, 0x35, 0xEE, 0x68, 0x71, 0x22, 0xA0, 0xB4, + 0xCC, 0x99, 0x86, 0x1B, 0x1B, 0xC8, 0x27, 0xFC, + 0x6D, 0x8D, 0xE7, 0x8B, 0xC3, 0x40, 0x3D, 0xA8, + 0xCB, 0x9B, 0xC4, 0x12, 0x07, 0xDD, 0xA1, 0x92, + 0xE5, 0x80, 0x7A, 0xF4, 0xDB, 0x4C, 0xE6, 0xEE, + 0xF9, 0xD5, 0x1C, 0x20, 0x18, 0xD3, 0x8F, 0xDF, + 0x1C, 0xD3, 0x51, 0x4E, 0x0E, 0xED, 0x06, 0x61, + 0xF7, 0xBA, 0x81, 0x3A, 0x2F, 0xEA, 0xED, 0x70, + 0xA9, 0xD9, 0x54, 0x4D, 0xFC, 0x1D, 0x19, 0xEA, + 0xA6, 0x39, 0x8C, 0x6C, 0x78, 0xA8, 0x05, 0xEB, + 0xF2, 0xB5, 0xDE, 0x06, 0x9D, 0x8A, 0x78, 0x2A, + 0xF5, 0x50, 0xA4, 0xBD, 0x9B, 0xDA, 0xCA, 0x66, + 0xC0, 0x23, 0xAB, 0xE8, 0x95, 0x7E, 0xC9, 0xD2, + 0x6F, 0x09, 0xF2, 0x9A, 0x17, 0x89, 0xDA, 0x47, + 0x65, 0x8C, 0x20, 0xFA, 0x4E, 0x86, 0x18, 0xEB, + 0x7C, 0x08, 0xEC, 0x8A, 0x05, 0x54, 0x96, 0xD2, + 0x7A, 0x8A, 0x81, 0x58, 0x75, 0x8C, 0x7B, 0x02, + 0xEE, 0x1F, 0x51, 0x88, 0xD0, 0xD1, 0x90, 0x99, + 0x0C, 0xAE, 0x51, 0x2E, 0x54, 0x3E, 0xB1, 0x7D, + 0xBC, 0xE8, 0x54, 0x93, 0x6D, 0x10, 0x3C, 0xC6, + 0x71, 0xF6, 0xF5, 0x0B, 0x07, 0x0A, 0x6E, 0x59, + 0x20, 0x45, 0x21, 0x7D, 0x37, 0x64, 0x92, 0x09, + 0xA7, 0xE2, 0x34, 0x6F, 0xFC, 0xCC, 0x66, 0x0E, + 0x88, 0x1B, 0x19, 0x86, 0x11, 0xD7, 0x81, 0x25, + 0xF1, 0x8A, 0x03, 0xB7, 0x7A, 0xF0, 0x98, 0x4A, + 0x5C, 0xA1, 0x6D, 0x85, 0xA4, 0x8C, 0x4B, 0x65, + 0x9F, 0x72, 0x64, 0x14, 0xBA, 0x74, 0xEE, 0xA3, + 0x88, 0xFE, 0x1B, 0xCF, 0x11, 0x4F, 0xD1, 0xAC, + 0xFA, 0x14, 0xC3, 0xA7, 0xDD, 0x06, 0x85, 0x4E, + 0x64, 0x06, 0x92, 0x9C, 0xDF, 0x06, 0x09, 0xF1, + 0x4D, 0xE8, 0xF8, 0x2F, 0x69, 0xB6, 0x8A, 0xAF, + 0x25, 0x21, 0xB5, 0x48, 0x59, 0xF8, 0x9D, 0x60, + 0xAE, 0x42, 0x11, 0x7A, 0x68, 0x4D, 0x7E, 0x76, + 0xB0, 0xD2, 0xE3, 0xD9, 0x24, 0x16, 0x20, 0x0A, + 0xEB, 0xE0, 0x68, 0xCB, 0xBC, 0xAB, 0x67, 0xE4, + 0xF3, 0x25, 0x1F, 0xD3, 0x85, 0xA7, 0x1D, 0x7E, + 0x3C, 0x63, 0xCB, 0xC2, 0x50, 0x90, 0x0F, 0x4B, + 0x6E, 0x68, 0x06, 0x84, 0x65, 0xF7, 0xD0, 0xD4, + 0x12, 0xED, 0xFA, 0xC9, 0x40, 0xE2, 0xC0, 0xC9, + 0x46, 0x22, 0x47, 0x5E, 0x6D, 0xC1, 0x63, 0xDB, + 0x51, 0x98, 0xDA, 0x1A, 0xC4, 0xB9, 0xED, 0xE9, + 0x09, 0xB9, 0xCF, 0x91, 0x04, 0x1C, 0x63, 0xD8, + 0xC5, 0xA5, 0xAE, 0x53, 0x7B, 0xA1, 0x29, 0x83, + 0x37, 0xFB, 0xBF, 0x96, 0xBB, 0x24, 0x3D, 0x77, + 0x8C, 0x0F, 0xB3, 0x4B, 0x66, 0x9C, 0x54, 0xBB, + 0xF6, 0xDD, 0xD1, 0xB4, 0xD2, 0xF6, 0xAA, 0xED, + 0x18, 0x56, 0x63, 0x3E, 0x0B, 0xCA, 0xAB, 0x70, + 0xBB, 0x63, 0xEA, 0xB1, 0x00, 0x65, 0x90, 0x18, + 0xB8, 0x63, 0xA2, 0xF2, 0xB6, 0x1E, 0x61, 0x7B, + 0xD5, 0x01, 0xD9, 0x4D, 0xC9, 0x9D, 0x99, 0xC1, + 0x57, 0x9D, 0x6F, 0xAE, 0x64, 0xE4, 0x0C, 0x7E, + 0xFA, 0x15, 0x5E, 0xB6, 0x43, 0xB8, 0x8B, 0x89, + 0x87, 0xCD, 0x4F, 0xAD, 0x30, 0x1E, 0xA5, 0x03, + 0x7A, 0xC2, 0x10, 0x42, 0x14, 0x88, 0xD6, 0x7A, + 0x6D, 0x56, 0x52, 0x2E, 0x8D, 0x1B, 0x5D, 0x36, + 0x27, 0xA0, 0x21, 0x4B, 0x64, 0xF0, 0xC5, 0x41, + 0xAD, 0x05, 0x4A, 0x24, 0xE4, 0x70, 0x88, 0x63, + 0x12, 0xD0, 0xBC, 0x05, 0x38, 0xD9, 0x41, 0x68, + 0x9F, 0x16, 0x9A, 0x54, 0x09, 0x21, 0x64, 0x36, + 0x63, 0x97, 0x3A, 0xB5, 0xE0, 0x25, 0x43, 0x8A, + 0x6A, 0x59, 0x97, 0xC1, 0x31, 0xA5, 0x66, 0xD2, + 0xF0, 0x1C, 0xDF, 0x97, 0x51, 0xD0, 0x61, 0xBA, + 0x55, 0x5F, 0xD7, 0x0D, 0xD4, 0x75, 0x8E, 0x79, + 0x04, 0x75, 0x00, 0xB9, 0xC0, 0x7A, 0x66, 0x05, + 0x9F, 0x2B, 0x44, 0x42, 0x75, 0x0F, 0xD5, 0x15, + 0xD6, 0x16, 0x8F, 0x6C, 0x6E, 0xD4, 0x37, 0xCF, + 0xB4, 0xDA, 0x93, 0x00, 0x11, 0xFB, 0xBE, 0xEE, + 0x3B, 0x6D, 0x1D, 0xBA, 0x33, 0xD1, 0x52, 0x8B, + 0x16, 0x39, 0x42, 0x27, 0xE6, 0x56, 0x4C, 0x41, + 0x91, 0xB0, 0x98, 0xAE, 0x9B, 0x2D, 0x9B, 0x23, + 0x80, 0x4C, 0xEA, 0x98, 0x57, 0x95, 0x28, 0x94, + 0x43, 0xD3, 0x88, 0x12, 0xDF, 0x89, 0x5A, 0x7B, + 0xC5, 0xCB, 0x36, 0x54, 0x65, 0x74, 0xB8, 0x4E, + 0xE2, 0x4D, 0x01, 0xD5, 0x9C, 0x82, 0xB9, 0x1A, + 0x09, 0xD2, 0xCE, 0x04, 0x36, 0xD8, 0x41, 0xAC, + 0x4C, 0xAD, 0xC6, 0x52, 0x91, 0x1A, 0x06, 0x6D, + 0xFC, 0xAB, 0x29, 0x93, 0x87, 0x88, 0xB9, 0x8C, + 0xFA, 0x57, 0x2B, 0x05, 0x03, 0xD0, 0x18, 0xED, + 0x7A, 0x7B, 0x81, 0x6A, 0x97, 0x65, 0x5B, 0x90, + 0xDE, 0xA9, 0xFC, 0x8F, 0xFC, 0xBB, 0x98, 0xD8, + 0xFA, 0x32, 0x3F, 0x3F, 0x7F, 0x74, 0x65, 0x38, + 0xC4, 0x28, 0xEC, 0x27, 0x1F, 0x28, 0x01, 0xB1, + 0xAF, 0x2B, 0x8A, 0x05, 0x38, 0x7B, 0x77, 0xC9, + 0x61, 0x77, 0x34, 0x2C, 0x22, 0xE5, 0xEB, 0xDC, + 0x9D, 0x18, 0x6E, 0x23, 0x25, 0x52, 0x69, 0xB7, + 0x05, 0xDB, 0x66, 0x5D, 0xEA, 0x76, 0x83, 0x82, + 0x97, 0x39, 0xAF, 0xC0, 0x50, 0x81, 0x18, 0x0D, + 0x22, 0xFA, 0xB7, 0x44, 0x5C, 0x3F, 0x69, 0xF3, + 0xAC, 0xC5, 0x63, 0x9F, 0xD8, 0x72, 0x7E, 0x9A, + 0xC2, 0xEB, 0x79, 0xD0, 0x74, 0x65, 0xE8, 0xCA, + 0xFD, 0xA8, 0x7D, 0x23, 0x07, 0x99, 0x3E, 0xAF, + 0xDB, 0x67, 0x10, 0xC0, 0xE5, 0x61, 0x77, 0xC6, + 0x8D, 0xC4, 0x0E, 0xAA, 0x55, 0xE3, 0xC0, 0xC7, + 0xA5, 0x36, 0x28, 0x61, 0xDB, 0x16, 0x96, 0x5E, + 0x01, 0x47, 0x82, 0xE3, 0xEB, 0x20, 0x3F, 0x10, + 0xFA, 0x5A, 0xBC, 0xD3, 0xF9, 0xCE, 0x04, 0x87, + 0x51, 0x07, 0xF9, 0xD0, 0xE7, 0x6D, 0xCB, 0xCC, + 0xC4, 0x15, 0x00, 0xE2, 0xDC, 0x8E, 0x7B, 0x5C, + 0x9A, 0xF2, 0x78, 0x70, 0x4D, 0xA1, 0xAA, 0xB5, + 0x13, 0xCC, 0x71, 0x66, 0x5A, 0x79, 0x13, 0x3B, + 0x12, 0xCD, 0x40, 0x30, 0x5A, 0x49, 0xD4, 0x20, + 0xED, 0xCF, 0x4A, 0x75, 0xE6, 0xD5, 0xDD, 0x0F, + 0xD4, 0xBE, 0x98, 0x9F, 0xD7, 0x1F, 0xC0, 0x02, + 0x31, 0xFA, 0x67, 0x37, 0x25, 0x86, 0x56, 0x85, + 0x2B, 0xA2, 0x57, 0xCD, 0x8E, 0x74, 0xE7, 0x69, + 0xEE, 0x33, 0x5A, 0x3F, 0xCD, 0x1E, 0xE3, 0xB9, + 0xAA, 0x52, 0xF5, 0x22, 0x4E, 0xE3, 0xFF, 0xC8, + 0xE3, 0x13, 0xA3, 0x9A, 0x63, 0x23, 0xC3, 0xD7, + 0xE5, 0x88, 0x3E, 0x0A, 0x4B, 0xA5, 0x01, 0xE6, + 0x13, 0xCF, 0xED, 0xEE, 0x2A, 0x58, 0x09, 0x3F, + 0x2F, 0x28, 0xE7, 0xC4, 0x6B, 0xEC, 0x49, 0x51, + 0x79, 0x8F, 0xD5, 0x19, 0x5D, 0xA5, 0x10, 0xCE, + 0x8E, 0xF6, 0x26, 0x78, 0x7A, 0xA8, 0x11, 0x52, + 0x5F, 0x97, 0x14, 0xC9, 0x29, 0x87, 0xB8, 0xA0, + 0x2D, 0xE6, 0xA7, 0x2A, 0xD4, 0xFF, 0xEB, 0xBA, + 0xFD, 0x58, 0x39, 0x33, 0xB1, 0xCE, 0x0E, 0x78, + 0x67, 0x1E, 0xA1, 0x92, 0x77, 0x63, 0xF8, 0xC0, + 0x02, 0x49, 0x73, 0xC0, 0xA1, 0x26, 0x83, 0x04, + 0x9A, 0x5D, 0x85, 0x68, 0x2A, 0x2F, 0xCB, 0x88, + 0x8D, 0x14, 0xB1, 0x33, 0xFA, 0xFB, 0xE9, 0x05, + 0xBE, 0x24, 0x1A, 0x6B, 0x29, 0x2B, 0x3F, 0x52, + 0x8F, 0xFB, 0xE6, 0x02, 0x77, 0x50, 0x71, 0xDB, + 0xE9, 0x92, 0x3F, 0xE1, 0x20, 0x62, 0x80, 0xAE, + 0xA4, 0x98, 0xC6, 0xCD, 0xE0, 0xB1, 0xC3, 0x33, + 0xB1, 0xC5, 0x91, 0x3C, 0x19, 0x34, 0xA8, 0xD9, + 0xB3, 0x25, 0x69, 0xE3, 0x9C, 0x5F, 0x78, 0xD0, + 0x83, 0x1F, 0xAB, 0x85, 0x13, 0x56, 0x69, 0xB5, + 0x06, 0x47, 0x62, 0x37, 0x27, 0x15, 0x14, 0x05, + 0x4A, 0xF4, 0x6A, 0x68, 0x2A, 0x6A, 0xC3, 0x5A, + 0xDF, 0xB5, 0xAE, 0x2F, 0x8D, 0x8F, 0x21, 0xDB, + 0x33, 0x00, 0x9B, 0xD4, 0xC4, 0x08, 0x3B, 0x81, + 0x63, 0x4C, 0xB0, 0x39, 0x4C, 0x0A, 0xD5, 0x71, + 0x3E, 0x5A, 0x50, 0x58, 0x9C, 0x07, 0x89, 0x79, + 0x79, 0x2F, 0x0B, 0xD9, 0x50, 0xBC, 0xCF, 0x46, + 0x7A, 0x68, 0x5C, 0xBF, 0x1E, 0x49, 0x77, 0x92, + 0x85, 0x11, 0x39, 0xA6, 0x2F, 0xDA, 0x7B, 0xFA, + 0x72, 0x87, 0x06, 0xCD, 0x84, 0x41, 0x20, 0x1B, + 0x66, 0x3F, 0x42, 0x0C, 0x9E, 0x19, 0xD3, 0x18, + 0x57, 0xA0, 0xEE, 0x16, 0x3A, 0xC7, 0xF9, 0xD3, + 0x8B, 0xC9, 0x24, 0x70, 0x70, 0x51, 0x7C, 0x06, + 0x68, 0xD3, 0x29, 0xC9, 0x85, 0x9A, 0x1C, 0xE6, + 0x8C, 0x17, 0xF4, 0x88, 0xDF, 0xEA, 0xFF, 0x44, + 0x8D, 0x54, 0xBE, 0x22, 0x07, 0xA5, 0x7C, 0x0C, + 0xF4, 0x8D, 0xB1, 0x0C, 0x07, 0xED, 0xBD, 0x28, + 0x19, 0xDA, 0x07, 0x71, 0xA8, 0xA1, 0xE0, 0xDD, + 0xEE, 0x08, 0x18, 0xA5, 0xBD, 0xDD, 0x32, 0x0B, + 0x70, 0x1C, 0xD9, 0xEE, 0x19, 0xC2, 0xAE, 0x5C, + 0xE3, 0x02, 0x74, 0x70, 0x96, 0x61, 0xB1, 0x73, + 0x3B, 0xD6, 0x74, 0xC0, 0x82, 0xA9, 0x1F, 0xE0, + 0xF1, 0x22, 0x50, 0xF3, 0x9F, 0xE5, 0x13, 0x92, + 0xFC, 0x0A, 0x1A, 0x3C, 0xB4, 0x46, 0xFB, 0x81, + 0x00, 0x84, 0xA4, 0x5E, 0x6B, 0x8C, 0x25, 0x6E, + 0xD7, 0xB7, 0x3B, 0x01, 0x65, 0xFB, 0x0B, 0x46, + 0x67, 0x27, 0x2D, 0x51, 0xAD, 0xB5, 0xE0, 0x85, + 0xC2, 0x95, 0xA3, 0xE3, 0x68, 0x4D, 0x9E, 0x8C, + 0x11, 0x53, 0xF0, 0xB2, 0x85, 0xFA, 0x52, 0x4E, + 0xEC, 0xF9, 0xB7, 0x3C, 0x89, 0x2C, 0x4D, 0x32, + 0x9A, 0xCB, 0x17, 0xF3, 0x16, 0xBF, 0x44, 0x40, + 0xE9, 0x5E, 0x51, 0x8C, 0x1E, 0x52, 0x0A, 0xC2, + 0xCD, 0xA5, 0xAA, 0x03, 0x27, 0xB0, 0x8F, 0x64, + 0xDB, 0xD7, 0x03, 0x01, 0x8A, 0x24, 0x28, 0x7E, + 0x53, 0x6F, 0x24, 0xFD, 0xAA, 0xE3, 0x78, 0xB6, + 0xA5, 0x5D, 0x5A, 0x67, 0x20, 0xE2, 0xBE, 0x3A, + 0x2B, 0xE7, 0x86, 0x11, 0xDD, 0x96, 0xCB, 0x09, + 0x65, 0xA0, 0x36, 0xF9, 0xB0, 0x20, 0x21, 0x8E, + 0xDB, 0xC0, 0x73, 0xC7, 0x79, 0xD8, 0xDA, 0xC2, + 0x66, 0x13, 0x64, 0x34, 0x0C, 0xE1, 0x22, 0x24, + 0x61, 0x67, 0x08, 0x39, 0x97, 0x3F, 0x33, 0x96, + 0xF2, 0x44, 0x18, 0x75, 0xBB, 0xF5, 0x6A, 0x5C, + 0x2C, 0xAE, 0x2A, 0x79, 0x3D, 0x47, 0x19, 0x53, + 0x50, 0x6C, 0x9F, 0xB3, 0x82, 0x55, 0x09, 0x78, + 0x7B, 0xAD, 0xBC, 0x05, 0x6F, 0xC8, 0x3D, 0xB6, + 0x7B, 0x30, 0xE6, 0xBB, 0x8B, 0xD0, 0x2F, 0xA6, + 0x15, 0xCC, 0x77, 0x8C, 0x21, 0xBA, 0x03, 0xED, + 0x56, 0x85, 0x82, 0x4F, 0x97, 0x8C, 0x59, 0x4F, + 0x53, 0x5A, 0xD2, 0x70, 0xD9, 0x07, 0xB3, 0xBD, + 0x1D, 0x3E, 0x97, 0xD4, 0x7D, 0x93, 0x35, 0xA4, + 0x82, 0x6E, 0xEA, 0x4B, 0xC8, 0x6C, 0xF5, 0xE6, + 0xEB, 0xAF, 0x11, 0xB0, 0xB4, 0x71, 0x8F, 0x7B, + 0xC4, 0x8C, 0xE2, 0x66, 0x51, 0x31, 0x99, 0x01, + 0x5B, 0xE7, 0x48, 0xF8, 0x4C, 0xE3, 0x9A, 0x77, + 0xF1, 0xC6, 0x09, 0xDE, 0x76, 0xD4, 0xE3, 0x5C, + 0xDF, 0xA3, 0xEC, 0x3C, 0x86, 0x7C, 0xA5, 0x3F, + 0x8D, 0x2A, 0xF3, 0x0B, 0x54, 0xB7, 0x54, 0xA2, + 0xC1, 0x69, 0xC0, 0x6F, 0x1C, 0x1C, 0x76, 0xD8, + 0x9F, 0x7A, 0x32, 0xB0, 0xA1, 0xA6, 0x9B, 0xB7, + 0x21, 0x56, 0x28, 0x2D, 0xB6, 0x97, 0x03, 0x5E, + 0x65, 0xE3, 0x74, 0x9A, 0x96, 0x7A, 0xF9, 0xF5, + 0xDD, 0x85, 0xCA, 0x4C, 0xB4, 0x03, 0x6A, 0xCD, + 0xB6, 0x01, 0xDC, 0x8B, 0xD8, 0x73, 0x8F, 0x4D, + 0x7F, 0xD6, 0x71, 0xEC, 0xD7, 0xC6, 0x0B, 0x5F, + 0x09, 0x21, 0xB2, 0x78, 0xA8, 0xAF, 0xAD, 0x2C, + 0xD4, 0x93, 0x9F, 0x71, 0xF7, 0x05, 0x89, 0x42, + 0xC9, 0x15, 0x6F, 0x2D, 0xE0, 0xBA, 0xC3, 0xD6, + 0xBF, 0xAC, 0xF8, 0x24, 0x58, 0x79, 0xA9, 0xC4, + 0xB4, 0x49, 0x3E, 0x0B, 0x9E, 0x5E, 0xE4, 0xA6, + 0x8B, 0xE8, 0xDE, 0xFB, 0x4A, 0xF1, 0x69, 0x9D, + 0x4F, 0x77, 0x83, 0x78, 0x55, 0x19, 0x42, 0x45, + 0xBF, 0xBD, 0xBD, 0x12, 0x0F, 0xEF, 0x8D, 0x04, + 0xD8, 0x5C, 0xF2, 0xC9, 0xF1, 0xA6, 0xE0, 0x3E, + 0x22, 0xA8, 0xA2, 0x5E, 0x66, 0xE9, 0xAB, 0xB4, + 0x71, 0xBE, 0x4B, 0x3F, 0xBE, 0xC4, 0xBA, 0x4A + }, + .len = 2048 + }, + .ciphertext = { + .data = { + 0x5E, 0x86, 0x02, 0x64, 0x32, 0xBF, 0x70, 0xC2, + 0x19, 0x99, 0x7F, 0x47, 0x0D, 0xA4, 0x91, 0xA8, + 0x7A, 0xC0, 0xA5, 0x7E, 0xA8, 0x6C, 0x88, 0x00, + 0xEA, 0xB5, 0x96, 0x6B, 0x25, 0xBD, 0xE7, 0x42, + 0xDB, 0x35, 0xE7, 0x92, 0x2B, 0x00, 0x82, 0x35, + 0xD4, 0x2C, 0xCF, 0x47, 0xC8, 0xB2, 0xB3, 0x57, + 0xF7, 0x24, 0x83, 0x7F, 0xC5, 0x2E, 0xF1, 0xC9, + 0x57, 0x1A, 0xEF, 0xC2, 0x3A, 0x8C, 0x1E, 0x92, + 0x88, 0x05, 0xAF, 0x55, 0xE6, 0x0C, 0xA7, 0x6B, + 0x59, 0x62, 0x32, 0x21, 0xF1, 0xFF, 0xB5, 0x5B, + 0x22, 0x26, 0x6F, 0x0A, 0x36, 0xDC, 0x0D, 0x16, + 0x3B, 0x4E, 0x7C, 0xA3, 0x75, 0x30, 0x3F, 0xB0, + 0x99, 0x38, 0x42, 0x8E, 0x89, 0xA3, 0x7C, 0x99, + 0x2F, 0x0A, 0xA1, 0xC7, 0xFD, 0x2D, 0x21, 0x8F, + 0xBD, 0xD4, 0x11, 0xEA, 0x55, 0xF5, 0x6A, 0x50, + 0x90, 0x3B, 0x60, 0x57, 0xE1, 0x86, 0x1E, 0x50, + 0x28, 0x67, 0x3F, 0xD2, 0xF3, 0xBD, 0xFA, 0xEE, + 0xD6, 0x5A, 0x38, 0x30, 0xA3, 0xDD, 0x78, 0xC4, + 0x37, 0x59, 0x52, 0xC0, 0x92, 0x54, 0xC7, 0x53, + 0xF0, 0xE6, 0xA9, 0x63, 0x1F, 0x9B, 0x97, 0xFB, + 0x40, 0x23, 0xFE, 0x52, 0x6A, 0xF0, 0x3A, 0x94, + 0xEB, 0x6A, 0x9E, 0x8F, 0xC5, 0x05, 0x9C, 0x04, + 0x1B, 0x00, 0x34, 0x96, 0x12, 0xDA, 0x60, 0xC6, + 0xAA, 0x1A, 0x3E, 0xEB, 0x70, 0x17, 0x10, 0xBC, + 0xF5, 0xC2, 0xE2, 0x71, 0xF3, 0xB8, 0x1D, 0xCE, + 0x47, 0x94, 0x21, 0x71, 0x34, 0x8C, 0xCC, 0xDD, + 0x27, 0xCE, 0x6F, 0x68, 0xFF, 0x91, 0x4E, 0xC4, + 0xA0, 0xCA, 0xB0, 0x4F, 0x17, 0x53, 0x73, 0x92, + 0x6C, 0xA8, 0x16, 0x06, 0xE3, 0xD9, 0x92, 0x99, + 0xBE, 0xB0, 0x7D, 0x56, 0xF2, 0x72, 0x30, 0xDA, + 0xC4, 0x4E, 0xF4, 0xA6, 0x8F, 0xD2, 0xC7, 0x8A, + 0xA2, 0xFC, 0xF5, 0x63, 0x17, 0x48, 0x56, 0x4D, + 0xBE, 0x94, 0xFE, 0xF5, 0xB1, 0xA9, 0x96, 0xAB, + 0x3F, 0x2D, 0xD4, 0x15, 0xEE, 0x4F, 0xFA, 0x2C, + 0xBE, 0x91, 0xB7, 0xBC, 0x18, 0xC8, 0xDB, 0x02, + 0x20, 0x29, 0xF1, 0xC1, 0x88, 0x8C, 0x8D, 0xD1, + 0xB3, 0x4E, 0x93, 0x96, 0xDD, 0x22, 0xAB, 0x55, + 0xB5, 0x9F, 0x8B, 0x20, 0xAE, 0xC6, 0x0E, 0x26, + 0xC6, 0xFE, 0x2D, 0x5F, 0x95, 0x89, 0x06, 0x15, + 0x3D, 0x88, 0x16, 0xEC, 0x9B, 0x4A, 0x1B, 0x5D, + 0x2E, 0xB2, 0x13, 0x56, 0x9F, 0x33, 0xB3, 0x45, + 0xBF, 0x5F, 0x25, 0x7E, 0x75, 0x22, 0xD2, 0xE6, + 0x9F, 0xAC, 0x2D, 0xFD, 0x99, 0xC2, 0x9B, 0xFC, + 0xD7, 0x7A, 0x9B, 0x05, 0x30, 0x0F, 0xB7, 0x4A, + 0xFE, 0x24, 0xDD, 0x39, 0x9B, 0xBB, 0x2F, 0xDD, + 0xF9, 0xFB, 0xCA, 0x6C, 0x87, 0xBA, 0x73, 0xD4, + 0x85, 0x7B, 0xB2, 0x6F, 0x5C, 0xD8, 0xFB, 0xE9, + 0x41, 0x24, 0x3A, 0x3B, 0x4F, 0x91, 0x77, 0xA2, + 0x35, 0x78, 0xE5, 0x4C, 0xFE, 0x8B, 0x04, 0x03, + 0xD3, 0x84, 0xA9, 0x1C, 0xA7, 0x7C, 0x45, 0x13, + 0x7D, 0xC5, 0x0A, 0x2F, 0x02, 0xF8, 0x56, 0xD5, + 0x5F, 0x35, 0xED, 0x06, 0xBF, 0x67, 0xBA, 0x51, + 0x02, 0x95, 0x36, 0xF2, 0x9A, 0xBA, 0x9D, 0xF6, + 0xD6, 0x77, 0x50, 0xC9, 0xFC, 0x1E, 0x32, 0xB5, + 0x2F, 0xEA, 0x3C, 0x76, 0xB4, 0xE1, 0xCC, 0x42, + 0xEB, 0x71, 0x79, 0xD3, 0x7D, 0xB7, 0xC0, 0x88, + 0x25, 0x81, 0xE8, 0xC0, 0xB8, 0x38, 0x7E, 0x7B, + 0xFD, 0x18, 0xAB, 0x08, 0xB2, 0x71, 0xA5, 0xAD, + 0xA7, 0xBE, 0x48, 0x5F, 0x86, 0xE2, 0x41, 0x3D, + 0x7C, 0x37, 0x7A, 0xAB, 0xDB, 0xE0, 0x3B, 0x3D, + 0xB6, 0xE8, 0x23, 0x7C, 0xF1, 0x8F, 0xBA, 0xB7, + 0xE9, 0x78, 0x0B, 0xCA, 0x67, 0xA8, 0x10, 0x36, + 0xEB, 0x72, 0xED, 0xDD, 0xF0, 0x5C, 0x74, 0x8E, + 0xE5, 0x2A, 0xAE, 0x6E, 0xC4, 0xF1, 0xFC, 0xD8, + 0xEE, 0x56, 0x07, 0x88, 0x02, 0xDC, 0x9D, 0xB7, + 0xF9, 0x13, 0xE1, 0xE1, 0x9D, 0x89, 0x26, 0x0B, + 0x23, 0x74, 0x4A, 0x43, 0xAA, 0xA0, 0xA8, 0x97, + 0x85, 0x15, 0x58, 0xAB, 0x2B, 0xB5, 0xDA, 0x1A, + 0xBA, 0x29, 0x62, 0xCF, 0xDD, 0xA3, 0xBA, 0x9D, + 0x7D, 0x83, 0xA5, 0x18, 0xD4, 0x03, 0x0F, 0x61, + 0x9F, 0xB1, 0x7E, 0xEC, 0xD2, 0x6E, 0xAF, 0xCF, + 0x1E, 0xC1, 0x88, 0x97, 0x99, 0xD6, 0xBF, 0x47, + 0xB9, 0x0A, 0x69, 0x11, 0x3A, 0x55, 0x8B, 0x1D, + 0x2D, 0xFF, 0x78, 0xC8, 0xDE, 0x82, 0x29, 0xD6, + 0x08, 0x3C, 0xC4, 0xCB, 0x2F, 0x01, 0xD0, 0xE8, + 0xB1, 0x75, 0x5E, 0x23, 0xE0, 0x37, 0x7C, 0x1C, + 0xB6, 0xD9, 0x47, 0xDE, 0x23, 0x87, 0xD3, 0x68, + 0x47, 0x46, 0x78, 0xF3, 0xBF, 0x54, 0xA3, 0xB9, + 0x54, 0xD5, 0xC5, 0x0A, 0x7C, 0x92, 0x2A, 0xC2, + 0x14, 0x76, 0xA6, 0x5C, 0x6D, 0x0B, 0x94, 0x56, + 0x00, 0x6B, 0x5C, 0x27, 0xDE, 0x77, 0x9B, 0xF1, + 0xB1, 0x8C, 0xA7, 0x49, 0x77, 0xFC, 0x4E, 0x29, + 0x23, 0x8F, 0x2F, 0xF7, 0x83, 0x8D, 0x36, 0xD9, + 0xAB, 0x0E, 0x78, 0xF5, 0x90, 0x05, 0xB9, 0x79, + 0x70, 0x88, 0x59, 0x6F, 0xE2, 0xC5, 0xD7, 0x80, + 0x95, 0x04, 0x29, 0xE0, 0xFA, 0x37, 0xE8, 0x8B, + 0xC5, 0x21, 0x51, 0x1A, 0x62, 0xCE, 0x93, 0xAF, + 0x1A, 0xFE, 0xC3, 0x6F, 0x86, 0x94, 0x5E, 0x13, + 0xA6, 0x9A, 0x26, 0xF0, 0xB5, 0x7C, 0x41, 0x9A, + 0x80, 0xB8, 0x84, 0x5A, 0x55, 0xA9, 0xB0, 0x6A, + 0xFA, 0xEB, 0x46, 0x32, 0x0B, 0xE2, 0x9C, 0x65, + 0x86, 0x11, 0x39, 0x7E, 0xAF, 0x93, 0x19, 0x09, + 0x70, 0x40, 0x80, 0x14, 0xBA, 0x1D, 0xB3, 0x62, + 0x5B, 0xF3, 0x9A, 0x21, 0x98, 0x7E, 0x63, 0xB6, + 0x1A, 0xBD, 0x65, 0x98, 0x35, 0x2A, 0xA9, 0x76, + 0x29, 0x59, 0x84, 0x25, 0x81, 0xB8, 0xDE, 0x25, + 0x32, 0x10, 0x50, 0xB7, 0xD3, 0xB3, 0x69, 0xC8, + 0xE1, 0x33, 0xCB, 0x9E, 0x9C, 0x7A, 0x7C, 0xD2, + 0x6C, 0x92, 0x97, 0xA9, 0xFA, 0xAF, 0x30, 0xBA, + 0x9A, 0xB3, 0x3D, 0x9A, 0xE5, 0x0A, 0x9B, 0x8D, + 0x89, 0xE2, 0x2B, 0xB8, 0xBC, 0xF0, 0x23, 0xFF, + 0x7B, 0x0D, 0x00, 0x36, 0xEE, 0x79, 0xCB, 0xA5, + 0x70, 0x4C, 0x66, 0x02, 0x79, 0x2E, 0x5B, 0x83, + 0xCE, 0x55, 0x8B, 0x89, 0xD6, 0xE3, 0x71, 0x63, + 0xBC, 0xB1, 0x5F, 0x67, 0xB4, 0x7E, 0x05, 0x0D, + 0xAC, 0x6D, 0x4E, 0x2C, 0xA5, 0xF4, 0x47, 0x89, + 0xAC, 0x5E, 0xBE, 0x2F, 0xFC, 0x9B, 0x2F, 0x0B, + 0xBE, 0x63, 0x54, 0x97, 0xBB, 0x23, 0x27, 0xCD, + 0xB9, 0xB2, 0x28, 0x0D, 0xA4, 0x78, 0x2C, 0xAB, + 0xD1, 0xC9, 0x94, 0x40, 0x54, 0xF2, 0x35, 0x61, + 0x49, 0x01, 0x87, 0x55, 0xA5, 0xB5, 0x1E, 0x84, + 0x92, 0x9E, 0xC1, 0xA4, 0x0B, 0x66, 0x2B, 0xF8, + 0xAF, 0xC3, 0x1E, 0xAF, 0x66, 0x3F, 0x6F, 0x5F, + 0x70, 0xEC, 0x25, 0x29, 0xE4, 0x65, 0xB2, 0x04, + 0x47, 0xF6, 0x3C, 0xB5, 0x5F, 0x66, 0x9F, 0xA4, + 0x1B, 0xFC, 0xA2, 0xD5, 0x3E, 0x84, 0xBA, 0x88, + 0x0D, 0xF1, 0x6A, 0xF2, 0xF6, 0x1D, 0xF1, 0xA3, + 0x45, 0xB2, 0x51, 0xD8, 0xA2, 0x8F, 0x55, 0xA6, + 0x89, 0xC4, 0x15, 0xD5, 0x73, 0xA8, 0xB1, 0x31, + 0x66, 0x9E, 0xC1, 0x43, 0xE1, 0x5D, 0x4E, 0x04, + 0x84, 0x8F, 0xF2, 0xBC, 0xE1, 0x4E, 0x4D, 0x60, + 0x81, 0xCA, 0x53, 0x34, 0x95, 0x17, 0x3B, 0xAE, + 0x8F, 0x95, 0xA7, 0xC6, 0x47, 0xC6, 0xAC, 0x32, + 0x12, 0x39, 0xCA, 0xEF, 0xE0, 0x07, 0xBF, 0x17, + 0x4F, 0xDC, 0x1B, 0x4E, 0x3C, 0x84, 0xF1, 0x9F, + 0x43, 0x70, 0x19, 0xE6, 0xF3, 0x8B, 0x8B, 0x5D, + 0xDB, 0xD2, 0x9D, 0xD4, 0xB2, 0x30, 0x45, 0x55, + 0xA2, 0x67, 0xA2, 0x76, 0x4A, 0x74, 0xAD, 0x88, + 0x71, 0xE6, 0x3E, 0x13, 0x06, 0x30, 0x17, 0xE1, + 0xEF, 0xAC, 0x71, 0xFB, 0x43, 0xCD, 0xF6, 0xFA, + 0x0E, 0x4C, 0x4E, 0x16, 0xF6, 0x6A, 0x09, 0x86, + 0x6B, 0xEA, 0x47, 0x6C, 0x70, 0xE7, 0xAD, 0xA2, + 0xE0, 0xFD, 0x7F, 0xF0, 0x5C, 0x21, 0x53, 0x0F, + 0x28, 0xA1, 0x43, 0xE1, 0x06, 0xCA, 0x0B, 0x31, + 0x88, 0x22, 0xA6, 0xE6, 0x34, 0x5B, 0xE6, 0xCF, + 0x25, 0x81, 0x63, 0xFF, 0x78, 0x66, 0x85, 0x19, + 0xE2, 0x0A, 0x7E, 0x81, 0x8A, 0x17, 0x1A, 0x18, + 0x8A, 0x5F, 0x5D, 0x9E, 0x82, 0x13, 0x10, 0xB9, + 0xD3, 0xE6, 0x93, 0x1C, 0xE4, 0x2C, 0xCB, 0x49, + 0x1E, 0xB6, 0x36, 0x13, 0xBF, 0x28, 0xEE, 0xCC, + 0x49, 0xF5, 0x79, 0xFC, 0x20, 0x65, 0xBD, 0xE8, + 0xF0, 0x1B, 0x4E, 0xC0, 0x0D, 0x3E, 0x89, 0x91, + 0xCC, 0x64, 0x10, 0xC0, 0x2A, 0x2B, 0xA3, 0xFA, + 0x60, 0x3D, 0xC3, 0x52, 0x2F, 0x93, 0xDE, 0xB7, + 0x6E, 0x8A, 0xDF, 0x6C, 0x08, 0xCC, 0x8B, 0x3B, + 0xC8, 0x50, 0xEF, 0x58, 0x64, 0x9A, 0x3D, 0x16, + 0x70, 0x94, 0x11, 0xD8, 0x94, 0x2B, 0x70, 0x91, + 0x10, 0x70, 0x88, 0xF0, 0x40, 0x75, 0x9A, 0x2B, + 0x39, 0xA1, 0x27, 0x3F, 0x2E, 0x91, 0xEA, 0xA1, + 0xCC, 0x12, 0xC1, 0x7F, 0x73, 0x8C, 0x5C, 0x6B, + 0xFC, 0xC5, 0x6A, 0x1C, 0x05, 0xF1, 0x3D, 0x30, + 0x82, 0x4A, 0x65, 0x35, 0xCE, 0x80, 0x10, 0xBB, + 0x41, 0x94, 0xFB, 0x84, 0x80, 0x7B, 0x91, 0xC4, + 0x4D, 0xA3, 0x5F, 0xB9, 0xFB, 0xF9, 0xC9, 0x1D, + 0x4F, 0x99, 0x1C, 0x1F, 0x47, 0x44, 0x89, 0x0E, + 0xED, 0x6D, 0xB5, 0x85, 0x41, 0x94, 0xEF, 0xF9, + 0x2E, 0xA0, 0xC8, 0xCA, 0xFB, 0x44, 0x02, 0xC6, + 0xBF, 0x96, 0x87, 0x80, 0x1D, 0xEF, 0x2A, 0x81, + 0xAB, 0xB2, 0x56, 0xDF, 0x54, 0x8B, 0xAB, 0xAF, + 0xFE, 0x18, 0x8C, 0xAA, 0xD4, 0x00, 0x17, 0xBE, + 0xCF, 0x06, 0xE5, 0xA6, 0xBF, 0x5A, 0x52, 0x3B, + 0x4E, 0xF5, 0x65, 0x60, 0x95, 0xDE, 0x8A, 0x25, + 0x88, 0xA5, 0x24, 0x96, 0x29, 0x13, 0x0D, 0x19, + 0x45, 0x95, 0x91, 0x08, 0xD2, 0x9C, 0x4C, 0x34, + 0x42, 0xF0, 0xA5, 0x72, 0xEB, 0xFB, 0x5E, 0xAA, + 0x68, 0x80, 0x82, 0xAC, 0x34, 0xAD, 0x89, 0xF6, + 0xAF, 0x54, 0x82, 0xCF, 0x98, 0x8C, 0x75, 0x63, + 0x8D, 0xBD, 0x1C, 0x2A, 0xD7, 0x00, 0xA7, 0x8E, + 0xB9, 0x33, 0xB6, 0x3B, 0x95, 0x9A, 0x59, 0x1D, + 0x3F, 0x23, 0x6B, 0x18, 0xF8, 0x4F, 0x1A, 0x8D, + 0xC0, 0x26, 0x9F, 0x87, 0x61, 0xB6, 0xC6, 0x60, + 0x38, 0x22, 0x73, 0x1C, 0x99, 0x23, 0xEF, 0xD9, + 0xFD, 0xCB, 0x54, 0x74, 0xBB, 0x77, 0x14, 0xA3, + 0xA9, 0xE6, 0x7C, 0x7E, 0x03, 0x3A, 0x13, 0x6E, + 0x1D, 0x6F, 0x64, 0xB3, 0xFA, 0xFB, 0x52, 0xDE, + 0xDF, 0x08, 0xFB, 0x6F, 0xC5, 0xFA, 0x51, 0x6A, + 0x69, 0x29, 0x9B, 0x96, 0xE8, 0x16, 0xC8, 0xD1, + 0xE4, 0x19, 0xBD, 0x14, 0x74, 0x27, 0xE7, 0x10, + 0xF0, 0xC3, 0xE2, 0xA7, 0x60, 0x48, 0xBF, 0xDD, + 0xC4, 0x0D, 0xD0, 0xF2, 0xEF, 0xA6, 0xC9, 0xA2, + 0x73, 0xD1, 0xCF, 0x41, 0xE1, 0x3B, 0xE5, 0x49, + 0x91, 0x5D, 0x09, 0xFD, 0x1D, 0x95, 0x29, 0xDB, + 0x52, 0x48, 0xEB, 0xF5, 0x1D, 0xF8, 0x06, 0x67, + 0x75, 0xF2, 0x57, 0xA4, 0x20, 0x60, 0xEA, 0xB0, + 0x85, 0x93, 0x7C, 0xDD, 0x52, 0x01, 0xD4, 0x57, + 0xA8, 0x31, 0x2D, 0xF9, 0x0A, 0xD2, 0x2A, 0xD1, + 0x34, 0x18, 0x35, 0x16, 0xB6, 0x8B, 0x0F, 0x0B, + 0xCF, 0x50, 0x80, 0xFE, 0x76, 0xCC, 0x4F, 0x30, + 0x98, 0x19, 0x16, 0x3D, 0x01, 0xEA, 0x8D, 0x8A, + 0x3D, 0xDC, 0xFB, 0x1F, 0x77, 0x8D, 0x72, 0x76, + 0x02, 0x3C, 0x5D, 0xEE, 0x55, 0x13, 0x5B, 0x6E, + 0x5A, 0x2D, 0xD5, 0x77, 0xD7, 0x01, 0x84, 0x7D, + 0x21, 0x8C, 0xDD, 0x94, 0x7D, 0x31, 0x3D, 0xF0, + 0xE7, 0x28, 0xF5, 0x72, 0x36, 0x60, 0xE0, 0x59, + 0x5F, 0xFE, 0x38, 0xF8, 0x2F, 0xDB, 0x9E, 0x55, + 0x5A, 0xD6, 0xBA, 0x6C, 0x87, 0xF3, 0xC0, 0x76, + 0x5F, 0xA3, 0x0A, 0xC3, 0xA3, 0x8D, 0x0E, 0x52, + 0xA8, 0xDA, 0x26, 0x3A, 0xF9, 0x3E, 0x36, 0xB1, + 0x06, 0xF8, 0x20, 0x2D, 0x1C, 0x0B, 0x93, 0xBB, + 0xD3, 0x64, 0x77, 0xCE, 0x11, 0xFC, 0xA2, 0x0E, + 0x1B, 0x5B, 0x9E, 0x13, 0x9F, 0x20, 0x8B, 0xAA, + 0xCD, 0x72, 0xD7, 0xA6, 0xF3, 0x1E, 0x4F, 0x72, + 0xC6, 0x49, 0x0F, 0x7B, 0xF0, 0x4C, 0x61, 0x1F, + 0x43, 0x0D, 0x4F, 0x0D, 0x33, 0x13, 0xED, 0x63, + 0xE5, 0xDB, 0x71, 0xAB, 0xA4, 0x83, 0xEF, 0xDC, + 0x86, 0x9D, 0x4B, 0xBD, 0x1B, 0x8A, 0xFE, 0x39, + 0xA8, 0x8B, 0xBA, 0x4C, 0x85, 0x28, 0xFC, 0xB3, + 0x62, 0x85, 0xD2, 0xF0, 0x38, 0xD0, 0x4B, 0xA4, + 0xD1, 0x3B, 0xD4, 0xD0, 0x2C, 0x78, 0x6C, 0x6A, + 0xC2, 0x64, 0x2C, 0x31, 0x4A, 0xD8, 0x69, 0x24, + 0xED, 0x77, 0x7D, 0x68, 0x9A, 0xA1, 0x78, 0x81, + 0xD9, 0x7E, 0x6C, 0xFE, 0x0A, 0x0D, 0x76, 0xF7, + 0x4B, 0x58, 0xE7, 0xC9, 0xB5, 0x11, 0x07, 0x87, + 0x88, 0x6A, 0x9F, 0x3D, 0xE0, 0xEE, 0xCC, 0x60, + 0x6B, 0x6B, 0xE6, 0xB5, 0x54, 0x8B, 0x32, 0x1F, + 0x04, 0x1D, 0x0E, 0x9E, 0xFA, 0x6D, 0xB0, 0xE0, + 0x6D, 0xF9, 0x79, 0xB4, 0xAB, 0x5E, 0xDF, 0x23, + 0x7F, 0x95, 0xAD, 0x80, 0x17, 0x23, 0x90, 0x1F, + 0xF0, 0xC3, 0xD9, 0x2D, 0xAC, 0x3F, 0x63, 0xF5, + 0x77, 0xC5, 0x05, 0xAC, 0x06, 0xB6, 0xA1, 0xB4, + 0xA2, 0x40, 0xB3, 0x99, 0x34, 0x7D, 0x31, 0xD4, + 0xB1, 0xD4, 0xC1, 0xBB, 0x71, 0x1E, 0xDA, 0x3F, + 0xA9, 0x12, 0x68, 0xFA, 0x5B, 0x20, 0x24, 0x6D, + 0x4D, 0x72, 0x43, 0x18, 0xBF, 0x66, 0x71, 0x69, + 0x26, 0x7D, 0x77, 0x78, 0xF8, 0xE5, 0x20, 0xAE, + 0x56, 0x6C, 0x0F, 0x72, 0x94, 0x42, 0x85, 0x4F, + 0xE4, 0xFB, 0x32, 0x26, 0x1B, 0x1C, 0x6E, 0x0B, + 0xF0, 0xB8, 0x58, 0x00, 0xD2, 0x36, 0x64, 0xAD, + 0xA9, 0x00, 0xCE, 0x35, 0x3C, 0x88, 0x79, 0x94, + 0x0C, 0x0C, 0x9B, 0xF2, 0xDA, 0xBD, 0xCA, 0x93, + 0x37, 0x26, 0xD3, 0x08, 0x54, 0xD2, 0x0D, 0xBC, + 0x5D, 0x43, 0x5F, 0xCF, 0x28, 0xB5, 0xAA, 0x15, + 0x28, 0x46, 0x45, 0x6B, 0xE8, 0xDF, 0xE8, 0xCE, + 0x8F, 0xC0, 0x1A, 0x53, 0x63, 0x3B, 0x53, 0x75, + 0xDD, 0x43, 0x1F, 0x07, 0x0A, 0xD5, 0xA1, 0x2A, + 0x6E, 0x28, 0xE1, 0xD7, 0xD0, 0x09, 0xCF, 0x62, + 0xC1, 0x5F, 0x21, 0xDB, 0xC5, 0x40, 0x99, 0x48, + 0x87, 0x6E, 0x11, 0xF5, 0x5A, 0x4E, 0xBC, 0xF9, + 0xA8, 0x02, 0x7C, 0x47, 0x39, 0xA5, 0xD8, 0x52, + 0xB1, 0x80, 0xDC, 0xFE, 0x08, 0x4B, 0x5D, 0x09, + 0xDE, 0x06, 0xF3, 0x2A, 0xAD, 0x14, 0x76, 0x40, + 0x2F, 0x82, 0x28, 0x6A, 0xB6, 0x43, 0xEF, 0x71, + 0x63, 0xC2, 0x56, 0xEB, 0x3B, 0x4B, 0x52, 0x2F, + 0x93, 0xD3, 0x18, 0x3E, 0x18, 0xA8, 0xF7, 0x58, + 0xFC, 0x8B, 0x3D, 0x4D, 0x4B, 0x72, 0xBD, 0xF7, + 0x04, 0xC9, 0xB8, 0xD7, 0x6C, 0x8C, 0x67, 0xBB, + 0x4C, 0x9B, 0x57, 0xF7, 0x22, 0x4E, 0x41, 0xB6, + 0xFD, 0xD9, 0xF8, 0x41, 0x62, 0x0F, 0xFF, 0xAA, + 0xC6, 0x87, 0x95, 0xFF, 0xFD, 0x58, 0xD9, 0xB2, + 0xBA, 0x47, 0x61, 0x24, 0xEA, 0x92, 0x6E, 0x74, + 0xB3, 0xDA, 0xE5, 0x83, 0x99, 0x24, 0xB1, 0x71, + 0x2A, 0x33, 0xB2, 0xD5, 0x8F, 0xF0, 0x32, 0xCE, + 0x37, 0xCF, 0xC7, 0x1C, 0xE8, 0xDE, 0x46, 0x78, + 0x96, 0x97, 0xF6, 0x73, 0x90, 0xE5, 0x71, 0x05, + 0xEA, 0x0D, 0xC2, 0x1D, 0x9E, 0x43, 0x34, 0xBC, + 0x8F, 0x45, 0xE5, 0x08, 0xCA, 0x20, 0x0C, 0x84 + }, + .len = 2048 + }, + .auth_tag = { + .data = { + 0xD0, 0x62, 0x1F, 0x20, 0x1C, 0xE8, 0xDD, 0x36, + 0x00, 0x74, 0xF6, 0xD7, 0xFD, 0x2C, 0xA0, 0xAF + }, + .len = 16 + } +}; + +/** AES-GCM-192 Test Vectors */ +static const struct aead_test_data gcm_test_case_192_1 = { + .algo = RTE_CRYPTO_AEAD_AES_GCM, + .key = { + .data = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + }, + .len = 24 + }, + .iv = { + .data = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00 + }, + .len = 12 + }, + .aad = { + .data = gcm_aad_zero_text, + .len = 0 + }, + .plaintext = { + .data = { 0x00 }, + .len = 0 + }, + .ciphertext = { + .data = { 0x00 }, + .len = 0 + }, + .auth_tag = { + .data = { + 0xCD, 0x33, 0xB2, 0x8A, 0xC7, 0x73, 0xF7, 0x4B, + 0xA0, 0x0E, 0xD1, 0xF3, 0x12, 0x57, 0x24, 0x35 + }, + .len = 16 + } +}; + +static const struct aead_test_data gcm_test_case_192_2 = { + .algo = RTE_CRYPTO_AEAD_AES_GCM, + .key = { + .data = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + }, + .len = 24 + }, + .iv = { + .data = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00 + }, + .len = 12 + }, + .aad = { + .data = gcm_aad_zero_text, + .len = 0 + }, + .plaintext = { + .data = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + }, + .len = 16 + }, + .ciphertext = { + .data = { + 0x98, 0xE7, 0x24, 0x7C, 0x07, 0xF0, 0xFE, 0x41, + 0x1C, 0x26, 0x7E, 0x43, 0x84, 0xB0, 0xF6, 0x00 + }, + .len = 16 + }, + .auth_tag = { + .data = { + 0x2F, 0xF5, 0x8D, 0x80, 0x03, 0x39, 0x27, 0xAB, + 0x8E, 0xF4, 0xD4, 0x58, 0x75, 0x14, 0xF0, 0xFB + + }, + .len = 16 + } +}; + +static const struct aead_test_data gcm_test_case_192_3 = { + .algo = RTE_CRYPTO_AEAD_AES_GCM, + .key = { + .data = { + 0xFE, 0xFF, 0xE9, 0x92, 0x86, 0x65, 0x73, 0x1C, + 0x6D, 0x6A, 0x8F, 0x94, 0x67, 0x30, 0x83, 0x08, + 0xFE, 0xFF, 0xE9, 0x92, 0x86, 0x65, 0x73, 0x1C + }, + .len = 24 + }, + .iv = { + .data = { + 0xCA, 0xFE, 0xBA, 0xBE, 0xFA, 0xCE, 0xDB, 0xAD, + 0xDE, 0xCA, 0xF8, 0x88 + }, + .len = 12 + }, + .aad = { + .data = gcm_aad_zero_text, + .len = 0 + }, + .plaintext = { + .data = { + 0xD9, 0x31, 0x32, 0x25, 0xF8, 0x84, 0x06, 0xE5, + 0xA5, 0x59, 0x09, 0xC5, 0xAF, 0xF5, 0x26, 0x9A, + 0x86, 0xA7, 0xA9, 0x53, 0x15, 0x34, 0xF7, 0xDA, + 0x2E, 0x4C, 0x30, 0x3D, 0x8A, 0x31, 0x8A, 0x72, + 0x1C, 0x3C, 0x0C, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2F, 0xCF, 0x0E, 0x24, 0x49, 0xA6, 0xB5, 0x25, + 0xB1, 0x6A, 0xED, 0xF5, 0xAA, 0x0D, 0xE6, 0x57, + 0xBA, 0x63, 0x7B, 0x39, 0x1A, 0xAF, 0xD2, 0x55 + }, + .len = 64 + }, + .ciphertext = { + .data = { + 0x39, 0x80, 0xCA, 0x0B, 0x3C, 0x00, 0xE8, 0x41, + 0xEB, 0x06, 0xFA, 0xC4, 0x87, 0x2A, 0x27, 0x57, + 0x85, 0x9E, 0x1C, 0xEA, 0xA6, 0xEF, 0xD9, 0x84, + 0x62, 0x85, 0x93, 0xB4, 0x0C, 0xA1, 0xE1, 0x9C, + 0x7D, 0x77, 0x3D, 0x00, 0xC1, 0x44, 0xC5, 0x25, + 0xAC, 0x61, 0x9D, 0x18, 0xC8, 0x4A, 0x3F, 0x47, + 0x18, 0xE2, 0x44, 0x8B, 0x2F, 0xE3, 0x24, 0xD9, + 0xCC, 0xDA, 0x27, 0x10, 0xAC, 0xAD, 0xE2, 0x56 + }, + .len = 64 + }, + .auth_tag = { + .data = { + 0x99, 0x24, 0xA7, 0xC8, 0x58, 0x73, 0x36, 0xBF, + 0xB1, 0x18, 0x02, 0x4D, 0xB8, 0x67, 0x4A, 0x14 + }, + .len = 16 + } +}; + +static const struct aead_test_data gcm_test_case_192_4 = { + .algo = RTE_CRYPTO_AEAD_AES_GCM, + .key = { + .data = { + 0xFE, 0xFF, 0xE9, 0x92, 0x86, 0x65, 0x73, 0x1C, + 0x6D, 0x6A, 0x8F, 0x94, 0x67, 0x30, 0x83, 0x08, + 0xFE, 0xFF, 0xE9, 0x92, 0x86, 0x65, 0x73, 0x1C + }, + .len = 24 + }, + .iv = { + .data = { + 0xCA, 0xFE, 0xBA, 0xBE, 0xFA, 0xCE, 0xDB, 0xAD, + 0xDE, 0xCA, 0xF8, 0x88 + }, + .len = 12 + }, + .aad = { + .data = gcm_aad_zero_text, + .len = 8 + }, + .plaintext = { + .data = { + 0xD9, 0x31, 0x32, 0x25, 0xF8, 0x84, 0x06, 0xE5, + 0xA5, 0x59, 0x09, 0xC5, 0xAF, 0xF5, 0x26, 0x9A, + 0x86, 0xA7, 0xA9, 0x53, 0x15, 0x34, 0xF7, 0xDA, + 0x2E, 0x4C, 0x30, 0x3D, 0x8A, 0x31, 0x8A, 0x72, + 0x1C, 0x3C, 0x0C, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2F, 0xCF, 0x0E, 0x24, 0x49, 0xA6, 0xB5, 0x25, + 0xB1, 0x6A, 0xED, 0xF5, 0xAA, 0x0D, 0xE6, 0x57, + 0xBA, 0x63, 0x7B, 0x39 + }, + .len = 60 + }, + .ciphertext = { + .data = { + 0x39, 0x80, 0xCA, 0x0B, 0x3C, 0x00, 0xE8, 0x41, + 0xEB, 0x06, 0xFA, 0xC4, 0x87, 0x2A, 0x27, 0x57, + 0x85, 0x9E, 0x1C, 0xEA, 0xA6, 0xEF, 0xD9, 0x84, + 0x62, 0x85, 0x93, 0xB4, 0x0C, 0xA1, 0xE1, 0x9C, + 0x7D, 0x77, 0x3D, 0x00, 0xC1, 0x44, 0xC5, 0x25, + 0xAC, 0x61, 0x9D, 0x18, 0xC8, 0x4A, 0x3F, 0x47, + 0x18, 0xE2, 0x44, 0x8B, 0x2F, 0xE3, 0x24, 0xD9, + 0xCC, 0xDA, 0x27, 0x10 + }, + .len = 60 + }, + .auth_tag = { + .data = { + 0x57, 0x5F, 0x03, 0xA0, 0x8D, 0x8F, 0x40, 0x26, + 0xE5, 0x64, 0x1F, 0x5B, 0x5C, 0xC2, 0xFD, 0x4B + }, + .len = 16 + } +}; + +static const struct aead_test_data gcm_test_case_192_5 = { + .algo = RTE_CRYPTO_AEAD_AES_GCM, + .key = { + .data = { + 0xFE, 0xFF, 0xE9, 0x92, 0x86, 0x65, 0x73, 0x1C, + 0x6D, 0x6A, 0x8F, 0x94, 0x67, 0x30, 0x83, 0x08, + 0xFE, 0xFF, 0xE9, 0x92, 0x86, 0x65, 0x73, 0x1C + }, + .len = 24 + }, + .iv = { + .data = { + 0xCA, 0xFE, 0xBA, 0xBE, 0xFA, 0xCE, 0xDB, 0xAD, + 0xDE, 0xCA, 0xF8, 0x88 + }, + .len = 12 + }, + .aad = { + .data = gcm_aad_text, + .len = 8 + }, + .plaintext = { + .data = { + 0xD9, 0x31, 0x32, 0x25, 0xF8, 0x84, 0x06, 0xE5, + 0xA5, 0x59, 0x09, 0xC5, 0xAF, 0xF5, 0x26, 0x9A, + 0x86, 0xA7, 0xA9, 0x53, 0x15, 0x34, 0xF7, 0xDA, + 0x2E, 0x4C, 0x30, 0x3D, 0x8A, 0x31, 0x8A, 0x72, + 0x1C, 0x3C, 0x0C, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2F, 0xCF, 0x0E, 0x24, 0x49, 0xA6, 0xB5, 0x25, + 0xB1, 0x6A, 0xED, 0xF5, 0xAA, 0x0D, 0xE6, 0x57, + 0xBA, 0x63, 0x7B, 0x39 + }, + .len = 60 + }, + .ciphertext = { + .data = { + 0x39, 0x80, 0xCA, 0x0B, 0x3C, 0x00, 0xE8, 0x41, + 0xEB, 0x06, 0xFA, 0xC4, 0x87, 0x2A, 0x27, 0x57, + 0x85, 0x9E, 0x1C, 0xEA, 0xA6, 0xEF, 0xD9, 0x84, + 0x62, 0x85, 0x93, 0xB4, 0x0C, 0xA1, 0xE1, 0x9C, + 0x7D, 0x77, 0x3D, 0x00, 0xC1, 0x44, 0xC5, 0x25, + 0xAC, 0x61, 0x9D, 0x18, 0xC8, 0x4A, 0x3F, 0x47, + 0x18, 0xE2, 0x44, 0x8B, 0x2F, 0xE3, 0x24, 0xD9, + 0xCC, 0xDA, 0x27, 0x10 + }, + .len = 60 + }, + .auth_tag = { + .data = { + 0xB6, 0x35, 0x56, 0xE7, 0xBA, 0x46, 0xA3, 0x38, + 0xED, 0xAD, 0x79, 0x9F, 0xB3, 0x5B, 0x34, 0xA8 + }, + .len = 16 + } +}; + +static const struct aead_test_data gcm_test_case_192_6 = { + .algo = RTE_CRYPTO_AEAD_AES_GCM, + .key = { + .data = { + 0xFE, 0xFF, 0xE9, 0x92, 0x86, 0x65, 0x73, 0x1C, + 0x6D, 0x6A, 0x8F, 0x94, 0x67, 0x30, 0x83, 0x08, + 0xFE, 0xFF, 0xE9, 0x92, 0x86, 0x65, 0x73, 0x1C + }, + .len = 24 + }, + .iv = { + .data = { + 0xCA, 0xFE, 0xBA, 0xBE, 0xFA, 0xCE, 0xDB, 0xAD, + 0xDE, 0xCA, 0xF8, 0x88 + }, + .len = 12 + }, + .aad = { + .data = gcm_aad_zero_text, + .len = 12 + }, + .plaintext = { + .data = { + 0xD9, 0x31, 0x32, 0x25, 0xF8, 0x84, 0x06, 0xE5, + 0xA5, 0x59, 0x09, 0xC5, 0xAF, 0xF5, 0x26, 0x9A, + 0x86, 0xA7, 0xA9, 0x53, 0x15, 0x34, 0xF7, 0xDA, + 0x2E, 0x4C, 0x30, 0x3D, 0x8A, 0x31, 0x8A, 0x72, + 0x1C, 0x3C, 0x0C, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2F, 0xCF, 0x0E, 0x24, 0x49, 0xA6, 0xB5, 0x25, + 0xB1, 0x6A, 0xED, 0xF5, 0xAA, 0x0D, 0xE6, 0x57, + 0xBA, 0x63, 0x7B, 0x39 + }, + .len = 60 + }, + .ciphertext = { + .data = { + 0x39, 0x80, 0xCA, 0x0B, 0x3C, 0x00, 0xE8, 0x41, + 0xEB, 0x06, 0xFA, 0xC4, 0x87, 0x2A, 0x27, 0x57, + 0x85, 0x9E, 0x1C, 0xEA, 0xA6, 0xEF, 0xD9, 0x84, + 0x62, 0x85, 0x93, 0xB4, 0x0C, 0xA1, 0xE1, 0x9C, + 0x7D, 0x77, 0x3D, 0x00, 0xC1, 0x44, 0xC5, 0x25, + 0xAC, 0x61, 0x9D, 0x18, 0xC8, 0x4A, 0x3F, 0x47, + 0x18, 0xE2, 0x44, 0x8B, 0x2F, 0xE3, 0x24, 0xD9, + 0xCC, 0xDA, 0x27, 0x10 + }, + .len = 60 + }, + .auth_tag = { + .data = { + 0xCA, 0x8A, 0x8A, 0x91, 0x5A, 0xF9, 0x76, 0xE3, + 0xFF, 0x2C, 0xE4, 0x7D, 0xE5, 0x62, 0x75, 0x18 + }, + .len = 16 + } +}; + +static const struct aead_test_data gcm_test_case_192_7 = { + .algo = RTE_CRYPTO_AEAD_AES_GCM, + .key = { + .data = { + 0xFE, 0xFF, 0xE9, 0x92, 0x86, 0x65, 0x73, 0x1C, + 0x6D, 0x6A, 0x8F, 0x94, 0x67, 0x30, 0x83, 0x08, + 0xFE, 0xFF, 0xE9, 0x92, 0x86, 0x65, 0x73, 0x1C + }, + .len = 24 + }, + .iv = { + .data = { + 0xCA, 0xFE, 0xBA, 0xBE, 0xFA, 0xCE, 0xDB, 0xAD, + 0xDE, 0xCA, 0xF8, 0x88 + }, + .len = 12 + }, + .aad = { + .data = gcm_aad_text, + .len = 12 + }, + .plaintext = { + .data = { + 0xD9, 0x31, 0x32, 0x25, 0xF8, 0x84, 0x06, 0xE5, + 0xA5, 0x59, 0x09, 0xC5, 0xAF, 0xF5, 0x26, 0x9A, + 0x86, 0xA7, 0xA9, 0x53, 0x15, 0x34, 0xF7, 0xDA, + 0x2E, 0x4C, 0x30, 0x3D, 0x8A, 0x31, 0x8A, 0x72, + 0x1C, 0x3C, 0x0C, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2F, 0xCF, 0x0E, 0x24, 0x49, 0xA6, 0xB5, 0x25, + 0xB1, 0x6A, 0xED, 0xF5, 0xAA, 0x0D, 0xE6, 0x57, + 0xBA, 0x63, 0x7B, 0x39 + }, + .len = 60 + }, + .ciphertext = { + .data = { + 0x39, 0x80, 0xCA, 0x0B, 0x3C, 0x00, 0xE8, 0x41, + 0xEB, 0x06, 0xFA, 0xC4, 0x87, 0x2A, 0x27, 0x57, + 0x85, 0x9E, 0x1C, 0xEA, 0xA6, 0xEF, 0xD9, 0x84, + 0x62, 0x85, 0x93, 0xB4, 0x0C, 0xA1, 0xE1, 0x9C, + 0x7D, 0x77, 0x3D, 0x00, 0xC1, 0x44, 0xC5, 0x25, + 0xAC, 0x61, 0x9D, 0x18, 0xC8, 0x4A, 0x3F, 0x47, + 0x18, 0xE2, 0x44, 0x8B, 0x2F, 0xE3, 0x24, 0xD9, + 0xCC, 0xDA, 0x27, 0x10 + }, + .len = 60 + }, + .auth_tag = { + .data = { + 0xC2, 0xD8, 0x4C, 0x6B, 0xA8, 0x3B, 0xA5, 0x6B, + 0x18, 0x9F, 0xE6, 0xEF, 0x66, 0x24, 0xDD, 0xDA + }, + .len = 16 + } +}; + +/** AES-GCM-256 Test Vectors */ +static const struct aead_test_data gcm_test_case_256_1 = { + .algo = RTE_CRYPTO_AEAD_AES_GCM, + .key = { + .data = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .len = 32 + }, + .iv = { + .data = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00 }, + .len = 12 + }, + .aad = { + .data = gcm_aad_zero_text, + .len = 0 + }, + .plaintext = { + .data = { 0x00 }, + .len = 0 + }, + .ciphertext = { + .data = { 0x00 }, + .len = 0 + }, + .auth_tag = { + .data = { + 0x53, 0x0F, 0x8A, 0xFB, 0xC7, 0x45, 0x36, 0xB9, + 0xA9, 0x63, 0xB4, 0xF1, 0xC4, 0xCB, 0x73, 0x8B }, + .len = 16 + } +}; + +static const struct aead_test_data gcm_test_case_256_2 = { + .algo = RTE_CRYPTO_AEAD_AES_GCM, + .key = { + .data = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .len = 32 + }, + .iv = { + .data = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00 }, + .len = 12 + }, + .aad = { + .data = gcm_aad_zero_text, + .len = 0 + }, + .plaintext = { + .data = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .len = 16 + }, + .ciphertext = { + .data = { + 0xCE, 0xA7, 0x40, 0x3D, 0x4D, 0x60, 0x6B, 0x6E, + 0x07, 0x4E, 0xC5, 0xD3, 0xBA, 0xF3, 0x9D, 0x18 }, + .len = 16 + }, + .auth_tag = { + .data = { + 0xD0, 0xD1, 0xC8, 0xA7, 0x99, 0x99, 0x6B, 0xF0, + 0x26, 0x5B, 0x98, 0xB5, 0xD4, 0x8A, 0xB9, 0x19 }, + .len = 16 + } +}; + +static const struct aead_test_data gcm_test_case_256_3 = { + .algo = RTE_CRYPTO_AEAD_AES_GCM, + .key = { + .data = { + 0xfe, 0xff, 0xe9, 0x92, 0x86, 0x65, 0x73, 0x1c, + 0x6d, 0x6a, 0x8f, 0x94, 0x67, 0x30, 0x83, 0x08, + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a }, + .len = 32 + }, + .iv = { + .data = { + 0xca, 0xfe, 0xba, 0xbe, 0xfa, 0xce, 0xdb, 0xad, + 0xde, 0xca, 0xf8, 0x88 }, + .len = 12 + }, + .aad = { + .data = gcm_aad_zero_text, + .len = 0 + }, + .plaintext = { + .data = { + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x55 }, + .len = 64 + }, + .ciphertext = { + .data = { + 0x05, 0xA2, 0x39, 0xA5, 0xE1, 0x1A, 0x74, 0xEA, + 0x6B, 0x2A, 0x55, 0xF6, 0xD7, 0x88, 0x44, 0x7E, + 0x93, 0x7E, 0x23, 0x64, 0x8D, 0xF8, 0xD4, 0x04, + 0x3B, 0x40, 0xEF, 0x6D, 0x7C, 0x6B, 0xF3, 0xB9, + 0x50, 0x15, 0x97, 0x5D, 0xB8, 0x28, 0xA1, 0xD5, + 0x22, 0xDE, 0x36, 0x26, 0xD0, 0x6A, 0x7A, 0xC0, + 0xB5, 0x14, 0x36, 0xAF, 0x3A, 0xC6, 0x50, 0xAB, + 0xFA, 0x47, 0xC8, 0x2E, 0xF0, 0x68, 0xE1, 0x3E }, + .len = 64 + }, + .auth_tag = { + .data = { + 0x64, 0xAF, 0x1D, 0xFB, 0xE8, 0x0D, 0x37, 0xD8, + 0x92, 0xC3, 0xB9, 0x1D, 0xD3, 0x08, 0xAB, 0xFC }, + .len = 16 + } +}; + +static const struct aead_test_data gcm_test_case_256_4 = { + .algo = RTE_CRYPTO_AEAD_AES_GCM, + .key = { + .data = { + 0xfe, 0xff, 0xe9, 0x92, 0x86, 0x65, 0x73, 0x1c, + 0x6d, 0x6a, 0x8f, 0x94, 0x67, 0x30, 0x83, 0x08, + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a }, + .len = 32 + }, + .iv = { + .data = { + 0xca, 0xfe, 0xba, 0xbe, 0xfa, 0xce, 0xdb, 0xad, + 0xde, 0xca, 0xf8, 0x88 }, + .len = 12 + }, + .aad = { + .data = gcm_aad_zero_text, + .len = 8 + }, + .plaintext = { + .data = { + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39 }, + .len = 60 + }, + .ciphertext = { + .data = { + 0x05, 0xA2, 0x39, 0xA5, 0xE1, 0x1A, 0x74, 0xEA, + 0x6B, 0x2A, 0x55, 0xF6, 0xD7, 0x88, 0x44, 0x7E, + 0x93, 0x7E, 0x23, 0x64, 0x8D, 0xF8, 0xD4, 0x04, + 0x3B, 0x40, 0xEF, 0x6D, 0x7C, 0x6B, 0xF3, 0xB9, + 0x50, 0x15, 0x97, 0x5D, 0xB8, 0x28, 0xA1, 0xD5, + 0x22, 0xDE, 0x36, 0x26, 0xD0, 0x6A, 0x7A, 0xC0, + 0xB5, 0x14, 0x36, 0xAF, 0x3A, 0xC6, 0x50, 0xAB, + 0xFA, 0x47, 0xC8, 0x2E }, + .len = 60 + }, + .auth_tag = { + .data = { + 0x63, 0x16, 0x91, 0xAE, 0x17, 0x05, 0x5E, 0xA6, + 0x6D, 0x0A, 0x51, 0xE2, 0x50, 0x21, 0x85, 0x4A }, + .len = 16 + } + +}; + +static const struct aead_test_data gcm_test_case_256_5 = { + .algo = RTE_CRYPTO_AEAD_AES_GCM, + .key = { + .data = { + 0xfe, 0xff, 0xe9, 0x92, 0x86, 0x65, 0x73, 0x1c, + 0x6d, 0x6a, 0x8f, 0x94, 0x67, 0x30, 0x83, 0x08, + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a }, + .len = 32 + }, + .iv = { + .data = { + 0xca, 0xfe, 0xba, 0xbe, 0xfa, 0xce, 0xdb, 0xad, + 0xde, 0xca, 0xf8, 0x88 }, + .len = 12 + }, + .aad = { + .data = gcm_aad_text, + .len = 8 + }, + .plaintext = { + .data = { + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39 }, + .len = 60 + }, + .ciphertext = { + .data = { + 0x05, 0xA2, 0x39, 0xA5, 0xE1, 0x1A, 0x74, 0xEA, + 0x6B, 0x2A, 0x55, 0xF6, 0xD7, 0x88, 0x44, 0x7E, + 0x93, 0x7E, 0x23, 0x64, 0x8D, 0xF8, 0xD4, 0x04, + 0x3B, 0x40, 0xEF, 0x6D, 0x7C, 0x6B, 0xF3, 0xB9, + 0x50, 0x15, 0x97, 0x5D, 0xB8, 0x28, 0xA1, 0xD5, + 0x22, 0xDE, 0x36, 0x26, 0xD0, 0x6A, 0x7A, 0xC0, + 0xB5, 0x14, 0x36, 0xAF, 0x3A, 0xC6, 0x50, 0xAB, + 0xFA, 0x47, 0xC8, 0x2E }, + .len = 60 + }, + .auth_tag = { + .data = { + 0xA7, 0x99, 0xAC, 0xB8, 0x27, 0xDA, 0xB1, 0x82, + 0x79, 0xFD, 0x83, 0x73, 0x52, 0x4D, 0xDB, 0xF1 }, + .len = 16 + } + +}; + +static const struct aead_test_data gcm_test_case_256_6 = { + .algo = RTE_CRYPTO_AEAD_AES_GCM, + .key = { + .data = { + 0xfe, 0xff, 0xe9, 0x92, 0x86, 0x65, 0x73, 0x1c, + 0x6d, 0x6a, 0x8f, 0x94, 0x67, 0x30, 0x83, 0x08, + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a }, + .len = 32 + }, + .iv = { + .data = { + 0xca, 0xfe, 0xba, 0xbe, 0xfa, 0xce, 0xdb, 0xad, + 0xde, 0xca, 0xf8, 0x88 }, + .len = 12 + }, + .aad = { + .data = gcm_aad_zero_text, + .len = 12 + }, + .plaintext = { + .data = { + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39 }, + .len = 60 + }, + .ciphertext = { + .data = { + 0x05, 0xA2, 0x39, 0xA5, 0xE1, 0x1A, 0x74, 0xEA, + 0x6B, 0x2A, 0x55, 0xF6, 0xD7, 0x88, 0x44, 0x7E, + 0x93, 0x7E, 0x23, 0x64, 0x8D, 0xF8, 0xD4, 0x04, + 0x3B, 0x40, 0xEF, 0x6D, 0x7C, 0x6B, 0xF3, 0xB9, + 0x50, 0x15, 0x97, 0x5D, 0xB8, 0x28, 0xA1, 0xD5, + 0x22, 0xDE, 0x36, 0x26, 0xD0, 0x6A, 0x7A, 0xC0, + 0xB5, 0x14, 0x36, 0xAF, 0x3A, 0xC6, 0x50, 0xAB, + 0xFA, 0x47, 0xC8, 0x2E }, + .len = 60 + }, + .auth_tag = { + .data = { + 0x5D, 0xA5, 0x0E, 0x53, 0x64, 0x7F, 0x3F, 0xAE, + 0x1A, 0x1F, 0xC0, 0xB0, 0xD8, 0xBE, 0xF2, 0x64 }, + .len = 16 + } +}; + +static const struct aead_test_data gcm_test_case_256_7 = { + .algo = RTE_CRYPTO_AEAD_AES_GCM, + .key = { + .data = { + 0xfe, 0xff, 0xe9, 0x92, 0x86, 0x65, 0x73, 0x1c, + 0x6d, 0x6a, 0x8f, 0x94, 0x67, 0x30, 0x83, 0x08, + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a }, + .len = 32 + }, + .iv = { + .data = { + 0xca, 0xfe, 0xba, 0xbe, 0xfa, 0xce, 0xdb, 0xad, + 0xde, 0xca, 0xf8, 0x88 }, + .len = 12 + }, + .aad = { + .data = gcm_aad_text, + .len = 12 + }, + .plaintext = { + .data = { + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39 }, + .len = 60 + }, + .ciphertext = { + .data = { + 0x05, 0xA2, 0x39, 0xA5, 0xE1, 0x1A, 0x74, 0xEA, + 0x6B, 0x2A, 0x55, 0xF6, 0xD7, 0x88, 0x44, 0x7E, + 0x93, 0x7E, 0x23, 0x64, 0x8D, 0xF8, 0xD4, 0x04, + 0x3B, 0x40, 0xEF, 0x6D, 0x7C, 0x6B, 0xF3, 0xB9, + 0x50, 0x15, 0x97, 0x5D, 0xB8, 0x28, 0xA1, 0xD5, + 0x22, 0xDE, 0x36, 0x26, 0xD0, 0x6A, 0x7A, 0xC0, + 0xB5, 0x14, 0x36, 0xAF, 0x3A, 0xC6, 0x50, 0xAB, + 0xFA, 0x47, 0xC8, 0x2E }, + .len = 60 + }, + .auth_tag = { + .data = { + 0x4E, 0xD0, 0x91, 0x95, 0x83, 0xA9, 0x38, 0x72, + 0x09, 0xA9, 0xCE, 0x5F, 0x89, 0x06, 0x4E, 0xC8 }, + .len = 16 + } +}; + +/** variable AAD AES-GCM-128 Test Vectors */ +static const struct aead_test_data gcm_test_case_aad_1 = { + .algo = RTE_CRYPTO_AEAD_AES_GCM, + .key = { + .data = { + 0xfe, 0xff, 0xe9, 0x92, 0x86, 0x65, 0x73, 0x1c, + 0x6d, 0x6a, 0x8f, 0x94, 0x67, 0x30, 0x83, 0x08 }, + .len = 16 + }, + .iv = { + .data = { + 0xca, 0xfe, 0xba, 0xbe, 0xfa, 0xce, 0xdb, 0xad, + 0xde, 0xca, 0xf8, 0x88 }, + .len = 12 + }, + .aad = { + .data = gcm_aad_text, + .len = GCM_LARGE_AAD_LENGTH + }, + .plaintext = { + .data = { + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x55 }, + .len = 64 + }, + .ciphertext = { + .data = { + 0x42, 0x83, 0x1E, 0xC2, 0x21, 0x77, 0x74, 0x24, + 0x4B, 0x72, 0x21, 0xB7, 0x84, 0xD0, 0xD4, 0x9C, + 0xE3, 0xAA, 0x21, 0x2F, 0x2C, 0x02, 0xA4, 0xE0, + 0x35, 0xC1, 0x7E, 0x23, 0x29, 0xAC, 0xA1, 0x2E, + 0x21, 0xD5, 0x14, 0xB2, 0x54, 0x66, 0x93, 0x1C, + 0x7D, 0x8F, 0x6A, 0x5A, 0xAC, 0x84, 0xAA, 0x05, + 0x1B, 0xA3, 0x0B, 0x39, 0x6A, 0x0A, 0xAC, 0x97, + 0x3D, 0x58, 0xE0, 0x91, 0x47, 0x3F, 0x59, 0x85 + }, + .len = 64 + }, + .auth_tag = { + .data = { + 0xCA, 0x70, 0xAF, 0x96, 0xA8, 0x5D, 0x40, 0x47, + 0x0C, 0x3C, 0x48, 0xF5, 0xF0, 0xF5, 0xA5, 0x7D + }, + .len = 16 + } +}; + +/** variable AAD AES-GCM-256 Test Vectors */ +static const struct aead_test_data gcm_test_case_aad_2 = { + .algo = RTE_CRYPTO_AEAD_AES_GCM, + .key = { + .data = { + 0xfe, 0xff, 0xe9, 0x92, 0x86, 0x65, 0x73, 0x1c, + 0x6d, 0x6a, 0x8f, 0x94, 0x67, 0x30, 0x83, 0x08, + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a }, + .len = 32 + }, + .iv = { + .data = { + 0xca, 0xfe, 0xba, 0xbe, 0xfa, 0xce, 0xdb, 0xad, + 0xde, 0xca, 0xf8, 0x88 }, + .len = 12 + }, + .aad = { + .data = gcm_aad_text, + .len = GCM_LARGE_AAD_LENGTH + }, + .plaintext = { + .data = { + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x55 }, + .len = 64 + }, + .ciphertext = { + .data = { + 0x05, 0xA2, 0x39, 0xA5, 0xE1, 0x1A, 0x74, 0xEA, + 0x6B, 0x2A, 0x55, 0xF6, 0xD7, 0x88, 0x44, 0x7E, + 0x93, 0x7E, 0x23, 0x64, 0x8D, 0xF8, 0xD4, 0x04, + 0x3B, 0x40, 0xEF, 0x6D, 0x7C, 0x6B, 0xF3, 0xB9, + 0x50, 0x15, 0x97, 0x5D, 0xB8, 0x28, 0xA1, 0xD5, + 0x22, 0xDE, 0x36, 0x26, 0xD0, 0x6A, 0x7A, 0xC0, + 0xB5, 0x14, 0x36, 0xAF, 0x3A, 0xC6, 0x50, 0xAB, + 0xFA, 0x47, 0xC8, 0x2E, 0xF0, 0x68, 0xE1, 0x3E + }, + .len = 64 + }, + .auth_tag = { + .data = { + 0xBA, 0x06, 0xDA, 0xA1, 0x91, 0xE1, 0xFE, 0x22, + 0x59, 0xDA, 0x67, 0xAF, 0x9D, 0xA5, 0x43, 0x94 + }, + .len = 16 + } +}; + +/** GMAC Test Vectors */ +static uint8_t gmac_plaintext[GMAC_LARGE_PLAINTEXT_LENGTH] = { + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, + 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, + 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, + 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, + 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, + 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, + 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, + 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, + 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, + 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, + 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10 +}; + +static const struct gmac_test_data gmac_test_case_1 = { + .key = { + .data = { + 0xfe, 0xff, 0xe9, 0x92, 0x86, 0x65, 0x73, 0x1c, + 0x6d, 0x6a, 0x8f, 0x94, 0x67, 0x30, 0x83, 0x08 + }, + .len = 16 + }, + .iv = { + .data = { + 0xca, 0xfe, 0xba, 0xbe, 0xfa, 0xce, 0xdb, 0xad, + 0xde, 0xca, 0xf8, 0x88 }, + .len = 12 + }, + .plaintext = { + .data = gmac_plaintext, + .len = 160 + }, + .gmac_tag = { + .data = { + 0x4C, 0x0C, 0x4F, 0x47, 0x2D, 0x78, 0xF6, 0xD8, + 0x03, 0x53, 0x20, 0x2F, 0x1A, 0xDF, 0x90, 0xD0 + }, + .len = 16 + }, +}; + +static const struct gmac_test_data gmac_test_case_2 = { + .key = { + .data = { + 0xaa, 0x74, 0x0a, 0xbf, 0xad, 0xcd, 0xa7, 0x79, + 0x22, 0x0d, 0x3b, 0x40, 0x6c, 0x5d, 0x7e, 0xc0, + 0x9a, 0x77, 0xfe, 0x9d, 0x94, 0x10, 0x45, 0x39, + }, + .len = 24 + }, + .iv = { + .data = { + 0xab, 0x22, 0x65, 0xb4, 0xc1, 0x68, 0x95, + 0x55, 0x61, 0xf0, 0x43, 0x15, }, + .len = 12 + }, + .plaintext = { + .data = gmac_plaintext, + .len = 80 + }, + .gmac_tag = { + .data = { + 0xCF, 0x82, 0x80, 0x64, 0x02, 0x46, 0xF4, 0xFB, + 0x33, 0xAE, 0x1D, 0x90, 0xEA, 0x48, 0x83, 0xDB + }, + .len = 16 + }, +}; + +static const struct gmac_test_data gmac_test_case_3 = { + .key = { + .data = { + 0xb5, 0x48, 0xe4, 0x93, 0x4f, 0x5c, 0x64, 0xd3, + 0xc0, 0xf0, 0xb7, 0x8f, 0x7b, 0x4d, 0x88, 0x24, + 0xaa, 0xc4, 0x6b, 0x3c, 0x8d, 0x2c, 0xc3, 0x5e, + 0xe4, 0xbf, 0xb2, 0x54, 0xe4, 0xfc, 0xba, 0xf7, + }, + .len = 32 + }, + .iv = { + .data = { + 0x2e, 0xed, 0xe1, 0xdc, 0x64, 0x47, 0xc7, + 0xaf, 0xc4, 0x41, 0x53, 0x58, + }, + .len = 12 + }, + .plaintext = { + .data = gmac_plaintext, + .len = 65 + }, + .gmac_tag = { + .data = { + 0x77, 0x46, 0x0D, 0x6F, 0xB1, 0x87, 0xDB, 0xA9, + 0x46, 0xAD, 0xCD, 0xFB, 0xB7, 0xF9, 0x13, 0xA1 + }, + .len = 16 + }, +}; + +/******* GCM PERF VECTORS ***********/ + +struct cryptodev_perf_test_data { + struct { + uint8_t data[64]; + unsigned len; + } key; + + struct { + uint8_t data[64] __rte_aligned(16); + unsigned len; + } iv; + + struct { + uint8_t data[64]; + unsigned len; + } aad; + + struct { + uint8_t data[2048]; + unsigned len; + } plaintext; + + struct { + uint8_t data[2048]; + unsigned len; + } ciphertext; + + struct { + uint8_t data[16]; + unsigned len; + } auth_tag; + + struct { + uint32_t size; + uint8_t data[16]; + unsigned len; + } auth_tags[7]; + +}; + +/* 2048B */ +static const struct cryptodev_perf_test_data AES_GCM_128_12IV_0AAD = { + .key = { + .data = { + 0xfe, 0xff, 0xe9, 0x92, 0x86, 0x65, 0x73, 0x1c, + 0x6d, 0x6a, 0x8f, 0x94, 0x67, 0x30, 0x83, 0x08 }, + .len = 16 + }, + .iv = { + .data = { + 0xca, 0xfe, 0xba, 0xbe, 0xfa, 0xce, 0xdb, 0xad, + 0xde, 0xca, 0xf8, 0x88 }, + .len = 12 + }, + .aad = { + .data = { 0 }, + .len = 0 + }, + .plaintext = { + .data = { + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x55, + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x55, + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x55, + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x55, + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x55, + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x55, + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x55, + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x55, + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x55, + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x55, + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x55, + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x55, + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x55, + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x55, + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x55, + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x55, + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x55, + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x55, + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x55, + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x55, + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x55, + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x55, + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x55, + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x55, + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x55, + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x55, + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x55, + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x55, + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x55, + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x55, + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x55, + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x55 + }, + .len = 2048 + }, + .ciphertext = { + .data = { + 0x42, 0x83, 0x1E, 0xC2, 0x21, 0x77, 0x74, 0x24, + 0x4B, 0x72, 0x21, 0xB7, 0x84, 0xD0, 0xD4, 0x9C, + 0xE3, 0xAA, 0x21, 0x2F, 0x2C, 0x02, 0xA4, 0xE0, + 0x35, 0xC1, 0x7E, 0x23, 0x29, 0xAC, 0xA1, 0x2E, + 0x21, 0xD5, 0x14, 0xB2, 0x54, 0x66, 0x93, 0x1C, + 0x7D, 0x8F, 0x6A, 0x5A, 0xAC, 0x84, 0xAA, 0x05, + 0x1B, 0xA3, 0x0B, 0x39, 0x6A, 0x0A, 0xAC, 0x97, + 0x3D, 0x58, 0xE0, 0x91, 0x47, 0x3F, 0x59, 0x85, + 0x04, 0x99, 0x55, 0xE1, 0x36, 0x76, 0xB7, 0x14, + 0x1D, 0xF0, 0xF6, 0x8C, 0x65, 0xD5, 0xAD, 0xFB, + 0x90, 0x7F, 0x5D, 0xA2, 0xD6, 0xFD, 0xD0, 0xE5, + 0x0D, 0x9B, 0x68, 0x21, 0x49, 0x42, 0x6E, 0x13, + 0xEC, 0x22, 0x50, 0x2A, 0x30, 0x47, 0x49, 0xA1, + 0x7F, 0xC3, 0x09, 0xE0, 0x56, 0x91, 0xC4, 0x54, + 0x70, 0xD7, 0x19, 0x40, 0xCA, 0x6B, 0x65, 0x27, + 0x3E, 0xE9, 0xD1, 0x0F, 0x1C, 0xB5, 0x45, 0x0D, + 0x27, 0xE7, 0xCF, 0x94, 0x10, 0xBF, 0xA2, 0xFA, + 0x86, 0x20, 0x3F, 0x6E, 0xE9, 0x95, 0x03, 0x5A, + 0x46, 0x11, 0x75, 0xD5, 0x37, 0x71, 0x7F, 0xE0, + 0xBC, 0x9F, 0xC8, 0xE9, 0xB1, 0x08, 0x2C, 0x59, + 0x6E, 0x51, 0x4A, 0x83, 0x38, 0xC1, 0xED, 0xE2, + 0x2E, 0x88, 0x90, 0xA5, 0x7D, 0xA4, 0x93, 0x9A, + 0x30, 0xD6, 0x96, 0x34, 0x0F, 0xC4, 0xD1, 0x7E, + 0xC9, 0x8F, 0xC5, 0xBB, 0x80, 0x50, 0x85, 0x75, + 0x7D, 0x82, 0x36, 0xDB, 0x62, 0x15, 0xAF, 0x4B, + 0x0A, 0x9D, 0xCD, 0x64, 0x00, 0xAB, 0x88, 0x28, + 0xA8, 0x35, 0x17, 0x70, 0x6F, 0x47, 0x44, 0xCD, + 0x65, 0xAE, 0xD5, 0x05, 0x0A, 0xA8, 0x2F, 0x48, + 0xAC, 0xA1, 0x72, 0x64, 0x1C, 0x7E, 0xD3, 0xF5, + 0xD8, 0x4E, 0x73, 0x17, 0x0C, 0xE5, 0x9F, 0xB6, + 0x00, 0xFA, 0xD7, 0x2C, 0x3D, 0x6A, 0x10, 0x47, + 0x7C, 0xF2, 0x6B, 0x13, 0x10, 0x8A, 0x76, 0x39, + 0xF8, 0x50, 0x33, 0xAC, 0x08, 0x1D, 0xA3, 0x48, + 0xE1, 0xD0, 0x05, 0x49, 0xB7, 0x76, 0x03, 0x72, + 0x07, 0xC5, 0xD3, 0x08, 0x79, 0x38, 0x66, 0xC1, + 0x52, 0xAF, 0x83, 0xCD, 0xF3, 0x86, 0x62, 0xBF, + 0x92, 0x24, 0x97, 0xBD, 0x5D, 0x7D, 0x81, 0x56, + 0x4C, 0xF3, 0xD2, 0x60, 0xC2, 0xDE, 0x61, 0xC1, + 0x39, 0x61, 0xDA, 0x07, 0x50, 0xC7, 0x98, 0x63, + 0x7E, 0xDD, 0x54, 0xCA, 0xDE, 0x12, 0xD2, 0xA8, + 0x19, 0x08, 0x6E, 0xF9, 0xFA, 0x6F, 0x58, 0x97, + 0xD4, 0x0B, 0x5C, 0x5B, 0xE5, 0x30, 0xE5, 0x4C, + 0x0E, 0x16, 0x87, 0xF0, 0x2C, 0xCB, 0x53, 0xB8, + 0x0C, 0xE5, 0xDF, 0x16, 0x7B, 0xE8, 0xC2, 0xCF, + 0xCC, 0xFF, 0x51, 0x24, 0xC1, 0xDD, 0x59, 0x9C, + 0xA7, 0x56, 0x03, 0xB9, 0x0A, 0x37, 0xA2, 0xAC, + 0x28, 0x8B, 0xEB, 0x51, 0x4E, 0xF1, 0xAE, 0xB5, + 0xC8, 0xB5, 0xCB, 0x8D, 0x23, 0xF6, 0x24, 0x2D, + 0xF6, 0x59, 0x62, 0xC0, 0xCB, 0xD3, 0x18, 0xE4, + 0xB7, 0x73, 0xEF, 0xDB, 0x13, 0x9A, 0xF5, 0xD3, + 0xD5, 0x61, 0x01, 0x14, 0xA5, 0xE5, 0x0D, 0x27, + 0xC9, 0xA5, 0x08, 0x1C, 0x60, 0xBA, 0x73, 0xFF, + 0xA9, 0xE0, 0x27, 0x86, 0x3F, 0xF7, 0x15, 0x03, + 0x69, 0xA7, 0x2B, 0x57, 0xAC, 0xA6, 0x70, 0x55, + 0xE9, 0xB5, 0x3F, 0xEB, 0x6F, 0xCE, 0x8A, 0xA1, + 0x9D, 0x8B, 0x84, 0xF1, 0x7C, 0xD0, 0x35, 0x21, + 0x91, 0x3D, 0x3D, 0x6E, 0x83, 0xFC, 0x45, 0x36, + 0x93, 0xDA, 0x66, 0xDF, 0x1A, 0x59, 0x22, 0xA5, + 0xC4, 0x99, 0x9B, 0xF8, 0x48, 0x9A, 0x50, 0x09, + 0xAB, 0xAE, 0x56, 0xB6, 0x49, 0x02, 0x3E, 0x90, + 0xB6, 0x07, 0x7E, 0xA7, 0x6A, 0x0A, 0xB5, 0x85, + 0x31, 0x0D, 0x84, 0xD4, 0x01, 0xE4, 0x48, 0x63, + 0xF3, 0xC1, 0x54, 0x65, 0xA6, 0x4C, 0x8B, 0x33, + 0xF9, 0x70, 0x59, 0x3B, 0xA6, 0xF6, 0x2B, 0x66, + 0xC5, 0xD2, 0xEB, 0xAB, 0x67, 0xD2, 0xE3, 0x78, + 0xA9, 0x1A, 0x4C, 0x99, 0xA9, 0xA6, 0xCA, 0xF7, + 0x65, 0xF0, 0x48, 0xF8, 0x2A, 0xEA, 0x96, 0x9F, + 0xC4, 0x50, 0x9A, 0x0C, 0xB6, 0x0D, 0x8A, 0x2F, + 0xC3, 0x99, 0x4E, 0xA0, 0x06, 0x4D, 0xAB, 0x25, + 0x2E, 0x44, 0x47, 0xB6, 0x98, 0xF1, 0x2C, 0x96, + 0x54, 0x51, 0x12, 0x41, 0x0D, 0xEF, 0x32, 0x9A, + 0x4A, 0xBD, 0xA2, 0x26, 0x53, 0xA8, 0xFD, 0x8B, + 0x6C, 0x95, 0x0A, 0x1A, 0x96, 0xEF, 0x3C, 0x85, + 0x34, 0x4E, 0x25, 0x9E, 0x1C, 0x67, 0x33, 0x8A, + 0xFF, 0x6D, 0x98, 0x93, 0x3D, 0x3F, 0x49, 0x6B, + 0xBF, 0x7C, 0x4F, 0x63, 0x5D, 0x62, 0x64, 0x67, + 0x0D, 0x07, 0x7F, 0x24, 0x4A, 0x23, 0xBC, 0x35, + 0xE0, 0x92, 0x6F, 0x51, 0xE7, 0x25, 0x97, 0xB9, + 0x14, 0x35, 0x2B, 0x48, 0xAC, 0x6F, 0x54, 0xDF, + 0xF2, 0xB4, 0xB0, 0xE0, 0xD3, 0x28, 0x0D, 0x66, + 0x46, 0x28, 0x0A, 0x16, 0x9C, 0x87, 0x73, 0xB7, + 0x9C, 0x2B, 0xB5, 0x43, 0xC9, 0x46, 0xB9, 0x1F, + 0x5F, 0x3C, 0x45, 0x03, 0x4B, 0xBF, 0x44, 0x4D, + 0xE1, 0x44, 0xDA, 0x54, 0xC5, 0x32, 0x3A, 0xFA, + 0x21, 0x5C, 0xAD, 0xD5, 0x1E, 0x1B, 0x54, 0x7C, + 0x9F, 0xEA, 0x92, 0x8C, 0xEA, 0x69, 0xC0, 0xCE, + 0xDA, 0x09, 0xAD, 0x95, 0xA0, 0x8E, 0x0B, 0x8E, + 0x10, 0x4F, 0x5B, 0x8F, 0xB8, 0x2D, 0xAC, 0x4C, + 0x94, 0x4B, 0x7C, 0x1E, 0xF1, 0x53, 0x20, 0x9B, + 0xD6, 0xC4, 0x92, 0x4C, 0x7F, 0xFB, 0x8B, 0x8E, + 0x40, 0x2F, 0x24, 0xA3, 0x4E, 0x46, 0x64, 0xF4, + 0xC6, 0x35, 0x0F, 0xC7, 0x40, 0x55, 0x43, 0xAF, + 0x7E, 0x91, 0x76, 0x48, 0x6F, 0x97, 0x7A, 0xF8, + 0x32, 0x1E, 0xD3, 0x5B, 0xBC, 0x19, 0xB5, 0x48, + 0xFA, 0x4F, 0x52, 0x77, 0x5B, 0x9E, 0xA2, 0xC8, + 0x9A, 0x83, 0x30, 0x8D, 0x9F, 0x0B, 0x6F, 0xA8, + 0x2E, 0x84, 0xCC, 0xC1, 0x50, 0x96, 0x46, 0xAE, + 0x73, 0x91, 0x7D, 0xCD, 0x88, 0xAB, 0x67, 0x3F, + 0x66, 0x3A, 0x8D, 0xB1, 0x89, 0x07, 0x93, 0xDB, + 0x42, 0x22, 0xDC, 0x13, 0xBD, 0xCD, 0xBB, 0x12, + 0x8D, 0x88, 0x44, 0x13, 0x22, 0x52, 0x81, 0xDC, + 0xEF, 0xA1, 0xE4, 0xA3, 0xA7, 0xBA, 0xEE, 0x98, + 0x79, 0x45, 0x29, 0x05, 0x65, 0x3D, 0xDC, 0xAF, + 0xA1, 0x37, 0x29, 0xFD, 0x05, 0xD1, 0x3A, 0xF7, + 0x32, 0x1D, 0x02, 0xEC, 0x28, 0x1E, 0x0F, 0x96, + 0xF3, 0x21, 0x19, 0x5F, 0x49, 0xB9, 0xEA, 0x9A, + 0xAD, 0x34, 0x58, 0xD1, 0xD9, 0xB1, 0x7D, 0xD2, + 0xEA, 0xED, 0x74, 0xE8, 0x25, 0x9A, 0x7B, 0xC5, + 0xC8, 0xD8, 0x76, 0xB6, 0xBC, 0x0B, 0x78, 0xCE, + 0xD9, 0xA6, 0xBB, 0x2F, 0x79, 0xA4, 0x45, 0x05, + 0x55, 0x6E, 0x20, 0x84, 0xEB, 0xC8, 0x70, 0xB0, + 0x3A, 0x2D, 0x06, 0x98, 0x29, 0x10, 0xB8, 0xC5, + 0xE9, 0xE4, 0xB6, 0xDE, 0x97, 0x9A, 0x0D, 0x8C, + 0xB6, 0x22, 0x16, 0x59, 0xAB, 0xB5, 0xD7, 0x14, + 0xAB, 0x08, 0x02, 0x27, 0x7B, 0xF7, 0x0E, 0xAC, + 0xC5, 0xAC, 0x4D, 0x7F, 0xE5, 0x65, 0x51, 0x40, + 0x44, 0x92, 0xB1, 0x6A, 0xB7, 0x00, 0x76, 0x89, + 0x6E, 0x08, 0x5F, 0x45, 0x2B, 0x53, 0x86, 0x86, + 0xA7, 0x85, 0xBC, 0x62, 0xAC, 0xAA, 0x82, 0x73, + 0x0A, 0xEB, 0x35, 0x16, 0x95, 0x26, 0xAB, 0x9E, + 0xE9, 0x64, 0x53, 0x99, 0x08, 0x31, 0xF5, 0x6B, + 0x1F, 0xFE, 0x47, 0x4B, 0x09, 0x33, 0x4F, 0xBF, + 0x1F, 0x0B, 0x4C, 0xB2, 0xB4, 0xA4, 0x17, 0xA9, + 0xAD, 0xC5, 0x62, 0x7C, 0xF1, 0x1B, 0xAE, 0x46, + 0xD3, 0xAC, 0xFD, 0x43, 0xFE, 0x79, 0xD0, 0x58, + 0x2F, 0x6C, 0x9F, 0xD0, 0x65, 0xA4, 0x64, 0x03, + 0xAF, 0x73, 0x46, 0x75, 0x7D, 0x49, 0x1B, 0x4C, + 0xFA, 0x49, 0xD8, 0x9A, 0xCC, 0x59, 0xC6, 0xC7, + 0xA1, 0x05, 0xC2, 0x32, 0xC8, 0x6C, 0x50, 0xA8, + 0x06, 0x58, 0xBE, 0x6C, 0x7D, 0x22, 0xD6, 0x0D, + 0x74, 0x40, 0xCE, 0xD6, 0x64, 0xD6, 0x47, 0xD0, + 0xBF, 0xF1, 0x5C, 0x54, 0xF9, 0x06, 0x3F, 0x3D, + 0x86, 0xBA, 0xF2, 0x0F, 0x5E, 0x2C, 0x01, 0xCC, + 0xD9, 0xC7, 0xB1, 0x4A, 0xB3, 0xD7, 0x26, 0xCC, + 0xC3, 0x7A, 0x74, 0x2C, 0xE1, 0x22, 0x65, 0xA0, + 0x5B, 0xCA, 0xF4, 0xE1, 0x7D, 0xE1, 0x56, 0xFD, + 0x94, 0x10, 0xC6, 0xA1, 0x4A, 0xE8, 0x6B, 0x34, + 0x4E, 0x71, 0x60, 0x77, 0x0F, 0x03, 0xDD, 0xFF, + 0xC8, 0x59, 0x54, 0x6C, 0xD4, 0x4A, 0x55, 0x24, + 0x35, 0x21, 0x60, 0x73, 0xDF, 0x6F, 0xE7, 0x3C, + 0xC2, 0xF0, 0xDA, 0xA9, 0xE5, 0x8C, 0xAC, 0xB6, + 0xFD, 0x2E, 0xF7, 0xA0, 0x18, 0xA7, 0x55, 0x47, + 0xD1, 0xCB, 0x9E, 0xAA, 0x58, 0x54, 0x3B, 0x37, + 0x18, 0xB5, 0xC1, 0xBB, 0x41, 0x59, 0xE4, 0x28, + 0x4A, 0x13, 0x90, 0x6A, 0xF7, 0xD1, 0xB3, 0x71, + 0xB6, 0x6E, 0xF6, 0x5D, 0x2E, 0x0E, 0x6C, 0x4A, + 0x7B, 0xF7, 0xB6, 0x21, 0xD4, 0xFC, 0x47, 0x8C, + 0x9B, 0x0A, 0x90, 0xAC, 0x11, 0x52, 0x86, 0x07, + 0x24, 0xDA, 0xA9, 0x49, 0x50, 0xD9, 0xDC, 0xE2, + 0x19, 0x87, 0x73, 0x88, 0xC3, 0xE4, 0xED, 0xC9, + 0x1C, 0xA8, 0x7E, 0x39, 0x48, 0x91, 0x10, 0xAB, + 0xFC, 0x3C, 0x1E, 0xEE, 0x08, 0xA1, 0xB9, 0xB2, + 0x02, 0x57, 0xB1, 0xD1, 0x35, 0x5E, 0x3D, 0x94, + 0xFB, 0x36, 0x27, 0x1A, 0x0E, 0x75, 0xFC, 0xBC, + 0xDB, 0xF3, 0xF5, 0x7C, 0x08, 0x39, 0xAA, 0xF4, + 0x2E, 0xEE, 0xCF, 0xCD, 0x2D, 0x70, 0xB8, 0x84, + 0xE6, 0x22, 0x5C, 0xC0, 0xB9, 0x33, 0xCB, 0x97, + 0xA1, 0xA3, 0xEE, 0x93, 0x71, 0xCF, 0xC9, 0x21, + 0x31, 0x7A, 0xEC, 0xE7, 0x70, 0xF2, 0xAA, 0x91, + 0xAA, 0x48, 0xAD, 0xAC, 0x03, 0xB1, 0x26, 0x52, + 0xBC, 0x65, 0x22, 0xA1, 0x09, 0x3D, 0xAB, 0x16, + 0x08, 0xBF, 0xCF, 0x3F, 0x59, 0x08, 0x6F, 0x68, + 0xEB, 0x8A, 0xB3, 0xCF, 0x77, 0x82, 0xFB, 0x25, + 0x78, 0x16, 0x4C, 0xDB, 0x72, 0xF5, 0xCF, 0x79, + 0x71, 0xE4, 0x4E, 0x23, 0x15, 0x7F, 0x1E, 0xA8, + 0x3E, 0xC0, 0x59, 0x91, 0x20, 0xAE, 0x2C, 0x1D, + 0x90, 0xC8, 0x49, 0x42, 0x48, 0x29, 0x82, 0x66, + 0x68, 0x49, 0x73, 0xDA, 0xE4, 0x28, 0xCD, 0x7B, + 0x4D, 0xE4, 0x23, 0x34, 0xB9, 0xE1, 0xB4, 0x42, + 0x67, 0x22, 0x5B, 0xEE, 0xE6, 0x74, 0x32, 0x6F, + 0x21, 0x9F, 0x97, 0x46, 0x03, 0xE1, 0xC9, 0x7A, + 0x14, 0x27, 0x30, 0xE1, 0xB2, 0x34, 0xE6, 0xAF, + 0x7B, 0xAA, 0xDD, 0x89, 0x04, 0x30, 0xD6, 0x78, + 0x0B, 0x3D, 0xC3, 0x69, 0xB0, 0x67, 0x4F, 0x4E, + 0x12, 0x21, 0x93, 0x2D, 0x79, 0xDD, 0x8B, 0xDB, + 0xEA, 0x90, 0x66, 0x54, 0xA8, 0x05, 0xF2, 0xE4, + 0x59, 0x8A, 0x96, 0x52, 0x30, 0xF0, 0x4E, 0x9A, + 0xE5, 0xD8, 0x72, 0x1C, 0x3B, 0x63, 0x02, 0xB9, + 0xC7, 0xA1, 0xDA, 0xC8, 0x6C, 0x48, 0xE0, 0xDE, + 0x59, 0x64, 0x89, 0x2C, 0xF9, 0xC8, 0x3B, 0x00, + 0xEC, 0xF2, 0x68, 0x51, 0x67, 0x05, 0x85, 0xAF, + 0xB8, 0xD5, 0x65, 0xEE, 0x73, 0x26, 0x88, 0xFB, + 0xA9, 0xD6, 0x6C, 0x68, 0x9D, 0x9F, 0x23, 0x6A, + 0x10, 0x24, 0x82, 0xB2, 0xB7, 0x40, 0x19, 0x3E, + 0x6F, 0xA2, 0xD5, 0x2C, 0x6E, 0x8D, 0xE9, 0x33, + 0x6E, 0x24, 0x94, 0x05, 0xE9, 0x2D, 0xD9, 0x3A, + 0x8C, 0xE5, 0xCC, 0x1D, 0x3F, 0xB8, 0x71, 0xA8, + 0x98, 0x33, 0xBB, 0x1A, 0xAC, 0x41, 0x0A, 0x04, + 0xFE, 0x4D, 0x46, 0x17, 0x8A, 0xCB, 0xF3, 0x4B, + 0x97, 0x02, 0xCC, 0x9D, 0x11, 0xF1, 0xBC, 0xA9, + 0xC1, 0xD1, 0xB6, 0xD6, 0x7B, 0x5F, 0x9D, 0x22, + 0x86, 0x71, 0xEC, 0x42, 0x53, 0xB7, 0x85, 0x30, + 0xAF, 0x1D, 0x01, 0xA7, 0xBF, 0x72, 0xC2, 0xC6, + 0xC9, 0xB8, 0xD8, 0xC7, 0xE9, 0xC4, 0xBA, 0xC5, + 0xB1, 0x8A, 0xB8, 0x62, 0xBF, 0x75, 0x75, 0x69, + 0xF8, 0x8D, 0x7E, 0xD9, 0xD2, 0x28, 0xB5, 0x40, + 0xCE, 0xCB, 0xB8, 0x74, 0x31, 0x40, 0x7B, 0x0D, + 0x73, 0x98, 0x99, 0x12, 0xB7, 0x75, 0x3E, 0xBC, + 0xAE, 0x48, 0xCA, 0xA9, 0x1E, 0xA7, 0x95, 0x31, + 0x87, 0x0F, 0x14, 0x52, 0xB6, 0x8E, 0x42, 0x50, + 0xB2, 0x76, 0x75, 0xD8, 0x7E, 0x66, 0x23, 0x13, + 0x8B, 0x29, 0xAA, 0x13, 0xCA, 0x8A, 0xD8, 0x9B, + 0x7B, 0x38, 0xD2, 0xE8, 0x67, 0xD1, 0x89, 0x25, + 0x9C, 0x63, 0x2F, 0xC3, 0x26, 0xC7, 0x74, 0x83, + 0x05, 0xED, 0x67, 0x02, 0x85, 0xAD, 0x1D, 0x0E, + 0xA9, 0xD6, 0xE1, 0xC7, 0x39, 0xA0, 0x6E, 0x72, + 0xCE, 0x56, 0x6C, 0xB8, 0x4A, 0xDE, 0x11, 0xA2, + 0xBF, 0xC1, 0x84, 0x98, 0x8F, 0xCA, 0x79, 0x74, + 0xCA, 0x9F, 0x45, 0x16, 0xBC, 0xB1, 0xF4, 0x03, + 0x76, 0x6E, 0xD5, 0x46, 0x60, 0xD7, 0x1D, 0xF0, + 0x87, 0x29, 0x63, 0x07, 0x06, 0xB9, 0xC2, 0x69, + 0x6D, 0xF9, 0x4B, 0x30, 0x96, 0x83, 0xB8, 0xC5, + 0xBE, 0x3A, 0xBA, 0xD0, 0x3E, 0x2B, 0x04, 0x16, + 0x6A, 0x00, 0x3B, 0x1A, 0x8E, 0xF8, 0xF6, 0x21, + 0x01, 0xD6, 0x08, 0x41, 0x74, 0xA2, 0xFC, 0x36, + 0xED, 0x11, 0x51, 0x5A, 0x4A, 0x21, 0x1A, 0x03, + 0x11, 0x95, 0x11, 0xF6, 0x73, 0x38, 0x67, 0xFC, + 0xF1, 0x2B, 0x22, 0x54, 0x65, 0x40, 0x7D, 0x8C, + 0x13, 0xC4, 0x46, 0x87, 0x09, 0x2B, 0xB5, 0xA1, + 0x82, 0x49, 0x46, 0x56, 0xF5, 0x5F, 0xF1, 0x04, + 0xD8, 0x6F, 0xDB, 0x38, 0xAD, 0xF4, 0x1A, 0xA3, + 0xFF, 0x7C, 0xC7, 0xA6, 0xAF, 0x87, 0x5C, 0x8C, + 0xEA, 0x3C, 0x9D, 0x7A, 0x4A, 0xD8, 0xA8, 0x66, + 0xDB, 0xBF, 0x12, 0x58, 0x98, 0x8E, 0xBA, 0x6F, + 0xAF, 0x20, 0xDA, 0xEE, 0x82, 0x34, 0x2F, 0x33, + 0x88, 0x98, 0xBA, 0xB2, 0x54, 0x7F, 0x9E, 0x63, + 0x19, 0x6C, 0x7D, 0xCE, 0x85, 0xF8, 0xB6, 0x77, + 0xCB, 0x38, 0x1F, 0xB1, 0x79, 0xBD, 0xED, 0x32, + 0xE3, 0xB9, 0x40, 0xEF, 0x3E, 0x6C, 0x29, 0x88, + 0x70, 0x99, 0x47, 0xA6, 0x4A, 0x1C, 0xCC, 0x0B, + 0x9B, 0x72, 0xA9, 0x29, 0x83, 0x4C, 0xDE, 0x4F, + 0x65, 0x4E, 0xCE, 0xBD, 0xFA, 0x76, 0x8D, 0xA6, + 0x1A, 0xD8, 0x66, 0xFE, 0xA4, 0x2A, 0x61, 0x50, + 0xEE, 0x15, 0xF1, 0xF0, 0x9D, 0xFF, 0xEC, 0xEE, + 0x00, 0x03, 0xFE, 0xAC, 0x53, 0x02, 0xCC, 0x87, + 0xB1, 0xA2, 0xD8, 0x34, 0x2C, 0xEC, 0xA6, 0x4C, + 0x02, 0xC0, 0xC1, 0x72, 0xD6, 0x54, 0x35, 0x24, + 0x25, 0x8B, 0xEC, 0xDA, 0x47, 0x5F, 0x5D, 0x7E, + 0xD8, 0x01, 0x51, 0xDD, 0x8F, 0xB4, 0x48, 0xDD, + 0x94, 0x99, 0x95, 0x77, 0xB3, 0x42, 0x14, 0xEB, + 0x26, 0x61, 0xE9, 0x22, 0xE3, 0x07, 0x73, 0xFB, + 0xEF, 0x38, 0x55, 0x35, 0x8F, 0xCC, 0x30, 0x1E, + 0x38, 0xE0, 0x35, 0xF4, 0x9A, 0x7C, 0xCF, 0x38, + 0x0B, 0x9E, 0xF4, 0x88, 0x4A, 0xEA, 0xF2, 0x67, + 0x9F, 0x61, 0x40, 0x34, 0x09, 0xDC, 0xBF, 0xFB, + 0x22, 0x27, 0x04, 0x8B, 0x8D, 0x85, 0x7F, 0xB2, + 0x29, 0x62, 0x25, 0x73, 0x7F, 0x46, 0x2E, 0xA3, + 0x8E, 0xAF, 0xEC, 0x55, 0x98, 0x1A, 0xEE, 0x29, + 0xA0, 0x1A, 0x5F, 0xFE, 0x5D, 0xA5, 0x76, 0x93, + 0xAB, 0x57, 0x56, 0xEA, 0xDB, 0x39, 0xAC, 0x48, + 0xBE, 0x95, 0x92, 0x2B, 0xC6, 0xE1, 0x2F, 0x36, + 0x4B, 0x08, 0x01, 0x90, 0x50, 0xD8, 0xFA, 0xF9, + 0x94, 0x4E, 0x76, 0x9B, 0x72, 0x59, 0xC2, 0x2F, + 0x61, 0x04, 0x0A, 0x9E, 0x28, 0xE5, 0x24, 0x1E, + 0x79, 0xCF, 0x8D, 0xB6, 0x52, 0xA7, 0x79, 0x5F, + 0x44, 0x98, 0xD5, 0x0E, 0x6E, 0x4B, 0x64, 0x9B, + }, + .len = 2048 + }, + .auth_tags[0] = { + .size = 64, + .data = { 0x4d, 0x5c, 0x2a, 0xf3, 0x27, 0xcd, 0x64, 0xa6, + 0x2c, 0xf3, 0x5a, 0xbd, 0x2b, 0xa6, 0xfa, 0xb4 }, + .len = 16 + }, + .auth_tags[1] = { + .size = 128, + .data = { 0xE9, 0xA9, 0x75, 0xB6, 0xEF, 0x6F, 0x8C, 0xF1, + 0xB3, 0xA9, 0x19, 0xA4, 0xAE, 0x66, 0xBD, 0x9E }, + .len = 16 + }, + .auth_tags[2] = { + .size = 256, + .data = { 0x29, 0xC3, 0x18, 0x96, 0x54, 0xCB, 0xF5, 0xAA, + 0x4E, 0x62, 0xB6, 0xFF, 0x45, 0xA6, 0x18, 0x0C }, + .len = 16 + }, + .auth_tags[3] = { + .size = 512, + .data = { 0x3B, 0xD7, 0xC3, 0x5F, 0xE4, 0x1B, 0xC2, 0xBC, + 0xE9, 0xAC, 0xF2, 0xCE, 0xA7, 0x7B, 0x1D, 0x70 }, + .len = 16 + }, + .auth_tags[4] = { + .size = 1024, + .data = { 0xCC, 0xBB, 0xBC, 0xCF, 0x86, 0x01, 0x4D, 0x93, + 0x4B, 0x68, 0x55, 0x19, 0xA1, 0x40, 0xCD, 0xEA }, + .len = 16 + }, + .auth_tags[5] = { + .size = 1536, + .data = { 0x67, 0x31, 0x11, 0xA2, 0x58, 0xB5, 0x1C, 0x23, + 0xC0, 0x41, 0x05, 0x30, 0xC6, 0xBA, 0xFA, 0x88 }, + .len = 16 + }, + .auth_tags[6] = { + .size = 2048, + .data = { 0x03, 0x9C, 0x6B, 0xB9, 0x57, 0xBF, 0x6E, 0x86, + 0x3A, 0x09, 0x5F, 0x08, 0xA9, 0xE4, 0xF2, 0x1F }, + .len = 16 + }, + .auth_tag = { + .data = { + 0x03, 0x9C, 0x6B, 0xB9, 0x57, 0xBF, 0x6E, 0x86, + 0x3A, 0x09, 0x5F, 0x08, 0xA9, 0xE4, 0xF2, 0x1F + }, + .len = 16 + }, +}; + +static const struct gmac_test_data gmac_test_case_4 = { + .key = { + .data = { + 0xfe, 0xff, 0xe9, 0x92, 0x86, 0x65, 0x73, 0x1c, + 0x6d, 0x6a, 0x8f, 0x94, 0x67, 0x30, 0x83, 0x08 + }, + .len = 16 + }, + .iv = { + .data = { + 0xca, 0xfe, 0xba, 0xbe, 0xfa, 0xce, 0xdb, 0xad, + 0xde, 0xca, 0xf8, 0x88 + }, + .len = 12 + }, + .plaintext = { + .data = gmac_plaintext, + .len = GMAC_LARGE_PLAINTEXT_LENGTH + }, + .gmac_tag = { + .data = { + 0x3f, 0x07, 0xcb, 0xb9, 0x86, 0x3a, 0xea, 0xc2, + 0x2f, 0x3a, 0x2a, 0x93, 0xd8, 0x09, 0x6b, 0xda + }, + .len = 16 + } +}; + +static const struct aead_test_data gcm_test_case_SGL_1 = { + .algo = RTE_CRYPTO_AEAD_AES_GCM, + .key = { + .data = { + 0xfe, 0xff, 0xe9, 0x92, 0x86, 0x65, 0x73, 0x1c, + 0x6d, 0x6a, 0x8f, 0x94, 0x67, 0x30, 0x83, 0x08 }, + .len = 16 + }, + .iv = { + .data = { + 0xca, 0xfe, 0xba, 0xbe, 0xfa, 0xce, 0xdb, 0xad, + 0xde, 0xca, 0xf8, 0x88 }, + .len = 12 + }, + .aad = { + .data = gcm_aad_zero_text, + .len = 0 + }, + .plaintext = { + .data = { + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x55, + 0xd8, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9b, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x54, + 0xd7, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9c, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x54, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x54, + 0xd7, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9c, + 0xd8, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9b, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x55, + 0xd8, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9b, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x54, + 0xd7, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9c, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x54, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x54, + 0xd7, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9c, + 0xd8, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9b, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x55, + 0xd8, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9b, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x54, + 0xd7, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9c, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x54, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x54, + 0xd7, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9c, + 0xd8, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9b, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x55, + 0xd8, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9b, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x54, + 0xd7, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9c, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x54, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x54, + 0xd7, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9c, + 0xd8, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9b, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x55, + 0xd8, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9b, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x54, + 0xd7, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9c, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x54, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x54, + 0xd7, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9c, + 0xd8, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9b, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x55, + 0xd8, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9b, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x54, + 0xd7, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9c, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x54, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x54, + 0xd7, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9c, + 0xd8, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9b, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x55, + 0xd8, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9b, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x54, + 0xd7, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9c, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x54, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x54, + 0xd7, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9c, + 0xd8, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9b, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x55, + 0xd8, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9b, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x54, + 0xd7, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9c, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x54, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x54, + 0xd7, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9c, + 0xd8, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9b, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x55, + 0xd8, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9b, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x54, + 0xd7, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9c, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x54, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39, 0x1a, 0xaf, 0xd2, 0x54, + 0xd7, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9c, + 0xd8, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9b, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + + }, + .len = 3120 + }, + .ciphertext = { + .data = { + 0x42, 0x83, 0x1E, 0xC2, 0x21, 0x77, 0x74, 0x24, + 0x4B, 0x72, 0x21, 0xB7, 0x84, 0xD0, 0xD4, 0x9C, + 0xE3, 0xAA, 0x21, 0x2F, 0x2C, 0x02, 0xA4, 0xE0, + 0x35, 0xC1, 0x7E, 0x23, 0x29, 0xAC, 0xA1, 0x2E, + 0x21, 0xD5, 0x14, 0xB2, 0x54, 0x66, 0x93, 0x1C, + 0x7D, 0x8F, 0x6A, 0x5A, 0xAC, 0x84, 0xAA, 0x05, + 0x1B, 0xA3, 0x0B, 0x39, 0x6A, 0x0A, 0xAC, 0x97, + 0x3D, 0x58, 0xE0, 0x91, 0x47, 0x3F, 0x59, 0x85, + 0x05, 0x99, 0x55, 0xE1, 0x36, 0x76, 0xB7, 0x14, + 0x1D, 0xF0, 0xF6, 0x8C, 0x65, 0xD5, 0xAD, 0xFA, + 0x90, 0x7F, 0x5D, 0xA2, 0xD6, 0xFD, 0xD0, 0xE5, + 0x0D, 0x9B, 0x68, 0x21, 0x49, 0x42, 0x6E, 0x13, + 0xEC, 0x22, 0x50, 0x2A, 0x30, 0x47, 0x49, 0xA1, + 0x7F, 0xC3, 0x09, 0xE0, 0x56, 0x91, 0xC4, 0x54, + 0x70, 0xD7, 0x19, 0x40, 0xCA, 0x6B, 0x65, 0x27, + 0x3E, 0xE9, 0xD1, 0x0F, 0x1C, 0xB5, 0x45, 0x0C, + 0x29, 0xE7, 0xCF, 0x94, 0x10, 0xBF, 0xA2, 0xFA, + 0x86, 0x20, 0x3F, 0x6E, 0xE9, 0x95, 0x03, 0x5C, + 0x46, 0x11, 0x75, 0xD5, 0x37, 0x71, 0x7F, 0xE0, + 0xBC, 0x9F, 0xC8, 0xE9, 0xB1, 0x08, 0x2C, 0x59, + 0x6E, 0x51, 0x4A, 0x83, 0x38, 0xC1, 0xED, 0xE2, + 0x2E, 0x88, 0x90, 0xA5, 0x7D, 0xA4, 0x93, 0x9A, + 0x30, 0xD6, 0x96, 0x34, 0x0F, 0xC4, 0xD1, 0x7E, + 0xC9, 0x8F, 0xC5, 0xBB, 0x80, 0x50, 0x85, 0x73, + 0x8B, 0x7C, 0x0A, 0xDA, 0xD3, 0x37, 0x1C, 0x8B, + 0x1E, 0xAE, 0x29, 0x54, 0x05, 0x53, 0x48, 0xE5, + 0x94, 0xF1, 0xC5, 0x1A, 0x60, 0xDC, 0x61, 0x43, + 0xCD, 0x45, 0x4C, 0x6B, 0x95, 0xAD, 0x52, 0xE0, + 0x9E, 0xD1, 0x4E, 0xCC, 0x03, 0x27, 0x50, 0xD4, + 0xEB, 0xBD, 0x71, 0xA6, 0xD0, 0x2B, 0x23, 0xC0, + 0x9E, 0x5F, 0x34, 0xFD, 0xDE, 0xC1, 0x43, 0x35, + 0x77, 0xFB, 0xFD, 0xDF, 0xA0, 0x28, 0x42, 0x3B, + 0x0F, 0x2D, 0x31, 0xB4, 0x7A, 0xA8, 0x2F, 0xDF, + 0x58, 0xB5, 0x00, 0x19, 0x8D, 0xEB, 0x2C, 0xBB, + 0xAE, 0xAD, 0x74, 0x7F, 0x25, 0xAA, 0x24, 0x3E, + 0xCD, 0x89, 0x5E, 0x05, 0xD3, 0xBA, 0x0E, 0x9A, + 0x34, 0x7B, 0xE0, 0x11, 0xD2, 0xBA, 0x5A, 0x51, + 0xB4, 0x0D, 0xEE, 0x61, 0x73, 0xFC, 0xD2, 0x01, + 0x2D, 0x52, 0x3E, 0x37, 0x55, 0x3F, 0x58, 0xA8, + 0x1C, 0x8F, 0x1D, 0xD6, 0x3C, 0x39, 0x06, 0x18, + 0x65, 0x60, 0x55, 0x19, 0xAD, 0x1E, 0x78, 0xE9, + 0xF7, 0xF5, 0xFC, 0xCD, 0x5F, 0xF1, 0x34, 0x0C, + 0xA6, 0xFD, 0x1E, 0x9E, 0xB3, 0xCE, 0x2E, 0x10, + 0xFB, 0x98, 0xDD, 0x0E, 0x09, 0x5D, 0x4E, 0x58, + 0x75, 0x9A, 0x54, 0x74, 0xFB, 0x40, 0x76, 0x55, + 0x0E, 0x3E, 0xA4, 0xCE, 0x56, 0xA5, 0xE0, 0x53, + 0xB7, 0xAD, 0x36, 0x99, 0x6E, 0xCD, 0xC2, 0x90, + 0x6E, 0xEA, 0xBC, 0x21, 0xAC, 0x31, 0xFF, 0x2B, + 0x00, 0xA7, 0x5E, 0xC1, 0x7A, 0xF1, 0xAB, 0x24, + 0xA3, 0x40, 0x0B, 0xEB, 0x16, 0x62, 0x35, 0x1E, + 0xE9, 0xA5, 0xD3, 0x7E, 0xAA, 0x7E, 0x28, 0xA8, + 0x3F, 0xD8, 0x0A, 0x04, 0x12, 0x0F, 0xFF, 0x68, + 0x10, 0x85, 0x22, 0xD6, 0x05, 0x6A, 0x3A, 0xCB, + 0xC0, 0xCF, 0x8C, 0x20, 0xF0, 0x34, 0x32, 0xAA, + 0x76, 0x93, 0xE2, 0x23, 0x4F, 0xF2, 0xE6, 0x84, + 0x3B, 0xD4, 0xF3, 0x5D, 0xF3, 0x17, 0xEE, 0x27, + 0x67, 0xC3, 0x01, 0x6F, 0x32, 0xDE, 0xF6, 0xF6, + 0x87, 0xE9, 0x82, 0xEF, 0x1F, 0xA1, 0xE2, 0x68, + 0xF8, 0x5D, 0x49, 0x92, 0x47, 0x01, 0x75, 0x87, + 0x52, 0xD3, 0x54, 0xAE, 0x3B, 0xB7, 0xB2, 0x07, + 0x0F, 0x62, 0x7B, 0xF7, 0x50, 0x97, 0x9A, 0x4A, + 0x98, 0x65, 0x23, 0xA3, 0x5D, 0x76, 0x0A, 0x9C, + 0x6C, 0xE7, 0x89, 0xAD, 0x86, 0x70, 0xE7, 0x16, + 0x5F, 0x2F, 0x2E, 0x97, 0x29, 0x31, 0xF0, 0x60, + 0x33, 0x2C, 0xD7, 0xAA, 0xD6, 0xF0, 0x50, 0xB8, + 0xBD, 0x29, 0xA8, 0xA9, 0xAC, 0x5E, 0x0A, 0x3A, + 0x59, 0x34, 0x9A, 0x92, 0x25, 0x71, 0xB3, 0x16, + 0xC5, 0xD3, 0xA4, 0x15, 0x75, 0x9A, 0xB5, 0x78, + 0x6E, 0xCF, 0xAF, 0xC0, 0x39, 0x28, 0x44, 0x21, + 0xBB, 0xE8, 0x32, 0xAB, 0xCB, 0xF8, 0x4B, 0xE7, + 0x63, 0x9C, 0x56, 0xE7, 0xB2, 0xD6, 0x23, 0x17, + 0xDE, 0x92, 0xE9, 0x22, 0xC3, 0x36, 0xA5, 0xAC, + 0xA9, 0x98, 0x34, 0xAA, 0xFB, 0x03, 0x33, 0x33, + 0xBE, 0xD8, 0x22, 0x7F, 0xFA, 0x34, 0xA0, 0x35, + 0xC8, 0xA0, 0xDC, 0x35, 0x82, 0x06, 0x58, 0xE6, + 0xBF, 0x7C, 0x4F, 0x63, 0x5D, 0x62, 0x64, 0x67, + 0x0D, 0x07, 0x7F, 0x24, 0x4A, 0x23, 0xBC, 0x35, + 0xE0, 0x92, 0x6F, 0x51, 0xE7, 0x25, 0x97, 0xB9, + 0x14, 0x35, 0x2B, 0x48, 0xAC, 0x6F, 0x54, 0xDF, + 0xF2, 0xB4, 0xB0, 0xE0, 0xD3, 0x28, 0x0D, 0x67, + 0x48, 0x28, 0x0A, 0x16, 0x9C, 0x87, 0x73, 0xB7, + 0x9C, 0x2B, 0xB5, 0x43, 0xC9, 0x46, 0xB9, 0x19, + 0x01, 0xAA, 0xDE, 0x75, 0xA6, 0x0F, 0xB5, 0x72, + 0x6A, 0x51, 0xE3, 0xAC, 0xE0, 0xF6, 0x96, 0x13, + 0xBB, 0xC7, 0x08, 0x13, 0x9E, 0x47, 0xAA, 0xF5, + 0x9E, 0x69, 0xAC, 0x95, 0x29, 0xFE, 0xFF, 0x99, + 0xB2, 0x52, 0x72, 0x45, 0xF2, 0x07, 0xEB, 0x3C, + 0x0F, 0x75, 0x29, 0x73, 0x0D, 0x77, 0x58, 0x83, + 0xCB, 0xDD, 0xE7, 0x68, 0x1C, 0xE3, 0xD1, 0xA4, + 0x5D, 0xD1, 0xAB, 0xB4, 0x5A, 0x3F, 0x27, 0x66, + 0xDA, 0xB4, 0x81, 0x65, 0xCE, 0x1A, 0x9A, 0x7D, + 0xC7, 0xB6, 0x31, 0xDE, 0x83, 0xC2, 0x7C, 0xF8, + 0xD3, 0xC7, 0x97, 0x28, 0x50, 0xF2, 0x95, 0xFC, + 0xA7, 0xB2, 0xA6, 0x46, 0xEF, 0x10, 0xD2, 0x38, + 0x93, 0x14, 0x8D, 0xA7, 0x09, 0x17, 0x42, 0x7A, + 0x85, 0xB9, 0x42, 0x71, 0x2A, 0x51, 0x9B, 0x66, + 0x71, 0x12, 0x57, 0xB7, 0xBD, 0x26, 0xB7, 0x91, + 0xF8, 0x84, 0x44, 0x35, 0xAD, 0x6F, 0xCB, 0xD7, + 0xFC, 0xA1, 0x28, 0x77, 0x09, 0x5B, 0x6D, 0x52, + 0x43, 0xA1, 0xE2, 0x0A, 0x7E, 0x5A, 0x84, 0x45, + 0x20, 0xDE, 0xA5, 0x73, 0x1D, 0x37, 0x6E, 0xD8, + 0x7A, 0x0D, 0x91, 0xBE, 0xF4, 0xB3, 0x89, 0xE9, + 0x1F, 0x1E, 0xF6, 0xD5, 0x37, 0xB4, 0x3C, 0x1D, + 0xBE, 0x0D, 0x5B, 0x01, 0xB0, 0x8B, 0xCE, 0x3E, + 0x6D, 0x8B, 0x99, 0x9A, 0xC5, 0xAE, 0xFE, 0xA9, + 0x78, 0x34, 0x20, 0xA7, 0x6C, 0x7D, 0x46, 0x72, + 0x37, 0xAF, 0xFD, 0x17, 0x59, 0xED, 0x83, 0x5B, + 0xEB, 0x6E, 0x4A, 0xF1, 0xE6, 0x0D, 0x44, 0x92, + 0x65, 0x8E, 0x97, 0xD6, 0x83, 0x6E, 0x97, 0xCA, + 0x4C, 0x0A, 0xCE, 0x32, 0x2A, 0xAD, 0x22, 0x73, + 0xCB, 0xCB, 0xC3, 0x55, 0x08, 0x63, 0x23, 0xC2, + 0x31, 0x24, 0x90, 0x54, 0x99, 0xB2, 0x8C, 0xC7, + 0x8A, 0xB6, 0xFF, 0xC2, 0x75, 0xB1, 0xD9, 0x3D, + 0x95, 0xDC, 0xB6, 0xCF, 0x11, 0x74, 0x06, 0x54, + 0x03, 0xE3, 0x9B, 0x49, 0xE4, 0xF2, 0x73, 0x04, + 0xF7, 0xDC, 0x71, 0xD7, 0xFA, 0x3C, 0xD2, 0x61, + 0x77, 0x61, 0xB3, 0xDB, 0x6B, 0xCE, 0xCA, 0xFF, + 0xF0, 0xAD, 0xBC, 0x94, 0xC8, 0xF8, 0xD5, 0xF4, + 0x38, 0xA3, 0x61, 0xAA, 0x8C, 0x96, 0xEE, 0x56, + 0xAC, 0xB4, 0x42, 0xBA, 0x1A, 0xE1, 0x70, 0x98, + 0x1F, 0x9A, 0x6F, 0x98, 0xB9, 0x13, 0x46, 0xAB, + 0x0B, 0xCD, 0xA3, 0x7B, 0x0C, 0xCB, 0x8F, 0x72, + 0x23, 0xCF, 0x9E, 0xD8, 0xBB, 0x3F, 0x32, 0x27, + 0x54, 0xB8, 0x60, 0x64, 0x83, 0xAE, 0x22, 0xD1, + 0x6A, 0xC9, 0xF8, 0x13, 0xC4, 0xE4, 0xFF, 0x97, + 0xD8, 0x92, 0xA3, 0xD1, 0xD4, 0x86, 0xD7, 0xC3, + 0xBB, 0x40, 0xA2, 0x45, 0x78, 0xB1, 0xDB, 0x80, + 0xC6, 0x8D, 0x0A, 0xF0, 0xC3, 0xC2, 0xE3, 0x48, + 0xA1, 0x05, 0xC2, 0x32, 0xC8, 0x6C, 0x50, 0xA8, + 0x06, 0x58, 0xBE, 0x6C, 0x7D, 0x22, 0xD6, 0x0D, + 0x74, 0x40, 0xCE, 0xD6, 0x64, 0xD6, 0x47, 0xD0, + 0xBF, 0xF1, 0x5C, 0x54, 0xF9, 0x06, 0x3F, 0x3D, + 0x86, 0xBA, 0xF2, 0x0F, 0x5E, 0x2C, 0x01, 0xCC, + 0xD9, 0xC7, 0xB1, 0x4A, 0xB3, 0xD7, 0x26, 0xCC, + 0xC3, 0x7A, 0x74, 0x2C, 0xE1, 0x22, 0x65, 0xA0, + 0x5B, 0xCA, 0xF4, 0xE1, 0x7D, 0xE1, 0x56, 0xFD, + 0x95, 0x10, 0xC6, 0xA1, 0x4A, 0xE8, 0x6B, 0x34, + 0x4E, 0x71, 0x60, 0x77, 0x0F, 0x03, 0xDD, 0xFE, + 0xC8, 0x59, 0x54, 0x6C, 0xD4, 0x4A, 0x55, 0x24, + 0x35, 0x21, 0x60, 0x73, 0xDF, 0x6F, 0xE7, 0x3C, + 0xC2, 0xF0, 0xDA, 0xA9, 0xE5, 0x8C, 0xAC, 0xB6, + 0xFD, 0x2E, 0xF7, 0xA0, 0x18, 0xA7, 0x55, 0x47, + 0xD1, 0xCB, 0x9E, 0xAA, 0x58, 0x54, 0x3B, 0x37, + 0x18, 0xB5, 0xC1, 0xBB, 0x41, 0x59, 0xE4, 0x29, + 0x44, 0x13, 0x90, 0x6A, 0xF7, 0xD1, 0xB3, 0x71, + 0xB6, 0x6E, 0xF6, 0x5D, 0x2E, 0x0E, 0x6C, 0x4C, + 0x7B, 0xF7, 0xB6, 0x21, 0xD4, 0xFC, 0x47, 0x8C, + 0x9B, 0x0A, 0x90, 0xAC, 0x11, 0x52, 0x86, 0x07, + 0x24, 0xDA, 0xA9, 0x49, 0x50, 0xD9, 0xDC, 0xE2, + 0x19, 0x87, 0x73, 0x88, 0xC3, 0xE4, 0xED, 0xC9, + 0x1C, 0xA8, 0x7E, 0x39, 0x48, 0x91, 0x10, 0xAB, + 0xFC, 0x3C, 0x1E, 0xEE, 0x08, 0xA1, 0xB9, 0xB4, + 0xF4, 0xA9, 0x8D, 0xD0, 0x84, 0x7C, 0x8E, 0x54, + 0xEF, 0x05, 0xC3, 0x2A, 0x0B, 0x8D, 0x3C, 0x71, + 0xE7, 0x37, 0x27, 0x16, 0x07, 0xA2, 0x8F, 0x7A, + 0x86, 0x05, 0x56, 0xA3, 0xB2, 0x75, 0xC5, 0x2C, + 0xD4, 0x52, 0x60, 0x68, 0xA6, 0x6A, 0x48, 0xB6, + 0x92, 0x50, 0xEC, 0x22, 0xAD, 0x01, 0x75, 0x57, + 0xAF, 0xDF, 0x0F, 0x36, 0x93, 0x59, 0xF9, 0xE3, + 0xA1, 0x41, 0x3B, 0x60, 0xB3, 0x13, 0x12, 0x50, + 0x4B, 0x18, 0x20, 0xB9, 0x7B, 0x88, 0x27, 0x81, + 0xB1, 0xDA, 0xCA, 0x6F, 0x63, 0x95, 0x40, 0xA1, + 0x42, 0xE2, 0x14, 0xB8, 0x2B, 0x10, 0xB9, 0xDA, + 0xE7, 0x30, 0x91, 0x13, 0x52, 0xC9, 0xA3, 0x5C, + 0xD7, 0xBB, 0x39, 0x8F, 0x9A, 0xB8, 0xC5, 0xAF, + 0xC6, 0x3E, 0x65, 0x90, 0x91, 0x8C, 0x9F, 0xDD, + 0x84, 0xFB, 0xAD, 0x72, 0x4D, 0xD1, 0x42, 0xAD, + 0x0A, 0x1B, 0x3A, 0xC6, 0x06, 0x03, 0x19, 0xCB, + 0x31, 0x8C, 0x18, 0xD4, 0xEE, 0x90, 0x94, 0x3C, + 0x44, 0xDC, 0xFB, 0x78, 0x5C, 0xB5, 0xE3, 0x2F, + 0x89, 0x74, 0x0E, 0x28, 0x9C, 0xE4, 0xB4, 0xD2, + 0xE3, 0x5A, 0x32, 0xF9, 0xC0, 0x81, 0x6A, 0x38, + 0xC2, 0xCF, 0xD8, 0xD9, 0x3E, 0xAD, 0xF9, 0xB1, + 0xA2, 0x55, 0x64, 0x1E, 0xEC, 0xF5, 0x0D, 0xB1, + 0x8D, 0x07, 0x4E, 0xE5, 0x59, 0xE1, 0xE7, 0xFE, + 0x4C, 0xCF, 0x11, 0xF8, 0x27, 0xC2, 0x29, 0xE2, + 0xAF, 0x74, 0xAA, 0x53, 0x81, 0xD2, 0xFD, 0x5A, + 0xF1, 0xEB, 0x96, 0x2C, 0x3E, 0x9B, 0xC2, 0x74, + 0xFB, 0x65, 0x08, 0xA2, 0x63, 0xD3, 0xC5, 0x51, + 0xAF, 0x19, 0x8B, 0x34, 0x8B, 0x7D, 0xB7, 0x97, + 0x55, 0x97, 0x6D, 0x01, 0x5D, 0x98, 0xAA, 0x67, + 0x11, 0xBD, 0xC2, 0x99, 0x2F, 0xB4, 0xCA, 0x04, + 0x36, 0xF0, 0xB1, 0xA0, 0xBD, 0xA3, 0x4F, 0x4F, + 0xB6, 0x7B, 0xF5, 0x1E, 0x38, 0x87, 0xC2, 0x38, + 0x99, 0x5C, 0xE9, 0x2D, 0xDF, 0xAF, 0x5A, 0xF3, + 0x7A, 0x17, 0x70, 0x35, 0xEC, 0xD5, 0x19, 0xF7, + 0xB0, 0x21, 0x1E, 0x77, 0x30, 0x23, 0x54, 0x26, + 0x61, 0x4E, 0xB9, 0x02, 0xDE, 0xF4, 0x86, 0x93, + 0x47, 0x28, 0x43, 0x47, 0xB0, 0x56, 0xDC, 0x84, + 0x3E, 0x6A, 0x6B, 0xEA, 0x4D, 0x63, 0xFE, 0x56, + 0x5E, 0xF7, 0x6B, 0x1E, 0x5B, 0x63, 0xF1, 0x07, + 0x20, 0x2E, 0x9B, 0xEE, 0xDC, 0x70, 0x5E, 0x36, + 0x59, 0xE3, 0x3D, 0xA6, 0x0E, 0x50, 0x71, 0x06, + 0xDD, 0x8B, 0x3C, 0xF7, 0xEC, 0x3C, 0x7A, 0x08, + 0x8D, 0x4E, 0x6A, 0x08, 0xB0, 0xEE, 0x50, 0xE0, + 0xF9, 0x0E, 0x40, 0xC0, 0x11, 0xBF, 0x8A, 0x17, + 0x63, 0x9D, 0x59, 0x14, 0x0E, 0x25, 0x94, 0x09, + 0xE6, 0x34, 0xEC, 0x0F, 0xE4, 0x7C, 0x59, 0xCD, + 0x99, 0x85, 0x8E, 0x0F, 0xA1, 0x9E, 0x84, 0xBC, + 0x13, 0x20, 0x5F, 0x56, 0x26, 0x10, 0x1A, 0x77, + 0x77, 0x7B, 0x4B, 0x68, 0x13, 0x8A, 0x2C, 0xA5, + 0x01, 0xBF, 0xAD, 0xF2, 0x2C, 0xD9, 0x4B, 0x24, + 0x4C, 0xF5, 0x96, 0x4E, 0xD8, 0xE8, 0x98, 0xA8, + 0x9C, 0x63, 0x2F, 0xC3, 0x26, 0xC7, 0x74, 0x83, + 0x05, 0xED, 0x67, 0x02, 0x85, 0xAD, 0x1D, 0x0E, + 0xA9, 0xD6, 0xE1, 0xC7, 0x39, 0xA0, 0x6E, 0x72, + 0xCE, 0x56, 0x6C, 0xB8, 0x4A, 0xDE, 0x11, 0xA2, + 0xBF, 0xC1, 0x84, 0x98, 0x8F, 0xCA, 0x79, 0x75, + 0xC4, 0x9F, 0x45, 0x16, 0xBC, 0xB1, 0xF4, 0x03, + 0x76, 0x6E, 0xD5, 0x46, 0x60, 0xD7, 0x1D, 0xF6, + 0xD9, 0xBF, 0xF8, 0x71, 0xEB, 0x09, 0x33, 0x56, + 0xE6, 0xEC, 0x72, 0xC8, 0xB3, 0x47, 0x14, 0x2C, + 0x24, 0xA1, 0x1F, 0x16, 0xBE, 0x77, 0xFA, 0x9F, + 0x6B, 0x83, 0x05, 0x03, 0x4D, 0x6F, 0xC9, 0x76, + 0x69, 0x8D, 0xD7, 0x91, 0x26, 0x2B, 0x1C, 0x84, + 0xF2, 0x2B, 0x23, 0xA6, 0xFF, 0x7B, 0xEE, 0xCC, + 0x4E, 0x03, 0x8A, 0x80, 0x9E, 0x88, 0x96, 0xC3, + 0x7A, 0x3E, 0x1B, 0xAC, 0x40, 0x84, 0xD1, 0x64, + 0x89, 0x5F, 0xE3, 0x41, 0x89, 0x77, 0x4B, 0x28, + 0x83, 0xCA, 0x78, 0x4F, 0x36, 0xC8, 0xCE, 0x53, + 0x75, 0x39, 0x3A, 0x58, 0x92, 0x91, 0xF5, 0xA7, + 0x6A, 0xD0, 0xB2, 0xBB, 0xFC, 0x8E, 0x3B, 0xFC, + 0x83, 0x67, 0x42, 0xAA, 0x18, 0x51, 0x48, 0xD4, + 0xC4, 0x85, 0x60, 0xA4, 0x2D, 0xD4, 0x4E, 0xA1, + 0xF0, 0xB6, 0x41, 0x98, 0x6F, 0x84, 0xDE, 0x0C, + 0x03, 0x8D, 0x83, 0x4A, 0x71, 0xBB, 0x32, 0x8B, + 0x83, 0xF7, 0xD8, 0x08, 0x05, 0xA4, 0x48, 0xFE, + 0xCA, 0xBB, 0x21, 0xA8, 0xBA, 0x2A, 0xD2, 0x65, + 0x4E, 0xEF, 0xA1, 0x8F, 0x01, 0x09, 0xC6, 0x8C, + 0xE5, 0x35, 0x32, 0xBB, 0x19, 0x15, 0xAB, 0x7A, + 0xFD, 0x29, 0x76, 0xF9, 0xD1, 0xC5, 0x3E, 0xFD, + 0x7A, 0x74, 0xBC, 0x41, 0x4F, 0x2C, 0x79, 0x6F, + 0x45, 0x4E, 0xFD, 0x88, 0x49, 0x9A, 0x90, 0x6F, + 0x65, 0x00, 0xC8, 0x08, 0xB8, 0x3B, 0x40, 0x06, + 0x9A, 0x98, 0x5B, 0x6A, 0xD3, 0x5E, 0x32, 0x0E, + 0xB0, 0x21, 0xE6, 0x2D, 0xEF, 0x7B, 0x99, 0x1B, + 0xAF, 0x96, 0x20, 0x12, 0xE9, 0x31, 0xDA, 0x20, + 0xB0, 0x27, 0x99, 0xC7, 0x14, 0x56, 0x3A, 0x08, + 0x46, 0xA4, 0xB2, 0x0C, 0x6C, 0x1F, 0x1B, 0xAF, + 0x9F, 0x90, 0x03, 0xBB, 0x03, 0xE0, 0x20, 0xE9, + 0x45, 0x33, 0xA0, 0x3E, 0x01, 0x2C, 0xA7, 0x4A, + 0xCC, 0xC6, 0xF5, 0xA3, 0x35, 0x0D, 0xE1, 0x5E, + 0x90, 0x0B, 0xAC, 0x9A, 0x05, 0x79, 0xB2, 0x90, + 0x39, 0xEE, 0xC8, 0x20, 0x55, 0xB3, 0x71, 0x46, + 0xAC, 0x92, 0x42, 0x85, 0xD5, 0x12, 0x03, 0x8D, + 0xBC, 0x82, 0xE7, 0x5A, 0x6E, 0x2E, 0x2C, 0xC0, + 0xB6, 0x44, 0xF8, 0xBB, 0x5F, 0x7A, 0x42, 0x86, + 0x28, 0xF0, 0x9B, 0xF9, 0x17, 0xDD, 0x35, 0x2F, + 0x56, 0xE4, 0x63, 0xFF, 0xEC, 0x87, 0xC5, 0x53, + 0xBF, 0x64, 0xB2, 0xDA, 0xDE, 0xC1, 0x6C, 0x85, + 0x82, 0x51, 0x40, 0x41, 0xC9, 0x7A, 0x0A, 0xB8, + 0xB2, 0x75, 0x03, 0x88, 0x22, 0x6D, 0x76, 0x6E, + 0x2D, 0x2B, 0x73, 0xCB, 0x48, 0xC4, 0xED, 0xE0, + 0x96, 0xFA, 0x36, 0x9F, 0x99, 0xC7, 0x97, 0xDE, + 0x6D, 0xFC, 0x69, 0x86, 0x57, 0x5F, 0xB9, 0x93, + 0x78, 0x5C, 0x07, 0x64, 0x61, 0xD0, 0x41, 0x14, + 0x32, 0xED, 0xC0, 0xE4, 0xAC, 0xFC, 0x10, 0x0D, + 0xAF, 0xEE, 0xDA, 0xB3, 0x6D, 0xB8, 0x7C, 0x10, + 0xD5, 0x3B, 0x88, 0xE1, 0x15, 0xE1, 0xA4, 0x27, + 0xFE, 0xEE, 0x0A, 0xC8, 0x95, 0xCF, 0xCA, 0x99, + 0x98, 0x1D, 0xF3, 0x0E, 0xB8, 0x03, 0xD5, 0x51, + 0x4B, 0x56, 0xB9, 0x07, 0x85, 0x58, 0x17, 0x51, + 0x16, 0xC4, 0x86, 0xBB, 0xD3, 0x50, 0x01, 0x0E, + 0x7B, 0x9C, 0xEF, 0xF0, 0x28, 0x4A, 0xD7, 0x3D, + 0x1E, 0x3A, 0xBB, 0xCF, 0x2C, 0x90, 0x12, 0x2A, + 0xB3, 0x90, 0x72, 0xE3, 0x93, 0x81, 0xE8, 0xA4, + 0xEF, 0x8F, 0xD9, 0x45, 0x4F, 0xB1, 0xD0, 0x21, + 0xDA, 0x20, 0x5C, 0xE9, 0x41, 0x41, 0x4E, 0x48, + 0x95, 0x4D, 0x5A, 0xB3, 0xE5, 0x8B, 0xFC, 0xDE, + 0xB9, 0x7B, 0x93, 0xBE, 0xA2, 0x74, 0x1B, 0xFA, + 0xED, 0xCC, 0x0E, 0xDD, 0x96, 0x13, 0x2C, 0xAC, + 0xDE, 0x2B, 0x2D, 0x8A, 0x30, 0x5A, 0xB8, 0x4B, + 0x08, 0x2C, 0x74, 0xF7, 0xB4, 0x45, 0xD3, 0xA5, + 0x62, 0x87, 0xCA, 0x16, 0xEB, 0x49, 0x46, 0x0C, + 0x87, 0x7F, 0x11, 0x1D, 0x22, 0x66, 0x0A, 0x38, + 0x90, 0x3A, 0x31, 0x38, 0x73, 0xB2, 0xD5, 0x5E, + 0x06, 0xC4, 0x1E, 0x3D, 0xB7, 0x52, 0xB8, 0xE5, + 0xC0, 0xF9, 0x72, 0xBC, 0x7A, 0x8A, 0xD3, 0xB4, + 0x1D, 0xA9, 0x93, 0x3B, 0x7E, 0xFF, 0x8E, 0xA0, + 0x96, 0x52, 0xE9, 0x9E, 0x60, 0x4C, 0x02, 0x90, + 0xE5, 0x46, 0x92, 0xB3, 0xB8, 0x24, 0xE9, 0xD0, + 0xCE, 0xD3, 0x0B, 0xCD, 0x8B, 0xE8, 0x72, 0xEA, + 0x6E, 0xBF, 0x2B, 0x99, 0x6F, 0xC0, 0x65, 0xE8, + 0x92, 0x30, 0x03, 0x28, 0xA9, 0xB0, 0xA7, 0x03, + 0x92, 0x2C, 0xC8, 0x38, 0x8C, 0x38, 0x56, 0xEE, + 0xDB, 0x39, 0xBD, 0x7E, 0xE9, 0x8D, 0xDB, 0xC1, + 0xD5, 0x71, 0xC7, 0x84, 0xF3, 0xB2, 0x23, 0x22, + 0xB5, 0x98, 0xB3, 0x36, 0xF1, 0xC4, 0xB1, 0xA4, + 0xF2, 0x84, 0x24, 0xE5, 0x97, 0x48, 0x34, 0x43, + 0xEF, 0xD9, 0xF4, 0x10, 0xE4, 0x13, 0xEE, 0x6C, + 0xE7, 0x5D, 0x9B, 0xBA, 0x35, 0xF5, 0x7D, 0xE5, + 0xBF, 0x8A, 0xCC, 0x3D, 0x28, 0xCF, 0xE8, 0x90, + 0xE3, 0xCF, 0x01, 0x69, 0xD7, 0xC0, 0xD2, 0x2C, + 0xC2, 0x9B, 0x89, 0xF2, 0xA9, 0x83, 0xA2, 0xA9, + 0x12, 0xAA, 0x56, 0xD8, 0xCB, 0xA5, 0x8B, 0x0A, + 0x03, 0xC1, 0xE1, 0x8E, 0x02, 0x36, 0x3D, 0x8F, + 0x58, 0x4D, 0xEB, 0x93, 0x91, 0xC6, 0xE7, 0x22, + 0xCE, 0xA8, 0x02, 0xD2, 0x82, 0x0D, 0x43, 0x4D, + 0x4E, 0x11, 0xF8, 0x7B, 0x45, 0xD0, 0x23, 0xF7, + 0x14, 0x35, 0x16, 0xA4, 0x0B, 0xAD, 0xFE, 0xE2, + 0x2B, 0xFD, 0xF7, 0x17, 0xA9, 0x93, 0x77, 0x82, + 0x45, 0x6E, 0x51, 0x1F, 0x5C, 0x2C, 0x5F, 0xFF, + 0x1A, 0xA3, 0x0E, 0x29, 0xA5, 0x1D, 0xFD, 0x0E, + 0xDD, 0x14, 0xF6, 0x69, 0x20, 0x15, 0xFD, 0xBB, + 0xF8, 0xAF, 0x3D, 0xF3, 0xCC, 0xB8, 0x7E, 0x64, + 0xED, 0x99, 0xF3, 0x1D, 0xFC, 0x96, 0xA2, 0x0A, + 0x9C, 0xC2, 0x9B, 0xD7, 0x03, 0xA6, 0x79, 0x3B, + 0x16, 0x0C, 0x6C, 0x5C, 0x2B, 0x61, 0x0E, 0x48, + 0x96, 0x5C, 0x46, 0x7F, 0xC3, 0xCD, 0x3C, 0x10, + 0x30, 0x8F, 0xC4, 0xB5, 0x92, 0x46, 0x1C, 0xDF, + 0x10, 0xEE, 0x43, 0x27, 0x42, 0x70, 0xD2, 0xC4, + 0x5E, 0x77, 0x78, 0x0E, 0x0E, 0xC3, 0x8B, 0x72, + 0xA0, 0xFC, 0x4C, 0x0F, 0x5D, 0xBE, 0xBE, 0x07, + 0x5B, 0x53, 0x38, 0xC8, 0x96, 0x82, 0x2D, 0x2D, + 0x8E, 0xA8, 0x6C, 0x68, 0x34, 0x42, 0x31, 0x90, + 0xD6, 0x4D, 0x29, 0xA9, 0x90, 0x95, 0x19, 0xD6, + 0x8F, 0x2F, 0xF4, 0xD3, 0x71, 0x21, 0xB7, 0x7D, + 0x51, 0xA6, 0x15, 0xE5, 0xDA, 0x08, 0x6A, 0x23, + 0xDE, 0x6C, 0xBA, 0xCF, 0x84, 0xF1, 0x47, 0x25, + 0x4A, 0xF1, 0x2F, 0x24, 0xED, 0x3B, 0xED, 0xF0, + 0xA7, 0x48, 0xAE, 0x58, 0x7F, 0x0B, 0x3B, 0x78, + 0xCE, 0x94, 0x32, 0x82, 0x63, 0x22, 0x67, 0xAA, + 0x45, 0x37, 0xCC, 0x43, 0xD5, 0x10, 0x59, 0x5B, + 0x09, 0xC6, 0x1C, 0x32, 0xCD, 0x19, 0xA2, 0x3C, + 0x2B, 0x84, 0x03, 0xD5, 0x97, 0x20, 0xE7, 0xFB, + 0x2D, 0x0A, 0x3C, 0x5C, 0xFD, 0x39, 0x9C, 0xDE, + 0x02, 0x3D, 0xC7, 0xDD, 0x51, 0xDE, 0x99, 0xB3, + 0x65, 0x00, 0x60, 0xCF, 0xAE, 0xCD, 0xE2, 0x83, + 0xD5, 0x36, 0x2C, 0x89, 0x28, 0x6D, 0xC3, 0x6A, + 0x80, 0xCD, 0x1A, 0xC3, 0x75, 0x11, 0x7E, 0x65, + 0x2A, 0x44, 0x9D, 0xB5, 0x12, 0x2A, 0x78, 0xD0, + 0x4D, 0xF8, 0x5E, 0xBF, 0xEC, 0x6B, 0x60, 0xD2, + 0x89, 0x92, 0x5E, 0x17, 0xDA, 0x33, 0x83, 0xDB, + 0xED, 0xF4, 0x5E, 0x82, 0xE9, 0x04, 0xD7, 0xE0, + 0xA4, 0x1B, 0xFE, 0x32, 0x93, 0x05, 0x2C, 0xCF, + 0xA2, 0xAE, 0x83, 0xCA, 0x2F, 0x5E, 0x47, 0x1C, + 0x85, 0x0D, 0x01, 0xE5, 0x44, 0x3D, 0xE4, 0x58, + 0x8E, 0xC0, 0x46, 0x05, 0x95, 0xBE, 0x59, 0xED, + 0x0F, 0x7B, 0xA1, 0xF7, 0xDB, 0x2C, 0x79, 0x86, + 0xE9, 0x54, 0x98, 0xA6, 0x2A, 0xD0, 0xFE, 0xC9, + 0x59, 0x1D, 0x31, 0xC6, 0x27, 0x83, 0x2C, 0x12, + 0x9C, 0xE1, 0x43, 0x3C, 0xEC, 0x65, 0x3B, 0xEF, + 0xFD, 0x92, 0xBC, 0x0E, 0x38, 0xBA, 0x56, 0x1C, + 0xC0, 0x81, 0x9E, 0xBE, 0x76, 0x59, 0x88, 0xA4, + 0x0C, 0x6B, 0xD9, 0x7C, 0xD6, 0x8C, 0x32, 0xCD, + 0x3F, 0xB6, 0xEF, 0xBF, 0xA6, 0xC7, 0xC9, 0xD3, + 0x02, 0xB0, 0x3B, 0xFF, 0xFC, 0x4A, 0x97, 0x14, + 0xFF, 0xF2, 0x48, 0xFE, 0x1B, 0xCE, 0x7D, 0x24, + 0xA1, 0xD6, 0x03, 0xB0, 0x2F, 0xAA, 0xF7, 0x71, + 0xC9, 0x0E, 0xCB, 0x57, 0xBA, 0xEF, 0xB5, 0x65, + 0xE1, 0x44, 0xE4, 0x6A, 0xEB, 0xE8, 0x2B, 0x8F, + 0x06, 0x23, 0x7A, 0xA9, 0x70, 0xAE, 0x48, 0x65, + 0x94, 0xEE, 0xA5, 0x94, 0x78, 0x7D, 0x09, 0xF8, + 0xB5, 0x4D, 0x64, 0x67, 0x10, 0x16, 0xA2, 0xFC, + 0x49, 0x93, 0x76, 0x71, 0xED, 0x56, 0x25, 0xB5, + 0x87, 0xE8, 0x84, 0x16, 0x55, 0xE1, 0x1E, 0x34, + 0xE3, 0xB2, 0x49, 0x8F, 0xDC, 0xDA, 0xC3, 0x17, + 0x82, 0x0E, 0x19, 0xD7, 0xE0, 0x09, 0xD7, 0xD9, + 0x59, 0x6B, 0x55, 0x60, 0x1C, 0x1B, 0x02, 0xE8, + 0xD1, 0x90, 0xF6, 0x3E, 0x94, 0x4A, 0x12, 0x0C, + 0xBB, 0x69, 0xFD, 0x7C, 0xA0, 0xDD, 0x5F, 0x93, + 0x9F, 0xFE, 0x2E, 0x79, 0xDB, 0xBE, 0x6F, 0x85, + 0xAD, 0x9B, 0xDE, 0xAA, 0x10, 0xCA, 0xDB, 0xF2, + 0xF9, 0xD0, 0x54, 0x15, 0x00, 0xF0, 0x6F, 0x86, + 0x16, 0xF6, 0xA8, 0xA4, 0x08, 0x7B, 0x50, 0xF1, + 0x35, 0xAC, 0xB6, 0xBB, 0x8B, 0xA0, 0x86, 0x3B, + 0x3B, 0xDA, 0x9F, 0x89, 0xB5, 0x9C, 0x44, 0x41, + 0x6A, 0xFD, 0x8A, 0x79, 0xA0, 0xFB, 0x7D, 0x1B, + 0xE8, 0xC4, 0xA7, 0x3F, 0x66, 0x97, 0xA9, 0xF8, + 0xEA, 0x0C, 0x30, 0x81, 0x63, 0xE4, 0xE3, 0x84, + 0x62, 0xC5, 0x19, 0xFB, 0x00, 0xD6, 0x72, 0xE6, + 0xC9, 0x6C, 0xDB, 0xEB, 0xF3, 0x6F, 0xDB, 0xE7, + 0x00, 0x53, 0xCE, 0x1D, 0xE5, 0xF5, 0x53, 0x18, + 0xE5, 0xAA, 0xDA, 0x90, 0x7B, 0xCB, 0x2B, 0x74, + 0xED, 0x70, 0xFE, 0x90, 0xA8, 0xC8, 0x80, 0x2B, + 0x93, 0x08, 0xDB, 0x6A, 0x0F, 0x3D, 0xA1, 0xFA, + 0xB6, 0x63, 0x18, 0xF8, 0x43, 0x68, 0x00, 0xD0, + 0x7A, 0x97, 0xCD, 0x5B, 0xB2, 0x84, 0x90, 0x06, + 0xB9, 0x81, 0xC5, 0x81, 0x05, 0x55, 0x8C, 0xC4, + 0x03, 0x89, 0xF5, 0x63, 0x87, 0x39, 0xEC, 0xD6, + 0x89, 0x01, 0xE7, 0x1C, 0x4C, 0xDF, 0x5D, 0x65, + 0xFE, 0x4B, 0x91, 0x04, 0x5B, 0x0E, 0x03, 0x38, + 0x2F, 0x21, 0xA8, 0x36, 0x58, 0x93, 0xAD, 0x1F, + 0xEB, 0xC3, 0x91, 0x90, 0x9B, 0x95, 0xCD, 0x53, + 0x81, 0xAA, 0xA9, 0x48, 0x4D, 0x2B, 0x22, 0xC7, + 0xBE, 0x1B, 0x38, 0x21, 0xA1, 0xFE, 0x23, 0xB4, + 0xAC, 0x66, 0x92, 0x9E, 0xF2, 0x27, 0xDC, 0x23, + 0x70, 0x6E, 0xBA, 0xF9, 0xED, 0x3B, 0xCE, 0x63, + 0xAD, 0x68, 0xF2, 0x80, 0xFA, 0x1B, 0x14, 0xB5, + 0xB4, 0x07, 0xE3, 0x5A, 0x81, 0x74, 0xE1, 0xF2, + }, + .len = 3120 + }, + .auth_tag = { + .data = { + 0xEA, 0xE9, 0x10, 0xB6, 0xB7, 0xAB, 0xEA, 0x90, + 0x8A, 0xD5, 0x63, 0x88, 0xDB, 0x2B, 0x8F, 0x23, + }, + .len = 16 + } +}; + +/** AES-CCM-128 Test Vectors */ +static const struct aead_test_data ccm_test_case_128_1 = { + .algo = RTE_CRYPTO_AEAD_AES_CCM, + .key = { + .data = { + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, + 0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F + }, + .len = 16 + }, + .iv = { + .data = { + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16 + }, + .len = 7 + }, + .aad = { + .data = ccm_aad_test_1, + .len = 8 + }, + .plaintext = { + .data = { + 0x20, 0x21, 0x22, 0x23 + }, + .len = 4 + }, + .ciphertext = { + .data = { + 0x71, 0x62, 0x01, 0x5B + }, + .len = 4 + }, + .auth_tag = { + .data = { + 0x4D, 0xAC, 0x25, 0x5D + }, + .len = 4 + } +}; + +static const struct aead_test_data ccm_test_case_128_2 = { + .algo = RTE_CRYPTO_AEAD_AES_CCM, + .key = { + .data = { + 0xC9, 0x7C, 0x1F, 0x67, 0xCE, 0x37, 0x11, 0x85, + 0x51, 0x4A, 0x8A, 0x19, 0xF2, 0xBD, 0xD5, 0x2F + }, + .len = 16 + }, + .iv = { + .data = { + 0x00, 0x50, 0x30, 0xF1, 0x84, 0x44, 0x08, 0xB5, + 0x03, 0x97, 0x76, 0xE7, 0x0C + }, + .len = 13 + }, + .aad = { + .data = ccm_aad_test_2, + .len = 22 + }, + .plaintext = { + .data = { + 0xF8, 0xBA, 0x1A, 0x55, 0xD0, 0x2F, 0x85, 0xAE, + 0x96, 0x7B, 0xB6, 0x2F, 0xB6, 0xCD, 0xA8, 0xEB, + 0x7E, 0x78, 0xA0, 0x50 + }, + .len = 20 + }, + .ciphertext = { + .data = { + 0xF3, 0xD0, 0xA2, 0xFE, 0x9A, 0x3D, 0xBF, 0x23, + 0x42, 0xA6, 0x43, 0xE4, 0x32, 0x46, 0xE8, 0x0C, + 0x3C, 0x04, 0xD0, 0x19 + }, + .len = 20 + }, + .auth_tag = { + .data = { + 0x78, 0x45, 0xCE, 0x0B, 0x16, 0xF9, 0x76, 0x23 + }, + .len = 8 + } +}; + +static const struct aead_test_data ccm_test_case_128_3 = { + .algo = RTE_CRYPTO_AEAD_AES_CCM, + .key = { + .data = { + 0xC9, 0x7C, 0x1F, 0x67, 0xCE, 0x37, 0x11, 0x85, + 0x51, 0x4A, 0x8A, 0x19, 0xF2, 0xBD, 0xD5, 0x2F + }, + .len = 16 + }, + .iv = { + .data = { + 0x00, 0x50, 0x30, 0xF1, 0x84, 0x44, 0x08, 0xB5, + 0x03, 0x97, 0x76, 0xE7, 0x0C + }, + .len = 13 + }, + .aad = { + .data = gcm_aad_zero_text, + .len = 0 + }, + .plaintext = { + .data = { + 0xF8, 0xBA, 0x1A, 0x55, 0xD0, 0x2F, 0x85, 0xAE, + 0x96, 0x7B, 0xB6, 0x2F, 0xB6, 0xCD, 0xA8, 0xEB, + 0x7E, 0x78, 0xA0, 0x50 + }, + .len = 20 + }, + .ciphertext = { + .data = { + 0xF3, 0xD0, 0xA2, 0xFE, 0x9A, 0x3D, 0xBF, 0x23, + 0x42, 0xA6, 0x43, 0xE4, 0x32, 0x46, 0xE8, 0x0C, + 0x3C, 0x04, 0xD0, 0x19 + }, + .len = 20 + }, + .auth_tag = { + .data = { + 0x41, 0x83, 0x21, 0x89, 0xA3, 0xD3, 0x1B, 0x43 + }, + .len = 8 + } +}; + +/** AES-CCM-192 Test Vectors */ +static const struct aead_test_data ccm_test_case_192_1 = { + .algo = RTE_CRYPTO_AEAD_AES_CCM, + .key = { + .data = { + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, + 0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57 + }, + .len = 24 + }, + .iv = { + .data = { + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16 + }, + .len = 7 + }, + .aad = { + .data = ccm_aad_test_1, + .len = 8 + }, + .plaintext = { + .data = { + 0x20, 0x21, 0x22, 0x23 + }, + .len = 4 + }, + .ciphertext = { + .data = { + 0x18, 0xEE, 0x17, 0x30 + }, + .len = 4 + }, + .auth_tag = { + .data = { + 0xC8, 0xC3, 0x26, 0xD5 + }, + .len = 4 + } +}; + +static const struct aead_test_data ccm_test_case_192_2 = { + .algo = RTE_CRYPTO_AEAD_AES_CCM, + .key = { + .data = { + 0xC9, 0x7C, 0x1F, 0x67, 0xCE, 0x37, 0x11, 0x85, + 0x51, 0x4A, 0x8A, 0x19, 0xF2, 0xBD, 0xD5, 0x2F, + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57 + }, + .len = 24 + }, + .iv = { + .data = { + 0x00, 0x50, 0x30, 0xF1, 0x84, 0x44, 0x08, 0xB5, + 0x03, 0x97, 0x76, 0xE7, 0x0C + }, + .len = 13 + }, + .aad = { + .data = ccm_aad_test_2, + .len = 22 + }, + .plaintext = { + .data = { + 0xF8, 0xBA, 0x1A, 0x55, 0xD0, 0x2F, 0x85, 0xAE, + 0x96, 0x7B, 0xB6, 0x2F, 0xB6, 0xCD, 0xA8, 0xEB, + 0x7E, 0x78, 0xA0, 0x50 + }, + .len = 20 + }, + .ciphertext = { + .data = { + 0x41, 0xC6, 0x2D, 0xD5, 0x31, 0xF2, 0xD5, 0xC8, + 0xCC, 0x57, 0x01, 0x2E, 0x7E, 0x2B, 0xF1, 0x26, + 0x6A, 0xC7, 0xCB, 0xA5 + }, + .len = 20 + }, + .auth_tag = { + .data = { + 0x77, 0xA3, 0x41, 0xD5, 0x2A, 0xE3, 0x25, 0x37 + }, + .len = 8 + } +}; + +static const struct aead_test_data ccm_test_case_192_3 = { + .algo = RTE_CRYPTO_AEAD_AES_CCM, + .key = { + .data = { + 0xC9, 0x7C, 0x1F, 0x67, 0xCE, 0x37, 0x11, 0x85, + 0x51, 0x4A, 0x8A, 0x19, 0xF2, 0xBD, 0xD5, 0x2F, + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57 + }, + .len = 24 + }, + .iv = { + .data = { + 0x00, 0x50, 0x30, 0xF1, 0x84, 0x44, 0x08, 0xB5, + 0x03, 0x97, 0x76, 0xE7, 0x0C + }, + .len = 13 + }, + .aad = { + .data = gcm_aad_zero_text, + .len = 0 + }, + .plaintext = { + .data = { + 0xF8, 0xBA, 0x1A, 0x55, 0xD0, 0x2F, 0x85, 0xAE, + 0x96, 0x7B, 0xB6, 0x2F, 0xB6, 0xCD, 0xA8, 0xEB, + 0x7E, 0x78, 0xA0, 0x50 + }, + .len = 20 + }, + .ciphertext = { + .data = { + 0x41, 0xC6, 0x2D, 0xD5, 0x31, 0xF2, 0xD5, 0xC8, + 0xCC, 0x57, 0x01, 0x2E, 0x7E, 0x2B, 0xF1, 0x26, + 0x6A, 0xC7, 0xCB, 0xA5 + }, + .len = 20 + }, + .auth_tag = { + .data = { + 0x84, 0x72, 0x6E, 0xE7, 0x8E, 0x8E, 0x3A, 0xC6 + }, + .len = 8 + } +}; + +/** AES-CCM-256 Test Vectors */ +static const struct aead_test_data ccm_test_case_256_1 = { + .algo = RTE_CRYPTO_AEAD_AES_CCM, + .key = { + .data = { + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, + 0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, + 0x58, 0x59, 0x5A, 0x5B, 0x5C, 0x5D, 0x5E, 0x5F + }, + .len = 32 + }, + .iv = { + .data = { + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16 + }, + .len = 7 + }, + .aad = { + .data = ccm_aad_test_1, + .len = 8 + }, + .plaintext = { + .data = { + 0x20, 0x21, 0x22, 0x23 + }, + .len = 4 + }, + .ciphertext = { + .data = { + 0x8A, 0xB1, 0xA8, 0x74 + }, + .len = 4 + }, + .auth_tag = { + .data = { + 0x95, 0xFC, 0x08, 0x20 + }, + .len = 4 + } +}; + +static const struct aead_test_data ccm_test_case_256_2 = { + .algo = RTE_CRYPTO_AEAD_AES_CCM, + .key = { + .data = { + 0xC9, 0x7C, 0x1F, 0x67, 0xCE, 0x37, 0x11, 0x85, + 0x51, 0x4A, 0x8A, 0x19, 0xF2, 0xBD, 0xD5, 0x2F, + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, + 0x58, 0x59, 0x5A, 0x5B, 0x5C, 0x5D, 0x5E, 0x5F + }, + .len = 32 + }, + .iv = { + .data = { + 0x00, 0x50, 0x30, 0xF1, 0x84, 0x44, 0x08, 0xB5, + 0x03, 0x97, 0x76, 0xE7, 0x0C + }, + .len = 13 + }, + .aad = { + .data = ccm_aad_test_2, + .len = 22 + }, + .plaintext = { + .data = { + 0xF8, 0xBA, 0x1A, 0x55, 0xD0, 0x2F, 0x85, 0xAE, + 0x96, 0x7B, 0xB6, 0x2F, 0xB6, 0xCD, 0xA8, 0xEB, + 0x7E, 0x78, 0xA0, 0x50 + }, + .len = 20 + }, + .ciphertext = { + .data = { + 0x25, 0x82, 0x89, 0x09, 0x3E, 0x39, 0x1F, 0x16, + 0xD2, 0x82, 0x3D, 0xF6, 0xCE, 0x97, 0x72, 0x07, + 0xEC, 0x23, 0x17, 0x98 + }, + .len = 20 + }, + .auth_tag = { + .data = { + 0xAB, 0x02, 0xE9, 0xD1, 0x16, 0x69, 0xED, 0x0A + }, + .len = 8 + } +}; + +static const struct aead_test_data ccm_test_case_256_3 = { + .algo = RTE_CRYPTO_AEAD_AES_CCM, + .key = { + .data = { + 0xC9, 0x7C, 0x1F, 0x67, 0xCE, 0x37, 0x11, 0x85, + 0x51, 0x4A, 0x8A, 0x19, 0xF2, 0xBD, 0xD5, 0x2F, + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, + 0x58, 0x59, 0x5A, 0x5B, 0x5C, 0x5D, 0x5E, 0x5F + }, + .len = 32 + }, + .iv = { + .data = { + 0x00, 0x50, 0x30, 0xF1, 0x84, 0x44, 0x08, 0xB5, + 0x03, 0x97, 0x76, 0xE7, 0x0C + }, + .len = 13 + }, + .aad = { + .data = gcm_aad_zero_text, + .len = 0 + }, + .plaintext = { + .data = { + 0xF8, 0xBA, 0x1A, 0x55, 0xD0, 0x2F, 0x85, 0xAE, + 0x96, 0x7B, 0xB6, 0x2F, 0xB6, 0xCD, 0xA8, 0xEB, + 0x7E, 0x78, 0xA0, 0x50 + }, + .len = 20 + }, + .ciphertext = { + .data = { + 0x25, 0x82, 0x89, 0x09, 0x3E, 0x39, 0x1F, 0x16, + 0xD2, 0x82, 0x3D, 0xF6, 0xCE, 0x97, 0x72, 0x07, + 0xEC, 0x23, 0x17, 0x98 + }, + .len = 20 + }, + .auth_tag = { + .data = { + 0x15, 0x80, 0xC4, 0xC9, 0x3F, 0xAB, 0x2A, 0xFD + }, + .len = 8 + } +}; +#endif /* TEST_CRYPTODEV_AEAD_TEST_VECTORS_H_ */ diff --git a/dpdk/test/test/test_cryptodev_aes_test_vectors.h b/dpdk/test/test/test_cryptodev_aes_test_vectors.h new file mode 100644 index 00000000..9c13041d --- /dev/null +++ b/dpdk/test/test/test_cryptodev_aes_test_vectors.h @@ -0,0 +1,1809 @@ +/* + * BSD LICENSE + * + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef TEST_CRYPTODEV_AES_TEST_VECTORS_H_ +#define TEST_CRYPTODEV_AES_TEST_VECTORS_H_ + +/* test vectors */ +static const uint8_t plaintext_aes128ctr[] = { + 0x6B, 0xC1, 0xBE, 0xE2, 0x2E, 0x40, 0x9F, 0x96, + 0xE9, 0x3D, 0x7E, 0x11, 0x73, 0x93, 0x17, 0x2A, + 0xAE, 0x2D, 0x8A, 0x57, 0x1E, 0x03, 0xAC, 0x9C, + 0x9E, 0xB7, 0x6F, 0xAC, 0x45, 0xAF, 0x8E, 0x51, + 0x30, 0xC8, 0x1C, 0x46, 0xA3, 0x5C, 0xE4, 0x11, + 0xE5, 0xFB, 0xC1, 0x19, 0x1A, 0x0A, 0x52, 0xEF, + 0xF6, 0x9F, 0x24, 0x45, 0xDF, 0x4F, 0x9B, 0x17, + 0xAD, 0x2B, 0x41, 0x7B, 0xE6, 0x6C, 0x37, 0x10 +}; + +static const uint8_t ciphertext64_aes128ctr[] = { + 0x87, 0x4D, 0x61, 0x91, 0xB6, 0x20, 0xE3, 0x26, + 0x1B, 0xEF, 0x68, 0x64, 0x99, 0x0D, 0xB6, 0xCE, + 0x98, 0x06, 0xF6, 0x6B, 0x79, 0x70, 0xFD, 0xFF, + 0x86, 0x17, 0x18, 0x7B, 0xB9, 0xFF, 0xFD, 0xFF, + 0x5A, 0xE4, 0xDF, 0x3E, 0xDB, 0xD5, 0xD3, 0x5E, + 0x5B, 0x4F, 0x09, 0x02, 0x0D, 0xB0, 0x3E, 0xAB, + 0x1E, 0x03, 0x1D, 0xDA, 0x2F, 0xBE, 0x03, 0xD1, + 0x79, 0x21, 0x70, 0xA0, 0xF3, 0x00, 0x9C, 0xEE +}; + +static const uint8_t ciphertext64_aes128ctr_IV_12bytes[] = { + 0x28, 0x80, 0x28, 0xC7, 0x15, 0x99, 0xC5, 0xA8, + 0xDD, 0x53, 0xC2, 0x67, 0x1B, 0x86, 0xB8, 0x13, + 0xAB, 0x25, 0x39, 0x7A, 0xD2, 0x1F, 0x8B, 0x4B, + 0x94, 0x89, 0x2B, 0x65, 0xCF, 0x89, 0x1E, 0xDD, + 0xD4, 0x7C, 0xFD, 0x8D, 0x0E, 0xCD, 0x23, 0xA4, + 0xEB, 0x8C, 0x05, 0x58, 0x45, 0x4A, 0x63, 0x44, + 0x11, 0x42, 0x07, 0x17, 0xB4, 0xD2, 0xCC, 0x75, + 0xB7, 0x23, 0x99, 0xA9, 0xC5, 0x89, 0x7F, 0x66 +}; + +static const uint8_t plaintext_aes_docsis_bpi_cfb[] = { + 0x00, 0x01, 0x02, 0x88, 0xEE, 0x59, 0x7E +}; + +static const uint8_t ciphertext_aes_docsis_bpi_cfb[] = { + 0xFC, 0x68, 0xA3, 0x55, 0x60, 0x37, 0xDC +}; + +static const uint8_t plaintext_aes_docsis_bpi_cbc_cfb[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x91, + 0xD2, 0xD1, 0x9F +}; + +static const uint8_t ciphertext_aes_docsis_bpi_cbc_cfb[] = { + 0x9D, 0xD1, 0x67, 0x4B, 0xBA, 0x61, 0x10, 0x1B, + 0x56, 0x75, 0x64, 0x74, 0x36, 0x4F, 0x10, 0x1D, + 0x44, 0xD4, 0x73 +}; + +static const uint8_t plaintext_aes192ctr[] = { + 0x01, 0x0F, 0x10, 0x1F, 0x20, 0x1C, 0x0E, 0xB8, + 0xFB, 0x5C, 0xCD, 0xCC, 0x1F, 0xF9, 0xAF, 0x0B, + 0x95, 0x03, 0x74, 0x99, 0x49, 0xE7, 0x62, 0x55, + 0xDA, 0xEA, 0x13, 0x20, 0x1D, 0xC6, 0xCC, 0xCC, + 0xD1, 0x70, 0x75, 0x47, 0x02, 0x2F, 0xFB, 0x86, + 0xBB, 0x6B, 0x23, 0xD2, 0xC9, 0x74, 0xD7, 0x7B, + 0x08, 0x03, 0x3B, 0x79, 0x39, 0xBB, 0x91, 0x29, + 0xDA, 0x14, 0x39, 0x8D, 0xFF, 0x81, 0x50, 0x96, +}; + +static const uint8_t ciphertext64_aes192ctr[] = { + 0x4A, 0x6C, 0xC8, 0xCC, 0x96, 0x2A, 0x13, 0x84, + 0x1C, 0x36, 0x88, 0xE9, 0xE5, 0x94, 0x70, 0xB2, + 0x14, 0x5B, 0x13, 0x80, 0xEA, 0xD8, 0x8D, 0x37, + 0xFD, 0x70, 0xA8, 0x83, 0xE8, 0x2B, 0x88, 0x1E, + 0xBA, 0x94, 0x3F, 0xF6, 0xB3, 0x1F, 0xDE, 0x34, + 0xF3, 0x5B, 0x80, 0xE9, 0xAB, 0xF5, 0x1C, 0x29, + 0xB6, 0xD9, 0x76, 0x2B, 0x06, 0xC6, 0x74, 0xF1, + 0x59, 0x5E, 0x9E, 0xA5, 0x7B, 0x2D, 0xD7, 0xF0 +}; + +static const uint8_t ciphertext64_aes192ctr_IV_12bytes[] = { + 0x67, 0x65, 0xa9, 0xee, 0xfd, 0x31, 0x62, 0xfc, + 0xad, 0xfd, 0xc7, 0x25, 0xb7, 0x25, 0x16, 0xbe, + 0x25, 0xce, 0xc0, 0x1d, 0xda, 0xa9, 0xd3, 0xda, + 0x1b, 0x7d, 0x68, 0x6a, 0x6f, 0x06, 0xea, 0x47, + 0xa0, 0xe0, 0x15, 0xf4, 0xbd, 0x1b, 0x70, 0x34, + 0xd4, 0x6d, 0x1c, 0x84, 0x17, 0x91, 0x46, 0x0c, + 0xe8, 0xbc, 0x7a, 0xfb, 0x9f, 0x2a, 0x8f, 0xb4, + 0xd4, 0xf3, 0x6e, 0x5b, 0x75, 0xa0, 0xce, 0x32 +}; + +static const uint8_t plaintext_aes256ctr[] = { + 0x6B, 0xC1, 0xBE, 0xE2, 0x2E, 0x40, 0x9F, 0x96, + 0xE9, 0x3D, 0x7E, 0x11, 0x73, 0x93, 0x17, 0x2A, + 0xAE, 0x2D, 0x8A, 0x57, 0x1E, 0x03, 0xAC, 0x9C, + 0x9E, 0xB7, 0x6F, 0xAC, 0x45, 0xAF, 0x8E, 0x51, + 0x30, 0xC8, 0x1C, 0x46, 0xA3, 0x5C, 0xE4, 0x11, + 0xE5, 0xFB, 0xC1, 0x19, 0x1A, 0x0A, 0x52, 0xEF, + 0xF6, 0x9F, 0x24, 0x45, 0xDF, 0x4F, 0x9B, 0x17, + 0xAD, 0x2B, 0x41, 0x7B, 0xE6, 0x6C, 0x37, 0x10 +}; + +static const uint8_t ciphertext64_aes256ctr[] = { + 0x60, 0x1E, 0xC3, 0x13, 0x77, 0x57, 0x89, 0xA5, + 0xB7, 0xA7, 0xF5, 0x04, 0xBB, 0xF3, 0xD2, 0x28, + 0xF4, 0x43, 0xE3, 0xCA, 0x4D, 0x62, 0xB5, 0x9A, + 0xCA, 0x84, 0xE9, 0x90, 0xCA, 0xCA, 0xF5, 0xC5, + 0x2B, 0x09, 0x30, 0xDA, 0xA2, 0x3D, 0xE9, 0x4C, + 0xE8, 0x70, 0x17, 0xBA, 0x2D, 0x84, 0x98, 0x8D, + 0xDF, 0xC9, 0xC5, 0x8D, 0xB6, 0x7A, 0xAD, 0xA6, + 0x13, 0xC2, 0xDD, 0x08, 0x45, 0x79, 0x41, 0xA6 +}; + +static const uint8_t ciphertext64_aes256ctr_IV_12bytes[] = { + 0x7B, 0x7A, 0x7D, 0x83, 0x85, 0xF8, 0x81, 0xF3, + 0x32, 0x33, 0xD9, 0xFB, 0x04, 0x73, 0xD4, 0x2F, + 0x70, 0xDE, 0x90, 0x3E, 0xD0, 0xA9, 0x93, 0x8A, + 0x91, 0xF3, 0xB5, 0x29, 0x4D, 0x2A, 0x74, 0xD0, + 0xDC, 0x4E, 0x5C, 0x9B, 0x97, 0x24, 0xD8, 0x02, + 0xFE, 0xAB, 0x38, 0xE8, 0x73, 0x51, 0x29, 0x7E, + 0xF1, 0xF9, 0x40, 0x78, 0xB1, 0x04, 0x7A, 0x78, + 0x61, 0x07, 0x47, 0xE6, 0x8C, 0x0F, 0xA8, 0x76 +}; + +static const uint8_t plaintext_aes_common[] = { + "What a lousy earth! He wondered how many people " + "were destitute that same night even in his own " + "prosperous country, how many homes were " + "shanties, how many husbands were drunk and " + "wives socked, and how many children were " + "bullied, abused, or abandoned. How many " + "families hungered for food they could not " + "afford to buy? How many hearts were broken? How " + "many suicides would take place that same night, " + "how many people would go insane? How many " + "cockroaches and landlords would triumph? How " + "many winners were losers, successes failures, " + "and rich men poor men? How many wise guys were " + "stupid? How many happy endings were unhappy " + "endings? How many honest men were liars, brave " + "men cowards, loyal men traitors, how many " + "sainted men were corrupt, how many people in " + "positions of trust had sold their souls to " + "bodyguards, how many had never had souls? How " + "many straight-and-narrow paths were crooked " + "paths? How many best families were worst " + "families and how many good people were bad " + "people? When you added them all up and then " + "subtracted, you might be left with only the " + "children, and perhaps with Albert Einstein and " + "an old violinist or sculptor somewhere." +}; + +static const uint8_t ciphertext512_aes128cbc[] = { + 0x8B, 0x4D, 0xDA, 0x1B, 0xCF, 0x04, 0xA0, 0x31, + 0xB4, 0xBF, 0xBD, 0x68, 0x43, 0x20, 0x7E, 0x76, + 0xB1, 0x96, 0x8B, 0xA2, 0x7C, 0xA2, 0x83, 0x9E, + 0x39, 0x5A, 0x2F, 0x7E, 0x92, 0xB4, 0x48, 0x1A, + 0x3F, 0x6B, 0x5D, 0xDF, 0x52, 0x85, 0x5F, 0x8E, + 0x42, 0x3C, 0xFB, 0xE9, 0x1A, 0x24, 0xD6, 0x08, + 0xDD, 0xFD, 0x16, 0xFB, 0xE9, 0x55, 0xEF, 0xF0, + 0xA0, 0x8D, 0x13, 0xAB, 0x81, 0xC6, 0x90, 0x01, + 0xB5, 0x18, 0x84, 0xB3, 0xF6, 0xE6, 0x11, 0x57, + 0xD6, 0x71, 0xC6, 0x3C, 0x3F, 0x2F, 0x33, 0xEE, + 0x24, 0x42, 0x6E, 0xAC, 0x0B, 0xCA, 0xEC, 0xF9, + 0x84, 0xF8, 0x22, 0xAA, 0x60, 0xF0, 0x32, 0xA9, + 0x75, 0x75, 0x3B, 0xCB, 0x70, 0x21, 0x0A, 0x8D, + 0x0F, 0xE0, 0xC4, 0x78, 0x2B, 0xF8, 0x97, 0xE3, + 0xE4, 0x26, 0x4B, 0x29, 0xDA, 0x88, 0xCD, 0x46, + 0xEC, 0xAA, 0xF9, 0x7F, 0xF1, 0x15, 0xEA, 0xC3, + 0x87, 0xE6, 0x31, 0xF2, 0xCF, 0xDE, 0x4D, 0x80, + 0x70, 0x91, 0x7E, 0x0C, 0xF7, 0x26, 0x3A, 0x92, + 0x4F, 0x18, 0x83, 0xC0, 0x8F, 0x59, 0x01, 0xA5, + 0x88, 0xD1, 0xDB, 0x26, 0x71, 0x27, 0x16, 0xF5, + 0xEE, 0x10, 0x82, 0xAC, 0x68, 0x26, 0x9B, 0xE2, + 0x6D, 0xD8, 0x9A, 0x80, 0xDF, 0x04, 0x31, 0xD5, + 0xF1, 0x35, 0x5C, 0x3B, 0xDD, 0x9A, 0x65, 0xBA, + 0x58, 0x34, 0x85, 0x61, 0x1C, 0x42, 0x10, 0x76, + 0x73, 0x02, 0x42, 0xC9, 0x23, 0x18, 0x8E, 0xB4, + 0x6F, 0xB4, 0xA3, 0x54, 0x6E, 0x88, 0x3B, 0x62, + 0x7C, 0x02, 0x8D, 0x4C, 0x9F, 0xC8, 0x45, 0xF4, + 0xC9, 0xDE, 0x4F, 0xEB, 0x22, 0x83, 0x1B, 0xE4, + 0x49, 0x37, 0xE4, 0xAD, 0xE7, 0xCD, 0x21, 0x54, + 0xBC, 0x1C, 0xC2, 0x04, 0x97, 0xB4, 0x10, 0x61, + 0xF0, 0xE4, 0xEF, 0x27, 0x63, 0x3A, 0xDA, 0x91, + 0x41, 0x25, 0x62, 0x1C, 0x5C, 0xB6, 0x38, 0x4A, + 0x88, 0x71, 0x59, 0x5A, 0x8D, 0xA0, 0x09, 0xAF, + 0x72, 0x94, 0xD7, 0x79, 0x5C, 0x60, 0x7C, 0x8F, + 0x4C, 0xF5, 0xD9, 0xA1, 0x39, 0x6D, 0x81, 0x28, + 0xEF, 0x13, 0x28, 0xDF, 0xF5, 0x3E, 0xF7, 0x8E, + 0x09, 0x9C, 0x78, 0x18, 0x79, 0xB8, 0x68, 0xD7, + 0xA8, 0x29, 0x62, 0xAD, 0xDE, 0xE1, 0x61, 0x76, + 0x1B, 0x05, 0x16, 0xCD, 0xBF, 0x02, 0x8E, 0xA6, + 0x43, 0x6E, 0x92, 0x55, 0x4F, 0x60, 0x9C, 0x03, + 0xB8, 0x4F, 0xA3, 0x02, 0xAC, 0xA8, 0xA7, 0x0C, + 0x1E, 0xB5, 0x6B, 0xF8, 0xC8, 0x4D, 0xDE, 0xD2, + 0xB0, 0x29, 0x6E, 0x40, 0xE6, 0xD6, 0xC9, 0xE6, + 0xB9, 0x0F, 0xB6, 0x63, 0xF5, 0xAA, 0x2B, 0x96, + 0xA7, 0x16, 0xAC, 0x4E, 0x0A, 0x33, 0x1C, 0xA6, + 0xE6, 0xBD, 0x8A, 0xCF, 0x40, 0xA9, 0xB2, 0xFA, + 0x63, 0x27, 0xFD, 0x9B, 0xD9, 0xFC, 0xD5, 0x87, + 0x8D, 0x4C, 0xB6, 0xA4, 0xCB, 0xE7, 0x74, 0x55, + 0xF4, 0xFB, 0x41, 0x25, 0xB5, 0x4B, 0x0A, 0x1B, + 0xB1, 0xD6, 0xB7, 0xD9, 0x47, 0x2A, 0xC3, 0x98, + 0x6A, 0xC4, 0x03, 0x73, 0x1F, 0x93, 0x6E, 0x53, + 0x19, 0x25, 0x64, 0x15, 0x83, 0xF9, 0x73, 0x2A, + 0x74, 0xB4, 0x93, 0x69, 0xC4, 0x72, 0xFC, 0x26, + 0xA2, 0x9F, 0x43, 0x45, 0xDD, 0xB9, 0xEF, 0x36, + 0xC8, 0x3A, 0xCD, 0x99, 0x9B, 0x54, 0x1A, 0x36, + 0xC1, 0x59, 0xF8, 0x98, 0xA8, 0xCC, 0x28, 0x0D, + 0x73, 0x4C, 0xEE, 0x98, 0xCB, 0x7C, 0x58, 0x7E, + 0x20, 0x75, 0x1E, 0xB7, 0xC9, 0xF8, 0xF2, 0x0E, + 0x63, 0x9E, 0x05, 0x78, 0x1A, 0xB6, 0xA8, 0x7A, + 0xF9, 0x98, 0x6A, 0xA6, 0x46, 0x84, 0x2E, 0xF6, + 0x4B, 0xDC, 0x9B, 0x8F, 0x9B, 0x8F, 0xEE, 0xB4, + 0xAA, 0x3F, 0xEE, 0xC0, 0x37, 0x27, 0x76, 0xC7, + 0x95, 0xBB, 0x26, 0x74, 0x69, 0x12, 0x7F, 0xF1, + 0xBB, 0xFF, 0xAE, 0xB5, 0x99, 0x6E, 0xCB, 0x0C +}; + +/* AES128-CTR-SHA1 test vector */ +static const struct blockcipher_test_data aes_test_data_1 = { + .crypto_algo = RTE_CRYPTO_CIPHER_AES_CTR, + .cipher_key = { + .data = { + 0x2B, 0x7E, 0x15, 0x16, 0x28, 0xAE, 0xD2, 0xA6, + 0xAB, 0xF7, 0x15, 0x88, 0x09, 0xCF, 0x4F, 0x3C + }, + .len = 16 + }, + .iv = { + .data = { + 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, + 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF + }, + .len = 16 + }, + .plaintext = { + .data = plaintext_aes128ctr, + .len = 64 + }, + .ciphertext = { + .data = ciphertext64_aes128ctr, + .len = 64 + }, + .auth_algo = RTE_CRYPTO_AUTH_SHA1_HMAC, + .auth_key = { + .data = { + 0xF8, 0x2A, 0xC7, 0x54, 0xDB, 0x96, 0x18, 0xAA, + 0xC3, 0xA1, 0x53, 0xF6, 0x1F, 0x17, 0x60, 0xBD, + 0xDE, 0xF4, 0xDE, 0xAD + }, + .len = 20 + }, + .digest = { + .data = { + 0x9B, 0x6F, 0x0C, 0x43, 0xF5, 0xC1, 0x3E, 0xB0, + 0xB1, 0x70, 0xB8, 0x2B, 0x33, 0x09, 0xD2, 0xB2, + 0x56, 0x20, 0xFB, 0xFE + }, + .len = 20, + .truncated_len = 12 + } +}; + +/** AES-192-CTR XCBC test vector */ +static const struct blockcipher_test_data aes_test_data_2 = { + .crypto_algo = RTE_CRYPTO_CIPHER_AES_CTR, + .cipher_key = { + .data = { + 0xCB, 0xC5, 0xED, 0x5B, 0xE7, 0x7C, 0xBD, 0x8C, + 0x50, 0xD9, 0x30, 0xF2, 0xB5, 0x6A, 0x0E, 0x5F, + 0xAA, 0xAE, 0xAD, 0xA2, 0x1F, 0x49, 0x52, 0xD4 + }, + .len = 24 + }, + .iv = { + .data = { + 0x3F, 0x69, 0xA8, 0xCD, 0xE8, 0xF0, 0xEF, 0x40, + 0xB8, 0x7A, 0x4B, 0xED, 0x2B, 0xAF, 0xBF, 0x57 + }, + .len = 16 + }, + .plaintext = { + .data = plaintext_aes192ctr, + .len = 64 + }, + .ciphertext = { + .data = ciphertext64_aes192ctr, + .len = 64 + }, + .auth_algo = RTE_CRYPTO_AUTH_AES_XCBC_MAC, + .auth_key = { + .data = { + 0x87, 0x61, 0x54, 0x53, 0xC4, 0x6D, 0xDD, 0x51, + 0xE1, 0x9F, 0x86, 0x64, 0x39, 0x0A, 0xE6, 0x59 + }, + .len = 16 + }, + .digest = { + .data = { + 0xCA, 0x33, 0xB3, 0x3B, 0x16, 0x94, 0xAA, 0x55, + 0x36, 0x6B, 0x45, 0x46 + }, + .len = 12, + .truncated_len = 12 + } +}; + +/** AES-256-CTR SHA1 test vector */ +static const struct blockcipher_test_data aes_test_data_3 = { + .crypto_algo = RTE_CRYPTO_CIPHER_AES_CTR, + .cipher_key = { + .data = { + 0x60, 0x3D, 0xEB, 0x10, 0x15, 0xCA, 0x71, 0xBE, + 0x2B, 0x73, 0xAE, 0xF0, 0x85, 0x7D, 0x77, 0x81, + 0x1F, 0x35, 0x2C, 0x07, 0x3B, 0x61, 0x08, 0xD7, + 0x2D, 0x98, 0x10, 0xA3, 0x09, 0x14, 0xDF, 0xF4 + }, + .len = 32 + }, + .iv = { + .data = { + 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, + 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF + }, + .len = 16 + }, + .plaintext = { + .data = plaintext_aes256ctr, + .len = 64 + }, + .ciphertext = { + .data = ciphertext64_aes256ctr, + .len = 64 + }, + .auth_algo = RTE_CRYPTO_AUTH_SHA1_HMAC, + .auth_key = { + .data = { + 0xF8, 0x2A, 0xC7, 0x54, 0xDB, 0x96, 0x18, 0xAA, + 0xC3, 0xA1, 0x53, 0xF6, 0x1F, 0x17, 0x60, 0xBD, + 0xDE, 0xF4, 0xDE, 0xAD + }, + .len = 20 + }, + .digest = { + .data = { + 0x3B, 0x1A, 0x9D, 0x82, 0x35, 0xD5, 0xDD, 0x64, + 0xCC, 0x1B, 0xA9, 0xC0, 0xEB, 0xE9, 0x42, 0x16, + 0xE7, 0x87, 0xA3, 0xEF + }, + .len = 20, + .truncated_len = 12 + } +}; + +/* AES128-CTR-SHA1 test vector (12-byte IV) */ +static const struct blockcipher_test_data aes_test_data_1_IV_12_bytes = { + .crypto_algo = RTE_CRYPTO_CIPHER_AES_CTR, + .cipher_key = { + .data = { + 0x2B, 0x7E, 0x15, 0x16, 0x28, 0xAE, 0xD2, 0xA6, + 0xAB, 0xF7, 0x15, 0x88, 0x09, 0xCF, 0x4F, 0x3C + }, + .len = 16 + }, + .iv = { + .data = { + 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, + 0xF8, 0xF9, 0xFA, 0xFB + }, + .len = 12 + }, + .plaintext = { + .data = plaintext_aes128ctr, + .len = 64 + }, + .ciphertext = { + .data = ciphertext64_aes128ctr_IV_12bytes, + .len = 64 + }, + .auth_algo = RTE_CRYPTO_AUTH_SHA1_HMAC, + .auth_key = { + .data = { + 0xF8, 0x2A, 0xC7, 0x54, 0xDB, 0x96, 0x18, 0xAA, + 0xC3, 0xA1, 0x53, 0xF6, 0x1F, 0x17, 0x60, 0xBD, + 0xDE, 0xF4, 0xDE, 0xAD + }, + .len = 20 + }, + .digest = { + .data = { + 0x5C, 0x34, 0x6B, 0xE4, 0x9A, 0x7F, 0x4A, 0xC3, + 0x82, 0xBE, 0xA0, 0x12, 0xD1, 0xF0, 0x15, 0xFA, + 0xCF, 0xC8, 0x7F, 0x60 + }, + .len = 20, + .truncated_len = 12 + } +}; + +/** AES-192-CTR XCBC test vector (12-byte IV) */ +static const struct blockcipher_test_data aes_test_data_2_IV_12_bytes = { + .crypto_algo = RTE_CRYPTO_CIPHER_AES_CTR, + .cipher_key = { + .data = { + 0xCB, 0xC5, 0xED, 0x5B, 0xE7, 0x7C, 0xBD, 0x8C, + 0x50, 0xD9, 0x30, 0xF2, 0xB5, 0x6A, 0x0E, 0x5F, + 0xAA, 0xAE, 0xAD, 0xA2, 0x1F, 0x49, 0x52, 0xD4 + }, + .len = 24 + }, + .iv = { + .data = { + 0x3F, 0x69, 0xA8, 0xCD, 0xE8, 0xF0, 0xEF, 0x40, + 0xB8, 0x7A, 0x4B, 0xED + }, + .len = 12 + }, + .plaintext = { + .data = plaintext_aes192ctr, + .len = 64 + }, + .ciphertext = { + .data = ciphertext64_aes192ctr_IV_12bytes, + .len = 64 + }, + .auth_algo = RTE_CRYPTO_AUTH_AES_XCBC_MAC, + .auth_key = { + .data = { + 0x87, 0x61, 0x54, 0x53, 0xC4, 0x6D, 0xDD, 0x51, + 0xE1, 0x9F, 0x86, 0x64, 0x39, 0x0A, 0xE6, 0x59 + }, + .len = 16 + }, + .digest = { + .data = { + 0x0C, 0xA1, 0xA5, 0xAF, 0x3E, 0x41, 0xD2, 0xF4, + 0x4C, 0x4C, 0xAB, 0x13 + }, + .len = 12, + .truncated_len = 12 + } +}; + +/** AES-256-CTR SHA1 test vector (12-byte IV) */ +static const struct blockcipher_test_data aes_test_data_3_IV_12_bytes = { + .crypto_algo = RTE_CRYPTO_CIPHER_AES_CTR, + .cipher_key = { + .data = { + 0x60, 0x3D, 0xEB, 0x10, 0x15, 0xCA, 0x71, 0xBE, + 0x2B, 0x73, 0xAE, 0xF0, 0x85, 0x7D, 0x77, 0x81, + 0x1F, 0x35, 0x2C, 0x07, 0x3B, 0x61, 0x08, 0xD7, + 0x2D, 0x98, 0x10, 0xA3, 0x09, 0x14, 0xDF, 0xF4 + }, + .len = 32 + }, + .iv = { + .data = { + 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, + 0xF8, 0xF9, 0xFA, 0xFB + }, + .len = 12 + }, + .plaintext = { + .data = plaintext_aes256ctr, + .len = 64 + }, + .ciphertext = { + .data = ciphertext64_aes256ctr_IV_12bytes, + .len = 64 + }, + .auth_algo = RTE_CRYPTO_AUTH_SHA1_HMAC, + .auth_key = { + .data = { + 0xF8, 0x2A, 0xC7, 0x54, 0xDB, 0x96, 0x18, 0xAA, + 0xC3, 0xA1, 0x53, 0xF6, 0x1F, 0x17, 0x60, 0xBD, + 0xDE, 0xF4, 0xDE, 0xAD + }, + .len = 20 + }, + .digest = { + .data = { + 0x57, 0x9A, 0x52, 0x6E, 0x31, 0x17, 0x57, 0x49, + 0xE7, 0xA1, 0x88, 0x6C, 0x2E, 0x36, 0x67, 0x63, + 0x3F, 0x2D, 0xA3, 0xEF + }, + .len = 20, + .truncated_len = 12 + } +}; +/** AES-128-CBC SHA1 test vector */ +static const struct blockcipher_test_data aes_test_data_4 = { + .crypto_algo = RTE_CRYPTO_CIPHER_AES_CBC, + .cipher_key = { + .data = { + 0xE4, 0x23, 0x33, 0x8A, 0x35, 0x64, 0x61, 0xE2, + 0x49, 0x03, 0xDD, 0xC6, 0xB8, 0xCA, 0x55, 0x7A + }, + .len = 16 + }, + .iv = { + .data = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F + }, + .len = 16 + }, + .plaintext = { + .data = plaintext_aes_common, + .len = 512 + }, + .ciphertext = { + .data = ciphertext512_aes128cbc, + .len = 512 + }, + .auth_algo = RTE_CRYPTO_AUTH_SHA1_HMAC, + .auth_key = { + .data = { + 0xF8, 0x2A, 0xC7, 0x54, 0xDB, 0x96, 0x18, 0xAA, + 0xC3, 0xA1, 0x53, 0xF6, 0x1F, 0x17, 0x60, 0xBD, + 0xDE, 0xF4, 0xDE, 0xAD + }, + .len = 20 + }, + .digest = { + .data = { + 0x9A, 0x4F, 0x88, 0x1B, 0xB6, 0x8F, 0xD8, 0x60, + 0x42, 0x1A, 0x7D, 0x3D, 0xF5, 0x82, 0x80, 0xF1, + 0x18, 0x8C, 0x1D, 0x32 + }, + .len = 20, + .truncated_len = 12 + } +}; + +/** AES-128-CBC SHA256 test vector */ +static const struct blockcipher_test_data aes_test_data_5 = { + .crypto_algo = RTE_CRYPTO_CIPHER_AES_CBC, + .cipher_key = { + .data = { + 0xE4, 0x23, 0x33, 0x8A, 0x35, 0x64, 0x61, 0xE2, + 0x49, 0x03, 0xDD, 0xC6, 0xB8, 0xCA, 0x55, 0x7A + }, + .len = 16 + }, + .iv = { + .data = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F + }, + .len = 16 + }, + .plaintext = { + .data = plaintext_aes_common, + .len = 512 + }, + .ciphertext = { + .data = ciphertext512_aes128cbc, + .len = 512 + }, + .auth_algo = RTE_CRYPTO_AUTH_SHA256_HMAC, + .auth_key = { + .data = { + 0x42, 0x1A, 0x7D, 0x3D, 0xF5, 0x82, 0x80, 0xF1, + 0xF1, 0x35, 0x5C, 0x3B, 0xDD, 0x9A, 0x65, 0xBA, + 0x58, 0x34, 0x85, 0x61, 0x1C, 0x42, 0x10, 0x76, + 0x9A, 0x4F, 0x88, 0x1B, 0xB6, 0x8F, 0xD8, 0x60 + }, + .len = 32 + }, + .digest = { + .data = { + 0xC8, 0x57, 0x57, 0x31, 0x03, 0xE0, 0x03, 0x55, + 0x07, 0xC8, 0x9E, 0x7F, 0x48, 0x9A, 0x61, 0x9A, + 0x68, 0xEE, 0x03, 0x0E, 0x71, 0x75, 0xC7, 0xF4, + 0x2E, 0x45, 0x26, 0x32, 0x7C, 0x12, 0x15, 0x15 + }, + .len = 32, + .truncated_len = 16 + } +}; + +/** AES-128-CBC SHA512 test vector */ +static const struct blockcipher_test_data aes_test_data_6 = { + .crypto_algo = RTE_CRYPTO_CIPHER_AES_CBC, + .cipher_key = { + .data = { + 0xE4, 0x23, 0x33, 0x8A, 0x35, 0x64, 0x61, 0xE2, + 0x49, 0x03, 0xDD, 0xC6, 0xB8, 0xCA, 0x55, 0x7A + }, + .len = 16 + }, + .iv = { + .data = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F + }, + .len = 16 + }, + .plaintext = { + .data = plaintext_aes_common, + .len = 512 + }, + .ciphertext = { + .data = ciphertext512_aes128cbc, + .len = 512 + }, + .auth_algo = RTE_CRYPTO_AUTH_SHA512_HMAC, + .auth_key = { + .data = { + 0x42, 0x1A, 0x7D, 0x3D, 0xF5, 0x82, 0x80, 0xF1, + 0xF1, 0x35, 0x5C, 0x3B, 0xDD, 0x9A, 0x65, 0xBA, + 0x58, 0x34, 0x85, 0x65, 0x1C, 0x42, 0x50, 0x76, + 0x9A, 0xAF, 0x88, 0x1B, 0xB6, 0x8F, 0xF8, 0x60, + 0xA2, 0x5A, 0x7F, 0x3F, 0xF4, 0x72, 0x70, 0xF1, + 0xF5, 0x35, 0x4C, 0x3B, 0xDD, 0x90, 0x65, 0xB0, + 0x47, 0x3A, 0x75, 0x61, 0x5C, 0xA2, 0x10, 0x76, + 0x9A, 0xAF, 0x77, 0x5B, 0xB6, 0x7F, 0xF7, 0x60 + }, + .len = 64 + }, + .digest = { + .data = { + 0x5D, 0x54, 0x66, 0xC1, 0x6E, 0xBC, 0x04, 0xB8, + 0x46, 0xB8, 0x08, 0x6E, 0xE0, 0xF0, 0x43, 0x48, + 0x37, 0x96, 0x9C, 0xC6, 0x9C, 0xC2, 0x1E, 0xE8, + 0xF2, 0x0C, 0x0B, 0xEF, 0x86, 0xA2, 0xE3, 0x70, + 0x95, 0xC8, 0xB3, 0x06, 0x47, 0xA9, 0x90, 0xE8, + 0xA0, 0xC6, 0x72, 0x69, 0x05, 0xC0, 0x0D, 0x0E, + 0x21, 0x96, 0x65, 0x93, 0x74, 0x43, 0x2A, 0x1D, + 0x2E, 0xBF, 0xC2, 0xC2, 0xEE, 0xCC, 0x2F, 0x0A + }, + .len = 64, + .truncated_len = 32 + } +}; + +/** AES-128-CBC XCBC test vector */ +static const struct blockcipher_test_data aes_test_data_7 = { + .crypto_algo = RTE_CRYPTO_CIPHER_AES_CBC, + .cipher_key = { + .data = { + 0xE4, 0x23, 0x33, 0x8A, 0x35, 0x64, 0x61, 0xE2, + 0x49, 0x03, 0xDD, 0xC6, 0xB8, 0xCA, 0x55, 0x7A + }, + .len = 16 + }, + .iv = { + .data = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F + }, + .len = 16 + }, + .plaintext = { + .data = plaintext_aes_common, + .len = 512 + }, + .ciphertext = { + .data = ciphertext512_aes128cbc, + .len = 512 + }, + .auth_algo = RTE_CRYPTO_AUTH_AES_XCBC_MAC, + .auth_key = { + .data = { + 0x87, 0x61, 0x54, 0x53, 0xC4, 0x6D, 0xDD, 0x51, + 0xE1, 0x9F, 0x86, 0x64, 0x39, 0x0A, 0xE6, 0x59 + }, + .len = 16 + }, + .digest = { + .data = { + 0xE0, 0xAC, 0x9A, 0xC4, 0x22, 0x64, 0x35, 0x89, + 0x77, 0x1D, 0x8B, 0x75 + }, + .len = 12, + .truncated_len = 12 + } +}; + +/** AES-128-CBC SHA224 test vector */ +static const struct blockcipher_test_data aes_test_data_8 = { + .crypto_algo = RTE_CRYPTO_CIPHER_AES_CBC, + .cipher_key = { + .data = { + 0xE4, 0x23, 0x33, 0x8A, 0x35, 0x64, 0x61, 0xE2, + 0x49, 0x03, 0xDD, 0xC6, 0xB8, 0xCA, 0x55, 0x7A + }, + .len = 16 + }, + .iv = { + .data = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F + }, + .len = 16 + }, + .plaintext = { + .data = plaintext_aes_common, + .len = 512 + }, + .ciphertext = { + .data = ciphertext512_aes128cbc, + .len = 512 + }, + .auth_algo = RTE_CRYPTO_AUTH_SHA224_HMAC, + .auth_key = { + .data = { + 0x42, 0x1A, 0x7D, 0x3D, 0xF5, 0x82, 0x80, 0xF1, + 0xF1, 0x35, 0x5C, 0x3B, 0xDD, 0x9A, 0x65, 0xBA, + 0x58, 0x34, 0x85, 0x65, 0x1C, 0x42, 0x50, 0x76, + 0x9A, 0xAF, 0x88, 0x1B, 0xB6, 0x8F, 0xF8, 0x60, + 0xA2, 0x5A, 0x7F, 0x3F, 0xF4, 0x72, 0x70, 0xF1, + 0xF5, 0x35, 0x4C, 0x3B, 0xDD, 0x90, 0x65, 0xB0, + 0x47, 0x3A, 0x75, 0x61, 0x5C, 0xA2, 0x10, 0x76, + 0x9A, 0xAF, 0x77, 0x5B, 0xB6, 0x7F, 0xF7, 0x60 + }, + .len = 64 + }, + .digest = { + .data = { + 0xA3, 0xCA, 0xC7, 0x1D, 0xA8, 0x61, 0x30, 0x98, + 0x3B, 0x8F, 0x01, 0x19, 0xAE, 0x8D, 0xBD, 0x34, + 0x40, 0x63, 0xA8, 0x2F, 0xDF, 0x85, 0x2B, 0x7F, + 0x63, 0x7C, 0xDD, 0xB7 + }, + .len = 28, + .truncated_len = 14 + } +}; + +/** AES-128-CBC SHA384 test vector */ +static const struct blockcipher_test_data aes_test_data_9 = { + .crypto_algo = RTE_CRYPTO_CIPHER_AES_CBC, + .cipher_key = { + .data = { + 0xE4, 0x23, 0x33, 0x8A, 0x35, 0x64, 0x61, 0xE2, + 0x49, 0x03, 0xDD, 0xC6, 0xB8, 0xCA, 0x55, 0x7A + }, + .len = 16 + }, + .iv = { + .data = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F + }, + .len = 16 + }, + .plaintext = { + .data = plaintext_aes_common, + .len = 512 + }, + .ciphertext = { + .data = ciphertext512_aes128cbc, + .len = 512 + }, + .auth_algo = RTE_CRYPTO_AUTH_SHA384_HMAC, + .auth_key = { + .data = { + 0x42, 0x1A, 0x7D, 0x3D, 0xF5, 0x82, 0x80, 0xF1, + 0xF1, 0x35, 0x5C, 0x3B, 0xDD, 0x9A, 0x65, 0xBA, + 0x58, 0x34, 0x85, 0x65, 0x1C, 0x42, 0x50, 0x76, + 0x9A, 0xAF, 0x88, 0x1B, 0xB6, 0x8F, 0xF8, 0x60, + 0xA2, 0x5A, 0x7F, 0x3F, 0xF4, 0x72, 0x70, 0xF1, + 0xF5, 0x35, 0x4C, 0x3B, 0xDD, 0x90, 0x65, 0xB0, + 0x47, 0x3A, 0x75, 0x61, 0x5C, 0xA2, 0x10, 0x76, + 0x9A, 0xAF, 0x77, 0x5B, 0xB6, 0x7F, 0xF7, 0x60, + 0x42, 0x1A, 0x7D, 0x3D, 0xF5, 0x82, 0x80, 0xF1, + 0xF1, 0x35, 0x5C, 0x3B, 0xDD, 0x9A, 0x65, 0xBA, + 0x58, 0x34, 0x85, 0x65, 0x1C, 0x42, 0x50, 0x76, + 0x9A, 0xAF, 0x88, 0x1B, 0xB6, 0x8F, 0xF8, 0x60, + 0xA2, 0x5A, 0x7F, 0x3F, 0xF4, 0x72, 0x70, 0xF1, + 0xF5, 0x35, 0x4C, 0x3B, 0xDD, 0x90, 0x65, 0xB0, + 0x47, 0x3A, 0x75, 0x61, 0x5C, 0xA2, 0x10, 0x76, + 0x9A, 0xAF, 0x77, 0x5B, 0xB6, 0x7F, 0xF7, 0x60 + }, + .len = 128 + }, + .digest = { + .data = { + 0x23, 0x60, 0xC8, 0xB1, 0x2D, 0x6C, 0x1E, 0x72, + 0x25, 0xAB, 0xF9, 0xC3, 0x9A, 0xA9, 0x4F, 0x8C, + 0x56, 0x38, 0x65, 0x0E, 0x74, 0xD5, 0x45, 0x9D, + 0xA3, 0xFD, 0x7E, 0x6D, 0x9E, 0x74, 0x88, 0x9D, + 0xA7, 0x12, 0x9D, 0xD8, 0x81, 0x3C, 0x86, 0x2F, + 0x4D, 0xF9, 0x6F, 0x0A, 0xB0, 0xC9, 0xEB, 0x0B + }, + .len = 48, + .truncated_len = 24 + } +}; + +static const uint8_t ciphertext512_aes192cbc[] = { + 0x45, 0xEE, 0x9A, 0xEA, 0x3C, 0x03, 0xFC, 0x4C, + 0x84, 0x36, 0xB0, 0xDA, 0xB0, 0xDC, 0xF3, 0x5B, + 0x75, 0xA7, 0xBE, 0x0E, 0xC0, 0x8D, 0x6C, 0xF8, + 0xC1, 0x0F, 0xD0, 0x35, 0x1D, 0x82, 0xAE, 0x7C, + 0x57, 0xC5, 0x7A, 0x55, 0x87, 0x1B, 0xD4, 0x03, + 0x0A, 0x64, 0xC9, 0xE0, 0xF4, 0xC7, 0x6F, 0x57, + 0x52, 0xC6, 0x73, 0xBA, 0x84, 0x0B, 0x5B, 0x89, + 0x21, 0xD2, 0x9B, 0x88, 0x68, 0xF5, 0xA9, 0x7F, + 0x3F, 0x49, 0xEB, 0xF4, 0xD4, 0x52, 0xD2, 0x64, + 0x80, 0xB2, 0x53, 0xDA, 0x19, 0xF6, 0x10, 0x24, + 0x23, 0x26, 0x7A, 0x7C, 0x07, 0x57, 0x4B, 0x0E, + 0x58, 0x49, 0x61, 0xD1, 0xDC, 0x9A, 0x32, 0x6B, + 0x0F, 0x43, 0x9E, 0x4D, 0xB4, 0x07, 0x4E, 0xB3, + 0x51, 0x74, 0xDE, 0x29, 0xBC, 0x98, 0xF9, 0xDF, + 0x78, 0x9A, 0x18, 0x9C, 0xD6, 0x7A, 0x55, 0x7C, + 0xE6, 0x1D, 0x5C, 0x1A, 0x99, 0xD2, 0xC3, 0x7B, + 0x9F, 0x96, 0x74, 0x2D, 0xE0, 0xEF, 0xD1, 0xE3, + 0x08, 0x9F, 0xAF, 0xE6, 0xED, 0xCA, 0xE1, 0xEA, + 0x23, 0x6F, 0x7C, 0x81, 0xA8, 0xC0, 0x5B, 0x8B, + 0x53, 0x90, 0x51, 0x2D, 0x0F, 0xF6, 0x7D, 0xA7, + 0x1C, 0xBD, 0x83, 0x84, 0x54, 0xA4, 0x15, 0xFB, + 0x3E, 0x25, 0xA7, 0x3A, 0x0A, 0x73, 0xD9, 0x88, + 0x6F, 0x80, 0x78, 0x95, 0x7F, 0x60, 0xAA, 0x86, + 0x8A, 0xFC, 0xDF, 0xC1, 0xCB, 0xDE, 0xBB, 0x25, + 0x52, 0x20, 0xC6, 0x79, 0xD4, 0x0F, 0x25, 0xE7, + 0xDB, 0xB2, 0x17, 0xA4, 0x6F, 0x3C, 0x6F, 0x91, + 0xF6, 0x44, 0x1E, 0xB6, 0x85, 0xBC, 0x7A, 0x14, + 0x10, 0x72, 0xBD, 0x16, 0x63, 0x39, 0x9E, 0x7B, + 0x84, 0x5B, 0x17, 0x61, 0xB1, 0x5D, 0x82, 0x0B, + 0x6D, 0x37, 0xD7, 0x79, 0xB8, 0x24, 0x91, 0x30, + 0x82, 0x91, 0x02, 0xB1, 0x18, 0x4B, 0xE0, 0xF4, + 0x13, 0x1B, 0xB2, 0x4C, 0xDA, 0xB8, 0x99, 0x96, + 0x83, 0x2F, 0xBE, 0x53, 0x8D, 0xDE, 0xFA, 0xAD, + 0xF6, 0x5C, 0xDB, 0xE5, 0x66, 0x26, 0x8F, 0x13, + 0x2B, 0x76, 0x47, 0x73, 0xDE, 0x1A, 0x74, 0xA6, + 0x30, 0xAF, 0x42, 0xA0, 0xE5, 0xD2, 0x8F, 0xC2, + 0xED, 0x3E, 0x9E, 0x29, 0x54, 0x3C, 0xDE, 0x9F, + 0x5D, 0x30, 0x2B, 0x63, 0xFB, 0xE3, 0xB1, 0x07, + 0xEE, 0x74, 0x4A, 0xAF, 0xB1, 0x20, 0x8D, 0xEC, + 0xE6, 0x78, 0x16, 0x8D, 0xA4, 0x6E, 0x34, 0x7D, + 0x47, 0xFB, 0x0B, 0xC1, 0x32, 0xD7, 0x0D, 0x6C, + 0x6F, 0x93, 0x9C, 0x5E, 0xEF, 0x1F, 0x9C, 0x45, + 0x80, 0x6B, 0x74, 0xA6, 0x81, 0xF2, 0xF6, 0xFA, + 0xAA, 0x9D, 0x4F, 0xCA, 0xB5, 0x90, 0x59, 0xB0, + 0x3B, 0xF2, 0xF0, 0x75, 0xFD, 0x8A, 0xD8, 0x97, + 0x65, 0x88, 0x56, 0x4C, 0x44, 0xDF, 0x73, 0xF7, + 0x56, 0x9C, 0x48, 0x7E, 0xB0, 0x1F, 0x1D, 0x7D, + 0x6A, 0x11, 0xF5, 0xC2, 0xF4, 0x17, 0xEF, 0x58, + 0xD8, 0x2A, 0xAF, 0x56, 0x2F, 0xCF, 0xEC, 0xA4, + 0x58, 0x8B, 0x60, 0xCE, 0xD4, 0x0F, 0x9C, 0x21, + 0xEC, 0x3E, 0x74, 0x7B, 0x81, 0x3D, 0x69, 0xC6, + 0x5E, 0x12, 0x83, 0xE9, 0xEF, 0x81, 0x58, 0x36, + 0x6A, 0x60, 0x0F, 0x54, 0x28, 0x11, 0xF9, 0x64, + 0x36, 0xAD, 0x79, 0xF5, 0x1C, 0x74, 0xD0, 0xC3, + 0x7B, 0x61, 0xE1, 0x92, 0xB0, 0x13, 0x91, 0x87, + 0x32, 0x1F, 0xF2, 0x5A, 0xDA, 0x25, 0x69, 0xEB, + 0xD7, 0x32, 0x7F, 0xF5, 0x23, 0x21, 0x54, 0x47, + 0x7B, 0x1B, 0x33, 0xB0, 0x3D, 0xF6, 0xE2, 0x7E, + 0x3E, 0xA2, 0x9E, 0xCA, 0x48, 0x0B, 0x4A, 0x29, + 0x81, 0xD4, 0x4E, 0xD5, 0x69, 0xFB, 0xCD, 0x37, + 0x8A, 0xC1, 0x5B, 0x50, 0xFF, 0xB5, 0x7D, 0x43, + 0x0F, 0xAE, 0xA6, 0xC2, 0xE5, 0x8F, 0x45, 0xB2, + 0x85, 0x99, 0x02, 0xA2, 0x9B, 0xBE, 0x90, 0x43, + 0x4F, 0x2F, 0x50, 0xE2, 0x77, 0x62, 0xD9, 0xCC +}; + +/** AES-192-CBC test vector */ +static const struct blockcipher_test_data aes_test_data_10 = { + .crypto_algo = RTE_CRYPTO_CIPHER_AES_CBC, + .cipher_key = { + .data = { + 0xE4, 0x23, 0x33, 0x8A, 0x35, 0x64, 0x61, 0xE2, + 0x49, 0x03, 0xDD, 0xC6, 0xB8, 0xCA, 0x55, 0x7A, + 0xD4, 0xC3, 0xA3, 0xAA, 0x33, 0x62, 0x61, 0xE0 + }, + .len = 24 + }, + .iv = { + .data = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F + }, + .len = 16 + }, + .plaintext = { + .data = plaintext_aes_common, + .len = 512 + }, + .ciphertext = { + .data = ciphertext512_aes192cbc, + .len = 512 + } +}; + +static const uint8_t ciphertext512_aes256cbc[] = { + 0xF3, 0xDD, 0xF0, 0x0B, 0xFF, 0xA2, 0x6A, 0x04, + 0xBE, 0xDA, 0x52, 0xA6, 0xFE, 0x6B, 0xA6, 0xA7, + 0x48, 0x1D, 0x7D, 0x98, 0x65, 0xDB, 0xEF, 0x06, + 0x26, 0xB5, 0x8E, 0xEB, 0x05, 0x0E, 0x77, 0x98, + 0x17, 0x8E, 0xD0, 0xD4, 0x7B, 0x92, 0x8F, 0x5C, + 0xD0, 0x74, 0x5C, 0xA8, 0x4B, 0x54, 0xB6, 0x2F, + 0x83, 0x72, 0x2C, 0xFF, 0x72, 0xE9, 0xE4, 0x15, + 0x4C, 0x32, 0xAF, 0xC8, 0xC9, 0x89, 0x3C, 0x6E, + 0x31, 0xD5, 0xC0, 0x16, 0xC0, 0x31, 0x7D, 0x11, + 0xAB, 0xCB, 0xDE, 0xD2, 0xD6, 0xAA, 0x76, 0x5E, + 0xBA, 0xF6, 0xE2, 0x92, 0xCB, 0x86, 0x07, 0xFA, + 0xD4, 0x9E, 0x83, 0xED, 0xFD, 0xB8, 0x70, 0x54, + 0x6B, 0xBE, 0xEC, 0x72, 0xDD, 0x28, 0x5E, 0x95, + 0x78, 0xA5, 0x28, 0x43, 0x3D, 0x6D, 0xB1, 0xD9, + 0x69, 0x1F, 0xC9, 0x66, 0x0E, 0x32, 0x44, 0x08, + 0xD2, 0xAE, 0x2C, 0x43, 0xF2, 0xD0, 0x7D, 0x26, + 0x70, 0xE5, 0xA1, 0xCA, 0x37, 0xE9, 0x7D, 0xC7, + 0xA3, 0xFA, 0x81, 0x91, 0x64, 0xAA, 0x64, 0x91, + 0x9A, 0x95, 0x2D, 0xC9, 0xF9, 0xCE, 0xFE, 0x9F, + 0xC4, 0xD8, 0x81, 0xBE, 0x57, 0x84, 0xC5, 0x02, + 0xDB, 0x30, 0xC1, 0xD9, 0x0E, 0xA0, 0xA6, 0x00, + 0xD6, 0xF3, 0x52, 0x7E, 0x0D, 0x23, 0x6B, 0x2B, + 0x34, 0x99, 0x1F, 0x70, 0x27, 0x6D, 0x58, 0x84, + 0x93, 0x77, 0xB8, 0x3E, 0xF1, 0x71, 0x58, 0x42, + 0x8B, 0x2B, 0xC8, 0x6D, 0x05, 0x84, 0xFF, 0x4E, + 0x85, 0xEF, 0x4A, 0x9D, 0x91, 0x6A, 0xD5, 0xE1, + 0xAF, 0x01, 0xEB, 0x83, 0x8F, 0x23, 0x7C, 0x7F, + 0x12, 0x91, 0x05, 0xF0, 0x4E, 0xD9, 0x17, 0x62, + 0x75, 0xBB, 0xAC, 0x97, 0xEE, 0x3B, 0x4E, 0xC7, + 0xE5, 0x92, 0xF8, 0x9D, 0x4C, 0xF9, 0xEE, 0x55, + 0x18, 0xBB, 0xCC, 0xB4, 0xF2, 0x59, 0xB9, 0xFC, + 0x7A, 0x0F, 0x98, 0xD4, 0x8B, 0xFE, 0xF7, 0x83, + 0x46, 0xE2, 0x83, 0x33, 0x3E, 0x95, 0x8D, 0x17, + 0x1E, 0x85, 0xF8, 0x8C, 0x51, 0xB0, 0x6C, 0xB5, + 0x5E, 0x95, 0xBA, 0x4B, 0x69, 0x1B, 0x48, 0x69, + 0x0B, 0x8F, 0xA5, 0x18, 0x13, 0xB9, 0x77, 0xD1, + 0x80, 0x32, 0x32, 0x6D, 0x53, 0xA1, 0x95, 0x40, + 0x96, 0x8A, 0xCC, 0xA3, 0x69, 0xF8, 0x9F, 0xB5, + 0x8E, 0xD2, 0x68, 0x07, 0x4F, 0xA7, 0xEC, 0xF8, + 0x20, 0x21, 0x58, 0xF8, 0xD8, 0x9E, 0x5F, 0x40, + 0xBA, 0xB9, 0x76, 0x57, 0x3B, 0x17, 0xAD, 0xEE, + 0xCB, 0xDF, 0x07, 0xC1, 0xDF, 0x66, 0xA8, 0x0D, + 0xC2, 0xCE, 0x8F, 0x79, 0xC3, 0x32, 0xE0, 0x8C, + 0xFE, 0x5A, 0xF3, 0x55, 0x27, 0x73, 0x6F, 0xA1, + 0x54, 0xC6, 0xFC, 0x28, 0x9D, 0xBE, 0x97, 0xB9, + 0x54, 0x97, 0x72, 0x3A, 0x61, 0xAF, 0x6F, 0xDE, + 0xF8, 0x0E, 0xBB, 0x6B, 0x96, 0x84, 0xDD, 0x9B, + 0x62, 0xBA, 0x47, 0xB5, 0xC9, 0x3B, 0x4E, 0x8C, + 0x78, 0x2A, 0xCC, 0x0A, 0x69, 0x54, 0x25, 0x5E, + 0x8B, 0xAC, 0x56, 0xD9, 0xFE, 0x48, 0xBA, 0xCE, + 0xA9, 0xCE, 0xA6, 0x1D, 0xBF, 0x3E, 0x3C, 0x66, + 0x40, 0x71, 0x79, 0xAD, 0x5B, 0x26, 0xAD, 0xBE, + 0x58, 0x13, 0x64, 0x60, 0x7C, 0x05, 0xFC, 0xE3, + 0x51, 0x7A, 0xF2, 0xCC, 0x54, 0x16, 0x2C, 0xA4, + 0xCE, 0x5F, 0x59, 0x12, 0x77, 0xEB, 0xD9, 0x23, + 0xE3, 0x86, 0xFB, 0xD7, 0x48, 0x76, 0x9D, 0xE3, + 0x89, 0x87, 0x39, 0xFA, 0x7B, 0x21, 0x0B, 0x76, + 0xB2, 0xED, 0x1C, 0x27, 0x4B, 0xD5, 0x27, 0x05, + 0x8C, 0x7D, 0x58, 0x6C, 0xCA, 0xA5, 0x54, 0x9A, + 0x0F, 0xCB, 0xE9, 0x88, 0x31, 0xAD, 0x49, 0xEE, + 0x38, 0xFB, 0xC9, 0xFB, 0xB4, 0x7A, 0x00, 0x58, + 0x20, 0x32, 0xD3, 0x53, 0x5A, 0xDD, 0x74, 0x95, + 0x60, 0x59, 0x09, 0xAE, 0x7E, 0xEC, 0x74, 0xA3, + 0xB7, 0x1C, 0x6D, 0xF2, 0xAE, 0x79, 0xA4, 0x7C +}; + +/** AES-256-CBC test vector */ +static const struct blockcipher_test_data aes_test_data_11 = { + .crypto_algo = RTE_CRYPTO_CIPHER_AES_CBC, + .cipher_key = { + .data = { + 0xE4, 0x23, 0x33, 0x8A, 0x35, 0x64, 0x61, 0xE2, + 0x49, 0x03, 0xDD, 0xC6, 0xB8, 0xCA, 0x55, 0x7A, + 0xD4, 0xC3, 0xA3, 0xAA, 0x33, 0x62, 0x61, 0xE0, + 0x37, 0x07, 0xB8, 0x23, 0xA2, 0xA3, 0xB5, 0x8D + }, + .len = 32 + }, + .iv = { + .data = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F + }, + .len = 16 + }, + .plaintext = { + .data = plaintext_aes_common, + .len = 512 + }, + .ciphertext = { + .data = ciphertext512_aes256cbc, + .len = 512 + } +}; + +/** AES-128-CBC SHA256 HMAC test vector (160 bytes) */ +static const struct blockcipher_test_data aes_test_data_12 = { + .crypto_algo = RTE_CRYPTO_CIPHER_AES_CBC, + .cipher_key = { + .data = { + 0xE4, 0x23, 0x33, 0x8A, 0x35, 0x64, 0x61, 0xE2, + 0x49, 0x03, 0xDD, 0xC6, 0xB8, 0xCA, 0x55, 0x7A + }, + .len = 16 + }, + .iv = { + .data = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F + }, + .len = 16 + }, + .plaintext = { + .data = plaintext_aes_common, + .len = 160 + }, + .ciphertext = { + .data = ciphertext512_aes128cbc, + .len = 160 + }, + .auth_algo = RTE_CRYPTO_AUTH_SHA256_HMAC, + .auth_key = { + .data = { + 0x42, 0x1A, 0x7D, 0x3D, 0xF5, 0x82, 0x80, 0xF1, + 0xF1, 0x35, 0x5C, 0x3B, 0xDD, 0x9A, 0x65, 0xBA, + 0x58, 0x34, 0x85, 0x61, 0x1C, 0x42, 0x10, 0x76, + 0x9A, 0x4F, 0x88, 0x1B, 0xB6, 0x8F, 0xD8, 0x60 + }, + .len = 32 + }, + .digest = { + .data = { + 0x92, 0xEC, 0x65, 0x9A, 0x52, 0xCC, 0x50, 0xA5, + 0xEE, 0x0E, 0xDF, 0x1E, 0xA4, 0xC9, 0xC1, 0x04, + 0xD5, 0xDC, 0x78, 0x90, 0xF4, 0xE3, 0x35, 0x62, + 0xAD, 0x95, 0x45, 0x28, 0x5C, 0xF8, 0x8C, 0x0B + }, + .len = 32, + .truncated_len = 16 + } +}; + +/** AES-128-CBC SHA1 HMAC test vector (160 bytes) */ +static const struct blockcipher_test_data aes_test_data_13 = { + .crypto_algo = RTE_CRYPTO_CIPHER_AES_CBC, + .cipher_key = { + .data = { + 0xE4, 0x23, 0x33, 0x8A, 0x35, 0x64, 0x61, 0xE2, + 0x49, 0x03, 0xDD, 0xC6, 0xB8, 0xCA, 0x55, 0x7A + }, + .len = 16 + }, + .iv = { + .data = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F + }, + .len = 16 + }, + .plaintext = { + .data = plaintext_aes_common, + .len = 160 + }, + .ciphertext = { + .data = ciphertext512_aes128cbc, + .len = 160 + }, + .auth_algo = RTE_CRYPTO_AUTH_SHA1_HMAC, + .auth_key = { + .data = { + 0xF8, 0x2A, 0xC7, 0x54, 0xDB, 0x96, 0x18, 0xAA, + 0xC3, 0xA1, 0x53, 0xF6, 0x1F, 0x17, 0x60, 0xBD, + 0xDE, 0xF4, 0xDE, 0xAD + }, + .len = 20 + }, + .digest = { + .data = { + 0x4F, 0x16, 0xEA, 0xF7, 0x4A, 0x88, 0xD3, 0xE0, + 0x0E, 0x12, 0x8B, 0xE7, 0x05, 0xD0, 0x86, 0x48, + 0x22, 0x43, 0x30, 0xA7 + }, + .len = 20, + .truncated_len = 12 + } +}; + +/* AES-DOCSIS-BPI test vectors */ + +/* Multiple of AES block size */ +static const struct blockcipher_test_data aes_test_data_docsis_1 = { + .crypto_algo = RTE_CRYPTO_CIPHER_AES_DOCSISBPI, + .cipher_key = { + .data = { + 0xE4, 0x23, 0x33, 0x8A, 0x35, 0x64, 0x61, 0xE2, + 0x49, 0x03, 0xDD, 0xC6, 0xB8, 0xCA, 0x55, 0x7A + }, + .len = 16 + }, + .iv = { + .data = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F + }, + .len = 16 + }, + .plaintext = { + .data = plaintext_aes_common, + .len = 512 + }, + .ciphertext = { + .data = ciphertext512_aes128cbc, + .len = 512 + } +}; + +/* Less than AES block size */ +static const struct blockcipher_test_data aes_test_data_docsis_2 = { + .crypto_algo = RTE_CRYPTO_CIPHER_AES_DOCSISBPI, + .cipher_key = { + .data = { + 0xE6, 0x60, 0x0F, 0xD8, 0x85, 0x2E, 0xF5, 0xAB, + 0xE6, 0x60, 0x0F, 0xD8, 0x85, 0x2E, 0xF5, 0xAB + }, + .len = 16 + }, + .iv = { + .data = { + 0x81, 0x0E, 0x52, 0x8E, 0x1C, 0x5F, 0xDA, 0x1A, + 0x81, 0x0E, 0x52, 0x8E, 0x1C, 0x5F, 0xDA, 0x1A + }, + .len = 16 + }, + .plaintext = { + .data = plaintext_aes_docsis_bpi_cfb, + .len = 7 + }, + .ciphertext = { + .data = ciphertext_aes_docsis_bpi_cfb, + .len = 7 + } +}; + +/* Not multiple of AES block size */ +static const struct blockcipher_test_data aes_test_data_docsis_3 = { + .crypto_algo = RTE_CRYPTO_CIPHER_AES_DOCSISBPI, + .cipher_key = { + .data = { + 0xE6, 0x60, 0x0F, 0xD8, 0x85, 0x2E, 0xF5, 0xAB, + 0xE6, 0x60, 0x0F, 0xD8, 0x85, 0x2E, 0xF5, 0xAB + }, + .len = 16 + }, + .iv = { + .data = { + 0x81, 0x0E, 0x52, 0x8E, 0x1C, 0x5F, 0xDA, 0x1A, + 0x81, 0x0E, 0x52, 0x8E, 0x1C, 0x5F, 0xDA, 0x1A + }, + .len = 16 + }, + .plaintext = { + .data = plaintext_aes_docsis_bpi_cbc_cfb, + .len = 19 + }, + .ciphertext = { + .data = ciphertext_aes_docsis_bpi_cbc_cfb, + .len = 19 + } +}; + +static const struct blockcipher_test_case aes_chain_test_cases[] = { + { + .test_descr = "AES-128-CTR HMAC-SHA1 Encryption Digest", + .test_data = &aes_test_data_1, + .op_mask = BLOCKCIPHER_TEST_OP_ENC_AUTH_GEN, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_MB | + BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT | + BLOCKCIPHER_TEST_TARGET_PMD_SCHEDULER | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_MRVL + }, + { + .test_descr = "AES-128-CTR HMAC-SHA1 Decryption Digest " + "Verify", + .test_data = &aes_test_data_1, + .op_mask = BLOCKCIPHER_TEST_OP_AUTH_VERIFY_DEC, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_MB | + BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT | + BLOCKCIPHER_TEST_TARGET_PMD_SCHEDULER | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_MRVL + }, + { + .test_descr = "AES-192-CTR XCBC Encryption Digest", + .test_data = &aes_test_data_2, + .op_mask = BLOCKCIPHER_TEST_OP_ENC_AUTH_GEN, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_MB | + BLOCKCIPHER_TEST_TARGET_PMD_QAT | + BLOCKCIPHER_TEST_TARGET_PMD_SCHEDULER + }, + { + .test_descr = "AES-192-CTR XCBC Decryption Digest Verify", + .test_data = &aes_test_data_2, + .op_mask = BLOCKCIPHER_TEST_OP_AUTH_VERIFY_DEC, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_MB | + BLOCKCIPHER_TEST_TARGET_PMD_QAT | + BLOCKCIPHER_TEST_TARGET_PMD_SCHEDULER + }, + { + .test_descr = "AES-192-CTR XCBC Decryption Digest Verify " + "Scatter Gather", + .test_data = &aes_test_data_2, + .op_mask = BLOCKCIPHER_TEST_OP_AUTH_VERIFY_DEC, + .feature_mask = BLOCKCIPHER_TEST_FEATURE_SG | + BLOCKCIPHER_TEST_FEATURE_OOP, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_MB | + BLOCKCIPHER_TEST_TARGET_PMD_QAT | + BLOCKCIPHER_TEST_TARGET_PMD_SCHEDULER + }, + { + .test_descr = "AES-256-CTR HMAC-SHA1 Encryption Digest", + .test_data = &aes_test_data_3, + .op_mask = BLOCKCIPHER_TEST_OP_ENC_AUTH_GEN, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_MB | + BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT | + BLOCKCIPHER_TEST_TARGET_PMD_SCHEDULER | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_MRVL + }, + { + .test_descr = "AES-256-CTR HMAC-SHA1 Decryption Digest " + "Verify", + .test_data = &aes_test_data_3, + .op_mask = BLOCKCIPHER_TEST_OP_AUTH_VERIFY_DEC, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_MB | + BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT | + BLOCKCIPHER_TEST_TARGET_PMD_SCHEDULER | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_MRVL + }, + { + .test_descr = "AES-128-CBC HMAC-SHA1 Encryption Digest", + .test_data = &aes_test_data_4, + .op_mask = BLOCKCIPHER_TEST_OP_ENC_AUTH_GEN, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_ARMV8 | + BLOCKCIPHER_TEST_TARGET_PMD_MB | + BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT | + BLOCKCIPHER_TEST_TARGET_PMD_SCHEDULER | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_MRVL + }, + { + .test_descr = "AES-128-CBC HMAC-SHA1 Encryption Digest " + "(short buffers)", + .test_data = &aes_test_data_13, + .op_mask = BLOCKCIPHER_TEST_OP_ENC_AUTH_GEN, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_ARMV8 | + BLOCKCIPHER_TEST_TARGET_PMD_MRVL + }, + { + .test_descr = "AES-128-CBC HMAC-SHA1 Encryption Digest " + "Scatter Gather", + .test_data = &aes_test_data_4, + .op_mask = BLOCKCIPHER_TEST_OP_ENC_AUTH_GEN, + .feature_mask = BLOCKCIPHER_TEST_FEATURE_SG | + BLOCKCIPHER_TEST_FEATURE_OOP, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_MB | + BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT | + BLOCKCIPHER_TEST_TARGET_PMD_SCHEDULER + }, + { + .test_descr = "AES-128-CBC HMAC-SHA1 Decryption Digest " + "Verify", + .test_data = &aes_test_data_4, + .op_mask = BLOCKCIPHER_TEST_OP_AUTH_VERIFY_DEC, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_ARMV8 | + BLOCKCIPHER_TEST_TARGET_PMD_MB | + BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT | + BLOCKCIPHER_TEST_TARGET_PMD_SCHEDULER | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_MRVL + }, + { + .test_descr = "AES-128-CBC HMAC-SHA1 Decryption Digest " + "Verify (short buffers)", + .test_data = &aes_test_data_13, + .op_mask = BLOCKCIPHER_TEST_OP_AUTH_VERIFY_DEC, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_ARMV8 | + BLOCKCIPHER_TEST_TARGET_PMD_MRVL + }, + { + .test_descr = "AES-128-CBC HMAC-SHA256 Encryption Digest", + .test_data = &aes_test_data_5, + .op_mask = BLOCKCIPHER_TEST_OP_ENC_AUTH_GEN, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_ARMV8 | + BLOCKCIPHER_TEST_TARGET_PMD_MB | + BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT | + BLOCKCIPHER_TEST_TARGET_PMD_SCHEDULER | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_MRVL + }, + { + .test_descr = "AES-128-CBC HMAC-SHA256 Encryption Digest " + "(short buffers)", + .test_data = &aes_test_data_12, + .op_mask = BLOCKCIPHER_TEST_OP_ENC_AUTH_GEN, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_ARMV8 | + BLOCKCIPHER_TEST_TARGET_PMD_MRVL + }, + { + .test_descr = "AES-128-CBC HMAC-SHA256 Decryption Digest " + "Verify", + .test_data = &aes_test_data_5, + .op_mask = BLOCKCIPHER_TEST_OP_AUTH_VERIFY_DEC, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_ARMV8 | + BLOCKCIPHER_TEST_TARGET_PMD_MB | + BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT | + BLOCKCIPHER_TEST_TARGET_PMD_SCHEDULER | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_MRVL + }, + { + .test_descr = "AES-128-CBC HMAC-SHA256 Decryption Digest " + "Verify (short buffers)", + .test_data = &aes_test_data_12, + .op_mask = BLOCKCIPHER_TEST_OP_AUTH_VERIFY_DEC, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_ARMV8 | + BLOCKCIPHER_TEST_TARGET_PMD_MRVL + }, + { + .test_descr = "AES-128-CBC HMAC-SHA512 Encryption Digest", + .test_data = &aes_test_data_6, + .op_mask = BLOCKCIPHER_TEST_OP_ENC_AUTH_GEN, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_MB | + BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT | + BLOCKCIPHER_TEST_TARGET_PMD_SCHEDULER | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_MRVL + }, + { + .test_descr = "AES-128-CBC HMAC-SHA512 Encryption Digest " + "Sessionless", + .test_data = &aes_test_data_6, + .op_mask = BLOCKCIPHER_TEST_OP_ENC_AUTH_GEN, + .feature_mask = BLOCKCIPHER_TEST_FEATURE_SESSIONLESS, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_MB | + BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL + }, + { + .test_descr = "AES-128-CBC HMAC-SHA512 Encryption Digest " + "Scatter Gather Sessionless", + .test_data = &aes_test_data_6, + .op_mask = BLOCKCIPHER_TEST_OP_ENC_AUTH_GEN, + .feature_mask = BLOCKCIPHER_TEST_FEATURE_SESSIONLESS | + BLOCKCIPHER_TEST_FEATURE_SG | + BLOCKCIPHER_TEST_FEATURE_OOP, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_MB | + BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL + }, + { + .test_descr = "AES-128-CBC HMAC-SHA512 Decryption Digest " + "Verify", + .test_data = &aes_test_data_6, + .op_mask = BLOCKCIPHER_TEST_OP_AUTH_VERIFY_DEC, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_MB | + BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT | + BLOCKCIPHER_TEST_TARGET_PMD_SCHEDULER | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_MRVL + }, + { + .test_descr = "AES-128-CBC HMAC-SHA512 Decryption Digest " + "Verify Scatter Gather", + .test_data = &aes_test_data_6, + .op_mask = BLOCKCIPHER_TEST_OP_AUTH_VERIFY_DEC, + .feature_mask = BLOCKCIPHER_TEST_FEATURE_SG | + BLOCKCIPHER_TEST_FEATURE_OOP, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_MB | + BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT | + BLOCKCIPHER_TEST_TARGET_PMD_SCHEDULER + }, + { + .test_descr = "AES-128-CBC XCBC Encryption Digest", + .test_data = &aes_test_data_7, + .op_mask = BLOCKCIPHER_TEST_OP_ENC_AUTH_GEN, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_MB | + BLOCKCIPHER_TEST_TARGET_PMD_QAT | + BLOCKCIPHER_TEST_TARGET_PMD_SCHEDULER + }, + { + .test_descr = "AES-128-CBC XCBC Decryption Digest Verify", + .test_data = &aes_test_data_7, + .op_mask = BLOCKCIPHER_TEST_OP_AUTH_VERIFY_DEC, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_MB | + BLOCKCIPHER_TEST_TARGET_PMD_QAT | + BLOCKCIPHER_TEST_TARGET_PMD_SCHEDULER + }, + { + .test_descr = "AES-128-CBC HMAC-SHA1 Encryption Digest " + "OOP", + .test_data = &aes_test_data_4, + .op_mask = BLOCKCIPHER_TEST_OP_ENC_AUTH_GEN, + .feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_ARMV8 | + BLOCKCIPHER_TEST_TARGET_PMD_QAT | + BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC + }, + { + .test_descr = "AES-128-CBC HMAC-SHA1 Decryption Digest " + "Verify OOP", + .test_data = &aes_test_data_4, + .op_mask = BLOCKCIPHER_TEST_OP_AUTH_VERIFY_DEC, + .feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_ARMV8 | + BLOCKCIPHER_TEST_TARGET_PMD_QAT | + BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC + }, + { + .test_descr = "AES-128-CBC HMAC-SHA224 Encryption Digest", + .test_data = &aes_test_data_8, + .op_mask = BLOCKCIPHER_TEST_OP_ENC_AUTH_GEN, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_MB | + BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT | + BLOCKCIPHER_TEST_TARGET_PMD_SCHEDULER | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC + }, + { + .test_descr = "AES-128-CBC HMAC-SHA224 Decryption Digest " + "Verify", + .test_data = &aes_test_data_8, + .op_mask = BLOCKCIPHER_TEST_OP_AUTH_VERIFY_DEC, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_MB | + BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT | + BLOCKCIPHER_TEST_TARGET_PMD_SCHEDULER | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC + }, + { + .test_descr = "AES-128-CBC HMAC-SHA384 Encryption Digest", + .test_data = &aes_test_data_9, + .op_mask = BLOCKCIPHER_TEST_OP_ENC_AUTH_GEN, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_MB | + BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT | + BLOCKCIPHER_TEST_TARGET_PMD_SCHEDULER | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_MRVL + }, + { + .test_descr = "AES-128-CBC HMAC-SHA384 Decryption Digest " + "Verify", + .test_data = &aes_test_data_9, + .op_mask = BLOCKCIPHER_TEST_OP_AUTH_VERIFY_DEC, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_MB | + BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT | + BLOCKCIPHER_TEST_TARGET_PMD_SCHEDULER | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_MRVL + }, + { + .test_descr = "AES-128-CBC HMAC-SHA1 Encryption Digest " + "Sessionless", + .test_data = &aes_test_data_4, + .op_mask = BLOCKCIPHER_TEST_OP_ENC_AUTH_GEN, + .feature_mask = BLOCKCIPHER_TEST_FEATURE_SESSIONLESS, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_ARMV8 | + BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL + }, + { + .test_descr = + "AES-128-CBC HMAC-SHA1 Decryption Digest " + "Verify Sessionless", + .test_data = &aes_test_data_4, + .op_mask = BLOCKCIPHER_TEST_OP_AUTH_VERIFY_DEC, + .feature_mask = BLOCKCIPHER_TEST_FEATURE_SESSIONLESS, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_ARMV8 | + BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL + }, +}; + +static const struct blockcipher_test_case aes_cipheronly_test_cases[] = { + { + .test_descr = "AES-128-CBC Encryption", + .test_data = &aes_test_data_4, + .op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT | + BLOCKCIPHER_TEST_TARGET_PMD_MB | + BLOCKCIPHER_TEST_TARGET_PMD_SCHEDULER | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_MRVL + }, + { + .test_descr = "AES-128-CBC Decryption", + .test_data = &aes_test_data_4, + .op_mask = BLOCKCIPHER_TEST_OP_DECRYPT, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT | + BLOCKCIPHER_TEST_TARGET_PMD_MB | + BLOCKCIPHER_TEST_TARGET_PMD_SCHEDULER | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_MRVL + }, + { + .test_descr = "AES-192-CBC Encryption", + .test_data = &aes_test_data_10, + .op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT | + BLOCKCIPHER_TEST_TARGET_PMD_MB | + BLOCKCIPHER_TEST_TARGET_PMD_SCHEDULER | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC + }, + { + .test_descr = "AES-192-CBC Encryption Scater gather", + .test_data = &aes_test_data_10, + .op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT, + .feature_mask = BLOCKCIPHER_TEST_FEATURE_SG | + BLOCKCIPHER_TEST_FEATURE_OOP, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL + }, + { + .test_descr = "AES-192-CBC Decryption", + .test_data = &aes_test_data_10, + .op_mask = BLOCKCIPHER_TEST_OP_DECRYPT, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT | + BLOCKCIPHER_TEST_TARGET_PMD_MB | + BLOCKCIPHER_TEST_TARGET_PMD_SCHEDULER | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC + }, + { + .test_descr = "AES-256-CBC Encryption", + .test_data = &aes_test_data_11, + .op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT | + BLOCKCIPHER_TEST_TARGET_PMD_MB | + BLOCKCIPHER_TEST_TARGET_PMD_SCHEDULER | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_MRVL + }, + { + .test_descr = "AES-256-CBC Decryption", + .test_data = &aes_test_data_11, + .op_mask = BLOCKCIPHER_TEST_OP_DECRYPT, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT | + BLOCKCIPHER_TEST_TARGET_PMD_MB | + BLOCKCIPHER_TEST_TARGET_PMD_SCHEDULER | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_MRVL + }, + { + .test_descr = "AES-256-CBC OOP Encryption", + .test_data = &aes_test_data_11, + .op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT, + .feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC + }, + { + .test_descr = "AES-256-CBC OOP Decryption", + .test_data = &aes_test_data_11, + .op_mask = BLOCKCIPHER_TEST_OP_DECRYPT, + .feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC + }, + { + .test_descr = "AES-128-CTR Encryption", + .test_data = &aes_test_data_1, + .op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT | + BLOCKCIPHER_TEST_TARGET_PMD_MB | + BLOCKCIPHER_TEST_TARGET_PMD_SCHEDULER | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_MRVL + }, + { + .test_descr = "AES-128-CTR Decryption", + .test_data = &aes_test_data_1, + .op_mask = BLOCKCIPHER_TEST_OP_DECRYPT, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT | + BLOCKCIPHER_TEST_TARGET_PMD_MB | + BLOCKCIPHER_TEST_TARGET_PMD_SCHEDULER | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_MRVL + }, + { + .test_descr = "AES-192-CTR Encryption", + .test_data = &aes_test_data_2, + .op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT | + BLOCKCIPHER_TEST_TARGET_PMD_MB | + BLOCKCIPHER_TEST_TARGET_PMD_SCHEDULER | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC + }, + { + .test_descr = "AES-192-CTR Decryption", + .test_data = &aes_test_data_2, + .op_mask = BLOCKCIPHER_TEST_OP_DECRYPT, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT | + BLOCKCIPHER_TEST_TARGET_PMD_MB | + BLOCKCIPHER_TEST_TARGET_PMD_SCHEDULER | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC + }, + { + .test_descr = "AES-256-CTR Encryption", + .test_data = &aes_test_data_3, + .op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT | + BLOCKCIPHER_TEST_TARGET_PMD_MB | + BLOCKCIPHER_TEST_TARGET_PMD_SCHEDULER | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_MRVL + }, + { + .test_descr = "AES-256-CTR Decryption", + .test_data = &aes_test_data_3, + .op_mask = BLOCKCIPHER_TEST_OP_DECRYPT, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT | + BLOCKCIPHER_TEST_TARGET_PMD_MB | + BLOCKCIPHER_TEST_TARGET_PMD_SCHEDULER | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_MRVL + }, + { + .test_descr = "AES-128-CTR Encryption (12-byte IV)", + .test_data = &aes_test_data_1_IV_12_bytes, + .op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_MB + }, + { + .test_descr = "AES-192-CTR Encryption (12-byte IV)", + .test_data = &aes_test_data_2_IV_12_bytes, + .op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_MB + }, + { + .test_descr = "AES-256-CTR Encryption (12-byte IV)", + .test_data = &aes_test_data_3_IV_12_bytes, + .op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_MB + } +}; + +static const struct blockcipher_test_case aes_docsis_test_cases[] = { + + { + .test_descr = "AES-DOCSIS-BPI Full Block Encryption", + .test_data = &aes_test_data_docsis_1, + .op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_MB | + BLOCKCIPHER_TEST_TARGET_PMD_QAT + }, + { + .test_descr = "AES-DOCSIS-BPI Runt Block Encryption", + .test_data = &aes_test_data_docsis_2, + .op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_MB | + BLOCKCIPHER_TEST_TARGET_PMD_QAT + }, + { + .test_descr = "AES-DOCSIS-BPI Uneven Encryption", + .test_data = &aes_test_data_docsis_3, + .op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_MB | + BLOCKCIPHER_TEST_TARGET_PMD_QAT + }, + { + .test_descr = "AES-DOCSIS-BPI Full Block Decryption", + .test_data = &aes_test_data_docsis_1, + .op_mask = BLOCKCIPHER_TEST_OP_DECRYPT, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_MB | + BLOCKCIPHER_TEST_TARGET_PMD_QAT + }, + { + .test_descr = "AES-DOCSIS-BPI Runt Block Decryption", + .test_data = &aes_test_data_docsis_2, + .op_mask = BLOCKCIPHER_TEST_OP_DECRYPT, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_MB | + BLOCKCIPHER_TEST_TARGET_PMD_QAT + }, + { + .test_descr = "AES-DOCSIS-BPI Uneven Decryption", + .test_data = &aes_test_data_docsis_3, + .op_mask = BLOCKCIPHER_TEST_OP_DECRYPT, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_MB | + BLOCKCIPHER_TEST_TARGET_PMD_QAT + }, + { + .test_descr = "AES-DOCSIS-BPI OOP Full Block Encryption", + .test_data = &aes_test_data_docsis_1, + .op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT, + .feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_QAT + }, + { + .test_descr = "AES-DOCSIS-BPI OOP Runt Block Encryption", + .test_data = &aes_test_data_docsis_2, + .op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT, + .feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_QAT + }, + { + .test_descr = "AES-DOCSIS-BPI OOP Uneven Block Encryption", + .test_data = &aes_test_data_docsis_3, + .op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT, + .feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_QAT + }, + { + .test_descr = "AES-DOCSIS-BPI OOP Full Block Decryption", + .test_data = &aes_test_data_docsis_1, + .op_mask = BLOCKCIPHER_TEST_OP_DECRYPT, + .feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_QAT + }, + { + .test_descr = "AES-DOCSIS-BPI OOP Runt Block Decryption", + .test_data = &aes_test_data_docsis_2, + .op_mask = BLOCKCIPHER_TEST_OP_DECRYPT, + .feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_QAT + }, + { + .test_descr = "AES-DOCSIS-BPI OOP Uneven Block Decryption", + .test_data = &aes_test_data_docsis_3, + .op_mask = BLOCKCIPHER_TEST_OP_DECRYPT, + .feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_QAT + } +}; +#endif /* TEST_CRYPTODEV_AES_TEST_VECTORS_H_ */ diff --git a/dpdk/test/test/test_cryptodev_blockcipher.c b/dpdk/test/test/test_cryptodev_blockcipher.c new file mode 100644 index 00000000..20f3296d --- /dev/null +++ b/dpdk/test/test/test_cryptodev_blockcipher.c @@ -0,0 +1,687 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2015-2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <rte_common.h> +#include <rte_hexdump.h> +#include <rte_mbuf.h> +#include <rte_malloc.h> +#include <rte_memcpy.h> +#include <rte_pause.h> + +#include <rte_crypto.h> +#include <rte_cryptodev.h> +#include <rte_cryptodev_pmd.h> + +#include "test.h" +#include "test_cryptodev.h" +#include "test_cryptodev_blockcipher.h" +#include "test_cryptodev_aes_test_vectors.h" +#include "test_cryptodev_des_test_vectors.h" +#include "test_cryptodev_hash_test_vectors.h" + +static int +test_blockcipher_one_case(const struct blockcipher_test_case *t, + struct rte_mempool *mbuf_pool, + struct rte_mempool *op_mpool, + struct rte_mempool *sess_mpool, + uint8_t dev_id, + int driver_id, + char *test_msg) +{ + struct rte_mbuf *ibuf = NULL; + struct rte_mbuf *obuf = NULL; + struct rte_mbuf *iobuf; + struct rte_crypto_sym_xform *cipher_xform = NULL; + struct rte_crypto_sym_xform *auth_xform = NULL; + struct rte_crypto_sym_xform *init_xform = NULL; + struct rte_crypto_sym_op *sym_op = NULL; + struct rte_crypto_op *op = NULL; + struct rte_cryptodev_info dev_info; + struct rte_cryptodev_sym_session *sess = NULL; + + int status = TEST_SUCCESS; + const struct blockcipher_test_data *tdata = t->test_data; + uint8_t cipher_key[tdata->cipher_key.len]; + uint8_t auth_key[tdata->auth_key.len]; + uint32_t buf_len = tdata->ciphertext.len; + uint32_t digest_len = 0; + char *buf_p = NULL; + uint8_t src_pattern = 0xa5; + uint8_t dst_pattern = 0xb6; + uint8_t tmp_src_buf[MBUF_SIZE]; + uint8_t tmp_dst_buf[MBUF_SIZE]; + + int openssl_pmd = rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_OPENSSL_PMD)); + int scheduler_pmd = rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_SCHEDULER_PMD)); + int armv8_pmd = rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_ARMV8_PMD)); + int aesni_mb_pmd = rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_AESNI_MB_PMD)); + int qat_pmd = rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_QAT_SYM_PMD)); + int dpaa2_sec_pmd = rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_DPAA2_SEC_PMD)); + int dpaa_sec_pmd = rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_DPAA_SEC_PMD)); + int mrvl_pmd = rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_MRVL_PMD)); + + int nb_segs = 1; + + if (t->feature_mask & BLOCKCIPHER_TEST_FEATURE_SG) { + rte_cryptodev_info_get(dev_id, &dev_info); + if (!(dev_info.feature_flags & + RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER)) { + printf("Device doesn't support scatter-gather. " + "Test Skipped.\n"); + return 0; + } + nb_segs = 3; + } + + if (tdata->cipher_key.len) + memcpy(cipher_key, tdata->cipher_key.data, + tdata->cipher_key.len); + if (tdata->auth_key.len) + memcpy(auth_key, tdata->auth_key.data, + tdata->auth_key.len); + + if (driver_id == dpaa2_sec_pmd || + driver_id == dpaa_sec_pmd || + driver_id == qat_pmd || + driver_id == openssl_pmd || + driver_id == armv8_pmd || + driver_id == mrvl_pmd) { /* Fall through */ + digest_len = tdata->digest.len; + } else if (driver_id == aesni_mb_pmd || + driver_id == scheduler_pmd) { + digest_len = tdata->digest.truncated_len; + } else { + snprintf(test_msg, BLOCKCIPHER_TEST_MSG_LEN, + "line %u FAILED: %s", + __LINE__, "Unsupported PMD type"); + status = TEST_FAILED; + goto error_exit; + } + + /* preparing data */ + if (t->op_mask & BLOCKCIPHER_TEST_OP_AUTH) + buf_len += digest_len; + + /* for contiguous mbuf, nb_segs is 1 */ + ibuf = create_segmented_mbuf(mbuf_pool, + tdata->ciphertext.len, nb_segs, src_pattern); + if (ibuf == NULL) { + snprintf(test_msg, BLOCKCIPHER_TEST_MSG_LEN, + "line %u FAILED: %s", + __LINE__, "Cannot create source mbuf"); + status = TEST_FAILED; + goto error_exit; + } + + /* only encryption requires plaintext.data input, + * decryption/(digest gen)/(digest verify) use ciphertext.data + * to be computed + */ + if (t->op_mask & BLOCKCIPHER_TEST_OP_ENCRYPT) + pktmbuf_write(ibuf, 0, tdata->plaintext.len, + tdata->plaintext.data); + else + pktmbuf_write(ibuf, 0, tdata->ciphertext.len, + tdata->ciphertext.data); + + buf_p = rte_pktmbuf_append(ibuf, digest_len); + if (t->op_mask & BLOCKCIPHER_TEST_OP_AUTH_VERIFY) + rte_memcpy(buf_p, tdata->digest.data, digest_len); + else + memset(buf_p, 0, digest_len); + + if (t->feature_mask & BLOCKCIPHER_TEST_FEATURE_OOP) { + obuf = rte_pktmbuf_alloc(mbuf_pool); + if (!obuf) { + snprintf(test_msg, BLOCKCIPHER_TEST_MSG_LEN, "line %u " + "FAILED: %s", __LINE__, + "Allocation of rte_mbuf failed"); + status = TEST_FAILED; + goto error_exit; + } + memset(obuf->buf_addr, dst_pattern, obuf->buf_len); + + buf_p = rte_pktmbuf_append(obuf, buf_len); + if (!buf_p) { + snprintf(test_msg, BLOCKCIPHER_TEST_MSG_LEN, "line %u " + "FAILED: %s", __LINE__, + "No room to append mbuf"); + status = TEST_FAILED; + goto error_exit; + } + memset(buf_p, 0, buf_len); + } + + /* Generate Crypto op data structure */ + op = rte_crypto_op_alloc(op_mpool, RTE_CRYPTO_OP_TYPE_SYMMETRIC); + if (!op) { + snprintf(test_msg, BLOCKCIPHER_TEST_MSG_LEN, + "line %u FAILED: %s", + __LINE__, "Failed to allocate symmetric crypto " + "operation struct"); + status = TEST_FAILED; + goto error_exit; + } + + sym_op = op->sym; + + sym_op->m_src = ibuf; + + if (t->feature_mask & BLOCKCIPHER_TEST_FEATURE_OOP) { + sym_op->m_dst = obuf; + iobuf = obuf; + } else { + sym_op->m_dst = NULL; + iobuf = ibuf; + } + + /* sessionless op requires allocate xform using + * rte_crypto_op_sym_xforms_alloc(), otherwise rte_zmalloc() + * is used + */ + if (t->feature_mask & BLOCKCIPHER_TEST_FEATURE_SESSIONLESS) { + uint32_t n_xforms = 0; + + if (t->op_mask & BLOCKCIPHER_TEST_OP_CIPHER) + n_xforms++; + if (t->op_mask & BLOCKCIPHER_TEST_OP_AUTH) + n_xforms++; + + if (rte_crypto_op_sym_xforms_alloc(op, n_xforms) + == NULL) { + snprintf(test_msg, BLOCKCIPHER_TEST_MSG_LEN, "line %u " + "FAILED: %s", __LINE__, "Failed to " + "allocate space for crypto transforms"); + status = TEST_FAILED; + goto error_exit; + } + } else { + cipher_xform = rte_zmalloc(NULL, + sizeof(struct rte_crypto_sym_xform), 0); + + auth_xform = rte_zmalloc(NULL, + sizeof(struct rte_crypto_sym_xform), 0); + + if (!cipher_xform || !auth_xform) { + snprintf(test_msg, BLOCKCIPHER_TEST_MSG_LEN, "line %u " + "FAILED: %s", __LINE__, "Failed to " + "allocate memory for crypto transforms"); + status = TEST_FAILED; + goto error_exit; + } + } + + /* preparing xform, for sessioned op, init_xform is initialized + * here and later as param in rte_cryptodev_sym_session_create() call + */ + if (t->op_mask == BLOCKCIPHER_TEST_OP_ENC_AUTH_GEN) { + if (t->feature_mask & BLOCKCIPHER_TEST_FEATURE_SESSIONLESS) { + cipher_xform = op->sym->xform; + auth_xform = cipher_xform->next; + auth_xform->next = NULL; + } else { + cipher_xform->next = auth_xform; + auth_xform->next = NULL; + init_xform = cipher_xform; + } + } else if (t->op_mask == BLOCKCIPHER_TEST_OP_AUTH_VERIFY_DEC) { + if (t->feature_mask & BLOCKCIPHER_TEST_FEATURE_SESSIONLESS) { + auth_xform = op->sym->xform; + cipher_xform = auth_xform->next; + cipher_xform->next = NULL; + } else { + auth_xform->next = cipher_xform; + cipher_xform->next = NULL; + init_xform = auth_xform; + } + } else if ((t->op_mask == BLOCKCIPHER_TEST_OP_ENCRYPT) || + (t->op_mask == BLOCKCIPHER_TEST_OP_DECRYPT)) { + if (t->feature_mask & BLOCKCIPHER_TEST_FEATURE_SESSIONLESS) + cipher_xform = op->sym->xform; + else + init_xform = cipher_xform; + cipher_xform->next = NULL; + } else if ((t->op_mask == BLOCKCIPHER_TEST_OP_AUTH_GEN) || + (t->op_mask == BLOCKCIPHER_TEST_OP_AUTH_VERIFY)) { + if (t->feature_mask & BLOCKCIPHER_TEST_FEATURE_SESSIONLESS) + auth_xform = op->sym->xform; + else + init_xform = auth_xform; + auth_xform->next = NULL; + } else { + snprintf(test_msg, BLOCKCIPHER_TEST_MSG_LEN, + "line %u FAILED: %s", + __LINE__, "Unrecognized operation"); + status = TEST_FAILED; + goto error_exit; + } + + /*configure xforms & sym_op cipher and auth data*/ + if (t->op_mask & BLOCKCIPHER_TEST_OP_CIPHER) { + cipher_xform->type = RTE_CRYPTO_SYM_XFORM_CIPHER; + cipher_xform->cipher.algo = tdata->crypto_algo; + if (t->op_mask & BLOCKCIPHER_TEST_OP_ENCRYPT) + cipher_xform->cipher.op = + RTE_CRYPTO_CIPHER_OP_ENCRYPT; + else + cipher_xform->cipher.op = + RTE_CRYPTO_CIPHER_OP_DECRYPT; + cipher_xform->cipher.key.data = cipher_key; + cipher_xform->cipher.key.length = tdata->cipher_key.len; + cipher_xform->cipher.iv.offset = IV_OFFSET; + cipher_xform->cipher.iv.length = tdata->iv.len; + + sym_op->cipher.data.offset = 0; + sym_op->cipher.data.length = tdata->ciphertext.len; + rte_memcpy(rte_crypto_op_ctod_offset(op, uint8_t *, IV_OFFSET), + tdata->iv.data, + tdata->iv.len); + } + + if (t->op_mask & BLOCKCIPHER_TEST_OP_AUTH) { + uint32_t digest_offset = tdata->ciphertext.len; + + auth_xform->type = RTE_CRYPTO_SYM_XFORM_AUTH; + auth_xform->auth.algo = tdata->auth_algo; + auth_xform->auth.key.length = tdata->auth_key.len; + auth_xform->auth.key.data = auth_key; + auth_xform->auth.digest_length = digest_len; + + if (t->op_mask & BLOCKCIPHER_TEST_OP_AUTH_GEN) { + auth_xform->auth.op = RTE_CRYPTO_AUTH_OP_GENERATE; + sym_op->auth.digest.data = pktmbuf_mtod_offset + (iobuf, digest_offset); + sym_op->auth.digest.phys_addr = + pktmbuf_iova_offset(iobuf, + digest_offset); + } else { + auth_xform->auth.op = RTE_CRYPTO_AUTH_OP_VERIFY; + sym_op->auth.digest.data = pktmbuf_mtod_offset + (sym_op->m_src, digest_offset); + sym_op->auth.digest.phys_addr = + pktmbuf_iova_offset(sym_op->m_src, + digest_offset); + } + + sym_op->auth.data.offset = 0; + sym_op->auth.data.length = tdata->ciphertext.len; + } + + /* create session for sessioned op */ + if (!(t->feature_mask & BLOCKCIPHER_TEST_FEATURE_SESSIONLESS)) { + sess = rte_cryptodev_sym_session_create(sess_mpool); + + rte_cryptodev_sym_session_init(dev_id, sess, init_xform, + sess_mpool); + if (!sess) { + snprintf(test_msg, BLOCKCIPHER_TEST_MSG_LEN, "line %u " + "FAILED: %s", __LINE__, + "Session creation failed"); + status = TEST_FAILED; + goto error_exit; + } + + /* attach symmetric crypto session to crypto operations */ + rte_crypto_op_attach_sym_session(op, sess); + } + + TEST_HEXDUMP(stdout, "m_src(before):", + sym_op->m_src->buf_addr, sym_op->m_src->buf_len); + rte_memcpy(tmp_src_buf, sym_op->m_src->buf_addr, + sym_op->m_src->buf_len); + if (t->feature_mask & BLOCKCIPHER_TEST_FEATURE_OOP) { + TEST_HEXDUMP(stdout, "m_dst(before):", + sym_op->m_dst->buf_addr, sym_op->m_dst->buf_len); + rte_memcpy(tmp_dst_buf, sym_op->m_dst->buf_addr, + sym_op->m_dst->buf_len); + } + + /* Process crypto operation */ + if (rte_cryptodev_enqueue_burst(dev_id, 0, &op, 1) != 1) { + snprintf(test_msg, BLOCKCIPHER_TEST_MSG_LEN, + "line %u FAILED: %s", + __LINE__, "Error sending packet for encryption"); + status = TEST_FAILED; + goto error_exit; + } + + op = NULL; + + while (rte_cryptodev_dequeue_burst(dev_id, 0, &op, 1) == 0) + rte_pause(); + + if (!op) { + snprintf(test_msg, BLOCKCIPHER_TEST_MSG_LEN, + "line %u FAILED: %s", + __LINE__, "Failed to process sym crypto op"); + status = TEST_FAILED; + goto error_exit; + } + + TEST_HEXDUMP(stdout, "m_src(after):", + sym_op->m_src->buf_addr, sym_op->m_src->buf_len); + if (t->feature_mask & BLOCKCIPHER_TEST_FEATURE_OOP) + TEST_HEXDUMP(stdout, "m_dst(after):", + sym_op->m_dst->buf_addr, sym_op->m_dst->buf_len); + + /* Verify results */ + if (op->status != RTE_CRYPTO_OP_STATUS_SUCCESS) { + if (t->op_mask & BLOCKCIPHER_TEST_OP_AUTH_VERIFY) + snprintf(test_msg, BLOCKCIPHER_TEST_MSG_LEN, "line %u " + "FAILED: Digest verification failed " + "(0x%X)", __LINE__, op->status); + else + snprintf(test_msg, BLOCKCIPHER_TEST_MSG_LEN, "line %u " + "FAILED: Digest verification failed " + "(0x%X)", __LINE__, op->status); + status = TEST_FAILED; + goto error_exit; + } + + if (t->op_mask & BLOCKCIPHER_TEST_OP_CIPHER) { + uint8_t buffer[2048]; + const uint8_t *compare_ref; + uint32_t compare_len; + + if (t->op_mask & BLOCKCIPHER_TEST_OP_ENCRYPT) { + compare_ref = tdata->ciphertext.data; + compare_len = tdata->ciphertext.len; + } else { + compare_ref = tdata->plaintext.data; + compare_len = tdata->plaintext.len; + } + + if (memcmp(rte_pktmbuf_read(iobuf, 0, compare_len, + buffer), compare_ref, compare_len)) { + snprintf(test_msg, BLOCKCIPHER_TEST_MSG_LEN, "line %u " + "FAILED: %s", __LINE__, + "Crypto data not as expected"); + status = TEST_FAILED; + goto error_exit; + } + } + + if (t->op_mask & BLOCKCIPHER_TEST_OP_AUTH_GEN) { + uint8_t *auth_res = pktmbuf_mtod_offset(iobuf, + tdata->ciphertext.len); + + if (memcmp(auth_res, tdata->digest.data, digest_len)) { + snprintf(test_msg, BLOCKCIPHER_TEST_MSG_LEN, "line %u " + "FAILED: %s", __LINE__, "Generated " + "digest data not as expected"); + status = TEST_FAILED; + goto error_exit; + } + } + + /* The only parts that should have changed in the buffer are + * plaintext/ciphertext and digest. + * In OOP only the dest buffer should change. + */ + if (t->feature_mask & BLOCKCIPHER_TEST_FEATURE_OOP) { + struct rte_mbuf *mbuf; + uint8_t value; + uint32_t head_unchanged_len, changed_len = 0; + uint32_t i; + + mbuf = sym_op->m_src; + head_unchanged_len = mbuf->buf_len; + + for (i = 0; i < mbuf->buf_len; i++) { + value = *((uint8_t *)(mbuf->buf_addr)+i); + if (value != tmp_src_buf[i]) { + snprintf(test_msg, BLOCKCIPHER_TEST_MSG_LEN, + "line %u FAILED: OOP src outer mbuf data (0x%x) not as expected (0x%x)", + __LINE__, value, tmp_src_buf[i]); + status = TEST_FAILED; + goto error_exit; + } + } + + mbuf = sym_op->m_dst; + if (t->op_mask & BLOCKCIPHER_TEST_OP_AUTH) { + head_unchanged_len = rte_pktmbuf_headroom(mbuf) + + sym_op->auth.data.offset; + changed_len = sym_op->auth.data.length; + if (t->op_mask & BLOCKCIPHER_TEST_OP_AUTH_GEN) + changed_len += digest_len; + } else { + /* cipher-only */ + head_unchanged_len = rte_pktmbuf_headroom(mbuf) + + sym_op->cipher.data.offset; + changed_len = sym_op->cipher.data.length; + } + + for (i = 0; i < mbuf->buf_len; i++) { + if (i == head_unchanged_len) + i += changed_len; + value = *((uint8_t *)(mbuf->buf_addr)+i); + if (value != tmp_dst_buf[i]) { + snprintf(test_msg, BLOCKCIPHER_TEST_MSG_LEN, + "line %u FAILED: OOP dst outer mbuf data " + "(0x%x) not as expected (0x%x)", + __LINE__, value, tmp_dst_buf[i]); + status = TEST_FAILED; + goto error_exit; + } + } + } else { + /* In-place operation */ + struct rte_mbuf *mbuf; + uint8_t value; + uint32_t head_unchanged_len = 0, changed_len = 0; + uint32_t i; + + mbuf = sym_op->m_src; + if (t->op_mask & BLOCKCIPHER_TEST_OP_CIPHER) { + head_unchanged_len = rte_pktmbuf_headroom(mbuf) + + sym_op->cipher.data.offset; + changed_len = sym_op->cipher.data.length; + } else { + /* auth-only */ + head_unchanged_len = rte_pktmbuf_headroom(mbuf) + + sym_op->auth.data.offset + + sym_op->auth.data.length; + changed_len = 0; + } + + if (t->op_mask & BLOCKCIPHER_TEST_OP_AUTH_GEN) + changed_len += digest_len; + + for (i = 0; i < mbuf->buf_len; i++) { + if (i == head_unchanged_len) + i += changed_len; + value = *((uint8_t *)(mbuf->buf_addr)+i); + if (value != tmp_src_buf[i]) { + snprintf(test_msg, BLOCKCIPHER_TEST_MSG_LEN, + "line %u FAILED: outer mbuf data (0x%x) " + "not as expected (0x%x)", + __LINE__, value, tmp_src_buf[i]); + status = TEST_FAILED; + goto error_exit; + } + } + } + + snprintf(test_msg, BLOCKCIPHER_TEST_MSG_LEN, "PASS"); + +error_exit: + if (!(t->feature_mask & BLOCKCIPHER_TEST_FEATURE_SESSIONLESS)) { + if (sess) { + rte_cryptodev_sym_session_clear(dev_id, sess); + rte_cryptodev_sym_session_free(sess); + } + if (cipher_xform) + rte_free(cipher_xform); + if (auth_xform) + rte_free(auth_xform); + } + + if (op) + rte_crypto_op_free(op); + + if (obuf) + rte_pktmbuf_free(obuf); + + if (ibuf) + rte_pktmbuf_free(ibuf); + + return status; +} + +int +test_blockcipher_all_tests(struct rte_mempool *mbuf_pool, + struct rte_mempool *op_mpool, + struct rte_mempool *sess_mpool, + uint8_t dev_id, + int driver_id, + enum blockcipher_test_type test_type) +{ + int status, overall_status = TEST_SUCCESS; + uint32_t i, test_index = 0; + char test_msg[BLOCKCIPHER_TEST_MSG_LEN + 1]; + uint32_t n_test_cases = 0; + uint32_t target_pmd_mask = 0; + const struct blockcipher_test_case *tcs = NULL; + + int openssl_pmd = rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_OPENSSL_PMD)); + int dpaa2_sec_pmd = rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_DPAA2_SEC_PMD)); + int dpaa_sec_pmd = rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_DPAA_SEC_PMD)); + int scheduler_pmd = rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_SCHEDULER_PMD)); + int armv8_pmd = rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_ARMV8_PMD)); + int aesni_mb_pmd = rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_AESNI_MB_PMD)); + int qat_pmd = rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_QAT_SYM_PMD)); + int mrvl_pmd = rte_cryptodev_driver_id_get( + RTE_STR(CRYPTODEV_NAME_MRVL_PMD)); + + switch (test_type) { + case BLKCIPHER_AES_CHAIN_TYPE: + n_test_cases = sizeof(aes_chain_test_cases) / + sizeof(aes_chain_test_cases[0]); + tcs = aes_chain_test_cases; + break; + case BLKCIPHER_AES_CIPHERONLY_TYPE: + n_test_cases = sizeof(aes_cipheronly_test_cases) / + sizeof(aes_cipheronly_test_cases[0]); + tcs = aes_cipheronly_test_cases; + break; + case BLKCIPHER_AES_DOCSIS_TYPE: + n_test_cases = sizeof(aes_docsis_test_cases) / + sizeof(aes_docsis_test_cases[0]); + tcs = aes_docsis_test_cases; + break; + case BLKCIPHER_3DES_CHAIN_TYPE: + n_test_cases = sizeof(triple_des_chain_test_cases) / + sizeof(triple_des_chain_test_cases[0]); + tcs = triple_des_chain_test_cases; + break; + case BLKCIPHER_3DES_CIPHERONLY_TYPE: + n_test_cases = sizeof(triple_des_cipheronly_test_cases) / + sizeof(triple_des_cipheronly_test_cases[0]); + tcs = triple_des_cipheronly_test_cases; + break; + case BLKCIPHER_DES_CIPHERONLY_TYPE: + n_test_cases = sizeof(des_cipheronly_test_cases) / + sizeof(des_cipheronly_test_cases[0]); + tcs = des_cipheronly_test_cases; + break; + case BLKCIPHER_DES_DOCSIS_TYPE: + n_test_cases = sizeof(des_docsis_test_cases) / + sizeof(des_docsis_test_cases[0]); + tcs = des_docsis_test_cases; + break; + case BLKCIPHER_AUTHONLY_TYPE: + n_test_cases = sizeof(hash_test_cases) / + sizeof(hash_test_cases[0]); + tcs = hash_test_cases; + break; + default: + break; + } + + if (driver_id == aesni_mb_pmd) + target_pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_MB; + else if (driver_id == qat_pmd) + target_pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_QAT; + else if (driver_id == openssl_pmd) + target_pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL; + else if (driver_id == armv8_pmd) + target_pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_ARMV8; + else if (driver_id == scheduler_pmd) + target_pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_SCHEDULER; + else if (driver_id == dpaa2_sec_pmd) + target_pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC; + else if (driver_id == dpaa_sec_pmd) + target_pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC; + else if (driver_id == mrvl_pmd) + target_pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_MRVL; + else + TEST_ASSERT(0, "Unrecognized cryptodev type"); + + for (i = 0; i < n_test_cases; i++) { + const struct blockcipher_test_case *tc = &tcs[i]; + + if (!(tc->pmd_mask & target_pmd_mask)) + continue; + + status = test_blockcipher_one_case(tc, mbuf_pool, op_mpool, + sess_mpool, dev_id, driver_id, test_msg); + + printf(" %u) TestCase %s %s\n", test_index ++, + tc->test_descr, test_msg); + + if (status != TEST_SUCCESS) { + if (overall_status == TEST_SUCCESS) + overall_status = status; + + if (tc->feature_mask & BLOCKCIPHER_TEST_FEATURE_STOPPER) + break; + } + } + + return overall_status; +} diff --git a/dpdk/test/test/test_cryptodev_blockcipher.h b/dpdk/test/test/test_cryptodev_blockcipher.h new file mode 100644 index 00000000..67c78d47 --- /dev/null +++ b/dpdk/test/test/test_cryptodev_blockcipher.h @@ -0,0 +1,135 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef TEST_CRYPTODEV_BLOCKCIPHER_H_ +#define TEST_CRYPTODEV_BLOCKCIPHER_H_ + +#ifndef BLOCKCIPHER_TEST_MSG_LEN +#define BLOCKCIPHER_TEST_MSG_LEN 256 +#endif + +#define BLOCKCIPHER_TEST_OP_ENCRYPT 0x01 +#define BLOCKCIPHER_TEST_OP_DECRYPT 0x02 +#define BLOCKCIPHER_TEST_OP_AUTH_GEN 0x04 +#define BLOCKCIPHER_TEST_OP_AUTH_VERIFY 0x08 + +#define BLOCKCIPHER_TEST_FEATURE_OOP 0x01 +#define BLOCKCIPHER_TEST_FEATURE_SESSIONLESS 0x02 +#define BLOCKCIPHER_TEST_FEATURE_STOPPER 0x04 /* stop upon failing */ +#define BLOCKCIPHER_TEST_FEATURE_SG 0x08 /* Scatter Gather */ + +#define BLOCKCIPHER_TEST_TARGET_PMD_MB 0x0001 /* Multi-buffer flag */ +#define BLOCKCIPHER_TEST_TARGET_PMD_QAT 0x0002 /* QAT flag */ +#define BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL 0x0004 /* SW OPENSSL flag */ +#define BLOCKCIPHER_TEST_TARGET_PMD_ARMV8 0x0008 /* ARMv8 flag */ +#define BLOCKCIPHER_TEST_TARGET_PMD_SCHEDULER 0x0010 /* Scheduler */ +#define BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC 0x0020 /* DPAA2_SEC flag */ +#define BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC 0x0040 /* DPAA_SEC flag */ +#define BLOCKCIPHER_TEST_TARGET_PMD_MRVL 0x0080 /* Marvell flag */ + +#define BLOCKCIPHER_TEST_OP_CIPHER (BLOCKCIPHER_TEST_OP_ENCRYPT | \ + BLOCKCIPHER_TEST_OP_DECRYPT) + +#define BLOCKCIPHER_TEST_OP_AUTH (BLOCKCIPHER_TEST_OP_AUTH_GEN | \ + BLOCKCIPHER_TEST_OP_AUTH_VERIFY) + +#define BLOCKCIPHER_TEST_OP_ENC_AUTH_GEN (BLOCKCIPHER_TEST_OP_ENCRYPT | \ + BLOCKCIPHER_TEST_OP_AUTH_GEN) + +#define BLOCKCIPHER_TEST_OP_AUTH_VERIFY_DEC (BLOCKCIPHER_TEST_OP_DECRYPT | \ + BLOCKCIPHER_TEST_OP_AUTH_VERIFY) + +enum blockcipher_test_type { + BLKCIPHER_AES_CHAIN_TYPE, /* use aes_chain_test_cases[] */ + BLKCIPHER_AES_CIPHERONLY_TYPE, /* use aes_cipheronly_test_cases[] */ + BLKCIPHER_AES_DOCSIS_TYPE, /* use aes_docsis_test_cases[] */ + BLKCIPHER_3DES_CHAIN_TYPE, /* use triple_des_chain_test_cases[] */ + BLKCIPHER_3DES_CIPHERONLY_TYPE, /* triple_des_cipheronly_test_cases[] */ + BLKCIPHER_AUTHONLY_TYPE, /* use hash_test_cases[] */ + BLKCIPHER_DES_CIPHERONLY_TYPE, /* use des_cipheronly_test_cases[] */ + BLKCIPHER_DES_DOCSIS_TYPE /* use des_docsis_test_cases[] */ +}; + +struct blockcipher_test_case { + const char *test_descr; /* test description */ + const struct blockcipher_test_data *test_data; + uint8_t op_mask; /* operation mask */ + uint8_t feature_mask; + uint32_t pmd_mask; +}; + +struct blockcipher_test_data { + enum rte_crypto_cipher_algorithm crypto_algo; + + struct { + uint8_t data[64]; + unsigned int len; + } cipher_key; + + struct { + uint8_t data[64] __rte_aligned(16); + unsigned int len; + } iv; + + struct { + const uint8_t *data; + unsigned int len; + } plaintext; + + struct { + const uint8_t *data; + unsigned int len; + } ciphertext; + + enum rte_crypto_auth_algorithm auth_algo; + + struct { + uint8_t data[128]; + unsigned int len; + } auth_key; + + struct { + uint8_t data[128]; + unsigned int len; /* for qat */ + unsigned int truncated_len; /* for mb */ + } digest; +}; + +int +test_blockcipher_all_tests(struct rte_mempool *mbuf_pool, + struct rte_mempool *op_mpool, + struct rte_mempool *sess_mpool, + uint8_t dev_id, + int driver_id, + enum blockcipher_test_type test_type); + +#endif /* TEST_CRYPTODEV_BLOCKCIPHER_H_ */ diff --git a/dpdk/test/test/test_cryptodev_des_test_vectors.h b/dpdk/test/test/test_cryptodev_des_test_vectors.h new file mode 100644 index 00000000..d09e23de --- /dev/null +++ b/dpdk/test/test/test_cryptodev_des_test_vectors.h @@ -0,0 +1,1293 @@ +/* + * BSD LICENSE + * + * Copyright(c) 2016 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef TEST_CRYPTODEV_DES_TEST_VECTORS_H_ +#define TEST_CRYPTODEV_DES_TEST_VECTORS_H_ + +static const uint8_t plaintext_des[] = { + "What a lousy earth! He wondered how many people " + "were destitute that same night even in his own " + "prosperous country, how many homes were " + "shanties, how many husbands were drunk and " + "wives socked, and how many children were " + "bullied, abused, or abandoned. How many " + "families hungered for food they could not " + "afford to buy? How many hearts were broken? How " + "many suicides would take place that same night, " + "how many people would go insane? How many " + "cockroaches and landlords would triumph? How " + "many winners were losers, successes failures, " + "and rich men poor men? How many wise guys were " + "stupid? How many happy endings were unhappy " + "endings? How many honest men were liars, brave " + "men cowards, loyal men traitors, how many " + "sainted men were corrupt, how many people in " + "positions of trust had sold their souls to " + "bodyguards, how many had never had souls? How " + "many straight-and-narrow paths were crooked " + "paths? How many best families were worst " + "families and how many good people were bad " + "people? When you added them all up and then " + "subtracted, you might be left with only the " + "children, and perhaps with Albert Einstein and " + "an old violinist or sculptor somewhere." +}; + +static const uint8_t ciphertext512_des128ctr[] = { + 0x13, 0x39, 0x3B, 0xBC, 0x1D, 0xE3, 0x23, 0x09, + 0x9B, 0x08, 0xD1, 0x09, 0x52, 0x93, 0x78, 0x29, + 0x11, 0x21, 0xBA, 0x01, 0x15, 0xCD, 0xEC, 0xAA, + 0x79, 0x77, 0x58, 0xAE, 0xAE, 0xBC, 0x97, 0x33, + 0x94, 0xA9, 0x2D, 0xC0, 0x0A, 0xA9, 0xA4, 0x4B, + 0x19, 0x07, 0x88, 0x06, 0x7E, 0x81, 0x0F, 0xB5, + 0x60, 0xCF, 0xA7, 0xC3, 0x2A, 0x43, 0xFF, 0x16, + 0x3A, 0x5F, 0x11, 0x2D, 0x11, 0x38, 0x37, 0x94, + 0x2A, 0xC8, 0x3D, 0x20, 0xBB, 0x93, 0x95, 0x54, + 0x12, 0xFF, 0x0C, 0x47, 0x89, 0x7D, 0x73, 0xD1, + 0x2E, 0x3A, 0x80, 0x52, 0xA8, 0x92, 0x93, 0x99, + 0x16, 0xB8, 0x12, 0x1B, 0x8B, 0xA8, 0xC1, 0x81, + 0x95, 0x18, 0x82, 0xD6, 0x5A, 0xA7, 0xFE, 0xCF, + 0xC4, 0xAC, 0x85, 0x91, 0x0C, 0x2F, 0x1D, 0x10, + 0x9A, 0x65, 0x07, 0xB0, 0x2E, 0x5A, 0x2D, 0x48, + 0x26, 0xF8, 0x17, 0x7A, 0x53, 0xD6, 0xB8, 0xDF, + 0xB1, 0x10, 0x48, 0x7E, 0x8F, 0xBE, 0x2E, 0xA1, + 0x0D, 0x9E, 0xA9, 0xF1, 0x3B, 0x3B, 0x33, 0xCD, + 0xDC, 0x52, 0x7E, 0xC0, 0x0E, 0xA0, 0xD8, 0xA7, + 0xC6, 0x34, 0x5A, 0xAA, 0x29, 0x8B, 0xA9, 0xAC, + 0x1F, 0x78, 0xAD, 0xEE, 0x34, 0x59, 0x30, 0xFB, + 0x2A, 0x20, 0x3D, 0x4D, 0x30, 0xA7, 0x7D, 0xD8, + 0xA0, 0xC6, 0xA2, 0xD3, 0x9A, 0xFB, 0x50, 0x97, + 0x4D, 0x25, 0xA2, 0x37, 0x51, 0x54, 0xB7, 0xEB, + 0xED, 0x77, 0xDB, 0x94, 0x35, 0x8B, 0x70, 0x95, + 0x4A, 0x00, 0xA7, 0xF1, 0x8A, 0x66, 0x0E, 0xC6, + 0x05, 0x7B, 0x69, 0x05, 0x42, 0x03, 0x96, 0x2C, + 0x55, 0x00, 0x1B, 0xC0, 0x19, 0x4D, 0x0D, 0x2E, + 0xF5, 0x81, 0x11, 0x64, 0xCA, 0xBB, 0xF2, 0x0F, + 0x9C, 0x60, 0xE2, 0xCC, 0x02, 0x6E, 0x83, 0xD5, + 0x24, 0xF4, 0x12, 0x0E, 0x6A, 0xEA, 0x4F, 0x6C, + 0x79, 0x69, 0x65, 0x67, 0xDB, 0xF7, 0xEA, 0x98, + 0x5D, 0x56, 0x98, 0xB7, 0x88, 0xE7, 0x23, 0xC9, + 0x17, 0x32, 0x92, 0x33, 0x5A, 0x0C, 0x15, 0x20, + 0x3B, 0x1C, 0xF9, 0x0F, 0x4D, 0xD1, 0xE8, 0xE6, + 0x9E, 0x5E, 0x24, 0x1B, 0xA4, 0xB8, 0xB9, 0xE9, + 0x2F, 0xFC, 0x89, 0xB4, 0xB9, 0xF4, 0xA6, 0xAD, + 0x55, 0xF4, 0xDF, 0x58, 0x63, 0x25, 0xE3, 0x41, + 0x70, 0xDF, 0x10, 0xE7, 0x13, 0x87, 0x8D, 0xB3, + 0x62, 0x4F, 0xF5, 0x86, 0x85, 0x8F, 0x59, 0xF0, + 0x21, 0x0E, 0x8F, 0x11, 0xAD, 0xBF, 0xDD, 0x61, + 0x68, 0x3F, 0x54, 0x57, 0x49, 0x38, 0xC8, 0x24, + 0x8E, 0x0A, 0xAC, 0xCA, 0x2C, 0x36, 0x3E, 0x5F, + 0x0A, 0xCE, 0xFD, 0x1A, 0x60, 0x63, 0x5A, 0xE6, + 0x06, 0x64, 0xB5, 0x94, 0x3C, 0xC9, 0xAF, 0x7C, + 0xCD, 0x49, 0x10, 0xCF, 0xAF, 0x0E, 0x2E, 0x79, + 0x27, 0xB2, 0x67, 0x02, 0xED, 0xEE, 0x80, 0x77, + 0x7C, 0x6D, 0x4B, 0xDB, 0xCF, 0x8D, 0x68, 0x00, + 0x2E, 0xD9, 0xF0, 0x8E, 0x08, 0xBF, 0xA6, 0x9B, + 0xFE, 0xA4, 0xFB, 0x19, 0x46, 0xAF, 0x1B, 0xA9, + 0xF8, 0x22, 0x81, 0x21, 0x97, 0xFC, 0xC0, 0x8A, + 0x26, 0x58, 0x13, 0x29, 0xB6, 0x69, 0x94, 0x4B, + 0xAB, 0xB3, 0x88, 0x0D, 0xA9, 0x48, 0x0E, 0xE8, + 0x70, 0xFC, 0xA1, 0x21, 0xC4, 0x2C, 0xE5, 0x99, + 0xB4, 0xF1, 0x6F, 0xB2, 0x4B, 0x4B, 0xCD, 0x48, + 0x15, 0x47, 0x2D, 0x72, 0x39, 0x99, 0x9D, 0x24, + 0x0C, 0x8B, 0xDC, 0xA1, 0xEE, 0xF6, 0xF4, 0x73, + 0xC3, 0xB8, 0x0C, 0x23, 0x0D, 0xA7, 0xC4, 0x7D, + 0x27, 0xE2, 0x14, 0x11, 0x53, 0x19, 0xE7, 0xCA, + 0x94, 0x4E, 0x0D, 0x2C, 0xF7, 0x36, 0x47, 0xDB, + 0x77, 0x3C, 0x22, 0xAC, 0xBE, 0xE1, 0x06, 0x55, + 0xE5, 0xDD, 0x8B, 0x65, 0xE8, 0xE9, 0x91, 0x52, + 0x59, 0x97, 0xFC, 0x8C, 0xEE, 0x96, 0x22, 0x60, + 0xEE, 0xBF, 0x82, 0xF0, 0xCA, 0x14, 0xF9, 0xD3 +}; + +static const struct blockcipher_test_data +triple_des128ctr_test_vector = { + .crypto_algo = RTE_CRYPTO_CIPHER_3DES_CTR, + .cipher_key = { + .data = { + 0xE4, 0x23, 0x33, 0x8A, 0x35, 0x64, 0x61, 0xE2, + 0x49, 0x03, 0xDD, 0xC6, 0xB8, 0xCA, 0x55, 0x7A + }, + .len = 16 + }, + .iv = { + .data = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07 + }, + .len = 8 + }, + .plaintext = { + .data = plaintext_des, + .len = 512 + }, + .ciphertext = { + .data = ciphertext512_des128ctr, + .len = 512 + } +}; + +static const struct blockcipher_test_data +triple_des128ctr_sha1_test_vector = { + .crypto_algo = RTE_CRYPTO_CIPHER_3DES_CTR, + .cipher_key = { + .data = { + 0xE4, 0x23, 0x33, 0x8A, 0x35, 0x64, 0x61, 0xE2, + 0x49, 0x03, 0xDD, 0xC6, 0xB8, 0xCA, 0x55, 0x7A + }, + .len = 16 + }, + .iv = { + .data = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07 + }, + .len = 8 + }, + .plaintext = { + .data = plaintext_des, + .len = 512 + }, + .ciphertext = { + .data = ciphertext512_des128ctr, + .len = 512 + }, + .auth_algo = RTE_CRYPTO_AUTH_SHA1, + .digest = { + .data = { + 0xC3, 0x40, 0xD5, 0xD9, 0x8F, 0x8A, 0xC0, 0xF0, + 0x46, 0x28, 0x02, 0x01, 0xB5, 0xC1, 0x87, 0x4D, + 0xAC, 0xFE, 0x48, 0x76 + }, + .len = 20 + } +}; + +static const struct blockcipher_test_data +triple_des128ctr_hmac_sha1_test_vector = { + .crypto_algo = RTE_CRYPTO_CIPHER_3DES_CTR, + .cipher_key = { + .data = { + 0xE4, 0x23, 0x33, 0x8A, 0x35, 0x64, 0x61, 0xE2, + 0x49, 0x03, 0xDD, 0xC6, 0xB8, 0xCA, 0x55, 0x7A + }, + .len = 16 + }, + .iv = { + .data = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07 + }, + .len = 8 + }, + .plaintext = { + .data = plaintext_des, + .len = 512 + }, + .ciphertext = { + .data = ciphertext512_des128ctr, + .len = 512 + }, + .auth_algo = RTE_CRYPTO_AUTH_SHA1_HMAC, + .auth_key = { + .data = { + 0xF8, 0x2A, 0xC7, 0x54, 0xDB, 0x96, 0x18, 0xAA, + 0xC3, 0xA1, 0x53, 0xF6, 0x1F, 0x17, 0x60, 0xBD, + 0xDE, 0xF4, 0xDE, 0xAD + }, + .len = 20 + }, + .digest = { + .data = { + 0xF1, 0xC1, 0xDB, 0x4D, 0xFA, 0x7F, 0x2F, 0xE5, + 0xF8, 0x49, 0xEA, 0x1D, 0x7F, 0xCB, 0x42, 0x59, + 0xC4, 0x1E, 0xB1, 0x18 + }, + .len = 20 + } +}; + +static const uint8_t ciphertext512_des192ctr[] = { + 0xFF, 0x32, 0x52, 0x97, 0x10, 0xBF, 0x0B, 0x10, + 0x68, 0x0F, 0x4F, 0x56, 0x8B, 0x2C, 0x7B, 0x8E, + 0x39, 0x1E, 0x1A, 0x2F, 0x83, 0xDE, 0x5E, 0x35, + 0xC8, 0x4B, 0xDF, 0xD5, 0xBC, 0x84, 0x50, 0x1A, + 0x02, 0xDF, 0xB3, 0x11, 0xE4, 0xDA, 0xB8, 0x0E, + 0x47, 0xC6, 0x0C, 0x51, 0x09, 0x62, 0x9C, 0x5D, + 0x71, 0x40, 0x49, 0xD8, 0x55, 0xBD, 0x7D, 0x90, + 0x71, 0xC5, 0xF7, 0x07, 0x6F, 0x08, 0x71, 0x2A, + 0xB1, 0x77, 0x9B, 0x0F, 0xA1, 0xB0, 0xD6, 0x10, + 0xB2, 0xE5, 0x31, 0xEC, 0x21, 0x13, 0x89, 0x2A, + 0x09, 0x7E, 0x30, 0xDB, 0xA0, 0xF0, 0xDC, 0xE4, + 0x74, 0x64, 0x39, 0xA3, 0xB0, 0xB1, 0x80, 0x66, + 0x52, 0xD4, 0x4E, 0xC9, 0x5A, 0x52, 0x6A, 0xC7, + 0xB5, 0x2B, 0x61, 0xD5, 0x17, 0xD5, 0xF3, 0xCC, + 0x41, 0x61, 0xD2, 0xA6, 0xF4, 0x51, 0x24, 0x3A, + 0x63, 0x5D, 0x23, 0xB1, 0xF0, 0x22, 0xE7, 0x45, + 0xFA, 0x5F, 0x7E, 0x99, 0x00, 0x11, 0x28, 0x35, + 0xA3, 0xF4, 0x61, 0x94, 0x0E, 0x98, 0xCE, 0x35, + 0xDD, 0x91, 0x1B, 0x0B, 0x4D, 0xEE, 0xFF, 0xFF, + 0x0B, 0xD4, 0xDC, 0x56, 0xFC, 0x71, 0xE9, 0xEC, + 0xE8, 0x36, 0x51, 0xF8, 0x8B, 0x6A, 0xE1, 0x8C, + 0x2B, 0x25, 0x91, 0x91, 0x9B, 0x92, 0x76, 0xB5, + 0x3D, 0x26, 0xA8, 0x53, 0xEA, 0x30, 0x5B, 0x4D, + 0xDA, 0x16, 0xDA, 0x7D, 0x04, 0x88, 0xF5, 0x22, + 0xA8, 0x0C, 0xB9, 0x41, 0xC7, 0x91, 0x64, 0x86, + 0x99, 0x7D, 0x18, 0xB9, 0x67, 0xA2, 0x6E, 0x05, + 0x1A, 0x82, 0x8F, 0xA2, 0xEB, 0x4D, 0x0B, 0x8C, + 0x88, 0x2D, 0xBA, 0x77, 0x87, 0x32, 0x50, 0x3C, + 0x4C, 0xD8, 0xD3, 0x50, 0x39, 0xFA, 0xDF, 0x48, + 0x3E, 0x30, 0xF5, 0x76, 0x06, 0xB0, 0x1A, 0x05, + 0x60, 0x2C, 0xD3, 0xA0, 0x63, 0x1A, 0x19, 0x2D, + 0x6B, 0x76, 0xF2, 0x31, 0x4C, 0xA7, 0xE6, 0x5C, + 0x1B, 0x23, 0x20, 0x41, 0x32, 0xE5, 0x83, 0x47, + 0x04, 0xB6, 0x3E, 0xE0, 0xFD, 0x49, 0x1E, 0x1B, + 0x75, 0x10, 0x11, 0x46, 0xE9, 0xF9, 0x96, 0x9A, + 0xD7, 0x59, 0xFE, 0x38, 0x31, 0xFE, 0x79, 0xC4, + 0xC8, 0x46, 0x88, 0xDE, 0x2E, 0xAE, 0x20, 0xED, + 0x77, 0x50, 0x40, 0x38, 0x26, 0xD3, 0x35, 0xF6, + 0x29, 0x55, 0x6A, 0x6B, 0x38, 0x69, 0xFE, 0x90, + 0x5B, 0xA7, 0xFA, 0x6B, 0x73, 0x4F, 0xB9, 0x5D, + 0xDC, 0x6F, 0x98, 0xC3, 0x6A, 0xC4, 0xB5, 0x09, + 0xC5, 0x84, 0xA5, 0x6A, 0x84, 0xA4, 0xB3, 0x8A, + 0x5F, 0xCA, 0x92, 0x64, 0x9E, 0xC3, 0x0F, 0x84, + 0x8B, 0x2D, 0x48, 0xC6, 0x67, 0xAE, 0x07, 0xE0, + 0x28, 0x38, 0x6D, 0xC4, 0x4D, 0x13, 0x87, 0xE0, + 0xB2, 0x2F, 0xAA, 0xC0, 0xCF, 0x68, 0xD7, 0x9C, + 0xB8, 0x07, 0xE4, 0x51, 0xD7, 0x75, 0x86, 0xFA, + 0x0C, 0x50, 0x74, 0x68, 0x00, 0x64, 0x2A, 0x27, + 0x59, 0xE9, 0x80, 0xEB, 0xC2, 0xA3, 0xFA, 0x58, + 0xCC, 0x03, 0xE7, 0x7B, 0x66, 0x53, 0xFF, 0x90, + 0xA0, 0x85, 0xE2, 0xF8, 0x82, 0xFE, 0xC6, 0x2B, + 0xFF, 0x5E, 0x70, 0x85, 0x34, 0xB7, 0x22, 0x38, + 0xDB, 0xBC, 0x15, 0x30, 0x59, 0xC1, 0x48, 0x42, + 0xE5, 0x38, 0x8D, 0x37, 0x59, 0xDB, 0xA3, 0x20, + 0x17, 0x36, 0x1D, 0x4B, 0xBF, 0x4E, 0xA4, 0x35, + 0xCC, 0xFE, 0xF5, 0x7A, 0x73, 0xB4, 0x6D, 0x20, + 0x1D, 0xC0, 0xE5, 0x21, 0x5C, 0xD2, 0x8A, 0x65, + 0x08, 0xB6, 0x63, 0xAC, 0x9A, 0x1E, 0x3F, 0x3C, + 0xAB, 0xB6, 0x6D, 0x34, 0xB2, 0x3A, 0x08, 0xDA, + 0x29, 0x63, 0xD1, 0xA4, 0x83, 0x52, 0xB0, 0x63, + 0x1B, 0x89, 0x35, 0x57, 0x59, 0x2C, 0x0F, 0x72, + 0x72, 0xFD, 0xA0, 0xAC, 0xDB, 0xB4, 0xA3, 0xA1, + 0x18, 0x10, 0x12, 0x97, 0x99, 0x63, 0x38, 0x98, + 0x96, 0xB5, 0x16, 0x07, 0x4E, 0xE9, 0x2C, 0x97 +}; + +static const struct blockcipher_test_data +triple_des192ctr_test_vector = { + .crypto_algo = RTE_CRYPTO_CIPHER_3DES_CTR, + .cipher_key = { + .data = { + 0xE4, 0x23, 0x33, 0x8A, 0x35, 0x64, 0x61, 0xE2, + 0x49, 0x03, 0xDD, 0xC6, 0xB8, 0xCA, 0x55, 0x7A, + 0xD4, 0xC3, 0xA3, 0xAA, 0x33, 0x62, 0x61, 0xE0 + }, + .len = 24 + }, + .iv = { + .data = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07 + }, + .len = 8 + }, + .plaintext = { + .data = plaintext_des, + .len = 512 + }, + .ciphertext = { + .data = ciphertext512_des192ctr, + .len = 512 + } +}; + +static const struct blockcipher_test_data +triple_des192ctr_sha1_test_vector = { + .crypto_algo = RTE_CRYPTO_CIPHER_3DES_CTR, + .cipher_key = { + .data = { + 0xE4, 0x23, 0x33, 0x8A, 0x35, 0x64, 0x61, 0xE2, + 0x49, 0x03, 0xDD, 0xC6, 0xB8, 0xCA, 0x55, 0x7A, + 0xD4, 0xC3, 0xA3, 0xAA, 0x33, 0x62, 0x61, 0xE0 + }, + .len = 24 + }, + .iv = { + .data = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07 + }, + .len = 8 + }, + .plaintext = { + .data = plaintext_des, + .len = 512 + }, + .ciphertext = { + .data = ciphertext512_des192ctr, + .len = 512 + }, + .auth_algo = RTE_CRYPTO_AUTH_SHA1, + .digest = { + .data = { + 0xEA, 0x62, 0xB9, 0xB2, 0x78, 0x6C, 0x8E, 0xDB, + 0xA3, 0xB6, 0xFF, 0x23, 0x3A, 0x47, 0xD8, 0xC8, + 0xED, 0x5E, 0x20, 0x1D + }, + .len = 20 + } +}; + +static const struct blockcipher_test_data +triple_des192ctr_hmac_sha1_test_vector = { + .crypto_algo = RTE_CRYPTO_CIPHER_3DES_CTR, + .cipher_key = { + .data = { + 0xE4, 0x23, 0x33, 0x8A, 0x35, 0x64, 0x61, 0xE2, + 0x49, 0x03, 0xDD, 0xC6, 0xB8, 0xCA, 0x55, 0x7A, + 0xD4, 0xC3, 0xA3, 0xAA, 0x33, 0x62, 0x61, 0xE0 + }, + .len = 24 + }, + .iv = { + .data = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07 + }, + .len = 8 + }, + .plaintext = { + .data = plaintext_des, + .len = 512 + }, + .ciphertext = { + .data = ciphertext512_des192ctr, + .len = 512 + }, + .auth_algo = RTE_CRYPTO_AUTH_SHA1_HMAC, + .auth_key = { + .data = { + 0xF8, 0x2A, 0xC7, 0x54, 0xDB, 0x96, 0x18, 0xAA, + 0xC3, 0xA1, 0x53, 0xF6, 0x1F, 0x17, 0x60, 0xBD, + 0xDE, 0xF4, 0xDE, 0xAD + }, + .len = 20 + }, + .digest = { + .data = { + 0x32, 0xD5, 0x19, 0x8F, 0x79, 0x3A, 0xAA, 0x7B, + 0x70, 0x67, 0x4E, 0x63, 0x88, 0xA3, 0x9A, 0x82, + 0x07, 0x33, 0x12, 0x94 + }, + .len = 20 + } +}; + +static const uint8_t ciphertext512_des128cbc[] = { + 0x28, 0x2a, 0xff, 0x15, 0x5c, 0xdf, 0xd9, 0x6b, + 0x54, 0xbc, 0x7b, 0xfb, 0xc5, 0x64, 0x4d, 0xdd, + 0x3e, 0xf2, 0x9e, 0xb7, 0x53, 0x65, 0x37, 0x05, + 0xe0, 0xdf, 0xae, 0xf7, 0xc9, 0x27, 0xe4, 0xec, + 0x11, 0x27, 0xc2, 0x9e, 0x02, 0x4e, 0x03, 0x3b, + 0x33, 0xf2, 0x66, 0x08, 0x24, 0x5f, 0xab, 0xc2, + 0x7e, 0x21, 0x19, 0x5d, 0x51, 0xc3, 0xe2, 0x97, + 0x6f, 0x2e, 0xb4, 0xaa, 0x34, 0x70, 0x88, 0x78, + 0x4e, 0xe7, 0x3d, 0xe1, 0x9f, 0x87, 0x1c, 0x8b, + 0xac, 0x8d, 0xa1, 0x1a, 0xcd, 0xb0, 0xf8, 0xb6, + 0x24, 0x36, 0xe3, 0x8c, 0x07, 0xe7, 0xe4, 0x92, + 0x13, 0x86, 0x6f, 0x13, 0xec, 0x04, 0x5c, 0xe9, + 0xb9, 0xca, 0x45, 0x8a, 0x2c, 0x46, 0xda, 0x54, + 0x1d, 0xb5, 0x81, 0xb1, 0xcd, 0xf3, 0x7d, 0x11, + 0x6b, 0xb3, 0x0a, 0x45, 0xe5, 0x6e, 0x51, 0x3e, + 0x2c, 0xac, 0x7c, 0xbc, 0xa7, 0x7e, 0x22, 0x4d, + 0xe6, 0x02, 0xe3, 0x3f, 0x77, 0xd7, 0x73, 0x72, + 0x0e, 0xfb, 0x42, 0x85, 0x80, 0xdf, 0xa8, 0x91, + 0x60, 0x40, 0x48, 0xcd, 0x1b, 0xd9, 0xbf, 0x2f, + 0xf2, 0xdf, 0xd0, 0xbd, 0x3f, 0x82, 0xce, 0x15, + 0x9d, 0x6e, 0xc6, 0x59, 0x6f, 0x27, 0x0d, 0xf9, + 0x26, 0xe2, 0x11, 0x29, 0x50, 0xc3, 0x0a, 0xb7, + 0xde, 0x9d, 0xe9, 0x55, 0xa1, 0xe9, 0x01, 0x33, + 0x56, 0x51, 0xa7, 0x3a, 0x9e, 0x63, 0xc5, 0x08, + 0x01, 0x3b, 0x03, 0x4b, 0xc6, 0xc4, 0xa1, 0xc0, + 0xc0, 0xd0, 0x0e, 0x48, 0xe5, 0x4c, 0x55, 0x6b, + 0x4a, 0xc1, 0x0a, 0x24, 0x4b, 0xd0, 0x02, 0xf4, + 0x31, 0x63, 0x11, 0xbd, 0xa6, 0x1f, 0xf4, 0xae, + 0x23, 0x5a, 0x40, 0x7e, 0x0e, 0x4e, 0x63, 0x8b, + 0x66, 0x3d, 0x55, 0x46, 0x6e, 0x5c, 0x76, 0xa7, + 0x68, 0x31, 0xce, 0x5d, 0xca, 0xe2, 0xb4, 0xb0, + 0xc1, 0x1f, 0x66, 0x18, 0x75, 0x64, 0x73, 0xa9, + 0x9e, 0xd5, 0x0e, 0x0e, 0xf7, 0x77, 0x61, 0xf8, + 0x89, 0xc6, 0xcf, 0x0c, 0x41, 0xd3, 0x8f, 0xfd, + 0x22, 0x52, 0x4f, 0x94, 0x5c, 0x19, 0x11, 0x3a, + 0xb5, 0x63, 0xe8, 0x81, 0x33, 0x13, 0x54, 0x3c, + 0x93, 0x36, 0xb5, 0x5b, 0x51, 0xaf, 0x51, 0xa2, + 0x08, 0xae, 0x83, 0x15, 0x77, 0x07, 0x28, 0x0d, + 0x98, 0xe1, 0x2f, 0x69, 0x0e, 0xfb, 0x9a, 0x2e, + 0x27, 0x27, 0xb0, 0xd5, 0xce, 0xf8, 0x16, 0x55, + 0xfd, 0xaa, 0xd7, 0x1a, 0x1b, 0x2e, 0x4c, 0x86, + 0x7a, 0x6a, 0x90, 0xf7, 0x0a, 0x07, 0xd3, 0x81, + 0x4b, 0x75, 0x6a, 0x79, 0xdb, 0x63, 0x45, 0x0f, + 0x31, 0x7e, 0xd0, 0x2a, 0x14, 0xff, 0xee, 0xcc, + 0x97, 0x8a, 0x7d, 0x74, 0xbd, 0x9d, 0xaf, 0x00, + 0xdb, 0x7e, 0xf3, 0xe6, 0x22, 0x76, 0x77, 0x58, + 0xba, 0x1c, 0x06, 0x96, 0xfb, 0x6f, 0x41, 0x71, + 0x66, 0x98, 0xae, 0x31, 0x7d, 0x29, 0x18, 0x71, + 0x0e, 0xe4, 0x98, 0x7e, 0x59, 0x5a, 0xc9, 0x78, + 0x9c, 0xfb, 0x6c, 0x81, 0x44, 0xb4, 0x0f, 0x5e, + 0x18, 0x53, 0xb8, 0x6f, 0xbc, 0x3b, 0x15, 0xf0, + 0x10, 0xdd, 0x0d, 0x4b, 0x0a, 0x36, 0x0e, 0xb4, + 0x76, 0x0f, 0x16, 0xa7, 0x5c, 0x9d, 0xcf, 0xb0, + 0x6d, 0x38, 0x02, 0x07, 0x05, 0xe9, 0xe9, 0x46, + 0x08, 0xb8, 0x52, 0xd6, 0xd9, 0x4c, 0x81, 0x63, + 0x1d, 0xe2, 0x5b, 0xd0, 0xf6, 0x5e, 0x1e, 0x81, + 0x48, 0x08, 0x66, 0x3a, 0x85, 0xed, 0x65, 0xfe, + 0xe8, 0x05, 0x7a, 0xe1, 0xe6, 0x12, 0xf2, 0x52, + 0x83, 0xdd, 0x82, 0xbe, 0xf6, 0x34, 0x8a, 0x6f, + 0xc5, 0x83, 0xcd, 0x3f, 0xbe, 0x58, 0x8b, 0x11, + 0x78, 0xdc, 0x0c, 0x83, 0x72, 0x5d, 0x05, 0x2a, + 0x01, 0x29, 0xee, 0x48, 0x9a, 0x67, 0x00, 0x6e, + 0x14, 0x60, 0x2d, 0x00, 0x52, 0x87, 0x98, 0x5e, + 0x43, 0xfe, 0xf1, 0x10, 0x14, 0xf1, 0x91, 0xcc +}; + + +static const uint8_t ciphertext512_des[] = { + 0x1A, 0x46, 0xDB, 0x69, 0x43, 0x45, 0x0F, 0x2F, + 0xDC, 0x27, 0xF9, 0x41, 0x0E, 0x01, 0x58, 0xB4, + 0x5E, 0xCC, 0x13, 0xF5, 0x92, 0x99, 0xE4, 0xF2, + 0xD5, 0xF9, 0x16, 0xFE, 0x0F, 0x7E, 0xDE, 0xA0, + 0xF5, 0x32, 0xFE, 0x20, 0x67, 0x93, 0xCA, 0xE1, + 0x8E, 0x4D, 0x72, 0xA3, 0x50, 0x72, 0x14, 0x15, + 0x70, 0xE7, 0xAB, 0x49, 0x25, 0x88, 0x0E, 0x01, + 0x5C, 0x52, 0x87, 0xE2, 0x27, 0xDC, 0xD4, 0xD1, + 0x14, 0x1B, 0x08, 0x9F, 0x42, 0x48, 0x93, 0xA9, + 0xD1, 0x2F, 0x2C, 0x69, 0x48, 0x16, 0x59, 0xCF, + 0x8B, 0xF6, 0x8B, 0xD9, 0x34, 0xD4, 0xD7, 0xE4, + 0xAE, 0x35, 0xFD, 0xDA, 0x73, 0xBE, 0xDC, 0x6B, + 0x10, 0x90, 0x75, 0x2D, 0x4C, 0x14, 0x37, 0x8B, + 0xC8, 0xC7, 0xDF, 0x6E, 0x6F, 0xED, 0xF3, 0xE3, + 0xD3, 0x21, 0x29, 0xCD, 0x06, 0xB6, 0x5B, 0xF4, + 0xB9, 0xBD, 0x77, 0xA2, 0xF7, 0x91, 0xF4, 0x95, + 0xF0, 0xE0, 0x62, 0x03, 0x46, 0xAE, 0x1B, 0xEB, + 0xE2, 0xA9, 0xCF, 0xB9, 0x0E, 0x3B, 0xB9, 0xDA, + 0x5C, 0x1B, 0x45, 0x3F, 0xDD, 0xCC, 0xCC, 0xB3, + 0xF0, 0xDD, 0x36, 0x26, 0x11, 0x57, 0x97, 0xA7, + 0xF6, 0xF4, 0xE1, 0x4F, 0xBB, 0x31, 0xBB, 0x07, + 0x4B, 0xA3, 0xB4, 0x83, 0xF9, 0x23, 0xA1, 0xCD, + 0x8C, 0x1C, 0x76, 0x92, 0x45, 0xA5, 0xEB, 0x7D, + 0xEB, 0x22, 0x88, 0xB1, 0x9F, 0xFB, 0xE9, 0x06, + 0x8F, 0x67, 0xA6, 0x8A, 0xB7, 0x0B, 0xCD, 0x8F, + 0x34, 0x40, 0x4F, 0x4F, 0xAD, 0xA0, 0xF2, 0xDC, + 0x2C, 0x53, 0xE1, 0xCA, 0xA5, 0x7A, 0x03, 0xEF, + 0x08, 0x00, 0xCC, 0x52, 0xA6, 0xAB, 0x56, 0xD2, + 0xF1, 0xCD, 0xC7, 0xED, 0xBE, 0xCB, 0x78, 0x37, + 0x4B, 0x61, 0xA9, 0xD2, 0x3C, 0x8D, 0xCC, 0xFD, + 0x21, 0xFD, 0x0F, 0xE4, 0x4E, 0x3D, 0x6F, 0x8F, + 0x2A, 0xEC, 0x69, 0xFA, 0x20, 0x50, 0x99, 0x35, + 0xA1, 0xCC, 0x3B, 0xFD, 0xD6, 0xAC, 0xE9, 0xBE, + 0x14, 0xF1, 0xBC, 0x71, 0x70, 0xFE, 0x13, 0xD1, + 0x48, 0xCC, 0xBE, 0x7B, 0xCB, 0xC0, 0x20, 0xD9, + 0x28, 0xD7, 0xD4, 0x0F, 0x66, 0x7A, 0x60, 0xAB, + 0x20, 0xA9, 0x23, 0x41, 0x03, 0x34, 0xC3, 0x63, + 0x91, 0x69, 0x02, 0xD5, 0xBC, 0x41, 0xDA, 0xA8, + 0xD1, 0x48, 0xC9, 0x8E, 0x4F, 0xCD, 0x0F, 0x21, + 0x5B, 0x4D, 0x5F, 0xF5, 0x1B, 0x2A, 0x44, 0x10, + 0x16, 0xA7, 0xFD, 0xC0, 0x55, 0xE1, 0x98, 0xBB, + 0x76, 0xB5, 0xAB, 0x39, 0x6B, 0x9B, 0xAB, 0x85, + 0x45, 0x4B, 0x9C, 0x64, 0x7D, 0x78, 0x3F, 0x61, + 0x22, 0xB1, 0xDE, 0x0E, 0x39, 0x2B, 0x21, 0x26, + 0xE2, 0x1D, 0x5A, 0xD7, 0xAC, 0xDF, 0xD4, 0x12, + 0x69, 0xD1, 0xE8, 0x9B, 0x1A, 0xCE, 0x6C, 0xA0, + 0x3B, 0x23, 0xDC, 0x03, 0x2B, 0x97, 0x16, 0xD0, + 0xD0, 0x46, 0x98, 0x36, 0x53, 0xCE, 0x88, 0x6E, + 0xCA, 0x2C, 0x15, 0x0E, 0x49, 0xED, 0xBE, 0xE5, + 0xBF, 0xBD, 0x7B, 0xC2, 0x21, 0xE1, 0x09, 0xFF, + 0x71, 0xA8, 0xBE, 0x8F, 0xB4, 0x1D, 0x25, 0x5C, + 0x37, 0xCA, 0x26, 0xD2, 0x1E, 0x63, 0xE1, 0x7F, + 0x0D, 0x89, 0x10, 0xEF, 0x78, 0xB0, 0xDB, 0xD0, + 0x72, 0x44, 0x60, 0x1D, 0xCF, 0x7C, 0x25, 0x1A, + 0xBB, 0xC3, 0x92, 0x53, 0x8E, 0x9F, 0x27, 0xC7, + 0xE8, 0x08, 0xFC, 0x5D, 0x50, 0x3E, 0xFC, 0xB0, + 0x00, 0xE2, 0x48, 0xB2, 0x4B, 0xF8, 0xF2, 0xE3, + 0xD3, 0x8B, 0x71, 0x64, 0xB8, 0xF0, 0x6E, 0x4A, + 0x23, 0xA0, 0xA4, 0x88, 0xA4, 0x36, 0x45, 0x6B, + 0x5A, 0xE7, 0x57, 0x65, 0xEA, 0xC9, 0xF8, 0xE8, + 0x7A, 0x80, 0x22, 0x67, 0x1A, 0x05, 0xF2, 0x78, + 0x81, 0x17, 0xCD, 0x87, 0xFB, 0x0D, 0x25, 0x84, + 0x49, 0x06, 0x25, 0xCE, 0xFC, 0x38, 0x06, 0x18, + 0x2E, 0x1D, 0xE1, 0x33, 0x97, 0xB6, 0x7E, 0xAB, +}; + + +static const struct blockcipher_test_data +triple_des128cbc_test_vector = { + .crypto_algo = RTE_CRYPTO_CIPHER_3DES_CBC, + .cipher_key = { + .data = { + 0xE4, 0x23, 0x33, 0x8A, 0x35, 0x64, 0x61, 0xE2, + 0x49, 0x03, 0xDD, 0xC6, 0xB8, 0xCA, 0x55, 0x7A + }, + .len = 16 + }, + .iv = { + .data = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07 + }, + .len = 8 + }, + .plaintext = { + .data = plaintext_des, + .len = 512 + }, + .ciphertext = { + .data = ciphertext512_des128cbc, + .len = 512 + } +}; + +static const struct blockcipher_test_data +triple_des128cbc_sha1_test_vector = { + .crypto_algo = RTE_CRYPTO_CIPHER_3DES_CBC, + .cipher_key = { + .data = { + 0xE4, 0x23, 0x33, 0x8A, 0x35, 0x64, 0x61, 0xE2, + 0x49, 0x03, 0xDD, 0xC6, 0xB8, 0xCA, 0x55, 0x7A + }, + .len = 16 + }, + .iv = { + .data = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07 + }, + .len = 8 + }, + .plaintext = { + .data = plaintext_des, + .len = 512 + }, + .ciphertext = { + .data = ciphertext512_des128cbc, + .len = 512 + }, + .auth_algo = RTE_CRYPTO_AUTH_SHA1, + .digest = { + .data = { + 0x94, 0x45, 0x7B, 0xDF, 0xFE, 0x80, 0xB9, 0xA6, + 0xA0, 0x7A, 0xE8, 0x93, 0x40, 0x7B, 0x85, 0x02, + 0x1C, 0xD7, 0xE8, 0x87 + }, + .len = 20 + } +}; + +static const struct blockcipher_test_data +triple_des128cbc_hmac_sha1_test_vector = { + .crypto_algo = RTE_CRYPTO_CIPHER_3DES_CBC, + .cipher_key = { + .data = { + 0xE4, 0x23, 0x33, 0x8A, 0x35, 0x64, 0x61, 0xE2, + 0x49, 0x03, 0xDD, 0xC6, 0xB8, 0xCA, 0x55, 0x7A + }, + .len = 16 + }, + .iv = { + .data = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07 + }, + .len = 8 + }, + .plaintext = { + .data = plaintext_des, + .len = 512 + }, + .ciphertext = { + .data = ciphertext512_des128cbc, + .len = 512 + }, + .auth_algo = RTE_CRYPTO_AUTH_SHA1_HMAC, + .auth_key = { + .data = { + 0xF8, 0x2A, 0xC7, 0x54, 0xDB, 0x96, 0x18, 0xAA, + 0xC3, 0xA1, 0x53, 0xF6, 0x1F, 0x17, 0x60, 0xBD, + 0xDE, 0xF4, 0xDE, 0xAD + }, + .len = 20 + }, + .digest = { + .data = { + 0x7E, 0xBA, 0xFF, 0x86, 0x8D, 0x65, 0xCD, 0x08, + 0x76, 0x34, 0x94, 0xE9, 0x9A, 0xCD, 0xB2, 0xBB, + 0xBF, 0x65, 0xF5, 0x42 + }, + .len = 20 + } +}; + +static const uint8_t ciphertext512_des192cbc[] = { + 0xd0, 0xc9, 0xdc, 0x51, 0x29, 0x97, 0x03, 0x64, + 0xcd, 0x22, 0xba, 0x3d, 0x2b, 0xbc, 0x21, 0x37, + 0x7b, 0x1e, 0x29, 0x23, 0xeb, 0x51, 0x6e, 0xac, + 0xbe, 0x5b, 0xd3, 0x67, 0xe0, 0x3f, 0xc3, 0xb5, + 0xe3, 0x04, 0x17, 0x42, 0x2b, 0xaa, 0xdd, 0xd6, + 0x0e, 0x69, 0xd0, 0x8f, 0x8a, 0xfc, 0xb4, 0x55, + 0x67, 0x06, 0x51, 0xbb, 0x00, 0x57, 0xee, 0x95, + 0x28, 0x79, 0x3f, 0xd9, 0x97, 0x2b, 0xb0, 0x02, + 0x35, 0x08, 0xce, 0x7a, 0xc3, 0x43, 0x2c, 0x87, + 0xaa, 0x97, 0x6a, 0xad, 0xf0, 0x26, 0xea, 0x1d, + 0xbb, 0x08, 0xe9, 0x52, 0x11, 0xd3, 0xaf, 0x36, + 0x17, 0x14, 0x21, 0xb2, 0xbc, 0x42, 0x51, 0x33, + 0x27, 0x8c, 0xd8, 0x45, 0xb9, 0x76, 0xa0, 0x11, + 0x24, 0x34, 0xde, 0x4d, 0x13, 0x67, 0x1b, 0xc3, + 0x31, 0x12, 0x66, 0x56, 0x59, 0xd2, 0xb1, 0x8f, + 0xec, 0x1e, 0xc0, 0x10, 0x7a, 0x86, 0xb1, 0x60, + 0xc3, 0x01, 0xd6, 0xa8, 0x55, 0xad, 0x58, 0x63, + 0xca, 0x68, 0xa9, 0x33, 0xe3, 0x93, 0x90, 0x7d, + 0x8f, 0xca, 0xf8, 0x1c, 0xc2, 0x9e, 0xfb, 0xde, + 0x9c, 0xc7, 0xf2, 0x6c, 0xff, 0xcc, 0x39, 0x17, + 0x49, 0x33, 0x0d, 0x7c, 0xed, 0x07, 0x99, 0x91, + 0x91, 0x6c, 0x5f, 0x3f, 0x02, 0x09, 0xdc, 0x70, + 0xf9, 0x3b, 0x8d, 0xaa, 0xf4, 0xbc, 0x0e, 0xec, + 0xf2, 0x26, 0xfb, 0xb2, 0x1c, 0x31, 0xae, 0xc6, + 0x72, 0xe8, 0x0b, 0x75, 0x05, 0x57, 0x58, 0x98, + 0x92, 0x37, 0x27, 0x8e, 0x3b, 0x0c, 0x25, 0xfb, + 0xcf, 0x82, 0x02, 0xd5, 0x0b, 0x1f, 0x89, 0x49, + 0xcd, 0x0f, 0xa1, 0xa7, 0x08, 0x63, 0x56, 0xa7, + 0x1f, 0x80, 0x3a, 0xef, 0x24, 0x89, 0x57, 0x1a, + 0x02, 0xdc, 0x2e, 0x51, 0xbd, 0x4a, 0x10, 0x23, + 0xfc, 0x02, 0x1a, 0x3f, 0x34, 0xbf, 0x1c, 0x98, + 0x1a, 0x40, 0x0a, 0x96, 0x8e, 0x41, 0xd5, 0x09, + 0x55, 0x37, 0xe9, 0x25, 0x11, 0x83, 0xf8, 0xf3, + 0xd4, 0xb0, 0xdb, 0x16, 0xd7, 0x51, 0x7e, 0x94, + 0xf7, 0xb4, 0x26, 0xe0, 0xf4, 0x80, 0x01, 0x65, + 0x51, 0xeb, 0xbc, 0xb0, 0x65, 0x8f, 0xdd, 0xb5, + 0xf7, 0x00, 0xec, 0x40, 0xab, 0x7d, 0x96, 0xcc, + 0x8d, 0xec, 0x89, 0x80, 0x31, 0x39, 0xa2, 0x5c, + 0xb0, 0x55, 0x4c, 0xee, 0xdd, 0x15, 0x2b, 0xa9, + 0x86, 0x4e, 0x23, 0x14, 0x36, 0xc5, 0x57, 0xf5, + 0xe3, 0xe8, 0x89, 0xc9, 0xb7, 0xf8, 0xeb, 0x08, + 0xe5, 0x93, 0x12, 0x5c, 0x0f, 0x79, 0xa1, 0x86, + 0xe4, 0xc2, 0xeb, 0xa6, 0xa0, 0x50, 0x6a, 0xec, + 0xd3, 0xce, 0x50, 0x78, 0x4e, 0x4f, 0x93, 0xd8, + 0xdc, 0xb4, 0xec, 0x02, 0xe9, 0xbd, 0x17, 0x99, + 0x1e, 0x16, 0x4e, 0xd7, 0xb0, 0x07, 0x02, 0x55, + 0x63, 0x24, 0x4f, 0x7b, 0x8f, 0xc5, 0x7a, 0x12, + 0x29, 0xff, 0x5d, 0xc1, 0xe7, 0xae, 0x48, 0xc8, + 0x57, 0x53, 0xe7, 0xcd, 0x10, 0x6c, 0x19, 0xfc, + 0xcc, 0xb9, 0xb1, 0xbe, 0x48, 0x9f, 0x2d, 0x3f, + 0x39, 0x2e, 0xdd, 0x71, 0xde, 0x1b, 0x54, 0xee, + 0x7d, 0x94, 0x8f, 0x27, 0x23, 0xe9, 0x74, 0x92, + 0x14, 0x93, 0x84, 0x65, 0xc9, 0x22, 0x7c, 0xa8, + 0x1b, 0x72, 0x73, 0xb1, 0x23, 0xa0, 0x6b, 0xcc, + 0xb5, 0x22, 0x06, 0x15, 0xe5, 0x96, 0x03, 0x4a, + 0x52, 0x8d, 0x1d, 0xbf, 0x3e, 0x82, 0x45, 0x9c, + 0x75, 0x9e, 0xa9, 0x3a, 0x97, 0xb6, 0x5d, 0xc4, + 0x75, 0x67, 0xa1, 0xf3, 0x0f, 0x7a, 0xfd, 0x71, + 0x58, 0x04, 0xf9, 0xa7, 0xc2, 0x56, 0x74, 0x04, + 0x74, 0x68, 0x6d, 0x8a, 0xf6, 0x6c, 0x5d, 0xd8, + 0xb5, 0xed, 0x70, 0x23, 0x32, 0x4d, 0x75, 0x92, + 0x88, 0x7b, 0x39, 0x37, 0x02, 0x4b, 0xb2, 0x1c, + 0x1f, 0x7e, 0x5b, 0x1b, 0x10, 0xfc, 0x17, 0x21, + 0x66, 0x62, 0x63, 0xc2, 0xcd, 0x16, 0x96, 0x3e +}; + +static const struct blockcipher_test_data +triple_des192cbc_test_vector = { + .crypto_algo = RTE_CRYPTO_CIPHER_3DES_CBC, + .cipher_key = { + .data = { + 0xE4, 0x23, 0x33, 0x8A, 0x35, 0x64, 0x61, 0xE2, + 0x49, 0x03, 0xDD, 0xC6, 0xB8, 0xCA, 0x55, 0x7A, + 0xD4, 0xC3, 0xA3, 0xAA, 0x33, 0x62, 0x61, 0xE0 + }, + .len = 24 + }, + .iv = { + .data = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07 + }, + .len = 8 + }, + .plaintext = { + .data = plaintext_des, + .len = 512 + }, + .ciphertext = { + .data = ciphertext512_des192cbc, + .len = 512 + } +}; + +static const struct blockcipher_test_data +triple_des192cbc_sha1_test_vector = { + .crypto_algo = RTE_CRYPTO_CIPHER_3DES_CBC, + .cipher_key = { + .data = { + 0xE4, 0x23, 0x33, 0x8A, 0x35, 0x64, 0x61, 0xE2, + 0x49, 0x03, 0xDD, 0xC6, 0xB8, 0xCA, 0x55, 0x7A, + 0xD4, 0xC3, 0xA3, 0xAA, 0x33, 0x62, 0x61, 0xE0 + }, + .len = 24 + }, + .iv = { + .data = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07 + }, + .len = 8 + }, + .plaintext = { + .data = plaintext_des, + .len = 512 + }, + .ciphertext = { + .data = ciphertext512_des192cbc, + .len = 512 + }, + .auth_algo = RTE_CRYPTO_AUTH_SHA1, + .digest = { + .data = { + 0x53, 0x27, 0xC0, 0xE6, 0xD6, 0x1B, 0xD6, 0x45, + 0x94, 0x2D, 0xCE, 0x8B, 0x29, 0xA3, 0x52, 0x14, + 0xC1, 0x6B, 0x87, 0x99 + }, + .len = 20 + } +}; + +static const struct blockcipher_test_data +triple_des192cbc_hmac_sha1_test_vector = { + .crypto_algo = RTE_CRYPTO_CIPHER_3DES_CBC, + .cipher_key = { + .data = { + 0xE4, 0x23, 0x33, 0x8A, 0x35, 0x64, 0x61, 0xE2, + 0x49, 0x03, 0xDD, 0xC6, 0xB8, 0xCA, 0x55, 0x7A, + 0xD4, 0xC3, 0xA3, 0xAA, 0x33, 0x62, 0x61, 0xE0 + }, + .len = 24 + }, + .iv = { + .data = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07 + }, + .len = 8 + }, + .plaintext = { + .data = plaintext_des, + .len = 512 + }, + .ciphertext = { + .data = ciphertext512_des192cbc, + .len = 512 + }, + .auth_algo = RTE_CRYPTO_AUTH_SHA1_HMAC, + .auth_key = { + .data = { + 0xF8, 0x2A, 0xC7, 0x54, 0xDB, 0x96, 0x18, 0xAA, + 0xC3, 0xA1, 0x53, 0xF6, 0x1F, 0x17, 0x60, 0xBD, + 0xDE, 0xF4, 0xDE, 0xAD + }, + .len = 20 + }, + .digest = { + .data = { + 0xBA, 0xAC, 0x74, 0x19, 0x43, 0xB0, 0x72, 0xB8, + 0x08, 0xF5, 0x24, 0xC4, 0x09, 0xBD, 0x48, 0xC1, + 0x3C, 0x50, 0x1C, 0xDD + }, + .len = 20 + } +}; + +static const struct blockcipher_test_data +des_cbc_test_vector = { + .crypto_algo = RTE_CRYPTO_CIPHER_DES_CBC, + .cipher_key = { + .data = { + 0xE4, 0x23, 0x33, 0x8A, 0x35, 0x64, 0x61, 0xE2 + }, + .len = 8 + }, + .iv = { + .data = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + }, + .len = 8 + }, + .plaintext = { + .data = plaintext_des, + .len = 512 + }, + .ciphertext = { + .data = ciphertext512_des, + .len = 512 + }, +}; + +static const struct blockcipher_test_case des_cipheronly_test_cases[] = { + { + .test_descr = "DES-CBC Encryption", + .test_data = &des_cbc_test_vector, + .op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_QAT | + BLOCKCIPHER_TEST_TARGET_PMD_MB | + BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_MRVL + }, + { + .test_descr = "DES-CBC Decryption", + .test_data = &des_cbc_test_vector, + .op_mask = BLOCKCIPHER_TEST_OP_DECRYPT, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_QAT | + BLOCKCIPHER_TEST_TARGET_PMD_MB | + BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_MRVL + }, + +}; + +/* DES-DOCSIS-BPI test vectors */ + +static const uint8_t plaintext_des_docsis_bpi_cfb[] = { + 0x00, 0x01, 0x02, 0x88, 0xEE, 0x59, 0x7E +}; + +static const uint8_t ciphertext_des_docsis_bpi_cfb[] = { + 0x17, 0x86, 0xA8, 0x03, 0xA0, 0x85, 0x75 +}; + +static const uint8_t plaintext_des_docsis_bpi_cbc_cfb[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x91, + 0xD2, 0xD1, 0x9F +}; + +static const uint8_t ciphertext_des_docsis_bpi_cbc_cfb[] = { + 0x0D, 0xDA, 0x5A, 0xCB, 0xD0, 0x5E, 0x55, 0x67, + 0x51, 0x47, 0x46, 0x86, 0x8A, 0x71, 0xE5, 0x77, + 0xEF, 0xAC, 0x88 +}; + +/* Multiple of DES block size */ +static const struct blockcipher_test_data des_test_data_1 = { + .crypto_algo = RTE_CRYPTO_CIPHER_DES_DOCSISBPI, + .cipher_key = { + .data = { + 0xE4, 0x23, 0x33, 0x8A, 0x35, 0x64, 0x61, 0xE2 + }, + .len = 8 + }, + .iv = { + .data = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07 + }, + .len = 8 + }, + .plaintext = { + .data = plaintext_des, + .len = 512 + }, + .ciphertext = { + .data = ciphertext512_des, + .len = 512 + }, +}; + +/* Less than DES block size */ +static const struct blockcipher_test_data des_test_data_2 = { + .crypto_algo = RTE_CRYPTO_CIPHER_DES_DOCSISBPI, + .cipher_key = { + .data = { + + 0xE6, 0x60, 0x0F, 0xD8, 0x85, 0x2E, 0xF5, 0xAB + }, + .len = 8 + }, + .iv = { + .data = { + 0x81, 0x0E, 0x52, 0x8E, 0x1C, 0x5F, 0xDA, 0x1A + }, + .len = 8 + }, + .plaintext = { + .data = plaintext_des_docsis_bpi_cfb, + .len = 7 + }, + .ciphertext = { + .data = ciphertext_des_docsis_bpi_cfb, + .len = 7 + } +}; + +/* Not multiple of DES block size */ +static const struct blockcipher_test_data des_test_data_3 = { + .crypto_algo = RTE_CRYPTO_CIPHER_DES_DOCSISBPI, + .cipher_key = { + .data = { + 0xE6, 0x60, 0x0F, 0xD8, 0x85, 0x2E, 0xF5, 0xAB + }, + .len = 8 + }, + .iv = { + .data = { + 0x81, 0x0E, 0x52, 0x8E, 0x1C, 0x5F, 0xDA, 0x1A + }, + .len = 8 + }, + .plaintext = { + .data = plaintext_des_docsis_bpi_cbc_cfb, + .len = 19 + }, + .ciphertext = { + .data = ciphertext_des_docsis_bpi_cbc_cfb, + .len = 19 + } +}; +static const struct blockcipher_test_case des_docsis_test_cases[] = { + { + .test_descr = "DES-DOCSIS-BPI Full Block Encryption", + .test_data = &des_test_data_1, + .op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT | + BLOCKCIPHER_TEST_TARGET_PMD_MB + }, + { + .test_descr = "DES-DOCSIS-BPI Runt Block Encryption", + .test_data = &des_test_data_2, + .op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT | + BLOCKCIPHER_TEST_TARGET_PMD_MB + }, + { + .test_descr = "DES-DOCSIS-BPI Uneven Encryption", + .test_data = &des_test_data_3, + .op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT | + BLOCKCIPHER_TEST_TARGET_PMD_MB + }, + { + .test_descr = "DES-DOCSIS-BPI Full Block Decryption", + .test_data = &des_test_data_1, + .op_mask = BLOCKCIPHER_TEST_OP_DECRYPT, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT | + BLOCKCIPHER_TEST_TARGET_PMD_MB + }, + { + .test_descr = "DES-DOCSIS-BPI Runt Block Decryption", + .test_data = &des_test_data_2, + .op_mask = BLOCKCIPHER_TEST_OP_DECRYPT, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT | + BLOCKCIPHER_TEST_TARGET_PMD_MB + }, + { + .test_descr = "DES-DOCSIS-BPI Uneven Decryption", + .test_data = &des_test_data_3, + .op_mask = BLOCKCIPHER_TEST_OP_DECRYPT, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT | + BLOCKCIPHER_TEST_TARGET_PMD_MB + }, + { + .test_descr = "DES-DOCSIS-BPI OOP Full Block Encryption", + .test_data = &des_test_data_1, + .op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT, + .feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT + }, + { + .test_descr = "DES-DOCSIS-BPI OOP Runt Block Encryption", + .test_data = &des_test_data_2, + .op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT, + .feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT + }, + { + .test_descr = "DES-DOCSIS-BPI OOP Uneven Encryption", + .test_data = &des_test_data_3, + .op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT, + .feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT + }, + { + .test_descr = "DES-DOCSIS-BPI OOP Full Block Decryption", + .test_data = &des_test_data_1, + .op_mask = BLOCKCIPHER_TEST_OP_DECRYPT, + .feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT + }, + { + .test_descr = "DES-DOCSIS-BPI OOP Runt Block Decryption", + .test_data = &des_test_data_2, + .op_mask = BLOCKCIPHER_TEST_OP_DECRYPT, + .feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT + }, + { + .test_descr = "DES-DOCSIS-BPI OOP Uneven Decryption", + .test_data = &des_test_data_3, + .op_mask = BLOCKCIPHER_TEST_OP_DECRYPT, + .feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT + } +}; + +static const struct blockcipher_test_case triple_des_chain_test_cases[] = { + { + .test_descr = "3DES-128-CBC HMAC-SHA1 Encryption Digest", + .test_data = &triple_des128cbc_hmac_sha1_test_vector, + .op_mask = BLOCKCIPHER_TEST_OP_ENC_AUTH_GEN, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC + }, + { + .test_descr = "3DES-128-CBC HMAC-SHA1 Decryption Digest Verify", + .test_data = &triple_des128cbc_hmac_sha1_test_vector, + .op_mask = BLOCKCIPHER_TEST_OP_AUTH_VERIFY_DEC, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC + }, + { + .test_descr = "3DES-128-CBC SHA1 Encryption Digest", + .test_data = &triple_des128cbc_sha1_test_vector, + .op_mask = BLOCKCIPHER_TEST_OP_ENC_AUTH_GEN, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL + }, + { + .test_descr = "3DES-128-CBC SHA1 Decryption Digest Verify", + .test_data = &triple_des128cbc_sha1_test_vector, + .op_mask = BLOCKCIPHER_TEST_OP_AUTH_VERIFY_DEC, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL + }, + { + .test_descr = "3DES-192-CBC HMAC-SHA1 Encryption Digest", + .test_data = &triple_des192cbc_hmac_sha1_test_vector, + .op_mask = BLOCKCIPHER_TEST_OP_ENC_AUTH_GEN, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_MRVL + }, + { + .test_descr = "3DES-192-CBC HMAC-SHA1 Decryption Digest Verify", + .test_data = &triple_des192cbc_hmac_sha1_test_vector, + .op_mask = BLOCKCIPHER_TEST_OP_AUTH_VERIFY_DEC, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_MRVL + }, + { + .test_descr = "3DES-192-CBC SHA1 Encryption Digest", + .test_data = &triple_des192cbc_sha1_test_vector, + .op_mask = BLOCKCIPHER_TEST_OP_ENC_AUTH_GEN, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_MRVL + }, + { + .test_descr = "3DES-192-CBC SHA1 Decryption Digest Verify", + .test_data = &triple_des192cbc_sha1_test_vector, + .op_mask = BLOCKCIPHER_TEST_OP_AUTH_VERIFY_DEC, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_MRVL + }, + { + .test_descr = "3DES-128-CTR HMAC-SHA1 Encryption Digest", + .test_data = &triple_des128ctr_hmac_sha1_test_vector, + .op_mask = BLOCKCIPHER_TEST_OP_ENC_AUTH_GEN, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT + }, + { + .test_descr = "3DES-128-CTR HMAC-SHA1 Decryption Digest Verify", + .test_data = &triple_des128ctr_hmac_sha1_test_vector, + .op_mask = BLOCKCIPHER_TEST_OP_AUTH_VERIFY_DEC, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT + }, + { + .test_descr = "3DES-128-CTR SHA1 Encryption Digest", + .test_data = &triple_des128ctr_sha1_test_vector, + .op_mask = BLOCKCIPHER_TEST_OP_ENC_AUTH_GEN, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL + }, + { + .test_descr = "3DES-128-CTR SHA1 Decryption Digest Verify", + .test_data = &triple_des128ctr_sha1_test_vector, + .op_mask = BLOCKCIPHER_TEST_OP_AUTH_VERIFY_DEC, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL + }, + { + .test_descr = "3DES-192-CTR HMAC-SHA1 Encryption Digest", + .test_data = &triple_des192ctr_hmac_sha1_test_vector, + .op_mask = BLOCKCIPHER_TEST_OP_ENC_AUTH_GEN, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT + }, + { + .test_descr = "3DES-192-CTR HMAC-SHA1 Decryption Digest Verify", + .test_data = &triple_des192ctr_hmac_sha1_test_vector, + .op_mask = BLOCKCIPHER_TEST_OP_AUTH_VERIFY_DEC, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT + }, + { + .test_descr = "3DES-192-CTR SHA1 Encryption Digest", + .test_data = &triple_des192ctr_sha1_test_vector, + .op_mask = BLOCKCIPHER_TEST_OP_ENC_AUTH_GEN, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL + }, + { + .test_descr = "3DES-192-CTR SHA1 Decryption Digest Verify", + .test_data = &triple_des192ctr_sha1_test_vector, + .op_mask = BLOCKCIPHER_TEST_OP_AUTH_VERIFY_DEC, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL + }, + { + .test_descr = "3DES-128-CBC HMAC-SHA1 Encryption Digest OOP", + .test_data = &triple_des128cbc_hmac_sha1_test_vector, + .op_mask = BLOCKCIPHER_TEST_OP_ENC_AUTH_GEN, + .feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC + }, + { + .test_descr = "3DES-128-CBC HMAC-SHA1 Decryption Digest" + " Verify OOP", + .test_data = &triple_des128cbc_hmac_sha1_test_vector, + .op_mask = BLOCKCIPHER_TEST_OP_AUTH_VERIFY_DEC, + .feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC + }, + { + .test_descr = "3DES-128-CBC HMAC-SHA1 Encryption Digest" + " Sessionless", + .test_data = &triple_des128cbc_hmac_sha1_test_vector, + .op_mask = BLOCKCIPHER_TEST_OP_ENC_AUTH_GEN, + .feature_mask = BLOCKCIPHER_TEST_FEATURE_SESSIONLESS, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL + }, + { + .test_descr = + "3DES-128-CBC HMAC-SHA1 Decryption Digest" + " Verify Sessionless", + .test_data = &triple_des128cbc_hmac_sha1_test_vector, + .op_mask = BLOCKCIPHER_TEST_OP_AUTH_VERIFY_DEC, + .feature_mask = BLOCKCIPHER_TEST_FEATURE_SESSIONLESS, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL + }, +}; + +static const struct blockcipher_test_case triple_des_cipheronly_test_cases[] = { + { + .test_descr = "3DES-128-CBC Encryption", + .test_data = &triple_des128cbc_test_vector, + .op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC + }, + { + .test_descr = "3DES-128-CBC Decryption", + .test_data = &triple_des128cbc_test_vector, + .op_mask = BLOCKCIPHER_TEST_OP_DECRYPT, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC + }, + { + .test_descr = "3DES-192-CBC Encryption", + .test_data = &triple_des192cbc_test_vector, + .op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_MRVL + }, + { + .test_descr = "3DES-192-CBC Decryption", + .test_data = &triple_des192cbc_test_vector, + .op_mask = BLOCKCIPHER_TEST_OP_DECRYPT, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_MRVL + }, + { + .test_descr = "3DES-128-CTR Encryption", + .test_data = &triple_des128ctr_test_vector, + .op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT + }, + { + .test_descr = "3DES-128-CTR Decryption", + .test_data = &triple_des128ctr_test_vector, + .op_mask = BLOCKCIPHER_TEST_OP_DECRYPT, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT + }, + { + .test_descr = "3DES-192-CTR Encryption", + .test_data = &triple_des192ctr_test_vector, + .op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT + }, + { + .test_descr = "3DES-192-CTR Decryption", + .test_data = &triple_des192ctr_test_vector, + .op_mask = BLOCKCIPHER_TEST_OP_DECRYPT, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_QAT + } +}; + +#endif /* TEST_CRYPTODEV_DES_TEST_VECTORS_H_ */ diff --git a/dpdk/test/test/test_cryptodev_hash_test_vectors.h b/dpdk/test/test/test_cryptodev_hash_test_vectors.h new file mode 100644 index 00000000..3b840ce6 --- /dev/null +++ b/dpdk/test/test/test_cryptodev_hash_test_vectors.h @@ -0,0 +1,557 @@ +/* + * BSD LICENSE + * + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef TEST_CRYPTODEV_HASH_TEST_VECTORS_H_ +#define TEST_CRYPTODEV_HASH_TEST_VECTORS_H_ + +static const uint8_t plaintext_hash[] = { + "What a lousy earth! He wondered how many people " + "were destitute that same night even in his own " + "prosperous country, how many homes were " + "shanties, how many husbands were drunk and " + "wives socked, and how many children were " + "bullied, abused, or abandoned. How many " + "families hungered for food they could not " + "afford to buy? How many hearts were broken? How " + "many suicides would take place that same night, " + "how many people would go insane? How many " + "cockroaches and landlords would triumph? How " + "many winners were losers, successes failures, " + "and rich men poor men? How many wise guys were " + "stupid? How many happy endings were unhappy " + "endings? How many honest men were liars, brave " + "men cowards, loyal men traitors, how many " + "sainted men were corrupt, how many people in " + "positions of trust had sold their souls to " + "bodyguards, how many had never had souls? How " + "many straight-and-narrow paths were crooked " + "paths? How many best families were worst " + "families and how many good people were bad " + "people? When you added them all up and then " + "subtracted, you might be left with only the " + "children, and perhaps with Albert Einstein and " + "an old violinist or sculptor somewhere." +}; + +static const struct blockcipher_test_data +md5_test_vector = { + .auth_algo = RTE_CRYPTO_AUTH_MD5, + .ciphertext = { + .data = plaintext_hash, + .len = 512 + }, + .digest = { + .data = { + 0xB3, 0xE6, 0xBB, 0x50, 0x41, 0x35, 0x3C, 0x6B, + 0x7A, 0xFF, 0xD2, 0x64, 0xAF, 0xD5, 0x1C, 0xB2 + }, + .len = 16 + } +}; + +static const struct blockcipher_test_data +hmac_md5_test_vector = { + .auth_algo = RTE_CRYPTO_AUTH_MD5_HMAC, + .ciphertext = { + .data = plaintext_hash, + .len = 512 + }, + .auth_key = { + .data = { + 0xF8, 0x2A, 0xC7, 0x54, 0xDB, 0x96, 0x18, 0xAA, + 0xC3, 0xA1, 0x53, 0xF6, 0x1F, 0x17, 0x60, 0xBD + }, + .len = 16 + }, + .digest = { + .data = { + 0x50, 0xE8, 0xDE, 0xC5, 0xC1, 0x76, 0xAC, 0xAE, + 0x15, 0x4A, 0xF1, 0x7F, 0x7E, 0x04, 0x42, 0x9B + }, + .len = 16, + .truncated_len = 12 + } +}; + +static const struct blockcipher_test_data +sha1_test_vector = { + .auth_algo = RTE_CRYPTO_AUTH_SHA1, + .ciphertext = { + .data = plaintext_hash, + .len = 512 + }, + .digest = { + .data = { + 0xA2, 0x8D, 0x40, 0x78, 0xDD, 0x9F, 0xBB, 0xD5, + 0x35, 0x62, 0xFB, 0xFA, 0x93, 0xFD, 0x7D, 0x70, + 0xA6, 0x7D, 0x45, 0xCA + }, + .len = 20 + } +}; + +static const struct blockcipher_test_data +hmac_sha1_test_vector = { + .auth_algo = RTE_CRYPTO_AUTH_SHA1_HMAC, + .ciphertext = { + .data = plaintext_hash, + .len = 512 + }, + .auth_key = { + .data = { + 0xF8, 0x2A, 0xC7, 0x54, 0xDB, 0x96, 0x18, 0xAA, + 0xC3, 0xA1, 0x53, 0xF6, 0x1F, 0x17, 0x60, 0xBD, + 0xDE, 0xF4, 0xDE, 0xAD + }, + .len = 20 + }, + .digest = { + .data = { + 0xC4, 0xB7, 0x0E, 0x6B, 0xDE, 0xD1, 0xE7, 0x77, + 0x7E, 0x2E, 0x8F, 0xFC, 0x48, 0x39, 0x46, 0x17, + 0x3F, 0x91, 0x64, 0x59 + }, + .len = 20, + .truncated_len = 12 + } +}; + +static const struct blockcipher_test_data +sha224_test_vector = { + .auth_algo = RTE_CRYPTO_AUTH_SHA224, + .ciphertext = { + .data = plaintext_hash, + .len = 512 + }, + .digest = { + .data = { + 0x91, 0xE7, 0xCD, 0x75, 0x14, 0x9C, 0xA9, 0xE9, + 0x2E, 0x46, 0x12, 0x20, 0x22, 0xF9, 0x68, 0x28, + 0x39, 0x26, 0xDF, 0xB5, 0x78, 0x62, 0xB2, 0x6E, + 0x5E, 0x8F, 0x25, 0x84 + }, + .len = 28 + } +}; + +static const struct blockcipher_test_data +hmac_sha224_test_vector = { + .auth_algo = RTE_CRYPTO_AUTH_SHA224_HMAC, + .ciphertext = { + .data = plaintext_hash, + .len = 512 + }, + .auth_key = { + .data = { + 0xF8, 0x2A, 0xC7, 0x54, 0xDB, 0x96, 0x18, 0xAA, + 0xC3, 0xA1, 0x53, 0xF6, 0x1F, 0x17, 0x60, 0xBD, + 0xDE, 0xF4, 0xDE, 0xAD, 0x26, 0xEB, 0xAB, 0x92, + 0xFB, 0xBF, 0xB0, 0x8C + }, + .len = 28 + }, + .digest = { + .data = { + 0x70, 0x0F, 0x04, 0x4D, 0x22, 0x02, 0x7D, 0x31, + 0x36, 0xDA, 0x77, 0x19, 0xB9, 0x66, 0x37, 0x7B, + 0xF1, 0x8A, 0x63, 0xBB, 0x5D, 0x1D, 0xE3, 0x9F, + 0x92, 0xF6, 0xAA, 0x19 + }, + .len = 28, + .truncated_len = 14 + } +}; + +static const struct blockcipher_test_data +sha256_test_vector = { + .auth_algo = RTE_CRYPTO_AUTH_SHA256, + .ciphertext = { + .data = plaintext_hash, + .len = 512 + }, + .digest = { + .data = { + 0x7F, 0xF1, 0x0C, 0xF5, 0x90, 0x97, 0x19, 0x0F, + 0x00, 0xE4, 0x83, 0x01, 0xCA, 0x59, 0x00, 0x2E, + 0x1F, 0xC7, 0x84, 0xEE, 0x76, 0xA6, 0x39, 0x15, + 0x76, 0x2F, 0x87, 0xF9, 0x01, 0x06, 0xF3, 0xB7 + }, + .len = 32 + } +}; + +static const struct blockcipher_test_data +hmac_sha256_test_vector = { + .auth_algo = RTE_CRYPTO_AUTH_SHA256_HMAC, + .ciphertext = { + .data = plaintext_hash, + .len = 512 + }, + .auth_key = { + .data = { + 0xF8, 0x2A, 0xC7, 0x54, 0xDB, 0x96, 0x18, 0xAA, + 0xC3, 0xA1, 0x53, 0xF6, 0x1F, 0x17, 0x60, 0xBD, + 0xDE, 0xF4, 0xDE, 0xAD, 0x26, 0xEB, 0xAB, 0x92, + 0xFB, 0xBF, 0xB0, 0x8C, 0x29, 0x87, 0x90, 0xAC + }, + .len = 32 + }, + .digest = { + .data = { + 0xAF, 0x8F, 0x70, 0x1B, 0x4B, 0xAF, 0x34, 0xCB, + 0x02, 0x24, 0x48, 0x45, 0x83, 0x52, 0x8F, 0x22, + 0x06, 0x4D, 0x64, 0x09, 0x0A, 0xCC, 0x02, 0x77, + 0x71, 0x83, 0x48, 0x71, 0x07, 0x02, 0x25, 0x17 + }, + .len = 32, + .truncated_len = 16 + } +}; + +static const struct blockcipher_test_data +sha384_test_vector = { + .auth_algo = RTE_CRYPTO_AUTH_SHA384, + .ciphertext = { + .data = plaintext_hash, + .len = 512 + }, + .digest = { + .data = { + 0x1D, 0xE7, 0x3F, 0x55, 0x86, 0xFE, 0x48, 0x9F, + 0xAC, 0xC6, 0x85, 0x32, 0xFA, 0x8E, 0xA6, 0x77, + 0x25, 0x84, 0xA5, 0x98, 0x8D, 0x0B, 0x80, 0xF4, + 0xEB, 0x2C, 0xFB, 0x6C, 0xEA, 0x7B, 0xFD, 0xD5, + 0xAD, 0x41, 0xAB, 0x15, 0xB0, 0x03, 0x15, 0xEC, + 0x9E, 0x3D, 0xED, 0xCB, 0x80, 0x7B, 0xF4, 0xB6 + }, + .len = 48 + } +}; + +static const struct blockcipher_test_data +hmac_sha384_test_vector = { + .auth_algo = RTE_CRYPTO_AUTH_SHA384_HMAC, + .ciphertext = { + .data = plaintext_hash, + .len = 512 + }, + .auth_key = { + .data = { + 0xF8, 0x2A, 0xC7, 0x54, 0xDB, 0x96, 0x18, 0xAA, + 0xC3, 0xA1, 0x53, 0xF6, 0x1F, 0x17, 0x60, 0xBD, + 0xDE, 0xF4, 0xDE, 0xAD, 0x26, 0xEB, 0xAB, 0x92, + 0xFB, 0xBF, 0xB0, 0x8C, 0x29, 0x87, 0x90, 0xAC, + 0x39, 0x8B, 0x5C, 0x49, 0x68, 0x1E, 0x3A, 0x05, + 0xCC, 0x68, 0x5C, 0x76, 0xCB, 0x3C, 0x71, 0x89 + }, + .len = 48 + }, + .digest = { + .data = { + 0xE2, 0x83, 0x18, 0x55, 0xB5, 0x8D, 0x94, 0x9B, + 0x01, 0xB6, 0xE2, 0x57, 0x7A, 0x62, 0xF5, 0xF4, + 0xAB, 0x39, 0xF3, 0x3C, 0x28, 0xA0, 0x0F, 0xCC, + 0xEE, 0x1C, 0xF1, 0xF8, 0x69, 0xF1, 0x24, 0x3B, + 0x10, 0x90, 0x0A, 0xE3, 0xF0, 0x59, 0xDD, 0xC0, + 0x6F, 0xE6, 0x8C, 0x84, 0xD5, 0x03, 0xF8, 0x9E + }, + .len = 48, + .truncated_len = 24 + } +}; + +static const struct blockcipher_test_data +sha512_test_vector = { + .auth_algo = RTE_CRYPTO_AUTH_SHA512, + .ciphertext = { + .data = plaintext_hash, + .len = 512 + }, + .digest = { + .data = { + 0xB9, 0xBA, 0x28, 0x48, 0x3C, 0xC2, 0xD3, 0x65, + 0x4A, 0xD6, 0x00, 0x1D, 0xCE, 0x61, 0x64, 0x54, + 0x45, 0x8C, 0x64, 0x0E, 0xED, 0x0E, 0xD8, 0x1C, + 0x72, 0xCE, 0xD2, 0x44, 0x91, 0xC8, 0xEB, 0xC7, + 0x99, 0xC5, 0xCA, 0x89, 0x72, 0x64, 0x96, 0x41, + 0xC8, 0xEA, 0xB2, 0x4E, 0xD1, 0x21, 0x13, 0x49, + 0x64, 0x4E, 0x15, 0x68, 0x12, 0x67, 0x26, 0x0F, + 0x2C, 0x3C, 0x83, 0x25, 0x27, 0x86, 0xF0, 0xDB + }, + .len = 64 + } +}; + +static const struct blockcipher_test_data +hmac_sha512_test_vector = { + .auth_algo = RTE_CRYPTO_AUTH_SHA512_HMAC, + .ciphertext = { + .data = plaintext_hash, + .len = 512 + }, + .auth_key = { + .data = { + 0xF8, 0x2A, 0xC7, 0x54, 0xDB, 0x96, 0x18, 0xAA, + 0xC3, 0xA1, 0x53, 0xF6, 0x1F, 0x17, 0x60, 0xBD, + 0xDE, 0xF4, 0xDE, 0xAD, 0x26, 0xEB, 0xAB, 0x92, + 0xFB, 0xBF, 0xB0, 0x8C, 0x29, 0x87, 0x90, 0xAC, + 0x39, 0x8B, 0x5C, 0x49, 0x68, 0x1E, 0x3A, 0x05, + 0xCC, 0x68, 0x5C, 0x76, 0xCB, 0x3C, 0x71, 0x89, + 0xDE, 0xAA, 0x36, 0x44, 0x98, 0x93, 0x97, 0x1E, + 0x6D, 0x53, 0x83, 0x87, 0xB3, 0xB7, 0x56, 0x41 + }, + .len = 64 + }, + .digest = { + .data = { + 0xB8, 0x0B, 0x35, 0x97, 0x3F, 0x24, 0x3F, 0x05, + 0x2A, 0x7F, 0x2F, 0xD8, 0xD7, 0x56, 0x58, 0xAD, + 0x6F, 0x8D, 0x1F, 0x4C, 0x30, 0xF9, 0xA8, 0x29, + 0x7A, 0xE0, 0x8D, 0x88, 0xF5, 0x2E, 0x94, 0xF5, + 0x06, 0xF7, 0x5D, 0x57, 0x32, 0xA8, 0x49, 0x29, + 0xEA, 0x6B, 0x6D, 0x95, 0xBD, 0x76, 0xF5, 0x79, + 0x97, 0x37, 0x0F, 0xBE, 0xC2, 0x45, 0xA0, 0x87, + 0xAF, 0x24, 0x27, 0x0C, 0x78, 0xBA, 0xBE, 0x20 + }, + .len = 64, + .truncated_len = 32 + } +}; + +static const struct blockcipher_test_case hash_test_cases[] = { + { + .test_descr = "MD5 Digest", + .test_data = &md5_test_vector, + .op_mask = BLOCKCIPHER_TEST_OP_AUTH_GEN, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL + }, + { + .test_descr = "MD5 Digest Verify", + .test_data = &md5_test_vector, + .op_mask = BLOCKCIPHER_TEST_OP_AUTH_VERIFY, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL + }, + { + .test_descr = "HMAC-MD5 Digest", + .test_data = &hmac_md5_test_vector, + .op_mask = BLOCKCIPHER_TEST_OP_AUTH_GEN, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_MB | + BLOCKCIPHER_TEST_TARGET_PMD_SCHEDULER | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_QAT + }, + { + .test_descr = "HMAC-MD5 Digest Verify", + .test_data = &hmac_md5_test_vector, + .op_mask = BLOCKCIPHER_TEST_OP_AUTH_VERIFY, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_MB | + BLOCKCIPHER_TEST_TARGET_PMD_SCHEDULER | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_QAT + }, + { + .test_descr = "SHA1 Digest", + .test_data = &sha1_test_vector, + .op_mask = BLOCKCIPHER_TEST_OP_AUTH_GEN, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL + }, + { + .test_descr = "SHA1 Digest Verify", + .test_data = &sha1_test_vector, + .op_mask = BLOCKCIPHER_TEST_OP_AUTH_VERIFY, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL + }, + { + .test_descr = "HMAC-SHA1 Digest", + .test_data = &hmac_sha1_test_vector, + .op_mask = BLOCKCIPHER_TEST_OP_AUTH_GEN, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_MB | + BLOCKCIPHER_TEST_TARGET_PMD_SCHEDULER | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_QAT + }, + { + .test_descr = "HMAC-SHA1 Digest Verify", + .test_data = &hmac_sha1_test_vector, + .op_mask = BLOCKCIPHER_TEST_OP_AUTH_VERIFY, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_MB | + BLOCKCIPHER_TEST_TARGET_PMD_SCHEDULER | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_QAT + }, + { + .test_descr = "SHA224 Digest", + .test_data = &sha224_test_vector, + .op_mask = BLOCKCIPHER_TEST_OP_AUTH_GEN, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL + }, + { + .test_descr = "SHA224 Digest Verify", + .test_data = &sha224_test_vector, + .op_mask = BLOCKCIPHER_TEST_OP_AUTH_VERIFY, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL + }, + { + .test_descr = "HMAC-SHA224 Digest", + .test_data = &hmac_sha224_test_vector, + .op_mask = BLOCKCIPHER_TEST_OP_AUTH_GEN, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_MB | + BLOCKCIPHER_TEST_TARGET_PMD_SCHEDULER | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_QAT + }, + { + .test_descr = "HMAC-SHA224 Digest Verify", + .test_data = &hmac_sha224_test_vector, + .op_mask = BLOCKCIPHER_TEST_OP_AUTH_VERIFY, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_MB | + BLOCKCIPHER_TEST_TARGET_PMD_SCHEDULER | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_QAT + }, + { + .test_descr = "SHA256 Digest", + .test_data = &sha256_test_vector, + .op_mask = BLOCKCIPHER_TEST_OP_AUTH_GEN, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL + }, + { + .test_descr = "SHA256 Digest Verify", + .test_data = &sha256_test_vector, + .op_mask = BLOCKCIPHER_TEST_OP_AUTH_VERIFY, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL + }, + { + .test_descr = "HMAC-SHA256 Digest", + .test_data = &hmac_sha256_test_vector, + .op_mask = BLOCKCIPHER_TEST_OP_AUTH_GEN, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_MB | + BLOCKCIPHER_TEST_TARGET_PMD_SCHEDULER | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_QAT + }, + { + .test_descr = "HMAC-SHA256 Digest Verify", + .test_data = &hmac_sha256_test_vector, + .op_mask = BLOCKCIPHER_TEST_OP_AUTH_VERIFY, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_MB | + BLOCKCIPHER_TEST_TARGET_PMD_SCHEDULER | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_QAT + }, + { + .test_descr = "SHA384 Digest", + .test_data = &sha384_test_vector, + .op_mask = BLOCKCIPHER_TEST_OP_AUTH_GEN, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL + }, + { + .test_descr = "SHA384 Digest Verify", + .test_data = &sha384_test_vector, + .op_mask = BLOCKCIPHER_TEST_OP_AUTH_VERIFY, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL + }, + { + .test_descr = "HMAC-SHA384 Digest", + .test_data = &hmac_sha384_test_vector, + .op_mask = BLOCKCIPHER_TEST_OP_AUTH_GEN, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_MB | + BLOCKCIPHER_TEST_TARGET_PMD_SCHEDULER | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_QAT + }, + { + .test_descr = "HMAC-SHA384 Digest Verify", + .test_data = &hmac_sha384_test_vector, + .op_mask = BLOCKCIPHER_TEST_OP_AUTH_VERIFY, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_MB | + BLOCKCIPHER_TEST_TARGET_PMD_SCHEDULER | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_QAT + }, + { + .test_descr = "SHA512 Digest", + .test_data = &sha512_test_vector, + .op_mask = BLOCKCIPHER_TEST_OP_AUTH_GEN, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL + }, + { + .test_descr = "SHA512 Digest Verify", + .test_data = &sha512_test_vector, + .op_mask = BLOCKCIPHER_TEST_OP_AUTH_VERIFY, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL + }, + { + .test_descr = "HMAC-SHA512 Digest", + .test_data = &hmac_sha512_test_vector, + .op_mask = BLOCKCIPHER_TEST_OP_AUTH_GEN, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_MB | + BLOCKCIPHER_TEST_TARGET_PMD_SCHEDULER | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_QAT + }, + { + .test_descr = "HMAC-SHA512 Digest Verify", + .test_data = &hmac_sha512_test_vector, + .op_mask = BLOCKCIPHER_TEST_OP_AUTH_VERIFY, + .pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL | + BLOCKCIPHER_TEST_TARGET_PMD_MB | + BLOCKCIPHER_TEST_TARGET_PMD_SCHEDULER | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC | + BLOCKCIPHER_TEST_TARGET_PMD_QAT + }, +}; + +#endif /* TEST_CRYPTODEV_HASH_TEST_VECTORS_H_ */ diff --git a/dpdk/test/test/test_cryptodev_hmac_test_vectors.h b/dpdk/test/test/test_cryptodev_hmac_test_vectors.h new file mode 100644 index 00000000..d30215fd --- /dev/null +++ b/dpdk/test/test/test_cryptodev_hmac_test_vectors.h @@ -0,0 +1,121 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef APP_TEST_TEST_CRYPTODEV_HMAC_TEST_VECTORS_H_ +#define APP_TEST_TEST_CRYPTODEV_HMAC_TEST_VECTORS_H_ + +/* *** MD5 test vectors *** */ + +#define MD5_DIGEST_LEN 16 + +struct HMAC_MD5_vector { + struct { + uint8_t data[64]; + uint16_t len; + } key; + + struct { + uint8_t data[1024]; + uint16_t len; + } plaintext; + + struct { + uint8_t data[16]; + uint16_t len; + } auth_tag; +}; + +static const struct +HMAC_MD5_vector HMAC_MD5_test_case_1 = { + .key = { + .data = { + 0xF8, 0x2A, 0xC7, 0x54, 0xDB, 0x96, 0x18, 0xAA, + 0xC3, 0xA1, 0x53, 0xF6, 0x1F, 0x17, 0x60, 0xBD + }, + .len = 16 + }, + .plaintext = { + .data = { + 0x87, 0x4D, 0x61, 0x91, 0xB6, 0x20, 0xE3, 0x26, + 0x1B, 0xEF, 0x68, 0x64, 0x99, 0x0D, 0xB6, 0xCE, + 0x98, 0x06, 0xF6, 0x6B, 0x79, 0x70, 0xFD, 0xFF, + 0x86, 0x17, 0x18, 0x7B, 0xB9, 0xFF, 0xFD, 0xFF, + 0x5A, 0xE4, 0xDF, 0x3E, 0xDB, 0xD5, 0xD3, 0x5E, + 0x5B, 0x4F, 0x09, 0x02, 0x0D, 0xB0, 0x3E, 0xAB, + 0x1E, 0x03, 0x1D, 0xDA, 0x2F, 0xBE, 0x03, 0xD1, + 0x79, 0x21, 0x70, 0xA0, 0xF3, 0x00, 0x9C, 0xEE + }, + .len = 64 + }, + .auth_tag = { + .data = { + 0x67, 0x83, 0xE1, 0x0F, 0xB0, 0xBF, 0x33, 0x49, + 0x22, 0x04, 0x89, 0xDF, 0x86, 0xD0, 0x5F, 0x0C + }, + .len = MD5_DIGEST_LEN + } +}; + +static const struct +HMAC_MD5_vector HMAC_MD5_test_case_2 = { + .key = { + .data = { + 0xF8, 0x2A, 0xC7, 0x54, 0xDB, 0x96, 0x18, 0xAA, + 0xC3, 0xA1, 0x53, 0xF6, 0x1F, 0x17, 0x60, 0xBD, + 0xF8, 0x2A, 0xC7, 0x54, 0xDB, 0x96, 0x18, 0xAA, + 0xC3, 0xA1, 0x53, 0xF6, 0x1F, 0x17, 0x60, 0xBD + }, + .len = 32 + }, + .plaintext = { + .data = { + 0x87, 0x4D, 0x61, 0x91, 0xB6, 0x20, 0xE3, 0x26, + 0x1B, 0xEF, 0x68, 0x64, 0x99, 0x0D, 0xB6, 0xCE, + 0x98, 0x06, 0xF6, 0x6B, 0x79, 0x70, 0xFD, 0xFF, + 0x86, 0x17, 0x18, 0x7B, 0xB9, 0xFF, 0xFD, 0xFF, + 0x5A, 0xE4, 0xDF, 0x3E, 0xDB, 0xD5, 0xD3, 0x5E, + 0x5B, 0x4F, 0x09, 0x02, 0x0D, 0xB0, 0x3E, 0xAB, + 0x1E, 0x03, 0x1D, 0xDA, 0x2F, 0xBE, 0x03, 0xD1, + 0x79, 0x21, 0x70, 0xA0, 0xF3, 0x00, 0x9C, 0xEE + }, + .len = 64 + }, + .auth_tag = { + .data = { + 0x39, 0x24, 0x70, 0x7A, 0x30, 0x38, 0x1E, 0x2B, + 0x9F, 0x6B, 0xD9, 0x3C, 0xAD, 0xC2, 0x73, 0x52 + }, + .len = MD5_DIGEST_LEN + } +}; + +#endif /* APP_TEST_TEST_CRYPTODEV_HMAC_TEST_VECTORS_H_ */ diff --git a/dpdk/test/test/test_cryptodev_kasumi_hash_test_vectors.h b/dpdk/test/test/test_cryptodev_kasumi_hash_test_vectors.h new file mode 100644 index 00000000..a6dd4f5c --- /dev/null +++ b/dpdk/test/test/test_cryptodev_kasumi_hash_test_vectors.h @@ -0,0 +1,248 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef TEST_CRYPTODEV_KASUMI_HASH_TEST_VECTORS_H_ +#define TEST_CRYPTODEV_KASUMI_HASH_TEST_VECTORS_H_ + +struct kasumi_hash_test_data { + struct { + uint8_t data[16]; + unsigned len; + } key; + + /* + * Includes COUNT (4 bytes), FRESH (4 bytes), message + * and DIRECTION (1 bit), plus 1 0*, with enough 0s, + * so total length is multiple of 8 or 64 bits + */ + struct { + uint8_t data[2056]; + unsigned len; /* length must be in Bits */ + } plaintext; + + struct { + uint8_t data[64]; + unsigned len; + } digest; +}; + +struct kasumi_hash_test_data kasumi_hash_test_case_1 = { + .key = { + .data = { + 0x2B, 0xD6, 0x45, 0x9F, 0x82, 0xC5, 0xB3, 0x00, + 0x95, 0x2C, 0x49, 0x10, 0x48, 0x81, 0xFF, 0x48 + }, + .len = 16 + }, + .plaintext = { + .data = { + 0x38, 0xA6, 0xF0, 0x56, 0x05, 0xD2, 0xEC, 0x49, + 0x6B, 0x22, 0x77, 0x37, 0x29, 0x6F, 0x39, 0x3C, + 0x80, 0x79, 0x35, 0x3E, 0xDC, 0x87, 0xE2, 0xE8, + 0x05, 0xD2, 0xEC, 0x49, 0xA4, 0xF2, 0xD8, 0xE2 + }, + .len = 256 + }, + .digest = { + .data = {0xF6, 0x3B, 0xD7, 0x2C}, + .len = 4 + } +}; + +struct kasumi_hash_test_data kasumi_hash_test_case_2 = { + .key = { + .data = { + 0xD4, 0x2F, 0x68, 0x24, 0x28, 0x20, 0x1C, 0xAF, + 0xCD, 0x9F, 0x97, 0x94, 0x5E, 0x6D, 0xE7, 0xB7 + }, + .len = 16 + }, + .plaintext = { + .data = { + 0x3E, 0xDC, 0x87, 0xE2, 0xA4, 0xF2, 0xD8, 0xE2, + 0xB5, 0x92, 0x43, 0x84, 0x32, 0x8A, 0x4A, 0xE0, + 0x0B, 0x73, 0x71, 0x09, 0xF8, 0xB6, 0xC8, 0xDD, + 0x2B, 0x4D, 0xB6, 0x3D, 0xD5, 0x33, 0x98, 0x1C, + 0xEB, 0x19, 0xAA, 0xD5, 0x2A, 0x5B, 0x2B, 0xC3 + }, + .len = 320 + }, + .digest = { + .data = {0xA9, 0xDA, 0xF1, 0xFF}, + .len = 4 + } +}; + +struct kasumi_hash_test_data kasumi_hash_test_case_3 = { + .key = { + .data = { + 0xFD, 0xB9, 0xCF, 0xDF, 0x28, 0x93, 0x6C, 0xC4, + 0x83, 0xA3, 0x18, 0x69, 0xD8, 0x1B, 0x8F, 0xAB + }, + .len = 16 + }, + .plaintext = { + .data = { + 0x36, 0xAF, 0x61, 0x44, 0x98, 0x38, 0xF0, 0x3A, + 0x59, 0x32, 0xBC, 0x0A, 0xCE, 0x2B, 0x0A, 0xBA, + 0x33, 0xD8, 0xAC, 0x18, 0x8A, 0xC5, 0x4F, 0x34, + 0x6F, 0xAD, 0x10, 0xBF, 0x9D, 0xEE, 0x29, 0x20, + 0xB4, 0x3B, 0xD0, 0xC5, 0x3A, 0x91, 0x5C, 0xB7, + 0xDF, 0x6C, 0xAA, 0x72, 0x05, 0x3A, 0xBF, 0xF3, + 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + }, + .len = 448 + }, + .digest = { + .data = {0x15, 0x37, 0xD3, 0x16}, + .len = 4 + } +}; + +struct kasumi_hash_test_data kasumi_hash_test_case_4 = { + .key = { + .data = { + 0xC7, 0x36, 0xC6, 0xAA, 0xB2, 0x2B, 0xFF, 0xF9, + 0x1E, 0x26, 0x98, 0xD2, 0xE2, 0x2A, 0xD5, 0x7E + }, + .len = 16 + }, + .plaintext = { + .data = { + 0x14, 0x79, 0x3E, 0x41, 0x03, 0x97, 0xE8, 0xFD, + 0xD0, 0xA7, 0xD4, 0x63, 0xDF, 0x9F, 0xB2, 0xB2, + 0x78, 0x83, 0x3F, 0xA0, 0x2E, 0x23, 0x5A, 0xA1, + 0x72, 0xBD, 0x97, 0x0C, 0x14, 0x73, 0xE1, 0x29, + 0x07, 0xFB, 0x64, 0x8B, 0x65, 0x99, 0xAA, 0xA0, + 0xB2, 0x4A, 0x03, 0x86, 0x65, 0x42, 0x2B, 0x20, + 0xA4, 0x99, 0x27, 0x6A, 0x50, 0x42, 0x70, 0x09, + 0xC0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + }, + .len = 512 + }, + .digest = { + .data = {0xDD, 0x7D, 0xFA, 0xDD }, + .len = 4 + } +}; + +struct kasumi_hash_test_data kasumi_hash_test_case_5 = { + .key = { + .data = { + 0xF4, 0xEB, 0xEC, 0x69, 0xE7, 0x3E, 0xAF, 0x2E, + 0xB2, 0xCF, 0x6A, 0xF4, 0xB3, 0x12, 0x0F, 0xFD + }, + .len = 16 + }, + .plaintext = { + .data = { + 0x29, 0x6F, 0x39, 0x3C, 0x6B, 0x22, 0x77, 0x37, + 0x10, 0xBF, 0xFF, 0x83, 0x9E, 0x0C, 0x71, 0x65, + 0x8D, 0xBB, 0x2D, 0x17, 0x07, 0xE1, 0x45, 0x72, + 0x4F, 0x41, 0xC1, 0x6F, 0x48, 0xBF, 0x40, 0x3C, + 0x3B, 0x18, 0xE3, 0x8F, 0xD5, 0xD1, 0x66, 0x3B, + 0x6F, 0x6D, 0x90, 0x01, 0x93, 0xE3, 0xCE, 0xA8, + 0xBB, 0x4F, 0x1B, 0x4F, 0x5B, 0xE8, 0x22, 0x03, + 0x22, 0x32, 0xA7, 0x8D, 0x7D, 0x75, 0x23, 0x8D, + 0x5E, 0x6D, 0xAE, 0xCD, 0x3B, 0x43, 0x22, 0xCF, + 0x59, 0xBC, 0x7E, 0xA8, 0x4A, 0xB1, 0x88, 0x11, + 0xB5, 0xBF, 0xB7, 0xBC, 0x55, 0x3F, 0x4F, 0xE4, + 0x44, 0x78, 0xCE, 0x28, 0x7A, 0x14, 0x87, 0x99, + 0x90, 0xD1, 0x8D, 0x12, 0xCA, 0x79, 0xD2, 0xC8, + 0x55, 0x14, 0x90, 0x21, 0xCD, 0x5C, 0xE8, 0xCA, + 0x03, 0x71, 0xCA, 0x04, 0xFC, 0xCE, 0x14, 0x3E, + 0x3D, 0x7C, 0xFE, 0xE9, 0x45, 0x85, 0xB5, 0x88, + 0x5C, 0xAC, 0x46, 0x06, 0x8B, 0xC0, 0x00, 0x00 + }, + .len = 1088 + }, + .digest = { + .data = {0xC3, 0x83, 0x83, 0x9D}, + .len = 4 + } +}; + +struct kasumi_hash_test_data kasumi_hash_test_case_6 = { + .key = { + .data = { + 0x83, 0xFD, 0x23, 0xA2, 0x44, 0xA7, 0x4C, 0xF3, + 0x58, 0xDA, 0x30, 0x19, 0xF1, 0x72, 0x26, 0x35 + }, + .len = 16 + }, + .plaintext = { + .data = { + 0x36, 0xAF, 0x61, 0x44, 0x4F, 0x30, 0x2A, 0xD2, + 0x35, 0xC6, 0x87, 0x16, 0x63, 0x3C, 0x66, 0xFB, + 0x75, 0x0C, 0x26, 0x68, 0x65, 0xD5, 0x3C, 0x11, + 0xEA, 0x05, 0xB1, 0xE9, 0xFA, 0x49, 0xC8, 0x39, + 0x8D, 0x48, 0xE1, 0xEF, 0xA5, 0x90, 0x9D, 0x39, + 0x47, 0x90, 0x28, 0x37, 0xF5, 0xAE, 0x96, 0xD5, + 0xA0, 0x5B, 0xC8, 0xD6, 0x1C, 0xA8, 0xDB, 0xEF, + 0x1B, 0x13, 0xA4, 0xB4, 0xAB, 0xFE, 0x4F, 0xB1, + 0x00, 0x60, 0x45, 0xB6, 0x74, 0xBB, 0x54, 0x72, + 0x93, 0x04, 0xC3, 0x82, 0xBE, 0x53, 0xA5, 0xAF, + 0x05, 0x55, 0x61, 0x76, 0xF6, 0xEA, 0xA2, 0xEF, + 0x1D, 0x05, 0xE4, 0xB0, 0x83, 0x18, 0x1E, 0xE6, + 0x74, 0xCD, 0xA5, 0xA4, 0x85, 0xF7, 0x4D, 0x7A, + 0xC0 + }, + .len = 840 + }, + .digest = { + .data = {0x95, 0xAE, 0x41, 0xBA}, + .len = 4 + } +}; + +struct kasumi_hash_test_data kasumi_hash_test_case_7 = { + .key = { + .data = { + 0x5A, 0xCB, 0x1D, 0x64, 0x4C, 0x0D, 0x51, 0x20, + 0x4E, 0xA5, 0xF1, 0x45, 0x10, 0x10, 0xD8, 0x52 + }, + .len = 16 + }, + .plaintext = { + .data = { + 0x38, 0xA6, 0xF0, 0x56, 0x05, 0xD2, 0xEC, 0x49, + 0xAD, 0x9C, 0x44, 0x1F, 0x89, 0x0B, 0x38, 0xC4, + 0x57, 0xA4, 0x9D, 0x42, 0x14, 0x07, 0xE8, 0xC0 + }, + .len = 192 + }, + .digest = { + .data = {0x87, 0x5F, 0xE4, 0x89}, + .len = 4 + } +}; +#endif /* TEST_CRYPTODEV_KASUMI_HASH_TEST_VECTORS_H_ */ diff --git a/dpdk/test/test/test_cryptodev_kasumi_test_vectors.h b/dpdk/test/test/test_cryptodev_kasumi_test_vectors.h new file mode 100644 index 00000000..0d042d0b --- /dev/null +++ b/dpdk/test/test/test_cryptodev_kasumi_test_vectors.h @@ -0,0 +1,387 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef TEST_CRYPTODEV_KASUMI_TEST_VECTORS_H_ +#define TEST_CRYPTODEV_KASUMI_TEST_VECTORS_H_ + +struct kasumi_test_data { + struct { + uint8_t data[64]; + unsigned len; + } key; + + struct { + uint8_t data[64] __rte_aligned(16); + unsigned len; + } cipher_iv; + + /* + * Data may include COUNT (4 bytes), FRESH (4 bytes), + * DIRECTION (1 bit), plus 1 0*, with enough 0s, + * so total length is multiple of 8 or 64 bits + */ + struct { + uint8_t data[1024]; + unsigned len; /* length must be in Bits */ + } plaintext; + + struct { + unsigned len; + } validDataLenInBits; + + struct { + uint8_t data[1024]; + unsigned len; /* length must be in Bits */ + } ciphertext; + + struct { + unsigned len; + } validCipherLenInBits; + + struct { + unsigned int len; + } validCipherOffsetInBits; + + /* Actual length of data to be hashed */ + struct { + unsigned len; + } validAuthLenInBits; + + struct { + uint8_t data[64]; + unsigned len; + } digest; + +}; + +struct kasumi_test_data kasumi_test_case_1 = { + .key = { + .data = { + 0x2B, 0xD6, 0x45, 0x9F, 0x82, 0xC5, 0xB3, 0x00, + 0x95, 0x2C, 0x49, 0x10, 0x48, 0x81, 0xFF, 0x48 + }, + .len = 16 + }, + .cipher_iv = { + .data = { + 0x72, 0xA4, 0xF2, 0x0F, 0x64, 0x00, 0x00, 0x00 + }, + .len = 8 + }, + .plaintext = { + .data = { + 0x7E, 0xC6, 0x12, 0x72, 0x74, 0x3B, 0xF1, 0x61, + 0x47, 0x26, 0x44, 0x6A, 0x6C, 0x38, 0xCE, 0xD1, + 0x66, 0xF6, 0xCA, 0x76, 0xEB, 0x54, 0x30, 0x04, + 0x42, 0x86, 0x34, 0x6C, 0xEF, 0x13, 0x0F, 0x92, + 0x92, 0x2B, 0x03, 0x45, 0x0D, 0x3A, 0x99, 0x75, + 0xE5, 0xBD, 0x2E, 0xA0, 0xEB, 0x55, 0xAD, 0x8E, + 0x1B, 0x19, 0x9E, 0x3E, 0xC4, 0x31, 0x60, 0x20, + 0xE9, 0xA1, 0xB2, 0x85, 0xE7, 0x62, 0x79, 0x53, + 0x59, 0xB7, 0xBD, 0xFD, 0x39, 0xBE, 0xF4, 0xB2, + 0x48, 0x45, 0x83, 0xD5, 0xAF, 0xE0, 0x82, 0xAE, + 0xE6, 0x38, 0xBF, 0x5F, 0xD5, 0xA6, 0x06, 0x19, + 0x39, 0x01, 0xA0, 0x8F, 0x4A, 0xB4, 0x1A, 0xAB, + 0x9B, 0x13, 0x48, 0x80 + }, + .len = 800 + }, + .ciphertext = { + .data = { + 0xD1, 0xE2, 0xDE, 0x70, 0xEE, 0xF8, 0x6C, 0x69, + 0x64, 0xFB, 0x54, 0x2B, 0xC2, 0xD4, 0x60, 0xAA, + 0xBF, 0xAA, 0x10, 0xA4, 0xA0, 0x93, 0x26, 0x2B, + 0x7D, 0x19, 0x9E, 0x70, 0x6F, 0xC2, 0xD4, 0x89, + 0x15, 0x53, 0x29, 0x69, 0x10, 0xF3, 0xA9, 0x73, + 0x01, 0x26, 0x82, 0xE4, 0x1C, 0x4E, 0x2B, 0x02, + 0xBE, 0x20, 0x17, 0xB7, 0x25, 0x3B, 0xBF, 0x93, + 0x09, 0xDE, 0x58, 0x19, 0xCB, 0x42, 0xE8, 0x19, + 0x56, 0xF4, 0xC9, 0x9B, 0xC9, 0x76, 0x5C, 0xAF, + 0x53, 0xB1, 0xD0, 0xBB, 0x82, 0x79, 0x82, 0x6A, + 0xDB, 0xBC, 0x55, 0x22, 0xE9, 0x15, 0xC1, 0x20, + 0xA6, 0x18, 0xA5, 0xA7, 0xF5, 0xE8, 0x97, 0x08, + 0x93, 0x39, 0x65, 0x0F + }, + .len = 800 + }, + .validCipherLenInBits = { + .len = 798 + }, + .validCipherOffsetInBits = { + .len = 0 + } +}; + +struct kasumi_test_data kasumi_test_case_2 = { + .key = { + .data = { + 0xEF, 0xA8, 0xB2, 0x22, 0x9E, 0x72, 0x0C, 0x2A, + 0x7C, 0x36, 0xEA, 0x55, 0xE9, 0x60, 0x56, 0x95 + }, + .len = 16 + }, + .cipher_iv = { + .data = { + 0xE2, 0x8B, 0xCF, 0x7B, 0xC0, 0x00, 0x00, 0x00 + }, + .len = 8 + }, + .plaintext = { + .data = { + 0x10, 0x11, 0x12, 0x31, 0xE0, 0x60, 0x25, 0x3A, + 0x43, 0xFD, 0x3F, 0x57, 0xE3, 0x76, 0x07, 0xAB, + 0x28, 0x27, 0xB5, 0x99, 0xB6, 0xB1, 0xBB, 0xDA, + 0x37, 0xA8, 0xAB, 0xCC, 0x5A, 0x8C, 0x55, 0x0D, + 0x1B, 0xFB, 0x2F, 0x49, 0x46, 0x24, 0xFB, 0x50, + 0x36, 0x7F, 0xA3, 0x6C, 0xE3, 0xBC, 0x68, 0xF1, + 0x1C, 0xF9, 0x3B, 0x15, 0x10, 0x37, 0x6B, 0x02, + 0x13, 0x0F, 0x81, 0x2A, 0x9F, 0xA1, 0x69, 0xD8 + }, + .len = 512 + }, + .ciphertext = { + .data = { + 0x3D, 0xEA, 0xCC, 0x7C, 0x15, 0x82, 0x1C, 0xAA, + 0x89, 0xEE, 0xCA, 0xDE, 0x9B, 0x5B, 0xD3, 0x61, + 0x4B, 0xD0, 0xC8, 0x41, 0x9D, 0x71, 0x03, 0x85, + 0xDD, 0xBE, 0x58, 0x49, 0xEF, 0x1B, 0xAC, 0x5A, + 0xE8, 0xB1, 0x4A, 0x5B, 0x0A, 0x67, 0x41, 0x52, + 0x1E, 0xB4, 0xE0, 0x0B, 0xB9, 0xEC, 0xF3, 0xE9, + 0xF7, 0xCC, 0xB9, 0xCA, 0xE7, 0x41, 0x52, 0xD7, + 0xF4, 0xE2, 0xA0, 0x34, 0xB6, 0xEA, 0x00, 0xEC + }, + .len = 512 + }, + .validCipherLenInBits = { + .len = 510 + }, + .validCipherOffsetInBits = { + .len = 0 + } +}; + +struct kasumi_test_data kasumi_test_case_3 = { + .key = { + .data = { + 0x5A, 0xCB, 0x1D, 0x64, 0x4C, 0x0D, 0x51, 0x20, + 0x4E, 0xA5, 0xF1, 0x45, 0x10, 0x10, 0xD8, 0x52 + }, + .len = 16 + }, + .cipher_iv = { + .data = { + 0xFA, 0x55, 0x6B, 0x26, 0x1C, 0x00, 0x00, 0x00 + }, + .len = 8 + }, + .plaintext = { + .data = { + 0x38, 0xA6, 0xF0, 0x56, 0x05, 0xD2, 0xEC, 0x49, + 0xAD, 0x9C, 0x44, 0x1F, 0x89, 0x0B, 0x38, 0xC4, + 0x57, 0xA4, 0x9D, 0x42, 0x14, 0x07, 0xE8, 0xC0 + }, + .len = 192 + }, + .ciphertext = { + .data = { + 0x38, 0xA6, 0xF0, 0x56, 0x05, 0xD2, 0xEC, 0x49, + 0x9B, 0xC9, 0x2C, 0xA8, 0x03, 0xC6, 0x7B, 0x28, + 0xA1, 0x1A, 0x4B, 0xEE, 0x5A, 0x0C, 0x25, 0xC0 + }, + .len = 192 + }, + .validDataLenInBits = { + .len = 192 + }, + .validCipherLenInBits = { + .len = 120 + }, + .validAuthLenInBits = { + .len = 192 + }, + .validCipherOffsetInBits = { + .len = 64 + }, + .digest = { + .data = {0x87, 0x5F, 0xE4, 0x89}, + .len = 4 + } +}; + +struct kasumi_test_data kasumi_test_case_4 = { + .key = { + .data = { + 0xD3, 0xC5, 0xD5, 0x92, 0x32, 0x7F, 0xB1, 0x1C, + 0x40, 0x35, 0xC6, 0x68, 0x0A, 0xF8, 0xC6, 0xD1 + }, + .len = 16 + }, + .cipher_iv = { + .data = { + 0x39, 0x8A, 0x59, 0xB4, 0x2C, 0x00, 0x00, 0x00, + }, + .len = 8 + }, + .plaintext = { + .data = { + 0x98, 0x1B, 0xA6, 0x82, 0x4C, 0x1B, 0xFB, 0x1A, + 0xB4, 0x85, 0x47, 0x20, 0x29, 0xB7, 0x1D, 0x80, + 0x8C, 0xE3, 0x3E, 0x2C, 0xC3, 0xC0, 0xB5, 0xFC, + 0x1F, 0x3D, 0xE8, 0xA6, 0xDC, 0x66, 0xB1, 0xF0 + }, + .len = 256 + }, + .ciphertext = { + .data = { + 0x5B, 0xB9, 0x43, 0x1B, 0xB1, 0xE9, 0x8B, 0xD1, + 0x1B, 0x93, 0xDB, 0x7C, 0x3D, 0x45, 0x13, 0x65, + 0x59, 0xBB, 0x86, 0xA2, 0x95, 0xAA, 0x20, 0x4E, + 0xCB, 0xEB, 0xF6, 0xF7, 0xA5, 0x10, 0x15, 0x10 + }, + .len = 256 + }, + .validCipherLenInBits = { + .len = 253 + }, + .validCipherOffsetInBits = { + .len = 0 + } +}; + +struct kasumi_test_data kasumi_test_case_5 = { + .key = { + .data = { + 0x60, 0x90, 0xEA, 0xE0, 0x4C, 0x83, 0x70, 0x6E, + 0xEC, 0xBF, 0x65, 0x2B, 0xE8, 0xE3, 0x65, 0x66 + }, + .len = 16 + }, + .cipher_iv = { + .data = { + 0x72, 0xA4, 0xF2, 0x0F, 0x48, 0x00, 0x00, 0x00 + }, + .len = 8 + }, + .plaintext = { + .data = { + 0x40, 0x98, 0x1B, 0xA6, 0x82, 0x4C, 0x1B, 0xFB, + 0x42, 0x86, 0xB2, 0x99, 0x78, 0x3D, 0xAF, 0x44, + 0x2C, 0x09, 0x9F, 0x7A, 0xB0, 0xF5, 0x8D, 0x5C, + 0x8E, 0x46, 0xB1, 0x04, 0xF0, 0x8F, 0x01, 0xB4, + 0x1A, 0xB4, 0x85, 0x47, 0x20, 0x29, 0xB7, 0x1D, + 0x36, 0xBD, 0x1A, 0x3D, 0x90, 0xDC, 0x3A, 0x41, + 0xB4, 0x6D, 0x51, 0x67, 0x2A, 0xC4, 0xC9, 0x66, + 0x3A, 0x2B, 0xE0, 0x63, 0xDA, 0x4B, 0xC8, 0xD2, + 0x80, 0x8C, 0xE3, 0x3E, 0x2C, 0xCC, 0xBF, 0xC6, + 0x34, 0xE1, 0xB2, 0x59, 0x06, 0x08, 0x76, 0xA0, + 0xFB, 0xB5, 0xA4, 0x37, 0xEB, 0xCC, 0x8D, 0x31, + 0xC1, 0x9E, 0x44, 0x54, 0x31, 0x87, 0x45, 0xE3, + 0x98, 0x76, 0x45, 0x98, 0x7A, 0x98, 0x6F, 0x2C, + 0xB0 + }, + .len = 840 + }, + .ciphertext = { + .data = { + 0xDD, 0xB3, 0x64, 0xDD, 0x2A, 0xAE, 0xC2, 0x4D, + 0xFF, 0x29, 0x19, 0x57, 0xB7, 0x8B, 0xAD, 0x06, + 0x3A, 0xC5, 0x79, 0xCD, 0x90, 0x41, 0xBA, 0xBE, + 0x89, 0xFD, 0x19, 0x5C, 0x05, 0x78, 0xCB, 0x9F, + 0xDE, 0x42, 0x17, 0x56, 0x61, 0x78, 0xD2, 0x02, + 0x40, 0x20, 0x6D, 0x07, 0xCF, 0xA6, 0x19, 0xEC, + 0x05, 0x9F, 0x63, 0x51, 0x44, 0x59, 0xFC, 0x10, + 0xD4, 0x2D, 0xC9, 0x93, 0x4E, 0x56, 0xEB, 0xC0, + 0xCB, 0xC6, 0x0D, 0x4D, 0x2D, 0xF1, 0x74, 0x77, + 0x4C, 0xBD, 0xCD, 0x5D, 0xA4, 0xA3, 0x50, 0x31, + 0x7A, 0x7F, 0x12, 0xE1, 0x94, 0x94, 0x71, 0xF8, + 0xA2, 0x95, 0xF2, 0x72, 0xE6, 0x8F, 0xC0, 0x71, + 0x59, 0xB0, 0x7D, 0x8E, 0x2D, 0x26, 0xE4, 0x59, + 0x9E + }, + .len = 840 + }, + .validCipherLenInBits = { + .len = 837 + }, + .validCipherOffsetInBits = { + .len = 0 + } +}; + +struct kasumi_test_data kasumi_test_case_6 = { + .key = { + .data = { + 0x5A, 0xCB, 0x1D, 0x64, 0x4C, 0x0D, 0x51, 0x20, + 0x4E, 0xA5, 0xF1, 0x45, 0x10, 0x10, 0xD8, 0x52 + }, + .len = 16 + }, + .cipher_iv = { + .data = { + 0xFA, 0x55, 0x6B, 0x26, 0x1C, 0x00, 0x00, 0x00 + }, + .len = 8 + }, + .plaintext = { + .data = { + 0x38, 0xA6, 0xF0, 0x56, 0x05, 0xD2, 0xEC, 0x49, + 0xAD, 0x9C, 0x44, 0x1F, 0x89, 0x0B, 0x38, 0xC4, + 0x57, 0xA4, 0x9D, 0x42, 0x14, 0x07, 0xE8, 0xC0 + }, + .len = 192 + }, + .ciphertext = { + .data = { + 0x38, 0xA6, 0xF0, 0x56, 0x05, 0xD2, 0xEC, 0x49, + 0x9B, 0xC9, 0x2C, 0xA8, 0x03, 0xC6, 0x7B, 0x28, + 0xA1, 0x1A, 0x4B, 0xEE, 0x5A, 0x0C, 0x25, 0xC0 + }, + .len = 192 + }, + .validDataLenInBits = { + .len = 192 + }, + .validCipherLenInBits = { + .len = 120 + }, + .validCipherOffsetInBits = { + .len = 64 + }, + .validAuthLenInBits = { + .len = 192 + }, + .digest = { + .data = {0x0F, 0xD2, 0xAA, 0xB5}, + .len = 4 + } +}; +#endif /* TEST_CRYPTODEV_KASUMI_TEST_VECTORS_H_ */ diff --git a/dpdk/test/test/test_cryptodev_snow3g_hash_test_vectors.h b/dpdk/test/test/test_cryptodev_snow3g_hash_test_vectors.h new file mode 100644 index 00000000..0bb274d1 --- /dev/null +++ b/dpdk/test/test/test_cryptodev_snow3g_hash_test_vectors.h @@ -0,0 +1,526 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef TEST_CRYPTODEV_SNOW3G_HASH_TEST_VECTORS_H_ +#define TEST_CRYPTODEV_SNOW3G_HASH_TEST_VECTORS_H_ + +struct snow3g_hash_test_data { + struct { + uint8_t data[64]; + unsigned len; + } key; + + struct { + uint8_t data[64]; + unsigned len; + } auth_iv; + + struct { + uint8_t data[2056]; + unsigned len; /* length must be in Bits */ + } plaintext; + + struct { + unsigned len; + } validAuthLenInBits; + + struct { + uint8_t data[64]; + unsigned len; + } digest; +}; + +struct snow3g_hash_test_data snow3g_hash_test_case_1 = { + .key = { + .data = { + 0xC7, 0x36, 0xC6, 0xAA, 0xB2, 0x2B, 0xFF, 0xF9, + 0x1E, 0x26, 0x98, 0xD2, 0xE2, 0x2A, 0xD5, 0x7E + }, + .len = 16 + }, + .auth_iv = { + .data = { + 0x14, 0x79, 0x3E, 0x41, 0x03, 0x97, 0xE8, 0xFD, + 0x94, 0x79, 0x3E, 0x41, 0x03, 0x97, 0x68, 0xFD + }, + .len = 16 + }, + .plaintext = { + .data = { + 0xD0, 0xA7, 0xD4, 0x63, 0xDF, 0x9F, 0xB2, 0xB2, + 0x78, 0x83, 0x3F, 0xA0, 0x2E, 0x23, 0x5A, 0xA1, + 0x72, 0xBD, 0x97, 0x0C, 0x14, 0x73, 0xE1, 0x29, + 0x07, 0xFB, 0x64, 0x8B, 0x65, 0x99, 0xAA, 0xA0, + 0xB2, 0x4A, 0x03, 0x86, 0x65, 0x42, 0x2B, 0x20, + 0xA4, 0x99, 0x27, 0x6A, 0x50, 0x42, 0x70, 0x09 + }, + .len = 384 + }, + .validAuthLenInBits = { + .len = 384 + }, + .digest = { + .data = {0x38, 0xB5, 0x54, 0xC0 }, + .len = 4 + } +}; + +struct snow3g_hash_test_data snow3g_hash_test_case_2 = { + .key = { + .data = { + 0xF4, 0xEB, 0xEC, 0x69, 0xE7, 0x3E, 0xAF, 0x2E, + 0xB2, 0xCF, 0x6A, 0xF4, 0xB3, 0x12, 0x0F, 0xFD + }, + .len = 16 + }, + .auth_iv = { + .data = { + 0x29, 0x6F, 0x39, 0x3C, 0x6B, 0x22, 0x77, 0x37, + 0xA9, 0x6F, 0x39, 0x3C, 0x6B, 0x22, 0xF7, 0x37 + }, + .len = 16 + }, + .plaintext = { + .data = { + 0x10, 0xBF, 0xFF, 0x83, 0x9E, 0x0C, 0x71, 0x65, + 0x8D, 0xBB, 0x2D, 0x17, 0x07, 0xE1, 0x45, 0x72, + 0x4F, 0x41, 0xC1, 0x6F, 0x48, 0xBF, 0x40, 0x3C, + 0x3B, 0x18, 0xE3, 0x8F, 0xD5, 0xD1, 0x66, 0x3B, + 0x6F, 0x6D, 0x90, 0x01, 0x93, 0xE3, 0xCE, 0xA8, + 0xBB, 0x4F, 0x1B, 0x4F, 0x5B, 0xE8, 0x22, 0x03, + 0x22, 0x32, 0xA7, 0x8D, 0x7D, 0x75, 0x23, 0x8D, + 0x5E, 0x6D, 0xAE, 0xCD, 0x3B, 0x43, 0x22, 0xCF, + 0x59, 0xBC, 0x7E, 0xA8, 0x4A, 0xB1, 0x88, 0x11, + 0xB5, 0xBF, 0xB7, 0xBC, 0x55, 0x3F, 0x4F, 0xE4, + 0x44, 0x78, 0xCE, 0x28, 0x7A, 0x14, 0x87, 0x99, + 0x90, 0xD1, 0x8D, 0x12, 0xCA, 0x79, 0xD2, 0xC8, + 0x55, 0x14, 0x90, 0x21, 0xCD, 0x5C, 0xE8, 0xCA, + 0x03, 0x71, 0xCA, 0x04, 0xFC, 0xCE, 0x14, 0x3E, + 0x3D, 0x7C, 0xFE, 0xE9, 0x45, 0x85, 0xB5, 0x88, + 0x5C, 0xAC, 0x46, 0x06, 0x8B + }, + .len = 1000 + }, + .validAuthLenInBits = { + .len = 1000 + }, + .digest = { + .data = {0x06, 0x17, 0x45, 0xAE}, + .len = 4 + } +}; + +struct snow3g_hash_test_data snow3g_hash_test_case_3 = { + .key = { + .data = { + 0xB3, 0x12, 0x0F, 0xFD, 0xB2, 0xCF, 0x6A, 0xF4, + 0xE7, 0x3E, 0xAF, 0x2E, 0xF4, 0xEB, 0xEC, 0x69 + }, + .len = 16 + }, + .auth_iv = { + .data = { + 0x29, 0x6F, 0x39, 0x3C, 0x6B, 0x22, 0x77, 0x37, + 0xA9, 0x6F, 0x39, 0x3C, 0x6B, 0x22, 0xF7, 0x37 + }, + .len = 16 + }, + .plaintext = { + .data = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0xE0, 0x95, 0x80, 0x45, 0xF3, 0xA0, 0xBB, 0xA4, + 0xE3, 0x96, 0x83, 0x46, 0xF0, 0xA3, 0xB8, 0xA7, + 0xC0, 0x2A, 0x01, 0x8A, 0xE6, 0x40, 0x76, 0x52, + 0x26, 0xB9, 0x87, 0xC9, 0x13, 0xE6, 0xCB, 0xF0, + 0x83, 0x57, 0x00, 0x16, 0xCF, 0x83, 0xEF, 0xBC, + 0x61, 0xC0, 0x82, 0x51, 0x3E, 0x21, 0x56, 0x1A, + 0x42, 0x7C, 0x00, 0x9D, 0x28, 0xC2, 0x98, 0xEF, + 0xAC, 0xE7, 0x8E, 0xD6, 0xD5, 0x6C, 0x2D, 0x45, + 0x05, 0xAD, 0x03, 0x2E, 0x9C, 0x04, 0xDC, 0x60, + 0xE7, 0x3A, 0x81, 0x69, 0x6D, 0xA6, 0x65, 0xC6, + 0xC4, 0x86, 0x03, 0xA5, 0x7B, 0x45, 0xAB, 0x33, + 0x22, 0x15, 0x85, 0xE6, 0x8E, 0xE3, 0x16, 0x91, + 0x87, 0xFB, 0x02, 0x39, 0x52, 0x86, 0x32, 0xDD, + 0x65, 0x6C, 0x80, 0x7E, 0xA3, 0x24, 0x8B, 0x7B, + 0x46, 0xD0, 0x02, 0xB2, 0xB5, 0xC7, 0x45, 0x8E, + 0xB8, 0x5B, 0x9C, 0xE9, 0x58, 0x79, 0xE0, 0x34, + 0x08, 0x59, 0x05, 0x5E, 0x3B, 0x0A, 0xBB, 0xC3, + 0xEA, 0xCE, 0x87, 0x19, 0xCA, 0xA8, 0x02, 0x65, + 0xC9, 0x72, 0x05, 0xD5, 0xDC, 0x4B, 0xCC, 0x90, + 0x2F, 0xE1, 0x83, 0x96, 0x29, 0xED, 0x71, 0x32, + 0x8A, 0x0F, 0x04, 0x49, 0xF5, 0x88, 0x55, 0x7E, + 0x68, 0x98, 0x86, 0x0E, 0x04, 0x2A, 0xEC, 0xD8, + 0x4B, 0x24, 0x04, 0xC2, 0x12, 0xC9, 0x22, 0x2D, + 0xA5, 0xBF, 0x8A, 0x89, 0xEF, 0x67, 0x97, 0x87, + 0x0C, 0xF5, 0x07, 0x71, 0xA6, 0x0F, 0x66, 0xA2, + 0xEE, 0x62, 0x85, 0x36, 0x57, 0xAD, 0xDF, 0x04, + 0xCD, 0xDE, 0x07, 0xFA, 0x41, 0x4E, 0x11, 0xF1, + 0x2B, 0x4D, 0x81, 0xB9, 0xB4, 0xE8, 0xAC, 0x53, + 0x8E, 0xA3, 0x06, 0x66, 0x68, 0x8D, 0x88, 0x1F, + 0x6C, 0x34, 0x84, 0x21, 0x99, 0x2F, 0x31, 0xB9, + 0x4F, 0x88, 0x06, 0xED, 0x8F, 0xCC, 0xFF, 0x4C, + 0x91, 0x23, 0xB8, 0x96, 0x42, 0x52, 0x7A, 0xD6, + 0x13, 0xB1, 0x09, 0xBF, 0x75, 0x16, 0x74, 0x85, + 0xF1, 0x26, 0x8B, 0xF8, 0x84, 0xB4, 0xCD, 0x23, + 0xD2, 0x9A, 0x09, 0x34, 0x92, 0x57, 0x03, 0xD6, + 0x34, 0x09, 0x8F, 0x77, 0x67, 0xF1, 0xBE, 0x74, + 0x91, 0xE7, 0x08, 0xA8, 0xBB, 0x94, 0x9A, 0x38, + 0x73, 0x70, 0x8A, 0xEF, 0x4A, 0x36, 0x23, 0x9E, + 0x50, 0xCC, 0x08, 0x23, 0x5C, 0xD5, 0xED, 0x6B, + 0xBE, 0x57, 0x86, 0x68, 0xA1, 0x7B, 0x58, 0xC1, + 0x17, 0x1D, 0x0B, 0x90, 0xE8, 0x13, 0xA9, 0xE4, + 0xF5, 0x8A, 0x89, 0xD7, 0x19, 0xB1, 0x10, 0x42, + 0xD6, 0x36, 0x0B, 0x1B, 0x0F, 0x52, 0xDE, 0xB7, + 0x30, 0xA5, 0x8D, 0x58, 0xFA, 0xF4, 0x63, 0x15, + 0x95, 0x4B, 0x0A, 0x87, 0x26, 0x91, 0x47, 0x59, + 0x77, 0xDC, 0x88, 0xC0, 0xD7, 0x33, 0xFE, 0xFF, + 0x54, 0x60, 0x0A, 0x0C, 0xC1, 0xD0, 0x30, 0x0A, + 0xAA, 0xEB, 0x94, 0x57, 0x2C, 0x6E, 0x95, 0xB0, + 0x1A, 0xE9, 0x0D, 0xE0, 0x4F, 0x1D, 0xCE, 0x47, + 0xF8, 0x7E, 0x8F, 0xA7, 0xBE, 0xBF, 0x77, 0xE1, + 0xDB, 0xC2, 0x0D, 0x6B, 0xA8, 0x5C, 0xB9, 0x14, + 0x3D, 0x51, 0x8B, 0x28, 0x5D, 0xFA, 0x04, 0xB6, + 0x98, 0xBF, 0x0C, 0xF7, 0x81, 0x9F, 0x20, 0xFA, + 0x7A, 0x28, 0x8E, 0xB0, 0x70, 0x3D, 0x99, 0x5C, + 0x59, 0x94, 0x0C, 0x7C, 0x66, 0xDE, 0x57, 0xA9, + 0xB7, 0x0F, 0x82, 0x37, 0x9B, 0x70, 0xE2, 0x03, + 0x1E, 0x45, 0x0F, 0xCF, 0xD2, 0x18, 0x13, 0x26, + 0xFC, 0xD2, 0x8D, 0x88, 0x23, 0xBA, 0xAA, 0x80, + 0xDF, 0x6E, 0x0F, 0x44, 0x35, 0x59, 0x64, 0x75, + 0x39, 0xFD, 0x89, 0x07, 0xC0, 0xFF, 0xD9, 0xD7, + 0x9C, 0x13, 0x0E, 0xD8, 0x1C, 0x9A, 0xFD, 0x9B, + 0x7E, 0x84, 0x8C, 0x9F, 0xED, 0x38, 0x44, 0x3D, + 0x5D, 0x38, 0x0E, 0x53, 0xFB, 0xDB, 0x8A, 0xC8, + 0xC3, 0xD3, 0xF0, 0x68, 0x76, 0x05, 0x4F, 0x12, + 0x24, 0x61, 0x10, 0x7D, 0xE9, 0x2F, 0xEA, 0x09, + 0xC6, 0xF6, 0x92, 0x3A, 0x18, 0x8D, 0x53, 0xAF, + 0xE5, 0x4A, 0x10, 0xF6, 0x0E, 0x6E, 0x9D, 0x5A, + 0x03, 0xD9, 0x96, 0xB5, 0xFB, 0xC8, 0x20, 0xF8, + 0xA6, 0x37, 0x11, 0x6A, 0x27, 0xAD, 0x04, 0xB4, + 0x44, 0xA0, 0x93, 0x2D, 0xD6, 0x0F, 0xBD, 0x12, + 0x67, 0x1C, 0x11, 0xE1, 0xC0, 0xEC, 0x73, 0xE7, + 0x89, 0x87, 0x9F, 0xAA, 0x3D, 0x42, 0xC6, 0x4D, + 0x20, 0xCD, 0x12, 0x52, 0x74, 0x2A, 0x37, 0x68, + 0xC2, 0x5A, 0x90, 0x15, 0x85, 0x88, 0x8E, 0xCE, + 0xE1, 0xE6, 0x12, 0xD9, 0x93, 0x6B, 0x40, 0x3B, + 0x07, 0x75, 0x94, 0x9A, 0x66, 0xCD, 0xFD, 0x99, + 0xA2, 0x9B, 0x13, 0x45, 0xBA, 0xA8, 0xD9, 0xD5, + 0x40, 0x0C, 0x91, 0x02, 0x4B, 0x0A, 0x60, 0x73, + 0x63, 0xB0, 0x13, 0xCE, 0x5D, 0xE9, 0xAE, 0x86, + 0x9D, 0x3B, 0x8D, 0x95, 0xB0, 0x57, 0x0B, 0x3C, + 0x2D, 0x39, 0x14, 0x22, 0xD3, 0x24, 0x50, 0xCB, + 0xCF, 0xAE, 0x96, 0x65, 0x22, 0x86, 0xE9, 0x6D, + 0xEC, 0x12, 0x14, 0xA9, 0x34, 0x65, 0x27, 0x98, + 0x0A, 0x81, 0x92, 0xEA, 0xC1, 0xC3, 0x9A, 0x3A, + 0xAF, 0x6F, 0x15, 0x35, 0x1D, 0xA6, 0xBE, 0x76, + 0x4D, 0xF8, 0x97, 0x72, 0xEC, 0x04, 0x07, 0xD0, + 0x6E, 0x44, 0x15, 0xBE, 0xFA, 0xE7, 0xC9, 0x25, + 0x80, 0xDF, 0x9B, 0xF5, 0x07, 0x49, 0x7C, 0x8F, + 0x29, 0x95, 0x16, 0x0D, 0x4E, 0x21, 0x8D, 0xAA, + 0xCB, 0x02, 0x94, 0x4A, 0xBF, 0x83, 0x34, 0x0C, + 0xE8, 0xBE, 0x16, 0x86, 0xA9, 0x60, 0xFA, 0xF9, + 0x0E, 0x2D, 0x90, 0xC5, 0x5C, 0xC6, 0x47, 0x5B, + 0xAB, 0xC3, 0x17, 0x1A, 0x80, 0xA3, 0x63, 0x17, + 0x49, 0x54, 0x95, 0x5D, 0x71, 0x01, 0xDA, 0xB1, + 0x6A, 0xE8, 0x17, 0x91, 0x67, 0xE2, 0x14, 0x44, + 0xB4, 0x43, 0xA9, 0xEA, 0xAA, 0x7C, 0x91, 0xDE, + 0x36, 0xD1, 0x18, 0xC3, 0x9D, 0x38, 0x9F, 0x8D, + 0xD4, 0x46, 0x9A, 0x84, 0x6C, 0x9A, 0x26, 0x2B, + 0xF7, 0xFA, 0x18, 0x48, 0x7A, 0x79, 0xE8, 0xDE, + 0x11, 0x69, 0x9E, 0x0B, 0x8F, 0xDF, 0x55, 0x7C, + 0xB4, 0x87, 0x19, 0xD4, 0x53, 0xBA, 0x71, 0x30, + 0x56, 0x10, 0x9B, 0x93, 0xA2, 0x18, 0xC8, 0x96, + 0x75, 0xAC, 0x19, 0x5F, 0xB4, 0xFB, 0x06, 0x63, + 0x9B, 0x37, 0x97, 0x14, 0x49, 0x55, 0xB3, 0xC9, + 0x32, 0x7D, 0x1A, 0xEC, 0x00, 0x3D, 0x42, 0xEC, + 0xD0, 0xEA, 0x98, 0xAB, 0xF1, 0x9F, 0xFB, 0x4A, + 0xF3, 0x56, 0x1A, 0x67, 0xE7, 0x7C, 0x35, 0xBF, + 0x15, 0xC5, 0x9C, 0x24, 0x12, 0xDA, 0x88, 0x1D, + 0xB0, 0x2B, 0x1B, 0xFB, 0xCE, 0xBF, 0xAC, 0x51, + 0x52, 0xBC, 0x99, 0xBC, 0x3F, 0x1D, 0x15, 0xF7, + 0x71, 0x00, 0x1B, 0x70, 0x29, 0xFE, 0xDB, 0x02, + 0x8F, 0x8B, 0x85, 0x2B, 0xC4, 0x40, 0x7E, 0xB8, + 0x3F, 0x89, 0x1C, 0x9C, 0xA7, 0x33, 0x25, 0x4F, + 0xDD, 0x1E, 0x9E, 0xDB, 0x56, 0x91, 0x9C, 0xE9, + 0xFE, 0xA2, 0x1C, 0x17, 0x40, 0x72, 0x52, 0x1C, + 0x18, 0x31, 0x9A, 0x54, 0xB5, 0xD4, 0xEF, 0xBE, + 0xBD, 0xDF, 0x1D, 0x8B, 0x69, 0xB1, 0xCB, 0xF2, + 0x5F, 0x48, 0x9F, 0xCC, 0x98, 0x13, 0x72, 0x54, + 0x7C, 0xF4, 0x1D, 0x00, 0x8E, 0xF0, 0xBC, 0xA1, + 0x92, 0x6F, 0x93, 0x4B, 0x73, 0x5E, 0x09, 0x0B, + 0x3B, 0x25, 0x1E, 0xB3, 0x3A, 0x36, 0xF8, 0x2E, + 0xD9, 0xB2, 0x9C, 0xF4, 0xCB, 0x94, 0x41, 0x88, + 0xFA, 0x0E, 0x1E, 0x38, 0xDD, 0x77, 0x8F, 0x7D, + 0x1C, 0x9D, 0x98, 0x7B, 0x28, 0xD1, 0x32, 0xDF, + 0xB9, 0x73, 0x1F, 0xA4, 0xF4, 0xB4, 0x16, 0x93, + 0x5B, 0xE4, 0x9D, 0xE3, 0x05, 0x16, 0xAF, 0x35, + 0x78, 0x58, 0x1F, 0x2F, 0x13, 0xF5, 0x61, 0xC0, + 0x66, 0x33, 0x61, 0x94, 0x1E, 0xAB, 0x24, 0x9A, + 0x4B, 0xC1, 0x23, 0xF8, 0xD1, 0x5C, 0xD7, 0x11, + 0xA9, 0x56, 0xA1, 0xBF, 0x20, 0xFE, 0x6E, 0xB7, + 0x8A, 0xEA, 0x23, 0x73, 0x36, 0x1D, 0xA0, 0x42, + 0x6C, 0x79, 0xA5, 0x30, 0xC3, 0xBB, 0x1D, 0xE0, + 0xC9, 0x97, 0x22, 0xEF, 0x1F, 0xDE, 0x39, 0xAC, + 0x2B, 0x00, 0xA0, 0xA8, 0xEE, 0x7C, 0x80, 0x0A, + 0x08, 0xBC, 0x22, 0x64, 0xF8, 0x9F, 0x4E, 0xFF, + 0xE6, 0x27, 0xAC, 0x2F, 0x05, 0x31, 0xFB, 0x55, + 0x4F, 0x6D, 0x21, 0xD7, 0x4C, 0x59, 0x0A, 0x70, + 0xAD, 0xFA, 0xA3, 0x90, 0xBD, 0xFB, 0xB3, 0xD6, + 0x8E, 0x46, 0x21, 0x5C, 0xAB, 0x18, 0x7D, 0x23, + 0x68, 0xD5, 0xA7, 0x1F, 0x5E, 0xBE, 0xC0, 0x81, + 0xCD, 0x3B, 0x20, 0xC0, 0x82, 0xDB, 0xE4, 0xCD, + 0x2F, 0xAC, 0xA2, 0x87, 0x73, 0x79, 0x5D, 0x6B, + 0x0C, 0x10, 0x20, 0x4B, 0x65, 0x9A, 0x93, 0x9E, + 0xF2, 0x9B, 0xBE, 0x10, 0x88, 0x24, 0x36, 0x24, + 0x42, 0x99, 0x27, 0xA7, 0xEB, 0x57, 0x6D, 0xD3, + 0xA0, 0x0E, 0xA5, 0xE0, 0x1A, 0xF5, 0xD4, 0x75, + 0x83, 0xB2, 0x27, 0x2C, 0x0C, 0x16, 0x1A, 0x80, + 0x65, 0x21, 0xA1, 0x6F, 0xF9, 0xB0, 0xA7, 0x22, + 0xC0, 0xCF, 0x26, 0xB0, 0x25, 0xD5, 0x83, 0x6E, + 0x22, 0x58, 0xA4, 0xF7, 0xD4, 0x77, 0x3A, 0xC8, + 0x01, 0xE4, 0x26, 0x3B, 0xC2, 0x94, 0xF4, 0x3D, + 0xEF, 0x7F, 0xA8, 0x70, 0x3F, 0x3A, 0x41, 0x97, + 0x46, 0x35, 0x25, 0x88, 0x76, 0x52, 0xB0, 0xB2, + 0xA4, 0xA2, 0xA7, 0xCF, 0x87, 0xF0, 0x09, 0x14, + 0x87, 0x1E, 0x25, 0x03, 0x91, 0x13, 0xC7, 0xE1, + 0x61, 0x8D, 0xA3, 0x40, 0x64, 0xB5, 0x7A, 0x43, + 0xC4, 0x63, 0x24, 0x9F, 0xB8, 0xD0, 0x5E, 0x0F, + 0x26, 0xF4, 0xA6, 0xD8, 0x49, 0x72, 0xE7, 0xA9, + 0x05, 0x48, 0x24, 0x14, 0x5F, 0x91, 0x29, 0x5C, + 0xDB, 0xE3, 0x9A, 0x6F, 0x92, 0x0F, 0xAC, 0xC6, + 0x59, 0x71, 0x2B, 0x46, 0xA5, 0x4B, 0xA2, 0x95, + 0xBB, 0xE6, 0xA9, 0x01, 0x54, 0xE9, 0x1B, 0x33, + 0x98, 0x5A, 0x2B, 0xCD, 0x42, 0x0A, 0xD5, 0xC6, + 0x7E, 0xC9, 0xAD, 0x8E, 0xB7, 0xAC, 0x68, 0x64, + 0xDB, 0x27, 0x2A, 0x51, 0x6B, 0xC9, 0x4C, 0x28, + 0x39, 0xB0, 0xA8, 0x16, 0x9A, 0x6B, 0xF5, 0x8E, + 0x1A, 0x0C, 0x2A, 0xDA, 0x8C, 0x88, 0x3B, 0x7B, + 0xF4, 0x97, 0xA4, 0x91, 0x71, 0x26, 0x8E, 0xD1, + 0x5D, 0xDD, 0x29, 0x69, 0x38, 0x4E, 0x7F, 0xF4, + 0xBF, 0x4A, 0xAB, 0x2E, 0xC9, 0xEC, 0xC6, 0x52, + 0x9C, 0xF6, 0x29, 0xE2, 0xDF, 0x0F, 0x08, 0xA7, + 0x7A, 0x65, 0xAF, 0xA1, 0x2A, 0xA9, 0xB5, 0x05, + 0xDF, 0x8B, 0x28, 0x7E, 0xF6, 0xCC, 0x91, 0x49, + 0x3D, 0x1C, 0xAA, 0x39, 0x07, 0x6E, 0x28, 0xEF, + 0x1E, 0xA0, 0x28, 0xF5, 0x11, 0x8D, 0xE6, 0x1A, + 0xE0, 0x2B, 0xB6, 0xAE, 0xFC, 0x33, 0x43, 0xA0, + 0x50, 0x29, 0x2F, 0x19, 0x9F, 0x40, 0x18, 0x57, + 0xB2, 0xBE, 0xAD, 0x5E, 0x6E, 0xE2, 0xA1, 0xF1, + 0x91, 0x02, 0x2F, 0x92, 0x78, 0x01, 0x6F, 0x04, + 0x77, 0x91, 0xA9, 0xD1, 0x8D, 0xA7, 0xD2, 0xA6, + 0xD2, 0x7F, 0x2E, 0x0E, 0x51, 0xC2, 0xF6, 0xEA, + 0x30, 0xE8, 0xAC, 0x49, 0xA0, 0x60, 0x4F, 0x4C, + 0x13, 0x54, 0x2E, 0x85, 0xB6, 0x83, 0x81, 0xB9, + 0xFD, 0xCF, 0xA0, 0xCE, 0x4B, 0x2D, 0x34, 0x13, + 0x54, 0x85, 0x2D, 0x36, 0x02, 0x45, 0xC5, 0x36, + 0xB6, 0x12, 0xAF, 0x71, 0xF3, 0xE7, 0x7C, 0x90, + 0x95, 0xAE, 0x2D, 0xBD, 0xE5, 0x04, 0xB2, 0x65, + 0x73, 0x3D, 0xAB, 0xFE, 0x10, 0xA2, 0x0F, 0xC7, + 0xD6, 0xD3, 0x2C, 0x21, 0xCC, 0xC7, 0x2B, 0x8B, + 0x34, 0x44, 0xAE, 0x66, 0x3D, 0x65, 0x92, 0x2D, + 0x17, 0xF8, 0x2C, 0xAA, 0x2B, 0x86, 0x5C, 0xD8, + 0x89, 0x13, 0xD2, 0x91, 0xA6, 0x58, 0x99, 0x02, + 0x6E, 0xA1, 0x32, 0x84, 0x39, 0x72, 0x3C, 0x19, + 0x8C, 0x36, 0xB0, 0xC3, 0xC8, 0xD0, 0x85, 0xBF, + 0xAF, 0x8A, 0x32, 0x0F, 0xDE, 0x33, 0x4B, 0x4A, + 0x49, 0x19, 0xB4, 0x4C, 0x2B, 0x95, 0xF6, 0xE8, + 0xEC, 0xF7, 0x33, 0x93, 0xF7, 0xF0, 0xD2, 0xA4, + 0x0E, 0x60, 0xB1, 0xD4, 0x06, 0x52, 0x6B, 0x02, + 0x2D, 0xDC, 0x33, 0x18, 0x10, 0xB1, 0xA5, 0xF7, + 0xC3, 0x47, 0xBD, 0x53, 0xED, 0x1F, 0x10, 0x5D, + 0x6A, 0x0D, 0x30, 0xAB, 0xA4, 0x77, 0xE1, 0x78, + 0x88, 0x9A, 0xB2, 0xEC, 0x55, 0xD5, 0x58, 0xDE, + 0xAB, 0x26, 0x30, 0x20, 0x43, 0x36, 0x96, 0x2B, + 0x4D, 0xB5, 0xB6, 0x63, 0xB6, 0x90, 0x2B, 0x89, + 0xE8, 0x5B, 0x31, 0xBC, 0x6A, 0xF5, 0x0F, 0xC5, + 0x0A, 0xCC, 0xB3, 0xFB, 0x9B, 0x57, 0xB6, 0x63, + 0x29, 0x70, 0x31, 0x37, 0x8D, 0xB4, 0x78, 0x96, + 0xD7, 0xFB, 0xAF, 0x6C, 0x60, 0x0A, 0xDD, 0x2C, + 0x67, 0xF9, 0x36, 0xDB, 0x03, 0x79, 0x86, 0xDB, + 0x85, 0x6E, 0xB4, 0x9C, 0xF2, 0xDB, 0x3F, 0x7D, + 0xA6, 0xD2, 0x36, 0x50, 0xE4, 0x38, 0xF1, 0x88, + 0x40, 0x41, 0xB0, 0x13, 0x11, 0x9E, 0x4C, 0x2A, + 0xE5, 0xAF, 0x37, 0xCC, 0xCD, 0xFB, 0x68, 0x66, + 0x07, 0x38, 0xB5, 0x8B, 0x3C, 0x59, 0xD1, 0xC0, + 0x24, 0x84, 0x37, 0x47, 0x2A, 0xBA, 0x1F, 0x35, + 0xCA, 0x1F, 0xB9, 0x0C, 0xD7, 0x14, 0xAA, 0x9F, + 0x63, 0x55, 0x34, 0xF4, 0x9E, 0x7C, 0x5B, 0xBA, + 0x81, 0xC2, 0xB6, 0xB3, 0x6F, 0xDE, 0xE2, 0x1C, + 0xA2, 0x7E, 0x34, 0x7F, 0x79, 0x3D, 0x2C, 0xE9, + 0x44, 0xED, 0xB2, 0x3C, 0x8C, 0x9B, 0x91, 0x4B, + 0xE1, 0x03, 0x35, 0xE3, 0x50, 0xFE, 0xB5, 0x07, + 0x03, 0x94, 0xB7, 0xA4, 0xA1, 0x5C, 0x0C, 0xA1, + 0x20, 0x28, 0x35, 0x68, 0xB7, 0xBF, 0xC2, 0x54, + 0xFE, 0x83, 0x8B, 0x13, 0x7A, 0x21, 0x47, 0xCE, + 0x7C, 0x11, 0x3A, 0x3A, 0x4D, 0x65, 0x49, 0x9D, + 0x9E, 0x86, 0xB8, 0x7D, 0xBC, 0xC7, 0xF0, 0x3B, + 0xBD, 0x3A, 0x3A, 0xB1, 0xAA, 0x24, 0x3E, 0xCE, + 0x5B, 0xA9, 0xBC, 0xF2, 0x5F, 0x82, 0x83, 0x6C, + 0xFE, 0x47, 0x3B, 0x2D, 0x83, 0xE7, 0xA7, 0x20, + 0x1C, 0xD0, 0xB9, 0x6A, 0x72, 0x45, 0x1E, 0x86, + 0x3F, 0x6C, 0x3B, 0xA6, 0x64, 0xA6, 0xD0, 0x73, + 0xD1, 0xF7, 0xB5, 0xED, 0x99, 0x08, 0x65, 0xD9, + 0x78, 0xBD, 0x38, 0x15, 0xD0, 0x60, 0x94, 0xFC, + 0x9A, 0x2A, 0xBA, 0x52, 0x21, 0xC2, 0x2D, 0x5A, + 0xB9, 0x96, 0x38, 0x9E, 0x37, 0x21, 0xE3, 0xAF, + 0x5F, 0x05, 0xBE, 0xDD, 0xC2, 0x87, 0x5E, 0x0D, + 0xFA, 0xEB, 0x39, 0x02, 0x1E, 0xE2, 0x7A, 0x41, + 0x18, 0x7C, 0xBB, 0x45, 0xEF, 0x40, 0xC3, 0xE7, + 0x3B, 0xC0, 0x39, 0x89, 0xF9, 0xA3, 0x0D, 0x12, + 0xC5, 0x4B, 0xA7, 0xD2, 0x14, 0x1D, 0xA8, 0xA8, + 0x75, 0x49, 0x3E, 0x65, 0x77, 0x6E, 0xF3, 0x5F, + 0x97, 0xDE, 0xBC, 0x22, 0x86, 0xCC, 0x4A, 0xF9, + 0xB4, 0x62, 0x3E, 0xEE, 0x90, 0x2F, 0x84, 0x0C, + 0x52, 0xF1, 0xB8, 0xAD, 0x65, 0x89, 0x39, 0xAE, + 0xF7, 0x1F, 0x3F, 0x72, 0xB9, 0xEC, 0x1D, 0xE2, + 0x15, 0x88, 0xBD, 0x35, 0x48, 0x4E, 0xA4, 0x44, + 0x36, 0x34, 0x3F, 0xF9, 0x5E, 0xAD, 0x6A, 0xB1, + 0xD8, 0xAF, 0xB1, 0xB2, 0xA3, 0x03, 0xDF, 0x1B, + 0x71, 0xE5, 0x3C, 0x4A, 0xEA, 0x6B, 0x2E, 0x3E, + 0x93, 0x72, 0xBE, 0x0D, 0x1B, 0xC9, 0x97, 0x98, + 0xB0, 0xCE, 0x3C, 0xC1, 0x0D, 0x2A, 0x59, 0x6D, + 0x56, 0x5D, 0xBA, 0x82, 0xF8, 0x8C, 0xE4, 0xCF, + 0xF3, 0xB3, 0x3D, 0x5D, 0x24, 0xE9, 0xC0, 0x83, + 0x11, 0x24, 0xBF, 0x1A, 0xD5, 0x4B, 0x79, 0x25, + 0x32, 0x98, 0x3D, 0xD6, 0xC3, 0xA8, 0xB7, 0xD0 + }, + .len = 16448 + }, + .validAuthLenInBits = { + .len = 16448 + }, + .digest = { + .data = {0x17, 0x9F, 0x2F, 0xA6}, + .len = 4 + } +}; + +struct snow3g_hash_test_data snow3g_hash_test_case_4 = { + .key = { + .data = { + 0x2B, 0xD6, 0x45, 0x9F, 0x82, 0xC5, 0xB3, 0x00, + 0x95, 0x2C, 0x49, 0x10, 0x48, 0x81, 0xFF, 0x48 + }, + .len = 16 + }, + .auth_iv = { + .data = { + 0x38, 0xA6, 0xF0, 0x56, 0x05, 0xD2, 0xEC, 0x49, + 0x38, 0xA6, 0xF0, 0x56, 0x05, 0xD2, 0xEC, 0x49, + }, + .len = 16 + }, + .plaintext = { + .data = { + 0x6B, 0x22, 0x77, 0x37, 0x29, 0x6F, 0x39, 0x3C, + 0x80, 0x79, 0x35, 0x3E, 0xDC, 0x87, 0xE2, 0xE8, + 0x05, 0xD2, 0xEC, 0x49, 0xA4, 0xF2, 0xD8, 0xE0 + }, + .len = 189 + }, + .validAuthLenInBits = { + .len = 189 + }, + .digest = { + .data = {0x2B, 0xCE, 0x18, 0x20}, + .len = 4 + } +}; + +struct snow3g_hash_test_data snow3g_hash_test_case_5 = { + .key = { + .data = { + 0xD4, 0x2F, 0x68, 0x24, 0x28, 0x20, 0x1C, 0xAF, + 0xCD, 0x9F, 0x97, 0x94, 0x5E, 0x6D, 0xE7, 0xB7 + }, + .len = 16 + }, + .auth_iv = { + .data = { + 0x3E, 0xDC, 0x87, 0xE2, 0xA4, 0xF2, 0xD8, 0xE2, + 0xBE, 0xDC, 0x87, 0xE2, 0xA4, 0xF2, 0x58, 0xE2 + }, + .len = 16 + }, + .plaintext = { + .data = { + 0xB5, 0x92, 0x43, 0x84, 0x32, 0x8A, 0x4A, 0xE0, + 0x0B, 0x73, 0x71, 0x09, 0xF8, 0xB6, 0xC8, 0xDD, + 0x2B, 0x4D, 0xB6, 0x3D, 0xD5, 0x33, 0x98, 0x1C, + 0xEB, 0x19, 0xAA, 0xD5, 0x2A, 0x5B, 0x2B, 0xC0 + }, + .len = 254 + }, + .validAuthLenInBits = { + .len = 254 + }, + .digest = { + .data = {0xFC, 0x7B, 0x18, 0xBD}, + .len = 4 + } +}; + +struct snow3g_hash_test_data snow3g_hash_test_case_6 = { + .key = { + .data = { + 0xFD, 0xB9, 0xCF, 0xDF, 0x28, 0x93, 0x6C, 0xC4, + 0x83, 0xA3, 0x18, 0x69, 0xD8, 0x1B, 0x8F, 0xAB + }, + .len = 16 + }, + .auth_iv = { + .data = { + 0x36, 0xAF, 0x61, 0x44, 0x98, 0x38, 0xF0, 0x3A, + 0xB6, 0xAF, 0x61, 0x44, 0x98, 0x38, 0x70, 0x3A + }, + .len = 16 + }, + .plaintext = { + .data = { + 0x59, 0x32, 0xBC, 0x0A, 0xCE, 0x2B, 0x0A, 0xBA, + 0x33, 0xD8, 0xAC, 0x18, 0x8A, 0xC5, 0x4F, 0x34, + 0x6F, 0xAD, 0x10, 0xBF, 0x9D, 0xEE, 0x29, 0x20, + 0xB4, 0x3B, 0xD0, 0xC5, 0x3A, 0x91, 0x5C, 0xB7, + 0xDF, 0x6C, 0xAA, 0x72, 0x05, 0x3A, 0xBF, 0xF2 + }, + .len = 319 + }, + .validAuthLenInBits = { + .len = 319 + }, + .digest = { + .data = {0x02, 0xF1, 0xFA, 0xAF}, + .len = 4 + } +}; +#endif /* TEST_CRYPTODEV_SNOW3G_HASH_TEST_VECTORS_H_ */ diff --git a/dpdk/test/test/test_cryptodev_snow3g_test_vectors.h b/dpdk/test/test/test_cryptodev_snow3g_test_vectors.h new file mode 100644 index 00000000..fed50dc2 --- /dev/null +++ b/dpdk/test/test/test_cryptodev_snow3g_test_vectors.h @@ -0,0 +1,411 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2015-2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef TEST_CRYPTODEV_SNOW3G_TEST_VECTORS_H_ +#define TEST_CRYPTODEV_SNOW3G_TEST_VECTORS_H_ + +struct snow3g_test_data { + struct { + uint8_t data[64]; + unsigned len; + } key; + + struct { + uint8_t data[64] __rte_aligned(16); + unsigned len; + } cipher_iv; + + struct { + uint8_t data[1024]; + unsigned len; /* length must be in Bits */ + } plaintext; + + struct { + uint8_t data[1024]; + unsigned len; /* length must be in Bits */ + } ciphertext; + + struct { + unsigned len; + } validDataLenInBits; + + struct { + unsigned len; + } validCipherLenInBits; + + struct { + unsigned len; + } validAuthLenInBits; + + struct { + uint8_t data[64]; + unsigned len; + } auth_iv; + + struct { + uint8_t data[64]; + unsigned len; + } digest; +}; +struct snow3g_test_data snow3g_test_case_1 = { + .key = { + .data = { + 0x2B, 0xD6, 0x45, 0x9F, 0x82, 0xC5, 0xB3, 0x00, + 0x95, 0x2C, 0x49, 0x10, 0x48, 0x81, 0xFF, 0x48 + }, + .len = 16 + }, + .cipher_iv = { + .data = { + 0x72, 0xA4, 0xF2, 0x0F, 0x64, 0x00, 0x00, 0x00, + 0x72, 0xA4, 0xF2, 0x0F, 0x64, 0x00, 0x00, 0x00 + }, + .len = 16 + }, + .plaintext = { + .data = { + 0x7E, 0xC6, 0x12, 0x72, 0x74, 0x3B, 0xF1, 0x61, + 0x47, 0x26, 0x44, 0x6A, 0x6C, 0x38, 0xCE, 0xD1, + 0x66, 0xF6, 0xCA, 0x76, 0xEB, 0x54, 0x30, 0x04, + 0x42, 0x86, 0x34, 0x6C, 0xEF, 0x13, 0x0F, 0x92, + 0x92, 0x2B, 0x03, 0x45, 0x0D, 0x3A, 0x99, 0x75, + 0xE5, 0xBD, 0x2E, 0xA0, 0xEB, 0x55, 0xAD, 0x8E, + 0x1B, 0x19, 0x9E, 0x3E, 0xC4, 0x31, 0x60, 0x20, + 0xE9, 0xA1, 0xB2, 0x85, 0xE7, 0x62, 0x79, 0x53, + 0x59, 0xB7, 0xBD, 0xFD, 0x39, 0xBE, 0xF4, 0xB2, + 0x48, 0x45, 0x83, 0xD5, 0xAF, 0xE0, 0x82, 0xAE, + 0xE6, 0x38, 0xBF, 0x5F, 0xD5, 0xA6, 0x06, 0x19, + 0x39, 0x01, 0xA0, 0x8F, 0x4A, 0xB4, 0x1A, 0xAB, + 0x9B, 0x13, 0x48, 0x80 + }, + .len = 800 + }, + .ciphertext = { + .data = { + 0x8C, 0xEB, 0xA6, 0x29, 0x43, 0xDC, 0xED, 0x3A, + 0x09, 0x90, 0xB0, 0x6E, 0xA1, 0xB0, 0xA2, 0xC4, + 0xFB, 0x3C, 0xED, 0xC7, 0x1B, 0x36, 0x9F, 0x42, + 0xBA, 0x64, 0xC1, 0xEB, 0x66, 0x65, 0xE7, 0x2A, + 0xA1, 0xC9, 0xBB, 0x0D, 0xEA, 0xA2, 0x0F, 0xE8, + 0x60, 0x58, 0xB8, 0xBA, 0xEE, 0x2C, 0x2E, 0x7F, + 0x0B, 0xEC, 0xCE, 0x48, 0xB5, 0x29, 0x32, 0xA5, + 0x3C, 0x9D, 0x5F, 0x93, 0x1A, 0x3A, 0x7C, 0x53, + 0x22, 0x59, 0xAF, 0x43, 0x25, 0xE2, 0xA6, 0x5E, + 0x30, 0x84, 0xAD, 0x5F, 0x6A, 0x51, 0x3B, 0x7B, + 0xDD, 0xC1, 0xB6, 0x5F, 0x0A, 0xA0, 0xD9, 0x7A, + 0x05, 0x3D, 0xB5, 0x5A, 0x88, 0xC4, 0xC4, 0xF9, + 0x60, 0x5E, 0x41, 0x40 + }, + .len = 800 + }, + .validDataLenInBits = { + .len = 798 + }, + .validCipherLenInBits = { + .len = 800 + }, + .auth_iv = { + .data = { + 0x72, 0xA4, 0xF2, 0x0F, 0x64, 0x00, 0x00, 0x00, + 0x72, 0xA4, 0xF2, 0x0F, 0x64, 0x00, 0x00, 0x00 + }, + .len = 16 + } +}; + +struct snow3g_test_data snow3g_test_case_2 = { + .key = { + .data = { + 0xEF, 0xA8, 0xB2, 0x22, 0x9E, 0x72, 0x0C, 0x2A, + 0x7C, 0x36, 0xEA, 0x55, 0xE9, 0x60, 0x56, 0x95 + }, + .len = 16 + }, + .cipher_iv = { + .data = { + 0xE2, 0x8B, 0xCF, 0x7B, 0xC0, 0x00, 0x00, 0x00, + 0xE2, 0x8B, 0xCF, 0x7B, 0xC0, 0x00, 0x00, 0x00 + }, + .len = 16 + }, + .plaintext = { + .data = { + 0x10, 0x11, 0x12, 0x31, 0xE0, 0x60, 0x25, 0x3A, + 0x43, 0xFD, 0x3F, 0x57, 0xE3, 0x76, 0x07, 0xAB, + 0x28, 0x27, 0xB5, 0x99, 0xB6, 0xB1, 0xBB, 0xDA, + 0x37, 0xA8, 0xAB, 0xCC, 0x5A, 0x8C, 0x55, 0x0D, + 0x1B, 0xFB, 0x2F, 0x49, 0x46, 0x24, 0xFB, 0x50, + 0x36, 0x7F, 0xA3, 0x6C, 0xE3, 0xBC, 0x68, 0xF1, + 0x1C, 0xF9, 0x3B, 0x15, 0x10, 0x37, 0x6B, 0x02, + 0x13, 0x0F, 0x81, 0x2A, 0x9F, 0xA1, 0x69, 0xD8 + }, + .len = 512 + }, + .ciphertext = { + .data = { + 0xE0, 0xDA, 0x15, 0xCA, 0x8E, 0x25, 0x54, 0xF5, + 0xE5, 0x6C, 0x94, 0x68, 0xDC, 0x6C, 0x7C, 0x12, + 0x9C, 0x56, 0x8A, 0xA5, 0x03, 0x23, 0x17, 0xE0, + 0x4E, 0x07, 0x29, 0x64, 0x6C, 0xAB, 0xEF, 0xA6, + 0x89, 0x86, 0x4C, 0x41, 0x0F, 0x24, 0xF9, 0x19, + 0xE6, 0x1E, 0x3D, 0xFD, 0xFA, 0xD7, 0x7E, 0x56, + 0x0D, 0xB0, 0xA9, 0xCD, 0x36, 0xC3, 0x4A, 0xE4, + 0x18, 0x14, 0x90, 0xB2, 0x9F, 0x5F, 0xA2, 0xFC + }, + .len = 512 + }, + .validDataLenInBits = { + .len = 510 + }, + .validCipherLenInBits = { + .len = 512 + }, + .auth_iv = { + .data = { + 0xE2, 0x8B, 0xCF, 0x7B, 0xC0, 0x00, 0x00, 0x00, + 0xE2, 0x8B, 0xCF, 0x7B, 0xC0, 0x00, 0x00, 0x00 + }, + .len = 16 + } +}; + +struct snow3g_test_data snow3g_test_case_3 = { + .key = { + .data = { + 0x5A, 0xCB, 0x1D, 0x64, 0x4C, 0x0D, 0x51, 0x20, + 0x4E, 0xA5, 0xF1, 0x45, 0x10, 0x10, 0xD8, 0x52 + }, + .len = 16 + }, + .cipher_iv = { + .data = { + 0xFA, 0x55, 0x6B, 0x26, 0x1C, 0x00, 0x00, 0x00, + 0xFA, 0x55, 0x6B, 0x26, 0x1C, 0x00, 0x00, 0x00 + }, + .len = 16 + }, + .plaintext = { + .data = { + 0xAD, 0x9C, 0x44, 0x1F, 0x89, 0x0B, 0x38, 0xC4, + 0x57, 0xA4, 0x9D, 0x42, 0x14, 0x07, 0xE8 + }, + .len = 120 + }, + .ciphertext = { + .data = { + 0xBA, 0x0F, 0x31, 0x30, 0x03, 0x34, 0xC5, 0x6B, + 0x52, 0xA7, 0x49, 0x7C, 0xBA, 0xC0, 0x46 + }, + .len = 120 + }, + .validDataLenInBits = { + .len = 120 + }, + .validCipherLenInBits = { + .len = 120 + }, + .auth_iv = { + .data = { + 0xFA, 0x55, 0x6B, 0x26, 0x1C, 0x00, 0x00, 0x00, + 0xFA, 0x55, 0x6B, 0x26, 0x1C, 0x00, 0x00, 0x00 + }, + .len = 16 + }, + .digest = { + .data = {0xE8, 0x60, 0x5A, 0x3E}, + .len = 4 + }, + .validAuthLenInBits = { + .len = 120 + } +}; + +struct snow3g_test_data snow3g_test_case_4 = { + .key = { + .data = { + 0xD3, 0xC5, 0xD5, 0x92, 0x32, 0x7F, 0xB1, 0x1C, + 0x40, 0x35, 0xC6, 0x68, 0x0A, 0xF8, 0xC6, 0xD1 + }, + .len = 16 + }, + .cipher_iv = { + .data = { + 0x39, 0x8A, 0x59, 0xB4, 0x2C, 0x00, 0x00, 0x00, + 0x39, 0x8A, 0x59, 0xB4, 0x2C, 0x00, 0x00, 0x00 + }, + .len = 16 + }, + .plaintext = { + .data = { + 0x98, 0x1B, 0xA6, 0x82, 0x4C, 0x1B, 0xFB, 0x1A, + 0xB4, 0x85, 0x47, 0x20, 0x29, 0xB7, 0x1D, 0x80, + 0x8C, 0xE3, 0x3E, 0x2C, 0xC3, 0xC0, 0xB5, 0xFC, + 0x1F, 0x3D, 0xE8, 0xA6, 0xDC, 0x66, 0xB1, 0xF0 + }, + .len = 256 + }, + .ciphertext = { + .data = { + 0x98, 0x9B, 0x71, 0x9C, 0xDC, 0x33, 0xCE, 0xB7, + 0xCF, 0x27, 0x6A, 0x52, 0x82, 0x7C, 0xEF, 0x94, + 0xA5, 0x6C, 0x40, 0xC0, 0xAB, 0x9D, 0x81, 0xF7, + 0xA2, 0xA9, 0xBA, 0xC6, 0x0E, 0x11, 0xC4, 0xB0 + }, + .len = 256 + }, + .validDataLenInBits = { + .len = 253 + }, + .validCipherLenInBits = { + .len = 256 + } +}; + +struct snow3g_test_data snow3g_test_case_5 = { + .key = { + .data = { + 0x60, 0x90, 0xEA, 0xE0, 0x4C, 0x83, 0x70, 0x6E, + 0xEC, 0xBF, 0x65, 0x2B, 0xE8, 0xE3, 0x65, 0x66 + }, + .len = 16 + }, + .cipher_iv = { + .data = { + 0x72, 0xA4, 0xF2, 0x0F, 0x48, 0x00, 0x00, 0x00, + 0x72, 0xA4, 0xF2, 0x0F, 0x48, 0x00, 0x00, 0x00 + }, + .len = 16}, + .plaintext = { + .data = { + 0x40, 0x98, 0x1B, 0xA6, 0x82, 0x4C, 0x1B, 0xFB, + 0x42, 0x86, 0xB2, 0x99, 0x78, 0x3D, 0xAF, 0x44, + 0x2C, 0x09, 0x9F, 0x7A, 0xB0, 0xF5, 0x8D, 0x5C, + 0x8E, 0x46, 0xB1, 0x04, 0xF0, 0x8F, 0x01, 0xB4, + 0x1A, 0xB4, 0x85, 0x47, 0x20, 0x29, 0xB7, 0x1D, + 0x36, 0xBD, 0x1A, 0x3D, 0x90, 0xDC, 0x3A, 0x41, + 0xB4, 0x6D, 0x51, 0x67, 0x2A, 0xC4, 0xC9, 0x66, + 0x3A, 0x2B, 0xE0, 0x63, 0xDA, 0x4B, 0xC8, 0xD2, + 0x80, 0x8C, 0xE3, 0x3E, 0x2C, 0xCC, 0xBF, 0xC6, + 0x34, 0xE1, 0xB2, 0x59, 0x06, 0x08, 0x76, 0xA0, + 0xFB, 0xB5, 0xA4, 0x37, 0xEB, 0xCC, 0x8D, 0x31, + 0xC1, 0x9E, 0x44, 0x54, 0x31, 0x87, 0x45, 0xE3, + 0x98, 0x76, 0x45, 0x98, 0x7A, 0x98, 0x6F, 0x2C, + 0xB0 + }, + .len = 840 + }, + .ciphertext = { + .data = { + 0x58, 0x92, 0xBB, 0xA8, 0x8B, 0xBB, 0xCA, 0xAE, + 0xAE, 0x76, 0x9A, 0xA0, 0x6B, 0x68, 0x3D, 0x3A, + 0x17, 0xCC, 0x04, 0xA3, 0x69, 0x88, 0x16, 0x97, + 0x43, 0x5E, 0x44, 0xFE, 0xD5, 0xFF, 0x9A, 0xF5, + 0x7B, 0x9E, 0x89, 0x0D, 0x4D, 0x5C, 0x64, 0x70, + 0x98, 0x85, 0xD4, 0x8A, 0xE4, 0x06, 0x90, 0xEC, + 0x04, 0x3B, 0xAA, 0xE9, 0x70, 0x57, 0x96, 0xE4, + 0xA9, 0xFF, 0x5A, 0x4B, 0x8D, 0x8B, 0x36, 0xD7, + 0xF3, 0xFE, 0x57, 0xCC, 0x6C, 0xFD, 0x6C, 0xD0, + 0x05, 0xCD, 0x38, 0x52, 0xA8, 0x5E, 0x94, 0xCE, + 0x6B, 0xCD, 0x90, 0xD0, 0xD0, 0x78, 0x39, 0xCE, + 0x09, 0x73, 0x35, 0x44, 0xCA, 0x8E, 0x35, 0x08, + 0x43, 0x24, 0x85, 0x50, 0x92, 0x2A, 0xC1, 0x28, + 0x18 + }, + .len = 840 + }, + .validDataLenInBits = { + .len = 837 + }, + .validCipherLenInBits = { + .len = 840 + }, +}; +struct snow3g_test_data snow3g_test_case_6 = { + .key = { + .data = { + 0xC7, 0x36, 0xC6, 0xAA, 0xB2, 0x2B, 0xFF, 0xF9, + 0x1E, 0x26, 0x98, 0xD2, 0xE2, 0x2A, 0xD5, 0x7E + }, + .len = 16 + }, + .cipher_iv = { + .data = { + 0x14, 0x79, 0x3E, 0x41, 0x03, 0x97, 0xE8, 0xFD, + 0x94, 0x79, 0x3E, 0x41, 0x03, 0x97, 0x68, 0xFD + }, + .len = 16 + }, + .auth_iv = { + .data = { + 0x14, 0x79, 0x3E, 0x41, 0x03, 0x97, 0xE8, 0xFD, + 0x94, 0x79, 0x3E, 0x41, 0x03, 0x97, 0x68, 0xFD + }, + .len = 16 + }, + .plaintext = { + .data = { + 0xD0, 0xA7, 0xD4, 0x63, 0xDF, 0x9F, 0xB2, 0xB2, + 0x78, 0x83, 0x3F, 0xA0, 0x2E, 0x23, 0x5A, 0xA1, + 0x72, 0xBD, 0x97, 0x0C, 0x14, 0x73, 0xE1, 0x29, + 0x07, 0xFB, 0x64, 0x8B, 0x65, 0x99, 0xAA, 0xA0, + 0xB2, 0x4A, 0x03, 0x86, 0x65, 0x42, 0x2B, 0x20, + 0xA4, 0x99, 0x27, 0x6A, 0x50, 0x42, 0x70, 0x09 + }, + .len = 384 + }, + .ciphertext = { + .data = { + 0x95, 0x2E, 0x5A, 0xE1, 0x50, 0xB8, 0x59, 0x2A, + 0x9B, 0xA0, 0x38, 0xA9, 0x8E, 0x2F, 0xED, 0xAB, + 0xFD, 0xC8, 0x3B, 0x47, 0x46, 0x0B, 0x50, 0x16, + 0xEC, 0x88, 0x45, 0xB6, 0x05, 0xC7, 0x54, 0xF8, + 0xBD, 0x91, 0xAA, 0xB6, 0xA4, 0xDC, 0x64, 0xB4, + 0xCB, 0xEB, 0x97, 0x06, 0x4C, 0xF7, 0x02, 0x3D + }, + .len = 384 + }, + .digest = { + .data = {0x38, 0xB5, 0x54, 0xC0 }, + .len = 4 + }, + .validDataLenInBits = { + .len = 384 + }, + .validCipherLenInBits = { + .len = 384 + }, + .validAuthLenInBits = { + .len = 384 + }, +}; + +#endif /* TEST_CRYPTODEV_SNOW3G_TEST_VECTORS_H_ */ diff --git a/dpdk/test/test/test_cryptodev_zuc_test_vectors.h b/dpdk/test/test/test_cryptodev_zuc_test_vectors.h new file mode 100644 index 00000000..a22e1edb --- /dev/null +++ b/dpdk/test/test/test_cryptodev_zuc_test_vectors.h @@ -0,0 +1,1071 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY ExPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, ExEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef TEST_CRYPTODEV_ZUC_TEST_VECTORS_H_ +#define TEST_CRYPTODEV_ZUC_TEST_VECTORS_H_ + +struct wireless_test_data { + struct { + uint8_t data[64]; + unsigned len; + } key; + + struct { + uint8_t data[64] __rte_aligned(16); + unsigned len; + } cipher_iv; + + struct { + uint8_t data[2048]; + unsigned len; /* length must be in Bits */ + } plaintext; + + struct { + uint8_t data[2048]; + unsigned len; /* length must be in Bits */ + } ciphertext; + + struct { + unsigned len; + } validDataLenInBits; + + struct { + unsigned len; + } validCipherLenInBits; + + struct { + unsigned len; + } validAuthLenInBits; + + struct { + uint8_t data[64]; + unsigned len; + } auth_iv; + + struct { + uint8_t data[64]; + unsigned len; + } digest; +}; +static struct wireless_test_data zuc_test_case_cipher_193b = { + .key = { + .data = { + 0x17, 0x3D, 0x14, 0xBA, 0x50, 0x03, 0x73, 0x1D, + 0x7A, 0x60, 0x04, 0x94, 0x70, 0xF0, 0x0A, 0x29 + }, + .len = 16 + }, + .cipher_iv = { + .data = { + 0x66, 0x03, 0x54, 0x92, 0x78, 0x00, 0x00, 0x00, + 0x66, 0x03, 0x54, 0x92, 0x78, 0x00, 0x00, 0x00 + }, + .len = 16 + }, + .plaintext = { + .data = { + 0x6C, 0xF6, 0x53, 0x40, 0x73, 0x55, 0x52, 0xAB, + 0x0C, 0x97, 0x52, 0xFA, 0x6F, 0x90, 0x25, 0xFE, + 0x0B, 0xD6, 0x75, 0xD9, 0x00, 0x58, 0x75, 0xB2, + 0x00 + }, + .len = 200 + }, + .ciphertext = { + .data = { + 0xA6, 0xC8, 0x5F, 0xC6, 0x6A, 0xFB, 0x85, 0x33, + 0xAA, 0xFC, 0x25, 0x18, 0xDF, 0xE7, 0x84, 0x94, + 0x0E, 0xE1, 0xE4, 0xB0, 0x30, 0x23, 0x8C, 0xC8, + 0x00 + }, + .len = 200 + }, + .validDataLenInBits = { + .len = 193 + }, + .validCipherLenInBits = { + .len = 193 + } +}; + +static struct wireless_test_data zuc_test_case_cipher_800b = { + .key = { + .data = { + 0xE5, 0xBD, 0x3E, 0xA0, 0xEB, 0x55, 0xAD, 0xE8, + 0x66, 0xC6, 0xAC, 0x58, 0xBD, 0x54, 0x30, 0x2A + }, + .len = 16 + }, + .cipher_iv = { + .data = { + 0x00, 0x05, 0x68, 0x23, 0xC4, 0x00, 0x00, 0x00, + 0x00, 0x05, 0x68, 0x23, 0xC4, 0x00, 0x00, 0x00 + }, + .len = 16 + }, + .plaintext = { + .data = { + 0x14, 0xA8, 0xEF, 0x69, 0x3D, 0x67, 0x85, 0x07, + 0xBB, 0xE7, 0x27, 0x0A, 0x7F, 0x67, 0xFF, 0x50, + 0x06, 0xC3, 0x52, 0x5B, 0x98, 0x07, 0xE4, 0x67, + 0xC4, 0xE5, 0x60, 0x00, 0xBA, 0x33, 0x8F, 0x5D, + 0x42, 0x95, 0x59, 0x03, 0x67, 0x51, 0x82, 0x22, + 0x46, 0xC8, 0x0D, 0x3B, 0x38, 0xF0, 0x7F, 0x4B, + 0xE2, 0xD8, 0xFF, 0x58, 0x05, 0xF5, 0x13, 0x22, + 0x29, 0xBD, 0xE9, 0x3B, 0xBB, 0xDC, 0xAF, 0x38, + 0x2B, 0xF1, 0xEE, 0x97, 0x2F, 0xBF, 0x99, 0x77, + 0xBA, 0xDA, 0x89, 0x45, 0x84, 0x7A, 0x2A, 0x6C, + 0x9A, 0xD3, 0x4A, 0x66, 0x75, 0x54, 0xE0, 0x4D, + 0x1F, 0x7F, 0xA2, 0xC3, 0x32, 0x41, 0xBD, 0x8F, + 0x01, 0xBA, 0x22, 0x0D + }, + .len = 800 + }, + .ciphertext = { + .data = { + 0x13, 0x1D, 0x43, 0xE0, 0xDE, 0xA1, 0xBE, 0x5C, + 0x5A, 0x1B, 0xFD, 0x97, 0x1D, 0x85, 0x2C, 0xBF, + 0x71, 0x2D, 0x7B, 0x4F, 0x57, 0x96, 0x1F, 0xEA, + 0x32, 0x08, 0xAF, 0xA8, 0xBC, 0xA4, 0x33, 0xF4, + 0x56, 0xAD, 0x09, 0xC7, 0x41, 0x7E, 0x58, 0xBC, + 0x69, 0xCF, 0x88, 0x66, 0xD1, 0x35, 0x3F, 0x74, + 0x86, 0x5E, 0x80, 0x78, 0x1D, 0x20, 0x2D, 0xFB, + 0x3E, 0xCF, 0xF7, 0xFC, 0xBC, 0x3B, 0x19, 0x0F, + 0xE8, 0x2A, 0x20, 0x4E, 0xD0, 0xE3, 0x50, 0xFC, + 0x0F, 0x6F, 0x26, 0x13, 0xB2, 0xF2, 0xBC, 0xA6, + 0xDF, 0x5A, 0x47, 0x3A, 0x57, 0xA4, 0xA0, 0x0D, + 0x98, 0x5E, 0xBA, 0xD8, 0x80, 0xD6, 0xF2, 0x38, + 0x64, 0xA0, 0x7B, 0x01 + }, + .len = 800 + }, + .validDataLenInBits = { + .len = 800 + }, + .validCipherLenInBits = { + .len = 800 + } +}; + +static struct wireless_test_data zuc_test_case_cipher_1570b = { + .key = { + .data = { + 0xD4, 0x55, 0x2A, 0x8F, 0xD6, 0xE6, 0x1C, 0xC8, + 0x1A, 0x20, 0x09, 0x14, 0x1A, 0x29, 0xC1, 0x0B + }, + .len = 16 + }, + .cipher_iv = { + .data = { + 0x76, 0x45, 0x2E, 0xC1, 0x14, 0x00, 0x00, 0x00, + 0x76, 0x45, 0x2E, 0xC1, 0x14, 0x00, 0x00, 0x00 + }, + .len = 16 + }, + .plaintext = { + .data = { + 0x38, 0xF0, 0x7F, 0x4B, 0xE2, 0xD8, 0xFF, 0x58, + 0x05, 0xF5, 0x13, 0x22, 0x29, 0xBD, 0xE9, 0x3B, + 0xBB, 0xDC, 0xAF, 0x38, 0x2B, 0xF1, 0xEE, 0x97, + 0x2F, 0xBF, 0x99, 0x77, 0xBA, 0xDA, 0x89, 0x45, + 0x84, 0x7A, 0x2A, 0x6C, 0x9A, 0xD3, 0x4A, 0x66, + 0x75, 0x54, 0xE0, 0x4D, 0x1F, 0x7F, 0xA2, 0xC3, + 0x32, 0x41, 0xBD, 0x8F, 0x01, 0xBA, 0x22, 0x0D, + 0x3C, 0xA4, 0xEC, 0x41, 0xE0, 0x74, 0x59, 0x5F, + 0x54, 0xAE, 0x2B, 0x45, 0x4F, 0xD9, 0x71, 0x43, + 0x20, 0x43, 0x60, 0x19, 0x65, 0xCC, 0xA8, 0x5C, + 0x24, 0x17, 0xED, 0x6C, 0xBE, 0xC3, 0xBA, 0xDA, + 0x84, 0xFC, 0x8A, 0x57, 0x9A, 0xEA, 0x78, 0x37, + 0xB0, 0x27, 0x11, 0x77, 0x24, 0x2A, 0x64, 0xDC, + 0x0A, 0x9D, 0xE7, 0x1A, 0x8E, 0xDE, 0xE8, 0x6C, + 0xA3, 0xD4, 0x7D, 0x03, 0x3D, 0x6B, 0xF5, 0x39, + 0x80, 0x4E, 0xCA, 0x86, 0xC5, 0x84, 0xA9, 0x05, + 0x2D, 0xE4, 0x6A, 0xD3, 0xFC, 0xED, 0x65, 0x54, + 0x3B, 0xD9, 0x02, 0x07, 0x37, 0x2B, 0x27, 0xAF, + 0xB7, 0x92, 0x34, 0xF5, 0xFF, 0x43, 0xEA, 0x87, + 0x08, 0x20, 0xE2, 0xC2, 0xB7, 0x8A, 0x8A, 0xAE, + 0x61, 0xCC, 0xE5, 0x2A, 0x05, 0x15, 0xE3, 0x48, + 0xD1, 0x96, 0x66, 0x4A, 0x34, 0x56, 0xB1, 0x82, + 0xA0, 0x7C, 0x40, 0x6E, 0x4A, 0x20, 0x79, 0x12, + 0x71, 0xCF, 0xED, 0xA1, 0x65, 0xD5, 0x35, 0xEC, + 0x5E, 0xA2, 0xD4, 0xDF, 0x40 + }, + .len = 1576 + }, + .ciphertext = { + .data = { + 0x83, 0x83, 0xB0, 0x22, 0x9F, 0xCC, 0x0B, 0x9D, + 0x22, 0x95, 0xEC, 0x41, 0xC9, 0x77, 0xE9, 0xC2, + 0xBB, 0x72, 0xE2, 0x20, 0x37, 0x81, 0x41, 0xF9, + 0xC8, 0x31, 0x8F, 0x3A, 0x27, 0x0D, 0xFB, 0xCD, + 0xEE, 0x64, 0x11, 0xC2, 0xB3, 0x04, 0x4F, 0x17, + 0x6D, 0xC6, 0xE0, 0x0F, 0x89, 0x60, 0xF9, 0x7A, + 0xFA, 0xCD, 0x13, 0x1A, 0xD6, 0xA3, 0xB4, 0x9B, + 0x16, 0xB7, 0xBA, 0xBC, 0xF2, 0xA5, 0x09, 0xEB, + 0xB1, 0x6A, 0x75, 0xDC, 0xAB, 0x14, 0xFF, 0x27, + 0x5D, 0xBE, 0xEE, 0xA1, 0xA2, 0xB1, 0x55, 0xF9, + 0xD5, 0x2C, 0x26, 0x45, 0x2D, 0x01, 0x87, 0xC3, + 0x10, 0xA4, 0xEE, 0x55, 0xBE, 0xAA, 0x78, 0xAB, + 0x40, 0x24, 0x61, 0x5B, 0xA9, 0xF5, 0xD5, 0xAD, + 0xC7, 0x72, 0x8F, 0x73, 0x56, 0x06, 0x71, 0xF0, + 0x13, 0xE5, 0xE5, 0x50, 0x08, 0x5D, 0x32, 0x91, + 0xDF, 0x7D, 0x5F, 0xEC, 0xED, 0xDE, 0xD5, 0x59, + 0x64, 0x1B, 0x6C, 0x2F, 0x58, 0x52, 0x33, 0xBC, + 0x71, 0xE9, 0x60, 0x2B, 0xD2, 0x30, 0x58, 0x55, + 0xBB, 0xD2, 0x5F, 0xFA, 0x7F, 0x17, 0xEC, 0xBC, + 0x04, 0x2D, 0xAA, 0xE3, 0x8C, 0x1F, 0x57, 0xAD, + 0x8E, 0x8E, 0xBD, 0x37, 0x34, 0x6F, 0x71, 0xBE, + 0xFD, 0xBB, 0x74, 0x32, 0xE0, 0xE0, 0xBB, 0x2C, + 0xFC, 0x09, 0xBC, 0xD9, 0x65, 0x70, 0xCB, 0x0C, + 0x0C, 0x39, 0xDF, 0x5E, 0x29, 0x29, 0x4E, 0x82, + 0x70, 0x3A, 0x63, 0x7F, 0x80 + }, + .len = 1576 + }, + .validDataLenInBits = { + .len = 1570 + }, + .validCipherLenInBits = { + .len = 1570 + } +}; + +static struct wireless_test_data zuc_test_case_cipher_2798b = { + .key = { + .data = { + 0xDB, 0x84, 0xB4, 0xFB, 0xCC, 0xDA, 0x56, 0x3B, + 0x66, 0x22, 0x7B, 0xFE, 0x45, 0x6F, 0x0F, 0x77 + }, + .len = 16 + }, + .cipher_iv = { + .data = { + 0xE4, 0x85, 0x0F, 0xE1, 0x84, 0x00, 0x00, 0x00, + 0xE4, 0x85, 0x0F, 0xE1, 0x84, 0x00, 0x00, 0x00 + }, + .len = 16 + }, + .plaintext = { + .data = { + 0xE5, 0x39, 0xF3, 0xB8, 0x97, 0x32, 0x40, 0xDA, + 0x03, 0xF2, 0xB8, 0xAA, 0x05, 0xEE, 0x0A, 0x00, + 0xDB, 0xAF, 0xC0, 0xE1, 0x82, 0x05, 0x5D, 0xFE, + 0x3D, 0x73, 0x83, 0xD9, 0x2C, 0xEF, 0x40, 0xE9, + 0x29, 0x28, 0x60, 0x5D, 0x52, 0xD0, 0x5F, 0x4F, + 0x90, 0x18, 0xA1, 0xF1, 0x89, 0xAE, 0x39, 0x97, + 0xCE, 0x19, 0x15, 0x5F, 0xB1, 0x22, 0x1D, 0xB8, + 0xBB, 0x09, 0x51, 0xA8, 0x53, 0xAD, 0x85, 0x2C, + 0xE1, 0x6C, 0xFF, 0x07, 0x38, 0x2C, 0x93, 0xA1, + 0x57, 0xDE, 0x00, 0xDD, 0xB1, 0x25, 0xC7, 0x53, + 0x9F, 0xD8, 0x50, 0x45, 0xE4, 0xEE, 0x07, 0xE0, + 0xC4, 0x3F, 0x9E, 0x9D, 0x6F, 0x41, 0x4F, 0xC4, + 0xD1, 0xC6, 0x29, 0x17, 0x81, 0x3F, 0x74, 0xC0, + 0x0F, 0xC8, 0x3F, 0x3E, 0x2E, 0xD7, 0xC4, 0x5B, + 0xA5, 0x83, 0x52, 0x64, 0xB4, 0x3E, 0x0B, 0x20, + 0xAF, 0xDA, 0x6B, 0x30, 0x53, 0xBF, 0xB6, 0x42, + 0x3B, 0x7F, 0xCE, 0x25, 0x47, 0x9F, 0xF5, 0xF1, + 0x39, 0xDD, 0x9B, 0x5B, 0x99, 0x55, 0x58, 0xE2, + 0xA5, 0x6B, 0xE1, 0x8D, 0xD5, 0x81, 0xCD, 0x01, + 0x7C, 0x73, 0x5E, 0x6F, 0x0D, 0x0D, 0x97, 0xC4, + 0xDD, 0xC1, 0xD1, 0xDA, 0x70, 0xC6, 0xDB, 0x4A, + 0x12, 0xCC, 0x92, 0x77, 0x8E, 0x2F, 0xBB, 0xD6, + 0xF3, 0xBA, 0x52, 0xAF, 0x91, 0xC9, 0xC6, 0xB6, + 0x4E, 0x8D, 0xA4, 0xF7, 0xA2, 0xC2, 0x66, 0xD0, + 0x2D, 0x00, 0x17, 0x53, 0xDF, 0x08, 0x96, 0x03, + 0x93, 0xC5, 0xD5, 0x68, 0x88, 0xBF, 0x49, 0xEB, + 0x5C, 0x16, 0xD9, 0xA8, 0x04, 0x27, 0xA4, 0x16, + 0xBC, 0xB5, 0x97, 0xDF, 0x5B, 0xFE, 0x6F, 0x13, + 0x89, 0x0A, 0x07, 0xEE, 0x13, 0x40, 0xE6, 0x47, + 0x6B, 0x0D, 0x9A, 0xA8, 0xF8, 0x22, 0xAB, 0x0F, + 0xD1, 0xAB, 0x0D, 0x20, 0x4F, 0x40, 0xB7, 0xCE, + 0x6F, 0x2E, 0x13, 0x6E, 0xB6, 0x74, 0x85, 0xE5, + 0x07, 0x80, 0x4D, 0x50, 0x45, 0x88, 0xAD, 0x37, + 0xFF, 0xD8, 0x16, 0x56, 0x8B, 0x2D, 0xC4, 0x03, + 0x11, 0xDF, 0xB6, 0x54, 0xCD, 0xEA, 0xD4, 0x7E, + 0x23, 0x85, 0xC3, 0x43, 0x62, 0x03, 0xDD, 0x83, + 0x6F, 0x9C, 0x64, 0xD9, 0x74, 0x62, 0xAD, 0x5D, + 0xFA, 0x63, 0xB5, 0xCF, 0xE0, 0x8A, 0xCB, 0x95, + 0x32, 0x86, 0x6F, 0x5C, 0xA7, 0x87, 0x56, 0x6F, + 0xCA, 0x93, 0xE6, 0xB1, 0x69, 0x3E, 0xE1, 0x5C, + 0xF6, 0xF7, 0xA2, 0xD6, 0x89, 0xD9, 0x74, 0x17, + 0x98, 0xDC, 0x1C, 0x23, 0x8E, 0x1B, 0xE6, 0x50, + 0x73, 0x3B, 0x18, 0xFB, 0x34, 0xFF, 0x88, 0x0E, + 0x16, 0xBB, 0xD2, 0x1B, 0x47, 0xAC + }, + .len = 2800 + }, + .ciphertext = { + .data = { + 0x4B, 0xBF, 0xA9, 0x1B, 0xA2, 0x5D, 0x47, 0xDB, + 0x9A, 0x9F, 0x19, 0x0D, 0x96, 0x2A, 0x19, 0xAB, + 0x32, 0x39, 0x26, 0xB3, 0x51, 0xFB, 0xD3, 0x9E, + 0x35, 0x1E, 0x05, 0xDA, 0x8B, 0x89, 0x25, 0xE3, + 0x0B, 0x1C, 0xCE, 0x0D, 0x12, 0x21, 0x10, 0x10, + 0x95, 0x81, 0x5C, 0xC7, 0xCB, 0x63, 0x19, 0x50, + 0x9E, 0xC0, 0xD6, 0x79, 0x40, 0x49, 0x19, 0x87, + 0xE1, 0x3F, 0x0A, 0xFF, 0xAC, 0x33, 0x2A, 0xA6, + 0xAA, 0x64, 0x62, 0x6D, 0x3E, 0x9A, 0x19, 0x17, + 0x51, 0x9E, 0x0B, 0x97, 0xB6, 0x55, 0xC6, 0xA1, + 0x65, 0xE4, 0x4C, 0xA9, 0xFE, 0xAC, 0x07, 0x90, + 0xD2, 0xA3, 0x21, 0xAD, 0x3D, 0x86, 0xB7, 0x9C, + 0x51, 0x38, 0x73, 0x9F, 0xA3, 0x8D, 0x88, 0x7E, + 0xC7, 0xDE, 0xF4, 0x49, 0xCE, 0x8A, 0xBD, 0xD3, + 0xE7, 0xF8, 0xDC, 0x4C, 0xA9, 0xE7, 0xB7, 0x33, + 0x14, 0xAD, 0x31, 0x0F, 0x90, 0x25, 0xE6, 0x19, + 0x46, 0xB3, 0xA5, 0x6D, 0xC6, 0x49, 0xEC, 0x0D, + 0xA0, 0xD6, 0x39, 0x43, 0xDF, 0xF5, 0x92, 0xCF, + 0x96, 0x2A, 0x7E, 0xFB, 0x2C, 0x85, 0x24, 0xE3, + 0x5A, 0x2A, 0x6E, 0x78, 0x79, 0xD6, 0x26, 0x04, + 0xEF, 0x26, 0x86, 0x95, 0xFA, 0x40, 0x03, 0x02, + 0x7E, 0x22, 0xE6, 0x08, 0x30, 0x77, 0x52, 0x20, + 0x64, 0xBD, 0x4A, 0x5B, 0x90, 0x6B, 0x5F, 0x53, + 0x12, 0x74, 0xF2, 0x35, 0xED, 0x50, 0x6C, 0xFF, + 0x01, 0x54, 0xC7, 0x54, 0x92, 0x8A, 0x0C, 0xE5, + 0x47, 0x6F, 0x2C, 0xB1, 0x02, 0x0A, 0x12, 0x22, + 0xD3, 0x2C, 0x14, 0x55, 0xEC, 0xAE, 0xF1, 0xE3, + 0x68, 0xFB, 0x34, 0x4D, 0x17, 0x35, 0xBF, 0xBE, + 0xDE, 0xB7, 0x1D, 0x0A, 0x33, 0xA2, 0xA5, 0x4B, + 0x1D, 0xA5, 0xA2, 0x94, 0xE6, 0x79, 0x14, 0x4D, + 0xDF, 0x11, 0xEB, 0x1A, 0x3D, 0xE8, 0xCF, 0x0C, + 0xC0, 0x61, 0x91, 0x79, 0x74, 0xF3, 0x5C, 0x1D, + 0x9C, 0xA0, 0xAC, 0x81, 0x80, 0x7F, 0x8F, 0xCC, + 0xE6, 0x19, 0x9A, 0x6C, 0x77, 0x12, 0xDA, 0x86, + 0x50, 0x21, 0xB0, 0x4C, 0xE0, 0x43, 0x95, 0x16, + 0xF1, 0xA5, 0x26, 0xCC, 0xDA, 0x9F, 0xD9, 0xAB, + 0xBD, 0x53, 0xC3, 0xA6, 0x84, 0xF9, 0xAE, 0x1E, + 0x7E, 0xE6, 0xB1, 0x1D, 0xA1, 0x38, 0xEA, 0x82, + 0x6C, 0x55, 0x16, 0xB5, 0xAA, 0xDF, 0x1A, 0xBB, + 0xE3, 0x6F, 0xA7, 0xFF, 0xF9, 0x2E, 0x3A, 0x11, + 0x76, 0x06, 0x4E, 0x8D, 0x95, 0xF2, 0xE4, 0x88, + 0x2B, 0x55, 0x00, 0xB9, 0x32, 0x28, 0xB2, 0x19, + 0x4A, 0x47, 0x5C, 0x1A, 0x27, 0xF6, 0x3F, 0x9F, + 0xFD, 0x26, 0x49, 0x89, 0xA1, 0xBC + }, + .len = 2800 + }, + .validDataLenInBits = { + .len = 2798 + }, + .validCipherLenInBits = { + .len = 2798 + } +}; + +static struct wireless_test_data zuc_test_case_cipher_4019b = { + .key = { + .data = { + 0xE1, 0x3F, 0xED, 0x21, 0xB4, 0x6E, 0x4E, 0x7E, + 0xC3, 0x12, 0x53, 0xB2, 0xBB, 0x17, 0xB3, 0xE0 + }, + .len = 16 + }, + .cipher_iv = { + .data = { + 0x27, 0x38, 0xCD, 0xAA, 0xD0, 0x00, 0x00, 0x00, + 0x27, 0x38, 0xCD, 0xAA, 0xD0, 0x00, 0x00, 0x00 + }, + .len = 16 + }, + .plaintext = { + .data = { + 0x8D, 0x74, 0xE2, 0x0D, 0x54, 0x89, 0x4E, 0x06, + 0xD3, 0xCB, 0x13, 0xCB, 0x39, 0x33, 0x06, 0x5E, + 0x86, 0x74, 0xBE, 0x62, 0xAD, 0xB1, 0xC7, 0x2B, + 0x3A, 0x64, 0x69, 0x65, 0xAB, 0x63, 0xCB, 0x7B, + 0x78, 0x54, 0xDF, 0xDC, 0x27, 0xE8, 0x49, 0x29, + 0xF4, 0x9C, 0x64, 0xB8, 0x72, 0xA4, 0x90, 0xB1, + 0x3F, 0x95, 0x7B, 0x64, 0x82, 0x7E, 0x71, 0xF4, + 0x1F, 0xBD, 0x42, 0x69, 0xA4, 0x2C, 0x97, 0xF8, + 0x24, 0x53, 0x70, 0x27, 0xF8, 0x6E, 0x9F, 0x4A, + 0xD8, 0x2D, 0x1D, 0xF4, 0x51, 0x69, 0x0F, 0xDD, + 0x98, 0xB6, 0xD0, 0x3F, 0x3A, 0x0E, 0xBE, 0x3A, + 0x31, 0x2D, 0x6B, 0x84, 0x0B, 0xA5, 0xA1, 0x82, + 0x0B, 0x2A, 0x2C, 0x97, 0x09, 0xC0, 0x90, 0xD2, + 0x45, 0xED, 0x26, 0x7C, 0xF8, 0x45, 0xAE, 0x41, + 0xFA, 0x97, 0x5D, 0x33, 0x33, 0xAC, 0x30, 0x09, + 0xFD, 0x40, 0xEB, 0xA9, 0xEB, 0x5B, 0x88, 0x57, + 0x14, 0xB7, 0x68, 0xB6, 0x97, 0x13, 0x8B, 0xAF, + 0x21, 0x38, 0x0E, 0xCA, 0x49, 0xF6, 0x44, 0xD4, + 0x86, 0x89, 0xE4, 0x21, 0x57, 0x60, 0xB9, 0x06, + 0x73, 0x9F, 0x0D, 0x2B, 0x3F, 0x09, 0x11, 0x33, + 0xCA, 0x15, 0xD9, 0x81, 0xCB, 0xE4, 0x01, 0xBA, + 0xF7, 0x2D, 0x05, 0xAC, 0xE0, 0x5C, 0xCC, 0xB2, + 0xD2, 0x97, 0xF4, 0xEF, 0x6A, 0x5F, 0x58, 0xD9, + 0x12, 0x46, 0xCF, 0xA7, 0x72, 0x15, 0xB8, 0x92, + 0xAB, 0x44, 0x1D, 0x52, 0x78, 0x45, 0x27, 0x95, + 0xCC, 0xB7, 0xF5, 0xD7, 0x90, 0x57, 0xA1, 0xC4, + 0xF7, 0x7F, 0x80, 0xD4, 0x6D, 0xB2, 0x03, 0x3C, + 0xB7, 0x9B, 0xED, 0xF8, 0xE6, 0x05, 0x51, 0xCE, + 0x10, 0xC6, 0x67, 0xF6, 0x2A, 0x97, 0xAB, 0xAF, + 0xAB, 0xBC, 0xD6, 0x77, 0x20, 0x18, 0xDF, 0x96, + 0xA2, 0x82, 0xEA, 0x73, 0x7C, 0xE2, 0xCB, 0x33, + 0x12, 0x11, 0xF6, 0x0D, 0x53, 0x54, 0xCE, 0x78, + 0xF9, 0x91, 0x8D, 0x9C, 0x20, 0x6C, 0xA0, 0x42, + 0xC9, 0xB6, 0x23, 0x87, 0xDD, 0x70, 0x96, 0x04, + 0xA5, 0x0A, 0xF1, 0x6D, 0x8D, 0x35, 0xA8, 0x90, + 0x6B, 0xE4, 0x84, 0xCF, 0x2E, 0x74, 0xA9, 0x28, + 0x99, 0x40, 0x36, 0x43, 0x53, 0x24, 0x9B, 0x27, + 0xB4, 0xC9, 0xAE, 0x29, 0xED, 0xDF, 0xC7, 0xDA, + 0x64, 0x18, 0x79, 0x1A, 0x4E, 0x7B, 0xAA, 0x06, + 0x60, 0xFA, 0x64, 0x51, 0x1F, 0x2D, 0x68, 0x5C, + 0xC3, 0xA5, 0xFF, 0x70, 0xE0, 0xD2, 0xB7, 0x42, + 0x92, 0xE3, 0xB8, 0xA0, 0xCD, 0x6B, 0x04, 0xB1, + 0xC7, 0x90, 0xB8, 0xEA, 0xD2, 0x70, 0x37, 0x08, + 0x54, 0x0D, 0xEA, 0x2F, 0xC0, 0x9C, 0x3D, 0xA7, + 0x70, 0xF6, 0x54, 0x49, 0xE8, 0x4D, 0x81, 0x7A, + 0x4F, 0x55, 0x10, 0x55, 0xE1, 0x9A, 0xB8, 0x50, + 0x18, 0xA0, 0x02, 0x8B, 0x71, 0xA1, 0x44, 0xD9, + 0x67, 0x91, 0xE9, 0xA3, 0x57, 0x79, 0x33, 0x50, + 0x4E, 0xEE, 0x00, 0x60, 0x34, 0x0C, 0x69, 0xD2, + 0x74, 0xE1, 0xBF, 0x9D, 0x80, 0x5D, 0xCB, 0xCC, + 0x1A, 0x6F, 0xAA, 0x97, 0x68, 0x00, 0xB6, 0xFF, + 0x2B, 0x67, 0x1D, 0xC4, 0x63, 0x65, 0x2F, 0xA8, + 0xA3, 0x3E, 0xE5, 0x09, 0x74, 0xC1, 0xC2, 0x1B, + 0xE0, 0x1E, 0xAB, 0xB2, 0x16, 0x74, 0x30, 0x26, + 0x9D, 0x72, 0xEE, 0x51, 0x1C, 0x9D, 0xDE, 0x30, + 0x79, 0x7C, 0x9A, 0x25, 0xD8, 0x6C, 0xE7, 0x4F, + 0x5B, 0x96, 0x1B, 0xE5, 0xFD, 0xFB, 0x68, 0x07, + 0x81, 0x40, 0x39, 0xE7, 0x13, 0x76, 0x36, 0xBD, + 0x1D, 0x7F, 0xA9, 0xE0, 0x9E, 0xFD, 0x20, 0x07, + 0x50, 0x59, 0x06, 0xA5, 0xAC, 0x45, 0xDF, 0xDE, + 0xED, 0x77, 0x57, 0xBB, 0xEE, 0x74, 0x57, 0x49, + 0xC2, 0x96, 0x33, 0x35, 0x0B, 0xEE, 0x0E, 0xA6, + 0xF4, 0x09, 0xDF, 0x45, 0x80, 0x16, 0x00 + }, + .len = 4024 + }, + .ciphertext = { + .data = { + 0x94, 0xEA, 0xA4, 0xAA, 0x30, 0xA5, 0x71, 0x37, + 0xDD, 0xF0, 0x9B, 0x97, 0xB2, 0x56, 0x18, 0xA2, + 0x0A, 0x13, 0xE2, 0xF1, 0x0F, 0xA5, 0xBF, 0x81, + 0x61, 0xA8, 0x79, 0xCC, 0x2A, 0xE7, 0x97, 0xA6, + 0xB4, 0xCF, 0x2D, 0x9D, 0xF3, 0x1D, 0xEB, 0xB9, + 0x90, 0x5C, 0xCF, 0xEC, 0x97, 0xDE, 0x60, 0x5D, + 0x21, 0xC6, 0x1A, 0xB8, 0x53, 0x1B, 0x7F, 0x3C, + 0x9D, 0xA5, 0xF0, 0x39, 0x31, 0xF8, 0xA0, 0x64, + 0x2D, 0xE4, 0x82, 0x11, 0xF5, 0xF5, 0x2F, 0xFE, + 0xA1, 0x0F, 0x39, 0x2A, 0x04, 0x76, 0x69, 0x98, + 0x5D, 0xA4, 0x54, 0xA2, 0x8F, 0x08, 0x09, 0x61, + 0xA6, 0xC2, 0xB6, 0x2D, 0xAA, 0x17, 0xF3, 0x3C, + 0xD6, 0x0A, 0x49, 0x71, 0xF4, 0x8D, 0x2D, 0x90, + 0x93, 0x94, 0xA5, 0x5F, 0x48, 0x11, 0x7A, 0xCE, + 0x43, 0xD7, 0x08, 0xE6, 0xB7, 0x7D, 0x3D, 0xC4, + 0x6D, 0x8B, 0xC0, 0x17, 0xD4, 0xD1, 0xAB, 0xB7, + 0x7B, 0x74, 0x28, 0xC0, 0x42, 0xB0, 0x6F, 0x2F, + 0x99, 0xD8, 0xD0, 0x7C, 0x98, 0x79, 0xD9, 0x96, + 0x00, 0x12, 0x7A, 0x31, 0x98, 0x5F, 0x10, 0x99, + 0xBB, 0xD7, 0xD6, 0xC1, 0x51, 0x9E, 0xDE, 0x8F, + 0x5E, 0xEB, 0x4A, 0x61, 0x0B, 0x34, 0x9A, 0xC0, + 0x1E, 0xA2, 0x35, 0x06, 0x91, 0x75, 0x6B, 0xD1, + 0x05, 0xC9, 0x74, 0xA5, 0x3E, 0xDD, 0xB3, 0x5D, + 0x1D, 0x41, 0x00, 0xB0, 0x12, 0xE5, 0x22, 0xAB, + 0x41, 0xF4, 0xC5, 0xF2, 0xFD, 0xE7, 0x6B, 0x59, + 0xCB, 0x8B, 0x96, 0xD8, 0x85, 0xCF, 0xE4, 0x08, + 0x0D, 0x13, 0x28, 0xA0, 0xD6, 0x36, 0xCC, 0x0E, + 0xDC, 0x05, 0x80, 0x0B, 0x76, 0xAC, 0xCA, 0x8F, + 0xEF, 0x67, 0x20, 0x84, 0xD1, 0xF5, 0x2A, 0x8B, + 0xBD, 0x8E, 0x09, 0x93, 0x32, 0x09, 0x92, 0xC7, + 0xFF, 0xBA, 0xE1, 0x7C, 0x40, 0x84, 0x41, 0xE0, + 0xEE, 0x88, 0x3F, 0xC8, 0xA8, 0xB0, 0x5E, 0x22, + 0xF5, 0xFF, 0x7F, 0x8D, 0x1B, 0x48, 0xC7, 0x4C, + 0x46, 0x8C, 0x46, 0x7A, 0x02, 0x8F, 0x09, 0xFD, + 0x7C, 0xE9, 0x11, 0x09, 0xA5, 0x70, 0xA2, 0xD5, + 0xC4, 0xD5, 0xF4, 0xFA, 0x18, 0xC5, 0xDD, 0x3E, + 0x45, 0x62, 0xAF, 0xE2, 0x4E, 0xF7, 0x71, 0x90, + 0x1F, 0x59, 0xAF, 0x64, 0x58, 0x98, 0xAC, 0xEF, + 0x08, 0x8A, 0xBA, 0xE0, 0x7E, 0x92, 0xD5, 0x2E, + 0xB2, 0xDE, 0x55, 0x04, 0x5B, 0xB1, 0xB7, 0xC4, + 0x16, 0x4E, 0xF2, 0xD7, 0xA6, 0xCA, 0xC1, 0x5E, + 0xEB, 0x92, 0x6D, 0x7E, 0xA2, 0xF0, 0x8B, 0x66, + 0xE1, 0xF7, 0x59, 0xF3, 0xAE, 0xE4, 0x46, 0x14, + 0x72, 0x5A, 0xA3, 0xC7, 0x48, 0x2B, 0x30, 0x84, + 0x4C, 0x14, 0x3F, 0xF8, 0x5B, 0x53, 0xF1, 0xE5, + 0x83, 0xC5, 0x01, 0x25, 0x7D, 0xDD, 0xD0, 0x96, + 0xB8, 0x12, 0x68, 0xDA, 0xA3, 0x03, 0xF1, 0x72, + 0x34, 0xC2, 0x33, 0x35, 0x41, 0xF0, 0xBB, 0x8E, + 0x19, 0x06, 0x48, 0xC5, 0x80, 0x7C, 0x86, 0x6D, + 0x71, 0x93, 0x22, 0x86, 0x09, 0xAD, 0xB9, 0x48, + 0x68, 0x6F, 0x7D, 0xE2, 0x94, 0xA8, 0x02, 0xCC, + 0x38, 0xF7, 0xFE, 0x52, 0x08, 0xF5, 0xEA, 0x31, + 0x96, 0xD0, 0x16, 0x7B, 0x9B, 0xDD, 0x02, 0xF0, + 0xD2, 0xA5, 0x22, 0x1C, 0xA5, 0x08, 0xF8, 0x93, + 0xAF, 0x5C, 0x4B, 0x4B, 0xB9, 0xF4, 0xF5, 0x20, + 0xFD, 0x84, 0x28, 0x9B, 0x3D, 0xBE, 0x7E, 0x61, + 0x49, 0x7A, 0x7E, 0x2A, 0x58, 0x40, 0x37, 0xEA, + 0x63, 0x7B, 0x69, 0x81, 0x12, 0x71, 0x74, 0xAF, + 0x57, 0xB4, 0x71, 0xDF, 0x4B, 0x27, 0x68, 0xFD, + 0x79, 0xC1, 0x54, 0x0F, 0xB3, 0xED, 0xF2, 0xEA, + 0x22, 0xCB, 0x69, 0xBE, 0xC0, 0xCF, 0x8D, 0x93, + 0x3D, 0x9C, 0x6F, 0xDD, 0x64, 0x5E, 0x85, 0x05, + 0x91, 0xCC, 0xA3, 0xD6, 0x2C, 0x0C, 0xC0 + }, + .len = 4024 + }, + .validDataLenInBits = { + .len = 4019 + }, + .validCipherLenInBits = { + .len = 4019 + } +}; + +static struct wireless_test_data zuc_test_case_cipher_200b_auth_200b = { + .key = { + .data = { + 0x17, 0x3D, 0x14, 0xBA, 0x50, 0x03, 0x73, 0x1D, + 0x7A, 0x60, 0x04, 0x94, 0x70, 0xF0, 0x0A, 0x29 + }, + .len = 16 + }, + .cipher_iv = { + .data = { + 0x66, 0x03, 0x54, 0x92, 0x78, 0x00, 0x00, 0x00, + 0x66, 0x03, 0x54, 0x92, 0x78, 0x00, 0x00, 0x00 + }, + .len = 16 + }, + .plaintext = { + .data = { + 0x6C, 0xF6, 0x53, 0x40, 0x73, 0x55, 0x52, 0xAB, + 0x0C, 0x97, 0x52, 0xFA, 0x6F, 0x90, 0x25, 0xFE, + 0x0B, 0xD6, 0x75, 0xD9, 0x00, 0x58, 0x75, 0xB2, + 0x00 + }, + .len = 200 + }, + .ciphertext = { + .data = { + 0xA6, 0xC8, 0x5F, 0xC6, 0x6A, 0xFB, 0x85, 0x33, + 0xAA, 0xFC, 0x25, 0x18, 0xDF, 0xE7, 0x84, 0x94, + 0x0E, 0xE1, 0xE4, 0xB0, 0x30, 0x23, 0x8C, 0xC8, + 0x10 + }, + .len = 200 + }, + .validDataLenInBits = { + .len = 200 + }, + .validCipherLenInBits = { + .len = 200 + }, + .auth_iv = { + .data = { + 0xFA, 0x55, 0x6B, 0x26, 0x1C, 0x00, 0x00, 0x00, + 0xFA, 0x55, 0x6B, 0x26, 0x1C, 0x00, 0x00, 0x00 + }, + .len = 16 + }, + .digest = { + .data = {0x01, 0xFE, 0x5E, 0x38}, + .len = 4 + }, + .validAuthLenInBits = { + .len = 200 + } +}; + +static struct wireless_test_data zuc_test_case_cipher_800b_auth_120b = { + .key = { + .data = { + 0xE5, 0xBD, 0x3E, 0xA0, 0xEB, 0x55, 0xAD, 0xE8, + 0x66, 0xC6, 0xAC, 0x58, 0xBD, 0x54, 0x30, 0x2A + }, + .len = 16 + }, + .cipher_iv = { + .data = { + 0x00, 0x05, 0x68, 0x23, 0xC4, 0x00, 0x00, 0x00, + 0x00, 0x05, 0x68, 0x23, 0xC4, 0x00, 0x00, 0x00 + }, + .len = 16 + }, + .plaintext = { + .data = { + 0x14, 0xA8, 0xEF, 0x69, 0x3D, 0x67, 0x85, 0x07, + 0xBB, 0xE7, 0x27, 0x0A, 0x7F, 0x67, 0xFF, 0x50, + 0x06, 0xC3, 0x52, 0x5B, 0x98, 0x07, 0xE4, 0x67, + 0xC4, 0xE5, 0x60, 0x00, 0xBA, 0x33, 0x8F, 0x5D, + 0x42, 0x95, 0x59, 0x03, 0x67, 0x51, 0x82, 0x22, + 0x46, 0xC8, 0x0D, 0x3B, 0x38, 0xF0, 0x7F, 0x4B, + 0xE2, 0xD8, 0xFF, 0x58, 0x05, 0xF5, 0x13, 0x22, + 0x29, 0xBD, 0xE9, 0x3B, 0xBB, 0xDC, 0xAF, 0x38, + 0x2B, 0xF1, 0xEE, 0x97, 0x2F, 0xBF, 0x99, 0x77, + 0xBA, 0xDA, 0x89, 0x45, 0x84, 0x7A, 0x2A, 0x6C, + 0x9A, 0xD3, 0x4A, 0x66, 0x75, 0x54, 0xE0, 0x4D, + 0x1F, 0x7F, 0xA2, 0xC3, 0x32, 0x41, 0xBD, 0x8F, + 0x01, 0xBA, 0x22, 0x0D + }, + .len = 800 + }, + .ciphertext = { + .data = { + 0x13, 0x1D, 0x43, 0xE0, 0xDE, 0xA1, 0xBE, 0x5C, + 0x5A, 0x1B, 0xFD, 0x97, 0x1D, 0x85, 0x2C, 0xBF, + 0x71, 0x2D, 0x7B, 0x4F, 0x57, 0x96, 0x1F, 0xEA, + 0x32, 0x08, 0xAF, 0xA8, 0xBC, 0xA4, 0x33, 0xF4, + 0x56, 0xAD, 0x09, 0xC7, 0x41, 0x7E, 0x58, 0xBC, + 0x69, 0xCF, 0x88, 0x66, 0xD1, 0x35, 0x3F, 0x74, + 0x86, 0x5E, 0x80, 0x78, 0x1D, 0x20, 0x2D, 0xFB, + 0x3E, 0xCF, 0xF7, 0xFC, 0xBC, 0x3B, 0x19, 0x0F, + 0xE8, 0x2A, 0x20, 0x4E, 0xD0, 0xE3, 0x50, 0xFC, + 0x0F, 0x6F, 0x26, 0x13, 0xB2, 0xF2, 0xBC, 0xA6, + 0xDF, 0x5A, 0x47, 0x3A, 0x57, 0xA4, 0xA0, 0x0D, + 0x98, 0x5E, 0xBA, 0xD8, 0x80, 0xD6, 0xF2, 0x38, + 0x64, 0xA0, 0x7B, 0x01 + }, + .len = 800 + }, + .validDataLenInBits = { + .len = 800 + }, + .validCipherLenInBits = { + .len = 800 + }, + .auth_iv = { + .data = { + 0xFA, 0x55, 0x6B, 0x26, 0x1C, 0x00, 0x00, 0x00, + 0xFA, 0x55, 0x6B, 0x26, 0x1C, 0x00, 0x00, 0x00 + }, + .len = 16 + }, + .digest = { + .data = {0x9D, 0x42, 0x1C, 0xEA}, + .len = 4 + }, + .validAuthLenInBits = { + .len = 120 + } +}; + +struct wireless_test_data zuc_test_case_auth_1b = { + .key = { + .data = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + }, + .len = 16 + }, + .auth_iv = { + .data = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + }, + .len = 16 + }, + .plaintext = { + .data = {0x00}, + .len = 8 + }, + .validAuthLenInBits = { + .len = 1 + }, + .digest = { + .data = {0xC8, 0xA9, 0x59, 0x5E}, + .len = 4 + } +}; + +struct wireless_test_data zuc_test_case_auth_90b = { + .key = { + .data = { + 0x47, 0x05, 0x41, 0x25, 0x56, 0x1E, 0xB2, 0xDD, + 0xA9, 0x40, 0x59, 0xDA, 0x05, 0x09, 0x78, 0x50 + }, + .len = 16 + }, + .auth_iv = { + .data = { + 0x56, 0x1E, 0xB2, 0xDD, 0xA0, 0x00, 0x00, 0x00, + 0x56, 0x1E, 0xB2, 0xDD, 0xA0, 0x00, 0x00, 0x00 + }, + .len = 16 + }, + .plaintext = { + .data = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00 + }, + .len = 96 + }, + .validAuthLenInBits = { + .len = 90 + }, + .digest = { + .data = {0x67, 0x19, 0xA0, 0x88}, + .len = 4 + } +}; + +struct wireless_test_data zuc_test_case_auth_577b = { + .key = { + .data = { + 0xC9, 0xE6, 0xCE, 0xC4, 0x60, 0x7C, 0x72, 0xDB, + 0x00, 0x0A, 0xEF, 0xA8, 0x83, 0x85, 0xAB, 0x0A + }, + .len = 16 + }, + .auth_iv = { + .data = { + 0xA9, 0x40, 0x59, 0xDA, 0x50, 0x00, 0x00, 0x00, + 0x29, 0x40, 0x59, 0xDA, 0x50, 0x00, 0x80, 0x00 + }, + .len = 16 + }, + .plaintext = { + .data = { + 0x98, 0x3B, 0x41, 0xD4, 0x7D, 0x78, 0x0C, 0x9E, + 0x1A, 0xD1, 0x1D, 0x7E, 0xB7, 0x03, 0x91, 0xB1, + 0xDE, 0x0B, 0x35, 0xDA, 0x2D, 0xC6, 0x2F, 0x83, + 0xE7, 0xB7, 0x8D, 0x63, 0x06, 0xCA, 0x0E, 0xA0, + 0x7E, 0x94, 0x1B, 0x7B, 0xE9, 0x13, 0x48, 0xF9, + 0xFC, 0xB1, 0x70, 0xE2, 0x21, 0x7F, 0xEC, 0xD9, + 0x7F, 0x9F, 0x68, 0xAD, 0xB1, 0x6E, 0x5D, 0x7D, + 0x21, 0xE5, 0x69, 0xD2, 0x80, 0xED, 0x77, 0x5C, + 0xEB, 0xDE, 0x3F, 0x40, 0x93, 0xC5, 0x38, 0x81, + 0x00 + }, + .len = 584 + }, + .validAuthLenInBits = { + .len = 577 + }, + .digest = { + .data = {0xFA, 0xE8, 0xFF, 0x0B}, + .len = 4 + } +}; + +struct wireless_test_data zuc_test_case_auth_2079b = { + .key = { + .data = { + 0xC8, 0xA4, 0x82, 0x62, 0xD0, 0xC2, 0xE2, 0xBA, + 0xC4, 0xB9, 0x6E, 0xF7, 0x7E, 0x80, 0xCA, 0x59 + }, + .len = 16 + }, + .auth_iv = { + .data = { + 0x05, 0x09, 0x78, 0x50, 0x80, 0x00, 0x00, 0x00, + 0x85, 0x09, 0x78, 0x50, 0x80, 0x00, 0x80, 0x00 + }, + .len = 16 + }, + .plaintext = { + .data = { + 0xB5, 0x46, 0x43, 0x0B, 0xF8, 0x7B, 0x4F, 0x1E, + 0xE8, 0x34, 0x70, 0x4C, 0xD6, 0x95, 0x1C, 0x36, + 0xE2, 0x6F, 0x10, 0x8C, 0xF7, 0x31, 0x78, 0x8F, + 0x48, 0xDC, 0x34, 0xF1, 0x67, 0x8C, 0x05, 0x22, + 0x1C, 0x8F, 0xA7, 0xFF, 0x2F, 0x39, 0xF4, 0x77, + 0xE7, 0xE4, 0x9E, 0xF6, 0x0A, 0x4E, 0xC2, 0xC3, + 0xDE, 0x24, 0x31, 0x2A, 0x96, 0xAA, 0x26, 0xE1, + 0xCF, 0xBA, 0x57, 0x56, 0x38, 0x38, 0xB2, 0x97, + 0xF4, 0x7E, 0x85, 0x10, 0xC7, 0x79, 0xFD, 0x66, + 0x54, 0xB1, 0x43, 0x38, 0x6F, 0xA6, 0x39, 0xD3, + 0x1E, 0xDB, 0xD6, 0xC0, 0x6E, 0x47, 0xD1, 0x59, + 0xD9, 0x43, 0x62, 0xF2, 0x6A, 0xEE, 0xED, 0xEE, + 0x0E, 0x4F, 0x49, 0xD9, 0xBF, 0x84, 0x12, 0x99, + 0x54, 0x15, 0xBF, 0xAD, 0x56, 0xEE, 0x82, 0xD1, + 0xCA, 0x74, 0x63, 0xAB, 0xF0, 0x85, 0xB0, 0x82, + 0xB0, 0x99, 0x04, 0xD6, 0xD9, 0x90, 0xD4, 0x3C, + 0xF2, 0xE0, 0x62, 0xF4, 0x08, 0x39, 0xD9, 0x32, + 0x48, 0xB1, 0xEB, 0x92, 0xCD, 0xFE, 0xD5, 0x30, + 0x0B, 0xC1, 0x48, 0x28, 0x04, 0x30, 0xB6, 0xD0, + 0xCA, 0xA0, 0x94, 0xB6, 0xEC, 0x89, 0x11, 0xAB, + 0x7D, 0xC3, 0x68, 0x24, 0xB8, 0x24, 0xDC, 0x0A, + 0xF6, 0x68, 0x2B, 0x09, 0x35, 0xFD, 0xE7, 0xB4, + 0x92, 0xA1, 0x4D, 0xC2, 0xF4, 0x36, 0x48, 0x03, + 0x8D, 0xA2, 0xCF, 0x79, 0x17, 0x0D, 0x2D, 0x50, + 0x13, 0x3F, 0xD4, 0x94, 0x16, 0xCB, 0x6E, 0x33, + 0xBE, 0xA9, 0x0B, 0x8B, 0xF4, 0x55, 0x9B, 0x03, + 0x73, 0x2A, 0x01, 0xEA, 0x29, 0x0E, 0x6D, 0x07, + 0x4F, 0x79, 0xBB, 0x83, 0xC1, 0x0E, 0x58, 0x00, + 0x15, 0xCC, 0x1A, 0x85, 0xB3, 0x6B, 0x55, 0x01, + 0x04, 0x6E, 0x9C, 0x4B, 0xDC, 0xAE, 0x51, 0x35, + 0x69, 0x0B, 0x86, 0x66, 0xBD, 0x54, 0xB7, 0xA7, + 0x03, 0xEA, 0x7B, 0x6F, 0x22, 0x0A, 0x54, 0x69, + 0xA5, 0x68, 0x02, 0x7E + }, + .len = 2080 + }, + .validAuthLenInBits = { + .len = 2079 + }, + .digest = { + .data = {0x00, 0x4A, 0xC4, 0xD6}, + .len = 4 + } +}; + +struct wireless_test_data zuc_test_auth_5670b = { + .key = { + .data = { + 0x6B, 0x8B, 0x08, 0xEE, 0x79, 0xE0, 0xB5, 0x98, + 0x2D, 0x6D, 0x12, 0x8E, 0xA9, 0xF2, 0x20, 0xCB + }, + .len = 16 + }, + .auth_iv = { + .data = { + 0x56, 0x1E, 0xB2, 0xDD, 0xE0, 0x00, 0x00, 0x00, + 0x56, 0x1E, 0xB2, 0xDD, 0xE0, 0x00, 0x00, 0x00 + }, + .len = 16 + }, + .plaintext = { + .data = { + 0x5B, 0xAD, 0x72, 0x47, 0x10, 0xBA, 0x1C, 0x56, + 0xD5, 0xA3, 0x15, 0xF8, 0xD4, 0x0F, 0x6E, 0x09, + 0x37, 0x80, 0xBE, 0x8E, 0x8D, 0xE0, 0x7B, 0x69, + 0x92, 0x43, 0x20, 0x18, 0xE0, 0x8E, 0xD9, 0x6A, + 0x57, 0x34, 0xAF, 0x8B, 0xAD, 0x8A, 0x57, 0x5D, + 0x3A, 0x1F, 0x16, 0x2F, 0x85, 0x04, 0x5C, 0xC7, + 0x70, 0x92, 0x55, 0x71, 0xD9, 0xF5, 0xB9, 0x4E, + 0x45, 0x4A, 0x77, 0xC1, 0x6E, 0x72, 0x93, 0x6B, + 0xF0, 0x16, 0xAE, 0x15, 0x74, 0x99, 0xF0, 0x54, + 0x3B, 0x5D, 0x52, 0xCA, 0xA6, 0xDB, 0xEA, 0xB6, + 0x97, 0xD2, 0xBB, 0x73, 0xE4, 0x1B, 0x80, 0x75, + 0xDC, 0xE7, 0x9B, 0x4B, 0x86, 0x04, 0x4F, 0x66, + 0x1D, 0x44, 0x85, 0xA5, 0x43, 0xDD, 0x78, 0x60, + 0x6E, 0x04, 0x19, 0xE8, 0x05, 0x98, 0x59, 0xD3, + 0xCB, 0x2B, 0x67, 0xCE, 0x09, 0x77, 0x60, 0x3F, + 0x81, 0xFF, 0x83, 0x9E, 0x33, 0x18, 0x59, 0x54, + 0x4C, 0xFB, 0xC8, 0xD0, 0x0F, 0xEF, 0x1A, 0x4C, + 0x85, 0x10, 0xFB, 0x54, 0x7D, 0x6B, 0x06, 0xC6, + 0x11, 0xEF, 0x44, 0xF1, 0xBC, 0xE1, 0x07, 0xCF, + 0xA4, 0x5A, 0x06, 0xAA, 0xB3, 0x60, 0x15, 0x2B, + 0x28, 0xDC, 0x1E, 0xBE, 0x6F, 0x7F, 0xE0, 0x9B, + 0x05, 0x16, 0xF9, 0xA5, 0xB0, 0x2A, 0x1B, 0xD8, + 0x4B, 0xB0, 0x18, 0x1E, 0x2E, 0x89, 0xE1, 0x9B, + 0xD8, 0x12, 0x59, 0x30, 0xD1, 0x78, 0x68, 0x2F, + 0x38, 0x62, 0xDC, 0x51, 0xB6, 0x36, 0xF0, 0x4E, + 0x72, 0x0C, 0x47, 0xC3, 0xCE, 0x51, 0xAD, 0x70, + 0xD9, 0x4B, 0x9B, 0x22, 0x55, 0xFB, 0xAE, 0x90, + 0x65, 0x49, 0xF4, 0x99, 0xF8, 0xC6, 0xD3, 0x99, + 0x47, 0xED, 0x5E, 0x5D, 0xF8, 0xE2, 0xDE, 0xF1, + 0x13, 0x25, 0x3E, 0x7B, 0x08, 0xD0, 0xA7, 0x6B, + 0x6B, 0xFC, 0x68, 0xC8, 0x12, 0xF3, 0x75, 0xC7, + 0x9B, 0x8F, 0xE5, 0xFD, 0x85, 0x97, 0x6A, 0xA6, + 0xD4, 0x6B, 0x4A, 0x23, 0x39, 0xD8, 0xAE, 0x51, + 0x47, 0xF6, 0x80, 0xFB, 0xE7, 0x0F, 0x97, 0x8B, + 0x38, 0xEF, 0xFD, 0x7B, 0x2F, 0x78, 0x66, 0xA2, + 0x25, 0x54, 0xE1, 0x93, 0xA9, 0x4E, 0x98, 0xA6, + 0x8B, 0x74, 0xBD, 0x25, 0xBB, 0x2B, 0x3F, 0x5F, + 0xB0, 0xA5, 0xFD, 0x59, 0x88, 0x7F, 0x9A, 0xB6, + 0x81, 0x59, 0xB7, 0x17, 0x8D, 0x5B, 0x7B, 0x67, + 0x7C, 0xB5, 0x46, 0xBF, 0x41, 0xEA, 0xDC, 0xA2, + 0x16, 0xFC, 0x10, 0x85, 0x01, 0x28, 0xF8, 0xBD, + 0xEF, 0x5C, 0x8D, 0x89, 0xF9, 0x6A, 0xFA, 0x4F, + 0xA8, 0xB5, 0x48, 0x85, 0x56, 0x5E, 0xD8, 0x38, + 0xA9, 0x50, 0xFE, 0xE5, 0xF1, 0xC3, 0xB0, 0xA4, + 0xF6, 0xFB, 0x71, 0xE5, 0x4D, 0xFD, 0x16, 0x9E, + 0x82, 0xCE, 0xCC, 0x72, 0x66, 0xC8, 0x50, 0xE6, + 0x7C, 0x5E, 0xF0, 0xBA, 0x96, 0x0F, 0x52, 0x14, + 0x06, 0x0E, 0x71, 0xEB, 0x17, 0x2A, 0x75, 0xFC, + 0x14, 0x86, 0x83, 0x5C, 0xBE, 0xA6, 0x53, 0x44, + 0x65, 0xB0, 0x55, 0xC9, 0x6A, 0x72, 0xE4, 0x10, + 0x52, 0x24, 0x18, 0x23, 0x25, 0xD8, 0x30, 0x41, + 0x4B, 0x40, 0x21, 0x4D, 0xAA, 0x80, 0x91, 0xD2, + 0xE0, 0xFB, 0x01, 0x0A, 0xE1, 0x5C, 0x6D, 0xE9, + 0x08, 0x50, 0x97, 0x3B, 0xDF, 0x1E, 0x42, 0x3B, + 0xE1, 0x48, 0xA2, 0x37, 0xB8, 0x7A, 0x0C, 0x9F, + 0x34, 0xD4, 0xB4, 0x76, 0x05, 0xB8, 0x03, 0xD7, + 0x43, 0xA8, 0x6A, 0x90, 0x39, 0x9A, 0x4A, 0xF3, + 0x96, 0xD3, 0xA1, 0x20, 0x0A, 0x62, 0xF3, 0xD9, + 0x50, 0x79, 0x62, 0xE8, 0xE5, 0xBE, 0xE6, 0xD3, + 0xDA, 0x2B, 0xB3, 0xF7, 0x23, 0x76, 0x64, 0xAC, + 0x7A, 0x29, 0x28, 0x23, 0x90, 0x0B, 0xC6, 0x35, + 0x03, 0xB2, 0x9E, 0x80, 0xD6, 0x3F, 0x60, 0x67, + 0xBF, 0x8E, 0x17, 0x16, 0xAC, 0x25, 0xBE, 0xBA, + 0x35, 0x0D, 0xEB, 0x62, 0xA9, 0x9F, 0xE0, 0x31, + 0x85, 0xEB, 0x4F, 0x69, 0x93, 0x7E, 0xCD, 0x38, + 0x79, 0x41, 0xFD, 0xA5, 0x44, 0xBA, 0x67, 0xDB, + 0x09, 0x11, 0x77, 0x49, 0x38, 0xB0, 0x18, 0x27, + 0xBC, 0xC6, 0x9C, 0x92, 0xB3, 0xF7, 0x72, 0xA9, + 0xD2, 0x85, 0x9E, 0xF0, 0x03, 0x39, 0x8B, 0x1F, + 0x6B, 0xBA, 0xD7, 0xB5, 0x74, 0xF7, 0x98, 0x9A, + 0x1D, 0x10, 0xB2, 0xDF, 0x79, 0x8E, 0x0D, 0xBF, + 0x30, 0xD6, 0x58, 0x74, 0x64, 0xD2, 0x48, 0x78, + 0xCD, 0x00, 0xC0, 0xEA, 0xEE, 0x8A, 0x1A, 0x0C, + 0xC7, 0x53, 0xA2, 0x79, 0x79, 0xE1, 0x1B, 0x41, + 0xDB, 0x1D, 0xE3, 0xD5, 0x03, 0x8A, 0xFA, 0xF4, + 0x9F, 0x5C, 0x68, 0x2C, 0x37, 0x48, 0xD8, 0xA3, + 0xA9, 0xEC, 0x54, 0xE6, 0xA3, 0x71, 0x27, 0x5F, + 0x16, 0x83, 0x51, 0x0F, 0x8E, 0x4F, 0x90, 0x93, + 0x8F, 0x9A, 0xB6, 0xE1, 0x34, 0xC2, 0xCF, 0xDF, + 0x48, 0x41, 0xCB, 0xA8, 0x8E, 0x0C, 0xFF, 0x2B, + 0x0B, 0xCC, 0x8E, 0x6A, 0xDC, 0xB7, 0x11, 0x09, + 0xB5, 0x19, 0x8F, 0xEC, 0xF1, 0xBB, 0x7E, 0x5C, + 0x53, 0x1A, 0xCA, 0x50, 0xA5, 0x6A, 0x8A, 0x3B, + 0x6D, 0xE5, 0x98, 0x62, 0xD4, 0x1F, 0xA1, 0x13, + 0xD9, 0xCD, 0x95, 0x78, 0x08, 0xF0, 0x85, 0x71, + 0xD9, 0xA4, 0xBB, 0x79, 0x2A, 0xF2, 0x71, 0xF6, + 0xCC, 0x6D, 0xBB, 0x8D, 0xC7, 0xEC, 0x36, 0xE3, + 0x6B, 0xE1, 0xED, 0x30, 0x81, 0x64, 0xC3, 0x1C, + 0x7C, 0x0A, 0xFC, 0x54, 0x1C + }, + .len = 5672 + }, + .validAuthLenInBits = { + .len = 5670 + }, + .digest = { + .data = {0x0C, 0xA1, 0x27, 0x92}, + .len = 4 + } +}; + +static struct wireless_test_data zuc_test_case_auth_128b = { + .key = { + .data = { 0x0 }, + .len = 16 + }, + .auth_iv = { + .data = { 0x0 }, + .len = 16 + }, + .plaintext = { + .data = { 0x0 }, + .len = 8 + }, + .validAuthLenInBits = { + .len = 8 + }, + .digest = { + .data = { 0x39, 0x0a, 0x91, 0xb7 }, + .len = 4 + } +}; + +static struct wireless_test_data zuc_test_case_auth_2080b = { + .key = { + .data = { + 0xC8, 0xA4, 0x82, 0x62, 0xD0, 0xC2, 0xE2, 0xBA, + 0xC4, 0xB9, 0x6E, 0xF7, 0x7E, 0x80, 0xCA, 0x59 + }, + .len = 16 + }, + .auth_iv = { + .data = { + 0x05, 0x09, 0x78, 0x50, 0x80, 0x00, 0x00, 0x00, + 0x85, 0x09, 0x78, 0x50, 0x80, 0x00, 0x80, 0x00 + }, + .len = 16 + }, + .plaintext = { + .data = { + 0xB5, 0x46, 0x43, 0x0B, 0xF8, 0x7B, 0x4F, 0x1E, + 0xE8, 0x34, 0x70, 0x4C, 0xD6, 0x95, 0x1C, 0x36, + 0xE2, 0x6F, 0x10, 0x8C, 0xF7, 0x31, 0x78, 0x8F, + 0x48, 0xDC, 0x34, 0xF1, 0x67, 0x8C, 0x05, 0x22, + 0x1C, 0x8F, 0xA7, 0xFF, 0x2F, 0x39, 0xF4, 0x77, + 0xE7, 0xE4, 0x9E, 0xF6, 0x0A, 0x4E, 0xC2, 0xC3, + 0xDE, 0x24, 0x31, 0x2A, 0x96, 0xAA, 0x26, 0xE1, + 0xCF, 0xBA, 0x57, 0x56, 0x38, 0x38, 0xB2, 0x97, + 0xF4, 0x7E, 0x85, 0x10, 0xC7, 0x79, 0xFD, 0x66, + 0x54, 0xB1, 0x43, 0x38, 0x6F, 0xA6, 0x39, 0xD3, + 0x1E, 0xDB, 0xD6, 0xC0, 0x6E, 0x47, 0xD1, 0x59, + 0xD9, 0x43, 0x62, 0xF2, 0x6A, 0xEE, 0xED, 0xEE, + 0x0E, 0x4F, 0x49, 0xD9, 0xBF, 0x84, 0x12, 0x99, + 0x54, 0x15, 0xBF, 0xAD, 0x56, 0xEE, 0x82, 0xD1, + 0xCA, 0x74, 0x63, 0xAB, 0xF0, 0x85, 0xB0, 0x82, + 0xB0, 0x99, 0x04, 0xD6, 0xD9, 0x90, 0xD4, 0x3C, + 0xF2, 0xE0, 0x62, 0xF4, 0x08, 0x39, 0xD9, 0x32, + 0x48, 0xB1, 0xEB, 0x92, 0xCD, 0xFE, 0xD5, 0x30, + 0x0B, 0xC1, 0x48, 0x28, 0x04, 0x30, 0xB6, 0xD0, + 0xCA, 0xA0, 0x94, 0xB6, 0xEC, 0x89, 0x11, 0xAB, + 0x7D, 0xC3, 0x68, 0x24, 0xB8, 0x24, 0xDC, 0x0A, + 0xF6, 0x68, 0x2B, 0x09, 0x35, 0xFD, 0xE7, 0xB4, + 0x92, 0xA1, 0x4D, 0xC2, 0xF4, 0x36, 0x48, 0x03, + 0x8D, 0xA2, 0xCF, 0x79, 0x17, 0x0D, 0x2D, 0x50, + 0x13, 0x3F, 0xD4, 0x94, 0x16, 0xCB, 0x6E, 0x33, + 0xBE, 0xA9, 0x0B, 0x8B, 0xF4, 0x55, 0x9B, 0x03, + 0x73, 0x2A, 0x01, 0xEA, 0x29, 0x0E, 0x6D, 0x07, + 0x4F, 0x79, 0xBB, 0x83, 0xC1, 0x0E, 0x58, 0x00, + 0x15, 0xCC, 0x1A, 0x85, 0xB3, 0x6B, 0x55, 0x01, + 0x04, 0x6E, 0x9C, 0x4B, 0xDC, 0xAE, 0x51, 0x35, + 0x69, 0x0B, 0x86, 0x66, 0xBD, 0x54, 0xB7, 0xA7, + 0x03, 0xEA, 0x7B, 0x6F, 0x22, 0x0A, 0x54, 0x69, + 0xA5, 0x68, 0x02, 0x7E + }, + .len = 2080 + }, + .validAuthLenInBits = { + .len = 2080 + }, + .digest = { + .data = {0x03, 0x95, 0x32, 0xe1}, + .len = 4 + } +}; + +static struct wireless_test_data zuc_test_case_auth_584b = { + .key = { + .data = { + 0xc9, 0xe6, 0xce, 0xc4, 0x60, 0x7c, 0x72, 0xdb, + 0x00, 0x0a, 0xef, 0xa8, 0x83, 0x85, 0xab, 0x0a + }, + .len = 16 + }, + .auth_iv = { + .data = { + 0xa9, 0x40, 0x59, 0xda, 0x50, 0x0, 0x0, 0x0, + 0x29, 0x40, 0x59, 0xda, 0x50, 0x0, 0x80, 0x0 + }, + .len = 16 + }, + .plaintext = { + .data = { + 0x98, 0x3b, 0x41, 0xd4, 0x7d, 0x78, 0x0c, 0x9e, + 0x1a, 0xd1, 0x1d, 0x7e, 0xb7, 0x03, 0x91, 0xb1, + 0xde, 0x0b, 0x35, 0xda, 0x2d, 0xc6, 0x2f, 0x83, + 0xe7, 0xb7, 0x8d, 0x63, 0x06, 0xca, 0x0e, 0xa0, + 0x7e, 0x94, 0x1b, 0x7b, 0xe9, 0x13, 0x48, 0xf9, + 0xfc, 0xb1, 0x70, 0xe2, 0x21, 0x7f, 0xec, 0xd9, + 0x7f, 0x9f, 0x68, 0xad, 0xb1, 0x6e, 0x5d, 0x7d, + 0x21, 0xe5, 0x69, 0xd2, 0x80, 0xed, 0x77, 0x5c, + 0xeb, 0xde, 0x3f, 0x40, 0x93, 0xc5, 0x38, 0x81, + 0x00, 0x00, 0x00, 0x00 + }, + .len = 584 + }, + .validAuthLenInBits = { + .len = 584 + }, + .digest = { + .data = {0x24, 0xa8, 0x42, 0xb3}, + .len = 4 + } +}; + +#endif /* TEST_CRYPTODEV_ZUC_TEST_VECTORS_H_ */ diff --git a/dpdk/test/test/test_cycles.c b/dpdk/test/test/test_cycles.c new file mode 100644 index 00000000..f1897979 --- /dev/null +++ b/dpdk/test/test/test_cycles.c @@ -0,0 +1,137 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <stdint.h> + +#include <rte_common.h> +#include <rte_cycles.h> + +#include "test.h" + +#define N 10000 + +/* + * Cycles test + * =========== + * + * - Loop N times and check that the timer always increments and + * never decrements during this loop. + * + * - Wait one second using rte_usleep() and check that the increment + * of cycles is correct with regard to the frequency of the timer. + */ + +static int +test_cycles(void) +{ + unsigned i; + uint64_t start_cycles, cycles, prev_cycles; + uint64_t hz = rte_get_timer_hz(); + uint64_t max_inc = (hz / 100); /* 10 ms max between 2 reads */ + + /* check that the timer is always incrementing */ + start_cycles = rte_get_timer_cycles(); + prev_cycles = start_cycles; + for (i=0; i<N; i++) { + cycles = rte_get_timer_cycles(); + if ((uint64_t)(cycles - prev_cycles) > max_inc) { + printf("increment too high or going backwards\n"); + return -1; + } + prev_cycles = cycles; + } + + /* check that waiting 1 second is precise */ + prev_cycles = rte_get_timer_cycles(); + rte_delay_us(1000000); + cycles = rte_get_timer_cycles(); + + if ((uint64_t)(cycles - prev_cycles) > (hz + max_inc)) { + printf("delay_us is not accurate: too long\n"); + return -1; + } + if ((uint64_t)(cycles - prev_cycles) < (hz - max_inc)) { + printf("delay_us is not accurate: too short\n"); + return -1; + } + + return 0; +} + +REGISTER_TEST_COMMAND(cycles_autotest, test_cycles); + +/* + * rte_delay_us_callback test + * + * - check if callback is correctly registered/unregistered + * + */ + +static unsigned int pattern; +static void my_rte_delay_us(unsigned int us) +{ + pattern += us; +} + +static int +test_user_delay_us(void) +{ + pattern = 0; + + rte_delay_us(2); + if (pattern != 0) + return -1; + + /* register custom delay function */ + rte_delay_us_callback_register(my_rte_delay_us); + + rte_delay_us(2); + if (pattern != 2) + return -1; + + rte_delay_us(3); + if (pattern != 5) + return -1; + + /* restore original delay function */ + rte_delay_us_callback_register(rte_delay_us_block); + + rte_delay_us(3); + if (pattern != 5) + return -1; + + return 0; +} + +REGISTER_TEST_COMMAND(user_delay_us, test_user_delay_us); diff --git a/dpdk/test/test/test_debug.c b/dpdk/test/test/test_debug.c new file mode 100644 index 00000000..0a3b2c46 --- /dev/null +++ b/dpdk/test/test/test_debug.c @@ -0,0 +1,149 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <stdint.h> +#include <sys/wait.h> +#include <unistd.h> + +#include <rte_debug.h> +#include <rte_common.h> +#include <rte_eal.h> + +#include "test.h" + +/* + * Debug test + * ========== + */ + +/* use fork() to test rte_panic() */ +static int +test_panic(void) +{ + int pid; + int status; + + pid = fork(); + + if (pid == 0) + rte_panic("Test Debug\n"); + else if (pid < 0){ + printf("Fork Failed\n"); + return -1; + } + wait(&status); + if(status == 0){ + printf("Child process terminated normally!\n"); + return -1; + } else + printf("Child process terminated as expected - Test passed!\n"); + + return 0; +} + +/* use fork() to test rte_exit() */ +static int +test_exit_val(int exit_val) +{ + int pid; + int status; + + pid = fork(); + + if (pid == 0) + rte_exit(exit_val, __func__); + else if (pid < 0){ + printf("Fork Failed\n"); + return -1; + } + wait(&status); + printf("Child process status: %d\n", status); +#ifndef RTE_EAL_ALWAYS_PANIC_ON_ERROR + if(!WIFEXITED(status) || WEXITSTATUS(status) != (uint8_t)exit_val){ + printf("Child process terminated with incorrect status (expected = %d)!\n", + exit_val); + return -1; + } +#endif + return 0; +} + +static int +test_exit(void) +{ + int test_vals[] = { 0, 1, 2, 255, -1 }; + unsigned i; + for (i = 0; i < sizeof(test_vals) / sizeof(test_vals[0]); i++){ + if (test_exit_val(test_vals[i]) < 0) + return -1; + } + printf("%s Passed\n", __func__); + return 0; +} + +static void +dummy_app_usage(const char *progname) +{ + RTE_SET_USED(progname); +} + +static int +test_usage(void) +{ + if (rte_set_application_usage_hook(dummy_app_usage) != NULL) { + printf("Non-NULL value returned for initial usage hook\n"); + return -1; + } + if (rte_set_application_usage_hook(NULL) != dummy_app_usage) { + printf("Incorrect value returned for application usage hook\n"); + return -1; + } + return 0; +} + +static int +test_debug(void) +{ + rte_dump_stack(); + rte_dump_registers(); + if (test_panic() < 0) + return -1; + if (test_exit() < 0) + return -1; + if (test_usage() < 0) + return -1; + return 0; +} + +REGISTER_TEST_COMMAND(debug_autotest, test_debug); diff --git a/dpdk/test/test/test_devargs.c b/dpdk/test/test/test_devargs.c new file mode 100644 index 00000000..18f54edc --- /dev/null +++ b/dpdk/test/test/test_devargs.c @@ -0,0 +1,131 @@ +/*- + * BSD LICENSE + * + * Copyright 2014 6WIND S.A. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of 6WIND S.A nor the names of its contributors + * may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/queue.h> + +#include <rte_debug.h> +#include <rte_devargs.h> + +#include "test.h" + +/* clear devargs list that was modified by the test */ +static void free_devargs_list(void) +{ + struct rte_devargs *devargs; + + while (!TAILQ_EMPTY(&devargs_list)) { + devargs = TAILQ_FIRST(&devargs_list); + TAILQ_REMOVE(&devargs_list, devargs, next); + free(devargs->args); + free(devargs); + } +} + +static int +test_devargs(void) +{ + struct rte_devargs_list save_devargs_list; + struct rte_devargs *devargs; + + /* save the real devargs_list, it is restored at the end of the test */ + save_devargs_list = devargs_list; + TAILQ_INIT(&devargs_list); + + /* test valid cases */ + if (rte_eal_devargs_add(RTE_DEVTYPE_WHITELISTED_PCI, "08:00.1") < 0) + goto fail; + if (rte_eal_devargs_add(RTE_DEVTYPE_WHITELISTED_PCI, "0000:5:00.0") < 0) + goto fail; + if (rte_eal_devargs_add(RTE_DEVTYPE_BLACKLISTED_PCI, "04:00.0,arg=val") < 0) + goto fail; + if (rte_eal_devargs_add(RTE_DEVTYPE_BLACKLISTED_PCI, "0000:01:00.1") < 0) + goto fail; + if (rte_eal_devargs_type_count(RTE_DEVTYPE_WHITELISTED_PCI) != 2) + goto fail; + if (rte_eal_devargs_type_count(RTE_DEVTYPE_BLACKLISTED_PCI) != 2) + goto fail; + if (rte_eal_devargs_type_count(RTE_DEVTYPE_VIRTUAL) != 0) + goto fail; + if (rte_eal_devargs_add(RTE_DEVTYPE_VIRTUAL, "net_ring0") < 0) + goto fail; + if (rte_eal_devargs_add(RTE_DEVTYPE_VIRTUAL, "net_ring1,key=val,k2=val2") < 0) + goto fail; + if (rte_eal_devargs_type_count(RTE_DEVTYPE_VIRTUAL) != 2) + goto fail; + free_devargs_list(); + + /* check virtual device with argument parsing */ + if (rte_eal_devargs_add(RTE_DEVTYPE_VIRTUAL, "net_ring1,k1=val,k2=val2") < 0) + goto fail; + devargs = TAILQ_FIRST(&devargs_list); + if (strncmp(devargs->name, "net_ring1", + sizeof(devargs->name)) != 0) + goto fail; + if (!devargs->args || strcmp(devargs->args, "k1=val,k2=val2") != 0) + goto fail; + free_devargs_list(); + + /* check PCI device with empty argument parsing */ + if (rte_eal_devargs_add(RTE_DEVTYPE_WHITELISTED_PCI, "04:00.1") < 0) + goto fail; + devargs = TAILQ_FIRST(&devargs_list); + if (strcmp(devargs->name, "04:00.1") != 0) + goto fail; + if (!devargs->args || strcmp(devargs->args, "") != 0) + goto fail; + free_devargs_list(); + + /* test error case: bad PCI address */ + if (rte_eal_devargs_add(RTE_DEVTYPE_WHITELISTED_PCI, "08:1") == 0) + goto fail; + if (rte_eal_devargs_add(RTE_DEVTYPE_WHITELISTED_PCI, "00.1") == 0) + goto fail; + if (rte_eal_devargs_add(RTE_DEVTYPE_WHITELISTED_PCI, "foo") == 0) + goto fail; + if (rte_eal_devargs_add(RTE_DEVTYPE_WHITELISTED_PCI, ",") == 0) + goto fail; + if (rte_eal_devargs_add(RTE_DEVTYPE_WHITELISTED_PCI, "000f:0:0") == 0) + goto fail; + + devargs_list = save_devargs_list; + return 0; + + fail: + free_devargs_list(); + devargs_list = save_devargs_list; + return -1; +} + +REGISTER_TEST_COMMAND(devargs_autotest, test_devargs); diff --git a/dpdk/test/test/test_distributor.c b/dpdk/test/test/test_distributor.c new file mode 100644 index 00000000..9fae688b --- /dev/null +++ b/dpdk/test/test/test_distributor.c @@ -0,0 +1,726 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "test.h" + +#include <unistd.h> +#include <string.h> +#include <rte_cycles.h> +#include <rte_errno.h> +#include <rte_mempool.h> +#include <rte_mbuf.h> +#include <rte_distributor.h> + +#define ITER_POWER 20 /* log 2 of how many iterations we do when timing. */ +#define BURST 32 +#define BIG_BATCH 1024 + +struct worker_params { + char name[64]; + struct rte_distributor *dist; +}; + +struct worker_params worker_params; + +/* statics - all zero-initialized by default */ +static volatile int quit; /**< general quit variable for all threads */ +static volatile int zero_quit; /**< var for when we just want thr0 to quit*/ +static volatile unsigned worker_idx; + +struct worker_stats { + volatile unsigned handled_packets; +} __rte_cache_aligned; +struct worker_stats worker_stats[RTE_MAX_LCORE]; + +/* returns the total count of the number of packets handled by the worker + * functions given below. + */ +static inline unsigned +total_packet_count(void) +{ + unsigned i, count = 0; + for (i = 0; i < worker_idx; i++) + count += worker_stats[i].handled_packets; + return count; +} + +/* resets the packet counts for a new test */ +static inline void +clear_packet_count(void) +{ + memset(&worker_stats, 0, sizeof(worker_stats)); +} + +/* this is the basic worker function for sanity test + * it does nothing but return packets and count them. + */ +static int +handle_work(void *arg) +{ + struct rte_mbuf *buf[8] __rte_cache_aligned; + struct worker_params *wp = arg; + struct rte_distributor *db = wp->dist; + unsigned int count = 0, num = 0; + unsigned int id = __sync_fetch_and_add(&worker_idx, 1); + int i; + + for (i = 0; i < 8; i++) + buf[i] = NULL; + num = rte_distributor_get_pkt(db, id, buf, buf, num); + while (!quit) { + worker_stats[id].handled_packets += num; + count += num; + num = rte_distributor_get_pkt(db, id, + buf, buf, num); + } + worker_stats[id].handled_packets += num; + count += num; + rte_distributor_return_pkt(db, id, buf, num); + return 0; +} + +/* do basic sanity testing of the distributor. This test tests the following: + * - send 32 packets through distributor with the same tag and ensure they + * all go to the one worker + * - send 32 packets through the distributor with two different tags and + * verify that they go equally to two different workers. + * - send 32 packets with different tags through the distributors and + * just verify we get all packets back. + * - send 1024 packets through the distributor, gathering the returned packets + * as we go. Then verify that we correctly got all 1024 pointers back again, + * not necessarily in the same order (as different flows). + */ +static int +sanity_test(struct worker_params *wp, struct rte_mempool *p) +{ + struct rte_distributor *db = wp->dist; + struct rte_mbuf *bufs[BURST]; + struct rte_mbuf *returns[BURST*2]; + unsigned int i, count; + unsigned int retries; + + printf("=== Basic distributor sanity tests ===\n"); + clear_packet_count(); + if (rte_mempool_get_bulk(p, (void *)bufs, BURST) != 0) { + printf("line %d: Error getting mbufs from pool\n", __LINE__); + return -1; + } + + /* now set all hash values in all buffers to zero, so all pkts go to the + * one worker thread */ + for (i = 0; i < BURST; i++) + bufs[i]->hash.usr = 0; + + rte_distributor_process(db, bufs, BURST); + count = 0; + do { + + rte_distributor_flush(db); + count += rte_distributor_returned_pkts(db, + returns, BURST*2); + } while (count < BURST); + + if (total_packet_count() != BURST) { + printf("Line %d: Error, not all packets flushed. " + "Expected %u, got %u\n", + __LINE__, BURST, total_packet_count()); + return -1; + } + + for (i = 0; i < rte_lcore_count() - 1; i++) + printf("Worker %u handled %u packets\n", i, + worker_stats[i].handled_packets); + printf("Sanity test with all zero hashes done.\n"); + + /* pick two flows and check they go correctly */ + if (rte_lcore_count() >= 3) { + clear_packet_count(); + for (i = 0; i < BURST; i++) + bufs[i]->hash.usr = (i & 1) << 8; + + rte_distributor_process(db, bufs, BURST); + count = 0; + do { + rte_distributor_flush(db); + count += rte_distributor_returned_pkts(db, + returns, BURST*2); + } while (count < BURST); + if (total_packet_count() != BURST) { + printf("Line %d: Error, not all packets flushed. " + "Expected %u, got %u\n", + __LINE__, BURST, total_packet_count()); + return -1; + } + + for (i = 0; i < rte_lcore_count() - 1; i++) + printf("Worker %u handled %u packets\n", i, + worker_stats[i].handled_packets); + printf("Sanity test with two hash values done\n"); + } + + /* give a different hash value to each packet, + * so load gets distributed */ + clear_packet_count(); + for (i = 0; i < BURST; i++) + bufs[i]->hash.usr = i+1; + + rte_distributor_process(db, bufs, BURST); + count = 0; + do { + rte_distributor_flush(db); + count += rte_distributor_returned_pkts(db, + returns, BURST*2); + } while (count < BURST); + if (total_packet_count() != BURST) { + printf("Line %d: Error, not all packets flushed. " + "Expected %u, got %u\n", + __LINE__, BURST, total_packet_count()); + return -1; + } + + for (i = 0; i < rte_lcore_count() - 1; i++) + printf("Worker %u handled %u packets\n", i, + worker_stats[i].handled_packets); + printf("Sanity test with non-zero hashes done\n"); + + rte_mempool_put_bulk(p, (void *)bufs, BURST); + + /* sanity test with BIG_BATCH packets to ensure they all arrived back + * from the returned packets function */ + clear_packet_count(); + struct rte_mbuf *many_bufs[BIG_BATCH], *return_bufs[BIG_BATCH]; + unsigned num_returned = 0; + + /* flush out any remaining packets */ + rte_distributor_flush(db); + rte_distributor_clear_returns(db); + + if (rte_mempool_get_bulk(p, (void *)many_bufs, BIG_BATCH) != 0) { + printf("line %d: Error getting mbufs from pool\n", __LINE__); + return -1; + } + for (i = 0; i < BIG_BATCH; i++) + many_bufs[i]->hash.usr = i << 2; + + printf("=== testing big burst (%s) ===\n", wp->name); + for (i = 0; i < BIG_BATCH/BURST; i++) { + rte_distributor_process(db, + &many_bufs[i*BURST], BURST); + count = rte_distributor_returned_pkts(db, + &return_bufs[num_returned], + BIG_BATCH - num_returned); + num_returned += count; + } + rte_distributor_flush(db); + count = rte_distributor_returned_pkts(db, + &return_bufs[num_returned], + BIG_BATCH - num_returned); + num_returned += count; + retries = 0; + do { + rte_distributor_flush(db); + count = rte_distributor_returned_pkts(db, + &return_bufs[num_returned], + BIG_BATCH - num_returned); + num_returned += count; + retries++; + } while ((num_returned < BIG_BATCH) && (retries < 100)); + + if (num_returned != BIG_BATCH) { + printf("line %d: Missing packets, expected %d\n", + __LINE__, num_returned); + return -1; + } + + /* big check - make sure all packets made it back!! */ + for (i = 0; i < BIG_BATCH; i++) { + unsigned j; + struct rte_mbuf *src = many_bufs[i]; + for (j = 0; j < BIG_BATCH; j++) { + if (return_bufs[j] == src) + break; + } + + if (j == BIG_BATCH) { + printf("Error: could not find source packet #%u\n", i); + return -1; + } + } + printf("Sanity test of returned packets done\n"); + + rte_mempool_put_bulk(p, (void *)many_bufs, BIG_BATCH); + + printf("\n"); + return 0; +} + + +/* to test that the distributor does not lose packets, we use this worker + * function which frees mbufs when it gets them. The distributor thread does + * the mbuf allocation. If distributor drops packets we'll eventually run out + * of mbufs. + */ +static int +handle_work_with_free_mbufs(void *arg) +{ + struct rte_mbuf *buf[8] __rte_cache_aligned; + struct worker_params *wp = arg; + struct rte_distributor *d = wp->dist; + unsigned int count = 0; + unsigned int i; + unsigned int num = 0; + unsigned int id = __sync_fetch_and_add(&worker_idx, 1); + + for (i = 0; i < 8; i++) + buf[i] = NULL; + num = rte_distributor_get_pkt(d, id, buf, buf, num); + while (!quit) { + worker_stats[id].handled_packets += num; + count += num; + for (i = 0; i < num; i++) + rte_pktmbuf_free(buf[i]); + num = rte_distributor_get_pkt(d, + id, buf, buf, num); + } + worker_stats[id].handled_packets += num; + count += num; + rte_distributor_return_pkt(d, id, buf, num); + return 0; +} + +/* Perform a sanity test of the distributor with a large number of packets, + * where we allocate a new set of mbufs for each burst. The workers then + * free the mbufs. This ensures that we don't have any packet leaks in the + * library. + */ +static int +sanity_test_with_mbuf_alloc(struct worker_params *wp, struct rte_mempool *p) +{ + struct rte_distributor *d = wp->dist; + unsigned i; + struct rte_mbuf *bufs[BURST]; + + printf("=== Sanity test with mbuf alloc/free (%s) ===\n", wp->name); + + clear_packet_count(); + for (i = 0; i < ((1<<ITER_POWER)); i += BURST) { + unsigned j; + while (rte_mempool_get_bulk(p, (void *)bufs, BURST) < 0) + rte_distributor_process(d, NULL, 0); + for (j = 0; j < BURST; j++) { + bufs[j]->hash.usr = (i+j) << 1; + rte_mbuf_refcnt_set(bufs[j], 1); + } + + rte_distributor_process(d, bufs, BURST); + } + + rte_distributor_flush(d); + + rte_delay_us(10000); + + if (total_packet_count() < (1<<ITER_POWER)) { + printf("Line %u: Packet count is incorrect, %u, expected %u\n", + __LINE__, total_packet_count(), + (1<<ITER_POWER)); + return -1; + } + + printf("Sanity test with mbuf alloc/free passed\n\n"); + return 0; +} + +static int +handle_work_for_shutdown_test(void *arg) +{ + struct rte_mbuf *pkt = NULL; + struct rte_mbuf *buf[8] __rte_cache_aligned; + struct worker_params *wp = arg; + struct rte_distributor *d = wp->dist; + unsigned int count = 0; + unsigned int num = 0; + unsigned int total = 0; + unsigned int i; + unsigned int returned = 0; + const unsigned int id = __sync_fetch_and_add(&worker_idx, 1); + + num = rte_distributor_get_pkt(d, id, buf, buf, num); + + /* wait for quit single globally, or for worker zero, wait + * for zero_quit */ + while (!quit && !(id == 0 && zero_quit)) { + worker_stats[id].handled_packets += num; + count += num; + for (i = 0; i < num; i++) + rte_pktmbuf_free(buf[i]); + num = rte_distributor_get_pkt(d, + id, buf, buf, num); + total += num; + } + worker_stats[id].handled_packets += num; + count += num; + returned = rte_distributor_return_pkt(d, id, buf, num); + + if (id == 0) { + /* for worker zero, allow it to restart to pick up last packet + * when all workers are shutting down. + */ + while (zero_quit) + usleep(100); + + num = rte_distributor_get_pkt(d, + id, buf, buf, num); + + while (!quit) { + worker_stats[id].handled_packets++, count++; + rte_pktmbuf_free(pkt); + num = rte_distributor_get_pkt(d, id, buf, buf, num); + } + returned = rte_distributor_return_pkt(d, + id, buf, num); + printf("Num returned = %d\n", returned); + } + return 0; +} + + +/* Perform a sanity test of the distributor with a large number of packets, + * where we allocate a new set of mbufs for each burst. The workers then + * free the mbufs. This ensures that we don't have any packet leaks in the + * library. + */ +static int +sanity_test_with_worker_shutdown(struct worker_params *wp, + struct rte_mempool *p) +{ + struct rte_distributor *d = wp->dist; + struct rte_mbuf *bufs[BURST]; + unsigned i; + + printf("=== Sanity test of worker shutdown ===\n"); + + clear_packet_count(); + + if (rte_mempool_get_bulk(p, (void *)bufs, BURST) != 0) { + printf("line %d: Error getting mbufs from pool\n", __LINE__); + return -1; + } + + /* + * Now set all hash values in all buffers to same value so all + * pkts go to the one worker thread + */ + for (i = 0; i < BURST; i++) + bufs[i]->hash.usr = 1; + + rte_distributor_process(d, bufs, BURST); + rte_distributor_flush(d); + + /* at this point, we will have processed some packets and have a full + * backlog for the other ones at worker 0. + */ + + /* get more buffers to queue up, again setting them to the same flow */ + if (rte_mempool_get_bulk(p, (void *)bufs, BURST) != 0) { + printf("line %d: Error getting mbufs from pool\n", __LINE__); + return -1; + } + for (i = 0; i < BURST; i++) + bufs[i]->hash.usr = 1; + + /* get worker zero to quit */ + zero_quit = 1; + rte_distributor_process(d, bufs, BURST); + + /* flush the distributor */ + rte_distributor_flush(d); + rte_delay_us(10000); + + for (i = 0; i < rte_lcore_count() - 1; i++) + printf("Worker %u handled %u packets\n", i, + worker_stats[i].handled_packets); + + if (total_packet_count() != BURST * 2) { + printf("Line %d: Error, not all packets flushed. " + "Expected %u, got %u\n", + __LINE__, BURST * 2, total_packet_count()); + return -1; + } + + printf("Sanity test with worker shutdown passed\n\n"); + return 0; +} + +/* Test that the flush function is able to move packets between workers when + * one worker shuts down.. + */ +static int +test_flush_with_worker_shutdown(struct worker_params *wp, + struct rte_mempool *p) +{ + struct rte_distributor *d = wp->dist; + struct rte_mbuf *bufs[BURST]; + unsigned i; + + printf("=== Test flush fn with worker shutdown (%s) ===\n", wp->name); + + clear_packet_count(); + if (rte_mempool_get_bulk(p, (void *)bufs, BURST) != 0) { + printf("line %d: Error getting mbufs from pool\n", __LINE__); + return -1; + } + + /* now set all hash values in all buffers to zero, so all pkts go to the + * one worker thread */ + for (i = 0; i < BURST; i++) + bufs[i]->hash.usr = 0; + + rte_distributor_process(d, bufs, BURST); + /* at this point, we will have processed some packets and have a full + * backlog for the other ones at worker 0. + */ + + /* get worker zero to quit */ + zero_quit = 1; + + /* flush the distributor */ + rte_distributor_flush(d); + + rte_delay_us(10000); + + zero_quit = 0; + for (i = 0; i < rte_lcore_count() - 1; i++) + printf("Worker %u handled %u packets\n", i, + worker_stats[i].handled_packets); + + if (total_packet_count() != BURST) { + printf("Line %d: Error, not all packets flushed. " + "Expected %u, got %u\n", + __LINE__, BURST, total_packet_count()); + return -1; + } + + printf("Flush test with worker shutdown passed\n\n"); + return 0; +} + +static +int test_error_distributor_create_name(void) +{ + struct rte_distributor *d = NULL; + struct rte_distributor *db = NULL; + char *name = NULL; + + d = rte_distributor_create(name, rte_socket_id(), + rte_lcore_count() - 1, + RTE_DIST_ALG_SINGLE); + if (d != NULL || rte_errno != EINVAL) { + printf("ERROR: No error on create() with NULL name param\n"); + return -1; + } + + db = rte_distributor_create(name, rte_socket_id(), + rte_lcore_count() - 1, + RTE_DIST_ALG_BURST); + if (db != NULL || rte_errno != EINVAL) { + printf("ERROR: No error on create() with NULL param\n"); + return -1; + } + + return 0; +} + + +static +int test_error_distributor_create_numworkers(void) +{ + struct rte_distributor *ds = NULL; + struct rte_distributor *db = NULL; + + ds = rte_distributor_create("test_numworkers", rte_socket_id(), + RTE_MAX_LCORE + 10, + RTE_DIST_ALG_SINGLE); + if (ds != NULL || rte_errno != EINVAL) { + printf("ERROR: No error on create() with num_workers > MAX\n"); + return -1; + } + + db = rte_distributor_create("test_numworkers", rte_socket_id(), + RTE_MAX_LCORE + 10, + RTE_DIST_ALG_BURST); + if (db != NULL || rte_errno != EINVAL) { + printf("ERROR: No error on create() num_workers > MAX\n"); + return -1; + } + + return 0; +} + + +/* Useful function which ensures that all worker functions terminate */ +static void +quit_workers(struct worker_params *wp, struct rte_mempool *p) +{ + struct rte_distributor *d = wp->dist; + const unsigned num_workers = rte_lcore_count() - 1; + unsigned i; + struct rte_mbuf *bufs[RTE_MAX_LCORE]; + rte_mempool_get_bulk(p, (void *)bufs, num_workers); + + zero_quit = 0; + quit = 1; + for (i = 0; i < num_workers; i++) + bufs[i]->hash.usr = i << 1; + rte_distributor_process(d, bufs, num_workers); + + rte_mempool_put_bulk(p, (void *)bufs, num_workers); + + rte_distributor_process(d, NULL, 0); + rte_distributor_flush(d); + rte_eal_mp_wait_lcore(); + quit = 0; + worker_idx = 0; +} + +static int +test_distributor(void) +{ + static struct rte_distributor *ds; + static struct rte_distributor *db; + static struct rte_distributor *dist[2]; + static struct rte_mempool *p; + int i; + + if (rte_lcore_count() < 2) { + printf("ERROR: not enough cores to test distributor\n"); + return -1; + } + + if (db == NULL) { + db = rte_distributor_create("Test_dist_burst", rte_socket_id(), + rte_lcore_count() - 1, + RTE_DIST_ALG_BURST); + if (db == NULL) { + printf("Error creating burst distributor\n"); + return -1; + } + } else { + rte_distributor_flush(db); + rte_distributor_clear_returns(db); + } + + if (ds == NULL) { + ds = rte_distributor_create("Test_dist_single", + rte_socket_id(), + rte_lcore_count() - 1, + RTE_DIST_ALG_SINGLE); + if (ds == NULL) { + printf("Error creating single distributor\n"); + return -1; + } + } else { + rte_distributor_flush(ds); + rte_distributor_clear_returns(ds); + } + + const unsigned nb_bufs = (511 * rte_lcore_count()) < BIG_BATCH ? + (BIG_BATCH * 2) - 1 : (511 * rte_lcore_count()); + if (p == NULL) { + p = rte_pktmbuf_pool_create("DT_MBUF_POOL", nb_bufs, BURST, + 0, RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id()); + if (p == NULL) { + printf("Error creating mempool\n"); + return -1; + } + } + + dist[0] = ds; + dist[1] = db; + + for (i = 0; i < 2; i++) { + + worker_params.dist = dist[i]; + if (i) + sprintf(worker_params.name, "burst"); + else + sprintf(worker_params.name, "single"); + + rte_eal_mp_remote_launch(handle_work, + &worker_params, SKIP_MASTER); + if (sanity_test(&worker_params, p) < 0) + goto err; + quit_workers(&worker_params, p); + + rte_eal_mp_remote_launch(handle_work_with_free_mbufs, + &worker_params, SKIP_MASTER); + if (sanity_test_with_mbuf_alloc(&worker_params, p) < 0) + goto err; + quit_workers(&worker_params, p); + + if (rte_lcore_count() > 2) { + rte_eal_mp_remote_launch(handle_work_for_shutdown_test, + &worker_params, + SKIP_MASTER); + if (sanity_test_with_worker_shutdown(&worker_params, + p) < 0) + goto err; + quit_workers(&worker_params, p); + + rte_eal_mp_remote_launch(handle_work_for_shutdown_test, + &worker_params, + SKIP_MASTER); + if (test_flush_with_worker_shutdown(&worker_params, + p) < 0) + goto err; + quit_workers(&worker_params, p); + + } else { + printf("Too few cores to run worker shutdown test\n"); + } + + } + + if (test_error_distributor_create_numworkers() == -1 || + test_error_distributor_create_name() == -1) { + printf("rte_distributor_create parameter check tests failed"); + return -1; + } + + return 0; + +err: + quit_workers(&worker_params, p); + return -1; +} + +REGISTER_TEST_COMMAND(distributor_autotest, test_distributor); diff --git a/dpdk/test/test/test_distributor_perf.c b/dpdk/test/test/test_distributor_perf.c new file mode 100644 index 00000000..7d69887b --- /dev/null +++ b/dpdk/test/test/test_distributor_perf.c @@ -0,0 +1,296 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "test.h" + +#include <unistd.h> +#include <string.h> +#include <rte_mempool.h> +#include <rte_cycles.h> +#include <rte_common.h> +#include <rte_mbuf.h> +#include <rte_distributor.h> +#include <rte_pause.h> + +#define ITER_POWER_CL 25 /* log 2 of how many iterations for Cache Line test */ +#define ITER_POWER 21 /* log 2 of how many iterations we do when timing. */ +#define BURST 64 +#define BIG_BATCH 1024 + +/* static vars - zero initialized by default */ +static volatile int quit; +static volatile unsigned worker_idx; + +struct worker_stats { + volatile unsigned handled_packets; +} __rte_cache_aligned; +struct worker_stats worker_stats[RTE_MAX_LCORE]; + +/* + * worker thread used for testing the time to do a round-trip of a cache + * line between two cores and back again + */ +static void +flip_bit(volatile uint64_t *arg) +{ + uint64_t old_val = 0; + while (old_val != 2) { + while (!*arg) + rte_pause(); + old_val = *arg; + *arg = 0; + } +} + +/* + * test case to time the number of cycles to round-trip a cache line between + * two cores and back again. + */ +static void +time_cache_line_switch(void) +{ + /* allocate a full cache line for data, we use only first byte of it */ + uint64_t data[RTE_CACHE_LINE_SIZE*3 / sizeof(uint64_t)]; + + unsigned i, slaveid = rte_get_next_lcore(rte_lcore_id(), 0, 0); + volatile uint64_t *pdata = &data[0]; + *pdata = 1; + rte_eal_remote_launch((lcore_function_t *)flip_bit, &data[0], slaveid); + while (*pdata) + rte_pause(); + + const uint64_t start_time = rte_rdtsc(); + for (i = 0; i < (1 << ITER_POWER_CL); i++) { + while (*pdata) + rte_pause(); + *pdata = 1; + } + const uint64_t end_time = rte_rdtsc(); + + while (*pdata) + rte_pause(); + *pdata = 2; + rte_eal_wait_lcore(slaveid); + printf("==== Cache line switch test ===\n"); + printf("Time for %u iterations = %"PRIu64" ticks\n", (1<<ITER_POWER_CL), + end_time-start_time); + printf("Ticks per iteration = %"PRIu64"\n\n", + (end_time-start_time) >> ITER_POWER_CL); +} + +/* + * returns the total count of the number of packets handled by the worker + * functions given below. + */ +static unsigned +total_packet_count(void) +{ + unsigned i, count = 0; + for (i = 0; i < worker_idx; i++) + count += worker_stats[i].handled_packets; + return count; +} + +/* resets the packet counts for a new test */ +static void +clear_packet_count(void) +{ + memset(&worker_stats, 0, sizeof(worker_stats)); +} + +/* + * This is the basic worker function for performance tests. + * it does nothing but return packets and count them. + */ +static int +handle_work(void *arg) +{ + struct rte_distributor *d = arg; + unsigned int count = 0; + unsigned int num = 0; + int i; + unsigned int id = __sync_fetch_and_add(&worker_idx, 1); + struct rte_mbuf *buf[8] __rte_cache_aligned; + + for (i = 0; i < 8; i++) + buf[i] = NULL; + + num = rte_distributor_get_pkt(d, id, buf, buf, num); + while (!quit) { + worker_stats[id].handled_packets += num; + count += num; + num = rte_distributor_get_pkt(d, id, buf, buf, num); + } + worker_stats[id].handled_packets += num; + count += num; + rte_distributor_return_pkt(d, id, buf, num); + return 0; +} + +/* + * This basic performance test just repeatedly sends in 32 packets at a time + * to the distributor and verifies at the end that we got them all in the worker + * threads and finally how long per packet the processing took. + */ +static inline int +perf_test(struct rte_distributor *d, struct rte_mempool *p) +{ + unsigned int i; + uint64_t start, end; + struct rte_mbuf *bufs[BURST]; + + clear_packet_count(); + if (rte_mempool_get_bulk(p, (void *)bufs, BURST) != 0) { + printf("Error getting mbufs from pool\n"); + return -1; + } + /* ensure we have different hash value for each pkt */ + for (i = 0; i < BURST; i++) + bufs[i]->hash.usr = i; + + start = rte_rdtsc(); + for (i = 0; i < (1<<ITER_POWER); i++) + rte_distributor_process(d, bufs, BURST); + end = rte_rdtsc(); + + do { + usleep(100); + rte_distributor_process(d, NULL, 0); + } while (total_packet_count() < (BURST << ITER_POWER)); + + rte_distributor_clear_returns(d); + + printf("Time per burst: %"PRIu64"\n", (end - start) >> ITER_POWER); + printf("Time per packet: %"PRIu64"\n\n", + ((end - start) >> ITER_POWER)/BURST); + rte_mempool_put_bulk(p, (void *)bufs, BURST); + + for (i = 0; i < rte_lcore_count() - 1; i++) + printf("Worker %u handled %u packets\n", i, + worker_stats[i].handled_packets); + printf("Total packets: %u (%x)\n", total_packet_count(), + total_packet_count()); + printf("=== Perf test done ===\n\n"); + + return 0; +} + +/* Useful function which ensures that all worker functions terminate */ +static void +quit_workers(struct rte_distributor *d, struct rte_mempool *p) +{ + const unsigned int num_workers = rte_lcore_count() - 1; + unsigned int i; + struct rte_mbuf *bufs[RTE_MAX_LCORE]; + + rte_mempool_get_bulk(p, (void *)bufs, num_workers); + + quit = 1; + for (i = 0; i < num_workers; i++) + bufs[i]->hash.usr = i << 1; + rte_distributor_process(d, bufs, num_workers); + + rte_mempool_put_bulk(p, (void *)bufs, num_workers); + + rte_distributor_process(d, NULL, 0); + rte_eal_mp_wait_lcore(); + quit = 0; + worker_idx = 0; +} + +static int +test_distributor_perf(void) +{ + static struct rte_distributor *ds; + static struct rte_distributor *db; + static struct rte_mempool *p; + + if (rte_lcore_count() < 2) { + printf("ERROR: not enough cores to test distributor\n"); + return -1; + } + + /* first time how long it takes to round-trip a cache line */ + time_cache_line_switch(); + + if (ds == NULL) { + ds = rte_distributor_create("Test_perf", rte_socket_id(), + rte_lcore_count() - 1, + RTE_DIST_ALG_SINGLE); + if (ds == NULL) { + printf("Error creating distributor\n"); + return -1; + } + } else { + rte_distributor_clear_returns(ds); + } + + if (db == NULL) { + db = rte_distributor_create("Test_burst", rte_socket_id(), + rte_lcore_count() - 1, + RTE_DIST_ALG_BURST); + if (db == NULL) { + printf("Error creating burst distributor\n"); + return -1; + } + } else { + rte_distributor_clear_returns(db); + } + + const unsigned nb_bufs = (511 * rte_lcore_count()) < BIG_BATCH ? + (BIG_BATCH * 2) - 1 : (511 * rte_lcore_count()); + if (p == NULL) { + p = rte_pktmbuf_pool_create("DPT_MBUF_POOL", nb_bufs, BURST, + 0, RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id()); + if (p == NULL) { + printf("Error creating mempool\n"); + return -1; + } + } + + printf("=== Performance test of distributor (single mode) ===\n"); + rte_eal_mp_remote_launch(handle_work, ds, SKIP_MASTER); + if (perf_test(ds, p) < 0) + return -1; + quit_workers(ds, p); + + printf("=== Performance test of distributor (burst mode) ===\n"); + rte_eal_mp_remote_launch(handle_work, db, SKIP_MASTER); + if (perf_test(db, p) < 0) + return -1; + quit_workers(db, p); + + return 0; +} + +REGISTER_TEST_COMMAND(distributor_perf_autotest, test_distributor_perf); diff --git a/dpdk/test/test/test_eal_flags.c b/dpdk/test/test/test_eal_flags.c new file mode 100644 index 00000000..310109e6 --- /dev/null +++ b/dpdk/test/test/test_eal_flags.c @@ -0,0 +1,1362 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * Copyright(c) 2014 6WIND S.A. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include <stdio.h> + +#include "test.h" + +#include <string.h> +#include <stdarg.h> +#include <libgen.h> +#include <stdlib.h> +#include <errno.h> +#include <unistd.h> +#include <dirent.h> +#include <sys/wait.h> +#include <sys/file.h> +#include <limits.h> + +#include <rte_debug.h> +#include <rte_string_fns.h> + +#include "process.h" + +#define DEFAULT_MEM_SIZE "18" +#define mp_flag "--proc-type=secondary" +#define no_hpet "--no-hpet" +#define no_huge "--no-huge" +#define no_shconf "--no-shconf" +#define pci_whitelist "--pci-whitelist" +#define vdev "--vdev" +#define memtest "memtest" +#define memtest1 "memtest1" +#define memtest2 "memtest2" +#define SOCKET_MEM_STRLEN (RTE_MAX_NUMA_NODES * 10) +#define launch_proc(ARGV) process_dup(ARGV, \ + sizeof(ARGV)/(sizeof(ARGV[0])), __func__) + +enum hugepage_action { + HUGEPAGE_CHECK_EXISTS = 0, + HUGEPAGE_CHECK_LOCKED, + HUGEPAGE_DELETE, + HUGEPAGE_INVALID +}; + +/* if string contains a hugepage path */ +static int +get_hugepage_path(char * src, int src_len, char * dst, int dst_len) +{ +#define NUM_TOKENS 4 + char *tokens[NUM_TOKENS]; + + /* if we couldn't properly split the string */ + if (rte_strsplit(src, src_len, tokens, NUM_TOKENS, ' ') < NUM_TOKENS) + return 0; + + if (strncmp(tokens[2], "hugetlbfs", sizeof("hugetlbfs")) == 0) { + snprintf(dst, dst_len, "%s", tokens[1]); + return 1; + } + return 0; +} + +/* + * Cycles through hugepage directories and looks for hugepage + * files associated with a given prefix. Depending on value of + * action, the hugepages are checked if they exist, checked if + * they can be locked, or are simply deleted. + * + * Returns 1 if it finds at least one hugepage matching the action + * Returns 0 if no matching hugepages were found + * Returns -1 if it encounters an error + */ +static int +process_hugefiles(const char * prefix, enum hugepage_action action) +{ + FILE * hugedir_handle = NULL; + DIR * hugepage_dir = NULL; + struct dirent *dirent = NULL; + + char hugefile_prefix[PATH_MAX] = {0}; + char hugedir[PATH_MAX] = {0}; + char line[PATH_MAX] = {0}; + + int fd, lck_result, result = 0; + + const int prefix_len = snprintf(hugefile_prefix, + sizeof(hugefile_prefix), "%smap_", prefix); + if (prefix_len <= 0 || prefix_len >= (int)sizeof(hugefile_prefix) + || prefix_len >= (int)sizeof(dirent->d_name)) { + printf("Error creating hugefile filename prefix\n"); + return -1; + } + + /* get hugetlbfs mountpoints from /proc/mounts */ + hugedir_handle = fopen("/proc/mounts", "r"); + + if (hugedir_handle == NULL) { + printf("Error parsing /proc/mounts!\n"); + return -1; + } + + /* read and parse script output */ + while (fgets(line, sizeof(line), hugedir_handle) != NULL) { + + /* check if we have a hugepage filesystem path */ + if (!get_hugepage_path(line, sizeof(line), hugedir, sizeof(hugedir))) + continue; + + /* check if directory exists */ + if ((hugepage_dir = opendir(hugedir)) == NULL) { + fclose(hugedir_handle); + printf("Error reading %s: %s\n", hugedir, strerror(errno)); + return -1; + } + + while ((dirent = readdir(hugepage_dir)) != NULL) { + if (memcmp(dirent->d_name, hugefile_prefix, prefix_len) != 0) + continue; + + switch (action) { + case HUGEPAGE_CHECK_EXISTS: + { + /* file exists, return */ + result = 1; + goto end; + } + break; + case HUGEPAGE_DELETE: + { + char file_path[PATH_MAX] = {0}; + + snprintf(file_path, sizeof(file_path), + "%s/%s", hugedir, dirent->d_name); + + /* remove file */ + if (remove(file_path) < 0) { + printf("Error deleting %s - %s!\n", + dirent->d_name, strerror(errno)); + closedir(hugepage_dir); + result = -1; + goto end; + } + result = 1; + } + break; + case HUGEPAGE_CHECK_LOCKED: + { + /* try and lock the file */ + fd = openat(dirfd(hugepage_dir), dirent->d_name, O_RDONLY); + + /* this shouldn't happen */ + if (fd == -1) { + printf("Error opening %s - %s!\n", + dirent->d_name, strerror(errno)); + closedir(hugepage_dir); + result = -1; + goto end; + } + + /* non-blocking lock */ + lck_result = flock(fd, LOCK_EX | LOCK_NB); + + /* if lock succeeds, there's something wrong */ + if (lck_result != -1) { + result = 0; + + /* unlock the resulting lock */ + flock(fd, LOCK_UN); + close(fd); + closedir(hugepage_dir); + goto end; + } + result = 1; + close(fd); + } + break; + /* shouldn't happen */ + default: + goto end; + } /* switch */ + + } /* read hugepage directory */ + closedir(hugepage_dir); + } /* read /proc/mounts */ +end: + fclose(hugedir_handle); + return result; +} + +#ifdef RTE_EXEC_ENV_LINUXAPP +/* + * count the number of "node*" files in /sys/devices/system/node/ + */ +static int +get_number_of_sockets(void) +{ + struct dirent *dirent = NULL; + const char * nodedir = "/sys/devices/system/node/"; + DIR * dir = NULL; + int result = 0; + + /* check if directory exists */ + if ((dir = opendir(nodedir)) == NULL) { + /* if errno==ENOENT this means we don't have NUMA support */ + if (errno == ENOENT) { + printf("No NUMA nodes detected: assuming 1 available socket\n"); + return 1; + } + printf("Error opening %s: %s\n", nodedir, strerror(errno)); + return -1; + } + + while ((dirent = readdir(dir)) != NULL) + if (strncmp(dirent->d_name, "node", sizeof("node") - 1) == 0) + result++; + + closedir(dir); + return result; +} +#endif + +static char* +get_current_prefix(char * prefix, int size) +{ + char path[PATH_MAX] = {0}; + char buf[PATH_MAX] = {0}; + + /* get file for config (fd is always 3) */ + snprintf(path, sizeof(path), "/proc/self/fd/%d", 3); + + /* return NULL on error */ + if (readlink(path, buf, sizeof(buf)) == -1) + return NULL; + + /* get the basename */ + snprintf(buf, sizeof(buf), "%s", basename(buf)); + + /* copy string all the way from second char up to start of _config */ + snprintf(prefix, size, "%.*s", + (int)(strnlen(buf, sizeof(buf)) - sizeof("_config")), + &buf[1]); + + return prefix; +} + +/* + * Test that the app doesn't run with invalid whitelist option. + * Final tests ensures it does run with valid options as sanity check (one + * test for with Domain+BDF, second for just with BDF) + */ +static int +test_whitelist_flag(void) +{ + unsigned i; +#ifdef RTE_EXEC_ENV_BSDAPP + /* BSD target doesn't support prefixes at this point */ + const char * prefix = ""; +#else + char prefix[PATH_MAX], tmp[PATH_MAX]; + if (get_current_prefix(tmp, sizeof(tmp)) == NULL) { + printf("Error - unable to get current prefix!\n"); + return -1; + } + snprintf(prefix, sizeof(prefix), "--file-prefix=%s", tmp); +#endif + + const char *wlinval[][11] = { + {prgname, prefix, mp_flag, "-n", "1", "-c", "1", + pci_whitelist, "error", "", ""}, + {prgname, prefix, mp_flag, "-n", "1", "-c", "1", + pci_whitelist, "0:0:0", "", ""}, + {prgname, prefix, mp_flag, "-n", "1", "-c", "1", + pci_whitelist, "0:error:0.1", "", ""}, + {prgname, prefix, mp_flag, "-n", "1", "-c", "1", + pci_whitelist, "0:0:0.1error", "", ""}, + {prgname, prefix, mp_flag, "-n", "1", "-c", "1", + pci_whitelist, "error0:0:0.1", "", ""}, + {prgname, prefix, mp_flag, "-n", "1", "-c", "1", + pci_whitelist, "0:0:0.1.2", "", ""}, + }; + /* Test with valid whitelist option */ + const char *wlval1[] = {prgname, prefix, mp_flag, "-n", "1", "-c", "1", + pci_whitelist, "00FF:09:0B.3"}; + const char *wlval2[] = {prgname, prefix, mp_flag, "-n", "1", "-c", "1", + pci_whitelist, "09:0B.3", pci_whitelist, "0a:0b.1"}; + const char *wlval3[] = {prgname, prefix, mp_flag, "-n", "1", "-c", "1", + pci_whitelist, "09:0B.3,type=test", + pci_whitelist, "08:00.1,type=normal", + }; + + for (i = 0; i < sizeof(wlinval) / sizeof(wlinval[0]); i++) { + if (launch_proc(wlinval[i]) == 0) { + printf("Error - process did run ok with invalid " + "whitelist parameter\n"); + return -1; + } + } + if (launch_proc(wlval1) != 0 ) { + printf("Error - process did not run ok with valid whitelist\n"); + return -1; + } + if (launch_proc(wlval2) != 0 ) { + printf("Error - process did not run ok with valid whitelist value set\n"); + return -1; + } + if (launch_proc(wlval3) != 0 ) { + printf("Error - process did not run ok with valid whitelist + args\n"); + return -1; + } + + return 0; +} + +/* + * Test that the app doesn't run with invalid blacklist option. + * Final test ensures it does run with valid options as sanity check + */ +static int +test_invalid_b_flag(void) +{ +#ifdef RTE_EXEC_ENV_BSDAPP + /* BSD target doesn't support prefixes at this point */ + const char * prefix = ""; +#else + char prefix[PATH_MAX], tmp[PATH_MAX]; + if (get_current_prefix(tmp, sizeof(tmp)) == NULL) { + printf("Error - unable to get current prefix!\n"); + return -1; + } + snprintf(prefix, sizeof(prefix), "--file-prefix=%s", tmp); +#endif + + const char *blinval[][9] = { + {prgname, prefix, mp_flag, "-n", "1", "-c", "1", "-b", "error"}, + {prgname, prefix, mp_flag, "-n", "1", "-c", "1", "-b", "0:0:0"}, + {prgname, prefix, mp_flag, "-n", "1", "-c", "1", "-b", "0:error:0.1"}, + {prgname, prefix, mp_flag, "-n", "1", "-c", "1", "-b", "0:0:0.1error"}, + {prgname, prefix, mp_flag, "-n", "1", "-c", "1", "-b", "error0:0:0.1"}, + {prgname, prefix, mp_flag, "-n", "1", "-c", "1", "-b", "0:0:0.1.2"}, + }; + /* Test with valid blacklist option */ + const char *blval[] = {prgname, prefix, mp_flag, "-n", "1", "-c", "1", "-b", "FF:09:0B.3"}; + + int i; + + for (i = 0; i != sizeof (blinval) / sizeof (blinval[0]); i++) { + if (launch_proc(blinval[i]) == 0) { + printf("Error - process did run ok with invalid " + "blacklist parameter\n"); + return -1; + } + } + if (launch_proc(blval) != 0) { + printf("Error - process did not run ok with valid blacklist value\n"); + return -1; + } + return 0; +} + +/* + * Test that the app doesn't run with invalid vdev option. + * Final test ensures it does run with valid options as sanity check + */ +#ifdef RTE_LIBRTE_PMD_RING +static int +test_invalid_vdev_flag(void) +{ +#ifdef RTE_EXEC_ENV_BSDAPP + /* BSD target doesn't support prefixes at this point, and we also need to + * run another primary process here */ + const char * prefix = no_shconf; +#else + const char * prefix = "--file-prefix=vdev"; +#endif + + /* Test with invalid vdev option */ + const char *vdevinval[] = {prgname, prefix, "-n", "1", + "-c", "1", vdev, "eth_dummy"}; + + /* Test with valid vdev option */ + const char *vdevval1[] = {prgname, prefix, "-n", "1", + "-c", "1", vdev, "net_ring0"}; + + const char *vdevval2[] = {prgname, prefix, "-n", "1", + "-c", "1", vdev, "net_ring0,args=test"}; + + const char *vdevval3[] = {prgname, prefix, "-n", "1", + "-c", "1", vdev, "net_ring0,nodeaction=r1:0:CREATE"}; + + if (launch_proc(vdevinval) == 0) { + printf("Error - process did run ok with invalid " + "vdev parameter\n"); + return -1; + } + + if (launch_proc(vdevval1) != 0) { + printf("Error - process did not run ok with valid vdev value\n"); + return -1; + } + + if (launch_proc(vdevval2) != 0) { + printf("Error - process did not run ok with valid vdev value," + "with dummy args\n"); + return -1; + } + + if (launch_proc(vdevval3) != 0) { + printf("Error - process did not run ok with valid vdev value," + "with valid args\n"); + return -1; + } + return 0; +} +#endif + +/* + * Test that the app doesn't run with invalid -r option. + */ +static int +test_invalid_r_flag(void) +{ +#ifdef RTE_EXEC_ENV_BSDAPP + /* BSD target doesn't support prefixes at this point */ + const char * prefix = ""; +#else + char prefix[PATH_MAX], tmp[PATH_MAX]; + if (get_current_prefix(tmp, sizeof(tmp)) == NULL) { + printf("Error - unable to get current prefix!\n"); + return -1; + } + snprintf(prefix, sizeof(prefix), "--file-prefix=%s", tmp); +#endif + + const char *rinval[][9] = { + {prgname, prefix, mp_flag, "-n", "1", "-c", "1", "-r", "error"}, + {prgname, prefix, mp_flag, "-n", "1", "-c", "1", "-r", "0"}, + {prgname, prefix, mp_flag, "-n", "1", "-c", "1", "-r", "-1"}, + {prgname, prefix, mp_flag, "-n", "1", "-c", "1", "-r", "17"}, + }; + /* Test with valid blacklist option */ + const char *rval[] = {prgname, prefix, mp_flag, "-n", "1", "-c", "1", "-r", "16"}; + + int i; + + for (i = 0; i != sizeof (rinval) / sizeof (rinval[0]); i++) { + if (launch_proc(rinval[i]) == 0) { + printf("Error - process did run ok with invalid " + "-r (rank) parameter\n"); + return -1; + } + } + if (launch_proc(rval) != 0) { + printf("Error - process did not run ok with valid -r (rank) value\n"); + return -1; + } + return 0; +} + +/* + * Test that the app doesn't run without the coremask/corelist flags. In all cases + * should give an error and fail to run + */ +static int +test_missing_c_flag(void) +{ +#ifdef RTE_EXEC_ENV_BSDAPP + /* BSD target doesn't support prefixes at this point */ + const char * prefix = ""; +#else + char prefix[PATH_MAX], tmp[PATH_MAX]; + if (get_current_prefix(tmp, sizeof(tmp)) == NULL) { + printf("Error - unable to get current prefix!\n"); + return -1; + } + snprintf(prefix, sizeof(prefix), "--file-prefix=%s", tmp); +#endif + + /* -c flag but no coremask value */ + const char *argv1[] = { prgname, prefix, mp_flag, "-n", "3", "-c"}; + /* No -c, -l or --lcores flag at all */ + const char *argv2[] = { prgname, prefix, mp_flag, "-n", "3"}; + /* bad coremask value */ + const char *argv3[] = { prgname, prefix, mp_flag, + "-n", "3", "-c", "error" }; + /* sanity check of tests - valid coremask value */ + const char *argv4[] = { prgname, prefix, mp_flag, + "-n", "3", "-c", "1" }; + /* -l flag but no corelist value */ + const char *argv5[] = { prgname, prefix, mp_flag, + "-n", "3", "-l"}; + const char *argv6[] = { prgname, prefix, mp_flag, + "-n", "3", "-l", " " }; + /* bad corelist values */ + const char *argv7[] = { prgname, prefix, mp_flag, + "-n", "3", "-l", "error" }; + const char *argv8[] = { prgname, prefix, mp_flag, + "-n", "3", "-l", "1-" }; + const char *argv9[] = { prgname, prefix, mp_flag, + "-n", "3", "-l", "1," }; + const char *argv10[] = { prgname, prefix, mp_flag, + "-n", "3", "-l", "1#2" }; + /* sanity check test - valid corelist value */ + const char *argv11[] = { prgname, prefix, mp_flag, + "-n", "3", "-l", "1-2,3" }; + + /* --lcores flag but no lcores value */ + const char *argv12[] = { prgname, prefix, mp_flag, + "-n", "3", "--lcores" }; + const char *argv13[] = { prgname, prefix, mp_flag, + "-n", "3", "--lcores", " " }; + /* bad lcores value */ + const char *argv14[] = { prgname, prefix, mp_flag, + "-n", "3", "--lcores", "1-3-5" }; + const char *argv15[] = { prgname, prefix, mp_flag, + "-n", "3", "--lcores", "0-1,,2" }; + const char *argv16[] = { prgname, prefix, mp_flag, + "-n", "3", "--lcores", "0-,1" }; + const char *argv17[] = { prgname, prefix, mp_flag, + "-n", "3", "--lcores", "(0-,2-4)" }; + const char *argv18[] = { prgname, prefix, mp_flag, + "-n", "3", "--lcores", "(-1,2)" }; + const char *argv19[] = { prgname, prefix, mp_flag, + "-n", "3", "--lcores", "(2-4)@(2-4-6)" }; + const char *argv20[] = { prgname, prefix, mp_flag, + "-n", "3", "--lcores", "(a,2)" }; + const char *argv21[] = { prgname, prefix, mp_flag, + "-n", "3", "--lcores", "1-3@(1,3)" }; + const char *argv22[] = { prgname, prefix, mp_flag, + "-n", "3", "--lcores", "3@((1,3)" }; + const char *argv23[] = { prgname, prefix, mp_flag, + "-n", "3", "--lcores", "(4-7)=(1,3)" }; + const char *argv24[] = { prgname, prefix, mp_flag, + "-n", "3", "--lcores", "[4-7]@(1,3)" }; + /* sanity check of tests - valid lcores value */ + const char *argv25[] = { prgname, prefix, mp_flag, + "-n", "3", "--lcores", + "0-1,2@(5-7),(3-5)@(0,2),(0,6),7"}; + + if (launch_proc(argv2) != 0) { + printf("Error - " + "process did not run ok when missing -c flag\n"); + return -1; + } + + if (launch_proc(argv1) == 0 + || launch_proc(argv3) == 0) { + printf("Error - " + "process ran without error with invalid -c flag\n"); + return -1; + } + if (launch_proc(argv4) != 0) { + printf("Error - " + "process did not run ok with valid coremask value\n"); + return -1; + } + + /* start -l test */ + if (launch_proc(argv5) == 0 + || launch_proc(argv6) == 0 + || launch_proc(argv7) == 0 + || launch_proc(argv8) == 0 + || launch_proc(argv9) == 0 + || launch_proc(argv10) == 0) { + printf("Error - " + "process ran without error with invalid -l flag\n"); + return -1; + } + if (launch_proc(argv11) != 0) { + printf("Error - " + "process did not run ok with valid corelist value\n"); + return -1; + } + + /* start --lcores tests */ + if (launch_proc(argv12) == 0 || launch_proc(argv13) == 0 || + launch_proc(argv14) == 0 || launch_proc(argv15) == 0 || + launch_proc(argv16) == 0 || launch_proc(argv17) == 0 || + launch_proc(argv18) == 0 || launch_proc(argv19) == 0 || + launch_proc(argv20) == 0 || launch_proc(argv21) == 0 || + launch_proc(argv21) == 0 || launch_proc(argv22) == 0 || + launch_proc(argv23) == 0 || launch_proc(argv24) == 0) { + printf("Error - " + "process ran without error with invalid --lcore flag\n"); + return -1; + } + + if (launch_proc(argv25) != 0) { + printf("Error - " + "process did not run ok with valid corelist value\n"); + return -1; + } + + return 0; +} + +/* + * Test --master-lcore option with matching coremask + */ +static int +test_master_lcore_flag(void) +{ +#ifdef RTE_EXEC_ENV_BSDAPP + /* BSD target doesn't support prefixes at this point */ + const char *prefix = ""; +#else + char prefix[PATH_MAX], tmp[PATH_MAX]; + if (get_current_prefix(tmp, sizeof(tmp)) == NULL) { + printf("Error - unable to get current prefix!\n"); + return -1; + } + snprintf(prefix, sizeof(prefix), "--file-prefix=%s", tmp); +#endif + + /* --master-lcore flag but no value */ + const char *argv1[] = { prgname, prefix, mp_flag, "-n", "1", "-c", "3", "--master-lcore"}; + /* --master-lcore flag with invalid value */ + const char *argv2[] = { prgname, prefix, mp_flag, "-n", "1", "-c", "3", "--master-lcore", "-1"}; + const char *argv3[] = { prgname, prefix, mp_flag, "-n", "1", "-c", "3", "--master-lcore", "X"}; + /* master lcore not in coremask */ + const char *argv4[] = { prgname, prefix, mp_flag, "-n", "1", "-c", "3", "--master-lcore", "2"}; + /* valid value */ + const char *argv5[] = { prgname, prefix, mp_flag, "-n", "1", "-c", "3", "--master-lcore", "1"}; + /* valid value set before coremask */ + const char *argv6[] = { prgname, prefix, mp_flag, "-n", "1", "--master-lcore", "1", "-c", "3"}; + + if (launch_proc(argv1) == 0 + || launch_proc(argv2) == 0 + || launch_proc(argv3) == 0 + || launch_proc(argv4) == 0) { + printf("Error - process ran without error with wrong --master-lcore\n"); + return -1; + } + if (launch_proc(argv5) != 0 + || launch_proc(argv6) != 0) { + printf("Error - process did not run ok with valid --master-lcore\n"); + return -1; + } + return 0; +} + +/* + * Test that the app doesn't run with invalid -n flag option. + * Final test ensures it does run with valid options as sanity check + * Since -n is not compulsory for MP, we instead use --no-huge and --no-shconf + * flags. + */ +static int +test_invalid_n_flag(void) +{ +#ifdef RTE_EXEC_ENV_BSDAPP + /* BSD target doesn't support prefixes at this point */ + const char * prefix = ""; +#else + char prefix[PATH_MAX], tmp[PATH_MAX]; + if (get_current_prefix(tmp, sizeof(tmp)) == NULL) { + printf("Error - unable to get current prefix!\n"); + return -1; + } + snprintf(prefix, sizeof(prefix), "--file-prefix=%s", tmp); +#endif + + /* -n flag but no value */ + const char *argv1[] = { prgname, prefix, no_huge, no_shconf, "-c", "1", "-n"}; + /* bad numeric value */ + const char *argv2[] = { prgname, prefix, no_huge, no_shconf, "-c", "1", "-n", "e" }; + /* zero is invalid */ + const char *argv3[] = { prgname, prefix, no_huge, no_shconf, "-c", "1", "-n", "0" }; + /* sanity test - check with good value */ + const char *argv4[] = { prgname, prefix, no_huge, no_shconf, "-c", "1", "-n", "2" }; + /* sanity test - check with no -n flag */ + const char *argv5[] = { prgname, prefix, no_huge, no_shconf, "-c", "1"}; + + if (launch_proc(argv1) == 0 + || launch_proc(argv2) == 0 + || launch_proc(argv3) == 0) { + printf("Error - process ran without error when" + "invalid -n flag\n"); + return -1; + } + if (launch_proc(argv4) != 0) { + printf("Error - process did not run ok with valid num-channel value\n"); + return -1; + } + if (launch_proc(argv5) != 0) { + printf("Error - process did not run ok without -n flag\n"); + return -1; + } + + return 0; +} + +/* + * Test that the app runs with HPET, and without HPET + */ +static int +test_no_hpet_flag(void) +{ + char prefix[PATH_MAX], tmp[PATH_MAX]; + +#ifdef RTE_EXEC_ENV_BSDAPP + return 0; +#endif + if (get_current_prefix(tmp, sizeof(tmp)) == NULL) { + printf("Error - unable to get current prefix!\n"); + return -1; + } + snprintf(prefix, sizeof(prefix), "--file-prefix=%s", tmp); + + /* With --no-hpet */ + const char *argv1[] = {prgname, prefix, mp_flag, no_hpet, "-c", "1", "-n", "2"}; + /* Without --no-hpet */ + const char *argv2[] = {prgname, prefix, mp_flag, "-c", "1", "-n", "2"}; + + if (launch_proc(argv1) != 0) { + printf("Error - process did not run ok with --no-hpet flag\n"); + return -1; + } + if (launch_proc(argv2) != 0) { + printf("Error - process did not run ok without --no-hpet flag\n"); + return -1; + } + return 0; +} + +/* + * Test that the app runs with --no-huge and doesn't run when --socket-mem are + * specified with --no-huge. + */ +static int +test_no_huge_flag(void) +{ +#ifdef RTE_EXEC_ENV_BSDAPP + /* BSD target doesn't support prefixes at this point, and we also need to + * run another primary process here */ + const char * prefix = no_shconf; +#else + const char * prefix = "--file-prefix=nohuge"; +#endif + + /* With --no-huge */ + const char *argv1[] = {prgname, prefix, no_huge, "-c", "1", "-n", "2"}; + /* With --no-huge and -m */ + const char *argv2[] = {prgname, prefix, no_huge, "-c", "1", "-n", "2", + "-m", DEFAULT_MEM_SIZE}; + + /* With --no-huge and --socket-mem */ + const char *argv3[] = {prgname, prefix, no_huge, "-c", "1", "-n", "2", + "--socket-mem=" DEFAULT_MEM_SIZE}; + /* With --no-huge, -m and --socket-mem */ + const char *argv4[] = {prgname, prefix, no_huge, "-c", "1", "-n", "2", + "-m", DEFAULT_MEM_SIZE, "--socket-mem=" DEFAULT_MEM_SIZE}; + if (launch_proc(argv1) != 0) { + printf("Error - process did not run ok with --no-huge flag\n"); + return -1; + } + if (launch_proc(argv2) != 0) { + printf("Error - process did not run ok with --no-huge and -m flags\n"); + return -1; + } +#ifdef RTE_EXEC_ENV_BSDAPP + /* BSD target does not support NUMA, hence no --socket-mem tests */ + return 0; +#endif + + if (launch_proc(argv3) == 0) { + printf("Error - process run ok with --no-huge and --socket-mem " + "flags\n"); + return -1; + } + if (launch_proc(argv4) == 0) { + printf("Error - process run ok with --no-huge, -m and " + "--socket-mem flags\n"); + return -1; + } + return 0; +} + +static int +test_misc_flags(void) +{ + char hugepath[PATH_MAX] = {0}; +#ifdef RTE_EXEC_ENV_BSDAPP + /* BSD target doesn't support prefixes at this point */ + const char * prefix = ""; + const char * nosh_prefix = ""; +#else + char prefix[PATH_MAX], tmp[PATH_MAX]; + const char * nosh_prefix = "--file-prefix=noshconf"; + FILE * hugedir_handle = NULL; + char line[PATH_MAX] = {0}; + unsigned i, isempty = 1; + if (get_current_prefix(tmp, sizeof(tmp)) == NULL) { + printf("Error - unable to get current prefix!\n"); + return -1; + } + snprintf(prefix, sizeof(prefix), "--file-prefix=%s", tmp); + + /* + * get first valid hugepage path + */ + + /* get hugetlbfs mountpoints from /proc/mounts */ + hugedir_handle = fopen("/proc/mounts", "r"); + + if (hugedir_handle == NULL) { + printf("Error opening /proc/mounts!\n"); + return -1; + } + + /* read /proc/mounts */ + while (fgets(line, sizeof(line), hugedir_handle) != NULL) { + + /* find first valid hugepath */ + if (get_hugepage_path(line, sizeof(line), hugepath, sizeof(hugepath))) + break; + } + + fclose(hugedir_handle); + + /* check if path is not empty */ + for (i = 0; i < sizeof(hugepath); i++) + if (hugepath[i] != '\0') + isempty = 0; + + if (isempty) { + printf("No mounted hugepage dir found!\n"); + return -1; + } +#endif + + + /* check that some general flags don't prevent things from working. + * All cases, apart from the first, app should run. + * No further testing of output done. + */ + /* sanity check - failure with invalid option */ + const char *argv0[] = {prgname, prefix, mp_flag, "-c", "1", "--invalid-opt"}; + + /* With --no-pci */ + const char *argv1[] = {prgname, prefix, mp_flag, "-c", "1", "--no-pci"}; + /* With -v */ + const char *argv2[] = {prgname, prefix, mp_flag, "-c", "1", "-v"}; + /* With valid --syslog */ + const char *argv3[] = {prgname, prefix, mp_flag, "-c", "1", + "--syslog", "syslog"}; + /* With empty --syslog (should fail) */ + const char *argv4[] = {prgname, prefix, mp_flag, "-c", "1", "--syslog"}; + /* With invalid --syslog */ + const char *argv5[] = {prgname, prefix, mp_flag, "-c", "1", "--syslog", "error"}; + /* With no-sh-conf */ + const char *argv6[] = {prgname, "-c", "1", "-n", "2", "-m", DEFAULT_MEM_SIZE, + no_shconf, nosh_prefix }; + +#ifdef RTE_EXEC_ENV_BSDAPP + return 0; +#endif + /* With --huge-dir */ + const char *argv7[] = {prgname, "-c", "1", "-n", "2", "-m", DEFAULT_MEM_SIZE, + "--file-prefix=hugedir", "--huge-dir", hugepath}; + /* With empty --huge-dir (should fail) */ + const char *argv8[] = {prgname, "-c", "1", "-n", "2", "-m", DEFAULT_MEM_SIZE, + "--file-prefix=hugedir", "--huge-dir"}; + /* With invalid --huge-dir */ + const char *argv9[] = {prgname, "-c", "1", "-n", "2", "-m", DEFAULT_MEM_SIZE, + "--file-prefix=hugedir", "--huge-dir", "invalid"}; + /* Secondary process with invalid --huge-dir (should run as flag has no + * effect on secondary processes) */ + const char *argv10[] = {prgname, prefix, mp_flag, "-c", "1", "--huge-dir", "invalid"}; + + /* try running with base-virtaddr param */ + const char *argv11[] = {prgname, "--file-prefix=virtaddr", + "-c", "1", "-n", "2", "--base-virtaddr=0x12345678"}; + + /* try running with --vfio-intr INTx flag */ + const char *argv12[] = {prgname, "--file-prefix=intr", + "-c", "1", "-n", "2", "--vfio-intr=legacy"}; + + /* try running with --vfio-intr MSI flag */ + const char *argv13[] = {prgname, "--file-prefix=intr", + "-c", "1", "-n", "2", "--vfio-intr=msi"}; + + /* try running with --vfio-intr MSI-X flag */ + const char *argv14[] = {prgname, "--file-prefix=intr", + "-c", "1", "-n", "2", "--vfio-intr=msix"}; + + /* try running with --vfio-intr invalid flag */ + const char *argv15[] = {prgname, "--file-prefix=intr", + "-c", "1", "-n", "2", "--vfio-intr=invalid"}; + + + if (launch_proc(argv0) == 0) { + printf("Error - process ran ok with invalid flag\n"); + return -1; + } + if (launch_proc(argv1) != 0) { + printf("Error - process did not run ok with --no-pci flag\n"); + return -1; + } + if (launch_proc(argv2) != 0) { + printf("Error - process did not run ok with -v flag\n"); + return -1; + } + if (launch_proc(argv3) != 0) { + printf("Error - process did not run ok with --syslog flag\n"); + return -1; + } + if (launch_proc(argv4) == 0) { + printf("Error - process run ok with empty --syslog flag\n"); + return -1; + } + if (launch_proc(argv5) == 0) { + printf("Error - process run ok with invalid --syslog flag\n"); + return -1; + } + if (launch_proc(argv6) != 0) { + printf("Error - process did not run ok with --no-shconf flag\n"); + return -1; + } +#ifdef RTE_EXEC_ENV_BSDAPP + return 0; +#endif + if (launch_proc(argv7) != 0) { + printf("Error - process did not run ok with --huge-dir flag\n"); + return -1; + } + if (launch_proc(argv8) == 0) { + printf("Error - process run ok with empty --huge-dir flag\n"); + return -1; + } + if (launch_proc(argv9) == 0) { + printf("Error - process run ok with invalid --huge-dir flag\n"); + return -1; + } + if (launch_proc(argv10) != 0) { + printf("Error - secondary process did not run ok with invalid --huge-dir flag\n"); + return -1; + } + if (launch_proc(argv11) != 0) { + printf("Error - process did not run ok with --base-virtaddr parameter\n"); + return -1; + } + if (launch_proc(argv12) != 0) { + printf("Error - process did not run ok with " + "--vfio-intr INTx parameter\n"); + return -1; + } + if (launch_proc(argv13) != 0) { + printf("Error - process did not run ok with " + "--vfio-intr MSI parameter\n"); + return -1; + } + if (launch_proc(argv14) != 0) { + printf("Error - process did not run ok with " + "--vfio-intr MSI-X parameter\n"); + return -1; + } + if (launch_proc(argv15) == 0) { + printf("Error - process run ok with " + "--vfio-intr invalid parameter\n"); + return -1; + } + return 0; +} + +static int +test_file_prefix(void) +{ + /* + * 1. check if current process hugefiles are locked + * 2. try to run secondary process without a corresponding primary process + * (while failing to run, it will also remove any unused hugepage files) + * 3. check if current process hugefiles are still in place and are locked + * 4. run a primary process with memtest1 prefix + * 5. check if memtest1 hugefiles are created + * 6. run a primary process with memtest2 prefix + * 7. check that only memtest2 hugefiles are present in the hugedir + */ + +#ifdef RTE_EXEC_ENV_BSDAPP + return 0; +#endif + + /* this should fail unless the test itself is run with "memtest" prefix */ + const char *argv0[] = {prgname, mp_flag, "-c", "1", "-n", "2", "-m", DEFAULT_MEM_SIZE, + "--file-prefix=" memtest }; + + /* primary process with memtest1 */ + const char *argv1[] = {prgname, "-c", "1", "-n", "2", "-m", DEFAULT_MEM_SIZE, + "--file-prefix=" memtest1 }; + + /* primary process with memtest2 */ + const char *argv2[] = {prgname, "-c", "1", "-n", "2", "-m", DEFAULT_MEM_SIZE, + "--file-prefix=" memtest2 }; + + char prefix[32]; + if (get_current_prefix(prefix, sizeof(prefix)) == NULL) { + printf("Error - unable to get current prefix!\n"); + return -1; + } + + /* check if files for current prefix are present */ + if (process_hugefiles(prefix, HUGEPAGE_CHECK_EXISTS) != 1) { + printf("Error - hugepage files for %s were not created!\n", prefix); + return -1; + } + + /* checks if files for current prefix are locked */ + if (process_hugefiles(prefix, HUGEPAGE_CHECK_LOCKED) != 1) { + printf("Error - hugepages for current process aren't locked!\n"); + return -1; + } + + /* check if files for secondary process are present */ + if (process_hugefiles(memtest, HUGEPAGE_CHECK_EXISTS) == 1) { + /* check if they are not locked */ + if (process_hugefiles(memtest, HUGEPAGE_CHECK_LOCKED) == 1) { + printf("Error - hugepages for current process are locked!\n"); + return -1; + } + /* they aren't locked, delete them */ + else { + if (process_hugefiles(memtest, HUGEPAGE_DELETE) != 1) { + printf("Error - deleting hugepages failed!\n"); + return -1; + } + } + } + + if (launch_proc(argv0) == 0) { + printf("Error - secondary process ran ok without primary process\n"); + return -1; + } + + /* check if files for current prefix are present */ + if (process_hugefiles(prefix, HUGEPAGE_CHECK_EXISTS) != 1) { + printf("Error - hugepage files for %s were not created!\n", prefix); + return -1; + } + + /* checks if files for current prefix are locked */ + if (process_hugefiles(prefix, HUGEPAGE_CHECK_LOCKED) != 1) { + printf("Error - hugepages for current process aren't locked!\n"); + return -1; + } + + if (launch_proc(argv1) != 0) { + printf("Error - failed to run with --file-prefix=%s\n", memtest); + return -1; + } + + /* check if memtest1_map0 is present */ + if (process_hugefiles(memtest1, HUGEPAGE_CHECK_EXISTS) != 1) { + printf("Error - hugepage files for %s were not created!\n", memtest1); + return -1; + } + + if (launch_proc(argv2) != 0) { + printf("Error - failed to run with --file-prefix=%s\n", memtest2); + return -1; + } + + /* check if hugefiles for memtest2 are present */ + if (process_hugefiles(memtest2, HUGEPAGE_CHECK_EXISTS) != 1) { + printf("Error - hugepage files for %s were not created!\n", memtest2); + return -1; + } + + /* check if hugefiles for memtest1 are present */ + if (process_hugefiles(memtest1, HUGEPAGE_CHECK_EXISTS) != 0) { + printf("Error - hugepage files for %s were not deleted!\n", memtest1); + return -1; + } + + return 0; +} + +/* + * Tests for correct handling of -m and --socket-mem flags + */ +static int +test_memory_flags(void) +{ +#ifdef RTE_EXEC_ENV_BSDAPP + /* BSD target doesn't support prefixes at this point */ + const char * prefix = ""; +#else + char prefix[PATH_MAX], tmp[PATH_MAX]; + if (get_current_prefix(tmp, sizeof(tmp)) == NULL) { + printf("Error - unable to get current prefix!\n"); + return -1; + } + snprintf(prefix, sizeof(prefix), "--file-prefix=%s", tmp); +#endif + + /* valid -m flag and mp flag */ + const char *argv0[] = {prgname, prefix, mp_flag, "-c", "10", + "-n", "2", "-m", DEFAULT_MEM_SIZE}; + + /* valid -m flag */ + const char *argv1[] = {prgname, "-c", "10", "-n", "2", + "--file-prefix=" memtest, "-m", DEFAULT_MEM_SIZE}; + + /* invalid (zero) --socket-mem flag */ + const char *argv2[] = {prgname, "-c", "10", "-n", "2", + "--file-prefix=" memtest, "--socket-mem=0,0,0,0"}; + + /* invalid (incomplete) --socket-mem flag */ + const char *argv3[] = {prgname, "-c", "10", "-n", "2", + "--file-prefix=" memtest, "--socket-mem=2,2,"}; + + /* invalid (mixed with invalid data) --socket-mem flag */ + const char *argv4[] = {prgname, "-c", "10", "-n", "2", + "--file-prefix=" memtest, "--socket-mem=2,2,Fred"}; + + /* invalid (with numeric value as last character) --socket-mem flag */ + const char *argv5[] = {prgname, "-c", "10", "-n", "2", + "--file-prefix=" memtest, "--socket-mem=2,2,Fred0"}; + + /* invalid (with empty socket) --socket-mem flag */ + const char *argv6[] = {prgname, "-c", "10", "-n", "2", + "--file-prefix=" memtest, "--socket-mem=2,,2"}; + + /* invalid (null) --socket-mem flag */ + const char *argv7[] = {prgname, "-c", "10", "-n", "2", + "--file-prefix=" memtest, "--socket-mem="}; + + /* valid --socket-mem specified together with -m flag */ + const char *argv8[] = {prgname, "-c", "10", "-n", "2", + "--file-prefix=" memtest, "-m", DEFAULT_MEM_SIZE, "--socket-mem=2,2"}; + + /* construct an invalid socket mask with 2 megs on each socket plus + * extra 2 megs on socket that doesn't exist on current system */ + char invalid_socket_mem[SOCKET_MEM_STRLEN]; + char buf[SOCKET_MEM_STRLEN]; /* to avoid copying string onto itself */ + +#ifdef RTE_EXEC_ENV_BSDAPP + int i, num_sockets = 1; +#else + int i, num_sockets = get_number_of_sockets(); +#endif + + if (num_sockets <= 0 || num_sockets > RTE_MAX_NUMA_NODES) { + printf("Error - cannot get number of sockets!\n"); + return -1; + } + + snprintf(invalid_socket_mem, sizeof(invalid_socket_mem), "--socket-mem="); + + /* add one extra socket */ + for (i = 0; i < num_sockets + 1; i++) { + snprintf(buf, sizeof(buf), "%s%s", invalid_socket_mem, DEFAULT_MEM_SIZE); + snprintf(invalid_socket_mem, sizeof(invalid_socket_mem), "%s", buf); + + if (num_sockets + 1 - i > 1) { + snprintf(buf, sizeof(buf), "%s,", invalid_socket_mem); + snprintf(invalid_socket_mem, sizeof(invalid_socket_mem), "%s", buf); + } + } + + /* construct a valid socket mask with 2 megs on each existing socket */ + char valid_socket_mem[SOCKET_MEM_STRLEN]; + + snprintf(valid_socket_mem, sizeof(valid_socket_mem), "--socket-mem="); + + /* add one extra socket */ + for (i = 0; i < num_sockets; i++) { + snprintf(buf, sizeof(buf), "%s%s", valid_socket_mem, DEFAULT_MEM_SIZE); + snprintf(valid_socket_mem, sizeof(valid_socket_mem), "%s", buf); + + if (num_sockets - i > 1) { + snprintf(buf, sizeof(buf), "%s,", valid_socket_mem); + snprintf(valid_socket_mem, sizeof(valid_socket_mem), "%s", buf); + } + } + + /* invalid --socket-mem flag (with extra socket) */ + const char *argv9[] = {prgname, "-c", "10", "-n", "2", + "--file-prefix=" memtest, invalid_socket_mem}; + + /* valid --socket-mem flag */ + const char *argv10[] = {prgname, "-c", "10", "-n", "2", + "--file-prefix=" memtest, valid_socket_mem}; + + if (launch_proc(argv0) != 0) { + printf("Error - secondary process failed with valid -m flag !\n"); + return -1; + } + +#ifdef RTE_EXEC_ENV_BSDAPP + /* no other tests are applicable to BSD */ + return 0; +#endif + + if (launch_proc(argv1) != 0) { + printf("Error - process failed with valid -m flag!\n"); + return -1; + } + if (launch_proc(argv2) == 0) { + printf("Error - process run ok with invalid (zero) --socket-mem!\n"); + return -1; + } + + if (launch_proc(argv3) == 0) { + printf("Error - process run ok with invalid " + "(incomplete) --socket-mem!\n"); + return -1; + } + + if (launch_proc(argv4) == 0) { + printf("Error - process run ok with invalid " + "(mixed with invalid input) --socket-mem!\n"); + return -1; + } + + if (launch_proc(argv5) == 0) { + printf("Error - process run ok with invalid " + "(mixed with invalid input with a numeric value as " + "last character) --socket-mem!\n"); + return -1; + } + + if (launch_proc(argv6) == 0) { + printf("Error - process run ok with invalid " + "(with empty socket) --socket-mem!\n"); + return -1; + } + + if (launch_proc(argv7) == 0) { + printf("Error - process run ok with invalid (null) --socket-mem!\n"); + return -1; + } + + if (launch_proc(argv8) == 0) { + printf("Error - process run ok with --socket-mem and -m specified!\n"); + return -1; + } + + if (launch_proc(argv9) == 0) { + printf("Error - process run ok with extra socket in --socket-mem!\n"); + return -1; + } + + if (launch_proc(argv10) != 0) { + printf("Error - process failed with valid --socket-mem!\n"); + return -1; + } + + return 0; +} + +static int +test_eal_flags(void) +{ + int ret = 0; + + ret = test_missing_c_flag(); + if (ret < 0) { + printf("Error in test_missing_c_flag()\n"); + return ret; + } + + ret = test_master_lcore_flag(); + if (ret < 0) { + printf("Error in test_master_lcore_flag()\n"); + return ret; + } + + ret = test_invalid_n_flag(); + if (ret < 0) { + printf("Error in test_invalid_n_flag()\n"); + return ret; + } + + ret = test_no_hpet_flag(); + if (ret < 0) { + printf("Error in test_no_hpet_flag()\n"); + return ret; + } + + ret = test_no_huge_flag(); + if (ret < 0) { + printf("Error in test_no_huge_flag()\n"); + return ret; + } + + ret = test_whitelist_flag(); + if (ret < 0) { + printf("Error in test_invalid_whitelist_flag()\n"); + return ret; + } + + ret = test_invalid_b_flag(); + if (ret < 0) { + printf("Error in test_invalid_b_flag()\n"); + return ret; + } + +#ifdef RTE_LIBRTE_PMD_RING + ret = test_invalid_vdev_flag(); + if (ret < 0) { + printf("Error in test_invalid_vdev_flag()\n"); + return ret; + } +#endif + ret = test_invalid_r_flag(); + if (ret < 0) { + printf("Error in test_invalid_r_flag()\n"); + return ret; + } + + ret = test_memory_flags(); + if (ret < 0) { + printf("Error in test_memory_flags()\n"); + return ret; + } + + ret = test_file_prefix(); + if (ret < 0) { + printf("Error in test_file_prefix()\n"); + return ret; + } + + ret = test_misc_flags(); + if (ret < 0) { + printf("Error in test_misc_flags()"); + return ret; + } + + return ret; +} + +REGISTER_TEST_COMMAND(eal_flags_autotest, test_eal_flags); diff --git a/dpdk/test/test/test_eal_fs.c b/dpdk/test/test/test_eal_fs.c new file mode 100644 index 00000000..78978120 --- /dev/null +++ b/dpdk/test/test/test_eal_fs.c @@ -0,0 +1,206 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "test.h" +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <errno.h> + +/* eal_filesystem.h is not a public header file, so use relative path */ +#include "../../lib/librte_eal/common/eal_filesystem.h" + +static int +test_parse_sysfs_value(void) +{ + char filename[PATH_MAX] = ""; + char proc_path[PATH_MAX]; + char file_template[] = "/tmp/eal_test_XXXXXX"; + int tmp_file_handle = -1; + FILE *fd = NULL; + unsigned valid_number; + unsigned long retval = 0; + +#ifdef RTE_EXEC_ENV_BSDAPP + /* BSD doesn't have /proc/pid/fd */ + return 0; +#endif + + printf("Testing function eal_parse_sysfs_value()\n"); + + /* get a temporary filename to use for all tests - create temp file handle and then + * use /proc to get the actual file that we can open */ + tmp_file_handle = mkstemp(file_template); + if (tmp_file_handle == -1) { + perror("mkstemp() failure"); + goto error; + } + snprintf(proc_path, sizeof(proc_path), "/proc/self/fd/%d", tmp_file_handle); + if (readlink(proc_path, filename, sizeof(filename)) < 0) { + perror("readlink() failure"); + goto error; + } + printf("Temporary file is: %s\n", filename); + + /* test we get an error value if we use file before it's created */ + printf("Test reading a missing file ...\n"); + if (eal_parse_sysfs_value("/dev/not-quite-null", &retval) == 0) { + printf("Error with eal_parse_sysfs_value() - returned success on reading empty file\n"); + goto error; + } + printf("Confirmed return error when reading empty file\n"); + + /* test reading a valid number value with "\n" on the end */ + printf("Test reading valid values ...\n"); + valid_number = 15; + fd = fopen(filename,"w"); + if (fd == NULL) { + printf("line %d, Error opening %s: %s\n", __LINE__, filename, strerror(errno)); + goto error; + } + fprintf(fd,"%u\n", valid_number); + fclose(fd); + fd = NULL; + if (eal_parse_sysfs_value(filename, &retval) < 0) { + printf("eal_parse_sysfs_value() returned error - test failed\n"); + goto error; + } + if (retval != valid_number) { + printf("Invalid value read by eal_parse_sysfs_value() - test failed\n"); + goto error; + } + printf("Read '%u\\n' ok\n", valid_number); + + /* test reading a valid hex number value with "\n" on the end */ + valid_number = 25; + fd = fopen(filename,"w"); + if (fd == NULL) { + printf("line %d, Error opening %s: %s\n", __LINE__, filename, strerror(errno)); + goto error; + } + fprintf(fd,"0x%x\n", valid_number); + fclose(fd); + fd = NULL; + if (eal_parse_sysfs_value(filename, &retval) < 0) { + printf("eal_parse_sysfs_value() returned error - test failed\n"); + goto error; + } + if (retval != valid_number) { + printf("Invalid value read by eal_parse_sysfs_value() - test failed\n"); + goto error; + } + printf("Read '0x%x\\n' ok\n", valid_number); + + printf("Test reading invalid values ...\n"); + + /* test reading an empty file - expect failure!*/ + fd = fopen(filename,"w"); + if (fd == NULL) { + printf("line %d, Error opening %s: %s\n", __LINE__, filename, strerror(errno)); + goto error; + } + fclose(fd); + fd = NULL; + if (eal_parse_sysfs_value(filename, &retval) == 0) { + printf("eal_parse_sysfs_value() read invalid value - test failed\n"); + goto error; + } + + /* test reading a valid number value *without* "\n" on the end - expect failure!*/ + valid_number = 3; + fd = fopen(filename,"w"); + if (fd == NULL) { + printf("line %d, Error opening %s: %s\n", __LINE__, filename, strerror(errno)); + goto error; + } + fprintf(fd,"%u", valid_number); + fclose(fd); + fd = NULL; + if (eal_parse_sysfs_value(filename, &retval) == 0) { + printf("eal_parse_sysfs_value() read invalid value - test failed\n"); + goto error; + } + + /* test reading a valid number value followed by string - expect failure!*/ + valid_number = 3; + fd = fopen(filename,"w"); + if (fd == NULL) { + printf("line %d, Error opening %s: %s\n", __LINE__, filename, strerror(errno)); + goto error; + } + fprintf(fd,"%uJ\n", valid_number); + fclose(fd); + fd = NULL; + if (eal_parse_sysfs_value(filename, &retval) == 0) { + printf("eal_parse_sysfs_value() read invalid value - test failed\n"); + goto error; + } + + /* test reading a non-numeric value - expect failure!*/ + fd = fopen(filename,"w"); + if (fd == NULL) { + printf("line %d, Error opening %s: %s\n", __LINE__, filename, strerror(errno)); + goto error; + } + fprintf(fd,"error\n"); + fclose(fd); + fd = NULL; + if (eal_parse_sysfs_value(filename, &retval) == 0) { + printf("eal_parse_sysfs_value() read invalid value - test failed\n"); + goto error; + } + + close(tmp_file_handle); + unlink(filename); + printf("eal_parse_sysfs_value() - OK\n"); + return 0; + +error: + if (fd) + fclose(fd); + if (tmp_file_handle > 0) + close(tmp_file_handle); + if (filename[0] != '\0') + unlink(filename); + return -1; +} + +static int +test_eal_fs(void) +{ + if (test_parse_sysfs_value() < 0) + return -1; + return 0; +} + +REGISTER_TEST_COMMAND(eal_fs_autotest, test_eal_fs); diff --git a/dpdk/test/test/test_efd.c b/dpdk/test/test/test_efd.c new file mode 100644 index 00000000..54430616 --- /dev/null +++ b/dpdk/test/test/test_efd.c @@ -0,0 +1,500 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <rte_memcpy.h> +#include <rte_malloc.h> +#include <rte_efd.h> +#include <rte_byteorder.h> +#include <rte_random.h> +#include <rte_debug.h> +#include <rte_ip.h> + +#include "test.h" + +#define EFD_TEST_KEY_LEN 8 +#define TABLE_SIZE (1 << 21) +#define ITERATIONS 3 + +#if RTE_EFD_VALUE_NUM_BITS == 32 +#define VALUE_BITMASK 0xffffffff +#else +#define VALUE_BITMASK ((1 << RTE_EFD_VALUE_NUM_BITS) - 1) +#endif +static unsigned int test_socket_id; + +/* 5-tuple key type */ +struct flow_key { + uint32_t ip_src; + uint32_t ip_dst; + uint16_t port_src; + uint16_t port_dst; + uint8_t proto; +} __attribute__((packed)); +/* + * Print out result of unit test efd operation. + */ +#if defined(UNIT_TEST_EFD_VERBOSE) + +static void print_key_info(const char *msg, const struct flow_key *key, + efd_value_t val) +{ + const uint8_t *p = (const uint8_t *) key; + unsigned int i; + + printf("%s key:0x", msg); + for (i = 0; i < sizeof(struct flow_key); i++) + printf("%02X", p[i]); + + printf(" @ val %d\n", val); +} +#else + +static void print_key_info(__attribute__((unused)) const char *msg, + __attribute__((unused)) const struct flow_key *key, + __attribute__((unused)) efd_value_t val) +{ +} +#endif + +/* Keys used by unit test functions */ +static struct flow_key keys[5] = { + { + .ip_src = IPv4(0x03, 0x02, 0x01, 0x00), + .ip_dst = IPv4(0x07, 0x06, 0x05, 0x04), + .port_src = 0x0908, + .port_dst = 0x0b0a, + .proto = 0x0c, + }, + { + .ip_src = IPv4(0x13, 0x12, 0x11, 0x10), + .ip_dst = IPv4(0x17, 0x16, 0x15, 0x14), + .port_src = 0x1918, + .port_dst = 0x1b1a, + .proto = 0x1c, + }, + { + .ip_src = IPv4(0x23, 0x22, 0x21, 0x20), + .ip_dst = IPv4(0x27, 0x26, 0x25, 0x24), + .port_src = 0x2928, + .port_dst = 0x2b2a, + .proto = 0x2c, + }, + { + .ip_src = IPv4(0x33, 0x32, 0x31, 0x30), + .ip_dst = IPv4(0x37, 0x36, 0x35, 0x34), + .port_src = 0x3938, + .port_dst = 0x3b3a, + .proto = 0x3c, + }, + { + .ip_src = IPv4(0x43, 0x42, 0x41, 0x40), + .ip_dst = IPv4(0x47, 0x46, 0x45, 0x44), + .port_src = 0x4948, + .port_dst = 0x4b4a, + .proto = 0x4c, + } +}; +/* Array to store the data */ +efd_value_t data[5]; + +static inline uint8_t efd_get_all_sockets_bitmask(void) +{ + uint8_t all_cpu_sockets_bitmask = 0; + unsigned int i; + unsigned int next_lcore = rte_get_master_lcore(); + const int val_true = 1, val_false = 0; + for (i = 0; i < rte_lcore_count(); i++) { + all_cpu_sockets_bitmask |= 1 << rte_lcore_to_socket_id(next_lcore); + next_lcore = rte_get_next_lcore(next_lcore, val_false, val_true); + } + + return all_cpu_sockets_bitmask; +} + +/* + * Basic sequence of operations for a single key: + * - add + * - lookup (hit) + * - delete + * Note: lookup (miss) is not applicable since this is a filter + */ +static int test_add_delete(void) +{ + struct rte_efd_table *handle; + /* test with standard add/lookup/delete functions */ + efd_value_t prev_value; + printf("Entering %s\n", __func__); + + handle = rte_efd_create("test_add_delete", + TABLE_SIZE, sizeof(struct flow_key), + efd_get_all_sockets_bitmask(), test_socket_id); + TEST_ASSERT_NOT_NULL(handle, "Error creating the EFD table\n"); + + data[0] = mrand48() & VALUE_BITMASK; + TEST_ASSERT_SUCCESS(rte_efd_update(handle, test_socket_id, &keys[0], + data[0]), + "Error inserting the key"); + print_key_info("Add", &keys[0], data[0]); + + TEST_ASSERT_EQUAL(rte_efd_lookup(handle, test_socket_id, &keys[0]), + data[0], + "failed to find key"); + + TEST_ASSERT_SUCCESS(rte_efd_delete(handle, test_socket_id, &keys[0], + &prev_value), + "failed to delete key"); + TEST_ASSERT_EQUAL(prev_value, data[0], + "failed to delete the expected value, got %d, " + "expected %d", prev_value, data[0]); + print_key_info("Del", &keys[0], data[0]); + + rte_efd_free(handle); + + return 0; +} + +/* + * Sequence of operations for a single key: + * - add + * - lookup: hit + * - add: update + * - lookup: hit (updated data) + * - delete: hit + */ +static int test_add_update_delete(void) +{ + struct rte_efd_table *handle; + printf("Entering %s\n", __func__); + /* test with standard add/lookup/delete functions */ + efd_value_t prev_value; + data[1] = mrand48() & VALUE_BITMASK; + + handle = rte_efd_create("test_add_update_delete", TABLE_SIZE, + sizeof(struct flow_key), + efd_get_all_sockets_bitmask(), test_socket_id); + TEST_ASSERT_NOT_NULL(handle, "Error creating the efd table\n"); + + TEST_ASSERT_SUCCESS(rte_efd_update(handle, test_socket_id, &keys[1], + data[1]), "Error inserting the key"); + print_key_info("Add", &keys[1], data[1]); + + TEST_ASSERT_EQUAL(rte_efd_lookup(handle, test_socket_id, &keys[1]), + data[1], "failed to find key"); + print_key_info("Lkp", &keys[1], data[1]); + + data[1] = data[1] + 1; + TEST_ASSERT_SUCCESS(rte_efd_update(handle, test_socket_id, &keys[1], + data[1]), "Error re-inserting the key"); + print_key_info("Add", &keys[1], data[1]); + + TEST_ASSERT_EQUAL(rte_efd_lookup(handle, test_socket_id, &keys[1]), + data[1], "failed to find key"); + print_key_info("Lkp", &keys[1], data[1]); + + TEST_ASSERT_SUCCESS(rte_efd_delete(handle, test_socket_id, &keys[1], + &prev_value), "failed to delete key"); + TEST_ASSERT_EQUAL(prev_value, data[1], + "failed to delete the expected value, got %d, " + "expected %d", prev_value, data[1]); + print_key_info("Del", &keys[1], data[1]); + + + rte_efd_free(handle); + return 0; +} + +/* + * Sequence of operations for find existing EFD table + * + * - create table + * - find existing table: hit + * - find non-existing table: miss + * + */ +static int test_efd_find_existing(void) +{ + struct rte_efd_table *handle = NULL, *result = NULL; + + printf("Entering %s\n", __func__); + + /* Create EFD table. */ + handle = rte_efd_create("efd_find_existing", TABLE_SIZE, + sizeof(struct flow_key), + efd_get_all_sockets_bitmask(), test_socket_id); + TEST_ASSERT_NOT_NULL(handle, "Error creating the efd table\n"); + + /* Try to find existing EFD table */ + result = rte_efd_find_existing("efd_find_existing"); + TEST_ASSERT_EQUAL(result, handle, "could not find existing efd table"); + + /* Try to find non-existing EFD table */ + result = rte_efd_find_existing("efd_find_non_existing"); + TEST_ASSERT_NULL(result, "found table that shouldn't exist"); + + /* Cleanup. */ + rte_efd_free(handle); + + return 0; +} + +/* + * Sequence of operations for 5 keys + * - add keys + * - lookup keys: hit (bulk) + * - add keys (update) + * - lookup keys: hit (updated data) + * - delete keys : hit + */ +static int test_five_keys(void) +{ + struct rte_efd_table *handle; + const void *key_array[5] = {0}; + efd_value_t result[5] = {0}; + efd_value_t prev_value; + unsigned int i; + printf("Entering %s\n", __func__); + + handle = rte_efd_create("test_five_keys", TABLE_SIZE, + sizeof(struct flow_key), + efd_get_all_sockets_bitmask(), test_socket_id); + TEST_ASSERT_NOT_NULL(handle, "Error creating the efd table\n"); + + /* Setup data */ + for (i = 0; i < 5; i++) + data[i] = mrand48() & VALUE_BITMASK; + + /* Add */ + for (i = 0; i < 5; i++) { + TEST_ASSERT_SUCCESS(rte_efd_update(handle, test_socket_id, + &keys[i], data[i]), + "Error inserting the key"); + print_key_info("Add", &keys[i], data[i]); + } + + /* Lookup */ + for (i = 0; i < 5; i++) + key_array[i] = &keys[i]; + + rte_efd_lookup_bulk(handle, test_socket_id, 5, + (void *) &key_array, result); + + for (i = 0; i < 5; i++) { + TEST_ASSERT_EQUAL(result[i], data[i], + "bulk: failed to find key. Expected %d, got %d", + data[i], result[i]); + print_key_info("Lkp", &keys[i], data[i]); + } + + /* Modify data (bulk) */ + for (i = 0; i < 5; i++) + data[i] = data[i] + 1; + + /* Add - update */ + for (i = 0; i < 5; i++) { + TEST_ASSERT_SUCCESS(rte_efd_update(handle, test_socket_id, + &keys[i], data[i]), + "Error inserting the key"); + print_key_info("Add", &keys[i], data[i]); + } + + /* Lookup */ + for (i = 0; i < 5; i++) { + TEST_ASSERT_EQUAL(rte_efd_lookup(handle, test_socket_id, + &keys[i]), data[i], + "failed to find key"); + print_key_info("Lkp", &keys[i], data[i]); + } + + /* Delete */ + for (i = 0; i < 5; i++) { + TEST_ASSERT_SUCCESS(rte_efd_delete(handle, test_socket_id, + &keys[i], &prev_value), + "failed to delete key"); + TEST_ASSERT_EQUAL(prev_value, data[i], + "failed to delete the expected value, got %d, " + "expected %d", prev_value, data[i]); + print_key_info("Del", &keys[i], data[i]); + } + + + rte_efd_free(handle); + + return 0; +} + +/* + * Test to see the average table utilization (entries added/max entries) + * before hitting a random entry that cannot be added + */ +static int test_average_table_utilization(void) +{ + struct rte_efd_table *handle = NULL; + uint32_t num_rules_in = TABLE_SIZE; + uint8_t simple_key[EFD_TEST_KEY_LEN]; + unsigned int i, j; + unsigned int added_keys, average_keys_added = 0; + + printf("Evaluating table utilization and correctness, please wait\n"); + fflush(stdout); + + for (j = 0; j < ITERATIONS; j++) { + handle = rte_efd_create("test_efd", num_rules_in, + EFD_TEST_KEY_LEN, efd_get_all_sockets_bitmask(), + test_socket_id); + if (handle == NULL) { + printf("efd table creation failed\n"); + return -1; + } + + unsigned int succeeded = 0; + unsigned int lost_keys = 0; + + /* Add random entries until key cannot be added */ + for (added_keys = 0; added_keys < num_rules_in; added_keys++) { + + for (i = 0; i < EFD_TEST_KEY_LEN; i++) + simple_key[i] = rte_rand() & 0xFF; + + efd_value_t val = simple_key[0]; + + if (rte_efd_update(handle, test_socket_id, simple_key, + val)) + break; /* continue;*/ + if (rte_efd_lookup(handle, test_socket_id, simple_key) + != val) + lost_keys++; + else + succeeded++; + } + + average_keys_added += succeeded; + + /* Reset the table */ + rte_efd_free(handle); + + /* Print progress on operations */ + printf("Added %10u Succeeded %10u Lost %10u\n", + added_keys, succeeded, lost_keys); + fflush(stdout); + } + + average_keys_added /= ITERATIONS; + + printf("\nAverage table utilization = %.2f%% (%u/%u)\n", + ((double) average_keys_added / num_rules_in * 100), + average_keys_added, num_rules_in); + + return 0; +} + +/* + * Do tests for EFD creation with bad parameters. + */ +static int test_efd_creation_with_bad_parameters(void) +{ + struct rte_efd_table *handle, *tmp; + printf("Entering %s, **Errors are expected **\n", __func__); + + handle = rte_efd_create("creation_with_bad_parameters_0", TABLE_SIZE, 0, + efd_get_all_sockets_bitmask(), test_socket_id); + if (handle != NULL) { + rte_efd_free(handle); + printf("Impossible creating EFD table successfully " + "if key_len in parameter is zero\n"); + return -1; + } + + handle = rte_efd_create("creation_with_bad_parameters_1", TABLE_SIZE, + sizeof(struct flow_key), 0, test_socket_id); + if (handle != NULL) { + rte_efd_free(handle); + printf("Impossible creating EFD table successfully " + "with invalid socket bitmask\n"); + return -1; + } + + handle = rte_efd_create("creation_with_bad_parameters_2", TABLE_SIZE, + sizeof(struct flow_key), efd_get_all_sockets_bitmask(), + 255); + if (handle != NULL) { + rte_efd_free(handle); + printf("Impossible creating EFD table successfully " + "with invalid socket\n"); + return -1; + } + + /* test with same name should fail */ + handle = rte_efd_create("same_name", TABLE_SIZE, + sizeof(struct flow_key), + efd_get_all_sockets_bitmask(), 0); + if (handle == NULL) { + printf("Cannot create first EFD table with 'same_name'\n"); + return -1; + } + tmp = rte_efd_create("same_name", TABLE_SIZE, sizeof(struct flow_key), + efd_get_all_sockets_bitmask(), 0); + if (tmp != NULL) { + printf("Creation of EFD table with same name should fail\n"); + rte_efd_free(handle); + rte_efd_free(tmp); + return -1; + } + rte_efd_free(handle); + + printf("# Test successful. No more errors expected\n"); + + return 0; +} + +static int +test_efd(void) +{ + + /* Unit tests */ + if (test_add_delete() < 0) + return -1; + if (test_efd_find_existing() < 0) + return -1; + if (test_add_update_delete() < 0) + return -1; + if (test_five_keys() < 0) + return -1; + if (test_efd_creation_with_bad_parameters() < 0) + return -1; + if (test_average_table_utilization() < 0) + return -1; + + return 0; +} + +REGISTER_TEST_COMMAND(efd_autotest, test_efd); diff --git a/dpdk/test/test/test_efd_perf.c b/dpdk/test/test/test_efd_perf.c new file mode 100644 index 00000000..2b8a8eac --- /dev/null +++ b/dpdk/test/test/test_efd_perf.c @@ -0,0 +1,414 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <inttypes.h> + +#include <rte_lcore.h> +#include <rte_cycles.h> +#include <rte_malloc.h> +#include <rte_random.h> +#include <rte_efd.h> +#include <rte_memcpy.h> +#include <rte_thash.h> + +#include "test.h" + +#define NUM_KEYSIZES 10 +#define NUM_SHUFFLES 10 +#define MAX_KEYSIZE 64 +#define MAX_ENTRIES (1 << 19) +#define KEYS_TO_ADD (MAX_ENTRIES * 3 / 4) /* 75% table utilization */ +#define NUM_LOOKUPS (KEYS_TO_ADD * 5) /* Loop among keys added, several times */ + +#if RTE_EFD_VALUE_NUM_BITS == 32 +#define VALUE_BITMASK 0xffffffff +#else +#define VALUE_BITMASK ((1 << RTE_EFD_VALUE_NUM_BITS) - 1) +#endif +static unsigned int test_socket_id; + +static inline uint8_t efd_get_all_sockets_bitmask(void) +{ + uint8_t all_cpu_sockets_bitmask = 0; + unsigned int i; + unsigned int next_lcore = rte_get_master_lcore(); + const int val_true = 1, val_false = 0; + for (i = 0; i < rte_lcore_count(); i++) { + all_cpu_sockets_bitmask |= 1 << rte_lcore_to_socket_id(next_lcore); + next_lcore = rte_get_next_lcore(next_lcore, val_false, val_true); + } + + return all_cpu_sockets_bitmask; +} + +enum operations { + ADD = 0, + LOOKUP, + LOOKUP_MULTI, + DELETE, + NUM_OPERATIONS +}; + +struct efd_perf_params { + struct rte_efd_table *efd_table; + uint32_t key_size; + unsigned int cycle; +}; + +static uint32_t hashtest_key_lens[] = { + /* standard key sizes */ + 4, 8, 16, 32, 48, 64, + /* IPv4 SRC + DST + protocol, unpadded */ + 9, + /* IPv4 5-tuple, unpadded */ + 13, + /* IPv6 5-tuple, unpadded */ + 37, + /* IPv6 5-tuple, padded to 8-byte boundary */ + 40 +}; + +/* Array to store number of cycles per operation */ +uint64_t cycles[NUM_KEYSIZES][NUM_OPERATIONS]; + +/* Array to store the data */ +efd_value_t data[KEYS_TO_ADD]; + +/* Array to store all input keys */ +uint8_t keys[KEYS_TO_ADD][MAX_KEYSIZE]; + +/* Shuffle the keys that have been added, so lookups will be totally random */ +static void +shuffle_input_keys(struct efd_perf_params *params) +{ + efd_value_t temp_data; + unsigned int i; + uint32_t swap_idx; + uint8_t temp_key[MAX_KEYSIZE]; + + for (i = KEYS_TO_ADD - 1; i > 0; i--) { + swap_idx = rte_rand() % i; + + memcpy(temp_key, keys[i], hashtest_key_lens[params->cycle]); + temp_data = data[i]; + + memcpy(keys[i], keys[swap_idx], hashtest_key_lens[params->cycle]); + data[i] = data[swap_idx]; + + memcpy(keys[swap_idx], temp_key, hashtest_key_lens[params->cycle]); + data[swap_idx] = temp_data; + } +} + +static int key_compare(const void *key1, const void *key2) +{ + return memcmp(key1, key2, MAX_KEYSIZE); +} + +/* + * TODO: we could "error proof" these as done in test_hash_perf.c ln 165: + * + * The current setup may give errors if too full in some cases which we check + * for. However, since EFD allows for ~99% capacity, these errors are rare for + * #"KEYS_TO_ADD" which is 75% capacity. + */ +static int +setup_keys_and_data(struct efd_perf_params *params, unsigned int cycle) +{ + unsigned int i, j; + int num_duplicates; + + params->key_size = hashtest_key_lens[cycle]; + params->cycle = cycle; + + /* Reset all arrays */ + for (i = 0; i < params->key_size; i++) + keys[0][i] = 0; + + /* Generate a list of keys, some of which may be duplicates */ + for (i = 0; i < KEYS_TO_ADD; i++) { + for (j = 0; j < params->key_size; j++) + keys[i][j] = rte_rand() & 0xFF; + + data[i] = rte_rand() & VALUE_BITMASK; + } + + /* Remove duplicates from the keys array */ + do { + num_duplicates = 0; + + /* Sort the list of keys to make it easier to find duplicates */ + qsort(keys, KEYS_TO_ADD, MAX_KEYSIZE, key_compare); + + /* Sift through the list of keys and look for duplicates */ + int num_duplicates = 0; + for (i = 0; i < KEYS_TO_ADD - 1; i++) { + if (memcmp(keys[i], keys[i + 1], params->key_size) == 0) { + /* This key already exists, try again */ + num_duplicates++; + for (j = 0; j < params->key_size; j++) + keys[i][j] = rte_rand() & 0xFF; + } + } + } while (num_duplicates != 0); + + /* Shuffle the random values again */ + shuffle_input_keys(params); + + params->efd_table = rte_efd_create("test_efd_perf", + MAX_ENTRIES, params->key_size, + efd_get_all_sockets_bitmask(), test_socket_id); + TEST_ASSERT_NOT_NULL(params->efd_table, "Error creating the efd table\n"); + + return 0; +} + +static int +timed_adds(struct efd_perf_params *params) +{ + const uint64_t start_tsc = rte_rdtsc(); + unsigned int i, a; + int32_t ret; + + for (i = 0; i < KEYS_TO_ADD; i++) { + ret = rte_efd_update(params->efd_table, test_socket_id, keys[i], + data[i]); + if (ret != 0) { + printf("Error %d in rte_efd_update - key=0x", ret); + for (a = 0; a < params->key_size; a++) + printf("%02x", keys[i][a]); + printf(" value=%d\n", data[i]); + + return -1; + } + } + + const uint64_t end_tsc = rte_rdtsc(); + const uint64_t time_taken = end_tsc - start_tsc; + + cycles[params->cycle][ADD] = time_taken / KEYS_TO_ADD; + return 0; +} + +static int +timed_lookups(struct efd_perf_params *params) +{ + unsigned int i, j, a; + const uint64_t start_tsc = rte_rdtsc(); + efd_value_t ret_data; + + for (i = 0; i < NUM_LOOKUPS / KEYS_TO_ADD; i++) { + for (j = 0; j < KEYS_TO_ADD; j++) { + ret_data = rte_efd_lookup(params->efd_table, + test_socket_id, keys[j]); + if (ret_data != data[j]) { + printf("Value mismatch using rte_efd_lookup: " + "key #%d (0x", i); + for (a = 0; a < params->key_size; a++) + printf("%02x", keys[i][a]); + printf(")\n"); + printf(" Expected %d, got %d\n", data[i], + ret_data); + + return -1; + } + + } + } + + const uint64_t end_tsc = rte_rdtsc(); + const uint64_t time_taken = end_tsc - start_tsc; + + cycles[params->cycle][LOOKUP] = time_taken / NUM_LOOKUPS; + + return 0; +} + +static int +timed_lookups_multi(struct efd_perf_params *params) +{ + unsigned int i, j, k, a; + efd_value_t result[RTE_EFD_BURST_MAX] = {0}; + const void *keys_burst[RTE_EFD_BURST_MAX]; + const uint64_t start_tsc = rte_rdtsc(); + + for (i = 0; i < NUM_LOOKUPS / KEYS_TO_ADD; i++) { + for (j = 0; j < KEYS_TO_ADD / RTE_EFD_BURST_MAX; j++) { + for (k = 0; k < RTE_EFD_BURST_MAX; k++) + keys_burst[k] = keys[j * RTE_EFD_BURST_MAX + k]; + + rte_efd_lookup_bulk(params->efd_table, test_socket_id, + RTE_EFD_BURST_MAX, + keys_burst, result); + + for (k = 0; k < RTE_EFD_BURST_MAX; k++) { + uint32_t data_idx = j * RTE_EFD_BURST_MAX + k; + if (result[k] != data[data_idx]) { + printf("Value mismatch using " + "rte_efd_lookup_bulk: key #%d " + "(0x", i); + for (a = 0; a < params->key_size; a++) + printf("%02x", + keys[data_idx][a]); + printf(")\n"); + printf(" Expected %d, got %d\n", + data[data_idx], result[k]); + + return -1; + } + } + } + } + + const uint64_t end_tsc = rte_rdtsc(); + const uint64_t time_taken = end_tsc - start_tsc; + + cycles[params->cycle][LOOKUP_MULTI] = time_taken / NUM_LOOKUPS; + + return 0; +} + +static int +timed_deletes(struct efd_perf_params *params) +{ + unsigned int i, a; + const uint64_t start_tsc = rte_rdtsc(); + int32_t ret; + + for (i = 0; i < KEYS_TO_ADD; i++) { + ret = rte_efd_delete(params->efd_table, test_socket_id, keys[i], + NULL); + + if (ret != 0) { + printf("Error %d in rte_efd_delete - key=0x", ret); + for (a = 0; a < params->key_size; a++) + printf("%02x", keys[i][a]); + printf("\n"); + + return -1; + } + } + + const uint64_t end_tsc = rte_rdtsc(); + const uint64_t time_taken = end_tsc - start_tsc; + + cycles[params->cycle][DELETE] = time_taken / KEYS_TO_ADD; + + return 0; +} + +static void +perform_frees(struct efd_perf_params *params) +{ + if (params->efd_table != NULL) { + rte_efd_free(params->efd_table); + params->efd_table = NULL; + } +} + +static int +exit_with_fail(const char *testname, struct efd_perf_params *params, + unsigned int i) +{ + + printf("<<<<<Test %s failed at keysize %d iteration %d >>>>>\n", + testname, hashtest_key_lens[params->cycle], i); + perform_frees(params); + return -1; +} + +static int +run_all_tbl_perf_tests(void) +{ + unsigned int i, j; + struct efd_perf_params params; + + printf("Measuring performance, please wait\n"); + fflush(stdout); + + test_socket_id = rte_socket_id(); + + for (i = 0; i < NUM_KEYSIZES; i++) { + + if (setup_keys_and_data(¶ms, i) < 0) { + printf("Could not create keys/data/table\n"); + return -1; + } + + if (timed_adds(¶ms) < 0) + return exit_with_fail("timed_adds", ¶ms, i); + + for (j = 0; j < NUM_SHUFFLES; j++) + shuffle_input_keys(¶ms); + + if (timed_lookups(¶ms) < 0) + return exit_with_fail("timed_lookups", ¶ms, i); + + if (timed_lookups_multi(¶ms) < 0) + return exit_with_fail("timed_lookups_multi", ¶ms, i); + + if (timed_deletes(¶ms) < 0) + return exit_with_fail("timed_deletes", ¶ms, i); + + /* Print a dot to show progress on operations */ + printf("."); + fflush(stdout); + + perform_frees(¶ms); + } + + printf("\nResults (in CPU cycles/operation)\n"); + printf("-----------------------------------\n"); + printf("\n%-18s%-18s%-18s%-18s%-18s\n", + "Keysize", "Add", "Lookup", "Lookup_bulk", "Delete"); + for (i = 0; i < NUM_KEYSIZES; i++) { + printf("%-18d", hashtest_key_lens[i]); + for (j = 0; j < NUM_OPERATIONS; j++) + printf("%-18"PRIu64, cycles[i][j]); + printf("\n"); + } + return 0; +} + +static int +test_efd_perf(void) +{ + + if (run_all_tbl_perf_tests() < 0) + return -1; + + return 0; +} + +REGISTER_TEST_COMMAND(efd_perf_autotest, test_efd_perf); diff --git a/dpdk/test/test/test_errno.c b/dpdk/test/test/test_errno.c new file mode 100644 index 00000000..388decbb --- /dev/null +++ b/dpdk/test/test/test_errno.c @@ -0,0 +1,116 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdint.h> +#include <stdio.h> +#include <stdarg.h> +#include <errno.h> +#include <string.h> +#include <rte_per_lcore.h> +#include <rte_errno.h> +#include <rte_string_fns.h> + +#include "test.h" + +static int +test_errno(void) +{ + const char *rte_retval; + const char *libc_retval; +#ifdef RTE_EXEC_ENV_BSDAPP + /* BSD has a colon in the string, unlike linux */ + const char unknown_code_result[] = "Unknown error: %d"; +#else + const char unknown_code_result[] = "Unknown error %d"; +#endif + char expected_libc_retval[sizeof(unknown_code_result)+3]; + + /* use a small selection of standard errors for testing */ + int std_errs[] = {EAGAIN, EBADF, EACCES, EINTR, EINVAL}; + /* test ALL registered RTE error codes for overlap */ + int rte_errs[] = {E_RTE_SECONDARY, E_RTE_NO_CONFIG}; + unsigned i; + + rte_errno = 0; + if (rte_errno != 0) + return -1; + /* check for standard errors we return the same as libc */ + for (i = 0; i < sizeof(std_errs)/sizeof(std_errs[0]); i++){ + rte_retval = rte_strerror(std_errs[i]); + libc_retval = strerror(std_errs[i]); + printf("rte_strerror: '%s', strerror: '%s'\n", + rte_retval, libc_retval); + if (strcmp(rte_retval, libc_retval) != 0) + return -1; + } + /* for rte-specific errors ensure we return a different string + * and that the string for libc is for an unknown error + */ + for (i = 0; i < sizeof(rte_errs)/sizeof(rte_errs[0]); i++){ + rte_retval = rte_strerror(rte_errs[i]); + libc_retval = strerror(rte_errs[i]); + printf("rte_strerror: '%s', strerror: '%s'\n", + rte_retval, libc_retval); + if (strcmp(rte_retval, libc_retval) == 0) + return -1; + /* generate appropriate error string for unknown error number + * and then check that this is what we got back. If not, we have + * a duplicate error number that conflicts with errno.h */ + snprintf(expected_libc_retval, sizeof(expected_libc_retval), + unknown_code_result, rte_errs[i]); + if ((strcmp(expected_libc_retval, libc_retval) != 0) && + (strcmp("", libc_retval) != 0)){ + printf("Error, duplicate error code %d\n", rte_errs[i]); + return -1; + } + } + + /* ensure that beyond RTE_MAX_ERRNO, we always get an unknown code */ + rte_retval = rte_strerror(RTE_MAX_ERRNO + 1); + libc_retval = strerror(RTE_MAX_ERRNO + 1); + snprintf(expected_libc_retval, sizeof(expected_libc_retval), + unknown_code_result, RTE_MAX_ERRNO + 1); + printf("rte_strerror: '%s', strerror: '%s'\n", + rte_retval, libc_retval); + if ((strcmp(rte_retval, libc_retval) != 0) || + (strcmp(expected_libc_retval, libc_retval) != 0)){ + if (strcmp("", libc_retval) != 0){ + printf("Failed test for RTE_MAX_ERRNO + 1 value\n"); + return -1; + } + } + + return 0; +} + +REGISTER_TEST_COMMAND(errno_autotest, test_errno); diff --git a/dpdk/test/test/test_event_eth_rx_adapter.c b/dpdk/test/test/test_event_eth_rx_adapter.c new file mode 100644 index 00000000..90a5c646 --- /dev/null +++ b/dpdk/test/test/test_event_eth_rx_adapter.c @@ -0,0 +1,454 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include <string.h> +#include <rte_common.h> +#include <rte_mempool.h> +#include <rte_mbuf.h> +#include <rte_ethdev.h> +#include <rte_eventdev.h> +#include <rte_bus_vdev.h> + +#include <rte_event_eth_rx_adapter.h> + +#include "test.h" + +#define MAX_NUM_RX_QUEUE 64 +#define NB_MBUFS (8192 * num_ports * MAX_NUM_RX_QUEUE) +#define MBUF_CACHE_SIZE 512 +#define MBUF_PRIV_SIZE 0 +#define TEST_INST_ID 0 +#define TEST_DEV_ID 0 +#define TEST_ETHDEV_ID 0 + +struct event_eth_rx_adapter_test_params { + struct rte_mempool *mp; + uint16_t rx_rings, tx_rings; + uint32_t caps; +}; + +static struct event_eth_rx_adapter_test_params default_params; + +static inline int +port_init(uint8_t port, struct rte_mempool *mp) +{ + static const struct rte_eth_conf port_conf_default = { + .rxmode = { + .mq_mode = ETH_MQ_RX_RSS, + .max_rx_pkt_len = ETHER_MAX_LEN + }, + .rx_adv_conf = { + .rss_conf = { + .rss_hf = ETH_RSS_IP | + ETH_RSS_TCP | + ETH_RSS_UDP, + } + } + }; + const uint16_t rx_ring_size = 512, tx_ring_size = 512; + struct rte_eth_conf port_conf = port_conf_default; + int retval; + uint16_t q; + struct rte_eth_dev_info dev_info; + + if (port >= rte_eth_dev_count()) + return -1; + + retval = rte_eth_dev_configure(port, 0, 0, &port_conf); + + rte_eth_dev_info_get(port, &dev_info); + + default_params.rx_rings = RTE_MIN(dev_info.max_rx_queues, + MAX_NUM_RX_QUEUE); + default_params.tx_rings = 1; + + /* Configure the Ethernet device. */ + retval = rte_eth_dev_configure(port, default_params.rx_rings, + default_params.tx_rings, &port_conf); + if (retval != 0) + return retval; + + for (q = 0; q < default_params.rx_rings; q++) { + retval = rte_eth_rx_queue_setup(port, q, rx_ring_size, + rte_eth_dev_socket_id(port), NULL, mp); + if (retval < 0) + return retval; + } + + /* Allocate and set up 1 TX queue per Ethernet port. */ + for (q = 0; q < default_params.tx_rings; q++) { + retval = rte_eth_tx_queue_setup(port, q, tx_ring_size, + rte_eth_dev_socket_id(port), NULL); + if (retval < 0) + return retval; + } + + /* Start the Ethernet port. */ + retval = rte_eth_dev_start(port); + if (retval < 0) + return retval; + + /* Display the port MAC address. */ + struct ether_addr addr; + rte_eth_macaddr_get(port, &addr); + printf("Port %u MAC: %02" PRIx8 " %02" PRIx8 " %02" PRIx8 + " %02" PRIx8 " %02" PRIx8 " %02" PRIx8 "\n", + (unsigned int)port, + addr.addr_bytes[0], addr.addr_bytes[1], + addr.addr_bytes[2], addr.addr_bytes[3], + addr.addr_bytes[4], addr.addr_bytes[5]); + + /* Enable RX in promiscuous mode for the Ethernet device. */ + rte_eth_promiscuous_enable(port); + + return 0; +} + +static int +init_ports(int num_ports) +{ + uint8_t portid; + int retval; + + default_params.mp = rte_pktmbuf_pool_create("packet_pool", + NB_MBUFS, + MBUF_CACHE_SIZE, + MBUF_PRIV_SIZE, + RTE_MBUF_DEFAULT_BUF_SIZE, + rte_socket_id()); + if (!default_params.mp) + return -ENOMEM; + + for (portid = 0; portid < num_ports; portid++) { + retval = port_init(portid, default_params.mp); + if (retval) + return retval; + } + + return 0; +} + +static int +testsuite_setup(void) +{ + int err; + uint8_t count; + struct rte_event_dev_info dev_info; + + count = rte_event_dev_count(); + if (!count) { + printf("Failed to find a valid event device," + " testing with event_skeleton device\n"); + rte_vdev_init("event_skeleton", NULL); + } + + struct rte_event_dev_config config = { + .nb_event_queues = 1, + .nb_event_ports = 1, + }; + + err = rte_event_dev_info_get(TEST_DEV_ID, &dev_info); + config.nb_event_queue_flows = dev_info.max_event_queue_flows; + config.nb_event_port_dequeue_depth = + dev_info.max_event_port_dequeue_depth; + config.nb_event_port_enqueue_depth = + dev_info.max_event_port_enqueue_depth; + config.nb_events_limit = + dev_info.max_num_events; + err = rte_event_dev_configure(TEST_DEV_ID, &config); + TEST_ASSERT(err == 0, "Event device initialization failed err %d\n", + err); + + /* + * eth devices like octeontx use event device to receive packets + * so rte_eth_dev_start invokes rte_event_dev_start internally, so + * call init_ports after rte_event_dev_configure + */ + err = init_ports(rte_eth_dev_count()); + TEST_ASSERT(err == 0, "Port initialization failed err %d\n", err); + + err = rte_event_eth_rx_adapter_caps_get(TEST_DEV_ID, TEST_ETHDEV_ID, + &default_params.caps); + TEST_ASSERT(err == 0, "Failed to get adapter cap err %d\n", + err); + + return err; +} + +static void +testsuite_teardown(void) +{ + uint32_t i; + for (i = 0; i < rte_eth_dev_count(); i++) + rte_eth_dev_stop(i); + + rte_mempool_free(default_params.mp); +} + +static int +adapter_create(void) +{ + int err; + struct rte_event_dev_info dev_info; + struct rte_event_port_conf rx_p_conf; + + err = rte_event_dev_info_get(TEST_DEV_ID, &dev_info); + TEST_ASSERT(err == 0, "Expected 0 got %d", err); + + rx_p_conf.new_event_threshold = dev_info.max_num_events; + rx_p_conf.dequeue_depth = dev_info.max_event_port_dequeue_depth; + rx_p_conf.enqueue_depth = dev_info.max_event_port_enqueue_depth; + err = rte_event_eth_rx_adapter_create(TEST_INST_ID, TEST_DEV_ID, + &rx_p_conf); + TEST_ASSERT(err == 0, "Expected 0 got %d", err); + + return err; +} + +static void +adapter_free(void) +{ + rte_event_eth_rx_adapter_free(TEST_INST_ID); +} + +static int +adapter_create_free(void) +{ + int err; + + struct rte_event_port_conf rx_p_conf = { + .dequeue_depth = 8, + .enqueue_depth = 8, + .new_event_threshold = 1200, + }; + + err = rte_event_eth_rx_adapter_create(TEST_INST_ID, TEST_DEV_ID, + NULL); + TEST_ASSERT(err == -EINVAL, "Expected -EINVAL got %d", err); + + err = rte_event_eth_rx_adapter_create(TEST_INST_ID, TEST_DEV_ID, + &rx_p_conf); + TEST_ASSERT(err == 0, "Expected 0 got %d", err); + + err = rte_event_eth_rx_adapter_create(TEST_INST_ID, + TEST_DEV_ID, &rx_p_conf); + TEST_ASSERT(err == -EEXIST, "Expected -EEXIST %d got %d", -EEXIST, err); + + err = rte_event_eth_rx_adapter_free(TEST_INST_ID); + TEST_ASSERT(err == 0, "Expected 0 got %d", err); + + err = rte_event_eth_rx_adapter_free(TEST_INST_ID); + TEST_ASSERT(err == -EINVAL, "Expected -EINVAL %d got %d", -EINVAL, err); + + err = rte_event_eth_rx_adapter_free(1); + TEST_ASSERT(err == -EINVAL, "Expected -EINVAL %d got %d", -EINVAL, err); + + return TEST_SUCCESS; +} + +static int +adapter_queue_add_del(void) +{ + int err; + struct rte_event ev; + uint32_t cap; + + struct rte_event_eth_rx_adapter_queue_conf queue_config; + + err = rte_event_eth_rx_adapter_caps_get(TEST_DEV_ID, TEST_ETHDEV_ID, + &cap); + TEST_ASSERT(err == 0, "Expected 0 got %d", err); + + ev.queue_id = 0; + ev.sched_type = RTE_SCHED_TYPE_ATOMIC; + ev.priority = 0; + + queue_config.rx_queue_flags = 0; + if (cap & RTE_EVENT_ETH_RX_ADAPTER_CAP_OVERRIDE_FLOW_ID) { + ev.flow_id = 1; + queue_config.rx_queue_flags = + RTE_EVENT_ETH_RX_ADAPTER_QUEUE_FLOW_ID_VALID; + } + queue_config.ev = ev; + queue_config.servicing_weight = 1; + + err = rte_event_eth_rx_adapter_queue_add(TEST_INST_ID, + rte_eth_dev_count(), + -1, &queue_config); + TEST_ASSERT(err == -EINVAL, "Expected -EINVAL got %d", err); + + if (cap & RTE_EVENT_ETH_RX_ADAPTER_CAP_MULTI_EVENTQ) { + err = rte_event_eth_rx_adapter_queue_add(TEST_INST_ID, + TEST_ETHDEV_ID, 0, + &queue_config); + TEST_ASSERT(err == 0, "Expected 0 got %d", err); + + err = rte_event_eth_rx_adapter_queue_del(TEST_INST_ID, + TEST_ETHDEV_ID, 0); + TEST_ASSERT(err == 0, "Expected 0 got %d", err); + + err = rte_event_eth_rx_adapter_queue_add(TEST_INST_ID, + TEST_ETHDEV_ID, + -1, + &queue_config); + TEST_ASSERT(err == 0, "Expected 0 got %d", err); + + err = rte_event_eth_rx_adapter_queue_del(TEST_INST_ID, + TEST_ETHDEV_ID, + -1); + TEST_ASSERT(err == 0, "Expected 0 got %d", err); + } else { + err = rte_event_eth_rx_adapter_queue_add(TEST_INST_ID, + TEST_ETHDEV_ID, + 0, + &queue_config); + TEST_ASSERT(err == -EINVAL, "Expected EINVAL got %d", err); + + err = rte_event_eth_rx_adapter_queue_add(TEST_INST_ID, + TEST_ETHDEV_ID, -1, + &queue_config); + TEST_ASSERT(err == 0, "Expected 0 got %d", err); + + err = rte_event_eth_rx_adapter_queue_del(TEST_INST_ID, + TEST_ETHDEV_ID, 0); + TEST_ASSERT(err == 0, "Expected 0 got %d", err); + + err = rte_event_eth_rx_adapter_queue_del(TEST_INST_ID, + TEST_ETHDEV_ID, -1); + TEST_ASSERT(err == 0, "Expected 0 got %d", err); + + err = rte_event_eth_rx_adapter_queue_del(TEST_INST_ID, + TEST_ETHDEV_ID, -1); + TEST_ASSERT(err == 0, "Expected 0 got %d", err); + } + + err = rte_event_eth_rx_adapter_queue_add(1, TEST_ETHDEV_ID, -1, + &queue_config); + TEST_ASSERT(err == -EINVAL, "Expected -EINVAL got %d", err); + + err = rte_event_eth_rx_adapter_queue_del(1, TEST_ETHDEV_ID, -1); + TEST_ASSERT(err == -EINVAL, "Expected -EINVAL got %d", err); + + return TEST_SUCCESS; +} + +static int +adapter_start_stop(void) +{ + int err; + struct rte_event ev; + + ev.queue_id = 0; + ev.sched_type = RTE_SCHED_TYPE_ATOMIC; + ev.priority = 0; + + struct rte_event_eth_rx_adapter_queue_conf queue_config; + + queue_config.rx_queue_flags = 0; + if (default_params.caps & + RTE_EVENT_ETH_RX_ADAPTER_CAP_OVERRIDE_FLOW_ID) { + ev.flow_id = 1; + queue_config.rx_queue_flags = + RTE_EVENT_ETH_RX_ADAPTER_QUEUE_FLOW_ID_VALID; + } + + queue_config.ev = ev; + queue_config.servicing_weight = 1; + + err = rte_event_eth_rx_adapter_queue_add(TEST_INST_ID, TEST_ETHDEV_ID, + -1, &queue_config); + TEST_ASSERT(err == 0, "Expected 0 got %d", err); + + err = rte_event_eth_rx_adapter_start(TEST_INST_ID); + TEST_ASSERT(err == 0, "Expected 0 got %d", err); + + err = rte_event_eth_rx_adapter_stop(TEST_INST_ID); + TEST_ASSERT(err == 0, "Expected 0 got %d", err); + + err = rte_event_eth_rx_adapter_queue_del(TEST_INST_ID, TEST_ETHDEV_ID, + -1); + TEST_ASSERT(err == 0, "Expected 0 got %d", err); + + err = rte_event_eth_rx_adapter_start(TEST_INST_ID); + TEST_ASSERT(err == 0, "Expected 0 got %d", err); + + err = rte_event_eth_rx_adapter_stop(TEST_INST_ID); + TEST_ASSERT(err == 0, "Expected 0 got %d", err); + + err = rte_event_eth_rx_adapter_start(1); + TEST_ASSERT(err == -EINVAL, "Expected -EINVAL got %d", err); + + err = rte_event_eth_rx_adapter_stop(1); + TEST_ASSERT(err == -EINVAL, "Expected -EINVAL got %d", err); + + return TEST_SUCCESS; +} + +static int +adapter_stats(void) +{ + int err; + struct rte_event_eth_rx_adapter_stats stats; + + err = rte_event_eth_rx_adapter_stats_get(TEST_INST_ID, NULL); + TEST_ASSERT(err == -EINVAL, "Expected -EINVAL got %d", err); + + err = rte_event_eth_rx_adapter_stats_get(TEST_INST_ID, &stats); + TEST_ASSERT(err == 0, "Expected 0 got %d", err); + + err = rte_event_eth_rx_adapter_stats_get(1, &stats); + TEST_ASSERT(err == -EINVAL, "Expected -EINVAL got %d", err); + + return TEST_SUCCESS; +} + +static struct unit_test_suite service_tests = { + .suite_name = "rx event eth adapter test suite", + .setup = testsuite_setup, + .teardown = testsuite_teardown, + .unit_test_cases = { + TEST_CASE_ST(NULL, NULL, adapter_create_free), + TEST_CASE_ST(adapter_create, adapter_free, + adapter_queue_add_del), + TEST_CASE_ST(adapter_create, adapter_free, adapter_start_stop), + TEST_CASE_ST(adapter_create, adapter_free, adapter_stats), + TEST_CASES_END() /**< NULL terminate unit test array */ + } +}; + +static int +test_event_eth_rx_adapter_common(void) +{ + return unit_test_suite_runner(&service_tests); +} + +REGISTER_TEST_COMMAND(event_eth_rx_adapter_autotest, + test_event_eth_rx_adapter_common); diff --git a/dpdk/test/test/test_event_ring.c b/dpdk/test/test/test_event_ring.c new file mode 100644 index 00000000..c158c9b0 --- /dev/null +++ b/dpdk/test/test/test_event_ring.c @@ -0,0 +1,276 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <string.h> + +#include <rte_event_ring.h> + +#include "test.h" + +/* + * Event Ring + * =========== + * + * Test some basic ops for the event rings. + * Does not fully test everything, since most code is reused from rte_ring + * library and tested as part of the normal ring autotests. + */ + +#define RING_SIZE 4096 +#define MAX_BULK 32 + +static struct rte_event_ring *r; + +/* + * ensure failure to create ring with a bad ring size + */ +static int +test_event_ring_creation_with_wrong_size(void) +{ + struct rte_event_ring *rp = NULL; + + /* Test if ring size is not power of 2 */ + rp = rte_event_ring_create("test_bad_ring_size", RING_SIZE + 1, + SOCKET_ID_ANY, 0); + if (rp != NULL) + return -1; + + /* Test if ring size is exceeding the limit */ + rp = rte_event_ring_create("test_bad_ring_size", (RTE_RING_SZ_MASK + 1), + SOCKET_ID_ANY, 0); + if (rp != NULL) + return -1; + return 0; +} + +/* + * Test to check if a non-power-of-2 count causes the create + * function to fail correctly + */ +static int +test_create_count_odd(void) +{ + struct rte_event_ring *r = rte_event_ring_create("test_event_ring_count", + 4097, SOCKET_ID_ANY, 0); + if (r != NULL) + return -1; + return 0; +} + +static int +test_lookup_null(void) +{ + struct rte_event_ring *rlp = rte_event_ring_lookup("ring_not_found"); + if (rlp == NULL && rte_errno != ENOENT) { + printf("test failed to return error on null pointer\n"); + return -1; + } + return 0; +} + +static int +test_basic_event_enqueue_dequeue(void) +{ + struct rte_event_ring *sr = NULL; + struct rte_event evs[16]; + uint16_t ret, free_count, used_count; + + memset(evs, 0, sizeof(evs)); + sr = rte_event_ring_create("spsc_ring", 32, rte_socket_id(), + RING_F_SP_ENQ | RING_F_SC_DEQ); + if (sr == NULL) { + printf("Failed to create sp/sc ring\n"); + return -1; + } + if (rte_event_ring_get_capacity(sr) != 31) { + printf("Error, invalid capacity\n"); + goto error; + } + + /* test sp/sc ring */ + if (rte_event_ring_count(sr) != 0) { + printf("Error, ring not empty as expected\n"); + goto error; + } + if (rte_event_ring_free_count(sr) != rte_event_ring_get_capacity(sr)) { + printf("Error, ring free count not as expected\n"); + goto error; + } + + ret = rte_event_ring_enqueue_burst(sr, evs, RTE_DIM(evs), &free_count); + if (ret != RTE_DIM(evs) || + free_count != rte_event_ring_get_capacity(sr) - ret) { + printf("Error, status after enqueue is unexpected\n"); + goto error; + } + + ret = rte_event_ring_enqueue_burst(sr, evs, RTE_DIM(evs), &free_count); + if (ret != RTE_DIM(evs) - 1 || + free_count != 0) { + printf("Error, status after enqueue is unexpected\n"); + goto error; + } + + ret = rte_event_ring_dequeue_burst(sr, evs, RTE_DIM(evs), &used_count); + if (ret != RTE_DIM(evs) || + used_count != rte_event_ring_get_capacity(sr) - ret) { + printf("Error, status after enqueue is unexpected\n"); + goto error; + } + ret = rte_event_ring_dequeue_burst(sr, evs, RTE_DIM(evs), &used_count); + if (ret != RTE_DIM(evs) - 1 || + used_count != 0) { + printf("Error, status after enqueue is unexpected\n"); + goto error; + } + + rte_event_ring_free(sr); + return 0; +error: + rte_event_ring_free(sr); + return -1; +} + +static int +test_event_ring_with_exact_size(void) +{ + struct rte_event_ring *std_ring, *exact_sz_ring; + struct rte_event ev = { .mbuf = NULL }; + struct rte_event ev_array[16]; + static const unsigned int ring_sz = RTE_DIM(ev_array); + unsigned int i; + + std_ring = rte_event_ring_create("std", ring_sz, rte_socket_id(), + RING_F_SP_ENQ | RING_F_SC_DEQ); + if (std_ring == NULL) { + printf("%s: error, can't create std ring\n", __func__); + return -1; + } + exact_sz_ring = rte_event_ring_create("exact sz", + ring_sz, rte_socket_id(), + RING_F_SP_ENQ | RING_F_SC_DEQ | RING_F_EXACT_SZ); + if (exact_sz_ring == NULL) { + printf("%s: error, can't create exact size ring\n", __func__); + return -1; + } + + /* + * Check that the exact size ring is bigger than the standard ring + */ + if (rte_event_ring_get_size(std_ring) >= + rte_event_ring_get_size(exact_sz_ring)) { + printf("%s: error, std ring (size: %u) is not smaller than exact size one (size %u)\n", + __func__, + rte_event_ring_get_size(std_ring), + rte_event_ring_get_size(exact_sz_ring)); + return -1; + } + /* + * check that the exact_sz_ring can hold one more element than the + * standard ring. (16 vs 15 elements) + */ + for (i = 0; i < ring_sz - 1; i++) { + rte_event_ring_enqueue_burst(std_ring, &ev, 1, NULL); + rte_event_ring_enqueue_burst(exact_sz_ring, &ev, 1, NULL); + } + if (rte_event_ring_enqueue_burst(std_ring, &ev, 1, NULL) != 0) { + printf("%s: error, unexpected successful enqueue\n", __func__); + return -1; + } + if (rte_event_ring_enqueue_burst(exact_sz_ring, &ev, 1, NULL) != 1) { + printf("%s: error, enqueue failed\n", __func__); + return -1; + } + + /* check that dequeue returns the expected number of elements */ + if (rte_event_ring_dequeue_burst(exact_sz_ring, ev_array, + RTE_DIM(ev_array), NULL) != ring_sz) { + printf("%s: error, failed to dequeue expected nb of elements\n", + __func__); + return -1; + } + + /* check that the capacity function returns expected value */ + if (rte_event_ring_get_capacity(exact_sz_ring) != ring_sz) { + printf("%s: error, incorrect ring capacity reported\n", + __func__); + return -1; + } + + rte_event_ring_free(std_ring); + rte_event_ring_free(exact_sz_ring); + return 0; +} + +static int +test_event_ring(void) +{ + if (r == NULL) + r = rte_event_ring_create("ev_test", RING_SIZE, + SOCKET_ID_ANY, 0); + if (r == NULL) + return -1; + + /* retrieve the ring from its name */ + if (rte_event_ring_lookup("ev_test") != r) { + printf("Cannot lookup ring from its name\n"); + return -1; + } + + /* basic operations */ + if (test_create_count_odd() < 0) { + printf("Test failed to detect odd count\n"); + return -1; + } + printf("Test detected odd count\n"); + + if (test_lookup_null() < 0) { + printf("Test failed to detect NULL ring lookup\n"); + return -1; + } + printf("Test detected NULL ring lookup\n"); + + /* test of creating ring with wrong size */ + if (test_event_ring_creation_with_wrong_size() < 0) + return -1; + + if (test_basic_event_enqueue_dequeue() < 0) + return -1; + + if (test_event_ring_with_exact_size() < 0) + return -1; + + return 0; +} + +REGISTER_TEST_COMMAND(event_ring_autotest, test_event_ring); diff --git a/dpdk/test/test/test_eventdev.c b/dpdk/test/test/test_eventdev.c new file mode 100644 index 00000000..1ed2a1dd --- /dev/null +++ b/dpdk/test/test/test_eventdev.c @@ -0,0 +1,1012 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016 Cavium, Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <rte_common.h> +#include <rte_hexdump.h> +#include <rte_mbuf.h> +#include <rte_malloc.h> +#include <rte_memcpy.h> +#include <rte_eventdev.h> +#include <rte_dev.h> +#include <rte_bus_vdev.h> + +#include "test.h" + +#define TEST_DEV_ID 0 + +static int +testsuite_setup(void) +{ + RTE_BUILD_BUG_ON(sizeof(struct rte_event) != 16); + uint8_t count; + count = rte_event_dev_count(); + if (!count) { + printf("Failed to find a valid event device," + " testing with event_skeleton device\n"); + return rte_vdev_init("event_skeleton", NULL); + } + return TEST_SUCCESS; +} + +static void +testsuite_teardown(void) +{ +} + +static int +test_eventdev_count(void) +{ + uint8_t count; + count = rte_event_dev_count(); + TEST_ASSERT(count > 0, "Invalid eventdev count %" PRIu8, count); + return TEST_SUCCESS; +} + +static int +test_eventdev_get_dev_id(void) +{ + int ret; + ret = rte_event_dev_get_dev_id("not_a_valid_eventdev_driver"); + TEST_ASSERT_FAIL(ret, "Expected <0 for invalid dev name ret=%d", ret); + return TEST_SUCCESS; +} + +static int +test_eventdev_socket_id(void) +{ + int socket_id; + socket_id = rte_event_dev_socket_id(TEST_DEV_ID); + TEST_ASSERT(socket_id != -EINVAL, "Failed to get socket_id %d", + socket_id); + socket_id = rte_event_dev_socket_id(RTE_EVENT_MAX_DEVS); + TEST_ASSERT(socket_id == -EINVAL, "Expected -EINVAL %d", socket_id); + + return TEST_SUCCESS; +} + +static int +test_eventdev_info_get(void) +{ + int ret; + struct rte_event_dev_info info; + ret = rte_event_dev_info_get(TEST_DEV_ID, NULL); + TEST_ASSERT(ret == -EINVAL, "Expected -EINVAL, %d", ret); + ret = rte_event_dev_info_get(TEST_DEV_ID, &info); + TEST_ASSERT_SUCCESS(ret, "Failed to get event dev info"); + TEST_ASSERT(info.max_event_ports > 0, + "Not enough event ports %d", info.max_event_ports); + TEST_ASSERT(info.max_event_queues > 0, + "Not enough event queues %d", info.max_event_queues); + return TEST_SUCCESS; +} + +static inline void +devconf_set_default_sane_values(struct rte_event_dev_config *dev_conf, + struct rte_event_dev_info *info) +{ + memset(dev_conf, 0, sizeof(struct rte_event_dev_config)); + dev_conf->dequeue_timeout_ns = info->min_dequeue_timeout_ns; + dev_conf->nb_event_ports = info->max_event_ports; + dev_conf->nb_event_queues = info->max_event_queues; + dev_conf->nb_event_queue_flows = info->max_event_queue_flows; + dev_conf->nb_event_port_dequeue_depth = + info->max_event_port_dequeue_depth; + dev_conf->nb_event_port_enqueue_depth = + info->max_event_port_enqueue_depth; + dev_conf->nb_event_port_enqueue_depth = + info->max_event_port_enqueue_depth; + dev_conf->nb_events_limit = + info->max_num_events; +} + +static int +test_ethdev_config_run(struct rte_event_dev_config *dev_conf, + struct rte_event_dev_info *info, + void (*fn)(struct rte_event_dev_config *dev_conf, + struct rte_event_dev_info *info)) +{ + devconf_set_default_sane_values(dev_conf, info); + fn(dev_conf, info); + return rte_event_dev_configure(TEST_DEV_ID, dev_conf); +} + +static void +max_dequeue_limit(struct rte_event_dev_config *dev_conf, + struct rte_event_dev_info *info) +{ + dev_conf->dequeue_timeout_ns = info->max_dequeue_timeout_ns + 1; +} + +static void +max_events_limit(struct rte_event_dev_config *dev_conf, + struct rte_event_dev_info *info) +{ + dev_conf->nb_events_limit = info->max_num_events + 1; +} + +static void +max_event_ports(struct rte_event_dev_config *dev_conf, + struct rte_event_dev_info *info) +{ + dev_conf->nb_event_ports = info->max_event_ports + 1; +} + +static void +max_event_queues(struct rte_event_dev_config *dev_conf, + struct rte_event_dev_info *info) +{ + dev_conf->nb_event_queues = info->max_event_queues + 1; +} + +static void +max_event_queue_flows(struct rte_event_dev_config *dev_conf, + struct rte_event_dev_info *info) +{ + dev_conf->nb_event_queue_flows = info->max_event_queue_flows + 1; +} + +static void +max_event_port_dequeue_depth(struct rte_event_dev_config *dev_conf, + struct rte_event_dev_info *info) +{ + dev_conf->nb_event_port_dequeue_depth = + info->max_event_port_dequeue_depth + 1; +} + +static void +max_event_port_enqueue_depth(struct rte_event_dev_config *dev_conf, + struct rte_event_dev_info *info) +{ + dev_conf->nb_event_port_enqueue_depth = + info->max_event_port_enqueue_depth + 1; +} + + +static int +test_eventdev_configure(void) +{ + int ret; + struct rte_event_dev_config dev_conf; + struct rte_event_dev_info info; + ret = rte_event_dev_configure(TEST_DEV_ID, NULL); + TEST_ASSERT(ret == -EINVAL, "Expected -EINVAL, %d", ret); + + ret = rte_event_dev_info_get(TEST_DEV_ID, &info); + TEST_ASSERT_SUCCESS(ret, "Failed to get event dev info"); + + /* Check limits */ + TEST_ASSERT_EQUAL(-EINVAL, + test_ethdev_config_run(&dev_conf, &info, max_dequeue_limit), + "Config negative test failed"); + TEST_ASSERT_EQUAL(-EINVAL, + test_ethdev_config_run(&dev_conf, &info, max_events_limit), + "Config negative test failed"); + TEST_ASSERT_EQUAL(-EINVAL, + test_ethdev_config_run(&dev_conf, &info, max_event_ports), + "Config negative test failed"); + TEST_ASSERT_EQUAL(-EINVAL, + test_ethdev_config_run(&dev_conf, &info, max_event_queues), + "Config negative test failed"); + TEST_ASSERT_EQUAL(-EINVAL, + test_ethdev_config_run(&dev_conf, &info, max_event_queue_flows), + "Config negative test failed"); + TEST_ASSERT_EQUAL(-EINVAL, + test_ethdev_config_run(&dev_conf, &info, + max_event_port_dequeue_depth), + "Config negative test failed"); + TEST_ASSERT_EQUAL(-EINVAL, + test_ethdev_config_run(&dev_conf, &info, + max_event_port_enqueue_depth), + "Config negative test failed"); + + /* Positive case */ + devconf_set_default_sane_values(&dev_conf, &info); + ret = rte_event_dev_configure(TEST_DEV_ID, &dev_conf); + TEST_ASSERT_SUCCESS(ret, "Failed to configure eventdev"); + + /* re-configure */ + devconf_set_default_sane_values(&dev_conf, &info); + dev_conf.nb_event_ports = RTE_MAX(info.max_event_ports/2, 1); + dev_conf.nb_event_queues = RTE_MAX(info.max_event_queues/2, 1); + ret = rte_event_dev_configure(TEST_DEV_ID, &dev_conf); + TEST_ASSERT_SUCCESS(ret, "Failed to re configure eventdev"); + + /* re-configure back to max_event_queues and max_event_ports */ + devconf_set_default_sane_values(&dev_conf, &info); + ret = rte_event_dev_configure(TEST_DEV_ID, &dev_conf); + TEST_ASSERT_SUCCESS(ret, "Failed to re-configure eventdev"); + + return TEST_SUCCESS; + +} + +static int +eventdev_configure_setup(void) +{ + int ret; + struct rte_event_dev_config dev_conf; + struct rte_event_dev_info info; + + ret = rte_event_dev_info_get(TEST_DEV_ID, &info); + TEST_ASSERT_SUCCESS(ret, "Failed to get event dev info"); + devconf_set_default_sane_values(&dev_conf, &info); + ret = rte_event_dev_configure(TEST_DEV_ID, &dev_conf); + TEST_ASSERT_SUCCESS(ret, "Failed to configure eventdev"); + + return TEST_SUCCESS; +} + +static int +test_eventdev_queue_default_conf_get(void) +{ + int i, ret; + struct rte_event_queue_conf qconf; + + ret = rte_event_queue_default_conf_get(TEST_DEV_ID, 0, NULL); + TEST_ASSERT(ret == -EINVAL, "Expected -EINVAL, %d", ret); + + uint32_t queue_count; + TEST_ASSERT_SUCCESS(rte_event_dev_attr_get(TEST_DEV_ID, + RTE_EVENT_DEV_ATTR_QUEUE_COUNT, &queue_count), + "Queue count get failed"); + + for (i = 0; i < (int)queue_count; i++) { + ret = rte_event_queue_default_conf_get(TEST_DEV_ID, i, + &qconf); + TEST_ASSERT_SUCCESS(ret, "Failed to get queue%d info", i); + } + + return TEST_SUCCESS; +} + +static int +test_eventdev_queue_setup(void) +{ + int i, ret; + struct rte_event_dev_info info; + struct rte_event_queue_conf qconf; + + ret = rte_event_dev_info_get(TEST_DEV_ID, &info); + TEST_ASSERT_SUCCESS(ret, "Failed to get event dev info"); + + /* Negative cases */ + ret = rte_event_queue_default_conf_get(TEST_DEV_ID, 0, &qconf); + TEST_ASSERT_SUCCESS(ret, "Failed to get queue0 info"); + qconf.event_queue_cfg = RTE_EVENT_QUEUE_CFG_ALL_TYPES; + qconf.nb_atomic_flows = info.max_event_queue_flows + 1; + ret = rte_event_queue_setup(TEST_DEV_ID, 0, &qconf); + TEST_ASSERT(ret == -EINVAL, "Expected -EINVAL, %d", ret); + + qconf.nb_atomic_flows = info.max_event_queue_flows; + qconf.schedule_type = RTE_SCHED_TYPE_ORDERED; + qconf.nb_atomic_order_sequences = info.max_event_queue_flows + 1; + ret = rte_event_queue_setup(TEST_DEV_ID, 0, &qconf); + TEST_ASSERT(ret == -EINVAL, "Expected -EINVAL, %d", ret); + + ret = rte_event_queue_setup(TEST_DEV_ID, info.max_event_queues, + &qconf); + TEST_ASSERT(ret == -EINVAL, "Expected -EINVAL, %d", ret); + + /* Positive case */ + ret = rte_event_queue_default_conf_get(TEST_DEV_ID, 0, &qconf); + TEST_ASSERT_SUCCESS(ret, "Failed to get queue0 info"); + ret = rte_event_queue_setup(TEST_DEV_ID, 0, &qconf); + TEST_ASSERT_SUCCESS(ret, "Failed to setup queue0"); + + uint32_t queue_count; + TEST_ASSERT_SUCCESS(rte_event_dev_attr_get(TEST_DEV_ID, + RTE_EVENT_DEV_ATTR_QUEUE_COUNT, &queue_count), + "Queue count get failed"); + + for (i = 0; i < (int)queue_count; i++) { + ret = rte_event_queue_setup(TEST_DEV_ID, i, NULL); + TEST_ASSERT_SUCCESS(ret, "Failed to setup queue%d", i); + } + + return TEST_SUCCESS; +} + +static int +test_eventdev_queue_count(void) +{ + int ret; + struct rte_event_dev_info info; + + ret = rte_event_dev_info_get(TEST_DEV_ID, &info); + TEST_ASSERT_SUCCESS(ret, "Failed to get event dev info"); + + uint32_t queue_count; + TEST_ASSERT_SUCCESS(rte_event_dev_attr_get(TEST_DEV_ID, + RTE_EVENT_DEV_ATTR_QUEUE_COUNT, &queue_count), + "Queue count get failed"); + TEST_ASSERT_EQUAL(queue_count, info.max_event_queues, + "Wrong queue count"); + + return TEST_SUCCESS; +} + +static int +test_eventdev_queue_attr_priority(void) +{ + int i, ret; + struct rte_event_dev_info info; + struct rte_event_queue_conf qconf; + uint8_t priority; + + ret = rte_event_dev_info_get(TEST_DEV_ID, &info); + TEST_ASSERT_SUCCESS(ret, "Failed to get event dev info"); + + uint32_t queue_count; + TEST_ASSERT_SUCCESS(rte_event_dev_attr_get(TEST_DEV_ID, + RTE_EVENT_DEV_ATTR_QUEUE_COUNT, &queue_count), + "Queue count get failed"); + + for (i = 0; i < (int)queue_count; i++) { + ret = rte_event_queue_default_conf_get(TEST_DEV_ID, i, + &qconf); + TEST_ASSERT_SUCCESS(ret, "Failed to get queue%d def conf", i); + qconf.priority = i % RTE_EVENT_DEV_PRIORITY_LOWEST; + ret = rte_event_queue_setup(TEST_DEV_ID, i, &qconf); + TEST_ASSERT_SUCCESS(ret, "Failed to setup queue%d", i); + } + + for (i = 0; i < (int)queue_count; i++) { + uint32_t tmp; + TEST_ASSERT_SUCCESS(rte_event_queue_attr_get(TEST_DEV_ID, i, + RTE_EVENT_QUEUE_ATTR_PRIORITY, &tmp), + "Queue priority get failed"); + priority = tmp; + + if (info.event_dev_cap & RTE_EVENT_DEV_CAP_QUEUE_QOS) + TEST_ASSERT_EQUAL(priority, + i % RTE_EVENT_DEV_PRIORITY_LOWEST, + "Wrong priority value for queue%d", i); + else + TEST_ASSERT_EQUAL(priority, + RTE_EVENT_DEV_PRIORITY_NORMAL, + "Wrong priority value for queue%d", i); + } + + return TEST_SUCCESS; +} + +static int +test_eventdev_queue_attr_nb_atomic_flows(void) +{ + int i, ret; + struct rte_event_dev_info info; + struct rte_event_queue_conf qconf; + uint32_t nb_atomic_flows; + + ret = rte_event_dev_info_get(TEST_DEV_ID, &info); + TEST_ASSERT_SUCCESS(ret, "Failed to get event dev info"); + + uint32_t queue_count; + TEST_ASSERT_SUCCESS(rte_event_dev_attr_get(TEST_DEV_ID, + RTE_EVENT_DEV_ATTR_QUEUE_COUNT, &queue_count), + "Queue count get failed"); + + ret = rte_event_queue_default_conf_get(TEST_DEV_ID, 0, &qconf); + TEST_ASSERT_SUCCESS(ret, "Failed to get queue 0's def conf"); + + if (qconf.nb_atomic_flows == 0) + /* Assume PMD doesn't support atomic flows, return early */ + return -ENOTSUP; + + qconf.schedule_type = RTE_SCHED_TYPE_ATOMIC; + + for (i = 0; i < (int)queue_count; i++) { + ret = rte_event_queue_setup(TEST_DEV_ID, i, &qconf); + TEST_ASSERT_SUCCESS(ret, "Failed to setup queue%d", i); + } + + for (i = 0; i < (int)queue_count; i++) { + TEST_ASSERT_SUCCESS(rte_event_queue_attr_get(TEST_DEV_ID, i, + RTE_EVENT_QUEUE_ATTR_NB_ATOMIC_FLOWS, + &nb_atomic_flows), + "Queue nb_atomic_flows get failed"); + + TEST_ASSERT_EQUAL(nb_atomic_flows, qconf.nb_atomic_flows, + "Wrong atomic flows value for queue%d", i); + } + + return TEST_SUCCESS; +} + +static int +test_eventdev_queue_attr_nb_atomic_order_sequences(void) +{ + int i, ret; + struct rte_event_dev_info info; + struct rte_event_queue_conf qconf; + uint32_t nb_atomic_order_sequences; + + ret = rte_event_dev_info_get(TEST_DEV_ID, &info); + TEST_ASSERT_SUCCESS(ret, "Failed to get event dev info"); + + uint32_t queue_count; + TEST_ASSERT_SUCCESS(rte_event_dev_attr_get(TEST_DEV_ID, + RTE_EVENT_DEV_ATTR_QUEUE_COUNT, &queue_count), + "Queue count get failed"); + + ret = rte_event_queue_default_conf_get(TEST_DEV_ID, 0, &qconf); + TEST_ASSERT_SUCCESS(ret, "Failed to get queue 0's def conf"); + + if (qconf.nb_atomic_order_sequences == 0) + /* Assume PMD doesn't support reordering */ + return -ENOTSUP; + + qconf.schedule_type = RTE_SCHED_TYPE_ORDERED; + + for (i = 0; i < (int)queue_count; i++) { + ret = rte_event_queue_setup(TEST_DEV_ID, i, &qconf); + TEST_ASSERT_SUCCESS(ret, "Failed to setup queue%d", i); + } + + for (i = 0; i < (int)queue_count; i++) { + TEST_ASSERT_SUCCESS(rte_event_queue_attr_get(TEST_DEV_ID, i, + RTE_EVENT_QUEUE_ATTR_NB_ATOMIC_ORDER_SEQUENCES, + &nb_atomic_order_sequences), + "Queue nb_atomic_order_sequencess get failed"); + + TEST_ASSERT_EQUAL(nb_atomic_order_sequences, + qconf.nb_atomic_order_sequences, + "Wrong atomic order sequences value for queue%d", + i); + } + + return TEST_SUCCESS; +} + +static int +test_eventdev_queue_attr_event_queue_cfg(void) +{ + int i, ret; + struct rte_event_dev_info info; + struct rte_event_queue_conf qconf; + uint32_t event_queue_cfg; + + ret = rte_event_dev_info_get(TEST_DEV_ID, &info); + TEST_ASSERT_SUCCESS(ret, "Failed to get event dev info"); + + uint32_t queue_count; + TEST_ASSERT_SUCCESS(rte_event_dev_attr_get(TEST_DEV_ID, + RTE_EVENT_DEV_ATTR_QUEUE_COUNT, &queue_count), + "Queue count get failed"); + + ret = rte_event_queue_default_conf_get(TEST_DEV_ID, 0, &qconf); + TEST_ASSERT_SUCCESS(ret, "Failed to get queue0 def conf"); + + qconf.event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK; + + for (i = 0; i < (int)queue_count; i++) { + ret = rte_event_queue_setup(TEST_DEV_ID, i, &qconf); + TEST_ASSERT_SUCCESS(ret, "Failed to setup queue%d", i); + } + + for (i = 0; i < (int)queue_count; i++) { + TEST_ASSERT_SUCCESS(rte_event_queue_attr_get(TEST_DEV_ID, i, + RTE_EVENT_QUEUE_ATTR_EVENT_QUEUE_CFG, + &event_queue_cfg), + "Queue event_queue_cfg get failed"); + + TEST_ASSERT_EQUAL(event_queue_cfg, qconf.event_queue_cfg, + "Wrong event_queue_cfg value for queue%d", + i); + } + + return TEST_SUCCESS; +} + +static int +test_eventdev_port_default_conf_get(void) +{ + int i, ret; + struct rte_event_port_conf pconf; + + ret = rte_event_port_default_conf_get(TEST_DEV_ID, 0, NULL); + TEST_ASSERT(ret == -EINVAL, "Expected -EINVAL, %d", ret); + + uint32_t port_count; + TEST_ASSERT_SUCCESS(rte_event_dev_attr_get(TEST_DEV_ID, + RTE_EVENT_DEV_ATTR_PORT_COUNT, + &port_count), "Port count get failed"); + + ret = rte_event_port_default_conf_get(TEST_DEV_ID, + port_count + 1, NULL); + TEST_ASSERT(ret == -EINVAL, "Expected -EINVAL, %d", ret); + + for (i = 0; i < (int)port_count; i++) { + ret = rte_event_port_default_conf_get(TEST_DEV_ID, i, + &pconf); + TEST_ASSERT_SUCCESS(ret, "Failed to get port%d info", i); + } + + return TEST_SUCCESS; +} + +static int +test_eventdev_port_setup(void) +{ + int i, ret; + struct rte_event_dev_info info; + struct rte_event_port_conf pconf; + + ret = rte_event_dev_info_get(TEST_DEV_ID, &info); + TEST_ASSERT_SUCCESS(ret, "Failed to get event dev info"); + + /* Negative cases */ + ret = rte_event_port_default_conf_get(TEST_DEV_ID, 0, &pconf); + TEST_ASSERT_SUCCESS(ret, "Failed to get port0 info"); + pconf.new_event_threshold = info.max_num_events + 1; + ret = rte_event_port_setup(TEST_DEV_ID, 0, &pconf); + TEST_ASSERT(ret == -EINVAL, "Expected -EINVAL, %d", ret); + + pconf.new_event_threshold = info.max_num_events; + pconf.dequeue_depth = info.max_event_port_dequeue_depth + 1; + ret = rte_event_port_setup(TEST_DEV_ID, 0, &pconf); + TEST_ASSERT(ret == -EINVAL, "Expected -EINVAL, %d", ret); + + pconf.dequeue_depth = info.max_event_port_dequeue_depth; + pconf.enqueue_depth = info.max_event_port_enqueue_depth + 1; + ret = rte_event_port_setup(TEST_DEV_ID, 0, &pconf); + TEST_ASSERT(ret == -EINVAL, "Expected -EINVAL, %d", ret); + + ret = rte_event_port_setup(TEST_DEV_ID, info.max_event_ports, + &pconf); + TEST_ASSERT(ret == -EINVAL, "Expected -EINVAL, %d", ret); + + /* Positive case */ + ret = rte_event_port_default_conf_get(TEST_DEV_ID, 0, &pconf); + TEST_ASSERT_SUCCESS(ret, "Failed to get port0 info"); + ret = rte_event_port_setup(TEST_DEV_ID, 0, &pconf); + TEST_ASSERT_SUCCESS(ret, "Failed to setup port0"); + + uint32_t port_count; + TEST_ASSERT_SUCCESS(rte_event_dev_attr_get(TEST_DEV_ID, + RTE_EVENT_DEV_ATTR_PORT_COUNT, + &port_count), "Port count get failed"); + + for (i = 0; i < (int)port_count; i++) { + ret = rte_event_port_setup(TEST_DEV_ID, i, NULL); + TEST_ASSERT_SUCCESS(ret, "Failed to setup port%d", i); + } + + return TEST_SUCCESS; +} + +static int +test_eventdev_port_attr_dequeue_depth(void) +{ + int ret; + struct rte_event_dev_info info; + struct rte_event_port_conf pconf; + + ret = rte_event_dev_info_get(TEST_DEV_ID, &info); + TEST_ASSERT_SUCCESS(ret, "Failed to get event dev info"); + + ret = rte_event_port_default_conf_get(TEST_DEV_ID, 0, &pconf); + TEST_ASSERT_SUCCESS(ret, "Failed to get port0 info"); + ret = rte_event_port_setup(TEST_DEV_ID, 0, &pconf); + TEST_ASSERT_SUCCESS(ret, "Failed to setup port0"); + + uint32_t value; + TEST_ASSERT_EQUAL(rte_event_port_attr_get(TEST_DEV_ID, 0, + RTE_EVENT_PORT_ATTR_DEQ_DEPTH, &value), + 0, "Call to get port dequeue depth failed"); + TEST_ASSERT_EQUAL(value, pconf.dequeue_depth, + "Wrong port dequeue depth"); + + return TEST_SUCCESS; +} + +static int +test_eventdev_port_attr_enqueue_depth(void) +{ + int ret; + struct rte_event_dev_info info; + struct rte_event_port_conf pconf; + + ret = rte_event_dev_info_get(TEST_DEV_ID, &info); + TEST_ASSERT_SUCCESS(ret, "Failed to get event dev info"); + + ret = rte_event_port_default_conf_get(TEST_DEV_ID, 0, &pconf); + TEST_ASSERT_SUCCESS(ret, "Failed to get port0 info"); + ret = rte_event_port_setup(TEST_DEV_ID, 0, &pconf); + TEST_ASSERT_SUCCESS(ret, "Failed to setup port0"); + + uint32_t value; + TEST_ASSERT_EQUAL(rte_event_port_attr_get(TEST_DEV_ID, 0, + RTE_EVENT_PORT_ATTR_ENQ_DEPTH, &value), + 0, "Call to get port enqueue depth failed"); + TEST_ASSERT_EQUAL(value, pconf.enqueue_depth, + "Wrong port enqueue depth"); + + return TEST_SUCCESS; +} + +static int +test_eventdev_port_attr_new_event_threshold(void) +{ + int ret; + struct rte_event_dev_info info; + struct rte_event_port_conf pconf; + + ret = rte_event_dev_info_get(TEST_DEV_ID, &info); + TEST_ASSERT_SUCCESS(ret, "Failed to get event dev info"); + + ret = rte_event_port_default_conf_get(TEST_DEV_ID, 0, &pconf); + TEST_ASSERT_SUCCESS(ret, "Failed to get port0 info"); + ret = rte_event_port_setup(TEST_DEV_ID, 0, &pconf); + TEST_ASSERT_SUCCESS(ret, "Failed to setup port0"); + + uint32_t value; + TEST_ASSERT_EQUAL(rte_event_port_attr_get(TEST_DEV_ID, 0, + RTE_EVENT_PORT_ATTR_NEW_EVENT_THRESHOLD, &value), + 0, "Call to get port new event threshold failed"); + TEST_ASSERT_EQUAL((int32_t) value, pconf.new_event_threshold, + "Wrong port new event threshold"); + + return TEST_SUCCESS; +} + +static int +test_eventdev_port_count(void) +{ + int ret; + struct rte_event_dev_info info; + + ret = rte_event_dev_info_get(TEST_DEV_ID, &info); + TEST_ASSERT_SUCCESS(ret, "Failed to get event dev info"); + + uint32_t port_count; + TEST_ASSERT_SUCCESS(rte_event_dev_attr_get(TEST_DEV_ID, + RTE_EVENT_DEV_ATTR_PORT_COUNT, + &port_count), "Port count get failed"); + TEST_ASSERT_EQUAL(port_count, info.max_event_ports, "Wrong port count"); + + return TEST_SUCCESS; +} + +static int +test_eventdev_timeout_ticks(void) +{ + int ret; + uint64_t timeout_ticks; + + ret = rte_event_dequeue_timeout_ticks(TEST_DEV_ID, 100, &timeout_ticks); + if (ret != -ENOTSUP) + TEST_ASSERT_SUCCESS(ret, "Fail to get timeout_ticks"); + + return ret; +} + + +static int +test_eventdev_start_stop(void) +{ + int i, ret; + + ret = eventdev_configure_setup(); + TEST_ASSERT_SUCCESS(ret, "Failed to configure eventdev"); + + uint32_t queue_count; + TEST_ASSERT_SUCCESS(rte_event_dev_attr_get(TEST_DEV_ID, + RTE_EVENT_DEV_ATTR_QUEUE_COUNT, &queue_count), + "Queue count get failed"); + for (i = 0; i < (int)queue_count; i++) { + ret = rte_event_queue_setup(TEST_DEV_ID, i, NULL); + TEST_ASSERT_SUCCESS(ret, "Failed to setup queue%d", i); + } + + uint32_t port_count; + TEST_ASSERT_SUCCESS(rte_event_dev_attr_get(TEST_DEV_ID, + RTE_EVENT_DEV_ATTR_PORT_COUNT, + &port_count), "Port count get failed"); + + for (i = 0; i < (int)port_count; i++) { + ret = rte_event_port_setup(TEST_DEV_ID, i, NULL); + TEST_ASSERT_SUCCESS(ret, "Failed to setup port%d", i); + } + + ret = rte_event_port_link(TEST_DEV_ID, 0, NULL, NULL, 0); + TEST_ASSERT(ret == (int)queue_count, "Failed to link port, device %d", + TEST_DEV_ID); + + ret = rte_event_dev_start(TEST_DEV_ID); + TEST_ASSERT_SUCCESS(ret, "Failed to start device%d", TEST_DEV_ID); + + rte_event_dev_stop(TEST_DEV_ID); + return TEST_SUCCESS; +} + + +static int +eventdev_setup_device(void) +{ + int i, ret; + + ret = eventdev_configure_setup(); + TEST_ASSERT_SUCCESS(ret, "Failed to configure eventdev"); + + uint32_t queue_count; + TEST_ASSERT_SUCCESS(rte_event_dev_attr_get(TEST_DEV_ID, + RTE_EVENT_DEV_ATTR_QUEUE_COUNT, &queue_count), + "Queue count get failed"); + for (i = 0; i < (int)queue_count; i++) { + ret = rte_event_queue_setup(TEST_DEV_ID, i, NULL); + TEST_ASSERT_SUCCESS(ret, "Failed to setup queue%d", i); + } + + uint32_t port_count; + TEST_ASSERT_SUCCESS(rte_event_dev_attr_get(TEST_DEV_ID, + RTE_EVENT_DEV_ATTR_PORT_COUNT, + &port_count), "Port count get failed"); + + for (i = 0; i < (int)port_count; i++) { + ret = rte_event_port_setup(TEST_DEV_ID, i, NULL); + TEST_ASSERT_SUCCESS(ret, "Failed to setup port%d", i); + } + + ret = rte_event_port_link(TEST_DEV_ID, 0, NULL, NULL, 0); + TEST_ASSERT(ret == (int)queue_count, "Failed to link port, device %d", + TEST_DEV_ID); + + ret = rte_event_dev_start(TEST_DEV_ID); + TEST_ASSERT_SUCCESS(ret, "Failed to start device%d", TEST_DEV_ID); + + return TEST_SUCCESS; +} + +static void +eventdev_stop_device(void) +{ + rte_event_dev_stop(TEST_DEV_ID); +} + +static int +test_eventdev_link(void) +{ + int ret, nb_queues, i; + uint8_t queues[RTE_EVENT_MAX_QUEUES_PER_DEV]; + uint8_t priorities[RTE_EVENT_MAX_QUEUES_PER_DEV]; + + ret = rte_event_port_link(TEST_DEV_ID, 0, NULL, NULL, 0); + TEST_ASSERT(ret >= 0, "Failed to link with NULL device%d", + TEST_DEV_ID); + + uint32_t queue_count; + TEST_ASSERT_SUCCESS(rte_event_dev_attr_get(TEST_DEV_ID, + RTE_EVENT_DEV_ATTR_QUEUE_COUNT, &queue_count), + "Queue count get failed"); + nb_queues = queue_count; + for (i = 0; i < nb_queues; i++) { + queues[i] = i; + priorities[i] = RTE_EVENT_DEV_PRIORITY_NORMAL; + } + + ret = rte_event_port_link(TEST_DEV_ID, 0, queues, + priorities, nb_queues); + TEST_ASSERT(ret == nb_queues, "Failed to link(device%d) ret=%d", + TEST_DEV_ID, ret); + return TEST_SUCCESS; +} + +static int +test_eventdev_unlink(void) +{ + int ret, nb_queues, i; + uint8_t queues[RTE_EVENT_MAX_QUEUES_PER_DEV]; + + ret = rte_event_port_unlink(TEST_DEV_ID, 0, NULL, 0); + TEST_ASSERT(ret >= 0, "Failed to unlink with NULL device%d", + TEST_DEV_ID); + + uint32_t queue_count; + TEST_ASSERT_SUCCESS(rte_event_dev_attr_get(TEST_DEV_ID, + RTE_EVENT_DEV_ATTR_QUEUE_COUNT, &queue_count), + "Queue count get failed"); + nb_queues = queue_count; + for (i = 0; i < nb_queues; i++) + queues[i] = i; + + ret = rte_event_port_link(TEST_DEV_ID, 0, NULL, NULL, 0); + TEST_ASSERT(ret >= 0, "Failed to link with NULL device%d", + TEST_DEV_ID); + + ret = rte_event_port_unlink(TEST_DEV_ID, 0, queues, nb_queues); + TEST_ASSERT(ret == nb_queues, "Failed to unlink(device%d) ret=%d", + TEST_DEV_ID, ret); + return TEST_SUCCESS; +} + +static int +test_eventdev_link_get(void) +{ + int ret, i; + uint8_t queues[RTE_EVENT_MAX_QUEUES_PER_DEV]; + uint8_t priorities[RTE_EVENT_MAX_QUEUES_PER_DEV]; + + /* link all queues */ + ret = rte_event_port_link(TEST_DEV_ID, 0, NULL, NULL, 0); + TEST_ASSERT(ret >= 0, "Failed to link with NULL device%d", + TEST_DEV_ID); + + uint32_t queue_count; + TEST_ASSERT_SUCCESS(rte_event_dev_attr_get(TEST_DEV_ID, + RTE_EVENT_DEV_ATTR_QUEUE_COUNT, &queue_count), + "Queue count get failed"); + const int nb_queues = queue_count; + for (i = 0; i < nb_queues; i++) + queues[i] = i; + + ret = rte_event_port_unlink(TEST_DEV_ID, 0, queues, nb_queues); + TEST_ASSERT(ret == nb_queues, "Failed to unlink(device%d) ret=%d", + TEST_DEV_ID, ret); + + ret = rte_event_port_links_get(TEST_DEV_ID, 0, queues, priorities); + TEST_ASSERT(ret == 0, "(%d)Wrong link get=%d", TEST_DEV_ID, ret); + + /* link all queues and get the links */ + for (i = 0; i < nb_queues; i++) { + queues[i] = i; + priorities[i] = RTE_EVENT_DEV_PRIORITY_NORMAL; + } + ret = rte_event_port_link(TEST_DEV_ID, 0, queues, priorities, + nb_queues); + TEST_ASSERT(ret == nb_queues, "Failed to link(device%d) ret=%d", + TEST_DEV_ID, ret); + ret = rte_event_port_links_get(TEST_DEV_ID, 0, queues, priorities); + TEST_ASSERT(ret == nb_queues, "(%d)Wrong link get ret=%d expected=%d", + TEST_DEV_ID, ret, nb_queues); + /* unlink all*/ + ret = rte_event_port_unlink(TEST_DEV_ID, 0, NULL, 0); + TEST_ASSERT(ret == nb_queues, "Failed to unlink(device%d) ret=%d", + TEST_DEV_ID, ret); + /* link just one queue */ + queues[0] = 0; + priorities[0] = RTE_EVENT_DEV_PRIORITY_NORMAL; + + ret = rte_event_port_link(TEST_DEV_ID, 0, queues, priorities, 1); + TEST_ASSERT(ret == 1, "Failed to link(device%d) ret=%d", + TEST_DEV_ID, ret); + ret = rte_event_port_links_get(TEST_DEV_ID, 0, queues, priorities); + TEST_ASSERT(ret == 1, "(%d)Wrong link get ret=%d expected=%d", + TEST_DEV_ID, ret, 1); + /* unlink the queue */ + ret = rte_event_port_unlink(TEST_DEV_ID, 0, NULL, 0); + TEST_ASSERT(ret == 1, "Failed to unlink(device%d) ret=%d", + TEST_DEV_ID, ret); + + /* 4links and 2 unlinks */ + if (nb_queues >= 4) { + for (i = 0; i < 4; i++) { + queues[i] = i; + priorities[i] = 0x40; + } + ret = rte_event_port_link(TEST_DEV_ID, 0, queues, priorities, + 4); + TEST_ASSERT(ret == 4, "Failed to link(device%d) ret=%d", + TEST_DEV_ID, ret); + + for (i = 0; i < 2; i++) + queues[i] = i; + + ret = rte_event_port_unlink(TEST_DEV_ID, 0, queues, 2); + TEST_ASSERT(ret == 2, "Failed to unlink(device%d) ret=%d", + TEST_DEV_ID, ret); + ret = rte_event_port_links_get(TEST_DEV_ID, 0, + queues, priorities); + TEST_ASSERT(ret == 2, "(%d)Wrong link get ret=%d expected=%d", + TEST_DEV_ID, ret, 2); + TEST_ASSERT(queues[0] == 2, "ret=%d expected=%d", ret, 2); + TEST_ASSERT(priorities[0] == 0x40, "ret=%d expected=%d", + ret, 0x40); + TEST_ASSERT(queues[1] == 3, "ret=%d expected=%d", ret, 3); + TEST_ASSERT(priorities[1] == 0x40, "ret=%d expected=%d", + ret, 0x40); + } + + return TEST_SUCCESS; +} + +static int +test_eventdev_close(void) +{ + rte_event_dev_stop(TEST_DEV_ID); + return rte_event_dev_close(TEST_DEV_ID); +} + +static struct unit_test_suite eventdev_common_testsuite = { + .suite_name = "eventdev common code unit test suite", + .setup = testsuite_setup, + .teardown = testsuite_teardown, + .unit_test_cases = { + TEST_CASE_ST(NULL, NULL, + test_eventdev_count), + TEST_CASE_ST(NULL, NULL, + test_eventdev_get_dev_id), + TEST_CASE_ST(NULL, NULL, + test_eventdev_socket_id), + TEST_CASE_ST(NULL, NULL, + test_eventdev_info_get), + TEST_CASE_ST(NULL, NULL, + test_eventdev_configure), + TEST_CASE_ST(eventdev_configure_setup, NULL, + test_eventdev_queue_default_conf_get), + TEST_CASE_ST(eventdev_configure_setup, NULL, + test_eventdev_queue_setup), + TEST_CASE_ST(eventdev_configure_setup, NULL, + test_eventdev_queue_count), + TEST_CASE_ST(eventdev_configure_setup, NULL, + test_eventdev_queue_attr_priority), + TEST_CASE_ST(eventdev_configure_setup, NULL, + test_eventdev_queue_attr_nb_atomic_flows), + TEST_CASE_ST(eventdev_configure_setup, NULL, + test_eventdev_queue_attr_nb_atomic_order_sequences), + TEST_CASE_ST(eventdev_configure_setup, NULL, + test_eventdev_queue_attr_event_queue_cfg), + TEST_CASE_ST(eventdev_configure_setup, NULL, + test_eventdev_port_default_conf_get), + TEST_CASE_ST(eventdev_configure_setup, NULL, + test_eventdev_port_setup), + TEST_CASE_ST(eventdev_configure_setup, NULL, + test_eventdev_port_attr_dequeue_depth), + TEST_CASE_ST(eventdev_configure_setup, NULL, + test_eventdev_port_attr_enqueue_depth), + TEST_CASE_ST(eventdev_configure_setup, NULL, + test_eventdev_port_attr_new_event_threshold), + TEST_CASE_ST(eventdev_configure_setup, NULL, + test_eventdev_port_count), + TEST_CASE_ST(eventdev_configure_setup, NULL, + test_eventdev_timeout_ticks), + TEST_CASE_ST(NULL, NULL, + test_eventdev_start_stop), + TEST_CASE_ST(eventdev_setup_device, eventdev_stop_device, + test_eventdev_link), + TEST_CASE_ST(eventdev_setup_device, eventdev_stop_device, + test_eventdev_unlink), + TEST_CASE_ST(eventdev_setup_device, eventdev_stop_device, + test_eventdev_link_get), + TEST_CASE_ST(eventdev_setup_device, NULL, + test_eventdev_close), + TEST_CASES_END() /**< NULL terminate unit test array */ + } +}; + +static int +test_eventdev_common(void) +{ + return unit_test_suite_runner(&eventdev_common_testsuite); +} + +REGISTER_TEST_COMMAND(eventdev_common_autotest, test_eventdev_common); diff --git a/dpdk/test/test/test_eventdev_octeontx.c b/dpdk/test/test/test_eventdev_octeontx.c new file mode 100644 index 00000000..8fddb4fd --- /dev/null +++ b/dpdk/test/test/test_eventdev_octeontx.c @@ -0,0 +1,1474 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Cavium, Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <rte_atomic.h> +#include <rte_common.h> +#include <rte_cycles.h> +#include <rte_debug.h> +#include <rte_eal.h> +#include <rte_ethdev.h> +#include <rte_eventdev.h> +#include <rte_hexdump.h> +#include <rte_mbuf.h> +#include <rte_malloc.h> +#include <rte_memcpy.h> +#include <rte_launch.h> +#include <rte_lcore.h> +#include <rte_per_lcore.h> +#include <rte_random.h> +#include <rte_bus_vdev.h> + +#include "test.h" + +#define NUM_PACKETS (1 << 18) +#define MAX_EVENTS (16 * 1024) + +static int evdev; +static struct rte_mempool *eventdev_test_mempool; + +struct event_attr { + uint32_t flow_id; + uint8_t event_type; + uint8_t sub_event_type; + uint8_t sched_type; + uint8_t queue; + uint8_t port; +}; + +static uint32_t seqn_list_index; +static int seqn_list[NUM_PACKETS]; + +static inline void +seqn_list_init(void) +{ + RTE_BUILD_BUG_ON(NUM_PACKETS < MAX_EVENTS); + memset(seqn_list, 0, sizeof(seqn_list)); + seqn_list_index = 0; +} + +static inline int +seqn_list_update(int val) +{ + if (seqn_list_index >= NUM_PACKETS) + return TEST_FAILED; + + seqn_list[seqn_list_index++] = val; + rte_smp_wmb(); + return TEST_SUCCESS; +} + +static inline int +seqn_list_check(int limit) +{ + int i; + + for (i = 0; i < limit; i++) { + if (seqn_list[i] != i) { + printf("Seqn mismatch %d %d\n", seqn_list[i], i); + return TEST_FAILED; + } + } + return TEST_SUCCESS; +} + +struct test_core_param { + rte_atomic32_t *total_events; + uint64_t dequeue_tmo_ticks; + uint8_t port; + uint8_t sched_type; +}; + +static int +testsuite_setup(void) +{ + const char *eventdev_name = "event_octeontx"; + + evdev = rte_event_dev_get_dev_id(eventdev_name); + if (evdev < 0) { + printf("%d: Eventdev %s not found - creating.\n", + __LINE__, eventdev_name); + if (rte_vdev_init(eventdev_name, NULL) < 0) { + printf("Error creating eventdev %s\n", eventdev_name); + return TEST_FAILED; + } + evdev = rte_event_dev_get_dev_id(eventdev_name); + if (evdev < 0) { + printf("Error finding newly created eventdev\n"); + return TEST_FAILED; + } + } + + return TEST_SUCCESS; +} + +static void +testsuite_teardown(void) +{ + rte_event_dev_close(evdev); +} + +static inline void +devconf_set_default_sane_values(struct rte_event_dev_config *dev_conf, + struct rte_event_dev_info *info) +{ + memset(dev_conf, 0, sizeof(struct rte_event_dev_config)); + dev_conf->dequeue_timeout_ns = info->min_dequeue_timeout_ns; + dev_conf->nb_event_ports = info->max_event_ports; + dev_conf->nb_event_queues = info->max_event_queues; + dev_conf->nb_event_queue_flows = info->max_event_queue_flows; + dev_conf->nb_event_port_dequeue_depth = + info->max_event_port_dequeue_depth; + dev_conf->nb_event_port_enqueue_depth = + info->max_event_port_enqueue_depth; + dev_conf->nb_event_port_enqueue_depth = + info->max_event_port_enqueue_depth; + dev_conf->nb_events_limit = + info->max_num_events; +} + +enum { + TEST_EVENTDEV_SETUP_DEFAULT, + TEST_EVENTDEV_SETUP_PRIORITY, + TEST_EVENTDEV_SETUP_DEQUEUE_TIMEOUT, +}; + +static inline int +_eventdev_setup(int mode) +{ + int i, ret; + struct rte_event_dev_config dev_conf; + struct rte_event_dev_info info; + const char *pool_name = "evdev_octeontx_test_pool"; + + /* Create and destrory pool for each test case to make it standalone */ + eventdev_test_mempool = rte_pktmbuf_pool_create(pool_name, + MAX_EVENTS, + 0 /*MBUF_CACHE_SIZE*/, + 0, + 512, /* Use very small mbufs */ + rte_socket_id()); + if (!eventdev_test_mempool) { + printf("ERROR creating mempool\n"); + return TEST_FAILED; + } + + ret = rte_event_dev_info_get(evdev, &info); + TEST_ASSERT_SUCCESS(ret, "Failed to get event dev info"); + TEST_ASSERT(info.max_num_events >= (int32_t)MAX_EVENTS, + "max_num_events=%d < max_events=%d", + info.max_num_events, MAX_EVENTS); + + devconf_set_default_sane_values(&dev_conf, &info); + if (mode == TEST_EVENTDEV_SETUP_DEQUEUE_TIMEOUT) + dev_conf.event_dev_cfg |= RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT; + + ret = rte_event_dev_configure(evdev, &dev_conf); + TEST_ASSERT_SUCCESS(ret, "Failed to configure eventdev"); + + uint32_t queue_count; + TEST_ASSERT_SUCCESS(rte_event_dev_attr_get(evdev, + RTE_EVENT_DEV_ATTR_QUEUE_COUNT, + &queue_count), "Queue count get failed"); + + if (mode == TEST_EVENTDEV_SETUP_PRIORITY) { + if (queue_count > 8) { + printf("test expects the unique priority per queue\n"); + return -ENOTSUP; + } + + /* Configure event queues(0 to n) with + * RTE_EVENT_DEV_PRIORITY_HIGHEST to + * RTE_EVENT_DEV_PRIORITY_LOWEST + */ + uint8_t step = (RTE_EVENT_DEV_PRIORITY_LOWEST + 1) / + queue_count; + for (i = 0; i < (int)queue_count; i++) { + struct rte_event_queue_conf queue_conf; + + ret = rte_event_queue_default_conf_get(evdev, i, + &queue_conf); + TEST_ASSERT_SUCCESS(ret, "Failed to get def_conf%d", i); + queue_conf.priority = i * step; + ret = rte_event_queue_setup(evdev, i, &queue_conf); + TEST_ASSERT_SUCCESS(ret, "Failed to setup queue=%d", i); + } + + } else { + /* Configure event queues with default priority */ + for (i = 0; i < (int)queue_count; i++) { + ret = rte_event_queue_setup(evdev, i, NULL); + TEST_ASSERT_SUCCESS(ret, "Failed to setup queue=%d", i); + } + } + /* Configure event ports */ + uint32_t port_count; + TEST_ASSERT_SUCCESS(rte_event_dev_attr_get(evdev, + RTE_EVENT_DEV_ATTR_PORT_COUNT, + &port_count), "Port count get failed"); + for (i = 0; i < (int)port_count; i++) { + ret = rte_event_port_setup(evdev, i, NULL); + TEST_ASSERT_SUCCESS(ret, "Failed to setup port=%d", i); + ret = rte_event_port_link(evdev, i, NULL, NULL, 0); + TEST_ASSERT(ret >= 0, "Failed to link all queues port=%d", i); + } + + ret = rte_event_dev_start(evdev); + TEST_ASSERT_SUCCESS(ret, "Failed to start device"); + + return TEST_SUCCESS; +} + +static inline int +eventdev_setup(void) +{ + return _eventdev_setup(TEST_EVENTDEV_SETUP_DEFAULT); +} + +static inline int +eventdev_setup_priority(void) +{ + return _eventdev_setup(TEST_EVENTDEV_SETUP_PRIORITY); +} + +static inline int +eventdev_setup_dequeue_timeout(void) +{ + return _eventdev_setup(TEST_EVENTDEV_SETUP_DEQUEUE_TIMEOUT); +} + +static inline void +eventdev_teardown(void) +{ + rte_event_dev_stop(evdev); + rte_mempool_free(eventdev_test_mempool); +} + +static inline void +update_event_and_validation_attr(struct rte_mbuf *m, struct rte_event *ev, + uint32_t flow_id, uint8_t event_type, + uint8_t sub_event_type, uint8_t sched_type, + uint8_t queue, uint8_t port) +{ + struct event_attr *attr; + + /* Store the event attributes in mbuf for future reference */ + attr = rte_pktmbuf_mtod(m, struct event_attr *); + attr->flow_id = flow_id; + attr->event_type = event_type; + attr->sub_event_type = sub_event_type; + attr->sched_type = sched_type; + attr->queue = queue; + attr->port = port; + + ev->flow_id = flow_id; + ev->sub_event_type = sub_event_type; + ev->event_type = event_type; + /* Inject the new event */ + ev->op = RTE_EVENT_OP_NEW; + ev->sched_type = sched_type; + ev->queue_id = queue; + ev->mbuf = m; +} + +static inline int +inject_events(uint32_t flow_id, uint8_t event_type, uint8_t sub_event_type, + uint8_t sched_type, uint8_t queue, uint8_t port, + unsigned int events) +{ + struct rte_mbuf *m; + unsigned int i; + + for (i = 0; i < events; i++) { + struct rte_event ev = {.event = 0, .u64 = 0}; + + m = rte_pktmbuf_alloc(eventdev_test_mempool); + TEST_ASSERT_NOT_NULL(m, "mempool alloc failed"); + + m->seqn = i; + update_event_and_validation_attr(m, &ev, flow_id, event_type, + sub_event_type, sched_type, queue, port); + rte_event_enqueue_burst(evdev, port, &ev, 1); + } + return 0; +} + +static inline int +check_excess_events(uint8_t port) +{ + int i; + uint16_t valid_event; + struct rte_event ev; + + /* Check for excess events, try for a few times and exit */ + for (i = 0; i < 32; i++) { + valid_event = rte_event_dequeue_burst(evdev, port, &ev, 1, 0); + + TEST_ASSERT_SUCCESS(valid_event, "Unexpected valid event=%d", + ev.mbuf->seqn); + } + return 0; +} + +static inline int +generate_random_events(const unsigned int total_events) +{ + struct rte_event_dev_info info; + unsigned int i; + int ret; + + uint32_t queue_count; + TEST_ASSERT_SUCCESS(rte_event_dev_attr_get(evdev, + RTE_EVENT_DEV_ATTR_QUEUE_COUNT, + &queue_count), "Queue count get failed"); + + ret = rte_event_dev_info_get(evdev, &info); + TEST_ASSERT_SUCCESS(ret, "Failed to get event dev info"); + for (i = 0; i < total_events; i++) { + ret = inject_events( + rte_rand() % info.max_event_queue_flows /*flow_id */, + RTE_EVENT_TYPE_CPU /* event_type */, + rte_rand() % 256 /* sub_event_type */, + rte_rand() % (RTE_SCHED_TYPE_PARALLEL + 1), + rte_rand() % queue_count /* queue */, + 0 /* port */, + 1 /* events */); + if (ret) + return TEST_FAILED; + } + return ret; +} + + +static inline int +validate_event(struct rte_event *ev) +{ + struct event_attr *attr; + + attr = rte_pktmbuf_mtod(ev->mbuf, struct event_attr *); + TEST_ASSERT_EQUAL(attr->flow_id, ev->flow_id, + "flow_id mismatch enq=%d deq =%d", + attr->flow_id, ev->flow_id); + TEST_ASSERT_EQUAL(attr->event_type, ev->event_type, + "event_type mismatch enq=%d deq =%d", + attr->event_type, ev->event_type); + TEST_ASSERT_EQUAL(attr->sub_event_type, ev->sub_event_type, + "sub_event_type mismatch enq=%d deq =%d", + attr->sub_event_type, ev->sub_event_type); + TEST_ASSERT_EQUAL(attr->sched_type, ev->sched_type, + "sched_type mismatch enq=%d deq =%d", + attr->sched_type, ev->sched_type); + TEST_ASSERT_EQUAL(attr->queue, ev->queue_id, + "queue mismatch enq=%d deq =%d", + attr->queue, ev->queue_id); + return 0; +} + +typedef int (*validate_event_cb)(uint32_t index, uint8_t port, + struct rte_event *ev); + +static inline int +consume_events(uint8_t port, const uint32_t total_events, validate_event_cb fn) +{ + int ret; + uint16_t valid_event; + uint32_t events = 0, forward_progress_cnt = 0, index = 0; + struct rte_event ev; + + while (1) { + if (++forward_progress_cnt > UINT16_MAX) { + printf("Detected deadlock\n"); + return TEST_FAILED; + } + + valid_event = rte_event_dequeue_burst(evdev, port, &ev, 1, 0); + if (!valid_event) + continue; + + forward_progress_cnt = 0; + ret = validate_event(&ev); + if (ret) + return TEST_FAILED; + + if (fn != NULL) { + ret = fn(index, port, &ev); + TEST_ASSERT_SUCCESS(ret, + "Failed to validate test specific event"); + } + + ++index; + + rte_pktmbuf_free(ev.mbuf); + if (++events >= total_events) + break; + } + + return check_excess_events(port); +} + +static int +validate_simple_enqdeq(uint32_t index, uint8_t port, struct rte_event *ev) +{ + RTE_SET_USED(port); + TEST_ASSERT_EQUAL(index, ev->mbuf->seqn, "index=%d != seqn=%d", index, + ev->mbuf->seqn); + return 0; +} + +static inline int +test_simple_enqdeq(uint8_t sched_type) +{ + int ret; + + ret = inject_events(0 /*flow_id */, + RTE_EVENT_TYPE_CPU /* event_type */, + 0 /* sub_event_type */, + sched_type, + 0 /* queue */, + 0 /* port */, + MAX_EVENTS); + if (ret) + return TEST_FAILED; + + return consume_events(0 /* port */, MAX_EVENTS, validate_simple_enqdeq); +} + +static int +test_simple_enqdeq_ordered(void) +{ + return test_simple_enqdeq(RTE_SCHED_TYPE_ORDERED); +} + +static int +test_simple_enqdeq_atomic(void) +{ + return test_simple_enqdeq(RTE_SCHED_TYPE_ATOMIC); +} + +static int +test_simple_enqdeq_parallel(void) +{ + return test_simple_enqdeq(RTE_SCHED_TYPE_PARALLEL); +} + +/* + * Generate a prescribed number of events and spread them across available + * queues. On dequeue, using single event port(port 0) verify the enqueued + * event attributes + */ +static int +test_multi_queue_enq_single_port_deq(void) +{ + int ret; + + ret = generate_random_events(MAX_EVENTS); + if (ret) + return TEST_FAILED; + + return consume_events(0 /* port */, MAX_EVENTS, NULL); +} + +/* + * Inject 0..MAX_EVENTS events over 0..queue_count with modulus + * operation + * + * For example, Inject 32 events over 0..7 queues + * enqueue events 0, 8, 16, 24 in queue 0 + * enqueue events 1, 9, 17, 25 in queue 1 + * .. + * .. + * enqueue events 7, 15, 23, 31 in queue 7 + * + * On dequeue, Validate the events comes in 0,8,16,24,1,9,17,25..,7,15,23,31 + * order from queue0(highest priority) to queue7(lowest_priority) + */ +static int +validate_queue_priority(uint32_t index, uint8_t port, struct rte_event *ev) +{ + uint32_t queue_count; + TEST_ASSERT_SUCCESS(rte_event_dev_attr_get(evdev, + RTE_EVENT_DEV_ATTR_QUEUE_COUNT, + &queue_count), "Queue count get failed"); + uint32_t range = MAX_EVENTS / queue_count; + uint32_t expected_val = (index % range) * queue_count; + + expected_val += ev->queue_id; + RTE_SET_USED(port); + TEST_ASSERT_EQUAL(ev->mbuf->seqn, expected_val, + "seqn=%d index=%d expected=%d range=%d nb_queues=%d max_event=%d", + ev->mbuf->seqn, index, expected_val, range, + queue_count, MAX_EVENTS); + return 0; +} + +static int +test_multi_queue_priority(void) +{ + uint8_t queue; + struct rte_mbuf *m; + int i, max_evts_roundoff; + + /* See validate_queue_priority() comments for priority validate logic */ + uint32_t queue_count; + TEST_ASSERT_SUCCESS(rte_event_dev_attr_get(evdev, + RTE_EVENT_DEV_ATTR_QUEUE_COUNT, + &queue_count), "Queue count get failed"); + max_evts_roundoff = MAX_EVENTS / queue_count; + max_evts_roundoff *= queue_count; + + for (i = 0; i < max_evts_roundoff; i++) { + struct rte_event ev = {.event = 0, .u64 = 0}; + + m = rte_pktmbuf_alloc(eventdev_test_mempool); + TEST_ASSERT_NOT_NULL(m, "mempool alloc failed"); + + m->seqn = i; + queue = i % queue_count; + update_event_and_validation_attr(m, &ev, 0, RTE_EVENT_TYPE_CPU, + 0, RTE_SCHED_TYPE_PARALLEL, queue, 0); + rte_event_enqueue_burst(evdev, 0, &ev, 1); + } + + return consume_events(0, max_evts_roundoff, validate_queue_priority); +} + +static int +worker_multi_port_fn(void *arg) +{ + struct test_core_param *param = arg; + struct rte_event ev; + uint16_t valid_event; + uint8_t port = param->port; + rte_atomic32_t *total_events = param->total_events; + int ret; + + while (rte_atomic32_read(total_events) > 0) { + valid_event = rte_event_dequeue_burst(evdev, port, &ev, 1, 0); + if (!valid_event) + continue; + + ret = validate_event(&ev); + TEST_ASSERT_SUCCESS(ret, "Failed to validate event"); + rte_pktmbuf_free(ev.mbuf); + rte_atomic32_sub(total_events, 1); + } + return 0; +} + +static inline int +wait_workers_to_join(int lcore, const rte_atomic32_t *count) +{ + uint64_t cycles, print_cycles; + + print_cycles = cycles = rte_get_timer_cycles(); + while (rte_eal_get_lcore_state(lcore) != FINISHED) { + uint64_t new_cycles = rte_get_timer_cycles(); + + if (new_cycles - print_cycles > rte_get_timer_hz()) { + printf("\r%s: events %d\n", __func__, + rte_atomic32_read(count)); + print_cycles = new_cycles; + } + if (new_cycles - cycles > rte_get_timer_hz() * 10) { + printf("%s: No schedules for seconds, deadlock (%d)\n", + __func__, + rte_atomic32_read(count)); + rte_event_dev_dump(evdev, stdout); + cycles = new_cycles; + return TEST_FAILED; + } + } + rte_eal_mp_wait_lcore(); + return TEST_SUCCESS; +} + + +static inline int +launch_workers_and_wait(int (*master_worker)(void *), + int (*slave_workers)(void *), uint32_t total_events, + uint8_t nb_workers, uint8_t sched_type) +{ + uint8_t port = 0; + int w_lcore; + int ret; + struct test_core_param *param; + rte_atomic32_t atomic_total_events; + uint64_t dequeue_tmo_ticks; + + if (!nb_workers) + return 0; + + rte_atomic32_set(&atomic_total_events, total_events); + seqn_list_init(); + + param = malloc(sizeof(struct test_core_param) * nb_workers); + if (!param) + return TEST_FAILED; + + ret = rte_event_dequeue_timeout_ticks(evdev, + rte_rand() % 10000000/* 10ms */, &dequeue_tmo_ticks); + if (ret) + return TEST_FAILED; + + param[0].total_events = &atomic_total_events; + param[0].sched_type = sched_type; + param[0].port = 0; + param[0].dequeue_tmo_ticks = dequeue_tmo_ticks; + rte_smp_wmb(); + + w_lcore = rte_get_next_lcore( + /* start core */ -1, + /* skip master */ 1, + /* wrap */ 0); + rte_eal_remote_launch(master_worker, ¶m[0], w_lcore); + + for (port = 1; port < nb_workers; port++) { + param[port].total_events = &atomic_total_events; + param[port].sched_type = sched_type; + param[port].port = port; + param[port].dequeue_tmo_ticks = dequeue_tmo_ticks; + rte_smp_wmb(); + w_lcore = rte_get_next_lcore(w_lcore, 1, 0); + rte_eal_remote_launch(slave_workers, ¶m[port], w_lcore); + } + + ret = wait_workers_to_join(w_lcore, &atomic_total_events); + free(param); + return ret; +} + +/* + * Generate a prescribed number of events and spread them across available + * queues. Dequeue the events through multiple ports and verify the enqueued + * event attributes + */ +static int +test_multi_queue_enq_multi_port_deq(void) +{ + const unsigned int total_events = MAX_EVENTS; + uint32_t nr_ports; + int ret; + + ret = generate_random_events(total_events); + if (ret) + return TEST_FAILED; + + TEST_ASSERT_SUCCESS(rte_event_dev_attr_get(evdev, + RTE_EVENT_DEV_ATTR_PORT_COUNT, + &nr_ports), "Port count get failed"); + nr_ports = RTE_MIN(nr_ports, rte_lcore_count() - 1); + + if (!nr_ports) { + printf("%s: Not enough ports=%d or workers=%d\n", __func__, + nr_ports, rte_lcore_count() - 1); + return TEST_SUCCESS; + } + + return launch_workers_and_wait(worker_multi_port_fn, + worker_multi_port_fn, total_events, + nr_ports, 0xff /* invalid */); +} + +static int +validate_queue_to_port_single_link(uint32_t index, uint8_t port, + struct rte_event *ev) +{ + RTE_SET_USED(index); + TEST_ASSERT_EQUAL(port, ev->queue_id, + "queue mismatch enq=%d deq =%d", + port, ev->queue_id); + return 0; +} + +/* + * Link queue x to port x and check correctness of link by checking + * queue_id == x on dequeue on the specific port x + */ +static int +test_queue_to_port_single_link(void) +{ + int i, nr_links, ret; + + uint32_t port_count; + TEST_ASSERT_SUCCESS(rte_event_dev_attr_get(evdev, + RTE_EVENT_DEV_ATTR_PORT_COUNT, + &port_count), "Port count get failed"); + + /* Unlink all connections that created in eventdev_setup */ + for (i = 0; i < (int)port_count; i++) { + ret = rte_event_port_unlink(evdev, i, NULL, 0); + TEST_ASSERT(ret >= 0, "Failed to unlink all queues port=%d", i); + } + + uint32_t queue_count; + TEST_ASSERT_SUCCESS(rte_event_dev_attr_get(evdev, + RTE_EVENT_DEV_ATTR_QUEUE_COUNT, + &queue_count), "Queue count get failed"); + + nr_links = RTE_MIN(port_count, queue_count); + const unsigned int total_events = MAX_EVENTS / nr_links; + + /* Link queue x to port x and inject events to queue x through port x */ + for (i = 0; i < nr_links; i++) { + uint8_t queue = (uint8_t)i; + + ret = rte_event_port_link(evdev, i, &queue, NULL, 1); + TEST_ASSERT(ret == 1, "Failed to link queue to port %d", i); + + ret = inject_events( + 0x100 /*flow_id */, + RTE_EVENT_TYPE_CPU /* event_type */, + rte_rand() % 256 /* sub_event_type */, + rte_rand() % (RTE_SCHED_TYPE_PARALLEL + 1), + queue /* queue */, + i /* port */, + total_events /* events */); + if (ret) + return TEST_FAILED; + } + + /* Verify the events generated from correct queue */ + for (i = 0; i < nr_links; i++) { + ret = consume_events(i /* port */, total_events, + validate_queue_to_port_single_link); + if (ret) + return TEST_FAILED; + } + + return TEST_SUCCESS; +} + +static int +validate_queue_to_port_multi_link(uint32_t index, uint8_t port, + struct rte_event *ev) +{ + RTE_SET_USED(index); + TEST_ASSERT_EQUAL(port, (ev->queue_id & 0x1), + "queue mismatch enq=%d deq =%d", + port, ev->queue_id); + return 0; +} + +/* + * Link all even number of queues to port 0 and all odd number of queues to + * port 1 and verify the link connection on dequeue + */ +static int +test_queue_to_port_multi_link(void) +{ + int ret, port0_events = 0, port1_events = 0; + uint8_t queue, port; + uint32_t nr_queues = 0; + uint32_t nr_ports = 0; + + TEST_ASSERT_SUCCESS(rte_event_dev_attr_get(evdev, + RTE_EVENT_DEV_ATTR_QUEUE_COUNT, + &nr_queues), "Queue count get failed"); + + TEST_ASSERT_SUCCESS(rte_event_dev_attr_get(evdev, + RTE_EVENT_DEV_ATTR_QUEUE_COUNT, + &nr_queues), "Queue count get failed"); + TEST_ASSERT_SUCCESS(rte_event_dev_attr_get(evdev, + RTE_EVENT_DEV_ATTR_PORT_COUNT, + &nr_ports), "Port count get failed"); + + if (nr_ports < 2) { + printf("%s: Not enough ports to test ports=%d\n", + __func__, nr_ports); + return TEST_SUCCESS; + } + + /* Unlink all connections that created in eventdev_setup */ + for (port = 0; port < nr_ports; port++) { + ret = rte_event_port_unlink(evdev, port, NULL, 0); + TEST_ASSERT(ret >= 0, "Failed to unlink all queues port=%d", + port); + } + + const unsigned int total_events = MAX_EVENTS / nr_queues; + + /* Link all even number of queues to port0 and odd numbers to port 1*/ + for (queue = 0; queue < nr_queues; queue++) { + port = queue & 0x1; + ret = rte_event_port_link(evdev, port, &queue, NULL, 1); + TEST_ASSERT(ret == 1, "Failed to link queue=%d to port=%d", + queue, port); + + ret = inject_events( + 0x100 /*flow_id */, + RTE_EVENT_TYPE_CPU /* event_type */, + rte_rand() % 256 /* sub_event_type */, + rte_rand() % (RTE_SCHED_TYPE_PARALLEL + 1), + queue /* queue */, + port /* port */, + total_events /* events */); + if (ret) + return TEST_FAILED; + + if (port == 0) + port0_events += total_events; + else + port1_events += total_events; + } + + ret = consume_events(0 /* port */, port0_events, + validate_queue_to_port_multi_link); + if (ret) + return TEST_FAILED; + ret = consume_events(1 /* port */, port1_events, + validate_queue_to_port_multi_link); + if (ret) + return TEST_FAILED; + + return TEST_SUCCESS; +} + +static int +worker_flow_based_pipeline(void *arg) +{ + struct test_core_param *param = arg; + struct rte_event ev; + uint16_t valid_event; + uint8_t port = param->port; + uint8_t new_sched_type = param->sched_type; + rte_atomic32_t *total_events = param->total_events; + uint64_t dequeue_tmo_ticks = param->dequeue_tmo_ticks; + + while (rte_atomic32_read(total_events) > 0) { + valid_event = rte_event_dequeue_burst(evdev, port, &ev, 1, + dequeue_tmo_ticks); + if (!valid_event) + continue; + + /* Events from stage 0 */ + if (ev.sub_event_type == 0) { + /* Move to atomic flow to maintain the ordering */ + ev.flow_id = 0x2; + ev.event_type = RTE_EVENT_TYPE_CPU; + ev.sub_event_type = 1; /* stage 1 */ + ev.sched_type = new_sched_type; + ev.op = RTE_EVENT_OP_FORWARD; + rte_event_enqueue_burst(evdev, port, &ev, 1); + } else if (ev.sub_event_type == 1) { /* Events from stage 1*/ + if (seqn_list_update(ev.mbuf->seqn) == TEST_SUCCESS) { + rte_pktmbuf_free(ev.mbuf); + rte_atomic32_sub(total_events, 1); + } else { + printf("Failed to update seqn_list\n"); + return TEST_FAILED; + } + } else { + printf("Invalid ev.sub_event_type = %d\n", + ev.sub_event_type); + return TEST_FAILED; + } + } + return 0; +} + +static int +test_multiport_flow_sched_type_test(uint8_t in_sched_type, + uint8_t out_sched_type) +{ + const unsigned int total_events = MAX_EVENTS; + uint32_t nr_ports; + int ret; + + TEST_ASSERT_SUCCESS(rte_event_dev_attr_get(evdev, + RTE_EVENT_DEV_ATTR_PORT_COUNT, + &nr_ports), "Port count get failed"); + nr_ports = RTE_MIN(nr_ports, rte_lcore_count() - 1); + + if (!nr_ports) { + printf("%s: Not enough ports=%d or workers=%d\n", __func__, + nr_ports, rte_lcore_count() - 1); + return TEST_SUCCESS; + } + + /* Injects events with m->seqn=0 to total_events */ + ret = inject_events( + 0x1 /*flow_id */, + RTE_EVENT_TYPE_CPU /* event_type */, + 0 /* sub_event_type (stage 0) */, + in_sched_type, + 0 /* queue */, + 0 /* port */, + total_events /* events */); + if (ret) + return TEST_FAILED; + + ret = launch_workers_and_wait(worker_flow_based_pipeline, + worker_flow_based_pipeline, + total_events, nr_ports, out_sched_type); + if (ret) + return TEST_FAILED; + + if (in_sched_type != RTE_SCHED_TYPE_PARALLEL && + out_sched_type == RTE_SCHED_TYPE_ATOMIC) { + /* Check the events order maintained or not */ + return seqn_list_check(total_events); + } + return TEST_SUCCESS; +} + + +/* Multi port ordered to atomic transaction */ +static int +test_multi_port_flow_ordered_to_atomic(void) +{ + /* Ingress event order test */ + return test_multiport_flow_sched_type_test(RTE_SCHED_TYPE_ORDERED, + RTE_SCHED_TYPE_ATOMIC); +} + +static int +test_multi_port_flow_ordered_to_ordered(void) +{ + return test_multiport_flow_sched_type_test(RTE_SCHED_TYPE_ORDERED, + RTE_SCHED_TYPE_ORDERED); +} + +static int +test_multi_port_flow_ordered_to_parallel(void) +{ + return test_multiport_flow_sched_type_test(RTE_SCHED_TYPE_ORDERED, + RTE_SCHED_TYPE_PARALLEL); +} + +static int +test_multi_port_flow_atomic_to_atomic(void) +{ + /* Ingress event order test */ + return test_multiport_flow_sched_type_test(RTE_SCHED_TYPE_ATOMIC, + RTE_SCHED_TYPE_ATOMIC); +} + +static int +test_multi_port_flow_atomic_to_ordered(void) +{ + return test_multiport_flow_sched_type_test(RTE_SCHED_TYPE_ATOMIC, + RTE_SCHED_TYPE_ORDERED); +} + +static int +test_multi_port_flow_atomic_to_parallel(void) +{ + return test_multiport_flow_sched_type_test(RTE_SCHED_TYPE_ATOMIC, + RTE_SCHED_TYPE_PARALLEL); +} + +static int +test_multi_port_flow_parallel_to_atomic(void) +{ + return test_multiport_flow_sched_type_test(RTE_SCHED_TYPE_PARALLEL, + RTE_SCHED_TYPE_ATOMIC); +} + +static int +test_multi_port_flow_parallel_to_ordered(void) +{ + return test_multiport_flow_sched_type_test(RTE_SCHED_TYPE_PARALLEL, + RTE_SCHED_TYPE_ORDERED); +} + +static int +test_multi_port_flow_parallel_to_parallel(void) +{ + return test_multiport_flow_sched_type_test(RTE_SCHED_TYPE_PARALLEL, + RTE_SCHED_TYPE_PARALLEL); +} + +static int +worker_group_based_pipeline(void *arg) +{ + struct test_core_param *param = arg; + struct rte_event ev; + uint16_t valid_event; + uint8_t port = param->port; + uint8_t new_sched_type = param->sched_type; + rte_atomic32_t *total_events = param->total_events; + uint64_t dequeue_tmo_ticks = param->dequeue_tmo_ticks; + + while (rte_atomic32_read(total_events) > 0) { + valid_event = rte_event_dequeue_burst(evdev, port, &ev, 1, + dequeue_tmo_ticks); + if (!valid_event) + continue; + + /* Events from stage 0(group 0) */ + if (ev.queue_id == 0) { + /* Move to atomic flow to maintain the ordering */ + ev.flow_id = 0x2; + ev.event_type = RTE_EVENT_TYPE_CPU; + ev.sched_type = new_sched_type; + ev.queue_id = 1; /* Stage 1*/ + ev.op = RTE_EVENT_OP_FORWARD; + rte_event_enqueue_burst(evdev, port, &ev, 1); + } else if (ev.queue_id == 1) { /* Events from stage 1(group 1)*/ + if (seqn_list_update(ev.mbuf->seqn) == TEST_SUCCESS) { + rte_pktmbuf_free(ev.mbuf); + rte_atomic32_sub(total_events, 1); + } else { + printf("Failed to update seqn_list\n"); + return TEST_FAILED; + } + } else { + printf("Invalid ev.queue_id = %d\n", ev.queue_id); + return TEST_FAILED; + } + } + + + return 0; +} + +static int +test_multiport_queue_sched_type_test(uint8_t in_sched_type, + uint8_t out_sched_type) +{ + const unsigned int total_events = MAX_EVENTS; + uint32_t nr_ports; + int ret; + + TEST_ASSERT_SUCCESS(rte_event_dev_attr_get(evdev, + RTE_EVENT_DEV_ATTR_PORT_COUNT, + &nr_ports), "Port count get failed"); + + nr_ports = RTE_MIN(nr_ports, rte_lcore_count() - 1); + + uint32_t queue_count; + TEST_ASSERT_SUCCESS(rte_event_dev_attr_get(evdev, + RTE_EVENT_DEV_ATTR_QUEUE_COUNT, + &queue_count), "Queue count get failed"); + if (queue_count < 2 || !nr_ports) { + printf("%s: Not enough queues=%d ports=%d or workers=%d\n", + __func__, queue_count, nr_ports, + rte_lcore_count() - 1); + return TEST_SUCCESS; + } + + /* Injects events with m->seqn=0 to total_events */ + ret = inject_events( + 0x1 /*flow_id */, + RTE_EVENT_TYPE_CPU /* event_type */, + 0 /* sub_event_type (stage 0) */, + in_sched_type, + 0 /* queue */, + 0 /* port */, + total_events /* events */); + if (ret) + return TEST_FAILED; + + ret = launch_workers_and_wait(worker_group_based_pipeline, + worker_group_based_pipeline, + total_events, nr_ports, out_sched_type); + if (ret) + return TEST_FAILED; + + if (in_sched_type != RTE_SCHED_TYPE_PARALLEL && + out_sched_type == RTE_SCHED_TYPE_ATOMIC) { + /* Check the events order maintained or not */ + return seqn_list_check(total_events); + } + return TEST_SUCCESS; +} + +static int +test_multi_port_queue_ordered_to_atomic(void) +{ + /* Ingress event order test */ + return test_multiport_queue_sched_type_test(RTE_SCHED_TYPE_ORDERED, + RTE_SCHED_TYPE_ATOMIC); +} + +static int +test_multi_port_queue_ordered_to_ordered(void) +{ + return test_multiport_queue_sched_type_test(RTE_SCHED_TYPE_ORDERED, + RTE_SCHED_TYPE_ORDERED); +} + +static int +test_multi_port_queue_ordered_to_parallel(void) +{ + return test_multiport_queue_sched_type_test(RTE_SCHED_TYPE_ORDERED, + RTE_SCHED_TYPE_PARALLEL); +} + +static int +test_multi_port_queue_atomic_to_atomic(void) +{ + /* Ingress event order test */ + return test_multiport_queue_sched_type_test(RTE_SCHED_TYPE_ATOMIC, + RTE_SCHED_TYPE_ATOMIC); +} + +static int +test_multi_port_queue_atomic_to_ordered(void) +{ + return test_multiport_queue_sched_type_test(RTE_SCHED_TYPE_ATOMIC, + RTE_SCHED_TYPE_ORDERED); +} + +static int +test_multi_port_queue_atomic_to_parallel(void) +{ + return test_multiport_queue_sched_type_test(RTE_SCHED_TYPE_ATOMIC, + RTE_SCHED_TYPE_PARALLEL); +} + +static int +test_multi_port_queue_parallel_to_atomic(void) +{ + return test_multiport_queue_sched_type_test(RTE_SCHED_TYPE_PARALLEL, + RTE_SCHED_TYPE_ATOMIC); +} + +static int +test_multi_port_queue_parallel_to_ordered(void) +{ + return test_multiport_queue_sched_type_test(RTE_SCHED_TYPE_PARALLEL, + RTE_SCHED_TYPE_ORDERED); +} + +static int +test_multi_port_queue_parallel_to_parallel(void) +{ + return test_multiport_queue_sched_type_test(RTE_SCHED_TYPE_PARALLEL, + RTE_SCHED_TYPE_PARALLEL); +} + +static int +worker_flow_based_pipeline_max_stages_rand_sched_type(void *arg) +{ + struct test_core_param *param = arg; + struct rte_event ev; + uint16_t valid_event; + uint8_t port = param->port; + rte_atomic32_t *total_events = param->total_events; + + while (rte_atomic32_read(total_events) > 0) { + valid_event = rte_event_dequeue_burst(evdev, port, &ev, 1, 0); + if (!valid_event) + continue; + + if (ev.sub_event_type == 255) { /* last stage */ + rte_pktmbuf_free(ev.mbuf); + rte_atomic32_sub(total_events, 1); + } else { + ev.event_type = RTE_EVENT_TYPE_CPU; + ev.sub_event_type++; + ev.sched_type = + rte_rand() % (RTE_SCHED_TYPE_PARALLEL + 1); + ev.op = RTE_EVENT_OP_FORWARD; + rte_event_enqueue_burst(evdev, port, &ev, 1); + } + } + return 0; +} + +static int +launch_multi_port_max_stages_random_sched_type(int (*fn)(void *)) +{ + uint32_t nr_ports; + int ret; + + TEST_ASSERT_SUCCESS(rte_event_dev_attr_get(evdev, + RTE_EVENT_DEV_ATTR_PORT_COUNT, + &nr_ports), "Port count get failed"); + nr_ports = RTE_MIN(nr_ports, rte_lcore_count() - 1); + + if (!nr_ports) { + printf("%s: Not enough ports=%d or workers=%d\n", __func__, + nr_ports, rte_lcore_count() - 1); + return TEST_SUCCESS; + } + + /* Injects events with m->seqn=0 to total_events */ + ret = inject_events( + 0x1 /*flow_id */, + RTE_EVENT_TYPE_CPU /* event_type */, + 0 /* sub_event_type (stage 0) */, + rte_rand() % (RTE_SCHED_TYPE_PARALLEL + 1) /* sched_type */, + 0 /* queue */, + 0 /* port */, + MAX_EVENTS /* events */); + if (ret) + return TEST_FAILED; + + return launch_workers_and_wait(fn, fn, MAX_EVENTS, nr_ports, + 0xff /* invalid */); +} + +/* Flow based pipeline with maximum stages with random sched type */ +static int +test_multi_port_flow_max_stages_random_sched_type(void) +{ + return launch_multi_port_max_stages_random_sched_type( + worker_flow_based_pipeline_max_stages_rand_sched_type); +} + +static int +worker_queue_based_pipeline_max_stages_rand_sched_type(void *arg) +{ + struct test_core_param *param = arg; + struct rte_event ev; + uint16_t valid_event; + uint8_t port = param->port; + uint32_t queue_count; + TEST_ASSERT_SUCCESS(rte_event_dev_attr_get(evdev, + RTE_EVENT_DEV_ATTR_QUEUE_COUNT, + &queue_count), "Queue count get failed"); + uint8_t nr_queues = queue_count; + rte_atomic32_t *total_events = param->total_events; + + while (rte_atomic32_read(total_events) > 0) { + valid_event = rte_event_dequeue_burst(evdev, port, &ev, 1, 0); + if (!valid_event) + continue; + + if (ev.queue_id == nr_queues - 1) { /* last stage */ + rte_pktmbuf_free(ev.mbuf); + rte_atomic32_sub(total_events, 1); + } else { + ev.event_type = RTE_EVENT_TYPE_CPU; + ev.queue_id++; + ev.sched_type = + rte_rand() % (RTE_SCHED_TYPE_PARALLEL + 1); + ev.op = RTE_EVENT_OP_FORWARD; + rte_event_enqueue_burst(evdev, port, &ev, 1); + } + } + return 0; +} + +/* Queue based pipeline with maximum stages with random sched type */ +static int +test_multi_port_queue_max_stages_random_sched_type(void) +{ + return launch_multi_port_max_stages_random_sched_type( + worker_queue_based_pipeline_max_stages_rand_sched_type); +} + +static int +worker_mixed_pipeline_max_stages_rand_sched_type(void *arg) +{ + struct test_core_param *param = arg; + struct rte_event ev; + uint16_t valid_event; + uint8_t port = param->port; + uint32_t queue_count; + TEST_ASSERT_SUCCESS(rte_event_dev_attr_get(evdev, + RTE_EVENT_DEV_ATTR_QUEUE_COUNT, + &queue_count), "Queue count get failed"); + uint8_t nr_queues = queue_count; + rte_atomic32_t *total_events = param->total_events; + + while (rte_atomic32_read(total_events) > 0) { + valid_event = rte_event_dequeue_burst(evdev, port, &ev, 1, 0); + if (!valid_event) + continue; + + if (ev.queue_id == nr_queues - 1) { /* Last stage */ + rte_pktmbuf_free(ev.mbuf); + rte_atomic32_sub(total_events, 1); + } else { + ev.event_type = RTE_EVENT_TYPE_CPU; + ev.queue_id++; + ev.sub_event_type = rte_rand() % 256; + ev.sched_type = + rte_rand() % (RTE_SCHED_TYPE_PARALLEL + 1); + ev.op = RTE_EVENT_OP_FORWARD; + rte_event_enqueue_burst(evdev, port, &ev, 1); + } + } + return 0; +} + +/* Queue and flow based pipeline with maximum stages with random sched type */ +static int +test_multi_port_mixed_max_stages_random_sched_type(void) +{ + return launch_multi_port_max_stages_random_sched_type( + worker_mixed_pipeline_max_stages_rand_sched_type); +} + +static int +worker_ordered_flow_producer(void *arg) +{ + struct test_core_param *param = arg; + uint8_t port = param->port; + struct rte_mbuf *m; + int counter = 0; + + while (counter < NUM_PACKETS) { + m = rte_pktmbuf_alloc(eventdev_test_mempool); + if (m == NULL) + continue; + + m->seqn = counter++; + + struct rte_event ev = {.event = 0, .u64 = 0}; + + ev.flow_id = 0x1; /* Generate a fat flow */ + ev.sub_event_type = 0; + /* Inject the new event */ + ev.op = RTE_EVENT_OP_NEW; + ev.event_type = RTE_EVENT_TYPE_CPU; + ev.sched_type = RTE_SCHED_TYPE_ORDERED; + ev.queue_id = 0; + ev.mbuf = m; + rte_event_enqueue_burst(evdev, port, &ev, 1); + } + + return 0; +} + +static inline int +test_producer_consumer_ingress_order_test(int (*fn)(void *)) +{ + uint32_t nr_ports; + + TEST_ASSERT_SUCCESS(rte_event_dev_attr_get(evdev, + RTE_EVENT_DEV_ATTR_PORT_COUNT, + &nr_ports), "Port count get failed"); + nr_ports = RTE_MIN(nr_ports, rte_lcore_count() - 1); + + if (rte_lcore_count() < 3 || nr_ports < 2) { + printf("### Not enough cores for %s test.\n", __func__); + return TEST_SUCCESS; + } + + launch_workers_and_wait(worker_ordered_flow_producer, fn, + NUM_PACKETS, nr_ports, RTE_SCHED_TYPE_ATOMIC); + /* Check the events order maintained or not */ + return seqn_list_check(NUM_PACKETS); +} + +/* Flow based producer consumer ingress order test */ +static int +test_flow_producer_consumer_ingress_order_test(void) +{ + return test_producer_consumer_ingress_order_test( + worker_flow_based_pipeline); +} + +/* Queue based producer consumer ingress order test */ +static int +test_queue_producer_consumer_ingress_order_test(void) +{ + return test_producer_consumer_ingress_order_test( + worker_group_based_pipeline); +} + +static struct unit_test_suite eventdev_octeontx_testsuite = { + .suite_name = "eventdev octeontx unit test suite", + .setup = testsuite_setup, + .teardown = testsuite_teardown, + .unit_test_cases = { + TEST_CASE_ST(eventdev_setup, eventdev_teardown, + test_simple_enqdeq_ordered), + TEST_CASE_ST(eventdev_setup, eventdev_teardown, + test_simple_enqdeq_atomic), + TEST_CASE_ST(eventdev_setup, eventdev_teardown, + test_simple_enqdeq_parallel), + TEST_CASE_ST(eventdev_setup, eventdev_teardown, + test_multi_queue_enq_single_port_deq), + TEST_CASE_ST(eventdev_setup_priority, eventdev_teardown, + test_multi_queue_priority), + TEST_CASE_ST(eventdev_setup, eventdev_teardown, + test_multi_queue_enq_multi_port_deq), + TEST_CASE_ST(eventdev_setup, eventdev_teardown, + test_queue_to_port_single_link), + TEST_CASE_ST(eventdev_setup, eventdev_teardown, + test_queue_to_port_multi_link), + TEST_CASE_ST(eventdev_setup, eventdev_teardown, + test_multi_port_flow_ordered_to_atomic), + TEST_CASE_ST(eventdev_setup, eventdev_teardown, + test_multi_port_flow_ordered_to_ordered), + TEST_CASE_ST(eventdev_setup, eventdev_teardown, + test_multi_port_flow_ordered_to_parallel), + TEST_CASE_ST(eventdev_setup, eventdev_teardown, + test_multi_port_flow_atomic_to_atomic), + TEST_CASE_ST(eventdev_setup, eventdev_teardown, + test_multi_port_flow_atomic_to_ordered), + TEST_CASE_ST(eventdev_setup, eventdev_teardown, + test_multi_port_flow_atomic_to_parallel), + TEST_CASE_ST(eventdev_setup, eventdev_teardown, + test_multi_port_flow_parallel_to_atomic), + TEST_CASE_ST(eventdev_setup, eventdev_teardown, + test_multi_port_flow_parallel_to_ordered), + TEST_CASE_ST(eventdev_setup, eventdev_teardown, + test_multi_port_flow_parallel_to_parallel), + TEST_CASE_ST(eventdev_setup, eventdev_teardown, + test_multi_port_queue_ordered_to_atomic), + TEST_CASE_ST(eventdev_setup, eventdev_teardown, + test_multi_port_queue_ordered_to_ordered), + TEST_CASE_ST(eventdev_setup, eventdev_teardown, + test_multi_port_queue_ordered_to_parallel), + TEST_CASE_ST(eventdev_setup, eventdev_teardown, + test_multi_port_queue_atomic_to_atomic), + TEST_CASE_ST(eventdev_setup, eventdev_teardown, + test_multi_port_queue_atomic_to_ordered), + TEST_CASE_ST(eventdev_setup, eventdev_teardown, + test_multi_port_queue_atomic_to_parallel), + TEST_CASE_ST(eventdev_setup, eventdev_teardown, + test_multi_port_queue_parallel_to_atomic), + TEST_CASE_ST(eventdev_setup, eventdev_teardown, + test_multi_port_queue_parallel_to_ordered), + TEST_CASE_ST(eventdev_setup, eventdev_teardown, + test_multi_port_queue_parallel_to_parallel), + TEST_CASE_ST(eventdev_setup, eventdev_teardown, + test_multi_port_flow_max_stages_random_sched_type), + TEST_CASE_ST(eventdev_setup, eventdev_teardown, + test_multi_port_queue_max_stages_random_sched_type), + TEST_CASE_ST(eventdev_setup, eventdev_teardown, + test_multi_port_mixed_max_stages_random_sched_type), + TEST_CASE_ST(eventdev_setup, eventdev_teardown, + test_flow_producer_consumer_ingress_order_test), + TEST_CASE_ST(eventdev_setup, eventdev_teardown, + test_queue_producer_consumer_ingress_order_test), + /* Tests with dequeue timeout */ + TEST_CASE_ST(eventdev_setup_dequeue_timeout, eventdev_teardown, + test_multi_port_flow_ordered_to_atomic), + TEST_CASE_ST(eventdev_setup_dequeue_timeout, eventdev_teardown, + test_multi_port_queue_ordered_to_atomic), + TEST_CASES_END() /**< NULL terminate unit test array */ + } +}; + +static int +test_eventdev_octeontx(void) +{ + return unit_test_suite_runner(&eventdev_octeontx_testsuite); +} + +REGISTER_TEST_COMMAND(eventdev_octeontx_autotest, test_eventdev_octeontx); diff --git a/dpdk/test/test/test_eventdev_sw.c b/dpdk/test/test/test_eventdev_sw.c new file mode 100644 index 00000000..f524b6f8 --- /dev/null +++ b/dpdk/test/test/test_eventdev_sw.c @@ -0,0 +1,3261 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <string.h> +#include <stdint.h> +#include <errno.h> +#include <unistd.h> +#include <sys/queue.h> + +#include <rte_memory.h> +#include <rte_launch.h> +#include <rte_eal.h> +#include <rte_per_lcore.h> +#include <rte_lcore.h> +#include <rte_debug.h> +#include <rte_ethdev.h> +#include <rte_cycles.h> +#include <rte_eventdev.h> +#include <rte_pause.h> +#include <rte_service.h> +#include <rte_service_component.h> +#include <rte_bus_vdev.h> + +#include "test.h" + +#define MAX_PORTS 16 +#define MAX_QIDS 16 +#define NUM_PACKETS (1<<18) + +static int evdev; + +struct test { + struct rte_mempool *mbuf_pool; + uint8_t port[MAX_PORTS]; + uint8_t qid[MAX_QIDS]; + int nb_qids; + uint32_t service_id; +}; + +static struct rte_event release_ev; + +static inline struct rte_mbuf * +rte_gen_arp(int portid, struct rte_mempool *mp) +{ + /* + * len = 14 + 46 + * ARP, Request who-has 10.0.0.1 tell 10.0.0.2, length 46 + */ + static const uint8_t arp_request[] = { + /*0x0000:*/ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xec, 0xa8, + 0x6b, 0xfd, 0x02, 0x29, 0x08, 0x06, 0x00, 0x01, + /*0x0010:*/ 0x08, 0x00, 0x06, 0x04, 0x00, 0x01, 0xec, 0xa8, + 0x6b, 0xfd, 0x02, 0x29, 0x0a, 0x00, 0x00, 0x01, + /*0x0020:*/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0a, 0x00, + 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + /*0x0030:*/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00 + }; + struct rte_mbuf *m; + int pkt_len = sizeof(arp_request) - 1; + + m = rte_pktmbuf_alloc(mp); + if (!m) + return 0; + + memcpy((void *)((uintptr_t)m->buf_addr + m->data_off), + arp_request, pkt_len); + rte_pktmbuf_pkt_len(m) = pkt_len; + rte_pktmbuf_data_len(m) = pkt_len; + + RTE_SET_USED(portid); + + return m; +} + +static void +xstats_print(void) +{ + const uint32_t XSTATS_MAX = 1024; + uint32_t i; + uint32_t ids[XSTATS_MAX]; + uint64_t values[XSTATS_MAX]; + struct rte_event_dev_xstats_name xstats_names[XSTATS_MAX]; + + for (i = 0; i < XSTATS_MAX; i++) + ids[i] = i; + + /* Device names / values */ + int ret = rte_event_dev_xstats_names_get(evdev, + RTE_EVENT_DEV_XSTATS_DEVICE, 0, + xstats_names, ids, XSTATS_MAX); + if (ret < 0) { + printf("%d: xstats names get() returned error\n", + __LINE__); + return; + } + ret = rte_event_dev_xstats_get(evdev, + RTE_EVENT_DEV_XSTATS_DEVICE, + 0, ids, values, ret); + if (ret > (signed int)XSTATS_MAX) + printf("%s %d: more xstats available than space\n", + __func__, __LINE__); + for (i = 0; (signed int)i < ret; i++) { + printf("%d : %s : %"PRIu64"\n", + i, xstats_names[i].name, values[i]); + } + + /* Port names / values */ + ret = rte_event_dev_xstats_names_get(evdev, + RTE_EVENT_DEV_XSTATS_PORT, 0, + xstats_names, ids, XSTATS_MAX); + ret = rte_event_dev_xstats_get(evdev, + RTE_EVENT_DEV_XSTATS_PORT, 1, + ids, values, ret); + if (ret > (signed int)XSTATS_MAX) + printf("%s %d: more xstats available than space\n", + __func__, __LINE__); + for (i = 0; (signed int)i < ret; i++) { + printf("%d : %s : %"PRIu64"\n", + i, xstats_names[i].name, values[i]); + } + + /* Queue names / values */ + ret = rte_event_dev_xstats_names_get(evdev, + RTE_EVENT_DEV_XSTATS_QUEUE, 0, + xstats_names, ids, XSTATS_MAX); + ret = rte_event_dev_xstats_get(evdev, + RTE_EVENT_DEV_XSTATS_QUEUE, + 1, ids, values, ret); + if (ret > (signed int)XSTATS_MAX) + printf("%s %d: more xstats available than space\n", + __func__, __LINE__); + for (i = 0; (signed int)i < ret; i++) { + printf("%d : %s : %"PRIu64"\n", + i, xstats_names[i].name, values[i]); + } +} + +/* initialization and config */ +static inline int +init(struct test *t, int nb_queues, int nb_ports) +{ + struct rte_event_dev_config config = { + .nb_event_queues = nb_queues, + .nb_event_ports = nb_ports, + .nb_event_queue_flows = 1024, + .nb_events_limit = 4096, + .nb_event_port_dequeue_depth = 128, + .nb_event_port_enqueue_depth = 128, + }; + int ret; + + void *temp = t->mbuf_pool; /* save and restore mbuf pool */ + + memset(t, 0, sizeof(*t)); + t->mbuf_pool = temp; + + ret = rte_event_dev_configure(evdev, &config); + if (ret < 0) + printf("%d: Error configuring device\n", __LINE__); + return ret; +}; + +static inline int +create_ports(struct test *t, int num_ports) +{ + int i; + static const struct rte_event_port_conf conf = { + .new_event_threshold = 1024, + .dequeue_depth = 32, + .enqueue_depth = 64, + }; + if (num_ports > MAX_PORTS) + return -1; + + for (i = 0; i < num_ports; i++) { + if (rte_event_port_setup(evdev, i, &conf) < 0) { + printf("Error setting up port %d\n", i); + return -1; + } + t->port[i] = i; + } + + return 0; +} + +static inline int +create_lb_qids(struct test *t, int num_qids, uint32_t flags) +{ + int i; + + /* Q creation */ + const struct rte_event_queue_conf conf = { + .schedule_type = flags, + .priority = RTE_EVENT_DEV_PRIORITY_NORMAL, + .nb_atomic_flows = 1024, + .nb_atomic_order_sequences = 1024, + }; + + for (i = t->nb_qids; i < t->nb_qids + num_qids; i++) { + if (rte_event_queue_setup(evdev, i, &conf) < 0) { + printf("%d: error creating qid %d\n", __LINE__, i); + return -1; + } + t->qid[i] = i; + } + t->nb_qids += num_qids; + if (t->nb_qids > MAX_QIDS) + return -1; + + return 0; +} + +static inline int +create_atomic_qids(struct test *t, int num_qids) +{ + return create_lb_qids(t, num_qids, RTE_SCHED_TYPE_ATOMIC); +} + +static inline int +create_ordered_qids(struct test *t, int num_qids) +{ + return create_lb_qids(t, num_qids, RTE_SCHED_TYPE_ORDERED); +} + + +static inline int +create_unordered_qids(struct test *t, int num_qids) +{ + return create_lb_qids(t, num_qids, RTE_SCHED_TYPE_PARALLEL); +} + +static inline int +create_directed_qids(struct test *t, int num_qids, const uint8_t ports[]) +{ + int i; + + /* Q creation */ + static const struct rte_event_queue_conf conf = { + .priority = RTE_EVENT_DEV_PRIORITY_NORMAL, + .event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK, + }; + + for (i = t->nb_qids; i < t->nb_qids + num_qids; i++) { + if (rte_event_queue_setup(evdev, i, &conf) < 0) { + printf("%d: error creating qid %d\n", __LINE__, i); + return -1; + } + t->qid[i] = i; + + if (rte_event_port_link(evdev, ports[i - t->nb_qids], + &t->qid[i], NULL, 1) != 1) { + printf("%d: error creating link for qid %d\n", + __LINE__, i); + return -1; + } + } + t->nb_qids += num_qids; + if (t->nb_qids > MAX_QIDS) + return -1; + + return 0; +} + +/* destruction */ +static inline int +cleanup(struct test *t __rte_unused) +{ + rte_event_dev_stop(evdev); + rte_event_dev_close(evdev); + return 0; +}; + +struct test_event_dev_stats { + uint64_t rx_pkts; /**< Total packets received */ + uint64_t rx_dropped; /**< Total packets dropped (Eg Invalid QID) */ + uint64_t tx_pkts; /**< Total packets transmitted */ + + /** Packets received on this port */ + uint64_t port_rx_pkts[MAX_PORTS]; + /** Packets dropped on this port */ + uint64_t port_rx_dropped[MAX_PORTS]; + /** Packets inflight on this port */ + uint64_t port_inflight[MAX_PORTS]; + /** Packets transmitted on this port */ + uint64_t port_tx_pkts[MAX_PORTS]; + /** Packets received on this qid */ + uint64_t qid_rx_pkts[MAX_QIDS]; + /** Packets dropped on this qid */ + uint64_t qid_rx_dropped[MAX_QIDS]; + /** Packets transmitted on this qid */ + uint64_t qid_tx_pkts[MAX_QIDS]; +}; + +static inline int +test_event_dev_stats_get(int dev_id, struct test_event_dev_stats *stats) +{ + static uint32_t i; + static uint32_t total_ids[3]; /* rx, tx and drop */ + static uint32_t port_rx_pkts_ids[MAX_PORTS]; + static uint32_t port_rx_dropped_ids[MAX_PORTS]; + static uint32_t port_inflight_ids[MAX_PORTS]; + static uint32_t port_tx_pkts_ids[MAX_PORTS]; + static uint32_t qid_rx_pkts_ids[MAX_QIDS]; + static uint32_t qid_rx_dropped_ids[MAX_QIDS]; + static uint32_t qid_tx_pkts_ids[MAX_QIDS]; + + + stats->rx_pkts = rte_event_dev_xstats_by_name_get(dev_id, + "dev_rx", &total_ids[0]); + stats->rx_dropped = rte_event_dev_xstats_by_name_get(dev_id, + "dev_drop", &total_ids[1]); + stats->tx_pkts = rte_event_dev_xstats_by_name_get(dev_id, + "dev_tx", &total_ids[2]); + for (i = 0; i < MAX_PORTS; i++) { + char name[32]; + snprintf(name, sizeof(name), "port_%u_rx", i); + stats->port_rx_pkts[i] = rte_event_dev_xstats_by_name_get( + dev_id, name, &port_rx_pkts_ids[i]); + snprintf(name, sizeof(name), "port_%u_drop", i); + stats->port_rx_dropped[i] = rte_event_dev_xstats_by_name_get( + dev_id, name, &port_rx_dropped_ids[i]); + snprintf(name, sizeof(name), "port_%u_inflight", i); + stats->port_inflight[i] = rte_event_dev_xstats_by_name_get( + dev_id, name, &port_inflight_ids[i]); + snprintf(name, sizeof(name), "port_%u_tx", i); + stats->port_tx_pkts[i] = rte_event_dev_xstats_by_name_get( + dev_id, name, &port_tx_pkts_ids[i]); + } + for (i = 0; i < MAX_QIDS; i++) { + char name[32]; + snprintf(name, sizeof(name), "qid_%u_rx", i); + stats->qid_rx_pkts[i] = rte_event_dev_xstats_by_name_get( + dev_id, name, &qid_rx_pkts_ids[i]); + snprintf(name, sizeof(name), "qid_%u_drop", i); + stats->qid_rx_dropped[i] = rte_event_dev_xstats_by_name_get( + dev_id, name, &qid_rx_dropped_ids[i]); + snprintf(name, sizeof(name), "qid_%u_tx", i); + stats->qid_tx_pkts[i] = rte_event_dev_xstats_by_name_get( + dev_id, name, &qid_tx_pkts_ids[i]); + } + + return 0; +} + +/* run_prio_packet_test + * This performs a basic packet priority check on the test instance passed in. + * It is factored out of the main priority tests as the same tests must be + * performed to ensure prioritization of each type of QID. + * + * Requirements: + * - An initialized test structure, including mempool + * - t->port[0] is initialized for both Enq / Deq of packets to the QID + * - t->qid[0] is the QID to be tested + * - if LB QID, the CQ must be mapped to the QID. + */ +static int +run_prio_packet_test(struct test *t) +{ + int err; + const uint32_t MAGIC_SEQN[] = {4711, 1234}; + const uint32_t PRIORITY[] = { + RTE_EVENT_DEV_PRIORITY_NORMAL, + RTE_EVENT_DEV_PRIORITY_HIGHEST + }; + unsigned int i; + for (i = 0; i < RTE_DIM(MAGIC_SEQN); i++) { + /* generate pkt and enqueue */ + struct rte_event ev; + struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool); + if (!arp) { + printf("%d: gen of pkt failed\n", __LINE__); + return -1; + } + arp->seqn = MAGIC_SEQN[i]; + + ev = (struct rte_event){ + .priority = PRIORITY[i], + .op = RTE_EVENT_OP_NEW, + .queue_id = t->qid[0], + .mbuf = arp + }; + err = rte_event_enqueue_burst(evdev, t->port[0], &ev, 1); + if (err < 0) { + printf("%d: error failed to enqueue\n", __LINE__); + return -1; + } + } + + rte_service_run_iter_on_app_lcore(t->service_id, 1); + + struct test_event_dev_stats stats; + err = test_event_dev_stats_get(evdev, &stats); + if (err) { + printf("%d: error failed to get stats\n", __LINE__); + return -1; + } + + if (stats.port_rx_pkts[t->port[0]] != 2) { + printf("%d: error stats incorrect for directed port\n", + __LINE__); + rte_event_dev_dump(evdev, stdout); + return -1; + } + + struct rte_event ev, ev2; + uint32_t deq_pkts; + deq_pkts = rte_event_dequeue_burst(evdev, t->port[0], &ev, 1, 0); + if (deq_pkts != 1) { + printf("%d: error failed to deq\n", __LINE__); + rte_event_dev_dump(evdev, stdout); + return -1; + } + if (ev.mbuf->seqn != MAGIC_SEQN[1]) { + printf("%d: first packet out not highest priority\n", + __LINE__); + rte_event_dev_dump(evdev, stdout); + return -1; + } + rte_pktmbuf_free(ev.mbuf); + + deq_pkts = rte_event_dequeue_burst(evdev, t->port[0], &ev2, 1, 0); + if (deq_pkts != 1) { + printf("%d: error failed to deq\n", __LINE__); + rte_event_dev_dump(evdev, stdout); + return -1; + } + if (ev2.mbuf->seqn != MAGIC_SEQN[0]) { + printf("%d: second packet out not lower priority\n", + __LINE__); + rte_event_dev_dump(evdev, stdout); + return -1; + } + rte_pktmbuf_free(ev2.mbuf); + + cleanup(t); + return 0; +} + +static int +test_single_directed_packet(struct test *t) +{ + const int rx_enq = 0; + const int wrk_enq = 2; + int err; + + /* Create instance with 3 directed QIDs going to 3 ports */ + if (init(t, 3, 3) < 0 || + create_ports(t, 3) < 0 || + create_directed_qids(t, 3, t->port) < 0) + return -1; + + if (rte_event_dev_start(evdev) < 0) { + printf("%d: Error with start call\n", __LINE__); + return -1; + } + + /************** FORWARD ****************/ + struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool); + struct rte_event ev = { + .op = RTE_EVENT_OP_NEW, + .queue_id = wrk_enq, + .mbuf = arp, + }; + + if (!arp) { + printf("%d: gen of pkt failed\n", __LINE__); + return -1; + } + + const uint32_t MAGIC_SEQN = 4711; + arp->seqn = MAGIC_SEQN; + + /* generate pkt and enqueue */ + err = rte_event_enqueue_burst(evdev, rx_enq, &ev, 1); + if (err < 0) { + printf("%d: error failed to enqueue\n", __LINE__); + return -1; + } + + /* Run schedule() as dir packets may need to be re-ordered */ + rte_service_run_iter_on_app_lcore(t->service_id, 1); + + struct test_event_dev_stats stats; + err = test_event_dev_stats_get(evdev, &stats); + if (err) { + printf("%d: error failed to get stats\n", __LINE__); + return -1; + } + + if (stats.port_rx_pkts[rx_enq] != 1) { + printf("%d: error stats incorrect for directed port\n", + __LINE__); + return -1; + } + + uint32_t deq_pkts; + deq_pkts = rte_event_dequeue_burst(evdev, wrk_enq, &ev, 1, 0); + if (deq_pkts != 1) { + printf("%d: error failed to deq\n", __LINE__); + return -1; + } + + err = test_event_dev_stats_get(evdev, &stats); + if (stats.port_rx_pkts[wrk_enq] != 0 && + stats.port_rx_pkts[wrk_enq] != 1) { + printf("%d: error directed stats post-dequeue\n", __LINE__); + return -1; + } + + if (ev.mbuf->seqn != MAGIC_SEQN) { + printf("%d: error magic sequence number not dequeued\n", + __LINE__); + return -1; + } + + rte_pktmbuf_free(ev.mbuf); + cleanup(t); + return 0; +} + +static int +test_directed_forward_credits(struct test *t) +{ + uint32_t i; + int32_t err; + + if (init(t, 1, 1) < 0 || + create_ports(t, 1) < 0 || + create_directed_qids(t, 1, t->port) < 0) + return -1; + + if (rte_event_dev_start(evdev) < 0) { + printf("%d: Error with start call\n", __LINE__); + return -1; + } + + struct rte_event ev = { + .op = RTE_EVENT_OP_NEW, + .queue_id = 0, + }; + + for (i = 0; i < 1000; i++) { + err = rte_event_enqueue_burst(evdev, 0, &ev, 1); + if (err < 0) { + printf("%d: error failed to enqueue\n", __LINE__); + return -1; + } + rte_service_run_iter_on_app_lcore(t->service_id, 1); + + uint32_t deq_pkts; + deq_pkts = rte_event_dequeue_burst(evdev, 0, &ev, 1, 0); + if (deq_pkts != 1) { + printf("%d: error failed to deq\n", __LINE__); + return -1; + } + + /* re-write event to be a forward, and continue looping it */ + ev.op = RTE_EVENT_OP_FORWARD; + } + + cleanup(t); + return 0; +} + + +static int +test_priority_directed(struct test *t) +{ + if (init(t, 1, 1) < 0 || + create_ports(t, 1) < 0 || + create_directed_qids(t, 1, t->port) < 0) { + printf("%d: Error initializing device\n", __LINE__); + return -1; + } + + if (rte_event_dev_start(evdev) < 0) { + printf("%d: Error with start call\n", __LINE__); + return -1; + } + + return run_prio_packet_test(t); +} + +static int +test_priority_atomic(struct test *t) +{ + if (init(t, 1, 1) < 0 || + create_ports(t, 1) < 0 || + create_atomic_qids(t, 1) < 0) { + printf("%d: Error initializing device\n", __LINE__); + return -1; + } + + /* map the QID */ + if (rte_event_port_link(evdev, t->port[0], &t->qid[0], NULL, 1) != 1) { + printf("%d: error mapping qid to port\n", __LINE__); + return -1; + } + if (rte_event_dev_start(evdev) < 0) { + printf("%d: Error with start call\n", __LINE__); + return -1; + } + + return run_prio_packet_test(t); +} + +static int +test_priority_ordered(struct test *t) +{ + if (init(t, 1, 1) < 0 || + create_ports(t, 1) < 0 || + create_ordered_qids(t, 1) < 0) { + printf("%d: Error initializing device\n", __LINE__); + return -1; + } + + /* map the QID */ + if (rte_event_port_link(evdev, t->port[0], &t->qid[0], NULL, 1) != 1) { + printf("%d: error mapping qid to port\n", __LINE__); + return -1; + } + if (rte_event_dev_start(evdev) < 0) { + printf("%d: Error with start call\n", __LINE__); + return -1; + } + + return run_prio_packet_test(t); +} + +static int +test_priority_unordered(struct test *t) +{ + if (init(t, 1, 1) < 0 || + create_ports(t, 1) < 0 || + create_unordered_qids(t, 1) < 0) { + printf("%d: Error initializing device\n", __LINE__); + return -1; + } + + /* map the QID */ + if (rte_event_port_link(evdev, t->port[0], &t->qid[0], NULL, 1) != 1) { + printf("%d: error mapping qid to port\n", __LINE__); + return -1; + } + if (rte_event_dev_start(evdev) < 0) { + printf("%d: Error with start call\n", __LINE__); + return -1; + } + + return run_prio_packet_test(t); +} + +static int +burst_packets(struct test *t) +{ + /************** CONFIG ****************/ + uint32_t i; + int err; + int ret; + + /* Create instance with 2 ports and 2 queues */ + if (init(t, 2, 2) < 0 || + create_ports(t, 2) < 0 || + create_atomic_qids(t, 2) < 0) { + printf("%d: Error initializing device\n", __LINE__); + return -1; + } + + /* CQ mapping to QID */ + ret = rte_event_port_link(evdev, t->port[0], &t->qid[0], NULL, 1); + if (ret != 1) { + printf("%d: error mapping lb qid0\n", __LINE__); + return -1; + } + ret = rte_event_port_link(evdev, t->port[1], &t->qid[1], NULL, 1); + if (ret != 1) { + printf("%d: error mapping lb qid1\n", __LINE__); + return -1; + } + + if (rte_event_dev_start(evdev) < 0) { + printf("%d: Error with start call\n", __LINE__); + return -1; + } + + /************** FORWARD ****************/ + const uint32_t rx_port = 0; + const uint32_t NUM_PKTS = 2; + + for (i = 0; i < NUM_PKTS; i++) { + struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool); + if (!arp) { + printf("%d: error generating pkt\n", __LINE__); + return -1; + } + + struct rte_event ev = { + .op = RTE_EVENT_OP_NEW, + .queue_id = i % 2, + .flow_id = i % 3, + .mbuf = arp, + }; + /* generate pkt and enqueue */ + err = rte_event_enqueue_burst(evdev, t->port[rx_port], &ev, 1); + if (err < 0) { + printf("%d: Failed to enqueue\n", __LINE__); + return -1; + } + } + rte_service_run_iter_on_app_lcore(t->service_id, 1); + + /* Check stats for all NUM_PKTS arrived to sched core */ + struct test_event_dev_stats stats; + + err = test_event_dev_stats_get(evdev, &stats); + if (err) { + printf("%d: failed to get stats\n", __LINE__); + return -1; + } + if (stats.rx_pkts != NUM_PKTS || stats.tx_pkts != NUM_PKTS) { + printf("%d: Sched core didn't receive all %d pkts\n", + __LINE__, NUM_PKTS); + rte_event_dev_dump(evdev, stdout); + return -1; + } + + uint32_t deq_pkts; + int p; + + deq_pkts = 0; + /******** DEQ QID 1 *******/ + do { + struct rte_event ev; + p = rte_event_dequeue_burst(evdev, t->port[0], &ev, 1, 0); + deq_pkts += p; + rte_pktmbuf_free(ev.mbuf); + } while (p); + + if (deq_pkts != NUM_PKTS/2) { + printf("%d: Half of NUM_PKTS didn't arrive at port 1\n", + __LINE__); + return -1; + } + + /******** DEQ QID 2 *******/ + deq_pkts = 0; + do { + struct rte_event ev; + p = rte_event_dequeue_burst(evdev, t->port[1], &ev, 1, 0); + deq_pkts += p; + rte_pktmbuf_free(ev.mbuf); + } while (p); + if (deq_pkts != NUM_PKTS/2) { + printf("%d: Half of NUM_PKTS didn't arrive at port 2\n", + __LINE__); + return -1; + } + + cleanup(t); + return 0; +} + +static int +abuse_inflights(struct test *t) +{ + const int rx_enq = 0; + const int wrk_enq = 2; + int err; + + /* Create instance with 4 ports */ + if (init(t, 1, 4) < 0 || + create_ports(t, 4) < 0 || + create_atomic_qids(t, 1) < 0) { + printf("%d: Error initializing device\n", __LINE__); + return -1; + } + + /* CQ mapping to QID */ + err = rte_event_port_link(evdev, t->port[wrk_enq], NULL, NULL, 0); + if (err != 1) { + printf("%d: error mapping lb qid\n", __LINE__); + cleanup(t); + return -1; + } + + if (rte_event_dev_start(evdev) < 0) { + printf("%d: Error with start call\n", __LINE__); + return -1; + } + + /* Enqueue op only */ + err = rte_event_enqueue_burst(evdev, t->port[rx_enq], &release_ev, 1); + if (err < 0) { + printf("%d: Failed to enqueue\n", __LINE__); + return -1; + } + + /* schedule */ + rte_service_run_iter_on_app_lcore(t->service_id, 1); + + struct test_event_dev_stats stats; + + err = test_event_dev_stats_get(evdev, &stats); + if (err) { + printf("%d: failed to get stats\n", __LINE__); + return -1; + } + + if (stats.rx_pkts != 0 || + stats.tx_pkts != 0 || + stats.port_inflight[wrk_enq] != 0) { + printf("%d: Sched core didn't handle pkt as expected\n", + __LINE__); + return -1; + } + + cleanup(t); + return 0; +} + +static int +xstats_tests(struct test *t) +{ + const int wrk_enq = 2; + int err; + + /* Create instance with 4 ports */ + if (init(t, 1, 4) < 0 || + create_ports(t, 4) < 0 || + create_atomic_qids(t, 1) < 0) { + printf("%d: Error initializing device\n", __LINE__); + return -1; + } + + /* CQ mapping to QID */ + err = rte_event_port_link(evdev, t->port[wrk_enq], NULL, NULL, 0); + if (err != 1) { + printf("%d: error mapping lb qid\n", __LINE__); + cleanup(t); + return -1; + } + + if (rte_event_dev_start(evdev) < 0) { + printf("%d: Error with start call\n", __LINE__); + return -1; + } + + const uint32_t XSTATS_MAX = 1024; + + uint32_t i; + uint32_t ids[XSTATS_MAX]; + uint64_t values[XSTATS_MAX]; + struct rte_event_dev_xstats_name xstats_names[XSTATS_MAX]; + + for (i = 0; i < XSTATS_MAX; i++) + ids[i] = i; + + /* Device names / values */ + int ret = rte_event_dev_xstats_names_get(evdev, + RTE_EVENT_DEV_XSTATS_DEVICE, + 0, xstats_names, ids, XSTATS_MAX); + if (ret != 6) { + printf("%d: expected 6 stats, got return %d\n", __LINE__, ret); + return -1; + } + ret = rte_event_dev_xstats_get(evdev, + RTE_EVENT_DEV_XSTATS_DEVICE, + 0, ids, values, ret); + if (ret != 6) { + printf("%d: expected 6 stats, got return %d\n", __LINE__, ret); + return -1; + } + + /* Port names / values */ + ret = rte_event_dev_xstats_names_get(evdev, + RTE_EVENT_DEV_XSTATS_PORT, 0, + xstats_names, ids, XSTATS_MAX); + if (ret != 21) { + printf("%d: expected 21 stats, got return %d\n", __LINE__, ret); + return -1; + } + ret = rte_event_dev_xstats_get(evdev, + RTE_EVENT_DEV_XSTATS_PORT, 0, + ids, values, ret); + if (ret != 21) { + printf("%d: expected 21 stats, got return %d\n", __LINE__, ret); + return -1; + } + + /* Queue names / values */ + ret = rte_event_dev_xstats_names_get(evdev, + RTE_EVENT_DEV_XSTATS_QUEUE, + 0, xstats_names, ids, XSTATS_MAX); + if (ret != 17) { + printf("%d: expected 17 stats, got return %d\n", __LINE__, ret); + return -1; + } + + /* NEGATIVE TEST: with wrong queue passed, 0 stats should be returned */ + ret = rte_event_dev_xstats_get(evdev, + RTE_EVENT_DEV_XSTATS_QUEUE, + 1, ids, values, ret); + if (ret != -EINVAL) { + printf("%d: expected 0 stats, got return %d\n", __LINE__, ret); + return -1; + } + + ret = rte_event_dev_xstats_get(evdev, + RTE_EVENT_DEV_XSTATS_QUEUE, + 0, ids, values, ret); + if (ret != 17) { + printf("%d: expected 17 stats, got return %d\n", __LINE__, ret); + return -1; + } + + /* enqueue packets to check values */ + for (i = 0; i < 3; i++) { + struct rte_event ev; + struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool); + if (!arp) { + printf("%d: gen of pkt failed\n", __LINE__); + return -1; + } + ev.queue_id = t->qid[i]; + ev.op = RTE_EVENT_OP_NEW; + ev.mbuf = arp; + ev.flow_id = 7; + arp->seqn = i; + + int err = rte_event_enqueue_burst(evdev, t->port[0], &ev, 1); + if (err != 1) { + printf("%d: Failed to enqueue\n", __LINE__); + return -1; + } + } + + rte_service_run_iter_on_app_lcore(t->service_id, 1); + + /* Device names / values */ + int num_stats = rte_event_dev_xstats_names_get(evdev, + RTE_EVENT_DEV_XSTATS_DEVICE, 0, + xstats_names, ids, XSTATS_MAX); + if (num_stats < 0) + goto fail; + ret = rte_event_dev_xstats_get(evdev, + RTE_EVENT_DEV_XSTATS_DEVICE, + 0, ids, values, num_stats); + static const uint64_t expected[] = {3, 3, 0, 1, 0, 0}; + for (i = 0; (signed int)i < ret; i++) { + if (expected[i] != values[i]) { + printf( + "%d Error xstat %d (id %d) %s : %"PRIu64 + ", expect %"PRIu64"\n", + __LINE__, i, ids[i], xstats_names[i].name, + values[i], expected[i]); + goto fail; + } + } + + ret = rte_event_dev_xstats_reset(evdev, RTE_EVENT_DEV_XSTATS_DEVICE, + 0, NULL, 0); + + /* ensure reset statistics are zero-ed */ + static const uint64_t expected_zero[] = {0, 0, 0, 0, 0, 0}; + ret = rte_event_dev_xstats_get(evdev, + RTE_EVENT_DEV_XSTATS_DEVICE, + 0, ids, values, num_stats); + for (i = 0; (signed int)i < ret; i++) { + if (expected_zero[i] != values[i]) { + printf( + "%d Error, xstat %d (id %d) %s : %"PRIu64 + ", expect %"PRIu64"\n", + __LINE__, i, ids[i], xstats_names[i].name, + values[i], expected_zero[i]); + goto fail; + } + } + + /* port reset checks */ + num_stats = rte_event_dev_xstats_names_get(evdev, + RTE_EVENT_DEV_XSTATS_PORT, 0, + xstats_names, ids, XSTATS_MAX); + if (num_stats < 0) + goto fail; + ret = rte_event_dev_xstats_get(evdev, RTE_EVENT_DEV_XSTATS_PORT, + 0, ids, values, num_stats); + + static const uint64_t port_expected[] = { + 3 /* rx */, + 0 /* tx */, + 0 /* drop */, + 0 /* inflights */, + 0 /* avg pkt cycles */, + 29 /* credits */, + 0 /* rx ring used */, + 4096 /* rx ring free */, + 0 /* cq ring used */, + 32 /* cq ring free */, + 0 /* dequeue calls */, + /* 10 dequeue burst buckets */ + 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, + }; + if (ret != RTE_DIM(port_expected)) { + printf( + "%s %d: wrong number of port stats (%d), expected %zu\n", + __func__, __LINE__, ret, RTE_DIM(port_expected)); + } + + for (i = 0; (signed int)i < ret; i++) { + if (port_expected[i] != values[i]) { + printf( + "%s : %d: Error stat %s is %"PRIu64 + ", expected %"PRIu64"\n", + __func__, __LINE__, xstats_names[i].name, + values[i], port_expected[i]); + goto fail; + } + } + + ret = rte_event_dev_xstats_reset(evdev, RTE_EVENT_DEV_XSTATS_PORT, + 0, NULL, 0); + + /* ensure reset statistics are zero-ed */ + static const uint64_t port_expected_zero[] = { + 0 /* rx */, + 0 /* tx */, + 0 /* drop */, + 0 /* inflights */, + 0 /* avg pkt cycles */, + 29 /* credits */, + 0 /* rx ring used */, + 4096 /* rx ring free */, + 0 /* cq ring used */, + 32 /* cq ring free */, + 0 /* dequeue calls */, + /* 10 dequeue burst buckets */ + 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, + }; + ret = rte_event_dev_xstats_get(evdev, + RTE_EVENT_DEV_XSTATS_PORT, + 0, ids, values, num_stats); + for (i = 0; (signed int)i < ret; i++) { + if (port_expected_zero[i] != values[i]) { + printf( + "%d, Error, xstat %d (id %d) %s : %"PRIu64 + ", expect %"PRIu64"\n", + __LINE__, i, ids[i], xstats_names[i].name, + values[i], port_expected_zero[i]); + goto fail; + } + } + + /* QUEUE STATS TESTS */ + num_stats = rte_event_dev_xstats_names_get(evdev, + RTE_EVENT_DEV_XSTATS_QUEUE, 0, + xstats_names, ids, XSTATS_MAX); + ret = rte_event_dev_xstats_get(evdev, RTE_EVENT_DEV_XSTATS_QUEUE, + 0, ids, values, num_stats); + if (ret < 0) { + printf("xstats get returned %d\n", ret); + goto fail; + } + if ((unsigned int)ret > XSTATS_MAX) + printf("%s %d: more xstats available than space\n", + __func__, __LINE__); + + static const uint64_t queue_expected[] = { + 3 /* rx */, + 3 /* tx */, + 0 /* drop */, + 3 /* inflights */, + 512 /* iq size */, + 0, 0, 0, 0, /* iq 0, 1, 2, 3 used */ + /* QID-to-Port: pinned_flows, packets */ + 0, 0, + 0, 0, + 1, 3, + 0, 0, + }; + for (i = 0; (signed int)i < ret; i++) { + if (queue_expected[i] != values[i]) { + printf( + "%d, Error, xstat %d (id %d) %s : %"PRIu64 + ", expect %"PRIu64"\n", + __LINE__, i, ids[i], xstats_names[i].name, + values[i], queue_expected[i]); + goto fail; + } + } + + /* Reset the queue stats here */ + ret = rte_event_dev_xstats_reset(evdev, + RTE_EVENT_DEV_XSTATS_QUEUE, 0, + NULL, + 0); + + /* Verify that the resetable stats are reset, and others are not */ + static const uint64_t queue_expected_zero[] = { + 0 /* rx */, + 0 /* tx */, + 0 /* drop */, + 3 /* inflight */, + 512 /* iq size */, + 0, 0, 0, 0, /* 4 iq used */ + /* QID-to-Port: pinned_flows, packets */ + 0, 0, + 0, 0, + 1, 0, + 0, 0, + }; + + ret = rte_event_dev_xstats_get(evdev, RTE_EVENT_DEV_XSTATS_QUEUE, 0, + ids, values, num_stats); + int fails = 0; + for (i = 0; (signed int)i < ret; i++) { + if (queue_expected_zero[i] != values[i]) { + printf( + "%d, Error, xstat %d (id %d) %s : %"PRIu64 + ", expect %"PRIu64"\n", + __LINE__, i, ids[i], xstats_names[i].name, + values[i], queue_expected_zero[i]); + fails++; + } + } + if (fails) { + printf("%d : %d of values were not as expected above\n", + __LINE__, fails); + goto fail; + } + + cleanup(t); + return 0; + +fail: + rte_event_dev_dump(0, stdout); + cleanup(t); + return -1; +} + + +static int +xstats_id_abuse_tests(struct test *t) +{ + int err; + const uint32_t XSTATS_MAX = 1024; + const uint32_t link_port = 2; + + uint32_t ids[XSTATS_MAX]; + struct rte_event_dev_xstats_name xstats_names[XSTATS_MAX]; + + /* Create instance with 4 ports */ + if (init(t, 1, 4) < 0 || + create_ports(t, 4) < 0 || + create_atomic_qids(t, 1) < 0) { + printf("%d: Error initializing device\n", __LINE__); + goto fail; + } + + err = rte_event_port_link(evdev, t->port[link_port], NULL, NULL, 0); + if (err != 1) { + printf("%d: error mapping lb qid\n", __LINE__); + goto fail; + } + + if (rte_event_dev_start(evdev) < 0) { + printf("%d: Error with start call\n", __LINE__); + goto fail; + } + + /* no test for device, as it ignores the port/q number */ + int num_stats = rte_event_dev_xstats_names_get(evdev, + RTE_EVENT_DEV_XSTATS_PORT, + UINT8_MAX-1, xstats_names, ids, + XSTATS_MAX); + if (num_stats != 0) { + printf("%d: expected %d stats, got return %d\n", __LINE__, + 0, num_stats); + goto fail; + } + + num_stats = rte_event_dev_xstats_names_get(evdev, + RTE_EVENT_DEV_XSTATS_QUEUE, + UINT8_MAX-1, xstats_names, ids, + XSTATS_MAX); + if (num_stats != 0) { + printf("%d: expected %d stats, got return %d\n", __LINE__, + 0, num_stats); + goto fail; + } + + cleanup(t); + return 0; +fail: + cleanup(t); + return -1; +} + +static int +port_reconfig_credits(struct test *t) +{ + if (init(t, 1, 1) < 0) { + printf("%d: Error initializing device\n", __LINE__); + return -1; + } + + uint32_t i; + const uint32_t NUM_ITERS = 32; + for (i = 0; i < NUM_ITERS; i++) { + const struct rte_event_queue_conf conf = { + .schedule_type = RTE_SCHED_TYPE_ATOMIC, + .priority = RTE_EVENT_DEV_PRIORITY_NORMAL, + .nb_atomic_flows = 1024, + .nb_atomic_order_sequences = 1024, + }; + if (rte_event_queue_setup(evdev, 0, &conf) < 0) { + printf("%d: error creating qid\n", __LINE__); + return -1; + } + t->qid[0] = 0; + + static const struct rte_event_port_conf port_conf = { + .new_event_threshold = 128, + .dequeue_depth = 32, + .enqueue_depth = 64, + }; + if (rte_event_port_setup(evdev, 0, &port_conf) < 0) { + printf("%d Error setting up port\n", __LINE__); + return -1; + } + + int links = rte_event_port_link(evdev, 0, NULL, NULL, 0); + if (links != 1) { + printf("%d: error mapping lb qid\n", __LINE__); + goto fail; + } + + if (rte_event_dev_start(evdev) < 0) { + printf("%d: Error with start call\n", __LINE__); + goto fail; + } + + const uint32_t NPKTS = 1; + uint32_t j; + for (j = 0; j < NPKTS; j++) { + struct rte_event ev; + struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool); + if (!arp) { + printf("%d: gen of pkt failed\n", __LINE__); + goto fail; + } + ev.queue_id = t->qid[0]; + ev.op = RTE_EVENT_OP_NEW; + ev.mbuf = arp; + int err = rte_event_enqueue_burst(evdev, 0, &ev, 1); + if (err != 1) { + printf("%d: Failed to enqueue\n", __LINE__); + rte_event_dev_dump(0, stdout); + goto fail; + } + } + + rte_service_run_iter_on_app_lcore(t->service_id, 1); + + struct rte_event ev[NPKTS]; + int deq = rte_event_dequeue_burst(evdev, t->port[0], ev, + NPKTS, 0); + if (deq != 1) + printf("%d error; no packet dequeued\n", __LINE__); + + /* let cleanup below stop the device on last iter */ + if (i != NUM_ITERS-1) + rte_event_dev_stop(evdev); + } + + cleanup(t); + return 0; +fail: + cleanup(t); + return -1; +} + +static int +port_single_lb_reconfig(struct test *t) +{ + if (init(t, 2, 2) < 0) { + printf("%d: Error initializing device\n", __LINE__); + goto fail; + } + + static const struct rte_event_queue_conf conf_lb_atomic = { + .priority = RTE_EVENT_DEV_PRIORITY_NORMAL, + .schedule_type = RTE_SCHED_TYPE_ATOMIC, + .nb_atomic_flows = 1024, + .nb_atomic_order_sequences = 1024, + }; + if (rte_event_queue_setup(evdev, 0, &conf_lb_atomic) < 0) { + printf("%d: error creating qid\n", __LINE__); + goto fail; + } + + static const struct rte_event_queue_conf conf_single_link = { + .priority = RTE_EVENT_DEV_PRIORITY_NORMAL, + .event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK, + }; + if (rte_event_queue_setup(evdev, 1, &conf_single_link) < 0) { + printf("%d: error creating qid\n", __LINE__); + goto fail; + } + + struct rte_event_port_conf port_conf = { + .new_event_threshold = 128, + .dequeue_depth = 32, + .enqueue_depth = 64, + }; + if (rte_event_port_setup(evdev, 0, &port_conf) < 0) { + printf("%d Error setting up port\n", __LINE__); + goto fail; + } + if (rte_event_port_setup(evdev, 1, &port_conf) < 0) { + printf("%d Error setting up port\n", __LINE__); + goto fail; + } + + /* link port to lb queue */ + uint8_t queue_id = 0; + if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) { + printf("%d: error creating link for qid\n", __LINE__); + goto fail; + } + + int ret = rte_event_port_unlink(evdev, 0, &queue_id, 1); + if (ret != 1) { + printf("%d: Error unlinking lb port\n", __LINE__); + goto fail; + } + + queue_id = 1; + if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) { + printf("%d: error creating link for qid\n", __LINE__); + goto fail; + } + + queue_id = 0; + int err = rte_event_port_link(evdev, 1, &queue_id, NULL, 1); + if (err != 1) { + printf("%d: error mapping lb qid\n", __LINE__); + goto fail; + } + + if (rte_event_dev_start(evdev) < 0) { + printf("%d: Error with start call\n", __LINE__); + goto fail; + } + + cleanup(t); + return 0; +fail: + cleanup(t); + return -1; +} + +static int +xstats_brute_force(struct test *t) +{ + uint32_t i; + const uint32_t XSTATS_MAX = 1024; + uint32_t ids[XSTATS_MAX]; + uint64_t values[XSTATS_MAX]; + struct rte_event_dev_xstats_name xstats_names[XSTATS_MAX]; + + + /* Create instance with 4 ports */ + if (init(t, 1, 4) < 0 || + create_ports(t, 4) < 0 || + create_atomic_qids(t, 1) < 0) { + printf("%d: Error initializing device\n", __LINE__); + return -1; + } + + int err = rte_event_port_link(evdev, t->port[0], NULL, NULL, 0); + if (err != 1) { + printf("%d: error mapping lb qid\n", __LINE__); + goto fail; + } + + if (rte_event_dev_start(evdev) < 0) { + printf("%d: Error with start call\n", __LINE__); + goto fail; + } + + for (i = 0; i < XSTATS_MAX; i++) + ids[i] = i; + + for (i = 0; i < 3; i++) { + uint32_t mode = RTE_EVENT_DEV_XSTATS_DEVICE + i; + uint32_t j; + for (j = 0; j < UINT8_MAX; j++) { + rte_event_dev_xstats_names_get(evdev, mode, + j, xstats_names, ids, XSTATS_MAX); + + rte_event_dev_xstats_get(evdev, mode, j, ids, + values, XSTATS_MAX); + } + } + + cleanup(t); + return 0; +fail: + cleanup(t); + return -1; +} + +static int +xstats_id_reset_tests(struct test *t) +{ + const int wrk_enq = 2; + int err; + + /* Create instance with 4 ports */ + if (init(t, 1, 4) < 0 || + create_ports(t, 4) < 0 || + create_atomic_qids(t, 1) < 0) { + printf("%d: Error initializing device\n", __LINE__); + return -1; + } + + /* CQ mapping to QID */ + err = rte_event_port_link(evdev, t->port[wrk_enq], NULL, NULL, 0); + if (err != 1) { + printf("%d: error mapping lb qid\n", __LINE__); + goto fail; + } + + if (rte_event_dev_start(evdev) < 0) { + printf("%d: Error with start call\n", __LINE__); + goto fail; + } + +#define XSTATS_MAX 1024 + int ret; + uint32_t i; + uint32_t ids[XSTATS_MAX]; + uint64_t values[XSTATS_MAX]; + struct rte_event_dev_xstats_name xstats_names[XSTATS_MAX]; + + for (i = 0; i < XSTATS_MAX; i++) + ids[i] = i; + +#define NUM_DEV_STATS 6 + /* Device names / values */ + int num_stats = rte_event_dev_xstats_names_get(evdev, + RTE_EVENT_DEV_XSTATS_DEVICE, + 0, xstats_names, ids, XSTATS_MAX); + if (num_stats != NUM_DEV_STATS) { + printf("%d: expected %d stats, got return %d\n", __LINE__, + NUM_DEV_STATS, num_stats); + goto fail; + } + ret = rte_event_dev_xstats_get(evdev, + RTE_EVENT_DEV_XSTATS_DEVICE, + 0, ids, values, num_stats); + if (ret != NUM_DEV_STATS) { + printf("%d: expected %d stats, got return %d\n", __LINE__, + NUM_DEV_STATS, ret); + goto fail; + } + +#define NPKTS 7 + for (i = 0; i < NPKTS; i++) { + struct rte_event ev; + struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool); + if (!arp) { + printf("%d: gen of pkt failed\n", __LINE__); + goto fail; + } + ev.queue_id = t->qid[i]; + ev.op = RTE_EVENT_OP_NEW; + ev.mbuf = arp; + arp->seqn = i; + + int err = rte_event_enqueue_burst(evdev, t->port[0], &ev, 1); + if (err != 1) { + printf("%d: Failed to enqueue\n", __LINE__); + goto fail; + } + } + + rte_service_run_iter_on_app_lcore(t->service_id, 1); + + static const char * const dev_names[] = { + "dev_rx", "dev_tx", "dev_drop", "dev_sched_calls", + "dev_sched_no_iq_enq", "dev_sched_no_cq_enq", + }; + uint64_t dev_expected[] = {NPKTS, NPKTS, 0, 1, 0, 0}; + for (i = 0; (int)i < ret; i++) { + unsigned int id; + uint64_t val = rte_event_dev_xstats_by_name_get(evdev, + dev_names[i], + &id); + if (id != i) { + printf("%d: %s id incorrect, expected %d got %d\n", + __LINE__, dev_names[i], i, id); + goto fail; + } + if (val != dev_expected[i]) { + printf("%d: %s value incorrect, expected %" + PRIu64" got %d\n", __LINE__, dev_names[i], + dev_expected[i], id); + goto fail; + } + /* reset to zero */ + int reset_ret = rte_event_dev_xstats_reset(evdev, + RTE_EVENT_DEV_XSTATS_DEVICE, 0, + &id, + 1); + if (reset_ret) { + printf("%d: failed to reset successfully\n", __LINE__); + goto fail; + } + dev_expected[i] = 0; + /* check value again */ + val = rte_event_dev_xstats_by_name_get(evdev, dev_names[i], 0); + if (val != dev_expected[i]) { + printf("%d: %s value incorrect, expected %"PRIu64 + " got %"PRIu64"\n", __LINE__, dev_names[i], + dev_expected[i], val); + goto fail; + } + }; + +/* 48 is stat offset from start of the devices whole xstats. + * This WILL break every time we add a statistic to a port + * or the device, but there is no other way to test + */ +#define PORT_OFF 48 +/* num stats for the tested port. CQ size adds more stats to a port */ +#define NUM_PORT_STATS 21 +/* the port to test. */ +#define PORT 2 + num_stats = rte_event_dev_xstats_names_get(evdev, + RTE_EVENT_DEV_XSTATS_PORT, PORT, + xstats_names, ids, XSTATS_MAX); + if (num_stats != NUM_PORT_STATS) { + printf("%d: expected %d stats, got return %d\n", + __LINE__, NUM_PORT_STATS, num_stats); + goto fail; + } + ret = rte_event_dev_xstats_get(evdev, RTE_EVENT_DEV_XSTATS_PORT, PORT, + ids, values, num_stats); + + if (ret != NUM_PORT_STATS) { + printf("%d: expected %d stats, got return %d\n", + __LINE__, NUM_PORT_STATS, ret); + goto fail; + } + static const char * const port_names[] = { + "port_2_rx", + "port_2_tx", + "port_2_drop", + "port_2_inflight", + "port_2_avg_pkt_cycles", + "port_2_credits", + "port_2_rx_ring_used", + "port_2_rx_ring_free", + "port_2_cq_ring_used", + "port_2_cq_ring_free", + "port_2_dequeue_calls", + "port_2_dequeues_returning_0", + "port_2_dequeues_returning_1-4", + "port_2_dequeues_returning_5-8", + "port_2_dequeues_returning_9-12", + "port_2_dequeues_returning_13-16", + "port_2_dequeues_returning_17-20", + "port_2_dequeues_returning_21-24", + "port_2_dequeues_returning_25-28", + "port_2_dequeues_returning_29-32", + "port_2_dequeues_returning_33-36", + }; + uint64_t port_expected[] = { + 0, /* rx */ + NPKTS, /* tx */ + 0, /* drop */ + NPKTS, /* inflight */ + 0, /* avg pkt cycles */ + 0, /* credits */ + 0, /* rx ring used */ + 4096, /* rx ring free */ + NPKTS, /* cq ring used */ + 25, /* cq ring free */ + 0, /* dequeue zero calls */ + 0, 0, 0, 0, 0, /* 10 dequeue buckets */ + 0, 0, 0, 0, 0, + }; + uint64_t port_expected_zero[] = { + 0, /* rx */ + 0, /* tx */ + 0, /* drop */ + NPKTS, /* inflight */ + 0, /* avg pkt cycles */ + 0, /* credits */ + 0, /* rx ring used */ + 4096, /* rx ring free */ + NPKTS, /* cq ring used */ + 25, /* cq ring free */ + 0, /* dequeue zero calls */ + 0, 0, 0, 0, 0, /* 10 dequeue buckets */ + 0, 0, 0, 0, 0, + }; + if (RTE_DIM(port_expected) != NUM_PORT_STATS || + RTE_DIM(port_names) != NUM_PORT_STATS) { + printf("%d: port array of wrong size\n", __LINE__); + goto fail; + } + + int failed = 0; + for (i = 0; (int)i < ret; i++) { + unsigned int id; + uint64_t val = rte_event_dev_xstats_by_name_get(evdev, + port_names[i], + &id); + if (id != i + PORT_OFF) { + printf("%d: %s id incorrect, expected %d got %d\n", + __LINE__, port_names[i], i+PORT_OFF, + id); + failed = 1; + } + if (val != port_expected[i]) { + printf("%d: %s value incorrect, expected %"PRIu64 + " got %d\n", __LINE__, port_names[i], + port_expected[i], id); + failed = 1; + } + /* reset to zero */ + int reset_ret = rte_event_dev_xstats_reset(evdev, + RTE_EVENT_DEV_XSTATS_PORT, PORT, + &id, + 1); + if (reset_ret) { + printf("%d: failed to reset successfully\n", __LINE__); + failed = 1; + } + /* check value again */ + val = rte_event_dev_xstats_by_name_get(evdev, port_names[i], 0); + if (val != port_expected_zero[i]) { + printf("%d: %s value incorrect, expected %"PRIu64 + " got %"PRIu64"\n", __LINE__, port_names[i], + port_expected_zero[i], val); + failed = 1; + } + }; + if (failed) + goto fail; + +/* num queue stats */ +#define NUM_Q_STATS 17 +/* queue offset from start of the devices whole xstats. + * This will break every time we add a statistic to a device/port/queue + */ +#define QUEUE_OFF 90 + const uint32_t queue = 0; + num_stats = rte_event_dev_xstats_names_get(evdev, + RTE_EVENT_DEV_XSTATS_QUEUE, queue, + xstats_names, ids, XSTATS_MAX); + if (num_stats != NUM_Q_STATS) { + printf("%d: expected %d stats, got return %d\n", + __LINE__, NUM_Q_STATS, num_stats); + goto fail; + } + ret = rte_event_dev_xstats_get(evdev, RTE_EVENT_DEV_XSTATS_QUEUE, + queue, ids, values, num_stats); + if (ret != NUM_Q_STATS) { + printf("%d: expected 21 stats, got return %d\n", __LINE__, ret); + goto fail; + } + static const char * const queue_names[] = { + "qid_0_rx", + "qid_0_tx", + "qid_0_drop", + "qid_0_inflight", + "qid_0_iq_size", + "qid_0_iq_0_used", + "qid_0_iq_1_used", + "qid_0_iq_2_used", + "qid_0_iq_3_used", + "qid_0_port_0_pinned_flows", + "qid_0_port_0_packets", + "qid_0_port_1_pinned_flows", + "qid_0_port_1_packets", + "qid_0_port_2_pinned_flows", + "qid_0_port_2_packets", + "qid_0_port_3_pinned_flows", + "qid_0_port_3_packets", + }; + uint64_t queue_expected[] = { + 7, /* rx */ + 7, /* tx */ + 0, /* drop */ + 7, /* inflight */ + 512, /* iq size */ + 0, /* iq 0 used */ + 0, /* iq 1 used */ + 0, /* iq 2 used */ + 0, /* iq 3 used */ + /* QID-to-Port: pinned_flows, packets */ + 0, 0, + 0, 0, + 1, 7, + 0, 0, + }; + uint64_t queue_expected_zero[] = { + 0, /* rx */ + 0, /* tx */ + 0, /* drop */ + 7, /* inflight */ + 512, /* iq size */ + 0, /* iq 0 used */ + 0, /* iq 1 used */ + 0, /* iq 2 used */ + 0, /* iq 3 used */ + /* QID-to-Port: pinned_flows, packets */ + 0, 0, + 0, 0, + 1, 0, + 0, 0, + }; + if (RTE_DIM(queue_expected) != NUM_Q_STATS || + RTE_DIM(queue_expected_zero) != NUM_Q_STATS || + RTE_DIM(queue_names) != NUM_Q_STATS) { + printf("%d : queue array of wrong size\n", __LINE__); + goto fail; + } + + failed = 0; + for (i = 0; (int)i < ret; i++) { + unsigned int id; + uint64_t val = rte_event_dev_xstats_by_name_get(evdev, + queue_names[i], + &id); + if (id != i + QUEUE_OFF) { + printf("%d: %s id incorrect, expected %d got %d\n", + __LINE__, queue_names[i], i+QUEUE_OFF, + id); + failed = 1; + } + if (val != queue_expected[i]) { + printf("%d: %d: %s value , expected %"PRIu64 + " got %"PRIu64"\n", i, __LINE__, + queue_names[i], queue_expected[i], val); + failed = 1; + } + /* reset to zero */ + int reset_ret = rte_event_dev_xstats_reset(evdev, + RTE_EVENT_DEV_XSTATS_QUEUE, + queue, &id, 1); + if (reset_ret) { + printf("%d: failed to reset successfully\n", __LINE__); + failed = 1; + } + /* check value again */ + val = rte_event_dev_xstats_by_name_get(evdev, queue_names[i], + 0); + if (val != queue_expected_zero[i]) { + printf("%d: %s value incorrect, expected %"PRIu64 + " got %"PRIu64"\n", __LINE__, queue_names[i], + queue_expected_zero[i], val); + failed = 1; + } + }; + + if (failed) + goto fail; + + cleanup(t); + return 0; +fail: + cleanup(t); + return -1; +} + +static int +ordered_reconfigure(struct test *t) +{ + if (init(t, 1, 1) < 0 || + create_ports(t, 1) < 0) { + printf("%d: Error initializing device\n", __LINE__); + return -1; + } + + const struct rte_event_queue_conf conf = { + .schedule_type = RTE_SCHED_TYPE_ORDERED, + .priority = RTE_EVENT_DEV_PRIORITY_NORMAL, + .nb_atomic_flows = 1024, + .nb_atomic_order_sequences = 1024, + }; + + if (rte_event_queue_setup(evdev, 0, &conf) < 0) { + printf("%d: error creating qid\n", __LINE__); + goto failed; + } + + if (rte_event_queue_setup(evdev, 0, &conf) < 0) { + printf("%d: error creating qid, for 2nd time\n", __LINE__); + goto failed; + } + + rte_event_port_link(evdev, t->port[0], NULL, NULL, 0); + if (rte_event_dev_start(evdev) < 0) { + printf("%d: Error with start call\n", __LINE__); + return -1; + } + + cleanup(t); + return 0; +failed: + cleanup(t); + return -1; +} + +static int +qid_priorities(struct test *t) +{ + /* Test works by having a CQ with enough empty space for all packets, + * and enqueueing 3 packets to 3 QIDs. They must return based on the + * priority of the QID, not the ingress order, to pass the test + */ + unsigned int i; + /* Create instance with 1 ports, and 3 qids */ + if (init(t, 3, 1) < 0 || + create_ports(t, 1) < 0) { + printf("%d: Error initializing device\n", __LINE__); + return -1; + } + + for (i = 0; i < 3; i++) { + /* Create QID */ + const struct rte_event_queue_conf conf = { + .schedule_type = RTE_SCHED_TYPE_ATOMIC, + /* increase priority (0 == highest), as we go */ + .priority = RTE_EVENT_DEV_PRIORITY_NORMAL - i, + .nb_atomic_flows = 1024, + .nb_atomic_order_sequences = 1024, + }; + + if (rte_event_queue_setup(evdev, i, &conf) < 0) { + printf("%d: error creating qid %d\n", __LINE__, i); + return -1; + } + t->qid[i] = i; + } + t->nb_qids = i; + /* map all QIDs to port */ + rte_event_port_link(evdev, t->port[0], NULL, NULL, 0); + + if (rte_event_dev_start(evdev) < 0) { + printf("%d: Error with start call\n", __LINE__); + return -1; + } + + /* enqueue 3 packets, setting seqn and QID to check priority */ + for (i = 0; i < 3; i++) { + struct rte_event ev; + struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool); + if (!arp) { + printf("%d: gen of pkt failed\n", __LINE__); + return -1; + } + ev.queue_id = t->qid[i]; + ev.op = RTE_EVENT_OP_NEW; + ev.mbuf = arp; + arp->seqn = i; + + int err = rte_event_enqueue_burst(evdev, t->port[0], &ev, 1); + if (err != 1) { + printf("%d: Failed to enqueue\n", __LINE__); + return -1; + } + } + + rte_service_run_iter_on_app_lcore(t->service_id, 1); + + /* dequeue packets, verify priority was upheld */ + struct rte_event ev[32]; + uint32_t deq_pkts = + rte_event_dequeue_burst(evdev, t->port[0], ev, 32, 0); + if (deq_pkts != 3) { + printf("%d: failed to deq packets\n", __LINE__); + rte_event_dev_dump(evdev, stdout); + return -1; + } + for (i = 0; i < 3; i++) { + if (ev[i].mbuf->seqn != 2-i) { + printf( + "%d: qid priority test: seqn %d incorrectly prioritized\n", + __LINE__, i); + } + } + + cleanup(t); + return 0; +} + +static int +load_balancing(struct test *t) +{ + const int rx_enq = 0; + int err; + uint32_t i; + + if (init(t, 1, 4) < 0 || + create_ports(t, 4) < 0 || + create_atomic_qids(t, 1) < 0) { + printf("%d: Error initializing device\n", __LINE__); + return -1; + } + + for (i = 0; i < 3; i++) { + /* map port 1 - 3 inclusive */ + if (rte_event_port_link(evdev, t->port[i+1], &t->qid[0], + NULL, 1) != 1) { + printf("%d: error mapping qid to port %d\n", + __LINE__, i); + return -1; + } + } + + if (rte_event_dev_start(evdev) < 0) { + printf("%d: Error with start call\n", __LINE__); + return -1; + } + + /************** FORWARD ****************/ + /* + * Create a set of flows that test the load-balancing operation of the + * implementation. Fill CQ 0 and 1 with flows 0 and 1, and test + * with a new flow, which should be sent to the 3rd mapped CQ + */ + static uint32_t flows[] = {0, 1, 1, 0, 0, 2, 2, 0, 2}; + + for (i = 0; i < RTE_DIM(flows); i++) { + struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool); + if (!arp) { + printf("%d: gen of pkt failed\n", __LINE__); + return -1; + } + + struct rte_event ev = { + .op = RTE_EVENT_OP_NEW, + .queue_id = t->qid[0], + .flow_id = flows[i], + .mbuf = arp, + }; + /* generate pkt and enqueue */ + err = rte_event_enqueue_burst(evdev, t->port[rx_enq], &ev, 1); + if (err < 0) { + printf("%d: Failed to enqueue\n", __LINE__); + return -1; + } + } + + rte_service_run_iter_on_app_lcore(t->service_id, 1); + + struct test_event_dev_stats stats; + err = test_event_dev_stats_get(evdev, &stats); + if (err) { + printf("%d: failed to get stats\n", __LINE__); + return -1; + } + + if (stats.port_inflight[1] != 4) { + printf("%d:%s: port 1 inflight not correct\n", __LINE__, + __func__); + return -1; + } + if (stats.port_inflight[2] != 2) { + printf("%d:%s: port 2 inflight not correct\n", __LINE__, + __func__); + return -1; + } + if (stats.port_inflight[3] != 3) { + printf("%d:%s: port 3 inflight not correct\n", __LINE__, + __func__); + return -1; + } + + cleanup(t); + return 0; +} + +static int +load_balancing_history(struct test *t) +{ + struct test_event_dev_stats stats = {0}; + const int rx_enq = 0; + int err; + uint32_t i; + + /* Create instance with 1 atomic QID going to 3 ports + 1 prod port */ + if (init(t, 1, 4) < 0 || + create_ports(t, 4) < 0 || + create_atomic_qids(t, 1) < 0) + return -1; + + /* CQ mapping to QID */ + if (rte_event_port_link(evdev, t->port[1], &t->qid[0], NULL, 1) != 1) { + printf("%d: error mapping port 1 qid\n", __LINE__); + return -1; + } + if (rte_event_port_link(evdev, t->port[2], &t->qid[0], NULL, 1) != 1) { + printf("%d: error mapping port 2 qid\n", __LINE__); + return -1; + } + if (rte_event_port_link(evdev, t->port[3], &t->qid[0], NULL, 1) != 1) { + printf("%d: error mapping port 3 qid\n", __LINE__); + return -1; + } + if (rte_event_dev_start(evdev) < 0) { + printf("%d: Error with start call\n", __LINE__); + return -1; + } + + /* + * Create a set of flows that test the load-balancing operation of the + * implementation. Fill CQ 0, 1 and 2 with flows 0, 1 and 2, drop + * the packet from CQ 0, send in a new set of flows. Ensure that: + * 1. The new flow 3 gets into the empty CQ0 + * 2. packets for existing flow gets added into CQ1 + * 3. Next flow 0 pkt is now onto CQ2, since CQ0 and CQ1 now contain + * more outstanding pkts + * + * This test makes sure that when a flow ends (i.e. all packets + * have been completed for that flow), that the flow can be moved + * to a different CQ when new packets come in for that flow. + */ + static uint32_t flows1[] = {0, 1, 1, 2}; + + for (i = 0; i < RTE_DIM(flows1); i++) { + struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool); + struct rte_event ev = { + .flow_id = flows1[i], + .op = RTE_EVENT_OP_NEW, + .queue_id = t->qid[0], + .event_type = RTE_EVENT_TYPE_CPU, + .priority = RTE_EVENT_DEV_PRIORITY_NORMAL, + .mbuf = arp + }; + + if (!arp) { + printf("%d: gen of pkt failed\n", __LINE__); + return -1; + } + arp->hash.rss = flows1[i]; + err = rte_event_enqueue_burst(evdev, t->port[rx_enq], &ev, 1); + if (err < 0) { + printf("%d: Failed to enqueue\n", __LINE__); + return -1; + } + } + + /* call the scheduler */ + rte_service_run_iter_on_app_lcore(t->service_id, 1); + + /* Dequeue the flow 0 packet from port 1, so that we can then drop */ + struct rte_event ev; + if (!rte_event_dequeue_burst(evdev, t->port[1], &ev, 1, 0)) { + printf("%d: failed to dequeue\n", __LINE__); + return -1; + } + if (ev.mbuf->hash.rss != flows1[0]) { + printf("%d: unexpected flow received\n", __LINE__); + return -1; + } + + /* drop the flow 0 packet from port 1 */ + rte_event_enqueue_burst(evdev, t->port[1], &release_ev, 1); + + /* call the scheduler */ + rte_service_run_iter_on_app_lcore(t->service_id, 1); + + /* + * Set up the next set of flows, first a new flow to fill up + * CQ 0, so that the next flow 0 packet should go to CQ2 + */ + static uint32_t flows2[] = { 3, 3, 3, 1, 1, 0 }; + + for (i = 0; i < RTE_DIM(flows2); i++) { + struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool); + struct rte_event ev = { + .flow_id = flows2[i], + .op = RTE_EVENT_OP_NEW, + .queue_id = t->qid[0], + .event_type = RTE_EVENT_TYPE_CPU, + .priority = RTE_EVENT_DEV_PRIORITY_NORMAL, + .mbuf = arp + }; + + if (!arp) { + printf("%d: gen of pkt failed\n", __LINE__); + return -1; + } + arp->hash.rss = flows2[i]; + + err = rte_event_enqueue_burst(evdev, t->port[rx_enq], &ev, 1); + if (err < 0) { + printf("%d: Failed to enqueue\n", __LINE__); + return -1; + } + } + + /* schedule */ + rte_service_run_iter_on_app_lcore(t->service_id, 1); + + err = test_event_dev_stats_get(evdev, &stats); + if (err) { + printf("%d:failed to get stats\n", __LINE__); + return -1; + } + + /* + * Now check the resulting inflights on each port. + */ + if (stats.port_inflight[1] != 3) { + printf("%d:%s: port 1 inflight not correct\n", __LINE__, + __func__); + printf("Inflights, ports 1, 2, 3: %u, %u, %u\n", + (unsigned int)stats.port_inflight[1], + (unsigned int)stats.port_inflight[2], + (unsigned int)stats.port_inflight[3]); + return -1; + } + if (stats.port_inflight[2] != 4) { + printf("%d:%s: port 2 inflight not correct\n", __LINE__, + __func__); + printf("Inflights, ports 1, 2, 3: %u, %u, %u\n", + (unsigned int)stats.port_inflight[1], + (unsigned int)stats.port_inflight[2], + (unsigned int)stats.port_inflight[3]); + return -1; + } + if (stats.port_inflight[3] != 2) { + printf("%d:%s: port 3 inflight not correct\n", __LINE__, + __func__); + printf("Inflights, ports 1, 2, 3: %u, %u, %u\n", + (unsigned int)stats.port_inflight[1], + (unsigned int)stats.port_inflight[2], + (unsigned int)stats.port_inflight[3]); + return -1; + } + + for (i = 1; i <= 3; i++) { + struct rte_event ev; + while (rte_event_dequeue_burst(evdev, i, &ev, 1, 0)) + rte_event_enqueue_burst(evdev, i, &release_ev, 1); + } + rte_service_run_iter_on_app_lcore(t->service_id, 1); + + cleanup(t); + return 0; +} + +static int +invalid_qid(struct test *t) +{ + struct test_event_dev_stats stats; + const int rx_enq = 0; + int err; + uint32_t i; + + if (init(t, 1, 4) < 0 || + create_ports(t, 4) < 0 || + create_atomic_qids(t, 1) < 0) { + printf("%d: Error initializing device\n", __LINE__); + return -1; + } + + /* CQ mapping to QID */ + for (i = 0; i < 4; i++) { + err = rte_event_port_link(evdev, t->port[i], &t->qid[0], + NULL, 1); + if (err != 1) { + printf("%d: error mapping port 1 qid\n", __LINE__); + return -1; + } + } + + if (rte_event_dev_start(evdev) < 0) { + printf("%d: Error with start call\n", __LINE__); + return -1; + } + + /* + * Send in a packet with an invalid qid to the scheduler. + * We should see the packed enqueued OK, but the inflights for + * that packet should not be incremented, and the rx_dropped + * should be incremented. + */ + static uint32_t flows1[] = {20}; + + for (i = 0; i < RTE_DIM(flows1); i++) { + struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool); + if (!arp) { + printf("%d: gen of pkt failed\n", __LINE__); + return -1; + } + + struct rte_event ev = { + .op = RTE_EVENT_OP_NEW, + .queue_id = t->qid[0] + flows1[i], + .flow_id = i, + .mbuf = arp, + }; + /* generate pkt and enqueue */ + err = rte_event_enqueue_burst(evdev, t->port[rx_enq], &ev, 1); + if (err < 0) { + printf("%d: Failed to enqueue\n", __LINE__); + return -1; + } + } + + /* call the scheduler */ + rte_service_run_iter_on_app_lcore(t->service_id, 1); + + err = test_event_dev_stats_get(evdev, &stats); + if (err) { + printf("%d: failed to get stats\n", __LINE__); + return -1; + } + + /* + * Now check the resulting inflights on the port, and the rx_dropped. + */ + if (stats.port_inflight[0] != 0) { + printf("%d:%s: port 1 inflight count not correct\n", __LINE__, + __func__); + rte_event_dev_dump(evdev, stdout); + return -1; + } + if (stats.port_rx_dropped[0] != 1) { + printf("%d:%s: port 1 drops\n", __LINE__, __func__); + rte_event_dev_dump(evdev, stdout); + return -1; + } + /* each packet drop should only be counted in one place - port or dev */ + if (stats.rx_dropped != 0) { + printf("%d:%s: port 1 dropped count not correct\n", __LINE__, + __func__); + rte_event_dev_dump(evdev, stdout); + return -1; + } + + cleanup(t); + return 0; +} + +static int +single_packet(struct test *t) +{ + const uint32_t MAGIC_SEQN = 7321; + struct rte_event ev; + struct test_event_dev_stats stats; + const int rx_enq = 0; + const int wrk_enq = 2; + int err; + + /* Create instance with 4 ports */ + if (init(t, 1, 4) < 0 || + create_ports(t, 4) < 0 || + create_atomic_qids(t, 1) < 0) { + printf("%d: Error initializing device\n", __LINE__); + return -1; + } + + /* CQ mapping to QID */ + err = rte_event_port_link(evdev, t->port[wrk_enq], NULL, NULL, 0); + if (err != 1) { + printf("%d: error mapping lb qid\n", __LINE__); + cleanup(t); + return -1; + } + + if (rte_event_dev_start(evdev) < 0) { + printf("%d: Error with start call\n", __LINE__); + return -1; + } + + /************** Gen pkt and enqueue ****************/ + struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool); + if (!arp) { + printf("%d: gen of pkt failed\n", __LINE__); + return -1; + } + + ev.op = RTE_EVENT_OP_NEW; + ev.priority = RTE_EVENT_DEV_PRIORITY_NORMAL; + ev.mbuf = arp; + ev.queue_id = 0; + ev.flow_id = 3; + arp->seqn = MAGIC_SEQN; + + err = rte_event_enqueue_burst(evdev, t->port[rx_enq], &ev, 1); + if (err < 0) { + printf("%d: Failed to enqueue\n", __LINE__); + return -1; + } + + rte_service_run_iter_on_app_lcore(t->service_id, 1); + + err = test_event_dev_stats_get(evdev, &stats); + if (err) { + printf("%d: failed to get stats\n", __LINE__); + return -1; + } + + if (stats.rx_pkts != 1 || + stats.tx_pkts != 1 || + stats.port_inflight[wrk_enq] != 1) { + printf("%d: Sched core didn't handle pkt as expected\n", + __LINE__); + rte_event_dev_dump(evdev, stdout); + return -1; + } + + uint32_t deq_pkts; + + deq_pkts = rte_event_dequeue_burst(evdev, t->port[wrk_enq], &ev, 1, 0); + if (deq_pkts < 1) { + printf("%d: Failed to deq\n", __LINE__); + return -1; + } + + err = test_event_dev_stats_get(evdev, &stats); + if (err) { + printf("%d: failed to get stats\n", __LINE__); + return -1; + } + + err = test_event_dev_stats_get(evdev, &stats); + if (ev.mbuf->seqn != MAGIC_SEQN) { + printf("%d: magic sequence number not dequeued\n", __LINE__); + return -1; + } + + rte_pktmbuf_free(ev.mbuf); + err = rte_event_enqueue_burst(evdev, t->port[wrk_enq], &release_ev, 1); + if (err < 0) { + printf("%d: Failed to enqueue\n", __LINE__); + return -1; + } + rte_service_run_iter_on_app_lcore(t->service_id, 1); + + err = test_event_dev_stats_get(evdev, &stats); + if (stats.port_inflight[wrk_enq] != 0) { + printf("%d: port inflight not correct\n", __LINE__); + return -1; + } + + cleanup(t); + return 0; +} + +static int +inflight_counts(struct test *t) +{ + struct rte_event ev; + struct test_event_dev_stats stats; + const int rx_enq = 0; + const int p1 = 1; + const int p2 = 2; + int err; + int i; + + /* Create instance with 4 ports */ + if (init(t, 2, 3) < 0 || + create_ports(t, 3) < 0 || + create_atomic_qids(t, 2) < 0) { + printf("%d: Error initializing device\n", __LINE__); + return -1; + } + + /* CQ mapping to QID */ + err = rte_event_port_link(evdev, t->port[p1], &t->qid[0], NULL, 1); + if (err != 1) { + printf("%d: error mapping lb qid\n", __LINE__); + cleanup(t); + return -1; + } + err = rte_event_port_link(evdev, t->port[p2], &t->qid[1], NULL, 1); + if (err != 1) { + printf("%d: error mapping lb qid\n", __LINE__); + cleanup(t); + return -1; + } + + if (rte_event_dev_start(evdev) < 0) { + printf("%d: Error with start call\n", __LINE__); + return -1; + } + + /************** FORWARD ****************/ +#define QID1_NUM 5 + for (i = 0; i < QID1_NUM; i++) { + struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool); + + if (!arp) { + printf("%d: gen of pkt failed\n", __LINE__); + goto err; + } + + ev.queue_id = t->qid[0]; + ev.op = RTE_EVENT_OP_NEW; + ev.mbuf = arp; + err = rte_event_enqueue_burst(evdev, t->port[rx_enq], &ev, 1); + if (err != 1) { + printf("%d: Failed to enqueue\n", __LINE__); + goto err; + } + } +#define QID2_NUM 3 + for (i = 0; i < QID2_NUM; i++) { + struct rte_mbuf *arp = rte_gen_arp(0, t->mbuf_pool); + + if (!arp) { + printf("%d: gen of pkt failed\n", __LINE__); + goto err; + } + ev.queue_id = t->qid[1]; + ev.op = RTE_EVENT_OP_NEW; + ev.mbuf = arp; + err = rte_event_enqueue_burst(evdev, t->port[rx_enq], &ev, 1); + if (err != 1) { + printf("%d: Failed to enqueue\n", __LINE__); + goto err; + } + } + + /* schedule */ + rte_service_run_iter_on_app_lcore(t->service_id, 1); + + err = test_event_dev_stats_get(evdev, &stats); + if (err) { + printf("%d: failed to get stats\n", __LINE__); + goto err; + } + + if (stats.rx_pkts != QID1_NUM + QID2_NUM || + stats.tx_pkts != QID1_NUM + QID2_NUM) { + printf("%d: Sched core didn't handle pkt as expected\n", + __LINE__); + goto err; + } + + if (stats.port_inflight[p1] != QID1_NUM) { + printf("%d: %s port 1 inflight not correct\n", __LINE__, + __func__); + goto err; + } + if (stats.port_inflight[p2] != QID2_NUM) { + printf("%d: %s port 2 inflight not correct\n", __LINE__, + __func__); + goto err; + } + + /************** DEQUEUE INFLIGHT COUNT CHECKS ****************/ + /* port 1 */ + struct rte_event events[QID1_NUM + QID2_NUM]; + uint32_t deq_pkts = rte_event_dequeue_burst(evdev, t->port[p1], events, + RTE_DIM(events), 0); + + if (deq_pkts != QID1_NUM) { + printf("%d: Port 1: DEQUEUE inflight failed\n", __LINE__); + goto err; + } + err = test_event_dev_stats_get(evdev, &stats); + if (stats.port_inflight[p1] != QID1_NUM) { + printf("%d: port 1 inflight decrement after DEQ != 0\n", + __LINE__); + goto err; + } + for (i = 0; i < QID1_NUM; i++) { + err = rte_event_enqueue_burst(evdev, t->port[p1], &release_ev, + 1); + if (err != 1) { + printf("%d: %s rte enqueue of inf release failed\n", + __LINE__, __func__); + goto err; + } + } + + /* + * As the scheduler core decrements inflights, it needs to run to + * process packets to act on the drop messages + */ + rte_service_run_iter_on_app_lcore(t->service_id, 1); + + err = test_event_dev_stats_get(evdev, &stats); + if (stats.port_inflight[p1] != 0) { + printf("%d: port 1 inflight NON NULL after DROP\n", __LINE__); + goto err; + } + + /* port2 */ + deq_pkts = rte_event_dequeue_burst(evdev, t->port[p2], events, + RTE_DIM(events), 0); + if (deq_pkts != QID2_NUM) { + printf("%d: Port 2: DEQUEUE inflight failed\n", __LINE__); + goto err; + } + err = test_event_dev_stats_get(evdev, &stats); + if (stats.port_inflight[p2] != QID2_NUM) { + printf("%d: port 1 inflight decrement after DEQ != 0\n", + __LINE__); + goto err; + } + for (i = 0; i < QID2_NUM; i++) { + err = rte_event_enqueue_burst(evdev, t->port[p2], &release_ev, + 1); + if (err != 1) { + printf("%d: %s rte enqueue of inf release failed\n", + __LINE__, __func__); + goto err; + } + } + + /* + * As the scheduler core decrements inflights, it needs to run to + * process packets to act on the drop messages + */ + rte_service_run_iter_on_app_lcore(t->service_id, 1); + + err = test_event_dev_stats_get(evdev, &stats); + if (stats.port_inflight[p2] != 0) { + printf("%d: port 2 inflight NON NULL after DROP\n", __LINE__); + goto err; + } + cleanup(t); + return 0; + +err: + rte_event_dev_dump(evdev, stdout); + cleanup(t); + return -1; +} + +static int +parallel_basic(struct test *t, int check_order) +{ + const uint8_t rx_port = 0; + const uint8_t w1_port = 1; + const uint8_t w3_port = 3; + const uint8_t tx_port = 4; + int err; + int i; + uint32_t deq_pkts, j; + struct rte_mbuf *mbufs[3]; + struct rte_mbuf *mbufs_out[3] = { 0 }; + const uint32_t MAGIC_SEQN = 1234; + + /* Create instance with 4 ports */ + if (init(t, 2, tx_port + 1) < 0 || + create_ports(t, tx_port + 1) < 0 || + (check_order ? create_ordered_qids(t, 1) : + create_unordered_qids(t, 1)) < 0 || + create_directed_qids(t, 1, &tx_port)) { + printf("%d: Error initializing device\n", __LINE__); + return -1; + } + + /* + * CQ mapping to QID + * We need three ports, all mapped to the same ordered qid0. Then we'll + * take a packet out to each port, re-enqueue in reverse order, + * then make sure the reordering has taken place properly when we + * dequeue from the tx_port. + * + * Simplified test setup diagram: + * + * rx_port w1_port + * \ / \ + * qid0 - w2_port - qid1 + * \ / \ + * w3_port tx_port + */ + /* CQ mapping to QID for LB ports (directed mapped on create) */ + for (i = w1_port; i <= w3_port; i++) { + err = rte_event_port_link(evdev, t->port[i], &t->qid[0], NULL, + 1); + if (err != 1) { + printf("%d: error mapping lb qid\n", __LINE__); + cleanup(t); + return -1; + } + } + + if (rte_event_dev_start(evdev) < 0) { + printf("%d: Error with start call\n", __LINE__); + return -1; + } + + /* Enqueue 3 packets to the rx port */ + for (i = 0; i < 3; i++) { + struct rte_event ev; + mbufs[i] = rte_gen_arp(0, t->mbuf_pool); + if (!mbufs[i]) { + printf("%d: gen of pkt failed\n", __LINE__); + return -1; + } + + ev.queue_id = t->qid[0]; + ev.op = RTE_EVENT_OP_NEW; + ev.mbuf = mbufs[i]; + mbufs[i]->seqn = MAGIC_SEQN + i; + + /* generate pkt and enqueue */ + err = rte_event_enqueue_burst(evdev, t->port[rx_port], &ev, 1); + if (err != 1) { + printf("%d: Failed to enqueue pkt %u, retval = %u\n", + __LINE__, i, err); + return -1; + } + } + + rte_service_run_iter_on_app_lcore(t->service_id, 1); + + /* use extra slot to make logic in loops easier */ + struct rte_event deq_ev[w3_port + 1]; + + /* Dequeue the 3 packets, one from each worker port */ + for (i = w1_port; i <= w3_port; i++) { + deq_pkts = rte_event_dequeue_burst(evdev, t->port[i], + &deq_ev[i], 1, 0); + if (deq_pkts != 1) { + printf("%d: Failed to deq\n", __LINE__); + rte_event_dev_dump(evdev, stdout); + return -1; + } + } + + /* Enqueue each packet in reverse order, flushing after each one */ + for (i = w3_port; i >= w1_port; i--) { + + deq_ev[i].op = RTE_EVENT_OP_FORWARD; + deq_ev[i].queue_id = t->qid[1]; + err = rte_event_enqueue_burst(evdev, t->port[i], &deq_ev[i], 1); + if (err != 1) { + printf("%d: Failed to enqueue\n", __LINE__); + return -1; + } + } + rte_service_run_iter_on_app_lcore(t->service_id, 1); + + /* dequeue from the tx ports, we should get 3 packets */ + deq_pkts = rte_event_dequeue_burst(evdev, t->port[tx_port], deq_ev, + 3, 0); + + /* Check to see if we've got all 3 packets */ + if (deq_pkts != 3) { + printf("%d: expected 3 pkts at tx port got %d from port %d\n", + __LINE__, deq_pkts, tx_port); + rte_event_dev_dump(evdev, stdout); + return 1; + } + + /* Check to see if the sequence numbers are in expected order */ + if (check_order) { + for (j = 0 ; j < deq_pkts ; j++) { + if (deq_ev[j].mbuf->seqn != MAGIC_SEQN + j) { + printf( + "%d: Incorrect sequence number(%d) from port %d\n", + __LINE__, mbufs_out[j]->seqn, tx_port); + return -1; + } + } + } + + /* Destroy the instance */ + cleanup(t); + return 0; +} + +static int +ordered_basic(struct test *t) +{ + return parallel_basic(t, 1); +} + +static int +unordered_basic(struct test *t) +{ + return parallel_basic(t, 0); +} + +static int +holb(struct test *t) /* test to check we avoid basic head-of-line blocking */ +{ + const struct rte_event new_ev = { + .op = RTE_EVENT_OP_NEW + /* all other fields zero */ + }; + struct rte_event ev = new_ev; + unsigned int rx_port = 0; /* port we get the first flow on */ + char rx_port_used_stat[64]; + char rx_port_free_stat[64]; + char other_port_used_stat[64]; + + if (init(t, 1, 2) < 0 || + create_ports(t, 2) < 0 || + create_atomic_qids(t, 1) < 0) { + printf("%d: Error initializing device\n", __LINE__); + return -1; + } + int nb_links = rte_event_port_link(evdev, t->port[1], NULL, NULL, 0); + if (rte_event_port_link(evdev, t->port[0], NULL, NULL, 0) != 1 || + nb_links != 1) { + printf("%d: Error links queue to ports\n", __LINE__); + goto err; + } + if (rte_event_dev_start(evdev) < 0) { + printf("%d: Error with start call\n", __LINE__); + goto err; + } + + /* send one packet and see where it goes, port 0 or 1 */ + if (rte_event_enqueue_burst(evdev, t->port[0], &ev, 1) != 1) { + printf("%d: Error doing first enqueue\n", __LINE__); + goto err; + } + rte_service_run_iter_on_app_lcore(t->service_id, 1); + + if (rte_event_dev_xstats_by_name_get(evdev, "port_0_cq_ring_used", NULL) + != 1) + rx_port = 1; + + snprintf(rx_port_used_stat, sizeof(rx_port_used_stat), + "port_%u_cq_ring_used", rx_port); + snprintf(rx_port_free_stat, sizeof(rx_port_free_stat), + "port_%u_cq_ring_free", rx_port); + snprintf(other_port_used_stat, sizeof(other_port_used_stat), + "port_%u_cq_ring_used", rx_port ^ 1); + if (rte_event_dev_xstats_by_name_get(evdev, rx_port_used_stat, NULL) + != 1) { + printf("%d: Error, first event not scheduled\n", __LINE__); + goto err; + } + + /* now fill up the rx port's queue with one flow to cause HOLB */ + do { + ev = new_ev; + if (rte_event_enqueue_burst(evdev, t->port[0], &ev, 1) != 1) { + printf("%d: Error with enqueue\n", __LINE__); + goto err; + } + rte_service_run_iter_on_app_lcore(t->service_id, 1); + } while (rte_event_dev_xstats_by_name_get(evdev, + rx_port_free_stat, NULL) != 0); + + /* one more packet, which needs to stay in IQ - i.e. HOLB */ + ev = new_ev; + if (rte_event_enqueue_burst(evdev, t->port[0], &ev, 1) != 1) { + printf("%d: Error with enqueue\n", __LINE__); + goto err; + } + rte_service_run_iter_on_app_lcore(t->service_id, 1); + + /* check that the other port still has an empty CQ */ + if (rte_event_dev_xstats_by_name_get(evdev, other_port_used_stat, NULL) + != 0) { + printf("%d: Error, second port CQ is not empty\n", __LINE__); + goto err; + } + /* check IQ now has one packet */ + if (rte_event_dev_xstats_by_name_get(evdev, "qid_0_iq_0_used", NULL) + != 1) { + printf("%d: Error, QID does not have exactly 1 packet\n", + __LINE__); + goto err; + } + + /* send another flow, which should pass the other IQ entry */ + ev = new_ev; + ev.flow_id = 1; + if (rte_event_enqueue_burst(evdev, t->port[0], &ev, 1) != 1) { + printf("%d: Error with enqueue\n", __LINE__); + goto err; + } + rte_service_run_iter_on_app_lcore(t->service_id, 1); + + if (rte_event_dev_xstats_by_name_get(evdev, other_port_used_stat, NULL) + != 1) { + printf("%d: Error, second flow did not pass out first\n", + __LINE__); + goto err; + } + + if (rte_event_dev_xstats_by_name_get(evdev, "qid_0_iq_0_used", NULL) + != 1) { + printf("%d: Error, QID does not have exactly 1 packet\n", + __LINE__); + goto err; + } + cleanup(t); + return 0; +err: + rte_event_dev_dump(evdev, stdout); + cleanup(t); + return -1; +} + +static int +worker_loopback_worker_fn(void *arg) +{ + struct test *t = arg; + uint8_t port = t->port[1]; + int count = 0; + int enqd; + + /* + * Takes packets from the input port and then loops them back through + * the Eventdev. Each packet gets looped through QIDs 0-8, 16 times + * so each packet goes through 8*16 = 128 times. + */ + printf("%d: \tWorker function started\n", __LINE__); + while (count < NUM_PACKETS) { +#define BURST_SIZE 32 + struct rte_event ev[BURST_SIZE]; + uint16_t i, nb_rx = rte_event_dequeue_burst(evdev, port, ev, + BURST_SIZE, 0); + if (nb_rx == 0) { + rte_pause(); + continue; + } + + for (i = 0; i < nb_rx; i++) { + ev[i].queue_id++; + if (ev[i].queue_id != 8) { + ev[i].op = RTE_EVENT_OP_FORWARD; + enqd = rte_event_enqueue_burst(evdev, port, + &ev[i], 1); + if (enqd != 1) { + printf("%d: Can't enqueue FWD!!\n", + __LINE__); + return -1; + } + continue; + } + + ev[i].queue_id = 0; + ev[i].mbuf->udata64++; + if (ev[i].mbuf->udata64 != 16) { + ev[i].op = RTE_EVENT_OP_FORWARD; + enqd = rte_event_enqueue_burst(evdev, port, + &ev[i], 1); + if (enqd != 1) { + printf("%d: Can't enqueue FWD!!\n", + __LINE__); + return -1; + } + continue; + } + /* we have hit 16 iterations through system - drop */ + rte_pktmbuf_free(ev[i].mbuf); + count++; + ev[i].op = RTE_EVENT_OP_RELEASE; + enqd = rte_event_enqueue_burst(evdev, port, &ev[i], 1); + if (enqd != 1) { + printf("%d drop enqueue failed\n", __LINE__); + return -1; + } + } + } + + return 0; +} + +static int +worker_loopback_producer_fn(void *arg) +{ + struct test *t = arg; + uint8_t port = t->port[0]; + uint64_t count = 0; + + printf("%d: \tProducer function started\n", __LINE__); + while (count < NUM_PACKETS) { + struct rte_mbuf *m = 0; + do { + m = rte_pktmbuf_alloc(t->mbuf_pool); + } while (m == NULL); + + m->udata64 = 0; + + struct rte_event ev = { + .op = RTE_EVENT_OP_NEW, + .queue_id = t->qid[0], + .flow_id = (uintptr_t)m & 0xFFFF, + .mbuf = m, + }; + + if (rte_event_enqueue_burst(evdev, port, &ev, 1) != 1) { + while (rte_event_enqueue_burst(evdev, port, &ev, 1) != + 1) + rte_pause(); + } + + count++; + } + + return 0; +} + +static int +worker_loopback(struct test *t) +{ + /* use a single producer core, and a worker core to see what happens + * if the worker loops packets back multiple times + */ + struct test_event_dev_stats stats; + uint64_t print_cycles = 0, cycles = 0; + uint64_t tx_pkts = 0; + int err; + int w_lcore, p_lcore; + + if (init(t, 8, 2) < 0 || + create_atomic_qids(t, 8) < 0) { + printf("%d: Error initializing device\n", __LINE__); + return -1; + } + + /* RX with low max events */ + static struct rte_event_port_conf conf = { + .dequeue_depth = 32, + .enqueue_depth = 64, + }; + /* beware: this cannot be initialized in the static above as it would + * only be initialized once - and this needs to be set for multiple runs + */ + conf.new_event_threshold = 512; + + if (rte_event_port_setup(evdev, 0, &conf) < 0) { + printf("Error setting up RX port\n"); + return -1; + } + t->port[0] = 0; + /* TX with higher max events */ + conf.new_event_threshold = 4096; + if (rte_event_port_setup(evdev, 1, &conf) < 0) { + printf("Error setting up TX port\n"); + return -1; + } + t->port[1] = 1; + + /* CQ mapping to QID */ + err = rte_event_port_link(evdev, t->port[1], NULL, NULL, 0); + if (err != 8) { /* should have mapped all queues*/ + printf("%d: error mapping port 2 to all qids\n", __LINE__); + return -1; + } + + if (rte_event_dev_start(evdev) < 0) { + printf("%d: Error with start call\n", __LINE__); + return -1; + } + + p_lcore = rte_get_next_lcore( + /* start core */ -1, + /* skip master */ 1, + /* wrap */ 0); + w_lcore = rte_get_next_lcore(p_lcore, 1, 0); + + rte_eal_remote_launch(worker_loopback_producer_fn, t, p_lcore); + rte_eal_remote_launch(worker_loopback_worker_fn, t, w_lcore); + + print_cycles = cycles = rte_get_timer_cycles(); + while (rte_eal_get_lcore_state(p_lcore) != FINISHED || + rte_eal_get_lcore_state(w_lcore) != FINISHED) { + + rte_service_run_iter_on_app_lcore(t->service_id, 1); + + uint64_t new_cycles = rte_get_timer_cycles(); + + if (new_cycles - print_cycles > rte_get_timer_hz()) { + test_event_dev_stats_get(evdev, &stats); + printf( + "%d: \tSched Rx = %"PRIu64", Tx = %"PRIu64"\n", + __LINE__, stats.rx_pkts, stats.tx_pkts); + + print_cycles = new_cycles; + } + if (new_cycles - cycles > rte_get_timer_hz() * 3) { + test_event_dev_stats_get(evdev, &stats); + if (stats.tx_pkts == tx_pkts) { + rte_event_dev_dump(evdev, stdout); + printf("Dumping xstats:\n"); + xstats_print(); + printf( + "%d: No schedules for seconds, deadlock\n", + __LINE__); + return -1; + } + tx_pkts = stats.tx_pkts; + cycles = new_cycles; + } + } + rte_service_run_iter_on_app_lcore(t->service_id, 1); + /* ensure all completions are flushed */ + + rte_eal_mp_wait_lcore(); + + cleanup(t); + return 0; +} + +static struct rte_mempool *eventdev_func_mempool; + +static int +test_sw_eventdev(void) +{ + struct test *t = malloc(sizeof(struct test)); + int ret; + + /* manually initialize the op, older gcc's complain on static + * initialization of struct elements that are a bitfield. + */ + release_ev.op = RTE_EVENT_OP_RELEASE; + + const char *eventdev_name = "event_sw0"; + evdev = rte_event_dev_get_dev_id(eventdev_name); + if (evdev < 0) { + printf("%d: Eventdev %s not found - creating.\n", + __LINE__, eventdev_name); + if (rte_vdev_init(eventdev_name, NULL) < 0) { + printf("Error creating eventdev\n"); + return -1; + } + evdev = rte_event_dev_get_dev_id(eventdev_name); + if (evdev < 0) { + printf("Error finding newly created eventdev\n"); + return -1; + } + } + + if (rte_event_dev_service_id_get(evdev, &t->service_id) < 0) { + printf("Failed to get service ID for software event dev\n"); + return -1; + } + + rte_service_runstate_set(t->service_id, 1); + rte_service_set_runstate_mapped_check(t->service_id, 0); + + /* Only create mbuf pool once, reuse for each test run */ + if (!eventdev_func_mempool) { + eventdev_func_mempool = rte_pktmbuf_pool_create( + "EVENTDEV_SW_SA_MBUF_POOL", + (1<<12), /* 4k buffers */ + 32 /*MBUF_CACHE_SIZE*/, + 0, + 512, /* use very small mbufs */ + rte_socket_id()); + if (!eventdev_func_mempool) { + printf("ERROR creating mempool\n"); + return -1; + } + } + t->mbuf_pool = eventdev_func_mempool; + printf("*** Running Single Directed Packet test...\n"); + ret = test_single_directed_packet(t); + if (ret != 0) { + printf("ERROR - Single Directed Packet test FAILED.\n"); + return ret; + } + printf("*** Running Directed Forward Credit test...\n"); + ret = test_directed_forward_credits(t); + if (ret != 0) { + printf("ERROR - Directed Forward Credit test FAILED.\n"); + return ret; + } + printf("*** Running Single Load Balanced Packet test...\n"); + ret = single_packet(t); + if (ret != 0) { + printf("ERROR - Single Packet test FAILED.\n"); + return ret; + } + printf("*** Running Unordered Basic test...\n"); + ret = unordered_basic(t); + if (ret != 0) { + printf("ERROR - Unordered Basic test FAILED.\n"); + return ret; + } + printf("*** Running Ordered Basic test...\n"); + ret = ordered_basic(t); + if (ret != 0) { + printf("ERROR - Ordered Basic test FAILED.\n"); + return ret; + } + printf("*** Running Burst Packets test...\n"); + ret = burst_packets(t); + if (ret != 0) { + printf("ERROR - Burst Packets test FAILED.\n"); + return ret; + } + printf("*** Running Load Balancing test...\n"); + ret = load_balancing(t); + if (ret != 0) { + printf("ERROR - Load Balancing test FAILED.\n"); + return ret; + } + printf("*** Running Prioritized Directed test...\n"); + ret = test_priority_directed(t); + if (ret != 0) { + printf("ERROR - Prioritized Directed test FAILED.\n"); + return ret; + } + printf("*** Running Prioritized Atomic test...\n"); + ret = test_priority_atomic(t); + if (ret != 0) { + printf("ERROR - Prioritized Atomic test FAILED.\n"); + return ret; + } + + printf("*** Running Prioritized Ordered test...\n"); + ret = test_priority_ordered(t); + if (ret != 0) { + printf("ERROR - Prioritized Ordered test FAILED.\n"); + return ret; + } + printf("*** Running Prioritized Unordered test...\n"); + ret = test_priority_unordered(t); + if (ret != 0) { + printf("ERROR - Prioritized Unordered test FAILED.\n"); + return ret; + } + printf("*** Running Invalid QID test...\n"); + ret = invalid_qid(t); + if (ret != 0) { + printf("ERROR - Invalid QID test FAILED.\n"); + return ret; + } + printf("*** Running Load Balancing History test...\n"); + ret = load_balancing_history(t); + if (ret != 0) { + printf("ERROR - Load Balancing History test FAILED.\n"); + return ret; + } + printf("*** Running Inflight Count test...\n"); + ret = inflight_counts(t); + if (ret != 0) { + printf("ERROR - Inflight Count test FAILED.\n"); + return ret; + } + printf("*** Running Abuse Inflights test...\n"); + ret = abuse_inflights(t); + if (ret != 0) { + printf("ERROR - Abuse Inflights test FAILED.\n"); + return ret; + } + printf("*** Running XStats test...\n"); + ret = xstats_tests(t); + if (ret != 0) { + printf("ERROR - XStats test FAILED.\n"); + return ret; + } + printf("*** Running XStats ID Reset test...\n"); + ret = xstats_id_reset_tests(t); + if (ret != 0) { + printf("ERROR - XStats ID Reset test FAILED.\n"); + return ret; + } + printf("*** Running XStats Brute Force test...\n"); + ret = xstats_brute_force(t); + if (ret != 0) { + printf("ERROR - XStats Brute Force test FAILED.\n"); + return ret; + } + printf("*** Running XStats ID Abuse test...\n"); + ret = xstats_id_abuse_tests(t); + if (ret != 0) { + printf("ERROR - XStats ID Abuse test FAILED.\n"); + return ret; + } + printf("*** Running QID Priority test...\n"); + ret = qid_priorities(t); + if (ret != 0) { + printf("ERROR - QID Priority test FAILED.\n"); + return ret; + } + printf("*** Running Ordered Reconfigure test...\n"); + ret = ordered_reconfigure(t); + if (ret != 0) { + printf("ERROR - Ordered Reconfigure test FAILED.\n"); + return ret; + } + printf("*** Running Port LB Single Reconfig test...\n"); + ret = port_single_lb_reconfig(t); + if (ret != 0) { + printf("ERROR - Port LB Single Reconfig test FAILED.\n"); + return ret; + } + printf("*** Running Port Reconfig Credits test...\n"); + ret = port_reconfig_credits(t); + if (ret != 0) { + printf("ERROR - Port Reconfig Credits Reset test FAILED.\n"); + return ret; + } + printf("*** Running Head-of-line-blocking test...\n"); + ret = holb(t); + if (ret != 0) { + printf("ERROR - Head-of-line-blocking test FAILED.\n"); + return ret; + } + if (rte_lcore_count() >= 3) { + printf("*** Running Worker loopback test...\n"); + ret = worker_loopback(t); + if (ret != 0) { + printf("ERROR - Worker loopback test FAILED.\n"); + return ret; + } + } else { + printf("### Not enough cores for worker loopback test.\n"); + printf("### Need at least 3 cores for test.\n"); + } + /* + * Free test instance, leaving mempool initialized, and a pointer to it + * in static eventdev_func_mempool, as it is re-used on re-runs + */ + free(t); + + return 0; +} + +REGISTER_TEST_COMMAND(eventdev_sw_autotest, test_sw_eventdev); diff --git a/dpdk/test/test/test_flow_classify.c b/dpdk/test/test/test_flow_classify.c new file mode 100644 index 00000000..9f331cd8 --- /dev/null +++ b/dpdk/test/test/test_flow_classify.c @@ -0,0 +1,672 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <string.h> +#include <errno.h> + +#include "test.h" + +#include <rte_string_fns.h> +#include <rte_mbuf.h> +#include <rte_byteorder.h> +#include <rte_ip.h> +#include <rte_acl.h> +#include <rte_common.h> +#include <rte_table_acl.h> +#include <rte_flow.h> +#include <rte_flow_classify.h> + +#include "packet_burst_generator.h" +#include "test_flow_classify.h" + + +#define FLOW_CLASSIFY_MAX_RULE_NUM 100 +struct flow_classifier *cls; + +struct flow_classifier { + struct rte_flow_classifier *cls; + uint32_t table_id[RTE_FLOW_CLASSIFY_TABLE_MAX]; + uint32_t n_tables; +}; + +struct flow_classifier_acl { + struct flow_classifier cls; +} __rte_cache_aligned; + +/* + * test functions by passing invalid or + * non-workable parameters. + */ +static int +test_invalid_parameters(void) +{ + struct rte_flow_classify_rule *rule; + int ret; + + rule = rte_flow_classify_table_entry_add(NULL, 1, NULL, NULL, NULL, + NULL, NULL); + if (rule) { + printf("Line %i: flow_classifier_table_entry_add", __LINE__); + printf(" with NULL param should have failed!\n"); + return -1; + } + + ret = rte_flow_classify_table_entry_delete(NULL, 1, NULL); + if (!ret) { + printf("Line %i: rte_flow_classify_table_entry_delete", + __LINE__); + printf(" with NULL param should have failed!\n"); + return -1; + } + + ret = rte_flow_classifier_query(NULL, 1, NULL, 0, NULL, NULL); + if (!ret) { + printf("Line %i: flow_classifier_query", __LINE__); + printf(" with NULL param should have failed!\n"); + return -1; + } + + rule = rte_flow_classify_table_entry_add(NULL, 1, NULL, NULL, NULL, + NULL, &error); + if (rule) { + printf("Line %i: flow_classify_table_entry_add ", __LINE__); + printf("with NULL param should have failed!\n"); + return -1; + } + + ret = rte_flow_classify_table_entry_delete(NULL, 1, NULL); + if (!ret) { + printf("Line %i: rte_flow_classify_table_entry_delete", + __LINE__); + printf("with NULL param should have failed!\n"); + return -1; + } + + ret = rte_flow_classifier_query(NULL, 1, NULL, 0, NULL, NULL); + if (!ret) { + printf("Line %i: flow_classifier_query", __LINE__); + printf(" with NULL param should have failed!\n"); + return -1; + } + return 0; +} + +static int +test_valid_parameters(void) +{ + struct rte_flow_classify_rule *rule; + int ret; + int key_found; + + /* + * set up parameters for rte_flow_classify_table_entry_add and + * rte_flow_classify_table_entry_delete + */ + + attr.ingress = 1; + attr.priority = 1; + pattern[0] = eth_item; + pattern[1] = ipv4_udp_item_1; + pattern[2] = udp_item_1; + pattern[3] = end_item; + actions[0] = count_action; + actions[1] = end_action; + + rule = rte_flow_classify_table_entry_add(cls->cls, 0, &key_found, + &attr, pattern, actions, &error); + if (!rule) { + printf("Line %i: flow_classify_table_entry_add", __LINE__); + printf(" should not have failed!\n"); + return -1; + } + + ret = rte_flow_classify_table_entry_delete(cls->cls, 0, rule); + if (ret) { + printf("Line %i: rte_flow_classify_table_entry_delete", + __LINE__); + printf(" should not have failed!\n"); + return -1; + } + return 0; +} + +static int +test_invalid_patterns(void) +{ + struct rte_flow_classify_rule *rule; + int ret; + int key_found; + + /* + * set up parameters for rte_flow_classify_table_entry_add and + * rte_flow_classify_table_entry_delete + */ + + attr.ingress = 1; + attr.priority = 1; + pattern[0] = eth_item_bad; + pattern[1] = ipv4_udp_item_1; + pattern[2] = udp_item_1; + pattern[3] = end_item; + actions[0] = count_action; + actions[1] = end_action; + + pattern[0] = eth_item; + pattern[1] = ipv4_udp_item_bad; + rule = rte_flow_classify_table_entry_add(cls->cls, 0, &key_found, + &attr, pattern, actions, &error); + if (rule) { + printf("Line %i: flow_classify_table_entry_add", __LINE__); + printf(" should have failed!\n"); + return -1; + } + + ret = rte_flow_classify_table_entry_delete(cls->cls, 0, rule); + if (!ret) { + printf("Line %i: rte_flow_classify_table_entry_delete", + __LINE__); + printf(" should have failed!\n"); + return -1; + } + + pattern[1] = ipv4_udp_item_1; + pattern[2] = udp_item_bad; + pattern[3] = end_item_bad; + rule = rte_flow_classify_table_entry_add(cls->cls, 0, &key_found, + &attr, pattern, actions, &error); + if (rule) { + printf("Line %i: flow_classify_table_entry_add", __LINE__); + printf(" should have failed!\n"); + return -1; + } + + ret = rte_flow_classify_table_entry_delete(cls->cls, 0, rule); + if (!ret) { + printf("Line %i: rte_flow_classify_table_entry_delete", + __LINE__); + printf(" should have failed!\n"); + return -1; + } + return 0; +} + +static int +test_invalid_actions(void) +{ + struct rte_flow_classify_rule *rule; + int ret; + int key_found; + + /* + * set up parameters for rte_flow_classify_table_entry_add and + * rte_flow_classify_table_entry_delete + */ + + attr.ingress = 1; + attr.priority = 1; + pattern[0] = eth_item; + pattern[1] = ipv4_udp_item_1; + pattern[2] = udp_item_1; + pattern[3] = end_item; + actions[0] = count_action_bad; + actions[1] = end_action; + + rule = rte_flow_classify_table_entry_add(cls->cls, 0, &key_found, + &attr, pattern, actions, &error); + if (rule) { + printf("Line %i: flow_classify_table_entry_add", __LINE__); + printf(" should have failed!\n"); + return -1; + } + + ret = rte_flow_classify_table_entry_delete(cls->cls, 0, rule); + if (!ret) { + printf("Line %i: rte_flow_classify_table_entry_delete", + __LINE__); + printf(" should have failed!\n"); + return -1; + } + + actions[0] = count_action; + actions[1] = end_action_bad; + + rule = rte_flow_classify_table_entry_add(cls->cls, 0, &key_found, + &attr, pattern, actions, &error); + if (rule) { + printf("Line %i: flow_classify_table_entry_add", __LINE__); + printf(" should have failed!\n"); + return -1; + } + + ret = rte_flow_classify_table_entry_delete(cls->cls, 0, rule); + if (!ret) { + printf("Line %i: rte_flow_classify_table_entry_delete", + __LINE__); + printf("should have failed!\n"); + return -1; + } + return 0; +} + +static int +init_ipv4_udp_traffic(struct rte_mempool *mp, + struct rte_mbuf **pkts_burst, uint32_t burst_size) +{ + struct ether_hdr pkt_eth_hdr; + struct ipv4_hdr pkt_ipv4_hdr; + struct udp_hdr pkt_udp_hdr; + uint32_t src_addr = IPV4_ADDR(2, 2, 2, 3); + uint32_t dst_addr = IPV4_ADDR(2, 2, 2, 7); + uint16_t src_port = 32; + uint16_t dst_port = 33; + uint16_t pktlen; + + static uint8_t src_mac[] = { 0x00, 0xFF, 0xAA, 0xFF, 0xAA, 0xFF }; + static uint8_t dst_mac[] = { 0x00, 0xAA, 0xFF, 0xAA, 0xFF, 0xAA }; + + printf("Set up IPv4 UDP traffic\n"); + initialize_eth_header(&pkt_eth_hdr, + (struct ether_addr *)src_mac, + (struct ether_addr *)dst_mac, ETHER_TYPE_IPv4, 0, 0); + pktlen = (uint16_t)(sizeof(struct ether_hdr)); + printf("ETH pktlen %u\n", pktlen); + + pktlen = initialize_ipv4_header(&pkt_ipv4_hdr, src_addr, dst_addr, + pktlen); + printf("ETH + IPv4 pktlen %u\n", pktlen); + + pktlen = initialize_udp_header(&pkt_udp_hdr, src_port, dst_port, + pktlen); + printf("ETH + IPv4 + UDP pktlen %u\n\n", pktlen); + + return generate_packet_burst(mp, pkts_burst, &pkt_eth_hdr, + 0, &pkt_ipv4_hdr, 1, + &pkt_udp_hdr, burst_size, + PACKET_BURST_GEN_PKT_LEN, 1); +} + +static int +init_ipv4_tcp_traffic(struct rte_mempool *mp, + struct rte_mbuf **pkts_burst, uint32_t burst_size) +{ + struct ether_hdr pkt_eth_hdr; + struct ipv4_hdr pkt_ipv4_hdr; + struct tcp_hdr pkt_tcp_hdr; + uint32_t src_addr = IPV4_ADDR(1, 2, 3, 4); + uint32_t dst_addr = IPV4_ADDR(5, 6, 7, 8); + uint16_t src_port = 16; + uint16_t dst_port = 17; + uint16_t pktlen; + + static uint8_t src_mac[] = { 0x00, 0xFF, 0xAA, 0xFF, 0xAA, 0xFF }; + static uint8_t dst_mac[] = { 0x00, 0xAA, 0xFF, 0xAA, 0xFF, 0xAA }; + + printf("Set up IPv4 TCP traffic\n"); + initialize_eth_header(&pkt_eth_hdr, + (struct ether_addr *)src_mac, + (struct ether_addr *)dst_mac, ETHER_TYPE_IPv4, 0, 0); + pktlen = (uint16_t)(sizeof(struct ether_hdr)); + printf("ETH pktlen %u\n", pktlen); + + pktlen = initialize_ipv4_header_proto(&pkt_ipv4_hdr, src_addr, + dst_addr, pktlen, IPPROTO_TCP); + printf("ETH + IPv4 pktlen %u\n", pktlen); + + pktlen = initialize_tcp_header(&pkt_tcp_hdr, src_port, dst_port, + pktlen); + printf("ETH + IPv4 + TCP pktlen %u\n\n", pktlen); + + return generate_packet_burst_proto(mp, pkts_burst, &pkt_eth_hdr, + 0, &pkt_ipv4_hdr, 1, IPPROTO_TCP, + &pkt_tcp_hdr, burst_size, + PACKET_BURST_GEN_PKT_LEN, 1); +} + +static int +init_ipv4_sctp_traffic(struct rte_mempool *mp, + struct rte_mbuf **pkts_burst, uint32_t burst_size) +{ + struct ether_hdr pkt_eth_hdr; + struct ipv4_hdr pkt_ipv4_hdr; + struct sctp_hdr pkt_sctp_hdr; + uint32_t src_addr = IPV4_ADDR(11, 12, 13, 14); + uint32_t dst_addr = IPV4_ADDR(15, 16, 17, 18); + uint16_t src_port = 10; + uint16_t dst_port = 11; + uint16_t pktlen; + + static uint8_t src_mac[] = { 0x00, 0xFF, 0xAA, 0xFF, 0xAA, 0xFF }; + static uint8_t dst_mac[] = { 0x00, 0xAA, 0xFF, 0xAA, 0xFF, 0xAA }; + + printf("Set up IPv4 SCTP traffic\n"); + initialize_eth_header(&pkt_eth_hdr, + (struct ether_addr *)src_mac, + (struct ether_addr *)dst_mac, ETHER_TYPE_IPv4, 0, 0); + pktlen = (uint16_t)(sizeof(struct ether_hdr)); + printf("ETH pktlen %u\n", pktlen); + + pktlen = initialize_ipv4_header_proto(&pkt_ipv4_hdr, src_addr, + dst_addr, pktlen, IPPROTO_SCTP); + printf("ETH + IPv4 pktlen %u\n", pktlen); + + pktlen = initialize_sctp_header(&pkt_sctp_hdr, src_port, dst_port, + pktlen); + printf("ETH + IPv4 + SCTP pktlen %u\n\n", pktlen); + + return generate_packet_burst_proto(mp, pkts_burst, &pkt_eth_hdr, + 0, &pkt_ipv4_hdr, 1, IPPROTO_SCTP, + &pkt_sctp_hdr, burst_size, + PACKET_BURST_GEN_PKT_LEN, 1); +} + +static int +init_mbufpool(void) +{ + int socketid; + int ret = 0; + unsigned int lcore_id; + char s[64]; + + for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { + if (rte_lcore_is_enabled(lcore_id) == 0) + continue; + + socketid = rte_lcore_to_socket_id(lcore_id); + if (socketid >= NB_SOCKETS) { + printf( + "Socket %d of lcore %u is out of range %d\n", + socketid, lcore_id, NB_SOCKETS); + ret = -1; + break; + } + if (mbufpool[socketid] == NULL) { + snprintf(s, sizeof(s), "mbuf_pool_%d", socketid); + mbufpool[socketid] = + rte_pktmbuf_pool_create(s, NB_MBUF, + MEMPOOL_CACHE_SIZE, 0, MBUF_SIZE, + socketid); + if (mbufpool[socketid]) { + printf("Allocated mbuf pool on socket %d\n", + socketid); + } else { + printf("Cannot init mbuf pool on socket %d\n", + socketid); + ret = -ENOMEM; + break; + } + } + } + return ret; +} + +static int +test_query_udp(void) +{ + struct rte_flow_error error; + struct rte_flow_classify_rule *rule; + int ret; + int i; + int key_found; + + ret = init_ipv4_udp_traffic(mbufpool[0], bufs, MAX_PKT_BURST); + if (ret != MAX_PKT_BURST) { + printf("Line %i: init_udp_ipv4_traffic has failed!\n", + __LINE__); + return -1; + } + + for (i = 0; i < MAX_PKT_BURST; i++) + bufs[i]->packet_type = RTE_PTYPE_L3_IPV4; + + /* + * set up parameters for rte_flow_classify_table_entry_add and + * rte_flow_classify_table_entry_delete + */ + + attr.ingress = 1; + attr.priority = 1; + pattern[0] = eth_item; + pattern[1] = ipv4_udp_item_1; + pattern[2] = udp_item_1; + pattern[3] = end_item; + actions[0] = count_action; + actions[1] = end_action; + + rule = rte_flow_classify_table_entry_add(cls->cls, 0, &key_found, + &attr, pattern, actions, &error); + if (!rule) { + printf("Line %i: flow_classify_table_entry_add", __LINE__); + printf(" should not have failed!\n"); + return -1; + } + + ret = rte_flow_classifier_query(cls->cls, 0, bufs, MAX_PKT_BURST, + rule, &udp_classify_stats); + if (ret) { + printf("Line %i: flow_classifier_query", __LINE__); + printf(" should not have failed!\n"); + return -1; + } + + ret = rte_flow_classify_table_entry_delete(cls->cls, 0, rule); + if (ret) { + printf("Line %i: rte_flow_classify_table_entry_delete", + __LINE__); + printf(" should not have failed!\n"); + return -1; + } + return 0; +} + +static int +test_query_tcp(void) +{ + struct rte_flow_classify_rule *rule; + int ret; + int i; + int key_found; + + ret = init_ipv4_tcp_traffic(mbufpool[0], bufs, MAX_PKT_BURST); + if (ret != MAX_PKT_BURST) { + printf("Line %i: init_ipv4_tcp_traffic has failed!\n", + __LINE__); + return -1; + } + + for (i = 0; i < MAX_PKT_BURST; i++) + bufs[i]->packet_type = RTE_PTYPE_L3_IPV4; + + /* + * set up parameters for rte_flow_classify_table_entry_add and + * rte_flow_classify_table_entry_delete + */ + + attr.ingress = 1; + attr.priority = 1; + pattern[0] = eth_item; + pattern[1] = ipv4_tcp_item_1; + pattern[2] = tcp_item_1; + pattern[3] = end_item; + actions[0] = count_action; + actions[1] = end_action; + + rule = rte_flow_classify_table_entry_add(cls->cls, 0, &key_found, + &attr, pattern, actions, &error); + if (!rule) { + printf("Line %i: flow_classify_table_entry_add", __LINE__); + printf(" should not have failed!\n"); + return -1; + } + + ret = rte_flow_classifier_query(cls->cls, 0, bufs, MAX_PKT_BURST, + rule, &tcp_classify_stats); + if (ret) { + printf("Line %i: flow_classifier_query", __LINE__); + printf(" should not have failed!\n"); + return -1; + } + + ret = rte_flow_classify_table_entry_delete(cls->cls, 0, rule); + if (ret) { + printf("Line %i: rte_flow_classify_table_entry_delete", + __LINE__); + printf(" should not have failed!\n"); + return -1; + } + return 0; +} + +static int +test_query_sctp(void) +{ + struct rte_flow_classify_rule *rule; + int ret; + int i; + int key_found; + + ret = init_ipv4_sctp_traffic(mbufpool[0], bufs, MAX_PKT_BURST); + if (ret != MAX_PKT_BURST) { + printf("Line %i: init_ipv4_tcp_traffic has failed!\n", + __LINE__); + return -1; + } + + for (i = 0; i < MAX_PKT_BURST; i++) + bufs[i]->packet_type = RTE_PTYPE_L3_IPV4; + + /* + * set up parameters rte_flow_classify_table_entry_add and + * rte_flow_classify_table_entry_delete + */ + + attr.ingress = 1; + attr.priority = 1; + pattern[0] = eth_item; + pattern[1] = ipv4_sctp_item_1; + pattern[2] = sctp_item_1; + pattern[3] = end_item; + actions[0] = count_action; + actions[1] = end_action; + + rule = rte_flow_classify_table_entry_add(cls->cls, 0, &key_found, + &attr, pattern, actions, &error); + if (!rule) { + printf("Line %i: flow_classify_table_entry_add", __LINE__); + printf(" should not have failed!\n"); + return -1; + } + + ret = rte_flow_classifier_query(cls->cls, 0, bufs, MAX_PKT_BURST, + rule, &sctp_classify_stats); + if (ret) { + printf("Line %i: flow_classifier_query", __LINE__); + printf(" should not have failed!\n"); + return -1; + } + + ret = rte_flow_classify_table_entry_delete(cls->cls, 0, rule); + if (ret) { + printf("Line %i: rte_flow_classify_table_entry_delete", + __LINE__); + printf(" should not have failed!\n"); + return -1; + } + return 0; +} + +static int +test_flow_classify(void) +{ + struct rte_table_acl_params table_acl_params; + struct rte_flow_classify_table_params cls_table_params; + struct rte_flow_classifier_params cls_params; + int socket_id; + int ret; + uint32_t size; + + socket_id = rte_eth_dev_socket_id(0); + + /* Memory allocation */ + size = RTE_CACHE_LINE_ROUNDUP(sizeof(struct flow_classifier_acl)); + cls = rte_zmalloc(NULL, size, RTE_CACHE_LINE_SIZE); + + cls_params.name = "flow_classifier"; + cls_params.socket_id = socket_id; + cls_params.type = RTE_FLOW_CLASSIFY_TABLE_TYPE_ACL; + cls->cls = rte_flow_classifier_create(&cls_params); + + /* initialise ACL table params */ + table_acl_params.n_rule_fields = RTE_DIM(ipv4_defs); + table_acl_params.name = "table_acl_ipv4_5tuple"; + table_acl_params.n_rules = FLOW_CLASSIFY_MAX_RULE_NUM; + memcpy(table_acl_params.field_format, ipv4_defs, sizeof(ipv4_defs)); + + /* initialise table create params */ + cls_table_params.ops = &rte_table_acl_ops, + cls_table_params.arg_create = &table_acl_params, + + ret = rte_flow_classify_table_create(cls->cls, &cls_table_params, + &cls->table_id[0]); + if (ret) { + printf("Line %i: f_create has failed!\n", __LINE__); + rte_flow_classifier_free(cls->cls); + rte_free(cls); + return -1; + } + printf("Created table_acl for for IPv4 five tuple packets\n"); + + ret = init_mbufpool(); + if (ret) { + printf("Line %i: init_mbufpool has failed!\n", __LINE__); + return -1; + } + + if (test_invalid_parameters() < 0) + return -1; + if (test_valid_parameters() < 0) + return -1; + if (test_invalid_patterns() < 0) + return -1; + if (test_invalid_actions() < 0) + return -1; + if (test_query_udp() < 0) + return -1; + if (test_query_tcp() < 0) + return -1; + if (test_query_sctp() < 0) + return -1; + + return 0; +} + +REGISTER_TEST_COMMAND(flow_classify_autotest, test_flow_classify); diff --git a/dpdk/test/test/test_flow_classify.h b/dpdk/test/test/test_flow_classify.h new file mode 100644 index 00000000..39535cfc --- /dev/null +++ b/dpdk/test/test/test_flow_classify.h @@ -0,0 +1,234 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef TEST_FLOW_CLASSIFY_H_ +#define TEST_FLOW_CLASSIFY_H_ + +#define MAX_PKT_BURST (32) +#define NB_SOCKETS (1) +#define MEMPOOL_CACHE_SIZE (256) +#define MBUF_SIZE (512) +#define NB_MBUF (512) + +/* test UDP, TCP and SCTP packets */ +static struct rte_mempool *mbufpool[NB_SOCKETS]; +static struct rte_mbuf *bufs[MAX_PKT_BURST]; + +/* ACL field definitions for IPv4 5 tuple rule */ + +enum { + PROTO_FIELD_IPV4, + SRC_FIELD_IPV4, + DST_FIELD_IPV4, + SRCP_FIELD_IPV4, + DSTP_FIELD_IPV4, + NUM_FIELDS_IPV4 +}; + +enum { + PROTO_INPUT_IPV4, + SRC_INPUT_IPV4, + DST_INPUT_IPV4, + SRCP_DESTP_INPUT_IPV4 +}; + +static struct rte_acl_field_def ipv4_defs[NUM_FIELDS_IPV4] = { + /* first input field - always one byte long. */ + { + .type = RTE_ACL_FIELD_TYPE_BITMASK, + .size = sizeof(uint8_t), + .field_index = PROTO_FIELD_IPV4, + .input_index = PROTO_INPUT_IPV4, + .offset = sizeof(struct ether_hdr) + + offsetof(struct ipv4_hdr, next_proto_id), + }, + /* next input field (IPv4 source address) - 4 consecutive bytes. */ + { + /* rte_flow uses a bit mask for IPv4 addresses */ + .type = RTE_ACL_FIELD_TYPE_BITMASK, + .size = sizeof(uint32_t), + .field_index = SRC_FIELD_IPV4, + .input_index = SRC_INPUT_IPV4, + .offset = sizeof(struct ether_hdr) + + offsetof(struct ipv4_hdr, src_addr), + }, + /* next input field (IPv4 destination address) - 4 consecutive bytes. */ + { + /* rte_flow uses a bit mask for IPv4 addresses */ + .type = RTE_ACL_FIELD_TYPE_BITMASK, + .size = sizeof(uint32_t), + .field_index = DST_FIELD_IPV4, + .input_index = DST_INPUT_IPV4, + .offset = sizeof(struct ether_hdr) + + offsetof(struct ipv4_hdr, dst_addr), + }, + /* + * Next 2 fields (src & dst ports) form 4 consecutive bytes. + * They share the same input index. + */ + { + /* rte_flow uses a bit mask for protocol ports */ + .type = RTE_ACL_FIELD_TYPE_BITMASK, + .size = sizeof(uint16_t), + .field_index = SRCP_FIELD_IPV4, + .input_index = SRCP_DESTP_INPUT_IPV4, + .offset = sizeof(struct ether_hdr) + + sizeof(struct ipv4_hdr) + + offsetof(struct tcp_hdr, src_port), + }, + { + /* rte_flow uses a bit mask for protocol ports */ + .type = RTE_ACL_FIELD_TYPE_BITMASK, + .size = sizeof(uint16_t), + .field_index = DSTP_FIELD_IPV4, + .input_index = SRCP_DESTP_INPUT_IPV4, + .offset = sizeof(struct ether_hdr) + + sizeof(struct ipv4_hdr) + + offsetof(struct tcp_hdr, dst_port), + }, +}; + +/* parameters for rte_flow_classify_validate and rte_flow_classify_create */ + +/* test UDP pattern: + * "eth / ipv4 src spec 2.2.2.3 src mask 255.255.255.00 dst spec 2.2.2.7 + * dst mask 255.255.255.00 / udp src is 32 dst is 33 / end" + */ +static struct rte_flow_item_ipv4 ipv4_udp_spec_1 = { + { 0, 0, 0, 0, 0, 0, IPPROTO_UDP, 0, IPv4(2, 2, 2, 3), IPv4(2, 2, 2, 7)} +}; +static const struct rte_flow_item_ipv4 ipv4_mask_24 = { + .hdr = { + .next_proto_id = 0xff, + .src_addr = 0xffffff00, + .dst_addr = 0xffffff00, + }, +}; +static struct rte_flow_item_udp udp_spec_1 = { + { 32, 33, 0, 0 } +}; + +static struct rte_flow_item eth_item = { RTE_FLOW_ITEM_TYPE_ETH, + 0, 0, 0 }; +static struct rte_flow_item eth_item_bad = { -1, 0, 0, 0 }; + +static struct rte_flow_item ipv4_udp_item_1 = { RTE_FLOW_ITEM_TYPE_IPV4, + &ipv4_udp_spec_1, 0, &ipv4_mask_24}; +static struct rte_flow_item ipv4_udp_item_bad = { RTE_FLOW_ITEM_TYPE_IPV4, + NULL, 0, NULL}; + +static struct rte_flow_item udp_item_1 = { RTE_FLOW_ITEM_TYPE_UDP, + &udp_spec_1, 0, &rte_flow_item_udp_mask}; +static struct rte_flow_item udp_item_bad = { RTE_FLOW_ITEM_TYPE_UDP, + NULL, 0, NULL}; + +static struct rte_flow_item end_item = { RTE_FLOW_ITEM_TYPE_END, + 0, 0, 0 }; +static struct rte_flow_item end_item_bad = { -1, 0, 0, 0 }; + +/* test TCP pattern: + * "eth / ipv4 src spec 1.2.3.4 src mask 255.255.255.00 dst spec 5.6.7.8 + * dst mask 255.255.255.00 / tcp src is 16 dst is 17 / end" + */ +static struct rte_flow_item_ipv4 ipv4_tcp_spec_1 = { + { 0, 0, 0, 0, 0, 0, IPPROTO_TCP, 0, IPv4(1, 2, 3, 4), IPv4(5, 6, 7, 8)} +}; + +static struct rte_flow_item_tcp tcp_spec_1 = { + { 16, 17, 0, 0, 0, 0, 0, 0, 0} +}; + +static struct rte_flow_item ipv4_tcp_item_1 = { RTE_FLOW_ITEM_TYPE_IPV4, + &ipv4_tcp_spec_1, 0, &ipv4_mask_24}; + +static struct rte_flow_item tcp_item_1 = { RTE_FLOW_ITEM_TYPE_TCP, + &tcp_spec_1, 0, &rte_flow_item_tcp_mask}; + +/* test SCTP pattern: + * "eth / ipv4 src spec 1.2.3.4 src mask 255.255.255.00 dst spec 5.6.7.8 + * dst mask 255.255.255.00 / sctp src is 16 dst is 17/ end" + */ +static struct rte_flow_item_ipv4 ipv4_sctp_spec_1 = { + { 0, 0, 0, 0, 0, 0, IPPROTO_SCTP, 0, IPv4(11, 12, 13, 14), + IPv4(15, 16, 17, 18)} +}; + +static struct rte_flow_item_sctp sctp_spec_1 = { + { 10, 11, 0, 0} +}; + +static struct rte_flow_item ipv4_sctp_item_1 = { RTE_FLOW_ITEM_TYPE_IPV4, + &ipv4_sctp_spec_1, 0, &ipv4_mask_24}; + +static struct rte_flow_item sctp_item_1 = { RTE_FLOW_ITEM_TYPE_SCTP, + &sctp_spec_1, 0, &rte_flow_item_sctp_mask}; + + +/* test actions: + * "actions count / end" + */ +static struct rte_flow_action count_action = { RTE_FLOW_ACTION_TYPE_COUNT, 0}; +static struct rte_flow_action count_action_bad = { -1, 0}; + +static struct rte_flow_action end_action = { RTE_FLOW_ACTION_TYPE_END, 0}; +static struct rte_flow_action end_action_bad = { -1, 0}; + +static struct rte_flow_action actions[2]; + +/* test attributes */ +static struct rte_flow_attr attr; + +/* test error */ +static struct rte_flow_error error; + +/* test pattern */ +static struct rte_flow_item pattern[4]; + +/* flow classify data for UDP burst */ +static struct rte_flow_classify_ipv4_5tuple_stats udp_ntuple_stats; +static struct rte_flow_classify_stats udp_classify_stats = { + .stats = (void *)&udp_ntuple_stats +}; + +/* flow classify data for TCP burst */ +static struct rte_flow_classify_ipv4_5tuple_stats tcp_ntuple_stats; +static struct rte_flow_classify_stats tcp_classify_stats = { + .stats = (void *)&tcp_ntuple_stats +}; + +/* flow classify data for SCTP burst */ +static struct rte_flow_classify_ipv4_5tuple_stats sctp_ntuple_stats; +static struct rte_flow_classify_stats sctp_classify_stats = { + .stats = (void *)&sctp_ntuple_stats +}; +#endif /* TEST_FLOW_CLASSIFY_H_ */ diff --git a/dpdk/test/test/test_func_reentrancy.c b/dpdk/test/test/test_func_reentrancy.c new file mode 100644 index 00000000..514c8761 --- /dev/null +++ b/dpdk/test/test/test_func_reentrancy.c @@ -0,0 +1,509 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <string.h> +#include <stdio.h> +#include <stdlib.h> +#include <stdint.h> +#include <inttypes.h> +#include <stdarg.h> +#include <errno.h> +#include <sys/queue.h> + +#include <rte_common.h> +#include <rte_log.h> +#include <rte_debug.h> +#include <rte_memory.h> +#include <rte_launch.h> +#include <rte_cycles.h> +#include <rte_eal.h> +#include <rte_per_lcore.h> +#include <rte_lcore.h> +#include <rte_atomic.h> +#include <rte_branch_prediction.h> +#include <rte_ring.h> +#include <rte_mempool.h> +#include <rte_spinlock.h> +#include <rte_malloc.h> + +#ifdef RTE_LIBRTE_HASH +#include <rte_hash.h> +#include <rte_fbk_hash.h> +#include <rte_jhash.h> +#endif /* RTE_LIBRTE_HASH */ + +#ifdef RTE_LIBRTE_LPM +#include <rte_lpm.h> +#endif /* RTE_LIBRTE_LPM */ + +#include <rte_string_fns.h> + +#include "test.h" + +typedef int (*case_func_t)(void* arg); +typedef void (*case_clean_t)(unsigned lcore_id); + +#define MAX_STRING_SIZE (256) +#define MAX_ITER_TIMES (16) +#define MAX_LPM_ITER_TIMES (8) + +#define MEMPOOL_ELT_SIZE (sizeof(uint32_t)) +#define MEMPOOL_SIZE (4) + +#define MAX_LCORES RTE_MAX_MEMZONE / (MAX_ITER_TIMES * 4U) + +static rte_atomic32_t obj_count = RTE_ATOMIC32_INIT(0); +static rte_atomic32_t synchro = RTE_ATOMIC32_INIT(0); + +#define WAIT_SYNCHRO_FOR_SLAVES() do{ \ + if (lcore_self != rte_get_master_lcore()) \ + while (rte_atomic32_read(&synchro) == 0); \ +} while(0) + +/* + * rte_eal_init only init once + */ +static int +test_eal_init_once(__attribute__((unused)) void *arg) +{ + unsigned lcore_self = rte_lcore_id(); + + WAIT_SYNCHRO_FOR_SLAVES(); + + rte_atomic32_set(&obj_count, 1); /* silent the check in the caller */ + if (rte_eal_init(0, NULL) != -1) + return -1; + + return 0; +} + +/* + * ring create/lookup reentrancy test + */ +static int +ring_create_lookup(__attribute__((unused)) void *arg) +{ + unsigned lcore_self = rte_lcore_id(); + struct rte_ring * rp; + char ring_name[MAX_STRING_SIZE]; + int i; + + WAIT_SYNCHRO_FOR_SLAVES(); + + /* create the same ring simultaneously on all threads */ + for (i = 0; i < MAX_ITER_TIMES; i++) { + rp = rte_ring_create("fr_test_once", 4096, SOCKET_ID_ANY, 0); + if (rp != NULL) + rte_atomic32_inc(&obj_count); + } + + /* create/lookup new ring several times */ + for (i = 0; i < MAX_ITER_TIMES; i++) { + snprintf(ring_name, sizeof(ring_name), "fr_test_%d_%d", lcore_self, i); + rp = rte_ring_create(ring_name, 4096, SOCKET_ID_ANY, 0); + if (NULL == rp) + return -1; + if (rte_ring_lookup(ring_name) != rp) + return -1; + } + + /* verify all ring created successful */ + for (i = 0; i < MAX_ITER_TIMES; i++) { + snprintf(ring_name, sizeof(ring_name), "fr_test_%d_%d", lcore_self, i); + if (rte_ring_lookup(ring_name) == NULL) + return -1; + } + + return 0; +} + +static void +my_obj_init(struct rte_mempool *mp, __attribute__((unused)) void *arg, + void *obj, unsigned i) +{ + uint32_t *objnum = obj; + memset(obj, 0, mp->elt_size); + *objnum = i; +} + +static int +mempool_create_lookup(__attribute__((unused)) void *arg) +{ + unsigned lcore_self = rte_lcore_id(); + struct rte_mempool * mp; + char mempool_name[MAX_STRING_SIZE]; + int i; + + WAIT_SYNCHRO_FOR_SLAVES(); + + /* create the same mempool simultaneously on all threads */ + for (i = 0; i < MAX_ITER_TIMES; i++) { + mp = rte_mempool_create("fr_test_once", MEMPOOL_SIZE, + MEMPOOL_ELT_SIZE, 0, 0, + NULL, NULL, + my_obj_init, NULL, + SOCKET_ID_ANY, 0); + if (mp != NULL) + rte_atomic32_inc(&obj_count); + } + + /* create/lookup new ring several times */ + for (i = 0; i < MAX_ITER_TIMES; i++) { + snprintf(mempool_name, sizeof(mempool_name), "fr_test_%d_%d", lcore_self, i); + mp = rte_mempool_create(mempool_name, MEMPOOL_SIZE, + MEMPOOL_ELT_SIZE, 0, 0, + NULL, NULL, + my_obj_init, NULL, + SOCKET_ID_ANY, 0); + if (NULL == mp) + return -1; + if (rte_mempool_lookup(mempool_name) != mp) + return -1; + } + + /* verify all ring created successful */ + for (i = 0; i < MAX_ITER_TIMES; i++) { + snprintf(mempool_name, sizeof(mempool_name), "fr_test_%d_%d", lcore_self, i); + if (rte_mempool_lookup(mempool_name) == NULL) + return -1; + } + + return 0; +} + +#ifdef RTE_LIBRTE_HASH +static void +hash_clean(unsigned lcore_id) +{ + char hash_name[MAX_STRING_SIZE]; + struct rte_hash *handle; + int i; + + for (i = 0; i < MAX_ITER_TIMES; i++) { + snprintf(hash_name, sizeof(hash_name), "fr_test_%d_%d", lcore_id, i); + + if ((handle = rte_hash_find_existing(hash_name)) != NULL) + rte_hash_free(handle); + } +} + +static int +hash_create_free(__attribute__((unused)) void *arg) +{ + unsigned lcore_self = rte_lcore_id(); + struct rte_hash *handle; + char hash_name[MAX_STRING_SIZE]; + int i; + struct rte_hash_parameters hash_params = { + .name = NULL, + .entries = 16, + .key_len = 4, + .hash_func = (rte_hash_function)rte_jhash_32b, + .hash_func_init_val = 0, + .socket_id = 0, + }; + + WAIT_SYNCHRO_FOR_SLAVES(); + + /* create the same hash simultaneously on all threads */ + hash_params.name = "fr_test_once"; + for (i = 0; i < MAX_ITER_TIMES; i++) { + handle = rte_hash_create(&hash_params); + if (handle != NULL) + rte_atomic32_inc(&obj_count); + } + + /* create mutiple times simultaneously */ + for (i = 0; i < MAX_ITER_TIMES; i++) { + snprintf(hash_name, sizeof(hash_name), "fr_test_%d_%d", lcore_self, i); + hash_params.name = hash_name; + + handle = rte_hash_create(&hash_params); + if (NULL == handle) + return -1; + + /* verify correct existing and then free all */ + if (handle != rte_hash_find_existing(hash_name)) + return -1; + + rte_hash_free(handle); + } + + /* verify free correct */ + for (i = 0; i < MAX_ITER_TIMES; i++) { + snprintf(hash_name, sizeof(hash_name), "fr_test_%d_%d", lcore_self, i); + + if (NULL != rte_hash_find_existing(hash_name)) + return -1; + } + + return 0; +} + +static void +fbk_clean(unsigned lcore_id) +{ + char fbk_name[MAX_STRING_SIZE]; + struct rte_fbk_hash_table *handle; + int i; + + for (i = 0; i < MAX_ITER_TIMES; i++) { + snprintf(fbk_name, sizeof(fbk_name), "fr_test_%d_%d", lcore_id, i); + + if ((handle = rte_fbk_hash_find_existing(fbk_name)) != NULL) + rte_fbk_hash_free(handle); + } +} + +static int +fbk_create_free(__attribute__((unused)) void *arg) +{ + unsigned lcore_self = rte_lcore_id(); + struct rte_fbk_hash_table *handle; + char fbk_name[MAX_STRING_SIZE]; + int i; + struct rte_fbk_hash_params fbk_params = { + .name = NULL, + .entries = 4, + .entries_per_bucket = 4, + .socket_id = 0, + .hash_func = rte_jhash_1word, + .init_val = RTE_FBK_HASH_INIT_VAL_DEFAULT, + }; + + WAIT_SYNCHRO_FOR_SLAVES(); + + /* create the same fbk hash table simultaneously on all threads */ + fbk_params.name = "fr_test_once"; + for (i = 0; i < MAX_ITER_TIMES; i++) { + handle = rte_fbk_hash_create(&fbk_params); + if (handle != NULL) + rte_atomic32_inc(&obj_count); + } + + /* create mutiple fbk tables simultaneously */ + for (i = 0; i < MAX_ITER_TIMES; i++) { + snprintf(fbk_name, sizeof(fbk_name), "fr_test_%d_%d", lcore_self, i); + fbk_params.name = fbk_name; + + handle = rte_fbk_hash_create(&fbk_params); + if (NULL == handle) + return -1; + + /* verify correct existing and then free all */ + if (handle != rte_fbk_hash_find_existing(fbk_name)) + return -1; + + rte_fbk_hash_free(handle); + } + + /* verify free correct */ + for (i = 0; i < MAX_ITER_TIMES; i++) { + snprintf(fbk_name, sizeof(fbk_name), "fr_test_%d_%d", lcore_self, i); + + if (NULL != rte_fbk_hash_find_existing(fbk_name)) + return -1; + } + + return 0; +} +#endif /* RTE_LIBRTE_HASH */ + +#ifdef RTE_LIBRTE_LPM +static void +lpm_clean(unsigned lcore_id) +{ + char lpm_name[MAX_STRING_SIZE]; + struct rte_lpm *lpm; + int i; + + for (i = 0; i < MAX_LPM_ITER_TIMES; i++) { + snprintf(lpm_name, sizeof(lpm_name), "fr_test_%d_%d", lcore_id, i); + + if ((lpm = rte_lpm_find_existing(lpm_name)) != NULL) + rte_lpm_free(lpm); + } +} + +static int +lpm_create_free(__attribute__((unused)) void *arg) +{ + unsigned lcore_self = rte_lcore_id(); + struct rte_lpm *lpm; + struct rte_lpm_config config; + + config.max_rules = 4; + config.number_tbl8s = 256; + config.flags = 0; + char lpm_name[MAX_STRING_SIZE]; + int i; + + WAIT_SYNCHRO_FOR_SLAVES(); + + /* create the same lpm simultaneously on all threads */ + for (i = 0; i < MAX_ITER_TIMES; i++) { + lpm = rte_lpm_create("fr_test_once", SOCKET_ID_ANY, &config); + if (lpm != NULL) + rte_atomic32_inc(&obj_count); + } + + /* create mutiple fbk tables simultaneously */ + for (i = 0; i < MAX_LPM_ITER_TIMES; i++) { + snprintf(lpm_name, sizeof(lpm_name), "fr_test_%d_%d", lcore_self, i); + lpm = rte_lpm_create(lpm_name, SOCKET_ID_ANY, &config); + if (NULL == lpm) + return -1; + + /* verify correct existing and then free all */ + if (lpm != rte_lpm_find_existing(lpm_name)) + return -1; + + rte_lpm_free(lpm); + } + + /* verify free correct */ + for (i = 0; i < MAX_LPM_ITER_TIMES; i++) { + snprintf(lpm_name, sizeof(lpm_name), "fr_test_%d_%d", lcore_self, i); + if (NULL != rte_lpm_find_existing(lpm_name)) + return -1; + } + + return 0; +} +#endif /* RTE_LIBRTE_LPM */ + +struct test_case{ + case_func_t func; + void* arg; + case_clean_t clean; + char name[MAX_STRING_SIZE]; +}; + +/* All test cases in the test suite */ +struct test_case test_cases[] = { + { test_eal_init_once, NULL, NULL, "eal init once" }, + { ring_create_lookup, NULL, NULL, "ring create/lookup" }, + { mempool_create_lookup, NULL, NULL, "mempool create/lookup" }, +#ifdef RTE_LIBRTE_HASH + { hash_create_free, NULL, hash_clean, "hash create/free" }, + { fbk_create_free, NULL, fbk_clean, "fbk create/free" }, +#endif /* RTE_LIBRTE_HASH */ +#ifdef RTE_LIBRTE_LPM + { lpm_create_free, NULL, lpm_clean, "lpm create/free" }, +#endif /* RTE_LIBRTE_LPM */ +}; + +/** + * launch test case in two separate thread + */ +static int +launch_test(struct test_case *pt_case) +{ + int ret = 0; + unsigned lcore_id; + unsigned cores_save = rte_lcore_count(); + unsigned cores = RTE_MIN(cores_save, MAX_LCORES); + unsigned count; + + if (pt_case->func == NULL) + return -1; + + rte_atomic32_set(&obj_count, 0); + rte_atomic32_set(&synchro, 0); + + RTE_LCORE_FOREACH_SLAVE(lcore_id) { + if (cores == 1) + break; + cores--; + rte_eal_remote_launch(pt_case->func, pt_case->arg, lcore_id); + } + + rte_atomic32_set(&synchro, 1); + + if (pt_case->func(pt_case->arg) < 0) + ret = -1; + + cores = cores_save; + RTE_LCORE_FOREACH_SLAVE(lcore_id) { + if (cores == 1) + break; + cores--; + if (rte_eal_wait_lcore(lcore_id) < 0) + ret = -1; + + if (pt_case->clean != NULL) + pt_case->clean(lcore_id); + } + + count = rte_atomic32_read(&obj_count); + if (count != 1) { + printf("%s: common object allocated %d times (should be 1)\n", + pt_case->name, count); + ret = -1; + } + + return ret; +} + +/** + * Main entry of func_reentrancy test + */ +static int +test_func_reentrancy(void) +{ + uint32_t case_id; + struct test_case *pt_case = NULL; + + if (rte_lcore_count() <= 1) { + printf("Not enough lcore for testing\n"); + return -1; + } + else if (rte_lcore_count() > MAX_LCORES) + printf("Too many lcores, some cores will be disabled\n"); + + for (case_id = 0; case_id < sizeof(test_cases)/sizeof(struct test_case); case_id ++) { + pt_case = &test_cases[case_id]; + if (pt_case->func == NULL) + continue; + + if (launch_test(pt_case) < 0) { + printf("Func-ReEnt CASE %"PRIu32": %s FAIL\n", case_id, pt_case->name); + return -1; + } + printf("Func-ReEnt CASE %"PRIu32": %s PASS\n", case_id, pt_case->name); + } + + return 0; +} + +REGISTER_TEST_COMMAND(func_reentrancy_autotest, test_func_reentrancy); diff --git a/dpdk/test/test/test_hash.c b/dpdk/test/test/test_hash.c new file mode 100644 index 00000000..4668cf1a --- /dev/null +++ b/dpdk/test/test/test_hash.c @@ -0,0 +1,1516 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <stdint.h> +#include <string.h> +#include <stdlib.h> +#include <stdarg.h> +#include <errno.h> +#include <sys/queue.h> + +#include <rte_common.h> +#include <rte_malloc.h> +#include <rte_cycles.h> +#include <rte_random.h> +#include <rte_memory.h> +#include <rte_eal.h> +#include <rte_ip.h> +#include <rte_string_fns.h> + +#include "test.h" + +#include <rte_hash.h> +#include <rte_fbk_hash.h> +#include <rte_jhash.h> +#include <rte_hash_crc.h> + +/******************************************************************************* + * Hash function performance test configuration section. Each performance test + * will be performed HASHTEST_ITERATIONS times. + * + * The five arrays below control what tests are performed. Every combination + * from the array entries is tested. + */ +static rte_hash_function hashtest_funcs[] = {rte_jhash, rte_hash_crc}; +static uint32_t hashtest_initvals[] = {0}; +static uint32_t hashtest_key_lens[] = {0, 2, 4, 5, 6, 7, 8, 10, 11, 15, 16, 21, 31, 32, 33, 63, 64}; +#define MAX_KEYSIZE 64 +/******************************************************************************/ +#define LOCAL_FBK_HASH_ENTRIES_MAX (1 << 15) + +/* + * Check condition and return an error if true. Assumes that "handle" is the + * name of the hash structure pointer to be freed. + */ +#define RETURN_IF_ERROR(cond, str, ...) do { \ + if (cond) { \ + printf("ERROR line %d: " str "\n", __LINE__, ##__VA_ARGS__); \ + if (handle) rte_hash_free(handle); \ + return -1; \ + } \ +} while(0) + +#define RETURN_IF_ERROR_FBK(cond, str, ...) do { \ + if (cond) { \ + printf("ERROR line %d: " str "\n", __LINE__, ##__VA_ARGS__); \ + if (handle) rte_fbk_hash_free(handle); \ + return -1; \ + } \ +} while(0) + +/* 5-tuple key type */ +struct flow_key { + uint32_t ip_src; + uint32_t ip_dst; + uint16_t port_src; + uint16_t port_dst; + uint8_t proto; +} __attribute__((packed)); + +/* + * Hash function that always returns the same value, to easily test what + * happens when a bucket is full. + */ +static uint32_t pseudo_hash(__attribute__((unused)) const void *keys, + __attribute__((unused)) uint32_t key_len, + __attribute__((unused)) uint32_t init_val) +{ + return 3; +} + +/* + * Print out result of unit test hash operation. + */ +#if defined(UNIT_TEST_HASH_VERBOSE) +static void print_key_info(const char *msg, const struct flow_key *key, + int32_t pos) +{ + uint8_t *p = (uint8_t *)key; + unsigned i; + + printf("%s key:0x", msg); + for (i = 0; i < sizeof(struct flow_key); i++) { + printf("%02X", p[i]); + } + printf(" @ pos %d\n", pos); +} +#else +static void print_key_info(__attribute__((unused)) const char *msg, + __attribute__((unused)) const struct flow_key *key, + __attribute__((unused)) int32_t pos) +{ +} +#endif + +/* Keys used by unit test functions */ +static struct flow_key keys[5] = { { + .ip_src = IPv4(0x03, 0x02, 0x01, 0x00), + .ip_dst = IPv4(0x07, 0x06, 0x05, 0x04), + .port_src = 0x0908, + .port_dst = 0x0b0a, + .proto = 0x0c, +}, { + .ip_src = IPv4(0x13, 0x12, 0x11, 0x10), + .ip_dst = IPv4(0x17, 0x16, 0x15, 0x14), + .port_src = 0x1918, + .port_dst = 0x1b1a, + .proto = 0x1c, +}, { + .ip_src = IPv4(0x23, 0x22, 0x21, 0x20), + .ip_dst = IPv4(0x27, 0x26, 0x25, 0x24), + .port_src = 0x2928, + .port_dst = 0x2b2a, + .proto = 0x2c, +}, { + .ip_src = IPv4(0x33, 0x32, 0x31, 0x30), + .ip_dst = IPv4(0x37, 0x36, 0x35, 0x34), + .port_src = 0x3938, + .port_dst = 0x3b3a, + .proto = 0x3c, +}, { + .ip_src = IPv4(0x43, 0x42, 0x41, 0x40), + .ip_dst = IPv4(0x47, 0x46, 0x45, 0x44), + .port_src = 0x4948, + .port_dst = 0x4b4a, + .proto = 0x4c, +} }; + +/* Parameters used for hash table in unit test functions. Name set later. */ +static struct rte_hash_parameters ut_params = { + .entries = 64, + .key_len = sizeof(struct flow_key), /* 13 */ + .hash_func = rte_jhash, + .hash_func_init_val = 0, + .socket_id = 0, +}; + +#define CRC32_ITERATIONS (1U << 10) +#define CRC32_DWORDS (1U << 6) +/* + * Test if all CRC32 implementations yield the same hash value + */ +static int +test_crc32_hash_alg_equiv(void) +{ + uint32_t hash_val; + uint32_t init_val; + uint64_t data64[CRC32_DWORDS]; + unsigned i, j; + size_t data_len; + + printf("\n# CRC32 implementations equivalence test\n"); + for (i = 0; i < CRC32_ITERATIONS; i++) { + /* Randomizing data_len of data set */ + data_len = (size_t) ((rte_rand() % sizeof(data64)) + 1); + init_val = (uint32_t) rte_rand(); + + /* Fill the data set */ + for (j = 0; j < CRC32_DWORDS; j++) + data64[j] = rte_rand(); + + /* Calculate software CRC32 */ + rte_hash_crc_set_alg(CRC32_SW); + hash_val = rte_hash_crc(data64, data_len, init_val); + + /* Check against 4-byte-operand sse4.2 CRC32 if available */ + rte_hash_crc_set_alg(CRC32_SSE42); + if (hash_val != rte_hash_crc(data64, data_len, init_val)) { + printf("Failed checking CRC32_SW against CRC32_SSE42\n"); + break; + } + + /* Check against 8-byte-operand sse4.2 CRC32 if available */ + rte_hash_crc_set_alg(CRC32_SSE42_x64); + if (hash_val != rte_hash_crc(data64, data_len, init_val)) { + printf("Failed checking CRC32_SW against CRC32_SSE42_x64\n"); + break; + } + + /* Check against 8-byte-operand ARM64 CRC32 if available */ + rte_hash_crc_set_alg(CRC32_ARM64); + if (hash_val != rte_hash_crc(data64, data_len, init_val)) { + printf("Failed checking CRC32_SW against CRC32_ARM64\n"); + break; + } + } + + /* Resetting to best available algorithm */ + rte_hash_crc_set_alg(CRC32_SSE42_x64); + + if (i == CRC32_ITERATIONS) + return 0; + + printf("Failed test data (hex, %zu bytes total):\n", data_len); + for (j = 0; j < data_len; j++) + printf("%02X%c", ((uint8_t *)data64)[j], + ((j+1) % 16 == 0 || j == data_len - 1) ? '\n' : ' '); + + return -1; +} + +/* + * Test a hash function. + */ +static void run_hash_func_test(rte_hash_function f, uint32_t init_val, + uint32_t key_len) +{ + static uint8_t key[MAX_KEYSIZE]; + unsigned i; + + + for (i = 0; i < key_len; i++) + key[i] = (uint8_t) rte_rand(); + + /* just to be on the safe side */ + if (!f) + return; + + f(key, key_len, init_val); +} + +/* + * Test all hash functions. + */ +static void run_hash_func_tests(void) +{ + unsigned i, j, k; + + for (i = 0; + i < sizeof(hashtest_funcs) / sizeof(rte_hash_function); + i++) { + for (j = 0; + j < sizeof(hashtest_initvals) / sizeof(uint32_t); + j++) { + for (k = 0; + k < sizeof(hashtest_key_lens) / sizeof(uint32_t); + k++) { + run_hash_func_test(hashtest_funcs[i], + hashtest_initvals[j], + hashtest_key_lens[k]); + } + } + } +} + +/* + * Basic sequence of operations for a single key: + * - add + * - lookup (hit) + * - delete + * - lookup (miss) + */ +static int test_add_delete(void) +{ + struct rte_hash *handle; + /* test with standard add/lookup/delete functions */ + int pos0, expectedPos0; + + ut_params.name = "test1"; + handle = rte_hash_create(&ut_params); + RETURN_IF_ERROR(handle == NULL, "hash creation failed"); + + pos0 = rte_hash_add_key(handle, &keys[0]); + print_key_info("Add", &keys[0], pos0); + RETURN_IF_ERROR(pos0 < 0, "failed to add key (pos0=%d)", pos0); + expectedPos0 = pos0; + + pos0 = rte_hash_lookup(handle, &keys[0]); + print_key_info("Lkp", &keys[0], pos0); + RETURN_IF_ERROR(pos0 != expectedPos0, + "failed to find key (pos0=%d)", pos0); + + pos0 = rte_hash_del_key(handle, &keys[0]); + print_key_info("Del", &keys[0], pos0); + RETURN_IF_ERROR(pos0 != expectedPos0, + "failed to delete key (pos0=%d)", pos0); + + pos0 = rte_hash_lookup(handle, &keys[0]); + print_key_info("Lkp", &keys[0], pos0); + RETURN_IF_ERROR(pos0 != -ENOENT, + "fail: found key after deleting! (pos0=%d)", pos0); + + rte_hash_free(handle); + + /* repeat test with precomputed hash functions */ + hash_sig_t hash_value; + int pos1, expectedPos1; + + handle = rte_hash_create(&ut_params); + RETURN_IF_ERROR(handle == NULL, "hash creation failed"); + + hash_value = rte_hash_hash(handle, &keys[0]); + pos1 = rte_hash_add_key_with_hash(handle, &keys[0], hash_value); + print_key_info("Add", &keys[0], pos1); + RETURN_IF_ERROR(pos1 < 0, "failed to add key (pos1=%d)", pos1); + expectedPos1 = pos1; + + pos1 = rte_hash_lookup_with_hash(handle, &keys[0], hash_value); + print_key_info("Lkp", &keys[0], pos1); + RETURN_IF_ERROR(pos1 != expectedPos1, + "failed to find key (pos1=%d)", pos1); + + pos1 = rte_hash_del_key_with_hash(handle, &keys[0], hash_value); + print_key_info("Del", &keys[0], pos1); + RETURN_IF_ERROR(pos1 != expectedPos1, + "failed to delete key (pos1=%d)", pos1); + + pos1 = rte_hash_lookup_with_hash(handle, &keys[0], hash_value); + print_key_info("Lkp", &keys[0], pos1); + RETURN_IF_ERROR(pos1 != -ENOENT, + "fail: found key after deleting! (pos1=%d)", pos1); + + rte_hash_free(handle); + + return 0; +} + +/* + * Sequence of operations for a single key: + * - delete: miss + * - add + * - lookup: hit + * - add: update + * - lookup: hit (updated data) + * - delete: hit + * - delete: miss + * - lookup: miss + */ +static int test_add_update_delete(void) +{ + struct rte_hash *handle; + int pos0, expectedPos0; + + ut_params.name = "test2"; + handle = rte_hash_create(&ut_params); + RETURN_IF_ERROR(handle == NULL, "hash creation failed"); + + pos0 = rte_hash_del_key(handle, &keys[0]); + print_key_info("Del", &keys[0], pos0); + RETURN_IF_ERROR(pos0 != -ENOENT, + "fail: found non-existent key (pos0=%d)", pos0); + + pos0 = rte_hash_add_key(handle, &keys[0]); + print_key_info("Add", &keys[0], pos0); + RETURN_IF_ERROR(pos0 < 0, "failed to add key (pos0=%d)", pos0); + expectedPos0 = pos0; + + pos0 = rte_hash_lookup(handle, &keys[0]); + print_key_info("Lkp", &keys[0], pos0); + RETURN_IF_ERROR(pos0 != expectedPos0, + "failed to find key (pos0=%d)", pos0); + + pos0 = rte_hash_add_key(handle, &keys[0]); + print_key_info("Add", &keys[0], pos0); + RETURN_IF_ERROR(pos0 != expectedPos0, + "failed to re-add key (pos0=%d)", pos0); + + pos0 = rte_hash_lookup(handle, &keys[0]); + print_key_info("Lkp", &keys[0], pos0); + RETURN_IF_ERROR(pos0 != expectedPos0, + "failed to find key (pos0=%d)", pos0); + + pos0 = rte_hash_del_key(handle, &keys[0]); + print_key_info("Del", &keys[0], pos0); + RETURN_IF_ERROR(pos0 != expectedPos0, + "failed to delete key (pos0=%d)", pos0); + + pos0 = rte_hash_del_key(handle, &keys[0]); + print_key_info("Del", &keys[0], pos0); + RETURN_IF_ERROR(pos0 != -ENOENT, + "fail: deleted already deleted key (pos0=%d)", pos0); + + pos0 = rte_hash_lookup(handle, &keys[0]); + print_key_info("Lkp", &keys[0], pos0); + RETURN_IF_ERROR(pos0 != -ENOENT, + "fail: found key after deleting! (pos0=%d)", pos0); + + rte_hash_free(handle); + return 0; +} + +/* + * Sequence of operations for retrieving a key with its position + * + * - create table + * - add key + * - get the key with its position: hit + * - delete key + * - try to get the deleted key: miss + * + */ +static int test_hash_get_key_with_position(void) +{ + struct rte_hash *handle = NULL; + int pos, expectedPos, result; + void *key; + + ut_params.name = "hash_get_key_w_pos"; + handle = rte_hash_create(&ut_params); + RETURN_IF_ERROR(handle == NULL, "hash creation failed"); + + pos = rte_hash_add_key(handle, &keys[0]); + print_key_info("Add", &keys[0], pos); + RETURN_IF_ERROR(pos < 0, "failed to add key (pos0=%d)", pos); + expectedPos = pos; + + result = rte_hash_get_key_with_position(handle, pos, &key); + RETURN_IF_ERROR(result != 0, "error retrieving a key"); + + pos = rte_hash_del_key(handle, &keys[0]); + print_key_info("Del", &keys[0], pos); + RETURN_IF_ERROR(pos != expectedPos, + "failed to delete key (pos0=%d)", pos); + + result = rte_hash_get_key_with_position(handle, pos, &key); + RETURN_IF_ERROR(result != -ENOENT, "non valid key retrieved"); + + rte_hash_free(handle); + return 0; +} + +/* + * Sequence of operations for find existing hash table + * + * - create table + * - find existing table: hit + * - find non-existing table: miss + * + */ +static int test_hash_find_existing(void) +{ + struct rte_hash *handle = NULL, *result = NULL; + + /* Create hash table. */ + ut_params.name = "hash_find_existing"; + handle = rte_hash_create(&ut_params); + RETURN_IF_ERROR(handle == NULL, "hash creation failed"); + + /* Try to find existing hash table */ + result = rte_hash_find_existing("hash_find_existing"); + RETURN_IF_ERROR(result != handle, "could not find existing hash table"); + + /* Try to find non-existing hash table */ + result = rte_hash_find_existing("hash_find_non_existing"); + RETURN_IF_ERROR(!(result == NULL), "found table that shouldn't exist"); + + /* Cleanup. */ + rte_hash_free(handle); + + return 0; +} + +/* + * Sequence of operations for 5 keys + * - add keys + * - lookup keys: hit + * - add keys (update) + * - lookup keys: hit (updated data) + * - delete keys : hit + * - lookup keys: miss + */ +static int test_five_keys(void) +{ + struct rte_hash *handle; + const void *key_array[5] = {0}; + int pos[5]; + int expected_pos[5]; + unsigned i; + int ret; + + ut_params.name = "test3"; + handle = rte_hash_create(&ut_params); + RETURN_IF_ERROR(handle == NULL, "hash creation failed"); + + /* Add */ + for (i = 0; i < 5; i++) { + pos[i] = rte_hash_add_key(handle, &keys[i]); + print_key_info("Add", &keys[i], pos[i]); + RETURN_IF_ERROR(pos[i] < 0, + "failed to add key (pos[%u]=%d)", i, pos[i]); + expected_pos[i] = pos[i]; + } + + /* Lookup */ + for(i = 0; i < 5; i++) + key_array[i] = &keys[i]; + + ret = rte_hash_lookup_bulk(handle, &key_array[0], 5, (int32_t *)pos); + if(ret == 0) + for(i = 0; i < 5; i++) { + print_key_info("Lkp", key_array[i], pos[i]); + RETURN_IF_ERROR(pos[i] != expected_pos[i], + "failed to find key (pos[%u]=%d)", i, pos[i]); + } + + /* Add - update */ + for (i = 0; i < 5; i++) { + pos[i] = rte_hash_add_key(handle, &keys[i]); + print_key_info("Add", &keys[i], pos[i]); + RETURN_IF_ERROR(pos[i] != expected_pos[i], + "failed to add key (pos[%u]=%d)", i, pos[i]); + } + + /* Lookup */ + for (i = 0; i < 5; i++) { + pos[i] = rte_hash_lookup(handle, &keys[i]); + print_key_info("Lkp", &keys[i], pos[i]); + RETURN_IF_ERROR(pos[i] != expected_pos[i], + "failed to find key (pos[%u]=%d)", i, pos[i]); + } + + /* Delete */ + for (i = 0; i < 5; i++) { + pos[i] = rte_hash_del_key(handle, &keys[i]); + print_key_info("Del", &keys[i], pos[i]); + RETURN_IF_ERROR(pos[i] != expected_pos[i], + "failed to delete key (pos[%u]=%d)", i, pos[i]); + } + + /* Lookup */ + for (i = 0; i < 5; i++) { + pos[i] = rte_hash_lookup(handle, &keys[i]); + print_key_info("Lkp", &keys[i], pos[i]); + RETURN_IF_ERROR(pos[i] != -ENOENT, + "found non-existent key (pos[%u]=%d)", i, pos[i]); + } + + /* Lookup multi */ + ret = rte_hash_lookup_bulk(handle, &key_array[0], 5, (int32_t *)pos); + if (ret == 0) + for (i = 0; i < 5; i++) { + print_key_info("Lkp", key_array[i], pos[i]); + RETURN_IF_ERROR(pos[i] != -ENOENT, + "found not-existent key (pos[%u]=%d)", i, pos[i]); + } + + rte_hash_free(handle); + + return 0; +} + +/* + * Add keys to the same bucket until bucket full. + * - add 5 keys to the same bucket (hash created with 4 keys per bucket): + * first 4 successful, 5th successful, pushing existing item in bucket + * - lookup the 5 keys: 5 hits + * - add the 5 keys again: 5 OK + * - lookup the 5 keys: 5 hits (updated data) + * - delete the 5 keys: 5 OK + * - lookup the 5 keys: 5 misses + */ +static int test_full_bucket(void) +{ + struct rte_hash_parameters params_pseudo_hash = { + .name = "test4", + .entries = 64, + .key_len = sizeof(struct flow_key), /* 13 */ + .hash_func = pseudo_hash, + .hash_func_init_val = 0, + .socket_id = 0, + }; + struct rte_hash *handle; + int pos[5]; + int expected_pos[5]; + unsigned i; + + handle = rte_hash_create(¶ms_pseudo_hash); + RETURN_IF_ERROR(handle == NULL, "hash creation failed"); + + /* Fill bucket */ + for (i = 0; i < 4; i++) { + pos[i] = rte_hash_add_key(handle, &keys[i]); + print_key_info("Add", &keys[i], pos[i]); + RETURN_IF_ERROR(pos[i] < 0, + "failed to add key (pos[%u]=%d)", i, pos[i]); + expected_pos[i] = pos[i]; + } + /* + * This should work and will push one of the items + * in the bucket because it is full + */ + pos[4] = rte_hash_add_key(handle, &keys[4]); + print_key_info("Add", &keys[4], pos[4]); + RETURN_IF_ERROR(pos[4] < 0, + "failed to add key (pos[4]=%d)", pos[4]); + expected_pos[4] = pos[4]; + + /* Lookup */ + for (i = 0; i < 5; i++) { + pos[i] = rte_hash_lookup(handle, &keys[i]); + print_key_info("Lkp", &keys[i], pos[i]); + RETURN_IF_ERROR(pos[i] != expected_pos[i], + "failed to find key (pos[%u]=%d)", i, pos[i]); + } + + /* Add - update */ + for (i = 0; i < 5; i++) { + pos[i] = rte_hash_add_key(handle, &keys[i]); + print_key_info("Add", &keys[i], pos[i]); + RETURN_IF_ERROR(pos[i] != expected_pos[i], + "failed to add key (pos[%u]=%d)", i, pos[i]); + } + + /* Lookup */ + for (i = 0; i < 5; i++) { + pos[i] = rte_hash_lookup(handle, &keys[i]); + print_key_info("Lkp", &keys[i], pos[i]); + RETURN_IF_ERROR(pos[i] != expected_pos[i], + "failed to find key (pos[%u]=%d)", i, pos[i]); + } + + /* Delete 1 key, check other keys are still found */ + pos[1] = rte_hash_del_key(handle, &keys[1]); + print_key_info("Del", &keys[1], pos[1]); + RETURN_IF_ERROR(pos[1] != expected_pos[1], + "failed to delete key (pos[1]=%d)", pos[1]); + pos[3] = rte_hash_lookup(handle, &keys[3]); + print_key_info("Lkp", &keys[3], pos[3]); + RETURN_IF_ERROR(pos[3] != expected_pos[3], + "failed lookup after deleting key from same bucket " + "(pos[3]=%d)", pos[3]); + + /* Go back to previous state */ + pos[1] = rte_hash_add_key(handle, &keys[1]); + print_key_info("Add", &keys[1], pos[1]); + expected_pos[1] = pos[1]; + RETURN_IF_ERROR(pos[1] < 0, "failed to add key (pos[1]=%d)", pos[1]); + + /* Delete */ + for (i = 0; i < 5; i++) { + pos[i] = rte_hash_del_key(handle, &keys[i]); + print_key_info("Del", &keys[i], pos[i]); + RETURN_IF_ERROR(pos[i] != expected_pos[i], + "failed to delete key (pos[%u]=%d)", i, pos[i]); + } + + /* Lookup */ + for (i = 0; i < 5; i++) { + pos[i] = rte_hash_lookup(handle, &keys[i]); + print_key_info("Lkp", &keys[i], pos[i]); + RETURN_IF_ERROR(pos[i] != -ENOENT, + "fail: found non-existent key (pos[%u]=%d)", i, pos[i]); + } + + rte_hash_free(handle); + + /* Cover the NULL case. */ + rte_hash_free(0); + return 0; +} + +/******************************************************************************/ +static int +fbk_hash_unit_test(void) +{ + struct rte_fbk_hash_params params = { + .name = "fbk_hash_test", + .entries = LOCAL_FBK_HASH_ENTRIES_MAX, + .entries_per_bucket = 4, + .socket_id = 0, + }; + + struct rte_fbk_hash_params invalid_params_1 = { + .name = "invalid_1", + .entries = LOCAL_FBK_HASH_ENTRIES_MAX + 1, /* Not power of 2 */ + .entries_per_bucket = 4, + .socket_id = 0, + }; + + struct rte_fbk_hash_params invalid_params_2 = { + .name = "invalid_2", + .entries = 4, + .entries_per_bucket = 3, /* Not power of 2 */ + .socket_id = 0, + }; + + struct rte_fbk_hash_params invalid_params_3 = { + .name = "invalid_3", + .entries = 0, /* Entries is 0 */ + .entries_per_bucket = 4, + .socket_id = 0, + }; + + struct rte_fbk_hash_params invalid_params_4 = { + .name = "invalid_4", + .entries = LOCAL_FBK_HASH_ENTRIES_MAX, + .entries_per_bucket = 0, /* Entries per bucket is 0 */ + .socket_id = 0, + }; + + struct rte_fbk_hash_params invalid_params_5 = { + .name = "invalid_5", + .entries = 4, + .entries_per_bucket = 8, /* Entries per bucket > entries */ + .socket_id = 0, + }; + + struct rte_fbk_hash_params invalid_params_6 = { + .name = "invalid_6", + .entries = RTE_FBK_HASH_ENTRIES_MAX * 2, /* Entries > max allowed */ + .entries_per_bucket = 4, + .socket_id = 0, + }; + + struct rte_fbk_hash_params invalid_params_7 = { + .name = "invalid_7", + .entries = RTE_FBK_HASH_ENTRIES_MAX, + .entries_per_bucket = RTE_FBK_HASH_ENTRIES_PER_BUCKET_MAX * 2, /* Entries > max allowed */ + .socket_id = 0, + }; + + struct rte_fbk_hash_params invalid_params_8 = { + .name = "invalid_7", + .entries = RTE_FBK_HASH_ENTRIES_MAX, + .entries_per_bucket = 4, + .socket_id = RTE_MAX_NUMA_NODES + 1, /* invalid socket */ + }; + + /* try to create two hashes with identical names + * in this case, trying to create a second one will not + * fail but will simply return pointer to the existing + * hash with that name. sort of like a "find hash by name" :-) + */ + struct rte_fbk_hash_params invalid_params_same_name_1 = { + .name = "same_name", /* hash with identical name */ + .entries = 4, + .entries_per_bucket = 2, + .socket_id = 0, + }; + + /* trying to create this hash should return a pointer to an existing hash */ + struct rte_fbk_hash_params invalid_params_same_name_2 = { + .name = "same_name", /* hash with identical name */ + .entries = RTE_FBK_HASH_ENTRIES_MAX, + .entries_per_bucket = 4, + .socket_id = 0, + }; + + /* this is a sanity check for "same name" test + * creating this hash will check if we are actually able to create + * multiple hashes with different names (instead of having just one). + */ + struct rte_fbk_hash_params different_name = { + .name = "different_name", /* different name */ + .entries = LOCAL_FBK_HASH_ENTRIES_MAX, + .entries_per_bucket = 4, + .socket_id = 0, + }; + + struct rte_fbk_hash_params params_jhash = { + .name = "valid", + .entries = LOCAL_FBK_HASH_ENTRIES_MAX, + .entries_per_bucket = 4, + .socket_id = 0, + .hash_func = rte_jhash_1word, /* Tests for different hash_func */ + .init_val = RTE_FBK_HASH_INIT_VAL_DEFAULT, + }; + + struct rte_fbk_hash_params params_nohash = { + .name = "valid nohash", + .entries = LOCAL_FBK_HASH_ENTRIES_MAX, + .entries_per_bucket = 4, + .socket_id = 0, + .hash_func = NULL, /* Tests for null hash_func */ + .init_val = RTE_FBK_HASH_INIT_VAL_DEFAULT, + }; + + struct rte_fbk_hash_table *handle, *tmp; + uint32_t keys[5] = + {0xc6e18639, 0xe67c201c, 0xd4c8cffd, 0x44728691, 0xd5430fa9}; + uint16_t vals[5] = {28108, 5699, 38490, 2166, 61571}; + int status; + unsigned i; + double used_entries; + + /* Try creating hashes with invalid parameters */ + printf("# Testing hash creation with invalid parameters " + "- expect error msgs\n"); + handle = rte_fbk_hash_create(&invalid_params_1); + RETURN_IF_ERROR_FBK(handle != NULL, "fbk hash creation should have failed"); + + handle = rte_fbk_hash_create(&invalid_params_2); + RETURN_IF_ERROR_FBK(handle != NULL, "fbk hash creation should have failed"); + + handle = rte_fbk_hash_create(&invalid_params_3); + RETURN_IF_ERROR_FBK(handle != NULL, "fbk hash creation should have failed"); + + handle = rte_fbk_hash_create(&invalid_params_4); + RETURN_IF_ERROR_FBK(handle != NULL, "fbk hash creation should have failed"); + + handle = rte_fbk_hash_create(&invalid_params_5); + RETURN_IF_ERROR_FBK(handle != NULL, "fbk hash creation should have failed"); + + handle = rte_fbk_hash_create(&invalid_params_6); + RETURN_IF_ERROR_FBK(handle != NULL, "fbk hash creation should have failed"); + + handle = rte_fbk_hash_create(&invalid_params_7); + RETURN_IF_ERROR_FBK(handle != NULL, "fbk hash creation should have failed"); + + handle = rte_fbk_hash_create(&invalid_params_8); + RETURN_IF_ERROR_FBK(handle != NULL, "fbk hash creation should have failed"); + + handle = rte_fbk_hash_create(&invalid_params_same_name_1); + RETURN_IF_ERROR_FBK(handle == NULL, "fbk hash creation should have succeeded"); + + tmp = rte_fbk_hash_create(&invalid_params_same_name_2); + if (tmp != NULL) + rte_fbk_hash_free(tmp); + RETURN_IF_ERROR_FBK(tmp != NULL, "fbk hash creation should have failed"); + + /* we are not freeing handle here because we need a hash list + * to be not empty for the next test */ + + /* create a hash in non-empty list - good for coverage */ + tmp = rte_fbk_hash_create(&different_name); + RETURN_IF_ERROR_FBK(tmp == NULL, "fbk hash creation should have succeeded"); + + /* free both hashes */ + rte_fbk_hash_free(handle); + rte_fbk_hash_free(tmp); + + /* Create empty jhash hash. */ + handle = rte_fbk_hash_create(¶ms_jhash); + RETURN_IF_ERROR_FBK(handle == NULL, "fbk jhash hash creation failed"); + + /* Cleanup. */ + rte_fbk_hash_free(handle); + + /* Create empty jhash hash. */ + handle = rte_fbk_hash_create(¶ms_nohash); + RETURN_IF_ERROR_FBK(handle == NULL, "fbk nohash hash creation failed"); + + /* Cleanup. */ + rte_fbk_hash_free(handle); + + /* Create empty hash. */ + handle = rte_fbk_hash_create(¶ms); + RETURN_IF_ERROR_FBK(handle == NULL, "fbk hash creation failed"); + + used_entries = rte_fbk_hash_get_load_factor(handle) * LOCAL_FBK_HASH_ENTRIES_MAX; + RETURN_IF_ERROR_FBK((unsigned)used_entries != 0, \ + "load factor right after creation is not zero but it should be"); + /* Add keys. */ + for (i = 0; i < 5; i++) { + status = rte_fbk_hash_add_key(handle, keys[i], vals[i]); + RETURN_IF_ERROR_FBK(status != 0, "fbk hash add failed"); + } + + used_entries = rte_fbk_hash_get_load_factor(handle) * LOCAL_FBK_HASH_ENTRIES_MAX; + RETURN_IF_ERROR_FBK((unsigned)used_entries != (unsigned)((((double)5)/LOCAL_FBK_HASH_ENTRIES_MAX)*LOCAL_FBK_HASH_ENTRIES_MAX), \ + "load factor now is not as expected"); + /* Find value of added keys. */ + for (i = 0; i < 5; i++) { + status = rte_fbk_hash_lookup(handle, keys[i]); + RETURN_IF_ERROR_FBK(status != vals[i], + "fbk hash lookup failed"); + } + + /* Change value of added keys. */ + for (i = 0; i < 5; i++) { + status = rte_fbk_hash_add_key(handle, keys[i], vals[4 - i]); + RETURN_IF_ERROR_FBK(status != 0, "fbk hash update failed"); + } + + /* Find new values. */ + for (i = 0; i < 5; i++) { + status = rte_fbk_hash_lookup(handle, keys[i]); + RETURN_IF_ERROR_FBK(status != vals[4-i], + "fbk hash lookup failed"); + } + + /* Delete keys individually. */ + for (i = 0; i < 5; i++) { + status = rte_fbk_hash_delete_key(handle, keys[i]); + RETURN_IF_ERROR_FBK(status != 0, "fbk hash delete failed"); + } + + used_entries = rte_fbk_hash_get_load_factor(handle) * LOCAL_FBK_HASH_ENTRIES_MAX; + RETURN_IF_ERROR_FBK((unsigned)used_entries != 0, \ + "load factor right after deletion is not zero but it should be"); + /* Lookup should now fail. */ + for (i = 0; i < 5; i++) { + status = rte_fbk_hash_lookup(handle, keys[i]); + RETURN_IF_ERROR_FBK(status == 0, + "fbk hash lookup should have failed"); + } + + /* Add keys again. */ + for (i = 0; i < 5; i++) { + status = rte_fbk_hash_add_key(handle, keys[i], vals[i]); + RETURN_IF_ERROR_FBK(status != 0, "fbk hash add failed"); + } + + /* Make sure they were added. */ + for (i = 0; i < 5; i++) { + status = rte_fbk_hash_lookup(handle, keys[i]); + RETURN_IF_ERROR_FBK(status != vals[i], + "fbk hash lookup failed"); + } + + /* Clear all entries. */ + rte_fbk_hash_clear_all(handle); + + /* Lookup should fail. */ + for (i = 0; i < 5; i++) { + status = rte_fbk_hash_lookup(handle, keys[i]); + RETURN_IF_ERROR_FBK(status == 0, + "fbk hash lookup should have failed"); + } + + /* coverage */ + + /* fill up the hash_table */ + for (i = 0; i < RTE_FBK_HASH_ENTRIES_MAX + 1; i++) + rte_fbk_hash_add_key(handle, i, (uint16_t) i); + + /* Find non-existent key in a full hashtable */ + status = rte_fbk_hash_lookup(handle, RTE_FBK_HASH_ENTRIES_MAX + 1); + RETURN_IF_ERROR_FBK(status != -ENOENT, + "fbk hash lookup succeeded"); + + /* Delete non-existent key in a full hashtable */ + status = rte_fbk_hash_delete_key(handle, RTE_FBK_HASH_ENTRIES_MAX + 1); + RETURN_IF_ERROR_FBK(status != -ENOENT, + "fbk hash delete succeeded"); + + /* Delete one key from a full hashtable */ + status = rte_fbk_hash_delete_key(handle, 1); + RETURN_IF_ERROR_FBK(status != 0, + "fbk hash delete failed"); + + /* Clear all entries. */ + rte_fbk_hash_clear_all(handle); + + /* Cleanup. */ + rte_fbk_hash_free(handle); + + /* Cover the NULL case. */ + rte_fbk_hash_free(0); + + return 0; +} + +/* + * Sequence of operations for find existing fbk hash table + * + * - create table + * - find existing table: hit + * - find non-existing table: miss + * + */ +static int test_fbk_hash_find_existing(void) +{ + struct rte_fbk_hash_params params = { + .name = "fbk_hash_find_existing", + .entries = LOCAL_FBK_HASH_ENTRIES_MAX, + .entries_per_bucket = 4, + .socket_id = 0, + }; + struct rte_fbk_hash_table *handle = NULL, *result = NULL; + + /* Create hash table. */ + handle = rte_fbk_hash_create(¶ms); + RETURN_IF_ERROR_FBK(handle == NULL, "fbk hash creation failed"); + + /* Try to find existing fbk hash table */ + result = rte_fbk_hash_find_existing("fbk_hash_find_existing"); + RETURN_IF_ERROR_FBK(result != handle, "could not find existing fbk hash table"); + + /* Try to find non-existing fbk hash table */ + result = rte_fbk_hash_find_existing("fbk_hash_find_non_existing"); + RETURN_IF_ERROR_FBK(!(result == NULL), "found fbk table that shouldn't exist"); + + /* Cleanup. */ + rte_fbk_hash_free(handle); + + return 0; +} + +#define BUCKET_ENTRIES 4 +/* + * Do tests for hash creation with bad parameters. + */ +static int test_hash_creation_with_bad_parameters(void) +{ + struct rte_hash *handle, *tmp; + struct rte_hash_parameters params; + + handle = rte_hash_create(NULL); + if (handle != NULL) { + rte_hash_free(handle); + printf("Impossible creating hash successfully without any parameter\n"); + return -1; + } + + memcpy(¶ms, &ut_params, sizeof(params)); + params.name = "creation_with_bad_parameters_0"; + params.entries = RTE_HASH_ENTRIES_MAX + 1; + handle = rte_hash_create(¶ms); + if (handle != NULL) { + rte_hash_free(handle); + printf("Impossible creating hash successfully with entries in parameter exceeded\n"); + return -1; + } + + memcpy(¶ms, &ut_params, sizeof(params)); + params.name = "creation_with_bad_parameters_2"; + params.entries = BUCKET_ENTRIES - 1; + handle = rte_hash_create(¶ms); + if (handle != NULL) { + rte_hash_free(handle); + printf("Impossible creating hash successfully if entries less than bucket_entries in parameter\n"); + return -1; + } + + memcpy(¶ms, &ut_params, sizeof(params)); + params.name = "creation_with_bad_parameters_3"; + params.key_len = 0; + handle = rte_hash_create(¶ms); + if (handle != NULL) { + rte_hash_free(handle); + printf("Impossible creating hash successfully if key_len in parameter is zero\n"); + return -1; + } + + memcpy(¶ms, &ut_params, sizeof(params)); + params.name = "creation_with_bad_parameters_4"; + params.socket_id = RTE_MAX_NUMA_NODES + 1; + handle = rte_hash_create(¶ms); + if (handle != NULL) { + rte_hash_free(handle); + printf("Impossible creating hash successfully with invalid socket\n"); + return -1; + } + + /* test with same name should fail */ + memcpy(¶ms, &ut_params, sizeof(params)); + params.name = "same_name"; + handle = rte_hash_create(¶ms); + if (handle == NULL) { + printf("Cannot create first hash table with 'same_name'\n"); + return -1; + } + tmp = rte_hash_create(¶ms); + if (tmp != NULL) { + printf("Creation of hash table with same name should fail\n"); + rte_hash_free(handle); + rte_hash_free(tmp); + return -1; + } + rte_hash_free(handle); + + printf("# Test successful. No more errors expected\n"); + + return 0; +} + +/* + * Do tests for hash creation with parameters that look incorrect + * but are actually valid. + */ +static int +test_hash_creation_with_good_parameters(void) +{ + struct rte_hash *handle; + struct rte_hash_parameters params; + + /* create with null hash function - should choose DEFAULT_HASH_FUNC */ + memcpy(¶ms, &ut_params, sizeof(params)); + params.name = "name"; + params.hash_func = NULL; + handle = rte_hash_create(¶ms); + if (handle == NULL) { + printf("Creating hash with null hash_func failed\n"); + return -1; + } + + rte_hash_free(handle); + + return 0; +} + +#define ITERATIONS 3 +/* + * Test to see the average table utilization (entries added/max entries) + * before hitting a random entry that cannot be added + */ +static int test_average_table_utilization(void) +{ + struct rte_hash *handle; + uint8_t simple_key[MAX_KEYSIZE]; + unsigned i, j; + unsigned added_keys, average_keys_added = 0; + int ret; + + printf("\n# Running test to determine average utilization" + "\n before adding elements begins to fail\n"); + printf("Measuring performance, please wait"); + fflush(stdout); + ut_params.entries = 1 << 16; + ut_params.name = "test_average_utilization"; + ut_params.hash_func = rte_jhash; + handle = rte_hash_create(&ut_params); + RETURN_IF_ERROR(handle == NULL, "hash creation failed"); + + for (j = 0; j < ITERATIONS; j++) { + ret = 0; + /* Add random entries until key cannot be added */ + for (added_keys = 0; ret >= 0; added_keys++) { + for (i = 0; i < ut_params.key_len; i++) + simple_key[i] = rte_rand() % 255; + ret = rte_hash_add_key(handle, simple_key); + } + if (ret != -ENOSPC) { + printf("Unexpected error when adding keys\n"); + rte_hash_free(handle); + return -1; + } + + average_keys_added += added_keys; + + /* Reset the table */ + rte_hash_reset(handle); + + /* Print a dot to show progress on operations */ + printf("."); + fflush(stdout); + } + + average_keys_added /= ITERATIONS; + + printf("\nAverage table utilization = %.2f%% (%u/%u)\n", + ((double) average_keys_added / ut_params.entries * 100), + average_keys_added, ut_params.entries); + rte_hash_free(handle); + + return 0; +} + +#define NUM_ENTRIES 256 +static int test_hash_iteration(void) +{ + struct rte_hash *handle; + unsigned i; + uint8_t keys[NUM_ENTRIES][MAX_KEYSIZE]; + const void *next_key; + void *next_data; + void *data[NUM_ENTRIES]; + unsigned added_keys; + uint32_t iter = 0; + int ret = 0; + + ut_params.entries = NUM_ENTRIES; + ut_params.name = "test_hash_iteration"; + ut_params.hash_func = rte_jhash; + ut_params.key_len = 16; + handle = rte_hash_create(&ut_params); + RETURN_IF_ERROR(handle == NULL, "hash creation failed"); + + /* Add random entries until key cannot be added */ + for (added_keys = 0; added_keys < NUM_ENTRIES; added_keys++) { + data[added_keys] = (void *) ((uintptr_t) rte_rand()); + for (i = 0; i < ut_params.key_len; i++) + keys[added_keys][i] = rte_rand() % 255; + ret = rte_hash_add_key_data(handle, keys[added_keys], data[added_keys]); + if (ret < 0) + break; + } + + /* Iterate through the hash table */ + while (rte_hash_iterate(handle, &next_key, &next_data, &iter) >= 0) { + /* Search for the key in the list of keys added */ + for (i = 0; i < NUM_ENTRIES; i++) { + if (memcmp(next_key, keys[i], ut_params.key_len) == 0) { + if (next_data != data[i]) { + printf("Data found in the hash table is" + "not the data added with the key\n"); + goto err; + } + added_keys--; + break; + } + } + if (i == NUM_ENTRIES) { + printf("Key found in the hash table was not added\n"); + goto err; + } + } + + /* Check if all keys have been iterated */ + if (added_keys != 0) { + printf("There were still %u keys to iterate\n", added_keys); + goto err; + } + + rte_hash_free(handle); + return 0; + +err: + rte_hash_free(handle); + return -1; +} + +static uint8_t key[16] = {0x00, 0x01, 0x02, 0x03, + 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, + 0x0c, 0x0d, 0x0e, 0x0f}; +static struct rte_hash_parameters hash_params_ex = { + .name = NULL, + .entries = 64, + .key_len = 0, + .hash_func = NULL, + .hash_func_init_val = 0, + .socket_id = 0, +}; + +/* + * add/delete key with jhash2 + */ +static int +test_hash_add_delete_jhash2(void) +{ + int ret = -1; + struct rte_hash *handle; + int32_t pos1, pos2; + + hash_params_ex.name = "hash_test_jhash2"; + hash_params_ex.key_len = 4; + hash_params_ex.hash_func = (rte_hash_function)rte_jhash_32b; + + handle = rte_hash_create(&hash_params_ex); + if (handle == NULL) { + printf("test_hash_add_delete_jhash2 fail to create hash\n"); + goto fail_jhash2; + } + pos1 = rte_hash_add_key(handle, (void *)&key[0]); + if (pos1 < 0) { + printf("test_hash_add_delete_jhash2 fail to add hash key\n"); + goto fail_jhash2; + } + + pos2 = rte_hash_del_key(handle, (void *)&key[0]); + if (pos2 < 0 || pos1 != pos2) { + printf("test_hash_add_delete_jhash2 delete different key from being added\n"); + goto fail_jhash2; + } + ret = 0; + +fail_jhash2: + if (handle != NULL) + rte_hash_free(handle); + + return ret; +} + +/* + * add/delete (2) key with jhash2 + */ +static int +test_hash_add_delete_2_jhash2(void) +{ + int ret = -1; + struct rte_hash *handle; + int32_t pos1, pos2; + + hash_params_ex.name = "hash_test_2_jhash2"; + hash_params_ex.key_len = 8; + hash_params_ex.hash_func = (rte_hash_function)rte_jhash_32b; + + handle = rte_hash_create(&hash_params_ex); + if (handle == NULL) + goto fail_2_jhash2; + + pos1 = rte_hash_add_key(handle, (void *)&key[0]); + if (pos1 < 0) + goto fail_2_jhash2; + + pos2 = rte_hash_del_key(handle, (void *)&key[0]); + if (pos2 < 0 || pos1 != pos2) + goto fail_2_jhash2; + + ret = 0; + +fail_2_jhash2: + if (handle != NULL) + rte_hash_free(handle); + + return ret; +} + +static uint32_t +test_hash_jhash_1word(const void *key, uint32_t length, uint32_t initval) +{ + const uint32_t *k = key; + + RTE_SET_USED(length); + + return rte_jhash_1word(k[0], initval); +} + +static uint32_t +test_hash_jhash_2word(const void *key, uint32_t length, uint32_t initval) +{ + const uint32_t *k = key; + + RTE_SET_USED(length); + + return rte_jhash_2words(k[0], k[1], initval); +} + +static uint32_t +test_hash_jhash_3word(const void *key, uint32_t length, uint32_t initval) +{ + const uint32_t *k = key; + + RTE_SET_USED(length); + + return rte_jhash_3words(k[0], k[1], k[2], initval); +} + +/* + * add/delete key with jhash 1word + */ +static int +test_hash_add_delete_jhash_1word(void) +{ + int ret = -1; + struct rte_hash *handle; + int32_t pos1, pos2; + + hash_params_ex.name = "hash_test_jhash_1word"; + hash_params_ex.key_len = 4; + hash_params_ex.hash_func = test_hash_jhash_1word; + + handle = rte_hash_create(&hash_params_ex); + if (handle == NULL) + goto fail_jhash_1word; + + pos1 = rte_hash_add_key(handle, (void *)&key[0]); + if (pos1 < 0) + goto fail_jhash_1word; + + pos2 = rte_hash_del_key(handle, (void *)&key[0]); + if (pos2 < 0 || pos1 != pos2) + goto fail_jhash_1word; + + ret = 0; + +fail_jhash_1word: + if (handle != NULL) + rte_hash_free(handle); + + return ret; +} + +/* + * add/delete key with jhash 2word + */ +static int +test_hash_add_delete_jhash_2word(void) +{ + int ret = -1; + struct rte_hash *handle; + int32_t pos1, pos2; + + hash_params_ex.name = "hash_test_jhash_2word"; + hash_params_ex.key_len = 8; + hash_params_ex.hash_func = test_hash_jhash_2word; + + handle = rte_hash_create(&hash_params_ex); + if (handle == NULL) + goto fail_jhash_2word; + + pos1 = rte_hash_add_key(handle, (void *)&key[0]); + if (pos1 < 0) + goto fail_jhash_2word; + + pos2 = rte_hash_del_key(handle, (void *)&key[0]); + if (pos2 < 0 || pos1 != pos2) + goto fail_jhash_2word; + + ret = 0; + +fail_jhash_2word: + if (handle != NULL) + rte_hash_free(handle); + + return ret; +} + +/* + * add/delete key with jhash 3word + */ +static int +test_hash_add_delete_jhash_3word(void) +{ + int ret = -1; + struct rte_hash *handle; + int32_t pos1, pos2; + + hash_params_ex.name = "hash_test_jhash_3word"; + hash_params_ex.key_len = 12; + hash_params_ex.hash_func = test_hash_jhash_3word; + + handle = rte_hash_create(&hash_params_ex); + if (handle == NULL) + goto fail_jhash_3word; + + pos1 = rte_hash_add_key(handle, (void *)&key[0]); + if (pos1 < 0) + goto fail_jhash_3word; + + pos2 = rte_hash_del_key(handle, (void *)&key[0]); + if (pos2 < 0 || pos1 != pos2) + goto fail_jhash_3word; + + ret = 0; + +fail_jhash_3word: + if (handle != NULL) + rte_hash_free(handle); + + return ret; +} + +/* + * Do all unit and performance tests. + */ +static int +test_hash(void) +{ + if (test_add_delete() < 0) + return -1; + if (test_hash_add_delete_jhash2() < 0) + return -1; + if (test_hash_add_delete_2_jhash2() < 0) + return -1; + if (test_hash_add_delete_jhash_1word() < 0) + return -1; + if (test_hash_add_delete_jhash_2word() < 0) + return -1; + if (test_hash_add_delete_jhash_3word() < 0) + return -1; + if (test_hash_get_key_with_position() < 0) + return -1; + if (test_hash_find_existing() < 0) + return -1; + if (test_add_update_delete() < 0) + return -1; + if (test_five_keys() < 0) + return -1; + if (test_full_bucket() < 0) + return -1; + + if (test_fbk_hash_find_existing() < 0) + return -1; + if (fbk_hash_unit_test() < 0) + return -1; + if (test_hash_creation_with_bad_parameters() < 0) + return -1; + if (test_hash_creation_with_good_parameters() < 0) + return -1; + if (test_average_table_utilization() < 0) + return -1; + if (test_hash_iteration() < 0) + return -1; + + run_hash_func_tests(); + + if (test_crc32_hash_alg_equiv() < 0) + return -1; + + return 0; +} + +REGISTER_TEST_COMMAND(hash_autotest, test_hash); diff --git a/dpdk/test/test/test_hash_functions.c b/dpdk/test/test/test_hash_functions.c new file mode 100644 index 00000000..9652b04d --- /dev/null +++ b/dpdk/test/test/test_hash_functions.c @@ -0,0 +1,322 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <stdint.h> +#include <string.h> +#include <stdlib.h> +#include <stdarg.h> +#include <errno.h> +#include <sys/queue.h> + +#include <rte_cycles.h> +#include <rte_random.h> +#include <rte_hash.h> +#include <rte_jhash.h> +#include <rte_hash_crc.h> + +#include "test.h" + +/* + * Hash values calculated for key sizes from array "hashtest_key_lens" + * and for initial values from array "hashtest_initvals. + * Each key will be formed by increasing each byte by 1: + * e.g.: key size = 4, key = 0x03020100 + * key size = 8, key = 0x0706050403020100 + */ +static uint32_t hash_values_jhash[2][12] = {{ + 0x8ba9414b, 0xdf0d39c9, + 0xe4cf1d42, 0xd4ccb93c, 0x5e84eafc, 0x21362cfe, + 0x2f4775ab, 0x9ff036cc, 0xeca51474, 0xbc9d6816, + 0x12926a31, 0x1c9fa888 +}, +{ + 0x5c62c303, 0x1b8cf784, + 0x8270ac65, 0x05fa6668, 0x762df861, 0xda088f2f, + 0x59614cd4, 0x7a94f690, 0xdc1e4993, 0x30825494, + 0x91d0e462, 0x768087fc +} +}; +static uint32_t hash_values_crc[2][12] = {{ + 0x00000000, 0xf26b8303, + 0x91545164, 0x06040eb1, 0x9bb99201, 0xcc4c4fe4, + 0x14a90993, 0xf8a5dd8c, 0xcaa1ad0b, 0x7ac1e03e, + 0x43f44466, 0x4a11475e +}, +{ + 0xbdfd3980, 0x70204542, + 0x98cd4c70, 0xd52c702f, 0x41fc0e1c, 0x3905f65c, + 0x94bff47f, 0x1bab102d, 0xf4a2c645, 0xbf441539, + 0x789c104f, 0x53028d3e +} +}; + +/******************************************************************************* + * Hash function performance test configuration section. Each performance test + * will be performed HASHTEST_ITERATIONS times. + * + * The three arrays below control what tests are performed. Every combination + * from the array entries is tested. + */ +#define HASHTEST_ITERATIONS 1000000 +#define MAX_KEYSIZE 64 +static rte_hash_function hashtest_funcs[] = {rte_jhash, rte_hash_crc}; +static uint32_t hashtest_initvals[] = {0, 0xdeadbeef}; +static uint32_t hashtest_key_lens[] = { + 1, 2, /* Unusual key sizes */ + 4, 8, 16, 32, 48, 64, /* standard key sizes */ + 9, /* IPv4 SRC + DST + protocol, unpadded */ + 13, /* IPv4 5-tuple, unpadded */ + 37, /* IPv6 5-tuple, unpadded */ + 40 /* IPv6 5-tuple, padded to 8-byte boundary */ +}; +/******************************************************************************/ + +/* + * To help print out name of hash functions. + */ +static const char * +get_hash_name(rte_hash_function f) +{ + if (f == rte_jhash) + return "jhash"; + + if (f == rte_hash_crc) + return "rte_hash_crc"; + + return "UnknownHash"; +} + +/* + * Test a hash function. + */ +static void +run_hash_func_perf_test(uint32_t key_len, uint32_t init_val, + rte_hash_function f) +{ + static uint8_t key[HASHTEST_ITERATIONS][MAX_KEYSIZE]; + uint64_t ticks, start, end; + unsigned i, j; + + for (i = 0; i < HASHTEST_ITERATIONS; i++) { + for (j = 0; j < key_len; j++) + key[i][j] = (uint8_t) rte_rand(); + } + + start = rte_rdtsc(); + for (i = 0; i < HASHTEST_ITERATIONS; i++) + f(key[i], key_len, init_val); + end = rte_rdtsc(); + ticks = end - start; + + printf("%-12s, %-18u, %-13u, %.02f\n", get_hash_name(f), (unsigned) key_len, + (unsigned) init_val, (double)ticks / HASHTEST_ITERATIONS); +} + +/* + * Test all hash functions. + */ +static void +run_hash_func_perf_tests(void) +{ + unsigned i, j, k; + + printf(" *** Hash function performance test results ***\n"); + printf(" Number of iterations for each test = %d\n", + HASHTEST_ITERATIONS); + printf("Hash Func. , Key Length (bytes), Initial value, Ticks/Op.\n"); + + for (i = 0; i < RTE_DIM(hashtest_initvals); i++) { + for (j = 0; j < RTE_DIM(hashtest_key_lens); j++) { + for (k = 0; k < RTE_DIM(hashtest_funcs); k++) { + run_hash_func_perf_test(hashtest_key_lens[j], + hashtest_initvals[i], + hashtest_funcs[k]); + } + } + } +} + +/* + * Verify that hash functions return what they are expected to return + * (using precalculated values stored above) + */ +static int +verify_precalculated_hash_func_tests(void) +{ + unsigned i, j; + uint8_t key[64]; + uint32_t hash; + + for (i = 0; i < 64; i++) + key[i] = (uint8_t) i; + + for (i = 0; i < sizeof(hashtest_key_lens) / sizeof(uint32_t); i++) { + for (j = 0; j < sizeof(hashtest_initvals) / sizeof(uint32_t); j++) { + hash = rte_jhash(key, hashtest_key_lens[i], + hashtest_initvals[j]); + if (hash != hash_values_jhash[j][i]) { + printf("jhash for %u bytes with initial value 0x%x." + "Expected 0x%x, but got 0x%x\n", + hashtest_key_lens[i], hashtest_initvals[j], + hash_values_jhash[j][i], hash); + return -1; + } + + hash = rte_hash_crc(key, hashtest_key_lens[i], + hashtest_initvals[j]); + if (hash != hash_values_crc[j][i]) { + printf("CRC for %u bytes with initial value 0x%x." + "Expected 0x%x, but got 0x%x\n", + hashtest_key_lens[i], hashtest_initvals[j], + hash_values_crc[j][i], hash); + return -1; + } + } + } + + return 0; +} + +/* + * Verify that rte_jhash and rte_jhash_32b return the same + */ +static int +verify_jhash_32bits(void) +{ + unsigned i, j; + uint8_t key[64]; + uint32_t hash, hash32; + + for (i = 0; i < 64; i++) + key[i] = rand() & 0xff; + + for (i = 0; i < sizeof(hashtest_key_lens) / sizeof(uint32_t); i++) { + for (j = 0; j < sizeof(hashtest_initvals) / sizeof(uint32_t); j++) { + /* Key size must be multiple of 4 (32 bits) */ + if ((hashtest_key_lens[i] & 0x3) == 0) { + hash = rte_jhash(key, hashtest_key_lens[i], + hashtest_initvals[j]); + /* Divide key length by 4 in rte_jhash for 32 bits */ + hash32 = rte_jhash_32b((const unaligned_uint32_t *)key, + hashtest_key_lens[i] >> 2, + hashtest_initvals[j]); + if (hash != hash32) { + printf("rte_jhash returns different value (0x%x)" + "than rte_jhash_32b (0x%x)\n", + hash, hash32); + return -1; + } + } + } + } + + return 0; +} + +/* + * Verify that rte_jhash and rte_jhash_1word, rte_jhash_2words + * and rte_jhash_3words return the same + */ +static int +verify_jhash_words(void) +{ + unsigned i; + uint32_t key[3]; + uint32_t hash, hash_words; + + for (i = 0; i < 3; i++) + key[i] = rand(); + + /* Test rte_jhash_1word */ + hash = rte_jhash(key, 4, 0); + hash_words = rte_jhash_1word(key[0], 0); + if (hash != hash_words) { + printf("rte_jhash returns different value (0x%x)" + "than rte_jhash_1word (0x%x)\n", + hash, hash_words); + return -1; + } + /* Test rte_jhash_2words */ + hash = rte_jhash(key, 8, 0); + hash_words = rte_jhash_2words(key[0], key[1], 0); + if (hash != hash_words) { + printf("rte_jhash returns different value (0x%x)" + "than rte_jhash_2words (0x%x)\n", + hash, hash_words); + return -1; + } + /* Test rte_jhash_3words */ + hash = rte_jhash(key, 12, 0); + hash_words = rte_jhash_3words(key[0], key[1], key[2], 0); + if (hash != hash_words) { + printf("rte_jhash returns different value (0x%x)" + "than rte_jhash_3words (0x%x)\n", + hash, hash_words); + return -1; + } + + return 0; +} + +/* + * Run all functional tests for hash functions + */ +static int +run_hash_func_tests(void) +{ + if (verify_precalculated_hash_func_tests() != 0) + return -1; + + if (verify_jhash_32bits() != 0) + return -1; + + if (verify_jhash_words() != 0) + return -1; + + return 0; + +} + +static int +test_hash_functions(void) +{ + if (run_hash_func_tests() != 0) + return -1; + + run_hash_func_perf_tests(); + + return 0; +} + +REGISTER_TEST_COMMAND(hash_functions_autotest, test_hash_functions); diff --git a/dpdk/test/test/test_hash_multiwriter.c b/dpdk/test/test/test_hash_multiwriter.c new file mode 100644 index 00000000..4dcbd9d5 --- /dev/null +++ b/dpdk/test/test/test_hash_multiwriter.c @@ -0,0 +1,281 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include <inttypes.h> +#include <locale.h> + +#include <rte_cycles.h> +#include <rte_hash.h> +#include <rte_hash_crc.h> +#include <rte_launch.h> +#include <rte_malloc.h> +#include <rte_random.h> +#include <rte_spinlock.h> + +#include "test.h" + +/* + * Check condition and return an error if true. Assumes that "handle" is the + * name of the hash structure pointer to be freed. + */ +#define RETURN_IF_ERROR(cond, str, ...) do { \ + if (cond) { \ + printf("ERROR line %d: " str "\n", __LINE__, \ + ##__VA_ARGS__); \ + if (handle) \ + rte_hash_free(handle); \ + return -1; \ + } \ +} while (0) + +#define RTE_APP_TEST_HASH_MULTIWRITER_FAILED 0 + +struct { + uint32_t *keys; + uint32_t *found; + uint32_t nb_tsx_insertion; + struct rte_hash *h; +} tbl_multiwriter_test_params; + +const uint32_t nb_entries = 16*1024*1024; +const uint32_t nb_total_tsx_insertion = 15*1024*1024; +uint32_t rounded_nb_total_tsx_insertion; + +static rte_atomic64_t gcycles; +static rte_atomic64_t ginsertions; + +static int use_htm; + +static int +test_hash_multiwriter_worker(__attribute__((unused)) void *arg) +{ + uint64_t i, offset; + uint32_t lcore_id = rte_lcore_id(); + uint64_t begin, cycles; + + offset = (lcore_id - rte_get_master_lcore()) + * tbl_multiwriter_test_params.nb_tsx_insertion; + + printf("Core #%d inserting %d: %'"PRId64" - %'"PRId64"\n", + lcore_id, tbl_multiwriter_test_params.nb_tsx_insertion, + offset, offset + tbl_multiwriter_test_params.nb_tsx_insertion); + + begin = rte_rdtsc_precise(); + + for (i = offset; + i < offset + tbl_multiwriter_test_params.nb_tsx_insertion; + i++) { + if (rte_hash_add_key(tbl_multiwriter_test_params.h, + tbl_multiwriter_test_params.keys + i) < 0) + break; + } + + cycles = rte_rdtsc_precise() - begin; + rte_atomic64_add(&gcycles, cycles); + rte_atomic64_add(&ginsertions, i - offset); + + for (; i < offset + tbl_multiwriter_test_params.nb_tsx_insertion; i++) + tbl_multiwriter_test_params.keys[i] + = RTE_APP_TEST_HASH_MULTIWRITER_FAILED; + + return 0; +} + + +static int +test_hash_multiwriter(void) +{ + unsigned int i, rounded_nb_total_tsx_insertion; + static unsigned calledCount = 1; + + uint32_t *keys; + uint32_t *found; + + struct rte_hash_parameters hash_params = { + .entries = nb_entries, + .key_len = sizeof(uint32_t), + .hash_func = rte_hash_crc, + .hash_func_init_val = 0, + .socket_id = rte_socket_id(), + }; + if (use_htm) + hash_params.extra_flag = + RTE_HASH_EXTRA_FLAGS_TRANS_MEM_SUPPORT + | RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD; + else + hash_params.extra_flag = + RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD; + + struct rte_hash *handle; + char name[RTE_HASH_NAMESIZE]; + + const void *next_key; + void *next_data; + uint32_t iter = 0; + + uint32_t duplicated_keys = 0; + uint32_t lost_keys = 0; + + snprintf(name, 32, "test%u", calledCount++); + hash_params.name = name; + + handle = rte_hash_create(&hash_params); + RETURN_IF_ERROR(handle == NULL, "hash creation failed"); + + tbl_multiwriter_test_params.h = handle; + tbl_multiwriter_test_params.nb_tsx_insertion = + nb_total_tsx_insertion / rte_lcore_count(); + + rounded_nb_total_tsx_insertion = (nb_total_tsx_insertion / + tbl_multiwriter_test_params.nb_tsx_insertion) + * tbl_multiwriter_test_params.nb_tsx_insertion; + + rte_srand(rte_rdtsc()); + + keys = rte_malloc(NULL, sizeof(uint32_t) * nb_entries, 0); + + if (keys == NULL) { + printf("RTE_MALLOC failed\n"); + goto err1; + } + + found = rte_zmalloc(NULL, sizeof(uint32_t) * nb_entries, 0); + if (found == NULL) { + printf("RTE_ZMALLOC failed\n"); + goto err2; + } + + for (i = 0; i < nb_entries; i++) + keys[i] = i; + + tbl_multiwriter_test_params.keys = keys; + tbl_multiwriter_test_params.found = found; + + rte_atomic64_init(&gcycles); + rte_atomic64_clear(&gcycles); + + rte_atomic64_init(&ginsertions); + rte_atomic64_clear(&ginsertions); + + /* Fire all threads. */ + rte_eal_mp_remote_launch(test_hash_multiwriter_worker, + NULL, CALL_MASTER); + rte_eal_mp_wait_lcore(); + + while (rte_hash_iterate(handle, &next_key, &next_data, &iter) >= 0) { + /* Search for the key in the list of keys added .*/ + i = *(const uint32_t *)next_key; + tbl_multiwriter_test_params.found[i]++; + } + + for (i = 0; i < rounded_nb_total_tsx_insertion; i++) { + if (tbl_multiwriter_test_params.keys[i] + != RTE_APP_TEST_HASH_MULTIWRITER_FAILED) { + if (tbl_multiwriter_test_params.found[i] > 1) { + duplicated_keys++; + break; + } + if (tbl_multiwriter_test_params.found[i] == 0) { + lost_keys++; + printf("key %d is lost\n", i); + break; + } + } + } + + if (duplicated_keys > 0) { + printf("%d key duplicated\n", duplicated_keys); + goto err3; + } + + if (lost_keys > 0) { + printf("%d key lost\n", lost_keys); + goto err3; + } + + printf("No key corrupted during multiwriter insertion.\n"); + + unsigned long long int cycles_per_insertion = + rte_atomic64_read(&gcycles)/ + rte_atomic64_read(&ginsertions); + + printf(" cycles per insertion: %llu\n", cycles_per_insertion); + + rte_free(tbl_multiwriter_test_params.found); + rte_free(tbl_multiwriter_test_params.keys); + rte_hash_free(handle); + return 0; + +err3: + rte_free(tbl_multiwriter_test_params.found); +err2: + rte_free(tbl_multiwriter_test_params.keys); +err1: + rte_hash_free(handle); + return -1; +} + +static int +test_hash_multiwriter_main(void) +{ + if (rte_lcore_count() == 1) { + printf("More than one lcore is required to do multiwriter test\n"); + return 0; + } + + + setlocale(LC_NUMERIC, ""); + + + if (!rte_tm_supported()) { + printf("Hardware transactional memory (lock elision) " + "is NOT supported\n"); + } else { + printf("Hardware transactional memory (lock elision) " + "is supported\n"); + + printf("Test multi-writer with Hardware transactional memory\n"); + + use_htm = 1; + if (test_hash_multiwriter() < 0) + return -1; + } + + printf("Test multi-writer without Hardware transactional memory\n"); + use_htm = 0; + if (test_hash_multiwriter() < 0) + return -1; + + return 0; +} + +REGISTER_TEST_COMMAND(hash_multiwriter_autotest, test_hash_multiwriter_main); diff --git a/dpdk/test/test/test_hash_perf.c b/dpdk/test/test/test_hash_perf.c new file mode 100644 index 00000000..c0051b20 --- /dev/null +++ b/dpdk/test/test/test_hash_perf.c @@ -0,0 +1,659 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <inttypes.h> + +#include <rte_lcore.h> +#include <rte_cycles.h> +#include <rte_malloc.h> +#include <rte_hash.h> +#include <rte_hash_crc.h> +#include <rte_jhash.h> +#include <rte_fbk_hash.h> +#include <rte_random.h> +#include <rte_string_fns.h> + +#include "test.h" + +#define MAX_ENTRIES (1 << 19) +#define KEYS_TO_ADD (MAX_ENTRIES * 3 / 4) /* 75% table utilization */ +#define NUM_LOOKUPS (KEYS_TO_ADD * 5) /* Loop among keys added, several times */ +#define BUCKET_SIZE 4 +#define NUM_BUCKETS (MAX_ENTRIES / BUCKET_SIZE) +#define MAX_KEYSIZE 64 +#define NUM_KEYSIZES 10 +#define NUM_SHUFFLES 10 +#define BURST_SIZE 16 + +enum operations { + ADD = 0, + LOOKUP, + LOOKUP_MULTI, + DELETE, + NUM_OPERATIONS +}; + +static uint32_t hashtest_key_lens[] = { + /* standard key sizes */ + 4, 8, 16, 32, 48, 64, + /* IPv4 SRC + DST + protocol, unpadded */ + 9, + /* IPv4 5-tuple, unpadded */ + 13, + /* IPv6 5-tuple, unpadded */ + 37, + /* IPv6 5-tuple, padded to 8-byte boundary */ + 40 +}; + +struct rte_hash *h[NUM_KEYSIZES]; + +/* Array that stores if a slot is full */ +uint8_t slot_taken[MAX_ENTRIES]; + +/* Array to store number of cycles per operation */ +uint64_t cycles[NUM_KEYSIZES][NUM_OPERATIONS][2][2]; + +/* Array to store all input keys */ +uint8_t keys[KEYS_TO_ADD][MAX_KEYSIZE]; + +/* Array to store the precomputed hash for 'keys' */ +hash_sig_t signatures[KEYS_TO_ADD]; + +/* Array to store how many busy entries have each bucket */ +uint8_t buckets[NUM_BUCKETS]; + +/* Array to store the positions where keys are added */ +int32_t positions[KEYS_TO_ADD]; + +/* Parameters used for hash table in unit test functions. */ +static struct rte_hash_parameters ut_params = { + .entries = MAX_ENTRIES, + .hash_func = rte_jhash, + .hash_func_init_val = 0, +}; + +static int +create_table(unsigned with_data, unsigned table_index) +{ + char name[RTE_HASH_NAMESIZE]; + + if (with_data) + /* Table will store 8-byte data */ + sprintf(name, "test_hash%d_data", hashtest_key_lens[table_index]); + else + sprintf(name, "test_hash%d", hashtest_key_lens[table_index]); + + ut_params.name = name; + ut_params.key_len = hashtest_key_lens[table_index]; + ut_params.socket_id = rte_socket_id(); + h[table_index] = rte_hash_find_existing(name); + if (h[table_index] != NULL) + /* + * If table was already created, free it to create it again, + * so we force it is empty + */ + rte_hash_free(h[table_index]); + h[table_index] = rte_hash_create(&ut_params); + if (h[table_index] == NULL) { + printf("Error creating table\n"); + return -1; + } + return 0; + +} + +/* Shuffle the keys that have been added, so lookups will be totally random */ +static void +shuffle_input_keys(unsigned table_index) +{ + unsigned i; + uint32_t swap_idx; + uint8_t temp_key[MAX_KEYSIZE]; + hash_sig_t temp_signature; + int32_t temp_position; + + for (i = KEYS_TO_ADD - 1; i > 0; i--) { + swap_idx = rte_rand() % i; + + memcpy(temp_key, keys[i], hashtest_key_lens[table_index]); + temp_signature = signatures[i]; + temp_position = positions[i]; + + memcpy(keys[i], keys[swap_idx], hashtest_key_lens[table_index]); + signatures[i] = signatures[swap_idx]; + positions[i] = positions[swap_idx]; + + memcpy(keys[swap_idx], temp_key, hashtest_key_lens[table_index]); + signatures[swap_idx] = temp_signature; + positions[swap_idx] = temp_position; + } +} + +/* + * Looks for random keys which + * ALL can fit in hash table (no errors) + */ +static int +get_input_keys(unsigned with_pushes, unsigned table_index) +{ + unsigned i, j; + unsigned bucket_idx, incr, success = 1; + uint8_t k = 0; + int32_t ret; + const uint32_t bucket_bitmask = NUM_BUCKETS - 1; + + /* Reset all arrays */ + for (i = 0; i < MAX_ENTRIES; i++) + slot_taken[i] = 0; + + for (i = 0; i < NUM_BUCKETS; i++) + buckets[i] = 0; + + for (j = 0; j < hashtest_key_lens[table_index]; j++) + keys[0][j] = 0; + + /* + * Add only entries that are not duplicated and that fits in the table + * (cannot store more than BUCKET_SIZE entries in a bucket). + * Regardless a key has been added correctly or not (success), + * the next one to try will be increased by 1. + */ + for (i = 0; i < KEYS_TO_ADD;) { + incr = 0; + if (i != 0) { + keys[i][0] = ++k; + /* Overflow, need to increment the next byte */ + if (keys[i][0] == 0) + incr = 1; + for (j = 1; j < hashtest_key_lens[table_index]; j++) { + /* Do not increase next byte */ + if (incr == 0) + if (success == 1) + keys[i][j] = keys[i - 1][j]; + else + keys[i][j] = keys[i][j]; + /* Increase next byte by one */ + else { + if (success == 1) + keys[i][j] = keys[i-1][j] + 1; + else + keys[i][j] = keys[i][j] + 1; + if (keys[i][j] == 0) + incr = 1; + else + incr = 0; + } + } + } + success = 0; + signatures[i] = rte_hash_hash(h[table_index], keys[i]); + bucket_idx = signatures[i] & bucket_bitmask; + /* + * If we are not inserting keys in secondary location, + * when bucket is full, do not try to insert the key + */ + if (with_pushes == 0) + if (buckets[bucket_idx] == BUCKET_SIZE) + continue; + + /* If key can be added, leave in successful key arrays "keys" */ + ret = rte_hash_add_key_with_hash(h[table_index], keys[i], + signatures[i]); + if (ret >= 0) { + /* If key is already added, ignore the entry and do not store */ + if (slot_taken[ret]) + continue; + else { + /* Store the returned position and mark slot as taken */ + slot_taken[ret] = 1; + positions[i] = ret; + buckets[bucket_idx]++; + success = 1; + i++; + } + } + } + + /* Reset the table, so we can measure the time to add all the entries */ + rte_hash_free(h[table_index]); + h[table_index] = rte_hash_create(&ut_params); + + return 0; +} + +static int +timed_adds(unsigned with_hash, unsigned with_data, unsigned table_index) +{ + unsigned i; + const uint64_t start_tsc = rte_rdtsc(); + void *data; + int32_t ret; + + for (i = 0; i < KEYS_TO_ADD; i++) { + data = (void *) ((uintptr_t) signatures[i]); + if (with_hash && with_data) { + ret = rte_hash_add_key_with_hash_data(h[table_index], + (const void *) keys[i], + signatures[i], data); + if (ret < 0) { + printf("Failed to add key number %u\n", ret); + return -1; + } + } else if (with_hash && !with_data) { + ret = rte_hash_add_key_with_hash(h[table_index], + (const void *) keys[i], + signatures[i]); + if (ret >= 0) + positions[i] = ret; + else { + printf("Failed to add key number %u\n", ret); + return -1; + } + } else if (!with_hash && with_data) { + ret = rte_hash_add_key_data(h[table_index], + (const void *) keys[i], + data); + if (ret < 0) { + printf("Failed to add key number %u\n", ret); + return -1; + } + } else { + ret = rte_hash_add_key(h[table_index], keys[i]); + if (ret >= 0) + positions[i] = ret; + else { + printf("Failed to add key number %u\n", ret); + return -1; + } + } + } + + const uint64_t end_tsc = rte_rdtsc(); + const uint64_t time_taken = end_tsc - start_tsc; + + cycles[table_index][ADD][with_hash][with_data] = time_taken/KEYS_TO_ADD; + + return 0; +} + +static int +timed_lookups(unsigned with_hash, unsigned with_data, unsigned table_index) +{ + unsigned i, j; + const uint64_t start_tsc = rte_rdtsc(); + void *ret_data; + void *expected_data; + int32_t ret; + + for (i = 0; i < NUM_LOOKUPS/KEYS_TO_ADD; i++) { + for (j = 0; j < KEYS_TO_ADD; j++) { + if (with_hash && with_data) { + ret = rte_hash_lookup_with_hash_data(h[table_index], + (const void *) keys[j], + signatures[j], &ret_data); + if (ret < 0) { + printf("Key number %u was not found\n", j); + return -1; + } + expected_data = (void *) ((uintptr_t) signatures[j]); + if (ret_data != expected_data) { + printf("Data returned for key number %u is %p," + " but should be %p\n", j, ret_data, + expected_data); + return -1; + } + } else if (with_hash && !with_data) { + ret = rte_hash_lookup_with_hash(h[table_index], + (const void *) keys[j], + signatures[j]); + if (ret < 0 || ret != positions[j]) { + printf("Key looked up in %d, should be in %d\n", + ret, positions[j]); + return -1; + } + } else if (!with_hash && with_data) { + ret = rte_hash_lookup_data(h[table_index], + (const void *) keys[j], &ret_data); + if (ret < 0) { + printf("Key number %u was not found\n", j); + return -1; + } + expected_data = (void *) ((uintptr_t) signatures[j]); + if (ret_data != expected_data) { + printf("Data returned for key number %u is %p," + " but should be %p\n", j, ret_data, + expected_data); + return -1; + } + } else { + ret = rte_hash_lookup(h[table_index], keys[j]); + if (ret < 0 || ret != positions[j]) { + printf("Key looked up in %d, should be in %d\n", + ret, positions[j]); + return -1; + } + } + } + } + + const uint64_t end_tsc = rte_rdtsc(); + const uint64_t time_taken = end_tsc - start_tsc; + + cycles[table_index][LOOKUP][with_hash][with_data] = time_taken/NUM_LOOKUPS; + + return 0; +} + +static int +timed_lookups_multi(unsigned with_data, unsigned table_index) +{ + unsigned i, j, k; + int32_t positions_burst[BURST_SIZE]; + const void *keys_burst[BURST_SIZE]; + void *expected_data[BURST_SIZE]; + void *ret_data[BURST_SIZE]; + uint64_t hit_mask; + int ret; + + const uint64_t start_tsc = rte_rdtsc(); + + for (i = 0; i < NUM_LOOKUPS/KEYS_TO_ADD; i++) { + for (j = 0; j < KEYS_TO_ADD/BURST_SIZE; j++) { + for (k = 0; k < BURST_SIZE; k++) + keys_burst[k] = keys[j * BURST_SIZE + k]; + if (with_data) { + ret = rte_hash_lookup_bulk_data(h[table_index], + (const void **) keys_burst, + BURST_SIZE, + &hit_mask, + ret_data); + if (ret != BURST_SIZE) { + printf("Expect to find %u keys," + " but found %d\n", BURST_SIZE, ret); + return -1; + } + for (k = 0; k < BURST_SIZE; k++) { + if ((hit_mask & (1ULL << k)) == 0) { + printf("Key number %u not found\n", + j * BURST_SIZE + k); + return -1; + } + expected_data[k] = (void *) ((uintptr_t) signatures[j * BURST_SIZE + k]); + if (ret_data[k] != expected_data[k]) { + printf("Data returned for key number %u is %p," + " but should be %p\n", j * BURST_SIZE + k, + ret_data[k], expected_data[k]); + return -1; + } + } + } else { + rte_hash_lookup_bulk(h[table_index], + (const void **) keys_burst, + BURST_SIZE, + positions_burst); + for (k = 0; k < BURST_SIZE; k++) { + if (positions_burst[k] != positions[j * BURST_SIZE + k]) { + printf("Key looked up in %d, should be in %d\n", + positions_burst[k], + positions[j * BURST_SIZE + k]); + return -1; + } + } + } + } + } + + const uint64_t end_tsc = rte_rdtsc(); + const uint64_t time_taken = end_tsc - start_tsc; + + cycles[table_index][LOOKUP_MULTI][0][with_data] = time_taken/NUM_LOOKUPS; + + return 0; +} + +static int +timed_deletes(unsigned with_hash, unsigned with_data, unsigned table_index) +{ + unsigned i; + const uint64_t start_tsc = rte_rdtsc(); + int32_t ret; + + for (i = 0; i < KEYS_TO_ADD; i++) { + /* There are no delete functions with data, so just call two functions */ + if (with_hash) + ret = rte_hash_del_key_with_hash(h[table_index], + (const void *) keys[i], + signatures[i]); + else + ret = rte_hash_del_key(h[table_index], + (const void *) keys[i]); + if (ret >= 0) + positions[i] = ret; + else { + printf("Failed to add key number %u\n", ret); + return -1; + } + } + + const uint64_t end_tsc = rte_rdtsc(); + const uint64_t time_taken = end_tsc - start_tsc; + + cycles[table_index][DELETE][with_hash][with_data] = time_taken/KEYS_TO_ADD; + + return 0; +} + +static void +free_table(unsigned table_index) +{ + rte_hash_free(h[table_index]); +} + +static void +reset_table(unsigned table_index) +{ + rte_hash_reset(h[table_index]); +} + +static int +run_all_tbl_perf_tests(unsigned with_pushes) +{ + unsigned i, j, with_data, with_hash; + + printf("Measuring performance, please wait"); + fflush(stdout); + + for (with_data = 0; with_data <= 1; with_data++) { + for (i = 0; i < NUM_KEYSIZES; i++) { + if (create_table(with_data, i) < 0) + return -1; + + if (get_input_keys(with_pushes, i) < 0) + return -1; + for (with_hash = 0; with_hash <= 1; with_hash++) { + if (timed_adds(with_hash, with_data, i) < 0) + return -1; + + for (j = 0; j < NUM_SHUFFLES; j++) + shuffle_input_keys(i); + + if (timed_lookups(with_hash, with_data, i) < 0) + return -1; + + if (timed_lookups_multi(with_data, i) < 0) + return -1; + + if (timed_deletes(with_hash, with_data, i) < 0) + return -1; + + /* Print a dot to show progress on operations */ + printf("."); + fflush(stdout); + + reset_table(i); + } + free_table(i); + } + } + + printf("\nResults (in CPU cycles/operation)\n"); + printf("-----------------------------------\n"); + for (with_data = 0; with_data <= 1; with_data++) { + if (with_data) + printf("\n Operations with 8-byte data\n"); + else + printf("\n Operations without data\n"); + for (with_hash = 0; with_hash <= 1; with_hash++) { + if (with_hash) + printf("\nWith pre-computed hash values\n"); + else + printf("\nWithout pre-computed hash values\n"); + + printf("\n%-18s%-18s%-18s%-18s%-18s\n", + "Keysize", "Add", "Lookup", "Lookup_bulk", "Delete"); + for (i = 0; i < NUM_KEYSIZES; i++) { + printf("%-18d", hashtest_key_lens[i]); + for (j = 0; j < NUM_OPERATIONS; j++) + printf("%-18"PRIu64, cycles[i][j][with_hash][with_data]); + printf("\n"); + } + } + } + return 0; +} + +/* Control operation of performance testing of fbk hash. */ +#define LOAD_FACTOR 0.667 /* How full to make the hash table. */ +#define TEST_SIZE 1000000 /* How many operations to time. */ +#define TEST_ITERATIONS 30 /* How many measurements to take. */ +#define ENTRIES (1 << 15) /* How many entries. */ + +static int +fbk_hash_perf_test(void) +{ + struct rte_fbk_hash_params params = { + .name = "fbk_hash_test", + .entries = ENTRIES, + .entries_per_bucket = 4, + .socket_id = rte_socket_id(), + }; + struct rte_fbk_hash_table *handle = NULL; + uint32_t *keys = NULL; + unsigned indexes[TEST_SIZE]; + uint64_t lookup_time = 0; + unsigned added = 0; + unsigned value = 0; + uint32_t key; + uint16_t val; + unsigned i, j; + + handle = rte_fbk_hash_create(¶ms); + if (handle == NULL) { + printf("Error creating table\n"); + return -1; + } + + keys = rte_zmalloc(NULL, ENTRIES * sizeof(*keys), 0); + if (keys == NULL) { + printf("fbk hash: memory allocation for key store failed\n"); + return -1; + } + + /* Generate random keys and values. */ + for (i = 0; i < ENTRIES; i++) { + key = (uint32_t)rte_rand(); + key = ((uint64_t)key << 32) | (uint64_t)rte_rand(); + val = (uint16_t)rte_rand(); + + if (rte_fbk_hash_add_key(handle, key, val) == 0) { + keys[added] = key; + added++; + } + if (added > (LOAD_FACTOR * ENTRIES)) + break; + } + + for (i = 0; i < TEST_ITERATIONS; i++) { + uint64_t begin; + uint64_t end; + + /* Generate random indexes into keys[] array. */ + for (j = 0; j < TEST_SIZE; j++) + indexes[j] = rte_rand() % added; + + begin = rte_rdtsc(); + /* Do lookups */ + for (j = 0; j < TEST_SIZE; j++) + value += rte_fbk_hash_lookup(handle, keys[indexes[j]]); + + end = rte_rdtsc(); + lookup_time += (double)(end - begin); + } + + printf("\n\n *** FBK Hash function performance test results ***\n"); + /* + * The use of the 'value' variable ensures that the hash lookup is not + * being optimised out by the compiler. + */ + if (value != 0) + printf("Number of ticks per lookup = %g\n", + (double)lookup_time / + ((double)TEST_ITERATIONS * (double)TEST_SIZE)); + + rte_fbk_hash_free(handle); + + return 0; +} + +static int +test_hash_perf(void) +{ + unsigned with_pushes; + + for (with_pushes = 0; with_pushes <= 1; with_pushes++) { + if (with_pushes == 0) + printf("\nALL ELEMENTS IN PRIMARY LOCATION\n"); + else + printf("\nELEMENTS IN PRIMARY OR SECONDARY LOCATION\n"); + if (run_all_tbl_perf_tests(with_pushes) < 0) + return -1; + } + if (fbk_hash_perf_test() < 0) + return -1; + + return 0; +} + +REGISTER_TEST_COMMAND(hash_perf_autotest, test_hash_perf); diff --git a/dpdk/test/test/test_hash_scaling.c b/dpdk/test/test/test_hash_scaling.c new file mode 100644 index 00000000..46c48e54 --- /dev/null +++ b/dpdk/test/test/test_hash_scaling.c @@ -0,0 +1,220 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2015 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> + +#include <rte_cycles.h> +#include <rte_hash.h> +#include <rte_hash_crc.h> +#include <rte_spinlock.h> +#include <rte_launch.h> + +#include "test.h" + +/* + * Check condition and return an error if true. Assumes that "handle" is the + * name of the hash structure pointer to be freed. + */ +#define RETURN_IF_ERROR(cond, str, ...) do { \ + if (cond) { \ + printf("ERROR line %d: " str "\n", __LINE__, \ + ##__VA_ARGS__); \ + if (handle) \ + rte_hash_free(handle); \ + return -1; \ + } \ +} while (0) + +enum locking_mode_t { + NORMAL_LOCK, + LOCK_ELISION, + NULL_LOCK +}; + +struct { + uint32_t num_iterations; + struct rte_hash *h; + rte_spinlock_t *lock; + int locking_mode; +} tbl_scaling_test_params; + +static rte_atomic64_t gcycles; + +static int test_hash_scaling_worker(__attribute__((unused)) void *arg) +{ + uint64_t i, key; + uint32_t thr_id = rte_sys_gettid(); + uint64_t begin, cycles = 0; + + switch (tbl_scaling_test_params.locking_mode) { + + case NORMAL_LOCK: + + for (i = 0; i < tbl_scaling_test_params.num_iterations; i++) { + /* different threads get different keys because + we use the thread-id in the key computation + */ + key = rte_hash_crc(&i, sizeof(i), thr_id); + begin = rte_rdtsc_precise(); + rte_spinlock_lock(tbl_scaling_test_params.lock); + rte_hash_add_key(tbl_scaling_test_params.h, &key); + rte_spinlock_unlock(tbl_scaling_test_params.lock); + cycles += rte_rdtsc_precise() - begin; + } + break; + + case LOCK_ELISION: + + for (i = 0; i < tbl_scaling_test_params.num_iterations; i++) { + key = rte_hash_crc(&i, sizeof(i), thr_id); + begin = rte_rdtsc_precise(); + rte_spinlock_lock_tm(tbl_scaling_test_params.lock); + rte_hash_add_key(tbl_scaling_test_params.h, &key); + rte_spinlock_unlock_tm(tbl_scaling_test_params.lock); + cycles += rte_rdtsc_precise() - begin; + } + break; + + default: + + for (i = 0; i < tbl_scaling_test_params.num_iterations; i++) { + key = rte_hash_crc(&i, sizeof(i), thr_id); + begin = rte_rdtsc_precise(); + rte_hash_add_key(tbl_scaling_test_params.h, &key); + cycles += rte_rdtsc_precise() - begin; + } + } + + rte_atomic64_add(&gcycles, cycles); + + return 0; +} + +/* + * Do scalability perf tests. + */ +static int +test_hash_scaling(int locking_mode) +{ + static unsigned calledCount = 1; + uint32_t num_iterations = 1024*1024; + uint64_t i, key; + struct rte_hash_parameters hash_params = { + .entries = num_iterations*2, + .key_len = sizeof(key), + .hash_func = rte_hash_crc, + .hash_func_init_val = 0, + .socket_id = rte_socket_id(), + .extra_flag = RTE_HASH_EXTRA_FLAGS_TRANS_MEM_SUPPORT + }; + struct rte_hash *handle; + char name[RTE_HASH_NAMESIZE]; + rte_spinlock_t lock; + + rte_spinlock_init(&lock); + + snprintf(name, 32, "test%u", calledCount++); + hash_params.name = name; + + handle = rte_hash_create(&hash_params); + RETURN_IF_ERROR(handle == NULL, "hash creation failed"); + + tbl_scaling_test_params.num_iterations = + num_iterations/rte_lcore_count(); + tbl_scaling_test_params.h = handle; + tbl_scaling_test_params.lock = &lock; + tbl_scaling_test_params.locking_mode = locking_mode; + + rte_atomic64_init(&gcycles); + rte_atomic64_clear(&gcycles); + + /* fill up to initial size */ + for (i = 0; i < num_iterations; i++) { + key = rte_hash_crc(&i, sizeof(i), 0xabcdabcd); + rte_hash_add_key(tbl_scaling_test_params.h, &key); + } + + rte_eal_mp_remote_launch(test_hash_scaling_worker, NULL, CALL_MASTER); + rte_eal_mp_wait_lcore(); + + unsigned long long int cycles_per_operation = + rte_atomic64_read(&gcycles)/ + (tbl_scaling_test_params.num_iterations*rte_lcore_count()); + const char *lock_name; + + switch (locking_mode) { + case NORMAL_LOCK: + lock_name = "normal spinlock"; + break; + case LOCK_ELISION: + lock_name = "lock elision"; + break; + default: + lock_name = "null lock"; + } + printf("--------------------------------------------------------\n"); + printf("Cores: %d; %s mode -> cycles per operation: %llu\n", + rte_lcore_count(), lock_name, cycles_per_operation); + printf("--------------------------------------------------------\n"); + /* CSV output */ + printf(">>>%d,%s,%llu\n", rte_lcore_count(), lock_name, + cycles_per_operation); + + rte_hash_free(handle); + return 0; +} + +static int +test_hash_scaling_main(void) +{ + int r = 0; + + if (rte_lcore_count() == 1) + r = test_hash_scaling(NULL_LOCK); + + if (r == 0) + r = test_hash_scaling(NORMAL_LOCK); + + if (!rte_tm_supported()) { + printf("Hardware transactional memory (lock elision) is NOT supported\n"); + return r; + } + printf("Hardware transactional memory (lock elision) is supported\n"); + + if (r == 0) + r = test_hash_scaling(LOCK_ELISION); + + return r; +} + +REGISTER_TEST_COMMAND(hash_scaling_autotest, test_hash_scaling_main); diff --git a/dpdk/test/test/test_interrupts.c b/dpdk/test/test/test_interrupts.c new file mode 100644 index 00000000..e0cd8a82 --- /dev/null +++ b/dpdk/test/test/test_interrupts.c @@ -0,0 +1,552 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <stdint.h> +#include <unistd.h> + +#include <rte_common.h> +#include <rte_cycles.h> +#include <rte_interrupts.h> + +#include "test.h" + +#define TEST_INTERRUPT_CHECK_INTERVAL 100 /* ms */ + +/* predefined interrupt handle types */ +enum test_interrupt_handle_type { + TEST_INTERRUPT_HANDLE_INVALID, + TEST_INTERRUPT_HANDLE_VALID, + TEST_INTERRUPT_HANDLE_VALID_UIO, + TEST_INTERRUPT_HANDLE_VALID_ALARM, + TEST_INTERRUPT_HANDLE_CASE1, + TEST_INTERRUPT_HANDLE_MAX +}; + +/* flag of if callback is called */ +static volatile int flag; +static struct rte_intr_handle intr_handles[TEST_INTERRUPT_HANDLE_MAX]; +static enum test_interrupt_handle_type test_intr_type = + TEST_INTERRUPT_HANDLE_MAX; + +#ifdef RTE_EXEC_ENV_LINUXAPP +union intr_pipefds{ + struct { + int pipefd[2]; + }; + struct { + int readfd; + int writefd; + }; +}; + +static union intr_pipefds pfds; + +/** + * Check if the interrupt handle is valid. + */ +static inline int +test_interrupt_handle_sanity_check(struct rte_intr_handle *intr_handle) +{ + if (!intr_handle || intr_handle->fd < 0) + return -1; + + return 0; +} + +/** + * Initialization for interrupt test. + */ +static int +test_interrupt_init(void) +{ + if (pipe(pfds.pipefd) < 0) + return -1; + + intr_handles[TEST_INTERRUPT_HANDLE_INVALID].fd = -1; + intr_handles[TEST_INTERRUPT_HANDLE_INVALID].type = + RTE_INTR_HANDLE_UNKNOWN; + + intr_handles[TEST_INTERRUPT_HANDLE_VALID].fd = pfds.readfd; + intr_handles[TEST_INTERRUPT_HANDLE_VALID].type = + RTE_INTR_HANDLE_UNKNOWN; + + intr_handles[TEST_INTERRUPT_HANDLE_VALID_UIO].fd = pfds.readfd; + intr_handles[TEST_INTERRUPT_HANDLE_VALID_UIO].type = + RTE_INTR_HANDLE_UIO; + + intr_handles[TEST_INTERRUPT_HANDLE_VALID_ALARM].fd = pfds.readfd; + intr_handles[TEST_INTERRUPT_HANDLE_VALID_ALARM].type = + RTE_INTR_HANDLE_ALARM; + + intr_handles[TEST_INTERRUPT_HANDLE_CASE1].fd = pfds.writefd; + intr_handles[TEST_INTERRUPT_HANDLE_CASE1].type = RTE_INTR_HANDLE_UIO; + + return 0; +} + +/** + * Deinitialization for interrupt test. + */ +static int +test_interrupt_deinit(void) +{ + close(pfds.pipefd[0]); + close(pfds.pipefd[1]); + + return 0; +} + +/** + * Write the pipe to simulate an interrupt. + */ +static int +test_interrupt_trigger_interrupt(void) +{ + if (write(pfds.writefd, "1", 1) < 0) + return -1; + + return 0; +} + +/** + * Check if two interrupt handles are the same. + */ +static int +test_interrupt_handle_compare(struct rte_intr_handle *intr_handle_l, + struct rte_intr_handle *intr_handle_r) +{ + if (!intr_handle_l || !intr_handle_r) + return -1; + + if (intr_handle_l->fd != intr_handle_r->fd || + intr_handle_l->type != intr_handle_r->type) + return -1; + + return 0; +} + +#else +/* to be implemented for bsd later */ +static inline int +test_interrupt_handle_sanity_check(struct rte_intr_handle *intr_handle) +{ + RTE_SET_USED(intr_handle); + + return 0; +} + +static int +test_interrupt_init(void) +{ + return 0; +} + +static int +test_interrupt_deinit(void) +{ + return 0; +} + +static int +test_interrupt_trigger_interrupt(void) +{ + return 0; +} + +static int +test_interrupt_handle_compare(struct rte_intr_handle *intr_handle_l, + struct rte_intr_handle *intr_handle_r) +{ + (void)intr_handle_l; + (void)intr_handle_r; + + return 0; +} +#endif /* RTE_EXEC_ENV_LINUXAPP */ + +/** + * Callback for the test interrupt. + */ +static void +test_interrupt_callback(void *arg) +{ + struct rte_intr_handle *intr_handle = arg; + if (test_intr_type >= TEST_INTERRUPT_HANDLE_MAX) { + printf("invalid interrupt type\n"); + flag = -1; + return; + } + + if (test_interrupt_handle_sanity_check(intr_handle) < 0) { + printf("null or invalid intr_handle for %s\n", __func__); + flag = -1; + return; + } + + if (rte_intr_callback_unregister(intr_handle, + test_interrupt_callback, arg) >= 0) { + printf("%s: unexpectedly able to unregister itself\n", + __func__); + flag = -1; + return; + } + + if (test_interrupt_handle_compare(intr_handle, + &(intr_handles[test_intr_type])) == 0) + flag = 1; +} + +/** + * Callback for the test interrupt. + */ +static void +test_interrupt_callback_1(void *arg) +{ + struct rte_intr_handle *intr_handle = arg; + if (test_interrupt_handle_sanity_check(intr_handle) < 0) { + printf("null or invalid intr_handle for %s\n", __func__); + flag = -1; + return; + } +} + +/** + * Tests for rte_intr_enable(). + */ +static int +test_interrupt_enable(void) +{ + struct rte_intr_handle test_intr_handle; + + /* check with null intr_handle */ + if (rte_intr_enable(NULL) == 0) { + printf("unexpectedly enable null intr_handle successfully\n"); + return -1; + } + + /* check with invalid intr_handle */ + test_intr_handle = intr_handles[TEST_INTERRUPT_HANDLE_INVALID]; + if (rte_intr_enable(&test_intr_handle) == 0) { + printf("unexpectedly enable invalid intr_handle " + "successfully\n"); + return -1; + } + + /* check with valid intr_handle */ + test_intr_handle = intr_handles[TEST_INTERRUPT_HANDLE_VALID]; + if (rte_intr_enable(&test_intr_handle) == 0) { + printf("unexpectedly enable a specific intr_handle " + "successfully\n"); + return -1; + } + + /* check with specific valid intr_handle */ + test_intr_handle = intr_handles[TEST_INTERRUPT_HANDLE_VALID_ALARM]; + if (rte_intr_enable(&test_intr_handle) == 0) { + printf("unexpectedly enable a specific intr_handle " + "successfully\n"); + return -1; + } + + /* check with valid handler and its type */ + test_intr_handle = intr_handles[TEST_INTERRUPT_HANDLE_CASE1]; + if (rte_intr_enable(&test_intr_handle) < 0) { + printf("fail to enable interrupt on a simulated handler\n"); + return -1; + } + + test_intr_handle = intr_handles[TEST_INTERRUPT_HANDLE_VALID_UIO]; + if (rte_intr_enable(&test_intr_handle) == 0) { + printf("unexpectedly enable a specific intr_handle " + "successfully\n"); + return -1; + } + + return 0; +} + +/** + * Tests for rte_intr_disable(). + */ +static int +test_interrupt_disable(void) +{ + struct rte_intr_handle test_intr_handle; + + /* check with null intr_handle */ + if (rte_intr_disable(NULL) == 0) { + printf("unexpectedly disable null intr_handle " + "successfully\n"); + return -1; + } + + /* check with invalid intr_handle */ + test_intr_handle = intr_handles[TEST_INTERRUPT_HANDLE_INVALID]; + if (rte_intr_disable(&test_intr_handle) == 0) { + printf("unexpectedly disable invalid intr_handle " + "successfully\n"); + return -1; + } + + /* check with valid intr_handle */ + test_intr_handle = intr_handles[TEST_INTERRUPT_HANDLE_VALID]; + if (rte_intr_disable(&test_intr_handle) == 0) { + printf("unexpectedly disable a specific intr_handle " + "successfully\n"); + return -1; + } + + /* check with specific valid intr_handle */ + test_intr_handle = intr_handles[TEST_INTERRUPT_HANDLE_VALID_ALARM]; + if (rte_intr_disable(&test_intr_handle) == 0) { + printf("unexpectedly disable a specific intr_handle " + "successfully\n"); + return -1; + } + + /* check with valid handler and its type */ + test_intr_handle = intr_handles[TEST_INTERRUPT_HANDLE_CASE1]; + if (rte_intr_disable(&test_intr_handle) < 0) { + printf("fail to disable interrupt on a simulated handler\n"); + return -1; + } + + test_intr_handle = intr_handles[TEST_INTERRUPT_HANDLE_VALID_UIO]; + if (rte_intr_disable(&test_intr_handle) == 0) { + printf("unexpectedly disable a specific intr_handle " + "successfully\n"); + return -1; + } + + return 0; +} + +/** + * Check the full path of a specified type of interrupt simulated. + */ +static int +test_interrupt_full_path_check(enum test_interrupt_handle_type intr_type) +{ + int count; + struct rte_intr_handle test_intr_handle; + + flag = 0; + test_intr_handle = intr_handles[intr_type]; + test_intr_type = intr_type; + if (rte_intr_callback_register(&test_intr_handle, + test_interrupt_callback, &test_intr_handle) < 0) { + printf("fail to register callback\n"); + return -1; + } + + if (test_interrupt_trigger_interrupt() < 0) + return -1; + + /* check flag */ + for (count = 0; flag == 0 && count < 3; count++) + rte_delay_ms(TEST_INTERRUPT_CHECK_INTERVAL); + + rte_delay_ms(TEST_INTERRUPT_CHECK_INTERVAL); + if (rte_intr_callback_unregister(&test_intr_handle, + test_interrupt_callback, &test_intr_handle) < 0) + return -1; + + if (flag == 0) { + printf("callback has not been called\n"); + return -1; + } else if (flag < 0) { + printf("it has internal error in callback\n"); + return -1; + } + + return 0; +} + +/** + * Main function of testing interrupt. + */ +static int +test_interrupt(void) +{ + int ret = -1; + struct rte_intr_handle test_intr_handle; + + if (test_interrupt_init() < 0) { + printf("fail to initialize for testing interrupt\n"); + return -1; + } + + printf("Check unknown valid interrupt full path\n"); + if (test_interrupt_full_path_check(TEST_INTERRUPT_HANDLE_VALID) < 0) { + printf("failure occurred during checking unknown valid " + "interrupt full path\n"); + goto out; + } + + printf("Check valid UIO interrupt full path\n"); + if (test_interrupt_full_path_check(TEST_INTERRUPT_HANDLE_VALID_UIO) + < 0) { + printf("failure occurred during checking valid UIO interrupt " + "full path\n"); + goto out; + } + + printf("Check valid alarm interrupt full path\n"); + if (test_interrupt_full_path_check(TEST_INTERRUPT_HANDLE_VALID_ALARM) + < 0) { + printf("failure occurred during checking valid alarm " + "interrupt full path\n"); + goto out; + } + + printf("start register/unregister test\n"); + /* check if it will fail to register cb with intr_handle = NULL */ + if (rte_intr_callback_register(NULL, test_interrupt_callback, + NULL) == 0) { + printf("unexpectedly register successfully with null " + "intr_handle\n"); + goto out; + } + + /* check if it will fail to register cb with invalid intr_handle */ + test_intr_handle = intr_handles[TEST_INTERRUPT_HANDLE_INVALID]; + if (rte_intr_callback_register(&test_intr_handle, + test_interrupt_callback, &test_intr_handle) == 0) { + printf("unexpectedly register successfully with invalid " + "intr_handle\n"); + goto out; + } + + /* check if it will fail to register without callback */ + test_intr_handle = intr_handles[TEST_INTERRUPT_HANDLE_VALID]; + if (rte_intr_callback_register(&test_intr_handle, NULL, &test_intr_handle) == 0) { + printf("unexpectedly register successfully with " + "null callback\n"); + goto out; + } + + /* check if it will fail to unregister cb with intr_handle = NULL */ + if (rte_intr_callback_unregister(NULL, + test_interrupt_callback, NULL) > 0) { + printf("unexpectedly unregister successfully with " + "null intr_handle\n"); + goto out; + } + + /* check if it will fail to unregister cb with invalid intr_handle */ + test_intr_handle = intr_handles[TEST_INTERRUPT_HANDLE_INVALID]; + if (rte_intr_callback_unregister(&test_intr_handle, + test_interrupt_callback, &test_intr_handle) > 0) { + printf("unexpectedly unregister successfully with " + "invalid intr_handle\n"); + goto out; + } + + /* check if it is ok to register the same intr_handle twice */ + test_intr_handle = intr_handles[TEST_INTERRUPT_HANDLE_VALID]; + if (rte_intr_callback_register(&test_intr_handle, + test_interrupt_callback, &test_intr_handle) < 0) { + printf("it fails to register test_interrupt_callback\n"); + goto out; + } + if (rte_intr_callback_register(&test_intr_handle, + test_interrupt_callback_1, &test_intr_handle) < 0) { + printf("it fails to register test_interrupt_callback_1\n"); + goto out; + } + /* check if it will fail to unregister with invalid parameter */ + if (rte_intr_callback_unregister(&test_intr_handle, + test_interrupt_callback, (void *)0xff) != 0) { + printf("unexpectedly unregisters successfully with " + "invalid arg\n"); + goto out; + } + if (rte_intr_callback_unregister(&test_intr_handle, + test_interrupt_callback, &test_intr_handle) <= 0) { + printf("it fails to unregister test_interrupt_callback\n"); + goto out; + } + if (rte_intr_callback_unregister(&test_intr_handle, + test_interrupt_callback_1, (void *)-1) <= 0) { + printf("it fails to unregister test_interrupt_callback_1 " + "for all\n"); + goto out; + } + rte_delay_ms(TEST_INTERRUPT_CHECK_INTERVAL); + + printf("start interrupt enable/disable test\n"); + /* check interrupt enable/disable functions */ + if (test_interrupt_enable() < 0) { + printf("fail to check interrupt enabling\n"); + goto out; + } + rte_delay_ms(TEST_INTERRUPT_CHECK_INTERVAL); + + if (test_interrupt_disable() < 0) { + printf("fail to check interrupt disabling\n"); + goto out; + } + rte_delay_ms(TEST_INTERRUPT_CHECK_INTERVAL); + + ret = 0; + +out: + printf("Clearing for interrupt tests\n"); + /* clear registered callbacks */ + test_intr_handle = intr_handles[TEST_INTERRUPT_HANDLE_VALID]; + rte_intr_callback_unregister(&test_intr_handle, + test_interrupt_callback, (void *)-1); + rte_intr_callback_unregister(&test_intr_handle, + test_interrupt_callback_1, (void *)-1); + + test_intr_handle = intr_handles[TEST_INTERRUPT_HANDLE_VALID_UIO]; + rte_intr_callback_unregister(&test_intr_handle, + test_interrupt_callback, (void *)-1); + rte_intr_callback_unregister(&test_intr_handle, + test_interrupt_callback_1, (void *)-1); + + test_intr_handle = intr_handles[TEST_INTERRUPT_HANDLE_VALID_ALARM]; + rte_intr_callback_unregister(&test_intr_handle, + test_interrupt_callback, (void *)-1); + rte_intr_callback_unregister(&test_intr_handle, + test_interrupt_callback_1, (void *)-1); + + rte_delay_ms(2 * TEST_INTERRUPT_CHECK_INTERVAL); + /* deinit */ + test_interrupt_deinit(); + + return ret; +} + +REGISTER_TEST_COMMAND(interrupt_autotest, test_interrupt); diff --git a/dpdk/test/test/test_kni.c b/dpdk/test/test/test_kni.c new file mode 100644 index 00000000..b9567275 --- /dev/null +++ b/dpdk/test/test/test_kni.c @@ -0,0 +1,637 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <stdint.h> +#include <unistd.h> +#include <string.h> +#include <sys/wait.h> + +#include "test.h" + +#include <rte_string_fns.h> +#include <rte_mempool.h> +#include <rte_ethdev.h> +#include <rte_bus_pci.h> +#include <rte_cycles.h> +#include <rte_kni.h> + +#define NB_MBUF 8192 +#define MAX_PACKET_SZ 2048 +#define MBUF_DATA_SZ (MAX_PACKET_SZ + RTE_PKTMBUF_HEADROOM) +#define PKT_BURST_SZ 32 +#define MEMPOOL_CACHE_SZ PKT_BURST_SZ +#define SOCKET 0 +#define NB_RXD 128 +#define NB_TXD 512 +#define KNI_TIMEOUT_MS 5000 /* ms */ + +#define IFCONFIG "/sbin/ifconfig " +#define TEST_KNI_PORT "test_kni_port" +#define KNI_TEST_MAX_PORTS 4 +/* The threshold number of mbufs to be transmitted or received. */ +#define KNI_NUM_MBUF_THRESHOLD 100 +static int kni_pkt_mtu = 0; + +struct test_kni_stats { + volatile uint64_t ingress; + volatile uint64_t egress; +}; + +static const struct rte_eth_rxconf rx_conf = { + .rx_thresh = { + .pthresh = 8, + .hthresh = 8, + .wthresh = 4, + }, + .rx_free_thresh = 0, +}; + +static const struct rte_eth_txconf tx_conf = { + .tx_thresh = { + .pthresh = 36, + .hthresh = 0, + .wthresh = 0, + }, + .tx_free_thresh = 0, + .tx_rs_thresh = 0, +}; + +static const struct rte_eth_conf port_conf = { + .rxmode = { + .header_split = 0, + .hw_ip_checksum = 0, + .hw_vlan_filter = 0, + .jumbo_frame = 0, + .hw_strip_crc = 1, + }, + .txmode = { + .mq_mode = ETH_DCB_NONE, + }, +}; + +static struct rte_kni_ops kni_ops = { + .change_mtu = NULL, + .config_network_if = NULL, +}; + +static unsigned lcore_master, lcore_ingress, lcore_egress; +static struct rte_kni *test_kni_ctx; +static struct test_kni_stats stats; + +static volatile uint32_t test_kni_processing_flag; + +static struct rte_mempool * +test_kni_create_mempool(void) +{ + struct rte_mempool * mp; + + mp = rte_mempool_lookup("kni_mempool"); + if (!mp) + mp = rte_pktmbuf_pool_create("kni_mempool", + NB_MBUF, + MEMPOOL_CACHE_SZ, 0, MBUF_DATA_SZ, + SOCKET); + + return mp; +} + +static struct rte_mempool * +test_kni_lookup_mempool(void) +{ + return rte_mempool_lookup("kni_mempool"); +} +/* Callback for request of changing MTU */ +static int +kni_change_mtu(uint16_t port_id, unsigned int new_mtu) +{ + printf("Change MTU of port %d to %u\n", port_id, new_mtu); + kni_pkt_mtu = new_mtu; + printf("Change MTU of port %d to %i successfully.\n", + port_id, kni_pkt_mtu); + return 0; +} +/** + * This loop fully tests the basic functions of KNI. e.g. transmitting, + * receiving to, from kernel space, and kernel requests. + * + * This is the loop to transmit/receive mbufs to/from kernel interface with + * supported by KNI kernel module. The ingress lcore will allocate mbufs and + * transmit them to kernel space; while the egress lcore will receive the mbufs + * from kernel space and free them. + * On the master lcore, several commands will be run to check handling the + * kernel requests. And it will finally set the flag to exit the KNI + * transmitting/receiving to/from the kernel space. + * + * Note: To support this testing, the KNI kernel module needs to be insmodded + * in one of its loopback modes. + */ +static int +test_kni_loop(__rte_unused void *arg) +{ + int ret = 0; + unsigned nb_rx, nb_tx, num, i; + const unsigned lcore_id = rte_lcore_id(); + struct rte_mbuf *pkts_burst[PKT_BURST_SZ]; + + if (lcore_id == lcore_master) { + rte_delay_ms(KNI_TIMEOUT_MS); + /* tests of handling kernel request */ + if (system(IFCONFIG TEST_KNI_PORT" up") == -1) + ret = -1; + if (system(IFCONFIG TEST_KNI_PORT" mtu 1400") == -1) + ret = -1; + if (system(IFCONFIG TEST_KNI_PORT" down") == -1) + ret = -1; + rte_delay_ms(KNI_TIMEOUT_MS); + test_kni_processing_flag = 1; + } else if (lcore_id == lcore_ingress) { + struct rte_mempool *mp = test_kni_lookup_mempool(); + + if (mp == NULL) + return -1; + + while (1) { + if (test_kni_processing_flag) + break; + + for (nb_rx = 0; nb_rx < PKT_BURST_SZ; nb_rx++) { + pkts_burst[nb_rx] = rte_pktmbuf_alloc(mp); + if (!pkts_burst[nb_rx]) + break; + } + + num = rte_kni_tx_burst(test_kni_ctx, pkts_burst, + nb_rx); + stats.ingress += num; + rte_kni_handle_request(test_kni_ctx); + if (num < nb_rx) { + for (i = num; i < nb_rx; i++) { + rte_pktmbuf_free(pkts_burst[i]); + } + } + rte_delay_ms(10); + } + } else if (lcore_id == lcore_egress) { + while (1) { + if (test_kni_processing_flag) + break; + num = rte_kni_rx_burst(test_kni_ctx, pkts_burst, + PKT_BURST_SZ); + stats.egress += num; + for (nb_tx = 0; nb_tx < num; nb_tx++) + rte_pktmbuf_free(pkts_burst[nb_tx]); + rte_delay_ms(10); + } + } + + return ret; +} + +static int +test_kni_allocate_lcores(void) +{ + unsigned i, count = 0; + + lcore_master = rte_get_master_lcore(); + printf("master lcore: %u\n", lcore_master); + for (i = 0; i < RTE_MAX_LCORE; i++) { + if (count >=2 ) + break; + if (rte_lcore_is_enabled(i) && i != lcore_master) { + count ++; + if (count == 1) + lcore_ingress = i; + else if (count == 2) + lcore_egress = i; + } + } + printf("count: %u\n", count); + + return count == 2 ? 0 : -1; +} + +static int +test_kni_register_handler_mp(void) +{ +#define TEST_KNI_HANDLE_REQ_COUNT 10 /* 5s */ +#define TEST_KNI_HANDLE_REQ_INTERVAL 500 /* ms */ +#define TEST_KNI_MTU 1450 +#define TEST_KNI_MTU_STR " 1450" + int pid; + + pid = fork(); + if (pid < 0) { + printf("Failed to fork a process\n"); + return -1; + } else if (pid == 0) { + int i; + struct rte_kni *kni = rte_kni_get(TEST_KNI_PORT); + struct rte_kni_ops ops = { + .change_mtu = kni_change_mtu, + .config_network_if = NULL, + }; + + if (!kni) { + printf("Failed to get KNI named %s\n", TEST_KNI_PORT); + exit(-1); + } + + kni_pkt_mtu = 0; + + /* Check with the invalid parameters */ + if (rte_kni_register_handlers(kni, NULL) == 0) { + printf("Unexpectedly register successuflly " + "with NULL ops pointer\n"); + exit(-1); + } + if (rte_kni_register_handlers(NULL, &ops) == 0) { + printf("Unexpectedly register successfully " + "to NULL KNI device pointer\n"); + exit(-1); + } + + if (rte_kni_register_handlers(kni, &ops)) { + printf("Fail to register ops\n"); + exit(-1); + } + + /* Check registering again after it has been registered */ + if (rte_kni_register_handlers(kni, &ops) == 0) { + printf("Unexpectedly register successfully after " + "it has already been registered\n"); + exit(-1); + } + + /** + * Handle the request of setting MTU, + * with registered handlers. + */ + for (i = 0; i < TEST_KNI_HANDLE_REQ_COUNT; i++) { + rte_kni_handle_request(kni); + if (kni_pkt_mtu == TEST_KNI_MTU) + break; + rte_delay_ms(TEST_KNI_HANDLE_REQ_INTERVAL); + } + if (i >= TEST_KNI_HANDLE_REQ_COUNT) { + printf("MTU has not been set\n"); + exit(-1); + } + + kni_pkt_mtu = 0; + if (rte_kni_unregister_handlers(kni) < 0) { + printf("Fail to unregister ops\n"); + exit(-1); + } + + /* Check with invalid parameter */ + if (rte_kni_unregister_handlers(NULL) == 0) { + exit(-1); + } + + /** + * Handle the request of setting MTU, + * without registered handlers. + */ + for (i = 0; i < TEST_KNI_HANDLE_REQ_COUNT; i++) { + rte_kni_handle_request(kni); + if (kni_pkt_mtu != 0) + break; + rte_delay_ms(TEST_KNI_HANDLE_REQ_INTERVAL); + } + if (kni_pkt_mtu != 0) { + printf("MTU shouldn't be set\n"); + exit(-1); + } + + exit(0); + } else { + int p_ret, status; + + rte_delay_ms(1000); + if (system(IFCONFIG TEST_KNI_PORT " mtu" TEST_KNI_MTU_STR) + == -1) + return -1; + + rte_delay_ms(1000); + if (system(IFCONFIG TEST_KNI_PORT " mtu" TEST_KNI_MTU_STR) + == -1) + return -1; + + p_ret = wait(&status); + if (!WIFEXITED(status)) { + printf("Child process (%d) exit abnormally\n", p_ret); + return -1; + } + if (WEXITSTATUS(status) != 0) { + printf("Child process exit with failure\n"); + return -1; + } + } + + return 0; +} + +static int +test_kni_processing(uint16_t port_id, struct rte_mempool *mp) +{ + int ret = 0; + unsigned i; + struct rte_kni *kni; + struct rte_kni_conf conf; + struct rte_eth_dev_info info; + struct rte_kni_ops ops; + + if (!mp) + return -1; + + memset(&conf, 0, sizeof(conf)); + memset(&info, 0, sizeof(info)); + memset(&ops, 0, sizeof(ops)); + + rte_eth_dev_info_get(port_id, &info); + conf.addr = info.pci_dev->addr; + conf.id = info.pci_dev->id; + snprintf(conf.name, sizeof(conf.name), TEST_KNI_PORT); + + /* core id 1 configured for kernel thread */ + conf.core_id = 1; + conf.force_bind = 1; + conf.mbuf_size = MAX_PACKET_SZ; + conf.group_id = port_id; + + ops = kni_ops; + ops.port_id = port_id; + + /* basic test of kni processing */ + kni = rte_kni_alloc(mp, &conf, &ops); + if (!kni) { + printf("fail to create kni\n"); + return -1; + } + + test_kni_ctx = kni; + test_kni_processing_flag = 0; + stats.ingress = 0; + stats.egress = 0; + + /** + * Check multiple processes support on + * registerring/unregisterring handlers. + */ + if (test_kni_register_handler_mp() < 0) { + printf("fail to check multiple process support\n"); + ret = -1; + goto fail_kni; + } + + rte_eal_mp_remote_launch(test_kni_loop, NULL, CALL_MASTER); + RTE_LCORE_FOREACH_SLAVE(i) { + if (rte_eal_wait_lcore(i) < 0) { + ret = -1; + goto fail_kni; + } + } + /** + * Check if the number of mbufs received from kernel space is equal + * to that of transmitted to kernel space + */ + if (stats.ingress < KNI_NUM_MBUF_THRESHOLD || + stats.egress < KNI_NUM_MBUF_THRESHOLD) { + printf("The ingress/egress number should not be " + "less than %u\n", (unsigned)KNI_NUM_MBUF_THRESHOLD); + ret = -1; + goto fail_kni; + } + + if (rte_kni_release(kni) < 0) { + printf("fail to release kni\n"); + return -1; + } + test_kni_ctx = NULL; + + /* test of releasing a released kni device */ + if (rte_kni_release(kni) == 0) { + printf("should not release a released kni device\n"); + return -1; + } + + /* test of reusing memzone */ + kni = rte_kni_alloc(mp, &conf, &ops); + if (!kni) { + printf("fail to create kni\n"); + return -1; + } + + /* Release the kni for following testing */ + if (rte_kni_release(kni) < 0) { + printf("fail to release kni\n"); + return -1; + } + + return ret; +fail_kni: + if (rte_kni_release(kni) < 0) { + printf("fail to release kni\n"); + ret = -1; + } + + return ret; +} + +static int +test_kni(void) +{ + int ret = -1; + uint16_t nb_ports, port_id; + struct rte_kni *kni; + struct rte_mempool *mp; + struct rte_kni_conf conf; + struct rte_eth_dev_info info; + struct rte_kni_ops ops; + + /* Initialize KNI subsytem */ + rte_kni_init(KNI_TEST_MAX_PORTS); + + if (test_kni_allocate_lcores() < 0) { + printf("No enough lcores for kni processing\n"); + return -1; + } + + mp = test_kni_create_mempool(); + if (!mp) { + printf("fail to create mempool for kni\n"); + return -1; + } + + nb_ports = rte_eth_dev_count(); + if (nb_ports == 0) { + printf("no supported nic port found\n"); + return -1; + } + + /* configuring port 0 for the test is enough */ + port_id = 0; + ret = rte_eth_dev_configure(port_id, 1, 1, &port_conf); + if (ret < 0) { + printf("fail to configure port %d\n", port_id); + return -1; + } + + ret = rte_eth_rx_queue_setup(port_id, 0, NB_RXD, SOCKET, &rx_conf, mp); + if (ret < 0) { + printf("fail to setup rx queue for port %d\n", port_id); + return -1; + } + + ret = rte_eth_tx_queue_setup(port_id, 0, NB_TXD, SOCKET, &tx_conf); + if (ret < 0) { + printf("fail to setup tx queue for port %d\n", port_id); + return -1; + } + + ret = rte_eth_dev_start(port_id); + if (ret < 0) { + printf("fail to start port %d\n", port_id); + return -1; + } + rte_eth_promiscuous_enable(port_id); + + /* basic test of kni processing */ + ret = test_kni_processing(port_id, mp); + if (ret < 0) + goto fail; + + /* test of allocating KNI with NULL mempool pointer */ + memset(&info, 0, sizeof(info)); + memset(&conf, 0, sizeof(conf)); + memset(&ops, 0, sizeof(ops)); + rte_eth_dev_info_get(port_id, &info); + conf.addr = info.pci_dev->addr; + conf.id = info.pci_dev->id; + conf.group_id = port_id; + conf.mbuf_size = MAX_PACKET_SZ; + + ops = kni_ops; + ops.port_id = port_id; + kni = rte_kni_alloc(NULL, &conf, &ops); + if (kni) { + ret = -1; + printf("unexpectedly creates kni successfully with NULL " + "mempool pointer\n"); + goto fail; + } + + /* test of allocating KNI without configurations */ + kni = rte_kni_alloc(mp, NULL, NULL); + if (kni) { + ret = -1; + printf("Unexpectedly allocate KNI device successfully " + "without configurations\n"); + goto fail; + } + + /* test of allocating KNI without a name */ + memset(&conf, 0, sizeof(conf)); + memset(&info, 0, sizeof(info)); + memset(&ops, 0, sizeof(ops)); + rte_eth_dev_info_get(port_id, &info); + conf.addr = info.pci_dev->addr; + conf.id = info.pci_dev->id; + conf.group_id = port_id; + conf.mbuf_size = MAX_PACKET_SZ; + + ops = kni_ops; + ops.port_id = port_id; + kni = rte_kni_alloc(mp, &conf, &ops); + if (kni) { + ret = -1; + printf("Unexpectedly allocate a KNI device successfully " + "without a name\n"); + goto fail; + } + + /* test of releasing NULL kni context */ + ret = rte_kni_release(NULL); + if (ret == 0) { + ret = -1; + printf("unexpectedly release kni successfully\n"); + goto fail; + } + + /* test of handling request on NULL device pointer */ + ret = rte_kni_handle_request(NULL); + if (ret == 0) { + ret = -1; + printf("Unexpectedly handle request on NULL device pointer\n"); + goto fail; + } + + /* test of getting KNI device with pointer to NULL */ + kni = rte_kni_get(NULL); + if (kni) { + ret = -1; + printf("Unexpectedly get a KNI device with " + "NULL name pointer\n"); + goto fail; + } + + /* test of getting KNI device with an zero length name string */ + memset(&conf, 0, sizeof(conf)); + kni = rte_kni_get(conf.name); + if (kni) { + ret = -1; + printf("Unexpectedly get a KNI device with " + "zero length name string\n"); + goto fail; + } + + /* test of getting KNI device with an invalid string name */ + memset(&conf, 0, sizeof(conf)); + snprintf(conf.name, sizeof(conf.name), "testing"); + kni = rte_kni_get(conf.name); + if (kni) { + ret = -1; + printf("Unexpectedly get a KNI device with " + "a never used name string\n"); + goto fail; + } + ret = 0; + +fail: + rte_eth_dev_stop(port_id); + + return ret; +} + +REGISTER_TEST_COMMAND(kni_autotest, test_kni); diff --git a/dpdk/test/test/test_kvargs.c b/dpdk/test/test/test_kvargs.c new file mode 100644 index 00000000..4d9e805b --- /dev/null +++ b/dpdk/test/test/test_kvargs.c @@ -0,0 +1,235 @@ +/* + * Copyright 2014 6WIND S.A. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * - Neither the name of 6WIND S.A. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdlib.h> +#include <stdio.h> +#include <string.h> + +#include <rte_common.h> +#include <rte_kvargs.h> + +#include "test.h" + +/* incrementd in handler, to check it is properly called once per + * key/value association */ +static unsigned count; + +/* this handler increment the "count" variable at each call and check + * that the key is "check" and the value is "value%d" */ +static int check_handler(const char *key, const char *value, + __rte_unused void *opaque) +{ + char buf[16]; + + /* we check that the value is "check" */ + if (strcmp(key, "check")) + return -1; + + /* we check that the value is "value$(count)" */ + snprintf(buf, sizeof(buf), "value%d", count); + if (strncmp(buf, value, sizeof(buf))) + return -1; + + count ++; + return 0; +} + +/* test a valid case */ +static int test_valid_kvargs(void) +{ + struct rte_kvargs *kvlist; + const char *args; + const char *valid_keys_list[] = { "foo", "check", NULL }; + const char **valid_keys; + + /* empty args is valid */ + args = ""; + valid_keys = NULL; + kvlist = rte_kvargs_parse(args, valid_keys); + if (kvlist == NULL) { + printf("rte_kvargs_parse() error"); + goto fail; + } + rte_kvargs_free(kvlist); + + /* first test without valid_keys */ + args = "foo=1234,check=value0,check=value1"; + valid_keys = NULL; + kvlist = rte_kvargs_parse(args, valid_keys); + if (kvlist == NULL) { + printf("rte_kvargs_parse() error"); + goto fail; + } + /* call check_handler() for all entries with key="check" */ + count = 0; + if (rte_kvargs_process(kvlist, "check", check_handler, NULL) < 0) { + printf("rte_kvargs_process() error\n"); + rte_kvargs_free(kvlist); + goto fail; + } + if (count != 2) { + printf("invalid count value %d after rte_kvargs_process(check)\n", + count); + rte_kvargs_free(kvlist); + goto fail; + } + count = 0; + /* call check_handler() for all entries with key="unexistant_key" */ + if (rte_kvargs_process(kvlist, "unexistant_key", check_handler, NULL) < 0) { + printf("rte_kvargs_process() error\n"); + rte_kvargs_free(kvlist); + goto fail; + } + if (count != 0) { + printf("invalid count value %d after rte_kvargs_process(unexistant_key)\n", + count); + rte_kvargs_free(kvlist); + goto fail; + } + /* count all entries with key="foo" */ + count = rte_kvargs_count(kvlist, "foo"); + if (count != 1) { + printf("invalid count value %d after rte_kvargs_count(foo)\n", + count); + rte_kvargs_free(kvlist); + goto fail; + } + /* count all entries */ + count = rte_kvargs_count(kvlist, NULL); + if (count != 3) { + printf("invalid count value %d after rte_kvargs_count(NULL)\n", + count); + rte_kvargs_free(kvlist); + goto fail; + } + /* count all entries with key="unexistant_key" */ + count = rte_kvargs_count(kvlist, "unexistant_key"); + if (count != 0) { + printf("invalid count value %d after rte_kvargs_count(unexistant_key)\n", + count); + rte_kvargs_free(kvlist); + goto fail; + } + rte_kvargs_free(kvlist); + + /* second test using valid_keys */ + args = "foo=droids,check=value0,check=value1,check=wrong_value"; + valid_keys = valid_keys_list; + kvlist = rte_kvargs_parse(args, valid_keys); + if (kvlist == NULL) { + printf("rte_kvargs_parse() error"); + goto fail; + } + /* call check_handler() on all entries with key="check", it + * should fail as the value is not recognized by the handler */ + if (rte_kvargs_process(kvlist, "check", check_handler, NULL) == 0) { + printf("rte_kvargs_process() is success bu should not\n"); + rte_kvargs_free(kvlist); + goto fail; + } + count = rte_kvargs_count(kvlist, "check"); + if (count != 3) { + printf("invalid count value %d after rte_kvargs_count(check)\n", + count); + rte_kvargs_free(kvlist); + goto fail; + } + rte_kvargs_free(kvlist); + + return 0; + + fail: + printf("while processing <%s>", args); + if (valid_keys != NULL && *valid_keys != NULL) { + printf(" using valid_keys=<%s", *valid_keys); + while (*(++valid_keys) != NULL) + printf(",%s", *valid_keys); + printf(">"); + } + printf("\n"); + return -1; +} + +/* test several error cases */ +static int test_invalid_kvargs(void) +{ + struct rte_kvargs *kvlist; + /* list of argument that should fail */ + const char *args_list[] = { + "wrong-key=x", /* key not in valid_keys_list */ + "foo=1,foo=", /* empty value */ + "foo=1,,foo=2", /* empty key/value */ + "foo=1,foo", /* no value */ + "foo=1,=2", /* no key */ + ",=", /* also test with a smiley */ + NULL }; + const char **args; + const char *valid_keys_list[] = { "foo", "check", NULL }; + const char **valid_keys = valid_keys_list; + + for (args = args_list; *args != NULL; args++) { + + kvlist = rte_kvargs_parse(*args, valid_keys); + if (kvlist != NULL) { + printf("rte_kvargs_parse() returned 0 (but should not)\n"); + rte_kvargs_free(kvlist); + goto fail; + } + return 0; + } + + fail: + printf("while processing <%s>", *args); + if (valid_keys != NULL && *valid_keys != NULL) { + printf(" using valid_keys=<%s", *valid_keys); + while (*(++valid_keys) != NULL) + printf(",%s", *valid_keys); + printf(">"); + } + printf("\n"); + return -1; +} + +static int +test_kvargs(void) +{ + printf("== test valid case ==\n"); + if (test_valid_kvargs() < 0) + return -1; + printf("== test invalid case ==\n"); + if (test_invalid_kvargs() < 0) + return -1; + return 0; +} + +REGISTER_TEST_COMMAND(kvargs_autotest, test_kvargs); diff --git a/dpdk/test/test/test_link_bonding.c b/dpdk/test/test/test_link_bonding.c new file mode 100644 index 00000000..c6e3a725 --- /dev/null +++ b/dpdk/test/test/test_link_bonding.c @@ -0,0 +1,4963 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "unistd.h" +#include <string.h> +#include <stdarg.h> +#include <stdio.h> +#include <stdlib.h> +#include <stdint.h> +#include <inttypes.h> +#include <errno.h> +#include <sys/queue.h> +#include <sys/time.h> +#include <rte_cycles.h> +#include <rte_byteorder.h> +#include <rte_common.h> +#include <rte_debug.h> +#include <rte_ethdev.h> +#include <rte_log.h> +#include <rte_lcore.h> +#include <rte_memory.h> +#include <rte_string_fns.h> +#include <rte_eth_bond.h> + +#include "virtual_pmd.h" +#include "packet_burst_generator.h" + +#include "test.h" + +#define TEST_MAX_NUMBER_OF_PORTS (6) + +#define RX_RING_SIZE 128 +#define RX_FREE_THRESH 32 +#define RX_PTHRESH 8 +#define RX_HTHRESH 8 +#define RX_WTHRESH 0 + +#define TX_RING_SIZE 512 +#define TX_FREE_THRESH 32 +#define TX_PTHRESH 32 +#define TX_HTHRESH 0 +#define TX_WTHRESH 0 +#define TX_RSBIT_THRESH 32 +#define TX_Q_FLAGS (ETH_TXQ_FLAGS_NOMULTSEGS | ETH_TXQ_FLAGS_NOVLANOFFL |\ + ETH_TXQ_FLAGS_NOXSUMSCTP | ETH_TXQ_FLAGS_NOXSUMUDP | \ + ETH_TXQ_FLAGS_NOXSUMTCP) + +#define MBUF_CACHE_SIZE (250) +#define BURST_SIZE (32) + +#define RTE_TEST_RX_DESC_MAX (2048) +#define RTE_TEST_TX_DESC_MAX (2048) +#define MAX_PKT_BURST (512) +#define DEF_PKT_BURST (16) + +#define BONDED_DEV_NAME ("net_bonding_ut") + +#define INVALID_SOCKET_ID (-1) +#define INVALID_PORT_ID (-1) +#define INVALID_BONDING_MODE (-1) + + +uint8_t slave_mac[] = {0x00, 0xFF, 0x00, 0xFF, 0x00, 0x00 }; +uint8_t bonded_mac[] = {0xAA, 0xFF, 0xAA, 0xFF, 0xAA, 0xFF }; + +struct link_bonding_unittest_params { + int16_t bonded_port_id; + int16_t slave_port_ids[TEST_MAX_NUMBER_OF_PORTS]; + uint16_t bonded_slave_count; + uint8_t bonding_mode; + + uint16_t nb_rx_q; + uint16_t nb_tx_q; + + struct rte_mempool *mbuf_pool; + + struct ether_addr *default_slave_mac; + struct ether_addr *default_bonded_mac; + + /* Packet Headers */ + struct ether_hdr *pkt_eth_hdr; + struct ipv4_hdr *pkt_ipv4_hdr; + struct ipv6_hdr *pkt_ipv6_hdr; + struct udp_hdr *pkt_udp_hdr; + +}; + +static struct ipv4_hdr pkt_ipv4_hdr; +static struct ipv6_hdr pkt_ipv6_hdr; +static struct udp_hdr pkt_udp_hdr; + +static struct link_bonding_unittest_params default_params = { + .bonded_port_id = -1, + .slave_port_ids = { -1 }, + .bonded_slave_count = 0, + .bonding_mode = BONDING_MODE_ROUND_ROBIN, + + .nb_rx_q = 1, + .nb_tx_q = 1, + + .mbuf_pool = NULL, + + .default_slave_mac = (struct ether_addr *)slave_mac, + .default_bonded_mac = (struct ether_addr *)bonded_mac, + + .pkt_eth_hdr = NULL, + .pkt_ipv4_hdr = &pkt_ipv4_hdr, + .pkt_ipv6_hdr = &pkt_ipv6_hdr, + .pkt_udp_hdr = &pkt_udp_hdr + +}; + +static struct link_bonding_unittest_params *test_params = &default_params; + +static uint8_t src_mac[] = { 0xFF, 0xAA, 0xFF, 0xAA, 0xFF, 0xAA }; +static uint8_t dst_mac_0[] = { 0xFF, 0xAA, 0xFF, 0xAA, 0xFF, 0xAA }; +static uint8_t dst_mac_1[] = { 0xFF, 0xAA, 0xFF, 0xAA, 0xFF, 0xAB }; + +static uint32_t src_addr = IPV4_ADDR(192, 168, 1, 98); +static uint32_t dst_addr_0 = IPV4_ADDR(192, 168, 1, 98); +static uint32_t dst_addr_1 = IPV4_ADDR(193, 166, 10, 97); + +static uint8_t src_ipv6_addr[] = { 0xFF, 0xAA, 0xFF, 0xAA, 0xFF, 0xAA, 0xFF, + 0xAA, 0xFF, 0xAA, 0xFF, 0xAA, 0xFF, 0xAA , 0xFF, 0xAA }; +static uint8_t dst_ipv6_addr_0[] = { 0xFF, 0xAA, 0xFF, 0xAA, 0xFF, 0xAA, 0xFF, + 0xAA, 0xFF, 0xAA, 0xFF, 0xAA , 0xFF, 0xAA, 0xFF, 0xAA }; +static uint8_t dst_ipv6_addr_1[] = { 0xFF, 0xAA, 0xFF, 0xAA, 0xFF, 0xAA, 0xFF, + 0xAA, 0xFF, 0xAA, 0xFF, 0xAA , 0xFF, 0xAA , 0xFF, 0xAB }; + +static uint16_t src_port = 1024; +static uint16_t dst_port_0 = 1024; +static uint16_t dst_port_1 = 2024; + +static uint16_t vlan_id = 0x100; + +struct rte_eth_rxmode rx_mode = { + .max_rx_pkt_len = ETHER_MAX_LEN, /**< Default maximum frame length. */ + .split_hdr_size = 0, + .header_split = 0, /**< Header Split disabled. */ + .hw_ip_checksum = 0, /**< IP checksum offload disabled. */ + .hw_vlan_filter = 1, /**< VLAN filtering enabled. */ + .hw_vlan_strip = 1, /**< VLAN strip enabled. */ + .hw_vlan_extend = 0, /**< Extended VLAN disabled. */ + .jumbo_frame = 0, /**< Jumbo Frame Support disabled. */ + .hw_strip_crc = 1, /**< CRC stripping by hardware enabled. */ +}; + +struct rte_fdir_conf fdir_conf = { + .mode = RTE_FDIR_MODE_NONE, + .pballoc = RTE_FDIR_PBALLOC_64K, + .status = RTE_FDIR_REPORT_STATUS, + .drop_queue = 127, +}; + +static struct rte_eth_conf default_pmd_conf = { + .rxmode = { + .mq_mode = ETH_MQ_RX_NONE, + .max_rx_pkt_len = ETHER_MAX_LEN, + .split_hdr_size = 0, + .header_split = 0, /**< Header Split disabled */ + .hw_ip_checksum = 0, /**< IP checksum offload enabled */ + .hw_vlan_filter = 0, /**< VLAN filtering disabled */ + .jumbo_frame = 0, /**< Jumbo Frame Support disabled */ + .hw_strip_crc = 1, /**< CRC stripped by hardware */ + }, + .txmode = { + .mq_mode = ETH_MQ_TX_NONE, + }, + .lpbk_mode = 0, +}; + +static const struct rte_eth_rxconf rx_conf_default = { + .rx_thresh = { + .pthresh = RX_PTHRESH, + .hthresh = RX_HTHRESH, + .wthresh = RX_WTHRESH, + }, + .rx_free_thresh = RX_FREE_THRESH, + .rx_drop_en = 0, +}; + +static struct rte_eth_txconf tx_conf_default = { + .tx_thresh = { + .pthresh = TX_PTHRESH, + .hthresh = TX_HTHRESH, + .wthresh = TX_WTHRESH, + }, + .tx_free_thresh = TX_FREE_THRESH, + .tx_rs_thresh = TX_RSBIT_THRESH, + .txq_flags = TX_Q_FLAGS + +}; + +static void free_virtualpmd_tx_queue(void); + + + +static int +configure_ethdev(uint16_t port_id, uint8_t start, uint8_t en_isr) +{ + int q_id; + + if (en_isr) + default_pmd_conf.intr_conf.lsc = 1; + else + default_pmd_conf.intr_conf.lsc = 0; + + TEST_ASSERT_SUCCESS(rte_eth_dev_configure(port_id, test_params->nb_rx_q, + test_params->nb_tx_q, &default_pmd_conf), + "rte_eth_dev_configure for port %d failed", port_id); + + for (q_id = 0; q_id < test_params->nb_rx_q; q_id++) + TEST_ASSERT_SUCCESS(rte_eth_rx_queue_setup(port_id, q_id, RX_RING_SIZE, + rte_eth_dev_socket_id(port_id), &rx_conf_default, + test_params->mbuf_pool) , + "rte_eth_rx_queue_setup for port %d failed", port_id); + + for (q_id = 0; q_id < test_params->nb_tx_q; q_id++) + TEST_ASSERT_SUCCESS(rte_eth_tx_queue_setup(port_id, q_id, TX_RING_SIZE, + rte_eth_dev_socket_id(port_id), &tx_conf_default), + "rte_eth_tx_queue_setup for port %d failed", port_id); + + if (start) + TEST_ASSERT_SUCCESS(rte_eth_dev_start(port_id), + "rte_eth_dev_start for port %d failed", port_id); + + return 0; +} + +static int slaves_initialized; + +static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t cvar = PTHREAD_COND_INITIALIZER; + + +static int +test_setup(void) +{ + int i, nb_mbuf_per_pool; + struct ether_addr *mac_addr = (struct ether_addr *)slave_mac; + + /* Allocate ethernet packet header with space for VLAN header */ + if (test_params->pkt_eth_hdr == NULL) { + test_params->pkt_eth_hdr = malloc(sizeof(struct ether_hdr) + + sizeof(struct vlan_hdr)); + + TEST_ASSERT_NOT_NULL(test_params->pkt_eth_hdr, + "Ethernet header struct allocation failed!"); + } + + nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX + DEF_PKT_BURST + + RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST; + if (test_params->mbuf_pool == NULL) { + test_params->mbuf_pool = rte_pktmbuf_pool_create("MBUF_POOL", + nb_mbuf_per_pool, MBUF_CACHE_SIZE, 0, + RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id()); + TEST_ASSERT_NOT_NULL(test_params->mbuf_pool, + "rte_mempool_create failed"); + } + + /* Create / Initialize virtual eth devs */ + if (!slaves_initialized) { + for (i = 0; i < TEST_MAX_NUMBER_OF_PORTS; i++) { + char pmd_name[RTE_ETH_NAME_MAX_LEN]; + + mac_addr->addr_bytes[ETHER_ADDR_LEN-1] = i; + + snprintf(pmd_name, RTE_ETH_NAME_MAX_LEN, "eth_virt_%d", i); + + test_params->slave_port_ids[i] = virtual_ethdev_create(pmd_name, + mac_addr, rte_socket_id(), 1); + TEST_ASSERT(test_params->slave_port_ids[i] >= 0, + "Failed to create virtual virtual ethdev %s", pmd_name); + + TEST_ASSERT_SUCCESS(configure_ethdev( + test_params->slave_port_ids[i], 1, 0), + "Failed to configure virtual ethdev %s", pmd_name); + } + slaves_initialized = 1; + } + + return 0; +} + +static int +test_create_bonded_device(void) +{ + int current_slave_count; + + uint16_t slaves[RTE_MAX_ETHPORTS]; + + /* Don't try to recreate bonded device if re-running test suite*/ + if (test_params->bonded_port_id == -1) { + test_params->bonded_port_id = rte_eth_bond_create(BONDED_DEV_NAME, + test_params->bonding_mode, rte_socket_id()); + + TEST_ASSERT(test_params->bonded_port_id >= 0, + "Failed to create bonded ethdev %s", BONDED_DEV_NAME); + + TEST_ASSERT_SUCCESS(configure_ethdev(test_params->bonded_port_id, 0, 0), + "Failed to configure bonded ethdev %s", BONDED_DEV_NAME); + } + + TEST_ASSERT_SUCCESS(rte_eth_bond_mode_set(test_params->bonded_port_id, + test_params->bonding_mode), "Failed to set ethdev %d to mode %d", + test_params->bonded_port_id, test_params->bonding_mode); + + current_slave_count = rte_eth_bond_slaves_get(test_params->bonded_port_id, + slaves, RTE_MAX_ETHPORTS); + + TEST_ASSERT_EQUAL(current_slave_count, 0, + "Number of slaves %d is great than expected %d.", + current_slave_count, 0); + + current_slave_count = rte_eth_bond_active_slaves_get( + test_params->bonded_port_id, slaves, RTE_MAX_ETHPORTS); + + TEST_ASSERT_EQUAL(current_slave_count, 0, + "Number of active slaves %d is great than expected %d.", + current_slave_count, 0); + + return 0; +} + + +static int +test_create_bonded_device_with_invalid_params(void) +{ + int port_id; + + test_params->bonding_mode = BONDING_MODE_ROUND_ROBIN; + + /* Invalid name */ + port_id = rte_eth_bond_create(NULL, test_params->bonding_mode, + rte_socket_id()); + TEST_ASSERT(port_id < 0, "Created bonded device unexpectedly"); + + test_params->bonding_mode = INVALID_BONDING_MODE; + + /* Invalid bonding mode */ + port_id = rte_eth_bond_create(BONDED_DEV_NAME, test_params->bonding_mode, + rte_socket_id()); + TEST_ASSERT(port_id < 0, "Created bonded device unexpectedly."); + + test_params->bonding_mode = BONDING_MODE_ROUND_ROBIN; + + /* Invalid socket id */ + port_id = rte_eth_bond_create(BONDED_DEV_NAME, test_params->bonding_mode, + INVALID_SOCKET_ID); + TEST_ASSERT(port_id < 0, "Created bonded device unexpectedly."); + + return 0; +} + +static int +test_add_slave_to_bonded_device(void) +{ + int current_slave_count; + + uint16_t slaves[RTE_MAX_ETHPORTS]; + + TEST_ASSERT_SUCCESS(rte_eth_bond_slave_add(test_params->bonded_port_id, + test_params->slave_port_ids[test_params->bonded_slave_count]), + "Failed to add slave (%d) to bonded port (%d).", + test_params->slave_port_ids[test_params->bonded_slave_count], + test_params->bonded_port_id); + + current_slave_count = rte_eth_bond_slaves_get(test_params->bonded_port_id, + slaves, RTE_MAX_ETHPORTS); + TEST_ASSERT_EQUAL(current_slave_count, test_params->bonded_slave_count + 1, + "Number of slaves (%d) is greater than expected (%d).", + current_slave_count, test_params->bonded_slave_count + 1); + + current_slave_count = rte_eth_bond_active_slaves_get( + test_params->bonded_port_id, slaves, RTE_MAX_ETHPORTS); + TEST_ASSERT_EQUAL(current_slave_count, 0, + "Number of active slaves (%d) is not as expected (%d).\n", + current_slave_count, 0); + + test_params->bonded_slave_count++; + + return 0; +} + +static int +test_add_slave_to_invalid_bonded_device(void) +{ + /* Invalid port ID */ + TEST_ASSERT_FAIL(rte_eth_bond_slave_add(test_params->bonded_port_id + 5, + test_params->slave_port_ids[test_params->bonded_slave_count]), + "Expected call to failed as invalid port specified."); + + /* Non bonded device */ + TEST_ASSERT_FAIL(rte_eth_bond_slave_add(test_params->slave_port_ids[0], + test_params->slave_port_ids[test_params->bonded_slave_count]), + "Expected call to failed as invalid port specified."); + + return 0; +} + + +static int +test_remove_slave_from_bonded_device(void) +{ + int current_slave_count; + struct ether_addr read_mac_addr, *mac_addr; + uint16_t slaves[RTE_MAX_ETHPORTS]; + + TEST_ASSERT_SUCCESS(rte_eth_bond_slave_remove(test_params->bonded_port_id, + test_params->slave_port_ids[test_params->bonded_slave_count-1]), + "Failed to remove slave %d from bonded port (%d).", + test_params->slave_port_ids[test_params->bonded_slave_count-1], + test_params->bonded_port_id); + + + current_slave_count = rte_eth_bond_slaves_get(test_params->bonded_port_id, + slaves, RTE_MAX_ETHPORTS); + + TEST_ASSERT_EQUAL(current_slave_count, test_params->bonded_slave_count - 1, + "Number of slaves (%d) is great than expected (%d).\n", + current_slave_count, test_params->bonded_slave_count - 1); + + + mac_addr = (struct ether_addr *)slave_mac; + mac_addr->addr_bytes[ETHER_ADDR_LEN-1] = + test_params->bonded_slave_count-1; + + rte_eth_macaddr_get( + test_params->slave_port_ids[test_params->bonded_slave_count-1], + &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(mac_addr, &read_mac_addr, sizeof(read_mac_addr)), + "bonded port mac address not set to that of primary port\n"); + + rte_eth_stats_reset( + test_params->slave_port_ids[test_params->bonded_slave_count-1]); + + virtual_ethdev_simulate_link_status_interrupt(test_params->bonded_port_id, + 0); + + test_params->bonded_slave_count--; + + return 0; +} + +static int +test_remove_slave_from_invalid_bonded_device(void) +{ + /* Invalid port ID */ + TEST_ASSERT_FAIL(rte_eth_bond_slave_remove( + test_params->bonded_port_id + 5, + test_params->slave_port_ids[test_params->bonded_slave_count - 1]), + "Expected call to failed as invalid port specified."); + + /* Non bonded device */ + TEST_ASSERT_FAIL(rte_eth_bond_slave_remove( + test_params->slave_port_ids[0], + test_params->slave_port_ids[test_params->bonded_slave_count - 1]), + "Expected call to failed as invalid port specified."); + + return 0; +} + +static int bonded_id = 2; + +static int +test_add_already_bonded_slave_to_bonded_device(void) +{ + int port_id, current_slave_count; + uint16_t slaves[RTE_MAX_ETHPORTS]; + char pmd_name[RTE_ETH_NAME_MAX_LEN]; + + test_add_slave_to_bonded_device(); + + current_slave_count = rte_eth_bond_slaves_get(test_params->bonded_port_id, + slaves, RTE_MAX_ETHPORTS); + TEST_ASSERT_EQUAL(current_slave_count, 1, + "Number of slaves (%d) is not that expected (%d).", + current_slave_count, 1); + + snprintf(pmd_name, RTE_ETH_NAME_MAX_LEN, "%s_%d", BONDED_DEV_NAME, ++bonded_id); + + port_id = rte_eth_bond_create(pmd_name, test_params->bonding_mode, + rte_socket_id()); + TEST_ASSERT(port_id >= 0, "Failed to create bonded device."); + + TEST_ASSERT(rte_eth_bond_slave_add(port_id, + test_params->slave_port_ids[test_params->bonded_slave_count - 1]) + < 0, + "Added slave (%d) to bonded port (%d) unexpectedly.", + test_params->slave_port_ids[test_params->bonded_slave_count-1], + port_id); + + return test_remove_slave_from_bonded_device(); +} + + +static int +test_get_slaves_from_bonded_device(void) +{ + int current_slave_count; + uint16_t slaves[RTE_MAX_ETHPORTS]; + + TEST_ASSERT_SUCCESS(test_add_slave_to_bonded_device(), + "Failed to add slave to bonded device"); + + /* Invalid port id */ + current_slave_count = rte_eth_bond_slaves_get(INVALID_PORT_ID, slaves, + RTE_MAX_ETHPORTS); + TEST_ASSERT(current_slave_count < 0, + "Invalid port id unexpectedly succeeded"); + + current_slave_count = rte_eth_bond_active_slaves_get(INVALID_PORT_ID, + slaves, RTE_MAX_ETHPORTS); + TEST_ASSERT(current_slave_count < 0, + "Invalid port id unexpectedly succeeded"); + + /* Invalid slaves pointer */ + current_slave_count = rte_eth_bond_slaves_get(test_params->bonded_port_id, + NULL, RTE_MAX_ETHPORTS); + TEST_ASSERT(current_slave_count < 0, + "Invalid slave array unexpectedly succeeded"); + + current_slave_count = rte_eth_bond_active_slaves_get( + test_params->bonded_port_id, NULL, RTE_MAX_ETHPORTS); + TEST_ASSERT(current_slave_count < 0, + "Invalid slave array unexpectedly succeeded"); + + /* non bonded device*/ + current_slave_count = rte_eth_bond_slaves_get( + test_params->slave_port_ids[0], NULL, RTE_MAX_ETHPORTS); + TEST_ASSERT(current_slave_count < 0, + "Invalid port id unexpectedly succeeded"); + + current_slave_count = rte_eth_bond_active_slaves_get( + test_params->slave_port_ids[0], NULL, RTE_MAX_ETHPORTS); + TEST_ASSERT(current_slave_count < 0, + "Invalid port id unexpectedly succeeded"); + + TEST_ASSERT_SUCCESS(test_remove_slave_from_bonded_device(), + "Failed to remove slaves from bonded device"); + + return 0; +} + + +static int +test_add_remove_multiple_slaves_to_from_bonded_device(void) +{ + int i; + + for (i = 0; i < TEST_MAX_NUMBER_OF_PORTS; i++) + TEST_ASSERT_SUCCESS(test_add_slave_to_bonded_device(), + "Failed to add slave to bonded device"); + + for (i = 0; i < TEST_MAX_NUMBER_OF_PORTS; i++) + TEST_ASSERT_SUCCESS(test_remove_slave_from_bonded_device(), + "Failed to remove slaves from bonded device"); + + return 0; +} + +static void +enable_bonded_slaves(void) +{ + int i; + + for (i = 0; i < test_params->bonded_slave_count; i++) { + virtual_ethdev_tx_burst_fn_set_success(test_params->slave_port_ids[i], + 1); + + virtual_ethdev_simulate_link_status_interrupt( + test_params->slave_port_ids[i], 1); + } +} + +static int +test_start_bonded_device(void) +{ + struct rte_eth_link link_status; + + int current_slave_count, current_bonding_mode, primary_port; + uint16_t slaves[RTE_MAX_ETHPORTS]; + + /* Add slave to bonded device*/ + TEST_ASSERT_SUCCESS(test_add_slave_to_bonded_device(), + "Failed to add slave to bonded device"); + + TEST_ASSERT_SUCCESS(rte_eth_dev_start(test_params->bonded_port_id), + "Failed to start bonded pmd eth device %d.", + test_params->bonded_port_id); + + /* Change link status of virtual pmd so it will be added to the active + * slave list of the bonded device*/ + virtual_ethdev_simulate_link_status_interrupt( + test_params->slave_port_ids[test_params->bonded_slave_count-1], 1); + + current_slave_count = rte_eth_bond_slaves_get(test_params->bonded_port_id, + slaves, RTE_MAX_ETHPORTS); + TEST_ASSERT_EQUAL(current_slave_count, test_params->bonded_slave_count, + "Number of slaves (%d) is not expected value (%d).", + current_slave_count, test_params->bonded_slave_count); + + current_slave_count = rte_eth_bond_active_slaves_get( + test_params->bonded_port_id, slaves, RTE_MAX_ETHPORTS); + TEST_ASSERT_EQUAL(current_slave_count, test_params->bonded_slave_count, + "Number of active slaves (%d) is not expected value (%d).", + current_slave_count, test_params->bonded_slave_count); + + current_bonding_mode = rte_eth_bond_mode_get(test_params->bonded_port_id); + TEST_ASSERT_EQUAL(current_bonding_mode, test_params->bonding_mode, + "Bonded device mode (%d) is not expected value (%d).\n", + current_bonding_mode, test_params->bonding_mode); + + primary_port = rte_eth_bond_primary_get(test_params->bonded_port_id); + TEST_ASSERT_EQUAL(primary_port, test_params->slave_port_ids[0], + "Primary port (%d) is not expected value (%d).", + primary_port, test_params->slave_port_ids[0]); + + rte_eth_link_get(test_params->bonded_port_id, &link_status); + TEST_ASSERT_EQUAL(link_status.link_status, 1, + "Bonded port (%d) status (%d) is not expected value (%d).\n", + test_params->bonded_port_id, link_status.link_status, 1); + + return 0; +} + +static int +test_stop_bonded_device(void) +{ + int current_slave_count; + uint16_t slaves[RTE_MAX_ETHPORTS]; + + struct rte_eth_link link_status; + + rte_eth_dev_stop(test_params->bonded_port_id); + + rte_eth_link_get(test_params->bonded_port_id, &link_status); + TEST_ASSERT_EQUAL(link_status.link_status, 0, + "Bonded port (%d) status (%d) is not expected value (%d).", + test_params->bonded_port_id, link_status.link_status, 0); + + current_slave_count = rte_eth_bond_slaves_get(test_params->bonded_port_id, + slaves, RTE_MAX_ETHPORTS); + TEST_ASSERT_EQUAL(current_slave_count, test_params->bonded_slave_count, + "Number of slaves (%d) is not expected value (%d).", + current_slave_count, test_params->bonded_slave_count); + + current_slave_count = rte_eth_bond_active_slaves_get( + test_params->bonded_port_id, slaves, RTE_MAX_ETHPORTS); + TEST_ASSERT_EQUAL(current_slave_count, 0, + "Number of active slaves (%d) is not expected value (%d).", + current_slave_count, 0); + + return 0; +} + +static int +remove_slaves_and_stop_bonded_device(void) +{ + /* Clean up and remove slaves from bonded device */ + free_virtualpmd_tx_queue(); + while (test_params->bonded_slave_count > 0) + TEST_ASSERT_SUCCESS(test_remove_slave_from_bonded_device(), + "test_remove_slave_from_bonded_device failed"); + + rte_eth_dev_stop(test_params->bonded_port_id); + rte_eth_stats_reset(test_params->bonded_port_id); + rte_eth_bond_mac_address_reset(test_params->bonded_port_id); + + return 0; +} + +static int +test_set_bonding_mode(void) +{ + int i, bonding_mode; + + int bonding_modes[] = { BONDING_MODE_ROUND_ROBIN, + BONDING_MODE_ACTIVE_BACKUP, + BONDING_MODE_BALANCE, + BONDING_MODE_BROADCAST + }; + + /* Test supported link bonding modes */ + for (i = 0; i < (int)RTE_DIM(bonding_modes); i++) { + /* Invalid port ID */ + TEST_ASSERT_FAIL(rte_eth_bond_mode_set(INVALID_PORT_ID, + bonding_modes[i]), + "Expected call to failed as invalid port (%d) specified.", + INVALID_PORT_ID); + + /* Non bonded device */ + TEST_ASSERT_FAIL(rte_eth_bond_mode_set(test_params->slave_port_ids[0], + bonding_modes[i]), + "Expected call to failed as invalid port (%d) specified.", + test_params->slave_port_ids[0]); + + TEST_ASSERT_SUCCESS(rte_eth_bond_mode_set(test_params->bonded_port_id, + bonding_modes[i]), + "Failed to set link bonding mode on port (%d) to (%d).", + test_params->bonded_port_id, bonding_modes[i]); + + bonding_mode = rte_eth_bond_mode_get(test_params->bonded_port_id); + TEST_ASSERT_EQUAL(bonding_mode, bonding_modes[i], + "Link bonding mode (%d) of port (%d) is not expected value (%d).", + bonding_mode, test_params->bonded_port_id, + bonding_modes[i]); + + /* Invalid port ID */ + bonding_mode = rte_eth_bond_mode_get(INVALID_PORT_ID); + TEST_ASSERT(bonding_mode < 0, + "Expected call to failed as invalid port (%d) specified.", + INVALID_PORT_ID); + + /* Non bonded device */ + bonding_mode = rte_eth_bond_mode_get(test_params->slave_port_ids[0]); + TEST_ASSERT(bonding_mode < 0, + "Expected call to failed as invalid port (%d) specified.", + test_params->slave_port_ids[0]); + } + + return remove_slaves_and_stop_bonded_device(); +} + +static int +test_set_primary_slave(void) +{ + int i, j, retval; + struct ether_addr read_mac_addr; + struct ether_addr *expected_mac_addr; + + /* Add 4 slaves to bonded device */ + for (i = test_params->bonded_slave_count; i < 4; i++) + TEST_ASSERT_SUCCESS(test_add_slave_to_bonded_device(), + "Failed to add slave to bonded device."); + + TEST_ASSERT_SUCCESS(rte_eth_bond_mode_set(test_params->bonded_port_id, + BONDING_MODE_ROUND_ROBIN), + "Failed to set link bonding mode on port (%d) to (%d).", + test_params->bonded_port_id, BONDING_MODE_ROUND_ROBIN); + + /* Invalid port ID */ + TEST_ASSERT_FAIL(rte_eth_bond_primary_set(INVALID_PORT_ID, + test_params->slave_port_ids[i]), + "Expected call to failed as invalid port specified."); + + /* Non bonded device */ + TEST_ASSERT_FAIL(rte_eth_bond_primary_set(test_params->slave_port_ids[i], + test_params->slave_port_ids[i]), + "Expected call to failed as invalid port specified."); + + /* Set slave as primary + * Verify slave it is now primary slave + * Verify that MAC address of bonded device is that of primary slave + * Verify that MAC address of all bonded slaves are that of primary slave + */ + for (i = 0; i < 4; i++) { + TEST_ASSERT_SUCCESS(rte_eth_bond_primary_set(test_params->bonded_port_id, + test_params->slave_port_ids[i]), + "Failed to set bonded port (%d) primary port to (%d)", + test_params->bonded_port_id, test_params->slave_port_ids[i]); + + retval = rte_eth_bond_primary_get(test_params->bonded_port_id); + TEST_ASSERT(retval >= 0, + "Failed to read primary port from bonded port (%d)\n", + test_params->bonded_port_id); + + TEST_ASSERT_EQUAL(retval, test_params->slave_port_ids[i], + "Bonded port (%d) primary port (%d) not expected value (%d)\n", + test_params->bonded_port_id, retval, + test_params->slave_port_ids[i]); + + /* stop/start bonded eth dev to apply new MAC */ + rte_eth_dev_stop(test_params->bonded_port_id); + + TEST_ASSERT_SUCCESS(rte_eth_dev_start(test_params->bonded_port_id), + "Failed to start bonded port %d", + test_params->bonded_port_id); + + expected_mac_addr = (struct ether_addr *)&slave_mac; + expected_mac_addr->addr_bytes[ETHER_ADDR_LEN-1] = i; + + /* Check primary slave MAC */ + rte_eth_macaddr_get(test_params->slave_port_ids[i], &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(expected_mac_addr, &read_mac_addr, + sizeof(read_mac_addr)), + "bonded port mac address not set to that of primary port\n"); + + /* Check bonded MAC */ + rte_eth_macaddr_get(test_params->bonded_port_id, &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(&read_mac_addr, &read_mac_addr, + sizeof(read_mac_addr)), + "bonded port mac address not set to that of primary port\n"); + + /* Check other slaves MACs */ + for (j = 0; j < 4; j++) { + if (j != i) { + rte_eth_macaddr_get(test_params->slave_port_ids[j], + &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(expected_mac_addr, &read_mac_addr, + sizeof(read_mac_addr)), + "slave port mac address not set to that of primary " + "port"); + } + } + } + + + /* Test with none existent port */ + TEST_ASSERT_FAIL(rte_eth_bond_primary_get(test_params->bonded_port_id + 10), + "read primary port from expectedly"); + + /* Test with slave port */ + TEST_ASSERT_FAIL(rte_eth_bond_primary_get(test_params->slave_port_ids[0]), + "read primary port from expectedly\n"); + + TEST_ASSERT_SUCCESS(remove_slaves_and_stop_bonded_device(), + "Failed to stop and remove slaves from bonded device"); + + /* No slaves */ + TEST_ASSERT(rte_eth_bond_primary_get(test_params->bonded_port_id) < 0, + "read primary port from expectedly\n"); + + return 0; +} + +static int +test_set_explicit_bonded_mac(void) +{ + int i; + struct ether_addr read_mac_addr; + struct ether_addr *mac_addr; + + uint8_t explicit_bonded_mac[] = { 0xDE, 0xAD, 0xBE, 0xEF, 0x00, 0x01 }; + + mac_addr = (struct ether_addr *)explicit_bonded_mac; + + /* Invalid port ID */ + TEST_ASSERT_FAIL(rte_eth_bond_mac_address_set(INVALID_PORT_ID, mac_addr), + "Expected call to failed as invalid port specified."); + + /* Non bonded device */ + TEST_ASSERT_FAIL(rte_eth_bond_mac_address_set( + test_params->slave_port_ids[0], mac_addr), + "Expected call to failed as invalid port specified."); + + /* NULL MAC address */ + TEST_ASSERT_FAIL(rte_eth_bond_mac_address_set( + test_params->bonded_port_id, NULL), + "Expected call to failed as NULL MAC specified"); + + TEST_ASSERT_SUCCESS(rte_eth_bond_mac_address_set( + test_params->bonded_port_id, mac_addr), + "Failed to set MAC address on bonded port (%d)", + test_params->bonded_port_id); + + /* Add 4 slaves to bonded device */ + for (i = test_params->bonded_slave_count; i < 4; i++) { + TEST_ASSERT_SUCCESS(test_add_slave_to_bonded_device(), + "Failed to add slave to bonded device.\n"); + } + + /* Check bonded MAC */ + rte_eth_macaddr_get(test_params->bonded_port_id, &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(mac_addr, &read_mac_addr, sizeof(read_mac_addr)), + "bonded port mac address not set to that of primary port"); + + /* Check other slaves MACs */ + for (i = 0; i < 4; i++) { + rte_eth_macaddr_get(test_params->slave_port_ids[i], &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(mac_addr, &read_mac_addr, + sizeof(read_mac_addr)), + "slave port mac address not set to that of primary port"); + } + + /* test resetting mac address on bonded device */ + TEST_ASSERT_SUCCESS( + rte_eth_bond_mac_address_reset(test_params->bonded_port_id), + "Failed to reset MAC address on bonded port (%d)", + test_params->bonded_port_id); + + TEST_ASSERT_FAIL( + rte_eth_bond_mac_address_reset(test_params->slave_port_ids[0]), + "Reset MAC address on bonded port (%d) unexpectedly", + test_params->slave_port_ids[1]); + + /* test resetting mac address on bonded device with no slaves */ + TEST_ASSERT_SUCCESS(remove_slaves_and_stop_bonded_device(), + "Failed to remove slaves and stop bonded device"); + + TEST_ASSERT_SUCCESS(rte_eth_bond_mac_address_reset(test_params->bonded_port_id), + "Failed to reset MAC address on bonded port (%d)", + test_params->bonded_port_id); + + return 0; +} + +#define BONDED_INIT_MAC_ASSIGNMENT_SLAVE_COUNT (3) + +static int +test_set_bonded_port_initialization_mac_assignment(void) +{ + int i, slave_count, bonded_port_id; + + uint16_t slaves[RTE_MAX_ETHPORTS]; + int slave_port_ids[BONDED_INIT_MAC_ASSIGNMENT_SLAVE_COUNT]; + + struct ether_addr slave_mac_addr, bonded_mac_addr, read_mac_addr; + + /* Initialize default values for MAC addresses */ + memcpy(&slave_mac_addr, slave_mac, sizeof(struct ether_addr)); + memcpy(&bonded_mac_addr, slave_mac, sizeof(struct ether_addr)); + + /* + * 1. a - Create / configure bonded / slave ethdevs + */ + bonded_port_id = rte_eth_bond_create("net_bonding_mac_ass_test", + BONDING_MODE_ACTIVE_BACKUP, rte_socket_id()); + TEST_ASSERT(bonded_port_id > 0, "failed to create bonded device"); + + TEST_ASSERT_SUCCESS(configure_ethdev(bonded_port_id, 0, 0), + "Failed to configure bonded ethdev"); + + for (i = 0; i < BONDED_INIT_MAC_ASSIGNMENT_SLAVE_COUNT; i++) { + char pmd_name[RTE_ETH_NAME_MAX_LEN]; + + slave_mac_addr.addr_bytes[ETHER_ADDR_LEN-1] = i + 100; + + snprintf(pmd_name, RTE_ETH_NAME_MAX_LEN, "eth_slave_%d", i); + + slave_port_ids[i] = virtual_ethdev_create(pmd_name, + &slave_mac_addr, rte_socket_id(), 1); + + TEST_ASSERT(slave_port_ids[i] >= 0, + "Failed to create slave ethdev %s", pmd_name); + + TEST_ASSERT_SUCCESS(configure_ethdev(slave_port_ids[i], 1, 0), + "Failed to configure virtual ethdev %s", + pmd_name); + } + + + /* + * 2. Add slave ethdevs to bonded device + */ + for (i = 0; i < BONDED_INIT_MAC_ASSIGNMENT_SLAVE_COUNT; i++) { + TEST_ASSERT_SUCCESS(rte_eth_bond_slave_add(bonded_port_id, + slave_port_ids[i]), + "Failed to add slave (%d) to bonded port (%d).", + slave_port_ids[i], bonded_port_id); + } + + slave_count = rte_eth_bond_slaves_get(bonded_port_id, slaves, + RTE_MAX_ETHPORTS); + TEST_ASSERT_EQUAL(BONDED_INIT_MAC_ASSIGNMENT_SLAVE_COUNT, slave_count, + "Number of slaves (%d) is not as expected (%d)", + slave_count, BONDED_INIT_MAC_ASSIGNMENT_SLAVE_COUNT); + + + /* + * 3. Set explicit MAC address on bonded ethdev + */ + bonded_mac_addr.addr_bytes[ETHER_ADDR_LEN-2] = 0xFF; + bonded_mac_addr.addr_bytes[ETHER_ADDR_LEN-1] = 0xAA; + + TEST_ASSERT_SUCCESS(rte_eth_bond_mac_address_set( + bonded_port_id, &bonded_mac_addr), + "Failed to set MAC address on bonded port (%d)", + bonded_port_id); + + + /* 4. a - Start bonded ethdev + * b - Enable slave devices + * c - Verify bonded/slaves ethdev MAC addresses + */ + TEST_ASSERT_SUCCESS(rte_eth_dev_start(bonded_port_id), + "Failed to start bonded pmd eth device %d.", + bonded_port_id); + + for (i = 0; i < BONDED_INIT_MAC_ASSIGNMENT_SLAVE_COUNT; i++) { + virtual_ethdev_simulate_link_status_interrupt( + slave_port_ids[i], 1); + } + + rte_eth_macaddr_get(bonded_port_id, &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(&bonded_mac_addr, &read_mac_addr, + sizeof(read_mac_addr)), + "bonded port mac address not as expected"); + + rte_eth_macaddr_get(slave_port_ids[0], &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(&bonded_mac_addr, &read_mac_addr, + sizeof(read_mac_addr)), + "slave port 0 mac address not as expected"); + + slave_mac_addr.addr_bytes[ETHER_ADDR_LEN-1] = 1 + 100; + rte_eth_macaddr_get(slave_port_ids[1], &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(&slave_mac_addr, &read_mac_addr, + sizeof(read_mac_addr)), + "slave port 1 mac address not as expected"); + + slave_mac_addr.addr_bytes[ETHER_ADDR_LEN-1] = 2 + 100; + rte_eth_macaddr_get(slave_port_ids[2], &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(&slave_mac_addr, &read_mac_addr, + sizeof(read_mac_addr)), + "slave port 2 mac address not as expected"); + + + /* 7. a - Change primary port + * b - Stop / Start bonded port + * d - Verify slave ethdev MAC addresses + */ + TEST_ASSERT_SUCCESS(rte_eth_bond_primary_set(bonded_port_id, + slave_port_ids[2]), + "failed to set primary port on bonded device."); + + rte_eth_dev_stop(bonded_port_id); + TEST_ASSERT_SUCCESS(rte_eth_dev_start(bonded_port_id), + "Failed to start bonded pmd eth device %d.", + bonded_port_id); + + rte_eth_macaddr_get(bonded_port_id, &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(&bonded_mac_addr, &read_mac_addr, + sizeof(read_mac_addr)), + "bonded port mac address not as expected"); + + slave_mac_addr.addr_bytes[ETHER_ADDR_LEN-1] = 0 + 100; + rte_eth_macaddr_get(slave_port_ids[0], &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(&slave_mac_addr, &read_mac_addr, + sizeof(read_mac_addr)), + "slave port 0 mac address not as expected"); + + slave_mac_addr.addr_bytes[ETHER_ADDR_LEN-1] = 1 + 100; + rte_eth_macaddr_get(slave_port_ids[1], &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(&slave_mac_addr, &read_mac_addr, + sizeof(read_mac_addr)), + "slave port 1 mac address not as expected"); + + rte_eth_macaddr_get(slave_port_ids[2], &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(&bonded_mac_addr, &read_mac_addr, + sizeof(read_mac_addr)), + "slave port 2 mac address not as expected"); + + /* 6. a - Stop bonded ethdev + * b - remove slave ethdevs + * c - Verify slave ethdevs MACs are restored + */ + rte_eth_dev_stop(bonded_port_id); + + for (i = 0; i < BONDED_INIT_MAC_ASSIGNMENT_SLAVE_COUNT; i++) { + TEST_ASSERT_SUCCESS(rte_eth_bond_slave_remove(bonded_port_id, + slave_port_ids[i]), + "Failed to remove slave %d from bonded port (%d).", + slave_port_ids[i], bonded_port_id); + } + + slave_count = rte_eth_bond_slaves_get(bonded_port_id, slaves, + RTE_MAX_ETHPORTS); + + TEST_ASSERT_EQUAL(slave_count, 0, + "Number of slaves (%d) is great than expected (%d).", + slave_count, 0); + + slave_mac_addr.addr_bytes[ETHER_ADDR_LEN-1] = 0 + 100; + rte_eth_macaddr_get(slave_port_ids[0], &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(&slave_mac_addr, &read_mac_addr, + sizeof(read_mac_addr)), + "slave port 0 mac address not as expected"); + + slave_mac_addr.addr_bytes[ETHER_ADDR_LEN-1] = 1 + 100; + rte_eth_macaddr_get(slave_port_ids[1], &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(&slave_mac_addr, &read_mac_addr, + sizeof(read_mac_addr)), + "slave port 1 mac address not as expected"); + + slave_mac_addr.addr_bytes[ETHER_ADDR_LEN-1] = 2 + 100; + rte_eth_macaddr_get(slave_port_ids[2], &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(&slave_mac_addr, &read_mac_addr, + sizeof(read_mac_addr)), + "slave port 2 mac address not as expected"); + + return 0; +} + + +static int +initialize_bonded_device_with_slaves(uint8_t bonding_mode, uint8_t bond_en_isr, + uint16_t number_of_slaves, uint8_t enable_slave) +{ + /* Configure bonded device */ + TEST_ASSERT_SUCCESS(configure_ethdev(test_params->bonded_port_id, 0, + bond_en_isr), "Failed to configure bonding port (%d) in mode %d " + "with (%d) slaves.", test_params->bonded_port_id, bonding_mode, + number_of_slaves); + + /* Add slaves to bonded device */ + while (number_of_slaves > test_params->bonded_slave_count) + TEST_ASSERT_SUCCESS(test_add_slave_to_bonded_device(), + "Failed to add slave (%d to bonding port (%d).", + test_params->bonded_slave_count - 1, + test_params->bonded_port_id); + + /* Set link bonding mode */ + TEST_ASSERT_SUCCESS(rte_eth_bond_mode_set(test_params->bonded_port_id, + bonding_mode), + "Failed to set link bonding mode on port (%d) to (%d).", + test_params->bonded_port_id, bonding_mode); + + TEST_ASSERT_SUCCESS(rte_eth_dev_start(test_params->bonded_port_id), + "Failed to start bonded pmd eth device %d.", + test_params->bonded_port_id); + + if (enable_slave) + enable_bonded_slaves(); + + return 0; +} + +static int +test_adding_slave_after_bonded_device_started(void) +{ + int i; + + TEST_ASSERT_SUCCESS(initialize_bonded_device_with_slaves( + BONDING_MODE_ROUND_ROBIN, 0, 4, 0), + "Failed to add slaves to bonded device"); + + /* Enabled slave devices */ + for (i = 0; i < test_params->bonded_slave_count + 1; i++) { + virtual_ethdev_simulate_link_status_interrupt( + test_params->slave_port_ids[i], 1); + } + + TEST_ASSERT_SUCCESS(rte_eth_bond_slave_add(test_params->bonded_port_id, + test_params->slave_port_ids[test_params->bonded_slave_count]), + "Failed to add slave to bonded port.\n"); + + rte_eth_stats_reset( + test_params->slave_port_ids[test_params->bonded_slave_count]); + + test_params->bonded_slave_count++; + + return remove_slaves_and_stop_bonded_device(); +} + +#define TEST_STATUS_INTERRUPT_SLAVE_COUNT 4 +#define TEST_LSC_WAIT_TIMEOUT_MS 500 + +int test_lsc_interrupt_count; + + +static int +test_bonding_lsc_event_callback(uint16_t port_id __rte_unused, + enum rte_eth_event_type type __rte_unused, + void *param __rte_unused, + void *ret_param __rte_unused) +{ + pthread_mutex_lock(&mutex); + test_lsc_interrupt_count++; + + pthread_cond_signal(&cvar); + pthread_mutex_unlock(&mutex); + + return 0; +} + +static inline int +lsc_timeout(int wait_us) +{ + int retval = 0; + + struct timespec ts; + struct timeval tp; + + gettimeofday(&tp, NULL); + + /* Convert from timeval to timespec */ + ts.tv_sec = tp.tv_sec; + ts.tv_nsec = tp.tv_usec * 1000; + ts.tv_nsec += wait_us * 1000; + + pthread_mutex_lock(&mutex); + if (test_lsc_interrupt_count < 1) + retval = pthread_cond_timedwait(&cvar, &mutex, &ts); + + pthread_mutex_unlock(&mutex); + + if (retval == 0 && test_lsc_interrupt_count < 1) + return -1; + + return retval; +} + +static int +test_status_interrupt(void) +{ + int slave_count; + uint16_t slaves[RTE_MAX_ETHPORTS]; + + /* initialized bonding device with T slaves */ + TEST_ASSERT_SUCCESS(initialize_bonded_device_with_slaves( + BONDING_MODE_ROUND_ROBIN, 1, + TEST_STATUS_INTERRUPT_SLAVE_COUNT, 1), + "Failed to initialise bonded device"); + + test_lsc_interrupt_count = 0; + + /* register link status change interrupt callback */ + rte_eth_dev_callback_register(test_params->bonded_port_id, + RTE_ETH_EVENT_INTR_LSC, test_bonding_lsc_event_callback, + &test_params->bonded_port_id); + + slave_count = rte_eth_bond_active_slaves_get(test_params->bonded_port_id, + slaves, RTE_MAX_ETHPORTS); + + TEST_ASSERT_EQUAL(slave_count, TEST_STATUS_INTERRUPT_SLAVE_COUNT, + "Number of active slaves (%d) is not as expected (%d)", + slave_count, TEST_STATUS_INTERRUPT_SLAVE_COUNT); + + /* Bring all 4 slaves link status to down and test that we have received a + * lsc interrupts */ + virtual_ethdev_simulate_link_status_interrupt( + test_params->slave_port_ids[0], 0); + virtual_ethdev_simulate_link_status_interrupt( + test_params->slave_port_ids[1], 0); + virtual_ethdev_simulate_link_status_interrupt( + test_params->slave_port_ids[2], 0); + + TEST_ASSERT_EQUAL(test_lsc_interrupt_count, 0, + "Received a link status change interrupt unexpectedly"); + + virtual_ethdev_simulate_link_status_interrupt( + test_params->slave_port_ids[3], 0); + + TEST_ASSERT(lsc_timeout(TEST_LSC_WAIT_TIMEOUT_MS) == 0, + "timed out waiting for interrupt"); + + TEST_ASSERT(test_lsc_interrupt_count > 0, + "Did not receive link status change interrupt"); + + slave_count = rte_eth_bond_active_slaves_get(test_params->bonded_port_id, + slaves, RTE_MAX_ETHPORTS); + + TEST_ASSERT_EQUAL(slave_count, 0, + "Number of active slaves (%d) is not as expected (%d)", + slave_count, 0); + + /* bring one slave port up so link status will change */ + test_lsc_interrupt_count = 0; + + virtual_ethdev_simulate_link_status_interrupt( + test_params->slave_port_ids[0], 1); + + TEST_ASSERT(lsc_timeout(TEST_LSC_WAIT_TIMEOUT_MS) == 0, + "timed out waiting for interrupt"); + + /* test that we have received another lsc interrupt */ + TEST_ASSERT(test_lsc_interrupt_count > 0, + "Did not receive link status change interrupt"); + + /* Verify that calling the same slave lsc interrupt doesn't cause another + * lsc interrupt from bonded device */ + test_lsc_interrupt_count = 0; + + virtual_ethdev_simulate_link_status_interrupt( + test_params->slave_port_ids[0], 1); + + TEST_ASSERT(lsc_timeout(TEST_LSC_WAIT_TIMEOUT_MS) != 0, + "received unexpected interrupt"); + + TEST_ASSERT_EQUAL(test_lsc_interrupt_count, 0, + "Did not receive link status change interrupt"); + + + /* unregister lsc callback before exiting */ + rte_eth_dev_callback_unregister(test_params->bonded_port_id, + RTE_ETH_EVENT_INTR_LSC, test_bonding_lsc_event_callback, + &test_params->bonded_port_id); + + /* Clean up and remove slaves from bonded device */ + return remove_slaves_and_stop_bonded_device(); +} + +static int +generate_test_burst(struct rte_mbuf **pkts_burst, uint16_t burst_size, + uint8_t vlan, uint8_t ipv4, uint8_t toggle_dst_mac, + uint8_t toggle_ip_addr, uint16_t toggle_udp_port) +{ + uint16_t pktlen, generated_burst_size, ether_type; + void *ip_hdr; + + if (ipv4) + ether_type = ETHER_TYPE_IPv4; + else + ether_type = ETHER_TYPE_IPv6; + + if (toggle_dst_mac) + initialize_eth_header(test_params->pkt_eth_hdr, + (struct ether_addr *)src_mac, (struct ether_addr *)dst_mac_1, + ether_type, vlan, vlan_id); + else + initialize_eth_header(test_params->pkt_eth_hdr, + (struct ether_addr *)src_mac, (struct ether_addr *)dst_mac_0, + ether_type, vlan, vlan_id); + + + if (toggle_udp_port) + pktlen = initialize_udp_header(test_params->pkt_udp_hdr, src_port, + dst_port_1, 64); + else + pktlen = initialize_udp_header(test_params->pkt_udp_hdr, src_port, + dst_port_0, 64); + + if (ipv4) { + if (toggle_ip_addr) + pktlen = initialize_ipv4_header(test_params->pkt_ipv4_hdr, src_addr, + dst_addr_1, pktlen); + else + pktlen = initialize_ipv4_header(test_params->pkt_ipv4_hdr, src_addr, + dst_addr_0, pktlen); + + ip_hdr = test_params->pkt_ipv4_hdr; + } else { + if (toggle_ip_addr) + pktlen = initialize_ipv6_header(test_params->pkt_ipv6_hdr, + (uint8_t *)src_ipv6_addr, (uint8_t *)dst_ipv6_addr_1, + pktlen); + else + pktlen = initialize_ipv6_header(test_params->pkt_ipv6_hdr, + (uint8_t *)src_ipv6_addr, (uint8_t *)dst_ipv6_addr_0, + pktlen); + + ip_hdr = test_params->pkt_ipv6_hdr; + } + + /* Generate burst of packets to transmit */ + generated_burst_size = generate_packet_burst(test_params->mbuf_pool, + pkts_burst, test_params->pkt_eth_hdr, vlan, ip_hdr, ipv4, + test_params->pkt_udp_hdr, burst_size, PACKET_BURST_GEN_PKT_LEN_128, + 1); + TEST_ASSERT_EQUAL(generated_burst_size, burst_size, + "Failed to generate packet burst"); + + return generated_burst_size; +} + +/** Round Robin Mode Tests */ + +static int +test_roundrobin_tx_burst(void) +{ + int i, burst_size; + struct rte_mbuf *pkt_burst[MAX_PKT_BURST]; + struct rte_eth_stats port_stats; + + TEST_ASSERT_SUCCESS(initialize_bonded_device_with_slaves( + BONDING_MODE_ROUND_ROBIN, 0, 2, 1), + "Failed to initialise bonded device"); + + burst_size = 20 * test_params->bonded_slave_count; + + TEST_ASSERT(burst_size <= MAX_PKT_BURST, + "Burst size specified is greater than supported."); + + /* Generate test bursts of packets to transmit */ + TEST_ASSERT_EQUAL(generate_test_burst(pkt_burst, burst_size, 0, 1, 0, 0, 0), + burst_size, "failed to generate test burst"); + + /* Send burst on bonded port */ + TEST_ASSERT_EQUAL(rte_eth_tx_burst( + test_params->bonded_port_id, 0, pkt_burst, burst_size), burst_size, + "tx burst failed"); + + /* Verify bonded port tx stats */ + rte_eth_stats_get(test_params->bonded_port_id, &port_stats); + TEST_ASSERT_EQUAL(port_stats.opackets, (uint64_t)burst_size, + "Bonded Port (%d) opackets value (%u) not as expected (%d)\n", + test_params->bonded_port_id, (unsigned int)port_stats.opackets, + burst_size); + + /* Verify slave ports tx stats */ + for (i = 0; i < test_params->bonded_slave_count; i++) { + rte_eth_stats_get(test_params->slave_port_ids[i], &port_stats); + TEST_ASSERT_EQUAL(port_stats.opackets, + (uint64_t)burst_size / test_params->bonded_slave_count, + "Slave Port (%d) opackets value (%u) not as expected (%d)\n", + test_params->bonded_port_id, (unsigned int)port_stats.opackets, + burst_size / test_params->bonded_slave_count); + } + + /* Put all slaves down and try and transmit */ + for (i = 0; i < test_params->bonded_slave_count; i++) { + virtual_ethdev_simulate_link_status_interrupt( + test_params->slave_port_ids[i], 0); + } + + /* Send burst on bonded port */ + TEST_ASSERT_EQUAL(rte_eth_tx_burst(test_params->bonded_port_id, 0, + pkt_burst, burst_size), 0, + "tx burst return unexpected value"); + + /* Clean up and remove slaves from bonded device */ + return remove_slaves_and_stop_bonded_device(); +} + +static int +verify_mbufs_ref_count(struct rte_mbuf **mbufs, int nb_mbufs, int val) +{ + int i, refcnt; + + for (i = 0; i < nb_mbufs; i++) { + refcnt = rte_mbuf_refcnt_read(mbufs[i]); + TEST_ASSERT_EQUAL(refcnt, val, + "mbuf ref count (%d)is not the expected value (%d)", + refcnt, val); + } + return 0; +} + +static void +free_mbufs(struct rte_mbuf **mbufs, int nb_mbufs) +{ + int i; + + for (i = 0; i < nb_mbufs; i++) + rte_pktmbuf_free(mbufs[i]); +} + +#define TEST_RR_SLAVE_TX_FAIL_SLAVE_COUNT (2) +#define TEST_RR_SLAVE_TX_FAIL_BURST_SIZE (64) +#define TEST_RR_SLAVE_TX_FAIL_PACKETS_COUNT (22) +#define TEST_RR_SLAVE_TX_FAIL_FAILING_SLAVE_IDX (1) + +static int +test_roundrobin_tx_burst_slave_tx_fail(void) +{ + struct rte_mbuf *pkt_burst[MAX_PKT_BURST]; + struct rte_mbuf *expected_tx_fail_pkts[MAX_PKT_BURST]; + + struct rte_eth_stats port_stats; + + int i, first_fail_idx, tx_count; + + TEST_ASSERT_SUCCESS(initialize_bonded_device_with_slaves( + BONDING_MODE_ROUND_ROBIN, 0, + TEST_RR_SLAVE_TX_FAIL_SLAVE_COUNT, 1), + "Failed to initialise bonded device"); + + /* Generate test bursts of packets to transmit */ + TEST_ASSERT_EQUAL(generate_test_burst(pkt_burst, + TEST_RR_SLAVE_TX_FAIL_BURST_SIZE, 0, 1, 0, 0, 0), + TEST_RR_SLAVE_TX_FAIL_BURST_SIZE, + "Failed to generate test packet burst"); + + /* Copy references to packets which we expect not to be transmitted */ + first_fail_idx = (TEST_RR_SLAVE_TX_FAIL_BURST_SIZE - + (TEST_RR_SLAVE_TX_FAIL_PACKETS_COUNT * + TEST_RR_SLAVE_TX_FAIL_SLAVE_COUNT)) + + TEST_RR_SLAVE_TX_FAIL_FAILING_SLAVE_IDX; + + for (i = 0; i < TEST_RR_SLAVE_TX_FAIL_PACKETS_COUNT; i++) { + expected_tx_fail_pkts[i] = pkt_burst[first_fail_idx + + (i * TEST_RR_SLAVE_TX_FAIL_SLAVE_COUNT)]; + } + + /* Set virtual slave to only fail transmission of + * TEST_RR_SLAVE_TX_FAIL_PACKETS_COUNT packets in burst */ + virtual_ethdev_tx_burst_fn_set_success( + test_params->slave_port_ids[TEST_RR_SLAVE_TX_FAIL_FAILING_SLAVE_IDX], + 0); + + virtual_ethdev_tx_burst_fn_set_tx_pkt_fail_count( + test_params->slave_port_ids[TEST_RR_SLAVE_TX_FAIL_FAILING_SLAVE_IDX], + TEST_RR_SLAVE_TX_FAIL_PACKETS_COUNT); + + tx_count = rte_eth_tx_burst(test_params->bonded_port_id, 0, pkt_burst, + TEST_RR_SLAVE_TX_FAIL_BURST_SIZE); + + TEST_ASSERT_EQUAL(tx_count, TEST_RR_SLAVE_TX_FAIL_BURST_SIZE - + TEST_RR_SLAVE_TX_FAIL_PACKETS_COUNT, + "Transmitted (%d) an unexpected (%d) number of packets", tx_count, + TEST_RR_SLAVE_TX_FAIL_BURST_SIZE - + TEST_RR_SLAVE_TX_FAIL_PACKETS_COUNT); + + /* Verify that failed packet are expected failed packets */ + for (i = 0; i < TEST_RR_SLAVE_TX_FAIL_PACKETS_COUNT; i++) { + TEST_ASSERT_EQUAL(expected_tx_fail_pkts[i], pkt_burst[i + tx_count], + "expected mbuf (%d) pointer %p not expected pointer %p", + i, expected_tx_fail_pkts[i], pkt_burst[i + tx_count]); + } + + /* Verify bonded port tx stats */ + rte_eth_stats_get(test_params->bonded_port_id, &port_stats); + + TEST_ASSERT_EQUAL(port_stats.opackets, + (uint64_t)TEST_RR_SLAVE_TX_FAIL_BURST_SIZE - + TEST_RR_SLAVE_TX_FAIL_PACKETS_COUNT, + "Bonded Port (%d) opackets value (%u) not as expected (%d)", + test_params->bonded_port_id, (unsigned int)port_stats.opackets, + TEST_RR_SLAVE_TX_FAIL_BURST_SIZE - + TEST_RR_SLAVE_TX_FAIL_PACKETS_COUNT); + + /* Verify slave ports tx stats */ + for (i = 0; i < test_params->bonded_slave_count; i++) { + int slave_expected_tx_count; + + rte_eth_stats_get(test_params->slave_port_ids[i], &port_stats); + + slave_expected_tx_count = TEST_RR_SLAVE_TX_FAIL_BURST_SIZE / + test_params->bonded_slave_count; + + if (i == TEST_RR_SLAVE_TX_FAIL_FAILING_SLAVE_IDX) + slave_expected_tx_count = slave_expected_tx_count - + TEST_RR_SLAVE_TX_FAIL_PACKETS_COUNT; + + TEST_ASSERT_EQUAL(port_stats.opackets, + (uint64_t)slave_expected_tx_count, + "Slave Port (%d) opackets value (%u) not as expected (%d)", + test_params->slave_port_ids[i], + (unsigned int)port_stats.opackets, slave_expected_tx_count); + } + + /* Verify that all mbufs have a ref value of zero */ + TEST_ASSERT_SUCCESS(verify_mbufs_ref_count(&pkt_burst[tx_count], + TEST_RR_SLAVE_TX_FAIL_PACKETS_COUNT, 1), + "mbufs refcnts not as expected"); + free_mbufs(&pkt_burst[tx_count], TEST_RR_SLAVE_TX_FAIL_PACKETS_COUNT); + + /* Clean up and remove slaves from bonded device */ + return remove_slaves_and_stop_bonded_device(); +} + +static int +test_roundrobin_rx_burst_on_single_slave(void) +{ + struct rte_mbuf *gen_pkt_burst[MAX_PKT_BURST] = { NULL }; + struct rte_mbuf *rx_pkt_burst[MAX_PKT_BURST] = { NULL }; + + struct rte_eth_stats port_stats; + + int i, j, burst_size = 25; + + /* Initialize bonded device with 4 slaves in round robin mode */ + TEST_ASSERT_SUCCESS(initialize_bonded_device_with_slaves( + BONDING_MODE_ROUND_ROBIN, 0, 4, 1), + "Failed to initialize bonded device with slaves"); + + /* Generate test bursts of packets to transmit */ + TEST_ASSERT_EQUAL(generate_test_burst( + gen_pkt_burst, burst_size, 0, 1, 0, 0, 0), burst_size, + "burst generation failed"); + + for (i = 0; i < test_params->bonded_slave_count; i++) { + /* Add rx data to slave */ + virtual_ethdev_add_mbufs_to_rx_queue(test_params->slave_port_ids[i], + &gen_pkt_burst[0], burst_size); + + /* Call rx burst on bonded device */ + /* Send burst on bonded port */ + TEST_ASSERT_EQUAL(rte_eth_rx_burst( + test_params->bonded_port_id, 0, rx_pkt_burst, + MAX_PKT_BURST), burst_size, + "round-robin rx burst failed"); + + /* Verify bonded device rx count */ + rte_eth_stats_get(test_params->bonded_port_id, &port_stats); + TEST_ASSERT_EQUAL(port_stats.ipackets, (uint64_t)burst_size, + "Bonded Port (%d) ipackets value (%u) not as expected (%d)", + test_params->bonded_port_id, + (unsigned int)port_stats.ipackets, burst_size); + + + + /* Verify bonded slave devices rx count */ + /* Verify slave ports tx stats */ + for (j = 0; j < test_params->bonded_slave_count; j++) { + rte_eth_stats_get(test_params->slave_port_ids[j], &port_stats); + + if (i == j) { + TEST_ASSERT_EQUAL(port_stats.ipackets, (uint64_t)burst_size, + "Slave Port (%d) ipackets value (%u) not as expected" + " (%d)", test_params->slave_port_ids[i], + (unsigned int)port_stats.ipackets, burst_size); + } else { + TEST_ASSERT_EQUAL(port_stats.ipackets, 0, + "Slave Port (%d) ipackets value (%u) not as expected" + " (%d)", test_params->slave_port_ids[i], + (unsigned int)port_stats.ipackets, 0); + } + + /* Reset bonded slaves stats */ + rte_eth_stats_reset(test_params->slave_port_ids[j]); + } + /* reset bonded device stats */ + rte_eth_stats_reset(test_params->bonded_port_id); + } + + /* free mbufs */ + for (i = 0; i < MAX_PKT_BURST; i++) { + if (rx_pkt_burst[i] != NULL) + rte_pktmbuf_free(rx_pkt_burst[i]); + } + + + /* Clean up and remove slaves from bonded device */ + return remove_slaves_and_stop_bonded_device(); +} + +#define TEST_ROUNDROBIN_TX_BURST_SLAVE_COUNT (3) + +static int +test_roundrobin_rx_burst_on_multiple_slaves(void) +{ + struct rte_mbuf *gen_pkt_burst[TEST_ROUNDROBIN_TX_BURST_SLAVE_COUNT][MAX_PKT_BURST]; + + struct rte_mbuf *rx_pkt_burst[MAX_PKT_BURST] = { NULL }; + struct rte_eth_stats port_stats; + + int burst_size[TEST_ROUNDROBIN_TX_BURST_SLAVE_COUNT] = { 15, 13, 36 }; + int i, nb_rx; + + /* Initialize bonded device with 4 slaves in round robin mode */ + TEST_ASSERT_SUCCESS(initialize_bonded_device_with_slaves( + BONDING_MODE_ROUND_ROBIN, 0, 4, 1), + "Failed to initialize bonded device with slaves"); + + /* Generate test bursts of packets to transmit */ + for (i = 0; i < TEST_ROUNDROBIN_TX_BURST_SLAVE_COUNT; i++) { + TEST_ASSERT_EQUAL(generate_test_burst( + &gen_pkt_burst[i][0], burst_size[i], 0, 1, 0, 0, 0), + burst_size[i], "burst generation failed"); + } + + /* Add rx data to slaves */ + for (i = 0; i < TEST_ROUNDROBIN_TX_BURST_SLAVE_COUNT; i++) { + virtual_ethdev_add_mbufs_to_rx_queue(test_params->slave_port_ids[i], + &gen_pkt_burst[i][0], burst_size[i]); + } + + /* Call rx burst on bonded device */ + /* Send burst on bonded port */ + nb_rx = rte_eth_rx_burst(test_params->bonded_port_id, 0, rx_pkt_burst, + MAX_PKT_BURST); + TEST_ASSERT_EQUAL(nb_rx , burst_size[0] + burst_size[1] + burst_size[2], + "round-robin rx burst failed (%d != %d)\n", nb_rx, + burst_size[0] + burst_size[1] + burst_size[2]); + + /* Verify bonded device rx count */ + rte_eth_stats_get(test_params->bonded_port_id, &port_stats); + TEST_ASSERT_EQUAL(port_stats.ipackets, + (uint64_t)(burst_size[0] + burst_size[1] + burst_size[2]), + "Bonded Port (%d) ipackets value (%u) not as expected (%d)", + test_params->bonded_port_id, (unsigned int)port_stats.ipackets, + burst_size[0] + burst_size[1] + burst_size[2]); + + /* Verify bonded slave devices rx counts */ + rte_eth_stats_get(test_params->slave_port_ids[0], &port_stats); + TEST_ASSERT_EQUAL(port_stats.ipackets, (uint64_t)burst_size[0], + "Slave Port (%d) ipackets value (%u) not as expected (%d)", + test_params->slave_port_ids[0], + (unsigned int)port_stats.ipackets, burst_size[0]); + + rte_eth_stats_get(test_params->slave_port_ids[1], &port_stats); + TEST_ASSERT_EQUAL(port_stats.ipackets, (uint64_t)burst_size[1], + "Slave Port (%d) ipackets value (%u) not as expected (%d)", + test_params->slave_port_ids[1], (unsigned int)port_stats.ipackets, + burst_size[1]); + + rte_eth_stats_get(test_params->slave_port_ids[2], &port_stats); + TEST_ASSERT_EQUAL(port_stats.ipackets, (uint64_t)burst_size[2], + "Slave Port (%d) ipackets value (%u) not as expected (%d)", + test_params->slave_port_ids[2], + (unsigned int)port_stats.ipackets, burst_size[2]); + + rte_eth_stats_get(test_params->slave_port_ids[3], &port_stats); + TEST_ASSERT_EQUAL(port_stats.ipackets, 0, + "Slave Port (%d) ipackets value (%u) not as expected (%d)", + test_params->slave_port_ids[3], + (unsigned int)port_stats.ipackets, 0); + + /* free mbufs */ + for (i = 0; i < MAX_PKT_BURST; i++) { + if (rx_pkt_burst[i] != NULL) + rte_pktmbuf_free(rx_pkt_burst[i]); + } + + /* Clean up and remove slaves from bonded device */ + return remove_slaves_and_stop_bonded_device(); +} + +static int +test_roundrobin_verify_mac_assignment(void) +{ + struct ether_addr read_mac_addr, expected_mac_addr_0, expected_mac_addr_2; + + int i; + + rte_eth_macaddr_get(test_params->slave_port_ids[0], &expected_mac_addr_0); + rte_eth_macaddr_get(test_params->slave_port_ids[2], &expected_mac_addr_2); + + /* Initialize bonded device with 4 slaves in round robin mode */ + TEST_ASSERT_SUCCESS(initialize_bonded_device_with_slaves( + BONDING_MODE_ROUND_ROBIN, 0, 4, 1), + "Failed to initialize bonded device with slaves"); + + /* Verify that all MACs are the same as first slave added to bonded dev */ + for (i = 0; i < test_params->bonded_slave_count; i++) { + rte_eth_macaddr_get(test_params->slave_port_ids[i], &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(&expected_mac_addr_0, &read_mac_addr, + sizeof(read_mac_addr)), + "slave port (%d) mac address not set to that of primary port", + test_params->slave_port_ids[i]); + } + + /* change primary and verify that MAC addresses haven't changed */ + TEST_ASSERT_SUCCESS(rte_eth_bond_primary_set(test_params->bonded_port_id, + test_params->slave_port_ids[2]), + "Failed to set bonded port (%d) primary port to (%d)", + test_params->bonded_port_id, test_params->slave_port_ids[i]); + + for (i = 0; i < test_params->bonded_slave_count; i++) { + rte_eth_macaddr_get(test_params->slave_port_ids[i], &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(&expected_mac_addr_0, &read_mac_addr, + sizeof(read_mac_addr)), + "slave port (%d) mac address has changed to that of primary" + " port without stop/start toggle of bonded device", + test_params->slave_port_ids[i]); + } + + /* stop / start bonded device and verify that primary MAC address is + * propagate to bonded device and slaves */ + rte_eth_dev_stop(test_params->bonded_port_id); + + TEST_ASSERT_SUCCESS(rte_eth_dev_start(test_params->bonded_port_id), + "Failed to start bonded device"); + + rte_eth_macaddr_get(test_params->bonded_port_id, &read_mac_addr); + TEST_ASSERT_SUCCESS( + memcmp(&expected_mac_addr_2, &read_mac_addr, sizeof(read_mac_addr)), + "bonded port (%d) mac address not set to that of new primary port", + test_params->slave_port_ids[i]); + + for (i = 0; i < test_params->bonded_slave_count; i++) { + rte_eth_macaddr_get(test_params->slave_port_ids[i], &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(&expected_mac_addr_2, &read_mac_addr, + sizeof(read_mac_addr)), + "slave port (%d) mac address not set to that of new primary" + " port", test_params->slave_port_ids[i]); + } + + /* Set explicit MAC address */ + TEST_ASSERT_SUCCESS(rte_eth_bond_mac_address_set( + test_params->bonded_port_id, (struct ether_addr *)bonded_mac), + "Failed to set MAC"); + + rte_eth_macaddr_get(test_params->bonded_port_id, &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(bonded_mac, &read_mac_addr, + sizeof(read_mac_addr)), + "bonded port (%d) mac address not set to that of new primary port", + test_params->slave_port_ids[i]); + + for (i = 0; i < test_params->bonded_slave_count; i++) { + rte_eth_macaddr_get(test_params->slave_port_ids[i], &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(bonded_mac, &read_mac_addr, + sizeof(read_mac_addr)), "slave port (%d) mac address not set to" + " that of new primary port\n", test_params->slave_port_ids[i]); + } + + /* Clean up and remove slaves from bonded device */ + return remove_slaves_and_stop_bonded_device(); +} + +static int +test_roundrobin_verify_promiscuous_enable_disable(void) +{ + int i, promiscuous_en; + + /* Initialize bonded device with 4 slaves in round robin mode */ + TEST_ASSERT_SUCCESS(initialize_bonded_device_with_slaves( + BONDING_MODE_ROUND_ROBIN, 0, 4, 1), + "Failed to initialize bonded device with slaves"); + + rte_eth_promiscuous_enable(test_params->bonded_port_id); + + promiscuous_en = rte_eth_promiscuous_get(test_params->bonded_port_id); + TEST_ASSERT_EQUAL(promiscuous_en, 1, + "Port (%d) promiscuous mode not enabled", + test_params->bonded_port_id); + + for (i = 0; i < test_params->bonded_slave_count; i++) { + promiscuous_en = rte_eth_promiscuous_get( + test_params->slave_port_ids[i]); + TEST_ASSERT_EQUAL(promiscuous_en, 1, + "slave port (%d) promiscuous mode not enabled", + test_params->slave_port_ids[i]); + } + + rte_eth_promiscuous_disable(test_params->bonded_port_id); + + promiscuous_en = rte_eth_promiscuous_get(test_params->bonded_port_id); + TEST_ASSERT_EQUAL(promiscuous_en, 0, + "Port (%d) promiscuous mode not disabled\n", + test_params->bonded_port_id); + + for (i = 0; i < test_params->bonded_slave_count; i++) { + promiscuous_en = rte_eth_promiscuous_get( + test_params->slave_port_ids[i]); + TEST_ASSERT_EQUAL(promiscuous_en, 0, + "Port (%d) promiscuous mode not disabled\n", + test_params->slave_port_ids[i]); + } + + /* Clean up and remove slaves from bonded device */ + return remove_slaves_and_stop_bonded_device(); +} + +#define TEST_RR_LINK_STATUS_SLAVE_COUNT (4) +#define TEST_RR_LINK_STATUS_EXPECTED_ACTIVE_SLAVE_COUNT (2) + +static int +test_roundrobin_verify_slave_link_status_change_behaviour(void) +{ + struct rte_mbuf *tx_pkt_burst[MAX_PKT_BURST] = { NULL }; + struct rte_mbuf *gen_pkt_burst[TEST_RR_LINK_STATUS_SLAVE_COUNT][MAX_PKT_BURST]; + struct rte_mbuf *rx_pkt_burst[MAX_PKT_BURST] = { NULL }; + + struct rte_eth_stats port_stats; + uint16_t slaves[RTE_MAX_ETHPORTS]; + + int i, burst_size, slave_count; + + /* NULL all pointers in array to simplify cleanup */ + memset(gen_pkt_burst, 0, sizeof(gen_pkt_burst)); + + /* Initialize bonded device with TEST_RR_LINK_STATUS_SLAVE_COUNT slaves + * in round robin mode */ + TEST_ASSERT_SUCCESS(initialize_bonded_device_with_slaves( + BONDING_MODE_ROUND_ROBIN, 0, TEST_RR_LINK_STATUS_SLAVE_COUNT, 1), + "Failed to initialize bonded device with slaves"); + + /* Verify Current Slaves Count /Active Slave Count is */ + slave_count = rte_eth_bond_slaves_get(test_params->bonded_port_id, slaves, + RTE_MAX_ETHPORTS); + TEST_ASSERT_EQUAL(slave_count, TEST_RR_LINK_STATUS_SLAVE_COUNT, + "Number of slaves (%d) is not as expected (%d).", + slave_count, TEST_RR_LINK_STATUS_SLAVE_COUNT); + + slave_count = rte_eth_bond_active_slaves_get(test_params->bonded_port_id, + slaves, RTE_MAX_ETHPORTS); + TEST_ASSERT_EQUAL(slave_count, TEST_RR_LINK_STATUS_SLAVE_COUNT, + "Number of active slaves (%d) is not as expected (%d).", + slave_count, TEST_RR_LINK_STATUS_SLAVE_COUNT); + + /* Set 2 slaves eth_devs link status to down */ + virtual_ethdev_simulate_link_status_interrupt( + test_params->slave_port_ids[1], 0); + virtual_ethdev_simulate_link_status_interrupt( + test_params->slave_port_ids[3], 0); + + slave_count = rte_eth_bond_active_slaves_get(test_params->bonded_port_id, + slaves, RTE_MAX_ETHPORTS); + TEST_ASSERT_EQUAL(slave_count, + TEST_RR_LINK_STATUS_EXPECTED_ACTIVE_SLAVE_COUNT, + "Number of active slaves (%d) is not as expected (%d).\n", + slave_count, TEST_RR_LINK_STATUS_EXPECTED_ACTIVE_SLAVE_COUNT); + + burst_size = 20; + + /* Verify that pkts are not sent on slaves with link status down: + * + * 1. Generate test burst of traffic + * 2. Transmit burst on bonded eth_dev + * 3. Verify stats for bonded eth_dev (opackets = burst_size) + * 4. Verify stats for slave eth_devs (s0 = 10, s1 = 0, s2 = 10, s3 = 0) + */ + TEST_ASSERT_EQUAL( + generate_test_burst(tx_pkt_burst, burst_size, 0, 1, 0, 0, 0), + burst_size, "generate_test_burst failed"); + + rte_eth_stats_reset(test_params->bonded_port_id); + + + TEST_ASSERT_EQUAL( + rte_eth_tx_burst(test_params->bonded_port_id, 0, tx_pkt_burst, + burst_size), burst_size, "rte_eth_tx_burst failed"); + + rte_eth_stats_get(test_params->bonded_port_id, &port_stats); + TEST_ASSERT_EQUAL(port_stats.opackets, (uint64_t)burst_size, + "Port (%d) opackets stats (%d) not expected (%d) value", + test_params->bonded_port_id, (int)port_stats.opackets, + burst_size); + + rte_eth_stats_get(test_params->slave_port_ids[0], &port_stats); + TEST_ASSERT_EQUAL(port_stats.opackets, (uint64_t)10, + "Port (%d) opackets stats (%d) not expected (%d) value", + test_params->slave_port_ids[0], (int)port_stats.opackets, 10); + + rte_eth_stats_get(test_params->slave_port_ids[1], &port_stats); + TEST_ASSERT_EQUAL(port_stats.opackets, (uint64_t)0, + "Port (%d) opackets stats (%d) not expected (%d) value", + test_params->slave_port_ids[1], (int)port_stats.opackets, 0); + + rte_eth_stats_get(test_params->slave_port_ids[2], &port_stats); + TEST_ASSERT_EQUAL(port_stats.opackets, (uint64_t)10, + "Port (%d) opackets stats (%d) not expected (%d) value", + test_params->slave_port_ids[2], (int)port_stats.opackets, 10); + + rte_eth_stats_get(test_params->slave_port_ids[3], &port_stats); + TEST_ASSERT_EQUAL(port_stats.opackets, (uint64_t)0, + "Port (%d) opackets stats (%d) not expected (%d) value", + test_params->slave_port_ids[3], (int)port_stats.opackets, 0); + + /* Verify that pkts are not sent on slaves with link status down: + * + * 1. Generate test bursts of traffic + * 2. Add bursts on to virtual eth_devs + * 3. Rx burst on bonded eth_dev, expected (burst_ size * + * TEST_RR_LINK_STATUS_EXPECTED_ACTIVE_SLAVE_COUNT) received + * 4. Verify stats for bonded eth_dev + * 6. Verify stats for slave eth_devs (s0 = 10, s1 = 0, s2 = 10, s3 = 0) + */ + for (i = 0; i < TEST_RR_LINK_STATUS_SLAVE_COUNT; i++) { + TEST_ASSERT_EQUAL(generate_test_burst( + &gen_pkt_burst[i][0], burst_size, 0, 1, 0, 0, 0), + burst_size, "failed to generate packet burst"); + + virtual_ethdev_add_mbufs_to_rx_queue(test_params->slave_port_ids[i], + &gen_pkt_burst[i][0], burst_size); + } + + TEST_ASSERT_EQUAL(rte_eth_rx_burst( + test_params->bonded_port_id, 0, rx_pkt_burst, MAX_PKT_BURST), + burst_size + burst_size, + "rte_eth_rx_burst failed"); + + /* Verify bonded device rx count */ + rte_eth_stats_get(test_params->bonded_port_id, &port_stats); + TEST_ASSERT_EQUAL(port_stats.ipackets , (uint64_t)(burst_size + burst_size), + "(%d) port_stats.ipackets not as expected\n", + test_params->bonded_port_id); + + /* free mbufs */ + for (i = 0; i < MAX_PKT_BURST; i++) { + if (rx_pkt_burst[i] != NULL) + rte_pktmbuf_free(rx_pkt_burst[i]); + } + + /* Clean up and remove slaves from bonded device */ + return remove_slaves_and_stop_bonded_device(); +} + +#define TEST_RR_POLLING_LINK_STATUS_SLAVE_COUNT (2) + +uint8_t polling_slave_mac[] = {0xDE, 0xAD, 0xBE, 0xEF, 0x00, 0x00 }; + + +int polling_test_slaves[TEST_RR_POLLING_LINK_STATUS_SLAVE_COUNT] = { -1, -1 }; + +static int +test_roundrobin_verfiy_polling_slave_link_status_change(void) +{ + struct ether_addr *mac_addr = (struct ether_addr *)polling_slave_mac; + char slave_name[RTE_ETH_NAME_MAX_LEN]; + + int i; + + for (i = 0; i < TEST_RR_POLLING_LINK_STATUS_SLAVE_COUNT; i++) { + /* Generate slave name / MAC address */ + snprintf(slave_name, RTE_ETH_NAME_MAX_LEN, "eth_virt_poll_%d", i); + mac_addr->addr_bytes[ETHER_ADDR_LEN-1] = i; + + /* Create slave devices with no ISR Support */ + if (polling_test_slaves[i] == -1) { + polling_test_slaves[i] = virtual_ethdev_create(slave_name, mac_addr, + rte_socket_id(), 0); + TEST_ASSERT(polling_test_slaves[i] >= 0, + "Failed to create virtual virtual ethdev %s\n", slave_name); + + /* Configure slave */ + TEST_ASSERT_SUCCESS(configure_ethdev(polling_test_slaves[i], 0, 0), + "Failed to configure virtual ethdev %s(%d)", slave_name, + polling_test_slaves[i]); + } + + /* Add slave to bonded device */ + TEST_ASSERT_SUCCESS(rte_eth_bond_slave_add(test_params->bonded_port_id, + polling_test_slaves[i]), + "Failed to add slave %s(%d) to bonded device %d", + slave_name, polling_test_slaves[i], + test_params->bonded_port_id); + } + + /* Initialize bonded device */ + TEST_ASSERT_SUCCESS(configure_ethdev(test_params->bonded_port_id, 1, 1), + "Failed to configure bonded device %d", + test_params->bonded_port_id); + + + /* Register link status change interrupt callback */ + rte_eth_dev_callback_register(test_params->bonded_port_id, + RTE_ETH_EVENT_INTR_LSC, test_bonding_lsc_event_callback, + &test_params->bonded_port_id); + + /* link status change callback for first slave link up */ + test_lsc_interrupt_count = 0; + + virtual_ethdev_set_link_status(polling_test_slaves[0], 1); + + TEST_ASSERT_SUCCESS(lsc_timeout(15000), "timed out waiting for interrupt"); + + + /* no link status change callback for second slave link up */ + test_lsc_interrupt_count = 0; + + virtual_ethdev_set_link_status(polling_test_slaves[1], 1); + + TEST_ASSERT_FAIL(lsc_timeout(15000), "unexpectedly succeeded"); + + /* link status change callback for both slave links down */ + test_lsc_interrupt_count = 0; + + virtual_ethdev_set_link_status(polling_test_slaves[0], 0); + virtual_ethdev_set_link_status(polling_test_slaves[1], 0); + + TEST_ASSERT_SUCCESS(lsc_timeout(20000), "timed out waiting for interrupt"); + + /* Un-Register link status change interrupt callback */ + rte_eth_dev_callback_unregister(test_params->bonded_port_id, + RTE_ETH_EVENT_INTR_LSC, test_bonding_lsc_event_callback, + &test_params->bonded_port_id); + + + /* Clean up and remove slaves from bonded device */ + for (i = 0; i < TEST_RR_POLLING_LINK_STATUS_SLAVE_COUNT; i++) { + + TEST_ASSERT_SUCCESS( + rte_eth_bond_slave_remove(test_params->bonded_port_id, + polling_test_slaves[i]), + "Failed to remove slave %d from bonded port (%d)", + polling_test_slaves[i], test_params->bonded_port_id); + } + + return remove_slaves_and_stop_bonded_device(); +} + + +/** Active Backup Mode Tests */ + +static int +test_activebackup_tx_burst(void) +{ + int i, pktlen, primary_port, burst_size; + struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; + struct rte_eth_stats port_stats; + + TEST_ASSERT_SUCCESS(initialize_bonded_device_with_slaves( + BONDING_MODE_ACTIVE_BACKUP, 0, 1, 1), + "Failed to initialize bonded device with slaves"); + + initialize_eth_header(test_params->pkt_eth_hdr, + (struct ether_addr *)src_mac, (struct ether_addr *)dst_mac_0, + ETHER_TYPE_IPv4, 0, 0); + pktlen = initialize_udp_header(test_params->pkt_udp_hdr, src_port, + dst_port_0, 16); + pktlen = initialize_ipv4_header(test_params->pkt_ipv4_hdr, src_addr, + dst_addr_0, pktlen); + + burst_size = 20 * test_params->bonded_slave_count; + + TEST_ASSERT(burst_size < MAX_PKT_BURST, + "Burst size specified is greater than supported."); + + /* Generate a burst of packets to transmit */ + TEST_ASSERT_EQUAL(generate_packet_burst(test_params->mbuf_pool, pkts_burst, + test_params->pkt_eth_hdr, 0, test_params->pkt_ipv4_hdr, 1, + test_params->pkt_udp_hdr, burst_size, PACKET_BURST_GEN_PKT_LEN, 1), + burst_size, "failed to generate burst correctly"); + + /* Send burst on bonded port */ + TEST_ASSERT_EQUAL(rte_eth_tx_burst(test_params->bonded_port_id, 0, pkts_burst, + burst_size), burst_size, "tx burst failed"); + + /* Verify bonded port tx stats */ + rte_eth_stats_get(test_params->bonded_port_id, &port_stats); + TEST_ASSERT_EQUAL(port_stats.opackets, (uint64_t)burst_size, + "Bonded Port (%d) opackets value (%u) not as expected (%d)", + test_params->bonded_port_id, (unsigned int)port_stats.opackets, + burst_size); + + primary_port = rte_eth_bond_primary_get(test_params->bonded_port_id); + + /* Verify slave ports tx stats */ + for (i = 0; i < test_params->bonded_slave_count; i++) { + rte_eth_stats_get(test_params->slave_port_ids[i], &port_stats); + if (test_params->slave_port_ids[i] == primary_port) { + TEST_ASSERT_EQUAL(port_stats.opackets, (uint64_t)burst_size, + "Slave Port (%d) opackets value (%u) not as expected (%d)", + test_params->bonded_port_id, + (unsigned int)port_stats.opackets, + burst_size / test_params->bonded_slave_count); + } else { + TEST_ASSERT_EQUAL(port_stats.opackets, 0, + "Slave Port (%d) opackets value (%u) not as expected (%d)", + test_params->bonded_port_id, + (unsigned int)port_stats.opackets, 0); + } + } + + /* Put all slaves down and try and transmit */ + for (i = 0; i < test_params->bonded_slave_count; i++) { + virtual_ethdev_simulate_link_status_interrupt( + test_params->slave_port_ids[i], 0); + } + + /* Send burst on bonded port */ + TEST_ASSERT_EQUAL(rte_eth_tx_burst(test_params->bonded_port_id, 0, + pkts_burst, burst_size), 0, "Sending empty burst failed"); + + /* Clean up and remove slaves from bonded device */ + return remove_slaves_and_stop_bonded_device(); +} + +#define TEST_ACTIVE_BACKUP_RX_BURST_SLAVE_COUNT (4) + +static int +test_activebackup_rx_burst(void) +{ + struct rte_mbuf *gen_pkt_burst[MAX_PKT_BURST] = { NULL }; + struct rte_mbuf *rx_pkt_burst[MAX_PKT_BURST] = { NULL }; + + struct rte_eth_stats port_stats; + + int primary_port; + + int i, j, burst_size = 17; + + TEST_ASSERT_SUCCESS(initialize_bonded_device_with_slaves( + BONDING_MODE_ACTIVE_BACKUP, 0, + TEST_ACTIVE_BACKUP_RX_BURST_SLAVE_COUNT, 1), + "Failed to initialize bonded device with slaves"); + + primary_port = rte_eth_bond_primary_get(test_params->bonded_port_id); + TEST_ASSERT(primary_port >= 0, + "failed to get primary slave for bonded port (%d)", + test_params->bonded_port_id); + + for (i = 0; i < test_params->bonded_slave_count; i++) { + /* Generate test bursts of packets to transmit */ + TEST_ASSERT_EQUAL(generate_test_burst( + &gen_pkt_burst[0], burst_size, 0, 1, 0, 0, 0), + burst_size, "burst generation failed"); + + /* Add rx data to slave */ + virtual_ethdev_add_mbufs_to_rx_queue(test_params->slave_port_ids[i], + &gen_pkt_burst[0], burst_size); + + /* Call rx burst on bonded device */ + TEST_ASSERT_EQUAL(rte_eth_rx_burst(test_params->bonded_port_id, 0, + &rx_pkt_burst[0], MAX_PKT_BURST), burst_size, + "rte_eth_rx_burst failed"); + + if (test_params->slave_port_ids[i] == primary_port) { + /* Verify bonded device rx count */ + rte_eth_stats_get(test_params->bonded_port_id, &port_stats); + TEST_ASSERT_EQUAL(port_stats.ipackets, (uint64_t)burst_size, + "Bonded Port (%d) ipackets value (%u) not as expected (%d)", + test_params->bonded_port_id, + (unsigned int)port_stats.ipackets, burst_size); + + /* Verify bonded slave devices rx count */ + for (j = 0; j < test_params->bonded_slave_count; j++) { + rte_eth_stats_get(test_params->slave_port_ids[j], &port_stats); + if (i == j) { + TEST_ASSERT_EQUAL(port_stats.ipackets, (uint64_t)burst_size, + "Slave Port (%d) ipackets value (%u) not as " + "expected (%d)", test_params->slave_port_ids[i], + (unsigned int)port_stats.ipackets, burst_size); + } else { + TEST_ASSERT_EQUAL(port_stats.ipackets, 0, + "Slave Port (%d) ipackets value (%u) not as " + "expected (%d)\n", test_params->slave_port_ids[i], + (unsigned int)port_stats.ipackets, 0); + } + } + } else { + for (j = 0; j < test_params->bonded_slave_count; j++) { + rte_eth_stats_get(test_params->slave_port_ids[j], &port_stats); + TEST_ASSERT_EQUAL(port_stats.ipackets, 0, + "Slave Port (%d) ipackets value (%u) not as expected " + "(%d)", test_params->slave_port_ids[i], + (unsigned int)port_stats.ipackets, 0); + } + } + + /* free mbufs */ + for (i = 0; i < MAX_PKT_BURST; i++) { + if (rx_pkt_burst[i] != NULL) { + rte_pktmbuf_free(rx_pkt_burst[i]); + rx_pkt_burst[i] = NULL; + } + } + + /* reset bonded device stats */ + rte_eth_stats_reset(test_params->bonded_port_id); + } + + /* Clean up and remove slaves from bonded device */ + return remove_slaves_and_stop_bonded_device(); +} + +static int +test_activebackup_verify_promiscuous_enable_disable(void) +{ + int i, primary_port, promiscuous_en; + + /* Initialize bonded device with 4 slaves in round robin mode */ + TEST_ASSERT_SUCCESS(initialize_bonded_device_with_slaves( + BONDING_MODE_ACTIVE_BACKUP, 0, 4, 1), + "Failed to initialize bonded device with slaves"); + + primary_port = rte_eth_bond_primary_get(test_params->bonded_port_id); + TEST_ASSERT(primary_port >= 0, + "failed to get primary slave for bonded port (%d)", + test_params->bonded_port_id); + + rte_eth_promiscuous_enable(test_params->bonded_port_id); + + TEST_ASSERT_EQUAL(rte_eth_promiscuous_get(test_params->bonded_port_id), 1, + "Port (%d) promiscuous mode not enabled", + test_params->bonded_port_id); + + for (i = 0; i < test_params->bonded_slave_count; i++) { + promiscuous_en = rte_eth_promiscuous_get( + test_params->slave_port_ids[i]); + if (primary_port == test_params->slave_port_ids[i]) { + TEST_ASSERT_EQUAL(promiscuous_en, 1, + "slave port (%d) promiscuous mode not enabled", + test_params->slave_port_ids[i]); + } else { + TEST_ASSERT_EQUAL(promiscuous_en, 0, + "slave port (%d) promiscuous mode enabled", + test_params->slave_port_ids[i]); + } + + } + + rte_eth_promiscuous_disable(test_params->bonded_port_id); + + TEST_ASSERT_EQUAL(rte_eth_promiscuous_get(test_params->bonded_port_id), 0, + "Port (%d) promiscuous mode not disabled\n", + test_params->bonded_port_id); + + for (i = 0; i < test_params->bonded_slave_count; i++) { + promiscuous_en = rte_eth_promiscuous_get( + test_params->slave_port_ids[i]); + TEST_ASSERT_EQUAL(promiscuous_en, 0, + "slave port (%d) promiscuous mode not disabled\n", + test_params->slave_port_ids[i]); + } + + /* Clean up and remove slaves from bonded device */ + return remove_slaves_and_stop_bonded_device(); +} + +static int +test_activebackup_verify_mac_assignment(void) +{ + struct ether_addr read_mac_addr, expected_mac_addr_0, expected_mac_addr_1; + + rte_eth_macaddr_get(test_params->slave_port_ids[0], &expected_mac_addr_0); + rte_eth_macaddr_get(test_params->slave_port_ids[1], &expected_mac_addr_1); + + /* Initialize bonded device with 2 slaves in active backup mode */ + TEST_ASSERT_SUCCESS(initialize_bonded_device_with_slaves( + BONDING_MODE_ACTIVE_BACKUP, 0, 2, 1), + "Failed to initialize bonded device with slaves"); + + /* Verify that bonded MACs is that of first slave and that the other slave + * MAC hasn't been changed */ + rte_eth_macaddr_get(test_params->bonded_port_id, &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(&expected_mac_addr_0, &read_mac_addr, + sizeof(read_mac_addr)), + "bonded port (%d) mac address not set to that of primary port", + test_params->bonded_port_id); + + rte_eth_macaddr_get(test_params->slave_port_ids[0], &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(&expected_mac_addr_0, &read_mac_addr, + sizeof(read_mac_addr)), + "slave port (%d) mac address not set to that of primary port", + test_params->slave_port_ids[0]); + + rte_eth_macaddr_get(test_params->slave_port_ids[1], &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(&expected_mac_addr_1, &read_mac_addr, + sizeof(read_mac_addr)), + "slave port (%d) mac address not as expected", + test_params->slave_port_ids[1]); + + /* change primary and verify that MAC addresses haven't changed */ + TEST_ASSERT_EQUAL(rte_eth_bond_primary_set(test_params->bonded_port_id, + test_params->slave_port_ids[1]), 0, + "Failed to set bonded port (%d) primary port to (%d)", + test_params->bonded_port_id, test_params->slave_port_ids[1]); + + rte_eth_macaddr_get(test_params->bonded_port_id, &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(&expected_mac_addr_0, &read_mac_addr, + sizeof(read_mac_addr)), + "bonded port (%d) mac address not set to that of primary port", + test_params->bonded_port_id); + + rte_eth_macaddr_get(test_params->slave_port_ids[0], &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(&expected_mac_addr_0, &read_mac_addr, + sizeof(read_mac_addr)), + "slave port (%d) mac address not set to that of primary port", + test_params->slave_port_ids[0]); + + rte_eth_macaddr_get(test_params->slave_port_ids[1], &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(&expected_mac_addr_1, &read_mac_addr, + sizeof(read_mac_addr)), + "slave port (%d) mac address not as expected", + test_params->slave_port_ids[1]); + + /* stop / start bonded device and verify that primary MAC address is + * propagated to bonded device and slaves */ + + rte_eth_dev_stop(test_params->bonded_port_id); + + TEST_ASSERT_SUCCESS(rte_eth_dev_start(test_params->bonded_port_id), + "Failed to start device"); + + rte_eth_macaddr_get(test_params->bonded_port_id, &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(&expected_mac_addr_1, &read_mac_addr, + sizeof(read_mac_addr)), + "bonded port (%d) mac address not set to that of primary port", + test_params->bonded_port_id); + + rte_eth_macaddr_get(test_params->slave_port_ids[0], &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(&expected_mac_addr_0, &read_mac_addr, + sizeof(read_mac_addr)), + "slave port (%d) mac address not as expected", + test_params->slave_port_ids[0]); + + rte_eth_macaddr_get(test_params->slave_port_ids[1], &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(&expected_mac_addr_1, &read_mac_addr, + sizeof(read_mac_addr)), + "slave port (%d) mac address not set to that of primary port", + test_params->slave_port_ids[1]); + + /* Set explicit MAC address */ + TEST_ASSERT_SUCCESS(rte_eth_bond_mac_address_set( + test_params->bonded_port_id, (struct ether_addr *)bonded_mac), + "failed to set MAC address"); + + rte_eth_macaddr_get(test_params->bonded_port_id, &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(&bonded_mac, &read_mac_addr, + sizeof(read_mac_addr)), + "bonded port (%d) mac address not set to that of bonded port", + test_params->bonded_port_id); + + rte_eth_macaddr_get(test_params->slave_port_ids[0], &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(&expected_mac_addr_0, &read_mac_addr, + sizeof(read_mac_addr)), + "slave port (%d) mac address not as expected", + test_params->slave_port_ids[0]); + + rte_eth_macaddr_get(test_params->slave_port_ids[1], &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(&bonded_mac, &read_mac_addr, + sizeof(read_mac_addr)), + "slave port (%d) mac address not set to that of bonded port", + test_params->slave_port_ids[1]); + + /* Clean up and remove slaves from bonded device */ + return remove_slaves_and_stop_bonded_device(); +} + +static int +test_activebackup_verify_slave_link_status_change_failover(void) +{ + struct rte_mbuf *pkt_burst[TEST_ACTIVE_BACKUP_RX_BURST_SLAVE_COUNT][MAX_PKT_BURST]; + struct rte_mbuf *rx_pkt_burst[MAX_PKT_BURST] = { NULL }; + struct rte_eth_stats port_stats; + + uint16_t slaves[RTE_MAX_ETHPORTS]; + + int i, burst_size, slave_count, primary_port; + + burst_size = 21; + + memset(pkt_burst, 0, sizeof(pkt_burst)); + + /* Generate packet burst for testing */ + TEST_ASSERT_EQUAL(generate_test_burst( + &pkt_burst[0][0], burst_size, 0, 1, 0, 0, 0), burst_size, + "generate_test_burst failed"); + + /* Initialize bonded device with 4 slaves in round robin mode */ + TEST_ASSERT_SUCCESS(initialize_bonded_device_with_slaves( + BONDING_MODE_ACTIVE_BACKUP, 0, + TEST_ACTIVE_BACKUP_RX_BURST_SLAVE_COUNT, 1), + "Failed to initialize bonded device with slaves"); + + /* Verify Current Slaves Count /Active Slave Count is */ + slave_count = rte_eth_bond_slaves_get(test_params->bonded_port_id, slaves, + RTE_MAX_ETHPORTS); + TEST_ASSERT_EQUAL(slave_count, 4, + "Number of slaves (%d) is not as expected (%d).", + slave_count, 4); + + slave_count = rte_eth_bond_active_slaves_get(test_params->bonded_port_id, + slaves, RTE_MAX_ETHPORTS); + TEST_ASSERT_EQUAL(slave_count, 4, + "Number of active slaves (%d) is not as expected (%d).", + slave_count, 4); + + primary_port = rte_eth_bond_primary_get(test_params->bonded_port_id); + TEST_ASSERT_EQUAL(primary_port, test_params->slave_port_ids[0], + "Primary port not as expected"); + + /* Bring 2 slaves down and verify active slave count */ + virtual_ethdev_simulate_link_status_interrupt( + test_params->slave_port_ids[1], 0); + virtual_ethdev_simulate_link_status_interrupt( + test_params->slave_port_ids[3], 0); + + TEST_ASSERT_EQUAL(rte_eth_bond_active_slaves_get( + test_params->bonded_port_id, slaves, RTE_MAX_ETHPORTS), 2, + "Number of active slaves (%d) is not as expected (%d).", + slave_count, 2); + + virtual_ethdev_simulate_link_status_interrupt( + test_params->slave_port_ids[1], 1); + virtual_ethdev_simulate_link_status_interrupt( + test_params->slave_port_ids[3], 1); + + + /* Bring primary port down, verify that active slave count is 3 and primary + * has changed */ + virtual_ethdev_simulate_link_status_interrupt( + test_params->slave_port_ids[0], 0); + + TEST_ASSERT_EQUAL(rte_eth_bond_active_slaves_get( + test_params->bonded_port_id, slaves, RTE_MAX_ETHPORTS), + 3, + "Number of active slaves (%d) is not as expected (%d).", + slave_count, 3); + + primary_port = rte_eth_bond_primary_get(test_params->bonded_port_id); + TEST_ASSERT_EQUAL(primary_port, test_params->slave_port_ids[2], + "Primary port not as expected"); + + /* Verify that pkts are sent on new primary slave */ + + TEST_ASSERT_EQUAL(rte_eth_tx_burst( + test_params->bonded_port_id, 0, &pkt_burst[0][0], + burst_size), burst_size, "rte_eth_tx_burst failed"); + + rte_eth_stats_get(test_params->slave_port_ids[2], &port_stats); + TEST_ASSERT_EQUAL(port_stats.opackets, (uint64_t)burst_size, + "(%d) port_stats.opackets not as expected", + test_params->slave_port_ids[2]); + + rte_eth_stats_get(test_params->slave_port_ids[0], &port_stats); + TEST_ASSERT_EQUAL(port_stats.opackets, 0, + "(%d) port_stats.opackets not as expected\n", + test_params->slave_port_ids[0]); + + rte_eth_stats_get(test_params->slave_port_ids[1], &port_stats); + TEST_ASSERT_EQUAL(port_stats.opackets, 0, + "(%d) port_stats.opackets not as expected\n", + test_params->slave_port_ids[1]); + + rte_eth_stats_get(test_params->slave_port_ids[3], &port_stats); + TEST_ASSERT_EQUAL(port_stats.opackets, 0, + "(%d) port_stats.opackets not as expected\n", + test_params->slave_port_ids[3]); + + /* Generate packet burst for testing */ + + for (i = 0; i < TEST_ACTIVE_BACKUP_RX_BURST_SLAVE_COUNT; i++) { + TEST_ASSERT_EQUAL(generate_test_burst( + &pkt_burst[i][0], burst_size, 0, 1, 0, 0, 0), burst_size, + "generate_test_burst failed"); + + virtual_ethdev_add_mbufs_to_rx_queue( + test_params->slave_port_ids[i], &pkt_burst[i][0], burst_size); + } + + TEST_ASSERT_EQUAL(rte_eth_rx_burst( + test_params->bonded_port_id, 0, rx_pkt_burst, MAX_PKT_BURST), + burst_size, "rte_eth_rx_burst\n"); + + /* Verify bonded device rx count */ + rte_eth_stats_get(test_params->bonded_port_id, &port_stats); + TEST_ASSERT_EQUAL(port_stats.ipackets, (uint64_t)burst_size, + "(%d) port_stats.ipackets not as expected", + test_params->bonded_port_id); + + rte_eth_stats_get(test_params->slave_port_ids[2], &port_stats); + TEST_ASSERT_EQUAL(port_stats.opackets, (uint64_t)burst_size, + "(%d) port_stats.opackets not as expected", + test_params->slave_port_ids[2]); + + rte_eth_stats_get(test_params->slave_port_ids[0], &port_stats); + TEST_ASSERT_EQUAL(port_stats.opackets, 0, + "(%d) port_stats.opackets not as expected", + test_params->slave_port_ids[0]); + + rte_eth_stats_get(test_params->slave_port_ids[1], &port_stats); + TEST_ASSERT_EQUAL(port_stats.opackets, 0, + "(%d) port_stats.opackets not as expected", + test_params->slave_port_ids[1]); + + rte_eth_stats_get(test_params->slave_port_ids[3], &port_stats); + TEST_ASSERT_EQUAL(port_stats.opackets, 0, + "(%d) port_stats.opackets not as expected", + test_params->slave_port_ids[3]); + + /* Clean up and remove slaves from bonded device */ + return remove_slaves_and_stop_bonded_device(); +} + +/** Balance Mode Tests */ + +static int +test_balance_xmit_policy_configuration(void) +{ + TEST_ASSERT_SUCCESS(initialize_bonded_device_with_slaves( + BONDING_MODE_ACTIVE_BACKUP, 0, 2, 1), + "Failed to initialize_bonded_device_with_slaves."); + + /* Invalid port id */ + TEST_ASSERT_FAIL(rte_eth_bond_xmit_policy_set( + INVALID_PORT_ID, BALANCE_XMIT_POLICY_LAYER2), + "Expected call to failed as invalid port specified."); + + /* Set xmit policy on non bonded device */ + TEST_ASSERT_FAIL(rte_eth_bond_xmit_policy_set( + test_params->slave_port_ids[0], BALANCE_XMIT_POLICY_LAYER2), + "Expected call to failed as invalid port specified."); + + + TEST_ASSERT_SUCCESS(rte_eth_bond_xmit_policy_set( + test_params->bonded_port_id, BALANCE_XMIT_POLICY_LAYER2), + "Failed to set balance xmit policy."); + + TEST_ASSERT_EQUAL(rte_eth_bond_xmit_policy_get(test_params->bonded_port_id), + BALANCE_XMIT_POLICY_LAYER2, "balance xmit policy not as expected."); + + + TEST_ASSERT_SUCCESS(rte_eth_bond_xmit_policy_set( + test_params->bonded_port_id, BALANCE_XMIT_POLICY_LAYER23), + "Failed to set balance xmit policy."); + + TEST_ASSERT_EQUAL(rte_eth_bond_xmit_policy_get(test_params->bonded_port_id), + BALANCE_XMIT_POLICY_LAYER23, + "balance xmit policy not as expected."); + + + TEST_ASSERT_SUCCESS(rte_eth_bond_xmit_policy_set( + test_params->bonded_port_id, BALANCE_XMIT_POLICY_LAYER34), + "Failed to set balance xmit policy."); + + TEST_ASSERT_EQUAL(rte_eth_bond_xmit_policy_get(test_params->bonded_port_id), + BALANCE_XMIT_POLICY_LAYER34, + "balance xmit policy not as expected."); + + /* Invalid port id */ + TEST_ASSERT_FAIL(rte_eth_bond_xmit_policy_get(INVALID_PORT_ID), + "Expected call to failed as invalid port specified."); + + /* Clean up and remove slaves from bonded device */ + return remove_slaves_and_stop_bonded_device(); +} + +#define TEST_BALANCE_L2_TX_BURST_SLAVE_COUNT (2) + +static int +test_balance_l2_tx_burst(void) +{ + struct rte_mbuf *pkts_burst[TEST_BALANCE_L2_TX_BURST_SLAVE_COUNT][MAX_PKT_BURST]; + int burst_size[TEST_BALANCE_L2_TX_BURST_SLAVE_COUNT] = { 10, 15 }; + + uint16_t pktlen; + int i; + struct rte_eth_stats port_stats; + + TEST_ASSERT_SUCCESS(initialize_bonded_device_with_slaves( + BONDING_MODE_BALANCE, 0, TEST_BALANCE_L2_TX_BURST_SLAVE_COUNT, 1), + "Failed to initialize_bonded_device_with_slaves."); + + TEST_ASSERT_SUCCESS(rte_eth_bond_xmit_policy_set( + test_params->bonded_port_id, BALANCE_XMIT_POLICY_LAYER2), + "Failed to set balance xmit policy."); + + initialize_eth_header(test_params->pkt_eth_hdr, + (struct ether_addr *)src_mac, (struct ether_addr *)dst_mac_0, + ETHER_TYPE_IPv4, 0, 0); + pktlen = initialize_udp_header(test_params->pkt_udp_hdr, src_port, + dst_port_0, 16); + pktlen = initialize_ipv4_header(test_params->pkt_ipv4_hdr, src_addr, + dst_addr_0, pktlen); + + /* Generate a burst 1 of packets to transmit */ + TEST_ASSERT_EQUAL(generate_packet_burst(test_params->mbuf_pool, &pkts_burst[0][0], + test_params->pkt_eth_hdr, 0, test_params->pkt_ipv4_hdr, 1, + test_params->pkt_udp_hdr, burst_size[0], + PACKET_BURST_GEN_PKT_LEN, 1), burst_size[0], + "failed to generate packet burst"); + + initialize_eth_header(test_params->pkt_eth_hdr, + (struct ether_addr *)src_mac, (struct ether_addr *)dst_mac_1, + ETHER_TYPE_IPv4, 0, 0); + + /* Generate a burst 2 of packets to transmit */ + TEST_ASSERT_EQUAL(generate_packet_burst(test_params->mbuf_pool, &pkts_burst[1][0], + test_params->pkt_eth_hdr, 0, test_params->pkt_ipv4_hdr, 1, + test_params->pkt_udp_hdr, burst_size[1], + PACKET_BURST_GEN_PKT_LEN, 1), burst_size[1], + "failed to generate packet burst"); + + /* Send burst 1 on bonded port */ + for (i = 0; i < TEST_BALANCE_L2_TX_BURST_SLAVE_COUNT; i++) { + TEST_ASSERT_EQUAL(rte_eth_tx_burst(test_params->bonded_port_id, 0, + &pkts_burst[i][0], burst_size[i]), + burst_size[i], "Failed to transmit packet burst"); + } + + /* Verify bonded port tx stats */ + rte_eth_stats_get(test_params->bonded_port_id, &port_stats); + TEST_ASSERT_EQUAL(port_stats.opackets, + (uint64_t)(burst_size[0] + burst_size[1]), + "Bonded Port (%d) opackets value (%u) not as expected (%d)", + test_params->bonded_port_id, (unsigned int)port_stats.opackets, + burst_size[0] + burst_size[1]); + + + /* Verify slave ports tx stats */ + rte_eth_stats_get(test_params->slave_port_ids[0], &port_stats); + TEST_ASSERT_EQUAL(port_stats.opackets, (uint64_t)burst_size[0], + "Slave Port (%d) opackets value (%u) not as expected (%d)", + test_params->slave_port_ids[0], (unsigned int)port_stats.opackets, + burst_size[0]); + + rte_eth_stats_get(test_params->slave_port_ids[1], &port_stats); + TEST_ASSERT_EQUAL(port_stats.opackets, (uint64_t)burst_size[1], + "Slave Port (%d) opackets value (%u) not as expected (%d)\n", + test_params->slave_port_ids[1], (unsigned int)port_stats.opackets, + burst_size[1]); + + /* Put all slaves down and try and transmit */ + for (i = 0; i < test_params->bonded_slave_count; i++) { + + virtual_ethdev_simulate_link_status_interrupt( + test_params->slave_port_ids[i], 0); + } + + /* Send burst on bonded port */ + TEST_ASSERT_EQUAL(rte_eth_tx_burst( + test_params->bonded_port_id, 0, &pkts_burst[0][0], burst_size[0]), + 0, "Expected zero packet"); + + /* Clean up and remove slaves from bonded device */ + return remove_slaves_and_stop_bonded_device(); +} + +static int +balance_l23_tx_burst(uint8_t vlan_enabled, uint8_t ipv4, + uint8_t toggle_mac_addr, uint8_t toggle_ip_addr) +{ + int i, burst_size_1, burst_size_2, nb_tx_1, nb_tx_2; + + struct rte_mbuf *pkts_burst_1[MAX_PKT_BURST]; + struct rte_mbuf *pkts_burst_2[MAX_PKT_BURST]; + + struct rte_eth_stats port_stats; + + TEST_ASSERT_SUCCESS(initialize_bonded_device_with_slaves( + BONDING_MODE_BALANCE, 0, 2, 1), + "Failed to initialize_bonded_device_with_slaves."); + + TEST_ASSERT_SUCCESS(rte_eth_bond_xmit_policy_set( + test_params->bonded_port_id, BALANCE_XMIT_POLICY_LAYER23), + "Failed to set balance xmit policy."); + + burst_size_1 = 20; + burst_size_2 = 10; + + TEST_ASSERT(burst_size_1 < MAX_PKT_BURST || burst_size_2 < MAX_PKT_BURST, + "Burst size specified is greater than supported."); + + /* Generate test bursts of packets to transmit */ + TEST_ASSERT_EQUAL(generate_test_burst( + pkts_burst_1, burst_size_1, vlan_enabled, ipv4, 0, 0, 0), + burst_size_1, "failed to generate packet burst"); + + TEST_ASSERT_EQUAL(generate_test_burst(pkts_burst_2, burst_size_2, vlan_enabled, ipv4, + toggle_mac_addr, toggle_ip_addr, 0), burst_size_2, + "failed to generate packet burst"); + + /* Send burst 1 on bonded port */ + nb_tx_1 = rte_eth_tx_burst(test_params->bonded_port_id, 0, pkts_burst_1, + burst_size_1); + TEST_ASSERT_EQUAL(nb_tx_1, burst_size_1, "tx burst failed"); + + /* Send burst 2 on bonded port */ + nb_tx_2 = rte_eth_tx_burst(test_params->bonded_port_id, 0, pkts_burst_2, + burst_size_2); + TEST_ASSERT_EQUAL(nb_tx_2, burst_size_2, "tx burst failed"); + + /* Verify bonded port tx stats */ + rte_eth_stats_get(test_params->bonded_port_id, &port_stats); + TEST_ASSERT_EQUAL(port_stats.opackets, (uint64_t)(nb_tx_1 + nb_tx_2), + "Bonded Port (%d) opackets value (%u) not as expected (%d)", + test_params->bonded_port_id, (unsigned int)port_stats.opackets, + nb_tx_1 + nb_tx_2); + + /* Verify slave ports tx stats */ + rte_eth_stats_get(test_params->slave_port_ids[0], &port_stats); + TEST_ASSERT_EQUAL(port_stats.opackets, (uint64_t)nb_tx_1, + "Slave Port (%d) opackets value (%u) not as expected (%d)", + test_params->slave_port_ids[0], (unsigned int)port_stats.opackets, + nb_tx_1); + + rte_eth_stats_get(test_params->slave_port_ids[1], &port_stats); + TEST_ASSERT_EQUAL(port_stats.opackets, (uint64_t)nb_tx_2, + "Slave Port (%d) opackets value (%u) not as expected (%d)", + test_params->slave_port_ids[1], (unsigned int)port_stats.opackets, + nb_tx_2); + + /* Put all slaves down and try and transmit */ + for (i = 0; i < test_params->bonded_slave_count; i++) { + + virtual_ethdev_simulate_link_status_interrupt( + test_params->slave_port_ids[i], 0); + } + + /* Send burst on bonded port */ + TEST_ASSERT_EQUAL(rte_eth_tx_burst( + test_params->bonded_port_id, 0, pkts_burst_1, + burst_size_1), 0, "Expected zero packet"); + + + /* Clean up and remove slaves from bonded device */ + return remove_slaves_and_stop_bonded_device(); +} + +static int +test_balance_l23_tx_burst_ipv4_toggle_ip_addr(void) +{ + return balance_l23_tx_burst(0, 1, 0, 1); +} + +static int +test_balance_l23_tx_burst_vlan_ipv4_toggle_ip_addr(void) +{ + return balance_l23_tx_burst(1, 1, 0, 1); +} + +static int +test_balance_l23_tx_burst_ipv6_toggle_ip_addr(void) +{ + return balance_l23_tx_burst(0, 0, 0, 1); +} + +static int +test_balance_l23_tx_burst_vlan_ipv6_toggle_ip_addr(void) +{ + return balance_l23_tx_burst(1, 0, 0, 1); +} + +static int +test_balance_l23_tx_burst_toggle_mac_addr(void) +{ + return balance_l23_tx_burst(0, 0, 1, 0); +} + +static int +balance_l34_tx_burst(uint8_t vlan_enabled, uint8_t ipv4, + uint8_t toggle_mac_addr, uint8_t toggle_ip_addr, + uint8_t toggle_udp_port) +{ + int i, burst_size_1, burst_size_2, nb_tx_1, nb_tx_2; + + struct rte_mbuf *pkts_burst_1[MAX_PKT_BURST]; + struct rte_mbuf *pkts_burst_2[MAX_PKT_BURST]; + + struct rte_eth_stats port_stats; + + TEST_ASSERT_SUCCESS(initialize_bonded_device_with_slaves( + BONDING_MODE_BALANCE, 0, 2, 1), + "Failed to initialize_bonded_device_with_slaves."); + + TEST_ASSERT_SUCCESS(rte_eth_bond_xmit_policy_set( + test_params->bonded_port_id, BALANCE_XMIT_POLICY_LAYER34), + "Failed to set balance xmit policy."); + + burst_size_1 = 20; + burst_size_2 = 10; + + TEST_ASSERT(burst_size_1 < MAX_PKT_BURST || burst_size_2 < MAX_PKT_BURST, + "Burst size specified is greater than supported."); + + /* Generate test bursts of packets to transmit */ + TEST_ASSERT_EQUAL(generate_test_burst( + pkts_burst_1, burst_size_1, vlan_enabled, ipv4, 0, 0, 0), + burst_size_1, "failed to generate burst"); + + TEST_ASSERT_EQUAL(generate_test_burst(pkts_burst_2, burst_size_2, + vlan_enabled, ipv4, toggle_mac_addr, toggle_ip_addr, + toggle_udp_port), burst_size_2, "failed to generate burst"); + + /* Send burst 1 on bonded port */ + nb_tx_1 = rte_eth_tx_burst(test_params->bonded_port_id, 0, pkts_burst_1, + burst_size_1); + TEST_ASSERT_EQUAL(nb_tx_1, burst_size_1, "tx burst failed"); + + /* Send burst 2 on bonded port */ + nb_tx_2 = rte_eth_tx_burst(test_params->bonded_port_id, 0, pkts_burst_2, + burst_size_2); + TEST_ASSERT_EQUAL(nb_tx_2, burst_size_2, "tx burst failed"); + + + /* Verify bonded port tx stats */ + rte_eth_stats_get(test_params->bonded_port_id, &port_stats); + TEST_ASSERT_EQUAL(port_stats.opackets, (uint64_t)(nb_tx_1 + nb_tx_2), + "Bonded Port (%d) opackets value (%u) not as expected (%d)", + test_params->bonded_port_id, (unsigned int)port_stats.opackets, + nb_tx_1 + nb_tx_2); + + /* Verify slave ports tx stats */ + rte_eth_stats_get(test_params->slave_port_ids[0], &port_stats); + TEST_ASSERT_EQUAL(port_stats.opackets, (uint64_t)nb_tx_1, + "Slave Port (%d) opackets value (%u) not as expected (%d)", + test_params->slave_port_ids[0], (unsigned int)port_stats.opackets, + nb_tx_1); + + rte_eth_stats_get(test_params->slave_port_ids[1], &port_stats); + TEST_ASSERT_EQUAL(port_stats.opackets, (uint64_t)nb_tx_2, + "Slave Port (%d) opackets value (%u) not as expected (%d)", + test_params->slave_port_ids[1], (unsigned int)port_stats.opackets, + nb_tx_2); + + /* Put all slaves down and try and transmit */ + for (i = 0; i < test_params->bonded_slave_count; i++) { + + virtual_ethdev_simulate_link_status_interrupt( + test_params->slave_port_ids[i], 0); + } + + /* Send burst on bonded port */ + TEST_ASSERT_EQUAL(rte_eth_tx_burst( + test_params->bonded_port_id, 0, pkts_burst_1, + burst_size_1), 0, "Expected zero packet"); + + /* Clean up and remove slaves from bonded device */ + return remove_slaves_and_stop_bonded_device(); +} + +static int +test_balance_l34_tx_burst_ipv4_toggle_ip_addr(void) +{ + return balance_l34_tx_burst(0, 1, 0, 1, 0); +} + +static int +test_balance_l34_tx_burst_ipv4_toggle_udp_port(void) +{ + return balance_l34_tx_burst(0, 1, 0, 0, 1); +} + +static int +test_balance_l34_tx_burst_vlan_ipv4_toggle_ip_addr(void) +{ + return balance_l34_tx_burst(1, 1, 0, 1, 0); +} + +static int +test_balance_l34_tx_burst_ipv6_toggle_ip_addr(void) +{ + return balance_l34_tx_burst(0, 0, 0, 1, 0); +} + +static int +test_balance_l34_tx_burst_vlan_ipv6_toggle_ip_addr(void) +{ + return balance_l34_tx_burst(1, 0, 0, 1, 0); +} + +static int +test_balance_l34_tx_burst_ipv6_toggle_udp_port(void) +{ + return balance_l34_tx_burst(0, 0, 0, 0, 1); +} + +#define TEST_BAL_SLAVE_TX_FAIL_SLAVE_COUNT (2) +#define TEST_BAL_SLAVE_TX_FAIL_BURST_SIZE_1 (40) +#define TEST_BAL_SLAVE_TX_FAIL_BURST_SIZE_2 (20) +#define TEST_BAL_SLAVE_TX_FAIL_PACKETS_COUNT (25) +#define TEST_BAL_SLAVE_TX_FAIL_FAILING_SLAVE_IDX (0) + +static int +test_balance_tx_burst_slave_tx_fail(void) +{ + struct rte_mbuf *pkts_burst_1[TEST_BAL_SLAVE_TX_FAIL_BURST_SIZE_1]; + struct rte_mbuf *pkts_burst_2[TEST_BAL_SLAVE_TX_FAIL_BURST_SIZE_2]; + + struct rte_mbuf *expected_fail_pkts[TEST_BAL_SLAVE_TX_FAIL_PACKETS_COUNT]; + + struct rte_eth_stats port_stats; + + int i, first_tx_fail_idx, tx_count_1, tx_count_2; + + TEST_ASSERT_SUCCESS(initialize_bonded_device_with_slaves( + BONDING_MODE_BALANCE, 0, + TEST_BAL_SLAVE_TX_FAIL_SLAVE_COUNT, 1), + "Failed to initialise bonded device"); + + TEST_ASSERT_SUCCESS(rte_eth_bond_xmit_policy_set( + test_params->bonded_port_id, BALANCE_XMIT_POLICY_LAYER2), + "Failed to set balance xmit policy."); + + + /* Generate test bursts for transmission */ + TEST_ASSERT_EQUAL(generate_test_burst(pkts_burst_1, + TEST_BAL_SLAVE_TX_FAIL_BURST_SIZE_1, 0, 0, 0, 0, 0), + TEST_BAL_SLAVE_TX_FAIL_BURST_SIZE_1, + "Failed to generate test packet burst 1"); + + first_tx_fail_idx = TEST_BAL_SLAVE_TX_FAIL_BURST_SIZE_1 - + TEST_BAL_SLAVE_TX_FAIL_PACKETS_COUNT; + + /* copy mbuf referneces for expected transmission failures */ + for (i = 0; i < TEST_BAL_SLAVE_TX_FAIL_PACKETS_COUNT; i++) + expected_fail_pkts[i] = pkts_burst_1[i + first_tx_fail_idx]; + + TEST_ASSERT_EQUAL(generate_test_burst(pkts_burst_2, + TEST_BAL_SLAVE_TX_FAIL_BURST_SIZE_2, 0, 0, 1, 0, 0), + TEST_BAL_SLAVE_TX_FAIL_BURST_SIZE_2, + "Failed to generate test packet burst 2"); + + + /* Set virtual slave TEST_BAL_SLAVE_TX_FAIL_FAILING_SLAVE_IDX to only fail + * transmission of TEST_BAL_SLAVE_TX_FAIL_PACKETS_COUNT packets of burst */ + virtual_ethdev_tx_burst_fn_set_success( + test_params->slave_port_ids[TEST_BAL_SLAVE_TX_FAIL_FAILING_SLAVE_IDX], + 0); + + virtual_ethdev_tx_burst_fn_set_tx_pkt_fail_count( + test_params->slave_port_ids[TEST_BAL_SLAVE_TX_FAIL_FAILING_SLAVE_IDX], + TEST_BAL_SLAVE_TX_FAIL_PACKETS_COUNT); + + + /* Transmit burst 1 */ + tx_count_1 = rte_eth_tx_burst(test_params->bonded_port_id, 0, pkts_burst_1, + TEST_BAL_SLAVE_TX_FAIL_BURST_SIZE_1); + + TEST_ASSERT_EQUAL(tx_count_1, TEST_BAL_SLAVE_TX_FAIL_BURST_SIZE_1 - + TEST_BAL_SLAVE_TX_FAIL_PACKETS_COUNT, + "Transmitted (%d) packets, expected to transmit (%d) packets", + tx_count_1, TEST_BAL_SLAVE_TX_FAIL_BURST_SIZE_1 - + TEST_BAL_SLAVE_TX_FAIL_PACKETS_COUNT); + + /* Verify that failed packet are expected failed packets */ + for (i = 0; i < TEST_RR_SLAVE_TX_FAIL_PACKETS_COUNT; i++) { + TEST_ASSERT_EQUAL(expected_fail_pkts[i], pkts_burst_1[i + tx_count_1], + "expected mbuf (%d) pointer %p not expected pointer %p", + i, expected_fail_pkts[i], pkts_burst_1[i + tx_count_1]); + } + + /* Transmit burst 2 */ + tx_count_2 = rte_eth_tx_burst(test_params->bonded_port_id, 0, pkts_burst_2, + TEST_BAL_SLAVE_TX_FAIL_BURST_SIZE_2); + + TEST_ASSERT_EQUAL(tx_count_2, TEST_BAL_SLAVE_TX_FAIL_BURST_SIZE_2, + "Transmitted (%d) packets, expected to transmit (%d) packets", + tx_count_2, TEST_BAL_SLAVE_TX_FAIL_BURST_SIZE_2); + + + /* Verify bonded port tx stats */ + rte_eth_stats_get(test_params->bonded_port_id, &port_stats); + + TEST_ASSERT_EQUAL(port_stats.opackets, + (uint64_t)((TEST_BAL_SLAVE_TX_FAIL_BURST_SIZE_1 - + TEST_BAL_SLAVE_TX_FAIL_PACKETS_COUNT) + + TEST_BAL_SLAVE_TX_FAIL_BURST_SIZE_2), + "Bonded Port (%d) opackets value (%u) not as expected (%d)", + test_params->bonded_port_id, (unsigned int)port_stats.opackets, + (TEST_BAL_SLAVE_TX_FAIL_BURST_SIZE_1 - + TEST_BAL_SLAVE_TX_FAIL_PACKETS_COUNT) + + TEST_BAL_SLAVE_TX_FAIL_BURST_SIZE_2); + + /* Verify slave ports tx stats */ + + rte_eth_stats_get(test_params->slave_port_ids[0], &port_stats); + + TEST_ASSERT_EQUAL(port_stats.opackets, (uint64_t) + TEST_BAL_SLAVE_TX_FAIL_BURST_SIZE_1 - + TEST_BAL_SLAVE_TX_FAIL_PACKETS_COUNT, + "Slave Port (%d) opackets value (%u) not as expected (%d)", + test_params->slave_port_ids[0], + (unsigned int)port_stats.opackets, + TEST_BAL_SLAVE_TX_FAIL_BURST_SIZE_1 - + TEST_BAL_SLAVE_TX_FAIL_PACKETS_COUNT); + + + + + rte_eth_stats_get(test_params->slave_port_ids[1], &port_stats); + + TEST_ASSERT_EQUAL(port_stats.opackets, + (uint64_t)TEST_BAL_SLAVE_TX_FAIL_BURST_SIZE_2, + "Slave Port (%d) opackets value (%u) not as expected (%d)", + test_params->slave_port_ids[1], + (unsigned int)port_stats.opackets, + TEST_BAL_SLAVE_TX_FAIL_BURST_SIZE_2); + + /* Verify that all mbufs have a ref value of zero */ + TEST_ASSERT_SUCCESS(verify_mbufs_ref_count(&pkts_burst_1[tx_count_1], + TEST_BAL_SLAVE_TX_FAIL_PACKETS_COUNT, 1), + "mbufs refcnts not as expected"); + + free_mbufs(&pkts_burst_1[tx_count_1], + TEST_BAL_SLAVE_TX_FAIL_PACKETS_COUNT); + + /* Clean up and remove slaves from bonded device */ + return remove_slaves_and_stop_bonded_device(); +} + +#define TEST_BALANCE_RX_BURST_SLAVE_COUNT (3) + +static int +test_balance_rx_burst(void) +{ + struct rte_mbuf *gen_pkt_burst[TEST_BALANCE_RX_BURST_SLAVE_COUNT][MAX_PKT_BURST]; + + struct rte_mbuf *rx_pkt_burst[MAX_PKT_BURST] = { NULL }; + struct rte_eth_stats port_stats; + + int burst_size[TEST_BALANCE_RX_BURST_SLAVE_COUNT] = { 10, 5, 30 }; + int i, j; + + memset(gen_pkt_burst, 0, sizeof(gen_pkt_burst)); + + /* Initialize bonded device with 4 slaves in round robin mode */ + TEST_ASSERT_SUCCESS(initialize_bonded_device_with_slaves( + BONDING_MODE_BALANCE, 0, 3, 1), + "Failed to initialise bonded device"); + + /* Generate test bursts of packets to transmit */ + for (i = 0; i < TEST_BALANCE_RX_BURST_SLAVE_COUNT; i++) { + TEST_ASSERT_EQUAL(generate_test_burst( + &gen_pkt_burst[i][0], burst_size[i], 0, 0, 1, + 0, 0), burst_size[i], + "failed to generate packet burst"); + } + + /* Add rx data to slaves */ + for (i = 0; i < TEST_BALANCE_RX_BURST_SLAVE_COUNT; i++) { + virtual_ethdev_add_mbufs_to_rx_queue(test_params->slave_port_ids[i], + &gen_pkt_burst[i][0], burst_size[i]); + } + + /* Call rx burst on bonded device */ + /* Send burst on bonded port */ + TEST_ASSERT_EQUAL(rte_eth_rx_burst(test_params->bonded_port_id, 0, + rx_pkt_burst, MAX_PKT_BURST), + burst_size[0] + burst_size[1] + burst_size[2], + "balance rx burst failed\n"); + + /* Verify bonded device rx count */ + rte_eth_stats_get(test_params->bonded_port_id, &port_stats); + TEST_ASSERT_EQUAL(port_stats.ipackets, + (uint64_t)(burst_size[0] + burst_size[1] + burst_size[2]), + "Bonded Port (%d) ipackets value (%u) not as expected (%d)", + test_params->bonded_port_id, (unsigned int)port_stats.ipackets, + burst_size[0] + burst_size[1] + burst_size[2]); + + + /* Verify bonded slave devices rx counts */ + rte_eth_stats_get(test_params->slave_port_ids[0], &port_stats); + TEST_ASSERT_EQUAL(port_stats.ipackets, (uint64_t)burst_size[0], + "Slave Port (%d) ipackets value (%u) not as expected (%d)", + test_params->slave_port_ids[0], + (unsigned int)port_stats.ipackets, burst_size[0]); + + rte_eth_stats_get(test_params->slave_port_ids[1], &port_stats); + TEST_ASSERT_EQUAL(port_stats.ipackets, (uint64_t)burst_size[1], + "Slave Port (%d) ipackets value (%u) not as expected (%d)", + test_params->slave_port_ids[1], (unsigned int)port_stats.ipackets, + burst_size[1]); + + rte_eth_stats_get(test_params->slave_port_ids[2], &port_stats); + TEST_ASSERT_EQUAL(port_stats.ipackets, (uint64_t)burst_size[2], + "Slave Port (%d) ipackets value (%u) not as expected (%d)", + test_params->slave_port_ids[2], (unsigned int)port_stats.ipackets, + burst_size[2]); + + rte_eth_stats_get(test_params->slave_port_ids[3], &port_stats); + TEST_ASSERT_EQUAL(port_stats.ipackets, 0, + "Slave Port (%d) ipackets value (%u) not as expected (%d)", + test_params->slave_port_ids[3], (unsigned int)port_stats.ipackets, + 0); + + /* free mbufs */ + for (i = 0; i < TEST_BALANCE_RX_BURST_SLAVE_COUNT; i++) { + for (j = 0; j < MAX_PKT_BURST; j++) { + if (gen_pkt_burst[i][j] != NULL) { + rte_pktmbuf_free(gen_pkt_burst[i][j]); + gen_pkt_burst[i][j] = NULL; + } + } + } + + /* Clean up and remove slaves from bonded device */ + return remove_slaves_and_stop_bonded_device(); +} + +static int +test_balance_verify_promiscuous_enable_disable(void) +{ + int i; + + /* Initialize bonded device with 4 slaves in round robin mode */ + TEST_ASSERT_SUCCESS(initialize_bonded_device_with_slaves( + BONDING_MODE_BALANCE, 0, 4, 1), + "Failed to initialise bonded device"); + + rte_eth_promiscuous_enable(test_params->bonded_port_id); + + TEST_ASSERT_EQUAL(rte_eth_promiscuous_get(test_params->bonded_port_id), 1, + "Port (%d) promiscuous mode not enabled", + test_params->bonded_port_id); + + for (i = 0; i < test_params->bonded_slave_count; i++) { + TEST_ASSERT_EQUAL(rte_eth_promiscuous_get( + test_params->slave_port_ids[i]), 1, + "Port (%d) promiscuous mode not enabled", + test_params->slave_port_ids[i]); + } + + rte_eth_promiscuous_disable(test_params->bonded_port_id); + + TEST_ASSERT_EQUAL(rte_eth_promiscuous_get(test_params->bonded_port_id), 0, + "Port (%d) promiscuous mode not disabled", + test_params->bonded_port_id); + + for (i = 0; i < test_params->bonded_slave_count; i++) { + TEST_ASSERT_EQUAL(rte_eth_promiscuous_get( + test_params->slave_port_ids[i]), 0, + "Port (%d) promiscuous mode not disabled", + test_params->slave_port_ids[i]); + } + + /* Clean up and remove slaves from bonded device */ + return remove_slaves_and_stop_bonded_device(); +} + +static int +test_balance_verify_mac_assignment(void) +{ + struct ether_addr read_mac_addr, expected_mac_addr_0, expected_mac_addr_1; + + rte_eth_macaddr_get(test_params->slave_port_ids[0], &expected_mac_addr_0); + rte_eth_macaddr_get(test_params->slave_port_ids[1], &expected_mac_addr_1); + + /* Initialize bonded device with 2 slaves in active backup mode */ + TEST_ASSERT_SUCCESS(initialize_bonded_device_with_slaves( + BONDING_MODE_BALANCE, 0, 2, 1), + "Failed to initialise bonded device"); + + /* Verify that bonded MACs is that of first slave and that the other slave + * MAC hasn't been changed */ + rte_eth_macaddr_get(test_params->bonded_port_id, &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(&expected_mac_addr_0, &read_mac_addr, + sizeof(read_mac_addr)), + "bonded port (%d) mac address not set to that of primary port", + test_params->bonded_port_id); + + rte_eth_macaddr_get(test_params->slave_port_ids[0], &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(&expected_mac_addr_0, &read_mac_addr, + sizeof(read_mac_addr)), + "slave port (%d) mac address not set to that of primary port", + test_params->slave_port_ids[0]); + + rte_eth_macaddr_get(test_params->slave_port_ids[1], &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(&expected_mac_addr_0, &read_mac_addr, + sizeof(read_mac_addr)), + "slave port (%d) mac address not set to that of primary port", + test_params->slave_port_ids[1]); + + /* change primary and verify that MAC addresses haven't changed */ + TEST_ASSERT_SUCCESS(rte_eth_bond_primary_set(test_params->bonded_port_id, + test_params->slave_port_ids[1]), + "Failed to set bonded port (%d) primary port to (%d)\n", + test_params->bonded_port_id, test_params->slave_port_ids[1]); + + rte_eth_macaddr_get(test_params->bonded_port_id, &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(&expected_mac_addr_0, &read_mac_addr, + sizeof(read_mac_addr)), + "bonded port (%d) mac address not set to that of primary port", + test_params->bonded_port_id); + + rte_eth_macaddr_get(test_params->slave_port_ids[0], &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(&expected_mac_addr_0, &read_mac_addr, + sizeof(read_mac_addr)), + "slave port (%d) mac address not set to that of primary port", + test_params->slave_port_ids[0]); + + rte_eth_macaddr_get(test_params->slave_port_ids[1], &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(&expected_mac_addr_0, &read_mac_addr, + sizeof(read_mac_addr)), + "slave port (%d) mac address not set to that of primary port", + test_params->slave_port_ids[1]); + + /* stop / start bonded device and verify that primary MAC address is + * propagated to bonded device and slaves */ + + rte_eth_dev_stop(test_params->bonded_port_id); + + TEST_ASSERT_SUCCESS(rte_eth_dev_start(test_params->bonded_port_id), + "Failed to start bonded device"); + + rte_eth_macaddr_get(test_params->bonded_port_id, &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(&expected_mac_addr_1, &read_mac_addr, + sizeof(read_mac_addr)), + "bonded port (%d) mac address not set to that of primary port", + test_params->bonded_port_id); + + rte_eth_macaddr_get(test_params->slave_port_ids[0], &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(&expected_mac_addr_1, &read_mac_addr, + sizeof(read_mac_addr)), + "slave port (%d) mac address not set to that of primary port", + test_params->slave_port_ids[0]); + + rte_eth_macaddr_get(test_params->slave_port_ids[1], &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(&expected_mac_addr_1, &read_mac_addr, + sizeof(read_mac_addr)), + "slave port (%d) mac address not set to that of primary port", + test_params->slave_port_ids[1]); + + /* Set explicit MAC address */ + TEST_ASSERT_SUCCESS(rte_eth_bond_mac_address_set( + test_params->bonded_port_id, (struct ether_addr *)bonded_mac), + "failed to set MAC"); + + rte_eth_macaddr_get(test_params->bonded_port_id, &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(&bonded_mac, &read_mac_addr, + sizeof(read_mac_addr)), + "bonded port (%d) mac address not set to that of bonded port", + test_params->bonded_port_id); + + rte_eth_macaddr_get(test_params->slave_port_ids[0], &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(&bonded_mac, &read_mac_addr, + sizeof(read_mac_addr)), + "slave port (%d) mac address not as expected\n", + test_params->slave_port_ids[0]); + + rte_eth_macaddr_get(test_params->slave_port_ids[1], &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(&bonded_mac, &read_mac_addr, + sizeof(read_mac_addr)), + "slave port (%d) mac address not set to that of bonded port", + test_params->slave_port_ids[1]); + + /* Clean up and remove slaves from bonded device */ + return remove_slaves_and_stop_bonded_device(); +} + +#define TEST_BALANCE_LINK_STATUS_SLAVE_COUNT (4) + +static int +test_balance_verify_slave_link_status_change_behaviour(void) +{ + struct rte_mbuf *pkt_burst[TEST_BALANCE_LINK_STATUS_SLAVE_COUNT][MAX_PKT_BURST]; + struct rte_mbuf *rx_pkt_burst[MAX_PKT_BURST] = { NULL }; + struct rte_eth_stats port_stats; + + uint16_t slaves[RTE_MAX_ETHPORTS]; + + int i, burst_size, slave_count; + + memset(pkt_burst, 0, sizeof(pkt_burst)); + + /* Initialize bonded device with 4 slaves in round robin mode */ + TEST_ASSERT_SUCCESS(initialize_bonded_device_with_slaves( + BONDING_MODE_BALANCE, 0, TEST_BALANCE_LINK_STATUS_SLAVE_COUNT, 1), + "Failed to initialise bonded device"); + + TEST_ASSERT_SUCCESS(rte_eth_bond_xmit_policy_set( + test_params->bonded_port_id, BALANCE_XMIT_POLICY_LAYER2), + "Failed to set balance xmit policy."); + + + /* Verify Current Slaves Count /Active Slave Count is */ + slave_count = rte_eth_bond_slaves_get(test_params->bonded_port_id, slaves, + RTE_MAX_ETHPORTS); + TEST_ASSERT_EQUAL(slave_count, TEST_BALANCE_LINK_STATUS_SLAVE_COUNT, + "Number of slaves (%d) is not as expected (%d).", + slave_count, TEST_BALANCE_LINK_STATUS_SLAVE_COUNT); + + slave_count = rte_eth_bond_active_slaves_get(test_params->bonded_port_id, + slaves, RTE_MAX_ETHPORTS); + TEST_ASSERT_EQUAL(slave_count, TEST_BALANCE_LINK_STATUS_SLAVE_COUNT, + "Number of active slaves (%d) is not as expected (%d).", + slave_count, TEST_BALANCE_LINK_STATUS_SLAVE_COUNT); + + /* Set 2 slaves link status to down */ + virtual_ethdev_simulate_link_status_interrupt( + test_params->slave_port_ids[1], 0); + virtual_ethdev_simulate_link_status_interrupt( + test_params->slave_port_ids[3], 0); + + TEST_ASSERT_EQUAL(rte_eth_bond_active_slaves_get( + test_params->bonded_port_id, slaves, RTE_MAX_ETHPORTS), 2, + "Number of active slaves (%d) is not as expected (%d).", + slave_count, 2); + + /* Send to sets of packet burst and verify that they are balanced across + * slaves */ + burst_size = 21; + + TEST_ASSERT_EQUAL(generate_test_burst( + &pkt_burst[0][0], burst_size, 0, 1, 0, 0, 0), burst_size, + "generate_test_burst failed"); + + TEST_ASSERT_EQUAL(generate_test_burst( + &pkt_burst[1][0], burst_size, 0, 1, 1, 0, 0), burst_size, + "generate_test_burst failed"); + + TEST_ASSERT_EQUAL(rte_eth_tx_burst( + test_params->bonded_port_id, 0, &pkt_burst[0][0], burst_size), + burst_size, "rte_eth_tx_burst failed"); + + TEST_ASSERT_EQUAL(rte_eth_tx_burst( + test_params->bonded_port_id, 0, &pkt_burst[1][0], burst_size), + burst_size, "rte_eth_tx_burst failed"); + + + rte_eth_stats_get(test_params->bonded_port_id, &port_stats); + TEST_ASSERT_EQUAL(port_stats.opackets, (uint64_t)(burst_size + burst_size), + "(%d) port_stats.opackets (%d) not as expected (%d).", + test_params->bonded_port_id, (int)port_stats.opackets, + burst_size + burst_size); + + rte_eth_stats_get(test_params->slave_port_ids[0], &port_stats); + TEST_ASSERT_EQUAL(port_stats.opackets, (uint64_t)burst_size, + "(%d) port_stats.opackets (%d) not as expected (%d).", + test_params->slave_port_ids[0], (int)port_stats.opackets, + burst_size); + + rte_eth_stats_get(test_params->slave_port_ids[2], &port_stats); + TEST_ASSERT_EQUAL(port_stats.opackets, (uint64_t)burst_size, + "(%d) port_stats.opackets (%d) not as expected (%d).", + test_params->slave_port_ids[2], (int)port_stats.opackets, + burst_size); + + /* verify that all packets get send on primary slave when no other slaves + * are available */ + virtual_ethdev_simulate_link_status_interrupt( + test_params->slave_port_ids[2], 0); + + TEST_ASSERT_EQUAL(rte_eth_bond_active_slaves_get( + test_params->bonded_port_id, slaves, RTE_MAX_ETHPORTS), 1, + "Number of active slaves (%d) is not as expected (%d).", + slave_count, 1); + + TEST_ASSERT_EQUAL(generate_test_burst( + &pkt_burst[1][0], burst_size, 0, 1, 1, 0, 0), burst_size, + "generate_test_burst failed"); + + TEST_ASSERT_EQUAL(rte_eth_tx_burst( + test_params->bonded_port_id, 0, &pkt_burst[1][0], burst_size), + burst_size, "rte_eth_tx_burst failed"); + + rte_eth_stats_get(test_params->bonded_port_id, &port_stats); + TEST_ASSERT_EQUAL(port_stats.opackets, + (uint64_t)(burst_size + burst_size + burst_size), + "(%d) port_stats.opackets (%d) not as expected (%d).\n", + test_params->bonded_port_id, (int)port_stats.opackets, + burst_size + burst_size + burst_size); + + rte_eth_stats_get(test_params->slave_port_ids[0], &port_stats); + TEST_ASSERT_EQUAL(port_stats.opackets, (uint64_t)(burst_size + burst_size), + "(%d) port_stats.opackets (%d) not as expected (%d).", + test_params->slave_port_ids[0], (int)port_stats.opackets, + burst_size + burst_size); + + virtual_ethdev_simulate_link_status_interrupt( + test_params->slave_port_ids[0], 0); + virtual_ethdev_simulate_link_status_interrupt( + test_params->slave_port_ids[1], 1); + virtual_ethdev_simulate_link_status_interrupt( + test_params->slave_port_ids[2], 1); + virtual_ethdev_simulate_link_status_interrupt( + test_params->slave_port_ids[3], 1); + + for (i = 0; i < TEST_BALANCE_LINK_STATUS_SLAVE_COUNT; i++) { + TEST_ASSERT_EQUAL(generate_test_burst( + &pkt_burst[i][0], burst_size, 0, 1, 0, 0, 0), burst_size, + "Failed to generate packet burst"); + + virtual_ethdev_add_mbufs_to_rx_queue(test_params->slave_port_ids[i], + &pkt_burst[i][0], burst_size); + } + + /* Verify that pkts are not received on slaves with link status down */ + + rte_eth_rx_burst(test_params->bonded_port_id, 0, rx_pkt_burst, + MAX_PKT_BURST); + + /* Verify bonded device rx count */ + rte_eth_stats_get(test_params->bonded_port_id, &port_stats); + TEST_ASSERT_EQUAL(port_stats.ipackets, (uint64_t)(burst_size * 3), + "(%d) port_stats.ipackets (%d) not as expected (%d)\n", + test_params->bonded_port_id, (int)port_stats.ipackets, + burst_size * 3); + + /* Clean up and remove slaves from bonded device */ + return remove_slaves_and_stop_bonded_device(); +} + +static int +test_broadcast_tx_burst(void) +{ + int i, pktlen, burst_size; + struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; + + struct rte_eth_stats port_stats; + + TEST_ASSERT_SUCCESS(initialize_bonded_device_with_slaves( + BONDING_MODE_BROADCAST, 0, 2, 1), + "Failed to initialise bonded device"); + + initialize_eth_header(test_params->pkt_eth_hdr, + (struct ether_addr *)src_mac, (struct ether_addr *)dst_mac_0, + ETHER_TYPE_IPv4, 0, 0); + + pktlen = initialize_udp_header(test_params->pkt_udp_hdr, src_port, + dst_port_0, 16); + pktlen = initialize_ipv4_header(test_params->pkt_ipv4_hdr, src_addr, + dst_addr_0, pktlen); + + burst_size = 20 * test_params->bonded_slave_count; + + TEST_ASSERT(burst_size < MAX_PKT_BURST, + "Burst size specified is greater than supported."); + + /* Generate a burst of packets to transmit */ + TEST_ASSERT_EQUAL(generate_packet_burst(test_params->mbuf_pool, + pkts_burst, test_params->pkt_eth_hdr, 0, test_params->pkt_ipv4_hdr, + 1, test_params->pkt_udp_hdr, burst_size, PACKET_BURST_GEN_PKT_LEN, + 1), burst_size, "Failed to generate packet burst"); + + /* Send burst on bonded port */ + TEST_ASSERT_EQUAL(rte_eth_tx_burst(test_params->bonded_port_id, 0, + pkts_burst, burst_size), burst_size, + "Bonded Port (%d) rx burst failed, packets transmitted value " + "not as expected (%d)", + test_params->bonded_port_id, burst_size); + + /* Verify bonded port tx stats */ + rte_eth_stats_get(test_params->bonded_port_id, &port_stats); + TEST_ASSERT_EQUAL(port_stats.opackets, + (uint64_t)burst_size * test_params->bonded_slave_count, + "Bonded Port (%d) opackets value (%u) not as expected (%d)", + test_params->bonded_port_id, (unsigned int)port_stats.opackets, + burst_size); + + /* Verify slave ports tx stats */ + for (i = 0; i < test_params->bonded_slave_count; i++) { + rte_eth_stats_get(test_params->slave_port_ids[i], &port_stats); + TEST_ASSERT_EQUAL(port_stats.opackets, (uint64_t)burst_size, + "Slave Port (%d) opackets value (%u) not as expected (%d)\n", + test_params->bonded_port_id, + (unsigned int)port_stats.opackets, burst_size); + } + + /* Put all slaves down and try and transmit */ + for (i = 0; i < test_params->bonded_slave_count; i++) { + + virtual_ethdev_simulate_link_status_interrupt( + test_params->slave_port_ids[i], 0); + } + + /* Send burst on bonded port */ + TEST_ASSERT_EQUAL(rte_eth_tx_burst( + test_params->bonded_port_id, 0, pkts_burst, burst_size), 0, + "transmitted an unexpected number of packets"); + + /* Clean up and remove slaves from bonded device */ + return remove_slaves_and_stop_bonded_device(); +} + + +#define TEST_BCAST_SLAVE_TX_FAIL_SLAVE_COUNT (3) +#define TEST_BCAST_SLAVE_TX_FAIL_BURST_SIZE (40) +#define TEST_BCAST_SLAVE_TX_FAIL_MAX_PACKETS_COUNT (15) +#define TEST_BCAST_SLAVE_TX_FAIL_MIN_PACKETS_COUNT (10) + +static int +test_broadcast_tx_burst_slave_tx_fail(void) +{ + struct rte_mbuf *pkts_burst[TEST_BCAST_SLAVE_TX_FAIL_BURST_SIZE]; + struct rte_mbuf *expected_fail_pkts[TEST_BCAST_SLAVE_TX_FAIL_MIN_PACKETS_COUNT]; + + struct rte_eth_stats port_stats; + + int i, tx_count; + + TEST_ASSERT_SUCCESS(initialize_bonded_device_with_slaves( + BONDING_MODE_BROADCAST, 0, + TEST_BCAST_SLAVE_TX_FAIL_SLAVE_COUNT, 1), + "Failed to initialise bonded device"); + + /* Generate test bursts for transmission */ + TEST_ASSERT_EQUAL(generate_test_burst(pkts_burst, + TEST_BCAST_SLAVE_TX_FAIL_BURST_SIZE, 0, 0, 0, 0, 0), + TEST_BCAST_SLAVE_TX_FAIL_BURST_SIZE, + "Failed to generate test packet burst"); + + for (i = 0; i < TEST_BCAST_SLAVE_TX_FAIL_MIN_PACKETS_COUNT; i++) { + expected_fail_pkts[i] = pkts_burst[TEST_BCAST_SLAVE_TX_FAIL_BURST_SIZE - + TEST_BCAST_SLAVE_TX_FAIL_MIN_PACKETS_COUNT + i]; + } + + /* Set virtual slave TEST_BAL_SLAVE_TX_FAIL_FAILING_SLAVE_IDX to only fail + * transmission of TEST_BAL_SLAVE_TX_FAIL_PACKETS_COUNT packets of burst */ + virtual_ethdev_tx_burst_fn_set_success( + test_params->slave_port_ids[0], + 0); + virtual_ethdev_tx_burst_fn_set_success( + test_params->slave_port_ids[1], + 0); + virtual_ethdev_tx_burst_fn_set_success( + test_params->slave_port_ids[2], + 0); + + virtual_ethdev_tx_burst_fn_set_tx_pkt_fail_count( + test_params->slave_port_ids[0], + TEST_BCAST_SLAVE_TX_FAIL_MAX_PACKETS_COUNT); + + virtual_ethdev_tx_burst_fn_set_tx_pkt_fail_count( + test_params->slave_port_ids[1], + TEST_BCAST_SLAVE_TX_FAIL_MIN_PACKETS_COUNT); + + virtual_ethdev_tx_burst_fn_set_tx_pkt_fail_count( + test_params->slave_port_ids[2], + TEST_BCAST_SLAVE_TX_FAIL_MAX_PACKETS_COUNT); + + /* Transmit burst */ + tx_count = rte_eth_tx_burst(test_params->bonded_port_id, 0, pkts_burst, + TEST_BCAST_SLAVE_TX_FAIL_BURST_SIZE); + + TEST_ASSERT_EQUAL(tx_count, TEST_BCAST_SLAVE_TX_FAIL_BURST_SIZE - + TEST_BCAST_SLAVE_TX_FAIL_MIN_PACKETS_COUNT, + "Transmitted (%d) packets, expected to transmit (%d) packets", + tx_count, TEST_BCAST_SLAVE_TX_FAIL_BURST_SIZE - + TEST_BCAST_SLAVE_TX_FAIL_MIN_PACKETS_COUNT); + + /* Verify that failed packet are expected failed packets */ + for (i = 0; i < TEST_BCAST_SLAVE_TX_FAIL_MIN_PACKETS_COUNT; i++) { + TEST_ASSERT_EQUAL(expected_fail_pkts[i], pkts_burst[i + tx_count], + "expected mbuf (%d) pointer %p not expected pointer %p", + i, expected_fail_pkts[i], pkts_burst[i + tx_count]); + } + + /* Verify slave ports tx stats */ + + rte_eth_stats_get(test_params->slave_port_ids[0], &port_stats); + + TEST_ASSERT_EQUAL(port_stats.opackets, + (uint64_t)TEST_BCAST_SLAVE_TX_FAIL_BURST_SIZE - + TEST_BCAST_SLAVE_TX_FAIL_MAX_PACKETS_COUNT, + "Port (%d) opackets value (%u) not as expected (%d)", + test_params->bonded_port_id, (unsigned int)port_stats.opackets, + TEST_BCAST_SLAVE_TX_FAIL_BURST_SIZE - + TEST_BCAST_SLAVE_TX_FAIL_MAX_PACKETS_COUNT); + + + rte_eth_stats_get(test_params->slave_port_ids[1], &port_stats); + + TEST_ASSERT_EQUAL(port_stats.opackets, + (uint64_t)TEST_BCAST_SLAVE_TX_FAIL_BURST_SIZE - + TEST_BCAST_SLAVE_TX_FAIL_MIN_PACKETS_COUNT, + "Port (%d) opackets value (%u) not as expected (%d)", + test_params->bonded_port_id, (unsigned int)port_stats.opackets, + TEST_BCAST_SLAVE_TX_FAIL_BURST_SIZE - + TEST_BCAST_SLAVE_TX_FAIL_MIN_PACKETS_COUNT); + + rte_eth_stats_get(test_params->slave_port_ids[2], &port_stats); + + TEST_ASSERT_EQUAL(port_stats.opackets, + (uint64_t)TEST_BCAST_SLAVE_TX_FAIL_BURST_SIZE - + TEST_BCAST_SLAVE_TX_FAIL_MAX_PACKETS_COUNT, + "Port (%d) opackets value (%u) not as expected (%d)", + test_params->bonded_port_id, (unsigned int)port_stats.opackets, + TEST_BCAST_SLAVE_TX_FAIL_BURST_SIZE - + TEST_BCAST_SLAVE_TX_FAIL_MAX_PACKETS_COUNT); + + + /* Verify that all mbufs who transmission failed have a ref value of one */ + TEST_ASSERT_SUCCESS(verify_mbufs_ref_count(&pkts_burst[tx_count], + TEST_BCAST_SLAVE_TX_FAIL_MIN_PACKETS_COUNT, 1), + "mbufs refcnts not as expected"); + + free_mbufs(&pkts_burst[tx_count], + TEST_BCAST_SLAVE_TX_FAIL_MIN_PACKETS_COUNT); + + /* Clean up and remove slaves from bonded device */ + return remove_slaves_and_stop_bonded_device(); +} + +#define BROADCAST_RX_BURST_NUM_OF_SLAVES (3) + +static int +test_broadcast_rx_burst(void) +{ + struct rte_mbuf *gen_pkt_burst[BROADCAST_RX_BURST_NUM_OF_SLAVES][MAX_PKT_BURST]; + + struct rte_mbuf *rx_pkt_burst[MAX_PKT_BURST] = { NULL }; + struct rte_eth_stats port_stats; + + int burst_size[BROADCAST_RX_BURST_NUM_OF_SLAVES] = { 10, 5, 30 }; + int i, j; + + memset(gen_pkt_burst, 0, sizeof(gen_pkt_burst)); + + /* Initialize bonded device with 4 slaves in round robin mode */ + TEST_ASSERT_SUCCESS(initialize_bonded_device_with_slaves( + BONDING_MODE_BROADCAST, 0, 3, 1), + "Failed to initialise bonded device"); + + /* Generate test bursts of packets to transmit */ + for (i = 0; i < BROADCAST_RX_BURST_NUM_OF_SLAVES; i++) { + TEST_ASSERT_EQUAL(generate_test_burst( + &gen_pkt_burst[i][0], burst_size[i], 0, 0, 1, 0, 0), + burst_size[i], "failed to generate packet burst"); + } + + /* Add rx data to slave 0 */ + for (i = 0; i < BROADCAST_RX_BURST_NUM_OF_SLAVES; i++) { + virtual_ethdev_add_mbufs_to_rx_queue(test_params->slave_port_ids[i], + &gen_pkt_burst[i][0], burst_size[i]); + } + + + /* Call rx burst on bonded device */ + /* Send burst on bonded port */ + TEST_ASSERT_EQUAL(rte_eth_rx_burst( + test_params->bonded_port_id, 0, rx_pkt_burst, MAX_PKT_BURST), + burst_size[0] + burst_size[1] + burst_size[2], + "rx burst failed"); + + /* Verify bonded device rx count */ + rte_eth_stats_get(test_params->bonded_port_id, &port_stats); + TEST_ASSERT_EQUAL(port_stats.ipackets, + (uint64_t)(burst_size[0] + burst_size[1] + burst_size[2]), + "Bonded Port (%d) ipackets value (%u) not as expected (%d)", + test_params->bonded_port_id, (unsigned int)port_stats.ipackets, + burst_size[0] + burst_size[1] + burst_size[2]); + + + /* Verify bonded slave devices rx counts */ + rte_eth_stats_get(test_params->slave_port_ids[0], &port_stats); + TEST_ASSERT_EQUAL(port_stats.ipackets, (uint64_t)burst_size[0], + "Slave Port (%d) ipackets value (%u) not as expected (%d)", + test_params->slave_port_ids[0], (unsigned int)port_stats.ipackets, + burst_size[0]); + + rte_eth_stats_get(test_params->slave_port_ids[1], &port_stats); + TEST_ASSERT_EQUAL(port_stats.ipackets, (uint64_t)burst_size[1], + "Slave Port (%d) ipackets value (%u) not as expected (%d)", + test_params->slave_port_ids[0], (unsigned int)port_stats.ipackets, + burst_size[1]); + + rte_eth_stats_get(test_params->slave_port_ids[2], &port_stats); + TEST_ASSERT_EQUAL(port_stats.ipackets, (uint64_t)burst_size[2], + "Slave Port (%d) ipackets value (%u) not as expected (%d)", + test_params->slave_port_ids[2], (unsigned int)port_stats.ipackets, + burst_size[2]); + + rte_eth_stats_get(test_params->slave_port_ids[3], &port_stats); + TEST_ASSERT_EQUAL(port_stats.ipackets, 0, + "Slave Port (%d) ipackets value (%u) not as expected (%d)", + test_params->slave_port_ids[3], (unsigned int)port_stats.ipackets, + 0); + + /* free mbufs allocate for rx testing */ + for (i = 0; i < BROADCAST_RX_BURST_NUM_OF_SLAVES; i++) { + for (j = 0; j < MAX_PKT_BURST; j++) { + if (gen_pkt_burst[i][j] != NULL) { + rte_pktmbuf_free(gen_pkt_burst[i][j]); + gen_pkt_burst[i][j] = NULL; + } + } + } + + /* Clean up and remove slaves from bonded device */ + return remove_slaves_and_stop_bonded_device(); +} + +static int +test_broadcast_verify_promiscuous_enable_disable(void) +{ + int i; + + /* Initialize bonded device with 4 slaves in round robin mode */ + TEST_ASSERT_SUCCESS(initialize_bonded_device_with_slaves( + BONDING_MODE_BROADCAST, 0, 4, 1), + "Failed to initialise bonded device"); + + rte_eth_promiscuous_enable(test_params->bonded_port_id); + + + TEST_ASSERT_EQUAL(rte_eth_promiscuous_get(test_params->bonded_port_id), 1, + "Port (%d) promiscuous mode not enabled", + test_params->bonded_port_id); + + for (i = 0; i < test_params->bonded_slave_count; i++) { + TEST_ASSERT_EQUAL(rte_eth_promiscuous_get( + test_params->slave_port_ids[i]), 1, + "Port (%d) promiscuous mode not enabled", + test_params->slave_port_ids[i]); + } + + rte_eth_promiscuous_disable(test_params->bonded_port_id); + + TEST_ASSERT_EQUAL(rte_eth_promiscuous_get(test_params->bonded_port_id), 0, + "Port (%d) promiscuous mode not disabled", + test_params->bonded_port_id); + + for (i = 0; i < test_params->bonded_slave_count; i++) { + TEST_ASSERT_EQUAL(rte_eth_promiscuous_get( + test_params->slave_port_ids[i]), 0, + "Port (%d) promiscuous mode not disabled", + test_params->slave_port_ids[i]); + } + + /* Clean up and remove slaves from bonded device */ + return remove_slaves_and_stop_bonded_device(); +} + +static int +test_broadcast_verify_mac_assignment(void) +{ + struct ether_addr read_mac_addr, expected_mac_addr_0, expected_mac_addr_1; + + int i; + + rte_eth_macaddr_get(test_params->slave_port_ids[0], &expected_mac_addr_0); + rte_eth_macaddr_get(test_params->slave_port_ids[2], &expected_mac_addr_1); + + /* Initialize bonded device with 4 slaves in round robin mode */ + TEST_ASSERT_SUCCESS(initialize_bonded_device_with_slaves( + BONDING_MODE_BROADCAST, 0, 4, 1), + "Failed to initialise bonded device"); + + /* Verify that all MACs are the same as first slave added to bonded + * device */ + for (i = 0; i < test_params->bonded_slave_count; i++) { + rte_eth_macaddr_get(test_params->slave_port_ids[i], &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(&expected_mac_addr_0, &read_mac_addr, + sizeof(read_mac_addr)), + "slave port (%d) mac address not set to that of primary port", + test_params->slave_port_ids[i]); + } + + /* change primary and verify that MAC addresses haven't changed */ + TEST_ASSERT_SUCCESS(rte_eth_bond_primary_set(test_params->bonded_port_id, + test_params->slave_port_ids[2]), + "Failed to set bonded port (%d) primary port to (%d)", + test_params->bonded_port_id, test_params->slave_port_ids[i]); + + for (i = 0; i < test_params->bonded_slave_count; i++) { + rte_eth_macaddr_get(test_params->slave_port_ids[i], &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(&expected_mac_addr_0, &read_mac_addr, + sizeof(read_mac_addr)), + "slave port (%d) mac address has changed to that of primary " + "port without stop/start toggle of bonded device", + test_params->slave_port_ids[i]); + } + + /* stop / start bonded device and verify that primary MAC address is + * propagated to bonded device and slaves */ + + rte_eth_dev_stop(test_params->bonded_port_id); + + TEST_ASSERT_SUCCESS(rte_eth_dev_start(test_params->bonded_port_id), + "Failed to start bonded device"); + + rte_eth_macaddr_get(test_params->bonded_port_id, &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(&expected_mac_addr_1, &read_mac_addr, + sizeof(read_mac_addr)), + "bonded port (%d) mac address not set to that of new primary port", + test_params->slave_port_ids[i]); + + for (i = 0; i < test_params->bonded_slave_count; i++) { + rte_eth_macaddr_get(test_params->slave_port_ids[i], &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(&expected_mac_addr_1, &read_mac_addr, + sizeof(read_mac_addr)), + "slave port (%d) mac address not set to that of new primary " + "port", test_params->slave_port_ids[i]); + } + + /* Set explicit MAC address */ + TEST_ASSERT_SUCCESS(rte_eth_bond_mac_address_set( + test_params->bonded_port_id, (struct ether_addr *)bonded_mac), + "Failed to set MAC address"); + + rte_eth_macaddr_get(test_params->bonded_port_id, &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(bonded_mac, &read_mac_addr, + sizeof(read_mac_addr)), + "bonded port (%d) mac address not set to that of new primary port", + test_params->slave_port_ids[i]); + + + for (i = 0; i < test_params->bonded_slave_count; i++) { + rte_eth_macaddr_get(test_params->slave_port_ids[i], &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(bonded_mac, &read_mac_addr, + sizeof(read_mac_addr)), + "slave port (%d) mac address not set to that of new primary " + "port", test_params->slave_port_ids[i]); + } + + /* Clean up and remove slaves from bonded device */ + return remove_slaves_and_stop_bonded_device(); +} + +#define BROADCAST_LINK_STATUS_NUM_OF_SLAVES (4) +static int +test_broadcast_verify_slave_link_status_change_behaviour(void) +{ + struct rte_mbuf *pkt_burst[BROADCAST_LINK_STATUS_NUM_OF_SLAVES][MAX_PKT_BURST]; + struct rte_mbuf *rx_pkt_burst[MAX_PKT_BURST] = { NULL }; + struct rte_eth_stats port_stats; + + uint16_t slaves[RTE_MAX_ETHPORTS]; + + int i, burst_size, slave_count; + + memset(pkt_burst, 0, sizeof(pkt_burst)); + + /* Initialize bonded device with 4 slaves in round robin mode */ + TEST_ASSERT_SUCCESS(initialize_bonded_device_with_slaves( + BONDING_MODE_BROADCAST, 0, BROADCAST_LINK_STATUS_NUM_OF_SLAVES, + 1), "Failed to initialise bonded device"); + + /* Verify Current Slaves Count /Active Slave Count is */ + slave_count = rte_eth_bond_slaves_get(test_params->bonded_port_id, slaves, + RTE_MAX_ETHPORTS); + TEST_ASSERT_EQUAL(slave_count, 4, + "Number of slaves (%d) is not as expected (%d).", + slave_count, 4); + + slave_count = rte_eth_bond_active_slaves_get(test_params->bonded_port_id, + slaves, RTE_MAX_ETHPORTS); + TEST_ASSERT_EQUAL(slave_count, 4, + "Number of active slaves (%d) is not as expected (%d).", + slave_count, 4); + + /* Set 2 slaves link status to down */ + virtual_ethdev_simulate_link_status_interrupt( + test_params->slave_port_ids[1], 0); + virtual_ethdev_simulate_link_status_interrupt( + test_params->slave_port_ids[3], 0); + + slave_count = rte_eth_bond_active_slaves_get(test_params->bonded_port_id, + slaves, RTE_MAX_ETHPORTS); + TEST_ASSERT_EQUAL(slave_count, 2, + "Number of active slaves (%d) is not as expected (%d).", + slave_count, 2); + + for (i = 0; i < test_params->bonded_slave_count; i++) + rte_eth_stats_reset(test_params->slave_port_ids[i]); + + /* Verify that pkts are not sent on slaves with link status down */ + burst_size = 21; + + TEST_ASSERT_EQUAL(generate_test_burst( + &pkt_burst[0][0], burst_size, 0, 0, 1, 0, 0), burst_size, + "generate_test_burst failed"); + + TEST_ASSERT_EQUAL(rte_eth_tx_burst(test_params->bonded_port_id, 0, + &pkt_burst[0][0], burst_size), burst_size, + "rte_eth_tx_burst failed\n"); + + rte_eth_stats_get(test_params->bonded_port_id, &port_stats); + TEST_ASSERT_EQUAL(port_stats.opackets, (uint64_t)(burst_size * slave_count), + "(%d) port_stats.opackets (%d) not as expected (%d)\n", + test_params->bonded_port_id, (int)port_stats.opackets, + burst_size * slave_count); + + rte_eth_stats_get(test_params->slave_port_ids[0], &port_stats); + TEST_ASSERT_EQUAL(port_stats.opackets, (uint64_t)burst_size, + "(%d) port_stats.opackets not as expected", + test_params->slave_port_ids[0]); + + rte_eth_stats_get(test_params->slave_port_ids[1], &port_stats); + TEST_ASSERT_EQUAL(port_stats.opackets, 0, + "(%d) port_stats.opackets not as expected", + test_params->slave_port_ids[1]); + + rte_eth_stats_get(test_params->slave_port_ids[2], &port_stats); + TEST_ASSERT_EQUAL(port_stats.opackets, (uint64_t)burst_size, + "(%d) port_stats.opackets not as expected", + test_params->slave_port_ids[2]); + + + rte_eth_stats_get(test_params->slave_port_ids[3], &port_stats); + TEST_ASSERT_EQUAL(port_stats.opackets, 0, + "(%d) port_stats.opackets not as expected", + test_params->slave_port_ids[3]); + + + for (i = 0; i < BROADCAST_LINK_STATUS_NUM_OF_SLAVES; i++) { + TEST_ASSERT_EQUAL(generate_test_burst( + &pkt_burst[i][0], burst_size, 0, 0, 1, 0, 0), + burst_size, "failed to generate packet burst"); + + virtual_ethdev_add_mbufs_to_rx_queue(test_params->slave_port_ids[i], + &pkt_burst[i][0], burst_size); + } + + /* Verify that pkts are not received on slaves with link status down */ + TEST_ASSERT_EQUAL(rte_eth_rx_burst( + test_params->bonded_port_id, 0, rx_pkt_burst, MAX_PKT_BURST), + burst_size + burst_size, "rte_eth_rx_burst failed"); + + + /* Verify bonded device rx count */ + rte_eth_stats_get(test_params->bonded_port_id, &port_stats); + TEST_ASSERT_EQUAL(port_stats.ipackets, (uint64_t)(burst_size + burst_size), + "(%d) port_stats.ipackets not as expected\n", + test_params->bonded_port_id); + + /* Clean up and remove slaves from bonded device */ + return remove_slaves_and_stop_bonded_device(); +} + +static int +test_reconfigure_bonded_device(void) +{ + test_params->nb_rx_q = 4; + test_params->nb_tx_q = 4; + + TEST_ASSERT_SUCCESS(configure_ethdev(test_params->bonded_port_id, 0, 0), + "failed to reconfigure bonded device"); + + test_params->nb_rx_q = 2; + test_params->nb_tx_q = 2; + + TEST_ASSERT_SUCCESS(configure_ethdev(test_params->bonded_port_id, 0, 0), + "failed to reconfigure bonded device with less rx/tx queues"); + + return 0; +} + + +static int +test_close_bonded_device(void) +{ + rte_eth_dev_close(test_params->bonded_port_id); + return 0; +} + +static void +testsuite_teardown(void) +{ + free(test_params->pkt_eth_hdr); + test_params->pkt_eth_hdr = NULL; + + /* Clean up and remove slaves from bonded device */ + remove_slaves_and_stop_bonded_device(); +} + +static void +free_virtualpmd_tx_queue(void) +{ + int i, slave_port, to_free_cnt; + struct rte_mbuf *pkts_to_free[MAX_PKT_BURST]; + + /* Free tx queue of virtual pmd */ + for (slave_port = 0; slave_port < test_params->bonded_slave_count; + slave_port++) { + to_free_cnt = virtual_ethdev_get_mbufs_from_tx_queue( + test_params->slave_port_ids[slave_port], + pkts_to_free, MAX_PKT_BURST); + for (i = 0; i < to_free_cnt; i++) + rte_pktmbuf_free(pkts_to_free[i]); + } +} + +static int +test_tlb_tx_burst(void) +{ + int i, burst_size, nb_tx; + uint64_t nb_tx2 = 0; + struct rte_mbuf *pkt_burst[MAX_PKT_BURST]; + struct rte_eth_stats port_stats[32]; + uint64_t sum_ports_opackets = 0, all_bond_opackets = 0, all_bond_obytes = 0; + uint16_t pktlen; + + TEST_ASSERT_SUCCESS(initialize_bonded_device_with_slaves + (BONDING_MODE_TLB, 1, 3, 1), + "Failed to initialise bonded device"); + + burst_size = 20 * test_params->bonded_slave_count; + + TEST_ASSERT(burst_size < MAX_PKT_BURST, + "Burst size specified is greater than supported.\n"); + + + /* Generate bursts of packets */ + for (i = 0; i < 400000; i++) { + /*test two types of mac src own(bonding) and others */ + if (i % 2 == 0) { + initialize_eth_header(test_params->pkt_eth_hdr, + (struct ether_addr *)src_mac, + (struct ether_addr *)dst_mac_0, ETHER_TYPE_IPv4, 0, 0); + } else { + initialize_eth_header(test_params->pkt_eth_hdr, + (struct ether_addr *)test_params->default_slave_mac, + (struct ether_addr *)dst_mac_0, ETHER_TYPE_IPv4, 0, 0); + } + pktlen = initialize_udp_header(test_params->pkt_udp_hdr, src_port, + dst_port_0, 16); + pktlen = initialize_ipv4_header(test_params->pkt_ipv4_hdr, src_addr, + dst_addr_0, pktlen); + generate_packet_burst(test_params->mbuf_pool, pkt_burst, + test_params->pkt_eth_hdr, 0, test_params->pkt_ipv4_hdr, + 1, test_params->pkt_udp_hdr, burst_size, 60, 1); + /* Send burst on bonded port */ + nb_tx = rte_eth_tx_burst(test_params->bonded_port_id, 0, pkt_burst, + burst_size); + nb_tx2 += nb_tx; + + free_virtualpmd_tx_queue(); + + TEST_ASSERT_EQUAL(nb_tx, burst_size, + "number of packet not equal burst size"); + + rte_delay_us(5); + } + + + /* Verify bonded port tx stats */ + rte_eth_stats_get(test_params->bonded_port_id, &port_stats[0]); + + all_bond_opackets = port_stats[0].opackets; + all_bond_obytes = port_stats[0].obytes; + + TEST_ASSERT_EQUAL(port_stats[0].opackets, (uint64_t)nb_tx2, + "Bonded Port (%d) opackets value (%u) not as expected (%d)\n", + test_params->bonded_port_id, (unsigned int)port_stats[0].opackets, + burst_size); + + + /* Verify slave ports tx stats */ + for (i = 0; i < test_params->bonded_slave_count; i++) { + rte_eth_stats_get(test_params->slave_port_ids[i], &port_stats[i]); + sum_ports_opackets += port_stats[i].opackets; + } + + TEST_ASSERT_EQUAL(sum_ports_opackets, (uint64_t)all_bond_opackets, + "Total packets sent by slaves is not equal to packets sent by bond interface"); + + /* checking if distribution of packets is balanced over slaves */ + for (i = 0; i < test_params->bonded_slave_count; i++) { + TEST_ASSERT(port_stats[i].obytes > 0 && + port_stats[i].obytes < all_bond_obytes, + "Packets are not balanced over slaves"); + } + + /* Put all slaves down and try and transmit */ + for (i = 0; i < test_params->bonded_slave_count; i++) { + virtual_ethdev_simulate_link_status_interrupt( + test_params->slave_port_ids[i], 0); + } + + /* Send burst on bonded port */ + nb_tx = rte_eth_tx_burst(test_params->bonded_port_id, 0, pkt_burst, + burst_size); + TEST_ASSERT_EQUAL(nb_tx, 0, " bad number of packet in burst"); + + /* Clean ugit checkout masterp and remove slaves from bonded device */ + return remove_slaves_and_stop_bonded_device(); +} + +#define TEST_ADAPTIVE_TRANSMIT_LOAD_BALANCING_RX_BURST_SLAVE_COUNT (4) + +static int +test_tlb_rx_burst(void) +{ + struct rte_mbuf *gen_pkt_burst[MAX_PKT_BURST] = { NULL }; + struct rte_mbuf *rx_pkt_burst[MAX_PKT_BURST] = { NULL }; + + struct rte_eth_stats port_stats; + + int primary_port; + + uint16_t i, j, nb_rx, burst_size = 17; + + /* Initialize bonded device with 4 slaves in transmit load balancing mode */ + TEST_ASSERT_SUCCESS(initialize_bonded_device_with_slaves( + BONDING_MODE_TLB, + TEST_ADAPTIVE_TRANSMIT_LOAD_BALANCING_RX_BURST_SLAVE_COUNT, 1, 1), + "Failed to initialize bonded device"); + + + primary_port = rte_eth_bond_primary_get(test_params->bonded_port_id); + TEST_ASSERT(primary_port >= 0, + "failed to get primary slave for bonded port (%d)", + test_params->bonded_port_id); + + for (i = 0; i < test_params->bonded_slave_count; i++) { + /* Generate test bursts of packets to transmit */ + TEST_ASSERT_EQUAL(generate_test_burst( + &gen_pkt_burst[0], burst_size, 0, 1, 0, 0, 0), burst_size, + "burst generation failed"); + + /* Add rx data to slave */ + virtual_ethdev_add_mbufs_to_rx_queue(test_params->slave_port_ids[i], + &gen_pkt_burst[0], burst_size); + + /* Call rx burst on bonded device */ + nb_rx = rte_eth_rx_burst(test_params->bonded_port_id, 0, + &rx_pkt_burst[0], MAX_PKT_BURST); + + TEST_ASSERT_EQUAL(nb_rx, burst_size, "rte_eth_rx_burst failed\n"); + + if (test_params->slave_port_ids[i] == primary_port) { + /* Verify bonded device rx count */ + rte_eth_stats_get(test_params->bonded_port_id, &port_stats); + TEST_ASSERT_EQUAL(port_stats.ipackets, (uint64_t)burst_size, + "Bonded Port (%d) ipackets value (%u) not as expected (%d)\n", + test_params->bonded_port_id, + (unsigned int)port_stats.ipackets, burst_size); + + /* Verify bonded slave devices rx count */ + for (j = 0; j < test_params->bonded_slave_count; j++) { + rte_eth_stats_get(test_params->slave_port_ids[j], &port_stats); + if (i == j) { + TEST_ASSERT_EQUAL(port_stats.ipackets, (uint64_t)burst_size, + "Slave Port (%d) ipackets value (%u) not as expected (%d)\n", + test_params->slave_port_ids[i], + (unsigned int)port_stats.ipackets, burst_size); + } else { + TEST_ASSERT_EQUAL(port_stats.ipackets, (uint64_t)0, + "Slave Port (%d) ipackets value (%u) not as expected (%d)\n", + test_params->slave_port_ids[i], + (unsigned int)port_stats.ipackets, 0); + } + } + } else { + for (j = 0; j < test_params->bonded_slave_count; j++) { + rte_eth_stats_get(test_params->slave_port_ids[j], &port_stats); + TEST_ASSERT_EQUAL(port_stats.ipackets, (uint64_t)0, + "Slave Port (%d) ipackets value (%u) not as expected (%d)\n", + test_params->slave_port_ids[i], + (unsigned int)port_stats.ipackets, 0); + } + } + + /* free mbufs */ + for (i = 0; i < burst_size; i++) + rte_pktmbuf_free(rx_pkt_burst[i]); + + /* reset bonded device stats */ + rte_eth_stats_reset(test_params->bonded_port_id); + } + + /* Clean up and remove slaves from bonded device */ + return remove_slaves_and_stop_bonded_device(); +} + +static int +test_tlb_verify_promiscuous_enable_disable(void) +{ + int i, primary_port, promiscuous_en; + + /* Initialize bonded device with 4 slaves in transmit load balancing mode */ + TEST_ASSERT_SUCCESS( initialize_bonded_device_with_slaves( + BONDING_MODE_TLB, 0, 4, 1), + "Failed to initialize bonded device"); + + primary_port = rte_eth_bond_primary_get(test_params->bonded_port_id); + TEST_ASSERT(primary_port >= 0, + "failed to get primary slave for bonded port (%d)", + test_params->bonded_port_id); + + rte_eth_promiscuous_enable(test_params->bonded_port_id); + + promiscuous_en = rte_eth_promiscuous_get(test_params->bonded_port_id); + TEST_ASSERT_EQUAL(promiscuous_en, (int)1, + "Port (%d) promiscuous mode not enabled\n", + test_params->bonded_port_id); + for (i = 0; i < test_params->bonded_slave_count; i++) { + promiscuous_en = rte_eth_promiscuous_get( + test_params->slave_port_ids[i]); + if (primary_port == test_params->slave_port_ids[i]) { + TEST_ASSERT_EQUAL(promiscuous_en, (int)1, + "Port (%d) promiscuous mode not enabled\n", + test_params->bonded_port_id); + } else { + TEST_ASSERT_EQUAL(promiscuous_en, (int)0, + "Port (%d) promiscuous mode enabled\n", + test_params->bonded_port_id); + } + + } + + rte_eth_promiscuous_disable(test_params->bonded_port_id); + + promiscuous_en = rte_eth_promiscuous_get(test_params->bonded_port_id); + TEST_ASSERT_EQUAL(promiscuous_en, (int)0, + "Port (%d) promiscuous mode not disabled\n", + test_params->bonded_port_id); + + for (i = 0; i < test_params->bonded_slave_count; i++) { + promiscuous_en = rte_eth_promiscuous_get( + test_params->slave_port_ids[i]); + TEST_ASSERT_EQUAL(promiscuous_en, (int)0, + "slave port (%d) promiscuous mode not disabled\n", + test_params->slave_port_ids[i]); + } + + /* Clean up and remove slaves from bonded device */ + return remove_slaves_and_stop_bonded_device(); +} + +static int +test_tlb_verify_mac_assignment(void) +{ + struct ether_addr read_mac_addr, expected_mac_addr_0, expected_mac_addr_1; + + rte_eth_macaddr_get(test_params->slave_port_ids[0], &expected_mac_addr_0); + rte_eth_macaddr_get(test_params->slave_port_ids[1], &expected_mac_addr_1); + + /* Initialize bonded device with 2 slaves in active backup mode */ + TEST_ASSERT_SUCCESS(initialize_bonded_device_with_slaves( + BONDING_MODE_TLB, 0, 2, 1), + "Failed to initialize bonded device"); + + /* Verify that bonded MACs is that of first slave and that the other slave + * MAC hasn't been changed */ + rte_eth_macaddr_get(test_params->bonded_port_id, &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(&expected_mac_addr_0, &read_mac_addr, + sizeof(read_mac_addr)), + "bonded port (%d) mac address not set to that of primary port", + test_params->bonded_port_id); + + rte_eth_macaddr_get(test_params->slave_port_ids[0], &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(&expected_mac_addr_0, &read_mac_addr, + sizeof(read_mac_addr)), + "slave port (%d) mac address not set to that of primary port", + test_params->slave_port_ids[0]); + + rte_eth_macaddr_get(test_params->slave_port_ids[1], &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(&expected_mac_addr_1, &read_mac_addr, + sizeof(read_mac_addr)), + "slave port (%d) mac address not as expected", + test_params->slave_port_ids[1]); + + /* change primary and verify that MAC addresses haven't changed */ + TEST_ASSERT_EQUAL(rte_eth_bond_primary_set(test_params->bonded_port_id, + test_params->slave_port_ids[1]), 0, + "Failed to set bonded port (%d) primary port to (%d)", + test_params->bonded_port_id, test_params->slave_port_ids[1]); + + rte_eth_macaddr_get(test_params->bonded_port_id, &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(&expected_mac_addr_0, &read_mac_addr, + sizeof(read_mac_addr)), + "bonded port (%d) mac address not set to that of primary port", + test_params->bonded_port_id); + + rte_eth_macaddr_get(test_params->slave_port_ids[0], &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(&expected_mac_addr_0, &read_mac_addr, + sizeof(read_mac_addr)), + "slave port (%d) mac address not set to that of primary port", + test_params->slave_port_ids[0]); + + rte_eth_macaddr_get(test_params->slave_port_ids[1], &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(&expected_mac_addr_1, &read_mac_addr, + sizeof(read_mac_addr)), + "slave port (%d) mac address not as expected", + test_params->slave_port_ids[1]); + + /* stop / start bonded device and verify that primary MAC address is + * propagated to bonded device and slaves */ + + rte_eth_dev_stop(test_params->bonded_port_id); + + TEST_ASSERT_SUCCESS(rte_eth_dev_start(test_params->bonded_port_id), + "Failed to start device"); + + rte_eth_macaddr_get(test_params->bonded_port_id, &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(&expected_mac_addr_1, &read_mac_addr, + sizeof(read_mac_addr)), + "bonded port (%d) mac address not set to that of primary port", + test_params->bonded_port_id); + + rte_eth_macaddr_get(test_params->slave_port_ids[0], &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(&expected_mac_addr_0, &read_mac_addr, + sizeof(read_mac_addr)), + "slave port (%d) mac address not as expected", + test_params->slave_port_ids[0]); + + rte_eth_macaddr_get(test_params->slave_port_ids[1], &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(&expected_mac_addr_1, &read_mac_addr, + sizeof(read_mac_addr)), + "slave port (%d) mac address not set to that of primary port", + test_params->slave_port_ids[1]); + + + /* Set explicit MAC address */ + TEST_ASSERT_SUCCESS(rte_eth_bond_mac_address_set( + test_params->bonded_port_id, (struct ether_addr *)bonded_mac), + "failed to set MAC address"); + + rte_eth_macaddr_get(test_params->bonded_port_id, &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(&bonded_mac, &read_mac_addr, + sizeof(read_mac_addr)), + "bonded port (%d) mac address not set to that of bonded port", + test_params->bonded_port_id); + + rte_eth_macaddr_get(test_params->slave_port_ids[0], &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(&expected_mac_addr_0, &read_mac_addr, + sizeof(read_mac_addr)), + "slave port (%d) mac address not as expected", + test_params->slave_port_ids[0]); + + rte_eth_macaddr_get(test_params->slave_port_ids[1], &read_mac_addr); + TEST_ASSERT_SUCCESS(memcmp(&bonded_mac, &read_mac_addr, + sizeof(read_mac_addr)), + "slave port (%d) mac address not set to that of bonded port", + test_params->slave_port_ids[1]); + + /* Clean up and remove slaves from bonded device */ + return remove_slaves_and_stop_bonded_device(); +} + +static int +test_tlb_verify_slave_link_status_change_failover(void) +{ + struct rte_mbuf *pkt_burst[TEST_ADAPTIVE_TRANSMIT_LOAD_BALANCING_RX_BURST_SLAVE_COUNT][MAX_PKT_BURST]; + struct rte_mbuf *rx_pkt_burst[MAX_PKT_BURST] = { NULL }; + struct rte_eth_stats port_stats; + + uint16_t slaves[RTE_MAX_ETHPORTS]; + + int i, burst_size, slave_count, primary_port; + + burst_size = 21; + + memset(pkt_burst, 0, sizeof(pkt_burst)); + + + + /* Initialize bonded device with 4 slaves in round robin mode */ + TEST_ASSERT_SUCCESS(initialize_bonded_device_with_slaves( + BONDING_MODE_TLB, 0, + TEST_ADAPTIVE_TRANSMIT_LOAD_BALANCING_RX_BURST_SLAVE_COUNT, 1), + "Failed to initialize bonded device with slaves"); + + /* Verify Current Slaves Count /Active Slave Count is */ + slave_count = rte_eth_bond_slaves_get(test_params->bonded_port_id, slaves, + RTE_MAX_ETHPORTS); + TEST_ASSERT_EQUAL(slave_count, 4, + "Number of slaves (%d) is not as expected (%d).\n", + slave_count, 4); + + slave_count = rte_eth_bond_active_slaves_get(test_params->bonded_port_id, + slaves, RTE_MAX_ETHPORTS); + TEST_ASSERT_EQUAL(slave_count, (int)4, + "Number of slaves (%d) is not as expected (%d).\n", + slave_count, 4); + + primary_port = rte_eth_bond_primary_get(test_params->bonded_port_id); + TEST_ASSERT_EQUAL(primary_port, test_params->slave_port_ids[0], + "Primary port not as expected"); + + /* Bring 2 slaves down and verify active slave count */ + virtual_ethdev_simulate_link_status_interrupt( + test_params->slave_port_ids[1], 0); + virtual_ethdev_simulate_link_status_interrupt( + test_params->slave_port_ids[3], 0); + + TEST_ASSERT_EQUAL(rte_eth_bond_active_slaves_get( + test_params->bonded_port_id, slaves, RTE_MAX_ETHPORTS), 2, + "Number of active slaves (%d) is not as expected (%d).", + slave_count, 2); + + virtual_ethdev_simulate_link_status_interrupt( + test_params->slave_port_ids[1], 1); + virtual_ethdev_simulate_link_status_interrupt( + test_params->slave_port_ids[3], 1); + + + /* Bring primary port down, verify that active slave count is 3 and primary + * has changed */ + virtual_ethdev_simulate_link_status_interrupt( + test_params->slave_port_ids[0], 0); + + TEST_ASSERT_EQUAL(rte_eth_bond_active_slaves_get( + test_params->bonded_port_id, slaves, RTE_MAX_ETHPORTS), 3, + "Number of active slaves (%d) is not as expected (%d).", + slave_count, 3); + + primary_port = rte_eth_bond_primary_get(test_params->bonded_port_id); + TEST_ASSERT_EQUAL(primary_port, test_params->slave_port_ids[2], + "Primary port not as expected"); + rte_delay_us(500000); + /* Verify that pkts are sent on new primary slave */ + for (i = 0; i < 4; i++) { + TEST_ASSERT_EQUAL(generate_test_burst( + &pkt_burst[0][0], burst_size, 0, 1, 0, 0, 0), burst_size, + "generate_test_burst failed\n"); + TEST_ASSERT_EQUAL(rte_eth_tx_burst( + test_params->bonded_port_id, 0, &pkt_burst[0][0], burst_size), burst_size, + "rte_eth_tx_burst failed\n"); + rte_delay_us(11000); + } + + rte_eth_stats_get(test_params->slave_port_ids[0], &port_stats); + TEST_ASSERT_EQUAL(port_stats.opackets, (int8_t)0, + "(%d) port_stats.opackets not as expected\n", + test_params->slave_port_ids[0]); + + rte_eth_stats_get(test_params->slave_port_ids[1], &port_stats); + TEST_ASSERT_NOT_EQUAL(port_stats.opackets, (int8_t)0, + "(%d) port_stats.opackets not as expected\n", + test_params->slave_port_ids[1]); + + rte_eth_stats_get(test_params->slave_port_ids[2], &port_stats); + TEST_ASSERT_NOT_EQUAL(port_stats.opackets, (int8_t)0, + "(%d) port_stats.opackets not as expected\n", + test_params->slave_port_ids[2]); + + rte_eth_stats_get(test_params->slave_port_ids[3], &port_stats); + TEST_ASSERT_NOT_EQUAL(port_stats.opackets, (int8_t)0, + "(%d) port_stats.opackets not as expected\n", + test_params->slave_port_ids[3]); + + + /* Generate packet burst for testing */ + + for (i = 0; i < TEST_ADAPTIVE_TRANSMIT_LOAD_BALANCING_RX_BURST_SLAVE_COUNT; i++) { + if (generate_test_burst(&pkt_burst[i][0], burst_size, 0, 1, 0, 0, 0) != + burst_size) + return -1; + + virtual_ethdev_add_mbufs_to_rx_queue( + test_params->slave_port_ids[i], &pkt_burst[i][0], burst_size); + } + + if (rte_eth_rx_burst(test_params->bonded_port_id, 0, rx_pkt_burst, + MAX_PKT_BURST) != burst_size) { + printf("rte_eth_rx_burst\n"); + return -1; + + } + + /* Verify bonded device rx count */ + rte_eth_stats_get(test_params->bonded_port_id, &port_stats); + TEST_ASSERT_EQUAL(port_stats.ipackets, (uint64_t)burst_size, + "(%d) port_stats.ipackets not as expected\n", + test_params->bonded_port_id); + + /* Clean up and remove slaves from bonded device */ + return remove_slaves_and_stop_bonded_device(); +} + +#define TEST_ALB_SLAVE_COUNT 2 + +static uint8_t mac_client1[] = {0x00, 0xAA, 0x55, 0xFF, 0xCC, 1}; +static uint8_t mac_client2[] = {0x00, 0xAA, 0x55, 0xFF, 0xCC, 2}; +static uint8_t mac_client3[] = {0x00, 0xAA, 0x55, 0xFF, 0xCC, 3}; +static uint8_t mac_client4[] = {0x00, 0xAA, 0x55, 0xFF, 0xCC, 4}; + +static uint32_t ip_host = IPV4_ADDR(192, 168, 0, 0); +static uint32_t ip_client1 = IPV4_ADDR(192, 168, 0, 1); +static uint32_t ip_client2 = IPV4_ADDR(192, 168, 0, 2); +static uint32_t ip_client3 = IPV4_ADDR(192, 168, 0, 3); +static uint32_t ip_client4 = IPV4_ADDR(192, 168, 0, 4); + +static int +test_alb_change_mac_in_reply_sent(void) +{ + struct rte_mbuf *pkt; + struct rte_mbuf *pkts_sent[MAX_PKT_BURST]; + + struct ether_hdr *eth_pkt; + struct arp_hdr *arp_pkt; + + int slave_idx, nb_pkts, pkt_idx; + int retval = 0; + + struct ether_addr bond_mac, client_mac; + struct ether_addr *slave_mac1, *slave_mac2; + + TEST_ASSERT_SUCCESS( + initialize_bonded_device_with_slaves(BONDING_MODE_ALB, + 0, TEST_ALB_SLAVE_COUNT, 1), + "Failed to initialize_bonded_device_with_slaves."); + + /* Flush tx queue */ + rte_eth_tx_burst(test_params->bonded_port_id, 0, NULL, 0); + for (slave_idx = 0; slave_idx < test_params->bonded_slave_count; + slave_idx++) { + nb_pkts = virtual_ethdev_get_mbufs_from_tx_queue( + test_params->slave_port_ids[slave_idx], pkts_sent, + MAX_PKT_BURST); + } + + ether_addr_copy( + rte_eth_devices[test_params->bonded_port_id].data->mac_addrs, + &bond_mac); + + /* + * Generating four packets with different mac and ip addresses and sending + * them through the bonding port. + */ + pkt = rte_pktmbuf_alloc(test_params->mbuf_pool); + memcpy(client_mac.addr_bytes, mac_client1, ETHER_ADDR_LEN); + eth_pkt = rte_pktmbuf_mtod(pkt, struct ether_hdr *); + initialize_eth_header(eth_pkt, &bond_mac, &client_mac, ETHER_TYPE_ARP, 0, + 0); + arp_pkt = (struct arp_hdr *)((char *)eth_pkt + sizeof(struct ether_hdr)); + initialize_arp_header(arp_pkt, &bond_mac, &client_mac, ip_host, ip_client1, + ARP_OP_REPLY); + rte_eth_tx_burst(test_params->bonded_port_id, 0, &pkt, 1); + + pkt = rte_pktmbuf_alloc(test_params->mbuf_pool); + memcpy(client_mac.addr_bytes, mac_client2, ETHER_ADDR_LEN); + eth_pkt = rte_pktmbuf_mtod(pkt, struct ether_hdr *); + initialize_eth_header(eth_pkt, &bond_mac, &client_mac, ETHER_TYPE_ARP, 0, + 0); + arp_pkt = (struct arp_hdr *)((char *)eth_pkt + sizeof(struct ether_hdr)); + initialize_arp_header(arp_pkt, &bond_mac, &client_mac, ip_host, ip_client2, + ARP_OP_REPLY); + rte_eth_tx_burst(test_params->bonded_port_id, 0, &pkt, 1); + + pkt = rte_pktmbuf_alloc(test_params->mbuf_pool); + memcpy(client_mac.addr_bytes, mac_client3, ETHER_ADDR_LEN); + eth_pkt = rte_pktmbuf_mtod(pkt, struct ether_hdr *); + initialize_eth_header(eth_pkt, &bond_mac, &client_mac, ETHER_TYPE_ARP, 0, + 0); + arp_pkt = (struct arp_hdr *)((char *)eth_pkt + sizeof(struct ether_hdr)); + initialize_arp_header(arp_pkt, &bond_mac, &client_mac, ip_host, ip_client3, + ARP_OP_REPLY); + rte_eth_tx_burst(test_params->bonded_port_id, 0, &pkt, 1); + + pkt = rte_pktmbuf_alloc(test_params->mbuf_pool); + memcpy(client_mac.addr_bytes, mac_client4, ETHER_ADDR_LEN); + eth_pkt = rte_pktmbuf_mtod(pkt, struct ether_hdr *); + initialize_eth_header(eth_pkt, &bond_mac, &client_mac, ETHER_TYPE_ARP, 0, + 0); + arp_pkt = (struct arp_hdr *)((char *)eth_pkt + sizeof(struct ether_hdr)); + initialize_arp_header(arp_pkt, &bond_mac, &client_mac, ip_host, ip_client4, + ARP_OP_REPLY); + rte_eth_tx_burst(test_params->bonded_port_id, 0, &pkt, 1); + + slave_mac1 = + rte_eth_devices[test_params->slave_port_ids[0]].data->mac_addrs; + slave_mac2 = + rte_eth_devices[test_params->slave_port_ids[1]].data->mac_addrs; + + /* + * Checking if packets are properly distributed on bonding ports. Packets + * 0 and 2 should be sent on port 0 and packets 1 and 3 on port 1. + */ + for (slave_idx = 0; slave_idx < test_params->bonded_slave_count; slave_idx++) { + nb_pkts = virtual_ethdev_get_mbufs_from_tx_queue( + test_params->slave_port_ids[slave_idx], pkts_sent, + MAX_PKT_BURST); + + for (pkt_idx = 0; pkt_idx < nb_pkts; pkt_idx++) { + eth_pkt = rte_pktmbuf_mtod(pkts_sent[pkt_idx], struct ether_hdr *); + arp_pkt = (struct arp_hdr *)((char *)eth_pkt + sizeof(struct ether_hdr)); + + if (slave_idx%2 == 0) { + if (!is_same_ether_addr(slave_mac1, &arp_pkt->arp_data.arp_sha)) { + retval = -1; + goto test_end; + } + } else { + if (!is_same_ether_addr(slave_mac2, &arp_pkt->arp_data.arp_sha)) { + retval = -1; + goto test_end; + } + } + } + } + +test_end: + retval += remove_slaves_and_stop_bonded_device(); + return retval; +} + +static int +test_alb_reply_from_client(void) +{ + struct ether_hdr *eth_pkt; + struct arp_hdr *arp_pkt; + + struct rte_mbuf *pkt; + struct rte_mbuf *pkts_sent[MAX_PKT_BURST]; + + int slave_idx, nb_pkts, pkt_idx, nb_pkts_sum = 0; + int retval = 0; + + struct ether_addr bond_mac, client_mac; + struct ether_addr *slave_mac1, *slave_mac2; + + TEST_ASSERT_SUCCESS( + initialize_bonded_device_with_slaves(BONDING_MODE_ALB, + 0, TEST_ALB_SLAVE_COUNT, 1), + "Failed to initialize_bonded_device_with_slaves."); + + /* Flush tx queue */ + rte_eth_tx_burst(test_params->bonded_port_id, 0, NULL, 0); + for (slave_idx = 0; slave_idx < test_params->bonded_slave_count; slave_idx++) { + nb_pkts = virtual_ethdev_get_mbufs_from_tx_queue( + test_params->slave_port_ids[slave_idx], pkts_sent, + MAX_PKT_BURST); + } + + ether_addr_copy( + rte_eth_devices[test_params->bonded_port_id].data->mac_addrs, + &bond_mac); + + /* + * Generating four packets with different mac and ip addresses and placing + * them in the rx queue to be received by the bonding driver on rx_burst. + */ + pkt = rte_pktmbuf_alloc(test_params->mbuf_pool); + memcpy(client_mac.addr_bytes, mac_client1, ETHER_ADDR_LEN); + eth_pkt = rte_pktmbuf_mtod(pkt, struct ether_hdr *); + initialize_eth_header(eth_pkt, &bond_mac, &client_mac, ETHER_TYPE_ARP, 0, + 0); + arp_pkt = (struct arp_hdr *)((char *)eth_pkt + sizeof(struct ether_hdr)); + initialize_arp_header(arp_pkt, &client_mac, &bond_mac, ip_client1, ip_host, + ARP_OP_REPLY); + virtual_ethdev_add_mbufs_to_rx_queue(test_params->slave_port_ids[0], &pkt, + 1); + + pkt = rte_pktmbuf_alloc(test_params->mbuf_pool); + memcpy(client_mac.addr_bytes, mac_client2, ETHER_ADDR_LEN); + eth_pkt = rte_pktmbuf_mtod(pkt, struct ether_hdr *); + initialize_eth_header(eth_pkt, &bond_mac, &client_mac, ETHER_TYPE_ARP, 0, + 0); + arp_pkt = (struct arp_hdr *)((char *)eth_pkt + sizeof(struct ether_hdr)); + initialize_arp_header(arp_pkt, &client_mac, &bond_mac, ip_client2, ip_host, + ARP_OP_REPLY); + virtual_ethdev_add_mbufs_to_rx_queue(test_params->slave_port_ids[0], &pkt, + 1); + + pkt = rte_pktmbuf_alloc(test_params->mbuf_pool); + memcpy(client_mac.addr_bytes, mac_client3, ETHER_ADDR_LEN); + eth_pkt = rte_pktmbuf_mtod(pkt, struct ether_hdr *); + initialize_eth_header(eth_pkt, &bond_mac, &client_mac, ETHER_TYPE_ARP, 0, + 0); + arp_pkt = (struct arp_hdr *)((char *)eth_pkt + sizeof(struct ether_hdr)); + initialize_arp_header(arp_pkt, &client_mac, &bond_mac, ip_client3, ip_host, + ARP_OP_REPLY); + virtual_ethdev_add_mbufs_to_rx_queue(test_params->slave_port_ids[0], &pkt, + 1); + + pkt = rte_pktmbuf_alloc(test_params->mbuf_pool); + memcpy(client_mac.addr_bytes, mac_client4, ETHER_ADDR_LEN); + eth_pkt = rte_pktmbuf_mtod(pkt, struct ether_hdr *); + initialize_eth_header(eth_pkt, &bond_mac, &client_mac, ETHER_TYPE_ARP, 0, + 0); + arp_pkt = (struct arp_hdr *)((char *)eth_pkt + sizeof(struct ether_hdr)); + initialize_arp_header(arp_pkt, &client_mac, &bond_mac, ip_client4, ip_host, + ARP_OP_REPLY); + virtual_ethdev_add_mbufs_to_rx_queue(test_params->slave_port_ids[0], &pkt, + 1); + + /* + * Issue rx_burst and tx_burst to force bonding driver to send update ARP + * packets to every client in alb table. + */ + rte_eth_rx_burst(test_params->bonded_port_id, 0, pkts_sent, MAX_PKT_BURST); + rte_eth_tx_burst(test_params->bonded_port_id, 0, NULL, 0); + + slave_mac1 = rte_eth_devices[test_params->slave_port_ids[0]].data->mac_addrs; + slave_mac2 = rte_eth_devices[test_params->slave_port_ids[1]].data->mac_addrs; + + /* + * Checking if update ARP packets were properly send on slave ports. + */ + for (slave_idx = 0; slave_idx < test_params->bonded_slave_count; slave_idx++) { + nb_pkts = virtual_ethdev_get_mbufs_from_tx_queue( + test_params->slave_port_ids[slave_idx], pkts_sent, MAX_PKT_BURST); + nb_pkts_sum += nb_pkts; + + for (pkt_idx = 0; pkt_idx < nb_pkts; pkt_idx++) { + eth_pkt = rte_pktmbuf_mtod(pkts_sent[pkt_idx], struct ether_hdr *); + arp_pkt = (struct arp_hdr *)((char *)eth_pkt + sizeof(struct ether_hdr)); + + if (slave_idx%2 == 0) { + if (!is_same_ether_addr(slave_mac1, &arp_pkt->arp_data.arp_sha)) { + retval = -1; + goto test_end; + } + } else { + if (!is_same_ether_addr(slave_mac2, &arp_pkt->arp_data.arp_sha)) { + retval = -1; + goto test_end; + } + } + } + } + + /* Check if proper number of packets was send */ + if (nb_pkts_sum < 4) { + retval = -1; + goto test_end; + } + +test_end: + retval += remove_slaves_and_stop_bonded_device(); + return retval; +} + +static int +test_alb_receive_vlan_reply(void) +{ + struct ether_hdr *eth_pkt; + struct vlan_hdr *vlan_pkt; + struct arp_hdr *arp_pkt; + + struct rte_mbuf *pkt; + struct rte_mbuf *pkts_sent[MAX_PKT_BURST]; + + int slave_idx, nb_pkts, pkt_idx; + int retval = 0; + + struct ether_addr bond_mac, client_mac; + + TEST_ASSERT_SUCCESS( + initialize_bonded_device_with_slaves(BONDING_MODE_ALB, + 0, TEST_ALB_SLAVE_COUNT, 1), + "Failed to initialize_bonded_device_with_slaves."); + + /* Flush tx queue */ + rte_eth_tx_burst(test_params->bonded_port_id, 0, NULL, 0); + for (slave_idx = 0; slave_idx < test_params->bonded_slave_count; slave_idx++) { + nb_pkts = virtual_ethdev_get_mbufs_from_tx_queue( + test_params->slave_port_ids[slave_idx], pkts_sent, + MAX_PKT_BURST); + } + + ether_addr_copy( + rte_eth_devices[test_params->bonded_port_id].data->mac_addrs, + &bond_mac); + + /* + * Generating packet with double VLAN header and placing it in the rx queue. + */ + pkt = rte_pktmbuf_alloc(test_params->mbuf_pool); + memcpy(client_mac.addr_bytes, mac_client1, ETHER_ADDR_LEN); + eth_pkt = rte_pktmbuf_mtod(pkt, struct ether_hdr *); + initialize_eth_header(eth_pkt, &bond_mac, &client_mac, ETHER_TYPE_VLAN, 0, + 0); + vlan_pkt = (struct vlan_hdr *)((char *)(eth_pkt + 1)); + vlan_pkt->vlan_tci = rte_cpu_to_be_16(1); + vlan_pkt->eth_proto = rte_cpu_to_be_16(ETHER_TYPE_VLAN); + vlan_pkt = vlan_pkt+1; + vlan_pkt->vlan_tci = rte_cpu_to_be_16(2); + vlan_pkt->eth_proto = rte_cpu_to_be_16(ETHER_TYPE_ARP); + arp_pkt = (struct arp_hdr *)((char *)(vlan_pkt + 1)); + initialize_arp_header(arp_pkt, &client_mac, &bond_mac, ip_client1, ip_host, + ARP_OP_REPLY); + virtual_ethdev_add_mbufs_to_rx_queue(test_params->slave_port_ids[0], &pkt, + 1); + + rte_eth_rx_burst(test_params->bonded_port_id, 0, pkts_sent, MAX_PKT_BURST); + rte_eth_tx_burst(test_params->bonded_port_id, 0, NULL, 0); + + /* + * Checking if VLAN headers in generated ARP Update packet are correct. + */ + for (slave_idx = 0; slave_idx < test_params->bonded_slave_count; slave_idx++) { + nb_pkts = virtual_ethdev_get_mbufs_from_tx_queue( + test_params->slave_port_ids[slave_idx], pkts_sent, + MAX_PKT_BURST); + + for (pkt_idx = 0; pkt_idx < nb_pkts; pkt_idx++) { + eth_pkt = rte_pktmbuf_mtod(pkts_sent[pkt_idx], struct ether_hdr *); + vlan_pkt = (struct vlan_hdr *)((char *)(eth_pkt + 1)); + if (vlan_pkt->vlan_tci != rte_cpu_to_be_16(1)) { + retval = -1; + goto test_end; + } + if (vlan_pkt->eth_proto != rte_cpu_to_be_16(ETHER_TYPE_VLAN)) { + retval = -1; + goto test_end; + } + vlan_pkt = vlan_pkt+1; + if (vlan_pkt->vlan_tci != rte_cpu_to_be_16(2)) { + retval = -1; + goto test_end; + } + if (vlan_pkt->eth_proto != rte_cpu_to_be_16(ETHER_TYPE_ARP)) { + retval = -1; + goto test_end; + } + } + } + +test_end: + retval += remove_slaves_and_stop_bonded_device(); + return retval; +} + +static int +test_alb_ipv4_tx(void) +{ + int burst_size, retval, pkts_send; + struct rte_mbuf *pkt_burst[MAX_PKT_BURST]; + + retval = 0; + + TEST_ASSERT_SUCCESS( + initialize_bonded_device_with_slaves(BONDING_MODE_ALB, + 0, TEST_ALB_SLAVE_COUNT, 1), + "Failed to initialize_bonded_device_with_slaves."); + + burst_size = 32; + + /* Generate test bursts of packets to transmit */ + if (generate_test_burst(pkt_burst, burst_size, 0, 1, 0, 0, 0) != burst_size) { + retval = -1; + goto test_end; + } + + /* + * Checking if ipv4 traffic is transmitted via TLB policy. + */ + pkts_send = rte_eth_tx_burst( + test_params->bonded_port_id, 0, pkt_burst, burst_size); + if (pkts_send != burst_size) { + retval = -1; + goto test_end; + } + +test_end: + retval += remove_slaves_and_stop_bonded_device(); + return retval; +} + +static struct unit_test_suite link_bonding_test_suite = { + .suite_name = "Link Bonding Unit Test Suite", + .setup = test_setup, + .teardown = testsuite_teardown, + .unit_test_cases = { + TEST_CASE(test_create_bonded_device), + TEST_CASE(test_create_bonded_device_with_invalid_params), + TEST_CASE(test_add_slave_to_bonded_device), + TEST_CASE(test_add_slave_to_invalid_bonded_device), + TEST_CASE(test_remove_slave_from_bonded_device), + TEST_CASE(test_remove_slave_from_invalid_bonded_device), + TEST_CASE(test_get_slaves_from_bonded_device), + TEST_CASE(test_add_already_bonded_slave_to_bonded_device), + TEST_CASE(test_add_remove_multiple_slaves_to_from_bonded_device), + TEST_CASE(test_start_bonded_device), + TEST_CASE(test_stop_bonded_device), + TEST_CASE(test_set_bonding_mode), + TEST_CASE(test_set_primary_slave), + TEST_CASE(test_set_explicit_bonded_mac), + TEST_CASE(test_set_bonded_port_initialization_mac_assignment), + TEST_CASE(test_status_interrupt), + TEST_CASE(test_adding_slave_after_bonded_device_started), + TEST_CASE(test_roundrobin_tx_burst), + TEST_CASE(test_roundrobin_tx_burst_slave_tx_fail), + TEST_CASE(test_roundrobin_rx_burst_on_single_slave), + TEST_CASE(test_roundrobin_rx_burst_on_multiple_slaves), + TEST_CASE(test_roundrobin_verify_promiscuous_enable_disable), + TEST_CASE(test_roundrobin_verify_mac_assignment), + TEST_CASE(test_roundrobin_verify_slave_link_status_change_behaviour), + TEST_CASE(test_roundrobin_verfiy_polling_slave_link_status_change), + TEST_CASE(test_activebackup_tx_burst), + TEST_CASE(test_activebackup_rx_burst), + TEST_CASE(test_activebackup_verify_promiscuous_enable_disable), + TEST_CASE(test_activebackup_verify_mac_assignment), + TEST_CASE(test_activebackup_verify_slave_link_status_change_failover), + TEST_CASE(test_balance_xmit_policy_configuration), + TEST_CASE(test_balance_l2_tx_burst), + TEST_CASE(test_balance_l23_tx_burst_ipv4_toggle_ip_addr), + TEST_CASE(test_balance_l23_tx_burst_vlan_ipv4_toggle_ip_addr), + TEST_CASE(test_balance_l23_tx_burst_ipv6_toggle_ip_addr), + TEST_CASE(test_balance_l23_tx_burst_vlan_ipv6_toggle_ip_addr), + TEST_CASE(test_balance_l23_tx_burst_toggle_mac_addr), + TEST_CASE(test_balance_l34_tx_burst_ipv4_toggle_ip_addr), + TEST_CASE(test_balance_l34_tx_burst_ipv4_toggle_udp_port), + TEST_CASE(test_balance_l34_tx_burst_vlan_ipv4_toggle_ip_addr), + TEST_CASE(test_balance_l34_tx_burst_ipv6_toggle_ip_addr), + TEST_CASE(test_balance_l34_tx_burst_vlan_ipv6_toggle_ip_addr), + TEST_CASE(test_balance_l34_tx_burst_ipv6_toggle_udp_port), + TEST_CASE(test_balance_tx_burst_slave_tx_fail), + TEST_CASE(test_balance_rx_burst), + TEST_CASE(test_balance_verify_promiscuous_enable_disable), + TEST_CASE(test_balance_verify_mac_assignment), + TEST_CASE(test_balance_verify_slave_link_status_change_behaviour), + TEST_CASE(test_tlb_tx_burst), + TEST_CASE(test_tlb_rx_burst), + TEST_CASE(test_tlb_verify_mac_assignment), + TEST_CASE(test_tlb_verify_promiscuous_enable_disable), + TEST_CASE(test_tlb_verify_slave_link_status_change_failover), + TEST_CASE(test_alb_change_mac_in_reply_sent), + TEST_CASE(test_alb_reply_from_client), + TEST_CASE(test_alb_receive_vlan_reply), + TEST_CASE(test_alb_ipv4_tx), + TEST_CASE(test_broadcast_tx_burst), + TEST_CASE(test_broadcast_tx_burst_slave_tx_fail), + TEST_CASE(test_broadcast_rx_burst), + TEST_CASE(test_broadcast_verify_promiscuous_enable_disable), + TEST_CASE(test_broadcast_verify_mac_assignment), + TEST_CASE(test_broadcast_verify_slave_link_status_change_behaviour), + TEST_CASE(test_reconfigure_bonded_device), + TEST_CASE(test_close_bonded_device), + + TEST_CASES_END() /**< NULL terminate unit test array */ + } +}; + + +static int +test_link_bonding(void) +{ + return unit_test_suite_runner(&link_bonding_test_suite); +} + +REGISTER_TEST_COMMAND(link_bonding_autotest, test_link_bonding); diff --git a/dpdk/test/test/test_link_bonding_mode4.c b/dpdk/test/test/test_link_bonding_mode4.c new file mode 100644 index 00000000..a7a3f8e2 --- /dev/null +++ b/dpdk/test/test/test_link_bonding_mode4.c @@ -0,0 +1,1679 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <string.h> +#include <stdarg.h> +#include <stdio.h> +#include <stdlib.h> +#include <stdint.h> +#include <inttypes.h> +#include <errno.h> +#include <rte_cycles.h> +#include <sys/queue.h> + +#include <rte_byteorder.h> +#include <rte_common.h> +#include <rte_debug.h> +#include <rte_ethdev.h> +#include <rte_log.h> +#include <rte_lcore.h> +#include <rte_memory.h> + +#include <rte_string_fns.h> + +#include <rte_eth_ring.h> +#include <rte_errno.h> +#include <rte_eth_bond.h> +#include <rte_eth_bond_8023ad.h> + +#include "packet_burst_generator.h" + +#include "test.h" + +#define SLAVE_COUNT (4) + +#define RX_RING_SIZE 128 +#define TX_RING_SIZE 512 + +#define MBUF_CACHE_SIZE (250) +#define BURST_SIZE (32) + +#define TEST_RX_DESC_MAX (2048) +#define TEST_TX_DESC_MAX (2048) +#define MAX_PKT_BURST (32) +#define DEF_PKT_BURST (16) + +#define BONDED_DEV_NAME ("net_bonding_m4_bond_dev") + +#define SLAVE_DEV_NAME_FMT ("net_virt_%d") +#define SLAVE_RX_QUEUE_FMT ("net_virt_%d_rx") +#define SLAVE_TX_QUEUE_FMT ("net_virt_%d_tx") + +#define INVALID_SOCKET_ID (-1) +#define INVALID_PORT_ID (0xFF) +#define INVALID_BONDING_MODE (-1) + +static const struct ether_addr slave_mac_default = { + { 0x00, 0xFF, 0x00, 0xFF, 0x00, 0x00 } +}; + +static const struct ether_addr parnter_mac_default = { + { 0x22, 0xBB, 0xFF, 0xBB, 0x00, 0x00 } +}; + +static const struct ether_addr parnter_system = { + { 0x33, 0xFF, 0xBB, 0xFF, 0x00, 0x00 } +}; + +static const struct ether_addr slow_protocol_mac_addr = { + { 0x01, 0x80, 0xC2, 0x00, 0x00, 0x02 } +}; + +struct slave_conf { + struct rte_ring *rx_queue; + struct rte_ring *tx_queue; + uint16_t port_id; + uint8_t bonded : 1; + + uint8_t lacp_parnter_state; +}; + +struct ether_vlan_hdr { + struct ether_hdr pkt_eth_hdr; + struct vlan_hdr vlan_hdr; +}; + +struct link_bonding_unittest_params { + uint8_t bonded_port_id; + struct slave_conf slave_ports[SLAVE_COUNT]; + + struct rte_mempool *mbuf_pool; +}; + +#define TEST_DEFAULT_SLAVE_COUNT RTE_DIM(test_params.slave_ports) +#define TEST_RX_SLAVE_COUT TEST_DEFAULT_SLAVE_COUNT +#define TEST_TX_SLAVE_COUNT TEST_DEFAULT_SLAVE_COUNT +#define TEST_MARKER_SLAVE_COUT TEST_DEFAULT_SLAVE_COUNT +#define TEST_EXPIRED_SLAVE_COUNT TEST_DEFAULT_SLAVE_COUNT +#define TEST_PROMISC_SLAVE_COUNT TEST_DEFAULT_SLAVE_COUNT + +static struct link_bonding_unittest_params test_params = { + .bonded_port_id = INVALID_PORT_ID, + .slave_ports = { [0 ... SLAVE_COUNT - 1] = { .port_id = INVALID_PORT_ID} }, + + .mbuf_pool = NULL, +}; + +static struct rte_eth_conf default_pmd_conf = { + .rxmode = { + .mq_mode = ETH_MQ_RX_NONE, + .max_rx_pkt_len = ETHER_MAX_LEN, + .split_hdr_size = 0, + .header_split = 0, /**< Header Split disabled */ + .hw_ip_checksum = 0, /**< IP checksum offload enabled */ + .hw_vlan_filter = 0, /**< VLAN filtering disabled */ + .jumbo_frame = 0, /**< Jumbo Frame Support disabled */ + .hw_strip_crc = 1, /**< CRC stripped by hardware */ + }, + .txmode = { + .mq_mode = ETH_MQ_TX_NONE, + }, + .lpbk_mode = 0, +}; + +static uint8_t lacpdu_rx_count[RTE_MAX_ETHPORTS] = {0, }; + +#define FOR_EACH(_i, _item, _array, _size) \ + for (_i = 0, _item = &_array[0]; _i < _size && (_item = &_array[_i]); _i++) + +/* Macro for iterating over every port that can be used as a slave + * in this test. + * _i variable used as an index in test_params->slave_ports + * _slave pointer to &test_params->slave_ports[_idx] + */ +#define FOR_EACH_PORT(_i, _port) \ + FOR_EACH(_i, _port, test_params.slave_ports, \ + RTE_DIM(test_params.slave_ports)) + +/* Macro for iterating over every port that can be used as a slave + * in this test and satisfy given condition. + * + * _i variable used as an index in test_params->slave_ports + * _slave pointer to &test_params->slave_ports[_idx] + * _condition condition that need to be checked + */ +#define FOR_EACH_PORT_IF(_i, _port, _condition) FOR_EACH_PORT((_i), (_port)) \ + if (!!(_condition)) + +/* Macro for iterating over every port that is currently a slave of a bonded + * device. + * _i variable used as an index in test_params->slave_ports + * _slave pointer to &test_params->slave_ports[_idx] + * */ +#define FOR_EACH_SLAVE(_i, _slave) \ + FOR_EACH_PORT_IF(_i, _slave, (_slave)->bonded != 0) + +/* + * Returns packets from slaves TX queue. + * slave slave port + * buffer for packets + * size size of buffer + * return number of packets or negative error number + */ +static int +slave_get_pkts(struct slave_conf *slave, struct rte_mbuf **buf, uint16_t size) +{ + return rte_ring_dequeue_burst(slave->tx_queue, (void **)buf, + size, NULL); +} + +/* + * Injects given packets into slaves RX queue. + * slave slave port + * buffer for packets + * size number of packets to be injected + * return number of queued packets or negative error number + */ +static int +slave_put_pkts(struct slave_conf *slave, struct rte_mbuf **buf, uint16_t size) +{ + return rte_ring_enqueue_burst(slave->rx_queue, (void **)buf, + size, NULL); +} + +static uint16_t +bond_rx(struct rte_mbuf **buf, uint16_t size) +{ + return rte_eth_rx_burst(test_params.bonded_port_id, 0, buf, size); +} + +static uint16_t +bond_tx(struct rte_mbuf **buf, uint16_t size) +{ + return rte_eth_tx_burst(test_params.bonded_port_id, 0, buf, size); +} + +static void +free_pkts(struct rte_mbuf **pkts, uint16_t count) +{ + uint16_t i; + + for (i = 0; i < count; i++) { + if (pkts[i] != NULL) + rte_pktmbuf_free(pkts[i]); + } +} + +static int +configure_ethdev(uint16_t port_id, uint8_t start) +{ + TEST_ASSERT(rte_eth_dev_configure(port_id, 1, 1, &default_pmd_conf) == 0, + "Failed to configure device %u", port_id); + + TEST_ASSERT(rte_eth_rx_queue_setup(port_id, 0, RX_RING_SIZE, + rte_eth_dev_socket_id(port_id), NULL, test_params.mbuf_pool) == 0, + "Failed to setup rx queue."); + + TEST_ASSERT(rte_eth_tx_queue_setup(port_id, 0, TX_RING_SIZE, + rte_eth_dev_socket_id(port_id), NULL) == 0, + "Failed to setup tx queue."); + + if (start) { + TEST_ASSERT(rte_eth_dev_start(port_id) == 0, + "Failed to start device (%d).", port_id); + } + return 0; +} + +static int +add_slave(struct slave_conf *slave, uint8_t start) +{ + struct ether_addr addr, addr_check; + + /* Some sanity check */ + RTE_VERIFY(test_params.slave_ports <= slave && + slave - test_params.slave_ports < (int)RTE_DIM(test_params.slave_ports)); + RTE_VERIFY(slave->bonded == 0); + RTE_VERIFY(slave->port_id != INVALID_PORT_ID); + + ether_addr_copy(&slave_mac_default, &addr); + addr.addr_bytes[ETHER_ADDR_LEN - 1] = slave->port_id; + + rte_eth_dev_mac_addr_remove(slave->port_id, &addr); + + TEST_ASSERT_SUCCESS(rte_eth_dev_mac_addr_add(slave->port_id, &addr, 0), + "Failed to set slave MAC address"); + + TEST_ASSERT_SUCCESS(rte_eth_bond_slave_add(test_params.bonded_port_id, + slave->port_id), + "Failed to add slave (idx=%u, id=%u) to bonding (id=%u)", + (uint8_t)(slave - test_params.slave_ports), slave->port_id, + test_params.bonded_port_id); + + slave->bonded = 1; + if (start) { + TEST_ASSERT_SUCCESS(rte_eth_dev_start(slave->port_id), + "Failed to start slave %u", slave->port_id); + } + + rte_eth_macaddr_get(slave->port_id, &addr_check); + TEST_ASSERT_EQUAL(is_same_ether_addr(&addr, &addr_check), 1, + "Slave MAC address is not as expected"); + + RTE_VERIFY(slave->lacp_parnter_state == 0); + return 0; +} + +static int +remove_slave(struct slave_conf *slave) +{ + ptrdiff_t slave_idx = slave - test_params.slave_ports; + + RTE_VERIFY(test_params.slave_ports <= slave && + slave_idx < (ptrdiff_t)RTE_DIM(test_params.slave_ports)); + + RTE_VERIFY(slave->bonded == 1); + RTE_VERIFY(slave->port_id != INVALID_PORT_ID); + + TEST_ASSERT_EQUAL(rte_ring_count(slave->rx_queue), 0, + "Slave %u tx queue not empty while removing from bonding.", + slave->port_id); + + TEST_ASSERT_EQUAL(rte_ring_count(slave->rx_queue), 0, + "Slave %u tx queue not empty while removing from bonding.", + slave->port_id); + + TEST_ASSERT_EQUAL(rte_eth_bond_slave_remove(test_params.bonded_port_id, + slave->port_id), 0, + "Failed to remove slave (idx=%u, id=%u) from bonding (id=%u)", + (uint8_t)slave_idx, slave->port_id, + test_params.bonded_port_id); + + slave->bonded = 0; + slave->lacp_parnter_state = 0; + return 0; +} + +static void +lacp_recv_cb(uint16_t slave_id, struct rte_mbuf *lacp_pkt) +{ + struct ether_hdr *hdr; + struct slow_protocol_frame *slow_hdr; + + RTE_VERIFY(lacp_pkt != NULL); + + hdr = rte_pktmbuf_mtod(lacp_pkt, struct ether_hdr *); + RTE_VERIFY(hdr->ether_type == rte_cpu_to_be_16(ETHER_TYPE_SLOW)); + + slow_hdr = rte_pktmbuf_mtod(lacp_pkt, struct slow_protocol_frame *); + RTE_VERIFY(slow_hdr->slow_protocol.subtype == SLOW_SUBTYPE_LACP); + + lacpdu_rx_count[slave_id]++; + rte_pktmbuf_free(lacp_pkt); +} + +static int +initialize_bonded_device_with_slaves(uint16_t slave_count, uint8_t external_sm) +{ + uint8_t i; + + RTE_VERIFY(test_params.bonded_port_id != INVALID_PORT_ID); + + for (i = 0; i < slave_count; i++) { + TEST_ASSERT_SUCCESS(add_slave(&test_params.slave_ports[i], 1), + "Failed to add port %u to bonded device.\n", + test_params.slave_ports[i].port_id); + } + + /* Reset mode 4 configuration */ + rte_eth_bond_8023ad_setup(test_params.bonded_port_id, NULL); + rte_eth_promiscuous_disable(test_params.bonded_port_id); + + if (external_sm) { + struct rte_eth_bond_8023ad_conf conf; + + rte_eth_bond_8023ad_conf_get(test_params.bonded_port_id, &conf); + conf.slowrx_cb = lacp_recv_cb; + rte_eth_bond_8023ad_setup(test_params.bonded_port_id, &conf); + + } + + TEST_ASSERT_SUCCESS(rte_eth_dev_start(test_params.bonded_port_id), + "Failed to start bonded device"); + + return TEST_SUCCESS; +} + +static int +remove_slaves_and_stop_bonded_device(void) +{ + struct slave_conf *slave; + int retval; + uint16_t slaves[RTE_MAX_ETHPORTS]; + uint16_t i; + + rte_eth_dev_stop(test_params.bonded_port_id); + + FOR_EACH_SLAVE(i, slave) + remove_slave(slave); + + retval = rte_eth_bond_slaves_get(test_params.bonded_port_id, slaves, + RTE_DIM(slaves)); + + TEST_ASSERT_EQUAL(retval, 0, + "Expected bonded device %u have 0 slaves but returned %d.", + test_params.bonded_port_id, retval); + + FOR_EACH_PORT(i, slave) { + rte_eth_dev_stop(slave->port_id); + + TEST_ASSERT(slave->bonded == 0, + "Port id=%u is still marked as enslaved.", slave->port_id); + } + + return TEST_SUCCESS; +} + +static int +test_setup(void) +{ + int retval, nb_mbuf_per_pool; + char name[RTE_ETH_NAME_MAX_LEN]; + struct slave_conf *port; + const uint8_t socket_id = rte_socket_id(); + uint16_t i; + + if (test_params.mbuf_pool == NULL) { + nb_mbuf_per_pool = TEST_RX_DESC_MAX + DEF_PKT_BURST + + TEST_TX_DESC_MAX + MAX_PKT_BURST; + test_params.mbuf_pool = rte_pktmbuf_pool_create("TEST_MODE4", + nb_mbuf_per_pool, MBUF_CACHE_SIZE, 0, + RTE_MBUF_DEFAULT_BUF_SIZE, socket_id); + + TEST_ASSERT(test_params.mbuf_pool != NULL, + "rte_mempool_create failed\n"); + } + + /* Create / initialize ring eth devs. */ + FOR_EACH_PORT(i, port) { + port = &test_params.slave_ports[i]; + + if (port->rx_queue == NULL) { + retval = snprintf(name, RTE_DIM(name), SLAVE_RX_QUEUE_FMT, i); + TEST_ASSERT(retval <= (int)RTE_DIM(name) - 1, "Name too long"); + port->rx_queue = rte_ring_create(name, RX_RING_SIZE, socket_id, 0); + TEST_ASSERT(port->rx_queue != NULL, + "Failed to allocate rx ring '%s': %s", name, + rte_strerror(rte_errno)); + } + + if (port->tx_queue == NULL) { + retval = snprintf(name, RTE_DIM(name), SLAVE_TX_QUEUE_FMT, i); + TEST_ASSERT(retval <= (int)RTE_DIM(name) - 1, "Name too long"); + port->tx_queue = rte_ring_create(name, TX_RING_SIZE, socket_id, 0); + TEST_ASSERT_NOT_NULL(port->tx_queue, + "Failed to allocate tx ring '%s': %s", name, + rte_strerror(rte_errno)); + } + + if (port->port_id == INVALID_PORT_ID) { + retval = snprintf(name, RTE_DIM(name), SLAVE_DEV_NAME_FMT, i); + TEST_ASSERT(retval < (int)RTE_DIM(name) - 1, "Name too long"); + retval = rte_eth_from_rings(name, &port->rx_queue, 1, + &port->tx_queue, 1, socket_id); + TEST_ASSERT(retval >= 0, + "Failed to create ring ethdev '%s'\n", name); + + port->port_id = rte_eth_dev_count() - 1; + } + + retval = configure_ethdev(port->port_id, 1); + TEST_ASSERT_SUCCESS(retval, "Failed to configure virtual ethdev %s\n", + name); + } + + if (test_params.bonded_port_id == INVALID_PORT_ID) { + retval = rte_eth_bond_create(BONDED_DEV_NAME, BONDING_MODE_8023AD, + socket_id); + + TEST_ASSERT(retval >= 0, "Failed to create bonded ethdev %s", + BONDED_DEV_NAME); + + test_params.bonded_port_id = retval; + TEST_ASSERT_SUCCESS(configure_ethdev(test_params.bonded_port_id, 0), + "Failed to configure bonded ethdev %s", BONDED_DEV_NAME); + } else if (rte_eth_bond_mode_get(test_params.bonded_port_id) != + BONDING_MODE_8023AD) { + TEST_ASSERT(rte_eth_bond_mode_set(test_params.bonded_port_id, + BONDING_MODE_8023AD) == 0, + "Failed to set ethdev %d to mode %d", + test_params.bonded_port_id, BONDING_MODE_8023AD); + } + + return 0; +} + +static void +testsuite_teardown(void) +{ + struct slave_conf *port; + uint8_t i; + + /* Only stop ports. + * Any cleanup/reset state is done when particular test is + * started. */ + + rte_eth_dev_stop(test_params.bonded_port_id); + + FOR_EACH_PORT(i, port) + rte_eth_dev_stop(port->port_id); +} + +/* + * Check if given LACP packet. If it is, make make replay packet to force + * COLLECTING state. + * return 0 when pkt is LACP frame, 1 if it is not slow frame, 2 if it is slow + * frame but not LACP + */ +static int +make_lacp_reply(struct slave_conf *slave, struct rte_mbuf *pkt) +{ + struct ether_hdr *hdr; + struct slow_protocol_frame *slow_hdr; + struct lacpdu *lacp; + + /* look for LACP */ + hdr = rte_pktmbuf_mtod(pkt, struct ether_hdr *); + if (hdr->ether_type != rte_cpu_to_be_16(ETHER_TYPE_SLOW)) + return 1; + + slow_hdr = rte_pktmbuf_mtod(pkt, struct slow_protocol_frame *); + /* ignore packets of other types */ + if (slow_hdr->slow_protocol.subtype != SLOW_SUBTYPE_LACP) + return 2; + + slow_hdr = rte_pktmbuf_mtod(pkt, struct slow_protocol_frame *); + + /* Change source address to partner address */ + ether_addr_copy(&parnter_mac_default, &slow_hdr->eth_hdr.s_addr); + slow_hdr->eth_hdr.s_addr.addr_bytes[ETHER_ADDR_LEN - 1] = slave->port_id; + + lacp = (struct lacpdu *) &slow_hdr->slow_protocol; + /* Save last received state */ + slave->lacp_parnter_state = lacp->actor.state; + /* Change it into LACP replay by matching parameters. */ + memcpy(&lacp->partner.port_params, &lacp->actor.port_params, + sizeof(struct port_params)); + + lacp->partner.state = lacp->actor.state; + + ether_addr_copy(&parnter_system, &lacp->actor.port_params.system); + lacp->actor.state = STATE_LACP_ACTIVE | + STATE_SYNCHRONIZATION | + STATE_AGGREGATION | + STATE_COLLECTING | + STATE_DISTRIBUTING; + + return 0; +} + +/* + * Reads packets from given slave, search for LACP packet and reply them. + * + * Receives burst of packets from slave. Looks for LACP packet. Drops + * all other packets. Prepares response LACP and sends it back. + * + * return number of LACP received and replied, -1 on error. + */ +static int +bond_handshake_reply(struct slave_conf *slave) +{ + int retval; + struct rte_mbuf *rx_buf[MAX_PKT_BURST]; + struct rte_mbuf *lacp_tx_buf[MAX_PKT_BURST]; + uint16_t lacp_tx_buf_cnt = 0, i; + + retval = slave_get_pkts(slave, rx_buf, RTE_DIM(rx_buf)); + TEST_ASSERT(retval >= 0, "Getting slave %u packets failed.", + slave->port_id); + + for (i = 0; i < (uint16_t)retval; i++) { + if (make_lacp_reply(slave, rx_buf[i]) == 0) { + /* reply with actor's LACP */ + lacp_tx_buf[lacp_tx_buf_cnt++] = rx_buf[i]; + } else + rte_pktmbuf_free(rx_buf[i]); + } + + if (lacp_tx_buf_cnt == 0) + return 0; + + retval = slave_put_pkts(slave, lacp_tx_buf, lacp_tx_buf_cnt); + if (retval <= lacp_tx_buf_cnt) { + /* retval might be negative */ + for (i = RTE_MAX(0, retval); retval < lacp_tx_buf_cnt; retval++) + rte_pktmbuf_free(lacp_tx_buf[i]); + } + + TEST_ASSERT_EQUAL(retval, lacp_tx_buf_cnt, + "Failed to equeue lacp packets into slave %u tx queue.", + slave->port_id); + + return lacp_tx_buf_cnt; +} + +/* + * Function check if given slave tx queue contains packets that make mode 4 + * handshake complete. It will drain slave queue. + * return 0 if handshake not completed, 1 if handshake was complete, + */ +static int +bond_handshake_done(struct slave_conf *slave) +{ + const uint8_t expected_state = STATE_LACP_ACTIVE | STATE_SYNCHRONIZATION | + STATE_AGGREGATION | STATE_COLLECTING | STATE_DISTRIBUTING; + + return slave->lacp_parnter_state == expected_state; +} + +static unsigned +bond_get_update_timeout_ms(void) +{ + struct rte_eth_bond_8023ad_conf conf; + + rte_eth_bond_8023ad_conf_get(test_params.bonded_port_id, &conf); + return conf.update_timeout_ms; +} + +/* + * Exchanges LACP packets with partner to achieve dynamic port configuration. + * return TEST_SUCCESS if initial handshake succeed, TEST_FAILED otherwise. + */ +static int +bond_handshake(void) +{ + struct slave_conf *slave; + struct rte_mbuf *buf[MAX_PKT_BURST]; + uint16_t nb_pkts; + uint8_t all_slaves_done, i, j; + uint8_t status[RTE_DIM(test_params.slave_ports)] = { 0 }; + const unsigned delay = bond_get_update_timeout_ms(); + + /* Exchange LACP frames */ + all_slaves_done = 0; + for (i = 0; i < 30 && all_slaves_done == 0; ++i) { + rte_delay_ms(delay); + + all_slaves_done = 1; + FOR_EACH_SLAVE(j, slave) { + /* If response already send, skip slave */ + if (status[j] != 0) + continue; + + if (bond_handshake_reply(slave) < 0) { + all_slaves_done = 0; + break; + } + + status[j] = bond_handshake_done(slave); + if (status[j] == 0) + all_slaves_done = 0; + } + + nb_pkts = bond_tx(NULL, 0); + TEST_ASSERT_EQUAL(nb_pkts, 0, "Packets transmitted unexpectedly"); + + nb_pkts = bond_rx(buf, RTE_DIM(buf)); + free_pkts(buf, nb_pkts); + TEST_ASSERT_EQUAL(nb_pkts, 0, "Packets received unexpectedly"); + } + /* If response didn't send - report failure */ + TEST_ASSERT_EQUAL(all_slaves_done, 1, "Bond handshake failed\n"); + + /* If flags doesn't match - report failure */ + return all_slaves_done == 1 ? TEST_SUCCESS : TEST_FAILED; +} + +#define TEST_LACP_SLAVE_COUT RTE_DIM(test_params.slave_ports) +static int +test_mode4_lacp(void) +{ + int retval; + + retval = initialize_bonded_device_with_slaves(TEST_LACP_SLAVE_COUT, 0); + TEST_ASSERT_SUCCESS(retval, "Failed to initialize bonded device"); + + /* Test LACP handshake function */ + retval = bond_handshake(); + TEST_ASSERT_SUCCESS(retval, "Initial handshake failed"); + + retval = remove_slaves_and_stop_bonded_device(); + TEST_ASSERT_SUCCESS(retval, "Test cleanup failed."); + + return TEST_SUCCESS; +} +static int +test_mode4_agg_mode_selection(void) +{ + int retval; + /* Test and verify for Stable mode */ + retval = initialize_bonded_device_with_slaves(TEST_LACP_SLAVE_COUT, 0); + TEST_ASSERT_SUCCESS(retval, "Failed to initialize bonded device"); + + + retval = rte_eth_bond_8023ad_agg_selection_set( + test_params.bonded_port_id, AGG_STABLE); + TEST_ASSERT_SUCCESS(retval, "Failed to initialize bond aggregation mode"); + retval = bond_handshake(); + TEST_ASSERT_SUCCESS(retval, "Initial handshake failed"); + + + retval = rte_eth_bond_8023ad_agg_selection_get( + test_params.bonded_port_id); + TEST_ASSERT_EQUAL(retval, AGG_STABLE, + "Wrong agg mode received from bonding device"); + + retval = remove_slaves_and_stop_bonded_device(); + TEST_ASSERT_SUCCESS(retval, "Test cleanup failed."); + + + /* test and verify for Bandwidth mode */ + retval = initialize_bonded_device_with_slaves(TEST_LACP_SLAVE_COUT, 0); + TEST_ASSERT_SUCCESS(retval, "Failed to initialize bonded device"); + + + retval = rte_eth_bond_8023ad_agg_selection_set( + test_params.bonded_port_id, + AGG_BANDWIDTH); + TEST_ASSERT_SUCCESS(retval, + "Failed to initialize bond aggregation mode"); + retval = bond_handshake(); + TEST_ASSERT_SUCCESS(retval, "Initial handshake failed"); + + retval = rte_eth_bond_8023ad_agg_selection_get( + test_params.bonded_port_id); + TEST_ASSERT_EQUAL(retval, AGG_BANDWIDTH, + "Wrong agg mode received from bonding device"); + + retval = remove_slaves_and_stop_bonded_device(); + TEST_ASSERT_SUCCESS(retval, "Test cleanup failed."); + + /* test and verify selection for count mode */ + retval = initialize_bonded_device_with_slaves(TEST_LACP_SLAVE_COUT, 0); + TEST_ASSERT_SUCCESS(retval, "Failed to initialize bonded device"); + + + retval = rte_eth_bond_8023ad_agg_selection_set( + test_params.bonded_port_id, AGG_COUNT); + TEST_ASSERT_SUCCESS(retval, + "Failed to initialize bond aggregation mode"); + retval = bond_handshake(); + TEST_ASSERT_SUCCESS(retval, "Initial handshake failed"); + + retval = rte_eth_bond_8023ad_agg_selection_get( + test_params.bonded_port_id); + TEST_ASSERT_EQUAL(retval, AGG_COUNT, + "Wrong agg mode received from bonding device"); + + retval = remove_slaves_and_stop_bonded_device(); + TEST_ASSERT_SUCCESS(retval, "Test cleanup failed."); + + return TEST_SUCCESS; +} + +static int +generate_packets(struct ether_addr *src_mac, + struct ether_addr *dst_mac, uint16_t count, struct rte_mbuf **buf) +{ + uint16_t pktlen = PACKET_BURST_GEN_PKT_LEN; + uint8_t vlan_enable = 0; + uint16_t vlan_id = 0; + uint8_t ip4_type = 1; /* 0 - ipv6 */ + + uint16_t src_port = 10, dst_port = 20; + + uint32_t ip_src[4] = { [0 ... 2] = 0xDEADBEEF, [3] = IPv4(192, 168, 0, 1) }; + uint32_t ip_dst[4] = { [0 ... 2] = 0xFEEDFACE, [3] = IPv4(192, 168, 0, 2) }; + + struct ether_hdr pkt_eth_hdr; + struct udp_hdr pkt_udp_hdr; + union { + struct ipv4_hdr v4; + struct ipv6_hdr v6; + } pkt_ip_hdr; + + int retval; + + initialize_eth_header(&pkt_eth_hdr, src_mac, dst_mac, ip4_type, + vlan_enable, vlan_id); + + if (ip4_type) + initialize_ipv4_header(&pkt_ip_hdr.v4, ip_src[3], ip_dst[3], pktlen); + else + initialize_ipv6_header(&pkt_ip_hdr.v6, (uint8_t *)ip_src, + (uint8_t *)&ip_dst, pktlen); + + initialize_udp_header(&pkt_udp_hdr, src_port, dst_port, 16); + + retval = generate_packet_burst(test_params.mbuf_pool, buf, + &pkt_eth_hdr, vlan_enable, &pkt_ip_hdr, 1, &pkt_udp_hdr, + count, pktlen, 1); + + if (retval > 0 && retval != count) + free_pkts(&buf[count - retval], retval); + + TEST_ASSERT_EQUAL(retval, count, "Failed to generate %u packets", + count); + + return count; +} + +static int +generate_and_put_packets(struct slave_conf *slave, struct ether_addr *src_mac, + struct ether_addr *dst_mac, uint16_t count) +{ + struct rte_mbuf *pkts[MAX_PKT_BURST]; + int retval; + + retval = generate_packets(src_mac, dst_mac, count, pkts); + if (retval != (int)count) + return retval; + + retval = slave_put_pkts(slave, pkts, count); + if (retval > 0 && retval != count) + free_pkts(&pkts[retval], count - retval); + + TEST_ASSERT_EQUAL(retval, count, + "Failed to enqueue packets into slave %u RX queue", slave->port_id); + + return TEST_SUCCESS; +} + +static int +test_mode4_rx(void) +{ + struct slave_conf *slave; + uint16_t i, j; + + uint16_t expected_pkts_cnt; + struct rte_mbuf *pkts[MAX_PKT_BURST]; + int retval; + unsigned delay; + + struct ether_hdr *hdr; + + struct ether_addr src_mac = { { 0x00, 0xFF, 0x00, 0xFF, 0x00, 0x00 } }; + struct ether_addr dst_mac; + struct ether_addr bonded_mac; + + retval = initialize_bonded_device_with_slaves(TEST_PROMISC_SLAVE_COUNT, + 0); + TEST_ASSERT_SUCCESS(retval, "Failed to initialize bonded device"); + + retval = bond_handshake(); + TEST_ASSERT_SUCCESS(retval, "Initial handshake failed"); + + rte_eth_macaddr_get(test_params.bonded_port_id, &bonded_mac); + ether_addr_copy(&bonded_mac, &dst_mac); + + /* Assert that dst address is not bonding address. Do not set the + * least significant bit of the zero byte as this would create a + * multicast address. + */ + dst_mac.addr_bytes[0] += 2; + + /* First try with promiscuous mode enabled. + * Add 2 packets to each slave. First with bonding MAC address, second with + * different. Check if we received all of them. */ + rte_eth_promiscuous_enable(test_params.bonded_port_id); + + expected_pkts_cnt = 0; + FOR_EACH_SLAVE(i, slave) { + retval = generate_and_put_packets(slave, &src_mac, &bonded_mac, 1); + TEST_ASSERT_SUCCESS(retval, "Failed to enqueue packets to slave %u", + slave->port_id); + + retval = generate_and_put_packets(slave, &src_mac, &dst_mac, 1); + TEST_ASSERT_SUCCESS(retval, "Failed to enqueue packets to slave %u", + slave->port_id); + + /* Expect 2 packets per slave */ + expected_pkts_cnt += 2; + } + + retval = rte_eth_rx_burst(test_params.bonded_port_id, 0, pkts, + RTE_DIM(pkts)); + + if (retval == expected_pkts_cnt) { + int cnt[2] = { 0, 0 }; + + for (i = 0; i < expected_pkts_cnt; i++) { + hdr = rte_pktmbuf_mtod(pkts[i], struct ether_hdr *); + cnt[is_same_ether_addr(&hdr->d_addr, &bonded_mac)]++; + } + + free_pkts(pkts, expected_pkts_cnt); + + /* For division by 2 expected_pkts_cnt must be even */ + RTE_VERIFY((expected_pkts_cnt & 1) == 0); + TEST_ASSERT(cnt[0] == expected_pkts_cnt / 2 && + cnt[1] == expected_pkts_cnt / 2, + "Expected %u packets with the same MAC and %u with different but " + "got %u with the same and %u with different MAC", + expected_pkts_cnt / 2, expected_pkts_cnt / 2, cnt[1], cnt[0]); + } else if (retval > 0) + free_pkts(pkts, retval); + + TEST_ASSERT_EQUAL(retval, expected_pkts_cnt, + "Expected %u packets but received only %d", expected_pkts_cnt, retval); + + /* Now, disable promiscuous mode. When promiscuous mode is disabled we + * expect to receive only packets that are directed to bonding port. */ + rte_eth_promiscuous_disable(test_params.bonded_port_id); + + expected_pkts_cnt = 0; + FOR_EACH_SLAVE(i, slave) { + retval = generate_and_put_packets(slave, &src_mac, &bonded_mac, 1); + TEST_ASSERT_SUCCESS(retval, "Failed to enqueue packets to slave %u", + slave->port_id); + + retval = generate_and_put_packets(slave, &src_mac, &dst_mac, 1); + TEST_ASSERT_SUCCESS(retval, "Failed to enqueue packets to slave %u", + slave->port_id); + + /* Expect only one packet per slave */ + expected_pkts_cnt += 1; + } + + retval = rte_eth_rx_burst(test_params.bonded_port_id, 0, pkts, + RTE_DIM(pkts)); + + if (retval == expected_pkts_cnt) { + int eq_cnt = 0; + + for (i = 0; i < expected_pkts_cnt; i++) { + hdr = rte_pktmbuf_mtod(pkts[i], struct ether_hdr *); + eq_cnt += is_same_ether_addr(&hdr->d_addr, &bonded_mac); + } + + free_pkts(pkts, expected_pkts_cnt); + TEST_ASSERT_EQUAL(eq_cnt, expected_pkts_cnt, "Packet address mismatch"); + } else if (retval > 0) + free_pkts(pkts, retval); + + TEST_ASSERT_EQUAL(retval, expected_pkts_cnt, + "Expected %u packets but received only %d", expected_pkts_cnt, retval); + + /* Link down test: simulate link down for first slave. */ + delay = bond_get_update_timeout_ms(); + + uint8_t slave_down_id = INVALID_PORT_ID; + + /* Find first slave and make link down on it*/ + FOR_EACH_SLAVE(i, slave) { + rte_eth_dev_set_link_down(slave->port_id); + slave_down_id = slave->port_id; + break; + } + + RTE_VERIFY(slave_down_id != INVALID_PORT_ID); + + /* Give some time to rearrange bonding */ + for (i = 0; i < 3; i++) { + rte_delay_ms(delay); + bond_handshake(); + } + + TEST_ASSERT_SUCCESS(bond_handshake(), "Handshake after link down failed"); + + /* Put packet to each slave */ + FOR_EACH_SLAVE(i, slave) { + void *pkt = NULL; + + dst_mac.addr_bytes[ETHER_ADDR_LEN - 1] = slave->port_id; + retval = generate_and_put_packets(slave, &src_mac, &dst_mac, 1); + TEST_ASSERT_SUCCESS(retval, "Failed to generate test packet burst."); + + src_mac.addr_bytes[ETHER_ADDR_LEN - 1] = slave->port_id; + retval = generate_and_put_packets(slave, &src_mac, &bonded_mac, 1); + TEST_ASSERT_SUCCESS(retval, "Failed to generate test packet burst."); + + retval = bond_rx(pkts, RTE_DIM(pkts)); + + /* Clean anything */ + if (retval > 0) + free_pkts(pkts, retval); + + while (rte_ring_dequeue(slave->rx_queue, (void **)&pkt) == 0) + rte_pktmbuf_free(pkt); + + if (slave_down_id == slave->port_id) + TEST_ASSERT_EQUAL(retval, 0, "Packets received unexpectedly."); + else + TEST_ASSERT_NOT_EQUAL(retval, 0, + "Expected to receive some packets on slave %u.", + slave->port_id); + rte_eth_dev_start(slave->port_id); + + for (j = 0; j < 5; j++) { + TEST_ASSERT(bond_handshake_reply(slave) >= 0, + "Handshake after link up"); + + if (bond_handshake_done(slave) == 1) + break; + } + + TEST_ASSERT(j < 5, "Failed to aggregate slave after link up"); + } + + remove_slaves_and_stop_bonded_device(); + return TEST_SUCCESS; +} + +static int +test_mode4_tx_burst(void) +{ + struct slave_conf *slave; + uint16_t i, j; + + uint16_t exp_pkts_cnt, pkts_cnt = 0; + struct rte_mbuf *pkts[MAX_PKT_BURST]; + int retval; + unsigned delay; + + struct ether_addr dst_mac = { { 0x00, 0xFF, 0x00, 0xFF, 0x00, 0x00 } }; + struct ether_addr bonded_mac; + + retval = initialize_bonded_device_with_slaves(TEST_TX_SLAVE_COUNT, 0); + TEST_ASSERT_SUCCESS(retval, "Failed to initialize bonded device"); + + retval = bond_handshake(); + TEST_ASSERT_SUCCESS(retval, "Initial handshake failed"); + + rte_eth_macaddr_get(test_params.bonded_port_id, &bonded_mac); + + /* Prepare burst */ + for (pkts_cnt = 0; pkts_cnt < RTE_DIM(pkts); pkts_cnt++) { + dst_mac.addr_bytes[ETHER_ADDR_LEN - 1] = pkts_cnt; + retval = generate_packets(&bonded_mac, &dst_mac, 1, &pkts[pkts_cnt]); + + if (retval != 1) + free_pkts(pkts, pkts_cnt); + + TEST_ASSERT_EQUAL(retval, 1, "Failed to generate packet %u", pkts_cnt); + } + exp_pkts_cnt = pkts_cnt; + + /* Transmit packets on bonded device */ + retval = bond_tx(pkts, pkts_cnt); + if (retval > 0 && retval < pkts_cnt) + free_pkts(&pkts[retval], pkts_cnt - retval); + + TEST_ASSERT_EQUAL(retval, pkts_cnt, "TX on bonded device failed"); + + /* Check if packets were transmitted properly. Every slave should have + * at least one packet, and sum must match. Under normal operation + * there should be no LACP nor MARKER frames. */ + pkts_cnt = 0; + FOR_EACH_SLAVE(i, slave) { + uint16_t normal_cnt, slow_cnt; + + retval = slave_get_pkts(slave, pkts, RTE_DIM(pkts)); + normal_cnt = 0; + slow_cnt = 0; + + for (j = 0; j < retval; j++) { + if (make_lacp_reply(slave, pkts[j]) == 1) + normal_cnt++; + else + slow_cnt++; + } + + free_pkts(pkts, normal_cnt + slow_cnt); + TEST_ASSERT_EQUAL(slow_cnt, 0, + "slave %u unexpectedly transmitted %d SLOW packets", slave->port_id, + slow_cnt); + + TEST_ASSERT_NOT_EQUAL(normal_cnt, 0, + "slave %u did not transmitted any packets", slave->port_id); + + pkts_cnt += normal_cnt; + } + + TEST_ASSERT_EQUAL(exp_pkts_cnt, pkts_cnt, + "Expected %u packets but transmitted only %d", exp_pkts_cnt, pkts_cnt); + + /* Link down test: + * simulate link down for first slave. */ + delay = bond_get_update_timeout_ms(); + + uint8_t slave_down_id = INVALID_PORT_ID; + + FOR_EACH_SLAVE(i, slave) { + rte_eth_dev_set_link_down(slave->port_id); + slave_down_id = slave->port_id; + break; + } + + RTE_VERIFY(slave_down_id != INVALID_PORT_ID); + + /* Give some time to rearrange bonding. */ + for (i = 0; i < 3; i++) { + bond_handshake(); + rte_delay_ms(delay); + } + + TEST_ASSERT_SUCCESS(bond_handshake(), "Handshake after link down failed"); + + /* Prepare burst. */ + for (pkts_cnt = 0; pkts_cnt < RTE_DIM(pkts); pkts_cnt++) { + dst_mac.addr_bytes[ETHER_ADDR_LEN - 1] = pkts_cnt; + retval = generate_packets(&bonded_mac, &dst_mac, 1, &pkts[pkts_cnt]); + + if (retval != 1) + free_pkts(pkts, pkts_cnt); + + TEST_ASSERT_EQUAL(retval, 1, "Failed to generate test packet %u", + pkts_cnt); + } + exp_pkts_cnt = pkts_cnt; + + /* Transmit packets on bonded device. */ + retval = bond_tx(pkts, pkts_cnt); + if (retval > 0 && retval < pkts_cnt) + free_pkts(&pkts[retval], pkts_cnt - retval); + + TEST_ASSERT_EQUAL(retval, pkts_cnt, "TX on bonded device failed"); + + /* Check if packets was transmitted properly. Every slave should have + * at least one packet, and sum must match. Under normal operation + * there should be no LACP nor MARKER frames. */ + pkts_cnt = 0; + FOR_EACH_SLAVE(i, slave) { + uint16_t normal_cnt, slow_cnt; + + retval = slave_get_pkts(slave, pkts, RTE_DIM(pkts)); + normal_cnt = 0; + slow_cnt = 0; + + for (j = 0; j < retval; j++) { + if (make_lacp_reply(slave, pkts[j]) == 1) + normal_cnt++; + else + slow_cnt++; + } + + free_pkts(pkts, normal_cnt + slow_cnt); + + if (slave_down_id == slave->port_id) { + TEST_ASSERT_EQUAL(normal_cnt + slow_cnt, 0, + "slave %u enexpectedly transmitted %u packets", + normal_cnt + slow_cnt, slave->port_id); + } else { + TEST_ASSERT_EQUAL(slow_cnt, 0, + "slave %u unexpectedly transmitted %d SLOW packets", + slave->port_id, slow_cnt); + + TEST_ASSERT_NOT_EQUAL(normal_cnt, 0, + "slave %u did not transmitted any packets", slave->port_id); + } + + pkts_cnt += normal_cnt; + } + + TEST_ASSERT_EQUAL(exp_pkts_cnt, pkts_cnt, + "Expected %u packets but transmitted only %d", exp_pkts_cnt, pkts_cnt); + + return remove_slaves_and_stop_bonded_device(); +} + +static void +init_marker(struct rte_mbuf *pkt, struct slave_conf *slave) +{ + struct marker_header *marker_hdr = rte_pktmbuf_mtod(pkt, + struct marker_header *); + + /* Copy multicast destination address */ + ether_addr_copy(&slow_protocol_mac_addr, &marker_hdr->eth_hdr.d_addr); + + /* Init source address */ + ether_addr_copy(&parnter_mac_default, &marker_hdr->eth_hdr.s_addr); + marker_hdr->eth_hdr.s_addr.addr_bytes[ETHER_ADDR_LEN-1] = slave->port_id; + + marker_hdr->eth_hdr.ether_type = rte_cpu_to_be_16(ETHER_TYPE_SLOW); + + marker_hdr->marker.subtype = SLOW_SUBTYPE_MARKER; + marker_hdr->marker.version_number = 1; + marker_hdr->marker.tlv_type_marker = MARKER_TLV_TYPE_INFO; + marker_hdr->marker.info_length = + offsetof(struct marker, reserved_90) - + offsetof(struct marker, requester_port); + RTE_VERIFY(marker_hdr->marker.info_length == 16); + marker_hdr->marker.requester_port = slave->port_id + 1; + marker_hdr->marker.tlv_type_terminator = TLV_TYPE_TERMINATOR_INFORMATION; + marker_hdr->marker.terminator_length = 0; +} + +static int +test_mode4_marker(void) +{ + struct slave_conf *slave; + struct rte_mbuf *pkts[MAX_PKT_BURST]; + struct rte_mbuf *marker_pkt; + struct marker_header *marker_hdr; + + unsigned delay; + int retval; + uint16_t nb_pkts; + uint8_t i, j; + const uint16_t ethtype_slow_be = rte_be_to_cpu_16(ETHER_TYPE_SLOW); + + retval = initialize_bonded_device_with_slaves(TEST_MARKER_SLAVE_COUT, + 0); + TEST_ASSERT_SUCCESS(retval, "Failed to initialize bonded device"); + + /* Test LACP handshake function */ + retval = bond_handshake(); + TEST_ASSERT_SUCCESS(retval, "Initial handshake failed"); + + delay = bond_get_update_timeout_ms(); + FOR_EACH_SLAVE(i, slave) { + marker_pkt = rte_pktmbuf_alloc(test_params.mbuf_pool); + TEST_ASSERT_NOT_NULL(marker_pkt, "Failed to allocate marker packet"); + init_marker(marker_pkt, slave); + + retval = slave_put_pkts(slave, &marker_pkt, 1); + if (retval != 1) + rte_pktmbuf_free(marker_pkt); + + TEST_ASSERT_EQUAL(retval, 1, + "Failed to send marker packet to slave %u", slave->port_id); + + for (j = 0; j < 20; ++j) { + rte_delay_ms(delay); + retval = rte_eth_rx_burst(test_params.bonded_port_id, 0, pkts, + RTE_DIM(pkts)); + + if (retval > 0) + free_pkts(pkts, retval); + + TEST_ASSERT_EQUAL(retval, 0, "Received packets unexpectedly"); + + retval = rte_eth_tx_burst(test_params.bonded_port_id, 0, NULL, 0); + TEST_ASSERT_EQUAL(retval, 0, + "Requested TX of 0 packets but %d transmitted", retval); + + /* Check if LACP packet was send by state machines + First and only packet must be a maker response */ + retval = slave_get_pkts(slave, pkts, MAX_PKT_BURST); + if (retval == 0) + continue; + if (retval > 1) + free_pkts(pkts, retval); + + TEST_ASSERT_EQUAL(retval, 1, "failed to get slave packets"); + nb_pkts = retval; + + marker_hdr = rte_pktmbuf_mtod(pkts[0], struct marker_header *); + /* Check if it's slow packet*/ + if (marker_hdr->eth_hdr.ether_type != ethtype_slow_be) + retval = -1; + /* Check if it's marker packet */ + else if (marker_hdr->marker.subtype != SLOW_SUBTYPE_MARKER) + retval = -2; + else if (marker_hdr->marker.tlv_type_marker != MARKER_TLV_TYPE_RESP) + retval = -3; + + free_pkts(pkts, nb_pkts); + + TEST_ASSERT_NOT_EQUAL(retval, -1, "Unexpected protocol type"); + TEST_ASSERT_NOT_EQUAL(retval, -2, "Unexpected sub protocol type"); + TEST_ASSERT_NOT_EQUAL(retval, -3, "Unexpected marker type"); + break; + } + + TEST_ASSERT(j < 20, "Marker response not found"); + } + + retval = remove_slaves_and_stop_bonded_device(); + TEST_ASSERT_SUCCESS(retval, "Test cleanup failed."); + + return TEST_SUCCESS; +} + +static int +test_mode4_expired(void) +{ + struct slave_conf *slave, *exp_slave = NULL; + struct rte_mbuf *pkts[MAX_PKT_BURST]; + int retval; + uint32_t old_delay; + + uint8_t i; + uint16_t j; + + struct rte_eth_bond_8023ad_conf conf; + + retval = initialize_bonded_device_with_slaves(TEST_EXPIRED_SLAVE_COUNT, + 0); + /* Set custom timeouts to make test last shorter. */ + rte_eth_bond_8023ad_conf_get(test_params.bonded_port_id, &conf); + conf.fast_periodic_ms = 100; + conf.slow_periodic_ms = 600; + conf.short_timeout_ms = 300; + conf.long_timeout_ms = 900; + conf.aggregate_wait_timeout_ms = 200; + conf.tx_period_ms = 100; + old_delay = conf.update_timeout_ms; + conf.update_timeout_ms = 10; + rte_eth_bond_8023ad_setup(test_params.bonded_port_id, &conf); + + /* Wait for new settings to be applied. */ + for (i = 0; i < old_delay/conf.update_timeout_ms * 2; i++) { + FOR_EACH_SLAVE(j, slave) + bond_handshake_reply(slave); + + rte_delay_ms(conf.update_timeout_ms); + } + + retval = bond_handshake(); + TEST_ASSERT_SUCCESS(retval, "Initial handshake failed"); + + /* Find first slave */ + FOR_EACH_SLAVE(i, slave) { + exp_slave = slave; + break; + } + + RTE_VERIFY(exp_slave != NULL); + + /* When one of partners do not send or respond to LACP frame in + * conf.long_timeout_ms time, internal state machines should detect this + * and transit to expired state. */ + for (j = 0; j < conf.long_timeout_ms/conf.update_timeout_ms + 2; j++) { + rte_delay_ms(conf.update_timeout_ms); + + retval = bond_tx(NULL, 0); + TEST_ASSERT_EQUAL(retval, 0, "Unexpectedly received %d packets", + retval); + + FOR_EACH_SLAVE(i, slave) { + retval = bond_handshake_reply(slave); + TEST_ASSERT(retval >= 0, "Handshake failed"); + + /* Remove replay for slave that suppose to be expired. */ + if (slave == exp_slave) { + while (rte_ring_count(slave->rx_queue) > 0) { + void *pkt = NULL; + + rte_ring_dequeue(slave->rx_queue, &pkt); + rte_pktmbuf_free(pkt); + } + } + } + + retval = bond_rx(pkts, RTE_DIM(pkts)); + if (retval > 0) + free_pkts(pkts, retval); + + TEST_ASSERT_EQUAL(retval, 0, "Unexpectedly received %d packets", + retval); + } + + /* After test only expected slave should be in EXPIRED state */ + FOR_EACH_SLAVE(i, slave) { + if (slave == exp_slave) + TEST_ASSERT(slave->lacp_parnter_state & STATE_EXPIRED, + "Slave %u should be in expired.", slave->port_id); + else + TEST_ASSERT_EQUAL(bond_handshake_done(slave), 1, + "Slave %u should be operational.", slave->port_id); + } + + retval = remove_slaves_and_stop_bonded_device(); + TEST_ASSERT_SUCCESS(retval, "Test cleanup failed."); + + return TEST_SUCCESS; +} + +static int +test_mode4_ext_ctrl(void) +{ + /* + * configure bonded interface without the external sm enabled + * . try to transmit lacpdu (should fail) + * . try to set collecting and distributing flags (should fail) + * reconfigure w/external sm + * . transmit one lacpdu on each slave using new api + * . make sure each slave receives one lacpdu using the callback api + * . transmit one data pdu on each slave (should fail) + * . enable distribution and collection, send one data pdu each again + */ + + int retval; + struct slave_conf *slave = NULL; + uint8_t i; + + struct rte_mbuf *lacp_tx_buf[SLAVE_COUNT]; + struct ether_addr src_mac, dst_mac; + struct lacpdu_header lacpdu = { + .lacpdu = { + .subtype = SLOW_SUBTYPE_LACP, + }, + }; + + ether_addr_copy(&parnter_system, &src_mac); + ether_addr_copy(&slow_protocol_mac_addr, &dst_mac); + + initialize_eth_header(&lacpdu.eth_hdr, &src_mac, &dst_mac, + ETHER_TYPE_SLOW, 0, 0); + + for (i = 0; i < SLAVE_COUNT; i++) { + lacp_tx_buf[i] = rte_pktmbuf_alloc(test_params.mbuf_pool); + rte_memcpy(rte_pktmbuf_mtod(lacp_tx_buf[i], char *), + &lacpdu, sizeof(lacpdu)); + rte_pktmbuf_pkt_len(lacp_tx_buf[i]) = sizeof(lacpdu); + } + + retval = initialize_bonded_device_with_slaves(TEST_TX_SLAVE_COUNT, 0); + TEST_ASSERT_SUCCESS(retval, "Failed to initialize bonded device"); + + FOR_EACH_SLAVE(i, slave) { + TEST_ASSERT_FAIL(rte_eth_bond_8023ad_ext_slowtx( + test_params.bonded_port_id, + slave->port_id, lacp_tx_buf[i]), + "Slave should not allow manual LACP xmit"); + TEST_ASSERT_FAIL(rte_eth_bond_8023ad_ext_collect( + test_params.bonded_port_id, + slave->port_id, 1), + "Slave should not allow external state controls"); + } + + free_pkts(lacp_tx_buf, RTE_DIM(lacp_tx_buf)); + + retval = remove_slaves_and_stop_bonded_device(); + TEST_ASSERT_SUCCESS(retval, "Bonded device cleanup failed."); + + return TEST_SUCCESS; +} + + +static int +test_mode4_ext_lacp(void) +{ + int retval; + struct slave_conf *slave = NULL; + uint8_t all_slaves_done = 0, i; + uint16_t nb_pkts; + const unsigned int delay = bond_get_update_timeout_ms(); + + struct rte_mbuf *lacp_tx_buf[SLAVE_COUNT]; + struct rte_mbuf *buf[SLAVE_COUNT]; + struct ether_addr src_mac, dst_mac; + struct lacpdu_header lacpdu = { + .lacpdu = { + .subtype = SLOW_SUBTYPE_LACP, + }, + }; + + ether_addr_copy(&parnter_system, &src_mac); + ether_addr_copy(&slow_protocol_mac_addr, &dst_mac); + + initialize_eth_header(&lacpdu.eth_hdr, &src_mac, &dst_mac, + ETHER_TYPE_SLOW, 0, 0); + + for (i = 0; i < SLAVE_COUNT; i++) { + lacp_tx_buf[i] = rte_pktmbuf_alloc(test_params.mbuf_pool); + rte_memcpy(rte_pktmbuf_mtod(lacp_tx_buf[i], char *), + &lacpdu, sizeof(lacpdu)); + rte_pktmbuf_pkt_len(lacp_tx_buf[i]) = sizeof(lacpdu); + } + + retval = initialize_bonded_device_with_slaves(TEST_TX_SLAVE_COUNT, 1); + TEST_ASSERT_SUCCESS(retval, "Failed to initialize bonded device"); + + memset(lacpdu_rx_count, 0, sizeof(lacpdu_rx_count)); + + /* Wait for new settings to be applied. */ + for (i = 0; i < 30; ++i) + rte_delay_ms(delay); + + FOR_EACH_SLAVE(i, slave) { + retval = rte_eth_bond_8023ad_ext_slowtx( + test_params.bonded_port_id, + slave->port_id, lacp_tx_buf[i]); + TEST_ASSERT_SUCCESS(retval, + "Slave should allow manual LACP xmit"); + } + + nb_pkts = bond_tx(NULL, 0); + TEST_ASSERT_EQUAL(nb_pkts, 0, "Packets transmitted unexpectedly"); + + FOR_EACH_SLAVE(i, slave) { + nb_pkts = slave_get_pkts(slave, buf, RTE_DIM(buf)); + TEST_ASSERT_EQUAL(nb_pkts, 1, "found %u packets on slave %d\n", + nb_pkts, i); + slave_put_pkts(slave, buf, nb_pkts); + } + + nb_pkts = bond_rx(buf, RTE_DIM(buf)); + free_pkts(buf, nb_pkts); + TEST_ASSERT_EQUAL(nb_pkts, 0, "Packets received unexpectedly"); + + /* wait for the periodic callback to run */ + for (i = 0; i < 30 && all_slaves_done == 0; ++i) { + uint8_t s, total = 0; + + rte_delay_ms(delay); + FOR_EACH_SLAVE(s, slave) { + total += lacpdu_rx_count[slave->port_id]; + } + + if (total >= SLAVE_COUNT) + all_slaves_done = 1; + } + + FOR_EACH_SLAVE(i, slave) { + TEST_ASSERT_EQUAL(lacpdu_rx_count[slave->port_id], 1, + "Slave port %u should have received 1 lacpdu (count=%u)", + slave->port_id, + lacpdu_rx_count[slave->port_id]); + } + + retval = remove_slaves_and_stop_bonded_device(); + TEST_ASSERT_SUCCESS(retval, "Test cleanup failed."); + + return TEST_SUCCESS; +} + +static int +check_environment(void) +{ + struct slave_conf *port; + uint8_t i, env_state; + uint16_t slaves[RTE_DIM(test_params.slave_ports)]; + int slaves_count; + + env_state = 0; + FOR_EACH_PORT(i, port) { + if (rte_ring_count(port->rx_queue) != 0) + env_state |= 0x01; + + if (rte_ring_count(port->tx_queue) != 0) + env_state |= 0x02; + + if (port->bonded != 0) + env_state |= 0x04; + + if (port->lacp_parnter_state != 0) + env_state |= 0x08; + + if (env_state != 0) + break; + } + + slaves_count = rte_eth_bond_slaves_get(test_params.bonded_port_id, + slaves, RTE_DIM(slaves)); + + if (slaves_count != 0) + env_state |= 0x10; + + TEST_ASSERT_EQUAL(env_state, 0, + "Environment not clean (port %u):%s%s%s%s%s", + port->port_id, + env_state & 0x01 ? " slave rx queue not clean" : "", + env_state & 0x02 ? " slave tx queue not clean" : "", + env_state & 0x04 ? " port marked as enslaved" : "", + env_state & 0x80 ? " slave state is not reset" : "", + env_state & 0x10 ? " slave count not equal 0" : "."); + + + return TEST_SUCCESS; +} + +static int +test_mode4_executor(int (*test_func)(void)) +{ + struct slave_conf *port; + int test_result; + uint8_t i; + void *pkt; + + /* Check if environment is clean. Fail to launch a test if there was + * a critical error before that prevented to reset environment. */ + TEST_ASSERT_SUCCESS(check_environment(), + "Refusing to launch test in dirty environment."); + + RTE_VERIFY(test_func != NULL); + test_result = (*test_func)(); + + /* If test succeed check if environment wast left in good condition. */ + if (test_result == TEST_SUCCESS) + test_result = check_environment(); + + /* Reset environment in case test failed to do that. */ + if (test_result != TEST_SUCCESS) { + TEST_ASSERT_SUCCESS(remove_slaves_and_stop_bonded_device(), + "Failed to stop bonded device"); + + FOR_EACH_PORT(i, port) { + while (rte_ring_count(port->rx_queue) != 0) { + if (rte_ring_dequeue(port->rx_queue, &pkt) == 0) + rte_pktmbuf_free(pkt); + } + + while (rte_ring_count(port->tx_queue) != 0) { + if (rte_ring_dequeue(port->tx_queue, &pkt) == 0) + rte_pktmbuf_free(pkt); + } + } + } + + return test_result; +} + +static int +test_mode4_agg_mode_selection_wrapper(void){ + return test_mode4_executor(&test_mode4_agg_mode_selection); +} + +static int +test_mode4_lacp_wrapper(void) +{ + return test_mode4_executor(&test_mode4_lacp); +} + +static int +test_mode4_marker_wrapper(void) +{ + return test_mode4_executor(&test_mode4_marker); +} + +static int +test_mode4_rx_wrapper(void) +{ + return test_mode4_executor(&test_mode4_rx); +} + +static int +test_mode4_tx_burst_wrapper(void) +{ + return test_mode4_executor(&test_mode4_tx_burst); +} + +static int +test_mode4_expired_wrapper(void) +{ + return test_mode4_executor(&test_mode4_expired); +} + +static int +test_mode4_ext_ctrl_wrapper(void) +{ + return test_mode4_executor(&test_mode4_ext_ctrl); +} + +static int +test_mode4_ext_lacp_wrapper(void) +{ + return test_mode4_executor(&test_mode4_ext_lacp); +} + +static struct unit_test_suite link_bonding_mode4_test_suite = { + .suite_name = "Link Bonding mode 4 Unit Test Suite", + .setup = test_setup, + .teardown = testsuite_teardown, + .unit_test_cases = { + TEST_CASE_NAMED("test_mode4_agg_mode_selection", + test_mode4_agg_mode_selection_wrapper), + TEST_CASE_NAMED("test_mode4_lacp", test_mode4_lacp_wrapper), + TEST_CASE_NAMED("test_mode4_rx", test_mode4_rx_wrapper), + TEST_CASE_NAMED("test_mode4_tx_burst", test_mode4_tx_burst_wrapper), + TEST_CASE_NAMED("test_mode4_marker", test_mode4_marker_wrapper), + TEST_CASE_NAMED("test_mode4_expired", test_mode4_expired_wrapper), + TEST_CASE_NAMED("test_mode4_ext_ctrl", + test_mode4_ext_ctrl_wrapper), + TEST_CASE_NAMED("test_mode4_ext_lacp", + test_mode4_ext_lacp_wrapper), + + TEST_CASES_END() /**< NULL terminate unit test array */ + } +}; + +static int +test_link_bonding_mode4(void) +{ + return unit_test_suite_runner(&link_bonding_mode4_test_suite); +} + +REGISTER_TEST_COMMAND(link_bonding_mode4_autotest, test_link_bonding_mode4); diff --git a/dpdk/test/test/test_link_bonding_rssconf.c b/dpdk/test/test/test_link_bonding_rssconf.c new file mode 100644 index 00000000..54cbf128 --- /dev/null +++ b/dpdk/test/test/test_link_bonding_rssconf.c @@ -0,0 +1,673 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2015 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <string.h> +#include <stdarg.h> +#include <stdio.h> +#include <stdlib.h> +#include <stdint.h> +#include <inttypes.h> +#include <errno.h> +#include <rte_cycles.h> +#include <sys/queue.h> + +#include <rte_byteorder.h> +#include <rte_common.h> +#include <rte_debug.h> +#include <rte_ethdev.h> +#include <rte_log.h> +#include <rte_lcore.h> +#include <rte_memory.h> +#include <rte_bus_vdev.h> + +#include <rte_string_fns.h> +#include <rte_errno.h> +#include <rte_eth_bond.h> + +#include "test.h" + +#define SLAVE_COUNT (4) + +#define RXTX_RING_SIZE 1024 +#define RXTX_QUEUE_COUNT 4 + +#define BONDED_DEV_NAME ("net_bonding_rss") + +#define SLAVE_DEV_NAME_FMT ("net_null%d") +#define SLAVE_RXTX_QUEUE_FMT ("rssconf_slave%d_q%d") + +#define NUM_MBUFS 8191 +#define MBUF_SIZE (1600 + RTE_PKTMBUF_HEADROOM) +#define MBUF_CACHE_SIZE 250 +#define BURST_SIZE 32 + +#define INVALID_SOCKET_ID (-1) +#define INVALID_PORT_ID (0xFF) +#define INVALID_BONDING_MODE (-1) + +struct slave_conf { + uint16_t port_id; + struct rte_eth_dev_info dev_info; + + struct rte_eth_rss_conf rss_conf; + uint8_t rss_key[40]; + struct rte_eth_rss_reta_entry64 reta_conf[512 / RTE_RETA_GROUP_SIZE]; + + uint8_t is_slave; + struct rte_ring *rxtx_queue[RXTX_QUEUE_COUNT]; +}; + +struct link_bonding_rssconf_unittest_params { + uint8_t bond_port_id; + struct rte_eth_dev_info bond_dev_info; + struct rte_eth_rss_reta_entry64 bond_reta_conf[512 / RTE_RETA_GROUP_SIZE]; + struct slave_conf slave_ports[SLAVE_COUNT]; + + struct rte_mempool *mbuf_pool; +}; + +static struct link_bonding_rssconf_unittest_params test_params = { + .bond_port_id = INVALID_PORT_ID, + .slave_ports = { + [0 ... SLAVE_COUNT - 1] = { .port_id = INVALID_PORT_ID, .is_slave = 0} + }, + .mbuf_pool = NULL, +}; + +/** + * Default port configuration with RSS turned off + */ +static struct rte_eth_conf default_pmd_conf = { + .rxmode = { + .mq_mode = ETH_MQ_RX_NONE, + .max_rx_pkt_len = ETHER_MAX_LEN, + .split_hdr_size = 0, + .header_split = 0, /**< Header Split disabled */ + .hw_ip_checksum = 0, /**< IP checksum offload enabled */ + .hw_vlan_filter = 0, /**< VLAN filtering disabled */ + .jumbo_frame = 0, /**< Jumbo Frame Support disabled */ + .hw_strip_crc = 1, /**< CRC stripped by hardware */ + }, + .txmode = { + .mq_mode = ETH_MQ_TX_NONE, + }, + .lpbk_mode = 0, +}; + +static struct rte_eth_conf rss_pmd_conf = { + .rxmode = { + .mq_mode = ETH_MQ_RX_RSS, + .max_rx_pkt_len = ETHER_MAX_LEN, + .split_hdr_size = 0, + .header_split = 0, /**< Header Split disabled */ + .hw_ip_checksum = 0, /**< IP checksum offload enabled */ + .hw_vlan_filter = 0, /**< VLAN filtering disabled */ + .jumbo_frame = 0, /**< Jumbo Frame Support disabled */ + .hw_strip_crc = 1, /**< CRC stripped by hardware */ + }, + .txmode = { + .mq_mode = ETH_MQ_TX_NONE, + }, + .rx_adv_conf = { + .rss_conf = { + .rss_key = NULL, + .rss_hf = ETH_RSS_IPV6, + }, + }, + .lpbk_mode = 0, +}; + +#define FOR_EACH(_i, _item, _array, _size) \ + for (_i = 0, _item = &_array[0]; _i < _size && (_item = &_array[_i]); _i++) + +/* Macro for iterating over every port that can be used as a slave + * in this test. + * _i variable used as an index in test_params->slave_ports + * _slave pointer to &test_params->slave_ports[_idx] + */ +#define FOR_EACH_PORT(_i, _port) \ + FOR_EACH(_i, _port, test_params.slave_ports, \ + RTE_DIM(test_params.slave_ports)) + +static int +configure_ethdev(uint16_t port_id, struct rte_eth_conf *eth_conf, + uint8_t start) +{ + int rxq, txq; + + TEST_ASSERT(rte_eth_dev_configure(port_id, RXTX_QUEUE_COUNT, + RXTX_QUEUE_COUNT, eth_conf) == 0, "Failed to configure device %u", + port_id); + + for (rxq = 0; rxq < RXTX_QUEUE_COUNT; rxq++) { + TEST_ASSERT(rte_eth_rx_queue_setup(port_id, rxq, RXTX_RING_SIZE, + rte_eth_dev_socket_id(port_id), NULL, + test_params.mbuf_pool) == 0, "Failed to setup rx queue."); + } + + for (txq = 0; txq < RXTX_QUEUE_COUNT; txq++) { + TEST_ASSERT(rte_eth_tx_queue_setup(port_id, txq, RXTX_RING_SIZE, + rte_eth_dev_socket_id(port_id), NULL) == 0, + "Failed to setup tx queue."); + } + + if (start) { + TEST_ASSERT(rte_eth_dev_start(port_id) == 0, + "Failed to start device (%d).", port_id); + } + + return 0; +} + +/** + * Remove all slaves from bonding + */ +static int +remove_slaves(void) +{ + unsigned n; + struct slave_conf *port; + + FOR_EACH_PORT(n, port) { + port = &test_params.slave_ports[n]; + if (port->is_slave) { + TEST_ASSERT_SUCCESS(rte_eth_bond_slave_remove( + test_params.bond_port_id, port->port_id), + "Cannot remove slave %d from bonding", port->port_id); + port->is_slave = 0; + } + } + + return 0; +} + +static int +remove_slaves_and_stop_bonded_device(void) +{ + TEST_ASSERT_SUCCESS(remove_slaves(), "Removing slaves"); + rte_eth_dev_stop(test_params.bond_port_id); + return TEST_SUCCESS; +} + +/** + * Add all slaves to bonding + */ +static int +bond_slaves(void) +{ + unsigned n; + struct slave_conf *port; + + FOR_EACH_PORT(n, port) { + port = &test_params.slave_ports[n]; + if (!port->is_slave) { + TEST_ASSERT_SUCCESS(rte_eth_bond_slave_add(test_params.bond_port_id, + port->port_id), "Cannot attach slave %d to the bonding", + port->port_id); + port->is_slave = 1; + } + } + + return 0; +} + +/** + * Set all RETA values in port_id to value + */ +static int +reta_set(uint16_t port_id, uint8_t value, int reta_size) +{ + struct rte_eth_rss_reta_entry64 reta_conf[512/RTE_RETA_GROUP_SIZE]; + int i, j; + + for (i = 0; i < reta_size / RTE_RETA_GROUP_SIZE; i++) { + /* select all fields to set */ + reta_conf[i].mask = ~0LL; + for (j = 0; j < RTE_RETA_GROUP_SIZE; j++) + reta_conf[i].reta[j] = value; + } + + return rte_eth_dev_rss_reta_update(port_id, reta_conf, reta_size); +} + +/** + * Check if slaves RETA is synchronized with bonding port. Returns 1 if slave + * port is synced with bonding port. + */ +static int +reta_check_synced(struct slave_conf *port) +{ + unsigned i; + + for (i = 0; i < test_params.bond_dev_info.reta_size; + i++) { + + int index = i / RTE_RETA_GROUP_SIZE; + int shift = i % RTE_RETA_GROUP_SIZE; + + if (port->reta_conf[index].reta[shift] != + test_params.bond_reta_conf[index].reta[shift]) + return 0; + + } + + return 1; +} + +/** + * Fetch bonding ports RETA + */ +static int +bond_reta_fetch(void) { + unsigned j; + + for (j = 0; j < test_params.bond_dev_info.reta_size / RTE_RETA_GROUP_SIZE; + j++) + test_params.bond_reta_conf[j].mask = ~0LL; + + TEST_ASSERT_SUCCESS(rte_eth_dev_rss_reta_query(test_params.bond_port_id, + test_params.bond_reta_conf, test_params.bond_dev_info.reta_size), + "Cannot take bonding ports RSS configuration"); + return 0; +} + +/** + * Fetch slaves RETA + */ +static int +slave_reta_fetch(struct slave_conf *port) { + unsigned j; + + for (j = 0; j < port->dev_info.reta_size / RTE_RETA_GROUP_SIZE; j++) + port->reta_conf[j].mask = ~0LL; + + TEST_ASSERT_SUCCESS(rte_eth_dev_rss_reta_query(port->port_id, + port->reta_conf, port->dev_info.reta_size), + "Cannot take bonding ports RSS configuration"); + return 0; +} + +/** + * Remove and add slave to check if slaves configuration is synced with + * the bonding ports values after adding new slave. + */ +static int +slave_remove_and_add(void) +{ + struct slave_conf *port = &(test_params.slave_ports[0]); + + /* 1. Remove first slave from bonding */ + TEST_ASSERT_SUCCESS(rte_eth_bond_slave_remove(test_params.bond_port_id, + port->port_id), "Cannot remove slave #d from bonding"); + + /* 2. Change removed (ex-)slave and bonding configuration to different + * values + */ + reta_set(test_params.bond_port_id, 1, test_params.bond_dev_info.reta_size); + bond_reta_fetch(); + + reta_set(port->port_id, 2, port->dev_info.reta_size); + slave_reta_fetch(port); + + TEST_ASSERT(reta_check_synced(port) == 0, + "Removed slave didn't should be synchronized with bonding port"); + + /* 3. Add (ex-)slave and check if configuration changed*/ + TEST_ASSERT_SUCCESS(rte_eth_bond_slave_add(test_params.bond_port_id, + port->port_id), "Cannot add slave"); + + bond_reta_fetch(); + slave_reta_fetch(port); + + return reta_check_synced(port); +} + +/** + * Test configuration propagation over slaves. + */ +static int +test_propagate(void) +{ + unsigned i; + uint8_t n; + struct slave_conf *port; + uint8_t bond_rss_key[40]; + struct rte_eth_rss_conf bond_rss_conf; + + int retval = 0; + uint64_t rss_hf = 0; + uint64_t default_rss_hf = 0; + + rte_eth_dev_info_get(test_params.bond_port_id, &test_params.bond_dev_info); + + /* + * Test hash function propagation + */ + for (i = 0; i < sizeof(test_params.bond_dev_info.flow_type_rss_offloads)*8; + i++) { + + rss_hf = test_params.bond_dev_info.flow_type_rss_offloads & (1<<i); + if (rss_hf) { + bond_rss_conf.rss_key = NULL; + bond_rss_conf.rss_hf = rss_hf; + + retval = rte_eth_dev_rss_hash_update(test_params.bond_port_id, + &bond_rss_conf); + TEST_ASSERT_SUCCESS(retval, "Cannot set slaves hash function"); + + FOR_EACH_PORT(n, port) { + port = &test_params.slave_ports[n]; + + retval = rte_eth_dev_rss_hash_conf_get(port->port_id, + &port->rss_conf); + TEST_ASSERT_SUCCESS(retval, + "Cannot take slaves RSS configuration"); + + TEST_ASSERT(port->rss_conf.rss_hf == rss_hf, + "Hash function not propagated for slave %d", + port->port_id); + } + + default_rss_hf = rss_hf; + } + + } + + /* + * Test key propagation + */ + for (i = 1; i < 10; i++) { + + /* Set all keys to zero */ + FOR_EACH_PORT(n, port) { + port = &test_params.slave_ports[n]; + memset(port->rss_conf.rss_key, 0, 40); + retval = rte_eth_dev_rss_hash_update(port->port_id, + &port->rss_conf); + TEST_ASSERT_SUCCESS(retval, "Cannot set slaves RSS keys"); + } + + memset(bond_rss_key, i, sizeof(bond_rss_key)); + bond_rss_conf.rss_hf = default_rss_hf, + bond_rss_conf.rss_key = bond_rss_key; + bond_rss_conf.rss_key_len = 40; + + retval = rte_eth_dev_rss_hash_update(test_params.bond_port_id, + &bond_rss_conf); + TEST_ASSERT_SUCCESS(retval, "Cannot set bonded port RSS keys"); + + FOR_EACH_PORT(n, port) { + port = &test_params.slave_ports[n]; + + retval = rte_eth_dev_rss_hash_conf_get(port->port_id, + &(port->rss_conf)); + + TEST_ASSERT_SUCCESS(retval, + "Cannot take slaves RSS configuration"); + + /* compare keys */ + retval = memcmp(port->rss_conf.rss_key, bond_rss_key, + sizeof(bond_rss_key)); + TEST_ASSERT(retval == 0, "Key value not propagated for slave %d", + port->port_id); + } + } + + /* + * Test RETA propagation + */ + for (i = 0; i < RXTX_QUEUE_COUNT; i++) { + + /* Set all keys to zero */ + FOR_EACH_PORT(n, port) { + port = &test_params.slave_ports[n]; + retval = reta_set(port->port_id, (i + 1) % RXTX_QUEUE_COUNT, + port->dev_info.reta_size); + TEST_ASSERT_SUCCESS(retval, "Cannot set slaves RETA"); + } + + TEST_ASSERT_SUCCESS(reta_set(test_params.bond_port_id, + i % RXTX_QUEUE_COUNT, test_params.bond_dev_info.reta_size), + "Cannot set bonded port RETA"); + + bond_reta_fetch(); + + FOR_EACH_PORT(n, port) { + port = &test_params.slave_ports[n]; + + slave_reta_fetch(port); + TEST_ASSERT(reta_check_synced(port) == 1, "RETAs inconsistent"); + } + } + + return TEST_SUCCESS; +} + +/** + * Test propagation logic, when RX_RSS mq_mode is turned on for bonding port + */ +static int +test_rss(void) +{ + /** + * Configure bonding port in RSS mq mode + */ + TEST_ASSERT_SUCCESS(configure_ethdev(test_params.bond_port_id, + &rss_pmd_conf, 0), "Failed to configure bonding device\n"); + + rte_eth_dev_info_get(test_params.bond_port_id, &test_params.bond_dev_info); + + TEST_ASSERT_SUCCESS(bond_slaves(), "Bonding slaves failed"); + + TEST_ASSERT_SUCCESS(rte_eth_dev_start(test_params.bond_port_id), + "Failed to start bonding port (%d).", test_params.bond_port_id); + + TEST_ASSERT_SUCCESS(test_propagate(), "Propagation test failed"); + + TEST_ASSERT(slave_remove_and_add() == 1, "New slave should be synced"); + + remove_slaves_and_stop_bonded_device(); + + return TEST_SUCCESS; +} + +/** + * Test propagation logic, when RX_RSS mq_mode is turned off for bonding port + */ +static int +test_rss_lazy(void) +{ + TEST_ASSERT_SUCCESS(configure_ethdev(test_params.bond_port_id, + &default_pmd_conf, 0), "Failed to configure bonding device\n"); + + rte_eth_dev_info_get(test_params.bond_port_id, &test_params.bond_dev_info); + + TEST_ASSERT_SUCCESS(bond_slaves(), "Bonding slaves failed"); + + TEST_ASSERT_SUCCESS(rte_eth_dev_start(test_params.bond_port_id), + "Failed to start bonding port (%d).", test_params.bond_port_id); + + TEST_ASSERT_SUCCESS(test_propagate(), "Propagation test failed"); + + TEST_ASSERT(slave_remove_and_add() == 0, "New slave shouldn't be synced"); + + remove_slaves_and_stop_bonded_device(); + + return TEST_SUCCESS; +} + +static int +test_setup(void) +{ + unsigned n; + int retval; + int port_id; + char name[256]; + struct slave_conf *port; + + if (test_params.mbuf_pool == NULL) { + + test_params.mbuf_pool = rte_pktmbuf_pool_create( + "RSS_MBUF_POOL", NUM_MBUFS * SLAVE_COUNT, + MBUF_CACHE_SIZE, 0, MBUF_SIZE, rte_socket_id()); + + TEST_ASSERT(test_params.mbuf_pool != NULL, + "rte_pktmbuf_pool_create failed\n"); + } + + /* Create / initialize ring eth devs. */ + FOR_EACH_PORT(n, port) { + port = &test_params.slave_ports[n]; + + port_id = rte_eth_dev_count(); + snprintf(name, sizeof(name), SLAVE_DEV_NAME_FMT, port_id); + + retval = rte_vdev_init(name, "size=64,copy=0"); + TEST_ASSERT_SUCCESS(retval, "Failed to create null device '%s'\n", + name); + + port->port_id = port_id; + + port->rss_conf.rss_key = port->rss_key; + port->rss_conf.rss_key_len = 40; + + retval = configure_ethdev(port->port_id, &default_pmd_conf, 0); + TEST_ASSERT_SUCCESS(retval, "Failed to configure virtual ethdev %s\n", + name); + + rte_eth_dev_info_get(port->port_id, &port->dev_info); + } + + if (test_params.bond_port_id == INVALID_PORT_ID) { + retval = rte_eth_bond_create(BONDED_DEV_NAME, 0, rte_socket_id()); + + TEST_ASSERT(retval >= 0, "Failed to create bonded ethdev %s", + BONDED_DEV_NAME); + + test_params.bond_port_id = retval; + + TEST_ASSERT_SUCCESS(configure_ethdev(test_params.bond_port_id, + &default_pmd_conf, 0), "Failed to configure bonding device\n"); + + rte_eth_dev_info_get(test_params.bond_port_id, + &test_params.bond_dev_info); + } + + return TEST_SUCCESS; +} + +static void +testsuite_teardown(void) +{ + struct slave_conf *port; + uint8_t i; + + /* Only stop ports. + * Any cleanup/reset state is done when particular test is + * started. */ + + rte_eth_dev_stop(test_params.bond_port_id); + + FOR_EACH_PORT(i, port) + rte_eth_dev_stop(port->port_id); +} + +static int +check_environment(void) +{ + return TEST_SUCCESS; +} + +static int +test_rssconf_executor(int (*test_func)(void)) +{ + int test_result; + + /* Check if environment is clean. Fail to launch a test if there was + * a critical error before that prevented to reset environment. */ + TEST_ASSERT_SUCCESS(check_environment(), + "Refusing to launch test in dirty environment."); + + RTE_VERIFY(test_func != NULL); + test_result = (*test_func)(); + + /* If test succeed check if environment wast left in good condition. */ + if (test_result == TEST_SUCCESS) + test_result = check_environment(); + + /* Reset environment in case test failed to do that. */ + if (test_result != TEST_SUCCESS) { + TEST_ASSERT_SUCCESS(remove_slaves_and_stop_bonded_device(), + "Failed to stop bonded device"); + } + + return test_result; +} + +static int +test_setup_wrapper(void) +{ + return test_rssconf_executor(&test_setup); +} + +static int +test_rss_wrapper(void) +{ + return test_rssconf_executor(&test_rss); +} + +static int +test_rss_lazy_wrapper(void) +{ + return test_rssconf_executor(&test_rss_lazy); +} + +static struct unit_test_suite link_bonding_rssconf_test_suite = { + .suite_name = "RSS Dynamic Configuration for Bonding Unit Test Suite", + .teardown = testsuite_teardown, + .unit_test_cases = { + TEST_CASE_NAMED("test_setup", test_setup_wrapper), + TEST_CASE_NAMED("test_rss", test_rss_wrapper), + TEST_CASE_NAMED("test_rss_lazy", test_rss_lazy_wrapper), + + TEST_CASES_END() + } +}; + +static int +test_link_bonding_rssconf(void) +{ + return unit_test_suite_runner(&link_bonding_rssconf_test_suite); +} + +REGISTER_TEST_COMMAND(link_bonding_rssconf_autotest, test_link_bonding_rssconf); diff --git a/dpdk/test/test/test_logs.c b/dpdk/test/test/test_logs.c new file mode 100644 index 00000000..d6e4973a --- /dev/null +++ b/dpdk/test/test/test_logs.c @@ -0,0 +1,88 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <stdint.h> +#include <stdarg.h> +#include <sys/queue.h> + +#include <rte_log.h> +#include <rte_memory.h> +#include <rte_launch.h> +#include <rte_eal.h> +#include <rte_per_lcore.h> +#include <rte_lcore.h> + +#include "test.h" + +#define RTE_LOGTYPE_TESTAPP1 RTE_LOGTYPE_USER1 +#define RTE_LOGTYPE_TESTAPP2 RTE_LOGTYPE_USER2 + +/* + * Logs + * ==== + * + * - Enable log types. + * - Set log level. + * - Send logs with different types and levels, some should not be displayed. + */ + +static int +test_logs(void) +{ + /* set logtype level low to so we can test global level */ + rte_log_set_level(RTE_LOGTYPE_TESTAPP1, RTE_LOG_DEBUG); + rte_log_set_level(RTE_LOGTYPE_TESTAPP2, RTE_LOG_DEBUG); + + /* log in error level */ + rte_log_set_global_level(RTE_LOG_ERR); + RTE_LOG(ERR, TESTAPP1, "error message\n"); + RTE_LOG(CRIT, TESTAPP1, "critical message\n"); + + /* log in critical level */ + rte_log_set_global_level(RTE_LOG_CRIT); + RTE_LOG(ERR, TESTAPP2, "error message (not displayed)\n"); + RTE_LOG(CRIT, TESTAPP2, "critical message\n"); + + /* bump up single log type level above global to test it */ + rte_log_set_level(RTE_LOGTYPE_TESTAPP2, RTE_LOG_EMERG); + + /* log in error level */ + rte_log_set_global_level(RTE_LOG_ERR); + RTE_LOG(ERR, TESTAPP1, "error message\n"); + RTE_LOG(ERR, TESTAPP2, "error message (not displayed)\n"); + + return 0; +} + +REGISTER_TEST_COMMAND(logs_autotest, test_logs); diff --git a/dpdk/test/test/test_lpm.c b/dpdk/test/test/test_lpm.c new file mode 100644 index 00000000..41ae80fe --- /dev/null +++ b/dpdk/test/test/test_lpm.c @@ -0,0 +1,1319 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <stdint.h> +#include <stdlib.h> + +#include <rte_ip.h> +#include <rte_lpm.h> + +#include "test.h" +#include "test_xmmt_ops.h" + +#define TEST_LPM_ASSERT(cond) do { \ + if (!(cond)) { \ + printf("Error at line %d: \n", __LINE__); \ + return -1; \ + } \ +} while(0) + +typedef int32_t (*rte_lpm_test)(void); + +static int32_t test0(void); +static int32_t test1(void); +static int32_t test2(void); +static int32_t test3(void); +static int32_t test4(void); +static int32_t test5(void); +static int32_t test6(void); +static int32_t test7(void); +static int32_t test8(void); +static int32_t test9(void); +static int32_t test10(void); +static int32_t test11(void); +static int32_t test12(void); +static int32_t test13(void); +static int32_t test14(void); +static int32_t test15(void); +static int32_t test16(void); +static int32_t test17(void); +static int32_t test18(void); + +rte_lpm_test tests[] = { +/* Test Cases */ + test0, + test1, + test2, + test3, + test4, + test5, + test6, + test7, + test8, + test9, + test10, + test11, + test12, + test13, + test14, + test15, + test16, + test17, + test18 +}; + +#define NUM_LPM_TESTS (sizeof(tests)/sizeof(tests[0])) +#define MAX_DEPTH 32 +#define MAX_RULES 256 +#define NUMBER_TBL8S 256 +#define PASS 0 + +/* + * Check that rte_lpm_create fails gracefully for incorrect user input + * arguments + */ +int32_t +test0(void) +{ + struct rte_lpm *lpm = NULL; + struct rte_lpm_config config; + + config.max_rules = MAX_RULES; + config.number_tbl8s = NUMBER_TBL8S; + config.flags = 0; + + /* rte_lpm_create: lpm name == NULL */ + lpm = rte_lpm_create(NULL, SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm == NULL); + + /* rte_lpm_create: max_rules = 0 */ + /* Note: __func__ inserts the function name, in this case "test0". */ + config.max_rules = 0; + lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm == NULL); + + /* socket_id < -1 is invalid */ + config.max_rules = MAX_RULES; + lpm = rte_lpm_create(__func__, -2, &config); + TEST_LPM_ASSERT(lpm == NULL); + + return PASS; +} + +/* + * Create lpm table then delete lpm table 100 times + * Use a slightly different rules size each time + * */ +int32_t +test1(void) +{ + struct rte_lpm *lpm = NULL; + struct rte_lpm_config config; + + config.number_tbl8s = NUMBER_TBL8S; + config.flags = 0; + int32_t i; + + /* rte_lpm_free: Free NULL */ + for (i = 0; i < 100; i++) { + config.max_rules = MAX_RULES - i; + lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm != NULL); + + rte_lpm_free(lpm); + } + + /* Can not test free so return success */ + return PASS; +} + +/* + * Call rte_lpm_free for NULL pointer user input. Note: free has no return and + * therefore it is impossible to check for failure but this test is added to + * increase function coverage metrics and to validate that freeing null does + * not crash. + */ +int32_t +test2(void) +{ + struct rte_lpm *lpm = NULL; + struct rte_lpm_config config; + + config.max_rules = MAX_RULES; + config.number_tbl8s = NUMBER_TBL8S; + config.flags = 0; + + lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm != NULL); + + rte_lpm_free(lpm); + rte_lpm_free(NULL); + return PASS; +} + +/* + * Check that rte_lpm_add fails gracefully for incorrect user input arguments + */ +int32_t +test3(void) +{ + struct rte_lpm *lpm = NULL; + struct rte_lpm_config config; + + config.max_rules = MAX_RULES; + config.number_tbl8s = NUMBER_TBL8S; + config.flags = 0; + uint32_t ip = IPv4(0, 0, 0, 0), next_hop = 100; + uint8_t depth = 24; + int32_t status = 0; + + /* rte_lpm_add: lpm == NULL */ + status = rte_lpm_add(NULL, ip, depth, next_hop); + TEST_LPM_ASSERT(status < 0); + + /*Create vaild lpm to use in rest of test. */ + lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm != NULL); + + /* rte_lpm_add: depth < 1 */ + status = rte_lpm_add(lpm, ip, 0, next_hop); + TEST_LPM_ASSERT(status < 0); + + /* rte_lpm_add: depth > MAX_DEPTH */ + status = rte_lpm_add(lpm, ip, (MAX_DEPTH + 1), next_hop); + TEST_LPM_ASSERT(status < 0); + + rte_lpm_free(lpm); + + return PASS; +} + +/* + * Check that rte_lpm_delete fails gracefully for incorrect user input + * arguments + */ +int32_t +test4(void) +{ + struct rte_lpm *lpm = NULL; + struct rte_lpm_config config; + + config.max_rules = MAX_RULES; + config.number_tbl8s = NUMBER_TBL8S; + config.flags = 0; + uint32_t ip = IPv4(0, 0, 0, 0); + uint8_t depth = 24; + int32_t status = 0; + + /* rte_lpm_delete: lpm == NULL */ + status = rte_lpm_delete(NULL, ip, depth); + TEST_LPM_ASSERT(status < 0); + + /*Create vaild lpm to use in rest of test. */ + lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm != NULL); + + /* rte_lpm_delete: depth < 1 */ + status = rte_lpm_delete(lpm, ip, 0); + TEST_LPM_ASSERT(status < 0); + + /* rte_lpm_delete: depth > MAX_DEPTH */ + status = rte_lpm_delete(lpm, ip, (MAX_DEPTH + 1)); + TEST_LPM_ASSERT(status < 0); + + rte_lpm_free(lpm); + + return PASS; +} + +/* + * Check that rte_lpm_lookup fails gracefully for incorrect user input + * arguments + */ +int32_t +test5(void) +{ +#if defined(RTE_LIBRTE_LPM_DEBUG) + struct rte_lpm *lpm = NULL; + struct rte_lpm_config config; + + config.max_rules = MAX_RULES; + config.number_tbl8s = NUMBER_TBL8S; + config.flags = 0; + uint32_t ip = IPv4(0, 0, 0, 0), next_hop_return = 0; + int32_t status = 0; + + /* rte_lpm_lookup: lpm == NULL */ + status = rte_lpm_lookup(NULL, ip, &next_hop_return); + TEST_LPM_ASSERT(status < 0); + + /*Create vaild lpm to use in rest of test. */ + lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm != NULL); + + /* rte_lpm_lookup: depth < 1 */ + status = rte_lpm_lookup(lpm, ip, NULL); + TEST_LPM_ASSERT(status < 0); + + rte_lpm_free(lpm); +#endif + return PASS; +} + + + +/* + * Call add, lookup and delete for a single rule with depth <= 24 + */ +int32_t +test6(void) +{ + struct rte_lpm *lpm = NULL; + struct rte_lpm_config config; + + config.max_rules = MAX_RULES; + config.number_tbl8s = NUMBER_TBL8S; + config.flags = 0; + uint32_t ip = IPv4(0, 0, 0, 0), next_hop_add = 100, next_hop_return = 0; + uint8_t depth = 24; + int32_t status = 0; + + lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm != NULL); + + status = rte_lpm_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add)); + + status = rte_lpm_delete(lpm, ip, depth); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT(status == -ENOENT); + + rte_lpm_free(lpm); + + return PASS; +} + +/* + * Call add, lookup and delete for a single rule with depth > 24 + */ + +int32_t +test7(void) +{ + xmm_t ipx4; + uint32_t hop[4]; + struct rte_lpm *lpm = NULL; + struct rte_lpm_config config; + + config.max_rules = MAX_RULES; + config.number_tbl8s = NUMBER_TBL8S; + config.flags = 0; + uint32_t ip = IPv4(0, 0, 0, 0), next_hop_add = 100, next_hop_return = 0; + uint8_t depth = 32; + int32_t status = 0; + + lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm != NULL); + + status = rte_lpm_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add)); + + ipx4 = vect_set_epi32(ip, ip + 0x100, ip - 0x100, ip); + rte_lpm_lookupx4(lpm, ipx4, hop, UINT32_MAX); + TEST_LPM_ASSERT(hop[0] == next_hop_add); + TEST_LPM_ASSERT(hop[1] == UINT32_MAX); + TEST_LPM_ASSERT(hop[2] == UINT32_MAX); + TEST_LPM_ASSERT(hop[3] == next_hop_add); + + status = rte_lpm_delete(lpm, ip, depth); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT(status == -ENOENT); + + rte_lpm_free(lpm); + + return PASS; +} + +/* + * Use rte_lpm_add to add rules which effect only the second half of the lpm + * table. Use all possible depths ranging from 1..32. Set the next hop = to the + * depth. Check lookup hit for on every add and check for lookup miss on the + * first half of the lpm table after each add. Finally delete all rules going + * backwards (i.e. from depth = 32 ..1) and carry out a lookup after each + * delete. The lookup should return the next_hop_add value related to the + * previous depth value (i.e. depth -1). + */ +int32_t +test8(void) +{ + xmm_t ipx4; + uint32_t hop[4]; + struct rte_lpm *lpm = NULL; + struct rte_lpm_config config; + + config.max_rules = MAX_RULES; + config.number_tbl8s = NUMBER_TBL8S; + config.flags = 0; + uint32_t ip1 = IPv4(127, 255, 255, 255), ip2 = IPv4(128, 0, 0, 0); + uint32_t next_hop_add, next_hop_return; + uint8_t depth; + int32_t status = 0; + + lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm != NULL); + + /* Loop with rte_lpm_add. */ + for (depth = 1; depth <= 32; depth++) { + /* Let the next_hop_add value = depth. Just for change. */ + next_hop_add = depth; + + status = rte_lpm_add(lpm, ip2, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + /* Check IP in first half of tbl24 which should be empty. */ + status = rte_lpm_lookup(lpm, ip1, &next_hop_return); + TEST_LPM_ASSERT(status == -ENOENT); + + status = rte_lpm_lookup(lpm, ip2, &next_hop_return); + TEST_LPM_ASSERT((status == 0) && + (next_hop_return == next_hop_add)); + + ipx4 = vect_set_epi32(ip2, ip1, ip2, ip1); + rte_lpm_lookupx4(lpm, ipx4, hop, UINT32_MAX); + TEST_LPM_ASSERT(hop[0] == UINT32_MAX); + TEST_LPM_ASSERT(hop[1] == next_hop_add); + TEST_LPM_ASSERT(hop[2] == UINT32_MAX); + TEST_LPM_ASSERT(hop[3] == next_hop_add); + } + + /* Loop with rte_lpm_delete. */ + for (depth = 32; depth >= 1; depth--) { + next_hop_add = (uint8_t) (depth - 1); + + status = rte_lpm_delete(lpm, ip2, depth); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm_lookup(lpm, ip2, &next_hop_return); + + if (depth != 1) { + TEST_LPM_ASSERT((status == 0) && + (next_hop_return == next_hop_add)); + } else { + TEST_LPM_ASSERT(status == -ENOENT); + } + + status = rte_lpm_lookup(lpm, ip1, &next_hop_return); + TEST_LPM_ASSERT(status == -ENOENT); + + ipx4 = vect_set_epi32(ip1, ip1, ip2, ip2); + rte_lpm_lookupx4(lpm, ipx4, hop, UINT32_MAX); + if (depth != 1) { + TEST_LPM_ASSERT(hop[0] == next_hop_add); + TEST_LPM_ASSERT(hop[1] == next_hop_add); + } else { + TEST_LPM_ASSERT(hop[0] == UINT32_MAX); + TEST_LPM_ASSERT(hop[1] == UINT32_MAX); + } + TEST_LPM_ASSERT(hop[2] == UINT32_MAX); + TEST_LPM_ASSERT(hop[3] == UINT32_MAX); + } + + rte_lpm_free(lpm); + + return PASS; +} + +/* + * - Add & lookup to hit invalid TBL24 entry + * - Add & lookup to hit valid TBL24 entry not extended + * - Add & lookup to hit valid extended TBL24 entry with invalid TBL8 entry + * - Add & lookup to hit valid extended TBL24 entry with valid TBL8 entry + * + */ +int32_t +test9(void) +{ + struct rte_lpm *lpm = NULL; + struct rte_lpm_config config; + + config.max_rules = MAX_RULES; + config.number_tbl8s = NUMBER_TBL8S; + config.flags = 0; + uint32_t ip, ip_1, ip_2; + uint8_t depth, depth_1, depth_2; + uint32_t next_hop_add, next_hop_add_1, next_hop_add_2, next_hop_return; + int32_t status = 0; + + /* Add & lookup to hit invalid TBL24 entry */ + ip = IPv4(128, 0, 0, 0); + depth = 24; + next_hop_add = 100; + + lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm != NULL); + + status = rte_lpm_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add)); + + status = rte_lpm_delete(lpm, ip, depth); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT(status == -ENOENT); + + rte_lpm_delete_all(lpm); + + /* Add & lookup to hit valid TBL24 entry not extended */ + ip = IPv4(128, 0, 0, 0); + depth = 23; + next_hop_add = 100; + + status = rte_lpm_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add)); + + depth = 24; + next_hop_add = 101; + + status = rte_lpm_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add)); + + depth = 24; + + status = rte_lpm_delete(lpm, ip, depth); + TEST_LPM_ASSERT(status == 0); + + depth = 23; + + status = rte_lpm_delete(lpm, ip, depth); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT(status == -ENOENT); + + rte_lpm_delete_all(lpm); + + /* Add & lookup to hit valid extended TBL24 entry with invalid TBL8 + * entry */ + ip = IPv4(128, 0, 0, 0); + depth = 32; + next_hop_add = 100; + + status = rte_lpm_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add)); + + ip = IPv4(128, 0, 0, 5); + depth = 32; + next_hop_add = 101; + + status = rte_lpm_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add)); + + status = rte_lpm_delete(lpm, ip, depth); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT(status == -ENOENT); + + ip = IPv4(128, 0, 0, 0); + depth = 32; + next_hop_add = 100; + + status = rte_lpm_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add)); + + status = rte_lpm_delete(lpm, ip, depth); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT(status == -ENOENT); + + rte_lpm_delete_all(lpm); + + /* Add & lookup to hit valid extended TBL24 entry with valid TBL8 + * entry */ + ip_1 = IPv4(128, 0, 0, 0); + depth_1 = 25; + next_hop_add_1 = 101; + + ip_2 = IPv4(128, 0, 0, 5); + depth_2 = 32; + next_hop_add_2 = 102; + + next_hop_return = 0; + + status = rte_lpm_add(lpm, ip_1, depth_1, next_hop_add_1); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm_lookup(lpm, ip_1, &next_hop_return); + TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add_1)); + + status = rte_lpm_add(lpm, ip_2, depth_2, next_hop_add_2); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm_lookup(lpm, ip_2, &next_hop_return); + TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add_2)); + + status = rte_lpm_delete(lpm, ip_2, depth_2); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm_lookup(lpm, ip_2, &next_hop_return); + TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add_1)); + + status = rte_lpm_delete(lpm, ip_1, depth_1); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm_lookup(lpm, ip_1, &next_hop_return); + TEST_LPM_ASSERT(status == -ENOENT); + + rte_lpm_free(lpm); + + return PASS; +} + + +/* + * - Add rule that covers a TBL24 range previously invalid & lookup (& delete & + * lookup) + * - Add rule that extends a TBL24 invalid entry & lookup (& delete & lookup) + * - Add rule that extends a TBL24 valid entry & lookup for both rules (& + * delete & lookup) + * - Add rule that updates the next hop in TBL24 & lookup (& delete & lookup) + * - Add rule that updates the next hop in TBL8 & lookup (& delete & lookup) + * - Delete a rule that is not present in the TBL24 & lookup + * - Delete a rule that is not present in the TBL8 & lookup + * + */ +int32_t +test10(void) +{ + + struct rte_lpm *lpm = NULL; + struct rte_lpm_config config; + + config.max_rules = MAX_RULES; + config.number_tbl8s = NUMBER_TBL8S; + config.flags = 0; + uint32_t ip, next_hop_add, next_hop_return; + uint8_t depth; + int32_t status = 0; + + /* Add rule that covers a TBL24 range previously invalid & lookup + * (& delete & lookup) */ + lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm != NULL); + + ip = IPv4(128, 0, 0, 0); + depth = 16; + next_hop_add = 100; + + status = rte_lpm_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add)); + + status = rte_lpm_delete(lpm, ip, depth); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT(status == -ENOENT); + + rte_lpm_delete_all(lpm); + + ip = IPv4(128, 0, 0, 0); + depth = 25; + next_hop_add = 100; + + status = rte_lpm_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add)); + + status = rte_lpm_delete(lpm, ip, depth); + TEST_LPM_ASSERT(status == 0); + + rte_lpm_delete_all(lpm); + + /* Add rule that extends a TBL24 valid entry & lookup for both rules + * (& delete & lookup) */ + + ip = IPv4(128, 0, 0, 0); + depth = 24; + next_hop_add = 100; + + status = rte_lpm_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + ip = IPv4(128, 0, 0, 10); + depth = 32; + next_hop_add = 101; + + status = rte_lpm_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add)); + + ip = IPv4(128, 0, 0, 0); + next_hop_add = 100; + + status = rte_lpm_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add)); + + ip = IPv4(128, 0, 0, 0); + depth = 24; + + status = rte_lpm_delete(lpm, ip, depth); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT(status == -ENOENT); + + ip = IPv4(128, 0, 0, 10); + depth = 32; + + status = rte_lpm_delete(lpm, ip, depth); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT(status == -ENOENT); + + rte_lpm_delete_all(lpm); + + /* Add rule that updates the next hop in TBL24 & lookup + * (& delete & lookup) */ + + ip = IPv4(128, 0, 0, 0); + depth = 24; + next_hop_add = 100; + + status = rte_lpm_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add)); + + next_hop_add = 101; + + status = rte_lpm_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add)); + + status = rte_lpm_delete(lpm, ip, depth); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT(status == -ENOENT); + + rte_lpm_delete_all(lpm); + + /* Add rule that updates the next hop in TBL8 & lookup + * (& delete & lookup) */ + + ip = IPv4(128, 0, 0, 0); + depth = 32; + next_hop_add = 100; + + status = rte_lpm_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add)); + + next_hop_add = 101; + + status = rte_lpm_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add)); + + status = rte_lpm_delete(lpm, ip, depth); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT(status == -ENOENT); + + rte_lpm_delete_all(lpm); + + /* Delete a rule that is not present in the TBL24 & lookup */ + + ip = IPv4(128, 0, 0, 0); + depth = 24; + + status = rte_lpm_delete(lpm, ip, depth); + TEST_LPM_ASSERT(status < 0); + + status = rte_lpm_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT(status == -ENOENT); + + rte_lpm_delete_all(lpm); + + /* Delete a rule that is not present in the TBL8 & lookup */ + + ip = IPv4(128, 0, 0, 0); + depth = 32; + + status = rte_lpm_delete(lpm, ip, depth); + TEST_LPM_ASSERT(status < 0); + + status = rte_lpm_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT(status == -ENOENT); + + rte_lpm_free(lpm); + + return PASS; +} + +/* + * Add two rules, lookup to hit the more specific one, lookup to hit the less + * specific one delete the less specific rule and lookup previous values again; + * add a more specific rule than the existing rule, lookup again + * + * */ +int32_t +test11(void) +{ + + struct rte_lpm *lpm = NULL; + struct rte_lpm_config config; + + config.max_rules = MAX_RULES; + config.number_tbl8s = NUMBER_TBL8S; + config.flags = 0; + uint32_t ip, next_hop_add, next_hop_return; + uint8_t depth; + int32_t status = 0; + + lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm != NULL); + + ip = IPv4(128, 0, 0, 0); + depth = 24; + next_hop_add = 100; + + status = rte_lpm_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + ip = IPv4(128, 0, 0, 10); + depth = 32; + next_hop_add = 101; + + status = rte_lpm_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add)); + + ip = IPv4(128, 0, 0, 0); + next_hop_add = 100; + + status = rte_lpm_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add)); + + ip = IPv4(128, 0, 0, 0); + depth = 24; + + status = rte_lpm_delete(lpm, ip, depth); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT(status == -ENOENT); + + ip = IPv4(128, 0, 0, 10); + depth = 32; + + status = rte_lpm_delete(lpm, ip, depth); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT(status == -ENOENT); + + rte_lpm_free(lpm); + + return PASS; +} + +/* + * Add an extended rule (i.e. depth greater than 24, lookup (hit), delete, + * lookup (miss) in a for loop of 1000 times. This will check tbl8 extension + * and contraction. + * + * */ + +int32_t +test12(void) +{ + xmm_t ipx4; + uint32_t hop[4]; + struct rte_lpm *lpm = NULL; + struct rte_lpm_config config; + + config.max_rules = MAX_RULES; + config.number_tbl8s = NUMBER_TBL8S; + config.flags = 0; + uint32_t ip, i, next_hop_add, next_hop_return; + uint8_t depth; + int32_t status = 0; + + lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm != NULL); + + ip = IPv4(128, 0, 0, 0); + depth = 32; + next_hop_add = 100; + + for (i = 0; i < 1000; i++) { + status = rte_lpm_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT((status == 0) && + (next_hop_return == next_hop_add)); + + ipx4 = vect_set_epi32(ip, ip + 1, ip, ip - 1); + rte_lpm_lookupx4(lpm, ipx4, hop, UINT32_MAX); + TEST_LPM_ASSERT(hop[0] == UINT32_MAX); + TEST_LPM_ASSERT(hop[1] == next_hop_add); + TEST_LPM_ASSERT(hop[2] == UINT32_MAX); + TEST_LPM_ASSERT(hop[3] == next_hop_add); + + status = rte_lpm_delete(lpm, ip, depth); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT(status == -ENOENT); + } + + rte_lpm_free(lpm); + + return PASS; +} + +/* + * Add a rule to tbl24, lookup (hit), then add a rule that will extend this + * tbl24 entry, lookup (hit). delete the rule that caused the tbl24 extension, + * lookup (miss) and repeat for loop of 1000 times. This will check tbl8 + * extension and contraction. + * + * */ + +int32_t +test13(void) +{ + struct rte_lpm *lpm = NULL; + struct rte_lpm_config config; + + config.max_rules = MAX_RULES; + config.number_tbl8s = NUMBER_TBL8S; + config.flags = 0; + uint32_t ip, i, next_hop_add_1, next_hop_add_2, next_hop_return; + uint8_t depth; + int32_t status = 0; + + lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm != NULL); + + ip = IPv4(128, 0, 0, 0); + depth = 24; + next_hop_add_1 = 100; + + status = rte_lpm_add(lpm, ip, depth, next_hop_add_1); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add_1)); + + depth = 32; + next_hop_add_2 = 101; + + for (i = 0; i < 1000; i++) { + status = rte_lpm_add(lpm, ip, depth, next_hop_add_2); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT((status == 0) && + (next_hop_return == next_hop_add_2)); + + status = rte_lpm_delete(lpm, ip, depth); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT((status == 0) && + (next_hop_return == next_hop_add_1)); + } + + depth = 24; + + status = rte_lpm_delete(lpm, ip, depth); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT(status == -ENOENT); + + rte_lpm_free(lpm); + + return PASS; +} + +/* + * Fore TBL8 extension exhaustion. Add 256 rules that require a tbl8 extension. + * No more tbl8 extensions will be allowed. Now add one more rule that required + * a tbl8 extension and get fail. + * */ +int32_t +test14(void) +{ + + /* We only use depth = 32 in the loop below so we must make sure + * that we have enough storage for all rules at that depth*/ + + struct rte_lpm *lpm = NULL; + struct rte_lpm_config config; + + config.max_rules = 256 * 32; + config.number_tbl8s = NUMBER_TBL8S; + config.flags = 0; + uint32_t ip, next_hop_add, next_hop_return; + uint8_t depth; + int32_t status = 0; + + /* Add enough space for 256 rules for every depth */ + lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm != NULL); + + depth = 32; + next_hop_add = 100; + ip = IPv4(0, 0, 0, 0); + + /* Add 256 rules that require a tbl8 extension */ + for (; ip <= IPv4(0, 0, 255, 0); ip += 256) { + status = rte_lpm_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT((status == 0) && + (next_hop_return == next_hop_add)); + } + + /* All tbl8 extensions have been used above. Try to add one more and + * we get a fail */ + ip = IPv4(1, 0, 0, 0); + depth = 32; + + status = rte_lpm_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status < 0); + + rte_lpm_free(lpm); + + return PASS; +} + +/* + * Sequence of operations for find existing lpm table + * + * - create table + * - find existing table: hit + * - find non-existing table: miss + * + */ +int32_t +test15(void) +{ + struct rte_lpm *lpm = NULL, *result = NULL; + struct rte_lpm_config config; + + config.max_rules = 256 * 32; + config.number_tbl8s = NUMBER_TBL8S; + config.flags = 0; + + /* Create lpm */ + lpm = rte_lpm_create("lpm_find_existing", SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm != NULL); + + /* Try to find existing lpm */ + result = rte_lpm_find_existing("lpm_find_existing"); + TEST_LPM_ASSERT(result == lpm); + + /* Try to find non-existing lpm */ + result = rte_lpm_find_existing("lpm_find_non_existing"); + TEST_LPM_ASSERT(result == NULL); + + /* Cleanup. */ + rte_lpm_delete_all(lpm); + rte_lpm_free(lpm); + + return PASS; +} + +/* + * test failure condition of overloading the tbl8 so no more will fit + * Check we get an error return value in that case + */ +int32_t +test16(void) +{ + uint32_t ip; + struct rte_lpm_config config; + + config.max_rules = 256 * 32; + config.number_tbl8s = NUMBER_TBL8S; + config.flags = 0; + struct rte_lpm *lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config); + + /* ip loops through all possibilities for top 24 bits of address */ + for (ip = 0; ip < 0xFFFFFF; ip++) { + /* add an entry within a different tbl8 each time, since + * depth >24 and the top 24 bits are different */ + if (rte_lpm_add(lpm, (ip << 8) + 0xF0, 30, 0) < 0) + break; + } + + if (ip != NUMBER_TBL8S) { + printf("Error, unexpected failure with filling tbl8 groups\n"); + printf("Failed after %u additions, expected after %u\n", + (unsigned)ip, (unsigned)NUMBER_TBL8S); + } + + rte_lpm_free(lpm); + return 0; +} + +/* + * Test for overwriting of tbl8: + * - add rule /32 and lookup + * - add new rule /24 and lookup + * - add third rule /25 and lookup + * - lookup /32 and /24 rule to ensure the table has not been overwritten. + */ +int32_t +test17(void) +{ + struct rte_lpm *lpm = NULL; + struct rte_lpm_config config; + + config.max_rules = MAX_RULES; + config.number_tbl8s = NUMBER_TBL8S; + config.flags = 0; + const uint32_t ip_10_32 = IPv4(10, 10, 10, 2); + const uint32_t ip_10_24 = IPv4(10, 10, 10, 0); + const uint32_t ip_20_25 = IPv4(10, 10, 20, 2); + const uint8_t d_ip_10_32 = 32, + d_ip_10_24 = 24, + d_ip_20_25 = 25; + const uint32_t next_hop_ip_10_32 = 100, + next_hop_ip_10_24 = 105, + next_hop_ip_20_25 = 111; + uint32_t next_hop_return = 0; + int32_t status = 0; + + lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm != NULL); + + if ((status = rte_lpm_add(lpm, ip_10_32, d_ip_10_32, + next_hop_ip_10_32)) < 0) + return -1; + + status = rte_lpm_lookup(lpm, ip_10_32, &next_hop_return); + uint32_t test_hop_10_32 = next_hop_return; + TEST_LPM_ASSERT(status == 0); + TEST_LPM_ASSERT(next_hop_return == next_hop_ip_10_32); + + if ((status = rte_lpm_add(lpm, ip_10_24, d_ip_10_24, + next_hop_ip_10_24)) < 0) + return -1; + + status = rte_lpm_lookup(lpm, ip_10_24, &next_hop_return); + uint32_t test_hop_10_24 = next_hop_return; + TEST_LPM_ASSERT(status == 0); + TEST_LPM_ASSERT(next_hop_return == next_hop_ip_10_24); + + if ((status = rte_lpm_add(lpm, ip_20_25, d_ip_20_25, + next_hop_ip_20_25)) < 0) + return -1; + + status = rte_lpm_lookup(lpm, ip_20_25, &next_hop_return); + uint32_t test_hop_20_25 = next_hop_return; + TEST_LPM_ASSERT(status == 0); + TEST_LPM_ASSERT(next_hop_return == next_hop_ip_20_25); + + if (test_hop_10_32 == test_hop_10_24) { + printf("Next hop return equal\n"); + return -1; + } + + if (test_hop_10_24 == test_hop_20_25) { + printf("Next hop return equal\n"); + return -1; + } + + status = rte_lpm_lookup(lpm, ip_10_32, &next_hop_return); + TEST_LPM_ASSERT(status == 0); + TEST_LPM_ASSERT(next_hop_return == next_hop_ip_10_32); + + status = rte_lpm_lookup(lpm, ip_10_24, &next_hop_return); + TEST_LPM_ASSERT(status == 0); + TEST_LPM_ASSERT(next_hop_return == next_hop_ip_10_24); + + rte_lpm_free(lpm); + + return PASS; +} + +/* + * Test for recycle of tbl8 + * - step 1: add a rule with depth=28 (> 24) + * - step 2: add a rule with same 24-bit prefix and depth=23 (< 24) + * - step 3: delete the first rule + * - step 4: check tbl8 is freed + * - step 5: add a rule same as the first one (depth=28) + * - step 6: check same tbl8 is allocated + * - step 7: add a rule with same 24-bit prefix and depth=24 + * - step 8: delete the rule (depth=28) added in step 5 + * - step 9: check tbl8 is freed + * - step 10: add a rule with same 24-bit prefix and depth = 28 + * - setp 11: check same tbl8 is allocated again + */ +int32_t +test18(void) +{ +#define group_idx next_hop + struct rte_lpm *lpm = NULL; + struct rte_lpm_config config; + uint32_t ip, next_hop; + uint8_t depth; + uint32_t tbl8_group_index; + + config.max_rules = MAX_RULES; + config.number_tbl8s = NUMBER_TBL8S; + config.flags = 0; + + lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm != NULL); + + ip = IPv4(192, 168, 100, 100); + depth = 28; + next_hop = 1; + rte_lpm_add(lpm, ip, depth, next_hop); + + TEST_LPM_ASSERT(lpm->tbl24[ip>>8].valid_group); + tbl8_group_index = lpm->tbl24[ip>>8].group_idx; + + depth = 23; + next_hop = 2; + rte_lpm_add(lpm, ip, depth, next_hop); + TEST_LPM_ASSERT(lpm->tbl24[ip>>8].valid_group); + + depth = 28; + rte_lpm_delete(lpm, ip, depth); + + TEST_LPM_ASSERT(!lpm->tbl24[ip>>8].valid_group); + + next_hop = 3; + rte_lpm_add(lpm, ip, depth, next_hop); + + TEST_LPM_ASSERT(lpm->tbl24[ip>>8].valid_group); + TEST_LPM_ASSERT(tbl8_group_index == lpm->tbl24[ip>>8].group_idx); + + depth = 24; + next_hop = 4; + rte_lpm_add(lpm, ip, depth, next_hop); + TEST_LPM_ASSERT(lpm->tbl24[ip>>8].valid_group); + + depth = 28; + rte_lpm_delete(lpm, ip, depth); + + TEST_LPM_ASSERT(!lpm->tbl24[ip>>8].valid_group); + + next_hop = 5; + rte_lpm_add(lpm, ip, depth, next_hop); + + TEST_LPM_ASSERT(lpm->tbl24[ip>>8].valid_group); + TEST_LPM_ASSERT(tbl8_group_index == lpm->tbl24[ip>>8].group_idx); + + rte_lpm_free(lpm); +#undef group_idx + return PASS; +} + +/* + * Do all unit tests. + */ + +static int +test_lpm(void) +{ + unsigned i; + int status, global_status = 0; + + for (i = 0; i < NUM_LPM_TESTS; i++) { + status = tests[i](); + if (status < 0) { + printf("ERROR: LPM Test %u: FAIL\n", i); + global_status = status; + } + } + + return global_status; +} + +REGISTER_TEST_COMMAND(lpm_autotest, test_lpm); diff --git a/dpdk/test/test/test_lpm6.c b/dpdk/test/test/test_lpm6.c new file mode 100644 index 00000000..e0e7bf02 --- /dev/null +++ b/dpdk/test/test/test_lpm6.c @@ -0,0 +1,1825 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <stdint.h> +#include <stdlib.h> +#include <string.h> + +#include <rte_memory.h> +#include <rte_lpm6.h> + +#include "test.h" +#include "test_lpm6_data.h" + +#define TEST_LPM_ASSERT(cond) do { \ + if (!(cond)) { \ + printf("Error at line %d: \n", __LINE__); \ + return -1; \ + } \ +} while(0) + +typedef int32_t (* rte_lpm6_test)(void); + +static int32_t test0(void); +static int32_t test1(void); +static int32_t test2(void); +static int32_t test3(void); +static int32_t test4(void); +static int32_t test5(void); +static int32_t test6(void); +static int32_t test7(void); +static int32_t test8(void); +static int32_t test9(void); +static int32_t test10(void); +static int32_t test11(void); +static int32_t test12(void); +static int32_t test13(void); +static int32_t test14(void); +static int32_t test15(void); +static int32_t test16(void); +static int32_t test17(void); +static int32_t test18(void); +static int32_t test19(void); +static int32_t test20(void); +static int32_t test21(void); +static int32_t test22(void); +static int32_t test23(void); +static int32_t test24(void); +static int32_t test25(void); +static int32_t test26(void); +static int32_t test27(void); +static int32_t test28(void); + +rte_lpm6_test tests6[] = { +/* Test Cases */ + test0, + test1, + test2, + test3, + test4, + test5, + test6, + test7, + test8, + test9, + test10, + test11, + test12, + test13, + test14, + test15, + test16, + test17, + test18, + test19, + test20, + test21, + test22, + test23, + test24, + test25, + test26, + test27, + test28, +}; + +#define NUM_LPM6_TESTS (sizeof(tests6)/sizeof(tests6[0])) +#define MAX_DEPTH 128 +#define MAX_RULES 1000000 +#define NUMBER_TBL8S (1 << 16) +#define MAX_NUM_TBL8S (1 << 21) +#define PASS 0 + +static void +IPv6(uint8_t *ip, uint8_t b1, uint8_t b2, uint8_t b3, uint8_t b4, uint8_t b5, + uint8_t b6, uint8_t b7, uint8_t b8, uint8_t b9, uint8_t b10, + uint8_t b11, uint8_t b12, uint8_t b13, uint8_t b14, uint8_t b15, + uint8_t b16) +{ + ip[0] = b1; + ip[1] = b2; + ip[2] = b3; + ip[3] = b4; + ip[4] = b5; + ip[5] = b6; + ip[6] = b7; + ip[7] = b8; + ip[8] = b9; + ip[9] = b10; + ip[10] = b11; + ip[11] = b12; + ip[12] = b13; + ip[13] = b14; + ip[14] = b15; + ip[15] = b16; +} + +/* + * Check that rte_lpm6_create fails gracefully for incorrect user input + * arguments + */ +int32_t +test0(void) +{ + struct rte_lpm6 *lpm = NULL; + struct rte_lpm6_config config; + + config.max_rules = MAX_RULES; + config.number_tbl8s = NUMBER_TBL8S; + config.flags = 0; + + /* rte_lpm6_create: lpm name == NULL */ + lpm = rte_lpm6_create(NULL, SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm == NULL); + + /* rte_lpm6_create: max_rules = 0 */ + /* Note: __func__ inserts the function name, in this case "test0". */ + config.max_rules = 0; + lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm == NULL); + + /* socket_id < -1 is invalid */ + config.max_rules = MAX_RULES; + lpm = rte_lpm6_create(__func__, -2, &config); + TEST_LPM_ASSERT(lpm == NULL); + + /* rte_lpm6_create: number_tbl8s is bigger than the maximum */ + config.number_tbl8s = MAX_NUM_TBL8S + 1; + lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm == NULL); + + /* rte_lpm6_create: config = NULL */ + lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, NULL); + TEST_LPM_ASSERT(lpm == NULL); + + return PASS; +} + +/* + * Creates two different LPM tables. Tries to create a third one with the same + * name as the first one and expects the create function to return the same + * pointer. + */ +int32_t +test1(void) +{ + struct rte_lpm6 *lpm1 = NULL, *lpm2 = NULL, *lpm3 = NULL; + struct rte_lpm6_config config; + + config.max_rules = MAX_RULES; + config.number_tbl8s = NUMBER_TBL8S; + config.flags = 0; + + /* rte_lpm6_create: lpm name == LPM1 */ + lpm1 = rte_lpm6_create("LPM1", SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm1 != NULL); + + /* rte_lpm6_create: lpm name == LPM2 */ + lpm2 = rte_lpm6_create("LPM2", SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm2 != NULL); + + /* rte_lpm6_create: lpm name == LPM2 */ + lpm3 = rte_lpm6_create("LPM1", SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm3 == NULL); + + rte_lpm6_free(lpm1); + rte_lpm6_free(lpm2); + + return PASS; +} + +/* + * Create lpm table then delete lpm table 20 times + * Use a slightly different rules size each time + */ +int32_t +test2(void) +{ + struct rte_lpm6 *lpm = NULL; + struct rte_lpm6_config config; + int32_t i; + + config.number_tbl8s = NUMBER_TBL8S; + config.flags = 0; + + /* rte_lpm6_free: Free NULL */ + for (i = 0; i < 20; i++) { + config.max_rules = MAX_RULES - i; + lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm != NULL); + + rte_lpm6_free(lpm); + } + + /* Can not test free so return success */ + return PASS; +} + +/* + * Call rte_lpm6_free for NULL pointer user input. Note: free has no return and + * therefore it is impossible to check for failure but this test is added to + * increase function coverage metrics and to validate that freeing null does + * not crash. + */ +int32_t +test3(void) +{ + struct rte_lpm6 *lpm = NULL; + struct rte_lpm6_config config; + + config.max_rules = MAX_RULES; + config.number_tbl8s = NUMBER_TBL8S; + config.flags = 0; + + lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm != NULL); + + rte_lpm6_free(lpm); + rte_lpm6_free(NULL); + return PASS; +} + +/* + * Check that rte_lpm6_add fails gracefully for incorrect user input arguments + */ +int32_t +test4(void) +{ + struct rte_lpm6 *lpm = NULL; + struct rte_lpm6_config config; + + uint8_t ip[] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; + uint8_t depth = 24, next_hop = 100; + int32_t status = 0; + + config.max_rules = MAX_RULES; + config.number_tbl8s = NUMBER_TBL8S; + config.flags = 0; + + /* rte_lpm6_add: lpm == NULL */ + status = rte_lpm6_add(NULL, ip, depth, next_hop); + TEST_LPM_ASSERT(status < 0); + + /*Create vaild lpm to use in rest of test. */ + lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm != NULL); + + /* rte_lpm6_add: depth < 1 */ + status = rte_lpm6_add(lpm, ip, 0, next_hop); + TEST_LPM_ASSERT(status < 0); + + /* rte_lpm6_add: depth > MAX_DEPTH */ + status = rte_lpm6_add(lpm, ip, (MAX_DEPTH + 1), next_hop); + TEST_LPM_ASSERT(status < 0); + + rte_lpm6_free(lpm); + + return PASS; +} + +/* + * Check that rte_lpm6_delete fails gracefully for incorrect user input + * arguments + */ +int32_t +test5(void) +{ + struct rte_lpm6 *lpm = NULL; + struct rte_lpm6_config config; + uint8_t ip[] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; + uint8_t depth = 24; + int32_t status = 0; + + config.max_rules = MAX_RULES; + config.number_tbl8s = NUMBER_TBL8S; + config.flags = 0; + + /* rte_lpm_delete: lpm == NULL */ + status = rte_lpm6_delete(NULL, ip, depth); + TEST_LPM_ASSERT(status < 0); + + /*Create vaild lpm to use in rest of test. */ + lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm != NULL); + + /* rte_lpm_delete: depth < 1 */ + status = rte_lpm6_delete(lpm, ip, 0); + TEST_LPM_ASSERT(status < 0); + + /* rte_lpm_delete: depth > MAX_DEPTH */ + status = rte_lpm6_delete(lpm, ip, (MAX_DEPTH + 1)); + TEST_LPM_ASSERT(status < 0); + + rte_lpm6_free(lpm); + + return PASS; +} + +/* + * Check that rte_lpm6_lookup fails gracefully for incorrect user input + * arguments + */ +int32_t +test6(void) +{ + struct rte_lpm6 *lpm = NULL; + struct rte_lpm6_config config; + uint8_t ip[] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; + uint32_t next_hop_return = 0; + int32_t status = 0; + + config.max_rules = MAX_RULES; + config.number_tbl8s = NUMBER_TBL8S; + config.flags = 0; + + /* rte_lpm6_lookup: lpm == NULL */ + status = rte_lpm6_lookup(NULL, ip, &next_hop_return); + TEST_LPM_ASSERT(status < 0); + + /*Create vaild lpm to use in rest of test. */ + lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm != NULL); + + /* rte_lpm6_lookup: ip = NULL */ + status = rte_lpm6_lookup(lpm, NULL, &next_hop_return); + TEST_LPM_ASSERT(status < 0); + + /* rte_lpm6_lookup: next_hop = NULL */ + status = rte_lpm6_lookup(lpm, ip, NULL); + TEST_LPM_ASSERT(status < 0); + + rte_lpm6_free(lpm); + + return PASS; +} + +/* + * Checks that rte_lpm6_lookup_bulk_func fails gracefully for incorrect user + * input arguments + */ +int32_t +test7(void) +{ + struct rte_lpm6 *lpm = NULL; + struct rte_lpm6_config config; + uint8_t ip[10][16]; + int32_t next_hop_return[10]; + int32_t status = 0; + + config.max_rules = MAX_RULES; + config.number_tbl8s = NUMBER_TBL8S; + config.flags = 0; + + /* rte_lpm6_lookup: lpm == NULL */ + status = rte_lpm6_lookup_bulk_func(NULL, ip, next_hop_return, 10); + TEST_LPM_ASSERT(status < 0); + + /*Create vaild lpm to use in rest of test. */ + lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm != NULL); + + /* rte_lpm6_lookup: ip = NULL */ + status = rte_lpm6_lookup_bulk_func(lpm, NULL, next_hop_return, 10); + TEST_LPM_ASSERT(status < 0); + + /* rte_lpm6_lookup: next_hop = NULL */ + status = rte_lpm6_lookup_bulk_func(lpm, ip, NULL, 10); + TEST_LPM_ASSERT(status < 0); + + rte_lpm6_free(lpm); + + return PASS; +} + +/* + * Checks that rte_lpm6_delete_bulk_func fails gracefully for incorrect user + * input arguments + */ +int32_t +test8(void) +{ + struct rte_lpm6 *lpm = NULL; + struct rte_lpm6_config config; + uint8_t ip[10][16]; + uint8_t depth[10]; + int32_t status = 0; + + config.max_rules = MAX_RULES; + config.number_tbl8s = NUMBER_TBL8S; + config.flags = 0; + + /* rte_lpm6_delete: lpm == NULL */ + status = rte_lpm6_delete_bulk_func(NULL, ip, depth, 10); + TEST_LPM_ASSERT(status < 0); + + /*Create vaild lpm to use in rest of test. */ + lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm != NULL); + + /* rte_lpm6_delete: ip = NULL */ + status = rte_lpm6_delete_bulk_func(lpm, NULL, depth, 10); + TEST_LPM_ASSERT(status < 0); + + /* rte_lpm6_delete: next_hop = NULL */ + status = rte_lpm6_delete_bulk_func(lpm, ip, NULL, 10); + TEST_LPM_ASSERT(status < 0); + + rte_lpm6_free(lpm); + + return PASS; +} + +/* + * Call add, lookup and delete for a single rule with depth < 24. + * Check all the combinations for the first three bytes that result in a hit. + * Delete the rule and check that the same test returs a miss. + */ +int32_t +test9(void) +{ + struct rte_lpm6 *lpm = NULL; + struct rte_lpm6_config config; + uint8_t ip[] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; + uint8_t depth = 16; + uint32_t next_hop_add = 100, next_hop_return = 0; + int32_t status = 0; + uint8_t i; + + config.max_rules = MAX_RULES; + config.number_tbl8s = NUMBER_TBL8S; + config.flags = 0; + + lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm != NULL); + + status = rte_lpm6_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + for (i = 0; i < UINT8_MAX; i++) { + ip[2] = i; + status = rte_lpm6_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add)); + } + + status = rte_lpm6_delete(lpm, ip, depth); + TEST_LPM_ASSERT(status == 0); + + for (i = 0; i < UINT8_MAX; i++) { + ip[2] = i; + status = rte_lpm6_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT(status == -ENOENT); + } + + rte_lpm6_free(lpm); + + return PASS; +} + +/* + * Adds max_rules + 1 and expects a failure. Deletes a rule, then adds + * another one and expects success. + */ +int32_t +test10(void) +{ + struct rte_lpm6 *lpm = NULL; + struct rte_lpm6_config config; + uint8_t ip[] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; + uint8_t depth; + uint32_t next_hop_add = 100; + int32_t status = 0; + int i; + + config.max_rules = 127; + config.number_tbl8s = NUMBER_TBL8S; + config.flags = 0; + + lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm != NULL); + + for (i = 1; i < 128; i++) { + depth = (uint8_t)i; + status = rte_lpm6_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + } + + depth = 128; + status = rte_lpm6_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == -ENOSPC); + + depth = 127; + status = rte_lpm6_delete(lpm, ip, depth); + TEST_LPM_ASSERT(status == 0); + + depth = 128; + status = rte_lpm6_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + rte_lpm6_free(lpm); + + return PASS; +} + +/* + * Creates an LPM table with a small number of tbl8s and exhaust them in the + * middle of the process of creating a rule. + */ +int32_t +test11(void) +{ + struct rte_lpm6 *lpm = NULL; + struct rte_lpm6_config config; + uint8_t ip[] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; + uint8_t depth; + uint32_t next_hop_add = 100; + int32_t status = 0; + + config.max_rules = MAX_RULES; + config.number_tbl8s = 16; + config.flags = 0; + + lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm != NULL); + + depth = 128; + status = rte_lpm6_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + ip[0] = 1; + depth = 25; + status = rte_lpm6_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm6_delete(lpm, ip, depth); + TEST_LPM_ASSERT(status == 0); + + depth = 33; + status = rte_lpm6_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm6_delete(lpm, ip, depth); + TEST_LPM_ASSERT(status == 0); + + depth = 41; + status = rte_lpm6_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm6_delete(lpm, ip, depth); + TEST_LPM_ASSERT(status == 0); + + depth = 49; + status = rte_lpm6_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == -ENOSPC); + + depth = 41; + status = rte_lpm6_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + rte_lpm6_free(lpm); + + return PASS; +} + +/* + * Creates an LPM table with a small number of tbl8s and exhaust them in the + * middle of the process of adding a rule when there is already an existing rule + * in that position and needs to be extended. + */ +int32_t +test12(void) +{ + struct rte_lpm6 *lpm = NULL; + struct rte_lpm6_config config; + uint8_t ip[] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; + uint8_t depth; + uint32_t next_hop_add = 100; + int32_t status = 0; + + config.max_rules = MAX_RULES; + config.number_tbl8s = 16; + config.flags = 0; + + lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm != NULL); + + depth = 128; + status = rte_lpm6_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + ip[0] = 1; + depth = 41; + status = rte_lpm6_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + depth = 49; + status = rte_lpm6_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == -ENOSPC); + + rte_lpm6_free(lpm); + + return PASS; +} + +/* + * Creates an LPM table with max_rules = 2 and tries to add 3 rules. + * Delete one of the rules and tries to add the third one again. + */ +int32_t +test13(void) +{ + struct rte_lpm6 *lpm = NULL; + struct rte_lpm6_config config; + uint8_t ip[] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; + uint8_t depth; + uint32_t next_hop_add = 100; + int32_t status = 0; + + config.max_rules = 2; + config.number_tbl8s = NUMBER_TBL8S; + config.flags = 0; + + lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm != NULL); + + depth = 1; + status = rte_lpm6_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + depth = 2; + status = rte_lpm6_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + depth = 3; + status = rte_lpm6_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == -ENOSPC); + + depth = 2; + status = rte_lpm6_delete(lpm, ip, depth); + TEST_LPM_ASSERT(status == 0); + + depth = 3; + status = rte_lpm6_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + rte_lpm6_free(lpm); + + return PASS; +} + +/* + * Add 2^12 routes with different first 12 bits and depth 25. + * Add one more route with the same depth and check that results in a failure. + * After that delete the last rule and create the one that was attempted to be + * created. This checks tbl8 exhaustion. + */ +int32_t +test14(void) +{ + struct rte_lpm6 *lpm = NULL; + struct rte_lpm6_config config; + uint8_t ip[] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; + uint8_t depth = 25; + uint32_t next_hop_add = 100; + int32_t status = 0; + int i; + + config.max_rules = MAX_RULES; + config.number_tbl8s = 256; + config.flags = 0; + + lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm != NULL); + + for (i = 0; i < 256; i++) { + ip[0] = (uint8_t)i; + status = rte_lpm6_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + } + + ip[0] = 255; + ip[1] = 1; + status = rte_lpm6_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == -ENOSPC); + + ip[0] = 255; + ip[1] = 0; + status = rte_lpm6_delete(lpm, ip, depth); + TEST_LPM_ASSERT(status == 0); + + ip[0] = 255; + ip[1] = 1; + status = rte_lpm6_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + rte_lpm6_free(lpm); + + return PASS; +} + +/* + * Call add, lookup and delete for a single rule with depth = 24 + */ +int32_t +test15(void) +{ + struct rte_lpm6 *lpm = NULL; + struct rte_lpm6_config config; + uint8_t ip[] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; + uint8_t depth = 24; + uint32_t next_hop_add = 100, next_hop_return = 0; + int32_t status = 0; + + config.max_rules = MAX_RULES; + config.number_tbl8s = NUMBER_TBL8S; + config.flags = 0; + + lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm != NULL); + + status = rte_lpm6_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm6_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add)); + + status = rte_lpm6_delete(lpm, ip, depth); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm6_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT(status == -ENOENT); + + rte_lpm6_free(lpm); + + return PASS; +} + +/* + * Call add, lookup and delete for a single rule with depth > 24 + */ +int32_t +test16(void) +{ + struct rte_lpm6 *lpm = NULL; + struct rte_lpm6_config config; + uint8_t ip[] = {12,12,1,0,0,0,0,0,0,0,0,0,0,0,0,0}; + uint8_t depth = 128; + uint32_t next_hop_add = 100, next_hop_return = 0; + int32_t status = 0; + + config.max_rules = MAX_RULES; + config.number_tbl8s = NUMBER_TBL8S; + config.flags = 0; + + lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm != NULL); + + status = rte_lpm6_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm6_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add)); + + status = rte_lpm6_delete(lpm, ip, depth); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm6_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT(status == -ENOENT); + + rte_lpm6_free(lpm); + + return PASS; +} + +/* + * Use rte_lpm6_add to add rules which effect only the second half of the lpm + * table. Use all possible depths ranging from 1..32. Set the next hop = to the + * depth. Check lookup hit for on every add and check for lookup miss on the + * first half of the lpm table after each add. Finally delete all rules going + * backwards (i.e. from depth = 32 ..1) and carry out a lookup after each + * delete. The lookup should return the next_hop_add value related to the + * previous depth value (i.e. depth -1). + */ +int32_t +test17(void) +{ + struct rte_lpm6 *lpm = NULL; + struct rte_lpm6_config config; + uint8_t ip1[] = {127,255,255,255,255,255,255,255,255, + 255,255,255,255,255,255,255}; + uint8_t ip2[] = {128,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; + uint8_t depth; + uint32_t next_hop_add, next_hop_return; + int32_t status = 0; + + config.max_rules = MAX_RULES; + config.number_tbl8s = NUMBER_TBL8S; + config.flags = 0; + + lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm != NULL); + + /* Loop with rte_lpm6_add. */ + for (depth = 1; depth <= 16; depth++) { + /* Let the next_hop_add value = depth. Just for change. */ + next_hop_add = depth; + + status = rte_lpm6_add(lpm, ip2, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + /* Check IP in first half of tbl24 which should be empty. */ + status = rte_lpm6_lookup(lpm, ip1, &next_hop_return); + TEST_LPM_ASSERT(status == -ENOENT); + + status = rte_lpm6_lookup(lpm, ip2, &next_hop_return); + TEST_LPM_ASSERT((status == 0) && + (next_hop_return == next_hop_add)); + } + + /* Loop with rte_lpm6_delete. */ + for (depth = 16; depth >= 1; depth--) { + next_hop_add = (depth - 1); + + status = rte_lpm6_delete(lpm, ip2, depth); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm6_lookup(lpm, ip2, &next_hop_return); + + if (depth != 1) { + TEST_LPM_ASSERT((status == 0) && + (next_hop_return == next_hop_add)); + } + else { + TEST_LPM_ASSERT(status == -ENOENT); + } + + status = rte_lpm6_lookup(lpm, ip1, &next_hop_return); + TEST_LPM_ASSERT(status == -ENOENT); + } + + rte_lpm6_free(lpm); + + return PASS; +} + +/* + * - Add & lookup to hit invalid TBL24 entry + * - Add & lookup to hit valid TBL24 entry not extended + * - Add & lookup to hit valid extended TBL24 entry with invalid TBL8 entry + * - Add & lookup to hit valid extended TBL24 entry with valid TBL8 entry + */ +int32_t +test18(void) +{ + struct rte_lpm6 *lpm = NULL; + struct rte_lpm6_config config; + uint8_t ip[16], ip_1[16], ip_2[16]; + uint8_t depth, depth_1, depth_2; + uint32_t next_hop_add, next_hop_add_1, + next_hop_add_2, next_hop_return; + int32_t status = 0; + + config.max_rules = MAX_RULES; + config.number_tbl8s = NUMBER_TBL8S; + config.flags = 0; + + /* Add & lookup to hit invalid TBL24 entry */ + IPv6(ip, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + depth = 24; + next_hop_add = 100; + + lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm != NULL); + + status = rte_lpm6_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm6_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add)); + + status = rte_lpm6_delete(lpm, ip, depth); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm6_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT(status == -ENOENT); + + rte_lpm6_delete_all(lpm); + + /* Add & lookup to hit valid TBL24 entry not extended */ + IPv6(ip, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + depth = 23; + next_hop_add = 100; + + status = rte_lpm6_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm6_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add)); + + depth = 24; + next_hop_add = 101; + + status = rte_lpm6_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm6_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add)); + + depth = 24; + + status = rte_lpm6_delete(lpm, ip, depth); + TEST_LPM_ASSERT(status == 0); + + depth = 23; + + status = rte_lpm6_delete(lpm, ip, depth); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm6_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT(status == -ENOENT); + + rte_lpm6_delete_all(lpm); + + /* Add & lookup to hit valid extended TBL24 entry with invalid TBL8 + * entry. + */ + IPv6(ip, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + depth = 32; + next_hop_add = 100; + + status = rte_lpm6_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm6_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add)); + + IPv6(ip, 128, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + depth = 32; + next_hop_add = 101; + + status = rte_lpm6_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm6_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add)); + + IPv6(ip, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + depth = 32; + next_hop_add = 100; + + status = rte_lpm6_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add)); + + status = rte_lpm6_delete(lpm, ip, depth); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm6_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT(status == -ENOENT); + + rte_lpm6_delete_all(lpm); + + /* Add & lookup to hit valid extended TBL24 entry with valid TBL8 + * entry + */ + IPv6(ip_1, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + depth_1 = 25; + next_hop_add_1 = 101; + + IPv6(ip_2, 128, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + depth_2 = 32; + next_hop_add_2 = 102; + + next_hop_return = 0; + + status = rte_lpm6_add(lpm, ip_1, depth_1, next_hop_add_1); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm6_lookup(lpm, ip_1, &next_hop_return); + TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add_1)); + + status = rte_lpm6_add(lpm, ip_2, depth_2, next_hop_add_2); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm6_lookup(lpm, ip_2, &next_hop_return); + TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add_2)); + + status = rte_lpm6_delete(lpm, ip_2, depth_2); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm6_lookup(lpm, ip_2, &next_hop_return); + TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add_1)); + + status = rte_lpm6_delete(lpm, ip_1, depth_1); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm6_lookup(lpm, ip_1, &next_hop_return); + TEST_LPM_ASSERT(status == -ENOENT); + + rte_lpm6_free(lpm); + + return PASS; +} + +/* + * - Add rule that covers a TBL24 range previously invalid & lookup (& delete & + * lookup) + * - Add rule that extends a TBL24 invalid entry & lookup (& delete & lookup) + * - Add rule that extends a TBL24 valid entry & lookup for both rules (& + * delete & lookup) + * - Add rule that updates the next hop in TBL24 & lookup (& delete & lookup) + * - Add rule that updates the next hop in TBL8 & lookup (& delete & lookup) + * - Delete a rule that is not present in the TBL24 & lookup + * - Delete a rule that is not present in the TBL8 & lookup + */ +int32_t +test19(void) +{ + struct rte_lpm6 *lpm = NULL; + struct rte_lpm6_config config; + uint8_t ip[16]; + uint8_t depth; + uint32_t next_hop_add, next_hop_return; + int32_t status = 0; + + config.max_rules = MAX_RULES; + config.number_tbl8s = NUMBER_TBL8S; + config.flags = 0; + + /* Add rule that covers a TBL24 range previously invalid & lookup + * (& delete & lookup) + */ + lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm != NULL); + + IPv6(ip, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + depth = 16; + next_hop_add = 100; + + status = rte_lpm6_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm6_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add)); + + status = rte_lpm6_delete(lpm, ip, depth); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm6_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT(status == -ENOENT); + + rte_lpm6_delete_all(lpm); + + IPv6(ip, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + depth = 25; + next_hop_add = 100; + + status = rte_lpm6_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm6_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add)); + + status = rte_lpm6_delete(lpm, ip, depth); + TEST_LPM_ASSERT(status == 0); + + rte_lpm6_delete_all(lpm); + + /* + * Add rule that extends a TBL24 valid entry & lookup for both rules + * (& delete & lookup) + */ + + IPv6(ip, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + depth = 24; + next_hop_add = 100; + + status = rte_lpm6_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + IPv6(ip, 128, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + depth = 32; + next_hop_add = 101; + + status = rte_lpm6_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm6_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add)); + + IPv6(ip, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + next_hop_add = 100; + + status = rte_lpm6_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add)); + + IPv6(ip, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + depth = 24; + + status = rte_lpm6_delete(lpm, ip, depth); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm6_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT(status == -ENOENT); + + IPv6(ip, 128, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + depth = 32; + + status = rte_lpm6_delete(lpm, ip, depth); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm6_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT(status == -ENOENT); + + rte_lpm6_delete_all(lpm); + + /* + * Add rule that updates the next hop in TBL24 & lookup + * (& delete & lookup) + */ + + IPv6(ip, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + depth = 24; + next_hop_add = 100; + + status = rte_lpm6_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm6_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add)); + + next_hop_add = 101; + + status = rte_lpm6_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm6_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add)); + + status = rte_lpm6_delete(lpm, ip, depth); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm6_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT(status == -ENOENT); + + rte_lpm6_delete_all(lpm); + + /* + * Add rule that updates the next hop in TBL8 & lookup + * (& delete & lookup) + */ + + IPv6(ip, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + depth = 32; + next_hop_add = 100; + + status = rte_lpm6_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm6_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add)); + + next_hop_add = 101; + + status = rte_lpm6_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm6_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add)); + + status = rte_lpm6_delete(lpm, ip, depth); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm6_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT(status == -ENOENT); + + rte_lpm6_delete_all(lpm); + + /* Delete a rule that is not present in the TBL24 & lookup */ + + IPv6(ip, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + depth = 24; + next_hop_add = 100; + + status = rte_lpm6_delete(lpm, ip, depth); + TEST_LPM_ASSERT(status < 0); + + status = rte_lpm6_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT(status == -ENOENT); + + rte_lpm6_delete_all(lpm); + + /* Delete a rule that is not present in the TBL8 & lookup */ + + IPv6(ip, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + depth = 32; + next_hop_add = 100; + + status = rte_lpm6_delete(lpm, ip, depth); + TEST_LPM_ASSERT(status < 0); + + status = rte_lpm6_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT(status == -ENOENT); + + rte_lpm6_free(lpm); + + return PASS; +} + +/* + * Add two rules, lookup to hit the more specific one, lookup to hit the less + * specific one delete the less specific rule and lookup previous values again; + * add a more specific rule than the existing rule, lookup again + */ +int32_t +test20(void) +{ + struct rte_lpm6 *lpm = NULL; + struct rte_lpm6_config config; + uint8_t ip[16]; + uint8_t depth; + uint32_t next_hop_add, next_hop_return; + int32_t status = 0; + + config.max_rules = MAX_RULES; + config.number_tbl8s = NUMBER_TBL8S; + config.flags = 0; + + lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm != NULL); + + IPv6(ip, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + depth = 24; + next_hop_add = 100; + + status = rte_lpm6_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + IPv6(ip, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10); + depth = 128; + next_hop_add = 101; + + status = rte_lpm6_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm6_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add)); + + IPv6(ip, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + next_hop_add = 100; + + status = rte_lpm6_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add)); + + IPv6(ip, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + depth = 24; + + status = rte_lpm6_delete(lpm, ip, depth); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm6_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT(status == -ENOENT); + + IPv6(ip, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10); + depth = 128; + + status = rte_lpm6_delete(lpm, ip, depth); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm6_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT(status == -ENOENT); + + rte_lpm6_free(lpm); + + return PASS; +} + +/* + * Adds 3 rules and look them up through the lookup_bulk function. + * Includes in the lookup a fourth IP address that won't match + * and checks that the result is as expected. + */ +int32_t +test21(void) +{ + struct rte_lpm6 *lpm = NULL; + struct rte_lpm6_config config; + uint8_t ip_batch[4][16]; + uint8_t depth; + uint32_t next_hop_add; + int32_t next_hop_return[4]; + int32_t status = 0; + + config.max_rules = MAX_RULES; + config.number_tbl8s = NUMBER_TBL8S; + config.flags = 0; + + lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm != NULL); + + IPv6(ip_batch[0], 128, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + depth = 48; + next_hop_add = 100; + + status = rte_lpm6_add(lpm, ip_batch[0], depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + IPv6(ip_batch[1], 128, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + depth = 48; + next_hop_add = 101; + + status = rte_lpm6_add(lpm, ip_batch[1], depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + IPv6(ip_batch[2], 128, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + depth = 48; + next_hop_add = 102; + + status = rte_lpm6_add(lpm, ip_batch[2], depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + IPv6(ip_batch[3], 128, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + + status = rte_lpm6_lookup_bulk_func(lpm, ip_batch, + next_hop_return, 4); + TEST_LPM_ASSERT(status == 0 && next_hop_return[0] == 100 + && next_hop_return[1] == 101 && next_hop_return[2] == 102 + && next_hop_return[3] == -1); + + rte_lpm6_free(lpm); + + return PASS; +} + +/* + * Adds 5 rules and look them up. + * Use the delete_bulk function to delete two of them. Lookup again. + * Use the delete_bulk function to delete one more. Lookup again. + * Use the delete_bulk function to delete two more, one invalid. Lookup again. + * Use the delete_bulk function to delete the remaining one. Lookup again. + */ +int32_t +test22(void) +{ + struct rte_lpm6 *lpm = NULL; + struct rte_lpm6_config config; + uint8_t ip_batch[5][16]; + uint8_t depth[5]; + uint32_t next_hop_add; + int32_t next_hop_return[5]; + int32_t status = 0; + + config.max_rules = MAX_RULES; + config.number_tbl8s = NUMBER_TBL8S; + config.flags = 0; + + lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm != NULL); + + /* Adds 5 rules and look them up */ + + IPv6(ip_batch[0], 128, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + depth[0] = 48; + next_hop_add = 101; + + status = rte_lpm6_add(lpm, ip_batch[0], depth[0], next_hop_add); + TEST_LPM_ASSERT(status == 0); + + IPv6(ip_batch[1], 128, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + depth[1] = 48; + next_hop_add = 102; + + status = rte_lpm6_add(lpm, ip_batch[1], depth[1], next_hop_add); + TEST_LPM_ASSERT(status == 0); + + IPv6(ip_batch[2], 128, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + depth[2] = 48; + next_hop_add = 103; + + status = rte_lpm6_add(lpm, ip_batch[2], depth[2], next_hop_add); + TEST_LPM_ASSERT(status == 0); + + IPv6(ip_batch[3], 128, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + depth[3] = 48; + next_hop_add = 104; + + status = rte_lpm6_add(lpm, ip_batch[3], depth[3], next_hop_add); + TEST_LPM_ASSERT(status == 0); + + IPv6(ip_batch[4], 128, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + depth[4] = 48; + next_hop_add = 105; + + status = rte_lpm6_add(lpm, ip_batch[4], depth[4], next_hop_add); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm6_lookup_bulk_func(lpm, ip_batch, + next_hop_return, 5); + TEST_LPM_ASSERT(status == 0 && next_hop_return[0] == 101 + && next_hop_return[1] == 102 && next_hop_return[2] == 103 + && next_hop_return[3] == 104 && next_hop_return[4] == 105); + + /* Use the delete_bulk function to delete two of them. Lookup again */ + + status = rte_lpm6_delete_bulk_func(lpm, &ip_batch[0], depth, 2); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm6_lookup_bulk_func(lpm, ip_batch, + next_hop_return, 5); + TEST_LPM_ASSERT(status == 0 && next_hop_return[0] == -1 + && next_hop_return[1] == -1 && next_hop_return[2] == 103 + && next_hop_return[3] == 104 && next_hop_return[4] == 105); + + /* Use the delete_bulk function to delete one more. Lookup again */ + + status = rte_lpm6_delete_bulk_func(lpm, &ip_batch[2], depth, 1); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm6_lookup_bulk_func(lpm, ip_batch, + next_hop_return, 5); + TEST_LPM_ASSERT(status == 0 && next_hop_return[0] == -1 + && next_hop_return[1] == -1 && next_hop_return[2] == -1 + && next_hop_return[3] == 104 && next_hop_return[4] == 105); + + /* Use the delete_bulk function to delete two, one invalid. Lookup again */ + + IPv6(ip_batch[4], 128, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + status = rte_lpm6_delete_bulk_func(lpm, &ip_batch[3], depth, 2); + TEST_LPM_ASSERT(status == 0); + + IPv6(ip_batch[4], 128, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + status = rte_lpm6_lookup_bulk_func(lpm, ip_batch, + next_hop_return, 5); + TEST_LPM_ASSERT(status == 0 && next_hop_return[0] == -1 + && next_hop_return[1] == -1 && next_hop_return[2] == -1 + && next_hop_return[3] == -1 && next_hop_return[4] == 105); + + /* Use the delete_bulk function to delete the remaining one. Lookup again */ + + status = rte_lpm6_delete_bulk_func(lpm, &ip_batch[4], depth, 1); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm6_lookup_bulk_func(lpm, ip_batch, + next_hop_return, 5); + TEST_LPM_ASSERT(status == 0 && next_hop_return[0] == -1 + && next_hop_return[1] == -1 && next_hop_return[2] == -1 + && next_hop_return[3] == -1 && next_hop_return[4] == -1); + + rte_lpm6_free(lpm); + + return PASS; +} + +/* + * Add an extended rule (i.e. depth greater than 24, lookup (hit), delete, + * lookup (miss) in a for loop of 30 times. This will check tbl8 extension + * and contraction. + */ +int32_t +test23(void) +{ + struct rte_lpm6 *lpm = NULL; + struct rte_lpm6_config config; + uint32_t i; + uint8_t ip[16]; + uint8_t depth; + uint32_t next_hop_add, next_hop_return; + int32_t status = 0; + + config.max_rules = MAX_RULES; + config.number_tbl8s = NUMBER_TBL8S; + config.flags = 0; + + lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm != NULL); + + IPv6(ip, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + depth = 128; + next_hop_add = 100; + + for (i = 0; i < 30; i++) { + status = rte_lpm6_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm6_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT((status == 0) && + (next_hop_return == next_hop_add)); + + status = rte_lpm6_delete(lpm, ip, depth); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm6_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT(status == -ENOENT); + } + + rte_lpm6_free(lpm); + + return PASS; +} + +/* + * Sequence of operations for find existing lpm table + * + * - create table + * - find existing table: hit + * - find non-existing table: miss + */ +int32_t +test24(void) +{ + struct rte_lpm6 *lpm = NULL, *result = NULL; + struct rte_lpm6_config config; + + config.max_rules = 256 * 32; + config.number_tbl8s = NUMBER_TBL8S; + config.flags = 0; + + /* Create lpm */ + lpm = rte_lpm6_create("lpm_find_existing", SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm != NULL); + + /* Try to find existing lpm */ + result = rte_lpm6_find_existing("lpm_find_existing"); + TEST_LPM_ASSERT(result == lpm); + + /* Try to find non-existing lpm */ + result = rte_lpm6_find_existing("lpm_find_non_existing"); + TEST_LPM_ASSERT(result == NULL); + + /* Cleanup. */ + rte_lpm6_delete_all(lpm); + rte_lpm6_free(lpm); + + return PASS; +} + +/* + * Add a set of random routes with random depths. + * Lookup different IP addresses that match the routes previously added. + * Checks that the next hop is the expected one. + * The routes, IP addresses and expected result for every case have been + * precalculated by using a python script and stored in a .h file. + */ +int32_t +test25(void) +{ + struct rte_lpm6 *lpm = NULL; + struct rte_lpm6_config config; + uint8_t ip[16]; + uint32_t i; + uint8_t depth; + uint32_t next_hop_add, next_hop_return, next_hop_expected; + int32_t status = 0; + + config.max_rules = MAX_RULES; + config.number_tbl8s = NUMBER_TBL8S; + config.flags = 0; + + lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm != NULL); + + for (i = 0; i < 1000; i++) { + memcpy(ip, large_route_table[i].ip, 16); + depth = large_route_table[i].depth; + next_hop_add = large_route_table[i].next_hop; + status = rte_lpm6_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + } + + /* generate large IPS table and expected next_hops */ + generate_large_ips_table(1); + + for (i = 0; i < 100000; i++) { + memcpy(ip, large_ips_table[i].ip, 16); + next_hop_expected = large_ips_table[i].next_hop; + + status = rte_lpm6_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT((status == 0) && + (next_hop_return == next_hop_expected)); + } + + rte_lpm6_free(lpm); + + return PASS; +} + +/* + * Test for overwriting of tbl8: + * - add rule /32 and lookup + * - add new rule /24 and lookup + * - add third rule /25 and lookup + * - lookup /32 and /24 rule to ensure the table has not been overwritten. + */ +int32_t +test26(void) +{ + struct rte_lpm6 *lpm = NULL; + struct rte_lpm6_config config; + uint8_t ip_10_32[] = {10, 10, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + uint8_t ip_10_24[] = {10, 10, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + uint8_t ip_20_25[] = {10, 10, 20, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + uint8_t d_ip_10_32 = 32; + uint8_t d_ip_10_24 = 24; + uint8_t d_ip_20_25 = 25; + uint32_t next_hop_ip_10_32 = 100; + uint32_t next_hop_ip_10_24 = 105; + uint32_t next_hop_ip_20_25 = 111; + uint32_t next_hop_return = 0; + int32_t status = 0; + + config.max_rules = MAX_RULES; + config.number_tbl8s = NUMBER_TBL8S; + config.flags = 0; + + lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm != NULL); + + if ((status = rte_lpm6_add(lpm, ip_10_32, d_ip_10_32, + next_hop_ip_10_32)) < 0) + return -1; + + status = rte_lpm6_lookup(lpm, ip_10_32, &next_hop_return); + uint32_t test_hop_10_32 = next_hop_return; + TEST_LPM_ASSERT(status == 0); + TEST_LPM_ASSERT(next_hop_return == next_hop_ip_10_32); + + if ((status = rte_lpm6_add(lpm, ip_10_24, d_ip_10_24, + next_hop_ip_10_24)) < 0) + return -1; + + status = rte_lpm6_lookup(lpm, ip_10_24, &next_hop_return); + uint32_t test_hop_10_24 = next_hop_return; + TEST_LPM_ASSERT(status == 0); + TEST_LPM_ASSERT(next_hop_return == next_hop_ip_10_24); + + if ((status = rte_lpm6_add(lpm, ip_20_25, d_ip_20_25, + next_hop_ip_20_25)) < 0) + return -1; + + status = rte_lpm6_lookup(lpm, ip_20_25, &next_hop_return); + uint32_t test_hop_20_25 = next_hop_return; + TEST_LPM_ASSERT(status == 0); + TEST_LPM_ASSERT(next_hop_return == next_hop_ip_20_25); + + if (test_hop_10_32 == test_hop_10_24) { + printf("Next hop return equal\n"); + return -1; + } + + if (test_hop_10_24 == test_hop_20_25){ + printf("Next hop return equal\n"); + return -1; + } + + status = rte_lpm6_lookup(lpm, ip_10_32, &next_hop_return); + TEST_LPM_ASSERT(status == 0); + TEST_LPM_ASSERT(next_hop_return == next_hop_ip_10_32); + + status = rte_lpm6_lookup(lpm, ip_10_24, &next_hop_return); + TEST_LPM_ASSERT(status == 0); + TEST_LPM_ASSERT(next_hop_return == next_hop_ip_10_24); + + rte_lpm6_free(lpm); + + return PASS; +} + +/* + * Add a rule that reaches the end of the tree. + * Add a rule that is more generic than the first one. + * Check every possible combination that produces a match for the second rule. + * This tests tbl expansion. + */ +int32_t +test27(void) +{ + struct rte_lpm6 *lpm = NULL; + struct rte_lpm6_config config; + uint8_t ip[] = {128,128,128,128,128,128,128,128,128,128,128,128,128,128,0,0}; + uint8_t depth = 128; + uint32_t next_hop_add = 100, next_hop_return; + int32_t status = 0; + int i, j; + + config.max_rules = MAX_RULES; + config.number_tbl8s = NUMBER_TBL8S; + config.flags = 0; + + lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm != NULL); + + depth = 128; + next_hop_add = 128; + status = rte_lpm6_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + depth = 112; + next_hop_add = 112; + status = rte_lpm6_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + for (i = 0; i < 256; i++) { + ip[14] = (uint8_t)i; + for (j = 0; j < 256; j++) { + ip[15] = (uint8_t)j; + status = rte_lpm6_lookup(lpm, ip, &next_hop_return); + if (i == 0 && j == 0) + TEST_LPM_ASSERT(status == 0 && next_hop_return == 128); + else + TEST_LPM_ASSERT(status == 0 && next_hop_return == 112); + } + } + + rte_lpm6_free(lpm); + + return PASS; +} + +/* + * Call add, lookup and delete for a single rule with maximum 21bit next_hop + * size. + * Check that next_hop returned from lookup is equal to provisioned value. + * Delete the rule and check that the same test returs a miss. + */ +int32_t +test28(void) +{ + struct rte_lpm6 *lpm = NULL; + struct rte_lpm6_config config; + uint8_t ip[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; + uint8_t depth = 16; + uint32_t next_hop_add = 0x001FFFFF, next_hop_return = 0; + int32_t status = 0; + + config.max_rules = MAX_RULES; + config.number_tbl8s = NUMBER_TBL8S; + config.flags = 0; + + lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm != NULL); + + status = rte_lpm6_add(lpm, ip, depth, next_hop_add); + TEST_LPM_ASSERT(status == 0); + + status = rte_lpm6_lookup(lpm, ip, &next_hop_return); + TEST_LPM_ASSERT((status == 0) && (next_hop_return == next_hop_add)); + + status = rte_lpm6_delete(lpm, ip, depth); + TEST_LPM_ASSERT(status == 0); + rte_lpm6_free(lpm); + + return PASS; +} + +/* + * Do all unit tests. + */ +static int +test_lpm6(void) +{ + unsigned i; + int status = -1, global_status = 0; + + for (i = 0; i < NUM_LPM6_TESTS; i++) { + printf("# test %02d\n", i); + status = tests6[i](); + + if (status < 0) { + printf("ERROR: LPM Test %s: FAIL\n", RTE_STR(tests6[i])); + global_status = status; + } + } + + return global_status; +} + +REGISTER_TEST_COMMAND(lpm6_autotest, test_lpm6); diff --git a/dpdk/test/test/test_lpm6_data.h b/dpdk/test/test/test_lpm6_data.h new file mode 100644 index 00000000..c3573b2b --- /dev/null +++ b/dpdk/test/test/test_lpm6_data.h @@ -0,0 +1,1188 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef _TEST_LPM_ROUTES_H_ +#define _TEST_LPM_ROUTES_H_ + +#include <stdint.h> +#include <stdlib.h> + +struct rules_tbl_entry { + uint8_t ip[16]; + uint8_t depth; + uint8_t next_hop; +}; + +struct ips_tbl_entry { + uint8_t ip[16]; + uint8_t next_hop; +}; + +/* this large_route_table[ ] is the same as the one with same name + * in previous test_lpm6_routes.h . Because this table has only 1000 + * lines, keeping it doesn't make LPM6 test case so large and also + * make the algorithm to generate rule table unnecessary and the + * algorithm to genertate test input IPv6 and associated expected + * next_hop much simple. + */ + +static struct rules_tbl_entry large_route_table[] = { + {{66, 70, 154, 143, 197, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 50, 146}, + {{107, 79, 18, 235, 142, 84, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 54, 141}, + {{247, 132, 113, 1, 215, 247, 183, 239, 128, 0, 0, 0, 0, 0, 0, 0}, 67, 23}, + {{48, 19, 41, 12, 76, 101, 114, 160, 45, 103, 134, 146, 128, 0, 0, 0}, 97, 252}, + {{5, 70, 208, 170, 19, 0, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 54, 6}, + {{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 9, 137}, + {{12, 188, 26, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 31, 9}, + {{1, 235, 101, 202, 26, 92, 23, 22, 179, 223, 128, 0, 0, 0, 0, 0}, 82, 9}, + {{215, 19, 224, 102, 45, 133, 102, 249, 56, 20, 214, 219, 93, 125, 52, 0}, 120, 163}, + {{178, 183, 109, 64, 136, 84, 11, 53, 217, 102, 0, 0, 0, 0, 0, 0}, 79, 197}, + {{212, 39, 158, 71, 253, 98, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 54, 249}, + {{92, 58, 159, 130, 105, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 47, 88}, + {{118, 140, 65, 198, 212, 93, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 52, 104}, + {{86, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 10, 36}, + {{79, 135, 242, 193, 197, 11, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 54, 239}, + {{163, 228, 239, 80, 41, 66, 176, 176, 0, 0, 0, 0, 0, 0, 0, 0}, 67, 201}, + {{31, 9, 231, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 33, 94}, + {{108, 144, 205, 39, 215, 26, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 51, 241}, + {{247, 217, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 24, 239}, + {{24, 186, 73, 182, 240, 251, 125, 165, 0, 0, 0, 0, 0, 0, 0, 0}, 66, 151}, + {{245, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 12, 137}, + {{44, 94, 138, 224, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 41, 231}, + {{184, 221, 109, 135, 225, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 44, 11}, + {{51, 179, 136, 184, 30, 118, 24, 16, 26, 161, 206, 101, 0, 0, 0, 0}, 96, 20}, + {{48, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 15, 68}, + {{143, 235, 237, 220, 89, 119, 187, 143, 209, 94, 46, 58, 120, 0, 0, 0}, 101, 64}, + {{121, 190, 90, 177, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 33, 152}, + {{128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 6, 217}, + {{128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 2, 101}, + {{111, 214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 15, 58}, + {{162, 23, 52, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 27, 254}, + {{76, 103, 44, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 32, 148}, + {{80, 85, 219, 214, 12, 4, 65, 129, 162, 148, 208, 78, 39, 69, 94, 184}, 126, 126}, + {{80, 54, 251, 28, 152, 23, 244, 192, 151, 83, 6, 144, 223, 213, 224, 128}, 123, 76}, + {{39, 232, 237, 103, 191, 188, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 54, 240}, + {{20, 231, 89, 210, 167, 173, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 54, 33}, + {{125, 67, 198, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 25, 47}, + {{26, 239, 153, 5, 213, 121, 31, 114, 161, 46, 84, 15, 148, 160, 0, 0}, 109, 41}, + {{102, 212, 159, 118, 223, 115, 134, 172, 0, 0, 0, 0, 0, 0, 0, 0}, 62, 72}, + {{85, 181, 241, 127, 3, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 46, 43}, + {{61, 199, 131, 226, 3, 230, 94, 119, 240, 0, 0, 0, 0, 0, 0, 0}, 68, 26}, + {{0, 143, 160, 184, 162, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 42, 139}, + {{170, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 13, 219}, + {{61, 122, 24, 251, 124, 122, 202, 192, 0, 0, 0, 0, 0, 0, 0, 0}, 58, 105}, + {{33, 219, 226, 3, 180, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 47, 210}, + {{51, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 17, 151}, + {{106, 185, 11, 122, 197, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 42, 28}, + {{192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 9, 64}, + {{239, 195, 77, 239, 131, 156, 2, 246, 191, 178, 204, 160, 21, 213, 30, 128}, 121, 9}, + {{141, 207, 181, 99, 55, 245, 151, 228, 65, 50, 85, 16, 0, 0, 0, 0}, 92, 250}, + {{110, 159, 230, 251, 224, 210, 58, 49, 0, 0, 0, 0, 0, 0, 0, 0}, 66, 200}, + {{134, 26, 104, 32, 129, 41, 201, 50, 164, 69, 178, 156, 156, 133, 8, 218}, 127, 132}, + {{253, 207, 116, 105, 210, 166, 186, 99, 182, 0, 0, 0, 0, 0, 0, 0}, 71, 182}, + {{211, 73, 38, 80, 183, 168, 52, 138, 25, 214, 112, 8, 252, 0, 0, 0}, 102, 7}, + {{200, 244, 108, 238, 164, 141, 215, 39, 233, 249, 120, 80, 112, 0, 0, 0}, 100, 146}, + {{107, 44, 250, 202, 64, 37, 107, 105, 140, 0, 0, 0, 0, 0, 0, 0}, 70, 98}, + {{93, 86, 56, 27, 159, 195, 126, 39, 240, 201, 48, 0, 0, 0, 0, 0}, 86, 179}, + {{32, 202, 214, 242, 39, 141, 61, 146, 138, 96, 0, 0, 0, 0, 0, 0}, 77, 245}, + {{167, 77, 249, 28, 210, 196, 227, 241, 0, 0, 0, 0, 0, 0, 0, 0}, 64, 2}, + {{241, 59, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 17, 5}, + {{143, 68, 146, 210, 173, 155, 251, 173, 0, 0, 0, 0, 0, 0, 0, 0}, 66, 169}, + {{167, 180, 226, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 33, 52}, + {{241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 9, 177}, + {{238, 9, 168, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 27, 74}, + {{203, 148, 16, 96, 125, 18, 86, 1, 91, 244, 251, 20, 31, 14, 75, 128}, 122, 212}, + {{111, 227, 137, 94, 65, 21, 77, 137, 119, 130, 159, 19, 159, 45, 18, 192}, 122, 238}, + {{59, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 19, 18}, + {{110, 192, 255, 120, 84, 215, 3, 130, 38, 224, 0, 0, 0, 0, 0, 0}, 75, 155}, + {{152, 79, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 24, 97}, + {{118, 186, 157, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 32, 8}, + {{70, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 9, 123}, + {{253, 119, 114, 227, 18, 243, 81, 61, 238, 107, 190, 144, 0, 0, 0, 0}, 92, 11}, + {{166, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 13, 211}, + {{43, 95, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 20, 116}, + {{94, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 11, 57}, + {{182, 251, 195, 132, 66, 7, 208, 146, 223, 231, 211, 181, 25, 176, 0, 0}, 108, 178}, + {{152, 166, 111, 233, 194, 17, 230, 41, 221, 253, 69, 123, 108, 0, 0, 0}, 102, 93}, + {{106, 141, 235, 190, 82, 241, 152, 186, 195, 81, 86, 144, 0, 0, 0, 0}, 92, 3}, + {{32, 81, 210, 153, 151, 29, 11, 62, 127, 177, 194, 254, 103, 83, 58, 128}, 121, 162}, + {{79, 112, 224, 26, 174, 39, 98, 181, 115, 57, 209, 189, 136, 48, 0, 0}, 109, 125}, + {{106, 197, 83, 151, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 34, 33}, + {{190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 9, 254}, + {{156, 73, 249, 148, 55, 192, 20, 42, 142, 128, 0, 0, 0, 0, 0, 0}, 74, 66}, + {{64, 107, 36, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 30, 4}, + {{115, 148, 71, 250, 158, 174, 168, 249, 106, 110, 196, 0, 0, 0, 0, 0}, 86, 122}, + {{18, 139, 152, 44, 38, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 46, 59}, + {{55, 229, 117, 106, 146, 95, 74, 220, 122, 0, 84, 202, 183, 138, 120, 0}, 117, 99}, + {{153, 211, 3, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 27, 41}, + {{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 8, 112}, + {{49, 192, 102, 142, 216, 3, 114, 64, 165, 128, 168, 0, 0, 0, 0, 0}, 85, 255}, + {{201, 143, 240, 240, 209, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 44, 106}, + {{158, 19, 164, 196, 87, 162, 33, 120, 0, 0, 0, 0, 0, 0, 0, 0}, 62, 170}, + {{5, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 16, 86}, + {{34, 170, 246, 62, 198, 85, 193, 227, 252, 68, 0, 0, 0, 0, 0, 0}, 79, 155}, + {{21, 52, 9, 86, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 35, 65}, + {{203, 81, 49, 171, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 36, 39}, + {{211, 218, 87, 244, 93, 181, 118, 41, 156, 143, 254, 0, 0, 0, 0, 0}, 90, 162}, + {{77, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 10, 69}, + {{158, 219, 219, 39, 4, 219, 100, 63, 0, 0, 0, 0, 0, 0, 0, 0}, 64, 163}, + {{61, 50, 232, 1, 185, 252, 243, 54, 189, 240, 170, 192, 0, 0, 0, 0}, 90, 116}, + {{241, 143, 33, 19, 247, 55, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 53, 19}, + {{61, 28, 61, 252, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 36, 48}, + {{102, 112, 194, 108, 90, 253, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 49, 230}, + {{74, 88, 58, 66, 172, 41, 144, 204, 195, 240, 0, 0, 0, 0, 0, 0}, 78, 155}, + {{44, 148, 187, 58, 190, 59, 190, 187, 124, 138, 222, 131, 0, 0, 0, 0}, 96, 158}, + {{67, 7, 216, 139, 93, 224, 20, 135, 186, 86, 209, 111, 60, 80, 0, 0}, 113, 252}, + {{209, 26, 12, 174, 5, 101, 164, 181, 237, 63, 192, 57, 54, 120, 0, 0}, 110, 176}, + {{4, 66, 232, 52, 239, 56, 48, 58, 192, 0, 0, 0, 0, 0, 0, 0}, 66, 211}, + {{158, 165, 2, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 28, 15}, + {{85, 204, 245, 198, 68, 44, 39, 71, 32, 0, 0, 0, 0, 0, 0, 0}, 68, 95}, + {{181, 134, 25, 87, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 34, 169}, + {{26, 230, 61, 36, 79, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 44, 249}, + {{5, 170, 198, 139, 65, 186, 188, 45, 42, 253, 165, 89, 206, 0, 0, 0}, 105, 61}, + {{211, 245, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 23, 63}, + {{117, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 14, 43}, + {{103, 17, 123, 102, 70, 206, 90, 92, 124, 198, 0, 0, 0, 0, 0, 0}, 81, 228}, + {{192, 237, 88, 244, 53, 30, 61, 160, 143, 64, 0, 0, 0, 0, 0, 0}, 78, 165}, + {{199, 82, 217, 183, 2, 179, 195, 6, 0, 0, 0, 0, 0, 0, 0, 0}, 64, 3}, + {{157, 230, 79, 162, 57, 125, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 57, 211}, + {{27, 67, 64, 235, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 33, 210}, + {{72, 158, 163, 106, 193, 137, 190, 7, 250, 165, 249, 73, 64, 0, 0, 0}, 99, 61}, + {{34, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 10, 120}, + {{215, 141, 95, 192, 189, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 47, 94}, + {{31, 181, 56, 141, 120, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 41, 153}, + {{153, 73, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 18, 221}, + {{162, 107, 41, 189, 165, 155, 22, 139, 165, 72, 96, 0, 0, 0, 0, 0}, 87, 163}, + {{218, 17, 204, 165, 217, 251, 107, 45, 29, 15, 192, 167, 75, 0, 0, 0}, 106, 188}, + {{200, 124, 238, 213, 35, 228, 94, 141, 86, 187, 101, 60, 115, 52, 131, 16}, 124, 15}, + {{74, 237, 160, 56, 141, 217, 191, 16, 0, 0, 0, 0, 0, 0, 0, 0}, 63, 28}, + {{163, 47, 242, 103, 173, 217, 88, 154, 38, 200, 32, 0, 0, 0, 0, 0}, 84, 240}, + {{20, 227, 128, 28, 144, 147, 22, 13, 94, 129, 107, 88, 0, 0, 0, 0}, 93, 59}, + {{95, 144, 229, 107, 218, 125, 204, 233, 161, 42, 180, 64, 0, 0, 0, 0}, 90, 195}, + {{155, 220, 83, 208, 108, 16, 134, 156, 128, 0, 0, 0, 0, 0, 0, 0}, 66, 10}, + {{179, 138, 55, 80, 190, 153, 12, 237, 22, 120, 69, 0, 0, 0, 0, 0}, 88, 206}, + {{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 2, 137}, + {{3, 119, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 22, 225}, + {{13, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 10, 223}, + {{117, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 15, 29}, + {{164, 19, 195, 47, 136, 190, 156, 255, 30, 74, 143, 134, 162, 0, 0, 0}, 103, 166}, + {{40, 235, 94, 135, 135, 230, 71, 33, 64, 233, 0, 0, 0, 0, 0, 0}, 80, 178}, + {{222, 151, 166, 97, 129, 250, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 55, 38}, + {{174, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 9, 141}, + {{6, 189, 100, 150, 250, 13, 46, 98, 228, 139, 50, 52, 52, 196, 128, 0}, 116, 230}, + {{75, 252, 89, 205, 37, 52, 106, 79, 188, 120, 54, 119, 160, 0, 0, 0}, 99, 124}, + {{38, 18, 146, 6, 63, 64, 231, 10, 152, 199, 5, 143, 147, 4, 252, 0}, 118, 54}, + {{111, 119, 169, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 32, 162}, + {{105, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 13, 32}, + {{143, 57, 57, 101, 98, 182, 74, 227, 205, 143, 253, 237, 8, 0, 0, 0}, 102, 237}, + {{30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 7, 215}, + {{14, 232, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 22, 138}, + {{14, 53, 67, 216, 229, 155, 149, 139, 31, 253, 184, 126, 133, 108, 40, 0}, 118, 73}, + {{22, 58, 40, 143, 188, 132, 239, 14, 181, 252, 81, 192, 0, 0, 0, 0}, 90, 43}, + {{11, 222, 185, 243, 248, 150, 79, 230, 214, 213, 3, 23, 193, 196, 0, 0}, 112, 88}, + {{14, 226, 198, 117, 84, 93, 22, 96, 77, 241, 173, 68, 68, 204, 72, 0}, 119, 91}, + {{15, 103, 247, 219, 150, 142, 92, 50, 144, 0, 0, 0, 0, 0, 0, 0}, 69, 140}, + {{0, 213, 77, 244, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 37, 65}, + {{178, 174, 174, 239, 72, 181, 36, 217, 40, 169, 12, 104, 149, 157, 125, 128}, 122, 201}, + {{118, 53, 55, 17, 97, 227, 243, 176, 2, 0, 0, 0, 0, 0, 0, 0}, 72, 69}, + {{21, 253, 4, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 35, 170}, + {{5, 249, 186, 133, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 40, 192}, + {{47, 79, 35, 66, 11, 178, 161, 28, 87, 180, 45, 128, 0, 0, 0, 0}, 89, 21}, + {{242, 227, 20, 73, 150, 196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 46, 35}, + {{121, 169, 102, 118, 157, 192, 154, 186, 126, 0, 0, 0, 0, 0, 0, 0}, 71, 235}, + {{9, 138, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 21, 240}, + {{45, 173, 14, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 30, 136}, + {{127, 47, 51, 201, 236, 45, 142, 80, 0, 0, 0, 0, 0, 0, 0, 0}, 60, 186}, + {{247, 233, 34, 38, 181, 207, 127, 20, 224, 118, 59, 148, 0, 0, 0, 0}, 95, 174}, + {{126, 187, 198, 104, 245, 223, 219, 18, 31, 124, 0, 0, 0, 0, 0, 0}, 79, 153}, + {{3, 163, 107, 228, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 35, 118}, + {{167, 109, 2, 95, 11, 62, 45, 128, 0, 0, 0, 0, 0, 0, 0, 0}, 60, 113}, + {{76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 6, 58}, + {{58, 190, 204, 151, 222, 147, 47, 78, 38, 203, 9, 17, 64, 0, 0, 0}, 101, 206}, + {{254, 220, 254, 220, 204, 79, 35, 127, 242, 63, 106, 232, 127, 180, 0, 0}, 111, 42}, + {{77, 156, 8, 209, 181, 37, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 55, 230}, + {{65, 89, 137, 76, 208, 199, 166, 90, 128, 0, 0, 0, 0, 0, 0, 0}, 67, 6}, + {{47, 232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 13, 254}, + {{172, 154, 12, 108, 77, 37, 106, 8, 234, 7, 248, 212, 112, 160, 0, 0}, 108, 214}, + {{254, 117, 239, 244, 154, 89, 166, 241, 12, 108, 127, 153, 206, 160, 0, 0}, 107, 43}, + {{113, 160, 206, 52, 143, 12, 9, 148, 224, 0, 0, 0, 0, 0, 0, 0}, 67, 178}, + {{178, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 16, 179}, + {{229, 177, 28, 106, 59, 75, 182, 241, 36, 79, 224, 0, 0, 0, 0, 0}, 87, 236}, + {{156, 72, 93, 193, 50, 235, 75, 228, 88, 115, 89, 119, 128, 0, 0, 0}, 98, 184}, + {{28, 232, 28, 249, 83, 105, 211, 7, 136, 147, 231, 64, 0, 0, 0, 0}, 91, 95}, + {{217, 33, 23, 107, 74, 42, 135, 197, 144, 34, 40, 243, 13, 126, 36, 136}, 127, 152}, + {{64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 2, 113}, + {{85, 172, 121, 126, 213, 57, 225, 54, 197, 73, 85, 251, 9, 64, 0, 0}, 108, 137}, + {{104, 46, 25, 71, 86, 220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 46, 224}, + {{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 6, 61}, + {{241, 113, 254, 106, 53, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 41, 205}, + {{29, 36, 12, 244, 197, 127, 240, 8, 167, 134, 154, 248, 199, 123, 143, 240}, 124, 170}, + {{58, 29, 129, 94, 43, 139, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 53, 117}, + {{213, 124, 147, 196, 7, 82, 67, 70, 228, 0, 0, 0, 0, 0, 0, 0}, 70, 225}, + {{164, 168, 161, 140, 87, 85, 250, 41, 34, 0, 0, 0, 0, 0, 0, 0}, 72, 34}, + {{186, 142, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 17, 5}, + {{237, 249, 9, 70, 247, 97, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 50, 92}, + {{155, 92, 145, 218, 125, 226, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 55, 230}, + {{35, 169, 62, 156, 86, 4, 125, 219, 119, 113, 191, 75, 198, 113, 0, 0}, 112, 61}, + {{207, 63, 96, 186, 26, 68, 115, 161, 163, 59, 190, 166, 18, 78, 232, 0}, 117, 221}, + {{86, 40, 200, 199, 247, 86, 159, 179, 191, 184, 117, 173, 211, 158, 0, 128}, 121, 105}, + {{104, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 11, 181}, + {{205, 35, 123, 178, 36, 64, 62, 153, 195, 250, 0, 0, 0, 0, 0, 0}, 79, 110}, + {{117, 40, 57, 157, 138, 160, 223, 59, 155, 145, 64, 0, 0, 0, 0, 0}, 86, 103}, + {{74, 166, 140, 146, 74, 72, 229, 99, 167, 124, 107, 117, 217, 14, 246, 64}, 123, 218}, + {{12, 222, 244, 183, 83, 146, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 56, 146}, + {{11, 98, 146, 110, 95, 96, 80, 142, 224, 0, 0, 0, 0, 0, 0, 0}, 67, 90}, + {{235, 5, 187, 199, 30, 170, 82, 187, 228, 159, 22, 25, 204, 112, 0, 0}, 108, 197}, + {{35, 96, 146, 145, 155, 116, 252, 181, 29, 205, 230, 246, 30, 0, 0, 0}, 103, 158}, + {{174, 38, 56, 244, 227, 102, 252, 237, 128, 86, 0, 0, 0, 0, 0, 0}, 81, 118}, + {{65, 134, 37, 58, 90, 125, 60, 84, 0, 0, 0, 0, 0, 0, 0, 0}, 62, 95}, + {{253, 117, 135, 98, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 33, 152}, + {{111, 115, 188, 184, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 45, 239}, + {{202, 24, 89, 9, 149, 45, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 50, 48}, + {{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 5, 228}, + {{244, 98, 52, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 30, 247}, + {{151, 167, 43, 178, 116, 194, 173, 126, 236, 98, 40, 0, 0, 0, 0, 0}, 85, 12}, + {{60, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 10, 129}, + {{208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 4, 50}, + {{126, 11, 216, 242, 7, 45, 121, 208, 110, 135, 210, 75, 59, 182, 228, 42}, 128, 250}, + {{217, 26, 184, 146, 3, 18, 240, 15, 135, 8, 0, 0, 0, 0, 0, 0}, 77, 249}, + {{128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 1, 230}, + {{145, 28, 29, 184, 2, 85, 234, 135, 98, 111, 136, 32, 0, 0, 0, 0}, 92, 228}, + {{108, 104, 255, 254, 34, 95, 72, 157, 0, 0, 0, 0, 0, 0, 0, 0}, 64, 181}, + {{153, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 16, 206}, + {{22, 250, 130, 201, 132, 248, 189, 108, 0, 0, 0, 0, 0, 0, 0, 0}, 63, 122}, + {{158, 165, 234, 18, 44, 61, 82, 61, 235, 0, 0, 0, 0, 0, 0, 0}, 72, 81}, + {{236, 57, 124, 110, 124, 218, 82, 70, 142, 78, 18, 128, 0, 0, 0, 0}, 95, 175}, + {{94, 209, 200, 201, 149, 162, 248, 134, 239, 226, 1, 237, 16, 134, 56, 0}, 118, 170}, + {{187, 42, 31, 144, 236, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 47, 174}, + {{90, 214, 185, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 29, 104}, + {{194, 220, 211, 212, 211, 32, 196, 98, 71, 62, 153, 103, 80, 35, 128, 0}, 114, 113}, + {{24, 255, 158, 64, 180, 148, 10, 81, 243, 247, 0, 0, 0, 0, 0, 0}, 80, 89}, + {{231, 155, 100, 242, 112, 160, 160, 95, 98, 253, 219, 21, 239, 90, 0, 0}, 113, 151}, + {{225, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 11, 108}, + {{136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 7, 224}, + {{250, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 9, 95}, + {{72, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 14, 173}, + {{185, 51, 51, 167, 18, 44, 36, 59, 35, 135, 20, 104, 0, 0, 0, 0}, 93, 176}, + {{57, 146, 252, 60, 197, 68, 39, 162, 80, 198, 137, 50, 97, 92, 124, 0}, 119, 84}, + {{254, 46, 242, 105, 86, 94, 96, 14, 130, 176, 0, 0, 0, 0, 0, 0}, 78, 104}, + {{247, 202, 176, 76, 69, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 49, 236}, + {{50, 233, 203, 77, 42, 21, 115, 163, 166, 138, 192, 52, 178, 37, 112, 0}, 116, 153}, + {{62, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 11, 190}, + {{53, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 13, 202}, + {{198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 8, 54}, + {{189, 234, 106, 247, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 34, 156}, + {{110, 24, 228, 65, 216, 147, 9, 48, 60, 179, 172, 91, 115, 185, 227, 96}, 126, 245}, + {{128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 6, 218}, + {{74, 177, 89, 218, 248, 18, 176, 39, 118, 173, 201, 152, 0, 0, 0, 0}, 93, 72}, + {{31, 13, 153, 92, 27, 122, 150, 232, 88, 95, 202, 171, 208, 158, 0, 0}, 112, 183}, + {{128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 2, 183}, + {{63, 37, 46, 158, 139, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 42, 241}, + {{53, 209, 59, 13, 202, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 47, 106}, + {{184, 44, 149, 221, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 40, 180}, + {{222, 134, 37, 62, 223, 193, 39, 246, 15, 151, 200, 146, 0, 0, 0, 0}, 96, 142}, + {{199, 176, 189, 37, 233, 177, 252, 216, 94, 175, 253, 119, 96, 0, 0, 0}, 100, 6}, + {{44, 195, 201, 106, 209, 120, 122, 38, 43, 30, 142, 22, 196, 175, 100, 0}, 118, 33}, + {{33, 166, 10, 174, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 34, 224}, + {{54, 1, 189, 195, 133, 49, 36, 80, 138, 200, 0, 0, 0, 0, 0, 0}, 78, 14}, + {{241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 10, 149}, + {{221, 131, 4, 247, 112, 89, 187, 119, 219, 80, 122, 156, 216, 160, 0, 0}, 108, 131}, + {{102, 20, 46, 129, 202, 247, 129, 1, 237, 71, 103, 58, 217, 44, 4, 0}, 121, 133}, + {{107, 156, 151, 44, 215, 98, 171, 126, 85, 32, 42, 128, 0, 0, 0, 0}, 89, 33}, + {{54, 25, 70, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 28, 204}, + {{149, 211, 242, 14, 112, 219, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 52, 43}, + {{95, 26, 143, 193, 8, 76, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 51, 168}, + {{63, 102, 244, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 28, 180}, + {{64, 85, 124, 226, 59, 239, 64, 130, 68, 122, 93, 74, 32, 37, 0, 0}, 112, 208}, + {{113, 90, 253, 149, 3, 218, 34, 215, 3, 143, 192, 64, 0, 0, 0, 0}, 90, 25}, + {{75, 231, 33, 5, 11, 94, 117, 104, 150, 60, 72, 161, 96, 38, 0, 0}, 111, 50}, + {{52, 13, 248, 1, 251, 14, 50, 29, 212, 123, 130, 177, 101, 96, 0, 0}, 109, 110}, + {{248, 221, 150, 132, 252, 82, 96, 2, 80, 232, 97, 239, 253, 64, 0, 0}, 109, 21}, + {{136, 77, 164, 161, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 36, 147}, + {{1, 33, 66, 254, 144, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 43, 56}, + {{181, 25, 186, 225, 109, 190, 76, 158, 118, 122, 20, 64, 125, 55, 8, 0}, 117, 144}, + {{191, 187, 160, 140, 17, 6, 80, 120, 236, 212, 104, 144, 128, 0, 0, 0}, 100, 198}, + {{201, 61, 150, 254, 70, 77, 214, 211, 171, 163, 245, 64, 0, 0, 0, 0}, 90, 235}, + {{143, 226, 190, 50, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 38, 105}, + {{65, 168, 226, 36, 201, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 45, 138}, + {{136, 40, 65, 90, 47, 16, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 49, 122}, + {{94, 189, 224, 200, 170, 11, 79, 172, 0, 0, 0, 0, 0, 0, 0, 0}, 65, 193}, + {{236, 41, 169, 234, 14, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 43, 231}, + {{1, 40, 140, 95, 81, 173, 250, 248, 0, 0, 0, 0, 0, 0, 0, 0}, 64, 250}, + {{83, 176, 146, 112, 89, 156, 57, 220, 125, 48, 44, 0, 0, 0, 0, 0}, 86, 24}, + {{76, 125, 228, 249, 243, 160, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 55, 191}, + {{10, 203, 204, 49, 212, 115, 125, 4, 239, 122, 81, 34, 1, 198, 216, 0}, 117, 111}, + {{74, 214, 23, 44, 211, 40, 161, 61, 237, 190, 155, 59, 173, 42, 0, 0}, 111, 205}, + {{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 1, 133}, + {{127, 0, 130, 61, 209, 5, 232, 35, 35, 42, 114, 52, 169, 234, 191, 0}, 122, 122}, + {{201, 107, 210, 13, 187, 62, 145, 28, 31, 189, 56, 0, 0, 0, 0, 0}, 87, 227}, + {{147, 171, 63, 145, 47, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 46, 53}, + {{93, 232, 10, 97, 21, 243, 213, 135, 200, 0, 0, 0, 0, 0, 0, 0}, 72, 224}, + {{144, 121, 41, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 26, 199}, + {{116, 105, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 17, 79}, + {{142, 149, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 21, 19}, + {{97, 0, 228, 158, 50, 233, 251, 249, 0, 66, 197, 226, 0, 0, 0, 0}, 96, 211}, + {{114, 228, 199, 155, 175, 104, 26, 213, 66, 249, 120, 218, 164, 252, 212, 0}, 120, 6}, + {{224, 166, 76, 200, 121, 60, 110, 65, 60, 95, 137, 190, 92, 218, 218, 0}, 121, 143}, + {{139, 219, 92, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 31, 135}, + {{203, 237, 64, 189, 28, 13, 75, 197, 219, 243, 172, 3, 142, 32, 0, 0}, 109, 21}, + {{237, 186, 88, 254, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 38, 220}, + {{182, 230, 93, 162, 129, 25, 56, 196, 112, 0, 0, 0, 0, 0, 0, 0}, 68, 151}, + {{245, 45, 69, 226, 90, 212, 254, 16, 0, 0, 0, 0, 0, 0, 0, 0}, 60, 111}, + {{107, 229, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 20, 63}, + {{119, 208, 177, 235, 222, 252, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 57, 112}, + {{178, 151, 220, 162, 120, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 41, 48}, + {{109, 26, 95, 170, 166, 151, 137, 83, 226, 82, 5, 114, 253, 210, 18, 12}, 126, 100}, + {{126, 27, 252, 19, 219, 129, 121, 48, 0, 0, 0, 0, 0, 0, 0, 0}, 60, 156}, + {{211, 195, 152, 145, 154, 93, 228, 215, 135, 101, 28, 82, 0, 0, 0, 0}, 95, 120}, + {{252, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 16, 5}, + {{192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 4, 103}, + {{64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 4, 84}, + {{225, 179, 43, 43, 222, 145, 205, 238, 164, 158, 147, 229, 56, 0, 0, 0}, 101, 24}, + {{208, 127, 151, 24, 64, 113, 47, 85, 209, 79, 144, 0, 0, 0, 0, 0}, 86, 81}, + {{178, 144, 203, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 29, 96}, + {{56, 227, 139, 4, 86, 87, 180, 1, 215, 167, 237, 156, 111, 64, 47, 0}, 121, 6}, + {{80, 76, 204, 119, 172, 169, 254, 81, 104, 166, 219, 44, 173, 161, 212, 0}, 119, 40}, + {{129, 141, 139, 34, 241, 101, 223, 144, 0, 0, 0, 0, 0, 0, 0, 0}, 62, 143}, + {{85, 102, 137, 98, 65, 103, 54, 142, 144, 0, 0, 0, 0, 0, 0, 0}, 68, 69}, + {{56, 31, 159, 13, 201, 139, 161, 31, 89, 137, 4, 0, 0, 0, 0, 0}, 92, 48}, + {{229, 221, 54, 216, 223, 27, 196, 1, 0, 0, 0, 0, 0, 0, 0, 0}, 64, 115}, + {{5, 144, 176, 43, 180, 187, 20, 49, 59, 73, 108, 34, 83, 32, 192, 0}, 115, 130}, + {{24, 217, 205, 193, 74, 123, 160, 106, 103, 74, 200, 0, 0, 0, 0, 0}, 86, 57}, + {{247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 8, 97}, + {{12, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 9, 146}, + {{160, 28, 201, 119, 148, 93, 251, 118, 28, 179, 123, 52, 71, 232, 48, 0}, 117, 194}, + {{152, 126, 17, 54, 101, 56, 130, 1, 205, 41, 207, 90, 151, 123, 128, 0}, 114, 129}, + {{77, 165, 29, 239, 95, 242, 34, 1, 11, 204, 135, 239, 128, 0, 0, 0}, 97, 159}, + {{183, 108, 146, 118, 74, 190, 7, 141, 9, 92, 2, 2, 8, 218, 120, 0}, 117, 242}, + {{37, 152, 29, 239, 242, 53, 56, 143, 219, 22, 14, 158, 49, 0, 0, 0}, 104, 162}, + {{198, 53, 241, 102, 240, 244, 97, 203, 62, 128, 213, 214, 220, 0, 0, 0}, 102, 140}, + {{144, 89, 48, 42, 249, 231, 189, 178, 232, 199, 30, 58, 63, 57, 0, 0}, 113, 77}, + {{68, 212, 177, 123, 44, 224, 19, 172, 89, 87, 192, 0, 0, 0, 0, 0}, 82, 121}, + {{252, 29, 179, 224, 4, 121, 205, 67, 152, 0, 0, 0, 0, 0, 0, 0}, 69, 102}, + {{28, 110, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 23, 28}, + {{24, 88, 231, 1, 4, 71, 71, 241, 252, 14, 197, 0, 0, 0, 0, 0}, 89, 154}, + {{63, 131, 43, 76, 58, 140, 163, 74, 158, 80, 0, 0, 0, 0, 0, 0}, 76, 39}, + {{56, 28, 147, 149, 98, 93, 216, 216, 203, 156, 0, 0, 0, 0, 0, 0}, 78, 163}, + {{134, 169, 6, 103, 161, 244, 134, 117, 16, 0, 0, 0, 0, 0, 0, 0}, 68, 42}, + {{143, 247, 125, 190, 106, 50, 204, 98, 250, 151, 161, 96, 0, 0, 0, 0}, 92, 207}, + {{235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 8, 25}, + {{46, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 18, 150}, + {{171, 35, 128, 117, 74, 29, 199, 67, 109, 176, 0, 0, 0, 0, 0, 0}, 76, 103}, + {{220, 233, 236, 112, 135, 136, 215, 43, 42, 0, 0, 0, 0, 0, 0, 0}, 71, 155}, + {{228, 11, 144, 117, 206, 192, 118, 25, 141, 78, 4, 105, 0, 0, 0, 0}, 96, 142}, + {{195, 67, 194, 229, 14, 53, 129, 7, 30, 208, 38, 100, 182, 59, 0, 0}, 112, 2}, + {{25, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 19, 59}, + {{160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 4, 112}, + {{26, 203, 217, 152, 16, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 48, 166}, + {{250, 213, 14, 235, 110, 171, 174, 23, 102, 128, 0, 0, 0, 0, 0, 0}, 73, 62}, + {{175, 230, 160, 13, 187, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 50, 176}, + {{92, 155, 156, 93, 191, 73, 28, 82, 187, 129, 57, 5, 16, 0, 0, 0}, 100, 6}, + {{45, 203, 3, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 29, 26}, + {{120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 5, 6}, + {{216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 5, 13}, + {{135, 215, 0, 71, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 37, 41}, + {{221, 149, 1, 40, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 36, 135}, + {{95, 143, 255, 194, 2, 157, 191, 113, 10, 229, 204, 56, 0, 0, 0, 0}, 93, 171}, + {{202, 212, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 19, 20}, + {{147, 203, 238, 120, 194, 23, 25, 58, 208, 177, 169, 0, 0, 0, 0, 0}, 89, 119}, + {{137, 170, 113, 252, 215, 194, 224, 146, 233, 87, 86, 192, 26, 46, 0, 0}, 112, 49}, + {{224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 4, 141}, + {{250, 90, 241, 174, 163, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 47, 132}, + {{66, 190, 202, 144, 122, 86, 22, 103, 107, 164, 57, 54, 228, 128, 0, 0}, 105, 176}, + {{76, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 12, 186}, + {{120, 246, 1, 52, 187, 163, 78, 105, 224, 0, 0, 0, 0, 0, 0, 0}, 67, 93}, + {{137, 242, 136, 71, 98, 10, 53, 97, 160, 85, 132, 127, 185, 222, 0, 0}, 111, 242}, + {{255, 133, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 24, 163}, + {{128, 177, 92, 155, 91, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 42, 184}, + {{45, 120, 186, 192, 240, 199, 178, 95, 32, 0, 0, 0, 0, 0, 0, 0}, 68, 188}, + {{151, 98, 103, 254, 90, 6, 10, 109, 14, 158, 69, 29, 140, 237, 40, 232}, 126, 193}, + {{148, 164, 81, 85, 76, 14, 84, 64, 89, 176, 0, 0, 0, 0, 0, 0}, 78, 63}, + {{145, 187, 165, 136, 88, 30, 107, 191, 205, 120, 119, 216, 158, 123, 64, 0}, 115, 160}, + {{78, 120, 28, 243, 216, 180, 87, 19, 253, 16, 110, 33, 228, 24, 232, 0}, 117, 251}, + {{74, 6, 166, 166, 183, 157, 96, 84, 151, 0, 0, 0, 0, 0, 0, 0}, 72, 228}, + {{89, 96, 4, 221, 214, 253, 58, 49, 9, 0, 0, 0, 0, 0, 0, 0}, 72, 168}, + {{97, 9, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 18, 194}, + {{213, 215, 45, 200, 170, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 47, 166}, + {{5, 14, 92, 0, 28, 245, 130, 202, 32, 40, 207, 77, 166, 170, 246, 64}, 122, 210}, + {{77, 45, 43, 71, 202, 0, 157, 146, 59, 91, 225, 0, 0, 0, 0, 0}, 89, 254}, + {{101, 174, 94, 168, 162, 171, 71, 12, 16, 224, 0, 0, 0, 0, 0, 0}, 75, 49}, + {{58, 17, 187, 194, 87, 73, 215, 103, 180, 12, 40, 66, 0, 0, 0, 0}, 96, 95}, + {{160, 91, 68, 81, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 33, 193}, + {{94, 112, 249, 13, 167, 245, 101, 64, 0, 0, 0, 0, 0, 0, 0, 0}, 58, 155}, + {{236, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 15, 133}, + {{168, 243, 103, 221, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 38, 10}, + {{86, 194, 218, 188, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 33, 31}, + {{232, 3, 134, 67, 63, 196, 86, 14, 170, 243, 77, 134, 187, 140, 72, 18}, 127, 98}, + {{55, 253, 19, 201, 199, 71, 229, 218, 54, 64, 12, 162, 0, 0, 0, 0}, 96, 22}, + {{142, 34, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 20, 214}, + {{213, 16, 208, 50, 100, 33, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 50, 217}, + {{117, 237, 132, 185, 184, 246, 79, 42, 103, 98, 162, 243, 128, 0, 0, 0}, 98, 102}, + {{120, 25, 214, 222, 61, 157, 203, 102, 3, 146, 192, 0, 0, 0, 0, 0}, 83, 169}, + {{222, 46, 254, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 27, 152}, + {{254, 70, 158, 171, 11, 245, 223, 97, 70, 17, 27, 192, 186, 0, 0, 0}, 103, 214}, + {{192, 128, 228, 17, 68, 20, 44, 31, 52, 34, 212, 1, 224, 0, 0, 0}, 99, 178}, + {{237, 229, 203, 8, 121, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 45, 164}, + {{6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 7, 15}, + {{71, 197, 251, 122, 138, 232, 12, 241, 116, 240, 0, 0, 0, 0, 0, 0}, 76, 94}, + {{18, 241, 135, 210, 233, 54, 121, 185, 4, 0, 0, 0, 0, 0, 0, 0}, 70, 239}, + {{32, 50, 213, 63, 73, 217, 180, 21, 187, 128, 0, 0, 0, 0, 0, 0}, 73, 82}, + {{203, 166, 233, 73, 92, 182, 212, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 55, 54}, + {{56, 162, 126, 4, 18, 195, 192, 64, 164, 156, 119, 196, 64, 0, 0, 0}, 98, 47}, + {{120, 87, 81, 136, 180, 179, 68, 148, 243, 38, 80, 0, 0, 0, 0, 0}, 84, 214}, + {{64, 244, 193, 50, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 37, 215}, + {{91, 168, 253, 158, 131, 83, 159, 163, 113, 169, 112, 0, 0, 0, 0, 0}, 84, 153}, + {{159, 103, 102, 132, 111, 46, 18, 77, 36, 15, 137, 33, 177, 31, 243, 192}, 122, 245}, + {{123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 8, 118}, + {{67, 81, 226, 190, 7, 79, 71, 250, 155, 245, 44, 81, 215, 213, 171, 224}, 123, 128}, + {{103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 8, 7}, + {{246, 44, 168, 200, 198, 238, 52, 196, 125, 115, 0, 0, 0, 0, 0, 0}, 80, 152}, + {{205, 14, 186, 252, 239, 213, 59, 119, 105, 37, 140, 209, 4, 231, 0, 0}, 114, 248}, + {{70, 91, 254, 106, 94, 71, 170, 19, 158, 242, 192, 0, 0, 0, 0, 0}, 85, 143}, + {{250, 86, 233, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 30, 159}, + {{122, 222, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 24, 11}, + {{27, 224, 235, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 31, 110}, + {{239, 100, 224, 3, 46, 127, 150, 251, 204, 120, 228, 64, 0, 0, 0, 0}, 97, 181}, + {{144, 115, 182, 206, 146, 13, 21, 111, 37, 70, 179, 129, 173, 82, 93, 128}, 121, 4}, + {{73, 190, 57, 243, 49, 51, 15, 209, 0, 0, 0, 0, 0, 0, 0, 0}, 67, 101}, + {{18, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 11, 38}, + {{23, 37, 236, 177, 186, 7, 209, 135, 114, 44, 0, 0, 0, 0, 0, 0}, 78, 57}, + {{200, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 17, 142}, + {{181, 255, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 24, 184}, + {{135, 168, 6, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 27, 91}, + {{200, 224, 33, 245, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 41, 224}, + {{70, 111, 10, 62, 200, 224, 38, 204, 14, 164, 0, 0, 0, 0, 0, 0}, 78, 114}, + {{158, 133, 252, 18, 242, 12, 16, 60, 5, 52, 251, 179, 38, 235, 12, 0}, 118, 184}, + {{2, 23, 116, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 28, 215}, + {{33, 25, 170, 74, 215, 134, 151, 181, 175, 232, 20, 155, 189, 242, 13, 0}, 120, 167}, + {{160, 186, 218, 183, 167, 84, 59, 152, 13, 137, 80, 128, 0, 0, 0, 0}, 89, 233}, + {{32, 141, 196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 29, 101}, + {{207, 24, 202, 226, 191, 136, 78, 124, 160, 0, 0, 0, 0, 0, 0, 0}, 67, 139}, + {{210, 173, 172, 27, 197, 57, 114, 146, 169, 32, 0, 0, 0, 0, 0, 0}, 79, 32}, + {{95, 113, 12, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 32, 57}, + {{129, 108, 186, 28, 19, 229, 96, 134, 199, 254, 199, 64, 0, 0, 0, 0}, 91, 151}, + {{103, 226, 38, 123, 35, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 49, 0}, + {{41, 117, 43, 35, 208, 115, 73, 64, 0, 0, 0, 0, 0, 0, 0, 0}, 63, 227}, + {{42, 220, 61, 34, 199, 183, 42, 16, 223, 135, 0, 135, 213, 150, 100, 0}, 118, 124}, + {{165, 227, 96, 243, 112, 171, 117, 106, 50, 37, 82, 60, 80, 0, 0, 0}, 104, 228}, + {{158, 60, 111, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 27, 64}, + {{124, 108, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 25, 179}, + {{232, 68, 132, 159, 156, 103, 95, 190, 76, 0, 0, 0, 0, 0, 0, 0}, 70, 107}, + {{70, 77, 240, 209, 72, 63, 63, 45, 125, 79, 77, 41, 13, 0, 0, 0}, 104, 206}, + {{146, 254, 7, 5, 68, 240, 67, 237, 112, 0, 0, 0, 0, 0, 0, 0}, 68, 95}, + {{162, 223, 117, 27, 2, 156, 94, 170, 157, 114, 162, 50, 0, 0, 0, 0}, 96, 219}, + {{161, 62, 191, 68, 239, 73, 100, 37, 168, 254, 139, 202, 252, 65, 74, 0}, 119, 138}, + {{248, 122, 115, 81, 15, 158, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 53, 84}, + {{8, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 11, 161}, + {{142, 96, 105, 133, 251, 57, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 52, 25}, + {{138, 196, 139, 131, 233, 93, 65, 242, 86, 169, 7, 72, 82, 128, 0, 0}, 107, 113}, + {{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 2, 46}, + {{175, 151, 75, 238, 26, 12, 100, 186, 0, 0, 0, 0, 0, 0, 0, 0}, 63, 72}, + {{82, 205, 211, 176, 170, 79, 57, 153, 161, 218, 32, 48, 0, 0, 0, 0}, 93, 230}, + {{227, 123, 232, 74, 236, 202, 211, 121, 200, 8, 59, 189, 81, 219, 144, 0}, 117, 142}, + {{205, 196, 89, 90, 103, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 41, 134}, + {{63, 145, 23, 127, 102, 216, 49, 36, 168, 164, 59, 133, 18, 146, 0, 0}, 112, 100}, + {{213, 72, 154, 16, 230, 236, 218, 203, 223, 51, 31, 251, 103, 64, 0, 0}, 109, 45}, + {{126, 148, 232, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 30, 219}, + {{160, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 16, 52}, + {{137, 38, 146, 20, 99, 188, 83, 123, 159, 159, 64, 0, 0, 0, 0, 0}, 83, 240}, + {{123, 228, 36, 44, 242, 29, 51, 228, 140, 60, 237, 0, 0, 0, 0, 0}, 90, 13}, + {{163, 169, 25, 89, 190, 114, 165, 158, 140, 210, 192, 0, 0, 0, 0, 0}, 84, 191}, + {{225, 38, 70, 89, 218, 236, 60, 5, 69, 163, 248, 50, 163, 64, 0, 0}, 106, 95}, + {{91, 94, 36, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 28, 65}, + {{209, 238, 110, 0, 2, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 45, 195}, + {{57, 17, 224, 164, 69, 95, 138, 172, 111, 55, 239, 167, 160, 0, 0, 0}, 103, 21}, + {{112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 4, 114}, + {{102, 96, 223, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 31, 92}, + {{137, 204, 150, 75, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 42, 237}, + {{136, 56, 252, 240, 85, 48, 248, 231, 17, 49, 47, 238, 15, 233, 159, 184}, 125, 172}, + {{57, 31, 132, 123, 234, 255, 37, 82, 167, 204, 37, 158, 128, 0, 0, 0}, 98, 116}, + {{55, 198, 139, 219, 161, 156, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 55, 54}, + {{44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 8, 203}, + {{53, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 16, 74}, + {{227, 62, 107, 236, 118, 156, 60, 34, 31, 179, 76, 221, 0, 0, 0, 0}, 96, 220}, + {{105, 40, 240, 216, 91, 61, 19, 128, 224, 0, 0, 0, 0, 0, 0, 0}, 67, 219}, + {{96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 3, 179}, + {{118, 142, 251, 249, 128, 105, 113, 16, 0, 0, 0, 0, 0, 0, 0, 0}, 61, 194}, + {{101, 70, 196, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 32, 187}, + {{245, 173, 165, 177, 200, 161, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 50, 79}, + {{0, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 19, 87}, + {{92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 8, 126}, + {{125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 11, 106}, + {{56, 59, 35, 82, 101, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 50, 96}, + {{184, 72, 77, 251, 8, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 47, 45}, + {{143, 74, 132, 205, 218, 247, 30, 160, 145, 199, 138, 12, 89, 220, 0, 0}, 110, 8}, + {{30, 178, 111, 225, 73, 79, 173, 52, 0, 0, 0, 0, 0, 0, 0, 0}, 62, 226}, + {{224, 48, 154, 231, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 36, 222}, + {{123, 144, 170, 143, 85, 169, 130, 245, 214, 0, 0, 0, 0, 0, 0, 0}, 71, 218}, + {{166, 224, 212, 100, 149, 55, 35, 210, 246, 108, 41, 245, 127, 174, 128, 0}, 116, 59}, + {{75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 8, 80}, + {{197, 128, 190, 87, 47, 53, 92, 64, 0, 0, 0, 0, 0, 0, 0, 0}, 58, 177}, + {{249, 10, 76, 217, 225, 20, 124, 205, 44, 159, 190, 8, 0, 0, 0, 0}, 98, 44}, + {{180, 226, 0, 167, 137, 232, 174, 120, 113, 95, 22, 184, 0, 0, 0, 0}, 93, 206}, + {{123, 153, 102, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 27, 64}, + {{5, 144, 206, 158, 239, 189, 171, 120, 69, 46, 128, 237, 0, 0, 0, 0}, 96, 236}, + {{159, 235, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 22, 101}, + {{42, 194, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 26, 49}, + {{205, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 11, 179}, + {{19, 65, 141, 20, 127, 77, 70, 205, 151, 115, 157, 23, 118, 128, 0, 0}, 109, 112}, + {{96, 11, 214, 40, 245, 251, 61, 64, 128, 241, 183, 183, 0, 0, 0, 0}, 96, 31}, + {{120, 4, 235, 112, 34, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 41, 111}, + {{110, 127, 207, 76, 100, 148, 130, 206, 249, 2, 104, 0, 0, 0, 0, 0}, 86, 65}, + {{226, 190, 191, 249, 173, 96, 127, 200, 62, 20, 0, 0, 0, 0, 0, 0}, 78, 222}, + {{89, 88, 182, 14, 78, 122, 213, 192, 0, 0, 0, 0, 0, 0, 0, 0}, 58, 4}, + {{167, 94, 163, 227, 28, 111, 117, 103, 224, 0, 0, 0, 0, 0, 0, 0}, 67, 67}, + {{57, 220, 53, 116, 243, 184, 242, 134, 16, 70, 83, 61, 161, 128, 0, 0}, 109, 197}, + {{63, 235, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 22, 121}, + {{128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 1, 167}, + {{15, 159, 42, 167, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 38, 140}, + {{216, 252, 113, 40, 239, 46, 172, 48, 103, 250, 82, 179, 136, 64, 0, 0}, 106, 193}, + {{158, 147, 16, 44, 124, 56, 44, 48, 138, 64, 169, 0, 0, 0, 0, 0}, 90, 47}, + {{238, 238, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 21, 187}, + {{63, 159, 177, 162, 106, 212, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 56, 102}, + {{59, 40, 252, 185, 187, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 46, 237}, + {{2, 218, 11, 68, 173, 196, 16, 223, 2, 18, 122, 215, 154, 0, 0, 0}, 103, 237}, + {{3, 9, 206, 73, 108, 196, 183, 119, 141, 162, 10, 180, 115, 32, 0, 0}, 107, 115}, + {{17, 227, 208, 146, 63, 201, 73, 239, 29, 79, 80, 0, 0, 0, 0, 0}, 84, 217}, + {{115, 180, 176, 241, 52, 209, 6, 64, 189, 76, 0, 0, 0, 0, 0, 0}, 79, 21}, + {{191, 88, 98, 245, 91, 46, 137, 254, 170, 80, 11, 55, 212, 28, 128, 0}, 113, 3}, + {{97, 141, 171, 175, 22, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 48, 62}, + {{32, 204, 102, 191, 164, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 47, 80}, + {{29, 133, 210, 252, 124, 66, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 51, 184}, + {{207, 179, 54, 144, 116, 67, 29, 64, 13, 199, 0, 0, 0, 0, 0, 0}, 80, 197}, + {{129, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 13, 63}, + {{50, 152, 249, 143, 174, 234, 240, 48, 158, 255, 80, 105, 0, 0, 0, 0}, 99, 62}, + {{105, 208, 95, 218, 44, 11, 87, 134, 109, 18, 138, 66, 17, 69, 128, 0}, 114, 231}, + {{151, 79, 158, 220, 122, 101, 210, 164, 64, 0, 0, 0, 0, 0, 0, 0}, 67, 158}, + {{236, 97, 87, 155, 254, 137, 122, 208, 168, 201, 194, 118, 224, 0, 0, 0}, 101, 118}, + {{14, 229, 193, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 30, 237}, + {{46, 154, 50, 80, 92, 147, 158, 86, 1, 112, 0, 0, 0, 0, 0, 0}, 79, 15}, + {{88, 131, 21, 84, 62, 86, 7, 110, 142, 251, 242, 110, 194, 175, 247, 0}, 122, 84}, + {{229, 216, 111, 92, 173, 32, 63, 70, 36, 84, 6, 74, 136, 166, 38, 0}, 119, 205}, + {{121, 147, 216, 245, 37, 189, 146, 63, 145, 74, 128, 0, 0, 0, 0, 0}, 82, 220}, + {{44, 26, 254, 11, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 40, 42}, + {{209, 114, 97, 249, 227, 159, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 51, 144}, + {{184, 244, 43, 117, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 37, 74}, + {{60, 81, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 19, 89}, + {{18, 40, 21, 113, 226, 91, 195, 88, 161, 19, 142, 0, 0, 0, 0, 0}, 88, 77}, + {{57, 0, 212, 158, 56, 51, 108, 198, 59, 5, 137, 196, 0, 0, 0, 0}, 94, 2}, + {{168, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 14, 75}, + {{64, 181, 254, 103, 1, 230, 117, 199, 128, 0, 0, 0, 0, 0, 0, 0}, 65, 18}, + {{212, 48, 214, 127, 78, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 46, 246}, + {{155, 185, 236, 163, 204, 49, 129, 120, 183, 47, 10, 243, 65, 92, 192, 0}, 114, 10}, + {{94, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 14, 207}, + {{19, 210, 136, 113, 73, 79, 132, 196, 224, 0, 0, 0, 0, 0, 0, 0}, 68, 41}, + {{24, 203, 246, 242, 241, 223, 150, 237, 213, 202, 11, 128, 0, 0, 0, 0}, 89, 102}, + {{115, 59, 171, 221, 172, 181, 170, 67, 115, 205, 44, 107, 162, 67, 56, 0}, 118, 118}, + {{250, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 10, 146}, + {{203, 240, 28, 158, 182, 12, 86, 182, 142, 47, 143, 57, 239, 0, 0, 0}, 104, 122}, + {{196, 218, 109, 52, 2, 0, 64, 153, 34, 250, 240, 185, 117, 0, 0, 0}, 107, 6}, + {{137, 131, 191, 40, 72, 209, 74, 64, 0, 0, 0, 0, 0, 0, 0, 0}, 58, 18}, + {{236, 126, 167, 37, 185, 20, 34, 207, 76, 0, 0, 0, 0, 0, 0, 0}, 70, 83}, + {{129, 192, 245, 137, 251, 52, 75, 68, 81, 112, 146, 133, 64, 0, 0, 0}, 99, 90}, + {{7, 31, 148, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 32, 140}, + {{112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 4, 242}, + {{167, 50, 202, 179, 74, 146, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 50, 31}, + {{44, 188, 186, 250, 229, 71, 28, 118, 35, 253, 245, 191, 199, 18, 0, 0}, 111, 9}, + {{128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 1, 230}, + {{156, 163, 215, 175, 71, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 47, 50}, + {{67, 24, 151, 198, 242, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 41, 34}, + {{134, 107, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 17, 11}, + {{35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 8, 71}, + {{46, 196, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 22, 146}, + {{82, 172, 8, 26, 154, 34, 125, 188, 5, 149, 159, 44, 78, 222, 236, 176}, 124, 249}, + {{78, 157, 79, 70, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 39, 143}, + {{231, 5, 210, 247, 198, 5, 157, 191, 206, 225, 149, 142, 207, 40, 0, 0}, 110, 17}, + {{38, 254, 235, 199, 191, 60, 43, 159, 190, 243, 203, 185, 184, 218, 132, 0}, 119, 60}, + {{128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 1, 162}, + {{95, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 12, 5}, + {{17, 128, 244, 178, 160, 78, 83, 92, 0, 0, 0, 0, 0, 0, 0, 0}, 62, 139}, + {{18, 102, 62, 251, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 39, 8}, + {{30, 75, 108, 40, 231, 166, 233, 220, 163, 176, 252, 210, 60, 30, 128, 0}, 114, 246}, + {{18, 3, 207, 64, 25, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 42, 171}, + {{52, 83, 235, 61, 164, 236, 83, 173, 143, 105, 14, 0, 0, 0, 0, 0}, 88, 206}, + {{166, 175, 186, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 28, 163}, + {{221, 154, 82, 98, 41, 126, 85, 52, 0, 0, 0, 0, 0, 0, 0, 0}, 62, 166}, + {{94, 84, 182, 120, 204, 232, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 51, 128}, + {{27, 174, 227, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 31, 59}, + {{218, 12, 4, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 32, 179}, + {{9, 5, 190, 195, 60, 216, 80, 150, 128, 117, 86, 128, 128, 112, 98, 208}, 124, 87}, + {{7, 226, 104, 112, 212, 9, 172, 124, 209, 121, 170, 229, 44, 178, 128, 0}, 114, 29}, + {{47, 71, 174, 76, 52, 83, 23, 18, 106, 48, 56, 32, 0, 0, 0, 0}, 91, 184}, + {{51, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 14, 45}, + {{28, 182, 167, 124, 28, 22, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 55, 144}, + {{34, 61, 14, 51, 253, 17, 19, 170, 49, 206, 188, 207, 247, 167, 192, 0}, 114, 119}, + {{2, 235, 18, 14, 195, 66, 237, 30, 0, 0, 0, 0, 0, 0, 0, 0}, 64, 113}, + {{51, 182, 142, 133, 127, 96, 159, 132, 99, 161, 64, 0, 0, 0, 0, 0}, 82, 50}, + {{170, 145, 230, 123, 215, 189, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 56, 207}, + {{151, 166, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 21, 3}, + {{16, 141, 196, 129, 132, 207, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 58, 13}, + {{205, 25, 184, 191, 201, 206, 109, 224, 0, 0, 0, 0, 0, 0, 0, 0}, 59, 42}, + {{48, 114, 33, 103, 247, 255, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 50, 31}, + {{179, 156, 119, 146, 125, 21, 42, 146, 237, 213, 191, 132, 0, 0, 0, 0}, 94, 30}, + {{179, 129, 186, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 29, 94}, + {{17, 179, 217, 188, 128, 212, 4, 4, 152, 0, 0, 0, 0, 0, 0, 0}, 71, 190}, + {{132, 63, 74, 89, 209, 64, 63, 192, 0, 0, 0, 0, 0, 0, 0, 0}, 59, 238}, + {{16, 50, 248, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 30, 20}, + {{189, 96, 58, 53, 191, 235, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 51, 84}, + {{111, 98, 6, 65, 35, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 42, 108}, + {{118, 223, 83, 220, 110, 122, 23, 112, 185, 155, 73, 0, 0, 0, 0, 0}, 89, 136}, + {{173, 191, 150, 197, 204, 35, 169, 79, 31, 214, 251, 240, 0, 0, 0, 0}, 93, 196}, + {{26, 76, 129, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 28, 67}, + {{231, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 12, 104}, + {{93, 172, 223, 252, 203, 0, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 55, 15}, + {{53, 142, 203, 124, 104, 51, 241, 12, 161, 17, 101, 245, 120, 110, 192, 199}, 128, 237}, + {{9, 77, 120, 197, 193, 10, 237, 174, 233, 2, 165, 11, 229, 47, 144, 0}, 116, 224}, + {{99, 161, 189, 88, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 36, 179}, + {{18, 8, 76, 66, 2, 185, 206, 132, 224, 0, 0, 0, 0, 0, 0, 0}, 67, 84}, + {{169, 53, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 22, 65}, + {{136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 5, 178}, + {{131, 162, 144, 124, 12, 98, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 55, 154}, + {{75, 50, 129, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 27, 106}, + {{212, 183, 40, 225, 152, 136, 174, 91, 0, 0, 0, 0, 0, 0, 0, 0}, 67, 125}, + {{158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 9, 118}, + {{7, 48, 132, 149, 169, 212, 198, 137, 202, 0, 0, 0, 0, 0, 0, 0}, 73, 52}, + {{173, 195, 129, 163, 141, 249, 40, 64, 0, 0, 0, 0, 0, 0, 0, 0}, 58, 173}, + {{109, 79, 75, 219, 205, 182, 22, 245, 223, 17, 146, 78, 109, 119, 128, 0}, 113, 8}, + {{174, 195, 24, 182, 215, 198, 214, 86, 34, 128, 0, 0, 0, 0, 0, 0}, 74, 211}, + {{22, 40, 51, 109, 70, 91, 152, 56, 0, 0, 0, 0, 0, 0, 0, 0}, 61, 253}, + {{169, 115, 246, 126, 65, 118, 219, 192, 0, 0, 0, 0, 0, 0, 0, 0}, 59, 47}, + {{154, 37, 70, 124, 107, 123, 232, 241, 164, 142, 71, 226, 182, 126, 0, 0}, 112, 73}, + {{6, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 19, 192}, + {{216, 167, 158, 158, 222, 19, 96, 28, 40, 6, 70, 12, 147, 27, 85, 240}, 128, 55}, + {{72, 222, 52, 69, 69, 206, 163, 106, 235, 206, 80, 128, 0, 0, 0, 0}, 94, 147}, + {{150, 112, 106, 56, 15, 243, 154, 97, 134, 110, 160, 20, 183, 144, 234, 8}, 125, 86}, + {{58, 186, 106, 58, 124, 171, 53, 85, 33, 100, 64, 0, 0, 0, 0, 0}, 82, 16}, + {{7, 195, 22, 31, 62, 217, 209, 46, 90, 49, 189, 50, 168, 126, 0, 0}, 111, 167}, + {{92, 44, 159, 198, 185, 94, 231, 177, 64, 0, 0, 0, 0, 0, 0, 0}, 67, 148}, + {{169, 108, 190, 162, 23, 39, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 54, 66}, + {{161, 5, 3, 11, 158, 157, 166, 212, 246, 22, 140, 101, 92, 0, 0, 0}, 104, 70}, + {{71, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 16, 166}, + {{48, 136, 194, 145, 57, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 44, 109}, + {{144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 5, 226}, + {{223, 209, 10, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 32, 8}, + {{154, 79, 170, 9, 43, 139, 249, 176, 186, 72, 216, 0, 0, 0, 0, 0}, 85, 218}, + {{1, 8, 123, 205, 167, 134, 128, 102, 10, 72, 0, 0, 0, 0, 0, 0}, 78, 54}, + {{31, 105, 48, 77, 103, 187, 99, 67, 96, 0, 0, 0, 0, 0, 0, 0}, 67, 48}, + {{14, 73, 54, 76, 232, 35, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 51, 244}, + {{14, 109, 251, 190, 36, 253, 99, 120, 94, 64, 0, 0, 0, 0, 0, 0}, 74, 50}, + {{122, 170, 9, 134, 124, 91, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 54, 173}, + {{246, 10, 85, 88, 82, 217, 95, 56, 216, 203, 160, 0, 0, 0, 0, 0}, 84, 245}, + {{77, 100, 114, 207, 150, 177, 69, 134, 74, 131, 147, 117, 177, 64, 210, 128}, 121, 54}, + {{171, 123, 22, 138, 132, 229, 250, 81, 186, 227, 146, 27, 170, 205, 128, 0}, 113, 86}, + {{128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 2, 115}, + {{12, 35, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 24, 144}, + {{255, 124, 179, 165, 169, 250, 66, 171, 223, 125, 247, 0, 0, 0, 0, 0}, 89, 171}, + {{244, 235, 211, 10, 251, 255, 206, 6, 198, 12, 50, 136, 0, 0, 0, 0}, 93, 231}, + {{221, 77, 237, 41, 50, 33, 103, 24, 25, 127, 208, 0, 0, 0, 0, 0}, 88, 34}, + {{216, 69, 47, 53, 117, 24, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 53, 225}, + {{180, 87, 25, 236, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 38, 174}, + {{110, 32, 24, 34, 116, 133, 245, 128, 123, 95, 125, 122, 100, 129, 128, 0}, 113, 37}, + {{27, 117, 179, 112, 133, 137, 110, 193, 246, 201, 219, 65, 56, 234, 106, 128}, 121, 39}, + {{186, 117, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 23, 59}, + {{243, 119, 54, 16, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 34, 96}, + {{160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 3, 147}, + {{78, 48, 117, 200, 245, 118, 115, 240, 170, 125, 84, 103, 33, 168, 0, 0}, 110, 56}, + {{201, 253, 184, 254, 143, 81, 95, 42, 243, 147, 96, 145, 23, 26, 0, 0}, 111, 234}, + {{41, 215, 84, 136, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 46, 199}, + {{91, 244, 137, 184, 231, 95, 135, 10, 184, 0, 0, 0, 0, 0, 0, 0}, 69, 191}, + {{113, 31, 181, 245, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 40, 235}, + {{181, 216, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 20, 45}, + {{87, 26, 119, 229, 97, 255, 9, 43, 32, 0, 0, 0, 0, 0, 0, 0}, 67, 164}, + {{205, 112, 67, 163, 196, 148, 5, 105, 8, 138, 144, 3, 171, 213, 159, 128}, 121, 130}, + {{136, 27, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 18, 166}, + {{2, 175, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 24, 140}, + {{222, 131, 85, 218, 16, 229, 44, 230, 243, 76, 250, 139, 1, 203, 108, 0}, 118, 47}, + {{101, 180, 77, 142, 194, 73, 196, 246, 107, 100, 194, 72, 204, 124, 0, 0}, 111, 148}, + {{96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 3, 103}, + {{46, 62, 191, 130, 110, 128, 235, 62, 68, 39, 58, 152, 207, 204, 96, 0}, 116, 94}, + {{111, 11, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 19, 85}, + {{58, 43, 14, 93, 102, 210, 117, 208, 222, 171, 130, 41, 16, 16, 0, 0}, 109, 250}, + {{141, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 16, 153}, + {{170, 153, 160, 170, 144, 235, 122, 8, 106, 34, 24, 32, 102, 57, 12, 168}, 125, 182}, + {{34, 113, 163, 107, 61, 177, 39, 172, 242, 2, 130, 0, 0, 0, 0, 0}, 94, 23}, + {{222, 191, 239, 110, 162, 191, 195, 181, 80, 50, 85, 240, 88, 32, 0, 0}, 108, 38}, + {{179, 82, 253, 151, 212, 0, 72, 253, 175, 22, 34, 78, 53, 32, 0, 0}, 110, 121}, + {{10, 162, 20, 46, 164, 64, 88, 1, 202, 204, 124, 0, 0, 0, 0, 0}, 87, 146}, + {{210, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 16, 138}, + {{183, 200, 1, 2, 51, 6, 66, 142, 20, 77, 48, 244, 0, 0, 0, 0}, 94, 149}, + {{29, 20, 224, 57, 204, 161, 131, 254, 53, 133, 163, 0, 0, 0, 0, 0}, 88, 232}, + {{75, 58, 170, 52, 146, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 45, 255}, + {{92, 21, 1, 113, 185, 88, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 51, 148}, + {{103, 180, 222, 187, 129, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 48, 117}, + {{32, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 14, 237}, + {{7, 60, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 19, 113}, + {{167, 122, 205, 185, 21, 199, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 57, 162}, + {{21, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 21, 225}, + {{92, 159, 167, 169, 136, 176, 95, 255, 87, 137, 112, 16, 0, 0, 0, 0}, 92, 210}, + {{84, 120, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 22, 34}, + {{126, 5, 126, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 29, 224}, + {{4, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 16, 143}, + {{239, 154, 181, 182, 189, 211, 244, 53, 144, 0, 0, 0, 0, 0, 0, 0}, 68, 216}, + {{254, 188, 139, 167, 135, 47, 147, 239, 187, 106, 228, 156, 234, 234, 102, 0}, 120, 239}, + {{225, 168, 138, 92, 193, 255, 47, 233, 11, 154, 205, 86, 209, 88, 0, 0}, 111, 54}, + {{223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 9, 35}, + {{235, 252, 115, 10, 151, 104, 193, 207, 38, 228, 229, 245, 42, 13, 108, 0}, 119, 230}, + {{1, 137, 53, 36, 210, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 45, 234}, + {{149, 182, 72, 197, 92, 229, 9, 10, 220, 128, 72, 19, 4, 58, 192, 0}, 115, 70}, + {{105, 73, 57, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 32, 246}, + {{189, 61, 230, 24, 235, 82, 58, 102, 97, 111, 121, 252, 156, 94, 191, 166}, 127, 217}, + {{193, 108, 231, 86, 140, 14, 192, 4, 135, 80, 129, 166, 158, 61, 230, 20}, 128, 201}, + {{110, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 17, 49}, + {{3, 102, 36, 231, 15, 242, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 57, 2}, + {{81, 189, 220, 168, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 38, 64}, + {{168, 75, 133, 180, 91, 165, 77, 232, 0, 0, 0, 0, 0, 0, 0, 0}, 62, 239}, + {{106, 179, 186, 109, 81, 234, 233, 167, 101, 160, 90, 102, 174, 234, 208, 0}, 116, 47}, + {{46, 105, 234, 21, 23, 247, 169, 33, 47, 5, 0, 0, 0, 0, 0, 0}, 80, 43}, + {{152, 144, 100, 142, 129, 23, 227, 50, 67, 81, 249, 116, 0, 0, 0, 0}, 94, 17}, + {{109, 74, 145, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 26, 5}, + {{100, 243, 22, 230, 38, 44, 128, 86, 132, 57, 0, 0, 0, 0, 0, 0}, 81, 240}, + {{153, 251, 115, 65, 104, 179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 50, 197}, + {{43, 113, 60, 224, 36, 20, 42, 161, 24, 223, 192, 0, 0, 0, 0, 0}, 84, 192}, + {{61, 77, 121, 176, 138, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 47, 160}, + {{119, 194, 146, 49, 59, 242, 25, 220, 122, 104, 80, 0, 0, 0, 0, 0}, 84, 199}, + {{254, 162, 155, 47, 187, 3, 1, 114, 142, 191, 152, 44, 144, 26, 202, 0}, 127, 217}, + {{176, 1, 114, 42, 191, 145, 43, 1, 141, 18, 64, 0, 0, 0, 0, 0}, 83, 75}, + {{170, 244, 67, 132, 145, 163, 76, 213, 85, 237, 248, 22, 207, 64, 0, 0}, 106, 222}, + {{102, 190, 58, 32, 75, 15, 89, 163, 64, 7, 168, 0, 0, 0, 0, 0}, 85, 39}, + {{124, 170, 35, 47, 152, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 45, 9}, + {{192, 221, 20, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 33, 217}, + {{208, 178, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 25, 142}, + {{188, 68, 77, 30, 68, 153, 102, 180, 0, 0, 0, 0, 0, 0, 0, 0}, 62, 18}, + {{114, 178, 121, 188, 205, 233, 35, 77, 34, 197, 158, 174, 101, 0, 0, 0}, 104, 180}, + {{195, 98, 67, 12, 13, 43, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 56, 205}, + {{146, 190, 42, 222, 14, 54, 28, 128, 0, 0, 0, 0, 0, 0, 0, 0}, 57, 251}, + {{185, 202, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 22, 178}, + {{138, 30, 129, 95, 224, 161, 120, 128, 0, 0, 0, 0, 0, 0, 0, 0}, 59, 198}, + {{69, 181, 5, 227, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 34, 84}, + {{90, 180, 0, 164, 227, 75, 174, 119, 128, 0, 0, 0, 0, 0, 0, 0}, 66, 128}, + {{20, 60, 58, 119, 245, 177, 162, 186, 13, 112, 211, 239, 128, 0, 0, 0}, 97, 75}, + {{158, 124, 157, 25, 230, 139, 51, 212, 76, 109, 236, 210, 48, 0, 0, 0}, 101, 192}, + {{125, 108, 242, 36, 94, 13, 36, 106, 90, 51, 83, 217, 131, 151, 0, 0}, 114, 60}, + {{222, 218, 162, 158, 15, 53, 191, 178, 0, 0, 0, 0, 0, 0, 0, 0}, 63, 169}, + {{104, 202, 127, 109, 73, 16, 17, 12, 0, 0, 0, 0, 0, 0, 0, 0}, 62, 10}, + {{172, 171, 246, 26, 176, 34, 22, 152, 246, 56, 173, 120, 105, 60, 92, 0}, 118, 64}, + {{190, 22, 171, 206, 109, 186, 179, 128, 253, 182, 108, 212, 220, 167, 171, 180}, 127, 182}, + {{119, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 13, 29}, + {{160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 3, 39}, + {{170, 144, 64, 2, 107, 166, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 51, 93}, + {{234, 9, 96, 20, 156, 157, 1, 34, 88, 0, 0, 0, 0, 0, 0, 0}, 75, 228}, + {{147, 237, 16, 120, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 34, 236}, + {{182, 189, 162, 158, 223, 90, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 56, 190}, + {{116, 148, 142, 240, 10, 253, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 49, 217}, + {{211, 73, 140, 69, 252, 27, 75, 46, 37, 6, 147, 32, 0, 0, 0, 0}, 93, 74}, + {{148, 61, 120, 49, 220, 65, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 61, 180}, + {{172, 35, 202, 180, 129, 75, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 55, 91}, + {{215, 109, 147, 157, 32, 28, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 54, 230}, + {{151, 26, 182, 112, 205, 220, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 52, 175}, + {{73, 91, 93, 61, 196, 3, 66, 26, 149, 96, 0, 0, 0, 0, 0, 0}, 75, 171}, + {{203, 163, 52, 247, 28, 119, 56, 223, 138, 70, 174, 97, 77, 59, 46, 0}, 120, 202}, + {{251, 50, 228, 178, 202, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 49, 113}, + {{217, 159, 164, 199, 14, 237, 170, 184, 100, 231, 92, 222, 0, 0, 0, 0}, 96, 187}, + {{16, 161, 85, 193, 202, 21, 3, 155, 63, 116, 124, 203, 34, 13, 215, 0}, 120, 38}, + {{111, 52, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 26, 35}, + {{69, 12, 116, 151, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 38, 115}, + {{187, 60, 97, 40, 112, 101, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 51, 18}, + {{230, 194, 136, 255, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 39, 34}, + {{179, 239, 170, 107, 3, 13, 212, 67, 177, 69, 8, 0, 0, 0, 0, 0}, 87, 75}, + {{11, 58, 130, 89, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 33, 232}, + {{217, 178, 43, 203, 234, 20, 234, 186, 157, 88, 146, 192, 0, 0, 0, 0}, 91, 154}, + {{6, 180, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 19, 195}, + {{157, 154, 218, 158, 39, 224, 103, 230, 164, 0, 0, 0, 0, 0, 0, 0}, 70, 122}, + {{225, 10, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 24, 97}, + {{16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 5, 220}, + {{166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 7, 80}, + {{29, 190, 131, 215, 232, 246, 41, 226, 52, 192, 0, 0, 0, 0, 0, 0}, 77, 133}, + {{138, 74, 163, 93, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 38, 93}, + {{229, 64, 97, 41, 28, 243, 249, 185, 97, 35, 49, 27, 175, 24, 0, 0}, 110, 176}, + {{6, 73, 94, 160, 186, 216, 84, 117, 233, 169, 146, 234, 0, 0, 0, 0}, 95, 68}, + {{163, 40, 242, 81, 224, 35, 72, 194, 176, 78, 224, 174, 12, 0, 0, 0}, 103, 247}, + {{2, 205, 40, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 29, 240}, + {{174, 225, 240, 160, 212, 8, 246, 67, 36, 0, 0, 0, 0, 0, 0, 0}, 74, 83}, + {{5, 117, 182, 141, 166, 249, 196, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 55, 132}, + {{46, 152, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 24, 217}, + {{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 3, 214}, + {{233, 202, 159, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 34, 193}, + {{172, 54, 159, 5, 14, 245, 106, 182, 2, 0, 0, 0, 0, 0, 0, 0}, 71, 61}, + {{241, 222, 251, 114, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 39, 65}, + {{31, 243, 190, 4, 207, 198, 249, 59, 167, 127, 93, 64, 0, 0, 0, 0}, 91, 108}, + {{201, 35, 222, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 28, 244}, + {{187, 105, 13, 114, 238, 197, 145, 23, 169, 116, 91, 28, 0, 0, 0, 0}, 95, 194}, + {{251, 251, 121, 168, 152, 178, 147, 188, 229, 123, 154, 242, 190, 165, 173, 48}, 124, 82}, + {{66, 187, 191, 164, 31, 196, 40, 186, 148, 115, 134, 57, 222, 254, 48, 0}, 116, 45}, + {{209, 17, 111, 41, 154, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 48, 224}, + {{40, 245, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 22, 17}, + {{72, 121, 151, 83, 170, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 44, 133}, + {{171, 172, 101, 238, 201, 148, 23, 81, 4, 11, 64, 0, 0, 0, 0, 0}, 85, 125}, + {{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 3, 42}, + {{20, 46, 27, 93, 195, 184, 6, 162, 109, 225, 22, 152, 0, 0, 0, 0}, 96, 140}, + {{243, 122, 30, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 28, 91}, + {{89, 250, 80, 72, 148, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 48, 92}, + {{187, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 13, 125}, + {{172, 160, 143, 114, 128, 239, 174, 133, 176, 154, 159, 134, 10, 0, 0, 0}, 106, 249}, + {{254, 202, 113, 112, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 40, 202}, + {{80, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 13, 107}, + {{222, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 13, 124}, + {{219, 138, 253, 12, 188, 197, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 53, 57}, + {{124, 41, 173, 8, 202, 192, 61, 254, 174, 48, 239, 112, 0, 0, 0, 0}, 92, 181}, + {{195, 236, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 24, 107}, + {{83, 82, 42, 244, 136, 191, 197, 81, 91, 154, 216, 85, 29, 150, 198, 22}, 128, 101}, + {{112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 4, 102}, + {{44, 30, 219, 248, 214, 88, 225, 132, 0, 0, 0, 0, 0, 0, 0, 0}, 62, 136}, + {{41, 171, 206, 178, 195, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 45, 114}, + {{159, 15, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 22, 215}, + {{42, 188, 37, 174, 86, 40, 4, 84, 174, 216, 0, 0, 0, 0, 0, 0}, 79, 249}, + {{185, 227, 85, 177, 219, 95, 250, 227, 69, 154, 118, 0, 0, 0, 0, 0}, 88, 29}, + {{22, 185, 238, 100, 25, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 44, 71}, + {{122, 149, 117, 77, 88, 250, 187, 203, 136, 22, 85, 42, 105, 234, 79, 8}, 127, 112}, + {{93, 152, 229, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 31, 72}, + {{129, 37, 165, 167, 241, 24, 37, 40, 2, 128, 0, 0, 0, 0, 0, 0}, 73, 155}, + {{30, 202, 177, 3, 253, 202, 164, 248, 0, 0, 0, 0, 0, 0, 0, 0}, 61, 66}, + {{176, 25, 220, 120, 194, 228, 10, 45, 225, 142, 192, 96, 0, 0, 0, 0}, 91, 77}, + {{96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 3, 109}, + {{82, 56, 12, 204, 61, 45, 147, 240, 221, 0, 0, 0, 0, 0, 0, 0}, 72, 37}, + {{242, 38, 240, 41, 140, 75, 250, 37, 175, 115, 97, 224, 0, 0, 0, 0}, 91, 56}, + {{251, 192, 23, 90, 135, 56, 252, 56, 79, 219, 80, 167, 22, 0, 0, 0}, 103, 5}, + {{62, 128, 139, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 25, 15}, + {{214, 1, 84, 232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 29, 183}, + {{207, 90, 237, 137, 171, 140, 227, 88, 250, 26, 197, 162, 163, 0, 0, 0}, 105, 171}, + {{196, 151, 235, 232, 114, 248, 1, 207, 193, 184, 186, 71, 157, 0, 0, 0}, 112, 202}, + {{152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 5, 136}, + {{9, 174, 211, 200, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 37, 107}, + {{89, 150, 95, 28, 209, 13, 125, 159, 254, 244, 110, 0, 0, 0, 0, 0}, 87, 193}, + {{23, 28, 202, 10, 90, 158, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 52, 4}, + {{48, 25, 180, 9, 84, 236, 6, 144, 30, 198, 41, 56, 0, 0, 0, 0}, 96, 68}, + {{252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 7, 40}, + {{20, 165, 57, 130, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 39, 255}, + {{167, 56, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 18, 108}, + {{91, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 14, 219}, + {{24, 46, 9, 4, 170, 150, 56, 130, 127, 120, 118, 104, 168, 48, 0, 0}, 108, 12}, + {{156, 60, 245, 247, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 40, 84}, + {{148, 104, 187, 174, 129, 28, 127, 162, 92, 222, 52, 18, 0, 0, 0, 0}, 96, 33}, + {{38, 253, 182, 153, 233, 194, 159, 41, 94, 193, 254, 160, 0, 0, 0, 0}, 91, 199}, + {{156, 77, 105, 235, 145, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 46, 52}, + {{100, 211, 238, 147, 65, 222, 99, 73, 252, 113, 46, 113, 52, 136, 0, 0}, 113, 184}, + {{13, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 18, 124}, + {{29, 240, 141, 230, 78, 237, 25, 135, 131, 6, 65, 77, 77, 248, 0, 0}, 109, 128}, + {{15, 192, 109, 31, 149, 221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 49, 255}, + {{80, 185, 170, 71, 41, 58, 158, 106, 253, 7, 2, 184, 173, 0, 0, 0}, 105, 146}, + {{16, 229, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 24, 172}, + {{169, 2, 153, 9, 169, 203, 245, 154, 184, 0, 0, 0, 0, 0, 0, 0}, 70, 116}, + {{144, 135, 239, 164, 142, 187, 64, 109, 0, 0, 0, 0, 0, 0, 0, 0}, 66, 189}, + {{170, 78, 252, 227, 242, 199, 130, 251, 200, 0, 0, 0, 0, 0, 0, 0}, 70, 10}, + {{232, 18, 15, 126, 166, 126, 58, 25, 209, 62, 76, 79, 0, 0, 0, 0}, 98, 184}, + {{170, 82, 72, 53, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 33, 98}, + {{152, 100, 37, 122, 242, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 42, 37}, + {{174, 231, 230, 33, 71, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 46, 174}, + {{74, 225, 252, 153, 202, 8, 162, 39, 64, 0, 0, 0, 0, 0, 0, 0}, 67, 251}, + {{167, 186, 101, 187, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 42, 115}, + {{83, 7, 21, 122, 243, 67, 171, 146, 145, 160, 168, 103, 223, 64, 0, 0}, 107, 252}, + {{83, 132, 219, 86, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 40, 176}, + {{22, 113, 72, 102, 73, 16, 236, 57, 197, 122, 31, 0, 0, 0, 0, 0}, 91, 155}, + {{250, 59, 64, 35, 72, 112, 159, 85, 200, 5, 193, 39, 152, 185, 148, 16}, 124, 36}, + {{220, 21, 48, 164, 224, 121, 17, 69, 10, 118, 106, 0, 0, 0, 0, 0}, 88, 202}, + {{160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 5, 208}, + {{247, 64, 83, 125, 195, 225, 50, 76, 18, 104, 0, 0, 0, 0, 0, 0}, 77, 158}, + {{78, 91, 31, 202, 189, 25, 13, 133, 220, 0, 0, 0, 0, 0, 0, 0}, 72, 136}, + {{105, 197, 26, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 28, 191}, + {{14, 31, 154, 242, 241, 231, 55, 151, 223, 56, 134, 255, 113, 206, 69, 0}, 120, 126}, + {{247, 193, 58, 176, 16, 71, 31, 120, 213, 104, 231, 83, 26, 118, 91, 135}, 128, 139}, + {{136, 32, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 25, 216}, + {{100, 238, 112, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 29, 93}, + {{80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 4, 196}, + {{233, 224, 254, 57, 33, 205, 140, 217, 181, 72, 0, 0, 0, 0, 0, 0}, 81, 119}, + {{107, 75, 65, 158, 128, 142, 191, 188, 188, 240, 148, 243, 116, 0, 0, 0}, 104, 93}, + {{39, 70, 120, 114, 69, 237, 95, 48, 233, 176, 91, 154, 0, 0, 0, 0}, 96, 183}, + {{10, 61, 43, 101, 64, 102, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 52, 207}, + {{151, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 9, 102}, + {{210, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 19, 36}, + {{52, 222, 249, 31, 108, 137, 199, 1, 242, 173, 184, 144, 0, 0, 0, 0}, 93, 41}, + {{123, 111, 88, 192, 69, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 47, 70}, + {{180, 82, 188, 125, 140, 8, 196, 74, 0, 0, 0, 0, 0, 0, 0, 0}, 63, 218}, + {{77, 158, 34, 101, 196, 102, 56, 220, 42, 143, 181, 187, 240, 64, 161, 0}, 120, 226}, + {{88, 220, 222, 38, 23, 108, 5, 148, 185, 110, 20, 14, 67, 61, 0, 0}, 114, 25}, + {{90, 65, 220, 165, 197, 133, 110, 92, 228, 19, 2, 17, 0, 0, 0, 0}, 98, 6}, + {{35, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 11, 26}, + {{103, 123, 49, 209, 228, 229, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 52, 149}, + {{50, 244, 58, 191, 95, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 46, 127}, + {{140, 169, 75, 77, 78, 86, 40, 16, 0, 0, 0, 0, 0, 0, 0, 0}, 62, 144}, + {{99, 176, 175, 83, 114, 50, 214, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 56, 213}, + {{19, 208, 211, 76, 85, 176, 247, 64, 0, 0, 0, 0, 0, 0, 0, 0}, 58, 115}, + {{153, 28, 188, 113, 211, 116, 7, 178, 136, 205, 96, 0, 0, 0, 0, 0}, 83, 146}, + {{160, 180, 220, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 26, 58}, + {{234, 6, 112, 19, 61, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 47, 222}, + {{97, 110, 34, 117, 149, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 50, 16}, + {{99, 173, 119, 73, 250, 30, 144, 30, 128, 0, 0, 0, 0, 0, 0, 0}, 65, 169}, + {{169, 134, 111, 89, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 40, 175}, + {{134, 80, 227, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 32, 3}, + {{231, 243, 35, 80, 75, 207, 128, 137, 54, 170, 71, 238, 0, 0, 0, 0}, 96, 2}, + {{189, 190, 121, 135, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 36, 193}, + {{143, 155, 216, 193, 239, 205, 204, 153, 143, 236, 69, 23, 200, 211, 0, 0}, 118, 151}, + {{32, 1, 115, 244, 33, 219, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 51, 182}, + {{220, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 10, 148}, + {{206, 87, 135, 235, 116, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 42, 53}, + {{152, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 11, 87}, + {{58, 146, 188, 233, 230, 236, 192, 214, 168, 128, 0, 0, 0, 0, 0, 0}, 73, 235}, + {{84, 220, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 23, 51}, + {{106, 145, 142, 42, 186, 186, 58, 1, 48, 98, 165, 131, 48, 156, 192, 0}, 116, 11}, + {{53, 219, 120, 242, 166, 214, 81, 130, 64, 0, 0, 0, 0, 0, 0, 0}, 68, 28}, + {{240, 120, 76, 163, 32, 197, 181, 251, 98, 220, 29, 226, 0, 0, 0, 0}, 96, 73}, + {{234, 197, 12, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 28, 216}, + {{191, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 16, 99}, + {{200, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 18, 35}, + {{29, 129, 47, 83, 19, 75, 158, 1, 28, 24, 26, 147, 82, 119, 140, 100}, 127, 195}, + {{241, 174, 26, 53, 152, 112, 200, 134, 84, 187, 177, 176, 42, 64, 0, 0}, 108, 176}, + {{77, 171, 145, 48, 195, 84, 190, 36, 122, 199, 18, 0, 0, 0, 0, 0}, 87, 217}, + {{105, 104, 135, 53, 226, 118, 238, 169, 9, 253, 132, 162, 217, 123, 191, 96}, 126, 244}, + {{160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 3, 125}, + {{41, 85, 143, 128, 91, 137, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 50, 219}, + {{116, 110, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 18, 165}, + {{75, 213, 44, 16, 43, 157, 34, 171, 98, 117, 109, 151, 5, 60, 224, 0}, 117, 6}, + {{229, 23, 116, 61, 80, 139, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 53, 47}, + {{83, 123, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 23, 73}, + {{151, 243, 45, 217, 216, 158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 47, 98}, + {{171, 184, 110, 211, 237, 114, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 52, 21}, + {{7, 246, 199, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 32, 142}, + {{103, 47, 70, 17, 31, 232, 44, 75, 145, 155, 100, 216, 0, 0, 0, 0}, 93, 34}, + {{65, 170, 169, 100, 167, 147, 142, 251, 20, 64, 0, 0, 0, 0, 0, 0}, 74, 41}, + {{235, 6, 229, 248, 151, 137, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 55, 80}, + {{156, 39, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 22, 11}, + {{92, 188, 82, 192, 142, 249, 190, 128, 0, 0, 0, 0, 0, 0, 0, 0}, 58, 254}, + {{253, 218, 181, 46, 134, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 45, 95}, + {{189, 19, 31, 244, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 40, 8}, + {{30, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 14, 212}, + {{81, 226, 13, 173, 79, 123, 223, 124, 108, 80, 83, 238, 0, 0, 0, 0}, 95, 217}, + {{126, 211, 206, 82, 147, 215, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 57, 15}, + {{42, 229, 135, 197, 196, 243, 94, 181, 133, 34, 16, 0, 0, 0, 0, 0}, 84, 66}, + {{68, 210, 158, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 28, 122}, + {{183, 63, 223, 94, 81, 41, 203, 20, 236, 212, 220, 199, 0, 0, 0, 0}, 97, 12}, + {{131, 146, 2, 125, 174, 43, 231, 20, 194, 0, 0, 0, 0, 0, 0, 0}, 71, 171}, + {{31, 180, 246, 158, 28, 192, 236, 39, 237, 55, 74, 195, 171, 192, 0, 0}, 106, 42}, + {{179, 10, 70, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 28, 194}, + {{147, 51, 85, 185, 234, 209, 236, 87, 147, 17, 7, 68, 148, 32, 0, 0}, 107, 237}, + {{177, 178, 6, 40, 46, 166, 87, 198, 214, 234, 23, 224, 0, 0, 0, 0}, 93, 151}, + {{201, 53, 40, 20, 49, 4, 38, 139, 133, 217, 214, 134, 89, 200, 0, 0}, 109, 238}, + {{4, 26, 181, 37, 206, 129, 233, 32, 0, 0, 0, 0, 0, 0, 0, 0}, 59, 128}, + {{81, 58, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 26, 227}, + {{18, 238, 250, 161, 57, 246, 208, 118, 14, 76, 73, 25, 65, 22, 152, 120}, 127, 138}, + {{31, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 10, 60}, + {{115, 195, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 18, 148}, + {{116, 22, 75, 33, 16, 129, 35, 124, 10, 112, 31, 213, 181, 108, 177, 46}, 128, 129}, + {{117, 214, 20, 80, 83, 51, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 49, 202}, + {{120, 75, 124, 149, 120, 123, 242, 151, 181, 164, 128, 0, 0, 0, 0, 0}, 81, 88}, + {{87, 238, 168, 62, 88, 166, 52, 104, 219, 169, 93, 128, 0, 0, 0, 0}, 90, 3}, + {{237, 44, 224, 146, 52, 85, 245, 192, 65, 137, 37, 95, 156, 176, 0, 0}, 108, 243}, + {{214, 241, 51, 63, 73, 61, 193, 165, 23, 108, 0, 0, 0, 0, 0, 0}, 80, 95}, + {{87, 242, 21, 157, 45, 188, 36, 62, 66, 243, 64, 0, 0, 0, 0, 0}, 87, 255}, + {{0, 97, 220, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 26, 48}, + {{227, 206, 189, 31, 222, 8, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 50, 38}, + {{174, 27, 0, 16, 13, 150, 33, 122, 154, 59, 236, 35, 248, 178, 64, 0}, 115, 20}, + {{39, 20, 125, 69, 252, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 43, 41}, + {{141, 232, 1, 12, 125, 229, 168, 14, 125, 116, 180, 0, 0, 0, 0, 0}, 92, 133}, + {{93, 238, 40, 228, 254, 203, 251, 6, 60, 82, 243, 242, 0, 0, 0, 0}, 95, 189}, + {{44, 115, 200, 17, 146, 223, 115, 253, 126, 206, 152, 90, 0, 0, 0, 0}, 95, 151}, + {{213, 58, 235, 255, 6, 163, 61, 10, 224, 0, 0, 0, 0, 0, 0, 0}, 68, 100}, + {{25, 86, 139, 116, 190, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 49, 118}, + {{113, 40, 65, 141, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 34, 164}, + {{149, 205, 200, 186, 19, 126, 215, 199, 94, 37, 100, 32, 128, 0, 0, 0}, 98, 71}, + {{39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 9, 251}, + {{81, 87, 80, 173, 163, 166, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 57, 51}, + {{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 3, 185}, + {{140, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 10, 144}, + {{6, 42, 1, 178, 250, 53, 186, 178, 114, 121, 192, 0, 0, 0, 0, 0}, 84, 51}, + {{2, 17, 234, 51, 169, 5, 219, 149, 245, 237, 4, 0, 0, 0, 0, 0}, 87, 32}, + {{112, 187, 173, 17, 229, 171, 225, 170, 8, 0, 0, 0, 0, 0, 0, 0}, 70, 137}, + {{203, 71, 140, 237, 113, 96, 123, 16, 0, 0, 0, 0, 0, 0, 0, 0}, 60, 2}, + {{99, 138, 207, 2, 244, 25, 211, 98, 0, 0, 0, 0, 0, 0, 0, 0}, 63, 163}, + {{114, 42, 98, 246, 252, 48, 233, 118, 63, 226, 157, 226, 192, 0, 0, 0}, 100, 162}, + {{161, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 10, 192}, + {{233, 70, 240, 45, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 36, 185}, + {{28, 123, 31, 176, 235, 229, 169, 192, 0, 0, 0, 0, 0, 0, 0, 0}, 59, 51}, + {{146, 197, 243, 235, 243, 56, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 54, 93}, + {{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 2, 159}, + {{141, 92, 13, 27, 87, 241, 171, 143, 220, 0, 0, 0, 0, 0, 0, 0}, 72, 189}, + {{164, 151, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 21, 248}, + {{35, 188, 248, 79, 39, 151, 232, 215, 248, 245, 185, 144, 78, 102, 173, 128}, 123, 38}, + {{193, 232, 166, 60, 62, 80, 230, 225, 165, 240, 0, 0, 0, 0, 0, 0}, 76, 167}, + {{109, 229, 118, 155, 43, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 51, 28}, + {{160, 62, 63, 212, 218, 138, 154, 108, 163, 127, 197, 237, 183, 44, 140, 192}, 125, 37}, + {{196, 37, 51, 146, 26, 85, 53, 31, 216, 141, 52, 218, 153, 32, 0, 0}, 107, 234}, + {{228, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 9, 70}, + {{154, 248, 20, 242, 154, 244, 63, 17, 121, 52, 70, 84, 118, 208, 0, 0}, 108, 50}, + {{41, 100, 27, 84, 106, 112, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 51, 171}, + {{81, 99, 197, 139, 30, 150, 230, 216, 81, 190, 84, 165, 29, 64, 128, 0}, 113, 236}, + {{112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 4, 3}, + {{164, 119, 253, 126, 160, 249, 183, 191, 119, 111, 224, 0, 0, 0, 0, 0}, 86, 64}, + {{138, 58, 198, 254, 0, 197, 60, 91, 132, 199, 181, 251, 78, 160, 0, 0}, 108, 213}, + {{209, 89, 168, 236, 146, 169, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 54, 15}, + {{131, 210, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 20, 145}, + {{165, 190, 157, 7, 131, 5, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 57, 27}, + {{179, 226, 57, 204, 187, 70, 52, 81, 119, 162, 229, 42, 47, 185, 9, 162}, 127, 75}, + {{98, 235, 155, 51, 107, 167, 127, 137, 254, 246, 162, 171, 180, 13, 233, 0}, 123, 76}, + {{107, 79, 76, 90, 94, 151, 155, 31, 33, 115, 19, 204, 98, 115, 0, 0}, 113, 247}, + {{143, 46, 30, 175, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 43, 121}, + {{155, 85, 217, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 30, 214}, + {{58, 62, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 26, 221}, + {{92, 155, 53, 3, 39, 108, 155, 200, 0, 0, 0, 0, 0, 0, 0, 0}, 63, 102}, + {{64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 2, 191}, + {{63, 134, 251, 59, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 39, 197}, + {{234, 149, 220, 106, 0, 144, 214, 128, 35, 102, 0, 0, 0, 0, 0, 0}, 79, 106}, +}; + +#define NUM_ROUTE_ENTRIES \ + (sizeof(large_route_table) / sizeof(large_route_table[0])) + +#define NUM_IPS_ENTRIES (NUM_ROUTE_ENTRIES * 100) + +/* clear the previous large_ips_table and reduce LPM6 test case much smaller, + * keep same size as previous one to make test case run similar input data. + */ +static struct ips_tbl_entry large_ips_table[NUM_IPS_ENTRIES]; + +/* let the most significant depth bits of ip_out[] same as ip_in[] + * in the same bit position. ip_out[] and ip_in[] are IPv6 address. + */ +static inline void mask_ip6_prefix(uint8_t *ip_out, + const uint8_t *ip_in, uint8_t depth) +{ + int k; + uint8_t mask_in, mask_out; + + for (k = 0; k < 16; k++) { + if (depth >= 8) + ip_out[k] = ip_in[k]; + else if (depth > 0) { + mask_in = (uint8_t)((unsigned int)(-1) << (8 - depth)); + mask_out = ~mask_in; + ip_out[k] = (ip_in[k] & mask_in) + | (ip_out[k] & mask_out); + } else + return; + + depth -= 8; + } +} + +/* check if IPv6 address ip[] match the rule with IPv6 address ip_rule[] + * and depth. if matched, return 0, else return -1. + */ +static inline int check_lpm6_rule(uint8_t *ip, + const uint8_t *ip_rule, uint8_t depth) +{ + int k; + uint8_t mask; + + for (k = 0; k < 16; k++) { + if (depth >= 8) { + if (ip[k] != ip_rule[k]) + return -1; + } else if (depth > 0) { + mask = (uint8_t)((unsigned int)(-1) << (8 - depth)); + if ((ip[k] & mask) == (ip_rule[k] & mask)) + return 0; + else + return -1; + } else + return 0; + + depth -= 8; + } + + return 0; +} + +/* check input IPv6 address ip[] with each one in rule[] and + * output the *next_hop from the matched item in rule[] with + * longest depth. The count of items in rule[ ] is rule_num. + * if found that some item in rule[] is matched return 0, + * else return -1; + */ +static int get_next_hop(uint8_t *ip, uint8_t *next_hop, + const struct rules_tbl_entry *rule, int rule_num) +{ + int i; + int result; + uint8_t max_depth = 0; + + for (i = 0; i < rule_num; i++) { + if (rule[i].depth >= max_depth) { + result = check_lpm6_rule(ip, rule[i].ip, rule[i].depth); + if (result == 0) { + *next_hop = rule[i].next_hop; + max_depth = rule[i].depth; + } + } + } + + if (max_depth > 0) + return 0; + else + return -1; +} + +/* the implementation of algorithm to generate large IPS table + * at run time. if gen_expected_nex_hop is non-zero, the expected + * next_hop of the IPv6 address of each item in IPS table is computed. + */ +static void generate_large_ips_table(int gen_expected_next_hop) +{ + uint32_t i, j, k; + + for (i = 0; i < NUM_IPS_ENTRIES; i++) { + for (j = 0; j < 16; j++) + large_ips_table[i].ip[j] = lrand48(); + } + + for (k = j = 0, i = 0; i < NUM_IPS_ENTRIES; i++) { + mask_ip6_prefix(large_ips_table[i].ip, + large_route_table[j].ip, large_route_table[j].depth); + k++; + if (k == (NUM_IPS_ENTRIES / NUM_ROUTE_ENTRIES)) { + j++; + k = 0; + } + if (j == NUM_ROUTE_ENTRIES) + j = 0; + } + + if (gen_expected_next_hop == 0) + return; + + for (k = 0; k < NUM_IPS_ENTRIES; k++) + get_next_hop(large_ips_table[k].ip, + &(large_ips_table[k].next_hop), + large_route_table, + NUM_ROUTE_ENTRIES); + +} + +#endif /* _TEST_LPM_ROUTES_H_ */ diff --git a/dpdk/test/test/test_lpm6_perf.c b/dpdk/test/test/test_lpm6_perf.c new file mode 100644 index 00000000..30be430f --- /dev/null +++ b/dpdk/test/test/test_lpm6_perf.c @@ -0,0 +1,192 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <stdint.h> +#include <stdlib.h> +#include <string.h> + +#include <rte_cycles.h> +#include <rte_random.h> +#include <rte_memory.h> +#include <rte_lpm6.h> + +#include "test.h" +#include "test_lpm6_data.h" + +#define TEST_LPM_ASSERT(cond) do { \ + if (!(cond)) { \ + printf("Error at line %d: \n", __LINE__); \ + return -1; \ + } \ +} while(0) + +#define ITERATIONS (1 << 10) +#define BATCH_SIZE 100000 +#define NUMBER_TBL8S (1 << 16) + +static void +print_route_distribution(const struct rules_tbl_entry *table, uint32_t n) +{ + unsigned i, j; + + printf("Route distribution per prefix width: \n"); + printf("DEPTH QUANTITY (PERCENT)\n"); + printf("--------------------------- \n"); + + /* Count depths. */ + for(i = 1; i <= 128; i++) { + unsigned depth_counter = 0; + double percent_hits; + + for (j = 0; j < n; j++) + if (table[j].depth == (uint8_t) i) + depth_counter++; + + percent_hits = ((double)depth_counter)/((double)n) * 100; + printf("%.2u%15u (%.2f)\n", i, depth_counter, percent_hits); + } + printf("\n"); +} + +static int +test_lpm6_perf(void) +{ + struct rte_lpm6 *lpm = NULL; + struct rte_lpm6_config config; + uint64_t begin, total_time; + unsigned i, j; + uint32_t next_hop_add = 0xAA, next_hop_return = 0; + int status = 0; + int64_t count = 0; + + config.max_rules = 1000000; + config.number_tbl8s = NUMBER_TBL8S; + config.flags = 0; + + rte_srand(rte_rdtsc()); + + printf("No. routes = %u\n", (unsigned) NUM_ROUTE_ENTRIES); + + print_route_distribution(large_route_table, (uint32_t) NUM_ROUTE_ENTRIES); + + /* Only generate IPv6 address of each item in large IPS table, + * here next_hop is not needed. + */ + generate_large_ips_table(0); + + lpm = rte_lpm6_create(__func__, SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm != NULL); + + /* Measure add. */ + begin = rte_rdtsc(); + + for (i = 0; i < NUM_ROUTE_ENTRIES; i++) { + if (rte_lpm6_add(lpm, large_route_table[i].ip, + large_route_table[i].depth, next_hop_add) == 0) + status++; + } + /* End Timer. */ + total_time = rte_rdtsc() - begin; + + printf("Unique added entries = %d\n", status); + printf("Average LPM Add: %g cycles\n", + (double)total_time / NUM_ROUTE_ENTRIES); + + /* Measure single Lookup */ + total_time = 0; + count = 0; + + for (i = 0; i < ITERATIONS; i ++) { + begin = rte_rdtsc(); + + for (j = 0; j < NUM_IPS_ENTRIES; j ++) { + if (rte_lpm6_lookup(lpm, large_ips_table[j].ip, + &next_hop_return) != 0) + count++; + } + + total_time += rte_rdtsc() - begin; + + } + printf("Average LPM Lookup: %.1f cycles (fails = %.1f%%)\n", + (double)total_time / ((double)ITERATIONS * BATCH_SIZE), + (count * 100.0) / (double)(ITERATIONS * BATCH_SIZE)); + + /* Measure bulk Lookup */ + total_time = 0; + count = 0; + + uint8_t ip_batch[NUM_IPS_ENTRIES][16]; + int32_t next_hops[NUM_IPS_ENTRIES]; + + for (i = 0; i < NUM_IPS_ENTRIES; i++) + memcpy(ip_batch[i], large_ips_table[i].ip, 16); + + for (i = 0; i < ITERATIONS; i ++) { + + /* Lookup per batch */ + begin = rte_rdtsc(); + rte_lpm6_lookup_bulk_func(lpm, ip_batch, next_hops, NUM_IPS_ENTRIES); + total_time += rte_rdtsc() - begin; + + for (j = 0; j < NUM_IPS_ENTRIES; j++) + if (next_hops[j] < 0) + count++; + } + printf("BULK LPM Lookup: %.1f cycles (fails = %.1f%%)\n", + (double)total_time / ((double)ITERATIONS * BATCH_SIZE), + (count * 100.0) / (double)(ITERATIONS * BATCH_SIZE)); + + /* Delete */ + status = 0; + begin = rte_rdtsc(); + + for (i = 0; i < NUM_ROUTE_ENTRIES; i++) { + /* rte_lpm_delete(lpm, ip, depth) */ + status += rte_lpm6_delete(lpm, large_route_table[i].ip, + large_route_table[i].depth); + } + + total_time += rte_rdtsc() - begin; + + printf("Average LPM Delete: %g cycles\n", + (double)total_time / NUM_ROUTE_ENTRIES); + + rte_lpm6_delete_all(lpm); + rte_lpm6_free(lpm); + + return 0; +} + +REGISTER_TEST_COMMAND(lpm6_perf_autotest, test_lpm6_perf); diff --git a/dpdk/test/test/test_lpm_perf.c b/dpdk/test/test/test_lpm_perf.c new file mode 100644 index 00000000..e7e1281b --- /dev/null +++ b/dpdk/test/test/test_lpm_perf.c @@ -0,0 +1,513 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <stdint.h> +#include <stdlib.h> +#include <math.h> + +#include <rte_cycles.h> +#include <rte_random.h> +#include <rte_branch_prediction.h> +#include <rte_ip.h> +#include <rte_lpm.h> + +#include "test.h" +#include "test_xmmt_ops.h" + +#define TEST_LPM_ASSERT(cond) do { \ + if (!(cond)) { \ + printf("Error at line %d: \n", __LINE__); \ + return -1; \ + } \ +} while(0) + +#define ITERATIONS (1 << 10) +#define BATCH_SIZE (1 << 12) +#define BULK_SIZE 32 + +#define MAX_RULE_NUM (1200000) + +struct route_rule { + uint32_t ip; + uint8_t depth; +}; + +struct route_rule large_route_table[MAX_RULE_NUM]; + +static uint32_t num_route_entries; +#define NUM_ROUTE_ENTRIES num_route_entries + +enum { + IP_CLASS_A, + IP_CLASS_B, + IP_CLASS_C +}; + +/* struct route_rule_count defines the total number of rules in following a/b/c + * each item in a[]/b[]/c[] is the number of common IP address class A/B/C, not + * including the ones for private local network. + */ +struct route_rule_count { + uint32_t a[RTE_LPM_MAX_DEPTH]; + uint32_t b[RTE_LPM_MAX_DEPTH]; + uint32_t c[RTE_LPM_MAX_DEPTH]; +}; + +/* All following numbers of each depth of each common IP class are just + * got from previous large constant table in app/test/test_lpm_routes.h . + * In order to match similar performance, they keep same depth and IP + * address coverage as previous constant table. These numbers don't + * include any private local IP address. As previous large const rule + * table was just dumped from a real router, there are no any IP address + * in class C or D. + */ +static struct route_rule_count rule_count = { + .a = { /* IP class A in which the most significant bit is 0 */ + 0, /* depth = 1 */ + 0, /* depth = 2 */ + 1, /* depth = 3 */ + 0, /* depth = 4 */ + 2, /* depth = 5 */ + 1, /* depth = 6 */ + 3, /* depth = 7 */ + 185, /* depth = 8 */ + 26, /* depth = 9 */ + 16, /* depth = 10 */ + 39, /* depth = 11 */ + 144, /* depth = 12 */ + 233, /* depth = 13 */ + 528, /* depth = 14 */ + 866, /* depth = 15 */ + 3856, /* depth = 16 */ + 3268, /* depth = 17 */ + 5662, /* depth = 18 */ + 17301, /* depth = 19 */ + 22226, /* depth = 20 */ + 11147, /* depth = 21 */ + 16746, /* depth = 22 */ + 17120, /* depth = 23 */ + 77578, /* depth = 24 */ + 401, /* depth = 25 */ + 656, /* depth = 26 */ + 1107, /* depth = 27 */ + 1121, /* depth = 28 */ + 2316, /* depth = 29 */ + 717, /* depth = 30 */ + 10, /* depth = 31 */ + 66 /* depth = 32 */ + }, + .b = { /* IP class A in which the most 2 significant bits are 10 */ + 0, /* depth = 1 */ + 0, /* depth = 2 */ + 0, /* depth = 3 */ + 0, /* depth = 4 */ + 1, /* depth = 5 */ + 1, /* depth = 6 */ + 1, /* depth = 7 */ + 3, /* depth = 8 */ + 3, /* depth = 9 */ + 30, /* depth = 10 */ + 25, /* depth = 11 */ + 168, /* depth = 12 */ + 305, /* depth = 13 */ + 569, /* depth = 14 */ + 1129, /* depth = 15 */ + 50800, /* depth = 16 */ + 1645, /* depth = 17 */ + 1820, /* depth = 18 */ + 3506, /* depth = 19 */ + 3258, /* depth = 20 */ + 3424, /* depth = 21 */ + 4971, /* depth = 22 */ + 6885, /* depth = 23 */ + 39771, /* depth = 24 */ + 424, /* depth = 25 */ + 170, /* depth = 26 */ + 433, /* depth = 27 */ + 92, /* depth = 28 */ + 366, /* depth = 29 */ + 377, /* depth = 30 */ + 2, /* depth = 31 */ + 200 /* depth = 32 */ + }, + .c = { /* IP class A in which the most 3 significant bits are 110 */ + 0, /* depth = 1 */ + 0, /* depth = 2 */ + 0, /* depth = 3 */ + 0, /* depth = 4 */ + 0, /* depth = 5 */ + 0, /* depth = 6 */ + 0, /* depth = 7 */ + 12, /* depth = 8 */ + 8, /* depth = 9 */ + 9, /* depth = 10 */ + 33, /* depth = 11 */ + 69, /* depth = 12 */ + 237, /* depth = 13 */ + 1007, /* depth = 14 */ + 1717, /* depth = 15 */ + 14663, /* depth = 16 */ + 8070, /* depth = 17 */ + 16185, /* depth = 18 */ + 48261, /* depth = 19 */ + 36870, /* depth = 20 */ + 33960, /* depth = 21 */ + 50638, /* depth = 22 */ + 61422, /* depth = 23 */ + 466549, /* depth = 24 */ + 1829, /* depth = 25 */ + 4824, /* depth = 26 */ + 4927, /* depth = 27 */ + 5914, /* depth = 28 */ + 10254, /* depth = 29 */ + 4905, /* depth = 30 */ + 1, /* depth = 31 */ + 716 /* depth = 32 */ + } +}; + +static void generate_random_rule_prefix(uint32_t ip_class, uint8_t depth) +{ +/* IP address class A, the most significant bit is 0 */ +#define IP_HEAD_MASK_A 0x00000000 +#define IP_HEAD_BIT_NUM_A 1 + +/* IP address class B, the most significant 2 bits are 10 */ +#define IP_HEAD_MASK_B 0x80000000 +#define IP_HEAD_BIT_NUM_B 2 + +/* IP address class C, the most significant 3 bits are 110 */ +#define IP_HEAD_MASK_C 0xC0000000 +#define IP_HEAD_BIT_NUM_C 3 + + uint32_t class_depth; + uint32_t range; + uint32_t mask; + uint32_t step; + uint32_t start; + uint32_t fixed_bit_num; + uint32_t ip_head_mask; + uint32_t rule_num; + uint32_t k; + struct route_rule *ptr_rule; + + if (ip_class == IP_CLASS_A) { /* IP Address class A */ + fixed_bit_num = IP_HEAD_BIT_NUM_A; + ip_head_mask = IP_HEAD_MASK_A; + rule_num = rule_count.a[depth - 1]; + } else if (ip_class == IP_CLASS_B) { /* IP Address class B */ + fixed_bit_num = IP_HEAD_BIT_NUM_B; + ip_head_mask = IP_HEAD_MASK_B; + rule_num = rule_count.b[depth - 1]; + } else { /* IP Address class C */ + fixed_bit_num = IP_HEAD_BIT_NUM_C; + ip_head_mask = IP_HEAD_MASK_C; + rule_num = rule_count.c[depth - 1]; + } + + if (rule_num == 0) + return; + + /* the number of rest bits which don't include the most significant + * fixed bits for this IP address class + */ + class_depth = depth - fixed_bit_num; + + /* range is the maximum number of rules for this depth and + * this IP address class + */ + range = 1 << class_depth; + + /* only mask the most depth significant generated bits + * except fixed bits for IP address class + */ + mask = range - 1; + + /* Widen coverage of IP address in generated rules */ + if (range <= rule_num) + step = 1; + else + step = round((double)range / rule_num); + + /* Only generate rest bits except the most significant + * fixed bits for IP address class + */ + start = lrand48() & mask; + ptr_rule = &large_route_table[num_route_entries]; + for (k = 0; k < rule_num; k++) { + ptr_rule->ip = (start << (RTE_LPM_MAX_DEPTH - depth)) + | ip_head_mask; + ptr_rule->depth = depth; + ptr_rule++; + start = (start + step) & mask; + } + num_route_entries += rule_num; +} + +static void insert_rule_in_random_pos(uint32_t ip, uint8_t depth) +{ + uint32_t pos; + int try_count = 0; + struct route_rule tmp; + + do { + pos = lrand48(); + try_count++; + } while ((try_count < 10) && (pos > num_route_entries)); + + if ((pos > num_route_entries) || (pos >= MAX_RULE_NUM)) + pos = num_route_entries >> 1; + + tmp = large_route_table[pos]; + large_route_table[pos].ip = ip; + large_route_table[pos].depth = depth; + if (num_route_entries < MAX_RULE_NUM) + large_route_table[num_route_entries++] = tmp; +} + +static void generate_large_route_rule_table(void) +{ + uint32_t ip_class; + uint8_t depth; + + num_route_entries = 0; + memset(large_route_table, 0, sizeof(large_route_table)); + + for (ip_class = IP_CLASS_A; ip_class <= IP_CLASS_C; ip_class++) { + for (depth = 1; depth <= RTE_LPM_MAX_DEPTH; depth++) { + generate_random_rule_prefix(ip_class, depth); + } + } + + /* Add following rules to keep same as previous large constant table, + * they are 4 rules with private local IP address and 1 all-zeros prefix + * with depth = 8. + */ + insert_rule_in_random_pos(IPv4(0, 0, 0, 0), 8); + insert_rule_in_random_pos(IPv4(10, 2, 23, 147), 32); + insert_rule_in_random_pos(IPv4(192, 168, 100, 10), 24); + insert_rule_in_random_pos(IPv4(192, 168, 25, 100), 24); + insert_rule_in_random_pos(IPv4(192, 168, 129, 124), 32); +} + +static void +print_route_distribution(const struct route_rule *table, uint32_t n) +{ + unsigned i, j; + + printf("Route distribution per prefix width: \n"); + printf("DEPTH QUANTITY (PERCENT)\n"); + printf("--------------------------- \n"); + + /* Count depths. */ + for (i = 1; i <= 32; i++) { + unsigned depth_counter = 0; + double percent_hits; + + for (j = 0; j < n; j++) + if (table[j].depth == (uint8_t) i) + depth_counter++; + + percent_hits = ((double)depth_counter)/((double)n) * 100; + printf("%.2u%15u (%.2f)\n", i, depth_counter, percent_hits); + } + printf("\n"); +} + +static int +test_lpm_perf(void) +{ + struct rte_lpm *lpm = NULL; + struct rte_lpm_config config; + + config.max_rules = 2000000; + config.number_tbl8s = 2048; + config.flags = 0; + uint64_t begin, total_time, lpm_used_entries = 0; + unsigned i, j; + uint32_t next_hop_add = 0xAA, next_hop_return = 0; + int status = 0; + uint64_t cache_line_counter = 0; + int64_t count = 0; + + rte_srand(rte_rdtsc()); + + generate_large_route_rule_table(); + + printf("No. routes = %u\n", (unsigned) NUM_ROUTE_ENTRIES); + + print_route_distribution(large_route_table, (uint32_t) NUM_ROUTE_ENTRIES); + + lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config); + TEST_LPM_ASSERT(lpm != NULL); + + /* Measue add. */ + begin = rte_rdtsc(); + + for (i = 0; i < NUM_ROUTE_ENTRIES; i++) { + if (rte_lpm_add(lpm, large_route_table[i].ip, + large_route_table[i].depth, next_hop_add) == 0) + status++; + } + /* End Timer. */ + total_time = rte_rdtsc() - begin; + + printf("Unique added entries = %d\n", status); + /* Obtain add statistics. */ + for (i = 0; i < RTE_LPM_TBL24_NUM_ENTRIES; i++) { + if (lpm->tbl24[i].valid) + lpm_used_entries++; + + if (i % 32 == 0) { + if ((uint64_t)count < lpm_used_entries) { + cache_line_counter++; + count = lpm_used_entries; + } + } + } + + printf("Used table 24 entries = %u (%g%%)\n", + (unsigned) lpm_used_entries, + (lpm_used_entries * 100.0) / RTE_LPM_TBL24_NUM_ENTRIES); + printf("64 byte Cache entries used = %u (%u bytes)\n", + (unsigned) cache_line_counter, (unsigned) cache_line_counter * 64); + + printf("Average LPM Add: %g cycles\n", + (double)total_time / NUM_ROUTE_ENTRIES); + + /* Measure single Lookup */ + total_time = 0; + count = 0; + + for (i = 0; i < ITERATIONS; i++) { + static uint32_t ip_batch[BATCH_SIZE]; + + for (j = 0; j < BATCH_SIZE; j++) + ip_batch[j] = rte_rand(); + + /* Lookup per batch */ + begin = rte_rdtsc(); + + for (j = 0; j < BATCH_SIZE; j++) { + if (rte_lpm_lookup(lpm, ip_batch[j], &next_hop_return) != 0) + count++; + } + + total_time += rte_rdtsc() - begin; + + } + printf("Average LPM Lookup: %.1f cycles (fails = %.1f%%)\n", + (double)total_time / ((double)ITERATIONS * BATCH_SIZE), + (count * 100.0) / (double)(ITERATIONS * BATCH_SIZE)); + + /* Measure bulk Lookup */ + total_time = 0; + count = 0; + for (i = 0; i < ITERATIONS; i++) { + static uint32_t ip_batch[BATCH_SIZE]; + uint32_t next_hops[BULK_SIZE]; + + /* Create array of random IP addresses */ + for (j = 0; j < BATCH_SIZE; j++) + ip_batch[j] = rte_rand(); + + /* Lookup per batch */ + begin = rte_rdtsc(); + for (j = 0; j < BATCH_SIZE; j += BULK_SIZE) { + unsigned k; + rte_lpm_lookup_bulk(lpm, &ip_batch[j], next_hops, BULK_SIZE); + for (k = 0; k < BULK_SIZE; k++) + if (unlikely(!(next_hops[k] & RTE_LPM_LOOKUP_SUCCESS))) + count++; + } + + total_time += rte_rdtsc() - begin; + } + printf("BULK LPM Lookup: %.1f cycles (fails = %.1f%%)\n", + (double)total_time / ((double)ITERATIONS * BATCH_SIZE), + (count * 100.0) / (double)(ITERATIONS * BATCH_SIZE)); + + /* Measure LookupX4 */ + total_time = 0; + count = 0; + for (i = 0; i < ITERATIONS; i++) { + static uint32_t ip_batch[BATCH_SIZE]; + uint32_t next_hops[4]; + + /* Create array of random IP addresses */ + for (j = 0; j < BATCH_SIZE; j++) + ip_batch[j] = rte_rand(); + + /* Lookup per batch */ + begin = rte_rdtsc(); + for (j = 0; j < BATCH_SIZE; j += RTE_DIM(next_hops)) { + unsigned k; + xmm_t ipx4; + + ipx4 = vect_loadu_sil128((xmm_t *)(ip_batch + j)); + ipx4 = *(xmm_t *)(ip_batch + j); + rte_lpm_lookupx4(lpm, ipx4, next_hops, UINT32_MAX); + for (k = 0; k < RTE_DIM(next_hops); k++) + if (unlikely(next_hops[k] == UINT32_MAX)) + count++; + } + + total_time += rte_rdtsc() - begin; + } + printf("LPM LookupX4: %.1f cycles (fails = %.1f%%)\n", + (double)total_time / ((double)ITERATIONS * BATCH_SIZE), + (count * 100.0) / (double)(ITERATIONS * BATCH_SIZE)); + + /* Delete */ + status = 0; + begin = rte_rdtsc(); + + for (i = 0; i < NUM_ROUTE_ENTRIES; i++) { + /* rte_lpm_delete(lpm, ip, depth) */ + status += rte_lpm_delete(lpm, large_route_table[i].ip, + large_route_table[i].depth); + } + + total_time += rte_rdtsc() - begin; + + printf("Average LPM Delete: %g cycles\n", + (double)total_time / NUM_ROUTE_ENTRIES); + + rte_lpm_delete_all(lpm); + rte_lpm_free(lpm); + + return 0; +} + +REGISTER_TEST_COMMAND(lpm_perf_autotest, test_lpm_perf); diff --git a/dpdk/test/test/test_malloc.c b/dpdk/test/test/test_malloc.c new file mode 100644 index 00000000..4572cafb --- /dev/null +++ b/dpdk/test/test/test_malloc.c @@ -0,0 +1,958 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <stdint.h> +#include <string.h> +#include <stdarg.h> +#include <errno.h> +#include <stdlib.h> +#include <sys/queue.h> + +#include <rte_common.h> +#include <rte_memory.h> +#include <rte_per_lcore.h> +#include <rte_launch.h> +#include <rte_eal.h> +#include <rte_lcore.h> +#include <rte_malloc.h> +#include <rte_cycles.h> +#include <rte_random.h> +#include <rte_string_fns.h> + +#include "test.h" + +#define N 10000 + +/* + * Malloc + * ====== + * + * Allocate some dynamic memory from heap (3 areas). Check that areas + * don't overlap and that alignment constraints match. This test is + * done many times on different lcores simultaneously. + */ + +/* Test if memory overlaps: return 1 if true, or 0 if false. */ +static int +is_memory_overlap(void *p1, size_t len1, void *p2, size_t len2) +{ + unsigned long ptr1 = (unsigned long)p1; + unsigned long ptr2 = (unsigned long)p2; + + if (ptr2 >= ptr1 && (ptr2 - ptr1) < len1) + return 1; + else if (ptr2 < ptr1 && (ptr1 - ptr2) < len2) + return 1; + return 0; +} + +static int +is_aligned(void *p, int align) +{ + unsigned long addr = (unsigned long)p; + unsigned mask = align - 1; + + if (addr & mask) + return 0; + return 1; +} + +static int +test_align_overlap_per_lcore(__attribute__((unused)) void *arg) +{ + const unsigned align1 = 8, + align2 = 64, + align3 = 2048; + unsigned i,j; + void *p1 = NULL, *p2 = NULL, *p3 = NULL; + int ret = 0; + + for (i = 0; i < N; i++) { + p1 = rte_zmalloc("dummy", 1000, align1); + if (!p1){ + printf("rte_zmalloc returned NULL (i=%u)\n", i); + ret = -1; + break; + } + for(j = 0; j < 1000 ; j++) { + if( *(char *)p1 != 0) { + printf("rte_zmalloc didn't zero the allocated memory\n"); + ret = -1; + } + } + p2 = rte_malloc("dummy", 1000, align2); + if (!p2){ + printf("rte_malloc returned NULL (i=%u)\n", i); + ret = -1; + rte_free(p1); + break; + } + p3 = rte_malloc("dummy", 1000, align3); + if (!p3){ + printf("rte_malloc returned NULL (i=%u)\n", i); + ret = -1; + rte_free(p1); + rte_free(p2); + break; + } + if (is_memory_overlap(p1, 1000, p2, 1000)) { + printf("p1 and p2 overlaps\n"); + ret = -1; + } + if (is_memory_overlap(p2, 1000, p3, 1000)) { + printf("p2 and p3 overlaps\n"); + ret = -1; + } + if (is_memory_overlap(p1, 1000, p3, 1000)) { + printf("p1 and p3 overlaps\n"); + ret = -1; + } + if (!is_aligned(p1, align1)) { + printf("p1 is not aligned\n"); + ret = -1; + } + if (!is_aligned(p2, align2)) { + printf("p2 is not aligned\n"); + ret = -1; + } + if (!is_aligned(p3, align3)) { + printf("p3 is not aligned\n"); + ret = -1; + } + rte_free(p1); + rte_free(p2); + rte_free(p3); + } + rte_malloc_dump_stats(stdout, "dummy"); + + return ret; +} + +static int +test_reordered_free_per_lcore(__attribute__((unused)) void *arg) +{ + const unsigned align1 = 8, + align2 = 64, + align3 = 2048; + unsigned i,j; + void *p1, *p2, *p3; + int ret = 0; + + for (i = 0; i < 30; i++) { + p1 = rte_zmalloc("dummy", 1000, align1); + if (!p1){ + printf("rte_zmalloc returned NULL (i=%u)\n", i); + ret = -1; + break; + } + for(j = 0; j < 1000 ; j++) { + if( *(char *)p1 != 0) { + printf("rte_zmalloc didn't zero the allocated memory\n"); + ret = -1; + } + } + /* use calloc to allocate 1000 16-byte items this time */ + p2 = rte_calloc("dummy", 1000, 16, align2); + /* for third request use regular malloc again */ + p3 = rte_malloc("dummy", 1000, align3); + if (!p2 || !p3){ + printf("rte_malloc returned NULL (i=%u)\n", i); + ret = -1; + break; + } + if (is_memory_overlap(p1, 1000, p2, 1000)) { + printf("p1 and p2 overlaps\n"); + ret = -1; + } + if (is_memory_overlap(p2, 1000, p3, 1000)) { + printf("p2 and p3 overlaps\n"); + ret = -1; + } + if (is_memory_overlap(p1, 1000, p3, 1000)) { + printf("p1 and p3 overlaps\n"); + ret = -1; + } + if (!is_aligned(p1, align1)) { + printf("p1 is not aligned\n"); + ret = -1; + } + if (!is_aligned(p2, align2)) { + printf("p2 is not aligned\n"); + ret = -1; + } + if (!is_aligned(p3, align3)) { + printf("p3 is not aligned\n"); + ret = -1; + } + /* try freeing in every possible order */ + switch (i%6){ + case 0: + rte_free(p1); + rte_free(p2); + rte_free(p3); + break; + case 1: + rte_free(p1); + rte_free(p3); + rte_free(p2); + break; + case 2: + rte_free(p2); + rte_free(p1); + rte_free(p3); + break; + case 3: + rte_free(p2); + rte_free(p3); + rte_free(p1); + break; + case 4: + rte_free(p3); + rte_free(p1); + rte_free(p2); + break; + case 5: + rte_free(p3); + rte_free(p2); + rte_free(p1); + break; + } + } + rte_malloc_dump_stats(stdout, "dummy"); + + return ret; +} + +/* test function inside the malloc lib*/ +static int +test_str_to_size(void) +{ + struct { + const char *str; + uint64_t value; + } test_values[] = + {{ "5G", (uint64_t)5 * 1024 * 1024 *1024 }, + {"0x20g", (uint64_t)0x20 * 1024 * 1024 *1024}, + {"10M", 10 * 1024 * 1024}, + {"050m", 050 * 1024 * 1024}, + {"8K", 8 * 1024}, + {"15k", 15 * 1024}, + {"0200", 0200}, + {"0x103", 0x103}, + {"432", 432}, + {"-1", 0}, /* negative values return 0 */ + {" -2", 0}, + {" -3MB", 0}, + {"18446744073709551616", 0} /* ULLONG_MAX + 1 == out of range*/ + }; + unsigned i; + for (i = 0; i < sizeof(test_values)/sizeof(test_values[0]); i++) + if (rte_str_to_size(test_values[i].str) != test_values[i].value) + return -1; + return 0; +} + +static int +test_multi_alloc_statistics(void) +{ + int socket = 0; + struct rte_malloc_socket_stats pre_stats, post_stats ,first_stats, second_stats; + size_t size = 2048; + int align = 1024; +#ifndef RTE_MALLOC_DEBUG + int trailer_size = 0; +#else + int trailer_size = RTE_CACHE_LINE_SIZE; +#endif + int overhead = RTE_CACHE_LINE_SIZE + trailer_size; + + rte_malloc_get_socket_stats(socket, &pre_stats); + + void *p1 = rte_malloc_socket("stats", size , align, socket); + if (!p1) + return -1; + rte_free(p1); + rte_malloc_dump_stats(stdout, "stats"); + + rte_malloc_get_socket_stats(socket,&post_stats); + /* Check statistics reported are correct */ + /* All post stats should be equal to pre stats after alloc freed */ + if ((post_stats.heap_totalsz_bytes != pre_stats.heap_totalsz_bytes) && + (post_stats.heap_freesz_bytes!=pre_stats.heap_freesz_bytes) && + (post_stats.heap_allocsz_bytes!=pre_stats.heap_allocsz_bytes)&& + (post_stats.alloc_count!=pre_stats.alloc_count)&& + (post_stats.free_count!=pre_stats.free_count)) { + printf("Malloc statistics are incorrect - freed alloc\n"); + return -1; + } + /* Check two consecutive allocations */ + size = 1024; + align = 0; + rte_malloc_get_socket_stats(socket,&pre_stats); + void *p2 = rte_malloc_socket("add", size ,align, socket); + if (!p2) + return -1; + rte_malloc_get_socket_stats(socket,&first_stats); + + void *p3 = rte_malloc_socket("add2", size,align, socket); + if (!p3) + return -1; + + rte_malloc_get_socket_stats(socket,&second_stats); + + rte_free(p2); + rte_free(p3); + + /* After freeing both allocations check stats return to original */ + rte_malloc_get_socket_stats(socket, &post_stats); + + if(second_stats.heap_totalsz_bytes != first_stats.heap_totalsz_bytes) { + printf("Incorrect heap statistics: Total size \n"); + return -1; + } + /* Check allocated size is equal to two additions plus overhead */ + if(second_stats.heap_allocsz_bytes != + size + overhead + first_stats.heap_allocsz_bytes) { + printf("Incorrect heap statistics: Allocated size \n"); + return -1; + } + /* Check that allocation count increments correctly i.e. +1 */ + if (second_stats.alloc_count != first_stats.alloc_count + 1) { + printf("Incorrect heap statistics: Allocated count \n"); + return -1; + } + + if (second_stats.free_count != first_stats.free_count){ + printf("Incorrect heap statistics: Free count \n"); + return -1; + } + + /* Make sure that we didn't touch our greatest chunk: 2 * 11M) */ + if (post_stats.greatest_free_size != pre_stats.greatest_free_size) { + printf("Incorrect heap statistics: Greatest free size \n"); + return -1; + } + /* Free size must equal the original free size minus the new allocation*/ + if (first_stats.heap_freesz_bytes <= second_stats.heap_freesz_bytes) { + printf("Incorrect heap statistics: Free size \n"); + return -1; + } + + if ((post_stats.heap_totalsz_bytes != pre_stats.heap_totalsz_bytes) && + (post_stats.heap_freesz_bytes!=pre_stats.heap_freesz_bytes) && + (post_stats.heap_allocsz_bytes!=pre_stats.heap_allocsz_bytes)&& + (post_stats.alloc_count!=pre_stats.alloc_count)&& + (post_stats.free_count!=pre_stats.free_count)) { + printf("Malloc statistics are incorrect - freed alloc\n"); + return -1; + } + return 0; +} + +static int +test_rte_malloc_type_limits(void) +{ + /* The type-limits functionality is not yet implemented, + * so always return 0 no matter what the retval. + */ + const char *typename = "limit_test"; + rte_malloc_set_limit(typename, 64 * 1024); + rte_malloc_dump_stats(stdout, typename); + return 0; +} + +static int +test_realloc(void) +{ + const char hello_str[] = "Hello, world!"; + const unsigned size1 = 1024; + const unsigned size2 = size1 + 1024; + const unsigned size3 = size2; + const unsigned size4 = size3 + 1024; + + /* test data is the same even if element is moved*/ + char *ptr1 = rte_zmalloc(NULL, size1, RTE_CACHE_LINE_SIZE); + if (!ptr1){ + printf("NULL pointer returned from rte_zmalloc\n"); + return -1; + } + snprintf(ptr1, size1, "%s" ,hello_str); + char *ptr2 = rte_realloc(ptr1, size2, RTE_CACHE_LINE_SIZE); + if (!ptr2){ + rte_free(ptr1); + printf("NULL pointer returned from rte_realloc\n"); + return -1; + } + if (ptr1 == ptr2){ + printf("unexpected - ptr1 == ptr2\n"); + } + if (strcmp(ptr2, hello_str) != 0){ + printf("Error - lost data from pointed area\n"); + rte_free(ptr2); + return -1; + } + unsigned i; + for (i = strnlen(hello_str, sizeof(hello_str)); i < size1; i++) + if (ptr2[i] != 0){ + printf("Bad data in realloc\n"); + rte_free(ptr2); + return -1; + } + /* now allocate third element, free the second + * and resize third. It should not move. (ptr1 is now invalid) + */ + char *ptr3 = rte_zmalloc(NULL, size3, RTE_CACHE_LINE_SIZE); + if (!ptr3){ + printf("NULL pointer returned from rte_zmalloc\n"); + rte_free(ptr2); + return -1; + } + for (i = 0; i < size3; i++) + if (ptr3[i] != 0){ + printf("Bad data in zmalloc\n"); + rte_free(ptr3); + rte_free(ptr2); + return -1; + } + rte_free(ptr2); + /* first resize to half the size of the freed block */ + char *ptr4 = rte_realloc(ptr3, size4, RTE_CACHE_LINE_SIZE); + if (!ptr4){ + printf("NULL pointer returned from rte_realloc\n"); + rte_free(ptr3); + return -1; + } + if (ptr3 != ptr4){ + printf("Unexpected - ptr4 != ptr3\n"); + rte_free(ptr4); + return -1; + } + /* now resize again to the full size of the freed block */ + ptr4 = rte_realloc(ptr3, size3 + size2 + size1, RTE_CACHE_LINE_SIZE); + if (ptr3 != ptr4){ + printf("Unexpected - ptr4 != ptr3 on second resize\n"); + rte_free(ptr4); + return -1; + } + rte_free(ptr4); + + /* now try a resize to a smaller size, see if it works */ + const unsigned size5 = 1024; + const unsigned size6 = size5 / 2; + char *ptr5 = rte_malloc(NULL, size5, RTE_CACHE_LINE_SIZE); + if (!ptr5){ + printf("NULL pointer returned from rte_malloc\n"); + return -1; + } + char *ptr6 = rte_realloc(ptr5, size6, RTE_CACHE_LINE_SIZE); + if (!ptr6){ + printf("NULL pointer returned from rte_realloc\n"); + rte_free(ptr5); + return -1; + } + if (ptr5 != ptr6){ + printf("Error, resizing to a smaller size moved data\n"); + rte_free(ptr6); + return -1; + } + rte_free(ptr6); + + /* check for behaviour changing alignment */ + const unsigned size7 = 1024; + const unsigned orig_align = RTE_CACHE_LINE_SIZE; + unsigned new_align = RTE_CACHE_LINE_SIZE * 2; + char *ptr7 = rte_malloc(NULL, size7, orig_align); + if (!ptr7){ + printf("NULL pointer returned from rte_malloc\n"); + return -1; + } + /* calc an alignment we don't already have */ + while(RTE_PTR_ALIGN(ptr7, new_align) == ptr7) + new_align *= 2; + char *ptr8 = rte_realloc(ptr7, size7, new_align); + if (!ptr8){ + printf("NULL pointer returned from rte_realloc\n"); + rte_free(ptr7); + return -1; + } + if (RTE_PTR_ALIGN(ptr8, new_align) != ptr8){ + printf("Failure to re-align data\n"); + rte_free(ptr8); + return -1; + } + rte_free(ptr8); + + /* test behaviour when there is a free block after current one, + * but its not big enough + */ + unsigned size9 = 1024, size10 = 1024; + unsigned size11 = size9 + size10 + 256; + char *ptr9 = rte_malloc(NULL, size9, RTE_CACHE_LINE_SIZE); + if (!ptr9){ + printf("NULL pointer returned from rte_malloc\n"); + return -1; + } + char *ptr10 = rte_malloc(NULL, size10, RTE_CACHE_LINE_SIZE); + if (!ptr10){ + printf("NULL pointer returned from rte_malloc\n"); + return -1; + } + rte_free(ptr9); + char *ptr11 = rte_realloc(ptr10, size11, RTE_CACHE_LINE_SIZE); + if (!ptr11){ + printf("NULL pointer returned from rte_realloc\n"); + rte_free(ptr10); + return -1; + } + if (ptr11 == ptr10){ + printf("Error, unexpected that realloc has not created new buffer\n"); + rte_free(ptr11); + return -1; + } + rte_free(ptr11); + + /* check we don't crash if we pass null to realloc + * We should get a malloc of the size requested*/ + const size_t size12 = 1024; + size_t size12_check; + char *ptr12 = rte_realloc(NULL, size12, RTE_CACHE_LINE_SIZE); + if (!ptr12){ + printf("NULL pointer returned from rte_realloc\n"); + return -1; + } + if (rte_malloc_validate(ptr12, &size12_check) < 0 || + size12_check != size12){ + rte_free(ptr12); + return -1; + } + rte_free(ptr12); + return 0; +} + +static int +test_random_alloc_free(void *_ __attribute__((unused))) +{ + struct mem_list { + struct mem_list *next; + char data[0]; + } *list_head = NULL; + unsigned i; + unsigned count = 0; + + rte_srand((unsigned)rte_rdtsc()); + + for (i = 0; i < N; i++){ + unsigned free_mem = 0; + size_t allocated_size; + while (!free_mem){ + const unsigned mem_size = sizeof(struct mem_list) + \ + rte_rand() % (64 * 1024); + const unsigned align = 1 << (rte_rand() % 12); /* up to 4k alignment */ + struct mem_list *entry = rte_malloc(NULL, + mem_size, align); + if (entry == NULL) + return -1; + if (RTE_PTR_ALIGN(entry, align)!= entry) + return -1; + if (rte_malloc_validate(entry, &allocated_size) == -1 + || allocated_size < mem_size) + return -1; + memset(entry->data, rte_lcore_id(), + mem_size - sizeof(*entry)); + entry->next = list_head; + if (rte_malloc_validate(entry, NULL) == -1) + return -1; + list_head = entry; + + count++; + /* switch to freeing the memory with a 20% probability */ + free_mem = ((rte_rand() % 10) >= 8); + } + while (list_head){ + struct mem_list *entry = list_head; + list_head = list_head->next; + rte_free(entry); + } + } + printf("Lcore %u allocated/freed %u blocks\n", rte_lcore_id(), count); + return 0; +} + +#define err_return() do { \ + printf("%s: %d - Error\n", __func__, __LINE__); \ + goto err_return; \ +} while (0) + +static int +test_rte_malloc_validate(void) +{ + const size_t request_size = 1024; + size_t allocated_size; + char *data_ptr = rte_malloc(NULL, request_size, RTE_CACHE_LINE_SIZE); +#ifdef RTE_MALLOC_DEBUG + int retval; + char *over_write_vals = NULL; +#endif + + if (data_ptr == NULL) { + printf("%s: %d - Allocation error\n", __func__, __LINE__); + return -1; + } + + /* check that a null input returns -1 */ + if (rte_malloc_validate(NULL, NULL) != -1) + err_return(); + + /* check that we get ok on a valid pointer */ + if (rte_malloc_validate(data_ptr, &allocated_size) < 0) + err_return(); + + /* check that the returned size is ok */ + if (allocated_size < request_size) + err_return(); + +#ifdef RTE_MALLOC_DEBUG + + /****** change the header to be bad */ + char save_buf[64]; + over_write_vals = (char *)((uintptr_t)data_ptr - sizeof(save_buf)); + /* first save the data as a backup before overwriting it */ + memcpy(save_buf, over_write_vals, sizeof(save_buf)); + memset(over_write_vals, 1, sizeof(save_buf)); + /* then run validate */ + retval = rte_malloc_validate(data_ptr, NULL); + /* finally restore the data again */ + memcpy(over_write_vals, save_buf, sizeof(save_buf)); + /* check we previously had an error */ + if (retval != -1) + err_return(); + + /* check all ok again */ + if (rte_malloc_validate(data_ptr, &allocated_size) < 0) + err_return(); + + /**** change the trailer to be bad */ + over_write_vals = (char *)((uintptr_t)data_ptr + allocated_size); + /* first save the data as a backup before overwriting it */ + memcpy(save_buf, over_write_vals, sizeof(save_buf)); + memset(over_write_vals, 1, sizeof(save_buf)); + /* then run validate */ + retval = rte_malloc_validate(data_ptr, NULL); + /* finally restore the data again */ + memcpy(over_write_vals, save_buf, sizeof(save_buf)); + if (retval != -1) + err_return(); + + /* check all ok again */ + if (rte_malloc_validate(data_ptr, &allocated_size) < 0) + err_return(); +#endif + + rte_free(data_ptr); + return 0; + +err_return: + /*clean up */ + rte_free(data_ptr); + return -1; +} + +static int +test_zero_aligned_alloc(void) +{ + char *p1 = rte_malloc(NULL,1024, 0); + if (!p1) + goto err_return; + if (!rte_is_aligned(p1, RTE_CACHE_LINE_SIZE)) + goto err_return; + rte_free(p1); + return 0; + +err_return: + /*clean up */ + if (p1) rte_free(p1); + return -1; +} + +static int +test_malloc_bad_params(void) +{ + const char *type = NULL; + size_t size = 0; + unsigned align = RTE_CACHE_LINE_SIZE; + + /* rte_malloc expected to return null with inappropriate size */ + char *bad_ptr = rte_malloc(type, size, align); + if (bad_ptr != NULL) + goto err_return; + + /* rte_malloc expected to return null with inappropriate alignment */ + align = 17; + size = 1024; + + bad_ptr = rte_malloc(type, size, align); + if (bad_ptr != NULL) + goto err_return; + + return 0; + +err_return: + /* clean up pointer */ + if (bad_ptr) + rte_free(bad_ptr); + return -1; +} + +/* Check if memory is available on a specific socket */ +static int +is_mem_on_socket(int32_t socket) +{ + const struct rte_memseg *ms = rte_eal_get_physmem_layout(); + unsigned i; + + for (i = 0; i < RTE_MAX_MEMSEG; i++) { + if (socket == ms[i].socket_id) + return 1; + } + return 0; +} + +/* + * Find what socket a memory address is on. Only works for addresses within + * memsegs, not heap or stack... + */ +static int32_t +addr_to_socket(void * addr) +{ + const struct rte_memseg *ms = rte_eal_get_physmem_layout(); + unsigned i; + + for (i = 0; i < RTE_MAX_MEMSEG; i++) { + if ((ms[i].addr <= addr) && + ((uintptr_t)addr < + ((uintptr_t)ms[i].addr + (uintptr_t)ms[i].len))) + return ms[i].socket_id; + } + return -1; +} + +/* Test using rte_[c|m|zm]alloc_socket() on a specific socket */ +static int +test_alloc_single_socket(int32_t socket) +{ + const char *type = NULL; + const size_t size = 10; + const unsigned align = 0; + char *mem = NULL; + int32_t desired_socket = (socket == SOCKET_ID_ANY) ? + (int32_t)rte_socket_id() : socket; + + /* Test rte_calloc_socket() */ + mem = rte_calloc_socket(type, size, sizeof(char), align, socket); + if (mem == NULL) + return -1; + if (addr_to_socket(mem) != desired_socket) { + rte_free(mem); + return -1; + } + rte_free(mem); + + /* Test rte_malloc_socket() */ + mem = rte_malloc_socket(type, size, align, socket); + if (mem == NULL) + return -1; + if (addr_to_socket(mem) != desired_socket) { + return -1; + } + rte_free(mem); + + /* Test rte_zmalloc_socket() */ + mem = rte_zmalloc_socket(type, size, align, socket); + if (mem == NULL) + return -1; + if (addr_to_socket(mem) != desired_socket) { + rte_free(mem); + return -1; + } + rte_free(mem); + + return 0; +} + +static int +test_alloc_socket(void) +{ + unsigned socket_count = 0; + unsigned i; + + if (test_alloc_single_socket(SOCKET_ID_ANY) < 0) + return -1; + + for (i = 0; i < RTE_MAX_NUMA_NODES; i++) { + if (is_mem_on_socket(i)) { + socket_count++; + if (test_alloc_single_socket(i) < 0) { + printf("Fail: rte_malloc_socket(..., %u) did not succeed\n", + i); + return -1; + } + } + else { + if (test_alloc_single_socket(i) == 0) { + printf("Fail: rte_malloc_socket(..., %u) succeeded\n", + i); + return -1; + } + } + } + + /* Print warnign if only a single socket, but don't fail the test */ + if (socket_count < 2) { + printf("WARNING: alloc_socket test needs memory on multiple sockets!\n"); + } + + return 0; +} + +static int +test_malloc(void) +{ + unsigned lcore_id; + int ret = 0; + + if (test_str_to_size() < 0){ + printf("test_str_to_size() failed\n"); + return -1; + } + else printf("test_str_to_size() passed\n"); + + if (test_zero_aligned_alloc() < 0){ + printf("test_zero_aligned_alloc() failed\n"); + return -1; + } + else printf("test_zero_aligned_alloc() passed\n"); + + if (test_malloc_bad_params() < 0){ + printf("test_malloc_bad_params() failed\n"); + return -1; + } + else printf("test_malloc_bad_params() passed\n"); + + if (test_realloc() < 0){ + printf("test_realloc() failed\n"); + return -1; + } + else printf("test_realloc() passed\n"); + + /*----------------------------*/ + RTE_LCORE_FOREACH_SLAVE(lcore_id) { + rte_eal_remote_launch(test_align_overlap_per_lcore, NULL, lcore_id); + } + + RTE_LCORE_FOREACH_SLAVE(lcore_id) { + if (rte_eal_wait_lcore(lcore_id) < 0) + ret = -1; + } + if (ret < 0){ + printf("test_align_overlap_per_lcore() failed\n"); + return ret; + } + else printf("test_align_overlap_per_lcore() passed\n"); + + /*----------------------------*/ + RTE_LCORE_FOREACH_SLAVE(lcore_id) { + rte_eal_remote_launch(test_reordered_free_per_lcore, NULL, lcore_id); + } + + RTE_LCORE_FOREACH_SLAVE(lcore_id) { + if (rte_eal_wait_lcore(lcore_id) < 0) + ret = -1; + } + if (ret < 0){ + printf("test_reordered_free_per_lcore() failed\n"); + return ret; + } + else printf("test_reordered_free_per_lcore() passed\n"); + + /*----------------------------*/ + RTE_LCORE_FOREACH_SLAVE(lcore_id) { + rte_eal_remote_launch(test_random_alloc_free, NULL, lcore_id); + } + + RTE_LCORE_FOREACH_SLAVE(lcore_id) { + if (rte_eal_wait_lcore(lcore_id) < 0) + ret = -1; + } + if (ret < 0){ + printf("test_random_alloc_free() failed\n"); + return ret; + } + else printf("test_random_alloc_free() passed\n"); + + /*----------------------------*/ + ret = test_rte_malloc_type_limits(); + if (ret < 0){ + printf("test_rte_malloc_type_limits() failed\n"); + return ret; + } + /* TODO: uncomment following line once type limits are valid */ + /*else printf("test_rte_malloc_type_limits() passed\n");*/ + + /*----------------------------*/ + ret = test_rte_malloc_validate(); + if (ret < 0){ + printf("test_rte_malloc_validate() failed\n"); + return ret; + } + else printf("test_rte_malloc_validate() passed\n"); + + ret = test_alloc_socket(); + if (ret < 0){ + printf("test_alloc_socket() failed\n"); + return ret; + } + else printf("test_alloc_socket() passed\n"); + + ret = test_multi_alloc_statistics(); + if (ret < 0) { + printf("test_multi_alloc_statistics() failed\n"); + return ret; + } + else + printf("test_multi_alloc_statistics() passed\n"); + + return 0; +} + +REGISTER_TEST_COMMAND(malloc_autotest, test_malloc); diff --git a/dpdk/test/test/test_mbuf.c b/dpdk/test/test/test_mbuf.c new file mode 100644 index 00000000..414509b2 --- /dev/null +++ b/dpdk/test/test/test_mbuf.c @@ -0,0 +1,1166 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <string.h> +#include <stdarg.h> +#include <stdio.h> +#include <stdlib.h> +#include <stdint.h> +#include <inttypes.h> +#include <errno.h> +#include <sys/queue.h> + +#include <rte_common.h> +#include <rte_debug.h> +#include <rte_log.h> +#include <rte_memory.h> +#include <rte_memcpy.h> +#include <rte_launch.h> +#include <rte_eal.h> +#include <rte_per_lcore.h> +#include <rte_lcore.h> +#include <rte_atomic.h> +#include <rte_branch_prediction.h> +#include <rte_ring.h> +#include <rte_mempool.h> +#include <rte_mbuf.h> +#include <rte_random.h> +#include <rte_cycles.h> + +#include "test.h" + +#define MBUF_DATA_SIZE 2048 +#define NB_MBUF 128 +#define MBUF_TEST_DATA_LEN 1464 +#define MBUF_TEST_DATA_LEN2 50 +#define MBUF_TEST_HDR1_LEN 20 +#define MBUF_TEST_HDR2_LEN 30 +#define MBUF_TEST_ALL_HDRS_LEN (MBUF_TEST_HDR1_LEN+MBUF_TEST_HDR2_LEN) + +/* size of private data for mbuf in pktmbuf_pool2 */ +#define MBUF2_PRIV_SIZE 128 + +#define REFCNT_MAX_ITER 64 +#define REFCNT_MAX_TIMEOUT 10 +#define REFCNT_MAX_REF (RTE_MAX_LCORE) +#define REFCNT_MBUF_NUM 64 +#define REFCNT_RING_SIZE (REFCNT_MBUF_NUM * REFCNT_MAX_REF) + +#define MAGIC_DATA 0x42424242 + +#define MAKE_STRING(x) # x + +#ifdef RTE_MBUF_REFCNT_ATOMIC + +static volatile uint32_t refcnt_stop_slaves; +static unsigned refcnt_lcore[RTE_MAX_LCORE]; + +#endif + +/* + * MBUF + * ==== + * + * #. Allocate a mbuf pool. + * + * - The pool contains NB_MBUF elements, where each mbuf is MBUF_SIZE + * bytes long. + * + * #. Test multiple allocations of mbufs from this pool. + * + * - Allocate NB_MBUF and store pointers in a table. + * - If an allocation fails, return an error. + * - Free all these mbufs. + * - Repeat the same test to check that mbufs were freed correctly. + * + * #. Test data manipulation in pktmbuf. + * + * - Alloc an mbuf. + * - Append data using rte_pktmbuf_append(). + * - Test for error in rte_pktmbuf_append() when len is too large. + * - Trim data at the end of mbuf using rte_pktmbuf_trim(). + * - Test for error in rte_pktmbuf_trim() when len is too large. + * - Prepend a header using rte_pktmbuf_prepend(). + * - Test for error in rte_pktmbuf_prepend() when len is too large. + * - Remove data at the beginning of mbuf using rte_pktmbuf_adj(). + * - Test for error in rte_pktmbuf_adj() when len is too large. + * - Check that appended data is not corrupt. + * - Free the mbuf. + * - Between all these tests, check data_len and pkt_len, and + * that the mbuf is contiguous. + * - Repeat the test to check that allocation operations + * reinitialize the mbuf correctly. + * + * #. Test packet cloning + * - Clone a mbuf and verify the data + * - Clone the cloned mbuf and verify the data + * - Attach a mbuf to another that does not have the same priv_size. + */ + +#define GOTO_FAIL(str, ...) do { \ + printf("mbuf test FAILED (l.%d): <" str ">\n", \ + __LINE__, ##__VA_ARGS__); \ + goto fail; \ +} while(0) + +/* + * test data manipulation in mbuf with non-ascii data + */ +static int +test_pktmbuf_with_non_ascii_data(struct rte_mempool *pktmbuf_pool) +{ + struct rte_mbuf *m = NULL; + char *data; + + m = rte_pktmbuf_alloc(pktmbuf_pool); + if (m == NULL) + GOTO_FAIL("Cannot allocate mbuf"); + if (rte_pktmbuf_pkt_len(m) != 0) + GOTO_FAIL("Bad length"); + + data = rte_pktmbuf_append(m, MBUF_TEST_DATA_LEN); + if (data == NULL) + GOTO_FAIL("Cannot append data"); + if (rte_pktmbuf_pkt_len(m) != MBUF_TEST_DATA_LEN) + GOTO_FAIL("Bad pkt length"); + if (rte_pktmbuf_data_len(m) != MBUF_TEST_DATA_LEN) + GOTO_FAIL("Bad data length"); + memset(data, 0xff, rte_pktmbuf_pkt_len(m)); + if (!rte_pktmbuf_is_contiguous(m)) + GOTO_FAIL("Buffer should be continuous"); + rte_pktmbuf_dump(stdout, m, MBUF_TEST_DATA_LEN); + + rte_pktmbuf_free(m); + + return 0; + +fail: + if(m) { + rte_pktmbuf_free(m); + } + return -1; +} + +/* + * test data manipulation in mbuf + */ +static int +test_one_pktmbuf(struct rte_mempool *pktmbuf_pool) +{ + struct rte_mbuf *m = NULL; + char *data, *data2, *hdr; + unsigned i; + + printf("Test pktmbuf API\n"); + + /* alloc a mbuf */ + + m = rte_pktmbuf_alloc(pktmbuf_pool); + if (m == NULL) + GOTO_FAIL("Cannot allocate mbuf"); + if (rte_pktmbuf_pkt_len(m) != 0) + GOTO_FAIL("Bad length"); + + rte_pktmbuf_dump(stdout, m, 0); + + /* append data */ + + data = rte_pktmbuf_append(m, MBUF_TEST_DATA_LEN); + if (data == NULL) + GOTO_FAIL("Cannot append data"); + if (rte_pktmbuf_pkt_len(m) != MBUF_TEST_DATA_LEN) + GOTO_FAIL("Bad pkt length"); + if (rte_pktmbuf_data_len(m) != MBUF_TEST_DATA_LEN) + GOTO_FAIL("Bad data length"); + memset(data, 0x66, rte_pktmbuf_pkt_len(m)); + if (!rte_pktmbuf_is_contiguous(m)) + GOTO_FAIL("Buffer should be continuous"); + rte_pktmbuf_dump(stdout, m, MBUF_TEST_DATA_LEN); + rte_pktmbuf_dump(stdout, m, 2*MBUF_TEST_DATA_LEN); + + /* this append should fail */ + + data2 = rte_pktmbuf_append(m, (uint16_t)(rte_pktmbuf_tailroom(m) + 1)); + if (data2 != NULL) + GOTO_FAIL("Append should not succeed"); + + /* append some more data */ + + data2 = rte_pktmbuf_append(m, MBUF_TEST_DATA_LEN2); + if (data2 == NULL) + GOTO_FAIL("Cannot append data"); + if (rte_pktmbuf_pkt_len(m) != MBUF_TEST_DATA_LEN + MBUF_TEST_DATA_LEN2) + GOTO_FAIL("Bad pkt length"); + if (rte_pktmbuf_data_len(m) != MBUF_TEST_DATA_LEN + MBUF_TEST_DATA_LEN2) + GOTO_FAIL("Bad data length"); + if (!rte_pktmbuf_is_contiguous(m)) + GOTO_FAIL("Buffer should be continuous"); + + /* trim data at the end of mbuf */ + + if (rte_pktmbuf_trim(m, MBUF_TEST_DATA_LEN2) < 0) + GOTO_FAIL("Cannot trim data"); + if (rte_pktmbuf_pkt_len(m) != MBUF_TEST_DATA_LEN) + GOTO_FAIL("Bad pkt length"); + if (rte_pktmbuf_data_len(m) != MBUF_TEST_DATA_LEN) + GOTO_FAIL("Bad data length"); + if (!rte_pktmbuf_is_contiguous(m)) + GOTO_FAIL("Buffer should be continuous"); + + /* this trim should fail */ + + if (rte_pktmbuf_trim(m, (uint16_t)(rte_pktmbuf_data_len(m) + 1)) == 0) + GOTO_FAIL("trim should not succeed"); + + /* prepend one header */ + + hdr = rte_pktmbuf_prepend(m, MBUF_TEST_HDR1_LEN); + if (hdr == NULL) + GOTO_FAIL("Cannot prepend"); + if (data - hdr != MBUF_TEST_HDR1_LEN) + GOTO_FAIL("Prepend failed"); + if (rte_pktmbuf_pkt_len(m) != MBUF_TEST_DATA_LEN + MBUF_TEST_HDR1_LEN) + GOTO_FAIL("Bad pkt length"); + if (rte_pktmbuf_data_len(m) != MBUF_TEST_DATA_LEN + MBUF_TEST_HDR1_LEN) + GOTO_FAIL("Bad data length"); + if (!rte_pktmbuf_is_contiguous(m)) + GOTO_FAIL("Buffer should be continuous"); + memset(hdr, 0x55, MBUF_TEST_HDR1_LEN); + + /* prepend another header */ + + hdr = rte_pktmbuf_prepend(m, MBUF_TEST_HDR2_LEN); + if (hdr == NULL) + GOTO_FAIL("Cannot prepend"); + if (data - hdr != MBUF_TEST_ALL_HDRS_LEN) + GOTO_FAIL("Prepend failed"); + if (rte_pktmbuf_pkt_len(m) != MBUF_TEST_DATA_LEN + MBUF_TEST_ALL_HDRS_LEN) + GOTO_FAIL("Bad pkt length"); + if (rte_pktmbuf_data_len(m) != MBUF_TEST_DATA_LEN + MBUF_TEST_ALL_HDRS_LEN) + GOTO_FAIL("Bad data length"); + if (!rte_pktmbuf_is_contiguous(m)) + GOTO_FAIL("Buffer should be continuous"); + memset(hdr, 0x55, MBUF_TEST_HDR2_LEN); + + rte_mbuf_sanity_check(m, 1); + rte_mbuf_sanity_check(m, 0); + rte_pktmbuf_dump(stdout, m, 0); + + /* this prepend should fail */ + + hdr = rte_pktmbuf_prepend(m, (uint16_t)(rte_pktmbuf_headroom(m) + 1)); + if (hdr != NULL) + GOTO_FAIL("prepend should not succeed"); + + /* remove data at beginning of mbuf (adj) */ + + if (data != rte_pktmbuf_adj(m, MBUF_TEST_ALL_HDRS_LEN)) + GOTO_FAIL("rte_pktmbuf_adj failed"); + if (rte_pktmbuf_pkt_len(m) != MBUF_TEST_DATA_LEN) + GOTO_FAIL("Bad pkt length"); + if (rte_pktmbuf_data_len(m) != MBUF_TEST_DATA_LEN) + GOTO_FAIL("Bad data length"); + if (!rte_pktmbuf_is_contiguous(m)) + GOTO_FAIL("Buffer should be continuous"); + + /* this adj should fail */ + + if (rte_pktmbuf_adj(m, (uint16_t)(rte_pktmbuf_data_len(m) + 1)) != NULL) + GOTO_FAIL("rte_pktmbuf_adj should not succeed"); + + /* check data */ + + if (!rte_pktmbuf_is_contiguous(m)) + GOTO_FAIL("Buffer should be continuous"); + + for (i=0; i<MBUF_TEST_DATA_LEN; i++) { + if (data[i] != 0x66) + GOTO_FAIL("Data corrupted at offset %u", i); + } + + /* free mbuf */ + + rte_pktmbuf_free(m); + m = NULL; + return 0; + +fail: + if (m) + rte_pktmbuf_free(m); + return -1; +} + +static int +testclone_testupdate_testdetach(struct rte_mempool *pktmbuf_pool) +{ + struct rte_mbuf *m = NULL; + struct rte_mbuf *clone = NULL; + struct rte_mbuf *clone2 = NULL; + unaligned_uint32_t *data; + + /* alloc a mbuf */ + m = rte_pktmbuf_alloc(pktmbuf_pool); + if (m == NULL) + GOTO_FAIL("ooops not allocating mbuf"); + + if (rte_pktmbuf_pkt_len(m) != 0) + GOTO_FAIL("Bad length"); + + rte_pktmbuf_append(m, sizeof(uint32_t)); + data = rte_pktmbuf_mtod(m, unaligned_uint32_t *); + *data = MAGIC_DATA; + + /* clone the allocated mbuf */ + clone = rte_pktmbuf_clone(m, pktmbuf_pool); + if (clone == NULL) + GOTO_FAIL("cannot clone data\n"); + + data = rte_pktmbuf_mtod(clone, unaligned_uint32_t *); + if (*data != MAGIC_DATA) + GOTO_FAIL("invalid data in clone\n"); + + if (rte_mbuf_refcnt_read(m) != 2) + GOTO_FAIL("invalid refcnt in m\n"); + + /* free the clone */ + rte_pktmbuf_free(clone); + clone = NULL; + + /* same test with a chained mbuf */ + m->next = rte_pktmbuf_alloc(pktmbuf_pool); + if (m->next == NULL) + GOTO_FAIL("Next Pkt Null\n"); + + rte_pktmbuf_append(m->next, sizeof(uint32_t)); + data = rte_pktmbuf_mtod(m->next, unaligned_uint32_t *); + *data = MAGIC_DATA; + + clone = rte_pktmbuf_clone(m, pktmbuf_pool); + if (clone == NULL) + GOTO_FAIL("cannot clone data\n"); + + data = rte_pktmbuf_mtod(clone, unaligned_uint32_t *); + if (*data != MAGIC_DATA) + GOTO_FAIL("invalid data in clone\n"); + + data = rte_pktmbuf_mtod(clone->next, unaligned_uint32_t *); + if (*data != MAGIC_DATA) + GOTO_FAIL("invalid data in clone->next\n"); + + if (rte_mbuf_refcnt_read(m) != 2) + GOTO_FAIL("invalid refcnt in m\n"); + + if (rte_mbuf_refcnt_read(m->next) != 2) + GOTO_FAIL("invalid refcnt in m->next\n"); + + /* try to clone the clone */ + + clone2 = rte_pktmbuf_clone(clone, pktmbuf_pool); + if (clone2 == NULL) + GOTO_FAIL("cannot clone the clone\n"); + + data = rte_pktmbuf_mtod(clone2, unaligned_uint32_t *); + if (*data != MAGIC_DATA) + GOTO_FAIL("invalid data in clone2\n"); + + data = rte_pktmbuf_mtod(clone2->next, unaligned_uint32_t *); + if (*data != MAGIC_DATA) + GOTO_FAIL("invalid data in clone2->next\n"); + + if (rte_mbuf_refcnt_read(m) != 3) + GOTO_FAIL("invalid refcnt in m\n"); + + if (rte_mbuf_refcnt_read(m->next) != 3) + GOTO_FAIL("invalid refcnt in m->next\n"); + + /* free mbuf */ + rte_pktmbuf_free(m); + rte_pktmbuf_free(clone); + rte_pktmbuf_free(clone2); + + m = NULL; + clone = NULL; + clone2 = NULL; + printf("%s ok\n", __func__); + return 0; + +fail: + if (m) + rte_pktmbuf_free(m); + if (clone) + rte_pktmbuf_free(clone); + if (clone2) + rte_pktmbuf_free(clone2); + return -1; +} + +static int +test_attach_from_different_pool(struct rte_mempool *pktmbuf_pool, + struct rte_mempool *pktmbuf_pool2) +{ + struct rte_mbuf *m = NULL; + struct rte_mbuf *clone = NULL; + struct rte_mbuf *clone2 = NULL; + char *data, *c_data, *c_data2; + + /* alloc a mbuf */ + m = rte_pktmbuf_alloc(pktmbuf_pool); + if (m == NULL) + GOTO_FAIL("cannot allocate mbuf"); + + if (rte_pktmbuf_pkt_len(m) != 0) + GOTO_FAIL("Bad length"); + + data = rte_pktmbuf_mtod(m, char *); + + /* allocate a new mbuf from the second pool, and attach it to the first + * mbuf */ + clone = rte_pktmbuf_alloc(pktmbuf_pool2); + if (clone == NULL) + GOTO_FAIL("cannot allocate mbuf from second pool\n"); + + /* check data room size and priv size, and erase priv */ + if (rte_pktmbuf_data_room_size(clone->pool) != 0) + GOTO_FAIL("data room size should be 0\n"); + if (rte_pktmbuf_priv_size(clone->pool) != MBUF2_PRIV_SIZE) + GOTO_FAIL("data room size should be %d\n", MBUF2_PRIV_SIZE); + memset(clone + 1, 0, MBUF2_PRIV_SIZE); + + /* save data pointer to compare it after detach() */ + c_data = rte_pktmbuf_mtod(clone, char *); + if (c_data != (char *)clone + sizeof(*clone) + MBUF2_PRIV_SIZE) + GOTO_FAIL("bad data pointer in clone"); + if (rte_pktmbuf_headroom(clone) != 0) + GOTO_FAIL("bad headroom in clone"); + + rte_pktmbuf_attach(clone, m); + + if (rte_pktmbuf_mtod(clone, char *) != data) + GOTO_FAIL("clone was not attached properly\n"); + if (rte_pktmbuf_headroom(clone) != RTE_PKTMBUF_HEADROOM) + GOTO_FAIL("bad headroom in clone after attach"); + if (rte_mbuf_refcnt_read(m) != 2) + GOTO_FAIL("invalid refcnt in m\n"); + + /* allocate a new mbuf from the second pool, and attach it to the first + * cloned mbuf */ + clone2 = rte_pktmbuf_alloc(pktmbuf_pool2); + if (clone2 == NULL) + GOTO_FAIL("cannot allocate clone2 from second pool\n"); + + /* check data room size and priv size, and erase priv */ + if (rte_pktmbuf_data_room_size(clone2->pool) != 0) + GOTO_FAIL("data room size should be 0\n"); + if (rte_pktmbuf_priv_size(clone2->pool) != MBUF2_PRIV_SIZE) + GOTO_FAIL("data room size should be %d\n", MBUF2_PRIV_SIZE); + memset(clone2 + 1, 0, MBUF2_PRIV_SIZE); + + /* save data pointer to compare it after detach() */ + c_data2 = rte_pktmbuf_mtod(clone2, char *); + if (c_data2 != (char *)clone2 + sizeof(*clone2) + MBUF2_PRIV_SIZE) + GOTO_FAIL("bad data pointer in clone2"); + if (rte_pktmbuf_headroom(clone2) != 0) + GOTO_FAIL("bad headroom in clone2"); + + rte_pktmbuf_attach(clone2, clone); + + if (rte_pktmbuf_mtod(clone2, char *) != data) + GOTO_FAIL("clone2 was not attached properly\n"); + if (rte_pktmbuf_headroom(clone2) != RTE_PKTMBUF_HEADROOM) + GOTO_FAIL("bad headroom in clone2 after attach"); + if (rte_mbuf_refcnt_read(m) != 3) + GOTO_FAIL("invalid refcnt in m\n"); + + /* detach the clones */ + rte_pktmbuf_detach(clone); + if (c_data != rte_pktmbuf_mtod(clone, char *)) + GOTO_FAIL("clone was not detached properly\n"); + if (rte_mbuf_refcnt_read(m) != 2) + GOTO_FAIL("invalid refcnt in m\n"); + + rte_pktmbuf_detach(clone2); + if (c_data2 != rte_pktmbuf_mtod(clone2, char *)) + GOTO_FAIL("clone2 was not detached properly\n"); + if (rte_mbuf_refcnt_read(m) != 1) + GOTO_FAIL("invalid refcnt in m\n"); + + /* free the clones and the initial mbuf */ + rte_pktmbuf_free(clone2); + rte_pktmbuf_free(clone); + rte_pktmbuf_free(m); + printf("%s ok\n", __func__); + return 0; + +fail: + if (m) + rte_pktmbuf_free(m); + if (clone) + rte_pktmbuf_free(clone); + if (clone2) + rte_pktmbuf_free(clone2); + return -1; +} +#undef GOTO_FAIL + +/* + * test allocation and free of mbufs + */ +static int +test_pktmbuf_pool(struct rte_mempool *pktmbuf_pool) +{ + unsigned i; + struct rte_mbuf *m[NB_MBUF]; + int ret = 0; + + for (i=0; i<NB_MBUF; i++) + m[i] = NULL; + + /* alloc NB_MBUF mbufs */ + for (i=0; i<NB_MBUF; i++) { + m[i] = rte_pktmbuf_alloc(pktmbuf_pool); + if (m[i] == NULL) { + printf("rte_pktmbuf_alloc() failed (%u)\n", i); + ret = -1; + } + } + struct rte_mbuf *extra = NULL; + extra = rte_pktmbuf_alloc(pktmbuf_pool); + if(extra != NULL) { + printf("Error pool not empty"); + ret = -1; + } + extra = rte_pktmbuf_clone(m[0], pktmbuf_pool); + if(extra != NULL) { + printf("Error pool not empty"); + ret = -1; + } + /* free them */ + for (i=0; i<NB_MBUF; i++) { + if (m[i] != NULL) + rte_pktmbuf_free(m[i]); + } + + return ret; +} + +/* + * test that the pointer to the data on a packet mbuf is set properly + */ +static int +test_pktmbuf_pool_ptr(struct rte_mempool *pktmbuf_pool) +{ + unsigned i; + struct rte_mbuf *m[NB_MBUF]; + int ret = 0; + + for (i=0; i<NB_MBUF; i++) + m[i] = NULL; + + /* alloc NB_MBUF mbufs */ + for (i=0; i<NB_MBUF; i++) { + m[i] = rte_pktmbuf_alloc(pktmbuf_pool); + if (m[i] == NULL) { + printf("rte_pktmbuf_alloc() failed (%u)\n", i); + ret = -1; + break; + } + m[i]->data_off += 64; + } + + /* free them */ + for (i=0; i<NB_MBUF; i++) { + if (m[i] != NULL) + rte_pktmbuf_free(m[i]); + } + + for (i=0; i<NB_MBUF; i++) + m[i] = NULL; + + /* alloc NB_MBUF mbufs */ + for (i=0; i<NB_MBUF; i++) { + m[i] = rte_pktmbuf_alloc(pktmbuf_pool); + if (m[i] == NULL) { + printf("rte_pktmbuf_alloc() failed (%u)\n", i); + ret = -1; + break; + } + if (m[i]->data_off != RTE_PKTMBUF_HEADROOM) { + printf("invalid data_off\n"); + ret = -1; + } + } + + /* free them */ + for (i=0; i<NB_MBUF; i++) { + if (m[i] != NULL) + rte_pktmbuf_free(m[i]); + } + + return ret; +} + +static int +test_pktmbuf_free_segment(struct rte_mempool *pktmbuf_pool) +{ + unsigned i; + struct rte_mbuf *m[NB_MBUF]; + int ret = 0; + + for (i=0; i<NB_MBUF; i++) + m[i] = NULL; + + /* alloc NB_MBUF mbufs */ + for (i=0; i<NB_MBUF; i++) { + m[i] = rte_pktmbuf_alloc(pktmbuf_pool); + if (m[i] == NULL) { + printf("rte_pktmbuf_alloc() failed (%u)\n", i); + ret = -1; + } + } + + /* free them */ + for (i=0; i<NB_MBUF; i++) { + if (m[i] != NULL) { + struct rte_mbuf *mb, *mt; + + mb = m[i]; + while(mb != NULL) { + mt = mb; + mb = mb->next; + rte_pktmbuf_free_seg(mt); + } + } + } + + return ret; +} + +/* + * Stress test for rte_mbuf atomic refcnt. + * Implies that RTE_MBUF_REFCNT_ATOMIC is defined. + * For more efficiency, recommended to run with RTE_LIBRTE_MBUF_DEBUG defined. + */ + +#ifdef RTE_MBUF_REFCNT_ATOMIC + +static int +test_refcnt_slave(void *arg) +{ + unsigned lcore, free; + void *mp = 0; + struct rte_ring *refcnt_mbuf_ring = arg; + + lcore = rte_lcore_id(); + printf("%s started at lcore %u\n", __func__, lcore); + + free = 0; + while (refcnt_stop_slaves == 0) { + if (rte_ring_dequeue(refcnt_mbuf_ring, &mp) == 0) { + free++; + rte_pktmbuf_free(mp); + } + } + + refcnt_lcore[lcore] += free; + printf("%s finished at lcore %u, " + "number of freed mbufs: %u\n", + __func__, lcore, free); + return 0; +} + +static void +test_refcnt_iter(unsigned int lcore, unsigned int iter, + struct rte_mempool *refcnt_pool, + struct rte_ring *refcnt_mbuf_ring) +{ + uint16_t ref; + unsigned i, n, tref, wn; + struct rte_mbuf *m; + + tref = 0; + + /* For each mbuf in the pool: + * - allocate mbuf, + * - increment it's reference up to N+1, + * - enqueue it N times into the ring for slave cores to free. + */ + for (i = 0, n = rte_mempool_avail_count(refcnt_pool); + i != n && (m = rte_pktmbuf_alloc(refcnt_pool)) != NULL; + i++) { + ref = RTE_MAX(rte_rand() % REFCNT_MAX_REF, 1UL); + tref += ref; + if ((ref & 1) != 0) { + rte_pktmbuf_refcnt_update(m, ref); + while (ref-- != 0) + rte_ring_enqueue(refcnt_mbuf_ring, m); + } else { + while (ref-- != 0) { + rte_pktmbuf_refcnt_update(m, 1); + rte_ring_enqueue(refcnt_mbuf_ring, m); + } + } + rte_pktmbuf_free(m); + } + + if (i != n) + rte_panic("(lcore=%u, iter=%u): was able to allocate only " + "%u from %u mbufs\n", lcore, iter, i, n); + + /* wait till slave lcores will consume all mbufs */ + while (!rte_ring_empty(refcnt_mbuf_ring)) + ; + + /* check that all mbufs are back into mempool by now */ + for (wn = 0; wn != REFCNT_MAX_TIMEOUT; wn++) { + if ((i = rte_mempool_avail_count(refcnt_pool)) == n) { + refcnt_lcore[lcore] += tref; + printf("%s(lcore=%u, iter=%u) completed, " + "%u references processed\n", + __func__, lcore, iter, tref); + return; + } + rte_delay_ms(100); + } + + rte_panic("(lcore=%u, iter=%u): after %us only " + "%u of %u mbufs left free\n", lcore, iter, wn, i, n); +} + +static int +test_refcnt_master(struct rte_mempool *refcnt_pool, + struct rte_ring *refcnt_mbuf_ring) +{ + unsigned i, lcore; + + lcore = rte_lcore_id(); + printf("%s started at lcore %u\n", __func__, lcore); + + for (i = 0; i != REFCNT_MAX_ITER; i++) + test_refcnt_iter(lcore, i, refcnt_pool, refcnt_mbuf_ring); + + refcnt_stop_slaves = 1; + rte_wmb(); + + printf("%s finished at lcore %u\n", __func__, lcore); + return 0; +} + +#endif + +static int +test_refcnt_mbuf(void) +{ +#ifdef RTE_MBUF_REFCNT_ATOMIC + unsigned lnum, master, slave, tref; + int ret = -1; + struct rte_mempool *refcnt_pool = NULL; + struct rte_ring *refcnt_mbuf_ring = NULL; + + if ((lnum = rte_lcore_count()) == 1) { + printf("skipping %s, number of lcores: %u is not enough\n", + __func__, lnum); + return 0; + } + + printf("starting %s, at %u lcores\n", __func__, lnum); + + /* create refcnt pool & ring if they don't exist */ + + refcnt_pool = rte_pktmbuf_pool_create(MAKE_STRING(refcnt_pool), + REFCNT_MBUF_NUM, 0, 0, 0, + SOCKET_ID_ANY); + if (refcnt_pool == NULL) { + printf("%s: cannot allocate " MAKE_STRING(refcnt_pool) "\n", + __func__); + return -1; + } + + refcnt_mbuf_ring = rte_ring_create("refcnt_mbuf_ring", + rte_align32pow2(REFCNT_RING_SIZE), SOCKET_ID_ANY, + RING_F_SP_ENQ); + if (refcnt_mbuf_ring == NULL) { + printf("%s: cannot allocate " MAKE_STRING(refcnt_mbuf_ring) + "\n", __func__); + goto err; + } + + refcnt_stop_slaves = 0; + memset(refcnt_lcore, 0, sizeof (refcnt_lcore)); + + rte_eal_mp_remote_launch(test_refcnt_slave, refcnt_mbuf_ring, + SKIP_MASTER); + + test_refcnt_master(refcnt_pool, refcnt_mbuf_ring); + + rte_eal_mp_wait_lcore(); + + /* check that we porcessed all references */ + tref = 0; + master = rte_get_master_lcore(); + + RTE_LCORE_FOREACH_SLAVE(slave) + tref += refcnt_lcore[slave]; + + if (tref != refcnt_lcore[master]) + rte_panic("refernced mbufs: %u, freed mbufs: %u\n", + tref, refcnt_lcore[master]); + + rte_mempool_dump(stdout, refcnt_pool); + rte_ring_dump(stdout, refcnt_mbuf_ring); + + ret = 0; + +err: + rte_mempool_free(refcnt_pool); + rte_ring_free(refcnt_mbuf_ring); + return ret; +#else + return 0; +#endif +} + +#include <unistd.h> +#include <sys/wait.h> + +/* use fork() to test mbuf errors panic */ +static int +verify_mbuf_check_panics(struct rte_mbuf *buf) +{ + int pid; + int status; + + pid = fork(); + + if (pid == 0) { + rte_mbuf_sanity_check(buf, 1); /* should panic */ + exit(0); /* return normally if it doesn't panic */ + } else if (pid < 0){ + printf("Fork Failed\n"); + return -1; + } + wait(&status); + if(status == 0) + return -1; + + return 0; +} + +static int +test_failing_mbuf_sanity_check(struct rte_mempool *pktmbuf_pool) +{ + struct rte_mbuf *buf; + struct rte_mbuf badbuf; + + printf("Checking rte_mbuf_sanity_check for failure conditions\n"); + + /* get a good mbuf to use to make copies */ + buf = rte_pktmbuf_alloc(pktmbuf_pool); + if (buf == NULL) + return -1; + printf("Checking good mbuf initially\n"); + if (verify_mbuf_check_panics(buf) != -1) + return -1; + + printf("Now checking for error conditions\n"); + + if (verify_mbuf_check_panics(NULL)) { + printf("Error with NULL mbuf test\n"); + return -1; + } + + badbuf = *buf; + badbuf.pool = NULL; + if (verify_mbuf_check_panics(&badbuf)) { + printf("Error with bad-pool mbuf test\n"); + return -1; + } + + badbuf = *buf; + badbuf.buf_iova = 0; + if (verify_mbuf_check_panics(&badbuf)) { + printf("Error with bad-physaddr mbuf test\n"); + return -1; + } + + badbuf = *buf; + badbuf.buf_addr = NULL; + if (verify_mbuf_check_panics(&badbuf)) { + printf("Error with bad-addr mbuf test\n"); + return -1; + } + + badbuf = *buf; + badbuf.refcnt = 0; + if (verify_mbuf_check_panics(&badbuf)) { + printf("Error with bad-refcnt(0) mbuf test\n"); + return -1; + } + + badbuf = *buf; + badbuf.refcnt = UINT16_MAX; + if (verify_mbuf_check_panics(&badbuf)) { + printf("Error with bad-refcnt(MAX) mbuf test\n"); + return -1; + } + + return 0; +} + +static int +test_mbuf_linearize(struct rte_mempool *pktmbuf_pool, int pkt_len, + int nb_segs) +{ + + struct rte_mbuf *m = NULL, *mbuf = NULL; + uint8_t *data; + int data_len = 0; + int remain; + int seg, seg_len; + int i; + + if (pkt_len < 1) { + printf("Packet size must be 1 or more (is %d)\n", pkt_len); + return -1; + } + + if (nb_segs < 1) { + printf("Number of segments must be 1 or more (is %d)\n", + nb_segs); + return -1; + } + + seg_len = pkt_len / nb_segs; + if (seg_len == 0) + seg_len = 1; + + remain = pkt_len; + + /* Create chained mbuf_src and fill it generated data */ + for (seg = 0; remain > 0; seg++) { + + m = rte_pktmbuf_alloc(pktmbuf_pool); + if (m == NULL) { + printf("Cannot create segment for source mbuf"); + goto fail; + } + + /* Make sure if tailroom is zeroed */ + memset(rte_pktmbuf_mtod(m, uint8_t *), 0, + rte_pktmbuf_tailroom(m)); + + data_len = remain; + if (data_len > seg_len) + data_len = seg_len; + + data = (uint8_t *)rte_pktmbuf_append(m, data_len); + if (data == NULL) { + printf("Cannot append %d bytes to the mbuf\n", + data_len); + goto fail; + } + + for (i = 0; i < data_len; i++) + data[i] = (seg * seg_len + i) % 0x0ff; + + if (seg == 0) + mbuf = m; + else + rte_pktmbuf_chain(mbuf, m); + + remain -= data_len; + } + + /* Create destination buffer to store coalesced data */ + if (rte_pktmbuf_linearize(mbuf)) { + printf("Mbuf linearization failed\n"); + goto fail; + } + + if (!rte_pktmbuf_is_contiguous(mbuf)) { + printf("Source buffer should be contiguous after " + "linearization\n"); + goto fail; + } + + data = rte_pktmbuf_mtod(mbuf, uint8_t *); + + for (i = 0; i < pkt_len; i++) + if (data[i] != (i % 0x0ff)) { + printf("Incorrect data in linearized mbuf\n"); + goto fail; + } + + rte_pktmbuf_free(mbuf); + return 0; + +fail: + if (mbuf) + rte_pktmbuf_free(mbuf); + return -1; +} + +static int +test_mbuf_linearize_check(struct rte_mempool *pktmbuf_pool) +{ + struct test_mbuf_array { + int size; + int nb_segs; + } mbuf_array[] = { + { 128, 1 }, + { 64, 64 }, + { 512, 10 }, + { 250, 11 }, + { 123, 8 }, + }; + unsigned int i; + + printf("Test mbuf linearize API\n"); + + for (i = 0; i < RTE_DIM(mbuf_array); i++) + if (test_mbuf_linearize(pktmbuf_pool, mbuf_array[i].size, + mbuf_array[i].nb_segs)) { + printf("Test failed for %d, %d\n", mbuf_array[i].size, + mbuf_array[i].nb_segs); + return -1; + } + + return 0; +} + +static int +test_mbuf(void) +{ + int ret = -1; + struct rte_mempool *pktmbuf_pool = NULL; + struct rte_mempool *pktmbuf_pool2 = NULL; + + + RTE_BUILD_BUG_ON(sizeof(struct rte_mbuf) != RTE_CACHE_LINE_MIN_SIZE * 2); + + /* create pktmbuf pool if it does not exist */ + pktmbuf_pool = rte_pktmbuf_pool_create("test_pktmbuf_pool", + NB_MBUF, 32, 0, MBUF_DATA_SIZE, SOCKET_ID_ANY); + + if (pktmbuf_pool == NULL) { + printf("cannot allocate mbuf pool\n"); + goto err; + } + + /* create a specific pktmbuf pool with a priv_size != 0 and no data + * room size */ + pktmbuf_pool2 = rte_pktmbuf_pool_create("test_pktmbuf_pool2", + NB_MBUF, 32, MBUF2_PRIV_SIZE, 0, SOCKET_ID_ANY); + + if (pktmbuf_pool2 == NULL) { + printf("cannot allocate mbuf pool\n"); + goto err; + } + + /* test multiple mbuf alloc */ + if (test_pktmbuf_pool(pktmbuf_pool) < 0) { + printf("test_mbuf_pool() failed\n"); + goto err; + } + + /* do it another time to check that all mbufs were freed */ + if (test_pktmbuf_pool(pktmbuf_pool) < 0) { + printf("test_mbuf_pool() failed (2)\n"); + goto err; + } + + /* test that the pointer to the data on a packet mbuf is set properly */ + if (test_pktmbuf_pool_ptr(pktmbuf_pool) < 0) { + printf("test_pktmbuf_pool_ptr() failed\n"); + goto err; + } + + /* test data manipulation in mbuf */ + if (test_one_pktmbuf(pktmbuf_pool) < 0) { + printf("test_one_mbuf() failed\n"); + goto err; + } + + + /* + * do it another time, to check that allocation reinitialize + * the mbuf correctly + */ + if (test_one_pktmbuf(pktmbuf_pool) < 0) { + printf("test_one_mbuf() failed (2)\n"); + goto err; + } + + if (test_pktmbuf_with_non_ascii_data(pktmbuf_pool) < 0) { + printf("test_pktmbuf_with_non_ascii_data() failed\n"); + goto err; + } + + /* test free pktmbuf segment one by one */ + if (test_pktmbuf_free_segment(pktmbuf_pool) < 0) { + printf("test_pktmbuf_free_segment() failed.\n"); + goto err; + } + + if (testclone_testupdate_testdetach(pktmbuf_pool) < 0) { + printf("testclone_and_testupdate() failed \n"); + goto err; + } + + if (test_attach_from_different_pool(pktmbuf_pool, pktmbuf_pool2) < 0) { + printf("test_attach_from_different_pool() failed\n"); + goto err; + } + + if (test_refcnt_mbuf()<0){ + printf("test_refcnt_mbuf() failed \n"); + goto err; + } + + if (test_failing_mbuf_sanity_check(pktmbuf_pool) < 0) { + printf("test_failing_mbuf_sanity_check() failed\n"); + goto err; + } + + if (test_mbuf_linearize_check(pktmbuf_pool) < 0) { + printf("test_mbuf_linearize_check() failed\n"); + goto err; + } + ret = 0; + +err: + rte_mempool_free(pktmbuf_pool); + rte_mempool_free(pktmbuf_pool2); + return ret; +} + +REGISTER_TEST_COMMAND(mbuf_autotest, test_mbuf); diff --git a/dpdk/test/test/test_member.c b/dpdk/test/test/test_member.c new file mode 100644 index 00000000..02375fdc --- /dev/null +++ b/dpdk/test/test/test_member.c @@ -0,0 +1,744 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* This test is for membership library's simple feature test */ + +#include <rte_memcpy.h> +#include <rte_malloc.h> +#include <rte_member.h> +#include <rte_byteorder.h> +#include <rte_random.h> +#include <rte_debug.h> +#include <rte_ip.h> + +#include "test.h" + +struct rte_member_setsum *setsum_ht; +struct rte_member_setsum *setsum_cache; +struct rte_member_setsum *setsum_vbf; + +/* 5-tuple key type */ +struct flow_key { + uint32_t ip_src; + uint32_t ip_dst; + uint16_t port_src; + uint16_t port_dst; + uint8_t proto; +} __attribute__((packed)); + +/* Set ID Macros for multimatch test usage */ +#define M_MATCH_S 1 /* Not start with 0 since by default 0 means no match */ +#define M_MATCH_E 15 +#define M_MATCH_STEP 2 +#define M_MATCH_CNT \ + (1 + (M_MATCH_E - M_MATCH_S) / M_MATCH_STEP) + + +#define NUM_SAMPLES 5 +#define MAX_MATCH 32 + +/* Keys used by unit test functions */ +static struct flow_key keys[NUM_SAMPLES] = { + { + .ip_src = IPv4(0x03, 0x02, 0x01, 0x00), + .ip_dst = IPv4(0x07, 0x06, 0x05, 0x04), + .port_src = 0x0908, + .port_dst = 0x0b0a, + .proto = 0x0c, + }, + { + .ip_src = IPv4(0x13, 0x12, 0x11, 0x10), + .ip_dst = IPv4(0x17, 0x16, 0x15, 0x14), + .port_src = 0x1918, + .port_dst = 0x1b1a, + .proto = 0x1c, + }, + { + .ip_src = IPv4(0x23, 0x22, 0x21, 0x20), + .ip_dst = IPv4(0x27, 0x26, 0x25, 0x24), + .port_src = 0x2928, + .port_dst = 0x2b2a, + .proto = 0x2c, + }, + { + .ip_src = IPv4(0x33, 0x32, 0x31, 0x30), + .ip_dst = IPv4(0x37, 0x36, 0x35, 0x34), + .port_src = 0x3938, + .port_dst = 0x3b3a, + .proto = 0x3c, + }, + { + .ip_src = IPv4(0x43, 0x42, 0x41, 0x40), + .ip_dst = IPv4(0x47, 0x46, 0x45, 0x44), + .port_src = 0x4948, + .port_dst = 0x4b4a, + .proto = 0x4c, + } +}; + +uint32_t test_set[NUM_SAMPLES] = {1, 2, 3, 4, 5}; + +#define ITERATIONS 3 +#define KEY_SIZE 4 + +#define MAX_ENTRIES (1 << 16) +uint8_t generated_keys[MAX_ENTRIES][KEY_SIZE]; + +static struct rte_member_parameters params = { + .num_keys = MAX_ENTRIES, /* Total hash table entries. */ + .key_len = KEY_SIZE, /* Length of hash key. */ + + /* num_set and false_positive_rate only relevant to vBF */ + .num_set = 16, + .false_positive_rate = 0.03, + .prim_hash_seed = 1, + .sec_hash_seed = 11, + .socket_id = 0 /* NUMA Socket ID for memory. */ +}; + +/* + * Sequence of operations for find existing setsummary + * + * - create setsum + * - find existing setsum: hit + * - find non-existing setsum: miss + * + */ +static int +test_member_find_existing(void) +{ + struct rte_member_setsum *tmp_setsum = NULL, *result = NULL; + struct rte_member_parameters tmp_params = { + .name = "member_find_existing", + .num_keys = MAX_ENTRIES, /* Total hash table entries. */ + .key_len = KEY_SIZE, /* Length of hash key. */ + .type = RTE_MEMBER_TYPE_HT, + .num_set = 32, + .false_positive_rate = 0.03, + .prim_hash_seed = 1, + .sec_hash_seed = 11, + .socket_id = 0 /* NUMA Socket ID for memory. */ + }; + + /* Create */ + tmp_setsum = rte_member_create(&tmp_params); + TEST_ASSERT(tmp_setsum != NULL, "setsum creation failed"); + + /* Try to find existing hash table */ + result = rte_member_find_existing("member_find_existing"); + TEST_ASSERT(result == tmp_setsum, "could not find existing setsum"); + + /* Try to find non-existing hash table */ + result = rte_member_find_existing("member_find_non_existing"); + TEST_ASSERT(result == NULL, "found setsum that shouldn't exist"); + + /* Cleanup. */ + rte_member_free(tmp_setsum); + + return 0; +} + +/* + * Test for bad creating parameters + */ +static int +test_member_create_bad_param(void) +{ + struct rte_member_setsum *bad_setsum = NULL; + struct rte_member_parameters bad_params = { + .num_keys = MAX_ENTRIES, /* Total hash table entries. */ + .key_len = KEY_SIZE, /* Length of hash key. */ + .type = RTE_MEMBER_TYPE_HT, + .num_set = 32, + .false_positive_rate = 0.03, + .prim_hash_seed = 1, + .sec_hash_seed = 11, + .socket_id = 0 /* NUMA Socket ID for memory. */ + }; + + printf("Expected error section begin...\n"); + bad_params.name = "bad_param1"; + bad_params.num_set = 0; + bad_params.type = RTE_MEMBER_TYPE_VBF; + /* Test with 0 set for vBF should fail */ + bad_setsum = rte_member_create(&bad_params); + if (bad_setsum != NULL) { + rte_member_free(bad_setsum); + printf("Impossible creating setsum successfully with invalid " + "number of set for vBF\n"); + return -1; + } + + bad_params.name = "bad_param2"; + bad_params.false_positive_rate = 0; + bad_params.num_set = 32; + /* Test with 0 false positive for vBF should fail */ + bad_setsum = rte_member_create(&bad_params); + if (bad_setsum != NULL) { + rte_member_free(bad_setsum); + printf("Impossible creating setsum successfully with invalid " + "false positive rate for vBF\n"); + return -1; + } + + bad_params.name = "bad_param3"; + bad_params.false_positive_rate = 0.03; + bad_params.num_keys = 0; + /* Test with 0 key per BF for vBF should fail */ + bad_setsum = rte_member_create(&bad_params); + if (bad_setsum != NULL) { + rte_member_free(bad_setsum); + printf("Impossible creating setsum successfully with invalid " + "num_keys for vBF\n"); + return -1; + } + + bad_params.name = "bad_param4"; + bad_params.type = RTE_MEMBER_TYPE_HT; + bad_params.num_keys = RTE_MEMBER_BUCKET_ENTRIES / 2; + /* Test with less than 1 bucket for HTSS should fail */ + bad_setsum = rte_member_create(&bad_params); + if (bad_setsum != NULL) { + rte_member_free(bad_setsum); + printf("Impossible creating setsum successfully with too few " + "number of keys(entries) for HT\n"); + return -1; + } + + bad_params.name = "bad_param5"; + bad_params.num_keys = RTE_MEMBER_ENTRIES_MAX + 1; + /* Test with more than maximum entries for HTSS should fail */ + bad_setsum = rte_member_create(&bad_params); + if (bad_setsum != NULL) { + rte_member_free(bad_setsum); + printf("Impossible creating setsum successfully with to many " + "number of keys(entries) for HT\n"); + return -1; + } + + bad_params.name = "bad_param5"; + /* Test with same name should fail */ + bad_setsum = rte_member_create(&bad_params); + if (bad_setsum != NULL) { + rte_member_free(bad_setsum); + printf("Impossible creating setsum successfully with existed " + "name\n"); + return -1; + } + printf("Expected error section end...\n"); + rte_member_free(bad_setsum); + return 0; +} + +/* Create test setsummaries. */ +static int test_member_create(void) +{ + params.key_len = sizeof(struct flow_key); + + params.name = "test_member_ht"; + params.is_cache = 0; + params.type = RTE_MEMBER_TYPE_HT; + setsum_ht = rte_member_create(¶ms); + + params.name = "test_member_cache"; + params.is_cache = 1; + setsum_cache = rte_member_create(¶ms); + + params.name = "test_member_vbf"; + params.type = RTE_MEMBER_TYPE_VBF; + setsum_vbf = rte_member_create(¶ms); + + if (setsum_ht == NULL || setsum_cache == NULL || setsum_vbf == NULL) { + printf("Creation of setsums fail\n"); + return -1; + } + printf("Creation of setsums success\n"); + return 0; +} + +static int test_member_insert(void) +{ + int ret_ht, ret_cache, ret_vbf, i; + + for (i = 0; i < NUM_SAMPLES; i++) { + ret_ht = rte_member_add(setsum_ht, &keys[i], test_set[i]); + ret_cache = rte_member_add(setsum_cache, &keys[i], + test_set[i]); + ret_vbf = rte_member_add(setsum_vbf, &keys[i], test_set[i]); + TEST_ASSERT(ret_ht >= 0 && ret_cache >= 0 && ret_vbf >= 0, + "insert error"); + } + printf("insert key success\n"); + return 0; +} + +static int test_member_lookup(void) +{ + int ret_ht, ret_cache, ret_vbf, i; + uint16_t set_ht, set_cache, set_vbf; + member_set_t set_ids_ht[NUM_SAMPLES] = {0}; + member_set_t set_ids_cache[NUM_SAMPLES] = {0}; + member_set_t set_ids_vbf[NUM_SAMPLES] = {0}; + + uint32_t num_key_ht = NUM_SAMPLES; + uint32_t num_key_cache = NUM_SAMPLES; + uint32_t num_key_vbf = NUM_SAMPLES; + + const void *key_array[NUM_SAMPLES]; + + /* Single lookup test */ + for (i = 0; i < NUM_SAMPLES; i++) { + ret_ht = rte_member_lookup(setsum_ht, &keys[i], &set_ht); + ret_cache = rte_member_lookup(setsum_cache, &keys[i], + &set_cache); + ret_vbf = rte_member_lookup(setsum_vbf, &keys[i], &set_vbf); + TEST_ASSERT(ret_ht >= 0 && ret_cache >= 0 && ret_vbf >= 0, + "single lookup function error"); + + TEST_ASSERT(set_ht == test_set[i] && + set_cache == test_set[i] && + set_vbf == test_set[i], + "single lookup set value error"); + } + printf("lookup single key success\n"); + + /* Bulk lookup test */ + for (i = 0; i < NUM_SAMPLES; i++) + key_array[i] = &keys[i]; + + ret_ht = rte_member_lookup_bulk(setsum_ht, key_array, + num_key_ht, set_ids_ht); + + ret_cache = rte_member_lookup_bulk(setsum_cache, key_array, + num_key_cache, set_ids_cache); + + ret_vbf = rte_member_lookup_bulk(setsum_vbf, key_array, + num_key_vbf, set_ids_vbf); + + TEST_ASSERT(ret_ht >= 0 && ret_cache >= 0 && ret_vbf >= 0, + "bulk lookup function error"); + + for (i = 0; i < NUM_SAMPLES; i++) { + TEST_ASSERT((set_ids_ht[i] == test_set[i]) && + (set_ids_cache[i] == test_set[i]) && + (set_ids_vbf[i] == test_set[i]), + "bulk lookup result error"); + } + + return 0; +} + +static int test_member_delete(void) +{ + int ret_ht, ret_cache, ret_vbf, i; + uint16_t set_ht, set_cache, set_vbf; + const void *key_array[NUM_SAMPLES]; + member_set_t set_ids_ht[NUM_SAMPLES] = {0}; + member_set_t set_ids_cache[NUM_SAMPLES] = {0}; + member_set_t set_ids_vbf[NUM_SAMPLES] = {0}; + uint32_t num_key_ht = NUM_SAMPLES; + uint32_t num_key_cache = NUM_SAMPLES; + uint32_t num_key_vbf = NUM_SAMPLES; + + /* Delete part of all inserted keys */ + for (i = 0; i < NUM_SAMPLES / 2; i++) { + ret_ht = rte_member_delete(setsum_ht, &keys[i], test_set[i]); + ret_cache = rte_member_delete(setsum_cache, &keys[i], + test_set[i]); + ret_vbf = rte_member_delete(setsum_vbf, &keys[i], test_set[i]); + /* VBF does not support delete yet, so return error code */ + TEST_ASSERT(ret_ht >= 0 && ret_cache >= 0, + "key deletion function error"); + TEST_ASSERT(ret_vbf < 0, + "vbf does not support deletion, error"); + } + + for (i = 0; i < NUM_SAMPLES; i++) + key_array[i] = &keys[i]; + + ret_ht = rte_member_lookup_bulk(setsum_ht, key_array, + num_key_ht, set_ids_ht); + + ret_cache = rte_member_lookup_bulk(setsum_cache, key_array, + num_key_cache, set_ids_cache); + + ret_vbf = rte_member_lookup_bulk(setsum_vbf, key_array, + num_key_vbf, set_ids_vbf); + + TEST_ASSERT(ret_ht >= 0 && ret_cache >= 0 && ret_vbf >= 0, + "bulk lookup function error"); + + for (i = 0; i < NUM_SAMPLES / 2; i++) { + TEST_ASSERT((set_ids_ht[i] == RTE_MEMBER_NO_MATCH) && + (set_ids_cache[i] == RTE_MEMBER_NO_MATCH), + "bulk lookup result error"); + } + + for (i = NUM_SAMPLES / 2; i < NUM_SAMPLES; i++) { + TEST_ASSERT((set_ids_ht[i] == test_set[i]) && + (set_ids_cache[i] == test_set[i]) && + (set_ids_vbf[i] == test_set[i]), + "bulk lookup result error"); + } + + /* Delete the left of inserted keys */ + for (i = NUM_SAMPLES / 2; i < NUM_SAMPLES; i++) { + ret_ht = rte_member_delete(setsum_ht, &keys[i], test_set[i]); + ret_cache = rte_member_delete(setsum_cache, &keys[i], + test_set[i]); + ret_vbf = rte_member_delete(setsum_vbf, &keys[i], test_set[i]); + /* VBF does not support delete yet, so return error code */ + TEST_ASSERT(ret_ht >= 0 && ret_cache >= 0, + "key deletion function error"); + TEST_ASSERT(ret_vbf < 0, + "vbf does not support deletion, error"); + } + + for (i = 0; i < NUM_SAMPLES; i++) { + ret_ht = rte_member_lookup(setsum_ht, &keys[i], &set_ht); + ret_cache = rte_member_lookup(setsum_cache, &keys[i], + &set_cache); + ret_vbf = rte_member_lookup(setsum_vbf, &keys[i], &set_vbf); + TEST_ASSERT(ret_ht >= 0 && ret_cache >= 0, + "key lookup function error"); + TEST_ASSERT(set_ht == RTE_MEMBER_NO_MATCH && + ret_cache == RTE_MEMBER_NO_MATCH, + "key deletion failed"); + } + /* Reset vbf for other following tests */ + rte_member_reset(setsum_vbf); + + printf("delete success\n"); + return 0; +} + +static int test_member_multimatch(void) +{ + int ret_ht, ret_vbf, ret_cache; + member_set_t set_ids_ht[MAX_MATCH] = {0}; + member_set_t set_ids_vbf[MAX_MATCH] = {0}; + member_set_t set_ids_cache[MAX_MATCH] = {0}; + + member_set_t set_ids_ht_m[NUM_SAMPLES][MAX_MATCH] = {{0} }; + member_set_t set_ids_vbf_m[NUM_SAMPLES][MAX_MATCH] = {{0} }; + member_set_t set_ids_cache_m[NUM_SAMPLES][MAX_MATCH] = {{0} }; + + uint32_t match_count_ht[NUM_SAMPLES]; + uint32_t match_count_vbf[NUM_SAMPLES]; + uint32_t match_count_cache[NUM_SAMPLES]; + + uint32_t num_key_ht = NUM_SAMPLES; + uint32_t num_key_vbf = NUM_SAMPLES; + uint32_t num_key_cache = NUM_SAMPLES; + + const void *key_array[NUM_SAMPLES]; + + uint32_t i, j; + + /* Same key at most inserted 2*entry_per_bucket times for HT mode */ + for (i = M_MATCH_S; i <= M_MATCH_E; i += M_MATCH_STEP) { + for (j = 0; j < NUM_SAMPLES; j++) { + ret_ht = rte_member_add(setsum_ht, &keys[j], i); + ret_vbf = rte_member_add(setsum_vbf, &keys[j], i); + ret_cache = rte_member_add(setsum_cache, &keys[j], i); + + TEST_ASSERT(ret_ht >= 0 && ret_vbf >= 0 && + ret_cache >= 0, + "insert function error"); + } + } + + /* Single multimatch test */ + for (i = 0; i < NUM_SAMPLES; i++) { + ret_vbf = rte_member_lookup_multi(setsum_vbf, &keys[i], + MAX_MATCH, set_ids_vbf); + ret_ht = rte_member_lookup_multi(setsum_ht, &keys[i], + MAX_MATCH, set_ids_ht); + ret_cache = rte_member_lookup_multi(setsum_cache, &keys[i], + MAX_MATCH, set_ids_cache); + /* + * For cache mode, keys overwrite when signature same. + * the mutimatch should work like single match. + */ + TEST_ASSERT(ret_ht == M_MATCH_CNT && ret_vbf == M_MATCH_CNT && + ret_cache == 1, + "single lookup_multi error"); + TEST_ASSERT(set_ids_cache[0] == M_MATCH_E, + "single lookup_multi cache error"); + + for (j = 1; j <= M_MATCH_CNT; j++) { + TEST_ASSERT(set_ids_ht[j-1] == j * M_MATCH_STEP - 1 && + set_ids_vbf[j-1] == + j * M_MATCH_STEP - 1, + "single multimatch lookup error"); + } + } + printf("lookup single key for multimatch success\n"); + + /* Bulk multimatch test */ + for (i = 0; i < NUM_SAMPLES; i++) + key_array[i] = &keys[i]; + ret_vbf = rte_member_lookup_multi_bulk(setsum_vbf, + &key_array[0], num_key_ht, MAX_MATCH, match_count_vbf, + (member_set_t *)set_ids_vbf_m); + + ret_ht = rte_member_lookup_multi_bulk(setsum_ht, + &key_array[0], num_key_vbf, MAX_MATCH, match_count_ht, + (member_set_t *)set_ids_ht_m); + + ret_cache = rte_member_lookup_multi_bulk(setsum_cache, + &key_array[0], num_key_cache, MAX_MATCH, + match_count_cache, (member_set_t *)set_ids_cache_m); + + + for (j = 0; j < NUM_SAMPLES; j++) { + TEST_ASSERT(match_count_ht[j] == M_MATCH_CNT, + "bulk multimatch lookup HT match count error"); + TEST_ASSERT(match_count_vbf[j] == M_MATCH_CNT, + "bulk multimatch lookup vBF match count error"); + TEST_ASSERT(match_count_cache[j] == 1, + "bulk multimatch lookup CACHE match count error"); + TEST_ASSERT(set_ids_cache_m[j][0] == M_MATCH_E, + "bulk multimatch lookup CACHE set value error"); + + for (i = 1; i <= M_MATCH_CNT; i++) { + TEST_ASSERT(set_ids_ht_m[j][i-1] == + i * M_MATCH_STEP - 1, + "bulk multimatch lookup HT set value error"); + TEST_ASSERT(set_ids_vbf_m[j][i-1] == + i * M_MATCH_STEP - 1, + "bulk multimatch lookup vBF set value error"); + } + } + + printf("lookup for bulk multimatch success\n"); + + return 0; +} + +static int key_compare(const void *key1, const void *key2) +{ + return memcmp(key1, key2, KEY_SIZE); +} + +static void +setup_keys_and_data(void) +{ + unsigned int i, j; + int num_duplicates; + + /* Reset all arrays */ + for (i = 0; i < KEY_SIZE; i++) + generated_keys[0][i] = 0; + + /* Generate a list of keys, some of which may be duplicates */ + for (i = 0; i < MAX_ENTRIES; i++) { + for (j = 0; j < KEY_SIZE; j++) + generated_keys[i][j] = rte_rand() & 0xFF; + } + + /* Remove duplicates from the keys array */ + do { + num_duplicates = 0; + /* Sort the list of keys to make it easier to find duplicates */ + qsort(generated_keys, MAX_ENTRIES, KEY_SIZE, key_compare); + + /* Sift through the list of keys and look for duplicates */ + int num_duplicates = 0; + for (i = 0; i < MAX_ENTRIES - 1; i++) { + if (memcmp(generated_keys[i], generated_keys[i + 1], + KEY_SIZE) == 0) { + /* This key already exists, try again */ + num_duplicates++; + for (j = 0; j < KEY_SIZE; j++) + generated_keys[i][j] = + rte_rand() & 0xFF; + } + } + } while (num_duplicates != 0); +} + +static inline int +add_generated_keys(struct rte_member_setsum *setsum, unsigned int *added_keys) +{ + int ret = 0; + + for (*added_keys = 0; ret >= 0 && *added_keys < MAX_ENTRIES; + (*added_keys)++) { + uint16_t set = (rte_rand() & 0xf) + 1; + ret = rte_member_add(setsum, &generated_keys[*added_keys], set); + } + return ret; +} + +static inline int +add_generated_keys_cache(struct rte_member_setsum *setsum, + unsigned int *added_keys) +{ + int ret = 0; + + for (*added_keys = 0; ret == 0 && *added_keys < MAX_ENTRIES; + (*added_keys)++) { + uint16_t set = (rte_rand() & 0xf) + 1; + ret = rte_member_add(setsum, &generated_keys[*added_keys], set); + } + return ret; +} + +static int +test_member_loadfactor(void) +{ + unsigned int j; + unsigned int added_keys, average_keys_added = 0; + int ret; + + setup_keys_and_data(); + + rte_member_free(setsum_ht); + rte_member_free(setsum_cache); + rte_member_free(setsum_vbf); + + params.key_len = KEY_SIZE; + params.name = "test_member_ht"; + params.is_cache = 0; + params.type = RTE_MEMBER_TYPE_HT; + setsum_ht = rte_member_create(¶ms); + + params.name = "test_member_cache"; + params.is_cache = 1; + setsum_cache = rte_member_create(¶ms); + + + if (setsum_ht == NULL || setsum_cache == NULL) { + printf("Creation of setsums fail\n"); + return -1; + } + /* Test HT non-cache mode */ + for (j = 0; j < ITERATIONS; j++) { + /* Add random entries until key cannot be added */ + ret = add_generated_keys(setsum_ht, &added_keys); + if (ret != -ENOSPC) { + printf("Unexpected error when adding keys\n"); + return -1; + } + average_keys_added += added_keys; + + /* Reset the table */ + rte_member_reset(setsum_ht); + + /* Print a dot to show progress on operations */ + printf("."); + fflush(stdout); + } + + average_keys_added /= ITERATIONS; + + printf("\nKeys inserted when no space(non-cache) = %.2f%% (%u/%u)\n", + ((double) average_keys_added / params.num_keys * 100), + average_keys_added, params.num_keys); + + /* Test cache mode */ + added_keys = average_keys_added = 0; + for (j = 0; j < ITERATIONS; j++) { + /* Add random entries until key cannot be added */ + ret = add_generated_keys_cache(setsum_cache, &added_keys); + if (ret != 1) { + printf("Unexpected error when adding keys\n"); + return -1; + } + average_keys_added += added_keys; + + /* Reset the table */ + rte_member_reset(setsum_cache); + + /* Print a dot to show progress on operations */ + printf("."); + fflush(stdout); + } + + average_keys_added /= ITERATIONS; + + printf("\nKeys inserted when eviction happens(cache)= %.2f%% (%u/%u)\n", + ((double) average_keys_added / params.num_keys * 100), + average_keys_added, params.num_keys); + return 0; +} + +static void +perform_free(void) +{ + rte_member_free(setsum_ht); + rte_member_free(setsum_cache); + rte_member_free(setsum_vbf); +} + +static int +test_member(void) +{ + if (test_member_create_bad_param() < 0) + return -1; + + if (test_member_find_existing() < 0) + return -1; + + if (test_member_create() < 0) { + perform_free(); + return -1; + } + if (test_member_insert() < 0) { + perform_free(); + return -1; + } + if (test_member_lookup() < 0) { + perform_free(); + return -1; + } + if (test_member_delete() < 0) { + perform_free(); + return -1; + } + if (test_member_multimatch() < 0) { + perform_free(); + return -1; + } + if (test_member_loadfactor() < 0) { + rte_member_free(setsum_ht); + rte_member_free(setsum_cache); + return -1; + } + + perform_free(); + return 0; +} + +REGISTER_TEST_COMMAND(member_autotest, test_member); diff --git a/dpdk/test/test/test_member_perf.c b/dpdk/test/test/test_member_perf.c new file mode 100644 index 00000000..e13066f1 --- /dev/null +++ b/dpdk/test/test/test_member_perf.c @@ -0,0 +1,654 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <inttypes.h> + +#include <rte_lcore.h> +#include <rte_cycles.h> +#include <rte_malloc.h> +#include <rte_random.h> +#include <rte_memcpy.h> +#include <rte_thash.h> +#include <rte_member.h> + +#include "test.h" + +#define NUM_KEYSIZES 10 +#define NUM_SHUFFLES 10 +#define MAX_KEYSIZE 64 +#define MAX_ENTRIES (1 << 19) +#define KEYS_TO_ADD (MAX_ENTRIES * 75 / 100) /* 75% table utilization */ +#define NUM_LOOKUPS (KEYS_TO_ADD * 5) /* Loop among keys added, several times */ +#define VBF_SET_CNT 16 +#define BURST_SIZE 64 +#define VBF_FALSE_RATE 0.03 + +static unsigned int test_socket_id; + +enum sstype { + HT = 0, + CACHE, + VBF, + NUM_TYPE +}; + +enum operations { + ADD = 0, + LOOKUP, + LOOKUP_BULK, + LOOKUP_MULTI, + LOOKUP_MULTI_BULK, + DELETE, + LOOKUP_MISS, + NUM_OPERATIONS +}; + +struct member_perf_params { + struct rte_member_setsum *setsum[NUM_TYPE]; + uint32_t key_size; + unsigned int cycle; +}; + +static uint32_t hashtest_key_lens[] = { + /* standard key sizes */ + 4, 8, 16, 32, 48, 64, + /* IPv4 SRC + DST + protocol, unpadded */ + 9, + /* IPv4 5-tuple, unpadded */ + 13, + /* IPv6 5-tuple, unpadded */ + 37, + /* IPv6 5-tuple, padded to 8-byte boundary */ + 40 +}; + +/* Array to store number of cycles per operation */ +uint64_t cycles[NUM_TYPE][NUM_KEYSIZES][NUM_OPERATIONS]; +uint64_t false_data[NUM_TYPE][NUM_KEYSIZES]; +uint64_t false_data_bulk[NUM_TYPE][NUM_KEYSIZES]; +uint64_t false_data_multi[NUM_TYPE][NUM_KEYSIZES]; +uint64_t false_data_multi_bulk[NUM_TYPE][NUM_KEYSIZES]; + +uint64_t false_hit[NUM_TYPE][NUM_KEYSIZES]; + +member_set_t data[NUM_TYPE][/* Array to store the data */KEYS_TO_ADD]; + +/* Array to store all input keys */ +uint8_t keys[KEYS_TO_ADD][MAX_KEYSIZE]; + +/* Shuffle the keys that have been added, so lookups will be totally random */ +static void +shuffle_input_keys(struct member_perf_params *params) +{ + member_set_t temp_data; + unsigned int i, j; + uint32_t swap_idx; + uint8_t temp_key[MAX_KEYSIZE]; + + for (i = KEYS_TO_ADD - 1; i > 0; i--) { + swap_idx = rte_rand() % i; + memcpy(temp_key, keys[i], hashtest_key_lens[params->cycle]); + memcpy(keys[i], keys[swap_idx], + hashtest_key_lens[params->cycle]); + memcpy(keys[swap_idx], temp_key, + hashtest_key_lens[params->cycle]); + for (j = 0; j < NUM_TYPE; j++) { + temp_data = data[j][i]; + data[j][i] = data[j][swap_idx]; + data[j][swap_idx] = temp_data; + } + } +} + +static int key_compare(const void *key1, const void *key2) +{ + return memcmp(key1, key2, MAX_KEYSIZE); +} + +struct rte_member_parameters member_params = { + .num_keys = MAX_ENTRIES, /* Total hash table entries. */ + .key_len = 4, /* Length of hash key. */ + + /* num_set and false_positive_rate only relevant to vBF */ + .num_set = VBF_SET_CNT, + .false_positive_rate = 0.03, + .prim_hash_seed = 0, + .sec_hash_seed = 1, + .socket_id = 0, /* NUMA Socket ID for memory. */ + }; + +static int +setup_keys_and_data(struct member_perf_params *params, unsigned int cycle, + int miss) +{ + unsigned int i, j; + int num_duplicates; + + params->key_size = hashtest_key_lens[cycle]; + params->cycle = cycle; + + /* Reset all arrays */ + for (i = 0; i < params->key_size; i++) + keys[0][i] = 0; + + /* Generate a list of keys, some of which may be duplicates */ + for (i = 0; i < KEYS_TO_ADD; i++) { + for (j = 0; j < params->key_size; j++) + keys[i][j] = rte_rand() & 0xFF; + + data[HT][i] = data[CACHE][i] = (rte_rand() & 0x7FFE) + 1; + data[VBF][i] = rte_rand() % VBF_SET_CNT + 1; + } + + /* Remove duplicates from the keys array */ + do { + num_duplicates = 0; + + /* Sort the list of keys to make it easier to find duplicates */ + qsort(keys, KEYS_TO_ADD, MAX_KEYSIZE, key_compare); + + /* Sift through the list of keys and look for duplicates */ + int num_duplicates = 0; + for (i = 0; i < KEYS_TO_ADD - 1; i++) { + if (memcmp(keys[i], keys[i + 1], + params->key_size) == 0) { + /* This key already exists, try again */ + num_duplicates++; + for (j = 0; j < params->key_size; j++) + keys[i][j] = rte_rand() & 0xFF; + } + } + } while (num_duplicates != 0); + + /* Shuffle the random values again */ + shuffle_input_keys(params); + + /* For testing miss lookup, we insert half and lookup the other half */ + unsigned int entry_cnt, bf_key_cnt; + if (!miss) { + entry_cnt = MAX_ENTRIES; + bf_key_cnt = KEYS_TO_ADD; + } else { + entry_cnt = MAX_ENTRIES / 2; + bf_key_cnt = KEYS_TO_ADD / 2; + } + member_params.false_positive_rate = VBF_FALSE_RATE; + member_params.key_len = params->key_size; + member_params.socket_id = test_socket_id; + member_params.num_keys = entry_cnt; + member_params.name = "test_member_ht"; + member_params.is_cache = 0; + member_params.type = RTE_MEMBER_TYPE_HT; + params->setsum[HT] = rte_member_create(&member_params); + if (params->setsum[HT] == NULL) + fprintf(stderr, "ht create fail\n"); + + member_params.name = "test_member_cache"; + member_params.is_cache = 1; + params->setsum[CACHE] = rte_member_create(&member_params); + if (params->setsum[CACHE] == NULL) + fprintf(stderr, "CACHE create fail\n"); + + member_params.name = "test_member_vbf"; + member_params.type = RTE_MEMBER_TYPE_VBF; + member_params.num_keys = bf_key_cnt; + params->setsum[VBF] = rte_member_create(&member_params); + if (params->setsum[VBF] == NULL) + fprintf(stderr, "VBF create fail\n"); + for (i = 0; i < NUM_TYPE; i++) { + if (params->setsum[i] == NULL) + return -1; + } + + return 0; +} + +static int +timed_adds(struct member_perf_params *params, int type) +{ + const uint64_t start_tsc = rte_rdtsc(); + unsigned int i, a; + int32_t ret; + + for (i = 0; i < KEYS_TO_ADD; i++) { + ret = rte_member_add(params->setsum[type], &keys[i], + data[type][i]); + if (ret < 0) { + printf("Error %d in rte_member_add - key=0x", ret); + for (a = 0; a < params->key_size; a++) + printf("%02x", keys[i][a]); + printf(" value=%d, type: %d\n", data[type][i], type); + + return -1; + } + } + + const uint64_t end_tsc = rte_rdtsc(); + const uint64_t time_taken = end_tsc - start_tsc; + + cycles[type][params->cycle][ADD] = time_taken / KEYS_TO_ADD; + return 0; +} + +static int +timed_lookups(struct member_perf_params *params, int type) +{ + unsigned int i, j; + + false_data[type][params->cycle] = 0; + + const uint64_t start_tsc = rte_rdtsc(); + member_set_t result; + int ret; + + for (i = 0; i < NUM_LOOKUPS / KEYS_TO_ADD; i++) { + for (j = 0; j < KEYS_TO_ADD; j++) { + ret = rte_member_lookup(params->setsum[type], &keys[j], + &result); + if (ret < 0) { + printf("lookup wrong internally"); + return -1; + } + if (type == HT && result == RTE_MEMBER_NO_MATCH) { + printf("HT mode shouldn't have false negative"); + return -1; + } + if (result != data[type][j]) + false_data[type][params->cycle]++; + } + } + + const uint64_t end_tsc = rte_rdtsc(); + const uint64_t time_taken = end_tsc - start_tsc; + + cycles[type][params->cycle][LOOKUP] = time_taken / NUM_LOOKUPS; + + return 0; +} + +static int +timed_lookups_bulk(struct member_perf_params *params, int type) +{ + unsigned int i, j, k; + member_set_t result[BURST_SIZE] = {0}; + const void *keys_burst[BURST_SIZE]; + int ret; + + false_data_bulk[type][params->cycle] = 0; + + const uint64_t start_tsc = rte_rdtsc(); + + for (i = 0; i < NUM_LOOKUPS / KEYS_TO_ADD; i++) { + for (j = 0; j < KEYS_TO_ADD / BURST_SIZE; j++) { + for (k = 0; k < BURST_SIZE; k++) + keys_burst[k] = keys[j * BURST_SIZE + k]; + + ret = rte_member_lookup_bulk(params->setsum[type], + keys_burst, + BURST_SIZE, + result); + if (ret <= 0) { + printf("lookup bulk has wrong return value\n"); + return -1; + } + for (k = 0; k < BURST_SIZE; k++) { + uint32_t data_idx = j * BURST_SIZE + k; + if (type == HT && result[k] == + RTE_MEMBER_NO_MATCH) { + printf("HT mode shouldn't have " + "false negative"); + return -1; + } + if (result[k] != data[type][data_idx]) + false_data_bulk[type][params->cycle]++; + } + } + } + + const uint64_t end_tsc = rte_rdtsc(); + const uint64_t time_taken = end_tsc - start_tsc; + + cycles[type][params->cycle][LOOKUP_BULK] = time_taken / NUM_LOOKUPS; + + return 0; +} + +static int +timed_lookups_multimatch(struct member_perf_params *params, int type) +{ + unsigned int i, j; + member_set_t result[RTE_MEMBER_BUCKET_ENTRIES] = {0}; + int ret; + false_data_multi[type][params->cycle] = 0; + + const uint64_t start_tsc = rte_rdtsc(); + + for (i = 0; i < NUM_LOOKUPS / KEYS_TO_ADD; i++) { + for (j = 0; j < KEYS_TO_ADD; j++) { + ret = rte_member_lookup_multi(params->setsum[type], + &keys[j], RTE_MEMBER_BUCKET_ENTRIES, result); + if (type != CACHE && ret <= 0) { + printf("lookup multi has wrong return value %d," + "type %d\n", ret, type); + } + if (type == HT && ret == 0) { + printf("HT mode shouldn't have false negative"); + return -1; + } + /* + * For performance test purpose, we do not iterate all + * results here. We assume most likely each key can only + * find one match which is result[0]. + */ + if (result[0] != data[type][j]) + false_data_multi[type][params->cycle]++; + } + } + + const uint64_t end_tsc = rte_rdtsc(); + const uint64_t time_taken = end_tsc - start_tsc; + + cycles[type][params->cycle][LOOKUP_MULTI] = time_taken / NUM_LOOKUPS; + + return 0; +} + +static int +timed_lookups_multimatch_bulk(struct member_perf_params *params, int type) +{ + unsigned int i, j, k; + member_set_t result[BURST_SIZE][RTE_MEMBER_BUCKET_ENTRIES] = {{0} }; + const void *keys_burst[BURST_SIZE]; + uint32_t match_count[BURST_SIZE]; + int ret; + + false_data_multi_bulk[type][params->cycle] = 0; + + const uint64_t start_tsc = rte_rdtsc(); + + for (i = 0; i < NUM_LOOKUPS / KEYS_TO_ADD; i++) { + for (j = 0; j < KEYS_TO_ADD / BURST_SIZE; j++) { + for (k = 0; k < BURST_SIZE; k++) + keys_burst[k] = keys[j * BURST_SIZE + k]; + + ret = rte_member_lookup_multi_bulk( + params->setsum[type], + keys_burst, BURST_SIZE, + RTE_MEMBER_BUCKET_ENTRIES, match_count, + (member_set_t *)result); + if (ret < 0) { + printf("lookup multimatch bulk has wrong return" + " value\n"); + return -1; + } + for (k = 0; k < BURST_SIZE; k++) { + if (type != CACHE && match_count[k] == 0) { + printf("lookup multimatch bulk get " + "wrong match count\n"); + return -1; + } + if (type == HT && match_count[k] == 0) { + printf("HT mode shouldn't have " + "false negative"); + return -1; + } + uint32_t data_idx = j * BURST_SIZE + k; + if (result[k][0] != data[type][data_idx]) + false_data_multi_bulk[type][params->cycle]++; + } + } + } + + const uint64_t end_tsc = rte_rdtsc(); + const uint64_t time_taken = end_tsc - start_tsc; + + cycles[type][params->cycle][LOOKUP_MULTI_BULK] = time_taken / + NUM_LOOKUPS; + + return 0; +} + +static int +timed_deletes(struct member_perf_params *params, int type) +{ + unsigned int i; + int32_t ret; + + if (type == VBF) + return 0; + const uint64_t start_tsc = rte_rdtsc(); + for (i = 0; i < KEYS_TO_ADD; i++) { + ret = rte_member_delete(params->setsum[type], &keys[i], + data[type][i]); + if (type != CACHE && ret < 0) { + printf("delete error\n"); + return -1; + } + } + + const uint64_t end_tsc = rte_rdtsc(); + const uint64_t time_taken = end_tsc - start_tsc; + + cycles[type][params->cycle][DELETE] = time_taken / KEYS_TO_ADD; + + return 0; +} + +static int +timed_miss_lookup(struct member_perf_params *params, int type) +{ + unsigned int i, j; + int ret; + + false_hit[type][params->cycle] = 0; + + for (i = 0; i < KEYS_TO_ADD / 2; i++) { + ret = rte_member_add(params->setsum[type], &keys[i], + data[type][i]); + if (ret < 0) { + unsigned int a; + printf("Error %d in rte_member_add - key=0x", ret); + for (a = 0; a < params->key_size; a++) + printf("%02x", keys[i][a]); + printf(" value=%d, type: %d\n", data[type][i], type); + + return -1; + } + } + + const uint64_t start_tsc = rte_rdtsc(); + member_set_t result; + + for (i = 0; i < 2 * NUM_LOOKUPS / KEYS_TO_ADD; i++) { + for (j = KEYS_TO_ADD / 2; j < KEYS_TO_ADD; j++) { + ret = rte_member_lookup(params->setsum[type], &keys[j], + &result); + if (ret < 0) { + printf("lookup wrong internally"); + return -1; + } + if (result != RTE_MEMBER_NO_MATCH) + false_hit[type][params->cycle]++; + } + } + + const uint64_t end_tsc = rte_rdtsc(); + const uint64_t time_taken = end_tsc - start_tsc; + + cycles[type][params->cycle][LOOKUP_MISS] = time_taken / NUM_LOOKUPS; + + return 0; +} + +static void +perform_frees(struct member_perf_params *params) +{ + int i; + for (i = 0; i < NUM_TYPE; i++) { + if (params->setsum[i] != NULL) { + rte_member_free(params->setsum[i]); + params->setsum[i] = NULL; + } + } +} + +static int +exit_with_fail(const char *testname, struct member_perf_params *params, + unsigned int i, unsigned int j) +{ + printf("<<<<<Test %s failed at keysize %d iteration %d type %d>>>>>\n", + testname, hashtest_key_lens[params->cycle], i, j); + perform_frees(params); + return -1; +} + +static int +run_all_tbl_perf_tests(void) +{ + unsigned int i, j, k; + struct member_perf_params params; + + printf("Measuring performance, please wait\n"); + fflush(stdout); + + test_socket_id = rte_socket_id(); + + for (i = 0; i < NUM_KEYSIZES; i++) { + if (setup_keys_and_data(¶ms, i, 0) < 0) { + printf("Could not create keys/data/table\n"); + return -1; + } + for (j = 0; j < NUM_TYPE; j++) { + + if (timed_adds(¶ms, j) < 0) + return exit_with_fail("timed_adds", ¶ms, + i, j); + + for (k = 0; k < NUM_SHUFFLES; k++) + shuffle_input_keys(¶ms); + + if (timed_lookups(¶ms, j) < 0) + return exit_with_fail("timed_lookups", ¶ms, + i, j); + + if (timed_lookups_bulk(¶ms, j) < 0) + return exit_with_fail("timed_lookups_bulk", + ¶ms, i, j); + + if (timed_lookups_multimatch(¶ms, j) < 0) + return exit_with_fail("timed_lookups_multi", + ¶ms, i, j); + + if (timed_lookups_multimatch_bulk(¶ms, j) < 0) + return exit_with_fail("timed_lookups_multi_bulk", + ¶ms, i, j); + + if (timed_deletes(¶ms, j) < 0) + return exit_with_fail("timed_deletes", ¶ms, + i, j); + + /* Print a dot to show progress on operations */ + } + printf("."); + fflush(stdout); + + perform_frees(¶ms); + } + + /* Test false positive rate using un-inserted keys */ + for (i = 0; i < NUM_KEYSIZES; i++) { + if (setup_keys_and_data(¶ms, i, 1) < 0) { + printf("Could not create keys/data/table\n"); + return -1; + } + for (j = 0; j < NUM_TYPE; j++) { + if (timed_miss_lookup(¶ms, j) < 0) + return exit_with_fail("timed_miss_lookup", + ¶ms, i, j); + } + perform_frees(¶ms); + } + + printf("\nResults (in CPU cycles/operation)\n"); + printf("-----------------------------------\n"); + printf("\n%-18s%-18s%-18s%-18s%-18s%-18s%-18s%-18s%-18s\n", + "Keysize", "type", "Add", "Lookup", "Lookup_bulk", + "lookup_multi", "lookup_multi_bulk", "Delete", + "miss_lookup"); + for (i = 0; i < NUM_KEYSIZES; i++) { + for (j = 0; j < NUM_TYPE; j++) { + printf("%-18d", hashtest_key_lens[i]); + printf("%-18d", j); + for (k = 0; k < NUM_OPERATIONS; k++) + printf("%-18"PRIu64, cycles[j][i][k]); + printf("\n"); + } + } + + printf("\nFalse results rate (and false positive rate)\n"); + printf("-----------------------------------\n"); + printf("\n%-18s%-18s%-18s%-18s%-18s%-18s%-18s\n", + "Keysize", "type", "fr_single", "fr_bulk", "fr_multi", + "fr_multi_bulk", "false_positive_rate"); + /* Key size not influence False rate so just print out one key size */ + for (i = 0; i < 1; i++) { + for (j = 0; j < NUM_TYPE; j++) { + printf("%-18d", hashtest_key_lens[i]); + printf("%-18d", j); + printf("%-18f", (float)false_data[j][i] / NUM_LOOKUPS); + printf("%-18f", (float)false_data_bulk[j][i] / + NUM_LOOKUPS); + printf("%-18f", (float)false_data_multi[j][i] / + NUM_LOOKUPS); + printf("%-18f", (float)false_data_multi_bulk[j][i] / + NUM_LOOKUPS); + printf("%-18f", (float)false_hit[j][i] / + NUM_LOOKUPS); + printf("\n"); + } + } + return 0; +} + +static int +test_member_perf(void) +{ + + if (run_all_tbl_perf_tests() < 0) + return -1; + + return 0; +} + +REGISTER_TEST_COMMAND(member_perf_autotest, test_member_perf); diff --git a/dpdk/test/test/test_memcpy.c b/dpdk/test/test/test_memcpy.c new file mode 100644 index 00000000..1d93dd53 --- /dev/null +++ b/dpdk/test/test/test_memcpy.c @@ -0,0 +1,162 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdint.h> +#include <stdio.h> +#include <string.h> +#include <stdlib.h> + +#include <rte_common.h> +#include <rte_random.h> +#include <rte_memcpy.h> + +#include "test.h" + +/* + * Set this to the maximum buffer size you want to test. If it is 0, then the + * values in the buf_sizes[] array below will be used. + */ +#define TEST_VALUE_RANGE 0 + +/* List of buffer sizes to test */ +#if TEST_VALUE_RANGE == 0 +static size_t buf_sizes[] = { + 0, 1, 7, 8, 9, 15, 16, 17, 31, 32, 33, 63, 64, 65, 127, 128, 129, 255, + 256, 257, 320, 384, 511, 512, 513, 1023, 1024, 1025, 1518, 1522, 1600, + 2048, 3072, 4096, 5120, 6144, 7168, 8192 +}; +/* MUST be as large as largest packet size above */ +#define SMALL_BUFFER_SIZE 8192 +#else /* TEST_VALUE_RANGE != 0 */ +static size_t buf_sizes[TEST_VALUE_RANGE]; +#define SMALL_BUFFER_SIZE TEST_VALUE_RANGE +#endif /* TEST_VALUE_RANGE == 0 */ + +/* Data is aligned on this many bytes (power of 2) */ +#define ALIGNMENT_UNIT 32 + + +/* + * Create two buffers, and initialise one with random values. These are copied + * to the second buffer and then compared to see if the copy was successful. + * The bytes outside the copied area are also checked to make sure they were not + * changed. + */ +static int +test_single_memcpy(unsigned int off_src, unsigned int off_dst, size_t size) +{ + unsigned int i; + uint8_t dest[SMALL_BUFFER_SIZE + ALIGNMENT_UNIT]; + uint8_t src[SMALL_BUFFER_SIZE + ALIGNMENT_UNIT]; + void * ret; + + /* Setup buffers */ + for (i = 0; i < SMALL_BUFFER_SIZE + ALIGNMENT_UNIT; i++) { + dest[i] = 0; + src[i] = (uint8_t) rte_rand(); + } + + /* Do the copy */ + ret = rte_memcpy(dest + off_dst, src + off_src, size); + if (ret != (dest + off_dst)) { + printf("rte_memcpy() returned %p, not %p\n", + ret, dest + off_dst); + } + + /* Check nothing before offset is affected */ + for (i = 0; i < off_dst; i++) { + if (dest[i] != 0) { + printf("rte_memcpy() failed for %u bytes (offsets=%u,%u): " + "[modified before start of dst].\n", + (unsigned)size, off_src, off_dst); + return -1; + } + } + + /* Check everything was copied */ + for (i = 0; i < size; i++) { + if (dest[i + off_dst] != src[i + off_src]) { + printf("rte_memcpy() failed for %u bytes (offsets=%u,%u): " + "[didn't copy byte %u].\n", + (unsigned)size, off_src, off_dst, i); + return -1; + } + } + + /* Check nothing after copy was affected */ + for (i = size; i < SMALL_BUFFER_SIZE; i++) { + if (dest[i + off_dst] != 0) { + printf("rte_memcpy() failed for %u bytes (offsets=%u,%u): " + "[copied too many].\n", + (unsigned)size, off_src, off_dst); + return -1; + } + } + return 0; +} + +/* + * Check functionality for various buffer sizes and data offsets/alignments. + */ +static int +func_test(void) +{ + unsigned int off_src, off_dst, i; + unsigned int num_buf_sizes = sizeof(buf_sizes) / sizeof(buf_sizes[0]); + int ret; + + for (off_src = 0; off_src < ALIGNMENT_UNIT; off_src++) { + for (off_dst = 0; off_dst < ALIGNMENT_UNIT; off_dst++) { + for (i = 0; i < num_buf_sizes; i++) { + ret = test_single_memcpy(off_src, off_dst, + buf_sizes[i]); + if (ret != 0) + return -1; + } + } + } + return 0; +} + +static int +test_memcpy(void) +{ + int ret; + + ret = func_test(); + if (ret != 0) + return -1; + return 0; +} + +REGISTER_TEST_COMMAND(memcpy_autotest, test_memcpy); diff --git a/dpdk/test/test/test_memcpy_perf.c b/dpdk/test/test/test_memcpy_perf.c new file mode 100644 index 00000000..f1084ca1 --- /dev/null +++ b/dpdk/test/test/test_memcpy_perf.c @@ -0,0 +1,381 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdint.h> +#include <stdio.h> +#include <string.h> +#include <stdlib.h> +#include <sys/time.h> + +#include <rte_common.h> +#include <rte_cycles.h> +#include <rte_random.h> +#include <rte_malloc.h> + +#include <rte_memcpy.h> + +#include "test.h" + +/* + * Set this to the maximum buffer size you want to test. If it is 0, then the + * values in the buf_sizes[] array below will be used. + */ +#define TEST_VALUE_RANGE 0 + +/* List of buffer sizes to test */ +#if TEST_VALUE_RANGE == 0 +static size_t buf_sizes[] = { + 1, 2, 3, 4, 5, 6, 7, 8, 9, 12, 15, 16, 17, 31, 32, 33, 63, 64, 65, 127, 128, + 129, 191, 192, 193, 255, 256, 257, 319, 320, 321, 383, 384, 385, 447, 448, + 449, 511, 512, 513, 767, 768, 769, 1023, 1024, 1025, 1518, 1522, 1536, 1600, + 2048, 2560, 3072, 3584, 4096, 4608, 5120, 5632, 6144, 6656, 7168, 7680, 8192 +}; +/* MUST be as large as largest packet size above */ +#define SMALL_BUFFER_SIZE 8192 +#else /* TEST_VALUE_RANGE != 0 */ +static size_t buf_sizes[TEST_VALUE_RANGE]; +#define SMALL_BUFFER_SIZE TEST_VALUE_RANGE +#endif /* TEST_VALUE_RANGE == 0 */ + + +/* + * Arrays of this size are used for measuring uncached memory accesses by + * picking a random location within the buffer. Make this smaller if there are + * memory allocation errors. + */ +#define LARGE_BUFFER_SIZE (100 * 1024 * 1024) + +/* How many times to run timing loop for performance tests */ +#define TEST_ITERATIONS 1000000 +#define TEST_BATCH_SIZE 100 + +/* Data is aligned on this many bytes (power of 2) */ +#ifdef RTE_MACHINE_CPUFLAG_AVX512F +#define ALIGNMENT_UNIT 64 +#elif defined RTE_MACHINE_CPUFLAG_AVX2 +#define ALIGNMENT_UNIT 32 +#else /* RTE_MACHINE_CPUFLAG */ +#define ALIGNMENT_UNIT 16 +#endif /* RTE_MACHINE_CPUFLAG */ + +/* + * Pointers used in performance tests. The two large buffers are for uncached + * access where random addresses within the buffer are used for each + * memcpy. The two small buffers are for cached access. + */ +static uint8_t *large_buf_read, *large_buf_write; +static uint8_t *small_buf_read, *small_buf_write; + +/* Initialise data buffers. */ +static int +init_buffers(void) +{ + unsigned i; + + large_buf_read = rte_malloc("memcpy", LARGE_BUFFER_SIZE + ALIGNMENT_UNIT, ALIGNMENT_UNIT); + if (large_buf_read == NULL) + goto error_large_buf_read; + + large_buf_write = rte_malloc("memcpy", LARGE_BUFFER_SIZE + ALIGNMENT_UNIT, ALIGNMENT_UNIT); + if (large_buf_write == NULL) + goto error_large_buf_write; + + small_buf_read = rte_malloc("memcpy", SMALL_BUFFER_SIZE + ALIGNMENT_UNIT, ALIGNMENT_UNIT); + if (small_buf_read == NULL) + goto error_small_buf_read; + + small_buf_write = rte_malloc("memcpy", SMALL_BUFFER_SIZE + ALIGNMENT_UNIT, ALIGNMENT_UNIT); + if (small_buf_write == NULL) + goto error_small_buf_write; + + for (i = 0; i < LARGE_BUFFER_SIZE; i++) + large_buf_read[i] = rte_rand(); + for (i = 0; i < SMALL_BUFFER_SIZE; i++) + small_buf_read[i] = rte_rand(); + + return 0; + +error_small_buf_write: + rte_free(small_buf_read); +error_small_buf_read: + rte_free(large_buf_write); +error_large_buf_write: + rte_free(large_buf_read); +error_large_buf_read: + printf("ERROR: not enough memory\n"); + return -1; +} + +/* Cleanup data buffers */ +static void +free_buffers(void) +{ + rte_free(large_buf_read); + rte_free(large_buf_write); + rte_free(small_buf_read); + rte_free(small_buf_write); +} + +/* + * Get a random offset into large array, with enough space needed to perform + * max copy size. Offset is aligned, uoffset is used for unalignment setting. + */ +static inline size_t +get_rand_offset(size_t uoffset) +{ + return ((rte_rand() % (LARGE_BUFFER_SIZE - SMALL_BUFFER_SIZE)) & + ~(ALIGNMENT_UNIT - 1)) + uoffset; +} + +/* Fill in source and destination addresses. */ +static inline void +fill_addr_arrays(size_t *dst_addr, int is_dst_cached, size_t dst_uoffset, + size_t *src_addr, int is_src_cached, size_t src_uoffset) +{ + unsigned int i; + + for (i = 0; i < TEST_BATCH_SIZE; i++) { + dst_addr[i] = (is_dst_cached) ? dst_uoffset : get_rand_offset(dst_uoffset); + src_addr[i] = (is_src_cached) ? src_uoffset : get_rand_offset(src_uoffset); + } +} + +/* + * WORKAROUND: For some reason the first test doing an uncached write + * takes a very long time (~25 times longer than is expected). So we do + * it once without timing. + */ +static void +do_uncached_write(uint8_t *dst, int is_dst_cached, + const uint8_t *src, int is_src_cached, size_t size) +{ + unsigned i, j; + size_t dst_addrs[TEST_BATCH_SIZE], src_addrs[TEST_BATCH_SIZE]; + + for (i = 0; i < (TEST_ITERATIONS / TEST_BATCH_SIZE); i++) { + fill_addr_arrays(dst_addrs, is_dst_cached, 0, + src_addrs, is_src_cached, 0); + for (j = 0; j < TEST_BATCH_SIZE; j++) { + rte_memcpy(dst+dst_addrs[j], src+src_addrs[j], size); + } + } +} + +/* + * Run a single memcpy performance test. This is a macro to ensure that if + * the "size" parameter is a constant it won't be converted to a variable. + */ +#define SINGLE_PERF_TEST(dst, is_dst_cached, dst_uoffset, \ + src, is_src_cached, src_uoffset, size) \ +do { \ + unsigned int iter, t; \ + size_t dst_addrs[TEST_BATCH_SIZE], src_addrs[TEST_BATCH_SIZE]; \ + uint64_t start_time, total_time = 0; \ + uint64_t total_time2 = 0; \ + for (iter = 0; iter < (TEST_ITERATIONS / TEST_BATCH_SIZE); iter++) { \ + fill_addr_arrays(dst_addrs, is_dst_cached, dst_uoffset, \ + src_addrs, is_src_cached, src_uoffset); \ + start_time = rte_rdtsc(); \ + for (t = 0; t < TEST_BATCH_SIZE; t++) \ + rte_memcpy(dst+dst_addrs[t], src+src_addrs[t], size); \ + total_time += rte_rdtsc() - start_time; \ + } \ + for (iter = 0; iter < (TEST_ITERATIONS / TEST_BATCH_SIZE); iter++) { \ + fill_addr_arrays(dst_addrs, is_dst_cached, dst_uoffset, \ + src_addrs, is_src_cached, src_uoffset); \ + start_time = rte_rdtsc(); \ + for (t = 0; t < TEST_BATCH_SIZE; t++) \ + memcpy(dst+dst_addrs[t], src+src_addrs[t], size); \ + total_time2 += rte_rdtsc() - start_time; \ + } \ + printf("%3.0f -", (double)total_time / TEST_ITERATIONS); \ + printf("%3.0f", (double)total_time2 / TEST_ITERATIONS); \ + printf("(%6.2f%%) ", ((double)total_time - total_time2)*100/total_time2); \ +} while (0) + +/* Run aligned memcpy tests for each cached/uncached permutation */ +#define ALL_PERF_TESTS_FOR_SIZE(n) \ +do { \ + if (__builtin_constant_p(n)) \ + printf("\nC%6u", (unsigned)n); \ + else \ + printf("\n%7u", (unsigned)n); \ + SINGLE_PERF_TEST(small_buf_write, 1, 0, small_buf_read, 1, 0, n); \ + SINGLE_PERF_TEST(large_buf_write, 0, 0, small_buf_read, 1, 0, n); \ + SINGLE_PERF_TEST(small_buf_write, 1, 0, large_buf_read, 0, 0, n); \ + SINGLE_PERF_TEST(large_buf_write, 0, 0, large_buf_read, 0, 0, n); \ +} while (0) + +/* Run unaligned memcpy tests for each cached/uncached permutation */ +#define ALL_PERF_TESTS_FOR_SIZE_UNALIGNED(n) \ +do { \ + if (__builtin_constant_p(n)) \ + printf("\nC%6u", (unsigned)n); \ + else \ + printf("\n%7u", (unsigned)n); \ + SINGLE_PERF_TEST(small_buf_write, 1, 1, small_buf_read, 1, 5, n); \ + SINGLE_PERF_TEST(large_buf_write, 0, 1, small_buf_read, 1, 5, n); \ + SINGLE_PERF_TEST(small_buf_write, 1, 1, large_buf_read, 0, 5, n); \ + SINGLE_PERF_TEST(large_buf_write, 0, 1, large_buf_read, 0, 5, n); \ +} while (0) + +/* Run memcpy tests for constant length */ +#define ALL_PERF_TEST_FOR_CONSTANT \ +do { \ + TEST_CONSTANT(6U); TEST_CONSTANT(64U); TEST_CONSTANT(128U); \ + TEST_CONSTANT(192U); TEST_CONSTANT(256U); TEST_CONSTANT(512U); \ + TEST_CONSTANT(768U); TEST_CONSTANT(1024U); TEST_CONSTANT(1536U); \ +} while (0) + +/* Run all memcpy tests for aligned constant cases */ +static inline void +perf_test_constant_aligned(void) +{ +#define TEST_CONSTANT ALL_PERF_TESTS_FOR_SIZE + ALL_PERF_TEST_FOR_CONSTANT; +#undef TEST_CONSTANT +} + +/* Run all memcpy tests for unaligned constant cases */ +static inline void +perf_test_constant_unaligned(void) +{ +#define TEST_CONSTANT ALL_PERF_TESTS_FOR_SIZE_UNALIGNED + ALL_PERF_TEST_FOR_CONSTANT; +#undef TEST_CONSTANT +} + +/* Run all memcpy tests for aligned variable cases */ +static inline void +perf_test_variable_aligned(void) +{ + unsigned n = sizeof(buf_sizes) / sizeof(buf_sizes[0]); + unsigned i; + for (i = 0; i < n; i++) { + ALL_PERF_TESTS_FOR_SIZE((size_t)buf_sizes[i]); + } +} + +/* Run all memcpy tests for unaligned variable cases */ +static inline void +perf_test_variable_unaligned(void) +{ + unsigned n = sizeof(buf_sizes) / sizeof(buf_sizes[0]); + unsigned i; + for (i = 0; i < n; i++) { + ALL_PERF_TESTS_FOR_SIZE_UNALIGNED((size_t)buf_sizes[i]); + } +} + +/* Run all memcpy tests */ +static int +perf_test(void) +{ + int ret; + struct timeval tv_begin, tv_end; + double time_aligned, time_unaligned; + double time_aligned_const, time_unaligned_const; + + ret = init_buffers(); + if (ret != 0) + return ret; + +#if TEST_VALUE_RANGE != 0 + /* Set up buf_sizes array, if required */ + unsigned i; + for (i = 0; i < TEST_VALUE_RANGE; i++) + buf_sizes[i] = i; +#endif + + /* See function comment */ + do_uncached_write(large_buf_write, 0, small_buf_read, 1, SMALL_BUFFER_SIZE); + + printf("\n** rte_memcpy() - memcpy perf. tests (C = compile-time constant) **\n" + "======= ================= ================= ================= =================\n" + " Size Cache to cache Cache to mem Mem to cache Mem to mem\n" + "(bytes) (ticks) (ticks) (ticks) (ticks)\n" + "------- ----------------- ----------------- ----------------- -----------------"); + + printf("\n================================= %2dB aligned =================================", + ALIGNMENT_UNIT); + /* Do aligned tests where size is a variable */ + gettimeofday(&tv_begin, NULL); + perf_test_variable_aligned(); + gettimeofday(&tv_end, NULL); + time_aligned = (double)(tv_end.tv_sec - tv_begin.tv_sec) + + ((double)tv_end.tv_usec - tv_begin.tv_usec)/1000000; + printf("\n------- ----------------- ----------------- ----------------- -----------------"); + /* Do aligned tests where size is a compile-time constant */ + gettimeofday(&tv_begin, NULL); + perf_test_constant_aligned(); + gettimeofday(&tv_end, NULL); + time_aligned_const = (double)(tv_end.tv_sec - tv_begin.tv_sec) + + ((double)tv_end.tv_usec - tv_begin.tv_usec)/1000000; + printf("\n================================== Unaligned =================================="); + /* Do unaligned tests where size is a variable */ + gettimeofday(&tv_begin, NULL); + perf_test_variable_unaligned(); + gettimeofday(&tv_end, NULL); + time_unaligned = (double)(tv_end.tv_sec - tv_begin.tv_sec) + + ((double)tv_end.tv_usec - tv_begin.tv_usec)/1000000; + printf("\n------- ----------------- ----------------- ----------------- -----------------"); + /* Do unaligned tests where size is a compile-time constant */ + gettimeofday(&tv_begin, NULL); + perf_test_constant_unaligned(); + gettimeofday(&tv_end, NULL); + time_unaligned_const = (double)(tv_end.tv_sec - tv_begin.tv_sec) + + ((double)tv_end.tv_usec - tv_begin.tv_usec)/1000000; + printf("\n======= ================= ================= ================= =================\n\n"); + + printf("Test Execution Time (seconds):\n"); + printf("Aligned variable copy size = %8.3f\n", time_aligned); + printf("Aligned constant copy size = %8.3f\n", time_aligned_const); + printf("Unaligned variable copy size = %8.3f\n", time_unaligned); + printf("Unaligned constant copy size = %8.3f\n", time_unaligned_const); + free_buffers(); + + return 0; +} + +static int +test_memcpy_perf(void) +{ + int ret; + + ret = perf_test(); + if (ret != 0) + return -1; + return 0; +} + +REGISTER_TEST_COMMAND(memcpy_perf_autotest, test_memcpy_perf); diff --git a/dpdk/test/test/test_memory.c b/dpdk/test/test/test_memory.c new file mode 100644 index 00000000..921bdc88 --- /dev/null +++ b/dpdk/test/test/test_memory.c @@ -0,0 +1,89 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <stdint.h> + +#include <rte_memory.h> +#include <rte_common.h> + +#include "test.h" + +/* + * Memory + * ====== + * + * - Dump the mapped memory. The python-expect script checks that at + * least one line is dumped. + * + * - Check that memory size is different than 0. + * + * - Try to read all memory; it should not segfault. + */ + +static int +test_memory(void) +{ + uint64_t s; + unsigned i; + size_t j; + const struct rte_memseg *mem; + + /* + * dump the mapped memory: the python-expect script checks + * that at least one line is dumped + */ + printf("Dump memory layout\n"); + rte_dump_physmem_layout(stdout); + + /* check that memory size is != 0 */ + s = rte_eal_get_physmem_size(); + if (s == 0) { + printf("No memory detected\n"); + return -1; + } + + /* try to read memory (should not segfault) */ + mem = rte_eal_get_physmem_layout(); + for (i = 0; i < RTE_MAX_MEMSEG && mem[i].addr != NULL ; i++) { + + /* check memory */ + for (j = 0; j<mem[i].len; j++) { + *((volatile uint8_t *) mem[i].addr + j); + } + } + + return 0; +} + +REGISTER_TEST_COMMAND(memory_autotest, test_memory); diff --git a/dpdk/test/test/test_mempool.c b/dpdk/test/test/test_mempool.c new file mode 100644 index 00000000..37ead503 --- /dev/null +++ b/dpdk/test/test/test_mempool.c @@ -0,0 +1,652 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <string.h> +#include <stdio.h> +#include <stdlib.h> +#include <stdint.h> +#include <inttypes.h> +#include <stdarg.h> +#include <errno.h> +#include <sys/queue.h> + +#include <rte_common.h> +#include <rte_log.h> +#include <rte_debug.h> +#include <rte_memory.h> +#include <rte_launch.h> +#include <rte_cycles.h> +#include <rte_eal.h> +#include <rte_per_lcore.h> +#include <rte_lcore.h> +#include <rte_atomic.h> +#include <rte_branch_prediction.h> +#include <rte_mempool.h> +#include <rte_spinlock.h> +#include <rte_malloc.h> + +#include "test.h" + +/* + * Mempool + * ======= + * + * Basic tests: done on one core with and without cache: + * + * - Get one object, put one object + * - Get two objects, put two objects + * - Get all objects, test that their content is not modified and + * put them back in the pool. + */ + +#define MEMPOOL_ELT_SIZE 2048 +#define MAX_KEEP 16 +#define MEMPOOL_SIZE ((rte_lcore_count()*(MAX_KEEP+RTE_MEMPOOL_CACHE_MAX_SIZE))-1) + +#define LOG_ERR() printf("test failed at %s():%d\n", __func__, __LINE__) +#define RET_ERR() do { \ + LOG_ERR(); \ + return -1; \ + } while (0) +#define GOTO_ERR(var, label) do { \ + LOG_ERR(); \ + var = -1; \ + goto label; \ + } while (0) + +static rte_atomic32_t synchro; + +/* + * save the object number in the first 4 bytes of object data. All + * other bytes are set to 0. + */ +static void +my_obj_init(struct rte_mempool *mp, __attribute__((unused)) void *arg, + void *obj, unsigned i) +{ + uint32_t *objnum = obj; + + memset(obj, 0, mp->elt_size); + *objnum = i; +} + +/* basic tests (done on one core) */ +static int +test_mempool_basic(struct rte_mempool *mp, int use_external_cache) +{ + uint32_t *objnum; + void **objtable; + void *obj, *obj2; + char *obj_data; + int ret = 0; + unsigned i, j; + int offset; + struct rte_mempool_cache *cache; + + if (use_external_cache) { + /* Create a user-owned mempool cache. */ + cache = rte_mempool_cache_create(RTE_MEMPOOL_CACHE_MAX_SIZE, + SOCKET_ID_ANY); + if (cache == NULL) + RET_ERR(); + } else { + /* May be NULL if cache is disabled. */ + cache = rte_mempool_default_cache(mp, rte_lcore_id()); + } + + /* dump the mempool status */ + rte_mempool_dump(stdout, mp); + + printf("get an object\n"); + if (rte_mempool_generic_get(mp, &obj, 1, cache) < 0) + GOTO_ERR(ret, out); + rte_mempool_dump(stdout, mp); + + /* tests that improve coverage */ + printf("get object count\n"); + /* We have to count the extra caches, one in this case. */ + offset = use_external_cache ? 1 * cache->len : 0; + if (rte_mempool_avail_count(mp) + offset != MEMPOOL_SIZE - 1) + GOTO_ERR(ret, out); + + printf("get private data\n"); + if (rte_mempool_get_priv(mp) != (char *)mp + + MEMPOOL_HEADER_SIZE(mp, mp->cache_size)) + GOTO_ERR(ret, out); + +#ifndef RTE_EXEC_ENV_BSDAPP /* rte_mem_virt2iova() not supported on bsd */ + printf("get physical address of an object\n"); + if (rte_mempool_virt2iova(obj) != rte_mem_virt2iova(obj)) + GOTO_ERR(ret, out); +#endif + + printf("put the object back\n"); + rte_mempool_generic_put(mp, &obj, 1, cache); + rte_mempool_dump(stdout, mp); + + printf("get 2 objects\n"); + if (rte_mempool_generic_get(mp, &obj, 1, cache) < 0) + GOTO_ERR(ret, out); + if (rte_mempool_generic_get(mp, &obj2, 1, cache) < 0) { + rte_mempool_generic_put(mp, &obj, 1, cache); + GOTO_ERR(ret, out); + } + rte_mempool_dump(stdout, mp); + + printf("put the objects back\n"); + rte_mempool_generic_put(mp, &obj, 1, cache); + rte_mempool_generic_put(mp, &obj2, 1, cache); + rte_mempool_dump(stdout, mp); + + /* + * get many objects: we cannot get them all because the cache + * on other cores may not be empty. + */ + objtable = malloc(MEMPOOL_SIZE * sizeof(void *)); + if (objtable == NULL) + GOTO_ERR(ret, out); + + for (i = 0; i < MEMPOOL_SIZE; i++) { + if (rte_mempool_generic_get(mp, &objtable[i], 1, cache) < 0) + break; + } + + /* + * for each object, check that its content was not modified, + * and put objects back in pool + */ + while (i--) { + obj = objtable[i]; + obj_data = obj; + objnum = obj; + if (*objnum > MEMPOOL_SIZE) { + printf("bad object number(%d)\n", *objnum); + ret = -1; + break; + } + for (j = sizeof(*objnum); j < mp->elt_size; j++) { + if (obj_data[j] != 0) + ret = -1; + } + + rte_mempool_generic_put(mp, &objtable[i], 1, cache); + } + + free(objtable); + if (ret == -1) + printf("objects were modified!\n"); + +out: + if (use_external_cache) { + rte_mempool_cache_flush(cache, mp); + rte_mempool_cache_free(cache); + } + + return ret; +} + +static int test_mempool_creation_with_exceeded_cache_size(void) +{ + struct rte_mempool *mp_cov; + + mp_cov = rte_mempool_create("test_mempool_cache_too_big", + MEMPOOL_SIZE, + MEMPOOL_ELT_SIZE, + RTE_MEMPOOL_CACHE_MAX_SIZE + 32, 0, + NULL, NULL, + my_obj_init, NULL, + SOCKET_ID_ANY, 0); + + if (mp_cov != NULL) { + rte_mempool_free(mp_cov); + RET_ERR(); + } + + return 0; +} + +static struct rte_mempool *mp_spsc; +static rte_spinlock_t scsp_spinlock; +static void *scsp_obj_table[MAX_KEEP]; + +/* + * single producer function + */ +static int test_mempool_single_producer(void) +{ + unsigned int i; + void *obj = NULL; + uint64_t start_cycles, end_cycles; + uint64_t duration = rte_get_timer_hz() / 4; + + start_cycles = rte_get_timer_cycles(); + while (1) { + end_cycles = rte_get_timer_cycles(); + /* duration uses up, stop producing */ + if (start_cycles + duration < end_cycles) + break; + rte_spinlock_lock(&scsp_spinlock); + for (i = 0; i < MAX_KEEP; i ++) { + if (NULL != scsp_obj_table[i]) { + obj = scsp_obj_table[i]; + break; + } + } + rte_spinlock_unlock(&scsp_spinlock); + if (i >= MAX_KEEP) { + continue; + } + if (rte_mempool_from_obj(obj) != mp_spsc) { + printf("obj not owned by this mempool\n"); + RET_ERR(); + } + rte_mempool_put(mp_spsc, obj); + rte_spinlock_lock(&scsp_spinlock); + scsp_obj_table[i] = NULL; + rte_spinlock_unlock(&scsp_spinlock); + } + + return 0; +} + +/* + * single consumer function + */ +static int test_mempool_single_consumer(void) +{ + unsigned int i; + void * obj; + uint64_t start_cycles, end_cycles; + uint64_t duration = rte_get_timer_hz() / 8; + + start_cycles = rte_get_timer_cycles(); + while (1) { + end_cycles = rte_get_timer_cycles(); + /* duration uses up, stop consuming */ + if (start_cycles + duration < end_cycles) + break; + rte_spinlock_lock(&scsp_spinlock); + for (i = 0; i < MAX_KEEP; i ++) { + if (NULL == scsp_obj_table[i]) + break; + } + rte_spinlock_unlock(&scsp_spinlock); + if (i >= MAX_KEEP) + continue; + if (rte_mempool_get(mp_spsc, &obj) < 0) + break; + rte_spinlock_lock(&scsp_spinlock); + scsp_obj_table[i] = obj; + rte_spinlock_unlock(&scsp_spinlock); + } + + return 0; +} + +/* + * test function for mempool test based on singple consumer and single producer, + * can run on one lcore only + */ +static int +test_mempool_launch_single_consumer(__attribute__((unused)) void *arg) +{ + return test_mempool_single_consumer(); +} + +static void +my_mp_init(struct rte_mempool *mp, __attribute__((unused)) void *arg) +{ + printf("mempool name is %s\n", mp->name); + /* nothing to be implemented here*/ + return ; +} + +/* + * it tests the mempool operations based on singple producer and single consumer + */ +static int +test_mempool_sp_sc(void) +{ + int ret = 0; + unsigned lcore_id = rte_lcore_id(); + unsigned lcore_next; + + /* create a mempool with single producer/consumer ring */ + if (mp_spsc == NULL) { + mp_spsc = rte_mempool_create("test_mempool_sp_sc", MEMPOOL_SIZE, + MEMPOOL_ELT_SIZE, 0, 0, + my_mp_init, NULL, + my_obj_init, NULL, + SOCKET_ID_ANY, + MEMPOOL_F_NO_CACHE_ALIGN | MEMPOOL_F_SP_PUT | + MEMPOOL_F_SC_GET); + if (mp_spsc == NULL) + RET_ERR(); + } + if (rte_mempool_lookup("test_mempool_sp_sc") != mp_spsc) { + printf("Cannot lookup mempool from its name\n"); + rte_mempool_free(mp_spsc); + RET_ERR(); + } + lcore_next = rte_get_next_lcore(lcore_id, 0, 1); + if (lcore_next >= RTE_MAX_LCORE) { + rte_mempool_free(mp_spsc); + RET_ERR(); + } + if (rte_eal_lcore_role(lcore_next) != ROLE_RTE) { + rte_mempool_free(mp_spsc); + RET_ERR(); + } + rte_spinlock_init(&scsp_spinlock); + memset(scsp_obj_table, 0, sizeof(scsp_obj_table)); + rte_eal_remote_launch(test_mempool_launch_single_consumer, NULL, + lcore_next); + if (test_mempool_single_producer() < 0) + ret = -1; + + if (rte_eal_wait_lcore(lcore_next) < 0) + ret = -1; + rte_mempool_free(mp_spsc); + + return ret; +} + +/* + * it tests some more basic of mempool + */ +static int +test_mempool_basic_ex(struct rte_mempool *mp) +{ + unsigned i; + void **obj; + void *err_obj; + int ret = -1; + + if (mp == NULL) + return ret; + + obj = rte_calloc("test_mempool_basic_ex", MEMPOOL_SIZE, + sizeof(void *), 0); + if (obj == NULL) { + printf("test_mempool_basic_ex fail to rte_malloc\n"); + return ret; + } + printf("test_mempool_basic_ex now mempool (%s) has %u free entries\n", + mp->name, rte_mempool_in_use_count(mp)); + if (rte_mempool_full(mp) != 1) { + printf("test_mempool_basic_ex the mempool should be full\n"); + goto fail_mp_basic_ex; + } + + for (i = 0; i < MEMPOOL_SIZE; i ++) { + if (rte_mempool_get(mp, &obj[i]) < 0) { + printf("test_mp_basic_ex fail to get object for [%u]\n", + i); + goto fail_mp_basic_ex; + } + } + if (rte_mempool_get(mp, &err_obj) == 0) { + printf("test_mempool_basic_ex get an impossible obj\n"); + goto fail_mp_basic_ex; + } + printf("number: %u\n", i); + if (rte_mempool_empty(mp) != 1) { + printf("test_mempool_basic_ex the mempool should be empty\n"); + goto fail_mp_basic_ex; + } + + for (i = 0; i < MEMPOOL_SIZE; i++) + rte_mempool_put(mp, obj[i]); + + if (rte_mempool_full(mp) != 1) { + printf("test_mempool_basic_ex the mempool should be full\n"); + goto fail_mp_basic_ex; + } + + ret = 0; + +fail_mp_basic_ex: + if (obj != NULL) + rte_free((void *)obj); + + return ret; +} + +static int +test_mempool_same_name_twice_creation(void) +{ + struct rte_mempool *mp_tc, *mp_tc2; + + mp_tc = rte_mempool_create("test_mempool_same_name", MEMPOOL_SIZE, + MEMPOOL_ELT_SIZE, 0, 0, + NULL, NULL, + NULL, NULL, + SOCKET_ID_ANY, 0); + + if (mp_tc == NULL) + RET_ERR(); + + mp_tc2 = rte_mempool_create("test_mempool_same_name", MEMPOOL_SIZE, + MEMPOOL_ELT_SIZE, 0, 0, + NULL, NULL, + NULL, NULL, + SOCKET_ID_ANY, 0); + + if (mp_tc2 != NULL) { + rte_mempool_free(mp_tc); + rte_mempool_free(mp_tc2); + RET_ERR(); + } + + rte_mempool_free(mp_tc); + return 0; +} + +/* + * Basic test for mempool_xmem functions. + */ +static int +test_mempool_xmem_misc(void) +{ + uint32_t elt_num, total_size; + size_t sz; + ssize_t usz; + + elt_num = MAX_KEEP; + total_size = rte_mempool_calc_obj_size(MEMPOOL_ELT_SIZE, 0, NULL); + sz = rte_mempool_xmem_size(elt_num, total_size, MEMPOOL_PG_SHIFT_MAX, + 0); + + usz = rte_mempool_xmem_usage(NULL, elt_num, total_size, 0, 1, + MEMPOOL_PG_SHIFT_MAX, 0); + + if (sz != (size_t)usz) { + printf("failure @ %s: rte_mempool_xmem_usage(%u, %u) " + "returns: %#zx, while expected: %#zx;\n", + __func__, elt_num, total_size, sz, (size_t)usz); + return -1; + } + + return 0; +} + +static void +walk_cb(struct rte_mempool *mp, void *userdata __rte_unused) +{ + printf("\t%s\n", mp->name); +} + +static int +test_mempool(void) +{ + int ret = -1; + struct rte_mempool *mp_cache = NULL; + struct rte_mempool *mp_nocache = NULL; + struct rte_mempool *mp_stack = NULL; + struct rte_mempool *default_pool = NULL; + + rte_atomic32_init(&synchro); + + /* create a mempool (without cache) */ + mp_nocache = rte_mempool_create("test_nocache", MEMPOOL_SIZE, + MEMPOOL_ELT_SIZE, 0, 0, + NULL, NULL, + my_obj_init, NULL, + SOCKET_ID_ANY, 0); + + if (mp_nocache == NULL) { + printf("cannot allocate mp_nocache mempool\n"); + goto err; + } + + /* create a mempool (with cache) */ + mp_cache = rte_mempool_create("test_cache", MEMPOOL_SIZE, + MEMPOOL_ELT_SIZE, + RTE_MEMPOOL_CACHE_MAX_SIZE, 0, + NULL, NULL, + my_obj_init, NULL, + SOCKET_ID_ANY, 0); + + if (mp_cache == NULL) { + printf("cannot allocate mp_cache mempool\n"); + goto err; + } + + /* create a mempool with an external handler */ + mp_stack = rte_mempool_create_empty("test_stack", + MEMPOOL_SIZE, + MEMPOOL_ELT_SIZE, + RTE_MEMPOOL_CACHE_MAX_SIZE, 0, + SOCKET_ID_ANY, 0); + + if (mp_stack == NULL) { + printf("cannot allocate mp_stack mempool\n"); + goto err; + } + if (rte_mempool_set_ops_byname(mp_stack, "stack", NULL) < 0) { + printf("cannot set stack handler\n"); + goto err; + } + if (rte_mempool_populate_default(mp_stack) < 0) { + printf("cannot populate mp_stack mempool\n"); + goto err; + } + rte_mempool_obj_iter(mp_stack, my_obj_init, NULL); + + /* Create a mempool based on Default handler */ + printf("Testing %s mempool handler\n", + RTE_MBUF_DEFAULT_MEMPOOL_OPS); + default_pool = rte_mempool_create_empty("default_pool", + MEMPOOL_SIZE, + MEMPOOL_ELT_SIZE, + RTE_MEMPOOL_CACHE_MAX_SIZE, 0, + SOCKET_ID_ANY, 0); + + if (default_pool == NULL) { + printf("cannot allocate default mempool\n"); + goto err; + } + if (rte_mempool_set_ops_byname(default_pool, + RTE_MBUF_DEFAULT_MEMPOOL_OPS, NULL) < 0) { + printf("cannot set %s handler\n", + RTE_MBUF_DEFAULT_MEMPOOL_OPS); + goto err; + } + if (rte_mempool_populate_default(default_pool) < 0) { + printf("cannot populate %s mempool\n", + RTE_MBUF_DEFAULT_MEMPOOL_OPS); + goto err; + } + rte_mempool_obj_iter(default_pool, my_obj_init, NULL); + + /* retrieve the mempool from its name */ + if (rte_mempool_lookup("test_nocache") != mp_nocache) { + printf("Cannot lookup mempool from its name\n"); + goto err; + } + + printf("Walk into mempools:\n"); + rte_mempool_walk(walk_cb, NULL); + + rte_mempool_list_dump(stdout); + + /* basic tests without cache */ + if (test_mempool_basic(mp_nocache, 0) < 0) + goto err; + + /* basic tests with cache */ + if (test_mempool_basic(mp_cache, 0) < 0) + goto err; + + /* basic tests with user-owned cache */ + if (test_mempool_basic(mp_nocache, 1) < 0) + goto err; + + /* more basic tests without cache */ + if (test_mempool_basic_ex(mp_nocache) < 0) + goto err; + + /* mempool operation test based on single producer and single comsumer */ + if (test_mempool_sp_sc() < 0) + goto err; + + if (test_mempool_creation_with_exceeded_cache_size() < 0) + goto err; + + if (test_mempool_same_name_twice_creation() < 0) + goto err; + + if (test_mempool_xmem_misc() < 0) + goto err; + + /* test the stack handler */ + if (test_mempool_basic(mp_stack, 1) < 0) + goto err; + + if (test_mempool_basic(default_pool, 1) < 0) + goto err; + + rte_mempool_list_dump(stdout); + + ret = 0; + +err: + rte_mempool_free(mp_nocache); + rte_mempool_free(mp_cache); + rte_mempool_free(mp_stack); + rte_mempool_free(default_pool); + + return ret; +} + +REGISTER_TEST_COMMAND(mempool_autotest, test_mempool); diff --git a/dpdk/test/test/test_mempool_perf.c b/dpdk/test/test/test_mempool_perf.c new file mode 100644 index 00000000..749b364e --- /dev/null +++ b/dpdk/test/test/test_mempool_perf.c @@ -0,0 +1,428 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <string.h> +#include <stdio.h> +#include <stdlib.h> +#include <stdint.h> +#include <inttypes.h> +#include <stdarg.h> +#include <errno.h> +#include <sys/queue.h> + +#include <rte_common.h> +#include <rte_log.h> +#include <rte_debug.h> +#include <rte_memory.h> +#include <rte_launch.h> +#include <rte_cycles.h> +#include <rte_eal.h> +#include <rte_per_lcore.h> +#include <rte_lcore.h> +#include <rte_atomic.h> +#include <rte_branch_prediction.h> +#include <rte_mempool.h> +#include <rte_spinlock.h> +#include <rte_malloc.h> + +#include "test.h" + +/* + * Mempool performance + * ======= + * + * Each core get *n_keep* objects per bulk of *n_get_bulk*. Then, + * objects are put back in the pool per bulk of *n_put_bulk*. + * + * This sequence is done during TIME_S seconds. + * + * This test is done on the following configurations: + * + * - Cores configuration (*cores*) + * + * - One core with cache + * - Two cores with cache + * - Max. cores with cache + * - One core without cache + * - Two cores without cache + * - Max. cores without cache + * - One core with user-owned cache + * - Two cores with user-owned cache + * - Max. cores with user-owned cache + * + * - Bulk size (*n_get_bulk*, *n_put_bulk*) + * + * - Bulk get from 1 to 32 + * - Bulk put from 1 to 32 + * + * - Number of kept objects (*n_keep*) + * + * - 32 + * - 128 + */ + +#define N 65536 +#define TIME_S 5 +#define MEMPOOL_ELT_SIZE 2048 +#define MAX_KEEP 128 +#define MEMPOOL_SIZE ((rte_lcore_count()*(MAX_KEEP+RTE_MEMPOOL_CACHE_MAX_SIZE))-1) + +#define LOG_ERR() printf("test failed at %s():%d\n", __func__, __LINE__) +#define RET_ERR() do { \ + LOG_ERR(); \ + return -1; \ + } while (0) +#define GOTO_ERR(var, label) do { \ + LOG_ERR(); \ + var = -1; \ + goto label; \ + } while (0) + +static int use_external_cache; +static unsigned external_cache_size = RTE_MEMPOOL_CACHE_MAX_SIZE; + +static rte_atomic32_t synchro; + +/* number of objects in one bulk operation (get or put) */ +static unsigned n_get_bulk; +static unsigned n_put_bulk; + +/* number of objects retrived from mempool before putting them back */ +static unsigned n_keep; + +/* number of enqueues / dequeues */ +struct mempool_test_stats { + uint64_t enq_count; +} __rte_cache_aligned; + +static struct mempool_test_stats stats[RTE_MAX_LCORE]; + +/* + * save the object number in the first 4 bytes of object data. All + * other bytes are set to 0. + */ +static void +my_obj_init(struct rte_mempool *mp, __attribute__((unused)) void *arg, + void *obj, unsigned i) +{ + uint32_t *objnum = obj; + memset(obj, 0, mp->elt_size); + *objnum = i; +} + +static int +per_lcore_mempool_test(void *arg) +{ + void *obj_table[MAX_KEEP]; + unsigned i, idx; + struct rte_mempool *mp = arg; + unsigned lcore_id = rte_lcore_id(); + int ret = 0; + uint64_t start_cycles, end_cycles; + uint64_t time_diff = 0, hz = rte_get_timer_hz(); + struct rte_mempool_cache *cache; + + if (use_external_cache) { + /* Create a user-owned mempool cache. */ + cache = rte_mempool_cache_create(external_cache_size, + SOCKET_ID_ANY); + if (cache == NULL) + RET_ERR(); + } else { + /* May be NULL if cache is disabled. */ + cache = rte_mempool_default_cache(mp, lcore_id); + } + + /* n_get_bulk and n_put_bulk must be divisors of n_keep */ + if (((n_keep / n_get_bulk) * n_get_bulk) != n_keep) + GOTO_ERR(ret, out); + if (((n_keep / n_put_bulk) * n_put_bulk) != n_keep) + GOTO_ERR(ret, out); + + stats[lcore_id].enq_count = 0; + + /* wait synchro for slaves */ + if (lcore_id != rte_get_master_lcore()) + while (rte_atomic32_read(&synchro) == 0); + + start_cycles = rte_get_timer_cycles(); + + while (time_diff/hz < TIME_S) { + for (i = 0; likely(i < (N/n_keep)); i++) { + /* get n_keep objects by bulk of n_bulk */ + idx = 0; + while (idx < n_keep) { + ret = rte_mempool_generic_get(mp, + &obj_table[idx], + n_get_bulk, + cache); + if (unlikely(ret < 0)) { + rte_mempool_dump(stdout, mp); + /* in this case, objects are lost... */ + GOTO_ERR(ret, out); + } + idx += n_get_bulk; + } + + /* put the objects back */ + idx = 0; + while (idx < n_keep) { + rte_mempool_generic_put(mp, &obj_table[idx], + n_put_bulk, + cache); + idx += n_put_bulk; + } + } + end_cycles = rte_get_timer_cycles(); + time_diff = end_cycles - start_cycles; + stats[lcore_id].enq_count += N; + } + +out: + if (use_external_cache) { + rte_mempool_cache_flush(cache, mp); + rte_mempool_cache_free(cache); + } + + return ret; +} + +/* launch all the per-lcore test, and display the result */ +static int +launch_cores(struct rte_mempool *mp, unsigned int cores) +{ + unsigned lcore_id; + uint64_t rate; + int ret; + unsigned cores_save = cores; + + rte_atomic32_set(&synchro, 0); + + /* reset stats */ + memset(stats, 0, sizeof(stats)); + + printf("mempool_autotest cache=%u cores=%u n_get_bulk=%u " + "n_put_bulk=%u n_keep=%u ", + use_external_cache ? + external_cache_size : (unsigned) mp->cache_size, + cores, n_get_bulk, n_put_bulk, n_keep); + + if (rte_mempool_avail_count(mp) != MEMPOOL_SIZE) { + printf("mempool is not full\n"); + return -1; + } + + RTE_LCORE_FOREACH_SLAVE(lcore_id) { + if (cores == 1) + break; + cores--; + rte_eal_remote_launch(per_lcore_mempool_test, + mp, lcore_id); + } + + /* start synchro and launch test on master */ + rte_atomic32_set(&synchro, 1); + + ret = per_lcore_mempool_test(mp); + + cores = cores_save; + RTE_LCORE_FOREACH_SLAVE(lcore_id) { + if (cores == 1) + break; + cores--; + if (rte_eal_wait_lcore(lcore_id) < 0) + ret = -1; + } + + if (ret < 0) { + printf("per-lcore test returned -1\n"); + return -1; + } + + rate = 0; + for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) + rate += (stats[lcore_id].enq_count / TIME_S); + + printf("rate_persec=%" PRIu64 "\n", rate); + + return 0; +} + +/* for a given number of core, launch all test cases */ +static int +do_one_mempool_test(struct rte_mempool *mp, unsigned int cores) +{ + unsigned bulk_tab_get[] = { 1, 4, 32, 0 }; + unsigned bulk_tab_put[] = { 1, 4, 32, 0 }; + unsigned keep_tab[] = { 32, 128, 0 }; + unsigned *get_bulk_ptr; + unsigned *put_bulk_ptr; + unsigned *keep_ptr; + int ret; + + for (get_bulk_ptr = bulk_tab_get; *get_bulk_ptr; get_bulk_ptr++) { + for (put_bulk_ptr = bulk_tab_put; *put_bulk_ptr; put_bulk_ptr++) { + for (keep_ptr = keep_tab; *keep_ptr; keep_ptr++) { + + n_get_bulk = *get_bulk_ptr; + n_put_bulk = *put_bulk_ptr; + n_keep = *keep_ptr; + ret = launch_cores(mp, cores); + + if (ret < 0) + return -1; + } + } + } + return 0; +} + +static int +test_mempool_perf(void) +{ + struct rte_mempool *mp_cache = NULL; + struct rte_mempool *mp_nocache = NULL; + struct rte_mempool *default_pool = NULL; + int ret = -1; + + rte_atomic32_init(&synchro); + + /* create a mempool (without cache) */ + mp_nocache = rte_mempool_create("perf_test_nocache", MEMPOOL_SIZE, + MEMPOOL_ELT_SIZE, 0, 0, + NULL, NULL, + my_obj_init, NULL, + SOCKET_ID_ANY, 0); + if (mp_nocache == NULL) + goto err; + + /* create a mempool (with cache) */ + mp_cache = rte_mempool_create("perf_test_cache", MEMPOOL_SIZE, + MEMPOOL_ELT_SIZE, + RTE_MEMPOOL_CACHE_MAX_SIZE, 0, + NULL, NULL, + my_obj_init, NULL, + SOCKET_ID_ANY, 0); + if (mp_cache == NULL) + goto err; + + /* Create a mempool based on Default handler */ + default_pool = rte_mempool_create_empty("default_pool", + MEMPOOL_SIZE, + MEMPOOL_ELT_SIZE, + 0, 0, + SOCKET_ID_ANY, 0); + + if (default_pool == NULL) { + printf("cannot allocate %s mempool\n", + RTE_MBUF_DEFAULT_MEMPOOL_OPS); + goto err; + } + + if (rte_mempool_set_ops_byname(default_pool, + RTE_MBUF_DEFAULT_MEMPOOL_OPS, NULL) + < 0) { + printf("cannot set %s handler\n", RTE_MBUF_DEFAULT_MEMPOOL_OPS); + goto err; + } + + if (rte_mempool_populate_default(default_pool) < 0) { + printf("cannot populate %s mempool\n", + RTE_MBUF_DEFAULT_MEMPOOL_OPS); + goto err; + } + + rte_mempool_obj_iter(default_pool, my_obj_init, NULL); + + /* performance test with 1, 2 and max cores */ + printf("start performance test (without cache)\n"); + + if (do_one_mempool_test(mp_nocache, 1) < 0) + goto err; + + if (do_one_mempool_test(mp_nocache, 2) < 0) + goto err; + + if (do_one_mempool_test(mp_nocache, rte_lcore_count()) < 0) + goto err; + + /* performance test with 1, 2 and max cores */ + printf("start performance test for %s (without cache)\n", + RTE_MBUF_DEFAULT_MEMPOOL_OPS); + + if (do_one_mempool_test(default_pool, 1) < 0) + goto err; + + if (do_one_mempool_test(default_pool, 2) < 0) + goto err; + + if (do_one_mempool_test(default_pool, rte_lcore_count()) < 0) + goto err; + + /* performance test with 1, 2 and max cores */ + printf("start performance test (with cache)\n"); + + if (do_one_mempool_test(mp_cache, 1) < 0) + goto err; + + if (do_one_mempool_test(mp_cache, 2) < 0) + goto err; + + if (do_one_mempool_test(mp_cache, rte_lcore_count()) < 0) + goto err; + + /* performance test with 1, 2 and max cores */ + printf("start performance test (with user-owned cache)\n"); + use_external_cache = 1; + + if (do_one_mempool_test(mp_nocache, 1) < 0) + goto err; + + if (do_one_mempool_test(mp_nocache, 2) < 0) + goto err; + + if (do_one_mempool_test(mp_nocache, rte_lcore_count()) < 0) + goto err; + + rte_mempool_list_dump(stdout); + + ret = 0; + +err: + rte_mempool_free(mp_cache); + rte_mempool_free(mp_nocache); + rte_mempool_free(default_pool); + return ret; +} + +REGISTER_TEST_COMMAND(mempool_perf_autotest, test_mempool_perf); diff --git a/dpdk/test/test/test_memzone.c b/dpdk/test/test/test_memzone.c new file mode 100644 index 00000000..1115ee0f --- /dev/null +++ b/dpdk/test/test/test_memzone.c @@ -0,0 +1,1022 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <stdint.h> +#include <inttypes.h> +#include <sys/queue.h> + +#include <rte_random.h> +#include <rte_cycles.h> +#include <rte_memory.h> +#include <rte_memzone.h> +#include <rte_eal.h> +#include <rte_eal_memconfig.h> +#include <rte_common.h> +#include <rte_string_fns.h> +#include <rte_errno.h> +#include <rte_malloc.h> +#include "../../lib/librte_eal/common/malloc_elem.h" + +#include "test.h" + +/* + * Memzone + * ======= + * + * - Search for three reserved zones or reserve them if they do not exist: + * + * - One is on any socket id. + * - The second is on socket 0. + * - The last one is on socket 1 (if socket 1 exists). + * + * - Check that the zones exist. + * + * - Check that the zones are cache-aligned. + * + * - Check that zones do not overlap. + * + * - Check that the zones are on the correct socket id. + * + * - Check that a lookup of the first zone returns the same pointer. + * + * - Check that it is not possible to create another zone with the + * same name as an existing zone. + * + * - Check flags for specific huge page size reservation + */ + +/* Test if memory overlaps: return 1 if true, or 0 if false. */ +static int +is_memory_overlap(rte_iova_t ptr1, size_t len1, rte_iova_t ptr2, size_t len2) +{ + if (ptr2 >= ptr1 && (ptr2 - ptr1) < len1) + return 1; + else if (ptr2 < ptr1 && (ptr1 - ptr2) < len2) + return 1; + return 0; +} + +static int +test_memzone_invalid_alignment(void) +{ + const struct rte_memzone * mz; + + mz = rte_memzone_lookup("invalid_alignment"); + if (mz != NULL) { + printf("Zone with invalid alignment has been reserved\n"); + return -1; + } + + mz = rte_memzone_reserve_aligned("invalid_alignment", 100, + SOCKET_ID_ANY, 0, 100); + if (mz != NULL) { + printf("Zone with invalid alignment has been reserved\n"); + return -1; + } + return 0; +} + +static int +test_memzone_reserving_zone_size_bigger_than_the_maximum(void) +{ + const struct rte_memzone * mz; + + mz = rte_memzone_lookup("zone_size_bigger_than_the_maximum"); + if (mz != NULL) { + printf("zone_size_bigger_than_the_maximum has been reserved\n"); + return -1; + } + + mz = rte_memzone_reserve("zone_size_bigger_than_the_maximum", (size_t)-1, + SOCKET_ID_ANY, 0); + if (mz != NULL) { + printf("It is impossible to reserve such big a memzone\n"); + return -1; + } + + return 0; +} + +static int +test_memzone_reserve_flags(void) +{ + const struct rte_memzone *mz; + const struct rte_memseg *ms; + int hugepage_2MB_avail = 0; + int hugepage_1GB_avail = 0; + int hugepage_16MB_avail = 0; + int hugepage_16GB_avail = 0; + const size_t size = 100; + int i = 0; + ms = rte_eal_get_physmem_layout(); + for (i = 0; i < RTE_MAX_MEMSEG; i++) { + if (ms[i].hugepage_sz == RTE_PGSIZE_2M) + hugepage_2MB_avail = 1; + if (ms[i].hugepage_sz == RTE_PGSIZE_1G) + hugepage_1GB_avail = 1; + if (ms[i].hugepage_sz == RTE_PGSIZE_16M) + hugepage_16MB_avail = 1; + if (ms[i].hugepage_sz == RTE_PGSIZE_16G) + hugepage_16GB_avail = 1; + } + /* Display the availability of 2MB ,1GB, 16MB, 16GB pages */ + if (hugepage_2MB_avail) + printf("2MB Huge pages available\n"); + if (hugepage_1GB_avail) + printf("1GB Huge pages available\n"); + if (hugepage_16MB_avail) + printf("16MB Huge pages available\n"); + if (hugepage_16GB_avail) + printf("16GB Huge pages available\n"); + /* + * If 2MB pages available, check that a small memzone is correctly + * reserved from 2MB huge pages when requested by the RTE_MEMZONE_2MB flag. + * Also check that RTE_MEMZONE_SIZE_HINT_ONLY flag only defaults to an + * available page size (i.e 1GB ) when 2MB pages are unavailable. + */ + if (hugepage_2MB_avail) { + mz = rte_memzone_reserve("flag_zone_2M", size, SOCKET_ID_ANY, + RTE_MEMZONE_2MB); + if (mz == NULL) { + printf("MEMZONE FLAG 2MB\n"); + return -1; + } + if (mz->hugepage_sz != RTE_PGSIZE_2M) { + printf("hugepage_sz not equal 2M\n"); + return -1; + } + if (rte_memzone_free(mz)) { + printf("Fail memzone free\n"); + return -1; + } + + mz = rte_memzone_reserve("flag_zone_2M_HINT", size, SOCKET_ID_ANY, + RTE_MEMZONE_2MB|RTE_MEMZONE_SIZE_HINT_ONLY); + if (mz == NULL) { + printf("MEMZONE FLAG 2MB\n"); + return -1; + } + if (mz->hugepage_sz != RTE_PGSIZE_2M) { + printf("hugepage_sz not equal 2M\n"); + return -1; + } + if (rte_memzone_free(mz)) { + printf("Fail memzone free\n"); + return -1; + } + + /* Check if 1GB huge pages are unavailable, that function fails unless + * HINT flag is indicated + */ + if (!hugepage_1GB_avail) { + mz = rte_memzone_reserve("flag_zone_1G_HINT", size, SOCKET_ID_ANY, + RTE_MEMZONE_1GB|RTE_MEMZONE_SIZE_HINT_ONLY); + if (mz == NULL) { + printf("MEMZONE FLAG 1GB & HINT\n"); + return -1; + } + if (mz->hugepage_sz != RTE_PGSIZE_2M) { + printf("hugepage_sz not equal 2M\n"); + return -1; + } + if (rte_memzone_free(mz)) { + printf("Fail memzone free\n"); + return -1; + } + + mz = rte_memzone_reserve("flag_zone_1G", size, SOCKET_ID_ANY, + RTE_MEMZONE_1GB); + if (mz != NULL) { + printf("MEMZONE FLAG 1GB\n"); + return -1; + } + } + } + + /*As with 2MB tests above for 1GB huge page requests*/ + if (hugepage_1GB_avail) { + mz = rte_memzone_reserve("flag_zone_1G", size, SOCKET_ID_ANY, + RTE_MEMZONE_1GB); + if (mz == NULL) { + printf("MEMZONE FLAG 1GB\n"); + return -1; + } + if (mz->hugepage_sz != RTE_PGSIZE_1G) { + printf("hugepage_sz not equal 1G\n"); + return -1; + } + if (rte_memzone_free(mz)) { + printf("Fail memzone free\n"); + return -1; + } + + mz = rte_memzone_reserve("flag_zone_1G_HINT", size, SOCKET_ID_ANY, + RTE_MEMZONE_1GB|RTE_MEMZONE_SIZE_HINT_ONLY); + if (mz == NULL) { + printf("MEMZONE FLAG 1GB\n"); + return -1; + } + if (mz->hugepage_sz != RTE_PGSIZE_1G) { + printf("hugepage_sz not equal 1G\n"); + return -1; + } + if (rte_memzone_free(mz)) { + printf("Fail memzone free\n"); + return -1; + } + + /* Check if 1GB huge pages are unavailable, that function fails unless + * HINT flag is indicated + */ + if (!hugepage_2MB_avail) { + mz = rte_memzone_reserve("flag_zone_2M_HINT", size, SOCKET_ID_ANY, + RTE_MEMZONE_2MB|RTE_MEMZONE_SIZE_HINT_ONLY); + if (mz == NULL){ + printf("MEMZONE FLAG 2MB & HINT\n"); + return -1; + } + if (mz->hugepage_sz != RTE_PGSIZE_1G) { + printf("hugepage_sz not equal 1G\n"); + return -1; + } + if (rte_memzone_free(mz)) { + printf("Fail memzone free\n"); + return -1; + } + mz = rte_memzone_reserve("flag_zone_2M", size, SOCKET_ID_ANY, + RTE_MEMZONE_2MB); + if (mz != NULL) { + printf("MEMZONE FLAG 2MB\n"); + return -1; + } + } + + if (hugepage_2MB_avail && hugepage_1GB_avail) { + mz = rte_memzone_reserve("flag_zone_2M_HINT", size, SOCKET_ID_ANY, + RTE_MEMZONE_2MB|RTE_MEMZONE_1GB); + if (mz == NULL) { + printf("BOTH SIZES SET\n"); + return -1; + } + if (mz->hugepage_sz != RTE_PGSIZE_1G && + mz->hugepage_sz != RTE_PGSIZE_2M) { + printf("Wrong size when both sizes set\n"); + return -1; + } + if (rte_memzone_free(mz)) { + printf("Fail memzone free\n"); + return -1; + } + } + } + /* + * This option is for IBM Power. If 16MB pages available, check + * that a small memzone is correctly reserved from 16MB huge pages + * when requested by the RTE_MEMZONE_16MB flag. Also check that + * RTE_MEMZONE_SIZE_HINT_ONLY flag only defaults to an available + * page size (i.e 16GB ) when 16MB pages are unavailable. + */ + if (hugepage_16MB_avail) { + mz = rte_memzone_reserve("flag_zone_16M", size, SOCKET_ID_ANY, + RTE_MEMZONE_16MB); + if (mz == NULL) { + printf("MEMZONE FLAG 16MB\n"); + return -1; + } + if (mz->hugepage_sz != RTE_PGSIZE_16M) { + printf("hugepage_sz not equal 16M\n"); + return -1; + } + if (rte_memzone_free(mz)) { + printf("Fail memzone free\n"); + return -1; + } + + mz = rte_memzone_reserve("flag_zone_16M_HINT", size, + SOCKET_ID_ANY, RTE_MEMZONE_16MB|RTE_MEMZONE_SIZE_HINT_ONLY); + if (mz == NULL) { + printf("MEMZONE FLAG 2MB\n"); + return -1; + } + if (mz->hugepage_sz != RTE_PGSIZE_16M) { + printf("hugepage_sz not equal 16M\n"); + return -1; + } + if (rte_memzone_free(mz)) { + printf("Fail memzone free\n"); + return -1; + } + + /* Check if 1GB huge pages are unavailable, that function fails + * unless HINT flag is indicated + */ + if (!hugepage_16GB_avail) { + mz = rte_memzone_reserve("flag_zone_16G_HINT", size, + SOCKET_ID_ANY, + RTE_MEMZONE_16GB|RTE_MEMZONE_SIZE_HINT_ONLY); + if (mz == NULL) { + printf("MEMZONE FLAG 16GB & HINT\n"); + return -1; + } + if (mz->hugepage_sz != RTE_PGSIZE_16M) { + printf("hugepage_sz not equal 16M\n"); + return -1; + } + if (rte_memzone_free(mz)) { + printf("Fail memzone free\n"); + return -1; + } + + mz = rte_memzone_reserve("flag_zone_16G", size, + SOCKET_ID_ANY, RTE_MEMZONE_16GB); + if (mz != NULL) { + printf("MEMZONE FLAG 16GB\n"); + return -1; + } + } + } + /*As with 16MB tests above for 16GB huge page requests*/ + if (hugepage_16GB_avail) { + mz = rte_memzone_reserve("flag_zone_16G", size, SOCKET_ID_ANY, + RTE_MEMZONE_16GB); + if (mz == NULL) { + printf("MEMZONE FLAG 16GB\n"); + return -1; + } + if (mz->hugepage_sz != RTE_PGSIZE_16G) { + printf("hugepage_sz not equal 16G\n"); + return -1; + } + if (rte_memzone_free(mz)) { + printf("Fail memzone free\n"); + return -1; + } + + mz = rte_memzone_reserve("flag_zone_16G_HINT", size, + SOCKET_ID_ANY, RTE_MEMZONE_16GB|RTE_MEMZONE_SIZE_HINT_ONLY); + if (mz == NULL) { + printf("MEMZONE FLAG 16GB\n"); + return -1; + } + if (mz->hugepage_sz != RTE_PGSIZE_16G) { + printf("hugepage_sz not equal 16G\n"); + return -1; + } + if (rte_memzone_free(mz)) { + printf("Fail memzone free\n"); + return -1; + } + + /* Check if 1GB huge pages are unavailable, that function fails + * unless HINT flag is indicated + */ + if (!hugepage_16MB_avail) { + mz = rte_memzone_reserve("flag_zone_16M_HINT", size, + SOCKET_ID_ANY, + RTE_MEMZONE_16MB|RTE_MEMZONE_SIZE_HINT_ONLY); + if (mz == NULL) { + printf("MEMZONE FLAG 16MB & HINT\n"); + return -1; + } + if (mz->hugepage_sz != RTE_PGSIZE_16G) { + printf("hugepage_sz not equal 16G\n"); + return -1; + } + if (rte_memzone_free(mz)) { + printf("Fail memzone free\n"); + return -1; + } + mz = rte_memzone_reserve("flag_zone_16M", size, + SOCKET_ID_ANY, RTE_MEMZONE_16MB); + if (mz != NULL) { + printf("MEMZONE FLAG 16MB\n"); + return -1; + } + } + + if (hugepage_16MB_avail && hugepage_16GB_avail) { + mz = rte_memzone_reserve("flag_zone_16M_HINT", size, + SOCKET_ID_ANY, + RTE_MEMZONE_16MB|RTE_MEMZONE_16GB); + if (mz == NULL) { + printf("BOTH SIZES SET\n"); + return -1; + } + if (mz->hugepage_sz != RTE_PGSIZE_16G && + mz->hugepage_sz != RTE_PGSIZE_16M) { + printf("Wrong size when both sizes set\n"); + return -1; + } + if (rte_memzone_free(mz)) { + printf("Fail memzone free\n"); + return -1; + } + } + } + return 0; +} + + +/* Find the heap with the greatest free block size */ +static size_t +find_max_block_free_size(const unsigned _align) +{ + struct rte_malloc_socket_stats stats; + unsigned i, align = _align; + size_t len = 0; + + for (i = 0; i < RTE_MAX_NUMA_NODES; i++) { + rte_malloc_get_socket_stats(i, &stats); + if (stats.greatest_free_size > len) + len = stats.greatest_free_size; + } + + if (align < RTE_CACHE_LINE_SIZE) + align = RTE_CACHE_LINE_ROUNDUP(align+1); + + if (len <= MALLOC_ELEM_OVERHEAD + align) + return 0; + + return len - MALLOC_ELEM_OVERHEAD - align; +} + +static int +test_memzone_reserve_max(void) +{ + const struct rte_memzone *mz; + size_t maxlen; + + maxlen = find_max_block_free_size(0); + + if (maxlen == 0) { + printf("There is no space left!\n"); + return 0; + } + + mz = rte_memzone_reserve("max_zone", 0, SOCKET_ID_ANY, 0); + if (mz == NULL){ + printf("Failed to reserve a big chunk of memory - %s\n", + rte_strerror(rte_errno)); + rte_dump_physmem_layout(stdout); + rte_memzone_dump(stdout); + return -1; + } + + if (mz->len != maxlen) { + printf("Memzone reserve with 0 size did not return bigest block\n"); + printf("Expected size = %zu, actual size = %zu\n", maxlen, mz->len); + rte_dump_physmem_layout(stdout); + rte_memzone_dump(stdout); + return -1; + } + + if (rte_memzone_free(mz)) { + printf("Fail memzone free\n"); + return -1; + } + + return 0; +} + +static int +test_memzone_reserve_max_aligned(void) +{ + const struct rte_memzone *mz; + size_t maxlen = 0; + + /* random alignment */ + rte_srand((unsigned)rte_rdtsc()); + const unsigned align = 1 << ((rte_rand() % 8) + 5); /* from 128 up to 4k alignment */ + + maxlen = find_max_block_free_size(align); + + if (maxlen == 0) { + printf("There is no space left for biggest %u-aligned memzone!\n", align); + return 0; + } + + mz = rte_memzone_reserve_aligned("max_zone_aligned", 0, + SOCKET_ID_ANY, 0, align); + if (mz == NULL){ + printf("Failed to reserve a big chunk of memory - %s\n", + rte_strerror(rte_errno)); + rte_dump_physmem_layout(stdout); + rte_memzone_dump(stdout); + return -1; + } + + if (mz->len != maxlen) { + printf("Memzone reserve with 0 size and alignment %u did not return" + " bigest block\n", align); + printf("Expected size = %zu, actual size = %zu\n", + maxlen, mz->len); + rte_dump_physmem_layout(stdout); + rte_memzone_dump(stdout); + return -1; + } + + if (rte_memzone_free(mz)) { + printf("Fail memzone free\n"); + return -1; + } + + return 0; +} + +static int +test_memzone_aligned(void) +{ + const struct rte_memzone *memzone_aligned_32; + const struct rte_memzone *memzone_aligned_128; + const struct rte_memzone *memzone_aligned_256; + const struct rte_memzone *memzone_aligned_512; + const struct rte_memzone *memzone_aligned_1024; + + /* memzone that should automatically be adjusted to align on 64 bytes */ + memzone_aligned_32 = rte_memzone_reserve_aligned("aligned_32", 100, + SOCKET_ID_ANY, 0, 32); + + /* memzone that is supposed to be aligned on a 128 byte boundary */ + memzone_aligned_128 = rte_memzone_reserve_aligned("aligned_128", 100, + SOCKET_ID_ANY, 0, 128); + + /* memzone that is supposed to be aligned on a 256 byte boundary */ + memzone_aligned_256 = rte_memzone_reserve_aligned("aligned_256", 100, + SOCKET_ID_ANY, 0, 256); + + /* memzone that is supposed to be aligned on a 512 byte boundary */ + memzone_aligned_512 = rte_memzone_reserve_aligned("aligned_512", 100, + SOCKET_ID_ANY, 0, 512); + + /* memzone that is supposed to be aligned on a 1024 byte boundary */ + memzone_aligned_1024 = rte_memzone_reserve_aligned("aligned_1024", 100, + SOCKET_ID_ANY, 0, 1024); + + printf("check alignments and lengths\n"); + if (memzone_aligned_32 == NULL) { + printf("Unable to reserve 64-byte aligned memzone!\n"); + return -1; + } + if ((memzone_aligned_32->iova & RTE_CACHE_LINE_MASK) != 0) + return -1; + if (((uintptr_t) memzone_aligned_32->addr & RTE_CACHE_LINE_MASK) != 0) + return -1; + if ((memzone_aligned_32->len & RTE_CACHE_LINE_MASK) != 0) + return -1; + + if (memzone_aligned_128 == NULL) { + printf("Unable to reserve 128-byte aligned memzone!\n"); + return -1; + } + if ((memzone_aligned_128->iova & 127) != 0) + return -1; + if (((uintptr_t) memzone_aligned_128->addr & 127) != 0) + return -1; + if ((memzone_aligned_128->len & RTE_CACHE_LINE_MASK) != 0) + return -1; + + if (memzone_aligned_256 == NULL) { + printf("Unable to reserve 256-byte aligned memzone!\n"); + return -1; + } + if ((memzone_aligned_256->iova & 255) != 0) + return -1; + if (((uintptr_t) memzone_aligned_256->addr & 255) != 0) + return -1; + if ((memzone_aligned_256->len & RTE_CACHE_LINE_MASK) != 0) + return -1; + + if (memzone_aligned_512 == NULL) { + printf("Unable to reserve 512-byte aligned memzone!\n"); + return -1; + } + if ((memzone_aligned_512->iova & 511) != 0) + return -1; + if (((uintptr_t) memzone_aligned_512->addr & 511) != 0) + return -1; + if ((memzone_aligned_512->len & RTE_CACHE_LINE_MASK) != 0) + return -1; + + if (memzone_aligned_1024 == NULL) { + printf("Unable to reserve 1024-byte aligned memzone!\n"); + return -1; + } + if ((memzone_aligned_1024->iova & 1023) != 0) + return -1; + if (((uintptr_t) memzone_aligned_1024->addr & 1023) != 0) + return -1; + if ((memzone_aligned_1024->len & RTE_CACHE_LINE_MASK) != 0) + return -1; + + /* check that zones don't overlap */ + printf("check overlapping\n"); + if (is_memory_overlap(memzone_aligned_32->iova, memzone_aligned_32->len, + memzone_aligned_128->iova, memzone_aligned_128->len)) + return -1; + if (is_memory_overlap(memzone_aligned_32->iova, memzone_aligned_32->len, + memzone_aligned_256->iova, memzone_aligned_256->len)) + return -1; + if (is_memory_overlap(memzone_aligned_32->iova, memzone_aligned_32->len, + memzone_aligned_512->iova, memzone_aligned_512->len)) + return -1; + if (is_memory_overlap(memzone_aligned_32->iova, memzone_aligned_32->len, + memzone_aligned_1024->iova, memzone_aligned_1024->len)) + return -1; + if (is_memory_overlap(memzone_aligned_128->iova, memzone_aligned_128->len, + memzone_aligned_256->iova, memzone_aligned_256->len)) + return -1; + if (is_memory_overlap(memzone_aligned_128->iova, memzone_aligned_128->len, + memzone_aligned_512->iova, memzone_aligned_512->len)) + return -1; + if (is_memory_overlap(memzone_aligned_128->iova, memzone_aligned_128->len, + memzone_aligned_1024->iova, memzone_aligned_1024->len)) + return -1; + if (is_memory_overlap(memzone_aligned_256->iova, memzone_aligned_256->len, + memzone_aligned_512->iova, memzone_aligned_512->len)) + return -1; + if (is_memory_overlap(memzone_aligned_256->iova, memzone_aligned_256->len, + memzone_aligned_1024->iova, memzone_aligned_1024->len)) + return -1; + if (is_memory_overlap(memzone_aligned_512->iova, memzone_aligned_512->len, + memzone_aligned_1024->iova, memzone_aligned_1024->len)) + return -1; + + /* free all used zones */ + if (rte_memzone_free(memzone_aligned_32)) { + printf("Fail memzone free\n"); + return -1; + } + if (rte_memzone_free(memzone_aligned_128)) { + printf("Fail memzone free\n"); + return -1; + } + if (rte_memzone_free(memzone_aligned_256)) { + printf("Fail memzone free\n"); + return -1; + } + if (rte_memzone_free(memzone_aligned_512)) { + printf("Fail memzone free\n"); + return -1; + } + if (rte_memzone_free(memzone_aligned_1024)) { + printf("Fail memzone free\n"); + return -1; + } + return 0; +} + +static int +check_memzone_bounded(const char *name, uint32_t len, uint32_t align, + uint32_t bound) +{ + const struct rte_memzone *mz; + rte_iova_t bmask; + + bmask = ~((rte_iova_t)bound - 1); + + if ((mz = rte_memzone_reserve_bounded(name, len, SOCKET_ID_ANY, 0, + align, bound)) == NULL) { + printf("%s(%s): memzone creation failed\n", + __func__, name); + return -1; + } + + if ((mz->iova & ((rte_iova_t)align - 1)) != 0) { + printf("%s(%s): invalid phys addr alignment\n", + __func__, mz->name); + return -1; + } + + if (((uintptr_t) mz->addr & ((uintptr_t)align - 1)) != 0) { + printf("%s(%s): invalid virtual addr alignment\n", + __func__, mz->name); + return -1; + } + + if ((mz->len & RTE_CACHE_LINE_MASK) != 0 || mz->len < len || + mz->len < RTE_CACHE_LINE_SIZE) { + printf("%s(%s): invalid length\n", + __func__, mz->name); + return -1; + } + + if ((mz->iova & bmask) != + ((mz->iova + mz->len - 1) & bmask)) { + printf("%s(%s): invalid memzone boundary %u crossed\n", + __func__, mz->name, bound); + return -1; + } + + if (rte_memzone_free(mz)) { + printf("Fail memzone free\n"); + return -1; + } + + return 0; +} + +static int +test_memzone_bounded(void) +{ + const struct rte_memzone *memzone_err; + const char *name; + int rc; + + /* should fail as boundary is not power of two */ + name = "bounded_error_31"; + if ((memzone_err = rte_memzone_reserve_bounded(name, + 100, SOCKET_ID_ANY, 0, 32, UINT32_MAX)) != NULL) { + printf("%s(%s)created a memzone with invalid boundary " + "conditions\n", __func__, memzone_err->name); + return -1; + } + + /* should fail as len is greater then boundary */ + name = "bounded_error_32"; + if ((memzone_err = rte_memzone_reserve_bounded(name, + 100, SOCKET_ID_ANY, 0, 32, 32)) != NULL) { + printf("%s(%s)created a memzone with invalid boundary " + "conditions\n", __func__, memzone_err->name); + return -1; + } + + if ((rc = check_memzone_bounded("bounded_128", 100, 128, 128)) != 0) + return rc; + + if ((rc = check_memzone_bounded("bounded_256", 100, 256, 128)) != 0) + return rc; + + if ((rc = check_memzone_bounded("bounded_1K", 100, 64, 1024)) != 0) + return rc; + + if ((rc = check_memzone_bounded("bounded_1K_MAX", 0, 64, 1024)) != 0) + return rc; + + return 0; +} + +static int +test_memzone_free(void) +{ + const struct rte_memzone *mz[RTE_MAX_MEMZONE + 1]; + int i; + char name[20]; + + mz[0] = rte_memzone_reserve("tempzone0", 2000, SOCKET_ID_ANY, 0); + mz[1] = rte_memzone_reserve("tempzone1", 4000, SOCKET_ID_ANY, 0); + + if (mz[0] > mz[1]) + return -1; + if (!rte_memzone_lookup("tempzone0")) + return -1; + if (!rte_memzone_lookup("tempzone1")) + return -1; + + if (rte_memzone_free(mz[0])) { + printf("Fail memzone free - tempzone0\n"); + return -1; + } + if (rte_memzone_lookup("tempzone0")) { + printf("Found previously free memzone - tempzone0\n"); + return -1; + } + mz[2] = rte_memzone_reserve("tempzone2", 2000, SOCKET_ID_ANY, 0); + + if (mz[2] > mz[1]) { + printf("tempzone2 should have gotten the free entry from tempzone0\n"); + return -1; + } + if (rte_memzone_free(mz[2])) { + printf("Fail memzone free - tempzone2\n"); + return -1; + } + if (rte_memzone_lookup("tempzone2")) { + printf("Found previously free memzone - tempzone2\n"); + return -1; + } + if (rte_memzone_free(mz[1])) { + printf("Fail memzone free - tempzone1\n"); + return -1; + } + if (rte_memzone_lookup("tempzone1")) { + printf("Found previously free memzone - tempzone1\n"); + return -1; + } + + i = 0; + do { + snprintf(name, sizeof(name), "tempzone%u", i); + mz[i] = rte_memzone_reserve(name, 1, SOCKET_ID_ANY, 0); + } while (mz[i++] != NULL); + + if (rte_memzone_free(mz[0])) { + printf("Fail memzone free - tempzone0\n"); + return -1; + } + mz[0] = rte_memzone_reserve("tempzone0new", 0, SOCKET_ID_ANY, 0); + + if (mz[0] == NULL) { + printf("Fail to create memzone - tempzone0new - when MAX memzones were " + "created and one was free\n"); + return -1; + } + + for (i = i - 2; i >= 0; i--) { + if (rte_memzone_free(mz[i])) { + printf("Fail memzone free - tempzone%d\n", i); + return -1; + } + } + + return 0; +} + +static int +test_memzone_basic(void) +{ + const struct rte_memzone *memzone1; + const struct rte_memzone *memzone2; + const struct rte_memzone *memzone3; + const struct rte_memzone *memzone4; + const struct rte_memzone *mz; + + memzone1 = rte_memzone_reserve("testzone1", 100, + SOCKET_ID_ANY, 0); + + memzone2 = rte_memzone_reserve("testzone2", 1000, + 0, 0); + + memzone3 = rte_memzone_reserve("testzone3", 1000, + 1, 0); + + memzone4 = rte_memzone_reserve("testzone4", 1024, + SOCKET_ID_ANY, 0); + + /* memzone3 may be NULL if we don't have NUMA */ + if (memzone1 == NULL || memzone2 == NULL || memzone4 == NULL) + return -1; + + rte_memzone_dump(stdout); + + /* check cache-line alignments */ + printf("check alignments and lengths\n"); + + if ((memzone1->iova & RTE_CACHE_LINE_MASK) != 0) + return -1; + if ((memzone2->iova & RTE_CACHE_LINE_MASK) != 0) + return -1; + if (memzone3 != NULL && (memzone3->iova & RTE_CACHE_LINE_MASK) != 0) + return -1; + if ((memzone1->len & RTE_CACHE_LINE_MASK) != 0 || memzone1->len == 0) + return -1; + if ((memzone2->len & RTE_CACHE_LINE_MASK) != 0 || memzone2->len == 0) + return -1; + if (memzone3 != NULL && ((memzone3->len & RTE_CACHE_LINE_MASK) != 0 || + memzone3->len == 0)) + return -1; + if (memzone4->len != 1024) + return -1; + + /* check that zones don't overlap */ + printf("check overlapping\n"); + + if (is_memory_overlap(memzone1->iova, memzone1->len, + memzone2->iova, memzone2->len)) + return -1; + if (memzone3 != NULL && + is_memory_overlap(memzone1->iova, memzone1->len, + memzone3->iova, memzone3->len)) + return -1; + if (memzone3 != NULL && + is_memory_overlap(memzone2->iova, memzone2->len, + memzone3->iova, memzone3->len)) + return -1; + + printf("check socket ID\n"); + + /* memzone2 must be on socket id 0 and memzone3 on socket 1 */ + if (memzone2->socket_id != 0) + return -1; + if (memzone3 != NULL && memzone3->socket_id != 1) + return -1; + + printf("test zone lookup\n"); + mz = rte_memzone_lookup("testzone1"); + if (mz != memzone1) + return -1; + + printf("test duplcate zone name\n"); + mz = rte_memzone_reserve("testzone1", 100, + SOCKET_ID_ANY, 0); + if (mz != NULL) + return -1; + + if (rte_memzone_free(memzone1)) { + printf("Fail memzone free - memzone1\n"); + return -1; + } + if (rte_memzone_free(memzone2)) { + printf("Fail memzone free - memzone2\n"); + return -1; + } + if (memzone3 && rte_memzone_free(memzone3)) { + printf("Fail memzone free - memzone3\n"); + return -1; + } + if (rte_memzone_free(memzone4)) { + printf("Fail memzone free - memzone4\n"); + return -1; + } + + return 0; +} + +static int memzone_calk_called; +static void memzone_walk_clb(const struct rte_memzone *mz __rte_unused, + void *arg __rte_unused) +{ + memzone_calk_called = 1; +} + +static int +test_memzone(void) +{ + printf("test basic memzone API\n"); + if (test_memzone_basic() < 0) + return -1; + + printf("test free memzone\n"); + if (test_memzone_free() < 0) + return -1; + + printf("test reserving memzone with bigger size than the maximum\n"); + if (test_memzone_reserving_zone_size_bigger_than_the_maximum() < 0) + return -1; + + printf("test memzone_reserve flags\n"); + if (test_memzone_reserve_flags() < 0) + return -1; + + printf("test alignment for memzone_reserve\n"); + if (test_memzone_aligned() < 0) + return -1; + + printf("test boundary alignment for memzone_reserve\n"); + if (test_memzone_bounded() < 0) + return -1; + + printf("test invalid alignment for memzone_reserve\n"); + if (test_memzone_invalid_alignment() < 0) + return -1; + + printf("test reserving the largest size memzone possible\n"); + if (test_memzone_reserve_max() < 0) + return -1; + + printf("test reserving the largest size aligned memzone possible\n"); + if (test_memzone_reserve_max_aligned() < 0) + return -1; + + printf("check memzone cleanup\n"); + rte_memzone_walk(memzone_walk_clb, NULL); + if (memzone_calk_called) { + printf("there are some memzones left after test\n"); + rte_memzone_dump(stdout); + return -1; + } + + return 0; +} + +REGISTER_TEST_COMMAND(memzone_autotest, test_memzone); diff --git a/dpdk/test/test/test_meter.c b/dpdk/test/test/test_meter.c new file mode 100644 index 00000000..26b05657 --- /dev/null +++ b/dpdk/test/test/test_meter.c @@ -0,0 +1,497 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdlib.h> +#include <stdio.h> +#include <string.h> +#include <stdint.h> +#include <unistd.h> + +#include "test.h" + +#include <rte_cycles.h> +#include <rte_meter.h> + +#define mlog(format, ...) do{\ + printf("Line %d:",__LINE__);\ + printf(format, ##__VA_ARGS__);\ + printf("\n");\ + }while(0); + +#define melog(format, ...) do{\ + printf("Line %d:",__LINE__);\ + printf(format, ##__VA_ARGS__);\ + printf(" failed!\n");\ + return -1;\ + }while(0); + +#define TM_TEST_SRTCM_CIR_DF 46000000 +#define TM_TEST_SRTCM_CBS_DF 2048 +#define TM_TEST_SRTCM_EBS_DF 4096 + +#define TM_TEST_TRTCM_CIR_DF 46000000 +#define TM_TEST_TRTCM_PIR_DF 69000000 +#define TM_TEST_TRTCM_CBS_DF 2048 +#define TM_TEST_TRTCM_PBS_DF 4096 + +static struct rte_meter_srtcm_params sparams = + {.cir = TM_TEST_SRTCM_CIR_DF, + .cbs = TM_TEST_SRTCM_CBS_DF, + .ebs = TM_TEST_SRTCM_EBS_DF,}; + +static struct rte_meter_trtcm_params tparams= + {.cir = TM_TEST_TRTCM_CIR_DF, + .pir = TM_TEST_TRTCM_PIR_DF, + .cbs = TM_TEST_TRTCM_CBS_DF, + .pbs = TM_TEST_TRTCM_PBS_DF,}; + +/** + * functional test for rte_meter_srtcm_config + */ +static inline int +tm_test_srtcm_config(void) +{ +#define SRTCM_CFG_MSG "srtcm_config" + struct rte_meter_srtcm sm; + struct rte_meter_srtcm_params sparams1; + + /* invalid parameter test */ + if(rte_meter_srtcm_config(NULL, NULL) == 0) + melog(SRTCM_CFG_MSG); + if(rte_meter_srtcm_config(&sm, NULL) == 0) + melog(SRTCM_CFG_MSG); + if(rte_meter_srtcm_config(NULL, &sparams) == 0) + melog(SRTCM_CFG_MSG); + + /* cbs and ebs can't both be zero */ + sparams1 = sparams; + sparams1.cbs = 0; + sparams1.ebs = 0; + if(rte_meter_srtcm_config(&sm, &sparams1) == 0) + melog(SRTCM_CFG_MSG); + + /* cir should never be 0 */ + sparams1 = sparams; + sparams1.cir = 0; + if(rte_meter_srtcm_config(&sm, &sparams1) == 0) + melog(SRTCM_CFG_MSG); + + /* one of ebs and cbs can be zero, should be successful */ + sparams1 = sparams; + sparams1.ebs = 0; + if(rte_meter_srtcm_config(&sm, &sparams1) != 0) + melog(SRTCM_CFG_MSG); + + sparams1 = sparams; + sparams1.cbs = 0; + if(rte_meter_srtcm_config(&sm, &sparams1) != 0) + melog(SRTCM_CFG_MSG); + + /* usual parameter, should be successful */ + if(rte_meter_srtcm_config(&sm, &sparams) != 0) + melog(SRTCM_CFG_MSG); + + return 0; + +} + +/** + * functional test for rte_meter_trtcm_config + */ +static inline int +tm_test_trtcm_config(void) +{ + struct rte_meter_trtcm tm; + struct rte_meter_trtcm_params tparams1; +#define TRTCM_CFG_MSG "trtcm_config" + + /* invalid parameter test */ + if(rte_meter_trtcm_config(NULL, NULL) == 0) + melog(TRTCM_CFG_MSG); + if(rte_meter_trtcm_config(&tm, NULL) == 0) + melog(TRTCM_CFG_MSG); + if(rte_meter_trtcm_config(NULL, &tparams) == 0) + melog(TRTCM_CFG_MSG); + + /* cir, cbs, pir and pbs never be zero */ + tparams1 = tparams; + tparams1.cir = 0; + if(rte_meter_trtcm_config(&tm, &tparams1) == 0) + melog(TRTCM_CFG_MSG); + + tparams1 = tparams; + tparams1.cbs = 0; + if(rte_meter_trtcm_config(&tm, &tparams1) == 0) + melog(TRTCM_CFG_MSG); + + tparams1 = tparams; + tparams1.pbs = 0; + if(rte_meter_trtcm_config(&tm, &tparams1) == 0) + melog(TRTCM_CFG_MSG); + + tparams1 = tparams; + tparams1.pir = 0; + if(rte_meter_trtcm_config(&tm, &tparams1) == 0) + melog(TRTCM_CFG_MSG); + + /* pir should be greater or equal to cir */ + tparams1 = tparams; + tparams1.pir = tparams1.cir - 1; + if(rte_meter_trtcm_config(&tm, &tparams1) == 0) + melog(TRTCM_CFG_MSG" pir < cir test"); + + /* usual parameter, should be successful */ + if(rte_meter_trtcm_config(&tm, &tparams) != 0) + melog(TRTCM_CFG_MSG); + + return 0; +} + +/** + * functional test for rte_meter_srtcm_color_blind_check + */ +static inline int +tm_test_srtcm_color_blind_check(void) +{ +#define SRTCM_BLIND_CHECK_MSG "srtcm_blind_check" + struct rte_meter_srtcm sm; + uint64_t time; + uint64_t hz = rte_get_tsc_hz(); + + /* Test green */ + if(rte_meter_srtcm_config(&sm, &sparams) != 0) + melog(SRTCM_BLIND_CHECK_MSG); + time = rte_get_tsc_cycles() + hz; + if(rte_meter_srtcm_color_blind_check( + &sm, time, TM_TEST_SRTCM_CBS_DF - 1) + != e_RTE_METER_GREEN) + melog(SRTCM_BLIND_CHECK_MSG" GREEN"); + + /* Test yellow */ + if(rte_meter_srtcm_config(&sm, &sparams) != 0) + melog(SRTCM_BLIND_CHECK_MSG); + time = rte_get_tsc_cycles() + hz; + if(rte_meter_srtcm_color_blind_check( + &sm, time, TM_TEST_SRTCM_CBS_DF + 1) + != e_RTE_METER_YELLOW) + melog(SRTCM_BLIND_CHECK_MSG" YELLOW"); + + if(rte_meter_srtcm_config(&sm, &sparams) != 0) + melog(SRTCM_BLIND_CHECK_MSG); + time = rte_get_tsc_cycles() + hz; + if(rte_meter_srtcm_color_blind_check( + &sm, time, (uint32_t)sm.ebs - 1) != e_RTE_METER_YELLOW) + melog(SRTCM_BLIND_CHECK_MSG" YELLOW"); + + /* Test red */ + if(rte_meter_srtcm_config(&sm, &sparams) != 0) + melog(SRTCM_BLIND_CHECK_MSG); + time = rte_get_tsc_cycles() + hz; + if(rte_meter_srtcm_color_blind_check( + &sm, time, TM_TEST_SRTCM_EBS_DF + 1) + != e_RTE_METER_RED) + melog(SRTCM_BLIND_CHECK_MSG" RED"); + + return 0; + +} + +/** + * functional test for rte_meter_trtcm_color_blind_check + */ +static inline int +tm_test_trtcm_color_blind_check(void) +{ +#define TRTCM_BLIND_CHECK_MSG "trtcm_blind_check" + + uint64_t time; + struct rte_meter_trtcm tm; + uint64_t hz = rte_get_tsc_hz(); + + /* Test green */ + if(rte_meter_trtcm_config(&tm, &tparams) != 0) + melog(TRTCM_BLIND_CHECK_MSG); + time = rte_get_tsc_cycles() + hz; + if(rte_meter_trtcm_color_blind_check( + &tm, time, TM_TEST_TRTCM_CBS_DF - 1) + != e_RTE_METER_GREEN) + melog(TRTCM_BLIND_CHECK_MSG" GREEN"); + + /* Test yellow */ + if(rte_meter_trtcm_config(&tm, &tparams) != 0) + melog(TRTCM_BLIND_CHECK_MSG); + time = rte_get_tsc_cycles() + hz; + if(rte_meter_trtcm_color_blind_check( + &tm, time, TM_TEST_TRTCM_CBS_DF + 1) + != e_RTE_METER_YELLOW) + melog(TRTCM_BLIND_CHECK_MSG" YELLOW"); + + if(rte_meter_trtcm_config(&tm, &tparams) != 0) + melog(TRTCM_BLIND_CHECK_MSG); + time = rte_get_tsc_cycles() + hz; + if(rte_meter_trtcm_color_blind_check( + &tm, time, TM_TEST_TRTCM_PBS_DF - 1) + != e_RTE_METER_YELLOW) + melog(TRTCM_BLIND_CHECK_MSG" YELLOW"); + + /* Test red */ + if(rte_meter_trtcm_config(&tm, &tparams) != 0) + melog(TRTCM_BLIND_CHECK_MSG); + time = rte_get_tsc_cycles() + hz; + if(rte_meter_trtcm_color_blind_check( + &tm, time, TM_TEST_TRTCM_PBS_DF + 1) + != e_RTE_METER_RED) + melog(TRTCM_BLIND_CHECK_MSG" RED"); + + return 0; +} + + +/** + * @in[4] : the flags packets carries. + * @in[4] : the flags function expect to return. + * It will do blind check at the time of 1 second from beginning. + * At the time, it will use packets length of cbs -1, cbs + 1, + * ebs -1 and ebs +1 with flag in[0], in[1], in[2] and in[3] to do + * aware check, expect flag out[0], out[1], out[2] and out[3] + */ + +static inline int +tm_test_srtcm_aware_check +(enum rte_meter_color in[4], enum rte_meter_color out[4]) +{ +#define SRTCM_AWARE_CHECK_MSG "srtcm_aware_check" + struct rte_meter_srtcm sm; + uint64_t time; + uint64_t hz = rte_get_tsc_hz(); + + if(rte_meter_srtcm_config(&sm, &sparams) != 0) + melog(SRTCM_AWARE_CHECK_MSG); + time = rte_get_tsc_cycles() + hz; + if(rte_meter_srtcm_color_aware_check( + &sm, time, TM_TEST_SRTCM_CBS_DF - 1, in[0]) != out[0]) + melog(SRTCM_AWARE_CHECK_MSG" %u:%u", in[0], out[0]); + + if(rte_meter_srtcm_config(&sm, &sparams) != 0) + melog(SRTCM_AWARE_CHECK_MSG); + time = rte_get_tsc_cycles() + hz; + if(rte_meter_srtcm_color_aware_check( + &sm, time, TM_TEST_SRTCM_CBS_DF + 1, in[1]) != out[1]) + melog(SRTCM_AWARE_CHECK_MSG" %u:%u", in[1], out[1]); + + if(rte_meter_srtcm_config(&sm, &sparams) != 0) + melog(SRTCM_AWARE_CHECK_MSG); + time = rte_get_tsc_cycles() + hz; + if(rte_meter_srtcm_color_aware_check( + &sm, time, TM_TEST_SRTCM_EBS_DF - 1, in[2]) != out[2]) + melog(SRTCM_AWARE_CHECK_MSG" %u:%u", in[2], out[2]); + + if(rte_meter_srtcm_config(&sm, &sparams) != 0) + melog(SRTCM_AWARE_CHECK_MSG); + time = rte_get_tsc_cycles() + hz; + if(rte_meter_srtcm_color_aware_check( + &sm, time, TM_TEST_SRTCM_EBS_DF + 1, in[3]) != out[3]) + melog(SRTCM_AWARE_CHECK_MSG" %u:%u", in[3], out[3]); + + return 0; +} + + +/** + * functional test for rte_meter_srtcm_color_aware_check + */ +static inline int +tm_test_srtcm_color_aware_check(void) +{ + enum rte_meter_color in[4], out[4]; + + /** + * test 4 points that will produce green, yellow, yellow, red flag + * if using blind check + */ + + /* previouly have a green, test points should keep unchanged */ + in[0] = in[1] = in[2] = in[3] = e_RTE_METER_GREEN; + out[0] = e_RTE_METER_GREEN; + out[1] = e_RTE_METER_YELLOW; + out[2] = e_RTE_METER_YELLOW; + out[3] = e_RTE_METER_RED; + if(tm_test_srtcm_aware_check(in, out) != 0) + return -1; + + /** + * previously have a yellow, green & yellow = yellow + * yellow & red = red + */ + in[0] = in[1] = in[2] = in[3] = e_RTE_METER_YELLOW; + out[0] = e_RTE_METER_YELLOW; + out[1] = e_RTE_METER_YELLOW; + out[2] = e_RTE_METER_YELLOW; + out[3] = e_RTE_METER_RED; + if(tm_test_srtcm_aware_check(in, out) != 0) + return -1; + + /** + * previously have a red, red & green = red + * red & yellow = red + */ + in[0] = in[1] = in[2] = in[3] = e_RTE_METER_RED; + out[0] = e_RTE_METER_RED; + out[1] = e_RTE_METER_RED; + out[2] = e_RTE_METER_RED; + out[3] = e_RTE_METER_RED; + if(tm_test_srtcm_aware_check(in, out) != 0) + return -1; + + return 0; +} + +/** + * @in[4] : the flags packets carries. + * @in[4] : the flags function expect to return. + * It will do blind check at the time of 1 second from beginning. + * At the time, it will use packets length of cbs -1, cbs + 1, + * ebs -1 and ebs +1 with flag in[0], in[1], in[2] and in[3] to do + * aware check, expect flag out[0], out[1], out[2] and out[3] + */ +static inline int +tm_test_trtcm_aware_check +(enum rte_meter_color in[4], enum rte_meter_color out[4]) +{ +#define TRTCM_AWARE_CHECK_MSG "trtcm_aware_check" + struct rte_meter_trtcm tm; + uint64_t time; + uint64_t hz = rte_get_tsc_hz(); + + if(rte_meter_trtcm_config(&tm, &tparams) != 0) + melog(TRTCM_AWARE_CHECK_MSG); + time = rte_get_tsc_cycles() + hz; + if(rte_meter_trtcm_color_aware_check( + &tm, time, TM_TEST_TRTCM_CBS_DF - 1, in[0]) != out[0]) + melog(TRTCM_AWARE_CHECK_MSG" %u:%u", in[0], out[0]); + + if(rte_meter_trtcm_config(&tm, &tparams) != 0) + melog(TRTCM_AWARE_CHECK_MSG); + time = rte_get_tsc_cycles() + hz; + if(rte_meter_trtcm_color_aware_check( + &tm, time, TM_TEST_TRTCM_CBS_DF + 1, in[1]) != out[1]) + melog(TRTCM_AWARE_CHECK_MSG" %u:%u", in[1], out[1]); + + if(rte_meter_trtcm_config(&tm, &tparams) != 0) + melog(TRTCM_AWARE_CHECK_MSG); + time = rte_get_tsc_cycles() + hz; + if(rte_meter_trtcm_color_aware_check( + &tm, time, TM_TEST_TRTCM_PBS_DF - 1, in[2]) != out[2]) + melog(TRTCM_AWARE_CHECK_MSG" %u:%u", in[2], out[2]); + + if(rte_meter_trtcm_config(&tm, &tparams) != 0) + melog(TRTCM_AWARE_CHECK_MSG); + time = rte_get_tsc_cycles() + hz; + if(rte_meter_trtcm_color_aware_check( + &tm, time, TM_TEST_TRTCM_PBS_DF + 1, in[3]) != out[3]) + melog(TRTCM_AWARE_CHECK_MSG" %u:%u", in[3], out[3]); + + return 0; +} + + +/** + * functional test for rte_meter_trtcm_color_aware_check + */ + +static inline int +tm_test_trtcm_color_aware_check(void) +{ + enum rte_meter_color in[4], out[4]; + /** + * test 4 points that will produce green, yellow, yellow, red flag + * if using blind check + */ + + /* previouly have a green, test points should keep unchanged */ + in[0] = in[1] = in[2] = in[3] = e_RTE_METER_GREEN; + out[0] = e_RTE_METER_GREEN; + out[1] = e_RTE_METER_YELLOW; + out[2] = e_RTE_METER_YELLOW; + out[3] = e_RTE_METER_RED; + if(tm_test_trtcm_aware_check(in, out) != 0) + return -1; + + in[0] = in[1] = in[2] = in[3] = e_RTE_METER_YELLOW; + out[0] = e_RTE_METER_YELLOW; + out[1] = e_RTE_METER_YELLOW; + out[2] = e_RTE_METER_YELLOW; + out[3] = e_RTE_METER_RED; + if(tm_test_trtcm_aware_check(in, out) != 0) + return -1; + + in[0] = in[1] = in[2] = in[3] = e_RTE_METER_RED; + out[0] = e_RTE_METER_RED; + out[1] = e_RTE_METER_RED; + out[2] = e_RTE_METER_RED; + out[3] = e_RTE_METER_RED; + if(tm_test_trtcm_aware_check(in, out) != 0) + return -1; + + return 0; +} + +/** + * test main entrance for library meter + */ +static int +test_meter(void) +{ + if(tm_test_srtcm_config() != 0 ) + return -1; + + if(tm_test_trtcm_config() != 0 ) + return -1; + + if(tm_test_srtcm_color_blind_check() != 0) + return -1; + + if(tm_test_trtcm_color_blind_check()!= 0) + return -1; + + if(tm_test_srtcm_color_aware_check()!= 0) + return -1; + + if(tm_test_trtcm_color_aware_check()!= 0) + return -1; + + return 0; + +} + +REGISTER_TEST_COMMAND(meter_autotest, test_meter); diff --git a/dpdk/test/test/test_mp_secondary.c b/dpdk/test/test/test_mp_secondary.c new file mode 100644 index 00000000..0b93a2de --- /dev/null +++ b/dpdk/test/test/test_mp_secondary.c @@ -0,0 +1,264 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> + +#include "test.h" + +#include <stdint.h> +#include <stdlib.h> +#include <stdarg.h> +#include <inttypes.h> +#include <sys/queue.h> +#include <errno.h> +#include <string.h> +#include <unistd.h> +#include <sys/wait.h> +#include <libgen.h> +#include <dirent.h> +#include <limits.h> + +#include <rte_common.h> +#include <rte_memory.h> +#include <rte_memzone.h> +#include <rte_eal.h> +#include <rte_launch.h> +#include <rte_per_lcore.h> +#include <rte_lcore.h> +#include <rte_errno.h> +#include <rte_branch_prediction.h> +#include <rte_atomic.h> +#include <rte_ring.h> +#include <rte_debug.h> +#include <rte_log.h> +#include <rte_mempool.h> + +#ifdef RTE_LIBRTE_HASH +#include <rte_hash.h> +#include <rte_fbk_hash.h> +#endif /* RTE_LIBRTE_HASH */ + +#ifdef RTE_LIBRTE_LPM +#include <rte_lpm.h> +#endif /* RTE_LIBRTE_LPM */ + +#include <rte_string_fns.h> + +#include "process.h" + +#define launch_proc(ARGV) process_dup(ARGV, \ + sizeof(ARGV)/(sizeof(ARGV[0])), __func__) + +#ifdef RTE_EXEC_ENV_LINUXAPP +static char* +get_current_prefix(char * prefix, int size) +{ + char path[PATH_MAX] = {0}; + char buf[PATH_MAX] = {0}; + + /* get file for config (fd is always 3) */ + snprintf(path, sizeof(path), "/proc/self/fd/%d", 3); + + /* return NULL on error */ + if (readlink(path, buf, sizeof(buf)) == -1) + return NULL; + + /* get the basename */ + snprintf(buf, sizeof(buf), "%s", basename(buf)); + + /* copy string all the way from second char up to start of _config */ + snprintf(prefix, size, "%.*s", + (int)(strnlen(buf, sizeof(buf)) - sizeof("_config")), + &buf[1]); + + return prefix; +} +#endif + +/* + * This function is called in the primary i.e. main test, to spawn off secondary + * processes to run actual mp tests. Uses fork() and exec pair + */ +static int +run_secondary_instances(void) +{ + int ret = 0; + char coremask[10]; + +#ifdef RTE_EXEC_ENV_LINUXAPP + char tmp[PATH_MAX] = {0}; + char prefix[PATH_MAX] = {0}; + + get_current_prefix(tmp, sizeof(tmp)); + + snprintf(prefix, sizeof(prefix), "--file-prefix=%s", tmp); +#else + const char *prefix = ""; +#endif + + /* good case, using secondary */ + const char *argv1[] = { + prgname, "-c", coremask, "--proc-type=secondary", + prefix + }; + /* good case, using auto */ + const char *argv2[] = { + prgname, "-c", coremask, "--proc-type=auto", + prefix + }; + /* bad case, using invalid type */ + const char *argv3[] = { + prgname, "-c", coremask, "--proc-type=ERROR", + prefix + }; +#ifdef RTE_EXEC_ENV_LINUXAPP + /* bad case, using invalid file prefix */ + const char *argv4[] = { + prgname, "-c", coremask, "--proc-type=secondary", + "--file-prefix=ERROR" + }; +#endif + + snprintf(coremask, sizeof(coremask), "%x", \ + (1 << rte_get_master_lcore())); + + ret |= launch_proc(argv1); + ret |= launch_proc(argv2); + + ret |= !(launch_proc(argv3)); +#ifdef RTE_EXEC_ENV_LINUXAPP + ret |= !(launch_proc(argv4)); +#endif + + return ret; +} + +/* + * This function is run in the secondary instance to test that creation of + * objects fails in a secondary + */ +static int +run_object_creation_tests(void) +{ + const unsigned flags = 0; + const unsigned size = 1024; + const unsigned elt_size = 64; + const unsigned cache_size = 64; + const unsigned priv_data_size = 32; + + printf("### Testing object creation - expect lots of mz reserve errors!\n"); + + rte_errno = 0; + if ((rte_memzone_reserve("test_mz", size, rte_socket_id(), + flags) == NULL) && + (rte_memzone_lookup("test_mz") == NULL)) { + printf("Error: unexpected return value from rte_memzone_reserve\n"); + return -1; + } + printf("# Checked rte_memzone_reserve() OK\n"); + + rte_errno = 0; + if ((rte_ring_create( + "test_ring", size, rte_socket_id(), flags) == NULL) && + (rte_ring_lookup("test_ring") == NULL)){ + printf("Error: unexpected return value from rte_ring_create()\n"); + return -1; + } + printf("# Checked rte_ring_create() OK\n"); + + rte_errno = 0; + if ((rte_mempool_create("test_mp", size, elt_size, cache_size, + priv_data_size, NULL, NULL, NULL, NULL, + rte_socket_id(), flags) == NULL) && + (rte_mempool_lookup("test_mp") == NULL)){ + printf("Error: unexpected return value from rte_mempool_create()\n"); + return -1; + } + printf("# Checked rte_mempool_create() OK\n"); + +#ifdef RTE_LIBRTE_HASH + const struct rte_hash_parameters hash_params = { .name = "test_mp_hash" }; + rte_errno=0; + if ((rte_hash_create(&hash_params) != NULL) && + (rte_hash_find_existing(hash_params.name) == NULL)){ + printf("Error: unexpected return value from rte_hash_create()\n"); + return -1; + } + printf("# Checked rte_hash_create() OK\n"); + + const struct rte_fbk_hash_params fbk_params = { .name = "test_fbk_mp_hash" }; + rte_errno=0; + if ((rte_fbk_hash_create(&fbk_params) != NULL) && + (rte_fbk_hash_find_existing(fbk_params.name) == NULL)){ + printf("Error: unexpected return value from rte_fbk_hash_create()\n"); + return -1; + } + printf("# Checked rte_fbk_hash_create() OK\n"); +#endif + +#ifdef RTE_LIBRTE_LPM + rte_errno=0; + struct rte_lpm_config config; + + config.max_rules = rte_socket_id(); + config.number_tbl8s = 256; + config.flags = 0; + if ((rte_lpm_create("test_lpm", size, &config) != NULL) && + (rte_lpm_find_existing("test_lpm") == NULL)){ + printf("Error: unexpected return value from rte_lpm_create()\n"); + return -1; + } + printf("# Checked rte_lpm_create() OK\n"); +#endif + + return 0; +} + +/* if called in a primary process, just spawns off a secondary process to + * run validation tests - which brings us right back here again... + * if called in a secondary process, this runs a series of API tests to check + * how things run in a secondary instance. + */ +int +test_mp_secondary(void) +{ + if (rte_eal_process_type() == RTE_PROC_PRIMARY) { + return run_secondary_instances(); + } + + printf("IN SECONDARY PROCESS\n"); + + return run_object_creation_tests(); +} + +REGISTER_TEST_COMMAND(multiprocess_autotest, test_mp_secondary); diff --git a/dpdk/test/test/test_per_lcore.c b/dpdk/test/test/test_per_lcore.c new file mode 100644 index 00000000..4eb7fe2c --- /dev/null +++ b/dpdk/test/test/test_per_lcore.c @@ -0,0 +1,137 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <stdint.h> +#include <sys/queue.h> + +#include <rte_common.h> +#include <rte_memory.h> +#include <rte_per_lcore.h> +#include <rte_launch.h> +#include <rte_eal.h> +#include <rte_lcore.h> +#include <rte_cycles.h> + +#include "test.h" + +/* + * Per-lcore variables and lcore launch + * ==================================== + * + * - Use ``rte_eal_mp_remote_launch()`` to call ``assign_vars()`` on + * every available lcore. In this function, a per-lcore variable is + * assigned to the lcore_id. + * + * - Use ``rte_eal_mp_remote_launch()`` to call ``display_vars()`` on + * every available lcore. The function checks that the variable is + * correctly set, or returns -1. + * + * - If at least one per-core variable was not correct, the test function + * returns -1. + */ + +static RTE_DEFINE_PER_LCORE(unsigned, test) = 0x12345678; + +static int +assign_vars(__attribute__((unused)) void *arg) +{ + if (RTE_PER_LCORE(test) != 0x12345678) + return -1; + RTE_PER_LCORE(test) = rte_lcore_id(); + return 0; +} + +static int +display_vars(__attribute__((unused)) void *arg) +{ + unsigned lcore_id = rte_lcore_id(); + unsigned var = RTE_PER_LCORE(test); + unsigned socket_id = rte_lcore_to_socket_id(lcore_id); + + printf("on socket %u, on core %u, variable is %u\n", socket_id, lcore_id, var); + if (lcore_id != var) + return -1; + + RTE_PER_LCORE(test) = 0x12345678; + return 0; +} + +static int +test_per_lcore_delay(__attribute__((unused)) void *arg) +{ + rte_delay_ms(100); + printf("wait 100ms on lcore %u\n", rte_lcore_id()); + + return 0; +} + +static int +test_per_lcore(void) +{ + unsigned lcore_id; + int ret; + + rte_eal_mp_remote_launch(assign_vars, NULL, SKIP_MASTER); + RTE_LCORE_FOREACH_SLAVE(lcore_id) { + if (rte_eal_wait_lcore(lcore_id) < 0) + return -1; + } + + rte_eal_mp_remote_launch(display_vars, NULL, SKIP_MASTER); + RTE_LCORE_FOREACH_SLAVE(lcore_id) { + if (rte_eal_wait_lcore(lcore_id) < 0) + return -1; + } + + /* test if it could do remote launch twice at the same time or not */ + ret = rte_eal_mp_remote_launch(test_per_lcore_delay, NULL, SKIP_MASTER); + if (ret < 0) { + printf("It fails to do remote launch but it should able to do\n"); + return -1; + } + /* it should not be able to launch a lcore which is running */ + ret = rte_eal_mp_remote_launch(test_per_lcore_delay, NULL, SKIP_MASTER); + if (ret == 0) { + printf("It does remote launch successfully but it should not at this time\n"); + return -1; + } + RTE_LCORE_FOREACH_SLAVE(lcore_id) { + if (rte_eal_wait_lcore(lcore_id) < 0) + return -1; + } + + return 0; +} + +REGISTER_TEST_COMMAND(per_lcore_autotest, test_per_lcore); diff --git a/dpdk/test/test/test_pmd_perf.c b/dpdk/test/test/test_pmd_perf.c new file mode 100644 index 00000000..255f2607 --- /dev/null +++ b/dpdk/test/test/test_pmd_perf.c @@ -0,0 +1,916 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + + +#include <stdio.h> +#include <inttypes.h> +#include <signal.h> +#include <unistd.h> +#include <rte_cycles.h> +#include <rte_ethdev.h> +#include <rte_byteorder.h> +#include <rte_atomic.h> +#include <rte_malloc.h> +#include "packet_burst_generator.h" +#include "test.h" + +#define NB_ETHPORTS_USED (1) +#define NB_SOCKETS (2) +#define MEMPOOL_CACHE_SIZE 250 +#define MAX_PKT_BURST (32) +#define RTE_TEST_RX_DESC_DEFAULT (128) +#define RTE_TEST_TX_DESC_DEFAULT (512) +#define RTE_PORT_ALL (~(uint16_t)0x0) + +/* how long test would take at full line rate */ +#define RTE_TEST_DURATION (2) + +/* + * RX and TX Prefetch, Host, and Write-back threshold values should be + * carefully set for optimal performance. Consult the network + * controller's datasheet and supporting DPDK documentation for guidance + * on how these parameters should be set. + */ +#define RX_PTHRESH 8 /**< Default values of RX prefetch threshold reg. */ +#define RX_HTHRESH 8 /**< Default values of RX host threshold reg. */ +#define RX_WTHRESH 0 /**< Default values of RX write-back threshold reg. */ + +/* + * These default values are optimized for use with the Intel(R) 82599 10 GbE + * Controller and the DPDK ixgbe PMD. Consider using other values for other + * network controllers and/or network drivers. + */ +#define TX_PTHRESH 32 /**< Default values of TX prefetch threshold reg. */ +#define TX_HTHRESH 0 /**< Default values of TX host threshold reg. */ +#define TX_WTHRESH 0 /**< Default values of TX write-back threshold reg. */ + +#define MAX_TRAFFIC_BURST 2048 + +#define NB_MBUF RTE_MAX( \ + (unsigned)(nb_ports*nb_rx_queue*nb_rxd + \ + nb_ports*nb_lcores*MAX_PKT_BURST + \ + nb_ports*nb_tx_queue*nb_txd + \ + nb_lcores*MEMPOOL_CACHE_SIZE + \ + nb_ports*MAX_TRAFFIC_BURST), \ + (unsigned)8192) + + +static struct rte_mempool *mbufpool[NB_SOCKETS]; +/* ethernet addresses of ports */ +static struct ether_addr ports_eth_addr[RTE_MAX_ETHPORTS]; + +static struct rte_eth_conf port_conf = { + .rxmode = { + .mq_mode = ETH_MQ_RX_NONE, + .max_rx_pkt_len = ETHER_MAX_LEN, + .split_hdr_size = 0, + .header_split = 0, /**< Header Split disabled */ + .hw_ip_checksum = 0, /**< IP checksum offload enabled */ + .hw_vlan_filter = 0, /**< VLAN filtering disabled */ + .hw_vlan_strip = 0, /**< VLAN strip enabled. */ + .hw_vlan_extend = 0, /**< Extended VLAN disabled. */ + .jumbo_frame = 0, /**< Jumbo Frame Support disabled */ + .hw_strip_crc = 1, /**< CRC stripped by hardware */ + .enable_scatter = 0, /**< scatter rx disabled */ + }, + .txmode = { + .mq_mode = ETH_MQ_TX_NONE, + }, + .lpbk_mode = 1, /* enable loopback */ +}; + +static struct rte_eth_rxconf rx_conf = { + .rx_thresh = { + .pthresh = RX_PTHRESH, + .hthresh = RX_HTHRESH, + .wthresh = RX_WTHRESH, + }, + .rx_free_thresh = 32, +}; + +static struct rte_eth_txconf tx_conf = { + .tx_thresh = { + .pthresh = TX_PTHRESH, + .hthresh = TX_HTHRESH, + .wthresh = TX_WTHRESH, + }, + .tx_free_thresh = 32, /* Use PMD default values */ + .tx_rs_thresh = 32, /* Use PMD default values */ + .txq_flags = (ETH_TXQ_FLAGS_NOMULTSEGS | + ETH_TXQ_FLAGS_NOVLANOFFL | + ETH_TXQ_FLAGS_NOXSUMSCTP | + ETH_TXQ_FLAGS_NOXSUMUDP | + ETH_TXQ_FLAGS_NOXSUMTCP) +}; + +enum { + LCORE_INVALID = 0, + LCORE_AVAIL, + LCORE_USED, +}; + +struct lcore_conf { + uint8_t status; + uint8_t socketid; + uint16_t nb_ports; + uint16_t portlist[RTE_MAX_ETHPORTS]; +} __rte_cache_aligned; + +struct lcore_conf lcore_conf[RTE_MAX_LCORE]; + +static uint64_t link_mbps; + +enum { + SC_CONTINUOUS = 0, + SC_BURST_POLL_FIRST, + SC_BURST_XMIT_FIRST, +}; + +static uint32_t sc_flag; + +/* Check the link status of all ports in up to 3s, and print them finally */ +static void +check_all_ports_link_status(uint16_t port_num, uint32_t port_mask) +{ +#define CHECK_INTERVAL 100 /* 100ms */ +#define MAX_CHECK_TIME 30 /* 3s (30 * 100ms) in total */ + uint16_t portid; + uint8_t count, all_ports_up, print_flag = 0; + struct rte_eth_link link; + + printf("Checking link statuses...\n"); + fflush(stdout); + for (count = 0; count <= MAX_CHECK_TIME; count++) { + all_ports_up = 1; + for (portid = 0; portid < port_num; portid++) { + if ((port_mask & (1 << portid)) == 0) + continue; + memset(&link, 0, sizeof(link)); + rte_eth_link_get_nowait(portid, &link); + /* print link status if flag set */ + if (print_flag == 1) { + if (link.link_status) { + printf( + "Port%d Link Up. Speed %u Mbps - %s\n", + portid, link.link_speed, + (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? + ("full-duplex") : ("half-duplex\n")); + if (link_mbps == 0) + link_mbps = link.link_speed; + } else + printf("Port %d Link Down\n", portid); + continue; + } + /* clear all_ports_up flag if any link down */ + if (link.link_status == ETH_LINK_DOWN) { + all_ports_up = 0; + break; + } + } + /* after finally printing all link status, get out */ + if (print_flag == 1) + break; + + if (all_ports_up == 0) { + fflush(stdout); + rte_delay_ms(CHECK_INTERVAL); + } + + /* set the print_flag if all ports up or timeout */ + if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) + print_flag = 1; + } +} + +static void +print_ethaddr(const char *name, const struct ether_addr *eth_addr) +{ + char buf[ETHER_ADDR_FMT_SIZE]; + ether_format_addr(buf, ETHER_ADDR_FMT_SIZE, eth_addr); + printf("%s%s", name, buf); +} + +static int +init_traffic(struct rte_mempool *mp, + struct rte_mbuf **pkts_burst, uint32_t burst_size) +{ + struct ether_hdr pkt_eth_hdr; + struct ipv4_hdr pkt_ipv4_hdr; + struct udp_hdr pkt_udp_hdr; + uint32_t pktlen; + static uint8_t src_mac[] = { 0x00, 0xFF, 0xAA, 0xFF, 0xAA, 0xFF }; + static uint8_t dst_mac[] = { 0x00, 0xAA, 0xFF, 0xAA, 0xFF, 0xAA }; + + + initialize_eth_header(&pkt_eth_hdr, + (struct ether_addr *)src_mac, + (struct ether_addr *)dst_mac, ETHER_TYPE_IPv4, 0, 0); + + pktlen = initialize_ipv4_header(&pkt_ipv4_hdr, + IPV4_ADDR(10, 0, 0, 1), + IPV4_ADDR(10, 0, 0, 2), 26); + printf("IPv4 pktlen %u\n", pktlen); + + pktlen = initialize_udp_header(&pkt_udp_hdr, 0, 0, 18); + + printf("UDP pktlen %u\n", pktlen); + + return generate_packet_burst(mp, pkts_burst, &pkt_eth_hdr, + 0, &pkt_ipv4_hdr, 1, + &pkt_udp_hdr, burst_size, + PACKET_BURST_GEN_PKT_LEN, 1); +} + +static int +init_lcores(void) +{ + unsigned lcore_id; + + for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { + lcore_conf[lcore_id].socketid = + rte_lcore_to_socket_id(lcore_id); + if (rte_lcore_is_enabled(lcore_id) == 0) { + lcore_conf[lcore_id].status = LCORE_INVALID; + continue; + } else + lcore_conf[lcore_id].status = LCORE_AVAIL; + } + return 0; +} + +static int +init_mbufpool(unsigned nb_mbuf) +{ + int socketid; + unsigned lcore_id; + char s[64]; + + for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { + if (rte_lcore_is_enabled(lcore_id) == 0) + continue; + + socketid = rte_lcore_to_socket_id(lcore_id); + if (socketid >= NB_SOCKETS) { + rte_exit(EXIT_FAILURE, + "Socket %d of lcore %u is out of range %d\n", + socketid, lcore_id, NB_SOCKETS); + } + if (mbufpool[socketid] == NULL) { + snprintf(s, sizeof(s), "mbuf_pool_%d", socketid); + mbufpool[socketid] = + rte_pktmbuf_pool_create(s, nb_mbuf, + MEMPOOL_CACHE_SIZE, 0, + RTE_MBUF_DEFAULT_BUF_SIZE, socketid); + if (mbufpool[socketid] == NULL) + rte_exit(EXIT_FAILURE, + "Cannot init mbuf pool on socket %d\n", + socketid); + else + printf("Allocated mbuf pool on socket %d\n", + socketid); + } + } + return 0; +} + +static uint16_t +alloc_lcore(uint16_t socketid) +{ + unsigned lcore_id; + + for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { + if (LCORE_AVAIL != lcore_conf[lcore_id].status || + lcore_conf[lcore_id].socketid != socketid || + lcore_id == rte_get_master_lcore()) + continue; + lcore_conf[lcore_id].status = LCORE_USED; + lcore_conf[lcore_id].nb_ports = 0; + return lcore_id; + } + + return (uint16_t)-1; +} + +volatile uint64_t stop; +uint64_t count; +uint64_t drop; +uint64_t idle; + +static void +reset_count(void) +{ + count = 0; + drop = 0; + idle = 0; +} + +static void +stats_display(uint16_t port_id) +{ + struct rte_eth_stats stats; + rte_eth_stats_get(port_id, &stats); + + printf(" RX-packets: %-10"PRIu64" RX-missed: %-10"PRIu64" RX-bytes: " + "%-"PRIu64"\n", + stats.ipackets, stats.imissed, stats.ibytes); + printf(" RX-errors: %-10"PRIu64" RX-nombuf: %-10"PRIu64"\n", + stats.ierrors, stats.rx_nombuf); + printf(" TX-packets: %-10"PRIu64" TX-errors: %-10"PRIu64" TX-bytes: " + "%-"PRIu64"\n", + stats.opackets, stats.oerrors, stats.obytes); +} + +static void +signal_handler(int signum) +{ + /* USR1 signal, stop testing */ + if (signum == SIGUSR1) { + printf("Force Stop!\n"); + stop = 1; + } + + /* USR2 signal, print stats */ + if (signum == SIGUSR2) + stats_display(0); +} + +struct rte_mbuf **tx_burst; + +uint64_t (*do_measure)(struct lcore_conf *conf, + struct rte_mbuf *pkts_burst[], + uint64_t total_pkts); + +static uint64_t +measure_rxtx(struct lcore_conf *conf, + struct rte_mbuf *pkts_burst[], + uint64_t total_pkts) +{ + unsigned i, portid, nb_rx, nb_tx; + uint64_t prev_tsc, cur_tsc; + + prev_tsc = rte_rdtsc(); + + while (likely(!stop)) { + for (i = 0; i < conf->nb_ports; i++) { + portid = conf->portlist[i]; + nb_rx = rte_eth_rx_burst(portid, 0, + pkts_burst, MAX_PKT_BURST); + if (unlikely(nb_rx == 0)) { + idle++; + continue; + } + + count += nb_rx; + nb_tx = rte_eth_tx_burst(portid, 0, pkts_burst, nb_rx); + if (unlikely(nb_tx < nb_rx)) { + drop += (nb_rx - nb_tx); + do { + rte_pktmbuf_free(pkts_burst[nb_tx]); + } while (++nb_tx < nb_rx); + } + } + if (unlikely(count >= total_pkts)) + break; + } + + cur_tsc = rte_rdtsc(); + + return cur_tsc - prev_tsc; +} + +static uint64_t +measure_rxonly(struct lcore_conf *conf, + struct rte_mbuf *pkts_burst[], + uint64_t total_pkts) +{ + unsigned i, portid, nb_rx, nb_tx; + uint64_t diff_tsc, cur_tsc; + + diff_tsc = 0; + while (likely(!stop)) { + for (i = 0; i < conf->nb_ports; i++) { + portid = conf->portlist[i]; + + cur_tsc = rte_rdtsc(); + nb_rx = rte_eth_rx_burst(portid, 0, + pkts_burst, MAX_PKT_BURST); + if (unlikely(nb_rx == 0)) { + idle++; + continue; + } + diff_tsc += rte_rdtsc() - cur_tsc; + + count += nb_rx; + nb_tx = rte_eth_tx_burst(portid, 0, pkts_burst, nb_rx); + if (unlikely(nb_tx < nb_rx)) { + drop += (nb_rx - nb_tx); + do { + rte_pktmbuf_free(pkts_burst[nb_tx]); + } while (++nb_tx < nb_rx); + } + } + if (unlikely(count >= total_pkts)) + break; + } + + return diff_tsc; +} + +static uint64_t +measure_txonly(struct lcore_conf *conf, + struct rte_mbuf *pkts_burst[], + uint64_t total_pkts) +{ + unsigned i, portid, nb_rx, nb_tx; + uint64_t diff_tsc, cur_tsc; + + printf("do tx measure\n"); + diff_tsc = 0; + while (likely(!stop)) { + for (i = 0; i < conf->nb_ports; i++) { + portid = conf->portlist[i]; + nb_rx = rte_eth_rx_burst(portid, 0, + pkts_burst, MAX_PKT_BURST); + if (unlikely(nb_rx == 0)) { + idle++; + continue; + } + + count += nb_rx; + + cur_tsc = rte_rdtsc(); + nb_tx = rte_eth_tx_burst(portid, 0, pkts_burst, nb_rx); + if (unlikely(nb_tx < nb_rx)) { + drop += (nb_rx - nb_tx); + do { + rte_pktmbuf_free(pkts_burst[nb_tx]); + } while (++nb_tx < nb_rx); + } + diff_tsc += rte_rdtsc() - cur_tsc; + } + if (unlikely(count >= total_pkts)) + break; + } + + return diff_tsc; +} + +/* main processing loop */ +static int +main_loop(__rte_unused void *args) +{ +#define PACKET_SIZE 64 +#define FRAME_GAP 12 +#define MAC_PREAMBLE 8 + struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; + unsigned lcore_id; + unsigned i, portid, nb_rx = 0, nb_tx = 0; + struct lcore_conf *conf; + int pkt_per_port; + uint64_t diff_tsc; + uint64_t packets_per_second, total_packets; + + lcore_id = rte_lcore_id(); + conf = &lcore_conf[lcore_id]; + if (conf->status != LCORE_USED) + return 0; + + pkt_per_port = MAX_TRAFFIC_BURST; + + int idx = 0; + for (i = 0; i < conf->nb_ports; i++) { + int num = pkt_per_port; + portid = conf->portlist[i]; + printf("inject %d packet to port %d\n", num, portid); + while (num) { + nb_tx = RTE_MIN(MAX_PKT_BURST, num); + nb_tx = rte_eth_tx_burst(portid, 0, + &tx_burst[idx], nb_tx); + num -= nb_tx; + idx += nb_tx; + } + } + printf("Total packets inject to prime ports = %u\n", idx); + + packets_per_second = (link_mbps * 1000 * 1000) / + ((PACKET_SIZE + FRAME_GAP + MAC_PREAMBLE) * CHAR_BIT); + printf("Each port will do %"PRIu64" packets per second\n", + packets_per_second); + + total_packets = RTE_TEST_DURATION * conf->nb_ports * packets_per_second; + printf("Test will stop after at least %"PRIu64" packets received\n", + + total_packets); + + diff_tsc = do_measure(conf, pkts_burst, total_packets); + + for (i = 0; i < conf->nb_ports; i++) { + portid = conf->portlist[i]; + int nb_free = pkt_per_port; + do { /* dry out */ + nb_rx = rte_eth_rx_burst(portid, 0, + pkts_burst, MAX_PKT_BURST); + nb_tx = 0; + while (nb_tx < nb_rx) + rte_pktmbuf_free(pkts_burst[nb_tx++]); + nb_free -= nb_rx; + } while (nb_free != 0); + printf("free %d mbuf left in port %u\n", pkt_per_port, portid); + } + + if (count == 0) + return -1; + + printf("%"PRIu64" packet, %"PRIu64" drop, %"PRIu64" idle\n", + count, drop, idle); + printf("Result: %"PRIu64" cycles per packet\n", diff_tsc / count); + + return 0; +} + +rte_atomic64_t start; + +static inline int +poll_burst(void *args) +{ +#define MAX_IDLE (10000) + unsigned lcore_id; + struct rte_mbuf **pkts_burst; + uint64_t diff_tsc, cur_tsc; + uint16_t next[RTE_MAX_ETHPORTS]; + struct lcore_conf *conf; + uint32_t pkt_per_port = *((uint32_t *)args); + unsigned i, portid, nb_rx = 0; + uint64_t total; + uint64_t timeout = MAX_IDLE; + int num[RTE_MAX_ETHPORTS]; + + lcore_id = rte_lcore_id(); + conf = &lcore_conf[lcore_id]; + if (conf->status != LCORE_USED) + return 0; + + total = pkt_per_port * conf->nb_ports; + printf("start to receive total expect %"PRIu64"\n", total); + + pkts_burst = (struct rte_mbuf **) + rte_calloc_socket("poll_burst", + total, sizeof(void *), + RTE_CACHE_LINE_SIZE, conf->socketid); + if (!pkts_burst) + return -1; + + for (i = 0; i < conf->nb_ports; i++) { + portid = conf->portlist[i]; + next[portid] = i * pkt_per_port; + num[portid] = pkt_per_port; + } + + while (!rte_atomic64_read(&start)) + ; + + cur_tsc = rte_rdtsc(); + while (total) { + for (i = 0; i < conf->nb_ports; i++) { + portid = conf->portlist[i]; + nb_rx = rte_eth_rx_burst(portid, 0, + &pkts_burst[next[portid]], + RTE_MIN(MAX_PKT_BURST, num[portid])); + if (unlikely(nb_rx == 0)) { + timeout--; + if (unlikely(timeout == 0)) + goto timeout; + continue; + } + next[portid] += nb_rx; + num[portid] -= nb_rx; + total -= nb_rx; + } + } +timeout: + diff_tsc = rte_rdtsc() - cur_tsc; + + printf("%"PRIu64" packets lost, IDLE %"PRIu64" times\n", + total, MAX_IDLE - timeout); + /* clean up */ + total = pkt_per_port * conf->nb_ports - total; + for (i = 0; i < total; i++) + rte_pktmbuf_free(pkts_burst[i]); + + rte_free(pkts_burst); + + if (total > 0) + return diff_tsc / total; + else + return -1; +} + +static int +exec_burst(uint32_t flags, int lcore) +{ + unsigned i, portid, nb_tx = 0; + struct lcore_conf *conf; + uint32_t pkt_per_port; + int num, idx = 0; + int diff_tsc; + + conf = &lcore_conf[lcore]; + + pkt_per_port = MAX_TRAFFIC_BURST; + num = pkt_per_port * conf->nb_ports; + + rte_atomic64_init(&start); + + /* start polling thread, but not actually poll yet */ + rte_eal_remote_launch(poll_burst, + (void *)&pkt_per_port, lcore); + + /* Only when polling first */ + if (flags == SC_BURST_POLL_FIRST) + rte_atomic64_set(&start, 1); + + /* start xmit */ + while (num) { + nb_tx = RTE_MIN(MAX_PKT_BURST, num); + for (i = 0; i < conf->nb_ports; i++) { + portid = conf->portlist[i]; + nb_tx = rte_eth_tx_burst(portid, 0, + &tx_burst[idx], nb_tx); + idx += nb_tx; + num -= nb_tx; + } + + } + + sleep(5); + + /* only when polling second */ + if (flags == SC_BURST_XMIT_FIRST) + rte_atomic64_set(&start, 1); + + /* wait for polling finished */ + diff_tsc = rte_eal_wait_lcore(lcore); + if (diff_tsc < 0) { + printf("exec_burst: Failed to measure cycles per packet\n"); + return -1; + } + + printf("Result: %d cycles per packet\n", diff_tsc); + + return 0; +} + +static int +test_pmd_perf(void) +{ + uint16_t nb_ports, num, nb_lcores, slave_id = (uint16_t)-1; + uint16_t nb_rxd = MAX_TRAFFIC_BURST; + uint16_t nb_txd = MAX_TRAFFIC_BURST; + uint16_t portid; + uint16_t nb_rx_queue = 1, nb_tx_queue = 1; + int socketid = -1; + int ret; + + printf("Start PMD RXTX cycles cost test.\n"); + + signal(SIGUSR1, signal_handler); + signal(SIGUSR2, signal_handler); + + nb_ports = rte_eth_dev_count(); + if (nb_ports < NB_ETHPORTS_USED) { + printf("At least %u port(s) used for perf. test\n", + NB_ETHPORTS_USED); + return -1; + } + + nb_lcores = rte_lcore_count(); + + memset(lcore_conf, 0, sizeof(lcore_conf)); + init_lcores(); + + init_mbufpool(NB_MBUF); + + if (sc_flag == SC_CONTINUOUS) { + nb_rxd = RTE_TEST_RX_DESC_DEFAULT; + nb_txd = RTE_TEST_TX_DESC_DEFAULT; + } + printf("CONFIG RXD=%d TXD=%d\n", nb_rxd, nb_txd); + + reset_count(); + num = 0; + for (portid = 0; portid < nb_ports; portid++) { + if (socketid == -1) { + socketid = rte_eth_dev_socket_id(portid); + slave_id = alloc_lcore(socketid); + if (slave_id == (uint16_t)-1) { + printf("No avail lcore to run test\n"); + return -1; + } + printf("Performance test runs on lcore %u socket %u\n", + slave_id, socketid); + } + + if (socketid != rte_eth_dev_socket_id(portid)) { + printf("Skip port %d\n", portid); + continue; + } + + /* port configure */ + ret = rte_eth_dev_configure(portid, nb_rx_queue, + nb_tx_queue, &port_conf); + if (ret < 0) + rte_exit(EXIT_FAILURE, + "Cannot configure device: err=%d, port=%d\n", + ret, portid); + + rte_eth_macaddr_get(portid, &ports_eth_addr[portid]); + printf("Port %u ", portid); + print_ethaddr("Address:", &ports_eth_addr[portid]); + printf("\n"); + + /* tx queue setup */ + ret = rte_eth_tx_queue_setup(portid, 0, nb_txd, + socketid, &tx_conf); + if (ret < 0) + rte_exit(EXIT_FAILURE, + "rte_eth_tx_queue_setup: err=%d, " + "port=%d\n", ret, portid); + + /* rx queue steup */ + ret = rte_eth_rx_queue_setup(portid, 0, nb_rxd, + socketid, &rx_conf, + mbufpool[socketid]); + if (ret < 0) + rte_exit(EXIT_FAILURE, "rte_eth_rx_queue_setup: err=%d," + "port=%d\n", ret, portid); + + /* Start device */ + stop = 0; + ret = rte_eth_dev_start(portid); + if (ret < 0) + rte_exit(EXIT_FAILURE, + "rte_eth_dev_start: err=%d, port=%d\n", + ret, portid); + + /* always eanble promiscuous */ + rte_eth_promiscuous_enable(portid); + + lcore_conf[slave_id].portlist[num++] = portid; + lcore_conf[slave_id].nb_ports++; + } + check_all_ports_link_status(nb_ports, RTE_PORT_ALL); + + if (tx_burst == NULL) { + tx_burst = (struct rte_mbuf **) + rte_calloc_socket("tx_buff", + MAX_TRAFFIC_BURST * nb_ports, + sizeof(void *), + RTE_CACHE_LINE_SIZE, socketid); + if (!tx_burst) + return -1; + } + + init_traffic(mbufpool[socketid], + tx_burst, MAX_TRAFFIC_BURST * nb_ports); + + printf("Generate %d packets @socket %d\n", + MAX_TRAFFIC_BURST * nb_ports, socketid); + + if (sc_flag == SC_CONTINUOUS) { + /* do both rxtx by default */ + if (NULL == do_measure) + do_measure = measure_rxtx; + + rte_eal_remote_launch(main_loop, NULL, slave_id); + + if (rte_eal_wait_lcore(slave_id) < 0) + return -1; + } else if (sc_flag == SC_BURST_POLL_FIRST || + sc_flag == SC_BURST_XMIT_FIRST) + if (exec_burst(sc_flag, slave_id) < 0) + return -1; + + /* port tear down */ + for (portid = 0; portid < nb_ports; portid++) { + if (socketid != rte_eth_dev_socket_id(portid)) + continue; + + rte_eth_dev_stop(portid); + } + + return 0; +} + +int +test_set_rxtx_conf(cmdline_fixed_string_t mode) +{ + printf("mode switch to %s\n", mode); + + if (!strcmp(mode, "vector")) { + /* vector rx, tx */ + tx_conf.txq_flags = 0xf01; + tx_conf.tx_rs_thresh = 32; + tx_conf.tx_free_thresh = 32; + port_conf.rxmode.hw_ip_checksum = 0; + port_conf.rxmode.enable_scatter = 0; + return 0; + } else if (!strcmp(mode, "scalar")) { + /* bulk alloc rx, full-featured tx */ + tx_conf.txq_flags = 0; + tx_conf.tx_rs_thresh = 32; + tx_conf.tx_free_thresh = 32; + port_conf.rxmode.hw_ip_checksum = 1; + port_conf.rxmode.enable_scatter = 0; + return 0; + } else if (!strcmp(mode, "hybrid")) { + /* bulk alloc rx, vector tx + * when vec macro not define, + * using the same rx/tx as scalar + */ + tx_conf.txq_flags = 0xf01; + tx_conf.tx_rs_thresh = 32; + tx_conf.tx_free_thresh = 32; + port_conf.rxmode.hw_ip_checksum = 1; + port_conf.rxmode.enable_scatter = 0; + return 0; + } else if (!strcmp(mode, "full")) { + /* full feature rx,tx pair */ + tx_conf.txq_flags = 0x0; /* must condition */ + tx_conf.tx_rs_thresh = 32; + tx_conf.tx_free_thresh = 32; + port_conf.rxmode.hw_ip_checksum = 0; + port_conf.rxmode.enable_scatter = 1; /* must condition */ + return 0; + } + + return -1; +} + +int +test_set_rxtx_anchor(cmdline_fixed_string_t type) +{ + printf("type switch to %s\n", type); + + if (!strcmp(type, "rxtx")) { + do_measure = measure_rxtx; + return 0; + } else if (!strcmp(type, "rxonly")) { + do_measure = measure_rxonly; + return 0; + } else if (!strcmp(type, "txonly")) { + do_measure = measure_txonly; + return 0; + } + + return -1; +} + +int +test_set_rxtx_sc(cmdline_fixed_string_t type) +{ + printf("stream control switch to %s\n", type); + + if (!strcmp(type, "continuous")) { + sc_flag = SC_CONTINUOUS; + return 0; + } else if (!strcmp(type, "poll_before_xmit")) { + sc_flag = SC_BURST_POLL_FIRST; + return 0; + } else if (!strcmp(type, "poll_after_xmit")) { + sc_flag = SC_BURST_XMIT_FIRST; + return 0; + } + + return -1; +} + +REGISTER_TEST_COMMAND(pmd_perf_autotest, test_pmd_perf); diff --git a/dpdk/test/test/test_pmd_ring.c b/dpdk/test/test/test_pmd_ring.c new file mode 100644 index 00000000..2cdf60d1 --- /dev/null +++ b/dpdk/test/test/test_pmd_ring.c @@ -0,0 +1,529 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include "test.h" + +#include <stdio.h> + +#include <rte_eth_ring.h> +#include <rte_ethdev.h> + +static struct rte_mempool *mp; +static int tx_porta, rx_portb, rxtx_portc, rxtx_portd, rxtx_porte; + +#define SOCKET0 0 +#define RING_SIZE 256 +#define NUM_RINGS 2 +#define NB_MBUF 512 + + +static int +test_ethdev_configure_port(int port) +{ + struct rte_eth_conf null_conf; + struct rte_eth_link link; + + memset(&null_conf, 0, sizeof(struct rte_eth_conf)); + + if (rte_eth_dev_configure(port, 1, 2, &null_conf) < 0) { + printf("Configure failed for port %d\n", port); + return -1; + } + + /* Test queue release */ + if (rte_eth_dev_configure(port, 1, 1, &null_conf) < 0) { + printf("Configure failed for port %d\n", port); + return -1; + } + + if (rte_eth_tx_queue_setup(port, 0, RING_SIZE, SOCKET0, NULL) < 0) { + printf("TX queue setup failed port %d\n", port); + return -1; + } + + if (rte_eth_rx_queue_setup(port, 0, RING_SIZE, SOCKET0, + NULL, mp) < 0) { + printf("RX queue setup failed port %d\n", port); + return -1; + } + + if (rte_eth_dev_start(port) < 0) { + printf("Error starting port %d\n", port); + return -1; + } + + rte_eth_link_get(port, &link); + + return 0; +} + +static int +test_send_basic_packets(void) +{ + struct rte_mbuf bufs[RING_SIZE]; + struct rte_mbuf *pbufs[RING_SIZE]; + int i; + + printf("Testing send and receive RING_SIZE/2 packets (tx_porta -> rx_portb)\n"); + + for (i = 0; i < RING_SIZE/2; i++) + pbufs[i] = &bufs[i]; + + if (rte_eth_tx_burst(tx_porta, 0, pbufs, RING_SIZE/2) < RING_SIZE/2) { + printf("Failed to transmit packet burst port %d\n", tx_porta); + return -1; + } + + if (rte_eth_rx_burst(rx_portb, 0, pbufs, RING_SIZE) != RING_SIZE/2) { + printf("Failed to receive packet burst on port %d\n", rx_portb); + return -1; + } + + for (i = 0; i < RING_SIZE/2; i++) + if (pbufs[i] != &bufs[i]) { + printf("Error: received data does not match that transmitted\n"); + return -1; + } + + return 0; +} + +static int +test_send_basic_packets_port(int port) +{ + struct rte_mbuf bufs[RING_SIZE]; + struct rte_mbuf *pbufs[RING_SIZE]; + int i; + + printf("Testing send and receive RING_SIZE/2 packets (cmdl_port0 -> cmdl_port0)\n"); + + for (i = 0; i < RING_SIZE/2; i++) + pbufs[i] = &bufs[i]; + + if (rte_eth_tx_burst(port, 0, pbufs, RING_SIZE/2) < RING_SIZE/2) { + printf("Failed to transmit packet burst port %d\n", port); + return -1; + } + + if (rte_eth_rx_burst(port, 0, pbufs, RING_SIZE) != RING_SIZE/2) { + printf("Failed to receive packet burst on port %d\n", port); + return -1; + } + + for (i = 0; i < RING_SIZE/2; i++) + if (pbufs[i] != &bufs[i]) { + printf("Error: received data does not match that transmitted\n"); + return -1; + } + + return 0; +} + + +static int +test_get_stats(int port) +{ + struct rte_eth_stats stats; + struct rte_mbuf buf, *pbuf = &buf; + + printf("Testing ring PMD stats_get port %d\n", port); + + /* check stats of RXTX port, should all be zero */ + + rte_eth_stats_get(port, &stats); + if (stats.ipackets != 0 || stats.opackets != 0 || + stats.ibytes != 0 || stats.obytes != 0 || + stats.ierrors != 0 || stats.oerrors != 0) { + printf("Error: port %d stats are not zero\n", port); + return -1; + } + + /* send and receive 1 packet and check for stats update */ + if (rte_eth_tx_burst(port, 0, &pbuf, 1) != 1) { + printf("Error sending packet to port %d\n", port); + return -1; + } + + if (rte_eth_rx_burst(port, 0, &pbuf, 1) != 1) { + printf("Error receiving packet from port %d\n", port); + return -1; + } + + rte_eth_stats_get(port, &stats); + if (stats.ipackets != 1 || stats.opackets != 1 || + stats.ibytes != 0 || stats.obytes != 0 || + stats.ierrors != 0 || stats.oerrors != 0) { + printf("Error: port %d stats are not as expected\n", port); + return -1; + } + return 0; +} + +static int +test_stats_reset(int port) +{ + struct rte_eth_stats stats; + struct rte_mbuf buf, *pbuf = &buf; + + printf("Testing ring PMD stats_reset port %d\n", port); + + rte_eth_stats_reset(port); + + /* check stats of RXTX port, should all be zero */ + rte_eth_stats_get(port, &stats); + if (stats.ipackets != 0 || stats.opackets != 0 || + stats.ibytes != 0 || stats.obytes != 0 || + stats.ierrors != 0 || stats.oerrors != 0) { + printf("Error: port %d stats are not zero\n", port); + return -1; + } + + /* send and receive 1 packet and check for stats update */ + if (rte_eth_tx_burst(port, 0, &pbuf, 1) != 1) { + printf("Error sending packet to port %d\n", port); + return -1; + } + + if (rte_eth_rx_burst(port, 0, &pbuf, 1) != 1) { + printf("Error receiving packet from port %d\n", port); + return -1; + } + + rte_eth_stats_get(port, &stats); + if (stats.ipackets != 1 || stats.opackets != 1 || + stats.ibytes != 0 || stats.obytes != 0 || + stats.ierrors != 0 || stats.oerrors != 0) { + printf("Error: port %d stats are not as expected\n", port); + return -1; + } + + rte_eth_stats_reset(port); + + /* check stats of RXTX port, should all be zero */ + rte_eth_stats_get(port, &stats); + if (stats.ipackets != 0 || stats.opackets != 0 || + stats.ibytes != 0 || stats.obytes != 0 || + stats.ierrors != 0 || stats.oerrors != 0) { + printf("Error: port %d stats are not zero\n", port); + return -1; + } + + return 0; +} + +static int +test_pmd_ring_pair_create_attach(int portd, int porte) +{ + struct rte_eth_stats stats, stats2; + struct rte_mbuf buf, *pbuf = &buf; + struct rte_eth_conf null_conf; + + if ((rte_eth_dev_configure(portd, 1, 1, &null_conf) < 0) + || (rte_eth_dev_configure(porte, 1, 1, &null_conf) < 0)) { + printf("Configure failed for port\n"); + return -1; + } + + if ((rte_eth_tx_queue_setup(portd, 0, RING_SIZE, SOCKET0, NULL) < 0) + || (rte_eth_tx_queue_setup(porte, 0, RING_SIZE, SOCKET0, NULL) < 0)) { + printf("TX queue setup failed\n"); + return -1; + } + + if ((rte_eth_rx_queue_setup(portd, 0, RING_SIZE, SOCKET0, NULL, mp) < 0) + || (rte_eth_rx_queue_setup(porte, 0, RING_SIZE, SOCKET0, NULL, mp) < 0)) { + printf("RX queue setup failed\n"); + return -1; + } + + if ((rte_eth_dev_start(portd) < 0) + || (rte_eth_dev_start(porte) < 0)) { + printf("Error starting port\n"); + return -1; + } + + rte_eth_stats_reset(portd); + /* check stats of port, should all be zero */ + rte_eth_stats_get(portd, &stats); + if (stats.ipackets != 0 || stats.opackets != 0 || + stats.ibytes != 0 || stats.obytes != 0 || + stats.ierrors != 0 || stats.oerrors != 0) { + printf("Error: port %d stats are not zero\n", portd); + return -1; + } + + rte_eth_stats_reset(porte); + /* check stats of port, should all be zero */ + rte_eth_stats_get(porte, &stats2); + if (stats2.ipackets != 0 || stats2.opackets != 0 || + stats2.ibytes != 0 || stats2.obytes != 0 || + stats2.ierrors != 0 || stats2.oerrors != 0) { + printf("Error: port %d stats are not zero\n", porte); + return -1; + } + + /* + * send and receive 1 packet (portd -> porte) + * and check for stats update + */ + printf("Testing send and receive 1 packet (portd -> porte)\n"); + if (rte_eth_tx_burst(portd, 0, &pbuf, 1) != 1) { + printf("Error sending packet to port %d\n", portd); + return -1; + } + + if (rte_eth_rx_burst(porte, 0, &pbuf, 1) != 1) { + printf("Error receiving packet from port %d\n", porte); + return -1; + } + + rte_eth_stats_get(portd, &stats); + rte_eth_stats_get(porte, &stats2); + if (stats.ipackets != 0 || stats.opackets != 1 || + stats.ibytes != 0 || stats.obytes != 0 || + stats.ierrors != 0 || stats.oerrors != 0) { + printf("Error: port %d stats are not as expected\n", portd); + return -1; + } + + if (stats2.ipackets != 1 || stats2.opackets != 0 || + stats2.ibytes != 0 || stats2.obytes != 0 || + stats2.ierrors != 0 || stats2.oerrors != 0) { + printf("Error: port %d stats are not as expected\n", porte); + return -1; + } + + /* + * send and receive 1 packet (porte -> portd) + * and check for stats update + */ + printf("Testing send and receive 1 packet (porte -> portd)\n"); + if (rte_eth_tx_burst(porte, 0, &pbuf, 1) != 1) { + printf("Error sending packet to port %d\n", porte); + return -1; + } + + if (rte_eth_rx_burst(portd, 0, &pbuf, 1) != 1) { + printf("Error receiving packet from port %d\n", portd); + return -1; + } + + rte_eth_stats_get(portd, &stats); + rte_eth_stats_get(porte, &stats2); + if (stats.ipackets != 1 || stats.opackets != 1 || + stats.ibytes != 0 || stats.obytes != 0 || + stats.ierrors != 0 || stats.oerrors != 0) { + printf("Error: port %d stats are not as expected\n", portd); + return -1; + } + + if (stats2.ipackets != 1 || stats2.opackets != 1 || + stats2.ibytes != 0 || stats2.obytes != 0 || + stats2.ierrors != 0 || stats2.oerrors != 0) { + printf("Error: port %d stats are not as expected\n", porte); + return -1; + } + + /* + * send and receive 1 packet (portd -> portd) + * and check for stats update + */ + printf("Testing send and receive 1 packet (portd -> portd)\n"); + if (rte_eth_tx_burst(portd, 0, &pbuf, 1) != 1) { + printf("Error sending packet to port %d\n", portd); + return -1; + } + + if (rte_eth_rx_burst(portd, 0, &pbuf, 1) != 1) { + printf("Error receiving packet from port %d\n", porte); + return -1; + } + + rte_eth_stats_get(portd, &stats); + rte_eth_stats_get(porte, &stats2); + if (stats.ipackets != 2 || stats.opackets != 2 || + stats.ibytes != 0 || stats.obytes != 0 || + stats.ierrors != 0 || stats.oerrors != 0) { + printf("Error: port %d stats are not as expected\n", portd); + return -1; + } + + if (stats2.ipackets != 1 || stats2.opackets != 1 || + stats2.ibytes != 0 || stats2.obytes != 0 || + stats2.ierrors != 0 || stats2.oerrors != 0) { + printf("Error: port %d stats are not as expected\n", porte); + return -1; + } + + /* + * send and receive 1 packet (porte -> porte) + * and check for stats update + */ + printf("Testing send and receive 1 packet (porte -> porte)\n"); + if (rte_eth_tx_burst(porte, 0, &pbuf, 1) != 1) { + printf("Error sending packet to port %d\n", porte); + return -1; + } + + if (rte_eth_rx_burst(porte, 0, &pbuf, 1) != 1) { + printf("Error receiving packet from port %d\n", porte); + return -1; + } + + rte_eth_stats_get(portd, &stats); + rte_eth_stats_get(porte, &stats2); + if (stats.ipackets != 2 || stats.opackets != 2 || + stats.ibytes != 0 || stats.obytes != 0 || + stats.ierrors != 0 || stats.oerrors != 0) { + printf("Error: port %d stats are not as expected\n", portd); + return -1; + } + + if (stats2.ipackets != 2 || stats2.opackets != 2 || + stats2.ibytes != 0 || stats2.obytes != 0 || + stats2.ierrors != 0 || stats2.oerrors != 0) { + printf("Error: port %d stats are not as expected\n", porte); + return -1; + } + + rte_eth_dev_stop(portd); + rte_eth_dev_stop(porte); + + return 0; +} + +static int +test_pmd_ring(void) +{ + struct rte_ring *rxtx[NUM_RINGS]; + int port, cmdl_port0 = -1; + uint8_t nb_ports; + + nb_ports = rte_eth_dev_count(); + printf("nb_ports=%d\n", (int)nb_ports); + + /* create the rings and eth_rings in the test code. + * This does not test the rte_pmd_ring_devinit function. + * + * Test with the command line option --vdev=net_ring0 to test rte_pmd_ring_devinit. + */ + rxtx[0] = rte_ring_create("R0", RING_SIZE, SOCKET0, RING_F_SP_ENQ|RING_F_SC_DEQ); + if (rxtx[0] == NULL) { + printf("rte_ring_create R0 failed"); + return -1; + } + + rxtx[1] = rte_ring_create("R1", RING_SIZE, SOCKET0, RING_F_SP_ENQ|RING_F_SC_DEQ); + if (rxtx[1] == NULL) { + printf("rte_ring_create R1 failed"); + return -1; + } + + tx_porta = rte_eth_from_rings("net_ringa", rxtx, NUM_RINGS, rxtx, NUM_RINGS, SOCKET0); + rx_portb = rte_eth_from_rings("net_ringb", rxtx, NUM_RINGS, rxtx, NUM_RINGS, SOCKET0); + rxtx_portc = rte_eth_from_rings("net_ringc", rxtx, NUM_RINGS, rxtx, NUM_RINGS, SOCKET0); + rxtx_portd = rte_eth_from_rings("net_ringd", rxtx, NUM_RINGS, rxtx, NUM_RINGS, SOCKET0); + rxtx_porte = rte_eth_from_rings("net_ringe", rxtx, NUM_RINGS, rxtx, NUM_RINGS, SOCKET0); + + printf("tx_porta=%d rx_portb=%d rxtx_portc=%d rxtx_portd=%d rxtx_porte=%d\n", + tx_porta, rx_portb, rxtx_portc, rxtx_portd, rxtx_porte); + + if ((tx_porta == -1) || (rx_portb == -1) || (rxtx_portc == -1) + || (rxtx_portd == -1) || (rxtx_porte == -1)) { + printf("rte_eth_from rings failed\n"); + return -1; + } + + mp = rte_pktmbuf_pool_create("mbuf_pool", NB_MBUF, 32, + 0, RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id()); + if (mp == NULL) + return -1; + + if ((tx_porta >= RTE_MAX_ETHPORTS) || (rx_portb >= RTE_MAX_ETHPORTS) + || (rxtx_portc >= RTE_MAX_ETHPORTS) + || (rxtx_portd >= RTE_MAX_ETHPORTS) + || (rxtx_porte >= RTE_MAX_ETHPORTS)) { + printf(" port exceed max eth ports\n"); + return -1; + } + + if (test_ethdev_configure_port(tx_porta) < 0) + return -1; + + if (test_ethdev_configure_port(rx_portb) < 0) + return -1; + + if (test_ethdev_configure_port(rxtx_portc) < 0) + return -1; + + if (test_send_basic_packets() < 0) + return -1; + + if (test_get_stats(rxtx_portc) < 0) + return -1; + + if (test_stats_reset(rxtx_portc) < 0) + return -1; + + rte_eth_dev_stop(tx_porta); + rte_eth_dev_stop(rx_portb); + rte_eth_dev_stop(rxtx_portc); + + if (test_pmd_ring_pair_create_attach(rxtx_portd, rxtx_porte) < 0) + return -1; + + /* find a port created with the --vdev=net_ring0 command line option */ + for (port = 0; port < nb_ports; port++) { + struct rte_eth_dev_info dev_info; + + rte_eth_dev_info_get(port, &dev_info); + if (!strcmp(dev_info.driver_name, "Rings PMD")) { + printf("found a command line ring port=%d\n", port); + cmdl_port0 = port; + break; + } + } + if (cmdl_port0 != -1) { + if (test_ethdev_configure_port(cmdl_port0) < 0) + return -1; + if (test_send_basic_packets_port(cmdl_port0) < 0) + return -1; + if (test_stats_reset(cmdl_port0) < 0) + return -1; + if (test_get_stats(cmdl_port0) < 0) + return -1; + rte_eth_dev_stop(cmdl_port0); + } + return 0; +} + +REGISTER_TEST_COMMAND(ring_pmd_autotest, test_pmd_ring); diff --git a/dpdk/test/test/test_pmd_ring_perf.c b/dpdk/test/test/test_pmd_ring_perf.c new file mode 100644 index 00000000..8e9cd331 --- /dev/null +++ b/dpdk/test/test/test_pmd_ring_perf.c @@ -0,0 +1,186 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2015 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + + +#include <stdio.h> +#include <inttypes.h> +#include <rte_ring.h> +#include <rte_cycles.h> +#include <rte_launch.h> +#include <rte_ethdev.h> +#include <rte_eth_ring.h> + +#include "test.h" + +#define RING_NAME "RING_PERF" +#define RING_SIZE 4096 +#define MAX_BURST 32 + +/* + * the sizes to enqueue and dequeue in testing + * (marked volatile so they won't be seen as compile-time constants) + */ +static const volatile unsigned bulk_sizes[] = { 1, 8, 32 }; + +/* The ring structure used for tests */ +static struct rte_ring *r; +static uint16_t ring_ethdev_port; + +/* Get cycle counts for dequeuing from an empty ring. Should be 2 or 3 cycles */ +static void +test_empty_dequeue(void) +{ + const unsigned iter_shift = 26; + const unsigned iterations = 1 << iter_shift; + unsigned i = 0; + void *burst[MAX_BURST]; + + const uint64_t sc_start = rte_rdtsc(); + for (i = 0; i < iterations; i++) + rte_ring_sc_dequeue_bulk(r, burst, bulk_sizes[0], NULL); + const uint64_t sc_end = rte_rdtsc(); + + const uint64_t eth_start = rte_rdtsc(); + for (i = 0; i < iterations; i++) + rte_eth_rx_burst(ring_ethdev_port, 0, (void *)burst, + bulk_sizes[0]); + const uint64_t eth_end = rte_rdtsc(); + + printf("Ring empty dequeue : %.1F\n", + (double)(sc_end - sc_start) / iterations); + printf("Ethdev empty dequeue: %.1F\n", + (double)(eth_end - eth_start) / iterations); +} + +/* + * Test function that determines how long an enqueue + dequeue of a single item + * takes on a single lcore. Result is for comparison with the bulk enq+deq. + */ +static void +test_single_enqueue_dequeue(void) +{ + const unsigned iter_shift = 24; + const unsigned iterations = 1 << iter_shift; + unsigned i = 0; + void *burst = NULL; + struct rte_mbuf *mburst[1] = { NULL }; + + const uint64_t sc_start = rte_rdtsc_precise(); + rte_compiler_barrier(); + for (i = 0; i < iterations; i++) { + rte_ring_enqueue_bulk(r, &burst, 1, NULL); + rte_ring_dequeue_bulk(r, &burst, 1, NULL); + } + const uint64_t sc_end = rte_rdtsc_precise(); + rte_compiler_barrier(); + + const uint64_t eth_start = rte_rdtsc_precise(); + rte_compiler_barrier(); + for (i = 0; i < iterations; i++) { + rte_eth_tx_burst(ring_ethdev_port, 0, mburst, 1); + rte_eth_rx_burst(ring_ethdev_port, 0, mburst, 1); + } + const uint64_t eth_end = rte_rdtsc_precise(); + rte_compiler_barrier(); + + printf("Ring single enq/dequeue : %"PRIu64"\n", + (sc_end-sc_start) >> iter_shift); + printf("Ethdev single enq/dequeue: %"PRIu64"\n", + (eth_end-eth_start) >> iter_shift); +} + +/* Times enqueue and dequeue on a single lcore */ +static void +test_bulk_enqueue_dequeue(void) +{ + const unsigned iter_shift = 23; + const unsigned iterations = 1 << iter_shift; + unsigned sz, i = 0; + struct rte_mbuf *burst[MAX_BURST] = {0}; + + for (sz = 0; sz < sizeof(bulk_sizes)/sizeof(bulk_sizes[0]); sz++) { + const uint64_t sc_start = rte_rdtsc(); + for (i = 0; i < iterations; i++) { + rte_ring_sp_enqueue_bulk(r, (void *)burst, + bulk_sizes[sz], NULL); + rte_ring_sc_dequeue_bulk(r, (void *)burst, + bulk_sizes[sz], NULL); + } + const uint64_t sc_end = rte_rdtsc(); + + const uint64_t eth_start = rte_rdtsc_precise(); + rte_compiler_barrier(); + for (i = 0; i < iterations; i++) { + rte_eth_tx_burst(ring_ethdev_port, 0, burst, bulk_sizes[sz]); + rte_eth_rx_burst(ring_ethdev_port, 0, burst, bulk_sizes[sz]); + } + const uint64_t eth_end = rte_rdtsc_precise(); + rte_compiler_barrier(); + + double sc_avg = ((double)(sc_end-sc_start) / + (iterations * bulk_sizes[sz])); + double eth_avg = ((double)(eth_end-eth_start) / + (iterations * bulk_sizes[sz])); + + printf("ring bulk enq/deq (size: %u) : %.1F\n", bulk_sizes[sz], + sc_avg); + printf("ethdev bulk enq/deq (size:%u): %.1F\n", bulk_sizes[sz], + eth_avg); + + printf("\n"); + } +} + +static int +test_ring_pmd_perf(void) +{ + r = rte_ring_create(RING_NAME, RING_SIZE, rte_socket_id(), + RING_F_SP_ENQ|RING_F_SC_DEQ); + if (r == NULL && (r = rte_ring_lookup(RING_NAME)) == NULL) + return -1; + + ring_ethdev_port = rte_eth_from_ring(r); + + printf("\n### Testing const single element enq/deq ###\n"); + test_single_enqueue_dequeue(); + + printf("\n### Testing empty dequeue ###\n"); + test_empty_dequeue(); + + printf("\n### Testing using a single lcore ###\n"); + test_bulk_enqueue_dequeue(); + + return 0; +} + +REGISTER_TEST_COMMAND(ring_pmd_perf_autotest, test_ring_pmd_perf); diff --git a/dpdk/test/test/test_power.c b/dpdk/test/test/test_power.c new file mode 100644 index 00000000..b2e1344c --- /dev/null +++ b/dpdk/test/test/test_power.c @@ -0,0 +1,107 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <stdint.h> +#include <unistd.h> +#include <limits.h> +#include <string.h> + +#include "test.h" + +#include <rte_power.h> + +static int +test_power(void) +{ + int ret = -1; + enum power_management_env env; + + /* Test setting an invalid environment */ + ret = rte_power_set_env(PM_ENV_NOT_SET); + if (ret == 0) { + printf("Unexpectedly succeeded on setting an invalid environment\n"); + return -1; + } + + /* Test that the environment has not been set */ + env = rte_power_get_env(); + if (env != PM_ENV_NOT_SET) { + printf("Unexpectedly got a valid environment configuration\n"); + return -1; + } + + /* verify that function pointers are NULL */ + if (rte_power_freqs != NULL) { + printf("rte_power_freqs should be NULL, environment has not been " + "initialised\n"); + goto fail_all; + } + if (rte_power_get_freq != NULL) { + printf("rte_power_get_freq should be NULL, environment has not been " + "initialised\n"); + goto fail_all; + } + if (rte_power_set_freq != NULL) { + printf("rte_power_set_freq should be NULL, environment has not been " + "initialised\n"); + goto fail_all; + } + if (rte_power_freq_up != NULL) { + printf("rte_power_freq_up should be NULL, environment has not been " + "initialised\n"); + goto fail_all; + } + if (rte_power_freq_down != NULL) { + printf("rte_power_freq_down should be NULL, environment has not been " + "initialised\n"); + goto fail_all; + } + if (rte_power_freq_max != NULL) { + printf("rte_power_freq_max should be NULL, environment has not been " + "initialised\n"); + goto fail_all; + } + if (rte_power_freq_min != NULL) { + printf("rte_power_freq_min should be NULL, environment has not been " + "initialised\n"); + goto fail_all; + } + rte_power_unset_env(); + return 0; +fail_all: + rte_power_unset_env(); + return -1; +} + +REGISTER_TEST_COMMAND(power_autotest, test_power); diff --git a/dpdk/test/test/test_power_acpi_cpufreq.c b/dpdk/test/test/test_power_acpi_cpufreq.c new file mode 100644 index 00000000..64f5dd56 --- /dev/null +++ b/dpdk/test/test/test_power_acpi_cpufreq.c @@ -0,0 +1,540 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <stdint.h> +#include <unistd.h> +#include <limits.h> +#include <string.h> + +#include "test.h" + +#include <rte_power.h> + +#define TEST_POWER_LCORE_ID 2U +#define TEST_POWER_LCORE_INVALID ((unsigned)RTE_MAX_LCORE) +#define TEST_POWER_FREQS_NUM_MAX ((unsigned)RTE_MAX_LCORE_FREQS) + +#define TEST_POWER_SYSFILE_CUR_FREQ \ + "/sys/devices/system/cpu/cpu%u/cpufreq/scaling_cur_freq" + +static uint32_t total_freq_num; +static uint32_t freqs[TEST_POWER_FREQS_NUM_MAX]; + +static int +check_cur_freq(unsigned lcore_id, uint32_t idx) +{ +#define TEST_POWER_CONVERT_TO_DECIMAL 10 + FILE *f; + char fullpath[PATH_MAX]; + char buf[BUFSIZ]; + uint32_t cur_freq; + int ret = -1; + + if (snprintf(fullpath, sizeof(fullpath), + TEST_POWER_SYSFILE_CUR_FREQ, lcore_id) < 0) { + return 0; + } + f = fopen(fullpath, "r"); + if (f == NULL) { + return 0; + } + if (fgets(buf, sizeof(buf), f) == NULL) { + goto fail_get_cur_freq; + } + cur_freq = strtoul(buf, NULL, TEST_POWER_CONVERT_TO_DECIMAL); + ret = (freqs[idx] == cur_freq ? 0 : -1); + +fail_get_cur_freq: + fclose(f); + + return ret; +} + +/* Check rte_power_freqs() */ +static int +check_power_freqs(void) +{ + uint32_t ret; + + total_freq_num = 0; + memset(freqs, 0, sizeof(freqs)); + + /* test with an invalid lcore id */ + ret = rte_power_freqs(TEST_POWER_LCORE_INVALID, freqs, + TEST_POWER_FREQS_NUM_MAX); + if (ret > 0) { + printf("Unexpectedly get available freqs successfully on " + "lcore %u\n", TEST_POWER_LCORE_INVALID); + return -1; + } + + /* test with NULL buffer to save available freqs */ + ret = rte_power_freqs(TEST_POWER_LCORE_ID, NULL, + TEST_POWER_FREQS_NUM_MAX); + if (ret > 0) { + printf("Unexpectedly get available freqs successfully with " + "NULL buffer on lcore %u\n", TEST_POWER_LCORE_ID); + return -1; + } + + /* test of getting zero number of freqs */ + ret = rte_power_freqs(TEST_POWER_LCORE_ID, freqs, 0); + if (ret > 0) { + printf("Unexpectedly get available freqs successfully with " + "zero buffer size on lcore %u\n", TEST_POWER_LCORE_ID); + return -1; + } + + /* test with all valid input parameters */ + ret = rte_power_freqs(TEST_POWER_LCORE_ID, freqs, + TEST_POWER_FREQS_NUM_MAX); + if (ret == 0 || ret > TEST_POWER_FREQS_NUM_MAX) { + printf("Fail to get available freqs on lcore %u\n", + TEST_POWER_LCORE_ID); + return -1; + } + + /* Save the total number of available freqs */ + total_freq_num = ret; + + return 0; +} + +/* Check rte_power_get_freq() */ +static int +check_power_get_freq(void) +{ + int ret; + uint32_t count; + + /* test with an invalid lcore id */ + count = rte_power_get_freq(TEST_POWER_LCORE_INVALID); + if (count < TEST_POWER_FREQS_NUM_MAX) { + printf("Unexpectedly get freq index successfully on " + "lcore %u\n", TEST_POWER_LCORE_INVALID); + return -1; + } + + count = rte_power_get_freq(TEST_POWER_LCORE_ID); + if (count >= TEST_POWER_FREQS_NUM_MAX) { + printf("Fail to get the freq index on lcore %u\n", + TEST_POWER_LCORE_ID); + return -1; + } + + /* Check the current frequency */ + ret = check_cur_freq(TEST_POWER_LCORE_ID, count); + if (ret < 0) + return -1; + + return 0; +} + +/* Check rte_power_set_freq() */ +static int +check_power_set_freq(void) +{ + int ret; + + /* test with an invalid lcore id */ + ret = rte_power_set_freq(TEST_POWER_LCORE_INVALID, 0); + if (ret >= 0) { + printf("Unexpectedly set freq index successfully on " + "lcore %u\n", TEST_POWER_LCORE_INVALID); + return -1; + } + + /* test with an invalid freq index */ + ret = rte_power_set_freq(TEST_POWER_LCORE_ID, + TEST_POWER_FREQS_NUM_MAX); + if (ret >= 0) { + printf("Unexpectedly set an invalid freq index (%u)" + "successfully on lcore %u\n", TEST_POWER_FREQS_NUM_MAX, + TEST_POWER_LCORE_ID); + return -1; + } + + /** + * test with an invalid freq index which is right one bigger than + * total number of freqs + */ + ret = rte_power_set_freq(TEST_POWER_LCORE_ID, total_freq_num); + if (ret >= 0) { + printf("Unexpectedly set an invalid freq index (%u)" + "successfully on lcore %u\n", total_freq_num, + TEST_POWER_LCORE_ID); + return -1; + } + ret = rte_power_set_freq(TEST_POWER_LCORE_ID, total_freq_num - 1); + if (ret < 0) { + printf("Fail to set freq index on lcore %u\n", + TEST_POWER_LCORE_ID); + return -1; + } + + /* Check the current frequency */ + ret = check_cur_freq(TEST_POWER_LCORE_ID, total_freq_num - 1); + if (ret < 0) + return -1; + + return 0; +} + +/* Check rte_power_freq_down() */ +static int +check_power_freq_down(void) +{ + int ret; + + /* test with an invalid lcore id */ + ret = rte_power_freq_down(TEST_POWER_LCORE_INVALID); + if (ret >= 0) { + printf("Unexpectedly scale down successfully the freq on " + "lcore %u\n", TEST_POWER_LCORE_INVALID); + return -1; + } + + /* Scale down to min and then scale down one step */ + ret = rte_power_freq_min(TEST_POWER_LCORE_ID); + if (ret < 0) { + printf("Fail to scale down the freq to min on lcore %u\n", + TEST_POWER_LCORE_ID); + return -1; + } + ret = rte_power_freq_down(TEST_POWER_LCORE_ID); + if (ret < 0) { + printf("Fail to scale down the freq on lcore %u\n", + TEST_POWER_LCORE_ID); + return -1; + } + + /* Check the current frequency */ + ret = check_cur_freq(TEST_POWER_LCORE_ID, total_freq_num - 1); + if (ret < 0) + return -1; + + /* Scale up to max and then scale down one step */ + ret = rte_power_freq_max(TEST_POWER_LCORE_ID); + if (ret < 0) { + printf("Fail to scale up the freq to max on lcore %u\n", + TEST_POWER_LCORE_ID); + return -1; + } + ret = rte_power_freq_down(TEST_POWER_LCORE_ID); + if (ret < 0) { + printf("Fail to scale down the freq on lcore %u\n", + TEST_POWER_LCORE_ID); + return -1; + } + + /* Check the current frequency */ + ret = check_cur_freq(TEST_POWER_LCORE_ID, 1); + if (ret < 0) + return -1; + + return 0; +} + +/* Check rte_power_freq_up() */ +static int +check_power_freq_up(void) +{ + int ret; + + /* test with an invalid lcore id */ + ret = rte_power_freq_up(TEST_POWER_LCORE_INVALID); + if (ret >= 0) { + printf("Unexpectedly scale up successfully the freq on %u\n", + TEST_POWER_LCORE_INVALID); + return -1; + } + + /* Scale down to min and then scale up one step */ + ret = rte_power_freq_min(TEST_POWER_LCORE_ID); + if (ret < 0) { + printf("Fail to scale down the freq to min on lcore %u\n", + TEST_POWER_LCORE_ID); + return -1; + } + ret = rte_power_freq_up(TEST_POWER_LCORE_ID); + if (ret < 0) { + printf("Fail to scale up the freq on lcore %u\n", + TEST_POWER_LCORE_ID); + return -1; + } + + /* Check the current frequency */ + ret = check_cur_freq(TEST_POWER_LCORE_ID, total_freq_num - 2); + if (ret < 0) + return -1; + + /* Scale up to max and then scale up one step */ + ret = rte_power_freq_max(TEST_POWER_LCORE_ID); + if (ret < 0) { + printf("Fail to scale up the freq to max on lcore %u\n", + TEST_POWER_LCORE_ID); + return -1; + } + ret = rte_power_freq_up(TEST_POWER_LCORE_ID); + if (ret < 0) { + printf("Fail to scale up the freq on lcore %u\n", + TEST_POWER_LCORE_ID); + return -1; + } + + /* Check the current frequency */ + ret = check_cur_freq(TEST_POWER_LCORE_ID, 0); + if (ret < 0) + return -1; + + return 0; +} + +/* Check rte_power_freq_max() */ +static int +check_power_freq_max(void) +{ + int ret; + + /* test with an invalid lcore id */ + ret = rte_power_freq_max(TEST_POWER_LCORE_INVALID); + if (ret >= 0) { + printf("Unexpectedly scale up successfully the freq to max on " + "lcore %u\n", TEST_POWER_LCORE_INVALID); + return -1; + } + ret = rte_power_freq_max(TEST_POWER_LCORE_ID); + if (ret < 0) { + printf("Fail to scale up the freq to max on lcore %u\n", + TEST_POWER_LCORE_ID); + return -1; + } + + /* Check the current frequency */ + ret = check_cur_freq(TEST_POWER_LCORE_ID, 0); + if (ret < 0) + return -1; + + return 0; +} + +/* Check rte_power_freq_min() */ +static int +check_power_freq_min(void) +{ + int ret; + + /* test with an invalid lcore id */ + ret = rte_power_freq_min(TEST_POWER_LCORE_INVALID); + if (ret >= 0) { + printf("Unexpectedly scale down successfully the freq to min " + "on lcore %u\n", TEST_POWER_LCORE_INVALID); + return -1; + } + ret = rte_power_freq_min(TEST_POWER_LCORE_ID); + if (ret < 0) { + printf("Fail to scale down the freq to min on lcore %u\n", + TEST_POWER_LCORE_ID); + return -1; + } + + /* Check the current frequency */ + ret = check_cur_freq(TEST_POWER_LCORE_ID, total_freq_num - 1); + if (ret < 0) + return -1; + + return 0; +} + +static int +test_power_acpi_cpufreq(void) +{ + int ret = -1; + enum power_management_env env; + + ret = rte_power_set_env(PM_ENV_ACPI_CPUFREQ); + if (ret != 0) { + printf("Failed on setting environment to PM_ENV_ACPI_CPUFREQ, this " + "may occur if environment is not configured correctly or " + " operating in another valid Power management environment\n"); + return -1; + } + + /* Test environment configuration */ + env = rte_power_get_env(); + if (env != PM_ENV_ACPI_CPUFREQ) { + printf("Unexpectedly got an environment other than ACPI cpufreq\n"); + goto fail_all; + } + + /* verify that function pointers are not NULL */ + if (rte_power_freqs == NULL) { + printf("rte_power_freqs should not be NULL, environment has not been " + "initialised\n"); + goto fail_all; + } + if (rte_power_get_freq == NULL) { + printf("rte_power_get_freq should not be NULL, environment has not " + "been initialised\n"); + goto fail_all; + } + if (rte_power_set_freq == NULL) { + printf("rte_power_set_freq should not be NULL, environment has not " + "been initialised\n"); + goto fail_all; + } + if (rte_power_freq_up == NULL) { + printf("rte_power_freq_up should not be NULL, environment has not " + "been initialised\n"); + goto fail_all; + } + if (rte_power_freq_down == NULL) { + printf("rte_power_freq_down should not be NULL, environment has not " + "been initialised\n"); + goto fail_all; + } + if (rte_power_freq_max == NULL) { + printf("rte_power_freq_max should not be NULL, environment has not " + "been initialised\n"); + goto fail_all; + } + if (rte_power_freq_min == NULL) { + printf("rte_power_freq_min should not be NULL, environment has not " + "been initialised\n"); + goto fail_all; + } + + /* test of init power management for an invalid lcore */ + ret = rte_power_init(TEST_POWER_LCORE_INVALID); + if (ret == 0) { + printf("Unexpectedly initialise power management successfully " + "for lcore %u\n", TEST_POWER_LCORE_INVALID); + rte_power_unset_env(); + return -1; + } + + /* Test initialisation of a valid lcore */ + ret = rte_power_init(TEST_POWER_LCORE_ID); + if (ret < 0) { + printf("Cannot initialise power management for lcore %u, this " + "may occur if environment is not configured " + "correctly(APCI cpufreq) or operating in another valid " + "Power management environment\n", TEST_POWER_LCORE_ID); + rte_power_unset_env(); + return -1; + } + + /** + * test of initialising power management for the lcore which has + * been initialised + */ + ret = rte_power_init(TEST_POWER_LCORE_ID); + if (ret == 0) { + printf("Unexpectedly init successfully power twice on " + "lcore %u\n", TEST_POWER_LCORE_ID); + goto fail_all; + } + + ret = check_power_freqs(); + if (ret < 0) + goto fail_all; + + if (total_freq_num < 2) { + rte_power_exit(TEST_POWER_LCORE_ID); + printf("Frequency can not be changed due to CPU itself\n"); + rte_power_unset_env(); + return 0; + } + + ret = check_power_get_freq(); + if (ret < 0) + goto fail_all; + + ret = check_power_set_freq(); + if (ret < 0) + goto fail_all; + + ret = check_power_freq_down(); + if (ret < 0) + goto fail_all; + + ret = check_power_freq_up(); + if (ret < 0) + goto fail_all; + + ret = check_power_freq_max(); + if (ret < 0) + goto fail_all; + + ret = check_power_freq_min(); + if (ret < 0) + goto fail_all; + + ret = rte_power_exit(TEST_POWER_LCORE_ID); + if (ret < 0) { + printf("Cannot exit power management for lcore %u\n", + TEST_POWER_LCORE_ID); + rte_power_unset_env(); + return -1; + } + + /** + * test of exiting power management for the lcore which has been exited + */ + ret = rte_power_exit(TEST_POWER_LCORE_ID); + if (ret == 0) { + printf("Unexpectedly exit successfully power management twice " + "on lcore %u\n", TEST_POWER_LCORE_ID); + rte_power_unset_env(); + return -1; + } + + /* test of exit power management for an invalid lcore */ + ret = rte_power_exit(TEST_POWER_LCORE_INVALID); + if (ret == 0) { + printf("Unpectedly exit power management successfully for " + "lcore %u\n", TEST_POWER_LCORE_INVALID); + rte_power_unset_env(); + return -1; + } + rte_power_unset_env(); + return 0; + +fail_all: + rte_power_exit(TEST_POWER_LCORE_ID); + rte_power_unset_env(); + return -1; +} + +REGISTER_TEST_COMMAND(power_acpi_cpufreq_autotest, test_power_acpi_cpufreq); diff --git a/dpdk/test/test/test_power_kvm_vm.c b/dpdk/test/test/test_power_kvm_vm.c new file mode 100644 index 00000000..253a5f8b --- /dev/null +++ b/dpdk/test/test/test_power_kvm_vm.c @@ -0,0 +1,303 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <stdint.h> +#include <unistd.h> +#include <limits.h> +#include <string.h> + +#include "test.h" + +#include <rte_power.h> + +#define TEST_POWER_VM_LCORE_ID 0U +#define TEST_POWER_VM_LCORE_OUT_OF_BOUNDS (RTE_MAX_LCORE+1) +#define TEST_POWER_VM_LCORE_INVALID 1U + +static int +test_power_kvm_vm(void) +{ + int ret; + enum power_management_env env; + + ret = rte_power_set_env(PM_ENV_KVM_VM); + if (ret != 0) { + printf("Failed on setting environment to PM_ENV_KVM_VM\n"); + return -1; + } + + /* Test environment configuration */ + env = rte_power_get_env(); + if (env != PM_ENV_KVM_VM) { + printf("Unexpectedly got a Power Management environment other than " + "KVM VM\n"); + rte_power_unset_env(); + return -1; + } + + /* verify that function pointers are not NULL */ + if (rte_power_freqs == NULL) { + printf("rte_power_freqs should not be NULL, environment has not been " + "initialised\n"); + return -1; + } + if (rte_power_get_freq == NULL) { + printf("rte_power_get_freq should not be NULL, environment has not " + "been initialised\n"); + return -1; + } + if (rte_power_set_freq == NULL) { + printf("rte_power_set_freq should not be NULL, environment has not " + "been initialised\n"); + return -1; + } + if (rte_power_freq_up == NULL) { + printf("rte_power_freq_up should not be NULL, environment has not " + "been initialised\n"); + return -1; + } + if (rte_power_freq_down == NULL) { + printf("rte_power_freq_down should not be NULL, environment has not " + "been initialised\n"); + return -1; + } + if (rte_power_freq_max == NULL) { + printf("rte_power_freq_max should not be NULL, environment has not " + "been initialised\n"); + return -1; + } + if (rte_power_freq_min == NULL) { + printf("rte_power_freq_min should not be NULL, environment has not " + "been initialised\n"); + return -1; + } + /* Test initialisation of an out of bounds lcore */ + ret = rte_power_init(TEST_POWER_VM_LCORE_OUT_OF_BOUNDS); + if (ret != -1) { + printf("rte_power_init unexpectedly succeeded on an invalid lcore %u\n", + TEST_POWER_VM_LCORE_OUT_OF_BOUNDS); + rte_power_unset_env(); + return -1; + } + + /* Test initialisation of a valid lcore */ + ret = rte_power_init(TEST_POWER_VM_LCORE_ID); + if (ret < 0) { + printf("Cannot initialise power management for lcore %u, this " + "may occur if environment is not configured " + "correctly(KVM VM) or operating in another valid " + "Power management environment\n", TEST_POWER_VM_LCORE_ID); + rte_power_unset_env(); + return -1; + } + + /* Test initialisation of previously initialised lcore */ + ret = rte_power_init(TEST_POWER_VM_LCORE_ID); + if (ret == 0) { + printf("rte_power_init unexpectedly succeeded on calling init twice on" + " lcore %u\n", TEST_POWER_VM_LCORE_ID); + goto fail_all; + } + + /* Test frequency up of invalid lcore */ + ret = rte_power_freq_up(TEST_POWER_VM_LCORE_OUT_OF_BOUNDS); + if (ret == 1) { + printf("rte_power_freq_up unexpectedly succeeded on invalid lcore %u\n", + TEST_POWER_VM_LCORE_OUT_OF_BOUNDS); + goto fail_all; + } + + /* Test frequency down of invalid lcore */ + ret = rte_power_freq_down(TEST_POWER_VM_LCORE_OUT_OF_BOUNDS); + if (ret == 1) { + printf("rte_power_freq_down unexpectedly succeeded on invalid lcore " + "%u\n", TEST_POWER_VM_LCORE_OUT_OF_BOUNDS); + goto fail_all; + } + + /* Test frequency min of invalid lcore */ + ret = rte_power_freq_min(TEST_POWER_VM_LCORE_OUT_OF_BOUNDS); + if (ret == 1) { + printf("rte_power_freq_min unexpectedly succeeded on invalid lcore " + "%u\n", TEST_POWER_VM_LCORE_OUT_OF_BOUNDS); + goto fail_all; + } + + /* Test frequency max of invalid lcore */ + ret = rte_power_freq_max(TEST_POWER_VM_LCORE_OUT_OF_BOUNDS); + if (ret == 1) { + printf("rte_power_freq_max unexpectedly succeeded on invalid lcore " + "%u\n", TEST_POWER_VM_LCORE_OUT_OF_BOUNDS); + goto fail_all; + } + + /* Test frequency up of valid but uninitialised lcore */ + ret = rte_power_freq_up(TEST_POWER_VM_LCORE_INVALID); + if (ret == 1) { + printf("rte_power_freq_up unexpectedly succeeded on invalid lcore %u\n", + TEST_POWER_VM_LCORE_INVALID); + goto fail_all; + } + + /* Test frequency down of valid but uninitialised lcore */ + ret = rte_power_freq_down(TEST_POWER_VM_LCORE_INVALID); + if (ret == 1) { + printf("rte_power_freq_down unexpectedly succeeded on invalid lcore " + "%u\n", TEST_POWER_VM_LCORE_INVALID); + goto fail_all; + } + + /* Test frequency min of valid but uninitialised lcore */ + ret = rte_power_freq_min(TEST_POWER_VM_LCORE_INVALID); + if (ret == 1) { + printf("rte_power_freq_min unexpectedly succeeded on invalid lcore " + "%u\n", TEST_POWER_VM_LCORE_INVALID); + goto fail_all; + } + + /* Test frequency max of valid but uninitialised lcore */ + ret = rte_power_freq_max(TEST_POWER_VM_LCORE_INVALID); + if (ret == 1) { + printf("rte_power_freq_max unexpectedly succeeded on invalid lcore " + "%u\n", TEST_POWER_VM_LCORE_INVALID); + goto fail_all; + } + + /* Test frequency up of valid lcore */ + ret = rte_power_freq_up(TEST_POWER_VM_LCORE_ID); + if (ret != 1) { + printf("rte_power_freq_up unexpectedly failed on valid lcore %u\n", + TEST_POWER_VM_LCORE_ID); + goto fail_all; + } + + /* Test frequency down of valid lcore */ + ret = rte_power_freq_down(TEST_POWER_VM_LCORE_ID); + if (ret != 1) { + printf("rte_power_freq_down unexpectedly failed on valid lcore " + "%u\n", TEST_POWER_VM_LCORE_ID); + goto fail_all; + } + + /* Test frequency min of valid lcore */ + ret = rte_power_freq_min(TEST_POWER_VM_LCORE_ID); + if (ret != 1) { + printf("rte_power_freq_min unexpectedly failed on valid lcore " + "%u\n", TEST_POWER_VM_LCORE_ID); + goto fail_all; + } + + /* Test frequency max of valid lcore */ + ret = rte_power_freq_max(TEST_POWER_VM_LCORE_ID); + if (ret != 1) { + printf("rte_power_freq_max unexpectedly failed on valid lcore " + "%u\n", TEST_POWER_VM_LCORE_ID); + goto fail_all; + } + + /* Test unsupported rte_power_freqs */ + ret = rte_power_freqs(TEST_POWER_VM_LCORE_ID, NULL, 0); + if (ret != -ENOTSUP) { + printf("rte_power_freqs did not return the expected -ENOTSUP(%d) but " + "returned %d\n", -ENOTSUP, ret); + goto fail_all; + } + + /* Test unsupported rte_power_get_freq */ + ret = rte_power_get_freq(TEST_POWER_VM_LCORE_ID); + if (ret != -ENOTSUP) { + printf("rte_power_get_freq did not return the expected -ENOTSUP(%d) but" + " returned %d for lcore %u\n", + -ENOTSUP, ret, TEST_POWER_VM_LCORE_ID); + goto fail_all; + } + + /* Test unsupported rte_power_set_freq */ + ret = rte_power_set_freq(TEST_POWER_VM_LCORE_ID, 0); + if (ret != -ENOTSUP) { + printf("rte_power_set_freq did not return the expected -ENOTSUP(%d) but" + " returned %d for lcore %u\n", + -ENOTSUP, ret, TEST_POWER_VM_LCORE_ID); + goto fail_all; + } + + /* Test removing of an lcore */ + ret = rte_power_exit(TEST_POWER_VM_LCORE_ID); + if (ret != 0) { + printf("rte_power_exit unexpectedly failed on valid lcore %u," + "please ensure that the environment has been configured " + "correctly\n", TEST_POWER_VM_LCORE_ID); + goto fail_all; + } + + /* Test frequency up of previously removed lcore */ + ret = rte_power_freq_up(TEST_POWER_VM_LCORE_ID); + if (ret == 0) { + printf("rte_power_freq_up unexpectedly succeeded on a removed " + "lcore %u\n", TEST_POWER_VM_LCORE_ID); + return -1; + } + + /* Test frequency down of previously removed lcore */ + ret = rte_power_freq_down(TEST_POWER_VM_LCORE_ID); + if (ret == 0) { + printf("rte_power_freq_down unexpectedly succeeded on a removed " + "lcore %u\n", TEST_POWER_VM_LCORE_ID); + return -1; + } + + /* Test frequency min of previously removed lcore */ + ret = rte_power_freq_min(TEST_POWER_VM_LCORE_ID); + if (ret == 0) { + printf("rte_power_freq_min unexpectedly succeeded on a removed " + "lcore %u\n", TEST_POWER_VM_LCORE_ID); + return -1; + } + + /* Test frequency max of previously removed lcore */ + ret = rte_power_freq_max(TEST_POWER_VM_LCORE_ID); + if (ret == 0) { + printf("rte_power_freq_max unexpectedly succeeded on a removed " + "lcore %u\n", TEST_POWER_VM_LCORE_ID); + return -1; + } + rte_power_unset_env(); + return 0; +fail_all: + rte_power_exit(TEST_POWER_VM_LCORE_ID); + rte_power_unset_env(); + return -1; +} + +REGISTER_TEST_COMMAND(power_kvm_vm_autotest, test_power_kvm_vm); diff --git a/dpdk/test/test/test_prefetch.c b/dpdk/test/test/test_prefetch.c new file mode 100644 index 00000000..80afaaf3 --- /dev/null +++ b/dpdk/test/test/test_prefetch.c @@ -0,0 +1,61 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <stdint.h> + +#include <rte_prefetch.h> + +#include "test.h" + +/* + * Prefetch test + * ============= + * + * - Just test that the macro can be called and validate the compilation. + * The test always return success. + */ + +static int +test_prefetch(void) +{ + int a; + + rte_prefetch0(&a); + rte_prefetch1(&a); + rte_prefetch2(&a); + + return 0; +} + +REGISTER_TEST_COMMAND(prefetch_autotest, test_prefetch); diff --git a/dpdk/test/test/test_red.c b/dpdk/test/test/test_red.c new file mode 100644 index 00000000..348075dc --- /dev/null +++ b/dpdk/test/test/test_red.c @@ -0,0 +1,1885 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdlib.h> +#include <stdio.h> +#include <string.h> +#include <stdint.h> +#include <unistd.h> +#include <inttypes.h> +#include <sys/time.h> +#include <time.h> +#include <math.h> + +#include "test.h" + +#include <rte_red.h> + +#ifdef __INTEL_COMPILER +#pragma warning(disable:2259) /* conversion may lose significant bits */ +#pragma warning(disable:181) /* Arg incompatible with format string */ +#endif + +#define TEST_HZ_PER_KHZ 1000 +#define TEST_NSEC_MARGIN 500 /**< nanosecond margin when calculating clk freq */ + +#define MAX_QEMPTY_TIME_MSEC 50000 +#define MSEC_PER_SEC 1000 /**< Milli-seconds per second */ +#define USEC_PER_MSEC 1000 /**< Micro-seconds per milli-second */ +#define USEC_PER_SEC 1000000 /**< Micro-seconds per second */ +#define NSEC_PER_SEC (USEC_PER_SEC * 1000) /**< Nano-seconds per second */ + +/**< structures for testing rte_red performance and function */ +struct test_rte_red_config { /**< Test structure for RTE_RED config */ + struct rte_red_config *rconfig; /**< RTE_RED configuration parameters */ + uint8_t num_cfg; /**< Number of RTE_RED configs to test */ + uint8_t *wq_log2; /**< Test wq_log2 value to use */ + uint32_t min_th; /**< Queue minimum threshold */ + uint32_t max_th; /**< Queue maximum threshold */ + uint8_t *maxp_inv; /**< Inverse mark probability */ +}; + +struct test_queue { /**< Test structure for RTE_RED Queues */ + struct rte_red *rdata; /**< RTE_RED runtime data */ + uint32_t num_queues; /**< Number of RTE_RED queues to test */ + uint32_t *qconfig; /**< Configuration of RTE_RED queues for test */ + uint32_t *q; /**< Queue size */ + uint32_t q_ramp_up; /**< Num of enqueues to ramp up the queue */ + uint32_t avg_ramp_up; /**< Average num of enqueues to ramp up the queue */ + uint32_t avg_tolerance; /**< Tolerance in queue average */ + double drop_tolerance; /**< Drop tolerance of packets not enqueued */ +}; + +struct test_var { /**< Test variables used for testing RTE_RED */ + uint32_t wait_usec; /**< Micro second wait interval */ + uint32_t num_iterations; /**< Number of test iterations */ + uint32_t num_ops; /**< Number of test operations */ + uint64_t clk_freq; /**< CPU clock frequency */ + uint32_t sleep_sec; /**< Seconds to sleep */ + uint32_t *dropped; /**< Test operations dropped */ + uint32_t *enqueued; /**< Test operations enqueued */ +}; + +struct test_config { /**< Master test structure for RTE_RED */ + const char *ifname; /**< Interface name */ + const char *msg; /**< Test message for display */ + const char *htxt; /**< Header txt display for result output */ + struct test_rte_red_config *tconfig; /**< Test structure for RTE_RED config */ + struct test_queue *tqueue; /**< Test structure for RTE_RED Queues */ + struct test_var *tvar; /**< Test variables used for testing RTE_RED */ + uint32_t *tlevel; /**< Queue levels */ +}; + +enum test_result { + FAIL = 0, + PASS +}; + +/**< Test structure to define tests to run */ +struct tests { + struct test_config *testcfg; + enum test_result (*testfn)(struct test_config *); +}; + +struct rdtsc_prof { + uint64_t clk_start; + uint64_t clk_min; /**< min clocks */ + uint64_t clk_max; /**< max clocks */ + uint64_t clk_avgc; /**< count to calc average */ + double clk_avg; /**< cumulative sum to calc average */ + const char *name; +}; + +static const uint64_t port_speed_bytes = (10ULL*1000ULL*1000ULL*1000ULL)/8ULL; +static double inv_cycles_per_byte = 0; +static double pkt_time_usec = 0; + +static void init_port_ts(uint64_t cpu_clock) +{ + double cycles_per_byte = (double)(cpu_clock) / (double)(port_speed_bytes); + inv_cycles_per_byte = 1.0 / cycles_per_byte; + pkt_time_usec = 1000000.0 / ((double)port_speed_bytes / (double)RTE_RED_S); +} + +static uint64_t get_port_ts(void) +{ + return (uint64_t)((double)rte_rdtsc() * inv_cycles_per_byte); +} + +static void rdtsc_prof_init(struct rdtsc_prof *p, const char *name) +{ + p->clk_min = (uint64_t)(-1LL); + p->clk_max = 0; + p->clk_avg = 0; + p->clk_avgc = 0; + p->name = name; +} + +static inline void rdtsc_prof_start(struct rdtsc_prof *p) +{ + p->clk_start = rte_rdtsc_precise(); +} + +static inline void rdtsc_prof_end(struct rdtsc_prof *p) +{ + uint64_t clk_start = rte_rdtsc() - p->clk_start; + + p->clk_avgc++; + p->clk_avg += (double) clk_start; + + if (clk_start > p->clk_max) + p->clk_max = clk_start; + if (clk_start < p->clk_min) + p->clk_min = clk_start; +} + +static void rdtsc_prof_print(struct rdtsc_prof *p) +{ + if (p->clk_avgc>0) { + printf("RDTSC stats for %s: n=%" PRIu64 ", min=%" PRIu64 ", max=%" PRIu64 ", avg=%.1f\n", + p->name, + p->clk_avgc, + p->clk_min, + p->clk_max, + (p->clk_avg / ((double) p->clk_avgc))); + } +} + +static uint32_t rte_red_get_avg_int(const struct rte_red_config *red_cfg, + struct rte_red *red) +{ + /** + * scale by 1/n and convert from fixed-point to integer + */ + return red->avg >> (RTE_RED_SCALING + red_cfg->wq_log2); +} + +static double rte_red_get_avg_float(const struct rte_red_config *red_cfg, + struct rte_red *red) +{ + /** + * scale by 1/n and convert from fixed-point to floating-point + */ + return ldexp((double)red->avg, -(RTE_RED_SCALING + red_cfg->wq_log2)); +} + +static void rte_red_set_avg_int(const struct rte_red_config *red_cfg, + struct rte_red *red, + uint32_t avg) +{ + /** + * scale by n and convert from integer to fixed-point + */ + red->avg = avg << (RTE_RED_SCALING + red_cfg->wq_log2); +} + +static double calc_exp_avg_on_empty(double avg, uint32_t n, uint32_t time_diff) +{ + return avg * pow((1.0 - 1.0 / (double)n), (double)time_diff / pkt_time_usec); +} + +static double calc_drop_rate(uint32_t enqueued, uint32_t dropped) +{ + return (double)dropped / ((double)enqueued + (double)dropped); +} + +/** + * calculate the drop probability + */ +static double calc_drop_prob(uint32_t min_th, uint32_t max_th, + uint32_t maxp_inv, uint32_t avg) +{ + double drop_prob = 0.0; + + if (avg < min_th) { + drop_prob = 0.0; + } else if (avg < max_th) { + drop_prob = (1.0 / (double)maxp_inv) + * ((double)(avg - min_th) + / (double)(max_th - min_th)); + } else { + drop_prob = 1.0; + } + return drop_prob; +} + +/** + * check if drop rate matches drop probability within tolerance + */ +static int check_drop_rate(double *diff, double drop_rate, double drop_prob, double tolerance) +{ + double abs_diff = 0.0; + int ret = 1; + + abs_diff = fabs(drop_rate - drop_prob); + if ((int)abs_diff == 0) { + *diff = 0.0; + } else { + *diff = (abs_diff / drop_prob) * 100.0; + if (*diff > tolerance) { + ret = 0; + } + } + return ret; +} + +/** + * check if average queue size is within tolerance + */ +static int check_avg(double *diff, double avg, double exp_avg, double tolerance) +{ + double abs_diff = 0.0; + int ret = 1; + + abs_diff = fabs(avg - exp_avg); + if ((int)abs_diff == 0) { + *diff = 0.0; + } else { + *diff = (abs_diff / exp_avg) * 100.0; + if (*diff > tolerance) { + ret = 0; + } + } + return ret; +} + +/** + * initialize the test rte_red config + */ +static enum test_result +test_rte_red_init(struct test_config *tcfg) +{ + unsigned i = 0; + + tcfg->tvar->clk_freq = rte_get_timer_hz(); + init_port_ts( tcfg->tvar->clk_freq ); + + for (i = 0; i < tcfg->tconfig->num_cfg; i++) { + if (rte_red_config_init(&tcfg->tconfig->rconfig[i], + (uint16_t)tcfg->tconfig->wq_log2[i], + (uint16_t)tcfg->tconfig->min_th, + (uint16_t)tcfg->tconfig->max_th, + (uint16_t)tcfg->tconfig->maxp_inv[i]) != 0) { + return FAIL; + } + } + + *tcfg->tqueue->q = 0; + *tcfg->tvar->dropped = 0; + *tcfg->tvar->enqueued = 0; + return PASS; +} + +/** + * enqueue until actual queue size reaches target level + */ +static int +increase_actual_qsize(struct rte_red_config *red_cfg, + struct rte_red *red, + uint32_t *q, + uint32_t level, + uint32_t attempts) +{ + uint32_t i = 0; + + for (i = 0; i < attempts; i++) { + int ret = 0; + + /** + * enqueue + */ + ret = rte_red_enqueue(red_cfg, red, *q, get_port_ts() ); + if (ret == 0) { + if (++(*q) >= level) + break; + } + } + /** + * check if target actual queue size has been reached + */ + if (*q != level) + return -1; + /** + * success + */ + return 0; +} + +/** + * enqueue until average queue size reaches target level + */ +static int +increase_average_qsize(struct rte_red_config *red_cfg, + struct rte_red *red, + uint32_t *q, + uint32_t level, + uint32_t num_ops) +{ + uint32_t avg = 0; + uint32_t i = 0; + + for (i = 0; i < num_ops; i++) { + /** + * enqueue + */ + rte_red_enqueue(red_cfg, red, *q, get_port_ts()); + } + /** + * check if target average queue size has been reached + */ + avg = rte_red_get_avg_int(red_cfg, red); + if (avg != level) + return -1; + /** + * success + */ + return 0; +} + +/** + * setup default values for the functional test structures + */ +static struct rte_red_config ft_wrconfig[1]; +static struct rte_red ft_rtdata[1]; +static uint8_t ft_wq_log2[] = {9}; +static uint8_t ft_maxp_inv[] = {10}; +static uint32_t ft_qconfig[] = {0, 0, 1, 1}; +static uint32_t ft_q[] ={0}; +static uint32_t ft_dropped[] ={0}; +static uint32_t ft_enqueued[] ={0}; + +static struct test_rte_red_config ft_tconfig = { + .rconfig = ft_wrconfig, + .num_cfg = RTE_DIM(ft_wrconfig), + .wq_log2 = ft_wq_log2, + .min_th = 32, + .max_th = 128, + .maxp_inv = ft_maxp_inv, +}; + +static struct test_queue ft_tqueue = { + .rdata = ft_rtdata, + .num_queues = RTE_DIM(ft_rtdata), + .qconfig = ft_qconfig, + .q = ft_q, + .q_ramp_up = 1000000, + .avg_ramp_up = 1000000, + .avg_tolerance = 5, /* 5 percent */ + .drop_tolerance = 50, /* 50 percent */ +}; + +static struct test_var ft_tvar = { + .wait_usec = 10000, + .num_iterations = 5, + .num_ops = 10000, + .clk_freq = 0, + .dropped = ft_dropped, + .enqueued = ft_enqueued, + .sleep_sec = (MAX_QEMPTY_TIME_MSEC / MSEC_PER_SEC) + 2, +}; + +/** + * functional test enqueue/dequeue packets + */ +static void enqueue_dequeue_func(struct rte_red_config *red_cfg, + struct rte_red *red, + uint32_t *q, + uint32_t num_ops, + uint32_t *enqueued, + uint32_t *dropped) +{ + uint32_t i = 0; + + for (i = 0; i < num_ops; i++) { + int ret = 0; + + /** + * enqueue + */ + ret = rte_red_enqueue(red_cfg, red, *q, get_port_ts()); + if (ret == 0) + (*enqueued)++; + else + (*dropped)++; + } +} + +/** + * Test F1: functional test 1 + */ +static uint32_t ft1_tlevels[] = {6, 12, 18, 24, 30, 36, 42, 48, 54, 60, 66, 72, 78, 84, 90, 96, 102, 108, 114, 120, 126, 132, 138, 144}; + +static struct test_config func_test1_config = { + .ifname = "functional test 1 interface", + .msg = "functional test 1 : use one rte_red configuration,\n" + " increase average queue size to various levels,\n" + " compare drop rate to drop probability\n\n", + .htxt = " " + "avg queue size " + "enqueued " + "dropped " + "drop prob % " + "drop rate % " + "diff % " + "tolerance % " + "\n", + .tconfig = &ft_tconfig, + .tqueue = &ft_tqueue, + .tvar = &ft_tvar, + .tlevel = ft1_tlevels, +}; + +static enum test_result func_test1(struct test_config *tcfg) +{ + enum test_result result = PASS; + uint32_t i = 0; + + printf("%s", tcfg->msg); + + if (test_rte_red_init(tcfg) != PASS) { + result = FAIL; + goto out; + } + + printf("%s", tcfg->htxt); + + for (i = 0; i < RTE_DIM(ft1_tlevels); i++) { + const char *label = NULL; + uint32_t avg = 0; + double drop_rate = 0.0; + double drop_prob = 0.0; + double diff = 0.0; + + /** + * reset rte_red run-time data + */ + rte_red_rt_data_init(tcfg->tqueue->rdata); + *tcfg->tvar->enqueued = 0; + *tcfg->tvar->dropped = 0; + + if (increase_actual_qsize(tcfg->tconfig->rconfig, + tcfg->tqueue->rdata, + tcfg->tqueue->q, + tcfg->tlevel[i], + tcfg->tqueue->q_ramp_up) != 0) { + result = FAIL; + goto out; + } + + if (increase_average_qsize(tcfg->tconfig->rconfig, + tcfg->tqueue->rdata, + tcfg->tqueue->q, + tcfg->tlevel[i], + tcfg->tqueue->avg_ramp_up) != 0) { + result = FAIL; + goto out; + } + + enqueue_dequeue_func(tcfg->tconfig->rconfig, + tcfg->tqueue->rdata, + tcfg->tqueue->q, + tcfg->tvar->num_ops, + tcfg->tvar->enqueued, + tcfg->tvar->dropped); + + avg = rte_red_get_avg_int(tcfg->tconfig->rconfig, tcfg->tqueue->rdata); + if (avg != tcfg->tlevel[i]) { + fprintf(stderr, "Fail: avg != level\n"); + result = FAIL; + } + + drop_rate = calc_drop_rate(*tcfg->tvar->enqueued, *tcfg->tvar->dropped); + drop_prob = calc_drop_prob(tcfg->tconfig->min_th, tcfg->tconfig->max_th, + *tcfg->tconfig->maxp_inv, tcfg->tlevel[i]); + if (!check_drop_rate(&diff, drop_rate, drop_prob, (double)tcfg->tqueue->drop_tolerance)) + result = FAIL; + + if (tcfg->tlevel[i] == tcfg->tconfig->min_th) + label = "min thresh: "; + else if (tcfg->tlevel[i] == tcfg->tconfig->max_th) + label = "max thresh: "; + else + label = " "; + printf("%s%-15u%-15u%-15u%-15.4lf%-15.4lf%-15.4lf%-15.4lf\n", + label, avg, *tcfg->tvar->enqueued, *tcfg->tvar->dropped, + drop_prob * 100.0, drop_rate * 100.0, diff, + (double)tcfg->tqueue->drop_tolerance); + } +out: + return result; +} + +/** + * Test F2: functional test 2 + */ +static uint32_t ft2_tlevel[] = {127}; +static uint8_t ft2_wq_log2[] = {9, 9, 9, 9, 9, 9, 9, 9, 9, 9}; +static uint8_t ft2_maxp_inv[] = {10, 20, 30, 40, 50, 60, 70, 80, 90, 100}; +static struct rte_red_config ft2_rconfig[10]; + +static struct test_rte_red_config ft2_tconfig = { + .rconfig = ft2_rconfig, + .num_cfg = RTE_DIM(ft2_rconfig), + .wq_log2 = ft2_wq_log2, + .min_th = 32, + .max_th = 128, + .maxp_inv = ft2_maxp_inv, +}; + +static struct test_config func_test2_config = { + .ifname = "functional test 2 interface", + .msg = "functional test 2 : use several RED configurations,\n" + " increase average queue size to just below maximum threshold,\n" + " compare drop rate to drop probability\n\n", + .htxt = "RED config " + "avg queue size " + "min threshold " + "max threshold " + "drop prob % " + "drop rate % " + "diff % " + "tolerance % " + "\n", + .tconfig = &ft2_tconfig, + .tqueue = &ft_tqueue, + .tvar = &ft_tvar, + .tlevel = ft2_tlevel, +}; + +static enum test_result func_test2(struct test_config *tcfg) +{ + enum test_result result = PASS; + double prev_drop_rate = 1.0; + uint32_t i = 0; + + printf("%s", tcfg->msg); + + if (test_rte_red_init(tcfg) != PASS) { + result = FAIL; + goto out; + } + rte_red_rt_data_init(tcfg->tqueue->rdata); + + if (increase_actual_qsize(tcfg->tconfig->rconfig, + tcfg->tqueue->rdata, + tcfg->tqueue->q, + *tcfg->tlevel, + tcfg->tqueue->q_ramp_up) != 0) { + result = FAIL; + goto out; + } + + if (increase_average_qsize(tcfg->tconfig->rconfig, + tcfg->tqueue->rdata, + tcfg->tqueue->q, + *tcfg->tlevel, + tcfg->tqueue->avg_ramp_up) != 0) { + result = FAIL; + goto out; + } + printf("%s", tcfg->htxt); + + for (i = 0; i < tcfg->tconfig->num_cfg; i++) { + uint32_t avg = 0; + double drop_rate = 0.0; + double drop_prob = 0.0; + double diff = 0.0; + + *tcfg->tvar->dropped = 0; + *tcfg->tvar->enqueued = 0; + + enqueue_dequeue_func(&tcfg->tconfig->rconfig[i], + tcfg->tqueue->rdata, + tcfg->tqueue->q, + tcfg->tvar->num_ops, + tcfg->tvar->enqueued, + tcfg->tvar->dropped); + + avg = rte_red_get_avg_int(&tcfg->tconfig->rconfig[i], tcfg->tqueue->rdata); + if (avg != *tcfg->tlevel) + result = FAIL; + + drop_rate = calc_drop_rate(*tcfg->tvar->enqueued, *tcfg->tvar->dropped); + drop_prob = calc_drop_prob(tcfg->tconfig->min_th, tcfg->tconfig->max_th, + tcfg->tconfig->maxp_inv[i], *tcfg->tlevel); + if (!check_drop_rate(&diff, drop_rate, drop_prob, (double)tcfg->tqueue->drop_tolerance)) + result = FAIL; + /** + * drop rate should decrease as maxp_inv increases + */ + if (drop_rate > prev_drop_rate) + result = FAIL; + prev_drop_rate = drop_rate; + + printf("%-15u%-15u%-15u%-15u%-15.4lf%-15.4lf%-15.4lf%-15.4lf\n", + i, avg, tcfg->tconfig->min_th, tcfg->tconfig->max_th, + drop_prob * 100.0, drop_rate * 100.0, diff, + (double)tcfg->tqueue->drop_tolerance); + } +out: + return result; +} + +/** + * Test F3: functional test 3 + */ +static uint32_t ft3_tlevel[] = {1022}; + +static struct test_rte_red_config ft3_tconfig = { + .rconfig = ft_wrconfig, + .num_cfg = RTE_DIM(ft_wrconfig), + .wq_log2 = ft_wq_log2, + .min_th = 32, + .max_th = 1023, + .maxp_inv = ft_maxp_inv, +}; + +static struct test_config func_test3_config = { + .ifname = "functional test 3 interface", + .msg = "functional test 3 : use one RED configuration,\n" + " increase average queue size to target level,\n" + " dequeue all packets until queue is empty,\n" + " confirm that average queue size is computed correctly while queue is empty\n\n", + .htxt = "q avg before " + "q avg after " + "expected " + "difference % " + "tolerance % " + "result " + "\n", + .tconfig = &ft3_tconfig, + .tqueue = &ft_tqueue, + .tvar = &ft_tvar, + .tlevel = ft3_tlevel, +}; + +static enum test_result func_test3(struct test_config *tcfg) +{ + enum test_result result = PASS; + uint32_t i = 0; + + printf("%s", tcfg->msg); + + if (test_rte_red_init(tcfg) != PASS) { + result = FAIL; + goto out; + } + + rte_red_rt_data_init(tcfg->tqueue->rdata); + + if (increase_actual_qsize(tcfg->tconfig->rconfig, + tcfg->tqueue->rdata, + tcfg->tqueue->q, + *tcfg->tlevel, + tcfg->tqueue->q_ramp_up) != 0) { + result = FAIL; + goto out; + } + + if (increase_average_qsize(tcfg->tconfig->rconfig, + tcfg->tqueue->rdata, + tcfg->tqueue->q, + *tcfg->tlevel, + tcfg->tqueue->avg_ramp_up) != 0) { + result = FAIL; + goto out; + } + + printf("%s", tcfg->htxt); + + for (i = 0; i < tcfg->tvar->num_iterations; i++) { + double avg_before = 0; + double avg_after = 0; + double exp_avg = 0; + double diff = 0.0; + + avg_before = rte_red_get_avg_float(tcfg->tconfig->rconfig, tcfg->tqueue->rdata); + + /** + * empty the queue + */ + *tcfg->tqueue->q = 0; + rte_red_mark_queue_empty(tcfg->tqueue->rdata, get_port_ts()); + + rte_delay_us(tcfg->tvar->wait_usec); + + /** + * enqueue one packet to recalculate average queue size + */ + if (rte_red_enqueue(tcfg->tconfig->rconfig, + tcfg->tqueue->rdata, + *tcfg->tqueue->q, + get_port_ts()) == 0) { + (*tcfg->tqueue->q)++; + } else { + printf("%s:%d: packet enqueued on empty queue was dropped\n", __func__, __LINE__); + result = FAIL; + } + + exp_avg = calc_exp_avg_on_empty(avg_before, + (1 << *tcfg->tconfig->wq_log2), + tcfg->tvar->wait_usec); + avg_after = rte_red_get_avg_float(tcfg->tconfig->rconfig, + tcfg->tqueue->rdata); + if (!check_avg(&diff, avg_after, exp_avg, (double)tcfg->tqueue->avg_tolerance)) + result = FAIL; + + printf("%-15.4lf%-15.4lf%-15.4lf%-15.4lf%-15.4lf%-15s\n", + avg_before, avg_after, exp_avg, diff, + (double)tcfg->tqueue->avg_tolerance, + diff <= (double)tcfg->tqueue->avg_tolerance ? "pass" : "fail"); + } +out: + return result; +} + +/** + * Test F4: functional test 4 + */ +static uint32_t ft4_tlevel[] = {1022}; +static uint8_t ft4_wq_log2[] = {11}; + +static struct test_rte_red_config ft4_tconfig = { + .rconfig = ft_wrconfig, + .num_cfg = RTE_DIM(ft_wrconfig), + .min_th = 32, + .max_th = 1023, + .wq_log2 = ft4_wq_log2, + .maxp_inv = ft_maxp_inv, +}; + +static struct test_queue ft4_tqueue = { + .rdata = ft_rtdata, + .num_queues = RTE_DIM(ft_rtdata), + .qconfig = ft_qconfig, + .q = ft_q, + .q_ramp_up = 1000000, + .avg_ramp_up = 1000000, + .avg_tolerance = 0, /* 0 percent */ + .drop_tolerance = 50, /* 50 percent */ +}; + +static struct test_config func_test4_config = { + .ifname = "functional test 4 interface", + .msg = "functional test 4 : use one RED configuration,\n" + " increase average queue size to target level,\n" + " dequeue all packets until queue is empty,\n" + " confirm that average queue size is computed correctly while\n" + " queue is empty for more than 50 sec,\n" + " (this test takes 52 sec to run)\n\n", + .htxt = "q avg before " + "q avg after " + "expected " + "difference % " + "tolerance % " + "result " + "\n", + .tconfig = &ft4_tconfig, + .tqueue = &ft4_tqueue, + .tvar = &ft_tvar, + .tlevel = ft4_tlevel, +}; + +static enum test_result func_test4(struct test_config *tcfg) +{ + enum test_result result = PASS; + uint64_t time_diff = 0; + uint64_t start = 0; + double avg_before = 0.0; + double avg_after = 0.0; + double exp_avg = 0.0; + double diff = 0.0; + + printf("%s", tcfg->msg); + + if (test_rte_red_init(tcfg) != PASS) { + result = FAIL; + goto out; + } + + rte_red_rt_data_init(tcfg->tqueue->rdata); + + if (increase_actual_qsize(tcfg->tconfig->rconfig, + tcfg->tqueue->rdata, + tcfg->tqueue->q, + *tcfg->tlevel, + tcfg->tqueue->q_ramp_up) != 0) { + result = FAIL; + goto out; + } + + if (increase_average_qsize(tcfg->tconfig->rconfig, + tcfg->tqueue->rdata, + tcfg->tqueue->q, + *tcfg->tlevel, + tcfg->tqueue->avg_ramp_up) != 0) { + result = FAIL; + goto out; + } + + printf("%s", tcfg->htxt); + + avg_before = rte_red_get_avg_float(tcfg->tconfig->rconfig, tcfg->tqueue->rdata); + + /** + * empty the queue + */ + *tcfg->tqueue->q = 0; + rte_red_mark_queue_empty(tcfg->tqueue->rdata, get_port_ts()); + + /** + * record empty time locally + */ + start = rte_rdtsc(); + + sleep(tcfg->tvar->sleep_sec); + + /** + * enqueue one packet to recalculate average queue size + */ + if (rte_red_enqueue(tcfg->tconfig->rconfig, + tcfg->tqueue->rdata, + *tcfg->tqueue->q, + get_port_ts()) != 0) { + result = FAIL; + goto out; + } + (*tcfg->tqueue->q)++; + + /** + * calculate how long queue has been empty + */ + time_diff = ((rte_rdtsc() - start) / tcfg->tvar->clk_freq) + * MSEC_PER_SEC; + if (time_diff < MAX_QEMPTY_TIME_MSEC) { + /** + * this could happen if sleep was interrupted for some reason + */ + result = FAIL; + goto out; + } + + /** + * confirm that average queue size is now at expected level + */ + exp_avg = 0.0; + avg_after = rte_red_get_avg_float(tcfg->tconfig->rconfig, tcfg->tqueue->rdata); + if (!check_avg(&diff, avg_after, exp_avg, (double)tcfg->tqueue->avg_tolerance)) + result = FAIL; + + printf("%-15.4lf%-15.4lf%-15.4lf%-15.4lf%-15.4lf%-15s\n", + avg_before, avg_after, exp_avg, + diff, (double)tcfg->tqueue->avg_tolerance, + diff <= (double)tcfg->tqueue->avg_tolerance ? "pass" : "fail"); +out: + return result; +} + +/** + * Test F5: functional test 5 + */ +static uint32_t ft5_tlevel[] = {127}; +static uint8_t ft5_wq_log2[] = {9, 8}; +static uint8_t ft5_maxp_inv[] = {10, 20}; +static struct rte_red_config ft5_config[2]; +static struct rte_red ft5_data[4]; +static uint32_t ft5_q[4]; +static uint32_t ft5_dropped[] = {0, 0, 0, 0}; +static uint32_t ft5_enqueued[] = {0, 0, 0, 0}; + +static struct test_rte_red_config ft5_tconfig = { + .rconfig = ft5_config, + .num_cfg = RTE_DIM(ft5_config), + .min_th = 32, + .max_th = 128, + .wq_log2 = ft5_wq_log2, + .maxp_inv = ft5_maxp_inv, +}; + +static struct test_queue ft5_tqueue = { + .rdata = ft5_data, + .num_queues = RTE_DIM(ft5_data), + .qconfig = ft_qconfig, + .q = ft5_q, + .q_ramp_up = 1000000, + .avg_ramp_up = 1000000, + .avg_tolerance = 5, /* 10 percent */ + .drop_tolerance = 50, /* 50 percent */ +}; + +struct test_var ft5_tvar = { + .wait_usec = 0, + .num_iterations = 15, + .num_ops = 10000, + .clk_freq = 0, + .dropped = ft5_dropped, + .enqueued = ft5_enqueued, + .sleep_sec = 0, +}; + +static struct test_config func_test5_config = { + .ifname = "functional test 5 interface", + .msg = "functional test 5 : use several queues (each with its own run-time data),\n" + " use several RED configurations (such that each configuration is shared by multiple queues),\n" + " increase average queue size to just below maximum threshold,\n" + " compare drop rate to drop probability,\n" + " (this is a larger scale version of functional test 2)\n\n", + .htxt = "queue " + "config " + "avg queue size " + "min threshold " + "max threshold " + "drop prob % " + "drop rate % " + "diff % " + "tolerance % " + "\n", + .tconfig = &ft5_tconfig, + .tqueue = &ft5_tqueue, + .tvar = &ft5_tvar, + .tlevel = ft5_tlevel, +}; + +static enum test_result func_test5(struct test_config *tcfg) +{ + enum test_result result = PASS; + uint32_t j = 0; + + printf("%s", tcfg->msg); + + if (test_rte_red_init(tcfg) != PASS) { + result = FAIL; + goto out; + } + + printf("%s", tcfg->htxt); + + for (j = 0; j < tcfg->tqueue->num_queues; j++) { + rte_red_rt_data_init(&tcfg->tqueue->rdata[j]); + tcfg->tqueue->q[j] = 0; + + if (increase_actual_qsize(&tcfg->tconfig->rconfig[tcfg->tqueue->qconfig[j]], + &tcfg->tqueue->rdata[j], + &tcfg->tqueue->q[j], + *tcfg->tlevel, + tcfg->tqueue->q_ramp_up) != 0) { + result = FAIL; + goto out; + } + + if (increase_average_qsize(&tcfg->tconfig->rconfig[tcfg->tqueue->qconfig[j]], + &tcfg->tqueue->rdata[j], + &tcfg->tqueue->q[j], + *tcfg->tlevel, + tcfg->tqueue->avg_ramp_up) != 0) { + result = FAIL; + goto out; + } + } + + for (j = 0; j < tcfg->tqueue->num_queues; j++) { + uint32_t avg = 0; + double drop_rate = 0.0; + double drop_prob = 0.0; + double diff = 0.0; + + tcfg->tvar->dropped[j] = 0; + tcfg->tvar->enqueued[j] = 0; + + enqueue_dequeue_func(&tcfg->tconfig->rconfig[tcfg->tqueue->qconfig[j]], + &tcfg->tqueue->rdata[j], + &tcfg->tqueue->q[j], + tcfg->tvar->num_ops, + &tcfg->tvar->enqueued[j], + &tcfg->tvar->dropped[j]); + + avg = rte_red_get_avg_int(&tcfg->tconfig->rconfig[tcfg->tqueue->qconfig[j]], + &tcfg->tqueue->rdata[j]); + if (avg != *tcfg->tlevel) + result = FAIL; + + drop_rate = calc_drop_rate(tcfg->tvar->enqueued[j],tcfg->tvar->dropped[j]); + drop_prob = calc_drop_prob(tcfg->tconfig->min_th, tcfg->tconfig->max_th, + tcfg->tconfig->maxp_inv[tcfg->tqueue->qconfig[j]], + *tcfg->tlevel); + if (!check_drop_rate(&diff, drop_rate, drop_prob, (double)tcfg->tqueue->drop_tolerance)) + result = FAIL; + + printf("%-15u%-15u%-15u%-15u%-15u%-15.4lf%-15.4lf%-15.4lf%-15.4lf\n", + j, tcfg->tqueue->qconfig[j], avg, + tcfg->tconfig->min_th, tcfg->tconfig->max_th, + drop_prob * 100.0, drop_rate * 100.0, + diff, (double)tcfg->tqueue->drop_tolerance); + } +out: + return result; +} + +/** + * Test F6: functional test 6 + */ +static uint32_t ft6_tlevel[] = {1022}; +static uint8_t ft6_wq_log2[] = {9, 8}; +static uint8_t ft6_maxp_inv[] = {10, 20}; +static struct rte_red_config ft6_config[2]; +static struct rte_red ft6_data[4]; +static uint32_t ft6_q[4]; + +static struct test_rte_red_config ft6_tconfig = { + .rconfig = ft6_config, + .num_cfg = RTE_DIM(ft6_config), + .min_th = 32, + .max_th = 1023, + .wq_log2 = ft6_wq_log2, + .maxp_inv = ft6_maxp_inv, +}; + +static struct test_queue ft6_tqueue = { + .rdata = ft6_data, + .num_queues = RTE_DIM(ft6_data), + .qconfig = ft_qconfig, + .q = ft6_q, + .q_ramp_up = 1000000, + .avg_ramp_up = 1000000, + .avg_tolerance = 5, /* 10 percent */ + .drop_tolerance = 50, /* 50 percent */ +}; + +static struct test_config func_test6_config = { + .ifname = "functional test 6 interface", + .msg = "functional test 6 : use several queues (each with its own run-time data),\n" + " use several RED configurations (such that each configuration is sharte_red by multiple queues),\n" + " increase average queue size to target level,\n" + " dequeue all packets until queue is empty,\n" + " confirm that average queue size is computed correctly while queue is empty\n" + " (this is a larger scale version of functional test 3)\n\n", + .htxt = "queue " + "config " + "q avg before " + "q avg after " + "expected " + "difference % " + "tolerance % " + "result ""\n", + .tconfig = &ft6_tconfig, + .tqueue = &ft6_tqueue, + .tvar = &ft_tvar, + .tlevel = ft6_tlevel, +}; + +static enum test_result func_test6(struct test_config *tcfg) +{ + enum test_result result = PASS; + uint32_t j = 0; + + printf("%s", tcfg->msg); + if (test_rte_red_init(tcfg) != PASS) { + result = FAIL; + goto out; + } + printf("%s", tcfg->htxt); + + for (j = 0; j < tcfg->tqueue->num_queues; j++) { + rte_red_rt_data_init(&tcfg->tqueue->rdata[j]); + tcfg->tqueue->q[j] = 0; + + if (increase_actual_qsize(&tcfg->tconfig->rconfig[tcfg->tqueue->qconfig[j]], + &tcfg->tqueue->rdata[j], + &tcfg->tqueue->q[j], + *tcfg->tlevel, + tcfg->tqueue->q_ramp_up) != 0) { + result = FAIL; + goto out; + } + if (increase_average_qsize(&tcfg->tconfig->rconfig[tcfg->tqueue->qconfig[j]], + &tcfg->tqueue->rdata[j], + &tcfg->tqueue->q[j], + *tcfg->tlevel, + tcfg->tqueue->avg_ramp_up) != 0) { + result = FAIL; + goto out; + } + } + for (j = 0; j < tcfg->tqueue->num_queues; j++) { + double avg_before = 0; + double avg_after = 0; + double exp_avg = 0; + double diff = 0.0; + + avg_before = rte_red_get_avg_float(&tcfg->tconfig->rconfig[tcfg->tqueue->qconfig[j]], + &tcfg->tqueue->rdata[j]); + + /** + * empty the queue + */ + tcfg->tqueue->q[j] = 0; + rte_red_mark_queue_empty(&tcfg->tqueue->rdata[j], get_port_ts()); + rte_delay_us(tcfg->tvar->wait_usec); + + /** + * enqueue one packet to recalculate average queue size + */ + if (rte_red_enqueue(&tcfg->tconfig->rconfig[tcfg->tqueue->qconfig[j]], + &tcfg->tqueue->rdata[j], + tcfg->tqueue->q[j], + get_port_ts()) == 0) { + tcfg->tqueue->q[j]++; + } else { + printf("%s:%d: packet enqueued on empty queue was dropped\n", __func__, __LINE__); + result = FAIL; + } + + exp_avg = calc_exp_avg_on_empty(avg_before, + (1 << tcfg->tconfig->wq_log2[tcfg->tqueue->qconfig[j]]), + tcfg->tvar->wait_usec); + avg_after = rte_red_get_avg_float(&tcfg->tconfig->rconfig[tcfg->tqueue->qconfig[j]], + &tcfg->tqueue->rdata[j]); + if (!check_avg(&diff, avg_after, exp_avg, (double)tcfg->tqueue->avg_tolerance)) + result = FAIL; + + printf("%-15u%-15u%-15.4lf%-15.4lf%-15.4lf%-15.4lf%-15.4lf%-15s\n", + j, tcfg->tqueue->qconfig[j], avg_before, avg_after, + exp_avg, diff, (double)tcfg->tqueue->avg_tolerance, + diff <= tcfg->tqueue->avg_tolerance ? "pass" : "fail"); + } +out: + return result; +} + +/** + * setup default values for the performance test structures + */ +static struct rte_red_config pt_wrconfig[1]; +static struct rte_red pt_rtdata[1]; +static uint8_t pt_wq_log2[] = {9}; +static uint8_t pt_maxp_inv[] = {10}; +static uint32_t pt_qconfig[] = {0}; +static uint32_t pt_q[] = {0}; +static uint32_t pt_dropped[] = {0}; +static uint32_t pt_enqueued[] = {0}; + +static struct test_rte_red_config pt_tconfig = { + .rconfig = pt_wrconfig, + .num_cfg = RTE_DIM(pt_wrconfig), + .wq_log2 = pt_wq_log2, + .min_th = 32, + .max_th = 128, + .maxp_inv = pt_maxp_inv, +}; + +static struct test_queue pt_tqueue = { + .rdata = pt_rtdata, + .num_queues = RTE_DIM(pt_rtdata), + .qconfig = pt_qconfig, + .q = pt_q, + .q_ramp_up = 1000000, + .avg_ramp_up = 1000000, + .avg_tolerance = 5, /* 10 percent */ + .drop_tolerance = 50, /* 50 percent */ +}; + +/** + * enqueue/dequeue packets + */ +static void enqueue_dequeue_perf(struct rte_red_config *red_cfg, + struct rte_red *red, + uint32_t *q, + uint32_t num_ops, + uint32_t *enqueued, + uint32_t *dropped, + struct rdtsc_prof *prof) +{ + uint32_t i = 0; + + for (i = 0; i < num_ops; i++) { + uint64_t ts = 0; + int ret = 0; + /** + * enqueue + */ + ts = get_port_ts(); + rdtsc_prof_start(prof); + ret = rte_red_enqueue(red_cfg, red, *q, ts ); + rdtsc_prof_end(prof); + if (ret == 0) + (*enqueued)++; + else + (*dropped)++; + } +} + +/** + * Setup test structures for tests P1, P2, P3 + * performance tests 1, 2 and 3 + */ +static uint32_t pt1_tlevel[] = {16}; +static uint32_t pt2_tlevel[] = {80}; +static uint32_t pt3_tlevel[] = {144}; + +static struct test_var perf1_tvar = { + .wait_usec = 0, + .num_iterations = 15, + .num_ops = 50000000, + .clk_freq = 0, + .dropped = pt_dropped, + .enqueued = pt_enqueued, + .sleep_sec = 0 +}; + +static struct test_config perf1_test1_config = { + .ifname = "performance test 1 interface", + .msg = "performance test 1 : use one RED configuration,\n" + " set actual and average queue sizes to level below min threshold,\n" + " measure enqueue performance\n\n", + .tconfig = &pt_tconfig, + .tqueue = &pt_tqueue, + .tvar = &perf1_tvar, + .tlevel = pt1_tlevel, +}; + +static struct test_config perf1_test2_config = { + .ifname = "performance test 2 interface", + .msg = "performance test 2 : use one RED configuration,\n" + " set actual and average queue sizes to level in between min and max thresholds,\n" + " measure enqueue performance\n\n", + .tconfig = &pt_tconfig, + .tqueue = &pt_tqueue, + .tvar = &perf1_tvar, + .tlevel = pt2_tlevel, +}; + +static struct test_config perf1_test3_config = { + .ifname = "performance test 3 interface", + .msg = "performance test 3 : use one RED configuration,\n" + " set actual and average queue sizes to level above max threshold,\n" + " measure enqueue performance\n\n", + .tconfig = &pt_tconfig, + .tqueue = &pt_tqueue, + .tvar = &perf1_tvar, + .tlevel = pt3_tlevel, +}; + +/** + * Performance test function to measure enqueue performance. + * This runs performance tests 1, 2 and 3 + */ +static enum test_result perf1_test(struct test_config *tcfg) +{ + enum test_result result = PASS; + struct rdtsc_prof prof = {0, 0, 0, 0, 0.0, NULL}; + uint32_t total = 0; + + printf("%s", tcfg->msg); + + rdtsc_prof_init(&prof, "enqueue"); + + if (test_rte_red_init(tcfg) != PASS) { + result = FAIL; + goto out; + } + + /** + * set average queue size to target level + */ + *tcfg->tqueue->q = *tcfg->tlevel; + + /** + * initialize the rte_red run time data structure + */ + rte_red_rt_data_init(tcfg->tqueue->rdata); + + /** + * set the queue average + */ + rte_red_set_avg_int(tcfg->tconfig->rconfig, tcfg->tqueue->rdata, *tcfg->tlevel); + if (rte_red_get_avg_int(tcfg->tconfig->rconfig, tcfg->tqueue->rdata) + != *tcfg->tlevel) { + result = FAIL; + goto out; + } + + enqueue_dequeue_perf(tcfg->tconfig->rconfig, + tcfg->tqueue->rdata, + tcfg->tqueue->q, + tcfg->tvar->num_ops, + tcfg->tvar->enqueued, + tcfg->tvar->dropped, + &prof); + + total = *tcfg->tvar->enqueued + *tcfg->tvar->dropped; + + printf("\ntotal: %u, enqueued: %u (%.2lf%%), dropped: %u (%.2lf%%)\n", total, + *tcfg->tvar->enqueued, ((double)(*tcfg->tvar->enqueued) / (double)total) * 100.0, + *tcfg->tvar->dropped, ((double)(*tcfg->tvar->dropped) / (double)total) * 100.0); + + rdtsc_prof_print(&prof); +out: + return result; +} + +/** + * Setup test structures for tests P4, P5, P6 + * performance tests 4, 5 and 6 + */ +static uint32_t pt4_tlevel[] = {16}; +static uint32_t pt5_tlevel[] = {80}; +static uint32_t pt6_tlevel[] = {144}; + +static struct test_var perf2_tvar = { + .wait_usec = 500, + .num_iterations = 10000, + .num_ops = 10000, + .dropped = pt_dropped, + .enqueued = pt_enqueued, + .sleep_sec = 0 +}; + +static struct test_config perf2_test4_config = { + .ifname = "performance test 4 interface", + .msg = "performance test 4 : use one RED configuration,\n" + " set actual and average queue sizes to level below min threshold,\n" + " dequeue all packets until queue is empty,\n" + " measure enqueue performance when queue is empty\n\n", + .htxt = "iteration " + "q avg before " + "q avg after " + "expected " + "difference % " + "tolerance % " + "result ""\n", + .tconfig = &pt_tconfig, + .tqueue = &pt_tqueue, + .tvar = &perf2_tvar, + .tlevel = pt4_tlevel, +}; + +static struct test_config perf2_test5_config = { + .ifname = "performance test 5 interface", + .msg = "performance test 5 : use one RED configuration,\n" + " set actual and average queue sizes to level in between min and max thresholds,\n" + " dequeue all packets until queue is empty,\n" + " measure enqueue performance when queue is empty\n\n", + .htxt = "iteration " + "q avg before " + "q avg after " + "expected " + "difference " + "tolerance " + "result ""\n", + .tconfig = &pt_tconfig, + .tqueue = &pt_tqueue, + .tvar = &perf2_tvar, + .tlevel = pt5_tlevel, +}; + +static struct test_config perf2_test6_config = { + .ifname = "performance test 6 interface", + .msg = "performance test 6 : use one RED configuration,\n" + " set actual and average queue sizes to level above max threshold,\n" + " dequeue all packets until queue is empty,\n" + " measure enqueue performance when queue is empty\n\n", + .htxt = "iteration " + "q avg before " + "q avg after " + "expected " + "difference % " + "tolerance % " + "result ""\n", + .tconfig = &pt_tconfig, + .tqueue = &pt_tqueue, + .tvar = &perf2_tvar, + .tlevel = pt6_tlevel, +}; + +/** + * Performance test function to measure enqueue performance when the + * queue is empty. This runs performance tests 4, 5 and 6 + */ +static enum test_result perf2_test(struct test_config *tcfg) +{ + enum test_result result = PASS; + struct rdtsc_prof prof = {0, 0, 0, 0, 0.0, NULL}; + uint32_t total = 0; + uint32_t i = 0; + + printf("%s", tcfg->msg); + + rdtsc_prof_init(&prof, "enqueue"); + + if (test_rte_red_init(tcfg) != PASS) { + result = FAIL; + goto out; + } + + printf("%s", tcfg->htxt); + + for (i = 0; i < tcfg->tvar->num_iterations; i++) { + uint32_t count = 0; + uint64_t ts = 0; + double avg_before = 0; + int ret = 0; + + /** + * set average queue size to target level + */ + *tcfg->tqueue->q = *tcfg->tlevel; + count = (*tcfg->tqueue->rdata).count; + + /** + * initialize the rte_red run time data structure + */ + rte_red_rt_data_init(tcfg->tqueue->rdata); + (*tcfg->tqueue->rdata).count = count; + + /** + * set the queue average + */ + rte_red_set_avg_int(tcfg->tconfig->rconfig, tcfg->tqueue->rdata, *tcfg->tlevel); + avg_before = rte_red_get_avg_float(tcfg->tconfig->rconfig, tcfg->tqueue->rdata); + if ((avg_before < *tcfg->tlevel) || (avg_before > *tcfg->tlevel)) { + result = FAIL; + goto out; + } + + /** + * empty the queue + */ + *tcfg->tqueue->q = 0; + rte_red_mark_queue_empty(tcfg->tqueue->rdata, get_port_ts()); + + /** + * wait for specified period of time + */ + rte_delay_us(tcfg->tvar->wait_usec); + + /** + * measure performance of enqueue operation while queue is empty + */ + ts = get_port_ts(); + rdtsc_prof_start(&prof); + ret = rte_red_enqueue(tcfg->tconfig->rconfig, tcfg->tqueue->rdata, + *tcfg->tqueue->q, ts ); + rdtsc_prof_end(&prof); + + /** + * gather enqueued/dropped statistics + */ + if (ret == 0) + (*tcfg->tvar->enqueued)++; + else + (*tcfg->tvar->dropped)++; + + /** + * on first and last iteration, confirm that + * average queue size was computed correctly + */ + if ((i == 0) || (i == tcfg->tvar->num_iterations - 1)) { + double avg_after = 0; + double exp_avg = 0; + double diff = 0.0; + int ok = 0; + + avg_after = rte_red_get_avg_float(tcfg->tconfig->rconfig, tcfg->tqueue->rdata); + exp_avg = calc_exp_avg_on_empty(avg_before, + (1 << *tcfg->tconfig->wq_log2), + tcfg->tvar->wait_usec); + if (check_avg(&diff, avg_after, exp_avg, (double)tcfg->tqueue->avg_tolerance)) + ok = 1; + printf("%-15u%-15.4lf%-15.4lf%-15.4lf%-15.4lf%-15.4lf%-15s\n", + i, avg_before, avg_after, exp_avg, diff, + (double)tcfg->tqueue->avg_tolerance, ok ? "pass" : "fail"); + if (!ok) { + result = FAIL; + goto out; + } + } + } + total = *tcfg->tvar->enqueued + *tcfg->tvar->dropped; + printf("\ntotal: %u, enqueued: %u (%.2lf%%), dropped: %u (%.2lf%%)\n", total, + *tcfg->tvar->enqueued, ((double)(*tcfg->tvar->enqueued) / (double)total) * 100.0, + *tcfg->tvar->dropped, ((double)(*tcfg->tvar->dropped) / (double)total) * 100.0); + + rdtsc_prof_print(&prof); +out: + return result; +} + +/** + * setup default values for overflow test structures + */ +static uint32_t avg_max = 0; +static uint32_t avg_max_bits = 0; + +static struct rte_red_config ovfl_wrconfig[1]; +static struct rte_red ovfl_rtdata[1]; +static uint8_t ovfl_maxp_inv[] = {10}; +static uint32_t ovfl_qconfig[] = {0, 0, 1, 1}; +static uint32_t ovfl_q[] ={0}; +static uint32_t ovfl_dropped[] ={0}; +static uint32_t ovfl_enqueued[] ={0}; +static uint32_t ovfl_tlevel[] = {1023}; +static uint8_t ovfl_wq_log2[] = {12}; + +static struct test_rte_red_config ovfl_tconfig = { + .rconfig = ovfl_wrconfig, + .num_cfg = RTE_DIM(ovfl_wrconfig), + .wq_log2 = ovfl_wq_log2, + .min_th = 32, + .max_th = 1023, + .maxp_inv = ovfl_maxp_inv, +}; + +static struct test_queue ovfl_tqueue = { + .rdata = ovfl_rtdata, + .num_queues = RTE_DIM(ovfl_rtdata), + .qconfig = ovfl_qconfig, + .q = ovfl_q, + .q_ramp_up = 1000000, + .avg_ramp_up = 1000000, + .avg_tolerance = 5, /* 10 percent */ + .drop_tolerance = 50, /* 50 percent */ +}; + +static struct test_var ovfl_tvar = { + .wait_usec = 10000, + .num_iterations = 1, + .num_ops = 10000, + .clk_freq = 0, + .dropped = ovfl_dropped, + .enqueued = ovfl_enqueued, + .sleep_sec = 0 +}; + +static void ovfl_check_avg(uint32_t avg) +{ + if (avg > avg_max) { + double avg_log = 0; + uint32_t bits = 0; + avg_max = avg; + avg_log = log(((double)avg_max)); + avg_log = avg_log / log(2.0); + bits = (uint32_t)ceil(avg_log); + if (bits > avg_max_bits) + avg_max_bits = bits; + } +} + +static struct test_config ovfl_test1_config = { + .ifname = "queue avergage overflow test interface", + .msg = "overflow test 1 : use one RED configuration,\n" + " increase average queue size to target level,\n" + " check maximum number of bits requirte_red to represent avg_s\n\n", + .htxt = "avg queue size " + "wq_log2 " + "fraction bits " + "max queue avg " + "num bits " + "enqueued " + "dropped " + "drop prob % " + "drop rate % " + "\n", + .tconfig = &ovfl_tconfig, + .tqueue = &ovfl_tqueue, + .tvar = &ovfl_tvar, + .tlevel = ovfl_tlevel, +}; + +static enum test_result ovfl_test1(struct test_config *tcfg) +{ + enum test_result result = PASS; + uint32_t avg = 0; + uint32_t i = 0; + double drop_rate = 0.0; + double drop_prob = 0.0; + double diff = 0.0; + int ret = 0; + + printf("%s", tcfg->msg); + + if (test_rte_red_init(tcfg) != PASS) { + + result = FAIL; + goto out; + } + + /** + * reset rte_red run-time data + */ + rte_red_rt_data_init(tcfg->tqueue->rdata); + + /** + * increase actual queue size + */ + for (i = 0; i < tcfg->tqueue->q_ramp_up; i++) { + ret = rte_red_enqueue(tcfg->tconfig->rconfig, tcfg->tqueue->rdata, + *tcfg->tqueue->q, get_port_ts()); + + if (ret == 0) { + if (++(*tcfg->tqueue->q) >= *tcfg->tlevel) + break; + } + } + + /** + * enqueue + */ + for (i = 0; i < tcfg->tqueue->avg_ramp_up; i++) { + ret = rte_red_enqueue(tcfg->tconfig->rconfig, tcfg->tqueue->rdata, + *tcfg->tqueue->q, get_port_ts()); + ovfl_check_avg((*tcfg->tqueue->rdata).avg); + avg = rte_red_get_avg_int(tcfg->tconfig->rconfig, tcfg->tqueue->rdata); + if (avg == *tcfg->tlevel) { + if (ret == 0) + (*tcfg->tvar->enqueued)++; + else + (*tcfg->tvar->dropped)++; + } + } + + /** + * check if target average queue size has been reached + */ + avg = rte_red_get_avg_int(tcfg->tconfig->rconfig, tcfg->tqueue->rdata); + if (avg != *tcfg->tlevel) { + result = FAIL; + goto out; + } + + /** + * check drop rate against drop probability + */ + drop_rate = calc_drop_rate(*tcfg->tvar->enqueued, *tcfg->tvar->dropped); + drop_prob = calc_drop_prob(tcfg->tconfig->min_th, + tcfg->tconfig->max_th, + *tcfg->tconfig->maxp_inv, + *tcfg->tlevel); + if (!check_drop_rate(&diff, drop_rate, drop_prob, (double)tcfg->tqueue->drop_tolerance)) + result = FAIL; + + printf("%s", tcfg->htxt); + + printf("%-16u%-9u%-15u0x%08x %-10u%-10u%-10u%-13.2lf%-13.2lf\n", + avg, *tcfg->tconfig->wq_log2, RTE_RED_SCALING, + avg_max, avg_max_bits, + *tcfg->tvar->enqueued, *tcfg->tvar->dropped, + drop_prob * 100.0, drop_rate * 100.0); +out: + return result; +} + +/** + * define the functional and performance tests to be executed + */ +struct tests func_tests[] = { + { &func_test1_config, func_test1 }, + { &func_test2_config, func_test2 }, + { &func_test3_config, func_test3 }, + { &func_test4_config, func_test4 }, + { &func_test5_config, func_test5 }, + { &func_test6_config, func_test6 }, + { &ovfl_test1_config, ovfl_test1 }, +}; + +struct tests func_tests_quick[] = { + { &func_test1_config, func_test1 }, + { &func_test2_config, func_test2 }, + { &func_test3_config, func_test3 }, + /* no test 4 as it takes a lot of time */ + { &func_test5_config, func_test5 }, + { &func_test6_config, func_test6 }, + { &ovfl_test1_config, ovfl_test1 }, +}; + +struct tests perf_tests[] = { + { &perf1_test1_config, perf1_test }, + { &perf1_test2_config, perf1_test }, + { &perf1_test3_config, perf1_test }, + { &perf2_test4_config, perf2_test }, + { &perf2_test5_config, perf2_test }, + { &perf2_test6_config, perf2_test }, +}; + +/** + * function to execute the required_red tests + */ +static void run_tests(struct tests *test_type, uint32_t test_count, uint32_t *num_tests, uint32_t *num_pass) +{ + enum test_result result = PASS; + uint32_t i = 0; + + for (i = 0; i < test_count; i++) { + printf("\n--------------------------------------------------------------------------------\n"); + result = test_type[i].testfn(test_type[i].testcfg); + (*num_tests)++; + if (result == PASS) { + (*num_pass)++; + printf("-------------------------------------<pass>-------------------------------------\n"); + } else { + printf("-------------------------------------<fail>-------------------------------------\n"); + } + } + return; +} + +/** + * check if functions accept invalid parameters + * + * First, all functions will be called without initialized RED + * Then, all of them will be called with NULL/invalid parameters + * + * Some functions are not tested as they are performance-critical and thus + * don't do any parameter checking. + */ +static int +test_invalid_parameters(void) +{ + struct rte_red_config config; + + if (rte_red_rt_data_init(NULL) == 0) { + printf("rte_red_rt_data_init should have failed!\n"); + return -1; + } + + if (rte_red_config_init(NULL, 0, 0, 0, 0) == 0) { + printf("rte_red_config_init should have failed!\n"); + return -1; + } + + if (rte_red_rt_data_init(NULL) == 0) { + printf("rte_red_rt_data_init should have failed!\n"); + return -1; + } + + /* NULL config */ + if (rte_red_config_init(NULL, 0, 0, 0, 0) == 0) { + printf("%i: rte_red_config_init should have failed!\n", __LINE__); + return -1; + } + /* min_treshold == max_treshold */ + if (rte_red_config_init(&config, 0, 1, 1, 0) == 0) { + printf("%i: rte_red_config_init should have failed!\n", __LINE__); + return -1; + } + /* min_treshold > max_treshold */ + if (rte_red_config_init(&config, 0, 2, 1, 0) == 0) { + printf("%i: rte_red_config_init should have failed!\n", __LINE__); + return -1; + } + /* wq_log2 > RTE_RED_WQ_LOG2_MAX */ + if (rte_red_config_init(&config, + RTE_RED_WQ_LOG2_MAX + 1, 1, 2, 0) == 0) { + printf("%i: rte_red_config_init should have failed!\n", __LINE__); + return -1; + } + /* wq_log2 < RTE_RED_WQ_LOG2_MIN */ + if (rte_red_config_init(&config, + RTE_RED_WQ_LOG2_MIN - 1, 1, 2, 0) == 0) { + printf("%i: rte_red_config_init should have failed!\n", __LINE__); + return -1; + } + /* maxp_inv > RTE_RED_MAXP_INV_MAX */ + if (rte_red_config_init(&config, + RTE_RED_WQ_LOG2_MIN, 1, 2, RTE_RED_MAXP_INV_MAX + 1) == 0) { + printf("%i: rte_red_config_init should have failed!\n", __LINE__); + return -1; + } + /* maxp_inv < RTE_RED_MAXP_INV_MIN */ + if (rte_red_config_init(&config, + RTE_RED_WQ_LOG2_MIN, 1, 2, RTE_RED_MAXP_INV_MIN - 1) == 0) { + printf("%i: rte_red_config_init should have failed!\n", __LINE__); + return -1; + } + + return 0; +} + +static void +show_stats(const uint32_t num_tests, const uint32_t num_pass) +{ + if (num_pass == num_tests) + printf("[total: %u, pass: %u]\n", num_tests, num_pass); + else + printf("[total: %u, pass: %u, fail: %u]\n", num_tests, num_pass, + num_tests - num_pass); +} + +static int +tell_the_result(const uint32_t num_tests, const uint32_t num_pass) +{ + return (num_pass == num_tests) ? 0 : 1; +} + +static int +test_red(void) +{ + uint32_t num_tests = 0; + uint32_t num_pass = 0; + + if (test_invalid_parameters() < 0) + return -1; + run_tests(func_tests_quick, RTE_DIM(func_tests_quick), + &num_tests, &num_pass); + show_stats(num_tests, num_pass); + return tell_the_result(num_tests, num_pass); +} + +static int +test_red_perf(void) +{ + uint32_t num_tests = 0; + uint32_t num_pass = 0; + + run_tests(perf_tests, RTE_DIM(perf_tests), &num_tests, &num_pass); + show_stats(num_tests, num_pass); + return tell_the_result(num_tests, num_pass); +} + +static int +test_red_all(void) +{ + uint32_t num_tests = 0; + uint32_t num_pass = 0; + + if (test_invalid_parameters() < 0) + return -1; + + run_tests(func_tests, RTE_DIM(func_tests), &num_tests, &num_pass); + run_tests(perf_tests, RTE_DIM(perf_tests), &num_tests, &num_pass); + show_stats(num_tests, num_pass); + return tell_the_result(num_tests, num_pass); +} + +REGISTER_TEST_COMMAND(red_autotest, test_red); +REGISTER_TEST_COMMAND(red_perf, test_red_perf); +REGISTER_TEST_COMMAND(red_all, test_red_all); diff --git a/dpdk/test/test/test_reorder.c b/dpdk/test/test/test_reorder.c new file mode 100644 index 00000000..429f6eb4 --- /dev/null +++ b/dpdk/test/test/test_reorder.c @@ -0,0 +1,395 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <unistd.h> +#include <string.h> + +#include <rte_cycles.h> +#include <rte_errno.h> +#include <rte_mbuf.h> +#include <rte_reorder.h> +#include <rte_lcore.h> +#include <rte_malloc.h> + +#include "test.h" + +#define BURST 32 +#define REORDER_BUFFER_SIZE 16384 +#define NUM_MBUFS (2*REORDER_BUFFER_SIZE) +#define REORDER_BUFFER_SIZE_INVALID 2049 + +struct reorder_unittest_params { + struct rte_mempool *p; + struct rte_reorder_buffer *b; +}; + +static struct reorder_unittest_params default_params = { + .p = NULL, + .b = NULL +}; + +static struct reorder_unittest_params *test_params = &default_params; + +static int +test_reorder_create(void) +{ + struct rte_reorder_buffer *b = NULL; + + b = rte_reorder_create(NULL, rte_socket_id(), REORDER_BUFFER_SIZE); + TEST_ASSERT((b == NULL) && (rte_errno == EINVAL), + "No error on create() with NULL name"); + + b = rte_reorder_create("PKT", rte_socket_id(), REORDER_BUFFER_SIZE_INVALID); + TEST_ASSERT((b == NULL) && (rte_errno == EINVAL), + "No error on create() with invalid buffer size param."); + + b = rte_reorder_create("PKT_RO1", rte_socket_id(), REORDER_BUFFER_SIZE); + TEST_ASSERT_EQUAL(b, test_params->b, + "New reorder instance created with already existing name"); + + return 0; +} + +static int +test_reorder_init(void) +{ + struct rte_reorder_buffer *b = NULL; + unsigned int size; + /* + * The minimum memory area size that should be passed to library is, + * sizeof(struct rte_reorder_buffer) + (2 * size * sizeof(struct rte_mbuf *)); + * Otherwise error will be thrown + */ + + size = 100; + b = rte_reorder_init(b, size, "PKT1", REORDER_BUFFER_SIZE); + TEST_ASSERT((b == NULL) && (rte_errno == EINVAL), + "No error on init with NULL buffer."); + + b = rte_malloc(NULL, size, 0); + b = rte_reorder_init(b, size, "PKT1", REORDER_BUFFER_SIZE); + TEST_ASSERT((b == NULL) && (rte_errno == EINVAL), + "No error on init with invalid mem zone size."); + rte_free(b); + + size = 262336; + b = rte_malloc(NULL, size, 0); + b = rte_reorder_init(b, size, "PKT1", REORDER_BUFFER_SIZE_INVALID); + TEST_ASSERT((b == NULL) && (rte_errno == EINVAL), + "No error on init with invalid buffer size param."); + + b = rte_reorder_init(b, size, NULL, REORDER_BUFFER_SIZE); + TEST_ASSERT((b == NULL) && (rte_errno == EINVAL), + "No error on init with invalid name."); + rte_free(b); + + return 0; +} + +static int +test_reorder_find_existing(void) +{ + struct rte_reorder_buffer *b = NULL; + + /* Try to find existing reorder buffer instance */ + b = rte_reorder_find_existing("PKT_RO1"); + TEST_ASSERT_EQUAL(b, test_params->b, + "existing reorder buffer instance not found"); + + /* Try to find non existing reorder buffer instance */ + b = rte_reorder_find_existing("ro_find_non_existing"); + TEST_ASSERT((b == NULL) && (rte_errno == ENOENT), + "non existing reorder buffer instance found"); + + return 0; +} + +static int +test_reorder_free(void) +{ + struct rte_reorder_buffer *b1 = NULL, *b2 = NULL; + const char *name = "test_free"; + + b1 = rte_reorder_create(name, rte_socket_id(), 8); + TEST_ASSERT_NOT_NULL(b1, "Failed to create reorder buffer."); + + b2 = rte_reorder_find_existing(name); + TEST_ASSERT_EQUAL(b1, b2, "Failed to find existing reorder buffer"); + + rte_reorder_free(b1); + + b2 = rte_reorder_find_existing(name); + TEST_ASSERT((b2 == NULL) && (rte_errno == ENOENT), + "Found previously freed reorder buffer"); + + return 0; +} + +static int +test_reorder_insert(void) +{ + struct rte_reorder_buffer *b = NULL; + struct rte_mempool *p = test_params->p; + const unsigned int size = 4; + const unsigned int num_bufs = 7; + struct rte_mbuf *bufs[num_bufs]; + int ret = 0; + unsigned i; + + /* This would create a reorder buffer instance consisting of: + * reorder_seq = 0 + * ready_buf: RB[size] = {NULL, NULL, NULL, NULL} + * order_buf: OB[size] = {NULL, NULL, NULL, NULL} + */ + b = rte_reorder_create("test_insert", rte_socket_id(), size); + TEST_ASSERT_NOT_NULL(b, "Failed to create reorder buffer"); + + ret = rte_mempool_get_bulk(p, (void *)bufs, num_bufs); + TEST_ASSERT_SUCCESS(ret, "Error getting mbuf from pool"); + + for (i = 0; i < num_bufs; i++) + bufs[i]->seqn = i; + + /* This should fill up order buffer: + * reorder_seq = 0 + * RB[] = {NULL, NULL, NULL, NULL} + * OB[] = {0, 1, 2, 3} + */ + for (i = 0; i < size; i++) { + ret = rte_reorder_insert(b, bufs[i]); + if (ret != 0) { + printf("%s:%d: Error inserting packet with seqn less than size\n", + __func__, __LINE__); + ret = -1; + goto exit; + } + } + + /* early packet - should move mbufs to ready buf and move sequence window + * reorder_seq = 4 + * RB[] = {0, 1, 2, 3} + * OB[] = {4, NULL, NULL, NULL} + */ + ret = rte_reorder_insert(b, bufs[4]); + if (ret != 0) { + printf("%s:%d: Error inserting early packet with seqn: size\n", + __func__, __LINE__); + ret = -1; + goto exit; + } + + /* early packet from current sequence window - full ready buffer */ + bufs[5]->seqn = 2 * size; + ret = rte_reorder_insert(b, bufs[5]); + if (!((ret == -1) && (rte_errno == ENOSPC))) { + printf("%s:%d: No error inserting early packet with full ready buffer\n", + __func__, __LINE__); + ret = -1; + goto exit; + } + + /* late packet */ + bufs[6]->seqn = 3 * size; + ret = rte_reorder_insert(b, bufs[6]); + if (!((ret == -1) && (rte_errno == ERANGE))) { + printf("%s:%d: No error inserting late packet with seqn:" + " 3 * size\n", __func__, __LINE__); + ret = -1; + goto exit; + } + + ret = 0; +exit: + rte_mempool_put_bulk(p, (void *)bufs, num_bufs); + rte_reorder_free(b); + return ret; +} + +static int +test_reorder_drain(void) +{ + struct rte_reorder_buffer *b = NULL; + struct rte_mempool *p = test_params->p; + const unsigned int size = 4; + const unsigned int num_bufs = 8; + struct rte_mbuf *bufs[num_bufs]; + struct rte_mbuf *robufs[num_bufs]; + int ret = 0; + unsigned i, cnt; + + /* This would create a reorder buffer instance consisting of: + * reorder_seq = 0 + * ready_buf: RB[size] = {NULL, NULL, NULL, NULL} + * order_buf: OB[size] = {NULL, NULL, NULL, NULL} + */ + b = rte_reorder_create("test_drain", rte_socket_id(), size); + TEST_ASSERT_NOT_NULL(b, "Failed to create reorder buffer"); + + ret = rte_mempool_get_bulk(p, (void *)bufs, num_bufs); + TEST_ASSERT_SUCCESS(ret, "Error getting mbuf from pool"); + + /* Check no drained packets if reorder is empty */ + cnt = rte_reorder_drain(b, robufs, 1); + if (cnt != 0) { + printf("%s:%d: drained packets from empty reorder buffer\n", + __func__, __LINE__); + ret = -1; + goto exit; + } + + for (i = 0; i < num_bufs; i++) + bufs[i]->seqn = i; + + /* Insert packet with seqn 1: + * reorder_seq = 0 + * RB[] = {NULL, NULL, NULL, NULL} + * OB[] = {1, NULL, NULL, NULL} + */ + rte_reorder_insert(b, bufs[1]); + + cnt = rte_reorder_drain(b, robufs, 1); + if (cnt != 1) { + printf("%s:%d:%d: number of expected packets not drained\n", + __func__, __LINE__, cnt); + ret = -1; + goto exit; + } + + /* Insert more packets + * RB[] = {NULL, NULL, NULL, NULL} + * OB[] = {NULL, 2, 3, NULL} + */ + rte_reorder_insert(b, bufs[2]); + rte_reorder_insert(b, bufs[3]); + + /* Insert more packets + * RB[] = {NULL, NULL, NULL, NULL} + * OB[] = {NULL, 2, 3, 4} + */ + rte_reorder_insert(b, bufs[4]); + + /* Insert more packets + * RB[] = {2, 3, 4, NULL} + * OB[] = {NULL, NULL, 7, NULL} + */ + rte_reorder_insert(b, bufs[7]); + + /* drained expected packets */ + cnt = rte_reorder_drain(b, robufs, 4); + if (cnt != 3) { + printf("%s:%d:%d: number of expected packets not drained\n", + __func__, __LINE__, cnt); + ret = -1; + goto exit; + } + + /* + * RB[] = {NULL, NULL, NULL, NULL} + * OB[] = {NULL, NULL, 7, NULL} + */ + cnt = rte_reorder_drain(b, robufs, 1); + if (cnt != 0) { + printf("%s:%d:%d: number of expected packets not drained\n", + __func__, __LINE__, cnt); + ret = -1; + goto exit; + } + ret = 0; +exit: + rte_mempool_put_bulk(p, (void *)bufs, num_bufs); + rte_reorder_free(b); + return ret; +} + +static int +test_setup(void) +{ + /* reorder buffer instance creation */ + if (test_params->b == NULL) { + test_params->b = rte_reorder_create("PKT_RO1", rte_socket_id(), + REORDER_BUFFER_SIZE); + if (test_params->b == NULL) { + printf("%s: Error creating reorder buffer instance b\n", + __func__); + return -1; + } + } else + rte_reorder_reset(test_params->b); + + /* mempool creation */ + if (test_params->p == NULL) { + test_params->p = rte_pktmbuf_pool_create("RO_MBUF_POOL", + NUM_MBUFS, BURST, 0, RTE_MBUF_DEFAULT_BUF_SIZE, + rte_socket_id()); + if (test_params->p == NULL) { + printf("%s: Error creating mempool\n", __func__); + return -1; + } + } + return 0; +} + +static void +test_teardown(void) +{ + rte_reorder_free(test_params->b); + test_params->b = NULL; + rte_mempool_free(test_params->p); + test_params->p = NULL; +} + + +static struct unit_test_suite reorder_test_suite = { + + .setup = test_setup, + .teardown = test_teardown, + .suite_name = "Reorder Unit Test Suite", + .unit_test_cases = { + TEST_CASE(test_reorder_create), + TEST_CASE(test_reorder_init), + TEST_CASE(test_reorder_find_existing), + TEST_CASE(test_reorder_free), + TEST_CASE(test_reorder_insert), + TEST_CASE(test_reorder_drain), + TEST_CASES_END() + } +}; + +static int +test_reorder(void) +{ + return unit_test_suite_runner(&reorder_test_suite); +} + +REGISTER_TEST_COMMAND(reorder_autotest, test_reorder); diff --git a/dpdk/test/test/test_resource.c b/dpdk/test/test/test_resource.c new file mode 100644 index 00000000..a3a82f13 --- /dev/null +++ b/dpdk/test/test/test_resource.c @@ -0,0 +1,133 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2016 RehiveTech. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of RehiveTech nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <string.h> + +#include "test.h" +#include "resource.h" + +const char test_resource_dpdk_blob[] = { + '\x44', '\x50', '\x44', '\x4b', '\x00' +}; + +REGISTER_RESOURCE(test_resource_dpdk, + test_resource_dpdk_blob, test_resource_dpdk_blob + 4); + +static int test_resource_dpdk(void) +{ + const struct resource *r; + + r = resource_find("test_resource_dpdk"); + TEST_ASSERT_NOT_NULL(r, "Could not find test_resource_dpdk"); + TEST_ASSERT(!strcmp(r->name, "test_resource_dpdk"), + "Found resource %s, expected test_resource_dpdk", + r->name); + + TEST_ASSERT(!strncmp("DPDK", r->begin, 4), + "Unexpected payload: %.4s...", r->begin); + + return 0; +} + +REGISTER_LINKED_RESOURCE(test_resource_c); + +static int test_resource_c(void) +{ + const struct resource *r; + FILE *f; + + r = resource_find("test_resource_c"); + TEST_ASSERT_NOT_NULL(r, "No test_resource_c found"); + TEST_ASSERT(!strcmp(r->name, "test_resource_c"), + "Found resource %s, expected test_resource_c", + r->name); + + TEST_ASSERT_SUCCESS(resource_fwrite_file(r, "test_resource.c"), + "Failed to to write file %s", r->name); + + f = fopen("test_resource.c", "r"); + TEST_ASSERT_NOT_NULL(f, + "Missing extracted file resource.c"); + fclose(f); + remove("test_resource.c"); + + return 0; +} + +#ifdef RTE_APP_TEST_RESOURCE_TAR +REGISTER_LINKED_RESOURCE(test_resource_tar); + +static int test_resource_tar(void) +{ + const struct resource *r; + FILE *f; + + r = resource_find("test_resource_tar"); + TEST_ASSERT_NOT_NULL(r, "No test_resource_tar found"); + TEST_ASSERT(!strcmp(r->name, "test_resource_tar"), + "Found resource %s, expected test_resource_tar", + r->name); + + TEST_ASSERT_SUCCESS(resource_untar(r), + "Failed to to untar %s", r->name); + + f = fopen("test_resource.c", "r"); + TEST_ASSERT_NOT_NULL(f, + "Missing extracted file test_resource.c"); + fclose(f); + + TEST_ASSERT_SUCCESS(resource_rm_by_tar(r), + "Failed to remove extracted contents of %s", r->name); + return 0; +} + +#endif /* RTE_APP_TEST_RESOURCE_TAR */ + +static int test_resource(void) +{ + if (test_resource_dpdk()) + return -1; + + if (test_resource_c()) + return -1; + +#ifdef RTE_APP_TEST_RESOURCE_TAR + if (test_resource_tar()) + return -1; +#endif /* RTE_APP_TEST_RESOURCE_TAR */ + + return 0; +} + +REGISTER_TEST_COMMAND(resource_autotest, test_resource); diff --git a/dpdk/test/test/test_ring.c b/dpdk/test/test/test_ring.c new file mode 100644 index 00000000..d225e829 --- /dev/null +++ b/dpdk/test/test/test_ring.c @@ -0,0 +1,905 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <string.h> +#include <stdarg.h> +#include <stdio.h> +#include <stdlib.h> +#include <stdint.h> +#include <inttypes.h> +#include <errno.h> +#include <sys/queue.h> + +#include <rte_common.h> +#include <rte_log.h> +#include <rte_memory.h> +#include <rte_launch.h> +#include <rte_cycles.h> +#include <rte_eal.h> +#include <rte_per_lcore.h> +#include <rte_lcore.h> +#include <rte_atomic.h> +#include <rte_branch_prediction.h> +#include <rte_malloc.h> +#include <rte_ring.h> +#include <rte_random.h> +#include <rte_errno.h> +#include <rte_hexdump.h> + +#include "test.h" + +/* + * Ring + * ==== + * + * #. Basic tests: done on one core: + * + * - Using single producer/single consumer functions: + * + * - Enqueue one object, two objects, MAX_BULK objects + * - Dequeue one object, two objects, MAX_BULK objects + * - Check that dequeued pointers are correct + * + * - Using multi producers/multi consumers functions: + * + * - Enqueue one object, two objects, MAX_BULK objects + * - Dequeue one object, two objects, MAX_BULK objects + * - Check that dequeued pointers are correct + * + * #. Performance tests. + * + * Tests done in test_ring_perf.c + */ + +#define RING_SIZE 4096 +#define MAX_BULK 32 + +static rte_atomic32_t synchro; + +#define TEST_RING_VERIFY(exp) \ + if (!(exp)) { \ + printf("error at %s:%d\tcondition " #exp " failed\n", \ + __func__, __LINE__); \ + rte_ring_dump(stdout, r); \ + return -1; \ + } + +#define TEST_RING_FULL_EMTPY_ITER 8 + +/* + * helper routine for test_ring_basic + */ +static int +test_ring_basic_full_empty(struct rte_ring *r, void * const src[], void *dst[]) +{ + unsigned i, rand; + const unsigned rsz = RING_SIZE - 1; + + printf("Basic full/empty test\n"); + + for (i = 0; TEST_RING_FULL_EMTPY_ITER != i; i++) { + + /* random shift in the ring */ + rand = RTE_MAX(rte_rand() % RING_SIZE, 1UL); + printf("%s: iteration %u, random shift: %u;\n", + __func__, i, rand); + TEST_RING_VERIFY(rte_ring_enqueue_bulk(r, src, rand, + NULL) != 0); + TEST_RING_VERIFY(rte_ring_dequeue_bulk(r, dst, rand, + NULL) == rand); + + /* fill the ring */ + TEST_RING_VERIFY(rte_ring_enqueue_bulk(r, src, rsz, NULL) != 0); + TEST_RING_VERIFY(0 == rte_ring_free_count(r)); + TEST_RING_VERIFY(rsz == rte_ring_count(r)); + TEST_RING_VERIFY(rte_ring_full(r)); + TEST_RING_VERIFY(0 == rte_ring_empty(r)); + + /* empty the ring */ + TEST_RING_VERIFY(rte_ring_dequeue_bulk(r, dst, rsz, + NULL) == rsz); + TEST_RING_VERIFY(rsz == rte_ring_free_count(r)); + TEST_RING_VERIFY(0 == rte_ring_count(r)); + TEST_RING_VERIFY(0 == rte_ring_full(r)); + TEST_RING_VERIFY(rte_ring_empty(r)); + + /* check data */ + TEST_RING_VERIFY(0 == memcmp(src, dst, rsz)); + rte_ring_dump(stdout, r); + } + return 0; +} + +static int +test_ring_basic(struct rte_ring *r) +{ + void **src = NULL, **cur_src = NULL, **dst = NULL, **cur_dst = NULL; + int ret; + unsigned i, num_elems; + + /* alloc dummy object pointers */ + src = malloc(RING_SIZE*2*sizeof(void *)); + if (src == NULL) + goto fail; + + for (i = 0; i < RING_SIZE*2 ; i++) { + src[i] = (void *)(unsigned long)i; + } + cur_src = src; + + /* alloc some room for copied objects */ + dst = malloc(RING_SIZE*2*sizeof(void *)); + if (dst == NULL) + goto fail; + + memset(dst, 0, RING_SIZE*2*sizeof(void *)); + cur_dst = dst; + + printf("enqueue 1 obj\n"); + ret = rte_ring_sp_enqueue_bulk(r, cur_src, 1, NULL); + cur_src += 1; + if (ret == 0) + goto fail; + + printf("enqueue 2 objs\n"); + ret = rte_ring_sp_enqueue_bulk(r, cur_src, 2, NULL); + cur_src += 2; + if (ret == 0) + goto fail; + + printf("enqueue MAX_BULK objs\n"); + ret = rte_ring_sp_enqueue_bulk(r, cur_src, MAX_BULK, NULL); + cur_src += MAX_BULK; + if (ret == 0) + goto fail; + + printf("dequeue 1 obj\n"); + ret = rte_ring_sc_dequeue_bulk(r, cur_dst, 1, NULL); + cur_dst += 1; + if (ret == 0) + goto fail; + + printf("dequeue 2 objs\n"); + ret = rte_ring_sc_dequeue_bulk(r, cur_dst, 2, NULL); + cur_dst += 2; + if (ret == 0) + goto fail; + + printf("dequeue MAX_BULK objs\n"); + ret = rte_ring_sc_dequeue_bulk(r, cur_dst, MAX_BULK, NULL); + cur_dst += MAX_BULK; + if (ret == 0) + goto fail; + + /* check data */ + if (memcmp(src, dst, cur_dst - dst)) { + rte_hexdump(stdout, "src", src, cur_src - src); + rte_hexdump(stdout, "dst", dst, cur_dst - dst); + printf("data after dequeue is not the same\n"); + goto fail; + } + cur_src = src; + cur_dst = dst; + + printf("enqueue 1 obj\n"); + ret = rte_ring_mp_enqueue_bulk(r, cur_src, 1, NULL); + cur_src += 1; + if (ret == 0) + goto fail; + + printf("enqueue 2 objs\n"); + ret = rte_ring_mp_enqueue_bulk(r, cur_src, 2, NULL); + cur_src += 2; + if (ret == 0) + goto fail; + + printf("enqueue MAX_BULK objs\n"); + ret = rte_ring_mp_enqueue_bulk(r, cur_src, MAX_BULK, NULL); + cur_src += MAX_BULK; + if (ret == 0) + goto fail; + + printf("dequeue 1 obj\n"); + ret = rte_ring_mc_dequeue_bulk(r, cur_dst, 1, NULL); + cur_dst += 1; + if (ret == 0) + goto fail; + + printf("dequeue 2 objs\n"); + ret = rte_ring_mc_dequeue_bulk(r, cur_dst, 2, NULL); + cur_dst += 2; + if (ret == 0) + goto fail; + + printf("dequeue MAX_BULK objs\n"); + ret = rte_ring_mc_dequeue_bulk(r, cur_dst, MAX_BULK, NULL); + cur_dst += MAX_BULK; + if (ret == 0) + goto fail; + + /* check data */ + if (memcmp(src, dst, cur_dst - dst)) { + rte_hexdump(stdout, "src", src, cur_src - src); + rte_hexdump(stdout, "dst", dst, cur_dst - dst); + printf("data after dequeue is not the same\n"); + goto fail; + } + cur_src = src; + cur_dst = dst; + + printf("fill and empty the ring\n"); + for (i = 0; i<RING_SIZE/MAX_BULK; i++) { + ret = rte_ring_mp_enqueue_bulk(r, cur_src, MAX_BULK, NULL); + cur_src += MAX_BULK; + if (ret == 0) + goto fail; + ret = rte_ring_mc_dequeue_bulk(r, cur_dst, MAX_BULK, NULL); + cur_dst += MAX_BULK; + if (ret == 0) + goto fail; + } + + /* check data */ + if (memcmp(src, dst, cur_dst - dst)) { + rte_hexdump(stdout, "src", src, cur_src - src); + rte_hexdump(stdout, "dst", dst, cur_dst - dst); + printf("data after dequeue is not the same\n"); + goto fail; + } + + if (test_ring_basic_full_empty(r, src, dst) != 0) + goto fail; + + cur_src = src; + cur_dst = dst; + + printf("test default bulk enqueue / dequeue\n"); + num_elems = 16; + + cur_src = src; + cur_dst = dst; + + ret = rte_ring_enqueue_bulk(r, cur_src, num_elems, NULL); + cur_src += num_elems; + if (ret == 0) { + printf("Cannot enqueue\n"); + goto fail; + } + ret = rte_ring_enqueue_bulk(r, cur_src, num_elems, NULL); + cur_src += num_elems; + if (ret == 0) { + printf("Cannot enqueue\n"); + goto fail; + } + ret = rte_ring_dequeue_bulk(r, cur_dst, num_elems, NULL); + cur_dst += num_elems; + if (ret == 0) { + printf("Cannot dequeue\n"); + goto fail; + } + ret = rte_ring_dequeue_bulk(r, cur_dst, num_elems, NULL); + cur_dst += num_elems; + if (ret == 0) { + printf("Cannot dequeue2\n"); + goto fail; + } + + /* check data */ + if (memcmp(src, dst, cur_dst - dst)) { + rte_hexdump(stdout, "src", src, cur_src - src); + rte_hexdump(stdout, "dst", dst, cur_dst - dst); + printf("data after dequeue is not the same\n"); + goto fail; + } + + cur_src = src; + cur_dst = dst; + + ret = rte_ring_mp_enqueue(r, cur_src); + if (ret != 0) + goto fail; + + ret = rte_ring_mc_dequeue(r, cur_dst); + if (ret != 0) + goto fail; + + free(src); + free(dst); + return 0; + + fail: + free(src); + free(dst); + return -1; +} + +static int +test_ring_burst_basic(struct rte_ring *r) +{ + void **src = NULL, **cur_src = NULL, **dst = NULL, **cur_dst = NULL; + int ret; + unsigned i; + + /* alloc dummy object pointers */ + src = malloc(RING_SIZE*2*sizeof(void *)); + if (src == NULL) + goto fail; + + for (i = 0; i < RING_SIZE*2 ; i++) { + src[i] = (void *)(unsigned long)i; + } + cur_src = src; + + /* alloc some room for copied objects */ + dst = malloc(RING_SIZE*2*sizeof(void *)); + if (dst == NULL) + goto fail; + + memset(dst, 0, RING_SIZE*2*sizeof(void *)); + cur_dst = dst; + + printf("Test SP & SC basic functions \n"); + printf("enqueue 1 obj\n"); + ret = rte_ring_sp_enqueue_burst(r, cur_src, 1, NULL); + cur_src += 1; + if (ret != 1) + goto fail; + + printf("enqueue 2 objs\n"); + ret = rte_ring_sp_enqueue_burst(r, cur_src, 2, NULL); + cur_src += 2; + if (ret != 2) + goto fail; + + printf("enqueue MAX_BULK objs\n"); + ret = rte_ring_sp_enqueue_burst(r, cur_src, MAX_BULK, NULL); + cur_src += MAX_BULK; + if (ret != MAX_BULK) + goto fail; + + printf("dequeue 1 obj\n"); + ret = rte_ring_sc_dequeue_burst(r, cur_dst, 1, NULL); + cur_dst += 1; + if (ret != 1) + goto fail; + + printf("dequeue 2 objs\n"); + ret = rte_ring_sc_dequeue_burst(r, cur_dst, 2, NULL); + cur_dst += 2; + if (ret != 2) + goto fail; + + printf("dequeue MAX_BULK objs\n"); + ret = rte_ring_sc_dequeue_burst(r, cur_dst, MAX_BULK, NULL); + cur_dst += MAX_BULK; + if (ret != MAX_BULK) + goto fail; + + /* check data */ + if (memcmp(src, dst, cur_dst - dst)) { + rte_hexdump(stdout, "src", src, cur_src - src); + rte_hexdump(stdout, "dst", dst, cur_dst - dst); + printf("data after dequeue is not the same\n"); + goto fail; + } + + cur_src = src; + cur_dst = dst; + + printf("Test enqueue without enough memory space \n"); + for (i = 0; i< (RING_SIZE/MAX_BULK - 1); i++) { + ret = rte_ring_sp_enqueue_burst(r, cur_src, MAX_BULK, NULL); + cur_src += MAX_BULK; + if (ret != MAX_BULK) + goto fail; + } + + printf("Enqueue 2 objects, free entries = MAX_BULK - 2 \n"); + ret = rte_ring_sp_enqueue_burst(r, cur_src, 2, NULL); + cur_src += 2; + if (ret != 2) + goto fail; + + printf("Enqueue the remaining entries = MAX_BULK - 2 \n"); + /* Always one free entry left */ + ret = rte_ring_sp_enqueue_burst(r, cur_src, MAX_BULK, NULL); + cur_src += MAX_BULK - 3; + if (ret != MAX_BULK - 3) + goto fail; + + printf("Test if ring is full \n"); + if (rte_ring_full(r) != 1) + goto fail; + + printf("Test enqueue for a full entry \n"); + ret = rte_ring_sp_enqueue_burst(r, cur_src, MAX_BULK, NULL); + if (ret != 0) + goto fail; + + printf("Test dequeue without enough objects \n"); + for (i = 0; i<RING_SIZE/MAX_BULK - 1; i++) { + ret = rte_ring_sc_dequeue_burst(r, cur_dst, MAX_BULK, NULL); + cur_dst += MAX_BULK; + if (ret != MAX_BULK) + goto fail; + } + + /* Available memory space for the exact MAX_BULK entries */ + ret = rte_ring_sc_dequeue_burst(r, cur_dst, 2, NULL); + cur_dst += 2; + if (ret != 2) + goto fail; + + ret = rte_ring_sc_dequeue_burst(r, cur_dst, MAX_BULK, NULL); + cur_dst += MAX_BULK - 3; + if (ret != MAX_BULK - 3) + goto fail; + + printf("Test if ring is empty \n"); + /* Check if ring is empty */ + if (1 != rte_ring_empty(r)) + goto fail; + + /* check data */ + if (memcmp(src, dst, cur_dst - dst)) { + rte_hexdump(stdout, "src", src, cur_src - src); + rte_hexdump(stdout, "dst", dst, cur_dst - dst); + printf("data after dequeue is not the same\n"); + goto fail; + } + + cur_src = src; + cur_dst = dst; + + printf("Test MP & MC basic functions \n"); + + printf("enqueue 1 obj\n"); + ret = rte_ring_mp_enqueue_burst(r, cur_src, 1, NULL); + cur_src += 1; + if (ret != 1) + goto fail; + + printf("enqueue 2 objs\n"); + ret = rte_ring_mp_enqueue_burst(r, cur_src, 2, NULL); + cur_src += 2; + if (ret != 2) + goto fail; + + printf("enqueue MAX_BULK objs\n"); + ret = rte_ring_mp_enqueue_burst(r, cur_src, MAX_BULK, NULL); + cur_src += MAX_BULK; + if (ret != MAX_BULK) + goto fail; + + printf("dequeue 1 obj\n"); + ret = rte_ring_mc_dequeue_burst(r, cur_dst, 1, NULL); + cur_dst += 1; + if (ret != 1) + goto fail; + + printf("dequeue 2 objs\n"); + ret = rte_ring_mc_dequeue_burst(r, cur_dst, 2, NULL); + cur_dst += 2; + if (ret != 2) + goto fail; + + printf("dequeue MAX_BULK objs\n"); + ret = rte_ring_mc_dequeue_burst(r, cur_dst, MAX_BULK, NULL); + cur_dst += MAX_BULK; + if (ret != MAX_BULK) + goto fail; + + /* check data */ + if (memcmp(src, dst, cur_dst - dst)) { + rte_hexdump(stdout, "src", src, cur_src - src); + rte_hexdump(stdout, "dst", dst, cur_dst - dst); + printf("data after dequeue is not the same\n"); + goto fail; + } + + cur_src = src; + cur_dst = dst; + + printf("fill and empty the ring\n"); + for (i = 0; i<RING_SIZE/MAX_BULK; i++) { + ret = rte_ring_mp_enqueue_burst(r, cur_src, MAX_BULK, NULL); + cur_src += MAX_BULK; + if (ret != MAX_BULK) + goto fail; + ret = rte_ring_mc_dequeue_burst(r, cur_dst, MAX_BULK, NULL); + cur_dst += MAX_BULK; + if (ret != MAX_BULK) + goto fail; + } + + /* check data */ + if (memcmp(src, dst, cur_dst - dst)) { + rte_hexdump(stdout, "src", src, cur_src - src); + rte_hexdump(stdout, "dst", dst, cur_dst - dst); + printf("data after dequeue is not the same\n"); + goto fail; + } + + cur_src = src; + cur_dst = dst; + + printf("Test enqueue without enough memory space \n"); + for (i = 0; i<RING_SIZE/MAX_BULK - 1; i++) { + ret = rte_ring_mp_enqueue_burst(r, cur_src, MAX_BULK, NULL); + cur_src += MAX_BULK; + if (ret != MAX_BULK) + goto fail; + } + + /* Available memory space for the exact MAX_BULK objects */ + ret = rte_ring_mp_enqueue_burst(r, cur_src, 2, NULL); + cur_src += 2; + if (ret != 2) + goto fail; + + ret = rte_ring_mp_enqueue_burst(r, cur_src, MAX_BULK, NULL); + cur_src += MAX_BULK - 3; + if (ret != MAX_BULK - 3) + goto fail; + + + printf("Test dequeue without enough objects \n"); + for (i = 0; i<RING_SIZE/MAX_BULK - 1; i++) { + ret = rte_ring_mc_dequeue_burst(r, cur_dst, MAX_BULK, NULL); + cur_dst += MAX_BULK; + if (ret != MAX_BULK) + goto fail; + } + + /* Available objects - the exact MAX_BULK */ + ret = rte_ring_mc_dequeue_burst(r, cur_dst, 2, NULL); + cur_dst += 2; + if (ret != 2) + goto fail; + + ret = rte_ring_mc_dequeue_burst(r, cur_dst, MAX_BULK, NULL); + cur_dst += MAX_BULK - 3; + if (ret != MAX_BULK - 3) + goto fail; + + /* check data */ + if (memcmp(src, dst, cur_dst - dst)) { + rte_hexdump(stdout, "src", src, cur_src - src); + rte_hexdump(stdout, "dst", dst, cur_dst - dst); + printf("data after dequeue is not the same\n"); + goto fail; + } + + cur_src = src; + cur_dst = dst; + + printf("Covering rte_ring_enqueue_burst functions \n"); + + ret = rte_ring_enqueue_burst(r, cur_src, 2, NULL); + cur_src += 2; + if (ret != 2) + goto fail; + + ret = rte_ring_dequeue_burst(r, cur_dst, 2, NULL); + cur_dst += 2; + if (ret != 2) + goto fail; + + /* Free memory before test completed */ + free(src); + free(dst); + return 0; + + fail: + free(src); + free(dst); + return -1; +} + +/* + * it will always fail to create ring with a wrong ring size number in this function + */ +static int +test_ring_creation_with_wrong_size(void) +{ + struct rte_ring * rp = NULL; + + /* Test if ring size is not power of 2 */ + rp = rte_ring_create("test_bad_ring_size", RING_SIZE + 1, SOCKET_ID_ANY, 0); + if (NULL != rp) { + return -1; + } + + /* Test if ring size is exceeding the limit */ + rp = rte_ring_create("test_bad_ring_size", (RTE_RING_SZ_MASK + 1), SOCKET_ID_ANY, 0); + if (NULL != rp) { + return -1; + } + return 0; +} + +/* + * it tests if it would always fail to create ring with an used ring name + */ +static int +test_ring_creation_with_an_used_name(void) +{ + struct rte_ring * rp; + + rp = rte_ring_create("test", RING_SIZE, SOCKET_ID_ANY, 0); + if (NULL != rp) + return -1; + + return 0; +} + +/* + * Test to if a non-power of 2 count causes the create + * function to fail correctly + */ +static int +test_create_count_odd(void) +{ + struct rte_ring *r = rte_ring_create("test_ring_count", + 4097, SOCKET_ID_ANY, 0 ); + if(r != NULL){ + return -1; + } + return 0; +} + +static int +test_lookup_null(void) +{ + struct rte_ring *rlp = rte_ring_lookup("ring_not_found"); + if (rlp ==NULL) + if (rte_errno != ENOENT){ + printf( "test failed to returnn error on null pointer\n"); + return -1; + } + return 0; +} + +/* + * it tests some more basic ring operations + */ +static int +test_ring_basic_ex(void) +{ + int ret = -1; + unsigned i; + struct rte_ring *rp = NULL; + void **obj = NULL; + + obj = rte_calloc("test_ring_basic_ex_malloc", RING_SIZE, sizeof(void *), 0); + if (obj == NULL) { + printf("test_ring_basic_ex fail to rte_malloc\n"); + goto fail_test; + } + + rp = rte_ring_create("test_ring_basic_ex", RING_SIZE, SOCKET_ID_ANY, + RING_F_SP_ENQ | RING_F_SC_DEQ); + if (rp == NULL) { + printf("test_ring_basic_ex fail to create ring\n"); + goto fail_test; + } + + if (rte_ring_lookup("test_ring_basic_ex") != rp) { + goto fail_test; + } + + if (rte_ring_empty(rp) != 1) { + printf("test_ring_basic_ex ring is not empty but it should be\n"); + goto fail_test; + } + + printf("%u ring entries are now free\n", rte_ring_free_count(rp)); + + for (i = 0; i < RING_SIZE; i ++) { + rte_ring_enqueue(rp, obj[i]); + } + + if (rte_ring_full(rp) != 1) { + printf("test_ring_basic_ex ring is not full but it should be\n"); + goto fail_test; + } + + for (i = 0; i < RING_SIZE; i ++) { + rte_ring_dequeue(rp, &obj[i]); + } + + if (rte_ring_empty(rp) != 1) { + printf("test_ring_basic_ex ring is not empty but it should be\n"); + goto fail_test; + } + + /* Covering the ring burst operation */ + ret = rte_ring_enqueue_burst(rp, obj, 2, NULL); + if (ret != 2) { + printf("test_ring_basic_ex: rte_ring_enqueue_burst fails \n"); + goto fail_test; + } + + ret = rte_ring_dequeue_burst(rp, obj, 2, NULL); + if (ret != 2) { + printf("test_ring_basic_ex: rte_ring_dequeue_burst fails \n"); + goto fail_test; + } + + ret = 0; +fail_test: + rte_ring_free(rp); + if (obj != NULL) + rte_free(obj); + + return ret; +} + +static int +test_ring_with_exact_size(void) +{ + struct rte_ring *std_ring = NULL, *exact_sz_ring = NULL; + void *ptr_array[16]; + static const unsigned int ring_sz = RTE_DIM(ptr_array); + unsigned int i; + int ret = -1; + + std_ring = rte_ring_create("std", ring_sz, rte_socket_id(), + RING_F_SP_ENQ | RING_F_SC_DEQ); + if (std_ring == NULL) { + printf("%s: error, can't create std ring\n", __func__); + goto end; + } + exact_sz_ring = rte_ring_create("exact sz", ring_sz, rte_socket_id(), + RING_F_SP_ENQ | RING_F_SC_DEQ | RING_F_EXACT_SZ); + if (exact_sz_ring == NULL) { + printf("%s: error, can't create exact size ring\n", __func__); + goto end; + } + + /* + * Check that the exact size ring is bigger than the standard ring + */ + if (rte_ring_get_size(std_ring) >= rte_ring_get_size(exact_sz_ring)) { + printf("%s: error, std ring (size: %u) is not smaller than exact size one (size %u)\n", + __func__, + rte_ring_get_size(std_ring), + rte_ring_get_size(exact_sz_ring)); + goto end; + } + /* + * check that the exact_sz_ring can hold one more element than the + * standard ring. (16 vs 15 elements) + */ + for (i = 0; i < ring_sz - 1; i++) { + rte_ring_enqueue(std_ring, NULL); + rte_ring_enqueue(exact_sz_ring, NULL); + } + if (rte_ring_enqueue(std_ring, NULL) != -ENOBUFS) { + printf("%s: error, unexpected successful enqueue\n", __func__); + goto end; + } + if (rte_ring_enqueue(exact_sz_ring, NULL) == -ENOBUFS) { + printf("%s: error, enqueue failed\n", __func__); + goto end; + } + + /* check that dequeue returns the expected number of elements */ + if (rte_ring_dequeue_burst(exact_sz_ring, ptr_array, + RTE_DIM(ptr_array), NULL) != ring_sz) { + printf("%s: error, failed to dequeue expected nb of elements\n", + __func__); + goto end; + } + + /* check that the capacity function returns expected value */ + if (rte_ring_get_capacity(exact_sz_ring) != ring_sz) { + printf("%s: error, incorrect ring capacity reported\n", + __func__); + goto end; + } + + ret = 0; /* all ok if we get here */ +end: + rte_ring_free(std_ring); + rte_ring_free(exact_sz_ring); + return ret; +} + +static int +test_ring(void) +{ + struct rte_ring *r = NULL; + + /* some more basic operations */ + if (test_ring_basic_ex() < 0) + goto test_fail; + + rte_atomic32_init(&synchro); + + r = rte_ring_create("test", RING_SIZE, SOCKET_ID_ANY, 0); + if (r == NULL) + goto test_fail; + + /* retrieve the ring from its name */ + if (rte_ring_lookup("test") != r) { + printf("Cannot lookup ring from its name\n"); + goto test_fail; + } + + /* burst operations */ + if (test_ring_burst_basic(r) < 0) + goto test_fail; + + /* basic operations */ + if (test_ring_basic(r) < 0) + goto test_fail; + + /* basic operations */ + if ( test_create_count_odd() < 0){ + printf("Test failed to detect odd count\n"); + goto test_fail; + } else + printf("Test detected odd count\n"); + + if ( test_lookup_null() < 0){ + printf("Test failed to detect NULL ring lookup\n"); + goto test_fail; + } else + printf("Test detected NULL ring lookup\n"); + + /* test of creating ring with wrong size */ + if (test_ring_creation_with_wrong_size() < 0) + goto test_fail; + + /* test of creation ring with an used name */ + if (test_ring_creation_with_an_used_name() < 0) + goto test_fail; + + if (test_ring_with_exact_size() < 0) + goto test_fail; + + /* dump the ring status */ + rte_ring_list_dump(stdout); + + rte_ring_free(r); + + return 0; + +test_fail: + rte_ring_free(r); + + return -1; +} + +REGISTER_TEST_COMMAND(ring_autotest, test_ring); diff --git a/dpdk/test/test/test_ring_perf.c b/dpdk/test/test/test_ring_perf.c new file mode 100644 index 00000000..66aa51ad --- /dev/null +++ b/dpdk/test/test/test_ring_perf.c @@ -0,0 +1,430 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + + +#include <stdio.h> +#include <inttypes.h> +#include <rte_ring.h> +#include <rte_cycles.h> +#include <rte_launch.h> +#include <rte_pause.h> + +#include "test.h" + +/* + * Ring + * ==== + * + * Measures performance of various operations using rdtsc + * * Empty ring dequeue + * * Enqueue/dequeue of bursts in 1 threads + * * Enqueue/dequeue of bursts in 2 threads + */ + +#define RING_NAME "RING_PERF" +#define RING_SIZE 4096 +#define MAX_BURST 32 + +/* + * the sizes to enqueue and dequeue in testing + * (marked volatile so they won't be seen as compile-time constants) + */ +static const volatile unsigned bulk_sizes[] = { 8, 32 }; + +struct lcore_pair { + unsigned c1, c2; +}; + +static volatile unsigned lcore_count = 0; + +/**** Functions to analyse our core mask to get cores for different tests ***/ + +static int +get_two_hyperthreads(struct lcore_pair *lcp) +{ + unsigned id1, id2; + unsigned c1, c2, s1, s2; + RTE_LCORE_FOREACH(id1) { + /* inner loop just re-reads all id's. We could skip the first few + * elements, but since number of cores is small there is little point + */ + RTE_LCORE_FOREACH(id2) { + if (id1 == id2) + continue; + c1 = lcore_config[id1].core_id; + c2 = lcore_config[id2].core_id; + s1 = lcore_config[id1].socket_id; + s2 = lcore_config[id2].socket_id; + if ((c1 == c2) && (s1 == s2)){ + lcp->c1 = id1; + lcp->c2 = id2; + return 0; + } + } + } + return 1; +} + +static int +get_two_cores(struct lcore_pair *lcp) +{ + unsigned id1, id2; + unsigned c1, c2, s1, s2; + RTE_LCORE_FOREACH(id1) { + RTE_LCORE_FOREACH(id2) { + if (id1 == id2) + continue; + c1 = lcore_config[id1].core_id; + c2 = lcore_config[id2].core_id; + s1 = lcore_config[id1].socket_id; + s2 = lcore_config[id2].socket_id; + if ((c1 != c2) && (s1 == s2)){ + lcp->c1 = id1; + lcp->c2 = id2; + return 0; + } + } + } + return 1; +} + +static int +get_two_sockets(struct lcore_pair *lcp) +{ + unsigned id1, id2; + unsigned s1, s2; + RTE_LCORE_FOREACH(id1) { + RTE_LCORE_FOREACH(id2) { + if (id1 == id2) + continue; + s1 = lcore_config[id1].socket_id; + s2 = lcore_config[id2].socket_id; + if (s1 != s2){ + lcp->c1 = id1; + lcp->c2 = id2; + return 0; + } + } + } + return 1; +} + +/* Get cycle counts for dequeuing from an empty ring. Should be 2 or 3 cycles */ +static void +test_empty_dequeue(struct rte_ring *r) +{ + const unsigned iter_shift = 26; + const unsigned iterations = 1<<iter_shift; + unsigned i = 0; + void *burst[MAX_BURST]; + + const uint64_t sc_start = rte_rdtsc(); + for (i = 0; i < iterations; i++) + rte_ring_sc_dequeue_bulk(r, burst, bulk_sizes[0], NULL); + const uint64_t sc_end = rte_rdtsc(); + + const uint64_t mc_start = rte_rdtsc(); + for (i = 0; i < iterations; i++) + rte_ring_mc_dequeue_bulk(r, burst, bulk_sizes[0], NULL); + const uint64_t mc_end = rte_rdtsc(); + + printf("SC empty dequeue: %.2F\n", + (double)(sc_end-sc_start) / iterations); + printf("MC empty dequeue: %.2F\n", + (double)(mc_end-mc_start) / iterations); +} + +/* + * for the separate enqueue and dequeue threads they take in one param + * and return two. Input = burst size, output = cycle average for sp/sc & mp/mc + */ +struct thread_params { + struct rte_ring *r; + unsigned size; /* input value, the burst size */ + double spsc, mpmc; /* output value, the single or multi timings */ +}; + +/* + * Function that uses rdtsc to measure timing for ring enqueue. Needs pair + * thread running dequeue_bulk function + */ +static int +enqueue_bulk(void *p) +{ + const unsigned iter_shift = 23; + const unsigned iterations = 1<<iter_shift; + struct thread_params *params = p; + struct rte_ring *r = params->r; + const unsigned size = params->size; + unsigned i; + void *burst[MAX_BURST] = {0}; + + if ( __sync_add_and_fetch(&lcore_count, 1) != 2 ) + while(lcore_count != 2) + rte_pause(); + + const uint64_t sp_start = rte_rdtsc(); + for (i = 0; i < iterations; i++) + while (rte_ring_sp_enqueue_bulk(r, burst, size, NULL) == 0) + rte_pause(); + const uint64_t sp_end = rte_rdtsc(); + + const uint64_t mp_start = rte_rdtsc(); + for (i = 0; i < iterations; i++) + while (rte_ring_mp_enqueue_bulk(r, burst, size, NULL) == 0) + rte_pause(); + const uint64_t mp_end = rte_rdtsc(); + + params->spsc = ((double)(sp_end - sp_start))/(iterations*size); + params->mpmc = ((double)(mp_end - mp_start))/(iterations*size); + return 0; +} + +/* + * Function that uses rdtsc to measure timing for ring dequeue. Needs pair + * thread running enqueue_bulk function + */ +static int +dequeue_bulk(void *p) +{ + const unsigned iter_shift = 23; + const unsigned iterations = 1<<iter_shift; + struct thread_params *params = p; + struct rte_ring *r = params->r; + const unsigned size = params->size; + unsigned i; + void *burst[MAX_BURST] = {0}; + + if ( __sync_add_and_fetch(&lcore_count, 1) != 2 ) + while(lcore_count != 2) + rte_pause(); + + const uint64_t sc_start = rte_rdtsc(); + for (i = 0; i < iterations; i++) + while (rte_ring_sc_dequeue_bulk(r, burst, size, NULL) == 0) + rte_pause(); + const uint64_t sc_end = rte_rdtsc(); + + const uint64_t mc_start = rte_rdtsc(); + for (i = 0; i < iterations; i++) + while (rte_ring_mc_dequeue_bulk(r, burst, size, NULL) == 0) + rte_pause(); + const uint64_t mc_end = rte_rdtsc(); + + params->spsc = ((double)(sc_end - sc_start))/(iterations*size); + params->mpmc = ((double)(mc_end - mc_start))/(iterations*size); + return 0; +} + +/* + * Function that calls the enqueue and dequeue bulk functions on pairs of cores. + * used to measure ring perf between hyperthreads, cores and sockets. + */ +static void +run_on_core_pair(struct lcore_pair *cores, struct rte_ring *r, + lcore_function_t f1, lcore_function_t f2) +{ + struct thread_params param1 = {0}, param2 = {0}; + unsigned i; + for (i = 0; i < sizeof(bulk_sizes)/sizeof(bulk_sizes[0]); i++) { + lcore_count = 0; + param1.size = param2.size = bulk_sizes[i]; + param1.r = param2.r = r; + if (cores->c1 == rte_get_master_lcore()) { + rte_eal_remote_launch(f2, ¶m2, cores->c2); + f1(¶m1); + rte_eal_wait_lcore(cores->c2); + } else { + rte_eal_remote_launch(f1, ¶m1, cores->c1); + rte_eal_remote_launch(f2, ¶m2, cores->c2); + rte_eal_wait_lcore(cores->c1); + rte_eal_wait_lcore(cores->c2); + } + printf("SP/SC bulk enq/dequeue (size: %u): %.2F\n", bulk_sizes[i], + param1.spsc + param2.spsc); + printf("MP/MC bulk enq/dequeue (size: %u): %.2F\n", bulk_sizes[i], + param1.mpmc + param2.mpmc); + } +} + +/* + * Test function that determines how long an enqueue + dequeue of a single item + * takes on a single lcore. Result is for comparison with the bulk enq+deq. + */ +static void +test_single_enqueue_dequeue(struct rte_ring *r) +{ + const unsigned iter_shift = 24; + const unsigned iterations = 1<<iter_shift; + unsigned i = 0; + void *burst = NULL; + + const uint64_t sc_start = rte_rdtsc(); + for (i = 0; i < iterations; i++) { + rte_ring_sp_enqueue(r, burst); + rte_ring_sc_dequeue(r, &burst); + } + const uint64_t sc_end = rte_rdtsc(); + + const uint64_t mc_start = rte_rdtsc(); + for (i = 0; i < iterations; i++) { + rte_ring_mp_enqueue(r, burst); + rte_ring_mc_dequeue(r, &burst); + } + const uint64_t mc_end = rte_rdtsc(); + + printf("SP/SC single enq/dequeue: %"PRIu64"\n", + (sc_end-sc_start) >> iter_shift); + printf("MP/MC single enq/dequeue: %"PRIu64"\n", + (mc_end-mc_start) >> iter_shift); +} + +/* + * Test that does both enqueue and dequeue on a core using the burst() API calls + * instead of the bulk() calls used in other tests. Results should be the same + * as for the bulk function called on a single lcore. + */ +static void +test_burst_enqueue_dequeue(struct rte_ring *r) +{ + const unsigned iter_shift = 23; + const unsigned iterations = 1<<iter_shift; + unsigned sz, i = 0; + void *burst[MAX_BURST] = {0}; + + for (sz = 0; sz < sizeof(bulk_sizes)/sizeof(bulk_sizes[0]); sz++) { + const uint64_t sc_start = rte_rdtsc(); + for (i = 0; i < iterations; i++) { + rte_ring_sp_enqueue_burst(r, burst, + bulk_sizes[sz], NULL); + rte_ring_sc_dequeue_burst(r, burst, + bulk_sizes[sz], NULL); + } + const uint64_t sc_end = rte_rdtsc(); + + const uint64_t mc_start = rte_rdtsc(); + for (i = 0; i < iterations; i++) { + rte_ring_mp_enqueue_burst(r, burst, + bulk_sizes[sz], NULL); + rte_ring_mc_dequeue_burst(r, burst, + bulk_sizes[sz], NULL); + } + const uint64_t mc_end = rte_rdtsc(); + + uint64_t mc_avg = ((mc_end-mc_start) >> iter_shift) / bulk_sizes[sz]; + uint64_t sc_avg = ((sc_end-sc_start) >> iter_shift) / bulk_sizes[sz]; + + printf("SP/SC burst enq/dequeue (size: %u): %"PRIu64"\n", bulk_sizes[sz], + sc_avg); + printf("MP/MC burst enq/dequeue (size: %u): %"PRIu64"\n", bulk_sizes[sz], + mc_avg); + } +} + +/* Times enqueue and dequeue on a single lcore */ +static void +test_bulk_enqueue_dequeue(struct rte_ring *r) +{ + const unsigned iter_shift = 23; + const unsigned iterations = 1<<iter_shift; + unsigned sz, i = 0; + void *burst[MAX_BURST] = {0}; + + for (sz = 0; sz < sizeof(bulk_sizes)/sizeof(bulk_sizes[0]); sz++) { + const uint64_t sc_start = rte_rdtsc(); + for (i = 0; i < iterations; i++) { + rte_ring_sp_enqueue_bulk(r, burst, + bulk_sizes[sz], NULL); + rte_ring_sc_dequeue_bulk(r, burst, + bulk_sizes[sz], NULL); + } + const uint64_t sc_end = rte_rdtsc(); + + const uint64_t mc_start = rte_rdtsc(); + for (i = 0; i < iterations; i++) { + rte_ring_mp_enqueue_bulk(r, burst, + bulk_sizes[sz], NULL); + rte_ring_mc_dequeue_bulk(r, burst, + bulk_sizes[sz], NULL); + } + const uint64_t mc_end = rte_rdtsc(); + + double sc_avg = ((double)(sc_end-sc_start) / + (iterations * bulk_sizes[sz])); + double mc_avg = ((double)(mc_end-mc_start) / + (iterations * bulk_sizes[sz])); + + printf("SP/SC bulk enq/dequeue (size: %u): %.2F\n", bulk_sizes[sz], + sc_avg); + printf("MP/MC bulk enq/dequeue (size: %u): %.2F\n", bulk_sizes[sz], + mc_avg); + } +} + +static int +test_ring_perf(void) +{ + struct lcore_pair cores; + struct rte_ring *r = NULL; + + r = rte_ring_create(RING_NAME, RING_SIZE, rte_socket_id(), 0); + if (r == NULL) + return -1; + + printf("### Testing single element and burst enq/deq ###\n"); + test_single_enqueue_dequeue(r); + test_burst_enqueue_dequeue(r); + + printf("\n### Testing empty dequeue ###\n"); + test_empty_dequeue(r); + + printf("\n### Testing using a single lcore ###\n"); + test_bulk_enqueue_dequeue(r); + + if (get_two_hyperthreads(&cores) == 0) { + printf("\n### Testing using two hyperthreads ###\n"); + run_on_core_pair(&cores, r, enqueue_bulk, dequeue_bulk); + } + if (get_two_cores(&cores) == 0) { + printf("\n### Testing using two physical cores ###\n"); + run_on_core_pair(&cores, r, enqueue_bulk, dequeue_bulk); + } + if (get_two_sockets(&cores) == 0) { + printf("\n### Testing using two NUMA nodes ###\n"); + run_on_core_pair(&cores, r, enqueue_bulk, dequeue_bulk); + } + rte_ring_free(r); + return 0; +} + +REGISTER_TEST_COMMAND(ring_perf_autotest, test_ring_perf); diff --git a/dpdk/test/test/test_rwlock.c b/dpdk/test/test/test_rwlock.c new file mode 100644 index 00000000..08685d32 --- /dev/null +++ b/dpdk/test/test/test_rwlock.c @@ -0,0 +1,130 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <stdint.h> +#include <unistd.h> +#include <sys/queue.h> + +#include <rte_common.h> +#include <rte_memory.h> +#include <rte_per_lcore.h> +#include <rte_launch.h> +#include <rte_atomic.h> +#include <rte_rwlock.h> +#include <rte_eal.h> +#include <rte_lcore.h> +#include <rte_cycles.h> + +#include "test.h" + +/* + * rwlock test + * =========== + * + * - There is a global rwlock and a table of rwlocks (one per lcore). + * + * - The test function takes all of these locks and launches the + * ``test_rwlock_per_core()`` function on each core (except the master). + * + * - The function takes the global write lock, display something, + * then releases the global lock. + * - Then, it takes the per-lcore write lock, display something, and + * releases the per-core lock. + * - Finally, a read lock is taken during 100 ms, then released. + * + * - The main function unlocks the per-lcore locks sequentially and + * waits between each lock. This triggers the display of a message + * for each core, in the correct order. + * + * Then, it tries to take the global write lock and display the last + * message. The autotest script checks that the message order is correct. + */ + +static rte_rwlock_t sl; +static rte_rwlock_t sl_tab[RTE_MAX_LCORE]; + +static int +test_rwlock_per_core(__attribute__((unused)) void *arg) +{ + rte_rwlock_write_lock(&sl); + printf("Global write lock taken on core %u\n", rte_lcore_id()); + rte_rwlock_write_unlock(&sl); + + rte_rwlock_write_lock(&sl_tab[rte_lcore_id()]); + printf("Hello from core %u !\n", rte_lcore_id()); + rte_rwlock_write_unlock(&sl_tab[rte_lcore_id()]); + + rte_rwlock_read_lock(&sl); + printf("Global read lock taken on core %u\n", rte_lcore_id()); + rte_delay_ms(100); + printf("Release global read lock on core %u\n", rte_lcore_id()); + rte_rwlock_read_unlock(&sl); + + return 0; +} + +static int +test_rwlock(void) +{ + int i; + + rte_rwlock_init(&sl); + for (i=0; i<RTE_MAX_LCORE; i++) + rte_rwlock_init(&sl_tab[i]); + + rte_rwlock_write_lock(&sl); + + RTE_LCORE_FOREACH_SLAVE(i) { + rte_rwlock_write_lock(&sl_tab[i]); + rte_eal_remote_launch(test_rwlock_per_core, NULL, i); + } + + rte_rwlock_write_unlock(&sl); + + RTE_LCORE_FOREACH_SLAVE(i) { + rte_rwlock_write_unlock(&sl_tab[i]); + rte_delay_ms(100); + } + + rte_rwlock_write_lock(&sl); + /* this message should be the last message of test */ + printf("Global write lock taken on master core %u\n", rte_lcore_id()); + rte_rwlock_write_unlock(&sl); + + rte_eal_mp_wait_lcore(); + + return 0; +} + +REGISTER_TEST_COMMAND(rwlock_autotest, test_rwlock); diff --git a/dpdk/test/test/test_sched.c b/dpdk/test/test/test_sched.c new file mode 100644 index 00000000..bd2776d3 --- /dev/null +++ b/dpdk/test/test/test_sched.c @@ -0,0 +1,216 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdlib.h> +#include <stdio.h> +#include <string.h> +#include <stdint.h> +#include <unistd.h> + +#include "test.h" + +#include <rte_cycles.h> +#include <rte_ether.h> +#include <rte_ip.h> +#include <rte_byteorder.h> +#include <rte_sched.h> + + +#define SUBPORT 0 +#define PIPE 1 +#define TC 2 +#define QUEUE 3 + +static struct rte_sched_subport_params subport_param[] = { + { + .tb_rate = 1250000000, + .tb_size = 1000000, + + .tc_rate = {1250000000, 1250000000, 1250000000, 1250000000}, + .tc_period = 10, + }, +}; + +static struct rte_sched_pipe_params pipe_profile[] = { + { /* Profile #0 */ + .tb_rate = 305175, + .tb_size = 1000000, + + .tc_rate = {305175, 305175, 305175, 305175}, + .tc_period = 40, + + .wrr_weights = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}, + }, +}; + +static struct rte_sched_port_params port_param = { + .socket = 0, /* computed */ + .rate = 0, /* computed */ + .mtu = 1522, + .frame_overhead = RTE_SCHED_FRAME_OVERHEAD_DEFAULT, + .n_subports_per_port = 1, + .n_pipes_per_subport = 1024, + .qsize = {32, 32, 32, 32}, + .pipe_profiles = pipe_profile, + .n_pipe_profiles = 1, +}; + +#define NB_MBUF 32 +#define MBUF_DATA_SZ (2048 + RTE_PKTMBUF_HEADROOM) +#define MEMPOOL_CACHE_SZ 0 +#define SOCKET 0 + + +static struct rte_mempool * +create_mempool(void) +{ + struct rte_mempool * mp; + + mp = rte_mempool_lookup("test_sched"); + if (!mp) + mp = rte_pktmbuf_pool_create("test_sched", NB_MBUF, + MEMPOOL_CACHE_SZ, 0, MBUF_DATA_SZ, SOCKET); + + return mp; +} + +static void +prepare_pkt(struct rte_mbuf *mbuf) +{ + struct ether_hdr *eth_hdr; + struct vlan_hdr *vlan1, *vlan2; + struct ipv4_hdr *ip_hdr; + + /* Simulate a classifier */ + eth_hdr = rte_pktmbuf_mtod(mbuf, struct ether_hdr *); + vlan1 = (struct vlan_hdr *)(ð_hdr->ether_type ); + vlan2 = (struct vlan_hdr *)((uintptr_t)ð_hdr->ether_type + sizeof(struct vlan_hdr)); + eth_hdr = (struct ether_hdr *)((uintptr_t)ð_hdr->ether_type + 2 *sizeof(struct vlan_hdr)); + ip_hdr = (struct ipv4_hdr *)((uintptr_t)eth_hdr + sizeof(eth_hdr->ether_type)); + + vlan1->vlan_tci = rte_cpu_to_be_16(SUBPORT); + vlan2->vlan_tci = rte_cpu_to_be_16(PIPE); + eth_hdr->ether_type = rte_cpu_to_be_16(ETHER_TYPE_IPv4); + ip_hdr->dst_addr = IPv4(0,0,TC,QUEUE); + + + rte_sched_port_pkt_write(mbuf, SUBPORT, PIPE, TC, QUEUE, e_RTE_METER_YELLOW); + + /* 64 byte packet */ + mbuf->pkt_len = 60; + mbuf->data_len = 60; +} + + +/** + * test main entrance for library sched + */ +static int +test_sched(void) +{ + struct rte_mempool *mp = NULL; + struct rte_sched_port *port = NULL; + uint32_t pipe; + struct rte_mbuf *in_mbufs[10]; + struct rte_mbuf *out_mbufs[10]; + int i; + + int err; + + mp = create_mempool(); + TEST_ASSERT_NOT_NULL(mp, "Error creating mempool\n"); + + port_param.socket = 0; + port_param.rate = (uint64_t) 10000 * 1000 * 1000 / 8; + + port = rte_sched_port_config(&port_param); + TEST_ASSERT_NOT_NULL(port, "Error config sched port\n"); + + err = rte_sched_subport_config(port, SUBPORT, subport_param); + TEST_ASSERT_SUCCESS(err, "Error config sched, err=%d\n", err); + + for (pipe = 0; pipe < port_param.n_pipes_per_subport; pipe ++) { + err = rte_sched_pipe_config(port, SUBPORT, pipe, 0); + TEST_ASSERT_SUCCESS(err, "Error config sched pipe %u, err=%d\n", pipe, err); + } + + for (i = 0; i < 10; i++) { + in_mbufs[i] = rte_pktmbuf_alloc(mp); + TEST_ASSERT_NOT_NULL(in_mbufs[i], "Packet allocation failed\n"); + prepare_pkt(in_mbufs[i]); + } + + + err = rte_sched_port_enqueue(port, in_mbufs, 10); + TEST_ASSERT_EQUAL(err, 10, "Wrong enqueue, err=%d\n", err); + + err = rte_sched_port_dequeue(port, out_mbufs, 10); + TEST_ASSERT_EQUAL(err, 10, "Wrong dequeue, err=%d\n", err); + + for (i = 0; i < 10; i++) { + enum rte_meter_color color; + uint32_t subport, traffic_class, queue; + + color = rte_sched_port_pkt_read_color(out_mbufs[i]); + TEST_ASSERT_EQUAL(color, e_RTE_METER_YELLOW, "Wrong color\n"); + + rte_sched_port_pkt_read_tree_path(out_mbufs[i], + &subport, &pipe, &traffic_class, &queue); + + TEST_ASSERT_EQUAL(subport, SUBPORT, "Wrong subport\n"); + TEST_ASSERT_EQUAL(pipe, PIPE, "Wrong pipe\n"); + TEST_ASSERT_EQUAL(traffic_class, TC, "Wrong traffic_class\n"); + TEST_ASSERT_EQUAL(queue, QUEUE, "Wrong queue\n"); + + } + + + struct rte_sched_subport_stats subport_stats; + uint32_t tc_ov; + rte_sched_subport_read_stats(port, SUBPORT, &subport_stats, &tc_ov); +#if 0 + TEST_ASSERT_EQUAL(subport_stats.n_pkts_tc[TC-1], 10, "Wrong subport stats\n"); +#endif + struct rte_sched_queue_stats queue_stats; + uint16_t qlen; + rte_sched_queue_read_stats(port, QUEUE, &queue_stats, &qlen); +#if 0 + TEST_ASSERT_EQUAL(queue_stats.n_pkts, 10, "Wrong queue stats\n"); +#endif + + rte_sched_port_free(port); + + return 0; +} + +REGISTER_TEST_COMMAND(sched_autotest, test_sched); diff --git a/dpdk/test/test/test_service_cores.c b/dpdk/test/test/test_service_cores.c new file mode 100644 index 00000000..ebff8b0a --- /dev/null +++ b/dpdk/test/test/test_service_cores.c @@ -0,0 +1,743 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <rte_common.h> +#include <rte_hexdump.h> +#include <rte_mbuf.h> +#include <rte_malloc.h> +#include <rte_memcpy.h> +#include <rte_cycles.h> + +#include <rte_service.h> +#include <rte_service_component.h> + +#include "test.h" + +/* used as the service core ID */ +static uint32_t slcore_id; +/* used as timestamp to detect if a service core is running */ +static uint64_t service_tick; +/* used as a flag to check if a function was run */ +static uint32_t service_remote_launch_flag; + +#define SERVICE_DELAY 1 + +#define DUMMY_SERVICE_NAME "dummy_service" +#define MT_SAFE_SERVICE_NAME "mt_safe_service" + +static int +testsuite_setup(void) +{ + slcore_id = rte_get_next_lcore(/* start core */ -1, + /* skip master */ 1, + /* wrap */ 0); + + return TEST_SUCCESS; +} + +static void +testsuite_teardown(void) +{ + /* release service cores? */ +} + +static int32_t dummy_cb(void *args) +{ + RTE_SET_USED(args); + service_tick++; + rte_delay_ms(SERVICE_DELAY); + return 0; +} + +static int32_t dummy_mt_unsafe_cb(void *args) +{ + /* before running test, the initialization has set pass_test to 1. + * If the cmpset in service-cores is working correctly, the code here + * should never fail to take the lock. If the lock *is* taken, fail the + * test, because two threads are concurrently in a non-MT safe callback. + */ + uint32_t *test_params = args; + uint32_t *atomic_lock = &test_params[0]; + uint32_t *pass_test = &test_params[1]; + int lock_taken = rte_atomic32_cmpset(atomic_lock, 0, 1); + if (lock_taken) { + /* delay with the lock held */ + rte_delay_ms(250); + rte_atomic32_clear((rte_atomic32_t *)atomic_lock); + } else { + /* 2nd thread will fail to take lock, so set pass flag */ + *pass_test = 0; + } + + return 0; +} + + +static int32_t dummy_mt_safe_cb(void *args) +{ + /* Atomic checks to ensure MT safe services allow > 1 thread to + * concurrently run the callback. The concept is as follows; + * 1) if lock is available, take the lock then delay + * 2) if first lock is taken, and a thread arrives in the CB, we know + * that 2 threads are running the callback at the same time: MT safe + */ + uint32_t *test_params = args; + uint32_t *atomic_lock = &test_params[0]; + uint32_t *pass_test = &test_params[1]; + int lock_taken = rte_atomic32_cmpset(atomic_lock, 0, 1); + if (lock_taken) { + /* delay with the lock held */ + rte_delay_ms(250); + rte_atomic32_clear((rte_atomic32_t *)atomic_lock); + } else { + /* 2nd thread will fail to take lock, so set pass flag */ + *pass_test = 1; + } + + return 0; +} + +/* unregister all services */ +static int +unregister_all(void) +{ + uint32_t i; + + TEST_ASSERT_EQUAL(-EINVAL, rte_service_component_unregister(1000), + "Unregistered invalid service id"); + + uint32_t c = rte_service_get_count(); + for (i = 0; i < c; i++) { + TEST_ASSERT_EQUAL(0, rte_service_component_unregister(i), + "Error unregistering a valid service"); + } + + rte_service_lcore_reset_all(); + + return TEST_SUCCESS; +} + +/* register a single dummy service */ +static int +dummy_register(void) +{ + /* make sure there are no remains from previous tests */ + unregister_all(); + + struct rte_service_spec service; + memset(&service, 0, sizeof(struct rte_service_spec)); + + TEST_ASSERT_EQUAL(-EINVAL, + rte_service_component_register(&service, NULL), + "Invalid callback"); + service.callback = dummy_cb; + + TEST_ASSERT_EQUAL(-EINVAL, + rte_service_component_register(&service, NULL), + "Invalid name"); + snprintf(service.name, sizeof(service.name), DUMMY_SERVICE_NAME); + + uint32_t id; + TEST_ASSERT_EQUAL(0, rte_service_component_register(&service, &id), + "Failed to register valid service"); + + rte_service_component_runstate_set(id, 1); + + return TEST_SUCCESS; +} + +/* verify get_by_name() service lookup */ +static int +service_get_by_name(void) +{ + unregister_all(); + + uint32_t sid; + TEST_ASSERT_EQUAL(-ENODEV, + rte_service_get_by_name(DUMMY_SERVICE_NAME, &sid), + "get by name with invalid name should return -ENODEV"); + TEST_ASSERT_EQUAL(-EINVAL, + rte_service_get_by_name(DUMMY_SERVICE_NAME, 0x0), + "get by name with NULL ptr should return -ENODEV"); + + /* register service */ + struct rte_service_spec service; + memset(&service, 0, sizeof(struct rte_service_spec)); + TEST_ASSERT_EQUAL(-EINVAL, + rte_service_component_register(&service, NULL), + "Invalid callback"); + service.callback = dummy_cb; + TEST_ASSERT_EQUAL(-EINVAL, + rte_service_component_register(&service, NULL), + "Invalid name"); + snprintf(service.name, sizeof(service.name), DUMMY_SERVICE_NAME); + TEST_ASSERT_EQUAL(0, rte_service_component_register(&service, NULL), + "Failed to register valid service"); + + /* we unregistered all service, now registering 1, should be id 0 */ + uint32_t service_id_as_expected = 0; + TEST_ASSERT_EQUAL(0, rte_service_get_by_name(DUMMY_SERVICE_NAME, &sid), + "Service get_by_name should return 0 on valid inputs"); + TEST_ASSERT_EQUAL(service_id_as_expected, sid, + "Service get_by_name should equal expected id"); + + unregister_all(); + + /* ensure after unregister, get_by_name returns NULL */ + TEST_ASSERT_EQUAL(-ENODEV, + rte_service_get_by_name(DUMMY_SERVICE_NAME, &sid), + "get by name should return -ENODEV after unregister"); + + return TEST_SUCCESS; +} + +/* verify probe of capabilities */ +static int +service_probe_capability(void) +{ + unregister_all(); + + struct rte_service_spec service; + memset(&service, 0, sizeof(struct rte_service_spec)); + service.callback = dummy_cb; + snprintf(service.name, sizeof(service.name), DUMMY_SERVICE_NAME); + service.capabilities |= RTE_SERVICE_CAP_MT_SAFE; + TEST_ASSERT_EQUAL(0, rte_service_component_register(&service, NULL), + "Register of MT SAFE service failed"); + + /* verify flag is enabled */ + const uint32_t sid = 0; + int32_t mt = rte_service_probe_capability(sid, RTE_SERVICE_CAP_MT_SAFE); + TEST_ASSERT_EQUAL(1, mt, "MT SAFE capability flag not set."); + + + unregister_all(); + + memset(&service, 0, sizeof(struct rte_service_spec)); + service.callback = dummy_cb; + snprintf(service.name, sizeof(service.name), DUMMY_SERVICE_NAME); + TEST_ASSERT_EQUAL(0, rte_service_component_register(&service, NULL), + "Register of non-MT safe service failed"); + + /* verify flag is enabled */ + mt = rte_service_probe_capability(sid, RTE_SERVICE_CAP_MT_SAFE); + TEST_ASSERT_EQUAL(0, mt, "MT SAFE cap flag set on non MT SAFE service"); + + return unregister_all(); +} + +/* verify the service name */ +static int +service_name(void) +{ + const char *name = rte_service_get_name(0); + int equal = strcmp(name, DUMMY_SERVICE_NAME); + TEST_ASSERT_EQUAL(0, equal, "Error: Service name not correct"); + + return unregister_all(); +} + +/* verify service dump */ +static int +service_dump(void) +{ + const uint32_t sid = 0; + rte_service_set_stats_enable(sid, 1); + rte_service_dump(stdout, 0); + rte_service_set_stats_enable(sid, 0); + rte_service_dump(stdout, 0); + return unregister_all(); +} + +/* start and stop a service */ +static int +service_start_stop(void) +{ + const uint32_t sid = 0; + + /* runstate_get() returns if service is running and slcore is mapped */ + TEST_ASSERT_EQUAL(0, rte_service_lcore_add(slcore_id), + "Service core add did not return zero"); + int ret = rte_service_map_lcore_set(sid, slcore_id, 1); + TEST_ASSERT_EQUAL(0, ret, + "Enabling service core, expected 0 got %d", ret); + + TEST_ASSERT_EQUAL(0, rte_service_runstate_get(sid), + "Error: Service should be stopped"); + + TEST_ASSERT_EQUAL(0, rte_service_runstate_set(sid, 0), + "Error: Service stopped returned non-zero"); + + TEST_ASSERT_EQUAL(0, rte_service_runstate_get(sid), + "Error: Service is running - should be stopped"); + + TEST_ASSERT_EQUAL(0, rte_service_runstate_set(sid, 1), + "Error: Service start returned non-zero"); + + TEST_ASSERT_EQUAL(1, rte_service_runstate_get(sid), + "Error: Service is not running"); + + return unregister_all(); +} + + +static int +service_remote_launch_func(void *arg) +{ + RTE_SET_USED(arg); + service_remote_launch_flag = 1; + return 0; +} + +/* enable and disable a lcore for a service */ +static int +service_lcore_en_dis_able(void) +{ + const uint32_t sid = 0; + + /* expected failure cases */ + TEST_ASSERT_EQUAL(-EINVAL, rte_service_map_lcore_set(sid, 100000, 1), + "Enable on invalid core did not fail"); + TEST_ASSERT_EQUAL(-EINVAL, rte_service_map_lcore_set(sid, 100000, 0), + "Disable on invalid core did not fail"); + + /* add service core to allow enabling */ + TEST_ASSERT_EQUAL(0, rte_service_lcore_add(slcore_id), + "Add service core failed when not in use before"); + + /* valid enable */ + TEST_ASSERT_EQUAL(0, rte_service_map_lcore_set(sid, slcore_id, 1), + "Enabling valid service and core failed"); + TEST_ASSERT_EQUAL(1, rte_service_map_lcore_get(sid, slcore_id), + "Enabled core returned not-enabled"); + + /* valid disable */ + TEST_ASSERT_EQUAL(0, rte_service_map_lcore_set(sid, slcore_id, 0), + "Disabling valid service and lcore failed"); + TEST_ASSERT_EQUAL(0, rte_service_map_lcore_get(sid, slcore_id), + "Disabled core returned enabled"); + + /* call remote_launch to verify that app can launch ex-service lcore */ + service_remote_launch_flag = 0; + rte_eal_wait_lcore(slcore_id); + int ret = rte_eal_remote_launch(service_remote_launch_func, NULL, + slcore_id); + TEST_ASSERT_EQUAL(0, ret, "Ex-service core remote launch failed."); + rte_eal_mp_wait_lcore(); + TEST_ASSERT_EQUAL(1, service_remote_launch_flag, + "Ex-service core function call had no effect."); + + return unregister_all(); +} + +static int +service_lcore_running_check(void) +{ + uint64_t tick = service_tick; + rte_delay_ms(SERVICE_DELAY * 100); + /* if (tick != service_tick) we know the lcore as polled the service */ + return tick != service_tick; +} + +static int +service_lcore_add_del(void) +{ + /* check initial count */ + TEST_ASSERT_EQUAL(0, rte_service_lcore_count(), + "Service lcore count has value before adding a lcore"); + + /* check service lcore add */ + TEST_ASSERT_EQUAL(0, rte_service_lcore_add(slcore_id), + "Add service core failed when not in use before"); + TEST_ASSERT_EQUAL(-EALREADY, rte_service_lcore_add(slcore_id), + "Add service core failed to refuse in-use lcore"); + + /* check count */ + TEST_ASSERT_EQUAL(1, rte_service_lcore_count(), + "Service core count not equal to one"); + + /* retrieve core list, checking lcore ids */ + const uint32_t size = 4; + uint32_t service_core_ids[size]; + int32_t n = rte_service_lcore_list(service_core_ids, size); + TEST_ASSERT_EQUAL(1, n, "Service core list return should equal 1"); + TEST_ASSERT_EQUAL(slcore_id, service_core_ids[0], + "Service core list lcore must equal slcore_id"); + + /* recheck count, add more cores, and check count */ + TEST_ASSERT_EQUAL(1, rte_service_lcore_count(), + "Service core count not equal to one"); + uint32_t slcore_1 = rte_get_next_lcore(/* start core */ -1, + /* skip master */ 1, + /* wrap */ 0); + TEST_ASSERT_EQUAL(0, rte_service_lcore_add(slcore_1), + "Service core add did not return zero"); + uint32_t slcore_2 = rte_get_next_lcore(/* start core */ slcore_1, + /* skip master */ 1, + /* wrap */ 0); + TEST_ASSERT_EQUAL(0, rte_service_lcore_add(slcore_2), + "Service core add did not return zero"); + + uint32_t count = rte_service_lcore_count(); + const uint32_t cores_at_this_point = 3; + TEST_ASSERT_EQUAL(cores_at_this_point, count, + "Service core count %d, expected %d", count, + cores_at_this_point); + + /* check longer service core list */ + n = rte_service_lcore_list(service_core_ids, size); + TEST_ASSERT_EQUAL(3, n, "Service core list return should equal 3"); + TEST_ASSERT_EQUAL(slcore_id, service_core_ids[0], + "Service core list[0] lcore must equal 1"); + TEST_ASSERT_EQUAL(slcore_1, service_core_ids[1], + "Service core list[1] lcore must equal 2"); + TEST_ASSERT_EQUAL(slcore_2, service_core_ids[2], + "Service core list[2] lcore must equal 3"); + + /* recheck count, remove lcores, check remaining lcore_id is correct */ + TEST_ASSERT_EQUAL(3, rte_service_lcore_count(), + "Service core count not equal to three"); + TEST_ASSERT_EQUAL(0, rte_service_lcore_del(slcore_1), + "Service core add did not return zero"); + TEST_ASSERT_EQUAL(0, rte_service_lcore_del(slcore_2), + "Service core add did not return zero"); + TEST_ASSERT_EQUAL(1, rte_service_lcore_count(), + "Service core count not equal to one"); + n = rte_service_lcore_list(service_core_ids, size); + TEST_ASSERT_EQUAL(1, n, "Service core list return should equal one"); + TEST_ASSERT_EQUAL(slcore_id, service_core_ids[0], + "Service core list[0] lcore must equal %d", + slcore_id); + + return unregister_all(); +} + +static int +service_threaded_test(int mt_safe) +{ + unregister_all(); + + /* add next 2 cores */ + uint32_t slcore_1 = rte_get_next_lcore(/* start core */ -1, + /* skip master */ 1, + /* wrap */ 0); + TEST_ASSERT_EQUAL(0, rte_service_lcore_add(slcore_1), + "mt safe lcore add fail"); + uint32_t slcore_2 = rte_get_next_lcore(/* start core */ slcore_1, + /* skip master */ 1, + /* wrap */ 0); + TEST_ASSERT_EQUAL(0, rte_service_lcore_add(slcore_2), + "mt safe lcore add fail"); + + /* Use atomic locks to verify that two threads are in the same function + * at the same time. These are passed to the unit tests through the + * callback userdata parameter + */ + uint32_t test_params[2]; + memset(test_params, 0, sizeof(uint32_t) * 2); + + /* register MT safe service. */ + struct rte_service_spec service; + memset(&service, 0, sizeof(struct rte_service_spec)); + service.callback_userdata = test_params; + snprintf(service.name, sizeof(service.name), MT_SAFE_SERVICE_NAME); + + if (mt_safe) { + service.callback = dummy_mt_safe_cb; + service.capabilities |= RTE_SERVICE_CAP_MT_SAFE; + } else + service.callback = dummy_mt_unsafe_cb; + + uint32_t id; + TEST_ASSERT_EQUAL(0, rte_service_component_register(&service, &id), + "Register of MT SAFE service failed"); + + const uint32_t sid = 0; + TEST_ASSERT_EQUAL(0, rte_service_runstate_set(sid, 1), + "Starting valid service failed"); + TEST_ASSERT_EQUAL(0, rte_service_map_lcore_set(sid, slcore_1, 1), + "Failed to enable lcore 1 on mt safe service"); + TEST_ASSERT_EQUAL(0, rte_service_map_lcore_set(sid, slcore_2, 1), + "Failed to enable lcore 2 on mt safe service"); + rte_service_lcore_start(slcore_1); + rte_service_lcore_start(slcore_2); + + /* wait for the worker threads to run */ + rte_delay_ms(500); + rte_service_lcore_stop(slcore_1); + rte_service_lcore_stop(slcore_2); + + TEST_ASSERT_EQUAL(0, test_params[1], + "Service run with component runstate = 0"); + + /* enable backend runstate: the service should run after this */ + rte_service_component_runstate_set(id, 1); + + /* initialize to pass, see callback comment for details */ + if (!mt_safe) + test_params[1] = 1; + + /* wait for lcores before start() */ + rte_eal_wait_lcore(slcore_1); + rte_eal_wait_lcore(slcore_2); + + rte_service_lcore_start(slcore_1); + rte_service_lcore_start(slcore_2); + + /* wait for the worker threads to run */ + rte_delay_ms(500); + rte_service_lcore_stop(slcore_1); + rte_service_lcore_stop(slcore_2); + + TEST_ASSERT_EQUAL(1, test_params[1], + "MT Safe service not run by two cores concurrently"); + TEST_ASSERT_EQUAL(0, rte_service_runstate_set(sid, 0), + "Failed to stop MT Safe service"); + + rte_eal_wait_lcore(slcore_1); + rte_eal_wait_lcore(slcore_2); + unregister_all(); + + /* return the value of the callback pass_test variable to caller */ + return test_params[1]; +} + +/* tests an MT SAFE service with two cores. The callback function ensures that + * two threads access the callback concurrently. + */ +static int +service_mt_safe_poll(void) +{ + int mt_safe = 1; + TEST_ASSERT_EQUAL(1, service_threaded_test(mt_safe), + "Error: MT Safe service not run by two cores concurrently"); + return TEST_SUCCESS; +} + +/* tests a NON mt safe service with two cores, the callback is serialized + * using the atomic cmpset. + */ +static int +service_mt_unsafe_poll(void) +{ + int mt_safe = 0; + TEST_ASSERT_EQUAL(1, service_threaded_test(mt_safe), + "Error: NON MT Safe service run by two cores concurrently"); + return TEST_SUCCESS; +} + +static int32_t +delay_as_a_mt_safe_service(void *args) +{ + RTE_SET_USED(args); + uint32_t *params = args; + + /* retrieve done flag and atomic lock to inc/dec */ + uint32_t *done = ¶ms[0]; + rte_atomic32_t *lock = (rte_atomic32_t *)¶ms[1]; + + while (!*done) { + rte_atomic32_inc(lock); + rte_delay_us(500); + if (rte_atomic32_read(lock) > 1) + /* pass: second core has simultaneously incremented */ + *done = 1; + rte_atomic32_dec(lock); + } + + return 0; +} + +static int32_t +delay_as_a_service(void *args) +{ + uint32_t *done = (uint32_t *)args; + while (!*done) + rte_delay_ms(5); + return 0; +} + +static int +service_run_on_app_core_func(void *arg) +{ + uint32_t *delay_service_id = (uint32_t *)arg; + return rte_service_run_iter_on_app_lcore(*delay_service_id, 1); +} + +static int +service_app_lcore_poll_impl(const int mt_safe) +{ + uint32_t params[2] = {0}; + + struct rte_service_spec service; + memset(&service, 0, sizeof(struct rte_service_spec)); + snprintf(service.name, sizeof(service.name), MT_SAFE_SERVICE_NAME); + if (mt_safe) { + service.callback = delay_as_a_mt_safe_service; + service.callback_userdata = params; + service.capabilities |= RTE_SERVICE_CAP_MT_SAFE; + } else { + service.callback = delay_as_a_service; + service.callback_userdata = ¶ms; + } + + uint32_t id; + TEST_ASSERT_EQUAL(0, rte_service_component_register(&service, &id), + "Register of app lcore delay service failed"); + + rte_service_component_runstate_set(id, 1); + rte_service_runstate_set(id, 1); + + uint32_t app_core2 = rte_get_next_lcore(slcore_id, 1, 1); + rte_eal_wait_lcore(app_core2); + int app_core2_ret = rte_eal_remote_launch(service_run_on_app_core_func, + &id, app_core2); + + rte_delay_ms(100); + + int app_core1_ret = service_run_on_app_core_func(&id); + + /* flag done, then wait for the spawned 2nd core to return */ + params[0] = 1; + rte_eal_mp_wait_lcore(); + + /* core two gets launched first - and should hold the service lock */ + TEST_ASSERT_EQUAL(0, app_core2_ret, + "App core2 : run service didn't return zero"); + + if (mt_safe) { + /* mt safe should have both cores return 0 for success */ + TEST_ASSERT_EQUAL(0, app_core1_ret, + "MT Safe: App core1 didn't return 0"); + } else { + /* core one attempts to run later - should be blocked */ + TEST_ASSERT_EQUAL(-EBUSY, app_core1_ret, + "MT Unsafe: App core1 didn't return -EBUSY"); + } + + unregister_all(); + + return TEST_SUCCESS; +} + +static int +service_app_lcore_mt_safe(void) +{ + const int mt_safe = 1; + return service_app_lcore_poll_impl(mt_safe); +} + +static int +service_app_lcore_mt_unsafe(void) +{ + const int mt_safe = 0; + return service_app_lcore_poll_impl(mt_safe); +} + +/* start and stop a service core - ensuring it goes back to sleep */ +static int +service_lcore_start_stop(void) +{ + /* start service core and service, create mapping so tick() runs */ + const uint32_t sid = 0; + TEST_ASSERT_EQUAL(0, rte_service_runstate_set(sid, 1), + "Starting valid service failed"); + TEST_ASSERT_EQUAL(-EINVAL, rte_service_map_lcore_set(sid, slcore_id, 1), + "Enabling valid service on non-service core must fail"); + + /* core start */ + TEST_ASSERT_EQUAL(-EINVAL, rte_service_lcore_start(slcore_id), + "Service core start without add should return EINVAL"); + TEST_ASSERT_EQUAL(0, rte_service_lcore_add(slcore_id), + "Service core add did not return zero"); + TEST_ASSERT_EQUAL(0, rte_service_map_lcore_set(sid, slcore_id, 1), + "Enabling valid service on valid core failed"); + TEST_ASSERT_EQUAL(0, rte_service_lcore_start(slcore_id), + "Service core start after add failed"); + TEST_ASSERT_EQUAL(-EALREADY, rte_service_lcore_start(slcore_id), + "Service core expected as running but was stopped"); + + /* ensures core really is running the service function */ + TEST_ASSERT_EQUAL(1, service_lcore_running_check(), + "Service core expected to poll service but it didn't"); + + /* core stop */ + TEST_ASSERT_EQUAL(-EBUSY, rte_service_lcore_stop(slcore_id), + "Service core running a service should return -EBUSY"); + TEST_ASSERT_EQUAL(0, rte_service_runstate_set(sid, 0), + "Stopping valid service failed"); + TEST_ASSERT_EQUAL(-EINVAL, rte_service_lcore_stop(100000), + "Invalid Service core stop should return -EINVAL"); + TEST_ASSERT_EQUAL(0, rte_service_lcore_stop(slcore_id), + "Service core stop expected to return 0"); + TEST_ASSERT_EQUAL(-EALREADY, rte_service_lcore_stop(slcore_id), + "Already stopped service core should return -EALREADY"); + + /* ensure service is not longer running */ + TEST_ASSERT_EQUAL(0, service_lcore_running_check(), + "Service core expected to poll service but it didn't"); + + TEST_ASSERT_EQUAL(0, rte_service_lcore_del(slcore_id), + "Service core del did not return zero"); + + return unregister_all(); +} + +static struct unit_test_suite service_tests = { + .suite_name = "service core test suite", + .setup = testsuite_setup, + .teardown = testsuite_teardown, + .unit_test_cases = { + TEST_CASE_ST(dummy_register, NULL, unregister_all), + TEST_CASE_ST(dummy_register, NULL, service_name), + TEST_CASE_ST(dummy_register, NULL, service_get_by_name), + TEST_CASE_ST(dummy_register, NULL, service_dump), + TEST_CASE_ST(dummy_register, NULL, service_probe_capability), + TEST_CASE_ST(dummy_register, NULL, service_start_stop), + TEST_CASE_ST(dummy_register, NULL, service_lcore_add_del), + TEST_CASE_ST(dummy_register, NULL, service_lcore_start_stop), + TEST_CASE_ST(dummy_register, NULL, service_lcore_en_dis_able), + TEST_CASE_ST(dummy_register, NULL, service_mt_unsafe_poll), + TEST_CASE_ST(dummy_register, NULL, service_mt_safe_poll), + TEST_CASE_ST(dummy_register, NULL, service_app_lcore_mt_safe), + TEST_CASE_ST(dummy_register, NULL, service_app_lcore_mt_unsafe), + TEST_CASES_END() /**< NULL terminate unit test array */ + } +}; + +static int +test_service_common(void) +{ + return unit_test_suite_runner(&service_tests); +} + +REGISTER_TEST_COMMAND(service_autotest, test_service_common); diff --git a/dpdk/test/test/test_spinlock.c b/dpdk/test/test/test_spinlock.c new file mode 100644 index 00000000..bb34e0c6 --- /dev/null +++ b/dpdk/test/test/test_spinlock.c @@ -0,0 +1,334 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <stdint.h> +#include <inttypes.h> +#include <string.h> +#include <unistd.h> +#include <sys/queue.h> + +#include <rte_common.h> +#include <rte_memory.h> +#include <rte_per_lcore.h> +#include <rte_launch.h> +#include <rte_eal.h> +#include <rte_lcore.h> +#include <rte_cycles.h> +#include <rte_spinlock.h> +#include <rte_atomic.h> + +#include "test.h" + +/* + * Spinlock test + * ============= + * + * - There is a global spinlock and a table of spinlocks (one per lcore). + * + * - The test function takes all of these locks and launches the + * ``test_spinlock_per_core()`` function on each core (except the master). + * + * - The function takes the global lock, display something, then releases + * the global lock. + * - The function takes the per-lcore lock, display something, then releases + * the per-core lock. + * + * - The main function unlocks the per-lcore locks sequentially and + * waits between each lock. This triggers the display of a message + * for each core, in the correct order. The autotest script checks that + * this order is correct. + * + * - A load test is carried out, with all cores attempting to lock a single lock + * multiple times + */ + +static rte_spinlock_t sl, sl_try; +static rte_spinlock_t sl_tab[RTE_MAX_LCORE]; +static rte_spinlock_recursive_t slr; +static unsigned count = 0; + +static rte_atomic32_t synchro; + +static int +test_spinlock_per_core(__attribute__((unused)) void *arg) +{ + rte_spinlock_lock(&sl); + printf("Global lock taken on core %u\n", rte_lcore_id()); + rte_spinlock_unlock(&sl); + + rte_spinlock_lock(&sl_tab[rte_lcore_id()]); + printf("Hello from core %u !\n", rte_lcore_id()); + rte_spinlock_unlock(&sl_tab[rte_lcore_id()]); + + return 0; +} + +static int +test_spinlock_recursive_per_core(__attribute__((unused)) void *arg) +{ + unsigned id = rte_lcore_id(); + + rte_spinlock_recursive_lock(&slr); + printf("Global recursive lock taken on core %u - count = %d\n", + id, slr.count); + rte_spinlock_recursive_lock(&slr); + printf("Global recursive lock taken on core %u - count = %d\n", + id, slr.count); + rte_spinlock_recursive_lock(&slr); + printf("Global recursive lock taken on core %u - count = %d\n", + id, slr.count); + + printf("Hello from within recursive locks from core %u !\n", id); + + rte_spinlock_recursive_unlock(&slr); + printf("Global recursive lock released on core %u - count = %d\n", + id, slr.count); + rte_spinlock_recursive_unlock(&slr); + printf("Global recursive lock released on core %u - count = %d\n", + id, slr.count); + rte_spinlock_recursive_unlock(&slr); + printf("Global recursive lock released on core %u - count = %d\n", + id, slr.count); + + return 0; +} + +static rte_spinlock_t lk = RTE_SPINLOCK_INITIALIZER; +static uint64_t lock_count[RTE_MAX_LCORE] = {0}; + +#define TIME_MS 100 + +static int +load_loop_fn(void *func_param) +{ + uint64_t time_diff = 0, begin; + uint64_t hz = rte_get_timer_hz(); + uint64_t lcount = 0; + const int use_lock = *(int*)func_param; + const unsigned lcore = rte_lcore_id(); + + /* wait synchro for slaves */ + if (lcore != rte_get_master_lcore()) + while (rte_atomic32_read(&synchro) == 0); + + begin = rte_get_timer_cycles(); + while (time_diff < hz * TIME_MS / 1000) { + if (use_lock) + rte_spinlock_lock(&lk); + lcount++; + if (use_lock) + rte_spinlock_unlock(&lk); + /* delay to make lock duty cycle slighlty realistic */ + rte_delay_us(1); + time_diff = rte_get_timer_cycles() - begin; + } + lock_count[lcore] = lcount; + return 0; +} + +static int +test_spinlock_perf(void) +{ + unsigned int i; + uint64_t total = 0; + int lock = 0; + const unsigned lcore = rte_lcore_id(); + + printf("\nTest with no lock on single core...\n"); + load_loop_fn(&lock); + printf("Core [%u] count = %"PRIu64"\n", lcore, lock_count[lcore]); + memset(lock_count, 0, sizeof(lock_count)); + + printf("\nTest with lock on single core...\n"); + lock = 1; + load_loop_fn(&lock); + printf("Core [%u] count = %"PRIu64"\n", lcore, lock_count[lcore]); + memset(lock_count, 0, sizeof(lock_count)); + + printf("\nTest with lock on %u cores...\n", rte_lcore_count()); + + /* Clear synchro and start slaves */ + rte_atomic32_set(&synchro, 0); + rte_eal_mp_remote_launch(load_loop_fn, &lock, SKIP_MASTER); + + /* start synchro and launch test on master */ + rte_atomic32_set(&synchro, 1); + load_loop_fn(&lock); + + rte_eal_mp_wait_lcore(); + + RTE_LCORE_FOREACH(i) { + printf("Core [%u] count = %"PRIu64"\n", i, lock_count[i]); + total += lock_count[i]; + } + + printf("Total count = %"PRIu64"\n", total); + + return 0; +} + +/* + * Use rte_spinlock_trylock() to trylock a spinlock object, + * If it could not lock the object successfully, it would + * return immediately and the variable of "count" would be + * increased by one per times. the value of "count" could be + * checked as the result later. + */ +static int +test_spinlock_try(__attribute__((unused)) void *arg) +{ + if (rte_spinlock_trylock(&sl_try) == 0) { + rte_spinlock_lock(&sl); + count ++; + rte_spinlock_unlock(&sl); + } + + return 0; +} + + +/* + * Test rte_eal_get_lcore_state() in addition to spinlocks + * as we have "waiting" then "running" lcores. + */ +static int +test_spinlock(void) +{ + int ret = 0; + int i; + + /* slave cores should be waiting: print it */ + RTE_LCORE_FOREACH_SLAVE(i) { + printf("lcore %d state: %d\n", i, + (int) rte_eal_get_lcore_state(i)); + } + + rte_spinlock_init(&sl); + rte_spinlock_init(&sl_try); + rte_spinlock_recursive_init(&slr); + for (i=0; i<RTE_MAX_LCORE; i++) + rte_spinlock_init(&sl_tab[i]); + + rte_spinlock_lock(&sl); + + RTE_LCORE_FOREACH_SLAVE(i) { + rte_spinlock_lock(&sl_tab[i]); + rte_eal_remote_launch(test_spinlock_per_core, NULL, i); + } + + /* slave cores should be busy: print it */ + RTE_LCORE_FOREACH_SLAVE(i) { + printf("lcore %d state: %d\n", i, + (int) rte_eal_get_lcore_state(i)); + } + rte_spinlock_unlock(&sl); + + RTE_LCORE_FOREACH_SLAVE(i) { + rte_spinlock_unlock(&sl_tab[i]); + rte_delay_ms(10); + } + + rte_eal_mp_wait_lcore(); + + rte_spinlock_recursive_lock(&slr); + + /* + * Try to acquire a lock that we already own + */ + if(!rte_spinlock_recursive_trylock(&slr)) { + printf("rte_spinlock_recursive_trylock failed on a lock that " + "we already own\n"); + ret = -1; + } else + rte_spinlock_recursive_unlock(&slr); + + RTE_LCORE_FOREACH_SLAVE(i) { + rte_eal_remote_launch(test_spinlock_recursive_per_core, NULL, i); + } + rte_spinlock_recursive_unlock(&slr); + rte_eal_mp_wait_lcore(); + + /* + * Test if it could return immediately from try-locking a locked object. + * Here it will lock the spinlock object first, then launch all the slave + * lcores to trylock the same spinlock object. + * All the slave lcores should give up try-locking a locked object and + * return immediately, and then increase the "count" initialized with zero + * by one per times. + * We can check if the "count" is finally equal to the number of all slave + * lcores to see if the behavior of try-locking a locked spinlock object + * is correct. + */ + if (rte_spinlock_trylock(&sl_try) == 0) { + return -1; + } + count = 0; + RTE_LCORE_FOREACH_SLAVE(i) { + rte_eal_remote_launch(test_spinlock_try, NULL, i); + } + rte_eal_mp_wait_lcore(); + rte_spinlock_unlock(&sl_try); + if (rte_spinlock_is_locked(&sl)) { + printf("spinlock is locked but it should not be\n"); + return -1; + } + rte_spinlock_lock(&sl); + if (count != ( rte_lcore_count() - 1)) { + ret = -1; + } + rte_spinlock_unlock(&sl); + + /* + * Test if it can trylock recursively. + * Use rte_spinlock_recursive_trylock() to check if it can lock a spinlock + * object recursively. Here it will try to lock a spinlock object twice. + */ + if (rte_spinlock_recursive_trylock(&slr) == 0) { + printf("It failed to do the first spinlock_recursive_trylock but it should able to do\n"); + return -1; + } + if (rte_spinlock_recursive_trylock(&slr) == 0) { + printf("It failed to do the second spinlock_recursive_trylock but it should able to do\n"); + return -1; + } + rte_spinlock_recursive_unlock(&slr); + rte_spinlock_recursive_unlock(&slr); + + if (test_spinlock_perf() < 0) + return -1; + + return ret; +} + +REGISTER_TEST_COMMAND(spinlock_autotest, test_spinlock); diff --git a/dpdk/test/test/test_string_fns.c b/dpdk/test/test/test_string_fns.c new file mode 100644 index 00000000..8b4359aa --- /dev/null +++ b/dpdk/test/test/test_string_fns.c @@ -0,0 +1,169 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <stdarg.h> +#include <stddef.h> +#include <errno.h> +#include <string.h> + +#include <rte_string_fns.h> + +#include "test.h" + +#define LOG(...) do {\ + fprintf(stderr, "%s() ln %d: ", __func__, __LINE__); \ + fprintf(stderr, __VA_ARGS__); \ +} while(0) + +#define DATA_BYTE 'a' + +static int +test_rte_strsplit(void) +{ + int i; + do { + /* ======================================================= + * split a mac address correct number of splits requested + * =======================================================*/ + char test_string[] = "54:65:76:87:98:90"; + char *splits[6]; + + LOG("Source string: '%s', to split on ':'\n", test_string); + if (rte_strsplit(test_string, sizeof(test_string), + splits, 6, ':') != 6) { + LOG("Error splitting mac address\n"); + return -1; + } + for (i = 0; i < 6; i++) + LOG("Token %d = %s\n", i + 1, splits[i]); + } while (0); + + + do { + /* ======================================================= + * split on spaces smaller number of splits requested + * =======================================================*/ + char test_string[] = "54 65 76 87 98 90"; + char *splits[6]; + + LOG("Source string: '%s', to split on ' '\n", test_string); + if (rte_strsplit(test_string, sizeof(test_string), + splits, 3, ' ') != 3) { + LOG("Error splitting mac address for max 2 splits\n"); + return -1; + } + for (i = 0; i < 3; i++) + LOG("Token %d = %s\n", i + 1, splits[i]); + } while (0); + + do { + /* ======================================================= + * split on commas - more splits than commas requested + * =======================================================*/ + char test_string[] = "a,b,c,d"; + char *splits[6]; + + LOG("Source string: '%s', to split on ','\n", test_string); + if (rte_strsplit(test_string, sizeof(test_string), + splits, 6, ',') != 4) { + LOG("Error splitting %s on ','\n", test_string); + return -1; + } + for (i = 0; i < 4; i++) + LOG("Token %d = %s\n", i + 1, splits[i]); + } while(0); + + do { + /* ======================================================= + * Try splitting on non-existent character. + * =======================================================*/ + char test_string[] = "a,b,c,d"; + char *splits[6]; + + LOG("Source string: '%s', to split on ' '\n", test_string); + if (rte_strsplit(test_string, sizeof(test_string), + splits, 6, ' ') != 1) { + LOG("Error splitting %s on ' '\n", test_string); + return -1; + } + LOG("String not split\n"); + } while(0); + + do { + /* ======================================================= + * Invalid / edge case parameter checks + * =======================================================*/ + char test_string[] = "a,b,c,d"; + char *splits[6]; + + if (rte_strsplit(NULL, 0, splits, 6, ',') >= 0 + || errno != EINVAL){ + LOG("Error: rte_strsplit accepted NULL string parameter\n"); + return -1; + } + + if (rte_strsplit(test_string, sizeof(test_string), NULL, 0, ',') >= 0 + || errno != EINVAL){ + LOG("Error: rte_strsplit accepted NULL array parameter\n"); + return -1; + } + + errno = 0; + if (rte_strsplit(test_string, 0, splits, 6, ',') != 0 || errno != 0) { + LOG("Error: rte_strsplit did not accept 0 length string\n"); + return -1; + } + + if (rte_strsplit(test_string, sizeof(test_string), splits, 0, ',') != 0 + || errno != 0) { + LOG("Error: rte_strsplit did not accept 0 length array\n"); + return -1; + } + + LOG("Parameter test cases passed\n"); + } while(0); + + LOG("%s - PASSED\n", __func__); + return 0; +} + +static int +test_string_fns(void) +{ + if (test_rte_strsplit() < 0) + return -1; + return 0; +} + +REGISTER_TEST_COMMAND(string_autotest, test_string_fns); diff --git a/dpdk/test/test/test_table.c b/dpdk/test/test/test_table.c new file mode 100644 index 00000000..c5a6e00f --- /dev/null +++ b/dpdk/test/test/test_table.c @@ -0,0 +1,215 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <rte_byteorder.h> +#include <rte_hexdump.h> +#include <rte_string_fns.h> +#include <string.h> +#include "test.h" +#include "test_table.h" +#include "test_table_pipeline.h" +#include "test_table_ports.h" +#include "test_table_tables.h" +#include "test_table_combined.h" +#include "test_table_acl.h" + +/* Global variables */ +struct rte_pipeline *p; +struct rte_ring *rings_rx[N_PORTS]; +struct rte_ring *rings_tx[N_PORTS]; +struct rte_mempool *pool = NULL; + +uint32_t port_in_id[N_PORTS]; +uint32_t port_out_id[N_PORTS]; +uint32_t port_out_id_type[3]; +uint32_t table_id[N_PORTS*2]; +uint64_t override_hit_mask = 0xFFFFFFFF; +uint64_t override_miss_mask = 0xFFFFFFFF; +uint64_t non_reserved_actions_hit = 0; +uint64_t non_reserved_actions_miss = 0; +uint8_t connect_miss_action_to_port_out = 0; +uint8_t connect_miss_action_to_table = 0; +uint32_t table_entry_default_action = RTE_PIPELINE_ACTION_DROP; +uint32_t table_entry_hit_action = RTE_PIPELINE_ACTION_PORT; +uint32_t table_entry_miss_action = RTE_PIPELINE_ACTION_DROP; +rte_pipeline_port_in_action_handler port_in_action = NULL; +rte_pipeline_port_out_action_handler port_out_action = NULL; +rte_pipeline_table_action_handler_hit action_handler_hit = NULL; +rte_pipeline_table_action_handler_miss action_handler_miss = NULL; + +/* Function prototypes */ +static void app_init_rings(void); +static void app_init_mbuf_pools(void); + +uint64_t pipeline_test_hash(void *key, + __attribute__((unused)) void *key_mask, + __attribute__((unused)) uint32_t key_size, + __attribute__((unused)) uint64_t seed) +{ + uint32_t *k32 = key; + uint32_t ip_dst = rte_be_to_cpu_32(k32[0]); + uint64_t signature = ip_dst; + + return signature; +} + +static void +app_free_resources(void) { + int i; + for (i = 0; i < N_PORTS; i++) + rte_ring_free(rings_rx[i]); + rte_mempool_free(pool); +} + +static void +app_init_mbuf_pools(void) +{ + /* Init the buffer pool */ + printf("Getting/Creating the mempool ...\n"); + pool = rte_mempool_lookup("mempool"); + if (!pool) { + pool = rte_pktmbuf_pool_create( + "mempool", + POOL_SIZE, + POOL_CACHE_SIZE, 0, POOL_BUFFER_SIZE, + 0); + if (pool == NULL) + rte_panic("Cannot create mbuf pool\n"); + } +} + +static void +app_init_rings(void) +{ + uint32_t i; + + for (i = 0; i < N_PORTS; i++) { + char name[32]; + + snprintf(name, sizeof(name), "app_ring_rx_%u", i); + rings_rx[i] = rte_ring_lookup(name); + if (rings_rx[i] == NULL) { + rings_rx[i] = rte_ring_create( + name, + RING_RX_SIZE, + 0, + RING_F_SP_ENQ | RING_F_SC_DEQ); + } + if (rings_rx[i] == NULL) + rte_panic("Cannot create RX ring %u\n", i); + } + + for (i = 0; i < N_PORTS; i++) { + char name[32]; + + snprintf(name, sizeof(name), "app_ring_tx_%u", i); + rings_tx[i] = rte_ring_lookup(name); + if (rings_tx[i] == NULL) { + rings_tx[i] = rte_ring_create( + name, + RING_TX_SIZE, + 0, + RING_F_SP_ENQ | RING_F_SC_DEQ); + } + if (rings_tx[i] == NULL) + rte_panic("Cannot create TX ring %u\n", i); + } + +} + +static int +test_table(void) +{ + int status, ret; + unsigned i; + + ret = TEST_SUCCESS; + + app_init_rings(); + app_init_mbuf_pools(); + + printf("\n\n\n\n************Pipeline tests************\n"); + + if (test_table_pipeline() < 0) { + ret = TEST_FAILED; + goto end; + } + + printf("\n\n\n\n************Port tests************\n"); + for (i = 0; i < n_port_tests; i++) { + status = port_tests[i](); + if (status < 0) { + printf("\nPort test number %d failed (%d).\n", i, + status); + ret = TEST_FAILED; + goto end; + } + } + + printf("\n\n\n\n************Table tests************\n"); + for (i = 0; i < n_table_tests; i++) { + status = table_tests[i](); + if (status < 0) { + printf("\nTable test number %d failed (%d).\n", i, + status); + ret = TEST_FAILED; + goto end; + } + } + + printf("\n\n\n\n************Table tests************\n"); + for (i = 0; i < n_table_tests_combined; i++) { + status = table_tests_combined[i](); + if (status < 0) { + printf("\nCombined table test number %d failed with " + "reason number %d.\n", i, status); + ret = TEST_FAILED; + goto end; + } + } + +#ifdef RTE_LIBRTE_ACL + printf("\n\n\n\n************ACL tests************\n"); + if (test_table_acl() < 0) { + ret = TEST_FAILED; + goto end; + } +#endif + +end: + app_free_resources(); + + return ret; +} + +REGISTER_TEST_COMMAND(table_autotest, test_table); diff --git a/dpdk/test/test/test_table.h b/dpdk/test/test/test_table.h new file mode 100644 index 00000000..8c1df333 --- /dev/null +++ b/dpdk/test/test/test_table.h @@ -0,0 +1,207 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <rte_table_stub.h> +#include <rte_table_lpm.h> +#include <rte_table_lpm_ipv6.h> +#include <rte_table_hash.h> +#include <rte_table_array.h> +#include <rte_pipeline.h> + +#ifdef RTE_LIBRTE_ACL +#include <rte_table_acl.h> +#endif + +#include <rte_port_ring.h> +#include <rte_port_ethdev.h> +#include <rte_port_source_sink.h> + +#ifndef TEST_TABLE_H_ +#define TEST_TABLE_H_ + +#define RING_SIZE 4096 +#define MAX_BULK 32 +#define N 65536 +#define TIME_S 5 +#define TEST_RING_FULL_EMTPY_ITER 8 +#define N_PORTS 2 +#define N_PKTS 2 +#define N_PKTS_EXT 6 +#define RING_RX rings_rx[0] +#define RING_RX_2 rings_rx[1] +#define RING_TX rings_tx[0] +#define RING_TX_2 rings_tx[1] +#define PORT_RX_RING_SIZE 128 +#define PORT_TX_RING_SIZE 512 +#define RING_RX_SIZE 128 +#define RING_TX_SIZE 128 +#define POOL_BUFFER_SIZE RTE_MBUF_DEFAULT_BUF_SIZE +#define POOL_SIZE (32 * 1024) +#define POOL_CACHE_SIZE 256 +#define BURST_SIZE 8 +#define WORKER_TYPE 1 +#define MAX_DUMMY_PORTS 2 +#define MP_NAME "dummy_port_mempool" +#define MBUF_COUNT (8000 * MAX_DUMMY_PORTS) +#define MP_CACHE_SZ 256 +#define MP_SOCKET 0 +#define MP_FLAGS 0 + +/* Macros */ +#define APP_METADATA_OFFSET(offset) (sizeof(struct rte_mbuf) + (offset)) + +#define RING_ENQUEUE(ring, value) do { \ + struct rte_mbuf *m; \ + uint32_t *k32, *signature; \ + uint8_t *key; \ + \ + m = rte_pktmbuf_alloc(pool); \ + if (m == NULL) \ + return -1; \ + signature = RTE_MBUF_METADATA_UINT32_PTR(m, \ + APP_METADATA_OFFSET(0)); \ + key = RTE_MBUF_METADATA_UINT8_PTR(m, \ + APP_METADATA_OFFSET(32)); \ + k32 = (uint32_t *) key; \ + k32[0] = (value); \ + *signature = pipeline_test_hash(key, NULL, 0, 0); \ + rte_ring_enqueue((ring), m); \ +} while (0) + +#define RUN_PIPELINE(pipeline) do { \ + rte_pipeline_run((pipeline)); \ + rte_pipeline_flush((pipeline)); \ +} while (0) + +#define VERIFY(var, value) do { \ + if ((var) != -(value)) \ + return var; \ +} while (0) + +#define VERIFY_TRAFFIC(ring, sent, expected) do { \ + unsigned i, n = 0; \ + void *mbuf = NULL; \ + \ + for (i = 0; i < (sent); i++) { \ + if (!rte_ring_dequeue((ring), &mbuf)) { \ + if (mbuf == NULL) \ + continue; \ + n++; \ + rte_pktmbuf_free((struct rte_mbuf *)mbuf); \ + } \ + else \ + break; \ + } \ + printf("Expected %d, got %d\n", expected, n); \ + if (n != (expected)) { \ + return -21; \ + } \ +} while (0) + +/* Function definitions */ +uint64_t pipeline_test_hash( + void *key, + __attribute__((unused)) void *key_mask, + __attribute__((unused)) uint32_t key_size, + __attribute__((unused)) uint64_t seed); + +/* Extern variables */ +extern struct rte_pipeline *p; +extern struct rte_ring *rings_rx[N_PORTS]; +extern struct rte_ring *rings_tx[N_PORTS]; +extern struct rte_mempool *pool; +extern uint32_t port_in_id[N_PORTS]; +extern uint32_t port_out_id[N_PORTS]; +extern uint32_t port_out_id_type[3]; +extern uint32_t table_id[N_PORTS*2]; +extern uint64_t override_hit_mask; +extern uint64_t override_miss_mask; +extern uint64_t non_reserved_actions_hit; +extern uint64_t non_reserved_actions_miss; +extern uint8_t connect_miss_action_to_port_out; +extern uint8_t connect_miss_action_to_table; +extern uint32_t table_entry_default_action; +extern uint32_t table_entry_hit_action; +extern uint32_t table_entry_miss_action; +extern rte_pipeline_port_in_action_handler port_in_action; +extern rte_pipeline_port_out_action_handler port_out_action; +extern rte_pipeline_table_action_handler_hit action_handler_hit; +extern rte_pipeline_table_action_handler_miss action_handler_miss; + +/* Global data types */ +struct manage_ops { + uint32_t op_id; + void *op_data; + int expected_result; +}; + +/* Internal pipeline structures */ +struct rte_port_in { + struct rte_port_in_ops ops; + uint32_t burst_size; + uint32_t table_id; + void *h_port; +}; + +struct rte_port_out { + struct rte_port_out_ops ops; + void *h_port; +}; + +struct rte_table { + struct rte_table_ops ops; + rte_pipeline_table_action_handler_hit f_action; + uint32_t table_next_id; + uint32_t table_next_id_valid; + uint8_t actions_lookup_miss[RTE_CACHE_LINE_SIZE]; + uint32_t action_data_size; + void *h_table; +}; + +#define RTE_PIPELINE_MAX_NAME_SZ 124 + +struct rte_pipeline { + char name[RTE_PIPELINE_MAX_NAME_SZ]; + uint32_t socket_id; + struct rte_port_in ports_in[16]; + struct rte_port_out ports_out[16]; + struct rte_table tables[64]; + uint32_t num_ports_in; + uint32_t num_ports_out; + uint32_t num_tables; + struct rte_mbuf *pkts[RTE_PORT_IN_BURST_SIZE_MAX]; + struct rte_table_entry *actions[RTE_PORT_IN_BURST_SIZE_MAX]; + uint64_t mask_action[64]; + uint32_t mask_actions; +}; +#endif diff --git a/dpdk/test/test/test_table_acl.c b/dpdk/test/test/test_table_acl.c new file mode 100644 index 00000000..6fcf4cc3 --- /dev/null +++ b/dpdk/test/test/test_table_acl.c @@ -0,0 +1,762 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <rte_hexdump.h> +#include "test_table.h" +#include "test_table_acl.h" + +#define IPv4(a, b, c, d) ((uint32_t)(((a) & 0xff) << 24) | \ + (((b) & 0xff) << 16) | \ + (((c) & 0xff) << 8) | \ + ((d) & 0xff)) + +/* + * Rule and trace formats definitions. + **/ + +struct ipv4_5tuple { + uint8_t proto; + uint32_t ip_src; + uint32_t ip_dst; + uint16_t port_src; + uint16_t port_dst; +}; + +enum { + PROTO_FIELD_IPV4, + SRC_FIELD_IPV4, + DST_FIELD_IPV4, + SRCP_FIELD_IPV4, + DSTP_FIELD_IPV4, + NUM_FIELDS_IPV4 +}; + +struct rte_acl_field_def ipv4_defs[NUM_FIELDS_IPV4] = { + { + .type = RTE_ACL_FIELD_TYPE_BITMASK, + .size = sizeof(uint8_t), + .field_index = PROTO_FIELD_IPV4, + .input_index = PROTO_FIELD_IPV4, + .offset = offsetof(struct ipv4_5tuple, proto), + }, + { + .type = RTE_ACL_FIELD_TYPE_MASK, + .size = sizeof(uint32_t), + .field_index = SRC_FIELD_IPV4, + .input_index = SRC_FIELD_IPV4, + .offset = offsetof(struct ipv4_5tuple, ip_src), + }, + { + .type = RTE_ACL_FIELD_TYPE_MASK, + .size = sizeof(uint32_t), + .field_index = DST_FIELD_IPV4, + .input_index = DST_FIELD_IPV4, + .offset = offsetof(struct ipv4_5tuple, ip_dst), + }, + { + .type = RTE_ACL_FIELD_TYPE_RANGE, + .size = sizeof(uint16_t), + .field_index = SRCP_FIELD_IPV4, + .input_index = SRCP_FIELD_IPV4, + .offset = offsetof(struct ipv4_5tuple, port_src), + }, + { + .type = RTE_ACL_FIELD_TYPE_RANGE, + .size = sizeof(uint16_t), + .field_index = DSTP_FIELD_IPV4, + .input_index = SRCP_FIELD_IPV4, + .offset = offsetof(struct ipv4_5tuple, port_dst), + }, +}; + +struct rte_table_acl_rule_add_params table_acl_IPv4_rule; + +typedef int (*parse_5tuple)(char *text, + struct rte_table_acl_rule_add_params *rule); + +/* +* The order of the fields in the rule string after the initial '@' +*/ +enum { + CB_FLD_SRC_ADDR, + CB_FLD_DST_ADDR, + CB_FLD_SRC_PORT_RANGE, + CB_FLD_DST_PORT_RANGE, + CB_FLD_PROTO, + CB_FLD_NUM, +}; + + +#define GET_CB_FIELD(in, fd, base, lim, dlm) \ +do { \ + unsigned long val; \ + char *end; \ + \ + errno = 0; \ + val = strtoul((in), &end, (base)); \ + if (errno != 0 || end[0] != (dlm) || val > (lim)) \ + return -EINVAL; \ + (fd) = (typeof(fd)) val; \ + (in) = end + 1; \ +} while (0) + + + + +static int +parse_ipv4_net(const char *in, uint32_t *addr, uint32_t *mask_len) +{ + uint8_t a, b, c, d, m; + + GET_CB_FIELD(in, a, 0, UINT8_MAX, '.'); + GET_CB_FIELD(in, b, 0, UINT8_MAX, '.'); + GET_CB_FIELD(in, c, 0, UINT8_MAX, '.'); + GET_CB_FIELD(in, d, 0, UINT8_MAX, '/'); + GET_CB_FIELD(in, m, 0, sizeof(uint32_t) * CHAR_BIT, 0); + + addr[0] = IPv4(a, b, c, d); + mask_len[0] = m; + + return 0; +} + +static int +parse_port_range(const char *in, uint16_t *port_low, uint16_t *port_high) +{ + uint16_t a, b; + + GET_CB_FIELD(in, a, 0, UINT16_MAX, ':'); + GET_CB_FIELD(in, b, 0, UINT16_MAX, 0); + + port_low[0] = a; + port_high[0] = b; + + return 0; +} + +static int +parse_cb_ipv4_rule(char *str, struct rte_table_acl_rule_add_params *v) +{ + int i, rc; + char *s, *sp, *in[CB_FLD_NUM]; + static const char *dlm = " \t\n"; + + /* + ** Skip leading '@' + */ + if (strchr(str, '@') != str) + return -EINVAL; + + s = str + 1; + + /* + * Populate the 'in' array with the location of each + * field in the string we're parsing + */ + for (i = 0; i != DIM(in); i++) { + in[i] = strtok_r(s, dlm, &sp); + if (in[i] == NULL) + return -EINVAL; + s = NULL; + } + + /* Parse x.x.x.x/x */ + rc = parse_ipv4_net(in[CB_FLD_SRC_ADDR], + &v->field_value[SRC_FIELD_IPV4].value.u32, + &v->field_value[SRC_FIELD_IPV4].mask_range.u32); + if (rc != 0) { + RTE_LOG(ERR, PIPELINE, "failed to read src address/mask: %s\n", + in[CB_FLD_SRC_ADDR]); + return rc; + } + + printf("V=%u, mask=%u\n", v->field_value[SRC_FIELD_IPV4].value.u32, + v->field_value[SRC_FIELD_IPV4].mask_range.u32); + + /* Parse x.x.x.x/x */ + rc = parse_ipv4_net(in[CB_FLD_DST_ADDR], + &v->field_value[DST_FIELD_IPV4].value.u32, + &v->field_value[DST_FIELD_IPV4].mask_range.u32); + if (rc != 0) { + RTE_LOG(ERR, PIPELINE, "failed to read dest address/mask: %s\n", + in[CB_FLD_DST_ADDR]); + return rc; + } + + printf("V=%u, mask=%u\n", v->field_value[DST_FIELD_IPV4].value.u32, + v->field_value[DST_FIELD_IPV4].mask_range.u32); + /* Parse n:n */ + rc = parse_port_range(in[CB_FLD_SRC_PORT_RANGE], + &v->field_value[SRCP_FIELD_IPV4].value.u16, + &v->field_value[SRCP_FIELD_IPV4].mask_range.u16); + if (rc != 0) { + RTE_LOG(ERR, PIPELINE, "failed to read source port range: %s\n", + in[CB_FLD_SRC_PORT_RANGE]); + return rc; + } + + printf("V=%u, mask=%u\n", v->field_value[SRCP_FIELD_IPV4].value.u16, + v->field_value[SRCP_FIELD_IPV4].mask_range.u16); + /* Parse n:n */ + rc = parse_port_range(in[CB_FLD_DST_PORT_RANGE], + &v->field_value[DSTP_FIELD_IPV4].value.u16, + &v->field_value[DSTP_FIELD_IPV4].mask_range.u16); + if (rc != 0) { + RTE_LOG(ERR, PIPELINE, "failed to read dest port range: %s\n", + in[CB_FLD_DST_PORT_RANGE]); + return rc; + } + + printf("V=%u, mask=%u\n", v->field_value[DSTP_FIELD_IPV4].value.u16, + v->field_value[DSTP_FIELD_IPV4].mask_range.u16); + /* parse 0/0xnn */ + GET_CB_FIELD(in[CB_FLD_PROTO], + v->field_value[PROTO_FIELD_IPV4].value.u8, + 0, UINT8_MAX, '/'); + GET_CB_FIELD(in[CB_FLD_PROTO], + v->field_value[PROTO_FIELD_IPV4].mask_range.u8, + 0, UINT8_MAX, 0); + + printf("V=%u, mask=%u\n", + (unsigned int)v->field_value[PROTO_FIELD_IPV4].value.u8, + v->field_value[PROTO_FIELD_IPV4].mask_range.u8); + return 0; +} + +static int +parse_cb_ipv4_rule_del(char *str, struct rte_table_acl_rule_delete_params *v) +{ + int i, rc; + char *s, *sp, *in[CB_FLD_NUM]; + static const char *dlm = " \t\n"; + + /* + ** Skip leading '@' + */ + if (strchr(str, '@') != str) + return -EINVAL; + + s = str + 1; + + /* + * Populate the 'in' array with the location of each + * field in the string we're parsing + */ + for (i = 0; i != DIM(in); i++) { + in[i] = strtok_r(s, dlm, &sp); + if (in[i] == NULL) + return -EINVAL; + s = NULL; + } + + /* Parse x.x.x.x/x */ + rc = parse_ipv4_net(in[CB_FLD_SRC_ADDR], + &v->field_value[SRC_FIELD_IPV4].value.u32, + &v->field_value[SRC_FIELD_IPV4].mask_range.u32); + if (rc != 0) { + RTE_LOG(ERR, PIPELINE, "failed to read src address/mask: %s\n", + in[CB_FLD_SRC_ADDR]); + return rc; + } + + printf("V=%u, mask=%u\n", v->field_value[SRC_FIELD_IPV4].value.u32, + v->field_value[SRC_FIELD_IPV4].mask_range.u32); + + /* Parse x.x.x.x/x */ + rc = parse_ipv4_net(in[CB_FLD_DST_ADDR], + &v->field_value[DST_FIELD_IPV4].value.u32, + &v->field_value[DST_FIELD_IPV4].mask_range.u32); + if (rc != 0) { + RTE_LOG(ERR, PIPELINE, "failed to read dest address/mask: %s\n", + in[CB_FLD_DST_ADDR]); + return rc; + } + + printf("V=%u, mask=%u\n", v->field_value[DST_FIELD_IPV4].value.u32, + v->field_value[DST_FIELD_IPV4].mask_range.u32); + /* Parse n:n */ + rc = parse_port_range(in[CB_FLD_SRC_PORT_RANGE], + &v->field_value[SRCP_FIELD_IPV4].value.u16, + &v->field_value[SRCP_FIELD_IPV4].mask_range.u16); + if (rc != 0) { + RTE_LOG(ERR, PIPELINE, "failed to read source port range: %s\n", + in[CB_FLD_SRC_PORT_RANGE]); + return rc; + } + + printf("V=%u, mask=%u\n", v->field_value[SRCP_FIELD_IPV4].value.u16, + v->field_value[SRCP_FIELD_IPV4].mask_range.u16); + /* Parse n:n */ + rc = parse_port_range(in[CB_FLD_DST_PORT_RANGE], + &v->field_value[DSTP_FIELD_IPV4].value.u16, + &v->field_value[DSTP_FIELD_IPV4].mask_range.u16); + if (rc != 0) { + RTE_LOG(ERR, PIPELINE, "failed to read dest port range: %s\n", + in[CB_FLD_DST_PORT_RANGE]); + return rc; + } + + printf("V=%u, mask=%u\n", v->field_value[DSTP_FIELD_IPV4].value.u16, + v->field_value[DSTP_FIELD_IPV4].mask_range.u16); + /* parse 0/0xnn */ + GET_CB_FIELD(in[CB_FLD_PROTO], + v->field_value[PROTO_FIELD_IPV4].value.u8, + 0, UINT8_MAX, '/'); + GET_CB_FIELD(in[CB_FLD_PROTO], + v->field_value[PROTO_FIELD_IPV4].mask_range.u8, + 0, UINT8_MAX, 0); + + printf("V=%u, mask=%u\n", + (unsigned int)v->field_value[PROTO_FIELD_IPV4].value.u8, + v->field_value[PROTO_FIELD_IPV4].mask_range.u8); + return 0; +} + +/* + * The format for these rules DO NOT need the port ranges to be + * separated by ' : ', just ':'. It's a lot more readable and + * cleaner, IMO. + */ +char lines[][128] = { + "@0.0.0.0/0 0.0.0.0/0 0:65535 0:65535 2/0xff", /* Protocol check */ + "@192.168.3.1/32 0.0.0.0/0 0:65535 0:65535 0/0", /* Src IP checl */ + "@0.0.0.0/0 10.4.4.1/32 0:65535 0:65535 0/0", /* dst IP check */ + "@0.0.0.0/0 0.0.0.0/0 105:105 0:65535 0/0", /* src port check */ + "@0.0.0.0/0 0.0.0.0/0 0:65535 206:206 0/0", /* dst port check */ +}; + +char line[128]; + + +static int +setup_acl_pipeline(void) +{ + int ret; + int i; + struct rte_pipeline_params pipeline_params = { + .name = "PIPELINE", + .socket_id = 0, + }; + uint32_t n; + struct rte_table_acl_rule_add_params rule_params; + struct rte_pipeline_table_acl_rule_delete_params *delete_params; + parse_5tuple parser; + char acl_name[64]; + + /* Pipeline configuration */ + p = rte_pipeline_create(&pipeline_params); + if (p == NULL) { + RTE_LOG(INFO, PIPELINE, "%s: Failed to configure pipeline\n", + __func__); + goto fail; + } + + /* Input port configuration */ + for (i = 0; i < N_PORTS; i++) { + struct rte_port_ring_reader_params port_ring_params = { + .ring = rings_rx[i], + }; + + struct rte_pipeline_port_in_params port_params = { + .ops = &rte_port_ring_reader_ops, + .arg_create = (void *) &port_ring_params, + .f_action = NULL, + .burst_size = BURST_SIZE, + }; + + /* Put in action for some ports */ + if (i) + port_params.f_action = port_in_action; + + ret = rte_pipeline_port_in_create(p, &port_params, + &port_in_id[i]); + if (ret) { + rte_panic("Unable to configure input port %d, ret:%d\n", + i, ret); + goto fail; + } + } + + /* output Port configuration */ + for (i = 0; i < N_PORTS; i++) { + struct rte_port_ring_writer_params port_ring_params = { + .ring = rings_tx[i], + .tx_burst_sz = BURST_SIZE, + }; + + struct rte_pipeline_port_out_params port_params = { + .ops = &rte_port_ring_writer_ops, + .arg_create = (void *) &port_ring_params, + .f_action = NULL, + .arg_ah = NULL, + }; + + + if (rte_pipeline_port_out_create(p, &port_params, + &port_out_id[i])) { + rte_panic("Unable to configure output port %d\n", i); + goto fail; + } + } + + /* Table configuration */ + for (i = 0; i < N_PORTS; i++) { + struct rte_pipeline_table_params table_params; + + /* Set up defaults for stub */ + table_params.ops = &rte_table_stub_ops; + table_params.arg_create = NULL; + table_params.f_action_hit = action_handler_hit; + table_params.f_action_miss = NULL; + table_params.action_data_size = 0; + + RTE_LOG(INFO, PIPELINE, "miss_action=%x\n", + table_entry_miss_action); + + printf("RTE_ACL_RULE_SZ(%zu) = %zu\n", DIM(ipv4_defs), + RTE_ACL_RULE_SZ(DIM(ipv4_defs))); + + struct rte_table_acl_params acl_params; + + acl_params.n_rules = 1 << 5; + acl_params.n_rule_fields = DIM(ipv4_defs); + snprintf(acl_name, sizeof(acl_name), "ACL%d", i); + acl_params.name = acl_name; + memcpy(acl_params.field_format, ipv4_defs, sizeof(ipv4_defs)); + + table_params.ops = &rte_table_acl_ops; + table_params.arg_create = &acl_params; + + if (rte_pipeline_table_create(p, &table_params, &table_id[i])) { + rte_panic("Unable to configure table %u\n", i); + goto fail; + } + + if (connect_miss_action_to_table) { + if (rte_pipeline_table_create(p, &table_params, + &table_id[i+2])) { + rte_panic("Unable to configure table %u\n", i); + goto fail; + } + } + } + + for (i = 0; i < N_PORTS; i++) { + if (rte_pipeline_port_in_connect_to_table(p, port_in_id[i], + table_id[i])) { + rte_panic("Unable to connect input port %u to " + "table %u\n", + port_in_id[i], table_id[i]); + goto fail; + } + } + + /* Add bulk entries to tables */ + for (i = 0; i < N_PORTS; i++) { + struct rte_table_acl_rule_add_params keys[5]; + struct rte_pipeline_table_entry entries[5]; + struct rte_table_acl_rule_add_params *key_array[5]; + struct rte_pipeline_table_entry *table_entries[5]; + int key_found[5]; + struct rte_pipeline_table_entry *table_entries_ptr[5]; + struct rte_pipeline_table_entry entries_ptr[5]; + + parser = parse_cb_ipv4_rule; + for (n = 0; n < 5; n++) { + memset(&keys[n], 0, sizeof(struct rte_table_acl_rule_add_params)); + key_array[n] = &keys[n]; + + snprintf(line, sizeof(line), "%s", lines[n]); + printf("PARSING [%s]\n", line); + + ret = parser(line, &keys[n]); + if (ret != 0) { + RTE_LOG(ERR, PIPELINE, + "line %u: parse_cb_ipv4vlan_rule" + " failed, error code: %d (%s)\n", + n, ret, strerror(-ret)); + return ret; + } + + keys[n].priority = RTE_ACL_MAX_PRIORITY - n - 1; + + entries[n].action = RTE_PIPELINE_ACTION_PORT; + entries[n].port_id = port_out_id[i^1]; + table_entries[n] = &entries[n]; + table_entries_ptr[n] = &entries_ptr[n]; + } + + ret = rte_pipeline_table_entry_add_bulk(p, table_id[i], + (void **)key_array, table_entries, 5, key_found, table_entries_ptr); + if (ret < 0) { + rte_panic("Add entry bulk to table %u failed (%d)\n", + table_id[i], ret); + goto fail; + } + } + + /* Delete bulk entries from tables */ + for (i = 0; i < N_PORTS; i++) { + struct rte_table_acl_rule_delete_params keys[5]; + struct rte_table_acl_rule_delete_params *key_array[5]; + struct rte_pipeline_table_entry *table_entries[5]; + int key_found[5]; + + memset(table_entries, 0, sizeof(table_entries)); + + for (n = 0; n < 5; n++) { + memset(&keys[n], 0, sizeof(struct rte_table_acl_rule_delete_params)); + key_array[n] = &keys[n]; + + snprintf(line, sizeof(line), "%s", lines[n]); + printf("PARSING [%s]\n", line); + + ret = parse_cb_ipv4_rule_del(line, &keys[n]); + if (ret != 0) { + RTE_LOG(ERR, PIPELINE, + "line %u: parse_cb_ipv4vlan_rule" + " failed, error code: %d (%s)\n", + n, ret, strerror(-ret)); + return ret; + } + } + + ret = rte_pipeline_table_entry_delete_bulk(p, table_id[i], + (void **)key_array, 5, key_found, table_entries); + if (ret < 0) { + rte_panic("Delete bulk entries from table %u failed (%d)\n", + table_id[i], ret); + goto fail; + } else + printf("Bulk deleted rules.\n"); + } + + /* Add entries to tables */ + for (i = 0; i < N_PORTS; i++) { + struct rte_pipeline_table_entry table_entry = { + .action = RTE_PIPELINE_ACTION_PORT, + {.port_id = port_out_id[i^1]}, + }; + int key_found; + struct rte_pipeline_table_entry *entry_ptr; + + memset(&rule_params, 0, sizeof(rule_params)); + parser = parse_cb_ipv4_rule; + + for (n = 1; n <= 5; n++) { + snprintf(line, sizeof(line), "%s", lines[n-1]); + printf("PARSING [%s]\n", line); + + ret = parser(line, &rule_params); + if (ret != 0) { + RTE_LOG(ERR, PIPELINE, + "line %u: parse_cb_ipv4vlan_rule" + " failed, error code: %d (%s)\n", + n, ret, strerror(-ret)); + return ret; + } + + rule_params.priority = RTE_ACL_MAX_PRIORITY - n; + + ret = rte_pipeline_table_entry_add(p, table_id[i], + &rule_params, + &table_entry, &key_found, &entry_ptr); + if (ret < 0) { + rte_panic("Add entry to table %u failed (%d)\n", + table_id[i], ret); + goto fail; + } + } + + /* delete a few rules */ + for (n = 2; n <= 3; n++) { + snprintf(line, sizeof(line), "%s", lines[n-1]); + printf("PARSING [%s]\n", line); + + ret = parser(line, &rule_params); + if (ret != 0) { + RTE_LOG(ERR, PIPELINE, "line %u: parse rule " + " failed, error code: %d (%s)\n", + n, ret, strerror(-ret)); + return ret; + } + + delete_params = (struct + rte_pipeline_table_acl_rule_delete_params *) + &(rule_params.field_value[0]); + ret = rte_pipeline_table_entry_delete(p, table_id[i], + delete_params, &key_found, NULL); + if (ret < 0) { + rte_panic("Add entry to table %u failed (%d)\n", + table_id[i], ret); + goto fail; + } else + printf("Deleted Rule.\n"); + } + + + /* Try to add duplicates */ + for (n = 1; n <= 5; n++) { + snprintf(line, sizeof(line), "%s", lines[n-1]); + printf("PARSING [%s]\n", line); + + ret = parser(line, &rule_params); + if (ret != 0) { + RTE_LOG(ERR, PIPELINE, "line %u: parse rule" + " failed, error code: %d (%s)\n", + n, ret, strerror(-ret)); + return ret; + } + + rule_params.priority = RTE_ACL_MAX_PRIORITY - n; + + ret = rte_pipeline_table_entry_add(p, table_id[i], + &rule_params, + &table_entry, &key_found, &entry_ptr); + if (ret < 0) { + rte_panic("Add entry to table %u failed (%d)\n", + table_id[i], ret); + goto fail; + } + } + } + + /* Enable input ports */ + for (i = 0; i < N_PORTS ; i++) + if (rte_pipeline_port_in_enable(p, port_in_id[i])) + rte_panic("Unable to enable input port %u\n", + port_in_id[i]); + + /* Check pipeline consistency */ + if (rte_pipeline_check(p) < 0) { + rte_panic("Pipeline consistency check failed\n"); + goto fail; + } + + return 0; +fail: + + return -1; +} + +static int +test_pipeline_single_filter(int expected_count) +{ + int i, j, ret, tx_count; + struct ipv4_5tuple five_tuple; + + /* Allocate a few mbufs and manually insert into the rings. */ + for (i = 0; i < N_PORTS; i++) { + for (j = 0; j < 8; j++) { + struct rte_mbuf *mbuf; + + mbuf = rte_pktmbuf_alloc(pool); + if (mbuf == NULL) + /* this will cause test failure after cleanup + * of already enqueued mbufs, as the mbuf + * counts won't match */ + break; + memset(rte_pktmbuf_mtod(mbuf, char *), 0x00, + sizeof(struct ipv4_5tuple)); + + five_tuple.proto = j; + five_tuple.ip_src = rte_bswap32(IPv4(192, 168, j, 1)); + five_tuple.ip_dst = rte_bswap32(IPv4(10, 4, j, 1)); + five_tuple.port_src = rte_bswap16(100 + j); + five_tuple.port_dst = rte_bswap16(200 + j); + + memcpy(rte_pktmbuf_mtod(mbuf, char *), &five_tuple, + sizeof(struct ipv4_5tuple)); + RTE_LOG(INFO, PIPELINE, "%s: Enqueue onto ring %d\n", + __func__, i); + rte_ring_enqueue(rings_rx[i], mbuf); + } + } + + /* Run pipeline once */ + for (i = 0; i< N_PORTS; i++) + rte_pipeline_run(p); + + rte_pipeline_flush(p); + + tx_count = 0; + + for (i = 0; i < N_PORTS; i++) { + void *objs[RING_TX_SIZE]; + struct rte_mbuf *mbuf; + + ret = rte_ring_sc_dequeue_burst(rings_tx[i], objs, 10, NULL); + if (ret <= 0) { + printf("Got no objects from ring %d - error code %d\n", + i, ret); + } else { + printf("Got %d object(s) from ring %d!\n", ret, i); + for (j = 0; j < ret; j++) { + mbuf = objs[j]; + rte_hexdump(stdout, "mbuf", + rte_pktmbuf_mtod(mbuf, char *), 64); + rte_pktmbuf_free(mbuf); + } + tx_count += ret; + } + } + + if (tx_count != expected_count) { + RTE_LOG(INFO, PIPELINE, + "%s: Unexpected packets for ACL test, " + "expected %d, got %d\n", + __func__, expected_count, tx_count); + goto fail; + } + + rte_pipeline_free(p); + + return 0; +fail: + return -1; + +} + +int +test_table_acl(void) +{ + + + override_hit_mask = 0xFF; /* All packets are a hit */ + + setup_acl_pipeline(); + if (test_pipeline_single_filter(10) < 0) + return -1; + + return 0; +} diff --git a/dpdk/test/test/test_table_acl.h b/dpdk/test/test/test_table_acl.h new file mode 100644 index 00000000..a64c3e6c --- /dev/null +++ b/dpdk/test/test/test_table_acl.h @@ -0,0 +1,35 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* Test prototypes */ +int test_table_acl(void); diff --git a/dpdk/test/test/test_table_combined.c b/dpdk/test/test/test_table_combined.c new file mode 100644 index 00000000..e8637884 --- /dev/null +++ b/dpdk/test/test/test_table_combined.c @@ -0,0 +1,872 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <string.h> +#include "test_table_combined.h" +#include "test_table.h" +#include <rte_table_lpm_ipv6.h> + +#define MAX_TEST_KEYS 128 +#define N_PACKETS 50 + +enum check_table_result { + CHECK_TABLE_OK, + CHECK_TABLE_PORT_CONFIG, + CHECK_TABLE_PORT_ENABLE, + CHECK_TABLE_TABLE_CONFIG, + CHECK_TABLE_ENTRY_ADD, + CHECK_TABLE_DEFAULT_ENTRY_ADD, + CHECK_TABLE_CONNECT, + CHECK_TABLE_MANAGE_ERROR, + CHECK_TABLE_CONSISTENCY, + CHECK_TABLE_NO_TRAFFIC, + CHECK_TABLE_INVALID_PARAMETER, +}; + +struct table_packets { + uint32_t hit_packet[MAX_TEST_KEYS]; + uint32_t miss_packet[MAX_TEST_KEYS]; + uint32_t n_hit_packets; + uint32_t n_miss_packets; +}; + +combined_table_test table_tests_combined[] = { + test_table_lpm_combined, + test_table_lpm_ipv6_combined, + test_table_hash8lru, + test_table_hash8ext, + test_table_hash16lru, + test_table_hash16ext, + test_table_hash32lru, + test_table_hash32ext, + test_table_hash_cuckoo_combined, +}; + +unsigned n_table_tests_combined = RTE_DIM(table_tests_combined); + +/* Generic port tester function */ +static int +test_table_type(struct rte_table_ops *table_ops, void *table_args, + void *key, struct table_packets *table_packets, + struct manage_ops *manage_ops, unsigned n_ops) +{ + uint32_t ring_in_id, table_id, ring_out_id, ring_out_2_id; + unsigned i; + + RTE_SET_USED(manage_ops); + RTE_SET_USED(n_ops); + /* Create pipeline */ + struct rte_pipeline_params pipeline_params = { + .name = "pipeline", + .socket_id = 0, + }; + + struct rte_pipeline *pipeline = rte_pipeline_create(&pipeline_params); + + /* Create input ring */ + struct rte_port_ring_reader_params ring_params_rx = { + .ring = RING_RX, + }; + + struct rte_port_ring_writer_params ring_params_tx = { + .ring = RING_RX, + .tx_burst_sz = RTE_PORT_IN_BURST_SIZE_MAX, + }; + + struct rte_pipeline_port_in_params ring_in_params = { + .ops = &rte_port_ring_reader_ops, + .arg_create = (void *)&ring_params_rx, + .f_action = NULL, + .burst_size = RTE_PORT_IN_BURST_SIZE_MAX, + }; + + if (rte_pipeline_port_in_create(pipeline, &ring_in_params, + &ring_in_id) != 0) { + rte_pipeline_free(pipeline); + return -CHECK_TABLE_PORT_CONFIG; + } + + /* Create table */ + struct rte_pipeline_table_params table_params = { + .ops = table_ops, + .arg_create = table_args, + .f_action_hit = NULL, + .f_action_miss = NULL, + .arg_ah = NULL, + .action_data_size = 0, + }; + + if (rte_pipeline_table_create(pipeline, &table_params, + &table_id) != 0) { + rte_pipeline_free(pipeline); + return -CHECK_TABLE_TABLE_CONFIG; + } + + /* Create output ports */ + ring_params_tx.ring = RING_TX; + + struct rte_pipeline_port_out_params ring_out_params = { + .ops = &rte_port_ring_writer_ops, + .arg_create = (void *)&ring_params_tx, + .f_action = NULL, + }; + + if (rte_pipeline_port_out_create(pipeline, &ring_out_params, + &ring_out_id) != 0) { + rte_pipeline_free(pipeline); + return -CHECK_TABLE_PORT_CONFIG; + } + + ring_params_tx.ring = RING_TX_2; + + if (rte_pipeline_port_out_create(pipeline, &ring_out_params, + &ring_out_2_id) != 0) { + rte_pipeline_free(pipeline); + return -CHECK_TABLE_PORT_CONFIG; + } + + /* Add entry to the table */ + struct rte_pipeline_table_entry default_entry = { + .action = RTE_PIPELINE_ACTION_DROP, + {.table_id = ring_out_id}, + }; + + struct rte_pipeline_table_entry table_entry = { + .action = RTE_PIPELINE_ACTION_PORT, + {.table_id = ring_out_id}, + }; + + struct rte_pipeline_table_entry *default_entry_ptr, *entry_ptr; + + int key_found; + + if (rte_pipeline_table_default_entry_add(pipeline, table_id, + &default_entry, &default_entry_ptr) != 0) { + rte_pipeline_free(pipeline); + return -CHECK_TABLE_DEFAULT_ENTRY_ADD; + } + + if (rte_pipeline_table_entry_add(pipeline, table_id, + key ? key : &table_entry, &table_entry, &key_found, + &entry_ptr) != 0) { + rte_pipeline_free(pipeline); + return -CHECK_TABLE_ENTRY_ADD; + } + + /* Create connections and check consistency */ + if (rte_pipeline_port_in_connect_to_table(pipeline, ring_in_id, + table_id) != 0) { + rte_pipeline_free(pipeline); + return -CHECK_TABLE_CONNECT; + } + + if (rte_pipeline_port_in_enable(pipeline, ring_in_id) != 0) { + rte_pipeline_free(pipeline); + return -CHECK_TABLE_PORT_ENABLE; + } + + if (rte_pipeline_check(pipeline) != 0) { + rte_pipeline_free(pipeline); + return -CHECK_TABLE_CONSISTENCY; + } + + + + /* Flow test - All hits */ + if (table_packets->n_hit_packets) { + for (i = 0; i < table_packets->n_hit_packets; i++) + RING_ENQUEUE(RING_RX, table_packets->hit_packet[i]); + + RUN_PIPELINE(pipeline); + + VERIFY_TRAFFIC(RING_TX, table_packets->n_hit_packets, + table_packets->n_hit_packets); + } + + /* Flow test - All misses */ + if (table_packets->n_miss_packets) { + for (i = 0; i < table_packets->n_miss_packets; i++) + RING_ENQUEUE(RING_RX, table_packets->miss_packet[i]); + + RUN_PIPELINE(pipeline); + + VERIFY_TRAFFIC(RING_TX, table_packets->n_miss_packets, 0); + } + + /* Flow test - Half hits, half misses */ + if (table_packets->n_hit_packets && table_packets->n_miss_packets) { + for (i = 0; i < (table_packets->n_hit_packets) / 2; i++) + RING_ENQUEUE(RING_RX, table_packets->hit_packet[i]); + + for (i = 0; i < (table_packets->n_miss_packets) / 2; i++) + RING_ENQUEUE(RING_RX, table_packets->miss_packet[i]); + + RUN_PIPELINE(pipeline); + VERIFY_TRAFFIC(RING_TX, table_packets->n_hit_packets, + table_packets->n_hit_packets / 2); + } + + /* Flow test - Single packet */ + if (table_packets->n_hit_packets) { + RING_ENQUEUE(RING_RX, table_packets->hit_packet[0]); + RUN_PIPELINE(pipeline); + VERIFY_TRAFFIC(RING_TX, table_packets->n_hit_packets, 1); + } + if (table_packets->n_miss_packets) { + RING_ENQUEUE(RING_RX, table_packets->miss_packet[0]); + RUN_PIPELINE(pipeline); + VERIFY_TRAFFIC(RING_TX, table_packets->n_miss_packets, 0); + } + + + /* Change table entry action */ + printf("Change entry action\n"); + table_entry.table_id = ring_out_2_id; + + if (rte_pipeline_table_default_entry_add(pipeline, table_id, + &default_entry, &default_entry_ptr) != 0) { + rte_pipeline_free(pipeline); + return -CHECK_TABLE_ENTRY_ADD; + } + + if (rte_pipeline_table_entry_add(pipeline, table_id, + key ? key : &table_entry, &table_entry, &key_found, + &entry_ptr) != 0) { + rte_pipeline_free(pipeline); + return -CHECK_TABLE_ENTRY_ADD; + } + + /* Check that traffic destination has changed */ + if (table_packets->n_hit_packets) { + for (i = 0; i < table_packets->n_hit_packets; i++) + RING_ENQUEUE(RING_RX, table_packets->hit_packet[i]); + + RUN_PIPELINE(pipeline); + VERIFY_TRAFFIC(RING_TX, table_packets->n_hit_packets, 0); + VERIFY_TRAFFIC(RING_TX_2, table_packets->n_hit_packets, + table_packets->n_hit_packets); + } + + printf("delete entry\n"); + /* Delete table entry */ + rte_pipeline_table_entry_delete(pipeline, table_id, + key ? key : &table_entry, &key_found, NULL); + + rte_pipeline_free(pipeline); + + return 0; +} + +/* Table tests */ +int +test_table_stub_combined(void) +{ + int status, i; + struct table_packets table_packets; + + printf("--------------\n"); + printf("RUNNING TEST - %s\n", __func__); + printf("--------------\n"); + for (i = 0; i < N_PACKETS; i++) + table_packets.hit_packet[i] = i; + + table_packets.n_hit_packets = N_PACKETS; + table_packets.n_miss_packets = 0; + + status = test_table_type(&rte_table_stub_ops, NULL, NULL, + &table_packets, NULL, 1); + VERIFY(status, CHECK_TABLE_OK); + + return 0; +} + +int +test_table_lpm_combined(void) +{ + int status, i; + + /* Traffic flow */ + struct rte_table_lpm_params lpm_params = { + .name = "LPM", + .n_rules = 1 << 16, + .number_tbl8s = 1 << 8, + .flags = 0, + .entry_unique_size = 8, + .offset = APP_METADATA_OFFSET(0), + }; + + struct rte_table_lpm_key lpm_key = { + .ip = 0xadadadad, + .depth = 16, + }; + + struct table_packets table_packets; + + printf("--------------\n"); + printf("RUNNING TEST - %s\n", __func__); + printf("--------------\n"); + + for (i = 0; i < N_PACKETS; i++) + table_packets.hit_packet[i] = 0xadadadad; + + for (i = 0; i < N_PACKETS; i++) + table_packets.miss_packet[i] = 0xfefefefe; + + table_packets.n_hit_packets = N_PACKETS; + table_packets.n_miss_packets = N_PACKETS; + + status = test_table_type(&rte_table_lpm_ops, (void *)&lpm_params, + (void *)&lpm_key, &table_packets, NULL, 0); + VERIFY(status, CHECK_TABLE_OK); + + /* Invalid parameters */ + lpm_params.n_rules = 0; + + status = test_table_type(&rte_table_lpm_ops, (void *)&lpm_params, + (void *)&lpm_key, &table_packets, NULL, 0); + VERIFY(status, CHECK_TABLE_TABLE_CONFIG); + + lpm_params.n_rules = 1 << 24; + lpm_key.depth = 0; + + status = test_table_type(&rte_table_lpm_ops, (void *)&lpm_params, + (void *)&lpm_key, &table_packets, NULL, 0); + VERIFY(status, CHECK_TABLE_ENTRY_ADD); + + lpm_key.depth = 33; + + status = test_table_type(&rte_table_lpm_ops, (void *)&lpm_params, + (void *)&lpm_key, &table_packets, NULL, 0); + VERIFY(status, CHECK_TABLE_ENTRY_ADD); + + return 0; +} + +int +test_table_lpm_ipv6_combined(void) +{ + int status, i; + + /* Traffic flow */ + struct rte_table_lpm_ipv6_params lpm_ipv6_params = { + .name = "LPM", + .n_rules = 1 << 16, + .number_tbl8s = 1 << 13, + .entry_unique_size = 8, + .offset = APP_METADATA_OFFSET(32), + }; + + struct rte_table_lpm_ipv6_key lpm_ipv6_key = { + .depth = 16, + }; + memset(lpm_ipv6_key.ip, 0xad, 16); + + struct table_packets table_packets; + + printf("--------------\n"); + printf("RUNNING TEST - %s\n", __func__); + printf("--------------\n"); + for (i = 0; i < N_PACKETS; i++) + table_packets.hit_packet[i] = 0xadadadad; + + for (i = 0; i < N_PACKETS; i++) + table_packets.miss_packet[i] = 0xadadadab; + + table_packets.n_hit_packets = N_PACKETS; + table_packets.n_miss_packets = N_PACKETS; + + status = test_table_type(&rte_table_lpm_ipv6_ops, + (void *)&lpm_ipv6_params, + (void *)&lpm_ipv6_key, &table_packets, NULL, 0); + VERIFY(status, CHECK_TABLE_OK); + + /* Invalid parameters */ + lpm_ipv6_params.n_rules = 0; + + status = test_table_type(&rte_table_lpm_ipv6_ops, + (void *)&lpm_ipv6_params, + (void *)&lpm_ipv6_key, &table_packets, NULL, 0); + VERIFY(status, CHECK_TABLE_TABLE_CONFIG); + + lpm_ipv6_params.n_rules = 1 << 24; + lpm_ipv6_key.depth = 0; + + status = test_table_type(&rte_table_lpm_ipv6_ops, + (void *)&lpm_ipv6_params, + (void *)&lpm_ipv6_key, &table_packets, NULL, 0); + VERIFY(status, CHECK_TABLE_ENTRY_ADD); + + lpm_ipv6_key.depth = 129; + status = test_table_type(&rte_table_lpm_ipv6_ops, + (void *)&lpm_ipv6_params, + (void *)&lpm_ipv6_key, &table_packets, NULL, 0); + VERIFY(status, CHECK_TABLE_ENTRY_ADD); + + return 0; +} + +int +test_table_hash8lru(void) +{ + int status, i; + + /* Traffic flow */ + struct rte_table_hash_params key8lru_params = { + .name = "TABLE", + .key_size = 8, + .key_offset = APP_METADATA_OFFSET(32), + .key_mask = NULL, + .n_keys = 1 << 16, + .n_buckets = 1 << 16, + .f_hash = pipeline_test_hash, + .seed = 0, + }; + + uint8_t key8lru[8]; + uint32_t *k8lru = (uint32_t *) key8lru; + + memset(key8lru, 0, sizeof(key8lru)); + k8lru[0] = 0xadadadad; + + struct table_packets table_packets; + + printf("--------------\n"); + printf("RUNNING TEST - %s\n", __func__); + printf("--------------\n"); + for (i = 0; i < 50; i++) + table_packets.hit_packet[i] = 0xadadadad; + + for (i = 0; i < 50; i++) + table_packets.miss_packet[i] = 0xfefefefe; + + table_packets.n_hit_packets = 50; + table_packets.n_miss_packets = 50; + + status = test_table_type(&rte_table_hash_key8_lru_ops, + (void *)&key8lru_params, (void *)key8lru, &table_packets, + NULL, 0); + VERIFY(status, CHECK_TABLE_OK); + + /* Invalid parameters */ + key8lru_params.n_keys = 0; + + status = test_table_type(&rte_table_hash_key8_lru_ops, + (void *)&key8lru_params, (void *)key8lru, &table_packets, + NULL, 0); + VERIFY(status, CHECK_TABLE_TABLE_CONFIG); + + key8lru_params.n_keys = 1<<16; + key8lru_params.f_hash = NULL; + + status = test_table_type(&rte_table_hash_key8_lru_ops, + (void *)&key8lru_params, (void *)key8lru, &table_packets, + NULL, 0); + VERIFY(status, CHECK_TABLE_TABLE_CONFIG); + + return 0; +} + +int +test_table_hash16lru(void) +{ + int status, i; + + /* Traffic flow */ + struct rte_table_hash_params key16lru_params = { + .name = "TABLE", + .key_size = 16, + .key_offset = APP_METADATA_OFFSET(32), + .key_mask = NULL, + .n_keys = 1 << 16, + .n_buckets = 1 << 16, + .f_hash = pipeline_test_hash, + .seed = 0, + }; + + uint8_t key16lru[16]; + uint32_t *k16lru = (uint32_t *) key16lru; + + memset(key16lru, 0, sizeof(key16lru)); + k16lru[0] = 0xadadadad; + + struct table_packets table_packets; + + printf("--------------\n"); + printf("RUNNING TEST - %s\n", __func__); + printf("--------------\n"); + for (i = 0; i < 50; i++) + table_packets.hit_packet[i] = 0xadadadad; + + for (i = 0; i < 50; i++) + table_packets.miss_packet[i] = 0xfefefefe; + + table_packets.n_hit_packets = 50; + table_packets.n_miss_packets = 50; + + status = test_table_type(&rte_table_hash_key16_lru_ops, + (void *)&key16lru_params, (void *)key16lru, &table_packets, + NULL, 0); + VERIFY(status, CHECK_TABLE_OK); + + /* Invalid parameters */ + key16lru_params.n_keys = 0; + + status = test_table_type(&rte_table_hash_key16_lru_ops, + (void *)&key16lru_params, (void *)key16lru, &table_packets, + NULL, 0); + VERIFY(status, CHECK_TABLE_TABLE_CONFIG); + + key16lru_params.n_keys = 1<<16; + key16lru_params.f_hash = NULL; + + status = test_table_type(&rte_table_hash_key16_lru_ops, + (void *)&key16lru_params, (void *)key16lru, &table_packets, + NULL, 0); + VERIFY(status, CHECK_TABLE_TABLE_CONFIG); + + return 0; +} + +int +test_table_hash32lru(void) +{ + int status, i; + + /* Traffic flow */ + struct rte_table_hash_params key32lru_params = { + .name = "TABLE", + .key_size = 32, + .key_offset = APP_METADATA_OFFSET(32), + .key_mask = NULL, + .n_keys = 1 << 16, + .n_buckets = 1 << 16, + .f_hash = pipeline_test_hash, + .seed = 0, + }; + + uint8_t key32lru[32]; + uint32_t *k32lru = (uint32_t *) key32lru; + + memset(key32lru, 0, sizeof(key32lru)); + k32lru[0] = 0xadadadad; + + struct table_packets table_packets; + + printf("--------------\n"); + printf("RUNNING TEST - %s\n", __func__); + printf("--------------\n"); + for (i = 0; i < 50; i++) + table_packets.hit_packet[i] = 0xadadadad; + + for (i = 0; i < 50; i++) + table_packets.miss_packet[i] = 0xbdadadad; + + table_packets.n_hit_packets = 50; + table_packets.n_miss_packets = 50; + + status = test_table_type(&rte_table_hash_key32_lru_ops, + (void *)&key32lru_params, (void *)key32lru, &table_packets, + NULL, 0); + VERIFY(status, CHECK_TABLE_OK); + + /* Invalid parameters */ + key32lru_params.n_keys = 0; + + status = test_table_type(&rte_table_hash_key32_lru_ops, + (void *)&key32lru_params, (void *)key32lru, &table_packets, + NULL, 0); + VERIFY(status, CHECK_TABLE_TABLE_CONFIG); + + key32lru_params.n_keys = 1<<16; + key32lru_params.f_hash = NULL; + + status = test_table_type(&rte_table_hash_key32_lru_ops, + (void *)&key32lru_params, (void *)key32lru, &table_packets, + NULL, 0); + VERIFY(status, CHECK_TABLE_TABLE_CONFIG); + + return 0; +} + +int +test_table_hash8ext(void) +{ + int status, i; + + /* Traffic flow */ + struct rte_table_hash_params key8ext_params = { + .name = "TABLE", + .key_size = 8, + .key_offset = APP_METADATA_OFFSET(32), + .key_mask = NULL, + .n_keys = 1 << 16, + .n_buckets = 1 << 16, + .f_hash = pipeline_test_hash, + .seed = 0, + }; + + uint8_t key8ext[8]; + uint32_t *k8ext = (uint32_t *) key8ext; + + memset(key8ext, 0, sizeof(key8ext)); + k8ext[0] = 0xadadadad; + + struct table_packets table_packets; + + printf("--------------\n"); + printf("RUNNING TEST - %s\n", __func__); + printf("--------------\n"); + for (i = 0; i < 50; i++) + table_packets.hit_packet[i] = 0xadadadad; + + for (i = 0; i < 50; i++) + table_packets.miss_packet[i] = 0xbdadadad; + + table_packets.n_hit_packets = 50; + table_packets.n_miss_packets = 50; + + status = test_table_type(&rte_table_hash_key8_ext_ops, + (void *)&key8ext_params, (void *)key8ext, &table_packets, + NULL, 0); + VERIFY(status, CHECK_TABLE_OK); + + /* Invalid parameters */ + key8ext_params.n_keys = 0; + + status = test_table_type(&rte_table_hash_key8_ext_ops, + (void *)&key8ext_params, (void *)key8ext, &table_packets, + NULL, 0); + VERIFY(status, CHECK_TABLE_TABLE_CONFIG); + + key8ext_params.n_keys = 1<<16; + key8ext_params.f_hash = NULL; + + status = test_table_type(&rte_table_hash_key8_ext_ops, + (void *)&key8ext_params, (void *)key8ext, &table_packets, + NULL, 0); + VERIFY(status, CHECK_TABLE_TABLE_CONFIG); + + return 0; +} + +int +test_table_hash16ext(void) +{ + int status, i; + + /* Traffic flow */ + struct rte_table_hash_params key16ext_params = { + .name = "TABLE", + .key_size = 16, + .key_offset = APP_METADATA_OFFSET(32), + .key_mask = NULL, + .n_keys = 1 << 16, + .n_buckets = 1 << 16, + .f_hash = pipeline_test_hash, + .seed = 0, + }; + + uint8_t key16ext[16]; + uint32_t *k16ext = (uint32_t *) key16ext; + + memset(key16ext, 0, sizeof(key16ext)); + k16ext[0] = 0xadadadad; + + struct table_packets table_packets; + + printf("--------------\n"); + printf("RUNNING TEST - %s\n", __func__); + printf("--------------\n"); + for (i = 0; i < 50; i++) + table_packets.hit_packet[i] = 0xadadadad; + + for (i = 0; i < 50; i++) + table_packets.miss_packet[i] = 0xbdadadad; + + table_packets.n_hit_packets = 50; + table_packets.n_miss_packets = 50; + + status = test_table_type(&rte_table_hash_key16_ext_ops, + (void *)&key16ext_params, (void *)key16ext, &table_packets, + NULL, 0); + VERIFY(status, CHECK_TABLE_OK); + + /* Invalid parameters */ + key16ext_params.n_keys = 0; + + status = test_table_type(&rte_table_hash_key16_ext_ops, + (void *)&key16ext_params, (void *)key16ext, &table_packets, + NULL, 0); + VERIFY(status, CHECK_TABLE_TABLE_CONFIG); + + key16ext_params.n_keys = 1<<16; + key16ext_params.f_hash = NULL; + + status = test_table_type(&rte_table_hash_key16_ext_ops, + (void *)&key16ext_params, (void *)key16ext, &table_packets, + NULL, 0); + VERIFY(status, CHECK_TABLE_TABLE_CONFIG); + + return 0; +} + +int +test_table_hash32ext(void) +{ + int status, i; + + /* Traffic flow */ + struct rte_table_hash_params key32ext_params = { + .name = "TABLE", + .key_size = 32, + .key_offset = APP_METADATA_OFFSET(32), + .key_mask = NULL, + .n_keys = 1 << 16, + .n_buckets = 1 << 16, + .f_hash = pipeline_test_hash, + .seed = 0, + }; + + uint8_t key32ext[32]; + uint32_t *k32ext = (uint32_t *) key32ext; + + memset(key32ext, 0, sizeof(key32ext)); + k32ext[0] = 0xadadadad; + + struct table_packets table_packets; + + printf("--------------\n"); + printf("RUNNING TEST - %s\n", __func__); + printf("--------------\n"); + for (i = 0; i < 50; i++) + table_packets.hit_packet[i] = 0xadadadad; + + for (i = 0; i < 50; i++) + table_packets.miss_packet[i] = 0xbdadadad; + + table_packets.n_hit_packets = 50; + table_packets.n_miss_packets = 50; + + status = test_table_type(&rte_table_hash_key32_ext_ops, + (void *)&key32ext_params, (void *)key32ext, &table_packets, + NULL, 0); + VERIFY(status, CHECK_TABLE_OK); + + /* Invalid parameters */ + key32ext_params.n_keys = 0; + + status = test_table_type(&rte_table_hash_key32_ext_ops, + (void *)&key32ext_params, (void *)key32ext, &table_packets, + NULL, 0); + VERIFY(status, CHECK_TABLE_TABLE_CONFIG); + + key32ext_params.n_keys = 1<<16; + key32ext_params.f_hash = NULL; + + status = test_table_type(&rte_table_hash_key32_ext_ops, + (void *)&key32ext_params, (void *)key32ext, &table_packets, + NULL, 0); + VERIFY(status, CHECK_TABLE_TABLE_CONFIG); + + return 0; +} + +int +test_table_hash_cuckoo_combined(void) +{ + int status, i; + + /* Traffic flow */ + struct rte_table_hash_params cuckoo_params = { + .name = "TABLE", + .key_size = 32, + .key_offset = APP_METADATA_OFFSET(32), + .key_mask = NULL, + .n_keys = 1 << 16, + .n_buckets = 1 << 16, + .f_hash = pipeline_test_hash, + .seed = 0, + }; + + uint8_t key_cuckoo[32]; + uint32_t *kcuckoo = (uint32_t *) key_cuckoo; + + memset(key_cuckoo, 0, sizeof(key_cuckoo)); + kcuckoo[0] = 0xadadadad; + + struct table_packets table_packets; + + printf("--------------\n"); + printf("RUNNING TEST - %s\n", __func__); + printf("--------------\n"); + for (i = 0; i < 50; i++) + table_packets.hit_packet[i] = 0xadadadad; + + for (i = 0; i < 50; i++) + table_packets.miss_packet[i] = 0xbdadadad; + + table_packets.n_hit_packets = 50; + table_packets.n_miss_packets = 50; + + status = test_table_type(&rte_table_hash_cuckoo_ops, + (void *)&cuckoo_params, (void *)key_cuckoo, &table_packets, + NULL, 0); + VERIFY(status, CHECK_TABLE_OK); + + /* Invalid parameters */ + cuckoo_params.key_size = 0; + + status = test_table_type(&rte_table_hash_cuckoo_ops, + (void *)&cuckoo_params, (void *)key_cuckoo, &table_packets, + NULL, 0); + VERIFY(status, CHECK_TABLE_TABLE_CONFIG); + + cuckoo_params.key_size = 32; + cuckoo_params.n_keys = 0; + + status = test_table_type(&rte_table_hash_cuckoo_ops, + (void *)&cuckoo_params, (void *)key_cuckoo, &table_packets, + NULL, 0); + VERIFY(status, CHECK_TABLE_TABLE_CONFIG); + + cuckoo_params.n_keys = 1<<16; + cuckoo_params.f_hash = NULL; + + status = test_table_type(&rte_table_hash_cuckoo_ops, + (void *)&cuckoo_params, (void *)key_cuckoo, &table_packets, + NULL, 0); + VERIFY(status, CHECK_TABLE_TABLE_CONFIG); + + return 0; +} + diff --git a/dpdk/test/test/test_table_combined.h b/dpdk/test/test/test_table_combined.h new file mode 100644 index 00000000..e1619f92 --- /dev/null +++ b/dpdk/test/test/test_table_combined.h @@ -0,0 +1,56 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* Test prototypes */ +int test_table_stub_combined(void); +int test_table_lpm_combined(void); +int test_table_lpm_ipv6_combined(void); +#ifdef RTE_LIBRTE_ACL +int test_table_acl(void); +#endif +int test_table_hash8unoptimized(void); +int test_table_hash8lru(void); +int test_table_hash8ext(void); +int test_table_hash16unoptimized(void); +int test_table_hash16lru(void); +int test_table_hash16ext(void); +int test_table_hash32unoptimized(void); +int test_table_hash32lru(void); +int test_table_hash32ext(void); +int test_table_hash_cuckoo_combined(void); + +/* Extern variables */ +typedef int (*combined_table_test)(void); + +extern combined_table_test table_tests_combined[]; +extern unsigned n_table_tests_combined; diff --git a/dpdk/test/test/test_table_pipeline.c b/dpdk/test/test/test_table_pipeline.c new file mode 100644 index 00000000..a6fef721 --- /dev/null +++ b/dpdk/test/test/test_table_pipeline.c @@ -0,0 +1,600 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <string.h> +#include <rte_pipeline.h> +#include <rte_log.h> +#include <inttypes.h> +#include <rte_hexdump.h> +#include "test_table.h" +#include "test_table_pipeline.h" + +#if 0 + +static rte_pipeline_port_out_action_handler port_action_0x00 + (struct rte_mbuf **pkts, uint32_t n, uint64_t *pkts_mask, void *arg); +static rte_pipeline_port_out_action_handler port_action_0xFF + (struct rte_mbuf **pkts, uint32_t n, uint64_t *pkts_mask, void *arg); +static rte_pipeline_port_out_action_handler port_action_stub + (struct rte_mbuf **pkts, uint32_t n, uint64_t *pkts_mask, void *arg); + + +rte_pipeline_port_out_action_handler port_action_0x00(struct rte_mbuf **pkts, + uint32_t n, + uint64_t *pkts_mask, + void *arg) +{ + RTE_SET_USED(pkts); + RTE_SET_USED(n); + RTE_SET_USED(arg); + printf("Port Action 0x00\n"); + *pkts_mask = 0x00; + return 0; +} + +rte_pipeline_port_out_action_handler port_action_0xFF(struct rte_mbuf **pkts, + uint32_t n, + uint64_t *pkts_mask, + void *arg) +{ + RTE_SET_USED(pkts); + RTE_SET_USED(n); + RTE_SET_USED(arg); + printf("Port Action 0xFF\n"); + *pkts_mask = 0xFF; + return 0; +} + +rte_pipeline_port_out_action_handler port_action_stub(struct rte_mbuf **pkts, + uint32_t n, + uint64_t *pkts_mask, + void *arg) +{ + RTE_SET_USED(pkts); + RTE_SET_USED(n); + RTE_SET_USED(pkts_mask); + RTE_SET_USED(arg); + printf("Port Action stub\n"); + return 0; +} + +#endif + +rte_pipeline_table_action_handler_hit +table_action_0x00(struct rte_pipeline *p, struct rte_mbuf **pkts, + uint64_t pkts_mask, struct rte_pipeline_table_entry **entry, void *arg); + +rte_pipeline_table_action_handler_hit +table_action_stub_hit(struct rte_pipeline *p, struct rte_mbuf **pkts, + uint64_t pkts_mask, struct rte_pipeline_table_entry **entry, void *arg); + +rte_pipeline_table_action_handler_miss +table_action_stub_miss(struct rte_pipeline *p, struct rte_mbuf **pkts, + uint64_t pkts_mask, struct rte_pipeline_table_entry **entry, void *arg); + +rte_pipeline_table_action_handler_hit +table_action_0x00(__attribute__((unused)) struct rte_pipeline *p, + __attribute__((unused)) struct rte_mbuf **pkts, + uint64_t pkts_mask, + __attribute__((unused)) struct rte_pipeline_table_entry **entry, + __attribute__((unused)) void *arg) +{ + printf("Table Action, setting pkts_mask to 0x00\n"); + pkts_mask = ~0x00; + rte_pipeline_ah_packet_drop(p, pkts_mask); + return 0; +} + +rte_pipeline_table_action_handler_hit +table_action_stub_hit(__attribute__((unused)) struct rte_pipeline *p, + __attribute__((unused)) struct rte_mbuf **pkts, + uint64_t pkts_mask, + __attribute__((unused)) struct rte_pipeline_table_entry **entry, + __attribute__((unused)) void *arg) +{ + printf("STUB Table Action Hit - doing nothing\n"); + printf("STUB Table Action Hit - setting mask to 0x%"PRIx64"\n", + override_hit_mask); + pkts_mask = (~override_hit_mask) & 0x3; + rte_pipeline_ah_packet_drop(p, pkts_mask); + return 0; +} + +rte_pipeline_table_action_handler_miss +table_action_stub_miss(struct rte_pipeline *p, + __attribute__((unused)) struct rte_mbuf **pkts, + uint64_t pkts_mask, + __attribute__((unused)) struct rte_pipeline_table_entry **entry, + __attribute__((unused)) void *arg) +{ + printf("STUB Table Action Miss - setting mask to 0x%"PRIx64"\n", + override_miss_mask); + pkts_mask = (~override_miss_mask) & 0x3; + rte_pipeline_ah_packet_drop(p, pkts_mask); + return 0; +} + +enum e_test_type { + e_TEST_STUB = 0, + e_TEST_LPM, + e_TEST_LPM6, + e_TEST_HASH_LRU_8, + e_TEST_HASH_LRU_16, + e_TEST_HASH_LRU_32, + e_TEST_HASH_EXT_8, + e_TEST_HASH_EXT_16, + e_TEST_HASH_EXT_32 +}; + +char pipeline_test_names[][64] = { + "Stub", + "LPM", + "LPMv6", + "8-bit LRU Hash", + "16-bit LRU Hash", + "32-bit LRU Hash", + "16-bit Ext Hash", + "8-bit Ext Hash", + "32-bit Ext Hash", + "" +}; + + +static int +cleanup_pipeline(void) +{ + + rte_pipeline_free(p); + + return 0; +} + + +static int check_pipeline_invalid_params(void); + +static int +check_pipeline_invalid_params(void) +{ + struct rte_pipeline_params pipeline_params_1 = { + .name = NULL, + .socket_id = 0, + }; + struct rte_pipeline_params pipeline_params_2 = { + .name = "PIPELINE", + .socket_id = -1, + }; + struct rte_pipeline_params pipeline_params_3 = { + .name = "PIPELINE", + .socket_id = 127, + }; + + p = rte_pipeline_create(NULL); + if (p != NULL) { + RTE_LOG(INFO, PIPELINE, + "%s: configured pipeline with null params\n", + __func__); + goto fail; + } + p = rte_pipeline_create(&pipeline_params_1); + if (p != NULL) { + RTE_LOG(INFO, PIPELINE, "%s: Configure pipeline with NULL " + "name\n", __func__); + goto fail; + } + + p = rte_pipeline_create(&pipeline_params_2); + if (p != NULL) { + RTE_LOG(INFO, PIPELINE, "%s: Configure pipeline with invalid " + "socket\n", __func__); + goto fail; + } + + p = rte_pipeline_create(&pipeline_params_3); + if (p != NULL) { + RTE_LOG(INFO, PIPELINE, "%s: Configure pipeline with invalid " + "socket\n", __func__); + goto fail; + } + + /* Check pipeline consistency */ + if (!rte_pipeline_check(p)) { + rte_panic("Pipeline consistency reported as OK\n"); + goto fail; + } + + + return 0; +fail: + return -1; +} + + +static int +setup_pipeline(int test_type) +{ + int ret; + int i; + struct rte_pipeline_params pipeline_params = { + .name = "PIPELINE", + .socket_id = 0, + }; + + RTE_LOG(INFO, PIPELINE, "%s: **** Setting up %s test\n", + __func__, pipeline_test_names[test_type]); + + /* Pipeline configuration */ + p = rte_pipeline_create(&pipeline_params); + if (p == NULL) { + RTE_LOG(INFO, PIPELINE, "%s: Failed to configure pipeline\n", + __func__); + goto fail; + } + + ret = rte_pipeline_free(p); + if (ret != 0) { + RTE_LOG(INFO, PIPELINE, "%s: Failed to free pipeline\n", + __func__); + goto fail; + } + + /* Pipeline configuration */ + p = rte_pipeline_create(&pipeline_params); + if (p == NULL) { + RTE_LOG(INFO, PIPELINE, "%s: Failed to configure pipeline\n", + __func__); + goto fail; + } + + + /* Input port configuration */ + for (i = 0; i < N_PORTS; i++) { + struct rte_port_ring_reader_params port_ring_params = { + .ring = rings_rx[i], + }; + + struct rte_pipeline_port_in_params port_params = { + .ops = &rte_port_ring_reader_ops, + .arg_create = (void *) &port_ring_params, + .f_action = NULL, + .burst_size = BURST_SIZE, + }; + + /* Put in action for some ports */ + if (i) + port_params.f_action = NULL; + + ret = rte_pipeline_port_in_create(p, &port_params, + &port_in_id[i]); + if (ret) { + rte_panic("Unable to configure input port %d, ret:%d\n", + i, ret); + goto fail; + } + } + + /* output Port configuration */ + for (i = 0; i < N_PORTS; i++) { + struct rte_port_ring_writer_params port_ring_params = { + .ring = rings_tx[i], + .tx_burst_sz = BURST_SIZE, + }; + + struct rte_pipeline_port_out_params port_params = { + .ops = &rte_port_ring_writer_ops, + .arg_create = (void *) &port_ring_params, + .f_action = NULL, + .arg_ah = NULL, + }; + + if (i) + port_params.f_action = port_out_action; + + if (rte_pipeline_port_out_create(p, &port_params, + &port_out_id[i])) { + rte_panic("Unable to configure output port %d\n", i); + goto fail; + } + } + + /* Table configuration */ + for (i = 0; i < N_PORTS; i++) { + struct rte_pipeline_table_params table_params = { + .ops = &rte_table_stub_ops, + .arg_create = NULL, + .f_action_hit = action_handler_hit, + .f_action_miss = action_handler_miss, + .action_data_size = 0, + }; + + if (rte_pipeline_table_create(p, &table_params, &table_id[i])) { + rte_panic("Unable to configure table %u\n", i); + goto fail; + } + + if (connect_miss_action_to_table) + if (rte_pipeline_table_create(p, &table_params, + &table_id[i+2])) { + rte_panic("Unable to configure table %u\n", i); + goto fail; + } + } + + for (i = 0; i < N_PORTS; i++) + if (rte_pipeline_port_in_connect_to_table(p, port_in_id[i], + table_id[i])) { + rte_panic("Unable to connect input port %u to " + "table %u\n", port_in_id[i], table_id[i]); + goto fail; + } + + /* Add entries to tables */ + for (i = 0; i < N_PORTS; i++) { + struct rte_pipeline_table_entry default_entry = { + .action = (enum rte_pipeline_action) + table_entry_default_action, + {.port_id = port_out_id[i^1]}, + }; + struct rte_pipeline_table_entry *default_entry_ptr; + + if (connect_miss_action_to_table) { + printf("Setting first table to output to next table\n"); + default_entry.action = RTE_PIPELINE_ACTION_TABLE; + default_entry.table_id = table_id[i+2]; + } + + /* Add the default action for the table. */ + ret = rte_pipeline_table_default_entry_add(p, table_id[i], + &default_entry, &default_entry_ptr); + if (ret < 0) { + rte_panic("Unable to add default entry to table %u " + "code %d\n", table_id[i], ret); + goto fail; + } else + printf("Added default entry to table id %d with " + "action %x\n", + table_id[i], default_entry.action); + + if (connect_miss_action_to_table) { + /* We create a second table so the first can pass + traffic into it */ + struct rte_pipeline_table_entry default_entry = { + .action = RTE_PIPELINE_ACTION_PORT, + {.port_id = port_out_id[i^1]}, + }; + printf("Setting secont table to output to port\n"); + + /* Add the default action for the table. */ + ret = rte_pipeline_table_default_entry_add(p, + table_id[i+2], + &default_entry, &default_entry_ptr); + if (ret < 0) { + rte_panic("Unable to add default entry to " + "table %u code %d\n", + table_id[i], ret); + goto fail; + } else + printf("Added default entry to table id %d " + "with action %x\n", + table_id[i], default_entry.action); + } + } + + /* Enable input ports */ + for (i = 0; i < N_PORTS ; i++) + if (rte_pipeline_port_in_enable(p, port_in_id[i])) + rte_panic("Unable to enable input port %u\n", + port_in_id[i]); + + /* Check pipeline consistency */ + if (rte_pipeline_check(p) < 0) { + rte_panic("Pipeline consistency check failed\n"); + goto fail; + } else + printf("Pipeline Consistency OK!\n"); + + return 0; +fail: + + return -1; +} + +static int +test_pipeline_single_filter(int test_type, int expected_count) +{ + int i; + int j; + int ret; + int tx_count; + + RTE_LOG(INFO, PIPELINE, "%s: **** Running %s test\n", + __func__, pipeline_test_names[test_type]); + /* Run pipeline once */ + for (i = 0; i < N_PORTS; i++) + rte_pipeline_run(p); + + + ret = rte_pipeline_flush(NULL); + if (ret != -EINVAL) { + RTE_LOG(INFO, PIPELINE, + "%s: No pipeline flush error NULL pipeline (%d)\n", + __func__, ret); + goto fail; + } + + /* + * Allocate a few mbufs and manually insert into the rings. */ + for (i = 0; i < N_PORTS; i++) + for (j = 0; j < N_PORTS; j++) { + struct rte_mbuf *m; + uint8_t *key; + uint32_t *k32; + + m = rte_pktmbuf_alloc(pool); + if (m == NULL) { + rte_panic("Failed to alloc mbuf from pool\n"); + return -1; + } + key = RTE_MBUF_METADATA_UINT8_PTR(m, + APP_METADATA_OFFSET(32)); + + k32 = (uint32_t *) key; + k32[0] = 0xadadadad >> (j % 2); + + RTE_LOG(INFO, PIPELINE, "%s: Enqueue onto ring %d\n", + __func__, i); + rte_ring_enqueue(rings_rx[i], m); + } + + /* Run pipeline once */ + for (i = 0; i < N_PORTS; i++) + rte_pipeline_run(p); + + /* + * need to flush the pipeline, as there may be less hits than the burst + size and they will not have been flushed to the tx rings. */ + rte_pipeline_flush(p); + + /* + * Now we'll see what we got back on the tx rings. We should see whatever + * packets we had hits on that were destined for the output ports. + */ + tx_count = 0; + + for (i = 0; i < N_PORTS; i++) { + void *objs[RING_TX_SIZE]; + struct rte_mbuf *mbuf; + + ret = rte_ring_sc_dequeue_burst(rings_tx[i], objs, 10, NULL); + if (ret <= 0) + printf("Got no objects from ring %d - error code %d\n", + i, ret); + else { + printf("Got %d object(s) from ring %d!\n", ret, i); + for (j = 0; j < ret; j++) { + mbuf = objs[j]; + rte_hexdump(stdout, "Object:", + rte_pktmbuf_mtod(mbuf, char *), + mbuf->data_len); + rte_pktmbuf_free(mbuf); + } + tx_count += ret; + } + } + + if (tx_count != expected_count) { + RTE_LOG(INFO, PIPELINE, + "%s: Unexpected packets out for %s test, expected %d, " + "got %d\n", __func__, pipeline_test_names[test_type], + expected_count, tx_count); + goto fail; + } + + cleanup_pipeline(); + + return 0; +fail: + return -1; + +} + +int +test_table_pipeline(void) +{ + /* TEST - All packets dropped */ + action_handler_hit = NULL; + action_handler_miss = NULL; + table_entry_default_action = RTE_PIPELINE_ACTION_DROP; + setup_pipeline(e_TEST_STUB); + if (test_pipeline_single_filter(e_TEST_STUB, 0) < 0) + return -1; + + /* TEST - All packets passed through */ + table_entry_default_action = RTE_PIPELINE_ACTION_PORT; + setup_pipeline(e_TEST_STUB); + if (test_pipeline_single_filter(e_TEST_STUB, 4) < 0) + return -1; + + /* TEST - one packet per port */ + action_handler_hit = NULL; + action_handler_miss = + (rte_pipeline_table_action_handler_miss) table_action_stub_miss; + table_entry_default_action = RTE_PIPELINE_ACTION_PORT; + override_miss_mask = 0x01; /* one packet per port */ + setup_pipeline(e_TEST_STUB); + if (test_pipeline_single_filter(e_TEST_STUB, 2) < 0) + return -1; + + /* TEST - one packet per port */ + override_miss_mask = 0x02; /*all per port */ + setup_pipeline(e_TEST_STUB); + if (test_pipeline_single_filter(e_TEST_STUB, 2) < 0) + return -1; + + /* TEST - all packets per port */ + override_miss_mask = 0x03; /*all per port */ + setup_pipeline(e_TEST_STUB); + if (test_pipeline_single_filter(e_TEST_STUB, 4) < 0) + return -1; + + /* + * This test will set up two tables in the pipeline. the first table + * will forward to another table on miss, and the second table will + * forward to port. + */ + connect_miss_action_to_table = 1; + table_entry_default_action = RTE_PIPELINE_ACTION_TABLE; + action_handler_hit = NULL; /* not for stub, hitmask always zero */ + action_handler_miss = NULL; + setup_pipeline(e_TEST_STUB); + if (test_pipeline_single_filter(e_TEST_STUB, 4) < 0) + return -1; + connect_miss_action_to_table = 0; + + printf("TEST - two tables, hitmask override to 0x01\n"); + connect_miss_action_to_table = 1; + action_handler_miss = + (rte_pipeline_table_action_handler_miss)table_action_stub_miss; + override_miss_mask = 0x01; + setup_pipeline(e_TEST_STUB); + if (test_pipeline_single_filter(e_TEST_STUB, 2) < 0) + return -1; + connect_miss_action_to_table = 0; + + if (check_pipeline_invalid_params()) { + RTE_LOG(INFO, PIPELINE, "%s: Check pipeline invalid params " + "failed.\n", __func__); + return -1; + } + + return 0; +} diff --git a/dpdk/test/test/test_table_pipeline.h b/dpdk/test/test/test_table_pipeline.h new file mode 100644 index 00000000..b3f20ba3 --- /dev/null +++ b/dpdk/test/test/test_table_pipeline.h @@ -0,0 +1,35 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* Test prototypes */ +int test_table_pipeline(void); diff --git a/dpdk/test/test/test_table_ports.c b/dpdk/test/test/test_table_ports.c new file mode 100644 index 00000000..39592ce1 --- /dev/null +++ b/dpdk/test/test/test_table_ports.c @@ -0,0 +1,220 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "test_table_ports.h" +#include "test_table.h" + +port_test port_tests[] = { + test_port_ring_reader, + test_port_ring_writer, +}; + +unsigned n_port_tests = RTE_DIM(port_tests); + +/* Port tests */ +int +test_port_ring_reader(void) +{ + int status, i; + struct rte_port_ring_reader_params port_ring_reader_params; + void *port; + + /* Invalid params */ + port = rte_port_ring_reader_ops.f_create(NULL, 0); + if (port != NULL) + return -1; + + status = rte_port_ring_reader_ops.f_free(port); + if (status >= 0) + return -2; + + /* Create and free */ + port_ring_reader_params.ring = RING_RX; + port = rte_port_ring_reader_ops.f_create(&port_ring_reader_params, 0); + if (port == NULL) + return -3; + + status = rte_port_ring_reader_ops.f_free(port); + if (status != 0) + return -4; + + /* -- Traffic RX -- */ + int expected_pkts, received_pkts; + struct rte_mbuf *res_mbuf[RTE_PORT_IN_BURST_SIZE_MAX]; + void *mbuf[RTE_PORT_IN_BURST_SIZE_MAX]; + + port_ring_reader_params.ring = RING_RX; + port = rte_port_ring_reader_ops.f_create(&port_ring_reader_params, 0); + + /* Single packet */ + mbuf[0] = (void *)rte_pktmbuf_alloc(pool); + + expected_pkts = rte_ring_sp_enqueue_burst(port_ring_reader_params.ring, + mbuf, 1, NULL); + received_pkts = rte_port_ring_reader_ops.f_rx(port, res_mbuf, 1); + + if (received_pkts < expected_pkts) + return -5; + + rte_pktmbuf_free(res_mbuf[0]); + + /* Multiple packets */ + for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) + mbuf[i] = rte_pktmbuf_alloc(pool); + + expected_pkts = rte_ring_sp_enqueue_burst(port_ring_reader_params.ring, + (void * const *) mbuf, RTE_PORT_IN_BURST_SIZE_MAX, NULL); + received_pkts = rte_port_ring_reader_ops.f_rx(port, res_mbuf, + RTE_PORT_IN_BURST_SIZE_MAX); + + if (received_pkts < expected_pkts) + return -6; + + for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) + rte_pktmbuf_free(res_mbuf[i]); + + return 0; +} + +int +test_port_ring_writer(void) +{ + int status, i; + struct rte_port_ring_writer_params port_ring_writer_params; + void *port; + + /* Invalid params */ + port = rte_port_ring_writer_ops.f_create(NULL, 0); + if (port != NULL) + return -1; + + status = rte_port_ring_writer_ops.f_free(port); + if (status >= 0) + return -2; + + port_ring_writer_params.ring = NULL; + + port = rte_port_ring_writer_ops.f_create(&port_ring_writer_params, 0); + if (port != NULL) + return -3; + + port_ring_writer_params.ring = RING_TX; + port_ring_writer_params.tx_burst_sz = RTE_PORT_IN_BURST_SIZE_MAX + 1; + + port = rte_port_ring_writer_ops.f_create(&port_ring_writer_params, 0); + if (port != NULL) + return -4; + + /* Create and free */ + port_ring_writer_params.ring = RING_TX; + port_ring_writer_params.tx_burst_sz = RTE_PORT_IN_BURST_SIZE_MAX; + + port = rte_port_ring_writer_ops.f_create(&port_ring_writer_params, 0); + if (port == NULL) + return -5; + + status = rte_port_ring_writer_ops.f_free(port); + if (status != 0) + return -6; + + /* -- Traffic TX -- */ + int expected_pkts, received_pkts; + struct rte_mbuf *mbuf[RTE_PORT_IN_BURST_SIZE_MAX]; + struct rte_mbuf *res_mbuf[RTE_PORT_IN_BURST_SIZE_MAX]; + + port_ring_writer_params.ring = RING_TX; + port_ring_writer_params.tx_burst_sz = RTE_PORT_IN_BURST_SIZE_MAX; + port = rte_port_ring_writer_ops.f_create(&port_ring_writer_params, 0); + + /* Single packet */ + mbuf[0] = rte_pktmbuf_alloc(pool); + + rte_port_ring_writer_ops.f_tx(port, mbuf[0]); + rte_port_ring_writer_ops.f_flush(port); + expected_pkts = 1; + received_pkts = rte_ring_sc_dequeue_burst(port_ring_writer_params.ring, + (void **)res_mbuf, port_ring_writer_params.tx_burst_sz, NULL); + + if (received_pkts < expected_pkts) + return -7; + + rte_pktmbuf_free(res_mbuf[0]); + + /* Multiple packets */ + for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) { + mbuf[i] = rte_pktmbuf_alloc(pool); + rte_port_ring_writer_ops.f_tx(port, mbuf[i]); + } + + expected_pkts = RTE_PORT_IN_BURST_SIZE_MAX; + received_pkts = rte_ring_sc_dequeue_burst(port_ring_writer_params.ring, + (void **)res_mbuf, port_ring_writer_params.tx_burst_sz, NULL); + + if (received_pkts < expected_pkts) + return -8; + + for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) + rte_pktmbuf_free(res_mbuf[i]); + + /* TX Bulk */ + for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) + mbuf[i] = rte_pktmbuf_alloc(pool); + rte_port_ring_writer_ops.f_tx_bulk(port, mbuf, (uint64_t)-1); + + expected_pkts = RTE_PORT_IN_BURST_SIZE_MAX; + received_pkts = rte_ring_sc_dequeue_burst(port_ring_writer_params.ring, + (void **)res_mbuf, port_ring_writer_params.tx_burst_sz, NULL); + + if (received_pkts < expected_pkts) + return -8; + + for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) + rte_pktmbuf_free(res_mbuf[i]); + + for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) + mbuf[i] = rte_pktmbuf_alloc(pool); + rte_port_ring_writer_ops.f_tx_bulk(port, mbuf, (uint64_t)-3); + rte_port_ring_writer_ops.f_tx_bulk(port, mbuf, (uint64_t)2); + + expected_pkts = RTE_PORT_IN_BURST_SIZE_MAX; + received_pkts = rte_ring_sc_dequeue_burst(port_ring_writer_params.ring, + (void **)res_mbuf, port_ring_writer_params.tx_burst_sz, NULL); + + if (received_pkts < expected_pkts) + return -9; + + for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) + rte_pktmbuf_free(res_mbuf[i]); + + return 0; +} diff --git a/dpdk/test/test/test_table_ports.h b/dpdk/test/test/test_table_ports.h new file mode 100644 index 00000000..512b77fe --- /dev/null +++ b/dpdk/test/test/test_table_ports.h @@ -0,0 +1,42 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* Test prototypes */ +int test_port_ring_reader(void); +int test_port_ring_writer(void); + +/* Extern variables */ +typedef int (*port_test)(void); + +extern port_test port_tests[]; +extern unsigned n_port_tests; diff --git a/dpdk/test/test/test_table_tables.c b/dpdk/test/test/test_table_tables.c new file mode 100644 index 00000000..a1b0c582 --- /dev/null +++ b/dpdk/test/test/test_table_tables.c @@ -0,0 +1,1083 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <string.h> +#include <rte_byteorder.h> +#include <rte_table_lpm_ipv6.h> +#include <rte_lru.h> +#include <rte_cycles.h> +#include "test_table_tables.h" +#include "test_table.h" + +table_test table_tests[] = { + test_table_stub, + test_table_array, + test_table_lpm, + test_table_lpm_ipv6, + test_table_hash_lru, + test_table_hash_ext, + test_table_hash_cuckoo, +}; + +#define PREPARE_PACKET(mbuf, value) do { \ + uint32_t *k32, *signature; \ + uint8_t *key; \ + mbuf = rte_pktmbuf_alloc(pool); \ + signature = RTE_MBUF_METADATA_UINT32_PTR(mbuf, \ + APP_METADATA_OFFSET(0)); \ + key = RTE_MBUF_METADATA_UINT8_PTR(mbuf, \ + APP_METADATA_OFFSET(32)); \ + memset(key, 0, 32); \ + k32 = (uint32_t *) key; \ + k32[0] = (value); \ + *signature = pipeline_test_hash(key, NULL, 0, 0); \ +} while (0) + +unsigned n_table_tests = RTE_DIM(table_tests); + +/* Function prototypes */ +static int +test_table_hash_lru_generic(struct rte_table_ops *ops, uint32_t key_size); +static int +test_table_hash_ext_generic(struct rte_table_ops *ops, uint32_t key_size); + +struct rte_bucket_4_8 { + /* Cache line 0 */ + uint64_t signature; + uint64_t lru_list; + struct rte_bucket_4_8 *next; + uint64_t next_valid; + uint64_t key[4]; + /* Cache line 1 */ + uint8_t data[0]; +}; + +#if RTE_TABLE_HASH_LRU_STRATEGY == 3 +uint64_t shuffles = 0xfffffffdfffbfff9ULL; +#else +uint64_t shuffles = 0x0003000200010000ULL; +#endif + +static int test_lru_update(void) +{ + struct rte_bucket_4_8 b; + struct rte_bucket_4_8 *bucket; + uint32_t i; + uint64_t pos; + uint64_t iterations; + uint64_t j; + int poss; + + printf("---------------------------\n"); + printf("Testing lru_update macro...\n"); + printf("---------------------------\n"); + bucket = &b; + iterations = 10; +#if RTE_TABLE_HASH_LRU_STRATEGY == 3 + bucket->lru_list = 0xFFFFFFFFFFFFFFFFULL; +#else + bucket->lru_list = 0x0000000100020003ULL; +#endif + poss = 0; + for (j = 0; j < iterations; j++) + for (i = 0; i < 9; i++) { + uint32_t idx = i >> 1; + lru_update(bucket, idx); + pos = lru_pos(bucket); + poss += pos; + printf("%s: %d lru_list=%016"PRIx64", upd=%d, " + "pos=%"PRIx64"\n", + __func__, i, bucket->lru_list, i>>1, pos); + } + + if (bucket->lru_list != shuffles) { + printf("%s: ERROR: %d lru_list=%016"PRIx64", expected %016" + PRIx64"\n", + __func__, i, bucket->lru_list, shuffles); + return -1; + } + printf("%s: output checksum of results =%d\n", + __func__, poss); +#if 0 + if (poss != 126) { + printf("%s: ERROR output checksum of results =%d expected %d\n", + __func__, poss, 126); + return -1; + } +#endif + + fflush(stdout); + + uint64_t sc_start = rte_rdtsc(); + iterations = 100000000; + poss = 0; + for (j = 0; j < iterations; j++) { + for (i = 0; i < 4; i++) { + lru_update(bucket, i); + pos |= bucket->lru_list; + } + } + uint64_t sc_end = rte_rdtsc(); + + printf("%s: output checksum of results =%llu\n", + __func__, (long long unsigned int)pos); + printf("%s: start=%016"PRIx64", end=%016"PRIx64"\n", + __func__, sc_start, sc_end); + printf("\nlru_update: %lu cycles per loop iteration.\n\n", + (long unsigned int)((sc_end-sc_start)/(iterations*4))); + + return 0; +} + +/* Table tests */ +int +test_table_stub(void) +{ + int i; + uint64_t expected_mask = 0, result_mask; + struct rte_mbuf *mbufs[RTE_PORT_IN_BURST_SIZE_MAX]; + void *table; + char *entries[RTE_PORT_IN_BURST_SIZE_MAX]; + + /* Create */ + table = rte_table_stub_ops.f_create(NULL, 0, 1); + if (table == NULL) + return -1; + + /* Traffic flow */ + for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) + if (i % 2 == 0) + PREPARE_PACKET(mbufs[i], 0xadadadad); + else + PREPARE_PACKET(mbufs[i], 0xadadadab); + + expected_mask = 0; + rte_table_stub_ops.f_lookup(table, mbufs, -1, + &result_mask, (void **)entries); + if (result_mask != expected_mask) + return -2; + + /* Free resources */ + for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) + rte_pktmbuf_free(mbufs[i]); + + return 0; +} + +int +test_table_array(void) +{ + int status, i; + uint64_t result_mask; + struct rte_mbuf *mbufs[RTE_PORT_IN_BURST_SIZE_MAX]; + void *table; + char *entries[RTE_PORT_IN_BURST_SIZE_MAX]; + char entry1, entry2; + void *entry_ptr; + int key_found; + + /* Initialize params and create tables */ + struct rte_table_array_params array_params = { + .n_entries = 7, + .offset = APP_METADATA_OFFSET(1) + }; + + table = rte_table_array_ops.f_create(NULL, 0, 1); + if (table != NULL) + return -1; + + array_params.n_entries = 0; + + table = rte_table_array_ops.f_create(&array_params, 0, 1); + if (table != NULL) + return -2; + + array_params.n_entries = 7; + + table = rte_table_array_ops.f_create(&array_params, 0, 1); + if (table != NULL) + return -3; + + array_params.n_entries = 1 << 24; + array_params.offset = APP_METADATA_OFFSET(1); + + table = rte_table_array_ops.f_create(&array_params, 0, 1); + if (table == NULL) + return -4; + + array_params.offset = APP_METADATA_OFFSET(32); + + table = rte_table_array_ops.f_create(&array_params, 0, 1); + if (table == NULL) + return -5; + + /* Free */ + status = rte_table_array_ops.f_free(table); + if (status < 0) + return -6; + + status = rte_table_array_ops.f_free(NULL); + if (status == 0) + return -7; + + /* Add */ + struct rte_table_array_key array_key_1 = { + .pos = 10, + }; + struct rte_table_array_key array_key_2 = { + .pos = 20, + }; + entry1 = 'A'; + entry2 = 'B'; + + table = rte_table_array_ops.f_create(&array_params, 0, 1); + if (table == NULL) + return -8; + + status = rte_table_array_ops.f_add(NULL, (void *) &array_key_1, &entry1, + &key_found, &entry_ptr); + if (status == 0) + return -9; + + status = rte_table_array_ops.f_add(table, (void *) &array_key_1, NULL, + &key_found, &entry_ptr); + if (status == 0) + return -10; + + status = rte_table_array_ops.f_add(table, (void *) &array_key_1, + &entry1, &key_found, &entry_ptr); + if (status != 0) + return -11; + + /* Traffic flow */ + status = rte_table_array_ops.f_add(table, (void *) &array_key_2, + &entry2, &key_found, &entry_ptr); + if (status != 0) + return -12; + + for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) + if (i % 2 == 0) + PREPARE_PACKET(mbufs[i], 10); + else + PREPARE_PACKET(mbufs[i], 20); + + rte_table_array_ops.f_lookup(table, mbufs, -1, + &result_mask, (void **)entries); + + for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) + if (i % 2 == 0 && *entries[i] != 'A') + return -13; + else + if (i % 2 == 1 && *entries[i] != 'B') + return -13; + + /* Free resources */ + for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) + rte_pktmbuf_free(mbufs[i]); + + status = rte_table_array_ops.f_free(table); + + return 0; +} + +int +test_table_lpm(void) +{ + int status, i; + uint64_t expected_mask = 0, result_mask; + struct rte_mbuf *mbufs[RTE_PORT_IN_BURST_SIZE_MAX]; + void *table; + char *entries[RTE_PORT_IN_BURST_SIZE_MAX]; + char entry; + void *entry_ptr; + int key_found; + uint32_t entry_size = 1; + + /* Initialize params and create tables */ + struct rte_table_lpm_params lpm_params = { + .name = "LPM", + .n_rules = 1 << 24, + .number_tbl8s = 1 << 8, + .flags = 0, + .entry_unique_size = entry_size, + .offset = APP_METADATA_OFFSET(1) + }; + + table = rte_table_lpm_ops.f_create(NULL, 0, entry_size); + if (table != NULL) + return -1; + + lpm_params.name = NULL; + + table = rte_table_lpm_ops.f_create(&lpm_params, 0, entry_size); + if (table != NULL) + return -2; + + lpm_params.name = "LPM"; + lpm_params.n_rules = 0; + + table = rte_table_lpm_ops.f_create(&lpm_params, 0, entry_size); + if (table != NULL) + return -3; + + lpm_params.n_rules = 1 << 24; + lpm_params.offset = APP_METADATA_OFFSET(32); + lpm_params.entry_unique_size = 0; + + table = rte_table_lpm_ops.f_create(&lpm_params, 0, entry_size); + if (table != NULL) + return -4; + + lpm_params.entry_unique_size = entry_size + 1; + + table = rte_table_lpm_ops.f_create(&lpm_params, 0, entry_size); + if (table != NULL) + return -5; + + lpm_params.entry_unique_size = entry_size; + + table = rte_table_lpm_ops.f_create(&lpm_params, 0, entry_size); + if (table == NULL) + return -6; + + /* Free */ + status = rte_table_lpm_ops.f_free(table); + if (status < 0) + return -7; + + status = rte_table_lpm_ops.f_free(NULL); + if (status == 0) + return -8; + + /* Add */ + struct rte_table_lpm_key lpm_key; + lpm_key.ip = 0xadadadad; + + table = rte_table_lpm_ops.f_create(&lpm_params, 0, 1); + if (table == NULL) + return -9; + + status = rte_table_lpm_ops.f_add(NULL, &lpm_key, &entry, &key_found, + &entry_ptr); + if (status == 0) + return -10; + + status = rte_table_lpm_ops.f_add(table, NULL, &entry, &key_found, + &entry_ptr); + if (status == 0) + return -11; + + status = rte_table_lpm_ops.f_add(table, &lpm_key, NULL, &key_found, + &entry_ptr); + if (status == 0) + return -12; + + lpm_key.depth = 0; + status = rte_table_lpm_ops.f_add(table, &lpm_key, &entry, &key_found, + &entry_ptr); + if (status == 0) + return -13; + + lpm_key.depth = 33; + status = rte_table_lpm_ops.f_add(table, &lpm_key, &entry, &key_found, + &entry_ptr); + if (status == 0) + return -14; + + lpm_key.depth = 16; + status = rte_table_lpm_ops.f_add(table, &lpm_key, &entry, &key_found, + &entry_ptr); + if (status != 0) + return -15; + + /* Delete */ + status = rte_table_lpm_ops.f_delete(NULL, &lpm_key, &key_found, NULL); + if (status == 0) + return -16; + + status = rte_table_lpm_ops.f_delete(table, NULL, &key_found, NULL); + if (status == 0) + return -17; + + lpm_key.depth = 0; + status = rte_table_lpm_ops.f_delete(table, &lpm_key, &key_found, NULL); + if (status == 0) + return -18; + + lpm_key.depth = 33; + status = rte_table_lpm_ops.f_delete(table, &lpm_key, &key_found, NULL); + if (status == 0) + return -19; + + lpm_key.depth = 16; + status = rte_table_lpm_ops.f_delete(table, &lpm_key, &key_found, NULL); + if (status != 0) + return -20; + + status = rte_table_lpm_ops.f_delete(table, &lpm_key, &key_found, NULL); + if (status != 0) + return -21; + + /* Traffic flow */ + entry = 'A'; + status = rte_table_lpm_ops.f_add(table, &lpm_key, &entry, &key_found, + &entry_ptr); + if (status < 0) + return -22; + + for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) + if (i % 2 == 0) { + expected_mask |= (uint64_t)1 << i; + PREPARE_PACKET(mbufs[i], 0xadadadad); + } else + PREPARE_PACKET(mbufs[i], 0xadadadab); + + rte_table_lpm_ops.f_lookup(table, mbufs, -1, + &result_mask, (void **)entries); + if (result_mask != expected_mask) + return -23; + + /* Free resources */ + for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) + rte_pktmbuf_free(mbufs[i]); + + status = rte_table_lpm_ops.f_free(table); + + return 0; +} + +int +test_table_lpm_ipv6(void) +{ + int status, i; + uint64_t expected_mask = 0, result_mask; + struct rte_mbuf *mbufs[RTE_PORT_IN_BURST_SIZE_MAX]; + void *table; + char *entries[RTE_PORT_IN_BURST_SIZE_MAX]; + char entry; + void *entry_ptr; + int key_found; + uint32_t entry_size = 1; + + /* Initialize params and create tables */ + struct rte_table_lpm_ipv6_params lpm_params = { + .name = "LPM", + .n_rules = 1 << 24, + .number_tbl8s = 1 << 21, + .entry_unique_size = entry_size, + .offset = APP_METADATA_OFFSET(32) + }; + + table = rte_table_lpm_ipv6_ops.f_create(NULL, 0, entry_size); + if (table != NULL) + return -1; + + lpm_params.name = NULL; + + table = rte_table_lpm_ipv6_ops.f_create(&lpm_params, 0, entry_size); + if (table != NULL) + return -2; + + lpm_params.name = "LPM"; + lpm_params.n_rules = 0; + + table = rte_table_lpm_ipv6_ops.f_create(&lpm_params, 0, entry_size); + if (table != NULL) + return -3; + + lpm_params.n_rules = 1 << 24; + lpm_params.number_tbl8s = 0; + table = rte_table_lpm_ipv6_ops.f_create(&lpm_params, 0, entry_size); + if (table != NULL) + return -4; + + lpm_params.number_tbl8s = 1 << 21; + lpm_params.entry_unique_size = 0; + table = rte_table_lpm_ipv6_ops.f_create(&lpm_params, 0, entry_size); + if (table != NULL) + return -5; + + lpm_params.entry_unique_size = entry_size + 1; + table = rte_table_lpm_ipv6_ops.f_create(&lpm_params, 0, entry_size); + if (table != NULL) + return -6; + + lpm_params.entry_unique_size = entry_size; + lpm_params.offset = APP_METADATA_OFFSET(32); + + table = rte_table_lpm_ipv6_ops.f_create(&lpm_params, 0, entry_size); + if (table == NULL) + return -7; + + /* Free */ + status = rte_table_lpm_ipv6_ops.f_free(table); + if (status < 0) + return -8; + + status = rte_table_lpm_ipv6_ops.f_free(NULL); + if (status == 0) + return -9; + + /* Add */ + struct rte_table_lpm_ipv6_key lpm_key; + + lpm_key.ip[0] = 0xad; + lpm_key.ip[1] = 0xad; + lpm_key.ip[2] = 0xad; + lpm_key.ip[3] = 0xad; + + table = rte_table_lpm_ipv6_ops.f_create(&lpm_params, 0, entry_size); + if (table == NULL) + return -10; + + status = rte_table_lpm_ipv6_ops.f_add(NULL, &lpm_key, &entry, + &key_found, &entry_ptr); + if (status == 0) + return -11; + + status = rte_table_lpm_ipv6_ops.f_add(table, NULL, &entry, &key_found, + &entry_ptr); + if (status == 0) + return -12; + + status = rte_table_lpm_ipv6_ops.f_add(table, &lpm_key, NULL, &key_found, + &entry_ptr); + if (status == 0) + return -13; + + lpm_key.depth = 0; + status = rte_table_lpm_ipv6_ops.f_add(table, &lpm_key, &entry, + &key_found, &entry_ptr); + if (status == 0) + return -14; + + lpm_key.depth = 129; + status = rte_table_lpm_ipv6_ops.f_add(table, &lpm_key, &entry, + &key_found, &entry_ptr); + if (status == 0) + return -15; + + lpm_key.depth = 16; + status = rte_table_lpm_ipv6_ops.f_add(table, &lpm_key, &entry, + &key_found, &entry_ptr); + if (status != 0) + return -16; + + /* Delete */ + status = rte_table_lpm_ipv6_ops.f_delete(NULL, &lpm_key, &key_found, + NULL); + if (status == 0) + return -17; + + status = rte_table_lpm_ipv6_ops.f_delete(table, NULL, &key_found, NULL); + if (status == 0) + return -18; + + lpm_key.depth = 0; + status = rte_table_lpm_ipv6_ops.f_delete(table, &lpm_key, &key_found, + NULL); + if (status == 0) + return -19; + + lpm_key.depth = 129; + status = rte_table_lpm_ipv6_ops.f_delete(table, &lpm_key, &key_found, + NULL); + if (status == 0) + return -20; + + lpm_key.depth = 16; + status = rte_table_lpm_ipv6_ops.f_delete(table, &lpm_key, &key_found, + NULL); + if (status != 0) + return -21; + + status = rte_table_lpm_ipv6_ops.f_delete(table, &lpm_key, &key_found, + NULL); + if (status != 0) + return -22; + + /* Traffic flow */ + entry = 'A'; + status = rte_table_lpm_ipv6_ops.f_add(table, &lpm_key, &entry, + &key_found, &entry_ptr); + if (status < 0) + return -23; + + for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) + if (i % 2 == 0) { + expected_mask |= (uint64_t)1 << i; + PREPARE_PACKET(mbufs[i], 0xadadadad); + } else + PREPARE_PACKET(mbufs[i], 0xadadadab); + + rte_table_lpm_ipv6_ops.f_lookup(table, mbufs, -1, + &result_mask, (void **)entries); + if (result_mask != expected_mask) + return -24; + + /* Free resources */ + for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) + rte_pktmbuf_free(mbufs[i]); + + status = rte_table_lpm_ipv6_ops.f_free(table); + + return 0; +} + +static int +test_table_hash_lru_generic(struct rte_table_ops *ops, uint32_t key_size) +{ + int status, i; + uint64_t expected_mask = 0, result_mask; + struct rte_mbuf *mbufs[RTE_PORT_IN_BURST_SIZE_MAX]; + void *table; + char *entries[RTE_PORT_IN_BURST_SIZE_MAX]; + char entry; + void *entry_ptr; + int key_found; + + /* Initialize params and create tables */ + struct rte_table_hash_params hash_params = { + .name = "TABLE", + .key_size = key_size, + .key_offset = APP_METADATA_OFFSET(32), + .key_mask = NULL, + .n_keys = 1 << 10, + .n_buckets = 1 << 10, + .f_hash = pipeline_test_hash, + .seed = 0, + }; + + hash_params.n_keys = 0; + + table = ops->f_create(&hash_params, 0, 1); + if (table != NULL) + return -1; + + hash_params.n_keys = 1 << 10; + hash_params.f_hash = NULL; + + table = ops->f_create(&hash_params, 0, 1); + if (table != NULL) + return -4; + + hash_params.f_hash = pipeline_test_hash; + + table = ops->f_create(&hash_params, 0, 1); + if (table == NULL) + return -5; + + /* Free */ + status = ops->f_free(table); + if (status < 0) + return -6; + + status = ops->f_free(NULL); + if (status == 0) + return -7; + + /* Add */ + uint8_t key[32]; + uint32_t *k32 = (uint32_t *) &key; + + memset(key, 0, 32); + k32[0] = rte_be_to_cpu_32(0xadadadad); + + table = ops->f_create(&hash_params, 0, 1); + if (table == NULL) + return -8; + + entry = 'A'; + status = ops->f_add(table, &key, &entry, &key_found, &entry_ptr); + if (status != 0) + return -9; + + /* Delete */ + status = ops->f_delete(table, &key, &key_found, NULL); + if (status != 0) + return -10; + + status = ops->f_delete(table, &key, &key_found, NULL); + if (status != 0) + return -11; + + /* Traffic flow */ + entry = 'A'; + status = ops->f_add(table, &key, &entry, &key_found, &entry_ptr); + if (status < 0) + return -12; + + for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) + if (i % 2 == 0) { + expected_mask |= (uint64_t)1 << i; + PREPARE_PACKET(mbufs[i], 0xadadadad); + } else + PREPARE_PACKET(mbufs[i], 0xadadadab); + + ops->f_lookup(table, mbufs, -1, &result_mask, (void **)entries); + if (result_mask != expected_mask) + return -13; + + /* Free resources */ + for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) + rte_pktmbuf_free(mbufs[i]); + + status = ops->f_free(table); + + return 0; +} + +static int +test_table_hash_ext_generic(struct rte_table_ops *ops, uint32_t key_size) +{ + int status, i; + uint64_t expected_mask = 0, result_mask; + struct rte_mbuf *mbufs[RTE_PORT_IN_BURST_SIZE_MAX]; + void *table; + char *entries[RTE_PORT_IN_BURST_SIZE_MAX]; + char entry; + int key_found; + void *entry_ptr; + + /* Initialize params and create tables */ + struct rte_table_hash_params hash_params = { + .name = "TABLE", + .key_size = key_size, + .key_offset = APP_METADATA_OFFSET(32), + .key_mask = NULL, + .n_keys = 1 << 10, + .n_buckets = 1 << 10, + .f_hash = pipeline_test_hash, + .seed = 0, + }; + + hash_params.n_keys = 0; + + table = ops->f_create(&hash_params, 0, 1); + if (table != NULL) + return -1; + + hash_params.n_keys = 1 << 10; + hash_params.key_offset = APP_METADATA_OFFSET(1); + + table = ops->f_create(&hash_params, 0, 1); + if (table == NULL) + return -3; + + hash_params.key_offset = APP_METADATA_OFFSET(32); + hash_params.f_hash = NULL; + + table = ops->f_create(&hash_params, 0, 1); + if (table != NULL) + return -4; + + hash_params.f_hash = pipeline_test_hash; + + table = ops->f_create(&hash_params, 0, 1); + if (table == NULL) + return -5; + + /* Free */ + status = ops->f_free(table); + if (status < 0) + return -6; + + status = ops->f_free(NULL); + if (status == 0) + return -7; + + /* Add */ + uint8_t key[32]; + uint32_t *k32 = (uint32_t *) &key; + + memset(key, 0, 32); + k32[0] = rte_be_to_cpu_32(0xadadadad); + + table = ops->f_create(&hash_params, 0, 1); + if (table == NULL) + return -8; + + entry = 'A'; + status = ops->f_add(table, &key, &entry, &key_found, &entry_ptr); + if (status != 0) + return -9; + + /* Delete */ + status = ops->f_delete(table, &key, &key_found, NULL); + if (status != 0) + return -10; + + status = ops->f_delete(table, &key, &key_found, NULL); + if (status != 0) + return -11; + + /* Traffic flow */ + entry = 'A'; + status = ops->f_add(table, &key, &entry, &key_found, &entry_ptr); + if (status < 0) + return -12; + + for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) + if (i % 2 == 0) { + expected_mask |= (uint64_t)1 << i; + PREPARE_PACKET(mbufs[i], 0xadadadad); + } else + PREPARE_PACKET(mbufs[i], 0xadadadab); + + ops->f_lookup(table, mbufs, -1, &result_mask, (void **)entries); + if (result_mask != expected_mask) + return -13; + + /* Free resources */ + for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) + rte_pktmbuf_free(mbufs[i]); + + status = ops->f_free(table); + + return 0; +} + +int +test_table_hash_lru(void) +{ + int status; + + status = test_table_hash_lru_generic( + &rte_table_hash_key8_lru_ops, + 8); + if (status < 0) + return status; + + status = test_table_hash_lru_generic( + &rte_table_hash_key16_lru_ops, + 16); + if (status < 0) + return status; + + status = test_table_hash_lru_generic( + &rte_table_hash_key32_lru_ops, + 32); + if (status < 0) + return status; + + status = test_lru_update(); + if (status < 0) + return status; + + return 0; +} + +int +test_table_hash_ext(void) +{ + int status; + + status = test_table_hash_ext_generic(&rte_table_hash_key8_ext_ops, 8); + if (status < 0) + return status; + + status = test_table_hash_ext_generic(&rte_table_hash_key16_ext_ops, 16); + if (status < 0) + return status; + + status = test_table_hash_ext_generic(&rte_table_hash_key32_ext_ops, 32); + if (status < 0) + return status; + + return 0; +} + + +int +test_table_hash_cuckoo(void) +{ + int status, i; + uint64_t expected_mask = 0, result_mask; + struct rte_mbuf *mbufs[RTE_PORT_IN_BURST_SIZE_MAX]; + void *table; + char *entries[RTE_PORT_IN_BURST_SIZE_MAX]; + char entry; + void *entry_ptr; + int key_found; + uint32_t entry_size = 1; + + /* Initialize params and create tables */ + struct rte_table_hash_params cuckoo_params = { + .name = "TABLE", + .key_size = 32, + .key_offset = APP_METADATA_OFFSET(32), + .key_mask = NULL, + .n_keys = 1 << 16, + .n_buckets = 1 << 16, + .f_hash = (rte_table_hash_op_hash)pipeline_test_hash, + .seed = 0, + }; + + table = rte_table_hash_cuckoo_ops.f_create(NULL, 0, entry_size); + if (table != NULL) + return -1; + + cuckoo_params.key_size = 0; + + table = rte_table_hash_cuckoo_ops.f_create(&cuckoo_params, + 0, entry_size); + if (table != NULL) + return -2; + + cuckoo_params.key_size = 32; + cuckoo_params.n_keys = 0; + + table = rte_table_hash_cuckoo_ops.f_create(&cuckoo_params, + 0, entry_size); + if (table != NULL) + return -3; + + cuckoo_params.n_keys = 1 << 24; + cuckoo_params.f_hash = NULL; + + table = rte_table_hash_cuckoo_ops.f_create(&cuckoo_params, + 0, entry_size); + if (table != NULL) + return -4; + + cuckoo_params.f_hash = pipeline_test_hash; + cuckoo_params.name = NULL; + + table = rte_table_hash_cuckoo_ops.f_create(&cuckoo_params, + 0, entry_size); + if (table != NULL) + return -5; + + cuckoo_params.name = "CUCKOO"; + + table = rte_table_hash_cuckoo_ops.f_create(&cuckoo_params, + 0, entry_size); + if (table == NULL) + return -6; + + /* Free */ + status = rte_table_hash_cuckoo_ops.f_free(table); + if (status < 0) + return -7; + + status = rte_table_hash_cuckoo_ops.f_free(NULL); + if (status == 0) + return -8; + + /* Add */ + uint8_t key_cuckoo[32]; + uint32_t *kcuckoo = (uint32_t *) &key_cuckoo; + + memset(key_cuckoo, 0, 32); + kcuckoo[0] = rte_be_to_cpu_32(0xadadadad); + + table = rte_table_hash_cuckoo_ops.f_create(&cuckoo_params, 0, 1); + if (table == NULL) + return -9; + + entry = 'A'; + status = rte_table_hash_cuckoo_ops.f_add(NULL, &key_cuckoo, + &entry, &key_found, &entry_ptr); + if (status == 0) + return -10; + + status = rte_table_hash_cuckoo_ops.f_add(table, NULL, &entry, + &key_found, &entry_ptr); + if (status == 0) + return -11; + + status = rte_table_hash_cuckoo_ops.f_add(table, &key_cuckoo, + NULL, &key_found, &entry_ptr); + if (status == 0) + return -12; + + status = rte_table_hash_cuckoo_ops.f_add(table, &key_cuckoo, + &entry, &key_found, &entry_ptr); + if (status != 0) + return -13; + + status = rte_table_hash_cuckoo_ops.f_add(table, &key_cuckoo, + &entry, &key_found, &entry_ptr); + if (status != 0) + return -14; + + /* Delete */ + status = rte_table_hash_cuckoo_ops.f_delete(NULL, &key_cuckoo, + &key_found, NULL); + if (status == 0) + return -15; + + status = rte_table_hash_cuckoo_ops.f_delete(table, NULL, + &key_found, NULL); + if (status == 0) + return -16; + + status = rte_table_hash_cuckoo_ops.f_delete(table, &key_cuckoo, + &key_found, NULL); + if (status != 0) + return -17; + + status = rte_table_hash_cuckoo_ops.f_delete(table, &key_cuckoo, + &key_found, NULL); + if (status != -ENOENT) + return -18; + + /* Traffic flow */ + entry = 'A'; + status = rte_table_hash_cuckoo_ops.f_add(table, &key_cuckoo, + &entry, &key_found, + &entry_ptr); + if (status < 0) + return -19; + + for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) + if (i % 2 == 0) { + expected_mask |= (uint64_t)1 << i; + PREPARE_PACKET(mbufs[i], 0xadadadad); + } else + PREPARE_PACKET(mbufs[i], 0xadadadab); + + rte_table_hash_cuckoo_ops.f_lookup(table, mbufs, -1, + &result_mask, (void **)entries); + if (result_mask != expected_mask) + return -20; + + /* Free resources */ + for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) + rte_pktmbuf_free(mbufs[i]); + + status = rte_table_hash_cuckoo_ops.f_free(table); + + return 0; +} + diff --git a/dpdk/test/test/test_table_tables.h b/dpdk/test/test/test_table_tables.h new file mode 100644 index 00000000..35311362 --- /dev/null +++ b/dpdk/test/test/test_table_tables.h @@ -0,0 +1,51 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* Test prototypes */ +int test_table_hash_cuckoo(void); +int test_table_lpm(void); +int test_table_lpm_ipv6(void); +int test_table_array(void); +#ifdef RTE_LIBRTE_ACL +int test_table_acl(void); +#endif +int test_table_hash_unoptimized(void); +int test_table_hash_lru(void); +int test_table_hash_ext(void); +int test_table_stub(void); + +/* Extern variables */ +typedef int (*table_test)(void); + +extern table_test table_tests[]; +extern unsigned n_table_tests; diff --git a/dpdk/test/test/test_tailq.c b/dpdk/test/test/test_tailq.c new file mode 100644 index 00000000..33a3e8a9 --- /dev/null +++ b/dpdk/test/test/test_tailq.c @@ -0,0 +1,157 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <stdint.h> +#include <stdarg.h> +#include <string.h> +#include <errno.h> +#include <sys/queue.h> + +#include <rte_eal.h> +#include <rte_eal_memconfig.h> +#include <rte_string_fns.h> + +#include "test.h" + +#define do_return(...) do { \ + printf("Error at %s, line %d: ", __func__, __LINE__); \ + printf(__VA_ARGS__); \ + return 1; \ +} while (0) + +static struct rte_tailq_elem rte_dummy_tailq = { + .name = "dummy", +}; +EAL_REGISTER_TAILQ(rte_dummy_tailq) + +static struct rte_tailq_elem rte_dummy_dyn_tailq = { + .name = "dummy_dyn", +}; +static struct rte_tailq_elem rte_dummy_dyn2_tailq = { + .name = "dummy_dyn", +}; + +static struct rte_tailq_entry d_elem; +static struct rte_tailq_entry d_dyn_elem; + +static int +test_tailq_early(void) +{ + struct rte_tailq_entry_head *d_head; + + d_head = RTE_TAILQ_CAST(rte_dummy_tailq.head, rte_tailq_entry_head); + if (d_head == NULL) + do_return("Error %s has not been initialised\n", + rte_dummy_tailq.name); + + /* check we can add an item to it */ + TAILQ_INSERT_TAIL(d_head, &d_elem, next); + + return 0; +} + +static int +test_tailq_create(void) +{ + struct rte_tailq_entry_head *d_head; + + /* create a tailq and check its non-null (since we are post-eal init) */ + if ((rte_eal_tailq_register(&rte_dummy_dyn_tailq) < 0) || + (rte_dummy_dyn_tailq.head == NULL)) + do_return("Error allocating %s\n", rte_dummy_dyn_tailq.name); + + d_head = RTE_TAILQ_CAST(rte_dummy_dyn_tailq.head, rte_tailq_entry_head); + + /* check we can add an item to it */ + TAILQ_INSERT_TAIL(d_head, &d_dyn_elem, next); + + if (strcmp(rte_dummy_dyn2_tailq.name, rte_dummy_dyn_tailq.name)) + do_return("Error, something is wrong in the tailq test\n"); + + /* try allocating again, and check for failure */ + if (!rte_eal_tailq_register(&rte_dummy_dyn2_tailq)) + do_return("Error, registering the same tailq %s did not fail\n", + rte_dummy_dyn2_tailq.name); + + return 0; +} + +static int +test_tailq_lookup(void) +{ + /* run successful test - check result is found */ + struct rte_tailq_entry_head *d_head; + struct rte_tailq_entry *d_ptr; + + d_head = RTE_TAILQ_LOOKUP(rte_dummy_tailq.name, rte_tailq_entry_head); + /* rte_dummy_tailq has been registered by EAL_REGISTER_TAILQ */ + if (d_head == NULL || + d_head != RTE_TAILQ_CAST(rte_dummy_tailq.head, rte_tailq_entry_head)) + do_return("Error with tailq lookup\n"); + + TAILQ_FOREACH(d_ptr, d_head, next) + if (d_ptr != &d_elem) + do_return("Error with tailq returned from lookup - " + "expected element not found\n"); + + d_head = RTE_TAILQ_LOOKUP(rte_dummy_dyn_tailq.name, rte_tailq_entry_head); + /* rte_dummy_dyn_tailq has been registered by test_tailq_create */ + if (d_head == NULL || + d_head != RTE_TAILQ_CAST(rte_dummy_dyn_tailq.head, rte_tailq_entry_head)) + do_return("Error with tailq lookup\n"); + + TAILQ_FOREACH(d_ptr, d_head, next) + if (d_ptr != &d_dyn_elem) + do_return("Error with tailq returned from lookup - " + "expected element not found\n"); + + /* now try a bad/error lookup */ + d_head = RTE_TAILQ_LOOKUP("coucou", rte_tailq_entry_head); + if (d_head != NULL) + do_return("Error, lookup does not return NULL for bad tailq name\n"); + + return 0; +} + +static int +test_tailq(void) +{ + int ret = 0; + ret |= test_tailq_early(); + ret |= test_tailq_create(); + ret |= test_tailq_lookup(); + return ret; +} + +REGISTER_TEST_COMMAND(tailq_autotest, test_tailq); diff --git a/dpdk/test/test/test_thash.c b/dpdk/test/test/test_thash.c new file mode 100644 index 00000000..61754a94 --- /dev/null +++ b/dpdk/test/test/test_thash.c @@ -0,0 +1,172 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2015 Vladimir Medvedkin <medvedkinv@gmail.com> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <rte_common.h> +#include <rte_eal.h> +#include <rte_ip.h> + +#include "test.h" + +#include <rte_thash.h> + +struct test_thash_v4 { + uint32_t dst_ip; + uint32_t src_ip; + uint16_t dst_port; + uint16_t src_port; + uint32_t hash_l3; + uint32_t hash_l3l4; +}; + +struct test_thash_v6 { + uint8_t dst_ip[16]; + uint8_t src_ip[16]; + uint16_t dst_port; + uint16_t src_port; + uint32_t hash_l3; + uint32_t hash_l3l4; +}; + +/*From 82599 Datasheet 7.1.2.8.3 RSS Verification Suite*/ +struct test_thash_v4 v4_tbl[] = { +{IPv4(161, 142, 100, 80), IPv4(66, 9, 149, 187), + 1766, 2794, 0x323e8fc2, 0x51ccc178}, +{IPv4(65, 69, 140, 83), IPv4(199, 92, 111, 2), + 4739, 14230, 0xd718262a, 0xc626b0ea}, +{IPv4(12, 22, 207, 184), IPv4(24, 19, 198, 95), + 38024, 12898, 0xd2d0a5de, 0x5c2b394a}, +{IPv4(209, 142, 163, 6), IPv4(38, 27, 205, 30), + 2217, 48228, 0x82989176, 0xafc7327f}, +{IPv4(202, 188, 127, 2), IPv4(153, 39, 163, 191), + 1303, 44251, 0x5d1809c5, 0x10e828a2}, +}; + +struct test_thash_v6 v6_tbl[] = { +/*3ffe:2501:200:3::1*/ +{{0x3f, 0xfe, 0x25, 0x01, 0x02, 0x00, 0x00, 0x03, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01,}, +/*3ffe:2501:200:1fff::7*/ +{0x3f, 0xfe, 0x25, 0x01, 0x02, 0x00, 0x1f, 0xff, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07,}, +1766, 2794, 0x2cc18cd5, 0x40207d3d}, +/*ff02::1*/ +{{0xff, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01,}, +/*3ffe:501:8::260:97ff:fe40:efab*/ +{0x3f, 0xfe, 0x05, 0x01, 0x00, 0x08, 0x00, 0x00, +0x02, 0x60, 0x97, 0xff, 0xfe, 0x40, 0xef, 0xab,}, +4739, 14230, 0x0f0c461c, 0xdde51bbf}, +/*fe80::200:f8ff:fe21:67cf*/ +{{0xfe, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x02, 0x00, 0xf8, 0xff, 0xfe, 0x21, 0x67, 0xcf,}, +/*3ffe:1900:4545:3:200:f8ff:fe21:67cf*/ +{0x3f, 0xfe, 0x19, 0x00, 0x45, 0x45, 0x00, 0x03, +0x02, 0x00, 0xf8, 0xff, 0xfe, 0x21, 0x67, 0xcf,}, +38024, 44251, 0x4b61e985, 0x02d1feef}, +}; + +uint8_t default_rss_key[] = { +0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, +0x41, 0x67, 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, +0xd0, 0xca, 0x2b, 0xcb, 0xae, 0x7b, 0x30, 0xb4, +0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30, 0xf2, 0x0c, +0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa, +}; + +static int +test_thash(void) +{ + uint32_t i, j; + union rte_thash_tuple tuple; + uint32_t rss_l3, rss_l3l4; + uint8_t rss_key_be[RTE_DIM(default_rss_key)]; + struct ipv6_hdr ipv6_hdr; + + /* Convert RSS key*/ + rte_convert_rss_key((uint32_t *)&default_rss_key, + (uint32_t *)rss_key_be, RTE_DIM(default_rss_key)); + + + for (i = 0; i < RTE_DIM(v4_tbl); i++) { + tuple.v4.src_addr = v4_tbl[i].src_ip; + tuple.v4.dst_addr = v4_tbl[i].dst_ip; + tuple.v4.sport = v4_tbl[i].src_port; + tuple.v4.dport = v4_tbl[i].dst_port; + /*Calculate hash with original key*/ + rss_l3 = rte_softrss((uint32_t *)&tuple, + RTE_THASH_V4_L3_LEN, default_rss_key); + rss_l3l4 = rte_softrss((uint32_t *)&tuple, + RTE_THASH_V4_L4_LEN, default_rss_key); + if ((rss_l3 != v4_tbl[i].hash_l3) || + (rss_l3l4 != v4_tbl[i].hash_l3l4)) + return -1; + /*Calculate hash with converted key*/ + rss_l3 = rte_softrss_be((uint32_t *)&tuple, + RTE_THASH_V4_L3_LEN, rss_key_be); + rss_l3l4 = rte_softrss_be((uint32_t *)&tuple, + RTE_THASH_V4_L4_LEN, rss_key_be); + if ((rss_l3 != v4_tbl[i].hash_l3) || + (rss_l3l4 != v4_tbl[i].hash_l3l4)) + return -1; + } + for (i = 0; i < RTE_DIM(v6_tbl); i++) { + /*Fill ipv6 hdr*/ + for (j = 0; j < RTE_DIM(ipv6_hdr.src_addr); j++) + ipv6_hdr.src_addr[j] = v6_tbl[i].src_ip[j]; + for (j = 0; j < RTE_DIM(ipv6_hdr.dst_addr); j++) + ipv6_hdr.dst_addr[j] = v6_tbl[i].dst_ip[j]; + /*Load and convert ipv6 address into tuple*/ + rte_thash_load_v6_addrs(&ipv6_hdr, &tuple); + tuple.v6.sport = v6_tbl[i].src_port; + tuple.v6.dport = v6_tbl[i].dst_port; + /*Calculate hash with original key*/ + rss_l3 = rte_softrss((uint32_t *)&tuple, + RTE_THASH_V6_L3_LEN, default_rss_key); + rss_l3l4 = rte_softrss((uint32_t *)&tuple, + RTE_THASH_V6_L4_LEN, default_rss_key); + if ((rss_l3 != v6_tbl[i].hash_l3) || + (rss_l3l4 != v6_tbl[i].hash_l3l4)) + return -1; + /*Calculate hash with converted key*/ + rss_l3 = rte_softrss_be((uint32_t *)&tuple, + RTE_THASH_V6_L3_LEN, rss_key_be); + rss_l3l4 = rte_softrss_be((uint32_t *)&tuple, + RTE_THASH_V6_L4_LEN, rss_key_be); + if ((rss_l3 != v6_tbl[i].hash_l3) || + (rss_l3l4 != v6_tbl[i].hash_l3l4)) + return -1; + } + return 0; +} + +REGISTER_TEST_COMMAND(thash_autotest, test_thash); diff --git a/dpdk/test/test/test_timer.c b/dpdk/test/test/test_timer.c new file mode 100644 index 00000000..04c23989 --- /dev/null +++ b/dpdk/test/test/test_timer.c @@ -0,0 +1,629 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "test.h" + +/* + * Timer + * ===== + * + * #. Stress test 1. + * + * The objective of the timer stress tests is to check that there are no + * race conditions in list and status management. This test launches, + * resets and stops the timer very often on many cores at the same + * time. + * + * - Only one timer is used for this test. + * - On each core, the rte_timer_manage() function is called from the main + * loop every 3 microseconds. + * - In the main loop, the timer may be reset (randomly, with a + * probability of 0.5 %) 100 microseconds later on a random core, or + * stopped (with a probability of 0.5 % also). + * - In callback, the timer is can be reset (randomly, with a + * probability of 0.5 %) 100 microseconds later on the same core or + * on another core (same probability), or stopped (same + * probability). + * + * # Stress test 2. + * + * The objective of this test is similar to the first in that it attempts + * to find if there are any race conditions in the timer library. However, + * it is less complex in terms of operations performed and duration, as it + * is designed to have a predictable outcome that can be tested. + * + * - A set of timers is initialized for use by the test + * - All cores then simultaneously are set to schedule all the timers at + * the same time, so conflicts should occur. + * - Then there is a delay while we wait for the timers to expire + * - Then the master lcore calls timer_manage() and we check that all + * timers have had their callbacks called exactly once - no more no less. + * - Then we repeat the process, except after setting up the timers, we have + * all cores randomly reschedule them. + * - Again we check that the expected number of callbacks has occurred when + * we call timer-manage. + * + * #. Basic test. + * + * This test performs basic functional checks of the timers. The test + * uses four different timers that are loaded and stopped under + * specific conditions in specific contexts. + * + * - Four timers are used for this test. + * - On each core, the rte_timer_manage() function is called from main loop + * every 3 microseconds. + * + * The autotest python script checks that the behavior is correct: + * + * - timer0 + * + * - At initialization, timer0 is loaded by the master core, on master core + * in "single" mode (time = 1 second). + * - In the first 19 callbacks, timer0 is reloaded on the same core, + * then, it is explicitly stopped at the 20th call. + * - At t=25s, timer0 is reloaded once by timer2. + * + * - timer1 + * + * - At initialization, timer1 is loaded by the master core, on the + * master core in "single" mode (time = 2 seconds). + * - In the first 9 callbacks, timer1 is reloaded on another + * core. After the 10th callback, timer1 is not reloaded anymore. + * + * - timer2 + * + * - At initialization, timer2 is loaded by the master core, on the + * master core in "periodical" mode (time = 1 second). + * - In the callback, when t=25s, it stops timer3 and reloads timer0 + * on the current core. + * + * - timer3 + * + * - At initialization, timer3 is loaded by the master core, on + * another core in "periodical" mode (time = 1 second). + * - It is stopped at t=25s by timer2. + */ + +#include <stdio.h> +#include <stdarg.h> +#include <string.h> +#include <stdlib.h> +#include <stdint.h> +#include <inttypes.h> +#include <sys/queue.h> +#include <math.h> + +#include <rte_common.h> +#include <rte_log.h> +#include <rte_memory.h> +#include <rte_launch.h> +#include <rte_cycles.h> +#include <rte_eal.h> +#include <rte_per_lcore.h> +#include <rte_lcore.h> +#include <rte_atomic.h> +#include <rte_timer.h> +#include <rte_random.h> +#include <rte_malloc.h> +#include <rte_pause.h> + +#define TEST_DURATION_S 1 /* in seconds */ +#define NB_TIMER 4 + +#define RTE_LOGTYPE_TESTTIMER RTE_LOGTYPE_USER3 + +static volatile uint64_t end_time; +static volatile int test_failed; + +struct mytimerinfo { + struct rte_timer tim; + unsigned id; + unsigned count; +}; + +static struct mytimerinfo mytiminfo[NB_TIMER]; + +static void timer_basic_cb(struct rte_timer *tim, void *arg); + +static void +mytimer_reset(struct mytimerinfo *timinfo, uint64_t ticks, + enum rte_timer_type type, unsigned tim_lcore, + rte_timer_cb_t fct) +{ + rte_timer_reset_sync(&timinfo->tim, ticks, type, tim_lcore, + fct, timinfo); +} + +/* timer callback for stress tests */ +static void +timer_stress_cb(__attribute__((unused)) struct rte_timer *tim, + __attribute__((unused)) void *arg) +{ + long r; + unsigned lcore_id = rte_lcore_id(); + uint64_t hz = rte_get_timer_hz(); + + if (rte_timer_pending(tim)) + return; + + r = rte_rand(); + if ((r & 0xff) == 0) { + mytimer_reset(&mytiminfo[0], hz, SINGLE, lcore_id, + timer_stress_cb); + } + else if ((r & 0xff) == 1) { + mytimer_reset(&mytiminfo[0], hz, SINGLE, + rte_get_next_lcore(lcore_id, 0, 1), + timer_stress_cb); + } + else if ((r & 0xff) == 2) { + rte_timer_stop(&mytiminfo[0].tim); + } +} + +static int +timer_stress_main_loop(__attribute__((unused)) void *arg) +{ + uint64_t hz = rte_get_timer_hz(); + unsigned lcore_id = rte_lcore_id(); + uint64_t cur_time; + int64_t diff = 0; + long r; + + while (diff >= 0) { + + /* call the timer handler on each core */ + rte_timer_manage(); + + /* simulate the processing of a packet + * (1 us = 2000 cycles at 2 Ghz) */ + rte_delay_us(1); + + /* randomly stop or reset timer */ + r = rte_rand(); + lcore_id = rte_get_next_lcore(lcore_id, 0, 1); + if ((r & 0xff) == 0) { + /* 100 us */ + mytimer_reset(&mytiminfo[0], hz/10000, SINGLE, lcore_id, + timer_stress_cb); + } + else if ((r & 0xff) == 1) { + rte_timer_stop_sync(&mytiminfo[0].tim); + } + cur_time = rte_get_timer_cycles(); + diff = end_time - cur_time; + } + + lcore_id = rte_lcore_id(); + RTE_LOG(INFO, TESTTIMER, "core %u finished\n", lcore_id); + + return 0; +} + +/* Need to synchronize slave lcores through multiple steps. */ +enum { SLAVE_WAITING = 1, SLAVE_RUN_SIGNAL, SLAVE_RUNNING, SLAVE_FINISHED }; +static rte_atomic16_t slave_state[RTE_MAX_LCORE]; + +static void +master_init_slaves(void) +{ + unsigned i; + + RTE_LCORE_FOREACH_SLAVE(i) { + rte_atomic16_set(&slave_state[i], SLAVE_WAITING); + } +} + +static void +master_start_slaves(void) +{ + unsigned i; + + RTE_LCORE_FOREACH_SLAVE(i) { + rte_atomic16_set(&slave_state[i], SLAVE_RUN_SIGNAL); + } + RTE_LCORE_FOREACH_SLAVE(i) { + while (rte_atomic16_read(&slave_state[i]) != SLAVE_RUNNING) + rte_pause(); + } +} + +static void +master_wait_for_slaves(void) +{ + unsigned i; + + RTE_LCORE_FOREACH_SLAVE(i) { + while (rte_atomic16_read(&slave_state[i]) != SLAVE_FINISHED) + rte_pause(); + } +} + +static void +slave_wait_to_start(void) +{ + unsigned lcore_id = rte_lcore_id(); + + while (rte_atomic16_read(&slave_state[lcore_id]) != SLAVE_RUN_SIGNAL) + rte_pause(); + rte_atomic16_set(&slave_state[lcore_id], SLAVE_RUNNING); +} + +static void +slave_finish(void) +{ + unsigned lcore_id = rte_lcore_id(); + + rte_atomic16_set(&slave_state[lcore_id], SLAVE_FINISHED); +} + + +static volatile int cb_count = 0; + +/* callback for second stress test. will only be called + * on master lcore */ +static void +timer_stress2_cb(struct rte_timer *tim __rte_unused, void *arg __rte_unused) +{ + cb_count++; +} + +#define NB_STRESS2_TIMERS 8192 + +static int +timer_stress2_main_loop(__attribute__((unused)) void *arg) +{ + static struct rte_timer *timers; + int i, ret; + uint64_t delay = rte_get_timer_hz() / 20; + unsigned lcore_id = rte_lcore_id(); + unsigned master = rte_get_master_lcore(); + int32_t my_collisions = 0; + static rte_atomic32_t collisions; + + if (lcore_id == master) { + cb_count = 0; + test_failed = 0; + rte_atomic32_set(&collisions, 0); + master_init_slaves(); + timers = rte_malloc(NULL, sizeof(*timers) * NB_STRESS2_TIMERS, 0); + if (timers == NULL) { + printf("Test Failed\n"); + printf("- Cannot allocate memory for timers\n" ); + test_failed = 1; + master_start_slaves(); + goto cleanup; + } + for (i = 0; i < NB_STRESS2_TIMERS; i++) + rte_timer_init(&timers[i]); + master_start_slaves(); + } else { + slave_wait_to_start(); + if (test_failed) + goto cleanup; + } + + /* have all cores schedule all timers on master lcore */ + for (i = 0; i < NB_STRESS2_TIMERS; i++) { + ret = rte_timer_reset(&timers[i], delay, SINGLE, master, + timer_stress2_cb, NULL); + /* there will be collisions when multiple cores simultaneously + * configure the same timers */ + if (ret != 0) + my_collisions++; + } + if (my_collisions != 0) + rte_atomic32_add(&collisions, my_collisions); + + /* wait long enough for timers to expire */ + rte_delay_ms(100); + + /* all cores rendezvous */ + if (lcore_id == master) { + master_wait_for_slaves(); + } else { + slave_finish(); + } + + /* now check that we get the right number of callbacks */ + if (lcore_id == master) { + my_collisions = rte_atomic32_read(&collisions); + if (my_collisions != 0) + printf("- %d timer reset collisions (OK)\n", my_collisions); + rte_timer_manage(); + if (cb_count != NB_STRESS2_TIMERS) { + printf("Test Failed\n"); + printf("- Stress test 2, part 1 failed\n"); + printf("- Expected %d callbacks, got %d\n", NB_STRESS2_TIMERS, + cb_count); + test_failed = 1; + master_start_slaves(); + goto cleanup; + } + cb_count = 0; + + /* proceed */ + master_start_slaves(); + } else { + /* proceed */ + slave_wait_to_start(); + if (test_failed) + goto cleanup; + } + + /* now test again, just stop and restart timers at random after init*/ + for (i = 0; i < NB_STRESS2_TIMERS; i++) + rte_timer_reset(&timers[i], delay, SINGLE, master, + timer_stress2_cb, NULL); + + /* pick random timer to reset, stopping them first half the time */ + for (i = 0; i < 100000; i++) { + int r = rand() % NB_STRESS2_TIMERS; + if (i % 2) + rte_timer_stop(&timers[r]); + rte_timer_reset(&timers[r], delay, SINGLE, master, + timer_stress2_cb, NULL); + } + + /* wait long enough for timers to expire */ + rte_delay_ms(100); + + /* now check that we get the right number of callbacks */ + if (lcore_id == master) { + master_wait_for_slaves(); + + rte_timer_manage(); + if (cb_count != NB_STRESS2_TIMERS) { + printf("Test Failed\n"); + printf("- Stress test 2, part 2 failed\n"); + printf("- Expected %d callbacks, got %d\n", NB_STRESS2_TIMERS, + cb_count); + test_failed = 1; + } else { + printf("Test OK\n"); + } + } + +cleanup: + if (lcore_id == master) { + master_wait_for_slaves(); + if (timers != NULL) { + rte_free(timers); + timers = NULL; + } + } else { + slave_finish(); + } + + return 0; +} + +/* timer callback for basic tests */ +static void +timer_basic_cb(struct rte_timer *tim, void *arg) +{ + struct mytimerinfo *timinfo = arg; + uint64_t hz = rte_get_timer_hz(); + unsigned lcore_id = rte_lcore_id(); + uint64_t cur_time = rte_get_timer_cycles(); + + if (rte_timer_pending(tim)) + return; + + timinfo->count ++; + + RTE_LOG(INFO, TESTTIMER, + "%"PRIu64": callback id=%u count=%u on core %u\n", + cur_time, timinfo->id, timinfo->count, lcore_id); + + /* reload timer 0 on same core */ + if (timinfo->id == 0 && timinfo->count < 20) { + mytimer_reset(timinfo, hz, SINGLE, lcore_id, timer_basic_cb); + return; + } + + /* reload timer 1 on next core */ + if (timinfo->id == 1 && timinfo->count < 10) { + mytimer_reset(timinfo, hz*2, SINGLE, + rte_get_next_lcore(lcore_id, 0, 1), + timer_basic_cb); + return; + } + + /* Explicitelly stop timer 0. Once stop() called, we can even + * erase the content of the structure: it is not referenced + * anymore by any code (in case of dynamic structure, it can + * be freed) */ + if (timinfo->id == 0 && timinfo->count == 20) { + + /* stop_sync() is not needed, because we know that the + * status of timer is only modified by this core */ + rte_timer_stop(tim); + memset(tim, 0xAA, sizeof(struct rte_timer)); + return; + } + + /* stop timer3, and restart a new timer0 (it was removed 5 + * seconds ago) for a single shot */ + if (timinfo->id == 2 && timinfo->count == 25) { + rte_timer_stop_sync(&mytiminfo[3].tim); + + /* need to reinit because structure was erased with 0xAA */ + rte_timer_init(&mytiminfo[0].tim); + mytimer_reset(&mytiminfo[0], hz, SINGLE, lcore_id, + timer_basic_cb); + } +} + +static int +timer_basic_main_loop(__attribute__((unused)) void *arg) +{ + uint64_t hz = rte_get_timer_hz(); + unsigned lcore_id = rte_lcore_id(); + uint64_t cur_time; + int64_t diff = 0; + + /* launch all timers on core 0 */ + if (lcore_id == rte_get_master_lcore()) { + mytimer_reset(&mytiminfo[0], hz/4, SINGLE, lcore_id, + timer_basic_cb); + mytimer_reset(&mytiminfo[1], hz/2, SINGLE, lcore_id, + timer_basic_cb); + mytimer_reset(&mytiminfo[2], hz/4, PERIODICAL, lcore_id, + timer_basic_cb); + mytimer_reset(&mytiminfo[3], hz/4, PERIODICAL, + rte_get_next_lcore(lcore_id, 0, 1), + timer_basic_cb); + } + + while (diff >= 0) { + + /* call the timer handler on each core */ + rte_timer_manage(); + + /* simulate the processing of a packet + * (3 us = 6000 cycles at 2 Ghz) */ + rte_delay_us(3); + + cur_time = rte_get_timer_cycles(); + diff = end_time - cur_time; + } + RTE_LOG(INFO, TESTTIMER, "core %u finished\n", lcore_id); + + return 0; +} + +static int +timer_sanity_check(void) +{ +#ifdef RTE_LIBEAL_USE_HPET + if (eal_timer_source != EAL_TIMER_HPET) { + printf("Not using HPET, can't sanity check timer sources\n"); + return 0; + } + + const uint64_t t_hz = rte_get_tsc_hz(); + const uint64_t h_hz = rte_get_hpet_hz(); + printf("Hertz values: TSC = %"PRIu64", HPET = %"PRIu64"\n", t_hz, h_hz); + + const uint64_t tsc_start = rte_get_tsc_cycles(); + const uint64_t hpet_start = rte_get_hpet_cycles(); + rte_delay_ms(100); /* delay 1/10 second */ + const uint64_t tsc_end = rte_get_tsc_cycles(); + const uint64_t hpet_end = rte_get_hpet_cycles(); + printf("Measured cycles: TSC = %"PRIu64", HPET = %"PRIu64"\n", + tsc_end-tsc_start, hpet_end-hpet_start); + + const double tsc_time = (double)(tsc_end - tsc_start)/t_hz; + const double hpet_time = (double)(hpet_end - hpet_start)/h_hz; + /* get the percentage that the times differ by */ + const double time_diff = fabs(tsc_time - hpet_time)*100/tsc_time; + printf("Measured time: TSC = %.4f, HPET = %.4f\n", tsc_time, hpet_time); + + printf("Elapsed time measured by TSC and HPET differ by %f%%\n", + time_diff); + if (time_diff > 0.1) { + printf("Error times differ by >0.1%%"); + return -1; + } +#endif + return 0; +} + +static int +test_timer(void) +{ + unsigned i; + uint64_t cur_time; + uint64_t hz; + + /* sanity check our timer sources and timer config values */ + if (timer_sanity_check() < 0) { + printf("Timer sanity checks failed\n"); + return TEST_FAILED; + } + + if (rte_lcore_count() < 2) { + printf("not enough lcores for this test\n"); + return TEST_FAILED; + } + + /* init timer */ + for (i=0; i<NB_TIMER; i++) { + memset(&mytiminfo[i], 0, sizeof(struct mytimerinfo)); + mytiminfo[i].id = i; + rte_timer_init(&mytiminfo[i].tim); + } + + /* calculate the "end of test" time */ + cur_time = rte_get_timer_cycles(); + hz = rte_get_timer_hz(); + end_time = cur_time + (hz * TEST_DURATION_S); + + /* start other cores */ + printf("Start timer stress tests\n"); + rte_eal_mp_remote_launch(timer_stress_main_loop, NULL, CALL_MASTER); + rte_eal_mp_wait_lcore(); + + /* stop timer 0 used for stress test */ + rte_timer_stop_sync(&mytiminfo[0].tim); + + /* run a second, slightly different set of stress tests */ + printf("\nStart timer stress tests 2\n"); + test_failed = 0; + rte_eal_mp_remote_launch(timer_stress2_main_loop, NULL, CALL_MASTER); + rte_eal_mp_wait_lcore(); + if (test_failed) + return TEST_FAILED; + + /* calculate the "end of test" time */ + cur_time = rte_get_timer_cycles(); + hz = rte_get_timer_hz(); + end_time = cur_time + (hz * TEST_DURATION_S); + + /* start other cores */ + printf("\nStart timer basic tests\n"); + rte_eal_mp_remote_launch(timer_basic_main_loop, NULL, CALL_MASTER); + rte_eal_mp_wait_lcore(); + + /* stop all timers */ + for (i=0; i<NB_TIMER; i++) { + rte_timer_stop_sync(&mytiminfo[i].tim); + } + + rte_timer_dump_stats(stdout); + + return TEST_SUCCESS; +} + +REGISTER_TEST_COMMAND(timer_autotest, test_timer); diff --git a/dpdk/test/test/test_timer_perf.c b/dpdk/test/test/test_timer_perf.c new file mode 100644 index 00000000..9804133f --- /dev/null +++ b/dpdk/test/test/test_timer_perf.c @@ -0,0 +1,163 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "test.h" + +#include <stdio.h> +#include <unistd.h> +#include <inttypes.h> +#include <rte_cycles.h> +#include <rte_timer.h> +#include <rte_common.h> +#include <rte_lcore.h> +#include <rte_random.h> +#include <rte_malloc.h> +#include <rte_pause.h> + +#define MAX_ITERATIONS 1000000 + +int outstanding_count = 0; + +static void +timer_cb(struct rte_timer *t __rte_unused, void *param __rte_unused) +{ + outstanding_count--; +} + +#define DELAY_SECONDS 1 + +#ifdef RTE_EXEC_ENV_LINUXAPP +#define do_delay() usleep(10) +#else +#define do_delay() rte_pause() +#endif + +static int +test_timer_perf(void) +{ + unsigned iterations = 100; + unsigned i; + struct rte_timer *tms; + uint64_t start_tsc, end_tsc, delay_start; + unsigned lcore_id = rte_lcore_id(); + + tms = rte_malloc(NULL, sizeof(*tms) * MAX_ITERATIONS, 0); + + for (i = 0; i < MAX_ITERATIONS; i++) + rte_timer_init(&tms[i]); + + const uint64_t ticks = rte_get_timer_hz() * DELAY_SECONDS; + const uint64_t ticks_per_ms = rte_get_tsc_hz()/1000; + const uint64_t ticks_per_us = ticks_per_ms/1000; + + while (iterations <= MAX_ITERATIONS) { + + printf("Appending %u timers\n", iterations); + start_tsc = rte_rdtsc(); + for (i = 0; i < iterations; i++) + rte_timer_reset(&tms[i], ticks, SINGLE, lcore_id, + timer_cb, NULL); + end_tsc = rte_rdtsc(); + printf("Time for %u timers: %"PRIu64" (%"PRIu64"ms), ", iterations, + end_tsc-start_tsc, (end_tsc-start_tsc+ticks_per_ms/2)/(ticks_per_ms)); + printf("Time per timer: %"PRIu64" (%"PRIu64"us)\n", + (end_tsc-start_tsc)/iterations, + ((end_tsc-start_tsc)/iterations+ticks_per_us/2)/(ticks_per_us)); + outstanding_count = iterations; + delay_start = rte_get_timer_cycles(); + while (rte_get_timer_cycles() < delay_start + ticks) + do_delay(); + + start_tsc = rte_rdtsc(); + while (outstanding_count) + rte_timer_manage(); + end_tsc = rte_rdtsc(); + printf("Time for %u callbacks: %"PRIu64" (%"PRIu64"ms), ", iterations, + end_tsc-start_tsc, (end_tsc-start_tsc+ticks_per_ms/2)/(ticks_per_ms)); + printf("Time per callback: %"PRIu64" (%"PRIu64"us)\n", + (end_tsc-start_tsc)/iterations, + ((end_tsc-start_tsc)/iterations+ticks_per_us/2)/(ticks_per_us)); + + printf("Resetting %u timers\n", iterations); + start_tsc = rte_rdtsc(); + for (i = 0; i < iterations; i++) + rte_timer_reset(&tms[i], rte_rand() % ticks, SINGLE, lcore_id, + timer_cb, NULL); + end_tsc = rte_rdtsc(); + printf("Time for %u timers: %"PRIu64" (%"PRIu64"ms), ", iterations, + end_tsc-start_tsc, (end_tsc-start_tsc+ticks_per_ms/2)/(ticks_per_ms)); + printf("Time per timer: %"PRIu64" (%"PRIu64"us)\n", + (end_tsc-start_tsc)/iterations, + ((end_tsc-start_tsc)/iterations+ticks_per_us/2)/(ticks_per_us)); + outstanding_count = iterations; + + delay_start = rte_get_timer_cycles(); + while (rte_get_timer_cycles() < delay_start + ticks) + do_delay(); + + rte_timer_manage(); + if (outstanding_count != 0) { + printf("Error: outstanding callback count = %d\n", outstanding_count); + return -1; + } + + iterations *= 10; + printf("\n"); + } + + printf("All timers processed ok\n"); + + /* measure time to poll an empty timer list */ + start_tsc = rte_rdtsc(); + for (i = 0; i < iterations; i++) + rte_timer_manage(); + end_tsc = rte_rdtsc(); + printf("\nTime per rte_timer_manage with zero timers: %"PRIu64" cycles\n", + (end_tsc - start_tsc + iterations/2) / iterations); + + /* measure time to poll a timer list with timers, but without + * calling any callbacks */ + rte_timer_reset(&tms[0], ticks * 100, SINGLE, lcore_id, + timer_cb, NULL); + start_tsc = rte_rdtsc(); + for (i = 0; i < iterations; i++) + rte_timer_manage(); + end_tsc = rte_rdtsc(); + printf("Time per rte_timer_manage with zero callbacks: %"PRIu64" cycles\n", + (end_tsc - start_tsc + iterations/2) / iterations); + + rte_free(tms); + return 0; +} + +REGISTER_TEST_COMMAND(timer_perf_autotest, test_timer_perf); diff --git a/dpdk/test/test/test_timer_racecond.c b/dpdk/test/test/test_timer_racecond.c new file mode 100644 index 00000000..5e08f06b --- /dev/null +++ b/dpdk/test/test/test_timer_racecond.c @@ -0,0 +1,206 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2015 Akamai Technologies. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "test.h" + +#include <stdio.h> +#include <unistd.h> +#include <inttypes.h> +#include <rte_cycles.h> +#include <rte_timer.h> +#include <rte_common.h> +#include <rte_lcore.h> +#include <rte_random.h> +#include <rte_malloc.h> +#include <rte_pause.h> + +#undef TEST_TIMER_RACECOND_VERBOSE + +#ifdef RTE_EXEC_ENV_LINUXAPP +#define usec_delay(us) usleep(us) +#else +#define usec_delay(us) rte_delay_us(us) +#endif + +#define BILLION (1UL << 30) + +#define TEST_DURATION_S 20 /* in seconds */ +#define N_TIMERS 50 + +static struct rte_timer timer[N_TIMERS]; +static unsigned timer_lcore_id[N_TIMERS]; + +static unsigned master; +static volatile unsigned stop_slaves; + +static int reload_timer(struct rte_timer *tim); + +static void +timer_cb(struct rte_timer *tim, void *arg __rte_unused) +{ + /* Simulate slow callback function, 100 us. */ + rte_delay_us(100); + +#ifdef TEST_TIMER_RACECOND_VERBOSE + if (tim == &timer[0]) + printf("------------------------------------------------\n"); + printf("timer_cb: core %u timer %lu\n", + rte_lcore_id(), tim - timer); +#endif + (void)reload_timer(tim); +} + +RTE_DEFINE_PER_LCORE(unsigned, n_reset_collisions); + +static int +reload_timer(struct rte_timer *tim) +{ + /* Make timer expire roughly when the TSC hits the next BILLION + * multiple. Add in timer's index to make them expire in nearly + * sorted order. This makes all timers somewhat synchronized, + * firing ~2-3 times per second, assuming 2-3 GHz TSCs. + */ + uint64_t ticks = BILLION - (rte_get_timer_cycles() % BILLION) + + (tim - timer); + int ret; + + ret = rte_timer_reset(tim, ticks, PERIODICAL, master, timer_cb, NULL); + if (ret != 0) { +#ifdef TEST_TIMER_RACECOND_VERBOSE + printf("- core %u failed to reset timer %lu (OK)\n", + rte_lcore_id(), tim - timer); +#endif + RTE_PER_LCORE(n_reset_collisions) += 1; + } + return ret; +} + +static int +slave_main_loop(__attribute__((unused)) void *arg) +{ + unsigned lcore_id = rte_lcore_id(); + unsigned i; + + RTE_PER_LCORE(n_reset_collisions) = 0; + + printf("Starting main loop on core %u\n", lcore_id); + + while (!stop_slaves) { + /* Wait until the timer manager is running. + * We know it's running when we see timer[0] NOT pending. + */ + if (rte_timer_pending(&timer[0])) { + rte_pause(); + continue; + } + + /* Now, go cause some havoc! + * Reload our timers. + */ + for (i = 0; i < N_TIMERS; i++) { + if (timer_lcore_id[i] == lcore_id) + (void)reload_timer(&timer[i]); + } + usec_delay(100*1000); /* sleep 100 ms */ + } + + if (RTE_PER_LCORE(n_reset_collisions) != 0) { + printf("- core %u, %u reset collisions (OK)\n", + lcore_id, RTE_PER_LCORE(n_reset_collisions)); + } + return 0; +} + +static int +test_timer_racecond(void) +{ + int ret; + uint64_t hz; + uint64_t cur_time; + uint64_t end_time; + int64_t diff = 0; + unsigned lcore_id; + unsigned i; + + master = lcore_id = rte_lcore_id(); + hz = rte_get_timer_hz(); + + /* init and start timers */ + for (i = 0; i < N_TIMERS; i++) { + rte_timer_init(&timer[i]); + ret = reload_timer(&timer[i]); + TEST_ASSERT(ret == 0, "reload_timer failed"); + + /* Distribute timers to slaves. + * Note that we assign timer[0] to the master. + */ + timer_lcore_id[i] = lcore_id; + lcore_id = rte_get_next_lcore(lcore_id, 1, 1); + } + + /* calculate the "end of test" time */ + cur_time = rte_get_timer_cycles(); + end_time = cur_time + (hz * TEST_DURATION_S); + + /* start slave cores */ + stop_slaves = 0; + printf("Start timer manage race condition test (%u seconds)\n", + TEST_DURATION_S); + rte_eal_mp_remote_launch(slave_main_loop, NULL, SKIP_MASTER); + + while (diff >= 0) { + /* run the timers */ + rte_timer_manage(); + + /* wait 100 ms */ + usec_delay(100*1000); + + cur_time = rte_get_timer_cycles(); + diff = end_time - cur_time; + } + + /* stop slave cores */ + printf("Stopping timer manage race condition test\n"); + stop_slaves = 1; + rte_eal_mp_wait_lcore(); + + /* stop timers */ + for (i = 0; i < N_TIMERS; i++) { + ret = rte_timer_stop(&timer[i]); + TEST_ASSERT(ret == 0, "rte_timer_stop failed"); + } + + return TEST_SUCCESS; +} + +REGISTER_TEST_COMMAND(timer_racecond_autotest, test_timer_racecond); diff --git a/dpdk/test/test/test_version.c b/dpdk/test/test/test_version.c new file mode 100644 index 00000000..afc0d0b8 --- /dev/null +++ b/dpdk/test/test/test_version.c @@ -0,0 +1,57 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <string.h> +#include <stdint.h> + +#include <rte_common.h> +#include <rte_version.h> + +#include "test.h" + + +static int +test_version(void) +{ + const char *version = rte_version(); + if (version == NULL) + return -1; + printf("Version string: '%s'\n", version); + if (*version == '\0' || + strncmp(version, RTE_VER_PREFIX, sizeof(RTE_VER_PREFIX)-1) != 0) + return -1; + return 0; +} + +REGISTER_TEST_COMMAND(version_autotest, test_version); diff --git a/dpdk/test/test/test_xmmt_ops.h b/dpdk/test/test/test_xmmt_ops.h new file mode 100644 index 00000000..2d6d800a --- /dev/null +++ b/dpdk/test/test/test_xmmt_ops.h @@ -0,0 +1,83 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2015 Cavium, Inc. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _TEST_XMMT_OPS_H_ +#define _TEST_XMMT_OPS_H_ + +#include <rte_vect.h> + +#if defined(RTE_ARCH_ARM) || defined(RTE_ARCH_ARM64) + +/* vect_* abstraction implementation using NEON */ + +/* loads the xmm_t value from address p(does not need to be 16-byte aligned)*/ +#define vect_loadu_sil128(p) vld1q_s32((const int32_t *)p) + +/* sets the 4 signed 32-bit integer values and returns the xmm_t variable */ +static __rte_always_inline xmm_t +vect_set_epi32(int i3, int i2, int i1, int i0) +{ + int32_t data[4] = {i0, i1, i2, i3}; + + return vld1q_s32(data); +} + +#elif defined(RTE_ARCH_X86) + +/* vect_* abstraction implementation using SSE */ + +/* loads the xmm_t value from address p(does not need to be 16-byte aligned)*/ +#define vect_loadu_sil128(p) _mm_loadu_si128(p) + +/* sets the 4 signed 32-bit integer values and returns the xmm_t variable */ +#define vect_set_epi32(i3, i2, i1, i0) _mm_set_epi32(i3, i2, i1, i0) + +#elif defined(RTE_ARCH_PPC_64) + +/* vect_* abstraction implementation using ALTIVEC */ + +/* loads the xmm_t value from address p(does not need to be 16-byte aligned)*/ +#define vect_loadu_sil128(p) vec_ld(0, p) + +/* sets the 4 signed 32-bit integer values and returns the xmm_t variable */ +static __rte_always_inline xmm_t +vect_set_epi32(int i3, int i2, int i1, int i0) +{ + xmm_t data = (xmm_t){i0, i1, i2, i3}; + + return data; +} + +#endif + +#endif /* _TEST_XMMT_OPS_H_ */ diff --git a/dpdk/test/test/virtual_pmd.c b/dpdk/test/test/virtual_pmd.c new file mode 100644 index 00000000..b57a9493 --- /dev/null +++ b/dpdk/test/test/virtual_pmd.c @@ -0,0 +1,624 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <rte_mbuf.h> +#include <rte_ethdev.h> +#include <rte_pci.h> +#include <rte_bus_pci.h> +#include <rte_malloc.h> +#include <rte_memcpy.h> +#include <rte_memory.h> +#include <rte_ring.h> + +#include "virtual_pmd.h" + +#define MAX_PKT_BURST 512 + +static const char *virtual_ethdev_driver_name = "Virtual PMD"; + +struct virtual_ethdev_private { + struct eth_dev_ops dev_ops; + struct rte_eth_stats eth_stats; + + struct rte_ring *rx_queue; + struct rte_ring *tx_queue; + + int tx_burst_fail_count; +}; + +struct virtual_ethdev_queue { + int port_id; + int queue_id; +}; + +static int +virtual_ethdev_start_success(struct rte_eth_dev *eth_dev __rte_unused) +{ + eth_dev->data->dev_started = 1; + + return 0; +} + +static int +virtual_ethdev_start_fail(struct rte_eth_dev *eth_dev __rte_unused) +{ + eth_dev->data->dev_started = 0; + + return -1; +} +static void virtual_ethdev_stop(struct rte_eth_dev *eth_dev __rte_unused) +{ + void *pkt = NULL; + struct virtual_ethdev_private *prv = eth_dev->data->dev_private; + + eth_dev->data->dev_link.link_status = ETH_LINK_DOWN; + eth_dev->data->dev_started = 0; + while (rte_ring_dequeue(prv->rx_queue, &pkt) != -ENOENT) + rte_pktmbuf_free(pkt); + + while (rte_ring_dequeue(prv->tx_queue, &pkt) != -ENOENT) + rte_pktmbuf_free(pkt); +} + +static void +virtual_ethdev_close(struct rte_eth_dev *dev __rte_unused) +{} + +static int +virtual_ethdev_configure_success(struct rte_eth_dev *dev __rte_unused) +{ + return 0; +} + +static int +virtual_ethdev_configure_fail(struct rte_eth_dev *dev __rte_unused) +{ + return -1; +} + +static void +virtual_ethdev_info_get(struct rte_eth_dev *dev __rte_unused, + struct rte_eth_dev_info *dev_info) +{ + dev_info->driver_name = virtual_ethdev_driver_name; + dev_info->max_mac_addrs = 1; + + dev_info->max_rx_pktlen = (uint32_t)2048; + + dev_info->max_rx_queues = (uint16_t)128; + dev_info->max_tx_queues = (uint16_t)512; + + dev_info->min_rx_bufsize = 0; +} + +static int +virtual_ethdev_rx_queue_setup_success(struct rte_eth_dev *dev, + uint16_t rx_queue_id, uint16_t nb_rx_desc __rte_unused, + unsigned int socket_id, + const struct rte_eth_rxconf *rx_conf __rte_unused, + struct rte_mempool *mb_pool __rte_unused) +{ + struct virtual_ethdev_queue *rx_q; + + rx_q = (struct virtual_ethdev_queue *)rte_zmalloc_socket(NULL, + sizeof(struct virtual_ethdev_queue), 0, socket_id); + + if (rx_q == NULL) + return -1; + + rx_q->port_id = dev->data->port_id; + rx_q->queue_id = rx_queue_id; + + dev->data->rx_queues[rx_queue_id] = rx_q; + + return 0; +} + +static int +virtual_ethdev_rx_queue_setup_fail(struct rte_eth_dev *dev __rte_unused, + uint16_t rx_queue_id __rte_unused, uint16_t nb_rx_desc __rte_unused, + unsigned int socket_id __rte_unused, + const struct rte_eth_rxconf *rx_conf __rte_unused, + struct rte_mempool *mb_pool __rte_unused) +{ + return -1; +} + +static int +virtual_ethdev_tx_queue_setup_success(struct rte_eth_dev *dev, + uint16_t tx_queue_id, uint16_t nb_tx_desc __rte_unused, + unsigned int socket_id, + const struct rte_eth_txconf *tx_conf __rte_unused) +{ + struct virtual_ethdev_queue *tx_q; + + tx_q = (struct virtual_ethdev_queue *)rte_zmalloc_socket(NULL, + sizeof(struct virtual_ethdev_queue), 0, socket_id); + + if (tx_q == NULL) + return -1; + + tx_q->port_id = dev->data->port_id; + tx_q->queue_id = tx_queue_id; + + dev->data->tx_queues[tx_queue_id] = tx_q; + + return 0; +} + +static int +virtual_ethdev_tx_queue_setup_fail(struct rte_eth_dev *dev __rte_unused, + uint16_t tx_queue_id __rte_unused, uint16_t nb_tx_desc __rte_unused, + unsigned int socket_id __rte_unused, + const struct rte_eth_txconf *tx_conf __rte_unused) +{ + return -1; +} + +static void +virtual_ethdev_rx_queue_release(void *q __rte_unused) +{ +} + +static void +virtual_ethdev_tx_queue_release(void *q __rte_unused) +{ +} + +static int +virtual_ethdev_link_update_success(struct rte_eth_dev *bonded_eth_dev, + int wait_to_complete __rte_unused) +{ + if (!bonded_eth_dev->data->dev_started) + bonded_eth_dev->data->dev_link.link_status = ETH_LINK_DOWN; + + return 0; +} + +static int +virtual_ethdev_link_update_fail(struct rte_eth_dev *bonded_eth_dev __rte_unused, + int wait_to_complete __rte_unused) +{ + return -1; +} + +static int +virtual_ethdev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) +{ + struct virtual_ethdev_private *dev_private = dev->data->dev_private; + + if (stats) + rte_memcpy(stats, &dev_private->eth_stats, sizeof(*stats)); + + return 0; +} + +static void +virtual_ethdev_stats_reset(struct rte_eth_dev *dev) +{ + struct virtual_ethdev_private *dev_private = dev->data->dev_private; + void *pkt = NULL; + + while (rte_ring_dequeue(dev_private->tx_queue, &pkt) == -ENOBUFS) + rte_pktmbuf_free(pkt); + + /* Reset internal statistics */ + memset(&dev_private->eth_stats, 0, sizeof(dev_private->eth_stats)); +} + +static void +virtual_ethdev_promiscuous_mode_enable(struct rte_eth_dev *dev __rte_unused) +{} + +static void +virtual_ethdev_promiscuous_mode_disable(struct rte_eth_dev *dev __rte_unused) +{} + + +static const struct eth_dev_ops virtual_ethdev_default_dev_ops = { + .dev_configure = virtual_ethdev_configure_success, + .dev_start = virtual_ethdev_start_success, + .dev_stop = virtual_ethdev_stop, + .dev_close = virtual_ethdev_close, + .dev_infos_get = virtual_ethdev_info_get, + .rx_queue_setup = virtual_ethdev_rx_queue_setup_success, + .tx_queue_setup = virtual_ethdev_tx_queue_setup_success, + .rx_queue_release = virtual_ethdev_rx_queue_release, + .tx_queue_release = virtual_ethdev_tx_queue_release, + .link_update = virtual_ethdev_link_update_success, + .stats_get = virtual_ethdev_stats_get, + .stats_reset = virtual_ethdev_stats_reset, + .promiscuous_enable = virtual_ethdev_promiscuous_mode_enable, + .promiscuous_disable = virtual_ethdev_promiscuous_mode_disable +}; + + +void +virtual_ethdev_start_fn_set_success(uint16_t port_id, uint8_t success) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + struct virtual_ethdev_private *dev_private = dev->data->dev_private; + struct eth_dev_ops *dev_ops = &dev_private->dev_ops; + + if (success) + dev_ops->dev_start = virtual_ethdev_start_success; + else + dev_ops->dev_start = virtual_ethdev_start_fail; + +} + +void +virtual_ethdev_configure_fn_set_success(uint16_t port_id, uint8_t success) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + struct virtual_ethdev_private *dev_private = dev->data->dev_private; + struct eth_dev_ops *dev_ops = &dev_private->dev_ops; + + if (success) + dev_ops->dev_configure = virtual_ethdev_configure_success; + else + dev_ops->dev_configure = virtual_ethdev_configure_fail; +} + +void +virtual_ethdev_rx_queue_setup_fn_set_success(uint16_t port_id, uint8_t success) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + struct virtual_ethdev_private *dev_private = dev->data->dev_private; + struct eth_dev_ops *dev_ops = &dev_private->dev_ops; + + if (success) + dev_ops->rx_queue_setup = virtual_ethdev_rx_queue_setup_success; + else + dev_ops->rx_queue_setup = virtual_ethdev_rx_queue_setup_fail; +} + +void +virtual_ethdev_tx_queue_setup_fn_set_success(uint16_t port_id, uint8_t success) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + struct virtual_ethdev_private *dev_private = dev->data->dev_private; + struct eth_dev_ops *dev_ops = &dev_private->dev_ops; + + if (success) + dev_ops->tx_queue_setup = virtual_ethdev_tx_queue_setup_success; + else + dev_ops->tx_queue_setup = virtual_ethdev_tx_queue_setup_fail; +} + +void +virtual_ethdev_link_update_fn_set_success(uint16_t port_id, uint8_t success) +{ + struct rte_eth_dev *dev = &rte_eth_devices[port_id]; + struct virtual_ethdev_private *dev_private = dev->data->dev_private; + struct eth_dev_ops *dev_ops = &dev_private->dev_ops; + + if (success) + dev_ops->link_update = virtual_ethdev_link_update_success; + else + dev_ops->link_update = virtual_ethdev_link_update_fail; +} + + +static uint16_t +virtual_ethdev_rx_burst_success(void *queue __rte_unused, + struct rte_mbuf **bufs, + uint16_t nb_pkts) +{ + struct rte_eth_dev *vrtl_eth_dev; + struct virtual_ethdev_queue *pq_map; + struct virtual_ethdev_private *dev_private; + + int rx_count, i; + + pq_map = (struct virtual_ethdev_queue *)queue; + vrtl_eth_dev = &rte_eth_devices[pq_map->port_id]; + dev_private = vrtl_eth_dev->data->dev_private; + + rx_count = rte_ring_dequeue_burst(dev_private->rx_queue, (void **) bufs, + nb_pkts, NULL); + + /* increments ipackets count */ + dev_private->eth_stats.ipackets += rx_count; + + /* increments ibytes count */ + for (i = 0; i < rx_count; i++) + dev_private->eth_stats.ibytes += rte_pktmbuf_pkt_len(bufs[i]); + + return rx_count; +} + +static uint16_t +virtual_ethdev_rx_burst_fail(void *queue __rte_unused, + struct rte_mbuf **bufs __rte_unused, + uint16_t nb_pkts __rte_unused) +{ + return 0; +} + +static uint16_t +virtual_ethdev_tx_burst_success(void *queue, struct rte_mbuf **bufs, + uint16_t nb_pkts) +{ + struct virtual_ethdev_queue *tx_q = queue; + + struct rte_eth_dev *vrtl_eth_dev; + struct virtual_ethdev_private *dev_private; + + int i; + + vrtl_eth_dev = &rte_eth_devices[tx_q->port_id]; + dev_private = vrtl_eth_dev->data->dev_private; + + if (!vrtl_eth_dev->data->dev_link.link_status) + nb_pkts = 0; + else + nb_pkts = rte_ring_enqueue_burst(dev_private->tx_queue, (void **)bufs, + nb_pkts, NULL); + + /* increment opacket count */ + dev_private->eth_stats.opackets += nb_pkts; + + /* increment obytes count */ + for (i = 0; i < nb_pkts; i++) + dev_private->eth_stats.obytes += rte_pktmbuf_pkt_len(bufs[i]); + + return nb_pkts; +} + +static uint16_t +virtual_ethdev_tx_burst_fail(void *queue, struct rte_mbuf **bufs, + uint16_t nb_pkts) +{ + struct rte_eth_dev *vrtl_eth_dev = NULL; + struct virtual_ethdev_queue *tx_q = NULL; + struct virtual_ethdev_private *dev_private = NULL; + + int i; + + tx_q = queue; + vrtl_eth_dev = &rte_eth_devices[tx_q->port_id]; + dev_private = vrtl_eth_dev->data->dev_private; + + if (dev_private->tx_burst_fail_count < nb_pkts) { + int successfully_txd = nb_pkts - dev_private->tx_burst_fail_count; + + /* increment opacket count */ + dev_private->eth_stats.opackets += successfully_txd; + + /* free packets in burst */ + for (i = 0; i < successfully_txd; i++) { + /* free packets in burst */ + if (bufs[i] != NULL) + rte_pktmbuf_free(bufs[i]); + + bufs[i] = NULL; + } + + return successfully_txd; + } + + return 0; +} + + +void +virtual_ethdev_rx_burst_fn_set_success(uint16_t port_id, uint8_t success) +{ + struct rte_eth_dev *vrtl_eth_dev = &rte_eth_devices[port_id]; + + if (success) + vrtl_eth_dev->rx_pkt_burst = virtual_ethdev_rx_burst_success; + else + vrtl_eth_dev->rx_pkt_burst = virtual_ethdev_rx_burst_fail; +} + + +void +virtual_ethdev_tx_burst_fn_set_success(uint16_t port_id, uint8_t success) +{ + struct virtual_ethdev_private *dev_private = NULL; + struct rte_eth_dev *vrtl_eth_dev = &rte_eth_devices[port_id]; + + dev_private = vrtl_eth_dev->data->dev_private; + + if (success) + vrtl_eth_dev->tx_pkt_burst = virtual_ethdev_tx_burst_success; + else + vrtl_eth_dev->tx_pkt_burst = virtual_ethdev_tx_burst_fail; + + dev_private->tx_burst_fail_count = 0; +} + +void +virtual_ethdev_tx_burst_fn_set_tx_pkt_fail_count(uint16_t port_id, + uint8_t packet_fail_count) +{ + struct virtual_ethdev_private *dev_private = NULL; + struct rte_eth_dev *vrtl_eth_dev = &rte_eth_devices[port_id]; + + + dev_private = vrtl_eth_dev->data->dev_private; + dev_private->tx_burst_fail_count = packet_fail_count; +} + +void +virtual_ethdev_set_link_status(uint16_t port_id, uint8_t link_status) +{ + struct rte_eth_dev *vrtl_eth_dev = &rte_eth_devices[port_id]; + + vrtl_eth_dev->data->dev_link.link_status = link_status; +} + +void +virtual_ethdev_simulate_link_status_interrupt(uint16_t port_id, + uint8_t link_status) +{ + struct rte_eth_dev *vrtl_eth_dev = &rte_eth_devices[port_id]; + + vrtl_eth_dev->data->dev_link.link_status = link_status; + + _rte_eth_dev_callback_process(vrtl_eth_dev, RTE_ETH_EVENT_INTR_LSC, + NULL, NULL); +} + +int +virtual_ethdev_add_mbufs_to_rx_queue(uint16_t port_id, + struct rte_mbuf **pkt_burst, int burst_length) +{ + struct rte_eth_dev *vrtl_eth_dev = &rte_eth_devices[port_id]; + struct virtual_ethdev_private *dev_private = + vrtl_eth_dev->data->dev_private; + + return rte_ring_enqueue_burst(dev_private->rx_queue, (void **)pkt_burst, + burst_length, NULL); +} + +int +virtual_ethdev_get_mbufs_from_tx_queue(uint16_t port_id, + struct rte_mbuf **pkt_burst, int burst_length) +{ + struct virtual_ethdev_private *dev_private; + struct rte_eth_dev *vrtl_eth_dev = &rte_eth_devices[port_id]; + + dev_private = vrtl_eth_dev->data->dev_private; + return rte_ring_dequeue_burst(dev_private->tx_queue, (void **)pkt_burst, + burst_length, NULL); +} + + +int +virtual_ethdev_create(const char *name, struct ether_addr *mac_addr, + uint8_t socket_id, uint8_t isr_support) +{ + struct rte_pci_device *pci_dev = NULL; + struct rte_eth_dev *eth_dev = NULL; + struct rte_pci_driver *pci_drv = NULL; + struct rte_pci_id *id_table = NULL; + struct virtual_ethdev_private *dev_private = NULL; + char name_buf[RTE_RING_NAMESIZE]; + + + /* now do all data allocation - for eth_dev structure, dummy pci driver + * and internal (dev_private) data + */ + + pci_dev = rte_zmalloc_socket(name, sizeof(*pci_dev), 0, socket_id); + if (pci_dev == NULL) + goto err; + + pci_drv = rte_zmalloc_socket(name, sizeof(*pci_drv), 0, socket_id); + if (pci_drv == NULL) + goto err; + + id_table = rte_zmalloc_socket(name, sizeof(*id_table), 0, socket_id); + if (id_table == NULL) + goto err; + id_table->device_id = 0xBEEF; + + dev_private = rte_zmalloc_socket(name, sizeof(*dev_private), 0, socket_id); + if (dev_private == NULL) + goto err; + + snprintf(name_buf, sizeof(name_buf), "%s_rxQ", name); + dev_private->rx_queue = rte_ring_create(name_buf, MAX_PKT_BURST, socket_id, + 0); + if (dev_private->rx_queue == NULL) + goto err; + + snprintf(name_buf, sizeof(name_buf), "%s_txQ", name); + dev_private->tx_queue = rte_ring_create(name_buf, MAX_PKT_BURST, socket_id, + 0); + if (dev_private->tx_queue == NULL) + goto err; + + /* reserve an ethdev entry */ + eth_dev = rte_eth_dev_allocate(name); + if (eth_dev == NULL) + goto err; + + pci_dev->device.numa_node = socket_id; + pci_dev->device.name = eth_dev->data->name; + pci_drv->driver.name = virtual_ethdev_driver_name; + pci_drv->id_table = id_table; + + if (isr_support) + pci_drv->drv_flags |= RTE_PCI_DRV_INTR_LSC; + else + pci_drv->drv_flags &= ~RTE_PCI_DRV_INTR_LSC; + + + eth_dev->device = &pci_dev->device; + eth_dev->device->driver = &pci_drv->driver; + + eth_dev->data->nb_rx_queues = (uint16_t)1; + eth_dev->data->nb_tx_queues = (uint16_t)1; + + eth_dev->data->dev_link.link_status = ETH_LINK_DOWN; + eth_dev->data->dev_link.link_speed = ETH_SPEED_NUM_10G; + eth_dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX; + + eth_dev->data->mac_addrs = rte_zmalloc(name, ETHER_ADDR_LEN, 0); + if (eth_dev->data->mac_addrs == NULL) + goto err; + + memcpy(eth_dev->data->mac_addrs, mac_addr, + sizeof(*eth_dev->data->mac_addrs)); + + eth_dev->data->dev_started = 0; + eth_dev->data->promiscuous = 0; + eth_dev->data->scattered_rx = 0; + eth_dev->data->all_multicast = 0; + + eth_dev->data->dev_private = dev_private; + + /* Copy default device operation functions */ + dev_private->dev_ops = virtual_ethdev_default_dev_ops; + eth_dev->dev_ops = &dev_private->dev_ops; + + pci_dev->device.driver = &pci_drv->driver; + eth_dev->device = &pci_dev->device; + + eth_dev->rx_pkt_burst = virtual_ethdev_rx_burst_success; + eth_dev->tx_pkt_burst = virtual_ethdev_tx_burst_success; + + return eth_dev->data->port_id; + +err: + rte_free(pci_dev); + rte_free(pci_drv); + rte_free(id_table); + rte_free(dev_private); + + return -1; +} diff --git a/dpdk/test/test/virtual_pmd.h b/dpdk/test/test/virtual_pmd.h new file mode 100644 index 00000000..7c53dd34 --- /dev/null +++ b/dpdk/test/test/virtual_pmd.h @@ -0,0 +1,106 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __VIRTUAL_ETHDEV_H_ +#define __VIRTUAL_ETHDEV_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <rte_ether.h> + +int +virtual_ethdev_init(void); + +int +virtual_ethdev_create(const char *name, struct ether_addr *mac_addr, + uint8_t socket_id, uint8_t isr_support); + +void +virtual_ethdev_set_link_status(uint16_t port_id, uint8_t link_status); + +void +virtual_ethdev_simulate_link_status_interrupt(uint16_t port_id, + uint8_t link_status); + +int +virtual_ethdev_add_mbufs_to_rx_queue(uint16_t port_id, + struct rte_mbuf **pkts_burst, int burst_length); + +int +virtual_ethdev_get_mbufs_from_tx_queue(uint16_t port_id, + struct rte_mbuf **pkt_burst, int burst_length); + +/** Control methods for the dev_ops functions pointer to control the behavior + * of the Virtual PMD */ + +void +virtual_ethdev_start_fn_set_success(uint16_t port_id, uint8_t success); + +void +virtual_ethdev_stop_fn_set_success(uint16_t port_id, uint8_t success); + +void +virtual_ethdev_configure_fn_set_success(uint16_t port_id, uint8_t success); + +void +virtual_ethdev_rx_queue_setup_fn_set_success(uint16_t port_id, + uint8_t success); + +void +virtual_ethdev_tx_queue_setup_fn_set_success(uint16_t port_id, + uint8_t success); + +void +virtual_ethdev_link_update_fn_set_success(uint16_t port_id, uint8_t success); + +void +virtual_ethdev_rx_burst_fn_set_success(uint16_t port_id, uint8_t success); + +void +virtual_ethdev_tx_burst_fn_set_success(uint16_t port_id, uint8_t success); + +/* if a value greater than zero is set for packet_fail_count then virtual + * device tx burst function will fail that many packet from burst or all + * packets if packet_fail_count is greater than the number of packets in the + * burst */ +void +virtual_ethdev_tx_burst_fn_set_tx_pkt_fail_count(uint16_t port_id, + uint8_t packet_fail_count); + +#ifdef __cplusplus +} +#endif + +#endif /* __VIRTUAL_ETHDEV_H_ */ diff --git a/dpdk/tools/cpu_layout.py b/dpdk/tools/cpu_layout.py deleted file mode 100755 index d38d0b5a..00000000 --- a/dpdk/tools/cpu_layout.py +++ /dev/null @@ -1,95 +0,0 @@ -#! /usr/bin/python -# -# BSD LICENSE -# -# Copyright(c) 2010-2014 Intel Corporation. All rights reserved. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in -# the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Intel Corporation nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# - -import sys - -sockets = [] -cores = [] -core_map = {} - -fd=open("/proc/cpuinfo") -lines = fd.readlines() -fd.close() - -core_details = [] -core_lines = {} -for line in lines: - if len(line.strip()) != 0: - name, value = line.split(":", 1) - core_lines[name.strip()] = value.strip() - else: - core_details.append(core_lines) - core_lines = {} - -for core in core_details: - for field in ["processor", "core id", "physical id"]: - if field not in core: - print "Error getting '%s' value from /proc/cpuinfo" % field - sys.exit(1) - core[field] = int(core[field]) - - if core["core id"] not in cores: - cores.append(core["core id"]) - if core["physical id"] not in sockets: - sockets.append(core["physical id"]) - key = (core["physical id"], core["core id"]) - if key not in core_map: - core_map[key] = [] - core_map[key].append(core["processor"]) - -print "============================================================" -print "Core and Socket Information (as reported by '/proc/cpuinfo')" -print "============================================================\n" -print "cores = ",cores -print "sockets = ", sockets -print "" - -max_processor_len = len(str(len(cores) * len(sockets) * 2 - 1)) -max_core_map_len = max_processor_len * 2 + len('[, ]') + len('Socket ') -max_core_id_len = len(str(max(cores))) - -print " ".ljust(max_core_id_len + len('Core ')), -for s in sockets: - print "Socket %s" % str(s).ljust(max_core_map_len - len('Socket ')), -print "" -print " ".ljust(max_core_id_len + len('Core ')), -for s in sockets: - print "--------".ljust(max_core_map_len), -print "" - -for c in cores: - print "Core %s" % str(c).ljust(max_core_id_len), - for s in sockets: - print str(core_map[(s,c)]).ljust(max_core_map_len), - print "" diff --git a/dpdk/tools/dpdk-devbind.py b/dpdk/tools/dpdk-devbind.py deleted file mode 100755 index 34be4953..00000000 --- a/dpdk/tools/dpdk-devbind.py +++ /dev/null @@ -1,577 +0,0 @@ -#! /usr/bin/python -# -# BSD LICENSE -# -# Copyright(c) 2010-2014 Intel Corporation. All rights reserved. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in -# the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Intel Corporation nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# - -import sys -import os -import getopt -import subprocess -from os.path import exists, abspath, dirname, basename - -# The PCI base class for NETWORK devices -NETWORK_BASE_CLASS = "02" - -# global dict ethernet devices present. Dictionary indexed by PCI address. -# Each device within this is itself a dictionary of device properties -devices = {} -# list of supported DPDK drivers -dpdk_drivers = ["igb_uio", "vfio-pci", "uio_pci_generic"] - -# command-line arg flags -b_flag = None -status_flag = False -force_flag = False -args = [] - - -def usage(): - '''Print usage information for the program''' - argv0 = basename(sys.argv[0]) - print(""" -Usage: ------- - - %(argv0)s [options] DEVICE1 DEVICE2 .... - -where DEVICE1, DEVICE2 etc, are specified via PCI "domain:bus:slot.func" syntax -or "bus:slot.func" syntax. For devices bound to Linux kernel drivers, they may -also be referred to by Linux interface name e.g. eth0, eth1, em0, em1, etc. - -Options: - --help, --usage: - Display usage information and quit - - -s, --status: - Print the current status of all known network interfaces. - For each device, it displays the PCI domain, bus, slot and function, - along with a text description of the device. Depending upon whether the - device is being used by a kernel driver, the igb_uio driver, or no - driver, other relevant information will be displayed: - * the Linux interface name e.g. if=eth0 - * the driver being used e.g. drv=igb_uio - * any suitable drivers not currently using that device - e.g. unused=igb_uio - NOTE: if this flag is passed along with a bind/unbind option, the - status display will always occur after the other operations have taken - place. - - -b driver, --bind=driver: - Select the driver to use or \"none\" to unbind the device - - -u, --unbind: - Unbind a device (Equivalent to \"-b none\") - - --force: - By default, devices which are used by Linux - as indicated by having - routes in the routing table - cannot be modified. Using the --force - flag overrides this behavior, allowing active links to be forcibly - unbound. - WARNING: This can lead to loss of network connection and should be used - with caution. - -Examples: ---------- - -To display current device status: - %(argv0)s --status - -To bind eth1 from the current driver and move to use igb_uio - %(argv0)s --bind=igb_uio eth1 - -To unbind 0000:01:00.0 from using any driver - %(argv0)s -u 0000:01:00.0 - -To bind 0000:02:00.0 and 0000:02:00.1 to the ixgbe kernel driver - %(argv0)s -b ixgbe 02:00.0 02:00.1 - - """ % locals()) # replace items from local variables - - -# This is roughly compatible with check_output function in subprocess module -# which is only available in python 2.7. -def check_output(args, stderr=None): - '''Run a command and capture its output''' - return subprocess.Popen(args, stdout=subprocess.PIPE, - stderr=stderr).communicate()[0] - - -def find_module(mod): - '''find the .ko file for kernel module named mod. - Searches the $RTE_SDK/$RTE_TARGET directory, the kernel - modules directory and finally under the parent directory of - the script ''' - # check $RTE_SDK/$RTE_TARGET directory - if 'RTE_SDK' in os.environ and 'RTE_TARGET' in os.environ: - path = "%s/%s/kmod/%s.ko" % (os.environ['RTE_SDK'], - os.environ['RTE_TARGET'], mod) - if exists(path): - return path - - # check using depmod - try: - depmod_out = check_output(["modinfo", "-n", mod], - stderr=subprocess.STDOUT).lower() - if "error" not in depmod_out: - path = depmod_out.strip() - if exists(path): - return path - except: # if modinfo can't find module, it fails, so continue - pass - - # check for a copy based off current path - tools_dir = dirname(abspath(sys.argv[0])) - if (tools_dir.endswith("tools")): - base_dir = dirname(tools_dir) - find_out = check_output(["find", base_dir, "-name", mod + ".ko"]) - if len(find_out) > 0: # something matched - path = find_out.splitlines()[0] - if exists(path): - return path - - -def check_modules(): - '''Checks that igb_uio is loaded''' - global dpdk_drivers - - # list of supported modules - mods = [{"Name": driver, "Found": False} for driver in dpdk_drivers] - - # first check if module is loaded - try: - # Get list of sysfs modules (both built-in and dynamically loaded) - sysfs_path = '/sys/module/' - - # Get the list of directories in sysfs_path - sysfs_mods = [os.path.join(sysfs_path, o) for o - in os.listdir(sysfs_path) - if os.path.isdir(os.path.join(sysfs_path, o))] - - # Extract the last element of '/sys/module/abc' in the array - sysfs_mods = [a.split('/')[-1] for a in sysfs_mods] - - # special case for vfio_pci (module is named vfio-pci, - # but its .ko is named vfio_pci) - sysfs_mods = map(lambda a: - a if a != 'vfio_pci' else 'vfio-pci', sysfs_mods) - - for mod in mods: - if mod["Name"] in sysfs_mods: - mod["Found"] = True - except: - pass - - # check if we have at least one loaded module - if True not in [mod["Found"] for mod in mods] and b_flag is not None: - if b_flag in dpdk_drivers: - print("Error - no supported modules(DPDK driver) are loaded") - sys.exit(1) - else: - print("Warning - no supported modules(DPDK driver) are loaded") - - # change DPDK driver list to only contain drivers that are loaded - dpdk_drivers = [mod["Name"] for mod in mods if mod["Found"]] - - -def has_driver(dev_id): - '''return true if a device is assigned to a driver. False otherwise''' - return "Driver_str" in devices[dev_id] - - -def get_pci_device_details(dev_id): - '''This function gets additional details for a PCI device''' - device = {} - - extra_info = check_output(["lspci", "-vmmks", dev_id]).splitlines() - - # parse lspci details - for line in extra_info: - if len(line) == 0: - continue - name, value = line.decode().split("\t", 1) - name = name.strip(":") + "_str" - device[name] = value - # check for a unix interface name - device["Interface"] = "" - for base, dirs, _ in os.walk("/sys/bus/pci/devices/%s/" % dev_id): - if "net" in dirs: - device["Interface"] = \ - ",".join(os.listdir(os.path.join(base, "net"))) - break - # check if a port is used for ssh connection - device["Ssh_if"] = False - device["Active"] = "" - - return device - - -def get_nic_details(): - '''This function populates the "devices" dictionary. The keys used are - the pci addresses (domain:bus:slot.func). The values are themselves - dictionaries - one for each NIC.''' - global devices - global dpdk_drivers - - # clear any old data - devices = {} - # first loop through and read details for all devices - # request machine readable format, with numeric IDs - dev = {} - dev_lines = check_output(["lspci", "-Dvmmn"]).splitlines() - for dev_line in dev_lines: - if (len(dev_line) == 0): - if dev["Class"][0:2] == NETWORK_BASE_CLASS: - # convert device and vendor ids to numbers, then add to global - dev["Vendor"] = int(dev["Vendor"], 16) - dev["Device"] = int(dev["Device"], 16) - # use dict to make copy of dev - devices[dev["Slot"]] = dict(dev) - else: - name, value = dev_line.decode().split("\t", 1) - dev[name.rstrip(":")] = value - - # check what is the interface if any for an ssh connection if - # any to this host, so we can mark it later. - ssh_if = [] - route = check_output(["ip", "-o", "route"]) - # filter out all lines for 169.254 routes - route = "\n".join(filter(lambda ln: not ln.startswith("169.254"), - route.decode().splitlines())) - rt_info = route.split() - for i in range(len(rt_info) - 1): - if rt_info[i] == "dev": - ssh_if.append(rt_info[i+1]) - - # based on the basic info, get extended text details - for d in devices.keys(): - # get additional info and add it to existing data - devices[d] = devices[d].copy() - devices[d].update(get_pci_device_details(d).items()) - - for _if in ssh_if: - if _if in devices[d]["Interface"].split(","): - devices[d]["Ssh_if"] = True - devices[d]["Active"] = "*Active*" - break - - # add igb_uio to list of supporting modules if needed - if "Module_str" in devices[d]: - for driver in dpdk_drivers: - if driver not in devices[d]["Module_str"]: - devices[d]["Module_str"] = \ - devices[d]["Module_str"] + ",%s" % driver - else: - devices[d]["Module_str"] = ",".join(dpdk_drivers) - - # make sure the driver and module strings do not have any duplicates - if has_driver(d): - modules = devices[d]["Module_str"].split(",") - if devices[d]["Driver_str"] in modules: - modules.remove(devices[d]["Driver_str"]) - devices[d]["Module_str"] = ",".join(modules) - - -def dev_id_from_dev_name(dev_name): - '''Take a device "name" - a string passed in by user to identify a NIC - device, and determine the device id - i.e. the domain:bus:slot.func - for - it, which can then be used to index into the devices array''' - - # check if it's already a suitable index - if dev_name in devices: - return dev_name - # check if it's an index just missing the domain part - elif "0000:" + dev_name in devices: - return "0000:" + dev_name - else: - # check if it's an interface name, e.g. eth1 - for d in devices.keys(): - if dev_name in devices[d]["Interface"].split(","): - return devices[d]["Slot"] - # if nothing else matches - error - print("Unknown device: %s. " - "Please specify device in \"bus:slot.func\" format" % dev_name) - sys.exit(1) - - -def unbind_one(dev_id, force): - '''Unbind the device identified by "dev_id" from its current driver''' - dev = devices[dev_id] - if not has_driver(dev_id): - print("%s %s %s is not currently managed by any driver\n" % - (dev["Slot"], dev["Device_str"], dev["Interface"])) - return - - # prevent us disconnecting ourselves - if dev["Ssh_if"] and not force: - print("Routing table indicates that interface %s is active. " - "Skipping unbind" % (dev_id)) - return - - # write to /sys to unbind - filename = "/sys/bus/pci/drivers/%s/unbind" % dev["Driver_str"] - try: - f = open(filename, "a") - except: - print("Error: unbind failed for %s - Cannot open %s" - % (dev_id, filename)) - sys.exit(1) - f.write(dev_id) - f.close() - - -def bind_one(dev_id, driver, force): - '''Bind the device given by "dev_id" to the driver "driver". If the device - is already bound to a different driver, it will be unbound first''' - dev = devices[dev_id] - saved_driver = None # used to rollback any unbind in case of failure - - # prevent disconnection of our ssh session - if dev["Ssh_if"] and not force: - print("Routing table indicates that interface %s is active. " - "Not modifying" % (dev_id)) - return - - # unbind any existing drivers we don't want - if has_driver(dev_id): - if dev["Driver_str"] == driver: - print("%s already bound to driver %s, skipping\n" - % (dev_id, driver)) - return - else: - saved_driver = dev["Driver_str"] - unbind_one(dev_id, force) - dev["Driver_str"] = "" # clear driver string - - # if we are binding to one of DPDK drivers, add PCI id's to that driver - if driver in dpdk_drivers: - filename = "/sys/bus/pci/drivers/%s/new_id" % driver - try: - f = open(filename, "w") - except: - print("Error: bind failed for %s - Cannot open %s" - % (dev_id, filename)) - return - try: - f.write("%04x %04x" % (dev["Vendor"], dev["Device"])) - f.close() - except: - print("Error: bind failed for %s - Cannot write new PCI ID to " - "driver %s" % (dev_id, driver)) - return - - # do the bind by writing to /sys - filename = "/sys/bus/pci/drivers/%s/bind" % driver - try: - f = open(filename, "a") - except: - print("Error: bind failed for %s - Cannot open %s" - % (dev_id, filename)) - if saved_driver is not None: # restore any previous driver - bind_one(dev_id, saved_driver, force) - return - try: - f.write(dev_id) - f.close() - except: - # for some reason, closing dev_id after adding a new PCI ID to new_id - # results in IOError. however, if the device was successfully bound, - # we don't care for any errors and can safely ignore IOError - tmp = get_pci_device_details(dev_id) - if "Driver_str" in tmp and tmp["Driver_str"] == driver: - return - print("Error: bind failed for %s - Cannot bind to driver %s" - % (dev_id, driver)) - if saved_driver is not None: # restore any previous driver - bind_one(dev_id, saved_driver, force) - return - - -def unbind_all(dev_list, force=False): - """Unbind method, takes a list of device locations""" - dev_list = map(dev_id_from_dev_name, dev_list) - for d in dev_list: - unbind_one(d, force) - - -def bind_all(dev_list, driver, force=False): - """Bind method, takes a list of device locations""" - global devices - - dev_list = map(dev_id_from_dev_name, dev_list) - - for d in dev_list: - bind_one(d, driver, force) - - # when binding devices to a generic driver (i.e. one that doesn't have a - # PCI ID table), some devices that are not bound to any other driver could - # be bound even if no one has asked them to. hence, we check the list of - # drivers again, and see if some of the previously-unbound devices were - # erroneously bound. - for d in devices.keys(): - # skip devices that were already bound or that we know should be bound - if "Driver_str" in devices[d] or d in dev_list: - continue - - # update information about this device - devices[d] = dict(devices[d].items() + - get_pci_device_details(d).items()) - - # check if updated information indicates that the device was bound - if "Driver_str" in devices[d]: - unbind_one(d, force) - - -def display_devices(title, dev_list, extra_params=None): - '''Displays to the user the details of a list of devices given in - "dev_list". The "extra_params" parameter, if given, should contain a string - with %()s fields in it for replacement by the named fields in each - device's dictionary.''' - strings = [] # this holds the strings to print. We sort before printing - print("\n%s" % title) - print("="*len(title)) - if len(dev_list) == 0: - strings.append("<none>") - else: - for dev in dev_list: - if extra_params is not None: - strings.append("%s '%s' %s" % (dev["Slot"], - dev["Device_str"], extra_params % dev)) - else: - strings.append("%s '%s'" % (dev["Slot"], dev["Device_str"])) - # sort before printing, so that the entries appear in PCI order - strings.sort() - print("\n".join(strings)) # print one per line - - -def show_status(): - '''Function called when the script is passed the "--status" option. - Displays to the user what devices are bound to the igb_uio driver, the - kernel driver or to no driver''' - global dpdk_drivers - kernel_drv = [] - dpdk_drv = [] - no_drv = [] - - # split our list of devices into the three categories above - for d in devices.keys(): - if not has_driver(d): - no_drv.append(devices[d]) - continue - if devices[d]["Driver_str"] in dpdk_drivers: - dpdk_drv.append(devices[d]) - else: - kernel_drv.append(devices[d]) - - # print each category separately, so we can clearly see what's used by DPDK - display_devices("Network devices using DPDK-compatible driver", dpdk_drv, - "drv=%(Driver_str)s unused=%(Module_str)s") - display_devices("Network devices using kernel driver", kernel_drv, - "if=%(Interface)s drv=%(Driver_str)s " - "unused=%(Module_str)s %(Active)s") - display_devices("Other network devices", no_drv, "unused=%(Module_str)s") - - -def parse_args(): - '''Parses the command-line arguments given by the user and takes the - appropriate action for each''' - global b_flag - global status_flag - global force_flag - global args - if len(sys.argv) <= 1: - usage() - sys.exit(0) - - try: - opts, args = getopt.getopt(sys.argv[1:], "b:us", - ["help", "usage", "status", "force", - "bind=", "unbind"]) - except getopt.GetoptError as error: - print(str(error)) - print("Run '%s --usage' for further information" % sys.argv[0]) - sys.exit(1) - - for opt, arg in opts: - if opt == "--help" or opt == "--usage": - usage() - sys.exit(0) - if opt == "--status" or opt == "-s": - status_flag = True - if opt == "--force": - force_flag = True - if opt == "-b" or opt == "-u" or opt == "--bind" or opt == "--unbind": - if b_flag is not None: - print("Error - Only one bind or unbind may be specified\n") - sys.exit(1) - if opt == "-u" or opt == "--unbind": - b_flag = "none" - else: - b_flag = arg - - -def do_arg_actions(): - '''do the actual action requested by the user''' - global b_flag - global status_flag - global force_flag - global args - - if b_flag is None and not status_flag: - print("Error: No action specified for devices." - "Please give a -b or -u option") - print("Run '%s --usage' for further information" % sys.argv[0]) - sys.exit(1) - - if b_flag is not None and len(args) == 0: - print("Error: No devices specified.") - print("Run '%s --usage' for further information" % sys.argv[0]) - sys.exit(1) - - if b_flag == "none" or b_flag == "None": - unbind_all(args, force_flag) - elif b_flag is not None: - bind_all(args, b_flag, force_flag) - if status_flag: - if b_flag is not None: - get_nic_details() # refresh if we have changed anything - show_status() - - -def main(): - '''program main function''' - parse_args() - check_modules() - get_nic_details() - do_arg_actions() - -if __name__ == "__main__": - main() diff --git a/dpdk/tools/dpdk-pmdinfo.py b/dpdk/tools/dpdk-pmdinfo.py deleted file mode 100755 index 3db9819c..00000000 --- a/dpdk/tools/dpdk-pmdinfo.py +++ /dev/null @@ -1,636 +0,0 @@ -#!/usr/bin/env python -# ------------------------------------------------------------------------- -# -# Utility to dump PMD_INFO_STRING support from an object file -# -# ------------------------------------------------------------------------- -import os -import sys -from optparse import OptionParser -import string -import json -import platform - -# For running from development directory. It should take precedence over the -# installed pyelftools. -sys.path.insert(0, '.') - - -from elftools import __version__ -from elftools.common.exceptions import ELFError -from elftools.common.py3compat import ( - ifilter, byte2int, bytes2str, itervalues, str2bytes) -from elftools.elf.elffile import ELFFile -from elftools.elf.dynamic import DynamicSection, DynamicSegment -from elftools.elf.enums import ENUM_D_TAG -from elftools.elf.segments import InterpSegment -from elftools.elf.sections import SymbolTableSection -from elftools.elf.gnuversions import ( - GNUVerSymSection, GNUVerDefSection, - GNUVerNeedSection, -) -from elftools.elf.relocation import RelocationSection -from elftools.elf.descriptions import ( - describe_ei_class, describe_ei_data, describe_ei_version, - describe_ei_osabi, describe_e_type, describe_e_machine, - describe_e_version_numeric, describe_p_type, describe_p_flags, - describe_sh_type, describe_sh_flags, - describe_symbol_type, describe_symbol_bind, describe_symbol_visibility, - describe_symbol_shndx, describe_reloc_type, describe_dyn_tag, - describe_ver_flags, -) -from elftools.elf.constants import E_FLAGS -from elftools.dwarf.dwarfinfo import DWARFInfo -from elftools.dwarf.descriptions import ( - describe_reg_name, describe_attr_value, set_global_machine_arch, - describe_CFI_instructions, describe_CFI_register_rule, - describe_CFI_CFA_rule, -) -from elftools.dwarf.constants import ( - DW_LNS_copy, DW_LNS_set_file, DW_LNE_define_file) -from elftools.dwarf.callframe import CIE, FDE - -raw_output = False -pcidb = None - -# =========================================== - - -class Vendor: - """ - Class for vendors. This is the top level class - for the devices belong to a specific vendor. - self.devices is the device dictionary - subdevices are in each device. - """ - - def __init__(self, vendorStr): - """ - Class initializes with the raw line from pci.ids - Parsing takes place inside __init__ - """ - self.ID = vendorStr.split()[0] - self.name = vendorStr.replace("%s " % self.ID, "").rstrip() - self.devices = {} - - def addDevice(self, deviceStr): - """ - Adds a device to self.devices - takes the raw line from pci.ids - """ - s = deviceStr.strip() - devID = s.split()[0] - if devID in self.devices: - pass - else: - self.devices[devID] = Device(deviceStr) - - def report(self): - print self.ID, self.name - for id, dev in self.devices.items(): - dev.report() - - def find_device(self, devid): - # convert to a hex string and remove 0x - devid = hex(devid)[2:] - try: - return self.devices[devid] - except: - return Device("%s Unknown Device" % devid) - - -class Device: - - def __init__(self, deviceStr): - """ - Class for each device. - Each vendor has its own devices dictionary. - """ - s = deviceStr.strip() - self.ID = s.split()[0] - self.name = s.replace("%s " % self.ID, "") - self.subdevices = {} - - def report(self): - print "\t%s\t%s" % (self.ID, self.name) - for subID, subdev in self.subdevices.items(): - subdev.report() - - def addSubDevice(self, subDeviceStr): - """ - Adds a subvendor, subdevice to device. - Uses raw line from pci.ids - """ - s = subDeviceStr.strip() - spl = s.split() - subVendorID = spl[0] - subDeviceID = spl[1] - subDeviceName = s.split(" ")[-1] - devID = "%s:%s" % (subVendorID, subDeviceID) - self.subdevices[devID] = SubDevice( - subVendorID, subDeviceID, subDeviceName) - - def find_subid(self, subven, subdev): - subven = hex(subven)[2:] - subdev = hex(subdev)[2:] - devid = "%s:%s" % (subven, subdev) - - try: - return self.subdevices[devid] - except: - if (subven == "ffff" and subdev == "ffff"): - return SubDevice("ffff", "ffff", "(All Subdevices)") - else: - return SubDevice(subven, subdev, "(Unknown Subdevice)") - - -class SubDevice: - """ - Class for subdevices. - """ - - def __init__(self, vendor, device, name): - """ - Class initializes with vendorid, deviceid and name - """ - self.vendorID = vendor - self.deviceID = device - self.name = name - - def report(self): - print "\t\t%s\t%s\t%s" % (self.vendorID, self.deviceID, self.name) - - -class PCIIds: - """ - Top class for all pci.ids entries. - All queries will be asked to this class. - PCIIds.vendors["0e11"].devices["0046"].\ - subdevices["0e11:4091"].name = "Smart Array 6i" - """ - - def __init__(self, filename): - """ - Prepares the directories. - Checks local data file. - Tries to load from local, if not found, downloads from web - """ - self.version = "" - self.date = "" - self.vendors = {} - self.contents = None - self.readLocal(filename) - self.parse() - - def reportVendors(self): - """Reports the vendors - """ - for vid, v in self.vendors.items(): - print v.ID, v.name - - def report(self, vendor=None): - """ - Reports everything for all vendors or a specific vendor - PCIIds.report() reports everything - PCIIDs.report("0e11") reports only "Compaq Computer Corporation" - """ - if vendor is not None: - self.vendors[vendor].report() - else: - for vID, v in self.vendors.items(): - v.report() - - def find_vendor(self, vid): - # convert vid to a hex string and remove the 0x - vid = hex(vid)[2:] - - try: - return self.vendors[vid] - except: - return Vendor("%s Unknown Vendor" % (vid)) - - def findDate(self, content): - for l in content: - if l.find("Date:") > -1: - return l.split()[-2].replace("-", "") - return None - - def parse(self): - if len(self.contents) < 1: - print "data/%s-pci.ids not found" % self.date - else: - vendorID = "" - deviceID = "" - for l in self.contents: - if l[0] == "#": - continue - elif len(l.strip()) == 0: - continue - else: - if l.find("\t\t") == 0: - self.vendors[vendorID].devices[ - deviceID].addSubDevice(l) - elif l.find("\t") == 0: - deviceID = l.strip().split()[0] - self.vendors[vendorID].addDevice(l) - else: - vendorID = l.split()[0] - self.vendors[vendorID] = Vendor(l) - - def readLocal(self, filename): - """ - Reads the local file - """ - self.contents = open(filename).readlines() - self.date = self.findDate(self.contents) - - def loadLocal(self): - """ - Loads database from local. If there is no file, - it creates a new one from web - """ - self.date = idsfile[0].split("/")[1].split("-")[0] - self.readLocal() - - -# ======================================= - -def search_file(filename, search_path): - """ Given a search path, find file with requested name """ - for path in string.split(search_path, ":"): - candidate = os.path.join(path, filename) - if os.path.exists(candidate): - return os.path.abspath(candidate) - return None - - -class ReadElf(object): - """ display_* methods are used to emit output into the output stream - """ - - def __init__(self, file, output): - """ file: - stream object with the ELF file to read - - output: - output stream to write to - """ - self.elffile = ELFFile(file) - self.output = output - - # Lazily initialized if a debug dump is requested - self._dwarfinfo = None - - self._versioninfo = None - - def _section_from_spec(self, spec): - """ Retrieve a section given a "spec" (either number or name). - Return None if no such section exists in the file. - """ - try: - num = int(spec) - if num < self.elffile.num_sections(): - return self.elffile.get_section(num) - else: - return None - except ValueError: - # Not a number. Must be a name then - return self.elffile.get_section_by_name(str2bytes(spec)) - - def pretty_print_pmdinfo(self, pmdinfo): - global pcidb - - for i in pmdinfo["pci_ids"]: - vendor = pcidb.find_vendor(i[0]) - device = vendor.find_device(i[1]) - subdev = device.find_subid(i[2], i[3]) - print("%s (%s) : %s (%s) %s" % - (vendor.name, vendor.ID, device.name, - device.ID, subdev.name)) - - def parse_pmd_info_string(self, mystring): - global raw_output - global pcidb - - optional_pmd_info = [{'id': 'params', 'tag': 'PMD PARAMETERS'}] - - i = mystring.index("=") - mystring = mystring[i + 2:] - pmdinfo = json.loads(mystring) - - if raw_output: - print(json.dumps(pmdinfo)) - return - - print("PMD NAME: " + pmdinfo["name"]) - for i in optional_pmd_info: - try: - print("%s: %s" % (i['tag'], pmdinfo[i['id']])) - except KeyError as e: - continue - - if (len(pmdinfo["pci_ids"]) != 0): - print("PMD HW SUPPORT:") - if pcidb is not None: - self.pretty_print_pmdinfo(pmdinfo) - else: - print("VENDOR\t DEVICE\t SUBVENDOR\t SUBDEVICE") - for i in pmdinfo["pci_ids"]: - print("0x%04x\t 0x%04x\t 0x%04x\t\t 0x%04x" % - (i[0], i[1], i[2], i[3])) - - print("") - - def display_pmd_info_strings(self, section_spec): - """ Display a strings dump of a section. section_spec is either a - section number or a name. - """ - section = self._section_from_spec(section_spec) - if section is None: - return - - data = section.data() - dataptr = 0 - - while dataptr < len(data): - while (dataptr < len(data) and - not (32 <= byte2int(data[dataptr]) <= 127)): - dataptr += 1 - - if dataptr >= len(data): - break - - endptr = dataptr - while endptr < len(data) and byte2int(data[endptr]) != 0: - endptr += 1 - - mystring = bytes2str(data[dataptr:endptr]) - rc = mystring.find("PMD_INFO_STRING") - if (rc != -1): - self.parse_pmd_info_string(mystring) - - dataptr = endptr - - def find_librte_eal(self, section): - for tag in section.iter_tags(): - if tag.entry.d_tag == 'DT_NEEDED': - if "librte_eal" in tag.needed: - return tag.needed - return None - - def search_for_autoload_path(self): - scanelf = self - scanfile = None - library = None - - section = self._section_from_spec(".dynamic") - try: - eallib = self.find_librte_eal(section) - if eallib is not None: - ldlibpath = os.environ.get('LD_LIBRARY_PATH') - if ldlibpath is None: - ldlibpath = "" - dtr = self.get_dt_runpath(section) - library = search_file(eallib, - dtr + ":" + ldlibpath + - ":/usr/lib64:/lib64:/usr/lib:/lib") - if library is None: - return (None, None) - if raw_output is False: - print("Scanning for autoload path in %s" % library) - scanfile = open(library, 'rb') - scanelf = ReadElf(scanfile, sys.stdout) - except AttributeError: - # Not a dynamic binary - pass - except ELFError: - scanfile.close() - return (None, None) - - section = scanelf._section_from_spec(".rodata") - if section is None: - if scanfile is not None: - scanfile.close() - return (None, None) - - data = section.data() - dataptr = 0 - - while dataptr < len(data): - while (dataptr < len(data) and - not (32 <= byte2int(data[dataptr]) <= 127)): - dataptr += 1 - - if dataptr >= len(data): - break - - endptr = dataptr - while endptr < len(data) and byte2int(data[endptr]) != 0: - endptr += 1 - - mystring = bytes2str(data[dataptr:endptr]) - rc = mystring.find("DPDK_PLUGIN_PATH") - if (rc != -1): - rc = mystring.find("=") - return (mystring[rc + 1:], library) - - dataptr = endptr - if scanfile is not None: - scanfile.close() - return (None, None) - - def get_dt_runpath(self, dynsec): - for tag in dynsec.iter_tags(): - if tag.entry.d_tag == 'DT_RUNPATH': - return tag.runpath - return "" - - def process_dt_needed_entries(self): - """ Look to see if there are any DT_NEEDED entries in the binary - And process those if there are - """ - global raw_output - runpath = "" - ldlibpath = os.environ.get('LD_LIBRARY_PATH') - if ldlibpath is None: - ldlibpath = "" - - dynsec = self._section_from_spec(".dynamic") - try: - runpath = self.get_dt_runpath(dynsec) - except AttributeError: - # dynsec is None, just return - return - - for tag in dynsec.iter_tags(): - if tag.entry.d_tag == 'DT_NEEDED': - rc = tag.needed.find("librte_pmd") - if (rc != -1): - library = search_file(tag.needed, - runpath + ":" + ldlibpath + - ":/usr/lib64:/lib64:/usr/lib:/lib") - if library is not None: - if raw_output is False: - print("Scanning %s for pmd information" % library) - with open(library, 'rb') as file: - try: - libelf = ReadElf(file, sys.stdout) - except ELFError as e: - print("%s is no an ELF file" % library) - continue - libelf.process_dt_needed_entries() - libelf.display_pmd_info_strings(".rodata") - file.close() - - -def scan_autoload_path(autoload_path): - global raw_output - - if os.path.exists(autoload_path) is False: - return - - try: - dirs = os.listdir(autoload_path) - except OSError as e: - # Couldn't read the directory, give up - return - - for d in dirs: - dpath = os.path.join(autoload_path, d) - if os.path.isdir(dpath): - scan_autoload_path(dpath) - if os.path.isfile(dpath): - try: - file = open(dpath, 'rb') - readelf = ReadElf(file, sys.stdout) - except ELFError as e: - # this is likely not an elf file, skip it - continue - except IOError as e: - # No permission to read the file, skip it - continue - - if raw_output is False: - print("Hw Support for library %s" % d) - readelf.display_pmd_info_strings(".rodata") - file.close() - - -def scan_for_autoload_pmds(dpdk_path): - """ - search the specified application or path for a pmd autoload path - then scan said path for pmds and report hw support - """ - global raw_output - - if (os.path.isfile(dpdk_path) is False): - if raw_output is False: - print("Must specify a file name") - return - - file = open(dpdk_path, 'rb') - try: - readelf = ReadElf(file, sys.stdout) - except ElfError as e: - if raw_output is False: - print("Unable to parse %s" % file) - return - - (autoload_path, scannedfile) = readelf.search_for_autoload_path() - if (autoload_path is None or autoload_path is ""): - if (raw_output is False): - print("No autoload path configured in %s" % dpdk_path) - return - if (raw_output is False): - if (scannedfile is None): - scannedfile = dpdk_path - print("Found autoload path %s in %s" % (autoload_path, scannedfile)) - - file.close() - if (raw_output is False): - print("Discovered Autoload HW Support:") - scan_autoload_path(autoload_path) - return - - -def main(stream=None): - global raw_output - global pcidb - - pcifile_default = "./pci.ids" # for unknown OS's assume local file - if platform.system() == 'Linux': - pcifile_default = "/usr/share/hwdata/pci.ids" - elif platform.system() == 'FreeBSD': - pcifile_default = "/usr/local/share/pciids/pci.ids" - if not os.path.exists(pcifile_default): - pcifile_default = "/usr/share/misc/pci_vendors" - - optparser = OptionParser( - usage='usage: %prog [-hrtp] [-d <pci id file] <elf-file>', - description="Dump pmd hardware support info", - add_help_option=True) - optparser.add_option('-r', '--raw', - action='store_true', dest='raw_output', - help='Dump raw json strings') - optparser.add_option("-d", "--pcidb", dest="pcifile", - help="specify a pci database " - "to get vendor names from", - default=pcifile_default, metavar="FILE") - optparser.add_option("-t", "--table", dest="tblout", - help="output information on hw support as a hex table", - action='store_true') - optparser.add_option("-p", "--plugindir", dest="pdir", - help="scan dpdk for autoload plugins", - action='store_true') - - options, args = optparser.parse_args() - - if options.raw_output: - raw_output = True - - if options.pcifile: - pcidb = PCIIds(options.pcifile) - if pcidb is None: - print("Pci DB file not found") - exit(1) - - if options.tblout: - options.pcifile = None - pcidb = None - - if (len(args) == 0): - optparser.print_usage() - exit(1) - - if options.pdir is True: - exit(scan_for_autoload_pmds(args[0])) - - ldlibpath = os.environ.get('LD_LIBRARY_PATH') - if (ldlibpath is None): - ldlibpath = "" - - if (os.path.exists(args[0]) is True): - myelffile = args[0] - else: - myelffile = search_file( - args[0], ldlibpath + ":/usr/lib64:/lib64:/usr/lib:/lib") - - if (myelffile is None): - print("File not found") - sys.exit(1) - - with open(myelffile, 'rb') as file: - try: - readelf = ReadElf(file, sys.stdout) - readelf.process_dt_needed_entries() - readelf.display_pmd_info_strings(".rodata") - sys.exit(0) - - except ELFError as ex: - sys.stderr.write('ELF error: %s\n' % ex) - sys.exit(1) - - -# ------------------------------------------------------------------------- -if __name__ == '__main__': - main() diff --git a/dpdk/tools/dpdk-setup.sh b/dpdk/tools/dpdk-setup.sh deleted file mode 100755 index ac81b2e4..00000000 --- a/dpdk/tools/dpdk-setup.sh +++ /dev/null @@ -1,634 +0,0 @@ -#! /bin/bash - -# BSD LICENSE -# -# Copyright(c) 2010-2014 Intel Corporation. All rights reserved. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in -# the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Intel Corporation nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# -# Run with "source /path/to/dpdk-setup.sh" -# - -# -# Change to DPDK directory ( <this-script's-dir>/.. ), and export it as RTE_SDK -# -cd $(dirname ${BASH_SOURCE[0]})/.. -export RTE_SDK=$PWD -echo "------------------------------------------------------------------------------" -echo " RTE_SDK exported as $RTE_SDK" -echo "------------------------------------------------------------------------------" - -HUGEPGSZ=`cat /proc/meminfo | grep Hugepagesize | cut -d : -f 2 | tr -d ' '` - -# -# Application EAL parameters for setting memory options (amount/channels/ranks). -# -EAL_PARAMS='-n 4' - -# -# Sets QUIT variable so script will finish. -# -quit() -{ - QUIT=$1 -} - -# Shortcut for quit. -q() -{ - quit -} - -# -# Sets up environmental variables for ICC. -# -setup_icc() -{ - DEFAULT_PATH=/opt/intel/bin/iccvars.sh - param=$1 - shpath=`which iccvars.sh 2> /dev/null` - if [ $? -eq 0 ] ; then - echo "Loading iccvars.sh from $shpath for $param" - source $shpath $param - elif [ -f $DEFAULT_PATH ] ; then - echo "Loading iccvars.sh from $DEFAULT_PATH for $param" - source $DEFAULT_PATH $param - else - echo "## ERROR: cannot find 'iccvars.sh' script to set up ICC." - echo "## To fix, please add the directory that contains" - echo "## iccvars.sh to your 'PATH' environment variable." - quit - fi -} - -# -# Sets RTE_TARGET and does a "make install". -# -setup_target() -{ - option=$1 - export RTE_TARGET=${TARGETS[option]} - - compiler=${RTE_TARGET##*-} - if [ "$compiler" == "icc" ] ; then - platform=${RTE_TARGET%%-*} - if [ "$platform" == "x86_64" ] ; then - setup_icc intel64 - else - setup_icc ia32 - fi - fi - if [ "$QUIT" == "0" ] ; then - make install T=${RTE_TARGET} - fi - echo "------------------------------------------------------------------------------" - echo " RTE_TARGET exported as $RTE_TARGET" - echo "------------------------------------------------------------------------------" -} - -# -# Creates hugepage filesystem. -# -create_mnt_huge() -{ - echo "Creating /mnt/huge and mounting as hugetlbfs" - sudo mkdir -p /mnt/huge - - grep -s '/mnt/huge' /proc/mounts > /dev/null - if [ $? -ne 0 ] ; then - sudo mount -t hugetlbfs nodev /mnt/huge - fi -} - -# -# Removes hugepage filesystem. -# -remove_mnt_huge() -{ - echo "Unmounting /mnt/huge and removing directory" - grep -s '/mnt/huge' /proc/mounts > /dev/null - if [ $? -eq 0 ] ; then - sudo umount /mnt/huge - fi - - if [ -d /mnt/huge ] ; then - sudo rm -R /mnt/huge - fi -} - -# -# Unloads igb_uio.ko. -# -remove_igb_uio_module() -{ - echo "Unloading any existing DPDK UIO module" - /sbin/lsmod | grep -s igb_uio > /dev/null - if [ $? -eq 0 ] ; then - sudo /sbin/rmmod igb_uio - fi -} - -# -# Loads new igb_uio.ko (and uio module if needed). -# -load_igb_uio_module() -{ - if [ ! -f $RTE_SDK/$RTE_TARGET/kmod/igb_uio.ko ];then - echo "## ERROR: Target does not have the DPDK UIO Kernel Module." - echo " To fix, please try to rebuild target." - return - fi - - remove_igb_uio_module - - /sbin/lsmod | grep -s uio > /dev/null - if [ $? -ne 0 ] ; then - modinfo uio > /dev/null - if [ $? -eq 0 ]; then - echo "Loading uio module" - sudo /sbin/modprobe uio - fi - fi - - # UIO may be compiled into kernel, so it may not be an error if it can't - # be loaded. - - echo "Loading DPDK UIO module" - sudo /sbin/insmod $RTE_SDK/$RTE_TARGET/kmod/igb_uio.ko - if [ $? -ne 0 ] ; then - echo "## ERROR: Could not load kmod/igb_uio.ko." - quit - fi -} - -# -# Unloads VFIO modules. -# -remove_vfio_module() -{ - echo "Unloading any existing VFIO module" - /sbin/lsmod | grep -s vfio > /dev/null - if [ $? -eq 0 ] ; then - sudo /sbin/rmmod vfio-pci - sudo /sbin/rmmod vfio_iommu_type1 - sudo /sbin/rmmod vfio - fi -} - -# -# Loads new vfio-pci (and vfio module if needed). -# -load_vfio_module() -{ - remove_vfio_module - - VFIO_PATH="kernel/drivers/vfio/pci/vfio-pci.ko" - - echo "Loading VFIO module" - /sbin/lsmod | grep -s vfio_pci > /dev/null - if [ $? -ne 0 ] ; then - if [ -f /lib/modules/$(uname -r)/$VFIO_PATH ] ; then - sudo /sbin/modprobe vfio-pci - fi - fi - - # make sure regular users can read /dev/vfio - echo "chmod /dev/vfio" - sudo chmod a+x /dev/vfio - if [ $? -ne 0 ] ; then - echo "FAIL" - quit - fi - echo "OK" - - # check if /dev/vfio/vfio exists - that way we - # know we either loaded the module, or it was - # compiled into the kernel - if [ ! -e /dev/vfio/vfio ] ; then - echo "## ERROR: VFIO not found!" - fi -} - -# -# Unloads the rte_kni.ko module. -# -remove_kni_module() -{ - echo "Unloading any existing DPDK KNI module" - /sbin/lsmod | grep -s rte_kni > /dev/null - if [ $? -eq 0 ] ; then - sudo /sbin/rmmod rte_kni - fi -} - -# -# Loads the rte_kni.ko module. -# -load_kni_module() -{ - # Check that the KNI module is already built. - if [ ! -f $RTE_SDK/$RTE_TARGET/kmod/rte_kni.ko ];then - echo "## ERROR: Target does not have the DPDK KNI Module." - echo " To fix, please try to rebuild target." - return - fi - - # Unload existing version if present. - remove_kni_module - - # Now try load the KNI module. - echo "Loading DPDK KNI module" - sudo /sbin/insmod $RTE_SDK/$RTE_TARGET/kmod/rte_kni.ko - if [ $? -ne 0 ] ; then - echo "## ERROR: Could not load kmod/rte_kni.ko." - quit - fi -} - -# -# Sets appropriate permissions on /dev/vfio/* files -# -set_vfio_permissions() -{ - # make sure regular users can read /dev/vfio - echo "chmod /dev/vfio" - sudo chmod a+x /dev/vfio - if [ $? -ne 0 ] ; then - echo "FAIL" - quit - fi - echo "OK" - - # make sure regular user can access everything inside /dev/vfio - echo "chmod /dev/vfio/*" - sudo chmod 0666 /dev/vfio/* - if [ $? -ne 0 ] ; then - echo "FAIL" - quit - fi - echo "OK" - - # since permissions are only to be set when running as - # regular user, we only check ulimit here - # - # warn if regular user is only allowed - # to memlock <64M of memory - MEMLOCK_AMNT=`ulimit -l` - - if [ "$MEMLOCK_AMNT" != "unlimited" ] ; then - MEMLOCK_MB=`expr $MEMLOCK_AMNT / 1024` - echo "" - echo "Current user memlock limit: ${MEMLOCK_MB} MB" - echo "" - echo "This is the maximum amount of memory you will be" - echo "able to use with DPDK and VFIO if run as current user." - echo -n "To change this, please adjust limits.conf memlock " - echo "limit for current user." - - if [ $MEMLOCK_AMNT -lt 65536 ] ; then - echo "" - echo "## WARNING: memlock limit is less than 64MB" - echo -n "## DPDK with VFIO may not be able to initialize " - echo "if run as current user." - fi - fi -} - -# -# Removes all reserved hugepages. -# -clear_huge_pages() -{ - echo > .echo_tmp - for d in /sys/devices/system/node/node? ; do - echo "echo 0 > $d/hugepages/hugepages-${HUGEPGSZ}/nr_hugepages" >> .echo_tmp - done - echo "Removing currently reserved hugepages" - sudo sh .echo_tmp - rm -f .echo_tmp - - remove_mnt_huge -} - -# -# Creates hugepages. -# -set_non_numa_pages() -{ - clear_huge_pages - - echo "" - echo " Input the number of ${HUGEPGSZ} hugepages" - echo " Example: to have 128MB of hugepages available in a 2MB huge page system," - echo " enter '64' to reserve 64 * 2MB pages" - echo -n "Number of pages: " - read Pages - - echo "echo $Pages > /sys/kernel/mm/hugepages/hugepages-${HUGEPGSZ}/nr_hugepages" > .echo_tmp - - echo "Reserving hugepages" - sudo sh .echo_tmp - rm -f .echo_tmp - - create_mnt_huge -} - -# -# Creates hugepages on specific NUMA nodes. -# -set_numa_pages() -{ - clear_huge_pages - - echo "" - echo " Input the number of ${HUGEPGSZ} hugepages for each node" - echo " Example: to have 128MB of hugepages available per node in a 2MB huge page system," - echo " enter '64' to reserve 64 * 2MB pages on each node" - - echo > .echo_tmp - for d in /sys/devices/system/node/node? ; do - node=$(basename $d) - echo -n "Number of pages for $node: " - read Pages - echo "echo $Pages > $d/hugepages/hugepages-${HUGEPGSZ}/nr_hugepages" >> .echo_tmp - done - echo "Reserving hugepages" - sudo sh .echo_tmp - rm -f .echo_tmp - - create_mnt_huge -} - -# -# Run unit test application. -# -run_test_app() -{ - echo "" - echo " Enter hex bitmask of cores to execute test app on" - echo " Example: to execute app on cores 0 to 7, enter 0xff" - echo -n "bitmask: " - read Bitmask - echo "Launching app" - sudo ${RTE_TARGET}/app/test -c $Bitmask $EAL_PARAMS -} - -# -# Run unit testpmd application. -# -run_testpmd_app() -{ - echo "" - echo " Enter hex bitmask of cores to execute testpmd app on" - echo " Example: to execute app on cores 0 to 7, enter 0xff" - echo -n "bitmask: " - read Bitmask - echo "Launching app" - sudo ${RTE_TARGET}/app/testpmd -c $Bitmask $EAL_PARAMS -- -i -} - -# -# Print hugepage information. -# -grep_meminfo() -{ - grep -i huge /proc/meminfo -} - -# -# Calls dpdk-devbind.py --status to show the NIC and what they -# are all bound to, in terms of drivers. -# -show_nics() -{ - if [ -d /sys/module/vfio_pci -o -d /sys/module/igb_uio ]; then - ${RTE_SDK}/tools/dpdk-devbind.py --status - else - echo "# Please load the 'igb_uio' or 'vfio-pci' kernel module before " - echo "# querying or adjusting NIC device bindings" - fi -} - -# -# Uses dpdk-devbind.py to move devices to work with vfio-pci -# -bind_nics_to_vfio() -{ - if [ -d /sys/module/vfio_pci ]; then - ${RTE_SDK}/tools/dpdk-devbind.py --status - echo "" - echo -n "Enter PCI address of device to bind to VFIO driver: " - read PCI_PATH - sudo ${RTE_SDK}/tools/dpdk-devbind.py -b vfio-pci $PCI_PATH && - echo "OK" - else - echo "# Please load the 'vfio-pci' kernel module before querying or " - echo "# adjusting NIC device bindings" - fi -} - -# -# Uses dpdk-devbind.py to move devices to work with igb_uio -# -bind_nics_to_igb_uio() -{ - if [ -d /sys/module/igb_uio ]; then - ${RTE_SDK}/tools/dpdk-devbind.py --status - echo "" - echo -n "Enter PCI address of device to bind to IGB UIO driver: " - read PCI_PATH - sudo ${RTE_SDK}/tools/dpdk-devbind.py -b igb_uio $PCI_PATH && echo "OK" - else - echo "# Please load the 'igb_uio' kernel module before querying or " - echo "# adjusting NIC device bindings" - fi -} - -# -# Uses dpdk-devbind.py to move devices to work with kernel drivers again -# -unbind_nics() -{ - ${RTE_SDK}/tools/dpdk-devbind.py --status - echo "" - echo -n "Enter PCI address of device to unbind: " - read PCI_PATH - echo "" - echo -n "Enter name of kernel driver to bind the device to: " - read DRV - sudo ${RTE_SDK}/tools/dpdk-devbind.py -b $DRV $PCI_PATH && echo "OK" -} - -# -# Options for building a target. Note that this step MUST be first as it sets -# up TARGETS[] starting from 1, and this is accessed in setup_target using the -# user entered option. -# -step1_func() -{ - TITLE="Select the DPDK environment to build" - CONFIG_NUM=1 - for cfg in config/defconfig_* ; do - cfg=${cfg/config\/defconfig_/} - TEXT[$CONFIG_NUM]="$cfg" - TARGETS[$CONFIG_NUM]=$cfg - FUNC[$CONFIG_NUM]="setup_target" - let "CONFIG_NUM+=1" - done -} - -# -# Options for setting up environment. -# -step2_func() -{ - TITLE="Setup linuxapp environment" - - TEXT[1]="Insert IGB UIO module" - FUNC[1]="load_igb_uio_module" - - TEXT[2]="Insert VFIO module" - FUNC[2]="load_vfio_module" - - TEXT[3]="Insert KNI module" - FUNC[3]="load_kni_module" - - TEXT[4]="Setup hugepage mappings for non-NUMA systems" - FUNC[4]="set_non_numa_pages" - - TEXT[5]="Setup hugepage mappings for NUMA systems" - FUNC[5]="set_numa_pages" - - TEXT[6]="Display current Ethernet device settings" - FUNC[6]="show_nics" - - TEXT[7]="Bind Ethernet device to IGB UIO module" - FUNC[7]="bind_nics_to_igb_uio" - - TEXT[8]="Bind Ethernet device to VFIO module" - FUNC[8]="bind_nics_to_vfio" - - TEXT[9]="Setup VFIO permissions" - FUNC[9]="set_vfio_permissions" -} - -# -# Options for running applications. -# -step3_func() -{ - TITLE="Run test application for linuxapp environment" - - TEXT[1]="Run test application (\$RTE_TARGET/app/test)" - FUNC[1]="run_test_app" - - TEXT[2]="Run testpmd application in interactive mode (\$RTE_TARGET/app/testpmd)" - FUNC[2]="run_testpmd_app" -} - -# -# Other options -# -step4_func() -{ - TITLE="Other tools" - - TEXT[1]="List hugepage info from /proc/meminfo" - FUNC[1]="grep_meminfo" - -} - -# -# Options for cleaning up the system -# -step5_func() -{ - TITLE="Uninstall and system cleanup" - - TEXT[1]="Unbind NICs from IGB UIO or VFIO driver" - FUNC[1]="unbind_nics" - - TEXT[2]="Remove IGB UIO module" - FUNC[2]="remove_igb_uio_module" - - TEXT[3]="Remove VFIO module" - FUNC[3]="remove_vfio_module" - - TEXT[4]="Remove KNI module" - FUNC[4]="remove_kni_module" - - TEXT[5]="Remove hugepage mappings" - FUNC[5]="clear_huge_pages" -} - -STEPS[1]="step1_func" -STEPS[2]="step2_func" -STEPS[3]="step3_func" -STEPS[4]="step4_func" -STEPS[5]="step5_func" - -QUIT=0 - -while [ "$QUIT" == "0" ]; do - OPTION_NUM=1 - - for s in $(seq ${#STEPS[@]}) ; do - ${STEPS[s]} - - echo "----------------------------------------------------------" - echo " Step $s: ${TITLE}" - echo "----------------------------------------------------------" - - for i in $(seq ${#TEXT[@]}) ; do - echo "[$OPTION_NUM] ${TEXT[i]}" - OPTIONS[$OPTION_NUM]=${FUNC[i]} - let "OPTION_NUM+=1" - done - - # Clear TEXT and FUNC arrays before next step - unset TEXT - unset FUNC - - echo "" - done - - echo "[$OPTION_NUM] Exit Script" - OPTIONS[$OPTION_NUM]="quit" - echo "" - echo -n "Option: " - read our_entry - echo "" - ${OPTIONS[our_entry]} ${our_entry} - - if [ "$QUIT" == "0" ] ; then - echo - echo -n "Press enter to continue ..."; read - fi - -done diff --git a/dpdk/usertools/cpu_layout.py b/dpdk/usertools/cpu_layout.py new file mode 100644 index 00000000..d3c8eba7 --- /dev/null +++ b/dpdk/usertools/cpu_layout.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python + +# +# BSD LICENSE +# +# Copyright(c) 2010-2014 Intel Corporation. All rights reserved. +# Copyright(c) 2017 Cavium, Inc. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +from __future__ import print_function +import sys +try: + xrange # Python 2 +except NameError: + xrange = range # Python 3 + +sockets = [] +cores = [] +core_map = {} +base_path = "/sys/devices/system/cpu" +fd = open("{}/kernel_max".format(base_path)) +max_cpus = int(fd.read()) +fd.close() +for cpu in xrange(max_cpus + 1): + try: + fd = open("{}/cpu{}/topology/core_id".format(base_path, cpu)) + except IOError: + continue + except: + break + core = int(fd.read()) + fd.close() + fd = open("{}/cpu{}/topology/physical_package_id".format(base_path, cpu)) + socket = int(fd.read()) + fd.close() + if core not in cores: + cores.append(core) + if socket not in sockets: + sockets.append(socket) + key = (socket, core) + if key not in core_map: + core_map[key] = [] + core_map[key].append(cpu) + +print(format("=" * (47 + len(base_path)))) +print("Core and Socket Information (as reported by '{}')".format(base_path)) +print("{}\n".format("=" * (47 + len(base_path)))) +print("cores = ", cores) +print("sockets = ", sockets) +print("") + +max_processor_len = len(str(len(cores) * len(sockets) * 2 - 1)) +max_thread_count = len(list(core_map.values())[0]) +max_core_map_len = (max_processor_len * max_thread_count) \ + + len(", ") * (max_thread_count - 1) \ + + len('[]') + len('Socket ') +max_core_id_len = len(str(max(cores))) + +output = " ".ljust(max_core_id_len + len('Core ')) +for s in sockets: + output += " Socket %s" % str(s).ljust(max_core_map_len - len('Socket ')) +print(output) + +output = " ".ljust(max_core_id_len + len('Core ')) +for s in sockets: + output += " --------".ljust(max_core_map_len) + output += " " +print(output) + +for c in cores: + output = "Core %s" % str(c).ljust(max_core_id_len) + for s in sockets: + if (s,c) in core_map: + output += " " + str(core_map[(s, c)]).ljust(max_core_map_len) + else: + output += " " * (max_core_map_len + 1) + print(output) diff --git a/dpdk/usertools/dpdk-devbind.py b/dpdk/usertools/dpdk-devbind.py new file mode 100644 index 00000000..df9b21a6 --- /dev/null +++ b/dpdk/usertools/dpdk-devbind.py @@ -0,0 +1,718 @@ +#! /usr/bin/env python +# +# BSD LICENSE +# +# Copyright(c) 2010-2014 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +import sys +import os +import getopt +import subprocess +from os.path import exists, abspath, dirname, basename + +# The PCI base class for all devices +network_class = {'Class': '02', 'Vendor': None, 'Device': None, + 'SVendor': None, 'SDevice': None} +encryption_class = {'Class': '10', 'Vendor': None, 'Device': None, + 'SVendor': None, 'SDevice': None} +intel_processor_class = {'Class': '0b', 'Vendor': '8086', 'Device': None, + 'SVendor': None, 'SDevice': None} +cavium_sso = {'Class': '08', 'Vendor': '177d', 'Device': 'a04b,a04d', + 'SVendor': None, 'SDevice': None} +cavium_fpa = {'Class': '08', 'Vendor': '177d', 'Device': 'a053', + 'SVendor': None, 'SDevice': None} +cavium_pkx = {'Class': '08', 'Vendor': '177d', 'Device': 'a0dd,a049', + 'SVendor': None, 'SDevice': None} + +network_devices = [network_class, cavium_pkx] +crypto_devices = [encryption_class, intel_processor_class] +eventdev_devices = [cavium_sso] +mempool_devices = [cavium_fpa] + +# global dict ethernet devices present. Dictionary indexed by PCI address. +# Each device within this is itself a dictionary of device properties +devices = {} +# list of supported DPDK drivers +dpdk_drivers = ["igb_uio", "vfio-pci", "uio_pci_generic"] + +# command-line arg flags +b_flag = None +status_flag = False +force_flag = False +args = [] + + +def usage(): + '''Print usage information for the program''' + argv0 = basename(sys.argv[0]) + print(""" +Usage: +------ + + %(argv0)s [options] DEVICE1 DEVICE2 .... + +where DEVICE1, DEVICE2 etc, are specified via PCI "domain:bus:slot.func" syntax +or "bus:slot.func" syntax. For devices bound to Linux kernel drivers, they may +also be referred to by Linux interface name e.g. eth0, eth1, em0, em1, etc. + +Options: + --help, --usage: + Display usage information and quit + + -s, --status: + Print the current status of all known network, crypto, event + and mempool devices. + For each device, it displays the PCI domain, bus, slot and function, + along with a text description of the device. Depending upon whether the + device is being used by a kernel driver, the igb_uio driver, or no + driver, other relevant information will be displayed: + * the Linux interface name e.g. if=eth0 + * the driver being used e.g. drv=igb_uio + * any suitable drivers not currently using that device + e.g. unused=igb_uio + NOTE: if this flag is passed along with a bind/unbind option, the + status display will always occur after the other operations have taken + place. + + --status-dev: + Print the status of given device group. Supported device groups are: + "net", "crypto", "event" and "mempool" + + -b driver, --bind=driver: + Select the driver to use or \"none\" to unbind the device + + -u, --unbind: + Unbind a device (Equivalent to \"-b none\") + + --force: + By default, network devices which are used by Linux - as indicated by + having routes in the routing table - cannot be modified. Using the + --force flag overrides this behavior, allowing active links to be + forcibly unbound. + WARNING: This can lead to loss of network connection and should be used + with caution. + +Examples: +--------- + +To display current device status: + %(argv0)s --status + +To display current network device status: + %(argv0)s --status-dev net + +To bind eth1 from the current driver and move to use igb_uio + %(argv0)s --bind=igb_uio eth1 + +To unbind 0000:01:00.0 from using any driver + %(argv0)s -u 0000:01:00.0 + +To bind 0000:02:00.0 and 0000:02:00.1 to the ixgbe kernel driver + %(argv0)s -b ixgbe 02:00.0 02:00.1 + + """ % locals()) # replace items from local variables + + +# This is roughly compatible with check_output function in subprocess module +# which is only available in python 2.7. +def check_output(args, stderr=None): + '''Run a command and capture its output''' + return subprocess.Popen(args, stdout=subprocess.PIPE, + stderr=stderr).communicate()[0] + + +def find_module(mod): + '''find the .ko file for kernel module named mod. + Searches the $RTE_SDK/$RTE_TARGET directory, the kernel + modules directory and finally under the parent directory of + the script ''' + # check $RTE_SDK/$RTE_TARGET directory + if 'RTE_SDK' in os.environ and 'RTE_TARGET' in os.environ: + path = "%s/%s/kmod/%s.ko" % (os.environ['RTE_SDK'], + os.environ['RTE_TARGET'], mod) + if exists(path): + return path + + # check using depmod + try: + with open(os.devnull, "w") as fnull: + path = check_output(["modinfo", "-n", mod], stderr=fnull).strip() + + if path and exists(path): + return path + except: # if modinfo can't find module, it fails, so continue + pass + + # check for a copy based off current path + tools_dir = dirname(abspath(sys.argv[0])) + if tools_dir.endswith("tools"): + base_dir = dirname(tools_dir) + find_out = check_output(["find", base_dir, "-name", mod + ".ko"]) + if len(find_out) > 0: # something matched + path = find_out.splitlines()[0] + if exists(path): + return path + + +def check_modules(): + '''Checks that igb_uio is loaded''' + global dpdk_drivers + + # list of supported modules + mods = [{"Name": driver, "Found": False} for driver in dpdk_drivers] + + # first check if module is loaded + try: + # Get list of sysfs modules (both built-in and dynamically loaded) + sysfs_path = '/sys/module/' + + # Get the list of directories in sysfs_path + sysfs_mods = [os.path.join(sysfs_path, o) for o + in os.listdir(sysfs_path) + if os.path.isdir(os.path.join(sysfs_path, o))] + + # Extract the last element of '/sys/module/abc' in the array + sysfs_mods = [a.split('/')[-1] for a in sysfs_mods] + + # special case for vfio_pci (module is named vfio-pci, + # but its .ko is named vfio_pci) + sysfs_mods = [a if a != 'vfio_pci' else 'vfio-pci' for a in sysfs_mods] + + for mod in mods: + if mod["Name"] in sysfs_mods: + mod["Found"] = True + except: + pass + + # check if we have at least one loaded module + if True not in [mod["Found"] for mod in mods] and b_flag is not None: + if b_flag in dpdk_drivers: + print("Error - no supported modules(DPDK driver) are loaded") + sys.exit(1) + else: + print("Warning - no supported modules(DPDK driver) are loaded") + + # change DPDK driver list to only contain drivers that are loaded + dpdk_drivers = [mod["Name"] for mod in mods if mod["Found"]] + + +def has_driver(dev_id): + '''return true if a device is assigned to a driver. False otherwise''' + return "Driver_str" in devices[dev_id] + + +def get_pci_device_details(dev_id, probe_lspci): + '''This function gets additional details for a PCI device''' + device = {} + + if probe_lspci: + extra_info = check_output(["lspci", "-vmmks", dev_id]).splitlines() + + # parse lspci details + for line in extra_info: + if len(line) == 0: + continue + name, value = line.decode().split("\t", 1) + name = name.strip(":") + "_str" + device[name] = value + # check for a unix interface name + device["Interface"] = "" + for base, dirs, _ in os.walk("/sys/bus/pci/devices/%s/" % dev_id): + if "net" in dirs: + device["Interface"] = \ + ",".join(os.listdir(os.path.join(base, "net"))) + break + # check if a port is used for ssh connection + device["Ssh_if"] = False + device["Active"] = "" + + return device + +def clear_data(): + '''This function clears any old data''' + devices = {} + +def get_device_details(devices_type): + '''This function populates the "devices" dictionary. The keys used are + the pci addresses (domain:bus:slot.func). The values are themselves + dictionaries - one for each NIC.''' + global devices + global dpdk_drivers + + # first loop through and read details for all devices + # request machine readable format, with numeric IDs and String + dev = {} + dev_lines = check_output(["lspci", "-Dvmmnnk"]).splitlines() + for dev_line in dev_lines: + if len(dev_line) == 0: + if device_type_match(dev, devices_type): + # Replace "Driver" with "Driver_str" to have consistency of + # of dictionary key names + if "Driver" in dev.keys(): + dev["Driver_str"] = dev.pop("Driver") + if "Module" in dev.keys(): + dev["Module_str"] = dev.pop("Module") + # use dict to make copy of dev + devices[dev["Slot"]] = dict(dev) + # Clear previous device's data + dev = {} + else: + name, value = dev_line.decode().split("\t", 1) + value_list = value.rsplit(' ', 1) + if len(value_list) > 1: + # String stored in <name>_str + dev[name.rstrip(":") + '_str'] = value_list[0] + # Numeric IDs + dev[name.rstrip(":")] = value_list[len(value_list) - 1] \ + .rstrip("]").lstrip("[") + + if devices_type == network_devices: + # check what is the interface if any for an ssh connection if + # any to this host, so we can mark it later. + ssh_if = [] + route = check_output(["ip", "-o", "route"]) + # filter out all lines for 169.254 routes + route = "\n".join(filter(lambda ln: not ln.startswith("169.254"), + route.decode().splitlines())) + rt_info = route.split() + for i in range(len(rt_info) - 1): + if rt_info[i] == "dev": + ssh_if.append(rt_info[i+1]) + + # based on the basic info, get extended text details + for d in devices.keys(): + if not device_type_match(devices[d], devices_type): + continue + + # get additional info and add it to existing data + devices[d] = devices[d].copy() + # No need to probe lspci + devices[d].update(get_pci_device_details(d, False).items()) + + if devices_type == network_devices: + for _if in ssh_if: + if _if in devices[d]["Interface"].split(","): + devices[d]["Ssh_if"] = True + devices[d]["Active"] = "*Active*" + break + + # add igb_uio to list of supporting modules if needed + if "Module_str" in devices[d]: + for driver in dpdk_drivers: + if driver not in devices[d]["Module_str"]: + devices[d]["Module_str"] = \ + devices[d]["Module_str"] + ",%s" % driver + else: + devices[d]["Module_str"] = ",".join(dpdk_drivers) + + # make sure the driver and module strings do not have any duplicates + if has_driver(d): + modules = devices[d]["Module_str"].split(",") + if devices[d]["Driver_str"] in modules: + modules.remove(devices[d]["Driver_str"]) + devices[d]["Module_str"] = ",".join(modules) + + +def device_type_match(dev, devices_type): + for i in range(len(devices_type)): + param_count = len( + [x for x in devices_type[i].values() if x is not None]) + match_count = 0 + if dev["Class"][0:2] == devices_type[i]["Class"]: + match_count = match_count + 1 + for key in devices_type[i].keys(): + if key != 'Class' and devices_type[i][key]: + value_list = devices_type[i][key].split(',') + for value in value_list: + if value.strip(' ') == dev[key]: + match_count = match_count + 1 + # count must be the number of non None parameters to match + if match_count == param_count: + return True + return False + +def dev_id_from_dev_name(dev_name): + '''Take a device "name" - a string passed in by user to identify a NIC + device, and determine the device id - i.e. the domain:bus:slot.func - for + it, which can then be used to index into the devices array''' + + # check if it's already a suitable index + if dev_name in devices: + return dev_name + # check if it's an index just missing the domain part + elif "0000:" + dev_name in devices: + return "0000:" + dev_name + else: + # check if it's an interface name, e.g. eth1 + for d in devices.keys(): + if dev_name in devices[d]["Interface"].split(","): + return devices[d]["Slot"] + # if nothing else matches - error + print("Unknown device: %s. " + "Please specify device in \"bus:slot.func\" format" % dev_name) + sys.exit(1) + + +def unbind_one(dev_id, force): + '''Unbind the device identified by "dev_id" from its current driver''' + dev = devices[dev_id] + if not has_driver(dev_id): + print("%s %s %s is not currently managed by any driver\n" % + (dev["Slot"], dev["Device_str"], dev["Interface"])) + return + + # prevent us disconnecting ourselves + if dev["Ssh_if"] and not force: + print("Routing table indicates that interface %s is active. " + "Skipping unbind" % (dev_id)) + return + + # write to /sys to unbind + filename = "/sys/bus/pci/drivers/%s/unbind" % dev["Driver_str"] + try: + f = open(filename, "a") + except: + print("Error: unbind failed for %s - Cannot open %s" + % (dev_id, filename)) + sys.exit(1) + f.write(dev_id) + f.close() + + +def bind_one(dev_id, driver, force): + '''Bind the device given by "dev_id" to the driver "driver". If the device + is already bound to a different driver, it will be unbound first''' + dev = devices[dev_id] + saved_driver = None # used to rollback any unbind in case of failure + + # prevent disconnection of our ssh session + if dev["Ssh_if"] and not force: + print("Routing table indicates that interface %s is active. " + "Not modifying" % (dev_id)) + return + + # unbind any existing drivers we don't want + if has_driver(dev_id): + if dev["Driver_str"] == driver: + print("%s already bound to driver %s, skipping\n" + % (dev_id, driver)) + return + else: + saved_driver = dev["Driver_str"] + unbind_one(dev_id, force) + dev["Driver_str"] = "" # clear driver string + + # For kernels >= 3.15 driver_override can be used to specify the driver + # for a device rather than relying on the driver to provide a positive + # match of the device. The existing process of looking up + # the vendor and device ID, adding them to the driver new_id, + # will erroneously bind other devices too which has the additional burden + # of unbinding those devices + if driver in dpdk_drivers: + filename = "/sys/bus/pci/devices/%s/driver_override" % dev_id + if os.path.exists(filename): + try: + f = open(filename, "w") + except: + print("Error: bind failed for %s - Cannot open %s" + % (dev_id, filename)) + return + try: + f.write("%s" % driver) + f.close() + except: + print("Error: bind failed for %s - Cannot write driver %s to " + "PCI ID " % (dev_id, driver)) + return + # For kernels < 3.15 use new_id to add PCI id's to the driver + else: + filename = "/sys/bus/pci/drivers/%s/new_id" % driver + try: + f = open(filename, "w") + except: + print("Error: bind failed for %s - Cannot open %s" + % (dev_id, filename)) + return + try: + # Convert Device and Vendor Id to int to write to new_id + f.write("%04x %04x" % (int(dev["Vendor"],16), + int(dev["Device"], 16))) + f.close() + except: + print("Error: bind failed for %s - Cannot write new PCI ID to " + "driver %s" % (dev_id, driver)) + return + + # do the bind by writing to /sys + filename = "/sys/bus/pci/drivers/%s/bind" % driver + try: + f = open(filename, "a") + except: + print("Error: bind failed for %s - Cannot open %s" + % (dev_id, filename)) + if saved_driver is not None: # restore any previous driver + bind_one(dev_id, saved_driver, force) + return + try: + f.write(dev_id) + f.close() + except: + # for some reason, closing dev_id after adding a new PCI ID to new_id + # results in IOError. however, if the device was successfully bound, + # we don't care for any errors and can safely ignore IOError + tmp = get_pci_device_details(dev_id, True) + if "Driver_str" in tmp and tmp["Driver_str"] == driver: + return + print("Error: bind failed for %s - Cannot bind to driver %s" + % (dev_id, driver)) + if saved_driver is not None: # restore any previous driver + bind_one(dev_id, saved_driver, force) + return + + # For kernels > 3.15 driver_override is used to bind a device to a driver. + # Before unbinding it, overwrite driver_override with empty string so that + # the device can be bound to any other driver + filename = "/sys/bus/pci/devices/%s/driver_override" % dev_id + if os.path.exists(filename): + try: + f = open(filename, "w") + except: + print("Error: unbind failed for %s - Cannot open %s" + % (dev_id, filename)) + sys.exit(1) + try: + f.write("\00") + f.close() + except: + print("Error: unbind failed for %s - Cannot open %s" + % (dev_id, filename)) + sys.exit(1) + + +def unbind_all(dev_list, force=False): + """Unbind method, takes a list of device locations""" + + if dev_list[0] == "dpdk": + for d in devices.keys(): + if "Driver_str" in devices[d]: + if devices[d]["Driver_str"] in dpdk_drivers: + unbind_one(devices[d]["Slot"], force) + return + + dev_list = map(dev_id_from_dev_name, dev_list) + for d in dev_list: + unbind_one(d, force) + + +def bind_all(dev_list, driver, force=False): + """Bind method, takes a list of device locations""" + global devices + + dev_list = map(dev_id_from_dev_name, dev_list) + + for d in dev_list: + bind_one(d, driver, force) + + # For kernels < 3.15 when binding devices to a generic driver + # (i.e. one that doesn't have a PCI ID table) using new_id, some devices + # that are not bound to any other driver could be bound even if no one has + # asked them to. hence, we check the list of drivers again, and see if + # some of the previously-unbound devices were erroneously bound. + if not os.path.exists("/sys/bus/pci/devices/%s/driver_override" % d): + for d in devices.keys(): + # skip devices that were already bound or that we know should be bound + if "Driver_str" in devices[d] or d in dev_list: + continue + + # update information about this device + devices[d] = dict(devices[d].items() + + get_pci_device_details(d, True).items()) + + # check if updated information indicates that the device was bound + if "Driver_str" in devices[d]: + unbind_one(d, force) + + +def display_devices(title, dev_list, extra_params=None): + '''Displays to the user the details of a list of devices given in + "dev_list". The "extra_params" parameter, if given, should contain a string + with %()s fields in it for replacement by the named fields in each + device's dictionary.''' + strings = [] # this holds the strings to print. We sort before printing + print("\n%s" % title) + print("="*len(title)) + if len(dev_list) == 0: + strings.append("<none>") + else: + for dev in dev_list: + if extra_params is not None: + strings.append("%s '%s %s' %s" % (dev["Slot"], + dev["Device_str"], + dev["Device"], + extra_params % dev)) + else: + strings.append("%s '%s'" % (dev["Slot"], dev["Device_str"])) + # sort before printing, so that the entries appear in PCI order + strings.sort() + print("\n".join(strings)) # print one per line + +def show_device_status(devices_type, device_name): + global dpdk_drivers + kernel_drv = [] + dpdk_drv = [] + no_drv = [] + + # split our list of network devices into the three categories above + for d in devices.keys(): + if device_type_match(devices[d], devices_type): + if not has_driver(d): + no_drv.append(devices[d]) + continue + if devices[d]["Driver_str"] in dpdk_drivers: + dpdk_drv.append(devices[d]) + else: + kernel_drv.append(devices[d]) + + # print each category separately, so we can clearly see what's used by DPDK + display_devices("%s devices using DPDK-compatible driver" % device_name, + dpdk_drv, "drv=%(Driver_str)s unused=%(Module_str)s") + display_devices("%s devices using kernel driver" % device_name, kernel_drv, + "if=%(Interface)s drv=%(Driver_str)s " + "unused=%(Module_str)s %(Active)s") + display_devices("Other %s devices" % device_name, no_drv, + "unused=%(Module_str)s") + +def show_status(): + '''Function called when the script is passed the "--status" option. + Displays to the user what devices are bound to the igb_uio driver, the + kernel driver or to no driver''' + + if status_dev == "net" or status_dev == "all": + show_device_status(network_devices, "Network") + + if status_dev == "crypto" or status_dev == "all": + show_device_status(crypto_devices, "Crypto") + + if status_dev == "event" or status_dev == "all": + show_device_status(eventdev_devices, "Eventdev") + + if status_dev == "mempool" or status_dev == "all": + show_device_status(mempool_devices, "Mempool") + +def parse_args(): + '''Parses the command-line arguments given by the user and takes the + appropriate action for each''' + global b_flag + global status_flag + global status_dev + global force_flag + global args + if len(sys.argv) <= 1: + usage() + sys.exit(0) + + try: + opts, args = getopt.getopt(sys.argv[1:], "b:us", + ["help", "usage", "status", "status-dev=", + "force", "bind=", "unbind", ]) + except getopt.GetoptError as error: + print(str(error)) + print("Run '%s --usage' for further information" % sys.argv[0]) + sys.exit(1) + + for opt, arg in opts: + if opt == "--help" or opt == "--usage": + usage() + sys.exit(0) + if opt == "--status-dev": + status_flag = True + status_dev = arg + if opt == "--status" or opt == "-s": + status_flag = True + status_dev = "all" + if opt == "--force": + force_flag = True + if opt == "-b" or opt == "-u" or opt == "--bind" or opt == "--unbind": + if b_flag is not None: + print("Error - Only one bind or unbind may be specified\n") + sys.exit(1) + if opt == "-u" or opt == "--unbind": + b_flag = "none" + else: + b_flag = arg + + +def do_arg_actions(): + '''do the actual action requested by the user''' + global b_flag + global status_flag + global force_flag + global args + + if b_flag is None and not status_flag: + print("Error: No action specified for devices." + "Please give a -b or -u option") + print("Run '%s --usage' for further information" % sys.argv[0]) + sys.exit(1) + + if b_flag is not None and len(args) == 0: + print("Error: No devices specified.") + print("Run '%s --usage' for further information" % sys.argv[0]) + sys.exit(1) + + if b_flag == "none" or b_flag == "None": + unbind_all(args, force_flag) + elif b_flag is not None: + bind_all(args, b_flag, force_flag) + if status_flag: + if b_flag is not None: + clear_data() + # refresh if we have changed anything + get_device_details(network_devices) + get_device_details(crypto_devices) + get_device_details(eventdev_devices) + get_device_details(mempool_devices) + show_status() + + +def main(): + '''program main function''' + parse_args() + check_modules() + clear_data() + get_device_details(network_devices) + get_device_details(crypto_devices) + get_device_details(eventdev_devices) + get_device_details(mempool_devices) + do_arg_actions() + +if __name__ == "__main__": + main() diff --git a/dpdk/usertools/dpdk-pmdinfo.py b/dpdk/usertools/dpdk-pmdinfo.py new file mode 100644 index 00000000..46c1be08 --- /dev/null +++ b/dpdk/usertools/dpdk-pmdinfo.py @@ -0,0 +1,610 @@ +#!/usr/bin/env python + +# ------------------------------------------------------------------------- +# +# Utility to dump PMD_INFO_STRING support from an object file +# +# ------------------------------------------------------------------------- +from __future__ import print_function +import json +import os +import platform +import string +import sys +from elftools.common.exceptions import ELFError +from elftools.common.py3compat import (byte2int, bytes2str, str2bytes) +from elftools.elf.elffile import ELFFile +from optparse import OptionParser + +# For running from development directory. It should take precedence over the +# installed pyelftools. +sys.path.insert(0, '.') + +raw_output = False +pcidb = None + +# =========================================== + + +class Vendor: + """ + Class for vendors. This is the top level class + for the devices belong to a specific vendor. + self.devices is the device dictionary + subdevices are in each device. + """ + + def __init__(self, vendorStr): + """ + Class initializes with the raw line from pci.ids + Parsing takes place inside __init__ + """ + self.ID = vendorStr.split()[0] + self.name = vendorStr.replace("%s " % self.ID, "").rstrip() + self.devices = {} + + def addDevice(self, deviceStr): + """ + Adds a device to self.devices + takes the raw line from pci.ids + """ + s = deviceStr.strip() + devID = s.split()[0] + if devID in self.devices: + pass + else: + self.devices[devID] = Device(deviceStr) + + def report(self): + print(self.ID, self.name) + for id, dev in self.devices.items(): + dev.report() + + def find_device(self, devid): + # convert to a hex string and remove 0x + devid = hex(devid)[2:] + try: + return self.devices[devid] + except: + return Device("%s Unknown Device" % devid) + + +class Device: + + def __init__(self, deviceStr): + """ + Class for each device. + Each vendor has its own devices dictionary. + """ + s = deviceStr.strip() + self.ID = s.split()[0] + self.name = s.replace("%s " % self.ID, "") + self.subdevices = {} + + def report(self): + print("\t%s\t%s" % (self.ID, self.name)) + for subID, subdev in self.subdevices.items(): + subdev.report() + + def addSubDevice(self, subDeviceStr): + """ + Adds a subvendor, subdevice to device. + Uses raw line from pci.ids + """ + s = subDeviceStr.strip() + spl = s.split() + subVendorID = spl[0] + subDeviceID = spl[1] + subDeviceName = s.split(" ")[-1] + devID = "%s:%s" % (subVendorID, subDeviceID) + self.subdevices[devID] = SubDevice( + subVendorID, subDeviceID, subDeviceName) + + def find_subid(self, subven, subdev): + subven = hex(subven)[2:] + subdev = hex(subdev)[2:] + devid = "%s:%s" % (subven, subdev) + + try: + return self.subdevices[devid] + except: + if (subven == "ffff" and subdev == "ffff"): + return SubDevice("ffff", "ffff", "(All Subdevices)") + else: + return SubDevice(subven, subdev, "(Unknown Subdevice)") + + +class SubDevice: + """ + Class for subdevices. + """ + + def __init__(self, vendor, device, name): + """ + Class initializes with vendorid, deviceid and name + """ + self.vendorID = vendor + self.deviceID = device + self.name = name + + def report(self): + print("\t\t%s\t%s\t%s" % (self.vendorID, self.deviceID, self.name)) + + +class PCIIds: + """ + Top class for all pci.ids entries. + All queries will be asked to this class. + PCIIds.vendors["0e11"].devices["0046"].\ + subdevices["0e11:4091"].name = "Smart Array 6i" + """ + + def __init__(self, filename): + """ + Prepares the directories. + Checks local data file. + Tries to load from local, if not found, downloads from web + """ + self.version = "" + self.date = "" + self.vendors = {} + self.contents = None + self.readLocal(filename) + self.parse() + + def reportVendors(self): + """Reports the vendors + """ + for vid, v in self.vendors.items(): + print(v.ID, v.name) + + def report(self, vendor=None): + """ + Reports everything for all vendors or a specific vendor + PCIIds.report() reports everything + PCIIDs.report("0e11") reports only "Compaq Computer Corporation" + """ + if vendor is not None: + self.vendors[vendor].report() + else: + for vID, v in self.vendors.items(): + v.report() + + def find_vendor(self, vid): + # convert vid to a hex string and remove the 0x + vid = hex(vid)[2:] + + try: + return self.vendors[vid] + except: + return Vendor("%s Unknown Vendor" % (vid)) + + def findDate(self, content): + for l in content: + if l.find("Date:") > -1: + return l.split()[-2].replace("-", "") + return None + + def parse(self): + if len(self.contents) < 1: + print("data/%s-pci.ids not found" % self.date) + else: + vendorID = "" + deviceID = "" + for l in self.contents: + if l[0] == "#": + continue + elif len(l.strip()) == 0: + continue + else: + if l.find("\t\t") == 0: + self.vendors[vendorID].devices[ + deviceID].addSubDevice(l) + elif l.find("\t") == 0: + deviceID = l.strip().split()[0] + self.vendors[vendorID].addDevice(l) + else: + vendorID = l.split()[0] + self.vendors[vendorID] = Vendor(l) + + def readLocal(self, filename): + """ + Reads the local file + """ + self.contents = open(filename).readlines() + self.date = self.findDate(self.contents) + + def loadLocal(self): + """ + Loads database from local. If there is no file, + it creates a new one from web + """ + self.date = idsfile[0].split("/")[1].split("-")[0] + self.readLocal() + + +# ======================================= + +def search_file(filename, search_path): + """ Given a search path, find file with requested name """ + for path in string.split(search_path, ":"): + candidate = os.path.join(path, filename) + if os.path.exists(candidate): + return os.path.abspath(candidate) + return None + + +class ReadElf(object): + """ display_* methods are used to emit output into the output stream + """ + + def __init__(self, file, output): + """ file: + stream object with the ELF file to read + + output: + output stream to write to + """ + self.elffile = ELFFile(file) + self.output = output + + # Lazily initialized if a debug dump is requested + self._dwarfinfo = None + + self._versioninfo = None + + def _section_from_spec(self, spec): + """ Retrieve a section given a "spec" (either number or name). + Return None if no such section exists in the file. + """ + try: + num = int(spec) + if num < self.elffile.num_sections(): + return self.elffile.get_section(num) + else: + return None + except ValueError: + # Not a number. Must be a name then + return self.elffile.get_section_by_name(str2bytes(spec)) + + def pretty_print_pmdinfo(self, pmdinfo): + global pcidb + + for i in pmdinfo["pci_ids"]: + vendor = pcidb.find_vendor(i[0]) + device = vendor.find_device(i[1]) + subdev = device.find_subid(i[2], i[3]) + print("%s (%s) : %s (%s) %s" % + (vendor.name, vendor.ID, device.name, + device.ID, subdev.name)) + + def parse_pmd_info_string(self, mystring): + global raw_output + global pcidb + + optional_pmd_info = [ + {'id': 'params', 'tag': 'PMD PARAMETERS'}, + {'id': 'kmod', 'tag': 'PMD KMOD DEPENDENCIES'} + ] + + i = mystring.index("=") + mystring = mystring[i + 2:] + pmdinfo = json.loads(mystring) + + if raw_output: + print(json.dumps(pmdinfo)) + return + + print("PMD NAME: " + pmdinfo["name"]) + for i in optional_pmd_info: + try: + print("%s: %s" % (i['tag'], pmdinfo[i['id']])) + except KeyError: + continue + + if (len(pmdinfo["pci_ids"]) != 0): + print("PMD HW SUPPORT:") + if pcidb is not None: + self.pretty_print_pmdinfo(pmdinfo) + else: + print("VENDOR\t DEVICE\t SUBVENDOR\t SUBDEVICE") + for i in pmdinfo["pci_ids"]: + print("0x%04x\t 0x%04x\t 0x%04x\t\t 0x%04x" % + (i[0], i[1], i[2], i[3])) + + print("") + + def display_pmd_info_strings(self, section_spec): + """ Display a strings dump of a section. section_spec is either a + section number or a name. + """ + section = self._section_from_spec(section_spec) + if section is None: + return + + data = section.data() + dataptr = 0 + + while dataptr < len(data): + while (dataptr < len(data) and + not (32 <= byte2int(data[dataptr]) <= 127)): + dataptr += 1 + + if dataptr >= len(data): + break + + endptr = dataptr + while endptr < len(data) and byte2int(data[endptr]) != 0: + endptr += 1 + + mystring = bytes2str(data[dataptr:endptr]) + rc = mystring.find("PMD_INFO_STRING") + if (rc != -1): + self.parse_pmd_info_string(mystring) + + dataptr = endptr + + def find_librte_eal(self, section): + for tag in section.iter_tags(): + if tag.entry.d_tag == 'DT_NEEDED': + if "librte_eal" in tag.needed: + return tag.needed + return None + + def search_for_autoload_path(self): + scanelf = self + scanfile = None + library = None + + section = self._section_from_spec(".dynamic") + try: + eallib = self.find_librte_eal(section) + if eallib is not None: + ldlibpath = os.environ.get('LD_LIBRARY_PATH') + if ldlibpath is None: + ldlibpath = "" + dtr = self.get_dt_runpath(section) + library = search_file(eallib, + dtr + ":" + ldlibpath + + ":/usr/lib64:/lib64:/usr/lib:/lib") + if library is None: + return (None, None) + if raw_output is False: + print("Scanning for autoload path in %s" % library) + scanfile = open(library, 'rb') + scanelf = ReadElf(scanfile, sys.stdout) + except AttributeError: + # Not a dynamic binary + pass + except ELFError: + scanfile.close() + return (None, None) + + section = scanelf._section_from_spec(".rodata") + if section is None: + if scanfile is not None: + scanfile.close() + return (None, None) + + data = section.data() + dataptr = 0 + + while dataptr < len(data): + while (dataptr < len(data) and + not (32 <= byte2int(data[dataptr]) <= 127)): + dataptr += 1 + + if dataptr >= len(data): + break + + endptr = dataptr + while endptr < len(data) and byte2int(data[endptr]) != 0: + endptr += 1 + + mystring = bytes2str(data[dataptr:endptr]) + rc = mystring.find("DPDK_PLUGIN_PATH") + if (rc != -1): + rc = mystring.find("=") + return (mystring[rc + 1:], library) + + dataptr = endptr + if scanfile is not None: + scanfile.close() + return (None, None) + + def get_dt_runpath(self, dynsec): + for tag in dynsec.iter_tags(): + if tag.entry.d_tag == 'DT_RUNPATH': + return tag.runpath + return "" + + def process_dt_needed_entries(self): + """ Look to see if there are any DT_NEEDED entries in the binary + And process those if there are + """ + global raw_output + runpath = "" + ldlibpath = os.environ.get('LD_LIBRARY_PATH') + if ldlibpath is None: + ldlibpath = "" + + dynsec = self._section_from_spec(".dynamic") + try: + runpath = self.get_dt_runpath(dynsec) + except AttributeError: + # dynsec is None, just return + return + + for tag in dynsec.iter_tags(): + if tag.entry.d_tag == 'DT_NEEDED': + rc = tag.needed.find(b"librte_pmd") + if (rc != -1): + library = search_file(tag.needed, + runpath + ":" + ldlibpath + + ":/usr/lib64:/lib64:/usr/lib:/lib") + if library is not None: + if raw_output is False: + print("Scanning %s for pmd information" % library) + with open(library, 'rb') as file: + try: + libelf = ReadElf(file, sys.stdout) + except ELFError: + print("%s is no an ELF file" % library) + continue + libelf.process_dt_needed_entries() + libelf.display_pmd_info_strings(".rodata") + file.close() + + +def scan_autoload_path(autoload_path): + global raw_output + + if os.path.exists(autoload_path) is False: + return + + try: + dirs = os.listdir(autoload_path) + except OSError: + # Couldn't read the directory, give up + return + + for d in dirs: + dpath = os.path.join(autoload_path, d) + if os.path.isdir(dpath): + scan_autoload_path(dpath) + if os.path.isfile(dpath): + try: + file = open(dpath, 'rb') + readelf = ReadElf(file, sys.stdout) + except ELFError: + # this is likely not an elf file, skip it + continue + except IOError: + # No permission to read the file, skip it + continue + + if raw_output is False: + print("Hw Support for library %s" % d) + readelf.display_pmd_info_strings(".rodata") + file.close() + + +def scan_for_autoload_pmds(dpdk_path): + """ + search the specified application or path for a pmd autoload path + then scan said path for pmds and report hw support + """ + global raw_output + + if (os.path.isfile(dpdk_path) is False): + if raw_output is False: + print("Must specify a file name") + return + + file = open(dpdk_path, 'rb') + try: + readelf = ReadElf(file, sys.stdout) + except ElfError: + if raw_output is False: + print("Unable to parse %s" % file) + return + + (autoload_path, scannedfile) = readelf.search_for_autoload_path() + if (autoload_path is None or autoload_path is ""): + if (raw_output is False): + print("No autoload path configured in %s" % dpdk_path) + return + if (raw_output is False): + if (scannedfile is None): + scannedfile = dpdk_path + print("Found autoload path %s in %s" % (autoload_path, scannedfile)) + + file.close() + if (raw_output is False): + print("Discovered Autoload HW Support:") + scan_autoload_path(autoload_path) + return + + +def main(stream=None): + global raw_output + global pcidb + + pcifile_default = "./pci.ids" # For unknown OS's assume local file + if platform.system() == 'Linux': + pcifile_default = "/usr/share/hwdata/pci.ids" + elif platform.system() == 'FreeBSD': + pcifile_default = "/usr/local/share/pciids/pci.ids" + if not os.path.exists(pcifile_default): + pcifile_default = "/usr/share/misc/pci_vendors" + + optparser = OptionParser( + usage='usage: %prog [-hrtp] [-d <pci id file] <elf-file>', + description="Dump pmd hardware support info", + add_help_option=True) + optparser.add_option('-r', '--raw', + action='store_true', dest='raw_output', + help='Dump raw json strings') + optparser.add_option("-d", "--pcidb", dest="pcifile", + help="specify a pci database " + "to get vendor names from", + default=pcifile_default, metavar="FILE") + optparser.add_option("-t", "--table", dest="tblout", + help="output information on hw support as a " + "hex table", + action='store_true') + optparser.add_option("-p", "--plugindir", dest="pdir", + help="scan dpdk for autoload plugins", + action='store_true') + + options, args = optparser.parse_args() + + if options.raw_output: + raw_output = True + + if options.pcifile: + pcidb = PCIIds(options.pcifile) + if pcidb is None: + print("Pci DB file not found") + exit(1) + + if options.tblout: + options.pcifile = None + pcidb = None + + if (len(args) == 0): + optparser.print_usage() + exit(1) + + if options.pdir is True: + exit(scan_for_autoload_pmds(args[0])) + + ldlibpath = os.environ.get('LD_LIBRARY_PATH') + if (ldlibpath is None): + ldlibpath = "" + + if (os.path.exists(args[0]) is True): + myelffile = args[0] + else: + myelffile = search_file( + args[0], ldlibpath + ":/usr/lib64:/lib64:/usr/lib:/lib") + + if (myelffile is None): + print("File not found") + sys.exit(1) + + with open(myelffile, 'rb') as file: + try: + readelf = ReadElf(file, sys.stdout) + readelf.process_dt_needed_entries() + readelf.display_pmd_info_strings(".rodata") + sys.exit(0) + + except ELFError as ex: + sys.stderr.write('ELF error: %s\n' % ex) + sys.exit(1) + + +# ------------------------------------------------------------------------- +if __name__ == '__main__': + main() diff --git a/dpdk/usertools/dpdk-setup.sh b/dpdk/usertools/dpdk-setup.sh new file mode 100644 index 00000000..c4fec5a6 --- /dev/null +++ b/dpdk/usertools/dpdk-setup.sh @@ -0,0 +1,634 @@ +#! /bin/bash + +# BSD LICENSE +# +# Copyright(c) 2010-2014 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# +# Run with "source /path/to/dpdk-setup.sh" +# + +# +# Change to DPDK directory ( <this-script's-dir>/.. ), and export it as RTE_SDK +# +cd $(dirname ${BASH_SOURCE[0]})/.. +export RTE_SDK=$PWD +echo "------------------------------------------------------------------------------" +echo " RTE_SDK exported as $RTE_SDK" +echo "------------------------------------------------------------------------------" + +HUGEPGSZ=`cat /proc/meminfo | grep Hugepagesize | cut -d : -f 2 | tr -d ' '` + +# +# Application EAL parameters for setting memory options (amount/channels/ranks). +# +EAL_PARAMS='-n 4' + +# +# Sets QUIT variable so script will finish. +# +quit() +{ + QUIT=$1 +} + +# Shortcut for quit. +q() +{ + quit +} + +# +# Sets up environmental variables for ICC. +# +setup_icc() +{ + DEFAULT_PATH=/opt/intel/bin/iccvars.sh + param=$1 + shpath=`which iccvars.sh 2> /dev/null` + if [ $? -eq 0 ] ; then + echo "Loading iccvars.sh from $shpath for $param" + source $shpath $param + elif [ -f $DEFAULT_PATH ] ; then + echo "Loading iccvars.sh from $DEFAULT_PATH for $param" + source $DEFAULT_PATH $param + else + echo "## ERROR: cannot find 'iccvars.sh' script to set up ICC." + echo "## To fix, please add the directory that contains" + echo "## iccvars.sh to your 'PATH' environment variable." + quit + fi +} + +# +# Sets RTE_TARGET and does a "make install". +# +setup_target() +{ + option=$1 + export RTE_TARGET=${TARGETS[option]} + + compiler=${RTE_TARGET##*-} + if [ "$compiler" == "icc" ] ; then + platform=${RTE_TARGET%%-*} + if [ "$platform" == "x86_64" ] ; then + setup_icc intel64 + else + setup_icc ia32 + fi + fi + if [ "$QUIT" == "0" ] ; then + make install T=${RTE_TARGET} + fi + echo "------------------------------------------------------------------------------" + echo " RTE_TARGET exported as $RTE_TARGET" + echo "------------------------------------------------------------------------------" +} + +# +# Creates hugepage filesystem. +# +create_mnt_huge() +{ + echo "Creating /mnt/huge and mounting as hugetlbfs" + sudo mkdir -p /mnt/huge + + grep -s '/mnt/huge' /proc/mounts > /dev/null + if [ $? -ne 0 ] ; then + sudo mount -t hugetlbfs nodev /mnt/huge + fi +} + +# +# Removes hugepage filesystem. +# +remove_mnt_huge() +{ + echo "Unmounting /mnt/huge and removing directory" + grep -s '/mnt/huge' /proc/mounts > /dev/null + if [ $? -eq 0 ] ; then + sudo umount /mnt/huge + fi + + if [ -d /mnt/huge ] ; then + sudo rm -R /mnt/huge + fi +} + +# +# Unloads igb_uio.ko. +# +remove_igb_uio_module() +{ + echo "Unloading any existing DPDK UIO module" + /sbin/lsmod | grep -s igb_uio > /dev/null + if [ $? -eq 0 ] ; then + sudo /sbin/rmmod igb_uio + fi +} + +# +# Loads new igb_uio.ko (and uio module if needed). +# +load_igb_uio_module() +{ + if [ ! -f $RTE_SDK/$RTE_TARGET/kmod/igb_uio.ko ];then + echo "## ERROR: Target does not have the DPDK UIO Kernel Module." + echo " To fix, please try to rebuild target." + return + fi + + remove_igb_uio_module + + /sbin/lsmod | grep -s uio > /dev/null + if [ $? -ne 0 ] ; then + modinfo uio > /dev/null + if [ $? -eq 0 ]; then + echo "Loading uio module" + sudo /sbin/modprobe uio + fi + fi + + # UIO may be compiled into kernel, so it may not be an error if it can't + # be loaded. + + echo "Loading DPDK UIO module" + sudo /sbin/insmod $RTE_SDK/$RTE_TARGET/kmod/igb_uio.ko + if [ $? -ne 0 ] ; then + echo "## ERROR: Could not load kmod/igb_uio.ko." + quit + fi +} + +# +# Unloads VFIO modules. +# +remove_vfio_module() +{ + echo "Unloading any existing VFIO module" + /sbin/lsmod | grep -s vfio > /dev/null + if [ $? -eq 0 ] ; then + sudo /sbin/rmmod vfio-pci + sudo /sbin/rmmod vfio_iommu_type1 + sudo /sbin/rmmod vfio + fi +} + +# +# Loads new vfio-pci (and vfio module if needed). +# +load_vfio_module() +{ + remove_vfio_module + + VFIO_PATH="kernel/drivers/vfio/pci/vfio-pci.ko" + + echo "Loading VFIO module" + /sbin/lsmod | grep -s vfio_pci > /dev/null + if [ $? -ne 0 ] ; then + if [ -f /lib/modules/$(uname -r)/$VFIO_PATH ] ; then + sudo /sbin/modprobe vfio-pci + fi + fi + + # make sure regular users can read /dev/vfio + echo "chmod /dev/vfio" + sudo chmod a+x /dev/vfio + if [ $? -ne 0 ] ; then + echo "FAIL" + quit + fi + echo "OK" + + # check if /dev/vfio/vfio exists - that way we + # know we either loaded the module, or it was + # compiled into the kernel + if [ ! -e /dev/vfio/vfio ] ; then + echo "## ERROR: VFIO not found!" + fi +} + +# +# Unloads the rte_kni.ko module. +# +remove_kni_module() +{ + echo "Unloading any existing DPDK KNI module" + /sbin/lsmod | grep -s rte_kni > /dev/null + if [ $? -eq 0 ] ; then + sudo /sbin/rmmod rte_kni + fi +} + +# +# Loads the rte_kni.ko module. +# +load_kni_module() +{ + # Check that the KNI module is already built. + if [ ! -f $RTE_SDK/$RTE_TARGET/kmod/rte_kni.ko ];then + echo "## ERROR: Target does not have the DPDK KNI Module." + echo " To fix, please try to rebuild target." + return + fi + + # Unload existing version if present. + remove_kni_module + + # Now try load the KNI module. + echo "Loading DPDK KNI module" + sudo /sbin/insmod $RTE_SDK/$RTE_TARGET/kmod/rte_kni.ko + if [ $? -ne 0 ] ; then + echo "## ERROR: Could not load kmod/rte_kni.ko." + quit + fi +} + +# +# Sets appropriate permissions on /dev/vfio/* files +# +set_vfio_permissions() +{ + # make sure regular users can read /dev/vfio + echo "chmod /dev/vfio" + sudo chmod a+x /dev/vfio + if [ $? -ne 0 ] ; then + echo "FAIL" + quit + fi + echo "OK" + + # make sure regular user can access everything inside /dev/vfio + echo "chmod /dev/vfio/*" + sudo chmod 0666 /dev/vfio/* + if [ $? -ne 0 ] ; then + echo "FAIL" + quit + fi + echo "OK" + + # since permissions are only to be set when running as + # regular user, we only check ulimit here + # + # warn if regular user is only allowed + # to memlock <64M of memory + MEMLOCK_AMNT=`ulimit -l` + + if [ "$MEMLOCK_AMNT" != "unlimited" ] ; then + MEMLOCK_MB=`expr $MEMLOCK_AMNT / 1024` + echo "" + echo "Current user memlock limit: ${MEMLOCK_MB} MB" + echo "" + echo "This is the maximum amount of memory you will be" + echo "able to use with DPDK and VFIO if run as current user." + echo -n "To change this, please adjust limits.conf memlock " + echo "limit for current user." + + if [ $MEMLOCK_AMNT -lt 65536 ] ; then + echo "" + echo "## WARNING: memlock limit is less than 64MB" + echo -n "## DPDK with VFIO may not be able to initialize " + echo "if run as current user." + fi + fi +} + +# +# Removes all reserved hugepages. +# +clear_huge_pages() +{ + echo > .echo_tmp + for d in /sys/devices/system/node/node? ; do + echo "echo 0 > $d/hugepages/hugepages-${HUGEPGSZ}/nr_hugepages" >> .echo_tmp + done + echo "Removing currently reserved hugepages" + sudo sh .echo_tmp + rm -f .echo_tmp + + remove_mnt_huge +} + +# +# Creates hugepages. +# +set_non_numa_pages() +{ + clear_huge_pages + + echo "" + echo " Input the number of ${HUGEPGSZ} hugepages" + echo " Example: to have 128MB of hugepages available in a 2MB huge page system," + echo " enter '64' to reserve 64 * 2MB pages" + echo -n "Number of pages: " + read Pages + + echo "echo $Pages > /sys/kernel/mm/hugepages/hugepages-${HUGEPGSZ}/nr_hugepages" > .echo_tmp + + echo "Reserving hugepages" + sudo sh .echo_tmp + rm -f .echo_tmp + + create_mnt_huge +} + +# +# Creates hugepages on specific NUMA nodes. +# +set_numa_pages() +{ + clear_huge_pages + + echo "" + echo " Input the number of ${HUGEPGSZ} hugepages for each node" + echo " Example: to have 128MB of hugepages available per node in a 2MB huge page system," + echo " enter '64' to reserve 64 * 2MB pages on each node" + + echo > .echo_tmp + for d in /sys/devices/system/node/node? ; do + node=$(basename $d) + echo -n "Number of pages for $node: " + read Pages + echo "echo $Pages > $d/hugepages/hugepages-${HUGEPGSZ}/nr_hugepages" >> .echo_tmp + done + echo "Reserving hugepages" + sudo sh .echo_tmp + rm -f .echo_tmp + + create_mnt_huge +} + +# +# Run unit test application. +# +run_test_app() +{ + echo "" + echo " Enter hex bitmask of cores to execute test app on" + echo " Example: to execute app on cores 0 to 7, enter 0xff" + echo -n "bitmask: " + read Bitmask + echo "Launching app" + sudo ${RTE_TARGET}/app/test -c $Bitmask $EAL_PARAMS +} + +# +# Run unit testpmd application. +# +run_testpmd_app() +{ + echo "" + echo " Enter hex bitmask of cores to execute testpmd app on" + echo " Example: to execute app on cores 0 to 7, enter 0xff" + echo -n "bitmask: " + read Bitmask + echo "Launching app" + sudo ${RTE_TARGET}/app/testpmd -c $Bitmask $EAL_PARAMS -- -i +} + +# +# Print hugepage information. +# +grep_meminfo() +{ + grep -i huge /proc/meminfo +} + +# +# Calls dpdk-devbind.py --status to show the devices and what they +# are all bound to, in terms of drivers. +# +show_devices() +{ + if [ -d /sys/module/vfio_pci -o -d /sys/module/igb_uio ]; then + ${RTE_SDK}/usertools/dpdk-devbind.py --status + else + echo "# Please load the 'igb_uio' or 'vfio-pci' kernel module before " + echo "# querying or adjusting device bindings" + fi +} + +# +# Uses dpdk-devbind.py to move devices to work with vfio-pci +# +bind_devices_to_vfio() +{ + if [ -d /sys/module/vfio_pci ]; then + ${RTE_SDK}/usertools/dpdk-devbind.py --status + echo "" + echo -n "Enter PCI address of device to bind to VFIO driver: " + read PCI_PATH + sudo ${RTE_SDK}/usertools/dpdk-devbind.py -b vfio-pci $PCI_PATH && + echo "OK" + else + echo "# Please load the 'vfio-pci' kernel module before querying or " + echo "# adjusting device bindings" + fi +} + +# +# Uses dpdk-devbind.py to move devices to work with igb_uio +# +bind_devices_to_igb_uio() +{ + if [ -d /sys/module/igb_uio ]; then + ${RTE_SDK}/usertools/dpdk-devbind.py --status + echo "" + echo -n "Enter PCI address of device to bind to IGB UIO driver: " + read PCI_PATH + sudo ${RTE_SDK}/usertools/dpdk-devbind.py -b igb_uio $PCI_PATH && echo "OK" + else + echo "# Please load the 'igb_uio' kernel module before querying or " + echo "# adjusting device bindings" + fi +} + +# +# Uses dpdk-devbind.py to move devices to work with kernel drivers again +# +unbind_devices() +{ + ${RTE_SDK}/usertools/dpdk-devbind.py --status + echo "" + echo -n "Enter PCI address of device to unbind: " + read PCI_PATH + echo "" + echo -n "Enter name of kernel driver to bind the device to: " + read DRV + sudo ${RTE_SDK}/usertools/dpdk-devbind.py -b $DRV $PCI_PATH && echo "OK" +} + +# +# Options for building a target. Note that this step MUST be first as it sets +# up TARGETS[] starting from 1, and this is accessed in setup_target using the +# user entered option. +# +step1_func() +{ + TITLE="Select the DPDK environment to build" + CONFIG_NUM=1 + for cfg in config/defconfig_* ; do + cfg=${cfg/config\/defconfig_/} + TEXT[$CONFIG_NUM]="$cfg" + TARGETS[$CONFIG_NUM]=$cfg + FUNC[$CONFIG_NUM]="setup_target" + let "CONFIG_NUM+=1" + done +} + +# +# Options for setting up environment. +# +step2_func() +{ + TITLE="Setup linuxapp environment" + + TEXT[1]="Insert IGB UIO module" + FUNC[1]="load_igb_uio_module" + + TEXT[2]="Insert VFIO module" + FUNC[2]="load_vfio_module" + + TEXT[3]="Insert KNI module" + FUNC[3]="load_kni_module" + + TEXT[4]="Setup hugepage mappings for non-NUMA systems" + FUNC[4]="set_non_numa_pages" + + TEXT[5]="Setup hugepage mappings for NUMA systems" + FUNC[5]="set_numa_pages" + + TEXT[6]="Display current Ethernet/Crypto device settings" + FUNC[6]="show_devices" + + TEXT[7]="Bind Ethernet/Crypto device to IGB UIO module" + FUNC[7]="bind_devices_to_igb_uio" + + TEXT[8]="Bind Ethernet/Crypto device to VFIO module" + FUNC[8]="bind_devices_to_vfio" + + TEXT[9]="Setup VFIO permissions" + FUNC[9]="set_vfio_permissions" +} + +# +# Options for running applications. +# +step3_func() +{ + TITLE="Run test application for linuxapp environment" + + TEXT[1]="Run test application (\$RTE_TARGET/app/test)" + FUNC[1]="run_test_app" + + TEXT[2]="Run testpmd application in interactive mode (\$RTE_TARGET/app/testpmd)" + FUNC[2]="run_testpmd_app" +} + +# +# Other options +# +step4_func() +{ + TITLE="Other tools" + + TEXT[1]="List hugepage info from /proc/meminfo" + FUNC[1]="grep_meminfo" + +} + +# +# Options for cleaning up the system +# +step5_func() +{ + TITLE="Uninstall and system cleanup" + + TEXT[1]="Unbind devices from IGB UIO or VFIO driver" + FUNC[1]="unbind_devices" + + TEXT[2]="Remove IGB UIO module" + FUNC[2]="remove_igb_uio_module" + + TEXT[3]="Remove VFIO module" + FUNC[3]="remove_vfio_module" + + TEXT[4]="Remove KNI module" + FUNC[4]="remove_kni_module" + + TEXT[5]="Remove hugepage mappings" + FUNC[5]="clear_huge_pages" +} + +STEPS[1]="step1_func" +STEPS[2]="step2_func" +STEPS[3]="step3_func" +STEPS[4]="step4_func" +STEPS[5]="step5_func" + +QUIT=0 + +while [ "$QUIT" == "0" ]; do + OPTION_NUM=1 + + for s in $(seq ${#STEPS[@]}) ; do + ${STEPS[s]} + + echo "----------------------------------------------------------" + echo " Step $s: ${TITLE}" + echo "----------------------------------------------------------" + + for i in $(seq ${#TEXT[@]}) ; do + echo "[$OPTION_NUM] ${TEXT[i]}" + OPTIONS[$OPTION_NUM]=${FUNC[i]} + let "OPTION_NUM+=1" + done + + # Clear TEXT and FUNC arrays before next step + unset TEXT + unset FUNC + + echo "" + done + + echo "[$OPTION_NUM] Exit Script" + OPTIONS[$OPTION_NUM]="quit" + echo "" + echo -n "Option: " + read our_entry + echo "" + ${OPTIONS[our_entry]} ${our_entry} + + if [ "$QUIT" == "0" ] ; then + echo + echo -n "Press enter to continue ..."; read + fi + +done diff --git a/example/Makefile b/example/Makefile index 9b990286..1e9bb424 100644 --- a/example/Makefile +++ b/example/Makefile @@ -8,11 +8,9 @@ ifeq ($(FF_DPDK),) FF_DPDK=${TOPDIR}/dpdk/x86_64-native-linuxapp-gcc endif -LIBS+= -L${FF_PATH}/lib -L${FF_DPDK}/lib -Wl,--whole-archive,-lfstack,--no-whole-archive -LIBS+= -Wl,--whole-archive -lrte_pmd_vmxnet3_uio -lrte_pmd_i40e -lrte_pmd_ixgbe -lrte_pmd_e1000 -lrte_pmd_ring -LIBS+= -Wl,--whole-archive -lrte_hash -lrte_kvargs -Wl,-lrte_mbuf -lethdev -lrte_eal -Wl,-lrte_mempool -LIBS+= -lrte_ring -lrte_cmdline -lrte_cfgfile -lrte_kni -lrte_timer -Wl,-lrte_pmd_virtio -LIBS+= -Wl,--no-whole-archive -lrt -lm -ldl -lcrypto -pthread +LIBS+= -L${FF_PATH}/lib -Wl,--whole-archive,-lfstack,--no-whole-archive +LIBS+= -L${FF_DPDK}/lib -Wl,--whole-archive,-ldpdk,--no-whole-archive +LIBS+= -Wl,--no-whole-archive -lrt -lm -ldl -lcrypto -pthread -lnuma TARGET="helloworld" all: diff --git a/lib/ff_dpdk_if.c b/lib/ff_dpdk_if.c index f9a35a9d..a05815c8 100644 --- a/lib/ff_dpdk_if.c +++ b/lib/ff_dpdk_if.c @@ -139,8 +139,8 @@ struct mbuf_table { }; struct lcore_rx_queue { - uint8_t port_id; - uint8_t queue_id; + uint16_t port_id; + uint16_t queue_id; } __rte_cache_aligned; struct lcore_conf { @@ -224,7 +224,8 @@ check_all_ports_link_status(void) #define CHECK_INTERVAL 100 /* 100ms */ #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ - uint8_t portid, count, all_ports_up, print_flag = 0; + uint16_t portid; + uint8_t count, all_ports_up, print_flag = 0; struct rte_eth_link link; printf("\nChecking link status"); @@ -235,7 +236,7 @@ check_all_ports_link_status(void) for (count = 0; count <= MAX_CHECK_TIME; count++) { all_ports_up = 1; for (i = 0; i < nb_ports; i++) { - uint8_t portid = ff_global_cfg.dpdk.portid_list[i]; + uint16_t portid = ff_global_cfg.dpdk.portid_list[i]; memset(&link, 0, sizeof(link)); rte_eth_link_get_nowait(portid, &link); @@ -544,7 +545,7 @@ init_kni(void) } static void -set_rss_table(uint8_t port_id, uint16_t reta_size, uint16_t nb_queues) +set_rss_table(uint16_t port_id, uint16_t reta_size, uint16_t nb_queues) { if (reta_size == 0) { return; @@ -955,7 +956,7 @@ pktmbuf_deep_clone(const struct rte_mbuf *md, } static inline void -process_packets(uint8_t port_id, uint16_t queue_id, struct rte_mbuf **bufs, +process_packets(uint16_t port_id, uint16_t queue_id, struct rte_mbuf **bufs, uint16_t count, const struct ff_dpdk_if_context *ctx, int pkts_from_ring) { struct lcore_conf *qconf = &lcore_conf; @@ -1036,13 +1037,13 @@ process_packets(uint8_t port_id, uint16_t queue_id, struct rte_mbuf **bufs, } static inline int -process_dispatch_ring(uint8_t port_id, uint16_t queue_id, +process_dispatch_ring(uint16_t port_id, uint16_t queue_id, struct rte_mbuf **pkts_burst, const struct ff_dpdk_if_context *ctx) { /* read packet from ring buf and to process */ uint16_t nb_rb; nb_rb = rte_ring_dequeue_burst(dispatch_ring[port_id][queue_id], - (void **)pkts_burst, MAX_PKT_BURST); + (void **)pkts_burst, MAX_PKT_BURST, NULL); if(nb_rb > 0) { process_packets(port_id, queue_id, pkts_burst, nb_rb, ctx, 1); @@ -1385,7 +1386,7 @@ main_loop(void *arg) struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; uint64_t prev_tsc, diff_tsc, cur_tsc, usch_tsc, div_tsc, usr_tsc, sys_tsc, end_tsc; int i, j, nb_rx, idle; - uint8_t port_id, queue_id; + uint16_t port_id, queue_id; struct lcore_conf *qconf; const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / US_PER_S * BURST_TX_DRAIN_US; diff --git a/lib/ff_dpdk_kni.c b/lib/ff_dpdk_kni.c index 7e9fdfcc..b3e59606 100644 --- a/lib/ff_dpdk_kni.c +++ b/lib/ff_dpdk_kni.c @@ -29,6 +29,7 @@ #include <rte_config.h> #include <rte_ether.h> +#include <rte_bus_pci.h> #include <rte_ethdev.h> #include <rte_kni.h> #include <rte_malloc.h> @@ -123,13 +124,13 @@ kni_set_bitmap(const char *p, unsigned char *port_bitmap) /* Currently we don't support change mtu. */ static int -kni_change_mtu(uint8_t port_id, unsigned new_mtu) +kni_change_mtu(uint16_t port_id, unsigned new_mtu) { return 0; } static int -kni_config_network_interface(uint8_t port_id, uint8_t if_up) +kni_config_network_interface(uint16_t port_id, uint8_t if_up) { int ret = 0; @@ -165,12 +166,12 @@ kni_config_network_interface(uint8_t port_id, uint8_t if_up) } static int -kni_process_tx(uint8_t port_id, uint16_t queue_id, +kni_process_tx(uint16_t port_id, uint16_t queue_id, struct rte_mbuf **pkts_burst, unsigned count) { /* read packet from kni ring(phy port) and transmit to kni */ uint16_t nb_tx, nb_kni_tx; - nb_tx = rte_ring_dequeue_burst(kni_rp[port_id], (void **)pkts_burst, count); + nb_tx = rte_ring_dequeue_burst(kni_rp[port_id], (void **)pkts_burst, count, NULL); /* NB. * if nb_tx is 0,it must call rte_kni_tx_burst @@ -192,7 +193,7 @@ kni_process_tx(uint8_t port_id, uint16_t queue_id, } static int -kni_process_rx(uint8_t port_id, uint16_t queue_id, +kni_process_rx(uint16_t port_id, uint16_t queue_id, struct rte_mbuf **pkts_burst, unsigned count) { uint16_t nb_kni_rx, nb_rx; @@ -331,7 +332,7 @@ ff_kni_init(uint16_t nb_ports, const char *tcp_ports, const char *udp_ports) } void -ff_kni_alloc(uint8_t port_id, unsigned socket_id, +ff_kni_alloc(uint16_t port_id, unsigned socket_id, struct rte_mempool *mbuf_pool, unsigned ring_queue_size) { if (rte_eal_process_type() == RTE_PROC_PRIMARY) { @@ -401,7 +402,7 @@ ff_kni_alloc(uint8_t port_id, unsigned socket_id, void -ff_kni_process(uint8_t port_id, uint16_t queue_id, +ff_kni_process(uint16_t port_id, uint16_t queue_id, struct rte_mbuf **pkts_burst, unsigned count) { kni_process_tx(port_id, queue_id, pkts_burst, count); @@ -410,7 +411,7 @@ ff_kni_process(uint8_t port_id, uint16_t queue_id, /* enqueue the packet, and own it */ int -ff_kni_enqueue(uint8_t port_id, struct rte_mbuf *pkt) +ff_kni_enqueue(uint16_t port_id, struct rte_mbuf *pkt) { int ret = rte_ring_enqueue(kni_rp[port_id], pkt); if (ret < 0) diff --git a/lib/ff_dpdk_kni.h b/lib/ff_dpdk_kni.h index 775f62f5..a51050ee 100644 --- a/lib/ff_dpdk_kni.h +++ b/lib/ff_dpdk_kni.h @@ -40,15 +40,15 @@ enum FilterReturn { void ff_kni_init(uint16_t nb_ports, const char *tcp_ports, const char *udp_ports); -void ff_kni_alloc(uint8_t port_id, unsigned socket_id, +void ff_kni_alloc(uint16_t port_id, unsigned socket_id, struct rte_mempool *mbuf_pool, unsigned ring_queue_size); -void ff_kni_process(uint8_t port_id, uint16_t queue_id, +void ff_kni_process(uint16_t port_id, uint16_t queue_id, struct rte_mbuf **pkts_burst, unsigned count); enum FilterReturn ff_kni_proto_filter(const void *data, uint16_t len); -int ff_kni_enqueue(uint8_t port_id, struct rte_mbuf *pkt); +int ff_kni_enqueue(uint16_t port_id, struct rte_mbuf *pkt); #endif /* ifndef _FSTACK_DPDK_KNI_H */ diff --git a/tools/prog.mk b/tools/prog.mk index 78ad8447..1c4f8e56 100644 --- a/tools/prog.mk +++ b/tools/prog.mk @@ -50,10 +50,9 @@ FF_PROG_CFLAGS+= -I${TOPDIR}/tools/compat/include -D__BSD_VISIBLE FF_PROG_CFLAGS+= -include ${FF_DPDK}/include/rte_config.h FF_PROG_CFLAGS+= -I${FF_DPDK}/include -FF_PROG_LIBS:= -L${TOPDIR}/tools/compat -Wl,--whole-archive -lffcompat -FF_PROG_LIBS+= -Wl,--no-whole-archive -L${FF_DPDK}/lib -FF_PROG_LIBS+= -Wl,--whole-archive -lrte_eal -lrte_mempool -lrte_ring -FF_PROG_LIBS+= -Wl,--no-whole-archive -lrt -lm -ldl -lcrypto -pthread +FF_PROG_LIBS:= -L${TOPDIR}/tools/compat -Wl,--whole-archive,-lffcompat,--no-whole-archive +FF_PROG_LIBS+= -L${FF_DPDK}/lib -Wl,--whole-archive,-ldpdk,--no-whole-archive +FF_PROG_LIBS+= -Wl,--no-whole-archive -lrt -lm -ldl -lcrypto -lpthread -lnuma CFLAGS+= -Wno-unused-but-set-variable -Wno-unused-variable CFLAGS+= ${FF_PROG_CFLAGS}